From 9e9f0c77c6c2ddc41097fa7a5f20ee3040ec9830 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Mon, 31 Aug 2020 12:09:47 +0200
Subject: [PATCH 0001/1008] Update the published JOSS information.

---
 CITING.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/CITING.md b/CITING.md
index 02fcec9cf13..da08593eafe 100644
--- a/CITING.md
+++ b/CITING.md
@@ -23,11 +23,14 @@ The Ginkgo software itself was reviewed and has a paper published in the Journal
 of Open Source Software, which can be cited with the following reference:
 
 ```bibtex
-@article{Joss2020,
-  doi = {10.21105.joss.02260},
+@article{GinkgoJoss2020,
+  doi = {10.21105/joss.02260},
   url = {https://doi.org/10.21105/joss.02260},
   year = {2020},
   publisher = {The Open Journal},
+  volume = {5},
+  number = {52},
+  pages = {2260},
   author = {Hartwig Anzt and Terry Cojean and Yen-Chen Chen and Goran Flegar and Fritz G\"{o}bel and Thomas Gr\"{u}tzmacher and Pratik Nayak and Tobias Ribizel and Yu-Hsiang Tsai},
   title = {Ginkgo: A high performance numerical linear algebra library},
   journal = {Journal of Open Source Software}

From ab7543d35c909e133470dc1bcd3c805be049d538 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Mon, 31 Aug 2020 12:14:04 +0200
Subject: [PATCH 0002/1008] Add A100 performance paper.

---
 CITING.md | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/CITING.md b/CITING.md
index da08593eafe..1ccaf82fdef 100644
--- a/CITING.md
+++ b/CITING.md
@@ -71,7 +71,7 @@ series = {PASC ’19}
 }
 ```
 
-### On SpMV performance
+### On SpMV or solvers performance
 
 ``` bibtex
 @InProceedings{tsai2020amdspmv,
@@ -92,7 +92,6 @@ abstract="Efficiently processing sparse matrices is a central and performance-cr
 isbn="978-3-030-50743-5"
 }
 
-
 @article{anzt2020spmv,
 author = {Anzt, Hartwig and Cojean, Terry and Yen-Chen, Chen and Dongarra, Jack and Flegar, Goran and Nayak, Pratik and Tomov, Stanimire and Tsai, Yuhsiang M. and Wang, Weichung},
 title = {Load-Balancing Sparse Matrix Vector Product Kernels on GPUs},
@@ -112,3 +111,14 @@ numpages = {26},
 keywords = {irregular matrices, GPUs, Sparse Matrix Vector Product (SpMV)}
 }
 ```
+
+```bibtex
+@misc{tsai2020evaluating,
+    title={Evaluating the Performance of NVIDIA's A100 Ampere GPU for Sparse Linear Algebra Computations},
+    author={Yuhsiang Mike Tsai and Terry Cojean and Hartwig Anzt},
+    year={2020},
+    eprint={2008.08478},
+    archivePrefix={arXiv},
+    primaryClass={cs.MS}
+}
+```

From 6a1075b25182542b2a45fc9255dde75a95c199df Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 1 Sep 2020 09:04:41 +0200
Subject: [PATCH 0003/1008] Add a badge for the JOSS paper.

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index a545ef04def..9e54903a92e 100644
--- a/README.md
+++ b/README.md
@@ -11,6 +11,7 @@
 [![Documentation](https://img.shields.io/badge/Documentation-latest-blue.svg)](https://ginkgo-project.github.io/ginkgo/doc/master/)
 [![License](https://img.shields.io/github/license/ginkgo-project/ginkgo.svg)](./LICENSE)
 [![c++ standard](https://img.shields.io/badge/c%2B%2B-14-blue.svg)](https://en.wikipedia.org/wiki/C%2B%2B#Standardization)
+[![DOI](https://joss.theoj.org/papers/10.21105/joss.02260/status.svg)](https://doi.org/10.21105/joss.02260)
 
 Ginkgo is a high-performance linear algebra library for manycore systems, with a
 focus on sparse solution of linear systems. It is implemented using modern C++

From 2017daddca62fabc5347d0b88400b625f86b5f22 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 14 Sep 2020 15:17:44 +0200
Subject: [PATCH 0004/1008] fix add_subdirectory builds for Ginkgo

Currently, we often use CMAKE_SOURCE_DIR instead of the
preferred PROJECT_SOURCE_DIR (relative to the last project() call)
or CMAKE_CURRENT_SOURCE_DIR (relative to the current CMakeLists.txt)

This causes issues when Ginkgo is built as a subdirectory instead
of installing it.
---
 CMakeLists.txt                |  6 +++---
 cmake/Modules/FindPAPI.cmake  |  6 +++---
 cmake/get_info.cmake          |  6 +++---
 core/test/CMakeLists.txt      |  2 +-
 cuda/test/CMakeLists.txt      |  2 +-
 doc/conf/Doxyfile.in          |  2 +-
 doc/examples/CMakeLists.txt   | 40 +++++++++++++++++------------------
 doc/helpers.cmake             |  6 +++---
 hip/test/CMakeLists.txt       |  2 +-
 omp/test/CMakeLists.txt       |  2 +-
 reference/test/CMakeLists.txt |  2 +-
 11 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e75f7d61fcc..7462fe9f766 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -125,7 +125,7 @@ if(GINKGO_WITH_IWYU)
     find_program(GINKGO_IWYU_PATH iwyu)
 endif()
 
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules/")
+list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules/")
 
 # Find important header files, store the definitions in include/ginkgo/config.h.in
 # For details, see https://gitlab.kitware.com/cmake/community/wikis/doc/tutorials/How-To-Write-Platform-Checks
@@ -311,8 +311,8 @@ include(CPack)
 # And finally, print the configuration to screen:
 #
 if(GINKGO_CONFIG_LOG_DETAILED)
-    FILE(READ ${CMAKE_BINARY_DIR}/detailed.log GINKGO_LOG_SUMMARY)
+    FILE(READ ${PROJECT_BINARY_DIR}/detailed.log GINKGO_LOG_SUMMARY)
 else()
-    FILE(READ ${CMAKE_BINARY_DIR}/minimal.log GINKGO_LOG_SUMMARY)
+    FILE(READ ${PROJECT_BINARY_DIR}/minimal.log GINKGO_LOG_SUMMARY)
 endif()
 MESSAGE("${GINKGO_LOG_SUMMARY}")
diff --git a/cmake/Modules/FindPAPI.cmake b/cmake/Modules/FindPAPI.cmake
index 3e16af2e125..c2648c252fb 100644
--- a/cmake/Modules/FindPAPI.cmake
+++ b/cmake/Modules/FindPAPI.cmake
@@ -70,7 +70,7 @@ if(PAPI_INCLUDE_DIR)
         # find the components
         enable_language(C)
         foreach(component IN LISTS PAPI_FIND_COMPONENTS)
-            file(WRITE "${CMAKE_BINARY_DIR}/papi_${component}_detect.c"
+            file(WRITE "${PROJECT_BINARY_DIR}/papi_${component}_detect.c"
                 "
                 #include <papi.h>
                 int main() {
@@ -85,8 +85,8 @@ if(PAPI_INCLUDE_DIR)
                 )
             try_run(PAPI_${component}_FOUND
                 gko_result_unused
-                "${CMAKE_BINARY_DIR}"
-                "${CMAKE_BINARY_DIR}/papi_${component}_detect.c"
+                "${PROJECT_BINARY_DIR}"
+                "${PROJECT_BINARY_DIR}/papi_${component}_detect.c"
                 LINK_LIBRARIES ${PAPI_LIBRARY}
                 )
 
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index 0860d5c03e9..f4773be50bd 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -1,5 +1,5 @@
-SET(detailed_log "${CMAKE_BINARY_DIR}/detailed.log")
-SET(minimal_log  "${CMAKE_BINARY_DIR}/minimal.log")
+SET(detailed_log "${PROJECT_BINARY_DIR}/detailed.log")
+SET(minimal_log  "${PROJECT_BINARY_DIR}/minimal.log")
 FILE(REMOVE ${detailed_log} ${minimal_log})
 
 MACRO(_both)
@@ -89,7 +89,7 @@ set(log_types "detailed_log;minimal_log")
 foreach(log_type ${log_types})
     ginkgo_print_module_footer(${${log_type}} "Ginkgo configuration:")
     set(print_var
-        "CMAKE_BUILD_TYPE;BUILD_SHARED_LIBS;CMAKE_INSTALL_PREFIX;CMAKE_SOURCE_DIR;CMAKE_BINARY_DIR"
+        "CMAKE_BUILD_TYPE;BUILD_SHARED_LIBS;CMAKE_INSTALL_PREFIX;PROJECT_SOURCE_DIR;PROJECT_BINARY_DIR"
         )
     foreach(var ${print_var})
         ginkgo_print_variable(${${log_type}} ${var} )
diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt
index f0e39d5c568..473d7005d27 100644
--- a/core/test/CMakeLists.txt
+++ b/core/test/CMakeLists.txt
@@ -1,4 +1,4 @@
-include(${CMAKE_SOURCE_DIR}/cmake/create_test.cmake)
+include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
 
 add_subdirectory(base)
 add_subdirectory(factorization)
diff --git a/cuda/test/CMakeLists.txt b/cuda/test/CMakeLists.txt
index 5b180f32c11..af845cb4c2a 100644
--- a/cuda/test/CMakeLists.txt
+++ b/cuda/test/CMakeLists.txt
@@ -1,4 +1,4 @@
-include(${CMAKE_SOURCE_DIR}/cmake/create_test.cmake)
+include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
 
 add_subdirectory(base)
 add_subdirectory(components)
diff --git a/doc/conf/Doxyfile.in b/doc/conf/Doxyfile.in
index d3442212a21..4ec815c0164 100644
--- a/doc/conf/Doxyfile.in
+++ b/doc/conf/Doxyfile.in
@@ -12,7 +12,7 @@ INPUT                  =
 
 INCLUDE_PATH           = @DIR_BASE@/include @DIR_BASE@
 OUTPUT_DIRECTORY       = @DIR_OUT@
-EXAMPLE_PATH           = @CMAKE_BINARY_DIR@/doc/examples
+EXAMPLE_PATH           = @PROJECT_BINARY_DIR@/doc/examples
 RECURSIVE              = YES
 EXAMPLE_RECURSIVE      = NO
 FILE_PATTERNS          = *.cpp *.cu *.hpp *.cuh *.md
diff --git a/doc/examples/CMakeLists.txt b/doc/examples/CMakeLists.txt
index bc256aac6c6..0bc282197b5 100644
--- a/doc/examples/CMakeLists.txt
+++ b/doc/examples/CMakeLists.txt
@@ -1,32 +1,32 @@
 # Collect all of the directory names for the examples programs
 FILE(GLOB _ginkgo_examples
-    ${CMAKE_SOURCE_DIR}/examples/*
+    ${PROJECT_SOURCE_DIR}/examples/*
     )
-LIST(REMOVE_ITEM _ginkgo_examples "${CMAKE_SOURCE_DIR}/examples/CMakeLists.txt" "${CMAKE_SOURCE_DIR}/examples/build-setup.sh")
+LIST(REMOVE_ITEM _ginkgo_examples "${CMAKE_CURRENT_SOURCE_DIR}/examples/CMakeLists.txt" "${PROJECT_SOURCE_DIR}/examples/build-setup.sh")
 
 ADD_CUSTOM_TARGET(examples)
 
 file(GLOB _ginkgo_examples_tooltip
-    ${CMAKE_SOURCE_DIR}/examples/*/doc/tooltip
+    ${PROJECT_SOURCE_DIR}/examples/*/doc/tooltip
     )
 
 file(GLOB _ginkgo_examples_kind
-    ${CMAKE_SOURCE_DIR}/examples/*/doc/kind
+    ${PROJECT_SOURCE_DIR}/examples/*/doc/kind
     )
 file(GLOB _ginkgo_examples_buildson
-    ${CMAKE_SOURCE_DIR}/examples/*/doc/builds-on
+    ${PROJECT_SOURCE_DIR}/examples/*/doc/builds-on
     )
 
 ADD_CUSTOM_COMMAND(
     OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/examples.hpp
     COMMAND ${PERL_EXECUTABLE}
     ARGS
-    ${CMAKE_SOURCE_DIR}/doc/scripts/examples.pl
+    ${PROJECT_SOURCE_DIR}/doc/scripts/examples.pl
     ${CMAKE_CURRENT_SOURCE_DIR}/examples.hpp.in
     ${_ginkgo_examples}
     > ${CMAKE_CURRENT_BINARY_DIR}/examples.hpp
     DEPENDS
-    ${CMAKE_SOURCE_DIR}/doc/scripts/examples.pl
+    ${PROJECT_SOURCE_DIR}/doc/scripts/examples.pl
     ${CMAKE_CURRENT_SOURCE_DIR}/examples.hpp.in
     ${_ginkgo_examples_tooltip}
     ${_ginkgo_examples_kind}
@@ -49,12 +49,12 @@ FOREACH(example ${_ginkgo_examples})
         OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${example}.cpp
         COMMAND ${PERL_EXECUTABLE}
         ARGS
-        ${CMAKE_SOURCE_DIR}/doc/scripts/program2plain
-        < ${CMAKE_SOURCE_DIR}/examples/${example}/${example}.cpp
+        ${PROJECT_SOURCE_DIR}/doc/scripts/program2plain
+        < ${PROJECT_SOURCE_DIR}/examples/${example}/${example}.cpp
         > ${CMAKE_CURRENT_BINARY_DIR}/${example}.cpp
         DEPENDS
-        ${CMAKE_SOURCE_DIR}/doc/scripts/program2plain
-        ${CMAKE_SOURCE_DIR}/examples/${example}/${example}.cpp
+        ${PROJECT_SOURCE_DIR}/doc/scripts/program2plain
+        ${PROJECT_SOURCE_DIR}/examples/${example}/${example}.cpp
         VERBATIM
         )
 
@@ -62,19 +62,19 @@ FOREACH(example ${_ginkgo_examples})
         OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${example}.hpp
         COMMAND ${PERL_EXECUTABLE}
         ARGS
-        ${CMAKE_SOURCE_DIR}/doc/scripts/make_example.pl
-        ${example} ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}
+        ${PROJECT_SOURCE_DIR}/doc/scripts/make_example.pl
+        ${example} ${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR}
         > ${CMAKE_CURRENT_BINARY_DIR}/${example}.hpp
         WORKING_DIRECTORY
         ${CMAKE_CURRENT_BINARY_DIR}
         DEPENDS
-        ${CMAKE_SOURCE_DIR}/doc/scripts/make_example.pl
-        ${CMAKE_SOURCE_DIR}/doc/scripts/intro2toc
-        ${CMAKE_SOURCE_DIR}/doc/scripts/create_anchors
-        ${CMAKE_SOURCE_DIR}/doc/scripts/program2doxygen
-        ${CMAKE_SOURCE_DIR}/examples/${example}/${example}.cpp
-        ${CMAKE_SOURCE_DIR}/examples/${example}/doc/intro.dox
-        ${CMAKE_SOURCE_DIR}/examples/${example}/doc/results.dox
+        ${PROJECT_SOURCE_DIR}/doc/scripts/make_example.pl
+        ${PROJECT_SOURCE_DIR}/doc/scripts/intro2toc
+        ${PROJECT_SOURCE_DIR}/doc/scripts/create_anchors
+        ${PROJECT_SOURCE_DIR}/doc/scripts/program2doxygen
+        ${PROJECT_SOURCE_DIR}/examples/${example}/${example}.cpp
+        ${PROJECT_SOURCE_DIR}/examples/${example}/doc/intro.dox
+        ${PROJECT_SOURCE_DIR}/examples/${example}/doc/results.dox
         )
 
     ADD_CUSTOM_TARGET(examples_${example}
diff --git a/doc/helpers.cmake b/doc/helpers.cmake
index 8d284019d07..f2a89f7eb09 100644
--- a/doc/helpers.cmake
+++ b/doc/helpers.cmake
@@ -39,7 +39,7 @@ endfunction()
 # generates the documentation named <name> with the additional
 # config file <in> in <pdf/html> format
 function(ginkgo_doc_gen name in pdf mainpage-in)
-    set(DIR_BASE "${CMAKE_SOURCE_DIR}")
+    set(DIR_BASE "${PROJECT_SOURCE_DIR}")
     set(DOC_BASE "${CMAKE_CURRENT_SOURCE_DIR}")
     set(DIR_SCRIPT "${DOC_BASE}/scripts")
     set(DIR_OUT "${CMAKE_CURRENT_BINARY_DIR}/${name}")
@@ -53,7 +53,7 @@ function(ginkgo_doc_gen name in pdf mainpage-in)
         "${DOC_BASE}/headers/"
         )
     list(APPEND doxygen_base_input
-        ${CMAKE_BINARY_DIR}/include/ginkgo/config.hpp
+        ${PROJECT_BINARY_DIR}/include/ginkgo/config.hpp
         ${DIR_BASE}/include
         ${MAINPAGE}
         )
@@ -77,7 +77,7 @@ function(ginkgo_doc_gen name in pdf mainpage-in)
         ${DIR_BASE}/include/ginkgo/**/*.hpp
         )
     list(APPEND doxygen_depend
-        ${CMAKE_BINARY_DIR}/include/ginkgo/config.hpp
+        ${PROJECT_BINARY_DIR}/include/ginkgo/config.hpp
         )
     if(GINKGO_DOC_GENERATE_EXAMPLES)
         list(APPEND doxygen_depend
diff --git a/hip/test/CMakeLists.txt b/hip/test/CMakeLists.txt
index fd1fa2941d8..9c8e39ca4ab 100644
--- a/hip/test/CMakeLists.txt
+++ b/hip/test/CMakeLists.txt
@@ -1,4 +1,4 @@
-include(${CMAKE_SOURCE_DIR}/cmake/create_test.cmake)
+include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
 
 add_subdirectory(base)
 add_subdirectory(components)
diff --git a/omp/test/CMakeLists.txt b/omp/test/CMakeLists.txt
index d746413f53f..d8257d9087c 100644
--- a/omp/test/CMakeLists.txt
+++ b/omp/test/CMakeLists.txt
@@ -1,4 +1,4 @@
-include(${CMAKE_SOURCE_DIR}/cmake/create_test.cmake)
+include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
 
 add_subdirectory(components)
 add_subdirectory(factorization)
diff --git a/reference/test/CMakeLists.txt b/reference/test/CMakeLists.txt
index b359d8146ed..44da92b0bd9 100644
--- a/reference/test/CMakeLists.txt
+++ b/reference/test/CMakeLists.txt
@@ -1,4 +1,4 @@
-include(${CMAKE_SOURCE_DIR}/cmake/create_test.cmake)
+include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
 
 add_subdirectory(base)
 add_subdirectory(components)

From ba0509cf4e1f2376e699fafc8b76e229b9d12ff2 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 15 Sep 2020 12:49:17 +0200
Subject: [PATCH 0005/1008] fix documentation build

---
 doc/examples/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/examples/CMakeLists.txt b/doc/examples/CMakeLists.txt
index 0bc282197b5..9363ea5e1dd 100644
--- a/doc/examples/CMakeLists.txt
+++ b/doc/examples/CMakeLists.txt
@@ -2,7 +2,7 @@
 FILE(GLOB _ginkgo_examples
     ${PROJECT_SOURCE_DIR}/examples/*
     )
-LIST(REMOVE_ITEM _ginkgo_examples "${CMAKE_CURRENT_SOURCE_DIR}/examples/CMakeLists.txt" "${PROJECT_SOURCE_DIR}/examples/build-setup.sh")
+LIST(REMOVE_ITEM _ginkgo_examples "${PROJECT_SOURCE_DIR}/examples/CMakeLists.txt" "${PROJECT_SOURCE_DIR}/examples/build-setup.sh")
 
 ADD_CUSTOM_TARGET(examples)
 

From 42ae32297623ba39829d7b50e06c429709eb12f1 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 15 Sep 2020 14:12:22 +0200
Subject: [PATCH 0006/1008] add subdir build test

---
 .gitlab-ci.yml             | 27 +++++++++++++++++++++++++--
 test_subdir/CMakeLists.txt |  9 +++++++++
 2 files changed, 34 insertions(+), 2 deletions(-)
 create mode 100644 test_subdir/CMakeLists.txt

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c2ab6d813a1..d8870d192b3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -25,6 +25,7 @@ stages:
   CONFIG_LOG: "ON"
   CXX_FLAGS: ""
   EXTRA_CMAKE_FLAGS: ""
+  CI_PROJECT_DIR_SUFFIX: ""
 
 .before_script_template: &default_before_script
   - export NUM_CORES=${CI_PARALLELISM}
@@ -50,7 +51,7 @@ stages:
       CUDA_ARCH_STR=-DGINKGO_CUDA_ARCHITECTURES=${CUDA_ARCH};
       CUDA_HOST_STR=-DCMAKE_CUDA_HOST_COMPILER=$(which ${CXX_COMPILER});
       fi
-    - cmake ${CI_PROJECT_DIR}
+    - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
         -GNinja
         -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${CXX_COMPILER}
         -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE}
@@ -76,7 +77,7 @@ stages:
       CUDA_ARCH_STR=-DGINKGO_CUDA_ARCHITECTURES=${CUDA_ARCH};
       CUDA_HOST_STR=-DCMAKE_CUDA_HOST_COMPILER=$(which ${CXX_COMPILER});
       fi
-    - cmake ${CI_PROJECT_DIR}
+    - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
         -GNinja
         -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${CXX_COMPILER}
         -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} -DCMAKE_BUILD_TYPE=${BUILD_TYPE}
@@ -675,6 +676,28 @@ no-circular-deps:
     - cuda
     - gpu
 
+# Ensure Ginkgo builds from a subdirectory
+subdir-build:
+  <<: *default_build
+  stage: code_quality
+  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  variables:
+    <<: *default_variables
+    BUILD_OMP: "ON"
+    BUILD_CUDA: "ON"
+    BUILD_HIP: "ON"
+    EXTRA_CMAKE_FLAGS: "-DGINKGO_BUILD_DOC=ON"
+    CI_PROJECT_PATH_SUFFIX: "/test_subdir"
+  only:
+    variables:
+      - $RUN_CI_TAG
+  dependencies: []
+  allow_failure: no
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
 # Run clang-tidy and iwyu
 clang-tidy:
   <<: *default_build
diff --git a/test_subdir/CMakeLists.txt b/test_subdir/CMakeLists.txt
new file mode 100644
index 00000000000..8596cc9a3d6
--- /dev/null
+++ b/test_subdir/CMakeLists.txt
@@ -0,0 +1,9 @@
+cmake_minimum_required(VERSION 3.14)
+project(GinkgoSubdirTest LANGUAGES CXX)
+file(CREATE_LINK "${CMAKE_CURRENT_SOURCE_DIR}/.." "${CMAKE_CURRENT_BINARY_DIR}/ginkgo" SYMBOLIC)
+
+add_subdirectory("${CMAKE_CURRENT_BINARY_DIR}/ginkgo")
+
+add_executable(test_subdir ../test_install/test_install.cpp)
+target_compile_features(test_subdir PUBLIC cxx_std_14)
+target_link_libraries(test_subdir PRIVATE Ginkgo::ginkgo)

From df22908deabf44bf7ec3d2a939d5388fe37787fc Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 15 Sep 2020 14:32:18 +0200
Subject: [PATCH 0007/1008] remove documentation from subdir build

---
 .gitlab-ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d8870d192b3..a2d229f67a1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -686,7 +686,6 @@ subdir-build:
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
-    EXTRA_CMAKE_FLAGS: "-DGINKGO_BUILD_DOC=ON"
     CI_PROJECT_PATH_SUFFIX: "/test_subdir"
   only:
     variables:

From f56b2e8a24bb5c47f8f1844ffd1b2ddccb8a1d00 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 16 Sep 2020 12:24:12 +0200
Subject: [PATCH 0008/1008] remove 27pt stencil from example overview

---
 doc/examples/examples.hpp.in | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/doc/examples/examples.hpp.in b/doc/examples/examples.hpp.in
index cf0b01f9a04..8c47be53d89 100644
--- a/doc/examples/examples.hpp.in
+++ b/doc/examples/examples.hpp.in
@@ -127,12 +127,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *       </td></tr>
  *
  *   <tr valign="top">
- *       <td>@ref twentyseven_pt_stencil_solver</td>
- *       <td> Using a twentyseven point 3D stencil to solve the poisson equation
- *            with array views.
- *       </td></tr>
- *
- *   <tr valign="top">
  *       <td>@ref external_lib_interfacing</td>
  *       <td> Using Ginkgo's solver with the external library deal.II.
  *       </td></tr>
@@ -228,7 +222,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *     <td>@ref poisson_solver,
  *         @ref three_pt_stencil_solver,
  *         @ref nine_pt_stencil_solver,
- *         @ref twentyseven_pt_stencil_solver,
  *         @ref custom_matrix_format
  *     </td>
  *   </tr>

From 4f93da9f852fb6d481175b6a2f19fcf5bb717ad7 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 1 Sep 2020 17:07:06 +0200
Subject: [PATCH 0009/1008] add template_convertor and remove_complex for class

---
 common/base/math.hpp.inc          |   6 +-
 core/test/base/math.cpp           |  40 +++++++++++
 include/ginkgo/core/base/math.hpp | 109 +++++++++++++++++++++++++++---
 3 files changed, 146 insertions(+), 9 deletions(-)

diff --git a/common/base/math.hpp.inc b/common/base/math.hpp.inc
index 3ba49b585c3..576ab32859c 100644
--- a/common/base/math.hpp.inc
+++ b/common/base/math.hpp.inc
@@ -54,10 +54,14 @@ struct is_complex_impl<thrust::complex<T>>
     : public std::integral_constant<bool, true> {};
 
 
+template <typename T>
+struct is_complex_or_scalar_impl<thrust::complex<T>> : std::is_scalar<T> {};
+
+
 template <typename T>
 struct truncate_type_impl<thrust::complex<T>> {
     using type = thrust::complex<typename truncate_type_impl<T>::type>;
 };
 
 
-}  // namespace detail
\ No newline at end of file
+}  // namespace detail
diff --git a/core/test/base/math.cpp b/core/test/base/math.cpp
index c63cd4ae8e9..0e54c87f788 100644
--- a/core/test/base/math.cpp
+++ b/core/test/base/math.cpp
@@ -53,6 +53,10 @@ static_assert(
     "imag must return a real type");
 
 
+template <typename ValueType, typename IndexType>
+class DummyClass {};
+
+
 template <typename T>
 void test_real_is_finite()
 {
@@ -135,4 +139,40 @@ TEST(Conjugate, DoubleComplex)
 }
 
 
+TEST(RemoveComplexClass, Float)
+{
+    using origin = DummyClass<float, int>;
+    using expect = DummyClass<float, int>;
+    bool check = std::is_same<expect, gko::remove_complex<origin>>::value;
+    ASSERT_TRUE(check);
+}
+
+
+TEST(RemoveComplexClass, Double)
+{
+    using origin = DummyClass<double, int>;
+    using expect = DummyClass<double, int>;
+    bool check = std::is_same<expect, gko::remove_complex<origin>>::value;
+    ASSERT_TRUE(check);
+}
+
+
+TEST(RemoveComplexClass, FloatComplex)
+{
+    using origin = DummyClass<std::complex<float>, int>;
+    using expect = DummyClass<float, int>;
+    bool check = std::is_same<expect, gko::remove_complex<origin>>::value;
+    ASSERT_TRUE(check);
+}
+
+
+TEST(RemoveComplexClass, DoubleComplex)
+{
+    using origin = DummyClass<std::complex<double>, int>;
+    using expect = DummyClass<double, int>;
+    bool check = std::is_same<expect, gko::remove_complex<origin>>::value;
+    ASSERT_TRUE(check);
+}
+
+
 }  // namespace
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 9f9dabb8f75..564635556cf 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -141,6 +141,65 @@ struct is_complex_impl<std::complex<T>>
     : public std::integral_constant<bool, true> {};
 
 
+template <typename T>
+struct is_complex_or_scalar_impl : std::is_scalar<T> {};
+
+template <typename T>
+struct is_complex_or_scalar_impl<std::complex<T>> : std::is_scalar<T> {};
+
+
+/**
+ * template_convertor is converting the template parameters of a class by
+ * convertor<type>.
+ *
+ * @tparam  convertor<type> which convert one type to another type
+ * @tparam  T  type
+ */
+template <template <typename> class convertor, typename T>
+struct template_convertor {};
+
+/**
+ * template_convertor is converting the template parameters of a class by
+ * convertor<type>. Converting class<T1, T2, ...> to class<convertor<T1>,
+ * convertor<T2>, convertor<...>>.
+ *
+ * @tparam  convertor<type> which convert one type to another type
+ * @tparam  template <...> T  class template base
+ * @tparam  ...Rest  the template parameter of T
+ */
+template <template <typename> class convertor, template <typename...> class T,
+          typename... Rest>
+struct template_convertor<convertor, T<Rest...>> {
+    using type = T<typename convertor<Rest>::type...>;
+};
+
+
+template <typename T, typename = void>
+struct remove_complex_s {};
+
+/**
+ * Obtains a real counterpart of a std::complex type, and leaves the type
+ * unchanged if it is not a complex type for complex/scalar type.
+ */
+template <typename T>
+struct remove_complex_s<
+    T, xstd::void_t<std::enable_if_t<is_complex_or_scalar_impl<T>::value>>> {
+    using type = typename detail::remove_complex_impl<T>::type;
+};
+
+/**
+ * Obtains a real counterpart of a class with template parameters, which
+ * converts complex parameters to real parameters.
+ */
+template <typename T>
+struct remove_complex_s<
+    T, xstd::void_t<std::enable_if_t<!is_complex_or_scalar_impl<T>::value>>> {
+    using type =
+        typename detail::template_convertor<detail::remove_complex_impl,
+                                            T>::type;
+};
+
+
 }  // namespace detail
 
 
@@ -167,14 +226,6 @@ struct cpx_real_type<std::complex<T>> {
 };
 
 
-/**
- * Obtains a real counterpart of a std::complex type, and leaves the type
- * unchanged if it is not a complex type.
- */
-template <typename T>
-using remove_complex = typename detail::remove_complex_impl<T>::type;
-
-
 /**
  * Allows to check if T is a complex value during compile time by accessing the
  * `value` attribute of this struct.
@@ -200,6 +251,48 @@ GKO_INLINE GKO_ATTRIBUTES constexpr bool is_complex()
 }
 
 
+/**
+ * Allows to check if T is a complex or scalar value during compile time by
+ * accessing the `value` attribute of this struct. If `value` is `true`, T is a
+ * complex/scalar type, if it is `false`, T is not a complex/scalar type.
+ *
+ * @tparam T  type to check
+ */
+template <typename T>
+using is_complex_or_scalar_s = detail::is_complex_or_scalar_impl<T>;
+
+/**
+ * Checks if T is a complex/scalar type.
+ *
+ * @tparam T  type to check
+ *
+ * @return `true` if T is a complex/scalar type, `false` otherwise
+ */
+template <typename T>
+GKO_INLINE GKO_ATTRIBUTES constexpr bool is_complex_or_scalar()
+{
+    return detail::is_complex_or_scalar_impl<T>::value;
+}
+
+
+/**
+ * Removes the complex of complex/scalar type or the template parameter of class
+ * by accessing the `type` attribute of this struct.
+ *
+ * @tparam T  type to remove complex
+ */
+using detail::remove_complex_s;
+
+/**
+ * Obtain the type which removed the complex of complex/scalar type or the
+ * template parameter of class by accessing the `type` attribute of this struct.
+ *
+ * @tparam T  type to remove complex
+ */
+template <typename T>
+using remove_complex = typename remove_complex_s<T>::type;
+
+
 namespace detail {
 
 

From 2bd9ca1b99bd77a940bcd4c64ec0be0b14fcc533 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 3 Sep 2020 23:49:23 +0200
Subject: [PATCH 0010/1008] implement absolute array

---
 common/components/absolute_array.hpp.inc     |  61 +++++++++
 core/components/absolute_array.hpp           | 109 ++++++++++++++++
 core/device_hooks/common_kernels.inc.cpp     |  11 ++
 cuda/CMakeLists.txt                          |   1 +
 cuda/components/absolute_array.cu            |  82 ++++++++++++
 cuda/test/components/CMakeLists.txt          |   1 +
 cuda/test/components/absolute_array.cpp      | 128 +++++++++++++++++++
 hip/CMakeLists.txt                           |   1 +
 hip/components/absolute_array.hip.cpp        |  86 +++++++++++++
 hip/test/components/CMakeLists.txt           |   1 +
 hip/test/components/absolute_array.hip.cpp   | 128 +++++++++++++++++++
 include/ginkgo/core/base/math.hpp            |  13 +-
 omp/CMakeLists.txt                           |   1 +
 omp/components/absolute_array.cpp            |  72 +++++++++++
 omp/test/components/CMakeLists.txt           |   1 +
 omp/test/components/absolute_array.cpp       | 128 +++++++++++++++++++
 reference/CMakeLists.txt                     |   1 +
 reference/components/absolute_array.cpp      |  70 ++++++++++
 reference/test/components/CMakeLists.txt     |   1 +
 reference/test/components/absolute_array.cpp | 102 +++++++++++++++
 20 files changed, 997 insertions(+), 1 deletion(-)
 create mode 100644 common/components/absolute_array.hpp.inc
 create mode 100644 core/components/absolute_array.hpp
 create mode 100644 cuda/components/absolute_array.cu
 create mode 100644 cuda/test/components/absolute_array.cpp
 create mode 100644 hip/components/absolute_array.hip.cpp
 create mode 100644 hip/test/components/absolute_array.hip.cpp
 create mode 100644 omp/components/absolute_array.cpp
 create mode 100644 omp/test/components/absolute_array.cpp
 create mode 100644 reference/components/absolute_array.cpp
 create mode 100644 reference/test/components/absolute_array.cpp

diff --git a/common/components/absolute_array.hpp.inc b/common/components/absolute_array.hpp.inc
new file mode 100644
index 00000000000..ee6079fb8fc
--- /dev/null
+++ b/common/components/absolute_array.hpp.inc
@@ -0,0 +1,61 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+namespace kernel {
+
+
+template <typename ValueType>
+__global__
+    __launch_bounds__(default_block_size) void inplace_absolute_array_kernel(
+        const size_type n, ValueType *__restrict__ array)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    if (tidx < n) {
+        array[tidx] = abs(array[tidx]);
+    }
+}
+
+
+template <typename ValueType>
+__global__
+    __launch_bounds__(default_block_size) void outplace_absolute_array_kernel(
+        const size_type n, const ValueType *__restrict__ in,
+        remove_complex<ValueType> *__restrict__ out)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    if (tidx < n) {
+        out[tidx] = abs(in[tidx]);
+    }
+}
+
+
+}  // namespace kernel
diff --git a/core/components/absolute_array.hpp b/core/components/absolute_array.hpp
new file mode 100644
index 00000000000..d48d04335fa
--- /dev/null
+++ b/core/components/absolute_array.hpp
@@ -0,0 +1,109 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_COMPONENTS_ABSOLUTE_ARRAY_HPP_
+#define GKO_CORE_COMPONENTS_ABSOLUTE_ARRAY_HPP_
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+namespace gko {
+namespace kernels {
+
+
+#define GKO_DECLARE_INPLACE_ABSOLUTE_ARRAY_KERNEL(ValueType)                 \
+    void inplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec, \
+                                ValueType *data, size_type num_entries)
+
+#define GKO_DECLARE_OUTPLACE_ABSOLUTE_ARRAY_KERNEL(ValueType)                 \
+    void outplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec, \
+                                 const ValueType *in, size_type num_entries,  \
+                                 remove_complex<ValueType> *out)
+
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES                      \
+    template <typename ValueType>                         \
+    GKO_DECLARE_INPLACE_ABSOLUTE_ARRAY_KERNEL(ValueType); \
+    template <typename ValueType>                         \
+    GKO_DECLARE_OUTPLACE_ABSOLUTE_ARRAY_KERNEL(ValueType)
+
+
+namespace omp {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace omp
+
+
+namespace cuda {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace cuda
+
+
+namespace reference {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace reference
+
+
+namespace hip {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace hip
+
+
+#undef GKO_DECLARE_ALL_AS_TEMPLATES
+
+
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_CORE_COMPONENTS_ABSOLUTE_ARRAY_HPP_
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 6adb0df0445..7015c9c644e 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 
 
+#include "core/components/absolute_array.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/components/precision_conversion.hpp"
 #include "core/components/prefix_sum.hpp"
@@ -94,6 +95,16 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 template GKO_DECLARE_FILL_ARRAY_KERNEL(size_type);
 
+template <typename ValueType>
+GKO_DECLARE_INPLACE_ABSOLUTE_ARRAY_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_ARRAY_KERNEL);
+
+template <typename ValueType>
+GKO_DECLARE_OUTPLACE_ABSOLUTE_ARRAY_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_ARRAY_KERNEL);
+
 
 }  // namespace components
 
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 52560f7be07..d3a73ab536c 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -70,6 +70,7 @@ target_sources(ginkgo_cuda
     base/exception.cpp
     base/executor.cpp
     base/version.cpp
+    components/absolute_array.cu
     components/fill_array.cu
     components/precision_conversion.cu
     components/prefix_sum.cu
diff --git a/cuda/components/absolute_array.cu b/cuda/components/absolute_array.cu
new file mode 100644
index 00000000000..8ef4901bf2c
--- /dev/null
+++ b/cuda/components/absolute_array.cu
@@ -0,0 +1,82 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/absolute_array.hpp"
+
+
+#include "cuda/base/types.hpp"
+#include "cuda/components/thread_ids.cuh"
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+namespace components {
+
+
+constexpr int default_block_size = 512;
+
+
+#include "common/components/absolute_array.hpp.inc"
+
+
+template <typename ValueType>
+void inplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec,
+                            ValueType *data, size_type n)
+{
+    const dim3 block_size(default_block_size, 1, 1);
+    const dim3 grid_size(ceildiv(n, block_size.x), 1, 1);
+    kernel::inplace_absolute_array_kernel<<<grid_size, block_size, 0, 0>>>(
+        n, as_cuda_type(data));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_ARRAY_KERNEL);
+
+
+template <typename ValueType>
+void outplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec,
+                             const ValueType *in, size_type n,
+                             remove_complex<ValueType> *out)
+{
+    const dim3 block_size(default_block_size, 1, 1);
+    const dim3 grid_size(ceildiv(n, block_size.x), 1, 1);
+    kernel::outplace_absolute_array_kernel<<<grid_size, block_size, 0, 0>>>(
+        n, as_cuda_type(in), as_cuda_type(out));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_ARRAY_KERNEL);
+
+
+}  // namespace components
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
diff --git a/cuda/test/components/CMakeLists.txt b/cuda/test/components/CMakeLists.txt
index 154a39e963e..1d76a251948 100644
--- a/cuda/test/components/CMakeLists.txt
+++ b/cuda/test/components/CMakeLists.txt
@@ -2,6 +2,7 @@ ginkgo_create_cuda_test(cooperative_groups_kernels)
 ginkgo_create_cuda_test(merging_kernels)
 ginkgo_create_cuda_test(searching_kernels)
 ginkgo_create_cuda_test(sorting_kernels)
+ginkgo_create_test(absolute_array)
 ginkgo_create_test(fill_array)
 ginkgo_create_test(precision_conversion)
 ginkgo_create_test(prefix_sum)
diff --git a/cuda/test/components/absolute_array.cpp b/cuda/test/components/absolute_array.cpp
new file mode 100644
index 00000000000..6bdef3df6dd
--- /dev/null
+++ b/cuda/test/components/absolute_array.cpp
@@ -0,0 +1,128 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/absolute_array.hpp"
+
+
+#include <memory>
+#include <random>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+
+
+#include "core/test/utils/assertions.hpp"
+
+
+namespace {
+
+
+class AbsoluteArray : public ::testing::Test {
+protected:
+    using value_type = double;
+    using complex_type = std::complex<double>;
+    AbsoluteArray()
+        : ref(gko::ReferenceExecutor::create()),
+          exec(gko::CudaExecutor::create(0, ref)),
+          total_size(6344),
+          vals(ref, total_size),
+          dvals(exec, total_size),
+          complex_vals(ref, total_size),
+          dcomplex_vals(exec, total_size)
+    {
+        std::fill_n(vals.get_data(), total_size, -1234.0);
+        dvals = vals;
+        std::fill_n(complex_vals.get_data(), total_size, complex_type{3, 4});
+        dcomplex_vals = complex_vals;
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<gko::CudaExecutor> exec;
+    gko::size_type total_size;
+    gko::Array<value_type> vals;
+    gko::Array<value_type> dvals;
+    gko::Array<complex_type> complex_vals;
+    gko::Array<complex_type> dcomplex_vals;
+};
+
+
+TEST_F(AbsoluteArray, InplaceEqualsReference)
+{
+    gko::kernels::cuda::components::inplace_absolute_array(
+        exec, dvals.get_data(), total_size);
+    gko::kernels::reference::components::inplace_absolute_array(
+        ref, vals.get_data(), total_size);
+    GKO_ASSERT_ARRAY_EQ(vals, dvals);
+}
+
+
+TEST_F(AbsoluteArray, InplaceComplexEqualsReference)
+{
+    gko::kernels::cuda::components::inplace_absolute_array(
+        exec, dcomplex_vals.get_data(), total_size);
+    gko::kernels::reference::components::inplace_absolute_array(
+        ref, complex_vals.get_data(), total_size);
+    GKO_ASSERT_ARRAY_EQ(complex_vals, dcomplex_vals);
+}
+
+
+TEST_F(AbsoluteArray, OutplaceEqualsReference)
+{
+    gko::Array<value_type> abs_vals(ref, total_size);
+    gko::Array<value_type> dabs_vals(exec, total_size);
+
+    gko::kernels::cuda::components::outplace_absolute_array(
+        exec, dvals.get_const_data(), total_size, dabs_vals.get_data());
+    gko::kernels::reference::components::outplace_absolute_array(
+        ref, vals.get_const_data(), total_size, abs_vals.get_data());
+    GKO_ASSERT_ARRAY_EQ(abs_vals, dabs_vals);
+}
+
+
+TEST_F(AbsoluteArray, OutplaceComplexEqualsReference)
+{
+    gko::Array<value_type> abs_vals(ref, total_size);
+    gko::Array<value_type> dabs_vals(exec, total_size);
+
+    gko::kernels::cuda::components::outplace_absolute_array(
+        exec, dcomplex_vals.get_const_data(), total_size, dabs_vals.get_data());
+    gko::kernels::reference::components::outplace_absolute_array(
+        ref, complex_vals.get_const_data(), total_size, abs_vals.get_data());
+    GKO_ASSERT_ARRAY_EQ(abs_vals, dabs_vals);
+}
+
+
+}  // namespace
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 57c6503ce36..20fcb0d79a0 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -143,6 +143,7 @@ set(GINKGO_HIP_SOURCES
     base/exception.hip.cpp
     base/executor.hip.cpp
     base/version.hip.cpp
+    components/absolute_array.hip.cpp
     components/fill_array.hip.cpp
     components/precision_conversion.hip.cpp
     components/prefix_sum.hip.cpp
diff --git a/hip/components/absolute_array.hip.cpp b/hip/components/absolute_array.hip.cpp
new file mode 100644
index 00000000000..3f7ee4c012c
--- /dev/null
+++ b/hip/components/absolute_array.hip.cpp
@@ -0,0 +1,86 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/absolute_array.hpp"
+
+
+#include <hip/hip_runtime.h>
+
+
+#include "hip/base/types.hip.hpp"
+#include "hip/components/thread_ids.hip.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace hip {
+namespace components {
+
+
+constexpr int default_block_size = 512;
+
+
+#include "common/components/absolute_array.hpp.inc"
+
+
+template <typename ValueType>
+void inplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec,
+                            ValueType *data, size_type n)
+{
+    const dim3 block_size(default_block_size, 1, 1);
+    const dim3 grid_size(ceildiv(n, block_size.x), 1, 1);
+    hipLaunchKernelGGL(kernel::inplace_absolute_array_kernel, dim3(grid_size),
+                       dim3(block_size), 0, 0, n, as_hip_type(data));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_ARRAY_KERNEL);
+
+
+template <typename ValueType>
+void outplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec,
+                             const ValueType *in, size_type n,
+                             remove_complex<ValueType> *out)
+{
+    const dim3 block_size(default_block_size, 1, 1);
+    const dim3 grid_size(ceildiv(n, block_size.x), 1, 1);
+    hipLaunchKernelGGL(kernel::outplace_absolute_array_kernel, dim3(grid_size),
+                       dim3(block_size), 0, 0, n, as_hip_type(in),
+                       as_hip_type(out));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_ARRAY_KERNEL);
+
+
+}  // namespace components
+}  // namespace hip
+}  // namespace kernels
+}  // namespace gko
diff --git a/hip/test/components/CMakeLists.txt b/hip/test/components/CMakeLists.txt
index b3bec2595f9..f0bae60fe60 100644
--- a/hip/test/components/CMakeLists.txt
+++ b/hip/test/components/CMakeLists.txt
@@ -1,3 +1,4 @@
+ginkgo_create_hip_test(absolute_array)
 ginkgo_create_hip_test(cooperative_groups_kernels)
 ginkgo_create_hip_test(fill_array)
 ginkgo_create_hip_test(merging_kernels)
diff --git a/hip/test/components/absolute_array.hip.cpp b/hip/test/components/absolute_array.hip.cpp
new file mode 100644
index 00000000000..8df8e79109d
--- /dev/null
+++ b/hip/test/components/absolute_array.hip.cpp
@@ -0,0 +1,128 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/absolute_array.hpp"
+
+
+#include <memory>
+#include <random>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+
+
+#include "core/test/utils/assertions.hpp"
+
+
+namespace {
+
+
+class AbsoluteArray : public ::testing::Test {
+protected:
+    using value_type = double;
+    using complex_type = std::complex<double>;
+    AbsoluteArray()
+        : ref(gko::ReferenceExecutor::create()),
+          exec(gko::HipExecutor::create(0, ref)),
+          total_size(6344),
+          vals(ref, total_size),
+          dvals(exec, total_size),
+          complex_vals(ref, total_size),
+          dcomplex_vals(exec, total_size)
+    {
+        std::fill_n(vals.get_data(), total_size, -1234.0);
+        dvals = vals;
+        std::fill_n(complex_vals.get_data(), total_size, complex_type{3, 4});
+        dcomplex_vals = complex_vals;
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<gko::HipExecutor> exec;
+    gko::size_type total_size;
+    gko::Array<value_type> vals;
+    gko::Array<value_type> dvals;
+    gko::Array<complex_type> complex_vals;
+    gko::Array<complex_type> dcomplex_vals;
+};
+
+
+TEST_F(AbsoluteArray, InplaceEqualsReference)
+{
+    gko::kernels::hip::components::inplace_absolute_array(
+        exec, dvals.get_data(), total_size);
+    gko::kernels::reference::components::inplace_absolute_array(
+        ref, vals.get_data(), total_size);
+    GKO_ASSERT_ARRAY_EQ(vals, dvals);
+}
+
+
+TEST_F(AbsoluteArray, InplaceComplexEqualsReference)
+{
+    gko::kernels::hip::components::inplace_absolute_array(
+        exec, dcomplex_vals.get_data(), total_size);
+    gko::kernels::reference::components::inplace_absolute_array(
+        ref, complex_vals.get_data(), total_size);
+    GKO_ASSERT_ARRAY_EQ(complex_vals, dcomplex_vals);
+}
+
+
+TEST_F(AbsoluteArray, OutplaceEqualsReference)
+{
+    gko::Array<value_type> abs_vals(ref, total_size);
+    gko::Array<value_type> dabs_vals(exec, total_size);
+
+    gko::kernels::hip::components::outplace_absolute_array(
+        exec, dvals.get_const_data(), total_size, dabs_vals.get_data());
+    gko::kernels::reference::components::outplace_absolute_array(
+        ref, vals.get_const_data(), total_size, abs_vals.get_data());
+    GKO_ASSERT_ARRAY_EQ(abs_vals, dabs_vals);
+}
+
+
+TEST_F(AbsoluteArray, OutplaceComplexEqualsReference)
+{
+    gko::Array<value_type> abs_vals(ref, total_size);
+    gko::Array<value_type> dabs_vals(exec, total_size);
+
+    gko::kernels::hip::components::outplace_absolute_array(
+        exec, dcomplex_vals.get_const_data(), total_size, dabs_vals.get_data());
+    gko::kernels::reference::components::outplace_absolute_array(
+        ref, complex_vals.get_const_data(), total_size, abs_vals.get_data());
+    GKO_ASSERT_ARRAY_EQ(abs_vals, dabs_vals);
+}
+
+
+}  // namespace
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 564635556cf..946b6b4608b 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -698,12 +698,23 @@ GKO_INLINE GKO_ATTRIBUTES constexpr T one(const T &)
  * @return x >= zero<T>() ? x : -x;
  */
 template <typename T>
-GKO_INLINE GKO_ATTRIBUTES constexpr T abs(const T &x)
+GKO_INLINE
+    GKO_ATTRIBUTES constexpr xstd::enable_if_t<!is_complex_s<T>::value, T>
+    abs(const T &x)
 {
     return x >= zero<T>() ? x : -x;
 }
 
 
+template <typename T>
+GKO_INLINE GKO_ATTRIBUTES constexpr xstd::enable_if_t<is_complex_s<T>::value,
+                                                      remove_complex<T>>
+abs(const T &x)
+{
+    return sqrt(squared_norm(x));
+}
+
+
 /**
  * Returns the larger of the arguments.
  *
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index 1f1f5aca534..bcfde819760 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -4,6 +4,7 @@ add_library(ginkgo_omp $<TARGET_OBJECTS:ginkgo_omp_device> "")
 target_sources(ginkgo_omp
     PRIVATE
     base/version.cpp
+    components/absolute_array.cpp
     components/fill_array.cpp
     components/precision_conversion.cpp
     components/prefix_sum.cpp
diff --git a/omp/components/absolute_array.cpp b/omp/components/absolute_array.cpp
new file mode 100644
index 00000000000..36d94ec8471
--- /dev/null
+++ b/omp/components/absolute_array.cpp
@@ -0,0 +1,72 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/absolute_array.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+namespace components {
+
+
+template <typename ValueType>
+void inplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec,
+                            ValueType *data, size_type n)
+{
+#pragma omp parallel for
+    for (size_type i = 0; i < n; ++i) {
+        data[i] = abs(data[i]);
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_ARRAY_KERNEL);
+
+
+template <typename ValueType>
+void outplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec,
+                             const ValueType *in, size_type n,
+                             remove_complex<ValueType> *out)
+{
+#pragma omp parallel for
+    for (size_type i = 0; i < n; ++i) {
+        out[i] = abs(in[i]);
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_ARRAY_KERNEL);
+
+
+}  // namespace components
+}  // namespace omp
+}  // namespace kernels
+}  // namespace gko
diff --git a/omp/test/components/CMakeLists.txt b/omp/test/components/CMakeLists.txt
index 9c1dca5bcfa..1e55ad12a4d 100644
--- a/omp/test/components/CMakeLists.txt
+++ b/omp/test/components/CMakeLists.txt
@@ -1,3 +1,4 @@
+ginkgo_create_test(absolute_array)
 ginkgo_create_test(fill_array)
 ginkgo_create_test(precision_conversion)
 ginkgo_create_test(prefix_sum)
diff --git a/omp/test/components/absolute_array.cpp b/omp/test/components/absolute_array.cpp
new file mode 100644
index 00000000000..9304def3fa9
--- /dev/null
+++ b/omp/test/components/absolute_array.cpp
@@ -0,0 +1,128 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/absolute_array.hpp"
+
+
+#include <memory>
+#include <random>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class AbsoluteArray : public ::testing::Test {
+protected:
+    using value_type = double;
+    using complex_type = std::complex<double>;
+    AbsoluteArray()
+        : ref(gko::ReferenceExecutor::create()),
+          exec(gko::OmpExecutor::create()),
+          total_size(6344),
+          vals(ref, total_size),
+          dvals(exec, total_size),
+          complex_vals(ref, total_size),
+          dcomplex_vals(exec, total_size)
+    {
+        std::fill_n(vals.get_data(), total_size, -1234.0);
+        dvals = vals;
+        std::fill_n(complex_vals.get_data(), total_size, complex_type{3, 4});
+        dcomplex_vals = complex_vals;
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<gko::OmpExecutor> exec;
+    gko::size_type total_size;
+    gko::Array<value_type> vals;
+    gko::Array<value_type> dvals;
+    gko::Array<complex_type> complex_vals;
+    gko::Array<complex_type> dcomplex_vals;
+};
+
+
+TEST_F(AbsoluteArray, InplaceEqualsReference)
+{
+    gko::kernels::omp::components::inplace_absolute_array(
+        exec, dvals.get_data(), total_size);
+    gko::kernels::reference::components::inplace_absolute_array(
+        ref, vals.get_data(), total_size);
+    GKO_ASSERT_ARRAY_EQ(vals, dvals);
+}
+
+
+TEST_F(AbsoluteArray, InplaceComplexEqualsReference)
+{
+    gko::kernels::omp::components::inplace_absolute_array(
+        exec, dcomplex_vals.get_data(), total_size);
+    gko::kernels::reference::components::inplace_absolute_array(
+        ref, complex_vals.get_data(), total_size);
+    GKO_ASSERT_ARRAY_EQ(complex_vals, dcomplex_vals);
+}
+
+
+TEST_F(AbsoluteArray, OutplaceEqualsReference)
+{
+    gko::Array<value_type> abs_vals(ref, total_size);
+    gko::Array<value_type> dabs_vals(exec, total_size);
+
+    gko::kernels::omp::components::outplace_absolute_array(
+        exec, dvals.get_const_data(), total_size, dabs_vals.get_data());
+    gko::kernels::reference::components::outplace_absolute_array(
+        ref, vals.get_const_data(), total_size, abs_vals.get_data());
+    GKO_ASSERT_ARRAY_EQ(abs_vals, dabs_vals);
+}
+
+
+TEST_F(AbsoluteArray, OutplaceComplexEqualsReference)
+{
+    gko::Array<value_type> abs_vals(ref, total_size);
+    gko::Array<value_type> dabs_vals(exec, total_size);
+
+    gko::kernels::omp::components::outplace_absolute_array(
+        exec, dcomplex_vals.get_const_data(), total_size, dabs_vals.get_data());
+    gko::kernels::reference::components::outplace_absolute_array(
+        ref, complex_vals.get_const_data(), total_size, abs_vals.get_data());
+    GKO_ASSERT_ARRAY_EQ(abs_vals, dabs_vals);
+}
+
+
+}  // namespace
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index e4bcd9ee2c2..f92060c036c 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -2,6 +2,7 @@ add_library(ginkgo_reference $<TARGET_OBJECTS:ginkgo_reference_device> "")
 target_sources(ginkgo_reference
     PRIVATE
     base/version.cpp
+    components/absolute_array.cpp
     components/fill_array.cpp
     components/precision_conversion.cpp
     components/prefix_sum.cpp
diff --git a/reference/components/absolute_array.cpp b/reference/components/absolute_array.cpp
new file mode 100644
index 00000000000..688945886f9
--- /dev/null
+++ b/reference/components/absolute_array.cpp
@@ -0,0 +1,70 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/absolute_array.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace reference {
+namespace components {
+
+
+template <typename ValueType>
+void inplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec,
+                            ValueType *data, size_type n)
+{
+    for (size_type i = 0; i < n; i++) {
+        data[i] = abs(data[i]);
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_ARRAY_KERNEL);
+
+
+template <typename ValueType>
+void outplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec,
+                             const ValueType *in, size_type n,
+                             remove_complex<ValueType> *out)
+{
+    for (size_type i = 0; i < n; i++) {
+        out[i] = abs(in[i]);
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_ARRAY_KERNEL);
+
+
+}  // namespace components
+}  // namespace reference
+}  // namespace kernels
+}  // namespace gko
diff --git a/reference/test/components/CMakeLists.txt b/reference/test/components/CMakeLists.txt
index 9c1dca5bcfa..1e55ad12a4d 100644
--- a/reference/test/components/CMakeLists.txt
+++ b/reference/test/components/CMakeLists.txt
@@ -1,3 +1,4 @@
+ginkgo_create_test(absolute_array)
 ginkgo_create_test(fill_array)
 ginkgo_create_test(precision_conversion)
 ginkgo_create_test(prefix_sum)
diff --git a/reference/test/components/absolute_array.cpp b/reference/test/components/absolute_array.cpp
new file mode 100644
index 00000000000..1c2f2952551
--- /dev/null
+++ b/reference/test/components/absolute_array.cpp
@@ -0,0 +1,102 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/absolute_array.hpp"
+
+
+#include <memory>
+#include <random>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/math.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+template <typename T>
+class AbsoluteArray : public ::testing::Test {
+protected:
+    using value_type = T;
+    using abs_type = gko::remove_complex<T>;
+    AbsoluteArray()
+        : ref(gko::ReferenceExecutor::create()),
+          total_size(6344),
+          inplace_expected(ref, total_size),
+          outplace_expected(ref, total_size),
+          vals(ref, total_size)
+    {
+        std::fill_n(inplace_expected.get_data(), total_size, T(6453));
+        std::fill_n(vals.get_data(), total_size, T(-6453));
+        std::fill_n(outplace_expected.get_data(), total_size, abs_type(6453));
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    gko::size_type total_size;
+    gko::Array<value_type> inplace_expected;
+    gko::Array<abs_type> outplace_expected;
+    gko::Array<value_type> vals;
+};
+
+TYPED_TEST_CASE(AbsoluteArray, gko::test::ValueTypes);
+
+
+TYPED_TEST(AbsoluteArray, InplaceEqualsReference)
+{
+    using T = typename TestFixture::value_type;
+    gko::kernels::reference::components::inplace_absolute_array(
+        this->ref, this->vals.get_data(), this->total_size);
+    GKO_ASSERT_ARRAY_EQ(this->vals, this->inplace_expected);
+}
+
+
+TYPED_TEST(AbsoluteArray, OutplaceEqualsReference)
+{
+    using T = typename TestFixture::value_type;
+    using AbsT = typename TestFixture::abs_type;
+    gko::Array<AbsT> abs_vals(this->ref, this->total_size);
+    gko::kernels::reference::components::outplace_absolute_array(
+        this->ref, this->vals.get_const_data(), this->total_size,
+        abs_vals.get_data());
+    GKO_ASSERT_ARRAY_EQ(abs_vals, this->outplace_expected);
+}
+
+
+}  // namespace

From 2ee4d64951fe4a12fa4a0f7fa964d9ae13ef0b2f Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 4 Sep 2020 13:18:47 +0200
Subject: [PATCH 0011/1008] implement Csr absolute

---
 core/matrix/csr.cpp                   | 36 +++++++++++-
 cuda/test/matrix/csr_kernels.cpp      | 46 ++++++++++++++-
 hip/test/matrix/csr_kernels.hip.cpp   | 60 +++++++++++++++++++-
 include/ginkgo/core/base/lin_op.hpp   | 36 ++++++++++++
 include/ginkgo/core/base/math.hpp     | 64 +++++++++++++++++++++
 include/ginkgo/core/matrix/csr.hpp    | 17 +++++-
 omp/test/matrix/csr_kernels.cpp       | 44 +++++++++++++++
 reference/test/matrix/csr_kernels.cpp | 80 +++++++++++++++++++++++++--
 8 files changed, 374 insertions(+), 9 deletions(-)

diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp
index 1863eea2959..0a51503c5f0 100644
--- a/core/matrix/csr.cpp
+++ b/core/matrix/csr.cpp
@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
+#include "core/components/absolute_array.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/matrix/csr_kernels.hpp"
 
@@ -81,7 +82,10 @@ GKO_REGISTER_OPERATION(is_sorted_by_column_index,
                        csr::is_sorted_by_column_index);
 GKO_REGISTER_OPERATION(extract_diagonal, csr::extract_diagonal);
 GKO_REGISTER_OPERATION(fill_array, components::fill_array);
-
+GKO_REGISTER_OPERATION(inplace_absolute_array,
+                       components::inplace_absolute_array);
+GKO_REGISTER_OPERATION(outplace_absolute_array,
+                       components::outplace_absolute_array);
 
 }  // namespace csr
 
@@ -482,6 +486,36 @@ Csr<ValueType, IndexType>::extract_diagonal() const
 }
 
 
+template <typename ValueType, typename IndexType>
+void Csr<ValueType, IndexType>::turn_absolute()
+{
+    auto exec = this->get_executor();
+
+    exec->run(csr::make_inplace_absolute_array(
+        this->get_values(), this->get_num_stored_elements()));
+}
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<typename Csr<ValueType, IndexType>::outplace_absolute_type>
+Csr<ValueType, IndexType>::get_absolute() const
+{
+    auto exec = this->get_executor();
+
+    auto abs_csr = outplace_absolute_type::create(
+        exec, this->get_size(), this->get_num_stored_elements());
+
+    abs_csr->col_idxs_ = col_idxs_;
+    abs_csr->row_ptrs_ = row_ptrs_;
+    exec->run(csr::make_outplace_absolute_array(this->get_const_values(),
+                                                this->get_num_stored_elements(),
+                                                abs_csr->get_values()));
+
+    convert_strategy_helper(abs_csr.get());
+    return abs_csr;
+}
+
+
 #define GKO_DECLARE_CSR_MATRIX(ValueType, IndexType) \
     class Csr<ValueType, IndexType>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_MATRIX);
diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index 2f1bb544ac3..b05ca46327c 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -60,8 +60,8 @@ namespace {
 
 class Csr : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Csr<>;
     using Vec = gko::matrix::Dense<>;
+    using Mtx = gko::matrix::Csr<>;
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
     using ComplexMtx = gko::matrix::Csr<std::complex<double>>;
 
@@ -740,4 +740,48 @@ TEST_F(Csr, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>(cuda));
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>(cuda));
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
+{
+    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(cuda));
+
+    complex_mtx->turn_absolute();
+    complex_dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(complex_mtx.get(), complex_dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
+{
+    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(cuda));
+
+    auto abs_mtx = complex_mtx->get_absolute();
+    auto dabs_mtx = complex_dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index 14ec43d6a4a..4a55b6a6ef8 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -60,8 +60,10 @@ namespace {
 
 class Csr : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Csr<>;
     using Vec = gko::matrix::Dense<>;
+    using Mtx = gko::matrix::Csr<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
+    using ComplexMtx = gko::matrix::Csr<std::complex<double>>;
 
     Csr() : mtx_size(532, 231), rand_engine(42) {}
 
@@ -114,6 +116,16 @@ class Csr : public ::testing::Test {
         dbeta->copy_from(beta.get());
     }
 
+    void set_up_apply_complex_data(
+        std::shared_ptr<ComplexMtx::strategy_type> strategy)
+    {
+        complex_mtx = ComplexMtx::create(ref, strategy);
+        complex_mtx->copy_from(
+            gen_mtx<ComplexVec>(mtx_size[0], mtx_size[1], 1));
+        complex_dmtx = ComplexMtx::create(hip, strategy);
+        complex_dmtx->copy_from(complex_mtx.get());
+    }
+
     struct matrix_pair {
         std::unique_ptr<Mtx> ref;
         std::unique_ptr<Mtx> hip;
@@ -153,6 +165,7 @@ class Csr : public ::testing::Test {
     std::ranlux48 rand_engine;
 
     std::unique_ptr<Mtx> mtx;
+    std::unique_ptr<ComplexMtx> complex_mtx;
     std::unique_ptr<Mtx> square_mtx;
     std::unique_ptr<Vec> expected;
     std::unique_ptr<Vec> y;
@@ -160,6 +173,7 @@ class Csr : public ::testing::Test {
     std::unique_ptr<Vec> beta;
 
     std::unique_ptr<Mtx> dmtx;
+    std::unique_ptr<ComplexMtx> complex_dmtx;
     std::unique_ptr<Mtx> square_dmtx;
     std::unique_ptr<Vec> dresult;
     std::unique_ptr<Vec> dy;
@@ -711,4 +725,48 @@ TEST_F(Csr, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
+{
+    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(hip));
+
+    complex_mtx->turn_absolute();
+    complex_dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(complex_mtx.get(), complex_dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
+{
+    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(hip));
+
+    auto abs_mtx = complex_mtx->get_absolute();
+    auto dabs_mtx = complex_dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index dcf5a1a88cf..2a38b84d31a 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/abstract_factory.hpp>
 #include <ginkgo/core/base/dim.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/matrix_data.hpp>
 #include <ginkgo/core/base/polymorphic_object.hpp>
 #include <ginkgo/core/base/types.hpp>
@@ -622,6 +623,41 @@ class DiagonalExtractable {
         const = 0;
 };
 
+
+/**
+ * The EnableAbsoluteComputation mixin implementing this interface can be
+ * computed. get_absolute gets a new LinOp which contains absolute values from a
+ * LinOp and the valuetype will be removed complex. turn_absolute applies
+ * absolute inplace, so it still keeps the valuetype of the class.
+ *
+ * @tparam ConcreteLinOp  the concrete LinOp which is being implemented
+ *                        [CRTP parameter]
+ *
+ * @ingroup LinOp
+ */
+template <typename ConcreteLinOp>
+class EnableAbsoluteComputation {
+public:
+    using outplace_type = remove_complex<ConcreteLinOp>;
+
+    virtual ~EnableAbsoluteComputation() = default;
+
+    /**
+     * Gets the absolute LinOp
+     *
+     * @return a pointer to the new absolute object
+     */
+    virtual std::unique_ptr<outplace_type> get_absolute() const = 0;
+
+    /**
+     * Turns the value
+     *
+     * @return a pointer to the new absolute object
+     */
+    virtual void turn_absolute() = 0;
+};
+
+
 /**
  * The EnableLinOp mixin can be used to provide sensible default implementations
  * of the majority of the LinOp and PolymorphicObject interface.
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 946b6b4608b..ce62ed82e7f 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -133,6 +133,27 @@ struct remove_complex_impl<std::complex<T>> {
 };
 
 
+/**
+ * Use the complex type if it is not complex.
+ *
+ * @tparam T  the type being made complex
+ */
+template <typename T>
+struct make_complex_impl {
+    using type = std::complex<T>;
+};
+
+/**
+ * Use the same type if it is complex type.
+ *
+ * @tparam T  the type being made complex
+ */
+template <typename T>
+struct make_complex_impl<std::complex<T>> {
+    using type = std::complex<T>;
+};
+
+
 template <typename T>
 struct is_complex_impl : public std::integral_constant<bool, false> {};
 
@@ -200,6 +221,31 @@ struct remove_complex_s<
 };
 
 
+template <typename T, typename = void>
+struct make_complex_s {};
+
+/**
+ * Obtains a complex counterpart of a real type, and leaves the type
+ * unchanged if it is a complex type for complex/scalar type.
+ */
+template <typename T>
+struct make_complex_s<
+    T, xstd::void_t<std::enable_if_t<is_complex_or_scalar_impl<T>::value>>> {
+    using type = typename detail::make_complex_impl<T>::type;
+};
+
+/**
+ * Obtains a complex counterpart of a class with template parameters, which
+ * converts real parameters to complex parameters.
+ */
+template <typename T>
+struct make_complex_s<
+    T, xstd::void_t<std::enable_if_t<!is_complex_or_scalar_impl<T>::value>>> {
+    using type =
+        typename detail::template_convertor<detail::make_complex_impl, T>::type;
+};
+
+
 }  // namespace detail
 
 
@@ -293,6 +339,24 @@ template <typename T>
 using remove_complex = typename remove_complex_s<T>::type;
 
 
+/**
+ * Removes the complex of complex/scalar type or the template parameter of class
+ * by accessing the `type` attribute of this struct.
+ *
+ * @tparam T  type to remove complex
+ */
+using detail::make_complex_s;
+
+/**
+ * Obtain the type which removed the complex of complex/scalar type or the
+ * template parameter of class by accessing the `type` attribute of this struct.
+ *
+ * @tparam T  type to remove complex
+ */
+template <typename T>
+using make_complex = typename make_complex_s<T>::type;
+
+
 namespace detail {
 
 
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index 11ef09907fb..ad9d77ee907 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -36,6 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/math.hpp>
 
 
 namespace gko {
@@ -129,7 +130,8 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
             public ReadableFromMatrixData<ValueType, IndexType>,
             public WritableToMatrixData<ValueType, IndexType>,
             public Transposable,
-            public Permutable<IndexType> {
+            public Permutable<IndexType>,
+            public EnableAbsoluteComputation<Csr<ValueType, IndexType>> {
     friend class EnableCreateMethod<Csr>;
     friend class EnablePolymorphicObject<Csr, LinOp>;
     friend class Coo<ValueType, IndexType>;
@@ -139,12 +141,21 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
     friend class Sellp<ValueType, IndexType>;
     friend class SparsityCsr<ValueType, IndexType>;
     friend class CsrBuilder<ValueType, IndexType>;
+    friend class Csr<make_complex<ValueType>, IndexType>;
+    // friend make_complex<Csr>;
+    //   error: can not recognize it is class correctly.
+    // friend class make_complex<Csr>;
+    //   error: using alias template specialization
+    // friend class make_complex_s<Csr<ValueType,IndexType>>::type;
+    //   error: can not recognize it is class correctly.
 
 public:
     using value_type = ValueType;
     using index_type = IndexType;
     using transposed_type = Csr<ValueType, IndexType>;
     using mat_data = matrix_data<ValueType, IndexType>;
+    using outplace_absolute_type =
+        typename EnableAbsoluteComputation<Csr>::outplace_type;
 
     class automatical;
 
@@ -701,6 +712,10 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
+    std::unique_ptr<outplace_absolute_type> get_absolute() const override;
+
+    void turn_absolute() override;
+
     /**
      * Sorts all (value, col_idx) pairs in each row by column index
      */
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index 745ad71a5f0..9674b2da1e3 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -549,4 +549,48 @@ TEST_F(Csr, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    complex_mtx->turn_absolute();
+    complex_dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(complex_mtx.get(), complex_dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_mtx = complex_mtx->get_absolute();
+    auto dabs_mtx = complex_dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp
index dbf8423b910..aef675a9f13 100644
--- a/reference/test/matrix/csr_kernels.cpp
+++ b/reference/test/matrix/csr_kernels.cpp
@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/diagonal.hpp>
@@ -1306,6 +1307,34 @@ TYPED_TEST(Csr, ExtractsDiagonal)
 }
 
 
+TYPED_TEST(Csr, OutplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto mtx = gko::initialize<Mtx>(
+        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
+
+    auto abs_mtx = mtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+    ASSERT_EQ(mtx->get_strategy()->get_name(),
+              abs_mtx->get_strategy()->get_name());
+}
+
+
+TYPED_TEST(Csr, InplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto mtx = gko::initialize<Mtx>(
+        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
+
+    mtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+}
+
+
 template <typename ValueIndexType>
 class CsrComplex : public ::testing::Test {
 protected:
@@ -1323,14 +1352,13 @@ TYPED_TEST(CsrComplex, MtxIsConjugateTransposable)
 {
     using Csr = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
-    using value_type = typename TestFixture::value_type;
 
     auto exec = gko::ReferenceExecutor::create();
     // clang-format off
-        auto mtx2 = gko::initialize<Csr>(
-            {{T{1.0, 2.0}, T{3.0, 0.0}, T{2.0, 0.0}},
-             {T{0.0, 0.0}, T{5.0, - 3.5}, T{0.0,0.0}},
-             {T{0.0, 0.0}, T{0.0, 1.5}, T{2.0,0.0}}}, exec);
+    auto mtx2 = gko::initialize<Csr>(
+        {{T{1.0, 2.0}, T{3.0, 0.0}, T{2.0, 0.0}},
+         {T{0.0, 0.0}, T{5.0, - 3.5}, T{0.0,0.0}},
+         {T{0.0, 0.0}, T{0.0, 1.5}, T{2.0,0.0}}}, exec);
     // clang-format on
 
     auto trans = mtx2->conj_transpose();
@@ -1345,4 +1373,46 @@ TYPED_TEST(CsrComplex, MtxIsConjugateTransposable)
 }
 
 
+TYPED_TEST(CsrComplex, OutplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    auto abs_mtx = mtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+    ASSERT_EQ(mtx->get_strategy()->get_name(),
+              abs_mtx->get_strategy()->get_name());
+}
+
+
+TYPED_TEST(CsrComplex, InplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    mtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+}
+
+
 }  // namespace

From 845b4ce8a7262b304c0c2dabb490af6775da3e23 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 4 Sep 2020 14:16:58 +0200
Subject: [PATCH 0012/1008] always use bigobj

---
 .github/workflows/windows-build.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml
index c1bda6bc2f3..126a029e942 100644
--- a/.github/workflows/windows-build.yml
+++ b/.github/workflows/windows-build.yml
@@ -22,6 +22,9 @@ jobs:
       if: matrix.config.version == 'latest'
       run: |
         choco install cuda -y
+    - name: bigobj_env
+      run: |
+        echo "::set-env name=CXXFLAGS::/bigobj"
     - name: configure
       run: |
         $env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.."
@@ -51,8 +54,7 @@ jobs:
       run: |
         echo "::set-env name=origin_path::$env:PATH"
         echo "::add-path::$pwd\build\windows_shared_library"
-    - name: debug_env
-      if: matrix.config.build_type == 'Debug'
+    - name: bigobj_env
       run: |
         echo "::set-env name=CXXFLAGS::/bigobj"
     - name: configure

From 2ca671346847114fff81fdc9d64851450a2b083a Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 4 Sep 2020 15:54:28 +0200
Subject: [PATCH 0013/1008] coo/ell/sellp/hybrid/diagonal absolute

---
 core/matrix/coo.cpp                        | 34 ++++++++++
 core/matrix/diagonal.cpp                   | 32 +++++++++
 core/matrix/ell.cpp                        | 34 ++++++++++
 core/matrix/hybrid.cpp                     | 33 +++++++++
 core/matrix/sellp.cpp                      | 36 ++++++++++
 cuda/test/matrix/coo_kernels.cpp           | 22 ++++++
 cuda/test/matrix/diagonal_kernels.cpp      | 22 ++++++
 cuda/test/matrix/ell_kernels.cpp           | 22 ++++++
 cuda/test/matrix/hybrid_kernels.cpp        | 22 ++++++
 cuda/test/matrix/sellp_kernels.cpp         | 22 ++++++
 hip/test/matrix/coo_kernels.hip.cpp        | 22 ++++++
 hip/test/matrix/diagonal_kernels.hip.cpp   | 22 ++++++
 hip/test/matrix/ell_kernels.hip.cpp        | 22 ++++++
 hip/test/matrix/hybrid_kernels.hip.cpp     | 22 ++++++
 hip/test/matrix/sellp_kernels.hip.cpp      | 22 ++++++
 include/ginkgo/core/matrix/coo.hpp         | 10 ++-
 include/ginkgo/core/matrix/diagonal.hpp    | 10 ++-
 include/ginkgo/core/matrix/ell.hpp         | 10 ++-
 include/ginkgo/core/matrix/hybrid.hpp      | 12 +++-
 include/ginkgo/core/matrix/sellp.hpp       | 10 ++-
 omp/test/matrix/coo_kernels.cpp            | 22 ++++++
 omp/test/matrix/diagonal_kernels.cpp       | 22 ++++++
 omp/test/matrix/ell_kernels.cpp            | 22 ++++++
 omp/test/matrix/hybrid_kernels.cpp         | 22 ++++++
 omp/test/matrix/sellp_kernels.cpp          | 22 ++++++
 reference/test/matrix/coo_kernels.cpp      | 79 ++++++++++++++++++++++
 reference/test/matrix/csr_kernels.cpp      | 32 ++++-----
 reference/test/matrix/diagonal_kernels.cpp | 67 ++++++++++++++++++
 reference/test/matrix/ell_kernels.cpp      | 79 ++++++++++++++++++++++
 reference/test/matrix/hybrid_kernels.cpp   | 79 ++++++++++++++++++++++
 reference/test/matrix/sellp_kernels.cpp    | 79 ++++++++++++++++++++++
 31 files changed, 945 insertions(+), 21 deletions(-)

diff --git a/core/matrix/coo.cpp b/core/matrix/coo.cpp
index dd5a4dda553..ecff61e6d4e 100644
--- a/core/matrix/coo.cpp
+++ b/core/matrix/coo.cpp
@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "core/components/absolute_array.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/matrix/coo_kernels.hpp"
 
@@ -64,6 +65,10 @@ GKO_REGISTER_OPERATION(convert_to_csr, coo::convert_to_csr);
 GKO_REGISTER_OPERATION(convert_to_dense, coo::convert_to_dense);
 GKO_REGISTER_OPERATION(extract_diagonal, coo::extract_diagonal);
 GKO_REGISTER_OPERATION(fill_array, components::fill_array);
+GKO_REGISTER_OPERATION(inplace_absolute_array,
+                       components::inplace_absolute_array);
+GKO_REGISTER_OPERATION(outplace_absolute_array,
+                       components::outplace_absolute_array);
 
 
 }  // namespace coo
@@ -233,6 +238,35 @@ Coo<ValueType, IndexType>::extract_diagonal() const
 }
 
 
+template <typename ValueType, typename IndexType>
+void Coo<ValueType, IndexType>::turn_absolute()
+{
+    auto exec = this->get_executor();
+
+    exec->run(coo::make_inplace_absolute_array(
+        this->get_values(), this->get_num_stored_elements()));
+}
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<typename Coo<ValueType, IndexType>::outplace_absolute_type>
+Coo<ValueType, IndexType>::get_absolute() const
+{
+    auto exec = this->get_executor();
+
+    auto abs_coo = outplace_absolute_type::create(
+        exec, this->get_size(), this->get_num_stored_elements());
+
+    abs_coo->col_idxs_ = col_idxs_;
+    abs_coo->row_idxs_ = row_idxs_;
+    exec->run(coo::make_outplace_absolute_array(this->get_const_values(),
+                                                this->get_num_stored_elements(),
+                                                abs_coo->get_values()));
+
+    return abs_coo;
+}
+
+
 #define GKO_DECLARE_COO_MATRIX(ValueType, IndexType) \
     class Coo<ValueType, IndexType>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_MATRIX);
diff --git a/core/matrix/diagonal.cpp b/core/matrix/diagonal.cpp
index 04130cd4604..83b8d826019 100644
--- a/core/matrix/diagonal.cpp
+++ b/core/matrix/diagonal.cpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "core/components/absolute_array.hpp"
 #include "core/matrix/diagonal_kernels.hpp"
 
 
@@ -52,6 +53,10 @@ GKO_REGISTER_OPERATION(apply_to_csr, diagonal::apply_to_csr);
 GKO_REGISTER_OPERATION(right_apply_to_csr, diagonal::right_apply_to_csr);
 GKO_REGISTER_OPERATION(convert_to_csr, diagonal::convert_to_csr);
 GKO_REGISTER_OPERATION(conj_transpose, diagonal::conj_transpose);
+GKO_REGISTER_OPERATION(inplace_absolute_array,
+                       components::inplace_absolute_array);
+GKO_REGISTER_OPERATION(outplace_absolute_array,
+                       components::outplace_absolute_array);
 
 
 }  // namespace diagonal
@@ -261,6 +266,33 @@ void Diagonal<ValueType>::write(mat_data32 &data) const
 }
 
 
+template <typename ValueType>
+void Diagonal<ValueType>::turn_absolute()
+{
+    auto exec = this->get_executor();
+
+    exec->run(diagonal::make_inplace_absolute_array(this->get_values(),
+                                                    this->get_size()[0]));
+}
+
+
+template <typename ValueType>
+std::unique_ptr<typename Diagonal<ValueType>::outplace_absolute_type>
+Diagonal<ValueType>::get_absolute() const
+{
+    auto exec = this->get_executor();
+
+    auto abs_diagonal =
+        outplace_absolute_type::create(exec, this->get_size()[0]);
+
+    exec->run(diagonal::make_outplace_absolute_array(
+        this->get_const_values(), this->get_size()[0],
+        abs_diagonal->get_values()));
+
+    return abs_diagonal;
+}
+
+
 #define GKO_DECLARE_DIAGONAL_MATRIX(value_type) class Diagonal<value_type>
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_MATRIX);
 
diff --git a/core/matrix/ell.cpp b/core/matrix/ell.cpp
index 5f7591421e1..a16ec65631b 100644
--- a/core/matrix/ell.cpp
+++ b/core/matrix/ell.cpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "core/components/absolute_array.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/matrix/ell_kernels.hpp"
 
@@ -62,6 +63,10 @@ GKO_REGISTER_OPERATION(calculate_nonzeros_per_row,
                        ell::calculate_nonzeros_per_row);
 GKO_REGISTER_OPERATION(extract_diagonal, ell::extract_diagonal);
 GKO_REGISTER_OPERATION(fill_array, components::fill_array);
+GKO_REGISTER_OPERATION(inplace_absolute_array,
+                       components::inplace_absolute_array);
+GKO_REGISTER_OPERATION(outplace_absolute_array,
+                       components::outplace_absolute_array);
 
 
 }  // namespace ell
@@ -251,6 +256,35 @@ Ell<ValueType, IndexType>::extract_diagonal() const
 }
 
 
+template <typename ValueType, typename IndexType>
+void Ell<ValueType, IndexType>::turn_absolute()
+{
+    auto exec = this->get_executor();
+
+    exec->run(ell::make_inplace_absolute_array(
+        this->get_values(), this->get_num_stored_elements()));
+}
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<typename Ell<ValueType, IndexType>::outplace_absolute_type>
+Ell<ValueType, IndexType>::get_absolute() const
+{
+    auto exec = this->get_executor();
+
+    auto abs_ell = outplace_absolute_type::create(
+        exec, this->get_size(), this->get_num_stored_elements_per_row(),
+        this->get_stride());
+
+    abs_ell->col_idxs_ = col_idxs_;
+    exec->run(ell::make_outplace_absolute_array(this->get_const_values(),
+                                                this->get_num_stored_elements(),
+                                                abs_ell->get_values()));
+
+    return abs_ell;
+}
+
+
 #define GKO_DECLARE_ELL_MATRIX(ValueType, IndexType) \
     class Ell<ValueType, IndexType>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ELL_MATRIX);
diff --git a/core/matrix/hybrid.cpp b/core/matrix/hybrid.cpp
index d5f012b79c7..b6d9552c9e5 100644
--- a/core/matrix/hybrid.cpp
+++ b/core/matrix/hybrid.cpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "core/components/absolute_array.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/matrix/coo_kernels.hpp"
 #include "core/matrix/ell_kernels.hpp"
@@ -60,6 +61,10 @@ GKO_REGISTER_OPERATION(count_nonzeros, hybrid::count_nonzeros);
 GKO_REGISTER_OPERATION(extract_coo_diagonal, coo::extract_diagonal);
 GKO_REGISTER_OPERATION(extract_ell_diagonal, ell::extract_diagonal);
 GKO_REGISTER_OPERATION(fill_array, components::fill_array);
+GKO_REGISTER_OPERATION(inplace_absolute_array,
+                       components::inplace_absolute_array);
+GKO_REGISTER_OPERATION(outplace_absolute_array,
+                       components::outplace_absolute_array);
 
 
 }  // namespace hybrid
@@ -285,6 +290,34 @@ Hybrid<ValueType, IndexType>::extract_diagonal() const
 }
 
 
+template <typename ValueType, typename IndexType>
+void Hybrid<ValueType, IndexType>::turn_absolute()
+{
+    auto exec = this->get_executor();
+
+    exec->run(hybrid::make_inplace_absolute_array(
+        this->get_ell_values(), this->get_ell_num_stored_elements()));
+    exec->run(hybrid::make_inplace_absolute_array(
+        this->get_coo_values(), this->get_coo_num_stored_elements()));
+}
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<typename Hybrid<ValueType, IndexType>::outplace_absolute_type>
+Hybrid<ValueType, IndexType>::get_absolute() const
+{
+    auto exec = this->get_executor();
+
+    // use default strategy
+    auto abs_hybrid = outplace_absolute_type::create(exec, this->get_size());
+
+    abs_hybrid->ell_->copy_from(ell_->get_absolute());
+    abs_hybrid->coo_->copy_from(coo_->get_absolute());
+
+    return abs_hybrid;
+}
+
+
 #define GKO_DECLARE_HYBRID_MATRIX(ValueType, IndexType) \
     class Hybrid<ValueType, IndexType>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_HYBRID_MATRIX);
diff --git a/core/matrix/sellp.cpp b/core/matrix/sellp.cpp
index 7164af154f4..916d531d072 100644
--- a/core/matrix/sellp.cpp
+++ b/core/matrix/sellp.cpp
@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/base/allocator.hpp"
+#include "core/components/absolute_array.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/matrix/sellp_kernels.hpp"
 
@@ -58,6 +59,10 @@ GKO_REGISTER_OPERATION(convert_to_csr, sellp::convert_to_csr);
 GKO_REGISTER_OPERATION(count_nonzeros, sellp::count_nonzeros);
 GKO_REGISTER_OPERATION(extract_diagonal, sellp::extract_diagonal);
 GKO_REGISTER_OPERATION(fill_array, components::fill_array);
+GKO_REGISTER_OPERATION(inplace_absolute_array,
+                       components::inplace_absolute_array);
+GKO_REGISTER_OPERATION(outplace_absolute_array,
+                       components::outplace_absolute_array);
 
 
 }  // namespace sellp
@@ -304,6 +309,37 @@ Sellp<ValueType, IndexType>::extract_diagonal() const
 }
 
 
+template <typename ValueType, typename IndexType>
+void Sellp<ValueType, IndexType>::turn_absolute()
+{
+    auto exec = this->get_executor();
+
+    exec->run(sellp::make_inplace_absolute_array(
+        this->get_values(), this->get_num_stored_elements()));
+}
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<typename Sellp<ValueType, IndexType>::outplace_absolute_type>
+Sellp<ValueType, IndexType>::get_absolute() const
+{
+    auto exec = this->get_executor();
+
+    auto abs_sellp = outplace_absolute_type::create(
+        exec, this->get_size(), this->get_slice_size(),
+        this->get_stride_factor(), this->get_total_cols());
+
+    abs_sellp->col_idxs_ = col_idxs_;
+    abs_sellp->slice_lengths_ = slice_lengths_;
+    abs_sellp->slice_sets_ = slice_sets_;
+    exec->run(sellp::make_outplace_absolute_array(
+        this->get_const_values(), this->get_num_stored_elements(),
+        abs_sellp->get_values()));
+
+    return abs_sellp;
+}
+
+
 #define GKO_DECLARE_SELLP_MATRIX(ValueType, IndexType) \
     class Sellp<ValueType, IndexType>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_SELLP_MATRIX);
diff --git a/cuda/test/matrix/coo_kernels.cpp b/cuda/test/matrix/coo_kernels.cpp
index 3522e918cc6..330e7d7301a 100644
--- a/cuda/test/matrix/coo_kernels.cpp
+++ b/cuda/test/matrix/coo_kernels.cpp
@@ -271,4 +271,26 @@ TEST_F(Coo, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Coo, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Coo, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/cuda/test/matrix/diagonal_kernels.cpp b/cuda/test/matrix/diagonal_kernels.cpp
index 8c8bbe207e2..8da64b3b8dc 100644
--- a/cuda/test/matrix/diagonal_kernels.cpp
+++ b/cuda/test/matrix/diagonal_kernels.cpp
@@ -249,4 +249,26 @@ TEST_F(Diagonal, ConjTransposeIsEquivalentToRef)
 }
 
 
+TEST_F(Diagonal, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    diag->turn_absolute();
+    ddiag->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 1e-14);
+}
+
+
+TEST_F(Diagonal, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_diag = diag->get_absolute();
+    auto dabs_diag = ddiag->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_diag.get(), dabs_diag.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/cuda/test/matrix/ell_kernels.cpp b/cuda/test/matrix/ell_kernels.cpp
index 960faaed20e..6a4d3b643f3 100644
--- a/cuda/test/matrix/ell_kernels.cpp
+++ b/cuda/test/matrix/ell_kernels.cpp
@@ -356,4 +356,26 @@ TEST_F(Ell, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Ell, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/cuda/test/matrix/hybrid_kernels.cpp b/cuda/test/matrix/hybrid_kernels.cpp
index 5e7048632bc..4ce0f86943c 100644
--- a/cuda/test/matrix/hybrid_kernels.cpp
+++ b/cuda/test/matrix/hybrid_kernels.cpp
@@ -231,4 +231,26 @@ TEST_F(Hybrid, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Hybrid, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/cuda/test/matrix/sellp_kernels.cpp b/cuda/test/matrix/sellp_kernels.cpp
index a3cdeed95ad..8b993e2f964 100644
--- a/cuda/test/matrix/sellp_kernels.cpp
+++ b/cuda/test/matrix/sellp_kernels.cpp
@@ -350,4 +350,26 @@ TEST_F(Sellp, ExtractDiagonalWithSliceSizeAndStrideFactorIsEquivalentToRef)
 }
 
 
+TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_matrix(32, 2);
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_matrix(32, 2);
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/hip/test/matrix/coo_kernels.hip.cpp b/hip/test/matrix/coo_kernels.hip.cpp
index 3e23bb6f4a4..a3bc99afd6a 100644
--- a/hip/test/matrix/coo_kernels.hip.cpp
+++ b/hip/test/matrix/coo_kernels.hip.cpp
@@ -271,4 +271,26 @@ TEST_F(Coo, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Coo, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Coo, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/hip/test/matrix/diagonal_kernels.hip.cpp b/hip/test/matrix/diagonal_kernels.hip.cpp
index 6b87034006e..1d3c118a271 100644
--- a/hip/test/matrix/diagonal_kernels.hip.cpp
+++ b/hip/test/matrix/diagonal_kernels.hip.cpp
@@ -249,4 +249,26 @@ TEST_F(Diagonal, ConjTransposeIsEquivalentToRef)
 }
 
 
+TEST_F(Diagonal, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    diag->turn_absolute();
+    ddiag->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 1e-14);
+}
+
+
+TEST_F(Diagonal, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_diag = diag->get_absolute();
+    auto dabs_diag = ddiag->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_diag.get(), dabs_diag.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/hip/test/matrix/ell_kernels.hip.cpp b/hip/test/matrix/ell_kernels.hip.cpp
index fa9a2df717a..0646691b88d 100644
--- a/hip/test/matrix/ell_kernels.hip.cpp
+++ b/hip/test/matrix/ell_kernels.hip.cpp
@@ -356,4 +356,26 @@ TEST_F(Ell, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Ell, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/hip/test/matrix/hybrid_kernels.hip.cpp b/hip/test/matrix/hybrid_kernels.hip.cpp
index a765cbc2717..04fcae927bf 100644
--- a/hip/test/matrix/hybrid_kernels.hip.cpp
+++ b/hip/test/matrix/hybrid_kernels.hip.cpp
@@ -231,4 +231,26 @@ TEST_F(Hybrid, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Hybrid, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/hip/test/matrix/sellp_kernels.hip.cpp b/hip/test/matrix/sellp_kernels.hip.cpp
index 80c3cf7c113..9a1d12fe3c7 100644
--- a/hip/test/matrix/sellp_kernels.hip.cpp
+++ b/hip/test/matrix/sellp_kernels.hip.cpp
@@ -349,4 +349,26 @@ TEST_F(Sellp, ExtractDiagonalWithSliceSizeAndStrideFactorIsEquivalentToRef)
 }
 
 
+TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_matrix(32, 2);
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_matrix(32, 2);
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/include/ginkgo/core/matrix/coo.hpp b/include/ginkgo/core/matrix/coo.hpp
index b897d50e6d3..7508d312fdc 100644
--- a/include/ginkgo/core/matrix/coo.hpp
+++ b/include/ginkgo/core/matrix/coo.hpp
@@ -81,12 +81,14 @@ class Coo : public EnableLinOp<Coo<ValueType, IndexType>>,
             public ConvertibleTo<Dense<ValueType>>,
             public DiagonalExtractable<ValueType>,
             public ReadableFromMatrixData<ValueType, IndexType>,
-            public WritableToMatrixData<ValueType, IndexType> {
+            public WritableToMatrixData<ValueType, IndexType>,
+            public EnableAbsoluteComputation<Coo<ValueType, IndexType>> {
     friend class EnableCreateMethod<Coo>;
     friend class EnablePolymorphicObject<Coo, LinOp>;
     friend class Csr<ValueType, IndexType>;
     friend class Dense<ValueType>;
     friend class CooBuilder<ValueType, IndexType>;
+    friend class Coo<make_complex<ValueType>, IndexType>;
 
 public:
     using EnableLinOp<Coo>::convert_to;
@@ -95,6 +97,8 @@ class Coo : public EnableLinOp<Coo<ValueType, IndexType>>,
     using value_type = ValueType;
     using index_type = IndexType;
     using mat_data = matrix_data<ValueType, IndexType>;
+    using outplace_absolute_type =
+        typename EnableAbsoluteComputation<Coo>::outplace_type;
 
     friend class Coo<next_precision<ValueType>, IndexType>;
 
@@ -117,6 +121,10 @@ class Coo : public EnableLinOp<Coo<ValueType, IndexType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
+    std::unique_ptr<outplace_absolute_type> get_absolute() const override;
+
+    void turn_absolute() override;
+
     /**
      * Returns the values of the matrix.
      *
diff --git a/include/ginkgo/core/matrix/diagonal.hpp b/include/ginkgo/core/matrix/diagonal.hpp
index c70c7889b37..38526439d95 100644
--- a/include/ginkgo/core/matrix/diagonal.hpp
+++ b/include/ginkgo/core/matrix/diagonal.hpp
@@ -72,11 +72,13 @@ class Diagonal : public EnableLinOp<Diagonal<ValueType>>,
                  public WritableToMatrixData<ValueType, int32>,
                  public WritableToMatrixData<ValueType, int64>,
                  public ReadableFromMatrixData<ValueType, int32>,
-                 public ReadableFromMatrixData<ValueType, int64> {
+                 public ReadableFromMatrixData<ValueType, int64>,
+                 public EnableAbsoluteComputation<Diagonal<ValueType>> {
     friend class EnablePolymorphicObject<Diagonal, LinOp>;
     friend class EnableCreateMethod<Diagonal>;
     friend class Csr<ValueType, int32>;
     friend class Csr<ValueType, int64>;
+    friend class Diagonal<make_complex<ValueType>>;
 
 public:
     using EnableLinOp<Diagonal>::convert_to;
@@ -86,6 +88,8 @@ class Diagonal : public EnableLinOp<Diagonal<ValueType>>,
     using index_type = int64;
     using mat_data = gko::matrix_data<ValueType, int64>;
     using mat_data32 = gko::matrix_data<ValueType, int32>;
+    using outplace_absolute_type =
+        typename EnableAbsoluteComputation<Diagonal>::outplace_type;
 
     std::unique_ptr<LinOp> transpose() const override;
 
@@ -99,6 +103,10 @@ class Diagonal : public EnableLinOp<Diagonal<ValueType>>,
 
     void move_to(Csr<ValueType, int64> *result) override;
 
+    std::unique_ptr<outplace_absolute_type> get_absolute() const override;
+
+    void turn_absolute() override;
+
     /**
      * Returns a pointer to the array of values of the matrix.
      *
diff --git a/include/ginkgo/core/matrix/ell.hpp b/include/ginkgo/core/matrix/ell.hpp
index b29aba22a46..f9c68fb0171 100644
--- a/include/ginkgo/core/matrix/ell.hpp
+++ b/include/ginkgo/core/matrix/ell.hpp
@@ -75,11 +75,13 @@ class Ell : public EnableLinOp<Ell<ValueType, IndexType>>,
             public ConvertibleTo<Csr<ValueType, IndexType>>,
             public DiagonalExtractable<ValueType>,
             public ReadableFromMatrixData<ValueType, IndexType>,
-            public WritableToMatrixData<ValueType, IndexType> {
+            public WritableToMatrixData<ValueType, IndexType>,
+            public EnableAbsoluteComputation<Ell<ValueType, IndexType>> {
     friend class EnableCreateMethod<Ell>;
     friend class EnablePolymorphicObject<Ell, LinOp>;
     friend class Dense<ValueType>;
     friend class Csr<ValueType, IndexType>;
+    friend class Ell<make_complex<ValueType>, IndexType>;
 
 public:
     using EnableLinOp<Ell>::convert_to;
@@ -88,6 +90,8 @@ class Ell : public EnableLinOp<Ell<ValueType, IndexType>>,
     using value_type = ValueType;
     using index_type = IndexType;
     using mat_data = matrix_data<ValueType, IndexType>;
+    using outplace_absolute_type =
+        typename EnableAbsoluteComputation<Ell>::outplace_type;
 
     friend class Ell<next_precision<ValueType>, IndexType>;
 
@@ -110,6 +114,10 @@ class Ell : public EnableLinOp<Ell<ValueType, IndexType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
+    std::unique_ptr<outplace_absolute_type> get_absolute() const override;
+
+    void turn_absolute() override;
+
     /**
      * Returns the values of the matrix.
      *
diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp
index 4d35165e10e..7f544da8b10 100644
--- a/include/ginkgo/core/matrix/hybrid.hpp
+++ b/include/ginkgo/core/matrix/hybrid.hpp
@@ -76,11 +76,14 @@ class Hybrid
       public ConvertibleTo<Csr<ValueType, IndexType>>,
       public DiagonalExtractable<ValueType>,
       public ReadableFromMatrixData<ValueType, IndexType>,
-      public WritableToMatrixData<ValueType, IndexType> {
+      public WritableToMatrixData<ValueType, IndexType>,
+      public EnableAbsoluteComputation<Hybrid<ValueType, IndexType>> {
     friend class EnableCreateMethod<Hybrid>;
     friend class EnablePolymorphicObject<Hybrid, LinOp>;
     friend class Dense<ValueType>;
     friend class Csr<ValueType, IndexType>;
+    friend class Hybrid<make_complex<ValueType>, IndexType>;
+
 
 public:
     using EnableLinOp<Hybrid>::convert_to;
@@ -91,6 +94,9 @@ class Hybrid
     using mat_data = matrix_data<ValueType, IndexType>;
     using coo_type = Coo<ValueType, IndexType>;
     using ell_type = Ell<ValueType, IndexType>;
+    using outplace_absolute_type =
+        typename EnableAbsoluteComputation<Hybrid>::outplace_type;
+
 
     /**
      * strategy_type is to decide how to set the hybrid config. It
@@ -357,6 +363,10 @@ class Hybrid
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
+    std::unique_ptr<outplace_absolute_type> get_absolute() const override;
+
+    void turn_absolute() override;
+
     /**
      * Returns the values of the ell part.
      *
diff --git a/include/ginkgo/core/matrix/sellp.hpp b/include/ginkgo/core/matrix/sellp.hpp
index caaa6c20fbb..3c6cb087be5 100644
--- a/include/ginkgo/core/matrix/sellp.hpp
+++ b/include/ginkgo/core/matrix/sellp.hpp
@@ -72,11 +72,13 @@ class Sellp : public EnableLinOp<Sellp<ValueType, IndexType>>,
               public ConvertibleTo<Csr<ValueType, IndexType>>,
               public DiagonalExtractable<ValueType>,
               public ReadableFromMatrixData<ValueType, IndexType>,
-              public WritableToMatrixData<ValueType, IndexType> {
+              public WritableToMatrixData<ValueType, IndexType>,
+              public EnableAbsoluteComputation<Sellp<ValueType, IndexType>> {
     friend class EnableCreateMethod<Sellp>;
     friend class EnablePolymorphicObject<Sellp, LinOp>;
     friend class Dense<ValueType>;
     friend class Csr<ValueType, IndexType>;
+    friend class Sellp<make_complex<ValueType>, IndexType>;
 
 public:
     using EnableLinOp<Sellp>::convert_to;
@@ -85,6 +87,8 @@ class Sellp : public EnableLinOp<Sellp<ValueType, IndexType>>,
     using value_type = ValueType;
     using index_type = IndexType;
     using mat_data = matrix_data<ValueType, IndexType>;
+    using outplace_absolute_type =
+        typename EnableAbsoluteComputation<Sellp>::outplace_type;
 
     friend class Sellp<next_precision<ValueType>, IndexType>;
 
@@ -107,6 +111,10 @@ class Sellp : public EnableLinOp<Sellp<ValueType, IndexType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
+    std::unique_ptr<outplace_absolute_type> get_absolute() const override;
+
+    void turn_absolute() override;
+
     /**
      * Returns the values of the matrix.
      *
diff --git a/omp/test/matrix/coo_kernels.cpp b/omp/test/matrix/coo_kernels.cpp
index a775a80b4cb..cc1da48c5ea 100644
--- a/omp/test/matrix/coo_kernels.cpp
+++ b/omp/test/matrix/coo_kernels.cpp
@@ -234,4 +234,26 @@ TEST_F(Coo, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Coo, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Coo, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/omp/test/matrix/diagonal_kernels.cpp b/omp/test/matrix/diagonal_kernels.cpp
index f4a8a77767f..d9075a450d2 100644
--- a/omp/test/matrix/diagonal_kernels.cpp
+++ b/omp/test/matrix/diagonal_kernels.cpp
@@ -248,4 +248,26 @@ TEST_F(Diagonal, ConjTransposeIsEquivalentToRef)
 }
 
 
+TEST_F(Diagonal, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    diag->turn_absolute();
+    ddiag->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 1e-14);
+}
+
+
+TEST_F(Diagonal, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_diag = diag->get_absolute();
+    auto dabs_diag = ddiag->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_diag.get(), dabs_diag.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/omp/test/matrix/ell_kernels.cpp b/omp/test/matrix/ell_kernels.cpp
index 03b4e42579b..f37c313591b 100644
--- a/omp/test/matrix/ell_kernels.cpp
+++ b/omp/test/matrix/ell_kernels.cpp
@@ -232,4 +232,26 @@ TEST_F(Ell, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Ell, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/omp/test/matrix/hybrid_kernels.cpp b/omp/test/matrix/hybrid_kernels.cpp
index 303853561b0..c0b3a329d93 100644
--- a/omp/test/matrix/hybrid_kernels.cpp
+++ b/omp/test/matrix/hybrid_kernels.cpp
@@ -230,4 +230,26 @@ TEST_F(Hybrid, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Hybrid, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/omp/test/matrix/sellp_kernels.cpp b/omp/test/matrix/sellp_kernels.cpp
index 996f4a672f2..1c758de6113 100644
--- a/omp/test/matrix/sellp_kernels.cpp
+++ b/omp/test/matrix/sellp_kernels.cpp
@@ -266,4 +266,26 @@ TEST_F(Sellp, ExtractDiagonalWithSliceSizeAndStrideFactorIsEquivalentToRef)
 }
 
 
+TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->turn_absolute();
+    dmtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+}
+
+
+TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_mtx = mtx->get_absolute();
+    auto dabs_mtx = dmtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/reference/test/matrix/coo_kernels.cpp b/reference/test/matrix/coo_kernels.cpp
index 704efff502d..e0666b7ccda 100644
--- a/reference/test/matrix/coo_kernels.cpp
+++ b/reference/test/matrix/coo_kernels.cpp
@@ -534,4 +534,83 @@ TYPED_TEST(Coo, ExtractsDiagonal)
 }
 
 
+TYPED_TEST(Coo, InplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto mtx = gko::initialize<Mtx>(
+        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
+
+    mtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+}
+
+
+TYPED_TEST(Coo, OutplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto mtx = gko::initialize<Mtx>(
+        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
+
+    auto abs_mtx = mtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+}
+
+
+template <typename ValueIndexType>
+class CooComplex : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using Mtx = gko::matrix::Coo<value_type, index_type>;
+};
+
+TYPED_TEST_CASE(CooComplex, gko::test::ComplexValueIndexTypes);
+
+
+TYPED_TEST(CooComplex, OutplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    auto abs_mtx = mtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+}
+
+
+TYPED_TEST(CooComplex, InplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    mtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+}
+
+
 }  // namespace
diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp
index aef675a9f13..3ab5fbe3269 100644
--- a/reference/test/matrix/csr_kernels.cpp
+++ b/reference/test/matrix/csr_kernels.cpp
@@ -1307,31 +1307,31 @@ TYPED_TEST(Csr, ExtractsDiagonal)
 }
 
 
-TYPED_TEST(Csr, OutplaceAbsolute)
+TYPED_TEST(Csr, InplaceAbsolute)
 {
     using Mtx = typename TestFixture::Mtx;
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    auto abs_mtx = mtx->get_absolute();
+    mtx->turn_absolute();
 
     GKO_ASSERT_MTX_NEAR(
-        abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
-    ASSERT_EQ(mtx->get_strategy()->get_name(),
-              abs_mtx->get_strategy()->get_name());
+        mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
 }
 
 
-TYPED_TEST(Csr, InplaceAbsolute)
+TYPED_TEST(Csr, OutplaceAbsolute)
 {
     using Mtx = typename TestFixture::Mtx;
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    mtx->turn_absolute();
+    auto abs_mtx = mtx->get_absolute();
 
     GKO_ASSERT_MTX_NEAR(
-        mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+        abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+    ASSERT_EQ(mtx->get_strategy()->get_name(),
+              abs_mtx->get_strategy()->get_name());
 }
 
 
@@ -1373,7 +1373,7 @@ TYPED_TEST(CsrComplex, MtxIsConjugateTransposable)
 }
 
 
-TYPED_TEST(CsrComplex, OutplaceAbsolute)
+TYPED_TEST(CsrComplex, InplaceAbsolute)
 {
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
@@ -1386,16 +1386,14 @@ TYPED_TEST(CsrComplex, OutplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    auto abs_mtx = mtx->get_absolute();
+    mtx->turn_absolute();
 
     GKO_ASSERT_MTX_NEAR(
-        abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
-    ASSERT_EQ(mtx->get_strategy()->get_name(),
-              abs_mtx->get_strategy()->get_name());
+        mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
 }
 
 
-TYPED_TEST(CsrComplex, InplaceAbsolute)
+TYPED_TEST(CsrComplex, OutplaceAbsolute)
 {
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
@@ -1408,10 +1406,12 @@ TYPED_TEST(CsrComplex, InplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    mtx->turn_absolute();
+    auto abs_mtx = mtx->get_absolute();
 
     GKO_ASSERT_MTX_NEAR(
-        mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+        abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+    ASSERT_EQ(mtx->get_strategy()->get_name(),
+              abs_mtx->get_strategy()->get_name());
 }
 
 
diff --git a/reference/test/matrix/diagonal_kernels.cpp b/reference/test/matrix/diagonal_kernels.cpp
index 6da642d6305..d2ab3f3453c 100644
--- a/reference/test/matrix/diagonal_kernels.cpp
+++ b/reference/test/matrix/diagonal_kernels.cpp
@@ -356,6 +356,31 @@ TYPED_TEST(Diagonal, ConvertsToCsr)
 }
 
 
+TYPED_TEST(Diagonal, InplaceAbsolute)
+{
+    using value_type = typename TestFixture::value_type;
+
+    this->diag1->turn_absolute();
+    auto values = this->diag1->get_values();
+
+    EXPECT_EQ(values[0], value_type(2.0));
+    EXPECT_EQ(values[1], value_type(3.0));
+}
+
+
+TYPED_TEST(Diagonal, OutplaceAbsolute)
+{
+    using value_type = typename TestFixture::value_type;
+    using abs_type = gko::remove_complex<value_type>;
+
+    auto abs_diag = this->diag1->get_absolute();
+    auto values = abs_diag->get_values();
+
+    EXPECT_EQ(values[0], value_type(2.0));
+    EXPECT_EQ(values[1], value_type(3.0));
+}
+
+
 template <typename ValueType>
 class DiagonalComplex : public ::testing::Test {
 protected:
@@ -388,4 +413,46 @@ TYPED_TEST(DiagonalComplex, MtxIsConjugateTransposable)
 }
 
 
+TYPED_TEST(DiagonalComplex, InplaceAbsolute)
+{
+    using Diag = typename TestFixture::Diag;
+    using value_type = typename TestFixture::value_type;
+
+    auto exec = gko::ReferenceExecutor::create();
+    auto diag = Diag::create(exec, 3);
+    auto diag_values = diag->get_values();
+    diag_values[0] = value_type{3.0, -4.0};
+    diag_values[1] = value_type{-3.0, 0.0};
+    diag_values[2] = value_type{0.0, -1.5};
+
+    diag->turn_absolute();
+
+    EXPECT_EQ(diag_values[0], (value_type{5.0, 0.0}));
+    EXPECT_EQ(diag_values[1], (value_type{3.0, 0.0}));
+    EXPECT_EQ(diag_values[2], (value_type{1.5, 0.0}));
+}
+
+
+TYPED_TEST(DiagonalComplex, OutplaceAbsolute)
+{
+    using Diag = typename TestFixture::Diag;
+    using value_type = typename TestFixture::value_type;
+    using abs_type = gko::remove_complex<value_type>;
+
+    auto exec = gko::ReferenceExecutor::create();
+    auto diag = Diag::create(exec, 3);
+    auto diag_values = diag->get_values();
+    diag_values[0] = value_type{3.0, -4.0};
+    diag_values[1] = value_type{-3.0, 0.0};
+    diag_values[2] = value_type{0.0, -1.5};
+
+    auto abs_diag = diag->get_absolute();
+    auto abs_values = abs_diag->get_values();
+
+    EXPECT_EQ(abs_values[0], (value_type{5.0}));
+    EXPECT_EQ(abs_values[1], (value_type{3.0}));
+    EXPECT_EQ(abs_values[2], (value_type{1.5}));
+}
+
+
 }  // namespace
diff --git a/reference/test/matrix/ell_kernels.cpp b/reference/test/matrix/ell_kernels.cpp
index 2e513b96f7d..47221ecfeb2 100644
--- a/reference/test/matrix/ell_kernels.cpp
+++ b/reference/test/matrix/ell_kernels.cpp
@@ -612,4 +612,83 @@ TYPED_TEST(Ell, ExtractsDiagonal)
 }
 
 
+TYPED_TEST(Ell, InplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto mtx = gko::initialize<Mtx>(
+        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
+
+    mtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+}
+
+
+TYPED_TEST(Ell, OutplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto mtx = gko::initialize<Mtx>(
+        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
+
+    auto abs_mtx = mtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+}
+
+
+template <typename ValueIndexType>
+class EllComplex : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using Mtx = gko::matrix::Ell<value_type, index_type>;
+};
+
+TYPED_TEST_CASE(EllComplex, gko::test::ComplexValueIndexTypes);
+
+
+TYPED_TEST(EllComplex, InplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    mtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+}
+
+
+TYPED_TEST(EllComplex, OutplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    auto abs_mtx = mtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+}
+
+
 }  // namespace
diff --git a/reference/test/matrix/hybrid_kernels.cpp b/reference/test/matrix/hybrid_kernels.cpp
index f108b3375e7..e5bd30fa00e 100644
--- a/reference/test/matrix/hybrid_kernels.cpp
+++ b/reference/test/matrix/hybrid_kernels.cpp
@@ -656,4 +656,83 @@ TYPED_TEST(Hybrid, ExtractsDiagonalWithoutZeros)
 }
 
 
+TYPED_TEST(Hybrid, InplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto mtx = gko::initialize<Mtx>(
+        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
+
+    mtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+}
+
+
+TYPED_TEST(Hybrid, OutplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto mtx = gko::initialize<Mtx>(
+        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
+
+    auto abs_mtx = mtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+}
+
+
+template <typename ValueIndexType>
+class HybridComplex : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using Mtx = gko::matrix::Hybrid<value_type, index_type>;
+};
+
+TYPED_TEST_CASE(HybridComplex, gko::test::ComplexValueIndexTypes);
+
+
+TYPED_TEST(HybridComplex, OutplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    auto abs_mtx = mtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+}
+
+
+TYPED_TEST(HybridComplex, InplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    mtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+}
+
+
 }  // namespace
diff --git a/reference/test/matrix/sellp_kernels.cpp b/reference/test/matrix/sellp_kernels.cpp
index 48a60f24a6b..6ab86e8ce1a 100644
--- a/reference/test/matrix/sellp_kernels.cpp
+++ b/reference/test/matrix/sellp_kernels.cpp
@@ -615,4 +615,83 @@ TYPED_TEST(Sellp, ExtractsDiagonalWithSliceSizeAndStrideFactor)
 }
 
 
+TYPED_TEST(Sellp, InplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto mtx = gko::initialize<Mtx>(
+        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
+
+    mtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+}
+
+
+TYPED_TEST(Sellp, OutplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto mtx = gko::initialize<Mtx>(
+        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
+
+    auto abs_mtx = mtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+}
+
+
+template <typename ValueIndexType>
+class SellpComplex : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using Mtx = gko::matrix::Sellp<value_type, index_type>;
+};
+
+TYPED_TEST_CASE(SellpComplex, gko::test::ComplexValueIndexTypes);
+
+
+TYPED_TEST(SellpComplex, OutplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    auto abs_mtx = mtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+}
+
+
+TYPED_TEST(SellpComplex, InplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    mtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+}
+
+
 }  // namespace

From 5bb1bd982ab5316ee65cb8598060f22b3d4f945c Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 4 Sep 2020 22:38:41 +0200
Subject: [PATCH 0014/1008] implement dense absolute

---
 common/matrix/dense_kernels.hpp.inc      |  29 +++++++
 core/device_hooks/common_kernels.inc.cpp |  10 +++
 core/matrix/dense.cpp                    |  26 ++++++
 core/matrix/dense_kernels.hpp            |  17 +++-
 cuda/matrix/dense_kernels.cu             |  32 +++++++
 cuda/test/matrix/dense_kernels.cpp       |  22 +++++
 hip/matrix/dense_kernels.hip.cpp         |  33 +++++++
 hip/test/matrix/dense_kernels.hip.cpp    |  22 +++++
 include/ginkgo/core/matrix/dense.hpp     |  10 ++-
 omp/matrix/dense_kernels.cpp             |  35 ++++++++
 omp/test/matrix/dense_kernels.cpp        |  22 +++++
 reference/matrix/dense_kernels.cpp       |  31 +++++++
 reference/test/matrix/dense_kernels.cpp  | 106 +++++++++++++++++++++++
 13 files changed, 393 insertions(+), 2 deletions(-)

diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index 9d0285e71f7..49be1ba3724 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -510,4 +510,33 @@ __global__ __launch_bounds__(default_block_size) void extract_diagonal(
 }
 
 
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void inplace_absolute_dense(
+    size_type num_rows, size_type num_cols, ValueType *__restrict__ data,
+    size_type stride)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    auto row = tidx / num_cols;
+    auto col = tidx % num_cols;
+    if (row < num_rows) {
+        data[row * stride + col] = abs(data[row * stride + col]);
+    }
+}
+
+
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void outplace_absolute_dense(
+    size_type num_rows, size_type num_cols, const ValueType *__restrict__ in,
+    size_type stride_in, remove_complex<ValueType> *__restrict__ out,
+    size_type stride_out)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    auto row = tidx / num_cols;
+    auto col = tidx % num_cols;
+    if (row < num_rows) {
+        out[row * stride_out + col] = abs(in[row * stride_out + col]);
+    }
+}
+
+
 }  // namespace kernel
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 7015c9c644e..2f52d52ce8f 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -244,6 +244,16 @@ GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL);
 
+template <typename ValueType>
+GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL);
+
+template <typename ValueType>
+GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
+
 
 }  // namespace dense
 
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index aab1d2e598a..306c83cde71 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -85,6 +85,8 @@ GKO_REGISTER_OPERATION(convert_to_hybrid, dense::convert_to_hybrid);
 GKO_REGISTER_OPERATION(convert_to_sellp, dense::convert_to_sellp);
 GKO_REGISTER_OPERATION(convert_to_sparsity_csr, dense::convert_to_sparsity_csr);
 GKO_REGISTER_OPERATION(extract_diagonal, dense::extract_diagonal);
+GKO_REGISTER_OPERATION(inplace_absolute_dense, dense::inplace_absolute_dense);
+GKO_REGISTER_OPERATION(outplace_absolute_dense, dense::outplace_absolute_dense);
 
 
 }  // namespace dense
@@ -752,6 +754,30 @@ std::unique_ptr<Diagonal<ValueType>> Dense<ValueType>::extract_diagonal() const
 }
 
 
+template <typename ValueType>
+void Dense<ValueType>::turn_absolute()
+{
+    auto exec = this->get_executor();
+
+    exec->run(dense::make_inplace_absolute_dense(this));
+}
+
+
+template <typename ValueType>
+std::unique_ptr<typename Dense<ValueType>::outplace_absolute_type>
+Dense<ValueType>::get_absolute() const
+{
+    auto exec = this->get_executor();
+
+    // do not inherit the stride
+    auto abs_dense = outplace_absolute_type::create(exec, this->get_size());
+
+    exec->run(dense::make_outplace_absolute_dense(this, abs_dense.get()));
+
+    return abs_dense;
+}
+
+
 #define GKO_DECLARE_DENSE_MATRIX(_type) class Dense<_type>
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_MATRIX);
 
diff --git a/core/matrix/dense_kernels.hpp b/core/matrix/dense_kernels.hpp
index 8edeec6878e..30ad6ca4729 100644
--- a/core/matrix/dense_kernels.hpp
+++ b/core/matrix/dense_kernels.hpp
@@ -172,6 +172,17 @@ namespace kernels {
                           const matrix::Dense<_vtype> *orig,           \
                           matrix::Diagonal<_vtype> *diag)
 
+#define GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL(_vtype)                    \
+    void inplace_absolute_dense(std::shared_ptr<const DefaultExecutor> exec, \
+                                matrix::Dense<_vtype> *source)
+
+#define GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL(_vtype) \
+    void outplace_absolute_dense(                          \
+        std::shared_ptr<const DefaultExecutor> exec,       \
+        const matrix::Dense<_vtype> *source,               \
+        matrix::Dense<remove_complex<_vtype>> *result)
+
+
 #define GKO_DECLARE_ALL_AS_TEMPLATES                                        \
     template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL(ValueType);                       \
@@ -220,7 +231,11 @@ namespace kernels {
     template <typename ValueType, typename IndexType>                       \
     GKO_DECLARE_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType);        \
     template <typename ValueType>                                           \
-    GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL(ValueType)
+    GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL(ValueType);                         \
+    template <typename ValueType>                                           \
+    GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL(ValueType);                   \
+    template <typename ValueType>                                           \
+    GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL(ValueType)
 
 
 namespace omp {
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index f0fe2cfae7e..b1c4ae747f8 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -701,6 +701,38 @@ void extract_diagonal(std::shared_ptr<const CudaExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL);
 
 
+template <typename ValueType>
+void inplace_absolute_dense(std::shared_ptr<const CudaExecutor> exec,
+                            matrix::Dense<ValueType> *source)
+{
+    auto dim = source->get_size();
+    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
+
+    kernel::inplace_absolute_dense<<<grid_dim, default_block_size>>>(
+        dim[0], dim[1], as_cuda_type(source->get_values()),
+        source->get_stride());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL);
+
+
+template <typename ValueType>
+void outplace_absolute_dense(std::shared_ptr<const CudaExecutor> exec,
+                             const matrix::Dense<ValueType> *source,
+                             matrix::Dense<remove_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
+
+    kernel::outplace_absolute_dense<<<grid_dim, default_block_size>>>(
+        dim[0], dim[1], as_cuda_type(source->get_const_values()),
+        source->get_stride(), as_cuda_type(result->get_values()),
+        result->get_stride());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
+
+
 }  // namespace dense
 }  // namespace cuda
 }  // namespace kernels
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index c520dfcf2a3..a2cbccfc8f3 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -587,4 +587,26 @@ TEST_F(Dense, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    x->turn_absolute();
+    dx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(x.get(), dx.get(), 1e-14);
+}
+
+
+TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_x = x->get_absolute();
+    auto dabs_x = dx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_x.get(), dabs_x.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index 824d2712b7f..2793d09832f 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -729,6 +729,39 @@ void extract_diagonal(std::shared_ptr<const HipExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL);
 
 
+template <typename ValueType>
+void inplace_absolute_dense(std::shared_ptr<const HipExecutor> exec,
+                            matrix::Dense<ValueType> *source)
+{
+    auto dim = source->get_size();
+    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
+
+    hipLaunchKernelGGL(kernel::inplace_absolute_dense, dim3(grid_dim),
+                       dim3(default_block_size), 0, 0, dim[0], dim[1],
+                       as_hip_type(source->get_values()), source->get_stride());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL);
+
+
+template <typename ValueType>
+void outplace_absolute_dense(std::shared_ptr<const HipExecutor> exec,
+                             const matrix::Dense<ValueType> *source,
+                             matrix::Dense<remove_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
+
+    hipLaunchKernelGGL(kernel::outplace_absolute_dense, dim3(grid_dim),
+                       dim3(default_block_size), 0, 0, dim[0], dim[1],
+                       as_hip_type(source->get_const_values()),
+                       source->get_stride(), as_hip_type(result->get_values()),
+                       result->get_stride());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
+
+
 }  // namespace dense
 }  // namespace hip
 }  // namespace kernels
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index 40de01b14e8..0abb142e1e9 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -569,4 +569,26 @@ TEST_F(Dense, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    x->turn_absolute();
+    dx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(x.get(), dx.get(), 1e-14);
+}
+
+
+TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_x = x->get_absolute();
+    auto dabs_x = dx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_x.get(), dabs_x.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index e4dc05e7017..912040345cd 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -110,7 +110,8 @@ class Dense : public EnableLinOp<Dense<ValueType>>,
               public WritableToMatrixData<ValueType, int64>,
               public Transposable,
               public Permutable<int32>,
-              public Permutable<int64> {
+              public Permutable<int64>,
+              public EnableAbsoluteComputation<Dense<ValueType>> {
     friend class EnableCreateMethod<Dense>;
     friend class EnablePolymorphicObject<Dense, LinOp>;
     friend class Coo<ValueType, int32>;
@@ -126,6 +127,7 @@ class Dense : public EnableLinOp<Dense<ValueType>>,
     friend class Sellp<ValueType, int64>;
     friend class SparsityCsr<ValueType, int32>;
     friend class SparsityCsr<ValueType, int64>;
+    friend class Dense<make_complex<ValueType>>;
 
 public:
     using EnableLinOp<Dense>::convert_to;
@@ -136,6 +138,8 @@ class Dense : public EnableLinOp<Dense<ValueType>>,
     using transposed_type = Dense<ValueType>;
     using mat_data = gko::matrix_data<ValueType, int64>;
     using mat_data32 = gko::matrix_data<ValueType, int32>;
+    using outplace_absolute_type =
+        typename EnableAbsoluteComputation<Dense>::outplace_type;
 
     using row_major_range = gko::range<gko::accessor::row_major<ValueType, 2>>;
 
@@ -245,6 +249,10 @@ class Dense : public EnableLinOp<Dense<ValueType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
+    std::unique_ptr<outplace_absolute_type> get_absolute() const override;
+
+    void turn_absolute() override;
+
     /**
      * Returns a pointer to the array of values of the matrix.
      *
diff --git a/omp/matrix/dense_kernels.cpp b/omp/matrix/dense_kernels.cpp
index abf8037bc9b..c3a5234d956 100644
--- a/omp/matrix/dense_kernels.cpp
+++ b/omp/matrix/dense_kernels.cpp
@@ -769,6 +769,41 @@ void extract_diagonal(std::shared_ptr<const OmpExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL);
 
 
+template <typename ValueType>
+void inplace_absolute_dense(std::shared_ptr<const OmpExecutor> exec,
+                            matrix::Dense<ValueType> *source)
+{
+    auto dim = source->get_size();
+
+#pragma omp parallel for
+    for (size_type row = 0; row < dim[0]; row++) {
+        for (size_type col = 0; col < dim[1]; col++) {
+            source->at(row, col) = abs(source->at(row, col));
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL);
+
+
+template <typename ValueType>
+void outplace_absolute_dense(std::shared_ptr<const OmpExecutor> exec,
+                             const matrix::Dense<ValueType> *source,
+                             matrix::Dense<remove_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+
+#pragma omp parallel for
+    for (size_type row = 0; row < dim[0]; row++) {
+        for (size_type col = 0; col < dim[1]; col++) {
+            result->at(row, col) = abs(source->at(row, col));
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
+
+
 }  // namespace dense
 }  // namespace omp
 }  // namespace kernels
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index 0d82dbd261b..42b91f5bdcd 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -702,4 +702,26 @@ TEST_F(Dense, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    x->turn_absolute();
+    dx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(x.get(), dx.get(), 1e-14);
+}
+
+
+TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_x = x->get_absolute();
+    auto dabs_x = dx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_x.get(), dabs_x.get(), 1e-14);
+}
+
+
 }  // namespace
diff --git a/reference/matrix/dense_kernels.cpp b/reference/matrix/dense_kernels.cpp
index db66b30e0df..75a6ddef342 100644
--- a/reference/matrix/dense_kernels.cpp
+++ b/reference/matrix/dense_kernels.cpp
@@ -684,6 +684,37 @@ void extract_diagonal(std::shared_ptr<const ReferenceExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL);
 
 
+template <typename ValueType>
+void inplace_absolute_dense(std::shared_ptr<const ReferenceExecutor> exec,
+                            matrix::Dense<ValueType> *source)
+{
+    auto dim = source->get_size();
+    for (size_type row = 0; row < dim[0]; row++) {
+        for (size_type col = 0; col < dim[1]; col++) {
+            source->at(row, col) = abs(source->at(row, col));
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL);
+
+
+template <typename ValueType>
+void outplace_absolute_dense(std::shared_ptr<const ReferenceExecutor> exec,
+                             const matrix::Dense<ValueType> *source,
+                             matrix::Dense<remove_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+    for (size_type row = 0; row < dim[0]; row++) {
+        for (size_type col = 0; col < dim[1]; col++) {
+            result->at(row, col) = abs(source->at(row, col));
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
+
+
 }  // namespace dense
 }  // namespace reference
 }  // namespace kernels
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 82fefe8f1df..c41a748d6d5 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -2009,6 +2009,74 @@ TYPED_TEST(Dense, ExtractsDiagonalFromNonSquareMatrix)
 }
 
 
+TYPED_TEST(Dense, InplaceAbsolute)
+{
+    using T = typename TestFixture::value_type;
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    this->mtx5->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(this->mtx5,
+                        l({{1.0, 1.0, 0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
+                        r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Dense, InplaceAbsoluteSubMatrix)
+{
+    using T = typename TestFixture::value_type;
+    // mtx takes the left top corner 2 x 2 matrix
+    auto mtx = this->mtx5->create_submatrix(gko::span{0, 2}, gko::span{0, 2});
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    mtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(this->mtx5,
+                        l({{1.0, 1.0, -0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
+                        r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Dense, OutplaceAbsolute)
+{
+    using T = typename TestFixture::value_type;
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    auto abs_mtx = this->mtx5->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx,
+                        l({{1.0, 1.0, 0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
+                        r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Dense, OutplaceAbsoluteSubMatrix)
+{
+    using T = typename TestFixture::value_type;
+    // mtx takes the left top corner 2 x 2 matrix
+    auto mtx = this->mtx5->create_submatrix(gko::span{0, 2}, gko::span{0, 2});
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    auto abs_mtx = mtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx, l({{1.0, 1.0}, {2.0, 2.0}}),
+                        r<TypeParam>::value);
+    GKO_ASSERT_EQ(abs_mtx->get_stride(), 2);
+}
+
+
 template <typename T>
 class DenseComplex : public ::testing::Test {
 protected:
@@ -2039,4 +2107,42 @@ TYPED_TEST(DenseComplex, NonSquareMatrixIsConjugateTransposable)
 }
 
 
+TYPED_TEST(DenseComplex, InplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    mtx->turn_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+}
+
+
+TYPED_TEST(DenseComplex, OutplaceAbsolute)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    auto abs_mtx = mtx->get_absolute();
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+}
+
+
 }  // namespace

From fbcc57d330995c7c1c0b3fb020d61e4dd0057990 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 7 Sep 2020 17:47:05 +0200
Subject: [PATCH 0015/1008] rename turn_absolute and add interface for LinOp 1.
 rename turn_absolute -> apply_absolute 2. add AbsoluteComputable for LinOp
 usage

---
 core/matrix/coo.cpp                        |   2 +-
 core/matrix/csr.cpp                        |   2 +-
 core/matrix/dense.cpp                      |   2 +-
 core/matrix/diagonal.cpp                   |   2 +-
 core/matrix/ell.cpp                        |   2 +-
 core/matrix/hybrid.cpp                     |   2 +-
 core/matrix/sellp.cpp                      |   2 +-
 core/test/base/lin_op.cpp                  | 102 ++++++++++++++++++++-
 cuda/test/matrix/coo_kernels.cpp           |   4 +-
 cuda/test/matrix/csr_kernels.cpp           |   8 +-
 cuda/test/matrix/dense_kernels.cpp         |   4 +-
 cuda/test/matrix/diagonal_kernels.cpp      |   4 +-
 cuda/test/matrix/ell_kernels.cpp           |   4 +-
 cuda/test/matrix/hybrid_kernels.cpp        |   4 +-
 cuda/test/matrix/sellp_kernels.cpp         |   4 +-
 hip/test/matrix/coo_kernels.hip.cpp        |   4 +-
 hip/test/matrix/csr_kernels.hip.cpp        |   8 +-
 hip/test/matrix/dense_kernels.hip.cpp      |   4 +-
 hip/test/matrix/diagonal_kernels.hip.cpp   |   4 +-
 hip/test/matrix/ell_kernels.hip.cpp        |   4 +-
 hip/test/matrix/hybrid_kernels.hip.cpp     |   4 +-
 hip/test/matrix/sellp_kernels.hip.cpp      |   4 +-
 include/ginkgo/core/base/lin_op.hpp        |  43 ++++++---
 include/ginkgo/core/base/math.hpp          |  54 +++++------
 include/ginkgo/core/matrix/coo.hpp         |   2 +-
 include/ginkgo/core/matrix/csr.hpp         |   2 +-
 include/ginkgo/core/matrix/dense.hpp       |   2 +-
 include/ginkgo/core/matrix/diagonal.hpp    |   2 +-
 include/ginkgo/core/matrix/ell.hpp         |   2 +-
 include/ginkgo/core/matrix/hybrid.hpp      |   2 +-
 include/ginkgo/core/matrix/sellp.hpp       |   2 +-
 omp/test/matrix/coo_kernels.cpp            |   4 +-
 omp/test/matrix/csr_kernels.cpp            |   8 +-
 omp/test/matrix/dense_kernels.cpp          |   4 +-
 omp/test/matrix/diagonal_kernels.cpp       |   4 +-
 omp/test/matrix/ell_kernels.cpp            |   4 +-
 omp/test/matrix/hybrid_kernels.cpp         |   4 +-
 omp/test/matrix/sellp_kernels.cpp          |   4 +-
 reference/test/matrix/coo_kernels.cpp      |   4 +-
 reference/test/matrix/csr_kernels.cpp      |   4 +-
 reference/test/matrix/dense_kernels.cpp    |   6 +-
 reference/test/matrix/diagonal_kernels.cpp |   4 +-
 reference/test/matrix/ell_kernels.cpp      |   4 +-
 reference/test/matrix/hybrid_kernels.cpp   |   4 +-
 reference/test/matrix/sellp_kernels.cpp    |   4 +-
 45 files changed, 236 insertions(+), 117 deletions(-)

diff --git a/core/matrix/coo.cpp b/core/matrix/coo.cpp
index ecff61e6d4e..b9648d1dc70 100644
--- a/core/matrix/coo.cpp
+++ b/core/matrix/coo.cpp
@@ -239,7 +239,7 @@ Coo<ValueType, IndexType>::extract_diagonal() const
 
 
 template <typename ValueType, typename IndexType>
-void Coo<ValueType, IndexType>::turn_absolute()
+void Coo<ValueType, IndexType>::apply_absolute()
 {
     auto exec = this->get_executor();
 
diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp
index 0a51503c5f0..8a6ad620885 100644
--- a/core/matrix/csr.cpp
+++ b/core/matrix/csr.cpp
@@ -487,7 +487,7 @@ Csr<ValueType, IndexType>::extract_diagonal() const
 
 
 template <typename ValueType, typename IndexType>
-void Csr<ValueType, IndexType>::turn_absolute()
+void Csr<ValueType, IndexType>::apply_absolute()
 {
     auto exec = this->get_executor();
 
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 306c83cde71..19152c62d9a 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -755,7 +755,7 @@ std::unique_ptr<Diagonal<ValueType>> Dense<ValueType>::extract_diagonal() const
 
 
 template <typename ValueType>
-void Dense<ValueType>::turn_absolute()
+void Dense<ValueType>::apply_absolute()
 {
     auto exec = this->get_executor();
 
diff --git a/core/matrix/diagonal.cpp b/core/matrix/diagonal.cpp
index 83b8d826019..e136e90a919 100644
--- a/core/matrix/diagonal.cpp
+++ b/core/matrix/diagonal.cpp
@@ -267,7 +267,7 @@ void Diagonal<ValueType>::write(mat_data32 &data) const
 
 
 template <typename ValueType>
-void Diagonal<ValueType>::turn_absolute()
+void Diagonal<ValueType>::apply_absolute()
 {
     auto exec = this->get_executor();
 
diff --git a/core/matrix/ell.cpp b/core/matrix/ell.cpp
index a16ec65631b..28acf7a3a39 100644
--- a/core/matrix/ell.cpp
+++ b/core/matrix/ell.cpp
@@ -257,7 +257,7 @@ Ell<ValueType, IndexType>::extract_diagonal() const
 
 
 template <typename ValueType, typename IndexType>
-void Ell<ValueType, IndexType>::turn_absolute()
+void Ell<ValueType, IndexType>::apply_absolute()
 {
     auto exec = this->get_executor();
 
diff --git a/core/matrix/hybrid.cpp b/core/matrix/hybrid.cpp
index b6d9552c9e5..cd5adcfbdcd 100644
--- a/core/matrix/hybrid.cpp
+++ b/core/matrix/hybrid.cpp
@@ -291,7 +291,7 @@ Hybrid<ValueType, IndexType>::extract_diagonal() const
 
 
 template <typename ValueType, typename IndexType>
-void Hybrid<ValueType, IndexType>::turn_absolute()
+void Hybrid<ValueType, IndexType>::apply_absolute()
 {
     auto exec = this->get_executor();
 
diff --git a/core/matrix/sellp.cpp b/core/matrix/sellp.cpp
index 916d531d072..e57555c067b 100644
--- a/core/matrix/sellp.cpp
+++ b/core/matrix/sellp.cpp
@@ -310,7 +310,7 @@ Sellp<ValueType, IndexType>::extract_diagonal() const
 
 
 template <typename ValueType, typename IndexType>
-void Sellp<ValueType, IndexType>::turn_absolute()
+void Sellp<ValueType, IndexType>::apply_absolute()
 {
     auto exec = this->get_executor();
 
diff --git a/core/test/base/lin_op.cpp b/core/test/base/lin_op.cpp
index 622158d9b42..e5eea7d7c70 100644
--- a/core/test/base/lin_op.cpp
+++ b/core/test/base/lin_op.cpp
@@ -30,8 +30,9 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include <complex>
 #include <ginkgo/core/base/lin_op.hpp>
-
+#include <ginkgo/core/base/math.hpp>
 
 #include <gtest/gtest.h>
 
@@ -314,4 +315,103 @@ TEST_F(EnableLinOpFactory, PassesParametersToLinOp)
 }
 
 
+template <typename Type>
+class DummyLinOpWithType
+    : public gko::EnableLinOp<DummyLinOpWithType<Type>>,
+      public gko::EnableCreateMethod<DummyLinOpWithType<Type>>,
+      public gko::EnableAbsoluteComputation<DummyLinOpWithType<Type>> {
+public:
+    using outplace_absolute_type = gko::remove_complex<DummyLinOpWithType>;
+    DummyLinOpWithType(std::shared_ptr<const gko::Executor> exec)
+        : gko::EnableLinOp<DummyLinOpWithType>(exec)
+    {}
+
+    DummyLinOpWithType(std::shared_ptr<const gko::Executor> exec,
+                       gko::dim<2> size, Type value)
+        : gko::EnableLinOp<DummyLinOpWithType>(exec, size), value_(value)
+    {}
+
+    void apply_absolute() override { value_ = gko::abs(value_); }
+
+    std::unique_ptr<outplace_absolute_type> get_absolute() const override
+    {
+        return std::make_unique<outplace_absolute_type>(
+            this->get_executor(), this->get_size(), gko::abs(value_));
+    }
+
+    Type get_value() const { return value_; }
+
+protected:
+    void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override {}
+
+    void apply_impl(const gko::LinOp *alpha, const gko::LinOp *b,
+                    const gko::LinOp *beta, gko::LinOp *x) const override
+    {}
+
+private:
+    Type value_;
+};
+
+
+class EnableAbsoluteComputation : public ::testing::Test {
+protected:
+    using dummy_type = DummyLinOpWithType<std::complex<double>>;
+    EnableAbsoluteComputation()
+        : ref{gko::ReferenceExecutor::create()},
+          op{dummy_type::create(ref, gko::dim<2>{1, 1},
+                                std::complex<double>{-3.0, 4.0})}
+    {}
+
+    std::shared_ptr<const gko::ReferenceExecutor> ref;
+    std::shared_ptr<dummy_type> op;
+};
+
+
+TEST_F(EnableAbsoluteComputation, InplaceAbsoluteOnConcreteType)
+{
+    op->apply_absolute();
+
+    ASSERT_EQ(op->get_value(), std::complex<double>{5.0});
+}
+
+
+TEST_F(EnableAbsoluteComputation, OutplaceAbsoluteOnConcreteType)
+{
+    auto abs_op = op->get_absolute();
+
+    ASSERT_EQ(typeid(abs_op),
+              typeid(std::unique_ptr<gko::remove_complex<dummy_type>>));
+    ASSERT_EQ(abs_op->get_value(), 5.0);
+}
+
+
+TEST_F(EnableAbsoluteComputation, InplaceAbsoluteOnAbsoluteComputable)
+{
+    auto linop = gko::as<gko::LinOp>(op);
+
+    gko::as<gko::AbsoluteComputable>(linop)->apply_absolute();
+
+    ASSERT_EQ(gko::as<dummy_type>(linop)->get_value(),
+              std::complex<double>{5.0});
+}
+
+
+TEST_F(EnableAbsoluteComputation, OutplaceAbsoluteOnAbsoluteComputable)
+{
+    auto abs_op = op->get_absolute();
+
+    ASSERT_EQ(typeid(abs_op),
+              typeid(std::unique_ptr<gko::remove_complex<dummy_type>>));
+    ASSERT_EQ(abs_op->get_value(), 5.0);
+}
+
+
+TEST_F(EnableAbsoluteComputation, ThrowWithoutAbsoluteComputableInterface)
+{
+    std::shared_ptr<gko::LinOp> linop = DummyLinOp::create(ref);
+
+    ASSERT_THROW(gko::as<gko::AbsoluteComputable>(linop), gko::NotSupported);
+}
+
+
 }  // namespace
diff --git a/cuda/test/matrix/coo_kernels.cpp b/cuda/test/matrix/coo_kernels.cpp
index 330e7d7301a..7575314ad7e 100644
--- a/cuda/test/matrix/coo_kernels.cpp
+++ b/cuda/test/matrix/coo_kernels.cpp
@@ -275,8 +275,8 @@ TEST_F(Coo, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index b05ca46327c..d51ea16bc33 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -744,8 +744,8 @@ TEST_F(Csr, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>(cuda));
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -766,8 +766,8 @@ TEST_F(Csr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
 {
     set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(cuda));
 
-    complex_mtx->turn_absolute();
-    complex_dmtx->turn_absolute();
+    complex_mtx->apply_absolute();
+    complex_dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(complex_mtx.get(), complex_dmtx.get(), 1e-14);
 }
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index a2cbccfc8f3..1121b994fd6 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -591,8 +591,8 @@ TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    x->turn_absolute();
-    dx->turn_absolute();
+    x->apply_absolute();
+    dx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(x.get(), dx.get(), 1e-14);
 }
diff --git a/cuda/test/matrix/diagonal_kernels.cpp b/cuda/test/matrix/diagonal_kernels.cpp
index 8da64b3b8dc..6a357769022 100644
--- a/cuda/test/matrix/diagonal_kernels.cpp
+++ b/cuda/test/matrix/diagonal_kernels.cpp
@@ -253,8 +253,8 @@ TEST_F(Diagonal, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    diag->turn_absolute();
-    ddiag->turn_absolute();
+    diag->apply_absolute();
+    ddiag->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 1e-14);
 }
diff --git a/cuda/test/matrix/ell_kernels.cpp b/cuda/test/matrix/ell_kernels.cpp
index 6a4d3b643f3..3df92e8c859 100644
--- a/cuda/test/matrix/ell_kernels.cpp
+++ b/cuda/test/matrix/ell_kernels.cpp
@@ -360,8 +360,8 @@ TEST_F(Ell, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
diff --git a/cuda/test/matrix/hybrid_kernels.cpp b/cuda/test/matrix/hybrid_kernels.cpp
index 4ce0f86943c..f9adc82ff53 100644
--- a/cuda/test/matrix/hybrid_kernels.cpp
+++ b/cuda/test/matrix/hybrid_kernels.cpp
@@ -235,8 +235,8 @@ TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
diff --git a/cuda/test/matrix/sellp_kernels.cpp b/cuda/test/matrix/sellp_kernels.cpp
index 8b993e2f964..43375ffb0cf 100644
--- a/cuda/test/matrix/sellp_kernels.cpp
+++ b/cuda/test/matrix/sellp_kernels.cpp
@@ -354,8 +354,8 @@ TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_matrix(32, 2);
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
diff --git a/hip/test/matrix/coo_kernels.hip.cpp b/hip/test/matrix/coo_kernels.hip.cpp
index a3bc99afd6a..470694c1ece 100644
--- a/hip/test/matrix/coo_kernels.hip.cpp
+++ b/hip/test/matrix/coo_kernels.hip.cpp
@@ -275,8 +275,8 @@ TEST_F(Coo, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index 4a55b6a6ef8..cbcb3c1b37e 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -729,8 +729,8 @@ TEST_F(Csr, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -751,8 +751,8 @@ TEST_F(Csr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
 {
     set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(hip));
 
-    complex_mtx->turn_absolute();
-    complex_dmtx->turn_absolute();
+    complex_mtx->apply_absolute();
+    complex_dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(complex_mtx.get(), complex_dmtx.get(), 1e-14);
 }
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index 0abb142e1e9..86e907d27ab 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -573,8 +573,8 @@ TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    x->turn_absolute();
-    dx->turn_absolute();
+    x->apply_absolute();
+    dx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(x.get(), dx.get(), 1e-14);
 }
diff --git a/hip/test/matrix/diagonal_kernels.hip.cpp b/hip/test/matrix/diagonal_kernels.hip.cpp
index 1d3c118a271..dd89c7945e5 100644
--- a/hip/test/matrix/diagonal_kernels.hip.cpp
+++ b/hip/test/matrix/diagonal_kernels.hip.cpp
@@ -253,8 +253,8 @@ TEST_F(Diagonal, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    diag->turn_absolute();
-    ddiag->turn_absolute();
+    diag->apply_absolute();
+    ddiag->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 1e-14);
 }
diff --git a/hip/test/matrix/ell_kernels.hip.cpp b/hip/test/matrix/ell_kernels.hip.cpp
index 0646691b88d..502035ba9c5 100644
--- a/hip/test/matrix/ell_kernels.hip.cpp
+++ b/hip/test/matrix/ell_kernels.hip.cpp
@@ -360,8 +360,8 @@ TEST_F(Ell, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
diff --git a/hip/test/matrix/hybrid_kernels.hip.cpp b/hip/test/matrix/hybrid_kernels.hip.cpp
index 04fcae927bf..be1cd12ae6c 100644
--- a/hip/test/matrix/hybrid_kernels.hip.cpp
+++ b/hip/test/matrix/hybrid_kernels.hip.cpp
@@ -235,8 +235,8 @@ TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
diff --git a/hip/test/matrix/sellp_kernels.hip.cpp b/hip/test/matrix/sellp_kernels.hip.cpp
index 9a1d12fe3c7..59d5efda437 100644
--- a/hip/test/matrix/sellp_kernels.hip.cpp
+++ b/hip/test/matrix/sellp_kernels.hip.cpp
@@ -353,8 +353,8 @@ TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_matrix(32, 2);
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index 2a38b84d31a..8c6819d0db5 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -625,10 +625,31 @@ class DiagonalExtractable {
 
 
 /**
- * The EnableAbsoluteComputation mixin implementing this interface can be
- * computed. get_absolute gets a new LinOp which contains absolute values from a
- * LinOp and the valuetype will be removed complex. turn_absolute applies
- * absolute inplace, so it still keeps the valuetype of the class.
+ * The AbsoluteComputable is an interface to allow get absolute from a LinOp.
+ * Use EnableAbsoluteComputation<ConcreteLinOp> to implement this interface.
+ */
+class AbsoluteComputable {
+public:
+    /**
+     * Gets the absolute LinOp
+     *
+     * @return a pointer to the new absolute LinOp
+     */
+    virtual std::unique_ptr<LinOp> get_absolute_linop() const = 0;
+
+    /**
+     * Apply absolute on each element.
+     */
+    virtual void apply_absolute() = 0;
+};
+
+
+/**
+ * The EnableAbsoluteComputation mixin provides the default implementations of
+ * `get_absolute_linop` and the absolute interface. `get_absolute` gets a new
+ * remove_complex<ConcreteLinOp> which removes the complex value_type by
+ * applying outplace absolute. `apply_absolute` applies absolute inplace, so it
+ * still keeps the value_type of the class.
  *
  * @tparam ConcreteLinOp  the concrete LinOp which is being implemented
  *                        [CRTP parameter]
@@ -636,25 +657,23 @@ class DiagonalExtractable {
  * @ingroup LinOp
  */
 template <typename ConcreteLinOp>
-class EnableAbsoluteComputation {
+class EnableAbsoluteComputation : public AbsoluteComputable {
 public:
     using outplace_type = remove_complex<ConcreteLinOp>;
 
     virtual ~EnableAbsoluteComputation() = default;
 
+    std::unique_ptr<LinOp> get_absolute_linop() const override
+    {
+        return this->get_absolute();
+    }
+
     /**
      * Gets the absolute LinOp
      *
      * @return a pointer to the new absolute object
      */
     virtual std::unique_ptr<outplace_type> get_absolute() const = 0;
-
-    /**
-     * Turns the value
-     *
-     * @return a pointer to the new absolute object
-     */
-    virtual void turn_absolute() = 0;
 };
 
 
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index ce62ed82e7f..85abf81a08e 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -752,33 +752,6 @@ GKO_INLINE GKO_ATTRIBUTES constexpr T one(const T &)
 #undef GKO_BIND_ZERO_ONE
 
 
-/**
- * Returns the absolute value of the object.
- *
- * @tparam T  the type of the object
- *
- * @param x  the object
- *
- * @return x >= zero<T>() ? x : -x;
- */
-template <typename T>
-GKO_INLINE
-    GKO_ATTRIBUTES constexpr xstd::enable_if_t<!is_complex_s<T>::value, T>
-    abs(const T &x)
-{
-    return x >= zero<T>() ? x : -x;
-}
-
-
-template <typename T>
-GKO_INLINE GKO_ATTRIBUTES constexpr xstd::enable_if_t<is_complex_s<T>::value,
-                                                      remove_complex<T>>
-abs(const T &x)
-{
-    return sqrt(squared_norm(x));
-}
-
-
 /**
  * Returns the larger of the arguments.
  *
@@ -902,6 +875,33 @@ GKO_INLINE GKO_ATTRIBUTES constexpr auto squared_norm(const T &x)
 }
 
 
+/**
+ * Returns the absolute value of the object.
+ *
+ * @tparam T  the type of the object
+ *
+ * @param x  the object
+ *
+ * @return x >= zero<T>() ? x : -x;
+ */
+template <typename T>
+GKO_INLINE
+    GKO_ATTRIBUTES constexpr xstd::enable_if_t<!is_complex_s<T>::value, T>
+    abs(const T &x)
+{
+    return x >= zero<T>() ? x : -x;
+}
+
+
+template <typename T>
+GKO_INLINE GKO_ATTRIBUTES constexpr xstd::enable_if_t<is_complex_s<T>::value,
+                                                      remove_complex<T>>
+abs(const T &x)
+{
+    return sqrt(squared_norm(x));
+}
+
+
 /**
  * Returns the position of the most significant bit of the number.
  *
diff --git a/include/ginkgo/core/matrix/coo.hpp b/include/ginkgo/core/matrix/coo.hpp
index 7508d312fdc..bde9046aef4 100644
--- a/include/ginkgo/core/matrix/coo.hpp
+++ b/include/ginkgo/core/matrix/coo.hpp
@@ -123,7 +123,7 @@ class Coo : public EnableLinOp<Coo<ValueType, IndexType>>,
 
     std::unique_ptr<outplace_absolute_type> get_absolute() const override;
 
-    void turn_absolute() override;
+    void apply_absolute() override;
 
     /**
      * Returns the values of the matrix.
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index ad9d77ee907..60b32cc2196 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -714,7 +714,7 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
 
     std::unique_ptr<outplace_absolute_type> get_absolute() const override;
 
-    void turn_absolute() override;
+    void apply_absolute() override;
 
     /**
      * Sorts all (value, col_idx) pairs in each row by column index
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 912040345cd..ddc7b4d9519 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -251,7 +251,7 @@ class Dense : public EnableLinOp<Dense<ValueType>>,
 
     std::unique_ptr<outplace_absolute_type> get_absolute() const override;
 
-    void turn_absolute() override;
+    void apply_absolute() override;
 
     /**
      * Returns a pointer to the array of values of the matrix.
diff --git a/include/ginkgo/core/matrix/diagonal.hpp b/include/ginkgo/core/matrix/diagonal.hpp
index 38526439d95..ceb97e363ca 100644
--- a/include/ginkgo/core/matrix/diagonal.hpp
+++ b/include/ginkgo/core/matrix/diagonal.hpp
@@ -105,7 +105,7 @@ class Diagonal : public EnableLinOp<Diagonal<ValueType>>,
 
     std::unique_ptr<outplace_absolute_type> get_absolute() const override;
 
-    void turn_absolute() override;
+    void apply_absolute() override;
 
     /**
      * Returns a pointer to the array of values of the matrix.
diff --git a/include/ginkgo/core/matrix/ell.hpp b/include/ginkgo/core/matrix/ell.hpp
index f9c68fb0171..02b3a3613c8 100644
--- a/include/ginkgo/core/matrix/ell.hpp
+++ b/include/ginkgo/core/matrix/ell.hpp
@@ -116,7 +116,7 @@ class Ell : public EnableLinOp<Ell<ValueType, IndexType>>,
 
     std::unique_ptr<outplace_absolute_type> get_absolute() const override;
 
-    void turn_absolute() override;
+    void apply_absolute() override;
 
     /**
      * Returns the values of the matrix.
diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp
index 7f544da8b10..6dcee07bbe2 100644
--- a/include/ginkgo/core/matrix/hybrid.hpp
+++ b/include/ginkgo/core/matrix/hybrid.hpp
@@ -365,7 +365,7 @@ class Hybrid
 
     std::unique_ptr<outplace_absolute_type> get_absolute() const override;
 
-    void turn_absolute() override;
+    void apply_absolute() override;
 
     /**
      * Returns the values of the ell part.
diff --git a/include/ginkgo/core/matrix/sellp.hpp b/include/ginkgo/core/matrix/sellp.hpp
index 3c6cb087be5..6f4b879a0d9 100644
--- a/include/ginkgo/core/matrix/sellp.hpp
+++ b/include/ginkgo/core/matrix/sellp.hpp
@@ -113,7 +113,7 @@ class Sellp : public EnableLinOp<Sellp<ValueType, IndexType>>,
 
     std::unique_ptr<outplace_absolute_type> get_absolute() const override;
 
-    void turn_absolute() override;
+    void apply_absolute() override;
 
     /**
      * Returns the values of the matrix.
diff --git a/omp/test/matrix/coo_kernels.cpp b/omp/test/matrix/coo_kernels.cpp
index cc1da48c5ea..03090fd2f54 100644
--- a/omp/test/matrix/coo_kernels.cpp
+++ b/omp/test/matrix/coo_kernels.cpp
@@ -238,8 +238,8 @@ TEST_F(Coo, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index 9674b2da1e3..8b6a54cdf37 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -553,8 +553,8 @@ TEST_F(Csr, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -575,8 +575,8 @@ TEST_F(Csr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    complex_mtx->turn_absolute();
-    complex_dmtx->turn_absolute();
+    complex_mtx->apply_absolute();
+    complex_dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(complex_mtx.get(), complex_dmtx.get(), 1e-14);
 }
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index 42b91f5bdcd..65afcbf8502 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -706,8 +706,8 @@ TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    x->turn_absolute();
-    dx->turn_absolute();
+    x->apply_absolute();
+    dx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(x.get(), dx.get(), 1e-14);
 }
diff --git a/omp/test/matrix/diagonal_kernels.cpp b/omp/test/matrix/diagonal_kernels.cpp
index d9075a450d2..571ca6fe7e6 100644
--- a/omp/test/matrix/diagonal_kernels.cpp
+++ b/omp/test/matrix/diagonal_kernels.cpp
@@ -252,8 +252,8 @@ TEST_F(Diagonal, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    diag->turn_absolute();
-    ddiag->turn_absolute();
+    diag->apply_absolute();
+    ddiag->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 1e-14);
 }
diff --git a/omp/test/matrix/ell_kernels.cpp b/omp/test/matrix/ell_kernels.cpp
index f37c313591b..584e88488c2 100644
--- a/omp/test/matrix/ell_kernels.cpp
+++ b/omp/test/matrix/ell_kernels.cpp
@@ -236,8 +236,8 @@ TEST_F(Ell, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
diff --git a/omp/test/matrix/hybrid_kernels.cpp b/omp/test/matrix/hybrid_kernels.cpp
index c0b3a329d93..046d4c254a5 100644
--- a/omp/test/matrix/hybrid_kernels.cpp
+++ b/omp/test/matrix/hybrid_kernels.cpp
@@ -234,8 +234,8 @@ TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
diff --git a/omp/test/matrix/sellp_kernels.cpp b/omp/test/matrix/sellp_kernels.cpp
index 1c758de6113..9d861d2db69 100644
--- a/omp/test/matrix/sellp_kernels.cpp
+++ b/omp/test/matrix/sellp_kernels.cpp
@@ -270,8 +270,8 @@ TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->turn_absolute();
-    dmtx->turn_absolute();
+    mtx->apply_absolute();
+    dmtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
diff --git a/reference/test/matrix/coo_kernels.cpp b/reference/test/matrix/coo_kernels.cpp
index e0666b7ccda..0f75f7a1eea 100644
--- a/reference/test/matrix/coo_kernels.cpp
+++ b/reference/test/matrix/coo_kernels.cpp
@@ -540,7 +540,7 @@ TYPED_TEST(Coo, InplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    mtx->turn_absolute();
+    mtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -606,7 +606,7 @@ TYPED_TEST(CooComplex, InplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    mtx->turn_absolute();
+    mtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp
index 3ab5fbe3269..e2fc18432aa 100644
--- a/reference/test/matrix/csr_kernels.cpp
+++ b/reference/test/matrix/csr_kernels.cpp
@@ -1313,7 +1313,7 @@ TYPED_TEST(Csr, InplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    mtx->turn_absolute();
+    mtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -1386,7 +1386,7 @@ TYPED_TEST(CsrComplex, InplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    mtx->turn_absolute();
+    mtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index c41a748d6d5..94cb1c42542 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -2017,7 +2017,7 @@ TYPED_TEST(Dense, InplaceAbsolute)
     // {-2.0, 2.0, 4.5},
     // {2.1, 3.4, 1.2}
     // clang-format on
-    this->mtx5->turn_absolute();
+    this->mtx5->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(this->mtx5,
                         l({{1.0, 1.0, 0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
@@ -2035,7 +2035,7 @@ TYPED_TEST(Dense, InplaceAbsoluteSubMatrix)
     // {-2.0, 2.0, 4.5},
     // {2.1, 3.4, 1.2}
     // clang-format on
-    mtx->turn_absolute();
+    mtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(this->mtx5,
                         l({{1.0, 1.0, -0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
@@ -2119,7 +2119,7 @@ TYPED_TEST(DenseComplex, InplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    mtx->turn_absolute();
+    mtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
diff --git a/reference/test/matrix/diagonal_kernels.cpp b/reference/test/matrix/diagonal_kernels.cpp
index d2ab3f3453c..69502cca362 100644
--- a/reference/test/matrix/diagonal_kernels.cpp
+++ b/reference/test/matrix/diagonal_kernels.cpp
@@ -360,7 +360,7 @@ TYPED_TEST(Diagonal, InplaceAbsolute)
 {
     using value_type = typename TestFixture::value_type;
 
-    this->diag1->turn_absolute();
+    this->diag1->apply_absolute();
     auto values = this->diag1->get_values();
 
     EXPECT_EQ(values[0], value_type(2.0));
@@ -425,7 +425,7 @@ TYPED_TEST(DiagonalComplex, InplaceAbsolute)
     diag_values[1] = value_type{-3.0, 0.0};
     diag_values[2] = value_type{0.0, -1.5};
 
-    diag->turn_absolute();
+    diag->apply_absolute();
 
     EXPECT_EQ(diag_values[0], (value_type{5.0, 0.0}));
     EXPECT_EQ(diag_values[1], (value_type{3.0, 0.0}));
diff --git a/reference/test/matrix/ell_kernels.cpp b/reference/test/matrix/ell_kernels.cpp
index 47221ecfeb2..13b7decd1f1 100644
--- a/reference/test/matrix/ell_kernels.cpp
+++ b/reference/test/matrix/ell_kernels.cpp
@@ -618,7 +618,7 @@ TYPED_TEST(Ell, InplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    mtx->turn_absolute();
+    mtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -664,7 +664,7 @@ TYPED_TEST(EllComplex, InplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    mtx->turn_absolute();
+    mtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
diff --git a/reference/test/matrix/hybrid_kernels.cpp b/reference/test/matrix/hybrid_kernels.cpp
index e5bd30fa00e..17f0aae4c50 100644
--- a/reference/test/matrix/hybrid_kernels.cpp
+++ b/reference/test/matrix/hybrid_kernels.cpp
@@ -662,7 +662,7 @@ TYPED_TEST(Hybrid, InplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    mtx->turn_absolute();
+    mtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -728,7 +728,7 @@ TYPED_TEST(HybridComplex, InplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    mtx->turn_absolute();
+    mtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
diff --git a/reference/test/matrix/sellp_kernels.cpp b/reference/test/matrix/sellp_kernels.cpp
index 6ab86e8ce1a..7eef159ddeb 100644
--- a/reference/test/matrix/sellp_kernels.cpp
+++ b/reference/test/matrix/sellp_kernels.cpp
@@ -621,7 +621,7 @@ TYPED_TEST(Sellp, InplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    mtx->turn_absolute();
+    mtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -687,7 +687,7 @@ TYPED_TEST(SellpComplex, InplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    mtx->turn_absolute();
+    mtx->apply_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);

From 83676e2f42732ef858ef31c3b5345770423d2f04 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 9 Sep 2020 15:49:01 +0200
Subject: [PATCH 0016/1008] get/apply_absolute -> compute_absolute(_inplace)

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/matrix/coo.cpp                        |  4 ++--
 core/matrix/csr.cpp                        |  4 ++--
 core/matrix/dense.cpp                      |  4 ++--
 core/matrix/diagonal.cpp                   |  4 ++--
 core/matrix/ell.cpp                        |  4 ++--
 core/matrix/hybrid.cpp                     |  8 ++++----
 core/matrix/sellp.cpp                      |  4 ++--
 core/test/base/lin_op.cpp                  | 12 ++++++------
 cuda/test/matrix/coo_kernels.cpp           |  8 ++++----
 cuda/test/matrix/csr_kernels.cpp           | 16 ++++++++--------
 cuda/test/matrix/dense_kernels.cpp         |  8 ++++----
 cuda/test/matrix/diagonal_kernels.cpp      |  8 ++++----
 cuda/test/matrix/ell_kernels.cpp           |  8 ++++----
 cuda/test/matrix/hybrid_kernels.cpp        |  8 ++++----
 cuda/test/matrix/sellp_kernels.cpp         |  8 ++++----
 hip/test/matrix/coo_kernels.hip.cpp        |  8 ++++----
 hip/test/matrix/csr_kernels.hip.cpp        | 16 ++++++++--------
 hip/test/matrix/dense_kernels.hip.cpp      |  8 ++++----
 hip/test/matrix/diagonal_kernels.hip.cpp   |  8 ++++----
 hip/test/matrix/ell_kernels.hip.cpp        |  8 ++++----
 hip/test/matrix/hybrid_kernels.hip.cpp     |  8 ++++----
 hip/test/matrix/sellp_kernels.hip.cpp      |  8 ++++----
 include/ginkgo/core/base/lin_op.hpp        | 22 +++++++++++-----------
 include/ginkgo/core/matrix/coo.hpp         |  4 ++--
 include/ginkgo/core/matrix/csr.hpp         |  4 ++--
 include/ginkgo/core/matrix/dense.hpp       |  4 ++--
 include/ginkgo/core/matrix/diagonal.hpp    |  4 ++--
 include/ginkgo/core/matrix/ell.hpp         |  4 ++--
 include/ginkgo/core/matrix/hybrid.hpp      |  4 ++--
 include/ginkgo/core/matrix/sellp.hpp       |  4 ++--
 omp/test/matrix/coo_kernels.cpp            |  8 ++++----
 omp/test/matrix/csr_kernels.cpp            | 16 ++++++++--------
 omp/test/matrix/dense_kernels.cpp          |  8 ++++----
 omp/test/matrix/diagonal_kernels.cpp       |  8 ++++----
 omp/test/matrix/ell_kernels.cpp            |  8 ++++----
 omp/test/matrix/hybrid_kernels.cpp         |  8 ++++----
 omp/test/matrix/sellp_kernels.cpp          |  8 ++++----
 reference/test/matrix/coo_kernels.cpp      |  8 ++++----
 reference/test/matrix/csr_kernels.cpp      |  8 ++++----
 reference/test/matrix/dense_kernels.cpp    | 12 ++++++------
 reference/test/matrix/diagonal_kernels.cpp |  8 ++++----
 reference/test/matrix/ell_kernels.cpp      |  8 ++++----
 reference/test/matrix/hybrid_kernels.cpp   |  8 ++++----
 reference/test/matrix/sellp_kernels.cpp    |  8 ++++----
 44 files changed, 173 insertions(+), 173 deletions(-)

diff --git a/core/matrix/coo.cpp b/core/matrix/coo.cpp
index b9648d1dc70..50b42d04ab4 100644
--- a/core/matrix/coo.cpp
+++ b/core/matrix/coo.cpp
@@ -239,7 +239,7 @@ Coo<ValueType, IndexType>::extract_diagonal() const
 
 
 template <typename ValueType, typename IndexType>
-void Coo<ValueType, IndexType>::apply_absolute()
+void Coo<ValueType, IndexType>::compute_absolute_inplace()
 {
     auto exec = this->get_executor();
 
@@ -250,7 +250,7 @@ void Coo<ValueType, IndexType>::apply_absolute()
 
 template <typename ValueType, typename IndexType>
 std::unique_ptr<typename Coo<ValueType, IndexType>::outplace_absolute_type>
-Coo<ValueType, IndexType>::get_absolute() const
+Coo<ValueType, IndexType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp
index 8a6ad620885..605f7b09fed 100644
--- a/core/matrix/csr.cpp
+++ b/core/matrix/csr.cpp
@@ -487,7 +487,7 @@ Csr<ValueType, IndexType>::extract_diagonal() const
 
 
 template <typename ValueType, typename IndexType>
-void Csr<ValueType, IndexType>::apply_absolute()
+void Csr<ValueType, IndexType>::compute_absolute_inplace()
 {
     auto exec = this->get_executor();
 
@@ -498,7 +498,7 @@ void Csr<ValueType, IndexType>::apply_absolute()
 
 template <typename ValueType, typename IndexType>
 std::unique_ptr<typename Csr<ValueType, IndexType>::outplace_absolute_type>
-Csr<ValueType, IndexType>::get_absolute() const
+Csr<ValueType, IndexType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 19152c62d9a..e51ad7bcff1 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -755,7 +755,7 @@ std::unique_ptr<Diagonal<ValueType>> Dense<ValueType>::extract_diagonal() const
 
 
 template <typename ValueType>
-void Dense<ValueType>::apply_absolute()
+void Dense<ValueType>::compute_absolute_inplace()
 {
     auto exec = this->get_executor();
 
@@ -765,7 +765,7 @@ void Dense<ValueType>::apply_absolute()
 
 template <typename ValueType>
 std::unique_ptr<typename Dense<ValueType>::outplace_absolute_type>
-Dense<ValueType>::get_absolute() const
+Dense<ValueType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
diff --git a/core/matrix/diagonal.cpp b/core/matrix/diagonal.cpp
index e136e90a919..e1cb767efbf 100644
--- a/core/matrix/diagonal.cpp
+++ b/core/matrix/diagonal.cpp
@@ -267,7 +267,7 @@ void Diagonal<ValueType>::write(mat_data32 &data) const
 
 
 template <typename ValueType>
-void Diagonal<ValueType>::apply_absolute()
+void Diagonal<ValueType>::compute_absolute_inplace()
 {
     auto exec = this->get_executor();
 
@@ -278,7 +278,7 @@ void Diagonal<ValueType>::apply_absolute()
 
 template <typename ValueType>
 std::unique_ptr<typename Diagonal<ValueType>::outplace_absolute_type>
-Diagonal<ValueType>::get_absolute() const
+Diagonal<ValueType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
diff --git a/core/matrix/ell.cpp b/core/matrix/ell.cpp
index 28acf7a3a39..26f518d2cbc 100644
--- a/core/matrix/ell.cpp
+++ b/core/matrix/ell.cpp
@@ -257,7 +257,7 @@ Ell<ValueType, IndexType>::extract_diagonal() const
 
 
 template <typename ValueType, typename IndexType>
-void Ell<ValueType, IndexType>::apply_absolute()
+void Ell<ValueType, IndexType>::compute_absolute_inplace()
 {
     auto exec = this->get_executor();
 
@@ -268,7 +268,7 @@ void Ell<ValueType, IndexType>::apply_absolute()
 
 template <typename ValueType, typename IndexType>
 std::unique_ptr<typename Ell<ValueType, IndexType>::outplace_absolute_type>
-Ell<ValueType, IndexType>::get_absolute() const
+Ell<ValueType, IndexType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
diff --git a/core/matrix/hybrid.cpp b/core/matrix/hybrid.cpp
index cd5adcfbdcd..9487bd99a4e 100644
--- a/core/matrix/hybrid.cpp
+++ b/core/matrix/hybrid.cpp
@@ -291,7 +291,7 @@ Hybrid<ValueType, IndexType>::extract_diagonal() const
 
 
 template <typename ValueType, typename IndexType>
-void Hybrid<ValueType, IndexType>::apply_absolute()
+void Hybrid<ValueType, IndexType>::compute_absolute_inplace()
 {
     auto exec = this->get_executor();
 
@@ -304,15 +304,15 @@ void Hybrid<ValueType, IndexType>::apply_absolute()
 
 template <typename ValueType, typename IndexType>
 std::unique_ptr<typename Hybrid<ValueType, IndexType>::outplace_absolute_type>
-Hybrid<ValueType, IndexType>::get_absolute() const
+Hybrid<ValueType, IndexType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
     // use default strategy
     auto abs_hybrid = outplace_absolute_type::create(exec, this->get_size());
 
-    abs_hybrid->ell_->copy_from(ell_->get_absolute());
-    abs_hybrid->coo_->copy_from(coo_->get_absolute());
+    abs_hybrid->ell_->copy_from(ell_->compute_absolute());
+    abs_hybrid->coo_->copy_from(coo_->compute_absolute());
 
     return abs_hybrid;
 }
diff --git a/core/matrix/sellp.cpp b/core/matrix/sellp.cpp
index e57555c067b..81af57b5dd6 100644
--- a/core/matrix/sellp.cpp
+++ b/core/matrix/sellp.cpp
@@ -310,7 +310,7 @@ Sellp<ValueType, IndexType>::extract_diagonal() const
 
 
 template <typename ValueType, typename IndexType>
-void Sellp<ValueType, IndexType>::apply_absolute()
+void Sellp<ValueType, IndexType>::compute_absolute_inplace()
 {
     auto exec = this->get_executor();
 
@@ -321,7 +321,7 @@ void Sellp<ValueType, IndexType>::apply_absolute()
 
 template <typename ValueType, typename IndexType>
 std::unique_ptr<typename Sellp<ValueType, IndexType>::outplace_absolute_type>
-Sellp<ValueType, IndexType>::get_absolute() const
+Sellp<ValueType, IndexType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
diff --git a/core/test/base/lin_op.cpp b/core/test/base/lin_op.cpp
index e5eea7d7c70..497396efaeb 100644
--- a/core/test/base/lin_op.cpp
+++ b/core/test/base/lin_op.cpp
@@ -331,9 +331,9 @@ class DummyLinOpWithType
         : gko::EnableLinOp<DummyLinOpWithType>(exec, size), value_(value)
     {}
 
-    void apply_absolute() override { value_ = gko::abs(value_); }
+    void compute_absolute_inplace() override { value_ = gko::abs(value_); }
 
-    std::unique_ptr<outplace_absolute_type> get_absolute() const override
+    std::unique_ptr<outplace_absolute_type> compute_absolute() const override
     {
         return std::make_unique<outplace_absolute_type>(
             this->get_executor(), this->get_size(), gko::abs(value_));
@@ -369,7 +369,7 @@ class EnableAbsoluteComputation : public ::testing::Test {
 
 TEST_F(EnableAbsoluteComputation, InplaceAbsoluteOnConcreteType)
 {
-    op->apply_absolute();
+    op->compute_absolute_inplace();
 
     ASSERT_EQ(op->get_value(), std::complex<double>{5.0});
 }
@@ -377,7 +377,7 @@ TEST_F(EnableAbsoluteComputation, InplaceAbsoluteOnConcreteType)
 
 TEST_F(EnableAbsoluteComputation, OutplaceAbsoluteOnConcreteType)
 {
-    auto abs_op = op->get_absolute();
+    auto abs_op = op->compute_absolute();
 
     ASSERT_EQ(typeid(abs_op),
               typeid(std::unique_ptr<gko::remove_complex<dummy_type>>));
@@ -389,7 +389,7 @@ TEST_F(EnableAbsoluteComputation, InplaceAbsoluteOnAbsoluteComputable)
 {
     auto linop = gko::as<gko::LinOp>(op);
 
-    gko::as<gko::AbsoluteComputable>(linop)->apply_absolute();
+    gko::as<gko::AbsoluteComputable>(linop)->compute_absolute_inplace();
 
     ASSERT_EQ(gko::as<dummy_type>(linop)->get_value(),
               std::complex<double>{5.0});
@@ -398,7 +398,7 @@ TEST_F(EnableAbsoluteComputation, InplaceAbsoluteOnAbsoluteComputable)
 
 TEST_F(EnableAbsoluteComputation, OutplaceAbsoluteOnAbsoluteComputable)
 {
-    auto abs_op = op->get_absolute();
+    auto abs_op = op->compute_absolute();
 
     ASSERT_EQ(typeid(abs_op),
               typeid(std::unique_ptr<gko::remove_complex<dummy_type>>));
diff --git a/cuda/test/matrix/coo_kernels.cpp b/cuda/test/matrix/coo_kernels.cpp
index 7575314ad7e..cc52e895097 100644
--- a/cuda/test/matrix/coo_kernels.cpp
+++ b/cuda/test/matrix/coo_kernels.cpp
@@ -275,8 +275,8 @@ TEST_F(Coo, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -286,8 +286,8 @@ TEST_F(Coo, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index d51ea16bc33..f8a171719a1 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -744,8 +744,8 @@ TEST_F(Csr, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>(cuda));
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -755,8 +755,8 @@ TEST_F(Csr, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>(cuda));
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
@@ -766,8 +766,8 @@ TEST_F(Csr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
 {
     set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(cuda));
 
-    complex_mtx->apply_absolute();
-    complex_dmtx->apply_absolute();
+    complex_mtx->compute_absolute_inplace();
+    complex_dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(complex_mtx.get(), complex_dmtx.get(), 1e-14);
 }
@@ -777,8 +777,8 @@ TEST_F(Csr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
 {
     set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(cuda));
 
-    auto abs_mtx = complex_mtx->get_absolute();
-    auto dabs_mtx = complex_dmtx->get_absolute();
+    auto abs_mtx = complex_mtx->compute_absolute();
+    auto dabs_mtx = complex_dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index 1121b994fd6..38273b107e1 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -591,8 +591,8 @@ TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    x->apply_absolute();
-    dx->apply_absolute();
+    x->compute_absolute_inplace();
+    dx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(x.get(), dx.get(), 1e-14);
 }
@@ -602,8 +602,8 @@ TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_x = x->get_absolute();
-    auto dabs_x = dx->get_absolute();
+    auto abs_x = x->compute_absolute();
+    auto dabs_x = dx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_x.get(), dabs_x.get(), 1e-14);
 }
diff --git a/cuda/test/matrix/diagonal_kernels.cpp b/cuda/test/matrix/diagonal_kernels.cpp
index 6a357769022..a3471be6929 100644
--- a/cuda/test/matrix/diagonal_kernels.cpp
+++ b/cuda/test/matrix/diagonal_kernels.cpp
@@ -253,8 +253,8 @@ TEST_F(Diagonal, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    diag->apply_absolute();
-    ddiag->apply_absolute();
+    diag->compute_absolute_inplace();
+    ddiag->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 1e-14);
 }
@@ -264,8 +264,8 @@ TEST_F(Diagonal, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_diag = diag->get_absolute();
-    auto dabs_diag = ddiag->get_absolute();
+    auto abs_diag = diag->compute_absolute();
+    auto dabs_diag = ddiag->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_diag.get(), dabs_diag.get(), 1e-14);
 }
diff --git a/cuda/test/matrix/ell_kernels.cpp b/cuda/test/matrix/ell_kernels.cpp
index 3df92e8c859..86a2ed1d099 100644
--- a/cuda/test/matrix/ell_kernels.cpp
+++ b/cuda/test/matrix/ell_kernels.cpp
@@ -360,8 +360,8 @@ TEST_F(Ell, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -371,8 +371,8 @@ TEST_F(Ell, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/cuda/test/matrix/hybrid_kernels.cpp b/cuda/test/matrix/hybrid_kernels.cpp
index f9adc82ff53..eb8b5ba7669 100644
--- a/cuda/test/matrix/hybrid_kernels.cpp
+++ b/cuda/test/matrix/hybrid_kernels.cpp
@@ -235,8 +235,8 @@ TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -246,8 +246,8 @@ TEST_F(Hybrid, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/cuda/test/matrix/sellp_kernels.cpp b/cuda/test/matrix/sellp_kernels.cpp
index 43375ffb0cf..91f063de36a 100644
--- a/cuda/test/matrix/sellp_kernels.cpp
+++ b/cuda/test/matrix/sellp_kernels.cpp
@@ -354,8 +354,8 @@ TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_matrix(32, 2);
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -365,8 +365,8 @@ TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_matrix(32, 2);
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/hip/test/matrix/coo_kernels.hip.cpp b/hip/test/matrix/coo_kernels.hip.cpp
index 470694c1ece..b235da23077 100644
--- a/hip/test/matrix/coo_kernels.hip.cpp
+++ b/hip/test/matrix/coo_kernels.hip.cpp
@@ -275,8 +275,8 @@ TEST_F(Coo, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -286,8 +286,8 @@ TEST_F(Coo, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index cbcb3c1b37e..2a2a8260cbf 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -729,8 +729,8 @@ TEST_F(Csr, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -740,8 +740,8 @@ TEST_F(Csr, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
@@ -751,8 +751,8 @@ TEST_F(Csr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
 {
     set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(hip));
 
-    complex_mtx->apply_absolute();
-    complex_dmtx->apply_absolute();
+    complex_mtx->compute_absolute_inplace();
+    complex_dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(complex_mtx.get(), complex_dmtx.get(), 1e-14);
 }
@@ -762,8 +762,8 @@ TEST_F(Csr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
 {
     set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(hip));
 
-    auto abs_mtx = complex_mtx->get_absolute();
-    auto dabs_mtx = complex_dmtx->get_absolute();
+    auto abs_mtx = complex_mtx->compute_absolute();
+    auto dabs_mtx = complex_dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index 86e907d27ab..a1dd4f34ec8 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -573,8 +573,8 @@ TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    x->apply_absolute();
-    dx->apply_absolute();
+    x->compute_absolute_inplace();
+    dx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(x.get(), dx.get(), 1e-14);
 }
@@ -584,8 +584,8 @@ TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_x = x->get_absolute();
-    auto dabs_x = dx->get_absolute();
+    auto abs_x = x->compute_absolute();
+    auto dabs_x = dx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_x.get(), dabs_x.get(), 1e-14);
 }
diff --git a/hip/test/matrix/diagonal_kernels.hip.cpp b/hip/test/matrix/diagonal_kernels.hip.cpp
index dd89c7945e5..d86e878dd66 100644
--- a/hip/test/matrix/diagonal_kernels.hip.cpp
+++ b/hip/test/matrix/diagonal_kernels.hip.cpp
@@ -253,8 +253,8 @@ TEST_F(Diagonal, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    diag->apply_absolute();
-    ddiag->apply_absolute();
+    diag->compute_absolute_inplace();
+    ddiag->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 1e-14);
 }
@@ -264,8 +264,8 @@ TEST_F(Diagonal, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_diag = diag->get_absolute();
-    auto dabs_diag = ddiag->get_absolute();
+    auto abs_diag = diag->compute_absolute();
+    auto dabs_diag = ddiag->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_diag.get(), dabs_diag.get(), 1e-14);
 }
diff --git a/hip/test/matrix/ell_kernels.hip.cpp b/hip/test/matrix/ell_kernels.hip.cpp
index 502035ba9c5..4ae03c3d00e 100644
--- a/hip/test/matrix/ell_kernels.hip.cpp
+++ b/hip/test/matrix/ell_kernels.hip.cpp
@@ -360,8 +360,8 @@ TEST_F(Ell, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -371,8 +371,8 @@ TEST_F(Ell, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/hip/test/matrix/hybrid_kernels.hip.cpp b/hip/test/matrix/hybrid_kernels.hip.cpp
index be1cd12ae6c..dd79f63c0f8 100644
--- a/hip/test/matrix/hybrid_kernels.hip.cpp
+++ b/hip/test/matrix/hybrid_kernels.hip.cpp
@@ -235,8 +235,8 @@ TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -246,8 +246,8 @@ TEST_F(Hybrid, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/hip/test/matrix/sellp_kernels.hip.cpp b/hip/test/matrix/sellp_kernels.hip.cpp
index 59d5efda437..8b39aedb189 100644
--- a/hip/test/matrix/sellp_kernels.hip.cpp
+++ b/hip/test/matrix/sellp_kernels.hip.cpp
@@ -353,8 +353,8 @@ TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_matrix(32, 2);
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -364,8 +364,8 @@ TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_matrix(32, 2);
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index 8c6819d0db5..a72f3685848 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -635,21 +635,21 @@ class AbsoluteComputable {
      *
      * @return a pointer to the new absolute LinOp
      */
-    virtual std::unique_ptr<LinOp> get_absolute_linop() const = 0;
+    virtual std::unique_ptr<LinOp> compute_absolute_linop() const = 0;
 
     /**
-     * Apply absolute on each element.
+     * Compute absolute inplace on each element.
      */
-    virtual void apply_absolute() = 0;
+    virtual void compute_absolute_inplace() = 0;
 };
 
 
 /**
  * The EnableAbsoluteComputation mixin provides the default implementations of
- * `get_absolute_linop` and the absolute interface. `get_absolute` gets a new
- * remove_complex<ConcreteLinOp> which removes the complex value_type by
- * applying outplace absolute. `apply_absolute` applies absolute inplace, so it
- * still keeps the value_type of the class.
+ * `compute_absolute_linop` and the absolute interface. `compute_absolute` gets
+ * a new remove_complex<ConcreteLinOp> which removes the complex value_type by
+ * applying outplace absolute. `compute_absolute_inplace` applies absolute
+ * inplace, so it still keeps the value_type of the class.
  *
  * @tparam ConcreteLinOp  the concrete LinOp which is being implemented
  *                        [CRTP parameter]
@@ -663,17 +663,17 @@ class EnableAbsoluteComputation : public AbsoluteComputable {
 
     virtual ~EnableAbsoluteComputation() = default;
 
-    std::unique_ptr<LinOp> get_absolute_linop() const override
+    std::unique_ptr<LinOp> compute_absolute_linop() const override
     {
-        return this->get_absolute();
+        return this->compute_absolute();
     }
 
     /**
-     * Gets the absolute LinOp
+     * Gets the absolute ConcreteLinOp
      *
      * @return a pointer to the new absolute object
      */
-    virtual std::unique_ptr<outplace_type> get_absolute() const = 0;
+    virtual std::unique_ptr<outplace_type> compute_absolute() const = 0;
 };
 
 
diff --git a/include/ginkgo/core/matrix/coo.hpp b/include/ginkgo/core/matrix/coo.hpp
index bde9046aef4..45b1c257c96 100644
--- a/include/ginkgo/core/matrix/coo.hpp
+++ b/include/ginkgo/core/matrix/coo.hpp
@@ -121,9 +121,9 @@ class Coo : public EnableLinOp<Coo<ValueType, IndexType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
-    std::unique_ptr<outplace_absolute_type> get_absolute() const override;
+    std::unique_ptr<outplace_absolute_type> compute_absolute() const override;
 
-    void apply_absolute() override;
+    void compute_absolute_inplace() override;
 
     /**
      * Returns the values of the matrix.
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index 60b32cc2196..bc41f528e23 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -712,9 +712,9 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
-    std::unique_ptr<outplace_absolute_type> get_absolute() const override;
+    std::unique_ptr<outplace_absolute_type> compute_absolute() const override;
 
-    void apply_absolute() override;
+    void compute_absolute_inplace() override;
 
     /**
      * Sorts all (value, col_idx) pairs in each row by column index
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index ddc7b4d9519..6f7c2b16c99 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -249,9 +249,9 @@ class Dense : public EnableLinOp<Dense<ValueType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
-    std::unique_ptr<outplace_absolute_type> get_absolute() const override;
+    std::unique_ptr<outplace_absolute_type> compute_absolute() const override;
 
-    void apply_absolute() override;
+    void compute_absolute_inplace() override;
 
     /**
      * Returns a pointer to the array of values of the matrix.
diff --git a/include/ginkgo/core/matrix/diagonal.hpp b/include/ginkgo/core/matrix/diagonal.hpp
index ceb97e363ca..aec604de9cc 100644
--- a/include/ginkgo/core/matrix/diagonal.hpp
+++ b/include/ginkgo/core/matrix/diagonal.hpp
@@ -103,9 +103,9 @@ class Diagonal : public EnableLinOp<Diagonal<ValueType>>,
 
     void move_to(Csr<ValueType, int64> *result) override;
 
-    std::unique_ptr<outplace_absolute_type> get_absolute() const override;
+    std::unique_ptr<outplace_absolute_type> compute_absolute() const override;
 
-    void apply_absolute() override;
+    void compute_absolute_inplace() override;
 
     /**
      * Returns a pointer to the array of values of the matrix.
diff --git a/include/ginkgo/core/matrix/ell.hpp b/include/ginkgo/core/matrix/ell.hpp
index 02b3a3613c8..6fa705f4ab9 100644
--- a/include/ginkgo/core/matrix/ell.hpp
+++ b/include/ginkgo/core/matrix/ell.hpp
@@ -114,9 +114,9 @@ class Ell : public EnableLinOp<Ell<ValueType, IndexType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
-    std::unique_ptr<outplace_absolute_type> get_absolute() const override;
+    std::unique_ptr<outplace_absolute_type> compute_absolute() const override;
 
-    void apply_absolute() override;
+    void compute_absolute_inplace() override;
 
     /**
      * Returns the values of the matrix.
diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp
index 6dcee07bbe2..8b19de70766 100644
--- a/include/ginkgo/core/matrix/hybrid.hpp
+++ b/include/ginkgo/core/matrix/hybrid.hpp
@@ -363,9 +363,9 @@ class Hybrid
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
-    std::unique_ptr<outplace_absolute_type> get_absolute() const override;
+    std::unique_ptr<outplace_absolute_type> compute_absolute() const override;
 
-    void apply_absolute() override;
+    void compute_absolute_inplace() override;
 
     /**
      * Returns the values of the ell part.
diff --git a/include/ginkgo/core/matrix/sellp.hpp b/include/ginkgo/core/matrix/sellp.hpp
index 6f4b879a0d9..fb5acd36526 100644
--- a/include/ginkgo/core/matrix/sellp.hpp
+++ b/include/ginkgo/core/matrix/sellp.hpp
@@ -111,9 +111,9 @@ class Sellp : public EnableLinOp<Sellp<ValueType, IndexType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
-    std::unique_ptr<outplace_absolute_type> get_absolute() const override;
+    std::unique_ptr<outplace_absolute_type> compute_absolute() const override;
 
-    void apply_absolute() override;
+    void compute_absolute_inplace() override;
 
     /**
      * Returns the values of the matrix.
diff --git a/omp/test/matrix/coo_kernels.cpp b/omp/test/matrix/coo_kernels.cpp
index 03090fd2f54..b2af927543a 100644
--- a/omp/test/matrix/coo_kernels.cpp
+++ b/omp/test/matrix/coo_kernels.cpp
@@ -238,8 +238,8 @@ TEST_F(Coo, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -249,8 +249,8 @@ TEST_F(Coo, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index 8b6a54cdf37..25cb647d98d 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -553,8 +553,8 @@ TEST_F(Csr, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -564,8 +564,8 @@ TEST_F(Csr, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
@@ -575,8 +575,8 @@ TEST_F(Csr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    complex_mtx->apply_absolute();
-    complex_dmtx->apply_absolute();
+    complex_mtx->compute_absolute_inplace();
+    complex_dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(complex_mtx.get(), complex_dmtx.get(), 1e-14);
 }
@@ -586,8 +586,8 @@ TEST_F(Csr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_mtx = complex_mtx->get_absolute();
-    auto dabs_mtx = complex_dmtx->get_absolute();
+    auto abs_mtx = complex_mtx->compute_absolute();
+    auto dabs_mtx = complex_dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index 65afcbf8502..77781b71aab 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -706,8 +706,8 @@ TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    x->apply_absolute();
-    dx->apply_absolute();
+    x->compute_absolute_inplace();
+    dx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(x.get(), dx.get(), 1e-14);
 }
@@ -717,8 +717,8 @@ TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_x = x->get_absolute();
-    auto dabs_x = dx->get_absolute();
+    auto abs_x = x->compute_absolute();
+    auto dabs_x = dx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_x.get(), dabs_x.get(), 1e-14);
 }
diff --git a/omp/test/matrix/diagonal_kernels.cpp b/omp/test/matrix/diagonal_kernels.cpp
index 571ca6fe7e6..2d03fb74f8f 100644
--- a/omp/test/matrix/diagonal_kernels.cpp
+++ b/omp/test/matrix/diagonal_kernels.cpp
@@ -252,8 +252,8 @@ TEST_F(Diagonal, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    diag->apply_absolute();
-    ddiag->apply_absolute();
+    diag->compute_absolute_inplace();
+    ddiag->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 1e-14);
 }
@@ -263,8 +263,8 @@ TEST_F(Diagonal, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_diag = diag->get_absolute();
-    auto dabs_diag = ddiag->get_absolute();
+    auto abs_diag = diag->compute_absolute();
+    auto dabs_diag = ddiag->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_diag.get(), dabs_diag.get(), 1e-14);
 }
diff --git a/omp/test/matrix/ell_kernels.cpp b/omp/test/matrix/ell_kernels.cpp
index 584e88488c2..06195abe138 100644
--- a/omp/test/matrix/ell_kernels.cpp
+++ b/omp/test/matrix/ell_kernels.cpp
@@ -236,8 +236,8 @@ TEST_F(Ell, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -247,8 +247,8 @@ TEST_F(Ell, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/omp/test/matrix/hybrid_kernels.cpp b/omp/test/matrix/hybrid_kernels.cpp
index 046d4c254a5..df85f18455f 100644
--- a/omp/test/matrix/hybrid_kernels.cpp
+++ b/omp/test/matrix/hybrid_kernels.cpp
@@ -234,8 +234,8 @@ TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -245,8 +245,8 @@ TEST_F(Hybrid, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/omp/test/matrix/sellp_kernels.cpp b/omp/test/matrix/sellp_kernels.cpp
index 9d861d2db69..69780f6071e 100644
--- a/omp/test/matrix/sellp_kernels.cpp
+++ b/omp/test/matrix/sellp_kernels.cpp
@@ -270,8 +270,8 @@ TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    mtx->apply_absolute();
-    dmtx->apply_absolute();
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
 }
@@ -281,8 +281,8 @@ TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto abs_mtx = mtx->get_absolute();
-    auto dabs_mtx = dmtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
 }
diff --git a/reference/test/matrix/coo_kernels.cpp b/reference/test/matrix/coo_kernels.cpp
index 0f75f7a1eea..d08ef5f9af9 100644
--- a/reference/test/matrix/coo_kernels.cpp
+++ b/reference/test/matrix/coo_kernels.cpp
@@ -540,7 +540,7 @@ TYPED_TEST(Coo, InplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    mtx->apply_absolute();
+    mtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -553,7 +553,7 @@ TYPED_TEST(Coo, OutplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    auto abs_mtx = mtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -586,7 +586,7 @@ TYPED_TEST(CooComplex, OutplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    auto abs_mtx = mtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
@@ -606,7 +606,7 @@ TYPED_TEST(CooComplex, InplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    mtx->apply_absolute();
+    mtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp
index e2fc18432aa..ac9ae9f64d8 100644
--- a/reference/test/matrix/csr_kernels.cpp
+++ b/reference/test/matrix/csr_kernels.cpp
@@ -1313,7 +1313,7 @@ TYPED_TEST(Csr, InplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    mtx->apply_absolute();
+    mtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -1326,7 +1326,7 @@ TYPED_TEST(Csr, OutplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    auto abs_mtx = mtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -1386,7 +1386,7 @@ TYPED_TEST(CsrComplex, InplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    mtx->apply_absolute();
+    mtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
@@ -1406,7 +1406,7 @@ TYPED_TEST(CsrComplex, OutplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    auto abs_mtx = mtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 94cb1c42542..cea66d04f43 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -2017,7 +2017,7 @@ TYPED_TEST(Dense, InplaceAbsolute)
     // {-2.0, 2.0, 4.5},
     // {2.1, 3.4, 1.2}
     // clang-format on
-    this->mtx5->apply_absolute();
+    this->mtx5->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(this->mtx5,
                         l({{1.0, 1.0, 0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
@@ -2035,7 +2035,7 @@ TYPED_TEST(Dense, InplaceAbsoluteSubMatrix)
     // {-2.0, 2.0, 4.5},
     // {2.1, 3.4, 1.2}
     // clang-format on
-    mtx->apply_absolute();
+    mtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(this->mtx5,
                         l({{1.0, 1.0, -0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
@@ -2051,7 +2051,7 @@ TYPED_TEST(Dense, OutplaceAbsolute)
     // {-2.0, 2.0, 4.5},
     // {2.1, 3.4, 1.2}
     // clang-format on
-    auto abs_mtx = this->mtx5->get_absolute();
+    auto abs_mtx = this->mtx5->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx,
                         l({{1.0, 1.0, 0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
@@ -2069,7 +2069,7 @@ TYPED_TEST(Dense, OutplaceAbsoluteSubMatrix)
     // {-2.0, 2.0, 4.5},
     // {2.1, 3.4, 1.2}
     // clang-format on
-    auto abs_mtx = mtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx, l({{1.0, 1.0}, {2.0, 2.0}}),
                         r<TypeParam>::value);
@@ -2119,7 +2119,7 @@ TYPED_TEST(DenseComplex, InplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    mtx->apply_absolute();
+    mtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
@@ -2138,7 +2138,7 @@ TYPED_TEST(DenseComplex, OutplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    auto abs_mtx = mtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
diff --git a/reference/test/matrix/diagonal_kernels.cpp b/reference/test/matrix/diagonal_kernels.cpp
index 69502cca362..fc207889de3 100644
--- a/reference/test/matrix/diagonal_kernels.cpp
+++ b/reference/test/matrix/diagonal_kernels.cpp
@@ -360,7 +360,7 @@ TYPED_TEST(Diagonal, InplaceAbsolute)
 {
     using value_type = typename TestFixture::value_type;
 
-    this->diag1->apply_absolute();
+    this->diag1->compute_absolute_inplace();
     auto values = this->diag1->get_values();
 
     EXPECT_EQ(values[0], value_type(2.0));
@@ -373,7 +373,7 @@ TYPED_TEST(Diagonal, OutplaceAbsolute)
     using value_type = typename TestFixture::value_type;
     using abs_type = gko::remove_complex<value_type>;
 
-    auto abs_diag = this->diag1->get_absolute();
+    auto abs_diag = this->diag1->compute_absolute();
     auto values = abs_diag->get_values();
 
     EXPECT_EQ(values[0], value_type(2.0));
@@ -425,7 +425,7 @@ TYPED_TEST(DiagonalComplex, InplaceAbsolute)
     diag_values[1] = value_type{-3.0, 0.0};
     diag_values[2] = value_type{0.0, -1.5};
 
-    diag->apply_absolute();
+    diag->compute_absolute_inplace();
 
     EXPECT_EQ(diag_values[0], (value_type{5.0, 0.0}));
     EXPECT_EQ(diag_values[1], (value_type{3.0, 0.0}));
@@ -446,7 +446,7 @@ TYPED_TEST(DiagonalComplex, OutplaceAbsolute)
     diag_values[1] = value_type{-3.0, 0.0};
     diag_values[2] = value_type{0.0, -1.5};
 
-    auto abs_diag = diag->get_absolute();
+    auto abs_diag = diag->compute_absolute();
     auto abs_values = abs_diag->get_values();
 
     EXPECT_EQ(abs_values[0], (value_type{5.0}));
diff --git a/reference/test/matrix/ell_kernels.cpp b/reference/test/matrix/ell_kernels.cpp
index 13b7decd1f1..cc87f580f9e 100644
--- a/reference/test/matrix/ell_kernels.cpp
+++ b/reference/test/matrix/ell_kernels.cpp
@@ -618,7 +618,7 @@ TYPED_TEST(Ell, InplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    mtx->apply_absolute();
+    mtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -631,7 +631,7 @@ TYPED_TEST(Ell, OutplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    auto abs_mtx = mtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -664,7 +664,7 @@ TYPED_TEST(EllComplex, InplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    mtx->apply_absolute();
+    mtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
@@ -684,7 +684,7 @@ TYPED_TEST(EllComplex, OutplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    auto abs_mtx = mtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
diff --git a/reference/test/matrix/hybrid_kernels.cpp b/reference/test/matrix/hybrid_kernels.cpp
index 17f0aae4c50..46123df227b 100644
--- a/reference/test/matrix/hybrid_kernels.cpp
+++ b/reference/test/matrix/hybrid_kernels.cpp
@@ -662,7 +662,7 @@ TYPED_TEST(Hybrid, InplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    mtx->apply_absolute();
+    mtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -675,7 +675,7 @@ TYPED_TEST(Hybrid, OutplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    auto abs_mtx = mtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -708,7 +708,7 @@ TYPED_TEST(HybridComplex, OutplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    auto abs_mtx = mtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
@@ -728,7 +728,7 @@ TYPED_TEST(HybridComplex, InplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    mtx->apply_absolute();
+    mtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
diff --git a/reference/test/matrix/sellp_kernels.cpp b/reference/test/matrix/sellp_kernels.cpp
index 7eef159ddeb..5c24e7afe3d 100644
--- a/reference/test/matrix/sellp_kernels.cpp
+++ b/reference/test/matrix/sellp_kernels.cpp
@@ -621,7 +621,7 @@ TYPED_TEST(Sellp, InplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    mtx->apply_absolute();
+    mtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -634,7 +634,7 @@ TYPED_TEST(Sellp, OutplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
 
-    auto abs_mtx = mtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
@@ -667,7 +667,7 @@ TYPED_TEST(SellpComplex, OutplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    auto abs_mtx = mtx->get_absolute();
+    auto abs_mtx = mtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(
         abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
@@ -687,7 +687,7 @@ TYPED_TEST(SellpComplex, InplaceAbsolute)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    mtx->apply_absolute();
+    mtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(
         mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);

From c39a692ded3a4795d96896ae90005cbb8249583a Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 11 Sep 2020 12:35:15 +0200
Subject: [PATCH 0017/1008] refine format

Co-authored-by: fritzgoebel <fritz.goebel@kit.edu>
---
 common/matrix/dense_kernels.hpp.inc          | 2 +-
 core/matrix/csr.cpp                          | 1 +
 core/test/base/lin_op.cpp                    | 9 +++++++--
 include/ginkgo/core/base/lin_op.hpp          | 5 +++--
 omp/test/matrix/diagonal_kernels.cpp         | 1 +
 reference/test/components/absolute_array.cpp | 4 ++--
 6 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index 49be1ba3724..c26203115fe 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -534,7 +534,7 @@ __global__ __launch_bounds__(default_block_size) void outplace_absolute_dense(
     auto row = tidx / num_cols;
     auto col = tidx % num_cols;
     if (row < num_rows) {
-        out[row * stride_out + col] = abs(in[row * stride_out + col]);
+        out[row * stride_out + col] = abs(in[row * stride_in + col]);
     }
 }
 
diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp
index 605f7b09fed..eae8599a3bb 100644
--- a/core/matrix/csr.cpp
+++ b/core/matrix/csr.cpp
@@ -87,6 +87,7 @@ GKO_REGISTER_OPERATION(inplace_absolute_array,
 GKO_REGISTER_OPERATION(outplace_absolute_array,
                        components::outplace_absolute_array);
 
+
 }  // namespace csr
 
 
diff --git a/core/test/base/lin_op.cpp b/core/test/base/lin_op.cpp
index 497396efaeb..144cbd33e42 100644
--- a/core/test/base/lin_op.cpp
+++ b/core/test/base/lin_op.cpp
@@ -30,13 +30,18 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <complex>
 #include <ginkgo/core/base/lin_op.hpp>
-#include <ginkgo/core/base/math.hpp>
+
+
+#include <complex>
+
 
 #include <gtest/gtest.h>
 
 
+#include <ginkgo/core/base/math.hpp>
+
+
 namespace {
 
 
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index a72f3685848..5be7dadcded 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -625,8 +625,9 @@ class DiagonalExtractable {
 
 
 /**
- * The AbsoluteComputable is an interface to allow get absolute from a LinOp.
- * Use EnableAbsoluteComputation<ConcreteLinOp> to implement this interface.
+ * The AbsoluteComputable is an interface that allows to get the component wise
+ * absolute of a LinOp. Use EnableAbsoluteComputation<ConcreteLinOp> to
+ * implement this interface.
  */
 class AbsoluteComputable {
 public:
diff --git a/omp/test/matrix/diagonal_kernels.cpp b/omp/test/matrix/diagonal_kernels.cpp
index 2d03fb74f8f..e58380ea773 100644
--- a/omp/test/matrix/diagonal_kernels.cpp
+++ b/omp/test/matrix/diagonal_kernels.cpp
@@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <random>
 #include <vector>
 
+
 #include <gtest/gtest.h>
 
 
diff --git a/reference/test/components/absolute_array.cpp b/reference/test/components/absolute_array.cpp
index 1c2f2952551..83cc444df20 100644
--- a/reference/test/components/absolute_array.cpp
+++ b/reference/test/components/absolute_array.cpp
@@ -78,7 +78,7 @@ class AbsoluteArray : public ::testing::Test {
 TYPED_TEST_CASE(AbsoluteArray, gko::test::ValueTypes);
 
 
-TYPED_TEST(AbsoluteArray, InplaceEqualsReference)
+TYPED_TEST(AbsoluteArray, InplaceEqualsExpected)
 {
     using T = typename TestFixture::value_type;
     gko::kernels::reference::components::inplace_absolute_array(
@@ -87,7 +87,7 @@ TYPED_TEST(AbsoluteArray, InplaceEqualsReference)
 }
 
 
-TYPED_TEST(AbsoluteArray, OutplaceEqualsReference)
+TYPED_TEST(AbsoluteArray, OutplaceEqualsExpected)
 {
     using T = typename TestFixture::value_type;
     using AbsT = typename TestFixture::abs_type;

From 004f39abf525efc1171141ac491b3a08a51735d8 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 15 Sep 2020 15:16:23 +0200
Subject: [PATCH 0018/1008] rename make_complex, use absolute_type only...etc

1. remove/add_complex
2. to_real/complex alias
3. using absolute_type not outplace_absolute_type
4. remove unneeded .get() in test

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/matrix/coo.cpp                      |  6 +--
 core/matrix/csr.cpp                      |  6 +--
 core/matrix/dense.cpp                    |  4 +-
 core/matrix/diagonal.cpp                 |  5 +--
 core/matrix/ell.cpp                      |  4 +-
 core/matrix/hybrid.cpp                   |  4 +-
 core/matrix/sellp.cpp                    |  4 +-
 core/test/base/lin_op.cpp                |  6 +--
 cuda/test/matrix/coo_kernels.cpp         |  4 +-
 cuda/test/matrix/csr_kernels.cpp         |  8 ++--
 cuda/test/matrix/dense_kernels.cpp       |  4 +-
 cuda/test/matrix/diagonal_kernels.cpp    |  4 +-
 cuda/test/matrix/ell_kernels.cpp         |  4 +-
 cuda/test/matrix/hybrid_kernels.cpp      |  4 +-
 cuda/test/matrix/sellp_kernels.cpp       |  4 +-
 hip/test/matrix/coo_kernels.hip.cpp      |  4 +-
 hip/test/matrix/csr_kernels.hip.cpp      |  8 ++--
 hip/test/matrix/dense_kernels.hip.cpp    |  4 +-
 hip/test/matrix/diagonal_kernels.hip.cpp |  4 +-
 hip/test/matrix/ell_kernels.hip.cpp      |  4 +-
 hip/test/matrix/hybrid_kernels.hip.cpp   |  4 +-
 hip/test/matrix/sellp_kernels.hip.cpp    |  4 +-
 include/ginkgo/core/base/math.hpp        | 52 ++++++++++++++++++++----
 include/ginkgo/core/matrix/coo.hpp       |  6 +--
 include/ginkgo/core/matrix/csr.hpp       | 12 +++---
 include/ginkgo/core/matrix/dense.hpp     |  6 +--
 include/ginkgo/core/matrix/diagonal.hpp  |  6 +--
 include/ginkgo/core/matrix/ell.hpp       |  6 +--
 include/ginkgo/core/matrix/hybrid.hpp    |  6 +--
 include/ginkgo/core/matrix/sellp.hpp     |  6 +--
 omp/test/matrix/coo_kernels.cpp          |  4 +-
 omp/test/matrix/csr_kernels.cpp          |  8 ++--
 omp/test/matrix/dense_kernels.cpp        |  4 +-
 omp/test/matrix/diagonal_kernels.cpp     |  4 +-
 omp/test/matrix/ell_kernels.cpp          |  4 +-
 omp/test/matrix/hybrid_kernels.cpp       |  4 +-
 omp/test/matrix/sellp_kernels.cpp        |  4 +-
 37 files changed, 134 insertions(+), 101 deletions(-)

diff --git a/core/matrix/coo.cpp b/core/matrix/coo.cpp
index 50b42d04ab4..6ab488d9988 100644
--- a/core/matrix/coo.cpp
+++ b/core/matrix/coo.cpp
@@ -249,13 +249,13 @@ void Coo<ValueType, IndexType>::compute_absolute_inplace()
 
 
 template <typename ValueType, typename IndexType>
-std::unique_ptr<typename Coo<ValueType, IndexType>::outplace_absolute_type>
+std::unique_ptr<typename Coo<ValueType, IndexType>::absolute_type>
 Coo<ValueType, IndexType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
-    auto abs_coo = outplace_absolute_type::create(
-        exec, this->get_size(), this->get_num_stored_elements());
+    auto abs_coo = absolute_type::create(exec, this->get_size(),
+                                         this->get_num_stored_elements());
 
     abs_coo->col_idxs_ = col_idxs_;
     abs_coo->row_idxs_ = row_idxs_;
diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp
index eae8599a3bb..32431850132 100644
--- a/core/matrix/csr.cpp
+++ b/core/matrix/csr.cpp
@@ -498,13 +498,13 @@ void Csr<ValueType, IndexType>::compute_absolute_inplace()
 
 
 template <typename ValueType, typename IndexType>
-std::unique_ptr<typename Csr<ValueType, IndexType>::outplace_absolute_type>
+std::unique_ptr<typename Csr<ValueType, IndexType>::absolute_type>
 Csr<ValueType, IndexType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
-    auto abs_csr = outplace_absolute_type::create(
-        exec, this->get_size(), this->get_num_stored_elements());
+    auto abs_csr = absolute_type::create(exec, this->get_size(),
+                                         this->get_num_stored_elements());
 
     abs_csr->col_idxs_ = col_idxs_;
     abs_csr->row_ptrs_ = row_ptrs_;
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index e51ad7bcff1..aaaaf2d01a5 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -764,13 +764,13 @@ void Dense<ValueType>::compute_absolute_inplace()
 
 
 template <typename ValueType>
-std::unique_ptr<typename Dense<ValueType>::outplace_absolute_type>
+std::unique_ptr<typename Dense<ValueType>::absolute_type>
 Dense<ValueType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
     // do not inherit the stride
-    auto abs_dense = outplace_absolute_type::create(exec, this->get_size());
+    auto abs_dense = absolute_type::create(exec, this->get_size());
 
     exec->run(dense::make_outplace_absolute_dense(this, abs_dense.get()));
 
diff --git a/core/matrix/diagonal.cpp b/core/matrix/diagonal.cpp
index e1cb767efbf..8df8af6a801 100644
--- a/core/matrix/diagonal.cpp
+++ b/core/matrix/diagonal.cpp
@@ -277,13 +277,12 @@ void Diagonal<ValueType>::compute_absolute_inplace()
 
 
 template <typename ValueType>
-std::unique_ptr<typename Diagonal<ValueType>::outplace_absolute_type>
+std::unique_ptr<typename Diagonal<ValueType>::absolute_type>
 Diagonal<ValueType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
-    auto abs_diagonal =
-        outplace_absolute_type::create(exec, this->get_size()[0]);
+    auto abs_diagonal = absolute_type::create(exec, this->get_size()[0]);
 
     exec->run(diagonal::make_outplace_absolute_array(
         this->get_const_values(), this->get_size()[0],
diff --git a/core/matrix/ell.cpp b/core/matrix/ell.cpp
index 26f518d2cbc..c802fe917e5 100644
--- a/core/matrix/ell.cpp
+++ b/core/matrix/ell.cpp
@@ -267,12 +267,12 @@ void Ell<ValueType, IndexType>::compute_absolute_inplace()
 
 
 template <typename ValueType, typename IndexType>
-std::unique_ptr<typename Ell<ValueType, IndexType>::outplace_absolute_type>
+std::unique_ptr<typename Ell<ValueType, IndexType>::absolute_type>
 Ell<ValueType, IndexType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
-    auto abs_ell = outplace_absolute_type::create(
+    auto abs_ell = absolute_type::create(
         exec, this->get_size(), this->get_num_stored_elements_per_row(),
         this->get_stride());
 
diff --git a/core/matrix/hybrid.cpp b/core/matrix/hybrid.cpp
index 9487bd99a4e..b4e83f3906e 100644
--- a/core/matrix/hybrid.cpp
+++ b/core/matrix/hybrid.cpp
@@ -303,13 +303,13 @@ void Hybrid<ValueType, IndexType>::compute_absolute_inplace()
 
 
 template <typename ValueType, typename IndexType>
-std::unique_ptr<typename Hybrid<ValueType, IndexType>::outplace_absolute_type>
+std::unique_ptr<typename Hybrid<ValueType, IndexType>::absolute_type>
 Hybrid<ValueType, IndexType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
     // use default strategy
-    auto abs_hybrid = outplace_absolute_type::create(exec, this->get_size());
+    auto abs_hybrid = absolute_type::create(exec, this->get_size());
 
     abs_hybrid->ell_->copy_from(ell_->compute_absolute());
     abs_hybrid->coo_->copy_from(coo_->compute_absolute());
diff --git a/core/matrix/sellp.cpp b/core/matrix/sellp.cpp
index 81af57b5dd6..94c30544832 100644
--- a/core/matrix/sellp.cpp
+++ b/core/matrix/sellp.cpp
@@ -320,12 +320,12 @@ void Sellp<ValueType, IndexType>::compute_absolute_inplace()
 
 
 template <typename ValueType, typename IndexType>
-std::unique_ptr<typename Sellp<ValueType, IndexType>::outplace_absolute_type>
+std::unique_ptr<typename Sellp<ValueType, IndexType>::absolute_type>
 Sellp<ValueType, IndexType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
-    auto abs_sellp = outplace_absolute_type::create(
+    auto abs_sellp = absolute_type::create(
         exec, this->get_size(), this->get_slice_size(),
         this->get_stride_factor(), this->get_total_cols());
 
diff --git a/core/test/base/lin_op.cpp b/core/test/base/lin_op.cpp
index 144cbd33e42..7ba5c794945 100644
--- a/core/test/base/lin_op.cpp
+++ b/core/test/base/lin_op.cpp
@@ -326,7 +326,7 @@ class DummyLinOpWithType
       public gko::EnableCreateMethod<DummyLinOpWithType<Type>>,
       public gko::EnableAbsoluteComputation<DummyLinOpWithType<Type>> {
 public:
-    using outplace_absolute_type = gko::remove_complex<DummyLinOpWithType>;
+    using absolute_type = gko::remove_complex<DummyLinOpWithType>;
     DummyLinOpWithType(std::shared_ptr<const gko::Executor> exec)
         : gko::EnableLinOp<DummyLinOpWithType>(exec)
     {}
@@ -338,9 +338,9 @@ class DummyLinOpWithType
 
     void compute_absolute_inplace() override { value_ = gko::abs(value_); }
 
-    std::unique_ptr<outplace_absolute_type> compute_absolute() const override
+    std::unique_ptr<absolute_type> compute_absolute() const override
     {
-        return std::make_unique<outplace_absolute_type>(
+        return std::make_unique<absolute_type>(
             this->get_executor(), this->get_size(), gko::abs(value_));
     }
 
diff --git a/cuda/test/matrix/coo_kernels.cpp b/cuda/test/matrix/coo_kernels.cpp
index cc52e895097..6815a9d3df1 100644
--- a/cuda/test/matrix/coo_kernels.cpp
+++ b/cuda/test/matrix/coo_kernels.cpp
@@ -278,7 +278,7 @@ TEST_F(Coo, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -289,7 +289,7 @@ TEST_F(Coo, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index f8a171719a1..4205f5c5d97 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -747,7 +747,7 @@ TEST_F(Csr, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -758,7 +758,7 @@ TEST_F(Csr, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
@@ -769,7 +769,7 @@ TEST_F(Csr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
     complex_mtx->compute_absolute_inplace();
     complex_dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(complex_mtx.get(), complex_dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(complex_mtx, complex_dmtx, 1e-14);
 }
 
 
@@ -780,7 +780,7 @@ TEST_F(Csr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
     auto abs_mtx = complex_mtx->compute_absolute();
     auto dabs_mtx = complex_dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index 38273b107e1..e8732c13136 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -594,7 +594,7 @@ TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
     x->compute_absolute_inplace();
     dx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(x.get(), dx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(x, dx, 1e-14);
 }
 
 
@@ -605,7 +605,7 @@ TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_x = x->compute_absolute();
     auto dabs_x = dx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_x.get(), dabs_x.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, 1e-14);
 }
 
 
diff --git a/cuda/test/matrix/diagonal_kernels.cpp b/cuda/test/matrix/diagonal_kernels.cpp
index a3471be6929..695eee1a880 100644
--- a/cuda/test/matrix/diagonal_kernels.cpp
+++ b/cuda/test/matrix/diagonal_kernels.cpp
@@ -256,7 +256,7 @@ TEST_F(Diagonal, InplaceAbsoluteMatrixIsEquivalentToRef)
     diag->compute_absolute_inplace();
     ddiag->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(diag, ddiag, 1e-14);
 }
 
 
@@ -267,7 +267,7 @@ TEST_F(Diagonal, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_diag = diag->compute_absolute();
     auto dabs_diag = ddiag->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_diag.get(), dabs_diag.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_diag, dabs_diag, 1e-14);
 }
 
 
diff --git a/cuda/test/matrix/ell_kernels.cpp b/cuda/test/matrix/ell_kernels.cpp
index 86a2ed1d099..ce48a80e634 100644
--- a/cuda/test/matrix/ell_kernels.cpp
+++ b/cuda/test/matrix/ell_kernels.cpp
@@ -363,7 +363,7 @@ TEST_F(Ell, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -374,7 +374,7 @@ TEST_F(Ell, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/cuda/test/matrix/hybrid_kernels.cpp b/cuda/test/matrix/hybrid_kernels.cpp
index eb8b5ba7669..7bcc7ad47b3 100644
--- a/cuda/test/matrix/hybrid_kernels.cpp
+++ b/cuda/test/matrix/hybrid_kernels.cpp
@@ -238,7 +238,7 @@ TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -249,7 +249,7 @@ TEST_F(Hybrid, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/cuda/test/matrix/sellp_kernels.cpp b/cuda/test/matrix/sellp_kernels.cpp
index 91f063de36a..7a00b63d0c1 100644
--- a/cuda/test/matrix/sellp_kernels.cpp
+++ b/cuda/test/matrix/sellp_kernels.cpp
@@ -357,7 +357,7 @@ TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -368,7 +368,7 @@ TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/hip/test/matrix/coo_kernels.hip.cpp b/hip/test/matrix/coo_kernels.hip.cpp
index b235da23077..6d0a0935cd6 100644
--- a/hip/test/matrix/coo_kernels.hip.cpp
+++ b/hip/test/matrix/coo_kernels.hip.cpp
@@ -278,7 +278,7 @@ TEST_F(Coo, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -289,7 +289,7 @@ TEST_F(Coo, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index 2a2a8260cbf..54b0f4cda65 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -732,7 +732,7 @@ TEST_F(Csr, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -743,7 +743,7 @@ TEST_F(Csr, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
@@ -754,7 +754,7 @@ TEST_F(Csr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
     complex_mtx->compute_absolute_inplace();
     complex_dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(complex_mtx.get(), complex_dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(complex_mtx, complex_dmtx, 1e-14);
 }
 
 
@@ -765,7 +765,7 @@ TEST_F(Csr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
     auto abs_mtx = complex_mtx->compute_absolute();
     auto dabs_mtx = complex_dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index a1dd4f34ec8..c14809ed79e 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -576,7 +576,7 @@ TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
     x->compute_absolute_inplace();
     dx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(x.get(), dx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(x, dx, 1e-14);
 }
 
 
@@ -587,7 +587,7 @@ TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_x = x->compute_absolute();
     auto dabs_x = dx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_x.get(), dabs_x.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, 1e-14);
 }
 
 
diff --git a/hip/test/matrix/diagonal_kernels.hip.cpp b/hip/test/matrix/diagonal_kernels.hip.cpp
index d86e878dd66..1bd1b3f0b8a 100644
--- a/hip/test/matrix/diagonal_kernels.hip.cpp
+++ b/hip/test/matrix/diagonal_kernels.hip.cpp
@@ -256,7 +256,7 @@ TEST_F(Diagonal, InplaceAbsoluteMatrixIsEquivalentToRef)
     diag->compute_absolute_inplace();
     ddiag->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(diag, ddiag, 1e-14);
 }
 
 
@@ -267,7 +267,7 @@ TEST_F(Diagonal, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_diag = diag->compute_absolute();
     auto dabs_diag = ddiag->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_diag.get(), dabs_diag.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_diag, dabs_diag, 1e-14);
 }
 
 
diff --git a/hip/test/matrix/ell_kernels.hip.cpp b/hip/test/matrix/ell_kernels.hip.cpp
index 4ae03c3d00e..74d22dbb738 100644
--- a/hip/test/matrix/ell_kernels.hip.cpp
+++ b/hip/test/matrix/ell_kernels.hip.cpp
@@ -363,7 +363,7 @@ TEST_F(Ell, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -374,7 +374,7 @@ TEST_F(Ell, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/hip/test/matrix/hybrid_kernels.hip.cpp b/hip/test/matrix/hybrid_kernels.hip.cpp
index dd79f63c0f8..5aa3b2d0258 100644
--- a/hip/test/matrix/hybrid_kernels.hip.cpp
+++ b/hip/test/matrix/hybrid_kernels.hip.cpp
@@ -238,7 +238,7 @@ TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -249,7 +249,7 @@ TEST_F(Hybrid, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/hip/test/matrix/sellp_kernels.hip.cpp b/hip/test/matrix/sellp_kernels.hip.cpp
index 8b39aedb189..6c99519b71d 100644
--- a/hip/test/matrix/sellp_kernels.hip.cpp
+++ b/hip/test/matrix/sellp_kernels.hip.cpp
@@ -356,7 +356,7 @@ TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -367,7 +367,7 @@ TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 85abf81a08e..dc222110fb4 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -139,7 +139,7 @@ struct remove_complex_impl<std::complex<T>> {
  * @tparam T  the type being made complex
  */
 template <typename T>
-struct make_complex_impl {
+struct add_complex_impl {
     using type = std::complex<T>;
 };
 
@@ -149,7 +149,7 @@ struct make_complex_impl {
  * @tparam T  the type being made complex
  */
 template <typename T>
-struct make_complex_impl<std::complex<T>> {
+struct add_complex_impl<std::complex<T>> {
     using type = std::complex<T>;
 };
 
@@ -222,16 +222,16 @@ struct remove_complex_s<
 
 
 template <typename T, typename = void>
-struct make_complex_s {};
+struct add_complex_s {};
 
 /**
  * Obtains a complex counterpart of a real type, and leaves the type
  * unchanged if it is a complex type for complex/scalar type.
  */
 template <typename T>
-struct make_complex_s<
+struct add_complex_s<
     T, xstd::void_t<std::enable_if_t<is_complex_or_scalar_impl<T>::value>>> {
-    using type = typename detail::make_complex_impl<T>::type;
+    using type = typename detail::add_complex_impl<T>::type;
 };
 
 /**
@@ -239,10 +239,10 @@ struct make_complex_s<
  * converts real parameters to complex parameters.
  */
 template <typename T>
-struct make_complex_s<
+struct add_complex_s<
     T, xstd::void_t<std::enable_if_t<!is_complex_or_scalar_impl<T>::value>>> {
     using type =
-        typename detail::template_convertor<detail::make_complex_impl, T>::type;
+        typename detail::template_convertor<detail::add_complex_impl, T>::type;
 };
 
 
@@ -345,7 +345,7 @@ using remove_complex = typename remove_complex_s<T>::type;
  *
  * @tparam T  type to remove complex
  */
-using detail::make_complex_s;
+using detail::add_complex_s;
 
 /**
  * Obtain the type which removed the complex of complex/scalar type or the
@@ -354,7 +354,41 @@ using detail::make_complex_s;
  * @tparam T  type to remove complex
  */
 template <typename T>
-using make_complex = typename make_complex_s<T>::type;
+using add_complex = typename add_complex_s<T>::type;
+
+
+/**
+ * to_real_s is alias of remove_complex_s
+ *
+ * @tparam T  type to real
+ */
+template <typename T>
+using to_real_s = remove_complex_s<T>;
+
+/**
+ * to_real is alias of remove_complex
+ *
+ * @tparam T  type to real
+ */
+template <typename T>
+using to_real = remove_complex<T>;
+
+
+/**
+ * to_complex_s is alias of add_complex_s
+ *
+ * @tparam T  type to complex
+ */
+template <typename T>
+using to_complex_s = add_complex_s<T>;
+
+/**
+ * to_complex is alias of add_complex
+ *
+ * @tparam T  type to complex
+ */
+template <typename T>
+using to_complex = add_complex<T>;
 
 
 namespace detail {
diff --git a/include/ginkgo/core/matrix/coo.hpp b/include/ginkgo/core/matrix/coo.hpp
index 45b1c257c96..6a9b972e7ef 100644
--- a/include/ginkgo/core/matrix/coo.hpp
+++ b/include/ginkgo/core/matrix/coo.hpp
@@ -88,7 +88,7 @@ class Coo : public EnableLinOp<Coo<ValueType, IndexType>>,
     friend class Csr<ValueType, IndexType>;
     friend class Dense<ValueType>;
     friend class CooBuilder<ValueType, IndexType>;
-    friend class Coo<make_complex<ValueType>, IndexType>;
+    friend class Coo<add_complex<ValueType>, IndexType>;
 
 public:
     using EnableLinOp<Coo>::convert_to;
@@ -97,7 +97,7 @@ class Coo : public EnableLinOp<Coo<ValueType, IndexType>>,
     using value_type = ValueType;
     using index_type = IndexType;
     using mat_data = matrix_data<ValueType, IndexType>;
-    using outplace_absolute_type =
+    using absolute_type =
         typename EnableAbsoluteComputation<Coo>::outplace_type;
 
     friend class Coo<next_precision<ValueType>, IndexType>;
@@ -121,7 +121,7 @@ class Coo : public EnableLinOp<Coo<ValueType, IndexType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
-    std::unique_ptr<outplace_absolute_type> compute_absolute() const override;
+    std::unique_ptr<absolute_type> compute_absolute() const override;
 
     void compute_absolute_inplace() override;
 
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index bc41f528e23..b71b194e6f3 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -141,12 +141,12 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
     friend class Sellp<ValueType, IndexType>;
     friend class SparsityCsr<ValueType, IndexType>;
     friend class CsrBuilder<ValueType, IndexType>;
-    friend class Csr<make_complex<ValueType>, IndexType>;
-    // friend make_complex<Csr>;
+    friend class Csr<add_complex<ValueType>, IndexType>;
+    // friend add_complex<Csr>;
     //   error: can not recognize it is class correctly.
-    // friend class make_complex<Csr>;
+    // friend class add_complex<Csr>;
     //   error: using alias template specialization
-    // friend class make_complex_s<Csr<ValueType,IndexType>>::type;
+    // friend class add_complex_s<Csr<ValueType,IndexType>>::type;
     //   error: can not recognize it is class correctly.
 
 public:
@@ -154,7 +154,7 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
     using index_type = IndexType;
     using transposed_type = Csr<ValueType, IndexType>;
     using mat_data = matrix_data<ValueType, IndexType>;
-    using outplace_absolute_type =
+    using absolute_type =
         typename EnableAbsoluteComputation<Csr>::outplace_type;
 
     class automatical;
@@ -712,7 +712,7 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
-    std::unique_ptr<outplace_absolute_type> compute_absolute() const override;
+    std::unique_ptr<absolute_type> compute_absolute() const override;
 
     void compute_absolute_inplace() override;
 
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 6f7c2b16c99..b0b1aa9712c 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -127,7 +127,7 @@ class Dense : public EnableLinOp<Dense<ValueType>>,
     friend class Sellp<ValueType, int64>;
     friend class SparsityCsr<ValueType, int32>;
     friend class SparsityCsr<ValueType, int64>;
-    friend class Dense<make_complex<ValueType>>;
+    friend class Dense<add_complex<ValueType>>;
 
 public:
     using EnableLinOp<Dense>::convert_to;
@@ -138,7 +138,7 @@ class Dense : public EnableLinOp<Dense<ValueType>>,
     using transposed_type = Dense<ValueType>;
     using mat_data = gko::matrix_data<ValueType, int64>;
     using mat_data32 = gko::matrix_data<ValueType, int32>;
-    using outplace_absolute_type =
+    using absolute_type =
         typename EnableAbsoluteComputation<Dense>::outplace_type;
 
     using row_major_range = gko::range<gko::accessor::row_major<ValueType, 2>>;
@@ -249,7 +249,7 @@ class Dense : public EnableLinOp<Dense<ValueType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
-    std::unique_ptr<outplace_absolute_type> compute_absolute() const override;
+    std::unique_ptr<absolute_type> compute_absolute() const override;
 
     void compute_absolute_inplace() override;
 
diff --git a/include/ginkgo/core/matrix/diagonal.hpp b/include/ginkgo/core/matrix/diagonal.hpp
index aec604de9cc..f9e42fb769f 100644
--- a/include/ginkgo/core/matrix/diagonal.hpp
+++ b/include/ginkgo/core/matrix/diagonal.hpp
@@ -78,7 +78,7 @@ class Diagonal : public EnableLinOp<Diagonal<ValueType>>,
     friend class EnableCreateMethod<Diagonal>;
     friend class Csr<ValueType, int32>;
     friend class Csr<ValueType, int64>;
-    friend class Diagonal<make_complex<ValueType>>;
+    friend class Diagonal<add_complex<ValueType>>;
 
 public:
     using EnableLinOp<Diagonal>::convert_to;
@@ -88,7 +88,7 @@ class Diagonal : public EnableLinOp<Diagonal<ValueType>>,
     using index_type = int64;
     using mat_data = gko::matrix_data<ValueType, int64>;
     using mat_data32 = gko::matrix_data<ValueType, int32>;
-    using outplace_absolute_type =
+    using absolute_type =
         typename EnableAbsoluteComputation<Diagonal>::outplace_type;
 
     std::unique_ptr<LinOp> transpose() const override;
@@ -103,7 +103,7 @@ class Diagonal : public EnableLinOp<Diagonal<ValueType>>,
 
     void move_to(Csr<ValueType, int64> *result) override;
 
-    std::unique_ptr<outplace_absolute_type> compute_absolute() const override;
+    std::unique_ptr<absolute_type> compute_absolute() const override;
 
     void compute_absolute_inplace() override;
 
diff --git a/include/ginkgo/core/matrix/ell.hpp b/include/ginkgo/core/matrix/ell.hpp
index 6fa705f4ab9..5fdfeb23770 100644
--- a/include/ginkgo/core/matrix/ell.hpp
+++ b/include/ginkgo/core/matrix/ell.hpp
@@ -81,7 +81,7 @@ class Ell : public EnableLinOp<Ell<ValueType, IndexType>>,
     friend class EnablePolymorphicObject<Ell, LinOp>;
     friend class Dense<ValueType>;
     friend class Csr<ValueType, IndexType>;
-    friend class Ell<make_complex<ValueType>, IndexType>;
+    friend class Ell<add_complex<ValueType>, IndexType>;
 
 public:
     using EnableLinOp<Ell>::convert_to;
@@ -90,7 +90,7 @@ class Ell : public EnableLinOp<Ell<ValueType, IndexType>>,
     using value_type = ValueType;
     using index_type = IndexType;
     using mat_data = matrix_data<ValueType, IndexType>;
-    using outplace_absolute_type =
+    using absolute_type =
         typename EnableAbsoluteComputation<Ell>::outplace_type;
 
     friend class Ell<next_precision<ValueType>, IndexType>;
@@ -114,7 +114,7 @@ class Ell : public EnableLinOp<Ell<ValueType, IndexType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
-    std::unique_ptr<outplace_absolute_type> compute_absolute() const override;
+    std::unique_ptr<absolute_type> compute_absolute() const override;
 
     void compute_absolute_inplace() override;
 
diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp
index 8b19de70766..453be83cdea 100644
--- a/include/ginkgo/core/matrix/hybrid.hpp
+++ b/include/ginkgo/core/matrix/hybrid.hpp
@@ -82,7 +82,7 @@ class Hybrid
     friend class EnablePolymorphicObject<Hybrid, LinOp>;
     friend class Dense<ValueType>;
     friend class Csr<ValueType, IndexType>;
-    friend class Hybrid<make_complex<ValueType>, IndexType>;
+    friend class Hybrid<add_complex<ValueType>, IndexType>;
 
 
 public:
@@ -94,7 +94,7 @@ class Hybrid
     using mat_data = matrix_data<ValueType, IndexType>;
     using coo_type = Coo<ValueType, IndexType>;
     using ell_type = Ell<ValueType, IndexType>;
-    using outplace_absolute_type =
+    using absolute_type =
         typename EnableAbsoluteComputation<Hybrid>::outplace_type;
 
 
@@ -363,7 +363,7 @@ class Hybrid
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
-    std::unique_ptr<outplace_absolute_type> compute_absolute() const override;
+    std::unique_ptr<absolute_type> compute_absolute() const override;
 
     void compute_absolute_inplace() override;
 
diff --git a/include/ginkgo/core/matrix/sellp.hpp b/include/ginkgo/core/matrix/sellp.hpp
index fb5acd36526..cbf06478bc6 100644
--- a/include/ginkgo/core/matrix/sellp.hpp
+++ b/include/ginkgo/core/matrix/sellp.hpp
@@ -78,7 +78,7 @@ class Sellp : public EnableLinOp<Sellp<ValueType, IndexType>>,
     friend class EnablePolymorphicObject<Sellp, LinOp>;
     friend class Dense<ValueType>;
     friend class Csr<ValueType, IndexType>;
-    friend class Sellp<make_complex<ValueType>, IndexType>;
+    friend class Sellp<add_complex<ValueType>, IndexType>;
 
 public:
     using EnableLinOp<Sellp>::convert_to;
@@ -87,7 +87,7 @@ class Sellp : public EnableLinOp<Sellp<ValueType, IndexType>>,
     using value_type = ValueType;
     using index_type = IndexType;
     using mat_data = matrix_data<ValueType, IndexType>;
-    using outplace_absolute_type =
+    using absolute_type =
         typename EnableAbsoluteComputation<Sellp>::outplace_type;
 
     friend class Sellp<next_precision<ValueType>, IndexType>;
@@ -111,7 +111,7 @@ class Sellp : public EnableLinOp<Sellp<ValueType, IndexType>>,
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
-    std::unique_ptr<outplace_absolute_type> compute_absolute() const override;
+    std::unique_ptr<absolute_type> compute_absolute() const override;
 
     void compute_absolute_inplace() override;
 
diff --git a/omp/test/matrix/coo_kernels.cpp b/omp/test/matrix/coo_kernels.cpp
index b2af927543a..5139e1c1334 100644
--- a/omp/test/matrix/coo_kernels.cpp
+++ b/omp/test/matrix/coo_kernels.cpp
@@ -241,7 +241,7 @@ TEST_F(Coo, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -252,7 +252,7 @@ TEST_F(Coo, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index 25cb647d98d..b9da8e7e81e 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -556,7 +556,7 @@ TEST_F(Csr, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -567,7 +567,7 @@ TEST_F(Csr, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
@@ -578,7 +578,7 @@ TEST_F(Csr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
     complex_mtx->compute_absolute_inplace();
     complex_dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(complex_mtx.get(), complex_dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(complex_mtx, complex_dmtx, 1e-14);
 }
 
 
@@ -589,7 +589,7 @@ TEST_F(Csr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
     auto abs_mtx = complex_mtx->compute_absolute();
     auto dabs_mtx = complex_dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index 77781b71aab..b9af3ec1027 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -709,7 +709,7 @@ TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
     x->compute_absolute_inplace();
     dx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(x.get(), dx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(x, dx, 1e-14);
 }
 
 
@@ -720,7 +720,7 @@ TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_x = x->compute_absolute();
     auto dabs_x = dx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_x.get(), dabs_x.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, 1e-14);
 }
 
 
diff --git a/omp/test/matrix/diagonal_kernels.cpp b/omp/test/matrix/diagonal_kernels.cpp
index e58380ea773..02ddd354241 100644
--- a/omp/test/matrix/diagonal_kernels.cpp
+++ b/omp/test/matrix/diagonal_kernels.cpp
@@ -256,7 +256,7 @@ TEST_F(Diagonal, InplaceAbsoluteMatrixIsEquivalentToRef)
     diag->compute_absolute_inplace();
     ddiag->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(diag, ddiag, 1e-14);
 }
 
 
@@ -267,7 +267,7 @@ TEST_F(Diagonal, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_diag = diag->compute_absolute();
     auto dabs_diag = ddiag->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_diag.get(), dabs_diag.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_diag, dabs_diag, 1e-14);
 }
 
 
diff --git a/omp/test/matrix/ell_kernels.cpp b/omp/test/matrix/ell_kernels.cpp
index 06195abe138..768cb87822e 100644
--- a/omp/test/matrix/ell_kernels.cpp
+++ b/omp/test/matrix/ell_kernels.cpp
@@ -239,7 +239,7 @@ TEST_F(Ell, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -250,7 +250,7 @@ TEST_F(Ell, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/omp/test/matrix/hybrid_kernels.cpp b/omp/test/matrix/hybrid_kernels.cpp
index df85f18455f..a57458b344e 100644
--- a/omp/test/matrix/hybrid_kernels.cpp
+++ b/omp/test/matrix/hybrid_kernels.cpp
@@ -237,7 +237,7 @@ TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -248,7 +248,7 @@ TEST_F(Hybrid, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/omp/test/matrix/sellp_kernels.cpp b/omp/test/matrix/sellp_kernels.cpp
index 69780f6071e..e776d9c88c1 100644
--- a/omp/test/matrix/sellp_kernels.cpp
+++ b/omp/test/matrix/sellp_kernels.cpp
@@ -273,7 +273,7 @@ TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx.get(), dmtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
 }
 
 
@@ -284,7 +284,7 @@ TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx.get(), dabs_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 

From 2e2d79ae8d5ad98520ad794e79f895c8fd1ef885 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 16 Sep 2020 18:33:30 +0200
Subject: [PATCH 0019/1008] update document, add hybrid get_strategy<Hyb>, test

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 core/matrix/hybrid.cpp                   |   4 +-
 core/test/base/lin_op.cpp                |   1 +
 core/test/matrix/hybrid.cpp              | 113 +++++++++++++++++++++++
 cuda/test/matrix/hybrid_kernels.cpp      |   9 +-
 hip/test/matrix/hybrid_kernels.hip.cpp   |   9 +-
 include/ginkgo/core/base/math.hpp        |  27 +++++-
 include/ginkgo/core/matrix/csr.hpp       |   6 --
 include/ginkgo/core/matrix/hybrid.hpp    |  79 ++++++++++++++++
 omp/test/matrix/hybrid_kernels.cpp       |   9 +-
 reference/test/matrix/hybrid_kernels.cpp |  13 ++-
 10 files changed, 253 insertions(+), 17 deletions(-)

diff --git a/core/matrix/hybrid.cpp b/core/matrix/hybrid.cpp
index b4e83f3906e..92768a09f89 100644
--- a/core/matrix/hybrid.cpp
+++ b/core/matrix/hybrid.cpp
@@ -308,8 +308,8 @@ Hybrid<ValueType, IndexType>::compute_absolute() const
 {
     auto exec = this->get_executor();
 
-    // use default strategy
-    auto abs_hybrid = absolute_type::create(exec, this->get_size());
+    auto abs_hybrid = absolute_type::create(
+        exec, this->get_size(), this->get_strategy<absolute_type>());
 
     abs_hybrid->ell_->copy_from(ell_->compute_absolute());
     abs_hybrid->coo_->copy_from(coo_->compute_absolute());
diff --git a/core/test/base/lin_op.cpp b/core/test/base/lin_op.cpp
index 7ba5c794945..8cb0c8855eb 100644
--- a/core/test/base/lin_op.cpp
+++ b/core/test/base/lin_op.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <complex>
+#include <memory>
 
 
 #include <gtest/gtest.h>
diff --git a/core/test/matrix/hybrid.cpp b/core/test/matrix/hybrid.cpp
index dac9da86167..9045e950a23 100644
--- a/core/test/matrix/hybrid.cpp
+++ b/core/test/matrix/hybrid.cpp
@@ -42,6 +42,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
+template <typename T>
+struct change_index_s {
+    using type = gko::int32;
+};
+
+template <>
+struct change_index_s<gko::int32> {
+    using type = gko::int64;
+};
+
+
+template <typename T>
+using change_index = typename change_index_s<T>::type;
+
+
 template <typename ValueIndexType>
 class Hybrid : public ::testing::Test {
 protected:
@@ -297,4 +312,102 @@ TYPED_TEST(Hybrid, GeneratesCorrectMatrixData)
 }
 
 
+TYPED_TEST(Hybrid, GetCorrectColumnLimit)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Mtx2 = gko::remove_complex<Mtx>;
+    using strategy = typename Mtx::column_limit;
+    using strategy2 = typename Mtx2::column_limit;
+
+    auto mtx = Mtx::create(this->exec, std::make_shared<strategy>(2));
+    auto mtx_stra = gko::as<strategy>(mtx->get_strategy());
+    auto mtx2_stra = gko::as<strategy2>(mtx->template get_strategy<Mtx2>());
+
+    EXPECT_EQ(mtx_stra->get_num_columns(), 2);
+    EXPECT_EQ(mtx2_stra->get_num_columns(), 2);
+}
+
+
+TYPED_TEST(Hybrid, GetCorrectImbalanceLimit)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Mtx2 = gko::remove_complex<Mtx>;
+    using strategy = typename Mtx::imbalance_limit;
+    using strategy2 = typename Mtx2::imbalance_limit;
+
+    auto mtx = Mtx::create(this->exec, std::make_shared<strategy>(0.4));
+    auto mtx_stra = gko::as<strategy>(mtx->get_strategy());
+    auto mtx2_stra = gko::as<strategy2>(mtx->template get_strategy<Mtx2>());
+
+    EXPECT_EQ(mtx_stra->get_percent(), 0.4f);
+    EXPECT_EQ(mtx2_stra->get_percent(), 0.4f);
+}
+
+
+TYPED_TEST(Hybrid, GetCorrectImbalanceBoundedLimit)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Mtx2 = gko::remove_complex<Mtx>;
+    using strategy = typename Mtx::imbalance_bounded_limit;
+    using strategy2 = typename Mtx2::imbalance_bounded_limit;
+
+    auto mtx = Mtx::create(this->exec, std::make_shared<strategy>(0.4, 0.1));
+    auto mtx_stra = gko::as<strategy>(mtx->get_strategy());
+    auto mtx2_stra = gko::as<strategy2>(mtx->template get_strategy<Mtx2>());
+
+    EXPECT_EQ(mtx_stra->get_percent(), 0.4f);
+    EXPECT_EQ(mtx_stra->get_ratio(), 0.1f);
+    EXPECT_EQ(mtx2_stra->get_percent(), 0.4f);
+    EXPECT_EQ(mtx2_stra->get_ratio(), 0.1f);
+}
+
+
+TYPED_TEST(Hybrid, GetCorrectMinimalStorageLimitWithDifferentHybType)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using Mtx2 = gko::matrix::Hybrid<value_type, change_index<index_type>>;
+    using strategy = typename Mtx::minimal_storage_limit;
+    using strategy2 = typename Mtx2::imbalance_limit;
+
+    auto mtx = Mtx::create(this->exec, std::make_shared<strategy>());
+    auto mtx_stra = gko::as<strategy>(mtx->get_strategy());
+    auto mtx2_stra = gko::as<strategy2>(mtx->template get_strategy<Mtx2>());
+
+    EXPECT_EQ(mtx2_stra->get_percent(), mtx_stra->get_percent());
+}
+
+
+TYPED_TEST(Hybrid, GetCorrectMinimalStorageLimitWithSameHybType)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Mtx2 = Mtx;
+    using strategy = typename Mtx::minimal_storage_limit;
+    using strategy2 = typename Mtx2::minimal_storage_limit;
+
+    auto mtx = Mtx::create(this->exec, std::make_shared<strategy>());
+    auto mtx_stra = gko::as<strategy>(mtx->get_strategy());
+    auto mtx2_stra = gko::as<strategy2>(mtx->template get_strategy<Mtx2>());
+
+    EXPECT_EQ(mtx2_stra->get_percent(), mtx_stra->get_percent());
+}
+
+
+TYPED_TEST(Hybrid, GetCorrectAutomatic)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using Mtx2 = Mtx;
+    using strategy = typename Mtx::automatic;
+    using strategy2 = typename Mtx2::automatic;
+
+    auto mtx = Mtx::create(this->exec, std::make_shared<strategy>());
+    auto mtx_stra = gko::as<strategy>(mtx->get_strategy());
+
+    ASSERT_NO_THROW(gko::as<strategy2>(mtx->template get_strategy<Mtx2>()));
+}
+
+
 }  // namespace
diff --git a/cuda/test/matrix/hybrid_kernels.cpp b/cuda/test/matrix/hybrid_kernels.cpp
index 7bcc7ad47b3..366686f27a0 100644
--- a/cuda/test/matrix/hybrid_kernels.cpp
+++ b/cuda/test/matrix/hybrid_kernels.cpp
@@ -244,12 +244,19 @@ TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
 
 TEST_F(Hybrid, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_data(1, std::make_shared<Mtx::column_limit>(2));
+    using AbsMtx = gko::remove_complex<Mtx>;
 
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
+    auto abs_strategy = gko::as<AbsMtx::column_limit>(abs_mtx->get_strategy());
+    auto dabs_strategy =
+        gko::as<AbsMtx::column_limit>(dabs_mtx->get_strategy());
 
     GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+    GKO_ASSERT_EQ(abs_strategy->get_num_columns(),
+                  dabs_strategy->get_num_columns());
+    GKO_ASSERT_EQ(abs_strategy->get_num_columns(), 2);
 }
 
 
diff --git a/hip/test/matrix/hybrid_kernels.hip.cpp b/hip/test/matrix/hybrid_kernels.hip.cpp
index 5aa3b2d0258..f7c60b59b3f 100644
--- a/hip/test/matrix/hybrid_kernels.hip.cpp
+++ b/hip/test/matrix/hybrid_kernels.hip.cpp
@@ -244,12 +244,19 @@ TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
 
 TEST_F(Hybrid, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_data(1, std::make_shared<Mtx::column_limit>(2));
+    using AbsMtx = gko::remove_complex<Mtx>;
 
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
+    auto abs_strategy = gko::as<AbsMtx::column_limit>(abs_mtx->get_strategy());
+    auto dabs_strategy =
+        gko::as<AbsMtx::column_limit>(dabs_mtx->get_strategy());
 
     GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+    GKO_ASSERT_EQ(abs_strategy->get_num_columns(),
+                  dabs_strategy->get_num_columns());
+    GKO_ASSERT_EQ(abs_strategy->get_num_columns(), 2);
 }
 
 
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index dc222110fb4..467d8668bd3 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -201,6 +201,8 @@ struct remove_complex_s {};
 /**
  * Obtains a real counterpart of a std::complex type, and leaves the type
  * unchanged if it is not a complex type for complex/scalar type.
+ *
+ * @tparam T  complex or scalar type
  */
 template <typename T>
 struct remove_complex_s<
@@ -211,6 +213,8 @@ struct remove_complex_s<
 /**
  * Obtains a real counterpart of a class with template parameters, which
  * converts complex parameters to real parameters.
+ *
+ * @tparam T  class with template parameters
  */
 template <typename T>
 struct remove_complex_s<
@@ -227,6 +231,8 @@ struct add_complex_s {};
 /**
  * Obtains a complex counterpart of a real type, and leaves the type
  * unchanged if it is a complex type for complex/scalar type.
+ *
+ * @tparam T  complex or scalar type
  */
 template <typename T>
 struct add_complex_s<
@@ -237,6 +243,8 @@ struct add_complex_s<
 /**
  * Obtains a complex counterpart of a class with template parameters, which
  * converts real parameters to complex parameters.
+ *
+ * @tparam T  class with template parameters
  */
 template <typename T>
 struct add_complex_s<
@@ -334,24 +342,35 @@ using detail::remove_complex_s;
  * template parameter of class by accessing the `type` attribute of this struct.
  *
  * @tparam T  type to remove complex
+ *
+ * @note remove_complex<class> can not be used in friend class declaration.
  */
 template <typename T>
 using remove_complex = typename remove_complex_s<T>::type;
 
 
 /**
- * Removes the complex of complex/scalar type or the template parameter of class
+ * Add the complex of complex/scalar type or the template parameter of class
  * by accessing the `type` attribute of this struct.
  *
- * @tparam T  type to remove complex
+ * @tparam T  type to add complex
  */
 using detail::add_complex_s;
 
 /**
- * Obtain the type which removed the complex of complex/scalar type or the
+ * Obtain the type which adds the complex of complex/scalar type or the
  * template parameter of class by accessing the `type` attribute of this struct.
  *
- * @tparam T  type to remove complex
+ * @tparam T  type to add complex
+ *
+ * @note add_complex<class> can not be used in friend class declaration.
+ *       the followings are the error message from different combination.
+ *       friend add_complex<Csr>;
+ *         error: can not recognize it is class correctly.
+ *       friend class add_complex<Csr>;
+ *         error: using alias template specialization
+ *       friend class add_complex_s<Csr<ValueType,IndexType>>::type;
+ *         error: can not recognize it is class correctly.
  */
 template <typename T>
 using add_complex = typename add_complex_s<T>::type;
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index b71b194e6f3..4e44b7cd4e2 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -142,12 +142,6 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
     friend class SparsityCsr<ValueType, IndexType>;
     friend class CsrBuilder<ValueType, IndexType>;
     friend class Csr<add_complex<ValueType>, IndexType>;
-    // friend add_complex<Csr>;
-    //   error: can not recognize it is class correctly.
-    // friend class add_complex<Csr>;
-    //   error: using alias template specialization
-    // friend class add_complex_s<Csr<ValueType,IndexType>>::type;
-    //   error: can not recognize it is class correctly.
 
 public:
     using value_type = ValueType;
diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp
index 453be83cdea..684b3c003e9 100644
--- a/include/ginkgo/core/matrix/hybrid.hpp
+++ b/include/ginkgo/core/matrix/hybrid.hpp
@@ -218,6 +218,13 @@ class Hybrid
             return num_columns_;
         }
 
+        /**
+         * Get the number of columns limit
+         *
+         * @return the number of columns limit
+         */
+        auto get_num_columns() const { return num_columns_; }
+
     private:
         size_type num_columns_;
     };
@@ -260,6 +267,13 @@ class Hybrid
             }
         }
 
+        /**
+         * Get the percent setting
+         *
+         * @retrun percent
+         */
+        auto get_percent() const { return percent_; }
+
     private:
         float percent_;
     };
@@ -288,6 +302,20 @@ class Hybrid
                             static_cast<size_type>(num_rows * ratio_));
         }
 
+        /**
+         * Get the percent setting
+         *
+         * @retrun percent
+         */
+        auto get_percent() const { return strategy_.get_percent(); }
+
+        /**
+         * Get the ratio setting
+         *
+         * @retrun ratio
+         */
+        auto get_ratio() const { return ratio_; }
+
     private:
         imbalance_limit strategy_;
         float ratio_;
@@ -316,6 +344,13 @@ class Hybrid
             return strategy_.compute_ell_num_stored_elements_per_row(row_nnz);
         }
 
+        /**
+         * Get the percent setting
+         *
+         * @retrun percent
+         */
+        auto get_percent() { return strategy_.get_percent(); }
+
     private:
         imbalance_limit strategy_;
     };
@@ -581,6 +616,50 @@ class Hybrid
         return strategy_;
     }
 
+    /**
+     * Returns the current strategy allowed in given hybrid format
+     *
+     * @tparam HybType  hybrid type
+     *
+     * @return the strategy
+     */
+    template <typename HybType>
+    std::shared_ptr<typename HybType::strategy_type> get_strategy() const
+    {
+        std::shared_ptr<typename HybType::strategy_type> strategy;
+        if (std::dynamic_pointer_cast<automatic>(strategy_)) {
+            strategy = std::make_shared<typename HybType::automatic>();
+        } else if (auto temp = std::dynamic_pointer_cast<minimal_storage_limit>(
+                       strategy_)) {
+            // minimal_storage_limit is related to ValueType and IndexType size.
+            if (sizeof(value_type) == sizeof(typename HybType::value_type) &&
+                sizeof(index_type) == sizeof(typename HybType::index_type)) {
+                strategy =
+                    std::make_shared<typename HybType::minimal_storage_limit>();
+            } else {
+                strategy = std::make_shared<typename HybType::imbalance_limit>(
+                    temp->get_percent());
+            }
+        } else if (auto temp =
+                       std::dynamic_pointer_cast<imbalance_bounded_limit>(
+                           strategy_)) {
+            strategy =
+                std::make_shared<typename HybType::imbalance_bounded_limit>(
+                    temp->get_percent(), temp->get_ratio());
+        } else if (auto temp =
+                       std::dynamic_pointer_cast<imbalance_limit>(strategy_)) {
+            strategy = std::make_shared<typename HybType::imbalance_limit>(
+                temp->get_percent());
+        } else if (auto temp =
+                       std::dynamic_pointer_cast<column_limit>(strategy_)) {
+            strategy = std::make_shared<typename HybType::column_limit>(
+                temp->get_num_columns());
+        } else {
+            GKO_NOT_SUPPORTED(strategy_);
+        }
+        return strategy;
+    }
+
     /**
      * Copies data from another Hybrid.
      *
diff --git a/omp/test/matrix/hybrid_kernels.cpp b/omp/test/matrix/hybrid_kernels.cpp
index a57458b344e..948c78060ca 100644
--- a/omp/test/matrix/hybrid_kernels.cpp
+++ b/omp/test/matrix/hybrid_kernels.cpp
@@ -243,12 +243,19 @@ TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
 
 TEST_F(Hybrid, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_data(1, std::make_shared<Mtx::column_limit>(2));
+    using AbsMtx = gko::remove_complex<Mtx>;
 
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
+    auto abs_strategy = gko::as<AbsMtx::column_limit>(abs_mtx->get_strategy());
+    auto dabs_strategy =
+        gko::as<AbsMtx::column_limit>(dabs_mtx->get_strategy());
 
     GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+    GKO_ASSERT_EQ(abs_strategy->get_num_columns(),
+                  dabs_strategy->get_num_columns());
+    GKO_ASSERT_EQ(abs_strategy->get_num_columns(), 2);
 }
 
 
diff --git a/reference/test/matrix/hybrid_kernels.cpp b/reference/test/matrix/hybrid_kernels.cpp
index 46123df227b..57125661cb9 100644
--- a/reference/test/matrix/hybrid_kernels.cpp
+++ b/reference/test/matrix/hybrid_kernels.cpp
@@ -673,12 +673,17 @@ TYPED_TEST(Hybrid, OutplaceAbsolute)
 {
     using Mtx = typename TestFixture::Mtx;
     auto mtx = gko::initialize<Mtx>(
-        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
+        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec,
+        std::make_shared<typename Mtx::column_limit>(2));
 
     auto abs_mtx = mtx->compute_absolute();
+    auto abs_strategy =
+        gko::as<typename gko::remove_complex<Mtx>::column_limit>(
+            abs_mtx->get_strategy());
 
     GKO_ASSERT_MTX_NEAR(
         abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+    GKO_ASSERT_EQ(abs_strategy->get_num_columns(), 2);
 }
 
 
@@ -705,13 +710,17 @@ TYPED_TEST(HybridComplex, OutplaceAbsolute)
     auto mtx = gko::initialize<Mtx>(
         {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
          {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
-         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec, std::make_shared<typename Mtx::column_limit>(2));
     // clang-format on
 
     auto abs_mtx = mtx->compute_absolute();
+    auto abs_strategy =
+        gko::as<typename gko::remove_complex<Mtx>::column_limit>(
+            abs_mtx->get_strategy());
 
     GKO_ASSERT_MTX_NEAR(
         abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+    GKO_ASSERT_EQ(abs_strategy->get_num_columns(), 2);
 }
 
 

From 7233fff0cde88ebe4b1b9567b5829934c7e7c234 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 17 Sep 2020 12:44:19 +0200
Subject: [PATCH 0020/1008] use absoluteLinOp template not concreteLinOp

---
 core/test/base/lin_op.cpp               |  3 +-
 include/ginkgo/core/base/lin_op.hpp     | 15 ++++---
 include/ginkgo/core/matrix/coo.hpp      |  6 +--
 include/ginkgo/core/matrix/csr.hpp      |  6 +--
 include/ginkgo/core/matrix/dense.hpp    | 52 ++++++++++++-------------
 include/ginkgo/core/matrix/diagonal.hpp | 24 ++++++------
 include/ginkgo/core/matrix/ell.hpp      |  6 +--
 include/ginkgo/core/matrix/hybrid.hpp   |  6 +--
 include/ginkgo/core/matrix/sellp.hpp    |  6 +--
 9 files changed, 62 insertions(+), 62 deletions(-)

diff --git a/core/test/base/lin_op.cpp b/core/test/base/lin_op.cpp
index 8cb0c8855eb..e73efa5e923 100644
--- a/core/test/base/lin_op.cpp
+++ b/core/test/base/lin_op.cpp
@@ -325,7 +325,8 @@ template <typename Type>
 class DummyLinOpWithType
     : public gko::EnableLinOp<DummyLinOpWithType<Type>>,
       public gko::EnableCreateMethod<DummyLinOpWithType<Type>>,
-      public gko::EnableAbsoluteComputation<DummyLinOpWithType<Type>> {
+      public gko::EnableAbsoluteComputation<
+          gko::remove_complex<DummyLinOpWithType<Type>>> {
 public:
     using absolute_type = gko::remove_complex<DummyLinOpWithType>;
     DummyLinOpWithType(std::shared_ptr<const gko::Executor> exec)
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index 5be7dadcded..6c873198b6e 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -626,7 +626,7 @@ class DiagonalExtractable {
 
 /**
  * The AbsoluteComputable is an interface that allows to get the component wise
- * absolute of a LinOp. Use EnableAbsoluteComputation<ConcreteLinOp> to
+ * absolute of a LinOp. Use EnableAbsoluteComputation<AbsoluteLinOp> to
  * implement this interface.
  */
 class AbsoluteComputable {
@@ -648,19 +648,18 @@ class AbsoluteComputable {
 /**
  * The EnableAbsoluteComputation mixin provides the default implementations of
  * `compute_absolute_linop` and the absolute interface. `compute_absolute` gets
- * a new remove_complex<ConcreteLinOp> which removes the complex value_type by
- * applying outplace absolute. `compute_absolute_inplace` applies absolute
+ * a new AbsoluteLinOp. `compute_absolute_inplace` applies absolute
  * inplace, so it still keeps the value_type of the class.
  *
- * @tparam ConcreteLinOp  the concrete LinOp which is being implemented
+ * @tparam AbsoluteLinOp  the absolute LinOp which is being returned
  *                        [CRTP parameter]
  *
  * @ingroup LinOp
  */
-template <typename ConcreteLinOp>
+template <typename AbsoluteLinOp>
 class EnableAbsoluteComputation : public AbsoluteComputable {
 public:
-    using outplace_type = remove_complex<ConcreteLinOp>;
+    using absolute_type = AbsoluteLinOp;
 
     virtual ~EnableAbsoluteComputation() = default;
 
@@ -670,11 +669,11 @@ class EnableAbsoluteComputation : public AbsoluteComputable {
     }
 
     /**
-     * Gets the absolute ConcreteLinOp
+     * Gets the AbsoluteLinOp
      *
      * @return a pointer to the new absolute object
      */
-    virtual std::unique_ptr<outplace_type> compute_absolute() const = 0;
+    virtual std::unique_ptr<absolute_type> compute_absolute() const = 0;
 };
 
 
diff --git a/include/ginkgo/core/matrix/coo.hpp b/include/ginkgo/core/matrix/coo.hpp
index 6a9b972e7ef..af7b75e1233 100644
--- a/include/ginkgo/core/matrix/coo.hpp
+++ b/include/ginkgo/core/matrix/coo.hpp
@@ -82,7 +82,8 @@ class Coo : public EnableLinOp<Coo<ValueType, IndexType>>,
             public DiagonalExtractable<ValueType>,
             public ReadableFromMatrixData<ValueType, IndexType>,
             public WritableToMatrixData<ValueType, IndexType>,
-            public EnableAbsoluteComputation<Coo<ValueType, IndexType>> {
+            public EnableAbsoluteComputation<
+                remove_complex<Coo<ValueType, IndexType>>> {
     friend class EnableCreateMethod<Coo>;
     friend class EnablePolymorphicObject<Coo, LinOp>;
     friend class Csr<ValueType, IndexType>;
@@ -97,8 +98,7 @@ class Coo : public EnableLinOp<Coo<ValueType, IndexType>>,
     using value_type = ValueType;
     using index_type = IndexType;
     using mat_data = matrix_data<ValueType, IndexType>;
-    using absolute_type =
-        typename EnableAbsoluteComputation<Coo>::outplace_type;
+    using absolute_type = remove_complex<Coo>;
 
     friend class Coo<next_precision<ValueType>, IndexType>;
 
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index 4e44b7cd4e2..659a47aa05f 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -131,7 +131,8 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
             public WritableToMatrixData<ValueType, IndexType>,
             public Transposable,
             public Permutable<IndexType>,
-            public EnableAbsoluteComputation<Csr<ValueType, IndexType>> {
+            public EnableAbsoluteComputation<
+                remove_complex<Csr<ValueType, IndexType>>> {
     friend class EnableCreateMethod<Csr>;
     friend class EnablePolymorphicObject<Csr, LinOp>;
     friend class Coo<ValueType, IndexType>;
@@ -148,8 +149,7 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
     using index_type = IndexType;
     using transposed_type = Csr<ValueType, IndexType>;
     using mat_data = matrix_data<ValueType, IndexType>;
-    using absolute_type =
-        typename EnableAbsoluteComputation<Csr>::outplace_type;
+    using absolute_type = remove_complex<Csr>;
 
     class automatical;
 
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index b0b1aa9712c..f639f0799ed 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -88,30 +88,31 @@ class SparsityCsr;
  * @ingroup LinOp
  */
 template <typename ValueType = default_precision>
-class Dense : public EnableLinOp<Dense<ValueType>>,
-              public EnableCreateMethod<Dense<ValueType>>,
-              public ConvertibleTo<Dense<next_precision<ValueType>>>,
-              public ConvertibleTo<Coo<ValueType, int32>>,
-              public ConvertibleTo<Coo<ValueType, int64>>,
-              public ConvertibleTo<Csr<ValueType, int32>>,
-              public ConvertibleTo<Csr<ValueType, int64>>,
-              public ConvertibleTo<Ell<ValueType, int32>>,
-              public ConvertibleTo<Ell<ValueType, int64>>,
-              public ConvertibleTo<Hybrid<ValueType, int32>>,
-              public ConvertibleTo<Hybrid<ValueType, int64>>,
-              public ConvertibleTo<Sellp<ValueType, int32>>,
-              public ConvertibleTo<Sellp<ValueType, int64>>,
-              public ConvertibleTo<SparsityCsr<ValueType, int32>>,
-              public ConvertibleTo<SparsityCsr<ValueType, int64>>,
-              public DiagonalExtractable<ValueType>,
-              public ReadableFromMatrixData<ValueType, int32>,
-              public ReadableFromMatrixData<ValueType, int64>,
-              public WritableToMatrixData<ValueType, int32>,
-              public WritableToMatrixData<ValueType, int64>,
-              public Transposable,
-              public Permutable<int32>,
-              public Permutable<int64>,
-              public EnableAbsoluteComputation<Dense<ValueType>> {
+class Dense
+    : public EnableLinOp<Dense<ValueType>>,
+      public EnableCreateMethod<Dense<ValueType>>,
+      public ConvertibleTo<Dense<next_precision<ValueType>>>,
+      public ConvertibleTo<Coo<ValueType, int32>>,
+      public ConvertibleTo<Coo<ValueType, int64>>,
+      public ConvertibleTo<Csr<ValueType, int32>>,
+      public ConvertibleTo<Csr<ValueType, int64>>,
+      public ConvertibleTo<Ell<ValueType, int32>>,
+      public ConvertibleTo<Ell<ValueType, int64>>,
+      public ConvertibleTo<Hybrid<ValueType, int32>>,
+      public ConvertibleTo<Hybrid<ValueType, int64>>,
+      public ConvertibleTo<Sellp<ValueType, int32>>,
+      public ConvertibleTo<Sellp<ValueType, int64>>,
+      public ConvertibleTo<SparsityCsr<ValueType, int32>>,
+      public ConvertibleTo<SparsityCsr<ValueType, int64>>,
+      public DiagonalExtractable<ValueType>,
+      public ReadableFromMatrixData<ValueType, int32>,
+      public ReadableFromMatrixData<ValueType, int64>,
+      public WritableToMatrixData<ValueType, int32>,
+      public WritableToMatrixData<ValueType, int64>,
+      public Transposable,
+      public Permutable<int32>,
+      public Permutable<int64>,
+      public EnableAbsoluteComputation<remove_complex<Dense<ValueType>>> {
     friend class EnableCreateMethod<Dense>;
     friend class EnablePolymorphicObject<Dense, LinOp>;
     friend class Coo<ValueType, int32>;
@@ -138,8 +139,7 @@ class Dense : public EnableLinOp<Dense<ValueType>>,
     using transposed_type = Dense<ValueType>;
     using mat_data = gko::matrix_data<ValueType, int64>;
     using mat_data32 = gko::matrix_data<ValueType, int32>;
-    using absolute_type =
-        typename EnableAbsoluteComputation<Dense>::outplace_type;
+    using absolute_type = remove_complex<Dense>;
 
     using row_major_range = gko::range<gko::accessor::row_major<ValueType, 2>>;
 
diff --git a/include/ginkgo/core/matrix/diagonal.hpp b/include/ginkgo/core/matrix/diagonal.hpp
index f9e42fb769f..f34a6488d09 100644
--- a/include/ginkgo/core/matrix/diagonal.hpp
+++ b/include/ginkgo/core/matrix/diagonal.hpp
@@ -64,16 +64,17 @@ class Dense;
  * @ingroup LinOp
  */
 template <typename ValueType = default_precision>
-class Diagonal : public EnableLinOp<Diagonal<ValueType>>,
-                 public EnableCreateMethod<Diagonal<ValueType>>,
-                 public ConvertibleTo<Csr<ValueType, int32>>,
-                 public ConvertibleTo<Csr<ValueType, int64>>,
-                 public Transposable,
-                 public WritableToMatrixData<ValueType, int32>,
-                 public WritableToMatrixData<ValueType, int64>,
-                 public ReadableFromMatrixData<ValueType, int32>,
-                 public ReadableFromMatrixData<ValueType, int64>,
-                 public EnableAbsoluteComputation<Diagonal<ValueType>> {
+class Diagonal
+    : public EnableLinOp<Diagonal<ValueType>>,
+      public EnableCreateMethod<Diagonal<ValueType>>,
+      public ConvertibleTo<Csr<ValueType, int32>>,
+      public ConvertibleTo<Csr<ValueType, int64>>,
+      public Transposable,
+      public WritableToMatrixData<ValueType, int32>,
+      public WritableToMatrixData<ValueType, int64>,
+      public ReadableFromMatrixData<ValueType, int32>,
+      public ReadableFromMatrixData<ValueType, int64>,
+      public EnableAbsoluteComputation<remove_complex<Diagonal<ValueType>>> {
     friend class EnablePolymorphicObject<Diagonal, LinOp>;
     friend class EnableCreateMethod<Diagonal>;
     friend class Csr<ValueType, int32>;
@@ -88,8 +89,7 @@ class Diagonal : public EnableLinOp<Diagonal<ValueType>>,
     using index_type = int64;
     using mat_data = gko::matrix_data<ValueType, int64>;
     using mat_data32 = gko::matrix_data<ValueType, int32>;
-    using absolute_type =
-        typename EnableAbsoluteComputation<Diagonal>::outplace_type;
+    using absolute_type = remove_complex<Diagonal>;
 
     std::unique_ptr<LinOp> transpose() const override;
 
diff --git a/include/ginkgo/core/matrix/ell.hpp b/include/ginkgo/core/matrix/ell.hpp
index 5fdfeb23770..d0ee6f80906 100644
--- a/include/ginkgo/core/matrix/ell.hpp
+++ b/include/ginkgo/core/matrix/ell.hpp
@@ -76,7 +76,8 @@ class Ell : public EnableLinOp<Ell<ValueType, IndexType>>,
             public DiagonalExtractable<ValueType>,
             public ReadableFromMatrixData<ValueType, IndexType>,
             public WritableToMatrixData<ValueType, IndexType>,
-            public EnableAbsoluteComputation<Ell<ValueType, IndexType>> {
+            public EnableAbsoluteComputation<
+                remove_complex<Ell<ValueType, IndexType>>> {
     friend class EnableCreateMethod<Ell>;
     friend class EnablePolymorphicObject<Ell, LinOp>;
     friend class Dense<ValueType>;
@@ -90,8 +91,7 @@ class Ell : public EnableLinOp<Ell<ValueType, IndexType>>,
     using value_type = ValueType;
     using index_type = IndexType;
     using mat_data = matrix_data<ValueType, IndexType>;
-    using absolute_type =
-        typename EnableAbsoluteComputation<Ell>::outplace_type;
+    using absolute_type = remove_complex<Ell>;
 
     friend class Ell<next_precision<ValueType>, IndexType>;
 
diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp
index 684b3c003e9..d53089d128a 100644
--- a/include/ginkgo/core/matrix/hybrid.hpp
+++ b/include/ginkgo/core/matrix/hybrid.hpp
@@ -77,7 +77,8 @@ class Hybrid
       public DiagonalExtractable<ValueType>,
       public ReadableFromMatrixData<ValueType, IndexType>,
       public WritableToMatrixData<ValueType, IndexType>,
-      public EnableAbsoluteComputation<Hybrid<ValueType, IndexType>> {
+      public EnableAbsoluteComputation<
+          remove_complex<Hybrid<ValueType, IndexType>>> {
     friend class EnableCreateMethod<Hybrid>;
     friend class EnablePolymorphicObject<Hybrid, LinOp>;
     friend class Dense<ValueType>;
@@ -94,8 +95,7 @@ class Hybrid
     using mat_data = matrix_data<ValueType, IndexType>;
     using coo_type = Coo<ValueType, IndexType>;
     using ell_type = Ell<ValueType, IndexType>;
-    using absolute_type =
-        typename EnableAbsoluteComputation<Hybrid>::outplace_type;
+    using absolute_type = remove_complex<Hybrid>;
 
 
     /**
diff --git a/include/ginkgo/core/matrix/sellp.hpp b/include/ginkgo/core/matrix/sellp.hpp
index cbf06478bc6..2e8fe4623b9 100644
--- a/include/ginkgo/core/matrix/sellp.hpp
+++ b/include/ginkgo/core/matrix/sellp.hpp
@@ -73,7 +73,8 @@ class Sellp : public EnableLinOp<Sellp<ValueType, IndexType>>,
               public DiagonalExtractable<ValueType>,
               public ReadableFromMatrixData<ValueType, IndexType>,
               public WritableToMatrixData<ValueType, IndexType>,
-              public EnableAbsoluteComputation<Sellp<ValueType, IndexType>> {
+              public EnableAbsoluteComputation<
+                  remove_complex<Sellp<ValueType, IndexType>>> {
     friend class EnableCreateMethod<Sellp>;
     friend class EnablePolymorphicObject<Sellp, LinOp>;
     friend class Dense<ValueType>;
@@ -87,8 +88,7 @@ class Sellp : public EnableLinOp<Sellp<ValueType, IndexType>>,
     using value_type = ValueType;
     using index_type = IndexType;
     using mat_data = matrix_data<ValueType, IndexType>;
-    using absolute_type =
-        typename EnableAbsoluteComputation<Sellp>::outplace_type;
+    using absolute_type = remove_complex<Sellp>;
 
     friend class Sellp<next_precision<ValueType>, IndexType>;
 

From ad34c1ccab18be9a8d7fe2fbb19c456804c0b84c Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 17 Sep 2020 14:11:53 +0200
Subject: [PATCH 0021/1008] to_complex/real and remove_complex only

---
 include/ginkgo/core/base/math.hpp       | 47 ++++++++-----------------
 include/ginkgo/core/matrix/coo.hpp      |  2 +-
 include/ginkgo/core/matrix/csr.hpp      |  2 +-
 include/ginkgo/core/matrix/dense.hpp    |  2 +-
 include/ginkgo/core/matrix/diagonal.hpp |  2 +-
 include/ginkgo/core/matrix/ell.hpp      |  2 +-
 include/ginkgo/core/matrix/hybrid.hpp   |  2 +-
 include/ginkgo/core/matrix/sellp.hpp    |  2 +-
 8 files changed, 22 insertions(+), 39 deletions(-)

diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 467d8668bd3..386f0bf737d 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -139,7 +139,7 @@ struct remove_complex_impl<std::complex<T>> {
  * @tparam T  the type being made complex
  */
 template <typename T>
-struct add_complex_impl {
+struct to_complex_impl {
     using type = std::complex<T>;
 };
 
@@ -149,7 +149,7 @@ struct add_complex_impl {
  * @tparam T  the type being made complex
  */
 template <typename T>
-struct add_complex_impl<std::complex<T>> {
+struct to_complex_impl<std::complex<T>> {
     using type = std::complex<T>;
 };
 
@@ -226,7 +226,7 @@ struct remove_complex_s<
 
 
 template <typename T, typename = void>
-struct add_complex_s {};
+struct to_complex_s {};
 
 /**
  * Obtains a complex counterpart of a real type, and leaves the type
@@ -235,9 +235,9 @@ struct add_complex_s {};
  * @tparam T  complex or scalar type
  */
 template <typename T>
-struct add_complex_s<
+struct to_complex_s<
     T, xstd::void_t<std::enable_if_t<is_complex_or_scalar_impl<T>::value>>> {
-    using type = typename detail::add_complex_impl<T>::type;
+    using type = typename detail::to_complex_impl<T>::type;
 };
 
 /**
@@ -247,10 +247,10 @@ struct add_complex_s<
  * @tparam T  class with template parameters
  */
 template <typename T>
-struct add_complex_s<
+struct to_complex_s<
     T, xstd::void_t<std::enable_if_t<!is_complex_or_scalar_impl<T>::value>>> {
     using type =
-        typename detail::template_convertor<detail::add_complex_impl, T>::type;
+        typename detail::template_convertor<detail::to_complex_impl, T>::type;
 };
 
 
@@ -353,27 +353,27 @@ using remove_complex = typename remove_complex_s<T>::type;
  * Add the complex of complex/scalar type or the template parameter of class
  * by accessing the `type` attribute of this struct.
  *
- * @tparam T  type to add complex
+ * @tparam T  type to complex_type
  */
-using detail::add_complex_s;
+using detail::to_complex_s;
 
 /**
  * Obtain the type which adds the complex of complex/scalar type or the
  * template parameter of class by accessing the `type` attribute of this struct.
  *
- * @tparam T  type to add complex
+ * @tparam T  type to complex_type
  *
- * @note add_complex<class> can not be used in friend class declaration.
+ * @note to_complex<class> can not be used in friend class declaration.
  *       the followings are the error message from different combination.
- *       friend add_complex<Csr>;
+ *       friend to_complex<Csr>;
  *         error: can not recognize it is class correctly.
- *       friend class add_complex<Csr>;
+ *       friend class to_complex<Csr>;
  *         error: using alias template specialization
- *       friend class add_complex_s<Csr<ValueType,IndexType>>::type;
+ *       friend class to_complex_s<Csr<ValueType,IndexType>>::type;
  *         error: can not recognize it is class correctly.
  */
 template <typename T>
-using add_complex = typename add_complex_s<T>::type;
+using to_complex = typename to_complex_s<T>::type;
 
 
 /**
@@ -393,23 +393,6 @@ template <typename T>
 using to_real = remove_complex<T>;
 
 
-/**
- * to_complex_s is alias of add_complex_s
- *
- * @tparam T  type to complex
- */
-template <typename T>
-using to_complex_s = add_complex_s<T>;
-
-/**
- * to_complex is alias of add_complex
- *
- * @tparam T  type to complex
- */
-template <typename T>
-using to_complex = add_complex<T>;
-
-
 namespace detail {
 
 
diff --git a/include/ginkgo/core/matrix/coo.hpp b/include/ginkgo/core/matrix/coo.hpp
index af7b75e1233..dd048fd5825 100644
--- a/include/ginkgo/core/matrix/coo.hpp
+++ b/include/ginkgo/core/matrix/coo.hpp
@@ -89,7 +89,7 @@ class Coo : public EnableLinOp<Coo<ValueType, IndexType>>,
     friend class Csr<ValueType, IndexType>;
     friend class Dense<ValueType>;
     friend class CooBuilder<ValueType, IndexType>;
-    friend class Coo<add_complex<ValueType>, IndexType>;
+    friend class Coo<to_complex<ValueType>, IndexType>;
 
 public:
     using EnableLinOp<Coo>::convert_to;
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index 659a47aa05f..a30f6f856e3 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -142,7 +142,7 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
     friend class Sellp<ValueType, IndexType>;
     friend class SparsityCsr<ValueType, IndexType>;
     friend class CsrBuilder<ValueType, IndexType>;
-    friend class Csr<add_complex<ValueType>, IndexType>;
+    friend class Csr<to_complex<ValueType>, IndexType>;
 
 public:
     using value_type = ValueType;
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index f639f0799ed..b6ec2421455 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -128,7 +128,7 @@ class Dense
     friend class Sellp<ValueType, int64>;
     friend class SparsityCsr<ValueType, int32>;
     friend class SparsityCsr<ValueType, int64>;
-    friend class Dense<add_complex<ValueType>>;
+    friend class Dense<to_complex<ValueType>>;
 
 public:
     using EnableLinOp<Dense>::convert_to;
diff --git a/include/ginkgo/core/matrix/diagonal.hpp b/include/ginkgo/core/matrix/diagonal.hpp
index f34a6488d09..b268421c163 100644
--- a/include/ginkgo/core/matrix/diagonal.hpp
+++ b/include/ginkgo/core/matrix/diagonal.hpp
@@ -79,7 +79,7 @@ class Diagonal
     friend class EnableCreateMethod<Diagonal>;
     friend class Csr<ValueType, int32>;
     friend class Csr<ValueType, int64>;
-    friend class Diagonal<add_complex<ValueType>>;
+    friend class Diagonal<to_complex<ValueType>>;
 
 public:
     using EnableLinOp<Diagonal>::convert_to;
diff --git a/include/ginkgo/core/matrix/ell.hpp b/include/ginkgo/core/matrix/ell.hpp
index d0ee6f80906..cbcd3552700 100644
--- a/include/ginkgo/core/matrix/ell.hpp
+++ b/include/ginkgo/core/matrix/ell.hpp
@@ -82,7 +82,7 @@ class Ell : public EnableLinOp<Ell<ValueType, IndexType>>,
     friend class EnablePolymorphicObject<Ell, LinOp>;
     friend class Dense<ValueType>;
     friend class Csr<ValueType, IndexType>;
-    friend class Ell<add_complex<ValueType>, IndexType>;
+    friend class Ell<to_complex<ValueType>, IndexType>;
 
 public:
     using EnableLinOp<Ell>::convert_to;
diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp
index d53089d128a..5e782ca8aa2 100644
--- a/include/ginkgo/core/matrix/hybrid.hpp
+++ b/include/ginkgo/core/matrix/hybrid.hpp
@@ -83,7 +83,7 @@ class Hybrid
     friend class EnablePolymorphicObject<Hybrid, LinOp>;
     friend class Dense<ValueType>;
     friend class Csr<ValueType, IndexType>;
-    friend class Hybrid<add_complex<ValueType>, IndexType>;
+    friend class Hybrid<to_complex<ValueType>, IndexType>;
 
 
 public:
diff --git a/include/ginkgo/core/matrix/sellp.hpp b/include/ginkgo/core/matrix/sellp.hpp
index 2e8fe4623b9..e28a5f3a73e 100644
--- a/include/ginkgo/core/matrix/sellp.hpp
+++ b/include/ginkgo/core/matrix/sellp.hpp
@@ -79,7 +79,7 @@ class Sellp : public EnableLinOp<Sellp<ValueType, IndexType>>,
     friend class EnablePolymorphicObject<Sellp, LinOp>;
     friend class Dense<ValueType>;
     friend class Csr<ValueType, IndexType>;
-    friend class Sellp<add_complex<ValueType>, IndexType>;
+    friend class Sellp<to_complex<ValueType>, IndexType>;
 
 public:
     using EnableLinOp<Sellp>::convert_to;

From 34823cd80d98bdd02576c501a43ce588b20c2f19 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 18 Sep 2020 12:42:35 +0200
Subject: [PATCH 0022/1008] split get_strategy<Hyb> impl, decl; check HybType

---
 include/ginkgo/core/matrix/hybrid.hpp | 78 +++++++++++++++------------
 1 file changed, 43 insertions(+), 35 deletions(-)

diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp
index 5e782ca8aa2..3b8653864dc 100644
--- a/include/ginkgo/core/matrix/hybrid.hpp
+++ b/include/ginkgo/core/matrix/hybrid.hpp
@@ -624,41 +624,7 @@ class Hybrid
      * @return the strategy
      */
     template <typename HybType>
-    std::shared_ptr<typename HybType::strategy_type> get_strategy() const
-    {
-        std::shared_ptr<typename HybType::strategy_type> strategy;
-        if (std::dynamic_pointer_cast<automatic>(strategy_)) {
-            strategy = std::make_shared<typename HybType::automatic>();
-        } else if (auto temp = std::dynamic_pointer_cast<minimal_storage_limit>(
-                       strategy_)) {
-            // minimal_storage_limit is related to ValueType and IndexType size.
-            if (sizeof(value_type) == sizeof(typename HybType::value_type) &&
-                sizeof(index_type) == sizeof(typename HybType::index_type)) {
-                strategy =
-                    std::make_shared<typename HybType::minimal_storage_limit>();
-            } else {
-                strategy = std::make_shared<typename HybType::imbalance_limit>(
-                    temp->get_percent());
-            }
-        } else if (auto temp =
-                       std::dynamic_pointer_cast<imbalance_bounded_limit>(
-                           strategy_)) {
-            strategy =
-                std::make_shared<typename HybType::imbalance_bounded_limit>(
-                    temp->get_percent(), temp->get_ratio());
-        } else if (auto temp =
-                       std::dynamic_pointer_cast<imbalance_limit>(strategy_)) {
-            strategy = std::make_shared<typename HybType::imbalance_limit>(
-                temp->get_percent());
-        } else if (auto temp =
-                       std::dynamic_pointer_cast<column_limit>(strategy_)) {
-            strategy = std::make_shared<typename HybType::column_limit>(
-                temp->get_num_columns());
-        } else {
-            GKO_NOT_SUPPORTED(strategy_);
-        }
-        return strategy;
-    }
+    std::shared_ptr<typename HybType::strategy_type> get_strategy() const;
 
     /**
      * Copies data from another Hybrid.
@@ -778,6 +744,48 @@ class Hybrid
 };
 
 
+template <typename ValueType, typename IndexType>
+template <typename HybType>
+std::shared_ptr<typename HybType::strategy_type>
+Hybrid<ValueType, IndexType>::get_strategy() const
+{
+    static_assert(
+        std::is_same<HybType, Hybrid<typename HybType::value_type,
+                                     typename HybType::index_type>>::value,
+        "The given `HybType` type must be of type `matrix::Hybrid`!");
+
+    std::shared_ptr<typename HybType::strategy_type> strategy;
+    if (std::dynamic_pointer_cast<automatic>(strategy_)) {
+        strategy = std::make_shared<typename HybType::automatic>();
+    } else if (auto temp = std::dynamic_pointer_cast<minimal_storage_limit>(
+                   strategy_)) {
+        // minimal_storage_limit is related to ValueType and IndexType size.
+        if (sizeof(value_type) == sizeof(typename HybType::value_type) &&
+            sizeof(index_type) == sizeof(typename HybType::index_type)) {
+            strategy =
+                std::make_shared<typename HybType::minimal_storage_limit>();
+        } else {
+            strategy = std::make_shared<typename HybType::imbalance_limit>(
+                temp->get_percent());
+        }
+    } else if (auto temp = std::dynamic_pointer_cast<imbalance_bounded_limit>(
+                   strategy_)) {
+        strategy = std::make_shared<typename HybType::imbalance_bounded_limit>(
+            temp->get_percent(), temp->get_ratio());
+    } else if (auto temp =
+                   std::dynamic_pointer_cast<imbalance_limit>(strategy_)) {
+        strategy = std::make_shared<typename HybType::imbalance_limit>(
+            temp->get_percent());
+    } else if (auto temp = std::dynamic_pointer_cast<column_limit>(strategy_)) {
+        strategy = std::make_shared<typename HybType::column_limit>(
+            temp->get_num_columns());
+    } else {
+        GKO_NOT_SUPPORTED(strategy_);
+    }
+    return strategy;
+}
+
+
 }  // namespace matrix
 }  // namespace gko
 

From 30aa10d2047cb81873ec0335c9a8e64658ff155c Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 25 Sep 2020 10:34:46 +0200
Subject: [PATCH 0023/1008] review update (remove unneeded, refine format) -
 remove unneed void_t on enable_if_t - remove the remove_complex_s,
 to_complex_s/real_s in gko namespace - refine code style on test according to
 AAA rule
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 core/base/extended_float.hpp                 |  7 +-
 core/test/base/lin_op.cpp                    | 13 ++--
 core/test/base/math.cpp                      |  8 +++
 core/test/matrix/hybrid.cpp                  | 12 ++--
 cuda/test/components/absolute_array.cpp      |  4 ++
 hip/test/components/absolute_array.hip.cpp   |  4 ++
 include/ginkgo/core/base/math.hpp            | 68 ++++++--------------
 include/ginkgo/core/matrix/hybrid.hpp        | 12 ++--
 omp/test/components/absolute_array.cpp       |  4 ++
 reference/test/components/absolute_array.cpp |  4 ++
 reference/test/matrix/dense_kernels.cpp      | 51 ++++++++++-----
 reference/test/matrix/diagonal_kernels.cpp   |  4 +-
 12 files changed, 105 insertions(+), 86 deletions(-)

diff --git a/core/base/extended_float.hpp b/core/base/extended_float.hpp
index 8ebf836641b..c83e7b81f20 100644
--- a/core/base/extended_float.hpp
+++ b/core/base/extended_float.hpp
@@ -70,18 +70,17 @@ template <std::size_t, typename = void>
 struct uint_of_impl {};
 
 template <std::size_t Bits>
-struct uint_of_impl<Bits, xstd::void_t<std::enable_if_t<(Bits <= 16)>>> {
+struct uint_of_impl<Bits, std::enable_if_t<(Bits <= 16)>> {
     using type = uint16;
 };
 
 template <std::size_t Bits>
-struct uint_of_impl<Bits,
-                    xstd::void_t<std::enable_if_t<(16 < Bits && Bits <= 32)>>> {
+struct uint_of_impl<Bits, std::enable_if_t<(16 < Bits && Bits <= 32)>> {
     using type = uint32;
 };
 
 template <std::size_t Bits>
-struct uint_of_impl<Bits, xstd::void_t<std::enable_if_t<(32 < Bits)>>> {
+struct uint_of_impl<Bits, std::enable_if_t<(32 < Bits)>> {
     using type = uint64;
 };
 
diff --git a/core/test/base/lin_op.cpp b/core/test/base/lin_op.cpp
index e73efa5e923..303e8bd6e89 100644
--- a/core/test/base/lin_op.cpp
+++ b/core/test/base/lin_op.cpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <complex>
 #include <memory>
+#include <type_traits>
 
 
 #include <gtest/gtest.h>
@@ -386,8 +387,10 @@ TEST_F(EnableAbsoluteComputation, OutplaceAbsoluteOnConcreteType)
 {
     auto abs_op = op->compute_absolute();
 
-    ASSERT_EQ(typeid(abs_op),
-              typeid(std::unique_ptr<gko::remove_complex<dummy_type>>));
+    static_assert(
+        std::is_same<decltype(abs_op),
+                     std::unique_ptr<gko::remove_complex<dummy_type>>>::value,
+        "Types must match.");
     ASSERT_EQ(abs_op->get_value(), 5.0);
 }
 
@@ -407,8 +410,10 @@ TEST_F(EnableAbsoluteComputation, OutplaceAbsoluteOnAbsoluteComputable)
 {
     auto abs_op = op->compute_absolute();
 
-    ASSERT_EQ(typeid(abs_op),
-              typeid(std::unique_ptr<gko::remove_complex<dummy_type>>));
+    static_assert(
+        std::is_same<decltype(abs_op),
+                     std::unique_ptr<gko::remove_complex<dummy_type>>>::value,
+        "Types must match.");
     ASSERT_EQ(abs_op->get_value(), 5.0);
 }
 
diff --git a/core/test/base/math.cpp b/core/test/base/math.cpp
index 0e54c87f788..288bea41576 100644
--- a/core/test/base/math.cpp
+++ b/core/test/base/math.cpp
@@ -143,7 +143,9 @@ TEST(RemoveComplexClass, Float)
 {
     using origin = DummyClass<float, int>;
     using expect = DummyClass<float, int>;
+
     bool check = std::is_same<expect, gko::remove_complex<origin>>::value;
+
     ASSERT_TRUE(check);
 }
 
@@ -152,7 +154,9 @@ TEST(RemoveComplexClass, Double)
 {
     using origin = DummyClass<double, int>;
     using expect = DummyClass<double, int>;
+
     bool check = std::is_same<expect, gko::remove_complex<origin>>::value;
+
     ASSERT_TRUE(check);
 }
 
@@ -161,7 +165,9 @@ TEST(RemoveComplexClass, FloatComplex)
 {
     using origin = DummyClass<std::complex<float>, int>;
     using expect = DummyClass<float, int>;
+
     bool check = std::is_same<expect, gko::remove_complex<origin>>::value;
+
     ASSERT_TRUE(check);
 }
 
@@ -170,7 +176,9 @@ TEST(RemoveComplexClass, DoubleComplex)
 {
     using origin = DummyClass<std::complex<double>, int>;
     using expect = DummyClass<double, int>;
+
     bool check = std::is_same<expect, gko::remove_complex<origin>>::value;
+
     ASSERT_TRUE(check);
 }
 
diff --git a/core/test/matrix/hybrid.cpp b/core/test/matrix/hybrid.cpp
index 9045e950a23..9a5ad45f0c7 100644
--- a/core/test/matrix/hybrid.cpp
+++ b/core/test/matrix/hybrid.cpp
@@ -339,8 +339,8 @@ TYPED_TEST(Hybrid, GetCorrectImbalanceLimit)
     auto mtx_stra = gko::as<strategy>(mtx->get_strategy());
     auto mtx2_stra = gko::as<strategy2>(mtx->template get_strategy<Mtx2>());
 
-    EXPECT_EQ(mtx_stra->get_percent(), 0.4f);
-    EXPECT_EQ(mtx2_stra->get_percent(), 0.4f);
+    EXPECT_EQ(mtx_stra->get_percentage(), 0.4f);
+    EXPECT_EQ(mtx2_stra->get_percentage(), 0.4f);
 }
 
 
@@ -355,9 +355,9 @@ TYPED_TEST(Hybrid, GetCorrectImbalanceBoundedLimit)
     auto mtx_stra = gko::as<strategy>(mtx->get_strategy());
     auto mtx2_stra = gko::as<strategy2>(mtx->template get_strategy<Mtx2>());
 
-    EXPECT_EQ(mtx_stra->get_percent(), 0.4f);
+    EXPECT_EQ(mtx_stra->get_percentage(), 0.4f);
     EXPECT_EQ(mtx_stra->get_ratio(), 0.1f);
-    EXPECT_EQ(mtx2_stra->get_percent(), 0.4f);
+    EXPECT_EQ(mtx2_stra->get_percentage(), 0.4f);
     EXPECT_EQ(mtx2_stra->get_ratio(), 0.1f);
 }
 
@@ -375,7 +375,7 @@ TYPED_TEST(Hybrid, GetCorrectMinimalStorageLimitWithDifferentHybType)
     auto mtx_stra = gko::as<strategy>(mtx->get_strategy());
     auto mtx2_stra = gko::as<strategy2>(mtx->template get_strategy<Mtx2>());
 
-    EXPECT_EQ(mtx2_stra->get_percent(), mtx_stra->get_percent());
+    EXPECT_EQ(mtx2_stra->get_percentage(), mtx_stra->get_percentage());
 }
 
 
@@ -390,7 +390,7 @@ TYPED_TEST(Hybrid, GetCorrectMinimalStorageLimitWithSameHybType)
     auto mtx_stra = gko::as<strategy>(mtx->get_strategy());
     auto mtx2_stra = gko::as<strategy2>(mtx->template get_strategy<Mtx2>());
 
-    EXPECT_EQ(mtx2_stra->get_percent(), mtx_stra->get_percent());
+    EXPECT_EQ(mtx2_stra->get_percentage(), mtx_stra->get_percentage());
 }
 
 
diff --git a/cuda/test/components/absolute_array.cpp b/cuda/test/components/absolute_array.cpp
index 6bdef3df6dd..919c042cff0 100644
--- a/cuda/test/components/absolute_array.cpp
+++ b/cuda/test/components/absolute_array.cpp
@@ -85,6 +85,7 @@ TEST_F(AbsoluteArray, InplaceEqualsReference)
         exec, dvals.get_data(), total_size);
     gko::kernels::reference::components::inplace_absolute_array(
         ref, vals.get_data(), total_size);
+
     GKO_ASSERT_ARRAY_EQ(vals, dvals);
 }
 
@@ -95,6 +96,7 @@ TEST_F(AbsoluteArray, InplaceComplexEqualsReference)
         exec, dcomplex_vals.get_data(), total_size);
     gko::kernels::reference::components::inplace_absolute_array(
         ref, complex_vals.get_data(), total_size);
+
     GKO_ASSERT_ARRAY_EQ(complex_vals, dcomplex_vals);
 }
 
@@ -108,6 +110,7 @@ TEST_F(AbsoluteArray, OutplaceEqualsReference)
         exec, dvals.get_const_data(), total_size, dabs_vals.get_data());
     gko::kernels::reference::components::outplace_absolute_array(
         ref, vals.get_const_data(), total_size, abs_vals.get_data());
+
     GKO_ASSERT_ARRAY_EQ(abs_vals, dabs_vals);
 }
 
@@ -121,6 +124,7 @@ TEST_F(AbsoluteArray, OutplaceComplexEqualsReference)
         exec, dcomplex_vals.get_const_data(), total_size, dabs_vals.get_data());
     gko::kernels::reference::components::outplace_absolute_array(
         ref, complex_vals.get_const_data(), total_size, abs_vals.get_data());
+
     GKO_ASSERT_ARRAY_EQ(abs_vals, dabs_vals);
 }
 
diff --git a/hip/test/components/absolute_array.hip.cpp b/hip/test/components/absolute_array.hip.cpp
index 8df8e79109d..f718805d5eb 100644
--- a/hip/test/components/absolute_array.hip.cpp
+++ b/hip/test/components/absolute_array.hip.cpp
@@ -85,6 +85,7 @@ TEST_F(AbsoluteArray, InplaceEqualsReference)
         exec, dvals.get_data(), total_size);
     gko::kernels::reference::components::inplace_absolute_array(
         ref, vals.get_data(), total_size);
+
     GKO_ASSERT_ARRAY_EQ(vals, dvals);
 }
 
@@ -95,6 +96,7 @@ TEST_F(AbsoluteArray, InplaceComplexEqualsReference)
         exec, dcomplex_vals.get_data(), total_size);
     gko::kernels::reference::components::inplace_absolute_array(
         ref, complex_vals.get_data(), total_size);
+
     GKO_ASSERT_ARRAY_EQ(complex_vals, dcomplex_vals);
 }
 
@@ -108,6 +110,7 @@ TEST_F(AbsoluteArray, OutplaceEqualsReference)
         exec, dvals.get_const_data(), total_size, dabs_vals.get_data());
     gko::kernels::reference::components::outplace_absolute_array(
         ref, vals.get_const_data(), total_size, abs_vals.get_data());
+
     GKO_ASSERT_ARRAY_EQ(abs_vals, dabs_vals);
 }
 
@@ -121,6 +124,7 @@ TEST_F(AbsoluteArray, OutplaceComplexEqualsReference)
         exec, dcomplex_vals.get_const_data(), total_size, dabs_vals.get_data());
     gko::kernels::reference::components::outplace_absolute_array(
         ref, complex_vals.get_const_data(), total_size, abs_vals.get_data());
+
     GKO_ASSERT_ARRAY_EQ(abs_vals, dabs_vals);
 }
 
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 386f0bf737d..0bb6ac9b5a9 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -170,28 +170,28 @@ struct is_complex_or_scalar_impl<std::complex<T>> : std::is_scalar<T> {};
 
 
 /**
- * template_convertor is converting the template parameters of a class by
- * convertor<type>.
+ * template_converter is converting the template parameters of a class by
+ * converter<type>.
  *
- * @tparam  convertor<type> which convert one type to another type
+ * @tparam  converter<type> which convert one type to another type
  * @tparam  T  type
  */
-template <template <typename> class convertor, typename T>
-struct template_convertor {};
+template <template <typename> class converter, typename T>
+struct template_converter {};
 
 /**
- * template_convertor is converting the template parameters of a class by
- * convertor<type>. Converting class<T1, T2, ...> to class<convertor<T1>,
- * convertor<T2>, convertor<...>>.
+ * template_converter is converting the template parameters of a class by
+ * converter<type>. Converting class<T1, T2, ...> to class<converter<T1>,
+ * converter<T2>, converter<...>>.
  *
- * @tparam  convertor<type> which convert one type to another type
+ * @tparam  converter<type> which convert one type to another type
  * @tparam  template <...> T  class template base
  * @tparam  ...Rest  the template parameter of T
  */
-template <template <typename> class convertor, template <typename...> class T,
+template <template <typename> class converter, template <typename...> class T,
           typename... Rest>
-struct template_convertor<convertor, T<Rest...>> {
-    using type = T<typename convertor<Rest>::type...>;
+struct template_converter<converter, T<Rest...>> {
+    using type = T<typename converter<Rest>::type...>;
 };
 
 
@@ -205,8 +205,8 @@ struct remove_complex_s {};
  * @tparam T  complex or scalar type
  */
 template <typename T>
-struct remove_complex_s<
-    T, xstd::void_t<std::enable_if_t<is_complex_or_scalar_impl<T>::value>>> {
+struct remove_complex_s<T,
+                        std::enable_if_t<is_complex_or_scalar_impl<T>::value>> {
     using type = typename detail::remove_complex_impl<T>::type;
 };
 
@@ -218,9 +218,9 @@ struct remove_complex_s<
  */
 template <typename T>
 struct remove_complex_s<
-    T, xstd::void_t<std::enable_if_t<!is_complex_or_scalar_impl<T>::value>>> {
+    T, std::enable_if_t<!is_complex_or_scalar_impl<T>::value>> {
     using type =
-        typename detail::template_convertor<detail::remove_complex_impl,
+        typename detail::template_converter<detail::remove_complex_impl,
                                             T>::type;
 };
 
@@ -235,8 +235,7 @@ struct to_complex_s {};
  * @tparam T  complex or scalar type
  */
 template <typename T>
-struct to_complex_s<
-    T, xstd::void_t<std::enable_if_t<is_complex_or_scalar_impl<T>::value>>> {
+struct to_complex_s<T, std::enable_if_t<is_complex_or_scalar_impl<T>::value>> {
     using type = typename detail::to_complex_impl<T>::type;
 };
 
@@ -247,10 +246,9 @@ struct to_complex_s<
  * @tparam T  class with template parameters
  */
 template <typename T>
-struct to_complex_s<
-    T, xstd::void_t<std::enable_if_t<!is_complex_or_scalar_impl<T>::value>>> {
+struct to_complex_s<T, std::enable_if_t<!is_complex_or_scalar_impl<T>::value>> {
     using type =
-        typename detail::template_convertor<detail::to_complex_impl, T>::type;
+        typename detail::template_converter<detail::to_complex_impl, T>::type;
 };
 
 
@@ -329,14 +327,6 @@ GKO_INLINE GKO_ATTRIBUTES constexpr bool is_complex_or_scalar()
 }
 
 
-/**
- * Removes the complex of complex/scalar type or the template parameter of class
- * by accessing the `type` attribute of this struct.
- *
- * @tparam T  type to remove complex
- */
-using detail::remove_complex_s;
-
 /**
  * Obtain the type which removed the complex of complex/scalar type or the
  * template parameter of class by accessing the `type` attribute of this struct.
@@ -346,17 +336,9 @@ using detail::remove_complex_s;
  * @note remove_complex<class> can not be used in friend class declaration.
  */
 template <typename T>
-using remove_complex = typename remove_complex_s<T>::type;
+using remove_complex = typename detail::remove_complex_s<T>::type;
 
 
-/**
- * Add the complex of complex/scalar type or the template parameter of class
- * by accessing the `type` attribute of this struct.
- *
- * @tparam T  type to complex_type
- */
-using detail::to_complex_s;
-
 /**
  * Obtain the type which adds the complex of complex/scalar type or the
  * template parameter of class by accessing the `type` attribute of this struct.
@@ -373,17 +355,9 @@ using detail::to_complex_s;
  *         error: can not recognize it is class correctly.
  */
 template <typename T>
-using to_complex = typename to_complex_s<T>::type;
+using to_complex = typename detail::to_complex_s<T>::type;
 
 
-/**
- * to_real_s is alias of remove_complex_s
- *
- * @tparam T  type to real
- */
-template <typename T>
-using to_real_s = remove_complex_s<T>;
-
 /**
  * to_real is alias of remove_complex
  *
diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp
index 3b8653864dc..c844907e8b2 100644
--- a/include/ginkgo/core/matrix/hybrid.hpp
+++ b/include/ginkgo/core/matrix/hybrid.hpp
@@ -272,7 +272,7 @@ class Hybrid
          *
          * @retrun percent
          */
-        auto get_percent() const { return percent_; }
+        auto get_percentage() const { return percent_; }
 
     private:
         float percent_;
@@ -307,7 +307,7 @@ class Hybrid
          *
          * @retrun percent
          */
-        auto get_percent() const { return strategy_.get_percent(); }
+        auto get_percentage() const { return strategy_.get_percentage(); }
 
         /**
          * Get the ratio setting
@@ -349,7 +349,7 @@ class Hybrid
          *
          * @retrun percent
          */
-        auto get_percent() { return strategy_.get_percent(); }
+        auto get_percentage() { return strategy_.get_percentage(); }
 
     private:
         imbalance_limit strategy_;
@@ -766,16 +766,16 @@ Hybrid<ValueType, IndexType>::get_strategy() const
                 std::make_shared<typename HybType::minimal_storage_limit>();
         } else {
             strategy = std::make_shared<typename HybType::imbalance_limit>(
-                temp->get_percent());
+                temp->get_percentage());
         }
     } else if (auto temp = std::dynamic_pointer_cast<imbalance_bounded_limit>(
                    strategy_)) {
         strategy = std::make_shared<typename HybType::imbalance_bounded_limit>(
-            temp->get_percent(), temp->get_ratio());
+            temp->get_percentage(), temp->get_ratio());
     } else if (auto temp =
                    std::dynamic_pointer_cast<imbalance_limit>(strategy_)) {
         strategy = std::make_shared<typename HybType::imbalance_limit>(
-            temp->get_percent());
+            temp->get_percentage());
     } else if (auto temp = std::dynamic_pointer_cast<column_limit>(strategy_)) {
         strategy = std::make_shared<typename HybType::column_limit>(
             temp->get_num_columns());
diff --git a/omp/test/components/absolute_array.cpp b/omp/test/components/absolute_array.cpp
index 9304def3fa9..92203777d9c 100644
--- a/omp/test/components/absolute_array.cpp
+++ b/omp/test/components/absolute_array.cpp
@@ -85,6 +85,7 @@ TEST_F(AbsoluteArray, InplaceEqualsReference)
         exec, dvals.get_data(), total_size);
     gko::kernels::reference::components::inplace_absolute_array(
         ref, vals.get_data(), total_size);
+
     GKO_ASSERT_ARRAY_EQ(vals, dvals);
 }
 
@@ -95,6 +96,7 @@ TEST_F(AbsoluteArray, InplaceComplexEqualsReference)
         exec, dcomplex_vals.get_data(), total_size);
     gko::kernels::reference::components::inplace_absolute_array(
         ref, complex_vals.get_data(), total_size);
+
     GKO_ASSERT_ARRAY_EQ(complex_vals, dcomplex_vals);
 }
 
@@ -108,6 +110,7 @@ TEST_F(AbsoluteArray, OutplaceEqualsReference)
         exec, dvals.get_const_data(), total_size, dabs_vals.get_data());
     gko::kernels::reference::components::outplace_absolute_array(
         ref, vals.get_const_data(), total_size, abs_vals.get_data());
+
     GKO_ASSERT_ARRAY_EQ(abs_vals, dabs_vals);
 }
 
@@ -121,6 +124,7 @@ TEST_F(AbsoluteArray, OutplaceComplexEqualsReference)
         exec, dcomplex_vals.get_const_data(), total_size, dabs_vals.get_data());
     gko::kernels::reference::components::outplace_absolute_array(
         ref, complex_vals.get_const_data(), total_size, abs_vals.get_data());
+
     GKO_ASSERT_ARRAY_EQ(abs_vals, dabs_vals);
 }
 
diff --git a/reference/test/components/absolute_array.cpp b/reference/test/components/absolute_array.cpp
index 83cc444df20..9793a6cf0ed 100644
--- a/reference/test/components/absolute_array.cpp
+++ b/reference/test/components/absolute_array.cpp
@@ -81,8 +81,10 @@ TYPED_TEST_CASE(AbsoluteArray, gko::test::ValueTypes);
 TYPED_TEST(AbsoluteArray, InplaceEqualsExpected)
 {
     using T = typename TestFixture::value_type;
+
     gko::kernels::reference::components::inplace_absolute_array(
         this->ref, this->vals.get_data(), this->total_size);
+
     GKO_ASSERT_ARRAY_EQ(this->vals, this->inplace_expected);
 }
 
@@ -92,9 +94,11 @@ TYPED_TEST(AbsoluteArray, OutplaceEqualsExpected)
     using T = typename TestFixture::value_type;
     using AbsT = typename TestFixture::abs_type;
     gko::Array<AbsT> abs_vals(this->ref, this->total_size);
+
     gko::kernels::reference::components::outplace_absolute_array(
         this->ref, this->vals.get_const_data(), this->total_size,
         abs_vals.get_data());
+
     GKO_ASSERT_ARRAY_EQ(abs_vals, this->outplace_expected);
 }
 
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index cea66d04f43..9ac4b9b448f 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -1726,9 +1726,10 @@ TYPED_TEST(Dense, SquareMatrixIsRowPermutable)
     using Mtx = typename TestFixture::Mtx;
     auto exec = this->mtx5->get_executor();
     gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
-    auto row_permute = this->mtx5->row_permute(&permute_idxs);
 
+    auto row_permute = this->mtx5->row_permute(&permute_idxs);
     auto row_permute_dense = static_cast<Mtx *>(row_permute.get());
+
     // clang-format off
     GKO_ASSERT_MTX_NEAR(row_permute_dense,
                         l({{-2.0, 2.0, 4.5},
@@ -1747,9 +1748,10 @@ TYPED_TEST(Dense, NonSquareMatrixIsRowPermutable)
     using Mtx = typename TestFixture::Mtx;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
-    auto row_permute = this->mtx4->row_permute(&permute_idxs);
 
+    auto row_permute = this->mtx4->row_permute(&permute_idxs);
     auto row_permute_dense = static_cast<Mtx *>(row_permute.get());
+
     // clang-format off
     GKO_ASSERT_MTX_NEAR(row_permute_dense,
                         l({{0.0, 5.0, 0.0},
@@ -1768,9 +1770,10 @@ TYPED_TEST(Dense, SquareMatrixIsColPermutable)
     using Mtx = typename TestFixture::Mtx;
     auto exec = this->mtx5->get_executor();
     gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
-    auto c_permute = this->mtx5->column_permute(&permute_idxs);
 
+    auto c_permute = this->mtx5->column_permute(&permute_idxs);
     auto c_permute_dense = static_cast<Mtx *>(c_permute.get());
+
     // clang-format off
     GKO_ASSERT_MTX_NEAR(c_permute_dense,
                         l({{-1.0, -0.5, 1.0}, {2.0, 4.5, -2.0}, {3.4, 1.2, 2.1}}), r<TypeParam>::value);
@@ -1787,9 +1790,10 @@ TYPED_TEST(Dense, NonSquareMatrixIsColPermutable)
     using Mtx = typename TestFixture::Mtx;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
-    auto c_permute = this->mtx4->column_permute(&permute_idxs);
 
+    auto c_permute = this->mtx4->column_permute(&permute_idxs);
     auto c_permute_dense = static_cast<Mtx *>(c_permute.get());
+
     // clang-format off
     GKO_ASSERT_MTX_NEAR(c_permute_dense,
                         l({{3.0, 2.0, 1.0},
@@ -1809,11 +1813,12 @@ TYPED_TEST(Dense, SquareMatrixIsInverseRowPermutable)
     using Mtx = typename TestFixture::Mtx;
     auto exec = this->mtx5->get_executor();
     gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 2, 0}};
+
     auto inverse_row_permute =
         this->mtx5->inverse_row_permute(&inverse_permute_idxs);
-
     auto inverse_row_permute_dense =
         static_cast<Mtx *>(inverse_row_permute.get());
+
     // clang-format off
     GKO_ASSERT_MTX_NEAR(inverse_row_permute_dense,
                         l({{2.1, 3.4, 1.2},
@@ -1832,11 +1837,12 @@ TYPED_TEST(Dense, NonSquareMatrixIsInverseRowPermutable)
     using Mtx = typename TestFixture::Mtx;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 0}};
+
     auto inverse_row_permute =
         this->mtx4->inverse_row_permute(&inverse_permute_idxs);
-
     auto inverse_row_permute_dense =
         static_cast<Mtx *>(inverse_row_permute.get());
+
     // clang-format off
     GKO_ASSERT_MTX_NEAR(inverse_row_permute_dense,
                         l({{0.0, 5.0, 0.0},
@@ -1855,10 +1861,11 @@ TYPED_TEST(Dense, SquareMatrixIsInverseColPermutable)
     using Mtx = typename TestFixture::Mtx;
     auto exec = this->mtx5->get_executor();
     gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 2, 0}};
+
     auto inverse_c_permute =
         this->mtx5->inverse_column_permute(&inverse_permute_idxs);
-
     auto inverse_c_permute_dense = static_cast<Mtx *>(inverse_c_permute.get());
+
     // clang-format off
     GKO_ASSERT_MTX_NEAR(inverse_c_permute_dense,
                         l({{-0.5, 1.0, -1.0},
@@ -1877,10 +1884,11 @@ TYPED_TEST(Dense, NonSquareMatrixIsInverseColPermutable)
     using Mtx = typename TestFixture::Mtx;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 2, 0}};
+
     auto inverse_c_permute =
         this->mtx4->inverse_column_permute(&inverse_permute_idxs);
-
     auto inverse_c_permute_dense = static_cast<Mtx *>(inverse_c_permute.get());
+
     // clang-format off
     GKO_ASSERT_MTX_NEAR(inverse_c_permute_dense,
                         l({{2.0, 1.0, 3.0},
@@ -1899,9 +1907,10 @@ TYPED_TEST(Dense, NonSquareMatrixIsRowPermutable64)
     using Mtx = typename TestFixture::Mtx;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
-    auto row_permute = this->mtx4->row_permute(&permute_idxs);
 
+    auto row_permute = this->mtx4->row_permute(&permute_idxs);
     auto row_permute_dense = static_cast<Mtx *>(row_permute.get());
+
     // clang-format off
     GKO_ASSERT_MTX_NEAR(row_permute_dense,
                         l({{0.0, 5.0, 0.0},
@@ -1919,9 +1928,10 @@ TYPED_TEST(Dense, NonSquareMatrixIsColPermutable64)
     using Mtx = typename TestFixture::Mtx;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
-    auto c_permute = this->mtx4->column_permute(&permute_idxs);
 
+    auto c_permute = this->mtx4->column_permute(&permute_idxs);
     auto c_permute_dense = static_cast<Mtx *>(c_permute.get());
+
     // clang-format off
     GKO_ASSERT_MTX_NEAR(c_permute_dense,
                         l({{3.0, 2.0, 1.0},
@@ -1940,11 +1950,12 @@ TYPED_TEST(Dense, NonSquareMatrixIsInverseRowPermutable64)
     using Mtx = typename TestFixture::Mtx;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int64> inverse_permute_idxs{exec, {1, 0}};
+
     auto inverse_row_permute =
         this->mtx4->inverse_row_permute(&inverse_permute_idxs);
-
     auto inverse_row_permute_dense =
         static_cast<Mtx *>(inverse_row_permute.get());
+
     // clang-format off
     GKO_ASSERT_MTX_NEAR(inverse_row_permute_dense,
                         l({{0.0, 5.0, 0.0},
@@ -1962,10 +1973,11 @@ TYPED_TEST(Dense, NonSquareMatrixIsInverseColPermutable64)
     using Mtx = typename TestFixture::Mtx;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int64> inverse_permute_idxs{exec, {1, 2, 0}};
+
     auto inverse_c_permute =
         this->mtx4->inverse_column_permute(&inverse_permute_idxs);
-
     auto inverse_c_permute_dense = static_cast<Mtx *>(inverse_c_permute.get());
+
     // clang-format off
     GKO_ASSERT_MTX_NEAR(inverse_c_permute_dense,
                         l({{2.0, 1.0, 3.0},
@@ -1978,6 +1990,7 @@ TYPED_TEST(Dense, NonSquareMatrixIsInverseColPermutable64)
 TYPED_TEST(Dense, ExtractsDiagonalFromSquareMatrix)
 {
     using T = typename TestFixture::value_type;
+
     // clang-format off
     // {1.0, -1.0, -0.5},
     // {-2.0, 2.0, 4.5},
@@ -1996,6 +2009,7 @@ TYPED_TEST(Dense, ExtractsDiagonalFromSquareMatrix)
 TYPED_TEST(Dense, ExtractsDiagonalFromNonSquareMatrix)
 {
     using T = typename TestFixture::value_type;
+
     // clang-format off
     // {1.0, 3.0, 2.0},
     // {0.0, 5.0, 0.0}
@@ -2017,6 +2031,7 @@ TYPED_TEST(Dense, InplaceAbsolute)
     // {-2.0, 2.0, 4.5},
     // {2.1, 3.4, 1.2}
     // clang-format on
+
     this->mtx5->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(this->mtx5,
@@ -2028,13 +2043,14 @@ TYPED_TEST(Dense, InplaceAbsolute)
 TYPED_TEST(Dense, InplaceAbsoluteSubMatrix)
 {
     using T = typename TestFixture::value_type;
-    // mtx takes the left top corner 2 x 2 matrix
-    auto mtx = this->mtx5->create_submatrix(gko::span{0, 2}, gko::span{0, 2});
     // clang-format off
     // {1.0, -1.0, -0.5},
     // {-2.0, 2.0, 4.5},
     // {2.1, 3.4, 1.2}
     // clang-format on
+    // mtx takes the left top corner 2 x 2 matrix
+    auto mtx = this->mtx5->create_submatrix(gko::span{0, 2}, gko::span{0, 2});
+
     mtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(this->mtx5,
@@ -2051,6 +2067,7 @@ TYPED_TEST(Dense, OutplaceAbsolute)
     // {-2.0, 2.0, 4.5},
     // {2.1, 3.4, 1.2}
     // clang-format on
+
     auto abs_mtx = this->mtx5->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx,
@@ -2062,13 +2079,14 @@ TYPED_TEST(Dense, OutplaceAbsolute)
 TYPED_TEST(Dense, OutplaceAbsoluteSubMatrix)
 {
     using T = typename TestFixture::value_type;
-    // mtx takes the left top corner 2 x 2 matrix
-    auto mtx = this->mtx5->create_submatrix(gko::span{0, 2}, gko::span{0, 2});
     // clang-format off
     // {1.0, -1.0, -0.5},
     // {-2.0, 2.0, 4.5},
     // {2.1, 3.4, 1.2}
     // clang-format on
+    // mtx takes the left top corner 2 x 2 matrix
+    auto mtx = this->mtx5->create_submatrix(gko::span{0, 2}, gko::span{0, 2});
+
     auto abs_mtx = mtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(abs_mtx, l({{1.0, 1.0}, {2.0, 2.0}}),
@@ -2097,6 +2115,7 @@ TYPED_TEST(DenseComplex, NonSquareMatrixIsConjugateTransposable)
                                        {T{-2.0, 1.5}, T{4.5, 0.0}},
                                        {T{1.0, 0.0}, T{0.0, 1.0}}},
                                       exec);
+
     auto trans = mtx->conj_transpose();
     auto trans_as_dense = static_cast<Dense *>(trans.get());
 
diff --git a/reference/test/matrix/diagonal_kernels.cpp b/reference/test/matrix/diagonal_kernels.cpp
index fc207889de3..1d39ee9367d 100644
--- a/reference/test/matrix/diagonal_kernels.cpp
+++ b/reference/test/matrix/diagonal_kernels.cpp
@@ -395,7 +395,6 @@ TYPED_TEST(DiagonalComplex, MtxIsConjugateTransposable)
 {
     using Diag = typename TestFixture::Diag;
     using value_type = typename TestFixture::value_type;
-
     auto exec = gko::ReferenceExecutor::create();
     auto diag = Diag::create(exec, 3);
     auto diag_values = diag->get_values();
@@ -406,6 +405,7 @@ TYPED_TEST(DiagonalComplex, MtxIsConjugateTransposable)
     auto trans = diag->conj_transpose();
     auto trans_as_diagonal = static_cast<Diag *>(trans.get());
     auto trans_values = trans_as_diagonal->get_values();
+
     EXPECT_EQ(trans->get_size(), gko::dim<2>(3));
     EXPECT_EQ(trans_values[0], (value_type{1.0, -2.0}));
     EXPECT_EQ(trans_values[1], (value_type{3.0, 0.0}));
@@ -417,7 +417,6 @@ TYPED_TEST(DiagonalComplex, InplaceAbsolute)
 {
     using Diag = typename TestFixture::Diag;
     using value_type = typename TestFixture::value_type;
-
     auto exec = gko::ReferenceExecutor::create();
     auto diag = Diag::create(exec, 3);
     auto diag_values = diag->get_values();
@@ -438,7 +437,6 @@ TYPED_TEST(DiagonalComplex, OutplaceAbsolute)
     using Diag = typename TestFixture::Diag;
     using value_type = typename TestFixture::value_type;
     using abs_type = gko::remove_complex<value_type>;
-
     auto exec = gko::ReferenceExecutor::create();
     auto diag = Diag::create(exec, 3);
     auto diag_values = diag->get_values();

From c821702bee5a3e43ec2e23451e9512e765da4a7f Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratik.nayak4@gmail.com>
Date: Thu, 3 Sep 2020 18:15:35 +0200
Subject: [PATCH 0024/1008] Add example targets and add better defaults as
 possible,

---
 examples/CMakeLists.txt                       | 46 ++++++++++---------
 .../CMakeLists.txt                            |  4 +-
 examples/custom-logger/CMakeLists.txt         |  4 +-
 examples/custom-matrix-format/CMakeLists.txt  | 11 +++--
 .../custom-stopping-criterion/CMakeLists.txt  |  9 ++--
 .../external-lib-interfacing/CMakeLists.txt   |  3 +-
 examples/ginkgo-overhead/CMakeLists.txt       |  4 +-
 examples/ginkgo-ranges/CMakeLists.txt         |  4 +-
 .../ilu-preconditioned-solver/CMakeLists.txt  |  4 +-
 examples/inverse-iteration/CMakeLists.txt     |  4 +-
 .../CMakeLists.txt                            |  4 +-
 examples/iterative-refinement/CMakeLists.txt  |  4 +-
 examples/minimal-cuda-solver/CMakeLists.txt   |  4 +-
 examples/mixed-precision-ir/CMakeLists.txt    |  4 +-
 .../nine-pt-stencil-solver/CMakeLists.txt     |  4 +-
 .../nine-pt-stencil-solver.cpp                |  6 +--
 examples/performance-debugging/CMakeLists.txt |  4 +-
 examples/poisson-solver/CMakeLists.txt        |  4 +-
 examples/poisson-solver/poisson-solver.cpp    |  6 +--
 examples/preconditioned-solver/CMakeLists.txt |  4 +-
 examples/preconditioner-export/CMakeLists.txt |  4 +-
 examples/simple-solver-logging/CMakeLists.txt |  4 +-
 examples/simple-solver/CMakeLists.txt         |  4 +-
 .../three-pt-stencil-solver/CMakeLists.txt    |  4 +-
 .../three-pt-stencil-solver.cpp               |  7 +--
 25 files changed, 65 insertions(+), 95 deletions(-)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index ea8a02e9072..f5413acb04d 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -1,25 +1,27 @@
 option(GINKGO_BUILD_EXTLIB_EXAMPLE "Build the external-lib-interfacing with deal.II, you need to link the deal.II library." OFF)
-add_subdirectory(custom-logger)
-add_subdirectory(custom-matrix-format)
-add_subdirectory(custom-stopping-criterion)
+option(GINKGO_RUN_EXAMPLES "Run the examples." OFF)
+
+function(setup_example_target target_name)
+    add_executable(${target_name} ${target_name}.cpp)
+    target_link_libraries(${target_name} ginkgo)
+    target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+endfunction()
+
+
+set(EXAMPLES_LIST "custom-logger;custom-matrix-format;custom-stopping-criterion;adaptiveprecision-blockjacobi;ginkgo-overhead;ginkgo-ranges;ilu-preconditioned-solver;ir-ilu-preconditioned-solver;inverse-iteration;iterative-refinement;minimal-cuda-solver;mixed-precision-ir;nine-pt-stencil-solver;papi-logging;performance-debugging;poisson-solver;preconditioned-solver;preconditioner-export;simple-solver;simple-solver-logging;three-pt-stencil-solver;")
 if(GINKGO_BUILD_EXTLIB_EXAMPLE)
-    add_subdirectory(external-lib-interfacing)
+    set(EXAMPLES_LIST "${EXAMPLES_LIST};external-lib-interfacing")
+endif()
+
+foreach(example ${EXAMPLES_LIST})
+    add_subdirectory(${example})
+endforeach()
+
+set(run_list "")
+if(GINKGO_RUN_EXAMPLES)
+    foreach(example ${EXAMPLES_LIST})
+        add_custom_target("run-${example}"
+            COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example} > ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example}.out
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${example})
+    endforeach()
 endif()
-add_subdirectory(adaptiveprecision-blockjacobi)
-add_subdirectory(ginkgo-overhead)
-add_subdirectory(ginkgo-ranges)
-add_subdirectory(ilu-preconditioned-solver)
-add_subdirectory(ir-ilu-preconditioned-solver)
-add_subdirectory(inverse-iteration)
-add_subdirectory(iterative-refinement)
-add_subdirectory(minimal-cuda-solver)
-add_subdirectory(mixed-precision-ir)
-add_subdirectory(nine-pt-stencil-solver)
-add_subdirectory(papi-logging)
-add_subdirectory(performance-debugging)
-add_subdirectory(poisson-solver)
-add_subdirectory(preconditioned-solver)
-add_subdirectory(preconditioner-export)
-add_subdirectory(simple-solver)
-add_subdirectory(simple-solver-logging)
-add_subdirectory(three-pt-stencil-solver)
diff --git a/examples/adaptiveprecision-blockjacobi/CMakeLists.txt b/examples/adaptiveprecision-blockjacobi/CMakeLists.txt
index d3188aaca12..e04e61ad6fa 100644
--- a/examples/adaptiveprecision-blockjacobi/CMakeLists.txt
+++ b/examples/adaptiveprecision-blockjacobi/CMakeLists.txt
@@ -1,4 +1,2 @@
-add_executable(adaptiveprecision-blockjacobi adaptiveprecision-blockjacobi.cpp)
-target_link_libraries(adaptiveprecision-blockjacobi ginkgo)
-target_include_directories(adaptiveprecision-blockjacobi PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("adaptiveprecision-blockjacobi")
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/custom-logger/CMakeLists.txt b/examples/custom-logger/CMakeLists.txt
index 5f15c06b0da..11b4182bbf6 100644
--- a/examples/custom-logger/CMakeLists.txt
+++ b/examples/custom-logger/CMakeLists.txt
@@ -1,6 +1,4 @@
-add_executable(custom-logger custom-logger.cpp)
-target_link_libraries(custom-logger ginkgo)
-target_include_directories(custom-logger PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("custom-logger")
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/custom-matrix-format/CMakeLists.txt b/examples/custom-matrix-format/CMakeLists.txt
index 81d8c9f7afb..1665e8e912a 100644
--- a/examples/custom-matrix-format/CMakeLists.txt
+++ b/examples/custom-matrix-format/CMakeLists.txt
@@ -2,12 +2,13 @@ if (GINKGO_BUILD_CUDA AND GINKGO_BUILD_OMP)
     enable_language(CUDA)
     set(CMAKE_CUDA_STANDARD 14)
     set(CMAKE_CUDA_STANDARD_REQUIRED ON)
-    add_executable(custom-matrix-format
-        custom-matrix-format.cpp
+    set(run_target "custom-matrix-format")
+    add_executable(${run_target}
+        ${run_target}.cpp
         stencil_kernel.cu)
-    target_link_libraries(custom-matrix-format ginkgo)
-    target_include_directories(custom-matrix-format PRIVATE
+    target_link_libraries(${run_target} ginkgo)
+    target_include_directories(${run_target} PRIVATE
         ${PROJECT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
     # workaround for clang-cuda/g++ interaction
-    set_target_properties(custom-matrix-format PROPERTIES POSITION_INDEPENDENT_CODE ON)
+    set_target_properties(${run_target} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endif()
diff --git a/examples/custom-stopping-criterion/CMakeLists.txt b/examples/custom-stopping-criterion/CMakeLists.txt
index d04f3b1a071..5fb0916afc6 100644
--- a/examples/custom-stopping-criterion/CMakeLists.txt
+++ b/examples/custom-stopping-criterion/CMakeLists.txt
@@ -1,10 +1,11 @@
+set(run_target "custom-stopping-criterion")
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
-add_executable(custom-stopping-criterion
-	custom-stopping-criterion.cpp)
-target_link_libraries(custom-stopping-criterion ginkgo
+add_executable(${run_target}
+	${run_target}.cpp)
+target_link_libraries(${run_target} ginkgo
 	Threads::Threads)
-target_include_directories(custom-stopping-criterion
+target_include_directories(${run_target}
 	PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
diff --git a/examples/external-lib-interfacing/CMakeLists.txt b/examples/external-lib-interfacing/CMakeLists.txt
index 7f97a03e630..9f2a75dcfbd 100644
--- a/examples/external-lib-interfacing/CMakeLists.txt
+++ b/examples/external-lib-interfacing/CMakeLists.txt
@@ -14,8 +14,9 @@ if(GINKGO_BUILD_EXTLIB_EXAMPLE)
     set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
 
+    set(run_target "external-lib-interfacing")
     add_executable(${PROJECT_NAME} "")
-    target_sources(${PROJECT_NAME} PRIVATE external-lib-interfacing.cpp)
+    target_sources(${PROJECT_NAME} PRIVATE ${run_target}.cpp)
     target_compile_options(${PROJECT_NAME} PRIVATE -g -Wall)
     target_compile_definitions(${PROJECT_NAME} PRIVATE OMPI_SKIP_MPICXX)
 
diff --git a/examples/ginkgo-overhead/CMakeLists.txt b/examples/ginkgo-overhead/CMakeLists.txt
index a52a2e31b3a..f8b5596796b 100644
--- a/examples/ginkgo-overhead/CMakeLists.txt
+++ b/examples/ginkgo-overhead/CMakeLists.txt
@@ -1,3 +1 @@
-add_executable(ginkgo-overhead ginkgo-overhead.cpp)
-target_link_libraries(ginkgo-overhead ginkgo)
-target_include_directories(ginkgo-overhead PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("ginkgo-overhead")
diff --git a/examples/ginkgo-ranges/CMakeLists.txt b/examples/ginkgo-ranges/CMakeLists.txt
index 34d5fd27654..6763cb0309c 100644
--- a/examples/ginkgo-ranges/CMakeLists.txt
+++ b/examples/ginkgo-ranges/CMakeLists.txt
@@ -1,3 +1 @@
-add_executable(ginkgo-ranges ginkgo-ranges.cpp)
-target_link_libraries(ginkgo-ranges ginkgo)
-target_include_directories(ginkgo-ranges PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("ginkgo-ranges")
diff --git a/examples/ilu-preconditioned-solver/CMakeLists.txt b/examples/ilu-preconditioned-solver/CMakeLists.txt
index 873dff6ace4..3df1b0cac01 100644
--- a/examples/ilu-preconditioned-solver/CMakeLists.txt
+++ b/examples/ilu-preconditioned-solver/CMakeLists.txt
@@ -1,6 +1,4 @@
-add_executable(ilu-preconditioned-solver ilu-preconditioned-solver.cpp)
-target_link_libraries(ilu-preconditioned-solver ginkgo)
-target_include_directories(ilu-preconditioned-solver PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("ilu-preconditioned-solver")
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/inverse-iteration/CMakeLists.txt b/examples/inverse-iteration/CMakeLists.txt
index 62c7fc725d0..04e037c4721 100644
--- a/examples/inverse-iteration/CMakeLists.txt
+++ b/examples/inverse-iteration/CMakeLists.txt
@@ -1,4 +1,2 @@
-add_executable(inverse-iteration inverse-iteration.cpp)
-target_link_libraries(inverse-iteration ginkgo)
-target_include_directories(inverse-iteration PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("inverse-iteration")
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/ir-ilu-preconditioned-solver/CMakeLists.txt b/examples/ir-ilu-preconditioned-solver/CMakeLists.txt
index dd77e163e59..56ca17eec40 100644
--- a/examples/ir-ilu-preconditioned-solver/CMakeLists.txt
+++ b/examples/ir-ilu-preconditioned-solver/CMakeLists.txt
@@ -1,4 +1,2 @@
-add_executable(ir-ilu-preconditioned-solver ir-ilu-preconditioned-solver.cpp)
-target_link_libraries(ir-ilu-preconditioned-solver ginkgo)
-target_include_directories(ir-ilu-preconditioned-solver PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("ir-ilu-preconditioned-solver")
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/iterative-refinement/CMakeLists.txt b/examples/iterative-refinement/CMakeLists.txt
index a21b54d2a96..5c90074b50c 100644
--- a/examples/iterative-refinement/CMakeLists.txt
+++ b/examples/iterative-refinement/CMakeLists.txt
@@ -1,4 +1,2 @@
-add_executable(iterative-refinement iterative-refinement.cpp)
-target_link_libraries(iterative-refinement ginkgo)
-target_include_directories(iterative-refinement PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("iterative-refinement")
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/minimal-cuda-solver/CMakeLists.txt b/examples/minimal-cuda-solver/CMakeLists.txt
index 60a26db0c40..4f9bfc3d1dc 100644
--- a/examples/minimal-cuda-solver/CMakeLists.txt
+++ b/examples/minimal-cuda-solver/CMakeLists.txt
@@ -1,6 +1,4 @@
-add_executable(minimal-cuda-solver minimal-cuda-solver.cpp)
-target_link_libraries(minimal-cuda-solver ginkgo)
-target_include_directories(minimal-cuda-solver PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("minimal-cuda-solver")
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/mixed-precision-ir/CMakeLists.txt b/examples/mixed-precision-ir/CMakeLists.txt
index e4f81ef2c55..a40f470449f 100644
--- a/examples/mixed-precision-ir/CMakeLists.txt
+++ b/examples/mixed-precision-ir/CMakeLists.txt
@@ -1,4 +1,2 @@
-add_executable(mixed-precision-ir mixed-precision-ir.cpp)
-target_link_libraries(mixed-precision-ir ginkgo)
-target_include_directories(mixed-precision-ir PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("mixed-precision-ir")
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/nine-pt-stencil-solver/CMakeLists.txt b/examples/nine-pt-stencil-solver/CMakeLists.txt
index 9902887e5c5..e3dfc6b78f0 100644
--- a/examples/nine-pt-stencil-solver/CMakeLists.txt
+++ b/examples/nine-pt-stencil-solver/CMakeLists.txt
@@ -1,3 +1 @@
-add_executable(nine-pt-stencil-solver nine-pt-stencil-solver.cpp)
-target_link_libraries(nine-pt-stencil-solver ginkgo)
-target_include_directories(nine-pt-stencil-solver PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("nine-pt-stencil-solver")
diff --git a/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp b/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
index 9f573ac79d3..4f594fa6f26 100644
--- a/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
+++ b/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
@@ -295,10 +295,9 @@ void solve_system(const std::string &executor_string,
 int main(int argc, char *argv[])
 {
     if (argc < 2) {
-        std::cerr << "Usage: " << argv[0] << " DISCRETIZATION_POINTS [executor]"
+        std::cout << "Usage: " << argv[0] << " DISCRETIZATION_POINTS [executor]"
                   << " [stencil_alpha] [stencil_beta] [stencil_gamma]"
                   << std::endl;
-        std::exit(-1);
     }
     using ValueType = double;
     using IndexType = int;
@@ -354,7 +353,8 @@ int main(int argc, char *argv[])
                 .count()) *
         1e-6;
 
-    print_solution(dp, u.data());
+    // Uncomment to print the solution
+    // print_solution(dp, u.data());
     std::cout << "The average relative error is "
               << calculate_error(dp, u.data(), correct_u) /
                      static_cast<gko::remove_complex<ValueType>>(dp_2)
diff --git a/examples/performance-debugging/CMakeLists.txt b/examples/performance-debugging/CMakeLists.txt
index f44e29ad39f..ebecae2d1d6 100644
--- a/examples/performance-debugging/CMakeLists.txt
+++ b/examples/performance-debugging/CMakeLists.txt
@@ -1,4 +1,2 @@
-add_executable(performance-debugging performance-debugging.cpp)
-target_link_libraries(performance-debugging ginkgo)
-target_include_directories(performance-debugging PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("performance-debugging")
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/poisson-solver/CMakeLists.txt b/examples/poisson-solver/CMakeLists.txt
index 945808cc4c6..a83dd611682 100644
--- a/examples/poisson-solver/CMakeLists.txt
+++ b/examples/poisson-solver/CMakeLists.txt
@@ -1,3 +1 @@
-add_executable(poisson-solver poisson-solver.cpp)
-target_link_libraries(poisson-solver ginkgo)
-target_include_directories(poisson-solver PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("poisson-solver")
diff --git a/examples/poisson-solver/poisson-solver.cpp b/examples/poisson-solver/poisson-solver.cpp
index 9124b13e352..7f80ce43f1a 100644
--- a/examples/poisson-solver/poisson-solver.cpp
+++ b/examples/poisson-solver/poisson-solver.cpp
@@ -123,9 +123,8 @@ int main(int argc, char *argv[])
     using bj = gko::preconditioner::Jacobi<ValueType, IndexType>;
 
     if (argc < 2) {
-        std::cerr << "Usage: " << argv[0] << " DISCRETIZATION_POINTS [executor]"
+        std::cout << "Usage: " << argv[0] << " DISCRETIZATION_POINTS [executor]"
                   << std::endl;
-        std::exit(-1);
     }
 
     // Get number of discretization points
@@ -185,7 +184,8 @@ int main(int argc, char *argv[])
         ->generate(clone(exec, matrix))  // copy the matrix to the executor
         ->apply(lend(rhs), lend(u));
 
-    print_solution<ValueType>(u0, u1, lend(u));
+    // Uncomment to print the solution
+    // print_solution<ValueType>(u0, u1, lend(u));
     std::cout << "The average relative error is "
               << calculate_error(discretization_points, lend(u), correct_u) /
                      static_cast<gko::remove_complex<ValueType>>(
diff --git a/examples/preconditioned-solver/CMakeLists.txt b/examples/preconditioned-solver/CMakeLists.txt
index 0bcbf2eeced..f80aeeee0a7 100644
--- a/examples/preconditioned-solver/CMakeLists.txt
+++ b/examples/preconditioned-solver/CMakeLists.txt
@@ -1,6 +1,4 @@
-add_executable(preconditioned-solver preconditioned-solver.cpp)
-target_link_libraries(preconditioned-solver ginkgo)
-target_include_directories(preconditioned-solver PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("preconditioned-solver")
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/preconditioner-export/CMakeLists.txt b/examples/preconditioner-export/CMakeLists.txt
index 3e35167bc9e..4a0ec3294ad 100644
--- a/examples/preconditioner-export/CMakeLists.txt
+++ b/examples/preconditioner-export/CMakeLists.txt
@@ -1,4 +1,2 @@
-add_executable(preconditioner-export preconditioner-export.cpp)
-target_link_libraries(preconditioner-export ginkgo)
-target_include_directories(preconditioner-export PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("preconditioner-export")
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/simple-solver-logging/CMakeLists.txt b/examples/simple-solver-logging/CMakeLists.txt
index f28413db303..6596011fae4 100644
--- a/examples/simple-solver-logging/CMakeLists.txt
+++ b/examples/simple-solver-logging/CMakeLists.txt
@@ -1,6 +1,4 @@
-add_executable(simple-solver-logging simple-solver-logging.cpp)
-target_link_libraries(simple-solver-logging ginkgo)
-target_include_directories(simple-solver-logging PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("simple-solver-logging")
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/simple-solver/CMakeLists.txt b/examples/simple-solver/CMakeLists.txt
index a4f54d59112..1f6a6373949 100644
--- a/examples/simple-solver/CMakeLists.txt
+++ b/examples/simple-solver/CMakeLists.txt
@@ -1,6 +1,4 @@
-add_executable(simple-solver simple-solver.cpp)
-target_link_libraries(simple-solver ginkgo)
-target_include_directories(simple-solver PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("simple-solver")
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/three-pt-stencil-solver/CMakeLists.txt b/examples/three-pt-stencil-solver/CMakeLists.txt
index 15f42fcb1c0..dd97f3a8946 100644
--- a/examples/three-pt-stencil-solver/CMakeLists.txt
+++ b/examples/three-pt-stencil-solver/CMakeLists.txt
@@ -1,3 +1 @@
-add_executable(three-pt-stencil-solver three-pt-stencil-solver.cpp)
-target_link_libraries(three-pt-stencil-solver ginkgo)
-target_include_directories(three-pt-stencil-solver PRIVATE ${PROJECT_SOURCE_DIR})
+setup_example_target("three-pt-stencil-solver")
diff --git a/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp b/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
index 7056e768ba5..6da4a4057c4 100644
--- a/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
+++ b/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
@@ -233,9 +233,8 @@ int main(int argc, char *argv[])
     using IndexType = int;
 
     if (argc < 2) {
-        std::cerr << "Usage: " << argv[0] << " DISCRETIZATION_POINTS [executor]"
+        std::cout << "Usage: " << argv[0] << " DISCRETIZATION_POINTS [executor]"
                   << std::endl;
-        std::exit(-1);
     }
 
     const IndexType discretization_points =
@@ -267,7 +266,9 @@ int main(int argc, char *argv[])
                  col_idxs.data(), values.data(), rhs.data(), u.data(),
                  reduction_factor);
 
-    print_solution<ValueType, IndexType>(discretization_points, 0, 1, u.data());
+    // Uncomment to print the solution
+    // print_solution<ValueType, IndexType>(discretization_points, 0, 1,
+    // u.data());
     std::cout << "The average relative error is "
               << calculate_error(discretization_points, u.data(), correct_u) /
                      discretization_points

From 97419a727049625b31b6c844c2166f7e37e52a20 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratik.nayak4@gmail.com>
Date: Thu, 24 Sep 2020 15:21:40 +0200
Subject: [PATCH 0025/1008] Add example runs to one CI job, minor stdout
 updates to examples.

---
 .gitlab-ci.yml                                |  10 +
 examples/CMakeLists.txt                       |  39 +-
 .../adaptiveprecision-blockjacobi.cpp         |   4 +-
 .../doc/results.dox                           |   2 +-
 examples/custom-logger/custom-logger.cpp      |   6 +-
 .../custom-matrix-format.cpp                  |   2 +-
 .../custom-stopping-criterion.cpp             |   2 +-
 examples/ginkgo-overhead/doc/results.dox      |   4 +-
 examples/ginkgo-overhead/ginkgo-overhead.cpp  |   5 +-
 .../ilu-preconditioned-solver.cpp             |   6 +-
 .../doc/results.dox                           |   2 +-
 .../ir-ilu-preconditioned-solver.cpp          |   9 +-
 examples/iterative-refinement/doc/results.dox |   2 +-
 .../iterative-refinement.cpp                  |   6 +-
 examples/mixed-precision-ir/doc/results.dox   |   2 +-
 .../mixed-precision-ir/mixed-precision-ir.cpp |   6 +-
 .../nine-pt-stencil-solver/doc/results.dox    |  19 +-
 .../nine-pt-stencil-solver.cpp                |   5 +-
 examples/papi-logging/papi-logging.cpp        |   2 +-
 .../performance-debugging.cpp                 |   2 +-
 examples/poisson-solver/doc/results.dox       |  25 +-
 examples/poisson-solver/poisson-solver.cpp    |   5 +-
 .../preconditioned-solver.cpp                 |   6 +-
 .../preconditioner-export.cpp                 |   2 +-
 .../simple-solver-logging/doc/results.dox     | 815 +++++++++++++++---
 .../simple-solver-logging.cpp                 |   6 +-
 examples/simple-solver/simple-solver.cpp      |   6 +-
 .../three-pt-stencil-solver/doc/results.dox   |  25 +-
 .../three-pt-stencil-solver.cpp               |   5 +-
 29 files changed, 807 insertions(+), 223 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a2d229f67a1..b332d501f47 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,6 +22,7 @@ stages:
   BUILD_OMP: "OFF"
   BUILD_CUDA: "OFF"
   BUILD_HIP: "OFF"
+  RUN_EXAMPLES: "OFF"
   CONFIG_LOG: "ON"
   CXX_FLAGS: ""
   EXTRA_CMAKE_FLAGS: ""
@@ -61,6 +62,7 @@ stages:
         -DGINKGO_BUILD_OMP=${BUILD_OMP} -DGINKGO_BUILD_CUDA=${BUILD_CUDA}
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
+        -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
     - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT}
   dependencies: []
@@ -87,12 +89,18 @@ stages:
         -DGINKGO_BUILD_OMP=${BUILD_OMP} -DGINKGO_BUILD_CUDA=${BUILD_CUDA}
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
+        -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
     - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT} install
     - |
         (( $(ctest -N | tail -1 | sed 's/Total Tests: //') != 0 )) || exit 1
     - ctest -V
     - ninja test_install
+    - |
+      if [ "${RUN_EXAMPLES}" == "ON" ]; then
+        ninja run_all_examples
+        ninja validate_all_examples
+      fi
   dependencies: []
   except:
       - schedules
@@ -122,6 +130,7 @@ sync:
 
 
 # Build jobs
+# Job with example runs.
 build/cuda90/gcc/all/debug/shared:
   <<: *default_build_with_test
   image: localhost:5000/gko-cuda90-gnu5-llvm39
@@ -131,6 +140,7 @@ build/cuda90/gcc/all/debug/shared:
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     BUILD_TYPE: "Debug"
+    RUN_EXAMPLES: "ON"
     CUDA_ARCH: 35
   only:
     variables:
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index f5413acb04d..b1f4abf622e 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -7,8 +7,10 @@ function(setup_example_target target_name)
     target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 endfunction()
 
+set(EXAMPLES_EXEC_LIST "adaptiveprecision-blockjacobi;custom-logger;ginkgo-overhead;ginkgo-ranges;ilu-preconditioned-solver;ir-ilu-preconditioned-solver;inverse-iteration;iterative-refinement;mixed-precision-ir;nine-pt-stencil-solver;poisson-solver;preconditioned-solver;simple-solver;three-pt-stencil-solver;")
+
+set(EXAMPLES_LIST "${EXAMPLES_EXEC_LIST};custom-matrix-format;custom-stopping-criterion;minimal-cuda-solver;papi-logging;performance-debugging;preconditioner-export;simple-solver-logging")
 
-set(EXAMPLES_LIST "custom-logger;custom-matrix-format;custom-stopping-criterion;adaptiveprecision-blockjacobi;ginkgo-overhead;ginkgo-ranges;ilu-preconditioned-solver;ir-ilu-preconditioned-solver;inverse-iteration;iterative-refinement;minimal-cuda-solver;mixed-precision-ir;nine-pt-stencil-solver;papi-logging;performance-debugging;poisson-solver;preconditioned-solver;preconditioner-export;simple-solver;simple-solver-logging;three-pt-stencil-solver;")
 if(GINKGO_BUILD_EXTLIB_EXAMPLE)
     set(EXAMPLES_LIST "${EXAMPLES_LIST};external-lib-interfacing")
 endif()
@@ -24,4 +26,39 @@ if(GINKGO_RUN_EXAMPLES)
             COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example} > ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example}.out
             WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${example})
     endforeach()
+    foreach(example ${EXAMPLES_LIST})
+        file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command "#!/bin/bash
+diff <(sed -n '7,$p' ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example}.out | sed -E 's/([0-9]+.)//g') <(sed -n '6,$p' ${CMAKE_SOURCE_DIR}/examples/${example}/doc/results.dox | head -n -4 | sed -E 's/([0-9]+.)//g')")
+        add_custom_target("diff-${example}"
+            COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command && ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${example})
+    endforeach()
+    file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/run_all_examples.sh
+        "NUM_EXAMPLES=$(wc -l < example-targets.txt)\n"
+        "for i in $(seq 1 $NUM_EXAMPLES)\n"
+        "do\n"
+        "  target_name=$(head -n $i example-targets.txt | tail -1)\n"
+        "  ex_target=run-\$\{target_name\}\n"
+        "  cmake --build ${CMAKE_BINARY_DIR} --target $ex_target\n"
+        "done\n")
+    add_custom_target("run_all_examples"
+        COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/run_all_examples.sh && ${CMAKE_CURRENT_BINARY_DIR}/run_all_examples.sh
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+    file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/validate_all_examples.sh
+        "NUM_EXAMPLES=$(wc -l < example-targets.txt)\n"
+        "for i in $(seq 1 $NUM_EXAMPLES)\n"
+        "do\n"
+        "  target_name=$(head -n $i example-targets.txt | tail -1)\n"
+        "  ex_target=diff-\$\{target_name\}\n"
+        "  cmake --build ${CMAKE_BINARY_DIR} --target $ex_target\n"
+        "done\n")
+    add_custom_target("validate_all_examples"
+        COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/validate_all_examples.sh && ${CMAKE_CURRENT_BINARY_DIR}/validate_all_examples.sh
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 endif()
+
+set(OUT_FILE "${CMAKE_BINARY_DIR}/examples/example-targets.txt")
+file(WRITE ${OUT_FILE} "")
+foreach(example ${EXAMPLES_EXEC_LIST})
+    file(APPEND ${OUT_FILE} "${example}\n")
+endforeach()
diff --git a/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp b/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
index e64107915f8..f00ae67236f 100644
--- a/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
+++ b/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
@@ -135,9 +135,9 @@ int main(int argc, char *argv[])
     A->apply(lend(one), lend(x), lend(neg_one), lend(b));
     b->compute_norm2(lend(res));
 
-    std::cout << "Initial residual norm sqrt(r^T r): \n";
+    std::cout << "Initial residual norm sqrt(r^T r):\n";
     write(std::cout, lend(initres));
-    std::cout << "Final residual norm sqrt(r^T r): \n";
+    std::cout << "Final residual norm sqrt(r^T r):\n";
     write(std::cout, lend(res));
 
     // Print solver statistics
diff --git a/examples/adaptiveprecision-blockjacobi/doc/results.dox b/examples/adaptiveprecision-blockjacobi/doc/results.dox
index 22a3a04c827..d3db070f8fa 100644
--- a/examples/adaptiveprecision-blockjacobi/doc/results.dox
+++ b/examples/adaptiveprecision-blockjacobi/doc/results.dox
@@ -12,7 +12,7 @@ Final residual norm sqrt(r^T r):
 1 1
 5.69384e-06
 CG iteration count:     5
-CG execution time [ms]: 2.04779
+CG execution time [ms]: 0.080041
 
 @endcode
 
diff --git a/examples/custom-logger/custom-logger.cpp b/examples/custom-logger/custom-logger.cpp
index e2f73ed4825..b3f526e0ba6 100644
--- a/examples/custom-logger/custom-logger.cpp
+++ b/examples/custom-logger/custom-logger.cpp
@@ -232,7 +232,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
     } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+        std::cerr << "Usage: executable [executor]" << std::endl;
         std::exit(-1);
     }
 
@@ -292,7 +292,7 @@ int main(int argc, char *argv[])
     solver->apply(gko::lend(b), gko::lend(x));
 
     // Print the solution to the command line.
-    std::cout << "Solution (x): \n";
+    std::cout << "Solution (x):\n";
     write(std::cout, gko::lend(x));
 
     // Print the table of the residuals obtained from the logger
@@ -311,6 +311,6 @@ int main(int argc, char *argv[])
     A->apply(gko::lend(one), gko::lend(x), gko::lend(neg_one), gko::lend(b));
     b->compute_norm2(gko::lend(res));
 
-    std::cout << "Residual norm sqrt(r^T r): \n";
+    std::cout << "Residual norm sqrt(r^T r):\n";
     write(std::cout, gko::lend(res));
 }
diff --git a/examples/custom-matrix-format/custom-matrix-format.cpp b/examples/custom-matrix-format/custom-matrix-format.cpp
index 3f73aeafca1..57eb4bac9c1 100644
--- a/examples/custom-matrix-format/custom-matrix-format.cpp
+++ b/examples/custom-matrix-format/custom-matrix-format.cpp
@@ -242,7 +242,7 @@ int main(int argc, char *argv[])
     using cg = gko::solver::Cg<ValueType>;
 
     if (argc < 2) {
-        std::cerr << "Usage: " << argv[0] << " DISCRETIZATION_POINTS [executor]"
+        std::cerr << "Usage: executable DISCRETIZATION_POINTS [executor]"
                   << std::endl;
         std::exit(-1);
     }
diff --git a/examples/custom-stopping-criterion/custom-stopping-criterion.cpp b/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
index b9a4519a027..eea5d5ce583 100644
--- a/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
+++ b/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
@@ -155,7 +155,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+        std::cerr << "Usage: executable [executor]" << std::endl;
         std::exit(-1);
     }
 
diff --git a/examples/ginkgo-overhead/doc/results.dox b/examples/ginkgo-overhead/doc/results.dox
index 7386ccd43da..431c19e88c5 100644
--- a/examples/ginkgo-overhead/doc/results.dox
+++ b/examples/ginkgo-overhead/doc/results.dox
@@ -3,8 +3,8 @@ This is the expected output:
 
 @code{.cpp}
 
-Running 1000000 iterations of the CG solver took a total of 1.62987 seconds.
-	Average library overhead:     1629.87 [nanoseconds / iteration]
+Running 1000000 iterations of the CG solver took a total of 1.60337 seconds.
+	Average library overhead:     1603.37 [nanoseconds / iteration]
 
 @endcode
 
diff --git a/examples/ginkgo-overhead/ginkgo-overhead.cpp b/examples/ginkgo-overhead/ginkgo-overhead.cpp
index b8bc7acc2b1..f087cba22e4 100644
--- a/examples/ginkgo-overhead/ginkgo-overhead.cpp
+++ b/examples/ginkgo-overhead/ginkgo-overhead.cpp
@@ -38,8 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <iostream>
 
 
-[[noreturn]] void print_usage_and_exit(const char *name)
-{
+[[noreturn]] void print_usage_and_exit(const char *name) {
     std::cerr << "Usage: " << name << " [NUM_ITERS]" << std::endl;
     std::exit(-1);
 }
@@ -56,7 +55,7 @@ int main(int argc, char *argv[])
 
     long unsigned num_iters = 1000000;
     if (argc > 2) {
-        print_usage_and_exit(argv[0]);
+        print_usage_and_exit("executable");
     }
     if (argc == 2) {
         num_iters = std::atol(argv[1]);
diff --git a/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp b/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
index 55d2946a123..91153157fde 100644
--- a/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
+++ b/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
@@ -68,7 +68,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+        std::cerr << "Usage: executable [executor]" << std::endl;
         std::exit(-1);
     }
 
@@ -116,7 +116,7 @@ int main(int argc, char *argv[])
     ilu_gmres->apply(gko::lend(b), gko::lend(x));
 
     // Print solution
-    std::cout << "Solution (x): \n";
+    std::cout << "Solution (x):\n";
     write(std::cout, gko::lend(x));
 
     // Calculate residual
@@ -126,6 +126,6 @@ int main(int argc, char *argv[])
     A->apply(gko::lend(one), gko::lend(x), gko::lend(neg_one), gko::lend(b));
     b->compute_norm2(gko::lend(res));
 
-    std::cout << "Residual norm sqrt(r^T r): \n";
+    std::cout << "Residual norm sqrt(r^T r):\n";
     write(std::cout, gko::lend(res));
 }
diff --git a/examples/ir-ilu-preconditioned-solver/doc/results.dox b/examples/ir-ilu-preconditioned-solver/doc/results.dox
index 37ddff94b2e..d42b40af236 100644
--- a/examples/ir-ilu-preconditioned-solver/doc/results.dox
+++ b/examples/ir-ilu-preconditioned-solver/doc/results.dox
@@ -27,7 +27,7 @@ Solution (x):
 0.0121141
 0.0123025
 GMRES iteration count:     8
-GMRES execution time [ms]: 3.89406
+GMRES execution time [ms]: 0.377673
 Residual norm sqrt(r^T r):
 %%MatrixMarket matrix array real general
 1 1
diff --git a/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp b/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
index 704794fd638..9b548611c41 100644
--- a/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
+++ b/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
@@ -70,8 +70,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: " << argv[0] << " [executor] [sweeps]"
-                  << std::endl;
+        std::cerr << "Usage: executable [executor] [sweeps]" << std::endl;
         std::exit(-1);
     }
     unsigned int sweeps = (argc == 3) ? atoi(argv[2]) : 5u;
@@ -165,10 +164,10 @@ int main(int argc, char *argv[])
         time += std::chrono::duration_cast<std::chrono::nanoseconds>(toc - tic);
     }
 
-    std::cout << "Using " << sweeps << " block-Jacobi sweeps. \n";
+    std::cout << "Using " << sweeps << " block-Jacobi sweeps.\n";
 
     // Print solution
-    std::cout << "Solution (x): \n";
+    std::cout << "Solution (x):\n";
     write(std::cout, gko::lend(x));
 
     // Calculate residual
@@ -182,6 +181,6 @@ int main(int argc, char *argv[])
               << "\n";
     std::cout << "GMRES execution time [ms]: "
               << static_cast<double>(time.count()) / 100000000.0 << "\n";
-    std::cout << "Residual norm sqrt(r^T r): \n";
+    std::cout << "Residual norm sqrt(r^T r):\n";
     write(std::cout, gko::lend(res));
 }
diff --git a/examples/iterative-refinement/doc/results.dox b/examples/iterative-refinement/doc/results.dox
index d15a5ca9f81..ebdbaa49b98 100644
--- a/examples/iterative-refinement/doc/results.dox
+++ b/examples/iterative-refinement/doc/results.dox
@@ -12,7 +12,7 @@ Final residual norm sqrt(r^T r):
 1 1
 4.23821e-11
 IR iteration count:     24
-IR execution time [ms]: 14.9084
+IR execution time [ms]: 0.794962
 
 @endcode
 
diff --git a/examples/iterative-refinement/iterative-refinement.cpp b/examples/iterative-refinement/iterative-refinement.cpp
index e0a7131f5d7..eb4b2f1440f 100644
--- a/examples/iterative-refinement/iterative-refinement.cpp
+++ b/examples/iterative-refinement/iterative-refinement.cpp
@@ -68,7 +68,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+        std::cerr << "Usage: executable [executor]" << std::endl;
         std::exit(-1);
     }
 
@@ -138,9 +138,9 @@ int main(int argc, char *argv[])
     A->apply(lend(one), lend(x), lend(neg_one), lend(b));
     b->compute_norm2(lend(res));
 
-    std::cout << "Initial residual norm sqrt(r^T r): \n";
+    std::cout << "Initial residual norm sqrt(r^T r):\n";
     write(std::cout, lend(initres));
-    std::cout << "Final residual norm sqrt(r^T r): \n";
+    std::cout << "Final residual norm sqrt(r^T r):\n";
     write(std::cout, lend(res));
 
     // Print solver statistics
diff --git a/examples/mixed-precision-ir/doc/results.dox b/examples/mixed-precision-ir/doc/results.dox
index 6cbe205c826..481cd38b641 100644
--- a/examples/mixed-precision-ir/doc/results.dox
+++ b/examples/mixed-precision-ir/doc/results.dox
@@ -12,7 +12,7 @@ Final residual norm sqrt(r^T r):
 1 1
 1.22728e-10
 MPIR iteration count:     25
-MPIR execution time [ms]: 14.2256
+MPIR execution time [ms]: 0.846559
 
 @endcode
 
diff --git a/examples/mixed-precision-ir/mixed-precision-ir.cpp b/examples/mixed-precision-ir/mixed-precision-ir.cpp
index e05ee0aa674..6b59b22ecf9 100644
--- a/examples/mixed-precision-ir/mixed-precision-ir.cpp
+++ b/examples/mixed-precision-ir/mixed-precision-ir.cpp
@@ -76,7 +76,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+        std::cerr << "Usage: executable [executor]" << std::endl;
         std::exit(-1);
     }
 
@@ -168,9 +168,9 @@ int main(int argc, char *argv[])
     A->apply(lend(one), lend(x), lend(neg_one), lend(b));
     b->compute_norm2(lend(res_vec));
 
-    std::cout << "Initial residual norm sqrt(r^T r): \n";
+    std::cout << "Initial residual norm sqrt(r^T r):\n";
     write(std::cout, lend(initres_vec));
-    std::cout << "Final residual norm sqrt(r^T r): \n";
+    std::cout << "Final residual norm sqrt(r^T r):\n";
     write(std::cout, lend(res_vec));
 
     // Print solver statistics
diff --git a/examples/nine-pt-stencil-solver/doc/results.dox b/examples/nine-pt-stencil-solver/doc/results.dox
index 352c92c070a..dfda3387d0c 100644
--- a/examples/nine-pt-stencil-solver/doc/results.dox
+++ b/examples/nine-pt-stencil-solver/doc/results.dox
@@ -1,22 +1,11 @@
 <h1>Results</h1>
-
-The expected output of the relative error at K=10 should be
+The expected output should be
 
 @code{.cpp}
 
-0.00150263 0.00676184 0.0210368 0.0488355 0.0946657 0.163035 0.258452 0.385425 0.54846 0.752066
-0.00676184 0.012021 0.026296 0.0540947 0.0999249 0.168295 0.263712 0.390684 0.553719 0.757325
-0.0210368 0.026296 0.040571 0.0683697 0.1142 0.18257 0.277987 0.404959 0.567994 0.7716
-0.0488354 0.0540947 0.0683697 0.0961683 0.141998 0.210368 0.305785 0.432757 0.595793 0.799399
-0.0946656 0.0999248 0.1142 0.141998 0.187829 0.256198 0.351615 0.478588 0.641623 0.845229
-0.163035 0.168295 0.182569 0.210368 0.256198 0.324568 0.419985 0.546957 0.709993 0.913599
-0.258452 0.263711 0.277987 0.305785 0.351615 0.419985 0.515402 0.642374 0.80541 1.00902
-0.385424 0.390684 0.404959 0.432757 0.478588 0.546957 0.642374 0.769346 0.932382 1.13599
-0.54846 0.553719 0.567994 0.595793 0.641623 0.709992 0.805409 0.932382 1.09542 1.29902
-0.752066 0.757325 0.7716 0.799399 0.845229 0.913599 1.00902 1.13599 1.29902 1.50263
-
-The average relative error is 1.4283e-07
-The runtime is 5.784994 ms
+Usage: executable DISCRETIZATION_POINTS [executor] [stencil_alpha] [stencil_beta] [stencil_gamma]
+The average relative error is 6.35715e-06
+The runtime is 167.320520 ms
 
 @endcode
 
diff --git a/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp b/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
index 4f594fa6f26..3db466f9015 100644
--- a/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
+++ b/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
@@ -294,8 +294,11 @@ void solve_system(const std::string &executor_string,
 
 int main(int argc, char *argv[])
 {
+    // Print version information
+    std::cout << gko::version_info::get() << std::endl;
+
     if (argc < 2) {
-        std::cout << "Usage: " << argv[0] << " DISCRETIZATION_POINTS [executor]"
+        std::cout << "Usage: executable DISCRETIZATION_POINTS [executor]"
                   << " [stencil_alpha] [stencil_beta] [stencil_gamma]"
                   << std::endl;
     }
diff --git a/examples/papi-logging/papi-logging.cpp b/examples/papi-logging/papi-logging.cpp
index 5d8019cc519..c36e80215da 100644
--- a/examples/papi-logging/papi-logging.cpp
+++ b/examples/papi-logging/papi-logging.cpp
@@ -151,7 +151,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+        std::cerr << "Usage: executable [executor]" << std::endl;
         std::exit(-1);
     }
 
diff --git a/examples/performance-debugging/performance-debugging.cpp b/examples/performance-debugging/performance-debugging.cpp
index 573346531cf..4d71b92a326 100644
--- a/examples/performance-debugging/performance-debugging.cpp
+++ b/examples/performance-debugging/performance-debugging.cpp
@@ -378,7 +378,7 @@ int main(int argc, char *argv[])
                gko::CudaExecutor::get_num_devices() > 0) {
         exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        print_usage(argv[0]);
+        print_usage("executable");
     }
 
     // Read the input matrix file directory
diff --git a/examples/poisson-solver/doc/results.dox b/examples/poisson-solver/doc/results.dox
index 5e093c1ac2f..6dd4180b920 100644
--- a/examples/poisson-solver/doc/results.dox
+++ b/examples/poisson-solver/doc/results.dox
@@ -3,29 +3,8 @@ This is the expected output:
 
 @code{.cpp}
 
-0
-0.00010798
-0.000863838
-0.00291545
-0.0069107
-0.0134975
-0.0233236
-0.037037
-0.0552856
-0.0787172
-0.10798
-0.143721
-0.186589
-0.237231
-0.296296
-0.364431
-0.442285
-0.530504
-0.629738
-0.740633
-0.863838
-1
-The average relative error is 1.87318e-15
+Usage: executable DISCRETIZATION_POINTS [executor]
+The average relative error is 2.52236e-11
 
 @endcode
 
diff --git a/examples/poisson-solver/poisson-solver.cpp b/examples/poisson-solver/poisson-solver.cpp
index 7f80ce43f1a..458178e505b 100644
--- a/examples/poisson-solver/poisson-solver.cpp
+++ b/examples/poisson-solver/poisson-solver.cpp
@@ -122,8 +122,11 @@ int main(int argc, char *argv[])
     using cg = gko::solver::Cg<ValueType>;
     using bj = gko::preconditioner::Jacobi<ValueType, IndexType>;
 
+    // Print version information
+    std::cout << gko::version_info::get() << std::endl;
+
     if (argc < 2) {
-        std::cout << "Usage: " << argv[0] << " DISCRETIZATION_POINTS [executor]"
+        std::cout << "Usage: executable DISCRETIZATION_POINTS [executor]"
                   << std::endl;
     }
 
diff --git a/examples/preconditioned-solver/preconditioned-solver.cpp b/examples/preconditioned-solver/preconditioned-solver.cpp
index 142614c73b0..181b3a1b01d 100644
--- a/examples/preconditioned-solver/preconditioned-solver.cpp
+++ b/examples/preconditioned-solver/preconditioned-solver.cpp
@@ -68,7 +68,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+        std::cerr << "Usage: executable [executor]" << std::endl;
         std::exit(-1);
     }
 
@@ -97,7 +97,7 @@ int main(int argc, char *argv[])
     solver->apply(lend(b), lend(x));
 
     // Print solution
-    std::cout << "Solution (x): \n";
+    std::cout << "Solution (x):\n";
     write(std::cout, lend(x));
 
     // Calculate residual
@@ -107,6 +107,6 @@ int main(int argc, char *argv[])
     A->apply(lend(one), lend(x), lend(neg_one), lend(b));
     b->compute_norm2(lend(res));
 
-    std::cout << "Residual norm sqrt(r^T r): \n";
+    std::cout << "Residual norm sqrt(r^T r):\n";
     write(std::cout, lend(res));
 }
diff --git a/examples/preconditioner-export/preconditioner-export.cpp b/examples/preconditioner-export/preconditioner-export.cpp
index 866b0e051f9..ee69235e5c0 100644
--- a/examples/preconditioner-export/preconditioner-export.cpp
+++ b/examples/preconditioner-export/preconditioner-export.cpp
@@ -81,7 +81,7 @@ int main(int argc, char *argv[])
 {
     // print usage message
     if (argc < 2 || executors.find(argv[1]) == executors.end()) {
-        std::cerr << "Usage: " << argv[0]
+        std::cerr << "Usage: executable"
                   << " <reference|omp|cuda|hip> [<matrix-file>] "
                      "[<jacobi|ilu|parilu|parilut|ilu-isai|parilu-isai|parilut-"
                      "isai] [<preconditioner args>]\n";
diff --git a/examples/simple-solver-logging/doc/results.dox b/examples/simple-solver-logging/doc/results.dox
index e77175e5f46..e70253b2079 100644
--- a/examples/simple-solver-logging/doc/results.dox
+++ b/examples/simple-solver-logging/doc/results.dox
@@ -3,96 +3,679 @@ This is the expected output:
 
 @code{.cpp}
 
-[LOG] >>> apply started on A LinOp[gko::solver::Cg<double>,0x562525b9cad0] with b LinOp[gko::matrix::Dense<double>,0x562525b9d670] and x LinOp[gko::matrix::Dense<double>,0x562525b9dca0]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[8]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9c350] with Bytes[8]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[8]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9c0f0] with Bytes[8]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[152]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba2a30] with Bytes[152]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[152]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba2c30] with Bytes[152]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[152]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba2e30] with Bytes[152]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[152]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3030] with Bytes[152]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[8]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3010] with Bytes[8]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[8]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3210] with Bytes[8]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[8]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3390] with Bytes[8]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[8]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3510] with Bytes[8]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[1]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3690] with Bytes[1]
-[LOG] >>> Operation[gko::solver::cg::initialize_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7fffc24b0d30] started on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> Operation[gko::solver::cg::initialize_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7fffc24b0d30] completed on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> Operation[gko::matrix::csr::advanced_spmv_operation<gko::matrix::Dense<double> const*, gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7fffc24b0a80] started on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> Operation[gko::matrix::csr::advanced_spmv_operation<gko::matrix::Dense<double> const*, gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7fffc24b0a80] completed on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[2]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba36d0] with Bytes[2]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[8]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba4360] with Bytes[8]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[8]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba43a0] with Bytes[8]
-[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7fffc24b0710] started on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7fffc24b0710] completed on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x562525b99ec0] to Executor[gko::ReferenceExecutor,0x562525b99ec0] from Location[0x562525ba2a30] to Location[0x562525ba2c30] with Bytes[152]
-[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x562525b99ec0] to Executor[gko::ReferenceExecutor,0x562525b99ec0] from Location[0x562525ba2a30] to Location[0x562525ba2c30] with Bytes[152]
-[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7fffc24b0a80] started on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7fffc24b0a80] completed on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> iteration 0 completed with solver LinOp[gko::solver::Cg<double>,0x562525b9cad0] with residual LinOp[gko::matrix::Dense<double>,0x562525b9e470], solution LinOp[gko::matrix::Dense<double>,0x562525b9dca0] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x562525ba42a0] at iteration 0 with ID 1 and finalized set to 1
-[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7fffc24b0800] started on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7fffc24b0800] completed on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7fffc24b0a00] started on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7fffc24b0a00] completed on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x562525ba42a0] at iteration 0 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[152]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba4790] with Bytes[152]
-
-.
-.
-.
-.
-.
-.
-
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba4830]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba4830]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba4790]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba4790]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba36d0]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba36d0]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba43a0]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba43a0]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba4360]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba4360]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3690]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3690]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3390]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3390]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3510]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3510]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3210]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3210]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3010]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3010]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3030]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba3030]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba2e30]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba2e30]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba2c30]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba2c30]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba2a30]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba2a30]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9c0f0]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9c0f0]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9c350]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9c350]
-[LOG] >>> apply completed on A LinOp[gko::solver::Cg<double>,0x562525b9cad0] with b LinOp[gko::matrix::Dense<double>,0x562525b9d670] and x LinOp[gko::matrix::Dense<double>,0x562525b9dca0]
-Last memory copied was of size 98 FROM executor 0x562525b99ec0 pointer 562525b9de80 TO executor 0x562525b99ec0 pointer 562525ba5170
+[LOG] >>> apply started on A LinOp[gko::solver::Cg<double>,0x2142d60] with b LinOp[gko::matrix::Dense<double>,0x2142140] and x LinOp[gko::matrix::Dense<double>,0x2143450]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[8]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2142280] with Bytes[8]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[8]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2143410] with Bytes[8]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21480a0] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21482f0] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21484d0] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21486b0] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[8]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148010] with Bytes[8]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[8]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148a60] with Bytes[8]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[8]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21482b0] with Bytes[8]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[8]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148a40] with Bytes[8]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[1]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2147c90] with Bytes[1]
+[LOG] >>> Operation[gko::solver::cg::initialize_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::initialize_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::advanced_spmv_operation<gko::matrix::Dense<double> const*, gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14aa0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::advanced_spmv_operation<gko::matrix::Dense<double> const*, gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14aa0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[2]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148ee0] with Bytes[2]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[8]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148e50] with Bytes[8]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[8]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2147ce0] with Bytes[8]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14a20] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14a20] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 0 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 0 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 0 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149550] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149550] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149550] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149730] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149730] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149730] with Bytes[152]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 1 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 1 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 1 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149980] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149980] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149980] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149b80] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149b80] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149b80] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149730]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149730]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149550]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149550]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 2 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 2 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 2 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149290] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149290] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149290] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149690] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149690] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149690] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149b80]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149b80]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149980]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149980]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 3 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 3 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 3 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149890] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149890] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149890] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149ae0] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149ae0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149ae0] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149690]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149690]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149290]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149290]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 4 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 4 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 4 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149200] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149200] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149200] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149310] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149310] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149310] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149ae0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149ae0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149890]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149890]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 5 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 5 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 5 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149890] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149890] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149890] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149cc0] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149cc0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149cc0] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149310]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149310]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149200]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149200]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 6 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 6 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 6 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149450] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149450] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149450] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21494f0] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x21494f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x21494f0] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149cc0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149cc0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149890]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149890]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 7 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 7 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 7 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149730] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149730] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149730] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21497d0] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x21497d0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x21497d0] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21494f0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21494f0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149450]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149450]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 8 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 8 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 8 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149200] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149200] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149200] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21492a0] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x21492a0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x21492a0] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21497d0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21497d0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149730]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149730]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 9 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 9 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 9 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149620] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149620] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149620] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21496c0] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x21496c0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x21496c0] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21492a0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21492a0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149200]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149200]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 10 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 10 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 10 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149450] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149450] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149450] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149760] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149760] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149760] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21496c0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21496c0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149620]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149620]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 11 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 11 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 11 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149860] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149860] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149860] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149900] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149900] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149900] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149760]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149760]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149450]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149450]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 12 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 12 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 12 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21499a0] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21499a0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21499a0] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21493d0] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x21493d0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x21493d0] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149900]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149900]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149860]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149860]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 13 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 13 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 13 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149490] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149490] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149490] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149580] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149580] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149580] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21493d0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21493d0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21499a0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21499a0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 14 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 14 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 14 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149b50] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149b50] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149b50] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21499c0] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x21499c0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x21499c0] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149580]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149580]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149490]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149490]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 15 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 15 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 15 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149a70] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149a70] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149a70] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149340] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149340] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149340] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21499c0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21499c0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149b50]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149b50]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 16 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 16 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 16 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149970] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149970] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149970] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149b10] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149b10] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149b10] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149340]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149340]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149a70]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149a70]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 17 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 17 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 17 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149780] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149780] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149780] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149890] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149890] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149890] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149b10]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149b10]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149970]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149970]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 18 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 18 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 18 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149620] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149620] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149620] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149cf0] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149cf0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149cf0] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149890]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149890]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149780]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149780]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_1_operation<gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::spmv_operation<gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14b80] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::solver::cg::step_2_operation<gko::matrix::Dense<double>*&, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::matrix::Dense<double>*, gko::Array<gko::stopping_status>*>,0x7ffd93d14ef0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21482f0] with Bytes[152]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> iteration 19 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 19 with ID 1 and finalized set to 1
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 19 with ID 1 and finalized set to 1. It changed one RHS 1, stopped the iteration process 1
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149890] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149890] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149890] with Bytes[152]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149340] with Bytes[152]
+[LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149340] with Bytes[152]
+[LOG] >>> copy completed from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x2143e90] to Location[0x2149340] with Bytes[152]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149cf0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149cf0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149620]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149620]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148ee0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148ee0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2147ce0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2147ce0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148e50]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148e50]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2147c90]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2147c90]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21482b0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21482b0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148a40]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148a40]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148a60]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148a60]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148010]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2148010]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21486b0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21486b0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21484d0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21484d0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21482f0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21482f0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21480a0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21480a0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2143410]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2143410]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2142280]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2142280]
+[LOG] >>> apply completed on A LinOp[gko::solver::Cg<double>,0x2142d60] with b LinOp[gko::matrix::Dense<double>,0x2142140] and x LinOp[gko::matrix::Dense<double>,0x2143450]
+Last memory copied was of size 98 FROM executor 0x21400d0 pointer 2143e90 TO executor 0x21400d0 pointer 2149340
 Residual = [
     8.1654e-19
     -1.51449e-17
@@ -136,36 +719,36 @@ Solution (x):
 0.0107016
 0.0121141
 0.0123025
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[8]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba5720] with Bytes[8]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[8]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba2e10] with Bytes[8]
-[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x562525b99ec0] with Bytes[8]
-[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba36b0] with Bytes[8]
-[LOG] >>> Operation[gko::matrix::csr::advanced_spmv_operation<gko::matrix::Dense<double> const*, gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7fffc24b0d10] started on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> Operation[gko::matrix::csr::advanced_spmv_operation<gko::matrix::Dense<double> const*, gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7fffc24b0d10] completed on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7fffc24b0d60] started on Executor[gko::ReferenceExecutor,0x562525b99ec0]
-[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7fffc24b0d60] completed on Executor[gko::ReferenceExecutor,0x562525b99ec0]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[8]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149bb0] with Bytes[8]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[8]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149870] with Bytes[8]
+[LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[8]
+[LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149500] with Bytes[8]
+[LOG] >>> Operation[gko::matrix::csr::advanced_spmv_operation<gko::matrix::Dense<double> const*, gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14e50] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::csr::advanced_spmv_operation<gko::matrix::Dense<double> const*, gko::matrix::Csr<double, int> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14e50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14f70] started on Executor[gko::ReferenceExecutor,0x21400d0]
+[LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14f70] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 Residual norm sqrt(r^T r):
 %%MatrixMarket matrix array real general
 1 1
 2.10788e-15
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba36b0]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba36b0]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba2e10]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba2e10]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba5720]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525ba5720]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9de80]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9de80]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9dde0]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9dde0]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9d1a0]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9d1a0]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9d3c0]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9d3c0]
-[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9dfb0]
-[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x562525b99ec0] at Location[0x562525b9dfb0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149500]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149500]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149870]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149870]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149bb0]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149bb0]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2143e90]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2143e90]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2143590]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2143590]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2142b10]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2142b10]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2143c30]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2143c30]
+[LOG] >>> free started on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2143790]
+[LOG] >>> free completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2143790]
 
 @endcode
 
diff --git a/examples/simple-solver-logging/simple-solver-logging.cpp b/examples/simple-solver-logging/simple-solver-logging.cpp
index 72739146881..e752324ba91 100644
--- a/examples/simple-solver-logging/simple-solver-logging.cpp
+++ b/examples/simple-solver-logging/simple-solver-logging.cpp
@@ -85,7 +85,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+        std::cerr << "Usage: executable [executor]" << std::endl;
         std::exit(-1);
     }
 
@@ -169,7 +169,7 @@ int main(int argc, char *argv[])
     print_vector("Residual", residual_d);
 
     // Print solution
-    std::cout << "Solution (x): \n";
+    std::cout << "Solution (x):\n";
     write(std::cout, lend(x));
 
     // Calculate residual
@@ -179,6 +179,6 @@ int main(int argc, char *argv[])
     A->apply(lend(one), lend(x), lend(neg_one), lend(b));
     b->compute_norm2(lend(res));
 
-    std::cout << "Residual norm sqrt(r^T r): \n";
+    std::cout << "Residual norm sqrt(r^T r):\n";
     write(std::cout, lend(res));
 }
diff --git a/examples/simple-solver/simple-solver.cpp b/examples/simple-solver/simple-solver.cpp
index 092b5ec4a5a..f07b6f130e7 100644
--- a/examples/simple-solver/simple-solver.cpp
+++ b/examples/simple-solver/simple-solver.cpp
@@ -87,7 +87,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+        std::cerr << "Usage: executable [executor]" << std::endl;
         std::exit(-1);
     }
 
@@ -137,7 +137,7 @@ int main(int argc, char *argv[])
     solver->apply(lend(b), lend(x));
 
     // Print the solution to the command line.
-    std::cout << "Solution (x): \n";
+    std::cout << "Solution (x):\n";
     write(std::cout, lend(x));
 
     // To measure if your solution has actually converged, you can measure the
@@ -153,6 +153,6 @@ int main(int argc, char *argv[])
     A->apply(lend(one), lend(x), lend(neg_one), lend(b));
     b->compute_norm2(lend(res));
 
-    std::cout << "Residual norm sqrt(r^T r): \n";
+    std::cout << "Residual norm sqrt(r^T r):\n";
     write(std::cout, lend(res));
 }
diff --git a/examples/three-pt-stencil-solver/doc/results.dox b/examples/three-pt-stencil-solver/doc/results.dox
index 5e093c1ac2f..6dd4180b920 100644
--- a/examples/three-pt-stencil-solver/doc/results.dox
+++ b/examples/three-pt-stencil-solver/doc/results.dox
@@ -3,29 +3,8 @@ This is the expected output:
 
 @code{.cpp}
 
-0
-0.00010798
-0.000863838
-0.00291545
-0.0069107
-0.0134975
-0.0233236
-0.037037
-0.0552856
-0.0787172
-0.10798
-0.143721
-0.186589
-0.237231
-0.296296
-0.364431
-0.442285
-0.530504
-0.629738
-0.740633
-0.863838
-1
-The average relative error is 1.87318e-15
+Usage: executable DISCRETIZATION_POINTS [executor]
+The average relative error is 2.52236e-11
 
 @endcode
 
diff --git a/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp b/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
index 6da4a4057c4..b539490d14a 100644
--- a/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
+++ b/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
@@ -232,8 +232,11 @@ int main(int argc, char *argv[])
     using ValueType = double;
     using IndexType = int;
 
+    // Print version information
+    std::cout << gko::version_info::get() << std::endl;
+
     if (argc < 2) {
-        std::cout << "Usage: " << argv[0] << " DISCRETIZATION_POINTS [executor]"
+        std::cout << "Usage: executable DISCRETIZATION_POINTS [executor]"
                   << std::endl;
     }
 

From f1ae65f408c7021915d8d07cb27601e625ee3eeb Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratik.nayak4@gmail.com>
Date: Mon, 28 Sep 2020 10:47:06 +0200
Subject: [PATCH 0026/1008] Review update.

+ Make example builds independent.
+ Use the correct Ginkgo::ginkgo target.
+ Use add_dependencies for the validate_all and run_all targets.
---
 examples/CMakeLists.txt                       | 43 +++----------------
 .../CMakeLists.txt                            |  5 ++-
 examples/custom-logger/CMakeLists.txt         |  5 ++-
 examples/custom-matrix-format/CMakeLists.txt  | 12 +++---
 .../custom-stopping-criterion/CMakeLists.txt  | 10 ++---
 examples/ginkgo-overhead/CMakeLists.txt       |  5 ++-
 examples/ginkgo-ranges/CMakeLists.txt         |  5 ++-
 .../ilu-preconditioned-solver/CMakeLists.txt  |  5 ++-
 examples/inverse-iteration/CMakeLists.txt     |  5 ++-
 .../CMakeLists.txt                            |  5 ++-
 examples/iterative-refinement/CMakeLists.txt  |  5 ++-
 examples/minimal-cuda-solver/CMakeLists.txt   |  5 ++-
 examples/mixed-precision-ir/CMakeLists.txt    |  5 ++-
 .../nine-pt-stencil-solver/CMakeLists.txt     |  5 ++-
 examples/papi-logging/CMakeLists.txt          |  7 +--
 examples/performance-debugging/CMakeLists.txt |  5 ++-
 examples/poisson-solver/CMakeLists.txt        |  5 ++-
 examples/preconditioned-solver/CMakeLists.txt |  5 ++-
 examples/preconditioner-export/CMakeLists.txt |  5 ++-
 examples/simple-solver-logging/CMakeLists.txt |  5 ++-
 examples/simple-solver/CMakeLists.txt         |  5 ++-
 .../three-pt-stencil-solver/CMakeLists.txt    |  5 ++-
 22 files changed, 94 insertions(+), 68 deletions(-)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index b1f4abf622e..9933f6fbd2b 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -1,12 +1,6 @@
 option(GINKGO_BUILD_EXTLIB_EXAMPLE "Build the external-lib-interfacing with deal.II, you need to link the deal.II library." OFF)
 option(GINKGO_RUN_EXAMPLES "Run the examples." OFF)
 
-function(setup_example_target target_name)
-    add_executable(${target_name} ${target_name}.cpp)
-    target_link_libraries(${target_name} ginkgo)
-    target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
-endfunction()
-
 set(EXAMPLES_EXEC_LIST "adaptiveprecision-blockjacobi;custom-logger;ginkgo-overhead;ginkgo-ranges;ilu-preconditioned-solver;ir-ilu-preconditioned-solver;inverse-iteration;iterative-refinement;mixed-precision-ir;nine-pt-stencil-solver;poisson-solver;preconditioned-solver;simple-solver;three-pt-stencil-solver;")
 
 set(EXAMPLES_LIST "${EXAMPLES_EXEC_LIST};custom-matrix-format;custom-stopping-criterion;minimal-cuda-solver;papi-logging;performance-debugging;preconditioner-export;simple-solver-logging")
@@ -19,7 +13,6 @@ foreach(example ${EXAMPLES_LIST})
     add_subdirectory(${example})
 endforeach()
 
-set(run_list "")
 if(GINKGO_RUN_EXAMPLES)
     foreach(example ${EXAMPLES_LIST})
         add_custom_target("run-${example}"
@@ -29,36 +22,14 @@ if(GINKGO_RUN_EXAMPLES)
     foreach(example ${EXAMPLES_LIST})
         file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command "#!/bin/bash
 diff <(sed -n '7,$p' ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example}.out | sed -E 's/([0-9]+.)//g') <(sed -n '6,$p' ${CMAKE_SOURCE_DIR}/examples/${example}/doc/results.dox | head -n -4 | sed -E 's/([0-9]+.)//g')")
-        add_custom_target("diff-${example}"
+        add_custom_target("validate-${example}"
             COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command && ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command
             WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${example})
     endforeach()
-    file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/run_all_examples.sh
-        "NUM_EXAMPLES=$(wc -l < example-targets.txt)\n"
-        "for i in $(seq 1 $NUM_EXAMPLES)\n"
-        "do\n"
-        "  target_name=$(head -n $i example-targets.txt | tail -1)\n"
-        "  ex_target=run-\$\{target_name\}\n"
-        "  cmake --build ${CMAKE_BINARY_DIR} --target $ex_target\n"
-        "done\n")
-    add_custom_target("run_all_examples"
-        COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/run_all_examples.sh && ${CMAKE_CURRENT_BINARY_DIR}/run_all_examples.sh
-        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-    file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/validate_all_examples.sh
-        "NUM_EXAMPLES=$(wc -l < example-targets.txt)\n"
-        "for i in $(seq 1 $NUM_EXAMPLES)\n"
-        "do\n"
-        "  target_name=$(head -n $i example-targets.txt | tail -1)\n"
-        "  ex_target=diff-\$\{target_name\}\n"
-        "  cmake --build ${CMAKE_BINARY_DIR} --target $ex_target\n"
-        "done\n")
-    add_custom_target("validate_all_examples"
-        COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/validate_all_examples.sh && ${CMAKE_CURRENT_BINARY_DIR}/validate_all_examples.sh
-        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+    add_custom_target(run_all_examples)
+    add_custom_target(validate_all_examples)
+    foreach(run_ex ${EXAMPLES_EXEC_LIST})
+        add_dependencies(run_all_examples "run-${run_ex}")
+        add_dependencies(validate_all_examples "validate-${run_ex}")
+    endforeach()
 endif()
-
-set(OUT_FILE "${CMAKE_BINARY_DIR}/examples/example-targets.txt")
-file(WRITE ${OUT_FILE} "")
-foreach(example ${EXAMPLES_EXEC_LIST})
-    file(APPEND ${OUT_FILE} "${example}\n")
-endforeach()
diff --git a/examples/adaptiveprecision-blockjacobi/CMakeLists.txt b/examples/adaptiveprecision-blockjacobi/CMakeLists.txt
index e04e61ad6fa..5073564c8d1 100644
--- a/examples/adaptiveprecision-blockjacobi/CMakeLists.txt
+++ b/examples/adaptiveprecision-blockjacobi/CMakeLists.txt
@@ -1,2 +1,5 @@
-setup_example_target("adaptiveprecision-blockjacobi")
+set(target_name "adaptiveprecision-blockjacobi")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/custom-logger/CMakeLists.txt b/examples/custom-logger/CMakeLists.txt
index 11b4182bbf6..d7978bed813 100644
--- a/examples/custom-logger/CMakeLists.txt
+++ b/examples/custom-logger/CMakeLists.txt
@@ -1,4 +1,7 @@
-setup_example_target("custom-logger")
+set(target_name "custom-logger")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/custom-matrix-format/CMakeLists.txt b/examples/custom-matrix-format/CMakeLists.txt
index 1665e8e912a..52e52c1bca4 100644
--- a/examples/custom-matrix-format/CMakeLists.txt
+++ b/examples/custom-matrix-format/CMakeLists.txt
@@ -2,13 +2,13 @@ if (GINKGO_BUILD_CUDA AND GINKGO_BUILD_OMP)
     enable_language(CUDA)
     set(CMAKE_CUDA_STANDARD 14)
     set(CMAKE_CUDA_STANDARD_REQUIRED ON)
-    set(run_target "custom-matrix-format")
-    add_executable(${run_target}
-        ${run_target}.cpp
+    set(target_name "custom-matrix-format")
+    add_executable(${target_name}
+        ${target_name}.cpp
         stencil_kernel.cu)
-    target_link_libraries(${run_target} ginkgo)
-    target_include_directories(${run_target} PRIVATE
+    target_link_libraries(${target_name} Ginkgo::ginkgo)
+    target_include_directories(${target_name} PRIVATE
         ${PROJECT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
     # workaround for clang-cuda/g++ interaction
-    set_target_properties(${run_target} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+    set_target_properties(${target_name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endif()
diff --git a/examples/custom-stopping-criterion/CMakeLists.txt b/examples/custom-stopping-criterion/CMakeLists.txt
index 5fb0916afc6..0a2196b814b 100644
--- a/examples/custom-stopping-criterion/CMakeLists.txt
+++ b/examples/custom-stopping-criterion/CMakeLists.txt
@@ -1,11 +1,11 @@
-set(run_target "custom-stopping-criterion")
+set(target_name "custom-stopping-criterion")
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
-add_executable(${run_target}
-	${run_target}.cpp)
-target_link_libraries(${run_target} ginkgo
+add_executable(${target_name}
+	${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo
 	Threads::Threads)
-target_include_directories(${run_target}
+target_include_directories(${target_name}
 	PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
diff --git a/examples/ginkgo-overhead/CMakeLists.txt b/examples/ginkgo-overhead/CMakeLists.txt
index f8b5596796b..6f42ba09eb1 100644
--- a/examples/ginkgo-overhead/CMakeLists.txt
+++ b/examples/ginkgo-overhead/CMakeLists.txt
@@ -1 +1,4 @@
-setup_example_target("ginkgo-overhead")
+set(target_name "ginkgo-overhead")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
diff --git a/examples/ginkgo-ranges/CMakeLists.txt b/examples/ginkgo-ranges/CMakeLists.txt
index 6763cb0309c..fc5ffa05f9c 100644
--- a/examples/ginkgo-ranges/CMakeLists.txt
+++ b/examples/ginkgo-ranges/CMakeLists.txt
@@ -1 +1,4 @@
-setup_example_target("ginkgo-ranges")
+set(target_name "ginkgo-ranges")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
diff --git a/examples/ilu-preconditioned-solver/CMakeLists.txt b/examples/ilu-preconditioned-solver/CMakeLists.txt
index 3df1b0cac01..f33f050c356 100644
--- a/examples/ilu-preconditioned-solver/CMakeLists.txt
+++ b/examples/ilu-preconditioned-solver/CMakeLists.txt
@@ -1,4 +1,7 @@
-setup_example_target("ilu-preconditioned-solver")
+set(target_name "ilu-preconditioned-solver")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/inverse-iteration/CMakeLists.txt b/examples/inverse-iteration/CMakeLists.txt
index 04e037c4721..b7f9789eb65 100644
--- a/examples/inverse-iteration/CMakeLists.txt
+++ b/examples/inverse-iteration/CMakeLists.txt
@@ -1,2 +1,5 @@
-setup_example_target("inverse-iteration")
+set(target_name "inverse-iteration")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/ir-ilu-preconditioned-solver/CMakeLists.txt b/examples/ir-ilu-preconditioned-solver/CMakeLists.txt
index 56ca17eec40..6b4c267d6a3 100644
--- a/examples/ir-ilu-preconditioned-solver/CMakeLists.txt
+++ b/examples/ir-ilu-preconditioned-solver/CMakeLists.txt
@@ -1,2 +1,5 @@
-setup_example_target("ir-ilu-preconditioned-solver")
+set(target_name "ir-ilu-preconditioned-solver")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/iterative-refinement/CMakeLists.txt b/examples/iterative-refinement/CMakeLists.txt
index 5c90074b50c..01d5600cdea 100644
--- a/examples/iterative-refinement/CMakeLists.txt
+++ b/examples/iterative-refinement/CMakeLists.txt
@@ -1,2 +1,5 @@
-setup_example_target("iterative-refinement")
+set(target_name "iterative-refinement")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/minimal-cuda-solver/CMakeLists.txt b/examples/minimal-cuda-solver/CMakeLists.txt
index 4f9bfc3d1dc..4aa13017904 100644
--- a/examples/minimal-cuda-solver/CMakeLists.txt
+++ b/examples/minimal-cuda-solver/CMakeLists.txt
@@ -1,4 +1,7 @@
-setup_example_target("minimal-cuda-solver")
+set(target_name "minimal-cuda-solver")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/mixed-precision-ir/CMakeLists.txt b/examples/mixed-precision-ir/CMakeLists.txt
index a40f470449f..109810c8657 100644
--- a/examples/mixed-precision-ir/CMakeLists.txt
+++ b/examples/mixed-precision-ir/CMakeLists.txt
@@ -1,2 +1,5 @@
-setup_example_target("mixed-precision-ir")
+set(target_name "mixed-precision-ir")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/nine-pt-stencil-solver/CMakeLists.txt b/examples/nine-pt-stencil-solver/CMakeLists.txt
index e3dfc6b78f0..bed21d0f93d 100644
--- a/examples/nine-pt-stencil-solver/CMakeLists.txt
+++ b/examples/nine-pt-stencil-solver/CMakeLists.txt
@@ -1 +1,4 @@
-setup_example_target("nine-pt-stencil-solver")
+set(target_name "nine-pt-stencil-solver")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
diff --git a/examples/papi-logging/CMakeLists.txt b/examples/papi-logging/CMakeLists.txt
index c378a0bd543..29fe170645b 100644
--- a/examples/papi-logging/CMakeLists.txt
+++ b/examples/papi-logging/CMakeLists.txt
@@ -1,7 +1,8 @@
 if (GINKGO_HAVE_PAPI_SDE)
-	add_executable(papi-logging papi-logging.cpp)
-	target_link_libraries(papi-logging ginkgo PAPI::PAPI)
-	target_include_directories(papi-logging PRIVATE ${PROJECT_SOURCE_DIR})
+    set(target_name "papi-logging")
+	add_executable(${target_name} ${target_name}.cpp)
+	target_link_libraries(${target_name} ginkgo PAPI::PAPI)
+	target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 	configure_file(data/A.mtx data/A.mtx COPYONLY)
 	configure_file(data/b.mtx data/b.mtx COPYONLY)
 	configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/performance-debugging/CMakeLists.txt b/examples/performance-debugging/CMakeLists.txt
index ebecae2d1d6..6ca05939515 100644
--- a/examples/performance-debugging/CMakeLists.txt
+++ b/examples/performance-debugging/CMakeLists.txt
@@ -1,2 +1,5 @@
-setup_example_target("performance-debugging")
+set(target_name "performance-debugging")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/poisson-solver/CMakeLists.txt b/examples/poisson-solver/CMakeLists.txt
index a83dd611682..5fc1e2ad83f 100644
--- a/examples/poisson-solver/CMakeLists.txt
+++ b/examples/poisson-solver/CMakeLists.txt
@@ -1 +1,4 @@
-setup_example_target("poisson-solver")
+set(target_name "poisson-solver")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
diff --git a/examples/preconditioned-solver/CMakeLists.txt b/examples/preconditioned-solver/CMakeLists.txt
index f80aeeee0a7..a2c7898fd49 100644
--- a/examples/preconditioned-solver/CMakeLists.txt
+++ b/examples/preconditioned-solver/CMakeLists.txt
@@ -1,4 +1,7 @@
-setup_example_target("preconditioned-solver")
+set(target_name "preconditioned-solver")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/preconditioner-export/CMakeLists.txt b/examples/preconditioner-export/CMakeLists.txt
index 4a0ec3294ad..04134b532af 100644
--- a/examples/preconditioner-export/CMakeLists.txt
+++ b/examples/preconditioner-export/CMakeLists.txt
@@ -1,2 +1,5 @@
-setup_example_target("preconditioner-export")
+set(target_name "preconditioner-export")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/simple-solver-logging/CMakeLists.txt b/examples/simple-solver-logging/CMakeLists.txt
index 6596011fae4..545a1d9977c 100644
--- a/examples/simple-solver-logging/CMakeLists.txt
+++ b/examples/simple-solver-logging/CMakeLists.txt
@@ -1,4 +1,7 @@
-setup_example_target("simple-solver-logging")
+set(target_name "simple-solver-logging")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/simple-solver/CMakeLists.txt b/examples/simple-solver/CMakeLists.txt
index 1f6a6373949..f30cb01ffca 100644
--- a/examples/simple-solver/CMakeLists.txt
+++ b/examples/simple-solver/CMakeLists.txt
@@ -1,4 +1,7 @@
-setup_example_target("simple-solver")
+set(target_name "simple-solver")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/three-pt-stencil-solver/CMakeLists.txt b/examples/three-pt-stencil-solver/CMakeLists.txt
index dd97f3a8946..c12b2ee7ff2 100644
--- a/examples/three-pt-stencil-solver/CMakeLists.txt
+++ b/examples/three-pt-stencil-solver/CMakeLists.txt
@@ -1 +1,4 @@
-setup_example_target("three-pt-stencil-solver")
+set(target_name "three-pt-stencil-solver")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})

From 9924947a861f72868f2c4a998213cb4d5ea2c6df Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratik.nayak4@gmail.com>
Date: Tue, 29 Sep 2020 10:45:27 +0200
Subject: [PATCH 0027/1008] Review update.

+ Formatting.
+ combie loops.
+ revert usage prints.
---
 .gitlab-ci.yml                                |  1 -
 examples/CMakeLists.txt                       | 63 ++++++++++++-------
 examples/custom-logger/custom-logger.cpp      |  2 +-
 .../custom-stopping-criterion.cpp             |  2 +-
 examples/ginkgo-overhead/ginkgo-overhead.cpp  |  2 +-
 .../ilu-preconditioned-solver.cpp             |  2 +-
 .../ir-ilu-preconditioned-solver.cpp          |  2 +-
 .../iterative-refinement.cpp                  |  2 +-
 .../mixed-precision-ir/mixed-precision-ir.cpp |  2 +-
 examples/papi-logging/papi-logging.cpp        |  2 +-
 .../performance-debugging.cpp                 |  2 +-
 .../preconditioned-solver.cpp                 |  2 +-
 .../simple-solver-logging.cpp                 |  2 +-
 examples/simple-solver/simple-solver.cpp      |  2 +-
 14 files changed, 51 insertions(+), 37 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b332d501f47..cad4790e431 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -89,7 +89,6 @@ stages:
         -DGINKGO_BUILD_OMP=${BUILD_OMP} -DGINKGO_BUILD_CUDA=${BUILD_CUDA}
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
-        -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
     - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT} install
     - |
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 9933f6fbd2b..4111cd9081b 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -1,35 +1,50 @@
 option(GINKGO_BUILD_EXTLIB_EXAMPLE "Build the external-lib-interfacing with deal.II, you need to link the deal.II library." OFF)
-option(GINKGO_RUN_EXAMPLES "Run the examples." OFF)
 
-set(EXAMPLES_EXEC_LIST "adaptiveprecision-blockjacobi;custom-logger;ginkgo-overhead;ginkgo-ranges;ilu-preconditioned-solver;ir-ilu-preconditioned-solver;inverse-iteration;iterative-refinement;mixed-precision-ir;nine-pt-stencil-solver;poisson-solver;preconditioned-solver;simple-solver;three-pt-stencil-solver;")
+set(EXAMPLES_EXEC_LIST
+    adaptiveprecision-blockjacobi
+    custom-logger
+    ginkgo-overhead
+    ginkgo-ranges
+    ilu-preconditioned-solver
+    ir-ilu-preconditioned-solver
+    inverse-iteration
+    iterative-refinement
+    mixed-precision-ir
+    nine-pt-stencil-solver
+    poisson-solver
+    preconditioned-solver
+    simple-solver
+    three-pt-stencil-solver)
 
-set(EXAMPLES_LIST "${EXAMPLES_EXEC_LIST};custom-matrix-format;custom-stopping-criterion;minimal-cuda-solver;papi-logging;performance-debugging;preconditioner-export;simple-solver-logging")
+set(EXAMPLES_LIST
+    ${EXAMPLES_EXEC_LIST}
+    custom-matrix-format
+    custom-stopping-criterion
+    minimal-cuda-solver
+    papi-logging
+    performance-debugging
+    preconditioner-export
+    simple-solver-logging)
 
 if(GINKGO_BUILD_EXTLIB_EXAMPLE)
-    set(EXAMPLES_LIST "${EXAMPLES_LIST};external-lib-interfacing")
+    list(APPEND EXAMPLES_LIST external-lib-interfacing)
 endif()
 
 foreach(example ${EXAMPLES_LIST})
     add_subdirectory(${example})
+    add_custom_target("run-${example}"
+        COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example} > ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example}.out
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${example})
+    file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command "#!/bin/bash
+diff <(sed -n '7,$p' ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example}.out | sed -E 's/([0-9]+.)//g') <(sed -n '6,$p' ${CMAKE_SOURCE_DIR}/examples/${example}/doc/results.dox | head -n -4 | sed -E 's/([0-9]+.)//g')")
+    add_custom_target("validate-${example}"
+        COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command && ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${example})
 endforeach()
 
-if(GINKGO_RUN_EXAMPLES)
-    foreach(example ${EXAMPLES_LIST})
-        add_custom_target("run-${example}"
-            COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example} > ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example}.out
-            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${example})
-    endforeach()
-    foreach(example ${EXAMPLES_LIST})
-        file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command "#!/bin/bash
-diff <(sed -n '7,$p' ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example}.out | sed -E 's/([0-9]+.)//g') <(sed -n '6,$p' ${CMAKE_SOURCE_DIR}/examples/${example}/doc/results.dox | head -n -4 | sed -E 's/([0-9]+.)//g')")
-        add_custom_target("validate-${example}"
-            COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command && ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command
-            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${example})
-    endforeach()
-    add_custom_target(run_all_examples)
-    add_custom_target(validate_all_examples)
-    foreach(run_ex ${EXAMPLES_EXEC_LIST})
-        add_dependencies(run_all_examples "run-${run_ex}")
-        add_dependencies(validate_all_examples "validate-${run_ex}")
-    endforeach()
-endif()
+add_custom_target(run_all_examples)
+add_custom_target(validate_all_examples)
+foreach(run_ex ${EXAMPLES_EXEC_LIST})
+    add_dependencies(run_all_examples "run-${run_ex}")
+    add_dependencies(validate_all_examples "validate-${run_ex}")
+endforeach()
diff --git a/examples/custom-logger/custom-logger.cpp b/examples/custom-logger/custom-logger.cpp
index b3f526e0ba6..6b79de5843e 100644
--- a/examples/custom-logger/custom-logger.cpp
+++ b/examples/custom-logger/custom-logger.cpp
@@ -232,7 +232,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
     } else {
-        std::cerr << "Usage: executable [executor]" << std::endl;
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
diff --git a/examples/custom-stopping-criterion/custom-stopping-criterion.cpp b/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
index eea5d5ce583..b9a4519a027 100644
--- a/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
+++ b/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
@@ -155,7 +155,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: executable [executor]" << std::endl;
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
diff --git a/examples/ginkgo-overhead/ginkgo-overhead.cpp b/examples/ginkgo-overhead/ginkgo-overhead.cpp
index f087cba22e4..17ce96d2900 100644
--- a/examples/ginkgo-overhead/ginkgo-overhead.cpp
+++ b/examples/ginkgo-overhead/ginkgo-overhead.cpp
@@ -55,7 +55,7 @@ int main(int argc, char *argv[])
 
     long unsigned num_iters = 1000000;
     if (argc > 2) {
-        print_usage_and_exit("executable");
+        print_usage_and_exit(argv[0]);
     }
     if (argc == 2) {
         num_iters = std::atol(argv[1]);
diff --git a/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp b/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
index 91153157fde..c3d8a17c998 100644
--- a/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
+++ b/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
@@ -68,7 +68,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: executable [executor]" << std::endl;
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
diff --git a/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp b/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
index 9b548611c41..47b9af91ec9 100644
--- a/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
+++ b/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
@@ -70,7 +70,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: executable [executor] [sweeps]" << std::endl;
+        std::cerr << "Usage: " << argv[0] << " [executor] [sweeps]" << std::endl;
         std::exit(-1);
     }
     unsigned int sweeps = (argc == 3) ? atoi(argv[2]) : 5u;
diff --git a/examples/iterative-refinement/iterative-refinement.cpp b/examples/iterative-refinement/iterative-refinement.cpp
index eb4b2f1440f..69116a48723 100644
--- a/examples/iterative-refinement/iterative-refinement.cpp
+++ b/examples/iterative-refinement/iterative-refinement.cpp
@@ -68,7 +68,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: executable [executor]" << std::endl;
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
diff --git a/examples/mixed-precision-ir/mixed-precision-ir.cpp b/examples/mixed-precision-ir/mixed-precision-ir.cpp
index 6b59b22ecf9..75e91d236ee 100644
--- a/examples/mixed-precision-ir/mixed-precision-ir.cpp
+++ b/examples/mixed-precision-ir/mixed-precision-ir.cpp
@@ -76,7 +76,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: executable [executor]" << std::endl;
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
diff --git a/examples/papi-logging/papi-logging.cpp b/examples/papi-logging/papi-logging.cpp
index c36e80215da..5d8019cc519 100644
--- a/examples/papi-logging/papi-logging.cpp
+++ b/examples/papi-logging/papi-logging.cpp
@@ -151,7 +151,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: executable [executor]" << std::endl;
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
diff --git a/examples/performance-debugging/performance-debugging.cpp b/examples/performance-debugging/performance-debugging.cpp
index 4d71b92a326..573346531cf 100644
--- a/examples/performance-debugging/performance-debugging.cpp
+++ b/examples/performance-debugging/performance-debugging.cpp
@@ -378,7 +378,7 @@ int main(int argc, char *argv[])
                gko::CudaExecutor::get_num_devices() > 0) {
         exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        print_usage("executable");
+        print_usage(argv[0]);
     }
 
     // Read the input matrix file directory
diff --git a/examples/preconditioned-solver/preconditioned-solver.cpp b/examples/preconditioned-solver/preconditioned-solver.cpp
index 181b3a1b01d..083553045ab 100644
--- a/examples/preconditioned-solver/preconditioned-solver.cpp
+++ b/examples/preconditioned-solver/preconditioned-solver.cpp
@@ -68,7 +68,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: executable [executor]" << std::endl;
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
diff --git a/examples/simple-solver-logging/simple-solver-logging.cpp b/examples/simple-solver-logging/simple-solver-logging.cpp
index e752324ba91..bf918d8a7e2 100644
--- a/examples/simple-solver-logging/simple-solver-logging.cpp
+++ b/examples/simple-solver-logging/simple-solver-logging.cpp
@@ -85,7 +85,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: executable [executor]" << std::endl;
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
diff --git a/examples/simple-solver/simple-solver.cpp b/examples/simple-solver/simple-solver.cpp
index f07b6f130e7..4cf6f0b8165 100644
--- a/examples/simple-solver/simple-solver.cpp
+++ b/examples/simple-solver/simple-solver.cpp
@@ -87,7 +87,7 @@ int main(int argc, char *argv[])
                gko::HipExecutor::get_num_devices() > 0) {
         exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
     } else {
-        std::cerr << "Usage: executable [executor]" << std::endl;
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 

From 7bc8d1f6026ea6312fd8de19b63bf3610222bdb6 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratik.nayak4@gmail.com>
Date: Tue, 29 Sep 2020 13:05:12 +0200
Subject: [PATCH 0028/1008] Review update.

---
 .gitlab-ci.yml                                |  1 +
 examples/CMakeLists.txt                       | 34 +++++++++++--------
 .../nine-pt-stencil-solver/doc/results.dox    |  1 -
 .../nine-pt-stencil-solver.cpp                |  8 +----
 examples/papi-logging/CMakeLists.txt          | 12 +++----
 .../three-pt-stencil-solver/doc/results.dox   |  1 -
 .../three-pt-stencil-solver.cpp               |  5 ---
 7 files changed, 28 insertions(+), 34 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cad4790e431..3c34509da2c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -90,6 +90,7 @@ stages:
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
+        -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
     - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT} install
     - |
         (( $(ctest -N | tail -1 | sed 's/Total Tests: //') != 0 )) || exit 1
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 4111cd9081b..4bee33a7c6f 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -1,4 +1,5 @@
 option(GINKGO_BUILD_EXTLIB_EXAMPLE "Build the external-lib-interfacing with deal.II, you need to link the deal.II library." OFF)
+option(GINKGO_RUN_EXAMPLES " Compile run and validation targets for the examples." ON)
 
 set(EXAMPLES_EXEC_LIST
     adaptiveprecision-blockjacobi
@@ -32,19 +33,24 @@ endif()
 
 foreach(example ${EXAMPLES_LIST})
     add_subdirectory(${example})
-    add_custom_target("run-${example}"
-        COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example} > ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example}.out
-        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${example})
-    file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command "#!/bin/bash
-diff <(sed -n '7,$p' ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example}.out | sed -E 's/([0-9]+.)//g') <(sed -n '6,$p' ${CMAKE_SOURCE_DIR}/examples/${example}/doc/results.dox | head -n -4 | sed -E 's/([0-9]+.)//g')")
-    add_custom_target("validate-${example}"
-        COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command && ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command
-        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${example})
 endforeach()
 
-add_custom_target(run_all_examples)
-add_custom_target(validate_all_examples)
-foreach(run_ex ${EXAMPLES_EXEC_LIST})
-    add_dependencies(run_all_examples "run-${run_ex}")
-    add_dependencies(validate_all_examples "validate-${run_ex}")
-endforeach()
+if(GINKGO_RUN_EXAMPLES)
+    foreach(example ${EXAMPLES_LIST})
+        add_custom_target("run-${example}"
+            COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example} > ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example}.out
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${example})
+        file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command "#!/bin/bash
+diff <(sed -n '7,$p' ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example}.out | sed -E 's/([0-9]+.)//g') <(sed -n '6,$p' ${CMAKE_SOURCE_DIR}/examples/${example}/doc/results.dox | head -n -4 | sed -E 's/([0-9]+.)//g')")
+        add_custom_target("validate-${example}"
+            COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command && ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${example})
+    endforeach()
+
+    add_custom_target(run_all_examples)
+    add_custom_target(validate_all_examples)
+    foreach(run_ex ${EXAMPLES_EXEC_LIST})
+        add_dependencies(run_all_examples "run-${run_ex}")
+        add_dependencies(validate_all_examples "validate-${run_ex}")
+    endforeach()
+endif()
diff --git a/examples/nine-pt-stencil-solver/doc/results.dox b/examples/nine-pt-stencil-solver/doc/results.dox
index dfda3387d0c..f9973a3d79e 100644
--- a/examples/nine-pt-stencil-solver/doc/results.dox
+++ b/examples/nine-pt-stencil-solver/doc/results.dox
@@ -3,7 +3,6 @@ The expected output should be
 
 @code{.cpp}
 
-Usage: executable DISCRETIZATION_POINTS [executor] [stencil_alpha] [stencil_beta] [stencil_gamma]
 The average relative error is 6.35715e-06
 The runtime is 167.320520 ms
 
diff --git a/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp b/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
index 3db466f9015..4f1c613428e 100644
--- a/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
+++ b/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
@@ -296,12 +296,6 @@ int main(int argc, char *argv[])
 {
     // Print version information
     std::cout << gko::version_info::get() << std::endl;
-
-    if (argc < 2) {
-        std::cout << "Usage: executable DISCRETIZATION_POINTS [executor]"
-                  << " [stencil_alpha] [stencil_beta] [stencil_gamma]"
-                  << std::endl;
-    }
     using ValueType = double;
     using IndexType = int;
 
@@ -314,7 +308,7 @@ int main(int argc, char *argv[])
     // clang-format off
     std::array<ValueType, 9> coefs{
         gamma_c, beta_c, gamma_c,
-	      beta_c, alpha_c, beta_c,
+	    beta_c, alpha_c, beta_c,
         gamma_c, beta_c, gamma_c};
     // clang-format on
 
diff --git a/examples/papi-logging/CMakeLists.txt b/examples/papi-logging/CMakeLists.txt
index 29fe170645b..a5f079025ba 100644
--- a/examples/papi-logging/CMakeLists.txt
+++ b/examples/papi-logging/CMakeLists.txt
@@ -1,9 +1,9 @@
 if (GINKGO_HAVE_PAPI_SDE)
     set(target_name "papi-logging")
-	add_executable(${target_name} ${target_name}.cpp)
-	target_link_libraries(${target_name} ginkgo PAPI::PAPI)
-	target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
-	configure_file(data/A.mtx data/A.mtx COPYONLY)
-	configure_file(data/b.mtx data/b.mtx COPYONLY)
-	configure_file(data/x0.mtx data/x0.mtx COPYONLY)
+    add_executable(${target_name} ${target_name}.cpp)
+    target_link_libraries(${target_name} ginkgo PAPI::PAPI)
+    target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+    configure_file(data/A.mtx data/A.mtx COPYONLY)
+    configure_file(data/b.mtx data/b.mtx COPYONLY)
+    configure_file(data/x0.mtx data/x0.mtx COPYONLY)
 endif()
diff --git a/examples/three-pt-stencil-solver/doc/results.dox b/examples/three-pt-stencil-solver/doc/results.dox
index 6dd4180b920..49e8cf79a2e 100644
--- a/examples/three-pt-stencil-solver/doc/results.dox
+++ b/examples/three-pt-stencil-solver/doc/results.dox
@@ -3,7 +3,6 @@ This is the expected output:
 
 @code{.cpp}
 
-Usage: executable DISCRETIZATION_POINTS [executor]
 The average relative error is 2.52236e-11
 
 @endcode
diff --git a/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp b/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
index b539490d14a..360d890ddae 100644
--- a/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
+++ b/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
@@ -235,11 +235,6 @@ int main(int argc, char *argv[])
     // Print version information
     std::cout << gko::version_info::get() << std::endl;
 
-    if (argc < 2) {
-        std::cout << "Usage: executable DISCRETIZATION_POINTS [executor]"
-                  << std::endl;
-    }
-
     const IndexType discretization_points =
         argc >= 2 ? std::atoi(argv[1]) : 100;
     const auto executor_string = argc >= 3 ? argv[2] : "reference";

From 1d30dbf8fa263bab1fe86c3d96e845721ea2568b Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratik.nayak4@gmail.com>
Date: Wed, 30 Sep 2020 23:06:35 +0200
Subject: [PATCH 0029/1008] Update with a better usage message and add executor
 as arg.

---
 .gitlab-ci.yml                                | 21 +++++++++-
 examples/CMakeLists.txt                       | 16 ++++----
 .../adaptiveprecision-blockjacobi.cpp         | 35 ++++++++++------
 examples/custom-logger/custom-logger.cpp      | 35 ++++++++++------
 .../custom-matrix-format.cpp                  | 17 ++++----
 .../custom-stopping-criterion.cpp             | 36 +++++++++++------
 .../ilu-preconditioned-solver.cpp             | 35 ++++++++++------
 .../inverse-iteration/inverse-iteration.cpp   | 33 +++++++++------
 .../ir-ilu-preconditioned-solver.cpp          | 39 +++++++++++-------
 .../iterative-refinement.cpp                  | 33 +++++++++------
 .../mixed-precision-ir/mixed-precision-ir.cpp | 33 +++++++++------
 .../nine-pt-stencil-solver.cpp                | 16 ++++++--
 examples/papi-logging/papi-logging.cpp        | 38 +++++++++++-------
 .../performance-debugging.cpp                 | 34 +++++++++++-----
 examples/poisson-solver/doc/results.dox       |  2 +-
 examples/poisson-solver/poisson-solver.cpp    | 11 ++---
 .../preconditioned-solver.cpp                 | 37 +++++++++++------
 .../simple-solver-logging.cpp                 | 35 ++++++++++------
 examples/simple-solver/simple-solver.cpp      | 40 ++++++++++++-------
 .../three-pt-stencil-solver.cpp               | 10 ++++-
 20 files changed, 359 insertions(+), 197 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3c34509da2c..3275706d2f9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -98,8 +98,24 @@ stages:
     - ninja test_install
     - |
       if [ "${RUN_EXAMPLES}" == "ON" ]; then
-        ninja run_all_examples
-        ninja validate_all_examples
+         export EX_ARG="reference"
+         ninja run_all_examples
+         ninja validate_all_examples
+         if [ "{BUILD_OMP}" == "ON" ]; then
+            export EX_ARG="omp"
+            ninja run_all_examples
+            ninja validate_all_examples
+        fi
+        if [ "{BUILD_CUDA}" == "ON" ]; then
+            export EX_ARG="cuda"
+            ninja run_all_examples
+            ninja validate_all_examples
+        fi
+        if [ "{BUILD_HIP}" == "ON" ]; then
+            export EX_ARG="hip"
+            ninja run_all_examples
+            ninja validate_all_examples
+        fi
       fi
   dependencies: []
   except:
@@ -517,6 +533,7 @@ build/amd/gcc/hip/debug/shared:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_HIP: "ON"
+    RUN_EXAMPLES: "ON"
     BUILD_TYPE: "Debug"
   only:
     variables:
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 4bee33a7c6f..b36429cfcf1 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -4,7 +4,6 @@ option(GINKGO_RUN_EXAMPLES " Compile run and validation targets for the examples
 set(EXAMPLES_EXEC_LIST
     adaptiveprecision-blockjacobi
     custom-logger
-    ginkgo-overhead
     ginkgo-ranges
     ilu-preconditioned-solver
     ir-ilu-preconditioned-solver
@@ -21,6 +20,7 @@ set(EXAMPLES_LIST
     ${EXAMPLES_EXEC_LIST}
     custom-matrix-format
     custom-stopping-criterion
+    ginkgo-overhead
     minimal-cuda-solver
     papi-logging
     performance-debugging
@@ -37,14 +37,16 @@ endforeach()
 
 if(GINKGO_RUN_EXAMPLES)
     foreach(example ${EXAMPLES_LIST})
+        set(example_path "${CMAKE_CURRENT_BINARY_DIR}/${example}")
+        file(WRITE ${example_path}/target-wrapper.sh "${example_path}/${example} \$\{EX_ARG\}")
         add_custom_target("run-${example}"
-            COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example} > ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example}.out
-            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${example})
-        file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command "#!/bin/bash
-diff <(sed -n '7,$p' ${CMAKE_CURRENT_BINARY_DIR}/${example}/${example}.out | sed -E 's/([0-9]+.)//g') <(sed -n '6,$p' ${CMAKE_SOURCE_DIR}/examples/${example}/doc/results.dox | head -n -4 | sed -E 's/([0-9]+.)//g')")
+            COMMAND chmod +x ${example_path}/target-wrapper.sh && ${example_path}/target-wrapper.sh > ${example_path}/${example}.out
+            WORKING_DIRECTORY ${example_path})
+        file(WRITE ${example_path}/diff-command "#!/bin/bash
+diff <(sed -n '7,$p' ${example_path}/${example}.out | sed -E 's/([^a-z,\":\\s\\)\\(\\{\\}_]+)//g') <(sed -n '6,$p' ${CMAKE_SOURCE_DIR}/examples/${example}/doc/results.dox | head -n -4 | sed -E 's/([^a-z,\":\\s\\)\\(\\{\\}_]+)//g')")
         add_custom_target("validate-${example}"
-            COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command && ${CMAKE_CURRENT_BINARY_DIR}/${example}/diff-command
-            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${example})
+            COMMAND chmod +x ${example_path}/diff-command && ${example_path}/diff-command
+            WORKING_DIRECTORY ${example_path})
     endforeach()
 
     add_custom_target(run_all_examples)
diff --git a/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp b/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
index f00ae67236f..06ddbd8f24b 100644
--- a/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
+++ b/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <fstream>
 #include <iomanip>
 #include <iostream>
+#include <map>
 #include <string>
 
 
@@ -55,23 +56,31 @@ int main(int argc, char *argv[])
     // Print version information
     std::cout << gko::version_info::get() << std::endl;
 
-    // Figure out where to run the code
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else {
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
         std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    // Figure out where to run the code
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
+
     // Read data
     auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
     // Create RHS and initial guess as 1
diff --git a/examples/custom-logger/custom-logger.cpp b/examples/custom-logger/custom-logger.cpp
index 6b79de5843e..afc48e0ea1f 100644
--- a/examples/custom-logger/custom-logger.cpp
+++ b/examples/custom-logger/custom-logger.cpp
@@ -37,6 +37,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 // Add the fstream header to read from data from files.
 #include <fstream>
+// Add the map header for storing the executor map.
+#include <map>
 // Add the C++ iomanip header to prettify the output.
 #include <iomanip>
 // Add formatting flag modification capabilities.
@@ -220,22 +222,31 @@ int main(int argc, char *argv[])
     // @note With the help of C++, you see that you only ever need to change the
     // executor and all the other functions/ routines within Ginkgo should
     // automatically work and run on the executor with any other changes.
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
-    } else {
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
         std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    // Figure out where to run the code
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
+
     // @sect3{Reading your data and transfer to the proper device.}
     // Read the matrix, right hand side and the initial solution using the @ref
     // read function.
diff --git a/examples/custom-matrix-format/custom-matrix-format.cpp b/examples/custom-matrix-format/custom-matrix-format.cpp
index 57eb4bac9c1..679ccb379c2 100644
--- a/examples/custom-matrix-format/custom-matrix-format.cpp
+++ b/examples/custom-matrix-format/custom-matrix-format.cpp
@@ -241,18 +241,15 @@ int main(int argc, char *argv[])
     using mtx = gko::matrix::Csr<ValueType, IndexType>;
     using cg = gko::solver::Cg<ValueType>;
 
-    if (argc < 2) {
-        std::cerr << "Usage: executable DISCRETIZATION_POINTS [executor]"
-                  << std::endl;
+    // Figure out where to run the code
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
-    // Get number of discretization points
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
     const unsigned int discretization_points =
-        argc >= 2 ? std::atoi(argv[1]) : 100u;
-    const auto executor_string = argc >= 3 ? argv[2] : "reference";
-
-    // Figure out where to run the code
+        argc >= 3 ? std::atoi(argv[2]) : 100u;
     std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
         exec_map{
             {"omp", [] { return gko::OmpExecutor::create(); }},
@@ -303,8 +300,8 @@ int main(int argc, char *argv[])
                                                     -1, 2, -1))
         ->apply(lend(rhs), lend(u));
 
-    print_solution(u0, u1, lend(u));
-    std::cout << "The average relative error is "
+    std::cout << "\nSolve complete."
+              << "\nThe average relative error is "
               << calculate_error(discretization_points, lend(u), correct_u) /
                      discretization_points
               << std::endl;
diff --git a/examples/custom-stopping-criterion/custom-stopping-criterion.cpp b/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
index b9a4519a027..5c7d7861b9b 100644
--- a/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
+++ b/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <fstream>
 #include <iostream>
+#include <map>
 #include <string>
 #include <thread>
 
@@ -143,22 +144,33 @@ int main(int argc, char *argv[])
     std::cout << gko::version_info::get() << std::endl;
 
     // Figure out where to run the code
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else {
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
         std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
+    // Figure out where to run the code
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+
+    // Figure out where to run the code
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
+
     // Declare a user controled boolean for the iteration process
     volatile bool stop_iteration_process{};
 
diff --git a/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp b/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
index c3d8a17c998..d6fbcc8ea06 100644
--- a/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
+++ b/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <cstdlib>
 #include <fstream>
 #include <iostream>
+#include <map>
 #include <string>
 
 
@@ -55,23 +56,31 @@ int main(int argc, char *argv[])
     // Print version information
     std::cout << gko::version_info::get() << std::endl;
 
-    // Figure out where to run the code
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else {
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
         std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    // Figure out where to run the code
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
+
     // Read data
     auto A = gko::share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
     auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
diff --git a/examples/inverse-iteration/inverse-iteration.cpp b/examples/inverse-iteration/inverse-iteration.cpp
index 68b1029d554..728b9363178 100644
--- a/examples/inverse-iteration/inverse-iteration.cpp
+++ b/examples/inverse-iteration/inverse-iteration.cpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <fstream>
 #include <iomanip>
 #include <iostream>
+#include <map>
 #include <string>
 
 
@@ -60,22 +61,30 @@ int main(int argc, char *argv[])
     std::cout << std::scientific << std::setprecision(8) << std::showpos;
 
     // Figure out where to run the code
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else {
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
         std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
+
     auto this_exec = exec->get_master();
 
     // linear system solver parameters
diff --git a/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp b/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
index 47b9af91ec9..627d11d88dd 100644
--- a/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
+++ b/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <cstdlib>
 #include <fstream>
 #include <iostream>
+#include <map>
 #include <string>
 
 
@@ -57,23 +58,31 @@ int main(int argc, char *argv[])
     // Print version information
     std::cout << gko::version_info::get() << std::endl;
 
-    // Figure out where to run the code and how many block-Jacobi sweeps to use
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if ((argc == 2 || argc == 3) && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if ((argc == 2 || argc == 3) && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else if ((argc == 2 || argc == 3) && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else {
-        std::cerr << "Usage: " << argv[0] << " [executor] [sweeps]" << std::endl;
+    // Figure out where to run the code
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
-    unsigned int sweeps = (argc == 3) ? atoi(argv[2]) : 5u;
+
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    const unsigned int sweeps = argc == 3 ? std::atoi(argv[2]) : 5u;
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
 
     // Read data
     auto A = gko::share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
diff --git a/examples/iterative-refinement/iterative-refinement.cpp b/examples/iterative-refinement/iterative-refinement.cpp
index 69116a48723..a07726ec82e 100644
--- a/examples/iterative-refinement/iterative-refinement.cpp
+++ b/examples/iterative-refinement/iterative-refinement.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <fstream>
 #include <iomanip>
 #include <iostream>
+#include <map>
 #include <string>
 
 
@@ -56,22 +57,30 @@ int main(int argc, char *argv[])
     std::cout << gko::version_info::get() << std::endl;
 
     // Figure out where to run the code
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else {
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
         std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
+
     // Read data
     auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
     // Create RHS and initial guess as 1
diff --git a/examples/mixed-precision-ir/mixed-precision-ir.cpp b/examples/mixed-precision-ir/mixed-precision-ir.cpp
index 75e91d236ee..a087baa3dfc 100644
--- a/examples/mixed-precision-ir/mixed-precision-ir.cpp
+++ b/examples/mixed-precision-ir/mixed-precision-ir.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <fstream>
 #include <iomanip>
 #include <iostream>
+#include <map>
 #include <string>
 
 
@@ -64,22 +65,30 @@ int main(int argc, char *argv[])
     std::cout << gko::version_info::get() << std::endl;
 
     // Figure out where to run the code
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else {
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
         std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
+
     // Read data
     auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
     // Create RHS and initial guess as 1
diff --git a/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp b/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
index 4f1c613428e..e1fd8fe886d 100644
--- a/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
+++ b/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
@@ -294,13 +294,21 @@ void solve_system(const std::string &executor_string,
 
 int main(int argc, char *argv[])
 {
-    // Print version information
-    std::cout << gko::version_info::get() << std::endl;
     using ValueType = double;
     using IndexType = int;
 
-    const int discretization_points = argc >= 2 ? std::atoi(argv[1]) : 100;
-    const auto executor_string = argc >= 3 ? argv[2] : "reference";
+    // Print version information
+    std::cout << gko::version_info::get() << std::endl;
+
+    if (argc == 2 && std::string(argv[1]) == "--help") {
+        std::cerr << "Usage: " << argv[0] << " [executor] [DISCRETIZATION_POINTS] [alpha] [beta] [gamma]"
+                  << std::endl;
+        std::exit(-1);
+    }
+
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    const IndexType discretization_points =
+        argc >= 3 ? std::atoi(argv[2]) : 100;
     const ValueType alpha_c = argc >= 4 ? std::atof(argv[3]) : default_alpha;
     const ValueType beta_c = argc >= 5 ? std::atof(argv[4]) : default_beta;
     const ValueType gamma_c = argc >= 6 ? std::atof(argv[5]) : default_gamma;
diff --git a/examples/papi-logging/papi-logging.cpp b/examples/papi-logging/papi-logging.cpp
index 5d8019cc519..2fd8cbbc8d4 100644
--- a/examples/papi-logging/papi-logging.cpp
+++ b/examples/papi-logging/papi-logging.cpp
@@ -36,6 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <papi.h>
 #include <fstream>
 #include <iostream>
+#include <map>
 #include <string>
 #include <thread>
 
@@ -138,23 +139,32 @@ int main(int argc, char *argv[])
     // Print version information
     std::cout << gko::version_info::get() << std::endl;
 
-    // Figure out where to run the code
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
+        std::cerr << "Usage: " << argv[0] << " [executor]"
+                  << std::endl;
         std::exit(-1);
     }
 
+    // Figure out where to run the code
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
+
     // Read data
     auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
     auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
diff --git a/examples/performance-debugging/performance-debugging.cpp b/examples/performance-debugging/performance-debugging.cpp
index 573346531cf..4a9fae8bcbd 100644
--- a/examples/performance-debugging/performance-debugging.cpp
+++ b/examples/performance-debugging/performance-debugging.cpp
@@ -369,18 +369,32 @@ int main(int argc, char *argv[])
     std::cout << gko::version_info::get() << std::endl;
 
     // Figure out where to run the code
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc > 1 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc > 1 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else {
-        print_usage(argv[0]);
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
+        std::cerr << "Usage: " << argv[0] << " [executor]"
+                  << std::endl;
+        std::exit(-1);
     }
 
+    // Figure out where to run the code
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
+
     // Read the input matrix file directory
     std::string input_mtx = "data/A.mtx";
     if (argc == 3) {
diff --git a/examples/poisson-solver/doc/results.dox b/examples/poisson-solver/doc/results.dox
index 6dd4180b920..7d123881371 100644
--- a/examples/poisson-solver/doc/results.dox
+++ b/examples/poisson-solver/doc/results.dox
@@ -3,7 +3,7 @@ This is the expected output:
 
 @code{.cpp}
 
-Usage: executable DISCRETIZATION_POINTS [executor]
+Solve complete.
 The average relative error is 2.52236e-11
 
 @endcode
diff --git a/examples/poisson-solver/poisson-solver.cpp b/examples/poisson-solver/poisson-solver.cpp
index 458178e505b..813fa5e8bad 100644
--- a/examples/poisson-solver/poisson-solver.cpp
+++ b/examples/poisson-solver/poisson-solver.cpp
@@ -125,15 +125,16 @@ int main(int argc, char *argv[])
     // Print version information
     std::cout << gko::version_info::get() << std::endl;
 
-    if (argc < 2) {
-        std::cout << "Usage: executable DISCRETIZATION_POINTS [executor]"
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
+        std::cerr << "Usage: " << argv[0] << " [executor] [DISCRETIZATION_POINTS]"
                   << std::endl;
+        std::exit(-1);
     }
 
     // Get number of discretization points
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
     const unsigned int discretization_points =
-        argc >= 2 ? std::atoi(argv[1]) : 100;
-    const auto executor_string = argc >= 3 ? argv[2] : "reference";
+        argc >= 3 ? std::atoi(argv[2]) : 100;
 
     // Figure out where to run the code
     std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
@@ -189,7 +190,7 @@ int main(int argc, char *argv[])
 
     // Uncomment to print the solution
     // print_solution<ValueType>(u0, u1, lend(u));
-    std::cout << "The average relative error is "
+    std::cout << "Solve complete.\nThe average relative error is "
               << calculate_error(discretization_points, lend(u), correct_u) /
                      static_cast<gko::remove_complex<ValueType>>(
                          discretization_points)
diff --git a/examples/preconditioned-solver/preconditioned-solver.cpp b/examples/preconditioned-solver/preconditioned-solver.cpp
index 083553045ab..e799a846f3f 100644
--- a/examples/preconditioned-solver/preconditioned-solver.cpp
+++ b/examples/preconditioned-solver/preconditioned-solver.cpp
@@ -36,6 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <fstream>
 #include <iostream>
+#include <map>
 #include <string>
 
 
@@ -56,22 +57,32 @@ int main(int argc, char *argv[])
     std::cout << gko::version_info::get() << std::endl;
 
     // Figure out where to run the code
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
+        std::cerr << "Usage: " << argv[0] << " [executor]"
+                  << std::endl;
         std::exit(-1);
     }
 
+    // Figure out where to run the code
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
+
     // Read data
     auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
     auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
diff --git a/examples/simple-solver-logging/simple-solver-logging.cpp b/examples/simple-solver-logging/simple-solver-logging.cpp
index bf918d8a7e2..96d32fa4b2a 100644
--- a/examples/simple-solver-logging/simple-solver-logging.cpp
+++ b/examples/simple-solver-logging/simple-solver-logging.cpp
@@ -36,6 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <fstream>
 #include <iomanip>
 #include <iostream>
+#include <map>
 #include <string>
 
 
@@ -72,23 +73,31 @@ int main(int argc, char *argv[])
     // Print version information
     std::cout << gko::version_info::get() << std::endl;
 
-    // Figure out where to run the code
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else {
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
         std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
+    // Figure out where to run the code
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
+
     // Read data
     auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
     auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
diff --git a/examples/simple-solver/simple-solver.cpp b/examples/simple-solver/simple-solver.cpp
index 4cf6f0b8165..baf18c49d45 100644
--- a/examples/simple-solver/simple-solver.cpp
+++ b/examples/simple-solver/simple-solver.cpp
@@ -39,6 +39,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <fstream>
 // Add the C++ iostream header to output information to the console.
 #include <iostream>
+// Add the STL map header for the executor selection
+#include <map>
 // Add the string manipulation header to handle strings.
 #include <string>
 
@@ -65,6 +67,11 @@ int main(int argc, char *argv[])
     // Print the ginkgo version information.
     std::cout << gko::version_info::get() << std::endl;
 
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
+        std::cerr << "Usage: " << argv[0] << " [executor] " << std::endl;
+        std::exit(-1);
+    }
+
     // @sect3{Where do you want to run your solver ?}
     // The gko::Executor class is one of the cornerstones of Ginkgo. Currently,
     // we have support for
@@ -75,21 +82,24 @@ int main(int argc, char *argv[])
     // @note With the help of C++, you see that you only ever need to change the
     // executor and all the other functions/ routines within Ginkgo should
     // automatically work and run on the executor with any other changes.
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create(), true);
-    } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
-        std::exit(-1);
-    }
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
 
     // @sect3{Reading your data and transfer to the proper device.}
     // Read the matrix, right hand side and the initial solution using the @ref
diff --git a/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp b/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
index 360d890ddae..3edb4e99e39 100644
--- a/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
+++ b/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
@@ -235,9 +235,15 @@ int main(int argc, char *argv[])
     // Print version information
     std::cout << gko::version_info::get() << std::endl;
 
+    if (argc == 2 && std::string(argv[1]) == "--help") {
+        std::cerr << "Usage: " << argv[0] << " [executor] [DISCRETIZATION_POINTS]"
+                  << std::endl;
+        std::exit(-1);
+    }
+
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
     const IndexType discretization_points =
-        argc >= 2 ? std::atoi(argv[1]) : 100;
-    const auto executor_string = argc >= 3 ? argv[2] : "reference";
+        argc >= 3 ? std::atoi(argv[2]) : 100;
 
     // problem:
     auto correct_u = [](ValueType x) { return x * x * x; };

From 5781e5db30eb9bc6ddc74cf5670b5e03b7c7c828 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 9 Oct 2020 11:54:22 +0200
Subject: [PATCH 0030/1008] add gko::dim overload for Identity

---
 benchmark/utils/overhead_linop.hpp      |  2 +-
 core/test/matrix/identity.cpp           | 30 +++++++++++++++++++++++++
 include/ginkgo/core/matrix/identity.hpp | 11 +++++++++
 include/ginkgo/core/solver/bicg.hpp     |  2 +-
 include/ginkgo/core/solver/bicgstab.hpp |  2 +-
 include/ginkgo/core/solver/cg.hpp       |  2 +-
 include/ginkgo/core/solver/cgs.hpp      |  2 +-
 include/ginkgo/core/solver/fcg.hpp      |  2 +-
 include/ginkgo/core/solver/gmres.hpp    |  2 +-
 include/ginkgo/core/solver/ir.hpp       |  2 +-
 10 files changed, 49 insertions(+), 8 deletions(-)

diff --git a/benchmark/utils/overhead_linop.hpp b/benchmark/utils/overhead_linop.hpp
index 0afbf57e12c..2c2aa4baccf 100644
--- a/benchmark/utils/overhead_linop.hpp
+++ b/benchmark/utils/overhead_linop.hpp
@@ -208,7 +208,7 @@ class Overhead : public EnableLinOp<Overhead<ValueType>>,
                 parameters_.preconditioner->generate(system_matrix_));
         } else {
             set_preconditioner(matrix::Identity<ValueType>::create(
-                this->get_executor(), this->get_size()[0]));
+                this->get_executor(), this->get_size()));
         }
         stop_criterion_factory_ =
             stop::combine(std::move(parameters_.criteria));
diff --git a/core/test/matrix/identity.cpp b/core/test/matrix/identity.cpp
index f890a9dd039..b145cf35705 100644
--- a/core/test/matrix/identity.cpp
+++ b/core/test/matrix/identity.cpp
@@ -36,6 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
+#include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
@@ -72,10 +73,29 @@ TYPED_TEST(Identity, CanBeConstructedWithSize)
 {
     using Id = typename TestFixture::Id;
     auto identity = Id::create(this->exec, 5);
+
     ASSERT_EQ(identity->get_size(), gko::dim<2>(5, 5));
 }
 
 
+TYPED_TEST(Identity, CanBeConstructedWithSquareSize)
+{
+    using Id = typename TestFixture::Id;
+    auto identity = Id::create(this->exec, gko::dim<2>(5, 5));
+
+    ASSERT_EQ(identity->get_size(), gko::dim<2>(5, 5));
+}
+
+
+TYPED_TEST(Identity, FailsConstructionWithRectangularSize)
+{
+    using Id = typename TestFixture::Id;
+
+    ASSERT_THROW(Id::create(this->exec, gko::dim<2>(5, 4)),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Identity, AppliesToVector)
 {
     using Id = typename TestFixture::Id;
@@ -127,4 +147,14 @@ TYPED_TEST(IdentityFactory, CanGenerateIdentityMatrix)
 }
 
 
+TYPED_TEST(IdentityFactory, FailsToGenerateRectangularIdentityMatrix)
+{
+    auto exec = gko::ReferenceExecutor::create();
+    auto id_factory = gko::matrix::IdentityFactory<TypeParam>::create(exec);
+    auto mtx = gko::matrix::Dense<TypeParam>::create(exec, gko::dim<2>{5, 4});
+
+    ASSERT_THROW(id_factory->generate(std::move(mtx)), gko::DimensionMismatch);
+}
+
+
 }  // namespace
diff --git a/include/ginkgo/core/matrix/identity.hpp b/include/ginkgo/core/matrix/identity.hpp
index 4cc9065d92f..06682c46abb 100644
--- a/include/ginkgo/core/matrix/identity.hpp
+++ b/include/ginkgo/core/matrix/identity.hpp
@@ -88,6 +88,17 @@ class Identity : public EnableLinOp<Identity<ValueType>>,
         : EnableLinOp<Identity>(exec)
     {}
 
+    /**
+     * Creates an Identity matrix of the specified size.
+     *
+     * @param size  size of the matrix (must be square)
+     */
+    Identity(std::shared_ptr<const Executor> exec, dim<2> size)
+        : EnableLinOp<Identity>(exec, size)
+    {
+        GKO_ASSERT_IS_SQUARE_MATRIX(this);
+    }
+
     /**
      * Creates an Identity matrix of the specified size.
      *
diff --git a/include/ginkgo/core/solver/bicg.hpp b/include/ginkgo/core/solver/bicg.hpp
index 7d8ecffa547..dedf9d82529 100644
--- a/include/ginkgo/core/solver/bicg.hpp
+++ b/include/ginkgo/core/solver/bicg.hpp
@@ -171,7 +171,7 @@ class Bicg : public EnableLinOp<Bicg<ValueType>>,
                 parameters_.preconditioner->generate(system_matrix_));
         } else {
             set_preconditioner(matrix::Identity<ValueType>::create(
-                this->get_executor(), this->get_size()[0]));
+                this->get_executor(), this->get_size()));
         }
         stop_criterion_factory_ =
             stop::combine(std::move(parameters_.criteria));
diff --git a/include/ginkgo/core/solver/bicgstab.hpp b/include/ginkgo/core/solver/bicgstab.hpp
index d300bd6a1cd..91e558461b1 100644
--- a/include/ginkgo/core/solver/bicgstab.hpp
+++ b/include/ginkgo/core/solver/bicgstab.hpp
@@ -176,7 +176,7 @@ class Bicgstab : public EnableLinOp<Bicgstab<ValueType>>,
                 parameters_.preconditioner->generate(system_matrix_));
         } else {
             set_preconditioner(matrix::Identity<ValueType>::create(
-                this->get_executor(), this->get_size()[0]));
+                this->get_executor(), this->get_size()));
         }
         stop_criterion_factory_ =
             stop::combine(std::move(parameters_.criteria));
diff --git a/include/ginkgo/core/solver/cg.hpp b/include/ginkgo/core/solver/cg.hpp
index ed5bc797885..f1b8dbc8116 100644
--- a/include/ginkgo/core/solver/cg.hpp
+++ b/include/ginkgo/core/solver/cg.hpp
@@ -172,7 +172,7 @@ class Cg : public EnableLinOp<Cg<ValueType>>,
                 parameters_.preconditioner->generate(system_matrix_));
         } else {
             set_preconditioner(matrix::Identity<ValueType>::create(
-                this->get_executor(), this->get_size()[0]));
+                this->get_executor(), this->get_size()));
         }
         stop_criterion_factory_ =
             stop::combine(std::move(parameters_.criteria));
diff --git a/include/ginkgo/core/solver/cgs.hpp b/include/ginkgo/core/solver/cgs.hpp
index e24c7bf3c3b..04906c6dbb7 100644
--- a/include/ginkgo/core/solver/cgs.hpp
+++ b/include/ginkgo/core/solver/cgs.hpp
@@ -169,7 +169,7 @@ class Cgs : public EnableLinOp<Cgs<ValueType>>,
                 parameters_.preconditioner->generate(system_matrix_));
         } else {
             set_preconditioner(matrix::Identity<ValueType>::create(
-                this->get_executor(), this->get_size()[0]));
+                this->get_executor(), this->get_size()));
         }
         stop_criterion_factory_ =
             stop::combine(std::move(parameters_.criteria));
diff --git a/include/ginkgo/core/solver/fcg.hpp b/include/ginkgo/core/solver/fcg.hpp
index 4fb8261631e..c20ed779e2d 100644
--- a/include/ginkgo/core/solver/fcg.hpp
+++ b/include/ginkgo/core/solver/fcg.hpp
@@ -177,7 +177,7 @@ class Fcg : public EnableLinOp<Fcg<ValueType>>,
                 parameters_.preconditioner->generate(system_matrix_));
         } else {
             set_preconditioner(matrix::Identity<ValueType>::create(
-                this->get_executor(), this->get_size()[0]));
+                this->get_executor(), this->get_size()));
         }
         stop_criterion_factory_ =
             stop::combine(std::move(parameters_.criteria));
diff --git a/include/ginkgo/core/solver/gmres.hpp b/include/ginkgo/core/solver/gmres.hpp
index 84fef57d266..deb0e7e8e0c 100644
--- a/include/ginkgo/core/solver/gmres.hpp
+++ b/include/ginkgo/core/solver/gmres.hpp
@@ -191,7 +191,7 @@ class Gmres : public EnableLinOp<Gmres<ValueType>>,
                 parameters_.preconditioner->generate(system_matrix_));
         } else {
             set_preconditioner(matrix::Identity<ValueType>::create(
-                this->get_executor(), this->get_size()[0]));
+                this->get_executor(), this->get_size()));
         }
         if (parameters_.krylov_dim) {
             krylov_dim_ = parameters_.krylov_dim;
diff --git a/include/ginkgo/core/solver/ir.hpp b/include/ginkgo/core/solver/ir.hpp
index 2d744a2960b..d396214924e 100644
--- a/include/ginkgo/core/solver/ir.hpp
+++ b/include/ginkgo/core/solver/ir.hpp
@@ -226,7 +226,7 @@ class Ir : public EnableLinOp<Ir<ValueType>>, public Transposable {
             solver_ = parameters_.solver->generate(system_matrix_);
         } else {
             solver_ = matrix::Identity<ValueType>::create(this->get_executor(),
-                                                          this->get_size()[0]);
+                                                          this->get_size());
         }
         relaxation_factor_ = gko::initialize<matrix::Dense<ValueType>>(
             {parameters_.relaxation_factor}, this->get_executor());

From 45cb8189223282d070b453c1eb363202b36857f6 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 9 Oct 2020 11:54:53 +0200
Subject: [PATCH 0031/1008] check for square matrix at solver construction

---
 core/solver/gmres.cpp                   | 2 --
 core/test/solver/bicg.cpp               | 2 +-
 core/test/solver/bicgstab.cpp           | 2 +-
 core/test/solver/cg.cpp                 | 2 +-
 core/test/solver/cgs.cpp                | 2 +-
 core/test/solver/fcg.cpp                | 2 +-
 core/test/solver/gmres.cpp              | 2 +-
 core/test/solver/ir.cpp                 | 4 ++--
 include/ginkgo/core/solver/bicg.hpp     | 1 +
 include/ginkgo/core/solver/bicgstab.hpp | 1 +
 include/ginkgo/core/solver/cg.hpp       | 1 +
 include/ginkgo/core/solver/cgs.hpp      | 1 +
 include/ginkgo/core/solver/fcg.hpp      | 1 +
 include/ginkgo/core/solver/gmres.hpp    | 1 +
 include/ginkgo/core/solver/ir.hpp       | 1 +
 15 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/core/solver/gmres.cpp b/core/solver/gmres.cpp
index 9e9c39c3848..b33d85db0de 100644
--- a/core/solver/gmres.cpp
+++ b/core/solver/gmres.cpp
@@ -94,8 +94,6 @@ std::unique_ptr<LinOp> Gmres<ValueType>::conj_transpose() const
 template <typename ValueType>
 void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix_);
-
     using Vector = matrix::Dense<ValueType>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
 
diff --git a/core/test/solver/bicg.cpp b/core/test/solver/bicg.cpp
index 0d8763cdc86..8e09f083dfc 100644
--- a/core/test/solver/bicg.cpp
+++ b/core/test/solver/bicg.cpp
@@ -257,7 +257,7 @@ TYPED_TEST(Bicg, ThrowsOnWrongPreconditionerInFactory)
     using Mtx = typename TestFixture::Mtx;
     using Solver = typename TestFixture::Solver;
     std::shared_ptr<Mtx> wrong_sized_mtx =
-        Mtx::create(this->exec, gko::dim<2>{1, 3});
+        Mtx::create(this->exec, gko::dim<2>{2, 2});
     std::shared_ptr<Solver> bicg_precond =
         Solver::build()
             .with_criteria(
diff --git a/core/test/solver/bicgstab.cpp b/core/test/solver/bicgstab.cpp
index 16d5b8a9bff..d3943d0ffd5 100644
--- a/core/test/solver/bicgstab.cpp
+++ b/core/test/solver/bicgstab.cpp
@@ -249,7 +249,7 @@ TYPED_TEST(Bicgstab, ThrowsOnWrongPreconditionerInFactory)
     using Mtx = typename TestFixture::Mtx;
     using Solver = typename TestFixture::Solver;
     std::shared_ptr<Mtx> wrong_sized_mtx =
-        Mtx::create(this->exec, gko::dim<2>{1, 3});
+        Mtx::create(this->exec, gko::dim<2>{2, 2});
     std::shared_ptr<Solver> bicgstab_precond =
         Solver::build()
             .with_criteria(
diff --git a/core/test/solver/cg.cpp b/core/test/solver/cg.cpp
index e6652defb0e..d77040fd027 100644
--- a/core/test/solver/cg.cpp
+++ b/core/test/solver/cg.cpp
@@ -256,7 +256,7 @@ TYPED_TEST(Cg, ThrowsOnWrongPreconditionerInFactory)
     using Mtx = typename TestFixture::Mtx;
     using Solver = typename TestFixture::Solver;
     std::shared_ptr<Mtx> wrong_sized_mtx =
-        Mtx::create(this->exec, gko::dim<2>{1, 3});
+        Mtx::create(this->exec, gko::dim<2>{2, 2});
     std::shared_ptr<Solver> cg_precond =
         Solver::build()
             .with_criteria(
diff --git a/core/test/solver/cgs.cpp b/core/test/solver/cgs.cpp
index 04f7c31aab9..e648f026617 100644
--- a/core/test/solver/cgs.cpp
+++ b/core/test/solver/cgs.cpp
@@ -256,7 +256,7 @@ TYPED_TEST(Cgs, ThrowsOnWrongPreconditionerInFactory)
     using Mtx = typename TestFixture::Mtx;
     using Solver = typename TestFixture::Solver;
     std::shared_ptr<Mtx> wrong_sized_mtx =
-        Mtx::create(this->exec, gko::dim<2>{1, 3});
+        Mtx::create(this->exec, gko::dim<2>{2, 2});
     std::shared_ptr<Solver> cgs_precond =
         Solver::build()
             .with_criteria(
diff --git a/core/test/solver/fcg.cpp b/core/test/solver/fcg.cpp
index 6b9c0e954a7..0b5e0c87290 100644
--- a/core/test/solver/fcg.cpp
+++ b/core/test/solver/fcg.cpp
@@ -241,7 +241,7 @@ TYPED_TEST(Fcg, ThrowsOnWrongPreconditionerInFactory)
     using Mtx = typename TestFixture::Mtx;
     using Solver = typename TestFixture::Solver;
     std::shared_ptr<Mtx> wrong_sized_mtx =
-        Mtx::create(this->exec, gko::dim<2>{1, 3});
+        Mtx::create(this->exec, gko::dim<2>{2, 2});
     std::shared_ptr<Solver> fcg_precond =
         Solver::build()
             .with_criteria(
diff --git a/core/test/solver/gmres.cpp b/core/test/solver/gmres.cpp
index 4765f07183b..52fe9818487 100644
--- a/core/test/solver/gmres.cpp
+++ b/core/test/solver/gmres.cpp
@@ -313,7 +313,7 @@ TYPED_TEST(Gmres, ThrowsOnWrongPreconditionerInFactory)
     using Mtx = typename TestFixture::Mtx;
     using Solver = typename TestFixture::Solver;
     std::shared_ptr<Mtx> wrong_sized_mtx =
-        Mtx::create(this->exec, gko::dim<2>{1, 3});
+        Mtx::create(this->exec, gko::dim<2>{2, 2});
     std::shared_ptr<Solver> gmres_precond =
         Solver::build()
             .with_criteria(
diff --git a/core/test/solver/ir.cpp b/core/test/solver/ir.cpp
index b711c511e97..ef77c78cc60 100644
--- a/core/test/solver/ir.cpp
+++ b/core/test/solver/ir.cpp
@@ -252,7 +252,7 @@ TYPED_TEST(Ir, ThrowsOnWrongInnerSolverInFactory)
     using Mtx = typename TestFixture::Mtx;
     using Solver = typename TestFixture::Solver;
     std::shared_ptr<Mtx> wrong_sized_mtx =
-        Mtx::create(this->exec, gko::dim<2>{1, 3});
+        Mtx::create(this->exec, gko::dim<2>{2, 2});
     std::shared_ptr<Solver> ir_solver =
         Solver::build()
             .with_criteria(
@@ -300,7 +300,7 @@ TYPED_TEST(Ir, ThrowOnWrongInnerSolverSet)
     using Mtx = typename TestFixture::Mtx;
     using Solver = typename TestFixture::Solver;
     std::shared_ptr<Mtx> wrong_sized_mtx =
-        Mtx::create(this->exec, gko::dim<2>{1, 3});
+        Mtx::create(this->exec, gko::dim<2>{2, 2});
     std::shared_ptr<Solver> ir_solver =
         Solver::build()
             .with_criteria(
diff --git a/include/ginkgo/core/solver/bicg.hpp b/include/ginkgo/core/solver/bicg.hpp
index dedf9d82529..e2960783a3e 100644
--- a/include/ginkgo/core/solver/bicg.hpp
+++ b/include/ginkgo/core/solver/bicg.hpp
@@ -162,6 +162,7 @@ class Bicg : public EnableLinOp<Bicg<ValueType>>,
           parameters_{factory->get_parameters()},
           system_matrix_{std::move(system_matrix)}
     {
+        GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix_);
         if (parameters_.generated_preconditioner) {
             GKO_ASSERT_EQUAL_DIMENSIONS(parameters_.generated_preconditioner,
                                         this);
diff --git a/include/ginkgo/core/solver/bicgstab.hpp b/include/ginkgo/core/solver/bicgstab.hpp
index 91e558461b1..e6fce0d9f53 100644
--- a/include/ginkgo/core/solver/bicgstab.hpp
+++ b/include/ginkgo/core/solver/bicgstab.hpp
@@ -167,6 +167,7 @@ class Bicgstab : public EnableLinOp<Bicgstab<ValueType>>,
           parameters_{factory->get_parameters()},
           system_matrix_{std::move(system_matrix)}
     {
+        GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix_);
         if (parameters_.generated_preconditioner) {
             GKO_ASSERT_EQUAL_DIMENSIONS(parameters_.generated_preconditioner,
                                         this);
diff --git a/include/ginkgo/core/solver/cg.hpp b/include/ginkgo/core/solver/cg.hpp
index f1b8dbc8116..fbe94677933 100644
--- a/include/ginkgo/core/solver/cg.hpp
+++ b/include/ginkgo/core/solver/cg.hpp
@@ -163,6 +163,7 @@ class Cg : public EnableLinOp<Cg<ValueType>>,
           parameters_{factory->get_parameters()},
           system_matrix_{std::move(system_matrix)}
     {
+        GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix_);
         if (parameters_.generated_preconditioner) {
             GKO_ASSERT_EQUAL_DIMENSIONS(parameters_.generated_preconditioner,
                                         this);
diff --git a/include/ginkgo/core/solver/cgs.hpp b/include/ginkgo/core/solver/cgs.hpp
index 04906c6dbb7..3d7c8c0a854 100644
--- a/include/ginkgo/core/solver/cgs.hpp
+++ b/include/ginkgo/core/solver/cgs.hpp
@@ -160,6 +160,7 @@ class Cgs : public EnableLinOp<Cgs<ValueType>>,
           parameters_{factory->get_parameters()},
           system_matrix_{std::move(system_matrix)}
     {
+        GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix_);
         if (parameters_.generated_preconditioner) {
             GKO_ASSERT_EQUAL_DIMENSIONS(parameters_.generated_preconditioner,
                                         this);
diff --git a/include/ginkgo/core/solver/fcg.hpp b/include/ginkgo/core/solver/fcg.hpp
index c20ed779e2d..3d25ff057bf 100644
--- a/include/ginkgo/core/solver/fcg.hpp
+++ b/include/ginkgo/core/solver/fcg.hpp
@@ -168,6 +168,7 @@ class Fcg : public EnableLinOp<Fcg<ValueType>>,
           parameters_{factory->get_parameters()},
           system_matrix_{std::move(system_matrix)}
     {
+        GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix_);
         if (parameters_.generated_preconditioner) {
             GKO_ASSERT_EQUAL_DIMENSIONS(parameters_.generated_preconditioner,
                                         this);
diff --git a/include/ginkgo/core/solver/gmres.hpp b/include/ginkgo/core/solver/gmres.hpp
index deb0e7e8e0c..36142219a7f 100644
--- a/include/ginkgo/core/solver/gmres.hpp
+++ b/include/ginkgo/core/solver/gmres.hpp
@@ -182,6 +182,7 @@ class Gmres : public EnableLinOp<Gmres<ValueType>>,
           parameters_{factory->get_parameters()},
           system_matrix_{std::move(system_matrix)}
     {
+        GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix_);
         if (parameters_.generated_preconditioner) {
             GKO_ASSERT_EQUAL_DIMENSIONS(parameters_.generated_preconditioner,
                                         this);
diff --git a/include/ginkgo/core/solver/ir.hpp b/include/ginkgo/core/solver/ir.hpp
index d396214924e..1059c733614 100644
--- a/include/ginkgo/core/solver/ir.hpp
+++ b/include/ginkgo/core/solver/ir.hpp
@@ -219,6 +219,7 @@ class Ir : public EnableLinOp<Ir<ValueType>>, public Transposable {
           parameters_{factory->get_parameters()},
           system_matrix_{std::move(system_matrix)}
     {
+        GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix_);
         if (parameters_.generated_solver) {
             solver_ = parameters_.generated_solver;
             GKO_ASSERT_EQUAL_DIMENSIONS(solver_, this);

From 61ae226ace3aa67122ea5a5c87af857a12c3b40b Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 9 Oct 2020 12:10:58 +0200
Subject: [PATCH 0032/1008] add solver core tests for rectangular matrices

---
 core/test/solver/bicg.cpp      | 12 ++++++++++++
 core/test/solver/bicgstab.cpp  | 12 ++++++++++++
 core/test/solver/cg.cpp        | 12 ++++++++++++
 core/test/solver/cgs.cpp       | 12 ++++++++++++
 core/test/solver/fcg.cpp       | 12 ++++++++++++
 core/test/solver/gmres.cpp     | 12 ++++++++++++
 core/test/solver/ir.cpp        | 12 ++++++++++++
 core/test/solver/lower_trs.cpp | 11 +++++++++++
 core/test/solver/upper_trs.cpp | 11 +++++++++++
 9 files changed, 106 insertions(+)

diff --git a/core/test/solver/bicg.cpp b/core/test/solver/bicg.cpp
index 8e09f083dfc..5d252091715 100644
--- a/core/test/solver/bicg.cpp
+++ b/core/test/solver/bicg.cpp
@@ -276,6 +276,18 @@ TYPED_TEST(Bicg, ThrowsOnWrongPreconditionerInFactory)
 }
 
 
+TYPED_TEST(Bicg, ThrowsOnRectangularMatrixInFactory)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<Mtx> rectangular_mtx =
+        Mtx::create(this->exec, gko::dim<2>{1, 2});
+
+    ASSERT_THROW(this->bicg_factory->generate(rectangular_mtx),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Bicg, CanSetPreconditioner)
 {
     using Solver = typename TestFixture::Solver;
diff --git a/core/test/solver/bicgstab.cpp b/core/test/solver/bicgstab.cpp
index d3943d0ffd5..7be7564d00f 100644
--- a/core/test/solver/bicgstab.cpp
+++ b/core/test/solver/bicgstab.cpp
@@ -268,6 +268,18 @@ TYPED_TEST(Bicgstab, ThrowsOnWrongPreconditionerInFactory)
 }
 
 
+TYPED_TEST(Bicgstab, ThrowsOnRectangularMatrixInFactory)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<Mtx> rectangular_mtx =
+        Mtx::create(this->exec, gko::dim<2>{1, 2});
+
+    ASSERT_THROW(this->bicgstab_factory->generate(rectangular_mtx),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Bicgstab, CanSetPreconditioner)
 {
     using Solver = typename TestFixture::Solver;
diff --git a/core/test/solver/cg.cpp b/core/test/solver/cg.cpp
index d77040fd027..0efb5af1bc8 100644
--- a/core/test/solver/cg.cpp
+++ b/core/test/solver/cg.cpp
@@ -275,6 +275,18 @@ TYPED_TEST(Cg, ThrowsOnWrongPreconditionerInFactory)
 }
 
 
+TYPED_TEST(Cg, ThrowsOnRectangularMatrixInFactory)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<Mtx> rectangular_mtx =
+        Mtx::create(this->exec, gko::dim<2>{1, 2});
+
+    ASSERT_THROW(this->cg_factory->generate(rectangular_mtx),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Cg, CanSetPreconditioner)
 {
     using Solver = typename TestFixture::Solver;
diff --git a/core/test/solver/cgs.cpp b/core/test/solver/cgs.cpp
index e648f026617..3acaf785bdc 100644
--- a/core/test/solver/cgs.cpp
+++ b/core/test/solver/cgs.cpp
@@ -275,6 +275,18 @@ TYPED_TEST(Cgs, ThrowsOnWrongPreconditionerInFactory)
 }
 
 
+TYPED_TEST(Cgs, ThrowsOnRectangularMatrixInFactory)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<Mtx> rectangular_mtx =
+        Mtx::create(this->exec, gko::dim<2>{1, 2});
+
+    ASSERT_THROW(this->cgs_factory->generate(rectangular_mtx),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Cgs, CanSetPreconditioner)
 {
     using Solver = typename TestFixture::Solver;
diff --git a/core/test/solver/fcg.cpp b/core/test/solver/fcg.cpp
index 0b5e0c87290..11048379e81 100644
--- a/core/test/solver/fcg.cpp
+++ b/core/test/solver/fcg.cpp
@@ -260,6 +260,18 @@ TYPED_TEST(Fcg, ThrowsOnWrongPreconditionerInFactory)
 }
 
 
+TYPED_TEST(Fcg, ThrowsOnRectangularMatrixInFactory)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<Mtx> rectangular_mtx =
+        Mtx::create(this->exec, gko::dim<2>{1, 2});
+
+    ASSERT_THROW(this->fcg_factory->generate(rectangular_mtx),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Fcg, CanSetPreconditioner)
 {
     using Solver = typename TestFixture::Solver;
diff --git a/core/test/solver/gmres.cpp b/core/test/solver/gmres.cpp
index 52fe9818487..1c9bf67b0ca 100644
--- a/core/test/solver/gmres.cpp
+++ b/core/test/solver/gmres.cpp
@@ -332,6 +332,18 @@ TYPED_TEST(Gmres, ThrowsOnWrongPreconditionerInFactory)
 }
 
 
+TYPED_TEST(Gmres, ThrowsOnRectangularMatrixInFactory)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<Mtx> rectangular_mtx =
+        Mtx::create(this->exec, gko::dim<2>{1, 2});
+
+    ASSERT_THROW(this->gmres_factory->generate(rectangular_mtx),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Gmres, CanSetPreconditioner)
 {
     using Solver = typename TestFixture::Solver;
diff --git a/core/test/solver/ir.cpp b/core/test/solver/ir.cpp
index ef77c78cc60..3e212430e29 100644
--- a/core/test/solver/ir.cpp
+++ b/core/test/solver/ir.cpp
@@ -319,6 +319,18 @@ TYPED_TEST(Ir, ThrowOnWrongInnerSolverSet)
 }
 
 
+TYPED_TEST(Ir, ThrowsOnRectangularMatrixInFactory)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<Mtx> rectangular_mtx =
+        Mtx::create(this->exec, gko::dim<2>{1, 2});
+
+    ASSERT_THROW(this->ir_factory->generate(rectangular_mtx),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Ir, DefaultRelaxationFactor)
 {
     using value_type = typename TestFixture::value_type;
diff --git a/core/test/solver/lower_trs.cpp b/core/test/solver/lower_trs.cpp
index be12f10ef53..0153b8aed2d 100644
--- a/core/test/solver/lower_trs.cpp
+++ b/core/test/solver/lower_trs.cpp
@@ -75,4 +75,15 @@ TYPED_TEST(LowerTrs, LowerTrsFactoryKnowsItsExecutor)
 }
 
 
+TYPED_TEST(LowerTrs, ThrowsOnRectangularMatrixInFactory)
+{
+    using Mtx = gko::matrix::Dense<typename TestFixture::value_type>;
+    std::shared_ptr<Mtx> rectangular_matrix =
+        Mtx::create(this->exec, gko::dim<2>{1, 2});
+
+    ASSERT_THROW(this->lower_trs_factory->generate(rectangular_matrix),
+                 gko::DimensionMismatch);
+}
+
+
 }  // namespace
diff --git a/core/test/solver/upper_trs.cpp b/core/test/solver/upper_trs.cpp
index 1ec759fa47d..10e2f727cc7 100644
--- a/core/test/solver/upper_trs.cpp
+++ b/core/test/solver/upper_trs.cpp
@@ -75,4 +75,15 @@ TYPED_TEST(UpperTrs, UpperTrsFactoryKnowsItsExecutor)
 }
 
 
+TYPED_TEST(UpperTrs, ThrowsOnRectangularMatrixInFactory)
+{
+    using Mtx = gko::matrix::Dense<typename TestFixture::value_type>;
+    std::shared_ptr<Mtx> rectangular_matrix =
+        Mtx::create(this->exec, gko::dim<2>{1, 2});
+
+    ASSERT_THROW(this->upper_trs_factory->generate(rectangular_matrix),
+                 gko::DimensionMismatch);
+}
+
+
 }  // namespace

From aa220efedfe2abb50f4dc04f1779750192bc55a7 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 13 Oct 2020 11:50:16 +0200
Subject: [PATCH 0033/1008] fix CuspCsrEx benchmark memory leak

---
 benchmark/utils/cuda_linops.hpp | 25 ++++---------------------
 1 file changed, 4 insertions(+), 21 deletions(-)

diff --git a/benchmark/utils/cuda_linops.hpp b/benchmark/utils/cuda_linops.hpp
index 08f5ea7b61d..17f386016df 100644
--- a/benchmark/utils/cuda_linops.hpp
+++ b/benchmark/utils/cuda_linops.hpp
@@ -328,21 +328,6 @@ class CuspCsrEx
         return csr_->get_num_stored_elements();
     }
 
-    ~CuspCsrEx() override
-    {
-        const auto id = this->get_gpu_exec()->get_device_id();
-        if (set_buffer_) {
-            try {
-                gko::cuda::device_guard g{id};
-                GKO_ASSERT_NO_CUDA_ERRORS(cudaFree(buffer_));
-            } catch (const std::exception &e) {
-                std::cerr
-                    << "Error when unallocating CuspCsrEx temporary buffer: "
-                    << e.what() << std::endl;
-            }
-        }
-    }
-
     CuspCsrEx(const CuspCsrEx &other) = delete;
 
     CuspCsrEx &operator=(const CuspCsrEx &other) = default;
@@ -370,14 +355,13 @@ class CuspCsrEx
             csr_->get_num_stored_elements(), &alpha, this->get_descr(),
             csr_->get_const_values(), csr_->get_const_row_ptrs(),
             csr_->get_const_col_idxs(), db, &beta, dx, &buffer_size);
-        GKO_ASSERT_NO_CUDA_ERRORS(cudaMalloc(&buffer_, buffer_size));
-        set_buffer_ = true;
+        buffer_.resize_and_reset(buffer_size);
 
         gko::kernels::cuda::cusparse::spmv<ValueType, IndexType>(
             handle, algmode_, trans_, this->get_size()[0], this->get_size()[1],
             csr_->get_num_stored_elements(), &alpha, this->get_descr(),
             csr_->get_const_values(), csr_->get_const_row_ptrs(),
-            csr_->get_const_col_idxs(), db, &beta, dx, buffer_);
+            csr_->get_const_col_idxs(), db, &beta, dx, buffer_.get_data());
 
         // Exiting the scope sets the pointer mode back to the default
         // DEVICE for Ginkgo
@@ -390,7 +374,7 @@ class CuspCsrEx
           csr_(std::move(
               csr::create(exec, std::make_shared<typename csr::classical>()))),
           trans_(CUSPARSE_OPERATION_NON_TRANSPOSE),
-          set_buffer_(false)
+          buffer_(exec)
     {
 #ifdef ALLOWMP
         algmode_ = CUSPARSE_ALG_MERGE_PATH;
@@ -401,8 +385,7 @@ class CuspCsrEx
     std::shared_ptr<csr> csr_;
     cusparseOperation_t trans_;
     cusparseAlgMode_t algmode_;
-    mutable void *buffer_;
-    mutable bool set_buffer_;
+    mutable gko::Array<char> buffer_;
 };
 
 

From 16275b690a0eef61dafc245eec369f4bb7f4c9d6 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 8 Apr 2020 13:53:52 +0200
Subject: [PATCH 0034/1008] Add reordering base framework.

---
 .../ginkgo/core/reorder/reordering_base.hpp   | 167 ++++++++++++++++++
 1 file changed, 167 insertions(+)
 create mode 100644 include/ginkgo/core/reorder/reordering_base.hpp

diff --git a/include/ginkgo/core/reorder/reordering_base.hpp b/include/ginkgo/core/reorder/reordering_base.hpp
new file mode 100644
index 00000000000..b97e9e6c341
--- /dev/null
+++ b/include/ginkgo/core/reorder/reordering_base.hpp
@@ -0,0 +1,167 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2019, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_REORDER_REORDERING_BASE_HPP_
+#define GKO_CORE_REORDER_REORDERING_BASE_HPP_
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/abstract_factory.hpp>
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/polymorphic_object.hpp>
+#include <ginkgo/core/base/utils.hpp>
+
+
+namespace gko {
+/**
+ * @brief The reordering namespace.
+ *
+ * @ingroup reorder
+ */
+namespace reorder {
+
+
+/**
+ * The ReorderingBase class is a base class for all the reordering algorithms.
+ * It contains a factory to instantiate the reorderings. It is up to each
+ * specific reordering to decide what to do with the data that is passed to it.
+ */
+class ReorderingBase : public EnableAbstractPolymorphicObject<ReorderingBase> {
+protected:
+    explicit ReorderingBase(std::shared_ptr<const gko::Executor> exec)
+        : EnableAbstractPolymorphicObject<ReorderingBase>(exec)
+    {}
+};
+
+
+/**
+ * This struct is used to pass parameters to the
+ * EnableDefaultReorderingBaseFactory::generate() method. It is the
+ * ComponentsType of ReorderingBaseFactory.
+ */
+struct ReorderingBaseArgs {
+    std::shared_ptr<LinOp> system_matrix;
+
+    ReorderingBaseArgs(std::shared_ptr<LinOp> system_matrix)
+        : system_matrix{system_matrix}
+    {}
+};
+
+
+/**
+ * Declares an Abstract Factory specialized for ReorderingBases
+ */
+using ReorderingBaseFactory =
+    AbstractFactory<ReorderingBase, ReorderingBaseArgs>;
+
+
+/**
+ * This is an alias for the EnableDefaultFactory mixin, which correctly sets the
+ * template parameters to enable a subclass of ReorderingBaseFactory.
+ *
+ * @tparam ConcreteFactory  the concrete factory which is being implemented
+ *                          [CRTP parmeter]
+ * @tparam ConcreteReorderingBase  the concrete ReorderingBase type which this
+ * factory produces, needs to have a constructor which takes a const
+ * ConcreteFactory *, and a const ReorderingBaseArgs * as parameters.
+ * @tparam ParametersType  a subclass of enable_parameters_type template which
+ *                         defines all of the parameters of the factory
+ * @tparam PolymorphicBase  parent of ConcreteFactory in the polymorphic
+ *                          hierarchy, has to be a subclass of
+ * ReorderingBaseFactory
+ */
+template <typename ConcreteFactory, typename ConcreteReorderingBase,
+          typename ParametersType,
+          typename PolymorphicBase = ReorderingBaseFactory>
+using EnableDefaultReorderingBaseFactory =
+    EnableDefaultFactory<ConcreteFactory, ConcreteReorderingBase,
+                         ParametersType, PolymorphicBase>;
+
+
+/**
+ * This macro will generate a default implementation of a ReorderingBaseFactory
+ * for the ReorderingBase subclass it is defined in.
+ *
+ * This macro is very similar to the macro #ENABLE_LIN_OP_FACTORY(). A more
+ * detailed description of the use of these type of macros can be found there.
+ *
+ * @param _reordering_base  concrete operator for which the factory is to be
+ * created [CRTP parameter]
+ * @param _parameters_name  name of the parameters member in the class
+ *                          (its type is `<_parameters_name>_type`, the
+ *                          protected member's name is `<_parameters_name>_`,
+ *                          and the public getter's name is
+ *                          `get_<_parameters_name>()`)
+ * @param _factory_name  name of the generated factory type
+ *
+ * @ingroup reorder
+ */
+#define GKO_ENABLE_REORDERING_BASE_FACTORY(_reordering_base, _parameters_name, \
+                                           _factory_name)                      \
+public:                                                                        \
+    const _parameters_name##_type &get_##_parameters_name() const              \
+    {                                                                          \
+        return _parameters_name##_;                                            \
+    }                                                                          \
+                                                                               \
+    class _factory_name                                                        \
+        : public ::gko::reorder::EnableDefaultReorderingBaseFactory<           \
+              _factory_name, _reordering_base, _parameters_name##_type> {      \
+        friend class ::gko::EnablePolymorphicObject<                           \
+            _factory_name, ::gko::reorder::ReorderingBaseFactory>;             \
+        friend class ::gko::enable_parameters_type<_parameters_name##_type,    \
+                                                   _factory_name>;             \
+        using ::gko::reorder::EnableDefaultReorderingBaseFactory<              \
+            _factory_name, _reordering_base,                                   \
+            _parameters_name##_type>::EnableDefaultReorderingBaseFactory;      \
+    };                                                                         \
+    friend ::gko::reorder::EnableDefaultReorderingBaseFactory<                 \
+        _factory_name, _reordering_base, _parameters_name##_type>;             \
+                                                                               \
+private:                                                                       \
+    _parameters_name##_type _parameters_name##_;                               \
+                                                                               \
+public:                                                                        \
+    static_assert(true,                                                        \
+                  "This assert is used to counter the false positive extra "   \
+                  "semi-colon warnings")
+
+
+}  // namespace reorder
+}  // namespace gko
+
+
+#endif  // GKO_CORE_REORDER_REORDERING_BASE_HPP_

From c40e24dd24d8325541f2483c5ff84baf5cdd7483 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 8 Apr 2020 13:54:49 +0200
Subject: [PATCH 0035/1008] Add Reverse-Cuthill-Mckee reordering.

+ Add framework.
+ Add reference kernels and tests.
+ Add omp kernels and tests.
+ Add 1138_bus as a test matrix.
---
 core/device_hooks/common_kernels.inc.cpp |   19 +
 core/reorder/rcm.cpp                     |   85 +
 core/reorder/rcm_kernels.hpp             |  122 +
 core/test/CMakeLists.txt                 |    1 +
 core/test/reorder/CMakeLists.txt         |    1 +
 core/test/reorder/rcm.cpp                |   72 +
 cuda/reorder/rcm_kernels.cu              |   86 +
 hip/CMakeLists.txt                       |    1 +
 hip/reorder/rcm_kernels.hip.cpp          |   86 +
 include/ginkgo/core/reorder/rcm.hpp      |  184 ++
 matrices/CMakeLists.txt                  |    1 +
 matrices/config.hpp.in                   |    1 +
 matrices/test/1138_bus.mtx               | 2599 ++++++++++++++++++++++
 omp/reorder/rcm_kernels.cpp              |  171 ++
 omp/test/CMakeLists.txt                  |    1 +
 omp/test/reorder/CMakeLists.txt          |    1 +
 omp/test/reorder/rcm_kernels.cpp         |  109 +
 reference/reorder/rcm_kernels.cpp        |  170 ++
 reference/test/CMakeLists.txt            |    1 +
 reference/test/reorder/CMakeLists.txt    |    2 +
 reference/test/reorder/rcm.cpp           |  107 +
 reference/test/reorder/rcm_kernels.cpp   |  124 ++
 22 files changed, 3944 insertions(+)
 create mode 100644 core/reorder/rcm.cpp
 create mode 100644 core/reorder/rcm_kernels.hpp
 create mode 100644 core/test/reorder/CMakeLists.txt
 create mode 100644 core/test/reorder/rcm.cpp
 create mode 100644 cuda/reorder/rcm_kernels.cu
 create mode 100644 hip/reorder/rcm_kernels.hip.cpp
 create mode 100644 include/ginkgo/core/reorder/rcm.hpp
 create mode 100644 matrices/test/1138_bus.mtx
 create mode 100644 omp/reorder/rcm_kernels.cpp
 create mode 100644 omp/test/reorder/CMakeLists.txt
 create mode 100644 omp/test/reorder/rcm_kernels.cpp
 create mode 100644 reference/reorder/rcm_kernels.cpp
 create mode 100644 reference/test/reorder/CMakeLists.txt
 create mode 100644 reference/test/reorder/rcm.cpp
 create mode 100644 reference/test/reorder/rcm_kernels.cpp

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 2f52d52ce8f..b2cdf8e7a59 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -52,6 +52,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/sparsity_csr_kernels.hpp"
 #include "core/preconditioner/isai_kernels.hpp"
 #include "core/preconditioner/jacobi_kernels.hpp"
+#include "core/reorder/rcm_kernels.hpp"
 #include "core/solver/bicg_kernels.hpp"
 #include "core/solver/bicgstab_kernels.hpp"
 #include "core/solver/cg_kernels.hpp"
@@ -1050,6 +1051,24 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 }  // namespace par_ilut_factorization
+namespace rcm {
+
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
+
+
+}  // namespace rcm
 
 
 namespace set_all_statuses {
diff --git a/core/reorder/rcm.cpp b/core/reorder/rcm.cpp
new file mode 100644
index 00000000000..f7e41ecb9b6
--- /dev/null
+++ b/core/reorder/rcm.cpp
@@ -0,0 +1,85 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2019, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/reorder/rcm.hpp>
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/polymorphic_object.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/base/utils.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/permutation.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+
+
+#include "core/matrix/csr_kernels.hpp"
+#include "core/reorder/rcm_kernels.hpp"
+
+
+namespace gko {
+namespace reorder {
+namespace rcm {
+
+
+GKO_REGISTER_OPERATION(get_permutation, rcm::get_permutation);
+GKO_REGISTER_OPERATION(get_degree_of_nodes, rcm::get_degree_of_nodes);
+
+
+}  // namespace rcm
+
+
+template <typename ValueType, typename IndexType>
+void Rcm<ValueType, IndexType>::generate() const
+{
+    IndexType num_rows = adjacency_matrix_->get_size()[0];
+    const auto exec = this->get_executor();
+    // RCM is only valid for symmetric matrices. Need to add an expensive check
+    // for symmetricity here ?
+    exec->run(rcm::make_get_degree_of_nodes(adjacency_matrix_, node_degrees_));
+    exec->run(rcm::make_get_permutation(num_rows, adjacency_matrix_,
+                                        node_degrees_, permutation_,
+                                        inv_permutation_));
+}
+
+
+#define GKO_DECLARE_RCM(ValueType, IndexType) class Rcm<ValueType, IndexType>
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_RCM);
+
+
+}  // namespace reorder
+}  // namespace gko
diff --git a/core/reorder/rcm_kernels.hpp b/core/reorder/rcm_kernels.hpp
new file mode 100644
index 00000000000..7abfeb5fd7c
--- /dev/null
+++ b/core/reorder/rcm_kernels.hpp
@@ -0,0 +1,122 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2019, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_REORDER_RCM_KERNELS_HPP_
+#define GKO_CORE_REORDER_RCM_KERNELS_HPP_
+
+
+#include <ginkgo/core/reorder/rcm.hpp>
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/permutation.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+
+
+namespace gko {
+namespace kernels {
+
+
+#define GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL(ValueType, IndexType)         \
+    void get_permutation(                                                    \
+        std::shared_ptr<const DefaultExecutor> exec, size_type num_vertices, \
+        std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>>           \
+            adjacency_matrix,                                                \
+        std::shared_ptr<Array<IndexType>> node_degrees,                      \
+        std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,     \
+        std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat)
+
+
+#define GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL(ValueType, IndexType) \
+    void get_degree_of_nodes(                                            \
+        std::shared_ptr<const DefaultExecutor> exec,                     \
+        std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>>       \
+            adjacency_matrix,                                            \
+        std::shared_ptr<gko::Array<IndexType>> node_degrees)
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES                                  \
+    template <typename ValueType, typename IndexType>                 \
+    GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL(ValueType, IndexType); \
+    template <typename ValueType, typename IndexType>                 \
+    GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL(ValueType, IndexType)
+
+
+namespace omp {
+namespace rcm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace rcm
+}  // namespace omp
+
+
+namespace cuda {
+namespace rcm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace rcm
+}  // namespace cuda
+
+
+namespace hip {
+namespace rcm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace rcm
+}  // namespace hip
+
+
+namespace reference {
+namespace rcm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace rcm
+}  // namespace reference
+
+
+#undef GKO_DECLARE_ALL_AS_TEMPLATES
+
+
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_CORE_REORDER_RCM_KERNELS_HPP_
diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt
index 473d7005d27..aeed9c81392 100644
--- a/core/test/CMakeLists.txt
+++ b/core/test/CMakeLists.txt
@@ -5,6 +5,7 @@ add_subdirectory(factorization)
 add_subdirectory(log)
 add_subdirectory(matrix)
 add_subdirectory(preconditioner)
+add_subdirectory(reorder)
 add_subdirectory(solver)
 add_subdirectory(stop)
 add_subdirectory(utils)
diff --git a/core/test/reorder/CMakeLists.txt b/core/test/reorder/CMakeLists.txt
new file mode 100644
index 00000000000..47af494c29b
--- /dev/null
+++ b/core/test/reorder/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_test(rcm)
diff --git a/core/test/reorder/rcm.cpp b/core/test/reorder/rcm.cpp
new file mode 100644
index 00000000000..cd36a6f6895
--- /dev/null
+++ b/core/test/reorder/rcm.cpp
@@ -0,0 +1,72 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2019, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/reorder/rcm.hpp>
+
+
+#include <memory>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+namespace {
+
+
+class Rcm : public ::testing::Test {
+protected:
+    using v_type = double;
+    using i_type = int;
+    using Mtx = gko::matrix::Dense<v_type>;
+    using reorder_factory_type = gko::reorder::Rcm<v_type, i_type>;
+
+    Rcm()
+        : exec(gko::ReferenceExecutor::create()),
+          rcm_factory(reorder_factory_type::build().on(exec))
+    {}
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::unique_ptr<reorder_factory_type::Factory> rcm_factory;
+};
+
+
+TEST_F(Rcm, RcmFactoryKnowsItsExecutor)
+{
+    ASSERT_EQ(rcm_factory->get_executor(), exec);
+}
+
+
+}  // namespace
diff --git a/cuda/reorder/rcm_kernels.cu b/cuda/reorder/rcm_kernels.cu
new file mode 100644
index 00000000000..c9bf37cd955
--- /dev/null
+++ b/cuda/reorder/rcm_kernels.cu
@@ -0,0 +1,86 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2019, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/reorder/rcm_kernels.hpp"
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/std_extensions.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/permutation.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+
+
+#include "cuda/base/math.hpp"
+#include "cuda/base/types.hpp"
+#include "cuda/components/prefix_sum.cuh"
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+/**
+ * @brief The parallel ilu factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace rcm {
+
+
+template <typename ValueType, typename IndexType>
+void get_degree_of_nodes(
+    std::shared_ptr<const CudaExecutor> exec,
+    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
+    std::shared_ptr<gko::Array<IndexType>> node_degrees) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void get_permutation(
+    std::shared_ptr<const CudaExecutor> exec, size_type num_vertices,
+    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
+    std::shared_ptr<Array<IndexType>> node_degrees,
+    std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,
+    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
+
+
+}  // namespace rcm
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 20fcb0d79a0..2bb1e6911ee 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -170,6 +170,7 @@ set(GINKGO_HIP_SOURCES
     preconditioner/jacobi_generate_kernel.hip.cpp
     preconditioner/jacobi_kernels.hip.cpp
     preconditioner/jacobi_simple_apply_kernel.hip.cpp
+    reorder/rcm_kernels.hip.cpp
     solver/bicg_kernels.hip.cpp
     solver/bicgstab_kernels.hip.cpp
     solver/cg_kernels.hip.cpp
diff --git a/hip/reorder/rcm_kernels.hip.cpp b/hip/reorder/rcm_kernels.hip.cpp
new file mode 100644
index 00000000000..319c13f8559
--- /dev/null
+++ b/hip/reorder/rcm_kernels.hip.cpp
@@ -0,0 +1,86 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2019, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/reorder/rcm_kernels.hpp"
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/std_extensions.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/permutation.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+
+
+#include "hip/base/math.hip.hpp"
+#include "hip/base/types.hip.hpp"
+#include "hip/components/prefix_sum.hip.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace hip {
+/**
+ * @brief The parallel ilu factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace rcm {
+
+
+template <typename ValueType, typename IndexType>
+void get_degree_of_nodes(
+    std::shared_ptr<const HipExecutor> exec,
+    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
+    std::shared_ptr<gko::Array<IndexType>> node_degrees) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void get_permutation(
+    std::shared_ptr<const HipExecutor> exec, size_type num_vertices,
+    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
+    std::shared_ptr<Array<IndexType>> node_degrees,
+    std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,
+    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
+
+
+}  // namespace rcm
+}  // namespace hip
+}  // namespace kernels
+}  // namespace gko
diff --git a/include/ginkgo/core/reorder/rcm.hpp b/include/ginkgo/core/reorder/rcm.hpp
new file mode 100644
index 00000000000..7b130b7ef51
--- /dev/null
+++ b/include/ginkgo/core/reorder/rcm.hpp
@@ -0,0 +1,184 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2019, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_REORDER_RCM_HPP_
+#define GKO_CORE_REORDER_RCM_HPP_
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/abstract_factory.hpp>
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/polymorphic_object.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/base/utils.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/identity.hpp>
+#include <ginkgo/core/matrix/permutation.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+#include <ginkgo/core/reorder/reordering_base.hpp>
+
+
+namespace gko {
+/**
+ * @brief The Reorder namespace.
+ *
+ * @ingroup reorder
+ */
+namespace reorder {
+
+
+/**
+ * Rcm is the reordering algorithm which uses METIS to compute a
+ * fill reducing re-ordering of the given sparse matrix. The METIS
+ * `Metis_NodeND` which is implements the multilevel nested dissection algorithm
+ * as in the METIS documentation.
+ *
+ * @note  This class is derives from polymorphic object but is not a LinOp as it
+ * does not make sense for this class to implement the apply methods. The
+ * objective of this class is to generate a reordering/permutation vector (in
+ * the form of the Permutation matrix), which can be used to apply to reorder a
+ * matrix as required.
+ *
+ * @tparam ValueType  Type of the values of all matrices used in this class
+ * @tparam IndexType  Type of the indices of all matrices used in this class
+ *
+ * @ingroup reorder
+ */
+template <typename ValueType = default_precision, typename IndexType = int32>
+class Rcm : public EnablePolymorphicObject<Rcm<ValueType, IndexType>,
+                                           ReorderingBase> {
+    friend class EnablePolymorphicObject<Rcm, ReorderingBase>;
+
+public:
+    using SparsityMatrix = matrix::SparsityCsr<ValueType, IndexType>;
+    using PermutationMatrix = matrix::Permutation<IndexType>;
+
+    /**
+     * Gets the adjacency matrix representation of the system operator (input
+     * matrix) of the linear operator.
+     *
+     * @return the system operator (matrix)
+     */
+    std::shared_ptr<const SparsityMatrix> get_adjacency_matrix() const
+    {
+        return adjacency_matrix_;
+    }
+
+    /**
+     * Gets the permutation (permutation matrix, output of the algorithm) of the
+     * linear operator.
+     *
+     * @return the permutation (permutation matrix)
+     */
+    std::shared_ptr<const PermutationMatrix> get_permutation() const
+    {
+        return permutation_;
+    }
+
+    /**
+     * Gets the inverse permutation (permutation matrix, output of the
+     * algorithm) of the linear operator.
+     *
+     * @return the inverse permutation (permutation matrix)
+     */
+    std::shared_ptr<const PermutationMatrix> get_inverse_permutation() const
+    {
+        return inv_permutation_;
+    }
+
+    GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
+    {
+        /**
+         * If this parameter is set then an inverse permutation matrix is also
+         * constructed along with the normal permutation matrix.
+         */
+        bool GKO_FACTORY_PARAMETER(construct_inverse_permutation, false);
+    };
+    GKO_ENABLE_REORDERING_BASE_FACTORY(Rcm, parameters, Factory);
+    GKO_ENABLE_BUILD_METHOD(Factory);
+
+protected:
+    /**
+     * Generates the permutation matrix and if required the inverse permutation
+     * matrix.
+     */
+    void generate() const;
+
+    explicit Rcm(std::shared_ptr<const Executor> exec)
+        : EnablePolymorphicObject<Rcm, ReorderingBase>(std::move(exec))
+    {}
+
+    explicit Rcm(const Factory *factory, const ReorderingBaseArgs &args)
+        : EnablePolymorphicObject<Rcm, ReorderingBase>(factory->get_executor()),
+          parameters_{factory->get_parameters()}
+    {
+        const auto exec = this->get_executor();
+        // The adjacency matrix has to be square.
+        GKO_ASSERT_IS_SQUARE_MATRIX(args.system_matrix);
+        // This is needed because it does not make sense to call the copy and
+        // convert if the existing matrix is empty.
+        if (!args.system_matrix->get_size()) {
+            adjacency_matrix_ = SparsityMatrix::create(exec);
+        } else {
+            auto tmp =
+                copy_and_convert_to<SparsityMatrix>(exec, args.system_matrix);
+            // This function provided within the Sparsity matrix format removes
+            // the diagonal elements and outputs an adjacency matrix.
+            adjacency_matrix_ = tmp->to_adjacency_matrix();
+        }
+        node_degrees_ = std::shared_ptr<Array<IndexType>>(
+            new Array<IndexType>(exec, adjacency_matrix_->get_size()[0]));
+        permutation_ =
+            PermutationMatrix::create(exec, adjacency_matrix_->get_size());
+        inv_permutation_ =
+            PermutationMatrix::create(exec, adjacency_matrix_->get_size());
+
+        this->generate();
+    }
+
+private:
+    std::shared_ptr<SparsityMatrix> adjacency_matrix_{};
+    std::shared_ptr<Array<IndexType>> node_degrees_{};
+    std::shared_ptr<PermutationMatrix> permutation_{};
+    std::shared_ptr<PermutationMatrix> inv_permutation_{};
+};
+
+
+}  // namespace reorder
+}  // namespace gko
+
+
+#endif  // GKO_CORE_REORDER_RCM_HPP_
diff --git a/matrices/CMakeLists.txt b/matrices/CMakeLists.txt
index ffe3602f83a..0281555c01f 100644
--- a/matrices/CMakeLists.txt
+++ b/matrices/CMakeLists.txt
@@ -13,3 +13,4 @@ configure_file("test/isai_u_excess.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai
 configure_file("test/isai_u_excess_rhs.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_u_excess_rhs.mtx")
 configure_file("test/isai_u_inv.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_u_inv.mtx")
 configure_file("test/isai_u_inv_partial.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_u_inv_partial.mtx")
+configure_file("test/1138_bus.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/1138_bus.mtx")
diff --git a/matrices/config.hpp.in b/matrices/config.hpp.in
index 93cd1adb874..ff7b586ac3c 100644
--- a/matrices/config.hpp.in
+++ b/matrices/config.hpp.in
@@ -41,6 +41,7 @@ namespace matrices {
 const char *location_ani1_mtx = "@Ginkgo_BINARY_DIR@/matrices/test/ani1.mtx";
 const char *location_ani4_mtx = "@Ginkgo_BINARY_DIR@/matrices/test/ani4.mtx";
 const char *location_isai_mtxs = "@Ginkgo_BINARY_DIR@/matrices/test/";
+const char *location_1138_bus_mtx = "@Ginkgo_BINARY_DIR@/matrices/test/1138_bus.mtx";
 
 
 }  // namespace matrices
diff --git a/matrices/test/1138_bus.mtx b/matrices/test/1138_bus.mtx
new file mode 100644
index 00000000000..86ec3950cd3
--- /dev/null
+++ b/matrices/test/1138_bus.mtx
@@ -0,0 +1,2599 @@
+%%MatrixMarket matrix coordinate real symmetric
+% Generated 21-Oct-2015
+1138 1138 2596
+1 1 1474.779
+5 1 -9.017133
+563 1 -5.730659
+2 2 9.136654
+10 2 -3.405995
+563 2 -5.730659
+3 3 69.61468
+11 3 -8.810573
+34 3 -31.15265
+35 3 -16.06684
+104 3 -4.86926
+475 3 -8.715357
+4 4 68.60106
+7 4 -34.62025
+27 4 -.4755112
+101 4 -28.66497
+102 4 -.7463244
+103 4 -4.093998
+5 5 13.88805
+9 5 -4.870921
+6 6 116.8288
+7 6 -10.96124
+37 6 -56.81818
+98 6 -10.88139
+103 6 -38.16794
+7 7 52.76726
+37 7 -4.773726
+101 7 -.7772424
+102 7 -1.072973
+103 7 -.5618293
+8 8 30.25788
+26 8 -3.752486
+35 8 -.4975867
+724 8 -26.0078
+9 9 7.174451
+10 9 -.905633
+104 9 -1.397898
+10 10 5.709526
+104 10 -1.397898
+11 11 27.18584
+12 11 -1.238697
+38 11 -7.418397
+566 11 -9.718173
+12 12 1.238697
+13 13 7.142646
+34 13 -1.85192
+104 13 -5.290726
+14 14 12.95337
+413 14 -12.95337
+15 15 1696.756
+16 15 -30.67485
+17 15 -30.4878
+18 15 -30.39514
+19 15 -30.39514
+411 15 -1574.803
+16 16 30.67485
+17 17 30.4878
+18 18 30.39514
+19 19 30.39514
+20 20 5746.122
+21 20 -5714.287
+37 20 -18.24817
+102 20 -13.58696
+21 21 5735.022
+22 21 -5.205622
+23 21 -5.186722
+24 21 -5.181347
+25 21 -5.162622
+22 22 5.205622
+23 23 5.186722
+24 24 5.181347
+25 25 5.162622
+26 26 4.371336
+35 26 -.6188502
+27 27 9.123474
+28 27 -2.373605
+29 27 -2.439024
+30 27 -1.190334
+101 27 -2.644999
+28 28 2.373605
+29 29 2.439024
+30 30 1.190334
+31 31 1.688214
+32 31 -.6254691
+100 31 -1.062744
+32 32 1.688214
+100 32 -1.062744
+33 33 .6581979
+100 33 -.6581979
+34 34 56.31459
+104 34 -10.13479
+553 34 -13.17523
+35 35 10018.3
+104 35 -1.112756
+710 35 -10000
+36 36 10000
+711 36 -10000
+37 37 80.33353
+102 37 -.4934373
+38 38 74.99123
+39 38 -15.67398
+98 38 -46.2963
+412 38 -5.602555
+39 39 16.41742
+99 39 -.7434391
+40 40 245.9335
+41 40 -126.5823
+43 40 -55.24862
+45 40 -64.10257
+41 41 988.9011
+42 41 -28.98551
+44 41 -833.3333
+42 42 28.98551
+43 43 4403.075
+146 43 -4347.826
+44 44 10833.33
+486 44 -10000
+45 45 118.7474
+49 45 -54.64481
+46 46 10000
+48 46 -10000
+47 47 104.9318
+48 47 -104.9318
+48 48 20183.36
+54 48 -78.43137
+506 48 -10000
+49 49 288.754
+50 49 -126.5823
+53 49 -107.5269
+50 50 988.9011
+51 50 -28.98551
+52 50 -833.3333
+51 51 28.98551
+52 52 874.3169
+125 52 -40.98361
+53 53 1586.098
+54 53 -1428.572
+55 53 -50
+54 54 1957.633
+425 54 -119.0476
+445 54 -184.5238
+447 54 -147.0588
+55 55 50
+56 56 29.32551
+63 56 -29.32551
+57 57 112.3596
+66 57 -112.3596
+58 58 15.64945
+70 58 -15.64945
+59 59 24.3309
+67 59 -24.3309
+60 60 104.1667
+68 60 -104.1667
+61 61 125
+130 61 -125
+62 62 50.76142
+129 62 -50.76142
+63 63 10059.54
+64 63 -30.21148
+65 63 -10000
+64 64 159.7102
+71 64 -40.98361
+131 64 -16.05136
+226 64 -72.46377
+65 65 10015.92
+1095 65 -15.92357
+66 66 442.0023
+67 66 -97.08738
+68 66 -24.27184
+74 66 -32.65326
+183 66 -147.0588
+226 66 -28.57143
+67 67 167.4071
+68 67 -30.67485
+69 67 -15.31394
+68 68 289.2932
+74 68 -47.5921
+90 68 -32.33649
+201 68 -50.25126
+69 69 190.7525
+70 69 -175.4386
+70 70 225.5077
+182 70 -4.955401
+196 70 -5.170631
+1063 70 -4.623209
+1065 70 -16.47446
+1074 70 -3.195909
+71 71 96.53917
+72 71 -55.55556
+72 72 84.92064
+73 72 -5.555555
+121 72 -23.80952
+73 73 5.555555
+74 74 384.9024
+75 74 -40.32258
+76 74 -25.44529
+78 74 -34.3871
+80 74 -15.82279
+119 74 -188.6792
+75 75 693.1041
+204 75 -645.1613
+810 75 -4.608295
+918 75 -3.012048
+76 76 147.3945
+77 76 -30.03003
+78 76 -36.36364
+127 76 -55.55556
+77 77 5030.03
+1050 77 -5000
+78 78 262.67
+79 78 -90.90909
+452 78 -101.0101
+79 79 564.4214
+416 79 -48.07692
+417 79 -48.07692
+454 79 -377.3585
+80 80 657.1666
+81 80 -67.1141
+447 80 -476.1905
+452 80 -98.03922
+81 81 420.0095
+448 81 -158.7302
+450 81 -175.4386
+464 81 -18.72659
+82 82 4.975124
+94 82 -4.975124
+83 83 17.24138
+94 83 -17.24138
+84 84 17.57469
+95 84 -17.57469
+85 85 439.0805
+87 85 -172.4138
+126 85 -266.6667
+86 86 6807.496
+87 86 -5000
+241 86 -46.51163
+251 86 -71.94245
+253 86 -74.62687
+267 86 -94.78674
+269 86 -277.7778
+283 86 -138.8889
+291 86 -49.01961
+293 86 -57.14286
+294 86 -57.14286
+302 86 -689.6552
+315 86 -250
+87 87 5319.472
+88 87 -147.0588
+88 88 147.0588
+89 89 65.3412
+105 89 -38.31417
+115 89 -27.02703
+90 90 64.83059
+91 90 -32.4941
+91 91 645.056
+92 91 -163.6515
+108 91 -67.56757
+115 91 -24.19992
+116 91 -357.1429
+92 92 258.8896
+93 92 -95.2381
+93 93 612.1849
+94 93 -10.90512
+252 93 -131.5789
+264 93 -12.67427
+271 93 -66.0066
+288 93 -65.35947
+292 93 -31.84714
+293 93 -38.31417
+294 93 -38.31417
+322 93 -32.15434
+323 93 -34.8432
+324 93 -24.03846
+325 93 -14.40922
+326 93 -16.50165
+94 94 464.0852
+96 94 -4.800768
+264 94 -239.4475
+318 94 -110.6696
+325 94 -76.04562
+95 95 76.57204
+97 95 -8.285004
+262 95 -9.67118
+268 95 -7.142857
+300 95 -33.8983
+96 96 10022.9
+145 96 -7.24113
+704 96 -10.85776
+705 96 -10000
+97 97 12.12083
+268 97 -3.835827
+98 98 57.17769
+99 99 3.134068
+413 99 -2.390629
+100 100 40.73998
+725 100 -35.58719
+732 100 -2.369107
+101 101 34.2926
+102 101 -.754091
+103 101 -1.451295
+102 102 47.04131
+103 102 -30.38753
+103 103 10074.66
+478 103 -10000
+104 104 24.20333
+105 105 420.7618
+106 105 -12.07729
+109 105 -370.3704
+106 106 12.07729
+107 107 32.15434
+108 107 -32.15434
+108 108 1008.813
+110 108 -909.0909
+109 109 10370.37
+1029 109 -10000
+110 110 10974.79
+111 110 -41.84558
+328 110 -10000
+367 110 -10.77006
+370 110 -13.08045
+111 111 10056.95
+367 111 -7.27802
+368 111 -7.824726
+701 111 -10000
+112 112 301.7592
+113 112 -7.420491
+370 112 -8.624407
+686 112 -285.7143
+113 113 10040.49
+114 113 -7.624856
+115 113 -25.44529
+690 113 -10000
+114 114 10007.63
+1009 114 -10000
+115 115 76.67225
+116 116 436.123
+117 116 -10.9529
+126 116 -68.02721
+117 117 10.9529
+118 118 5.521811
+119 118 -5.521811
+119 119 228.3307
+123 119 -34.12969
+120 120 126.5823
+121 120 -126.5823
+121 121 272.2888
+122 121 -70.87661
+504 121 -51.02041
+122 122 152.6848
+123 122 -40.48583
+124 122 -41.32232
+123 123 111.929
+125 123 -37.31343
+124 124 97.81158
+125 124 -32.67974
+127 124 -23.80952
+125 125 363.2857
+126 125 -81.38689
+129 125 -70.92198
+546 125 -100
+126 126 478.1926
+129 126 -62.1118
+127 127 84.92064
+128 127 -5.555555
+128 128 5.555555
+129 129 183.7952
+130 130 356.9032
+133 130 -136.9863
+173 130 -51.81347
+213 130 -43.10345
+131 131 896.646
+132 131 -75.18797
+133 131 -769.2308
+136 131 -36.17595
+132 132 101.0104
+144 132 -4.409171
+742 132 -21.41328
+133 133 918.7799
+134 133 -12.56281
+134 134 12.56281
+135 135 10030.19
+136 135 -25.44529
+139 135 -4.748338
+740 135 -10000
+136 136 99.82595
+141 136 -38.20471
+137 137 108.031
+139 137 -72.46377
+141 137 -27.39726
+761 137 -8.169934
+138 138 217.898
+141 138 -28.98551
+877 138 -59.88025
+882 138 -129.0323
+139 139 90.06557
+140 139 -12.85347
+140 140 45.52996
+829 140 -18.62891
+830 140 -14.04757
+141 141 94.58748
+142 142 36.70324
+258 142 -21.55172
+366 142 -15.15152
+143 143 20014.59
+144 143 -12.5
+743 143 -2.090738
+745 143 -10000
+826 143 -10000
+144 144 10016.91
+827 144 -10000
+145 145 10022.01
+318 145 -14.77105
+703 145 -10000
+146 146 7289.003
+147 146 -2941.177
+147 147 12941.18
+148 147 -10000
+148 148 10000
+149 149 5.205622
+166 149 -5.205622
+150 150 73.52941
+173 150 -73.52941
+151 151 13.29787
+175 151 -13.29787
+152 152 10.40583
+180 152 -10.40583
+153 153 34.48276
+183 153 -34.48276
+154 154 13.71742
+207 154 -13.71742
+155 155 28.98551
+198 155 -28.98551
+156 156 144.9275
+213 156 -144.9275
+157 157 38.46154
+219 157 -38.46154
+158 158 28.0112
+226 158 -28.0112
+159 159 9.64196
+182 159 -4.730369
+205 159 -4.911591
+160 160 21.37365
+172 160 -10.01001
+233 160 -11.36364
+161 161 10.90321
+222 161 -4.730369
+743 161 -6.17284
+162 162 87.51172
+163 162 -46.08295
+169 162 -11.53403
+209 162 -7.451564
+223 162 -12.61034
+232 162 -9.832842
+163 163 145.9383
+186 163 -46.51163
+201 163 -13.6612
+227 163 -39.68254
+164 164 5.205622
+166 164 -5.205622
+165 165 117.6942
+166 165 -60.60606
+167 165 -17.33102
+175 165 -11.18568
+194 165 -28.57143
+166 166 341.3369
+192 166 -80
+201 166 -14.88095
+218 166 -175.4386
+167 167 34.66204
+175 167 -17.33102
+168 168 69.63064
+180 168 -9.389671
+209 168 -60.24096
+169 169 25.6384
+187 169 -14.10437
+170 170 30.28655
+188 170 -22.77904
+194 170 -7.507507
+171 171 52.2535
+178 171 -39.21569
+217 171 -13.03781
+172 172 66.85269
+173 172 -42.55319
+222 172 -4.873294
+755 172 -9.416196
+173 173 206.2103
+184 173 -38.31417
+174 174 50.63389
+211 174 -12.7551
+216 174 -37.87879
+175 175 53.63491
+179 175 -11.82033
+176 176 860.8028
+177 176 -172.4138
+178 176 -20.98079
+207 176 -667.4082
+177 177 276.5804
+207 177 -104.1667
+178 178 88.17661
+214 178 -15.06024
+217 178 -12.9199
+179 179 23.64066
+203 179 -11.82033
+180 180 28.49115
+209 180 -8.695652
+181 181 23.14815
+183 181 -23.14815
+182 182 61.50294
+183 182 -29.94012
+205 182 -16.72241
+221 182 -5.154639
+183 183 398.0187
+184 183 -91.74312
+201 183 -35.14939
+226 183 -36.49635
+184 184 192.1349
+193 184 -41.40787
+213 184 -20.6697
+185 185 8.064516
+187 185 -8.064516
+186 186 126.2011
+187 186 -41.66667
+218 186 -38.02281
+187 187 107.524
+188 187 -29.67359
+206 187 -8.695652
+209 187 -5.319149
+188 188 52.45263
+189 189 41.49792
+211 189 -11.46789
+216 189 -30.03003
+190 190 33.0033
+199 190 -33.0033
+191 191 1149.633
+195 191 -10.71811
+197 191 -14.74926
+198 191 -1111.111
+211 191 -13.05483
+192 192 824.6421
+199 192 -666.6667
+218 192 -63.29114
+220 192 -14.68429
+193 193 247.007
+198 193 -60.60606
+199 193 -106.383
+208 193 -38.61004
+194 194 78.9974
+216 194 -42.91846
+195 195 36.89613
+211 195 -26.17801
+196 196 5.170631
+197 197 36.48839
+212 197 -21.73913
+198 198 1200.703
+199 199 806.053
+200 200 60.0913
+201 200 -43.87481
+202 200 -16.21649
+201 201 172.6985
+218 201 -14.88095
+202 202 20.9357
+203 202 -4.719207
+203 203 16.53954
+204 204 645.1613
+205 205 28.7262
+221 205 -7.092198
+206 206 18.89453
+216 206 -10.19888
+207 207 881.1484
+208 207 -84.38818
+224 207 -11.46789
+208 208 122.9982
+209 209 81.70734
+210 210 1796.925
+215 210 -11.21076
+217 210 -1666.667
+218 210 -119.0476
+211 211 84.37634
+212 211 -20.9205
+212 212 42.65963
+213 213 282.2301
+504 213 -73.52941
+214 214 65.82166
+217 214 -50.76142
+215 215 11.21076
+216 216 148.1264
+218 216 -27.10027
+217 217 1743.386
+218 218 437.7814
+219 219 150.9842
+220 219 -86.20689
+230 219 -26.31579
+220 220 100.8912
+221 221 18.99606
+222 221 -3.15856
+225 221 -3.590664
+222 222 91.18269
+225 222 -26.24672
+233 222 -49.50495
+743 222 -2.668802
+223 223 54.62715
+232 223 -42.01681
+224 224 18.73533
+1026 224 -7.267442
+225 225 29.83738
+226 226 165.5428
+227 227 85.76549
+228 227 -46.08295
+228 228 97.42727
+229 228 -27.62431
+231 228 -12.7551
+232 228 -10.96491
+229 229 27.62431
+230 230 26.31579
+231 231 31.20529
+232 231 -18.45018
+232 232 81.26475
+233 233 60.86859
+234 234 29.17592
+236 234 -14.55604
+307 234 -14.61988
+235 235 2117.081
+236 235 -67.56757
+243 235 -14.59854
+270 235 -2000
+272 235 -9.208103
+298 235 -12.85347
+299 235 -12.85347
+236 236 90.2339
+286 236 -8.1103
+237 237 123.4571
+261 237 -2.818489
+287 237 -18.97533
+309 237 -69.68641
+366 237 -4.004806
+702 237 -27.97203
+238 238 188.1504
+239 238 -41.25455
+270 238 -146.8959
+239 239 59.24802
+270 239 -15.33742
+281 239 -2.656042
+240 240 7.917656
+259 240 -7.917656
+241 241 2888.018
+242 241 -185.1852
+246 241 -89.28572
+253 241 -123.4568
+257 241 -92.59259
+266 241 -123.4568
+275 241 -13.44086
+279 241 -31.74603
+289 241 -30.03003
+291 241 -833.3333
+292 241 -833.3333
+293 241 -138.8889
+294 241 -138.8889
+299 241 -13.28021
+310 241 -16.80672
+321 241 -88.49557
+327 241 -89.28572
+242 242 314.7918
+244 242 -57.14286
+245 242 -72.46377
+243 243 14.59854
+244 244 118.4926
+272 244 -61.34969
+245 245 95.93795
+298 245 -23.47418
+246 246 113.1521
+310 246 -23.86635
+247 247 582.7703
+248 247 -270.2703
+255 247 -312.5
+248 248 502.8284
+255 248 -232.5581
+249 249 1666.667
+282 249 -1666.667
+250 250 1123.714
+257 250 -92.59259
+263 250 -78.74016
+289 250 -119.0476
+317 250 -833.3333
+251 251 210.8313
+267 251 -138.8889
+252 252 288.2195
+310 252 -21.50537
+313 252 -135.1351
+253 253 198.0837
+254 254 383.7719
+255 254 -208.3333
+285 254 -175.4386
+255 255 910.2564
+258 255 -31.44654
+260 255 -71.94245
+263 255 -53.47594
+256 256 10133
+290 256 -52.35602
+317 256 -10000
+321 256 -80.64516
+257 257 185.1852
+258 258 261.8486
+261 258 -128.2051
+327 258 -80.64516
+259 259 20.85427
+278 259 -12.93661
+260 260 190.9901
+289 260 -119.0476
+261 261 131.0236
+262 262 183.095
+268 262 -26.17801
+276 262 -62.5
+314 262 -84.74576
+263 263 132.2161
+264 264 383.7008
+311 264 -131.5789
+265 265 1017.575
+276 265 -999.9999
+300 265 -17.57469
+266 266 246.9136
+307 266 -123.4568
+267 267 566.0127
+282 267 -144.9275
+284 267 -77.51938
+302 267 -109.8901
+268 268 158.6071
+270 268 -53.88283
+301 268 -67.56757
+269 269 486.1111
+284 269 -208.3333
+270 270 2317.201
+275 270 -12.77139
+279 270 -8.257638
+280 270 -29.32551
+318 270 -34.36426
+320 270 -16.36661
+271 271 66.0066
+272 272 70.5578
+273 273 1295.094
+274 273 -303.0303
+303 273 -714.2858
+315 273 -277.7778
+274 274 606.0606
+312 274 -303.0303
+275 275 26.21225
+276 276 1062.5
+277 277 5086.957
+285 277 -86.95652
+289 277 -5000
+278 278 27.55649
+309 278 -14.61988
+279 279 40.00367
+280 280 29.32551
+281 281 2.656042
+282 282 1811.594
+283 283 214.6465
+292 283 -75.75758
+284 284 285.8527
+285 285 262.3951
+286 286 8.1103
+287 287 18.97533
+288 288 127.4713
+291 288 -62.1118
+289 289 5268.125
+290 290 52.35602
+291 291 944.4647
+292 292 940.938
+293 293 234.3459
+294 294 234.3459
+295 295 38.31417
+298 295 -38.31417
+296 296 101.0101
+297 296 -101.0101
+297 297 118.4317
+299 297 -17.4216
+298 298 74.64183
+299 299 43.55529
+300 300 122.395
+696 300 -70.92198
+301 301 153.7745
+314 301 -86.20689
+302 302 799.5453
+303 303 1428.572
+312 303 -714.2858
+304 304 1380.582
+305 304 -263.1579
+315 304 -208.3333
+329 304 -909.0909
+305 305 459.2363
+315 305 -196.0784
+306 306 37.87879
+322 306 -37.87879
+307 307 138.0767
+308 308 81.30082
+323 308 -81.30082
+309 309 84.30629
+310 310 62.17845
+311 311 200.5445
+326 311 -68.96552
+312 312 1295.094
+315 312 -277.7778
+313 313 135.1351
+314 314 170.9527
+315 315 1688.798
+316 315 -322.5806
+319 315 -125
+323 315 -31.25
+316 316 360.1746
+322 316 -37.59399
+317 317 10833.33
+318 318 169.825
+704 318 -10.02004
+319 319 1236.111
+329 319 -1111.111
+320 320 16.36661
+321 321 169.1407
+322 322 107.6271
+323 323 147.394
+324 324 51.89362
+325 324 -27.85515
+325 325 118.31
+326 326 85.46717
+327 327 169.9309
+328 328 10000
+329 329 2020.202
+330 330 2.409639
+335 330 -2.409639
+331 331 .8810573
+336 331 -.8810573
+332 332 1.666667
+337 332 -1.666667
+333 333 2.608922
+350 333 -2.608922
+334 334 1.446132
+350 334 -1.446132
+335 335 30.26479
+337 335 -27.85515
+336 336 218.2724
+337 336 -217.3913
+337 337 260.9173
+338 337 -3.840246
+339 337 -3.955696
+340 337 -6.208189
+338 338 19.08415
+340 338 -15.2439
+339 339 19.1996
+340 339 -15.2439
+340 340 59.70203
+341 340 -8.1103
+342 340 -14.89573
+341 341 20.88169
+342 341 -12.77139
+342 342 66.7209
+343 342 -5.675369
+344 342 -10.71237
+345 342 -4.215851
+370 342 -18.45018
+343 343 15.14507
+346 343 -9.469697
+344 344 28.10368
+346 344 -17.3913
+345 345 26.63738
+346 345 -22.42153
+346 346 102.5212
+347 346 -14.08451
+349 346 -20.70393
+367 346 -18.45018
+347 347 54.57034
+348 347 -40.48583
+348 348 79.09587
+349 348 -38.61004
+349 349 83.58582
+350 349 -24.27184
+350 350 28.3269
+351 351 7.374631
+366 351 -7.374631
+352 352 14.91256
+353 352 -5.289326
+408 352 -2.934273
+409 352 -6.688963
+353 353 17.04893
+363 353 -3.816794
+382 353 -7.942812
+354 354 28.97867
+382 354 -6.199628
+383 354 -22.77904
+355 355 2.164971
+375 355 -2.164971
+356 356 13.59556
+383 356 -5.344736
+398 356 -8.250825
+357 357 1.307306
+394 357 -.5700604
+395 357 -.7372456
+358 358 46.94836
+479 358 -46.94836
+359 359 18.55447
+474 359 -18.55447
+360 360 8.05153
+715 360 -8.05153
+361 361 509.4518
+479 361 -9.451796
+480 361 -500
+362 362 22.4661
+403 362 -11.03753
+714 362 -11.42857
+363 363 96.66204
+364 363 -9.149131
+365 363 -9.149131
+373 363 -9.149131
+374 363 -9.149131
+388 363 -34.96503
+395 363 -4.926108
+407 363 -10.75847
+474 363 -5.599104
+364 364 9.149131
+365 365 9.149131
+366 366 41.86838
+699 366 -15.33742
+367 367 307.3316
+368 367 -104.1667
+372 367 -166.6667
+368 368 111.9914
+369 369 72.9927
+371 369 -72.9927
+370 370 40.15504
+371 371 239.6594
+372 371 -166.6667
+372 372 333.3333
+373 373 9.149131
+374 374 9.149131
+375 375 122.031
+376 375 -2.019386
+377 375 -2.019386
+378 375 -2.019386
+379 375 -1.959632
+380 375 -1.959632
+381 375 -1.959632
+389 375 -105.8201
+404 375 -2.108815
+376 376 2.019386
+377 377 2.019386
+378 378 2.019386
+379 379 1.959632
+380 380 1.959632
+381 381 1.959632
+382 382 14.14244
+383 383 84.26632
+384 383 -8.143323
+385 383 -8.032128
+386 383 -16.50165
+387 383 -16.50165
+392 383 -6.963788
+384 384 8.143323
+385 385 8.032128
+386 386 16.50165
+387 387 16.50165
+388 388 94.03941
+389 388 -22.98851
+390 388 -7.077141
+391 388 -7.077141
+395 388 -4.739336
+401 388 -9.416196
+405 388 -7.77605
+389 389 128.8086
+390 390 7.077141
+391 391 7.077141
+392 392 10.41564
+395 392 -3.451847
+393 393 32.25806
+474 393 -16.12903
+479 393 -16.12903
+394 394 20.16254
+395 394 -19.59248
+395 395 785.9075
+398 395 -76.33588
+399 395 -77.51938
+405 395 -10.71811
+406 395 -473.7875
+407 395 -89.28572
+474 395 -24.8139
+396 396 97.03087
+406 396 -86.43042
+716 396 -5.336179
+717 396 -5.264266
+397 397 690.4763
+716 397 -357.1429
+717 397 -333.3333
+398 398 84.5867
+399 399 77.51938
+400 400 10000
+473 400 -10000
+401 401 29.10979
+402 401 -8.038586
+714 401 -11.65501
+402 402 8.038586
+403 403 11.03753
+404 404 5.644883
+409 404 -3.536068
+405 405 18.49416
+406 406 10591.18
+479 406 -23.97024
+483 406 -10000
+716 406 -3.519887
+717 406 -3.472463
+407 407 100.0442
+408 408 2.934273
+409 409 10.22503
+410 410 10155.49
+411 410 -10000
+412 410 -80.19246
+731 410 -75.30121
+411 411 11741.83
+476 411 -48.89976
+477 411 -68.37394
+486 411 -49.75124
+412 412 111.7064
+413 412 -9.425071
+478 412 -3.253196
+709 412 -.6465378
+724 412 -12.58653
+413 413 24.76907
+414 414 319.0159
+431 414 -227.2727
+433 414 -91.74312
+415 415 319.0159
+432 415 -227.2727
+433 415 -91.74312
+416 416 362.0423
+433 416 -91.74312
+462 416 -222.2222
+417 417 362.0423
+433 417 -91.74312
+462 417 -222.2222
+418 418 39.25825
+419 418 -9.389671
+545 418 -29.86858
+419 419 9.389671
+420 420 1716.987
+421 420 -38.75969
+422 420 -11.56069
+470 420 -1666.667
+421 421 38.75969
+422 422 11.56069
+423 423 606.9609
+425 423 -526.3158
+516 423 -80.64516
+424 424 293.4111
+466 424 -212.766
+516 424 -80.64516
+425 425 905.9952
+426 425 -73.52941
+427 425 -35.58719
+466 425 -151.5152
+426 426 536.205
+437 426 -454.5455
+519 426 -8.130081
+427 427 35.58719
+428 428 305.9395
+429 428 -25
+430 428 -37.03704
+431 428 -121.9512
+432 428 -121.9512
+429 429 25
+430 430 37.03704
+431 431 349.224
+432 432 349.224
+433 433 602.0284
+434 433 -32.05128
+510 433 -181.8182
+564 433 -21.18644
+434 434 32.05128
+435 435 83.68201
+510 435 -83.68201
+436 436 21.45923
+534 436 -21.45923
+437 437 548.2252
+438 437 -45.6621
+439 437 -6.222775
+441 437 -12.12121
+458 437 -29.67359
+438 438 45.6621
+439 439 22.0206
+440 439 -11.16819
+460 439 -4.62963
+440 440 11.16819
+441 441 45.41016
+442 441 -33.28895
+442 442 33.28895
+443 443 1625.394
+444 443 -73.52941
+470 443 -1111.111
+506 443 -155.0388
+516 443 -285.7143
+444 444 1323.529
+458 444 -1250
+445 445 298.3669
+446 445 -46.72897
+447 445 -67.1141
+446 446 46.72897
+447 447 1251.698
+449 447 -73.52941
+537 447 -487.8049
+448 448 178.454
+464 448 -19.72387
+449 449 2073.529
+450 449 -2000
+450 450 2196.806
+451 450 -21.36752
+451 451 21.36752
+452 452 237.3635
+453 452 -38.31417
+453 453 38.31417
+454 454 413.3297
+455 454 -35.97123
+455 455 35.97123
+456 456 105.5727
+457 456 -24.27184
+564 456 -81.30082
+457 457 24.27184
+458 458 1323.682
+459 458 -18.69159
+460 458 -25.31645
+459 459 18.69159
+460 460 51.68521
+461 460 -21.73913
+461 461 21.73913
+462 462 492.9882
+463 462 -48.54369
+463 463 48.54369
+464 464 86.29654
+465 464 -26.88172
+545 464 -20.96436
+465 465 92.6712
+968 465 -65.78947
+466 466 414.0324
+467 466 -49.75124
+467 467 49.75124
+468 468 10000
+544 468 -10000
+469 469 10000
+543 469 -10000
+470 470 2777.778
+471 471 71.30248
+472 471 -24.67105
+478 471 -11.01079
+481 471 -2.51756
+492 471 -2.24341
+707 471 -5.08647
+708 471 -25.7732
+472 472 24.67105
+473 473 10004.09
+474 473 -4.095004
+474 474 69.19151
+475 475 31.94728
+710 475 -12.28501
+724 475 -10.94691
+476 476 81.26222
+482 476 -32.36246
+477 477 228.7589
+478 477 -160.3849
+478 478 10195.93
+708 478 -20.67495
+709 478 -.6028454
+479 479 118.9729
+480 479 -9.02527
+714 479 -8.410429
+726 479 -5.037783
+480 480 509.0252
+481 481 26.89223
+484 481 -4.597701
+485 481 -4.425562
+491 481 -2.892765
+492 481 -3.441512
+733 481 -9.017133
+482 482 111.1026
+483 482 -78.74016
+483 483 10078.74
+484 484 22.31696
+709 484 -10.70664
+733 484 -7.012623
+485 485 12.73121
+491 485 -8.305648
+486 486 10101.3
+731 486 -51.54639
+487 487 5.743825
+491 487 -5.743825
+488 488 5.743825
+491 488 -5.743825
+489 489 6.968641
+492 489 -6.968641
+490 490 7.077141
+492 490 -7.077141
+491 491 41.56859
+492 491 -16.91332
+734 491 -1.969202
+492 492 10036.64
+707 492 -10000
+493 493 11.23596
+707 493 -11.23596
+494 494 121.9512
+505 494 -121.9512
+495 495 46.2963
+513 495 -46.2963
+496 496 303.0303
+521 496 -303.0303
+497 497 104.1667
+522 497 -104.1667
+498 498 20.08032
+523 498 -20.08032
+499 499 87.33171
+500 499 -39.37008
+819 499 -23.98081
+914 499 -23.98081
+500 500 39.37008
+501 501 227.9827
+502 501 -33.19632
+505 501 -18.58736
+507 501 -58.13953
+521 501 -32.67974
+531 501 -42.55319
+536 501 -42.82655
+502 502 120.8391
+607 502 -33.33334
+609 502 -11.02536
+613 502 -27.3224
+937 502 -15.96169
+503 503 121.5155
+507 503 -42.91846
+513 503 -55.55556
+553 503 -23.04147
+504 504 195.9784
+505 504 -71.42857
+505 505 292.5258
+521 505 -41.49377
+526 505 -17.60563
+578 505 -21.45923
+506 506 10155.04
+507 507 295.7078
+508 507 -12.03369
+509 507 -53.0504
+515 507 -20.4918
+521 507 -39.0625
+522 507 -40.16064
+547 507 -29.85075
+508 508 265.0443
+619 508 -175.4386
+639 508 -56.49718
+781 508 -21.07482
+509 509 912.198
+511 509 -12.0919
+515 509 -82.64463
+520 509 -238.0952
+534 509 -526.3158
+510 510 2646.453
+520 510 -2380.952
+511 511 29.78859
+914 511 -6.779661
+926 511 -10.91703
+512 512 99.0099
+520 512 -99.0099
+513 513 185.3068
+514 513 -9.380863
+540 513 -74.07407
+514 514 28.32451
+790 514 -11.28668
+795 514 -7.656968
+515 515 688.1429
+516 515 -416.6667
+517 515 -10.34126
+518 515 -18.18182
+536 515 -80.64516
+547 515 -59.1716
+516 516 863.6713
+517 517 27.53012
+993 517 -12.95337
+995 517 -4.235493
+518 518 18.18182
+519 519 93.04359
+957 519 -16.42036
+967 519 -68.49315
+520 520 2718.058
+521 521 520.433
+522 521 -104.1667
+522 522 260.4988
+523 522 -12.0048
+523 523 247.5269
+582 523 -196.0784
+770 523 -3.675795
+779 523 -11.60093
+802 523 -4.086637
+524 524 220.5431
+533 524 -158.7302
+985 524 -43.85965
+1001 524 -17.95332
+525 525 78.74016
+568 525 -78.74016
+526 526 71.7446
+527 526 -21.45923
+531 526 -32.67974
+527 527 149.7379
+921 527 -4.553734
+933 527 -5.621135
+1068 527 -35.27337
+1073 527 -30.67485
+1074 527 -25.83979
+1124 527 -26.31579
+528 528 336.2918
+529 528 -196.0784
+809 528 -56.17978
+819 528 -84.03362
+529 529 196.0784
+530 530 261.7177
+578 530 -9.708738
+583 530 -9.259259
+780 530 -5.390836
+797 530 -4.800768
+803 530 -232.5581
+531 531 204.1382
+532 531 -53.71729
+534 531 -75.18797
+532 532 527.8732
+554 532 -35.97123
+647 532 -113.6364
+652 532 -91.3242
+663 532 -45.87156
+668 532 -63.29114
+672 532 -12.97017
+673 532 -32.78688
+677 532 -28.24859
+908 532 -13.75516
+912 532 -18.48429
+918 532 -6.426735
+1047 532 -11.38952
+533 533 158.7302
+534 534 740.9194
+535 534 -21.45923
+536 534 -56.49718
+542 534 -40
+535 535 37.48487
+885 535 -16.02564
+536 536 698.459
+537 536 -416.6667
+538 536 -21.14165
+539 536 -18.18182
+542 536 -62.5
+537 537 904.4716
+538 538 105.9105
+809 538 -33.78378
+914 538 -7.002801
+948 538 -11.17318
+953 538 -10.48768
+964 538 -22.32143
+539 539 18.18182
+540 540 95.5333
+800 540 -21.45923
+541 541 13.9958
+543 541 -13.9958
+542 542 256.9678
+543 542 -42.91846
+545 542 -21.45923
+546 542 -90.0901
+543 543 10075.1
+544 543 -18.18182
+544 544 10018.18
+545 545 72.29217
+546 546 190.0901
+547 547 89.02235
+548 548 7.942812
+551 548 -7.942812
+549 549 8.532423
+563 549 -8.532423
+550 550 35.33569
+566 550 -35.33569
+551 551 65.62718
+552 551 -17.51313
+555 551 -15.47988
+566 551 -24.69136
+552 552 35.10244
+558 552 -3.012048
+560 552 -14.57726
+553 553 67.98992
+561 553 -26.73797
+710 553 -5.035247
+554 554 351.6514
+574 554 -93.45794
+909 554 -222.2222
+555 555 58.73003
+556 555 -16.26016
+557 555 -8.591065
+564 555 -10.9529
+566 555 -7.446017
+556 556 150.6284
+559 556 -64.93507
+579 556 -7.886436
+621 556 -45.04505
+903 556 -16.50165
+557 557 25.14736
+558 557 -16.55629
+558 558 103.3103
+979 558 -4.599816
+980 558 -12.40695
+986 558 -12.0919
+989 558 -14.4405
+991 558 -13.86001
+998 558 -8.061266
+1000 558 -18.28154
+559 559 73.91173
+579 559 -8.976661
+560 560 20.04772
+980 560 -5.470459
+561 561 200.6328
+562 561 -135.1351
+566 561 -38.75969
+562 562 143.5954
+563 562 -8.460238
+563 563 31.84037
+567 563 -3.386387
+564 564 113.4402
+565 565 5.017561
+567 565 -5.017561
+566 566 120.1828
+567 566 -4.231908
+567 567 12.63586
+568 568 164.0549
+773 568 -6.574622
+795 568 -78.74016
+569 569 20.08032
+785 569 -20.08032
+570 570 14.70588
+579 570 -14.70588
+571 571 11.42857
+776 571 -11.42857
+572 572 14.85884
+783 572 -14.85884
+573 573 18.51852
+796 573 -18.51852
+574 574 93.45794
+575 575 400
+581 575 -400
+576 576 23.74311
+582 576 -13.31558
+784 576 -10.42753
+577 577 897.2149
+786 577 -769.2308
+802 577 -8.93655
+1090 577 -119.0476
+578 578 45.98278
+780 578 -14.81481
+579 579 84.46193
+580 579 -2.915452
+782 579 -20.4918
+783 579 -7.85546
+895 579 -18.21494
+927 579 -3.4153
+580 580 16.74671
+587 580 -13.83126
+581 581 427.649
+776 581 -9.65717
+796 581 -4.690432
+797 581 -7.42942
+799 581 -5.87199
+582 582 209.394
+583 583 29.98347
+780 583 -13.03781
+803 583 -7.686395
+584 584 52.25406
+585 584 -45.6621
+1068 584 -6.591958
+585 585 121.8822
+607 585 -38.91051
+613 585 -34.0136
+659 585 -3.295979
+586 586 2.538071
+597 586 -2.538071
+587 587 66.82627
+598 587 -14.08451
+614 587 -38.91051
+588 588 10.00413
+595 588 -5.065856
+597 588 -4.938272
+589 589 6.075334
+591 589 -6.075334
+590 590 11.77992
+595 590 -4.405286
+616 590 -7.374631
+591 591 15.86007
+601 591 -9.784736
+592 592 7.564296
+611 592 -7.564296
+593 593 10.53741
+604 593 -10.53741
+594 594 63.69427
+600 594 -63.69427
+595 595 9.471143
+596 596 22.07258
+597 596 -4.837929
+603 596 -6.493506
+607 596 -10.74114
+597 597 12.31427
+598 598 14.08451
+599 599 6.038647
+1105 599 -6.038647
+600 600 346.1801
+601 600 -4.708098
+782 600 -277.7778
+601 601 23.91828
+602 601 -5.24659
+633 601 -4.178855
+602 602 10.19709
+640 602 -4.950495
+603 603 49.17807
+604 603 -4.837929
+605 603 -10.96491
+1092 603 -26.88172
+604 604 30.79998
+612 604 -7.843137
+615 604 -7.581501
+605 605 31.29012
+607 605 -20.3252
+606 606 3.743916
+610 606 -3.743916
+607 607 103.3102
+608 608 44.10909
+610 608 -41.15226
+1101 608 -2.95683
+609 609 20.31041
+1068 609 -9.285051
+610 610 44.89618
+611 611 19.35675
+615 611 -11.79245
+612 612 7.843137
+613 613 61.33601
+614 614 38.91051
+615 615 19.37395
+616 616 7.374631
+617 617 51.02041
+1123 617 -51.02041
+618 618 27.57334
+629 618 -10.09082
+639 618 -17.48252
+619 619 180.4386
+620 619 -5
+620 620 7.089864
+632 620 -2.089864
+621 621 137.2607
+624 621 -87.7193
+1061 621 -4.496403
+622 622 12.46672
+625 622 -10.67236
+641 622 -1.794366
+623 623 18.4185
+639 623 -10.37344
+642 623 -8.045053
+624 624 110.7608
+638 624 -23.04147
+625 625 17.12814
+635 625 -6.455778
+626 626 8.504332
+634 626 -5.627462
+635 626 -2.87687
+627 627 15.82279
+639 627 -15.82279
+628 628 43.10402
+631 628 -33.22259
+642 628 -9.881423
+629 629 5125.033
+637 629 -5000
+643 629 -114.9425
+630 630 27.86917
+631 630 -5.931198
+635 630 -4.347826
+796 630 -17.59015
+631 631 39.15379
+632 632 10.68662
+634 632 -6.830601
+641 632 -1.76616
+633 633 4.178855
+634 634 12.45806
+635 635 18.68799
+636 635 -5.007511
+636 636 8.039652
+1094 636 -3.03214
+637 637 5000
+638 638 23.04147
+639 639 100.1759
+640 640 4.950495
+641 641 3.560526
+642 642 17.92648
+643 643 114.9425
+644 644 87.81362
+652 644 -55.55556
+662 644 -32.25806
+645 645 32.23264
+650 645 -26.80965
+1060 645 -5.422993
+646 646 35.23614
+664 646 -30.4878
+1051 646 -4.748338
+647 647 132.4334
+1041 647 -18.79699
+648 648 87.5595
+678 648 -64.93507
+1042 648 -22.62444
+649 649 89.50495
+672 649 -40
+1055 649 -49.50495
+650 650 43.7588
+653 650 -16.94915
+651 651 35.83253
+670 651 -15.12859
+1045 651 -20.70393
+652 652 4092.357
+653 652 -4.440497
+663 652 -3333.333
+666 652 -97.08738
+668 652 -158.7302
+676 652 -18.55288
+1054 652 -333.3333
+653 653 31.9941
+1048 653 -10.60445
+654 654 184.7682
+677 654 -39.84064
+1057 654 -144.9275
+655 655 330.6451
+657 655 -80.64516
+665 655 -250
+656 656 4.040404
+678 656 -4.040404
+657 657 188.172
+1055 657 -107.5269
+658 658 298.2456
+671 658 -131.5789
+672 658 -166.6667
+659 659 65.79597
+677 659 -62.5
+660 660 36.60525
+1050 660 -17.03578
+1058 660 -19.56947
+661 661 500
+677 661 -500
+662 662 90.3976
+672 662 -58.13953
+663 663 3379.205
+664 664 60.0736
+1056 664 -29.5858
+665 665 340.0901
+682 665 -90.0901
+666 666 623.4032
+681 666 -526.3158
+667 667 5.494505
+1051 667 -5.494505
+668 668 222.0213
+669 669 400
+679 669 -66.66667
+1058 669 -333.3333
+670 670 69.18265
+1121 670 -54.05405
+671 671 184.7704
+679 671 -53.19149
+672 672 619.955
+673 672 -24.27184
+682 672 -117.647
+1049 672 -40
+1050 672 -16.80672
+1052 672 -18.34863
+1058 672 -19.84127
+1121 672 -105.2632
+673 673 57.05873
+674 674 340.5986
+675 674 -322.5806
+1046 674 -18.01802
+675 675 353.4448
+1050 675 -30.8642
+676 676 172.399
+1043 676 -153.8461
+677 677 945.5062
+678 677 -4.347826
+1041 677 -66.66667
+1059 677 -243.9024
+678 678 73.3233
+679 679 119.8582
+680 680 103.5542
+1047 680 -46.08295
+1057 680 -57.47126
+681 681 1637.427
+1053 681 -1111.111
+682 682 207.7372
+683 683 1.579031
+689 683 -1.579031
+684 684 1.923077
+689 684 -1.923077
+685 685 1.764913
+691 685 -1.764913
+686 686 328.5128
+687 686 -18.34863
+688 686 -24.44988
+687 687 18.34863
+688 688 71.2075
+689 688 -4.409431
+694 688 -8.312551
+699 688 -3.82995
+700 688 -7.374631
+702 688 -22.83105
+689 689 9.846897
+693 689 -1.935359
+690 690 10178.92
+691 690 -4.99002
+694 690 -161.2903
+697 690 -12.64223
+691 691 10040.88
+692 691 -10000
+695 691 -34.12969
+692 692 10000
+693 693 36.06505
+695 693 -34.12969
+694 694 187.9515
+695 694 -18.34863
+695 695 86.60801
+696 696 70.92198
+697 697 23.45304
+698 697 -10.81081
+698 698 10.81081
+699 699 27.1355
+700 699 -7.968128
+700 700 15.34276
+701 701 10000
+702 702 50.80308
+703 703 10000
+704 704 20.8778
+705 705 10000
+706 706 11.04972
+707 706 -11.04972
+707 707 10067.62
+726 707 -40.24145
+708 708 78.90948
+709 708 -32.46134
+709 709 44.41736
+710 710 10017.32
+711 711 10024.15
+724 711 -11.7096
+734 711 -12.43781
+712 712 10.78749
+714 712 -10.78749
+713 713 10.92896
+714 713 -10.92896
+714 714 102.2441
+715 714 -25.08151
+734 714 -23.9521
+715 715 33.13305
+716 716 370.5403
+717 716 -4.541326
+717 717 346.6114
+718 718 14.51379
+724 718 -14.51379
+719 719 14.51379
+724 719 -14.51379
+720 720 14.51379
+724 720 -14.51379
+721 721 14.51379
+724 721 -14.51379
+722 722 14.51379
+724 722 -14.51379
+723 723 14.51379
+724 723 -14.51379
+724 724 156.8111
+725 724 -2.262444
+734 724 -6.215041
+725 725 37.84963
+726 726 45.27923
+727 727 22.77734
+728 727 -1.437174
+731 727 -21.34016
+728 728 22.77734
+731 728 -21.34016
+729 729 22.54354
+730 729 -.5025378
+731 729 -22.041
+730 730 22.63133
+731 730 -22.12879
+731 731 213.6977
+732 732 2.369107
+733 733 16.02976
+734 734 44.57415
+735 735 3.749531
+743 735 -3.749531
+736 736 5.060729
+742 736 -5.060729
+737 737 10.41667
+743 737 -10.41667
+738 738 25.25253
+742 738 -25.25253
+739 739 6.666667
+744 739 -6.666667
+740 740 10119.13
+741 740 -29.06977
+742 740 -9.416196
+744 740 -80.64516
+741 741 66.38967
+749 741 -30.12048
+759 741 -7.199424
+742 742 97.15186
+743 742 -36.00912
+743 743 65.14973
+746 743 -4.042037
+744 744 120.6452
+756 744 -33.33334
+745 745 10004.53
+746 745 -4.531038
+746 746 8.573075
+747 747 83.11967
+748 747 -15.5521
+758 747 -67.56757
+748 748 35.75412
+757 748 -20.20202
+749 749 109.4856
+753 749 -79.36508
+750 750 36.15381
+751 750 -9.199632
+759 750 -26.95418
+751 751 9.199632
+752 752 14.23959
+757 752 -10.20408
+762 752 -4.035512
+753 753 79.36508
+754 754 101.7248
+757 754 -42.55319
+765 754 -59.1716
+755 755 34.41619
+764 755 -25
+756 756 61.14975
+757 756 -27.81641
+757 757 100.7757
+758 758 67.56757
+759 759 34.1536
+760 760 6.238303
+761 760 -6.238303
+761 761 27.13861
+766 761 -4.730369
+767 761 -8
+762 762 8.519817
+763 762 -4.484305
+763 763 4.484305
+764 764 25
+765 765 59.1716
+766 766 7.055951
+768 766 -2.325581
+767 767 8
+768 768 8.877011
+769 768 -3.030303
+772 768 -3.521127
+769 769 3.030303
+770 770 10003.68
+823 770 -10000
+771 771 4.694836
+772 771 -4.694836
+772 772 8.215962
+773 773 41.97304
+774 773 -9.191176
+795 773 -6.067961
+801 773 -5.646527
+802 773 -14.49275
+774 774 9.191176
+775 775 87.97093
+776 775 -79.36508
+781 775 -8.605852
+776 776 145.414
+781 776 -7.067138
+796 776 -2.511932
+797 776 -3.435246
+1127 776 -31.94888
+777 777 24.34617
+793 777 -3.907777
+795 777 -5.797101
+800 777 -14.64129
+778 778 27.51031
+779 778 -27.51031
+779 779 52.63487
+780 779 -3.543586
+787 779 -9.980041
+780 780 53.90431
+797 780 -2.540005
+949 780 -14.57726
+781 781 36.74781
+782 782 311.0084
+783 782 -12.73885
+783 783 35.45316
+784 784 25.00479
+802 784 -14.57726
+785 785 85.04397
+786 785 -26.66667
+788 785 -16.80672
+796 785 -5.941771
+797 785 -11.00715
+802 785 -4.541326
+786 786 1295.897
+1128 786 -500
+787 787 18.79838
+949 787 -8.818342
+788 788 201.3442
+789 788 -2.932551
+796 788 -9.191176
+1132 788 -172.4138
+789 789 6.317792
+1094 789 -3.38524
+790 790 67.99094
+791 790 -32.89474
+795 790 -23.80952
+791 791 32.89474
+792 792 18.23483
+799 792 -10.1833
+1110 792 -8.05153
+793 793 10007.82
+794 793 -3.907777
+825 793 -10000
+794 794 24.34617
+795 794 -5.797101
+800 794 -14.64129
+795 795 127.8688
+796 796 80.27805
+799 796 -21.83406
+797 797 107.3739
+798 797 -4.818348
+803 797 -4.5106
+805 797 -18.83239
+1119 797 -50
+798 798 10.82075
+1114 798 -6.002401
+799 799 37.88935
+800 800 50.74181
+801 801 5.646527
+802 802 10046.63
+1137 802 -10000
+803 803 290.2097
+804 803 -45.45454
+804 804 45.45454
+805 805 136.4794
+1138 805 -117.647
+806 806 4.975124
+821 806 -4.975124
+807 807 4.065041
+887 807 -4.065041
+808 808 13.33333
+897 808 -13.33333
+809 809 202.3231
+920 809 -112.3596
+810 810 39.62731
+811 810 -26.31579
+918 810 -8.70322
+811 811 26.31579
+812 812 500
+909 812 -500
+813 813 805.5555
+885 813 -11.13586
+940 813 -25.18892
+1122 813 -769.2308
+814 814 149.0395
+815 814 -112.3596
+909 814 -23.14815
+918 814 -13.5318
+815 815 112.3596
+816 816 67.98565
+817 816 -47.61905
+927 816 -20.3666
+817 817 139.7763
+818 817 -52.63158
+907 817 -39.52569
+818 818 52.63158
+819 819 340.5726
+820 819 -232.5581
+820 820 232.5581
+821 821 148.7724
+888 821 -22.72727
+897 821 -114.9425
+921 821 -6.127451
+822 822 133.5555
+880 822 -77.51938
+885 822 -25.64103
+935 822 -30.39514
+823 823 10000
+824 824 10000
+825 824 -10000
+825 825 20000
+826 826 10000
+827 827 10000
+828 828 12.59221
+837 828 -3.399953
+838 828 -4.568296
+874 828 -4.623957
+829 829 38.16908
+833 829 -9.785693
+839 829 -9.75447
+830 830 35.75804
+833 830 -12.15067
+840 830 -9.559799
+831 831 21.56662
+841 831 -10.30197
+869 831 -11.26464
+832 832 24.83065
+842 832 -6.55914
+869 832 -18.27151
+833 833 25.06136
+843 833 -3.125
+834 834 65.83865
+844 834 -9.816236
+874 834 -56.02241
+835 835 97.96513
+845 835 -1.626016
+862 835 -96.33911
+836 836 13.04241
+846 836 -6.806048
+869 836 -6.236358
+837 837 3.399953
+838 838 11.84409
+859 838 -6.666667
+860 838 -.6091247
+839 839 9.75447
+840 840 9.559799
+841 841 10.30197
+842 842 6.55914
+843 843 3.125
+844 844 9.816236
+845 845 1.626016
+846 846 6.806048
+847 847 6.595976
+848 847 -6.595976
+848 848 14.46535
+873 848 -7.869369
+849 849 7.853637
+854 849 -3.149802
+863 849 -2.352
+864 849 -2.351835
+850 850 9.019895
+855 850 -6.506006
+873 850 -2.513889
+851 851 1.965569
+856 851 -.805153
+858 851 -1.160416
+852 852 2.941501
+857 852 -1.692047
+869 852 -1.249453
+853 853 5.118024
+876 853 -5.118024
+854 854 3.149802
+855 855 6.506006
+856 856 .805153
+857 857 1.692047
+858 858 3.252203
+873 858 -2.091788
+859 859 6.666667
+860 860 1.422133
+861 860 -.8130081
+861 861 .8130081
+862 862 101.7126
+863 862 -5.373455
+863 863 10.56927
+869 863 -2.843818
+864 864 26.18864
+865 864 -20.9205
+869 864 -2.916302
+865 865 20.9205
+866 866 71.85119
+869 866 -67.1141
+877 866 -4.737092
+867 867 5.135919
+878 867 -5.135919
+868 868 79.89591
+869 868 -68.02721
+871 868 -7.077141
+877 868 -4.791567
+869 869 245.0375
+870 869 -67.1141
+870 870 78.94842
+871 870 -7.097232
+877 870 -4.737092
+871 871 14.17437
+872 872 3.439506
+882 872 -3.439506
+873 873 19.16737
+882 873 -6.692322
+874 874 65.74321
+878 874 -5.09684
+875 875 268.8182
+876 875 -227.2727
+882 875 -41.54549
+876 876 232.3908
+877 877 144.0989
+878 877 -54.4514
+883 877 -15.50147
+878 878 64.68417
+879 879 13.23977
+882 879 -13.23977
+880 880 77.51938
+881 881 15.55936
+882 881 -15.55936
+882 882 209.5087
+883 883 15.50147
+884 884 208.6253
+937 884 -114.2857
+1123 884 -94.33962
+885 885 67.07021
+914 885 -9.425071
+918 885 -4.842615
+886 886 34.37894
+895 886 -13.2626
+927 886 -6.146282
+937 886 -14.97006
+887 887 30.75514
+896 887 -15.94896
+897 887 -10.74114
+888 888 34.51973
+924 888 -11.79245
+889 889 526.3158
+917 889 -526.3158
+890 890 90.17906
+891 890 -55.55556
+908 890 -11.79245
+917 890 -22.83105
+891 891 55.55556
+892 892 185.8299
+893 892 -78.74016
+899 892 -68.02721
+927 892 -39.0625
+893 893 78.74016
+894 894 10037.21
+921 894 -24.93765
+936 894 -12.26994
+1126 894 -10000
+895 895 486.023
+1125 895 -454.5455
+896 896 48.8437
+897 896 -32.89474
+897 897 242.6641
+913 897 -54.05405
+917 897 -3.943218
+944 897 -12.7551
+898 898 119.8733
+914 898 -21.83406
+926 898 -98.03922
+899 899 208.295
+900 899 -36.10108
+908 899 -104.1667
+900 900 36.10108
+901 901 10010.86
+913 901 -10.85776
+1129 901 -10000
+902 902 400
+906 902 -400
+903 903 27.64992
+987 903 -11.14827
+904 904 479.299
+905 904 -256.4103
+939 904 -192.3077
+944 904 -30.58104
+905 905 256.4103
+906 906 613.7151
+922 906 -17.63668
+939 906 -196.0784
+907 907 58.35809
+923 907 -18.83239
+908 908 200.9013
+912 908 -43.47826
+917 908 -7.668712
+927 908 -20.04008
+909 909 941.4488
+910 909 -196.0784
+910 910 196.0784
+911 911 131.5789
+912 911 -131.5789
+912 912 193.5415
+913 913 64.91182
+914 914 181.033
+935 914 -29.23977
+938 914 -78.74016
+963 914 -4.030633
+915 915 103.0928
+926 915 -103.0928
+916 916 81.96722
+991 916 -81.96722
+917 917 754.9636
+918 917 -121.2121
+930 917 -72.9927
+918 918 232.832
+919 918 -52.63158
+1130 918 -22.47191
+919 919 69.87296
+940 919 -17.24138
+920 920 112.3596
+921 921 123.4495
+923 921 -22.98851
+924 921 -29.06977
+927 921 -5.208333
+933 921 -23.98081
+937 921 -6.583279
+922 922 10033.82
+930 922 -16.18123
+1133 922 -10000
+923 923 41.8209
+924 924 79.77273
+925 924 -38.91051
+925 925 38.91051
+926 926 212.049
+927 927 138.2073
+928 927 -37.17472
+937 927 -6.793478
+928 928 1045.494
+929 928 -999.9999
+937 928 -8.319468
+929 929 999.9999
+930 930 185.3278
+931 930 -96.15385
+931 931 96.15385
+932 932 78.74016
+949 932 -78.74016
+933 933 187.4276
+934 933 -131.5789
+942 933 -26.24672
+934 934 131.5789
+935 935 94.47811
+947 935 -34.8432
+936 936 10045.17
+937 936 -32.89474
+1134 936 -10000
+937 937 199.8084
+938 938 78.74016
+939 939 388.3861
+940 940 141.4402
+954 940 -99.0099
+941 941 400
+954 941 -400
+942 942 26.24672
+943 943 80.64516
+944 943 -80.64516
+944 944 123.9813
+945 945 3.323363
+961 945 -3.323363
+946 946 10.52632
+965 946 -10.52632
+947 947 34.8432
+948 948 110.8715
+952 948 -12.90323
+955 948 -10.88454
+957 948 -18.50139
+958 948 -35.08772
+964 948 -22.32143
+949 949 102.1358
+950 950 59.13499
+951 950 -4.399472
+955 950 -35.77818
+956 950 -18.95735
+951 951 26.76585
+965 951 -4.962779
+969 951 -10.17294
+975 951 -7.230658
+952 952 78.2627
+960 952 -65.35947
+953 953 32.95959
+955 953 -22.47191
+954 954 528.2497
+959 954 -29.23977
+955 955 69.13462
+956 956 18.95735
+957 957 153.211
+958 957 -17.85714
+967 957 -21.69197
+973 957 -78.74016
+958 958 52.94486
+959 959 29.23977
+960 960 465.3595
+973 960 -400
+961 961 17.72833
+962 961 -6.060606
+963 961 -2.007226
+968 961 -6.337135
+962 962 42.96098
+966 962 -36.90037
+963 963 10.88517
+972 963 -4.84731
+964 964 44.64286
+965 965 31.12432
+968 965 -5.945303
+969 965 -9.689922
+966 966 135.9103
+971 966 -99.0099
+967 967 90.18512
+968 968 78.07191
+969 969 26.28135
+970 969 -6.418485
+970 970 6.418485
+971 971 110.4385
+972 971 -11.42857
+972 972 16.27588
+973 973 478.7402
+974 974 999.9999
+983 974 -999.9999
+975 975 7.230658
+976 976 131.2277
+977 976 -72.9927
+989 976 -11.28668
+1001 976 -46.94836
+977 977 72.9927
+978 978 11.99041
+989 978 -11.99041
+979 979 212.9332
+983 979 -208.3333
+980 980 17.87741
+981 981 214.386
+982 981 -192.3077
+984 981 -12.04819
+998 981 -10.03009
+982 982 192.3077
+983 983 1216.031
+986 983 -7.69823
+984 984 12.04819
+985 985 73.44217
+987 985 -24.27184
+989 985 -5.310675
+986 986 19.79013
+987 987 74.94581
+999 987 -39.52569
+988 988 25.27583
+993 988 -12.42236
+995 988 -12.85347
+989 989 48.93494
+990 989 -5.906674
+990 990 5.906674
+991 991 98.75978
+992 991 -2.932551
+992 992 2.932551
+993 993 29.40311
+994 993 -4.027386
+994 994 4.027386
+995 995 28.39029
+997 995 -7.267442
+998 995 -4.033885
+996 996 78.74016
+997 996 -78.74016
+997 997 95.07378
+998 997 -9.066183
+998 998 31.19142
+999 999 39.52569
+1000 1000 18.28154
+1001 1001 64.90168
+1002 1002 8.319468
+1027 1002 -8.319468
+1003 1003 4.084967
+1035 1003 -4.084967
+1004 1004 3.15856
+1032 1004 -3.15856
+1005 1005 10.41667
+1039 1005 -10.41667
+1006 1006 125.0263
+1007 1006 -4.149378
+1013 1006 -19.26782
+1019 1006 -10.48218
+1021 1006 -55.24862
+1030 1006 -14.32665
+1035 1006 -21.55167
+1007 1007 4.149378
+1008 1008 24.42002
+1023 1008 -24.42002
+1009 1009 10093.68
+1010 1009 -1.567398
+1018 1009 -21.18644
+1019 1009 -6.297229
+1021 1009 -4.244482
+1022 1009 -57.14286
+1040 1009 -3.239391
+1010 1010 1.567398
+1011 1011 41.15226
+1040 1011 -41.15226
+1012 1012 5.87199
+1040 1012 -5.87199
+1013 1013 748.2476
+1023 1013 -15.07159
+1029 1013 -454.4287
+1030 1013 -32.57329
+1037 1013 -118.3432
+1038 1013 -102.5641
+1039 1013 -5.9988
+1014 1014 75.75758
+1039 1014 -75.75758
+1015 1015 34.60208
+1029 1015 -34.60208
+1016 1016 4.102139
+1024 1016 -2.283626
+1034 1016 -1.818512
+1017 1017 73.80074
+1029 1017 -73.80074
+1018 1018 21.18644
+1019 1019 60.25767
+1033 1019 -43.47826
+1020 1020 34.96503
+1021 1020 -34.96503
+1021 1021 94.45814
+1022 1022 57.14286
+1023 1023 66.38087
+1024 1023 -6.973501
+1025 1023 -7.352941
+1027 1023 -12.56281
+1024 1024 11.00568
+1034 1024 -1.748557
+1025 1025 11.7895
+1028 1025 -4.436557
+1026 1026 16.88283
+1027 1026 -9.615384
+1027 1027 38.77231
+1028 1027 -3.372681
+1035 1027 -4.901961
+1028 1028 7.809239
+1029 1029 10562.83
+1030 1030 56.26324
+1031 1030 -9.363297
+1031 1031 9.363297
+1032 1032 19.43661
+1034 1032 -5.455537
+1036 1032 -10.82251
+1033 1033 43.47826
+1034 1034 9.022608
+1035 1035 34.63528
+1036 1035 -4.096682
+1036 1036 14.91919
+1037 1037 118.3432
+1038 1038 102.5641
+1039 1039 92.17304
+1040 1040 50.26365
+1041 1041 85.46367
+1042 1042 22.62444
+1043 1043 179.8878
+1050 1043 -26.04167
+1044 1044 51.6919
+1050 1044 -33.8983
+1053 1044 -17.79359
+1045 1045 69.24763
+1047 1045 -48.54369
+1046 1046 71.20951
+1058 1046 -53.19149
+1047 1047 106.0162
+1048 1048 10.60445
+1049 1049 66.17801
+1054 1049 -26.17801
+1050 1050 5329.354
+1051 1050 -4.708098
+1052 1050 -200
+1051 1051 18.55844
+1056 1051 -3.607503
+1052 1052 218.3486
+1053 1053 1128.905
+1054 1054 359.5114
+1055 1055 267.5334
+1056 1055 -4.118616
+1058 1055 -106.383
+1056 1056 37.31192
+1057 1057 202.3988
+1058 1058 532.3185
+1059 1059 295.9858
+1061 1059 -52.08333
+1060 1060 5.422993
+1061 1061 56.57973
+1062 1062 9.573448
+1072 1062 -6.301197
+1085 1062 -3.272251
+1063 1063 14.97518
+1074 1063 -10.35197
+1064 1064 15.77324
+1075 1064 -11.49425
+1087 1064 -4.27899
+1065 1065 56.4111
+1077 1065 -7.256895
+1080 1065 -32.67974
+1066 1066 14.0056
+1068 1066 -14.0056
+1067 1067 2.654632
+1071 1067 -2.654632
+1068 1068 96.37038
+1069 1068 -21.64502
+1092 1068 -9.569378
+1069 1069 49.50018
+1086 1069 -27.85515
+1070 1070 27.63299
+1077 1070 -21.09705
+1113 1070 -6.535948
+1071 1071 6.953859
+1088 1071 -4.299226
+1072 1072 10.51705
+1075 1072 -4.215851
+1073 1073 194.6093
+1074 1073 -163.9344
+1074 1074 315.5717
+1075 1074 -5.040322
+1081 1074 -85.47009
+1086 1074 -21.73913
+1075 1075 25.04781
+1082 1075 -4.297379
+1076 1076 18.69159
+1077 1076 -18.69159
+1077 1077 82.05888
+1078 1077 -9.416196
+1080 1077 -9.328359
+1088 1077 -11.54734
+1095 1077 -4.721436
+1078 1078 30.82096
+1084 1078 -17.66784
+1085 1078 -3.736921
+1079 1079 3.007519
+1117 1079 -3.007519
+1080 1080 42.00809
+1081 1081 85.47009
+1082 1082 4.297379
+1083 1083 500
+1087 1083 -500
+1084 1084 17.66784
+1085 1085 7.009172
+1086 1086 49.59428
+1087 1087 504.279
+1088 1088 15.84657
+1089 1089 12.21374
+1106 1089 -9.225092
+1112 1089 -2.988643
+1090 1090 123.7557
+1091 1090 -4.708098
+1091 1091 14.65977
+1093 1091 -5.681818
+1108 1091 -4.269855
+1092 1092 36.4511
+1093 1093 14.98414
+1098 1093 -9.302325
+1094 1094 6.417381
+1095 1095 136.3807
+1096 1095 -4.440497
+1100 1095 -7.342144
+1105 1095 -16.23377
+1113 1095 -87.7193
+1096 1096 16.50748
+1103 1096 -7.581501
+1107 1096 -2.338634
+1112 1096 -2.146844
+1097 1097 99.0099
+1100 1097 -99.0099
+1098 1098 15.8597
+1099 1098 -6.557377
+1099 1099 6.557377
+1100 1100 110.9308
+1101 1100 -4.578754
+1101 1101 13.03916
+1120 1101 -5.503577
+1102 1102 4.524887
+1112 1102 -4.524887
+1103 1103 7.581501
+1104 1104 46.18962
+1118 1104 -23.14815
+1119 1104 -23.04147
+1105 1105 22.27241
+1106 1106 15.2311
+1109 1106 -6.006006
+1107 1107 2.338634
+1108 1108 4.269855
+1109 1109 6.006006
+1110 1110 10.67965
+1114 1110 -2.628121
+1111 1111 7.380074
+1119 1111 -7.380074
+1112 1112 9.660375
+1113 1113 94.25525
+1114 1114 15.25571
+1115 1114 -4.384042
+1116 1114 -2.241148
+1115 1115 4.384042
+1116 1116 2.241148
+1117 1117 7.461862
+1120 1117 -4.454343
+1118 1118 23.14815
+1119 1119 80.42155
+1120 1120 9.95792
+1121 1121 159.3172
+1122 1122 791.0173
+1136 1122 -21.78649
+1123 1123 145.36
+1124 1124 37.05693
+1135 1124 -10.74114
+1125 1125 454.5455
+1126 1126 10000
+1127 1127 31.94888
+1128 1128 500
+1129 1129 10015.82
+1135 1129 -15.82279
+1130 1130 22.47191
+1131 1131 24.39024
+1136 1131 -24.39024
+1132 1132 172.4138
+1133 1133 10000
+1134 1134 10000
+1135 1135 26.56392
+1136 1136 46.17674
+1137 1137 10000
+1138 1138 117.647
diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
new file mode 100644
index 00000000000..672afe4d2a5
--- /dev/null
+++ b/omp/reorder/rcm_kernels.cpp
@@ -0,0 +1,171 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2019, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/reorder/rcm_kernels.hpp"
+
+
+#include <algorithm>
+#include <iterator>
+#include <memory>
+#include <queue>
+#include <utility>
+#include <vector>
+
+
+#include <ginkgo/config.hpp>
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/permutation.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+/**
+ * @brief The reordering namespace.
+ *
+ * @ingroup reorder
+ */
+namespace rcm {
+
+
+template <typename ValueType, typename IndexType>
+void get_degree_of_nodes(
+    std::shared_ptr<const OmpExecutor> exec,
+    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
+    std::shared_ptr<gko::Array<IndexType>> node_degrees)
+{
+    auto num_rows = adjacency_matrix->get_size()[0];
+    auto adj_ptrs = adjacency_matrix->get_row_ptrs();
+    auto node_deg = node_degrees->get_data();
+
+#pragma omp parallel for
+    for (auto i = 0; i < num_rows; ++i) {
+        node_deg[i] = adj_ptrs[i + 1] - adj_ptrs[i];
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
+
+
+template <typename IndexType>
+IndexType find_index(std::vector<std::pair<IndexType, IndexType>> &a,
+                     IndexType x)
+{
+    for (auto i = 0; i < a.size(); i++)
+        if (a[i].first == x) return i;
+    return -1;
+}
+
+
+template <typename ValueType, typename IndexType>
+void get_permutation(
+    std::shared_ptr<const OmpExecutor> exec, size_type num_vertices,
+    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
+    std::shared_ptr<Array<IndexType>> node_degrees,
+    std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,
+    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat)
+{
+    IndexType num_vtxs = static_cast<IndexType>(num_vertices);
+    auto adj_ptrs = adjacency_matrix->get_row_ptrs();
+    auto adj_idxs = adjacency_matrix->get_col_idxs();
+    auto node_deg = node_degrees->get_data();
+    auto permutation_arr = permutation_mat->get_permutation();
+    auto inv_permutation_arr = inv_permutation_mat->get_permutation();
+
+    std::queue<IndexType> q;
+    std::vector<IndexType> r;
+    std::vector<std::pair<IndexType, IndexType>> not_visited;
+
+    for (auto i = 0; i < num_vtxs; ++i) {
+        not_visited.push_back(std::make_pair(i, node_deg[i]));
+    }
+
+    while (not_visited.size()) {
+        // choose this better.
+        IndexType min_node_index = 0;
+
+        for (auto i = 0; i < not_visited.size(); i++) {
+            if (not_visited[i].second < not_visited[min_node_index].second) {
+                min_node_index = i;
+            }
+        }
+        q.push(not_visited[min_node_index].first);
+
+        not_visited.erase(
+            not_visited.begin() +
+            find_index(not_visited, not_visited[q.front()].first));
+
+        // Simple BFS
+        while (!q.empty()) {
+            std::vector<IndexType> to_sort;
+
+            for (IndexType i = 0; i < num_vtxs; i++) {
+                if (i != q.front() && find_index(not_visited, i) != -1) {
+                    to_sort.push_back(i);
+                    not_visited.erase(not_visited.begin() +
+                                      find_index(not_visited, i));
+                }
+            }
+
+            std::sort(to_sort.begin(), to_sort.end(),
+                      [&node_deg](int i, int j) {
+                          return node_deg[i] < node_deg[j];
+                      });
+
+            for (auto i = 0; i < to_sort.size(); i++) q.push(to_sort[i]);
+
+            r.push_back(q.front());
+            q.pop();
+        }
+    }
+
+#pragma omp parallel for
+    for (auto i = 0; i < r.size(); ++i) {
+        permutation_arr[i] = r[i];
+        inv_permutation_arr[r[i]] = i;
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
+
+
+}  // namespace rcm
+}  // namespace omp
+}  // namespace kernels
+}  // namespace gko
diff --git a/omp/test/CMakeLists.txt b/omp/test/CMakeLists.txt
index d8257d9087c..2f506e51109 100644
--- a/omp/test/CMakeLists.txt
+++ b/omp/test/CMakeLists.txt
@@ -4,5 +4,6 @@ add_subdirectory(components)
 add_subdirectory(factorization)
 add_subdirectory(matrix)
 add_subdirectory(preconditioner)
+add_subdirectory(reorder)
 add_subdirectory(solver)
 add_subdirectory(stop)
diff --git a/omp/test/reorder/CMakeLists.txt b/omp/test/reorder/CMakeLists.txt
new file mode 100644
index 00000000000..8987ae28a48
--- /dev/null
+++ b/omp/test/reorder/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_test(rcm_kernels)
diff --git a/omp/test/reorder/rcm_kernels.cpp b/omp/test/reorder/rcm_kernels.cpp
new file mode 100644
index 00000000000..f6905a481c0
--- /dev/null
+++ b/omp/test/reorder/rcm_kernels.cpp
@@ -0,0 +1,109 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2019, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/reorder/rcm.hpp>
+
+
+#include <algorithm>
+#include <fstream>
+#include <memory>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/permutation.hpp>
+
+
+#include <iostream>
+#include "core/test/utils.hpp"
+#include "core/test/utils/assertions.hpp"
+#include "matrices/config.hpp"
+
+namespace {
+
+
+class Rcm : public ::testing::Test {
+protected:
+    using v_type = double;
+    using i_type = int;
+    using Mtx = gko::matrix::Dense<v_type>;
+    using CsrMtx = gko::matrix::Csr<v_type, i_type>;
+    using reorder_type = gko::reorder::Rcm<v_type, i_type>;
+    Rcm()
+        : ref(gko::ReferenceExecutor::create()),
+          omp(gko::OmpExecutor::create()),
+          o_1138_bus_mtx(gko::read<CsrMtx>(
+              std::ifstream(gko::matrices::location_1138_bus_mtx, std::ios::in),
+              ref)),
+          d_1138_bus_mtx(gko::read<CsrMtx>(
+              std::ifstream(gko::matrices::location_1138_bus_mtx, std::ios::in),
+              omp))
+    {}
+
+    static void assert_equal_permutations(
+        const gko::matrix::Permutation<i_type> *to_check,
+        const gko::matrix::Permutation<i_type> *orig)
+    {
+        auto o_p_size = orig->get_permutation_size();
+        auto d_p_size = to_check->get_permutation_size();
+        ASSERT_EQ(d_p_size, o_p_size);
+
+        for (auto i = 0; i < d_p_size; ++i) {
+            ASSERT_EQ(to_check->get_const_permutation()[i],
+                      orig->get_const_permutation()[i]);
+        }
+    }
+
+    std::shared_ptr<const gko::Executor> ref;
+    std::shared_ptr<const gko::Executor> omp;
+    std::shared_ptr<CsrMtx> o_1138_bus_mtx;
+    std::shared_ptr<CsrMtx> d_1138_bus_mtx;
+    std::unique_ptr<reorder_type> reorder_op;
+    std::unique_ptr<reorder_type> d_reorder_op;
+};
+
+TEST_F(Rcm, OmpPermutationIsEquivalentToRef)
+{
+    reorder_op = reorder_type::build().on(ref)->generate(o_1138_bus_mtx);
+    d_reorder_op = reorder_type::build().on(omp)->generate(d_1138_bus_mtx);
+
+    auto perm = reorder_op->get_permutation();
+    auto d_perm = d_reorder_op->get_permutation();
+
+    assert_equal_permutations(d_perm.get(), perm.get());
+}
+
+}  // namespace
diff --git a/reference/reorder/rcm_kernels.cpp b/reference/reorder/rcm_kernels.cpp
new file mode 100644
index 00000000000..495668f74ed
--- /dev/null
+++ b/reference/reorder/rcm_kernels.cpp
@@ -0,0 +1,170 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2019, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+
+#include "core/reorder/rcm_kernels.hpp"
+
+
+#include <algorithm>
+#include <iterator>
+#include <memory>
+#include <queue>
+#include <utility>
+#include <vector>
+
+
+#include <ginkgo/config.hpp>
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/permutation.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace reference {
+/**
+ * @brief The reordering namespace.
+ *
+ * @ingroup reorder
+ */
+namespace rcm {
+
+
+template <typename ValueType, typename IndexType>
+void get_degree_of_nodes(
+    std::shared_ptr<const ReferenceExecutor> exec,
+    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
+    std::shared_ptr<gko::Array<IndexType>> node_degrees)
+{
+    auto num_rows = adjacency_matrix->get_size()[0];
+    auto adj_ptrs = adjacency_matrix->get_row_ptrs();
+    auto node_deg = node_degrees->get_data();
+
+    for (auto i = 0; i < num_rows; ++i) {
+        node_deg[i] = adj_ptrs[i + 1] - adj_ptrs[i];
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
+
+
+template <typename IndexType>
+IndexType find_index(std::vector<std::pair<IndexType, IndexType>> &a,
+                     IndexType x)
+{
+    for (auto i = 0; i < a.size(); i++)
+        if (a[i].first == x) return i;
+    return -1;
+}
+
+
+template <typename ValueType, typename IndexType>
+void get_permutation(
+    std::shared_ptr<const ReferenceExecutor> exec, size_type num_vertices,
+    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
+    std::shared_ptr<Array<IndexType>> node_degrees,
+    std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,
+    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat)
+{
+    IndexType num_vtxs = static_cast<IndexType>(num_vertices);
+    auto adj_ptrs = adjacency_matrix->get_row_ptrs();
+    auto adj_idxs = adjacency_matrix->get_col_idxs();
+    auto node_deg = node_degrees->get_data();
+    auto permutation_arr = permutation_mat->get_permutation();
+    auto inv_permutation_arr = inv_permutation_mat->get_permutation();
+
+    std::queue<IndexType> q;
+    std::vector<IndexType> r;
+    std::vector<std::pair<IndexType, IndexType>> not_visited;
+
+    for (auto i = 0; i < num_vtxs; ++i) {
+        not_visited.push_back(std::make_pair(i, node_deg[i]));
+    }
+
+    while (not_visited.size()) {
+        // choose this better.
+        IndexType min_node_index = 0;
+
+        for (auto i = 0; i < not_visited.size(); i++) {
+            if (not_visited[i].second < not_visited[min_node_index].second) {
+                min_node_index = i;
+            }
+        }
+        q.push(not_visited[min_node_index].first);
+
+        not_visited.erase(
+            not_visited.begin() +
+            find_index(not_visited, not_visited[q.front()].first));
+
+        // Simple BFS
+        while (!q.empty()) {
+            std::vector<IndexType> to_sort;
+
+            for (IndexType i = 0; i < num_vtxs; i++) {
+                if (i != q.front() && find_index(not_visited, i) != -1) {
+                    to_sort.push_back(i);
+                    not_visited.erase(not_visited.begin() +
+                                      find_index(not_visited, i));
+                }
+            }
+
+            std::sort(to_sort.begin(), to_sort.end(),
+                      [&node_deg](IndexType i, IndexType j) {
+                          return node_deg[i] < node_deg[j];
+                      });
+
+            for (auto i = 0; i < to_sort.size(); i++) q.push(to_sort[i]);
+
+            r.push_back(q.front());
+            q.pop();
+        }
+    }
+
+    for (auto i = 0; i < r.size(); ++i) {
+        permutation_arr[i] = r[i];
+        inv_permutation_arr[r[i]] = i;
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
+
+
+}  // namespace rcm
+}  // namespace reference
+}  // namespace kernels
+}  // namespace gko
diff --git a/reference/test/CMakeLists.txt b/reference/test/CMakeLists.txt
index 44da92b0bd9..384ce384e08 100644
--- a/reference/test/CMakeLists.txt
+++ b/reference/test/CMakeLists.txt
@@ -6,6 +6,7 @@ add_subdirectory(factorization)
 add_subdirectory(log)
 add_subdirectory(matrix)
 add_subdirectory(preconditioner)
+add_subdirectory(reorder)
 add_subdirectory(solver)
 add_subdirectory(stop)
 add_subdirectory(utils)
diff --git a/reference/test/reorder/CMakeLists.txt b/reference/test/reorder/CMakeLists.txt
new file mode 100644
index 00000000000..fef614b98f6
--- /dev/null
+++ b/reference/test/reorder/CMakeLists.txt
@@ -0,0 +1,2 @@
+ginkgo_create_test(rcm)
+ginkgo_create_test(rcm_kernels)
diff --git a/reference/test/reorder/rcm.cpp b/reference/test/reorder/rcm.cpp
new file mode 100644
index 00000000000..b6c0bc29b5c
--- /dev/null
+++ b/reference/test/reorder/rcm.cpp
@@ -0,0 +1,107 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2019, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/reorder/rcm.hpp>
+
+
+#include <algorithm>
+#include <fstream>
+#include <memory>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+
+
+#include <iostream>
+#include "core/test/utils.hpp"
+#include "core/test/utils/assertions.hpp"
+#include "matrices/config.hpp"
+
+namespace {
+
+
+class Rcm : public ::testing::Test {
+protected:
+    using v_type = double;
+    using i_type = int;
+    using Mtx = gko::matrix::Dense<v_type>;
+    using CsrMtx = gko::matrix::Csr<v_type, i_type>;
+    using reorder_type = gko::reorder::Rcm<v_type, i_type>;
+    Rcm()
+        : exec(gko::ReferenceExecutor::create()),
+          rcm_factory(reorder_type::build().on(exec)),
+          // clang-format off
+          p_mtx(gko::initialize<CsrMtx>(
+                                        {{1.0, 2.0, 0.0, -1.3, 2.1},
+                                         {2.0, 5.0, 1.5, 0.0, 0.0},
+                                         {0.0, 1.5, 1.5, 1.1, 0.0},
+                                         {-1.3, 0.0, 1.1, 2.0, 0.0},
+                                         {2.1, 0.0, 0.0, 0.0, 1.0}},
+                                        exec)),
+          // clang-format on
+          reorder_op(rcm_factory->generate(p_mtx))
+    {}
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<CsrMtx> p_mtx;
+    std::unique_ptr<reorder_type::Factory> rcm_factory;
+    std::unique_ptr<reorder_type> reorder_op;
+};
+
+
+TEST_F(Rcm, FactoryCreatesCorrectReorderOp)
+{
+    auto adj_mtx = reorder_op->get_adjacency_matrix();
+    auto tmp = gko::matrix::SparsityCsr<v_type, i_type>::create(exec, p_mtx);
+    auto comp_mtx = tmp->to_adjacency_matrix();
+
+    ASSERT_NE(reorder_op->get_adjacency_matrix(), nullptr);
+    GKO_ASSERT_MTX_NEAR(adj_mtx.get(), comp_mtx.get(), 0);
+}
+
+
+TEST_F(Rcm, CanBeCleared)
+{
+    reorder_op->clear();
+
+    auto reorder_op_mtx = reorder_op->get_adjacency_matrix();
+    ASSERT_EQ(reorder_op_mtx, nullptr);
+}
+
+
+}  // namespace
diff --git a/reference/test/reorder/rcm_kernels.cpp b/reference/test/reorder/rcm_kernels.cpp
new file mode 100644
index 00000000000..5114d12bc14
--- /dev/null
+++ b/reference/test/reorder/rcm_kernels.cpp
@@ -0,0 +1,124 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2019, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/reorder/rcm.hpp>
+
+
+#include <algorithm>
+#include <fstream>
+#include <memory>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/permutation.hpp>
+
+
+#include "core/test/utils/assertions.hpp"
+#include "matrices/config.hpp"
+
+
+namespace {
+
+
+class Rcm : public ::testing::Test {
+protected:
+    using v_type = double;
+    using i_type = int;
+    using Mtx = gko::matrix::Dense<v_type>;
+    using CsrMtx = gko::matrix::Csr<v_type, i_type>;
+    using reorder_type = gko::reorder::Rcm<v_type, i_type>;
+    using perm_type = gko::matrix::Permutation<i_type>;
+    Rcm()
+        : exec(gko::ReferenceExecutor::create()),
+          // clang-format off
+          p_mtx(gko::initialize<CsrMtx>(
+                                        {{1.0, 2.0, 0.0, -1.3, 2.1},
+                                         {2.0, 5.0, 1.5, 0.0, 0.0},
+                                         {0.0, 1.5, 1.5, 1.1, 0.0},
+                                         {-1.3, 0.0, 1.1, 2.0, 0.0},
+                                         {2.1, 0.0, 0.0, 0.0, 1.0}},
+                                        exec)),
+          // clang-format on
+          rcm_factory(reorder_type::build().on(exec)),
+          reorder_op(rcm_factory->generate(p_mtx))
+    {}
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<CsrMtx> p_mtx;
+    std::unique_ptr<reorder_type::Factory> rcm_factory;
+    std::unique_ptr<reorder_type> reorder_op;
+
+    bool find_duplicates(i_type val, std::size_t index, const i_type *data,
+                         std::size_t length)
+    {
+        auto count = 0;
+        for (auto i = 0; i < length; ++i) {
+            if (i != index && val == data[i]) {
+                count++;
+            }
+        }
+        if (count == 0) {
+            return false;
+        } else {
+            return true;
+        }
+    }
+
+    void assert_correct_permutation(const perm_type *input_perm)
+    {
+        auto perm_data = input_perm->get_const_permutation();
+        auto perm_size = input_perm->get_permutation_size();
+
+        for (auto i = 0; i < perm_size; ++i) {
+            ASSERT_LT(perm_data[i], perm_size);
+            ASSERT_GE(perm_data[i], 0);
+            ASSERT_FALSE(
+                find_duplicates(perm_data[i], i, perm_data, perm_size));
+        }
+    }
+};
+
+
+TEST_F(Rcm, CreatesAPermutation)
+{
+    auto p = reorder_op->get_permutation();
+    assert_correct_permutation(p.get());
+}
+
+
+}  // namespace

From 643313527210ff3292f598d27831f0adcc130af2 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Wed, 15 Apr 2020 13:05:47 +0200
Subject: [PATCH 0036/1008] Add strategy parameter

---
 core/reorder/rcm.cpp                |  8 ++++----
 core/reorder/rcm_kernels.hpp        |  5 +++--
 include/ginkgo/core/reorder/rcm.hpp | 28 ++++++++++++++++++++--------
 include/ginkgo/ginkgo.hpp           |  3 +++
 omp/reorder/rcm_kernels.cpp         |  5 +++--
 5 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/core/reorder/rcm.cpp b/core/reorder/rcm.cpp
index f7e41ecb9b6..7671acb5ce1 100644
--- a/core/reorder/rcm.cpp
+++ b/core/reorder/rcm.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2019, the Ginkgo authors
+Copyright (c) 2017-2020, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -71,9 +71,9 @@ void Rcm<ValueType, IndexType>::generate() const
     // RCM is only valid for symmetric matrices. Need to add an expensive check
     // for symmetricity here ?
     exec->run(rcm::make_get_degree_of_nodes(adjacency_matrix_, node_degrees_));
-    exec->run(rcm::make_get_permutation(num_rows, adjacency_matrix_,
-                                        node_degrees_, permutation_,
-                                        inv_permutation_));
+    exec->run(rcm::make_get_permutation(
+        num_rows, adjacency_matrix_, node_degrees_, permutation_,
+        inv_permutation_, parameters_.starting_strategy));
 }
 
 
diff --git a/core/reorder/rcm_kernels.hpp b/core/reorder/rcm_kernels.hpp
index 7abfeb5fd7c..e69c4348da6 100644
--- a/core/reorder/rcm_kernels.hpp
+++ b/core/reorder/rcm_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2019, the Ginkgo authors
+Copyright (c) 2017-2020, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -59,7 +59,8 @@ namespace kernels {
             adjacency_matrix,                                                \
         std::shared_ptr<Array<IndexType>> node_degrees,                      \
         std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,     \
-        std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat)
+        std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat, \
+        const gko::reorder::starting_strategy strategy)
 
 
 #define GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL(ValueType, IndexType) \
diff --git a/include/ginkgo/core/reorder/rcm.hpp b/include/ginkgo/core/reorder/rcm.hpp
index 7b130b7ef51..71fd824e988 100644
--- a/include/ginkgo/core/reorder/rcm.hpp
+++ b/include/ginkgo/core/reorder/rcm.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2019, the Ginkgo authors
+Copyright (c) 2017-2020, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -59,6 +59,8 @@ namespace gko {
  */
 namespace reorder {
 
+enum class starting_strategy { minimum_degree, pseudo_peripheral };
+
 
 /**
  * Rcm is the reordering algorithm which uses METIS to compute a
@@ -72,6 +74,8 @@ namespace reorder {
  * the form of the Permutation matrix), which can be used to apply to reorder a
  * matrix as required.
  *
+ * <Explanation starting strategy here>
+ *
  * @tparam ValueType  Type of the values of all matrices used in this class
  * @tparam IndexType  Type of the indices of all matrices used in this class
  *
@@ -126,6 +130,13 @@ class Rcm : public EnablePolymorphicObject<Rcm<ValueType, IndexType>,
          * constructed along with the normal permutation matrix.
          */
         bool GKO_FACTORY_PARAMETER(construct_inverse_permutation, false);
+
+        /**
+         * This parameter controls the strategy used to determine a starting
+         * vertex.
+         */
+        starting_strategy GKO_FACTORY_PARAMETER(
+            starting_strategy, starting_strategy::pseudo_peripheral);
     };
     GKO_ENABLE_REORDERING_BASE_FACTORY(Rcm, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);
@@ -159,20 +170,21 @@ class Rcm : public EnablePolymorphicObject<Rcm<ValueType, IndexType>,
             // the diagonal elements and outputs an adjacency matrix.
             adjacency_matrix_ = tmp->to_adjacency_matrix();
         }
+        auto const dim = adjacency_matrix_->get_size();
         node_degrees_ = std::shared_ptr<Array<IndexType>>(
-            new Array<IndexType>(exec, adjacency_matrix_->get_size()[0]));
-        permutation_ =
-            PermutationMatrix::create(exec, adjacency_matrix_->get_size());
-        inv_permutation_ =
-            PermutationMatrix::create(exec, adjacency_matrix_->get_size());
+            new Array<IndexType>(exec, dim[0]));
+        permutation_ = PermutationMatrix::create(exec, dim);
+        if (parameters_.construct_inverse_permutation) {
+            inv_permutation_ = PermutationMatrix::create(exec, dim);
+        }
 
         this->generate();
     }
 
 private:
     std::shared_ptr<SparsityMatrix> adjacency_matrix_{};
-    std::shared_ptr<Array<IndexType>> node_degrees_{};
     std::shared_ptr<PermutationMatrix> permutation_{};
+    std::shared_ptr<Array<IndexType>> node_degrees_{};
     std::shared_ptr<PermutationMatrix> inv_permutation_{};
 };
 
@@ -181,4 +193,4 @@ class Rcm : public EnablePolymorphicObject<Rcm<ValueType, IndexType>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_REORDER_RCM_HPP_
+#endif  // GKO_CORE_REORDER_RCM_HPP_
\ No newline at end of file
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 99ddae5219d..71c6a94dfaa 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -85,6 +85,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/preconditioner/isai.hpp>
 #include <ginkgo/core/preconditioner/jacobi.hpp>
 
+#include <ginkgo/core/reorder/rcm.hpp>
+#include <ginkgo/core/reorder/reordering_base.hpp>
+
 #include <ginkgo/core/solver/bicg.hpp>
 #include <ginkgo/core/solver/bicgstab.hpp>
 #include <ginkgo/core/solver/cg.hpp>
diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index 672afe4d2a5..d8609868f01 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2019, the Ginkgo authors
+Copyright (c) 2017-2020, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -98,7 +98,8 @@ void get_permutation(
     std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
     std::shared_ptr<Array<IndexType>> node_degrees,
     std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,
-    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat)
+    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat,
+    const gko::reorder::starting_strategy strategy)
 {
     IndexType num_vtxs = static_cast<IndexType>(num_vertices);
     auto adj_ptrs = adjacency_matrix->get_row_ptrs();

From 36b8b388ce4dfcf94811e566f8a5011ad5a924ad Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Wed, 15 Apr 2020 13:06:42 +0200
Subject: [PATCH 0037/1008] Rewrite rcm reference kernel

---
 reference/reorder/rcm_kernels.cpp | 267 ++++++++++++++++++++++++------
 1 file changed, 214 insertions(+), 53 deletions(-)

diff --git a/reference/reorder/rcm_kernels.cpp b/reference/reorder/rcm_kernels.cpp
index 495668f74ed..aa0a6ac9987 100644
--- a/reference/reorder/rcm_kernels.cpp
+++ b/reference/reorder/rcm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2019, the Ginkgo authors
+Copyright (c) 2017-2020, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -82,81 +82,242 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 
 
-template <typename IndexType>
-IndexType find_index(std::vector<std::pair<IndexType, IndexType>> &a,
-                     IndexType x)
+/**
+ * Computes a level structure rooted at `root`, returning the height of the
+ * structure along with the index of the node of minimal degree in its last
+ * level.
+ */
+template <typename IndexType, typename ValueType>
+std::pair<IndexType, size_type> rls_contender_and_height(
+    std::shared_ptr<const ReferenceExecutor> exec, IndexType root,
+    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
+    std::shared_ptr<Array<IndexType>> node_degrees, size_type num_vertices)
 {
-    for (auto i = 0; i < a.size(); i++)
-        if (a[i].first == x) return i;
-    return -1;
+    auto adj_ptrs = adjacency_matrix->get_row_ptrs();
+    auto adj_idxs = adjacency_matrix->get_col_idxs();
+    auto node_degrees_p = node_degrees->get_data();
+
+    // This could actually be allocated in the calling scope, then reused.
+    auto visited_local =
+        std::unique_ptr<Array<bool>>(new Array<bool>(exec, num_vertices));
+    auto visited_local_p = visited_local->get_data();
+    for (auto i = 0; i < num_vertices; ++i) {
+        visited_local_p[i] = false;
+    }
+
+    // This stores the level structure, starting with the root node.
+    std::vector<IndexType> rls;
+    rls.reserve(num_vertices);
+    rls.push_back(root);
+    visited_local_p[root] = true;
+
+    auto rls_index = 0;
+
+    // Used to compute the height.
+    auto height = 0;
+    auto current_level_countdown = 1;
+    auto next_level_countup = 0;
+
+    // The last levels size is required to compute the contender.
+    auto last_level_size = 0;
+
+    while (rls_index < rls.size()) {
+        auto parent = rls[rls_index];
+        --current_level_countdown;
+
+
+        // Iterate through the parents neighbors.
+        auto row_start = adj_ptrs[parent];
+        auto row_end = adj_ptrs[parent + 1];
+        for (auto neighbor_idx = row_start; neighbor_idx < row_end;
+             ++neighbor_idx) {
+            auto neighbor = adj_idxs[neighbor_idx];
+
+            if (!visited_local_p[neighbor]) {
+                visited_local_p[neighbor] = true;
+                ++next_level_countup;
+                rls.push_back(neighbor);
+            }
+        }
+
+        if (current_level_countdown == 0) {
+            if (next_level_countup > 0) {
+                last_level_size = next_level_countup;
+            }
+
+            current_level_countdown = next_level_countup;
+            next_level_countup = 0;
+            ++height;
+        }
+
+        ++rls_index;
+    }
+
+    // Choose the contender.
+    auto rls_size = rls.size();
+    auto min_degree = std::numeric_limits<IndexType>::max();
+    auto contender = rls_size - last_level_size;
+    for (auto i = rls_size - last_level_size; i < rls_size; ++i) {
+        if (node_degrees_p[rls[i]] < min_degree) {
+            contender = i;
+            min_degree = node_degrees_p[rls[i]];
+        }
+    }
+
+    return std::pair<IndexType, size_type>(rls[contender], height);
 }
 
+template <typename IndexType, typename ValueType>
+IndexType find_starting_node(
+    std::shared_ptr<const ReferenceExecutor> exec,
+    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
+    std::shared_ptr<Array<IndexType>> node_degrees,
+    std::shared_ptr<Array<bool>> visited, size_type num_vertices,
+    gko::reorder::starting_strategy strategy)
+{
+    using strategies = gko::reorder::starting_strategy;
+
+    // Only those strategies are supported here yet, assert this.
+    GKO_ASSERT(strategy == strategies::minimum_degree ||
+               startegy == strategies::pseudo_peripheral);
+
+    auto node_degrees_p = node_degrees->get_data();
+    auto visited_p = visited->get_data();
+
+    // There must always be at least one unvisited node left.
+    auto index_min_node = 0;
+    auto min_node_degree = std::numeric_limits<IndexType>::max();
+    for (auto i = 0; i < num_vertices; ++i) {
+        if (!visited_p[i]) {
+            if (node_degrees_p[i] < min_node_degree) {
+                index_min_node = i;
+                min_node_degree = node_degrees_p[i];
+            }
+        }
+    }
 
+    // If that is all that is required, return.
+    if (strategy == gko::reorder::starting_strategy::minimum_degree) {
+        return index_min_node;
+    }
+
+    // Find a pseudo-peripheral node, starting from an node of minimum degree.
+    auto current = index_min_node;
+
+    // Isolated nodes are by definition perioheral.
+    if (min_node_degree == 0) {
+        return index_min_node;
+    }
+
+    // This algorithm is e. g. presented in
+    // http://heath.cs.illinois.edu/courses/cs598mh/george_liu.pdf, p. 70
+    // onwards.
+    auto contender_and_height = rls_contender_and_height<IndexType, ValueType>(
+        exec, current, adjacency_matrix, node_degrees, num_vertices);
+    while (true) {
+        auto contender_and_height_contender =
+            rls_contender_and_height<IndexType, ValueType>(
+                exec, contender_and_height.first, adjacency_matrix,
+                node_degrees, num_vertices);
+
+        if (contender_and_height_contender.second >
+            contender_and_height.second) {
+            contender_and_height.second = contender_and_height_contender.second;
+            current = contender_and_height.first;
+            contender_and_height.first = contender_and_height_contender.first;
+        } else {
+            return current;
+        }
+    }
+}
+
+/**
+ * Comutes a rcm reording using a naive sequential algorithm.
+ */
 template <typename ValueType, typename IndexType>
 void get_permutation(
     std::shared_ptr<const ReferenceExecutor> exec, size_type num_vertices,
     std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
     std::shared_ptr<Array<IndexType>> node_degrees,
     std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,
-    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat)
+    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat,
+    const gko::reorder::starting_strategy strategy)
 {
     IndexType num_vtxs = static_cast<IndexType>(num_vertices);
     auto adj_ptrs = adjacency_matrix->get_row_ptrs();
     auto adj_idxs = adjacency_matrix->get_col_idxs();
-    auto node_deg = node_degrees->get_data();
-    auto permutation_arr = permutation_mat->get_permutation();
-    auto inv_permutation_arr = inv_permutation_mat->get_permutation();
 
-    std::queue<IndexType> q;
-    std::vector<IndexType> r;
-    std::vector<std::pair<IndexType, IndexType>> not_visited;
-
-    for (auto i = 0; i < num_vtxs; ++i) {
-        not_visited.push_back(std::make_pair(i, node_deg[i]));
+    auto permutation = permutation_mat->get_permutation();
+    auto permutation_inv = inv_permutation_mat.get()
+                               ? inv_permutation_mat->get_permutation()
+                               : nullptr;
+
+    IndexType *degrees = node_degrees->get_data();
+
+    // Storing vertices left to proceess.
+    auto linear_queue = std::unique_ptr<Array<IndexType>>(
+        new Array<IndexType>(exec, num_vertices));
+    auto linear_queue_p = linear_queue->get_data();
+    auto head_offset = 0;
+    auto tail_offset = 0;
+
+    // Storing which vertices have already been visited.
+    auto visited = std::make_shared<Array<bool>>(exec, num_vertices);
+    auto visited_p = visited->get_data();
+    for (auto i = 0; i < num_vertices; ++i) {
+        visited_p[i] = false;
     }
 
-    while (not_visited.size()) {
-        // choose this better.
-        IndexType min_node_index = 0;
+    for (auto perm_index = 0; perm_index < num_vertices; ++perm_index) {
+        // Get the next vertex to process.
+        auto next_vertex = 0;
+        if (head_offset <= tail_offset) {
+            // Choose a new starting vertex, new componenet needs to be
+            // discovered.
+            next_vertex = find_starting_node<IndexType, ValueType>(
+                exec, adjacency_matrix, node_degrees, visited, num_vertices,
+                strategy);
+            visited_p[next_vertex] = true;
+        } else {
+            next_vertex = linear_queue_p[tail_offset];
+            ++tail_offset;
+        }
 
-        for (auto i = 0; i < not_visited.size(); i++) {
-            if (not_visited[i].second < not_visited[min_node_index].second) {
-                min_node_index = i;
+        // Get the neigbours of the next vertex,
+        // check if they have already been visited,
+        // if no, insert them to sort.
+        auto prev_head_offset = head_offset;
+
+        // Is this code correct for getting next_vertexs neighbors?
+        auto row_start = adj_ptrs[next_vertex];
+        auto row_end = adj_ptrs[next_vertex + 1];
+        for (auto neighbor_idx = row_start; neighbor_idx < row_end;
+             ++neighbor_idx) {
+            auto neighbor = adj_idxs[neighbor_idx];
+
+            // If the neighbour hasn't been visited yet, add it to the queue and
+            // mark it.
+            if (!visited_p[neighbor]) {
+                visited_p[neighbor] = true;
+                linear_queue_p[head_offset] = neighbor;
+                ++head_offset;
             }
         }
-        q.push(not_visited[min_node_index].first);
-
-        not_visited.erase(
-            not_visited.begin() +
-            find_index(not_visited, not_visited[q.front()].first));
-
-        // Simple BFS
-        while (!q.empty()) {
-            std::vector<IndexType> to_sort;
-
-            for (IndexType i = 0; i < num_vtxs; i++) {
-                if (i != q.front() && find_index(not_visited, i) != -1) {
-                    to_sort.push_back(i);
-                    not_visited.erase(not_visited.begin() +
-                                      find_index(not_visited, i));
-                }
-            }
 
-            std::sort(to_sort.begin(), to_sort.end(),
-                      [&node_deg](IndexType i, IndexType j) {
-                          return node_deg[i] < node_deg[j];
-                      });
+        auto count_added = head_offset - prev_head_offset;
+        std::sort(linear_queue_p + prev_head_offset,
+                  linear_queue_p + head_offset,
+                  [&degrees](IndexType i, IndexType j) {
+                      return degrees[i] < degrees[j];
+                  });
 
-            for (auto i = 0; i < to_sort.size(); i++) q.push(to_sort[i]);
-
-            r.push_back(q.front());
-            q.pop();
-        }
+        permutation[perm_index] = next_vertex;
     }
 
-    for (auto i = 0; i < r.size(); ++i) {
-        permutation_arr[i] = r[i];
-        inv_permutation_arr[r[i]] = i;
+    if (permutation_inv) {
+        for (auto i = 0; i < num_vertices; ++i) {
+            permutation_inv[permutation[i]] = i;
+        }
     }
 }
 
@@ -167,4 +328,4 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 }  // namespace rcm
 }  // namespace reference
 }  // namespace kernels
-}  // namespace gko
+}  // namespace gko
\ No newline at end of file

From 408af3915f0ebbecf1eda7b71551fba849a483fa Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Wed, 15 Apr 2020 13:49:13 +0200
Subject: [PATCH 0038/1008] Fix missing rcm kernel param

---
 cuda/reorder/rcm_kernels.cu     | 6 +++---
 hip/reorder/rcm_kernels.hip.cpp | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/cuda/reorder/rcm_kernels.cu b/cuda/reorder/rcm_kernels.cu
index c9bf37cd955..eecc192f2bd 100644
--- a/cuda/reorder/rcm_kernels.cu
+++ b/cuda/reorder/rcm_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2019, the Ginkgo authors
+Copyright (c) 2017-2020, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -73,8 +73,8 @@ void get_permutation(
     std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
     std::shared_ptr<Array<IndexType>> node_degrees,
     std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,
-    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat)
-    GKO_NOT_IMPLEMENTED;
+    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat,
+    const gko::reorder::starting_strategy strategy) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
diff --git a/hip/reorder/rcm_kernels.hip.cpp b/hip/reorder/rcm_kernels.hip.cpp
index 319c13f8559..3b475220465 100644
--- a/hip/reorder/rcm_kernels.hip.cpp
+++ b/hip/reorder/rcm_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2019, the Ginkgo authors
+Copyright (c) 2017-2020, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -73,8 +73,8 @@ void get_permutation(
     std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
     std::shared_ptr<Array<IndexType>> node_degrees,
     std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,
-    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat)
-    GKO_NOT_IMPLEMENTED;
+    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat,
+    const gko::reorder::starting_strategy strategy) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);

From ebd8d91dfff978c272129a8a3d37358fb2328e6a Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Sat, 18 Apr 2020 15:15:31 +0200
Subject: [PATCH 0039/1008] Changes by review

---
 core/reorder/rcm.cpp                | 11 ++--
 core/reorder/rcm_kernels.hpp        | 33 +++++-----
 cuda/reorder/rcm_kernels.cu         | 15 +++--
 hip/reorder/rcm_kernels.hip.cpp     | 15 +++--
 include/ginkgo/core/reorder/rcm.hpp |  4 +-
 reference/reorder/rcm_kernels.cpp   | 93 ++++++++++++-----------------
 6 files changed, 78 insertions(+), 93 deletions(-)

diff --git a/core/reorder/rcm.cpp b/core/reorder/rcm.cpp
index 7671acb5ce1..bae1aeb52de 100644
--- a/core/reorder/rcm.cpp
+++ b/core/reorder/rcm.cpp
@@ -66,14 +66,17 @@ GKO_REGISTER_OPERATION(get_degree_of_nodes, rcm::get_degree_of_nodes);
 template <typename ValueType, typename IndexType>
 void Rcm<ValueType, IndexType>::generate() const
 {
-    IndexType num_rows = adjacency_matrix_->get_size()[0];
+    const IndexType num_rows = adjacency_matrix_->get_size()[0];
     const auto exec = this->get_executor();
     // RCM is only valid for symmetric matrices. Need to add an expensive check
     // for symmetricity here ?
-    exec->run(rcm::make_get_degree_of_nodes(adjacency_matrix_, node_degrees_));
+    exec->run(rcm::make_get_degree_of_nodes(adjacency_matrix_.get(),
+                                            degrees_->get_data()));
     exec->run(rcm::make_get_permutation(
-        num_rows, adjacency_matrix_, node_degrees_, permutation_,
-        inv_permutation_, parameters_.starting_strategy));
+        num_rows, adjacency_matrix_.get(), degrees_->get_data(),
+        permutation_->get_permutation(),
+        inv_permutation_.get() ? inv_permutation_->get_permutation() : nullptr,
+        parameters_.starting_strategy));
 }
 
 
diff --git a/core/reorder/rcm_kernels.hpp b/core/reorder/rcm_kernels.hpp
index e69c4348da6..79a2a32acb6 100644
--- a/core/reorder/rcm_kernels.hpp
+++ b/core/reorder/rcm_kernels.hpp
@@ -52,23 +52,22 @@ namespace gko {
 namespace kernels {
 
 
-#define GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL(ValueType, IndexType)         \
-    void get_permutation(                                                    \
-        std::shared_ptr<const DefaultExecutor> exec, size_type num_vertices, \
-        std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>>           \
-            adjacency_matrix,                                                \
-        std::shared_ptr<Array<IndexType>> node_degrees,                      \
-        std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,     \
-        std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat, \
-        const gko::reorder::starting_strategy strategy)
-
-
-#define GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL(ValueType, IndexType) \
-    void get_degree_of_nodes(                                            \
-        std::shared_ptr<const DefaultExecutor> exec,                     \
-        std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>>       \
-            adjacency_matrix,                                            \
-        std::shared_ptr<gko::Array<IndexType>> node_degrees)
+#define GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL(ValueType, IndexType)     \
+    void get_permutation(std::shared_ptr<const DefaultExecutor> exec,    \
+                         const size_type num_vertices,                   \
+                         const matrix::SparsityCsr<ValueType, IndexType> \
+                             *const adjacency_matrix,                    \
+                         const IndexType *const degrees,                 \
+                         IndexType *const permutation,                   \
+                         IndexType *const inv_permutation,               \
+                         const gko::reorder::starting_strategy strategy)
+
+
+#define GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL(ValueType, IndexType)     \
+    void get_degree_of_nodes(std::shared_ptr<const DefaultExecutor> exec,    \
+                             const matrix::SparsityCsr<ValueType, IndexType> \
+                                 *const adjacency_matrix,                    \
+                             IndexType *const degrees)
 
 #define GKO_DECLARE_ALL_AS_TEMPLATES                                  \
     template <typename ValueType, typename IndexType>                 \
diff --git a/cuda/reorder/rcm_kernels.cu b/cuda/reorder/rcm_kernels.cu
index eecc192f2bd..3cca4f33021 100644
--- a/cuda/reorder/rcm_kernels.cu
+++ b/cuda/reorder/rcm_kernels.cu
@@ -59,9 +59,9 @@ namespace rcm {
 
 template <typename ValueType, typename IndexType>
 void get_degree_of_nodes(
-    std::shared_ptr<const CudaExecutor> exec,
-    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
-    std::shared_ptr<gko::Array<IndexType>> node_degrees) GKO_NOT_IMPLEMENTED;
+    std::shared_ptr<const DefaultExecutor> exec,
+    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
+    IndexType *const degrees) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
@@ -69,11 +69,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void get_permutation(
-    std::shared_ptr<const CudaExecutor> exec, size_type num_vertices,
-    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
-    std::shared_ptr<Array<IndexType>> node_degrees,
-    std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,
-    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat,
+    std::shared_ptr<const DefaultExecutor> exec, const size_type num_vertices,
+    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
+    const IndexType *const degrees, IndexType *const permutation,
+    IndexType *const inv_permutation,
     const gko::reorder::starting_strategy strategy) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
diff --git a/hip/reorder/rcm_kernels.hip.cpp b/hip/reorder/rcm_kernels.hip.cpp
index 3b475220465..66caa7fe54d 100644
--- a/hip/reorder/rcm_kernels.hip.cpp
+++ b/hip/reorder/rcm_kernels.hip.cpp
@@ -59,9 +59,9 @@ namespace rcm {
 
 template <typename ValueType, typename IndexType>
 void get_degree_of_nodes(
-    std::shared_ptr<const HipExecutor> exec,
-    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
-    std::shared_ptr<gko::Array<IndexType>> node_degrees) GKO_NOT_IMPLEMENTED;
+    std::shared_ptr<const DefaultExecutor> exec,
+    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
+    IndexType *const degrees) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
@@ -69,11 +69,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void get_permutation(
-    std::shared_ptr<const HipExecutor> exec, size_type num_vertices,
-    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
-    std::shared_ptr<Array<IndexType>> node_degrees,
-    std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,
-    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat,
+    std::shared_ptr<const DefaultExecutor> exec, const size_type num_vertices,
+    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
+    const IndexType *const degrees, IndexType *const permutation,
+    IndexType *const inv_permutation,
     const gko::reorder::starting_strategy strategy) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
diff --git a/include/ginkgo/core/reorder/rcm.hpp b/include/ginkgo/core/reorder/rcm.hpp
index 71fd824e988..9519d9107f8 100644
--- a/include/ginkgo/core/reorder/rcm.hpp
+++ b/include/ginkgo/core/reorder/rcm.hpp
@@ -171,7 +171,7 @@ class Rcm : public EnablePolymorphicObject<Rcm<ValueType, IndexType>,
             adjacency_matrix_ = tmp->to_adjacency_matrix();
         }
         auto const dim = adjacency_matrix_->get_size();
-        node_degrees_ = std::shared_ptr<Array<IndexType>>(
+        degrees_ = std::unique_ptr<Array<IndexType>>(
             new Array<IndexType>(exec, dim[0]));
         permutation_ = PermutationMatrix::create(exec, dim);
         if (parameters_.construct_inverse_permutation) {
@@ -184,7 +184,7 @@ class Rcm : public EnablePolymorphicObject<Rcm<ValueType, IndexType>,
 private:
     std::shared_ptr<SparsityMatrix> adjacency_matrix_{};
     std::shared_ptr<PermutationMatrix> permutation_{};
-    std::shared_ptr<Array<IndexType>> node_degrees_{};
+    std::unique_ptr<Array<IndexType>> degrees_{};
     std::shared_ptr<PermutationMatrix> inv_permutation_{};
 };
 
diff --git a/reference/reorder/rcm_kernels.cpp b/reference/reorder/rcm_kernels.cpp
index aa0a6ac9987..022bc5b32fc 100644
--- a/reference/reorder/rcm_kernels.cpp
+++ b/reference/reorder/rcm_kernels.cpp
@@ -66,15 +66,14 @@ namespace rcm {
 template <typename ValueType, typename IndexType>
 void get_degree_of_nodes(
     std::shared_ptr<const ReferenceExecutor> exec,
-    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
-    std::shared_ptr<gko::Array<IndexType>> node_degrees)
+    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
+    IndexType *const degrees)
 {
     auto num_rows = adjacency_matrix->get_size()[0];
-    auto adj_ptrs = adjacency_matrix->get_row_ptrs();
-    auto node_deg = node_degrees->get_data();
+    auto adj_ptrs = adjacency_matrix->get_const_row_ptrs();
 
     for (auto i = 0; i < num_rows; ++i) {
-        node_deg[i] = adj_ptrs[i + 1] - adj_ptrs[i];
+        degrees[i] = adj_ptrs[i + 1] - adj_ptrs[i];
     }
 }
 
@@ -89,18 +88,17 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
  */
 template <typename IndexType, typename ValueType>
 std::pair<IndexType, size_type> rls_contender_and_height(
-    std::shared_ptr<const ReferenceExecutor> exec, IndexType root,
-    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
-    std::shared_ptr<Array<IndexType>> node_degrees, size_type num_vertices)
+    std::shared_ptr<const ReferenceExecutor> exec, const size_type num_vertices,
+    IndexType root,
+    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
+    const IndexType *const degrees)
 {
-    auto adj_ptrs = adjacency_matrix->get_row_ptrs();
-    auto adj_idxs = adjacency_matrix->get_col_idxs();
-    auto node_degrees_p = node_degrees->get_data();
+    auto adj_ptrs = adjacency_matrix->get_const_row_ptrs();
+    auto adj_idxs = adjacency_matrix->get_const_col_idxs();
 
     // This could actually be allocated in the calling scope, then reused.
-    auto visited_local =
-        std::unique_ptr<Array<bool>>(new Array<bool>(exec, num_vertices));
-    auto visited_local_p = visited_local->get_data();
+    Array<bool> visited_local(exec, num_vertices);
+    auto visited_local_p = visited_local.get_data();
     for (auto i = 0; i < num_vertices; ++i) {
         visited_local_p[i] = false;
     }
@@ -154,13 +152,14 @@ std::pair<IndexType, size_type> rls_contender_and_height(
     }
 
     // Choose the contender.
+    // It's the node of minimum degree in the last level.
     auto rls_size = rls.size();
     auto min_degree = std::numeric_limits<IndexType>::max();
     auto contender = rls_size - last_level_size;
     for (auto i = rls_size - last_level_size; i < rls_size; ++i) {
-        if (node_degrees_p[rls[i]] < min_degree) {
+        if (degrees[rls[i]] < min_degree) {
             contender = i;
-            min_degree = node_degrees_p[rls[i]];
+            min_degree = degrees[rls[i]];
         }
     }
 
@@ -169,11 +168,10 @@ std::pair<IndexType, size_type> rls_contender_and_height(
 
 template <typename IndexType, typename ValueType>
 IndexType find_starting_node(
-    std::shared_ptr<const ReferenceExecutor> exec,
-    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
-    std::shared_ptr<Array<IndexType>> node_degrees,
-    std::shared_ptr<Array<bool>> visited, size_type num_vertices,
-    gko::reorder::starting_strategy strategy)
+    std::shared_ptr<const ReferenceExecutor> exec, const size_type num_vertices,
+    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
+    const IndexType *const degrees, const bool *visited,
+    const gko::reorder::starting_strategy strategy)
 {
     using strategies = gko::reorder::starting_strategy;
 
@@ -181,17 +179,14 @@ IndexType find_starting_node(
     GKO_ASSERT(strategy == strategies::minimum_degree ||
                startegy == strategies::pseudo_peripheral);
 
-    auto node_degrees_p = node_degrees->get_data();
-    auto visited_p = visited->get_data();
-
     // There must always be at least one unvisited node left.
     auto index_min_node = 0;
     auto min_node_degree = std::numeric_limits<IndexType>::max();
     for (auto i = 0; i < num_vertices; ++i) {
-        if (!visited_p[i]) {
-            if (node_degrees_p[i] < min_node_degree) {
+        if (!visited[i]) {
+            if (degrees[i] < min_node_degree) {
                 index_min_node = i;
-                min_node_degree = node_degrees_p[i];
+                min_node_degree = degrees[i];
             }
         }
     }
@@ -204,7 +199,7 @@ IndexType find_starting_node(
     // Find a pseudo-peripheral node, starting from an node of minimum degree.
     auto current = index_min_node;
 
-    // Isolated nodes are by definition perioheral.
+    // Isolated nodes are by definition peripheral.
     if (min_node_degree == 0) {
         return index_min_node;
     }
@@ -213,12 +208,12 @@ IndexType find_starting_node(
     // http://heath.cs.illinois.edu/courses/cs598mh/george_liu.pdf, p. 70
     // onwards.
     auto contender_and_height = rls_contender_and_height<IndexType, ValueType>(
-        exec, current, adjacency_matrix, node_degrees, num_vertices);
+        exec, num_vertices, current, adjacency_matrix, degrees);
     while (true) {
         auto contender_and_height_contender =
             rls_contender_and_height<IndexType, ValueType>(
-                exec, contender_and_height.first, adjacency_matrix,
-                node_degrees, num_vertices);
+                exec, num_vertices, contender_and_height.first,
+                adjacency_matrix, degrees);
 
         if (contender_and_height_contender.second >
             contender_and_height.second) {
@@ -236,34 +231,25 @@ IndexType find_starting_node(
  */
 template <typename ValueType, typename IndexType>
 void get_permutation(
-    std::shared_ptr<const ReferenceExecutor> exec, size_type num_vertices,
-    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
-    std::shared_ptr<Array<IndexType>> node_degrees,
-    std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,
-    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat,
+    std::shared_ptr<const DefaultExecutor> exec, const size_type num_vertices,
+    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
+    const IndexType *const degrees, IndexType *const permutation,
+    IndexType *const inv_permutation,
     const gko::reorder::starting_strategy strategy)
 {
     IndexType num_vtxs = static_cast<IndexType>(num_vertices);
-    auto adj_ptrs = adjacency_matrix->get_row_ptrs();
-    auto adj_idxs = adjacency_matrix->get_col_idxs();
-
-    auto permutation = permutation_mat->get_permutation();
-    auto permutation_inv = inv_permutation_mat.get()
-                               ? inv_permutation_mat->get_permutation()
-                               : nullptr;
-
-    IndexType *degrees = node_degrees->get_data();
+    auto adj_ptrs = adjacency_matrix->get_const_row_ptrs();
+    auto adj_idxs = adjacency_matrix->get_const_col_idxs();
 
     // Storing vertices left to proceess.
-    auto linear_queue = std::unique_ptr<Array<IndexType>>(
-        new Array<IndexType>(exec, num_vertices));
-    auto linear_queue_p = linear_queue->get_data();
+    Array<IndexType> linear_queue(exec, num_vertices);
+    auto linear_queue_p = linear_queue.get_data();
     auto head_offset = 0;
     auto tail_offset = 0;
 
     // Storing which vertices have already been visited.
-    auto visited = std::make_shared<Array<bool>>(exec, num_vertices);
-    auto visited_p = visited->get_data();
+    Array<bool> visited(exec, num_vertices);
+    auto visited_p = visited.get_data();
     for (auto i = 0; i < num_vertices; ++i) {
         visited_p[i] = false;
     }
@@ -275,7 +261,7 @@ void get_permutation(
             // Choose a new starting vertex, new componenet needs to be
             // discovered.
             next_vertex = find_starting_node<IndexType, ValueType>(
-                exec, adjacency_matrix, node_degrees, visited, num_vertices,
+                exec, num_vertices, adjacency_matrix, degrees, visited_p,
                 strategy);
             visited_p[next_vertex] = true;
         } else {
@@ -304,7 +290,6 @@ void get_permutation(
             }
         }
 
-        auto count_added = head_offset - prev_head_offset;
         std::sort(linear_queue_p + prev_head_offset,
                   linear_queue_p + head_offset,
                   [&degrees](IndexType i, IndexType j) {
@@ -314,9 +299,9 @@ void get_permutation(
         permutation[perm_index] = next_vertex;
     }
 
-    if (permutation_inv) {
+    if (inv_permutation) {
         for (auto i = 0; i < num_vertices; ++i) {
-            permutation_inv[permutation[i]] = i;
+            inv_permutation[permutation[i]] = i;
         }
     }
 }

From 6f651edede577deffdced315f90a119acbcc8aac Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Sat, 18 Apr 2020 15:49:58 +0200
Subject: [PATCH 0040/1008] Change vector to Array as requested

---
 reference/reorder/rcm_kernels.cpp | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/reference/reorder/rcm_kernels.cpp b/reference/reorder/rcm_kernels.cpp
index 022bc5b32fc..6749454ff9c 100644
--- a/reference/reorder/rcm_kernels.cpp
+++ b/reference/reorder/rcm_kernels.cpp
@@ -104,9 +104,10 @@ std::pair<IndexType, size_type> rls_contender_and_height(
     }
 
     // This stores the level structure, starting with the root node.
-    std::vector<IndexType> rls;
-    rls.reserve(num_vertices);
-    rls.push_back(root);
+    Array<IndexType> rls(exec, num_vertices);
+    auto rls_p = rls.get_data();
+    rls_p[0] = root;
+    auto rls_offset = 1;
     visited_local_p[root] = true;
 
     auto rls_index = 0;
@@ -119,25 +120,28 @@ std::pair<IndexType, size_type> rls_contender_and_height(
     // The last levels size is required to compute the contender.
     auto last_level_size = 0;
 
-    while (rls_index < rls.size()) {
-        auto parent = rls[rls_index];
+    // While there are still nodes who's neighbours haven't been inspected.
+    while (rls_index < rls_offset) {
+        auto parent = rls_p[rls_index];
         --current_level_countdown;
 
-
-        // Iterate through the parents neighbors.
+        // Iterate through parents neighbors.
         auto row_start = adj_ptrs[parent];
         auto row_end = adj_ptrs[parent + 1];
         for (auto neighbor_idx = row_start; neighbor_idx < row_end;
              ++neighbor_idx) {
             auto neighbor = adj_idxs[neighbor_idx];
 
+            // If this is a new node, add it to the rls and mark it as visited.
             if (!visited_local_p[neighbor]) {
                 visited_local_p[neighbor] = true;
                 ++next_level_countup;
-                rls.push_back(neighbor);
+                rls_p[rls_offset] = neighbor;
+                ++rls_offset;
             }
         }
 
+        // Machinery for computing the last levels length.
         if (current_level_countdown == 0) {
             if (next_level_countup > 0) {
                 last_level_size = next_level_countup;
@@ -153,17 +157,17 @@ std::pair<IndexType, size_type> rls_contender_and_height(
 
     // Choose the contender.
     // It's the node of minimum degree in the last level.
-    auto rls_size = rls.size();
+    auto rls_size = rls_offset;
     auto min_degree = std::numeric_limits<IndexType>::max();
     auto contender = rls_size - last_level_size;
     for (auto i = rls_size - last_level_size; i < rls_size; ++i) {
-        if (degrees[rls[i]] < min_degree) {
+        if (degrees[rls_p[i]] < min_degree) {
             contender = i;
-            min_degree = degrees[rls[i]];
+            min_degree = degrees[rls_p[i]];
         }
     }
 
-    return std::pair<IndexType, size_type>(rls[contender], height);
+    return std::pair<IndexType, size_type>(rls_p[contender], height);
 }
 
 template <typename IndexType, typename ValueType>

From 0d412b68a5fd913ca6ee5cd179d5d15f07a40356 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Wed, 3 Jun 2020 21:28:30 +0200
Subject: [PATCH 0041/1008] Fix name clashing

---
 core/reorder/rcm.cpp                | 2 +-
 include/ginkgo/core/reorder/rcm.hpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/reorder/rcm.cpp b/core/reorder/rcm.cpp
index bae1aeb52de..cde67e85faf 100644
--- a/core/reorder/rcm.cpp
+++ b/core/reorder/rcm.cpp
@@ -76,7 +76,7 @@ void Rcm<ValueType, IndexType>::generate() const
         num_rows, adjacency_matrix_.get(), degrees_->get_data(),
         permutation_->get_permutation(),
         inv_permutation_.get() ? inv_permutation_->get_permutation() : nullptr,
-        parameters_.starting_strategy));
+        parameters_.strategy));
 }
 
 
diff --git a/include/ginkgo/core/reorder/rcm.hpp b/include/ginkgo/core/reorder/rcm.hpp
index 9519d9107f8..c7aeb929742 100644
--- a/include/ginkgo/core/reorder/rcm.hpp
+++ b/include/ginkgo/core/reorder/rcm.hpp
@@ -136,7 +136,7 @@ class Rcm : public EnablePolymorphicObject<Rcm<ValueType, IndexType>,
          * vertex.
          */
         starting_strategy GKO_FACTORY_PARAMETER(
-            starting_strategy, starting_strategy::pseudo_peripheral);
+            strategy, starting_strategy::pseudo_peripheral);
     };
     GKO_ENABLE_REORDERING_BASE_FACTORY(Rcm, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);

From 51b3d2933b419b1882311ad30dc19d042ff139a0 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Sun, 7 Jun 2020 14:48:44 +0200
Subject: [PATCH 0042/1008] Remove rcm ValueType

Removes the unneccessary ValueType in the rcm algorithm.
Instead, it operates on raw non-owning pointers.
---
 core/device_hooks/common_kernels.inc.cpp |  14 ++--
 core/reorder/rcm.cpp                     |   7 +-
 core/reorder/rcm_kernels.hpp             |  37 ++++-----
 cuda/reorder/rcm_kernels.cu              |  22 +++--
 hip/reorder/rcm_kernels.hip.cpp          |  22 +++--
 omp/reorder/rcm_kernels.cpp              |  41 ++++-----
 reference/reorder/rcm_kernels.cpp        | 101 +++++++++++------------
 7 files changed, 113 insertions(+), 131 deletions(-)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index b2cdf8e7a59..a2285615224 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -1054,18 +1054,16 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 namespace rcm {
 
 
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL(ValueType, IndexType)
+template <typename IndexType>
+GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL(IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL(ValueType, IndexType)
+template <typename IndexType>
+GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL(IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 
 
 }  // namespace rcm
diff --git a/core/reorder/rcm.cpp b/core/reorder/rcm.cpp
index cde67e85faf..584b6bfe1f6 100644
--- a/core/reorder/rcm.cpp
+++ b/core/reorder/rcm.cpp
@@ -68,13 +68,14 @@ void Rcm<ValueType, IndexType>::generate() const
 {
     const IndexType num_rows = adjacency_matrix_->get_size()[0];
     const auto exec = this->get_executor();
+    const auto mtx = adjacency_matrix_.get();
     // RCM is only valid for symmetric matrices. Need to add an expensive check
     // for symmetricity here ?
-    exec->run(rcm::make_get_degree_of_nodes(adjacency_matrix_.get(),
+    exec->run(rcm::make_get_degree_of_nodes(num_rows, mtx->get_const_row_ptrs(),
                                             degrees_->get_data()));
     exec->run(rcm::make_get_permutation(
-        num_rows, adjacency_matrix_.get(), degrees_->get_data(),
-        permutation_->get_permutation(),
+        num_rows, mtx->get_const_row_ptrs(), mtx->get_const_col_idxs(),
+        degrees_->get_const_data(), permutation_->get_permutation(),
         inv_permutation_.get() ? inv_permutation_->get_permutation() : nullptr,
         parameters_.strategy));
 }
diff --git a/core/reorder/rcm_kernels.hpp b/core/reorder/rcm_kernels.hpp
index 79a2a32acb6..e0eda389b8f 100644
--- a/core/reorder/rcm_kernels.hpp
+++ b/core/reorder/rcm_kernels.hpp
@@ -52,28 +52,25 @@ namespace gko {
 namespace kernels {
 
 
-#define GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL(ValueType, IndexType)     \
-    void get_permutation(std::shared_ptr<const DefaultExecutor> exec,    \
-                         const size_type num_vertices,                   \
-                         const matrix::SparsityCsr<ValueType, IndexType> \
-                             *const adjacency_matrix,                    \
-                         const IndexType *const degrees,                 \
-                         IndexType *const permutation,                   \
-                         IndexType *const inv_permutation,               \
-                         const gko::reorder::starting_strategy strategy)
-
-
-#define GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL(ValueType, IndexType)     \
-    void get_degree_of_nodes(std::shared_ptr<const DefaultExecutor> exec,    \
-                             const matrix::SparsityCsr<ValueType, IndexType> \
-                                 *const adjacency_matrix,                    \
+#define GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL(IndexType)                \
+    void get_permutation(                                                \
+        std::shared_ptr<const DefaultExecutor> exec,                     \
+        const size_type num_vertices, const IndexType *const row_ptrs,   \
+        const IndexType *const col_idxs, const IndexType *const degrees, \
+        IndexType *const permutation, IndexType *const inv_permutation,  \
+        const gko::reorder::starting_strategy strategy)
+
+#define GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL(IndexType)             \
+    void get_degree_of_nodes(std::shared_ptr<const DefaultExecutor> exec, \
+                             const size_type num_vertices,                \
+                             const IndexType *const row_ptrs,             \
                              IndexType *const degrees)
 
-#define GKO_DECLARE_ALL_AS_TEMPLATES                                  \
-    template <typename ValueType, typename IndexType>                 \
-    GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL(ValueType, IndexType); \
-    template <typename ValueType, typename IndexType>                 \
-    GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL(ValueType, IndexType)
+#define GKO_DECLARE_ALL_AS_TEMPLATES                       \
+    template <typename IndexType>                          \
+    GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL(IndexType); \
+    template <typename IndexType>                          \
+    GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL(IndexType)
 
 
 namespace omp {
diff --git a/cuda/reorder/rcm_kernels.cu b/cuda/reorder/rcm_kernels.cu
index 3cca4f33021..3ccc267b755 100644
--- a/cuda/reorder/rcm_kernels.cu
+++ b/cuda/reorder/rcm_kernels.cu
@@ -57,26 +57,24 @@ namespace cuda {
 namespace rcm {
 
 
-template <typename ValueType, typename IndexType>
-void get_degree_of_nodes(
-    std::shared_ptr<const DefaultExecutor> exec,
-    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
-    IndexType *const degrees) GKO_NOT_IMPLEMENTED;
+template <typename IndexType>
+void get_degree_of_nodes(std::shared_ptr<const CudaExecutor> exec,
+                         const size_type num_vertices,
+                         const IndexType *const row_ptrs,
+                         IndexType *const degrees) GKO_NOT_IMPLEMENTED;
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
+template <typename IndexType>
 void get_permutation(
-    std::shared_ptr<const DefaultExecutor> exec, const size_type num_vertices,
-    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
+    std::shared_ptr<const CudaExecutor> exec, const size_type num_vertices,
+    const IndexType *const row_ptrs, const IndexType *const col_idxs,
     const IndexType *const degrees, IndexType *const permutation,
     IndexType *const inv_permutation,
     const gko::reorder::starting_strategy strategy) GKO_NOT_IMPLEMENTED;
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
 
 
 }  // namespace rcm
diff --git a/hip/reorder/rcm_kernels.hip.cpp b/hip/reorder/rcm_kernels.hip.cpp
index 66caa7fe54d..385ae8b9890 100644
--- a/hip/reorder/rcm_kernels.hip.cpp
+++ b/hip/reorder/rcm_kernels.hip.cpp
@@ -57,26 +57,24 @@ namespace hip {
 namespace rcm {
 
 
-template <typename ValueType, typename IndexType>
-void get_degree_of_nodes(
-    std::shared_ptr<const DefaultExecutor> exec,
-    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
-    IndexType *const degrees) GKO_NOT_IMPLEMENTED;
+template <typename IndexType>
+void get_degree_of_nodes(std::shared_ptr<const Hipxecutor> exec,
+                         const size_type num_vertices,
+                         const IndexType *const row_ptrs,
+                         IndexType *const degrees) GKO_NOT_IMPLEMENTED;
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
+template <typename IndexType>
 void get_permutation(
-    std::shared_ptr<const DefaultExecutor> exec, const size_type num_vertices,
-    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
+    std::shared_ptr<const HipExecutor> exec, const size_type num_vertices,
+    const IndexType *const row_ptrs, const IndexType *const col_idxs,
     const IndexType *const degrees, IndexType *const permutation,
     IndexType *const inv_permutation,
     const gko::reorder::starting_strategy strategy) GKO_NOT_IMPLEMENTED;
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
 
 
 }  // namespace rcm
diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index d8609868f01..c90096be567 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -62,24 +62,19 @@ namespace omp {
 namespace rcm {
 
 
-template <typename ValueType, typename IndexType>
-void get_degree_of_nodes(
-    std::shared_ptr<const OmpExecutor> exec,
-    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
-    std::shared_ptr<gko::Array<IndexType>> node_degrees)
+template <typename IndexType>
+void get_degree_of_nodes(std::shared_ptr<const OmpExecutor> exec,
+                         const size_type num_vertices,
+                         const IndexType *const row_ptrs,
+                         IndexType *const degrees)
 {
-    auto num_rows = adjacency_matrix->get_size()[0];
-    auto adj_ptrs = adjacency_matrix->get_row_ptrs();
-    auto node_deg = node_degrees->get_data();
-
 #pragma omp parallel for
-    for (auto i = 0; i < num_rows; ++i) {
-        node_deg[i] = adj_ptrs[i + 1] - adj_ptrs[i];
+    for (auto i = 0; i < num_vertices; ++i) {
+        degrees[i] = row_ptrs[i + 1] - row_ptrs[i];
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 
 
 template <typename IndexType>
@@ -92,14 +87,15 @@ IndexType find_index(std::vector<std::pair<IndexType, IndexType>> &a,
 }
 
 
-template <typename ValueType, typename IndexType>
-void get_permutation(
-    std::shared_ptr<const OmpExecutor> exec, size_type num_vertices,
-    std::shared_ptr<matrix::SparsityCsr<ValueType, IndexType>> adjacency_matrix,
-    std::shared_ptr<Array<IndexType>> node_degrees,
-    std::shared_ptr<matrix::Permutation<IndexType>> permutation_mat,
-    std::shared_ptr<matrix::Permutation<IndexType>> inv_permutation_mat,
-    const gko::reorder::starting_strategy strategy)
+template <typename IndexType>
+void get_permutation(std::shared_ptr<const OmpExecutor> exec,
+                     const size_type num_vertices,
+                     const IndexType *const row_ptrs,
+                     const IndexType *const col_idxs,
+                     const IndexType *const degrees,
+                     IndexType *const permutation,
+                     IndexType *const inv_permutation,
+                     const gko::reorder::starting_strategy strategy)
 {
     IndexType num_vtxs = static_cast<IndexType>(num_vertices);
     auto adj_ptrs = adjacency_matrix->get_row_ptrs();
@@ -162,8 +158,7 @@ void get_permutation(
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
 
 
 }  // namespace rcm
diff --git a/reference/reorder/rcm_kernels.cpp b/reference/reorder/rcm_kernels.cpp
index 6749454ff9c..110f7d01908 100644
--- a/reference/reorder/rcm_kernels.cpp
+++ b/reference/reorder/rcm_kernels.cpp
@@ -63,22 +63,18 @@ namespace reference {
 namespace rcm {
 
 
-template <typename ValueType, typename IndexType>
-void get_degree_of_nodes(
-    std::shared_ptr<const ReferenceExecutor> exec,
-    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
-    IndexType *const degrees)
+template <typename IndexType>
+void get_degree_of_nodes(std::shared_ptr<const ReferenceExecutor> exec,
+                         const size_type num_vertices,
+                         const IndexType *const row_ptrs,
+                         IndexType *const degrees)
 {
-    auto num_rows = adjacency_matrix->get_size()[0];
-    auto adj_ptrs = adjacency_matrix->get_const_row_ptrs();
-
-    for (auto i = 0; i < num_rows; ++i) {
-        degrees[i] = adj_ptrs[i + 1] - adj_ptrs[i];
+    for (auto i = 0; i < num_vertices; ++i) {
+        degrees[i] = row_ptrs[i + 1] - row_ptrs[i];
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 
 
 /**
@@ -86,16 +82,12 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
  * structure along with the index of the node of minimal degree in its last
  * level.
  */
-template <typename IndexType, typename ValueType>
+template <typename IndexType>
 std::pair<IndexType, size_type> rls_contender_and_height(
     std::shared_ptr<const ReferenceExecutor> exec, const size_type num_vertices,
-    IndexType root,
-    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
-    const IndexType *const degrees)
+    const IndexType root, const IndexType *const row_ptrs,
+    const IndexType *const col_idxs, const IndexType *const degrees)
 {
-    auto adj_ptrs = adjacency_matrix->get_const_row_ptrs();
-    auto adj_idxs = adjacency_matrix->get_const_col_idxs();
-
     // This could actually be allocated in the calling scope, then reused.
     Array<bool> visited_local(exec, num_vertices);
     auto visited_local_p = visited_local.get_data();
@@ -113,6 +105,8 @@ std::pair<IndexType, size_type> rls_contender_and_height(
     auto rls_index = 0;
 
     // Used to compute the height.
+    // Always count up the count of the next level,
+    // while counting down the current levels count.
     auto height = 0;
     auto current_level_countdown = 1;
     auto next_level_countup = 0;
@@ -126,11 +120,11 @@ std::pair<IndexType, size_type> rls_contender_and_height(
         --current_level_countdown;
 
         // Iterate through parents neighbors.
-        auto row_start = adj_ptrs[parent];
-        auto row_end = adj_ptrs[parent + 1];
+        auto row_start = row_ptrs[parent];
+        auto row_end = row_ptrs[parent + 1];
         for (auto neighbor_idx = row_start; neighbor_idx < row_end;
              ++neighbor_idx) {
-            auto neighbor = adj_idxs[neighbor_idx];
+            auto neighbor = col_idxs[neighbor_idx];
 
             // If this is a new node, add it to the rls and mark it as visited.
             if (!visited_local_p[neighbor]) {
@@ -170,18 +164,20 @@ std::pair<IndexType, size_type> rls_contender_and_height(
     return std::pair<IndexType, size_type>(rls_p[contender], height);
 }
 
-template <typename IndexType, typename ValueType>
-IndexType find_starting_node(
-    std::shared_ptr<const ReferenceExecutor> exec, const size_type num_vertices,
-    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
-    const IndexType *const degrees, const bool *visited,
-    const gko::reorder::starting_strategy strategy)
+template <typename IndexType>
+IndexType find_starting_node(std::shared_ptr<const ReferenceExecutor> exec,
+                             const size_type num_vertices,
+                             const IndexType *const row_ptrs,
+                             const IndexType *const col_idxs,
+                             const IndexType *const degrees,
+                             const bool *const visited,
+                             const gko::reorder::starting_strategy strategy)
 {
     using strategies = gko::reorder::starting_strategy;
 
     // Only those strategies are supported here yet, assert this.
     GKO_ASSERT(strategy == strategies::minimum_degree ||
-               startegy == strategies::pseudo_peripheral);
+               strategy == strategies::pseudo_peripheral);
 
     // There must always be at least one unvisited node left.
     auto index_min_node = 0;
@@ -211,13 +207,13 @@ IndexType find_starting_node(
     // This algorithm is e. g. presented in
     // http://heath.cs.illinois.edu/courses/cs598mh/george_liu.pdf, p. 70
     // onwards.
-    auto contender_and_height = rls_contender_and_height<IndexType, ValueType>(
-        exec, num_vertices, current, adjacency_matrix, degrees);
+    auto contender_and_height = rls_contender_and_height<IndexType>(
+        exec, num_vertices, current, row_ptrs, col_idxs, degrees);
     while (true) {
         auto contender_and_height_contender =
-            rls_contender_and_height<IndexType, ValueType>(
-                exec, num_vertices, contender_and_height.first,
-                adjacency_matrix, degrees);
+            rls_contender_and_height<IndexType>(exec, num_vertices,
+                                                contender_and_height.first,
+                                                row_ptrs, col_idxs, degrees);
 
         if (contender_and_height_contender.second >
             contender_and_height.second) {
@@ -233,18 +229,16 @@ IndexType find_starting_node(
 /**
  * Comutes a rcm reording using a naive sequential algorithm.
  */
-template <typename ValueType, typename IndexType>
-void get_permutation(
-    std::shared_ptr<const DefaultExecutor> exec, const size_type num_vertices,
-    const matrix::SparsityCsr<ValueType, IndexType> *const adjacency_matrix,
-    const IndexType *const degrees, IndexType *const permutation,
-    IndexType *const inv_permutation,
-    const gko::reorder::starting_strategy strategy)
+template <typename IndexType>
+void get_permutation(std::shared_ptr<const ReferenceExecutor> exec,
+                     const size_type num_vertices,
+                     const IndexType *const row_ptrs,
+                     const IndexType *const col_idxs,
+                     const IndexType *const degrees,
+                     IndexType *const permutation,
+                     IndexType *const inv_permutation,
+                     const gko::reorder::starting_strategy strategy)
 {
-    IndexType num_vtxs = static_cast<IndexType>(num_vertices);
-    auto adj_ptrs = adjacency_matrix->get_const_row_ptrs();
-    auto adj_idxs = adjacency_matrix->get_const_col_idxs();
-
     // Storing vertices left to proceess.
     Array<IndexType> linear_queue(exec, num_vertices);
     auto linear_queue_p = linear_queue.get_data();
@@ -262,10 +256,10 @@ void get_permutation(
         // Get the next vertex to process.
         auto next_vertex = 0;
         if (head_offset <= tail_offset) {
-            // Choose a new starting vertex, new componenet needs to be
+            // Choose a new starting vertex, new component needs to be
             // discovered.
-            next_vertex = find_starting_node<IndexType, ValueType>(
-                exec, num_vertices, adjacency_matrix, degrees, visited_p,
+            next_vertex = find_starting_node<IndexType>(
+                exec, num_vertices, row_ptrs, col_idxs, degrees, visited_p,
                 strategy);
             visited_p[next_vertex] = true;
         } else {
@@ -279,11 +273,11 @@ void get_permutation(
         auto prev_head_offset = head_offset;
 
         // Is this code correct for getting next_vertexs neighbors?
-        auto row_start = adj_ptrs[next_vertex];
-        auto row_end = adj_ptrs[next_vertex + 1];
+        auto row_start = row_ptrs[next_vertex];
+        auto row_end = row_ptrs[next_vertex + 1];
         for (auto neighbor_idx = row_start; neighbor_idx < row_end;
              ++neighbor_idx) {
-            auto neighbor = adj_idxs[neighbor_idx];
+            auto neighbor = col_idxs[neighbor_idx];
 
             // If the neighbour hasn't been visited yet, add it to the queue and
             // mark it.
@@ -294,12 +288,14 @@ void get_permutation(
             }
         }
 
+        // Sort all just-added neighbours by degree.
         std::sort(linear_queue_p + prev_head_offset,
                   linear_queue_p + head_offset,
                   [&degrees](IndexType i, IndexType j) {
                       return degrees[i] < degrees[j];
                   });
 
+        // Write out the processed vertex.
         permutation[perm_index] = next_vertex;
     }
 
@@ -310,8 +306,7 @@ void get_permutation(
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
 
 
 }  // namespace rcm

From 2702fe4b8f99f5fed417768aca0e1629e4dafba6 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Sun, 7 Jun 2020 14:51:03 +0200
Subject: [PATCH 0043/1008] Remove rcm omp kernel

Removes the old omp rcm kernel,
replacing it with a sketch of the intended layout of the new algorithm.
---
 omp/reorder/rcm_kernels.cpp | 76 +++++++++----------------------------
 1 file changed, 17 insertions(+), 59 deletions(-)

diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index c90096be567..bd7bca01717 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -97,65 +97,23 @@ void get_permutation(std::shared_ptr<const OmpExecutor> exec,
                      IndexType *const inv_permutation,
                      const gko::reorder::starting_strategy strategy)
 {
-    IndexType num_vtxs = static_cast<IndexType>(num_vertices);
-    auto adj_ptrs = adjacency_matrix->get_row_ptrs();
-    auto adj_idxs = adjacency_matrix->get_col_idxs();
-    auto node_deg = node_degrees->get_data();
-    auto permutation_arr = permutation_mat->get_permutation();
-    auto inv_permutation_arr = inv_permutation_mat->get_permutation();
-
-    std::queue<IndexType> q;
-    std::vector<IndexType> r;
-    std::vector<std::pair<IndexType, IndexType>> not_visited;
-
-    for (auto i = 0; i < num_vtxs; ++i) {
-        not_visited.push_back(std::make_pair(i, node_deg[i]));
-    }
-
-    while (not_visited.size()) {
-        // choose this better.
-        IndexType min_node_index = 0;
-
-        for (auto i = 0; i < not_visited.size(); i++) {
-            if (not_visited[i].second < not_visited[min_node_index].second) {
-                min_node_index = i;
-            }
-        }
-        q.push(not_visited[min_node_index].first);
-
-        not_visited.erase(
-            not_visited.begin() +
-            find_index(not_visited, not_visited[q.front()].first));
-
-        // Simple BFS
-        while (!q.empty()) {
-            std::vector<IndexType> to_sort;
-
-            for (IndexType i = 0; i < num_vtxs; i++) {
-                if (i != q.front() && find_index(not_visited, i) != -1) {
-                    to_sort.push_back(i);
-                    not_visited.erase(not_visited.begin() +
-                                      find_index(not_visited, i));
-                }
-            }
-
-            std::sort(to_sort.begin(), to_sort.end(),
-                      [&node_deg](int i, int j) {
-                          return node_deg[i] < node_deg[j];
-                      });
-
-            for (auto i = 0; i < to_sort.size(); i++) q.push(to_sort[i]);
-
-            r.push_back(q.front());
-            q.pop();
-        }
-    }
-
-#pragma omp parallel for
-    for (auto i = 0; i < r.size(); ++i) {
-        permutation_arr[i] = r[i];
-        inv_permutation_arr[r[i]] = i;
-    }
+    // Phase 1:
+    //     Compute the level of each node using UBFS.
+    //     Find all starting vertices.
+    // Phase 2 (for each connected component,
+    // starting once respective starting vertex is confirmed):
+    //     Get the primary offset into the perm.
+    //     Compute the level borders as prefix sum of the counts.
+    // Phase 3 (for each connected component)
+    //     Start by writing the starting node.
+    //     Threads watch their level:
+    //         If the thread to the left writes a new node to your level:
+    //         Write those neighbours of the node which are in the next level to
+    //         the next level, sorted by degree.
+    //     (Can either by implemented by spinning or tasks)
+    // Once the last node in the last level is written for each component,
+    // the algorithm is finished.
+    GKO_NOT_IMPLEMENTED;
 }
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);

From d7b96974709040dd229677c382d87a39328c5178 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Sun, 7 Jun 2020 14:52:52 +0200
Subject: [PATCH 0044/1008] Add rcm reference test.

This test could be run for both starting_strategies.
It asserts a perfect ordering for full banded matrices is always found.
---
 reference/test/reorder/rcm_kernels.cpp | 42 +++++++++++++++++++++-----
 1 file changed, 35 insertions(+), 7 deletions(-)

diff --git a/reference/test/reorder/rcm_kernels.cpp b/reference/test/reorder/rcm_kernels.cpp
index 5114d12bc14..924bcc05abc 100644
--- a/reference/test/reorder/rcm_kernels.cpp
+++ b/reference/test/reorder/rcm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2019, the Ginkgo authors
+Copyright (c) 2017-2020, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -63,25 +63,41 @@ class Rcm : public ::testing::Test {
     using CsrMtx = gko::matrix::Csr<v_type, i_type>;
     using reorder_type = gko::reorder::Rcm<v_type, i_type>;
     using perm_type = gko::matrix::Permutation<i_type>;
+
+
     Rcm()
         : exec(gko::ReferenceExecutor::create()),
           // clang-format off
-          p_mtx(gko::initialize<CsrMtx>(
+          p_mtx_0(gko::initialize<CsrMtx>(
                                         {{1.0, 2.0, 0.0, -1.3, 2.1},
                                          {2.0, 5.0, 1.5, 0.0, 0.0},
                                          {0.0, 1.5, 1.5, 1.1, 0.0},
                                          {-1.3, 0.0, 1.1, 2.0, 0.0},
                                          {2.1, 0.0, 0.0, 0.0, 1.0}},
                                         exec)),
+        p_mtx_1(gko::initialize<CsrMtx>(
+                                        {{1., 0., 0., 1., 1., 1., 0., 1., 1.},
+                                         {0., 1., 1., 1., 0., 1., 1., 0., 0.},
+                                         {0., 1., 1., 0., 0., 1., 1., 0., 0.},
+                                         {1., 1., 0., 1., 1., 1., 1., 0., 1.},
+                                         {1., 0., 0., 1., 1., 1., 1., 1., 1.},
+                                         {1., 1., 1., 1., 1., 1., 1., 0., 0.},
+                                         {0., 1., 1., 1., 1., 1., 1., 0., 0.},
+                                         {1., 0., 0., 0., 1., 0., 0., 1., 1.},
+                                         {1., 0., 0., 1., 1., 0., 0., 1., 1.},},
+                                        exec)),
           // clang-format on
           rcm_factory(reorder_type::build().on(exec)),
-          reorder_op(rcm_factory->generate(p_mtx))
+          reorder_op_0(rcm_factory->generate(p_mtx_0)),
+          reorder_op_1(rcm_factory->generate(p_mtx_1))
     {}
 
     std::shared_ptr<const gko::Executor> exec;
-    std::shared_ptr<CsrMtx> p_mtx;
     std::unique_ptr<reorder_type::Factory> rcm_factory;
-    std::unique_ptr<reorder_type> reorder_op;
+    std::shared_ptr<CsrMtx> p_mtx_0;
+    std::unique_ptr<reorder_type> reorder_op_0;
+    std::shared_ptr<CsrMtx> p_mtx_1;
+    std::unique_ptr<reorder_type> reorder_op_1;
 
     bool find_duplicates(i_type val, std::size_t index, const i_type *data,
                          std::size_t length)
@@ -113,12 +129,24 @@ class Rcm : public ::testing::Test {
     }
 };
 
-
 TEST_F(Rcm, CreatesAPermutation)
 {
-    auto p = reorder_op->get_permutation();
+    auto p = reorder_op_0->get_permutation();
     assert_correct_permutation(p.get());
 }
 
+TEST_F(Rcm, PermutesPerfectFullBand)
+{
+    auto p = reorder_op_1->get_permutation()->get_const_permutation();
+    // These perms are hand-crafted for the matrix given in the ctor.
+    // They are same graph renumbering, reversed.
+    std::vector<i_type> correct_0 = {2, 1, 6, 5, 3, 4, 0, 8, 7};
+    std::vector<i_type> correct_1 = {6, 7, 2, 3, 5, 4, 8, 0, 1};
+
+    auto eq_0 = std::equal(p, p + correct_0.size(), correct_0.begin());
+    auto eq_1 = std::equal(p, p + correct_1.size(), correct_1.begin());
+    const auto incl_or = [](bool a, bool b) { return a || b; };
+    ASSERT_PRED2(incl_or, eq_0, eq_1);
+}
 
 }  // namespace

From eccda0b5eb1e5130c77357ebc6e080f904271176 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Sun, 7 Jun 2020 15:54:56 +0200
Subject: [PATCH 0045/1008] Document starting strategy

---
 include/ginkgo/core/reorder/rcm.hpp | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/include/ginkgo/core/reorder/rcm.hpp b/include/ginkgo/core/reorder/rcm.hpp
index c7aeb929742..95ab03a7500 100644
--- a/include/ginkgo/core/reorder/rcm.hpp
+++ b/include/ginkgo/core/reorder/rcm.hpp
@@ -74,7 +74,13 @@ enum class starting_strategy { minimum_degree, pseudo_peripheral };
  * the form of the Permutation matrix), which can be used to apply to reorder a
  * matrix as required.
  *
- * <Explanation starting strategy here>
+ * There are two "starting strategies" currently available: minimum degree and
+ * pseudo-peripheral. These strategies control how a starting vertex for a
+ * connected component is choosen, which is then renumbered as first vertex in
+ * the component, starting the algorithm from there.
+ * In general, the bandwidths obtained by choosing a pseudo-peripheral node are
+ * slightly smaller than those obtained from choosing a node of minimum degree.
+ * On the other hand, this strategy is much more expensive, relatively.
  *
  * @tparam ValueType  Type of the values of all matrices used in this class
  * @tparam IndexType  Type of the indices of all matrices used in this class

From ff93fe6d00c8ff4875a2a101e18b502b2e78d20f Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Sun, 7 Jun 2020 17:57:42 +0200
Subject: [PATCH 0046/1008] Actually reverse the rcm ordering

---
 reference/reorder/rcm_kernels.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reference/reorder/rcm_kernels.cpp b/reference/reorder/rcm_kernels.cpp
index 110f7d01908..7b00e801737 100644
--- a/reference/reorder/rcm_kernels.cpp
+++ b/reference/reorder/rcm_kernels.cpp
@@ -296,7 +296,7 @@ void get_permutation(std::shared_ptr<const ReferenceExecutor> exec,
                   });
 
         // Write out the processed vertex.
-        permutation[perm_index] = next_vertex;
+        permutation[num_vertices - perm_index - 1] = next_vertex;
     }
 
     if (inv_permutation) {

From 213ad48e8e191ee6766522b2d65ee419c9ced278 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Sun, 7 Jun 2020 18:00:27 +0200
Subject: [PATCH 0047/1008] Adapt the rcm tests

---
 reference/test/reorder/rcm_kernels.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/reference/test/reorder/rcm_kernels.cpp b/reference/test/reorder/rcm_kernels.cpp
index 924bcc05abc..9fec95760c4 100644
--- a/reference/test/reorder/rcm_kernels.cpp
+++ b/reference/test/reorder/rcm_kernels.cpp
@@ -84,7 +84,7 @@ class Rcm : public ::testing::Test {
                                          {1., 1., 1., 1., 1., 1., 1., 0., 0.},
                                          {0., 1., 1., 1., 1., 1., 1., 0., 0.},
                                          {1., 0., 0., 0., 1., 0., 0., 1., 1.},
-                                         {1., 0., 0., 1., 1., 0., 0., 1., 1.},},
+                                         {1., 0., 0., 1., 1., 0., 0., 1., 1.}},
                                         exec)),
           // clang-format on
           rcm_factory(reorder_type::build().on(exec)),
@@ -141,7 +141,7 @@ TEST_F(Rcm, PermutesPerfectFullBand)
     // These perms are hand-crafted for the matrix given in the ctor.
     // They are same graph renumbering, reversed.
     std::vector<i_type> correct_0 = {2, 1, 6, 5, 3, 4, 0, 8, 7};
-    std::vector<i_type> correct_1 = {6, 7, 2, 3, 5, 4, 8, 0, 1};
+    std::vector<i_type> correct_1 = {7, 8, 0, 4, 3, 5, 6, 1, 2};
 
     auto eq_0 = std::equal(p, p + correct_0.size(), correct_0.begin());
     auto eq_1 = std::equal(p, p + correct_1.size(), correct_1.begin());

From c074b97c9ae2cb849ea08e3a8084b142454543cb Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Sun, 7 Jun 2020 18:14:25 +0200
Subject: [PATCH 0048/1008] Rcm documentation

---
 include/ginkgo/core/reorder/rcm.hpp | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/include/ginkgo/core/reorder/rcm.hpp b/include/ginkgo/core/reorder/rcm.hpp
index 95ab03a7500..c2992ed5721 100644
--- a/include/ginkgo/core/reorder/rcm.hpp
+++ b/include/ginkgo/core/reorder/rcm.hpp
@@ -63,10 +63,10 @@ enum class starting_strategy { minimum_degree, pseudo_peripheral };
 
 
 /**
- * Rcm is the reordering algorithm which uses METIS to compute a
- * fill reducing re-ordering of the given sparse matrix. The METIS
- * `Metis_NodeND` which is implements the multilevel nested dissection algorithm
- * as in the METIS documentation.
+ * Rcm is a reordering algorithm minimizing the bandwidth of a matrix. Such a
+ * reordering typically also significantly reduces fill-in, though usually not
+ * as effective as more complex algorithms, specifically AMD and nested
+ * dissection schemes. The advantage of this algorithm is its low runtime.
  *
  * @note  This class is derives from polymorphic object but is not a LinOp as it
  * does not make sense for this class to implement the apply methods. The
@@ -78,9 +78,12 @@ enum class starting_strategy { minimum_degree, pseudo_peripheral };
  * pseudo-peripheral. These strategies control how a starting vertex for a
  * connected component is choosen, which is then renumbered as first vertex in
  * the component, starting the algorithm from there.
- * In general, the bandwidths obtained by choosing a pseudo-peripheral node are
- * slightly smaller than those obtained from choosing a node of minimum degree.
- * On the other hand, this strategy is much more expensive, relatively.
+ * In general, the bandwidths obtained by choosing a pseudo-peripheral vertex
+ * are slightly smaller than those obtained from choosing a vertex of minimum
+ * degree. On the other hand, this strategy is much more expensive, relatively.
+ * The algorithm for finding a pseudo-peripheral vertex as
+ * described in "Computer Solution of Sparse Linear Systems" (George, Liu, Ng,
+ * Oak Ridge National Laboratory, 1994) is implemented here.
  *
  * @tparam ValueType  Type of the values of all matrices used in this class
  * @tparam IndexType  Type of the indices of all matrices used in this class
@@ -180,6 +183,9 @@ class Rcm : public EnablePolymorphicObject<Rcm<ValueType, IndexType>,
         degrees_ = std::unique_ptr<Array<IndexType>>(
             new Array<IndexType>(exec, dim[0]));
         permutation_ = PermutationMatrix::create(exec, dim);
+
+        // To make it explicit.
+        inv_permutation_ = nullptr;
         if (parameters_.construct_inverse_permutation) {
             inv_permutation_ = PermutationMatrix::create(exec, dim);
         }
@@ -188,10 +194,10 @@ class Rcm : public EnablePolymorphicObject<Rcm<ValueType, IndexType>,
     }
 
 private:
-    std::shared_ptr<SparsityMatrix> adjacency_matrix_{};
-    std::shared_ptr<PermutationMatrix> permutation_{};
-    std::unique_ptr<Array<IndexType>> degrees_{};
-    std::shared_ptr<PermutationMatrix> inv_permutation_{};
+    std::shared_ptr<SparsityMatrix> adjacency_matrix_;
+    std::shared_ptr<PermutationMatrix> permutation_;
+    std::unique_ptr<Array<IndexType>> degrees_;
+    std::shared_ptr<PermutationMatrix> inv_permutation_;
 };
 
 

From e597621527e3d4072540866a97a917bbf39d5cfb Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Sat, 22 Aug 2020 16:51:55 +0200
Subject: [PATCH 0049/1008] Implement urcm

Also implements ubfs as a building block of urcm.
---
 omp/reorder/rcm_kernels.cpp | 808 ++++++++++++++++++++++++++++++++++--
 1 file changed, 783 insertions(+), 25 deletions(-)

diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index bd7bca01717..6dea667674d 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -31,9 +31,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #include "core/reorder/rcm_kernels.hpp"
+#include "core/base/allocator.hpp"
+#include "core/components/prefix_sum.hpp"
 
 
+#include <immintrin.h>
 #include <algorithm>
+#include <cstring>
 #include <iterator>
 #include <memory>
 #include <queue>
@@ -41,6 +45,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <vector>
 
 
+#include <omp.h>
+
+
 #include <ginkgo/config.hpp>
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/math.hpp>
@@ -76,44 +83,793 @@ void get_degree_of_nodes(std::shared_ptr<const OmpExecutor> exec,
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 
+constexpr auto chunk_bound = 512;
+
+template <typename IndexType>
+struct UbfsLinearQueue {
+    gko::vector<IndexType> arr;
+    IndexType head;
+    IndexType tail;
+
+    // Those two are only there for safety, they are barely ever contended,
+    // due to work chunking.
+    omp_lock_t read_lock;
+    omp_lock_t write_lock;
+
+    UbfsLinearQueue() {}
+
+    UbfsLinearQueue(std::shared_ptr<const OmpExecutor> exec, size_type capacity)
+        : arr(gko::vector<IndexType>(capacity, exec)), head(0), tail(0)
+    {
+        omp_init_lock(&read_lock);
+        omp_init_lock(&write_lock);
+    }
+
+    ~UbfsLinearQueue()
+    {
+        omp_destroy_lock(&read_lock);
+        omp_destroy_lock(&write_lock);
+    }
+
+    /**
+     * Copies a chunk of nodes back into the work queue,
+     * in a thread-safe manner.
+     */
+    void enqueue_chunk(const IndexType *const chunk, size_type n)
+    {
+        const auto data = &arr[0];
+
+        omp_set_lock(&write_lock);
+
+        std::memcpy(&data[tail], chunk, n * sizeof(IndexType));
+
+        // Atomicity, order, and observability enforced by lock.
+        tail += n;
+
+        omp_unset_lock(&write_lock);
+    }
+
+    /**
+     * Computes the correct chunk size at a given moment.
+     * It is upper bounded by the chunk_bound and the half of all nodes.
+     */
+    IndexType chunk_size()
+    {
+        const auto available_nodes = tail - head;
+        return std::min((available_nodes + 1) / 2, (IndexType)chunk_bound);
+    }
+
+    /**
+     * Returns a pointer to an exclusively owned chunk of length <= n/2.
+     * Blocks in case no nodes are available,
+     * until all other threads are finished processing.
+     * If (nullptr,0) is returned, the algorithm is done.
+     */
+    std::pair<IndexType *, IndexType> dequeue_chunk(int *threads_working)
+    {
+        const auto data = &arr[0];
+        omp_set_lock(&read_lock);
+
+        const auto n = chunk_size();
+
+        if (n > 0) {
+// Interacts with other parts of the program.
+#pragma omp atomic update
+            ++(*threads_working);
+
+            const auto old_head = head;
+            head += n;
+
+            omp_unset_lock(&read_lock);
+            return std::make_pair(data + old_head, n);
+        } else {
+            // This "spinlock" is fine without cmpxchg, exclusive waiter-access
+            // is guaranteed by the outer lock.
+            while (true) {
+                int val_threads_working;
+#pragma omp atomic read
+                val_threads_working = *threads_working;
+
+                if (val_threads_working == 0) {
+                    break;
+                }
+
+                // No measureable effect on performance.
+                _mm_pause();
+            }
+
+            // Recalculate the chunk size, now that all threads are finished.
+            const auto n = chunk_size();
+            if (n > 0) {
+#pragma omp atomic update
+                ++(*threads_working);
+
+                IndexType old_head = head;
+                head += n;
+
+                omp_unset_lock(&read_lock);
+                return std::make_pair(data + old_head, n);
+            } else {
+                omp_unset_lock(&read_lock);
+                return std::make_pair(nullptr, 0);
+            }
+        }
+    }
+};
+
+/**
+ * Performs an unordered breadth-first search,
+ * thereby building a rooted level structure.
+ */
+template <typename IndexType>
+void ubfs(std::shared_ptr<const OmpExecutor> exec, const size_type num_vertices,
+          const IndexType *const row_ptrs, const IndexType *const col_idxs,
+          IndexType *const
+              levels,  // Must be inf/max in all nodes connected to source
+          const IndexType start, const IndexType max_degree)
+{
+    const auto max_threads = omp_get_max_threads();
+
+    // This is an upper bound for how many nodes may be in the work set
+    // at maximum at any time.
+    const auto work_bound = (size_type)max_threads * num_vertices;
+    UbfsLinearQueue<IndexType> q(exec, work_bound);
+
+    // Start from the given start node.
+    q.enqueue_chunk(&start, 1);
+    levels[start] = 0;
+
+    // Used for synchronization of (pseudo)remaining work.
+    int threads_working = 0;
+
+#pragma omp parallel firstprivate(max_degree)
+    {
+        // This is an upper bound, as '.deque_chunk...' is upper bounded by
+        // chunk_bound.
+        gko::vector<IndexType> local_to_insert(max_degree * chunk_bound, exec);
+        auto local_to_insert_size = 0;
+
+        while (true) {
+            // Get some nodes from the locked q.
+            const auto pair = q.dequeue_chunk(&threads_working);
+            const auto chunk = pair.first;
+            const auto chunk_size = pair.second;
+
+            // If there are nodes left to process.
+            // If there are no nodes returned that also means there are no nodes
+            // still in flight.
+            if (chunk_size > 0) {
+                // For each node in the chunk.
+                for (auto chunk_i = 0; chunk_i < chunk_size; ++chunk_i) {
+                    const auto node = chunk[chunk_i];
+
+                    IndexType level;
+#pragma omp atomic read
+                    level = levels[node];
+
+                    const auto new_neighbour_level = level + 1;
+
+                    // Go through neighbours.
+                    const auto row_start = row_ptrs[node];
+                    const auto row_end = row_ptrs[node + 1];
+
+                    for (auto neighbour_i = row_start; neighbour_i < row_end;
+                         ++neighbour_i) {
+                        const auto neighbour = col_idxs[neighbour_i];
+
+                        IndexType old_neighbour_level;
+                        do {
+#pragma omp atomic read
+                            old_neighbour_level = levels[neighbour];
+
+                            const auto is_new_smaller =
+                                new_neighbour_level < old_neighbour_level;
+
+                            if (!is_new_smaller) {
+                                goto continue_outer;
+                            }
+                        } while (!__atomic_compare_exchange_n(
+                            &levels[neighbour], &old_neighbour_level,
+                            new_neighbour_level, true,
+                            std::memory_order::memory_order_acq_rel,
+                            std::memory_order::memory_order_acquire));
+
+                        local_to_insert[local_to_insert_size++] = neighbour;
+
+                    continue_outer:;
+                    }
+                }
+
+                // Insert back to global work queue, done with this chunk.
+                q.enqueue_chunk(&local_to_insert[0], local_to_insert_size);
+
+#pragma omp atomic update
+                --threads_working;
+
+                local_to_insert_size = 0;
+            } else {
+                // Done!
+                break;
+            }
+        }
+    }
+}
+
+/**
+ * Finds a 'contender', meaning a node in the last level of the rls with minimum
+ * degree, returns it along with the rls height.
+ */
+template <typename IndexType>
+std::pair<IndexType, IndexType> rls_contender_and_height(
+    std::shared_ptr<const OmpExecutor> exec, const size_type num_vertices,
+    const IndexType *const row_ptrs, const IndexType *const col_idxs,
+    const IndexType *const degrees,
+    IndexType *const levels,  // Must be max/inf in all nodes connected to start
+    const IndexType start, const IndexType max_degree)
+{
+    // Create a level structure.
+    ubfs(exec, num_vertices, row_ptrs, col_idxs, levels, start, max_degree);
+
+    // Now find the node in the last level with minimal degree.
+    // TODO: Rewrite this as a custom reduction.
+
+    // First local ...
+    const auto num_threads = omp_get_max_threads();
+    gko::vector<IndexType> local_contenders(num_threads, exec);
+    gko::vector<IndexType> local_degrees(num_threads, exec);
+    gko::vector<IndexType> local_heights(num_threads, exec);
+
+#pragma omp parallel num_threads(num_threads)
+    {
+        const auto tid = omp_get_thread_num();
+        auto local_contender = 0;
+        auto local_degree = std::numeric_limits<IndexType>::max();
+        auto local_height = 0;
+
+#pragma omp for schedule(static)
+        for (auto i = 0; i < num_vertices; ++i) {
+            if (levels[i] > local_height) {
+                local_contender = i;
+                local_degree = degrees[i];
+                local_height = levels[i];
+            } else if (levels[i] == local_height && degrees[i] > local_degree) {
+                local_contender = i;
+                local_degree = degrees[i];
+            }
+        }
+
+        local_contenders[tid] = local_contender;
+        local_degrees[tid] = local_degree;
+        local_heights[tid] = local_height;
+    }
+
+    // ... then global.
+    auto global_contender = local_contenders[0];
+    auto global_degree = local_degrees[0];
+    auto global_height = local_heights[0];
+    for (auto i = 1; i < num_threads; ++i) {
+        if (local_heights[i] > global_height) {
+            global_contender = local_contenders[i];
+            global_degree = local_degrees[i];
+            global_height = local_heights[i];
+        } else if (local_heights[i] == global_height &&
+                   local_degrees[i] > global_degree) {
+            global_contender = local_contenders[i];
+            global_degree = local_degrees[i];
+        }
+    }
+
+    return std::make_pair(global_contender, global_height);
+}
+
+/**
+ * Finds the index of a node with minimum degree and the maximum degree.
+ */
+template <typename IndexType>
+std::pair<IndexType, IndexType> find_min_idx_and_max_val(
+    std::shared_ptr<const OmpExecutor> exec, const size_type num_vertices,
+    const IndexType *const row_ptrs, const IndexType *const col_idxs,
+    const IndexType *const degrees, gko::vector<IndexType> &levels,
+    const uint8 *const previous_component,
+    gko::reorder::starting_strategy strategy)
+{
+    // First find local extrema.
+    const auto num_threads = omp_get_max_threads();
+    gko::vector<std::pair<IndexType, IndexType>> local_min_vals_idxs(
+        num_threads, exec);
+    gko::vector<IndexType> local_max_vals(num_threads, exec);
+
+#pragma omp parallel num_threads(num_threads)
+    {
+        const auto tid = omp_get_thread_num();
+        auto local_min_val = std::numeric_limits<IndexType>::max();
+        auto local_min_idx = tid;
+        auto local_max_val = std::numeric_limits<IndexType>::min();
+        auto local_max_idx = tid;
+
+#pragma omp for schedule(static)
+        for (auto i = 0; i < num_vertices; ++i) {
+            // If this level hasn't been discovered before.
+            if (!previous_component[i]) {
+                if (degrees[i] < local_min_val) {
+                    local_min_val = degrees[i];
+                    local_min_idx = i;
+                }
+                if (degrees[i] > local_max_val) {
+                    local_max_val = degrees[i];
+                    local_max_idx = i;
+                }
+            }
+        }
+
+        local_min_vals_idxs[tid] = std::make_pair(local_min_val, local_min_idx);
+        local_max_vals[tid] = local_max_val;
+    }
+
+    // Then find global extrema.
+    auto global_min_idx = local_min_vals_idxs[0].second;
+    auto global_min_val = local_min_vals_idxs[0].first;
+    auto global_max_val = local_max_vals[0];
+    for (auto i = 1; i < num_threads; ++i) {
+        if (local_min_vals_idxs[i].first < global_min_val) {
+            global_min_val = local_min_vals_idxs[i].first;
+            global_min_idx = local_min_vals_idxs[i].second;
+        }
+        if (local_max_vals[i] > global_max_val) {
+            global_max_val = local_max_vals[i];
+        }
+    }
+
+    return std::make_pair(global_min_idx, global_max_val);
+}
+
+/**
+ * Finds a start node for the urcm algorithm, using parallel building blocks.
+ */
+template <typename IndexType>
+IndexType find_start_node(std::shared_ptr<const OmpExecutor> exec,
+                          const size_type num_vertices,
+                          const IndexType *const row_ptrs,
+                          const IndexType *const col_idxs,
+                          const IndexType *const degrees,
+                          gko::vector<IndexType> &levels,
+                          const uint8 *const previous_component,
+                          const gko::reorder::starting_strategy strategy)
+{
+    // Find the node with minimal degree and the maximum degree.
+    // That is necessary for every strategy.
+    const auto min_idx_and_max_val =
+        find_min_idx_and_max_val(exec, num_vertices, row_ptrs, col_idxs,
+                                 degrees, levels, previous_component, strategy);
+    const auto min_idx = min_idx_and_max_val.first;
+    const auto max_val = min_idx_and_max_val.second;
+
+    // Now is the time to return for the min degere strategy.
+    if (strategy == gko::reorder::starting_strategy::minimum_degree) {
+        ubfs(exec, num_vertices, row_ptrs, col_idxs, &levels[0], min_idx,
+             max_val);
+        return min_idx;
+    }
+
+    // Intermediate storage.
+    // We copy back to `levels` for the final pick.
+    gko::vector<IndexType> levels_clone(levels, exec);
+    gko::vector<IndexType> current_levels_clone(levels, exec);
+
+    auto current = min_idx;
+    const auto contender_and_height =
+        rls_contender_and_height(exec, num_vertices, row_ptrs, col_idxs,
+                                 degrees, &levels_clone[0], current, max_val);
+    current_levels_clone.swap(levels_clone);
+    auto current_contender = contender_and_height.first;
+    auto current_height = contender_and_height.second;
+
+    // This loop always terminates, as height needs to strictly increase.
+    while (true) {
+        const auto contender_contender_and_height = rls_contender_and_height(
+            exec, num_vertices, row_ptrs, col_idxs, degrees, &levels_clone[0],
+            current_contender, max_val);
+        auto contender_contender = contender_contender_and_height.first;
+        auto contender_height = contender_contender_and_height.second;
 
+        if (contender_height > current_height) {
+            current_height = contender_height;
+            current = current_contender;
+            current_contender = contender_contender;
+            current_levels_clone = gko::vector<IndexType>(levels, exec);
+            current_levels_clone.swap(levels_clone);
+        } else {
+            std::memcpy(&levels[0], &current_levels_clone[0],
+                        num_vertices * sizeof(IndexType));
+            return current;
+        }
+    }
+}
+
+/**
+ * Counts how many nodes there are per level.
+ */
+template <typename IndexType>
+gko::vector<IndexType> count_levels(std::shared_ptr<const OmpExecutor> exec,
+                                    const IndexType *const levels,
+                                    uint8 *const previous_component,
+                                    size_type num_vertices)
+{
+    const auto num_threads = omp_get_max_threads();
+
+    // This initialization is necessary due to nested custom allocator.
+    gko::vector<gko::vector<IndexType>> level_counts(exec);
+    for (auto i = 0; i < num_threads; ++i) {
+        level_counts.push_back(gko::vector<IndexType>(0, exec));
+    }
+
+#pragma omp parallel num_threads(num_threads)
+    {
+        const auto tid = omp_get_thread_num();
+        auto local_level_counts = &level_counts[tid];
+
+        auto current_size = 0;
+
+#pragma omp for schedule(static)
+        for (auto i = 0; i < num_vertices; ++i) {
+            const auto level = levels[i];
+
+            // If not part of previous component and actually discovered in this
+            // component.
+            if (!previous_component[i] &&
+                level != std::numeric_limits<IndexType>::max()) {
+                if (level >= current_size) {
+                    local_level_counts->resize(level + 1);
+                    current_size = level + 1;
+                }
+                previous_component[i] = true;
+                ++(*local_level_counts)[level];
+            }
+        }
+    }
+
+    gko::vector<size_type> level_count_sizes(num_threads, exec);
+    for (auto tid = 0; tid < num_threads; ++tid) {
+        level_count_sizes[tid] = level_counts[tid].size();
+    }
+    const auto max_size =
+        *std::max_element(level_count_sizes.begin(), level_count_sizes.end());
+
+    gko::vector<IndexType> final_level_counts(max_size + 1, exec);
+    auto i = 0;
+    while (true) {
+        auto exit = true;
+        for (auto tid = 0; tid < num_threads; ++tid) {
+            if (i < level_count_sizes[tid]) {
+                const auto count = level_counts[tid][i];
+                final_level_counts[i] += count;
+                exit = false;
+            }
+        }
+        if (exit) {
+            break;
+        }
+        ++i;
+    }
+
+    return final_level_counts;
+}
+
+/**
+ * Implements the two intermediate phases of urcm,
+ * counting the nodes per level and computing the level offsets,
+ * as a prefix sum.
+ */
+template <typename IndexType>
+gko::vector<IndexType> compute_level_offsets(
+    std::shared_ptr<const OmpExecutor> exec, const IndexType *const levels,
+    size_type num_vertices, uint8 *const previous_component)
+{
+    auto counts = count_levels(exec, levels, previous_component, num_vertices);
+    components::prefix_sum(exec, &counts[0], counts.size());
+    return counts;
+}
+
+/**
+ * Helper macros for the helper function below.
+ */
+#ifdef __GNUG__
+// GCC can only be convinced to use cmovs here by inline asm.
+#define GKO_COMPARATOR(x, y)                                       \
+    {                                                              \
+        IndexType tmp;                                             \
+        const auto degree_x = degrees[neighbours[x]];              \
+        const auto degree_y = degrees[neighbours[y]];              \
+        asm("mov %0, %2\n\t"                                       \
+            "cmp %4, %3\n\t"                                       \
+            "cmova %1, %0\n\t"                                     \
+            "cmova %2, %1\n\t"                                     \
+            : "+r"(neighbours[x]), "+r"(neighbours[y]), "=&r"(tmp) \
+            : "r"(degree_x), "r"(degree_y)                         \
+            : "cc");                                               \
+    }
+#else
+#define GKO_COMPARATOR(x, y)                                         \
+    {                                                                \
+        const auto degree_x = degrees[neighbours[x]];                \
+        const auto degree_y = degrees[neighbours[y]];                \
+        const auto should_swap = degree_x > degree_y;                \
+        const auto tmp = neighbours[x];                              \
+        neighbours[x] = should_swap ? neighbours[y] : neighbours[x]; \
+        neighbours[y] = should_swap ? tmp : neighbours[y];           \
+    }
+#endif
+
+/**
+ * Helper fucntion for quickly sorting small vectors by degree.
+ */
+template <typename IndexType>
+inline void sort_by_degree(IndexType *const neighbours, size_type count,
+                           const IndexType *const degrees)
+{
+    if (count < 2) {
+        return;
+    }
+
+    if (count == 2) {
+        GKO_COMPARATOR(0, 1);
+        return;
+    }
+
+    if (count == 3) {
+        GKO_COMPARATOR(0, 1);
+        GKO_COMPARATOR(0, 2);
+        GKO_COMPARATOR(1, 2);
+        return;
+    }
+
+    if (count == 4) {
+        GKO_COMPARATOR(0, 1);
+        GKO_COMPARATOR(2, 3);
+        GKO_COMPARATOR(0, 2);
+        GKO_COMPARATOR(1, 3);
+        GKO_COMPARATOR(1, 2);
+        return;
+    }
+
+    if (count < 16) {
+        int i, j;
+        for (i = 1; i < count; i++) {
+            int tmp = neighbours[i];
+            for (j = i; j >= 1 && degrees[tmp] < degrees[neighbours[j - 1]];
+                 --j) {
+                neighbours[j] = neighbours[j - 1];
+            }
+            neighbours[j] = tmp;
+        }
+        return;
+    }
+
+    std::sort(neighbours, neighbours + count,
+              [&](IndexType left, IndexType right) {
+                  return degrees[left] < degrees[right];
+              });
+}
+
+/**
+ * Implements the last phase of urcm,
+ * writing the permutation levels in parallel.
+ */
 template <typename IndexType>
-IndexType find_index(std::vector<std::pair<IndexType, IndexType>> &a,
-                     IndexType x)
+void write_permutation(std::shared_ptr<const OmpExecutor> exec,
+                       const IndexType *const row_ptrs,
+                       const IndexType *const col_idxs, IndexType *const levels,
+                       const IndexType *const degrees,
+                       const gko::vector<IndexType> &offsets,
+                       IndexType *const perm, const std::size_t num_vertices,
+                       const IndexType base_offset, const IndexType start)
 {
-    for (auto i = 0; i < a.size(); i++)
-        if (a[i].first == x) return i;
-    return -1;
+    const auto offsets_size = offsets.size();
+    perm[base_offset] = start;
+
+    const auto num_threads = omp_get_max_threads();
+#pragma omp parallel num_threads(num_threads) firstprivate(offsets_size)
+    {
+        const auto tid = omp_get_thread_num();
+        gko::vector<IndexType> valid_neighbours(0, exec);
+
+        // Go through the levels assigned to this thread.
+        for (auto level = tid; level < offsets_size - 1; level += num_threads) {
+            const auto next_level_idx = offsets[level + 1];
+            const auto level_idx = offsets[level];
+            const auto level_len = next_level_idx - level_idx;
+            auto write_offset = 0;
+
+            // Go through one level.
+            for (auto read_offset = level_idx; read_offset < next_level_idx;
+                 ++read_offset) {
+                // Wait until a value is written at that index.
+                IndexType written;
+#pragma omp atomic read
+                written = perm[base_offset + read_offset];
+                while (written == std::numeric_limits<IndexType>::max()) {
+                    _mm_pause();
+#pragma omp atomic read
+                    written = perm[base_offset + read_offset];
+                }
+
+                // Collect valid neighbours.
+                const auto row_start = row_ptrs[written];
+                const auto row_end = row_ptrs[written + 1];
+                const auto write_level = level + 1;
+                for (auto neighbour_i = row_start; neighbour_i < row_end;
+                     ++neighbour_i) {
+                    const auto neighbour = col_idxs[neighbour_i];
+
+                    // Will not be written by multiple threads, but can be read
+                    // while written. This is only necessary to guarantee the
+                    // abscence of reads-while-writes.
+                    IndexType neighbour_level;
+#pragma omp atomic read
+                    neighbour_level = levels[neighbour];
+
+                    // ::max() is not a valid value here (as ther cannot be
+                    // ::max() nodes), therefore can be used for signaling.
+                    if (neighbour_level == write_level) {
+// Protect against writing the same node as neighbour of different nodes in the
+// previous level by the same thread.
+#pragma omp atomic write
+                        levels[neighbour] =
+                            std::numeric_limits<IndexType>::max();
+
+                        valid_neighbours.push_back(neighbour);
+                    }
+                }
+
+                // Sort neighbours.
+                const auto size = valid_neighbours.size();
+                sort_by_degree(&valid_neighbours[0], size, degrees);
+
+                // Write the processed neighbours.
+                const auto base_write_offset =
+                    base_offset + next_level_idx + write_offset;
+
+                // Memcpy would be nice, but we need element-wise atomicity.
+                for (auto i = 0; i < size; ++i) {
+#pragma omp atomic write
+                    perm[base_write_offset + i] = valid_neighbours[i];
+                }
+
+                write_offset += size;
+                valid_neighbours.clear();
+            }
+        }
+    }
 }
 
+/**
+ * Processes all isolated nodes, returning their count.
+ */
+template <typename IndexType>
+IndexType handle_isolated_nodes(std::shared_ptr<const OmpExecutor> exec,
+                                const IndexType *const row_ptrs,
+                                const IndexType *const col_idxs,
+                                const IndexType *const degrees,
+                                IndexType *const perm, size_type num_vertices,
+                                gko::vector<uint8> &previous_component)
+{
+    struct IsolatedNodes {
+        // Using a gko::vector here makes clang and gcc segfault,
+        // if we then use 'omp_priv = omp_orig'.
+        std::vector<IndexType> nodes;
+        IsolatedNodes() : nodes(std::vector<IndexType>(0)) {}
+        IsolatedNodes &operator+=(const IsolatedNodes &rhs)
+        {
+            nodes.reserve(nodes.size() + rhs.nodes.size());
+            nodes.insert(nodes.end(), rhs.nodes.begin(), rhs.nodes.end());
+            return *this;
+        }
+    };
+
+#pragma omp declare reduction(FindIsolated         \
+                              : IsolatedNodes      \
+                              : omp_out += omp_in) \
+    initializer(omp_priv = IsolatedNodes())
+    auto isolated = IsolatedNodes();
+#pragma omp parallel for reduction(FindIsolated : isolated)
+    for (auto i = 0; i < num_vertices; ++i) {
+        if (degrees[i] == 0
+            /* || (degrees[i] == 1 && col_idxs[row_ptrs[i]] == i) */) {
+            isolated.nodes.push_back(i);
+            previous_component[i] = true;
+        }
+    }
+
+    std::memcpy(perm, &isolated.nodes[0],
+                isolated.nodes.size() * sizeof(IndexType));
+    return isolated.nodes.size();
+}
 
+/**
+ * Computes a rcm permutation, employing the parallel unordered rcm algorithm.
+ */
 template <typename IndexType>
 void get_permutation(std::shared_ptr<const OmpExecutor> exec,
                      const size_type num_vertices,
                      const IndexType *const row_ptrs,
                      const IndexType *const col_idxs,
-                     const IndexType *const degrees,
-                     IndexType *const permutation,
-                     IndexType *const inv_permutation,
+                     const IndexType *const degrees, IndexType *const perm,
+                     IndexType *const inv_perm,
                      const gko::reorder::starting_strategy strategy)
 {
-    // Phase 1:
-    //     Compute the level of each node using UBFS.
-    //     Find all starting vertices.
-    // Phase 2 (for each connected component,
-    // starting once respective starting vertex is confirmed):
-    //     Get the primary offset into the perm.
-    //     Compute the level borders as prefix sum of the counts.
-    // Phase 3 (for each connected component)
-    //     Start by writing the starting node.
-    //     Threads watch their level:
-    //         If the thread to the left writes a new node to your level:
-    //         Write those neighbours of the node which are in the next level to
-    //         the next level, sorted by degree.
-    //     (Can either by implemented by spinning or tasks)
-    // Once the last node in the last level is written for each component,
-    // the algorithm is finished.
-    GKO_NOT_IMPLEMENTED;
+    // Initialize the perm to all "signal value".
+    std::fill(perm, perm + num_vertices, std::numeric_limits<IndexType>::max());
+
+    // For multiple components.
+    IndexType base_offset = 0;
+
+    // Stores for each node if it is part of an already discovered component.
+    gko::vector<uint8> previous_component(num_vertices, exec);
+
+    // First handle all isolated nodes. That reduces complexity later on.
+    base_offset +=
+        handle_isolated_nodes(exec, row_ptrs, col_idxs, degrees, perm,
+                              num_vertices, previous_component);
+
+    // Stores the level structure. Initialized to all "infinity".
+    gko::vector<IndexType> levels(num_vertices, exec);
+    std::fill(levels.begin(), levels.end(),
+              std::numeric_limits<IndexType>::max());
+
+    // Are we done yet?
+    auto done = false;
+
+    while (!done) {
+        // Phase 1:
+        // Finds a start node, while also filling the level structure.
+
+        const auto start =
+            find_start_node(exec, num_vertices, row_ptrs, col_idxs, degrees,
+                            levels, &previous_component[0], strategy);
+
+        // Phase 2:
+        // Generate the level offsets.
+
+        // Will contain 0 -- 1 -- level_count(2) + 1 -- level_count(2) +
+        // level_count(3) + 1 -- ... -- total_sum
+        gko::vector<IndexType> offsets = compute_level_offsets(
+            exec, &levels[0], num_vertices, &previous_component[0]);
+
+
+        // Phase 3:
+        //     Start by writing the starting node.
+        //     Threads watch their level:
+        //          If the thread to the left writes a new node to your level:
+        //              Write those neighbours of the node which are in the next
+        //              level (and havent been written to that next level yet)
+        //              to the next level, sorted by degree.
+        //  Once the last node in the last level is written for the last
+        //  component, the algorithm is finished.
+
+        write_permutation(exec, row_ptrs, col_idxs, &levels[0], degrees,
+                          offsets, perm, num_vertices, base_offset, start);
+
+        // Are we done yet?
+        done = base_offset + offsets.back() == num_vertices;
+        base_offset += offsets.back();
+    }
+
+// Finally reverse the order.
+// Could in principle also be already written in reverse order, but ... just no.
+// This is ridiculoulsly fast anyway.
+#pragma omp parallel for schedule(static)
+    for (auto i = 0; i < num_vertices / 2; ++i) {
+        const auto tmp = perm[i];
+        perm[i] = perm[num_vertices - i - 1];
+        perm[num_vertices - i - 1] = tmp;
+    }
+
+    if (inv_perm) {
+#pragma omp parallel for schedule(static)
+        for (auto i = 0; i < num_vertices; ++i) {
+            inv_perm[perm[i]] = i;
+        }
+    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
@@ -123,3 +879,5 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
 }  // namespace omp
 }  // namespace kernels
 }  // namespace gko
+
+#undef GKO_COMPARATOR
\ No newline at end of file

From 940790c2d466124e82f32511d8785b308499a5e9 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Mon, 24 Aug 2020 12:38:09 +0200
Subject: [PATCH 0050/1008] Add rcm to cmake

Adds the rcm and rcm_kernel sources to the cmake target_sources.
---
 core/CMakeLists.txt      | 1 +
 cuda/CMakeLists.txt      | 1 +
 omp/CMakeLists.txt       | 1 +
 reference/CMakeLists.txt | 1 +
 4 files changed, 4 insertions(+)

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index f5cff2e407d..f16d63ff617 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -31,6 +31,7 @@ target_sources(ginkgo
     matrix/sparsity_csr.cpp
     preconditioner/isai.cpp
     preconditioner/jacobi.cpp
+    reorder/rcm.cpp
     solver/bicg.cpp
     solver/bicgstab.cpp
     solver/cg.cpp
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index d3a73ab536c..b19651b6a59 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -97,6 +97,7 @@ target_sources(ginkgo_cuda
     preconditioner/jacobi_generate_kernel.cu
     preconditioner/jacobi_kernels.cu
     preconditioner/jacobi_simple_apply_kernel.cu
+    reorder/rcm_kernels.cu
     solver/bicg_kernels.cu
     solver/bicgstab_kernels.cu
     solver/cg_kernels.cu
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index bcfde819760..005e55ec24d 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -23,6 +23,7 @@ target_sources(ginkgo_omp
     matrix/sparsity_csr_kernels.cpp
     preconditioner/isai_kernels.cpp
     preconditioner/jacobi_kernels.cpp
+    reorder/rcm_kernels.cpp
     solver/bicg_kernels.cpp
     solver/bicgstab_kernels.cpp
     solver/cg_kernels.cpp
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index f92060c036c..ba3d22a9abf 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -21,6 +21,7 @@ target_sources(ginkgo_reference
     matrix/sparsity_csr_kernels.cpp
     preconditioner/isai_kernels.cpp
     preconditioner/jacobi_kernels.cpp
+    reorder/rcm_kernels.cpp
     solver/bicg_kernels.cpp
     solver/bicgstab_kernels.cpp
     solver/cg_kernels.cpp

From d35913ca9d9e63c16ff932c803780016c58003ad Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Mon, 24 Aug 2020 12:43:04 +0200
Subject: [PATCH 0051/1008] Add definitive rcm_kernel test

The added test for the omp rcm_kernel asserts if a perm is in rcm order.
---
 omp/test/reorder/rcm_kernels.cpp | 257 +++++++++++++++++++++++++++++--
 1 file changed, 243 insertions(+), 14 deletions(-)

diff --git a/omp/test/reorder/rcm_kernels.cpp b/omp/test/reorder/rcm_kernels.cpp
index f6905a481c0..cbcea22e9ac 100644
--- a/omp/test/reorder/rcm_kernels.cpp
+++ b/omp/test/reorder/rcm_kernels.cpp
@@ -34,6 +34,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
+#include <cstring>
+#include <deque>
 #include <fstream>
 #include <memory>
 
@@ -62,6 +64,7 @@ class Rcm : public ::testing::Test {
     using Mtx = gko::matrix::Dense<v_type>;
     using CsrMtx = gko::matrix::Csr<v_type, i_type>;
     using reorder_type = gko::reorder::Rcm<v_type, i_type>;
+    using strategy = gko::reorder::starting_strategy;
     Rcm()
         : ref(gko::ReferenceExecutor::create()),
           omp(gko::OmpExecutor::create()),
@@ -73,18 +76,243 @@ class Rcm : public ::testing::Test {
               omp))
     {}
 
-    static void assert_equal_permutations(
-        const gko::matrix::Permutation<i_type> *to_check,
-        const gko::matrix::Permutation<i_type> *orig)
+    static void ubfs_reference(
+        std::shared_ptr<CsrMtx> mtx,
+        i_type
+            *const levels,  // Must be inf/max in all nodes connected to source
+        const i_type start)
     {
-        auto o_p_size = orig->get_permutation_size();
-        auto d_p_size = to_check->get_permutation_size();
-        ASSERT_EQ(d_p_size, o_p_size);
+        const auto row_ptrs = mtx->get_const_row_ptrs();
+        const auto col_idxs = mtx->get_const_col_idxs();
 
-        for (auto i = 0; i < d_p_size; ++i) {
-            ASSERT_EQ(to_check->get_const_permutation()[i],
-                      orig->get_const_permutation()[i]);
+        std::deque<i_type> q(0);
+        q.push_back(start);
+        levels[start] = 0;
+
+        while (!q.empty()) {
+            const auto node = q.front();
+            q.pop_front();
+
+            const auto level = levels[node];
+            const auto neighbours_level = level + 1;
+            const auto row_start = row_ptrs[node];
+            const auto row_end = row_ptrs[node + 1];
+
+            for (auto neighbour_i = row_start; neighbour_i < row_end;
+                 ++neighbour_i) {
+                const auto neighbour = col_idxs[neighbour_i];
+                if (neighbours_level < levels[neighbour]) {
+                    levels[neighbour] = neighbours_level;
+                    q.push_back(neighbour);
+                }
+            }
+        }
+    }
+
+    static bool is_valid_start_node(std::shared_ptr<CsrMtx> mtx,
+                                    std::shared_ptr<reorder_type> reorder,
+                                    i_type start,
+                                    std::vector<bool> &already_visited)
+    {
+        if (already_visited[start]) {
+            return false;
+        }
+
+        const auto n = reorder->get_permutation()->get_permutation_size();
+        auto degrees = std::vector<i_type>(n);
+        for (auto i = 0; i < n; ++i) {
+            degrees[i] =
+                mtx->get_const_row_ptrs()[i + 1] - mtx->get_const_row_ptrs()[i];
+        }
+
+        switch (reorder->get_parameters().strategy) {
+        case strategy::minimum_degree: {
+            auto min_degree = std::numeric_limits<i_type>::max();
+            for (auto i = 0; i < n; ++i) {
+                if (!already_visited[i] && degrees[i] < min_degree) {
+                    min_degree = degrees[i];
+                }
+            }
+            if (min_degree != degrees[start]) {
+                return false;
+            }
+            break;
+        }
+
+        case strategy::pseudo_peripheral: {
+            // Check if any valid contender has a lowereq height than the
+            // selected start node.
+
+            std::vector<i_type> reference_current_levels(n);
+            std::fill(reference_current_levels.begin(),
+                      reference_current_levels.end(),
+                      std::numeric_limits<i_type>::max());
+            ubfs_reference(mtx, &reference_current_levels[0], start);
+
+            std::vector<i_type> reference_contenders(0);
+            auto current_height = std::numeric_limits<i_type>::min();
+            for (auto i = 0; i < n; ++i) {
+                if (reference_current_levels[i] !=
+                        std::numeric_limits<i_type>::max() &&
+                    reference_current_levels[i] >= current_height) {
+                    if (reference_current_levels[i] > current_height) {
+                        reference_contenders.clear();
+                    }
+                    reference_contenders.push_back(i);
+                    current_height = reference_current_levels[i];
+                }
+            }
+
+            std::vector<std::vector<i_type>> reference_contenders_levels(
+                reference_contenders.size());
+            for (auto i = 0; i < reference_contenders.size(); ++i) {
+                std::vector<i_type> reference_contender_levels(n);
+                std::fill(reference_contender_levels.begin(),
+                          reference_contender_levels.end(),
+                          std::numeric_limits<i_type>::max());
+                ubfs_reference(mtx, &reference_contender_levels[0],
+                               reference_contenders[i]);
+                reference_contenders_levels[i] = reference_contender_levels;
+            }
+
+            for (auto i = 0; i < reference_contenders.size(); ++i) {
+                auto contender_height = std::numeric_limits<i_type>::min();
+                for (auto j = 0; j < n; ++j) {
+                    if (reference_contenders_levels[i][j] !=
+                            std::numeric_limits<i_type>::max() &&
+                        reference_contenders_levels[i][j] > contender_height) {
+                        contender_height = reference_contenders_levels[i][j];
+                    }
+                }
+                if (contender_height <= current_height) {
+                    return true;
+                }
+            }
+            return false;
+            break;
+        }
         }
+        return true;
+    }
+
+    static bool is_rcm_ordered(std::shared_ptr<CsrMtx> mtx,
+                               std::shared_ptr<reorder_type> reorder)
+    {
+        const auto n = reorder->get_permutation()->get_permutation_size();
+        const auto row_ptrs = mtx->get_const_row_ptrs();
+        const auto col_idxs = mtx->get_const_col_idxs();
+        auto degrees = std::vector<i_type>(n);
+        for (auto i = 0; i < n; ++i) {
+            degrees[i] =
+                mtx->get_const_row_ptrs()[i + 1] - mtx->get_const_row_ptrs()[i];
+        }
+
+        // Following checks for cm ordering, therefore create a reversed perm.
+        auto perm = std::vector<i_type>(n);
+        std::memcpy(&perm[0],
+                    reorder->get_permutation()->get_const_permutation(),
+                    n * sizeof(i_type));
+        for (auto i = 0; i < n / 2; ++i) {
+            const auto tmp = perm[i];
+            perm[i] = perm[n - i - 1];
+            perm[n - i - 1] = tmp;
+        }
+
+        // Now check for cm ordering.
+
+        auto base_offset = 0;
+        std::vector<bool> already_visited(n);
+        while (base_offset != n) {
+            // Assert valid start node.
+            if (!is_valid_start_node(mtx, reorder, perm[base_offset],
+                                     already_visited)) {
+                return false;
+            }
+
+            // Assert valid level structure.
+            // Also update base_offset and mark as visited while at it.
+            std::vector<i_type> levels(n);
+            std::fill(levels.begin(), levels.end(),
+                      std::numeric_limits<i_type>::max());
+            ubfs_reference(mtx, &levels[0], perm[base_offset]);
+
+            auto current_level = 0;
+            const auto previous_base_offset = base_offset;
+            for (auto i = 0; i < n; ++i) {
+                const auto node = perm[i];
+                if (levels[node] != std::numeric_limits<i_type>::max() &&
+                    !already_visited[node]) {
+                    already_visited[node] = true;
+                    ++base_offset;
+
+                    if (levels[node] == current_level) {
+                        continue;
+                    }
+                    if (levels[node] == current_level + 1) {
+                        ++current_level;
+                        continue;
+                    }
+                    return false;
+                }
+            }
+
+            // Assert cm order within levels.
+            for (auto i = previous_base_offset + 1 /* Skip start node */;
+                 i < base_offset - 1; ++i) {
+                const auto x = perm[i];
+                const auto y = perm[i + 1];
+                if (levels[x] != levels[y]) {
+                    continue;  // Skip if on level border
+                }
+                const auto level = levels[x];
+
+                // Get first neighbour of x in the previous level.
+                auto x_first_neighbour =
+                    perm[n - 1];  // There is always a neighbour, this is valid.
+                const auto x_row_start = row_ptrs[x];
+                const auto x_row_end = row_ptrs[x + 1];
+                for (auto x_neighbour_idx = x_row_start;
+                     x_neighbour_idx < x_row_end; ++x_neighbour_idx) {
+                    const auto x_neighbour = col_idxs[x_neighbour_idx];
+                    if (levels[x_neighbour] == level - 1) {
+                        if (std::find(perm.begin(), perm.end(), x_neighbour) <
+                            std::find(perm.begin(), perm.end(),
+                                      x_first_neighbour)) {
+                            x_first_neighbour = x_neighbour;
+                        }
+                    }
+                }
+                // Same again, for y.
+                auto y_first_neighbour = perm[n - 1];
+                const auto y_row_start = row_ptrs[y];
+                const auto y_row_end = row_ptrs[y + 1];
+                for (auto y_neighbour_idx = y_row_start;
+                     y_neighbour_idx < y_row_end; ++y_neighbour_idx) {
+                    const auto y_neighbour = col_idxs[y_neighbour_idx];
+                    if (levels[y_neighbour] == level - 1) {
+                        if (std::find(perm.begin(), perm.end(), y_neighbour) <
+                            std::find(perm.begin(), perm.end(),
+                                      y_first_neighbour)) {
+                            y_first_neighbour = y_neighbour;
+                        }
+                    }
+                }
+
+                // Assert the ... is not after the ... in the previous level.
+                if (std::find(perm.begin(), perm.end(), y_first_neighbour) <
+                    std::find(perm.begin(), perm.end(), x_first_neighbour)) {
+                    return false;
+                }
+
+                if (y_first_neighbour == x_first_neighbour) {
+                    if (degrees[y] < degrees[x]) {
+                        return false;
+                    }
+                }
+            }
+        }
+
+        return true;
     }
 
     std::shared_ptr<const gko::Executor> ref;
@@ -95,15 +323,16 @@ class Rcm : public ::testing::Test {
     std::unique_ptr<reorder_type> d_reorder_op;
 };
 
-TEST_F(Rcm, OmpPermutationIsEquivalentToRef)
+TEST_F(Rcm, OmpPermutationIsRcmOrdered)
 {
-    reorder_op = reorder_type::build().on(ref)->generate(o_1138_bus_mtx);
     d_reorder_op = reorder_type::build().on(omp)->generate(d_1138_bus_mtx);
 
-    auto perm = reorder_op->get_permutation();
-    auto d_perm = d_reorder_op->get_permutation();
+    auto perm = d_reorder_op->get_permutation();
 
-    assert_equal_permutations(d_perm.get(), perm.get());
+    // Can't std::move parameter when using ASSERT_PREDN, no perfect forwarding.
+    const auto d_reorder_op_shared =
+        std::shared_ptr<reorder_type>(d_reorder_op.release());
+    ASSERT_PRED2(is_rcm_ordered, d_1138_bus_mtx, d_reorder_op_shared);
 }
 
 }  // namespace

From a4b8b665cb7f6427cfacb8305ec179e6def71dfb Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Wed, 26 Aug 2020 10:30:36 +0200
Subject: [PATCH 0052/1008] Apply suggestions from code review

Change formatting, mostly newlines.

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 cuda/reorder/rcm_kernels.cu         |  4 ++--
 hip/reorder/rcm_kernels.hip.cpp     |  6 +++---
 include/ginkgo/core/reorder/rcm.hpp |  5 +++--
 omp/reorder/rcm_kernels.cpp         | 26 ++++++++++++++++++++------
 4 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/cuda/reorder/rcm_kernels.cu b/cuda/reorder/rcm_kernels.cu
index 3ccc267b755..1dce73ce28d 100644
--- a/cuda/reorder/rcm_kernels.cu
+++ b/cuda/reorder/rcm_kernels.cu
@@ -50,9 +50,9 @@ namespace gko {
 namespace kernels {
 namespace cuda {
 /**
- * @brief The parallel ilu factorization namespace.
+ * @brief The reordering namespace.
  *
- * @ingroup factor
+ * @ingroup reorder
  */
 namespace rcm {
 
diff --git a/hip/reorder/rcm_kernels.hip.cpp b/hip/reorder/rcm_kernels.hip.cpp
index 385ae8b9890..c02d839b4aa 100644
--- a/hip/reorder/rcm_kernels.hip.cpp
+++ b/hip/reorder/rcm_kernels.hip.cpp
@@ -50,15 +50,15 @@ namespace gko {
 namespace kernels {
 namespace hip {
 /**
- * @brief The parallel ilu factorization namespace.
+ * @brief The reordering namespace.
  *
- * @ingroup factor
+ * @ingroup reorder
  */
 namespace rcm {
 
 
 template <typename IndexType>
-void get_degree_of_nodes(std::shared_ptr<const Hipxecutor> exec,
+void get_degree_of_nodes(std::shared_ptr<const HipExecutor> exec,
                          const size_type num_vertices,
                          const IndexType *const row_ptrs,
                          IndexType *const degrees) GKO_NOT_IMPLEMENTED;
diff --git a/include/ginkgo/core/reorder/rcm.hpp b/include/ginkgo/core/reorder/rcm.hpp
index c2992ed5721..732ea73c19f 100644
--- a/include/ginkgo/core/reorder/rcm.hpp
+++ b/include/ginkgo/core/reorder/rcm.hpp
@@ -59,6 +59,7 @@ namespace gko {
  */
 namespace reorder {
 
+
 enum class starting_strategy { minimum_degree, pseudo_peripheral };
 
 
@@ -68,7 +69,7 @@ enum class starting_strategy { minimum_degree, pseudo_peripheral };
  * as effective as more complex algorithms, specifically AMD and nested
  * dissection schemes. The advantage of this algorithm is its low runtime.
  *
- * @note  This class is derives from polymorphic object but is not a LinOp as it
+ * @note  This class is derived from polymorphic object but is not a LinOp as it
  * does not make sense for this class to implement the apply methods. The
  * objective of this class is to generate a reordering/permutation vector (in
  * the form of the Permutation matrix), which can be used to apply to reorder a
@@ -205,4 +206,4 @@ class Rcm : public EnablePolymorphicObject<Rcm<ValueType, IndexType>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_REORDER_RCM_HPP_
\ No newline at end of file
+#endif  // GKO_CORE_REORDER_RCM_HPP_
diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index 6dea667674d..78a5fabaf38 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -83,8 +83,10 @@ void get_degree_of_nodes(std::shared_ptr<const OmpExecutor> exec,
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 
+
 constexpr auto chunk_bound = 512;
 
+
 template <typename IndexType>
 struct UbfsLinearQueue {
     gko::vector<IndexType> arr;
@@ -197,6 +199,7 @@ struct UbfsLinearQueue {
     }
 };
 
+
 /**
  * Performs an unordered breadth-first search,
  * thereby building a rooted level structure.
@@ -295,6 +298,7 @@ void ubfs(std::shared_ptr<const OmpExecutor> exec, const size_type num_vertices,
     }
 }
 
+
 /**
  * Finds a 'contender', meaning a node in the last level of the rls with minimum
  * degree, returns it along with the rls height.
@@ -362,6 +366,7 @@ std::pair<IndexType, IndexType> rls_contender_and_height(
     return std::make_pair(global_contender, global_height);
 }
 
+
 /**
  * Finds the index of a node with minimum degree and the maximum degree.
  */
@@ -423,6 +428,7 @@ std::pair<IndexType, IndexType> find_min_idx_and_max_val(
     return std::make_pair(global_min_idx, global_max_val);
 }
 
+
 /**
  * Finds a start node for the urcm algorithm, using parallel building blocks.
  */
@@ -486,6 +492,7 @@ IndexType find_start_node(std::shared_ptr<const OmpExecutor> exec,
     }
 }
 
+
 /**
  * Counts how many nodes there are per level.
  */
@@ -538,15 +545,15 @@ gko::vector<IndexType> count_levels(std::shared_ptr<const OmpExecutor> exec,
     gko::vector<IndexType> final_level_counts(max_size + 1, exec);
     auto i = 0;
     while (true) {
-        auto exit = true;
+        auto done = true;
         for (auto tid = 0; tid < num_threads; ++tid) {
             if (i < level_count_sizes[tid]) {
                 const auto count = level_counts[tid][i];
                 final_level_counts[i] += count;
-                exit = false;
+                done = false;
             }
         }
-        if (exit) {
+        if (done) {
             break;
         }
         ++i;
@@ -555,6 +562,7 @@ gko::vector<IndexType> count_levels(std::shared_ptr<const OmpExecutor> exec,
     return final_level_counts;
 }
 
+
 /**
  * Implements the two intermediate phases of urcm,
  * counting the nodes per level and computing the level offsets,
@@ -570,6 +578,7 @@ gko::vector<IndexType> compute_level_offsets(
     return counts;
 }
 
+
 /**
  * Helper macros for the helper function below.
  */
@@ -600,6 +609,7 @@ gko::vector<IndexType> compute_level_offsets(
     }
 #endif
 
+
 /**
  * Helper fucntion for quickly sorting small vectors by degree.
  */
@@ -651,6 +661,7 @@ inline void sort_by_degree(IndexType *const neighbours, size_type count,
               });
 }
 
+
 /**
  * Implements the last phase of urcm,
  * writing the permutation levels in parallel.
@@ -742,6 +753,7 @@ void write_permutation(std::shared_ptr<const OmpExecutor> exec,
     }
 }
 
+
 /**
  * Processes all isolated nodes, returning their count.
  */
@@ -773,8 +785,9 @@ IndexType handle_isolated_nodes(std::shared_ptr<const OmpExecutor> exec,
     auto isolated = IsolatedNodes();
 #pragma omp parallel for reduction(FindIsolated : isolated)
     for (auto i = 0; i < num_vertices; ++i) {
-        if (degrees[i] == 0
-            /* || (degrees[i] == 1 && col_idxs[row_ptrs[i]] == i) */) {
+        // No need to check for diagonal elements (only self-neighbouring) here,
+        // those are already removed from the matrix.
+        if (degrees[i] == 0) {
             isolated.nodes.push_back(i);
             previous_component[i] = true;
         }
@@ -785,6 +798,7 @@ IndexType handle_isolated_nodes(std::shared_ptr<const OmpExecutor> exec,
     return isolated.nodes.size();
 }
 
+
 /**
  * Computes a rcm permutation, employing the parallel unordered rcm algorithm.
  */
@@ -880,4 +894,4 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
 }  // namespace kernels
 }  // namespace gko
 
-#undef GKO_COMPARATOR
\ No newline at end of file
+#undef GKO_COMPARATOR

From 8efa934a5e2e895c4c00c551c0d2e3de56a23691 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Wed, 26 Aug 2020 16:21:52 +0200
Subject: [PATCH 0053/1008] Fix rcm declare reduction error

The omp reduction initializer interacted badly with the wrapper ctor.
---
 omp/reorder/rcm_kernels.cpp | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index 78a5fabaf38..ae7581d58aa 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -769,7 +769,6 @@ IndexType handle_isolated_nodes(std::shared_ptr<const OmpExecutor> exec,
         // Using a gko::vector here makes clang and gcc segfault,
         // if we then use 'omp_priv = omp_orig'.
         std::vector<IndexType> nodes;
-        IsolatedNodes() : nodes(std::vector<IndexType>(0)) {}
         IsolatedNodes &operator+=(const IsolatedNodes &rhs)
         {
             nodes.reserve(nodes.size() + rhs.nodes.size());
@@ -778,10 +777,7 @@ IndexType handle_isolated_nodes(std::shared_ptr<const OmpExecutor> exec,
         }
     };
 
-#pragma omp declare reduction(FindIsolated         \
-                              : IsolatedNodes      \
-                              : omp_out += omp_in) \
-    initializer(omp_priv = IsolatedNodes())
+#pragma omp declare reduction(FindIsolated : IsolatedNodes : omp_out += omp_in)
     auto isolated = IsolatedNodes();
 #pragma omp parallel for reduction(FindIsolated : isolated)
     for (auto i = 0; i < num_vertices; ++i) {

From fa4860a01fc14a5fabaf2d0fdea4c6a3ab875482 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Thu, 27 Aug 2020 13:49:23 +0200
Subject: [PATCH 0054/1008] Make Rcm ConvertibleTo itself

---
 include/ginkgo/core/reorder/rcm.hpp | 37 +++++++++++++++++++++++++++--
 1 file changed, 35 insertions(+), 2 deletions(-)

diff --git a/include/ginkgo/core/reorder/rcm.hpp b/include/ginkgo/core/reorder/rcm.hpp
index 732ea73c19f..62f6792bae4 100644
--- a/include/ginkgo/core/reorder/rcm.hpp
+++ b/include/ginkgo/core/reorder/rcm.hpp
@@ -92,8 +92,9 @@ enum class starting_strategy { minimum_degree, pseudo_peripheral };
  * @ingroup reorder
  */
 template <typename ValueType = default_precision, typename IndexType = int32>
-class Rcm : public EnablePolymorphicObject<Rcm<ValueType, IndexType>,
-                                           ReorderingBase> {
+class Rcm
+    : public EnablePolymorphicObject<Rcm<ValueType, IndexType>, ReorderingBase>,
+      public ConvertibleTo<Rcm<ValueType, IndexType>> {
     friend class EnablePolymorphicObject<Rcm, ReorderingBase>;
 
 public:
@@ -151,6 +152,38 @@ class Rcm : public EnablePolymorphicObject<Rcm<ValueType, IndexType>,
     GKO_ENABLE_REORDERING_BASE_FACTORY(Rcm, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);
 
+    /**
+     * Copies this Rcm into another Rcm.
+     *
+     * @param result  the object used to emplace the result of the conversion
+     */
+    void convert_to(Rcm<ValueType, IndexType> *result) const
+    {
+        result->permutation_ =
+            PermutationMatrix::create(*this->permutation_.get());
+        result->inv_permutation_ =
+            !this->inv_permutation_
+                ? nullptr
+                : PermutationMatrix::create(*this->inv_permutation_.get());
+        result->degrees_ = std::unique_ptr<gko::Array<IndexType>>(
+            new gko::Array<IndexType>(*this->degrees_.get()));
+        result->adjacency_matrix_ =
+            std::make_shared<SparsityMatrix>(*this->adjacency_matrix_.get());
+        result->parameters_.strategy = this->parameters_.strategy;
+        result->parameters_.construct_inverse_permutation =
+            this->parameters_.construct_inverse_permutation;
+    }
+
+    /**
+     * Moves this Rcm into another Rcm.
+     *
+     * @param result  the object used to emplace the result of the conversion
+     */
+    void move_to(Rcm<ValueType, IndexType> *result)
+    {
+        *result = std::move(*this);
+    }
+
 protected:
     /**
      * Generates the permutation matrix and if required the inverse permutation

From 89d5316b86cc920ba804ddb18f62a3b70d173117 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Thu, 27 Aug 2020 13:51:29 +0200
Subject: [PATCH 0055/1008] Implement standard tests for Rcm

---
 core/test/reorder/rcm.cpp | 105 +++++++++++++++++++++++++++++++++++---
 1 file changed, 99 insertions(+), 6 deletions(-)

diff --git a/core/test/reorder/rcm.cpp b/core/test/reorder/rcm.cpp
index cd36a6f6895..db19dd9eb26 100644
--- a/core/test/reorder/rcm.cpp
+++ b/core/test/reorder/rcm.cpp
@@ -50,23 +50,116 @@ class Rcm : public ::testing::Test {
 protected:
     using v_type = double;
     using i_type = int;
-    using Mtx = gko::matrix::Dense<v_type>;
-    using reorder_factory_type = gko::reorder::Rcm<v_type, i_type>;
+    using reorder_type = gko::reorder::Rcm<v_type, i_type>;
 
     Rcm()
         : exec(gko::ReferenceExecutor::create()),
-          rcm_factory(reorder_factory_type::build().on(exec))
+          rcm_factory(reorder_type::build().on(exec))
     {}
 
     std::shared_ptr<const gko::Executor> exec;
-    std::unique_ptr<reorder_factory_type::Factory> rcm_factory;
+    std::unique_ptr<reorder_type::Factory> rcm_factory;
 };
 
-
 TEST_F(Rcm, RcmFactoryKnowsItsExecutor)
 {
-    ASSERT_EQ(rcm_factory->get_executor(), exec);
+    ASSERT_EQ(rcm_factory->get_executor(), this->exec);
+}
+
+TEST_F(Rcm, CanBeCopied)
+{
+    auto rcm =
+        rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+            3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
+            this->exec));
+    auto rcm_copy =
+        rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+            3, {{1.0, 0.0, 1.0}, {0.0, 1.0, 0.0}, {1.0, 0.0, 1.0}},
+            this->exec));
+
+    rcm_copy->copy_from(rcm.get());
+
+    ASSERT_EQ(rcm_copy->get_permutation()->get_const_permutation()[0], 2);
+    ASSERT_EQ(rcm_copy->get_permutation()->get_const_permutation()[1], 1);
+    ASSERT_EQ(rcm_copy->get_permutation()->get_const_permutation()[2], 0);
+    ASSERT_NE(rcm->get_adjacency_matrix(), rcm_copy->get_adjacency_matrix());
+}
+
+
+TEST_F(Rcm, CanBeMoved)
+{
+    auto rcm =
+        rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+            3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
+            this->exec));
+    auto rcm_move =
+        rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+            3, {{1.0, 0.0, 1.0}, {0.0, 1.0, 0.0}, {1.0, 0.0, 1.0}},
+            this->exec));
+
+    rcm->move_to(rcm_move.get());
+
+    ASSERT_EQ(rcm_move->get_permutation()->get_const_permutation()[0], 2);
+    ASSERT_EQ(rcm_move->get_permutation()->get_const_permutation()[1], 1);
+    ASSERT_EQ(rcm_move->get_permutation()->get_const_permutation()[2], 0);
+}
+
+
+TEST_F(Rcm, CanBeCloned)
+{
+    auto rcm =
+        rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+            3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
+            this->exec));
+
+    auto rcm_clone = rcm->clone();
+
+    ASSERT_EQ(rcm_clone->get_permutation()->get_const_permutation()[0], 2);
+    ASSERT_EQ(rcm_clone->get_permutation()->get_const_permutation()[1], 1);
+    ASSERT_EQ(rcm_clone->get_permutation()->get_const_permutation()[2], 0);
+    ASSERT_NE(rcm->get_adjacency_matrix(), rcm_clone->get_adjacency_matrix());
+}
+
+TEST_F(Rcm, HasSensibleDefaults)
+{
+    auto rcm = reorder_type::build()
+                   .on(this->exec)
+                   ->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+                       3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
+                       this->exec));
+
+    ASSERT_EQ(rcm->get_parameters().construct_inverse_permutation, false);
+    ASSERT_EQ(rcm->get_parameters().strategy,
+              gko::reorder::starting_strategy::pseudo_peripheral);
 }
 
+TEST_F(Rcm, CanBeCreatedWithStartingStrategy)
+{
+    auto rcm =
+        reorder_type::build()
+            .with_strategy(gko::reorder::starting_strategy::minimum_degree)
+            .on(this->exec)
+            ->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+                3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
+                this->exec));
+
+    ASSERT_EQ(rcm->get_parameters().strategy,
+              gko::reorder::starting_strategy::minimum_degree);
+}
+
+TEST_F(Rcm, CanBeCreatedWithConstructInversePermutation)
+{
+    auto rcm = reorder_type::build()
+                   .with_construct_inverse_permutation(true)
+                   .on(this->exec)
+                   ->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+                       3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
+                       this->exec));
+
+    ASSERT_EQ(rcm->get_inverse_permutation()->get_const_permutation()[0], 2);
+    ASSERT_EQ(rcm->get_inverse_permutation()->get_const_permutation()[1], 1);
+    ASSERT_EQ(rcm->get_inverse_permutation()->get_const_permutation()[2], 0);
+    ASSERT_EQ(rcm->get_parameters().construct_inverse_permutation, true);
+}
 
 }  // namespace

From cac500626419ff041e93ea21899f2e84b9e2a4c4 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Thu, 27 Aug 2020 14:36:48 +0200
Subject: [PATCH 0056/1008] Use EnablePolymorphicAssignment for rcm

---
 core/test/reorder/rcm.cpp           |  2 --
 include/ginkgo/core/reorder/rcm.hpp | 36 ++---------------------------
 2 files changed, 2 insertions(+), 36 deletions(-)

diff --git a/core/test/reorder/rcm.cpp b/core/test/reorder/rcm.cpp
index db19dd9eb26..eef3af92dfd 100644
--- a/core/test/reorder/rcm.cpp
+++ b/core/test/reorder/rcm.cpp
@@ -82,7 +82,6 @@ TEST_F(Rcm, CanBeCopied)
     ASSERT_EQ(rcm_copy->get_permutation()->get_const_permutation()[0], 2);
     ASSERT_EQ(rcm_copy->get_permutation()->get_const_permutation()[1], 1);
     ASSERT_EQ(rcm_copy->get_permutation()->get_const_permutation()[2], 0);
-    ASSERT_NE(rcm->get_adjacency_matrix(), rcm_copy->get_adjacency_matrix());
 }
 
 
@@ -117,7 +116,6 @@ TEST_F(Rcm, CanBeCloned)
     ASSERT_EQ(rcm_clone->get_permutation()->get_const_permutation()[0], 2);
     ASSERT_EQ(rcm_clone->get_permutation()->get_const_permutation()[1], 1);
     ASSERT_EQ(rcm_clone->get_permutation()->get_const_permutation()[2], 0);
-    ASSERT_NE(rcm->get_adjacency_matrix(), rcm_clone->get_adjacency_matrix());
 }
 
 TEST_F(Rcm, HasSensibleDefaults)
diff --git a/include/ginkgo/core/reorder/rcm.hpp b/include/ginkgo/core/reorder/rcm.hpp
index 62f6792bae4..2c6c27d0337 100644
--- a/include/ginkgo/core/reorder/rcm.hpp
+++ b/include/ginkgo/core/reorder/rcm.hpp
@@ -94,7 +94,7 @@ enum class starting_strategy { minimum_degree, pseudo_peripheral };
 template <typename ValueType = default_precision, typename IndexType = int32>
 class Rcm
     : public EnablePolymorphicObject<Rcm<ValueType, IndexType>, ReorderingBase>,
-      public ConvertibleTo<Rcm<ValueType, IndexType>> {
+      public EnablePolymorphicAssignment<Rcm<ValueType, IndexType>> {
     friend class EnablePolymorphicObject<Rcm, ReorderingBase>;
 
 public:
@@ -152,38 +152,6 @@ class Rcm
     GKO_ENABLE_REORDERING_BASE_FACTORY(Rcm, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);
 
-    /**
-     * Copies this Rcm into another Rcm.
-     *
-     * @param result  the object used to emplace the result of the conversion
-     */
-    void convert_to(Rcm<ValueType, IndexType> *result) const
-    {
-        result->permutation_ =
-            PermutationMatrix::create(*this->permutation_.get());
-        result->inv_permutation_ =
-            !this->inv_permutation_
-                ? nullptr
-                : PermutationMatrix::create(*this->inv_permutation_.get());
-        result->degrees_ = std::unique_ptr<gko::Array<IndexType>>(
-            new gko::Array<IndexType>(*this->degrees_.get()));
-        result->adjacency_matrix_ =
-            std::make_shared<SparsityMatrix>(*this->adjacency_matrix_.get());
-        result->parameters_.strategy = this->parameters_.strategy;
-        result->parameters_.construct_inverse_permutation =
-            this->parameters_.construct_inverse_permutation;
-    }
-
-    /**
-     * Moves this Rcm into another Rcm.
-     *
-     * @param result  the object used to emplace the result of the conversion
-     */
-    void move_to(Rcm<ValueType, IndexType> *result)
-    {
-        *result = std::move(*this);
-    }
-
 protected:
     /**
      * Generates the permutation matrix and if required the inverse permutation
@@ -230,7 +198,7 @@ class Rcm
 private:
     std::shared_ptr<SparsityMatrix> adjacency_matrix_;
     std::shared_ptr<PermutationMatrix> permutation_;
-    std::unique_ptr<Array<IndexType>> degrees_;
+    std::shared_ptr<Array<IndexType>> degrees_;
     std::shared_ptr<PermutationMatrix> inv_permutation_;
 };
 

From 24ff2979068f1025a6b655a3a4fba027b6734653 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Thu, 27 Aug 2020 15:00:34 +0200
Subject: [PATCH 0057/1008] Make rcm tests TYPED_TESTs

---
 core/test/reorder/rcm.cpp | 51 ++++++++++++++++++++++++++-------------
 1 file changed, 34 insertions(+), 17 deletions(-)

diff --git a/core/test/reorder/rcm.cpp b/core/test/reorder/rcm.cpp
index eef3af92dfd..88669d99b02 100644
--- a/core/test/reorder/rcm.cpp
+++ b/core/test/reorder/rcm.cpp
@@ -43,13 +43,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
-namespace {
+#include "core/test/utils.hpp"
+
 
+namespace {
 
+template <typename ValueIndexType>
 class Rcm : public ::testing::Test {
 protected:
-    using v_type = double;
-    using i_type = int;
+    using v_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using i_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using reorder_type = gko::reorder::Rcm<v_type, i_type>;
 
     Rcm()
@@ -58,22 +63,25 @@ class Rcm : public ::testing::Test {
     {}
 
     std::shared_ptr<const gko::Executor> exec;
-    std::unique_ptr<reorder_type::Factory> rcm_factory;
+    std::unique_ptr<typename reorder_type::Factory> rcm_factory;
 };
 
-TEST_F(Rcm, RcmFactoryKnowsItsExecutor)
+TYPED_TEST_CASE(Rcm, gko::test::ValueIndexTypes);
+
+TYPED_TEST(Rcm, RcmFactoryKnowsItsExecutor)
 {
-    ASSERT_EQ(rcm_factory->get_executor(), this->exec);
+    ASSERT_EQ(this->rcm_factory->get_executor(), this->exec);
 }
 
-TEST_F(Rcm, CanBeCopied)
+TYPED_TEST(Rcm, CanBeCopied)
 {
+    using v_type = typename TestFixture::v_type;
     auto rcm =
-        rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
             3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
             this->exec));
     auto rcm_copy =
-        rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
             3, {{1.0, 0.0, 1.0}, {0.0, 1.0, 0.0}, {1.0, 0.0, 1.0}},
             this->exec));
 
@@ -85,14 +93,15 @@ TEST_F(Rcm, CanBeCopied)
 }
 
 
-TEST_F(Rcm, CanBeMoved)
+TYPED_TEST(Rcm, CanBeMoved)
 {
+    using v_type = typename TestFixture::v_type;
     auto rcm =
-        rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
             3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
             this->exec));
     auto rcm_move =
-        rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
             3, {{1.0, 0.0, 1.0}, {0.0, 1.0, 0.0}, {1.0, 0.0, 1.0}},
             this->exec));
 
@@ -104,10 +113,11 @@ TEST_F(Rcm, CanBeMoved)
 }
 
 
-TEST_F(Rcm, CanBeCloned)
+TYPED_TEST(Rcm, CanBeCloned)
 {
+    using v_type = typename TestFixture::v_type;
     auto rcm =
-        rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
             3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
             this->exec));
 
@@ -118,8 +128,11 @@ TEST_F(Rcm, CanBeCloned)
     ASSERT_EQ(rcm_clone->get_permutation()->get_const_permutation()[2], 0);
 }
 
-TEST_F(Rcm, HasSensibleDefaults)
+TYPED_TEST(Rcm, HasSensibleDefaults)
 {
+    using v_type = typename TestFixture::v_type;
+    using reorder_type = typename TestFixture::reorder_type;
+
     auto rcm = reorder_type::build()
                    .on(this->exec)
                    ->generate(gko::initialize<gko::matrix::Dense<v_type>>(
@@ -131,8 +144,10 @@ TEST_F(Rcm, HasSensibleDefaults)
               gko::reorder::starting_strategy::pseudo_peripheral);
 }
 
-TEST_F(Rcm, CanBeCreatedWithStartingStrategy)
+TYPED_TEST(Rcm, CanBeCreatedWithStartingStrategy)
 {
+    using v_type = typename TestFixture::v_type;
+    using reorder_type = typename TestFixture::reorder_type;
     auto rcm =
         reorder_type::build()
             .with_strategy(gko::reorder::starting_strategy::minimum_degree)
@@ -145,8 +160,10 @@ TEST_F(Rcm, CanBeCreatedWithStartingStrategy)
               gko::reorder::starting_strategy::minimum_degree);
 }
 
-TEST_F(Rcm, CanBeCreatedWithConstructInversePermutation)
+TYPED_TEST(Rcm, CanBeCreatedWithConstructInversePermutation)
 {
+    using v_type = typename TestFixture::v_type;
+    using reorder_type = typename TestFixture::reorder_type;
     auto rcm = reorder_type::build()
                    .with_construct_inverse_permutation(true)
                    .on(this->exec)

From e3f29e00415e0f4675a788d3f48a8f980fbe7205 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Thu, 27 Aug 2020 16:03:41 +0200
Subject: [PATCH 0058/1008] Move rcm tests to reference

---
 core/test/reorder/rcm.cpp      | 117 +---------------------------
 reference/test/reorder/rcm.cpp | 134 +++++++++++++++++++++++++++++----
 2 files changed, 125 insertions(+), 126 deletions(-)

diff --git a/core/test/reorder/rcm.cpp b/core/test/reorder/rcm.cpp
index 88669d99b02..2f452f09ae7 100644
--- a/core/test/reorder/rcm.cpp
+++ b/core/test/reorder/rcm.cpp
@@ -48,13 +48,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace {
 
-template <typename ValueIndexType>
 class Rcm : public ::testing::Test {
 protected:
-    using v_type =
-        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
-    using i_type =
-        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using v_type = double;
+    using i_type = int;
     using reorder_type = gko::reorder::Rcm<v_type, i_type>;
 
     Rcm()
@@ -63,118 +60,12 @@ class Rcm : public ::testing::Test {
     {}
 
     std::shared_ptr<const gko::Executor> exec;
-    std::unique_ptr<typename reorder_type::Factory> rcm_factory;
+    std::unique_ptr<reorder_type::Factory> rcm_factory;
 };
 
-TYPED_TEST_CASE(Rcm, gko::test::ValueIndexTypes);
-
-TYPED_TEST(Rcm, RcmFactoryKnowsItsExecutor)
+TEST_F(Rcm, RcmFactoryKnowsItsExecutor)
 {
     ASSERT_EQ(this->rcm_factory->get_executor(), this->exec);
 }
 
-TYPED_TEST(Rcm, CanBeCopied)
-{
-    using v_type = typename TestFixture::v_type;
-    auto rcm =
-        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-            3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
-            this->exec));
-    auto rcm_copy =
-        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-            3, {{1.0, 0.0, 1.0}, {0.0, 1.0, 0.0}, {1.0, 0.0, 1.0}},
-            this->exec));
-
-    rcm_copy->copy_from(rcm.get());
-
-    ASSERT_EQ(rcm_copy->get_permutation()->get_const_permutation()[0], 2);
-    ASSERT_EQ(rcm_copy->get_permutation()->get_const_permutation()[1], 1);
-    ASSERT_EQ(rcm_copy->get_permutation()->get_const_permutation()[2], 0);
-}
-
-
-TYPED_TEST(Rcm, CanBeMoved)
-{
-    using v_type = typename TestFixture::v_type;
-    auto rcm =
-        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-            3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
-            this->exec));
-    auto rcm_move =
-        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-            3, {{1.0, 0.0, 1.0}, {0.0, 1.0, 0.0}, {1.0, 0.0, 1.0}},
-            this->exec));
-
-    rcm->move_to(rcm_move.get());
-
-    ASSERT_EQ(rcm_move->get_permutation()->get_const_permutation()[0], 2);
-    ASSERT_EQ(rcm_move->get_permutation()->get_const_permutation()[1], 1);
-    ASSERT_EQ(rcm_move->get_permutation()->get_const_permutation()[2], 0);
-}
-
-
-TYPED_TEST(Rcm, CanBeCloned)
-{
-    using v_type = typename TestFixture::v_type;
-    auto rcm =
-        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-            3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
-            this->exec));
-
-    auto rcm_clone = rcm->clone();
-
-    ASSERT_EQ(rcm_clone->get_permutation()->get_const_permutation()[0], 2);
-    ASSERT_EQ(rcm_clone->get_permutation()->get_const_permutation()[1], 1);
-    ASSERT_EQ(rcm_clone->get_permutation()->get_const_permutation()[2], 0);
-}
-
-TYPED_TEST(Rcm, HasSensibleDefaults)
-{
-    using v_type = typename TestFixture::v_type;
-    using reorder_type = typename TestFixture::reorder_type;
-
-    auto rcm = reorder_type::build()
-                   .on(this->exec)
-                   ->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-                       3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
-                       this->exec));
-
-    ASSERT_EQ(rcm->get_parameters().construct_inverse_permutation, false);
-    ASSERT_EQ(rcm->get_parameters().strategy,
-              gko::reorder::starting_strategy::pseudo_peripheral);
-}
-
-TYPED_TEST(Rcm, CanBeCreatedWithStartingStrategy)
-{
-    using v_type = typename TestFixture::v_type;
-    using reorder_type = typename TestFixture::reorder_type;
-    auto rcm =
-        reorder_type::build()
-            .with_strategy(gko::reorder::starting_strategy::minimum_degree)
-            .on(this->exec)
-            ->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-                3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
-                this->exec));
-
-    ASSERT_EQ(rcm->get_parameters().strategy,
-              gko::reorder::starting_strategy::minimum_degree);
-}
-
-TYPED_TEST(Rcm, CanBeCreatedWithConstructInversePermutation)
-{
-    using v_type = typename TestFixture::v_type;
-    using reorder_type = typename TestFixture::reorder_type;
-    auto rcm = reorder_type::build()
-                   .with_construct_inverse_permutation(true)
-                   .on(this->exec)
-                   ->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-                       3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
-                       this->exec));
-
-    ASSERT_EQ(rcm->get_inverse_permutation()->get_const_permutation()[0], 2);
-    ASSERT_EQ(rcm->get_inverse_permutation()->get_const_permutation()[1], 1);
-    ASSERT_EQ(rcm->get_inverse_permutation()->get_const_permutation()[2], 0);
-    ASSERT_EQ(rcm->get_parameters().construct_inverse_permutation, true);
-}
-
 }  // namespace
diff --git a/reference/test/reorder/rcm.cpp b/reference/test/reorder/rcm.cpp
index b6c0bc29b5c..89010bbcafa 100644
--- a/reference/test/reorder/rcm.cpp
+++ b/reference/test/reorder/rcm.cpp
@@ -54,14 +54,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace {
 
-
+template <typename ValueIndexType>
 class Rcm : public ::testing::Test {
 protected:
-    using v_type = double;
-    using i_type = int;
+    using v_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using i_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using reorder_type = gko::reorder::Rcm<v_type, i_type>;
     using Mtx = gko::matrix::Dense<v_type>;
     using CsrMtx = gko::matrix::Csr<v_type, i_type>;
-    using reorder_type = gko::reorder::Rcm<v_type, i_type>;
     Rcm()
         : exec(gko::ReferenceExecutor::create()),
           rcm_factory(reorder_type::build().on(exec)),
@@ -79,29 +81,135 @@ class Rcm : public ::testing::Test {
 
     std::shared_ptr<const gko::Executor> exec;
     std::shared_ptr<CsrMtx> p_mtx;
-    std::unique_ptr<reorder_type::Factory> rcm_factory;
+    std::unique_ptr<typename reorder_type::Factory> rcm_factory;
     std::unique_ptr<reorder_type> reorder_op;
 };
 
+TYPED_TEST_CASE(Rcm, gko::test::ValueIndexTypes);
 
-TEST_F(Rcm, FactoryCreatesCorrectReorderOp)
+TYPED_TEST(Rcm, FactoryCreatesCorrectReorderOp)
 {
-    auto adj_mtx = reorder_op->get_adjacency_matrix();
-    auto tmp = gko::matrix::SparsityCsr<v_type, i_type>::create(exec, p_mtx);
+    using i_type = typename TestFixture::i_type;
+    using v_type = typename TestFixture::v_type;
+    auto adj_mtx = this->reorder_op->get_adjacency_matrix();
+    auto tmp = gko::matrix::SparsityCsr<v_type, i_type>::create(this->exec,
+                                                                this->p_mtx);
     auto comp_mtx = tmp->to_adjacency_matrix();
 
-    ASSERT_NE(reorder_op->get_adjacency_matrix(), nullptr);
+    ASSERT_NE(this->reorder_op->get_adjacency_matrix(), nullptr);
     GKO_ASSERT_MTX_NEAR(adj_mtx.get(), comp_mtx.get(), 0);
 }
 
-
-TEST_F(Rcm, CanBeCleared)
+TYPED_TEST(Rcm, CanBeCleared)
 {
-    reorder_op->clear();
+    this->reorder_op->clear();
 
-    auto reorder_op_mtx = reorder_op->get_adjacency_matrix();
+    auto reorder_op_mtx = this->reorder_op->get_adjacency_matrix();
     ASSERT_EQ(reorder_op_mtx, nullptr);
 }
 
+TYPED_TEST(Rcm, CanBeCopied)
+{
+    using v_type = typename TestFixture::v_type;
+    auto rcm =
+        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+            3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
+            this->exec));
+    auto rcm_copy =
+        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+            3, {{1.0, 0.0, 1.0}, {0.0, 1.0, 0.0}, {1.0, 0.0, 1.0}},
+            this->exec));
+
+    rcm_copy->copy_from(rcm.get());
+
+    ASSERT_EQ(rcm_copy->get_permutation()->get_const_permutation()[0], 2);
+    ASSERT_EQ(rcm_copy->get_permutation()->get_const_permutation()[1], 1);
+    ASSERT_EQ(rcm_copy->get_permutation()->get_const_permutation()[2], 0);
+}
+
+
+TYPED_TEST(Rcm, CanBeMoved)
+{
+    using v_type = typename TestFixture::v_type;
+    auto rcm =
+        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+            3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
+            this->exec));
+    auto rcm_move =
+        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+            3, {{1.0, 0.0, 1.0}, {0.0, 1.0, 0.0}, {1.0, 0.0, 1.0}},
+            this->exec));
+
+    rcm->move_to(rcm_move.get());
+
+    ASSERT_EQ(rcm_move->get_permutation()->get_const_permutation()[0], 2);
+    ASSERT_EQ(rcm_move->get_permutation()->get_const_permutation()[1], 1);
+    ASSERT_EQ(rcm_move->get_permutation()->get_const_permutation()[2], 0);
+}
+
+
+TYPED_TEST(Rcm, CanBeCloned)
+{
+    using v_type = typename TestFixture::v_type;
+    auto rcm =
+        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+            3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
+            this->exec));
+
+    auto rcm_clone = rcm->clone();
+
+    ASSERT_EQ(rcm_clone->get_permutation()->get_const_permutation()[0], 2);
+    ASSERT_EQ(rcm_clone->get_permutation()->get_const_permutation()[1], 1);
+    ASSERT_EQ(rcm_clone->get_permutation()->get_const_permutation()[2], 0);
+}
+
+TYPED_TEST(Rcm, HasSensibleDefaults)
+{
+    using v_type = typename TestFixture::v_type;
+    using reorder_type = typename TestFixture::reorder_type;
+
+    auto rcm = reorder_type::build()
+                   .on(this->exec)
+                   ->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+                       3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
+                       this->exec));
+
+    ASSERT_EQ(rcm->get_parameters().construct_inverse_permutation, false);
+    ASSERT_EQ(rcm->get_parameters().strategy,
+              gko::reorder::starting_strategy::pseudo_peripheral);
+}
+
+TYPED_TEST(Rcm, CanBeCreatedWithStartingStrategy)
+{
+    using v_type = typename TestFixture::v_type;
+    using reorder_type = typename TestFixture::reorder_type;
+    auto rcm =
+        reorder_type::build()
+            .with_strategy(gko::reorder::starting_strategy::minimum_degree)
+            .on(this->exec)
+            ->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+                3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
+                this->exec));
+
+    ASSERT_EQ(rcm->get_parameters().strategy,
+              gko::reorder::starting_strategy::minimum_degree);
+}
+
+TYPED_TEST(Rcm, CanBeCreatedWithConstructInversePermutation)
+{
+    using v_type = typename TestFixture::v_type;
+    using reorder_type = typename TestFixture::reorder_type;
+    auto rcm = reorder_type::build()
+                   .with_construct_inverse_permutation(true)
+                   .on(this->exec)
+                   ->generate(gko::initialize<gko::matrix::Dense<v_type>>(
+                       3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
+                       this->exec));
+
+    ASSERT_EQ(rcm->get_inverse_permutation()->get_const_permutation()[0], 2);
+    ASSERT_EQ(rcm->get_inverse_permutation()->get_const_permutation()[1], 1);
+    ASSERT_EQ(rcm->get_inverse_permutation()->get_const_permutation()[2], 0);
+    ASSERT_EQ(rcm->get_parameters().construct_inverse_permutation, true);
+}
 
 }  // namespace

From 8f3bec0ec4a84297e47f0b41d43267f1d48939cb Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Thu, 27 Aug 2020 19:45:48 +0200
Subject: [PATCH 0059/1008] Remove include of <immintrin.h>

---
 omp/reorder/rcm_kernels.cpp | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index ae7581d58aa..ad10b208671 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -35,7 +35,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/prefix_sum.hpp"
 
 
-#include <immintrin.h>
 #include <algorithm>
 #include <cstring>
 #include <iterator>
@@ -69,6 +68,12 @@ namespace omp {
 namespace rcm {
 
 
+#if defined(__x86_64__) && (defined(__GNUG__) || defined(__clang__)))
+#define GKO_MM_PAUSE() asm("rep nop" :::);
+#else
+#define GKO_MM_PAUSE()  // No equivalent instruction.
+#endif
+
 template <typename IndexType>
 void get_degree_of_nodes(std::shared_ptr<const OmpExecutor> exec,
                          const size_type num_vertices,
@@ -177,7 +182,7 @@ struct UbfsLinearQueue {
                 }
 
                 // No measureable effect on performance.
-                _mm_pause();
+                GKO_MM_PAUSE();
             }
 
             // Recalculate the chunk size, now that all threads are finished.
@@ -582,7 +587,7 @@ gko::vector<IndexType> compute_level_offsets(
 /**
  * Helper macros for the helper function below.
  */
-#ifdef __GNUG__
+#if defined(__GNUG__) && defined(__x86_64__)
 // GCC can only be convinced to use cmovs here by inline asm.
 #define GKO_COMPARATOR(x, y)                                       \
     {                                                              \
@@ -699,7 +704,7 @@ void write_permutation(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp atomic read
                 written = perm[base_offset + read_offset];
                 while (written == std::numeric_limits<IndexType>::max()) {
-                    _mm_pause();
+                    GKO_MM_PAUSE();
 #pragma omp atomic read
                     written = perm[base_offset + read_offset];
                 }
@@ -890,4 +895,5 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
 }  // namespace kernels
 }  // namespace gko
 
+#undef GKO_MM_PAUSE
 #undef GKO_COMPARATOR

From 1e8f239e07a50f92b3f30a6f733c2c2661e1e603 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Thu, 27 Aug 2020 20:47:55 +0200
Subject: [PATCH 0060/1008] Fix typo extra parenthesis

---
 omp/reorder/rcm_kernels.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index ad10b208671..0b27e591817 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -68,7 +68,7 @@ namespace omp {
 namespace rcm {
 
 
-#if defined(__x86_64__) && (defined(__GNUG__) || defined(__clang__)))
+#if defined(__x86_64__) && (defined(__GNUG__) || defined(__clang__))
 #define GKO_MM_PAUSE() asm("rep nop" :::);
 #else
 #define GKO_MM_PAUSE()  // No equivalent instruction.

From e5450d8942224e90849afe8b0208309df8164f50 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Fri, 28 Aug 2020 09:40:41 +0200
Subject: [PATCH 0061/1008] Explicitly delete copy ctor of UbfsLinearQueue

The copy assignment is also explicitly deleted.
Move operators will be implicitly deleted.
---
 omp/reorder/rcm_kernels.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index 0b27e591817..d53cbbfd663 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -118,6 +118,9 @@ struct UbfsLinearQueue {
         omp_destroy_lock(&write_lock);
     }
 
+    UbfsLinearQueue(UbfsLinearQueue &other) = delete;
+    UbfsLinearQueue &operator=(const UbfsLinearQueue &other) = delete;
+
     /**
      * Copies a chunk of nodes back into the work queue,
      * in a thread-safe manner.

From 02c800e0677ce29d8afdfa70257932f3357540e4 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Fri, 28 Aug 2020 10:29:40 +0200
Subject: [PATCH 0062/1008] Conditionally include immintrin.h for rcm

See also the discussion under #500.
---
 omp/reorder/rcm_kernels.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index d53cbbfd663..042ec3e1a1a 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -68,8 +68,11 @@ namespace omp {
 namespace rcm {
 
 
-#if defined(__x86_64__) && (defined(__GNUG__) || defined(__clang__))
-#define GKO_MM_PAUSE() asm("rep nop" :::);
+#ifdef __x86_64__
+#if defined(__GNUG__) || defined(__clang__)
+#include <immintrin.h>
+#endif
+#define GKO_MM_PAUSE() _mm_pause()
 #else
 #define GKO_MM_PAUSE()  // No equivalent instruction.
 #endif

From 65bf265baafb89a6a2942c55e51c1e90a08e0203 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Fri, 28 Aug 2020 15:28:59 +0200
Subject: [PATCH 0063/1008] Apply suggestions from code review

Everything following are changes to the rcm implementation.
Remove unneccessary consts from rcm declaration.
Do not store adjacency matrix and degrees as members in the class.
Replace explicit gko::vector with vector.
replace std::memcpy with std::copy_n.
Various documentation improvments.
Reverse description order of return paramters.
Make IndexType explicit instead of auto.
Use std::min_elelemt
Fix various spelling errors, typos, rewordings.
Removes the 'explicit' from the ExectorAllocators rebinding construtor.
Changes occurences of array<bool> to vector<bool>, to save memory.
Minor cleanup
Remove the unnecessary test for the rcm adjacency matrix.
Add a test case for the correct rcm result.
Rewrite the assert_correct_permutation function to comparing with iota.
Move test matrices to test class.
Improve nested vector initialization.
Allocate degrees inside rcm::generate.
Change from goto to immediately evaluated lambda expression.
Refactor loop body into an inline helper function.
Relace some autos.
Reorganize includes to conform to include order.
Replace autos with IndexType where necessary.
Replace size_type with IndexType for num_vertices.
Refactor for the number of levels to be more explicit.
Make perm signal value -1.
Factor out sort_by_degree in a generalized, fast small sorting function.
Make level_processed a constexpr.
In the small_sort make some types explicit.
Wrap omp locks in a omp_mutex, use RAII guards.

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/base/allocator.hpp                |   2 +-
 core/reorder/rcm.cpp                   |  12 +-
 core/reorder/rcm_kernels.hpp           |  18 +-
 cuda/reorder/rcm_kernels.cu            |   4 +-
 hip/reorder/rcm_kernels.hip.cpp        |   4 +-
 include/ginkgo/core/reorder/rcm.hpp    |  31 +-
 omp/components/omp_mutex.hpp           |  62 ++++
 omp/components/sort_small.hpp          | 190 ++++++++++++
 omp/reorder/rcm_kernels.cpp            | 408 +++++++++++--------------
 omp/test/reorder/rcm_kernels.cpp       |   7 +-
 reference/reorder/rcm_kernels.cpp      | 103 +++----
 reference/test/reorder/rcm.cpp         |  74 ++---
 reference/test/reorder/rcm_kernels.cpp |  61 ++--
 13 files changed, 548 insertions(+), 428 deletions(-)
 create mode 100644 omp/components/omp_mutex.hpp
 create mode 100644 omp/components/sort_small.hpp

diff --git a/core/base/allocator.hpp b/core/base/allocator.hpp
index 0c62f5deccb..3f618974b46 100644
--- a/core/base/allocator.hpp
+++ b/core/base/allocator.hpp
@@ -86,7 +86,7 @@ class ExecutorAllocator {
      * @tparam U  the element type of the allocator to be constructed.
      */
     template <typename U>
-    explicit ExecutorAllocator(const ExecutorAllocator<U> &other)
+    ExecutorAllocator(const ExecutorAllocator<U> &other)
         : exec_{other.get_executor()}
     {}
 
diff --git a/core/reorder/rcm.cpp b/core/reorder/rcm.cpp
index 584b6bfe1f6..fcac288039f 100644
--- a/core/reorder/rcm.cpp
+++ b/core/reorder/rcm.cpp
@@ -64,18 +64,20 @@ GKO_REGISTER_OPERATION(get_degree_of_nodes, rcm::get_degree_of_nodes);
 
 
 template <typename ValueType, typename IndexType>
-void Rcm<ValueType, IndexType>::generate() const
+void Rcm<ValueType, IndexType>::generate(
+    std::unique_ptr<SparsityMatrix> adjacency_matrix) const
 {
-    const IndexType num_rows = adjacency_matrix_->get_size()[0];
+    const IndexType num_rows = adjacency_matrix->get_size()[0];
+    const auto mtx = adjacency_matrix.get();
     const auto exec = this->get_executor();
-    const auto mtx = adjacency_matrix_.get();
+    auto degrees = Array<IndexType>(exec, num_rows);
     // RCM is only valid for symmetric matrices. Need to add an expensive check
     // for symmetricity here ?
     exec->run(rcm::make_get_degree_of_nodes(num_rows, mtx->get_const_row_ptrs(),
-                                            degrees_->get_data()));
+                                            degrees.get_data()));
     exec->run(rcm::make_get_permutation(
         num_rows, mtx->get_const_row_ptrs(), mtx->get_const_col_idxs(),
-        degrees_->get_const_data(), permutation_->get_permutation(),
+        degrees.get_const_data(), permutation_->get_permutation(),
         inv_permutation_.get() ? inv_permutation_->get_permutation() : nullptr,
         parameters_.strategy));
 }
diff --git a/core/reorder/rcm_kernels.hpp b/core/reorder/rcm_kernels.hpp
index e0eda389b8f..9c8b049a280 100644
--- a/core/reorder/rcm_kernels.hpp
+++ b/core/reorder/rcm_kernels.hpp
@@ -52,19 +52,17 @@ namespace gko {
 namespace kernels {
 
 
-#define GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL(IndexType)                \
-    void get_permutation(                                                \
-        std::shared_ptr<const DefaultExecutor> exec,                     \
-        const size_type num_vertices, const IndexType *const row_ptrs,   \
-        const IndexType *const col_idxs, const IndexType *const degrees, \
-        IndexType *const permutation, IndexType *const inv_permutation,  \
-        const gko::reorder::starting_strategy strategy)
+#define GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL(IndexType)                     \
+    void get_permutation(std::shared_ptr<const DefaultExecutor> exec,         \
+                         IndexType num_vertices, const IndexType *row_ptrs,   \
+                         const IndexType *col_idxs, const IndexType *degrees, \
+                         IndexType *permutation, IndexType *inv_permutation,  \
+                         gko::reorder::starting_strategy strategy)
 
 #define GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL(IndexType)             \
     void get_degree_of_nodes(std::shared_ptr<const DefaultExecutor> exec, \
-                             const size_type num_vertices,                \
-                             const IndexType *const row_ptrs,             \
-                             IndexType *const degrees)
+                             IndexType num_vertices,                      \
+                             const IndexType *row_ptrs, IndexType *degrees)
 
 #define GKO_DECLARE_ALL_AS_TEMPLATES                       \
     template <typename IndexType>                          \
diff --git a/cuda/reorder/rcm_kernels.cu b/cuda/reorder/rcm_kernels.cu
index 1dce73ce28d..e1f35511178 100644
--- a/cuda/reorder/rcm_kernels.cu
+++ b/cuda/reorder/rcm_kernels.cu
@@ -59,7 +59,7 @@ namespace rcm {
 
 template <typename IndexType>
 void get_degree_of_nodes(std::shared_ptr<const CudaExecutor> exec,
-                         const size_type num_vertices,
+                         const IndexType num_vertices,
                          const IndexType *const row_ptrs,
                          IndexType *const degrees) GKO_NOT_IMPLEMENTED;
 
@@ -68,7 +68,7 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 
 template <typename IndexType>
 void get_permutation(
-    std::shared_ptr<const CudaExecutor> exec, const size_type num_vertices,
+    std::shared_ptr<const CudaExecutor> exec, const IndexType num_vertices,
     const IndexType *const row_ptrs, const IndexType *const col_idxs,
     const IndexType *const degrees, IndexType *const permutation,
     IndexType *const inv_permutation,
diff --git a/hip/reorder/rcm_kernels.hip.cpp b/hip/reorder/rcm_kernels.hip.cpp
index c02d839b4aa..fe07008d2c9 100644
--- a/hip/reorder/rcm_kernels.hip.cpp
+++ b/hip/reorder/rcm_kernels.hip.cpp
@@ -59,7 +59,7 @@ namespace rcm {
 
 template <typename IndexType>
 void get_degree_of_nodes(std::shared_ptr<const HipExecutor> exec,
-                         const size_type num_vertices,
+                         const IndexType num_vertices,
                          const IndexType *const row_ptrs,
                          IndexType *const degrees) GKO_NOT_IMPLEMENTED;
 
@@ -68,7 +68,7 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 
 template <typename IndexType>
 void get_permutation(
-    std::shared_ptr<const HipExecutor> exec, const size_type num_vertices,
+    std::shared_ptr<const HipExecutor> exec, const IndexType num_vertices,
     const IndexType *const row_ptrs, const IndexType *const col_idxs,
     const IndexType *const degrees, IndexType *const permutation,
     IndexType *const inv_permutation,
diff --git a/include/ginkgo/core/reorder/rcm.hpp b/include/ginkgo/core/reorder/rcm.hpp
index 2c6c27d0337..28629bcbd4e 100644
--- a/include/ginkgo/core/reorder/rcm.hpp
+++ b/include/ginkgo/core/reorder/rcm.hpp
@@ -101,17 +101,6 @@ class Rcm
     using SparsityMatrix = matrix::SparsityCsr<ValueType, IndexType>;
     using PermutationMatrix = matrix::Permutation<IndexType>;
 
-    /**
-     * Gets the adjacency matrix representation of the system operator (input
-     * matrix) of the linear operator.
-     *
-     * @return the system operator (matrix)
-     */
-    std::shared_ptr<const SparsityMatrix> get_adjacency_matrix() const
-    {
-        return adjacency_matrix_;
-    }
-
     /**
      * Gets the permutation (permutation matrix, output of the algorithm) of the
      * linear operator.
@@ -157,7 +146,7 @@ class Rcm
      * Generates the permutation matrix and if required the inverse permutation
      * matrix.
      */
-    void generate() const;
+    void generate(std::unique_ptr<SparsityMatrix> adjacency_matrix) const;
 
     explicit Rcm(std::shared_ptr<const Executor> exec)
         : EnablePolymorphicObject<Rcm, ReorderingBase>(std::move(exec))
@@ -168,22 +157,22 @@ class Rcm
           parameters_{factory->get_parameters()}
     {
         const auto exec = this->get_executor();
+        auto adjacency_matrix = SparsityMatrix::create(exec);
+        Array<IndexType> degrees;
+
         // The adjacency matrix has to be square.
         GKO_ASSERT_IS_SQUARE_MATRIX(args.system_matrix);
         // This is needed because it does not make sense to call the copy and
         // convert if the existing matrix is empty.
-        if (!args.system_matrix->get_size()) {
-            adjacency_matrix_ = SparsityMatrix::create(exec);
-        } else {
+        if (args.system_matrix->get_size()) {
             auto tmp =
                 copy_and_convert_to<SparsityMatrix>(exec, args.system_matrix);
             // This function provided within the Sparsity matrix format removes
             // the diagonal elements and outputs an adjacency matrix.
-            adjacency_matrix_ = tmp->to_adjacency_matrix();
+            adjacency_matrix = tmp->to_adjacency_matrix();
         }
-        auto const dim = adjacency_matrix_->get_size();
-        degrees_ = std::unique_ptr<Array<IndexType>>(
-            new Array<IndexType>(exec, dim[0]));
+
+        auto const dim = adjacency_matrix->get_size();
         permutation_ = PermutationMatrix::create(exec, dim);
 
         // To make it explicit.
@@ -192,13 +181,11 @@ class Rcm
             inv_permutation_ = PermutationMatrix::create(exec, dim);
         }
 
-        this->generate();
+        this->generate(std::move(adjacency_matrix));
     }
 
 private:
-    std::shared_ptr<SparsityMatrix> adjacency_matrix_;
     std::shared_ptr<PermutationMatrix> permutation_;
-    std::shared_ptr<Array<IndexType>> degrees_;
     std::shared_ptr<PermutationMatrix> inv_permutation_;
 };
 
diff --git a/omp/components/omp_mutex.hpp b/omp/components/omp_mutex.hpp
new file mode 100644
index 00000000000..5f195be50ec
--- /dev/null
+++ b/omp/components/omp_mutex.hpp
@@ -0,0 +1,62 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <omp.h>
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+
+
+struct omp_mutex {
+    omp_mutex() { omp_init_lock(&lock_); }
+    omp_mutex(omp_lock_hint_t hint) { omp_init_lock_with_hint(&lock_, hint); }
+    ~omp_mutex() { omp_destroy_lock(&lock_); }
+
+    omp_mutex(const omp_mutex &) = delete;
+    omp_mutex(omp_mutex &&) = delete;
+    omp_mutex &operator=(const omp_mutex &) = delete;
+    omp_mutex &operator=(omp_mutex &&) = delete;
+
+    void lock() { omp_set_lock(&lock_); }
+
+    void unlock() { omp_unset_lock(&lock_); }
+
+private:
+    omp_lock_t lock_;
+};
+
+
+}  // namespace omp
+}  // namespace kernels
+}  // namespace gko
diff --git a/omp/components/sort_small.hpp b/omp/components/sort_small.hpp
new file mode 100644
index 00000000000..186de94907f
--- /dev/null
+++ b/omp/components/sort_small.hpp
@@ -0,0 +1,190 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <algorithm>
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+
+
+#ifdef __GNUG__
+#define GKO_GCC_EXPECT_LIKELY(v) __builtin_expect(v, true)
+#else
+#define GKO_GCC_EXPECT_LIKELY(v) v
+#endif
+
+#ifdef _MSC_VER
+#define GKO_COMPARATOR(x, y)                            \
+    {                                                   \
+        const auto should_swap = !comp(arr[x], arr[y]); \
+        const auto tmp = arr[x];                        \
+        arr[x] = should_swap ? arr[y] : arr[x];         \
+        arr[y] = should_swap ? tmp : arr[y];            \
+    }
+#else
+#define GKO_COMPARATOR(x, y)                                         \
+    {                                                                \
+        const auto should_swap = !comp(arr[x], arr[y]);              \
+        const auto tx = arr[x];                                      \
+        const auto ty = arr[y];                                      \
+        arr[x] = ((-(should_swap)) & ty) | ((should_swap - 1) & tx); \
+        arr[y] = ((-(should_swap)) & tx) | ((should_swap - 1) & ty); \
+    }
+#endif
+
+
+/**
+ * Sorts elements, such that comp(arr[i]), arr[i + 1]) is true for all
+ * i up to n - 1.
+ * Features a fast path for n < 128, especially performant for n < 8.
+ * For larger sizes delegates to std::sort.
+ */
+template <typename TargetType, typename Functor>
+void sort_small(TargetType *arr, size_type n, Functor comp)
+{
+    if (n < 2) {
+        return;
+    }
+    if (n == 2) {
+        GKO_COMPARATOR(0, 1);
+        return;
+    }
+    if (n == 3) {
+        GKO_COMPARATOR(1, 2);
+        GKO_COMPARATOR(0, 2);
+        GKO_COMPARATOR(0, 1);
+        return;
+    }
+    if (n == 4) {
+        // clang-format off
+        GKO_COMPARATOR(0, 1)GKO_COMPARATOR(2, 3)
+        GKO_COMPARATOR(0, 2)GKO_COMPARATOR(1, 3)
+        GKO_COMPARATOR(1, 2);
+        // clang-format on
+        return;
+    }
+    if (n == 5) {
+        // clang-format off
+        GKO_COMPARATOR(0, 4)GKO_COMPARATOR(1, 3)
+        GKO_COMPARATOR(0, 2)
+        GKO_COMPARATOR(2, 4)GKO_COMPARATOR(0, 1)
+        GKO_COMPARATOR(2, 3)GKO_COMPARATOR(1, 4)
+        GKO_COMPARATOR(1, 2)GKO_COMPARATOR(3, 4);
+        // clang-format on
+        return;
+    }
+    if (n == 6) {
+        // clang-format off
+        GKO_COMPARATOR(0, 4)GKO_COMPARATOR(1, 5)
+        GKO_COMPARATOR(0, 2)GKO_COMPARATOR(1, 3)
+        GKO_COMPARATOR(2, 4)GKO_COMPARATOR(3, 5)GKO_COMPARATOR(0, 1)
+        GKO_COMPARATOR(2, 3)GKO_COMPARATOR(4, 5)
+        GKO_COMPARATOR(1, 4)
+        GKO_COMPARATOR(1, 2)GKO_COMPARATOR(3, 4);
+        // clang-format on
+        return;
+    }
+    if (n == 7) {
+        // clang-format off
+        GKO_COMPARATOR(0,4)GKO_COMPARATOR(1,5)GKO_COMPARATOR(2,6)
+        GKO_COMPARATOR(0,2)GKO_COMPARATOR(1,3)GKO_COMPARATOR(4,6)
+        GKO_COMPARATOR(2,4)GKO_COMPARATOR(3,5)GKO_COMPARATOR(0,1)
+        GKO_COMPARATOR(2,3)GKO_COMPARATOR(4,5)
+        GKO_COMPARATOR(1,4)GKO_COMPARATOR(3,6)
+        GKO_COMPARATOR(1,2)GKO_COMPARATOR(3,4)GKO_COMPARATOR(5,6);
+        // clang-format on
+        return;
+    }
+    if (n < 128) {
+        // clang-format off
+        GKO_COMPARATOR(0,4)GKO_COMPARATOR(1,5)GKO_COMPARATOR(2,6)GKO_COMPARATOR(3,7)
+        GKO_COMPARATOR(0,2)GKO_COMPARATOR(1,3)GKO_COMPARATOR(4,6)GKO_COMPARATOR(5,7)
+        GKO_COMPARATOR(2,4)GKO_COMPARATOR(3,5)GKO_COMPARATOR(0,1)GKO_COMPARATOR(6,7)
+        GKO_COMPARATOR(2,3)GKO_COMPARATOR(4,5)
+        GKO_COMPARATOR(1,4)GKO_COMPARATOR(3,6)
+        GKO_COMPARATOR(1,2)GKO_COMPARATOR(3,4)GKO_COMPARATOR(5,6);
+        // clang-format on
+
+        const int8 m = n < 48 ? n : 48;
+        for (int8 i = 8; i < m; ++i) {
+            auto j = i;
+            const auto tmp = arr[j];
+            for (; GKO_GCC_EXPECT_LIKELY(j >= 2 && comp(tmp, arr[j - 2]));
+                 j -= 2) {
+                std::swap(arr[j - 1], arr[j]);
+                std::swap(arr[j - 2], arr[j - 1]);
+            }
+
+            if (j == 0) {
+                continue;
+            }
+            GKO_COMPARATOR(j - 1, j);
+        }
+        for (int8 i = 48; i < n; ++i) {
+            auto j = i;
+            const auto tmp = arr[j];
+            for (; GKO_GCC_EXPECT_LIKELY(j >= 4 && comp(tmp, arr[j - 4]));
+                 j -= 4) {
+                std::swap(arr[j - 1], arr[j]);
+                std::swap(arr[j - 2], arr[j - 1]);
+                std::swap(arr[j - 3], arr[j - 2]);
+                std::swap(arr[j - 4], arr[j - 3]);
+            }
+
+            if (j == 0) {
+                continue;
+            }
+            GKO_COMPARATOR(j - 1, j);
+            if (j == 1) {
+                continue;
+            }
+            GKO_COMPARATOR(j - 2, j - 1);
+            if (j == 2) {
+                continue;
+            }
+            GKO_COMPARATOR(j - 3, j - 2);
+        }
+        return;
+    }
+    std::sort(arr, arr + n, comp);
+}
+
+
+#undef GKO_GCC_EXPECT_LIKELY
+#undef GKO_COMPARATOR
+
+
+}  // namespace omp
+}  // namespace kernels
+}  // namespace gko
diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index 042ec3e1a1a..8f8862606ec 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -31,12 +31,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #include "core/reorder/rcm_kernels.hpp"
-#include "core/base/allocator.hpp"
-#include "core/components/prefix_sum.hpp"
 
 
 #include <algorithm>
-#include <cstring>
 #include <iterator>
 #include <memory>
 #include <queue>
@@ -57,6 +54,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
+#include "components/omp_mutex.hpp"
+#include "components/sort_small.hpp"
+#include "core/base/allocator.hpp"
+#include "core/components/prefix_sum.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace omp {
@@ -79,12 +82,12 @@ namespace rcm {
 
 template <typename IndexType>
 void get_degree_of_nodes(std::shared_ptr<const OmpExecutor> exec,
-                         const size_type num_vertices,
+                         const IndexType num_vertices,
                          const IndexType *const row_ptrs,
                          IndexType *const degrees)
 {
 #pragma omp parallel for
-    for (auto i = 0; i < num_vertices; ++i) {
+    for (IndexType i = 0; i < num_vertices; ++i) {
         degrees[i] = row_ptrs[i + 1] - row_ptrs[i];
     }
 }
@@ -92,34 +95,31 @@ void get_degree_of_nodes(std::shared_ptr<const OmpExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 
 
+// This constant controls how many nodes can be dequeued from the
+// UbfsLinearQueue at once at most. Increasing it reduces lock contention and
+// "unneccesary work", but disturbs queue ordering, generating extra work.
 constexpr auto chunk_bound = 512;
 
 
 template <typename IndexType>
 struct UbfsLinearQueue {
-    gko::vector<IndexType> arr;
+    vector<IndexType> arr;
     IndexType head;
     IndexType tail;
 
     // Those two are only there for safety, they are barely ever contended,
     // due to work chunking.
-    omp_lock_t read_lock;
-    omp_lock_t write_lock;
+    omp_mutex read_mutex;
+    omp_mutex write_mutex;
 
-    UbfsLinearQueue() {}
 
     UbfsLinearQueue(std::shared_ptr<const OmpExecutor> exec, size_type capacity)
-        : arr(gko::vector<IndexType>(capacity, exec)), head(0), tail(0)
-    {
-        omp_init_lock(&read_lock);
-        omp_init_lock(&write_lock);
-    }
-
-    ~UbfsLinearQueue()
-    {
-        omp_destroy_lock(&read_lock);
-        omp_destroy_lock(&write_lock);
-    }
+        : arr(vector<IndexType>(capacity, exec)),
+          head(0),
+          tail(0),
+          read_mutex(omp_lock_hint_uncontended),
+          write_mutex(omp_lock_hint_uncontended)
+    {}
 
     UbfsLinearQueue(UbfsLinearQueue &other) = delete;
     UbfsLinearQueue &operator=(const UbfsLinearQueue &other) = delete;
@@ -132,14 +132,12 @@ struct UbfsLinearQueue {
     {
         const auto data = &arr[0];
 
-        omp_set_lock(&write_lock);
+        std::lock_guard<omp_mutex> write_guard{write_mutex};
 
-        std::memcpy(&data[tail], chunk, n * sizeof(IndexType));
+        std::copy_n(chunk, n, &data[tail]);
 
         // Atomicity, order, and observability enforced by lock.
         tail += n;
-
-        omp_unset_lock(&write_lock);
     }
 
     /**
@@ -149,19 +147,20 @@ struct UbfsLinearQueue {
     IndexType chunk_size()
     {
         const auto available_nodes = tail - head;
-        return std::min((available_nodes + 1) / 2, (IndexType)chunk_bound);
+        return std::min<IndexType>((available_nodes + 1) / 2, chunk_bound);
     }
 
     /**
      * Returns a pointer to an exclusively owned chunk of length <= n/2.
-     * Blocks in case no nodes are available,
-     * until all other threads are finished processing.
-     * If (nullptr,0) is returned, the algorithm is done.
+     * Blocks in case no nodes are available and other threads are still
+     * working, as indicated by `threads_working`. That way, if (nullptr,0) is
+     * finally returned, after all threads stopped working and still no nodes
+     * are available, the algorithm is definitely done.
      */
     std::pair<IndexType *, IndexType> dequeue_chunk(int *threads_working)
     {
         const auto data = &arr[0];
-        omp_set_lock(&read_lock);
+        std::lock_guard<omp_mutex> read_guard{read_mutex};
 
         const auto n = chunk_size();
 
@@ -173,7 +172,6 @@ struct UbfsLinearQueue {
             const auto old_head = head;
             head += n;
 
-            omp_unset_lock(&read_lock);
             return std::make_pair(data + old_head, n);
         } else {
             // This "spinlock" is fine without cmpxchg, exclusive waiter-access
@@ -200,10 +198,8 @@ struct UbfsLinearQueue {
                 IndexType old_head = head;
                 head += n;
 
-                omp_unset_lock(&read_lock);
                 return std::make_pair(data + old_head, n);
             } else {
-                omp_unset_lock(&read_lock);
                 return std::make_pair(nullptr, 0);
             }
         }
@@ -211,12 +207,69 @@ struct UbfsLinearQueue {
 };
 
 
+template <typename IndexType>
+inline void reduce_neighbours_levels(const IndexType num_vertices,
+                                     const IndexType *const row_ptrs,
+                                     const IndexType *const col_idxs,
+                                     IndexType *const local_to_insert,
+                                     size_type *const local_to_insert_size,
+                                     IndexType *const levels,
+                                     const IndexType node)
+{
+    IndexType level;
+#pragma omp atomic read
+    level = levels[node];
+
+    const auto new_neighbour_level = level + 1;
+
+    // Go through neighbours.
+    const auto row_start = row_ptrs[node];
+    const auto row_end = row_ptrs[node + 1];
+
+    for (auto neighbour_i = row_start; neighbour_i < row_end; ++neighbour_i) {
+        const auto neighbour = col_idxs[neighbour_i];
+
+        // Immediately evaluated lambda expression,
+        // to enable continuing the outer loop.
+        const auto reduce_level_atomic = [&]() {
+            // Atomically check if we have lower new level,
+            // then set, implemented with cmpxchg.
+
+            IndexType old_neighbour_level;
+            do {
+#pragma omp atomic read
+                old_neighbour_level = levels[neighbour];
+
+                const auto is_new_smaller =
+                    new_neighbour_level < old_neighbour_level;
+
+                if (!is_new_smaller) {
+                    return true;
+                }
+
+            } while (!__atomic_compare_exchange_n(
+                &levels[neighbour], &old_neighbour_level, new_neighbour_level,
+                true, std::memory_order::memory_order_acq_rel,
+                std::memory_order::memory_order_acquire));
+            return false;
+        };
+
+        if (reduce_level_atomic()) {
+            continue;
+        }
+
+        local_to_insert[*local_to_insert_size] = neighbour;
+        ++(*local_to_insert_size);
+    }
+}
+
+
 /**
  * Performs an unordered breadth-first search,
  * thereby building a rooted level structure.
  */
 template <typename IndexType>
-void ubfs(std::shared_ptr<const OmpExecutor> exec, const size_type num_vertices,
+void ubfs(std::shared_ptr<const OmpExecutor> exec, const IndexType num_vertices,
           const IndexType *const row_ptrs, const IndexType *const col_idxs,
           IndexType *const
               levels,  // Must be inf/max in all nodes connected to source
@@ -226,7 +279,7 @@ void ubfs(std::shared_ptr<const OmpExecutor> exec, const size_type num_vertices,
 
     // This is an upper bound for how many nodes may be in the work set
     // at maximum at any time.
-    const auto work_bound = (size_type)max_threads * num_vertices;
+    const auto work_bound = static_cast<size_type>(max_threads) * num_vertices;
     UbfsLinearQueue<IndexType> q(exec, work_bound);
 
     // Start from the given start node.
@@ -240,8 +293,8 @@ void ubfs(std::shared_ptr<const OmpExecutor> exec, const size_type num_vertices,
     {
         // This is an upper bound, as '.deque_chunk...' is upper bounded by
         // chunk_bound.
-        gko::vector<IndexType> local_to_insert(max_degree * chunk_bound, exec);
-        auto local_to_insert_size = 0;
+        vector<IndexType> local_to_insert(max_degree * chunk_bound, exec);
+        size_type local_to_insert_size = 0;
 
         while (true) {
             // Get some nodes from the locked q.
@@ -253,45 +306,16 @@ void ubfs(std::shared_ptr<const OmpExecutor> exec, const size_type num_vertices,
             // If there are no nodes returned that also means there are no nodes
             // still in flight.
             if (chunk_size > 0) {
-                // For each node in the chunk.
-                for (auto chunk_i = 0; chunk_i < chunk_size; ++chunk_i) {
+                for (IndexType chunk_i = 0; chunk_i < chunk_size; ++chunk_i) {
                     const auto node = chunk[chunk_i];
 
-                    IndexType level;
-#pragma omp atomic read
-                    level = levels[node];
-
-                    const auto new_neighbour_level = level + 1;
-
-                    // Go through neighbours.
-                    const auto row_start = row_ptrs[node];
-                    const auto row_end = row_ptrs[node + 1];
-
-                    for (auto neighbour_i = row_start; neighbour_i < row_end;
-                         ++neighbour_i) {
-                        const auto neighbour = col_idxs[neighbour_i];
-
-                        IndexType old_neighbour_level;
-                        do {
-#pragma omp atomic read
-                            old_neighbour_level = levels[neighbour];
-
-                            const auto is_new_smaller =
-                                new_neighbour_level < old_neighbour_level;
-
-                            if (!is_new_smaller) {
-                                goto continue_outer;
-                            }
-                        } while (!__atomic_compare_exchange_n(
-                            &levels[neighbour], &old_neighbour_level,
-                            new_neighbour_level, true,
-                            std::memory_order::memory_order_acq_rel,
-                            std::memory_order::memory_order_acquire));
-
-                        local_to_insert[local_to_insert_size++] = neighbour;
-
-                    continue_outer:;
-                    }
+                    // For each node in the chunk, process all neighbours,
+                    // by reducing their level if a new shortest path was found,
+                    // in that case also writing the neighbour into
+                    // local_to_insert.
+                    reduce_neighbours_levels(
+                        num_vertices, row_ptrs, col_idxs, &local_to_insert[0],
+                        &local_to_insert_size, levels, node);
                 }
 
                 // Insert back to global work queue, done with this chunk.
@@ -316,7 +340,7 @@ void ubfs(std::shared_ptr<const OmpExecutor> exec, const size_type num_vertices,
  */
 template <typename IndexType>
 std::pair<IndexType, IndexType> rls_contender_and_height(
-    std::shared_ptr<const OmpExecutor> exec, const size_type num_vertices,
+    std::shared_ptr<const OmpExecutor> exec, const IndexType num_vertices,
     const IndexType *const row_ptrs, const IndexType *const col_idxs,
     const IndexType *const degrees,
     IndexType *const levels,  // Must be max/inf in all nodes connected to start
@@ -330,19 +354,19 @@ std::pair<IndexType, IndexType> rls_contender_and_height(
 
     // First local ...
     const auto num_threads = omp_get_max_threads();
-    gko::vector<IndexType> local_contenders(num_threads, exec);
-    gko::vector<IndexType> local_degrees(num_threads, exec);
-    gko::vector<IndexType> local_heights(num_threads, exec);
+    vector<IndexType> local_contenders(num_threads, exec);
+    vector<IndexType> local_degrees(num_threads, exec);
+    vector<IndexType> local_heights(num_threads, exec);
 
 #pragma omp parallel num_threads(num_threads)
     {
         const auto tid = omp_get_thread_num();
-        auto local_contender = 0;
-        auto local_degree = std::numeric_limits<IndexType>::max();
-        auto local_height = 0;
+        IndexType local_contender = 0;
+        IndexType local_degree = std::numeric_limits<IndexType>::max();
+        IndexType local_height = 0;
 
 #pragma omp for schedule(static)
-        for (auto i = 0; i < num_vertices; ++i) {
+        for (IndexType i = 0; i < num_vertices; ++i) {
             if (levels[i] > local_height) {
                 local_contender = i;
                 local_degree = degrees[i];
@@ -362,7 +386,7 @@ std::pair<IndexType, IndexType> rls_contender_and_height(
     auto global_contender = local_contenders[0];
     auto global_degree = local_degrees[0];
     auto global_height = local_heights[0];
-    for (auto i = 1; i < num_threads; ++i) {
+    for (IndexType i = 1; i < num_threads; ++i) {
         if (local_heights[i] > global_height) {
             global_contender = local_contenders[i];
             global_degree = local_degrees[i];
@@ -383,28 +407,28 @@ std::pair<IndexType, IndexType> rls_contender_and_height(
  */
 template <typename IndexType>
 std::pair<IndexType, IndexType> find_min_idx_and_max_val(
-    std::shared_ptr<const OmpExecutor> exec, const size_type num_vertices,
+    std::shared_ptr<const OmpExecutor> exec, const IndexType num_vertices,
     const IndexType *const row_ptrs, const IndexType *const col_idxs,
-    const IndexType *const degrees, gko::vector<IndexType> &levels,
+    const IndexType *const degrees, vector<IndexType> &levels,
     const uint8 *const previous_component,
     gko::reorder::starting_strategy strategy)
 {
     // First find local extrema.
     const auto num_threads = omp_get_max_threads();
-    gko::vector<std::pair<IndexType, IndexType>> local_min_vals_idxs(
-        num_threads, exec);
-    gko::vector<IndexType> local_max_vals(num_threads, exec);
+    vector<std::pair<IndexType, IndexType>> local_min_vals_idxs(num_threads,
+                                                                exec);
+    vector<IndexType> local_max_vals(num_threads, exec);
 
 #pragma omp parallel num_threads(num_threads)
     {
         const auto tid = omp_get_thread_num();
         auto local_min_val = std::numeric_limits<IndexType>::max();
-        auto local_min_idx = tid;
+        auto local_min_idx = IndexType{};
         auto local_max_val = std::numeric_limits<IndexType>::min();
-        auto local_max_idx = tid;
+        auto local_max_idx = IndexType{};
 
 #pragma omp for schedule(static)
-        for (auto i = 0; i < num_vertices; ++i) {
+        for (IndexType i = 0; i < num_vertices; ++i) {
             // If this level hasn't been discovered before.
             if (!previous_component[i]) {
                 if (degrees[i] < local_min_val) {
@@ -426,7 +450,7 @@ std::pair<IndexType, IndexType> find_min_idx_and_max_val(
     auto global_min_idx = local_min_vals_idxs[0].second;
     auto global_min_val = local_min_vals_idxs[0].first;
     auto global_max_val = local_max_vals[0];
-    for (auto i = 1; i < num_threads; ++i) {
+    for (IndexType i = 1; i < num_threads; ++i) {
         if (local_min_vals_idxs[i].first < global_min_val) {
             global_min_val = local_min_vals_idxs[i].first;
             global_min_idx = local_min_vals_idxs[i].second;
@@ -445,11 +469,11 @@ std::pair<IndexType, IndexType> find_min_idx_and_max_val(
  */
 template <typename IndexType>
 IndexType find_start_node(std::shared_ptr<const OmpExecutor> exec,
-                          const size_type num_vertices,
+                          const IndexType num_vertices,
                           const IndexType *const row_ptrs,
                           const IndexType *const col_idxs,
                           const IndexType *const degrees,
-                          gko::vector<IndexType> &levels,
+                          vector<IndexType> &levels,
                           const uint8 *const previous_component,
                           const gko::reorder::starting_strategy strategy)
 {
@@ -470,8 +494,8 @@ IndexType find_start_node(std::shared_ptr<const OmpExecutor> exec,
 
     // Intermediate storage.
     // We copy back to `levels` for the final pick.
-    gko::vector<IndexType> levels_clone(levels, exec);
-    gko::vector<IndexType> current_levels_clone(levels, exec);
+    vector<IndexType> levels_clone(levels, exec);
+    vector<IndexType> current_levels_clone(levels, exec);
 
     auto current = min_idx;
     const auto contender_and_height =
@@ -493,11 +517,11 @@ IndexType find_start_node(std::shared_ptr<const OmpExecutor> exec,
             current_height = contender_height;
             current = current_contender;
             current_contender = contender_contender;
-            current_levels_clone = gko::vector<IndexType>(levels, exec);
+            current_levels_clone = vector<IndexType>(levels, exec);
             current_levels_clone.swap(levels_clone);
         } else {
-            std::memcpy(&levels[0], &current_levels_clone[0],
-                        num_vertices * sizeof(IndexType));
+            std::copy_n(current_levels_clone.begin(), num_vertices,
+                        levels.begin());
             return current;
         }
     }
@@ -508,37 +532,30 @@ IndexType find_start_node(std::shared_ptr<const OmpExecutor> exec,
  * Counts how many nodes there are per level.
  */
 template <typename IndexType>
-gko::vector<IndexType> count_levels(std::shared_ptr<const OmpExecutor> exec,
-                                    const IndexType *const levels,
-                                    uint8 *const previous_component,
-                                    size_type num_vertices)
+vector<IndexType> count_levels(std::shared_ptr<const OmpExecutor> exec,
+                               const IndexType *const levels,
+                               uint8 *const previous_component,
+                               IndexType num_vertices)
 {
     const auto num_threads = omp_get_max_threads();
-
-    // This initialization is necessary due to nested custom allocator.
-    gko::vector<gko::vector<IndexType>> level_counts(exec);
-    for (auto i = 0; i < num_threads; ++i) {
-        level_counts.push_back(gko::vector<IndexType>(0, exec));
-    }
+    vector<vector<IndexType>> level_counts(num_threads, vector<IndexType>(exec),
+                                           exec);
 
 #pragma omp parallel num_threads(num_threads)
     {
         const auto tid = omp_get_thread_num();
         auto local_level_counts = &level_counts[tid];
 
-        auto current_size = 0;
-
 #pragma omp for schedule(static)
-        for (auto i = 0; i < num_vertices; ++i) {
+        for (IndexType i = 0; i < num_vertices; ++i) {
             const auto level = levels[i];
 
             // If not part of previous component and actually discovered in this
             // component.
             if (!previous_component[i] &&
                 level != std::numeric_limits<IndexType>::max()) {
-                if (level >= current_size) {
+                if (level >= local_level_counts->size()) {
                     local_level_counts->resize(level + 1);
-                    current_size = level + 1;
                 }
                 previous_component[i] = true;
                 ++(*local_level_counts)[level];
@@ -546,18 +563,19 @@ gko::vector<IndexType> count_levels(std::shared_ptr<const OmpExecutor> exec,
         }
     }
 
-    gko::vector<size_type> level_count_sizes(num_threads, exec);
+    vector<size_type> level_count_sizes(num_threads, exec);
     for (auto tid = 0; tid < num_threads; ++tid) {
         level_count_sizes[tid] = level_counts[tid].size();
     }
     const auto max_size =
         *std::max_element(level_count_sizes.begin(), level_count_sizes.end());
 
-    gko::vector<IndexType> final_level_counts(max_size + 1, exec);
+    vector<IndexType> final_level_counts(max_size + 1, exec);
     auto i = 0;
     while (true) {
         auto done = true;
-        for (auto tid = 0; tid < num_threads; ++tid) {
+        for (std::remove_const_t<decltype(num_threads)> tid = 0;
+             tid < num_threads; ++tid) {
             if (i < level_count_sizes[tid]) {
                 const auto count = level_counts[tid][i];
                 final_level_counts[i] += count;
@@ -580,9 +598,10 @@ gko::vector<IndexType> count_levels(std::shared_ptr<const OmpExecutor> exec,
  * as a prefix sum.
  */
 template <typename IndexType>
-gko::vector<IndexType> compute_level_offsets(
-    std::shared_ptr<const OmpExecutor> exec, const IndexType *const levels,
-    size_type num_vertices, uint8 *const previous_component)
+vector<IndexType> compute_level_offsets(std::shared_ptr<const OmpExecutor> exec,
+                                        const IndexType *const levels,
+                                        IndexType num_vertices,
+                                        uint8 *const previous_component)
 {
     auto counts = count_levels(exec, levels, previous_component, num_vertices);
     components::prefix_sum(exec, &counts[0], counts.size());
@@ -590,88 +609,16 @@ gko::vector<IndexType> compute_level_offsets(
 }
 
 
-/**
- * Helper macros for the helper function below.
- */
-#if defined(__GNUG__) && defined(__x86_64__)
-// GCC can only be convinced to use cmovs here by inline asm.
-#define GKO_COMPARATOR(x, y)                                       \
-    {                                                              \
-        IndexType tmp;                                             \
-        const auto degree_x = degrees[neighbours[x]];              \
-        const auto degree_y = degrees[neighbours[y]];              \
-        asm("mov %0, %2\n\t"                                       \
-            "cmp %4, %3\n\t"                                       \
-            "cmova %1, %0\n\t"                                     \
-            "cmova %2, %1\n\t"                                     \
-            : "+r"(neighbours[x]), "+r"(neighbours[y]), "=&r"(tmp) \
-            : "r"(degree_x), "r"(degree_y)                         \
-            : "cc");                                               \
-    }
-#else
-#define GKO_COMPARATOR(x, y)                                         \
-    {                                                                \
-        const auto degree_x = degrees[neighbours[x]];                \
-        const auto degree_y = degrees[neighbours[y]];                \
-        const auto should_swap = degree_x > degree_y;                \
-        const auto tmp = neighbours[x];                              \
-        neighbours[x] = should_swap ? neighbours[y] : neighbours[x]; \
-        neighbours[y] = should_swap ? tmp : neighbours[y];           \
-    }
-#endif
-
-
-/**
- * Helper fucntion for quickly sorting small vectors by degree.
- */
-template <typename IndexType>
-inline void sort_by_degree(IndexType *const neighbours, size_type count,
-                           const IndexType *const degrees)
-{
-    if (count < 2) {
-        return;
-    }
-
-    if (count == 2) {
-        GKO_COMPARATOR(0, 1);
-        return;
-    }
-
-    if (count == 3) {
-        GKO_COMPARATOR(0, 1);
-        GKO_COMPARATOR(0, 2);
-        GKO_COMPARATOR(1, 2);
-        return;
-    }
-
-    if (count == 4) {
-        GKO_COMPARATOR(0, 1);
-        GKO_COMPARATOR(2, 3);
-        GKO_COMPARATOR(0, 2);
-        GKO_COMPARATOR(1, 3);
-        GKO_COMPARATOR(1, 2);
-        return;
-    }
-
-    if (count < 16) {
-        int i, j;
-        for (i = 1; i < count; i++) {
-            int tmp = neighbours[i];
-            for (j = i; j >= 1 && degrees[tmp] < degrees[neighbours[j - 1]];
-                 --j) {
-                neighbours[j] = neighbours[j - 1];
-            }
-            neighbours[j] = tmp;
-        }
-        return;
-    }
-
-    std::sort(neighbours, neighbours + count,
-              [&](IndexType left, IndexType right) {
-                  return degrees[left] < degrees[right];
-              });
-}
+// Signal value to wich the entire permutation is intialized.
+// Threads spin on this value, until it is replaced by another value,
+// written by another thread.
+constexpr auto perm_untouched = -1;
 
+// Signal value which a thread writes to a level (as in the level of a node
+// becomes -1), to signal that it has been processed. This information is only
+// relevant local to the thread, since only a single thread is responsible for a
+// node.
+constexpr auto level_processed = -1;
 
 /**
  * Implements the last phase of urcm,
@@ -682,25 +629,26 @@ void write_permutation(std::shared_ptr<const OmpExecutor> exec,
                        const IndexType *const row_ptrs,
                        const IndexType *const col_idxs, IndexType *const levels,
                        const IndexType *const degrees,
-                       const gko::vector<IndexType> &offsets,
-                       IndexType *const perm, const std::size_t num_vertices,
+                       const vector<IndexType> &offsets, IndexType *const perm,
+                       const IndexType num_vertices,
                        const IndexType base_offset, const IndexType start)
 {
-    const auto offsets_size = offsets.size();
+    // There can not be more levels than nodes, therefore IndexType.
+    const IndexType num_levels = offsets.size() - 1;
     perm[base_offset] = start;
 
     const auto num_threads = omp_get_max_threads();
-#pragma omp parallel num_threads(num_threads) firstprivate(offsets_size)
+#pragma omp parallel num_threads(num_threads) firstprivate(num_levels)
     {
         const auto tid = omp_get_thread_num();
-        gko::vector<IndexType> valid_neighbours(0, exec);
+        vector<IndexType> valid_neighbours(0, exec);
 
         // Go through the levels assigned to this thread.
-        for (auto level = tid; level < offsets_size - 1; level += num_threads) {
+        for (IndexType level = tid; level < num_levels; level += num_threads) {
             const auto next_level_idx = offsets[level + 1];
             const auto level_idx = offsets[level];
             const auto level_len = next_level_idx - level_idx;
-            auto write_offset = 0;
+            IndexType write_offset = 0;
 
             // Go through one level.
             for (auto read_offset = level_idx; read_offset < next_level_idx;
@@ -709,7 +657,7 @@ void write_permutation(std::shared_ptr<const OmpExecutor> exec,
                 IndexType written;
 #pragma omp atomic read
                 written = perm[base_offset + read_offset];
-                while (written == std::numeric_limits<IndexType>::max()) {
+                while (written == perm_untouched) {
                     GKO_MM_PAUSE();
 #pragma omp atomic read
                     written = perm[base_offset + read_offset];
@@ -730,29 +678,30 @@ void write_permutation(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp atomic read
                     neighbour_level = levels[neighbour];
 
-                    // ::max() is not a valid value here (as ther cannot be
-                    // ::max() nodes), therefore can be used for signaling.
+                    // level_processed is not a valid value for a node,
+                    // implicitly filtered out here.
                     if (neighbour_level == write_level) {
 // Protect against writing the same node as neighbour of different nodes in the
 // previous level by the same thread.
 #pragma omp atomic write
-                        levels[neighbour] =
-                            std::numeric_limits<IndexType>::max();
-
+                        levels[neighbour] = level_processed;
                         valid_neighbours.push_back(neighbour);
                     }
                 }
 
-                // Sort neighbours.
-                const auto size = valid_neighbours.size();
-                sort_by_degree(&valid_neighbours[0], size, degrees);
+                // Sort neighbours. Can not be more than there are nodes.
+                const IndexType size = valid_neighbours.size();
+                sort_small(&valid_neighbours[0], size,
+                           [&](IndexType l, IndexType r) {
+                               return degrees[l] < degrees[r];
+                           });
 
                 // Write the processed neighbours.
                 const auto base_write_offset =
                     base_offset + next_level_idx + write_offset;
 
                 // Memcpy would be nice, but we need element-wise atomicity.
-                for (auto i = 0; i < size; ++i) {
+                for (IndexType i = 0; i < size; ++i) {
 #pragma omp atomic write
                     perm[base_write_offset + i] = valid_neighbours[i];
                 }
@@ -773,8 +722,8 @@ IndexType handle_isolated_nodes(std::shared_ptr<const OmpExecutor> exec,
                                 const IndexType *const row_ptrs,
                                 const IndexType *const col_idxs,
                                 const IndexType *const degrees,
-                                IndexType *const perm, size_type num_vertices,
-                                gko::vector<uint8> &previous_component)
+                                IndexType *const perm, IndexType num_vertices,
+                                vector<uint8> &previous_component)
 {
     struct IsolatedNodes {
         // Using a gko::vector here makes clang and gcc segfault,
@@ -791,7 +740,7 @@ IndexType handle_isolated_nodes(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp declare reduction(FindIsolated : IsolatedNodes : omp_out += omp_in)
     auto isolated = IsolatedNodes();
 #pragma omp parallel for reduction(FindIsolated : isolated)
-    for (auto i = 0; i < num_vertices; ++i) {
+    for (IndexType i = 0; i < num_vertices; ++i) {
         // No need to check for diagonal elements (only self-neighbouring) here,
         // those are already removed from the matrix.
         if (degrees[i] == 0) {
@@ -800,8 +749,7 @@ IndexType handle_isolated_nodes(std::shared_ptr<const OmpExecutor> exec,
         }
     }
 
-    std::memcpy(perm, &isolated.nodes[0],
-                isolated.nodes.size() * sizeof(IndexType));
+    std::copy_n(isolated.nodes.begin(), isolated.nodes.size(), perm);
     return isolated.nodes.size();
 }
 
@@ -811,7 +759,7 @@ IndexType handle_isolated_nodes(std::shared_ptr<const OmpExecutor> exec,
  */
 template <typename IndexType>
 void get_permutation(std::shared_ptr<const OmpExecutor> exec,
-                     const size_type num_vertices,
+                     const IndexType num_vertices,
                      const IndexType *const row_ptrs,
                      const IndexType *const col_idxs,
                      const IndexType *const degrees, IndexType *const perm,
@@ -819,13 +767,13 @@ void get_permutation(std::shared_ptr<const OmpExecutor> exec,
                      const gko::reorder::starting_strategy strategy)
 {
     // Initialize the perm to all "signal value".
-    std::fill(perm, perm + num_vertices, std::numeric_limits<IndexType>::max());
+    std::fill(perm, perm + num_vertices, perm_untouched);
 
     // For multiple components.
     IndexType base_offset = 0;
 
     // Stores for each node if it is part of an already discovered component.
-    gko::vector<uint8> previous_component(num_vertices, exec);
+    vector<uint8> previous_component(num_vertices, exec);
 
     // First handle all isolated nodes. That reduces complexity later on.
     base_offset +=
@@ -833,7 +781,7 @@ void get_permutation(std::shared_ptr<const OmpExecutor> exec,
                               num_vertices, previous_component);
 
     // Stores the level structure. Initialized to all "infinity".
-    gko::vector<IndexType> levels(num_vertices, exec);
+    vector<IndexType> levels(num_vertices, exec);
     std::fill(levels.begin(), levels.end(),
               std::numeric_limits<IndexType>::max());
 
@@ -853,7 +801,7 @@ void get_permutation(std::shared_ptr<const OmpExecutor> exec,
 
         // Will contain 0 -- 1 -- level_count(2) + 1 -- level_count(2) +
         // level_count(3) + 1 -- ... -- total_sum
-        gko::vector<IndexType> offsets = compute_level_offsets(
+        vector<IndexType> offsets = compute_level_offsets(
             exec, &levels[0], num_vertices, &previous_component[0]);
 
 
@@ -876,10 +824,8 @@ void get_permutation(std::shared_ptr<const OmpExecutor> exec,
     }
 
 // Finally reverse the order.
-// Could in principle also be already written in reverse order, but ... just no.
-// This is ridiculoulsly fast anyway.
 #pragma omp parallel for schedule(static)
-    for (auto i = 0; i < num_vertices / 2; ++i) {
+    for (IndexType i = 0; i < num_vertices / 2; ++i) {
         const auto tmp = perm[i];
         perm[i] = perm[num_vertices - i - 1];
         perm[num_vertices - i - 1] = tmp;
@@ -887,7 +833,7 @@ void get_permutation(std::shared_ptr<const OmpExecutor> exec,
 
     if (inv_perm) {
 #pragma omp parallel for schedule(static)
-        for (auto i = 0; i < num_vertices; ++i) {
+        for (IndexType i = 0; i < num_vertices; ++i) {
             inv_perm[perm[i]] = i;
         }
     }
diff --git a/omp/test/reorder/rcm_kernels.cpp b/omp/test/reorder/rcm_kernels.cpp
index cbcea22e9ac..c145f98433f 100644
--- a/omp/test/reorder/rcm_kernels.cpp
+++ b/omp/test/reorder/rcm_kernels.cpp
@@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
-#include <cstring>
 #include <deque>
 #include <fstream>
 #include <memory>
@@ -45,7 +44,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/permutation.hpp>
 
 
@@ -209,9 +207,8 @@ class Rcm : public ::testing::Test {
 
         // Following checks for cm ordering, therefore create a reversed perm.
         auto perm = std::vector<i_type>(n);
-        std::memcpy(&perm[0],
-                    reorder->get_permutation()->get_const_permutation(),
-                    n * sizeof(i_type));
+        std::copy_n(reorder->get_permutation()->get_const_permutation(), n,
+                    perm.begin());
         for (auto i = 0; i < n / 2; ++i) {
             const auto tmp = perm[i];
             perm[i] = perm[n - i - 1];
diff --git a/reference/reorder/rcm_kernels.cpp b/reference/reorder/rcm_kernels.cpp
index 7b00e801737..2cb0727c131 100644
--- a/reference/reorder/rcm_kernels.cpp
+++ b/reference/reorder/rcm_kernels.cpp
@@ -52,6 +52,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
+#include "core/base/allocator.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace reference {
@@ -65,11 +68,11 @@ namespace rcm {
 
 template <typename IndexType>
 void get_degree_of_nodes(std::shared_ptr<const ReferenceExecutor> exec,
-                         const size_type num_vertices,
+                         const IndexType num_vertices,
                          const IndexType *const row_ptrs,
                          IndexType *const degrees)
 {
-    for (auto i = 0; i < num_vertices; ++i) {
+    for (IndexType i = 0; i < num_vertices; ++i) {
         degrees[i] = row_ptrs[i + 1] - row_ptrs[i];
     }
 }
@@ -78,43 +81,38 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 
 
 /**
- * Computes a level structure rooted at `root`, returning the height of the
- * structure along with the index of the node of minimal degree in its last
- * level.
+ * Computes a level structure rooted at `root`, returning a node of minimal
+ * degree in its last level, along with the height of the structure.
  */
 template <typename IndexType>
 std::pair<IndexType, size_type> rls_contender_and_height(
-    std::shared_ptr<const ReferenceExecutor> exec, const size_type num_vertices,
+    std::shared_ptr<const ReferenceExecutor> exec, const IndexType num_vertices,
     const IndexType root, const IndexType *const row_ptrs,
     const IndexType *const col_idxs, const IndexType *const degrees)
 {
     // This could actually be allocated in the calling scope, then reused.
-    Array<bool> visited_local(exec, num_vertices);
-    auto visited_local_p = visited_local.get_data();
-    for (auto i = 0; i < num_vertices; ++i) {
-        visited_local_p[i] = false;
-    }
+    vector<bool> visited_local(num_vertices, false, exec);
 
-    // This stores the level structure, starting with the root node.
+    // This stores a reordering in bfs order, starting with the root node.
     Array<IndexType> rls(exec, num_vertices);
     auto rls_p = rls.get_data();
     rls_p[0] = root;
-    auto rls_offset = 1;
-    visited_local_p[root] = true;
+    IndexType rls_offset = 1;
+    visited_local[root] = true;
 
-    auto rls_index = 0;
+    IndexType rls_index = 0;
 
     // Used to compute the height.
     // Always count up the count of the next level,
     // while counting down the current levels count.
-    auto height = 0;
-    auto current_level_countdown = 1;
-    auto next_level_countup = 0;
+    IndexType height = 0;
+    IndexType current_level_countdown = 1;
+    IndexType next_level_countup = 0;
 
     // The last levels size is required to compute the contender.
-    auto last_level_size = 0;
+    IndexType last_level_size = 0;
 
-    // While there are still nodes who's neighbours haven't been inspected.
+    // While there are still nodes whose neighbours haven't been inspected.
     while (rls_index < rls_offset) {
         auto parent = rls_p[rls_index];
         --current_level_countdown;
@@ -127,8 +125,8 @@ std::pair<IndexType, size_type> rls_contender_and_height(
             auto neighbor = col_idxs[neighbor_idx];
 
             // If this is a new node, add it to the rls and mark it as visited.
-            if (!visited_local_p[neighbor]) {
-                visited_local_p[neighbor] = true;
+            if (!visited_local[neighbor]) {
+                visited_local[neighbor] = true;
                 ++next_level_countup;
                 rls_p[rls_offset] = neighbor;
                 ++rls_offset;
@@ -152,25 +150,20 @@ std::pair<IndexType, size_type> rls_contender_and_height(
     // Choose the contender.
     // It's the node of minimum degree in the last level.
     auto rls_size = rls_offset;
-    auto min_degree = std::numeric_limits<IndexType>::max();
-    auto contender = rls_size - last_level_size;
-    for (auto i = rls_size - last_level_size; i < rls_size; ++i) {
-        if (degrees[rls_p[i]] < min_degree) {
-            contender = i;
-            min_degree = degrees[rls_p[i]];
-        }
-    }
+    auto contender = *std::min_element(
+        rls_p + rls_size - last_level_size, rls_p + rls_size,
+        [&](IndexType u, IndexType v) { return degrees[u] < degrees[v]; });
 
-    return std::pair<IndexType, size_type>(rls_p[contender], height);
+    return std::pair<IndexType, IndexType>(contender, height);
 }
 
 template <typename IndexType>
 IndexType find_starting_node(std::shared_ptr<const ReferenceExecutor> exec,
-                             const size_type num_vertices,
+                             const IndexType num_vertices,
                              const IndexType *const row_ptrs,
                              const IndexType *const col_idxs,
                              const IndexType *const degrees,
-                             const bool *const visited,
+                             const vector<bool> &visited,
                              const gko::reorder::starting_strategy strategy)
 {
     using strategies = gko::reorder::starting_strategy;
@@ -180,9 +173,9 @@ IndexType find_starting_node(std::shared_ptr<const ReferenceExecutor> exec,
                strategy == strategies::pseudo_peripheral);
 
     // There must always be at least one unvisited node left.
-    auto index_min_node = 0;
-    auto min_node_degree = std::numeric_limits<IndexType>::max();
-    for (auto i = 0; i < num_vertices; ++i) {
+    IndexType index_min_node = 0;
+    IndexType min_node_degree = std::numeric_limits<IndexType>::max();
+    for (IndexType i = 0; i < num_vertices; ++i) {
         if (!visited[i]) {
             if (degrees[i] < min_node_degree) {
                 index_min_node = i;
@@ -231,7 +224,7 @@ IndexType find_starting_node(std::shared_ptr<const ReferenceExecutor> exec,
  */
 template <typename IndexType>
 void get_permutation(std::shared_ptr<const ReferenceExecutor> exec,
-                     const size_type num_vertices,
+                     const IndexType num_vertices,
                      const IndexType *const row_ptrs,
                      const IndexType *const col_idxs,
                      const IndexType *const degrees,
@@ -242,26 +235,22 @@ void get_permutation(std::shared_ptr<const ReferenceExecutor> exec,
     // Storing vertices left to proceess.
     Array<IndexType> linear_queue(exec, num_vertices);
     auto linear_queue_p = linear_queue.get_data();
-    auto head_offset = 0;
-    auto tail_offset = 0;
+    IndexType head_offset = 0;
+    IndexType tail_offset = 0;
 
     // Storing which vertices have already been visited.
-    Array<bool> visited(exec, num_vertices);
-    auto visited_p = visited.get_data();
-    for (auto i = 0; i < num_vertices; ++i) {
-        visited_p[i] = false;
-    }
+    vector<bool> visited(num_vertices, false, exec);
 
-    for (auto perm_index = 0; perm_index < num_vertices; ++perm_index) {
+    for (IndexType perm_index = 0; perm_index < num_vertices; ++perm_index) {
         // Get the next vertex to process.
-        auto next_vertex = 0;
+        IndexType next_vertex = 0;
         if (head_offset <= tail_offset) {
             // Choose a new starting vertex, new component needs to be
             // discovered.
             next_vertex = find_starting_node<IndexType>(
-                exec, num_vertices, row_ptrs, col_idxs, degrees, visited_p,
+                exec, num_vertices, row_ptrs, col_idxs, degrees, visited,
                 strategy);
-            visited_p[next_vertex] = true;
+            visited[next_vertex] = true;
         } else {
             next_vertex = linear_queue_p[tail_offset];
             ++tail_offset;
@@ -272,7 +261,7 @@ void get_permutation(std::shared_ptr<const ReferenceExecutor> exec,
         // if no, insert them to sort.
         auto prev_head_offset = head_offset;
 
-        // Is this code correct for getting next_vertexs neighbors?
+        // Get the next vertex neighbours.
         auto row_start = row_ptrs[next_vertex];
         auto row_end = row_ptrs[next_vertex + 1];
         for (auto neighbor_idx = row_start; neighbor_idx < row_end;
@@ -281,26 +270,24 @@ void get_permutation(std::shared_ptr<const ReferenceExecutor> exec,
 
             // If the neighbour hasn't been visited yet, add it to the queue and
             // mark it.
-            if (!visited_p[neighbor]) {
-                visited_p[neighbor] = true;
+            if (!visited[neighbor]) {
+                visited[neighbor] = true;
                 linear_queue_p[head_offset] = neighbor;
                 ++head_offset;
             }
         }
 
         // Sort all just-added neighbours by degree.
-        std::sort(linear_queue_p + prev_head_offset,
-                  linear_queue_p + head_offset,
-                  [&degrees](IndexType i, IndexType j) {
-                      return degrees[i] < degrees[j];
-                  });
+        std::sort(
+            linear_queue_p + prev_head_offset, linear_queue_p + head_offset,
+            [&](IndexType i, IndexType j) { return degrees[i] < degrees[j]; });
 
         // Write out the processed vertex.
         permutation[num_vertices - perm_index - 1] = next_vertex;
     }
 
     if (inv_permutation) {
-        for (auto i = 0; i < num_vertices; ++i) {
+        for (IndexType i = 0; i < num_vertices; ++i) {
             inv_permutation[permutation[i]] = i;
         }
     }
@@ -312,4 +299,4 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
 }  // namespace rcm
 }  // namespace reference
 }  // namespace kernels
-}  // namespace gko
\ No newline at end of file
+}  // namespace gko
diff --git a/reference/test/reorder/rcm.cpp b/reference/test/reorder/rcm.cpp
index 89010bbcafa..53075365f5b 100644
--- a/reference/test/reorder/rcm.cpp
+++ b/reference/test/reorder/rcm.cpp
@@ -47,10 +47,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
-#include <iostream>
 #include "core/test/utils.hpp"
 #include "core/test/utils/assertions.hpp"
-#include "matrices/config.hpp"
 
 namespace {
 
@@ -68,57 +66,40 @@ class Rcm : public ::testing::Test {
         : exec(gko::ReferenceExecutor::create()),
           rcm_factory(reorder_type::build().on(exec)),
           // clang-format off
-          p_mtx(gko::initialize<CsrMtx>(
-                                        {{1.0, 2.0, 0.0, -1.3, 2.1},
-                                         {2.0, 5.0, 1.5, 0.0, 0.0},
-                                         {0.0, 1.5, 1.5, 1.1, 0.0},
-                                         {-1.3, 0.0, 1.1, 2.0, 0.0},
-                                         {2.1, 0.0, 0.0, 0.0, 1.0}},
-                                        exec)),
+          id3_mtx(gko::initialize<CsrMtx>(
+              {{1.0, 0.0, 0.0}, 
+              {0.0, 1.0, 0.0}, 
+              {0.0, 0.0, 1.0}}, exec)),
+          not_id3_mtx(gko::initialize<CsrMtx>(
+              {{1.0, 0.0, 1.0}, 
+              {0.0, 1.0, 0.0}, 
+              {1.0, 0.0, 1.0}}, exec)),
           // clang-format on
-          reorder_op(rcm_factory->generate(p_mtx))
+          reorder_op(rcm_factory->generate(id3_mtx))
     {}
 
     std::shared_ptr<const gko::Executor> exec;
-    std::shared_ptr<CsrMtx> p_mtx;
+    std::shared_ptr<CsrMtx> id3_mtx;
+    std::shared_ptr<CsrMtx> not_id3_mtx;
     std::unique_ptr<typename reorder_type::Factory> rcm_factory;
     std::unique_ptr<reorder_type> reorder_op;
 };
 
 TYPED_TEST_CASE(Rcm, gko::test::ValueIndexTypes);
 
-TYPED_TEST(Rcm, FactoryCreatesCorrectReorderOp)
-{
-    using i_type = typename TestFixture::i_type;
-    using v_type = typename TestFixture::v_type;
-    auto adj_mtx = this->reorder_op->get_adjacency_matrix();
-    auto tmp = gko::matrix::SparsityCsr<v_type, i_type>::create(this->exec,
-                                                                this->p_mtx);
-    auto comp_mtx = tmp->to_adjacency_matrix();
-
-    ASSERT_NE(this->reorder_op->get_adjacency_matrix(), nullptr);
-    GKO_ASSERT_MTX_NEAR(adj_mtx.get(), comp_mtx.get(), 0);
-}
-
 TYPED_TEST(Rcm, CanBeCleared)
 {
     this->reorder_op->clear();
 
-    auto reorder_op_mtx = this->reorder_op->get_adjacency_matrix();
-    ASSERT_EQ(reorder_op_mtx, nullptr);
+    auto reorder_op_perm = this->reorder_op->get_permutation();
+    ASSERT_EQ(reorder_op_perm, nullptr);
 }
 
 TYPED_TEST(Rcm, CanBeCopied)
 {
     using v_type = typename TestFixture::v_type;
-    auto rcm =
-        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-            3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
-            this->exec));
-    auto rcm_copy =
-        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-            3, {{1.0, 0.0, 1.0}, {0.0, 1.0, 0.0}, {1.0, 0.0, 1.0}},
-            this->exec));
+    auto rcm = this->rcm_factory->generate(this->id3_mtx);
+    auto rcm_copy = this->rcm_factory->generate(this->not_id3_mtx);
 
     rcm_copy->copy_from(rcm.get());
 
@@ -131,14 +112,8 @@ TYPED_TEST(Rcm, CanBeCopied)
 TYPED_TEST(Rcm, CanBeMoved)
 {
     using v_type = typename TestFixture::v_type;
-    auto rcm =
-        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-            3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
-            this->exec));
-    auto rcm_move =
-        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-            3, {{1.0, 0.0, 1.0}, {0.0, 1.0, 0.0}, {1.0, 0.0, 1.0}},
-            this->exec));
+    auto rcm = this->rcm_factory->generate(this->id3_mtx);
+    auto rcm_move = this->rcm_factory->generate(this->not_id3_mtx);
 
     rcm->move_to(rcm_move.get());
 
@@ -151,10 +126,7 @@ TYPED_TEST(Rcm, CanBeMoved)
 TYPED_TEST(Rcm, CanBeCloned)
 {
     using v_type = typename TestFixture::v_type;
-    auto rcm =
-        this->rcm_factory->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-            3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
-            this->exec));
+    auto rcm = this->rcm_factory->generate(this->id3_mtx);
 
     auto rcm_clone = rcm->clone();
 
@@ -168,11 +140,7 @@ TYPED_TEST(Rcm, HasSensibleDefaults)
     using v_type = typename TestFixture::v_type;
     using reorder_type = typename TestFixture::reorder_type;
 
-    auto rcm = reorder_type::build()
-                   .on(this->exec)
-                   ->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-                       3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
-                       this->exec));
+    auto rcm = reorder_type::build().on(this->exec)->generate(this->id3_mtx);
 
     ASSERT_EQ(rcm->get_parameters().construct_inverse_permutation, false);
     ASSERT_EQ(rcm->get_parameters().strategy,
@@ -202,9 +170,7 @@ TYPED_TEST(Rcm, CanBeCreatedWithConstructInversePermutation)
     auto rcm = reorder_type::build()
                    .with_construct_inverse_permutation(true)
                    .on(this->exec)
-                   ->generate(gko::initialize<gko::matrix::Dense<v_type>>(
-                       3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}},
-                       this->exec));
+                   ->generate(this->id3_mtx);
 
     ASSERT_EQ(rcm->get_inverse_permutation()->get_const_permutation()[0], 2);
     ASSERT_EQ(rcm->get_inverse_permutation()->get_const_permutation()[1], 1);
diff --git a/reference/test/reorder/rcm_kernels.cpp b/reference/test/reorder/rcm_kernels.cpp
index 9fec95760c4..393627a53bf 100644
--- a/reference/test/reorder/rcm_kernels.cpp
+++ b/reference/test/reorder/rcm_kernels.cpp
@@ -44,7 +44,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/permutation.hpp>
 
 
@@ -59,7 +58,6 @@ class Rcm : public ::testing::Test {
 protected:
     using v_type = double;
     using i_type = int;
-    using Mtx = gko::matrix::Dense<v_type>;
     using CsrMtx = gko::matrix::Csr<v_type, i_type>;
     using reorder_type = gko::reorder::Rcm<v_type, i_type>;
     using perm_type = gko::matrix::Permutation<i_type>;
@@ -99,54 +97,41 @@ class Rcm : public ::testing::Test {
     std::shared_ptr<CsrMtx> p_mtx_1;
     std::unique_ptr<reorder_type> reorder_op_1;
 
-    bool find_duplicates(i_type val, std::size_t index, const i_type *data,
-                         std::size_t length)
+    static bool is_permutation(const perm_type *input_perm)
     {
-        auto count = 0;
-        for (auto i = 0; i < length; ++i) {
-            if (i != index && val == data[i]) {
-                count++;
-            }
-        }
-        if (count == 0) {
-            return false;
-        } else {
-            return true;
-        }
-    }
-
-    void assert_correct_permutation(const perm_type *input_perm)
-    {
-        auto perm_data = input_perm->get_const_permutation();
-        auto perm_size = input_perm->get_permutation_size();
-
-        for (auto i = 0; i < perm_size; ++i) {
-            ASSERT_LT(perm_data[i], perm_size);
-            ASSERT_GE(perm_data[i], 0);
-            ASSERT_FALSE(
-                find_duplicates(perm_data[i], i, perm_data, perm_size));
-        }
+        const auto perm_size = input_perm->get_permutation_size();
+        auto perm_sorted = std::vector<i_type>(perm_size);
+        std::copy_n(input_perm->get_const_permutation(), perm_size,
+                    perm_sorted.begin());
+        std::sort(perm_sorted.begin(), perm_sorted.end());
+        auto identity = std::vector<i_type>(perm_size);
+        std::iota(identity.begin(), identity.end(), 0);
+        return identity == perm_sorted;
     }
 };
 
 TEST_F(Rcm, CreatesAPermutation)
 {
     auto p = reorder_op_0->get_permutation();
-    assert_correct_permutation(p.get());
+    ASSERT_PRED1(is_permutation, p.get());
+}
+
+TEST_F(Rcm, CreatesCorrectPermutation)
+{
+    std::vector<i_type> correct = {2, 3, 1, 0, 4};
+
+    auto p = reorder_op_0->get_permutation()->get_const_permutation();
+
+    ASSERT_TRUE(std::equal(p, p + correct.size(), correct.begin()));
 }
 
 TEST_F(Rcm, PermutesPerfectFullBand)
 {
+    std::vector<i_type> correct = {7, 8, 0, 4, 3, 5, 6, 1, 2};
+
     auto p = reorder_op_1->get_permutation()->get_const_permutation();
-    // These perms are hand-crafted for the matrix given in the ctor.
-    // They are same graph renumbering, reversed.
-    std::vector<i_type> correct_0 = {2, 1, 6, 5, 3, 4, 0, 8, 7};
-    std::vector<i_type> correct_1 = {7, 8, 0, 4, 3, 5, 6, 1, 2};
-
-    auto eq_0 = std::equal(p, p + correct_0.size(), correct_0.begin());
-    auto eq_1 = std::equal(p, p + correct_1.size(), correct_1.begin());
-    const auto incl_or = [](bool a, bool b) { return a || b; };
-    ASSERT_PRED2(incl_or, eq_0, eq_1);
+
+    ASSERT_TRUE(std::equal(p, p + correct.size(), correct.begin()));
 }
 
 }  // namespace

From 89fd7a3bac4218638b5101580c7347ee68373a5c Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Wed, 9 Sep 2020 08:35:54 +0200
Subject: [PATCH 0064/1008] Fix intel include error

Includes immintrin.h only if explicitly not intel compiler.
---
 omp/reorder/rcm_kernels.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index 8f8862606ec..97f6ee7c00a 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -72,7 +72,7 @@ namespace rcm {
 
 
 #ifdef __x86_64__
-#if defined(__GNUG__) || defined(__clang__)
+#if (defined(__GNUG__) || defined(__clang__)) && !defined(__INTEL_COMPILER)
 #include <immintrin.h>
 #endif
 #define GKO_MM_PAUSE() _mm_pause()

From 7e3e206417af5dcda5cbab2dfec6012f19cdb3c3 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Wed, 9 Sep 2020 11:53:58 +0200
Subject: [PATCH 0065/1008] Remove use of omp 5.0 feature

omp_init_lock_with_hint is an OpenMP 5.0 feature.
This is not significant from a performance perspective.
---
 omp/components/omp_mutex.hpp | 1 -
 omp/reorder/rcm_kernels.cpp  | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/omp/components/omp_mutex.hpp b/omp/components/omp_mutex.hpp
index 5f195be50ec..5c6db0212f6 100644
--- a/omp/components/omp_mutex.hpp
+++ b/omp/components/omp_mutex.hpp
@@ -40,7 +40,6 @@ namespace omp {
 
 struct omp_mutex {
     omp_mutex() { omp_init_lock(&lock_); }
-    omp_mutex(omp_lock_hint_t hint) { omp_init_lock_with_hint(&lock_, hint); }
     ~omp_mutex() { omp_destroy_lock(&lock_); }
 
     omp_mutex(const omp_mutex &) = delete;
diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index 97f6ee7c00a..b430f4f1eb6 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -117,8 +117,8 @@ struct UbfsLinearQueue {
         : arr(vector<IndexType>(capacity, exec)),
           head(0),
           tail(0),
-          read_mutex(omp_lock_hint_uncontended),
-          write_mutex(omp_lock_hint_uncontended)
+          read_mutex(),
+          write_mutex()
     {}
 
     UbfsLinearQueue(UbfsLinearQueue &other) = delete;

From 856dd9e095c62cf0d3672b0cc0e14b78655a64cc Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Fri, 11 Sep 2020 20:09:20 +0200
Subject: [PATCH 0066/1008] Apply suggestions from code review
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Include types.hpp to make CI happy.
Add include guards to omp-only headers.
Remove an unnecessary #undef.
Refactor count_lelvels, including explicit types, simpler control flow.
Move all responsibility for prefix sum out of count_levels.
Make some constants types explicit.
Remove unnecessary includes.
Rewrite three reductions as non-complex omp reductions.
Make some types related to omp functions explicit.
Update to GKO_FACTORY_PARAMETER_SCALAR.
Minor changes, mostly explicit types and formatting.
Make the comparator macro a function.
Introduce builtin cmpxchg support for all supported compilers.
Manually format omp reductions to 80 chars.
Add documenting comment to omp_mutex.
Remove unnecessary undef of macro in cpp file.
Minor changes to formatting, naming, documentation.
Formatting changes, introduce type aliases v_type, i_type to Rcm.
Remove unnecessary break.
Use gko::share instead of releasing unique_ptr.
Remove autos in rcm tests.
Move comparator to detail namespace.
Rewrite omp reductions as manual reductions.
Fix typo.

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 core/test/reorder/rcm.cpp              |   1 -
 include/ginkgo/core/reorder/rcm.hpp    |   6 +-
 omp/components/omp_mutex.hpp           |  14 +-
 omp/components/sort_small.hpp          | 137 ++++++------
 omp/reorder/rcm_kernels.cpp            | 278 ++++++++++++++-----------
 omp/test/reorder/rcm_kernels.cpp       |  28 ++-
 reference/reorder/rcm_kernels.cpp      |   2 +-
 reference/test/reorder/rcm.cpp         |  14 +-
 reference/test/reorder/rcm_kernels.cpp |   6 +-
 9 files changed, 274 insertions(+), 212 deletions(-)

diff --git a/core/test/reorder/rcm.cpp b/core/test/reorder/rcm.cpp
index 2f452f09ae7..030db177fa3 100644
--- a/core/test/reorder/rcm.cpp
+++ b/core/test/reorder/rcm.cpp
@@ -40,7 +40,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
 
 
 #include "core/test/utils.hpp"
diff --git a/include/ginkgo/core/reorder/rcm.hpp b/include/ginkgo/core/reorder/rcm.hpp
index 28629bcbd4e..a22abaf8bcb 100644
--- a/include/ginkgo/core/reorder/rcm.hpp
+++ b/include/ginkgo/core/reorder/rcm.hpp
@@ -100,6 +100,8 @@ class Rcm
 public:
     using SparsityMatrix = matrix::SparsityCsr<ValueType, IndexType>;
     using PermutationMatrix = matrix::Permutation<IndexType>;
+    using value_type = ValueType;
+    using index_type = IndexType;
 
     /**
      * Gets the permutation (permutation matrix, output of the algorithm) of the
@@ -129,13 +131,13 @@ class Rcm
          * If this parameter is set then an inverse permutation matrix is also
          * constructed along with the normal permutation matrix.
          */
-        bool GKO_FACTORY_PARAMETER(construct_inverse_permutation, false);
+        bool GKO_FACTORY_PARAMETER_SCALAR(construct_inverse_permutation, false);
 
         /**
          * This parameter controls the strategy used to determine a starting
          * vertex.
          */
-        starting_strategy GKO_FACTORY_PARAMETER(
+        starting_strategy GKO_FACTORY_PARAMETER_SCALAR(
             strategy, starting_strategy::pseudo_peripheral);
     };
     GKO_ENABLE_REORDERING_BASE_FACTORY(Rcm, parameters, Factory);
diff --git a/omp/components/omp_mutex.hpp b/omp/components/omp_mutex.hpp
index 5c6db0212f6..2f1c410b9a5 100644
--- a/omp/components/omp_mutex.hpp
+++ b/omp/components/omp_mutex.hpp
@@ -30,6 +30,10 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#ifndef GKO_OMP_COMPONENTS_OMP_MUTEX_HPP_
+#define GKO_OMP_COMPONENTS_OMP_MUTEX_HPP_
+
+
 #include <omp.h>
 
 
@@ -37,7 +41,12 @@ namespace gko {
 namespace kernels {
 namespace omp {
 
-
+/**
+ * Wrapper class for the OpenMP mutex omp_lock_t.
+ *
+ * Fulfills the BasicLockable requirement, which means it can be used with
+ * std::lock_guard<omp_mutex>, making RAII possible.
+ */
 struct omp_mutex {
     omp_mutex() { omp_init_lock(&lock_); }
     ~omp_mutex() { omp_destroy_lock(&lock_); }
@@ -59,3 +68,6 @@ struct omp_mutex {
 }  // namespace omp
 }  // namespace kernels
 }  // namespace gko
+
+
+#endif  // GKO_OMP_COMPONENTS_OMP_MUTEX_HPP_
diff --git a/omp/components/sort_small.hpp b/omp/components/sort_small.hpp
index 186de94907f..cb93b55e001 100644
--- a/omp/components/sort_small.hpp
+++ b/omp/components/sort_small.hpp
@@ -30,38 +30,54 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#ifndef GKO_OMP_COMPONENTS_SORT_SMALL_HPP_
+#define GKO_OMP_COMPONENTS_SORT_SMALL_HPP_
+
+
 #include <algorithm>
 
 
+#include <ginkgo/core/base/types.hpp>
+
+
 namespace gko {
 namespace kernels {
 namespace omp {
 
 
 #ifdef __GNUG__
-#define GKO_GCC_EXPECT_LIKELY(v) __builtin_expect(v, true)
+#define GKO_GNU_EXPECT_LIKELY(v) __builtin_expect(v, true)
 #else
-#define GKO_GCC_EXPECT_LIKELY(v) v
-#endif
+#define GKO_GNU_EXPECT_LIKELY(v) v
+#endif  // defined __GNUG__
+
+
+namespace detail {
+
 
 #ifdef _MSC_VER
-#define GKO_COMPARATOR(x, y)                            \
-    {                                                   \
-        const auto should_swap = !comp(arr[x], arr[y]); \
-        const auto tmp = arr[x];                        \
-        arr[x] = should_swap ? arr[y] : arr[x];         \
-        arr[y] = should_swap ? tmp : arr[y];            \
-    }
+template <typename TargetType, typename Functor>
+inline void comparator(TargetType *arr, size_type x, size_type y, Functor comp)
+{
+    const auto should_swap = !comp(arr[x], arr[y]);
+    const auto tmp = arr[x];
+    arr[x] = should_swap ? arr[y] : arr[x];
+    arr[y] = should_swap ? tmp : arr[y];
+}
 #else
-#define GKO_COMPARATOR(x, y)                                         \
-    {                                                                \
-        const auto should_swap = !comp(arr[x], arr[y]);              \
-        const auto tx = arr[x];                                      \
-        const auto ty = arr[y];                                      \
-        arr[x] = ((-(should_swap)) & ty) | ((should_swap - 1) & tx); \
-        arr[y] = ((-(should_swap)) & tx) | ((should_swap - 1) & ty); \
-    }
-#endif
+template <typename TargetType, typename Functor>
+inline void comparator(TargetType *arr, size_type x, size_type y, Functor comp)
+{
+    const auto tx = arr[x];
+    const auto ty = arr[y];
+    const auto should_swap = !comp(tx, ty);
+    arr[x] = ((-should_swap) & ty) | ((should_swap - 1) & tx);
+    arr[y] = ((-should_swap) & tx) | ((should_swap - 1) & ty);
+}
+#endif  // defined _MSC_VER
+
+
+}  // namespace detail
 
 
 /**
@@ -73,74 +89,78 @@ namespace omp {
 template <typename TargetType, typename Functor>
 void sort_small(TargetType *arr, size_type n, Functor comp)
 {
+    const auto sort2 = [&](size_type x, size_type y) {
+        detail::comparator(arr, x, y, comp);
+    };
+
     if (n < 2) {
         return;
     }
     if (n == 2) {
-        GKO_COMPARATOR(0, 1);
+        sort2(0, 1);
         return;
     }
     if (n == 3) {
-        GKO_COMPARATOR(1, 2);
-        GKO_COMPARATOR(0, 2);
-        GKO_COMPARATOR(0, 1);
+        sort2(1, 2);
+        sort2(0, 2);
+        sort2(0, 1);
         return;
     }
     if (n == 4) {
         // clang-format off
-        GKO_COMPARATOR(0, 1)GKO_COMPARATOR(2, 3)
-        GKO_COMPARATOR(0, 2)GKO_COMPARATOR(1, 3)
-        GKO_COMPARATOR(1, 2);
+        sort2(0,1);sort2(2,3);
+        sort2(0,2);sort2(1,3);
+        sort2(1,2);
         // clang-format on
         return;
     }
     if (n == 5) {
         // clang-format off
-        GKO_COMPARATOR(0, 4)GKO_COMPARATOR(1, 3)
-        GKO_COMPARATOR(0, 2)
-        GKO_COMPARATOR(2, 4)GKO_COMPARATOR(0, 1)
-        GKO_COMPARATOR(2, 3)GKO_COMPARATOR(1, 4)
-        GKO_COMPARATOR(1, 2)GKO_COMPARATOR(3, 4);
+        sort2(0,4);sort2(1,3);
+        sort2(0,2);
+        sort2(2,4);sort2(0,1);
+        sort2(2,3);sort2(1,4);
+        sort2(1,2);sort2(3,4);
         // clang-format on
         return;
     }
     if (n == 6) {
         // clang-format off
-        GKO_COMPARATOR(0, 4)GKO_COMPARATOR(1, 5)
-        GKO_COMPARATOR(0, 2)GKO_COMPARATOR(1, 3)
-        GKO_COMPARATOR(2, 4)GKO_COMPARATOR(3, 5)GKO_COMPARATOR(0, 1)
-        GKO_COMPARATOR(2, 3)GKO_COMPARATOR(4, 5)
-        GKO_COMPARATOR(1, 4)
-        GKO_COMPARATOR(1, 2)GKO_COMPARATOR(3, 4);
+        sort2(0,4);sort2(1,5);
+        sort2(0,2);sort2(1,3);
+        sort2(2,4);sort2(3,5);sort2(0,1);
+        sort2(2,3);sort2(4,5);
+        sort2(1,4);
+        sort2(1,2);sort2(3,4);
         // clang-format on
         return;
     }
     if (n == 7) {
         // clang-format off
-        GKO_COMPARATOR(0,4)GKO_COMPARATOR(1,5)GKO_COMPARATOR(2,6)
-        GKO_COMPARATOR(0,2)GKO_COMPARATOR(1,3)GKO_COMPARATOR(4,6)
-        GKO_COMPARATOR(2,4)GKO_COMPARATOR(3,5)GKO_COMPARATOR(0,1)
-        GKO_COMPARATOR(2,3)GKO_COMPARATOR(4,5)
-        GKO_COMPARATOR(1,4)GKO_COMPARATOR(3,6)
-        GKO_COMPARATOR(1,2)GKO_COMPARATOR(3,4)GKO_COMPARATOR(5,6);
+        sort2(0,4);sort2(1,5);sort2(2,6);
+        sort2(0,2);sort2(1,3);sort2(4,6);
+        sort2(2,4);sort2(3,5);sort2(0,1);
+        sort2(2,3);sort2(4,5);
+        sort2(1,4);sort2(3,6);
+        sort2(1,2);sort2(3,4);sort2(5,6);
         // clang-format on
         return;
     }
     if (n < 128) {
         // clang-format off
-        GKO_COMPARATOR(0,4)GKO_COMPARATOR(1,5)GKO_COMPARATOR(2,6)GKO_COMPARATOR(3,7)
-        GKO_COMPARATOR(0,2)GKO_COMPARATOR(1,3)GKO_COMPARATOR(4,6)GKO_COMPARATOR(5,7)
-        GKO_COMPARATOR(2,4)GKO_COMPARATOR(3,5)GKO_COMPARATOR(0,1)GKO_COMPARATOR(6,7)
-        GKO_COMPARATOR(2,3)GKO_COMPARATOR(4,5)
-        GKO_COMPARATOR(1,4)GKO_COMPARATOR(3,6)
-        GKO_COMPARATOR(1,2)GKO_COMPARATOR(3,4)GKO_COMPARATOR(5,6);
+        sort2(0,4);sort2(1,5);sort2(2,6);sort2(3,7);
+        sort2(0,2);sort2(1,3);sort2(4,6);sort2(5,7);
+        sort2(2,4);sort2(3,5);sort2(0,1);sort2(6,7);
+        sort2(2,3);sort2(4,5);
+        sort2(1,4);sort2(3,6);
+        sort2(1,2);sort2(3,4);sort2(5,6);
         // clang-format on
 
         const int8 m = n < 48 ? n : 48;
         for (int8 i = 8; i < m; ++i) {
             auto j = i;
             const auto tmp = arr[j];
-            for (; GKO_GCC_EXPECT_LIKELY(j >= 2 && comp(tmp, arr[j - 2]));
+            for (; GKO_GNU_EXPECT_LIKELY(j >= 2 && comp(tmp, arr[j - 2]));
                  j -= 2) {
                 std::swap(arr[j - 1], arr[j]);
                 std::swap(arr[j - 2], arr[j - 1]);
@@ -149,12 +169,12 @@ void sort_small(TargetType *arr, size_type n, Functor comp)
             if (j == 0) {
                 continue;
             }
-            GKO_COMPARATOR(j - 1, j);
+            sort2(j - 1, j);
         }
         for (int8 i = 48; i < n; ++i) {
             auto j = i;
             const auto tmp = arr[j];
-            for (; GKO_GCC_EXPECT_LIKELY(j >= 4 && comp(tmp, arr[j - 4]));
+            for (; GKO_GNU_EXPECT_LIKELY(j >= 4 && comp(tmp, arr[j - 4]));
                  j -= 4) {
                 std::swap(arr[j - 1], arr[j]);
                 std::swap(arr[j - 2], arr[j - 1]);
@@ -165,15 +185,15 @@ void sort_small(TargetType *arr, size_type n, Functor comp)
             if (j == 0) {
                 continue;
             }
-            GKO_COMPARATOR(j - 1, j);
+            sort2(j - 1, j);
             if (j == 1) {
                 continue;
             }
-            GKO_COMPARATOR(j - 2, j - 1);
+            sort2(j - 2, j - 1);
             if (j == 2) {
                 continue;
             }
-            GKO_COMPARATOR(j - 3, j - 2);
+            sort2(j - 3, j - 2);
         }
         return;
     }
@@ -181,10 +201,9 @@ void sort_small(TargetType *arr, size_type n, Functor comp)
 }
 
 
-#undef GKO_GCC_EXPECT_LIKELY
-#undef GKO_COMPARATOR
-
-
 }  // namespace omp
 }  // namespace kernels
 }  // namespace gko
+
+
+#endif  // GKO_OMP_COMPONENTS_SORT_SMALL_HPP_
diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index b430f4f1eb6..a5966fddef2 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -74,11 +74,13 @@ namespace rcm {
 #ifdef __x86_64__
 #if (defined(__GNUG__) || defined(__clang__)) && !defined(__INTEL_COMPILER)
 #include <immintrin.h>
-#endif
+#endif  // (defined(__GNUG__) || defined(__clang__)) &&
+        // !defined(__INTEL_COMPILER)
 #define GKO_MM_PAUSE() _mm_pause()
 #else
-#define GKO_MM_PAUSE()  // No equivalent instruction.
-#endif
+// No equivalent instruction.
+#define GKO_MM_PAUSE()
+#endif  // defined __x86_64__
 
 template <typename IndexType>
 void get_degree_of_nodes(std::shared_ptr<const OmpExecutor> exec,
@@ -98,7 +100,7 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 // This constant controls how many nodes can be dequeued from the
 // UbfsLinearQueue at once at most. Increasing it reduces lock contention and
 // "unneccesary work", but disturbs queue ordering, generating extra work.
-constexpr auto chunk_bound = 512;
+constexpr int32 chunk_bound = 512;
 
 
 template <typename IndexType>
@@ -207,6 +209,50 @@ struct UbfsLinearQueue {
 };
 
 
+#ifdef _MSC_VER
+#define GKO_CMPXCHG_IMPL(ptr, ptr_expected, replace_with)                      \
+    if (sizeof replace_with == 8) {                                            \
+        return _InterlockedCompareExchange64(reinterpret_cast<int64_t *>(ptr), \
+                                             replace_with,                     \
+                                             *ptr_expected) == *ptr_expected;  \
+    }                                                                          \
+    if (sizeof replace_with == 4) {                                            \
+        return _InterlockedCompareExchange(reinterpret_cast<long *>(ptr),      \
+                                           replace_with,                       \
+                                           *ptr_expected) == *ptr_expected;    \
+    }                                                                          \
+    if (sizeof replace_with == 2) {                                            \
+        return _InterlockedCompareExchange16(reinterpret_cast<short *>(ptr),   \
+                                             replace_with,                     \
+                                             *ptr_expected) == *ptr_expected;  \
+    }                                                                          \
+    if (sizeof replace_with == 1) {                                            \
+        return _InterlockedCompareExchange8(reinterpret_cast<char *>(ptr),     \
+                                            replace_with,                      \
+                                            *ptr_expected) == *ptr_expected;   \
+    }
+#else
+#define GKO_CMPXCHG_IMPL(ptr, ptr_expected, replace_with) \
+    return __atomic_compare_exchange_n(                   \
+        ptr, ptr_expected, replace_with, true,            \
+        std::memory_order::memory_order_acq_rel,          \
+        std::memory_order::memory_order_acquire);
+#endif
+
+/**
+ * Basic building block for CAS loops.
+ * Note that "weak" and "acqrel" are only the minimum guarantees made.
+ * Usage with types of size > 8 bytes is undefined behaviour.
+ * Usage with non-primitive types is explicitly discouraged.
+ */
+template <typename TargetType>
+inline bool compare_exchange_weak_acqrel(TargetType *value, TargetType old,
+                                         TargetType newer)
+{
+    GKO_CMPXCHG_IMPL(value, &old, newer)
+}
+
+
 template <typename IndexType>
 inline void reduce_neighbours_levels(const IndexType num_vertices,
                                      const IndexType *const row_ptrs,
@@ -247,10 +293,8 @@ inline void reduce_neighbours_levels(const IndexType num_vertices,
                     return true;
                 }
 
-            } while (!__atomic_compare_exchange_n(
-                &levels[neighbour], &old_neighbour_level, new_neighbour_level,
-                true, std::memory_order::memory_order_acq_rel,
-                std::memory_order::memory_order_acquire));
+            } while (!compare_exchange_weak_acqrel(
+                &levels[neighbour], old_neighbour_level, new_neighbour_level));
             return false;
         };
 
@@ -275,7 +319,7 @@ void ubfs(std::shared_ptr<const OmpExecutor> exec, const IndexType num_vertices,
               levels,  // Must be inf/max in all nodes connected to source
           const IndexType start, const IndexType max_degree)
 {
-    const auto max_threads = omp_get_max_threads();
+    const int32 max_threads = omp_get_max_threads();
 
     // This is an upper bound for how many nodes may be in the work set
     // at maximum at any time.
@@ -346,59 +390,50 @@ std::pair<IndexType, IndexType> rls_contender_and_height(
     IndexType *const levels,  // Must be max/inf in all nodes connected to start
     const IndexType start, const IndexType max_degree)
 {
+    // Layout: ((level, degree), idx).
+    using contending = std::pair<std::pair<IndexType, IndexType>, IndexType>;
+
     // Create a level structure.
     ubfs(exec, num_vertices, row_ptrs, col_idxs, levels, start, max_degree);
 
-    // Now find the node in the last level with minimal degree.
-    // TODO: Rewrite this as a custom reduction.
-
-    // First local ...
-    const auto num_threads = omp_get_max_threads();
-    vector<IndexType> local_contenders(num_threads, exec);
-    vector<IndexType> local_degrees(num_threads, exec);
-    vector<IndexType> local_heights(num_threads, exec);
+    // Find a node in the last level with minimal degree, the "contender".
+    // Implement this through a tie-max reduction. First reduce local ...
+    const int32 num_threads = omp_get_max_threads();
+    const auto initial_value =
+        std::make_pair(std::make_pair(levels[0], degrees[0]), 0);
+    vector<contending> local_contenders(num_threads, initial_value, exec);
 
 #pragma omp parallel num_threads(num_threads)
     {
-        const auto tid = omp_get_thread_num();
-        IndexType local_contender = 0;
-        IndexType local_degree = std::numeric_limits<IndexType>::max();
-        IndexType local_height = 0;
+        const int32 tid = omp_get_thread_num();
+        auto local_contender = initial_value;
 
 #pragma omp for schedule(static)
-        for (IndexType i = 0; i < num_vertices; ++i) {
-            if (levels[i] > local_height) {
-                local_contender = i;
-                local_degree = degrees[i];
-                local_height = levels[i];
-            } else if (levels[i] == local_height && degrees[i] > local_degree) {
-                local_contender = i;
-                local_degree = degrees[i];
+        for (IndexType i = 1; i < num_vertices; ++i) {
+            if (std::tie(levels[i], degrees[i]) >
+                std::tie(local_contender.first.first,
+                         local_contender.first.second)) {
+                local_contender.first = std::make_pair(levels[i], degrees[i]);
+                local_contender.second = i;
             }
         }
 
         local_contenders[tid] = local_contender;
-        local_degrees[tid] = local_degree;
-        local_heights[tid] = local_height;
     }
 
     // ... then global.
-    auto global_contender = local_contenders[0];
-    auto global_degree = local_degrees[0];
-    auto global_height = local_heights[0];
-    for (IndexType i = 1; i < num_threads; ++i) {
-        if (local_heights[i] > global_height) {
+    auto global_contender = initial_value;
+    for (int32 i = 0; i < num_threads; ++i) {
+        if (std::tie(local_contenders[i].first.first,
+                     local_contenders[i].first.second) >
+            std::tie(global_contender.first.first,
+                     global_contender.first.second)) {
             global_contender = local_contenders[i];
-            global_degree = local_degrees[i];
-            global_height = local_heights[i];
-        } else if (local_heights[i] == global_height &&
-                   local_degrees[i] > global_degree) {
-            global_contender = local_contenders[i];
-            global_degree = local_degrees[i];
         }
     }
 
-    return std::make_pair(global_contender, global_height);
+    return std::make_pair(global_contender.second,
+                          global_contender.first.first);
 }
 
 
@@ -413,54 +448,56 @@ std::pair<IndexType, IndexType> find_min_idx_and_max_val(
     const uint8 *const previous_component,
     gko::reorder::starting_strategy strategy)
 {
-    // First find local extrema.
-    const auto num_threads = omp_get_max_threads();
-    vector<std::pair<IndexType, IndexType>> local_min_vals_idxs(num_threads,
-                                                                exec);
-    vector<IndexType> local_max_vals(num_threads, exec);
+    // Layout: ((min_val, min_idx), (max_val, max_idx)).
+    using minmax = std::pair<std::pair<IndexType, IndexType>,
+                             std::pair<IndexType, IndexType>>;
+
+    // Min-and-max reduction: First local ...
+    const int32 num_threads = omp_get_max_threads();
+    const auto initial_value = std::make_pair(
+        std::make_pair(std::numeric_limits<IndexType>::max(), IndexType{}),
+        std::make_pair(std::numeric_limits<IndexType>::min(), IndexType{}));
+    vector<minmax> local_minmaxs(num_threads, initial_value, exec);
 
 #pragma omp parallel num_threads(num_threads)
     {
-        const auto tid = omp_get_thread_num();
-        auto local_min_val = std::numeric_limits<IndexType>::max();
-        auto local_min_idx = IndexType{};
-        auto local_max_val = std::numeric_limits<IndexType>::min();
-        auto local_max_idx = IndexType{};
+        const int32 tid = omp_get_thread_num();
+        auto local_minmax = initial_value;
 
 #pragma omp for schedule(static)
         for (IndexType i = 0; i < num_vertices; ++i) {
             // If this level hasn't been discovered before.
             if (!previous_component[i]) {
-                if (degrees[i] < local_min_val) {
-                    local_min_val = degrees[i];
-                    local_min_idx = i;
+                if (degrees[i] < local_minmax.first.first) {
+                    local_minmax.first = std::make_pair(degrees[i], i);
                 }
-                if (degrees[i] > local_max_val) {
-                    local_max_val = degrees[i];
-                    local_max_idx = i;
+                if (degrees[i] > local_minmax.second.first) {
+                    local_minmax.second = std::make_pair(degrees[i], i);
                 }
             }
         }
 
-        local_min_vals_idxs[tid] = std::make_pair(local_min_val, local_min_idx);
-        local_max_vals[tid] = local_max_val;
+        local_minmaxs[tid] = local_minmax;
     }
 
-    // Then find global extrema.
-    auto global_min_idx = local_min_vals_idxs[0].second;
-    auto global_min_val = local_min_vals_idxs[0].first;
-    auto global_max_val = local_max_vals[0];
-    for (IndexType i = 1; i < num_threads; ++i) {
-        if (local_min_vals_idxs[i].first < global_min_val) {
-            global_min_val = local_min_vals_idxs[i].first;
-            global_min_idx = local_min_vals_idxs[i].second;
+    // ... then global.
+    auto global_minmax = initial_value;
+    for (IndexType i = 0; i < num_threads; ++i) {
+        // If this level hasn't been discovered before.
+        if (!previous_component[local_minmaxs[i].first.second]) {
+            if (local_minmaxs[i].first.first < global_minmax.first.first) {
+                global_minmax.first = local_minmaxs[i].first;
+            }
         }
-        if (local_max_vals[i] > global_max_val) {
-            global_max_val = local_max_vals[i];
+        if (!previous_component[local_minmaxs[i].second.second]) {
+            if (local_minmaxs[i].second.first > global_minmax.second.first) {
+                global_minmax.second = local_minmaxs[i].second;
+            }
         }
     }
 
-    return std::make_pair(global_min_idx, global_max_val);
+    return std::make_pair(global_minmax.first.second,
+                          global_minmax.second.first);
 }
 
 
@@ -537,13 +574,13 @@ vector<IndexType> count_levels(std::shared_ptr<const OmpExecutor> exec,
                                uint8 *const previous_component,
                                IndexType num_vertices)
 {
-    const auto num_threads = omp_get_max_threads();
+    const int32 num_threads = omp_get_max_threads();
     vector<vector<IndexType>> level_counts(num_threads, vector<IndexType>(exec),
                                            exec);
 
 #pragma omp parallel num_threads(num_threads)
     {
-        const auto tid = omp_get_thread_num();
+        const int32 tid = omp_get_thread_num();
         auto local_level_counts = &level_counts[tid];
 
 #pragma omp for schedule(static)
@@ -563,29 +600,15 @@ vector<IndexType> count_levels(std::shared_ptr<const OmpExecutor> exec,
         }
     }
 
-    vector<size_type> level_count_sizes(num_threads, exec);
-    for (auto tid = 0; tid < num_threads; ++tid) {
-        level_count_sizes[tid] = level_counts[tid].size();
-    }
-    const auto max_size =
-        *std::max_element(level_count_sizes.begin(), level_count_sizes.end());
-
-    vector<IndexType> final_level_counts(max_size + 1, exec);
-    auto i = 0;
-    while (true) {
-        auto done = true;
-        for (std::remove_const_t<decltype(num_threads)> tid = 0;
-             tid < num_threads; ++tid) {
-            if (i < level_count_sizes[tid]) {
-                const auto count = level_counts[tid][i];
-                final_level_counts[i] += count;
-                done = false;
+    vector<IndexType> final_level_counts(exec);
+    for (int32 tid = 0; tid < num_threads; ++tid) {
+        for (IndexType i = 0; i < level_counts[tid].size(); ++i) {
+            if (final_level_counts.size() <= i) {
+                final_level_counts.push_back(0);
             }
+            const auto local_count = level_counts[tid][i];
+            final_level_counts[i] += local_count;
         }
-        if (done) {
-            break;
-        }
-        ++i;
     }
 
     return final_level_counts;
@@ -604,21 +627,22 @@ vector<IndexType> compute_level_offsets(std::shared_ptr<const OmpExecutor> exec,
                                         uint8 *const previous_component)
 {
     auto counts = count_levels(exec, levels, previous_component, num_vertices);
+    counts.push_back(0);
     components::prefix_sum(exec, &counts[0], counts.size());
     return counts;
 }
 
 
-// Signal value to wich the entire permutation is intialized.
+// Signal value to which the entire permutation is intialized.
 // Threads spin on this value, until it is replaced by another value,
 // written by another thread.
-constexpr auto perm_untouched = -1;
+constexpr int32 perm_untouched = -1;
 
 // Signal value which a thread writes to a level (as in the level of a node
 // becomes -1), to signal that it has been processed. This information is only
 // relevant local to the thread, since only a single thread is responsible for a
 // node.
-constexpr auto level_processed = -1;
+constexpr int32 level_processed = -1;
 
 /**
  * Implements the last phase of urcm,
@@ -637,10 +661,10 @@ void write_permutation(std::shared_ptr<const OmpExecutor> exec,
     const IndexType num_levels = offsets.size() - 1;
     perm[base_offset] = start;
 
-    const auto num_threads = omp_get_max_threads();
+    const int32 num_threads = omp_get_max_threads();
 #pragma omp parallel num_threads(num_threads) firstprivate(num_levels)
     {
-        const auto tid = omp_get_thread_num();
+        const int32 tid = omp_get_thread_num();
         vector<IndexType> valid_neighbours(0, exec);
 
         // Go through the levels assigned to this thread.
@@ -725,32 +749,35 @@ IndexType handle_isolated_nodes(std::shared_ptr<const OmpExecutor> exec,
                                 IndexType *const perm, IndexType num_vertices,
                                 vector<uint8> &previous_component)
 {
-    struct IsolatedNodes {
-        // Using a gko::vector here makes clang and gcc segfault,
-        // if we then use 'omp_priv = omp_orig'.
-        std::vector<IndexType> nodes;
-        IsolatedNodes &operator+=(const IsolatedNodes &rhs)
-        {
-            nodes.reserve(nodes.size() + rhs.nodes.size());
-            nodes.insert(nodes.end(), rhs.nodes.begin(), rhs.nodes.end());
-            return *this;
-        }
-    };
+    const int32 num_threads = omp_get_max_threads();
+    vector<vector<IndexType>> local_isolated_nodes(
+        num_threads, vector<IndexType>(exec), exec);
 
-#pragma omp declare reduction(FindIsolated : IsolatedNodes : omp_out += omp_in)
-    auto isolated = IsolatedNodes();
-#pragma omp parallel for reduction(FindIsolated : isolated)
-    for (IndexType i = 0; i < num_vertices; ++i) {
-        // No need to check for diagonal elements (only self-neighbouring) here,
-        // those are already removed from the matrix.
-        if (degrees[i] == 0) {
-            isolated.nodes.push_back(i);
-            previous_component[i] = true;
+#pragma omp parallel
+    {
+        const int32 tid = omp_get_thread_num();
+
+#pragma omp for schedule(static)
+        for (IndexType i = 0; i < num_vertices; ++i) {
+            // No need to check for diagonal elements (only self-neighbouring)
+            // here, those are already removed from the matrix.
+            if (degrees[i] == 0) {
+                local_isolated_nodes[tid].push_back(i);
+                previous_component[i] = true;
+            }
         }
     }
 
-    std::copy_n(isolated.nodes.begin(), isolated.nodes.size(), perm);
-    return isolated.nodes.size();
+    const auto isolated_nodes = std::accumulate(
+        local_isolated_nodes.begin(), local_isolated_nodes.end(),
+        vector<IndexType>(exec), [](vector<IndexType> a, vector<IndexType> b) {
+            a.reserve(a.size() + b.size());
+            a.insert(a.end(), b.begin(), b.end());
+            return a;
+        });
+
+    std::copy_n(isolated_nodes.begin(), isolated_nodes.size(), perm);
+    return isolated_nodes.size();
 }
 
 
@@ -773,7 +800,7 @@ void get_permutation(std::shared_ptr<const OmpExecutor> exec,
     IndexType base_offset = 0;
 
     // Stores for each node if it is part of an already discovered component.
-    vector<uint8> previous_component(num_vertices, exec);
+    vector<uint8> previous_component(num_vertices, 0, exec);
 
     // First handle all isolated nodes. That reduces complexity later on.
     base_offset +=
@@ -845,7 +872,4 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
 }  // namespace rcm
 }  // namespace omp
 }  // namespace kernels
-}  // namespace gko
-
-#undef GKO_MM_PAUSE
-#undef GKO_COMPARATOR
+}  // namespace gko
\ No newline at end of file
diff --git a/omp/test/reorder/rcm_kernels.cpp b/omp/test/reorder/rcm_kernels.cpp
index c145f98433f..8fcad63a4f4 100644
--- a/omp/test/reorder/rcm_kernels.cpp
+++ b/omp/test/reorder/rcm_kernels.cpp
@@ -47,11 +47,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/permutation.hpp>
 
 
-#include <iostream>
 #include "core/test/utils.hpp"
 #include "core/test/utils/assertions.hpp"
 #include "matrices/config.hpp"
 
+
 namespace {
 
 
@@ -118,7 +118,7 @@ class Rcm : public ::testing::Test {
 
         const auto n = reorder->get_permutation()->get_permutation_size();
         auto degrees = std::vector<i_type>(n);
-        for (auto i = 0; i < n; ++i) {
+        for (gko::size_type i = 0; i < n; ++i) {
             degrees[i] =
                 mtx->get_const_row_ptrs()[i + 1] - mtx->get_const_row_ptrs()[i];
         }
@@ -126,7 +126,7 @@ class Rcm : public ::testing::Test {
         switch (reorder->get_parameters().strategy) {
         case strategy::minimum_degree: {
             auto min_degree = std::numeric_limits<i_type>::max();
-            for (auto i = 0; i < n; ++i) {
+            for (gko::size_type i = 0; i < n; ++i) {
                 if (!already_visited[i] && degrees[i] < min_degree) {
                     min_degree = degrees[i];
                 }
@@ -149,7 +149,7 @@ class Rcm : public ::testing::Test {
 
             std::vector<i_type> reference_contenders(0);
             auto current_height = std::numeric_limits<i_type>::min();
-            for (auto i = 0; i < n; ++i) {
+            for (gko::size_type i = 0; i < n; ++i) {
                 if (reference_current_levels[i] !=
                         std::numeric_limits<i_type>::max() &&
                     reference_current_levels[i] >= current_height) {
@@ -163,7 +163,7 @@ class Rcm : public ::testing::Test {
 
             std::vector<std::vector<i_type>> reference_contenders_levels(
                 reference_contenders.size());
-            for (auto i = 0; i < reference_contenders.size(); ++i) {
+            for (gko::size_type i = 0; i < reference_contenders.size(); ++i) {
                 std::vector<i_type> reference_contender_levels(n);
                 std::fill(reference_contender_levels.begin(),
                           reference_contender_levels.end(),
@@ -173,9 +173,9 @@ class Rcm : public ::testing::Test {
                 reference_contenders_levels[i] = reference_contender_levels;
             }
 
-            for (auto i = 0; i < reference_contenders.size(); ++i) {
+            for (gko::size_type i = 0; i < reference_contenders.size(); ++i) {
                 auto contender_height = std::numeric_limits<i_type>::min();
-                for (auto j = 0; j < n; ++j) {
+                for (gko::size_type j = 0; j < n; ++j) {
                     if (reference_contenders_levels[i][j] !=
                             std::numeric_limits<i_type>::max() &&
                         reference_contenders_levels[i][j] > contender_height) {
@@ -187,7 +187,6 @@ class Rcm : public ::testing::Test {
                 }
             }
             return false;
-            break;
         }
         }
         return true;
@@ -200,7 +199,7 @@ class Rcm : public ::testing::Test {
         const auto row_ptrs = mtx->get_const_row_ptrs();
         const auto col_idxs = mtx->get_const_col_idxs();
         auto degrees = std::vector<i_type>(n);
-        for (auto i = 0; i < n; ++i) {
+        for (gko::size_type i = 0; i < n; ++i) {
             degrees[i] =
                 mtx->get_const_row_ptrs()[i + 1] - mtx->get_const_row_ptrs()[i];
         }
@@ -209,7 +208,7 @@ class Rcm : public ::testing::Test {
         auto perm = std::vector<i_type>(n);
         std::copy_n(reorder->get_permutation()->get_const_permutation(), n,
                     perm.begin());
-        for (auto i = 0; i < n / 2; ++i) {
+        for (gko::size_type i = 0; i < n / 2; ++i) {
             const auto tmp = perm[i];
             perm[i] = perm[n - i - 1];
             perm[n - i - 1] = tmp;
@@ -217,7 +216,7 @@ class Rcm : public ::testing::Test {
 
         // Now check for cm ordering.
 
-        auto base_offset = 0;
+        gko::size_type base_offset = 0;
         std::vector<bool> already_visited(n);
         while (base_offset != n) {
             // Assert valid start node.
@@ -233,9 +232,9 @@ class Rcm : public ::testing::Test {
                       std::numeric_limits<i_type>::max());
             ubfs_reference(mtx, &levels[0], perm[base_offset]);
 
-            auto current_level = 0;
+            i_type current_level = 0;
             const auto previous_base_offset = base_offset;
-            for (auto i = 0; i < n; ++i) {
+            for (gko::size_type i = 0; i < n; ++i) {
                 const auto node = perm[i];
                 if (levels[node] != std::numeric_limits<i_type>::max() &&
                     !already_visited[node]) {
@@ -327,8 +326,7 @@ TEST_F(Rcm, OmpPermutationIsRcmOrdered)
     auto perm = d_reorder_op->get_permutation();
 
     // Can't std::move parameter when using ASSERT_PREDN, no perfect forwarding.
-    const auto d_reorder_op_shared =
-        std::shared_ptr<reorder_type>(d_reorder_op.release());
+    auto d_reorder_op_shared = gko::share(d_reorder_op);
     ASSERT_PRED2(is_rcm_ordered, d_1138_bus_mtx, d_reorder_op_shared);
 }
 
diff --git a/reference/reorder/rcm_kernels.cpp b/reference/reorder/rcm_kernels.cpp
index 2cb0727c131..452b8c877ac 100644
--- a/reference/reorder/rcm_kernels.cpp
+++ b/reference/reorder/rcm_kernels.cpp
@@ -220,7 +220,7 @@ IndexType find_starting_node(std::shared_ptr<const ReferenceExecutor> exec,
 }
 
 /**
- * Comutes a rcm reording using a naive sequential algorithm.
+ * Computes a RCM reordering using a naive sequential algorithm.
  */
 template <typename IndexType>
 void get_permutation(std::shared_ptr<const ReferenceExecutor> exec,
diff --git a/reference/test/reorder/rcm.cpp b/reference/test/reorder/rcm.cpp
index 53075365f5b..b8c8ea332ec 100644
--- a/reference/test/reorder/rcm.cpp
+++ b/reference/test/reorder/rcm.cpp
@@ -50,8 +50,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/test/utils.hpp"
 #include "core/test/utils/assertions.hpp"
 
+
 namespace {
 
+
 template <typename ValueIndexType>
 class Rcm : public ::testing::Test {
 protected:
@@ -87,17 +89,19 @@ class Rcm : public ::testing::Test {
 
 TYPED_TEST_CASE(Rcm, gko::test::ValueIndexTypes);
 
+
 TYPED_TEST(Rcm, CanBeCleared)
 {
     this->reorder_op->clear();
 
     auto reorder_op_perm = this->reorder_op->get_permutation();
+
     ASSERT_EQ(reorder_op_perm, nullptr);
 }
 
+
 TYPED_TEST(Rcm, CanBeCopied)
 {
-    using v_type = typename TestFixture::v_type;
     auto rcm = this->rcm_factory->generate(this->id3_mtx);
     auto rcm_copy = this->rcm_factory->generate(this->not_id3_mtx);
 
@@ -111,7 +115,6 @@ TYPED_TEST(Rcm, CanBeCopied)
 
 TYPED_TEST(Rcm, CanBeMoved)
 {
-    using v_type = typename TestFixture::v_type;
     auto rcm = this->rcm_factory->generate(this->id3_mtx);
     auto rcm_move = this->rcm_factory->generate(this->not_id3_mtx);
 
@@ -125,7 +128,6 @@ TYPED_TEST(Rcm, CanBeMoved)
 
 TYPED_TEST(Rcm, CanBeCloned)
 {
-    using v_type = typename TestFixture::v_type;
     auto rcm = this->rcm_factory->generate(this->id3_mtx);
 
     auto rcm_clone = rcm->clone();
@@ -135,9 +137,9 @@ TYPED_TEST(Rcm, CanBeCloned)
     ASSERT_EQ(rcm_clone->get_permutation()->get_const_permutation()[2], 0);
 }
 
+
 TYPED_TEST(Rcm, HasSensibleDefaults)
 {
-    using v_type = typename TestFixture::v_type;
     using reorder_type = typename TestFixture::reorder_type;
 
     auto rcm = reorder_type::build().on(this->exec)->generate(this->id3_mtx);
@@ -147,6 +149,7 @@ TYPED_TEST(Rcm, HasSensibleDefaults)
               gko::reorder::starting_strategy::pseudo_peripheral);
 }
 
+
 TYPED_TEST(Rcm, CanBeCreatedWithStartingStrategy)
 {
     using v_type = typename TestFixture::v_type;
@@ -163,9 +166,9 @@ TYPED_TEST(Rcm, CanBeCreatedWithStartingStrategy)
               gko::reorder::starting_strategy::minimum_degree);
 }
 
+
 TYPED_TEST(Rcm, CanBeCreatedWithConstructInversePermutation)
 {
-    using v_type = typename TestFixture::v_type;
     using reorder_type = typename TestFixture::reorder_type;
     auto rcm = reorder_type::build()
                    .with_construct_inverse_permutation(true)
@@ -178,4 +181,5 @@ TYPED_TEST(Rcm, CanBeCreatedWithConstructInversePermutation)
     ASSERT_EQ(rcm->get_parameters().construct_inverse_permutation, true);
 }
 
+
 }  // namespace
diff --git a/reference/test/reorder/rcm_kernels.cpp b/reference/test/reorder/rcm_kernels.cpp
index 393627a53bf..a12c94ec7ae 100644
--- a/reference/test/reorder/rcm_kernels.cpp
+++ b/reference/test/reorder/rcm_kernels.cpp
@@ -48,7 +48,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/test/utils/assertions.hpp"
-#include "matrices/config.hpp"
 
 
 namespace {
@@ -110,12 +109,15 @@ class Rcm : public ::testing::Test {
     }
 };
 
+
 TEST_F(Rcm, CreatesAPermutation)
 {
     auto p = reorder_op_0->get_permutation();
+    
     ASSERT_PRED1(is_permutation, p.get());
 }
 
+
 TEST_F(Rcm, CreatesCorrectPermutation)
 {
     std::vector<i_type> correct = {2, 3, 1, 0, 4};
@@ -125,6 +127,7 @@ TEST_F(Rcm, CreatesCorrectPermutation)
     ASSERT_TRUE(std::equal(p, p + correct.size(), correct.begin()));
 }
 
+
 TEST_F(Rcm, PermutesPerfectFullBand)
 {
     std::vector<i_type> correct = {7, 8, 0, 4, 3, 5, 6, 1, 2};
@@ -134,4 +137,5 @@ TEST_F(Rcm, PermutesPerfectFullBand)
     ASSERT_TRUE(std::equal(p, p + correct.size(), correct.begin()));
 }
 
+
 }  // namespace

From 29bcb96938d10ac1a0a273d4abddc914077f6442 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Wed, 14 Oct 2020 20:59:40 +0200
Subject: [PATCH 0067/1008] Execute rcm on cpu exector forcibly

---
 core/reorder/rcm.cpp                |  2 +-
 cuda/test/CMakeLists.txt            |  1 +
 cuda/test/reorder/CMakeLists.txt    |  1 +
 cuda/test/reorder/rcm_kernels.cpp   | 85 +++++++++++++++++++++++++++++
 include/ginkgo/core/reorder/rcm.hpp | 35 +++++++++---
 5 files changed, 115 insertions(+), 9 deletions(-)
 create mode 100644 cuda/test/reorder/CMakeLists.txt
 create mode 100644 cuda/test/reorder/rcm_kernels.cpp

diff --git a/core/reorder/rcm.cpp b/core/reorder/rcm.cpp
index fcac288039f..885c2982878 100644
--- a/core/reorder/rcm.cpp
+++ b/core/reorder/rcm.cpp
@@ -65,11 +65,11 @@ GKO_REGISTER_OPERATION(get_degree_of_nodes, rcm::get_degree_of_nodes);
 
 template <typename ValueType, typename IndexType>
 void Rcm<ValueType, IndexType>::generate(
+    std::shared_ptr<const Executor> &exec,
     std::unique_ptr<SparsityMatrix> adjacency_matrix) const
 {
     const IndexType num_rows = adjacency_matrix->get_size()[0];
     const auto mtx = adjacency_matrix.get();
-    const auto exec = this->get_executor();
     auto degrees = Array<IndexType>(exec, num_rows);
     // RCM is only valid for symmetric matrices. Need to add an expensive check
     // for symmetricity here ?
diff --git a/cuda/test/CMakeLists.txt b/cuda/test/CMakeLists.txt
index af845cb4c2a..83d7b39f35e 100644
--- a/cuda/test/CMakeLists.txt
+++ b/cuda/test/CMakeLists.txt
@@ -5,6 +5,7 @@ add_subdirectory(components)
 add_subdirectory(factorization)
 add_subdirectory(matrix)
 add_subdirectory(preconditioner)
+add_subdirectory(reorder)
 add_subdirectory(solver)
 add_subdirectory(stop)
 add_subdirectory(utils)
diff --git a/cuda/test/reorder/CMakeLists.txt b/cuda/test/reorder/CMakeLists.txt
new file mode 100644
index 00000000000..108e3b57dd5
--- /dev/null
+++ b/cuda/test/reorder/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_test(rcm_kernels)
\ No newline at end of file
diff --git a/cuda/test/reorder/rcm_kernels.cpp b/cuda/test/reorder/rcm_kernels.cpp
new file mode 100644
index 00000000000..d4822b36859
--- /dev/null
+++ b/cuda/test/reorder/rcm_kernels.cpp
@@ -0,0 +1,85 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+
+#include <ginkgo/core/reorder/rcm.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+#include "core/test/utils/assertions.hpp"
+
+
+namespace {
+
+
+class Rcm : public ::testing::Test {
+protected:
+    using v_type = double;
+    using i_type = int;
+    using CsrMtx = gko::matrix::Csr<v_type, i_type>;
+    using reorder_type = gko::reorder::Rcm<v_type, i_type>;
+    using perm_type = gko::matrix::Permutation<i_type>;
+
+
+    Rcm()
+        : exec(gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true)),
+          // clang-format off
+          p_mtx(gko::initialize<CsrMtx>({{1.0, 2.0, 0.0, -1.3, 2.1},
+                                         {2.0, 5.0, 1.5, 0.0, 0.0},
+                                         {0.0, 1.5, 1.5, 1.1, 0.0},
+                                         {-1.3, 0.0, 1.1, 2.0, 0.0},
+                                         {2.1, 0.0, 0.0, 0.0, 1.0}},
+                                        exec)),
+          // clang-format on
+          rcm_factory(reorder_type::build().on(exec)),
+          reorder_op(rcm_factory->generate(p_mtx))
+    {}
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::unique_ptr<reorder_type::Factory> rcm_factory;
+    std::shared_ptr<CsrMtx> p_mtx;
+    std::unique_ptr<reorder_type> reorder_op;
+};
+
+
+TEST_F(Rcm, IsExecutedOnCpuExecutor)
+{
+    // This only executes successfully if computed on cpu executor.
+    auto p = reorder_op->get_permutation();
+
+    ASSERT_TRUE(true);
+}
+
+
+}  // namespace
diff --git a/include/ginkgo/core/reorder/rcm.hpp b/include/ginkgo/core/reorder/rcm.hpp
index a22abaf8bcb..3a36eccf465 100644
--- a/include/ginkgo/core/reorder/rcm.hpp
+++ b/include/ginkgo/core/reorder/rcm.hpp
@@ -148,7 +148,8 @@ class Rcm
      * Generates the permutation matrix and if required the inverse permutation
      * matrix.
      */
-    void generate(std::unique_ptr<SparsityMatrix> adjacency_matrix) const;
+    void generate(std::shared_ptr<const Executor> &exec,
+                  std::unique_ptr<SparsityMatrix> adjacency_matrix) const;
 
     explicit Rcm(std::shared_ptr<const Executor> exec)
         : EnablePolymorphicObject<Rcm, ReorderingBase>(std::move(exec))
@@ -158,8 +159,13 @@ class Rcm
         : EnablePolymorphicObject<Rcm, ReorderingBase>(factory->get_executor()),
           parameters_{factory->get_parameters()}
     {
-        const auto exec = this->get_executor();
-        auto adjacency_matrix = SparsityMatrix::create(exec);
+        // Always execute the reordering on the cpu.
+        const auto is_gpu_executor =
+            this->get_executor() != this->get_executor()->get_master();
+        auto cpu_exec = is_gpu_executor ? this->get_executor()->get_master()
+                                        : this->get_executor();
+
+        auto adjacency_matrix = SparsityMatrix::create(cpu_exec);
         Array<IndexType> degrees;
 
         // The adjacency matrix has to be square.
@@ -167,23 +173,36 @@ class Rcm
         // This is needed because it does not make sense to call the copy and
         // convert if the existing matrix is empty.
         if (args.system_matrix->get_size()) {
-            auto tmp =
-                copy_and_convert_to<SparsityMatrix>(exec, args.system_matrix);
+            auto tmp = copy_and_convert_to<SparsityMatrix>(cpu_exec,
+                                                           args.system_matrix);
             // This function provided within the Sparsity matrix format removes
             // the diagonal elements and outputs an adjacency matrix.
             adjacency_matrix = tmp->to_adjacency_matrix();
         }
 
         auto const dim = adjacency_matrix->get_size();
-        permutation_ = PermutationMatrix::create(exec, dim);
+        permutation_ = PermutationMatrix::create(cpu_exec, dim);
 
         // To make it explicit.
         inv_permutation_ = nullptr;
         if (parameters_.construct_inverse_permutation) {
-            inv_permutation_ = PermutationMatrix::create(exec, dim);
+            inv_permutation_ = PermutationMatrix::create(cpu_exec, dim);
         }
 
-        this->generate(std::move(adjacency_matrix));
+        this->generate(cpu_exec, std::move(adjacency_matrix));
+
+        // Copy back results to gpu if necessary.
+        if (is_gpu_executor) {
+            const auto gpu_exec = this->get_executor();
+            auto gpu_perm = PermutationMatrix::create(gpu_exec, dim);
+            gpu_perm->copy_from(permutation_.get());
+            permutation_ = gko::share(gpu_perm);
+            if (inv_permutation_) {
+                auto gpu_inv_perm = PermutationMatrix::create(gpu_exec, dim);
+                gpu_inv_perm->copy_from(inv_permutation_.get());
+                inv_permutation_ = gko::share(gpu_inv_perm);
+            }
+        }
     }
 
 private:

From 7fbdefffc4d28baed7b0edc9c5d4ac2e773ecb05 Mon Sep 17 00:00:00 2001
From: Lukas Riemer <lksriemer@gmail.com>
Date: Thu, 15 Oct 2020 18:01:30 +0200
Subject: [PATCH 0068/1008] Add to contributors.txt

---
 contributors.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contributors.txt b/contributors.txt
index 5631bd6cc43..ce7f8045f4f 100644
--- a/contributors.txt
+++ b/contributors.txt
@@ -17,4 +17,5 @@ Maier Matthias <matthias@43-1.org> Texas A&M University
 Nayak Pratik <pratik.nayak@kit.edu> Karlsruhe Institute of Technology
 Olenik Gregor <go@hpsim.de> HPSim
 Ribizel Tobias <mail@upsj.de> Karlsruhe Institute of Technology
+Riemer Lukas <lksriemer@gmail.com> Karlsruhe Institute of Technology
 Tsai Yuhsiang <yhmtsai@gmail.com> National Taiwan University

From c0ce1737187441120f615c5d1d269549a3c96bf1 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 16 Oct 2020 10:56:44 +0200
Subject: [PATCH 0069/1008] fix omp local header and reorder_factory

---
 include/ginkgo/core/reorder/reordering_base.hpp | 14 +++++++++++---
 omp/reorder/rcm_kernels.cpp                     |  6 +++---
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/include/ginkgo/core/reorder/reordering_base.hpp b/include/ginkgo/core/reorder/reordering_base.hpp
index b97e9e6c341..e4ee5c9e8f1 100644
--- a/include/ginkgo/core/reorder/reordering_base.hpp
+++ b/include/ginkgo/core/reorder/reordering_base.hpp
@@ -144,9 +144,17 @@ public:                                                                        \
             _factory_name, ::gko::reorder::ReorderingBaseFactory>;             \
         friend class ::gko::enable_parameters_type<_parameters_name##_type,    \
                                                    _factory_name>;             \
-        using ::gko::reorder::EnableDefaultReorderingBaseFactory<              \
-            _factory_name, _reordering_base,                                   \
-            _parameters_name##_type>::EnableDefaultReorderingBaseFactory;      \
+        explicit _factory_name(std::shared_ptr<const ::gko::Executor> exec)    \
+            : ::gko::reorder::EnableDefaultReorderingBaseFactory<              \
+                  _factory_name, _reordering_base, _parameters_name##_type>(   \
+                  std::move(exec))                                             \
+        {}                                                                     \
+        explicit _factory_name(std::shared_ptr<const ::gko::Executor> exec,    \
+                               const _parameters_name##_type &parameters)      \
+            : ::gko::reorder::EnableDefaultReorderingBaseFactory<              \
+                  _factory_name, _reordering_base, _parameters_name##_type>(   \
+                  std::move(exec), parameters)                                 \
+        {}                                                                     \
     };                                                                         \
     friend ::gko::reorder::EnableDefaultReorderingBaseFactory<                 \
         _factory_name, _reordering_base, _parameters_name##_type>;             \
diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index a5966fddef2..a51c57cef41 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -54,10 +54,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
-#include "components/omp_mutex.hpp"
-#include "components/sort_small.hpp"
 #include "core/base/allocator.hpp"
 #include "core/components/prefix_sum.hpp"
+#include "omp/components/omp_mutex.hpp"
+#include "omp/components/sort_small.hpp"
 
 
 namespace gko {
@@ -872,4 +872,4 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
 }  // namespace rcm
 }  // namespace omp
 }  // namespace kernels
-}  // namespace gko
\ No newline at end of file
+}  // namespace gko

From 8fdc56916bf0916a139fdc5e23f94c46d55e517e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 16 Oct 2020 14:41:34 +0200
Subject: [PATCH 0070/1008] Make gko::dim conversion to bool explicit

Currently, gko::dim is implicitly convertible to bool.
C++ allows for so-called contextual conversions to also
use explicit conversion operators.
<expression> is contextually converted to bool in the following cases:

1. if(<expression>), while(<expression>), for(...; <expression>; ...)
2. !<expression>, ... && <expression>, || <expression>
3. <expression> ? ... : ...
3. static_assert/noexcept(<expression>)
---
 include/ginkgo/core/base/dim.hpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/include/ginkgo/core/base/dim.hpp b/include/ginkgo/core/base/dim.hpp
index c0256df30dc..e0aaf25666c 100644
--- a/include/ginkgo/core/base/dim.hpp
+++ b/include/ginkgo/core/base/dim.hpp
@@ -118,8 +118,11 @@ struct dim {
      * different than zero.
      *
      * @return true if and only if all dimensions evaluate to true
+     *
+     * @note This operator is explicit to avoid implicit dim-to-int casts.
+     *       It will still be used in contextual conversions (if, &&, ||, !)
      */
-    constexpr GKO_ATTRIBUTES operator bool() const
+    explicit constexpr GKO_ATTRIBUTES operator bool() const
     {
         return static_cast<bool>(first_) && static_cast<bool>(rest_);
     }
@@ -183,7 +186,7 @@ struct dim<1u, DimensionType> {
         return GKO_ASSERT(dimension == 0), first_;
     }
 
-    constexpr GKO_ATTRIBUTES operator bool() const
+    explicit constexpr GKO_ATTRIBUTES operator bool() const
     {
         return static_cast<bool>(first_);
     }

From 82b0e3caf2a37c8319d1faa054377f9018ccdfa3 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Fri, 16 Oct 2020 22:59:50 +0200
Subject: [PATCH 0071/1008] Change the transpose to conj_transpose in the BiCG
 solver.

+ Add some doc clarification and a reference.
---
 core/solver/bicg.cpp                | 25 +++++++++++++------------
 include/ginkgo/core/solver/bicg.hpp | 10 ++++++++--
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp
index a7519f48a33..02abd90bf4e 100644
--- a/core/solver/bicg.cpp
+++ b/core/solver/bicg.cpp
@@ -95,7 +95,7 @@ std::unique_ptr<LinOp> Bicg<ValueType>::conj_transpose() const
  *                  before transposing it
  */
 template <typename CsrType>
-std::unique_ptr<LinOp> transpose_with_csr(const LinOp *mtx)
+std::unique_ptr<LinOp> conj_transpose_with_csr(const LinOp *mtx)
 {
     auto csr_matrix_unique_ptr = copy_and_convert_to<CsrType>(
         mtx->get_executor(), const_cast<LinOp *>(mtx));
@@ -103,7 +103,7 @@ std::unique_ptr<LinOp> transpose_with_csr(const LinOp *mtx)
     csr_matrix_unique_ptr->set_strategy(
         std::make_shared<typename CsrType::classical>());
 
-    return csr_matrix_unique_ptr->transpose();
+    return csr_matrix_unique_ptr->conj_transpose();
 }
 
 
@@ -149,12 +149,12 @@ void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     // r = r2 = dense_b
     // z2 = p2 = q2 = 0
 
-    std::unique_ptr<LinOp> trans_A;
-    auto transposable_system_matrix =
+    std::unique_ptr<LinOp> conj_trans_A;
+    auto conj_transposable_system_matrix =
         dynamic_cast<const Transposable *>(system_matrix_.get());
 
-    if (transposable_system_matrix) {
-        trans_A = transposable_system_matrix->transpose();
+    if (conj_transposable_system_matrix) {
+        conj_trans_A = conj_transposable_system_matrix->conj_transpose();
     } else {
         // TODO Extend when adding more IndexTypes
         // Try to figure out the IndexType that can be used for the CSR matrix
@@ -163,15 +163,16 @@ void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         auto supports_int64 =
             dynamic_cast<const ConvertibleTo<Csr64> *>(system_matrix_.get());
         if (supports_int64) {
-            trans_A = transpose_with_csr<Csr64>(system_matrix_.get());
+            conj_trans_A = conj_transpose_with_csr<Csr64>(system_matrix_.get());
         } else {
-            trans_A = transpose_with_csr<Csr32>(system_matrix_.get());
+            conj_trans_A = conj_transpose_with_csr<Csr32>(system_matrix_.get());
         }
     }
 
-    auto trans_preconditioner_tmp =
+    auto conj_trans_preconditioner_tmp =
         as<const Transposable>(get_preconditioner().get());
-    auto trans_preconditioner = trans_preconditioner_tmp->transpose();
+    auto conj_trans_preconditioner =
+        conj_trans_preconditioner_tmp->conj_transpose();
 
     system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(), r.get());
     // r = r - Ax =  -1.0 * A*dense_x + 1.0*r
@@ -185,7 +186,7 @@ void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     while (true) {
         get_preconditioner()->apply(r.get(), z.get());
-        trans_preconditioner->apply(r2.get(), z2.get());
+        conj_trans_preconditioner->apply(r2.get(), z2.get());
         z->compute_dot(r2.get(), rho.get());
 
         ++iter;
@@ -205,7 +206,7 @@ void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         // p = z + tmp * p
         // p2 = z2 + tmp * p2
         system_matrix_->apply(p.get(), q.get());
-        trans_A->apply(p2.get(), q2.get());
+        conj_trans_A->apply(p2.get(), q2.get());
         p2->compute_dot(q.get(), beta.get());
         exec->run(bicg::make_step_2(dense_x, r.get(), r2.get(), p.get(),
                                     q.get(), q2.get(), beta.get(), rho.get(),
diff --git a/include/ginkgo/core/solver/bicg.hpp b/include/ginkgo/core/solver/bicg.hpp
index e2960783a3e..349fd469b9e 100644
--- a/include/ginkgo/core/solver/bicg.hpp
+++ b/include/ginkgo/core/solver/bicg.hpp
@@ -59,8 +59,14 @@ namespace solver {
  * Being a generic solver, it is capable of solving general matrices, including
  * non-s.p.d matrices. Though, the memory and the computational requirement of
  * the BiCG solver are higher than of its s.p.d solver counterpart, it has
- * the capability to solve generic systems. BiCG is the unstable version of
- * BiCGSTAB.
+ * the capability to solve generic systems.
+ *
+ * BiCG is based on the bi-Lanczos tridiagonalization method and in exact
+ * arithmetic should terminate in atmost N iterations (2N MV's, with A and A^H).
+ * It forms the basis of many of the cheaper methods such as BiCGSTAB and CGS.
+ *
+ * Reference: R.Fletcher, Conjugate gradient methods for indefinite systems,
+ * doi: 10.1007/BFb0080116
  *
  * @tparam ValueType  precision of matrix elements
  *

From 422d1a925f314d739f00cb190445c62f4a0d7e36 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Fri, 16 Oct 2020 23:00:28 +0200
Subject: [PATCH 0072/1008] Fix conj_transpose for CSR kernel in HIP.

---
 hip/matrix/csr_kernels.hip.cpp      |  2 +-
 hip/test/matrix/csr_kernels.hip.cpp | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/hip/matrix/csr_kernels.hip.cpp b/hip/matrix/csr_kernels.hip.cpp
index 145a84271d5..f133507092e 100644
--- a/hip/matrix/csr_kernels.hip.cpp
+++ b/hip/matrix/csr_kernels.hip.cpp
@@ -953,7 +953,7 @@ void conj_transpose(std::shared_ptr<const HipExecutor> exec,
             orig->get_size()[1], orig->get_num_stored_elements(),
             orig->get_const_values(), orig->get_const_row_ptrs(),
             orig->get_const_col_idxs(), trans->get_values(),
-            trans->get_col_idxs(), trans->get_row_ptrs(), copyValues, idxBase);
+            trans->get_row_ptrs(), trans->get_col_idxs(), copyValues, idxBase);
 
         hipLaunchKernelGGL(conjugate_kernel, dim3(grid_size), dim3(block_size),
                            0, 0, trans->get_num_stored_elements(),
diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index 54b0f4cda65..e80eaa940dc 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -420,6 +420,19 @@ TEST_F(Csr, TransposeIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, ConjugateTransposeIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+
+    auto ctrans = mtx->conj_transpose();
+    auto d_ctrans = dmtx->conj_transpose();
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(d_ctrans.get()),
+                        static_cast<Mtx *>(ctrans.get()), 0.0);
+    ASSERT_TRUE(static_cast<Mtx *>(d_ctrans.get())->is_sorted_by_column_index());
+}
+
+
 TEST_F(Csr, ConvertToDenseIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());

From 4b45ea3c0312ee9563c4c7ef71f857db7f53bc2c Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Sat, 17 Oct 2020 13:20:48 +0200
Subject: [PATCH 0073/1008] Review update.

---
 core/solver/bicg.cpp                | 8 ++++----
 include/ginkgo/core/solver/bicg.hpp | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp
index 02abd90bf4e..37568284703 100644
--- a/core/solver/bicg.cpp
+++ b/core/solver/bicg.cpp
@@ -87,12 +87,12 @@ std::unique_ptr<LinOp> Bicg<ValueType>::conj_transpose() const
 
 /**
  * @internal
- * Transposes the matrix by converting it into a CSR matrix of type
- * CsrType, followed by transposing.
+ * (Conjugate-)Transposes the matrix by converting it into a CSR matrix of type
+ * CsrType, followed by (conjugate-)transposing.
  *
- * @param mtx  Matrix to transpose
+ * @param mtx  Matrix to (conjugate-)transpose
  * @tparam CsrType  Matrix format in which the matrix mtx is converted into
- *                  before transposing it
+ *                  before (conjugate-)transposing it
  */
 template <typename CsrType>
 std::unique_ptr<LinOp> conj_transpose_with_csr(const LinOp *mtx)
diff --git a/include/ginkgo/core/solver/bicg.hpp b/include/ginkgo/core/solver/bicg.hpp
index 349fd469b9e..19a92365050 100644
--- a/include/ginkgo/core/solver/bicg.hpp
+++ b/include/ginkgo/core/solver/bicg.hpp
@@ -62,7 +62,7 @@ namespace solver {
  * the capability to solve generic systems.
  *
  * BiCG is based on the bi-Lanczos tridiagonalization method and in exact
- * arithmetic should terminate in atmost N iterations (2N MV's, with A and A^H).
+ * arithmetic should terminate in at most N iterations (2N MV's, with A and A^H).
  * It forms the basis of many of the cheaper methods such as BiCGSTAB and CGS.
  *
  * Reference: R.Fletcher, Conjugate gradient methods for indefinite systems,

From f4738cdb3ead27145478cd02e4ef897eca8b2016 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sun, 18 Oct 2020 18:51:28 +0200
Subject: [PATCH 0074/1008] add DENSE_ prefix to some kernel DECLARE_ macros

---
 core/device_hooks/common_kernels.inc.cpp | 29 ++++++++++++------------
 core/matrix/dense_kernels.hpp            | 28 +++++++++++------------
 cuda/matrix/dense_kernels.cu             | 15 ++++++------
 hip/matrix/dense_kernels.hip.cpp         | 15 ++++++------
 omp/matrix/dense_kernels.cpp             | 15 ++++++------
 reference/matrix/dense_kernels.cpp       | 15 ++++++------
 6 files changed, 61 insertions(+), 56 deletions(-)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index a2285615224..09082c5d507 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -208,42 +208,43 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_DENSE_CALCULATE_TOTAL_COLS_KERNEL);
 
 template <typename ValueType>
-GKO_DECLARE_TRANSPOSE_KERNEL(ValueType)
+GKO_DECLARE_DENSE_TRANSPOSE_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_TRANSPOSE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_TRANSPOSE_KERNEL);
 
 template <typename ValueType>
-GKO_DECLARE_CONJ_TRANSPOSE_KERNEL(ValueType)
+GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CONJ_TRANSPOSE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 template <typename ValueType, typename IndexType>
-GKO_DECLARE_ROW_PERMUTE_KERNEL(ValueType, IndexType)
+GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ROW_PERMUTE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL);
 
 template <typename ValueType, typename IndexType>
-GKO_DECLARE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)
+GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL);
 
 template <typename ValueType, typename IndexType>
-GKO_DECLARE_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType)
+GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_INVERSE_ROW_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL);
 
 template <typename ValueType, typename IndexType>
-GKO_DECLARE_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)
+GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_INVERSE_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL);
 
 template <typename ValueType>
-GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL(ValueType)
+GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL);
 
 template <typename ValueType>
 GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL(ValueType)
diff --git a/core/matrix/dense_kernels.hpp b/core/matrix/dense_kernels.hpp
index 30ad6ca4729..bf86205fbef 100644
--- a/core/matrix/dense_kernels.hpp
+++ b/core/matrix/dense_kernels.hpp
@@ -133,41 +133,41 @@ namespace kernels {
                               size_type *result, size_type stride_factor,  \
                               size_type slice_size)
 
-#define GKO_DECLARE_TRANSPOSE_KERNEL(_type)                     \
+#define GKO_DECLARE_DENSE_TRANSPOSE_KERNEL(_type)               \
     void transpose(std::shared_ptr<const DefaultExecutor> exec, \
                    const matrix::Dense<_type> *orig,            \
                    matrix::Dense<_type> *trans)
 
-#define GKO_DECLARE_CONJ_TRANSPOSE_KERNEL(_type)                     \
+#define GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL(_type)               \
     void conj_transpose(std::shared_ptr<const DefaultExecutor> exec, \
                         const matrix::Dense<_type> *orig,            \
                         matrix::Dense<_type> *trans)
 
-#define GKO_DECLARE_ROW_PERMUTE_KERNEL(_vtype, _itype)            \
+#define GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL(_vtype, _itype)      \
     void row_permute(std::shared_ptr<const DefaultExecutor> exec, \
                      const Array<_itype> *permutation_indices,    \
                      const matrix::Dense<_vtype> *orig,           \
                      matrix::Dense<_vtype> *row_permuted)
 
-#define GKO_DECLARE_COLUMN_PERMUTE_KERNEL(_vtype, _itype)            \
+#define GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL(_vtype, _itype)      \
     void column_permute(std::shared_ptr<const DefaultExecutor> exec, \
                         const Array<_itype> *permutation_indices,    \
                         const matrix::Dense<_vtype> *orig,           \
                         matrix::Dense<_vtype> *column_permuted)
 
-#define GKO_DECLARE_INVERSE_ROW_PERMUTE_KERNEL(_vtype, _itype)            \
+#define GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL(_vtype, _itype)          \
     void inverse_row_permute(std::shared_ptr<const DefaultExecutor> exec, \
                              const Array<_itype> *permutation_indices,    \
                              const matrix::Dense<_vtype> *orig,           \
                              matrix::Dense<_vtype> *row_permuted)
 
-#define GKO_DECLARE_INVERSE_COLUMN_PERMUTE_KERNEL(_vtype, _itype)            \
+#define GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL(_vtype, _itype)          \
     void inverse_column_permute(std::shared_ptr<const DefaultExecutor> exec, \
                                 const Array<_itype> *permutation_indices,    \
                                 const matrix::Dense<_vtype> *orig,           \
                                 matrix::Dense<_vtype> *column_permuted)
 
-#define GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL(_vtype)                    \
+#define GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL(_vtype)              \
     void extract_diagonal(std::shared_ptr<const DefaultExecutor> exec, \
                           const matrix::Dense<_vtype> *orig,           \
                           matrix::Diagonal<_vtype> *diag)
@@ -219,19 +219,19 @@ namespace kernels {
     template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_CALCULATE_TOTAL_COLS_KERNEL(ValueType);               \
     template <typename ValueType>                                           \
-    GKO_DECLARE_TRANSPOSE_KERNEL(ValueType);                                \
+    GKO_DECLARE_DENSE_TRANSPOSE_KERNEL(ValueType);                          \
     template <typename ValueType>                                           \
-    GKO_DECLARE_CONJ_TRANSPOSE_KERNEL(ValueType);                           \
+    GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL(ValueType);                     \
     template <typename ValueType, typename IndexType>                       \
-    GKO_DECLARE_ROW_PERMUTE_KERNEL(ValueType, IndexType);                   \
+    GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL(ValueType, IndexType);             \
     template <typename ValueType, typename IndexType>                       \
-    GKO_DECLARE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType);                \
+    GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType);          \
     template <typename ValueType, typename IndexType>                       \
-    GKO_DECLARE_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType);           \
+    GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL(ValueType, IndexType);         \
     template <typename ValueType, typename IndexType>                       \
-    GKO_DECLARE_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType);        \
+    GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL(ValueType, IndexType);      \
     template <typename ValueType>                                           \
-    GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL(ValueType);                         \
+    GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL(ValueType);                   \
     template <typename ValueType>                                           \
     GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL(ValueType);                   \
     template <typename ValueType>                                           \
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index b1c4ae747f8..72d557e7dd1 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -574,7 +574,7 @@ void transpose(std::shared_ptr<const CudaExecutor> exec,
     }
 };
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_TRANSPOSE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType>
@@ -599,7 +599,7 @@ void conj_transpose(std::shared_ptr<const CudaExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CONJ_TRANSPOSE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -619,7 +619,8 @@ void row_permute(std::shared_ptr<const CudaExecutor> exec,
         as_cuda_type(row_permuted->get_values()), row_permuted->get_stride());
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ROW_PERMUTE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -641,7 +642,7 @@ void column_permute(std::shared_ptr<const CudaExecutor> exec,
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -662,7 +663,7 @@ void inverse_row_permute(std::shared_ptr<const CudaExecutor> exec,
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_INVERSE_ROW_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -684,7 +685,7 @@ void inverse_column_permute(std::shared_ptr<const CudaExecutor> exec,
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_INVERSE_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL);
 
 
 template <typename ValueType>
@@ -698,7 +699,7 @@ void extract_diagonal(std::shared_ptr<const CudaExecutor> exec,
         orig->get_stride(), as_cuda_type(diag->get_values()));
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL);
 
 
 template <typename ValueType>
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index 2793d09832f..fa732e1e86a 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -597,7 +597,7 @@ void transpose(std::shared_ptr<const HipExecutor> exec,
     }
 };
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_TRANSPOSE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType>
@@ -622,7 +622,7 @@ void conj_transpose(std::shared_ptr<const HipExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CONJ_TRANSPOSE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -643,7 +643,8 @@ void row_permute(std::shared_ptr<const HipExecutor> exec,
         as_hip_type(row_permuted->get_values()), row_permuted->get_stride());
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ROW_PERMUTE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -666,7 +667,7 @@ void column_permute(std::shared_ptr<const HipExecutor> exec,
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -688,7 +689,7 @@ void inverse_row_permute(std::shared_ptr<const HipExecutor> exec,
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_INVERSE_ROW_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -711,7 +712,7 @@ void inverse_column_permute(std::shared_ptr<const HipExecutor> exec,
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_INVERSE_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL);
 
 
 template <typename ValueType>
@@ -726,7 +727,7 @@ void extract_diagonal(std::shared_ptr<const HipExecutor> exec,
                        orig->get_stride(), as_hip_type(diag->get_values()));
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL);
 
 
 template <typename ValueType>
diff --git a/omp/matrix/dense_kernels.cpp b/omp/matrix/dense_kernels.cpp
index c3a5234d956..afe74e3c5e4 100644
--- a/omp/matrix/dense_kernels.cpp
+++ b/omp/matrix/dense_kernels.cpp
@@ -660,7 +660,7 @@ void transpose(std::shared_ptr<const OmpExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_TRANSPOSE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType>
@@ -676,7 +676,7 @@ void conj_transpose(std::shared_ptr<const OmpExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CONJ_TRANSPOSE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -694,7 +694,8 @@ void row_permute(std::shared_ptr<const OmpExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ROW_PERMUTE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -713,7 +714,7 @@ void column_permute(std::shared_ptr<const OmpExecutor> exec,
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -732,7 +733,7 @@ void inverse_row_permute(std::shared_ptr<const OmpExecutor> exec,
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_INVERSE_ROW_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -751,7 +752,7 @@ void inverse_column_permute(std::shared_ptr<const OmpExecutor> exec,
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_INVERSE_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL);
 
 
 template <typename ValueType>
@@ -766,7 +767,7 @@ void extract_diagonal(std::shared_ptr<const OmpExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL);
 
 
 template <typename ValueType>
diff --git a/reference/matrix/dense_kernels.cpp b/reference/matrix/dense_kernels.cpp
index 75a6ddef342..f0c8f938d05 100644
--- a/reference/matrix/dense_kernels.cpp
+++ b/reference/matrix/dense_kernels.cpp
@@ -581,7 +581,7 @@ void transpose(std::shared_ptr<const ReferenceExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_TRANSPOSE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType>
@@ -596,7 +596,7 @@ void conj_transpose(std::shared_ptr<const ReferenceExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CONJ_TRANSPOSE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -613,7 +613,8 @@ void row_permute(std::shared_ptr<const ReferenceExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ROW_PERMUTE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -631,7 +632,7 @@ void column_permute(std::shared_ptr<const ReferenceExecutor> exec,
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -649,7 +650,7 @@ void inverse_row_permute(std::shared_ptr<const ReferenceExecutor> exec,
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_INVERSE_ROW_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -667,7 +668,7 @@ void inverse_column_permute(std::shared_ptr<const ReferenceExecutor> exec,
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_INVERSE_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL);
 
 
 template <typename ValueType>
@@ -681,7 +682,7 @@ void extract_diagonal(std::shared_ptr<const ReferenceExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL);
 
 
 template <typename ValueType>

From bef228f7be6a304b7bb22396705fe5ceb3561f6f Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Mon, 21 Sep 2020 16:02:03 +0200
Subject: [PATCH 0075/1008] Add matrix_assembly_data with option of adding to
 matrix entries

---
 core/test/base/CMakeLists.txt                 |   1 +
 core/test/base/matrix_assembly_data.cpp       | 130 ++++++++++
 .../ginkgo/core/base/matrix_assembly_data.hpp | 224 ++++++++++++++++++
 include/ginkgo/ginkgo.hpp                     |   1 +
 4 files changed, 356 insertions(+)
 create mode 100644 core/test/base/matrix_assembly_data.cpp
 create mode 100644 include/ginkgo/core/base/matrix_assembly_data.hpp

diff --git a/core/test/base/CMakeLists.txt b/core/test/base/CMakeLists.txt
index 1183b339e07..6f8c7291165 100644
--- a/core/test/base/CMakeLists.txt
+++ b/core/test/base/CMakeLists.txt
@@ -11,6 +11,7 @@ ginkgo_create_test(executor)
 ginkgo_create_test(iterator_factory)
 ginkgo_create_test(lin_op)
 ginkgo_create_test(math)
+ginkgo_create_test(matrix_assembly_data)
 ginkgo_create_test(matrix_data)
 ginkgo_create_test(mtx_io)
 ginkgo_create_test(perturbation)
diff --git a/core/test/base/matrix_assembly_data.cpp b/core/test/base/matrix_assembly_data.cpp
new file mode 100644
index 00000000000..05157ce2efe
--- /dev/null
+++ b/core/test/base/matrix_assembly_data.cpp
@@ -0,0 +1,130 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/base/matrix_assembly_data.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+namespace {
+
+
+TEST(MatrixAssemblyData, InitializesWithZeros)
+{
+    gko::matrix_assembly_data<double, int> m(gko::dim<2>{3, 5});
+
+    ASSERT_EQ(m.get_size(), gko::dim<2>(3, 5));
+    ASSERT_EQ(m.get_num_stored_elements(), 0);
+    ASSERT_EQ(m.get_value(0, 0), 0.0);
+    ASSERT_FALSE(m.contains(0, 0));
+}
+
+
+TEST(MatrixAssemblyData, InsertsValuesWithoutAdding)
+{
+    gko::matrix_assembly_data<double, int> m(gko::dim<2>{3, 5});
+
+    m.add_value(0, 0, 1.3);
+    m.add_value(2, 3, 2.2);
+    m.add_value(1, 4, 1.1);
+    m.add_value(1, 2, 3.6);
+
+    ASSERT_EQ(m.get_size(), gko::dim<2>(3, 5));
+    ASSERT_EQ(m.get_num_stored_elements(), 4);
+    ASSERT_EQ(m.get_value(0, 0), 1.3);
+    ASSERT_EQ(m.get_value(2, 3), 2.2);
+    ASSERT_EQ(m.get_value(1, 4), 1.1);
+    ASSERT_EQ(m.get_value(1, 2), 3.6);
+    ASSERT_TRUE(m.contains(0, 0));
+}
+
+
+TEST(MatrixAssemblyData, InsertsValuesWithAdding)
+{
+    gko::matrix_assembly_data<double, int> m(gko::dim<2>{3, 5});
+
+    m.add_value(0, 0, 1.3);
+    m.add_value(2, 3, 2.2);
+    m.add_value(1, 4, 1.1);
+    m.add_value(1, 2, 3.6);
+    m.add_value(1, 4, 9.1);
+    m.add_value(2, 3, 1.3);
+
+    ASSERT_EQ(m.get_size(), gko::dim<2>(3, 5));
+    ASSERT_EQ(m.get_num_stored_elements(), 4);
+    ASSERT_EQ(m.get_value(0, 0), 1.3);
+    ASSERT_EQ(m.get_value(2, 3), 3.5);
+    ASSERT_EQ(m.get_value(1, 4), 10.2);
+    ASSERT_EQ(m.get_value(1, 2), 3.6);
+}
+
+
+TEST(MatrixAssemblyData, OverwritesValuesWhenNotAdding)
+{
+    gko::matrix_assembly_data<double, int> m(gko::dim<2>{3, 5});
+
+    m.set_value(0, 0, 1.3);
+    m.set_value(2, 3, 2.2);
+    m.set_value(1, 4, 1.1);
+    m.set_value(1, 2, 3.6);
+    m.set_value(1, 4, 9.1);
+    m.set_value(2, 3, 1.4);
+
+    ASSERT_EQ(m.get_size(), gko::dim<2>(3, 5));
+    ASSERT_EQ(m.get_num_stored_elements(), 4);
+    ASSERT_EQ(m.get_value(0, 0), 1.3);
+    ASSERT_EQ(m.get_value(2, 3), 1.4);
+    ASSERT_EQ(m.get_value(1, 4), 9.1);
+    ASSERT_EQ(m.get_value(1, 2), 3.6);
+}
+
+
+TEST(MatrixAssemblyData, GetsSortedData)
+{
+    gko::matrix_assembly_data<double, int> m(gko::dim<2>{3, 5});
+    std::vector<gko::matrix_data<double, int>::nonzero_type> reference{
+        {0, 0, 1.3}, {1, 2, 3.6}, {1, 4, 1.1}, {2, 3, 2.2}};
+
+    m.set_value(0, 0, 1.3);
+    m.set_value(2, 3, 2.2);
+    m.set_value(1, 4, 1.1);
+    m.set_value(1, 2, 3.6);
+
+    auto sorted = m.get_ordered_data();
+
+    ASSERT_EQ(sorted.size, m.get_size());
+    ASSERT_EQ(sorted.nonzeros, reference);
+}
+
+
+}  // namespace
diff --git a/include/ginkgo/core/base/matrix_assembly_data.hpp b/include/ginkgo/core/base/matrix_assembly_data.hpp
new file mode 100644
index 00000000000..1fbf7df67dc
--- /dev/null
+++ b/include/ginkgo/core/base/matrix_assembly_data.hpp
@@ -0,0 +1,224 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_BASE_MATRIX_ASSEMBLY_DATA_HPP_
+#define GKO_CORE_BASE_MATRIX_ASSEMBLY_DATA_HPP_
+
+
+#include <algorithm>
+#include <iterator>
+#include <numeric>
+#include <tuple>
+#include <unordered_map>
+
+
+#include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/matrix_data.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/base/utils.hpp>
+
+
+namespace gko {
+
+
+namespace detail {
+
+
+/**
+ * Type used to store indices used as keys in the unordered_map storing
+ * the nonzeros.
+ */
+template <typename IndexType>
+struct symbolic_nonzero {
+    using index_type = IndexType;
+    symbolic_nonzero() = default;
+
+    symbolic_nonzero(index_type r, index_type c) noexcept : row(r), column(c) {}
+
+    bool operator==(symbolic_nonzero other) const noexcept
+    {
+        return std::tie(this->row, this->column) ==
+               std::tie(other.row, other.column);
+    }
+
+    bool operator!=(symbolic_nonzero other) const noexcept
+    {
+        return !(*this == other);
+    }
+
+    index_type row;
+    index_type column;
+};
+
+
+template <typename IndexType>
+struct symbolic_nonzero_hash {
+    explicit symbolic_nonzero_hash(size_type num_cols) noexcept
+        : num_cols_{num_cols}
+    {}
+
+    std::size_t operator()(symbolic_nonzero<IndexType> nnz) const noexcept
+    {
+        return nnz.row * num_cols_ + nnz.column;
+    }
+
+    std::size_t num_cols_;
+};
+
+
+}  // namespace detail
+
+
+/**
+ * This structure is used as an intermediate type to assemble a sparse matrix.
+ *
+ * The matrix is stored as a set of nonzero elements, where each element is
+ * a triple of the form (row_index, column_index, value).
+ *
+ * New values can be added by using the matrix_assembly_data::add_entry or
+ * matrix_assembly_data::add_entries functions.
+ *
+ * @tparam ValueType  type of matrix values stored in the structure
+ * @tparam IndexType  type of matrix indexes stored in the structure
+ */
+template <typename ValueType = default_precision, typename IndexType = int32>
+class matrix_assembly_data {
+public:
+    using value_type = ValueType;
+    using index_type = IndexType;
+
+    explicit matrix_assembly_data(dim<2> size)
+        : size_{size},
+          nonzeros_(0, detail::symbolic_nonzero_hash<index_type>(size_[1]))
+    {}
+
+    /**
+     * Sets the matrix value at (row, col).
+     * If there is an existing value, it will be replaced by the sum of the
+     * existing and new value otherwise it will be.
+     */
+    void add_value(index_type row, index_type col, value_type val)
+    {
+        auto ind = detail::symbolic_nonzero<index_type>{row, col};
+        nonzeros_[ind] += val;
+    }
+
+    /**
+     * Sets the matrix value at (row, col).
+     * If there is an existing value, it will be overwritten by the new value.
+     */
+    void set_value(index_type row, index_type col, value_type val)
+    {
+        auto ind = detail::symbolic_nonzero<index_type>{row, col};
+        nonzeros_[ind] = val;
+    }
+
+    /**
+     * Gets the matrix value at (row, col).
+     * Returns zero if it doesn't exist.
+     */
+    value_type get_value(index_type row, index_type col)
+    {
+        auto ind = detail::symbolic_nonzero<index_type>{row, col};
+        auto it = nonzeros_.find(ind);
+        if (it == nonzeros_.end()) {
+            return zero<value_type>();
+        } else {
+            return it->second;
+        }
+    }
+
+    /**
+     * Returns true iff the matrix has an entry at (row, col).
+     */
+    bool contains(index_type row, index_type col)
+    {
+        auto ind = detail::symbolic_nonzero<index_type>{row, col};
+        auto it = nonzeros_.find(ind);
+        return it != nonzeros_.end();
+    }
+
+    /** Returns the size of the matrix being assembled. */
+    dim<2> get_size() const noexcept { return size_; }
+
+    /** Returns the number of non-zeros in the (partially) assembled matrix. */
+    size_type get_num_stored_elements() const noexcept
+    {
+        return nonzeros_.size();
+    }
+
+    /**
+     * Returns a matrix_data instance containing the assembled non-zeros in
+     * row-major order to be used by all matrix formats.
+     */
+    matrix_data<ValueType, IndexType> get_ordered_data() const
+    {
+        using output_type = matrix_data<ValueType, IndexType>;
+        using nonzero_type = typename output_type::nonzero_type;
+        using entry_type =
+            std::pair<detail::symbolic_nonzero<index_type>, value_type>;
+        output_type data{size_};
+        data.nonzeros.reserve(nonzeros_.size());
+        std::transform(nonzeros_.begin(), nonzeros_.end(),
+                       std::back_inserter(data.nonzeros), [](entry_type entry) {
+                           return nonzero_type{entry.first.row,
+                                               entry.first.column,
+                                               entry.second};
+                       });
+        data.ensure_row_major_order();
+        return data;
+    }
+
+private:
+    /**
+     * Size of the matrix.
+     */
+    dim<2> size_;
+
+    /**
+     * An unordered map storing the non-zeros of the matrix.
+     *
+     * The keys of the elements in the map are the row index and the column
+     * index of a matrix element, and its value is the value at that
+     * position.
+     */
+    std::unordered_map<detail::symbolic_nonzero<index_type>, value_type,
+                       detail::symbolic_nonzero_hash<index_type>>
+        nonzeros_;
+};
+
+
+}  // namespace gko
+
+
+#endif  // GKO_CORE_BASE_MATRIX_ASSEMBLY_DATA_HPP_
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 71c6a94dfaa..755e7753379 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -47,6 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/matrix_assembly_data.hpp>
 #include <ginkgo/core/base/matrix_data.hpp>
 #include <ginkgo/core/base/mtx_io.hpp>
 #include <ginkgo/core/base/name_demangling.hpp>

From 2849296871e0f00330f221b5a78913041a9692ff Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 17 Oct 2020 14:40:36 +0200
Subject: [PATCH 0076/1008] add matrix assembly data to matrix formats

Co-authored-by: Fritz Goebel <goebel.fritz@googlemail.com>
---
 core/test/matrix/coo.cpp                    |  21 ++++
 core/test/matrix/csr.cpp                    |  22 ++++
 core/test/matrix/dense.cpp                  |  26 +++++
 core/test/matrix/ell.cpp                    |  21 ++++
 core/test/matrix/hybrid.cpp                 | 109 ++++++++++++++++++++
 core/test/matrix/sellp.cpp                  |  43 ++++++++
 include/ginkgo/core/base/lin_op.hpp         |  11 ++
 include/ginkgo/core/matrix/coo.hpp          |   1 +
 include/ginkgo/core/matrix/csr.hpp          |   2 +
 include/ginkgo/core/matrix/dense.hpp        |   2 +
 include/ginkgo/core/matrix/ell.hpp          |   1 +
 include/ginkgo/core/matrix/hybrid.hpp       |   1 +
 include/ginkgo/core/matrix/sellp.hpp        |   1 +
 include/ginkgo/core/matrix/sparsity_csr.hpp |   1 +
 14 files changed, 262 insertions(+)

diff --git a/core/test/matrix/coo.cpp b/core/test/matrix/coo.cpp
index 92a999febb6..ab8e0902877 100644
--- a/core/test/matrix/coo.cpp
+++ b/core/test/matrix/coo.cpp
@@ -211,6 +211,27 @@ TYPED_TEST(Coo, CanBeReadFromMatrixData)
 }
 
 
+TYPED_TEST(Coo, CanBeReadFromMatrixAssemblyData)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto m = Mtx::create(this->exec);
+
+    gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
+    data.set_value(0, 0, 1.0);
+    data.set_value(0, 1, 3.0);
+    data.set_value(0, 2, 2.0);
+    data.set_value(1, 0, 0.0);
+    data.set_value(1, 1, 5.0);
+    data.set_value(1, 2, 0.0);
+
+    m->read(data);
+
+    this->assert_equal_to_original_mtx(m.get());
+}
+
+
 TYPED_TEST(Coo, GeneratesCorrectMatrixData)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/core/test/matrix/csr.cpp b/core/test/matrix/csr.cpp
index f927861afce..809be5ec922 100644
--- a/core/test/matrix/csr.cpp
+++ b/core/test/matrix/csr.cpp
@@ -221,6 +221,28 @@ TYPED_TEST(Csr, CanBeReadFromMatrixData)
 }
 
 
+TYPED_TEST(Csr, CanBeReadFromMatrixAssemblyData)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto m = Mtx::create(this->exec,
+                         std::make_shared<typename Mtx::load_balance>(2));
+
+    gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
+    data.set_value(0, 0, 1.0);
+    data.set_value(0, 1, 3.0);
+    data.set_value(0, 2, 2.0);
+    data.set_value(1, 0, 0.0);
+    data.set_value(1, 1, 5.0);
+    data.set_value(1, 2, 0.0);
+
+    m->read(data);
+
+    this->assert_equal_to_original_mtx(m.get());
+}
+
+
 TYPED_TEST(Csr, GeneratesCorrectMatrixData)
 {
     using value_type = typename TestFixture::value_type;
diff --git a/core/test/matrix/dense.cpp b/core/test/matrix/dense.cpp
index c89f9a740e5..81549795416 100644
--- a/core/test/matrix/dense.cpp
+++ b/core/test/matrix/dense.cpp
@@ -278,6 +278,32 @@ TYPED_TEST(Dense, GeneratesCorrectMatrixData)
 }
 
 
+TYPED_TEST(Dense, CanBeReadFromMatrixAssemblyData)
+{
+    using value_type = typename TestFixture::value_type;
+    auto m = gko::matrix::Dense<TypeParam>::create(this->exec);
+
+    gko::matrix_assembly_data<TypeParam> data(gko::dim<2>{2, 3});
+    data.set_value(0, 0, 1.0);
+    data.set_value(0, 1, 3.0);
+    data.set_value(0, 2, 2.0);
+    data.set_value(1, 0, 0.0);
+    data.set_value(1, 1, 5.0);
+    data.set_value(1, 2, 0.0);
+
+    m->read(data);
+
+    ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
+    ASSERT_EQ(m->get_num_stored_elements(), 6);
+    EXPECT_EQ(m->at(0, 0), value_type{1.0});
+    EXPECT_EQ(m->at(1, 0), value_type{0.0});
+    EXPECT_EQ(m->at(0, 1), value_type{3.0});
+    EXPECT_EQ(m->at(1, 1), value_type{5.0});
+    EXPECT_EQ(m->at(0, 2), value_type{2.0});
+    ASSERT_EQ(m->at(1, 2), value_type{0.0});
+}
+
+
 TYPED_TEST(Dense, CanCreateSubmatrix)
 {
     using value_type = typename TestFixture::value_type;
diff --git a/core/test/matrix/ell.cpp b/core/test/matrix/ell.cpp
index 6e92f1251ba..970ea83e6ba 100644
--- a/core/test/matrix/ell.cpp
+++ b/core/test/matrix/ell.cpp
@@ -231,4 +231,25 @@ TYPED_TEST(Ell, GeneratesCorrectMatrixData)
 }
 
 
+TYPED_TEST(Ell, CanBeReadFromMatrixAssemblyData)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto m = Mtx::create(this->exec);
+
+    gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
+    data.set_value(0, 0, 1.0);
+    data.set_value(0, 1, 3.0);
+    data.set_value(0, 2, 2.0);
+    data.set_value(1, 0, 0.0);
+    data.set_value(1, 1, 5.0);
+    data.set_value(1, 2, 0.0);
+
+    m->read(data);
+
+    this->assert_equal_to_original_mtx(m.get());
+}
+
+
 }  // namespace
diff --git a/core/test/matrix/hybrid.cpp b/core/test/matrix/hybrid.cpp
index 9a5ad45f0c7..64ed2576f0b 100644
--- a/core/test/matrix/hybrid.cpp
+++ b/core/test/matrix/hybrid.cpp
@@ -294,6 +294,115 @@ TYPED_TEST(Hybrid, CanBeReadFromMatrixDataByPercent40)
 }
 
 
+TYPED_TEST(Hybrid, CanBeReadFromMatrixAssemblyDataAutomatically)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto m =
+        Mtx::create(this->exec, std::make_shared<typename Mtx::automatic>());
+
+    gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
+    data.set_value(0, 0, 1.0);
+    data.set_value(0, 1, 3.0);
+    data.set_value(0, 2, 2.0);
+    data.set_value(1, 0, 0.0);
+    data.set_value(1, 1, 5.0);
+    data.set_value(1, 2, 0.0);
+
+    m->read(data);
+
+    auto v = m->get_const_coo_values();
+    auto c = m->get_const_coo_col_idxs();
+    auto r = m->get_const_coo_row_idxs();
+    auto n = m->get_ell_num_stored_elements_per_row();
+    auto p = m->get_ell_stride();
+    ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
+    ASSERT_EQ(m->get_ell_num_stored_elements(), 0);
+    ASSERT_EQ(m->get_coo_num_stored_elements(), 4);
+    EXPECT_EQ(n, 0);
+    EXPECT_EQ(p, 2);
+    EXPECT_EQ(r[0], 0);
+    EXPECT_EQ(r[1], 0);
+    EXPECT_EQ(r[2], 0);
+    EXPECT_EQ(r[3], 1);
+    EXPECT_EQ(c[0], 0);
+    EXPECT_EQ(c[1], 1);
+    EXPECT_EQ(c[2], 2);
+    EXPECT_EQ(c[3], 1);
+    EXPECT_EQ(v[0], value_type{1.0});
+    EXPECT_EQ(v[1], value_type{3.0});
+    EXPECT_EQ(v[2], value_type{2.0});
+    EXPECT_EQ(v[3], value_type{5.0});
+}
+
+
+TYPED_TEST(Hybrid, CanBeReadFromMatrixAssemblyDataByColumns2)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto m = Mtx::create(this->exec,
+                         std::make_shared<typename Mtx::column_limit>(2));
+
+    gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
+    data.set_value(0, 0, 1.0);
+    data.set_value(0, 1, 3.0);
+    data.set_value(0, 2, 2.0);
+    data.set_value(1, 0, 0.0);
+    data.set_value(1, 1, 5.0);
+    data.set_value(1, 2, 0.0);
+
+    m->read(data);
+
+    this->assert_equal_to_original_mtx(m.get());
+}
+
+
+TYPED_TEST(Hybrid, CanBeReadFromMatrixAssemblyDataByPercent40)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto m = Mtx::create(this->exec,
+                         std::make_shared<typename Mtx::imbalance_limit>(0.4));
+
+    gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
+    data.set_value(0, 0, 1.0);
+    data.set_value(0, 1, 3.0);
+    data.set_value(0, 2, 2.0);
+    data.set_value(1, 0, 0.0);
+    data.set_value(1, 1, 5.0);
+    data.set_value(1, 2, 0.0);
+
+    m->read(data);
+
+    auto v = m->get_const_ell_values();
+    auto c = m->get_const_ell_col_idxs();
+    auto n = m->get_ell_num_stored_elements_per_row();
+    auto p = m->get_ell_stride();
+    ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
+    ASSERT_EQ(m->get_ell_num_stored_elements(), 2);
+    EXPECT_EQ(n, 1);
+    EXPECT_EQ(p, 2);
+    EXPECT_EQ(c[0], 0);
+    EXPECT_EQ(c[1], 1);
+    EXPECT_EQ(v[0], value_type{1.0});
+    EXPECT_EQ(v[1], value_type{5.0});
+
+    auto coo_v = m->get_const_coo_values();
+    auto coo_c = m->get_const_coo_col_idxs();
+    auto coo_r = m->get_const_coo_row_idxs();
+    ASSERT_EQ(m->get_coo_num_stored_elements(), 2);
+    EXPECT_EQ(coo_v[0], value_type{3.0});
+    EXPECT_EQ(coo_v[1], value_type{2.0});
+    EXPECT_EQ(coo_c[0], 1);
+    EXPECT_EQ(coo_c[1], 2);
+    EXPECT_EQ(coo_r[0], 0);
+    EXPECT_EQ(coo_r[1], 0);
+}
+
+
 TYPED_TEST(Hybrid, GeneratesCorrectMatrixData)
 {
     using value_type = typename TestFixture::value_type;
diff --git a/core/test/matrix/sellp.cpp b/core/test/matrix/sellp.cpp
index d6f139ba82e..26b58077131 100644
--- a/core/test/matrix/sellp.cpp
+++ b/core/test/matrix/sellp.cpp
@@ -274,4 +274,47 @@ TYPED_TEST(Sellp, GeneratesCorrectMatrixData)
 }
 
 
+TYPED_TEST(Sellp, CanBeReadFromMatrixAssemblyData)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto m = Mtx::create(this->exec);
+
+    gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
+    data.set_value(0, 0, 1.0);
+    data.set_value(0, 1, 3.0);
+    data.set_value(0, 2, 2.0);
+    data.set_value(1, 0, 0.0);
+    data.set_value(1, 1, 5.0);
+    data.set_value(1, 2, 0.0);
+
+    m->read(data);
+
+    this->assert_equal_to_original_mtx(m.get());
+}
+
+
+TYPED_TEST(Sellp, CanBeReadFromMatrixAssemblyDataWithSliceSizeAndStrideFactor)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto m = Mtx::create(this->exec, gko::dim<2>{2, 3}, 2, 2, 3);
+
+    gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
+    data.set_value(0, 0, 1.0);
+    data.set_value(0, 1, 3.0);
+    data.set_value(0, 2, 2.0);
+    data.set_value(1, 0, 0.0);
+    data.set_value(1, 1, 5.0);
+    data.set_value(1, 2, 0.0);
+
+    m->read(data);
+
+    this->assert_equal_to_original_mtx_with_slice_size_and_stride_factor(
+        m.get());
+}
+
+
 }  // namespace
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index 6c873198b6e..7ce4cef3e31 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/dim.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/matrix_assembly_data.hpp>
 #include <ginkgo/core/base/matrix_data.hpp>
 #include <ginkgo/core/base/polymorphic_object.hpp>
 #include <ginkgo/core/base/types.hpp>
@@ -538,6 +539,16 @@ class ReadableFromMatrixData {
      * @param data  the matrix_data structure
      */
     virtual void read(const matrix_data<ValueType, IndexType> &data) = 0;
+
+    /**
+     * Reads a matrix from a matrix_assembly_data structure.
+     *
+     * @param data  the matrix_assembly_data structure
+     */
+    void read(const matrix_assembly_data<ValueType, IndexType> &data)
+    {
+        this->read(data.get_ordered_data());
+    }
 };
 
 
diff --git a/include/ginkgo/core/matrix/coo.hpp b/include/ginkgo/core/matrix/coo.hpp
index dd048fd5825..7fe9fe02533 100644
--- a/include/ginkgo/core/matrix/coo.hpp
+++ b/include/ginkgo/core/matrix/coo.hpp
@@ -94,6 +94,7 @@ class Coo : public EnableLinOp<Coo<ValueType, IndexType>>,
 public:
     using EnableLinOp<Coo>::convert_to;
     using EnableLinOp<Coo>::move_to;
+    using ReadableFromMatrixData<ValueType, IndexType>::read;
 
     using value_type = ValueType;
     using index_type = IndexType;
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index a30f6f856e3..c0f6d914465 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -145,6 +145,8 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
     friend class Csr<to_complex<ValueType>, IndexType>;
 
 public:
+    using ReadableFromMatrixData<ValueType, IndexType>::read;
+
     using value_type = ValueType;
     using index_type = IndexType;
     using transposed_type = Csr<ValueType, IndexType>;
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index b6ec2421455..ecc155b9ff2 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -133,6 +133,8 @@ class Dense
 public:
     using EnableLinOp<Dense>::convert_to;
     using EnableLinOp<Dense>::move_to;
+    using ReadableFromMatrixData<ValueType, int32>::read;
+    using ReadableFromMatrixData<ValueType, int64>::read;
 
     using value_type = ValueType;
     using index_type = int64;
diff --git a/include/ginkgo/core/matrix/ell.hpp b/include/ginkgo/core/matrix/ell.hpp
index cbcd3552700..2fab5c9de42 100644
--- a/include/ginkgo/core/matrix/ell.hpp
+++ b/include/ginkgo/core/matrix/ell.hpp
@@ -85,6 +85,7 @@ class Ell : public EnableLinOp<Ell<ValueType, IndexType>>,
     friend class Ell<to_complex<ValueType>, IndexType>;
 
 public:
+    using ReadableFromMatrixData<ValueType, IndexType>::read;
     using EnableLinOp<Ell>::convert_to;
     using EnableLinOp<Ell>::move_to;
 
diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp
index c844907e8b2..91a62ac1a5a 100644
--- a/include/ginkgo/core/matrix/hybrid.hpp
+++ b/include/ginkgo/core/matrix/hybrid.hpp
@@ -87,6 +87,7 @@ class Hybrid
 
 
 public:
+    using ReadableFromMatrixData<ValueType, IndexType>::read;
     using EnableLinOp<Hybrid>::convert_to;
     using EnableLinOp<Hybrid>::move_to;
 
diff --git a/include/ginkgo/core/matrix/sellp.hpp b/include/ginkgo/core/matrix/sellp.hpp
index e28a5f3a73e..623342c1cf5 100644
--- a/include/ginkgo/core/matrix/sellp.hpp
+++ b/include/ginkgo/core/matrix/sellp.hpp
@@ -82,6 +82,7 @@ class Sellp : public EnableLinOp<Sellp<ValueType, IndexType>>,
     friend class Sellp<to_complex<ValueType>, IndexType>;
 
 public:
+    using ReadableFromMatrixData<ValueType, IndexType>::read;
     using EnableLinOp<Sellp>::convert_to;
     using EnableLinOp<Sellp>::move_to;
 
diff --git a/include/ginkgo/core/matrix/sparsity_csr.hpp b/include/ginkgo/core/matrix/sparsity_csr.hpp
index 25374a251b8..07595ae2698 100644
--- a/include/ginkgo/core/matrix/sparsity_csr.hpp
+++ b/include/ginkgo/core/matrix/sparsity_csr.hpp
@@ -79,6 +79,7 @@ class SparsityCsr
     friend class Csr<ValueType, IndexType>;
 
 public:
+    using ReadableFromMatrixData<ValueType, IndexType>::read;
     using EnableLinOp<SparsityCsr>::convert_to;
     using EnableLinOp<SparsityCsr>::move_to;
 

From 97130f67a686f89af3946850815c20c47bb16058 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 23 Oct 2020 20:19:46 +0200
Subject: [PATCH 0077/1008] review updates
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Yuhsiang M. Tsai <yhmtsai@gmail.com>
Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 core/test/base/matrix_assembly_data.cpp       |  1 -
 core/test/matrix/coo.cpp                      |  1 -
 core/test/matrix/csr.cpp                      |  1 -
 core/test/matrix/dense.cpp                    |  1 -
 core/test/matrix/ell.cpp                      |  1 -
 core/test/matrix/hybrid.cpp                   | 21 +++-----
 core/test/matrix/sellp.cpp                    |  2 -
 .../ginkgo/core/base/matrix_assembly_data.hpp | 50 ++++++++++++-------
 include/ginkgo/core/matrix/ell.hpp            |  2 +-
 include/ginkgo/core/matrix/hybrid.hpp         |  2 +-
 include/ginkgo/core/matrix/sellp.hpp          |  2 +-
 include/ginkgo/core/matrix/sparsity_csr.hpp   |  2 +-
 12 files changed, 44 insertions(+), 42 deletions(-)

diff --git a/core/test/base/matrix_assembly_data.cpp b/core/test/base/matrix_assembly_data.cpp
index 05157ce2efe..8ae072e39d6 100644
--- a/core/test/base/matrix_assembly_data.cpp
+++ b/core/test/base/matrix_assembly_data.cpp
@@ -114,7 +114,6 @@ TEST(MatrixAssemblyData, GetsSortedData)
     gko::matrix_assembly_data<double, int> m(gko::dim<2>{3, 5});
     std::vector<gko::matrix_data<double, int>::nonzero_type> reference{
         {0, 0, 1.3}, {1, 2, 3.6}, {1, 4, 1.1}, {2, 3, 2.2}};
-
     m.set_value(0, 0, 1.3);
     m.set_value(2, 3, 2.2);
     m.set_value(1, 4, 1.1);
diff --git a/core/test/matrix/coo.cpp b/core/test/matrix/coo.cpp
index ab8e0902877..978f9ffc17a 100644
--- a/core/test/matrix/coo.cpp
+++ b/core/test/matrix/coo.cpp
@@ -217,7 +217,6 @@ TYPED_TEST(Coo, CanBeReadFromMatrixAssemblyData)
     using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
     auto m = Mtx::create(this->exec);
-
     gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
     data.set_value(0, 0, 1.0);
     data.set_value(0, 1, 3.0);
diff --git a/core/test/matrix/csr.cpp b/core/test/matrix/csr.cpp
index 809be5ec922..79797fde20d 100644
--- a/core/test/matrix/csr.cpp
+++ b/core/test/matrix/csr.cpp
@@ -228,7 +228,6 @@ TYPED_TEST(Csr, CanBeReadFromMatrixAssemblyData)
     using index_type = typename TestFixture::index_type;
     auto m = Mtx::create(this->exec,
                          std::make_shared<typename Mtx::load_balance>(2));
-
     gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
     data.set_value(0, 0, 1.0);
     data.set_value(0, 1, 3.0);
diff --git a/core/test/matrix/dense.cpp b/core/test/matrix/dense.cpp
index 81549795416..dad8c64cfa2 100644
--- a/core/test/matrix/dense.cpp
+++ b/core/test/matrix/dense.cpp
@@ -282,7 +282,6 @@ TYPED_TEST(Dense, CanBeReadFromMatrixAssemblyData)
 {
     using value_type = typename TestFixture::value_type;
     auto m = gko::matrix::Dense<TypeParam>::create(this->exec);
-
     gko::matrix_assembly_data<TypeParam> data(gko::dim<2>{2, 3});
     data.set_value(0, 0, 1.0);
     data.set_value(0, 1, 3.0);
diff --git a/core/test/matrix/ell.cpp b/core/test/matrix/ell.cpp
index 970ea83e6ba..9fe00a82621 100644
--- a/core/test/matrix/ell.cpp
+++ b/core/test/matrix/ell.cpp
@@ -237,7 +237,6 @@ TYPED_TEST(Ell, CanBeReadFromMatrixAssemblyData)
     using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
     auto m = Mtx::create(this->exec);
-
     gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
     data.set_value(0, 0, 1.0);
     data.set_value(0, 1, 3.0);
diff --git a/core/test/matrix/hybrid.cpp b/core/test/matrix/hybrid.cpp
index 64ed2576f0b..e882b2cde96 100644
--- a/core/test/matrix/hybrid.cpp
+++ b/core/test/matrix/hybrid.cpp
@@ -272,19 +272,18 @@ TYPED_TEST(Hybrid, CanBeReadFromMatrixDataByPercent40)
     auto c = m->get_const_ell_col_idxs();
     auto n = m->get_ell_num_stored_elements_per_row();
     auto p = m->get_ell_stride();
+    auto coo_v = m->get_const_coo_values();
+    auto coo_c = m->get_const_coo_col_idxs();
+    auto coo_r = m->get_const_coo_row_idxs();
     ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
     ASSERT_EQ(m->get_ell_num_stored_elements(), 2);
+    ASSERT_EQ(m->get_coo_num_stored_elements(), 2);
     EXPECT_EQ(n, 1);
     EXPECT_EQ(p, 2);
     EXPECT_EQ(c[0], 0);
     EXPECT_EQ(c[1], 1);
     EXPECT_EQ(v[0], value_type{1.0});
     EXPECT_EQ(v[1], value_type{5.0});
-
-    auto coo_v = m->get_const_coo_values();
-    auto coo_c = m->get_const_coo_col_idxs();
-    auto coo_r = m->get_const_coo_row_idxs();
-    ASSERT_EQ(m->get_coo_num_stored_elements(), 2);
     EXPECT_EQ(coo_v[0], value_type{3.0});
     EXPECT_EQ(coo_v[1], value_type{2.0});
     EXPECT_EQ(coo_c[0], 1);
@@ -301,7 +300,6 @@ TYPED_TEST(Hybrid, CanBeReadFromMatrixAssemblyDataAutomatically)
     using index_type = typename TestFixture::index_type;
     auto m =
         Mtx::create(this->exec, std::make_shared<typename Mtx::automatic>());
-
     gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
     data.set_value(0, 0, 1.0);
     data.set_value(0, 1, 3.0);
@@ -344,7 +342,6 @@ TYPED_TEST(Hybrid, CanBeReadFromMatrixAssemblyDataByColumns2)
     using index_type = typename TestFixture::index_type;
     auto m = Mtx::create(this->exec,
                          std::make_shared<typename Mtx::column_limit>(2));
-
     gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
     data.set_value(0, 0, 1.0);
     data.set_value(0, 1, 3.0);
@@ -366,7 +363,6 @@ TYPED_TEST(Hybrid, CanBeReadFromMatrixAssemblyDataByPercent40)
     using index_type = typename TestFixture::index_type;
     auto m = Mtx::create(this->exec,
                          std::make_shared<typename Mtx::imbalance_limit>(0.4));
-
     gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
     data.set_value(0, 0, 1.0);
     data.set_value(0, 1, 3.0);
@@ -381,19 +377,18 @@ TYPED_TEST(Hybrid, CanBeReadFromMatrixAssemblyDataByPercent40)
     auto c = m->get_const_ell_col_idxs();
     auto n = m->get_ell_num_stored_elements_per_row();
     auto p = m->get_ell_stride();
+    auto coo_v = m->get_const_coo_values();
+    auto coo_c = m->get_const_coo_col_idxs();
+    auto coo_r = m->get_const_coo_row_idxs();
     ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
     ASSERT_EQ(m->get_ell_num_stored_elements(), 2);
+    ASSERT_EQ(m->get_coo_num_stored_elements(), 2);
     EXPECT_EQ(n, 1);
     EXPECT_EQ(p, 2);
     EXPECT_EQ(c[0], 0);
     EXPECT_EQ(c[1], 1);
     EXPECT_EQ(v[0], value_type{1.0});
     EXPECT_EQ(v[1], value_type{5.0});
-
-    auto coo_v = m->get_const_coo_values();
-    auto coo_c = m->get_const_coo_col_idxs();
-    auto coo_r = m->get_const_coo_row_idxs();
-    ASSERT_EQ(m->get_coo_num_stored_elements(), 2);
     EXPECT_EQ(coo_v[0], value_type{3.0});
     EXPECT_EQ(coo_v[1], value_type{2.0});
     EXPECT_EQ(coo_c[0], 1);
diff --git a/core/test/matrix/sellp.cpp b/core/test/matrix/sellp.cpp
index 26b58077131..7b9439e820c 100644
--- a/core/test/matrix/sellp.cpp
+++ b/core/test/matrix/sellp.cpp
@@ -280,7 +280,6 @@ TYPED_TEST(Sellp, CanBeReadFromMatrixAssemblyData)
     using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
     auto m = Mtx::create(this->exec);
-
     gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
     data.set_value(0, 0, 1.0);
     data.set_value(0, 1, 3.0);
@@ -301,7 +300,6 @@ TYPED_TEST(Sellp, CanBeReadFromMatrixAssemblyDataWithSliceSizeAndStrideFactor)
     using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
     auto m = Mtx::create(this->exec, gko::dim<2>{2, 3}, 2, 2, 3);
-
     gko::matrix_assembly_data<value_type, index_type> data(gko::dim<2>{2, 3});
     data.set_value(0, 0, 1.0);
     data.set_value(0, 1, 3.0);
diff --git a/include/ginkgo/core/base/matrix_assembly_data.hpp b/include/ginkgo/core/base/matrix_assembly_data.hpp
index 1fbf7df67dc..557e9e160eb 100644
--- a/include/ginkgo/core/base/matrix_assembly_data.hpp
+++ b/include/ginkgo/core/base/matrix_assembly_data.hpp
@@ -49,8 +49,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 namespace gko {
-
-
 namespace detail {
 
 
@@ -61,6 +59,7 @@ namespace detail {
 template <typename IndexType>
 struct symbolic_nonzero {
     using index_type = IndexType;
+
     symbolic_nonzero() = default;
 
     symbolic_nonzero(index_type r, index_type c) noexcept : row(r), column(c) {}
@@ -89,10 +88,10 @@ struct symbolic_nonzero_hash {
 
     std::size_t operator()(symbolic_nonzero<IndexType> nnz) const noexcept
     {
-        return nnz.row * num_cols_ + nnz.column;
+        return static_cast<std::size_t>(nnz.row) * num_cols_ + nnz.column;
     }
 
-    std::size_t num_cols_;
+    size_type num_cols_;
 };
 
 
@@ -103,10 +102,10 @@ struct symbolic_nonzero_hash {
  * This structure is used as an intermediate type to assemble a sparse matrix.
  *
  * The matrix is stored as a set of nonzero elements, where each element is
- * a triple of the form (row_index, column_index, value).
+ * a triplet of the form (row_index, column_index, value).
  *
- * New values can be added by using the matrix_assembly_data::add_entry or
- * matrix_assembly_data::add_entries functions.
+ * New values can be added by using the matrix_assembly_data::add_value or
+ * matrix_assembly_data::set_value
  *
  * @tparam ValueType  type of matrix values stored in the structure
  * @tparam IndexType  type of matrix indexes stored in the structure
@@ -124,8 +123,12 @@ class matrix_assembly_data {
 
     /**
      * Sets the matrix value at (row, col).
-     * If there is an existing value, it will be replaced by the sum of the
-     * existing and new value otherwise it will be.
+     * If there is an existing value, it will be set to the sum of the
+     * existing and new value, otherwise the value will be inserted.
+     *
+     * @param row  the row where the value should be added
+     * @param col  the column where the value should be added
+     * @param val  the value to be added to (row, col)
      */
     void add_value(index_type row, index_type col, value_type val)
     {
@@ -136,6 +139,10 @@ class matrix_assembly_data {
     /**
      * Sets the matrix value at (row, col).
      * If there is an existing value, it will be overwritten by the new value.
+     *
+     * @param row  the row index
+     * @param col  the column index
+     * @param val  the value to be written to (row, col)
      */
     void set_value(index_type row, index_type col, value_type val)
     {
@@ -145,12 +152,15 @@ class matrix_assembly_data {
 
     /**
      * Gets the matrix value at (row, col).
-     * Returns zero if it doesn't exist.
+     *
+     * @param row  the row index
+     * @param col  the column index
+     * @return the value at (row, col) or 0 if it doesn't exist.
      */
     value_type get_value(index_type row, index_type col)
     {
-        auto ind = detail::symbolic_nonzero<index_type>{row, col};
-        auto it = nonzeros_.find(ind);
+        const auto ind = detail::symbolic_nonzero<index_type>{row, col};
+        const auto it = nonzeros_.find(ind);
         if (it == nonzeros_.end()) {
             return zero<value_type>();
         } else {
@@ -159,26 +169,30 @@ class matrix_assembly_data {
     }
 
     /**
-     * Returns true iff the matrix has an entry at (row, col).
+     * Returns true iff the matrix contains an entry at (row, col).
+     *
+     * @param row  the row index
+     * @param col  the column index
+     * @return true if the value at (row, col) exists, false otherwise
      */
     bool contains(index_type row, index_type col)
     {
-        auto ind = detail::symbolic_nonzero<index_type>{row, col};
-        auto it = nonzeros_.find(ind);
+        const auto ind = detail::symbolic_nonzero<index_type>{row, col};
+        const auto it = nonzeros_.find(ind);
         return it != nonzeros_.end();
     }
 
-    /** Returns the size of the matrix being assembled. */
+    /** @return the dimensions of the matrix being assembled */
     dim<2> get_size() const noexcept { return size_; }
 
-    /** Returns the number of non-zeros in the (partially) assembled matrix. */
+    /** @return the number of non-zeros in the (partially) assembled matrix */
     size_type get_num_stored_elements() const noexcept
     {
         return nonzeros_.size();
     }
 
     /**
-     * Returns a matrix_data instance containing the assembled non-zeros in
+     * @return a matrix_data instance containing the assembled non-zeros in
      * row-major order to be used by all matrix formats.
      */
     matrix_data<ValueType, IndexType> get_ordered_data() const
diff --git a/include/ginkgo/core/matrix/ell.hpp b/include/ginkgo/core/matrix/ell.hpp
index 2fab5c9de42..d8970d2b1c3 100644
--- a/include/ginkgo/core/matrix/ell.hpp
+++ b/include/ginkgo/core/matrix/ell.hpp
@@ -85,9 +85,9 @@ class Ell : public EnableLinOp<Ell<ValueType, IndexType>>,
     friend class Ell<to_complex<ValueType>, IndexType>;
 
 public:
-    using ReadableFromMatrixData<ValueType, IndexType>::read;
     using EnableLinOp<Ell>::convert_to;
     using EnableLinOp<Ell>::move_to;
+    using ReadableFromMatrixData<ValueType, IndexType>::read;
 
     using value_type = ValueType;
     using index_type = IndexType;
diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp
index 91a62ac1a5a..babff97efbc 100644
--- a/include/ginkgo/core/matrix/hybrid.hpp
+++ b/include/ginkgo/core/matrix/hybrid.hpp
@@ -87,9 +87,9 @@ class Hybrid
 
 
 public:
-    using ReadableFromMatrixData<ValueType, IndexType>::read;
     using EnableLinOp<Hybrid>::convert_to;
     using EnableLinOp<Hybrid>::move_to;
+    using ReadableFromMatrixData<ValueType, IndexType>::read;
 
     using value_type = ValueType;
     using index_type = IndexType;
diff --git a/include/ginkgo/core/matrix/sellp.hpp b/include/ginkgo/core/matrix/sellp.hpp
index 623342c1cf5..8540b7f1c5d 100644
--- a/include/ginkgo/core/matrix/sellp.hpp
+++ b/include/ginkgo/core/matrix/sellp.hpp
@@ -82,9 +82,9 @@ class Sellp : public EnableLinOp<Sellp<ValueType, IndexType>>,
     friend class Sellp<to_complex<ValueType>, IndexType>;
 
 public:
-    using ReadableFromMatrixData<ValueType, IndexType>::read;
     using EnableLinOp<Sellp>::convert_to;
     using EnableLinOp<Sellp>::move_to;
+    using ReadableFromMatrixData<ValueType, IndexType>::read;
 
     using value_type = ValueType;
     using index_type = IndexType;
diff --git a/include/ginkgo/core/matrix/sparsity_csr.hpp b/include/ginkgo/core/matrix/sparsity_csr.hpp
index 07595ae2698..f55da0bfa83 100644
--- a/include/ginkgo/core/matrix/sparsity_csr.hpp
+++ b/include/ginkgo/core/matrix/sparsity_csr.hpp
@@ -79,9 +79,9 @@ class SparsityCsr
     friend class Csr<ValueType, IndexType>;
 
 public:
-    using ReadableFromMatrixData<ValueType, IndexType>::read;
     using EnableLinOp<SparsityCsr>::convert_to;
     using EnableLinOp<SparsityCsr>::move_to;
+    using ReadableFromMatrixData<ValueType, IndexType>::read;
 
     using value_type = ValueType;
     using index_type = IndexType;

From 2ac237e510bae5d633493b88d953842ca7939d9b Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 23 Oct 2020 20:38:16 +0200
Subject: [PATCH 0078/1008] use pair for symbolic non-zero location

---
 .../ginkgo/core/base/matrix_assembly_data.hpp | 51 ++++---------------
 1 file changed, 10 insertions(+), 41 deletions(-)

diff --git a/include/ginkgo/core/base/matrix_assembly_data.hpp b/include/ginkgo/core/base/matrix_assembly_data.hpp
index 557e9e160eb..b54d10f4db1 100644
--- a/include/ginkgo/core/base/matrix_assembly_data.hpp
+++ b/include/ginkgo/core/base/matrix_assembly_data.hpp
@@ -52,43 +52,15 @@ namespace gko {
 namespace detail {
 
 
-/**
- * Type used to store indices used as keys in the unordered_map storing
- * the nonzeros.
- */
-template <typename IndexType>
-struct symbolic_nonzero {
-    using index_type = IndexType;
-
-    symbolic_nonzero() = default;
-
-    symbolic_nonzero(index_type r, index_type c) noexcept : row(r), column(c) {}
-
-    bool operator==(symbolic_nonzero other) const noexcept
-    {
-        return std::tie(this->row, this->column) ==
-               std::tie(other.row, other.column);
-    }
-
-    bool operator!=(symbolic_nonzero other) const noexcept
-    {
-        return !(*this == other);
-    }
-
-    index_type row;
-    index_type column;
-};
-
-
 template <typename IndexType>
 struct symbolic_nonzero_hash {
     explicit symbolic_nonzero_hash(size_type num_cols) noexcept
         : num_cols_{num_cols}
     {}
 
-    std::size_t operator()(symbolic_nonzero<IndexType> nnz) const noexcept
+    std::size_t operator()(std::pair<IndexType, IndexType> nnz) const noexcept
     {
-        return static_cast<std::size_t>(nnz.row) * num_cols_ + nnz.column;
+        return static_cast<std::size_t>(nnz.first) * num_cols_ + nnz.second;
     }
 
     size_type num_cols_;
@@ -132,7 +104,7 @@ class matrix_assembly_data {
      */
     void add_value(index_type row, index_type col, value_type val)
     {
-        auto ind = detail::symbolic_nonzero<index_type>{row, col};
+        auto ind = std::make_pair(row, col);
         nonzeros_[ind] += val;
     }
 
@@ -146,7 +118,7 @@ class matrix_assembly_data {
      */
     void set_value(index_type row, index_type col, value_type val)
     {
-        auto ind = detail::symbolic_nonzero<index_type>{row, col};
+        auto ind = std::make_pair(row, col);
         nonzeros_[ind] = val;
     }
 
@@ -159,8 +131,7 @@ class matrix_assembly_data {
      */
     value_type get_value(index_type row, index_type col)
     {
-        const auto ind = detail::symbolic_nonzero<index_type>{row, col};
-        const auto it = nonzeros_.find(ind);
+        const auto it = nonzeros_.find(std::make_pair(row, col));
         if (it == nonzeros_.end()) {
             return zero<value_type>();
         } else {
@@ -177,9 +148,7 @@ class matrix_assembly_data {
      */
     bool contains(index_type row, index_type col)
     {
-        const auto ind = detail::symbolic_nonzero<index_type>{row, col};
-        const auto it = nonzeros_.find(ind);
-        return it != nonzeros_.end();
+        return nonzeros_.find(std::make_pair(row, col)) != nonzeros_.end();
     }
 
     /** @return the dimensions of the matrix being assembled */
@@ -200,13 +169,13 @@ class matrix_assembly_data {
         using output_type = matrix_data<ValueType, IndexType>;
         using nonzero_type = typename output_type::nonzero_type;
         using entry_type =
-            std::pair<detail::symbolic_nonzero<index_type>, value_type>;
+            std::pair<std::pair<index_type, index_type>, value_type>;
         output_type data{size_};
         data.nonzeros.reserve(nonzeros_.size());
         std::transform(nonzeros_.begin(), nonzeros_.end(),
                        std::back_inserter(data.nonzeros), [](entry_type entry) {
-                           return nonzero_type{entry.first.row,
-                                               entry.first.column,
+                           return nonzero_type{entry.first.first,
+                                               entry.first.second,
                                                entry.second};
                        });
         data.ensure_row_major_order();
@@ -226,7 +195,7 @@ class matrix_assembly_data {
      * index of a matrix element, and its value is the value at that
      * position.
      */
-    std::unordered_map<detail::symbolic_nonzero<index_type>, value_type,
+    std::unordered_map<std::pair<index_type, index_type>, value_type,
                        detail::symbolic_nonzero_hash<index_type>>
         nonzeros_;
 };

From a55c2a967c8acfdd666e1f68f03b73dc87fc26fa Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Tue, 20 Oct 2020 10:54:33 +0200
Subject: [PATCH 0079/1008] Add view_as_real to dense

---
 core/matrix/dense.cpp                | 13 +++++++--
 include/ginkgo/core/matrix/dense.hpp | 43 ++++++++++++++++++++++++++++
 include/ginkgo/ginkgo.hpp            |  1 +
 3 files changed, 55 insertions(+), 2 deletions(-)

diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index aaaaf2d01a5..83da73f9960 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
+#include <type_traits>
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -215,8 +216,16 @@ inline void conversion_helper(SparsityCsr<ValueType, IndexType> *result,
 template <typename ValueType>
 void Dense<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    this->get_executor()->run(dense::make_simple_apply(
-        this, as<Dense<ValueType>>(b), as<Dense<ValueType>>(x)));
+    if (dynamic_cast<const Dense<ValueType> *>(b)) {
+        this->get_executor()->run(dense::make_simple_apply(
+            this, as<Dense<ValueType>>(b), as<Dense<ValueType>>(x)));
+    } else {
+        auto dense_b = as<const Dense<to_complex<ValueType>>>(b);
+        auto dense_x = as<Dense<to_complex<ValueType>>>(x);
+        this->get_executor()->run(
+            dense::make_simple_apply(this, dense_b->view_as_real().get(),
+                                     dense_x->view_as_real().get()));
+    }
 }
 
 
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index ecc155b9ff2..c3de3caa48c 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -443,6 +443,49 @@ class Dense
         return create_submatrix(rows, columns, this->get_stride());
     }
 
+    /*
+     * Create a real view of the (potentially) complex original matrix.
+     */
+    std::unique_ptr<Dense<remove_complex<ValueType>>> view_as_real()
+    {
+        const auto num_rows = this->get_size()[0];
+        const bool complex = is_complex<ValueType>();
+        const auto num_cols =
+            complex ? this->get_size()[1] : 2 * this->get_size()[1];
+        const auto stride =
+            complex ? this->get_stride() : 2 * this->get_stride();
+
+        return Dense<remove_complex<ValueType>>::create(
+            this->get_executor(), dim<2>{num_rows, num_cols},
+            Array<remove_complex<ValueType>>::view(
+                this->get_executor(), num_rows * stride,
+                reinterpret_cast<remove_complex<ValueType> *>(
+                    this->get_values())),
+            stride);
+    }
+
+    /*
+     * Create a constant real view of the (potentially) complex original matrix.
+     */
+    std::unique_ptr<Dense<remove_complex<ValueType>>> view_as_real() const
+    {
+        const auto num_rows = this->get_size()[0];
+        const bool complex = is_complex<ValueType>();
+        const auto num_cols =
+            complex ? this->get_size()[1] : 2 * this->get_size()[1];
+        const auto stride =
+            complex ? this->get_stride() : 2 * this->get_stride();
+
+        return Dense<remove_complex<ValueType>>::create(
+            this->get_executor(), dim<2>{num_rows, num_cols},
+            Array<remove_complex<ValueType>>::view(
+                this->get_executor(), num_rows * stride,
+                const_cast<remove_complex<ValueType *>>(
+                    reinterpret_cast<remove_complex<const ValueType> *>(
+                        this->get_const_values()))),
+            stride);
+    }
+
 protected:
     /**
      * Creates an uninitialized Dense matrix of the specified size.
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 755e7753379..4c44166afff 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -55,6 +55,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/polymorphic_object.hpp>
 #include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/base/range_accessors.hpp>
+#include <ginkgo/core/base/readable_data.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/base/utils.hpp>

From 4febe38e6c2d8cd7084ae3849e0daa21fdca8b30 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Tue, 20 Oct 2020 18:15:35 +0200
Subject: [PATCH 0080/1008] Add application to complex vectors to matrix
 formats. Add tests

---
 core/matrix/coo.cpp                      | 67 +++++++++++++++---
 core/matrix/csr.cpp                      | 46 +++++++++----
 core/matrix/dense.cpp                    | 25 +++++--
 core/matrix/ell.cpp                      | 34 +++++++--
 core/matrix/sellp.cpp                    | 35 ++++++++--
 core/test/matrix/dense.cpp               | 36 ++++++++++
 cuda/test/matrix/coo_kernels.cpp         | 54 ++++++++++++++-
 cuda/test/matrix/csr_kernels.cpp         | 32 +++++++++
 cuda/test/matrix/dense_kernels.cpp       | 32 +++++++++
 cuda/test/matrix/ell_kernels.cpp         | 38 ++++++++++-
 cuda/test/matrix/hybrid_kernels.cpp      | 39 ++++++++++-
 cuda/test/matrix/sellp_kernels.cpp       | 38 ++++++++++-
 hip/test/matrix/coo_kernels.hip.cpp      | 54 ++++++++++++++-
 hip/test/matrix/csr_kernels.hip.cpp      | 35 +++++++++-
 hip/test/matrix/dense_kernels.hip.cpp    | 32 +++++++++
 hip/test/matrix/ell_kernels.hip.cpp      | 38 ++++++++++-
 hip/test/matrix/hybrid_kernels.hip.cpp   | 39 ++++++++++-
 hip/test/matrix/sellp_kernels.hip.cpp    | 38 ++++++++++-
 include/ginkgo/core/matrix/dense.hpp     | 14 ++--
 include/ginkgo/ginkgo.hpp                |  1 -
 omp/test/matrix/coo_kernels.cpp          | 55 ++++++++++++++-
 omp/test/matrix/csr_kernels.cpp          | 32 +++++++++
 omp/test/matrix/dense_kernels.cpp        | 32 +++++++++
 omp/test/matrix/ell_kernels.cpp          | 39 ++++++++++-
 omp/test/matrix/hybrid_kernels.cpp       | 39 ++++++++++-
 omp/test/matrix/sellp_kernels.cpp        | 38 ++++++++++-
 reference/test/matrix/coo_kernels.cpp    | 87 ++++++++++++++++++++++++
 reference/test/matrix/csr_kernels.cpp    | 57 ++++++++++++++++
 reference/test/matrix/dense_kernels.cpp  | 56 +++++++++++++++
 reference/test/matrix/ell_kernels.cpp    | 57 ++++++++++++++++
 reference/test/matrix/hybrid_kernels.cpp | 57 ++++++++++++++++
 reference/test/matrix/sellp_kernels.cpp  | 57 ++++++++++++++++
 32 files changed, 1259 insertions(+), 74 deletions(-)

diff --git a/core/matrix/coo.cpp b/core/matrix/coo.cpp
index 6ab488d9988..158addd6d6b 100644
--- a/core/matrix/coo.cpp
+++ b/core/matrix/coo.cpp
@@ -77,8 +77,18 @@ GKO_REGISTER_OPERATION(outplace_absolute_array,
 template <typename ValueType, typename IndexType>
 void Coo<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    using Dense = Dense<ValueType>;
-    this->get_executor()->run(coo::make_spmv(this, as<Dense>(b), as<Dense>(x)));
+    using ComplexDense = Dense<to_complex<ValueType>>;
+
+    if (dynamic_cast<const Dense<ValueType> *>(b)) {
+        this->get_executor()->run(coo::make_spmv(this, as<Dense<ValueType>>(b),
+                                                 as<Dense<ValueType>>(x)));
+    } else {
+        auto dense_b = as<ComplexDense>(b);
+        auto dense_x = as<ComplexDense>(x);
+        this->get_executor()->run(coo::make_spmv(
+            as<Coo<remove_complex<ValueType>, IndexType>>(this),
+            dense_b->view_as_real().get(), dense_x->view_as_real().get()));
+    }
 }
 
 
@@ -86,18 +96,41 @@ template <typename ValueType, typename IndexType>
 void Coo<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                            const LinOp *beta, LinOp *x) const
 {
-    using Dense = Dense<ValueType>;
-    this->get_executor()->run(coo::make_advanced_spmv(
-        as<Dense>(alpha), this, as<Dense>(b), as<Dense>(beta), as<Dense>(x)));
+    using ComplexDense = Dense<to_complex<ValueType>>;
+    using RealDense = Dense<remove_complex<ValueType>>;
+
+    if (dynamic_cast<const Dense<ValueType> *>(b)) {
+        this->get_executor()->run(coo::make_advanced_spmv(
+            as<Dense<ValueType>>(alpha), this, as<Dense<ValueType>>(b),
+            as<Dense<ValueType>>(beta), as<Dense<ValueType>>(x)));
+    } else {
+        auto dense_b = as<ComplexDense>(b);
+        auto dense_x = as<ComplexDense>(x);
+        auto dense_alpha = as<RealDense>(alpha);
+        auto dense_beta = as<RealDense>(beta);
+        this->get_executor()->run(coo::make_advanced_spmv(
+            dense_alpha, as<Coo<remove_complex<ValueType>, IndexType>>(this),
+            dense_b->view_as_real().get(), dense_beta,
+            dense_x->view_as_real().get()));
+    }
 }
 
 
 template <typename ValueType, typename IndexType>
 void Coo<ValueType, IndexType>::apply2_impl(const LinOp *b, LinOp *x) const
 {
-    using Dense = Dense<ValueType>;
-    this->get_executor()->run(
-        coo::make_spmv2(this, as<Dense>(b), as<Dense>(x)));
+    using ComplexDense = Dense<to_complex<ValueType>>;
+
+    if (dynamic_cast<const Dense<ValueType> *>(b)) {
+        this->get_executor()->run(coo::make_spmv2(this, as<Dense<ValueType>>(b),
+                                                  as<Dense<ValueType>>(x)));
+    } else {
+        auto dense_b = as<ComplexDense>(b);
+        auto dense_x = as<ComplexDense>(x);
+        this->get_executor()->run(coo::make_spmv2(
+            as<Coo<remove_complex<ValueType>, IndexType>>(this),
+            dense_b->view_as_real().get(), dense_x->view_as_real().get()));
+    }
 }
 
 
@@ -105,9 +138,21 @@ template <typename ValueType, typename IndexType>
 void Coo<ValueType, IndexType>::apply2_impl(const LinOp *alpha, const LinOp *b,
                                             LinOp *x) const
 {
-    using Dense = Dense<ValueType>;
-    this->get_executor()->run(coo::make_advanced_spmv2(
-        as<Dense>(alpha), this, as<Dense>(b), as<Dense>(x)));
+    using ComplexDense = Dense<to_complex<ValueType>>;
+    using RealDense = Dense<remove_complex<ValueType>>;
+
+    if (dynamic_cast<const Dense<ValueType> *>(b)) {
+        this->get_executor()->run(coo::make_advanced_spmv2(
+            as<Dense<ValueType>>(alpha), this, as<Dense<ValueType>>(b),
+            as<Dense<ValueType>>(x)));
+    } else {
+        auto dense_b = as<ComplexDense>(b);
+        auto dense_x = as<ComplexDense>(x);
+        auto dense_alpha = as<RealDense>(alpha);
+        this->get_executor()->run(coo::make_advanced_spmv2(
+            dense_alpha, as<Coo<remove_complex<ValueType>, IndexType>>(this),
+            dense_b->view_as_real().get(), dense_x->view_as_real().get()));
+    }
 }
 
 
diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp
index 32431850132..67690eeac31 100644
--- a/core/matrix/csr.cpp
+++ b/core/matrix/csr.cpp
@@ -94,7 +94,7 @@ GKO_REGISTER_OPERATION(outplace_absolute_array,
 template <typename ValueType, typename IndexType>
 void Csr<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    using Dense = Dense<ValueType>;
+    using ComplexDense = Dense<to_complex<ValueType>>;
     using TCsr = Csr<ValueType, IndexType>;
     if (auto b_csr = dynamic_cast<const TCsr *>(b)) {
         // if b is a CSR matrix, we compute a SpGeMM
@@ -102,8 +102,16 @@ void Csr<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
         this->get_executor()->run(csr::make_spgemm(this, b_csr, x_csr));
     } else {
         // otherwise we assume that b is dense and compute a SpMV/SpMM
-        this->get_executor()->run(
-            csr::make_spmv(this, as<Dense>(b), as<Dense>(x)));
+        if (dynamic_cast<const Dense<ValueType> *>(b)) {
+            this->get_executor()->run(csr::make_spmv(
+                this, as<Dense<ValueType>>(b), as<Dense<ValueType>>(x)));
+        } else {
+            auto dense_b = as<ComplexDense>(b);
+            auto dense_x = as<ComplexDense>(x);
+            this->get_executor()->run(csr::make_spmv(
+                as<Csr<remove_complex<ValueType>, IndexType>>(this),
+                dense_b->view_as_real().get(), dense_x->view_as_real().get()));
+        }
     }
 }
 
@@ -112,26 +120,40 @@ template <typename ValueType, typename IndexType>
 void Csr<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                            const LinOp *beta, LinOp *x) const
 {
-    using Dense = Dense<ValueType>;
+    using ComplexDense = Dense<to_complex<ValueType>>;
+    using RealDense = Dense<remove_complex<ValueType>>;
     using TCsr = Csr<ValueType, IndexType>;
     if (auto b_csr = dynamic_cast<const TCsr *>(b)) {
         // if b is a CSR matrix, we compute a SpGeMM
         auto x_csr = as<TCsr>(x);
         auto x_copy = x_csr->clone();
-        this->get_executor()->run(
-            csr::make_advanced_spgemm(as<Dense>(alpha), this, b_csr,
-                                      as<Dense>(beta), x_copy.get(), x_csr));
+        this->get_executor()->run(csr::make_advanced_spgemm(
+            as<Dense<ValueType>>(alpha), this, b_csr,
+            as<Dense<ValueType>>(beta), x_copy.get(), x_csr));
     } else if (dynamic_cast<const Identity<ValueType> *>(b)) {
         // if b is an identity matrix, we compute an SpGEAM
         auto x_csr = as<TCsr>(x);
         auto x_copy = x_csr->clone();
-        this->get_executor()->run(csr::make_spgeam(
-            as<Dense>(alpha), this, as<Dense>(beta), lend(x_copy), x_csr));
+        this->get_executor()->run(
+            csr::make_spgeam(as<Dense<ValueType>>(alpha), this,
+                             as<Dense<ValueType>>(beta), lend(x_copy), x_csr));
     } else {
         // otherwise we assume that b is dense and compute a SpMV/SpMM
-        this->get_executor()->run(
-            csr::make_advanced_spmv(as<Dense>(alpha), this, as<Dense>(b),
-                                    as<Dense>(beta), as<Dense>(x)));
+        if (dynamic_cast<const Dense<ValueType> *>(b)) {
+            this->get_executor()->run(csr::make_advanced_spmv(
+                as<Dense<ValueType>>(alpha), this, as<Dense<ValueType>>(b),
+                as<Dense<ValueType>>(beta), as<Dense<ValueType>>(x)));
+        } else {
+            auto dense_b = as<ComplexDense>(b);
+            auto dense_x = as<ComplexDense>(x);
+            auto dense_alpha = as<RealDense>(alpha);
+            auto dense_beta = as<RealDense>(beta);
+            this->get_executor()->run(csr::make_advanced_spmv(
+                dense_alpha,
+                as<Csr<remove_complex<ValueType>, IndexType>>(this),
+                dense_b->view_as_real().get(), dense_beta,
+                dense_x->view_as_real().get()));
+        }
     }
 }
 
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 83da73f9960..4ce43777f4b 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -220,11 +220,11 @@ void Dense<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         this->get_executor()->run(dense::make_simple_apply(
             this, as<Dense<ValueType>>(b), as<Dense<ValueType>>(x)));
     } else {
-        auto dense_b = as<const Dense<to_complex<ValueType>>>(b);
+        auto dense_b = as<Dense<to_complex<ValueType>>>(b);
         auto dense_x = as<Dense<to_complex<ValueType>>>(x);
-        this->get_executor()->run(
-            dense::make_simple_apply(this, dense_b->view_as_real().get(),
-                                     dense_x->view_as_real().get()));
+        this->get_executor()->run(dense::make_simple_apply(
+            as<Dense<remove_complex<ValueType>>>(this),
+            dense_b->view_as_real().get(), dense_x->view_as_real().get()));
     }
 }
 
@@ -233,9 +233,20 @@ template <typename ValueType>
 void Dense<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                   const LinOp *beta, LinOp *x) const
 {
-    this->get_executor()->run(dense::make_apply(
-        as<Dense<ValueType>>(alpha), this, as<Dense<ValueType>>(b),
-        as<Dense<ValueType>>(beta), as<Dense<ValueType>>(x)));
+    if (dynamic_cast<const Dense<ValueType> *>(b)) {
+        this->get_executor()->run(dense::make_apply(
+            as<Dense<ValueType>>(alpha), this, as<Dense<ValueType>>(b),
+            as<Dense<ValueType>>(beta), as<Dense<ValueType>>(x)));
+    } else {
+        auto dense_b = as<Dense<to_complex<ValueType>>>(b);
+        auto dense_x = as<Dense<to_complex<ValueType>>>(x);
+        auto dense_alpha = as<Dense<remove_complex<ValueType>>>(alpha);
+        auto dense_beta = as<Dense<remove_complex<ValueType>>>(beta);
+        this->get_executor()->run(dense::make_apply(
+            dense_alpha, as<Dense<remove_complex<ValueType>>>(this),
+            dense_b->view_as_real().get(), dense_beta,
+            dense_x->view_as_real().get()));
+    }
 }
 
 
diff --git a/core/matrix/ell.cpp b/core/matrix/ell.cpp
index c802fe917e5..ee1c0fb6d6a 100644
--- a/core/matrix/ell.cpp
+++ b/core/matrix/ell.cpp
@@ -101,8 +101,18 @@ size_type calculate_max_nnz_per_row(
 template <typename ValueType, typename IndexType>
 void Ell<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    using Dense = Dense<ValueType>;
-    this->get_executor()->run(ell::make_spmv(this, as<Dense>(b), as<Dense>(x)));
+    using ComplexDense = Dense<to_complex<ValueType>>;
+
+    if (dynamic_cast<const Dense<ValueType> *>(b)) {
+        this->get_executor()->run(ell::make_spmv(this, as<Dense<ValueType>>(b),
+                                                 as<Dense<ValueType>>(x)));
+    } else {
+        auto dense_b = as<ComplexDense>(b);
+        auto dense_x = as<ComplexDense>(x);
+        this->get_executor()->run(ell::make_spmv(
+            as<Ell<remove_complex<ValueType>, IndexType>>(this),
+            dense_b->view_as_real().get(), dense_x->view_as_real().get()));
+    }
 }
 
 
@@ -110,9 +120,23 @@ template <typename ValueType, typename IndexType>
 void Ell<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                            const LinOp *beta, LinOp *x) const
 {
-    using Dense = Dense<ValueType>;
-    this->get_executor()->run(ell::make_advanced_spmv(
-        as<Dense>(alpha), this, as<Dense>(b), as<Dense>(beta), as<Dense>(x)));
+    using ComplexDense = Dense<to_complex<ValueType>>;
+    using RealDense = Dense<remove_complex<ValueType>>;
+
+    if (dynamic_cast<const Dense<ValueType> *>(b)) {
+        this->get_executor()->run(ell::make_advanced_spmv(
+            as<Dense<ValueType>>(alpha), this, as<Dense<ValueType>>(b),
+            as<Dense<ValueType>>(beta), as<Dense<ValueType>>(x)));
+    } else {
+        auto dense_b = as<ComplexDense>(b);
+        auto dense_x = as<ComplexDense>(x);
+        auto dense_alpha = as<RealDense>(alpha);
+        auto dense_beta = as<RealDense>(beta);
+        this->get_executor()->run(ell::make_advanced_spmv(
+            dense_alpha, as<Ell<remove_complex<ValueType>, IndexType>>(this),
+            dense_b->view_as_real().get(), dense_beta,
+            dense_x->view_as_real().get()));
+    }
 }
 
 
diff --git a/core/matrix/sellp.cpp b/core/matrix/sellp.cpp
index 94c30544832..fb9a24f548a 100644
--- a/core/matrix/sellp.cpp
+++ b/core/matrix/sellp.cpp
@@ -112,9 +112,18 @@ size_type calculate_total_cols(const matrix_data<ValueType, IndexType> &data,
 template <typename ValueType, typename IndexType>
 void Sellp<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    using Dense = Dense<ValueType>;
-    this->get_executor()->run(
-        sellp::make_spmv(this, as<Dense>(b), as<Dense>(x)));
+    using ComplexDense = Dense<to_complex<ValueType>>;
+
+    if (dynamic_cast<const Dense<ValueType> *>(b)) {
+        this->get_executor()->run(sellp::make_spmv(
+            this, as<Dense<ValueType>>(b), as<Dense<ValueType>>(x)));
+    } else {
+        auto dense_b = as<ComplexDense>(b);
+        auto dense_x = as<ComplexDense>(x);
+        this->get_executor()->run(sellp::make_spmv(
+            as<Sellp<remove_complex<ValueType>, IndexType>>(this),
+            dense_b->view_as_real().get(), dense_x->view_as_real().get()));
+    }
 }
 
 
@@ -122,9 +131,23 @@ template <typename ValueType, typename IndexType>
 void Sellp<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                              const LinOp *beta, LinOp *x) const
 {
-    using Dense = Dense<ValueType>;
-    this->get_executor()->run(sellp::make_advanced_spmv(
-        as<Dense>(alpha), this, as<Dense>(b), as<Dense>(beta), as<Dense>(x)));
+    using ComplexDense = Dense<to_complex<ValueType>>;
+    using RealDense = Dense<remove_complex<ValueType>>;
+
+    if (dynamic_cast<const Dense<ValueType> *>(b)) {
+        this->get_executor()->run(sellp::make_advanced_spmv(
+            as<Dense<ValueType>>(alpha), this, as<Dense<ValueType>>(b),
+            as<Dense<ValueType>>(beta), as<Dense<ValueType>>(x)));
+    } else {
+        auto dense_b = as<ComplexDense>(b);
+        auto dense_x = as<ComplexDense>(x);
+        auto dense_alpha = as<RealDense>(alpha);
+        auto dense_beta = as<RealDense>(beta);
+        this->get_executor()->run(sellp::make_advanced_spmv(
+            dense_alpha, as<Sellp<remove_complex<ValueType>, IndexType>>(this),
+            dense_b->view_as_real().get(), dense_beta,
+            dense_x->view_as_real().get()));
+    }
 }
 
 
diff --git a/core/test/matrix/dense.cpp b/core/test/matrix/dense.cpp
index dad8c64cfa2..bac536ab9c9 100644
--- a/core/test/matrix/dense.cpp
+++ b/core/test/matrix/dense.cpp
@@ -328,4 +328,40 @@ TYPED_TEST(Dense, CanCreateSubmatrixWithStride)
 }
 
 
+TYPED_TEST(Dense, CanBeViewedAsReal)
+{
+    using value_type = typename TestFixture::value_type;
+    using real_type = gko::remove_complex<value_type>;
+    auto real_view = this->mtx->view_as_real();
+
+    if (gko::is_complex<value_type>()) {
+        EXPECT_EQ(real_view->get_size()[0], this->mtx->get_size()[0]);
+        EXPECT_EQ(real_view->get_size()[1], 2 * this->mtx->get_size()[1]);
+        EXPECT_EQ(real_view->get_stride(), 2 * this->mtx->get_stride());
+        EXPECT_EQ(real_view->at(0, 0), real_type{1.0});
+        EXPECT_EQ(real_view->at(0, 1), real_type{0.0});
+        EXPECT_EQ(real_view->at(0, 2), real_type{2.0});
+        EXPECT_EQ(real_view->at(0, 3), real_type{0.0});
+        EXPECT_EQ(real_view->at(0, 4), real_type{3.0});
+        EXPECT_EQ(real_view->at(0, 5), real_type{0.0});
+        EXPECT_EQ(real_view->at(1, 0), real_type{1.5});
+        EXPECT_EQ(real_view->at(1, 1), real_type{0.0});
+        EXPECT_EQ(real_view->at(1, 2), real_type{2.5});
+        EXPECT_EQ(real_view->at(1, 3), real_type{0.0});
+        EXPECT_EQ(real_view->at(1, 4), real_type{3.5});
+        EXPECT_EQ(real_view->at(1, 5), real_type{0.0});
+    } else {
+        EXPECT_EQ(real_view->get_size()[0], this->mtx->get_size()[0]);
+        EXPECT_EQ(real_view->get_size()[1], this->mtx->get_size()[1]);
+        EXPECT_EQ(real_view->get_stride(), this->mtx->get_stride());
+        EXPECT_EQ(real_view->at(0, 0), real_type{1.0});
+        EXPECT_EQ(real_view->at(0, 1), real_type{2.0});
+        EXPECT_EQ(real_view->at(0, 2), real_type{3.0});
+        EXPECT_EQ(real_view->at(1, 0), real_type{1.5});
+        EXPECT_EQ(real_view->at(1, 1), real_type{2.5});
+        EXPECT_EQ(real_view->at(1, 2), real_type{3.5});
+    }
+}
+
+
 }  // namespace
diff --git a/cuda/test/matrix/coo_kernels.cpp b/cuda/test/matrix/coo_kernels.cpp
index 6815a9d3df1..fd16a583181 100644
--- a/cuda/test/matrix/coo_kernels.cpp
+++ b/cuda/test/matrix/coo_kernels.cpp
@@ -58,6 +58,7 @@ class Coo : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Coo<>;
     using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Coo() : rand_engine(42) {}
 
@@ -75,9 +76,10 @@ class Coo : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Vec> gen_mtx(int num_rows, int num_cols)
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols)
     {
-        return gko::test::generate_random_matrix<Vec>(
+        return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols, std::uniform_int_distribution<>(1, num_cols),
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
     }
@@ -231,6 +233,54 @@ TEST_F(Coo, AdvancedApplyAddToLargeDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+TEST_F(Coo, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Coo, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Coo, ApplyAddToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply2(alpha.get(), complex_b.get(), complex_x.get());
+    dmtx->apply2(dalpha.get(), dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Coo, ConvertToDenseIsEquivalentToRef)
 {
diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index 4205f5c5d97..d9806fe42dc 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -407,6 +407,38 @@ TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
     ASSERT_TRUE(db->is_sorted_by_column_index());
 }
 
+TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>());
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Csr, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>());
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Csr, TransposeIsEquivalentToRef)
 {
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index e8732c13136..a30c707b942 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -320,6 +320,38 @@ TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(25, 1);
+    auto dcomplex_b = ComplexMtx::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(65, 1);
+    auto dcomplex_x = ComplexMtx::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(complex_b.get(), complex_x.get());
+    dx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(25, 1);
+    auto dcomplex_b = ComplexMtx::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(65, 1);
+    auto dcomplex_x = ComplexMtx::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Dense, IsTransposable)
 {
diff --git a/cuda/test/matrix/ell_kernels.cpp b/cuda/test/matrix/ell_kernels.cpp
index ce48a80e634..4fab3b064aa 100644
--- a/cuda/test/matrix/ell_kernels.cpp
+++ b/cuda/test/matrix/ell_kernels.cpp
@@ -59,6 +59,7 @@ class Ell : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Ell<>;
     using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Ell() : rand_engine(42) {}
 
@@ -76,9 +77,10 @@ class Ell : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Vec> gen_mtx(int num_rows, int num_cols)
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols)
     {
-        return gko::test::generate_random_matrix<Vec>(
+        return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols, std::uniform_int_distribution<>(1, num_cols),
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
     }
@@ -278,6 +280,38 @@ TEST_F(Ell, AdvancedApplyOnSmallMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Ell, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Ell, ConvertToDenseIsEquivalentToRef)
 {
diff --git a/cuda/test/matrix/hybrid_kernels.cpp b/cuda/test/matrix/hybrid_kernels.cpp
index 366686f27a0..4f752229812 100644
--- a/cuda/test/matrix/hybrid_kernels.cpp
+++ b/cuda/test/matrix/hybrid_kernels.cpp
@@ -57,6 +57,7 @@ class Hybrid : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Hybrid<>;
     using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Hybrid() : rand_engine(42) {}
 
@@ -74,9 +75,11 @@ class Hybrid : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Vec> gen_mtx(int num_rows, int num_cols, int min_nnz_row)
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row)
     {
-        return gko::test::generate_random_matrix<Vec>(
+        return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(min_nnz_row, num_cols),
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
@@ -180,6 +183,38 @@ TEST_F(Hybrid, AdvancedApplyToDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+TEST_F(Hybrid, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Hybrid, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Hybrid, CountNonzerosIsEquivalentToRef)
 {
diff --git a/cuda/test/matrix/sellp_kernels.cpp b/cuda/test/matrix/sellp_kernels.cpp
index 7a00b63d0c1..f8fa4f09a58 100644
--- a/cuda/test/matrix/sellp_kernels.cpp
+++ b/cuda/test/matrix/sellp_kernels.cpp
@@ -59,6 +59,7 @@ class Sellp : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Sellp<>;
     using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Sellp() : rand_engine(42) {}
 
@@ -76,9 +77,10 @@ class Sellp : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Vec> gen_mtx(int num_rows, int num_cols)
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols)
     {
-        return gko::test::generate_random_matrix<Vec>(
+        return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols, std::uniform_int_distribution<>(1, num_cols),
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
     }
@@ -257,6 +259,38 @@ TEST_F(Sellp,
     GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
 }
 
+TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_matrix();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_matrix();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 {
diff --git a/hip/test/matrix/coo_kernels.hip.cpp b/hip/test/matrix/coo_kernels.hip.cpp
index 6d0a0935cd6..57758ba283c 100644
--- a/hip/test/matrix/coo_kernels.hip.cpp
+++ b/hip/test/matrix/coo_kernels.hip.cpp
@@ -58,6 +58,7 @@ class Coo : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Coo<>;
     using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Coo() : rand_engine(42) {}
 
@@ -75,9 +76,10 @@ class Coo : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Vec> gen_mtx(int num_rows, int num_cols)
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols)
     {
-        return gko::test::generate_random_matrix<Vec>(
+        return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols, std::uniform_int_distribution<>(1, num_cols),
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
     }
@@ -231,6 +233,54 @@ TEST_F(Coo, AdvancedApplyAddToLargeDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+TEST_F(Coo, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Coo, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Coo, ApplyAddToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply2(alpha.get(), complex_b.get(), complex_x.get());
+    dmtx->apply2(dalpha.get(), dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Coo, ConvertToDenseIsEquivalentToRef)
 {
diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index e80eaa940dc..6d846d8f205 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -406,6 +406,38 @@ TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
     ASSERT_TRUE(db->is_sorted_by_column_index());
 }
 
+TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Csr, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Csr, TransposeIsEquivalentToRef)
 {
@@ -429,7 +461,8 @@ TEST_F(Csr, ConjugateTransposeIsEquivalentToRef)
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(d_ctrans.get()),
                         static_cast<Mtx *>(ctrans.get()), 0.0);
-    ASSERT_TRUE(static_cast<Mtx *>(d_ctrans.get())->is_sorted_by_column_index());
+    ASSERT_TRUE(
+        static_cast<Mtx *>(d_ctrans.get())->is_sorted_by_column_index());
 }
 
 
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index c14809ed79e..be72bc9deb8 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -314,6 +314,38 @@ TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(25, 1);
+    auto dcomplex_b = ComplexMtx::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(65, 1);
+    auto dcomplex_x = ComplexMtx::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(complex_b.get(), complex_x.get());
+    dx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(25, 1);
+    auto dcomplex_b = ComplexMtx::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(65, 1);
+    auto dcomplex_x = ComplexMtx::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Dense, IsTransposable)
 {
diff --git a/hip/test/matrix/ell_kernels.hip.cpp b/hip/test/matrix/ell_kernels.hip.cpp
index 74d22dbb738..e076fe90143 100644
--- a/hip/test/matrix/ell_kernels.hip.cpp
+++ b/hip/test/matrix/ell_kernels.hip.cpp
@@ -59,6 +59,7 @@ class Ell : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Ell<>;
     using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Ell() : rand_engine(42) {}
 
@@ -76,9 +77,10 @@ class Ell : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Vec> gen_mtx(int num_rows, int num_cols)
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols)
     {
-        return gko::test::generate_random_matrix<Vec>(
+        return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols, std::uniform_int_distribution<>(1, num_cols),
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
     }
@@ -278,6 +280,38 @@ TEST_F(Ell, AdvancedApplyOnSmallMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Ell, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Ell, ConvertToDenseIsEquivalentToRef)
 {
diff --git a/hip/test/matrix/hybrid_kernels.hip.cpp b/hip/test/matrix/hybrid_kernels.hip.cpp
index f7c60b59b3f..49b6b12aa93 100644
--- a/hip/test/matrix/hybrid_kernels.hip.cpp
+++ b/hip/test/matrix/hybrid_kernels.hip.cpp
@@ -57,6 +57,7 @@ class Hybrid : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Hybrid<>;
     using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Hybrid() : rand_engine(42) {}
 
@@ -74,9 +75,11 @@ class Hybrid : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Vec> gen_mtx(int num_rows, int num_cols, int min_nnz_row)
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row)
     {
-        return gko::test::generate_random_matrix<Vec>(
+        return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(min_nnz_row, num_cols),
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
@@ -180,6 +183,38 @@ TEST_F(Hybrid, AdvancedApplyToDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+TEST_F(Hybrid, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Hybrid, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Hybrid, CountNonzerosIsEquivalentToRef)
 {
diff --git a/hip/test/matrix/sellp_kernels.hip.cpp b/hip/test/matrix/sellp_kernels.hip.cpp
index 6c99519b71d..d49b4fa0e5f 100644
--- a/hip/test/matrix/sellp_kernels.hip.cpp
+++ b/hip/test/matrix/sellp_kernels.hip.cpp
@@ -58,6 +58,7 @@ class Sellp : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Sellp<>;
     using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Sellp() : rand_engine(42) {}
 
@@ -75,9 +76,10 @@ class Sellp : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Vec> gen_mtx(int num_rows, int num_cols)
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols)
     {
-        return gko::test::generate_random_matrix<Vec>(
+        return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols, std::uniform_int_distribution<>(1, num_cols),
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
     }
@@ -256,6 +258,38 @@ TEST_F(Sellp,
     GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
 }
 
+TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_matrix();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_matrix();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 {
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index c3de3caa48c..58d71ea6a28 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -451,9 +451,9 @@ class Dense
         const auto num_rows = this->get_size()[0];
         const bool complex = is_complex<ValueType>();
         const auto num_cols =
-            complex ? this->get_size()[1] : 2 * this->get_size()[1];
+            complex ? 2 * this->get_size()[1] : this->get_size()[1];
         const auto stride =
-            complex ? this->get_stride() : 2 * this->get_stride();
+            complex ? 2 * this->get_stride() : this->get_stride();
 
         return Dense<remove_complex<ValueType>>::create(
             this->get_executor(), dim<2>{num_rows, num_cols},
@@ -467,21 +467,21 @@ class Dense
     /*
      * Create a constant real view of the (potentially) complex original matrix.
      */
-    std::unique_ptr<Dense<remove_complex<ValueType>>> view_as_real() const
+    std::unique_ptr<const Dense<remove_complex<ValueType>>> view_as_real() const
     {
         const auto num_rows = this->get_size()[0];
         const bool complex = is_complex<ValueType>();
         const auto num_cols =
-            complex ? this->get_size()[1] : 2 * this->get_size()[1];
+            complex ? 2 * this->get_size()[1] : this->get_size()[1];
         const auto stride =
-            complex ? this->get_stride() : 2 * this->get_stride();
+            complex ? 2 * this->get_stride() : this->get_stride();
 
         return Dense<remove_complex<ValueType>>::create(
             this->get_executor(), dim<2>{num_rows, num_cols},
             Array<remove_complex<ValueType>>::view(
                 this->get_executor(), num_rows * stride,
-                const_cast<remove_complex<ValueType *>>(
-                    reinterpret_cast<remove_complex<const ValueType> *>(
+                const_cast<remove_complex<ValueType> *>(
+                    reinterpret_cast<const remove_complex<ValueType> *>(
                         this->get_const_values()))),
             stride);
     }
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 4c44166afff..755e7753379 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -55,7 +55,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/polymorphic_object.hpp>
 #include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/base/range_accessors.hpp>
-#include <ginkgo/core/base/readable_data.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/base/utils.hpp>
diff --git a/omp/test/matrix/coo_kernels.cpp b/omp/test/matrix/coo_kernels.cpp
index 5139e1c1334..449661b6d93 100644
--- a/omp/test/matrix/coo_kernels.cpp
+++ b/omp/test/matrix/coo_kernels.cpp
@@ -58,6 +58,7 @@ class Coo : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Coo<>;
     using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Coo() : rand_engine(42) {}
 
@@ -74,9 +75,11 @@ class Coo : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Vec> gen_mtx(int num_rows, int num_cols, int min_nnz_row)
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row)
     {
-        return gko::test::generate_random_matrix<Vec>(
+        return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(min_nnz_row, num_cols),
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
@@ -209,6 +212,54 @@ TEST_F(Coo, AdvancedApplyAddToDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+TEST_F(Coo, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Coo, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Coo, ApplyAddToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply2(alpha.get(), complex_b.get(), complex_x.get());
+    dmtx->apply2(dalpha.get(), dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Coo, ConvertToCsrIsEquivalentToRef)
 {
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index b9da8e7e81e..29f0c1f41f1 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -290,6 +290,38 @@ TEST_F(Csr, AdvancedApplyToDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data(3);
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Csr, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data(3);
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Csr, TransposeIsEquivalentToRef)
 {
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index b9af3ec1027..1519ce0af98 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -332,6 +332,38 @@ TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(25, 1);
+    auto dcomplex_b = ComplexMtx::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(40, 1);
+    auto dcomplex_x = ComplexMtx::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(complex_b.get(), complex_x.get());
+    dx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(25, 1);
+    auto dcomplex_b = ComplexMtx::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(40, 1);
+    auto dcomplex_x = ComplexMtx::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Dense, ConvertToCooIsEquivalentToRef)
 {
diff --git a/omp/test/matrix/ell_kernels.cpp b/omp/test/matrix/ell_kernels.cpp
index 768cb87822e..a371c564289 100644
--- a/omp/test/matrix/ell_kernels.cpp
+++ b/omp/test/matrix/ell_kernels.cpp
@@ -57,6 +57,7 @@ class Ell : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Ell<>;
     using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Ell() : rand_engine(42) {}
 
@@ -73,9 +74,11 @@ class Ell : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Vec> gen_mtx(int num_rows, int num_cols, int min_nnz_row)
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row)
     {
-        return gko::test::generate_random_matrix<Vec>(
+        return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(min_nnz_row, num_cols),
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
@@ -206,6 +209,38 @@ TEST_F(Ell, AdvancedApplyWithPaddingToDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data(300, 600);
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Ell, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data(300, 600);
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Ell, CountNonzerosIsEquivalentToRef)
 {
diff --git a/omp/test/matrix/hybrid_kernels.cpp b/omp/test/matrix/hybrid_kernels.cpp
index 948c78060ca..dfb8e6ff7f1 100644
--- a/omp/test/matrix/hybrid_kernels.cpp
+++ b/omp/test/matrix/hybrid_kernels.cpp
@@ -57,6 +57,7 @@ class Hybrid : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Hybrid<>;
     using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Hybrid() : rand_engine(42) {}
 
@@ -73,9 +74,11 @@ class Hybrid : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Vec> gen_mtx(int num_rows, int num_cols, int min_nnz_row)
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row)
     {
-        return gko::test::generate_random_matrix<Vec>(
+        return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(min_nnz_row, num_cols),
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
@@ -179,6 +182,38 @@ TEST_F(Hybrid, AdvancedApplyToDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+TEST_F(Hybrid, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Hybrid, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Hybrid, CountNonzerosIsEquivalentToRef)
 {
diff --git a/omp/test/matrix/sellp_kernels.cpp b/omp/test/matrix/sellp_kernels.cpp
index e776d9c88c1..848b24217d4 100644
--- a/omp/test/matrix/sellp_kernels.cpp
+++ b/omp/test/matrix/sellp_kernels.cpp
@@ -56,6 +56,7 @@ class Sellp : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Sellp<>;
     using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Sellp() : rand_engine(42) {}
 
@@ -72,9 +73,10 @@ class Sellp : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Vec> gen_mtx(int num_rows, int num_cols)
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols)
     {
-        return gko::test::generate_random_matrix<Vec>(
+        return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols, std::uniform_int_distribution<>(1, num_cols),
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
     }
@@ -213,6 +215,38 @@ TEST_F(Sellp,
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
 
 TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 {
diff --git a/reference/test/matrix/coo_kernels.cpp b/reference/test/matrix/coo_kernels.cpp
index d08ef5f9af9..7c6ec33b423 100644
--- a/reference/test/matrix/coo_kernels.cpp
+++ b/reference/test/matrix/coo_kernels.cpp
@@ -560,6 +560,93 @@ TYPED_TEST(Coo, OutplaceAbsolute)
 }
 
 
+TYPED_TEST(Coo, AppliesToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Mtx = typename TestFixture::Mtx;
+    using Vec = typename gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = Vec::create(exec, gko::dim<2>{2,2});
+    // clang-format on
+
+    this->mtx->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{13.0, 14.0}, complex_type{19.0, 20.0}},
+           {complex_type{10.0, 10.0}, complex_type{15.0, 15.0}}}),
+        0.0);
+}
+
+
+TYPED_TEST(Coo, AdvancedAppliesToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Mtx = typename TestFixture::Mtx;
+    using Scal = typename gko::matrix::Dense<value_type>;
+    using Vec = typename gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
+    auto alpha = gko::initialize<Scal>({-1.0}, this->exec);
+    auto beta = gko::initialize<Scal>({2.0}, this->exec);
+    // clang-format on
+
+    this->mtx->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{-11.0, -14.0}, complex_type{-15.0, -18.0}},
+           {complex_type{-6.0, -6.0}, complex_type{-9.0, -9.0}}}),
+        0.0);
+}
+
+
+TYPED_TEST(Coo, ApplyAddsToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Mtx = typename TestFixture::Mtx;
+    using Scal = typename gko::matrix::Dense<value_type>;
+    using Vec = typename gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
+    auto alpha = gko::initialize<Scal>({-1.0}, this->exec);
+    // clang-format on
+
+    this->mtx->apply2(alpha.get(), b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{-12.0, -14.0}, complex_type{-17.0, -19.0}},
+           {complex_type{-8.0, -8.0}, complex_type{-12.0, -12.0}}}),
+        0.0);
+}
+
+
 template <typename ValueIndexType>
 class CooComplex : public ::testing::Test {
 protected:
diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp
index ac9ae9f64d8..3cedeba21d4 100644
--- a/reference/test/matrix/csr_kernels.cpp
+++ b/reference/test/matrix/csr_kernels.cpp
@@ -1335,6 +1335,63 @@ TYPED_TEST(Csr, OutplaceAbsolute)
 }
 
 
+TYPED_TEST(Csr, AppliesToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Mtx = typename TestFixture::Mtx;
+    using Vec = typename gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = Vec::create(exec, gko::dim<2>{2,2});
+    // clang-format on
+
+    this->mtx->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{13.0, 14.0}, complex_type{19.0, 20.0}},
+           {complex_type{10.0, 10.0}, complex_type{15.0, 15.0}}}),
+        0.0);
+}
+
+
+TYPED_TEST(Csr, AdvancedAppliesToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Mtx = typename TestFixture::Mtx;
+    using Scal = typename gko::matrix::Dense<value_type>;
+    using Vec = typename gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
+    auto alpha = gko::initialize<Scal>({-1.0}, this->exec);
+    auto beta = gko::initialize<Scal>({2.0}, this->exec);
+    // clang-format on
+
+    this->mtx->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{-11.0, -14.0}, complex_type{-15.0, -18.0}},
+           {complex_type{-6.0, -6.0}, complex_type{-9.0, -9.0}}}),
+        0.0);
+}
+
+
 template <typename ValueIndexType>
 class CsrComplex : public ::testing::Test {
 protected:
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 9ac4b9b448f..5e8a8ac48da 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -2095,6 +2095,62 @@ TYPED_TEST(Dense, OutplaceAbsoluteSubMatrix)
 }
 
 
+TYPED_TEST(Dense, AppliesToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Mtx = typename TestFixture::Mtx;
+    using Vec = typename gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = Vec::create(exec, gko::dim<2>{2,2});
+    // clang-format on
+
+    this->mtx1->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{14.0, 16.0}, complex_type{20.0, 22.0}},
+           {complex_type{17.0, 19.0}, complex_type{24.5, 26.5}}}),
+        0.0);
+}
+
+
+TYPED_TEST(Dense, AdvancedAppliesToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Mtx = typename TestFixture::Mtx;
+    using Vec = typename gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
+    auto alpha = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({2.0}, this->exec);
+    // clang-format on
+
+    this->mtx1->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{-12.0, -16.0}, complex_type{-16.0, -20.0}},
+           {complex_type{-13.0, -15.0}, complex_type{-18.5, -20.5}}}),
+        0.0);
+}
+
+
 template <typename T>
 class DenseComplex : public ::testing::Test {
 protected:
diff --git a/reference/test/matrix/ell_kernels.cpp b/reference/test/matrix/ell_kernels.cpp
index cc87f580f9e..b987001aed3 100644
--- a/reference/test/matrix/ell_kernels.cpp
+++ b/reference/test/matrix/ell_kernels.cpp
@@ -638,6 +638,63 @@ TYPED_TEST(Ell, OutplaceAbsolute)
 }
 
 
+TYPED_TEST(Ell, AppliesToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Mtx = typename TestFixture::Mtx;
+    using Vec = typename gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = Vec::create(exec, gko::dim<2>{2,2});
+    // clang-format on
+
+    this->mtx1->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{13.0, 14.0}, complex_type{19.0, 20.0}},
+           {complex_type{10.0, 10.0}, complex_type{15.0, 15.0}}}),
+        0.0);
+}
+
+
+TYPED_TEST(Ell, AdvancedAppliesToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Mtx = typename TestFixture::Mtx;
+    using Scal = typename gko::matrix::Dense<value_type>;
+    using Vec = typename gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
+    auto alpha = gko::initialize<Scal>({-1.0}, this->exec);
+    auto beta = gko::initialize<Scal>({2.0}, this->exec);
+    // clang-format on
+
+    this->mtx1->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{-11.0, -14.0}, complex_type{-15.0, -18.0}},
+           {complex_type{-6.0, -6.0}, complex_type{-9.0, -9.0}}}),
+        0.0);
+}
+
+
 template <typename ValueIndexType>
 class EllComplex : public ::testing::Test {
 protected:
diff --git a/reference/test/matrix/hybrid_kernels.cpp b/reference/test/matrix/hybrid_kernels.cpp
index 57125661cb9..8fa81df287e 100644
--- a/reference/test/matrix/hybrid_kernels.cpp
+++ b/reference/test/matrix/hybrid_kernels.cpp
@@ -687,6 +687,63 @@ TYPED_TEST(Hybrid, OutplaceAbsolute)
 }
 
 
+TYPED_TEST(Hybrid, AppliesToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Mtx = typename TestFixture::Mtx;
+    using Vec = typename gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = Vec::create(exec, gko::dim<2>{2,2});
+    // clang-format on
+
+    this->mtx1->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{13.0, 14.0}, complex_type{19.0, 20.0}},
+           {complex_type{10.0, 10.0}, complex_type{15.0, 15.0}}}),
+        0.0);
+}
+
+
+TYPED_TEST(Hybrid, AdvancedAppliesToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Mtx = typename TestFixture::Mtx;
+    using Scal = typename gko::matrix::Dense<value_type>;
+    using Vec = typename gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
+    auto alpha = gko::initialize<Scal>({-1.0}, this->exec);
+    auto beta = gko::initialize<Scal>({2.0}, this->exec);
+    // clang-format on
+
+    this->mtx1->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{-11.0, -14.0}, complex_type{-15.0, -18.0}},
+           {complex_type{-6.0, -6.0}, complex_type{-9.0, -9.0}}}),
+        0.0);
+}
+
+
 template <typename ValueIndexType>
 class HybridComplex : public ::testing::Test {
 protected:
diff --git a/reference/test/matrix/sellp_kernels.cpp b/reference/test/matrix/sellp_kernels.cpp
index 5c24e7afe3d..218609eda2c 100644
--- a/reference/test/matrix/sellp_kernels.cpp
+++ b/reference/test/matrix/sellp_kernels.cpp
@@ -641,6 +641,63 @@ TYPED_TEST(Sellp, OutplaceAbsolute)
 }
 
 
+TYPED_TEST(Sellp, AppliesToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Mtx = typename TestFixture::Mtx;
+    using Vec = typename gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = Vec::create(exec, gko::dim<2>{2,2});
+    // clang-format on
+
+    this->mtx1->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{13.0, 14.0}, complex_type{19.0, 20.0}},
+           {complex_type{10.0, 10.0}, complex_type{15.0, 15.0}}}),
+        0.0);
+}
+
+
+TYPED_TEST(Sellp, AdvancedAppliesToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Mtx = typename TestFixture::Mtx;
+    using Scal = typename gko::matrix::Dense<value_type>;
+    using Vec = typename gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<Vec>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
+    auto alpha = gko::initialize<Scal>({-1.0}, this->exec);
+    auto beta = gko::initialize<Scal>({2.0}, this->exec);
+    // clang-format on
+
+    this->mtx1->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{-11.0, -14.0}, complex_type{-15.0, -18.0}},
+           {complex_type{-6.0, -6.0}, complex_type{-9.0, -9.0}}}),
+        0.0);
+}
+
+
 template <typename ValueIndexType>
 class SellpComplex : public ::testing::Test {
 protected:

From 3a7c98f29b4a4e471d8c4061e75cd23056ff88ba Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Wed, 21 Oct 2020 10:05:34 +0200
Subject: [PATCH 0081/1008] fix hip/test/matrix/dense_kernels

---
 hip/test/matrix/dense_kernels.hip.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index be72bc9deb8..23548de4fe7 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -63,6 +63,7 @@ class Dense : public ::testing::Test {
     using Mtx = gko::matrix::Dense<vtype>;
     using NormVector = gko::matrix::Dense<gko::remove_complex<vtype>>;
     using Arr = gko::Array<itype>;
+    using ComplexMtx = gko::matrix::Dense<std::complex<vtype>>;
 
     Dense() : rand_engine(15) {}
 

From 94b3ae5994c2263a0b09e19b14273f0f41bd8a7f Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 22 Oct 2020 16:28:27 +0200
Subject: [PATCH 0082/1008] Apply suggestions from code review

---
 core/matrix/coo.cpp                    | 21 ++++++++-------------
 core/matrix/csr.cpp                    | 12 ++++--------
 core/matrix/dense.cpp                  | 11 ++++-------
 core/matrix/ell.cpp                    | 11 ++++-------
 core/matrix/sellp.cpp                  | 11 ++++-------
 core/test/matrix/dense.cpp             |  4 ++--
 cuda/test/matrix/coo_kernels.cpp       |  3 +++
 cuda/test/matrix/csr_kernels.cpp       |  2 ++
 cuda/test/matrix/dense_kernels.cpp     |  2 ++
 cuda/test/matrix/ell_kernels.cpp       |  2 ++
 cuda/test/matrix/hybrid_kernels.cpp    |  2 ++
 cuda/test/matrix/sellp_kernels.cpp     |  2 ++
 hip/test/matrix/coo_kernels.hip.cpp    |  3 +++
 hip/test/matrix/csr_kernels.hip.cpp    |  2 ++
 hip/test/matrix/dense_kernels.hip.cpp  |  2 ++
 hip/test/matrix/ell_kernels.hip.cpp    |  2 ++
 hip/test/matrix/hybrid_kernels.hip.cpp |  2 ++
 hip/test/matrix/sellp_kernels.hip.cpp  |  2 ++
 include/ginkgo/core/matrix/dense.hpp   | 11 ++++++++---
 omp/test/matrix/coo_kernels.cpp        |  3 +++
 omp/test/matrix/csr_kernels.cpp        |  2 ++
 omp/test/matrix/dense_kernels.cpp      |  2 ++
 omp/test/matrix/ell_kernels.cpp        |  2 ++
 omp/test/matrix/hybrid_kernels.cpp     |  2 ++
 omp/test/matrix/sellp_kernels.cpp      |  2 ++
 25 files changed, 73 insertions(+), 47 deletions(-)

diff --git a/core/matrix/coo.cpp b/core/matrix/coo.cpp
index 158addd6d6b..b256f4e4190 100644
--- a/core/matrix/coo.cpp
+++ b/core/matrix/coo.cpp
@@ -85,9 +85,8 @@ void Coo<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
     } else {
         auto dense_b = as<ComplexDense>(b);
         auto dense_x = as<ComplexDense>(x);
-        this->get_executor()->run(coo::make_spmv(
-            as<Coo<remove_complex<ValueType>, IndexType>>(this),
-            dense_b->view_as_real().get(), dense_x->view_as_real().get()));
+        this->apply(dense_b->create_real_view().get(),
+                    dense_x->create_real_view().get());
     }
 }
 
@@ -108,10 +107,8 @@ void Coo<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
         auto dense_x = as<ComplexDense>(x);
         auto dense_alpha = as<RealDense>(alpha);
         auto dense_beta = as<RealDense>(beta);
-        this->get_executor()->run(coo::make_advanced_spmv(
-            dense_alpha, as<Coo<remove_complex<ValueType>, IndexType>>(this),
-            dense_b->view_as_real().get(), dense_beta,
-            dense_x->view_as_real().get()));
+        this->apply(dense_alpha, dense_b->create_real_view().get(), dense_beta,
+                    dense_x->create_real_view().get());
     }
 }
 
@@ -127,9 +124,8 @@ void Coo<ValueType, IndexType>::apply2_impl(const LinOp *b, LinOp *x) const
     } else {
         auto dense_b = as<ComplexDense>(b);
         auto dense_x = as<ComplexDense>(x);
-        this->get_executor()->run(coo::make_spmv2(
-            as<Coo<remove_complex<ValueType>, IndexType>>(this),
-            dense_b->view_as_real().get(), dense_x->view_as_real().get()));
+        this->apply2(dense_b->create_real_view().get(),
+                     dense_x->create_real_view().get());
     }
 }
 
@@ -149,9 +145,8 @@ void Coo<ValueType, IndexType>::apply2_impl(const LinOp *alpha, const LinOp *b,
         auto dense_b = as<ComplexDense>(b);
         auto dense_x = as<ComplexDense>(x);
         auto dense_alpha = as<RealDense>(alpha);
-        this->get_executor()->run(coo::make_advanced_spmv2(
-            dense_alpha, as<Coo<remove_complex<ValueType>, IndexType>>(this),
-            dense_b->view_as_real().get(), dense_x->view_as_real().get()));
+        this->apply2(dense_alpha, dense_b->create_real_view().get(),
+                     dense_x->create_real_view().get());
     }
 }
 
diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp
index 67690eeac31..6a31b3bffae 100644
--- a/core/matrix/csr.cpp
+++ b/core/matrix/csr.cpp
@@ -108,9 +108,8 @@ void Csr<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
         } else {
             auto dense_b = as<ComplexDense>(b);
             auto dense_x = as<ComplexDense>(x);
-            this->get_executor()->run(csr::make_spmv(
-                as<Csr<remove_complex<ValueType>, IndexType>>(this),
-                dense_b->view_as_real().get(), dense_x->view_as_real().get()));
+            this->apply(dense_b->create_real_view().get(),
+                        dense_x->create_real_view().get());
         }
     }
 }
@@ -148,11 +147,8 @@ void Csr<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
             auto dense_x = as<ComplexDense>(x);
             auto dense_alpha = as<RealDense>(alpha);
             auto dense_beta = as<RealDense>(beta);
-            this->get_executor()->run(csr::make_advanced_spmv(
-                dense_alpha,
-                as<Csr<remove_complex<ValueType>, IndexType>>(this),
-                dense_b->view_as_real().get(), dense_beta,
-                dense_x->view_as_real().get()));
+            this->apply(dense_alpha, dense_b->create_real_view().get(),
+                        dense_beta, dense_x->create_real_view().get());
         }
     }
 }
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 4ce43777f4b..2e0b663966f 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -222,9 +222,8 @@ void Dense<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     } else {
         auto dense_b = as<Dense<to_complex<ValueType>>>(b);
         auto dense_x = as<Dense<to_complex<ValueType>>>(x);
-        this->get_executor()->run(dense::make_simple_apply(
-            as<Dense<remove_complex<ValueType>>>(this),
-            dense_b->view_as_real().get(), dense_x->view_as_real().get()));
+        this->apply(dense_b->create_real_view().get(),
+                    dense_x->create_real_view().get());
     }
 }
 
@@ -242,10 +241,8 @@ void Dense<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
         auto dense_x = as<Dense<to_complex<ValueType>>>(x);
         auto dense_alpha = as<Dense<remove_complex<ValueType>>>(alpha);
         auto dense_beta = as<Dense<remove_complex<ValueType>>>(beta);
-        this->get_executor()->run(dense::make_apply(
-            dense_alpha, as<Dense<remove_complex<ValueType>>>(this),
-            dense_b->view_as_real().get(), dense_beta,
-            dense_x->view_as_real().get()));
+        this->apply(dense_alpha, dense_b->create_real_view().get(), dense_beta,
+                    dense_x->create_real_view().get());
     }
 }
 
diff --git a/core/matrix/ell.cpp b/core/matrix/ell.cpp
index ee1c0fb6d6a..04eb5eab482 100644
--- a/core/matrix/ell.cpp
+++ b/core/matrix/ell.cpp
@@ -109,9 +109,8 @@ void Ell<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
     } else {
         auto dense_b = as<ComplexDense>(b);
         auto dense_x = as<ComplexDense>(x);
-        this->get_executor()->run(ell::make_spmv(
-            as<Ell<remove_complex<ValueType>, IndexType>>(this),
-            dense_b->view_as_real().get(), dense_x->view_as_real().get()));
+        this->apply(dense_b->create_real_view().get(),
+                    dense_x->create_real_view().get());
     }
 }
 
@@ -132,10 +131,8 @@ void Ell<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
         auto dense_x = as<ComplexDense>(x);
         auto dense_alpha = as<RealDense>(alpha);
         auto dense_beta = as<RealDense>(beta);
-        this->get_executor()->run(ell::make_advanced_spmv(
-            dense_alpha, as<Ell<remove_complex<ValueType>, IndexType>>(this),
-            dense_b->view_as_real().get(), dense_beta,
-            dense_x->view_as_real().get()));
+        this->apply(dense_alpha, dense_b->create_real_view().get(), dense_beta,
+                    dense_x->create_real_view().get());
     }
 }
 
diff --git a/core/matrix/sellp.cpp b/core/matrix/sellp.cpp
index fb9a24f548a..b34e6119e59 100644
--- a/core/matrix/sellp.cpp
+++ b/core/matrix/sellp.cpp
@@ -120,9 +120,8 @@ void Sellp<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
     } else {
         auto dense_b = as<ComplexDense>(b);
         auto dense_x = as<ComplexDense>(x);
-        this->get_executor()->run(sellp::make_spmv(
-            as<Sellp<remove_complex<ValueType>, IndexType>>(this),
-            dense_b->view_as_real().get(), dense_x->view_as_real().get()));
+        this->apply(dense_b->create_real_view().get(),
+                    dense_x->create_real_view().get());
     }
 }
 
@@ -143,10 +142,8 @@ void Sellp<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
         auto dense_x = as<ComplexDense>(x);
         auto dense_alpha = as<RealDense>(alpha);
         auto dense_beta = as<RealDense>(beta);
-        this->get_executor()->run(sellp::make_advanced_spmv(
-            dense_alpha, as<Sellp<remove_complex<ValueType>, IndexType>>(this),
-            dense_b->view_as_real().get(), dense_beta,
-            dense_x->view_as_real().get()));
+        this->apply(dense_alpha, dense_b->create_real_view().get(), dense_beta,
+                    dense_x->create_real_view().get());
     }
 }
 
diff --git a/core/test/matrix/dense.cpp b/core/test/matrix/dense.cpp
index bac536ab9c9..9439687d763 100644
--- a/core/test/matrix/dense.cpp
+++ b/core/test/matrix/dense.cpp
@@ -328,11 +328,11 @@ TYPED_TEST(Dense, CanCreateSubmatrixWithStride)
 }
 
 
-TYPED_TEST(Dense, CanBeViewedAsReal)
+TYPED_TEST(Dense, CanCreateRealView)
 {
     using value_type = typename TestFixture::value_type;
     using real_type = gko::remove_complex<value_type>;
-    auto real_view = this->mtx->view_as_real();
+    auto real_view = this->mtx->create_real_view();
 
     if (gko::is_complex<value_type>()) {
         EXPECT_EQ(real_view->get_size()[0], this->mtx->get_size()[0]);
diff --git a/cuda/test/matrix/coo_kernels.cpp b/cuda/test/matrix/coo_kernels.cpp
index fd16a583181..1c65df1b692 100644
--- a/cuda/test/matrix/coo_kernels.cpp
+++ b/cuda/test/matrix/coo_kernels.cpp
@@ -233,6 +233,7 @@ TEST_F(Coo, AdvancedApplyAddToLargeDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+
 TEST_F(Coo, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -249,6 +250,7 @@ TEST_F(Coo, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Coo, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -265,6 +267,7 @@ TEST_F(Coo, AdvancedApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Coo, ApplyAddToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index d9806fe42dc..9a4a9075933 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -407,6 +407,7 @@ TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
     ASSERT_TRUE(db->is_sorted_by_column_index());
 }
 
+
 TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>());
@@ -423,6 +424,7 @@ TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Csr, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>());
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index a30c707b942..1dee85ba561 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -320,6 +320,7 @@ TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+
 TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -336,6 +337,7 @@ TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/cuda/test/matrix/ell_kernels.cpp b/cuda/test/matrix/ell_kernels.cpp
index 4fab3b064aa..11ec8dbbea5 100644
--- a/cuda/test/matrix/ell_kernels.cpp
+++ b/cuda/test/matrix/ell_kernels.cpp
@@ -280,6 +280,7 @@ TEST_F(Ell, AdvancedApplyOnSmallMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+
 TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -296,6 +297,7 @@ TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Ell, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/cuda/test/matrix/hybrid_kernels.cpp b/cuda/test/matrix/hybrid_kernels.cpp
index 4f752229812..75ebab55443 100644
--- a/cuda/test/matrix/hybrid_kernels.cpp
+++ b/cuda/test/matrix/hybrid_kernels.cpp
@@ -183,6 +183,7 @@ TEST_F(Hybrid, AdvancedApplyToDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+
 TEST_F(Hybrid, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -199,6 +200,7 @@ TEST_F(Hybrid, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Hybrid, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/cuda/test/matrix/sellp_kernels.cpp b/cuda/test/matrix/sellp_kernels.cpp
index f8fa4f09a58..390c0de211b 100644
--- a/cuda/test/matrix/sellp_kernels.cpp
+++ b/cuda/test/matrix/sellp_kernels.cpp
@@ -259,6 +259,7 @@ TEST_F(Sellp,
     GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
 }
 
+
 TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_matrix();
@@ -275,6 +276,7 @@ TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_matrix();
diff --git a/hip/test/matrix/coo_kernels.hip.cpp b/hip/test/matrix/coo_kernels.hip.cpp
index 57758ba283c..b2dd2039926 100644
--- a/hip/test/matrix/coo_kernels.hip.cpp
+++ b/hip/test/matrix/coo_kernels.hip.cpp
@@ -233,6 +233,7 @@ TEST_F(Coo, AdvancedApplyAddToLargeDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+
 TEST_F(Coo, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -249,6 +250,7 @@ TEST_F(Coo, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Coo, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -265,6 +267,7 @@ TEST_F(Coo, AdvancedApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Coo, ApplyAddToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index 6d846d8f205..99e6bca6f2a 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -406,6 +406,7 @@ TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
     ASSERT_TRUE(db->is_sorted_by_column_index());
 }
 
+
 TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
@@ -422,6 +423,7 @@ TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Csr, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index 23548de4fe7..fb40a3819fa 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -315,6 +315,7 @@ TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+
 TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -331,6 +332,7 @@ TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/hip/test/matrix/ell_kernels.hip.cpp b/hip/test/matrix/ell_kernels.hip.cpp
index e076fe90143..d1eb395aec1 100644
--- a/hip/test/matrix/ell_kernels.hip.cpp
+++ b/hip/test/matrix/ell_kernels.hip.cpp
@@ -280,6 +280,7 @@ TEST_F(Ell, AdvancedApplyOnSmallMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+
 TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -296,6 +297,7 @@ TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Ell, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/hip/test/matrix/hybrid_kernels.hip.cpp b/hip/test/matrix/hybrid_kernels.hip.cpp
index 49b6b12aa93..31b35e1906c 100644
--- a/hip/test/matrix/hybrid_kernels.hip.cpp
+++ b/hip/test/matrix/hybrid_kernels.hip.cpp
@@ -183,6 +183,7 @@ TEST_F(Hybrid, AdvancedApplyToDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+
 TEST_F(Hybrid, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -199,6 +200,7 @@ TEST_F(Hybrid, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Hybrid, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/hip/test/matrix/sellp_kernels.hip.cpp b/hip/test/matrix/sellp_kernels.hip.cpp
index d49b4fa0e5f..36547a221da 100644
--- a/hip/test/matrix/sellp_kernels.hip.cpp
+++ b/hip/test/matrix/sellp_kernels.hip.cpp
@@ -258,6 +258,7 @@ TEST_F(Sellp,
     GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
 }
 
+
 TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_matrix();
@@ -274,6 +275,7 @@ TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_matrix();
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 58d71ea6a28..65113ec204b 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -445,8 +445,12 @@ class Dense
 
     /*
      * Create a real view of the (potentially) complex original matrix.
+     * If the original matrix is real, nothing changes. If the original matrix
+     * is complex, the result is created by viewing the complex matrix with as
+     * real with a reinterpret_cast with twice the number of columns and
+     * double the stride.
      */
-    std::unique_ptr<Dense<remove_complex<ValueType>>> view_as_real()
+    std::unique_ptr<Dense<remove_complex<ValueType>>> create_real_view()
     {
         const auto num_rows = this->get_size()[0];
         const bool complex = is_complex<ValueType>();
@@ -465,9 +469,10 @@ class Dense
     }
 
     /*
-     * Create a constant real view of the (potentially) complex original matrix.
+     * @copydoc create_real_view()
      */
-    std::unique_ptr<const Dense<remove_complex<ValueType>>> view_as_real() const
+    std::unique_ptr<const Dense<remove_complex<ValueType>>> create_real_view()
+        const
     {
         const auto num_rows = this->get_size()[0];
         const bool complex = is_complex<ValueType>();
diff --git a/omp/test/matrix/coo_kernels.cpp b/omp/test/matrix/coo_kernels.cpp
index 449661b6d93..fae58704ab2 100644
--- a/omp/test/matrix/coo_kernels.cpp
+++ b/omp/test/matrix/coo_kernels.cpp
@@ -212,6 +212,7 @@ TEST_F(Coo, AdvancedApplyAddToDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+
 TEST_F(Coo, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -228,6 +229,7 @@ TEST_F(Coo, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Coo, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -244,6 +246,7 @@ TEST_F(Coo, AdvancedApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Coo, ApplyAddToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index 29f0c1f41f1..6462720003f 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -290,6 +290,7 @@ TEST_F(Csr, AdvancedApplyToDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+
 TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(3);
@@ -306,6 +307,7 @@ TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Csr, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(3);
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index 1519ce0af98..06999748b9f 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -332,6 +332,7 @@ TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+
 TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -348,6 +349,7 @@ TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/omp/test/matrix/ell_kernels.cpp b/omp/test/matrix/ell_kernels.cpp
index a371c564289..2d11a61f68b 100644
--- a/omp/test/matrix/ell_kernels.cpp
+++ b/omp/test/matrix/ell_kernels.cpp
@@ -209,6 +209,7 @@ TEST_F(Ell, AdvancedApplyWithPaddingToDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+
 TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(300, 600);
@@ -225,6 +226,7 @@ TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Ell, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(300, 600);
diff --git a/omp/test/matrix/hybrid_kernels.cpp b/omp/test/matrix/hybrid_kernels.cpp
index dfb8e6ff7f1..a0dd242e1f7 100644
--- a/omp/test/matrix/hybrid_kernels.cpp
+++ b/omp/test/matrix/hybrid_kernels.cpp
@@ -182,6 +182,7 @@ TEST_F(Hybrid, AdvancedApplyToDenseMatrixIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+
 TEST_F(Hybrid, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -198,6 +199,7 @@ TEST_F(Hybrid, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Hybrid, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/omp/test/matrix/sellp_kernels.cpp b/omp/test/matrix/sellp_kernels.cpp
index 848b24217d4..802abdbdfdf 100644
--- a/omp/test/matrix/sellp_kernels.cpp
+++ b/omp/test/matrix/sellp_kernels.cpp
@@ -215,6 +215,7 @@ TEST_F(Sellp,
     GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
+
 TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -231,6 +232,7 @@ TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
 }
 
+
 TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();

From be99b5324c3f2d146681aeeaf31c2c113f60d647 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sun, 25 Oct 2020 16:32:12 +0100
Subject: [PATCH 0083/1008] add GPU permutation kernels

---
 common/matrix/csr_kernels.hpp.inc        | 117 +++++++++++++++++
 core/device_hooks/common_kernels.inc.cpp |  11 +-
 core/matrix/csr.cpp                      |  24 ++--
 core/matrix/csr_kernels.hpp              |  21 ++-
 cuda/matrix/csr_kernels.cu               |  75 ++++++++---
 cuda/test/matrix/csr_kernels.cpp         |  73 +++++++++++
 hip/matrix/csr_kernels.hip.cpp           |  80 +++++++++---
 hip/test/matrix/csr_kernels.hip.cpp      |  73 +++++++++++
 omp/matrix/csr_kernels.cpp               | 140 +++++++++-----------
 omp/test/matrix/csr_kernels.cpp          |  55 ++++----
 reference/matrix/csr_kernels.cpp         | 158 ++++++++++-------------
 11 files changed, 577 insertions(+), 250 deletions(-)

diff --git a/common/matrix/csr_kernels.hpp.inc b/common/matrix/csr_kernels.hpp.inc
index bb7e6b27b57..8f7e1a3f5a2 100644
--- a/common/matrix/csr_kernels.hpp.inc
+++ b/common/matrix/csr_kernels.hpp.inc
@@ -946,3 +946,120 @@ __global__ __launch_bounds__(default_block_size) void conjugate_kernel(
 
 
 }  //  namespace
+
+
+template <typename IndexType>
+__global__ __launch_bounds__(default_block_size) void inv_permutation_kernel(
+    size_type size, const IndexType *__restrict__ permutation,
+    IndexType *__restrict__ inv_permutation)
+{
+    auto tid = thread::get_thread_id_flat();
+    if (tid >= size) {
+        return;
+    }
+    inv_permutation[permutation[tid]] = tid;
+}
+
+
+template <typename ValueType, typename IndexType>
+__global__ __launch_bounds__(default_block_size) void col_permute_kernel(
+    size_type num_rows, size_type num_nonzeros,
+    const IndexType *__restrict__ permutation,
+    const IndexType *__restrict__ in_row_ptrs,
+    const IndexType *__restrict__ in_cols,
+    const ValueType *__restrict__ in_vals, IndexType *__restrict__ out_row_ptrs,
+    IndexType *__restrict__ out_cols, ValueType *__restrict__ out_vals)
+{
+    auto tid = thread::get_thread_id_flat();
+    if (tid < num_nonzeros) {
+        out_cols[tid] = permutation[in_cols[tid]];
+        out_vals[tid] = in_vals[tid];
+    }
+    if (tid <= num_rows) {
+        out_row_ptrs[tid] = in_row_ptrs[tid];
+    }
+}
+
+
+template <typename IndexType>
+__global__ __launch_bounds__(default_block_size) void row_ptr_permute_kernel(
+    size_type num_rows, const IndexType *__restrict__ permutation,
+    const IndexType *__restrict__ in_row_ptrs, IndexType *__restrict__ out_nnz)
+{
+    auto tid = thread::get_thread_id_flat();
+    if (tid >= num_rows) {
+        return;
+    }
+    auto in_row = permutation[tid];
+    auto out_row = tid;
+    out_nnz[out_row] = in_row_ptrs[in_row + 1] - in_row_ptrs[in_row];
+}
+
+
+template <typename IndexType>
+__global__
+    __launch_bounds__(default_block_size) void inv_row_ptr_permute_kernel(
+        size_type num_rows, const IndexType *__restrict__ permutation,
+        const IndexType *__restrict__ in_row_ptrs,
+        IndexType *__restrict__ out_nnz)
+{
+    auto tid = thread::get_thread_id_flat();
+    if (tid >= num_rows) {
+        return;
+    }
+    auto in_row = tid;
+    auto out_row = permutation[tid];
+    out_nnz[out_row] = in_row_ptrs[in_row + 1] - in_row_ptrs[in_row];
+}
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+__global__ __launch_bounds__(default_block_size) void row_permute_kernel(
+    size_type num_rows, const IndexType *__restrict__ permutation,
+    const IndexType *__restrict__ in_row_ptrs,
+    const IndexType *__restrict__ in_cols,
+    const ValueType *__restrict__ in_vals,
+    const IndexType *__restrict__ out_row_ptrs,
+    IndexType *__restrict__ out_cols, ValueType *__restrict__ out_vals)
+{
+    auto tid = thread::get_subwarp_id_flat<subwarp_size>();
+    if (tid >= num_rows) {
+        return;
+    }
+    auto lane = threadIdx.x % subwarp_size;
+    auto in_row = permutation[tid];
+    auto out_row = tid;
+    auto in_begin = in_row_ptrs[in_row];
+    auto in_size = in_row_ptrs[in_row + 1] - in_begin;
+    auto out_begin = out_row_ptrs[out_row];
+    for (IndexType i = lane; i < in_size; i += subwarp_size) {
+        out_cols[out_begin + i] = in_cols[in_begin + i];
+        out_vals[out_begin + i] = in_vals[in_begin + i];
+    }
+}
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+__global__ __launch_bounds__(default_block_size) void inv_row_permute_kernel(
+    size_type num_rows, const IndexType *__restrict__ permutation,
+    const IndexType *__restrict__ in_row_ptrs,
+    const IndexType *__restrict__ in_cols,
+    const ValueType *__restrict__ in_vals,
+    const IndexType *__restrict__ out_row_ptrs,
+    IndexType *__restrict__ out_cols, ValueType *__restrict__ out_vals)
+{
+    auto tid = thread::get_subwarp_id_flat<subwarp_size>();
+    if (tid >= num_rows) {
+        return;
+    }
+    auto lane = threadIdx.x % subwarp_size;
+    auto in_row = tid;
+    auto out_row = permutation[tid];
+    auto in_begin = in_row_ptrs[in_row];
+    auto in_size = in_row_ptrs[in_row + 1] - in_begin;
+    auto out_begin = out_row_ptrs[out_row];
+    for (IndexType i = lane; i < in_size; i += subwarp_size) {
+        out_cols[out_begin + i] = in_cols[in_begin + i];
+        out_vals[out_begin + i] = in_vals[in_begin + i];
+    }
+}
\ No newline at end of file
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 09082c5d507..a981372c630 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -647,10 +647,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL);
 
 template <typename ValueType, typename IndexType>
-GKO_DECLARE_CSR_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)
+GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
 
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType)
@@ -658,11 +658,10 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL);
 
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)
+template <typename IndexType>
+GKO_DECLARE_INVERT_PERMUTATION_KERNEL(IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
 
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_CSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL(ValueType, IndexType)
diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp
index 6a31b3bffae..c6d410aee16 100644
--- a/core/matrix/csr.cpp
+++ b/core/matrix/csr.cpp
@@ -70,9 +70,9 @@ GKO_REGISTER_OPERATION(convert_to_hybrid, csr::convert_to_hybrid);
 GKO_REGISTER_OPERATION(transpose, csr::transpose);
 GKO_REGISTER_OPERATION(conj_transpose, csr::conj_transpose);
 GKO_REGISTER_OPERATION(row_permute, csr::row_permute);
-GKO_REGISTER_OPERATION(column_permute, csr::column_permute);
 GKO_REGISTER_OPERATION(inverse_row_permute, csr::inverse_row_permute);
 GKO_REGISTER_OPERATION(inverse_column_permute, csr::inverse_column_permute);
+GKO_REGISTER_OPERATION(invert_permutation, csr::invert_permutation);
 GKO_REGISTER_OPERATION(calculate_max_nnz_per_row,
                        csr::calculate_max_nnz_per_row);
 GKO_REGISTER_OPERATION(calculate_nonzeros_per_row,
@@ -412,8 +412,8 @@ std::unique_ptr<LinOp> Csr<ValueType, IndexType>::row_permute(
         Csr::create(exec, this->get_size(), this->get_num_stored_elements(),
                     this->get_strategy());
 
-    exec->run(
-        csr::make_row_permute(permutation_indices, this, permute_cpy.get()));
+    exec->run(csr::make_row_permute(permutation_indices->get_const_data(), this,
+                                    permute_cpy.get()));
     permute_cpy->make_srow();
     return std::move(permute_cpy);
 }
@@ -428,10 +428,15 @@ std::unique_ptr<LinOp> Csr<ValueType, IndexType>::column_permute(
     auto permute_cpy =
         Csr::create(exec, this->get_size(), this->get_num_stored_elements(),
                     this->get_strategy());
+    Array<IndexType> inv_permutation(exec, this->get_size()[1]);
 
-    exec->run(
-        csr::make_column_permute(permutation_indices, this, permute_cpy.get()));
+    exec->run(csr::make_invert_permutation(
+        this->get_size()[1], permutation_indices->get_const_data(),
+        inv_permutation.get_data()));
+    exec->run(csr::make_inverse_column_permute(inv_permutation.get_const_data(),
+                                               this, permute_cpy.get()));
     permute_cpy->make_srow();
+    permute_cpy->sort_by_column_index();
     return std::move(permute_cpy);
 }
 
@@ -447,8 +452,9 @@ std::unique_ptr<LinOp> Csr<ValueType, IndexType>::inverse_row_permute(
         Csr::create(exec, this->get_size(), this->get_num_stored_elements(),
                     this->get_strategy());
 
-    exec->run(csr::make_inverse_row_permute(inverse_permutation_indices, this,
-                                            inverse_permute_cpy.get()));
+    exec->run(csr::make_inverse_row_permute(
+        inverse_permutation_indices->get_const_data(), this,
+        inverse_permute_cpy.get()));
     inverse_permute_cpy->make_srow();
     return std::move(inverse_permute_cpy);
 }
@@ -466,8 +472,10 @@ std::unique_ptr<LinOp> Csr<ValueType, IndexType>::inverse_column_permute(
                     this->get_strategy());
 
     exec->run(csr::make_inverse_column_permute(
-        inverse_permutation_indices, this, inverse_permute_cpy.get()));
+        inverse_permutation_indices->get_const_data(), this,
+        inverse_permute_cpy.get()));
     inverse_permute_cpy->make_srow();
+    inverse_permute_cpy->sort_by_column_index();
     return std::move(inverse_permute_cpy);
 }
 
diff --git a/core/matrix/csr_kernels.hpp b/core/matrix/csr_kernels.hpp
index 92d9f462dfd..ff8c2d9ef26 100644
--- a/core/matrix/csr_kernels.hpp
+++ b/core/matrix/csr_kernels.hpp
@@ -131,29 +131,28 @@ namespace kernels {
 
 #define GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL(ValueType, IndexType)    \
     void row_permute(std::shared_ptr<const DefaultExecutor> exec,   \
-                     const Array<IndexType> *permutation_indices,   \
+                     const IndexType *permutation_indices,          \
                      const matrix::Csr<ValueType, IndexType> *orig, \
                      matrix::Csr<ValueType, IndexType> *row_permuted)
 
-#define GKO_DECLARE_CSR_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)    \
-    void column_permute(std::shared_ptr<const DefaultExecutor> exec,   \
-                        const Array<IndexType> *permutation_indices,   \
-                        const matrix::Csr<ValueType, IndexType> *orig, \
-                        matrix::Csr<ValueType, IndexType> *column_permuted)
-
 #define GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType)    \
     void inverse_row_permute(std::shared_ptr<const DefaultExecutor> exec,   \
-                             const Array<IndexType> *permutation_indices,   \
+                             const IndexType *permutation_indices,          \
                              const matrix::Csr<ValueType, IndexType> *orig, \
                              matrix::Csr<ValueType, IndexType> *row_permuted)
 
 #define GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType) \
     void inverse_column_permute(                                            \
         std::shared_ptr<const DefaultExecutor> exec,                        \
-        const Array<IndexType> *permutation_indices,                        \
+        const IndexType *permutation_indices,                               \
         const matrix::Csr<ValueType, IndexType> *orig,                      \
         matrix::Csr<ValueType, IndexType> *column_permuted)
 
+#define GKO_DECLARE_INVERT_PERMUTATION_KERNEL(IndexType)             \
+    void invert_permutation(                                         \
+        std::shared_ptr<const DefaultExecutor> exec, size_type size, \
+        const IndexType *permutation_indices, IndexType *inv_permutation)
+
 #define GKO_DECLARE_CSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL(ValueType, IndexType) \
     void calculate_max_nnz_per_row(                                            \
         std::shared_ptr<const DefaultExecutor> exec,                           \
@@ -210,11 +209,11 @@ namespace kernels {
     template <typename ValueType, typename IndexType>                        \
     GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL(ValueType, IndexType);                \
     template <typename ValueType, typename IndexType>                        \
-    GKO_DECLARE_CSR_COLUMN_PERMUTE_KERNEL(ValueType, IndexType);             \
-    template <typename ValueType, typename IndexType>                        \
     GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType);        \
     template <typename ValueType, typename IndexType>                        \
     GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType);     \
+    template <typename IndexType>                                            \
+    GKO_DECLARE_INVERT_PERMUTATION_KERNEL(IndexType);                        \
     template <typename ValueType, typename IndexType>                        \
     GKO_DECLARE_CSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL(ValueType, IndexType);  \
     template <typename ValueType, typename IndexType>                        \
diff --git a/cuda/matrix/csr_kernels.cu b/cuda/matrix/csr_kernels.cu
index 7316ecfa890..b90745fc8d9 100644
--- a/cuda/matrix/csr_kernels.cu
+++ b/cuda/matrix/csr_kernels.cu
@@ -1143,34 +1143,66 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONJ_TRANSPOSE_KERNEL);
 
 
+template <typename IndexType>
+void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
+                        size_type size, const IndexType *permutation_indices,
+                        IndexType *inv_permutation)
+{
+    auto num_blocks = ceildiv(size, default_block_size);
+    inv_permutation_kernel<<<num_blocks, default_block_size>>>(
+        size, permutation_indices, inv_permutation);
+}
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void row_permute(std::shared_ptr<const CudaExecutor> exec,
-                 const Array<IndexType> *permutation_indices,
+                 const IndexType *perm,
                  const matrix::Csr<ValueType, IndexType> *orig,
                  matrix::Csr<ValueType, IndexType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
+{
+    auto num_rows = orig->get_size()[0];
+    auto count_num_blocks = ceildiv(num_rows, default_block_size);
+    row_ptr_permute_kernel<<<count_num_blocks, default_block_size>>>(
+        num_rows, perm, orig->get_const_row_ptrs(),
+        row_permuted->get_row_ptrs());
+    components::prefix_sum(exec, row_permuted->get_row_ptrs(), num_rows + 1);
+    auto copy_num_blocks =
+        ceildiv(num_rows, default_block_size / config::warp_size);
+    row_permute_kernel<config::warp_size>
+        <<<copy_num_blocks, default_block_size>>>(
+            num_rows, perm, orig->get_const_row_ptrs(),
+            orig->get_const_col_idxs(), as_cuda_type(orig->get_const_values()),
+            row_permuted->get_row_ptrs(), row_permuted->get_col_idxs(),
+            as_cuda_type(row_permuted->get_values()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void column_permute(std::shared_ptr<const CudaExecutor> exec,
-                    const Array<IndexType> *permutation_indices,
-                    const matrix::Csr<ValueType, IndexType> *orig,
-                    matrix::Csr<ValueType, IndexType> *column_permuted)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_COLUMN_PERMUTE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void inverse_row_permute(std::shared_ptr<const CudaExecutor> exec,
-                         const Array<IndexType> *permutation_indices,
+                         const IndexType *perm,
                          const matrix::Csr<ValueType, IndexType> *orig,
                          matrix::Csr<ValueType, IndexType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
+{
+    auto num_rows = orig->get_size()[0];
+    auto count_num_blocks = ceildiv(num_rows, default_block_size);
+    inv_row_ptr_permute_kernel<<<count_num_blocks, default_block_size>>>(
+        num_rows, perm, orig->get_const_row_ptrs(),
+        row_permuted->get_row_ptrs());
+    components::prefix_sum(exec, row_permuted->get_row_ptrs(), num_rows + 1);
+    auto copy_num_blocks =
+        ceildiv(num_rows, default_block_size / config::warp_size);
+    inv_row_permute_kernel<config::warp_size>
+        <<<copy_num_blocks, default_block_size>>>(
+            num_rows, perm, orig->get_const_row_ptrs(),
+            orig->get_const_col_idxs(), as_cuda_type(orig->get_const_values()),
+            row_permuted->get_row_ptrs(), row_permuted->get_col_idxs(),
+            as_cuda_type(row_permuted->get_values()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL);
@@ -1178,10 +1210,19 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void inverse_column_permute(std::shared_ptr<const CudaExecutor> exec,
-                            const Array<IndexType> *permutation_indices,
+                            const IndexType *perm,
                             const matrix::Csr<ValueType, IndexType> *orig,
                             matrix::Csr<ValueType, IndexType> *column_permuted)
-    GKO_NOT_IMPLEMENTED;
+{
+    auto num_rows = orig->get_size()[0];
+    auto nnz = orig->get_num_stored_elements();
+    auto num_blocks = ceildiv(std::max(num_rows, nnz), default_block_size);
+    col_permute_kernel<<<num_blocks, default_block_size>>>(
+        num_rows, nnz, perm, orig->get_const_row_ptrs(),
+        orig->get_const_col_idxs(), as_cuda_type(orig->get_const_values()),
+        column_permuted->get_row_ptrs(), column_permuted->get_col_idxs(),
+        as_cuda_type(column_permuted->get_values()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index 9a4a9075933..38af50b1c21 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -60,6 +60,7 @@ namespace {
 
 class Csr : public ::testing::Test {
 protected:
+    using Arr = gko::Array<int>;
     using Vec = gko::matrix::Dense<>;
     using Mtx = gko::matrix::Csr<>;
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
@@ -114,6 +115,18 @@ class Csr : public ::testing::Test {
         dalpha->copy_from(alpha.get());
         dbeta = Vec::create(cuda);
         dbeta->copy_from(beta.get());
+
+        std::vector<int> tmp(mtx->get_size()[0], 0);
+        auto rng = std::default_random_engine{};
+        std::iota(tmp.begin(), tmp.end(), 0);
+        std::shuffle(tmp.begin(), tmp.end(), rng);
+        std::vector<int> tmp2(mtx->get_size()[1], 0);
+        std::iota(tmp2.begin(), tmp2.end(), 0);
+        std::shuffle(tmp2.begin(), tmp2.end(), rng);
+        rpermute_idxs = std::make_unique<Arr>(ref, tmp.begin(), tmp.end());
+        drpermute_idxs = std::make_unique<Arr>(cuda, tmp.begin(), tmp.end());
+        cpermute_idxs = std::make_unique<Arr>(ref, tmp2.begin(), tmp2.end());
+        dcpermute_idxs = std::make_unique<Arr>(cuda, tmp2.begin(), tmp2.end());
     }
 
     void set_up_apply_complex_data(
@@ -179,6 +192,10 @@ class Csr : public ::testing::Test {
     std::unique_ptr<Vec> dy;
     std::unique_ptr<Vec> dalpha;
     std::unique_ptr<Vec> dbeta;
+    std::unique_ptr<Arr> rpermute_idxs;
+    std::unique_ptr<Arr> drpermute_idxs;
+    std::unique_ptr<Arr> cpermute_idxs;
+    std::unique_ptr<Arr> dcpermute_idxs;
 };
 
 
@@ -688,6 +705,62 @@ TEST_F(Csr, MoveToHybridIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, IsRowPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+    auto r_permute = gko::share(mtx->row_permute(rpermute_idxs.get()));
+    auto dr_permute = gko::share(dmtx->row_permute(drpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(r_permute),
+                               gko::as<Mtx>(dr_permute));
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(r_permute), gko::as<Mtx>(dr_permute), 0);
+}
+
+
+TEST_F(Csr, IsColPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+    auto c_permute = gko::share(mtx->column_permute(cpermute_idxs.get()));
+    auto dc_permute = gko::share(dmtx->column_permute(dcpermute_idxs.get()));
+
+    ASSERT_TRUE(gko::as<Mtx>(dc_permute)->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(c_permute),
+                               gko::as<Mtx>(dc_permute));
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(c_permute), gko::as<Mtx>(dc_permute), 0);
+}
+
+
+TEST_F(Csr, IsInverseRowPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+    auto inverse_r_permute =
+        gko::share(mtx->inverse_row_permute(rpermute_idxs.get()));
+    auto d_inverse_r_permute =
+        gko::share(dmtx->inverse_row_permute(drpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(inverse_r_permute),
+                               gko::as<Mtx>(d_inverse_r_permute));
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(inverse_r_permute),
+                        gko::as<Mtx>(d_inverse_r_permute), 0);
+}
+
+
+TEST_F(Csr, IsInverseColPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+    auto inverse_c_permute =
+        gko::share(mtx->inverse_column_permute(cpermute_idxs.get()));
+    auto d_inverse_c_permute =
+        gko::share(dmtx->inverse_column_permute(dcpermute_idxs.get()));
+
+    ASSERT_TRUE(gko::as<Mtx>(d_inverse_c_permute)->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(inverse_c_permute),
+                               gko::as<Mtx>(d_inverse_c_permute));
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(inverse_c_permute),
+                        gko::as<Mtx>(d_inverse_c_permute), 0);
+}
+
+
 TEST_F(Csr, RecognizeSortedMatrixIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>());
diff --git a/hip/matrix/csr_kernels.hip.cpp b/hip/matrix/csr_kernels.hip.cpp
index f133507092e..6effd739123 100644
--- a/hip/matrix/csr_kernels.hip.cpp
+++ b/hip/matrix/csr_kernels.hip.cpp
@@ -967,34 +967,68 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void row_permute(std::shared_ptr<const HipExecutor> exec,
-                 const Array<IndexType> *permutation_indices,
-                 const matrix::Csr<ValueType, IndexType> *orig,
-                 matrix::Csr<ValueType, IndexType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
+template <typename IndexType>
+void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
+                        size_type size, const IndexType *permutation_indices,
+                        IndexType *inv_permutation)
+{
+    auto num_blocks = ceildiv(size, default_block_size);
+    hipLaunchKernelGGL(HIP_KERNEL_NAME(inv_permutation_kernel), num_blocks,
+                       default_block_size, 0, 0, size, permutation_indices,
+                       inv_permutation);
+}
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void column_permute(std::shared_ptr<const HipExecutor> exec,
-                    const Array<IndexType> *permutation_indices,
-                    const matrix::Csr<ValueType, IndexType> *orig,
-                    matrix::Csr<ValueType, IndexType> *column_permuted)
-    GKO_NOT_IMPLEMENTED;
+void row_permute(std::shared_ptr<const HipExecutor> exec, const IndexType *perm,
+                 const matrix::Csr<ValueType, IndexType> *orig,
+                 matrix::Csr<ValueType, IndexType> *row_permuted)
+{
+    auto num_rows = orig->get_size()[0];
+    auto count_num_blocks = ceildiv(num_rows, default_block_size);
+    hipLaunchKernelGGL(HIP_KERNEL_NAME(row_ptr_permute_kernel),
+                       count_num_blocks, default_block_size, 0, 0, num_rows,
+                       perm, orig->get_const_row_ptrs(),
+                       row_permuted->get_row_ptrs());
+    components::prefix_sum(exec, row_permuted->get_row_ptrs(), num_rows + 1);
+    auto copy_num_blocks =
+        ceildiv(num_rows, default_block_size / config::warp_size);
+    hipLaunchKernelGGL(
+        HIP_KERNEL_NAME(row_permute_kernel<config::warp_size>), copy_num_blocks,
+        default_block_size, 0, 0, num_rows, perm, orig->get_const_row_ptrs(),
+        orig->get_const_col_idxs(), as_hip_type(orig->get_const_values()),
+        row_permuted->get_row_ptrs(), row_permuted->get_col_idxs(),
+        as_hip_type(row_permuted->get_values()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
 void inverse_row_permute(std::shared_ptr<const HipExecutor> exec,
-                         const Array<IndexType> *permutation_indices,
+                         const IndexType *perm,
                          const matrix::Csr<ValueType, IndexType> *orig,
                          matrix::Csr<ValueType, IndexType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
+{
+    auto num_rows = orig->get_size()[0];
+    auto count_num_blocks = ceildiv(num_rows, default_block_size);
+    hipLaunchKernelGGL(HIP_KERNEL_NAME(inv_row_ptr_permute_kernel),
+                       count_num_blocks, default_block_size, 0, 0, num_rows,
+                       perm, orig->get_const_row_ptrs(),
+                       row_permuted->get_row_ptrs());
+    components::prefix_sum(exec, row_permuted->get_row_ptrs(), num_rows + 1);
+    auto copy_num_blocks =
+        ceildiv(num_rows, default_block_size / config::warp_size);
+    hipLaunchKernelGGL(
+        HIP_KERNEL_NAME(inv_row_permute_kernel<config::warp_size>),
+        copy_num_blocks, default_block_size, 0, 0, num_rows, perm,
+        orig->get_const_row_ptrs(), orig->get_const_col_idxs(),
+        as_hip_type(orig->get_const_values()), row_permuted->get_row_ptrs(),
+        row_permuted->get_col_idxs(), as_hip_type(row_permuted->get_values()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL);
@@ -1002,10 +1036,20 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void inverse_column_permute(std::shared_ptr<const HipExecutor> exec,
-                            const Array<IndexType> *permutation_indices,
+                            const IndexType *perm,
                             const matrix::Csr<ValueType, IndexType> *orig,
                             matrix::Csr<ValueType, IndexType> *column_permuted)
-    GKO_NOT_IMPLEMENTED;
+{
+    auto num_rows = orig->get_size()[0];
+    auto nnz = orig->get_num_stored_elements();
+    auto num_blocks = ceildiv(std::max(num_rows, nnz), default_block_size);
+    hipLaunchKernelGGL(
+        HIP_KERNEL_NAME(col_permute_kernel), num_blocks, default_block_size, 0,
+        0, num_rows, nnz, perm, orig->get_const_row_ptrs(),
+        orig->get_const_col_idxs(), as_hip_type(orig->get_const_values()),
+        column_permuted->get_row_ptrs(), column_permuted->get_col_idxs(),
+        as_hip_type(column_permuted->get_values()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index 99e6bca6f2a..510189ad70e 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -60,6 +60,7 @@ namespace {
 
 class Csr : public ::testing::Test {
 protected:
+    using Arr = gko::Array<int>;
     using Vec = gko::matrix::Dense<>;
     using Mtx = gko::matrix::Csr<>;
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
@@ -114,6 +115,18 @@ class Csr : public ::testing::Test {
         dalpha->copy_from(alpha.get());
         dbeta = Vec::create(hip);
         dbeta->copy_from(beta.get());
+
+        std::vector<int> tmp(mtx->get_size()[0], 0);
+        auto rng = std::default_random_engine{};
+        std::iota(tmp.begin(), tmp.end(), 0);
+        std::shuffle(tmp.begin(), tmp.end(), rng);
+        std::vector<int> tmp2(mtx->get_size()[1], 0);
+        std::iota(tmp2.begin(), tmp2.end(), 0);
+        std::shuffle(tmp2.begin(), tmp2.end(), rng);
+        rpermute_idxs = std::make_unique<Arr>(ref, tmp.begin(), tmp.end());
+        drpermute_idxs = std::make_unique<Arr>(hip, tmp.begin(), tmp.end());
+        cpermute_idxs = std::make_unique<Arr>(ref, tmp2.begin(), tmp2.end());
+        dcpermute_idxs = std::make_unique<Arr>(hip, tmp2.begin(), tmp2.end());
     }
 
     void set_up_apply_complex_data(
@@ -179,6 +192,10 @@ class Csr : public ::testing::Test {
     std::unique_ptr<Vec> dy;
     std::unique_ptr<Vec> dalpha;
     std::unique_ptr<Vec> dbeta;
+    std::unique_ptr<Arr> rpermute_idxs;
+    std::unique_ptr<Arr> drpermute_idxs;
+    std::unique_ptr<Arr> cpermute_idxs;
+    std::unique_ptr<Arr> dcpermute_idxs;
 };
 
 
@@ -687,6 +704,62 @@ TEST_F(Csr, MoveToHybridIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, IsRowPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+    auto r_permute = gko::share(mtx->row_permute(rpermute_idxs.get()));
+    auto dr_permute = gko::share(dmtx->row_permute(drpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(r_permute),
+                               gko::as<Mtx>(dr_permute));
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(r_permute), gko::as<Mtx>(dr_permute), 0);
+}
+
+
+TEST_F(Csr, IsColPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+    auto c_permute = gko::share(mtx->column_permute(cpermute_idxs.get()));
+    auto dc_permute = gko::share(dmtx->column_permute(dcpermute_idxs.get()));
+
+    ASSERT_TRUE(gko::as<Mtx>(dc_permute)->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(c_permute),
+                               gko::as<Mtx>(dc_permute));
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(c_permute), gko::as<Mtx>(dc_permute), 0);
+}
+
+
+TEST_F(Csr, IsInverseRowPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+    auto inverse_r_permute =
+        gko::share(mtx->inverse_row_permute(rpermute_idxs.get()));
+    auto d_inverse_r_permute =
+        gko::share(dmtx->inverse_row_permute(drpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(inverse_r_permute),
+                               gko::as<Mtx>(d_inverse_r_permute));
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(inverse_r_permute),
+                        gko::as<Mtx>(d_inverse_r_permute), 0);
+}
+
+
+TEST_F(Csr, IsInverseColPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+    auto inverse_c_permute =
+        gko::share(mtx->inverse_column_permute(cpermute_idxs.get()));
+    auto d_inverse_c_permute =
+        gko::share(dmtx->inverse_column_permute(dcpermute_idxs.get()));
+
+    ASSERT_TRUE(gko::as<Mtx>(d_inverse_c_permute)->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(inverse_c_permute),
+                               gko::as<Mtx>(d_inverse_c_permute));
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(inverse_c_permute),
+                        gko::as<Mtx>(d_inverse_c_permute), 0);
+}
+
+
 TEST_F(Csr, RecognizeSortedMatrixIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
diff --git a/omp/matrix/csr_kernels.cpp b/omp/matrix/csr_kernels.cpp
index a4d467a0693..32007104cc1 100644
--- a/omp/matrix/csr_kernels.cpp
+++ b/omp/matrix/csr_kernels.cpp
@@ -598,13 +598,25 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONVERT_TO_HYBRID_KERNEL);
 
 
+template <typename IndexType>
+void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
+                        size_type size, const IndexType *permutation_indices,
+                        IndexType *inv_permutation)
+{
+#pragma omp parallel for
+    for (size_type i = 0; i < size; ++i) {
+        inv_permutation[permutation_indices[i]] = i;
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
-void row_permute_impl(std::shared_ptr<const OmpExecutor> exec,
-                      const Array<IndexType> *permutation_indices,
-                      const matrix::Csr<ValueType, IndexType> *orig,
-                      matrix::Csr<ValueType, IndexType> *row_permuted)
+void row_permute(std::shared_ptr<const OmpExecutor> exec, const IndexType *perm,
+                 const matrix::Csr<ValueType, IndexType> *orig,
+                 matrix::Csr<ValueType, IndexType> *row_permuted)
 {
-    auto perm = permutation_indices->get_const_data();
     auto orig_row_ptrs = orig->get_const_row_ptrs();
     auto orig_col_idxs = orig->get_const_col_idxs();
     auto orig_vals = orig->get_const_values();
@@ -614,60 +626,65 @@ void row_permute_impl(std::shared_ptr<const OmpExecutor> exec,
     size_type num_rows = orig->get_size()[0];
     size_type num_nnz = orig->get_num_stored_elements();
 
-    size_type cur_ptr = 0;
-    rp_row_ptrs[0] = cur_ptr;
-    vector<size_type> orig_num_nnz_per_row(num_rows, 0, exec);
 #pragma omp parallel for
     for (size_type row = 0; row < num_rows; ++row) {
-        orig_num_nnz_per_row[row] = orig_row_ptrs[row + 1] - orig_row_ptrs[row];
+        auto src_row = perm[row];
+        auto dst_row = row;
+        rp_row_ptrs[dst_row] =
+            orig_row_ptrs[src_row + 1] - orig_row_ptrs[src_row];
     }
-    for (size_type row = 0; row < num_rows; ++row) {
-        rp_row_ptrs[row + 1] =
-            rp_row_ptrs[row] + orig_num_nnz_per_row[perm[row]];
-    }
-    rp_row_ptrs[num_rows] = orig_row_ptrs[num_rows];
+    components::prefix_sum(exec, rp_row_ptrs, num_rows + 1);
 #pragma omp parallel for
     for (size_type row = 0; row < num_rows; ++row) {
-        auto new_row = perm[row];
-        auto new_k = orig_row_ptrs[new_row];
-        for (size_type k = rp_row_ptrs[row];
-             k < size_type(rp_row_ptrs[row + 1]); ++k) {
-            rp_col_idxs[k] = orig_col_idxs[new_k];
-            rp_vals[k] = orig_vals[new_k];
-            new_k++;
-        }
+        auto src_row = perm[row];
+        auto dst_row = row;
+        auto src_begin = orig_row_ptrs[src_row];
+        auto dst_begin = rp_row_ptrs[dst_row];
+        auto row_size = orig_row_ptrs[src_row + 1] - src_begin;
+        std::copy_n(orig_col_idxs + src_begin, row_size,
+                    rp_col_idxs + dst_begin);
+        std::copy_n(orig_vals + src_begin, row_size, rp_vals + dst_begin);
     }
 }
 
-
-template <typename ValueType, typename IndexType>
-void row_permute(std::shared_ptr<const OmpExecutor> exec,
-                 const Array<IndexType> *permutation_indices,
-                 const matrix::Csr<ValueType, IndexType> *orig,
-                 matrix::Csr<ValueType, IndexType> *row_permuted)
-{
-    row_permute_impl(exec, permutation_indices, orig, row_permuted);
-}
-
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
 void inverse_row_permute(std::shared_ptr<const OmpExecutor> exec,
-                         const Array<IndexType> *permutation_indices,
+                         const IndexType *perm,
                          const matrix::Csr<ValueType, IndexType> *orig,
                          matrix::Csr<ValueType, IndexType> *row_permuted)
 {
-    auto perm = permutation_indices->get_const_data();
-    Array<IndexType> inv_perm(*permutation_indices);
-    auto iperm = inv_perm.get_data();
+    auto orig_row_ptrs = orig->get_const_row_ptrs();
+    auto orig_col_idxs = orig->get_const_col_idxs();
+    auto orig_vals = orig->get_const_values();
+    auto rp_row_ptrs = row_permuted->get_row_ptrs();
+    auto rp_col_idxs = row_permuted->get_col_idxs();
+    auto rp_vals = row_permuted->get_values();
+    size_type num_rows = orig->get_size()[0];
+    size_type num_nnz = orig->get_num_stored_elements();
+
 #pragma omp parallel for
-    for (size_type ind = 0; ind < inv_perm.get_num_elems(); ++ind) {
-        iperm[perm[ind]] = ind;
+    for (size_type row = 0; row < num_rows; ++row) {
+        auto src_row = row;
+        auto dst_row = perm[row];
+        rp_row_ptrs[dst_row] =
+            orig_row_ptrs[src_row + 1] - orig_row_ptrs[src_row];
+    }
+    components::prefix_sum(exec, rp_row_ptrs, num_rows + 1);
+#pragma omp parallel for
+    for (size_type row = 0; row < num_rows; ++row) {
+        auto src_row = row;
+        auto dst_row = perm[row];
+        auto src_begin = orig_row_ptrs[src_row];
+        auto dst_begin = rp_row_ptrs[dst_row];
+        auto row_size = orig_row_ptrs[src_row + 1] - src_begin;
+        std::copy_n(orig_col_idxs + src_begin, row_size,
+                    rp_col_idxs + dst_begin);
+        std::copy_n(orig_vals + src_begin, row_size, rp_vals + dst_begin);
     }
-
-    row_permute_impl(exec, &inv_perm, orig, row_permuted);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -675,26 +692,23 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void column_permute_impl(const Array<IndexType> *permutation_indices,
-                         const matrix::Csr<ValueType, IndexType> *orig,
-                         matrix::Csr<ValueType, IndexType> *column_permuted)
+void inverse_column_permute(std::shared_ptr<const OmpExecutor> exec,
+                            const IndexType *perm,
+                            const matrix::Csr<ValueType, IndexType> *orig,
+                            matrix::Csr<ValueType, IndexType> *column_permuted)
 {
-    auto perm = permutation_indices->get_const_data();
     auto orig_row_ptrs = orig->get_const_row_ptrs();
     auto orig_col_idxs = orig->get_const_col_idxs();
     auto orig_vals = orig->get_const_values();
     auto cp_row_ptrs = column_permuted->get_row_ptrs();
     auto cp_col_idxs = column_permuted->get_col_idxs();
     auto cp_vals = column_permuted->get_values();
-    auto num_nnz = orig->get_num_stored_elements();
     size_type num_rows = orig->get_size()[0];
-    size_type num_cols = orig->get_size()[1];
 
 #pragma omp parallel for
     for (size_type row = 0; row < num_rows; ++row) {
         cp_row_ptrs[row] = orig_row_ptrs[row];
-        for (size_type k = orig_row_ptrs[row];
-             k < size_type(orig_row_ptrs[row + 1]); ++k) {
+        for (auto k = orig_row_ptrs[row]; k < orig_row_ptrs[row + 1]; ++k) {
             cp_col_idxs[k] = perm[orig_col_idxs[k]];
             cp_vals[k] = orig_vals[k];
         }
@@ -702,36 +716,6 @@ void column_permute_impl(const Array<IndexType> *permutation_indices,
     cp_row_ptrs[num_rows] = orig_row_ptrs[num_rows];
 }
 
-
-template <typename ValueType, typename IndexType>
-void column_permute(std::shared_ptr<const OmpExecutor> exec,
-                    const Array<IndexType> *permutation_indices,
-                    const matrix::Csr<ValueType, IndexType> *orig,
-                    matrix::Csr<ValueType, IndexType> *column_permuted)
-{
-    auto perm = permutation_indices->get_const_data();
-    Array<IndexType> inv_perm(*permutation_indices);
-    auto iperm = inv_perm.get_data();
-#pragma omp parallel for
-    for (size_type ind = 0; ind < inv_perm.get_num_elems(); ++ind) {
-        iperm[perm[ind]] = ind;
-    }
-    column_permute_impl(&inv_perm, orig, column_permuted);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_COLUMN_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(std::shared_ptr<const OmpExecutor> exec,
-                            const Array<IndexType> *permutation_indices,
-                            const matrix::Csr<ValueType, IndexType> *orig,
-                            matrix::Csr<ValueType, IndexType> *column_permuted)
-{
-    column_permute_impl(permutation_indices, orig, column_permuted);
-}
-
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
 
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index 6462720003f..0dcfaac7844 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -126,14 +126,10 @@ class Csr : public ::testing::Test {
         std::vector<int> tmp2(mtx->get_size()[1], 0);
         std::iota(tmp2.begin(), tmp2.end(), 0);
         std::shuffle(tmp2.begin(), tmp2.end(), rng);
-        rpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{ref, tmp.begin(), tmp.end()});
-        drpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{omp, tmp.begin(), tmp.end()});
-        cpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{ref, tmp2.begin(), tmp2.end()});
-        dcpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{omp, tmp2.begin(), tmp2.end()});
+        rpermute_idxs = std::make_unique<Arr>(ref, tmp.begin(), tmp.end());
+        drpermute_idxs = std::make_unique<Arr>(omp, tmp.begin(), tmp.end());
+        cpermute_idxs = std::make_unique<Arr>(ref, tmp2.begin(), tmp2.end());
+        dcpermute_idxs = std::make_unique<Arr>(omp, tmp2.begin(), tmp2.end());
     }
 
     struct matrix_pair {
@@ -480,45 +476,56 @@ TEST_F(Csr, MoveToHybridIsEquivalentToRef)
 TEST_F(Csr, IsRowPermutable)
 {
     set_up_apply_data();
-    auto r_permute = mtx->row_permute(rpermute_idxs.get());
-    auto dr_permute = dmtx->row_permute(drpermute_idxs.get());
+    auto r_permute = gko::share(mtx->row_permute(rpermute_idxs.get()));
+    auto dr_permute = gko::share(dmtx->row_permute(drpermute_idxs.get()));
 
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(r_permute.get()),
-                        static_cast<Mtx *>(dr_permute.get()), 0);
+    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(r_permute),
+                               gko::as<Mtx>(dr_permute));
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(r_permute), gko::as<Mtx>(dr_permute), 0);
 }
 
 
 TEST_F(Csr, IsColPermutable)
 {
     set_up_apply_data();
-    auto c_permute = mtx->column_permute(cpermute_idxs.get());
-    auto dc_permute = dmtx->column_permute(dcpermute_idxs.get());
+    auto c_permute = gko::share(mtx->column_permute(cpermute_idxs.get()));
+    auto dc_permute = gko::share(dmtx->column_permute(dcpermute_idxs.get()));
 
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(c_permute.get()),
-                        static_cast<Mtx *>(dc_permute.get()), 0);
+    ASSERT_TRUE(gko::as<Mtx>(dc_permute)->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(c_permute),
+                               gko::as<Mtx>(dc_permute));
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(c_permute), gko::as<Mtx>(dc_permute), 0);
 }
 
 
 TEST_F(Csr, IsInverseRowPermutable)
 {
     set_up_apply_data();
-    auto inverse_r_permute = mtx->inverse_row_permute(rpermute_idxs.get());
-    auto d_inverse_r_permute = dmtx->inverse_row_permute(drpermute_idxs.get());
+    auto inverse_r_permute =
+        gko::share(mtx->inverse_row_permute(rpermute_idxs.get()));
+    auto d_inverse_r_permute =
+        gko::share(dmtx->inverse_row_permute(drpermute_idxs.get()));
 
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_r_permute.get()),
-                        static_cast<Mtx *>(d_inverse_r_permute.get()), 0);
+    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(inverse_r_permute),
+                               gko::as<Mtx>(d_inverse_r_permute));
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(inverse_r_permute),
+                        gko::as<Mtx>(d_inverse_r_permute), 0);
 }
 
 
 TEST_F(Csr, IsInverseColPermutable)
 {
     set_up_apply_data();
-    auto inverse_c_permute = mtx->inverse_column_permute(cpermute_idxs.get());
+    auto inverse_c_permute =
+        gko::share(mtx->inverse_column_permute(cpermute_idxs.get()));
     auto d_inverse_c_permute =
-        dmtx->inverse_column_permute(dcpermute_idxs.get());
+        gko::share(dmtx->inverse_column_permute(dcpermute_idxs.get()));
 
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_c_permute.get()),
-                        static_cast<Mtx *>(d_inverse_c_permute.get()), 0);
+    ASSERT_TRUE(gko::as<Mtx>(d_inverse_c_permute)->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(inverse_c_permute),
+                               gko::as<Mtx>(d_inverse_c_permute));
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(inverse_c_permute),
+                        gko::as<Mtx>(d_inverse_c_permute), 0);
 }
 
 
diff --git a/reference/matrix/csr_kernels.cpp b/reference/matrix/csr_kernels.cpp
index e5d5b7ce151..1be2ae79cb5 100644
--- a/reference/matrix/csr_kernels.cpp
+++ b/reference/matrix/csr_kernels.cpp
@@ -689,73 +689,85 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONVERT_TO_HYBRID_KERNEL);
 
 
+template <typename IndexType>
+void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
+                        size_type size, const IndexType *permutation_indices,
+                        IndexType *inv_permutation)
+{
+    for (size_type i = 0; i < size; ++i) {
+        inv_permutation[permutation_indices[i]] = i;
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
-void row_permute_impl(std::shared_ptr<const ReferenceExecutor> exec,
-                      const Array<IndexType> *permutation_indices,
-                      const matrix::Csr<ValueType, IndexType> *orig,
-                      matrix::Csr<ValueType, IndexType> *row_permuted)
+void row_permute(std::shared_ptr<const ReferenceExecutor> exec,
+                 const IndexType *perm,
+                 const matrix::Csr<ValueType, IndexType> *orig,
+                 matrix::Csr<ValueType, IndexType> *row_permuted)
 {
-    auto perm = permutation_indices->get_const_data();
-    auto orig_row_ptrs = orig->get_const_row_ptrs();
-    auto orig_col_idxs = orig->get_const_col_idxs();
-    auto orig_vals = orig->get_const_values();
+    auto in_row_ptrs = orig->get_const_row_ptrs();
+    auto in_col_idxs = orig->get_const_col_idxs();
+    auto in_vals = orig->get_const_values();
     auto rp_row_ptrs = row_permuted->get_row_ptrs();
     auto rp_col_idxs = row_permuted->get_col_idxs();
     auto rp_vals = row_permuted->get_values();
     size_type num_rows = orig->get_size()[0];
     size_type num_nnz = orig->get_num_stored_elements();
 
-    size_type cur_ptr = 0;
-    rp_row_ptrs[0] = cur_ptr;
-    vector<size_type> orig_num_nnz_per_row(num_rows, 0, exec);
-    for (size_type row = 0; row < num_rows; ++row) {
-        orig_num_nnz_per_row[row] = orig_row_ptrs[row + 1] - orig_row_ptrs[row];
-    }
     for (size_type row = 0; row < num_rows; ++row) {
-        rp_row_ptrs[row + 1] =
-            rp_row_ptrs[row] + orig_num_nnz_per_row[perm[row]];
+        auto src_row = perm[row];
+        auto dst_row = row;
+        rp_row_ptrs[dst_row] = in_row_ptrs[src_row + 1] - in_row_ptrs[src_row];
     }
-    rp_row_ptrs[num_rows] = orig_row_ptrs[num_rows];
+    components::prefix_sum(exec, rp_row_ptrs, num_rows + 1);
     for (size_type row = 0; row < num_rows; ++row) {
-        auto new_row = perm[row];
-        auto new_k = orig_row_ptrs[new_row];
-        for (size_type k = rp_row_ptrs[row];
-             k < size_type(rp_row_ptrs[row + 1]); ++k) {
-            rp_col_idxs[k] = orig_col_idxs[new_k];
-            rp_vals[k] = orig_vals[new_k];
-            new_k++;
-        }
+        auto src_row = perm[row];
+        auto dst_row = row;
+        auto src_begin = in_row_ptrs[src_row];
+        auto dst_begin = rp_row_ptrs[dst_row];
+        auto row_size = in_row_ptrs[src_row + 1] - src_begin;
+        std::copy_n(in_col_idxs + src_begin, row_size, rp_col_idxs + dst_begin);
+        std::copy_n(in_vals + src_begin, row_size, rp_vals + dst_begin);
     }
 }
 
-
-template <typename ValueType, typename IndexType>
-void row_permute(std::shared_ptr<const ReferenceExecutor> exec,
-                 const Array<IndexType> *permutation_indices,
-                 const matrix::Csr<ValueType, IndexType> *orig,
-                 matrix::Csr<ValueType, IndexType> *row_permuted)
-{
-    row_permute_impl(exec, permutation_indices, orig, row_permuted);
-}
-
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
 void inverse_row_permute(std::shared_ptr<const ReferenceExecutor> exec,
-                         const Array<IndexType> *permutation_indices,
+                         const IndexType *perm,
                          const matrix::Csr<ValueType, IndexType> *orig,
                          matrix::Csr<ValueType, IndexType> *row_permuted)
 {
-    auto perm = permutation_indices->get_const_data();
-    Array<IndexType> inv_perm(*permutation_indices);
-    auto iperm = inv_perm.get_data();
-    for (size_type ind = 0; ind < inv_perm.get_num_elems(); ++ind) {
-        iperm[perm[ind]] = ind;
-    }
+    auto in_row_ptrs = orig->get_const_row_ptrs();
+    auto in_col_idxs = orig->get_const_col_idxs();
+    auto in_vals = orig->get_const_values();
+    auto rp_row_ptrs = row_permuted->get_row_ptrs();
+    auto rp_col_idxs = row_permuted->get_col_idxs();
+    auto rp_vals = row_permuted->get_values();
+    size_type num_rows = orig->get_size()[0];
+    size_type num_nnz = orig->get_num_stored_elements();
 
-    row_permute_impl(exec, &inv_perm, orig, row_permuted);
+    for (size_type row = 0; row < num_rows; ++row) {
+        auto src_row = row;
+        auto dst_row = perm[row];
+        rp_row_ptrs[dst_row] = in_row_ptrs[src_row + 1] - in_row_ptrs[src_row];
+    }
+    components::prefix_sum(exec, rp_row_ptrs, num_rows + 1);
+    for (size_type row = 0; row < num_rows; ++row) {
+        auto src_row = row;
+        auto dst_row = perm[row];
+        auto src_begin = in_row_ptrs[src_row];
+        auto dst_begin = rp_row_ptrs[dst_row];
+        auto row_size = in_row_ptrs[src_row + 1] - src_begin;
+        std::copy_n(in_col_idxs + src_begin, row_size, rp_col_idxs + dst_begin);
+        std::copy_n(in_vals + src_begin, row_size, rp_vals + dst_begin);
+    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -763,59 +775,29 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void column_permute_impl(const Array<IndexType> *permutation_indices,
-                         const matrix::Csr<ValueType, IndexType> *orig,
-                         matrix::Csr<ValueType, IndexType> *column_permuted)
+void inverse_column_permute(std::shared_ptr<const ReferenceExecutor> exec,
+                            const IndexType *perm,
+                            const matrix::Csr<ValueType, IndexType> *orig,
+                            matrix::Csr<ValueType, IndexType> *column_permuted)
 {
-    auto perm = permutation_indices->get_const_data();
-    auto orig_row_ptrs = orig->get_const_row_ptrs();
-    auto orig_col_idxs = orig->get_const_col_idxs();
-    auto orig_vals = orig->get_const_values();
+    auto in_row_ptrs = orig->get_const_row_ptrs();
+    auto in_col_idxs = orig->get_const_col_idxs();
+    auto in_vals = orig->get_const_values();
     auto cp_row_ptrs = column_permuted->get_row_ptrs();
     auto cp_col_idxs = column_permuted->get_col_idxs();
     auto cp_vals = column_permuted->get_values();
-    auto num_nnz = orig->get_num_stored_elements();
-    size_type num_rows = orig->get_size()[0];
-    size_type num_cols = orig->get_size()[1];
+    auto num_rows = orig->get_size()[0];
 
     for (size_type row = 0; row < num_rows; ++row) {
-        cp_row_ptrs[row] = orig_row_ptrs[row];
-        for (size_type k = orig_row_ptrs[row];
-             k < size_type(orig_row_ptrs[row + 1]); ++k) {
-            cp_col_idxs[k] = perm[orig_col_idxs[k]];
-            cp_vals[k] = orig_vals[k];
+        auto row_begin = in_row_ptrs[row];
+        auto row_end = in_row_ptrs[row + 1];
+        cp_row_ptrs[row] = in_row_ptrs[row];
+        for (auto k = row_begin; k < row_end; ++k) {
+            cp_col_idxs[k] = perm[in_col_idxs[k]];
+            cp_vals[k] = in_vals[k];
         }
     }
-    cp_row_ptrs[num_rows] = orig_row_ptrs[num_rows];
-}
-
-
-template <typename ValueType, typename IndexType>
-void column_permute(std::shared_ptr<const ReferenceExecutor> exec,
-                    const Array<IndexType> *permutation_indices,
-                    const matrix::Csr<ValueType, IndexType> *orig,
-                    matrix::Csr<ValueType, IndexType> *column_permuted)
-{
-    auto perm = permutation_indices->get_const_data();
-    Array<IndexType> inv_perm(*permutation_indices);
-    auto iperm = inv_perm.get_data();
-    for (size_type ind = 0; ind < inv_perm.get_num_elems(); ++ind) {
-        iperm[perm[ind]] = ind;
-    }
-    column_permute_impl(&inv_perm, orig, column_permuted);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_COLUMN_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(std::shared_ptr<const ReferenceExecutor> exec,
-                            const Array<IndexType> *permutation_indices,
-                            const matrix::Csr<ValueType, IndexType> *orig,
-                            matrix::Csr<ValueType, IndexType> *column_permuted)
-{
-    column_permute_impl(permutation_indices, orig, column_permuted);
+    cp_row_ptrs[num_rows] = in_row_ptrs[num_rows];
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(

From 82fb7a8a79c9805032ca802cb94368adc6b5e6c5 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 26 Oct 2020 12:20:05 +0100
Subject: [PATCH 0084/1008] improve code style and fix sonarqube warnings

---
 cuda/test/matrix/csr_kernels.cpp      | 65 +++++++++++++--------------
 hip/test/matrix/csr_kernels.hip.cpp   | 65 +++++++++++++--------------
 omp/matrix/csr_kernels.cpp            |  4 +-
 omp/test/matrix/csr_kernels.cpp       | 65 +++++++++++++--------------
 reference/matrix/csr_kernels.cpp      |  4 +-
 reference/test/matrix/csr_kernels.cpp | 43 +++++++-----------
 6 files changed, 107 insertions(+), 139 deletions(-)

diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index 38af50b1c21..f0a1416b6bd 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -463,12 +463,11 @@ TEST_F(Csr, TransposeIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>(cuda));
 
-    auto trans = mtx->transpose();
-    auto d_trans = dmtx->transpose();
+    auto trans = gko::as<Mtx>(mtx->transpose());
+    auto d_trans = gko::as<Mtx>(dmtx->transpose());
 
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(d_trans.get()),
-                        static_cast<Mtx *>(trans.get()), 0.0);
-    ASSERT_TRUE(static_cast<Mtx *>(d_trans.get())->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_NEAR(d_trans, trans, 0.0);
+    ASSERT_TRUE(d_trans->is_sorted_by_column_index());
 }
 
 
@@ -476,13 +475,11 @@ TEST_F(Csr, ConjugateTransposeIsEquivalentToRef)
 {
     set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(cuda));
 
-    auto trans = complex_mtx->conj_transpose();
-    auto d_trans = complex_dmtx->conj_transpose();
+    auto trans = gko::as<ComplexMtx>(complex_mtx->conj_transpose());
+    auto d_trans = gko::as<ComplexMtx>(complex_dmtx->conj_transpose());
 
-    GKO_ASSERT_MTX_NEAR(static_cast<ComplexMtx *>(d_trans.get()),
-                        static_cast<ComplexMtx *>(trans.get()), 0.0);
-    ASSERT_TRUE(
-        static_cast<ComplexMtx *>(d_trans.get())->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_NEAR(d_trans, trans, 0.0);
+    ASSERT_TRUE(d_trans->is_sorted_by_column_index());
 }
 
 
@@ -708,56 +705,54 @@ TEST_F(Csr, MoveToHybridIsEquivalentToRef)
 TEST_F(Csr, IsRowPermutable)
 {
     set_up_apply_data(std::make_shared<Mtx::classical>());
-    auto r_permute = gko::share(mtx->row_permute(rpermute_idxs.get()));
-    auto dr_permute = gko::share(dmtx->row_permute(drpermute_idxs.get()));
 
-    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(r_permute),
-                               gko::as<Mtx>(dr_permute));
-    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(r_permute), gko::as<Mtx>(dr_permute), 0);
+    auto r_permute = gko::as<Mtx>(mtx->row_permute(rpermute_idxs.get()));
+    auto dr_permute = gko::as<Mtx>(dmtx->row_permute(drpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(r_permute, dr_permute);
+    GKO_ASSERT_MTX_NEAR(r_permute, dr_permute, 0);
 }
 
 
 TEST_F(Csr, IsColPermutable)
 {
     set_up_apply_data(std::make_shared<Mtx::classical>());
-    auto c_permute = gko::share(mtx->column_permute(cpermute_idxs.get()));
-    auto dc_permute = gko::share(dmtx->column_permute(dcpermute_idxs.get()));
 
-    ASSERT_TRUE(gko::as<Mtx>(dc_permute)->is_sorted_by_column_index());
-    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(c_permute),
-                               gko::as<Mtx>(dc_permute));
-    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(c_permute), gko::as<Mtx>(dc_permute), 0);
+    auto c_permute = gko::as<Mtx>(mtx->column_permute(cpermute_idxs.get()));
+    auto dc_permute = gko::as<Mtx>(dmtx->column_permute(dcpermute_idxs.get()));
+
+    ASSERT_TRUE(dc_permute->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_EQ_SPARSITY(c_permute, dc_permute);
+    GKO_ASSERT_MTX_NEAR(c_permute, dc_permute, 0);
 }
 
 
 TEST_F(Csr, IsInverseRowPermutable)
 {
     set_up_apply_data(std::make_shared<Mtx::classical>());
+
     auto inverse_r_permute =
-        gko::share(mtx->inverse_row_permute(rpermute_idxs.get()));
+        gko::as<Mtx>(mtx->inverse_row_permute(rpermute_idxs.get()));
     auto d_inverse_r_permute =
-        gko::share(dmtx->inverse_row_permute(drpermute_idxs.get()));
+        gko::as<Mtx>(dmtx->inverse_row_permute(drpermute_idxs.get()));
 
-    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(inverse_r_permute),
-                               gko::as<Mtx>(d_inverse_r_permute));
-    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(inverse_r_permute),
-                        gko::as<Mtx>(d_inverse_r_permute), 0);
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse_r_permute, d_inverse_r_permute);
+    GKO_ASSERT_MTX_NEAR(inverse_r_permute, d_inverse_r_permute, 0);
 }
 
 
 TEST_F(Csr, IsInverseColPermutable)
 {
     set_up_apply_data(std::make_shared<Mtx::classical>());
+
     auto inverse_c_permute =
-        gko::share(mtx->inverse_column_permute(cpermute_idxs.get()));
+        gko::as<Mtx>(mtx->inverse_column_permute(cpermute_idxs.get()));
     auto d_inverse_c_permute =
-        gko::share(dmtx->inverse_column_permute(dcpermute_idxs.get()));
+        gko::as<Mtx>(dmtx->inverse_column_permute(dcpermute_idxs.get()));
 
-    ASSERT_TRUE(gko::as<Mtx>(d_inverse_c_permute)->is_sorted_by_column_index());
-    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(inverse_c_permute),
-                               gko::as<Mtx>(d_inverse_c_permute));
-    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(inverse_c_permute),
-                        gko::as<Mtx>(d_inverse_c_permute), 0);
+    ASSERT_TRUE(d_inverse_c_permute->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse_c_permute, d_inverse_c_permute);
+    GKO_ASSERT_MTX_NEAR(inverse_c_permute, d_inverse_c_permute, 0);
 }
 
 
diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index 510189ad70e..1fb01640dfa 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -462,12 +462,11 @@ TEST_F(Csr, TransposeIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
 
-    auto trans = mtx->transpose();
-    auto d_trans = dmtx->transpose();
+    auto trans = gko::as<Mtx>(mtx->transpose());
+    auto d_trans = gko::as<Mtx>(dmtx->transpose());
 
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(d_trans.get()),
-                        static_cast<Mtx *>(trans.get()), 0.0);
-    ASSERT_TRUE(static_cast<Mtx *>(d_trans.get())->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_NEAR(d_trans, trans, 0.0);
+    ASSERT_TRUE(d_trans->is_sorted_by_column_index());
 }
 
 
@@ -475,13 +474,11 @@ TEST_F(Csr, ConjugateTransposeIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
 
-    auto ctrans = mtx->conj_transpose();
-    auto d_ctrans = dmtx->conj_transpose();
+    auto trans = gko::as<Mtx>(mtx->conj_transpose());
+    auto d_trans = gko::as<Mtx>(dmtx->conj_transpose());
 
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(d_ctrans.get()),
-                        static_cast<Mtx *>(ctrans.get()), 0.0);
-    ASSERT_TRUE(
-        static_cast<Mtx *>(d_ctrans.get())->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_NEAR(d_trans, trans, 0.0);
+    ASSERT_TRUE(d_trans->is_sorted_by_column_index());
 }
 
 
@@ -707,56 +704,54 @@ TEST_F(Csr, MoveToHybridIsEquivalentToRef)
 TEST_F(Csr, IsRowPermutable)
 {
     set_up_apply_data(std::make_shared<Mtx::classical>());
-    auto r_permute = gko::share(mtx->row_permute(rpermute_idxs.get()));
-    auto dr_permute = gko::share(dmtx->row_permute(drpermute_idxs.get()));
 
-    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(r_permute),
-                               gko::as<Mtx>(dr_permute));
-    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(r_permute), gko::as<Mtx>(dr_permute), 0);
+    auto r_permute = gko::as<Mtx>(mtx->row_permute(rpermute_idxs.get()));
+    auto dr_permute = gko::as<Mtx>(dmtx->row_permute(drpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(r_permute, dr_permute);
+    GKO_ASSERT_MTX_NEAR(r_permute, dr_permute, 0);
 }
 
 
 TEST_F(Csr, IsColPermutable)
 {
     set_up_apply_data(std::make_shared<Mtx::classical>());
-    auto c_permute = gko::share(mtx->column_permute(cpermute_idxs.get()));
-    auto dc_permute = gko::share(dmtx->column_permute(dcpermute_idxs.get()));
 
-    ASSERT_TRUE(gko::as<Mtx>(dc_permute)->is_sorted_by_column_index());
-    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(c_permute),
-                               gko::as<Mtx>(dc_permute));
-    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(c_permute), gko::as<Mtx>(dc_permute), 0);
+    auto c_permute = gko::as<Mtx>(mtx->column_permute(cpermute_idxs.get()));
+    auto dc_permute = gko::as<Mtx>(dmtx->column_permute(dcpermute_idxs.get()));
+
+    ASSERT_TRUE(dc_permute->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_EQ_SPARSITY(c_permute, dc_permute);
+    GKO_ASSERT_MTX_NEAR(c_permute, dc_permute, 0);
 }
 
 
 TEST_F(Csr, IsInverseRowPermutable)
 {
     set_up_apply_data(std::make_shared<Mtx::classical>());
+
     auto inverse_r_permute =
-        gko::share(mtx->inverse_row_permute(rpermute_idxs.get()));
+        gko::as<Mtx>(mtx->inverse_row_permute(rpermute_idxs.get()));
     auto d_inverse_r_permute =
-        gko::share(dmtx->inverse_row_permute(drpermute_idxs.get()));
+        gko::as<Mtx>(dmtx->inverse_row_permute(drpermute_idxs.get()));
 
-    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(inverse_r_permute),
-                               gko::as<Mtx>(d_inverse_r_permute));
-    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(inverse_r_permute),
-                        gko::as<Mtx>(d_inverse_r_permute), 0);
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse_r_permute, d_inverse_r_permute);
+    GKO_ASSERT_MTX_NEAR(inverse_r_permute, d_inverse_r_permute, 0);
 }
 
 
 TEST_F(Csr, IsInverseColPermutable)
 {
     set_up_apply_data(std::make_shared<Mtx::classical>());
+
     auto inverse_c_permute =
-        gko::share(mtx->inverse_column_permute(cpermute_idxs.get()));
+        gko::as<Mtx>(mtx->inverse_column_permute(cpermute_idxs.get()));
     auto d_inverse_c_permute =
-        gko::share(dmtx->inverse_column_permute(dcpermute_idxs.get()));
+        gko::as<Mtx>(dmtx->inverse_column_permute(dcpermute_idxs.get()));
 
-    ASSERT_TRUE(gko::as<Mtx>(d_inverse_c_permute)->is_sorted_by_column_index());
-    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(inverse_c_permute),
-                               gko::as<Mtx>(d_inverse_c_permute));
-    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(inverse_c_permute),
-                        gko::as<Mtx>(d_inverse_c_permute), 0);
+    ASSERT_TRUE(d_inverse_c_permute->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse_c_permute, d_inverse_c_permute);
+    GKO_ASSERT_MTX_NEAR(inverse_c_permute, d_inverse_c_permute, 0);
 }
 
 
diff --git a/omp/matrix/csr_kernels.cpp b/omp/matrix/csr_kernels.cpp
index 32007104cc1..0f681fa409b 100644
--- a/omp/matrix/csr_kernels.cpp
+++ b/omp/matrix/csr_kernels.cpp
@@ -604,7 +604,7 @@ void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
                         IndexType *inv_permutation)
 {
 #pragma omp parallel for
-    for (size_type i = 0; i < size; ++i) {
+    for (IndexType i = 0; i < static_cast<IndexType>(size); ++i) {
         inv_permutation[permutation_indices[i]] = i;
     }
 }
@@ -624,7 +624,6 @@ void row_permute(std::shared_ptr<const OmpExecutor> exec, const IndexType *perm,
     auto rp_col_idxs = row_permuted->get_col_idxs();
     auto rp_vals = row_permuted->get_values();
     size_type num_rows = orig->get_size()[0];
-    size_type num_nnz = orig->get_num_stored_elements();
 
 #pragma omp parallel for
     for (size_type row = 0; row < num_rows; ++row) {
@@ -664,7 +663,6 @@ void inverse_row_permute(std::shared_ptr<const OmpExecutor> exec,
     auto rp_col_idxs = row_permuted->get_col_idxs();
     auto rp_vals = row_permuted->get_values();
     size_type num_rows = orig->get_size()[0];
-    size_type num_nnz = orig->get_num_stored_elements();
 
 #pragma omp parallel for
     for (size_type row = 0; row < num_rows; ++row) {
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index 0dcfaac7844..064a129b9eb 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -325,12 +325,11 @@ TEST_F(Csr, TransposeIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto trans = mtx->transpose();
-    auto d_trans = dmtx->transpose();
+    auto trans = gko::as<Mtx>(mtx->transpose());
+    auto d_trans = gko::as<Mtx>(dmtx->transpose());
 
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(d_trans.get()),
-                        static_cast<Mtx *>(trans.get()), 0.0);
-    ASSERT_TRUE(static_cast<Mtx *>(d_trans.get())->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_NEAR(d_trans, trans, 0.0);
+    ASSERT_TRUE(d_trans->is_sorted_by_column_index());
 }
 
 
@@ -338,13 +337,11 @@ TEST_F(Csr, ConjugateTransposeIsEquivalentToRef)
 {
     set_up_apply_data();
 
-    auto trans = complex_mtx->conj_transpose();
-    auto d_trans = complex_dmtx->conj_transpose();
+    auto trans = gko::as<ComplexMtx>(complex_mtx->conj_transpose());
+    auto d_trans = gko::as<ComplexMtx>(complex_dmtx->conj_transpose());
 
-    GKO_ASSERT_MTX_NEAR(static_cast<ComplexMtx *>(d_trans.get()),
-                        static_cast<ComplexMtx *>(trans.get()), 0.0);
-    ASSERT_TRUE(
-        static_cast<ComplexMtx *>(d_trans.get())->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_NEAR(d_trans, trans, 0.0);
+    ASSERT_TRUE(d_trans->is_sorted_by_column_index());
 }
 
 
@@ -476,56 +473,54 @@ TEST_F(Csr, MoveToHybridIsEquivalentToRef)
 TEST_F(Csr, IsRowPermutable)
 {
     set_up_apply_data();
-    auto r_permute = gko::share(mtx->row_permute(rpermute_idxs.get()));
-    auto dr_permute = gko::share(dmtx->row_permute(drpermute_idxs.get()));
 
-    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(r_permute),
-                               gko::as<Mtx>(dr_permute));
-    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(r_permute), gko::as<Mtx>(dr_permute), 0);
+    auto r_permute = gko::as<Mtx>(mtx->row_permute(rpermute_idxs.get()));
+    auto dr_permute = gko::as<Mtx>(dmtx->row_permute(drpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(r_permute, dr_permute);
+    GKO_ASSERT_MTX_NEAR(r_permute, dr_permute, 0);
 }
 
 
 TEST_F(Csr, IsColPermutable)
 {
     set_up_apply_data();
-    auto c_permute = gko::share(mtx->column_permute(cpermute_idxs.get()));
-    auto dc_permute = gko::share(dmtx->column_permute(dcpermute_idxs.get()));
 
-    ASSERT_TRUE(gko::as<Mtx>(dc_permute)->is_sorted_by_column_index());
-    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(c_permute),
-                               gko::as<Mtx>(dc_permute));
-    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(c_permute), gko::as<Mtx>(dc_permute), 0);
+    auto c_permute = gko::as<Mtx>(mtx->column_permute(cpermute_idxs.get()));
+    auto dc_permute = gko::as<Mtx>(dmtx->column_permute(dcpermute_idxs.get()));
+
+    ASSERT_TRUE(dc_permute->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_EQ_SPARSITY(c_permute, dc_permute);
+    GKO_ASSERT_MTX_NEAR(c_permute, dc_permute, 0);
 }
 
 
 TEST_F(Csr, IsInverseRowPermutable)
 {
     set_up_apply_data();
+
     auto inverse_r_permute =
-        gko::share(mtx->inverse_row_permute(rpermute_idxs.get()));
+        gko::as<Mtx>(mtx->inverse_row_permute(rpermute_idxs.get()));
     auto d_inverse_r_permute =
-        gko::share(dmtx->inverse_row_permute(drpermute_idxs.get()));
+        gko::as<Mtx>(dmtx->inverse_row_permute(drpermute_idxs.get()));
 
-    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(inverse_r_permute),
-                               gko::as<Mtx>(d_inverse_r_permute));
-    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(inverse_r_permute),
-                        gko::as<Mtx>(d_inverse_r_permute), 0);
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse_r_permute, d_inverse_r_permute);
+    GKO_ASSERT_MTX_NEAR(inverse_r_permute, d_inverse_r_permute, 0);
 }
 
 
 TEST_F(Csr, IsInverseColPermutable)
 {
     set_up_apply_data();
+
     auto inverse_c_permute =
-        gko::share(mtx->inverse_column_permute(cpermute_idxs.get()));
+        gko::as<Mtx>(mtx->inverse_column_permute(cpermute_idxs.get()));
     auto d_inverse_c_permute =
-        gko::share(dmtx->inverse_column_permute(dcpermute_idxs.get()));
+        gko::as<Mtx>(dmtx->inverse_column_permute(dcpermute_idxs.get()));
 
-    ASSERT_TRUE(gko::as<Mtx>(d_inverse_c_permute)->is_sorted_by_column_index());
-    GKO_ASSERT_MTX_EQ_SPARSITY(gko::as<Mtx>(inverse_c_permute),
-                               gko::as<Mtx>(d_inverse_c_permute));
-    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(inverse_c_permute),
-                        gko::as<Mtx>(d_inverse_c_permute), 0);
+    ASSERT_TRUE(d_inverse_c_permute->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse_c_permute, d_inverse_c_permute);
+    GKO_ASSERT_MTX_NEAR(inverse_c_permute, d_inverse_c_permute, 0);
 }
 
 
diff --git a/reference/matrix/csr_kernels.cpp b/reference/matrix/csr_kernels.cpp
index 1be2ae79cb5..b22d7e3d66e 100644
--- a/reference/matrix/csr_kernels.cpp
+++ b/reference/matrix/csr_kernels.cpp
@@ -694,7 +694,7 @@ void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
                         size_type size, const IndexType *permutation_indices,
                         IndexType *inv_permutation)
 {
-    for (size_type i = 0; i < size; ++i) {
+    for (IndexType i = 0; i < static_cast<IndexType>(size); ++i) {
         inv_permutation[permutation_indices[i]] = i;
     }
 }
@@ -715,7 +715,6 @@ void row_permute(std::shared_ptr<const ReferenceExecutor> exec,
     auto rp_col_idxs = row_permuted->get_col_idxs();
     auto rp_vals = row_permuted->get_values();
     size_type num_rows = orig->get_size()[0];
-    size_type num_nnz = orig->get_num_stored_elements();
 
     for (size_type row = 0; row < num_rows; ++row) {
         auto src_row = perm[row];
@@ -751,7 +750,6 @@ void inverse_row_permute(std::shared_ptr<const ReferenceExecutor> exec,
     auto rp_col_idxs = row_permuted->get_col_idxs();
     auto rp_vals = row_permuted->get_values();
     size_type num_rows = orig->get_size()[0];
-    size_type num_nnz = orig->get_num_stored_elements();
 
     for (size_type row = 0; row < num_rows; ++row) {
         auto src_row = row;
diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp
index 3cedeba21d4..dfeb1b7ace3 100644
--- a/reference/test/matrix/csr_kernels.cpp
+++ b/reference/test/matrix/csr_kernels.cpp
@@ -1042,8 +1042,7 @@ TYPED_TEST(Csr, SquareMtxIsTransposable)
                  {0.0, 1.5, 2.0}}, this->exec);
     // clang-format on
 
-    auto trans = mtx2->transpose();
-    auto trans_as_csr = static_cast<Csr *>(trans.get());
+    auto trans_as_csr = gko::as<Csr>(mtx2->transpose());
 
     // clang-format off
     GKO_ASSERT_MTX_NEAR(trans_as_csr,
@@ -1057,8 +1056,7 @@ TYPED_TEST(Csr, SquareMtxIsTransposable)
 TYPED_TEST(Csr, NonSquareMtxIsTransposable)
 {
     using Csr = typename TestFixture::Mtx;
-    auto trans = this->mtx->transpose();
-    auto trans_as_csr = static_cast<Csr *>(trans.get());
+    auto trans_as_csr = gko::as<Csr>(this->mtx->transpose());
 
     // clang-format off
     GKO_ASSERT_MTX_NEAR(trans_as_csr,
@@ -1080,9 +1078,8 @@ TYPED_TEST(Csr, SquareMatrixIsRowPermutable)
     // clang-format on
     gko::Array<index_type> permute_idxs{this->exec, {1, 2, 0}};
 
-    auto row_permute = p_mtx->row_permute(&permute_idxs);
+    auto row_permute_csr = gko::as<Csr>(p_mtx->row_permute(&permute_idxs));
 
-    auto row_permute_csr = static_cast<Csr *>(row_permute.get());
     // clang-format off
     GKO_ASSERT_MTX_NEAR(row_permute_csr,
                         l({{0.0, 5.0, 0.0},
@@ -1103,9 +1100,8 @@ TYPED_TEST(Csr, NonSquareMatrixIsRowPermutable)
     // clang-format on
     gko::Array<index_type> permute_idxs{this->exec, {1, 0}};
 
-    auto row_permute = p_mtx->row_permute(&permute_idxs);
+    auto row_permute_csr = gko::as<Csr>(p_mtx->row_permute(&permute_idxs));
 
-    auto row_permute_csr = static_cast<Csr *>(row_permute.get());
     // clang-format off
     GKO_ASSERT_MTX_NEAR(row_permute_csr,
                         l({{0.0, 5.0, 0.0},
@@ -1126,9 +1122,8 @@ TYPED_TEST(Csr, SquareMatrixIsColPermutable)
     // clang-format on
     gko::Array<index_type> permute_idxs{this->exec, {1, 2, 0}};
 
-    auto c_permute = p_mtx->column_permute(&permute_idxs);
+    auto c_permute_csr = gko::as<Csr>(p_mtx->column_permute(&permute_idxs));
 
-    auto c_permute_csr = static_cast<Csr *>(c_permute.get());
     // clang-format off
     GKO_ASSERT_MTX_NEAR(c_permute_csr,
                         l({{3.0, 2.0, 1.0},
@@ -1149,9 +1144,8 @@ TYPED_TEST(Csr, NonSquareMatrixIsColPermutable)
     // clang-format on
     gko::Array<index_type> permute_idxs{this->exec, {1, 2, 0}};
 
-    auto c_permute = p_mtx->column_permute(&permute_idxs);
+    auto c_permute_csr = gko::as<Csr>(p_mtx->column_permute(&permute_idxs));
 
-    auto c_permute_csr = static_cast<Csr *>(c_permute.get());
     // clang-format off
     GKO_ASSERT_MTX_NEAR(c_permute_csr,
                         l({{0.0, 2.0, 1.0},
@@ -1172,11 +1166,9 @@ TYPED_TEST(Csr, SquareMatrixIsInverseRowPermutable)
     // clang-format on
     gko::Array<index_type> inverse_permute_idxs{this->exec, {1, 2, 0}};
 
-    auto inverse_row_permute =
-        inverse_p_mtx->inverse_row_permute(&inverse_permute_idxs);
-
     auto inverse_row_permute_csr =
-        static_cast<Csr *>(inverse_row_permute.get());
+        gko::as<Csr>(inverse_p_mtx->inverse_row_permute(&inverse_permute_idxs));
+
     // clang-format off
     GKO_ASSERT_MTX_NEAR(inverse_row_permute_csr,
                         l({{0.0, 1.5, 2.0},
@@ -1197,11 +1189,9 @@ TYPED_TEST(Csr, NonSquareMatrixIsInverseRowPermutable)
     // clang-format on
     gko::Array<index_type> inverse_permute_idxs{this->exec, {1, 0}};
 
-    auto inverse_row_permute =
-        inverse_p_mtx->inverse_row_permute(&inverse_permute_idxs);
-
     auto inverse_row_permute_csr =
-        static_cast<Csr *>(inverse_row_permute.get());
+        gko::as<Csr>(inverse_p_mtx->inverse_row_permute(&inverse_permute_idxs));
+
     // clang-format off
     GKO_ASSERT_MTX_NEAR(inverse_row_permute_csr,
                         l({{0.0, 5.0, 0.0},
@@ -1222,10 +1212,9 @@ TYPED_TEST(Csr, SquareMatrixIsInverseColPermutable)
     // clang-format on
     gko::Array<index_type> inverse_permute_idxs{this->exec, {1, 2, 0}};
 
-    auto inverse_c_permute =
-        inverse_p_mtx->inverse_column_permute(&inverse_permute_idxs);
+    auto inverse_c_permute_csr = gko::as<Csr>(
+        inverse_p_mtx->inverse_column_permute(&inverse_permute_idxs));
 
-    auto inverse_c_permute_csr = static_cast<Csr *>(inverse_c_permute.get());
     // clang-format off
     GKO_ASSERT_MTX_NEAR(inverse_c_permute_csr,
                         l({{2.0, 1.0, 3.0},
@@ -1246,10 +1235,9 @@ TYPED_TEST(Csr, NonSquareMatrixIsInverseColPermutable)
     // clang-format on
     gko::Array<index_type> inverse_permute_idxs{this->exec, {1, 2, 0}};
 
-    auto inverse_c_permute =
-        inverse_p_mtx->inverse_column_permute(&inverse_permute_idxs);
+    auto inverse_c_permute_csr = gko::as<Csr>(
+        inverse_p_mtx->inverse_column_permute(&inverse_permute_idxs));
 
-    auto inverse_c_permute_csr = static_cast<Csr *>(inverse_c_permute.get());
     // clang-format off
     GKO_ASSERT_MTX_NEAR(inverse_c_permute_csr,
                         l({{2.0, 1.0, 3.0},
@@ -1418,8 +1406,7 @@ TYPED_TEST(CsrComplex, MtxIsConjugateTransposable)
          {T{0.0, 0.0}, T{0.0, 1.5}, T{2.0,0.0}}}, exec);
     // clang-format on
 
-    auto trans = mtx2->conj_transpose();
-    auto trans_as_csr = static_cast<Csr *>(trans.get());
+    auto trans_as_csr = gko::as<Csr>(mtx2->conj_transpose());
 
     // clang-format off
     GKO_ASSERT_MTX_NEAR(trans_as_csr,

From 4ab00a9dcba3f49e461961ffcd1ad2770c7d73b1 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 3 Nov 2020 13:56:52 +0100
Subject: [PATCH 0085/1008] hacky fix for rocThrust compilation issues

---
 benchmark/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
index f3a05ab5c9e..62cfb1b2538 100644
--- a/benchmark/CMakeLists.txt
+++ b/benchmark/CMakeLists.txt
@@ -25,6 +25,8 @@ function(ginkgo_benchmark_hipsp_linops name)
     target_compile_definitions("${name}" PRIVATE HAS_HIP=1)
     EXECUTE_PROCESS(COMMAND ${HIP_PATH}/bin/hipconfig --cpp_config OUTPUT_VARIABLE HIP_CXX_FLAGS)
     set_target_properties("${name}" PROPERTIES COMPILE_FLAGS ${HIP_CXX_FLAGS})
+    # use Thrust C++ device just for compilation, we don't use thrust::complex in the benchmarks
+    target_compile_definitions("${name}" PUBLIC -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CPP)
     # for some reason, HIP creates a dependency on Threads::Threads here, so we
     # need to find it
     find_package(Threads REQUIRED)

From 3b77aac28ab5d6593d7c4dbdaa36a3ff861d96c8 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 14 Oct 2020 17:59:05 +0200
Subject: [PATCH 0086/1008] Use a newer GTEST version.

+ `TYPED_TEST_CASE` is deprecated for the newer `TYPED_TEST_SUITE`
  version.
+ We can now use `GTEST_SKIP()` to skip specific tests.
---
 core/test/base/array.cpp                          | 2 +-
 core/test/base/combination.cpp                    | 2 +-
 core/test/base/composition.cpp                    | 2 +-
 core/test/base/iterator_factory.cpp               | 2 +-
 core/test/base/mtx_io.cpp                         | 4 ++--
 core/test/factorization/par_ilu.cpp               | 2 +-
 core/test/log/convergence.cpp                     | 2 +-
 core/test/log/papi.cpp                            | 2 +-
 core/test/log/stream.cpp                          | 2 +-
 core/test/matrix/coo.cpp                          | 2 +-
 core/test/matrix/coo_builder.cpp                  | 2 +-
 core/test/matrix/csr.cpp                          | 2 +-
 core/test/matrix/csr_builder.cpp                  | 2 +-
 core/test/matrix/dense.cpp                        | 2 +-
 core/test/matrix/diagonal.cpp                     | 2 +-
 core/test/matrix/ell.cpp                          | 2 +-
 core/test/matrix/hybrid.cpp                       | 2 +-
 core/test/matrix/identity.cpp                     | 4 ++--
 core/test/matrix/permutation.cpp                  | 2 +-
 core/test/matrix/sellp.cpp                        | 2 +-
 core/test/matrix/sparsity_csr.cpp                 | 2 +-
 core/test/preconditioner/ilu.cpp                  | 2 +-
 core/test/preconditioner/isai.cpp                 | 2 +-
 core/test/preconditioner/jacobi.cpp               | 4 ++--
 core/test/solver/bicg.cpp                         | 2 +-
 core/test/solver/bicgstab.cpp                     | 2 +-
 core/test/solver/cg.cpp                           | 2 +-
 core/test/solver/cgs.cpp                          | 2 +-
 core/test/solver/fcg.cpp                          | 2 +-
 core/test/solver/gmres.cpp                        | 2 +-
 core/test/solver/ir.cpp                           | 2 +-
 core/test/solver/lower_trs.cpp                    | 2 +-
 core/test/solver/upper_trs.cpp                    | 2 +-
 omp/test/components/fill_array.cpp                | 2 +-
 omp/test/components/prefix_sum.cpp                | 2 +-
 omp/test/factorization/par_ict_kernels.cpp        | 2 +-
 omp/test/factorization/par_ilu_kernels.cpp        | 2 +-
 omp/test/factorization/par_ilut_kernels.cpp       | 2 +-
 omp/test/stop/residual_norm_kernels.cpp           | 6 +++---
 reference/test/base/combination.cpp               | 2 +-
 reference/test/base/composition.cpp               | 2 +-
 reference/test/base/perturbation.cpp              | 2 +-
 reference/test/components/absolute_array.cpp      | 2 +-
 reference/test/components/fill_array.cpp          | 2 +-
 reference/test/components/prefix_sum.cpp          | 2 +-
 reference/test/factorization/par_ict_kernels.cpp  | 2 +-
 reference/test/factorization/par_ilu_kernels.cpp  | 2 +-
 reference/test/factorization/par_ilut_kernels.cpp | 2 +-
 reference/test/log/convergence.cpp                | 2 +-
 reference/test/log/papi.cpp                       | 2 +-
 reference/test/matrix/coo_kernels.cpp             | 4 ++--
 reference/test/matrix/csr_kernels.cpp             | 4 ++--
 reference/test/matrix/dense_kernels.cpp           | 4 ++--
 reference/test/matrix/diagonal_kernels.cpp        | 4 ++--
 reference/test/matrix/ell_kernels.cpp             | 4 ++--
 reference/test/matrix/hybrid_kernels.cpp          | 4 ++--
 reference/test/matrix/identity.cpp                | 2 +-
 reference/test/matrix/permutation.cpp             | 2 +-
 reference/test/matrix/sellp_kernels.cpp           | 4 ++--
 reference/test/matrix/sparsity_csr.cpp            | 2 +-
 reference/test/matrix/sparsity_csr_kernels.cpp    | 2 +-
 reference/test/preconditioner/ilu.cpp             | 2 +-
 reference/test/preconditioner/isai_kernels.cpp    | 2 +-
 reference/test/preconditioner/jacobi.cpp          | 2 +-
 reference/test/preconditioner/jacobi_kernels.cpp  | 2 +-
 reference/test/reorder/rcm.cpp                    | 2 +-
 reference/test/solver/bicg_kernels.cpp            | 2 +-
 reference/test/solver/bicgstab_kernels.cpp        | 2 +-
 reference/test/solver/cg_kernels.cpp              | 2 +-
 reference/test/solver/cgs_kernels.cpp             | 2 +-
 reference/test/solver/fcg_kernels.cpp             | 2 +-
 reference/test/solver/gmres_kernels.cpp           | 2 +-
 reference/test/solver/ir_kernels.cpp              | 2 +-
 reference/test/solver/lower_trs.cpp               | 2 +-
 reference/test/solver/lower_trs_kernels.cpp       | 2 +-
 reference/test/solver/upper_trs.cpp               | 2 +-
 reference/test/solver/upper_trs_kernels.cpp       | 2 +-
 reference/test/stop/residual_norm_kernels.cpp     | 6 +++---
 reference/test/utils/assertions_test.cpp          | 2 +-
 third_party/gtest/CMakeLists.txt                  | 8 ++++----
 80 files changed, 97 insertions(+), 97 deletions(-)

diff --git a/core/test/base/array.cpp b/core/test/base/array.cpp
index b1e04dd2f39..ca6db900f94 100644
--- a/core/test/base/array.cpp
+++ b/core/test/base/array.cpp
@@ -70,7 +70,7 @@ class Array : public ::testing::Test {
     gko::Array<T> x;
 };
 
-TYPED_TEST_CASE(Array, gko::test::ValueAndIndexTypes);
+TYPED_TEST_SUITE(Array, gko::test::ValueAndIndexTypes);
 
 
 TYPED_TEST(Array, CanBeCreatedWithoutAnExecutor)
diff --git a/core/test/base/combination.cpp b/core/test/base/combination.cpp
index 8c2eaf35b6f..22d41d5a0cd 100644
--- a/core/test/base/combination.cpp
+++ b/core/test/base/combination.cpp
@@ -74,7 +74,7 @@ class Combination : public ::testing::Test {
     std::vector<std::shared_ptr<gko::LinOp>> coefficients;
 };
 
-TYPED_TEST_CASE(Combination, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Combination, gko::test::ValueTypes);
 
 
 TYPED_TEST(Combination, CanBeEmpty)
diff --git a/core/test/base/composition.cpp b/core/test/base/composition.cpp
index aa9df458456..aca88922238 100644
--- a/core/test/base/composition.cpp
+++ b/core/test/base/composition.cpp
@@ -72,7 +72,7 @@ class Composition : public ::testing::Test {
     std::vector<std::shared_ptr<gko::LinOp>> operators;
 };
 
-TYPED_TEST_CASE(Composition, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Composition, gko::test::ValueTypes);
 
 
 TYPED_TEST(Composition, CanBeEmpty)
diff --git a/core/test/base/iterator_factory.cpp b/core/test/base/iterator_factory.cpp
index b2b873a627a..058a52ae58a 100644
--- a/core/test/base/iterator_factory.cpp
+++ b/core/test/base/iterator_factory.cpp
@@ -96,7 +96,7 @@ class IteratorFactory : public ::testing::Test {
     const std::vector<value_type> ordered_value;
 };
 
-TYPED_TEST_CASE(IteratorFactory, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(IteratorFactory, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(IteratorFactory, EmptyIterator)
diff --git a/core/test/base/mtx_io.cpp b/core/test/base/mtx_io.cpp
index a5a17beeefc..e8f26d73f19 100644
--- a/core/test/base/mtx_io.cpp
+++ b/core/test/base/mtx_io.cpp
@@ -640,7 +640,7 @@ class RealDummyLinOpTest : public ::testing::Test {
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
 };
 
-TYPED_TEST_CASE(RealDummyLinOpTest, gko::test::RealValueIndexTypes);
+TYPED_TEST_SUITE(RealDummyLinOpTest, gko::test::RealValueIndexTypes);
 
 
 TYPED_TEST(RealDummyLinOpTest, ReadsLinOpFromStream)
@@ -713,7 +713,7 @@ class ComplexDummyLinOpTest : public ::testing::Test {
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
 };
 
-TYPED_TEST_CASE(ComplexDummyLinOpTest, gko::test::ComplexValueIndexTypes);
+TYPED_TEST_SUITE(ComplexDummyLinOpTest, gko::test::ComplexValueIndexTypes);
 
 
 TYPED_TEST(ComplexDummyLinOpTest, ReadsLinOpFromStream)
diff --git a/core/test/factorization/par_ilu.cpp b/core/test/factorization/par_ilu.cpp
index 75c714fcab1..7d7226e7ca5 100644
--- a/core/test/factorization/par_ilu.cpp
+++ b/core/test/factorization/par_ilu.cpp
@@ -60,7 +60,7 @@ class ParIlu : public ::testing::Test {
     std::shared_ptr<const gko::ReferenceExecutor> ref;
 };
 
-TYPED_TEST_CASE(ParIlu, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(ParIlu, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(ParIlu, SetIterations)
diff --git a/core/test/log/convergence.cpp b/core/test/log/convergence.cpp
index bb05007817b..413c6dcd2a0 100644
--- a/core/test/log/convergence.cpp
+++ b/core/test/log/convergence.cpp
@@ -47,7 +47,7 @@ namespace {
 template <typename T>
 class Convergence : public ::testing::Test {};
 
-TYPED_TEST_CASE(Convergence, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Convergence, gko::test::ValueTypes);
 
 
 TYPED_TEST(Convergence, CanGetData)
diff --git a/core/test/log/papi.cpp b/core/test/log/papi.cpp
index d318a29f228..1b8dba4bfd7 100644
--- a/core/test/log/papi.cpp
+++ b/core/test/log/papi.cpp
@@ -119,7 +119,7 @@ class Papi : public ::testing::Test {
     int eventset;
 };
 
-TYPED_TEST_CASE(Papi, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Papi, gko::test::ValueTypes);
 
 
 TYPED_TEST(Papi, CatchesAllocationStarted)
diff --git a/core/test/log/stream.cpp b/core/test/log/stream.cpp
index 163a54fd74a..3e11f1f9669 100644
--- a/core/test/log/stream.cpp
+++ b/core/test/log/stream.cpp
@@ -59,7 +59,7 @@ constexpr int num_iters = 10;
 template <typename T>
 class Stream : public ::testing::Test {};
 
-TYPED_TEST_CASE(Stream, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Stream, gko::test::ValueTypes);
 
 
 TYPED_TEST(Stream, CatchesAllocationStarted)
diff --git a/core/test/matrix/coo.cpp b/core/test/matrix/coo.cpp
index 978f9ffc17a..7035b61ea74 100644
--- a/core/test/matrix/coo.cpp
+++ b/core/test/matrix/coo.cpp
@@ -107,7 +107,7 @@ class Coo : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Coo, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Coo, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Coo, KnowsItsSize)
diff --git a/core/test/matrix/coo_builder.cpp b/core/test/matrix/coo_builder.cpp
index de5844b0bbe..1027d84067f 100644
--- a/core/test/matrix/coo_builder.cpp
+++ b/core/test/matrix/coo_builder.cpp
@@ -63,7 +63,7 @@ class CooBuilder : public ::testing::Test {
     std::unique_ptr<Mtx> mtx;
 };
 
-TYPED_TEST_CASE(CooBuilder, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(CooBuilder, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(CooBuilder, ReturnsCorrectArrays)
diff --git a/core/test/matrix/csr.cpp b/core/test/matrix/csr.cpp
index 79797fde20d..56e88708b08 100644
--- a/core/test/matrix/csr.cpp
+++ b/core/test/matrix/csr.cpp
@@ -111,7 +111,7 @@ class Csr : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Csr, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Csr, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Csr, KnowsItsSize)
diff --git a/core/test/matrix/csr_builder.cpp b/core/test/matrix/csr_builder.cpp
index 9a1bfb6eb5f..30336778eb3 100644
--- a/core/test/matrix/csr_builder.cpp
+++ b/core/test/matrix/csr_builder.cpp
@@ -64,7 +64,7 @@ class CsrBuilder : public ::testing::Test {
     std::unique_ptr<Mtx> mtx;
 };
 
-TYPED_TEST_CASE(CsrBuilder, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(CsrBuilder, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(CsrBuilder, ReturnsCorrectArrays)
diff --git a/core/test/matrix/dense.cpp b/core/test/matrix/dense.cpp
index 9439687d763..2226f8faeec 100644
--- a/core/test/matrix/dense.cpp
+++ b/core/test/matrix/dense.cpp
@@ -80,7 +80,7 @@ class Dense : public ::testing::Test {
     std::unique_ptr<gko::matrix::Dense<value_type>> mtx;
 };
 
-TYPED_TEST_CASE(Dense, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Dense, gko::test::ValueTypes);
 
 
 TYPED_TEST(Dense, CanBeEmpty)
diff --git a/core/test/matrix/diagonal.cpp b/core/test/matrix/diagonal.cpp
index 3fafc05d83e..023d319a71e 100644
--- a/core/test/matrix/diagonal.cpp
+++ b/core/test/matrix/diagonal.cpp
@@ -77,7 +77,7 @@ class Diagonal : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Diagonal, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Diagonal, gko::test::ValueTypes);
 
 
 TYPED_TEST(Diagonal, KnowsItsSize)
diff --git a/core/test/matrix/ell.cpp b/core/test/matrix/ell.cpp
index 9fe00a82621..37cfe11e6d5 100644
--- a/core/test/matrix/ell.cpp
+++ b/core/test/matrix/ell.cpp
@@ -110,7 +110,7 @@ class Ell : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Ell, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Ell, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Ell, KnowsItsSize)
diff --git a/core/test/matrix/hybrid.cpp b/core/test/matrix/hybrid.cpp
index e882b2cde96..8d2fbe8efaa 100644
--- a/core/test/matrix/hybrid.cpp
+++ b/core/test/matrix/hybrid.cpp
@@ -127,7 +127,7 @@ class Hybrid : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Hybrid, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Hybrid, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Hybrid, KnowsItsSize)
diff --git a/core/test/matrix/identity.cpp b/core/test/matrix/identity.cpp
index b145cf35705..a224e373fce 100644
--- a/core/test/matrix/identity.cpp
+++ b/core/test/matrix/identity.cpp
@@ -58,7 +58,7 @@ class Identity : public ::testing::Test {
     std::shared_ptr<const gko::Executor> exec;
 };
 
-TYPED_TEST_CASE(Identity, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Identity, gko::test::ValueTypes);
 
 
 TYPED_TEST(Identity, CanBeEmpty)
@@ -132,7 +132,7 @@ class IdentityFactory : public ::testing::Test {
     using value_type = T;
 };
 
-TYPED_TEST_CASE(IdentityFactory, gko::test::ValueTypes);
+TYPED_TEST_SUITE(IdentityFactory, gko::test::ValueTypes);
 
 
 TYPED_TEST(IdentityFactory, CanGenerateIdentityMatrix)
diff --git a/core/test/matrix/permutation.cpp b/core/test/matrix/permutation.cpp
index c64f39fb3e2..2c4f5a33214 100644
--- a/core/test/matrix/permutation.cpp
+++ b/core/test/matrix/permutation.cpp
@@ -86,7 +86,7 @@ class Permutation : public ::testing::Test {
     std::unique_ptr<gko::matrix::Permutation<i_type>> mtx;
 };
 
-TYPED_TEST_CASE(Permutation, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Permutation, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Permutation, CanBeEmpty)
diff --git a/core/test/matrix/sellp.cpp b/core/test/matrix/sellp.cpp
index 7b9439e820c..276f93dc370 100644
--- a/core/test/matrix/sellp.cpp
+++ b/core/test/matrix/sellp.cpp
@@ -138,7 +138,7 @@ class Sellp : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Sellp, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Sellp, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Sellp, KnowsItsSize)
diff --git a/core/test/matrix/sparsity_csr.cpp b/core/test/matrix/sparsity_csr.cpp
index 7e26fee9c88..754e159f8cd 100644
--- a/core/test/matrix/sparsity_csr.cpp
+++ b/core/test/matrix/sparsity_csr.cpp
@@ -107,7 +107,7 @@ class SparsityCsr : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(SparsityCsr, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(SparsityCsr, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(SparsityCsr, KnowsItsSize)
diff --git a/core/test/preconditioner/ilu.cpp b/core/test/preconditioner/ilu.cpp
index 2103e00d958..66f858d1d89 100644
--- a/core/test/preconditioner/ilu.cpp
+++ b/core/test/preconditioner/ilu.cpp
@@ -69,7 +69,7 @@ class IluFactory : public ::testing::Test {
     std::shared_ptr<typename u_solver_type::Factory> u_factory;
 };
 
-TYPED_TEST_CASE(IluFactory, gko::test::ValueTypes);
+TYPED_TEST_SUITE(IluFactory, gko::test::ValueTypes);
 
 
 TYPED_TEST(IluFactory, KnowsItsExecutor)
diff --git a/core/test/preconditioner/isai.cpp b/core/test/preconditioner/isai.cpp
index 1a549bca874..f72e4f5ade5 100644
--- a/core/test/preconditioner/isai.cpp
+++ b/core/test/preconditioner/isai.cpp
@@ -86,7 +86,7 @@ class IsaiFactory : public ::testing::Test {
     std::unique_ptr<typename UpperIsai::Factory> upper_isai_factory;
 };
 
-TYPED_TEST_CASE(IsaiFactory, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(IsaiFactory, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(IsaiFactory, KnowsItsExecutor)
diff --git a/core/test/preconditioner/jacobi.cpp b/core/test/preconditioner/jacobi.cpp
index 0af14acbabf..36308f882a9 100644
--- a/core/test/preconditioner/jacobi.cpp
+++ b/core/test/preconditioner/jacobi.cpp
@@ -75,7 +75,7 @@ class JacobiFactory : public ::testing::Test {
     std::shared_ptr<gko::matrix::Csr<value_type, index_type>> mtx;
 };
 
-TYPED_TEST_CASE(JacobiFactory, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(JacobiFactory, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(JacobiFactory, KnowsItsExecutor)
@@ -156,7 +156,7 @@ class BlockInterleavedStorageScheme : public ::testing::Test {
                                                                         2};
 };
 
-TYPED_TEST_CASE(BlockInterleavedStorageScheme, gko::test::IndexTypes);
+TYPED_TEST_SUITE(BlockInterleavedStorageScheme, gko::test::IndexTypes);
 
 
 TYPED_TEST(BlockInterleavedStorageScheme, ComputesStorageSpace)
diff --git a/core/test/solver/bicg.cpp b/core/test/solver/bicg.cpp
index 5d252091715..f0e5af4bc4d 100644
--- a/core/test/solver/bicg.cpp
+++ b/core/test/solver/bicg.cpp
@@ -91,7 +91,7 @@ class Bicg : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Bicg, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Bicg, gko::test::ValueTypes);
 
 
 TYPED_TEST(Bicg, BicgFactoryKnowsItsExecutor)
diff --git a/core/test/solver/bicgstab.cpp b/core/test/solver/bicgstab.cpp
index 7be7564d00f..f871e33ff0c 100644
--- a/core/test/solver/bicgstab.cpp
+++ b/core/test/solver/bicgstab.cpp
@@ -89,7 +89,7 @@ class Bicgstab : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Bicgstab, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Bicgstab, gko::test::ValueTypes);
 
 
 TYPED_TEST(Bicgstab, BicgstabFactoryKnowsItsExecutor)
diff --git a/core/test/solver/cg.cpp b/core/test/solver/cg.cpp
index 0efb5af1bc8..150625b0385 100644
--- a/core/test/solver/cg.cpp
+++ b/core/test/solver/cg.cpp
@@ -91,7 +91,7 @@ class Cg : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Cg, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Cg, gko::test::ValueTypes);
 
 
 TYPED_TEST(Cg, CgFactoryKnowsItsExecutor)
diff --git a/core/test/solver/cgs.cpp b/core/test/solver/cgs.cpp
index 3acaf785bdc..d6726bdc94b 100644
--- a/core/test/solver/cgs.cpp
+++ b/core/test/solver/cgs.cpp
@@ -91,7 +91,7 @@ class Cgs : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Cgs, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Cgs, gko::test::ValueTypes);
 
 
 TYPED_TEST(Cgs, CgsFactoryKnowsItsExecutor)
diff --git a/core/test/solver/fcg.cpp b/core/test/solver/fcg.cpp
index 11048379e81..bd588d4b594 100644
--- a/core/test/solver/fcg.cpp
+++ b/core/test/solver/fcg.cpp
@@ -77,7 +77,7 @@ class Fcg : public ::testing::Test {
     std::unique_ptr<gko::LinOp> solver;
 };
 
-TYPED_TEST_CASE(Fcg, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Fcg, gko::test::ValueTypes);
 
 
 TYPED_TEST(Fcg, FcgFactoryKnowsItsExecutor)
diff --git a/core/test/solver/gmres.cpp b/core/test/solver/gmres.cpp
index 1c9bf67b0ca..1d3cf385407 100644
--- a/core/test/solver/gmres.cpp
+++ b/core/test/solver/gmres.cpp
@@ -110,7 +110,7 @@ class Gmres : public ::testing::Test {
 template <typename T>
 constexpr gko::remove_complex<T> Gmres<T>::reduction_factor;
 
-TYPED_TEST_CASE(Gmres, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Gmres, gko::test::ValueTypes);
 
 
 TYPED_TEST(Gmres, GmresFactoryKnowsItsExecutor)
diff --git a/core/test/solver/ir.cpp b/core/test/solver/ir.cpp
index 3e212430e29..6bd864b2a4a 100644
--- a/core/test/solver/ir.cpp
+++ b/core/test/solver/ir.cpp
@@ -91,7 +91,7 @@ class Ir : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Ir, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Ir, gko::test::ValueTypes);
 
 
 TYPED_TEST(Ir, IrFactoryKnowsItsExecutor)
diff --git a/core/test/solver/lower_trs.cpp b/core/test/solver/lower_trs.cpp
index 0153b8aed2d..8effe6c6f3d 100644
--- a/core/test/solver/lower_trs.cpp
+++ b/core/test/solver/lower_trs.cpp
@@ -66,7 +66,7 @@ class LowerTrs : public ::testing::Test {
     std::unique_ptr<typename Solver::Factory> lower_trs_factory;
 };
 
-TYPED_TEST_CASE(LowerTrs, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(LowerTrs, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(LowerTrs, LowerTrsFactoryKnowsItsExecutor)
diff --git a/core/test/solver/upper_trs.cpp b/core/test/solver/upper_trs.cpp
index 10e2f727cc7..ae41a2b65cc 100644
--- a/core/test/solver/upper_trs.cpp
+++ b/core/test/solver/upper_trs.cpp
@@ -66,7 +66,7 @@ class UpperTrs : public ::testing::Test {
     std::unique_ptr<typename Solver::Factory> upper_trs_factory;
 };
 
-TYPED_TEST_CASE(UpperTrs, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(UpperTrs, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(UpperTrs, UpperTrsFactoryKnowsItsExecutor)
diff --git a/omp/test/components/fill_array.cpp b/omp/test/components/fill_array.cpp
index ad657e7e6e2..a9bafc74ad5 100644
--- a/omp/test/components/fill_array.cpp
+++ b/omp/test/components/fill_array.cpp
@@ -71,7 +71,7 @@ class FillArray : public ::testing::Test {
     gko::Array<value_type> dvals;
 };
 
-TYPED_TEST_CASE(FillArray, gko::test::ValueAndIndexTypes);
+TYPED_TEST_SUITE(FillArray, gko::test::ValueAndIndexTypes);
 
 
 TYPED_TEST(FillArray, EqualsReference)
diff --git a/omp/test/components/prefix_sum.cpp b/omp/test/components/prefix_sum.cpp
index 277667b7801..af00ed9977a 100644
--- a/omp/test/components/prefix_sum.cpp
+++ b/omp/test/components/prefix_sum.cpp
@@ -86,7 +86,7 @@ class PrefixSum : public ::testing::Test {
     gko::Array<index_type> dvals;
 };
 
-TYPED_TEST_CASE(PrefixSum, gko::test::IndexTypes);
+TYPED_TEST_SUITE(PrefixSum, gko::test::IndexTypes);
 
 
 TYPED_TEST(PrefixSum, SmallEqualsReference) { this->test(100); }
diff --git a/omp/test/factorization/par_ict_kernels.cpp b/omp/test/factorization/par_ict_kernels.cpp
index 95ab07b4030..a6afed077a6 100644
--- a/omp/test/factorization/par_ict_kernels.cpp
+++ b/omp/test/factorization/par_ict_kernels.cpp
@@ -142,7 +142,7 @@ class ParIct : public ::testing::Test {
     std::unique_ptr<Csr> dmtx_l;
 };
 
-TYPED_TEST_CASE(ParIct, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(ParIct, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(ParIct, KernelAddCandidatesIsEquivalentToRef)
diff --git a/omp/test/factorization/par_ilu_kernels.cpp b/omp/test/factorization/par_ilu_kernels.cpp
index 41ff692b702..583a68c26e9 100644
--- a/omp/test/factorization/par_ilu_kernels.cpp
+++ b/omp/test/factorization/par_ilu_kernels.cpp
@@ -224,7 +224,7 @@ class ParIlu : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(ParIlu, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(ParIlu, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(ParIlu, OmpKernelAddDiagonalElementsSortedEquivalentToRef)
diff --git a/omp/test/factorization/par_ilut_kernels.cpp b/omp/test/factorization/par_ilut_kernels.cpp
index 7af808a258b..a30ffba8e45 100644
--- a/omp/test/factorization/par_ilut_kernels.cpp
+++ b/omp/test/factorization/par_ilut_kernels.cpp
@@ -279,7 +279,7 @@ class ParIlut : public ::testing::Test {
     std::unique_ptr<Csr> dmtx_u;
 };
 
-TYPED_TEST_CASE(ParIlut, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(ParIlut, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(ParIlut, KernelThresholdSelectIsEquivalentToRef)
diff --git a/omp/test/stop/residual_norm_kernels.cpp b/omp/test/stop/residual_norm_kernels.cpp
index 3d33fb59628..082dfd7bc57 100644
--- a/omp/test/stop/residual_norm_kernels.cpp
+++ b/omp/test/stop/residual_norm_kernels.cpp
@@ -64,7 +64,7 @@ class ResidualNormReduction : public ::testing::Test {
     std::shared_ptr<const gko::OmpExecutor> omp_;
 };
 
-TYPED_TEST_CASE(ResidualNormReduction, gko::test::ValueTypes);
+TYPED_TEST_SUITE(ResidualNormReduction, gko::test::ValueTypes);
 
 
 TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoal)
@@ -164,7 +164,7 @@ class RelativeResidualNorm : public ::testing::Test {
     std::shared_ptr<const gko::OmpExecutor> omp_;
 };
 
-TYPED_TEST_CASE(RelativeResidualNorm, gko::test::ValueTypes);
+TYPED_TEST_SUITE(RelativeResidualNorm, gko::test::ValueTypes);
 
 
 TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoal)
@@ -264,7 +264,7 @@ class AbsoluteResidualNorm : public ::testing::Test {
     std::shared_ptr<const gko::OmpExecutor> omp_;
 };
 
-TYPED_TEST_CASE(AbsoluteResidualNorm, gko::test::ValueTypes);
+TYPED_TEST_SUITE(AbsoluteResidualNorm, gko::test::ValueTypes);
 
 
 TYPED_TEST(AbsoluteResidualNorm, WaitsTillResidualGoal)
diff --git a/reference/test/base/combination.cpp b/reference/test/base/combination.cpp
index 830b031e83e..f490433ef8b 100644
--- a/reference/test/base/combination.cpp
+++ b/reference/test/base/combination.cpp
@@ -67,7 +67,7 @@ class Combination : public ::testing::Test {
     std::vector<std::shared_ptr<gko::LinOp>> operators;
 };
 
-TYPED_TEST_CASE(Combination, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Combination, gko::test::ValueTypes);
 
 
 TYPED_TEST(Combination, AppliesToVector)
diff --git a/reference/test/base/composition.cpp b/reference/test/base/composition.cpp
index 76d71734d12..5e381a8a48e 100644
--- a/reference/test/base/composition.cpp
+++ b/reference/test/base/composition.cpp
@@ -108,7 +108,7 @@ class Composition : public ::testing::Test {
     std::shared_ptr<Mtx> product;
 };
 
-TYPED_TEST_CASE(Composition, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Composition, gko::test::ValueTypes);
 
 
 TYPED_TEST(Composition, AppliesSingleToVector)
diff --git a/reference/test/base/perturbation.cpp b/reference/test/base/perturbation.cpp
index fe1cc8692fe..d2bc6a617f5 100644
--- a/reference/test/base/perturbation.cpp
+++ b/reference/test/base/perturbation.cpp
@@ -66,7 +66,7 @@ class Perturbation : public ::testing::Test {
     std::shared_ptr<gko::LinOp> scalar;
 };
 
-TYPED_TEST_CASE(Perturbation, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Perturbation, gko::test::ValueTypes);
 
 
 TYPED_TEST(Perturbation, AppliesToVector)
diff --git a/reference/test/components/absolute_array.cpp b/reference/test/components/absolute_array.cpp
index 9793a6cf0ed..8d08b6d8b95 100644
--- a/reference/test/components/absolute_array.cpp
+++ b/reference/test/components/absolute_array.cpp
@@ -75,7 +75,7 @@ class AbsoluteArray : public ::testing::Test {
     gko::Array<value_type> vals;
 };
 
-TYPED_TEST_CASE(AbsoluteArray, gko::test::ValueTypes);
+TYPED_TEST_SUITE(AbsoluteArray, gko::test::ValueTypes);
 
 
 TYPED_TEST(AbsoluteArray, InplaceEqualsExpected)
diff --git a/reference/test/components/fill_array.cpp b/reference/test/components/fill_array.cpp
index 51ec5d8dd09..7ad30a78aff 100644
--- a/reference/test/components/fill_array.cpp
+++ b/reference/test/components/fill_array.cpp
@@ -69,7 +69,7 @@ class FillArray : public ::testing::Test {
     gko::Array<value_type> vals;
 };
 
-TYPED_TEST_CASE(FillArray, gko::test::ValueAndIndexTypes);
+TYPED_TEST_SUITE(FillArray, gko::test::ValueAndIndexTypes);
 
 
 TYPED_TEST(FillArray, EqualsReference)
diff --git a/reference/test/components/prefix_sum.cpp b/reference/test/components/prefix_sum.cpp
index 2766326bc4a..58d40606bb2 100644
--- a/reference/test/components/prefix_sum.cpp
+++ b/reference/test/components/prefix_sum.cpp
@@ -62,7 +62,7 @@ class PrefixSum : public ::testing::Test {
     std::vector<index_type> expected;
 };
 
-TYPED_TEST_CASE(PrefixSum, gko::test::IndexTypes);
+TYPED_TEST_SUITE(PrefixSum, gko::test::IndexTypes);
 
 
 TYPED_TEST(PrefixSum, Works)
diff --git a/reference/test/factorization/par_ict_kernels.cpp b/reference/test/factorization/par_ict_kernels.cpp
index 9be9045492b..3cde2a85cab 100644
--- a/reference/test/factorization/par_ict_kernels.cpp
+++ b/reference/test/factorization/par_ict_kernels.cpp
@@ -168,7 +168,7 @@ class ParIct : public ::testing::Test {
     gko::remove_complex<value_type> tol;
 };
 
-TYPED_TEST_CASE(ParIct, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(ParIct, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(ParIct, KernelInitializeRowPtrsL)
diff --git a/reference/test/factorization/par_ilu_kernels.cpp b/reference/test/factorization/par_ilu_kernels.cpp
index b24309de53b..551f591b53d 100644
--- a/reference/test/factorization/par_ilu_kernels.cpp
+++ b/reference/test/factorization/par_ilu_kernels.cpp
@@ -212,7 +212,7 @@ class ParIlu : public ::testing::Test {
     std::unique_ptr<typename par_ilu_type::Factory> ilu_factory_sort;
 };
 
-TYPED_TEST_CASE(ParIlu, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(ParIlu, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(ParIlu, KernelAddDiagonalElementsEmpty)
diff --git a/reference/test/factorization/par_ilut_kernels.cpp b/reference/test/factorization/par_ilut_kernels.cpp
index a72dd6206f7..0a5aa7ae049 100644
--- a/reference/test/factorization/par_ilut_kernels.cpp
+++ b/reference/test/factorization/par_ilut_kernels.cpp
@@ -300,7 +300,7 @@ class ParIlut : public ::testing::Test {
     gko::remove_complex<value_type> tol;
 };  // namespace
 
-TYPED_TEST_CASE(ParIlut, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(ParIlut, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(ParIlut, KernelThresholdSelect)
diff --git a/reference/test/log/convergence.cpp b/reference/test/log/convergence.cpp
index 01a9b17c303..484e605864c 100644
--- a/reference/test/log/convergence.cpp
+++ b/reference/test/log/convergence.cpp
@@ -51,7 +51,7 @@ namespace {
 template <typename T>
 class Convergence : public ::testing::Test {};
 
-TYPED_TEST_CASE(Convergence, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Convergence, gko::test::ValueTypes);
 
 
 TYPED_TEST(Convergence, CatchesCriterionCheckCompleted)
diff --git a/reference/test/log/papi.cpp b/reference/test/log/papi.cpp
index 842b6214374..f1e98dd0a51 100644
--- a/reference/test/log/papi.cpp
+++ b/reference/test/log/papi.cpp
@@ -115,7 +115,7 @@ class Papi : public ::testing::Test {
     int eventset;
 };
 
-TYPED_TEST_CASE(Papi, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Papi, gko::test::ValueTypes);
 
 
 TYPED_TEST(Papi, CatchesCriterionCheckCompleted)
diff --git a/reference/test/matrix/coo_kernels.cpp b/reference/test/matrix/coo_kernels.cpp
index 7c6ec33b423..68657a8702e 100644
--- a/reference/test/matrix/coo_kernels.cpp
+++ b/reference/test/matrix/coo_kernels.cpp
@@ -96,7 +96,7 @@ class Coo : public ::testing::Test {
     std::unique_ptr<Mtx> mtx;
 };
 
-TYPED_TEST_CASE(Coo, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Coo, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Coo, ConvertsToPrecision)
@@ -657,7 +657,7 @@ class CooComplex : public ::testing::Test {
     using Mtx = gko::matrix::Coo<value_type, index_type>;
 };
 
-TYPED_TEST_CASE(CooComplex, gko::test::ComplexValueIndexTypes);
+TYPED_TEST_SUITE(CooComplex, gko::test::ComplexValueIndexTypes);
 
 
 TYPED_TEST(CooComplex, OutplaceAbsolute)
diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp
index dfeb1b7ace3..14f153ff8f3 100644
--- a/reference/test/matrix/csr_kernels.cpp
+++ b/reference/test/matrix/csr_kernels.cpp
@@ -349,7 +349,7 @@ class Csr : public ::testing::Test {
     std::unique_ptr<Mtx> mtx3_unsorted;
 };
 
-TYPED_TEST_CASE(Csr, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Csr, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Csr, AppliesToDenseVector)
@@ -1390,7 +1390,7 @@ class CsrComplex : public ::testing::Test {
     using Mtx = gko::matrix::Csr<value_type, index_type>;
 };
 
-TYPED_TEST_CASE(CsrComplex, gko::test::ComplexValueIndexTypes);
+TYPED_TEST_SUITE(CsrComplex, gko::test::ComplexValueIndexTypes);
 
 
 TYPED_TEST(CsrComplex, MtxIsConjugateTransposable)
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 5e8a8ac48da..b6195ffa8b0 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -104,7 +104,7 @@ class Dense : public ::testing::Test {
 };
 
 
-TYPED_TEST_CASE(Dense, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Dense, gko::test::ValueTypes);
 
 
 TYPED_TEST(Dense, AppliesToDense)
@@ -2159,7 +2159,7 @@ class DenseComplex : public ::testing::Test {
 };
 
 
-TYPED_TEST_CASE(DenseComplex, gko::test::ComplexValueTypes);
+TYPED_TEST_SUITE(DenseComplex, gko::test::ComplexValueTypes);
 
 
 TYPED_TEST(DenseComplex, NonSquareMatrixIsConjugateTransposable)
diff --git a/reference/test/matrix/diagonal_kernels.cpp b/reference/test/matrix/diagonal_kernels.cpp
index 1d39ee9367d..440719f33bc 100644
--- a/reference/test/matrix/diagonal_kernels.cpp
+++ b/reference/test/matrix/diagonal_kernels.cpp
@@ -103,7 +103,7 @@ class Diagonal : public ::testing::Test {
     std::unique_ptr<Dense> dense2;
 };
 
-TYPED_TEST_CASE(Diagonal, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Diagonal, gko::test::ValueTypes);
 
 
 TYPED_TEST(Diagonal, AppliesToDense)
@@ -388,7 +388,7 @@ class DiagonalComplex : public ::testing::Test {
     using Diag = gko::matrix::Diagonal<value_type>;
 };
 
-TYPED_TEST_CASE(DiagonalComplex, gko::test::ComplexValueTypes);
+TYPED_TEST_SUITE(DiagonalComplex, gko::test::ComplexValueTypes);
 
 
 TYPED_TEST(DiagonalComplex, MtxIsConjugateTransposable)
diff --git a/reference/test/matrix/ell_kernels.cpp b/reference/test/matrix/ell_kernels.cpp
index b987001aed3..a16f3c5b125 100644
--- a/reference/test/matrix/ell_kernels.cpp
+++ b/reference/test/matrix/ell_kernels.cpp
@@ -105,7 +105,7 @@ class Ell : public ::testing::Test {
     std::unique_ptr<Mtx> mtx2;
 };
 
-TYPED_TEST_CASE(Ell, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Ell, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Ell, AppliesToDenseVector)
@@ -705,7 +705,7 @@ class EllComplex : public ::testing::Test {
     using Mtx = gko::matrix::Ell<value_type, index_type>;
 };
 
-TYPED_TEST_CASE(EllComplex, gko::test::ComplexValueIndexTypes);
+TYPED_TEST_SUITE(EllComplex, gko::test::ComplexValueIndexTypes);
 
 
 TYPED_TEST(EllComplex, InplaceAbsolute)
diff --git a/reference/test/matrix/hybrid_kernels.cpp b/reference/test/matrix/hybrid_kernels.cpp
index 8fa81df287e..c261eebaec3 100644
--- a/reference/test/matrix/hybrid_kernels.cpp
+++ b/reference/test/matrix/hybrid_kernels.cpp
@@ -131,7 +131,7 @@ class Hybrid : public ::testing::Test {
     std::unique_ptr<Mtx> mtx3;
 };
 
-TYPED_TEST_CASE(Hybrid, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Hybrid, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Hybrid, AppliesToDenseVector)
@@ -754,7 +754,7 @@ class HybridComplex : public ::testing::Test {
     using Mtx = gko::matrix::Hybrid<value_type, index_type>;
 };
 
-TYPED_TEST_CASE(HybridComplex, gko::test::ComplexValueIndexTypes);
+TYPED_TEST_SUITE(HybridComplex, gko::test::ComplexValueIndexTypes);
 
 
 TYPED_TEST(HybridComplex, OutplaceAbsolute)
diff --git a/reference/test/matrix/identity.cpp b/reference/test/matrix/identity.cpp
index 0ade3bce9f1..586d8ed232b 100644
--- a/reference/test/matrix/identity.cpp
+++ b/reference/test/matrix/identity.cpp
@@ -58,7 +58,7 @@ class Identity : public ::testing::Test {
 };
 
 
-TYPED_TEST_CASE(Identity, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Identity, gko::test::ValueTypes);
 
 
 TYPED_TEST(Identity, AppliesLinearCombinationToVector)
diff --git a/reference/test/matrix/permutation.cpp b/reference/test/matrix/permutation.cpp
index 1c7e93fe115..4535cd6a292 100644
--- a/reference/test/matrix/permutation.cpp
+++ b/reference/test/matrix/permutation.cpp
@@ -63,7 +63,7 @@ class Permutation : public ::testing::Test {
     std::shared_ptr<const gko::Executor> exec;
 };
 
-TYPED_TEST_CASE(Permutation, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Permutation, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Permutation, AppliesRowPermutationToDense)
diff --git a/reference/test/matrix/sellp_kernels.cpp b/reference/test/matrix/sellp_kernels.cpp
index 218609eda2c..736264d873d 100644
--- a/reference/test/matrix/sellp_kernels.cpp
+++ b/reference/test/matrix/sellp_kernels.cpp
@@ -81,7 +81,7 @@ class Sellp : public ::testing::Test {
     std::unique_ptr<Mtx> mtx2;
 };
 
-TYPED_TEST_CASE(Sellp, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Sellp, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Sellp, AppliesToDenseVector)
@@ -708,7 +708,7 @@ class SellpComplex : public ::testing::Test {
     using Mtx = gko::matrix::Sellp<value_type, index_type>;
 };
 
-TYPED_TEST_CASE(SellpComplex, gko::test::ComplexValueIndexTypes);
+TYPED_TEST_SUITE(SellpComplex, gko::test::ComplexValueIndexTypes);
 
 
 TYPED_TEST(SellpComplex, OutplaceAbsolute)
diff --git a/reference/test/matrix/sparsity_csr.cpp b/reference/test/matrix/sparsity_csr.cpp
index 69c8f580ffd..8569072711a 100644
--- a/reference/test/matrix/sparsity_csr.cpp
+++ b/reference/test/matrix/sparsity_csr.cpp
@@ -79,7 +79,7 @@ class SparsityCsr : public ::testing::Test {
     std::unique_ptr<Mtx> mtx;
 };
 
-TYPED_TEST_CASE(SparsityCsr, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(SparsityCsr, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(SparsityCsr, CanBeCreatedFromExistingCsrMatrix)
diff --git a/reference/test/matrix/sparsity_csr_kernels.cpp b/reference/test/matrix/sparsity_csr_kernels.cpp
index 7c40ceb41bc..141013061c0 100644
--- a/reference/test/matrix/sparsity_csr_kernels.cpp
+++ b/reference/test/matrix/sparsity_csr_kernels.cpp
@@ -157,7 +157,7 @@ class SparsityCsr : public ::testing::Test {
     std::unique_ptr<Mtx> mtx3_unsorted;
 };
 
-TYPED_TEST_CASE(SparsityCsr, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(SparsityCsr, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(SparsityCsr, AppliesToDenseVector)
diff --git a/reference/test/preconditioner/ilu.cpp b/reference/test/preconditioner/ilu.cpp
index b44791098c1..373906be719 100644
--- a/reference/test/preconditioner/ilu.cpp
+++ b/reference/test/preconditioner/ilu.cpp
@@ -125,7 +125,7 @@ class Ilu : public ::testing::Test {
     std::shared_ptr<typename ilu_rev_prec_type::Factory> ilu_rev_pre_factory;
 };
 
-TYPED_TEST_CASE(Ilu, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Ilu, gko::test::ValueTypes);
 
 
 TYPED_TEST(Ilu, BuildsDefaultWithoutThrowing)
diff --git a/reference/test/preconditioner/isai_kernels.cpp b/reference/test/preconditioner/isai_kernels.cpp
index 06ad239732d..6fc099189df 100644
--- a/reference/test/preconditioner/isai_kernels.cpp
+++ b/reference/test/preconditioner/isai_kernels.cpp
@@ -221,7 +221,7 @@ class Isai : public ::testing::Test {
     std::shared_ptr<Csr> u_sparse_inv2;
 };
 
-TYPED_TEST_CASE(Isai, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Isai, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Isai, KernelGenerateL1)
diff --git a/reference/test/preconditioner/jacobi.cpp b/reference/test/preconditioner/jacobi.cpp
index 29d5ddd477f..586825a2d3b 100644
--- a/reference/test/preconditioner/jacobi.cpp
+++ b/reference/test/preconditioner/jacobi.cpp
@@ -176,7 +176,7 @@ class Jacobi : public ::testing::Test {
     std::unique_ptr<Bj> adaptive_bj;
 };
 
-TYPED_TEST_CASE(Jacobi, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Jacobi, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Jacobi, GeneratesCorrectStorageScheme)
diff --git a/reference/test/preconditioner/jacobi_kernels.cpp b/reference/test/preconditioner/jacobi_kernels.cpp
index 9fc4a028f27..48f66a19723 100644
--- a/reference/test/preconditioner/jacobi_kernels.cpp
+++ b/reference/test/preconditioner/jacobi_kernels.cpp
@@ -117,7 +117,7 @@ class Jacobi : public ::testing::Test {
     std::shared_ptr<gko::matrix::Csr<value_type, index_type>> mtx;
 };
 
-TYPED_TEST_CASE(Jacobi, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Jacobi, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Jacobi, CanBeGenerated)
diff --git a/reference/test/reorder/rcm.cpp b/reference/test/reorder/rcm.cpp
index b8c8ea332ec..b8358f4edd3 100644
--- a/reference/test/reorder/rcm.cpp
+++ b/reference/test/reorder/rcm.cpp
@@ -87,7 +87,7 @@ class Rcm : public ::testing::Test {
     std::unique_ptr<reorder_type> reorder_op;
 };
 
-TYPED_TEST_CASE(Rcm, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Rcm, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Rcm, CanBeCleared)
diff --git a/reference/test/solver/bicg_kernels.cpp b/reference/test/solver/bicg_kernels.cpp
index e1d5b35da65..9ff64123176 100644
--- a/reference/test/solver/bicg_kernels.cpp
+++ b/reference/test/solver/bicg_kernels.cpp
@@ -104,7 +104,7 @@ class Bicg : public ::testing::Test {
     std::unique_ptr<typename Solver::Factory> bicg_factory_non_symmetric;
 };
 
-TYPED_TEST_CASE(Bicg, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Bicg, gko::test::ValueTypes);
 
 
 TYPED_TEST(Bicg, SolvesStencilSystem)
diff --git a/reference/test/solver/bicgstab_kernels.cpp b/reference/test/solver/bicgstab_kernels.cpp
index 45836e2ac01..3e01f6d12aa 100644
--- a/reference/test/solver/bicgstab_kernels.cpp
+++ b/reference/test/solver/bicgstab_kernels.cpp
@@ -93,7 +93,7 @@ class Bicgstab : public ::testing::Test {
     std::unique_ptr<typename Solver::Factory> bicgstab_factory_precision;
 };
 
-TYPED_TEST_CASE(Bicgstab, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Bicgstab, gko::test::ValueTypes);
 
 
 TYPED_TEST(Bicgstab, SolvesDenseSystem)
diff --git a/reference/test/solver/cg_kernels.cpp b/reference/test/solver/cg_kernels.cpp
index adeea72b0ae..7de370c020e 100644
--- a/reference/test/solver/cg_kernels.cpp
+++ b/reference/test/solver/cg_kernels.cpp
@@ -98,7 +98,7 @@ class Cg : public ::testing::Test {
     std::unique_ptr<typename Solver::Factory> cg_factory_big;
 };
 
-TYPED_TEST_CASE(Cg, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Cg, gko::test::ValueTypes);
 
 
 TYPED_TEST(Cg, SolvesStencilSystem)
diff --git a/reference/test/solver/cgs_kernels.cpp b/reference/test/solver/cgs_kernels.cpp
index 8a15055bea9..105062ff187 100644
--- a/reference/test/solver/cgs_kernels.cpp
+++ b/reference/test/solver/cgs_kernels.cpp
@@ -97,7 +97,7 @@ class Cgs : public ::testing::Test {
     std::unique_ptr<typename Solver::Factory> cgs_factory_big;
 };
 
-TYPED_TEST_CASE(Cgs, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Cgs, gko::test::ValueTypes);
 
 
 TYPED_TEST(Cgs, SolvesDenseSystem)
diff --git a/reference/test/solver/fcg_kernels.cpp b/reference/test/solver/fcg_kernels.cpp
index 843ea5a6037..eb21327256e 100644
--- a/reference/test/solver/fcg_kernels.cpp
+++ b/reference/test/solver/fcg_kernels.cpp
@@ -99,7 +99,7 @@ class Fcg : public ::testing::Test {
     std::unique_ptr<typename Solver::Factory> fcg_factory_big;
 };
 
-TYPED_TEST_CASE(Fcg, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Fcg, gko::test::ValueTypes);
 
 
 TYPED_TEST(Fcg, SolvesStencilSystem)
diff --git a/reference/test/solver/gmres_kernels.cpp b/reference/test/solver/gmres_kernels.cpp
index 8eb06cf8dce..9232971c98f 100644
--- a/reference/test/solver/gmres_kernels.cpp
+++ b/reference/test/solver/gmres_kernels.cpp
@@ -108,7 +108,7 @@ class Gmres : public ::testing::Test {
     std::unique_ptr<typename Solver::Factory> gmres_factory_big;
 };
 
-TYPED_TEST_CASE(Gmres, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Gmres, gko::test::ValueTypes);
 
 
 TYPED_TEST(Gmres, SolvesStencilSystem)
diff --git a/reference/test/solver/ir_kernels.cpp b/reference/test/solver/ir_kernels.cpp
index 208d4f6b9d9..76c66873d00 100644
--- a/reference/test/solver/ir_kernels.cpp
+++ b/reference/test/solver/ir_kernels.cpp
@@ -80,7 +80,7 @@ class Ir : public ::testing::Test {
     std::unique_ptr<typename Solver::Factory> ir_factory;
 };
 
-TYPED_TEST_CASE(Ir, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Ir, gko::test::ValueTypes);
 
 
 TYPED_TEST(Ir, SolvesTriangularSystem)
diff --git a/reference/test/solver/lower_trs.cpp b/reference/test/solver/lower_trs.cpp
index 65a7aab0d42..fe0deeca70c 100644
--- a/reference/test/solver/lower_trs.cpp
+++ b/reference/test/solver/lower_trs.cpp
@@ -78,7 +78,7 @@ class LowerTrs : public ::testing::Test {
     std::unique_ptr<Solver> solver;
 };
 
-TYPED_TEST_CASE(LowerTrs, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(LowerTrs, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(LowerTrs, LowerTrsFactoryCreatesCorrectSolver)
diff --git a/reference/test/solver/lower_trs_kernels.cpp b/reference/test/solver/lower_trs_kernels.cpp
index 6379dca192d..88158a9b288 100644
--- a/reference/test/solver/lower_trs_kernels.cpp
+++ b/reference/test/solver/lower_trs_kernels.cpp
@@ -93,7 +93,7 @@ class LowerTrs : public ::testing::Test {
     std::unique_ptr<typename Solver::Factory> lower_trs_factory_big;
 };
 
-TYPED_TEST_CASE(LowerTrs, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(LowerTrs, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(LowerTrs, RefLowerTrsFlagCheckIsCorrect)
diff --git a/reference/test/solver/upper_trs.cpp b/reference/test/solver/upper_trs.cpp
index 178fa5aff3c..5476897c6ce 100644
--- a/reference/test/solver/upper_trs.cpp
+++ b/reference/test/solver/upper_trs.cpp
@@ -78,7 +78,7 @@ class UpperTrs : public ::testing::Test {
     std::unique_ptr<Solver> upper_trs_solver;
 };
 
-TYPED_TEST_CASE(UpperTrs, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(UpperTrs, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(UpperTrs, UpperTrsFactoryCreatesCorrectSolver)
diff --git a/reference/test/solver/upper_trs_kernels.cpp b/reference/test/solver/upper_trs_kernels.cpp
index 81da3158442..d0077e63398 100644
--- a/reference/test/solver/upper_trs_kernels.cpp
+++ b/reference/test/solver/upper_trs_kernels.cpp
@@ -91,7 +91,7 @@ class UpperTrs : public ::testing::Test {
     std::unique_ptr<typename Solver::Factory> upper_trs_factory_mrhs;
 };
 
-TYPED_TEST_CASE(UpperTrs, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(UpperTrs, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(UpperTrs, RefUpperTrsFlagCheckIsCorrect)
diff --git a/reference/test/stop/residual_norm_kernels.cpp b/reference/test/stop/residual_norm_kernels.cpp
index bf2de2bc20c..8a2b31d8b38 100644
--- a/reference/test/stop/residual_norm_kernels.cpp
+++ b/reference/test/stop/residual_norm_kernels.cpp
@@ -68,7 +68,7 @@ class ResidualNormReduction : public ::testing::Test {
     std::shared_ptr<const gko::Executor> exec_;
 };
 
-TYPED_TEST_CASE(ResidualNormReduction, gko::test::ValueTypes);
+TYPED_TEST_SUITE(ResidualNormReduction, gko::test::ValueTypes);
 
 
 TYPED_TEST(ResidualNormReduction, CanCreateFactory)
@@ -195,7 +195,7 @@ class RelativeResidualNorm : public ::testing::Test {
     std::shared_ptr<const gko::Executor> exec_;
 };
 
-TYPED_TEST_CASE(RelativeResidualNorm, gko::test::ValueTypes);
+TYPED_TEST_SUITE(RelativeResidualNorm, gko::test::ValueTypes);
 
 
 TYPED_TEST(RelativeResidualNorm, CanCreateFactory)
@@ -321,7 +321,7 @@ class AbsoluteResidualNorm : public ::testing::Test {
     std::shared_ptr<const gko::Executor> exec_;
 };
 
-TYPED_TEST_CASE(AbsoluteResidualNorm, gko::test::ValueTypes);
+TYPED_TEST_SUITE(AbsoluteResidualNorm, gko::test::ValueTypes);
 
 
 TYPED_TEST(AbsoluteResidualNorm, CanCreateFactory)
diff --git a/reference/test/utils/assertions_test.cpp b/reference/test/utils/assertions_test.cpp
index ffa09c8d431..ffc467e06aa 100644
--- a/reference/test/utils/assertions_test.cpp
+++ b/reference/test/utils/assertions_test.cpp
@@ -48,7 +48,7 @@ namespace {
 template <typename T>
 class MatricesNear : public ::testing::Test {};
 
-TYPED_TEST_CASE(MatricesNear, gko::test::ValueTypes);
+TYPED_TEST_SUITE(MatricesNear, gko::test::ValueTypes);
 
 
 TYPED_TEST(MatricesNear, CanPassAnyMatrixType)
diff --git a/third_party/gtest/CMakeLists.txt b/third_party/gtest/CMakeLists.txt
index 48976e93f69..f68db6d2060 100644
--- a/third_party/gtest/CMakeLists.txt
+++ b/third_party/gtest/CMakeLists.txt
@@ -2,19 +2,19 @@ if(MSVC)
     # Force using shared runtime library when MSVC builds shared libraries
     ginkgo_load_git_package(gtest_external
         "https://github.com/google/googletest.git"
-        "df428ec11891f12c81e2872c0432e342b5403a34"
+        "ee3aa831172090fd5442820f215cb04ab6062756"
         # Work around the linking errors when compiling gtest with CUDA
         "-Dgtest_disable_pthreads=ON" "-Dgtest_force_shared_crt=${BUILD_SHARED_LIBS}")
 else()
     ginkgo_load_git_package(gtest_external
         "https://github.com/google/googletest.git"
-        "df428ec11891f12c81e2872c0432e342b5403a34"
+        "ee3aa831172090fd5442820f215cb04ab6062756"
         # Work around the linking errors when compiling gtest with CUDA
         "-Dgtest_disable_pthreads=ON"
         "-DCMAKE_CXX_FLAGS=-fPIC")
 endif()
 
-ginkgo_add_external_target(GTest::GTest gtest src/googletest/include build/googlemock/gtest
+ginkgo_add_external_target(GTest::GTest gtest src/googletest/include build/lib
     STATIC "d" gtest_external FALSE)
-ginkgo_add_external_target(GTest::Main gtest_main src/googletest/include build/googlemock/gtest
+ginkgo_add_external_target(GTest::Main gtest_main src/googletest/include build/lib
     STATIC "d" gtest_external FALSE)

From 03c8b0a3cb74320e181a60efaac819adce717062 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Fri, 9 Oct 2020 16:01:34 +0200
Subject: [PATCH 0087/1008] First version of the DPC++ executor.

+ Add a new `Dpcpp` executor. Use it in all `gko::Operation` and
  wherever appropriate.
+ Add the general architecture with all kernels set as `GKO_NOT_IMPLEMENTED`
+ Cope with unsupported copies between devices by going through the
  master executors. This (should) allow to copy from an AMD GPU to a
  NVIDIA GPU, or from Intel to AMD/NVIDIA (untested).
+ Implement a few simple kernels in the components and `stop/criterion`
  to ensure kernel execution works properly.
+ Implement DPC++ specific executor tests.
+ Circumvent some issues with `abs` and `sqrt` my using the sycl
  specific functions when appropriate. Note that it's still needed to
  namespace `abs` (and probably `sqrt` as well) calls inside DPC++
  kernels, otherwise the default (std) version is still used.
---
 BENCHMARKING.md                               |   8 +-
 CMakeLists.txt                                |  27 +-
 CONTRIBUTING.md                               |  16 +-
 INSTALL.md                                    |   4 +
 README.md                                     |   9 +-
 benchmark/utils/general.hpp                   |   7 +-
 benchmark/utils/overhead_linop.hpp            |   9 +
 cmake/GinkgoConfig.cmake.in                   |   1 +
 cmake/autodetect_executors.cmake              |  11 +
 cmake/create_test.cmake                       |  20 +
 cmake/get_info.cmake                          |  43 +-
 core/CMakeLists.txt                           |   2 +-
 core/base/executor.cpp                        |   4 +
 core/components/absolute_array.hpp            |   9 +
 core/components/fill_array.hpp                |   9 +
 core/components/precision_conversion.hpp      |   9 +
 core/components/prefix_sum.hpp                |   9 +
 core/device_hooks/CMakeLists.txt              |  29 +-
 core/device_hooks/cuda_hooks.cpp              |   5 +
 core/device_hooks/dpcpp_hooks.cpp             | 118 +++++
 core/device_hooks/hip_hooks.cpp               |   5 +
 core/devices/CMakeLists.txt                   |   3 +-
 core/devices/dpcpp/CMakeLists.txt             |   2 +
 core/devices/dpcpp/executor.cpp               |  59 +++
 core/factorization/factorization_kernels.hpp  |   9 +
 core/factorization/ilu_kernels.hpp            |   9 +
 core/factorization/par_ict_kernels.hpp        |   9 +
 core/factorization/par_ilu_kernels.hpp        |   9 +
 core/factorization/par_ilut_kernels.hpp       |   9 +
 core/matrix/coo_kernels.hpp                   |   9 +
 core/matrix/csr_kernels.hpp                   |   9 +
 core/matrix/dense_kernels.hpp                 |   9 +
 core/matrix/diagonal_kernels.hpp              |   9 +
 core/matrix/ell_kernels.hpp                   |   9 +
 core/matrix/hybrid_kernels.hpp                |   9 +
 core/matrix/sellp_kernels.hpp                 |   9 +
 core/matrix/sparsity_csr_kernels.hpp          |   9 +
 core/preconditioner/isai_kernels.hpp          |   9 +
 core/preconditioner/jacobi_kernels.hpp        |   9 +
 core/reorder/rcm_kernels.hpp                  |   9 +
 core/solver/bicg_kernels.hpp                  |   9 +
 core/solver/bicgstab_kernels.hpp              |   9 +
 core/solver/cg_kernels.hpp                    |   9 +
 core/solver/cgs_kernels.hpp                   |   9 +
 core/solver/fcg_kernels.hpp                   |   9 +
 core/solver/gmres_kernels.hpp                 |   9 +
 core/solver/ir_kernels.hpp                    |   9 +
 core/solver/lower_trs_kernels.hpp             |   9 +
 core/solver/upper_trs_kernels.hpp             |   9 +
 core/stop/criterion_kernels.hpp               |   9 +
 core/stop/residual_norm_kernels.hpp           |   9 +
 core/test/base/executor.cpp                   |  62 ++-
 cuda/CMakeLists.txt                           |   3 -
 cuda/base/executor.cpp                        |  19 +-
 cuda/test/base/cuda_executor.cu               |   7 +-
 doc/examples/examples.hpp.in                  |  12 +-
 doc/headers/dpcpp_executor.hpp                |  40 ++
 doc/headers/executors.hpp                     |   6 +-
 doc/helpers.cmake                             |   1 +
 dpcpp/CMakeLists.txt                          |  58 +++
 dpcpp/base/executor.dp.cpp                    | 182 ++++++++
 dpcpp/base/version.dp.cpp                     |  48 ++
 dpcpp/components/absolute_array.dp.cpp        |  82 ++++
 dpcpp/components/csr_spgeam.dp.hpp            |  76 ++++
 dpcpp/components/fill_array.dp.cpp            |  65 +++
 dpcpp/components/format_conversion.dp.hpp     |  91 ++++
 dpcpp/components/matrix_operations.dp.hpp     |  65 +++
 dpcpp/components/precision_conversion.dp.cpp  |  63 +++
 dpcpp/components/prefix_sum.dp.cpp            |  64 +++
 .../factorization_kernels.dp.cpp              | 144 ++++++
 dpcpp/factorization/ilu_kernels.dp.cpp        |  58 +++
 dpcpp/factorization/par_ict_kernels.dp.cpp    |  95 ++++
 dpcpp/factorization/par_ilu_kernels.dp.cpp    |  72 +++
 dpcpp/factorization/par_ilut_kernels.dp.cpp   | 168 +++++++
 dpcpp/get_info.cmake                          |   4 +
 dpcpp/matrix/coo_kernels.dp.cpp               | 163 +++++++
 dpcpp/matrix/csr_kernels.dp.cpp               | 425 ++++++++++++++++++
 dpcpp/matrix/dense_kernels.dp.cpp             | 375 ++++++++++++++++
 dpcpp/matrix/diagonal_kernels.dp.cpp          | 131 ++++++
 dpcpp/matrix/ell_kernels.dp.cpp               | 142 ++++++
 dpcpp/matrix/hybrid_kernels.dp.cpp            |  99 ++++
 dpcpp/matrix/sellp_kernels.dp.cpp             | 125 ++++++
 dpcpp/matrix/sparsity_csr_kernels.dp.cpp      | 175 ++++++++
 dpcpp/preconditioner/isai_kernels.dp.cpp      | 129 ++++++
 dpcpp/preconditioner/jacobi_kernels.dp.cpp    | 345 ++++++++++++++
 dpcpp/reorder/rcm_kernels.dp.cpp              |  81 ++++
 dpcpp/solver/bicg_kernels.dp.cpp              | 105 +++++
 dpcpp/solver/bicgstab_kernels.dp.cpp          | 137 ++++++
 dpcpp/solver/cg_kernels.dp.cpp                | 101 +++++
 dpcpp/solver/cgs_kernels.dp.cpp               | 119 +++++
 dpcpp/solver/fcg_kernels.dp.cpp               | 101 +++++
 dpcpp/solver/gmres_kernels.dp.cpp             | 186 ++++++++
 dpcpp/solver/ir_kernels.dp.cpp                |  60 +++
 dpcpp/solver/lower_trs_kernels.dp.cpp         | 112 +++++
 dpcpp/solver/upper_trs_kernels.dp.cpp         | 112 +++++
 dpcpp/stop/criterion_kernels.dp.cpp           |  71 +++
 dpcpp/stop/residual_norm_kernels.dp.cpp       |  77 ++++
 dpcpp/test/CMakeLists.txt                     |   5 +
 dpcpp/test/base/CMakeLists.txt                |   1 +
 dpcpp/test/base/executor.dp.cpp               | 303 +++++++++++++
 dpcpp/test/components/CMakeLists.txt          |   3 +
 dpcpp/test/components/absolute_array.cpp      | 132 ++++++
 dpcpp/test/components/fill_array.cpp          |  86 ++++
 .../test/components/precision_conversion.cpp  | 173 +++++++
 dpcpp/test/stop/CMakeLists.txt                |   1 +
 dpcpp/test/stop/criterion_kernels.cpp         | 107 +++++
 dpcpp/test_dpcpp.dp.cpp                       |  42 ++
 .../adaptiveprecision-blockjacobi.cpp         |   5 +
 examples/build-setup.sh                       |   6 +-
 examples/custom-logger/custom-logger.cpp      |   5 +
 examples/custom-matrix-format/build.sh        |   4 +-
 .../custom-matrix-format.cpp                  |   5 +
 .../custom-stopping-criterion.cpp             |   5 +
 .../ilu-preconditioned-solver.cpp             |   5 +
 .../inverse-iteration/inverse-iteration.cpp   |   5 +
 .../ir-ilu-preconditioned-solver.cpp          |   5 +
 .../iterative-refinement.cpp                  |   5 +
 .../mixed-precision-ir/mixed-precision-ir.cpp |   5 +
 .../nine-pt-stencil-solver.cpp                |  11 +-
 examples/papi-logging/papi-logging.cpp        |   8 +-
 .../performance-debugging.cpp                 |   8 +-
 examples/poisson-solver/poisson-solver.cpp    |   9 +-
 .../preconditioned-solver.cpp                 |   8 +-
 .../preconditioner-export/doc/results.dox     |   2 +-
 .../preconditioner-export.cpp                 |  10 +-
 .../simple-solver-logging.cpp                 |   5 +
 examples/simple-solver/simple-solver.cpp      |   5 +
 .../three-pt-stencil-solver.cpp               |   9 +-
 hip/base/executor.hip.cpp                     |   9 +-
 hip/test/base/hip_executor.hip.cpp            |   7 +-
 include/ginkgo/core/base/executor.hpp         | 282 +++++++++---
 include/ginkgo/core/base/math.hpp             |  32 ++
 include/ginkgo/core/base/types.hpp            |   1 +
 include/ginkgo/core/base/version.hpp          |  12 +-
 omp/CMakeLists.txt                            |   2 +
 135 files changed, 6690 insertions(+), 155 deletions(-)
 create mode 100644 core/device_hooks/dpcpp_hooks.cpp
 create mode 100644 core/devices/dpcpp/CMakeLists.txt
 create mode 100644 core/devices/dpcpp/executor.cpp
 create mode 100644 doc/headers/dpcpp_executor.hpp
 create mode 100644 dpcpp/CMakeLists.txt
 create mode 100644 dpcpp/base/executor.dp.cpp
 create mode 100644 dpcpp/base/version.dp.cpp
 create mode 100644 dpcpp/components/absolute_array.dp.cpp
 create mode 100644 dpcpp/components/csr_spgeam.dp.hpp
 create mode 100644 dpcpp/components/fill_array.dp.cpp
 create mode 100644 dpcpp/components/format_conversion.dp.hpp
 create mode 100644 dpcpp/components/matrix_operations.dp.hpp
 create mode 100644 dpcpp/components/precision_conversion.dp.cpp
 create mode 100644 dpcpp/components/prefix_sum.dp.cpp
 create mode 100644 dpcpp/factorization/factorization_kernels.dp.cpp
 create mode 100644 dpcpp/factorization/ilu_kernels.dp.cpp
 create mode 100644 dpcpp/factorization/par_ict_kernels.dp.cpp
 create mode 100644 dpcpp/factorization/par_ilu_kernels.dp.cpp
 create mode 100644 dpcpp/factorization/par_ilut_kernels.dp.cpp
 create mode 100644 dpcpp/get_info.cmake
 create mode 100644 dpcpp/matrix/coo_kernels.dp.cpp
 create mode 100644 dpcpp/matrix/csr_kernels.dp.cpp
 create mode 100644 dpcpp/matrix/dense_kernels.dp.cpp
 create mode 100644 dpcpp/matrix/diagonal_kernels.dp.cpp
 create mode 100644 dpcpp/matrix/ell_kernels.dp.cpp
 create mode 100644 dpcpp/matrix/hybrid_kernels.dp.cpp
 create mode 100644 dpcpp/matrix/sellp_kernels.dp.cpp
 create mode 100644 dpcpp/matrix/sparsity_csr_kernels.dp.cpp
 create mode 100644 dpcpp/preconditioner/isai_kernels.dp.cpp
 create mode 100644 dpcpp/preconditioner/jacobi_kernels.dp.cpp
 create mode 100644 dpcpp/reorder/rcm_kernels.dp.cpp
 create mode 100644 dpcpp/solver/bicg_kernels.dp.cpp
 create mode 100644 dpcpp/solver/bicgstab_kernels.dp.cpp
 create mode 100644 dpcpp/solver/cg_kernels.dp.cpp
 create mode 100644 dpcpp/solver/cgs_kernels.dp.cpp
 create mode 100644 dpcpp/solver/fcg_kernels.dp.cpp
 create mode 100644 dpcpp/solver/gmres_kernels.dp.cpp
 create mode 100644 dpcpp/solver/ir_kernels.dp.cpp
 create mode 100644 dpcpp/solver/lower_trs_kernels.dp.cpp
 create mode 100644 dpcpp/solver/upper_trs_kernels.dp.cpp
 create mode 100644 dpcpp/stop/criterion_kernels.dp.cpp
 create mode 100644 dpcpp/stop/residual_norm_kernels.dp.cpp
 create mode 100644 dpcpp/test/CMakeLists.txt
 create mode 100644 dpcpp/test/base/CMakeLists.txt
 create mode 100644 dpcpp/test/base/executor.dp.cpp
 create mode 100644 dpcpp/test/components/CMakeLists.txt
 create mode 100644 dpcpp/test/components/absolute_array.cpp
 create mode 100644 dpcpp/test/components/fill_array.cpp
 create mode 100644 dpcpp/test/components/precision_conversion.cpp
 create mode 100644 dpcpp/test/stop/CMakeLists.txt
 create mode 100644 dpcpp/test/stop/criterion_kernels.cpp
 create mode 100644 dpcpp/test_dpcpp.dp.cpp

diff --git a/BENCHMARKING.md b/BENCHMARKING.md
index 6ce05bf072c..ff5306ea432 100644
--- a/BENCHMARKING.md
+++ b/BENCHMARKING.md
@@ -147,8 +147,8 @@ collection. For details, see the [available benchmark options](### 5: Available
 benchmark options). Here are the most important options:
 * `BENCHMARK={spmv, solver, preconditioner}` - allows to select the type of
     benchmark to be ran.
-* `EXECUTOR={reference,cuda,hip,omp}` - select the executor and platform the
-    benchmarks should be ran on.
+* `EXECUTOR={reference,cuda,hip,omp,dpcpp}` - select the executor and platform
+    the benchmarks should be ran on.
 * `SYSTEM_NAME=<name>` - a name which will be used to designate this platform
     (e.g. V100, RadeonVII, ...).
 * `SEGMENTS=<N>` - Split the benchmarked matrix space into `<N>` segments. If
@@ -256,8 +256,8 @@ The supported environment variables are described in the following list:
                  benchmark.
     *   `preconditioner` - Runs the preconditioner benchmarks on artificially
                  generated block-diagonal matrices.
-* `EXECUTOR={reference,cuda,hip,omp}` - select the executor and platform the
-    benchmarks should be ran on. Default is `cuda`.
+* `EXECUTOR={reference,cuda,hip,omp,dpcpp}` - select the executor and platform
+    the benchmarks should be ran on. Default is `cuda`.
 * `SYSTEM_NAME=<name>` - a name which will be used to designate this platform
     (e.g. V100, RadeonVII, ...) and not overwrite previous results. Default is
     `unknown`.
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7462fe9f766..dd1c32c23f4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -27,6 +27,7 @@ option(GINKGO_BUILD_EXAMPLES "Build Ginkgo's examples" ON)
 option(GINKGO_BUILD_BENCHMARKS "Build Ginkgo's benchmarks" ON)
 option(GINKGO_BUILD_REFERENCE "Compile reference CPU kernels" ON)
 option(GINKGO_BUILD_OMP "Compile OpenMP kernels for CPU" ${GINKGO_HAS_OMP})
+option(GINKGO_BUILD_DPCPP "Compile DPC++ kernels for CPU" ${GINKGO_HAS_DPCPP})
 option(GINKGO_BUILD_CUDA "Compile kernels for NVIDIA GPUs" ${GINKGO_HAS_CUDA})
 option(GINKGO_BUILD_HIP "Compile kernels for AMD or NVIDIA GPUs" ${GINKGO_HAS_HIP})
 option(GINKGO_BUILD_DOC "Generate documentation" OFF)
@@ -72,7 +73,8 @@ option(BUILD_SHARED_LIBS "Build shared (.so, .dylib, .dll) libraries" ON)
 set(GINKGO_CIRCULAR_DEPS_FLAGS "-Wl,--no-undefined")
 
 if(BUILD_SHARED_LIBS AND (WIN32 OR CYGWIN) AND (GINKGO_BUILD_TESTS OR GINKGO_BUILD_EXAMPLES OR GINKGO_BUILD_BENCHMARKS))
-    # Change shared libraries output only if this build has executable program with shared libraries.
+    # Change shared libraries output only if this build has executable program
+    # with shared libraries.
     set(GINKGO_CHANGED_SHARED_LIBRARY TRUE)
     option(GINKGO_CHECK_PATH "Tell Ginkgo to check if the environment variable PATH is available for this build." ON)
     set(GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH "windows_shared_library" CACHE STRING
@@ -83,7 +85,7 @@ else()
     set(GINKGO_CHANGED_SHARED_LIBRARY FALSE)
 endif()
 
-if(GINKGO_BUILD_TESTS AND (GINKGO_BUILD_CUDA OR GINKGO_BUILD_OMP OR GINKGO_BUILD_HIP))
+if(GINKGO_BUILD_TESTS AND (GINKGO_BUILD_CUDA OR GINKGO_BUILD_OMP OR GINKGO_BUILD_HIP OR GINKGO_BUILD_DPCPP))
     message(STATUS "GINKGO_BUILD_TESTS is ON, enabling GINKGO_BUILD_REFERENCE")
     set(GINKGO_BUILD_REFERENCE ON CACHE BOOL "Compile reference CPU kernels" FORCE)
 endif()
@@ -127,8 +129,9 @@ endif()
 
 list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules/")
 
-# Find important header files, store the definitions in include/ginkgo/config.h.in
-# For details, see https://gitlab.kitware.com/cmake/community/wikis/doc/tutorials/How-To-Write-Platform-Checks
+# Find important header files, store the definitions in
+# include/ginkgo/config.h.in For details, see
+# https://gitlab.kitware.com/cmake/community/wikis/doc/tutorials/How-To-Write-Platform-Checks
 include(CheckIncludeFileCXX)
 check_include_file_cxx(cxxabi.h GKO_HAVE_CXXABI_H)
 
@@ -147,7 +150,8 @@ if(GINKGO_BUILD_HIP)
     if(DEFINED ENV{HIP_PLATFORM})
         set(GINKGO_HIP_PLATFORM "$ENV{HIP_PLATFORM}")
     elseif(GINKGO_HIPCONFIG_PATH)
-        execute_process(COMMAND ${GINKGO_HIPCONFIG_PATH} --platform OUTPUT_VARIABLE GINKGO_HIP_PLATFORM)
+        execute_process(COMMAND ${GINKGO_HIPCONFIG_PATH}
+            --platform OUTPUT_VARIABLE GINKGO_HIP_PLATFORM)
     else()
         message(FATAL_ERROR "No platform could be found for HIP. "
             "Set and export the environment variable HIP_PLATFORM.")
@@ -170,7 +174,8 @@ include(cmake/hip_helpers.cmake)
 include(cmake/install_helpers.cmake)
 include(cmake/windows_helpers.cmake)
 
-# This is modified from https://gitlab.kitware.com/cmake/community/wikis/FAQ#dynamic-replace
+# This is modified from
+# https://gitlab.kitware.com/cmake/community/wikis/FAQ#dynamic-replace
 if(MSVC)
     if(BUILD_SHARED_LIBS)
         ginkgo_switch_to_windows_dynamic("CXX")
@@ -204,6 +209,9 @@ endif()
 if (GINKGO_BUILD_OMP)
     add_subdirectory(omp)        # High-performance omp kernels
 endif()
+if (GINKGO_BUILD_DPCPP)
+    add_subdirectory(dpcpp)        # High-performance DPC++ kernels
+endif()
 # HIP needs to be last because it builds the GINKGO_RPATH_FOR_HIP variable
 # which needs to know the `ginkgo` target.
 if(GINKGO_BUILD_HIP)
@@ -271,7 +279,8 @@ endif()
 configure_file(${Ginkgo_SOURCE_DIR}/cmake/ginkgo.pc.in
     ${Ginkgo_BINARY_DIR}/ginkgo.pc @ONLY)
 
-# WINDOWS NVCC has " inside the string, add escape charater to avoid config problem.
+# WINDOWS NVCC has " inside the string, add escape character
+# to avoid config problem.
 ginkgo_modify_flags(CMAKE_CUDA_FLAGS)
 ginkgo_modify_flags(CMAKE_CUDA_FLAGS_DEBUG)
 ginkgo_modify_flags(CMAKE_CUDA_FLAGS_RELEASE)
@@ -296,7 +305,8 @@ add_custom_target(test_install
     -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
     -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
     # `--config cfg` is ignored by single-configuration generator.
-    # `$<CONFIG>` is always be the same as `CMAKE_BUILD_TYPE` in single-configuration generator.
+    # `$<CONFIG>` is always be the same as `CMAKE_BUILD_TYPE` in
+    # single-configuration generator.
     COMMAND ${CMAKE_COMMAND} --build ${Ginkgo_BINARY_DIR}/test_install --config $<CONFIG>
     COMMAND ${GINKGO_TEST_INSTALL_COMMAND}
     COMMENT "Running a test on the installed binaries. This requires running `(sudo) make install` first.")
@@ -309,7 +319,6 @@ set(CPACK_PACKAGE_CONTACT "ginkgo.library@gmail.com")
 include(CPack)
 
 # And finally, print the configuration to screen:
-#
 if(GINKGO_CONFIG_LOG_DETAILED)
     FILE(READ ${PROJECT_BINARY_DIR}/detailed.log GINKGO_LOG_SUMMARY)
 else()
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 9fcdc25ed13..07aec3f9649 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -68,8 +68,8 @@ look at our coding guidelines before proposing a pull request.
 
 Ginkgo is divided into a `core` module with common functionalities independent
 of the architecture, and several kernel modules (`reference`, `omp`, `cuda`,
-`hip`) which contain low-level computational routines for each supported
-architecture.
+`hip`, `dpcpp`) which contain low-level computational routines for each
+supported architecture.
 
 ### Extended header files
 
@@ -521,12 +521,12 @@ existing code has been broken.
   need to be performed with data that can be as small as possible. For example,
   matrices lesser than 5x5 are acceptable. This allows the reviewers to verify
   the results for exactness with tools such as MATLAB.
-* OpenMP, CUDA and HIP kernels have to be tested against the reference kernels.
-  Hence data for the tests of these kernels can be generated in the test files
-  using helper functions or by using external files to be read through the
-  standard input. In particular for CUDA and HIP, the data size should be at
-  least bigger than the architecture's warp size to ensure there is no corner
-  case in the kernels.
+* OpenMP, CUDA, HIP and DPC++ kernels have to be tested against the reference
+  kernels. Hence data for the tests of these kernels can be generated in the
+  test files using helper functions or by using external files to be read
+  through the standard input. In particular for CUDA, HIP and DPC++ the data
+  size should be at least bigger than the architecture's warp size to ensure
+  there is no corner case in the kernels.
 
 
 ## Documentation style
diff --git a/INSTALL.md b/INSTALL.md
index 2a0056bf3a4..7ef20b65aef 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -32,6 +32,10 @@ Ginkgo adds the following additional switches to control what is being built:
 *   `-DGINKGO_BUILD_CUDA={ON, OFF}` builds optimized cuda versions of the kernels
     (requires CUDA), default is `ON` if a CUDA compiler could be detected,
     `OFF` otherwise.
+*   `-DGINKGO_BUILD_DPCPP={ON, OFF}` builds optimized DPC++ versions of the
+    kernels (requires `CMAKE_CXX_COMPILER` to be set to the `dpcpp` compiler).
+    The default is `ON` if `CMAKE_CXX_COMPILER` is a DPC++ compiler, `OFF`
+    otherwise.
 *   `-DGINKGO_BUILD_HIP={ON, OFF}` builds optimized HIP versions of the kernels
     (requires HIP), default is `ON` if an installation of HIP could be detected,
     `OFF` otherwise.
diff --git a/README.md b/README.md
index 9e54903a92e..3012dc5dd7b 100644
--- a/README.md
+++ b/README.md
@@ -118,9 +118,12 @@ mkdir build; cd build
 cmake -G "Unix Makefiles" .. && make
 ```
 
-By default, `GINKGO_BUILD_REFERENCE` is enabled. You should be able to run examples with this
-executor. You would need to explicitly compile with the OpenMP and CUDA modules enabled
-to run with these executors. Please refer to the [Installation page](./INSTALL.md).
+By default, `GINKGO_BUILD_REFERENCE` is enabled. You should be able to run
+examples with this executor. By default, Ginkgo tries to enable the relevant
+modules depending on your machine environment (present of CUDA, ...). You can
+also explicitly compile with the OpenMP, CUDA, HIP or DPC++ modules enabled to
+run the examples with these executors. Please refer to the [Installation
+page](./INSTALL.md) for more details.
 
 After the installation, CMake can find ginkgo with `find_package(Ginkgo)`.
 An example can be found in the [`test_install`](test_install/CMakeLists.txt).
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index f61b92baeaf..fcf3b5df379 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -263,9 +263,14 @@ const std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
              return gko::CudaExecutor::create(FLAGS_device_id,
                                               gko::OmpExecutor::create(), true);
          }},
-        {"hip", [] {
+        {"hip",
+         [] {
              return gko::HipExecutor::create(FLAGS_device_id,
                                              gko::OmpExecutor::create(), true);
+         }},
+        {"dpcpp", [] {
+             return gko::DpcppExecutor::create(FLAGS_device_id,
+                                               gko::OmpExecutor::create());
          }}};
 
 
diff --git a/benchmark/utils/overhead_linop.hpp b/benchmark/utils/overhead_linop.hpp
index 2c2aa4baccf..5bddad88325 100644
--- a/benchmark/utils/overhead_linop.hpp
+++ b/benchmark/utils/overhead_linop.hpp
@@ -109,6 +109,15 @@ GKO_DECLARE_ALL;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace overhead {
+
+GKO_DECLARE_ALL;
+
+}  // namespace overhead
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL
 
 
diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 0348f956e7b..0d644c9e34b 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -36,6 +36,7 @@ set(GINKGO_BUILD_REFERENCE @GINKGO_BUILD_REFERENCE@)
 set(GINKGO_BUILD_OMP @GINKGO_BUILD_OMP@)
 set(GINKGO_BUILD_CUDA @GINKGO_BUILD_CUDA@)
 set(GINKGO_BUILD_HIP @GINKGO_BUILD_HIP@)
+set(GINKGO_BUILD_DPCPP @GINKGO_BUILD_DPCPP@)
 
 set(GINKGO_DEVEL_TOOLS @GINKGO_DEVEL_TOOLS@)
 set(GINKGO_BUILD_TESTS @GINKGO_BUILD_TESTS@)
diff --git a/cmake/autodetect_executors.cmake b/cmake/autodetect_executors.cmake
index 1f90640acb9..c432a92d593 100644
--- a/cmake/autodetect_executors.cmake
+++ b/cmake/autodetect_executors.cmake
@@ -1,9 +1,13 @@
 set(GINKGO_HAS_OMP OFF)
 set(GINKGO_HAS_CUDA OFF)
+set(GINKGO_HAS_DPCPP OFF)
 set(GINKGO_HAS_HIP OFF)
 find_package(OpenMP)
 include(CheckLanguage)
 check_language(CUDA)
+try_compile(GKO_CAN_COMPILE_DPCPP ${PROJECT_BINARY_DIR}/dpcpp
+    SOURCES ${PROJECT_SOURCE_DIR}/dpcpp/test_dpcpp.dp.cpp
+    CXX_STANDARD 17)
 
 if(OpenMP_CXX_FOUND)
     if(NOT DEFINED GINKGO_BUILD_OMP)
@@ -25,3 +29,10 @@ if(GINKGO_HIPCONFIG_PATH)
     endif()
     set(GINKGO_HAS_HIP ON)
 endif()
+
+if (GKO_CAN_COMPILE_DPCPP)
+    if(NOT DEFINED GINKGO_BUILD_DPCPP)
+        message(STATUS "Enabling DPCPP executor")
+    endif()
+    set(GINKGO_HAS_DPCPP ON)
+endif()
diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index f2ff8496e64..a2d2ef5d978 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -17,6 +17,26 @@ function(ginkgo_create_test test_name)
     add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
 endfunction(ginkgo_create_test)
 
+function(ginkgo_create_dpcpp_test test_name)
+    file(RELATIVE_PATH REL_BINARY_DIR
+        ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+    string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
+    add_executable(${TEST_TARGET_NAME} ${test_name}.dp.cpp)
+    target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_17)
+    target_compile_options("${TEST_TARGET_NAME}" PRIVATE "${GINKGO_DPCPP_FLAGS}")
+    target_include_directories("${TEST_TARGET_NAME}"
+        PRIVATE
+        "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
+        )
+    set_target_properties(${TEST_TARGET_NAME} PROPERTIES
+        OUTPUT_NAME ${test_name})
+    if (GINKGO_CHECK_CIRCULAR_DEPS)
+        target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
+    endif()
+    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
+    add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
+endfunction(ginkgo_create_dpcpp_test)
+
 function(ginkgo_create_thread_test test_name)
     set(THREADS_PREFER_PTHREAD_FLAG ON)
     find_package(Threads REQUIRED)
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index f4773be50bd..94f898a24c4 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -72,6 +72,12 @@ function(ginkgo_print_variable log_type var_name)
     FILE(APPEND ${log_type} "${upd_string}")
 endfunction()
 
+macro(ginkgo_print_foreach_variable variables)
+    foreach(var ${variables})
+        ginkgo_print_variable(${${log_type}} ${var} )
+    endforeach()
+endmacro()
+
 IF("${GINKGO_GIT_SHORTREV}" STREQUAL "")
     set(to_print "Summary of Configuration for Ginkgo (version ${Ginkgo_VERSION} with tag ${Ginkgo_VERSION_TAG})
 --"
@@ -88,12 +94,9 @@ ENDIF()
 set(log_types "detailed_log;minimal_log")
 foreach(log_type ${log_types})
     ginkgo_print_module_footer(${${log_type}} "Ginkgo configuration:")
-    set(print_var
-        "CMAKE_BUILD_TYPE;BUILD_SHARED_LIBS;CMAKE_INSTALL_PREFIX;PROJECT_SOURCE_DIR;PROJECT_BINARY_DIR"
-        )
-    foreach(var ${print_var})
-        ginkgo_print_variable(${${log_type}} ${var} )
-    endforeach()
+    ginkgo_print_foreach_variable(
+        "CMAKE_BUILD_TYPE;BUILD_SHARED_LIBS;CMAKE_INSTALL_PREFIX"
+        "PROJECT_SOURCE_DIR;PROJECT_BINARY_DIR")
     string(SUBSTRING
         "
 --        CMAKE_CXX_COMPILER:                                                   " 0 55 print_string)
@@ -108,31 +111,17 @@ foreach(log_type ${log_types})
     FILE(APPEND ${${log_type}} "${print_string}")
     ginkgo_print_module_footer(${${log_type}} "User configuration:")
     ginkgo_print_module_footer(${${log_type}} "  Enabled modules:")
-    set(print_var
-        "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP"
-        )
-    foreach(var ${print_var})
-        ginkgo_print_variable(${${log_type}} ${var} )
-    endforeach()
+    ginkgo_print_foreach_variable(
+        "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP")
     ginkgo_print_module_footer(${${log_type}} "  Tests, benchmarks and examples:")
-    set(print_var
+    ginkgo_print_foreach_variable(
         "GINKGO_BUILD_TESTS;GINKGO_BUILD_EXAMPLES;GINKGO_EXTLIB_EXAMPLE;GINKGO_BUILD_BENCHMARKS")
-    foreach(var ${print_var})
-        ginkgo_print_variable(${${log_type}} ${var} )
-    endforeach()
     ginkgo_print_module_footer(${${log_type}} "  Documentation:")
-    set(print_var
-        "GINKGO_BUILD_DOC;GINKGO_VERBOSE_LEVEL")
-    foreach(var ${print_var})
-        ginkgo_print_variable(${${log_type}} ${var} )
-    endforeach()
+    ginkgo_print_foreach_variable("GINKGO_BUILD_DOC;GINKGO_VERBOSE_LEVEL")
     ginkgo_print_module_footer(${${log_type}} "  Developer helpers:")
-    set(print_var
+    ginkgo_print_foreach_variable(
         "GINKGO_DEVEL_TOOLS;GINKGO_WITH_CLANG_TIDY;GINKGO_WITH_IWYU"
         "GINKGO_CHECK_CIRCULAR_DEPS;GINKGO_CHECK_PATH")
-    foreach(var ${print_var})
-        ginkgo_print_variable(${${log_type}} ${var} )
-    endforeach()
     ginkgo_print_module_footer(${${log_type}} "")
 endforeach()
 
@@ -160,6 +149,10 @@ IF(GINKGO_BUILD_HIP)
     include(hip/get_info.cmake)
 ENDIF()
 
+IF(GINKGO_BUILD_DPCPP)
+    include(dpcpp/get_info.cmake)
+ENDIF()
+
 ginkgo_print_generic_header(${detailed_log} "Optional Components:")
 ginkgo_print_variable(${detailed_log} "GKO_HAVE_PAPI_SDE")
 if(PAPI_sde_FOUND)
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index f16d63ff617..31639009e5a 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -58,7 +58,7 @@ target_compile_options(ginkgo PRIVATE "${GINKGO_COMPILER_FLAGS}")
 # regardless of whether it is installed or added as a subdirectory
 add_library(Ginkgo::ginkgo ALIAS ginkgo)
 target_link_libraries(ginkgo
-    PUBLIC ginkgo_omp ginkgo_cuda ginkgo_reference ginkgo_hip)
+    PUBLIC ginkgo_omp ginkgo_cuda ginkgo_reference ginkgo_hip ginkgo_dpcpp)
 # The PAPI dependency needs to be exposed to the user.
 if (GINKGO_HAVE_PAPI_SDE)
     target_link_libraries(ginkgo PUBLIC PAPI::PAPI)
diff --git a/core/base/executor.cpp b/core/base/executor.cpp
index 9d80ad818f0..90342b30e43 100644
--- a/core/base/executor.cpp
+++ b/core/base/executor.cpp
@@ -53,6 +53,10 @@ void Operation::run(std::shared_ptr<const HipExecutor> executor) const
     GKO_NOT_IMPLEMENTED;
 
 
+void Operation::run(std::shared_ptr<const DpcppExecutor> executor) const
+    GKO_NOT_IMPLEMENTED;
+
+
 void Operation::run(std::shared_ptr<const ReferenceExecutor> executor) const
 {
     this->run(static_cast<std::shared_ptr<const OmpExecutor>>(executor));
diff --git a/core/components/absolute_array.hpp b/core/components/absolute_array.hpp
index d48d04335fa..746c1de733a 100644
--- a/core/components/absolute_array.hpp
+++ b/core/components/absolute_array.hpp
@@ -99,6 +99,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/components/fill_array.hpp b/core/components/fill_array.hpp
index 7bafb8aecb4..452661bed84 100644
--- a/core/components/fill_array.hpp
+++ b/core/components/fill_array.hpp
@@ -91,6 +91,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/components/precision_conversion.hpp b/core/components/precision_conversion.hpp
index 719c596c34e..82b684d4be1 100644
--- a/core/components/precision_conversion.hpp
+++ b/core/components/precision_conversion.hpp
@@ -93,6 +93,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/components/prefix_sum.hpp b/core/components/prefix_sum.hpp
index d171be831aa..f226ce4c282 100644
--- a/core/components/prefix_sum.hpp
+++ b/core/components/prefix_sum.hpp
@@ -91,6 +91,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/device_hooks/CMakeLists.txt b/core/device_hooks/CMakeLists.txt
index 94dfc8ab9f0..5b8f8bffb22 100644
--- a/core/device_hooks/CMakeLists.txt
+++ b/core/device_hooks/CMakeLists.txt
@@ -3,11 +3,28 @@ if(NOT GINKGO_BUILD_CUDA)
         $<TARGET_OBJECTS:ginkgo_cuda_device>
         cuda_hooks.cpp)
     ginkgo_compile_features(ginkgo_cuda)
-    target_link_libraries(ginkgo_cuda PUBLIC ginkgo_hip)
     ginkgo_default_includes(ginkgo_cuda)
     ginkgo_install_library(ginkgo_cuda cuda)
 endif()
 
+if (NOT GINKGO_BUILD_DPCPP)
+    add_library(ginkgo_dpcpp
+        $<TARGET_OBJECTS:ginkgo_dpcpp_device>
+        dpcpp_hooks.cpp)
+    ginkgo_compile_features(ginkgo_dpcpp)
+    ginkgo_default_includes(ginkgo_dpcpp)
+    ginkgo_install_library(ginkgo_dpcpp dpcpp)
+endif()
+
+if(NOT GINKGO_BUILD_HIP)
+    add_library(ginkgo_hip
+        $<TARGET_OBJECTS:ginkgo_hip_device>
+        hip_hooks.cpp)
+    ginkgo_compile_features(ginkgo_hip)
+    ginkgo_default_includes(ginkgo_hip)
+    ginkgo_install_library(ginkgo_hip hip)
+endif()
+
 if (NOT GINKGO_BUILD_OMP)
     add_library(ginkgo_omp
         $<TARGET_OBJECTS:ginkgo_omp_device>
@@ -15,6 +32,7 @@ if (NOT GINKGO_BUILD_OMP)
     ginkgo_compile_features(ginkgo_omp)
     target_link_libraries(ginkgo_omp PUBLIC ginkgo_cuda)
     target_link_libraries(ginkgo_omp PUBLIC ginkgo_hip)
+    target_link_libraries(ginkgo_omp PUBLIC ginkgo_dpcpp)
     ginkgo_default_includes(ginkgo_omp)
     ginkgo_install_library(ginkgo_omp omp)
 endif()
@@ -27,12 +45,3 @@ if (NOT GINKGO_BUILD_REFERENCE)
     ginkgo_default_includes(ginkgo_reference)
     ginkgo_install_library(ginkgo_reference reference)
 endif()
-
-if(NOT GINKGO_BUILD_HIP)
-    add_library(ginkgo_hip
-        $<TARGET_OBJECTS:ginkgo_hip_device>
-        hip_hooks.cpp)
-    ginkgo_compile_features(ginkgo_hip)
-    ginkgo_default_includes(ginkgo_hip)
-    ginkgo_install_library(ginkgo_hip hip)
-endif()
diff --git a/core/device_hooks/cuda_hooks.cpp b/core/device_hooks/cuda_hooks.cpp
index d41d77d24d9..5bd7f5cb09f 100644
--- a/core/device_hooks/cuda_hooks.cpp
+++ b/core/device_hooks/cuda_hooks.cpp
@@ -93,6 +93,11 @@ void CudaExecutor::raw_copy_to(const HipExecutor *, size_type num_bytes,
     GKO_NOT_COMPILED(cuda);
 
 
+void CudaExecutor::raw_copy_to(const DpcppExecutor *, size_type num_bytes,
+                               const void *src_ptr, void *dest_ptr) const
+    GKO_NOT_COMPILED(cuda);
+
+
 void CudaExecutor::synchronize() const GKO_NOT_COMPILED(cuda);
 
 
diff --git a/core/device_hooks/dpcpp_hooks.cpp b/core/device_hooks/dpcpp_hooks.cpp
new file mode 100644
index 00000000000..56d61760fed
--- /dev/null
+++ b/core/device_hooks/dpcpp_hooks.cpp
@@ -0,0 +1,118 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+
+#include <memory>
+#include <string>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/base/version.hpp>
+
+
+namespace gko {
+
+version version_info::get_dpcpp_version() noexcept
+{
+    // We just return the version with a special "not compiled" tag in
+    // placeholder modules.
+    return {GKO_VERSION_STR, "not compiled"};
+}
+
+
+std::shared_ptr<DpcppExecutor> DpcppExecutor::create(
+    int device_id, std::shared_ptr<Executor> master, std::string device_type)
+{
+    return std::shared_ptr<DpcppExecutor>(
+        new DpcppExecutor(device_id, std::move(master), device_type));
+}
+
+
+void OmpExecutor::raw_copy_to(const DpcppExecutor *, size_type num_bytes,
+                              const void *src_ptr, void *dest_ptr) const
+    GKO_NOT_COMPILED(dpcpp);
+
+
+void DpcppExecutor::raw_free(void *ptr) const noexcept
+{
+    // Free must never fail, as it can be called in destructors.
+    // If the nvidia module was not compiled, the library couldn't have
+    // allocated the memory, so there is no need to deallocate it.
+}
+
+
+void *DpcppExecutor::raw_alloc(size_type num_bytes) const
+    GKO_NOT_COMPILED(dpcpp);
+
+
+void DpcppExecutor::raw_copy_to(const OmpExecutor *, size_type num_bytes,
+                                const void *src_ptr, void *dest_ptr) const
+    GKO_NOT_COMPILED(dpcpp);
+
+
+void DpcppExecutor::raw_copy_to(const CudaExecutor *, size_type num_bytes,
+                                const void *src_ptr, void *dest_ptr) const
+    GKO_NOT_COMPILED(dpcpp);
+
+void DpcppExecutor::raw_copy_to(const HipExecutor *, size_type num_bytes,
+                                const void *src_ptr, void *dest_ptr) const
+    GKO_NOT_COMPILED(dpcpp);
+
+void DpcppExecutor::raw_copy_to(const DpcppExecutor *, size_type num_bytes,
+                                const void *src_ptr, void *dest_ptr) const
+    GKO_NOT_COMPILED(dpcpp);
+
+
+void DpcppExecutor::synchronize() const GKO_NOT_COMPILED(dpcpp);
+
+
+void DpcppExecutor::run(const Operation &op) const
+{
+    op.run(std::static_pointer_cast<const DpcppExecutor>(
+        this->shared_from_this()));
+}
+
+
+int DpcppExecutor::get_num_devices(std::string) { return 0; }
+
+
+void DpcppExecutor::set_device_property() {}
+
+
+}  // namespace gko
+
+
+#define GKO_HOOK_MODULE dpcpp
+#include "core/device_hooks/common_kernels.inc.cpp"
+#undef GKO_HOOK_MODULE
diff --git a/core/device_hooks/hip_hooks.cpp b/core/device_hooks/hip_hooks.cpp
index a2e288b4157..a60b657bac7 100644
--- a/core/device_hooks/hip_hooks.cpp
+++ b/core/device_hooks/hip_hooks.cpp
@@ -90,6 +90,11 @@ void HipExecutor::raw_copy_to(const HipExecutor *, size_type num_bytes,
     GKO_NOT_COMPILED(hip);
 
 
+void HipExecutor::raw_copy_to(const DpcppExecutor *, size_type num_bytes,
+                              const void *src_ptr, void *dest_ptr) const
+    GKO_NOT_COMPILED(hip);
+
+
 void HipExecutor::synchronize() const GKO_NOT_COMPILED(hip);
 
 
diff --git a/core/devices/CMakeLists.txt b/core/devices/CMakeLists.txt
index 2a5626c0018..adfd93beb03 100644
--- a/core/devices/CMakeLists.txt
+++ b/core/devices/CMakeLists.txt
@@ -6,7 +6,8 @@ function(ginkgo_add_object_library name)
     set_target_properties(${name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endfunction()
 
-add_subdirectory(omp)
 add_subdirectory(cuda)
+add_subdirectory(dpcpp)
 add_subdirectory(hip)
+add_subdirectory(omp)
 add_subdirectory(reference)
diff --git a/core/devices/dpcpp/CMakeLists.txt b/core/devices/dpcpp/CMakeLists.txt
new file mode 100644
index 00000000000..f6fdb354ff2
--- /dev/null
+++ b/core/devices/dpcpp/CMakeLists.txt
@@ -0,0 +1,2 @@
+ginkgo_add_object_library(ginkgo_dpcpp_device
+    executor.cpp)
diff --git a/core/devices/dpcpp/executor.cpp b/core/devices/dpcpp/executor.cpp
new file mode 100644
index 00000000000..90b7eb67616
--- /dev/null
+++ b/core/devices/dpcpp/executor.cpp
@@ -0,0 +1,59 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include <cstdlib>
+#include <cstring>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
+namespace gko {
+
+
+std::shared_ptr<Executor> DpcppExecutor::get_master() noexcept
+{
+    return master_;
+}
+
+
+std::shared_ptr<const Executor> DpcppExecutor::get_master() const noexcept
+{
+    return master_;
+}
+
+
+}  // namespace gko
diff --git a/core/factorization/factorization_kernels.hpp b/core/factorization/factorization_kernels.hpp
index f7c25964dde..96e87ed4362 100644
--- a/core/factorization/factorization_kernels.hpp
+++ b/core/factorization/factorization_kernels.hpp
@@ -132,6 +132,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace factorization
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/factorization/ilu_kernels.hpp b/core/factorization/ilu_kernels.hpp
index 17602ac4ab4..1c71ceba897 100644
--- a/core/factorization/ilu_kernels.hpp
+++ b/core/factorization/ilu_kernels.hpp
@@ -95,6 +95,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace ilu_factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace ilu_factorization
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/factorization/par_ict_kernels.hpp b/core/factorization/par_ict_kernels.hpp
index f02b6ac7bb6..5cde5fa14e2 100644
--- a/core/factorization/par_ict_kernels.hpp
+++ b/core/factorization/par_ict_kernels.hpp
@@ -106,6 +106,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace par_ict_factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace par_ict_factorization
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/factorization/par_ilu_kernels.hpp b/core/factorization/par_ilu_kernels.hpp
index 09bc1dd2596..6d8451b32e7 100644
--- a/core/factorization/par_ilu_kernels.hpp
+++ b/core/factorization/par_ilu_kernels.hpp
@@ -97,6 +97,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace par_ilu_factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace par_ilu_factorization
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/factorization/par_ilut_kernels.hpp b/core/factorization/par_ilut_kernels.hpp
index 9bb19596c3f..714e797e553 100644
--- a/core/factorization/par_ilut_kernels.hpp
+++ b/core/factorization/par_ilut_kernels.hpp
@@ -143,6 +143,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace par_ilut_factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace par_ilut_factorization
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/matrix/coo_kernels.hpp b/core/matrix/coo_kernels.hpp
index 752798c4e2e..8ae0466db26 100644
--- a/core/matrix/coo_kernels.hpp
+++ b/core/matrix/coo_kernels.hpp
@@ -140,6 +140,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace coo {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace coo
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/matrix/csr_kernels.hpp b/core/matrix/csr_kernels.hpp
index ff8c2d9ef26..6f7b72312d9 100644
--- a/core/matrix/csr_kernels.hpp
+++ b/core/matrix/csr_kernels.hpp
@@ -262,6 +262,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace csr {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace csr
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/matrix/dense_kernels.hpp b/core/matrix/dense_kernels.hpp
index bf86205fbef..1f9bdae1cc2 100644
--- a/core/matrix/dense_kernels.hpp
+++ b/core/matrix/dense_kernels.hpp
@@ -274,6 +274,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace dense {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace dense
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/matrix/diagonal_kernels.hpp b/core/matrix/diagonal_kernels.hpp
index bb8d4c14e7d..556ab66d8a3 100644
--- a/core/matrix/diagonal_kernels.hpp
+++ b/core/matrix/diagonal_kernels.hpp
@@ -135,6 +135,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace diagonal {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace diagonal
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/matrix/ell_kernels.hpp b/core/matrix/ell_kernels.hpp
index 049ed280b66..aa78e17881e 100644
--- a/core/matrix/ell_kernels.hpp
+++ b/core/matrix/ell_kernels.hpp
@@ -139,6 +139,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace ell {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace ell
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/matrix/hybrid_kernels.hpp b/core/matrix/hybrid_kernels.hpp
index 788fe66e15b..fb813a0e428 100644
--- a/core/matrix/hybrid_kernels.hpp
+++ b/core/matrix/hybrid_kernels.hpp
@@ -104,6 +104,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace hybrid {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace hybrid
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/matrix/sellp_kernels.hpp b/core/matrix/sellp_kernels.hpp
index 7b9b6fdf551..f57cf7087a8 100644
--- a/core/matrix/sellp_kernels.hpp
+++ b/core/matrix/sellp_kernels.hpp
@@ -130,6 +130,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace sellp {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace sellp
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/matrix/sparsity_csr_kernels.hpp b/core/matrix/sparsity_csr_kernels.hpp
index 58ec58e789f..8d5e3570699 100644
--- a/core/matrix/sparsity_csr_kernels.hpp
+++ b/core/matrix/sparsity_csr_kernels.hpp
@@ -144,6 +144,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace sparsity_csr {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace sparsity_csr
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/preconditioner/isai_kernels.hpp b/core/preconditioner/isai_kernels.hpp
index ce53d51cd3c..8093f454255 100644
--- a/core/preconditioner/isai_kernels.hpp
+++ b/core/preconditioner/isai_kernels.hpp
@@ -111,6 +111,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace isai {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace isai
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/preconditioner/jacobi_kernels.hpp b/core/preconditioner/jacobi_kernels.hpp
index 12d232c26f8..96ae123df12 100644
--- a/core/preconditioner/jacobi_kernels.hpp
+++ b/core/preconditioner/jacobi_kernels.hpp
@@ -176,6 +176,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace jacobi {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace jacobi
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/reorder/rcm_kernels.hpp b/core/reorder/rcm_kernels.hpp
index 9c8b049a280..2eeaa63a3ab 100644
--- a/core/reorder/rcm_kernels.hpp
+++ b/core/reorder/rcm_kernels.hpp
@@ -98,6 +98,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace rcm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace rcm
+}  // namespace dpcpp
+
+
 namespace reference {
 namespace rcm {
 
diff --git a/core/solver/bicg_kernels.hpp b/core/solver/bicg_kernels.hpp
index 9ef21b3a243..c11c7f4e4a1 100644
--- a/core/solver/bicg_kernels.hpp
+++ b/core/solver/bicg_kernels.hpp
@@ -124,6 +124,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace bicg {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace bicg
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/solver/bicgstab_kernels.hpp b/core/solver/bicgstab_kernels.hpp
index 8b48151a50f..0ebd591a1fe 100644
--- a/core/solver/bicgstab_kernels.hpp
+++ b/core/solver/bicgstab_kernels.hpp
@@ -148,6 +148,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace bicgstab {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace bicgstab
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/solver/cg_kernels.hpp b/core/solver/cg_kernels.hpp
index 3a52974033a..1042b24cccd 100644
--- a/core/solver/cg_kernels.hpp
+++ b/core/solver/cg_kernels.hpp
@@ -119,6 +119,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace cg {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace cg
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/solver/cgs_kernels.hpp b/core/solver/cgs_kernels.hpp
index 1404303b2ce..e2622419573 100644
--- a/core/solver/cgs_kernels.hpp
+++ b/core/solver/cgs_kernels.hpp
@@ -135,6 +135,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace cgs {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace cgs
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/solver/fcg_kernels.hpp b/core/solver/fcg_kernels.hpp
index dc269f2fa19..89c3e1c3504 100644
--- a/core/solver/fcg_kernels.hpp
+++ b/core/solver/fcg_kernels.hpp
@@ -120,6 +120,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace fcg {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace fcg
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/solver/gmres_kernels.hpp b/core/solver/gmres_kernels.hpp
index 644a8cf708e..2aaa0fb509e 100644
--- a/core/solver/gmres_kernels.hpp
+++ b/core/solver/gmres_kernels.hpp
@@ -135,6 +135,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace gmres {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace gmres
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/solver/ir_kernels.hpp b/core/solver/ir_kernels.hpp
index 9fe59ba4a6c..ac76627755a 100644
--- a/core/solver/ir_kernels.hpp
+++ b/core/solver/ir_kernels.hpp
@@ -92,6 +92,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace ir {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace ir
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/solver/lower_trs_kernels.hpp b/core/solver/lower_trs_kernels.hpp
index 799c50129e0..4430de5b468 100644
--- a/core/solver/lower_trs_kernels.hpp
+++ b/core/solver/lower_trs_kernels.hpp
@@ -123,6 +123,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace lower_trs {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace lower_trs
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/solver/upper_trs_kernels.hpp b/core/solver/upper_trs_kernels.hpp
index cce48ea2812..e49fe8cc8e7 100644
--- a/core/solver/upper_trs_kernels.hpp
+++ b/core/solver/upper_trs_kernels.hpp
@@ -123,6 +123,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace upper_trs {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace upper_trs
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/stop/criterion_kernels.hpp b/core/stop/criterion_kernels.hpp
index 07eb8f2798c..c894e6337ca 100644
--- a/core/stop/criterion_kernels.hpp
+++ b/core/stop/criterion_kernels.hpp
@@ -87,6 +87,15 @@ GKO_DECLARE_SET_ALL_STATUSES_KERNEL();
 
 }  // namespace set_all_statuses
 }  // namespace hip
+
+
+namespace dpcpp {
+namespace set_all_statuses {
+
+GKO_DECLARE_SET_ALL_STATUSES_KERNEL();
+
+}  // namespace set_all_statuses
+}  // namespace dpcpp
 }  // namespace kernels
 }  // namespace gko
 
diff --git a/core/stop/residual_norm_kernels.hpp b/core/stop/residual_norm_kernels.hpp
index 30407cf9b9f..94c9b7001ad 100644
--- a/core/stop/residual_norm_kernels.hpp
+++ b/core/stop/residual_norm_kernels.hpp
@@ -99,6 +99,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace residual_norm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace residual_norm
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 }  // namespace kernels
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 1b2e1b0698e..f43e375cf19 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -63,10 +63,14 @@ class ExampleOperation : public gko::Operation {
     {
         value = 3;
     }
-    void run(std::shared_ptr<const gko::ReferenceExecutor>) const override
+    void run(std::shared_ptr<const gko::DpcppExecutor>) const override
     {
         value = 4;
     }
+    void run(std::shared_ptr<const gko::ReferenceExecutor>) const override
+    {
+        value = 5;
+    }
 
     int &value;
 };
@@ -88,9 +92,10 @@ TEST(OmpExecutor, RunsCorrectLambdaOperation)
     auto omp_lambda = [&value]() { value = 1; };
     auto cuda_lambda = [&value]() { value = 2; };
     auto hip_lambda = [&value]() { value = 3; };
+    auto dpcpp_lambda = [&value]() { value = 4; };
     exec_ptr omp = gko::OmpExecutor::create();
 
-    omp->run(omp_lambda, cuda_lambda, hip_lambda);
+    omp->run(omp_lambda, cuda_lambda, hip_lambda, dpcpp_lambda);
     ASSERT_EQ(1, value);
 }
 
@@ -155,7 +160,7 @@ TEST(ReferenceExecutor, RunsCorrectOperation)
     exec_ptr ref = gko::ReferenceExecutor::create();
 
     ref->run(ExampleOperation(value));
-    ASSERT_EQ(4, value);
+    ASSERT_EQ(5, value);
 }
 
 
@@ -165,9 +170,10 @@ TEST(ReferenceExecutor, RunsCorrectLambdaOperation)
     auto omp_lambda = [&value]() { value = 1; };
     auto cuda_lambda = [&value]() { value = 2; };
     auto hip_lambda = [&value]() { value = 3; };
+    auto dpcpp_lambda = [&value]() { value = 4; };
     exec_ptr ref = gko::ReferenceExecutor::create();
 
-    ref->run(omp_lambda, cuda_lambda, hip_lambda);
+    ref->run(omp_lambda, cuda_lambda, hip_lambda, dpcpp_lambda);
     ASSERT_EQ(1, value);
 }
 
@@ -289,10 +295,11 @@ TEST(CudaExecutor, RunsCorrectLambdaOperation)
     auto omp_lambda = [&value]() { value = 1; };
     auto cuda_lambda = [&value]() { value = 2; };
     auto hip_lambda = [&value]() { value = 3; };
+    auto dpcpp_lambda = [&value]() { value = 4; };
     exec_ptr cuda =
         gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
 
-    cuda->run(omp_lambda, cuda_lambda, hip_lambda);
+    cuda->run(omp_lambda, cuda_lambda, hip_lambda, dpcpp_lambda);
     ASSERT_EQ(2, value);
 }
 
@@ -360,9 +367,10 @@ TEST(HipExecutor, RunsCorrectLambdaOperation)
     auto omp_lambda = [&value]() { value = 1; };
     auto cuda_lambda = [&value]() { value = 2; };
     auto hip_lambda = [&value]() { value = 3; };
+    auto dpcpp_lambda = [&value]() { value = 4; };
     exec_ptr hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
 
-    hip->run(omp_lambda, cuda_lambda, hip_lambda);
+    hip->run(omp_lambda, cuda_lambda, hip_lambda, dpcpp_lambda);
     ASSERT_EQ(3, value);
 }
 
@@ -414,6 +422,48 @@ TEST(HipExecutor, CanSetDeviceResetBoolean)
 }
 
 
+TEST(DpcppExecutor, RunsCorrectOperation)
+{
+    int value = 0;
+    exec_ptr dpcpp = gko::DpcppExecutor::create(0, gko::OmpExecutor::create());
+
+    dpcpp->run(ExampleOperation(value));
+    ASSERT_EQ(4, value);
+}
+
+
+TEST(DpcppExecutor, RunsCorrectLambdaOperation)
+{
+    int value = 0;
+    auto omp_lambda = [&value]() { value = 1; };
+    auto cuda_lambda = [&value]() { value = 2; };
+    auto hip_lambda = [&value]() { value = 3; };
+    auto dpcpp_lambda = [&value]() { value = 4; };
+    exec_ptr dpcpp = gko::DpcppExecutor::create(0, gko::OmpExecutor::create());
+
+    dpcpp->run(omp_lambda, cuda_lambda, hip_lambda, dpcpp_lambda);
+    ASSERT_EQ(4, value);
+}
+
+
+TEST(DpcppExecutor, KnowsItsMaster)
+{
+    auto omp = gko::OmpExecutor::create();
+    exec_ptr dpcpp = gko::DpcppExecutor::create(0, omp);
+
+    ASSERT_EQ(omp, dpcpp->get_master());
+}
+
+
+TEST(DpcppExecutor, KnowsItsDeviceId)
+{
+    auto omp = gko::OmpExecutor::create();
+    auto dpcpp = gko::DpcppExecutor::create(0, omp);
+
+    ASSERT_EQ(0, dpcpp->get_device_id());
+}
+
+
 template <typename T>
 struct mock_free : T {
     /**
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index b19651b6a59..021f17a081d 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -151,9 +151,6 @@ target_include_directories(ginkgo_cuda
     SYSTEM PRIVATE ${CUDA_INCLUDE_DIRS})
 target_link_libraries(ginkgo_cuda PRIVATE ${CUDA_RUNTIME_LIBS} ${CUBLAS} ${CUSPARSE})
 
-# Need to link against ginkgo_hip for the `raw_copy_to(HipExecutor ...)` method
-target_link_libraries(ginkgo_cuda PUBLIC ginkgo_hip)
-
 cas_target_cuda_architectures(ginkgo_cuda
     ARCHITECTURES ${GINKGO_CUDA_ARCHITECTURES}
     UNSUPPORTED "20" "21")
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 543e78131e0..7926ae98a21 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -127,31 +127,38 @@ void CudaExecutor::raw_copy_to(const OmpExecutor *, size_type num_bytes,
 }
 
 
-void CudaExecutor::raw_copy_to(const CudaExecutor *dest, size_type num_bytes,
+void CudaExecutor::raw_copy_to(const HipExecutor *dest, size_type num_bytes,
                                const void *src_ptr, void *dest_ptr) const
 {
+#if GINKGO_HIP_PLATFORM_NVCC == 1
     if (num_bytes > 0) {
         cuda::device_guard g(this->get_device_id());
         GKO_ASSERT_NO_CUDA_ERRORS(
             cudaMemcpyPeer(dest_ptr, dest->get_device_id(), src_ptr,
                            this->get_device_id(), num_bytes));
     }
+#else
+    GKO_NOT_SUPPORTED(dest);
+#endif
 }
 
 
-void CudaExecutor::raw_copy_to(const HipExecutor *dest, size_type num_bytes,
+void CudaExecutor::raw_copy_to(const DpcppExecutor *dest, size_type num_bytes,
+                               const void *src_ptr, void *dest_ptr) const
+{
+    GKO_NOT_SUPPORTED(dest);
+}
+
+
+void CudaExecutor::raw_copy_to(const CudaExecutor *dest, size_type num_bytes,
                                const void *src_ptr, void *dest_ptr) const
 {
-#if GINKGO_HIP_PLATFORM_NVCC == 1
     if (num_bytes > 0) {
         cuda::device_guard g(this->get_device_id());
         GKO_ASSERT_NO_CUDA_ERRORS(
             cudaMemcpyPeer(dest_ptr, dest->get_device_id(), src_ptr,
                            this->get_device_id(), num_bytes));
     }
-#else
-    GKO_NOT_SUPPORTED(this);
-#endif
 }
 
 
diff --git a/cuda/test/base/cuda_executor.cu b/cuda/test/base/cuda_executor.cu
index 2bcf5961bbd..7b944fd13a1 100644
--- a/cuda/test/base/cuda_executor.cu
+++ b/cuda/test/base/cuda_executor.cu
@@ -69,6 +69,11 @@ public:
         value = -3;
     }
 
+    void run(std::shared_ptr<const gko::DpcppExecutor>) const override
+    {
+        value = -4;
+    }
+
     void run(std::shared_ptr<const gko::CudaExecutor>) const override
     {
         cudaGetDevice(&value);
@@ -239,7 +244,7 @@ TEST_F(CudaExecutor, CopiesDataFromCudaToCuda)
     omp->copy_from(cuda2.get(), 2, copy_cuda2, copy);
     EXPECT_EQ(3, copy[0]);
     ASSERT_EQ(8, copy[1]);
-    cuda->free(copy_cuda2);
+    cuda2->free(copy_cuda2);
     cuda->free(orig);
 }
 
diff --git a/doc/examples/examples.hpp.in b/doc/examples/examples.hpp.in
index 8c47be53d89..ce5b5add69a 100644
--- a/doc/examples/examples.hpp.in
+++ b/doc/examples/examples.hpp.in
@@ -57,8 +57,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * By default, Ginkgo is compiled with at least
  * <code>-DGINKGO_BUILD_REFERENCE=ON</code>.
- * To execute on a GPU, you need to have a GPU on the system and must have
- * compiled Ginkgo with the <code>-DGINKGO_BUILD_CUDA=ON</code> option.
+ * Ginkgo also tries to detect your environment setup (presence of CUDA, ...) to
+ * enable the relevant accelerator modules. If you want to target a specific
+ * GPU, make sure that Ginkgo is compiled with the accelerator specific module
+ * enabled, such as:
+ * <ol>
+ *   <li> <code>-DGINKGO_BUILD_CUDA=ON</code> option for NVIDIA GPUs.
+ *   <li> <code>-DGINKGO_BUILD_HIP=ON</code> option for AMD or NVIDIA GPUs.
+ *   <li> <code>-DGINKGO_BUILD_SYCL=ON</code> option for Intel GPUs (and
+ *        possibly any other platform).
+ * </ol
  *
  * <a name="graph"></a>
  * @anchor ExampleConnectionGraph
diff --git a/doc/headers/dpcpp_executor.hpp b/doc/headers/dpcpp_executor.hpp
new file mode 100644
index 00000000000..bef39d461db
--- /dev/null
+++ b/doc/headers/dpcpp_executor.hpp
@@ -0,0 +1,40 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+/**
+ * @defgroup exec_dpcpp DPC++ Executor
+ *
+ * @brief A module dedicated to the implementation and usage of the DPC++
+ * executor in Ginkgo.
+ *
+ * @ingroup Executor
+ */
diff --git a/doc/headers/executors.hpp b/doc/headers/executors.hpp
index 002f64230e9..1a07f985395 100644
--- a/doc/headers/executors.hpp
+++ b/doc/headers/executors.hpp
@@ -47,12 +47,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  * Ginkgo currently supports three different executor types:
  *
  * +    @ref exec_omp specifies that the data should be stored and the
- * associated operations executed on an OpenMP-supporting device (e.g. host
- * CPU);
+ *      associated operations executed on an OpenMP-supporting device (e.g. host
+ *      CPU);
  * +    @ref exec_cuda specifies that the data should be stored and the
  *      operations executed on the NVIDIA GPU accelerator;
  * +    @ref exec_hip uses the HIP library to compile code for either NVIDIA or
  *      AMD GPU accelerator;
+ * +    @ref exec_dpcpp uses the DPC++ compiler for any DPC++ supported hardware
+ *      (e.g. Intel CPUs, GPU, FPGAs, ...);
  * +    @ref exec_ref executes a non-optimized reference implementation,
  *      which can be used to debug the library.
  */
diff --git a/doc/helpers.cmake b/doc/helpers.cmake
index f2a89f7eb09..c0b4c7b23a7 100644
--- a/doc/helpers.cmake
+++ b/doc/helpers.cmake
@@ -69,6 +69,7 @@ function(ginkgo_doc_gen name in pdf mainpage-in)
         ${DIR_BASE}/omp
         ${DIR_BASE}/cuda
         ${DIR_BASE}/hip
+        ${DIR_BASE}/dpcpp
         ${DIR_BASE}/reference
         )
     set(doxygen_image_path "")
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
new file mode 100644
index 00000000000..4eb501e8d35
--- /dev/null
+++ b/dpcpp/CMakeLists.txt
@@ -0,0 +1,58 @@
+if (NOT GKO_CAN_COMPILE_DPCPP)
+    message(FATAL_ERROR "The CMAKE_CXX_COMPILER compiler, which is "
+        "${CMAKE_CXX_COMPILER} cannot compile DPC++ code!")
+endif()
+
+add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_dpcpp_device> "")
+target_sources(ginkgo_dpcpp
+    PRIVATE
+    base/version.dp.cpp
+    base/executor.dp.cpp
+    components/absolute_array.dp.cpp
+    components/fill_array.dp.cpp
+    components/precision_conversion.dp.cpp
+    components/prefix_sum.dp.cpp
+    factorization/ilu_kernels.dp.cpp
+    factorization/factorization_kernels.dp.cpp
+    factorization/par_ict_kernels.dp.cpp
+    factorization/par_ilu_kernels.dp.cpp
+    factorization/par_ilut_kernels.dp.cpp
+    matrix/coo_kernels.dp.cpp
+    matrix/csr_kernels.dp.cpp
+    matrix/dense_kernels.dp.cpp
+    matrix/diagonal_kernels.dp.cpp
+    matrix/ell_kernels.dp.cpp
+    matrix/hybrid_kernels.dp.cpp
+    matrix/sellp_kernels.dp.cpp
+    matrix/sparsity_csr_kernels.dp.cpp
+    preconditioner/isai_kernels.dp.cpp
+    preconditioner/jacobi_kernels.dp.cpp
+    reorder/rcm_kernels.dp.cpp
+    solver/bicg_kernels.dp.cpp
+    solver/bicgstab_kernels.dp.cpp
+    solver/cg_kernels.dp.cpp
+    solver/cgs_kernels.dp.cpp
+    solver/fcg_kernels.dp.cpp
+    solver/gmres_kernels.dp.cpp
+    solver/ir_kernels.dp.cpp
+    solver/lower_trs_kernels.dp.cpp
+    solver/upper_trs_kernels.dp.cpp
+    stop/criterion_kernels.dp.cpp
+    stop/residual_norm_kernels.dp.cpp)
+
+ginkgo_compile_features(ginkgo_dpcpp)
+
+set(GINKGO_DPCPP_FLAGS ${GINKGO_COMPILER_FLAGS} -fsycl)
+target_compile_options(ginkgo_dpcpp PRIVATE "${GINKGO_DPCPP_FLAGS}")
+target_compile_features(ginkgo_dpcpp PRIVATE cxx_std_17)
+
+ginkgo_default_includes(ginkgo_dpcpp)
+ginkgo_install_library(ginkgo_dpcpp dpcpp)
+
+if (GINKGO_CHECK_CIRCULAR_DEPS)
+    ginkgo_check_headers(ginkgo_dpcpp)
+endif()
+
+if(GINKGO_BUILD_TESTS)
+    add_subdirectory(test)
+endif()
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
new file mode 100644
index 00000000000..208d7255fb3
--- /dev/null
+++ b/dpcpp/base/executor.dp.cpp
@@ -0,0 +1,182 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include <iostream>
+#include <map>
+#include <string>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/config.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
+namespace gko {
+namespace detail {
+
+
+const std::vector<sycl::device> get_devices(std::string device_type)
+{
+    std::map<std::string, sycl::info::device_type> device_type_map{
+        {"accelerator", sycl::info::device_type::accelerator},
+        {"all", sycl::info::device_type::all},
+        {"cpu", sycl::info::device_type::cpu},
+        {"gpu", sycl::info::device_type::gpu}};
+    return sycl::device::get_devices(device_type_map.at(device_type));
+}
+
+
+}  // namespace detail
+
+
+void OmpExecutor::raw_copy_to(const DpcppExecutor *dest, size_type num_bytes,
+                              const void *src_ptr, void *dest_ptr) const
+{
+    if (num_bytes > 0) {
+        dest->get_queue()->memcpy(dest_ptr, src_ptr, num_bytes).wait();
+    }
+}
+
+
+std::shared_ptr<DpcppExecutor> DpcppExecutor::create(
+    int device_id, std::shared_ptr<Executor> master, std::string device_type)
+{
+    return std::shared_ptr<DpcppExecutor>(
+        new DpcppExecutor(device_id, std::move(master), device_type));
+}
+
+
+void DpcppExecutor::raw_free(void *ptr) const noexcept
+{
+    sycl::free(ptr, queue_->get_context());
+}
+
+
+void *DpcppExecutor::raw_alloc(size_type num_bytes) const
+{
+    void *dev_ptr = sycl::malloc_device(num_bytes, *queue_.get());
+    GKO_ENSURE_ALLOCATED(dev_ptr, "DPC++", num_bytes);
+    return dev_ptr;
+}
+
+
+void DpcppExecutor::raw_copy_to(const OmpExecutor *, size_type num_bytes,
+                                const void *src_ptr, void *dest_ptr) const
+{
+    if (num_bytes > 0) {
+        queue_->memcpy(dest_ptr, src_ptr, num_bytes).wait();
+    }
+}
+
+
+void DpcppExecutor::raw_copy_to(const CudaExecutor *dest, size_type num_bytes,
+                                const void *src_ptr, void *dest_ptr) const
+{
+    // TODO: later when possible, if we have DPC++ with a CUDA backend
+    // support/compiler, we could maybe support native copies?
+    GKO_NOT_SUPPORTED(dest);
+}
+
+
+void DpcppExecutor::raw_copy_to(const HipExecutor *dest, size_type num_bytes,
+                                const void *src_ptr, void *dest_ptr) const
+{
+    GKO_NOT_SUPPORTED(dest);
+}
+
+
+void DpcppExecutor::raw_copy_to(const DpcppExecutor *dest, size_type num_bytes,
+                                const void *src_ptr, void *dest_ptr) const
+{
+    if (num_bytes > 0) {
+        // TODO: does this work? Or is it needed to go through host?
+        dest->get_queue()->memcpy(dest_ptr, src_ptr, num_bytes).wait();
+    }
+}
+
+
+void DpcppExecutor::synchronize() const { queue_->wait_and_throw(); }
+
+
+void DpcppExecutor::run(const Operation &op) const
+{
+    this->template log<log::Logger::operation_launched>(this, &op);
+    op.run(std::static_pointer_cast<const DpcppExecutor>(
+        this->shared_from_this()));
+    this->template log<log::Logger::operation_completed>(this, &op);
+}
+
+
+int DpcppExecutor::get_num_devices(std::string device_type)
+{
+    return detail::get_devices(device_type).size();
+}
+
+
+void delete_queue(sycl::queue *queue)
+{
+    queue->wait();
+    delete queue;
+}
+
+
+void DpcppExecutor::set_device_property()
+{
+    assert(device_id_ < DpcppExecutor::get_num_devices(device_type_));
+    auto device = detail::get_devices(device_type_)[device_id_];
+    try {
+        subgroup_sizes_ =
+            device.get_info<cl::sycl::info::device::sub_group_sizes>();
+    } catch (cl::sycl::runtime_error &err) {
+        GKO_NOT_SUPPORTED(device);
+    }
+    num_computing_units_ =
+        device.get_info<sycl::info::device::max_compute_units>();
+    auto max_workitem_sizes =
+        device.get_info<sycl::info::device::max_work_item_sizes>();
+    // There is no way to get the dimension of a sycl::id object
+    for (std::size_t i = 0; i < 3; i++) {
+        max_workitem_sizes_.push_back(max_workitem_sizes[i]);
+    }
+    max_workgroup_size_ =
+        device.get_info<sycl::info::device::max_work_group_size>();
+    auto *queue = new sycl::queue{device, sycl::property::queue::in_order{}};
+    queue_ = std::move(queue_manager<sycl::queue>{queue, delete_queue});
+}
+
+
+}  // namespace gko
diff --git a/dpcpp/base/version.dp.cpp b/dpcpp/base/version.dp.cpp
new file mode 100644
index 00000000000..229a72ca72c
--- /dev/null
+++ b/dpcpp/base/version.dp.cpp
@@ -0,0 +1,48 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/base/version.hpp>
+
+
+namespace gko {
+
+
+version version_info::get_dpcpp_version() noexcept
+{
+    // When compiling the module, the header version is the same as the library
+    // version. Mismatch between the header and the module versions may happen
+    // if using shared libraries from different versions of Ginkgo.
+    return version_info::get_header_version();
+}
+
+
+}  // namespace gko
diff --git a/dpcpp/components/absolute_array.dp.cpp b/dpcpp/components/absolute_array.dp.cpp
new file mode 100644
index 00000000000..9603dacbc8b
--- /dev/null
+++ b/dpcpp/components/absolute_array.dp.cpp
@@ -0,0 +1,82 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/absolute_array.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/math.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+namespace components {
+
+
+template <typename ValueType>
+void inplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec,
+                            ValueType *data, size_type n)
+{
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{n}, [=](sycl::id<1> idx_id) {
+            const int idx = idx_id[0];
+            data[idx] = dpcpp::abs(data[idx]);
+        });
+    });
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_ARRAY_KERNEL);
+
+
+template <typename ValueType>
+void outplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec,
+                             const ValueType *in, size_type n,
+                             remove_complex<ValueType> *out)
+{
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{n}, [=](sycl::id<1> idx_id) {
+            const int idx = idx_id[0];
+            out[idx] = dpcpp::abs(in[idx]);
+        });
+    });
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_ARRAY_KERNEL);
+
+
+}  // namespace components
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/components/csr_spgeam.dp.hpp b/dpcpp/components/csr_spgeam.dp.hpp
new file mode 100644
index 00000000000..77303d7dd7a
--- /dev/null
+++ b/dpcpp/components/csr_spgeam.dp.hpp
@@ -0,0 +1,76 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_COMPONENTS_CSR_SPGEAM_HPP_
+#define GKO_DPCPP_COMPONENTS_CSR_SPGEAM_HPP_
+
+
+#include <limits>
+
+
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+#include "core/base/utils.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+/**
+ * Adds two (sorted) sparse matrices.
+ *
+ * Calls begin_cb(row) on each row to initialize row-local data
+ * Calls entry_cb(row, col, a_val, b_val, local_data) on each output non-zero
+ * Calls end_cb(row, local_data) on each row to finalize row-local data
+ *
+ * If the three functions are thread-safe, the whole invocation is.
+ */
+template <typename ValueType, typename IndexType, typename BeginCallback,
+          typename EntryCallback, typename EndCallback>
+void abstract_spgeam(const matrix::Csr<ValueType, IndexType> *a,
+                     const matrix::Csr<ValueType, IndexType> *b,
+                     BeginCallback begin_cb, EntryCallback entry_cb,
+                     EndCallback end_cb)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_DPCPP_COMPONENTS_CSR_SPGEAM_HPP_
diff --git a/dpcpp/components/fill_array.dp.cpp b/dpcpp/components/fill_array.dp.cpp
new file mode 100644
index 00000000000..f8c260cdab6
--- /dev/null
+++ b/dpcpp/components/fill_array.dp.cpp
@@ -0,0 +1,65 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/fill_array.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+namespace components {
+
+
+template <typename ValueType>
+void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
+                size_type n, ValueType val)
+{
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{n}, [=](sycl::id<1> idx_id) {
+            const int idx = idx_id[0];
+            array[idx] = val;
+        });
+    });
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
+template GKO_DECLARE_FILL_ARRAY_KERNEL(size_type);
+
+
+}  // namespace components
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/components/format_conversion.dp.hpp b/dpcpp/components/format_conversion.dp.hpp
new file mode 100644
index 00000000000..f1dba5eed78
--- /dev/null
+++ b/dpcpp/components/format_conversion.dp.hpp
@@ -0,0 +1,91 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <algorithm>
+#include <numeric>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/types.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+/**
+ * @internal
+ *
+ * Converts an array of indexes `idxs` in any order to an array of pointers
+ * `ptrs`. This is used for transposing a csr matrix when calculating the row
+ * pointers of the transposed matrix out of the column indices of the original
+ * matrix.
+ */
+template <typename IndexType>
+inline void convert_unsorted_idxs_to_ptrs(const IndexType *idxs,
+                                          size_type num_nonzeros,
+                                          IndexType *ptrs, size_type length)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+/**
+ * @internal
+ *
+ * Converts an array of indexes `idxs` which are already stored in an increasing
+ * order to an array of pointers `ptrs`. This is used to calculate the row
+ * pointers when converting a coo matrix to a csr matrix.
+ */
+template <typename IndexType>
+inline void convert_sorted_idxs_to_ptrs(const IndexType *idxs,
+                                        size_type num_nonzeros, IndexType *ptrs,
+                                        size_type length)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename IndexType>
+inline void convert_ptrs_to_idxs(const IndexType *ptrs, size_type num_rows,
+                                 IndexType *idxs)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/components/matrix_operations.dp.hpp b/dpcpp/components/matrix_operations.dp.hpp
new file mode 100644
index 00000000000..4efb2d839f2
--- /dev/null
+++ b/dpcpp/components/matrix_operations.dp.hpp
@@ -0,0 +1,65 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_COMPONENTS_MATRIX_OPERATIONS_HPP_
+#define GKO_DPCPP_COMPONENTS_MATRIX_OPERATIONS_HPP_
+
+
+#include <ginkgo/core/base/math.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+/**
+ * @internal
+ *
+ * Computes the infinity norm of a column-major matrix.
+ */
+template <typename ValueType>
+remove_complex<ValueType> compute_inf_norm(size_type num_rows,
+                                           size_type num_cols,
+                                           const ValueType *matrix,
+                                           size_type stride)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_DPCPP_COMPONENTS_MATRIX_OPERATIONS_HPP_
diff --git a/dpcpp/components/precision_conversion.dp.cpp b/dpcpp/components/precision_conversion.dp.cpp
new file mode 100644
index 00000000000..c508e853ad6
--- /dev/null
+++ b/dpcpp/components/precision_conversion.dp.cpp
@@ -0,0 +1,63 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/precision_conversion.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+namespace components {
+
+
+template <typename SourceType, typename TargetType>
+void convert_precision(std::shared_ptr<const DefaultExecutor> exec,
+                       size_type size, const SourceType *in, TargetType *out)
+{
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{size}, [=](sycl::id<1> idx_id) {
+            const int idx = idx_id[0];
+            out[idx] = in[idx];
+        });
+    });
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_CONVERT_PRECISION_KERNEL);
+
+
+}  // namespace components
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/components/prefix_sum.dp.cpp b/dpcpp/components/prefix_sum.dp.cpp
new file mode 100644
index 00000000000..21ca279b696
--- /dev/null
+++ b/dpcpp/components/prefix_sum.dp.cpp
@@ -0,0 +1,64 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/prefix_sum.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+namespace components {
+
+
+template <typename IndexType>
+void prefix_sum(std::shared_ptr<const DefaultExecutor> exec, IndexType *counts,
+                size_type num_entries)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_PREFIX_SUM_KERNEL);
+
+// instantiate for size_type as well, as this is used in the Sellp format
+template GKO_DECLARE_PREFIX_SUM_KERNEL(size_type);
+
+
+}  // namespace components
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/factorization/factorization_kernels.dp.cpp b/dpcpp/factorization/factorization_kernels.dp.cpp
new file mode 100644
index 00000000000..74386b58376
--- /dev/null
+++ b/dpcpp/factorization/factorization_kernels.dp.cpp
@@ -0,0 +1,144 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/factorization_kernels.hpp"
+
+
+#include <algorithm>
+#include <memory>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/csr_builder.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace factorization {
+
+
+template <bool IsSorted, typename ValueType, typename IndexType>
+void find_missing_diagonal_elements(
+    const matrix::Csr<ValueType, IndexType> *mtx,
+    IndexType *elements_to_add_per_row, bool *changes_required)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void add_missing_diagonal_elements(const matrix::Csr<ValueType, IndexType> *mtx,
+                                   ValueType *new_values,
+                                   IndexType *new_col_idxs,
+                                   const IndexType *row_ptrs_addition)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void add_diagonal_elements(std::shared_ptr<const DpcppExecutor> exec,
+                           matrix::Csr<ValueType, IndexType> *mtx,
+                           bool is_sorted)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FACTORIZATION_ADD_DIAGONAL_ELEMENTS_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void initialize_row_ptrs_l_u(
+    std::shared_ptr<const DpcppExecutor> exec,
+    const matrix::Csr<ValueType, IndexType> *system_matrix,
+    IndexType *l_row_ptrs, IndexType *u_row_ptrs)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FACTORIZATION_INITIALIZE_ROW_PTRS_L_U_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void initialize_l_u(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Csr<ValueType, IndexType> *system_matrix,
+                    matrix::Csr<ValueType, IndexType> *csr_l,
+                    matrix::Csr<ValueType, IndexType> *csr_u)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FACTORIZATION_INITIALIZE_L_U_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void initialize_row_ptrs_l(
+    std::shared_ptr<const DpcppExecutor> exec,
+    const matrix::Csr<ValueType, IndexType> *system_matrix,
+    IndexType *l_row_ptrs)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FACTORIZATION_INITIALIZE_ROW_PTRS_L_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void initialize_l(std::shared_ptr<const DpcppExecutor> exec,
+                  const matrix::Csr<ValueType, IndexType> *system_matrix,
+                  matrix::Csr<ValueType, IndexType> *csr_l, bool diag_sqrt)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FACTORIZATION_INITIALIZE_L_KERNEL);
+
+
+}  // namespace factorization
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/factorization/ilu_kernels.dp.cpp b/dpcpp/factorization/ilu_kernels.dp.cpp
new file mode 100644
index 00000000000..1d854a35a3d
--- /dev/null
+++ b/dpcpp/factorization/ilu_kernels.dp.cpp
@@ -0,0 +1,58 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/ilu_kernels.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The ilu factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace ilu_factorization {
+
+
+template <typename ValueType, typename IndexType>
+void compute_lu(std::shared_ptr<const DefaultExecutor> exec,
+                matrix::Csr<ValueType, IndexType> *m) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ILU_COMPUTE_LU_KERNEL);
+
+
+}  // namespace ilu_factorization
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/factorization/par_ict_kernels.dp.cpp b/dpcpp/factorization/par_ict_kernels.dp.cpp
new file mode 100644
index 00000000000..6e7dd7e5563
--- /dev/null
+++ b/dpcpp/factorization/par_ict_kernels.dp.cpp
@@ -0,0 +1,95 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/par_ict_kernels.hpp"
+
+
+#include <algorithm>
+#include <tuple>
+#include <unordered_map>
+#include <unordered_set>
+
+
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/base/utils.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/csr_builder.hpp"
+#include "dpcpp/components/csr_spgeam.dp.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The parallel ICT factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace par_ict_factorization {
+
+
+template <typename ValueType, typename IndexType>
+void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
+                    const matrix::Csr<ValueType, IndexType> *a,
+                    matrix::Csr<ValueType, IndexType> *l,
+                    const matrix::Coo<ValueType, IndexType> *)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_ICT_COMPUTE_FACTOR_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
+                    const matrix::Csr<ValueType, IndexType> *llt,
+                    const matrix::Csr<ValueType, IndexType> *a,
+                    const matrix::Csr<ValueType, IndexType> *l,
+                    matrix::Csr<ValueType, IndexType> *l_new)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_ICT_ADD_CANDIDATES_KERNEL);
+
+
+}  // namespace par_ict_factorization
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/factorization/par_ilu_kernels.dp.cpp b/dpcpp/factorization/par_ilu_kernels.dp.cpp
new file mode 100644
index 00000000000..baa8b4a4481
--- /dev/null
+++ b/dpcpp/factorization/par_ilu_kernels.dp.cpp
@@ -0,0 +1,72 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/par_ilu_kernels.hpp"
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The parallel ILU factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace par_ilu_factorization {
+
+
+template <typename ValueType, typename IndexType>
+void compute_l_u_factors(std::shared_ptr<const DpcppExecutor> exec,
+                         size_type iterations,
+                         const matrix::Coo<ValueType, IndexType> *system_matrix,
+                         matrix::Csr<ValueType, IndexType> *l_factor,
+                         matrix::Csr<ValueType, IndexType> *u_factor)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_ILU_COMPUTE_L_U_FACTORS_KERNEL);
+
+
+}  // namespace par_ilu_factorization
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/factorization/par_ilut_kernels.dp.cpp b/dpcpp/factorization/par_ilut_kernels.dp.cpp
new file mode 100644
index 00000000000..6b7f5b0ac17
--- /dev/null
+++ b/dpcpp/factorization/par_ilut_kernels.dp.cpp
@@ -0,0 +1,168 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/par_ilut_kernels.hpp"
+
+
+#include <algorithm>
+#include <tuple>
+#include <unordered_map>
+#include <unordered_set>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/base/utils.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/coo_builder.hpp"
+#include "core/matrix/csr_builder.hpp"
+#include "dpcpp/components/csr_spgeam.dp.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The parallel ILUT factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace par_ilut_factorization {
+
+
+template <typename ValueType, typename IndexType>
+void threshold_select(std::shared_ptr<const DefaultExecutor> exec,
+                      const matrix::Csr<ValueType, IndexType> *m,
+                      IndexType rank, Array<ValueType> &tmp,
+                      Array<remove_complex<ValueType>> &,
+                      remove_complex<ValueType> &threshold)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_ILUT_THRESHOLD_SELECT_KERNEL);
+
+
+/**
+ * Removes all the elements from the input matrix for which pred is false.
+ * Stores the result in m_out and (if non-null) m_out_coo.
+ * pred(row, nz) is called for each entry, where nz is the index in
+ * values/col_idxs.
+ */
+template <typename Predicate, typename ValueType, typename IndexType>
+void abstract_filter(std::shared_ptr<const DefaultExecutor> exec,
+                     const matrix::Csr<ValueType, IndexType> *m,
+                     matrix::Csr<ValueType, IndexType> *m_out,
+                     matrix::Coo<ValueType, IndexType> *m_out_coo,
+                     Predicate pred)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void threshold_filter(std::shared_ptr<const DefaultExecutor> exec,
+                      const matrix::Csr<ValueType, IndexType> *m,
+                      remove_complex<ValueType> threshold,
+                      matrix::Csr<ValueType, IndexType> *m_out,
+                      matrix::Coo<ValueType, IndexType> *m_out_coo, bool)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_ILUT_THRESHOLD_FILTER_KERNEL);
+
+
+constexpr auto bucket_count = 1 << sampleselect_searchtree_height;
+constexpr auto sample_size = bucket_count * sampleselect_oversampling;
+
+
+template <typename ValueType, typename IndexType>
+void threshold_filter_approx(std::shared_ptr<const DefaultExecutor> exec,
+                             const matrix::Csr<ValueType, IndexType> *m,
+                             IndexType rank, Array<ValueType> &tmp,
+                             remove_complex<ValueType> &threshold,
+                             matrix::Csr<ValueType, IndexType> *m_out,
+                             matrix::Coo<ValueType, IndexType> *m_out_coo)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_ILUT_THRESHOLD_FILTER_APPROX_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void compute_l_u_factors(std::shared_ptr<const DefaultExecutor> exec,
+                         const matrix::Csr<ValueType, IndexType> *a,
+                         matrix::Csr<ValueType, IndexType> *l,
+                         const matrix::Coo<ValueType, IndexType> *,
+                         matrix::Csr<ValueType, IndexType> *u,
+                         const matrix::Coo<ValueType, IndexType> *,
+                         matrix::Csr<ValueType, IndexType> *u_csc)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_ILUT_COMPUTE_LU_FACTORS_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
+                    const matrix::Csr<ValueType, IndexType> *lu,
+                    const matrix::Csr<ValueType, IndexType> *a,
+                    const matrix::Csr<ValueType, IndexType> *l,
+                    const matrix::Csr<ValueType, IndexType> *u,
+                    matrix::Csr<ValueType, IndexType> *l_new,
+                    matrix::Csr<ValueType, IndexType> *u_new)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_ILUT_ADD_CANDIDATES_KERNEL);
+
+
+}  // namespace par_ilut_factorization
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/get_info.cmake b/dpcpp/get_info.cmake
new file mode 100644
index 00000000000..475a53f5dcf
--- /dev/null
+++ b/dpcpp/get_info.cmake
@@ -0,0 +1,4 @@
+ginkgo_print_module_header(${detailed_log} "DPCPP")
+ginkgo_print_module_footer(${detailed_log} "DPCPP variables:")
+ginkgo_print_variable(${detailed_log} "GINKGO_DPCPP_FLAGS")
+ginkgo_print_module_footer(${detailed_log} "")
diff --git a/dpcpp/matrix/coo_kernels.dp.cpp b/dpcpp/matrix/coo_kernels.dp.cpp
new file mode 100644
index 00000000000..b72b24d1fa5
--- /dev/null
+++ b/dpcpp/matrix/coo_kernels.dp.cpp
@@ -0,0 +1,163 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/coo_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "dpcpp/components/format_conversion.dp.hpp"
+
+
+namespace gko {
+namespace kernels {
+/**
+ * @brief DPCPP namespace.
+ *
+ * @ingroup dpcpp
+ */
+namespace dpcpp {
+/**
+ * @brief The Coordinate matrix format namespace.
+ *
+ * @ingroup coo
+ */
+namespace coo {
+
+
+template <typename ValueType, typename IndexType>
+void spmv(std::shared_ptr<const DpcppExecutor> exec,
+          const matrix::Coo<ValueType, IndexType> *a,
+          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
+                   const matrix::Dense<ValueType> *alpha,
+                   const matrix::Coo<ValueType, IndexType> *a,
+                   const matrix::Dense<ValueType> *b,
+                   const matrix::Dense<ValueType> *beta,
+                   matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_COO_ADVANCED_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void spmv2(std::shared_ptr<const DpcppExecutor> exec,
+           const matrix::Coo<ValueType, IndexType> *a,
+           const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_SPMV2_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spmv2(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Dense<ValueType> *alpha,
+                    const matrix::Coo<ValueType, IndexType> *a,
+                    const matrix::Dense<ValueType> *b,
+                    matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_COO_ADVANCED_SPMV2_KERNEL);
+
+
+template <typename IndexType>
+void convert_row_idxs_to_ptrs(std::shared_ptr<const DpcppExecutor> exec,
+                              const IndexType *idxs, size_type num_nonzeros,
+                              IndexType *ptrs, size_type length)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Coo<ValueType, IndexType> *source,
+                    matrix::Csr<ValueType, IndexType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_COO_CONVERT_TO_CSR_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Coo<ValueType, IndexType> *source,
+                      matrix::Dense<ValueType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_COO_CONVERT_TO_DENSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Coo<ValueType, IndexType> *orig,
+                      matrix::Diagonal<ValueType> *diag)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_COO_EXTRACT_DIAGONAL_KERNEL);
+
+
+}  // namespace coo
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
new file mode 100644
index 00000000000..3516b7efc04
--- /dev/null
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -0,0 +1,425 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/csr_kernels.hpp"
+
+
+#include <algorithm>
+#include <numeric>
+#include <utility>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/hybrid.hpp>
+
+
+#include "core/base/allocator.hpp"
+#include "core/base/iterator_factory.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/csr_builder.hpp"
+#include "dpcpp/components/csr_spgeam.dp.hpp"
+#include "dpcpp/components/format_conversion.dp.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The Compressed sparse row matrix format namespace.
+ *
+ * @ingroup csr
+ */
+namespace csr {
+
+
+template <typename ValueType, typename IndexType>
+void spmv(std::shared_ptr<const DpcppExecutor> exec,
+          const matrix::Csr<ValueType, IndexType> *a,
+          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
+                   const matrix::Dense<ValueType> *alpha,
+                   const matrix::Csr<ValueType, IndexType> *a,
+                   const matrix::Dense<ValueType> *b,
+                   const matrix::Dense<ValueType> *beta,
+                   matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_ADVANCED_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_insert_row(unordered_set<IndexType> &cols,
+                       const matrix::Csr<ValueType, IndexType> *c,
+                       size_type row)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_insert_row2(unordered_set<IndexType> &cols,
+                        const matrix::Csr<ValueType, IndexType> *a,
+                        const matrix::Csr<ValueType, IndexType> *b,
+                        size_type row)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_accumulate_row(map<IndexType, ValueType> &cols,
+                           const matrix::Csr<ValueType, IndexType> *c,
+                           ValueType scale, size_type row)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_accumulate_row2(map<IndexType, ValueType> &cols,
+                            const matrix::Csr<ValueType, IndexType> *a,
+                            const matrix::Csr<ValueType, IndexType> *b,
+                            ValueType scale, size_type row)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void spgemm(std::shared_ptr<const DpcppExecutor> exec,
+            const matrix::Csr<ValueType, IndexType> *a,
+            const matrix::Csr<ValueType, IndexType> *b,
+            matrix::Csr<ValueType, IndexType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEMM_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spgemm(std::shared_ptr<const DpcppExecutor> exec,
+                     const matrix::Dense<ValueType> *alpha,
+                     const matrix::Csr<ValueType, IndexType> *a,
+                     const matrix::Csr<ValueType, IndexType> *b,
+                     const matrix::Dense<ValueType> *beta,
+                     const matrix::Csr<ValueType, IndexType> *d,
+                     matrix::Csr<ValueType, IndexType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_ADVANCED_SPGEMM_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void spgeam(std::shared_ptr<const DpcppExecutor> exec,
+            const matrix::Dense<ValueType> *alpha,
+            const matrix::Csr<ValueType, IndexType> *a,
+            const matrix::Dense<ValueType> *beta,
+            const matrix::Csr<ValueType, IndexType> *b,
+            matrix::Csr<ValueType, IndexType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEAM_KERNEL);
+
+
+template <typename IndexType>
+void convert_row_ptrs_to_idxs(std::shared_ptr<const DpcppExecutor> exec,
+                              const IndexType *ptrs, size_type num_rows,
+                              IndexType *idxs)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_coo(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Csr<ValueType, IndexType> *source,
+                    matrix::Coo<ValueType, IndexType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_CONVERT_TO_COO_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Csr<ValueType, IndexType> *source,
+                      matrix::Dense<ValueType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_CONVERT_TO_DENSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Csr<ValueType, IndexType> *source,
+                      matrix::Sellp<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_CONVERT_TO_SELLP_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_ell(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Csr<ValueType, IndexType> *source,
+                    matrix::Ell<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_CONVERT_TO_ELL_KERNEL);
+
+
+template <typename ValueType, typename IndexType, typename UnaryOperator>
+inline void convert_csr_to_csc(size_type num_rows, const IndexType *row_ptrs,
+                               const IndexType *col_idxs,
+                               const ValueType *csr_vals, IndexType *row_idxs,
+                               IndexType *col_ptrs, ValueType *csc_vals,
+                               UnaryOperator op)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType, typename UnaryOperator>
+void transpose_and_transform(std::shared_ptr<const DpcppExecutor> exec,
+                             matrix::Csr<ValueType, IndexType> *trans,
+                             const matrix::Csr<ValueType, IndexType> *orig,
+                             UnaryOperator op)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void transpose(std::shared_ptr<const DpcppExecutor> exec,
+               const matrix::Csr<ValueType, IndexType> *orig,
+               matrix::Csr<ValueType, IndexType> *trans)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Csr<ValueType, IndexType> *orig,
+                    matrix::Csr<ValueType, IndexType> *trans)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_CONJ_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
+                          const matrix::Csr<ValueType, IndexType> *source,
+                          size_type *result, size_type stride_factor,
+                          size_type slice_size) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_CALCULATE_TOTAL_COLS_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_max_nnz_per_row(std::shared_ptr<const DpcppExecutor> exec,
+                               const matrix::Csr<ValueType, IndexType> *source,
+                               size_type *result) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_hybrid(std::shared_ptr<const DpcppExecutor> exec,
+                       const matrix::Csr<ValueType, IndexType> *source,
+                       matrix::Hybrid<ValueType, IndexType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_CONVERT_TO_HYBRID_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void row_permute_impl(std::shared_ptr<const DpcppExecutor> exec,
+                      const Array<IndexType> *permutation_indices,
+                      const matrix::Csr<ValueType, IndexType> *orig,
+                      matrix::Csr<ValueType, IndexType> *row_permuted)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void row_permute(std::shared_ptr<const DpcppExecutor> exec,
+                 const Array<IndexType> *permutation_indices,
+                 const matrix::Csr<ValueType, IndexType> *orig,
+                 matrix::Csr<ValueType, IndexType> *row_permuted)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_row_permute(std::shared_ptr<const DpcppExecutor> exec,
+                         const Array<IndexType> *permutation_indices,
+                         const matrix::Csr<ValueType, IndexType> *orig,
+                         matrix::Csr<ValueType, IndexType> *row_permuted)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void column_permute_impl(const Array<IndexType> *permutation_indices,
+                         const matrix::Csr<ValueType, IndexType> *orig,
+                         matrix::Csr<ValueType, IndexType> *column_permuted)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void column_permute(std::shared_ptr<const DpcppExecutor> exec,
+                    const Array<IndexType> *permutation_indices,
+                    const matrix::Csr<ValueType, IndexType> *orig,
+                    matrix::Csr<ValueType, IndexType> *column_permuted)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_column_permute(std::shared_ptr<const DpcppExecutor> exec,
+                            const Array<IndexType> *permutation_indices,
+                            const matrix::Csr<ValueType, IndexType> *orig,
+                            matrix::Csr<ValueType, IndexType> *column_permuted)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_nonzeros_per_row(std::shared_ptr<const DpcppExecutor> exec,
+                                const matrix::Csr<ValueType, IndexType> *source,
+                                Array<size_type> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void sort_by_column_index(std::shared_ptr<const DpcppExecutor> exec,
+                          matrix::Csr<ValueType, IndexType> *to_sort)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_SORT_BY_COLUMN_INDEX);
+
+
+template <typename ValueType, typename IndexType>
+void is_sorted_by_column_index(
+    std::shared_ptr<const DpcppExecutor> exec,
+    const matrix::Csr<ValueType, IndexType> *to_check, bool *is_sorted)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_IS_SORTED_BY_COLUMN_INDEX);
+
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Csr<ValueType, IndexType> *orig,
+                      matrix::Diagonal<ValueType> *diag)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_EXTRACT_DIAGONAL);
+
+
+}  // namespace csr
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
new file mode 100644
index 00000000000..5f76e1b5b97
--- /dev/null
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -0,0 +1,375 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/dense_kernels.hpp"
+
+
+#include <algorithm>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/range_accessors.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+#include <ginkgo/core/matrix/ell.hpp>
+#include <ginkgo/core/matrix/hybrid.hpp>
+#include <ginkgo/core/matrix/sellp.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+
+
+#include "core/components/prefix_sum.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The Dense matrix format namespace.
+ *
+ * @ingroup dense
+ */
+namespace dense {
+
+
+template <typename ValueType>
+void simple_apply(std::shared_ptr<const DpcppExecutor> exec,
+                  const matrix::Dense<ValueType> *a,
+                  const matrix::Dense<ValueType> *b,
+                  matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL);
+
+
+template <typename ValueType>
+void apply(std::shared_ptr<const DpcppExecutor> exec,
+           const matrix::Dense<ValueType> *alpha,
+           const matrix::Dense<ValueType> *a, const matrix::Dense<ValueType> *b,
+           const matrix::Dense<ValueType> *beta, matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
+
+
+template <typename ValueType>
+void scale(std::shared_ptr<const DpcppExecutor> exec,
+           const matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *x)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SCALE_KERNEL);
+
+
+template <typename ValueType>
+void add_scaled(std::shared_ptr<const DpcppExecutor> exec,
+                const matrix::Dense<ValueType> *alpha,
+                const matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *y)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_KERNEL);
+
+
+template <typename ValueType>
+void add_scaled_diag(std::shared_ptr<const DpcppExecutor> exec,
+                     const matrix::Dense<ValueType> *alpha,
+                     const matrix::Diagonal<ValueType> *x,
+                     matrix::Dense<ValueType> *y)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL);
+
+
+template <typename ValueType>
+void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
+                 const matrix::Dense<ValueType> *x,
+                 const matrix::Dense<ValueType> *y,
+                 matrix::Dense<ValueType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL);
+
+
+template <typename ValueType>
+void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
+                   const matrix::Dense<ValueType> *x,
+                   matrix::Dense<remove_complex<ValueType>> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_NORM2_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_coo(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Dense<ValueType> *source,
+                    matrix::Coo<ValueType, IndexType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_CONVERT_TO_COO_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Dense<ValueType> *source,
+                    matrix::Csr<ValueType, IndexType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_CONVERT_TO_CSR_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_ell(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Dense<ValueType> *source,
+                    matrix::Ell<ValueType, IndexType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_CONVERT_TO_ELL_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_hybrid(std::shared_ptr<const DpcppExecutor> exec,
+                       const matrix::Dense<ValueType> *source,
+                       matrix::Hybrid<ValueType, IndexType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_CONVERT_TO_HYBRID_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Dense<ValueType> *source,
+                      matrix::Sellp<ValueType, IndexType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_CONVERT_TO_SELLP_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_sparsity_csr(std::shared_ptr<const DpcppExecutor> exec,
+                             const matrix::Dense<ValueType> *source,
+                             matrix::SparsityCsr<ValueType, IndexType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_CONVERT_TO_SPARSITY_CSR_KERNEL);
+
+
+template <typename ValueType>
+void count_nonzeros(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Dense<ValueType> *source, size_type *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COUNT_NONZEROS_KERNEL);
+
+
+template <typename ValueType>
+void calculate_max_nnz_per_row(std::shared_ptr<const DpcppExecutor> exec,
+                               const matrix::Dense<ValueType> *source,
+                               size_type *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_DENSE_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
+
+
+template <typename ValueType>
+void calculate_nonzeros_per_row(std::shared_ptr<const DpcppExecutor> exec,
+                                const matrix::Dense<ValueType> *source,
+                                Array<size_type> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_DENSE_CALCULATE_NONZEROS_PER_ROW_KERNEL);
+
+
+template <typename ValueType>
+void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
+                          const matrix::Dense<ValueType> *source,
+                          size_type *result, size_type stride_factor,
+                          size_type slice_size)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_DENSE_CALCULATE_TOTAL_COLS_KERNEL);
+
+
+template <typename ValueType>
+void transpose(std::shared_ptr<const DpcppExecutor> exec,
+               const matrix::Dense<ValueType> *orig,
+               matrix::Dense<ValueType> *trans)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType>
+void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Dense<ValueType> *orig,
+                    matrix::Dense<ValueType> *trans)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CONJ_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void row_permute(std::shared_ptr<const DpcppExecutor> exec,
+                 const Array<IndexType> *permutation_indices,
+                 const matrix::Dense<ValueType> *orig,
+                 matrix::Dense<ValueType> *row_permuted)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void column_permute(std::shared_ptr<const DpcppExecutor> exec,
+                    const Array<IndexType> *permutation_indices,
+                    const matrix::Dense<ValueType> *orig,
+                    matrix::Dense<ValueType> *column_permuted)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_row_permute(std::shared_ptr<const DpcppExecutor> exec,
+                         const Array<IndexType> *permutation_indices,
+                         const matrix::Dense<ValueType> *orig,
+                         matrix::Dense<ValueType> *row_permuted)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_INVERSE_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_column_permute(std::shared_ptr<const DpcppExecutor> exec,
+                            const Array<IndexType> *permutation_indices,
+                            const matrix::Dense<ValueType> *orig,
+                            matrix::Dense<ValueType> *column_permuted)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_INVERSE_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType>
+void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Dense<ValueType> *orig,
+                      matrix::Diagonal<ValueType> *diag)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL);
+
+
+template <typename ValueType>
+void inplace_absolute_dense(std::shared_ptr<const DpcppExecutor> exec,
+                            matrix::Dense<ValueType> *source)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL);
+
+
+template <typename ValueType>
+void outplace_absolute_dense(std::shared_ptr<const DpcppExecutor> exec,
+                             const matrix::Dense<ValueType> *source,
+                             matrix::Dense<remove_complex<ValueType>> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
+
+
+}  // namespace dense
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/matrix/diagonal_kernels.dp.cpp b/dpcpp/matrix/diagonal_kernels.dp.cpp
new file mode 100644
index 00000000000..c2090a17dc4
--- /dev/null
+++ b/dpcpp/matrix/diagonal_kernels.dp.cpp
@@ -0,0 +1,131 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/diagonal_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The Diagonal matrix format namespace.
+ *
+ * @ingroup diagonal
+ */
+namespace diagonal {
+
+
+template <typename ValueType>
+void apply_to_dense(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Diagonal<ValueType> *a,
+                    const matrix::Dense<ValueType> *b,
+                    matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_APPLY_TO_DENSE_KERNEL);
+
+
+template <typename ValueType>
+void right_apply_to_dense(std::shared_ptr<const DpcppExecutor> exec,
+                          const matrix::Diagonal<ValueType> *a,
+                          const matrix::Dense<ValueType> *b,
+                          matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_DENSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void apply_to_csr(std::shared_ptr<const DpcppExecutor> exec,
+                  const matrix::Diagonal<ValueType> *a,
+                  const matrix::Csr<ValueType, IndexType> *b,
+                  matrix::Csr<ValueType, IndexType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DIAGONAL_APPLY_TO_CSR_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void right_apply_to_csr(std::shared_ptr<const DpcppExecutor> exec,
+                        const matrix::Diagonal<ValueType> *a,
+                        const matrix::Csr<ValueType, IndexType> *b,
+                        matrix::Csr<ValueType, IndexType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_CSR_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Diagonal<ValueType> *source,
+                    matrix::Csr<ValueType, IndexType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DIAGONAL_CONVERT_TO_CSR_KERNEL);
+
+
+template <typename ValueType>
+void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Diagonal<ValueType> *orig,
+                    matrix::Diagonal<ValueType> *trans)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_CONJ_TRANSPOSE_KERNEL);
+
+
+}  // namespace diagonal
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/matrix/ell_kernels.dp.cpp b/dpcpp/matrix/ell_kernels.dp.cpp
new file mode 100644
index 00000000000..d27a19408b4
--- /dev/null
+++ b/dpcpp/matrix/ell_kernels.dp.cpp
@@ -0,0 +1,142 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/ell_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "dpcpp/components/format_conversion.dp.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The ELL matrix format namespace.
+ *
+ * @ingroup ell
+ */
+namespace ell {
+
+
+template <typename ValueType, typename IndexType>
+void spmv(std::shared_ptr<const DpcppExecutor> exec,
+          const matrix::Ell<ValueType, IndexType> *a,
+          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ELL_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
+                   const matrix::Dense<ValueType> *alpha,
+                   const matrix::Ell<ValueType, IndexType> *a,
+                   const matrix::Dense<ValueType> *b,
+                   const matrix::Dense<ValueType> *beta,
+                   matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Ell<ValueType, IndexType> *source,
+                      matrix::Dense<ValueType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ELL_CONVERT_TO_DENSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Ell<ValueType, IndexType> *source,
+                    matrix::Csr<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ELL_CONVERT_TO_CSR_KERNEL);
+
+template <typename ValueType, typename IndexType>
+void count_nonzeros(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Ell<ValueType, IndexType> *source,
+                    size_type *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ELL_COUNT_NONZEROS_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_nonzeros_per_row(std::shared_ptr<const DpcppExecutor> exec,
+                                const matrix::Ell<ValueType, IndexType> *source,
+                                Array<size_type> *result) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ELL_CALCULATE_NONZEROS_PER_ROW_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Ell<ValueType, IndexType> *orig,
+                      matrix::Diagonal<ValueType> *diag)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ELL_EXTRACT_DIAGONAL_KERNEL);
+
+
+}  // namespace ell
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/matrix/hybrid_kernels.dp.cpp b/dpcpp/matrix/hybrid_kernels.dp.cpp
new file mode 100644
index 00000000000..f740541cccc
--- /dev/null
+++ b/dpcpp/matrix/hybrid_kernels.dp.cpp
@@ -0,0 +1,99 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/hybrid_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/matrix/ell_kernels.hpp"
+#include "dpcpp/components/format_conversion.dp.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The Hybrid matrix format namespace.
+ *
+ * @ingroup hybrid
+ */
+namespace hybrid {
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Hybrid<ValueType, IndexType> *source,
+                      matrix::Dense<ValueType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_HYBRID_CONVERT_TO_DENSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Hybrid<ValueType, IndexType> *source,
+                    matrix::Csr<ValueType, IndexType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_HYBRID_CONVERT_TO_CSR_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void count_nonzeros(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Hybrid<ValueType, IndexType> *source,
+                    size_type *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_HYBRID_COUNT_NONZEROS_KERNEL);
+
+
+}  // namespace hybrid
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/matrix/sellp_kernels.dp.cpp b/dpcpp/matrix/sellp_kernels.dp.cpp
new file mode 100644
index 00000000000..20f6adcc2a0
--- /dev/null
+++ b/dpcpp/matrix/sellp_kernels.dp.cpp
@@ -0,0 +1,125 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/sellp_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The SELL-P matrix format namespace.
+ *
+ * @ingroup sellp
+ */
+namespace sellp {
+
+
+template <typename ValueType, typename IndexType>
+void spmv(std::shared_ptr<const DpcppExecutor> exec,
+          const matrix::Sellp<ValueType, IndexType> *a,
+          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_SELLP_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
+                   const matrix::Dense<ValueType> *alpha,
+                   const matrix::Sellp<ValueType, IndexType> *a,
+                   const matrix::Dense<ValueType> *b,
+                   const matrix::Dense<ValueType> *beta,
+                   matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_SELLP_ADVANCED_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Sellp<ValueType, IndexType> *source,
+                      matrix::Dense<ValueType> *result)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_SELLP_CONVERT_TO_DENSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Sellp<ValueType, IndexType> *source,
+                    matrix::Csr<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_SELLP_CONVERT_TO_CSR_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void count_nonzeros(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Sellp<ValueType, IndexType> *source,
+                    size_type *result) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_SELLP_COUNT_NONZEROS_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Sellp<ValueType, IndexType> *orig,
+                      matrix::Diagonal<ValueType> *diag)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_SELLP_EXTRACT_DIAGONAL_KERNEL);
+
+
+}  // namespace sellp
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/matrix/sparsity_csr_kernels.dp.cpp b/dpcpp/matrix/sparsity_csr_kernels.dp.cpp
new file mode 100644
index 00000000000..ba682e194e7
--- /dev/null
+++ b/dpcpp/matrix/sparsity_csr_kernels.dp.cpp
@@ -0,0 +1,175 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/sparsity_csr_kernels.hpp"
+
+
+#include <algorithm>
+#include <numeric>
+#include <utility>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/base/iterator_factory.hpp"
+#include "dpcpp/components/format_conversion.dp.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The SparsityCsr pattern format namespace.
+ *
+ * @ingroup sparsity
+ */
+namespace sparsity_csr {
+
+
+template <typename ValueType, typename IndexType>
+void spmv(std::shared_ptr<const DpcppExecutor> exec,
+          const matrix::SparsityCsr<ValueType, IndexType> *a,
+          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_SPARSITY_CSR_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
+                   const matrix::Dense<ValueType> *alpha,
+                   const matrix::SparsityCsr<ValueType, IndexType> *a,
+                   const matrix::Dense<ValueType> *b,
+                   const matrix::Dense<ValueType> *beta,
+                   matrix::Dense<ValueType> *c)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_SPARSITY_CSR_ADVANCED_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void count_num_diagonal_elements(
+    std::shared_ptr<const DpcppExecutor> exec,
+    const matrix::SparsityCsr<ValueType, IndexType> *matrix,
+    size_type *num_diagonal_elements)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_SPARSITY_CSR_COUNT_NUM_DIAGONAL_ELEMENTS_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void remove_diagonal_elements(std::shared_ptr<const DpcppExecutor> exec,
+                              const IndexType *row_ptrs,
+                              const IndexType *col_idxs,
+                              matrix::SparsityCsr<ValueType, IndexType> *matrix)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_SPARSITY_CSR_REMOVE_DIAGONAL_ELEMENTS_KERNEL);
+
+
+template <typename IndexType>
+inline void convert_sparsity_to_csc(size_type num_rows,
+                                    const IndexType *row_ptrs,
+                                    const IndexType *col_idxs,
+                                    IndexType *row_idxs, IndexType *col_ptrs)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void transpose_and_transform(
+    std::shared_ptr<const DpcppExecutor> exec,
+    matrix::SparsityCsr<ValueType, IndexType> *trans,
+    const matrix::SparsityCsr<ValueType, IndexType> *orig)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void transpose(std::shared_ptr<const DpcppExecutor> exec,
+               const matrix::SparsityCsr<ValueType, IndexType> *orig,
+               matrix::SparsityCsr<ValueType, IndexType> *trans)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_SPARSITY_CSR_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void sort_by_column_index(std::shared_ptr<const DpcppExecutor> exec,
+                          matrix::SparsityCsr<ValueType, IndexType> *to_sort)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_SPARSITY_CSR_SORT_BY_COLUMN_INDEX);
+
+
+template <typename ValueType, typename IndexType>
+void is_sorted_by_column_index(
+    std::shared_ptr<const DpcppExecutor> exec,
+    const matrix::SparsityCsr<ValueType, IndexType> *to_check, bool *is_sorted)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_SPARSITY_CSR_IS_SORTED_BY_COLUMN_INDEX);
+
+
+}  // namespace sparsity_csr
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/preconditioner/isai_kernels.dp.cpp b/dpcpp/preconditioner/isai_kernels.dp.cpp
new file mode 100644
index 00000000000..481979eed3b
--- /dev/null
+++ b/dpcpp/preconditioner/isai_kernels.dp.cpp
@@ -0,0 +1,129 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/preconditioner/isai_kernels.hpp"
+
+
+#include <algorithm>
+#include <memory>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/csr_builder.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The Isai preconditioner namespace.
+ *
+ * @ingroup isai
+ */
+namespace isai {
+
+
+template <typename IndexType, typename Callback>
+void forall_matching(const IndexType *fst, IndexType fst_size,
+                     const IndexType *snd, IndexType snd_size, Callback cb)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType, typename Callable>
+void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
+                      const matrix::Csr<ValueType, IndexType> *mtx,
+                      matrix::Csr<ValueType, IndexType> *inverse_mtx,
+                      IndexType *excess_rhs_ptrs, IndexType *excess_nz_ptrs,
+                      Callable trs_solve)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+void generate_tri_inverse(std::shared_ptr<const DefaultExecutor> exec,
+                          const matrix::Csr<ValueType, IndexType> *mtx,
+                          matrix::Csr<ValueType, IndexType> *inverse_mtx,
+                          IndexType *excess_rhs_ptrs, IndexType *excess_nz_ptrs,
+                          bool lower)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_GENERATE_TRI_INVERSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
+                            const matrix::Csr<ValueType, IndexType> *input,
+                            const matrix::Csr<ValueType, IndexType> *inverse,
+                            const IndexType *excess_rhs_ptrs,
+                            const IndexType *excess_nz_ptrs,
+                            matrix::Csr<ValueType, IndexType> *excess_system,
+                            matrix::Dense<ValueType> *excess_rhs)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void scatter_excess_solution(std::shared_ptr<const DefaultExecutor>,
+                             const IndexType *excess_block_ptrs,
+                             const matrix::Dense<ValueType> *excess_solution,
+                             matrix::Csr<ValueType, IndexType> *inverse)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_SCATTER_EXCESS_SOLUTION_KERNEL);
+
+
+}  // namespace isai
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/preconditioner/jacobi_kernels.dp.cpp b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
new file mode 100644
index 00000000000..06b00835e82
--- /dev/null
+++ b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
@@ -0,0 +1,345 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/preconditioner/jacobi_kernels.hpp"
+
+
+#include <algorithm>
+#include <cmath>
+#include <iterator>
+#include <numeric>
+#include <vector>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/base/allocator.hpp"
+#include "core/base/extended_float.hpp"
+#include "core/preconditioner/jacobi_utils.hpp"
+#include "dpcpp/components/matrix_operations.dp.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The Jacobi preconditioner namespace.
+ *
+ * @ingroup jacobi
+ */
+namespace jacobi {
+
+
+void initialize_precisions(std::shared_ptr<const DpcppExecutor> exec,
+                           const Array<precision_reduction> &source,
+                           Array<precision_reduction> &precisions)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+namespace {
+
+
+template <typename IndexType>
+inline bool has_same_nonzero_pattern(const IndexType *prev_row_ptr,
+                                     const IndexType *curr_row_ptr,
+                                     const IndexType *next_row_ptr)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+size_type find_natural_blocks(const matrix::Csr<ValueType, IndexType> *mtx,
+                              uint32 max_block_size, IndexType *block_ptrs)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename IndexType>
+inline size_type agglomerate_supervariables(uint32 max_block_size,
+                                            size_type num_natural_blocks,
+                                            IndexType *block_ptrs)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+}  // namespace
+
+
+template <typename ValueType, typename IndexType>
+void find_blocks(std::shared_ptr<const DpcppExecutor> exec,
+                 const matrix::Csr<ValueType, IndexType> *system_matrix,
+                 uint32 max_block_size, size_type &num_blocks,
+                 Array<IndexType> &block_pointers)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_JACOBI_FIND_BLOCKS_KERNEL);
+
+
+namespace {
+
+
+template <typename ValueType, typename IndexType>
+inline void extract_block(const matrix::Csr<ValueType, IndexType> *mtx,
+                          IndexType block_size, IndexType block_start,
+                          ValueType *block, size_type stride)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+inline IndexType choose_pivot(IndexType block_size, const ValueType *block,
+                              size_type stride)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+inline void swap_rows(IndexType row1, IndexType row2, IndexType block_size,
+                      ValueType *block, size_type stride)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+inline bool apply_gauss_jordan_transform(IndexType row, IndexType col,
+                                         IndexType block_size, ValueType *block,
+                                         size_type stride)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename SourceValueType, typename ResultValueType,
+          typename IndexType,
+          typename ValueConverter =
+              default_converter<SourceValueType, ResultValueType>>
+inline void transpose_block(IndexType block_size, const SourceValueType *from,
+                            size_type from_stride, ResultValueType *to,
+                            size_type to_stride,
+                            ValueConverter converter = {}) noexcept
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename SourceValueType, typename ResultValueType,
+          typename IndexType,
+          typename ValueConverter =
+              default_converter<SourceValueType, ResultValueType>>
+inline void conj_transpose_block(IndexType block_size,
+                                 const SourceValueType *from,
+                                 size_type from_stride, ResultValueType *to,
+                                 size_type to_stride,
+                                 ValueConverter converter = {}) noexcept
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename SourceValueType, typename ResultValueType,
+          typename IndexType,
+          typename ValueConverter =
+              default_converter<SourceValueType, ResultValueType>>
+inline void permute_and_transpose_block(IndexType block_size,
+                                        const IndexType *col_perm,
+                                        const SourceValueType *source,
+                                        size_type source_stride,
+                                        ResultValueType *result,
+                                        size_type result_stride,
+                                        ValueConverter converter = {})
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType, typename IndexType>
+inline bool invert_block(IndexType block_size, IndexType *perm,
+                         ValueType *block, size_type stride)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ReducedType, typename ValueType, typename IndexType>
+inline bool validate_precision_reduction_feasibility(
+    std::shared_ptr<const DpcppExecutor> exec, IndexType block_size,
+    const ValueType *block, size_type stride)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+}  // namespace
+
+
+template <typename ValueType, typename IndexType>
+void generate(std::shared_ptr<const DpcppExecutor> exec,
+              const matrix::Csr<ValueType, IndexType> *system_matrix,
+              size_type num_blocks, uint32 max_block_size,
+              remove_complex<ValueType> accuracy,
+              const preconditioner::block_interleaved_storage_scheme<IndexType>
+                  &storage_scheme,
+              Array<remove_complex<ValueType>> &conditioning,
+              Array<precision_reduction> &block_precisions,
+              const Array<IndexType> &block_pointers, Array<ValueType> &blocks)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_JACOBI_GENERATE_KERNEL);
+
+
+namespace {
+
+
+template <
+    typename ValueType, typename BlockValueType,
+    typename ValueConverter = default_converter<BlockValueType, ValueType>>
+inline void apply_block(size_type block_size, size_type num_rhs,
+                        const BlockValueType *block, size_type stride,
+                        ValueType alpha, const ValueType *b, size_type stride_b,
+                        ValueType beta, ValueType *x, size_type stride_x,
+                        ValueConverter converter = {})
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+}  // namespace
+
+
+template <typename ValueType, typename IndexType>
+void apply(std::shared_ptr<const DpcppExecutor> exec, size_type num_blocks,
+           uint32 max_block_size,
+           const preconditioner::block_interleaved_storage_scheme<IndexType>
+               &storage_scheme,
+           const Array<precision_reduction> &block_precisions,
+           const Array<IndexType> &block_pointers,
+           const Array<ValueType> &blocks,
+           const matrix::Dense<ValueType> *alpha,
+           const matrix::Dense<ValueType> *b,
+           const matrix::Dense<ValueType> *beta, matrix::Dense<ValueType> *x)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_JACOBI_APPLY_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void simple_apply(
+    std::shared_ptr<const DpcppExecutor> exec, size_type num_blocks,
+    uint32 max_block_size,
+    const preconditioner::block_interleaved_storage_scheme<IndexType>
+        &storage_scheme,
+    const Array<precision_reduction> &block_precisions,
+    const Array<IndexType> &block_pointers, const Array<ValueType> &blocks,
+    const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *x)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_JACOBI_SIMPLE_APPLY_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void transpose_jacobi(
+    std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks,
+    uint32 max_block_size, const Array<precision_reduction> &block_precisions,
+    const Array<IndexType> &block_pointers, const Array<ValueType> &blocks,
+    const preconditioner::block_interleaved_storage_scheme<IndexType>
+        &storage_scheme,
+    Array<ValueType> &out_blocks)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_JACOBI_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void conj_transpose_jacobi(
+    std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks,
+    uint32 max_block_size, const Array<precision_reduction> &block_precisions,
+    const Array<IndexType> &block_pointers, const Array<ValueType> &blocks,
+    const preconditioner::block_interleaved_storage_scheme<IndexType>
+        &storage_scheme,
+    Array<ValueType> &out_blocks)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_JACOBI_CONJ_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_dense(
+    std::shared_ptr<const DpcppExecutor> exec, size_type num_blocks,
+    const Array<precision_reduction> &block_precisions,
+    const Array<IndexType> &block_pointers, const Array<ValueType> &blocks,
+    const preconditioner::block_interleaved_storage_scheme<IndexType>
+        &storage_scheme,
+    ValueType *result_values, size_type result_stride)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL);
+
+
+}  // namespace jacobi
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/reorder/rcm_kernels.dp.cpp b/dpcpp/reorder/rcm_kernels.dp.cpp
new file mode 100644
index 00000000000..ef05c1f07be
--- /dev/null
+++ b/dpcpp/reorder/rcm_kernels.dp.cpp
@@ -0,0 +1,81 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/reorder/rcm_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/std_extensions.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/permutation.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The reordering namespace.
+ *
+ * @ingroup reorder
+ */
+namespace rcm {
+
+
+template <typename IndexType>
+void get_degree_of_nodes(std::shared_ptr<const DpcppExecutor> exec,
+                         const IndexType num_vertices,
+                         const IndexType *const row_ptrs,
+                         IndexType *const degrees) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
+
+
+template <typename IndexType>
+void get_permutation(
+    std::shared_ptr<const DpcppExecutor> exec, const IndexType num_vertices,
+    const IndexType *const row_ptrs, const IndexType *const col_idxs,
+    const IndexType *const degrees, IndexType *const permutation,
+    IndexType *const inv_permutation,
+    const gko::reorder::starting_strategy strategy) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_PERMUTATION_KERNEL);
+
+
+}  // namespace rcm
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/solver/bicg_kernels.dp.cpp b/dpcpp/solver/bicg_kernels.dp.cpp
new file mode 100644
index 00000000000..11a5471002a
--- /dev/null
+++ b/dpcpp/solver/bicg_kernels.dp.cpp
@@ -0,0 +1,105 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/bicg_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The BICG solver namespace.
+ *
+ * @ingroup bicg
+ */
+namespace bicg {
+
+
+template <typename ValueType>
+void initialize(std::shared_ptr<const DpcppExecutor> exec,
+                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
+                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
+                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
+                matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *r2,
+                matrix::Dense<ValueType> *z2, matrix::Dense<ValueType> *p2,
+                matrix::Dense<ValueType> *q2,
+                Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_INITIALIZE_KERNEL);
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const DpcppExecutor> exec,
+            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
+            matrix::Dense<ValueType> *p2, const matrix::Dense<ValueType> *z2,
+            const matrix::Dense<ValueType> *rho,
+            const matrix::Dense<ValueType> *prev_rho,
+            const Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_1_KERNEL);
+
+
+template <typename ValueType>
+void step_2(std::shared_ptr<const DpcppExecutor> exec,
+            matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
+            matrix::Dense<ValueType> *r2, const matrix::Dense<ValueType> *p,
+            const matrix::Dense<ValueType> *q,
+            const matrix::Dense<ValueType> *q2,
+            const matrix::Dense<ValueType> *beta,
+            const matrix::Dense<ValueType> *rho,
+            const Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_2_KERNEL);
+
+
+}  // namespace bicg
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/solver/bicgstab_kernels.dp.cpp b/dpcpp/solver/bicgstab_kernels.dp.cpp
new file mode 100644
index 00000000000..af4d42289cd
--- /dev/null
+++ b/dpcpp/solver/bicgstab_kernels.dp.cpp
@@ -0,0 +1,137 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/bicgstab_kernels.hpp"
+
+
+#include <algorithm>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The BICGSTAB solver namespace.
+ *
+ * @ingroup bicgstab
+ */
+namespace bicgstab {
+
+
+template <typename ValueType>
+void initialize(std::shared_ptr<const DpcppExecutor> exec,
+                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
+                matrix::Dense<ValueType> *rr, matrix::Dense<ValueType> *y,
+                matrix::Dense<ValueType> *s, matrix::Dense<ValueType> *t,
+                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *v,
+                matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *prev_rho,
+                matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *alpha,
+                matrix::Dense<ValueType> *beta, matrix::Dense<ValueType> *gamma,
+                matrix::Dense<ValueType> *omega,
+                Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_INITIALIZE_KERNEL);
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const DpcppExecutor> exec,
+            const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *p,
+            const matrix::Dense<ValueType> *v,
+            const matrix::Dense<ValueType> *rho,
+            const matrix::Dense<ValueType> *prev_rho,
+            const matrix::Dense<ValueType> *alpha,
+            const matrix::Dense<ValueType> *omega,
+            const Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_1_KERNEL);
+
+
+template <typename ValueType>
+void step_2(std::shared_ptr<const DpcppExecutor> exec,
+            const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *s,
+            const matrix::Dense<ValueType> *v,
+            const matrix::Dense<ValueType> *rho,
+            matrix::Dense<ValueType> *alpha,
+            const matrix::Dense<ValueType> *beta,
+            const Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_2_KERNEL);
+
+
+template <typename ValueType>
+void step_3(
+    std::shared_ptr<const DpcppExecutor> exec, matrix::Dense<ValueType> *x,
+    matrix::Dense<ValueType> *r, const matrix::Dense<ValueType> *s,
+    const matrix::Dense<ValueType> *t, const matrix::Dense<ValueType> *y,
+    const matrix::Dense<ValueType> *z, const matrix::Dense<ValueType> *alpha,
+    const matrix::Dense<ValueType> *beta, const matrix::Dense<ValueType> *gamma,
+    matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_3_KERNEL);
+
+
+template <typename ValueType>
+void finalize(std::shared_ptr<const DpcppExecutor> exec,
+              matrix::Dense<ValueType> *x, const matrix::Dense<ValueType> *y,
+              const matrix::Dense<ValueType> *alpha,
+              Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_FINALIZE_KERNEL);
+
+
+}  // namespace bicgstab
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/solver/cg_kernels.dp.cpp b/dpcpp/solver/cg_kernels.dp.cpp
new file mode 100644
index 00000000000..9cb2e0f30da
--- /dev/null
+++ b/dpcpp/solver/cg_kernels.dp.cpp
@@ -0,0 +1,101 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/cg_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The CG solver namespace.
+ *
+ * @ingroup cg
+ */
+namespace cg {
+
+
+template <typename ValueType>
+void initialize(std::shared_ptr<const DpcppExecutor> exec,
+                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
+                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
+                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
+                matrix::Dense<ValueType> *rho,
+                Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const DpcppExecutor> exec,
+            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
+            const matrix::Dense<ValueType> *rho,
+            const matrix::Dense<ValueType> *prev_rho,
+            const Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
+
+
+template <typename ValueType>
+void step_2(std::shared_ptr<const DpcppExecutor> exec,
+            matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
+            const matrix::Dense<ValueType> *p,
+            const matrix::Dense<ValueType> *q,
+            const matrix::Dense<ValueType> *beta,
+            const matrix::Dense<ValueType> *rho,
+            const Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
+
+
+}  // namespace cg
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/solver/cgs_kernels.dp.cpp b/dpcpp/solver/cgs_kernels.dp.cpp
new file mode 100644
index 00000000000..04d81b682c5
--- /dev/null
+++ b/dpcpp/solver/cgs_kernels.dp.cpp
@@ -0,0 +1,119 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/cgs_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The CGS solver namespace.
+ *
+ * @ingroup cgs
+ */
+namespace cgs {
+
+
+template <typename ValueType>
+void initialize(std::shared_ptr<const DpcppExecutor> exec,
+                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
+                matrix::Dense<ValueType> *r_tld, matrix::Dense<ValueType> *p,
+                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *u,
+                matrix::Dense<ValueType> *u_hat,
+                matrix::Dense<ValueType> *v_hat, matrix::Dense<ValueType> *t,
+                matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *beta,
+                matrix::Dense<ValueType> *gamma,
+                matrix::Dense<ValueType> *prev_rho,
+                matrix::Dense<ValueType> *rho,
+                Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_INITIALIZE_KERNEL);
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const DpcppExecutor> exec,
+            const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *u,
+            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *q,
+            matrix::Dense<ValueType> *beta, const matrix::Dense<ValueType> *rho,
+            const matrix::Dense<ValueType> *rho_prev,
+            const Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_1_KERNEL);
+
+
+template <typename ValueType>
+void step_2(std::shared_ptr<const DpcppExecutor> exec,
+            const matrix::Dense<ValueType> *u,
+            const matrix::Dense<ValueType> *v_hat, matrix::Dense<ValueType> *q,
+            matrix::Dense<ValueType> *t, matrix::Dense<ValueType> *alpha,
+            const matrix::Dense<ValueType> *rho,
+            const matrix::Dense<ValueType> *gamma,
+            const Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_2_KERNEL);
+
+template <typename ValueType>
+void step_3(std::shared_ptr<const DefaultExecutor> exec,
+            const matrix::Dense<ValueType> *t,
+            const matrix::Dense<ValueType> *u_hat, matrix::Dense<ValueType> *r,
+            matrix::Dense<ValueType> *x, const matrix::Dense<ValueType> *alpha,
+            const Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_3_KERNEL);
+
+
+}  // namespace cgs
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/solver/fcg_kernels.dp.cpp b/dpcpp/solver/fcg_kernels.dp.cpp
new file mode 100644
index 00000000000..79422ed1960
--- /dev/null
+++ b/dpcpp/solver/fcg_kernels.dp.cpp
@@ -0,0 +1,101 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/fcg_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The FCG solver namespace.
+ *
+ * @ingroup fcg
+ */
+namespace fcg {
+
+
+template <typename ValueType>
+void initialize(std::shared_ptr<const DpcppExecutor> exec,
+                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
+                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
+                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *t,
+                matrix::Dense<ValueType> *prev_rho,
+                matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *rho_t,
+                Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_INITIALIZE_KERNEL);
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const DpcppExecutor> exec,
+            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
+            const matrix::Dense<ValueType> *rho_t,
+            const matrix::Dense<ValueType> *prev_rho,
+            const Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_1_KERNEL);
+
+
+template <typename ValueType>
+void step_2(std::shared_ptr<const DpcppExecutor> exec,
+            matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
+            matrix::Dense<ValueType> *t, const matrix::Dense<ValueType> *p,
+            const matrix::Dense<ValueType> *q,
+            const matrix::Dense<ValueType> *beta,
+            const matrix::Dense<ValueType> *rho,
+            const Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_2_KERNEL);
+
+
+}  // namespace fcg
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/solver/gmres_kernels.dp.cpp b/dpcpp/solver/gmres_kernels.dp.cpp
new file mode 100644
index 00000000000..c8924b0b67f
--- /dev/null
+++ b/dpcpp/solver/gmres_kernels.dp.cpp
@@ -0,0 +1,186 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/gmres_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/solver/gmres.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The GMRES solver namespace.
+ *
+ * @ingroup gmres
+ */
+namespace gmres {
+
+
+namespace {
+
+
+template <typename ValueType>
+void finish_arnoldi(size_type num_rows, matrix::Dense<ValueType> *krylov_bases,
+                    matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
+                    const stopping_status *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType>
+void calculate_sin_and_cos(matrix::Dense<ValueType> *givens_sin,
+                           matrix::Dense<ValueType> *givens_cos,
+                           matrix::Dense<ValueType> *hessenberg_iter,
+                           size_type iter, const size_type rhs)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType>
+void givens_rotation(matrix::Dense<ValueType> *givens_sin,
+                     matrix::Dense<ValueType> *givens_cos,
+                     matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
+                     const stopping_status *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType>
+void calculate_next_residual_norm(
+    matrix::Dense<ValueType> *givens_sin, matrix::Dense<ValueType> *givens_cos,
+    matrix::Dense<remove_complex<ValueType>> *residual_norm,
+    matrix::Dense<ValueType> *residual_norm_collection, size_type iter,
+    const stopping_status *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType>
+void solve_upper_triangular(
+    const matrix::Dense<ValueType> *residual_norm_collection,
+    const matrix::Dense<ValueType> *hessenberg, matrix::Dense<ValueType> *y,
+    const size_type *final_iter_nums)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+template <typename ValueType>
+void calculate_qy(const matrix::Dense<ValueType> *krylov_bases,
+                  const matrix::Dense<ValueType> *y,
+                  matrix::Dense<ValueType> *before_preconditioner,
+                  const size_type *final_iter_nums)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+}  // namespace
+
+
+template <typename ValueType>
+void initialize_1(std::shared_ptr<const DpcppExecutor> exec,
+                  const matrix::Dense<ValueType> *b,
+                  matrix::Dense<ValueType> *residual,
+                  matrix::Dense<ValueType> *givens_sin,
+                  matrix::Dense<ValueType> *givens_cos,
+                  Array<stopping_status> *stop_status, size_type krylov_dim)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_INITIALIZE_1_KERNEL);
+
+
+template <typename ValueType>
+void initialize_2(std::shared_ptr<const DpcppExecutor> exec,
+                  const matrix::Dense<ValueType> *residual,
+                  matrix::Dense<remove_complex<ValueType>> *residual_norm,
+                  matrix::Dense<ValueType> *residual_norm_collection,
+                  matrix::Dense<ValueType> *krylov_bases,
+                  Array<size_type> *final_iter_nums, size_type krylov_dim)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_INITIALIZE_2_KERNEL);
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const DpcppExecutor> exec, size_type num_rows,
+            matrix::Dense<ValueType> *givens_sin,
+            matrix::Dense<ValueType> *givens_cos,
+            matrix::Dense<remove_complex<ValueType>> *residual_norm,
+            matrix::Dense<ValueType> *residual_norm_collection,
+            matrix::Dense<ValueType> *krylov_bases,
+            matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
+            Array<size_type> *final_iter_nums,
+            const Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_STEP_1_KERNEL);
+
+
+template <typename ValueType>
+void step_2(std::shared_ptr<const DpcppExecutor> exec,
+            const matrix::Dense<ValueType> *residual_norm_collection,
+            const matrix::Dense<ValueType> *krylov_bases,
+            const matrix::Dense<ValueType> *hessenberg,
+            matrix::Dense<ValueType> *y,
+            matrix::Dense<ValueType> *before_preconditioner,
+            const Array<size_type> *final_iter_nums)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_STEP_2_KERNEL);
+
+
+}  // namespace gmres
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/solver/ir_kernels.dp.cpp b/dpcpp/solver/ir_kernels.dp.cpp
new file mode 100644
index 00000000000..bd3c6a98bc8
--- /dev/null
+++ b/dpcpp/solver/ir_kernels.dp.cpp
@@ -0,0 +1,60 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/ir_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The IR solver namespace.
+ *
+ * @ingroup ir
+ */
+namespace ir {
+
+
+void initialize(std::shared_ptr<const DpcppExecutor> exec,
+                Array<stopping_status> *stop_status)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+}  // namespace ir
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/solver/lower_trs_kernels.dp.cpp b/dpcpp/solver/lower_trs_kernels.dp.cpp
new file mode 100644
index 00000000000..6f0546ed6a4
--- /dev/null
+++ b/dpcpp/solver/lower_trs_kernels.dp.cpp
@@ -0,0 +1,112 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/lower_trs_kernels.hpp"
+
+
+#include <memory>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/solver/lower_trs.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The LOWER_TRS solver namespace.
+ *
+ * @ingroup lower_trs
+ */
+namespace lower_trs {
+
+
+void should_perform_transpose(std::shared_ptr<const DpcppExecutor> exec,
+                              bool &do_transpose)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+
+void init_struct(std::shared_ptr<const DpcppExecutor> exec,
+                 std::shared_ptr<solver::SolveStruct> &solve_struct)
+{
+    // This init kernel is here to allow initialization of the solve struct for
+    // a more sophisticated implementation as for other executors.
+}
+
+
+template <typename ValueType, typename IndexType>
+void generate(std::shared_ptr<const DpcppExecutor> exec,
+              const matrix::Csr<ValueType, IndexType> *matrix,
+              solver::SolveStruct *solve_struct, const gko::size_type num_rhs)
+{
+    // This generate kernel is here to allow for a more sophisticated
+    // implementation as for other executors. This kernel would perform the
+    // "analysis" phase for the triangular matrix.
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_LOWER_TRS_GENERATE_KERNEL);
+
+
+/**
+ * The parameters trans_x and trans_b are used only in the CUDA executor for
+ * versions <=9.1 due to a limitation in the cssrsm_solve algorithm
+ */
+template <typename ValueType, typename IndexType>
+void solve(std::shared_ptr<const DpcppExecutor> exec,
+           const matrix::Csr<ValueType, IndexType> *matrix,
+           const solver::SolveStruct *solve_struct,
+           matrix::Dense<ValueType> *trans_b, matrix::Dense<ValueType> *trans_x,
+           const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *x)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_LOWER_TRS_SOLVE_KERNEL);
+
+
+}  // namespace lower_trs
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/solver/upper_trs_kernels.dp.cpp b/dpcpp/solver/upper_trs_kernels.dp.cpp
new file mode 100644
index 00000000000..67efc9896f6
--- /dev/null
+++ b/dpcpp/solver/upper_trs_kernels.dp.cpp
@@ -0,0 +1,112 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/upper_trs_kernels.hpp"
+
+
+#include <memory>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/solver/upper_trs.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The UPPER_TRS solver namespace.
+ *
+ * @ingroup upper_trs
+ */
+namespace upper_trs {
+
+
+void should_perform_transpose(std::shared_ptr<const DpcppExecutor> exec,
+                              bool &do_transpose)
+{
+    do_transpose = false;
+}
+
+
+void init_struct(std::shared_ptr<const DpcppExecutor> exec,
+                 std::shared_ptr<solver::SolveStruct> &solve_struct)
+{
+    // This init kernel is here to allow initialization of the solve struct for
+    // a more sophisticated implementation as for other executors.
+}
+
+
+template <typename ValueType, typename IndexType>
+void generate(std::shared_ptr<const DpcppExecutor> exec,
+              const matrix::Csr<ValueType, IndexType> *matrix,
+              solver::SolveStruct *solve_struct, const gko::size_type num_rhs)
+{
+    // This generate kernel is here to allow for a more sophisticated
+    // implementation as for other executors. This kernel would perform the
+    // "analysis" phase for the triangular matrix.
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_UPPER_TRS_GENERATE_KERNEL);
+
+
+/**
+ * The parameters trans_x and trans_b are used only in the CUDA executor for
+ * versions <=9.1 due to a limitation in the cssrsm_solve algorithm
+ */
+template <typename ValueType, typename IndexType>
+void solve(std::shared_ptr<const DpcppExecutor> exec,
+           const matrix::Csr<ValueType, IndexType> *matrix,
+           const solver::SolveStruct *solve_struct,
+           matrix::Dense<ValueType> *trans_b, matrix::Dense<ValueType> *trans_x,
+           const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *x)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_UPPER_TRS_SOLVE_KERNEL);
+
+
+}  // namespace upper_trs
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/stop/criterion_kernels.dp.cpp b/dpcpp/stop/criterion_kernels.dp.cpp
new file mode 100644
index 00000000000..dce2c8ae84c
--- /dev/null
+++ b/dpcpp/stop/criterion_kernels.dp.cpp
@@ -0,0 +1,71 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/stop/criterion_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The Setting of all statuses namespace.
+ * @ref set_status
+ * @ingroup set_all_statuses
+ */
+namespace set_all_statuses {
+
+
+void set_all_statuses(std::shared_ptr<const DpcppExecutor> exec,
+                      uint8 stoppingId, bool setFinalized,
+                      Array<stopping_status> *stop_status)
+{
+    auto size = stop_status->get_num_elems();
+    stopping_status *__restrict__ stop_status_ptr = stop_status->get_data();
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{size}, [=](sycl::id<1> idx_id) {
+            const int idx = idx_id[0];
+            stop_status_ptr[idx].stop(stoppingId, setFinalized);
+        });
+    });
+}
+
+
+}  // namespace set_all_statuses
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/stop/residual_norm_kernels.dp.cpp b/dpcpp/stop/residual_norm_kernels.dp.cpp
new file mode 100644
index 00000000000..f0f205d438d
--- /dev/null
+++ b/dpcpp/stop/residual_norm_kernels.dp.cpp
@@ -0,0 +1,77 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/stop/residual_norm_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The Residual norm stopping criterion namespace.
+ * @ref resnorm
+ * @ingroup resnorm
+ */
+namespace residual_norm {
+
+
+constexpr int default_group_size = 512;
+
+
+template <typename ValueType>
+void residual_norm(std::shared_ptr<const DpcppExecutor> exec,
+                   const matrix::Dense<ValueType> *tau,
+                   const matrix::Dense<ValueType> *orig_tau,
+                   ValueType rel_residual_goal, uint8 stoppingId,
+                   bool setFinalized, Array<stopping_status> *stop_status,
+                   Array<bool> *device_storage, bool *all_converged,
+                   bool *one_changed)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
+    GKO_DECLARE_RESIDUAL_NORM_KERNEL);
+
+
+}  // namespace residual_norm
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/test/CMakeLists.txt b/dpcpp/test/CMakeLists.txt
new file mode 100644
index 00000000000..8e34766a1c5
--- /dev/null
+++ b/dpcpp/test/CMakeLists.txt
@@ -0,0 +1,5 @@
+include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
+
+add_subdirectory(base)
+add_subdirectory(components)
+add_subdirectory(stop)
diff --git a/dpcpp/test/base/CMakeLists.txt b/dpcpp/test/base/CMakeLists.txt
new file mode 100644
index 00000000000..adb2f2505b5
--- /dev/null
+++ b/dpcpp/test/base/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_dpcpp_test(executor)
diff --git a/dpcpp/test/base/executor.dp.cpp b/dpcpp/test/base/executor.dp.cpp
new file mode 100644
index 00000000000..090c9bf671f
--- /dev/null
+++ b/dpcpp/test/base/executor.dp.cpp
@@ -0,0 +1,303 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include <exception>
+#include <memory>
+#include <type_traits>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
+namespace {
+
+
+class DpcppExecutor : public ::testing::Test {
+protected:
+    DpcppExecutor()
+        : omp(gko::OmpExecutor::create()), dpcpp(nullptr), dpcpp2(nullptr)
+    {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices("cpu"), 0);
+        dpcpp = gko::DpcppExecutor::create(0, omp, "cpu");
+        if (gko::DpcppExecutor::get_num_devices("gpu") > 0) {
+            dpcpp2 = gko::DpcppExecutor::create(0, omp, "gpu");
+        }
+    }
+
+    void TearDown()
+    {
+        if (dpcpp != nullptr) {
+            // ensure that previous calls finished and didn't throw an error
+            ASSERT_NO_THROW(dpcpp->synchronize());
+        }
+    }
+
+    std::shared_ptr<gko::Executor> omp;
+    std::shared_ptr<gko::DpcppExecutor> dpcpp;
+    std::shared_ptr<gko::DpcppExecutor> dpcpp2;
+};
+
+
+TEST_F(DpcppExecutor, CanInstantiateTwoExecutorsOnOneDevice)
+{
+    auto dpcpp = gko::DpcppExecutor::create(0, omp);
+    auto dpcpp2 = gko::DpcppExecutor::create(0, omp);
+
+    // We want automatic deinitialization to not create any error
+}
+
+
+TEST_F(DpcppExecutor, KnowsNumberOfDevicesOfTypeAll)
+{
+    auto count = sycl::device::get_devices(sycl::info::device_type::all).size();
+
+    auto num_devices = gko::DpcppExecutor::get_num_devices("all");
+
+    ASSERT_EQ(count, num_devices);
+}
+
+
+TEST_F(DpcppExecutor, KnowsNumberOfDevicesOfTypeCPU)
+{
+    auto count = sycl::device::get_devices(sycl::info::device_type::cpu).size();
+
+    auto num_devices = gko::DpcppExecutor::get_num_devices("cpu");
+
+    ASSERT_EQ(count, num_devices);
+}
+
+
+TEST_F(DpcppExecutor, KnowsNumberOfDevicesOfTypeGPU)
+{
+    auto count = sycl::device::get_devices(sycl::info::device_type::gpu).size();
+
+    auto num_devices = gko::DpcppExecutor::get_num_devices("gpu");
+
+    ASSERT_EQ(count, num_devices);
+}
+
+
+TEST_F(DpcppExecutor, KnowsNumberOfDevicesOfTypeAccelerator)
+{
+    auto count =
+        sycl::device::get_devices(sycl::info::device_type::accelerator).size();
+
+    auto num_devices = gko::DpcppExecutor::get_num_devices("accelerator");
+
+    ASSERT_EQ(count, num_devices);
+}
+
+
+TEST_F(DpcppExecutor, AllocatesAndFreesMemoryOnCPU)
+{
+    int *ptr = nullptr;
+
+    ASSERT_NO_THROW(ptr = dpcpp->alloc<int>(2));
+    ASSERT_NO_THROW(dpcpp->free(ptr));
+}
+
+
+TEST_F(DpcppExecutor, AllocatesAndFreesMemoryOnGPU)
+{
+    if (!dpcpp2) {
+        GTEST_SKIP() << "No DPC++ compatible GPU.";
+    }
+    int *ptr = nullptr;
+
+    ASSERT_NO_THROW(ptr = dpcpp2->alloc<int>(2));
+    ASSERT_NO_THROW(dpcpp2->free(ptr));
+}
+
+
+TEST_F(DpcppExecutor, FailsWhenOverallocating)
+{
+    const gko::size_type num_elems = 1ll << 50;  // 4PB of integers
+    int *ptr = nullptr;
+
+    ASSERT_THROW(
+        {
+            ptr = dpcpp->alloc<int>(num_elems);
+            dpcpp->synchronize();
+        },
+        gko::AllocationError);
+
+    dpcpp->free(ptr);
+}
+
+
+void check_data(int *data, bool *result)
+{
+    *result = false;
+    if (data[0] == 3 && data[1] == 8) {
+        *result = true;
+    }
+}
+
+TEST_F(DpcppExecutor, CopiesDataToCPU)
+{
+    int orig[] = {3, 8};
+    auto *copy = dpcpp->alloc<int>(2);
+    gko::Array<bool> is_set(omp, 1);
+
+    dpcpp->copy_from(omp.get(), 2, orig, copy);
+
+    is_set.set_executor(dpcpp);
+    ASSERT_NO_THROW(dpcpp->synchronize());
+    ASSERT_NO_THROW(dpcpp->get_queue()->submit([&](sycl::handler &cgh) {
+        auto *is_set_ptr = is_set.get_data();
+        cgh.single_task([=]() { check_data(copy, is_set_ptr); });
+    }));
+    is_set.set_executor(omp);
+    ASSERT_EQ(*is_set.get_data(), true);
+    ASSERT_NO_THROW(dpcpp->synchronize());
+    dpcpp->free(copy);
+}
+
+
+TEST_F(DpcppExecutor, CopiesDataToGPU)
+{
+    if (!dpcpp2) {
+        GTEST_SKIP() << "No DPC++ compatible GPU.";
+    }
+    int orig[] = {3, 8};
+    auto *copy = dpcpp2->alloc<int>(2);
+    gko::Array<bool> is_set(omp, 1);
+
+    dpcpp2->copy_from(omp.get(), 2, orig, copy);
+
+    is_set.set_executor(dpcpp2);
+    ASSERT_NO_THROW(dpcpp2->get_queue()->submit([&](sycl::handler &cgh) {
+        auto *is_set_ptr = is_set.get_data();
+        cgh.single_task([=]() { check_data(copy, is_set_ptr); });
+    }));
+    is_set.set_executor(omp);
+    ASSERT_EQ(*is_set.get_data(), true);
+    ASSERT_NO_THROW(dpcpp2->synchronize());
+    dpcpp2->free(copy);
+}
+
+
+void init_data(int *data)
+{
+    data[0] = 3;
+    data[1] = 8;
+}
+
+TEST_F(DpcppExecutor, CopiesDataFromCPU)
+{
+    int copy[2];
+    auto orig = dpcpp->alloc<int>(2);
+    dpcpp->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.single_task([=]() { init_data(orig); });
+    });
+
+    omp->copy_from(dpcpp.get(), 2, orig, copy);
+
+    EXPECT_EQ(3, copy[0]);
+    ASSERT_EQ(8, copy[1]);
+    dpcpp->free(orig);
+}
+
+
+TEST_F(DpcppExecutor, CopiesDataFromGPU)
+{
+    if (!dpcpp2) {
+        GTEST_SKIP() << "No DPC++ compatible GPU.";
+    }
+    int copy[2];
+    auto orig = dpcpp2->alloc<int>(2);
+    dpcpp2->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.single_task([=]() { init_data(orig); });
+    });
+
+    omp->copy_from(dpcpp2.get(), 2, orig, copy);
+
+    EXPECT_EQ(3, copy[0]);
+    ASSERT_EQ(8, copy[1]);
+    dpcpp2->free(orig);
+}
+
+
+TEST_F(DpcppExecutor, CopiesDataFromDpcppToDpcpp)
+{
+    if (!dpcpp2) {
+        GTEST_SKIP() << "No DPC++ compatible GPU.";
+    }
+    int copy[2];
+    gko::Array<bool> is_set(omp, 1);
+    auto orig = dpcpp->alloc<int>(2);
+    dpcpp->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.single_task([=]() { init_data(orig); });
+    });
+
+    auto copy_dpcpp2 = dpcpp2->alloc<int>(2);
+    dpcpp2->copy_from(dpcpp.get(), 2, orig, copy_dpcpp2);
+    // Check that the data is really on GPU
+    is_set.set_executor(dpcpp2);
+    ASSERT_NO_THROW(dpcpp2->get_queue()->submit([&](sycl::handler &cgh) {
+        auto *is_set_ptr = is_set.get_data();
+        cgh.single_task([=]() { check_data(copy_dpcpp2, is_set_ptr); });
+    }));
+    is_set.set_executor(omp);
+    ASSERT_EQ(*is_set.get_data(), true);
+
+    // Put the results on OpenMP and run CPU side assertions
+    omp->copy_from(dpcpp2.get(), 2, copy_dpcpp2, copy);
+    EXPECT_EQ(3, copy[0]);
+    ASSERT_EQ(8, copy[1]);
+    dpcpp2->free(copy_dpcpp2);
+    dpcpp->free(orig);
+}
+
+
+TEST_F(DpcppExecutor, Synchronizes)
+{
+    // Todo design a proper unit test once we support streams
+    ASSERT_NO_THROW(dpcpp->synchronize());
+}
+
+
+}  // namespace
diff --git a/dpcpp/test/components/CMakeLists.txt b/dpcpp/test/components/CMakeLists.txt
new file mode 100644
index 00000000000..f6ca175d105
--- /dev/null
+++ b/dpcpp/test/components/CMakeLists.txt
@@ -0,0 +1,3 @@
+ginkgo_create_test(absolute_array)
+ginkgo_create_test(fill_array)
+ginkgo_create_test(precision_conversion)
diff --git a/dpcpp/test/components/absolute_array.cpp b/dpcpp/test/components/absolute_array.cpp
new file mode 100644
index 00000000000..7dfe989de4b
--- /dev/null
+++ b/dpcpp/test/components/absolute_array.cpp
@@ -0,0 +1,132 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/absolute_array.hpp"
+
+
+#include <memory>
+#include <random>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class AbsoluteArray : public ::testing::Test {
+protected:
+    using value_type = double;
+    using complex_type = std::complex<double>;
+    AbsoluteArray()
+        : ref(gko::ReferenceExecutor::create()),
+          exec(gko::DpcppExecutor::create(0, ref)),
+          total_size(6344),
+          vals(ref, total_size),
+          dvals(exec, total_size),
+          complex_vals(ref, total_size),
+          dcomplex_vals(exec, total_size)
+    {
+        std::fill_n(vals.get_data(), total_size, -1234.0);
+        dvals = vals;
+        std::fill_n(complex_vals.get_data(), total_size, complex_type{3, 4});
+        dcomplex_vals = complex_vals;
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<gko::DpcppExecutor> exec;
+    gko::size_type total_size;
+    gko::Array<value_type> vals;
+    gko::Array<value_type> dvals;
+    gko::Array<complex_type> complex_vals;
+    gko::Array<complex_type> dcomplex_vals;
+};
+
+
+TEST_F(AbsoluteArray, InplaceEqualsReference)
+{
+    gko::kernels::dpcpp::components::inplace_absolute_array(
+        exec, dvals.get_data(), total_size);
+    gko::kernels::reference::components::inplace_absolute_array(
+        ref, vals.get_data(), total_size);
+
+    GKO_ASSERT_ARRAY_EQ(vals, dvals);
+}
+
+
+TEST_F(AbsoluteArray, InplaceComplexEqualsReference)
+{
+    gko::kernels::dpcpp::components::inplace_absolute_array(
+        exec, dcomplex_vals.get_data(), total_size);
+    gko::kernels::reference::components::inplace_absolute_array(
+        ref, complex_vals.get_data(), total_size);
+
+    GKO_ASSERT_ARRAY_EQ(complex_vals, dcomplex_vals);
+}
+
+
+TEST_F(AbsoluteArray, OutplaceEqualsReference)
+{
+    gko::Array<value_type> abs_vals(ref, total_size);
+    gko::Array<value_type> dabs_vals(exec, total_size);
+
+    gko::kernels::dpcpp::components::outplace_absolute_array(
+        exec, dvals.get_const_data(), total_size, dabs_vals.get_data());
+    gko::kernels::reference::components::outplace_absolute_array(
+        ref, vals.get_const_data(), total_size, abs_vals.get_data());
+
+    GKO_ASSERT_ARRAY_EQ(abs_vals, dabs_vals);
+}
+
+
+TEST_F(AbsoluteArray, OutplaceComplexEqualsReference)
+{
+    gko::Array<value_type> abs_vals(ref, total_size);
+    gko::Array<value_type> dabs_vals(exec, total_size);
+
+    gko::kernels::dpcpp::components::outplace_absolute_array(
+        exec, dcomplex_vals.get_const_data(), total_size, dabs_vals.get_data());
+    gko::kernels::reference::components::outplace_absolute_array(
+        ref, complex_vals.get_const_data(), total_size, abs_vals.get_data());
+
+    GKO_ASSERT_ARRAY_EQ(abs_vals, dabs_vals);
+}
+
+
+}  // namespace
diff --git a/dpcpp/test/components/fill_array.cpp b/dpcpp/test/components/fill_array.cpp
new file mode 100644
index 00000000000..5b2be7d06f7
--- /dev/null
+++ b/dpcpp/test/components/fill_array.cpp
@@ -0,0 +1,86 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/fill_array.hpp"
+
+
+#include <memory>
+#include <random>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+template <typename T>
+class FillArray : public ::testing::Test {
+protected:
+    using value_type = T;
+    FillArray()
+        : ref(gko::ReferenceExecutor::create()),
+          exec(gko::DpcppExecutor::create(0, ref)),
+          total_size(63531),
+          vals(ref, total_size),
+          dvals(exec, total_size)
+    {
+        std::fill_n(vals.get_data(), total_size, T(1523));
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<gko::DpcppExecutor> exec;
+    gko::size_type total_size;
+    gko::Array<value_type> vals;
+    gko::Array<value_type> dvals;
+};
+
+TYPED_TEST_SUITE(FillArray, gko::test::ValueAndIndexTypes);
+
+
+TYPED_TEST(FillArray, EqualsReference)
+{
+    using T = typename TestFixture::value_type;
+    gko::kernels::dpcpp::components::fill_array(
+        this->exec, this->dvals.get_data(), this->total_size, T(1523));
+    GKO_ASSERT_ARRAY_EQ(this->vals, this->dvals);
+}
+
+
+}  // namespace
diff --git a/dpcpp/test/components/precision_conversion.cpp b/dpcpp/test/components/precision_conversion.cpp
new file mode 100644
index 00000000000..660042f1afb
--- /dev/null
+++ b/dpcpp/test/components/precision_conversion.cpp
@@ -0,0 +1,173 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <cstring>
+#include <limits>
+#include <memory>
+#include <random>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class PrecisionConversion : public ::testing::Test {
+protected:
+    PrecisionConversion()
+        : ref(gko::ReferenceExecutor::create()),
+          exec(gko::DpcppExecutor::create(0, ref)),
+          rand(293),
+          total_size(42793),
+          vals(ref, total_size),
+          cvals(ref, total_size),
+          vals2(ref, 1),
+          expected_float(ref, 1),
+          expected_double(ref, 1),
+          dvals(exec),
+          dcvals(exec),
+          dvals2(exec)
+    {
+        auto maxval = 1e10f;
+        std::uniform_real_distribution<float> dist(-maxval, maxval);
+        for (gko::size_type i = 0; i < total_size; ++i) {
+            vals.get_data()[i] = dist(rand);
+            cvals.get_data()[i] = {dist(rand), dist(rand)};
+        }
+        dvals = vals;
+        dcvals = cvals;
+        gko::uint64 rawdouble{0x4218888000889111ULL};
+        gko::uint32 rawfloat{0x50c44400UL};
+        gko::uint64 rawrounded{0x4218888000000000ULL};
+        std::memcpy(vals2.get_data(), &rawdouble, sizeof(double));
+        std::memcpy(expected_float.get_data(), &rawfloat, sizeof(float));
+        std::memcpy(expected_double.get_data(), &rawrounded, sizeof(double));
+        dvals2 = vals2;
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<gko::DpcppExecutor> exec;
+    std::default_random_engine rand;
+    gko::size_type total_size;
+    gko::Array<float> vals;
+    gko::Array<float> dvals;
+    gko::Array<double> vals2;
+    gko::Array<double> dvals2;
+    gko::Array<float> expected_float;
+    gko::Array<double> expected_double;
+    gko::Array<std::complex<float>> cvals;
+    gko::Array<std::complex<float>> dcvals;
+};
+
+
+TEST_F(PrecisionConversion, ConvertsReal)
+{
+    gko::Array<double> dtmp;
+    gko::Array<float> dout;
+
+    dtmp = dvals;
+    dout = dtmp;
+
+    GKO_ASSERT_ARRAY_EQ(dvals, dout);
+}
+
+
+TEST_F(PrecisionConversion, ConvertsRealViaRef)
+{
+    gko::Array<double> tmp{ref};
+    gko::Array<float> dout;
+
+    tmp = dvals;
+    dout = tmp;
+
+    GKO_ASSERT_ARRAY_EQ(dvals, dout);
+}
+
+
+TEST_F(PrecisionConversion, ConvertsComplex)
+{
+    gko::Array<std::complex<double>> dtmp;
+    gko::Array<std::complex<float>> dout;
+
+    dtmp = dcvals;
+    dout = dtmp;
+
+    GKO_ASSERT_ARRAY_EQ(dcvals, dout);
+}
+
+
+TEST_F(PrecisionConversion, ConversionRounds)
+{
+    gko::Array<float> dtmp;
+    gko::Array<double> dout;
+
+    dtmp = dvals2;
+    dout = dtmp;
+
+    GKO_ASSERT_ARRAY_EQ(dtmp, expected_float);
+    GKO_ASSERT_ARRAY_EQ(dout, expected_double);
+}
+
+
+TEST_F(PrecisionConversion, ConvertsRealFromRef)
+{
+    gko::Array<double> dtmp;
+    gko::Array<float> dout;
+
+    dtmp = vals;
+    dout = dtmp;
+
+    GKO_ASSERT_ARRAY_EQ(dvals, dout);
+}
+
+
+TEST_F(PrecisionConversion, ConvertsComplexFromRef)
+{
+    gko::Array<std::complex<double>> dtmp;
+    gko::Array<std::complex<float>> dout;
+
+    dtmp = cvals;
+    dout = dtmp;
+
+    GKO_ASSERT_ARRAY_EQ(dcvals, dout);
+}
+
+
+}  // namespace
diff --git a/dpcpp/test/stop/CMakeLists.txt b/dpcpp/test/stop/CMakeLists.txt
new file mode 100644
index 00000000000..dd96e64d469
--- /dev/null
+++ b/dpcpp/test/stop/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_test(criterion_kernels)
diff --git a/dpcpp/test/stop/criterion_kernels.cpp b/dpcpp/test/stop/criterion_kernels.cpp
new file mode 100644
index 00000000000..5bdfc7a9d3e
--- /dev/null
+++ b/dpcpp/test/stop/criterion_kernels.cpp
@@ -0,0 +1,107 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/stop/criterion.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/stop/iteration.hpp>
+
+
+namespace {
+
+
+constexpr gko::size_type test_iterations = 10;
+
+
+class Criterion : public ::testing::Test {
+protected:
+    Criterion()
+    {
+        ref_ = gko::ReferenceExecutor::create();
+        dpcpp_ = gko::DpcppExecutor::create(0, ref_);
+        // Actually use an iteration stopping criterion because Criterion is an
+        // abstract class
+        factory_ = gko::stop::Iteration::build()
+                       .with_max_iters(test_iterations)
+                       .on(dpcpp_);
+    }
+
+    std::unique_ptr<gko::stop::Iteration::Factory> factory_;
+    std::shared_ptr<gko::ReferenceExecutor> ref_;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp_;
+};
+
+
+TEST_F(Criterion, SetsOneStopStatus)
+{
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    auto criterion = factory_->generate(nullptr, nullptr, nullptr);
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+
+    stop_status.set_executor(dpcpp_);
+    criterion->update()
+        .num_iterations(test_iterations)
+        .check(RelativeStoppingId, true, &stop_status, &one_changed);
+    stop_status.set_executor(ref_);
+
+    ASSERT_EQ(stop_status.get_data()[0].has_stopped(), true);
+}
+
+
+TEST_F(Criterion, SetsMultipleStopStatuses)
+{
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    auto criterion = factory_->generate(nullptr, nullptr, nullptr);
+    gko::Array<gko::stopping_status> stop_status(ref_, 3);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.get_data()[2].reset();
+
+    stop_status.set_executor(dpcpp_);
+    criterion->update()
+        .num_iterations(test_iterations)
+        .check(RelativeStoppingId, true, &stop_status, &one_changed);
+    stop_status.set_executor(ref_);
+
+    ASSERT_EQ(stop_status.get_data()[0].has_stopped(), true);
+    ASSERT_EQ(stop_status.get_data()[1].has_stopped(), true);
+    ASSERT_EQ(stop_status.get_data()[2].has_stopped(), true);
+}
+
+
+}  // namespace
diff --git a/dpcpp/test_dpcpp.dp.cpp b/dpcpp/test_dpcpp.dp.cpp
new file mode 100644
index 00000000000..1e4d05fde18
--- /dev/null
+++ b/dpcpp/test_dpcpp.dp.cpp
@@ -0,0 +1,42 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <CL/sycl.hpp>
+
+namespace sycl = cl::sycl;
+
+int main()
+{
+    // Use the queue property `in_order` which is DPC++ only
+    sycl::queue myQueue{sycl::property::queue::in_order{}};
+    return 0;
+}
diff --git a/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp b/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
index 06ddbd8f24b..2e5cd58ba3a 100644
--- a/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
+++ b/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
@@ -76,6 +76,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/build-setup.sh b/examples/build-setup.sh
index 8a2305192cb..0fa7639a508 100644
--- a/examples/build-setup.sh
+++ b/examples/build-setup.sh
@@ -2,7 +2,7 @@
 
 # copy libraries
 LIBRARY_DIRS="core core/device_hooks reference omp cuda hip"
-LIBRARY_NAMES="ginkgo ginkgo_reference ginkgo_omp ginkgo_cuda ginkgo_hip"
+LIBRARY_NAMES="ginkgo ginkgo_reference ginkgo_omp ginkgo_cuda ginkgo_hip ginkgo_dpcpp"
 SUFFIXES=".so .dylib .dll d.so d.dylib d.dll"
 VERSION="1.3.0"
 for prefix in ${LIBRARY_DIRS}; do
@@ -20,9 +20,9 @@ done
 
 # figure out correct compiler flags
 if ls ${THIS_DIR} | grep -F "libginkgo." >/dev/null; then
-    LINK_FLAGS="-lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference -lginkgo_hip -Wl,-rpath,${THIS_DIR}"
+    LINK_FLAGS="-lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference -lginkgo_hip -lginkgo_dpcpp -Wl,-rpath,${THIS_DIR}"
 else
-    LINK_FLAGS="-lginkgod -lginkgo_ompd -lginkgo_cudad -lginkgo_referenced -lginkgo_hipd -Wl,-rpath,${THIS_DIR}"
+    LINK_FLAGS="-lginkgod -lginkgo_ompd -lginkgo_cudad -lginkgo_referenced -lginkgo_hipd -lginkgo_dpcppd -Wl,-rpath,${THIS_DIR}"
 fi
 if [ -z "${CXX}" ]; then
     CXX="c++"
diff --git a/examples/custom-logger/custom-logger.cpp b/examples/custom-logger/custom-logger.cpp
index afc48e0ea1f..7944acfeb6e 100644
--- a/examples/custom-logger/custom-logger.cpp
+++ b/examples/custom-logger/custom-logger.cpp
@@ -242,6 +242,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/custom-matrix-format/build.sh b/examples/custom-matrix-format/build.sh
index e56cb0cd25d..1821aa72bd5 100755
--- a/examples/custom-matrix-format/build.sh
+++ b/examples/custom-matrix-format/build.sh
@@ -14,9 +14,9 @@ CXX="nvcc"
 
 # figure out correct compiler flags
 if ls ${THIS_DIR} | grep -F "libginkgo." >/dev/null; then
-    LINK_FLAGS="-lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference -lginkgo_hip -Xlinker -rpath -Xlinker ${THIS_DIR}"
+    LINK_FLAGS="-lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference -lginkgo_dpcpp -lginkgo_hip -Xlinker -rpath -Xlinker ${THIS_DIR}"
 else
-    LINK_FLAGS="-lginkgod -lginkgo_ompd -lginkgo_cudad -lginkgo_referenced -lginkgo_hipd -Xlinker -rpath -Xlinker ${THIS_DIR}"
+    LINK_FLAGS="-lginkgod -lginkgo_ompd -lginkgo_cudad -lginkgo_referenced -lginkgo_dpcppd -lginkgo_hipd -Xlinker -rpath -Xlinker ${THIS_DIR}"
 fi
 
 
diff --git a/examples/custom-matrix-format/custom-matrix-format.cpp b/examples/custom-matrix-format/custom-matrix-format.cpp
index 679ccb379c2..96e9235773c 100644
--- a/examples/custom-matrix-format/custom-matrix-format.cpp
+++ b/examples/custom-matrix-format/custom-matrix-format.cpp
@@ -263,6 +263,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/custom-stopping-criterion/custom-stopping-criterion.cpp b/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
index 5c7d7861b9b..fe1086a1bfe 100644
--- a/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
+++ b/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
@@ -166,6 +166,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp b/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
index d6fbcc8ea06..7df73d3694b 100644
--- a/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
+++ b/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
@@ -76,6 +76,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/inverse-iteration/inverse-iteration.cpp b/examples/inverse-iteration/inverse-iteration.cpp
index 728b9363178..80575f02d9e 100644
--- a/examples/inverse-iteration/inverse-iteration.cpp
+++ b/examples/inverse-iteration/inverse-iteration.cpp
@@ -80,6 +80,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp b/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
index 627d11d88dd..f5f215a87a8 100644
--- a/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
+++ b/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
@@ -79,6 +79,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/iterative-refinement/iterative-refinement.cpp b/examples/iterative-refinement/iterative-refinement.cpp
index a07726ec82e..14bafe34ec5 100644
--- a/examples/iterative-refinement/iterative-refinement.cpp
+++ b/examples/iterative-refinement/iterative-refinement.cpp
@@ -76,6 +76,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/mixed-precision-ir/mixed-precision-ir.cpp b/examples/mixed-precision-ir/mixed-precision-ir.cpp
index a087baa3dfc..9cd21696253 100644
--- a/examples/mixed-precision-ir/mixed-precision-ir.cpp
+++ b/examples/mixed-precision-ir/mixed-precision-ir.cpp
@@ -84,6 +84,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp b/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
index e1fd8fe886d..fe194e99ae4 100644
--- a/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
+++ b/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
@@ -238,6 +238,11 @@ void solve_system(const std::string &executor_string,
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
@@ -301,8 +306,10 @@ int main(int argc, char *argv[])
     std::cout << gko::version_info::get() << std::endl;
 
     if (argc == 2 && std::string(argv[1]) == "--help") {
-        std::cerr << "Usage: " << argv[0] << " [executor] [DISCRETIZATION_POINTS] [alpha] [beta] [gamma]"
-                  << std::endl;
+        std::cerr
+            << "Usage: " << argv[0]
+            << " [executor] [DISCRETIZATION_POINTS] [alpha] [beta] [gamma]"
+            << std::endl;
         std::exit(-1);
     }
 
diff --git a/examples/papi-logging/papi-logging.cpp b/examples/papi-logging/papi-logging.cpp
index 2fd8cbbc8d4..e50fe72c210 100644
--- a/examples/papi-logging/papi-logging.cpp
+++ b/examples/papi-logging/papi-logging.cpp
@@ -140,8 +140,7 @@ int main(int argc, char *argv[])
     std::cout << gko::version_info::get() << std::endl;
 
     if (argc == 2 && (std::string(argv[1]) == "--help")) {
-        std::cerr << "Usage: " << argv[0] << " [executor]"
-                  << std::endl;
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
@@ -160,6 +159,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/performance-debugging/performance-debugging.cpp b/examples/performance-debugging/performance-debugging.cpp
index 4a9fae8bcbd..eb88bd28fee 100644
--- a/examples/performance-debugging/performance-debugging.cpp
+++ b/examples/performance-debugging/performance-debugging.cpp
@@ -370,8 +370,7 @@ int main(int argc, char *argv[])
 
     // Figure out where to run the code
     if (argc == 2 && (std::string(argv[1]) == "--help")) {
-        std::cerr << "Usage: " << argv[0] << " [executor]"
-                  << std::endl;
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
@@ -390,6 +389,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/poisson-solver/poisson-solver.cpp b/examples/poisson-solver/poisson-solver.cpp
index 813fa5e8bad..5079d9f5645 100644
--- a/examples/poisson-solver/poisson-solver.cpp
+++ b/examples/poisson-solver/poisson-solver.cpp
@@ -126,8 +126,8 @@ int main(int argc, char *argv[])
     std::cout << gko::version_info::get() << std::endl;
 
     if (argc == 2 && (std::string(argv[1]) == "--help")) {
-        std::cerr << "Usage: " << argv[0] << " [executor] [DISCRETIZATION_POINTS]"
-                  << std::endl;
+        std::cerr << "Usage: " << argv[0]
+                  << " [executor] [DISCRETIZATION_POINTS]" << std::endl;
         std::exit(-1);
     }
 
@@ -150,6 +150,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/preconditioned-solver/preconditioned-solver.cpp b/examples/preconditioned-solver/preconditioned-solver.cpp
index e799a846f3f..00ff326c5e1 100644
--- a/examples/preconditioned-solver/preconditioned-solver.cpp
+++ b/examples/preconditioned-solver/preconditioned-solver.cpp
@@ -58,8 +58,7 @@ int main(int argc, char *argv[])
 
     // Figure out where to run the code
     if (argc == 2 && (std::string(argv[1]) == "--help")) {
-        std::cerr << "Usage: " << argv[0] << " [executor]"
-                  << std::endl;
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
     }
 
@@ -78,6 +77,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/preconditioner-export/doc/results.dox b/examples/preconditioner-export/doc/results.dox
index 66e257949ed..6287655ea57 100644
--- a/examples/preconditioner-export/doc/results.dox
+++ b/examples/preconditioner-export/doc/results.dox
@@ -3,7 +3,7 @@ This is the expected output:
 
 @code{.cpp}
 
-Usage: ./preconditioner-export <reference|omp|cuda|hip> [<matrix-file>] [<jacobi|ilu|parilu|parilut|ilu-isai|parilu-isai|parilut-isai] [<preconditioner args>]
+Usage: ./preconditioner-export <reference|omp|cuda|hip|dpcpp> [<matrix-file>] [<jacobi|ilu|parilu|parilut|ilu-isai|parilu-isai|parilut-isai] [<preconditioner args>]
 Jacobi parameters: [<max-block-size>] [<accuracy>] [<storage-optimization:auto|0|1|2>]
 ParILU parameters: [<iteration-count>]
 ParILUT parameters: [<iteration-count>] [<fill-in-limit>]
diff --git a/examples/preconditioner-export/preconditioner-export.cpp b/examples/preconditioner-export/preconditioner-export.cpp
index ee69235e5c0..0cb2eb73384 100644
--- a/examples/preconditioner-export/preconditioner-export.cpp
+++ b/examples/preconditioner-export/preconditioner-export.cpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <functional>
 #include <iostream>
 #include <map>
+#include <memory>
 #include <string>
 
 
@@ -49,9 +50,14 @@ const std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
                    return gko::CudaExecutor::create(
                        0, gko::ReferenceExecutor::create());
                }},
-              {"hip", [] {
+              {"hip",
+               [] {
                    return gko::HipExecutor::create(
                        0, gko::ReferenceExecutor::create());
+               }},
+              {"dpcpp", [] {
+                   return gko::DpcppExecutor::create(
+                       0, gko::ReferenceExecutor::create());
                }}};
 
 
@@ -82,7 +88,7 @@ int main(int argc, char *argv[])
     // print usage message
     if (argc < 2 || executors.find(argv[1]) == executors.end()) {
         std::cerr << "Usage: executable"
-                  << " <reference|omp|cuda|hip> [<matrix-file>] "
+                  << " <reference|omp|cuda|hip|dpcpp> [<matrix-file>] "
                      "[<jacobi|ilu|parilu|parilut|ilu-isai|parilu-isai|parilut-"
                      "isai] [<preconditioner args>]\n";
         std::cerr << "Jacobi parameters: [<max-block-size>] [<accuracy>] "
diff --git a/examples/simple-solver-logging/simple-solver-logging.cpp b/examples/simple-solver-logging/simple-solver-logging.cpp
index 96d32fa4b2a..8fc26d4a65b 100644
--- a/examples/simple-solver-logging/simple-solver-logging.cpp
+++ b/examples/simple-solver-logging/simple-solver-logging.cpp
@@ -93,6 +93,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/simple-solver/simple-solver.cpp b/examples/simple-solver/simple-solver.cpp
index baf18c49d45..fec576052e8 100644
--- a/examples/simple-solver/simple-solver.cpp
+++ b/examples/simple-solver/simple-solver.cpp
@@ -96,6 +96,11 @@ int main(int argc, char *argv[])
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
diff --git a/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp b/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
index 3edb4e99e39..e413556d909 100644
--- a/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
+++ b/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
@@ -173,6 +173,11 @@ void solve_system(const std::string &executor_string,
                  return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
                                                  true);
              }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
             {"reference", [] { return gko::ReferenceExecutor::create(); }}};
 
     // executor where Ginkgo will perform the computation
@@ -236,8 +241,8 @@ int main(int argc, char *argv[])
     std::cout << gko::version_info::get() << std::endl;
 
     if (argc == 2 && std::string(argv[1]) == "--help") {
-        std::cerr << "Usage: " << argv[0] << " [executor] [DISCRETIZATION_POINTS]"
-                  << std::endl;
+        std::cerr << "Usage: " << argv[0]
+                  << " [executor] [DISCRETIZATION_POINTS]" << std::endl;
         std::exit(-1);
     }
 
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index f41fb69f46c..3e400986e2b 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -138,11 +138,18 @@ void HipExecutor::raw_copy_to(const CudaExecutor *dest, size_type num_bytes,
                                                num_bytes));
     }
 #else
-    GKO_NOT_SUPPORTED(this);
+    GKO_NOT_SUPPORTED(dest);
 #endif
 }
 
 
+void HipExecutor::raw_copy_to(const DpcppExecutor *dest, size_type num_bytes,
+                              const void *src_ptr, void *dest_ptr) const
+{
+    GKO_NOT_SUPPORTED(dest);
+}
+
+
 void HipExecutor::raw_copy_to(const HipExecutor *dest, size_type num_bytes,
                               const void *src_ptr, void *dest_ptr) const
 {
diff --git a/hip/test/base/hip_executor.hip.cpp b/hip/test/base/hip_executor.hip.cpp
index 635639fc21e..e0003e91c4b 100644
--- a/hip/test/base/hip_executor.hip.cpp
+++ b/hip/test/base/hip_executor.hip.cpp
@@ -75,6 +75,11 @@ class ExampleOperation : public gko::Operation {
         value = -3;
     }
 
+    void run(std::shared_ptr<const gko::DpcppExecutor>) const override
+    {
+        value = -4;
+    }
+
     void run(std::shared_ptr<const gko::HipExecutor>) const override
     {
         hipGetDevice(&value);
@@ -248,7 +253,7 @@ TEST_F(HipExecutor, CopiesDataFromHipToHip)
     omp->copy_from(hip2.get(), 2, copy_hip2, copy);
     EXPECT_EQ(3, copy[0]);
     ASSERT_EQ(8, copy[1]);
-    hip->free(copy_hip2);
+    hip2->free(copy_hip2);
     hip->free(orig);
 }
 
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 1df29abc59c..ec8628d64de 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <memory>
 #include <mutex>
 #include <sstream>
+#include <string>
 #include <tuple>
 #include <type_traits>
 
@@ -46,6 +47,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/synthesizer/containers.hpp>
 
 
+inline namespace cl {
+namespace sycl {
+
+class queue;
+
+}  // namespace sycl
+}  // namespace cl
+
+
 struct cublasContext;
 
 struct cusparseContext;
@@ -114,6 +124,9 @@ class ExecutorBase;
  *     void run(const gko::HipExecutor *exec) const override
  *     { os_ << "HIP(" << exec->get_device_id() << ")"; }
  *
+ *     void run(const gko::DpcppExecutor *exec) const override
+ *     { os_ << "DPC++(" << exec->get_device_id() << ")"; }
+ *
  *     // This is optional, if not overloaded, defaults to OmpExecutor overload
  *     void run(const gko::ReferenceExecutor *) const override
  *     { os_ << "Reference CPU"; }
@@ -142,6 +155,7 @@ class ExecutorBase;
  * std::cout << *omp << std::endl
  *           << *gko::CudaExecutor::create(0, omp) << std::endl
  *           << *gko::HipExecutor::create(0, omp) << std::endl
+ *           << *gko::DpcppExecutor::create(0, omp) << std::endl
  *           << *gko::ReferenceExecutor::create() << std::endl;
  * ```
  *
@@ -151,15 +165,16 @@ class ExecutorBase;
  * OMP
  * CUDA(0)
  * HIP(0)
+ * DPC++(0)
  * Reference CPU
  * ```
  *
  * One might feel that this code is too complicated for such a simple task.
  * Luckily, there is an overload of the Executor::run() method, which is
  * designed to facilitate writing simple operations like this one. The method
- * takes three closures as input: one which is run for OMP, one for
- * CUDA executors, and the last one for HIP executors. Using this method, there
- * is no need to implement an Operation subclass:
+ * takes three closures as input: one which is run for OMP, one for CUDA
+ * executors, one for HIP executors, and the last one for DPC++ executors. Using
+ * this method, there is no need to implement an Operation subclass:
  *
  * ```
  * std::ostream& operator<<(std::ostream &os, const gko::Executor &exec)
@@ -174,6 +189,10 @@ class ExecutorBase;
  *                    << static_cast<gko::HipExecutor&>(exec)
  *                         .get_device_id()
  *                    << ")"; });
+ *         [&]() { os << "DPC++("    // DPC++ closure
+ *                    << static_cast<gko::DpcppExecutor&>(exec)
+ *                         .get_device_id()
+ *                    << ")"; });
  *     return os;
  * }
  * ```
@@ -250,7 +269,7 @@ private:                                                                     \
  * kernel when the operation is executed.
  *
  * The kernels used to bind the operation are searched in `kernels::DEV_TYPE`
- * namespace, where `DEV_TYPE` is replaced by `omp`, `cuda`, `hip` and
+ * namespace, where `DEV_TYPE` is replaced by `omp`, `cuda`, `hip`, `dpcpp` and
  * `reference`.
  *
  * @param _name  operation name
@@ -278,6 +297,11 @@ private:                                                                     \
  *      // hip code
  * }
  * }
+ * namespace dpcpp {
+ * void my_kernel(int x) {
+ *      // dpcpp code
+ * }
+ * }
  * namespace reference {
  * void my_kernel(int x) {
  *     // reference code
@@ -292,6 +316,7 @@ private:                                                                     \
  *     auto omp = OmpExecutor::create();
  *     auto cuda = CudaExecutor::create(omp, 0);
  *     auto hip = HipExecutor::create(omp, 0);
+ *     auto dpcpp = DpcppExecutor::create(omp, 0);
  *     auto ref = ReferenceExecutor::create();
  *
  *     // create the operation
@@ -300,50 +325,52 @@ private:                                                                     \
  *     omp->run(op);  // run omp kernel
  *     cuda->run(op);  // run cuda kernel
  *     hip->run(op);  // run hip kernel
+ *     dpcpp->run(op);  // run DPC++ kernel
  *     ref->run(op);  // run reference kernel
  * }
  * ```
  *
  * @ingroup Executor
  */
-#define GKO_REGISTER_OPERATION(_name, _kernel)                               \
-    template <typename... Args>                                              \
-    class _name##_operation : public Operation {                             \
-        using counts =                                                       \
-            ::gko::syn::as_list<::gko::syn::range<0, sizeof...(Args)>>;      \
-                                                                             \
-    public:                                                                  \
-        explicit _name##_operation(Args &&... args)                          \
-            : data(std::forward<Args>(args)...)                              \
-        {}                                                                   \
-                                                                             \
-        const char *get_name() const noexcept override                       \
-        {                                                                    \
-            static auto name = [this] {                                      \
-                std::ostringstream oss;                                      \
-                oss << #_kernel << '#' << sizeof...(Args);                   \
-                return oss.str();                                            \
-            }();                                                             \
-            return name.c_str();                                             \
-        }                                                                    \
-                                                                             \
-        GKO_KERNEL_DETAIL_DEFINE_RUN_OVERLOAD(OmpExecutor, omp, _kernel);    \
-        GKO_KERNEL_DETAIL_DEFINE_RUN_OVERLOAD(CudaExecutor, cuda, _kernel);  \
-        GKO_KERNEL_DETAIL_DEFINE_RUN_OVERLOAD(HipExecutor, hip, _kernel);    \
-        GKO_KERNEL_DETAIL_DEFINE_RUN_OVERLOAD(ReferenceExecutor, reference,  \
-                                              _kernel);                      \
-                                                                             \
-    private:                                                                 \
-        mutable std::tuple<Args &&...> data;                                 \
-    };                                                                       \
-                                                                             \
-    template <typename... Args>                                              \
-    static _name##_operation<Args...> make_##_name(Args &&... args)          \
-    {                                                                        \
-        return _name##_operation<Args...>(std::forward<Args>(args)...);      \
-    }                                                                        \
-    static_assert(true,                                                      \
-                  "This assert is used to counter the false positive extra " \
+#define GKO_REGISTER_OPERATION(_name, _kernel)                                \
+    template <typename... Args>                                               \
+    class _name##_operation : public Operation {                              \
+        using counts =                                                        \
+            ::gko::syn::as_list<::gko::syn::range<0, sizeof...(Args)>>;       \
+                                                                              \
+    public:                                                                   \
+        explicit _name##_operation(Args &&... args)                           \
+            : data(std::forward<Args>(args)...)                               \
+        {}                                                                    \
+                                                                              \
+        const char *get_name() const noexcept override                        \
+        {                                                                     \
+            static auto name = [this] {                                       \
+                std::ostringstream oss;                                       \
+                oss << #_kernel << '#' << sizeof...(Args);                    \
+                return oss.str();                                             \
+            }();                                                              \
+            return name.c_str();                                              \
+        }                                                                     \
+                                                                              \
+        GKO_KERNEL_DETAIL_DEFINE_RUN_OVERLOAD(OmpExecutor, omp, _kernel);     \
+        GKO_KERNEL_DETAIL_DEFINE_RUN_OVERLOAD(CudaExecutor, cuda, _kernel);   \
+        GKO_KERNEL_DETAIL_DEFINE_RUN_OVERLOAD(HipExecutor, hip, _kernel);     \
+        GKO_KERNEL_DETAIL_DEFINE_RUN_OVERLOAD(DpcppExecutor, dpcpp, _kernel); \
+        GKO_KERNEL_DETAIL_DEFINE_RUN_OVERLOAD(ReferenceExecutor, reference,   \
+                                              _kernel);                       \
+                                                                              \
+    private:                                                                  \
+        mutable std::tuple<Args &&...> data;                                  \
+    };                                                                        \
+                                                                              \
+    template <typename... Args>                                               \
+    static _name##_operation<Args...> make_##_name(Args &&... args)           \
+    {                                                                         \
+        return _name##_operation<Args...>(std::forward<Args>(args)...);       \
+    }                                                                         \
+    static_assert(true,                                                       \
+                  "This assert is used to counter the false positive extra "  \
                   "semi-colon warnings")
 
 
@@ -359,6 +386,8 @@ private:                                                                     \
  *      operations executed on the NVIDIA GPU accelerator;
  * +    HipExecutor specifies that the data should be stored and the
  *      operations executed on either an NVIDIA or AMD GPU accelerator;
+ * +    DpcppExecutor specifies that the data should be stored and the
+ *      operations executed on an hardware supporting DPC++;
  * +    ReferenceExecutor executes a non-optimized reference implementation,
  *      which can be used to debug the library.
  *
@@ -436,6 +465,12 @@ class Executor : public log::EnableLogging<Executor> {
     template <typename T>
     friend class detail::ExecutorBase;
 
+#define GKO_DECLARE_EXECUTOR_FRIEND(_type, ...) friend class _type
+
+    GKO_ENABLE_FOR_ALL_EXECUTORS(GKO_DECLARE_EXECUTOR_FRIEND);
+
+#undef GKO_DECLARE_EXECUTOR_FRIEND
+
 public:
     virtual ~Executor() = default;
 
@@ -464,12 +499,13 @@ class Executor : public log::EnableLogging<Executor> {
      * @param op_cuda  functor to run in case of a CudaExecutor
      * @param op_hip  functor to run in case of a HipExecutor
      */
-    template <typename ClosureOmp, typename ClosureCuda, typename ClosureHip>
+    template <typename ClosureOmp, typename ClosureCuda, typename ClosureHip,
+              typename ClosureDpcpp>
     void run(const ClosureOmp &op_omp, const ClosureCuda &op_cuda,
-             const ClosureHip &op_hip) const
+             const ClosureHip &op_hip, const ClosureDpcpp &op_dpcpp) const
     {
-        LambdaOperation<ClosureOmp, ClosureCuda, ClosureHip> op(op_omp, op_cuda,
-                                                                op_hip);
+        LambdaOperation<ClosureOmp, ClosureCuda, ClosureHip, ClosureDpcpp> op(
+            op_omp, op_cuda, op_hip, op_dpcpp);
         this->run(op);
     }
 
@@ -530,7 +566,20 @@ class Executor : public log::EnableLogging<Executor> {
         this->template log<log::Logger::copy_started>(
             src_exec, this, reinterpret_cast<uintptr>(src_ptr),
             reinterpret_cast<uintptr>(dest_ptr), num_elems * sizeof(T));
-        this->raw_copy_from(src_exec, num_elems * sizeof(T), src_ptr, dest_ptr);
+        try {
+            this->raw_copy_from(src_exec, num_elems * sizeof(T), src_ptr,
+                                dest_ptr);
+        } catch (NotSupported &err) {
+            // Unoptimized copy. Try to go through the masters.
+            auto src_master = src_exec->get_master().get();
+            if (num_elems > 0 && src_master != src_exec) {
+                auto *master_ptr = src_exec->get_master()->alloc<T>(num_elems);
+                src_master->copy_from<T>(src_exec, num_elems, src_ptr,
+                                         master_ptr);
+                this->copy_from<T>(src_master, num_elems, master_ptr, dest_ptr);
+                src_master->free(master_ptr);
+            }
+        }
         this->template log<log::Logger::copy_completed>(
             src_exec, this, reinterpret_cast<uintptr>(src_ptr),
             reinterpret_cast<uintptr>(dest_ptr), num_elems * sizeof(T));
@@ -650,8 +699,10 @@ class Executor : public log::EnableLogging<Executor> {
      * @tparam ClosureOmp  the type of the first functor
      * @tparam ClosureCuda  the type of the second functor
      * @tparam ClosureHip  the type of the third functor
+     * @tparam ClosureDpcpp  the type of the fourth functor
      */
-    template <typename ClosureOmp, typename ClosureCuda, typename ClosureHip>
+    template <typename ClosureOmp, typename ClosureCuda, typename ClosureHip,
+              typename ClosureDpcpp>
     class LambdaOperation : public Operation {
     public:
         /**
@@ -661,10 +712,15 @@ class Executor : public log::EnableLogging<Executor> {
          *                and ReferenceExecutor
          * @param op_cuda  a functor object which will be called by CudaExecutor
          * @param op_hip  a functor object which will be called by HipExecutor
+         * @param op_dpcpp  a functor object which will be called by
+         * DpcppExecutor
          */
         LambdaOperation(const ClosureOmp &op_omp, const ClosureCuda &op_cuda,
-                        const ClosureHip &op_hip)
-            : op_omp_(op_omp), op_cuda_(op_cuda), op_hip_(op_hip)
+                        const ClosureHip &op_hip, const ClosureDpcpp &op_dpcpp)
+            : op_omp_(op_omp),
+              op_cuda_(op_cuda),
+              op_hip_(op_hip),
+              op_dpcpp_(op_dpcpp)
         {}
 
         void run(std::shared_ptr<const OmpExecutor>) const override
@@ -682,10 +738,16 @@ class Executor : public log::EnableLogging<Executor> {
             op_hip_();
         }
 
+        void run(std::shared_ptr<const DpcppExecutor>) const override
+        {
+            op_dpcpp_();
+        }
+
     private:
         ClosureOmp op_omp_;
         ClosureCuda op_cuda_;
         ClosureHip op_hip_;
+        ClosureDpcpp op_dpcpp_;
     };
 };
 
@@ -755,6 +817,12 @@ namespace detail {
 
 template <typename ConcreteExecutor>
 class ExecutorBase : public Executor {
+#define GKO_DECLARE_EXECUTOR_FRIEND(_type, ...) friend class _type
+
+    GKO_ENABLE_FOR_ALL_EXECUTORS(GKO_DECLARE_EXECUTOR_FRIEND);
+
+#undef GKO_DECLARE_EXECUTOR_FRIEND
+
 public:
     void run(const Operation &op) const override
     {
@@ -1238,6 +1306,120 @@ using DefaultExecutor = HipExecutor;
 }  // namespace kernels
 
 
+/**
+ * This is the Executor subclass which represents a DPC++ enhanced device.
+ *
+ * @ingroup exec_dpcpp
+ * @ingroup Executor
+ */
+class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
+                      public std::enable_shared_from_this<DpcppExecutor> {
+    friend class detail::ExecutorBase<DpcppExecutor>;
+
+public:
+    /**
+     * Creates a new DpcppExecutor.
+     *
+     * @param device_id  the DPCPP device id of this device
+     * @param master  an executor on the host that is used to invoke the device
+     *                kernels
+     * @param device_type  a string representing the type of device to consider
+     *                     (accelerator, cpu, gpu or all).
+     */
+    static std::shared_ptr<DpcppExecutor> create(
+        int device_id, std::shared_ptr<Executor> master,
+        std::string device_type = "all");
+
+    std::shared_ptr<Executor> get_master() noexcept override;
+
+    std::shared_ptr<const Executor> get_master() const noexcept override;
+
+    void synchronize() const override;
+
+    void run(const Operation &op) const override;
+
+    /**
+     * Get the DPCPP device id of the device associated to this executor.
+     */
+    int get_device_id() const noexcept { return device_id_; }
+
+    ::cl::sycl::queue *get_queue() const { return queue_.get(); }
+
+    /**
+     * Get the number of devices present on the system.
+     */
+    static int get_num_devices(std::string device_type);
+
+    /**
+     * Get the available subgroup sizes for this device.
+     */
+    const std::vector<size_type> &get_subgroup_sizes() const noexcept
+    {
+        return subgroup_sizes_;
+    }
+
+    /**
+     * Get the number of Computing Units of this executor.
+     */
+    size_type get_num_computing_units() const noexcept
+    {
+        return num_computing_units_;
+    }
+
+    /**
+     * Get the maximum work item sizes.
+     */
+    const std::vector<size_type> &get_max_workitem_sizes() const noexcept
+    {
+        return max_workitem_sizes_;
+    }
+
+    /**
+     * Get the maximum workgroup size.
+     */
+    size_type get_max_workgroup_size() const noexcept
+    {
+        return max_workgroup_size_;
+    }
+
+protected:
+    void set_device_property();
+
+    DpcppExecutor(int device_id, std::shared_ptr<Executor> master,
+                  std::string device_type = "all")
+        : device_id_(device_id), master_(master), device_type_(device_type)
+    {
+        this->set_device_property();
+    }
+
+    void *raw_alloc(size_type size) const override;
+
+    void raw_free(void *ptr) const noexcept override;
+
+    GKO_ENABLE_FOR_ALL_EXECUTORS(GKO_OVERRIDE_RAW_COPY_TO);
+
+private:
+    int device_id_;
+    std::shared_ptr<Executor> master_;
+    std::string device_type_;
+    int num_computing_units_{};
+    std::vector<size_type> subgroup_sizes_{};
+    std::vector<size_type> max_workitem_sizes_{};
+    size_type max_workgroup_size_{};
+
+    template <typename T>
+    using queue_manager = std::unique_ptr<T, std::function<void(T *)>>;
+    queue_manager<::cl::sycl::queue> queue_;
+};
+
+
+namespace kernels {
+namespace dpcpp {
+using DefaultExecutor = DpcppExecutor;
+}  // namespace dpcpp
+}  // namespace kernels
+
+
 #undef GKO_OVERRIDE_RAW_COPY_TO
 
 
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 0bb6ac9b5a9..9217e00f800 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -41,6 +41,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <type_traits>
 
 
+#ifdef CL_SYCL_LANGUAGE_VERSION
+#include <CL/sycl.hpp>
+#endif
+
+
 #include <ginkgo/config.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/base/utils.hpp>
@@ -908,7 +913,11 @@ GKO_INLINE GKO_ATTRIBUTES constexpr xstd::enable_if_t<is_complex_s<T>::value,
                                                       remove_complex<T>>
 abs(const T &x)
 {
+#ifdef CL_SYCL_LANGUAGE_VERSION
+    return cl::sycl::sqrt(real(x) * real(x) + imag(x) * imag(x));
+#else
     return sqrt(squared_norm(x));
+#endif
 }
 
 
@@ -966,7 +975,11 @@ GKO_INLINE GKO_ATTRIBUTES std::enable_if_t<!is_complex_s<T>::value, bool>
 is_finite(const T &value)
 {
     constexpr T infinity{detail::infinity_impl<T>::value};
+#ifdef CL_SYCL_LANGUAGE_VERSION
+    return ::gko::abs(value) < infinity;
+#else
     return abs(value) < infinity;
+#endif
 }
 
 
@@ -989,6 +1002,25 @@ is_finite(const T &value)
 }
 
 
+namespace kernels {
+namespace dpcpp {
+
+
+// For now this seems to be useless. Somehow, DPC++ doesn't use this
+// declaration and anyway always replace calls to `abs` by `std::abs`. To
+// reference this declaration, use `dpcpp::abs`.
+using ::gko::abs;
+
+
+#ifdef CL_SYCL_LANGUAGE_VERSION
+using cl::sycl::sqrt;
+#endif
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+
+
 }  // namespace gko
 
 
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 10ef4d5d4cf..d63623fea40 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -392,6 +392,7 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
 #define GKO_ENABLE_FOR_ALL_EXECUTORS(_enable_macro) \
     _enable_macro(OmpExecutor, omp);                \
     _enable_macro(HipExecutor, hip);                \
+    _enable_macro(DpcppExecutor, dpcpp);            \
     _enable_macro(CudaExecutor, cuda)
 
 
diff --git a/include/ginkgo/core/base/version.hpp b/include/ginkgo/core/base/version.hpp
index 52731ab56e3..33ed03a8872 100644
--- a/include/ginkgo/core/base/version.hpp
+++ b/include/ginkgo/core/base/version.hpp
@@ -212,6 +212,13 @@ class version_info {
      */
     version hip_version;
 
+    /**
+     * Contains version information of the DPC++ module.
+     *
+     * This is the version of the static/shared library called "ginkgo_dpcpp".
+     */
+    version dpcpp_version;
+
 private:
     static constexpr version get_header_version() noexcept
     {
@@ -229,13 +236,16 @@ class version_info {
 
     static version get_hip_version() noexcept;
 
+    static version get_dpcpp_version() noexcept;
+
     version_info()
         : header_version{get_header_version()},
           core_version{get_core_version()},
           reference_version{get_reference_version()},
           omp_version{get_omp_version()},
           cuda_version{get_cuda_version()},
-          hip_version{get_hip_version()}
+          hip_version{get_hip_version()},
+          dpcpp_version{get_dpcpp_version()}
     {}
 };
 
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index 005e55ec24d..b0468af7cd2 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -50,6 +50,8 @@ target_compile_options(ginkgo_omp PRIVATE "${GINKGO_COMPILER_FLAGS}")
 target_link_libraries(ginkgo_omp PUBLIC ginkgo_cuda)
 # Need to link against ginkgo_hip for the `raw_copy_to(HipExecutor ...)` method
 target_link_libraries(ginkgo_omp PUBLIC ginkgo_hip)
+# Need to link against ginkgo_dpcpp for the `raw_copy_to(DpcppExecutor ...)` method
+target_link_libraries(ginkgo_omp PUBLIC ginkgo_dpcpp)
 
 ginkgo_default_includes(ginkgo_omp)
 ginkgo_install_library(ginkgo_omp omp)

From 154c06ba2210edfb9ef6ccc5b098e55215676e7a Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Thu, 15 Oct 2020 10:47:35 +0200
Subject: [PATCH 0088/1008] Update the JOSS paper.

---
 doc/joss/figures/ginkgo-hierarchy.pdf | Bin 62348 -> 62020 bytes
 doc/joss/figures/ginkgo-hierarchy.tex |   7 ++++++-
 doc/joss/paper.md                     |  14 +++++++-------
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/doc/joss/figures/ginkgo-hierarchy.pdf b/doc/joss/figures/ginkgo-hierarchy.pdf
index 945bacfe263e7f24984ad54d6ff763b3b5f453e7..d171dfe03a58aa681eeac8cac9c7f96b2e5f5196 100644
GIT binary patch
delta 12968
zcmajFRZv~Q7A+baf<u6fh2ZY)!GpWIyW0jfEZp6L1}DMYJ-B;tm*B1eF6X>+U%kKg
zx4NpkderPOXU{c8M<2la-i4|D1`or@!jp8_O%0s->bg=6I`8uX{azHNFsH4CM1-*^
zcEf2-TCxexyY4Js$F>mHgL)X?$s=BGGndv<zN_o|e#l7;-ybmk?Zjfp`HPLdYot~4
zSKo6G-@hHaqmVAne}=?2J=($`V)W_a9xC5vpNH+Y&zog{Ky<EO9(-n!@V~$Q7wLgr
zw?2T}XOmbNQ!Fzs3X+(xu8PKK;+##*xBZ=GlGnkFw~vWv=fdM}?QM*2|77>w3vP|_
zHO`!{$A6@7Jzn@%hX{zuNN<*4atERF;s?MS!XMj*k`xfDTum$*fz&>pIdNNjtwwbi
z;$2Cs`xYLgJhB|bfc_ibC8L&V3$eY-jT8j9P6YK5VBvqg8@AL<p{ORq-IsP!-c}1J
zbf6WV8>zJ*?Gvv5HNhrQ^!9YozgXao+_xzvNP{d4Du6$aH~HZ2RU&+K&=w{p-&Mt%
zn}~XF#ff*D^VTWtcd=A}i~oyqAZR$~ek3^PE=UDm>(%W9DFfv&@^&jvs8=yD>8cu7
zM_Cea4>5vI6WqT#cI<k{+z=(4wHcy2?rRbz8nN6fbKD?)eHm4pj3eoKigSG1i`sz5
z;y02g*rd&Ao8YGZDqLy&@h?6Q4kyqP|8<5hv^uZGAZx4ET#{lk&>r*qay3?#dY_)B
z6`})nZSq$<mW^c8eJUmX=JdR687L;epB+WvI}CE~AAL%$9qQjd1EK_)1Dup`v<eAa
z9f(j%oqlcQAJ6zeHSpc?HmK$~V3se+1Z0XxSQShqi;bmm&1CYDB*=HAc;B2&x6R0i
z3GvW~x9l2N6|bXd@n-7$lttphe^a4n%$={exx5+NQC(43Q9SnWW=DQ`3MJSCx)61~
zzS&h9(`Lr3&8aYtD|>vS4YYM{J=H^U<kRden;_fk8GwbYm?}pgshhTcBlz?wNpGbM
z9<GR@+*;d6=|W^kM{|w7!2NBc`NT-#^PGpcHFPFy*^{P5!I5#u#9W-~8pE;YQh%-5
z!LwGp^U$>8shI>9eMk9%1B-?az~Sm4ruNZK&iLDh&bjHMpE2_D=0JY^>O4UyR>+Mj
zW+bw${^>vMM;p7}B)PGkaU|<&!W+VVT3q{gnnweAg?PPPa~UNLm;;Qsul&gduClhB
z9D{^qHyEZe&L_$o>CSfJXp-u*p{4jjG1YLL1Dp8pW2m%;HE#jAwf8mfK;DeYg{m!f
z^QiL_Yicx{5EmN1g?ztPyedLE{I}_Ml5&y~)p5n=#$m1yTblfE@;50RDVx%6ZDU35
zmCjTRN;=9H_*J05IBBm#qcG#1S<k*H!w`vPAQF_oY&Szmk}xil6#BRkU(WVLM}>!8
zuIe<&@rMwCaO_Gq%r3GIU``TRI$hV>Jyg`7;Pv_o!ohVAwsKJNEZj8S{8;PIFJXfF
zkjCcGISnNg%GA$2s~;prn~pMI(Q4aWIWeFme<D{K#Oh7NI`dC4(4>`qbCbaX!(u{f
zlclv@GVZ&QGZb+t+RkoVi0Mt`LMZmY`uZU0irQX&Pw}#86TNIXFs0qeYCEv^TeDYS
zmboaR#jzykKts>kpzxT>bdG;7oLwv{)Ii{c@iE*^BcsOoi-!i)CCWp?I0yxHg4T{u
z?>Nh7F5i$qFKyC*wb?<qLe~ZJ36HlqH9LLcqTqWdSiMRLNj_~1Nxdl$Z36+Da1L!+
z%5>R*w>NHt*Kx%U;LTtG9q5-o?ODzIkV9)cO2@bNwotHYwMd9NQEwUh$VkSO%}(!b
zgI$WY>*Rl2rh7ihlu;zGA%%cJu{Z8pKV(tl%%6@Ro>?P@?w(@e{ZhRWFoDLeRnB#O
z8`9^1f$8H7gY9p&2){Nfb<JIc@1ohI#C<{5gz*eP299q5t0YFbg#iQR0Q2g#&23y1
zeEAD<YUF5HfvF~%!^U!d_uxxS6@oTYCnkax8di1+RFYU^DHMLEI4Tfl1(EbBDwCEu
zdOibQb&<qa+HbNGg`=;BNXqE0_S}ro<1AeK$Hfg*D)GM~l}wA_sBjc;bdquQl#1MJ
z+-<5;e$^=fYn)7nr-BN_TgX-pXIB!HyyOXrT`dGoj8ix?acjaT%vs*IHubFVPwm5h
zL{>(ZZTv-I;&>*M{lZBr>KCrw6T_Xe^`%;SkB+Twi23ICpW-o&r)=*jRNo6Yub9!=
zi99cVh4{3(9lJk#>2EO3ln_ynWB}S~<?o*2ZW%!!_QpsO>QzgWB`&bia(T%ZWI;Zq
zpqAxo&J%dZx7iDmva14*|B`N>5LweEgEXKR{!GflUCDucCQ6hwygWKXSN)A^Uyg^~
z<=16?9Zk$}v2K@YW{M8pmbd*vFIecea}4IeG#<0vT5`0w&<j`8C?D^Vvk~-$MNzk`
zahDw+r@+jigX}P>(6F}?(r1|-fbMr=KJ_P2hfpYOazxWKQ&6zeG1Kl@f7XmTBBWcj
z2r~ZMVahJc7gj^fF>S1v{)1l56gg8Ub7g7MoIUfc=;FLDpDKO}%~$mistne`f@QT=
zLNKdu*V=y9B4XQVk~;m3_RtQoBLFFLMqvk7MYTVy4_^I|RA*X3D;(_nW5MPmoB}0v
zk;krBwEGH*j`VXdPw<-bG*Soye^(bxAvwx!)2DwFJ^7opxTy`Cakl-;c-lWa0G@Bm
zFM$nJcaKY_bwq<=@T8f}RpU!;TQr}EoA3`@oj-Jq84QqI#lyG>@<fMyhf}Z`l`2RA
z9&-(~ulIKT4u%bu!~xA+&ujOFYq0lwz~}QEyr8ER4CT-Z!8(HSkQ@Fza=3Zqy4D}e
z?*ERuyWYHW-Wu8o+_omH(=sf?oYou=Q!8NMn+L*vv9`dzIA63_^L-A@TAwJhhE@o=
zL$jZxe2k7VoERDq)0zh(RuG`)Wt;E;(DUOZ-wYc_X6jUlvm<pmND~iT@H{yebV0Uj
z8(!SO6=zJ{@o1JPzMcqT!l<T&SJQ`Qezz|r2PG?Q2VE~K8-6K$X&19QvvIaL3T9n5
zZWO@e^qRQd5201F^t`etyu@NTp17tfF!yqg6QD-Z(gV&takKicSgxmmMUn>W8f!$)
z!a@(e_?&#r6An{A#Dk`Q=5VD;>jMW{W~ELX)8Tv`98ZNwNxH@mWD@=s@m`WYeF0;+
zUC@#fO5lE!T6n-t>r=wC9!oZw*;MPR`=u6H8gCvm!wRKQk9e!vu|99)=vX}=f&}96
z%A&Az%3jhxeEx*uU)}2Dh@uI!KX*=cTXF1)JzCxPzWswhTKnNvt|uIszV0%|$&T?=
zf8=+PS^g@Vnk2(G%qvoxrEJarZ<@kXFDRoY;N#D|Ru3CJ{vpPt2KJz7n_W3tj9+YV
zVVS{J>>5cOKR6d3?<0QJeuBdNZqrzYRRV9B(!!$5pG{zp*!iugM_L~hXvT)gan><v
zB-QDfw)JvK=n3JhN-^F=3L1MXn6~xxd2@oK^kHdx5~X4Q=%48`Ncqff122-5Fmt9x
z9Hx6%=39#KH<!yVCm0ZOVRA^Ej71($R+WZXH)4AGQGy6@MY$wxC)f?@4EB4I_j+M9
zC~U11e{s~~MY5kt5W61%UeZ3ej!_4#1TKc50tY>b_%6}9f*`pfXL{HEpd!{92#0s|
z28r-m*E7WDscEnT!OdW2#B^l#quE_2<o4`(txU>NG%}~oRNiy*q`L?me}67&JsaDM
z(72wB?H?#cxl=?$+0?~hCH!*Al&;EdMG|FVC1uit%fv|1v2F&abf5_4iV40bP6*cv
z->l_;!d*Jj=pD%-$WlMp7H4#~X`Pk%)`@y{W;3$({VKYToeQ1MA=_#j$=2z}77d-b
zqUinPJj|)nEnhJ`abx=*V`9-EsmKj0VkHlBan!#L!2)?lp9KuAf*QdcW%g9bN}SlD
z^z<bxX4zsefjK~TZvS$WI?|yv#6~xQc*(LOed14o4_H=qWWZ`o22`aRF&2WzBx@RP
zW~+llJ6n&nkVP0#TA(1vOo$ZK)W4yR+QKZBbL|0sAU&#BM~llkzEy89Lc69m7v8$O
zd2I1;bp|o}>+kQE{>w=-#qr$S>lO%V&|usb%KAcxPynR3zc<_w=1^fuN@~;{>Cc;}
zN3htT{YKrr{#v{wNmj(46=(MK(BVP$ItW|x#~3Y6Vi-k8)k7LhXd3%-o8^3X)HXj=
zXS4T3uIw*dHC4|hk}U<YZ@n+@9CL6L!v;kX4fyHvl|OmXki;mQoRg3^$=$!agzdCx
z^0F(74+5D;oK!O!lS%)^{3@C_LoZxb5aLXkxx<&`3nvV)bIuJM<g)49)_aalT-`l>
zpV8IqNiLZbRMZhJt~6&FN?6@LDsT$W{yD?HvvCrd8Gq7mE%4%;oV3r`3-IASyqtIr
zU6VbyGRpQbzMt7LF+{vQ+p9W1U(a>tqdb2d*aOBHURNCTJer#5m^9lAI%2Zd(UcsU
zO7q<sw&qvDb<Q*y?;C6sqSh)s%F5gF7S{aAr1kCH-97J9+x*uE>|1>Qa_!CQIoemM
zE7n`fChAQuF-o^zm>HL=*h}ZrKixo{U&>aM0o~f>0%AK^5AF(^z|oHKbxC<gsgkB=
zRREi)m6W*F;Q@6mEym`gr}PI0u~jlGvr^v$v~1+8BeJ&(Emjts4Gj}_Rco8a=vs$E
zHWQ5j?8<gxC9V&7RE19wtn1)7Ordj-qpyE(@6kIOGqO7zz!Yp}kL0+12(Lq%44=HA
zRO>RgBG*u&aK2@S#@h{H^<|z7%P+1_0$K33iS~yvx?Jm<{b<UG{RTZ6|18R>cy<D}
z$b!5L?U<zfDb&ZEHS+hHm70_L=V!{6Qg;}Z;hWqzR|yFn=K1llNnP9K`K@C{D!izj
z>X6!J|4pA{3w2BgI!_SaQxt9l`Fy-cQLNF9G-|fs?di3ETEv%^xbi<vxYl)p24Z=p
zVqIr3Sq#KJ@JT0%oqtfZGD?q+s1Dr?mQ8ep56sm|gpE=|9hQy=-7R#ajifKqtU``W
zGSkhPowsweZggM#bju>NT06|S<?BJbD2Uy;!UM4L6>oI9Xh&S1K2qZPBKQsey2QV*
zCK0g{#L&acGH#iUa-<~C7XiQa0(6GhMBE>Zy!o35ps3@iLTM0%9CAeiv0*L$8-z_V
zm=-}fO6lrv>eyK+Je=U^rceDTGi8gZn+!|vU2dE!nHl!;%-S(+XS!=xLajx&%XF&2
zB9nO#_NO8M0_N*<7^0{>Yo<!|L(f5yh*f?L-G*KKM6Go;w#}ZooB@37YYL7=LLKaT
z*jaUgwYwz#lT0xPGZNS&5uy~#NkpyZlEsd3vzthvRcOPbAj3gq8Df2({i{f`4mmN&
zN*6La@8oFP?0z1s_${M-T5v1H5(05B|C2l@ad9>6mU;14eL~u7)m4Gv-{*GK#vp{1
zu+UxDMG9)IbQE+e{45|(h9NL^xj$p~;f_Q|V%3h7|8T|)kq$YLKP6P4gH2MDRU;|O
zS;Qp)>RL6KqK8af%ta-il$T!b+r)plu*<NFd;ZRa6@qsz@R}PxwYXoHG0K_L=@JSU
zCx3K(Py6-1h9sATGr0rghl(5dncn$5bDG>wqt`-FQOCdoL*5J;B()CH>~?mKx`!>0
z{)n9c#hN{oJ;H@-D+``j-RiEeCb0gyp%hm2ASXX_Qz5Ziq>lNI?EjGDF<5s0og}+z
z@!j8wGakxC<so?v|0BtEk{O}E!aGT98r^}t?RdREhgEVKJILL>0Gw%#8%=KX`Vj#5
z{(G30pH$CwSItk($uyB)e|0Mo_z)E~MkI8(_L;Ld_^6aMk6Ya_Op$>QV%{ivic!I+
z7#<EAJ%W|2yZX~<8I8$``b>={otf~!*S92a`l*z4=T_PDJF==$GF3O3uJ2rW3NI&5
z8B-{So@{o8!Dmj3$Hpvzt_&B)VgXmcW|j6rWY|uSGj=6|w<?REi3Mk3OGzk$Dl*sM
z+|W}etL`id7jOG~^DK%muH@$`TdtEd1e(5%SNQj#y4@7dE-2aHV*MFPa%GwUd|q>B
zhtXtI6Xr~-(2n)uE<(7Dy?$Y^OXjW2B=0ZL7<ykQ*XNVRhc}o~U%rQSEZqvAqMg|E
z3u<3xonJY0_Z6-{(t37G{BEgSLH_h%@6A@*>)zej<@2H!bAs2TPYtxsg#p;TTc@LW
zVFU^18mYoA?XrK+iVy=Gy@a<^GsRypeKbl+mc>ECLzP4j=V0(m<>%8hmZa@Ay8*W0
zzGxOs2y2{l?K=>eKA(k5Yk=9NRCQE#yUtB_b#Xr0hi!-VEhKI)xOyJwA>y&^@-RiG
zn&=@Kk(&K;e1rdj1Nqfo;vFelQ0(~n{4ZpTVcV(>!Xkv}`+PGwn=~bFpb;l)eq91I
zC2BH}v`?he<_KzzxVOM)9$i9<-*~WwZo)3X<5yUjT}t8_;8soh16zc+_EV-nM*egZ
z*J1Au)fp3vt-*<vKi(KUR<npmjs$r_A%uA}|A}5JP9jqYyhVQthq6?*_gTFtmSYfm
z2Z4)asKAaw2m2;XW9M#B9f_#h!J75sA@jLHo7W5H>d&dV9R&&fv!3kjc~s&)lW%=U
z>hu)#`O_d&wZ$qx!3)K^Ku%AfdAkPScIC*TBQ<4#G2ctKAd=%T6ownIn*E66sPhg&
zH=ZU&B(cIeq)=`aMcb80eonb0hrc5J_sX_HRp@FO>12y4q<C<z;%v^e@J|Xk1vqe0
z8=xWS*i{&pJ>y_rv-Sjk)F%X<J((Z(7avtM>JJoK*Z|M!bCVy29+;D6i%={KH;+>(
zmMh}j)oAAhCMYbJ(0$(R8zuu^F+RbF1+pQAc9+qBuu51!`8=^fq-IG0EQL&Lp9a8}
z{l7uxhY^<wv7=ld?JQV{uP70t@gN+9$f)h+Nbg$#?r%RIij3tdJyt7-`L||`Kb5(z
ze%humE(DhGptGWVzstD$CjlLA7i|)SgS1V4JV|7yk6<d$Joch)BOJ$f*Ci40q)R!l
z!o$A)6^PZ{+h!B~X}sqto<c}cWdU@^zU_J%90wJ2*|{d)t7|P|i)5aTK5nUmKaNID
zG<m+u`cW`c@td%+Tqte`iqN<3?@he>7l>Wn2!KX6cxX-qBEzSfrG7^qWk$Js9i8;K
z1fDuY!dZ{A<EhAhOyZPmRq-xV^Z0bsJNs~sKrc4uvx>`pFTcK$E4-m1GK<7GJ-Qn#
zh%q)mVL)-;a%7;7BQC1@TGb;fGg?M)baeZyhjS}^rYK!()Iq`Oa2&~w_JmrFV_9Yl
z=ml{td*K^&cUPt#;BCTXA)vBj>pK%|1P}Lws3l}ghwf!fP3i5D)6k8SPmvk*xoMQ<
zwN4qN4&TmIoTr{L?BH6uHykjXdJXtm3n(6rNxl^sB|2(OjKM-4ap9e>xswo{Z|e8k
zdCFq%v1ZPCeM3-SCbF&r&yuHRvay1JpZNpSD{k|!`TeFjt;RmTZ3Dgi_`Yw!;?oEK
zIyfPD1*%_pi_C%>?Mi}EPqX68Xwztwv8v5|rb{UKkC~!&b&eM90#7KmsSey5_Fv1r
zCfozSw9WRFqwlq(O1~tG1JYUnVQ;Mjc>EsR^qdXI>T<44yzV#KQG_-GPsIUg8Jy-k
z?|>z_DKS*#IMQ;s{V)^;t8|BG;;g#sB}OQpi+u+^?GZ}jT#Feuwv>LddSgCn4mrF>
z#K=fC`g+7aEZI1G4M>F@40UrVB{{kobr9XGh51PS+HHJGbCmi%S-jC9?Y}JP4R{qD
z@R9y?8C~DA=3h>q2hnHCUIYQ4|Iz(l3WgWn>j*-$zsYLgaq?8{dtf>sL}_>wpXPq}
zHjbMhcH&^`%Z%HEBY}v82EBQI8#!9oS1yWy?EfktF|P5FNUJi{iAMP|u1V*mo)xw;
zB-;MeT|pj53eWtvFrV>Ux6k{A)4R#V<jM>Byx&~=hr~&fm3V}^;2C%tw;p$mDEH2Z
z3SJA@>Ai-qX7=YJQrD}Mz;Y#rMARTHEPZ4TLZ0A?B5TIWuFHwLXD>$$#Z<ff4Ndn!
zv1{>#BLH<4$uWH5JEC9Y6@^SMr+}rof<3e~Q^bV(T*Yzf;rrQCxpVGK$axKz8?%&@
za!;UV4~rq%UiYz-h69rX@*B}Y$GLw0UjM$XWvpzk@;S&Nwj*?zUv3hiWV_3HC|C_m
zAoXfZpc*_nPp8!QBgk8*utj@*D{qh|iBUEs`loh3-AyfUCnO|DDNbHr#7|idIe<_o
z|3^H7ZVaXl(dn(btCg!PaqwkERAh5-Hqi7ro5?v4Natqh1&~(+(&d}?np}SN22l$p
zGZ(Ylp2w10{9d|#VSl~diKNDExf8%g?nG7$m0AdliIR6yu==iu)N6y}8El$5O3g)2
z)L3lMXw062BE!WoHYjMGd2FykE8t<OUmiYbB?eKj3TM7b2reR{uMS_l%9*qFnrxL_
zcV%?9ZQB!!0-A^8^eb9~ZQogmk8WhI_tabM!dJoI%iHdZiM~fD#N`7afNAyh>+J8|
zv8{}*J8NNkZ{;dcq~fUxLAqD2qGEmu7{zrDi8VhJka%qoU<711AG1}(<bG{+_OVk*
z`)DSScn^XgeJ{|BbIjmT$cQ+LMzUsJogM0>c8V&H`PF+At0lCRv*U|jf>RV;+kbzA
zce*dEsj9eg61xA|McP_YaKwH>-^-j1@O2f)7OHG!NYEZMNAmLAs?G0@g0ZQQ?nh#r
zr4{)@XA){q@-Zq1CM3fvEU4xuTe1GJI}+XGpG$#y8u~5Loj>KbY|Dw+SUQcaLWw1M
z79jvGlx}B|DC35<k2ck!YEvg3gc_gQtFlb+-fGen4F!FmEr#A09(`)1p&P~l!d44m
z=dF%wQq0}g-hB#SPOzU$soC`b?=t=&eW(EvM06ZcwAhd0eTj>4F2bKDw7dPcTJQ>J
zRBS`SwJNaErgi8cu|;$;`4}ZCbojOEHUOz<BXlJ~G&W;F=ekvH7{*|lTPaIK#L)ZX
zJY9N-iuR{|&e^iH@0Tijvg9WQ6;15NjKsL1$~CO^$USA3bGQZyf`oyKG~Am(?ok-+
z$lRQki*c>G6n@X8Z8IO#i8O}`=@QF+5bw-r5$)7VO7JYwiNL#6V9Ru?Ywr@h0rgmM
z1+5xKPFkA7f+Hd<YM<2e$eL|az7(`AAQiSUB$T};vTUazLGr!Lc((3!^!4(mhP3v@
zrP2<9e=#cb%07z%TEa?>m_7&r-t_E4%~d#MX1kihUwi4Jl*Q)=aJCG~U@<>54#+xb
zXqzCV>O@fIcWXR_vsB|C(P9s40!)>nydb-fQLAcqXj*33-N>T_&#=3+fy)j8w>Nmf
zgP}&)CvzzerYYud_Vi}Nn`i6!F!!M6M0B6$&CVHE^?0q}pLBBNU%u?j=n(NVh?<xG
z;S&*$s=%U&EMrIv{G|as5?_-d!)8%VkvAlTq49}U8K1XN0VB_}47L(l8=%tEY+6YB
z-6e2$K3~}dYRkE_$_#|5lj6rC0tsdiPHshQc6aILYToTY;YE02pf}eBL^J4O$_}Ka
za8R_fX(Bc01d(;yrC^BYRqdOKt3$gv`=M3HP@0ffca$BFMK}I_zc?PU;wTvKwDTl-
zQy<vY?Xs0@zTyt2%vW5a0q4>~@~<a0^DjELS==jFis|i3E#O{);%Km)dNB6qK95S?
zItLWvD|1H@F)f`B_xf@w*Q{hcbPA=~lg%8s4s)fOwGLC^laXW85oUq#za&iYqIxvi
zNTqZuei=3YIV>nk3&OY8nnXf9mJkZ8&@+L<eI%HvlPuv1pnC@a{09Z)vGx#S=Q{{u
z-P>s9KK7Rh2jcI%I|I7nFA4b3^>biE;K?fXs^w^urz(~m`|7Osdkbx2ze!9wHo-Es
zf%YgAF_zGjZh<OBR6hZ0&aVlL=jtDV7}6`=l7U<1bptEtDUjgC`u9NGWyS5%r)UL8
zbf(@FD$Dv*0w1&~G&Pzg`0J@jxq0i#)bDNQT_?tTh8&K2zsrLGBX_e<MSG$V?iU&#
z+e-P}Rabk47bZR3UVc??zf_6DXsis$uV8cjauz(d*-1;9L5U747?z9q^{2T;C=}~Z
zF;{Xzr~%=i$aD>tZ$grplHEx1<QR{Lcadb2CDOf(0M4hBWyx-wFN9G4fsq#DAF{f;
zT&b~Er9Rohg~9T-Vu!<ac2i8p)!u>ye}`C+q+)>Iz}+#+XR@OPUU;~}_eImy#gA_?
z+Vd@QKxwm;_EUXrZb-Z6OpS<O=jWezjHH|7N35_bjlmxtOF@J8aHZWw#AXFja9TW)
z=Qid5k4Ph|{+=q$Nf{Y^{Wlujz%;(>W!PWP%2etaC5>t0{*(|aHcyZzcXO@pu(}*l
ziR{|e{OeoSpSS3p-%;oT4#e0Q(dRE!4#iYc1da%DB;%PsZ5>p0=be*8*VR)5m=0ek
zVlIFS=BKs-<6cVf%WvO=D5hANYiw!q!-5@wG)00}ZqSY7?cUn#<=x}MTH6kKQ?BCV
zo30lez(I#jX|x*wIyzU3BZ=*pgj9C<{Cn8(6c!K^YNs0AIi01iZGU;VyxF0z_4#)(
zxHEh0IT^<cJYPHv?u>-VZ=J_YZumZ3&R$<`wf#S~T2~hD>&RdU`Ox**XXY7Vz_4Jq
zMDs<Z<i|?MO4!cz<=<KQ6yDcI@crZCWy%uJ`c}0wYPjC%bHDcY^5t36`pJ#R7I^e+
znrH3usff*(LyK*T;z#@}V8L}grfE>V8N5G%Kz-p0=Xlw|nPL~RwPMG|g`Nv8VHJwD
z{(RS8s=o=Af2n3hDKOrSYeJ+0rV;JWAWovL2V%SAXEx#gZK+ltjY9s@LNJf+dcT(=
zpMxOtRhO^>n@RoU>oMZbD5k}}>mFeU&8#3(4HBDyaM?Y=Nc&C~XQ|!!mS*zbYkkDx
zB$q|##JG;nE~2BJG8m1}%rWO%Pw!fXfT-)z0yRGcj@-2a_JD#4K$r0}{sEMX#(Mm;
zhYhV;i`Tx#XHG^tvZro^Sy|*gU1ZL8%qY7{aI|m}_*bG=KK9sVNFV<yYf#1kG+ASR
zrS%{hW1vmji-2KlqST3rIlc(NJy^KbX1Mqa(U>v?|7-?VMPL*Ba)l<Ow-QJ$No=<y
zMESfi`?JLox#6Qe@FRme2ri4IT;)X%TVYcF%5)W;6h9Aji8s%<ZAj`OIjX#y%%IOh
zqa?3wN*{N)VXopd^^9Q~cksR4MXT@{@X3QX9|w&S1f5h0{z2(eIv&r1Tzg!O`NSr|
zo%6^jbGJDq3z;gK&*WPxi>}nuh+x1d?pmMgg=n8h*|XCCh@|LzX~+a9QV*mEx?RDa
z^NDwQ|Fqg;E;lmDSgqh5kQ2iFx8t_gD^yx(*80D<bWD`l>MmW?-D-j0B(H4zWJrZ_
zFjn;Vm9`--aOF`m;)5CW?IfpUH*zVcB-j8vG*HrCcAPa;&!AwWAs&KEg5xhsLC1Z*
z4@ktUFz|BQ3Mun%7L43xD9-tPn=a$y!7Grj+YT<)j(~`}Vwh`KVOCpflu!MXPghHW
zb3t{tZM9cr*u*9AYS^7%@EYKkgb4Z^vsYR~AGd82-~J%Af89kDi@RUHCvD+BQ=S`J
zIrmIY4%<E930D$RuxOVrCRdXt$o-nr17HXGeUP-8q#5`f=@kpdwK?Ek&Vq2uwRVR=
zBdHg<pI0J)KX6R*O<FnDGbl_l6b>f<v9&Ci8pOs4iu}WC3Tww!-ECA3V)|wxK{b}L
zmEZkSGU`|U4;Aw4r!1ZLx&K%&zFYHJj<)3?ZDR7A6PL~i*E9ajQ2zoNc$E6}ftiNz
zW+wkZp|M=KFru8Z+&IN-Y6IEOtc}GK$80l;gbkZ`s$are=8=D{zE5RL!6LJ7nYw`_
zzlvgtMJjBS!J=?{6H%Phmj+e&rGqF7Tt7TIag32+s<CHmW=z8(i>ioV(X-7*Q=!_(
z;OR?<s8M$C={sfoUf%S7*Kg^gfC?W{nkmArdE@{RdB1tot%y}O8j|t6mO+G)uC90U
zrRgY&XEBG}WY}{v+DMD{!LFD^UydTw)`gu3B~&ER{&#f;4&rqPsD#iOtLZb5d%8vi
zJrNrcCdng4zwwYcdIK5HgqFO^fNd%})3=c#btB9QD!wuon~VG=hFF_YKz9_mu?|c~
zG&djVQ@h<dE4q}pc2u7nz?x@3!oO#SzE@yfiD+7K2J^UE!!w#QXi{^(StA<E>yRbt
zh9ru)>XhwdT^Wx6Qfq6464x_fnY|>B5L-g?ZcRPXM(^2_fYBzn7_l~ODC9Fnokwyk
zRcJP&O`+-`)9!OqDKzE-tQlnsp8hUeCjO;Z!v)+MwjAuh1G@bjhq8+GNo`jms1~Q<
z&tT3Ek?%s{+0p%=;{CbMM$YT$<zHiglrmDxrYu(Rs{6eH&)oBgM-F9&#gZBqekU7K
z8UvD`N<!E^_aGxZEt~>!%dmT1It;u^3qR@ABT^mHWnAO#FH1UreuW1p%2V`W9Ey~{
z*geXq-2pQinakB0M@ziy{Ta#s8z|k;D5{=qdna2>`m^E4<5+u$JcFZ$UGsJI-5IH0
z$n@ceRNW+B56em14)Nf^u)97eZOYYHG)Azw+CS`At{U|b$ze#bx!w?Qv(CAt4*HTw
zDXTJ18W7++<j=Q0_yV5q)oQ1rih}X1LJ^856DKbx72bMZl-6GK?s11N#OJD64=H+&
zZkL9(HndX+M>O?^=vOq8)8{xFf6;B`*e7UV3Dcxn46ID48j&hMexAS2{j8XqYq-}b
zex`rlPuG4bcxsY<33f{-r&7DIY%Uhw=?w${dw5i{KW(r*+BLWa!SVdc2B<m`8%z&;
z=<cP7Wrybi4l$Wu(jfS4WgNzOQybC{i^`-B>2kz@T~C!bRk);ye45H^2UO<L$$UXI
zFT#*2$`XO4;Iib4*^t*Wn+wCro<;ovVsxtXipuFjk*ZMFzV1yW)xCmgPQQ0+QO_L+
z$<m#~`IxYn%!c-ad!5!2H7;X=khHnl6{X)<yM#Wjelkh2u_n~DoqD=P66^VPz^PgN
zrlQ%_ckJlVL{_eU`l`*$^0rq+i4C(frw`B8wf_1e_Ihe=tiB%ZluR**6tYVn@`Is(
zTKom07aU&x!y|gPfaR&8r{BquViy6(Uc32N@U_M}C8Lp;0tKJO3a9o{bTIBG$pUk)
zj~<%HhDJl5LVduYwvhJxZl13rUuyRPDb&P!<?xxVa_Q=6=%T)t6{uz=QKx$2CiorA
zZQ1;4;+W6J7MF03-BF<xvUO1=0VP3&q1&@<@9f%^xM8h#LlwTen#CJqIhD77D5Y#d
z!HuYdUFuT>E=MpNk<f?8qPbwIW($s<UbT3e`g@#lQu30ZQPiIu#K~8@oP~GsK_4G9
zaMOaT$hRXjhi!UoG_dvRGK_}jRTAIdd2m)phh!`kR?tM){KqP>Nzx;1pPILEv~F+@
zgH;J>o+b&|4K3C5=)@LMv6}(%ARNStkN?i@D03-XP9G_-AP7vfkEF}itrS*jQyI%}
zIH@wu7m|-!#`xr_K5H!eJI*pyu6-w2Jt8|DB1@xm=__FKGJ{4Vd1l?s;xxwUP0AFC
z8ZxzUJC!^OE^B%jx|Erpv*uE|h0B{UmGC~~K^6RD>Ep%aP2Oad7w*7%C3)n-t68^y
z{tkBm$y3<ht@XAz62lwGWY<ajnhX95NpL=$xv;&b3=`%$$`ewWWS=aUbx<v6CvMiy
zlJjmpNY%M@P<v~$P`$S1^NUY<AA#X@R=3AL!wU~+B`4n1l5M(ow=yZ)otG1=tcNS5
zK38!tVfG`u$$Z<RVF#!@0b6Js%>7_S&Hp-qJP;bBjhr9PVic_SAMv~MT@#V$qGUUr
zIMKQi1XqSf8M89t(jz*8-ZNSICZ5C~psX8~h{dbM3aTR&J<sh4I<A@TOtt?Z*pu-x
zL4}qu0CflHKeH+5aWn(o1^CNC**>t!|6ow=NCWMRImb4G`b(G52fgW+OLn(<&Bn=B
z83zP?GbHbMeDJi^H<}a0e(KGX3PH$0hM~(KQnvz4g2%xFdn#oC+gM8Z@Tdb*3pK_9
zk~DJ4VICbgV&ZfH26VE0o@u_}^O5vHoLl%+NR!Qa9E?TM(G+Em6B!)HRfUZVF?6N6
zz$kKOog3l1O7-9Jzu4<v;`a=e-Le4dWHJU)V4{CiAQ?x119*gpg;$9UZ}hX=Cq0oK
z?>F02di`}<D18G6C-Af~80piWbyIAnP`o~F(<qhA8JVJ-28Q>SF|mf&Nc<LEG!BwZ
zya=9$P&XIz_XPgbQ-i~8A!wb{;Q-92LnxPI$8U~2MBEeYqPkjj&SDrn2c(R}@f+Ml
zMwbN2B?J^UBh&*g4{pYZD&f$H%UdWXDvndMwu!-%$k*Pe#4>G9J>YSNfuf%*<*?C3
zrl<o@Lq^N<=*KMojuSVSKA4)g;B*m#FKVJJdn`(g@1wbPbCjh`>58E(695~VI`|TJ
z=A)6?Qj^w(w!(Xcm3LdOVQpo_VU2z3p32Tm4*VoX*wLwFUQ_U0IHT$i=qALS<%}x1
z=ZV(2RwcX6DKLS%+bkiIbctE0*Q0LV;@g^WU_~FX)pYOUvQFnAwA8H>oBTA2ynr?N
z;2*6ruT7YFd5Lcck?@2j1bVU8m2#Y<wdfQADRle1XNrskX_=x5=#Cby6K^TjadX`p
zcHTKU0BnWs?kA+&ehr>az!9oPOU>@YbgcDlH@wwOHe58X8S~*0HD}qEysHt65d*st
zg*kcM4fx|ta4i$u1dTG)9}|Hdkq3V9ap?{IF{XN~IlwCcE*dfcyI&z_@aGvp5^z;5
z6U_qZV*Tny2^GBG#acO}T)#PM6R9gV<>2XAUN#c_YF&2L^w7K4rqOlN)hxf57<dSG
z*{}|<h3^=Vx{jP*+nH@s&+UJ_%}T}aMpr;)NaOgxxP;_5q|nTv+1+Hq0y2inZ<|D}
zl;Z!2`u7f$*BA8_pj!y@B=WG;&OG`;8icxt<k+kLcK=%a>A%@>ERhm=^gpJK%Kl1n
z)7AT*v>OJOzc)@jKZx8=Mm>LdmE!rIbgl06^5a_<D(7WLFkCdQnd(?!Q!2q6Faml>
z<{z}E7Z$*cJMi3J=N_191}!&>#hi950p#vdQXc+}(ChK6Yc+Q6TJMLppZTvZ7bN7b
zXlAdIf04NlcNrX}Q*$wdV&0gC{$7Ojafel**LKv95sWB-sBs*hX}R^$9c#$I&yJ;I
z$?br2+||-#UTEWvzWq6!1c))8IYE#v>65{?+yQvv<YetH{AvTBFyR(8+CG{FAQrAx
z${bqGF(^0xyStK#MZk#?eiV0Zv}}Vph*GNY>s@Jd98b9B`^da%mF#^fEhv#+*T)<>
zY7^NgF2kr^4vsul2S-#o4X&2N+#wi=Yz)zv<!&l$iG#6L)?J|UZ?e7B9BKNT_Lnv_
z>YZyDZ@&B0AnEM)ZQa{dVSs41lD>GsX2dRWkDn8u@)}OGyh1tT6J<PBtIkKb*bK*~
zzWq$Y!)EQFZ0N*6PWe#d&%8}TpXUR#_c26GGofdc>K^ZzaE6g`vvL<QeOPGg-XX`j
z<Hs?yN)m@!SGRoi;1G`jYfEnR{_N{Jcz^dm^?3QBL%*d8rweKM4N%ogO+&Zo92#-5
z|I-Lpy`j!S?`g=m@EIwgWx(qUN3!6srFmU&vvZJG>n{<G8z3-Dj+W#mz=r5lggZ`H
z+Bxs_(5<P2UPjdoNT42CnmB||{o5|Te6VYDwrL_m9$=p2+twSazZJPq90d#5AI^ia
zD|GC|SIvY&ao!{<0Pfg?)q0N|MiW-=qwW%&H=GA^spaP<%NZqImJFoX@9}=31rs*C
zowv~}(JsSGcEd;&Ikre`3$s>`BK>h<KkhtbOHShrZ+LdI^Bd9ez$Tr_%{C1z?y8uG
zS=6TgyZB}b4`CG*q9to}t0()XDQ9WhI;dKyZnFYNrGKbQaRWbTO4WT~q~<s(r#jJ{
zyv#Dn@rZi*?^e&Vv%jMi$2rA*P(H~LL)B2-wyK~TMgtmHcjZN4+Qef8DpvJo?aCIM
z#*2-~ZPWVg4SPDhISu8FKik0kciOUl8d7(l@_0@YJudqZ9Vv|`b+!o?Wh~cdq@DP`
z09rks6(*rx$tsPfq9DU)Zw)MuilU|mI~$IQ)L>@GYI>x$&o|)A(9%FdxdL9}>4b8M
z6J15nFWQO@!$ax+1%n;L53V>hsh&Wp;46C%4mM0->ECY-c#bLBefQ;NYDn#|F32-3
z|Bd9?&F`HK(!VSQXTO8~_yK;su%>0d%;^59?R5Vs?5#iEN$yXZl3Z*LEfeU@t!txL
zn*^wCS;rdge$IDxm)y!c?n5Rf3VR9#7=J|wfP^b>A5|7GhqmSK>=%M_7os4fj#cml
zxAmuAYQHpN613?jx!T?cMcWpbLRvJH13F*5(*0{V@_gGQodXriE(DM{Bd246TxVs^
z-^rp_rv9R`FeOiQpmT=>aUK{jy1D<`Ts9Q2na1|oDh_^PAue+yORopsgg?vJAl4=G
zEYw1?u_W0YQ+;G*VdZ3HVc}$9OUgRN|NjoVjx}l7SiaDxnQMKKv-U9m!p6eNOZ&yv
z$=J=x)!g`thq;TZwWGrqE+%#+);i%60(dbtHg0iV4sHovUNKR2ULF>99v(4q2@w`H
zP7!V{4puRtF9QF6Q<Ac-cmaMiC}mLprR)A4FM6o4ID@Du<Ln`hH*#Lx2mW}glO9~P
z{F<QStnIY7;j~P>FX))HdC_Ka$x1zrDBFATSKE@&eBAJnACgt1i9wDKMSscX(`;ya
zBbVz5e|MwzC;w0v0L)+Zm8Zt|7LM9v4Hw$N<jA&;4SBG4vuaM21JC$DPxf$R?uXa2
y2N<;!Q2s}A2QybU7jt9#|9m-`*nD(#Gj?$UyO>*i<YMJwWB*7=DWNF&@&5o!5=j*R

delta 13276
zcmajFRaBh85-u9t83^tYB>3R&1cJM}y9Xbb5S$<h9yCA*1a}F};0{3t9o*gZ&ffc+
zb=SHt_n{yDe|4{}uBz^;`o8MjLW<i(s{V+IB*ZI_td778J`OhaW*qR`5e1`A4=qTw
zo{$Xm_bZtxHbN{pKAa@acfaAL<zM9uGsKpE`+O)^_qU9Pt@YszS`;OpU;@3IMF*$O
zcKb={!NCBV&vg{n41py7lfLOboBgxRr?yxDRe}AoB|r10HMaNHQDs_JSp19GPqXG#
zc%0^ERWIRr;NpzJ={pS8tfa(#h<cgpCw;(KG58iXK=bk_`GhKA`h4ekzk96uygOky
z-!PXPW7JZ1=)hD_Kcs((*QlZJk%jw7!-Mz!Te79lXSdo_yE}d;3*#vW5UE(SUg@Tc
zg45WoDr|4;Y)7+UlTv#V+l02IB0Q!mxIyl}bUtkXE*DnEWb9@Ae2_fR=p192%3faA
zof;#HCgAJ*PKDKZNG1ZDR?f*6ItdQC?wkM6I@|N}XKqS_d7NglpM%w#&w;zImuv28
zK45afh2JeQ**|Q`GQcmHXHaVXTmGr5ch7BVAs379TFJIZHdBX^Z->L#j+u*injw&t
z=t3JQ@cZ{dfG=1R77kKv*LK6WTjXDUnA{kQxOYBpoQ43#OJ-Aa=k(WJJaoU#YOQs&
zJT!4hO00tK4%RxKcez4tUCT6xk8Eo-RFDO~W9Q&O0W1a<bpGkEs5$O%-nNH6qDoVt
zg4Bvpo&rUz%ZL&tm1R>5^=gc<mQYTMb{2|qaD2Y67kAjY6B4J?47S=2Hu~`Dr39BA
zKy=f`t^->2g5M(5TDnfe_!tI4S039wR6AYus2AB(nKi~36fL|9b{hEEWFU!NP^sY3
z=BkVSDyW@r5K7TnH}FCqsJ|k-#fCQXK7E<s<>)wEOKc>^RB6)nzIeHX_;BU^<t)7w
zjJ7{nTY@(tb9E^$()CL3fgs)?4bTyw2uf4#te0QyD?;a9a9lJMH2E}-bNV+gg_(lD
zz$?m5{5~T-P25*wQpKyb6BpA+@x9}!g@y*LO+_ni)?~*3Q?}n?dBToOIe+9>xR1C%
z&D8Z`+^zr*?PB?M>q&h~>1MmYq|o*rSkky@47#+})Zm+311%J{5&TfJK2P`?d0pGI
zDL2P6tt^<>xNv;w^ARER&(}$lg@d+qs|KC(8vwws>GpK3Ef^H^8ghJnp&lMbd_Z=t
z-txRW)$?(%=e5#F5|psiS>Nwmc138dRCA?CK4Y-F+AYM<hsWO?g=h>sA{FZfe{Nwx
zF%9Iq;|QV&!V6Ce%Uel;aSkXB7_iy-pVJ$cx{U?=)JDq)^_Ye`toO7>IaDfGLjIFR
zX|L4go4(10`Ft#&rlQzLK{s4P_wu@%%s5mnv+O+X)e?$_4qiNO_9R@GY*-T5ajgoc
z_C^3ivs0p7JNyQKaTvSb(*6YIM-REM9YoKk@2t+dwTvaV=8qdPfgbFnYc*(`Kl%wK
z0^DDjo=sDZ`m|V>zJNI(H_kYjbCZez+>C-kLSY!rgxX$6fj$ELZWoS8IdN0A)zF<1
zN*ir2R5be<W^Uy^?5NNR{=YiYnOLNbKJN*1=o2QFaw#4(`T-JZi@(8@#5f|e)S_Oi
zR#BW+sihdZ$=<mCoa9{N$0pFc%$D0X+q49){I*25qm#9=5neB}ub{Nf?vS;3q?cNr
z7`y$6Yrjc`@K%r2X0u&nB&O$?<__YZ{$?{`q9nuU7u|%zfTj}OC#Rh$+xf&sKtXab
zz*HeZJPg2uIiohEU=6lWpn>UcXE)UrCo-w@2avSpN@Vh}B!?aU4V3v{u1+IoKXCwi
z=4)8YPnDZmA<#y<&mZ~>0Dd%+ZK)CsF1YZoqWSzSMbf-#0=g89h`Yl&WydfcrlGAe
zL31vOmjL0U*gk9zfko4N?5zn4ZBeK--h6JE)Y=g)d_Dn$e*zQ#C0iVj?RKu}$#Suj
zF?^Up&Xh)jq>7MMWUhZb*CU-T``M&5iEo5@i0+$|(w~4{@J+sOJL5-E<QH3_I14n-
z8RWtpTVO)#5K7q*k@E=3r=jR;fm(9kkpPE11Wn2N^&Q;~6#3VBH?zSq9MAH%x~vr*
z*09O^RKqznf?=>!X`Q`aIj0;B$&tm=x@Ql7v1?`7-k`?V$Ob3j87fu%W&A?`WkA<p
z{~!CRZVKz4BsmrI4v##=xWL>C{3wHAb?#p{O_hC4f5!Oz4K)Ufrd1dur<w{ce*~<H
z-wxSd9dMPDvCE4_9dLi-jWbf3x6DBr!HI;Rg<HNkpp^i7Q+~qSDI4*?UY6UX#v~Am
zVR`5Og(~jsr}Cv+WvT^`?L9W)uQbYpYRj&zs%X|8H&G^ZN!pX*brVB#_hu|lFDB;+
z<HIf`!#^#s-a+Ygtch~|q%n<-I&bevHqUpvjZ(JJ&3Lx|QiAXL9*+*8^v)hf%gyqs
z$sE@7SYdC$#o}_jduMS&`xvK<c<9IMdd2@nvPGp0t^*D8S;AWKz31WA-SY(UoH*TN
zE6kE6!~M>mczC|v-Ry~bpHgYfsQ}W!FTI-?%QoIe9F;A0JFsnz2{x->2(4TPM$ejS
zFaU$Xzf)IgY+SKSV2RH=6Dnj%bW`d|-_C02bWDRSn(;<7qtv_yRYio2|5}>iMgBA@
zn{M!2P&3?&Zs$kL1ki^2_YFEOa4@&$YizHd>c?Y8hd#`O(+C-mnJ8OwRM>tTdug6V
zoPLJIMqM)32Tx7d&rbx}y&(#B8OANF+Yg`o;elYq>t&RYvEECzW9&JcfI<yUrSp|)
zitq+U6zOp2Y>dA?4QFzl0wmj8!FOl7O-<mZm$Jk4j;F-drGeQ|gmlzU+c{J$1uXks
zQox6XZ-_AvO5SXNcMqBRQs7t6zt<u}s!y+z;;jr~f4bY*_%lRt;|k&dIp7xL;D9qj
z|KuoGY10TjZP-8LFr+IDry+ZjI`7CwZ3tKd?BuYkjzh)AOIvbVq7=NV6+aOwg!;ue
zK(}yjTl2J_d|qSD7yZf&vsg|lzIA>HA1^Y5JgR|i5rdg_6TLa5G(rjaY3!DUhQ`f}
z=Jn+i{Hlv<-s64?kaR_n)O!}W<4=>XIThK^dYttUAc(z+K=7yxKk%=h^{=2By#iy(
z4ypD(+Uz0`HS6uvZul>nd{z`FEe_pwLGrDeJe2JW$HJpT81ZJEFHFRl5E?O^^A-_H
zb1f{JQ;URk4%h<JG?CmzQOU&!FzwQ&qzs8B6E5y38e}yX|L8#1`@8d_Nc4=^RA`-J
z8s8E2Ixrs0Ncn3n9J0B12q5=sQG<7v$qF2Oiz8%78cBxBL-CykFUM&(|0Qn1v`UFt
z&(HFxPpREJ&hPxCbbxqBR5cIxf4)a>SvjMHDpBm6cxfq;O9Zc{slT6ResN0TR{QxA
z*Md&*ab7J|H;SD~N|cwZB9>|vRXU}G64m+_hr#I^qT}PdmkvWN_lVHc->YDx=$Nps
z7+a?}MSEIU!#kaWadZ<BloPp71w^tE*Vvnsx}au8`_b<o(=3IIB0B>xIbA-~w!nRc
zezD1YZAF|dx~3FOI<BTZ>OJuoVNPw}HuACTE;7#XCo(GZ6=4`Tx5|{iy`I`PB<gSv
z%WY(`qYsP<5*CD<@ubVlmi1tK=pzEGo%zgYFzyd`q%J%?;?oOf1NILQWyGO+nYAK*
z_fz9*eeY!+D@Ht<^usHzSdJe>yXW1dE_S_8^%anJz+9jJY691l=CQgT&05F(C-1xY
z$0$G2$ziD7@FltZO-EYb<^IiLp^$vv`!{cIMbSUw?G$5Qp3lDjECKfDq^s8!dm@k5
zH2c|A!Y!=a$xR)ox1E|AV(l?r1+wEM+l1KEO;`G5M`8{*HhWO&Df0%dk|VLXw(sKc
zqJ<J1&QK;#OHCt9I9}!8BXX54=ddOsOD}8Bp8ry0?cMWtaftFm{s;O<_2pF$FI#z5
zM%iDqi}ISQIlxmN$qI}Si8o-fCrgJqZ2v)8zMmG8=jIj<B{{FYPt<SP2O2f*Lk@M6
zh+sLSL`DaFQLUKsDp#_o?PuN>ZZ!CH?ZRsFReU2}NR1*NF3xHYZdDPPxO;q;eAZcZ
zf7}`6MmA;CkOS>*!zNz9Zg1CD`rD#GYka&!6S-h0kLldW9++nZ6f&rml|}6Rqv+#G
zZL=gO@aUa+_V9CS`iG9Wm-lD*^P=b=Vt|}snchF9hpy*{Ni`VmDNEm9>KP3S9p?5-
z0hsD7vUmJ!Mt4uMp)d`BH)&#2xcA8bb~t7fTw$boSof&T@u8`WtCrC};J)VLl%VKH
zyB|8h_b`qQ4kpn(QA<_$l6M_=!R)#y(>tf|yNi6u=W;*wR}*dpdy!NA+xZvR3su5X
z2l(;jWUOw3!r2OMoz%XL%1?a~w4)&i{U+<*z8D@gF~6kcb?8(Li6~y2U$TtCHN6=_
zZFLR{ZiZ*H9pX*URf;2oghs4%G>)y4jr=Hk)NM8J_Qx+H2)-MqMUN#U*C4m+@+y*M
z#dLjiwH01|UoQn;a!O9ZjpO#hpx+HTP4*8<O-+*2ICA6LLmS4zkmKpYh@$y5PAa!j
zpPQBAhWN&0`A4-v@zU|pR@cU<oB6<Wm-ei=CE+FOWQtD^*EZO1bN9U!)Flsf&-PoP
zl#+VzZMs{6*~~Tc02-%O>>j>hg74n#7Ex?rL0FQvfL>AbH(mo<orcD}1FRQK?jAXZ
z*AO-~3DYxlzXGAT){FB&708c;ujK(Nsknw%Sy67|^16Y4=QJ|aExZb=zEwY9de^yw
z$5hgcVCyp9fI%I$0?TYBr!2U(e|a927KFi;2L(0y(+j@8lz_)#uCzpSK<XHsyFm0+
zJeH-{C8R&_tIjuZlq(o#MU-eo$Xy>sYU_cD)gH0Wk-I*=fnn#IEuu=d_)R!65sg8x
zle7!QXP8_L3|M}Dfyz~0oyLAc*jN|P6IjTcZ75#$b1mN3$jYLyv?<7{ped-|!xj9s
zmnW&bK8XG}PQyoYnT^ej*NO~XYB=x0B)nVSGvNa}di|d^qJr<RifI1y(ulO^>+=A*
zuTIqA?EO>JGJ2o2BtF#COMAHA<JbjaH>Ws?Kmg5KEo}mW>(OCX&CLX1=R7DGq=KM5
zZi@#Ty#WWO4B%6wH>4OdV@yk#vK*{~0dZGBhqyn@{r>p=T}$Orho-Mu*WZ$3{@F&V
z%7WbCN1@_ywq;Zy;>zi#Z|EJHO;i=tlUl!#%hn{lf5aY8^|4ynqty(TUSvPD1$X<3
zkuQ^pcY{XAXgc`*hMh5>6Q0CV3%;{(YV&^^Mxa4pc`{DkV`_Z2o-su*M<EV|jqVwu
zfC~RM^mZ%hx?lD7O0TD`K?yNz8T1{m<QB<l4hkCeb(ieQYsm#4)eReBwIdIj^egDG
zgVLBM>sO8HStFc&ye)k>JAs`9ykpb5rZvKs4MNxZ=oJUWDs%U6zsA`K>b;9AZTt;6
z>t?e40({g6F$KkR6R+K?#$<tmrHsy;*x!{}_8d`+kL9vj5G!+dR%<bic5u^;=90SY
zzR5I@PcmH$es`Sh0k_a0tDC3=EG%~?rTh&Ojk$edfSLV(=o-8r?ca0a)WsUpXQs&|
zsy65={QLp?n9j;pG@v?)7Q3^%9VVFZ2R|{9EPL6D9WfEww|aIy^#KeDF*@=U-MjyU
z&dLk%!=wpgt!m^iKtIIE^RT-sN31oRdyB@V+jB-i&~p|8bGl<l`)*SNBXHc6EC0G-
zrtg`BKf;2=JD=Qd0ECHL+(lqzR3kSySgK{O%+Wf-<w=dc=DPL&c2KoV`3z|y{Ywq~
zd06KBynQYtKZdhG3byt}^Bi47uYb7zO;Xo}>USaq)UF)Mc+Y$xjNUO$l7wj$M2Yfi
z?4p`5Oq>%TutW6Z=j9A*7z<A(Ia6CcY?-12(eAE-E|Nbp(4Lk!(B9(c|2R^1zNW7{
zVQa~;F5HnICfP}%+vf@(vQe5J%snN{FpMBwY6ofsaJANNVwUqmaPZyP?DczpDjbvY
zRLynzRkK&2_maI|i#I4GW;qj;n*b}G`;lgT!So%UUC;2>aT~Pj8$0TIBrN6$QjATB
zJfUiw4g;03LK%(}yTM@H@X42EKG?ITqZt+%<9}aY_&2vVo>0v=ssuw(=&$*KqL>bE
zHE@TyPK%(fTMmdzRxet5D7!Hvu~<wXvBNlnXBTLWG%#bZS$L}a$H*bqDx@voG49>z
zORR3Y$c0-asj_JEzs=zQk<=nVzHCuhBvD-v$-X^ZLI=Zaoxp9Z)lKb@sd7+v;gyR;
z`sbQ?PnPk<{=n;4Omm0kGZa0nL+3yIs9-&4I2WH?I2Um4kMP@RqhTa<hnXDw8BU|+
zWa7j=-cu&A=fIwI4olyjck`qWn^^GXy+CKc_n`ln*8gTgu<3S9lB=L*SOGsi11c*A
z5yhW7n5{5-m<*7W6%DonY4aK(Y+&rw|B5)2uu$K<IB-qB2x-oGyl4?N|EpUBerPg9
zKh6m~TP5@77)vm;wpohjfnmiH!LWJ-TM&~crG}mDX%EIR97*_36ypc1$U9gF)qZ@F
zA-z5yJeJEapMa;KUmQDqL_PoG{Ic~BW^79how|a+aiL#3Jg+`1-E3clG%8SrTbBg^
zuY8>lj)U62<*ghaUYF!eTZqB_W}g@(6sA&pTf_Da&aD)#eEDa70+u;2j@d^xj(OYI
z)(R0wb=jJ(7LI>ATn%%WW_(i0jOPdKZ^j=FkcVS-x-_3D!$a)<jJ(QC!G(rEusK7X
zJS8=bG~lw8gKy3iyZnPbQO@8L8#(swQo$g{vXOG-9Nie}`3WEFYcPLF?;^s|!y|!)
zpBCc>z$bS9wMVLlMDs8yY$q!^!a8K*^J<c=j(NxDdCeL>p!qpedSIBFmnl^vEfF;Q
z4RZfdp(_(%Enp(VR2GpP+aPaIs48S<E$`3dsaC;FhO;H+F)F_*jDJ%w7|e2E5Z>1m
zx6Gnocttq&-LVBcOpCzq&-w%wpUmE%!|};CeC!Zil(JU_?IZrxE?F;+yaHX?R%=*%
zPSPKajXxI>f-w=_5kB25#%?jp!N!&{#3t0N#8;u*j0y_M>OIoAdEqeAFK-i?uo=sR
zl{9-CHieahxWtqdTgI)e;>6fKwm5Fp;zSA#NT-#BZ;Ba%6{7_E4ZRr(zGsurY{zx-
zK}J6$w$5&f%ovd#9#E=JNNf^fk2a~Vr<`EtX-Q^={7yTVo_HErx&@mQU8`gW1iV{R
z*@pQyl8s)+jtK4+Wf?jxN_YJ5suX7iNdp5$3q>V6J#eUwY=S>ha>X?B+xf)bd4xce
zI~?1L{d3O1jXBeqvSd42Bl(6hbr9smk+QIG8OaR-az<ySu)KUzbI(jT@N+qNw$U`K
z#yYSUz?fl45x!60o)DH=o5>C8rl3I!6-<L4A@e)PSmJcpm+@qaG+=7Aqh4^(u##?T
z@4lPGUah4J)lxM4c#_m&REW8tdsnqT)RU!-&=8#K&$b(QVrzrEO^P0)Los3AaVEun
zL}voq`&X0^R`35ntCB16?WXz^h@WiEJKEzQ2(>NB>$dRZ=5jyh6;dbv&HuM92qemk
zMxteZ7mhhn4CuKs6d!dl%%U>FAe>Xj#<;X!z7$YYlMPS?(2`GJSdavtXyqx$WcEW6
zPQWRbqRk9}TP>eIt7D`uf5YsbqZsf@lW7>GoRyGpLE$TP!kZ#RYejdYgLLujwt0o#
zNdJ|3?3y9rm+&hVZHgwIC~|@!;8)2VY<D|-A<+4G*5xu!7?<gT`H{J}JbxMqinfoS
z186dFN-m6-+DAyG*?-e3>YntAFXd(!fgBuTk5d0t;I??o8iqZn3`^M*_180TjpEH{
zPZzVG4mmfTz+nGRgdKkwSb0qJqd<t)6~J`pg-|1AcI*G!r2@Ly5fSg0TWV2V*=Wx&
zMhHWmQo4D<uPHzgGmF~}WMSfR{(>}-MQPQiyp9qo$~OEu29m*ea}x)ab|cUu@TAhu
z>JY0vT1mc>fGk}lBw^)$%n#!jHY6M;AHOOfW^k`ZF>`dnKGeP2<gV(b2Y%iM{-{Xr
zP5+OEx>i@8{9i-Vegb1Hb=GJjxdnXjA)u}&zW$1%&fss<lM+Cx2ZJ;=A;Oto!Hrg)
zC~Gq@oQySptz>#t(-yd@NRO_;<zvqenEBHva5W+700d2)>SWnMr%}jd;p4JRF=<)@
z%EslukV`be30=+;NbIY}WAdA>C8Pc=#?#kZp_C$m&EdVgpkaVH%ftBV`J|d?%s{u$
zVUBd+0}yf<$(=Pzj}`oFXHDYU1-nYAYW27;sol7*9((ONr~i0h3%E$CS|Ec~>pJ-@
z-W%$T)5l9x%tQBPU`T2BBD-^9VZ4y#WACVRh(osbCNmk1Ek}IkcYi&G))FuKQn9!F
zz-PUg;!zHcQLN1yT|xA6)--9~T$zPT<MIF4Y~ba*_>^X~V-eF<XNfk2o9PB`ggy5t
z%(5vBSBfSaX%g)>SZbH43S_vz$Zips)u(~cUk02rno74Wph4@r4i;V@&0QU^am=od
zR3dSp`32ArU-thqhFgKG#O#2r$V>~j8AEY?o4(qlgBbHq)2kNcVCt_<t6d+@evQys
zVg@|g<V0D-#*}#!dTu&>WYJO{wLe&0ZXY1<OmwF?A5f<N*Tm0?#Jctlb6QbQ{|WnZ
z?<ATh@zLCYCWD&bM0H@4#M(tqs9Du2*NQjSL`eYMylcfZ>#<{nMU2?S^vEPs1BQ7e
z-HcckmF3}Fi^LW%O+YR>O%UyOg27OfUth$G7i0gS@qhm9$+4+f{b@sVZ@}!Z*O}gP
zoTTJmKUfB;Rg#up*x?6Y<X9^npimL(Bw8&w(DZ+oaD%t%k?klGt0{sUjE5^tlOe*<
zLF4+DS7jVJ^x3s&l`dObTSe(E4DydtF=$>C`6{2=9|j;wInnUk$EkR0*H_+-YwAES
zE>%vs71b~3{+)p@<!YFWHDXt;@xPVmJ82xSdG*D?uYcqjjP^l{I9!T*0dCnFbC&Tf
z8XVeIP(i;aNL2j$epU)O?*1Om0)zUDU7ZTw{=G21mz&T(Tvicg#kdd^De12mU&H9i
zdE_~nT9JEr8z~R%Tp7iPsQRpU@~2;x(DvW_e_T5{Xq9Qm@%?Ek7>kd50@DuuBx&Zs
zgM03lU-9pSFR<+$DICD-W@A{MpZow)16&$jv<846fF>Pd@5Yf&@A!QizF?3)E!P6e
z-89a5T-!V<AEnP=$HWkpe=pA2s72SAMuH{>hkI0YZyV+Ap8!#iV)lrU)4hwZu9zg(
z?ic1g6chCf=4^2BUHhDbI$at`l#w;1wPT?0_~n)F-6)T9ECX-S$p%XHcKUDa4hMI8
zs4*J%sHe4cpmc}ruH{Wzd!J)U)|q4H=6#y;AKi{^-Mt<HCjs|g29+jlA+ffVDV&a6
zO_iLkT)ZE8S`b<CT@!1U+gH$X+nZiB>JBfM(ruck0#d*|P1aFfcV0m*{J=6>RiF1G
zTT8*~e?X6bc%N1J7{~X=4S+U2q{|OkqVqf7Kl=v8+8r`eH5O-*7kt%HFLJ0Xie$-b
zDu4>HdE45e8U7HRJ!9z1DLUutTxO+Mb*w4sJl0G*Y<ldfeZKJ@>m4yJ|KTZmVKBUw
zK$CXa#5)Ss!)UT}s+489bdS26>3ojr7jyrP%%5`GMP!y)e_t^9%ywnqAu3!}_jUU?
z?ifZcv39Scw`-(i<MYx?HXr#T+t?)NS!LXW!&KhF&E8ba!Y(D;8%HZ`8FxD-^bx?$
z?{S-7bhNG7i2au;R7}!i;H^5|C{FS$Brb!0NHZ7A)co3Yf$T@~5xGtE?Yzc;v%P;n
z!R}WxFn^%<+ikT8<m0msUqO7sE%%(=-&H0XbkZ>%ZsA0X=f&=DA8wMQUm(meT-16q
zCcMQ_piSFytYRsQJ<H>k4zb0I!#?{jV3O#8G11K#|55AER5FoHM@06KHYfA|Ca<FE
z>re#N5J6JsTn3lCT~|%2j^v@alX-VhPU<&x5YiCH-jkCpMsBzAFc@Qk!>d@7xSL=$
zd^R)1NOPzMqS*QB20OmK{NP=*?whFSB)rv&9WQZ}ASE<Ua{~CYUQ6uIyQ`n(t@ad#
zdib@!axSaFS39B6_*eH#*ObD1l@I|dWvB!gUBTiSi1eFSw0fHP>9df?AVBp^o8sF!
zjU8tNTKWkLcan+|gEEC4idAKgn-qlX0ps)~*x2Z)Op6z<lbHWM|4<9g>R+xaan3pu
zwAI{DKm3K|Y%iOJCl#7-EWWmXlyTowd*%~^K8<tK8in)|z!sfo^f=$h7r9EtbW`?#
z4L8T)6FUC)ykfzhS6}0=R0jLWoRdOHoKlGYM=8xxaHVwl{!b~@0f5<@PszhwG?Wxg
zuy^nLT}Z2BU2vC`@{R1W-+8h@Ycdt45=fyR>XrRbRJf|<JX)P;!p-^Gw3#tSQ8*$I
z$e^B@Rm+I|eNf{(@GoNv84=1+iW8EIc3focDm4BFbfOSx_=4)d4Cy+imLGzcMb<iL
zzs$eAX*T}`H<9@x3VwiIeXD~{r;#}E@#n5z>SX<fq@n>q4A37#Db9q%-ibvO9)fVR
zLcrU|JyNVRnD8RHELmoCzSCg+^3u`;geoocF(T-;#Gx3hjTelrs1O<m3fHb`d$%ro
z!;Q?={!U7qwf|y9#u;dUHoMpJ247*%bpktheKo%|u+;FeatODA(ChRpyEyD=QmzZ?
z(nJ0+=$Kp%CaT#2Ssw159n*U2`RL*A7~Qt_`bzRlgYnzn8DiiEwWjy)4+zqJhmg39
zi_B!Yu>2RC14krB=8pCDY<zHJm^UqxRTB@t6pucgYn_yibiH<ULe(4Je%=MtoT}B7
zM+;P5>LON#v6+6fWo7{BgzP+Yc6}Bu<#MMvaic!%2=cy#j1@yRU-itoU->bSjk?(B
zxUaSWx2LD_eV~((74Y4DD-HF@nt1VWYgUxKP5}5427wjZ>-fFCaQOie*Th#PO>b77
z%})d{WBb8vK~E=}aP;eRHTd>q=NxvtmRm{rx&?tyO<wMBtP-~c;U?4MT`Pn|x>38W
zu86OSgFsBYe&E*D=PUo$(F4hj=ZV&0<<+);%N5wm08L)sKE(9<%ipI-f}swD+=XQq
zFk000Vg)APJAalf{a97(B%0^f2#HA?Z1?k7kyMu`7=fQite<7uzl@QAXulNheR5=n
zf1QQcvS&k#ZAeq!=QRE^KiZhRoZ9y>{68$A6#NG%{2ImoFT}V9e2s7r+tr)?kk>l1
z``uVE!%OWE1ck+X+R}UL_5+-@giWU1k}>92Q=`H0uNLZV^1;dnr;?tV_{QVkv;qkI
zL)5JH@RjrfJn6kc`%aNZb3+xWo`dW;vcN3LNEGLPfJtD=LZtfo84=r|IK$@^sDf04
zq+<dFmxX6>!0F*sA;*_g636G>{~LfG*dpvv>b}zZ!J%MHFni0N>`Kw?&vMvH%IbG6
zHi{=v>UVqbsl`Pa91yQX_df1YSHgkl81dekpUNTC&4C-S66lx3fL~od&+mRJJw<u~
z{U<e;Jt3sOFine$ZVJh=fk=9<Jw$4^rWcGksD<VEJqUYYXK(ZXb*4`LNl|)`qN!kr
zaek`|ijMEWuKS2`8^Ya31!GiBmmlS1A2t+Etsyjf?~J2>A`P|*2+1t&?TPJ#rM6~V
zsn75I-@?MyhqfBNuJ7~S!23X!D}ukB%t-*PyX(TBqVtx-kH&1M0J<&3-N5beb@a|H
zi^qXI*?Pu^#N+3R{OL7#n(M9u@FV<RHQ*5srrY=E6?aeG;!C+5hLOkHM@)Q`xGkBm
zMj+ht?tXeg;A0?#Y!{w;vs?rQ94HJGkuMhRDfU2RufkUYhx?0XA-7l3{uIHq$}yNE
zS|V@KO>~F>c=GW}qQYPKf0a-zlng(UYt4ixO_fPBk>~f!?uB{q>jAv+=j@{N5F^UK
z__ApCmUcOUAK#KI89k8U`>QRtIe6J;ioNB2C!rTIjn#tsyRP3=K}CGuOR1W{XEizl
zj?@oo<Ntfg4%ofarc2(V4p(8Sm|xT8W>Ftn<XPxqpZ&f3;~&0U((u=RSpHIz>K2Sp
zf>@R@D>JPY)#ey#(b3T5okuq8zE4Z*V4lFrWQ#V7FuP&sx!c9M|AGc%iF6ayIZmV!
zj$O9zI4Notww}d~m}CQ8f^KHD-WN8cDC&yx+Q?Elcppf6thZ5&2)~3`%Jtu2qX?-j
zokOoRZyx3x`K}6bunku)3OZECCLFc#f_lO>js7y=YgALBA0TYk8>yGl`I%@Ie9JG%
z*;FlU(l%$1;K%^G{`OwAEx$cpFvAho{2C&MOBtIX<f|spZ&4jbc=aSt8g<U|HRd}-
z06c-m&5ZeldM&@YzB))T>Qc|&Tcf1yz7-#~I?Ps8p3ju@T}9t+bx`CTlPTYg5&06X
z%H&T$W^pYY)Ncxi1k_~x!ouN8;4MMfe{lB5D5*9bt)uh541*o0z(@fhws}$d^OvdJ
zt-2q{<=;K=mnp<xDlr+kP<fT{<xt>kv}cFk!1={zoeYwGQA$<}d~-3&G0R$4fR?4N
zx`F#$_-&fH5U+_-1rnQM`#Av{TBv?UfVZlQEK8Nx=RwhT4yN_Z%p71hG@f4^iv)Ec
z51&4Tr*d1%YM@3Y2L{DOz=@B5ObqLC%cL*FeI>)1gzxlj@cLxJ_dT--%DLY;S69DA
zdJ{#8b^9-0tKob0^^6u53gNnn?%@~R+l!-w-7@++TR5vxl(pOos$_s65%+@G#%|8>
zVW~Wp&++*1;k;^;44%;FcRqBV$svGniY&+DCdyT3Fa=L{g9;ONzO&wv{p@k3nXzfa
z-I<5?gJ4*K|32Q`j1^}=I#(sz*wAA5LA@*PD&t4N99KL4j%$Plpj7?BR{m(Al?)~~
z+<U8v=qEYh9={z&zvfVx>F!IJ5SX|mYywjf=1BJ$Q3;bOz_TkGF*W`xDCQOxzxc~<
z-}VG?(PbM7HOz&k)~~8Yr-pU0t+ll<<>#m9yUD=Otb_j|gH2(k0bh=Pp)nu~E9~Up
zc`0p3B|cIapDN60HB7$k#|0WaDz{9rY_0-JHL(gy`h}Kcb+G)eCXm^l^1FLz$j5cb
z5yL%AG>*YiZBvB1%6^oIxPOx!wysdP;z+-yZ}@vpBE1je8U#+0Gd<<Q{<A{0<v%j<
z!2ob<5D@RaYLYyuG3*R>*vcW8h0FAvI8l!OCx;lD?W#OF<j(8(?8}~BNShIyn4~)r
zTm!eki8YYkomB$>LQ-?~w`sb}YB=<y8f;UsH=4J;`h;Om-djHg+*Qj_U+@xk8>;KR
z{o9sVA8S@gBMRWr8S>dV6>}TR>eBy)(RwUIm;Ix2^UkUzL9o9@iIl8!6GSP}d_I~<
zzmLt>#uH3`Y^pD)BGHARo$Dh*?|KJ|I6o;kGq37tk}c#=pixt5i`vyrV4t)!1N(I@
zZ~e>qbDQ=b`=+mT1p76SQU7Vw^tW+w;dY|D65L5|4M<Qpn8{9G>V6VVbkEffak)uu
z_J7%jWMvK5FSB$!U#=X#28>5s<{i=p`M-$!D)!8MxS0qvecbH~Unx)05wBOOZJC$k
z*#h-LjBSgsL5ve&OdvXrh@b4>d^EXdLK)!=JxyaCb{0_1z&iA=Amv5!xLe40QvQ3h
zqMaNZTv-&%wl7UuwR0>gK$R<<qekYA9t{?3taCoYfa$^>p>M-dwMCIaI*L6?7?oEv
zBK17OS%$sb-*zZ#g!58=X%~}{=(hhZ)yrYzDfMPAE5BP9QOq(la?JyR)%~5(qd4(t
z!q!B9I`CYG(dgMNCWMEl&*rDOr+&FS^$%9I-XUj>NMMB!NR!BTKfbNIJrz!606NHz
zgY6$O7Uj^=Bl3epS+>33MxNxO5}oOCJ7MHy_458*o~D>H4h#!Yb_X`CLEjFrYPSCo
zBn~uoU#?Bd)RFfx{SgE%Jd7LsGzE2bauxlEpi7tAXq<X=^W!A^RD82qfB_BZH6G^{
z?nIgDxWR$mmpa*24G9}M)^=!?toJO?fQLSnvV!&&{S-&8z9EWj>nPh%sgs848|-AV
z^f=<V+{-PK$e-gu)+oW54RfDXJdr+tmp!KF1hW^No#U-jt5?Ar6cp`!f(eG&x<I5B
zcp@M_M+T<^8|G)Az8PQc-4mFPOIul5tUWQC*_&{w<cNuyHfH|AwtF7XtcCdVsAv`y
z0zZr%(W=2}swn>r?_iMpi;VQ<*kG~+{u$`{_$4Rr<rH;-Jc8HU^w=YMm347JokmRM
zOMKJwQ~nn(M6jl(a9D{@9#{R2NcHp_h2*&x-d&HNdj*EW`|WO<0xiGs-qV6UD#!D|
zY&_o6?39rn2Xq?+rifC8<9<+W#*?Tk%75Dt5Ep6I#K{H?iXZ`Wo^??li9G{2e$QSg
zKq3Hq*@xUR9s>UDRY(L25kqCR-@?n5`UCcnF{+%wMoU%U3?L{!9PO4yiXh-K5t4?q
zJU9MF8m8=@=k(b>xyQ4d>B)x!j#1*<ysZ}D@bL8JcL&seHPifJqM0yJ#HGU847vqa
zYbUi|)l5I8K8G$A$1XUeIjA`Z9CT~*v9L6XF5Unt{gD@Ur<FwkhJkZJmER3goR1h%
z0B0d*{3bO`@_q|%Gmt_wyhDf!mG)gUJV8U9h+ti6k%k0vIe45R<Lt22Rs6v%|8y-O
z1&ZEY>F0B=P_k!OOGV>@j-%U<93M*uEx|_oRaShaeF#9^!9nbHqrZkjz_pCGhyLu1
z@HUyWaKh%$W}-TS;KGc)y9L=%B@BWBu&v`Y5wV`RWo3K$Z-)vcafjbxyX!V3x|lsa
z*wZ^4X5*pX1MM$~<-2`z*Z6A5=axWWL0RrVUVW!hBsSBY9TLv)KEcSAKK-FMo7UPp
z`9<OLgMt?Qx#jN$5&OX@60RuN@*WEMT`sKmk{349H;AOfM#JoaptVhw!~2qJFeFiE
z2O4KwabUSXpXQ6#Sl%F|I&EYi32VP6825?6khHj>3IcBYxxGi09LRs{$yU+}d~*f<
z)5b8A6*kWEzRF}55r<G>(gTK!vx~Zx32y~6K1Jyrfzzm-t0n|f8spCD@HZO2mosoV
zow`Yu{3Sx%@h-yIg`0{D408rgetgrhTrK{R3BUVC;w1q0CVH|mPr_kRwjgv{{53Y?
z$j;?aa!!+<Fq0U3Z>PU6eBV^b)oUxpkroN7&u2qvM=8+cXDad*;ug6*F~=31f1Z7g
z!~!@739dqoqTu~3NPKGW(HP*YDg_VyR6jNS)M#5BJQa-K@TCV9m@ntkI9omnC5E-F
zGT|kpXPw7A=h8?CUX-ye96v*{#%SfQal%NsYon~q$akQtRqELMCX{4OLXtGdGGC-i
z{KTVA=SpV((Ajr6n3Q1ghHp9xbKdMm0>!iYZ#$~~{!<}pBZ|snU0srdJ#DEwCxpcd
zcm|f7(<cD<`vbhes&cZzRh00AWOsf3zpm{|D+^7+EJmr_OX%l5K8#4)YuY4DgLhd`
zKEf-olbsbi%EAYE&$x4L5d{Dpu&4a8>aL6xfLZWnq~n;MRx|Zh2j0TVpXbZcr}su;
zY5nNws9#WdtggDUrMG06y%pU0aQmebA@kLm6ULMS&hG2>dd7jt-n)Om)dzPx>>M1j
zorL#BM+y+`r1n0_43CkhFzR8XzvAK&2;3LnmGnuVQ9#p{a#wxg@Qf7A-v!w)v9wJ)
zSxu<OQt-0bM2(Veo(ZOl*rB_w*2vrps(ij9ddlv0{48LnH;DVA#-F*TdB!<%l6X*W
zkow0t82|R8P8FF93#&4GsYQqHrVC9fg#m^+LeLg+9V#nn_*FSy{duj-4P=yGTRPh$
zA@Jk@4fsc%zSU-6Q<gixjEl58K8L7Q+4n5PW%`mr>Roe99em#eKc_F6JD?IrWc`i#
zLC!^%p}k*T<j{=oUDN@VVLONBu#)>gF&A*hOKL(E*$T_gb0HWH^1net`XC!LyC2%<
zGJ-@dva08&raVam9ihjC7De6hKZt4-e9K3Gkuz%5vK|yJmEWjvrs#9S^qX@J=&9}W
zFU#0E#8MoEL3<Wo-N_eviA5Ny%OY^Ccp>HyLRF63A>_+-7~o=1=06M=Z_rQqx(M)w
z`B<^3z=A2!vlyi_D4CO-L&)vSQX6lLvT9*tI84_l6C1`=NlwE5GcU&Q?aCR6zN<=R
zZG7EYEh{=WU>1MZN`APoyI9u*7b8HoR>`L<K0512nRVNIHqd%Vs;Gf&9%XUQ@eNJj
zi{N@O(tgR6{f36IMkdB3QQtXQ_c(YbD)^Gjl5{)CbU#VrqePR$QKovBD5D|=)yr9!
zh5aR@;~v{?cy*sBizwa1ZN1g*ZJrb_Doq@y=~}wF?%r|@6u9?QcATxS2#_>U0eIZu
zU1Irbi|-t+%qe(g%DaDdm6Kp+TYvR`CNq%&@5N4u<0b#QRiZkWfLB6>F;dF;YF-49
zUhToB2)~2;)djS`n@(-#158ws_E2oLu)Bm*`V9FMMm5iD&qaxQFS=?2JcVx-AF7=>
zW!e4HOawsuJxh1h_MGRbCPj6`Dgp+$=mKY^bs3h_MJcJK=&&-)iT)^4?|lx564Lms
zT38wW5&Rnv2o}qY)$XqPJ#}R{`yIZmU>;CM+bF1Wqb~Rx$Srt*Kd*aX$k|mt%;$l2
zaq+3zS%SOo%SAw+T^}Q_@m=nZI9BNyO}GDhK0nY#%-Xk4@0&g#QKOpWYIOG|u>Qc6
zyb?m*T;GY>%c<2K?!aacSYKy8*8Qm9Pec3)78^A#2DX>0e+lRTEXmyO#QY`b_)!p7
z{<kOUm^iOw@dwawxd~5~AY{SiBBr!GGj8tkH&R)CcY)Q>C)IJ8eF|V@Tw03m1W$zC
z!=(Y@?;~Zm+$nBc3eZBz0Kwb7-cq^%L(edCGtVAT1#TxpZ%AJ0!(ry&_A}_ll!5je
zn&d0Zc~OR3or>UGEh4vYGQ$BA2T+KIpBu<0$j{9N<Y5K^nOQKoRNZalEI--OzLyi>
z0rCL<-zmv{2m1Kj!nE&oYz=5t9DQwRd4SwP$!iBB2*A4A19DVZZUG(vVPT*=FOVB3
zC(OskFT^J+$15xSL5PoAKuADPf>!MRZ%VSxxiEMPA3>Tl9Coru)9IXUYgmK~V?aW~
zN0&joSRhO(!|tU;aWjzv{spMFo0_Z6zWIPX7{!ystto@jseq^Mo|HpDs^*5MLa(93
z=wnlREsjEdobHEO|9yW<6iwj!;$kaW*DKDi3DEA67V-3;k`apyeUZ+x`Lo-wvWNZA
nsvF+ez)H?YW**Yr*E^&LcGALR$_r{tVSa8*M#c~7@|gb*FSyuw

diff --git a/doc/joss/figures/ginkgo-hierarchy.tex b/doc/joss/figures/ginkgo-hierarchy.tex
index eadc3affdb2..d72b7839e7d 100644
--- a/doc/joss/figures/ginkgo-hierarchy.tex
+++ b/doc/joss/figures/ginkgo-hierarchy.tex
@@ -350,7 +350,11 @@
         {
             \textbf{CudaExecutor}
         };
-    \node (hip) [abstract, rectangle, below=of cuda]
+    \node (dpcpp) [abstract, rectangle, below=of cuda]
+        {
+          \textbf{DpcppExecutor}
+        };
+    \node (hip) [abstract, rectangle, below=of dpcpp]
         {
           \textbf{HipExecutor}
         };
@@ -460,6 +464,7 @@
 
 
     \draw[myarrow] (cuda.west) -- ++(-0.5,0) -- ++(0,0.6) -| (exec.south);
+    \draw[line] (dpcpp.west) -- ++(-0.5,0) -- ([xshift=-0.5cm] dpcpp.west);
     \draw[line] (hip.west) -- ++(-0.5,0) -- ([xshift=-0.5cm] cuda.west);
     \draw[line] (omp.west) -- ++(-0.5,0) -- ([xshift=-0.5cm] cuda.west);
     \draw[line] (ref.west) -- ++(-0.5,0) -- ([xshift=-0.5cm] cuda.west);
diff --git a/doc/joss/paper.md b/doc/joss/paper.md
index ad764e7f8a5..499874e8fec 100644
--- a/doc/joss/paper.md
+++ b/doc/joss/paper.md
@@ -53,13 +53,13 @@ Ginkgo is a production-ready sparse linear algebra library for high
 performance computing on GPU-centric architectures with a high level of
 performance portability and focuses on software sustainability. 
 
-The library focuses on solving sparse linear systems and accommodates a large variety
-of matrix formats, state-of-the-art iterative (Krylov) solvers and preconditioners, 
-which make the library suitable for a variety of scientific applications. Ginkgo
-supports many architectures such as multi-threaded CPU, NVIDIA GPUs, and AMD GPUs.
-The heavy use of modern C++ features simplifies the addition of new executor
-paradigms and algorithmic functionality without introducing significant
-performance overhead. 
+The library focuses on solving sparse linear systems and accommodates a large
+variety of matrix formats, state-of-the-art iterative (Krylov) solvers and
+preconditioners, which make the library suitable for a variety of scientific
+applications. Ginkgo supports many architectures such as multi-threaded CPU,
+NVIDIA GPUs, AMD GPUs, and Intel CPU/GPUs. The heavy use of modern C++ features
+simplifies the addition of new executor paradigms and algorithmic functionality
+without introducing significant performance overhead.
 
 Solving linear systems is usually one of the most computationally
 and memory intensive aspects of any application. Hence there has been a significant

From d2f702de0654180b092afdfc2f3ac31350e1e34f Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Thu, 15 Oct 2020 19:09:42 +0200
Subject: [PATCH 0089/1008] Fix the gtest issue with MSVC.

`using T = x;` is banned inside any gtest class used in a `TYPED_TEST`
fashion because gtest also defines a `T` of its own and somehow `MSVC`
confuses the two.
---
 core/test/utils.hpp                      |  3 +++
 include/ginkgo/core/base/executor.hpp    |  2 +-
 include/ginkgo/core/base/lin_op.hpp      | 16 ++++++++--------
 include/ginkgo/core/matrix/csr.hpp       | 16 ++++++++--------
 include/ginkgo/core/stop/criterion.hpp   |  4 ++--
 reference/test/matrix/ell_kernels.cpp    |  9 ++++-----
 reference/test/matrix/hybrid_kernels.cpp |  9 ++++-----
 third_party/gtest/CMakeLists.txt         |  4 ++--
 8 files changed, 32 insertions(+), 31 deletions(-)

diff --git a/core/test/utils.hpp b/core/test/utils.hpp
index 89b135a01f3..d6ae92bb951 100644
--- a/core/test/utils.hpp
+++ b/core/test/utils.hpp
@@ -39,6 +39,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <type_traits>
 
 
+#include <gtest/gtest.h>
+
+
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
 
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index ec8628d64de..3a2cf9bdf09 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -569,7 +569,7 @@ class Executor : public log::EnableLogging<Executor> {
         try {
             this->raw_copy_from(src_exec, num_elems * sizeof(T), src_ptr,
                                 dest_ptr);
-        } catch (NotSupported &err) {
+        } catch (NotSupported &) {
             // Unoptimized copy. Try to go through the masters.
             auto src_master = src_exec->get_master().get();
             if (num_elems > 0 && src_master != src_exec) {
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index 7ce4cef3e31..a1530ff2880 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -821,12 +821,12 @@ using EnableDefaultLinOpFactory =
  *
  * @ingroup LinOp
  */
-#define GKO_CREATE_FACTORY_PARAMETERS(_parameters_name, _factory_name) \
-public:                                                                \
-    class _factory_name;                                               \
-    struct _parameters_name##_type                                     \
-        : ::gko::enable_parameters_type<_parameters_name##_type,       \
-                                        _factory_name>
+#define GKO_CREATE_FACTORY_PARAMETERS(_parameters_name, _factory_name)  \
+public:                                                                 \
+    class _factory_name;                                                \
+    struct _parameters_name##_type                                      \
+        : public ::gko::enable_parameters_type<_parameters_name##_type, \
+                                               _factory_name>
 
 
 /**
@@ -916,8 +916,8 @@ public:                                                                      \
                                                   _parameters_name##_type> { \
         friend class ::gko::EnablePolymorphicObject<_factory_name,           \
                                                     ::gko::LinOpFactory>;    \
-        friend class ::gko::enable_parameters_type<_parameters_name##_type,  \
-                                                   _factory_name>;           \
+        friend struct ::gko::enable_parameters_type<_parameters_name##_type, \
+                                                    _factory_name>;          \
         explicit _factory_name(std::shared_ptr<const ::gko::Executor> exec)  \
             : ::gko::EnableDefaultLinOpFactory<_factory_name, _lin_op,       \
                                                _parameters_name##_type>(     \
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index c0f6d914465..f6e486e9f5d 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -439,22 +439,22 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
         {
             if (warp_size_ > 0) {
                 int multiple = 8;
-                if (nnz >= 2e8) {
+                if (nnz >= static_cast<int64_t>(2e8)) {
                     multiple = 2048;
-                } else if (nnz >= 2e7) {
+                } else if (nnz >= static_cast<int64_t>(2e7)) {
                     multiple = 512;
-                } else if (nnz >= 2e6) {
+                } else if (nnz >= static_cast<int64_t>(2e6)) {
                     multiple = 128;
-                } else if (nnz >= 2e5) {
+                } else if (nnz >= static_cast<int64_t>(2e5)) {
                     multiple = 32;
                 }
 
 #if GINKGO_HIP_PLATFORM_HCC
                 if (!cuda_strategy_) {
                     multiple = 8;
-                    if (nnz >= 1e7) {
+                    if (nnz >= static_cast<int64_t>(1e7)) {
                         multiple = 64;
-                    } else if (nnz >= 1e6) {
+                    } else if (nnz >= static_cast<int64_t>(1e6)) {
                         multiple = 16;
                     }
                 }
@@ -486,13 +486,13 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
         const index_type nvidia_row_len_limit = 1024;
         /* Use imbalance strategy when the matrix has more more than 1e6 on
          * NVIDIA hardware */
-        const index_type nvidia_nnz_limit = 1e6;
+        const index_type nvidia_nnz_limit{static_cast<index_type>(1e6)};
         /* Use imbalance strategy when the maximum number of nonzero per row is
          * more than 768 on AMD hardware */
         const index_type amd_row_len_limit = 768;
         /* Use imbalance strategy when the matrix has more more than 1e8 on AMD
          * hardware */
-        const index_type amd_nnz_limit = 1e8;
+        const index_type amd_nnz_limit{static_cast<index_type>(1e8)};
 
         /**
          * Creates an automatical strategy.
diff --git a/include/ginkgo/core/stop/criterion.hpp b/include/ginkgo/core/stop/criterion.hpp
index 35c28aefdd9..41d48b2d9ae 100644
--- a/include/ginkgo/core/stop/criterion.hpp
+++ b/include/ginkgo/core/stop/criterion.hpp
@@ -297,8 +297,8 @@ public:                                                                      \
               _factory_name, _criterion, _parameters_name##_type> {          \
         friend class ::gko::EnablePolymorphicObject<                         \
             _factory_name, ::gko::stop::CriterionFactory>;                   \
-        friend class ::gko::enable_parameters_type<_parameters_name##_type,  \
-                                                   _factory_name>;           \
+        friend struct ::gko::enable_parameters_type<_parameters_name##_type, \
+                                                    _factory_name>;          \
         explicit _factory_name(std::shared_ptr<const ::gko::Executor> exec)  \
             : ::gko::stop::EnableDefaultCriterionFactory<                    \
                   _factory_name, _criterion, _parameters_name##_type>(       \
diff --git a/reference/test/matrix/ell_kernels.cpp b/reference/test/matrix/ell_kernels.cpp
index a16f3c5b125..03e0b6417d4 100644
--- a/reference/test/matrix/ell_kernels.cpp
+++ b/reference/test/matrix/ell_kernels.cpp
@@ -60,7 +60,6 @@ class Ell : public ::testing::Test {
         typename std::tuple_element<0, decltype(ValueIndexType())>::type;
     using index_type =
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
-    using T = value_type;
     using Mtx = gko::matrix::Ell<value_type, index_type>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
     using Vec = gko::matrix::Dense<value_type>;
@@ -94,10 +93,10 @@ class Ell : public ::testing::Test {
         EXPECT_EQ(c[1], 1);
         EXPECT_EQ(c[2], 2);
         EXPECT_EQ(c[3], 1);
-        EXPECT_EQ(v[0], T{1.0});
-        EXPECT_EQ(v[1], T{3.0});
-        EXPECT_EQ(v[2], T{2.0});
-        EXPECT_EQ(v[3], T{5.0});
+        EXPECT_EQ(v[0], value_type{1.0});
+        EXPECT_EQ(v[1], value_type{3.0});
+        EXPECT_EQ(v[2], value_type{2.0});
+        EXPECT_EQ(v[3], value_type{5.0});
     }
 
     std::shared_ptr<const gko::Executor> exec;
diff --git a/reference/test/matrix/hybrid_kernels.cpp b/reference/test/matrix/hybrid_kernels.cpp
index c261eebaec3..da8a5dded83 100644
--- a/reference/test/matrix/hybrid_kernels.cpp
+++ b/reference/test/matrix/hybrid_kernels.cpp
@@ -61,7 +61,6 @@ class Hybrid : public ::testing::Test {
         typename std::tuple_element<0, decltype(ValueIndexType())>::type;
     using index_type =
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
-    using T = value_type;
     using Mtx = gko::matrix::Hybrid<value_type, index_type>;
     using Vec = gko::matrix::Dense<value_type>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
@@ -119,10 +118,10 @@ class Hybrid : public ::testing::Test {
         EXPECT_EQ(c[1], 1);
         EXPECT_EQ(c[2], 2);
         EXPECT_EQ(c[3], 1);
-        EXPECT_EQ(v[0], T{1.0});
-        EXPECT_EQ(v[1], T{3.0});
-        EXPECT_EQ(v[2], T{2.0});
-        EXPECT_EQ(v[3], T{5.0});
+        EXPECT_EQ(v[0], value_type{1.0});
+        EXPECT_EQ(v[1], value_type{3.0});
+        EXPECT_EQ(v[2], value_type{2.0});
+        EXPECT_EQ(v[3], value_type{5.0});
     }
 
     std::shared_ptr<const gko::ReferenceExecutor> exec;
diff --git a/third_party/gtest/CMakeLists.txt b/third_party/gtest/CMakeLists.txt
index f68db6d2060..9d13c970cc7 100644
--- a/third_party/gtest/CMakeLists.txt
+++ b/third_party/gtest/CMakeLists.txt
@@ -2,13 +2,13 @@ if(MSVC)
     # Force using shared runtime library when MSVC builds shared libraries
     ginkgo_load_git_package(gtest_external
         "https://github.com/google/googletest.git"
-        "ee3aa831172090fd5442820f215cb04ab6062756"
+        "6a7ed316a5cdc07b6d26362c90770787513822d4"
         # Work around the linking errors when compiling gtest with CUDA
         "-Dgtest_disable_pthreads=ON" "-Dgtest_force_shared_crt=${BUILD_SHARED_LIBS}")
 else()
     ginkgo_load_git_package(gtest_external
         "https://github.com/google/googletest.git"
-        "ee3aa831172090fd5442820f215cb04ab6062756"
+        "6a7ed316a5cdc07b6d26362c90770787513822d4"
         # Work around the linking errors when compiling gtest with CUDA
         "-Dgtest_disable_pthreads=ON"
         "-DCMAKE_CXX_FLAGS=-fPIC")

From 1f0dcb21daa38263d29ff38779c8c6938829d638 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Mon, 19 Oct 2020 10:00:02 +0200
Subject: [PATCH 0090/1008] Add a CPU DPCPP job.

---
 .gitlab-ci.yml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3275706d2f9..8f5f3818c0d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -52,6 +52,7 @@ stages:
       CUDA_ARCH_STR=-DGINKGO_CUDA_ARCHITECTURES=${CUDA_ARCH};
       CUDA_HOST_STR=-DCMAKE_CUDA_HOST_COMPILER=$(which ${CXX_COMPILER});
       fi
+    - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then export SYCL_DEVICE_TYPE; fi
     - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
         -GNinja
         -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${CXX_COMPILER}
@@ -65,6 +66,7 @@ stages:
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
     - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT}
+    - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then unset SYCL_DEVICE_TYPE; fi
   dependencies: []
   except:
       - schedules
@@ -79,6 +81,7 @@ stages:
       CUDA_ARCH_STR=-DGINKGO_CUDA_ARCHITECTURES=${CUDA_ARCH};
       CUDA_HOST_STR=-DCMAKE_CUDA_HOST_COMPILER=$(which ${CXX_COMPILER});
       fi
+    - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then export SYCL_DEVICE_TYPE; fi
     - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
         -GNinja
         -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${CXX_COMPILER}
@@ -117,6 +120,7 @@ stages:
             ninja validate_all_examples
         fi
       fi
+    - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then unset SYCL_DEVICE_TYPE; fi
   dependencies: []
   except:
       - schedules
@@ -660,6 +664,24 @@ build/nocuda/intel/omp/release/static:
     - cuda
     - cpu
 
+build/dpcpp/cpu/release/static:
+  <<: *default_build_with_test
+  image: localhost:5000/gko-oneapi
+  variables:
+    <<: *default_variables
+    C_COMPILER: "gcc"
+    CXX_COMPILER: "dpcpp"
+    BUILD_DPCPP: "ON"
+    BUILD_TYPE: "Release"
+    BUILD_SHARED_LIBS: "OFF"
+    SYCL_DEVICE_TYPE: "CPU"
+  only:
+    variables:
+      - $RUN_CI_TAG
+  tags:
+    - private_ci
+    - cuda
+    - cpu
 
 # Job with important warnings as error
 warnings:

From 030e0bcef58564b09ccf65685fbd4f053691a7ef Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Mon, 26 Oct 2020 10:10:51 +0100
Subject: [PATCH 0091/1008] Review updates.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

+ Fix some documentation issues.
+ Fix some spacing issues.
+ Do not duplicate `GKO_DECLARE_EXECUTOR_FRIEND` macro.

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 CMakeLists.txt                        |  3 ++-
 core/device_hooks/dpcpp_hooks.cpp     |  3 +++
 core/test/base/executor.cpp           | 11 +++++++++++
 doc/examples/examples.hpp.in          |  2 +-
 include/ginkgo/core/base/executor.hpp | 11 +++--------
 5 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index dd1c32c23f4..5c2016d90bf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -27,7 +27,8 @@ option(GINKGO_BUILD_EXAMPLES "Build Ginkgo's examples" ON)
 option(GINKGO_BUILD_BENCHMARKS "Build Ginkgo's benchmarks" ON)
 option(GINKGO_BUILD_REFERENCE "Compile reference CPU kernels" ON)
 option(GINKGO_BUILD_OMP "Compile OpenMP kernels for CPU" ${GINKGO_HAS_OMP})
-option(GINKGO_BUILD_DPCPP "Compile DPC++ kernels for CPU" ${GINKGO_HAS_DPCPP})
+option(GINKGO_BUILD_DPCPP
+    "Compile DPC++ kernels for Intel GPUs or other DPC++ enabled hardware" ${GINKGO_HAS_DPCPP})
 option(GINKGO_BUILD_CUDA "Compile kernels for NVIDIA GPUs" ${GINKGO_HAS_CUDA})
 option(GINKGO_BUILD_HIP "Compile kernels for AMD or NVIDIA GPUs" ${GINKGO_HAS_HIP})
 option(GINKGO_BUILD_DOC "Generate documentation" OFF)
diff --git a/core/device_hooks/dpcpp_hooks.cpp b/core/device_hooks/dpcpp_hooks.cpp
index 56d61760fed..cce8fdec27e 100644
--- a/core/device_hooks/dpcpp_hooks.cpp
+++ b/core/device_hooks/dpcpp_hooks.cpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace gko {
 
+
 version version_info::get_dpcpp_version() noexcept
 {
     // We just return the version with a special "not compiled" tag in
@@ -85,10 +86,12 @@ void DpcppExecutor::raw_copy_to(const CudaExecutor *, size_type num_bytes,
                                 const void *src_ptr, void *dest_ptr) const
     GKO_NOT_COMPILED(dpcpp);
 
+
 void DpcppExecutor::raw_copy_to(const HipExecutor *, size_type num_bytes,
                                 const void *src_ptr, void *dest_ptr) const
     GKO_NOT_COMPILED(dpcpp);
 
+
 void DpcppExecutor::raw_copy_to(const DpcppExecutor *, size_type num_bytes,
                                 const void *src_ptr, void *dest_ptr) const
     GKO_NOT_COMPILED(dpcpp);
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index f43e375cf19..b968bbe210a 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -82,6 +82,7 @@ TEST(OmpExecutor, RunsCorrectOperation)
     exec_ptr omp = gko::OmpExecutor::create();
 
     omp->run(ExampleOperation(value));
+
     ASSERT_EQ(1, value);
 }
 
@@ -96,6 +97,7 @@ TEST(OmpExecutor, RunsCorrectLambdaOperation)
     exec_ptr omp = gko::OmpExecutor::create();
 
     omp->run(omp_lambda, cuda_lambda, hip_lambda, dpcpp_lambda);
+
     ASSERT_EQ(1, value);
 }
 
@@ -160,6 +162,7 @@ TEST(ReferenceExecutor, RunsCorrectOperation)
     exec_ptr ref = gko::ReferenceExecutor::create();
 
     ref->run(ExampleOperation(value));
+
     ASSERT_EQ(5, value);
 }
 
@@ -174,6 +177,7 @@ TEST(ReferenceExecutor, RunsCorrectLambdaOperation)
     exec_ptr ref = gko::ReferenceExecutor::create();
 
     ref->run(omp_lambda, cuda_lambda, hip_lambda, dpcpp_lambda);
+
     ASSERT_EQ(1, value);
 }
 
@@ -285,6 +289,7 @@ TEST(CudaExecutor, RunsCorrectOperation)
         gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
 
     cuda->run(ExampleOperation(value));
+
     ASSERT_EQ(2, value);
 }
 
@@ -300,6 +305,7 @@ TEST(CudaExecutor, RunsCorrectLambdaOperation)
         gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true);
 
     cuda->run(omp_lambda, cuda_lambda, hip_lambda, dpcpp_lambda);
+
     ASSERT_EQ(2, value);
 }
 
@@ -357,6 +363,7 @@ TEST(HipExecutor, RunsCorrectOperation)
     exec_ptr hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
 
     hip->run(ExampleOperation(value));
+
     ASSERT_EQ(3, value);
 }
 
@@ -371,6 +378,7 @@ TEST(HipExecutor, RunsCorrectLambdaOperation)
     exec_ptr hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
 
     hip->run(omp_lambda, cuda_lambda, hip_lambda, dpcpp_lambda);
+
     ASSERT_EQ(3, value);
 }
 
@@ -428,6 +436,7 @@ TEST(DpcppExecutor, RunsCorrectOperation)
     exec_ptr dpcpp = gko::DpcppExecutor::create(0, gko::OmpExecutor::create());
 
     dpcpp->run(ExampleOperation(value));
+
     ASSERT_EQ(4, value);
 }
 
@@ -442,6 +451,7 @@ TEST(DpcppExecutor, RunsCorrectLambdaOperation)
     exec_ptr dpcpp = gko::DpcppExecutor::create(0, gko::OmpExecutor::create());
 
     dpcpp->run(omp_lambda, cuda_lambda, hip_lambda, dpcpp_lambda);
+
     ASSERT_EQ(4, value);
 }
 
@@ -498,6 +508,7 @@ TEST(ExecutorDeleter, DeletesObject)
 TEST(ExecutorDeleter, AvoidsDeletionForNullExecutor)
 {
     int x[5];
+
     ASSERT_NO_THROW(gko::executor_deleter<int>{nullptr}(x));
 }
 
diff --git a/doc/examples/examples.hpp.in b/doc/examples/examples.hpp.in
index ce5b5add69a..5290b4ecaf6 100644
--- a/doc/examples/examples.hpp.in
+++ b/doc/examples/examples.hpp.in
@@ -64,7 +64,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  * <ol>
  *   <li> <code>-DGINKGO_BUILD_CUDA=ON</code> option for NVIDIA GPUs.
  *   <li> <code>-DGINKGO_BUILD_HIP=ON</code> option for AMD or NVIDIA GPUs.
- *   <li> <code>-DGINKGO_BUILD_SYCL=ON</code> option for Intel GPUs (and
+ *   <li> <code>-DGINKGO_BUILD_DPCPP=ON</code> option for Intel GPUs (and
  *        possibly any other platform).
  * </ol
  *
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 3a2cf9bdf09..950e7e1b2ed 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -374,6 +374,8 @@ private:                                                                     \
                   "semi-colon warnings")
 
 
+#define GKO_DECLARE_EXECUTOR_FRIEND(_type, ...) friend class _type
+
 /**
  * The first step in using the Ginkgo library consists of creating an
  * executor. Executors are used to specify the location for the data of linear
@@ -465,12 +467,8 @@ class Executor : public log::EnableLogging<Executor> {
     template <typename T>
     friend class detail::ExecutorBase;
 
-#define GKO_DECLARE_EXECUTOR_FRIEND(_type, ...) friend class _type
-
     GKO_ENABLE_FOR_ALL_EXECUTORS(GKO_DECLARE_EXECUTOR_FRIEND);
 
-#undef GKO_DECLARE_EXECUTOR_FRIEND
-
 public:
     virtual ~Executor() = default;
 
@@ -817,12 +815,8 @@ namespace detail {
 
 template <typename ConcreteExecutor>
 class ExecutorBase : public Executor {
-#define GKO_DECLARE_EXECUTOR_FRIEND(_type, ...) friend class _type
-
     GKO_ENABLE_FOR_ALL_EXECUTORS(GKO_DECLARE_EXECUTOR_FRIEND);
 
-#undef GKO_DECLARE_EXECUTOR_FRIEND
-
 public:
     void run(const Operation &op) const override
     {
@@ -850,6 +844,7 @@ class ExecutorBase : public Executor {
     }
 };
 
+#undef GKO_DECLARE_EXECUTOR_FRIEND
 
 /**
  * Controls whether the DeviceReset function should be called thanks to a

From b0b83c67a8eb0eead7ca0892b76b58204c3acf61 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Mon, 26 Oct 2020 10:41:11 +0100
Subject: [PATCH 0092/1008] Fix defines and header scripts.

Co-authored-by: Yuhsiang M. Tsai <yhmtsai@gmail.com>
---
 dev_tools/scripts/config                  | 6 +++---
 dev_tools/scripts/format_header.sh        | 4 ++--
 dev_tools/scripts/regroup                 | 2 +-
 dpcpp/components/csr_spgeam.dp.hpp        | 6 +++---
 dpcpp/components/matrix_operations.dp.hpp | 6 +++---
 5 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/dev_tools/scripts/config b/dev_tools/scripts/config
index 768f3693327..ba29fed413b 100644
--- a/dev_tools/scripts/config
+++ b/dev_tools/scripts/config
@@ -4,9 +4,9 @@
     - FixInclude: "ginkgo/core/base/executor.hpp"
 - "hip/base/config.hip.hpp"
     - FixInclude: "hip/hip_runtime.h"
-- "(cuda|hip|omp)/test/factorization/par_ilu_kernels"
+- "(cuda|hip|omp|dpcpp)/test/factorization/par_ilu_kernels"
     - FixInclude: "core/factorization/par_ilu_kernels.hpp"
-- "(cuda|hip)/preconditioner/jacobi_"
+- "(cuda|hip|dpcpp)/preconditioner/jacobi_"
     - FixInclude: "core/preconditioner/jacobi_kernels.hpp"
 - "core/test/base/(extended_float|iterator_factory)"
     - RemoveTest: "true"
@@ -14,7 +14,7 @@
     - RemoveTest: "true"
 - "_builder\.hpp"
     - CoreSuffix: "_builder"
-- "components.*_kernels(\.hip)?\.(cu|cpp|hpp|cuh)"
+- "components.*_kernels(\.hip|\.dp)?\.(cu|cpp|hpp|cuh)"
     - CoreSuffix: "_kernels"
     - RemoveTest: "true"
 - "components"
diff --git a/dev_tools/scripts/format_header.sh b/dev_tools/scripts/format_header.sh
index 21b92419ccd..c250af38a85 100755
--- a/dev_tools/scripts/format_header.sh
+++ b/dev_tools/scripts/format_header.sh
@@ -104,9 +104,9 @@ get_include_regex () {
         if [ ! -z "${path_prefix}" ]; then
             path_prefix="${path_prefix}/"
         fi
-        local_output=$(echo "${file}" | sed -E "s~\.hip~~g;s~$path_regex~$path_prefix\2~g")
+        local_output=$(echo "${file}" | sed -E "s~\.(hip|dp)~~g;s~$path_regex~$path_prefix\2~g")
         local_output=$(echo "${local_output}" | sed -E "s~$core_suffix$~~g")
-        local_output="#include (<|\")$local_output\.(hpp|hip\.hpp|cuh)(\"|>)"
+        local_output="#include (<|\")$local_output\.(hpp|hip\.hpp|dp\.hpp|cuh)(\"|>)"
         if [ "${remove_test}" = "true" ]; then
             local_output=$(echo "${local_output}" | sed -E "s~test/~~g")
         fi
diff --git a/dev_tools/scripts/regroup b/dev_tools/scripts/regroup
index 036d5d81588..5351dc7009e 100644
--- a/dev_tools/scripts/regroup
+++ b/dev_tools/scripts/regroup
@@ -2,7 +2,7 @@ IncludeBlocks: Regroup
 IncludeCategories:
   - Regex: '^<(rapidjson|gflags|gtest|papi).*'
     Priority: 3
-  - Regex: '^<(omp|cu|hip|thrust).*'
+  - Regex: '^<(omp|cu|hip|thrust|CL).*'
     Priority: 2
   - Regex: '^<ginkgo.*'
     Priority: 5
diff --git a/dpcpp/components/csr_spgeam.dp.hpp b/dpcpp/components/csr_spgeam.dp.hpp
index 77303d7dd7a..013ee10bbf8 100644
--- a/dpcpp/components/csr_spgeam.dp.hpp
+++ b/dpcpp/components/csr_spgeam.dp.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_DPCPP_COMPONENTS_CSR_SPGEAM_HPP_
-#define GKO_DPCPP_COMPONENTS_CSR_SPGEAM_HPP_
+#ifndef GKO_DPCPP_COMPONENTS_CSR_SPGEAM_DP_HPP_
+#define GKO_DPCPP_COMPONENTS_CSR_SPGEAM_DP_HPP_
 
 
 #include <limits>
@@ -73,4 +73,4 @@ void abstract_spgeam(const matrix::Csr<ValueType, IndexType> *a,
 }  // namespace gko
 
 
-#endif  // GKO_DPCPP_COMPONENTS_CSR_SPGEAM_HPP_
+#endif  // GKO_DPCPP_COMPONENTS_CSR_SPGEAM_DP_HPP_
diff --git a/dpcpp/components/matrix_operations.dp.hpp b/dpcpp/components/matrix_operations.dp.hpp
index 4efb2d839f2..d4cbcd245b8 100644
--- a/dpcpp/components/matrix_operations.dp.hpp
+++ b/dpcpp/components/matrix_operations.dp.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_DPCPP_COMPONENTS_MATRIX_OPERATIONS_HPP_
-#define GKO_DPCPP_COMPONENTS_MATRIX_OPERATIONS_HPP_
+#ifndef GKO_DPCPP_COMPONENTS_MATRIX_OPERATIONS_DP_HPP_
+#define GKO_DPCPP_COMPONENTS_MATRIX_OPERATIONS_DP_HPP_
 
 
 #include <ginkgo/core/base/math.hpp>
@@ -62,4 +62,4 @@ remove_complex<ValueType> compute_inf_norm(size_type num_rows,
 }  // namespace gko
 
 
-#endif  // GKO_DPCPP_COMPONENTS_MATRIX_OPERATIONS_HPP_
+#endif  // GKO_DPCPP_COMPONENTS_MATRIX_OPERATIONS_DP_HPP_

From 7d11b91d2314e7a2e3d3a00c16a38f0e75cd80af Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Mon, 26 Oct 2020 16:09:41 +0100
Subject: [PATCH 0093/1008] Review updates.

+ Add some environment variable output to `get_info.cmake`.
+ Fix formatting of `GKO_NOT_IMPLEMENTED` declarations.
+ Allow lowercase in the `DPC++` strings passed to executors.
+ Fix kernels storing SYCL `id` into `int` types.
+ Add some extra comments to tricky parts of the code.
+ Improve some other code formatting issues.

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
Co-authored-by: Yuhsiang M. Tsai <yhmtsai@gmail.com>
---
 cmake/get_info.cmake                          |  15 ++
 core/test/reorder/rcm.cpp                     |   2 +-
 dpcpp/base/executor.dp.cpp                    |  18 +-
 dpcpp/components/absolute_array.dp.cpp        |   4 +-
 dpcpp/components/csr_spgeam.dp.hpp            |  76 --------
 dpcpp/components/fill_array.dp.cpp            |   2 +-
 dpcpp/components/format_conversion.dp.hpp     |  16 +-
 dpcpp/components/matrix_operations.dp.hpp     |  10 +-
 dpcpp/components/precision_conversion.dp.cpp  |   2 +-
 dpcpp/components/prefix_sum.dp.cpp            |   6 +-
 .../factorization_kernels.dp.cpp              |  40 ++---
 dpcpp/factorization/par_ict_kernels.dp.cpp    |   5 -
 dpcpp/factorization/par_ilu_kernels.dp.cpp    |  13 +-
 dpcpp/factorization/par_ilut_kernels.dp.cpp   |  36 ++--
 dpcpp/get_info.cmake                          |   3 +
 dpcpp/matrix/coo_kernels.dp.cpp               |  40 ++---
 dpcpp/matrix/csr_kernels.dp.cpp               | 165 ++++--------------
 dpcpp/matrix/dense_kernels.dp.cpp             | 113 +++---------
 dpcpp/matrix/diagonal_kernels.dp.cpp          |  24 +--
 dpcpp/matrix/ell_kernels.dp.cpp               |  26 +--
 dpcpp/matrix/hybrid_kernels.dp.cpp            |  12 +-
 dpcpp/matrix/sellp_kernels.dp.cpp             |  21 +--
 dpcpp/matrix/sparsity_csr_kernels.dp.cpp      |  48 ++---
 dpcpp/preconditioner/isai_kernels.dp.cpp      |  44 ++---
 dpcpp/preconditioner/jacobi_kernels.dp.cpp    | 139 +++++----------
 dpcpp/solver/bicg_kernels.dp.cpp              |  15 +-
 dpcpp/solver/bicgstab_kernels.dp.cpp          |  26 +--
 dpcpp/solver/cg_kernels.dp.cpp                |  15 +-
 dpcpp/solver/cgs_kernels.dp.cpp               |  20 +--
 dpcpp/solver/fcg_kernels.dp.cpp               |  15 +-
 dpcpp/solver/gmres_kernels.dp.cpp             |  53 ++----
 dpcpp/solver/ir_kernels.dp.cpp                |   5 +-
 dpcpp/solver/lower_trs_kernels.dp.cpp         |  11 +-
 dpcpp/solver/upper_trs_kernels.dp.cpp         |   6 +-
 dpcpp/stop/criterion_kernels.dp.cpp           |   2 +-
 dpcpp/stop/residual_norm_kernels.dp.cpp       |   5 +-
 include/ginkgo/core/base/math.hpp             |   5 +
 .../ginkgo/core/reorder/reordering_base.hpp   |   2 +-
 omp/get_info.cmake                            |   2 +
 omp/test/reorder/rcm_kernels.cpp              |   2 +-
 reference/test/reorder/rcm.cpp                |   2 +-
 41 files changed, 281 insertions(+), 785 deletions(-)
 delete mode 100644 dpcpp/components/csr_spgeam.dp.hpp

diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index 94f898a24c4..dcaf07f334b 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -72,6 +72,21 @@ function(ginkgo_print_variable log_type var_name)
     FILE(APPEND ${log_type} "${upd_string}")
 endfunction()
 
+
+function(ginkgo_print_env_variable log_type var_name)
+    string(SUBSTRING
+        "
+--        ${var_name}:                                                          " 0 55 upd_string)
+    if(NOT ENV{${var_name}})
+        set(str_value "<empty>")
+    else()
+        set(str_value "$ENV{${var_name}}")
+    endif()
+    string(APPEND upd_string "${str_value}")
+    FILE(APPEND ${log_type} "${upd_string}")
+endfunction()
+
+
 macro(ginkgo_print_foreach_variable variables)
     foreach(var ${variables})
         ginkgo_print_variable(${${log_type}} ${var} )
diff --git a/core/test/reorder/rcm.cpp b/core/test/reorder/rcm.cpp
index 030db177fa3..4d05bb0574f 100644
--- a/core/test/reorder/rcm.cpp
+++ b/core/test/reorder/rcm.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2019, the Ginkgo authors
+Copyright (c) 2017-2020, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 208d7255fb3..7e311d489c7 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -33,6 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
+#include <algorithm>
+#include <cctype>
 #include <iostream>
 #include <map>
 #include <string>
@@ -56,6 +58,8 @@ const std::vector<sycl::device> get_devices(std::string device_type)
         {"all", sycl::info::device_type::all},
         {"cpu", sycl::info::device_type::cpu},
         {"gpu", sycl::info::device_type::gpu}};
+    std::for_each(device_type.begin(), device_type.end(),
+                  [](char &c) { c = std::tolower(c); });
     return sycl::device::get_devices(device_type_map.at(device_type));
 }
 
@@ -123,7 +127,6 @@ void DpcppExecutor::raw_copy_to(const DpcppExecutor *dest, size_type num_bytes,
                                 const void *src_ptr, void *dest_ptr) const
 {
     if (num_bytes > 0) {
-        // TODO: does this work? Or is it needed to go through host?
         dest->get_queue()->memcpy(dest_ptr, src_ptr, num_bytes).wait();
     }
 }
@@ -147,6 +150,9 @@ int DpcppExecutor::get_num_devices(std::string device_type)
 }
 
 
+namespace detail {
+
+
 void delete_queue(sycl::queue *queue)
 {
     queue->wait();
@@ -154,6 +160,9 @@ void delete_queue(sycl::queue *queue)
 }
 
 
+}  // namespace detail
+
+
 void DpcppExecutor::set_device_property()
 {
     assert(device_id_ < DpcppExecutor::get_num_devices(device_type_));
@@ -174,8 +183,13 @@ void DpcppExecutor::set_device_property()
     }
     max_workgroup_size_ =
         device.get_info<sycl::info::device::max_work_group_size>();
+    // Here we declare the queue with the property `in_order` which ensures the
+    // kernels are executed in the submission order. Otherwise, calls to
+    // `wait()` would be needed after every call to a DPC++ function or kernel.
+    // For example, without `in_order`, doing a copy, a kernel, and a copy, will
+    // not necessarily happen in that order by default, which we need to avoid.
     auto *queue = new sycl::queue{device, sycl::property::queue::in_order{}};
-    queue_ = std::move(queue_manager<sycl::queue>{queue, delete_queue});
+    queue_ = std::move(queue_manager<sycl::queue>{queue, detail::delete_queue});
 }
 
 
diff --git a/dpcpp/components/absolute_array.dp.cpp b/dpcpp/components/absolute_array.dp.cpp
index 9603dacbc8b..830752de343 100644
--- a/dpcpp/components/absolute_array.dp.cpp
+++ b/dpcpp/components/absolute_array.dp.cpp
@@ -51,7 +51,7 @@ void inplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec,
 {
     exec->get_queue()->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl::range<1>{n}, [=](sycl::id<1> idx_id) {
-            const int idx = idx_id[0];
+            const auto idx = idx_id[0];
             data[idx] = dpcpp::abs(data[idx]);
         });
     });
@@ -67,7 +67,7 @@ void outplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec,
 {
     exec->get_queue()->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl::range<1>{n}, [=](sycl::id<1> idx_id) {
-            const int idx = idx_id[0];
+            const auto idx = idx_id[0];
             out[idx] = dpcpp::abs(in[idx]);
         });
     });
diff --git a/dpcpp/components/csr_spgeam.dp.hpp b/dpcpp/components/csr_spgeam.dp.hpp
deleted file mode 100644
index 013ee10bbf8..00000000000
--- a/dpcpp/components/csr_spgeam.dp.hpp
+++ /dev/null
@@ -1,76 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GKO_DPCPP_COMPONENTS_CSR_SPGEAM_DP_HPP_
-#define GKO_DPCPP_COMPONENTS_CSR_SPGEAM_DP_HPP_
-
-
-#include <limits>
-
-
-#include <ginkgo/core/matrix/csr.hpp>
-
-
-#include "core/base/utils.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace dpcpp {
-
-
-/**
- * Adds two (sorted) sparse matrices.
- *
- * Calls begin_cb(row) on each row to initialize row-local data
- * Calls entry_cb(row, col, a_val, b_val, local_data) on each output non-zero
- * Calls end_cb(row, local_data) on each row to finalize row-local data
- *
- * If the three functions are thread-safe, the whole invocation is.
- */
-template <typename ValueType, typename IndexType, typename BeginCallback,
-          typename EntryCallback, typename EndCallback>
-void abstract_spgeam(const matrix::Csr<ValueType, IndexType> *a,
-                     const matrix::Csr<ValueType, IndexType> *b,
-                     BeginCallback begin_cb, EntryCallback entry_cb,
-                     EndCallback end_cb)
-{
-    GKO_NOT_IMPLEMENTED;
-}
-
-
-}  // namespace dpcpp
-}  // namespace kernels
-}  // namespace gko
-
-
-#endif  // GKO_DPCPP_COMPONENTS_CSR_SPGEAM_DP_HPP_
diff --git a/dpcpp/components/fill_array.dp.cpp b/dpcpp/components/fill_array.dp.cpp
index f8c260cdab6..98328484be2 100644
--- a/dpcpp/components/fill_array.dp.cpp
+++ b/dpcpp/components/fill_array.dp.cpp
@@ -48,7 +48,7 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
 {
     exec->get_queue()->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl::range<1>{n}, [=](sycl::id<1> idx_id) {
-            const int idx = idx_id[0];
+            const auto idx = idx_id[0];
             array[idx] = val;
         });
     });
diff --git a/dpcpp/components/format_conversion.dp.hpp b/dpcpp/components/format_conversion.dp.hpp
index f1dba5eed78..0b603e7bcf2 100644
--- a/dpcpp/components/format_conversion.dp.hpp
+++ b/dpcpp/components/format_conversion.dp.hpp
@@ -56,10 +56,8 @@ namespace dpcpp {
 template <typename IndexType>
 inline void convert_unsorted_idxs_to_ptrs(const IndexType *idxs,
                                           size_type num_nonzeros,
-                                          IndexType *ptrs, size_type length)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                                          IndexType *ptrs,
+                                          size_type length) GKO_NOT_IMPLEMENTED;
 
 
 /**
@@ -72,18 +70,12 @@ inline void convert_unsorted_idxs_to_ptrs(const IndexType *idxs,
 template <typename IndexType>
 inline void convert_sorted_idxs_to_ptrs(const IndexType *idxs,
                                         size_type num_nonzeros, IndexType *ptrs,
-                                        size_type length)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                                        size_type length) GKO_NOT_IMPLEMENTED;
 
 
 template <typename IndexType>
 inline void convert_ptrs_to_idxs(const IndexType *ptrs, size_type num_rows,
-                                 IndexType *idxs)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                                 IndexType *idxs) GKO_NOT_IMPLEMENTED;
 
 
 }  // namespace dpcpp
diff --git a/dpcpp/components/matrix_operations.dp.hpp b/dpcpp/components/matrix_operations.dp.hpp
index d4cbcd245b8..7c39735f485 100644
--- a/dpcpp/components/matrix_operations.dp.hpp
+++ b/dpcpp/components/matrix_operations.dp.hpp
@@ -48,13 +48,9 @@ namespace dpcpp {
  * Computes the infinity norm of a column-major matrix.
  */
 template <typename ValueType>
-remove_complex<ValueType> compute_inf_norm(size_type num_rows,
-                                           size_type num_cols,
-                                           const ValueType *matrix,
-                                           size_type stride)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+remove_complex<ValueType> compute_inf_norm(
+    size_type num_rows, size_type num_cols, const ValueType *matrix,
+    size_type stride) GKO_NOT_IMPLEMENTED;
 
 
 }  // namespace dpcpp
diff --git a/dpcpp/components/precision_conversion.dp.cpp b/dpcpp/components/precision_conversion.dp.cpp
index c508e853ad6..41f9c66833a 100644
--- a/dpcpp/components/precision_conversion.dp.cpp
+++ b/dpcpp/components/precision_conversion.dp.cpp
@@ -48,7 +48,7 @@ void convert_precision(std::shared_ptr<const DefaultExecutor> exec,
 {
     exec->get_queue()->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl::range<1>{size}, [=](sycl::id<1> idx_id) {
-            const int idx = idx_id[0];
+            const auto idx = idx_id[0];
             out[idx] = in[idx];
         });
     });
diff --git a/dpcpp/components/prefix_sum.dp.cpp b/dpcpp/components/prefix_sum.dp.cpp
index 21ca279b696..6740a7e966c 100644
--- a/dpcpp/components/prefix_sum.dp.cpp
+++ b/dpcpp/components/prefix_sum.dp.cpp
@@ -47,13 +47,9 @@ namespace components {
 
 template <typename IndexType>
 void prefix_sum(std::shared_ptr<const DefaultExecutor> exec, IndexType *counts,
-                size_type num_entries)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                size_type num_entries) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_PREFIX_SUM_KERNEL);
-
 // instantiate for size_type as well, as this is used in the Sellp format
 template GKO_DECLARE_PREFIX_SUM_KERNEL(size_type);
 
diff --git a/dpcpp/factorization/factorization_kernels.dp.cpp b/dpcpp/factorization/factorization_kernels.dp.cpp
index 74386b58376..5e33d368733 100644
--- a/dpcpp/factorization/factorization_kernels.dp.cpp
+++ b/dpcpp/factorization/factorization_kernels.dp.cpp
@@ -59,29 +59,21 @@ namespace factorization {
 template <bool IsSorted, typename ValueType, typename IndexType>
 void find_missing_diagonal_elements(
     const matrix::Csr<ValueType, IndexType> *mtx,
-    IndexType *elements_to_add_per_row, bool *changes_required)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    IndexType *elements_to_add_per_row,
+    bool *changes_required) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
-void add_missing_diagonal_elements(const matrix::Csr<ValueType, IndexType> *mtx,
-                                   ValueType *new_values,
-                                   IndexType *new_col_idxs,
-                                   const IndexType *row_ptrs_addition)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+void add_missing_diagonal_elements(
+    const matrix::Csr<ValueType, IndexType> *mtx, ValueType *new_values,
+    IndexType *new_col_idxs,
+    const IndexType *row_ptrs_addition) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 void add_diagonal_elements(std::shared_ptr<const DpcppExecutor> exec,
                            matrix::Csr<ValueType, IndexType> *mtx,
-                           bool is_sorted)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                           bool is_sorted) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FACTORIZATION_ADD_DIAGONAL_ELEMENTS_KERNEL);
@@ -91,10 +83,7 @@ template <typename ValueType, typename IndexType>
 void initialize_row_ptrs_l_u(
     std::shared_ptr<const DpcppExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *system_matrix,
-    IndexType *l_row_ptrs, IndexType *u_row_ptrs)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    IndexType *l_row_ptrs, IndexType *u_row_ptrs) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FACTORIZATION_INITIALIZE_ROW_PTRS_L_U_KERNEL);
@@ -105,9 +94,7 @@ void initialize_l_u(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Csr<ValueType, IndexType> *system_matrix,
                     matrix::Csr<ValueType, IndexType> *csr_l,
                     matrix::Csr<ValueType, IndexType> *csr_u)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FACTORIZATION_INITIALIZE_L_U_KERNEL);
@@ -117,10 +104,7 @@ template <typename ValueType, typename IndexType>
 void initialize_row_ptrs_l(
     std::shared_ptr<const DpcppExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *system_matrix,
-    IndexType *l_row_ptrs)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    IndexType *l_row_ptrs) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FACTORIZATION_INITIALIZE_ROW_PTRS_L_KERNEL);
@@ -129,10 +113,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void initialize_l(std::shared_ptr<const DpcppExecutor> exec,
                   const matrix::Csr<ValueType, IndexType> *system_matrix,
-                  matrix::Csr<ValueType, IndexType> *csr_l, bool diag_sqrt)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                  matrix::Csr<ValueType, IndexType> *csr_l,
+                  bool diag_sqrt) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FACTORIZATION_INITIALIZE_L_KERNEL);
diff --git a/dpcpp/factorization/par_ict_kernels.dp.cpp b/dpcpp/factorization/par_ict_kernels.dp.cpp
index 6e7dd7e5563..c99d4e813cd 100644
--- a/dpcpp/factorization/par_ict_kernels.dp.cpp
+++ b/dpcpp/factorization/par_ict_kernels.dp.cpp
@@ -48,7 +48,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/base/utils.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/csr_builder.hpp"
-#include "dpcpp/components/csr_spgeam.dp.hpp"
 
 
 namespace gko {
@@ -67,9 +66,7 @@ void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
                     const matrix::Csr<ValueType, IndexType> *a,
                     matrix::Csr<ValueType, IndexType> *l,
                     const matrix::Coo<ValueType, IndexType> *)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_PAR_ICT_COMPUTE_FACTOR_KERNEL);
@@ -81,9 +78,7 @@ void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
                     const matrix::Csr<ValueType, IndexType> *a,
                     const matrix::Csr<ValueType, IndexType> *l,
                     matrix::Csr<ValueType, IndexType> *l_new)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_PAR_ICT_ADD_CANDIDATES_KERNEL);
diff --git a/dpcpp/factorization/par_ilu_kernels.dp.cpp b/dpcpp/factorization/par_ilu_kernels.dp.cpp
index baa8b4a4481..cd1ae44ce37 100644
--- a/dpcpp/factorization/par_ilu_kernels.dp.cpp
+++ b/dpcpp/factorization/par_ilu_kernels.dp.cpp
@@ -53,14 +53,11 @@ namespace par_ilu_factorization {
 
 
 template <typename ValueType, typename IndexType>
-void compute_l_u_factors(std::shared_ptr<const DpcppExecutor> exec,
-                         size_type iterations,
-                         const matrix::Coo<ValueType, IndexType> *system_matrix,
-                         matrix::Csr<ValueType, IndexType> *l_factor,
-                         matrix::Csr<ValueType, IndexType> *u_factor)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+void compute_l_u_factors(
+    std::shared_ptr<const DpcppExecutor> exec, size_type iterations,
+    const matrix::Coo<ValueType, IndexType> *system_matrix,
+    matrix::Csr<ValueType, IndexType> *l_factor,
+    matrix::Csr<ValueType, IndexType> *u_factor) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_PAR_ILU_COMPUTE_L_U_FACTORS_KERNEL);
diff --git a/dpcpp/factorization/par_ilut_kernels.dp.cpp b/dpcpp/factorization/par_ilut_kernels.dp.cpp
index 6b7f5b0ac17..80b4e11c219 100644
--- a/dpcpp/factorization/par_ilut_kernels.dp.cpp
+++ b/dpcpp/factorization/par_ilut_kernels.dp.cpp
@@ -52,7 +52,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/coo_builder.hpp"
 #include "core/matrix/csr_builder.hpp"
-#include "dpcpp/components/csr_spgeam.dp.hpp"
 
 
 namespace gko {
@@ -71,10 +70,7 @@ void threshold_select(std::shared_ptr<const DefaultExecutor> exec,
                       const matrix::Csr<ValueType, IndexType> *m,
                       IndexType rank, Array<ValueType> &tmp,
                       Array<remove_complex<ValueType>> &,
-                      remove_complex<ValueType> &threshold)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                      remove_complex<ValueType> &threshold) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_PAR_ILUT_THRESHOLD_SELECT_KERNEL);
@@ -91,10 +87,7 @@ void abstract_filter(std::shared_ptr<const DefaultExecutor> exec,
                      const matrix::Csr<ValueType, IndexType> *m,
                      matrix::Csr<ValueType, IndexType> *m_out,
                      matrix::Coo<ValueType, IndexType> *m_out_coo,
-                     Predicate pred)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                     Predicate pred) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
@@ -102,10 +95,8 @@ void threshold_filter(std::shared_ptr<const DefaultExecutor> exec,
                       const matrix::Csr<ValueType, IndexType> *m,
                       remove_complex<ValueType> threshold,
                       matrix::Csr<ValueType, IndexType> *m_out,
-                      matrix::Coo<ValueType, IndexType> *m_out_coo, bool)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                      matrix::Coo<ValueType, IndexType> *m_out_coo,
+                      bool) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_PAR_ILUT_THRESHOLD_FILTER_KERNEL);
@@ -116,15 +107,12 @@ constexpr auto sample_size = bucket_count * sampleselect_oversampling;
 
 
 template <typename ValueType, typename IndexType>
-void threshold_filter_approx(std::shared_ptr<const DefaultExecutor> exec,
-                             const matrix::Csr<ValueType, IndexType> *m,
-                             IndexType rank, Array<ValueType> &tmp,
-                             remove_complex<ValueType> &threshold,
-                             matrix::Csr<ValueType, IndexType> *m_out,
-                             matrix::Coo<ValueType, IndexType> *m_out_coo)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+void threshold_filter_approx(
+    std::shared_ptr<const DefaultExecutor> exec,
+    const matrix::Csr<ValueType, IndexType> *m, IndexType rank,
+    Array<ValueType> &tmp, remove_complex<ValueType> &threshold,
+    matrix::Csr<ValueType, IndexType> *m_out,
+    matrix::Coo<ValueType, IndexType> *m_out_coo) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_PAR_ILUT_THRESHOLD_FILTER_APPROX_KERNEL);
@@ -138,9 +126,7 @@ void compute_l_u_factors(std::shared_ptr<const DefaultExecutor> exec,
                          matrix::Csr<ValueType, IndexType> *u,
                          const matrix::Coo<ValueType, IndexType> *,
                          matrix::Csr<ValueType, IndexType> *u_csc)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_PAR_ILUT_COMPUTE_LU_FACTORS_KERNEL);
@@ -154,9 +140,7 @@ void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
                     const matrix::Csr<ValueType, IndexType> *u,
                     matrix::Csr<ValueType, IndexType> *l_new,
                     matrix::Csr<ValueType, IndexType> *u_new)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_PAR_ILUT_ADD_CANDIDATES_KERNEL);
diff --git a/dpcpp/get_info.cmake b/dpcpp/get_info.cmake
index 475a53f5dcf..7cbfa34f7d0 100644
--- a/dpcpp/get_info.cmake
+++ b/dpcpp/get_info.cmake
@@ -1,4 +1,7 @@
 ginkgo_print_module_header(${detailed_log} "DPCPP")
 ginkgo_print_module_footer(${detailed_log} "DPCPP variables:")
 ginkgo_print_variable(${detailed_log} "GINKGO_DPCPP_FLAGS")
+ginkgo_print_module_footer(${detailed_log} "DPCPP environment variables:")
+ginkgo_print_env_variable(${detailed_log} "SYCL_DEVICE_TYPE")
+ginkgo_print_env_variable(${detailed_log} "SYCL_BE")
 ginkgo_print_module_footer(${detailed_log} "")
diff --git a/dpcpp/matrix/coo_kernels.dp.cpp b/dpcpp/matrix/coo_kernels.dp.cpp
index b72b24d1fa5..57f70edf499 100644
--- a/dpcpp/matrix/coo_kernels.dp.cpp
+++ b/dpcpp/matrix/coo_kernels.dp.cpp
@@ -64,10 +64,8 @@ namespace coo {
 template <typename ValueType, typename IndexType>
 void spmv(std::shared_ptr<const DpcppExecutor> exec,
           const matrix::Coo<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+          const matrix::Dense<ValueType> *b,
+          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_SPMV_KERNEL);
 
@@ -78,10 +76,7 @@ void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::Coo<ValueType, IndexType> *a,
                    const matrix::Dense<ValueType> *b,
                    const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_ADVANCED_SPMV_KERNEL);
@@ -90,10 +85,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void spmv2(std::shared_ptr<const DpcppExecutor> exec,
            const matrix::Coo<ValueType, IndexType> *a,
-           const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+           const matrix::Dense<ValueType> *b,
+           matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_SPMV2_KERNEL);
 
@@ -103,10 +96,7 @@ void advanced_spmv2(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Dense<ValueType> *alpha,
                     const matrix::Coo<ValueType, IndexType> *a,
                     const matrix::Dense<ValueType> *b,
-                    matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                    matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_ADVANCED_SPMV2_KERNEL);
@@ -115,19 +105,15 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename IndexType>
 void convert_row_idxs_to_ptrs(std::shared_ptr<const DpcppExecutor> exec,
                               const IndexType *idxs, size_type num_nonzeros,
-                              IndexType *ptrs, size_type length)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                              IndexType *ptrs,
+                              size_type length) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Coo<ValueType, IndexType> *source,
                     matrix::Csr<ValueType, IndexType> *result)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_CONVERT_TO_CSR_KERNEL);
@@ -136,10 +122,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Coo<ValueType, IndexType> *source,
-                      matrix::Dense<ValueType> *result)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_CONVERT_TO_DENSE_KERNEL);
@@ -148,10 +131,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Coo<ValueType, IndexType> *orig,
-                      matrix::Diagonal<ValueType> *diag)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_EXTRACT_DIAGONAL_KERNEL);
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index 3516b7efc04..7863539f7a2 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -53,7 +53,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/base/iterator_factory.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/csr_builder.hpp"
-#include "dpcpp/components/csr_spgeam.dp.hpp"
 #include "dpcpp/components/format_conversion.dp.hpp"
 
 
@@ -71,10 +70,8 @@ namespace csr {
 template <typename ValueType, typename IndexType>
 void spmv(std::shared_ptr<const DpcppExecutor> exec,
           const matrix::Csr<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+          const matrix::Dense<ValueType> *b,
+          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPMV_KERNEL);
 
@@ -85,10 +82,7 @@ void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::Csr<ValueType, IndexType> *a,
                    const matrix::Dense<ValueType> *b,
                    const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_ADVANCED_SPMV_KERNEL);
@@ -97,49 +91,34 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void spgemm_insert_row(unordered_set<IndexType> &cols,
                        const matrix::Csr<ValueType, IndexType> *c,
-                       size_type row)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                       size_type row) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 void spgemm_insert_row2(unordered_set<IndexType> &cols,
                         const matrix::Csr<ValueType, IndexType> *a,
                         const matrix::Csr<ValueType, IndexType> *b,
-                        size_type row)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                        size_type row) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 void spgemm_accumulate_row(map<IndexType, ValueType> &cols,
                            const matrix::Csr<ValueType, IndexType> *c,
-                           ValueType scale, size_type row)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                           ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 void spgemm_accumulate_row2(map<IndexType, ValueType> &cols,
                             const matrix::Csr<ValueType, IndexType> *a,
                             const matrix::Csr<ValueType, IndexType> *b,
-                            ValueType scale, size_type row)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                            ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 void spgemm(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Csr<ValueType, IndexType> *a,
             const matrix::Csr<ValueType, IndexType> *b,
-            matrix::Csr<ValueType, IndexType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            matrix::Csr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEMM_KERNEL);
 
@@ -151,10 +130,7 @@ void advanced_spgemm(std::shared_ptr<const DpcppExecutor> exec,
                      const matrix::Csr<ValueType, IndexType> *b,
                      const matrix::Dense<ValueType> *beta,
                      const matrix::Csr<ValueType, IndexType> *d,
-                     matrix::Csr<ValueType, IndexType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                     matrix::Csr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_ADVANCED_SPGEMM_KERNEL);
@@ -166,10 +142,7 @@ void spgeam(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Csr<ValueType, IndexType> *a,
             const matrix::Dense<ValueType> *beta,
             const matrix::Csr<ValueType, IndexType> *b,
-            matrix::Csr<ValueType, IndexType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            matrix::Csr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEAM_KERNEL);
 
@@ -177,19 +150,14 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEAM_KERNEL);
 template <typename IndexType>
 void convert_row_ptrs_to_idxs(std::shared_ptr<const DpcppExecutor> exec,
                               const IndexType *ptrs, size_type num_rows,
-                              IndexType *idxs)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                              IndexType *idxs) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 void convert_to_coo(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Csr<ValueType, IndexType> *source,
                     matrix::Coo<ValueType, IndexType> *result)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONVERT_TO_COO_KERNEL);
@@ -198,10 +166,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Csr<ValueType, IndexType> *source,
-                      matrix::Dense<ValueType> *result)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONVERT_TO_DENSE_KERNEL);
@@ -232,29 +197,20 @@ inline void convert_csr_to_csc(size_type num_rows, const IndexType *row_ptrs,
                                const IndexType *col_idxs,
                                const ValueType *csr_vals, IndexType *row_idxs,
                                IndexType *col_ptrs, ValueType *csc_vals,
-                               UnaryOperator op)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                               UnaryOperator op) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType, typename UnaryOperator>
 void transpose_and_transform(std::shared_ptr<const DpcppExecutor> exec,
                              matrix::Csr<ValueType, IndexType> *trans,
                              const matrix::Csr<ValueType, IndexType> *orig,
-                             UnaryOperator op)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                             UnaryOperator op) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 void transpose(std::shared_ptr<const DpcppExecutor> exec,
                const matrix::Csr<ValueType, IndexType> *orig,
-               matrix::Csr<ValueType, IndexType> *trans)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+               matrix::Csr<ValueType, IndexType> *trans) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_TRANSPOSE_KERNEL);
 
@@ -263,9 +219,7 @@ template <typename ValueType, typename IndexType>
 void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Csr<ValueType, IndexType> *orig,
                     matrix::Csr<ValueType, IndexType> *trans)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONJ_TRANSPOSE_KERNEL);
@@ -294,80 +248,45 @@ template <typename ValueType, typename IndexType>
 void convert_to_hybrid(std::shared_ptr<const DpcppExecutor> exec,
                        const matrix::Csr<ValueType, IndexType> *source,
                        matrix::Hybrid<ValueType, IndexType> *result)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONVERT_TO_HYBRID_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void row_permute_impl(std::shared_ptr<const DpcppExecutor> exec,
-                      const Array<IndexType> *permutation_indices,
-                      const matrix::Csr<ValueType, IndexType> *orig,
-                      matrix::Csr<ValueType, IndexType> *row_permuted)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+template <typename IndexType>
+void invert_permutation(std::shared_ptr<const DpcppExecutor> exec,
+                        size_type size, const IndexType *permutation_indices,
+                        IndexType *inv_permutation) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void row_permute(std::shared_ptr<const DpcppExecutor> exec,
-                 const Array<IndexType> *permutation_indices,
-                 const matrix::Csr<ValueType, IndexType> *orig,
-                 matrix::Csr<ValueType, IndexType> *row_permuted)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+void row_permute(
+    std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
+    const matrix::Csr<ValueType, IndexType> *orig,
+    matrix::Csr<ValueType, IndexType> *row_permuted) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void inverse_row_permute(std::shared_ptr<const DpcppExecutor> exec,
-                         const Array<IndexType> *permutation_indices,
-                         const matrix::Csr<ValueType, IndexType> *orig,
-                         matrix::Csr<ValueType, IndexType> *row_permuted)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+void inverse_row_permute(
+    std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
+    const matrix::Csr<ValueType, IndexType> *orig,
+    matrix::Csr<ValueType, IndexType> *row_permuted) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void column_permute_impl(const Array<IndexType> *permutation_indices,
-                         const matrix::Csr<ValueType, IndexType> *orig,
-                         matrix::Csr<ValueType, IndexType> *column_permuted)
-{
-    GKO_NOT_IMPLEMENTED;
-}
-
-
-template <typename ValueType, typename IndexType>
-void column_permute(std::shared_ptr<const DpcppExecutor> exec,
-                    const Array<IndexType> *permutation_indices,
-                    const matrix::Csr<ValueType, IndexType> *orig,
-                    matrix::Csr<ValueType, IndexType> *column_permuted)
-{
-    GKO_NOT_IMPLEMENTED;
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_COLUMN_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(std::shared_ptr<const DpcppExecutor> exec,
-                            const Array<IndexType> *permutation_indices,
-                            const matrix::Csr<ValueType, IndexType> *orig,
-                            matrix::Csr<ValueType, IndexType> *column_permuted)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+void inverse_column_permute(
+    std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
+    const matrix::Csr<ValueType, IndexType> *orig,
+    matrix::Csr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
@@ -376,10 +295,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void calculate_nonzeros_per_row(std::shared_ptr<const DpcppExecutor> exec,
                                 const matrix::Csr<ValueType, IndexType> *source,
-                                Array<size_type> *result)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                                Array<size_type> *result) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
@@ -388,9 +304,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void sort_by_column_index(std::shared_ptr<const DpcppExecutor> exec,
                           matrix::Csr<ValueType, IndexType> *to_sort)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_SORT_BY_COLUMN_INDEX);
@@ -399,10 +313,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void is_sorted_by_column_index(
     std::shared_ptr<const DpcppExecutor> exec,
-    const matrix::Csr<ValueType, IndexType> *to_check, bool *is_sorted)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    const matrix::Csr<ValueType, IndexType> *to_check,
+    bool *is_sorted) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_IS_SORTED_BY_COLUMN_INDEX);
@@ -411,10 +323,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Csr<ValueType, IndexType> *orig,
-                      matrix::Diagonal<ValueType> *diag)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_EXTRACT_DIAGONAL);
 
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 5f76e1b5b97..e84519e8192 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -69,10 +69,7 @@ template <typename ValueType>
 void simple_apply(std::shared_ptr<const DpcppExecutor> exec,
                   const matrix::Dense<ValueType> *a,
                   const matrix::Dense<ValueType> *b,
-                  matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                  matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL);
 
@@ -81,20 +78,16 @@ template <typename ValueType>
 void apply(std::shared_ptr<const DpcppExecutor> exec,
            const matrix::Dense<ValueType> *alpha,
            const matrix::Dense<ValueType> *a, const matrix::Dense<ValueType> *b,
-           const matrix::Dense<ValueType> *beta, matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+           const matrix::Dense<ValueType> *beta,
+           matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
 template <typename ValueType>
 void scale(std::shared_ptr<const DpcppExecutor> exec,
-           const matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *x)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+           const matrix::Dense<ValueType> *alpha,
+           matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SCALE_KERNEL);
 
@@ -102,10 +95,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SCALE_KERNEL);
 template <typename ValueType>
 void add_scaled(std::shared_ptr<const DpcppExecutor> exec,
                 const matrix::Dense<ValueType> *alpha,
-                const matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *y)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                const matrix::Dense<ValueType> *x,
+                matrix::Dense<ValueType> *y) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_KERNEL);
 
@@ -114,10 +105,7 @@ template <typename ValueType>
 void add_scaled_diag(std::shared_ptr<const DpcppExecutor> exec,
                      const matrix::Dense<ValueType> *alpha,
                      const matrix::Diagonal<ValueType> *x,
-                     matrix::Dense<ValueType> *y)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                     matrix::Dense<ValueType> *y) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL);
 
@@ -126,10 +114,7 @@ template <typename ValueType>
 void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
                  const matrix::Dense<ValueType> *x,
                  const matrix::Dense<ValueType> *y,
-                 matrix::Dense<ValueType> *result)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                 matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL);
 
@@ -138,9 +123,7 @@ template <typename ValueType>
 void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::Dense<ValueType> *x,
                    matrix::Dense<remove_complex<ValueType>> *result)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_NORM2_KERNEL);
 
@@ -149,9 +132,7 @@ template <typename ValueType, typename IndexType>
 void convert_to_coo(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Dense<ValueType> *source,
                     matrix::Coo<ValueType, IndexType> *result)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DENSE_CONVERT_TO_COO_KERNEL);
@@ -161,9 +142,7 @@ template <typename ValueType, typename IndexType>
 void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Dense<ValueType> *source,
                     matrix::Csr<ValueType, IndexType> *result)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DENSE_CONVERT_TO_CSR_KERNEL);
@@ -173,9 +152,7 @@ template <typename ValueType, typename IndexType>
 void convert_to_ell(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Dense<ValueType> *source,
                     matrix::Ell<ValueType, IndexType> *result)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DENSE_CONVERT_TO_ELL_KERNEL);
@@ -185,9 +162,7 @@ template <typename ValueType, typename IndexType>
 void convert_to_hybrid(std::shared_ptr<const DpcppExecutor> exec,
                        const matrix::Dense<ValueType> *source,
                        matrix::Hybrid<ValueType, IndexType> *result)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DENSE_CONVERT_TO_HYBRID_KERNEL);
@@ -197,9 +172,7 @@ template <typename ValueType, typename IndexType>
 void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Dense<ValueType> *source,
                       matrix::Sellp<ValueType, IndexType> *result)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DENSE_CONVERT_TO_SELLP_KERNEL);
@@ -209,9 +182,7 @@ template <typename ValueType, typename IndexType>
 void convert_to_sparsity_csr(std::shared_ptr<const DpcppExecutor> exec,
                              const matrix::Dense<ValueType> *source,
                              matrix::SparsityCsr<ValueType, IndexType> *result)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DENSE_CONVERT_TO_SPARSITY_CSR_KERNEL);
@@ -219,10 +190,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType>
 void count_nonzeros(std::shared_ptr<const DpcppExecutor> exec,
-                    const matrix::Dense<ValueType> *source, size_type *result)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                    const matrix::Dense<ValueType> *source,
+                    size_type *result) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COUNT_NONZEROS_KERNEL);
 
@@ -230,10 +199,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COUNT_NONZEROS_KERNEL);
 template <typename ValueType>
 void calculate_max_nnz_per_row(std::shared_ptr<const DpcppExecutor> exec,
                                const matrix::Dense<ValueType> *source,
-                               size_type *result)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                               size_type *result) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_DENSE_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
@@ -242,10 +208,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
 template <typename ValueType>
 void calculate_nonzeros_per_row(std::shared_ptr<const DpcppExecutor> exec,
                                 const matrix::Dense<ValueType> *source,
-                                Array<size_type> *result)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                                Array<size_type> *result) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_DENSE_CALCULATE_NONZEROS_PER_ROW_KERNEL);
@@ -255,10 +218,7 @@ template <typename ValueType>
 void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
                           const matrix::Dense<ValueType> *source,
                           size_type *result, size_type stride_factor,
-                          size_type slice_size)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                          size_type slice_size) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_DENSE_CALCULATE_TOTAL_COLS_KERNEL);
@@ -267,35 +227,27 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
 template <typename ValueType>
 void transpose(std::shared_ptr<const DpcppExecutor> exec,
                const matrix::Dense<ValueType> *orig,
-               matrix::Dense<ValueType> *trans)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+               matrix::Dense<ValueType> *trans) GKO_NOT_IMPLEMENTED;
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_TRANSPOSE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType>
 void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Dense<ValueType> *orig,
-                    matrix::Dense<ValueType> *trans)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                    matrix::Dense<ValueType> *trans) GKO_NOT_IMPLEMENTED;
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CONJ_TRANSPOSE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
 void row_permute(std::shared_ptr<const DpcppExecutor> exec,
                  const Array<IndexType> *permutation_indices,
                  const matrix::Dense<ValueType> *orig,
-                 matrix::Dense<ValueType> *row_permuted)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                 matrix::Dense<ValueType> *row_permuted) GKO_NOT_IMPLEMENTED;
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ROW_PERMUTE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -303,12 +255,10 @@ void column_permute(std::shared_ptr<const DpcppExecutor> exec,
                     const Array<IndexType> *permutation_indices,
                     const matrix::Dense<ValueType> *orig,
                     matrix::Dense<ValueType> *column_permuted)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -316,12 +266,10 @@ void inverse_row_permute(std::shared_ptr<const DpcppExecutor> exec,
                          const Array<IndexType> *permutation_indices,
                          const matrix::Dense<ValueType> *orig,
                          matrix::Dense<ValueType> *row_permuted)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_INVERSE_ROW_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -329,31 +277,24 @@ void inverse_column_permute(std::shared_ptr<const DpcppExecutor> exec,
                             const Array<IndexType> *permutation_indices,
                             const matrix::Dense<ValueType> *orig,
                             matrix::Dense<ValueType> *column_permuted)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_INVERSE_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL);
 
 
 template <typename ValueType>
 void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Dense<ValueType> *orig,
-                      matrix::Diagonal<ValueType> *diag)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_EXTRACT_DIAGONAL_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL);
 
 
 template <typename ValueType>
 void inplace_absolute_dense(std::shared_ptr<const DpcppExecutor> exec,
                             matrix::Dense<ValueType> *source)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL);
 
@@ -362,9 +303,7 @@ template <typename ValueType>
 void outplace_absolute_dense(std::shared_ptr<const DpcppExecutor> exec,
                              const matrix::Dense<ValueType> *source,
                              matrix::Dense<remove_complex<ValueType>> *result)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
 
diff --git a/dpcpp/matrix/diagonal_kernels.dp.cpp b/dpcpp/matrix/diagonal_kernels.dp.cpp
index c2090a17dc4..8825daf65c7 100644
--- a/dpcpp/matrix/diagonal_kernels.dp.cpp
+++ b/dpcpp/matrix/diagonal_kernels.dp.cpp
@@ -55,10 +55,7 @@ template <typename ValueType>
 void apply_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Diagonal<ValueType> *a,
                     const matrix::Dense<ValueType> *b,
-                    matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                    matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_APPLY_TO_DENSE_KERNEL);
 
@@ -67,10 +64,7 @@ template <typename ValueType>
 void right_apply_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                           const matrix::Diagonal<ValueType> *a,
                           const matrix::Dense<ValueType> *b,
-                          matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_DENSE_KERNEL);
@@ -80,10 +74,7 @@ template <typename ValueType, typename IndexType>
 void apply_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                   const matrix::Diagonal<ValueType> *a,
                   const matrix::Csr<ValueType, IndexType> *b,
-                  matrix::Csr<ValueType, IndexType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                  matrix::Csr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DIAGONAL_APPLY_TO_CSR_KERNEL);
@@ -94,9 +85,7 @@ void right_apply_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                         const matrix::Diagonal<ValueType> *a,
                         const matrix::Csr<ValueType, IndexType> *b,
                         matrix::Csr<ValueType, IndexType> *c)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_CSR_KERNEL);
@@ -106,9 +95,7 @@ template <typename ValueType, typename IndexType>
 void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Diagonal<ValueType> *source,
                     matrix::Csr<ValueType, IndexType> *result)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DIAGONAL_CONVERT_TO_CSR_KERNEL);
@@ -117,10 +104,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType>
 void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Diagonal<ValueType> *orig,
-                    matrix::Diagonal<ValueType> *trans)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                    matrix::Diagonal<ValueType> *trans) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_CONJ_TRANSPOSE_KERNEL);
 
diff --git a/dpcpp/matrix/ell_kernels.dp.cpp b/dpcpp/matrix/ell_kernels.dp.cpp
index d27a19408b4..f0bb1dbf6f6 100644
--- a/dpcpp/matrix/ell_kernels.dp.cpp
+++ b/dpcpp/matrix/ell_kernels.dp.cpp
@@ -59,10 +59,8 @@ namespace ell {
 template <typename ValueType, typename IndexType>
 void spmv(std::shared_ptr<const DpcppExecutor> exec,
           const matrix::Ell<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+          const matrix::Dense<ValueType> *b,
+          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ELL_SPMV_KERNEL);
 
@@ -73,10 +71,7 @@ void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::Ell<ValueType, IndexType> *a,
                    const matrix::Dense<ValueType> *b,
                    const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL);
@@ -85,10 +80,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Ell<ValueType, IndexType> *source,
-                      matrix::Dense<ValueType> *result)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_CONVERT_TO_DENSE_KERNEL);
@@ -106,10 +98,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void count_nonzeros(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Ell<ValueType, IndexType> *source,
-                    size_type *result)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                    size_type *result) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_COUNT_NONZEROS_KERNEL);
@@ -127,10 +116,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Ell<ValueType, IndexType> *orig,
-                      matrix::Diagonal<ValueType> *diag)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_EXTRACT_DIAGONAL_KERNEL);
diff --git a/dpcpp/matrix/hybrid_kernels.dp.cpp b/dpcpp/matrix/hybrid_kernels.dp.cpp
index f740541cccc..327903c51ab 100644
--- a/dpcpp/matrix/hybrid_kernels.dp.cpp
+++ b/dpcpp/matrix/hybrid_kernels.dp.cpp
@@ -60,10 +60,7 @@ namespace hybrid {
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Hybrid<ValueType, IndexType> *source,
-                      matrix::Dense<ValueType> *result)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_HYBRID_CONVERT_TO_DENSE_KERNEL);
@@ -73,9 +70,7 @@ template <typename ValueType, typename IndexType>
 void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Hybrid<ValueType, IndexType> *source,
                     matrix::Csr<ValueType, IndexType> *result)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_HYBRID_CONVERT_TO_CSR_KERNEL);
@@ -84,10 +79,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void count_nonzeros(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Hybrid<ValueType, IndexType> *source,
-                    size_type *result)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                    size_type *result) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_HYBRID_COUNT_NONZEROS_KERNEL);
diff --git a/dpcpp/matrix/sellp_kernels.dp.cpp b/dpcpp/matrix/sellp_kernels.dp.cpp
index 20f6adcc2a0..310c3706e88 100644
--- a/dpcpp/matrix/sellp_kernels.dp.cpp
+++ b/dpcpp/matrix/sellp_kernels.dp.cpp
@@ -53,10 +53,8 @@ namespace sellp {
 template <typename ValueType, typename IndexType>
 void spmv(std::shared_ptr<const DpcppExecutor> exec,
           const matrix::Sellp<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+          const matrix::Dense<ValueType> *b,
+          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_SELLP_SPMV_KERNEL);
 
@@ -67,10 +65,7 @@ void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::Sellp<ValueType, IndexType> *a,
                    const matrix::Dense<ValueType> *b,
                    const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SELLP_ADVANCED_SPMV_KERNEL);
@@ -79,10 +74,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Sellp<ValueType, IndexType> *source,
-                      matrix::Dense<ValueType> *result)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SELLP_CONVERT_TO_DENSE_KERNEL);
@@ -110,10 +102,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Sellp<ValueType, IndexType> *orig,
-                      matrix::Diagonal<ValueType> *diag)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SELLP_EXTRACT_DIAGONAL_KERNEL);
diff --git a/dpcpp/matrix/sparsity_csr_kernels.dp.cpp b/dpcpp/matrix/sparsity_csr_kernels.dp.cpp
index ba682e194e7..cfc21f27727 100644
--- a/dpcpp/matrix/sparsity_csr_kernels.dp.cpp
+++ b/dpcpp/matrix/sparsity_csr_kernels.dp.cpp
@@ -64,10 +64,8 @@ namespace sparsity_csr {
 template <typename ValueType, typename IndexType>
 void spmv(std::shared_ptr<const DpcppExecutor> exec,
           const matrix::SparsityCsr<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+          const matrix::Dense<ValueType> *b,
+          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SPARSITY_CSR_SPMV_KERNEL);
@@ -79,10 +77,7 @@ void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::SparsityCsr<ValueType, IndexType> *a,
                    const matrix::Dense<ValueType> *b,
                    const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SPARSITY_CSR_ADVANCED_SPMV_KERNEL);
@@ -92,23 +87,17 @@ template <typename ValueType, typename IndexType>
 void count_num_diagonal_elements(
     std::shared_ptr<const DpcppExecutor> exec,
     const matrix::SparsityCsr<ValueType, IndexType> *matrix,
-    size_type *num_diagonal_elements)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    size_type *num_diagonal_elements) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SPARSITY_CSR_COUNT_NUM_DIAGONAL_ELEMENTS_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void remove_diagonal_elements(std::shared_ptr<const DpcppExecutor> exec,
-                              const IndexType *row_ptrs,
-                              const IndexType *col_idxs,
-                              matrix::SparsityCsr<ValueType, IndexType> *matrix)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+void remove_diagonal_elements(
+    std::shared_ptr<const DpcppExecutor> exec, const IndexType *row_ptrs,
+    const IndexType *col_idxs,
+    matrix::SparsityCsr<ValueType, IndexType> *matrix) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SPARSITY_CSR_REMOVE_DIAGONAL_ELEMENTS_KERNEL);
@@ -118,29 +107,22 @@ template <typename IndexType>
 inline void convert_sparsity_to_csc(size_type num_rows,
                                     const IndexType *row_ptrs,
                                     const IndexType *col_idxs,
-                                    IndexType *row_idxs, IndexType *col_ptrs)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                                    IndexType *row_idxs,
+                                    IndexType *col_ptrs) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 void transpose_and_transform(
     std::shared_ptr<const DpcppExecutor> exec,
     matrix::SparsityCsr<ValueType, IndexType> *trans,
-    const matrix::SparsityCsr<ValueType, IndexType> *orig)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    const matrix::SparsityCsr<ValueType, IndexType> *orig) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 void transpose(std::shared_ptr<const DpcppExecutor> exec,
                const matrix::SparsityCsr<ValueType, IndexType> *orig,
                matrix::SparsityCsr<ValueType, IndexType> *trans)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SPARSITY_CSR_TRANSPOSE_KERNEL);
@@ -149,9 +131,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void sort_by_column_index(std::shared_ptr<const DpcppExecutor> exec,
                           matrix::SparsityCsr<ValueType, IndexType> *to_sort)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SPARSITY_CSR_SORT_BY_COLUMN_INDEX);
@@ -160,10 +140,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void is_sorted_by_column_index(
     std::shared_ptr<const DpcppExecutor> exec,
-    const matrix::SparsityCsr<ValueType, IndexType> *to_check, bool *is_sorted)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    const matrix::SparsityCsr<ValueType, IndexType> *to_check,
+    bool *is_sorted) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SPARSITY_CSR_IS_SORTED_BY_COLUMN_INDEX);
diff --git a/dpcpp/preconditioner/isai_kernels.dp.cpp b/dpcpp/preconditioner/isai_kernels.dp.cpp
index 481979eed3b..a8aa955174c 100644
--- a/dpcpp/preconditioner/isai_kernels.dp.cpp
+++ b/dpcpp/preconditioner/isai_kernels.dp.cpp
@@ -63,10 +63,8 @@ namespace isai {
 
 template <typename IndexType, typename Callback>
 void forall_matching(const IndexType *fst, IndexType fst_size,
-                     const IndexType *snd, IndexType snd_size, Callback cb)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                     const IndexType *snd, IndexType snd_size,
+                     Callback cb) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType, typename Callable>
@@ -74,10 +72,7 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                       const matrix::Csr<ValueType, IndexType> *mtx,
                       matrix::Csr<ValueType, IndexType> *inverse_mtx,
                       IndexType *excess_rhs_ptrs, IndexType *excess_nz_ptrs,
-                      Callable trs_solve)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                      Callable trs_solve) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
@@ -85,39 +80,30 @@ void generate_tri_inverse(std::shared_ptr<const DefaultExecutor> exec,
                           const matrix::Csr<ValueType, IndexType> *mtx,
                           matrix::Csr<ValueType, IndexType> *inverse_mtx,
                           IndexType *excess_rhs_ptrs, IndexType *excess_nz_ptrs,
-                          bool lower)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                          bool lower) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_TRI_INVERSE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
-                            const matrix::Csr<ValueType, IndexType> *input,
-                            const matrix::Csr<ValueType, IndexType> *inverse,
-                            const IndexType *excess_rhs_ptrs,
-                            const IndexType *excess_nz_ptrs,
-                            matrix::Csr<ValueType, IndexType> *excess_system,
-                            matrix::Dense<ValueType> *excess_rhs)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+void generate_excess_system(
+    std::shared_ptr<const DefaultExecutor>,
+    const matrix::Csr<ValueType, IndexType> *input,
+    const matrix::Csr<ValueType, IndexType> *inverse,
+    const IndexType *excess_rhs_ptrs, const IndexType *excess_nz_ptrs,
+    matrix::Csr<ValueType, IndexType> *excess_system,
+    matrix::Dense<ValueType> *excess_rhs) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void scatter_excess_solution(std::shared_ptr<const DefaultExecutor>,
-                             const IndexType *excess_block_ptrs,
-                             const matrix::Dense<ValueType> *excess_solution,
-                             matrix::Csr<ValueType, IndexType> *inverse)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+void scatter_excess_solution(
+    std::shared_ptr<const DefaultExecutor>, const IndexType *excess_block_ptrs,
+    const matrix::Dense<ValueType> *excess_solution,
+    matrix::Csr<ValueType, IndexType> *inverse) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_SCATTER_EXCESS_SOLUTION_KERNEL);
diff --git a/dpcpp/preconditioner/jacobi_kernels.dp.cpp b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
index 06b00835e82..635f08ca87e 100644
--- a/dpcpp/preconditioner/jacobi_kernels.dp.cpp
+++ b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
@@ -69,38 +69,28 @@ namespace jacobi {
 void initialize_precisions(std::shared_ptr<const DpcppExecutor> exec,
                            const Array<precision_reduction> &source,
                            Array<precision_reduction> &precisions)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 
 namespace {
 
 
 template <typename IndexType>
-inline bool has_same_nonzero_pattern(const IndexType *prev_row_ptr,
-                                     const IndexType *curr_row_ptr,
-                                     const IndexType *next_row_ptr)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+inline bool has_same_nonzero_pattern(
+    const IndexType *prev_row_ptr, const IndexType *curr_row_ptr,
+    const IndexType *next_row_ptr) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 size_type find_natural_blocks(const matrix::Csr<ValueType, IndexType> *mtx,
-                              uint32 max_block_size, IndexType *block_ptrs)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                              uint32 max_block_size,
+                              IndexType *block_ptrs) GKO_NOT_IMPLEMENTED;
 
 
 template <typename IndexType>
-inline size_type agglomerate_supervariables(uint32 max_block_size,
-                                            size_type num_natural_blocks,
-                                            IndexType *block_ptrs)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+inline size_type agglomerate_supervariables(
+    uint32 max_block_size, size_type num_natural_blocks,
+    IndexType *block_ptrs) GKO_NOT_IMPLEMENTED;
 
 
 }  // namespace
@@ -110,10 +100,7 @@ template <typename ValueType, typename IndexType>
 void find_blocks(std::shared_ptr<const DpcppExecutor> exec,
                  const matrix::Csr<ValueType, IndexType> *system_matrix,
                  uint32 max_block_size, size_type &num_blocks,
-                 Array<IndexType> &block_pointers)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                 Array<IndexType> &block_pointers) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_FIND_BLOCKS_KERNEL);
@@ -125,95 +112,67 @@ namespace {
 template <typename ValueType, typename IndexType>
 inline void extract_block(const matrix::Csr<ValueType, IndexType> *mtx,
                           IndexType block_size, IndexType block_start,
-                          ValueType *block, size_type stride)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                          ValueType *block,
+                          size_type stride) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 inline IndexType choose_pivot(IndexType block_size, const ValueType *block,
-                              size_type stride)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                              size_type stride) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 inline void swap_rows(IndexType row1, IndexType row2, IndexType block_size,
-                      ValueType *block, size_type stride)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                      ValueType *block, size_type stride) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 inline bool apply_gauss_jordan_transform(IndexType row, IndexType col,
                                          IndexType block_size, ValueType *block,
-                                         size_type stride)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                                         size_type stride) GKO_NOT_IMPLEMENTED;
 
 
 template <typename SourceValueType, typename ResultValueType,
           typename IndexType,
           typename ValueConverter =
               default_converter<SourceValueType, ResultValueType>>
-inline void transpose_block(IndexType block_size, const SourceValueType *from,
-                            size_type from_stride, ResultValueType *to,
-                            size_type to_stride,
-                            ValueConverter converter = {}) noexcept
-{
-    GKO_NOT_IMPLEMENTED;
-}
+inline void transpose_block(
+    IndexType block_size, const SourceValueType *from, size_type from_stride,
+    ResultValueType *to, size_type to_stride,
+    ValueConverter converter = {}) noexcept GKO_NOT_IMPLEMENTED;
 
 
 template <typename SourceValueType, typename ResultValueType,
           typename IndexType,
           typename ValueConverter =
               default_converter<SourceValueType, ResultValueType>>
-inline void conj_transpose_block(IndexType block_size,
-                                 const SourceValueType *from,
-                                 size_type from_stride, ResultValueType *to,
-                                 size_type to_stride,
-                                 ValueConverter converter = {}) noexcept
-{
-    GKO_NOT_IMPLEMENTED;
-}
+inline void conj_transpose_block(
+    IndexType block_size, const SourceValueType *from, size_type from_stride,
+    ResultValueType *to, size_type to_stride,
+    ValueConverter converter = {}) noexcept GKO_NOT_IMPLEMENTED;
 
 
 template <typename SourceValueType, typename ResultValueType,
           typename IndexType,
           typename ValueConverter =
               default_converter<SourceValueType, ResultValueType>>
-inline void permute_and_transpose_block(IndexType block_size,
-                                        const IndexType *col_perm,
-                                        const SourceValueType *source,
-                                        size_type source_stride,
-                                        ResultValueType *result,
-                                        size_type result_stride,
-                                        ValueConverter converter = {})
-{
-    GKO_NOT_IMPLEMENTED;
-}
+inline void permute_and_transpose_block(
+    IndexType block_size, const IndexType *col_perm,
+    const SourceValueType *source, size_type source_stride,
+    ResultValueType *result, size_type result_stride,
+    ValueConverter converter = {}) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 inline bool invert_block(IndexType block_size, IndexType *perm,
-                         ValueType *block, size_type stride)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                         ValueType *block,
+                         size_type stride) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ReducedType, typename ValueType, typename IndexType>
 inline bool validate_precision_reduction_feasibility(
     std::shared_ptr<const DpcppExecutor> exec, IndexType block_size,
-    const ValueType *block, size_type stride)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    const ValueType *block, size_type stride) GKO_NOT_IMPLEMENTED;
 
 
 }  // namespace
@@ -228,10 +187,8 @@ void generate(std::shared_ptr<const DpcppExecutor> exec,
                   &storage_scheme,
               Array<remove_complex<ValueType>> &conditioning,
               Array<precision_reduction> &block_precisions,
-              const Array<IndexType> &block_pointers, Array<ValueType> &blocks)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+              const Array<IndexType> &block_pointers,
+              Array<ValueType> &blocks) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_GENERATE_KERNEL);
@@ -247,10 +204,7 @@ inline void apply_block(size_type block_size, size_type num_rhs,
                         const BlockValueType *block, size_type stride,
                         ValueType alpha, const ValueType *b, size_type stride_b,
                         ValueType beta, ValueType *x, size_type stride_x,
-                        ValueConverter converter = {})
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                        ValueConverter converter = {}) GKO_NOT_IMPLEMENTED;
 
 
 }  // namespace
@@ -266,10 +220,8 @@ void apply(std::shared_ptr<const DpcppExecutor> exec, size_type num_blocks,
            const Array<ValueType> &blocks,
            const matrix::Dense<ValueType> *alpha,
            const matrix::Dense<ValueType> *b,
-           const matrix::Dense<ValueType> *beta, matrix::Dense<ValueType> *x)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+           const matrix::Dense<ValueType> *beta,
+           matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_JACOBI_APPLY_KERNEL);
 
@@ -282,10 +234,8 @@ void simple_apply(
         &storage_scheme,
     const Array<precision_reduction> &block_precisions,
     const Array<IndexType> &block_pointers, const Array<ValueType> &blocks,
-    const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *x)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    const matrix::Dense<ValueType> *b,
+    matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_SIMPLE_APPLY_KERNEL);
@@ -298,10 +248,7 @@ void transpose_jacobi(
     const Array<IndexType> &block_pointers, const Array<ValueType> &blocks,
     const preconditioner::block_interleaved_storage_scheme<IndexType>
         &storage_scheme,
-    Array<ValueType> &out_blocks)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    Array<ValueType> &out_blocks) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_TRANSPOSE_KERNEL);
@@ -314,10 +261,7 @@ void conj_transpose_jacobi(
     const Array<IndexType> &block_pointers, const Array<ValueType> &blocks,
     const preconditioner::block_interleaved_storage_scheme<IndexType>
         &storage_scheme,
-    Array<ValueType> &out_blocks)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    Array<ValueType> &out_blocks) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONJ_TRANSPOSE_KERNEL);
@@ -330,10 +274,7 @@ void convert_to_dense(
     const Array<IndexType> &block_pointers, const Array<ValueType> &blocks,
     const preconditioner::block_interleaved_storage_scheme<IndexType>
         &storage_scheme,
-    ValueType *result_values, size_type result_stride)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    ValueType *result_values, size_type result_stride) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL);
diff --git a/dpcpp/solver/bicg_kernels.dp.cpp b/dpcpp/solver/bicg_kernels.dp.cpp
index 11a5471002a..bb1b6e0f49d 100644
--- a/dpcpp/solver/bicg_kernels.dp.cpp
+++ b/dpcpp/solver/bicg_kernels.dp.cpp
@@ -61,10 +61,7 @@ void initialize(std::shared_ptr<const DpcppExecutor> exec,
                 matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *r2,
                 matrix::Dense<ValueType> *z2, matrix::Dense<ValueType> *p2,
                 matrix::Dense<ValueType> *q2,
-                Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_INITIALIZE_KERNEL);
 
@@ -75,10 +72,7 @@ void step_1(std::shared_ptr<const DpcppExecutor> exec,
             matrix::Dense<ValueType> *p2, const matrix::Dense<ValueType> *z2,
             const matrix::Dense<ValueType> *rho,
             const matrix::Dense<ValueType> *prev_rho,
-            const Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_1_KERNEL);
 
@@ -91,10 +85,7 @@ void step_2(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Dense<ValueType> *q2,
             const matrix::Dense<ValueType> *beta,
             const matrix::Dense<ValueType> *rho,
-            const Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_2_KERNEL);
 
diff --git a/dpcpp/solver/bicgstab_kernels.dp.cpp b/dpcpp/solver/bicgstab_kernels.dp.cpp
index af4d42289cd..25b133f087e 100644
--- a/dpcpp/solver/bicgstab_kernels.dp.cpp
+++ b/dpcpp/solver/bicgstab_kernels.dp.cpp
@@ -65,10 +65,7 @@ void initialize(std::shared_ptr<const DpcppExecutor> exec,
                 matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *alpha,
                 matrix::Dense<ValueType> *beta, matrix::Dense<ValueType> *gamma,
                 matrix::Dense<ValueType> *omega,
-                Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_INITIALIZE_KERNEL);
 
@@ -81,10 +78,7 @@ void step_1(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Dense<ValueType> *prev_rho,
             const matrix::Dense<ValueType> *alpha,
             const matrix::Dense<ValueType> *omega,
-            const Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_1_KERNEL);
 
@@ -96,10 +90,7 @@ void step_2(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Dense<ValueType> *rho,
             matrix::Dense<ValueType> *alpha,
             const matrix::Dense<ValueType> *beta,
-            const Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_2_KERNEL);
 
@@ -111,10 +102,8 @@ void step_3(
     const matrix::Dense<ValueType> *t, const matrix::Dense<ValueType> *y,
     const matrix::Dense<ValueType> *z, const matrix::Dense<ValueType> *alpha,
     const matrix::Dense<ValueType> *beta, const matrix::Dense<ValueType> *gamma,
-    matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    matrix::Dense<ValueType> *omega,
+    const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_3_KERNEL);
 
@@ -123,10 +112,7 @@ template <typename ValueType>
 void finalize(std::shared_ptr<const DpcppExecutor> exec,
               matrix::Dense<ValueType> *x, const matrix::Dense<ValueType> *y,
               const matrix::Dense<ValueType> *alpha,
-              Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+              Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_FINALIZE_KERNEL);
 
diff --git a/dpcpp/solver/cg_kernels.dp.cpp b/dpcpp/solver/cg_kernels.dp.cpp
index 9cb2e0f30da..2e5bf4bc47b 100644
--- a/dpcpp/solver/cg_kernels.dp.cpp
+++ b/dpcpp/solver/cg_kernels.dp.cpp
@@ -59,10 +59,7 @@ void initialize(std::shared_ptr<const DpcppExecutor> exec,
                 matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
                 matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
                 matrix::Dense<ValueType> *rho,
-                Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
 
@@ -72,10 +69,7 @@ void step_1(std::shared_ptr<const DpcppExecutor> exec,
             matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
             const matrix::Dense<ValueType> *rho,
             const matrix::Dense<ValueType> *prev_rho,
-            const Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
 
@@ -87,10 +81,7 @@ void step_2(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Dense<ValueType> *q,
             const matrix::Dense<ValueType> *beta,
             const matrix::Dense<ValueType> *rho,
-            const Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
 
diff --git a/dpcpp/solver/cgs_kernels.dp.cpp b/dpcpp/solver/cgs_kernels.dp.cpp
index 04d81b682c5..42e1ff9b37e 100644
--- a/dpcpp/solver/cgs_kernels.dp.cpp
+++ b/dpcpp/solver/cgs_kernels.dp.cpp
@@ -64,10 +64,7 @@ void initialize(std::shared_ptr<const DpcppExecutor> exec,
                 matrix::Dense<ValueType> *gamma,
                 matrix::Dense<ValueType> *prev_rho,
                 matrix::Dense<ValueType> *rho,
-                Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_INITIALIZE_KERNEL);
 
@@ -78,10 +75,7 @@ void step_1(std::shared_ptr<const DpcppExecutor> exec,
             matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *q,
             matrix::Dense<ValueType> *beta, const matrix::Dense<ValueType> *rho,
             const matrix::Dense<ValueType> *rho_prev,
-            const Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_1_KERNEL);
 
@@ -93,10 +87,7 @@ void step_2(std::shared_ptr<const DpcppExecutor> exec,
             matrix::Dense<ValueType> *t, matrix::Dense<ValueType> *alpha,
             const matrix::Dense<ValueType> *rho,
             const matrix::Dense<ValueType> *gamma,
-            const Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_2_KERNEL);
 
@@ -105,10 +96,7 @@ void step_3(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *t,
             const matrix::Dense<ValueType> *u_hat, matrix::Dense<ValueType> *r,
             matrix::Dense<ValueType> *x, const matrix::Dense<ValueType> *alpha,
-            const Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_3_KERNEL);
 
diff --git a/dpcpp/solver/fcg_kernels.dp.cpp b/dpcpp/solver/fcg_kernels.dp.cpp
index 79422ed1960..70bff10d329 100644
--- a/dpcpp/solver/fcg_kernels.dp.cpp
+++ b/dpcpp/solver/fcg_kernels.dp.cpp
@@ -59,10 +59,7 @@ void initialize(std::shared_ptr<const DpcppExecutor> exec,
                 matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *t,
                 matrix::Dense<ValueType> *prev_rho,
                 matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *rho_t,
-                Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_INITIALIZE_KERNEL);
 
@@ -72,10 +69,7 @@ void step_1(std::shared_ptr<const DpcppExecutor> exec,
             matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
             const matrix::Dense<ValueType> *rho_t,
             const matrix::Dense<ValueType> *prev_rho,
-            const Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_1_KERNEL);
 
@@ -87,10 +81,7 @@ void step_2(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Dense<ValueType> *q,
             const matrix::Dense<ValueType> *beta,
             const matrix::Dense<ValueType> *rho,
-            const Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_2_KERNEL);
 
diff --git a/dpcpp/solver/gmres_kernels.dp.cpp b/dpcpp/solver/gmres_kernels.dp.cpp
index c8924b0b67f..8a174cb22f8 100644
--- a/dpcpp/solver/gmres_kernels.dp.cpp
+++ b/dpcpp/solver/gmres_kernels.dp.cpp
@@ -60,30 +60,22 @@ namespace {
 template <typename ValueType>
 void finish_arnoldi(size_type num_rows, matrix::Dense<ValueType> *krylov_bases,
                     matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
-                    const stopping_status *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                    const stopping_status *stop_status) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType>
 void calculate_sin_and_cos(matrix::Dense<ValueType> *givens_sin,
                            matrix::Dense<ValueType> *givens_cos,
                            matrix::Dense<ValueType> *hessenberg_iter,
-                           size_type iter, const size_type rhs)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                           size_type iter,
+                           const size_type rhs) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType>
 void givens_rotation(matrix::Dense<ValueType> *givens_sin,
                      matrix::Dense<ValueType> *givens_cos,
                      matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
-                     const stopping_status *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                     const stopping_status *stop_status) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType>
@@ -91,30 +83,21 @@ void calculate_next_residual_norm(
     matrix::Dense<ValueType> *givens_sin, matrix::Dense<ValueType> *givens_cos,
     matrix::Dense<remove_complex<ValueType>> *residual_norm,
     matrix::Dense<ValueType> *residual_norm_collection, size_type iter,
-    const stopping_status *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    const stopping_status *stop_status) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType>
 void solve_upper_triangular(
     const matrix::Dense<ValueType> *residual_norm_collection,
     const matrix::Dense<ValueType> *hessenberg, matrix::Dense<ValueType> *y,
-    const size_type *final_iter_nums)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    const size_type *final_iter_nums) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType>
 void calculate_qy(const matrix::Dense<ValueType> *krylov_bases,
                   const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
-                  const size_type *final_iter_nums)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                  const size_type *final_iter_nums) GKO_NOT_IMPLEMENTED;
 
 
 }  // namespace
@@ -126,10 +109,8 @@ void initialize_1(std::shared_ptr<const DpcppExecutor> exec,
                   matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *givens_sin,
                   matrix::Dense<ValueType> *givens_cos,
-                  Array<stopping_status> *stop_status, size_type krylov_dim)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                  Array<stopping_status> *stop_status,
+                  size_type krylov_dim) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_INITIALIZE_1_KERNEL);
 
@@ -140,10 +121,8 @@ void initialize_2(std::shared_ptr<const DpcppExecutor> exec,
                   matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
                   matrix::Dense<ValueType> *krylov_bases,
-                  Array<size_type> *final_iter_nums, size_type krylov_dim)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                  Array<size_type> *final_iter_nums,
+                  size_type krylov_dim) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_INITIALIZE_2_KERNEL);
 
@@ -157,10 +136,7 @@ void step_1(std::shared_ptr<const DpcppExecutor> exec, size_type num_rows,
             matrix::Dense<ValueType> *krylov_bases,
             matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
             Array<size_type> *final_iter_nums,
-            const Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_STEP_1_KERNEL);
 
@@ -172,10 +148,7 @@ void step_2(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
-            const Array<size_type> *final_iter_nums)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+            const Array<size_type> *final_iter_nums) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_STEP_2_KERNEL);
 
diff --git a/dpcpp/solver/ir_kernels.dp.cpp b/dpcpp/solver/ir_kernels.dp.cpp
index bd3c6a98bc8..03f1150e004 100644
--- a/dpcpp/solver/ir_kernels.dp.cpp
+++ b/dpcpp/solver/ir_kernels.dp.cpp
@@ -48,10 +48,7 @@ namespace ir {
 
 
 void initialize(std::shared_ptr<const DpcppExecutor> exec,
-                Array<stopping_status> *stop_status)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 
 }  // namespace ir
diff --git a/dpcpp/solver/lower_trs_kernels.dp.cpp b/dpcpp/solver/lower_trs_kernels.dp.cpp
index 6f0546ed6a4..6b6d71b30fd 100644
--- a/dpcpp/solver/lower_trs_kernels.dp.cpp
+++ b/dpcpp/solver/lower_trs_kernels.dp.cpp
@@ -60,10 +60,7 @@ namespace lower_trs {
 
 
 void should_perform_transpose(std::shared_ptr<const DpcppExecutor> exec,
-                              bool &do_transpose)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                              bool &do_transpose) GKO_NOT_IMPLEMENTED;
 
 
 void init_struct(std::shared_ptr<const DpcppExecutor> exec,
@@ -97,10 +94,8 @@ void solve(std::shared_ptr<const DpcppExecutor> exec,
            const matrix::Csr<ValueType, IndexType> *matrix,
            const solver::SolveStruct *solve_struct,
            matrix::Dense<ValueType> *trans_b, matrix::Dense<ValueType> *trans_x,
-           const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *x)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+           const matrix::Dense<ValueType> *b,
+           matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_LOWER_TRS_SOLVE_KERNEL);
diff --git a/dpcpp/solver/upper_trs_kernels.dp.cpp b/dpcpp/solver/upper_trs_kernels.dp.cpp
index 67efc9896f6..2d7afa5dbc8 100644
--- a/dpcpp/solver/upper_trs_kernels.dp.cpp
+++ b/dpcpp/solver/upper_trs_kernels.dp.cpp
@@ -97,10 +97,8 @@ void solve(std::shared_ptr<const DpcppExecutor> exec,
            const matrix::Csr<ValueType, IndexType> *matrix,
            const solver::SolveStruct *solve_struct,
            matrix::Dense<ValueType> *trans_b, matrix::Dense<ValueType> *trans_x,
-           const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *x)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+           const matrix::Dense<ValueType> *b,
+           matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_UPPER_TRS_SOLVE_KERNEL);
diff --git a/dpcpp/stop/criterion_kernels.dp.cpp b/dpcpp/stop/criterion_kernels.dp.cpp
index dce2c8ae84c..bc59cc1cae8 100644
--- a/dpcpp/stop/criterion_kernels.dp.cpp
+++ b/dpcpp/stop/criterion_kernels.dp.cpp
@@ -58,7 +58,7 @@ void set_all_statuses(std::shared_ptr<const DpcppExecutor> exec,
     stopping_status *__restrict__ stop_status_ptr = stop_status->get_data();
     exec->get_queue()->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl::range<1>{size}, [=](sycl::id<1> idx_id) {
-            const int idx = idx_id[0];
+            const auto idx = idx_id[0];
             stop_status_ptr[idx].stop(stoppingId, setFinalized);
         });
     });
diff --git a/dpcpp/stop/residual_norm_kernels.dp.cpp b/dpcpp/stop/residual_norm_kernels.dp.cpp
index f0f205d438d..e42861c38bc 100644
--- a/dpcpp/stop/residual_norm_kernels.dp.cpp
+++ b/dpcpp/stop/residual_norm_kernels.dp.cpp
@@ -62,10 +62,7 @@ void residual_norm(std::shared_ptr<const DpcppExecutor> exec,
                    ValueType rel_residual_goal, uint8 stoppingId,
                    bool setFinalized, Array<stopping_status> *stop_status,
                    Array<bool> *device_storage, bool *all_converged,
-                   bool *one_changed)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+                   bool *one_changed) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
     GKO_DECLARE_RESIDUAL_NORM_KERNEL);
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 9217e00f800..c210718f13c 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -914,6 +914,11 @@ GKO_INLINE GKO_ATTRIBUTES constexpr xstd::enable_if_t<is_complex_s<T>::value,
 abs(const T &x)
 {
 #ifdef CL_SYCL_LANGUAGE_VERSION
+    // FIXME: This implementation is due to two DPC++ issues
+    // 1) plain `sqrt` call evaluates to `std::sqrt` which fails on GPUs
+    // 2) DPC++ GPUs require the OpenCL backend (`SYCL_BE=OPENCL`) to support
+    //    complex multiplication, on top of extra binary objects. For now, we
+    //    avoid this issue with the current implementation.
     return cl::sycl::sqrt(real(x) * real(x) + imag(x) * imag(x));
 #else
     return sqrt(squared_norm(x));
diff --git a/include/ginkgo/core/reorder/reordering_base.hpp b/include/ginkgo/core/reorder/reordering_base.hpp
index e4ee5c9e8f1..7b93d575813 100644
--- a/include/ginkgo/core/reorder/reordering_base.hpp
+++ b/include/ginkgo/core/reorder/reordering_base.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2019, the Ginkgo authors
+Copyright (c) 2017-2020, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/get_info.cmake b/omp/get_info.cmake
index 4fe24965649..3dc866e13fc 100644
--- a/omp/get_info.cmake
+++ b/omp/get_info.cmake
@@ -4,4 +4,6 @@ ginkgo_print_variable(${detailed_log} "OpenMP_CXX_LIB_NAMES")
 ginkgo_print_variable(${detailed_log} "OpenMP_CXX_LIBRARIES")
 ginkgo_print_module_footer(${detailed_log} "OMP variables:")
 ginkgo_print_variable(${detailed_log} "GINKGO_COMPILER_FLAGS")
+ginkgo_print_module_footer(${detailed_log} "OMP environment variables:")
+ginkgo_print_env_variable(${detailed_log} "OMP_NUM_THREADS")
 ginkgo_print_module_footer(${detailed_log} "")
diff --git a/omp/test/reorder/rcm_kernels.cpp b/omp/test/reorder/rcm_kernels.cpp
index 8fcad63a4f4..9f7a891cb89 100644
--- a/omp/test/reorder/rcm_kernels.cpp
+++ b/omp/test/reorder/rcm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2019, the Ginkgo authors
+Copyright (c) 2017-2020, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/reorder/rcm.cpp b/reference/test/reorder/rcm.cpp
index b8358f4edd3..c81045c8fd6 100644
--- a/reference/test/reorder/rcm.cpp
+++ b/reference/test/reorder/rcm.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2019, the Ginkgo authors
+Copyright (c) 2017-2020, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

From b447ced37be96baef7cda2fee413e075a6502df4 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 22 Oct 2020 15:27:37 +0200
Subject: [PATCH 0094/1008] Add conversion to complex and extraction of real
 and imaginary part to dense

---
 common/matrix/dense_kernels.hpp.inc      |  45 ++++++++++
 core/device_hooks/common_kernels.inc.cpp |  15 ++++
 core/matrix/dense.cpp                    |  45 ++++++++++
 core/matrix/dense_kernels.hpp            |  23 ++++-
 cuda/matrix/dense_kernels.cu             |  51 +++++++++++
 cuda/test/matrix/dense_kernels.cpp       |  33 ++++++++
 hip/matrix/dense_kernels.hip.cpp         |  54 ++++++++++++
 hip/test/matrix/dense_kernels.hip.cpp    |  33 ++++++++
 include/ginkgo/core/matrix/dense.hpp     |   7 ++
 omp/matrix/dense_kernels.cpp             |  54 ++++++++++++
 omp/test/matrix/dense_kernels.cpp        |  33 ++++++++
 reference/matrix/dense_kernels.cpp       |  48 +++++++++++
 reference/test/matrix/dense_kernels.cpp  | 103 +++++++++++++++++++++++
 13 files changed, 543 insertions(+), 1 deletion(-)

diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index c26203115fe..b4649c40fdd 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -539,4 +539,49 @@ __global__ __launch_bounds__(default_block_size) void outplace_absolute_dense(
 }
 
 
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void make_complex(
+    size_type num_rows, size_type num_cols, const ValueType *__restrict__ in,
+    size_type stride_in, to_complex<ValueType> *__restrict__ out,
+    size_type stride_out)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    auto row = tidx / num_cols;
+    auto col = tidx % num_cols;
+    if (row < num_rows) {
+        out[row * stride_out + col] = to_complex{in[row * stride_in + col]};
+    }
+}
+
+
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void get_real(
+    size_type num_rows, size_type num_cols, const ValueType *__restrict__ in,
+    size_type stride_in, to_complex<ValueType> *__restrict__ out,
+    size_type stride_out)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    auto row = tidx / num_cols;
+    auto col = tidx % num_cols;
+    if (row < num_rows) {
+        out[row * stride_out + col] = real(in[row * stride_in + col]);
+    }
+}
+
+
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void get_imag(
+    size_type num_rows, size_type num_cols, const ValueType *__restrict__ in,
+    size_type stride_in, to_complex<ValueType> *__restrict__ out,
+    size_type stride_out)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    auto row = tidx / num_cols;
+    auto col = tidx % num_cols;
+    if (row < num_rows) {
+        out[row * stride_out + col] = imag(in[row * stride_in + col]);
+    }
+}
+
+
 }  // namespace kernel
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index a981372c630..34992284498 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -256,6 +256,21 @@ GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
 
+template <typename ValueType>
+GKO_DECLARE_MAKE_COMPLEX_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MAKE_COMPLEX_KERNEL);
+
+template <typename ValueType>
+GKO_DECLARE_GET_REAL_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_REAL_KERNEL);
+
+template <typename ValueType>
+GKO_DECLARE_GET_IMAG_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_IMAG_KERNEL);
+
 
 }  // namespace dense
 
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 2e0b663966f..398b5192035 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -88,6 +88,9 @@ GKO_REGISTER_OPERATION(convert_to_sparsity_csr, dense::convert_to_sparsity_csr);
 GKO_REGISTER_OPERATION(extract_diagonal, dense::extract_diagonal);
 GKO_REGISTER_OPERATION(inplace_absolute_dense, dense::inplace_absolute_dense);
 GKO_REGISTER_OPERATION(outplace_absolute_dense, dense::outplace_absolute_dense);
+GKO_REGISTER_OPERATION(make_complex, dense::make_complex);
+GKO_REGISTER_OPERATION(get_real, dense::get_real);
+GKO_REGISTER_OPERATION(get_imag, dense::get_imag);
 
 
 }  // namespace dense
@@ -795,6 +798,48 @@ Dense<ValueType>::compute_absolute() const
 }
 
 
+template <typename ValueType>
+std::unique_ptr<typename Dense<ValueType>::complex_type>
+Dense<ValueType>::make_complex() const
+{
+    auto exec = this->get_executor();
+
+    auto complex_dense = complex_type::create(exec, this->get_size());
+
+    exec->run(dense::make_make_complex(this, complex_dense.get()));
+
+    return complex_dense;
+}
+
+
+template <typename ValueType>
+std::unique_ptr<typename Dense<ValueType>::absolute_type>
+Dense<ValueType>::get_real() const
+{
+    auto exec = this->get_executor();
+
+    auto real_dense = absolute_type::create(exec, this->get_size());
+
+    exec->run(dense::make_get_real(this, real_dense.get()));
+
+    return real_dense;
+}
+
+
+template <typename ValueType>
+std::unique_ptr<typename Dense<ValueType>::absolute_type>
+Dense<ValueType>::get_imag() const
+{
+    auto exec = this->get_executor();
+
+    auto imag_dense = absolute_type::create(exec, this->get_size());
+
+    exec->run(dense::make_get_imag(this, imag_dense.get()));
+
+    return imag_dense;
+}
+
+
 #define GKO_DECLARE_DENSE_MATRIX(_type) class Dense<_type>
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_MATRIX);
 
diff --git a/core/matrix/dense_kernels.hpp b/core/matrix/dense_kernels.hpp
index 1f9bdae1cc2..e99d455aa77 100644
--- a/core/matrix/dense_kernels.hpp
+++ b/core/matrix/dense_kernels.hpp
@@ -182,6 +182,21 @@ namespace kernels {
         const matrix::Dense<_vtype> *source,               \
         matrix::Dense<remove_complex<_vtype>> *result)
 
+#define GKO_DECLARE_MAKE_COMPLEX_KERNEL(_vtype)                    \
+    void make_complex(std::shared_ptr<const DefaultExecutor> exec, \
+                      const matrix::Dense<_vtype> *source,         \
+                      matrix::Dense<to_complex<_vtype>> *result)
+
+#define GKO_DECLARE_GET_REAL_KERNEL(_vtype)                    \
+    void get_real(std::shared_ptr<const DefaultExecutor> exec, \
+                  const matrix::Dense<_vtype> *source,         \
+                  matrix::Dense<remove_complex<_vtype>> *result)
+
+#define GKO_DECLARE_GET_IMAG_KERNEL(_vtype)                    \
+    void get_imag(std::shared_ptr<const DefaultExecutor> exec, \
+                  const matrix::Dense<_vtype> *source,         \
+                  matrix::Dense<remove_complex<_vtype>> *result)
+
 
 #define GKO_DECLARE_ALL_AS_TEMPLATES                                        \
     template <typename ValueType>                                           \
@@ -235,7 +250,13 @@ namespace kernels {
     template <typename ValueType>                                           \
     GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL(ValueType);                   \
     template <typename ValueType>                                           \
-    GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL(ValueType)
+    GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL(ValueType);                  \
+    template <typename ValueType>                                           \
+    GKO_DECLARE_MAKE_COMPLEX_KERNEL(ValueType);                             \
+    template <typename ValueType>                                           \
+    GKO_DECLARE_GET_REAL_KERNEL(ValueType);                                 \
+    template <typename ValueType>                                           \
+    GKO_DECLARE_GET_IMAG_KERNEL(ValueType)
 
 
 namespace omp {
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index 72d557e7dd1..83804b43083 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -734,6 +734,57 @@ void outplace_absolute_dense(std::shared_ptr<const CudaExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
 
 
+template <typename ValueType>
+void make_complex(std::shared_ptr<const CudaExecutor> exec,
+                  const matrix::Dense<ValueType> *source,
+                  matrix::Dense<to_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
+
+    kernel::make_complex<<<grid_dim, default_block_size>>>(
+        dim[0], dim[1], as_cuda_type(source->get_const_values()),
+        source->get_stride(), as_cuda_type(result->get_values()),
+        result->get_stride());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MAKE_COMPLEX_KERNEL);
+
+
+template <typename ValueType>
+void get_real(std::shared_ptr<const CudaExecutor> exec,
+              const matrix::Dense<ValueType> *source,
+              matrix::Dense<remove_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
+
+    kernel::get_real<<<grid_dim, default_block_size>>>(
+        dim[0], dim[1], as_cuda_type(source->get_const_values()),
+        source->get_stride(), as_cuda_type(result->get_values()),
+        result->get_stride());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_REAL_KERNEL);
+
+
+template <typename ValueType>
+void get_imag(std::shared_ptr<const CudaExecutor> exec,
+              const matrix::Dense<ValueType> *source,
+              matrix::Dense<remove_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
+
+    kernel::get_imag<<<grid_dim, default_block_size>>>(
+        dim[0], dim[1], as_cuda_type(source->get_const_values()),
+        source->get_stride(), as_cuda_type(result->get_values()),
+        result->get_stride());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_IMAG_KERNEL);
+
+
 }  // namespace dense
 }  // namespace cuda
 }  // namespace kernels
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index 1dee85ba561..ac8f0262978 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -643,4 +643,37 @@ TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, MakeComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto complex_x = x->make_complex();
+    auto dcomplex_x = dx->make_complex();
+
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 1e-14);
+}
+
+
+TEST_F(Dense, GetRealIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto real_x = x->get_real();
+    auto dreal_x = dx->get_real();
+
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 1e-14);
+}
+
+
+TEST_F(Dense, GetImagIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto imag_x = x->get_imag();
+    auto dimag_x = dx->get_imag();
+
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 1e-14);
+}
+
+
 }  // namespace
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index fa732e1e86a..e5de9f7df83 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -763,6 +763,60 @@ void outplace_absolute_dense(std::shared_ptr<const HipExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
 
 
+template <typename ValueType>
+void make_complex(std::shared_ptr<const HipExecutor> exec,
+                  const matrix::Dense<ValueType> *source,
+                  matrix::Dense<to_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
+
+    hipLaunchKernelGGL(kernel::make_complex, dim3(grid_dim),
+                       dim3(default_block_size), 0, 0, dim[0], dim[1],
+                       as_hip_type(source->get_const_values()),
+                       source->get_stride(), as_hip_type(result->get_values()),
+                       result->get_stride());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MAKE_COMPLEX_KERNEL);
+
+
+template <typename ValueType>
+void get_real(std::shared_ptr<const HipExecutor> exec,
+              const matrix::Dense<ValueType> *source,
+              matrix::Dense<remove_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
+
+    hipLaunchKernelGGL(kernel::get_real, dim3(grid_dim),
+                       dim3(default_block_size), 0, 0, dim[0], dim[1],
+                       as_hip_type(source->get_const_values()),
+                       source->get_stride(), as_hip_type(result->get_values()),
+                       result->get_stride());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_REAL_KERNEL);
+
+
+template <typename ValueType>
+void get_imag(std::shared_ptr<const HipExecutor> exec,
+              const matrix::Dense<ValueType> *source,
+              matrix::Dense<remove_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
+
+    hipLaunchKernelGGL(kernel::get_imag, dim3(grid_dim),
+                       dim3(default_block_size), 0, 0, dim[0], dim[1],
+                       as_hip_type(source->get_const_values()),
+                       source->get_stride(), as_hip_type(result->get_values()),
+                       result->get_stride());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_IMAG_KERNEL);
+
+
 }  // namespace dense
 }  // namespace hip
 }  // namespace kernels
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index fb40a3819fa..8385fd21205 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -626,4 +626,37 @@ TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, MakeComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto complex_x = x->make_complex();
+    auto dcomplex_x = dx->make_complex();
+
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 1e-14);
+}
+
+
+TEST_F(Dense, GetRealIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto real_x = x->get_real();
+    auto dreal_x = dx->get_real();
+
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 1e-14);
+}
+
+
+TEST_F(Dense, GetImagIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto imag_x = x->get_imag();
+    auto dimag_x = dx->get_imag();
+
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 1e-14);
+}
+
+
 }  // namespace
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 65113ec204b..ce9e85d5d16 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -142,6 +142,7 @@ class Dense
     using mat_data = gko::matrix_data<ValueType, int64>;
     using mat_data32 = gko::matrix_data<ValueType, int32>;
     using absolute_type = remove_complex<Dense>;
+    using complex_type = to_complex<Dense>;
 
     using row_major_range = gko::range<gko::accessor::row_major<ValueType, 2>>;
 
@@ -255,6 +256,12 @@ class Dense
 
     void compute_absolute_inplace() override;
 
+    std::unique_ptr<complex_type> make_complex() const;
+
+    std::unique_ptr<absolute_type> get_real() const;
+
+    std::unique_ptr<absolute_type> get_imag() const;
+
     /**
      * Returns a pointer to the array of values of the matrix.
      *
diff --git a/omp/matrix/dense_kernels.cpp b/omp/matrix/dense_kernels.cpp
index afe74e3c5e4..15a9d6e9a99 100644
--- a/omp/matrix/dense_kernels.cpp
+++ b/omp/matrix/dense_kernels.cpp
@@ -805,6 +805,60 @@ void outplace_absolute_dense(std::shared_ptr<const OmpExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
 
 
+template <typename ValueType>
+void make_complex(std::shared_ptr<const OmpExecutor> exec,
+                  const matrix::Dense<ValueType> *source,
+                  matrix::Dense<to_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+
+#pragma omp parallel for
+    for (size_type row = 0; row < dim[0]; row++) {
+        for (size_type col = 0; col < dim[1]; col++) {
+            result->at(row, col) = to_complex<ValueType>{source->at(row, col)};
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MAKE_COMPLEX_KERNEL);
+
+
+template <typename ValueType>
+void get_real(std::shared_ptr<const OmpExecutor> exec,
+              const matrix::Dense<ValueType> *source,
+              matrix::Dense<remove_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+
+#pragma omp parallel for
+    for (size_type row = 0; row < dim[0]; row++) {
+        for (size_type col = 0; col < dim[1]; col++) {
+            result->at(row, col) = real(source->at(row, col));
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_REAL_KERNEL);
+
+
+template <typename ValueType>
+void get_imag(std::shared_ptr<const OmpExecutor> exec,
+              const matrix::Dense<ValueType> *source,
+              matrix::Dense<remove_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+
+#pragma omp parallel for
+    for (size_type row = 0; row < dim[0]; row++) {
+        for (size_type col = 0; col < dim[1]; col++) {
+            result->at(row, col) = imag(source->at(row, col));
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_IMAG_KERNEL);
+
+
 }  // namespace dense
 }  // namespace omp
 }  // namespace kernels
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index 06999748b9f..90c7880bd85 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -758,4 +758,37 @@ TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, MakeComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto complex_x = x->make_complex();
+    auto dcomplex_x = dx->make_complex();
+
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 1e-14);
+}
+
+
+TEST_F(Dense, GetRealIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto real_x = x->get_real();
+    auto dreal_x = dx->get_real();
+
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 1e-14);
+}
+
+
+TEST_F(Dense, GetImagIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto imag_x = x->get_imag();
+    auto dimag_x = dx->get_imag();
+
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 1e-14);
+}
+
+
 }  // namespace
diff --git a/reference/matrix/dense_kernels.cpp b/reference/matrix/dense_kernels.cpp
index f0c8f938d05..147204157e7 100644
--- a/reference/matrix/dense_kernels.cpp
+++ b/reference/matrix/dense_kernels.cpp
@@ -716,6 +716,54 @@ void outplace_absolute_dense(std::shared_ptr<const ReferenceExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
 
 
+template <typename ValueType>
+void make_complex(std::shared_ptr<const ReferenceExecutor> exec,
+                  const matrix::Dense<ValueType> *source,
+                  matrix::Dense<to_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+    for (size_type row = 0; row < dim[0]; row++) {
+        for (size_type col = 0; col < dim[1]; col++) {
+            result->at(row, col) = to_complex<ValueType>{source->at(row, col)};
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MAKE_COMPLEX_KERNEL);
+
+
+template <typename ValueType>
+void get_real(std::shared_ptr<const ReferenceExecutor> exec,
+              const matrix::Dense<ValueType> *source,
+              matrix::Dense<remove_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+    for (size_type row = 0; row < dim[0]; row++) {
+        for (size_type col = 0; col < dim[1]; col++) {
+            result->at(row, col) = real(source->at(row, col));
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_REAL_KERNEL);
+
+
+template <typename ValueType>
+void get_imag(std::shared_ptr<const ReferenceExecutor> exec,
+              const matrix::Dense<ValueType> *source,
+              matrix::Dense<remove_complex<ValueType>> *result)
+{
+    auto dim = source->get_size();
+    for (size_type row = 0; row < dim[0]; row++) {
+        for (size_type col = 0; col < dim[1]; col++) {
+            result->at(row, col) = imag(source->at(row, col));
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_IMAG_KERNEL);
+
+
 }  // namespace dense
 }  // namespace reference
 }  // namespace kernels
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index b6195ffa8b0..b31fd51e52a 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -2151,6 +2151,53 @@ TYPED_TEST(Dense, AdvancedAppliesToComplex)
 }
 
 
+TYPED_TEST(Dense, MakeComplex)
+{
+    using T = typename TestFixture::value_type;
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+
+    auto abs_mtx = this->mtx5->make_complex();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx, this->mtx5, r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Dense, GetReal)
+{
+    using T = typename TestFixture::value_type;
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+
+    auto abs_mtx = this->mtx5->get_real();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx, this->mtx5, r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Dense, GetImag)
+{
+    using T = typename TestFixture::value_type;
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+
+    auto abs_mtx = this->mtx5->get_imag();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx,
+                        l({{0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}}),
+                        r<TypeParam>::value);
+}
+
+
 template <typename T>
 class DenseComplex : public ::testing::Test {
 protected:
@@ -2220,4 +2267,60 @@ TYPED_TEST(DenseComplex, OutplaceAbsolute)
 }
 
 
+TYPED_TEST(DenseComplex, MakeComplex)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    auto abs_mtx = mtx->make_complex();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx, mtx, 0.0);
+}
+
+
+TYPED_TEST(DenseComplex, GetReal)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    auto abs_mtx = mtx->get_real();
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l({{1.0, 3.0, 0.0}, {-4.0, -1.0, 0.0}, {0.0, 0.0, 2.0}}), 0.0);
+}
+
+
+TYPED_TEST(DenseComplex, GetImag)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    auto abs_mtx = mtx->get_imag();
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l({{0.0, 4.0, 2.0}, {-3.0, 0.0, 0.0}, {0.0, -1.5, 0.0}}), 0.0);
+}
+
+
 }  // namespace

From 658d76be6046d496ccb31eab0a8467484d196319 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Mon, 26 Oct 2020 11:55:49 +0100
Subject: [PATCH 0095/1008] Fix wrong parameter types in gpu kernels

---
 common/matrix/dense_kernels.hpp.inc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index b4649c40fdd..8adeaed9521 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -557,7 +557,7 @@ __global__ __launch_bounds__(default_block_size) void make_complex(
 template <typename ValueType>
 __global__ __launch_bounds__(default_block_size) void get_real(
     size_type num_rows, size_type num_cols, const ValueType *__restrict__ in,
-    size_type stride_in, to_complex<ValueType> *__restrict__ out,
+    size_type stride_in, remove_complex<ValueType> *__restrict__ out,
     size_type stride_out)
 {
     const auto tidx = thread::get_thread_id_flat();
@@ -572,7 +572,7 @@ __global__ __launch_bounds__(default_block_size) void get_real(
 template <typename ValueType>
 __global__ __launch_bounds__(default_block_size) void get_imag(
     size_type num_rows, size_type num_cols, const ValueType *__restrict__ in,
-    size_type stride_in, to_complex<ValueType> *__restrict__ out,
+    size_type stride_in, remove_complex<ValueType> *__restrict__ out,
     size_type stride_out)
 {
     const auto tidx = thread::get_thread_id_flat();

From 8c8774eb458c2006283cd765cb522767a75ee7a2 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Tue, 27 Oct 2020 15:34:48 +0100
Subject: [PATCH 0096/1008] Apply suggestions from code review

---
 common/matrix/dense_kernels.hpp.inc   |  7 +++----
 cuda/test/matrix/dense_kernels.cpp    |  6 +++---
 hip/test/matrix/dense_kernels.hip.cpp |  6 +++---
 include/ginkgo/core/matrix/dense.hpp  | 12 ++++++++++++
 omp/test/matrix/dense_kernels.cpp     |  6 +++---
 5 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index 8adeaed9521..a9947ad891f 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -539,17 +539,16 @@ __global__ __launch_bounds__(default_block_size) void outplace_absolute_dense(
 }
 
 
-template <typename ValueType>
+template <typename ValueType, typename ComplexType>
 __global__ __launch_bounds__(default_block_size) void make_complex(
     size_type num_rows, size_type num_cols, const ValueType *__restrict__ in,
-    size_type stride_in, to_complex<ValueType> *__restrict__ out,
-    size_type stride_out)
+    size_type stride_in, ComplexType *__restrict__ out, size_type stride_out)
 {
     const auto tidx = thread::get_thread_id_flat();
     auto row = tidx / num_cols;
     auto col = tidx % num_cols;
     if (row < num_rows) {
-        out[row * stride_out + col] = to_complex{in[row * stride_in + col]};
+        out[row * stride_out + col] = in[row * stride_in + col];
     }
 }
 
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index ac8f0262978..ea067231ece 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -650,7 +650,7 @@ TEST_F(Dense, MakeComplexIsEquivalentToRef)
     auto complex_x = x->make_complex();
     auto dcomplex_x = dx->make_complex();
 
-    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
 }
 
 
@@ -661,7 +661,7 @@ TEST_F(Dense, GetRealIsEquivalentToRef)
     auto real_x = x->get_real();
     auto dreal_x = dx->get_real();
 
-    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
 }
 
 
@@ -672,7 +672,7 @@ TEST_F(Dense, GetImagIsEquivalentToRef)
     auto imag_x = x->get_imag();
     auto dimag_x = dx->get_imag();
 
-    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
 }
 
 
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index 8385fd21205..5268e6d1878 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -633,7 +633,7 @@ TEST_F(Dense, MakeComplexIsEquivalentToRef)
     auto complex_x = x->make_complex();
     auto dcomplex_x = dx->make_complex();
 
-    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
 }
 
 
@@ -644,7 +644,7 @@ TEST_F(Dense, GetRealIsEquivalentToRef)
     auto real_x = x->get_real();
     auto dreal_x = dx->get_real();
 
-    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
 }
 
 
@@ -655,7 +655,7 @@ TEST_F(Dense, GetImagIsEquivalentToRef)
     auto imag_x = x->get_imag();
     auto dimag_x = dx->get_imag();
 
-    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
 }
 
 
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index ce9e85d5d16..b7359599104 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -256,10 +256,22 @@ class Dense
 
     void compute_absolute_inplace() override;
 
+    /**
+     * Creates a complex copy of the original matrix. If the original matrix
+     * was real, the imaginary part of the result will be zero.
+     */
     std::unique_ptr<complex_type> make_complex() const;
 
+    /**
+     * Creates a new real matrix and extracts the real part of the original
+     * matrix into that.
+     */
     std::unique_ptr<absolute_type> get_real() const;
 
+    /**
+     * Creates a new real matrix and extracts the imaginary part of the
+     * original matrix into that.
+     */
     std::unique_ptr<absolute_type> get_imag() const;
 
     /**
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index 90c7880bd85..fc5089fcf8a 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -765,7 +765,7 @@ TEST_F(Dense, MakeComplexIsEquivalentToRef)
     auto complex_x = x->make_complex();
     auto dcomplex_x = dx->make_complex();
 
-    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
 }
 
 
@@ -776,7 +776,7 @@ TEST_F(Dense, GetRealIsEquivalentToRef)
     auto real_x = x->get_real();
     auto dreal_x = dx->get_real();
 
-    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
 }
 
 
@@ -787,7 +787,7 @@ TEST_F(Dense, GetImagIsEquivalentToRef)
     auto imag_x = x->get_imag();
     auto dimag_x = dx->get_imag();
 
-    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
 }
 
 

From 9c7436d65bd6f4a89c5c66956636f4bb9b9543c5 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Wed, 28 Oct 2020 11:13:28 +0100
Subject: [PATCH 0097/1008] Add conversions into given matrix

---
 core/matrix/dense.cpp                   |  33 ++++
 cuda/test/matrix/dense_kernels.cpp      |  39 +++++
 hip/test/matrix/dense_kernels.hip.cpp   |  39 +++++
 include/ginkgo/core/matrix/dense.hpp    |  18 +++
 omp/test/matrix/dense_kernels.cpp       |  39 +++++
 reference/test/matrix/dense_kernels.cpp | 200 ++++++++++++++++++++++--
 6 files changed, 354 insertions(+), 14 deletions(-)

diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 398b5192035..2cb02fb4361 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -812,6 +812,17 @@ Dense<ValueType>::make_complex() const
 }
 
 
+template <typename ValueType>
+void Dense<ValueType>::make_complex(Dense<to_complex<ValueType>> *result) const
+{
+    auto exec = this->get_executor();
+
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, result);
+
+    exec->run(dense::make_make_complex(this, result));
+}
+
+
 template <typename ValueType>
 std::unique_ptr<typename Dense<ValueType>::absolute_type>
 Dense<ValueType>::get_real() const
@@ -826,6 +837,17 @@ Dense<ValueType>::get_real() const
 }
 
 
+template <typename ValueType>
+void Dense<ValueType>::get_real(Dense<remove_complex<ValueType>> *result) const
+{
+    auto exec = this->get_executor();
+
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, result);
+
+    exec->run(dense::make_get_real(this, result));
+}
+
+
 template <typename ValueType>
 std::unique_ptr<typename Dense<ValueType>::absolute_type>
 Dense<ValueType>::get_imag() const
@@ -840,6 +862,17 @@ Dense<ValueType>::get_imag() const
 }
 
 
+template <typename ValueType>
+void Dense<ValueType>::get_imag(Dense<remove_complex<ValueType>> *result) const
+{
+    auto exec = this->get_executor();
+
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, result);
+
+    exec->run(dense::make_get_imag(this, result));
+}
+
+
 #define GKO_DECLARE_DENSE_MATRIX(_type) class Dense<_type>
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_MATRIX);
 
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index ea067231ece..823aabe9b2b 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -654,6 +654,19 @@ TEST_F(Dense, MakeComplexIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, MakeComplexWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto complex_x = ComplexMtx::create(ref, x->get_size());
+    x->make_complex(complex_x.get());
+    auto dcomplex_x = ComplexMtx::create(cuda, x->get_size());
+    dx->make_complex(dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
+}
+
+
 TEST_F(Dense, GetRealIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -665,6 +678,19 @@ TEST_F(Dense, GetRealIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, GetRealWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto real_x = Mtx::create(ref, x->get_size());
+    x->get_real(real_x.get());
+    auto dreal_x = Mtx::create(cuda, dx->get_size());
+    dx->get_real(dreal_x.get());
+
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
+}
+
+
 TEST_F(Dense, GetImagIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -676,4 +702,17 @@ TEST_F(Dense, GetImagIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, GetImagWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto imag_x = Mtx::create(ref, x->get_size());
+    x->get_imag(imag_x.get());
+    auto dimag_x = Mtx::create(cuda, dx->get_size());
+    dx->get_imag(dimag_x.get());
+
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
+}
+
+
 }  // namespace
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index 5268e6d1878..a4f4b4e58f1 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -637,6 +637,19 @@ TEST_F(Dense, MakeComplexIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, MakeComplexWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto complex_x = ComplexMtx::create(ref, x->get_size());
+    x->make_complex(complex_x.get());
+    auto dcomplex_x = ComplexMtx::create(hip, x->get_size());
+    dx->make_complex(dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
+}
+
+
 TEST_F(Dense, GetRealIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -648,6 +661,19 @@ TEST_F(Dense, GetRealIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, GetRealWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto real_x = Mtx::create(ref, x->get_size());
+    x->get_real(real_x.get());
+    auto dreal_x = Mtx::create(hip, dx->get_size());
+    dx->get_real(dreal_x.get());
+
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
+}
+
+
 TEST_F(Dense, GetImagIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -659,4 +685,17 @@ TEST_F(Dense, GetImagIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, GetImagWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto imag_x = Mtx::create(ref, x->get_size());
+    x->get_imag(imag_x.get());
+    auto dimag_x = Mtx::create(hip, dx->get_size());
+    dx->get_imag(dimag_x.get());
+
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
+}
+
+
 }  // namespace
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index b7359599104..7bb439afc34 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -262,18 +262,36 @@ class Dense
      */
     std::unique_ptr<complex_type> make_complex() const;
 
+    /**
+     * Writes a complex copy of the original matrix to a given complex matrix.
+     * If the original matrix was real, the imaginary part of the result will
+     * be zero.
+     */
+    void make_complex(Dense<to_complex<ValueType>> *result) const;
+
     /**
      * Creates a new real matrix and extracts the real part of the original
      * matrix into that.
      */
     std::unique_ptr<absolute_type> get_real() const;
 
+    /**
+     * Extracts the real part of the original matrix into a given real matrix.
+     */
+    void get_real(Dense<remove_complex<ValueType>> *result) const;
+
     /**
      * Creates a new real matrix and extracts the imaginary part of the
      * original matrix into that.
      */
     std::unique_ptr<absolute_type> get_imag() const;
 
+    /**
+     * Extracts the imaginary part of the original matrix into a given real
+     * matrix.
+     */
+    void get_imag(Dense<remove_complex<ValueType>> *result) const;
+
     /**
      * Returns a pointer to the array of values of the matrix.
      *
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index fc5089fcf8a..cf2d975c2ef 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -769,6 +769,19 @@ TEST_F(Dense, MakeComplexIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, MakeComplexWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto complex_x = ComplexMtx::create(ref, x->get_size());
+    x->make_complex(complex_x.get());
+    auto dcomplex_x = ComplexMtx::create(omp, x->get_size());
+    dx->make_complex(dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
+}
+
+
 TEST_F(Dense, GetRealIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -780,6 +793,19 @@ TEST_F(Dense, GetRealIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, GetRealWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto real_x = Mtx::create(ref, x->get_size());
+    x->get_real(real_x.get());
+    auto dreal_x = Mtx::create(omp, dx->get_size());
+    dx->get_real(dreal_x.get());
+
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
+}
+
+
 TEST_F(Dense, GetImagIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -791,4 +817,17 @@ TEST_F(Dense, GetImagIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, GetImagWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto imag_x = Mtx::create(ref, x->get_size());
+    x->get_imag(imag_x.get());
+    auto dimag_x = Mtx::create(omp, dx->get_size());
+    dx->get_imag(dimag_x.get());
+
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
+}
+
+
 }  // namespace
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index b31fd51e52a..562176aefbe 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -65,6 +65,8 @@ class Dense : public ::testing::Test {
 protected:
     using value_type = T;
     using Mtx = gko::matrix::Dense<value_type>;
+    using ComplexMtx = gko::matrix::Dense<gko::to_complex<value_type>>;
+    using RealMtx = gko::matrix::Dense<gko::remove_complex<value_type>>;
     Dense()
         : exec(gko::ReferenceExecutor::create()),
           mtx1(gko::initialize<Mtx>(4, {{1.0, 2.0, 3.0}, {1.5, 2.5, 3.5}},
@@ -2160,9 +2162,45 @@ TYPED_TEST(Dense, MakeComplex)
     // {2.1, 3.4, 1.2}
     // clang-format on
 
-    auto abs_mtx = this->mtx5->make_complex();
+    auto complex_mtx = this->mtx5->make_complex();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx, this->mtx5, r<TypeParam>::value);
+    GKO_ASSERT_MTX_NEAR(complex_mtx, this->mtx5, 0);
+}
+
+
+TYPED_TEST(Dense, MakeComplexWithGivenResult)
+{
+    using T = typename TestFixture::value_type;
+    using ComplexMtx = typename TestFixture::ComplexMtx;
+    auto exec = this->mtx5->get_executor();
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+
+    auto complex_mtx = ComplexMtx::create(exec, this->mtx5->get_size());
+    this->mtx5->make_complex(complex_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(complex_mtx, this->mtx5, 0);
+}
+
+
+TYPED_TEST(Dense, MakeComplexWithGivenResultFailsForWrongDimensions)
+{
+    using T = typename TestFixture::value_type;
+    using ComplexMtx = typename TestFixture::ComplexMtx;
+    auto exec = this->mtx5->get_executor();
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+
+    auto complex_mtx = ComplexMtx::create(exec);
+
+    ASSERT_THROW(this->mtx5->make_complex(complex_mtx.get()),
+                 gko::DimensionMismatch);
 }
 
 
@@ -2175,9 +2213,43 @@ TYPED_TEST(Dense, GetReal)
     // {2.1, 3.4, 1.2}
     // clang-format on
 
-    auto abs_mtx = this->mtx5->get_real();
+    auto real_mtx = this->mtx5->get_real();
+
+    GKO_ASSERT_MTX_NEAR(real_mtx, this->mtx5, 0);
+}
+
+
+TYPED_TEST(Dense, GetRealWithGivenResult)
+{
+    using T = typename TestFixture::value_type;
+    using RealMtx = typename TestFixture::RealMtx;
+    auto exec = this->mtx5->get_executor();
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+
+    auto real_mtx = RealMtx::create(exec, this->mtx5->get_size());
+    this->mtx5->get_real(real_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx, this->mtx5, r<TypeParam>::value);
+    GKO_ASSERT_MTX_NEAR(real_mtx, this->mtx5, 0);
+}
+
+
+TYPED_TEST(Dense, GetRealWithGivenResultFailsForWrongDimensions)
+{
+    using T = typename TestFixture::value_type;
+    using RealMtx = typename TestFixture::RealMtx;
+    auto exec = this->mtx5->get_executor();
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+
+    auto real_mtx = RealMtx::create(exec);
+    ASSERT_THROW(this->mtx5->get_real(real_mtx.get()), gko::DimensionMismatch);
 }
 
 
@@ -2190,11 +2262,45 @@ TYPED_TEST(Dense, GetImag)
     // {2.1, 3.4, 1.2}
     // clang-format on
 
-    auto abs_mtx = this->mtx5->get_imag();
+    auto imag_mtx = this->mtx5->get_imag();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx,
-                        l({{0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}}),
-                        r<TypeParam>::value);
+    GKO_ASSERT_MTX_NEAR(
+        imag_mtx, l({{0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}}), 0);
+}
+
+
+TYPED_TEST(Dense, GetImagWithGivenResult)
+{
+    using T = typename TestFixture::value_type;
+    using RealMtx = typename TestFixture::RealMtx;
+    auto exec = this->mtx5->get_executor();
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+
+    auto imag_mtx = RealMtx::create(exec, this->mtx5->get_size());
+    this->mtx5->get_imag(imag_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        imag_mtx, l({{0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}}), 0);
+}
+
+
+TYPED_TEST(Dense, GetImagWithGivenResultFailsForWrongDimensions)
+{
+    using T = typename TestFixture::value_type;
+    using RealMtx = typename TestFixture::RealMtx;
+    auto exec = this->mtx5->get_executor();
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+
+    auto imag_mtx = RealMtx::create(exec);
+    ASSERT_THROW(this->mtx5->get_imag(imag_mtx.get()), gko::DimensionMismatch);
 }
 
 
@@ -2203,6 +2309,7 @@ class DenseComplex : public ::testing::Test {
 protected:
     using value_type = T;
     using Mtx = gko::matrix::Dense<value_type>;
+    using RealMtx = gko::matrix::Dense<gko::remove_complex<value_type>>;
 };
 
 
@@ -2279,9 +2386,28 @@ TYPED_TEST(DenseComplex, MakeComplex)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    auto abs_mtx = mtx->make_complex();
+    auto complex_mtx = mtx->make_complex();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx, mtx, 0.0);
+    GKO_ASSERT_MTX_NEAR(complex_mtx, mtx, 0.0);
+}
+
+
+TYPED_TEST(DenseComplex, MakeComplexWithGivenResult)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    auto complex_mtx = Mtx::create(exec, mtx->get_size());
+    mtx->make_complex(complex_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(complex_mtx, mtx, 0.0);
 }
 
 
@@ -2297,10 +2423,33 @@ TYPED_TEST(DenseComplex, GetReal)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    auto abs_mtx = mtx->get_real();
+    auto real_mtx = mtx->get_real();
+
+    GKO_ASSERT_MTX_NEAR(
+        real_mtx, l({{1.0, 3.0, 0.0}, {-4.0, -1.0, 0.0}, {0.0, 0.0, 2.0}}),
+        0.0);
+}
+
+
+TYPED_TEST(DenseComplex, GetRealWithGivenResult)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using RealMtx = typename TestFixture::RealMtx;
+    using T = typename TestFixture::value_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    auto real_mtx = RealMtx::create(exec, mtx->get_size());
+    mtx->get_real(real_mtx.get());
 
     GKO_ASSERT_MTX_NEAR(
-        abs_mtx, l({{1.0, 3.0, 0.0}, {-4.0, -1.0, 0.0}, {0.0, 0.0, 2.0}}), 0.0);
+        real_mtx, l({{1.0, 3.0, 0.0}, {-4.0, -1.0, 0.0}, {0.0, 0.0, 2.0}}),
+        0.0);
 }
 
 
@@ -2316,10 +2465,33 @@ TYPED_TEST(DenseComplex, GetImag)
          {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
     // clang-format on
 
-    auto abs_mtx = mtx->get_imag();
+    auto imag_mtx = mtx->get_imag();
 
     GKO_ASSERT_MTX_NEAR(
-        abs_mtx, l({{0.0, 4.0, 2.0}, {-3.0, 0.0, 0.0}, {0.0, -1.5, 0.0}}), 0.0);
+        imag_mtx, l({{0.0, 4.0, 2.0}, {-3.0, 0.0, 0.0}, {0.0, -1.5, 0.0}}),
+        0.0);
+}
+
+
+TYPED_TEST(DenseComplex, GetImagWithGivenResult)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using RealMtx = typename TestFixture::RealMtx;
+    using T = typename TestFixture::value_type;
+    auto exec = gko::ReferenceExecutor::create();
+    // clang-format off
+    auto mtx = gko::initialize<Mtx>(
+        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+    // clang-format on
+
+    auto imag_mtx = RealMtx::create(exec, mtx->get_size());
+    mtx->get_imag(imag_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        imag_mtx, l({{0.0, 4.0, 2.0}, {-3.0, 0.0, 0.0}, {0.0, -1.5, 0.0}}),
+        0.0);
 }
 
 

From b7d2106befd1f3b75cf3c468ff6b03c3bc7373ee Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 29 Oct 2020 13:22:54 +0100
Subject: [PATCH 0098/1008] Use temporary clone in conversions with given
 result

---
 core/matrix/dense.cpp | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 2cb02fb4361..cb350c31628 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -819,7 +819,8 @@ void Dense<ValueType>::make_complex(Dense<to_complex<ValueType>> *result) const
 
     GKO_ASSERT_EQUAL_DIMENSIONS(this, result);
 
-    exec->run(dense::make_make_complex(this, result));
+    exec->run(dense::make_make_complex(
+        this, make_temporary_clone(exec, result).get()));
 }
 
 
@@ -844,7 +845,8 @@ void Dense<ValueType>::get_real(Dense<remove_complex<ValueType>> *result) const
 
     GKO_ASSERT_EQUAL_DIMENSIONS(this, result);
 
-    exec->run(dense::make_get_real(this, result));
+    exec->run(
+        dense::make_get_real(this, make_temporary_clone(exec, result).get()));
 }
 
 
@@ -869,7 +871,8 @@ void Dense<ValueType>::get_imag(Dense<remove_complex<ValueType>> *result) const
 
     GKO_ASSERT_EQUAL_DIMENSIONS(this, result);
 
-    exec->run(dense::make_get_imag(this, result));
+    exec->run(
+        dense::make_get_imag(this, make_temporary_clone(exec, result).get()));
 }
 
 

From 9f81755305b3a50339ba4d3d762f5fe4e3db739c Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 5 Nov 2020 14:49:57 +0100
Subject: [PATCH 0099/1008] Add dpcpp GKO_NOT_IMPLEMENTED kernels, code review

---
 dpcpp/matrix/dense_kernels.dp.cpp       | 27 +++++++++++++++++++++++++
 reference/test/matrix/dense_kernels.cpp |  4 ++--
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index e84519e8192..3f2f823d799 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -308,6 +308,33 @@ void outplace_absolute_dense(std::shared_ptr<const DpcppExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
 
 
+template <typename ValueType>
+void make_complex(std::shared_ptr<const DpcppExecutor> exec,
+                  const matrix::Dense<ValueType> *source,
+                  matrix::Dense<to_complex<ValueType>> *result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MAKE_COMPLEX_KERNEL);
+
+
+template <typename ValueType>
+void get_real(std::shared_ptr<const DpcppExecutor> exec,
+              const matrix::Dense<ValueType> *source,
+              matrix::Dense<remove_complex<ValueType>> *result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_REAL_KERNEL);
+
+
+template <typename ValueType>
+void get_imag(std::shared_ptr<const CudaExecutor> exec,
+              const matrix::Dense<ValueType> *source,
+              matrix::Dense<remove_complex<ValueType>> *result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_IMAG_KERNEL);
+
+
 }  // namespace dense
 }  // namespace dpcpp
 }  // namespace kernels
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 562176aefbe..d6618866de0 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -65,8 +65,8 @@ class Dense : public ::testing::Test {
 protected:
     using value_type = T;
     using Mtx = gko::matrix::Dense<value_type>;
-    using ComplexMtx = gko::matrix::Dense<gko::to_complex<value_type>>;
-    using RealMtx = gko::matrix::Dense<gko::remove_complex<value_type>>;
+    using ComplexMtx = gko::to_complex<Mtx>;
+    using RealMtx = gko::remove_complex<Mtx>;
     Dense()
         : exec(gko::ReferenceExecutor::create()),
           mtx1(gko::initialize<Mtx>(4, {{1.0, 2.0, 3.0}, {1.5, 2.5, 3.5}},

From bea5744e1d5c37eacdd0ef825089c6cb7451421b Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 5 Nov 2020 19:43:31 +0100
Subject: [PATCH 0100/1008] Fix missed dpcpp kernel

---
 dpcpp/matrix/dense_kernels.dp.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 3f2f823d799..fac85ba3a14 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -327,7 +327,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_REAL_KERNEL);
 
 
 template <typename ValueType>
-void get_imag(std::shared_ptr<const CudaExecutor> exec,
+void get_imag(std::shared_ptr<const DpcppExecutor> exec,
               const matrix::Dense<ValueType> *source,
               matrix::Dense<remove_complex<ValueType>> *result)
     GKO_NOT_IMPLEMENTED;

From e35822a2bd35b04d09685c6bf9dbc19d0d82d13b Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 30 Oct 2020 15:35:22 +0100
Subject: [PATCH 0101/1008] extend solver benchmark: add ISAI, custom rhs

---
 benchmark/solver/solver.cpp | 152 +++++++++++++++++++++++++++++++-----
 1 file changed, 133 insertions(+), 19 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index f043977ab9a..81b55f1110f 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -47,6 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/general.hpp"
 #include "benchmark/utils/loggers.hpp"
 #include "benchmark/utils/overhead_linop.hpp"
+#include "ginkgo/core/preconditioner/isai.hpp"
 
 
 // some Ginkgo shortcuts
@@ -68,7 +69,7 @@ DEFINE_string(
     preconditioners, "none",
     "A comma-separated list of preconditioners to use. "
     "Supported values are: none, jacobi, adaptive-jacobi, parict, parilu, "
-    "parilut, ilu, overhead");
+    "parilut, ilu, parict-isai, parilu-isai, parilut-isai, ilu-isai, overhead");
 
 DEFINE_uint32(parilu_iterations, 5,
               "The number of iterations for ParICT/ParILU(T)");
@@ -78,10 +79,17 @@ DEFINE_bool(parilut_approx_select, true,
 
 DEFINE_double(parilut_limit, 2.0, "The fill-in limit for ParICT/ParILUT");
 
+DEFINE_int32(
+    isai_power, 1,
+    "Which power of the sparsity structure to use for ISAI preconditioners");
+
 DEFINE_uint32(
     nrhs, 1,
     "The number of right hand sides. Record the residual only when nrhs == 1.");
 
+DEFINE_bool(random_initial_guess, false,
+            "Use a random vector for the initial guess (otherwise use rhs)");
+
 // This allows to benchmark the overhead of a solver by using the following
 // data: A=[1.0], x=[0.0], b=[nan]. This data can be used to benchmark normal
 // solvers or using the argument --solvers=overhead, a minimal solver will be
@@ -93,13 +101,14 @@ DEFINE_bool(overhead, false,
 // input validation
 [[noreturn]] void print_config_error_and_exit()
 {
-    std::cerr << "Input has to be a JSON array of matrix configurations:\n"
-              << "  [\n"
-              << "    { \"filename\": \"my_file.mtx\",  \"optimal\": { "
-                 "\"spmv\": \"<matrix format>\" } },\n"
-              << "    { \"filename\": \"my_file2.mtx\", \"optimal\": { "
-                 "\"spmv\": \"<matrix format>\" } }\n"
-              << "  ]" << std::endl;
+    std::cerr
+        << "Input has to be a JSON array of matrix configurations:\n"
+        << "  [\n"
+        << "    { \"filename\": \"my_file.mtx\",  \"optimal\": { "
+           "\"spmv\": \"<matrix format>\" }, \"rhs\": \"my_file_rhs.mtx\" },\n"
+        << "    { \"filename\": \"my_file2.mtx\", \"optimal\": { "
+           "\"spmv\": \"<matrix format>\" } }\n"
+        << "  ]" << std::endl;
     std::exit(1);
 }
 
@@ -109,7 +118,8 @@ void validate_option_object(const rapidjson::Value &value)
     if (!value.IsObject() || !value.HasMember("optimal") ||
         !value["optimal"].HasMember("spmv") ||
         !value["optimal"]["spmv"].IsString() || !value.HasMember("filename") ||
-        !value["filename"].IsString()) {
+        !value["filename"].IsString() ||
+        (value.HasMember("rhs") && !value["rhs"].IsString())) {
         print_config_error_and_exit();
     }
 }
@@ -252,6 +262,100 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
              return std::unique_ptr<ReferenceFactoryWrapper>(
                  new ReferenceFactoryWrapper(f));
          }},
+        {"parict-isai",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact = std::shared_ptr<gko::LinOpFactory>(
+                 gko::factorization::ParIct<>::build()
+                     .with_iterations(FLAGS_parilu_iterations)
+                     .with_approximate_select(FLAGS_parilut_approx_select)
+                     .with_fill_in_limit(FLAGS_parilut_limit)
+                     .on(exec));
+             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             std::shared_ptr<const gko::LinOpFactory> f =
+                 gko::preconditioner::Ilu<
+                     gko::preconditioner::LowerIsai<>,
+                     gko::preconditioner::UpperIsai<>>::build()
+                     .with_factorization_factory(fact)
+                     .with_l_solver_factory(lisai)
+                     .with_u_solver_factory(uisai)
+                     .on(exec);
+             return std::unique_ptr<ReferenceFactoryWrapper>(
+                 new ReferenceFactoryWrapper(f));
+         }},
+        {"parilu-isai",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact = std::shared_ptr<gko::LinOpFactory>(
+                 gko::factorization::ParIlu<>::build()
+                     .with_iterations(FLAGS_parilu_iterations)
+                     .on(exec));
+             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             std::shared_ptr<const gko::LinOpFactory> f =
+                 gko::preconditioner::Ilu<
+                     gko::preconditioner::LowerIsai<>,
+                     gko::preconditioner::UpperIsai<>>::build()
+                     .with_factorization_factory(fact)
+                     .with_l_solver_factory(lisai)
+                     .with_u_solver_factory(uisai)
+                     .on(exec);
+             return std::unique_ptr<ReferenceFactoryWrapper>(
+                 new ReferenceFactoryWrapper(f));
+         }},
+        {"parilut-isai",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact = std::shared_ptr<gko::LinOpFactory>(
+                 gko::factorization::ParIlut<>::build()
+                     .with_iterations(FLAGS_parilu_iterations)
+                     .with_approximate_select(FLAGS_parilut_approx_select)
+                     .with_fill_in_limit(FLAGS_parilut_limit)
+                     .on(exec));
+             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             std::shared_ptr<const gko::LinOpFactory> f =
+                 gko::preconditioner::Ilu<
+                     gko::preconditioner::LowerIsai<>,
+                     gko::preconditioner::UpperIsai<>>::build()
+                     .with_factorization_factory(fact)
+                     .with_l_solver_factory(lisai)
+                     .with_u_solver_factory(uisai)
+                     .on(exec);
+             return std::unique_ptr<ReferenceFactoryWrapper>(
+                 new ReferenceFactoryWrapper(f));
+         }},
+        {"ilu-isai",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact = std::shared_ptr<gko::LinOpFactory>(
+                 gko::factorization::Ilu<>::build().on(exec));
+             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             std::shared_ptr<const gko::LinOpFactory> f =
+                 gko::preconditioner::Ilu<
+                     gko::preconditioner::LowerIsai<>,
+                     gko::preconditioner::UpperIsai<>>::build()
+                     .with_factorization_factory(fact)
+                     .with_l_solver_factory(lisai)
+                     .with_u_solver_factory(uisai)
+                     .on(exec);
+             return std::unique_ptr<ReferenceFactoryWrapper>(
+                 new ReferenceFactoryWrapper(f));
+         }},
         {"overhead", [](std::shared_ptr<const gko::Executor> exec) {
              std::shared_ptr<const gko::LinOpFactory> f =
                  gko::Overhead<>::build().on(exec);
@@ -330,7 +434,7 @@ void solve_system(const std::string &solver_name,
                           rapidjson::Value(rapidjson::kArrayType), allocator);
         add_or_set_member(solver_json, "iteration_timestamps",
                           rapidjson::Value(rapidjson::kArrayType), allocator);
-        if (FLAGS_nrhs == 1 && !FLAGS_overhead) {
+        if (b->get_size()[1] == 1 && !FLAGS_overhead) {
             auto rhs_norm = compute_norm2(lend(b));
             add_or_set_member(solver_json, "rhs_norm", rhs_norm, allocator);
         }
@@ -397,7 +501,7 @@ void solve_system(const std::string &solver_name,
                                      allocator, 1);
 
             // slow run, gets the recurrent and true residuals of each iteration
-            if (FLAGS_nrhs == 1) {
+            if (b->get_size()[1] == 1) {
                 x_clone = clone(x);
                 auto res_logger = std::make_shared<ResidualLogger<etype>>(
                     exec, lend(system_matrix), b,
@@ -439,7 +543,7 @@ void solve_system(const std::string &solver_name,
             apply_time += std::chrono::duration_cast<std::chrono::nanoseconds>(
                 a_tac - a_tic);
 
-            if (FLAGS_nrhs == 1 && i == FLAGS_repetitions - 1 &&
+            if (b->get_size()[1] == 1 && i == FLAGS_repetitions - 1 &&
                 !FLAGS_overhead) {
                 auto residual = compute_residual_norm(lend(system_matrix),
                                                       lend(b), lend(x_clone));
@@ -476,7 +580,7 @@ int main(int argc, char *argv[])
     std::string format =
         std::string() + "  [\n" +
         "    { \"filename\": \"my_file.mtx\",  \"optimal\": { "
-        "\"spmv\": \"<matrix format>\" } },\n" +
+        "\"spmv\": \"<matrix format>\" }, \"rhs\": \"my_file_rhs.mtx\" },\n" +
         "    { \"filename\": \"my_file2.mtx\", \"optimal\": { "
         "\"spmv\": \"<matrix format>\" } }\n" +
         "  ]\n\n" +
@@ -553,12 +657,22 @@ int main(int argc, char *argv[])
                 auto data = gko::read_raw<etype>(mtx_fd);
                 system_matrix = share(formats::matrix_factory.at(
                     test_case["optimal"]["spmv"].GetString())(exec, data));
-                b = create_matrix<etype>(
-                    exec, gko::dim<2>{system_matrix->get_size()[0], FLAGS_nrhs},
-                    engine);
-                x = create_matrix<etype>(
-                    exec,
-                    gko::dim<2>{system_matrix->get_size()[0], FLAGS_nrhs});
+                if (test_case.HasMember("rhs")) {
+                    std::ifstream rhs_fd{test_case["rhs"].GetString()};
+                    b = gko::read<Vec>(rhs_fd, exec);
+                } else {
+                    b = create_matrix<etype>(
+                        exec,
+                        gko::dim<2>{system_matrix->get_size()[0], FLAGS_nrhs});
+                }
+                if (FLAGS_random_initial_guess) {
+                    x = create_matrix<etype>(
+                        exec,
+                        gko::dim<2>{system_matrix->get_size()[0], FLAGS_nrhs},
+                        engine);
+                } else {
+                    x = b->clone();
+                }
             }
 
             std::clog << "Matrix is of size (" << system_matrix->get_size()[0]

From 5bef12b2842c9a426126dbeebcc407b6e8cb86b6 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 30 Oct 2020 21:15:05 +0100
Subject: [PATCH 0102/1008] add GMRES Krylov dimension flag

---
 benchmark/solver/solver.cpp | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 81b55f1110f..20f390dd16b 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -87,6 +87,9 @@ DEFINE_uint32(
     nrhs, 1,
     "The number of right hand sides. Record the residual only when nrhs == 1.");
 
+DEFINE_uint32(gmres_restart, 100,
+              "What maximum dimension of the Krylov space to use in GMRES");
+
 DEFINE_bool(random_initial_guess, false,
             "Use a random vector for the initial guess (otherwise use rhs)");
 
@@ -151,7 +154,21 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                    {"cg", create_solver<gko::solver::Cg<>>},
                    {"cgs", create_solver<gko::solver::Cgs<>>},
                    {"fcg", create_solver<gko::solver::Fcg<>>},
-                   {"gmres", create_solver<gko::solver::Gmres<>>},
+                   {"gmres",
+                    [](std::shared_ptr<const gko::Executor> exec,
+                       std::shared_ptr<const gko::LinOpFactory> precond) {
+                        return gko::solver::Gmres<>::build()
+                            .with_criteria(
+                                gko::stop::ResidualNormReduction<>::build()
+                                    .with_reduction_factor(FLAGS_rel_res_goal)
+                                    .on(exec),
+                                gko::stop::Iteration::build()
+                                    .with_max_iters(FLAGS_max_iters)
+                                    .on(exec))
+                            .with_krylov_dim(FLAGS_gmres_restart)
+                            .with_preconditioner(give(precond))
+                            .on(exec);
+                    }},
                    {"overhead", create_solver<gko::Overhead<>>}};
 
 

From 5eeef9ca60fa52b4685b2cb9e2fb72d499871525 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 3 Nov 2020 16:47:40 +0100
Subject: [PATCH 0103/1008] add benchmark relative residual stopping criterion

---
 benchmark/solver/solver.cpp | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 20f390dd16b..a46c5843ae5 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -60,6 +60,10 @@ DEFINE_uint32(max_iters, 1000,
 
 DEFINE_double(rel_res_goal, 1e-6, "The relative residual goal of the solver");
 
+DEFINE_bool(
+    rel_residual, false,
+    "Use relative residual instead of residual reduction stopping criterion");
+
 DEFINE_string(
     solvers, "cg",
     "A comma-separated list of solvers to run. "
@@ -134,13 +138,21 @@ std::unique_ptr<gko::LinOpFactory> create_solver(
     std::shared_ptr<const gko::Executor> exec,
     std::shared_ptr<const gko::LinOpFactory> precond)
 {
-    return SolverType::build()
-        .with_criteria(gko::stop::ResidualNormReduction<>::build()
+    std::shared_ptr<const gko::stop::CriterionFactory> residual_stop;
+    if (FLAGS_rel_residual) {
+        residual_stop = gko::share(gko::stop::RelativeResidualNorm<>::build()
+                                       .with_tolerance(FLAGS_rel_res_goal)
+                                       .on(exec));
+    } else {
+        residual_stop =
+            gko::share(gko::stop::ResidualNormReduction<>::build()
                            .with_reduction_factor(FLAGS_rel_res_goal)
-                           .on(exec),
-                       gko::stop::Iteration::build()
-                           .with_max_iters(FLAGS_max_iters)
-                           .on(exec))
+                           .on(exec));
+    }
+    return SolverType::build()
+        .with_criteria(residual_stop, gko::stop::Iteration::build()
+                                          .with_max_iters(FLAGS_max_iters)
+                                          .on(exec))
         .with_preconditioner(give(precond))
         .on(exec);
 }

From 00050cc0aaac36eabb36039ed54ec57294ab0a50 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 3 Nov 2020 16:48:53 +0100
Subject: [PATCH 0104/1008] add constant rhs to solver benchmark

---
 benchmark/solver/solver.cpp |  6 +++++-
 benchmark/utils/general.hpp | 10 +++++++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index a46c5843ae5..f68654b16bb 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -94,6 +94,9 @@ DEFINE_uint32(
 DEFINE_uint32(gmres_restart, 100,
               "What maximum dimension of the Krylov space to use in GMRES");
 
+DEFINE_bool(random_rhs, false,
+            "Use a random vector for the rhs (otherwise use all ones)");
+
 DEFINE_bool(random_initial_guess, false,
             "Use a random vector for the initial guess (otherwise use rhs)");
 
@@ -692,7 +695,8 @@ int main(int argc, char *argv[])
                 } else {
                     b = create_matrix<etype>(
                         exec,
-                        gko::dim<2>{system_matrix->get_size()[0], FLAGS_nrhs});
+                        gko::dim<2>{system_matrix->get_size()[0], FLAGS_nrhs},
+                        engine, FLAGS_random_rhs);
                 }
                 if (FLAGS_random_initial_guess) {
                     x = create_matrix<etype>(
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index fcf3b5df379..d4e6607ff5f 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -311,11 +311,15 @@ std::unique_ptr<vec<ValueType>> create_matrix(
 template <typename ValueType, typename RandomEngine>
 std::unique_ptr<vec<ValueType>> create_matrix(
     std::shared_ptr<const gko::Executor> exec, gko::dim<2> size,
-    RandomEngine &engine)
+    RandomEngine &engine, bool random = true)
 {
     auto res = vec<ValueType>::create(exec);
-    res->read(gko::matrix_data<ValueType>(
-        size, std::uniform_real_distribution<>(-1.0, 1.0), engine));
+    if (random) {
+        res->read(gko::matrix_data<ValueType>(
+            size, std::uniform_real_distribution<>(-1.0, 1.0), engine));
+    } else {
+        res->read(gko::matrix_data<ValueType>(size, gko::one<ValueType>()));
+    }
     return res;
 }
 

From ba96142206692d28b0f49a73dc6e8f27798e05ee Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 5 Nov 2020 11:23:59 +0100
Subject: [PATCH 0105/1008] consolidate preconditioner flags for benchmarks

---
 benchmark/preconditioner/preconditioner.cpp | 144 ++++---------
 benchmark/solver/solver.cpp                 | 228 +-------------------
 benchmark/utils/preconditioners.hpp         | 212 ++++++++++++++++++
 3 files changed, 250 insertions(+), 334 deletions(-)
 create mode 100644 benchmark/utils/preconditioners.hpp

diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index 11979fd6ba1..0745ebc6037 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -45,112 +45,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/formats.hpp"
 #include "benchmark/utils/general.hpp"
 #include "benchmark/utils/loggers.hpp"
+#include "benchmark/utils/preconditioners.hpp"
 #include "benchmark/utils/spmv_common.hpp"
 
 
-// Command-line arguments
-DEFINE_uint32(max_block_size, 32,
-              "Maximal block size of the block-Jacobi preconditioner");
-
-DEFINE_uint32(num_iterations, 5,
-              "Number of iterations for the ParICT/ParILU(T) preconditioner");
-
-DEFINE_bool(
-    approx_select, true,
-    "Use approximate selection for the threshold filtering in ParICT/ParILUT");
-
-DEFINE_double(fill_limit, 2.0, "The fill-in limit used in ParICT/ParILUT");
-
-DEFINE_string(preconditioners, "jacobi,parilu,parilut,ilu",
-              "A comma-separated list of preconditioners to run."
-              "Supported values are: jacobi, parict, parilu, parilut, ilu");
-
-DEFINE_string(storage_optimization, "0,0",
-              "Defines the kind of storage optimization to perform on "
-              "preconditioners that support it. Supported values are: "
-              "autodetect and <X>,<Y> where <X> and <Y> are the input "
-              "parameters used to construct a precision_reduction object.");
-
-DEFINE_double(accuracy, 1e-1,
-              "This value is used as the accuracy flag of the adaptive Jacobi "
-              "preconditioner.");
-
-
 // some shortcuts
 using etype = double;
 
 
-// parses the storage optimization command line argument
-gko::precision_reduction parse_storage_optimization(const std::string &flag)
-{
-    if (flag == "autodetect") {
-        return gko::precision_reduction::autodetect();
-    }
-    const auto parts = split(flag, ',');
-    if (parts.size() != 2) {
-        throw std::runtime_error(
-            "storage_optimization has to be a list of two integers");
-    }
-    return gko::precision_reduction(std::stoi(parts[0]), std::stoi(parts[1]));
-}
-
-
-// preconditioner mapping
-const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
-                                std::shared_ptr<const gko::Executor> exec)>>
-    precond_factory{
-        {"jacobi",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             return gko::preconditioner::Jacobi<etype>::build()
-                 .with_max_block_size(FLAGS_max_block_size)
-                 .with_storage_optimization(
-                     parse_storage_optimization(FLAGS_storage_optimization))
-                 .with_accuracy(FLAGS_accuracy)
-                 .on(exec);
-         }},
-        {"parict",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             auto ict_fact = std::shared_ptr<gko::LinOpFactory>(
-                 gko::factorization::ParIct<etype>::build()
-                     .with_iterations(FLAGS_num_iterations)
-                     .with_approximate_select(FLAGS_approx_select)
-                     .with_fill_in_limit(FLAGS_fill_limit)
-                     .on(exec));
-             return gko::preconditioner::Ilu<>::build()
-                 .with_factorization_factory(ict_fact)
-                 .on(exec);
-         }},
-        {"parilu",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             auto ilu_fact = std::shared_ptr<gko::LinOpFactory>(
-                 gko::factorization::ParIlu<etype>::build()
-                     .with_iterations(FLAGS_num_iterations)
-                     .on(exec));
-             return gko::preconditioner::Ilu<>::build()
-                 .with_factorization_factory(ilu_fact)
-                 .on(exec);
-         }},
-        {"parilut",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             auto ilut_fact = std::shared_ptr<gko::LinOpFactory>(
-                 gko::factorization::ParIlut<etype>::build()
-                     .with_iterations(FLAGS_num_iterations)
-                     .with_approximate_select(FLAGS_approx_select)
-                     .with_fill_in_limit(FLAGS_fill_limit)
-                     .on(exec));
-             return gko::preconditioner::Ilu<>::build()
-                 .with_factorization_factory(ilut_fact)
-                 .on(exec);
-         }},
-        {"ilu", [](std::shared_ptr<const gko::Executor> exec) {
-             auto ilu_fact = std::shared_ptr<gko::LinOpFactory>(
-                 gko::factorization::Ilu<etype>::build().on(exec));
-             return gko::preconditioner::Ilu<>::build()
-                 .with_factorization_factory(ilu_fact)
-                 .on(exec);
-         }}};
-
-
 // preconditioner generation and application
 
 std::string encode_parameters(const char *precond_name)
@@ -159,31 +61,59 @@ std::string encode_parameters(const char *precond_name)
         {"jacobi",
          [] {
              std::ostringstream oss;
-             oss << "jacobi-" << FLAGS_max_block_size << "-"
-                 << FLAGS_storage_optimization;
+             oss << "jacobi-" << FLAGS_jacobi_max_block_size << "-"
+                 << FLAGS_jacobi_storage;
              return oss.str();
          }},
         {"parict",
          [] {
              std::ostringstream oss;
-             oss << "parict-" << FLAGS_num_iterations << '-'
-                 << FLAGS_approx_select << '-' << FLAGS_fill_limit;
+             oss << "parict-" << FLAGS_parilu_iterations << '-'
+                 << FLAGS_parilut_approx_select << '-' << FLAGS_parilut_limit;
              return oss.str();
          }},
         {"parilu",
          [] {
              std::ostringstream oss;
-             oss << "parilu-" << FLAGS_num_iterations;
+             oss << "parilu-" << FLAGS_parilu_iterations;
              return oss.str();
          }},
         {"parilut",
          [] {
              std::ostringstream oss;
-             oss << "parilut-" << FLAGS_num_iterations << '-'
-                 << FLAGS_approx_select << '-' << FLAGS_fill_limit;
+             oss << "parilut-" << FLAGS_parilu_iterations << '-'
+                 << FLAGS_parilut_approx_select << '-' << FLAGS_parilut_limit;
              return oss.str();
          }},
-        {"ilu", [] { return std::string{"ilu"}; }}};
+        {"parict-isai",
+         [] {
+             std::ostringstream oss;
+             oss << "parict-isai-" << FLAGS_parilu_iterations << '-'
+                 << FLAGS_parilut_approx_select << '-' << FLAGS_parilut_limit
+                 << '-' << FLAGS_isai_power;
+             return oss.str();
+         }},
+        {"parilu-isai",
+         [] {
+             std::ostringstream oss;
+             oss << "parilu-isai-" << FLAGS_parilu_iterations << '-'
+                 << FLAGS_isai_power;
+             return oss.str();
+         }},
+        {"parilut-isai",
+         [] {
+             std::ostringstream oss;
+             oss << "parilut-isai-" << FLAGS_parilu_iterations << '-'
+                 << FLAGS_parilut_approx_select << '-' << FLAGS_parilut_limit
+                 << '-' << FLAGS_isai_power;
+             return oss.str();
+         }},
+        {"ilu-isai", [] {
+             return std::string{"ilu-isai-"} + std::to_string(FLAGS_isai_power);
+         }}};
+    if (encoder.find(precond_name) == encoder.end()) {
+        return precond_name;
+    }
     return encoder[precond_name]();
 }
 
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index f68654b16bb..91a571ec6dc 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -47,6 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/general.hpp"
 #include "benchmark/utils/loggers.hpp"
 #include "benchmark/utils/overhead_linop.hpp"
+#include "benchmark/utils/preconditioners.hpp"
 #include "ginkgo/core/preconditioner/isai.hpp"
 
 
@@ -69,24 +70,6 @@ DEFINE_string(
     "A comma-separated list of solvers to run. "
     "Supported values are: bicgstab, bicg, cg, cgs, fcg, gmres, overhead");
 
-DEFINE_string(
-    preconditioners, "none",
-    "A comma-separated list of preconditioners to use. "
-    "Supported values are: none, jacobi, adaptive-jacobi, parict, parilu, "
-    "parilut, ilu, parict-isai, parilu-isai, parilut-isai, ilu-isai, overhead");
-
-DEFINE_uint32(parilu_iterations, 5,
-              "The number of iterations for ParICT/ParILU(T)");
-
-DEFINE_bool(parilut_approx_select, true,
-            "Use approximate selection for ParICT/ParILUT");
-
-DEFINE_double(parilut_limit, 2.0, "The fill-in limit for ParICT/ParILUT");
-
-DEFINE_int32(
-    isai_power, 1,
-    "Which power of the sparsity structure to use for ISAI preconditioners");
-
 DEFINE_uint32(
     nrhs, 1,
     "The number of right hand sides. Record the residual only when nrhs == 1.");
@@ -187,215 +170,6 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                    {"overhead", create_solver<gko::Overhead<>>}};
 
 
-// TODO: Workaround until GPU matrix conversions are implemented
-//       The factory will wrap another factory, and make sure that the
-//       input operator is copied to the reference executor, and then sent
-//       through the generate function
-struct ReferenceFactoryWrapper
-    : gko::EnablePolymorphicObject<ReferenceFactoryWrapper, gko::LinOpFactory> {
-    ReferenceFactoryWrapper(std::shared_ptr<const gko::Executor> exec)
-        : gko::EnablePolymorphicObject<ReferenceFactoryWrapper,
-                                       gko::LinOpFactory>(exec)
-    {}
-
-    ReferenceFactoryWrapper(std::shared_ptr<const gko::LinOpFactory> f)
-        : gko::EnablePolymorphicObject<ReferenceFactoryWrapper,
-                                       gko::LinOpFactory>(f->get_executor()),
-          base_factory{f}
-    {}
-
-    std::shared_ptr<const gko::Executor> exec{gko::ReferenceExecutor::create()};
-    std::shared_ptr<const gko::LinOpFactory> base_factory;
-
-protected:
-    std::unique_ptr<gko::LinOp> generate_impl(
-        std::shared_ptr<const gko::LinOp> op) const override
-    {
-        return base_factory->generate(gko::clone(exec, op));
-    }
-};
-
-
-const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
-                                std::shared_ptr<const gko::Executor>)>>
-    precond_factory{
-        {"none",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             return gko::matrix::IdentityFactory<>::create(exec);
-         }},
-        {"jacobi",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             std::shared_ptr<const gko::LinOpFactory> f =
-                 gko::preconditioner::Jacobi<>::build().on(exec);
-             return std::unique_ptr<ReferenceFactoryWrapper>(
-                 new ReferenceFactoryWrapper(f));
-         }},
-        {"adaptive-jacobi",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             std::shared_ptr<const gko::LinOpFactory> f =
-                 gko::preconditioner::Jacobi<>::build()
-                     .with_storage_optimization(
-                         gko::precision_reduction::autodetect())
-                     .on(exec);
-             return std::unique_ptr<ReferenceFactoryWrapper>(
-                 new ReferenceFactoryWrapper(f));
-         }},
-        {"parict",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             auto fact = std::shared_ptr<gko::LinOpFactory>(
-                 gko::factorization::ParIct<>::build()
-                     .with_iterations(FLAGS_parilu_iterations)
-                     .with_approximate_select(FLAGS_parilut_approx_select)
-                     .with_fill_in_limit(FLAGS_parilut_limit)
-                     .on(exec));
-             std::shared_ptr<const gko::LinOpFactory> f =
-                 gko::preconditioner::Ilu<>::build()
-                     .with_factorization_factory(fact)
-                     .on(exec);
-             return std::unique_ptr<ReferenceFactoryWrapper>(
-                 new ReferenceFactoryWrapper(f));
-         }},
-        {"parilu",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             auto fact = std::shared_ptr<gko::LinOpFactory>(
-                 gko::factorization::ParIlu<>::build()
-                     .with_iterations(FLAGS_parilu_iterations)
-                     .on(exec));
-             std::shared_ptr<const gko::LinOpFactory> f =
-                 gko::preconditioner::Ilu<>::build()
-                     .with_factorization_factory(fact)
-                     .on(exec);
-             return std::unique_ptr<ReferenceFactoryWrapper>(
-                 new ReferenceFactoryWrapper(f));
-         }},
-        {"parilut",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             auto fact = std::shared_ptr<gko::LinOpFactory>(
-                 gko::factorization::ParIlut<>::build()
-                     .with_iterations(FLAGS_parilu_iterations)
-                     .with_approximate_select(FLAGS_parilut_approx_select)
-                     .with_fill_in_limit(FLAGS_parilut_limit)
-                     .on(exec));
-             std::shared_ptr<const gko::LinOpFactory> f =
-                 gko::preconditioner::Ilu<>::build()
-                     .with_factorization_factory(fact)
-                     .on(exec);
-             return std::unique_ptr<ReferenceFactoryWrapper>(
-                 new ReferenceFactoryWrapper(f));
-         }},
-        {"ilu",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             auto fact = std::shared_ptr<gko::LinOpFactory>(
-                 gko::factorization::Ilu<>::build().on(exec));
-             std::shared_ptr<const gko::LinOpFactory> f =
-                 gko::preconditioner::Ilu<>::build()
-                     .with_factorization_factory(fact)
-                     .on(exec);
-             return std::unique_ptr<ReferenceFactoryWrapper>(
-                 new ReferenceFactoryWrapper(f));
-         }},
-        {"parict-isai",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             auto fact = std::shared_ptr<gko::LinOpFactory>(
-                 gko::factorization::ParIct<>::build()
-                     .with_iterations(FLAGS_parilu_iterations)
-                     .with_approximate_select(FLAGS_parilut_approx_select)
-                     .with_fill_in_limit(FLAGS_parilut_limit)
-                     .on(exec));
-             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
-             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
-             std::shared_ptr<const gko::LinOpFactory> f =
-                 gko::preconditioner::Ilu<
-                     gko::preconditioner::LowerIsai<>,
-                     gko::preconditioner::UpperIsai<>>::build()
-                     .with_factorization_factory(fact)
-                     .with_l_solver_factory(lisai)
-                     .with_u_solver_factory(uisai)
-                     .on(exec);
-             return std::unique_ptr<ReferenceFactoryWrapper>(
-                 new ReferenceFactoryWrapper(f));
-         }},
-        {"parilu-isai",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             auto fact = std::shared_ptr<gko::LinOpFactory>(
-                 gko::factorization::ParIlu<>::build()
-                     .with_iterations(FLAGS_parilu_iterations)
-                     .on(exec));
-             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
-             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
-             std::shared_ptr<const gko::LinOpFactory> f =
-                 gko::preconditioner::Ilu<
-                     gko::preconditioner::LowerIsai<>,
-                     gko::preconditioner::UpperIsai<>>::build()
-                     .with_factorization_factory(fact)
-                     .with_l_solver_factory(lisai)
-                     .with_u_solver_factory(uisai)
-                     .on(exec);
-             return std::unique_ptr<ReferenceFactoryWrapper>(
-                 new ReferenceFactoryWrapper(f));
-         }},
-        {"parilut-isai",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             auto fact = std::shared_ptr<gko::LinOpFactory>(
-                 gko::factorization::ParIlut<>::build()
-                     .with_iterations(FLAGS_parilu_iterations)
-                     .with_approximate_select(FLAGS_parilut_approx_select)
-                     .with_fill_in_limit(FLAGS_parilut_limit)
-                     .on(exec));
-             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
-             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
-             std::shared_ptr<const gko::LinOpFactory> f =
-                 gko::preconditioner::Ilu<
-                     gko::preconditioner::LowerIsai<>,
-                     gko::preconditioner::UpperIsai<>>::build()
-                     .with_factorization_factory(fact)
-                     .with_l_solver_factory(lisai)
-                     .with_u_solver_factory(uisai)
-                     .on(exec);
-             return std::unique_ptr<ReferenceFactoryWrapper>(
-                 new ReferenceFactoryWrapper(f));
-         }},
-        {"ilu-isai",
-         [](std::shared_ptr<const gko::Executor> exec) {
-             auto fact = std::shared_ptr<gko::LinOpFactory>(
-                 gko::factorization::Ilu<>::build().on(exec));
-             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
-             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
-             std::shared_ptr<const gko::LinOpFactory> f =
-                 gko::preconditioner::Ilu<
-                     gko::preconditioner::LowerIsai<>,
-                     gko::preconditioner::UpperIsai<>>::build()
-                     .with_factorization_factory(fact)
-                     .with_l_solver_factory(lisai)
-                     .with_u_solver_factory(uisai)
-                     .on(exec);
-             return std::unique_ptr<ReferenceFactoryWrapper>(
-                 new ReferenceFactoryWrapper(f));
-         }},
-        {"overhead", [](std::shared_ptr<const gko::Executor> exec) {
-             std::shared_ptr<const gko::LinOpFactory> f =
-                 gko::Overhead<>::build().on(exec);
-             return std::unique_ptr<ReferenceFactoryWrapper>(
-                 new ReferenceFactoryWrapper(f));
-         }}};
-
-
 void write_precond_info(const gko::LinOp *precond,
                         rapidjson::Value &precond_info,
                         rapidjson::MemoryPoolAllocator<> &allocator)
diff --git a/benchmark/utils/preconditioners.hpp b/benchmark/utils/preconditioners.hpp
new file mode 100644
index 00000000000..9f61c99f9a7
--- /dev/null
+++ b/benchmark/utils/preconditioners.hpp
@@ -0,0 +1,212 @@
+
+#ifndef GKO_BENCHMARK_UTILS_PRECONDITIONERS_HPP_
+#define GKO_BENCHMARK_UTILS_PRECONDITIONERS_HPP_
+
+
+#include <ginkgo/ginkgo.hpp>
+
+
+#include <map>
+#include <string>
+
+
+#include <gflags/gflags.h>
+
+
+#include "benchmark/utils/general.hpp"
+#include "benchmark/utils/overhead_linop.hpp"
+
+
+DEFINE_string(
+    preconditioners, "none",
+    "A comma-separated list of preconditioners to use. "
+    "Supported values are: none, jacobi, parict, parilu, parilut, ilu, "
+    "parict-isai, parilu-isai, parilut-isai, ilu-isai, overhead");
+
+DEFINE_uint32(parilu_iterations, 5,
+              "The number of iterations for ParICT/ParILU(T)");
+
+DEFINE_bool(parilut_approx_select, true,
+            "Use approximate selection for ParICT/ParILUT");
+
+DEFINE_double(parilut_limit, 2.0, "The fill-in limit for ParICT/ParILUT");
+
+DEFINE_int32(
+    isai_power, 1,
+    "Which power of the sparsity structure to use for ISAI preconditioners");
+
+DEFINE_string(jacobi_storage, "0,0",
+              "Defines the kind of storage optimization to perform on "
+              "preconditioners that support it. Supported values are: "
+              "autodetect and <X>,<Y> where <X> and <Y> are the input "
+              "parameters used to construct a precision_reduction object.");
+
+DEFINE_double(jacobi_accuracy, 1e-1,
+              "This value is used as the accuracy flag of the adaptive Jacobi "
+              "preconditioner.");
+
+DEFINE_uint32(jacobi_max_block_size, 32,
+              "Maximal block size of the block-Jacobi preconditioner");
+
+
+// parses the Jacobi storage optimization command line argument
+gko::precision_reduction parse_storage_optimization(const std::string &flag)
+{
+    if (flag == "autodetect") {
+        return gko::precision_reduction::autodetect();
+    }
+    const auto parts = split(flag, ',');
+    if (parts.size() != 2) {
+        throw std::runtime_error(
+            "storage_optimization has to be a list of two integers");
+    }
+    return gko::precision_reduction(std::stoi(parts[0]), std::stoi(parts[1]));
+}
+
+
+const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
+                                std::shared_ptr<const gko::Executor>)>>
+    precond_factory{
+        {"none",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             return gko::matrix::IdentityFactory<>::create(exec);
+         }},
+        {"jacobi",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             return gko::preconditioner::Jacobi<>::build()
+                 .with_max_block_size(FLAGS_jacobi_max_block_size)
+                 .with_storage_optimization(
+                     parse_storage_optimization(FLAGS_jacobi_storage))
+                 .with_accuracy(FLAGS_jacobi_accuracy)
+                 .on(exec);
+         }},
+        {"parict",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact = gko::share(
+                 gko::factorization::ParIct<>::build()
+                     .with_iterations(FLAGS_parilu_iterations)
+                     .with_approximate_select(FLAGS_parilut_approx_select)
+                     .with_fill_in_limit(FLAGS_parilut_limit)
+                     .on(exec));
+             return gko::preconditioner::Ilu<>::build()
+                 .with_factorization_factory(fact)
+                 .on(exec);
+         }},
+        {"parilu",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact =
+                 gko::share(gko::factorization::ParIlu<>::build()
+                                .with_iterations(FLAGS_parilu_iterations)
+                                .on(exec));
+             return gko::preconditioner::Ilu<>::build()
+                 .with_factorization_factory(fact)
+                 .on(exec);
+         }},
+        {"parilut",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact = gko::share(
+                 gko::factorization::ParIlut<>::build()
+                     .with_iterations(FLAGS_parilu_iterations)
+                     .with_approximate_select(FLAGS_parilut_approx_select)
+                     .with_fill_in_limit(FLAGS_parilut_limit)
+                     .on(exec));
+             return gko::preconditioner::Ilu<>::build()
+                 .with_factorization_factory(fact)
+                 .on(exec);
+         }},
+        {"ilu",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact =
+                 gko::share(gko::factorization::Ilu<>::build().on(exec));
+             return gko::preconditioner::Ilu<>::build()
+                 .with_factorization_factory(fact)
+                 .on(exec);
+         }},
+        {"parict-isai",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact = gko::share(
+                 gko::factorization::ParIct<>::build()
+                     .with_iterations(FLAGS_parilu_iterations)
+                     .with_approximate_select(FLAGS_parilut_approx_select)
+                     .with_fill_in_limit(FLAGS_parilut_limit)
+                     .on(exec));
+             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             return gko::preconditioner::Ilu<
+                        gko::preconditioner::LowerIsai<>,
+                        gko::preconditioner::UpperIsai<>>::build()
+                 .with_factorization_factory(fact)
+                 .with_l_solver_factory(lisai)
+                 .with_u_solver_factory(uisai)
+                 .on(exec);
+         }},
+        {"parilu-isai",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact =
+                 gko::share(gko::factorization::ParIlu<>::build()
+                                .with_iterations(FLAGS_parilu_iterations)
+                                .on(exec));
+             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             return gko::preconditioner::Ilu<
+                        gko::preconditioner::LowerIsai<>,
+                        gko::preconditioner::UpperIsai<>>::build()
+                 .with_factorization_factory(fact)
+                 .with_l_solver_factory(lisai)
+                 .with_u_solver_factory(uisai)
+                 .on(exec);
+         }},
+        {"parilut-isai",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact = gko::share(
+                 gko::factorization::ParIlut<>::build()
+                     .with_iterations(FLAGS_parilu_iterations)
+                     .with_approximate_select(FLAGS_parilut_approx_select)
+                     .with_fill_in_limit(FLAGS_parilut_limit)
+                     .on(exec));
+             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             return gko::preconditioner::Ilu<
+                        gko::preconditioner::LowerIsai<>,
+                        gko::preconditioner::UpperIsai<>>::build()
+                 .with_factorization_factory(fact)
+                 .with_l_solver_factory(lisai)
+                 .with_u_solver_factory(uisai)
+                 .on(exec);
+         }},
+        {"ilu-isai",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact =
+                 gko::share(gko::factorization::Ilu<>::build().on(exec));
+             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
+                                         .with_sparsity_power(FLAGS_isai_power)
+                                         .on(exec));
+             return gko::preconditioner::Ilu<
+                        gko::preconditioner::LowerIsai<>,
+                        gko::preconditioner::UpperIsai<>>::build()
+                 .with_factorization_factory(fact)
+                 .with_l_solver_factory(lisai)
+                 .with_u_solver_factory(uisai)
+                 .on(exec);
+         }},
+        {"overhead", [](std::shared_ptr<const gko::Executor> exec) {
+             return gko::Overhead<>::build().on(exec);
+         }}};
+
+
+#endif  // GKO_BENCHMARK_UTILS_PRECONDITIONERS_HPP_

From 7f790a58e4f6ea7e3808dba9e89bbe51cf7ffe71 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 5 Nov 2020 20:19:21 +0100
Subject: [PATCH 0106/1008] update benchmark runner and documentation

---
 BENCHMARKING.md                 |  6 ++++--
 benchmark/run_all_benchmarks.sh | 17 ++++++++++++++---
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/BENCHMARKING.md b/BENCHMARKING.md
index ff5306ea432..a26cfe5ee1a 100644
--- a/BENCHMARKING.md
+++ b/BENCHMARKING.md
@@ -282,8 +282,8 @@ The supported environment variables are described in the following list:
     benchmark runs (downloads the collections, creates the result structure,
     etc.) and outputs the list of commands that would normally be run, but does
     not run the benchmarks themselves. Default is `false`.
-* `PRECONDS={jacobi,adaptive-jacobi,ilu,parict,parilu,parilut,none}` the
-    preconditioners to use for either `solver` or `preconditioner` benchmarks.
+* `PRECONDS={jacobi,ilu,parict,parilu,parilut,ilu-isai,parict-isai,parilu-isai,parilut-isai,none}`
+    the preconditioners to use for either `solver` or `preconditioner` benchmarks.
     Multiple options can be passed to this variable. Default is `none`.
 * `FORMATS={csr,coo,ell,hybrid,sellp,hybridxx,cusp_xx,hipsp_xx}` the matrix
     formats to benchmark for the `spmv` phase of the benchmark. Run
@@ -299,5 +299,7 @@ The supported environment variables are described in the following list:
     the solver should stop. The default is `1e-6`.
 * `SOLVERS_MAX_ITERATION=<number>` - the maximum number of iterations with which
     a solver should be ran. The default is `10000`.
+* `SOLVERS_RHS={unit, random}` - whether to use a vector of all ones or random
+    values as the right-hand side in solver benchmarks. Default is `unit`.
 * `DETAILED={0,1}` - selects whether detailed benchmarks should be ran for the
     solver benchmarks, can be either `0` (off) or `1` (on). The default is `0`.
diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index 6a782bac322..226fb4b2992 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -60,6 +60,17 @@ if [ ! "${DEVICE_ID}" ]; then
     DEVICE_ID="0"
 fi
 
+if [ ! "${SOLVERS_RHS}" ]; then
+    echo "SOLVERS_RHS environment variable not set - assuming \"unit\"" 1>&2
+    SOLVERS_RHS="unit"
+fi
+
+if [ "${SOLVERS_RHS}" == "random" ]; then
+    SOLVERS_RHS_FLAG="--random_rhs=true"
+else
+    SOLVERS_RHS_FLAG="--random_rhs=false"
+fi
+
 # Control whether to run detailed benchmarks or not.
 # Default setting is detailed=false. To activate, set DETAILED=1.
 if  [ ! "${DETAILED}" ] || [ "${DETAILED}" -eq 0 ]; then
@@ -167,7 +178,7 @@ run_solver_benchmarks() {
                     --executor="${EXECUTOR}" --solvers="${SOLVERS}" \
                     --preconditioners="${PRECONDS}" \
                     --max_iters=${SOLVERS_MAX_ITERATIONS} --rel_res_goal=${SOLVERS_PRECISION} \
-                    ${DETAILED_STR} --device_id="${DEVICE_ID}" \
+                    ${SOLVERS_RHS_FLAG} ${DETAILED_STR} --device_id="${DEVICE_ID}" \
                     <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }
@@ -191,8 +202,8 @@ run_preconditioner_benchmarks() {
             ./preconditioner/preconditioner \
                 --backup="$1.bkp" --double_buffer="$1.bkp2" \
                 --executor="${EXECUTOR}" --preconditioners="jacobi" \
-                --max_block_size="${bsize}" \
-                --storage_optimization="${prec}" \
+                --jacobi_max_block_size="${bsize}" \
+                --jacobi_storage="${prec}" \
                 --device_id="${DEVICE_ID}" \
                 <"$1.imd" 2>&1 >"$1"
             keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"

From bfa4292f461572d543a36d1347a159f524b3b78f Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 10 Nov 2020 10:33:51 +0100
Subject: [PATCH 0107/1008] improve solver benchmark error output

---
 benchmark/solver/solver.cpp | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 91a571ec6dc..f5cfeae39ea 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -94,14 +94,14 @@ DEFINE_bool(overhead, false,
 // input validation
 [[noreturn]] void print_config_error_and_exit()
 {
-    std::cerr
-        << "Input has to be a JSON array of matrix configurations:\n"
-        << "  [\n"
-        << "    { \"filename\": \"my_file.mtx\",  \"optimal\": { "
-           "\"spmv\": \"<matrix format>\" }, \"rhs\": \"my_file_rhs.mtx\" },\n"
-        << "    { \"filename\": \"my_file2.mtx\", \"optimal\": { "
-           "\"spmv\": \"<matrix format>\" } }\n"
-        << "  ]" << std::endl;
+    std::cerr << "Input has to be a JSON array of matrix configurations:\n"
+              << "  [\n"
+              << "    { \"filename\": \"my_file.mtx\",  \"optimal\": { "
+                 "\"spmv\": \"<matrix format>\" },\n"
+                 "      \"rhs\": \"my_file_rhs.mtx\" },\n"
+              << "    { \"filename\": \"my_file2.mtx\", \"optimal\": { "
+                 "\"spmv\": \"<matrix format>\" } }\n"
+              << "  ]" << std::endl;
     std::exit(1);
 }
 
@@ -386,7 +386,8 @@ int main(int argc, char *argv[])
     std::string format =
         std::string() + "  [\n" +
         "    { \"filename\": \"my_file.mtx\",  \"optimal\": { "
-        "\"spmv\": \"<matrix format>\" }, \"rhs\": \"my_file_rhs.mtx\" },\n" +
+        "\"spmv\": \"<matrix format>\" },\n"
+        "      \"rhs\": \"my_file_rhs.mtx\" },\n" +
         "    { \"filename\": \"my_file2.mtx\", \"optimal\": { "
         "\"spmv\": \"<matrix format>\" } }\n" +
         "  ]\n\n" +

From 7dd842bfaf1ac0915668af95b71e9902c16c9824 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Thu, 5 Nov 2020 17:13:15 +0100
Subject: [PATCH 0108/1008] Fix new algorithm script and DPC++ flags.

+ Add DPC++ to the new algorithm script.
+ Extract DPC++ version in CMake
+ Add the proper flag for GPU complex support.
---
 cmake/GinkgoConfig.cmake.in               |  4 ++++
 cmake/build_helpers.cmake                 | 20 ++++++++++++++++++++
 dev_tools/scripts/create_new_algorithm.sh | 14 ++++++++++++++
 dpcpp/CMakeLists.txt                      |  6 +++++-
 include/ginkgo/core/base/math.hpp         |  9 +++------
 5 files changed, 46 insertions(+), 7 deletions(-)

diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 0d644c9e34b..354158bf07a 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -67,10 +67,14 @@ set(GINKGO_CUDA_HOST_COMPILER @CMAKE_CUDA_HOST_COMPILER@)
 set(GINKGO_HIP_COMPILER_FLAGS @GINKGO_HIP_COMPILER_FLAGS@)
 set(GINKGO_HIP_HCC_COMPILER_FLAGS @GINKGO_HIP_HCC_COMPILER_FLAGS@)
 set(GINKGO_HIP_NVCC_COMPILER_FLAGS @GINKGO_HIP_NVCC_COMPILER_FLAGS@)
+set(GINKGO_HIP_CLANG_COMPILER_FLAGS @GINKGO_HIP_CLANG_COMPILER_FLAGS@)
 set(GINKGO_HIP_PLATFORM @GINKGO_HIP_PLATFORM@)
 set(GINKGO_HIP_AMDGPU @GINKGO_HIP_AMDGPU@)
 set(GINKGO_HIP_VERSION @GINKGO_HIP_VERSION@)
 
+set(GINKGO_DPCPP_VERSION @GINKGO_DPCPP_VERSION@)
+set(GINKGO_DPCPP_FLAGS @GINKGO_DPCPP_FLAGS@)
+
 set(GINKGO_HAVE_PAPI_SDE @GINKGO_HAVE_PAPI_SDE@)
 
 # Ginkgo external package variables
diff --git a/cmake/build_helpers.cmake b/cmake/build_helpers.cmake
index ef7d47f26ba..0323b47d309 100644
--- a/cmake/build_helpers.cmake
+++ b/cmake/build_helpers.cmake
@@ -156,3 +156,23 @@ function(ginkgo_extract_clang_version CLANG_COMPILER GINKGO_CLANG_VERSION)
     file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/extract_clang_ver.cpp)
     file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/extract_clang_ver)
 endfunction()
+
+# Extract the DPC++ version
+function(ginkgo_extract_dpcpp_version DPCPP_COMPILER GINKGO_DPCPP_VERSION)
+    set(DPCPP_VERSION_PROG "#include <CL/sycl.hpp>\n#include <iostream>\n"
+        "int main() {std::cout << __SYCL_COMPILER_VERSION << '\\n'\;"
+        "return 0\;}")
+    file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/extract_dpcpp_ver.cpp" ${DPCPP_VERSION_PROG})
+    execute_process(COMMAND ${DPCPP_COMPILER} ${CMAKE_CURRENT_BINARY_DIR}/extract_dpcpp_ver.cpp
+        -o ${CMAKE_CURRENT_BINARY_DIR}/extract_dpcpp_ver
+        ERROR_VARIABLE DPCPP_EXTRACT_VER_ERROR)
+    execute_process(COMMAND ${CMAKE_CURRENT_BINARY_DIR}/extract_dpcpp_ver
+        OUTPUT_VARIABLE FOUND_DPCPP_VERSION
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        ERROR_STRIP_TRAILING_WHITESPACE
+        )
+
+    set (${GINKGO_DPCPP_VERSION} "${FOUND_DPCPP_VERSION}" PARENT_SCOPE)
+    file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/extract_dpcpp_ver.cpp)
+    file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/extract_dpcpp_ver)
+endfunction()
diff --git a/dev_tools/scripts/create_new_algorithm.sh b/dev_tools/scripts/create_new_algorithm.sh
index f6893f68c82..707e5803283 100755
--- a/dev_tools/scripts/create_new_algorithm.sh
+++ b/dev_tools/scripts/create_new_algorithm.sh
@@ -99,11 +99,13 @@ TEMPLATE_FILES=(
     "${name}_kernels.cpp"
     "${name}_*.[ch]*"
     "${name}_kernels.hip.cpp"
+    "${name}_kernels.dp.cpp"
     "${name}.cpp"
     "${name}_kernels.cpp"
     "${name}_kernels.cpp"
     "${name}_kernels.cpp"
     "${name}_kernels.*"
+    "${name}_kernels.*"
 )
 CMAKE_FILES=(
     "core/CMakeLists.txt"
@@ -113,11 +115,13 @@ CMAKE_FILES=(
     "omp/CMakeLists.txt"
     "cuda/CMakeLists.txt"
     "hip/CMakeLists.txt"
+    "dpcpp/CMakeLists.txt"
     "core/test/$source_type/CMakeLists.txt"
     "reference/test/$source_type/CMakeLists.txt"
     "omp/test/$source_type/CMakeLists.txt"
     "cuda/test/$source_type/CMakeLists.txt"
     "hip/test/$source_type/CMakeLists.txt"
+    "dpcpp/test/$source_type/CMakeLists.txt"
 )
 TEMPLATE_FILES_LOCATIONS=(
     "core/$source_type"
@@ -127,11 +131,13 @@ TEMPLATE_FILES_LOCATIONS=(
     "omp/$source_type"
     "cuda/$source_type"
     "hip/$source_type"
+    "dpcpp/$source_type"
     "core/test/$source_type"
     "reference/test/$source_type"
     "omp/test/$source_type"
     "cuda/test/$source_type"
     "hip/test/$source_type"
+    "dpcpp/test/$source_type"
 )
 TEMPLATE_FILES_TYPES=(
     "$source_type file"
@@ -141,11 +147,13 @@ TEMPLATE_FILES_TYPES=(
     "OpenMP kernel file"
     "CUDA kernel file"
     "HIP kernel file"
+    "DPC++ kernel file"
     "unit tests for ${name} $source_type"
     "unit tests for ${name} reference kernels"
     "unit tests for ${name} OMP kernels"
     "unit tests for ${name} CUDA kernels"
     "unit tests for ${name} HIP kernels"
+    "unit tests for ${name} DPC++ kernels"
 )
 TEMPLATE_FILES_DESCRIPTIONS=(
     "This is where the ${name} algorithm needs to be implemented."
@@ -155,11 +163,13 @@ TEMPLATE_FILES_DESCRIPTIONS=(
     "OMP kernels for ${name} need to be implemented here."
     "CUDA kernels for ${name} need to be implemented here."
     "HIP kernels for ${name} need to be implemented here."
+    "DPC++ kernels for ${name} need to be implemented here."
     "This is where core related unit tests should be implemented, i.e. relating to the interface without executor usage."
     "This is where tests with the Reference executor should be implemented. Usually, this means comparing against previously known values."
     "This is where tests with the OpenMP executor should be implemented. Usually, this means comparing against a Reference execution."
     "This is where tests with the CUDA executor should be implemented. Usually, this means comparing against a Reference execution."
     "This is where tests with the HIP executor should be implemented. Usually, this means comparing against a Reference execution."
+    "This is where tests with the DPC++ executor should be implemented. Usually, this means comparing against a Reference execution."
 )
 
 mkdir ${TMPDIR}
@@ -389,6 +399,10 @@ then
     echo "hip/test/${source_type}/CMakeLists.txt"             | tee -a todo_${name}.txt
     echo ""                                                   | tee -a todo_${name}.txt
     echo ""                                                   | tee -a todo_${name}.txt
+    echo "dpcpp/CMakeLists.txt"                               | tee -a todo_${name}.txt
+    echo "dpcpp/test/${source_type}/CMakeLists.txt"           | tee -a todo_${name}.txt
+    echo ""                                                   | tee -a todo_${name}.txt
+    echo ""                                                   | tee -a todo_${name}.txt
     echo "The following header file has to be modified:"      | tee -a todo_${name}.txt
     echo "core/device_hooks/common_kernels.inc.cpp"           | tee -a todo_${name}.txt
     echo "Equivalent to the other solvers, the following part has to be appended:"  | tee -a todo_${name}.txt
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 4eb501e8d35..10f8668dbe2 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -3,6 +3,9 @@ if (NOT GKO_CAN_COMPILE_DPCPP)
         "${CMAKE_CXX_COMPILER} cannot compile DPC++ code!")
 endif()
 
+ginkgo_extract_dpcpp_version(${CMAKE_CXX_COMPILER} GINKGO_DPCPP_VERSION)
+set(GINKGO_DPCPP_VERSION ${GINKGO_DPCPP_VERSION} PARENT_SCOPE)
+
 add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_dpcpp_device> "")
 target_sources(ginkgo_dpcpp
     PRIVATE
@@ -42,7 +45,8 @@ target_sources(ginkgo_dpcpp
 
 ginkgo_compile_features(ginkgo_dpcpp)
 
-set(GINKGO_DPCPP_FLAGS ${GINKGO_COMPILER_FLAGS} -fsycl)
+set(GINKGO_DPCPP_FLAGS ${GINKGO_COMPILER_FLAGS} -fsycl -fsycl-device-lib=libc)
+set(GINKGO_DPCPP_FLAGS ${GINKGO_DPCPP_FLAGS} PARENT_SCOPE)
 target_compile_options(ginkgo_dpcpp PRIVATE "${GINKGO_DPCPP_FLAGS}")
 target_compile_features(ginkgo_dpcpp PRIVATE cxx_std_17)
 
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index c210718f13c..077830602a3 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -914,12 +914,9 @@ GKO_INLINE GKO_ATTRIBUTES constexpr xstd::enable_if_t<is_complex_s<T>::value,
 abs(const T &x)
 {
 #ifdef CL_SYCL_LANGUAGE_VERSION
-    // FIXME: This implementation is due to two DPC++ issues
-    // 1) plain `sqrt` call evaluates to `std::sqrt` which fails on GPUs
-    // 2) DPC++ GPUs require the OpenCL backend (`SYCL_BE=OPENCL`) to support
-    //    complex multiplication, on top of extra binary objects. For now, we
-    //    avoid this issue with the current implementation.
-    return cl::sycl::sqrt(real(x) * real(x) + imag(x) * imag(x));
+    // FIXME: This implementation is due to a DPC++ issue:
+    // plain `sqrt` call evaluates to `std::sqrt` which fails on GPUs
+    return cl::sycl::sqrt(squared_norm(x));
 #else
     return sqrt(squared_norm(x));
 #endif

From 8140e35bbdc112ea2f243f930b340095ef9fdcfc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 15 Oct 2020 17:53:12 +0200
Subject: [PATCH 0109/1008] Add sorting for the Jacobi preconditioner

Add utility header which (later) will be used to unsort matrices
---
 benchmark/utils/preconditioners.hpp           |  7 ++
 core/preconditioner/jacobi.cpp                | 23 +++++-
 core/test/utils/shuffle_matrix.hpp            | 79 +++++++++++++++++++
 include/ginkgo/core/preconditioner/jacobi.hpp | 34 ++++++--
 4 files changed, 134 insertions(+), 9 deletions(-)
 create mode 100644 core/test/utils/shuffle_matrix.hpp

diff --git a/benchmark/utils/preconditioners.hpp b/benchmark/utils/preconditioners.hpp
index 9f61c99f9a7..f55b9b0e13a 100644
--- a/benchmark/utils/preconditioners.hpp
+++ b/benchmark/utils/preconditioners.hpp
@@ -78,6 +78,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_storage_optimization(
                      parse_storage_optimization(FLAGS_jacobi_storage))
                  .with_accuracy(FLAGS_jacobi_accuracy)
+                 .with_skip_sorting(true)
                  .on(exec);
          }},
         {"parict",
@@ -87,6 +88,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                      .with_iterations(FLAGS_parilu_iterations)
                      .with_approximate_select(FLAGS_parilut_approx_select)
                      .with_fill_in_limit(FLAGS_parilut_limit)
+                     .with_skip_sorting(true)
                      .on(exec));
              return gko::preconditioner::Ilu<>::build()
                  .with_factorization_factory(fact)
@@ -97,6 +99,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
              auto fact =
                  gko::share(gko::factorization::ParIlu<>::build()
                                 .with_iterations(FLAGS_parilu_iterations)
+                                .with_skip_sorting(true)
                                 .on(exec));
              return gko::preconditioner::Ilu<>::build()
                  .with_factorization_factory(fact)
@@ -109,6 +112,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                      .with_iterations(FLAGS_parilu_iterations)
                      .with_approximate_select(FLAGS_parilut_approx_select)
                      .with_fill_in_limit(FLAGS_parilut_limit)
+                     .with_skip_sorting(true)
                      .on(exec));
              return gko::preconditioner::Ilu<>::build()
                  .with_factorization_factory(fact)
@@ -129,6 +133,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                      .with_iterations(FLAGS_parilu_iterations)
                      .with_approximate_select(FLAGS_parilut_approx_select)
                      .with_fill_in_limit(FLAGS_parilut_limit)
+                     .with_skip_sorting(true)
                      .on(exec));
              auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
                                          .with_sparsity_power(FLAGS_isai_power)
@@ -149,6 +154,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
              auto fact =
                  gko::share(gko::factorization::ParIlu<>::build()
                                 .with_iterations(FLAGS_parilu_iterations)
+                                .with_skip_sorting(true)
                                 .on(exec));
              auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
                                          .with_sparsity_power(FLAGS_isai_power)
@@ -171,6 +177,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                      .with_iterations(FLAGS_parilu_iterations)
                      .with_approximate_select(FLAGS_parilut_approx_select)
                      .with_fill_in_limit(FLAGS_parilut_limit)
+                     .with_skip_sorting(true)
                      .on(exec));
              auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
                                          .with_sparsity_power(FLAGS_isai_power)
diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index f7351cd779c..a8a8ad7ba2f 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -33,6 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/preconditioner/jacobi.hpp>
 
 
+#include <memory>
+
+
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
@@ -201,12 +204,26 @@ void Jacobi<ValueType, IndexType>::detect_blocks(
 
 
 template <typename ValueType, typename IndexType>
-void Jacobi<ValueType, IndexType>::generate(const LinOp *system_matrix)
+void Jacobi<ValueType, IndexType>::generate(const LinOp *system_matrix,
+                                            bool skip_sorting)
 {
     GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix);
+    using csr_type = matrix::Csr<ValueType, IndexType>;
     const auto exec = this->get_executor();
-    const auto csr_mtx = copy_and_convert_to<matrix::Csr<ValueType, IndexType>>(
-        exec, system_matrix);
+    decltype(copy_and_convert_to<csr_type>(exec, system_matrix)) csr_mtx{};
+
+    if (skip_sorting) {
+        csr_mtx = copy_and_convert_to<matrix::Csr<ValueType, IndexType>>(
+            exec, system_matrix);
+    } else {
+        auto editable_csr = csr_type::create(exec);
+        as<ConvertibleTo<csr_type>>(system_matrix)
+            ->convert_to(lend(editable_csr));
+        // Maybe check if it is sorted first
+        editable_csr->sort_by_column_index();
+        csr_mtx = decltype(csr_mtx){editable_csr.release(),
+                                    std::default_delete<const csr_type>{}};
+    }
 
     if (parameters_.block_pointers.get_data() == nullptr) {
         this->detect_blocks(csr_mtx.get());
diff --git a/core/test/utils/shuffle_matrix.hpp b/core/test/utils/shuffle_matrix.hpp
new file mode 100644
index 00000000000..44bcc45e502
--- /dev/null
+++ b/core/test/utils/shuffle_matrix.hpp
@@ -0,0 +1,79 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_TEST_UTILS_MATRIX_SHUFFLE_HPP_
+#define GKO_CORE_TEST_UTILS_MATRIX_SHUFFLE_HPP_
+
+#include <algorithm>
+#include <random>
+
+
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+namespace gko {
+namespace test {
+
+
+// Plan for now: shuffle values and column indices to unsort the given matrix
+// without changing the represented matrix.
+template <typename ValueType, typename IndexType, typename RandomEngine>
+void shuffle_nz_storage(matrix::Csr<ValueType, IndexType> *mtx,
+                        RandomEngine &&engine)
+{
+    using value_type = ValueType;
+    using index_type = IndexType;
+    auto size = mtx->get_size();
+    auto vals = mtx->get_values();
+    auto row_ptrs = mtx->get_row_ptrs();
+    auto cols = mtx->get_col_idxs();
+    for (index_type row = 0; row < size[0]; ++row) {
+        auto start = row_ptrs[row];
+        auto end = row_ptrs[row + 1] - 1;
+        std::uniform_int_distribution<index_type> dist{start, end};
+        for (index_type i = start; i < start + (start + end) / 2; ++i) {
+            auto a = dist(engine);
+            auto b = dist(engine);
+            using std::swap;
+            if (a != b) {
+                swap(vals[a], vals[b]);
+                swap(cols[a], cols[b]);
+            }
+        }
+    }
+}
+
+
+}  // namespace test
+}  // namespace gko
+
+#endif  // GKO_CORE_TEST_UTILS_MATRIX_SHUFFLE_HPP_
diff --git a/include/ginkgo/core/preconditioner/jacobi.hpp b/include/ginkgo/core/preconditioner/jacobi.hpp
index be82373c37c..60433b2f2cc 100644
--- a/include/ginkgo/core/preconditioner/jacobi.hpp
+++ b/include/ginkgo/core/preconditioner/jacobi.hpp
@@ -107,8 +107,8 @@ struct block_interleaved_storage_scheme {
      *        blocks is not known, for a special input `size_type{} - 1`
      *        the method returns `0` to avoid overallocation of memory.
      */
-    GKO_ATTRIBUTES size_type compute_storage_space(size_type num_blocks) const
-        noexcept
+    GKO_ATTRIBUTES size_type
+    compute_storage_space(size_type num_blocks) const noexcept
     {
         return (num_blocks + 1 == size_type{0})
                    ? size_type{0}
@@ -146,8 +146,8 @@ struct block_interleaved_storage_scheme {
      *
      * @return the offset of the block with ID `block_id`
      */
-    GKO_ATTRIBUTES IndexType get_global_block_offset(IndexType block_id) const
-        noexcept
+    GKO_ATTRIBUTES IndexType
+    get_global_block_offset(IndexType block_id) const noexcept
     {
         return this->get_group_offset(block_id) +
                this->get_block_offset(block_id);
@@ -313,6 +313,25 @@ class Jacobi : public EnableLinOp<Jacobi<ValueType, IndexType>>,
          */
         uint32 GKO_FACTORY_PARAMETER_SCALAR(max_block_stride, 0u);
 
+        /**
+         * @brief `true` means it is known that the matrix given to this
+         *        factory will be sorted first by row, then by column index,
+         *        `false` means it is unknown or not sorted, so an additional
+         *        sorting step will be performed during the preconditioner
+         *        generation (it will not change the matrix given).
+         *        The matrix must be sorted for this preconditioner to work.
+         *
+         * The `system_matrix`, which will be given to this factory, must be
+         * sorted (first by row, then by column) in order for the algorithm
+         * to work. If it is known that the matrix will be sorted, this
+         * parameter can be set to `true` to skip the sorting (therefore,
+         * shortening the runtime).
+         * However, if it is unknown or if the matrix is known to be not sorted,
+         * it must remain `false`, otherwise, this preconditioner might be
+         * incorrect.
+         */
+        bool GKO_FACTORY_PARAMETER_SCALAR(skip_sorting, false);
+
         /**
          * Starting (row / column) indexes of individual blocks.
          *
@@ -509,7 +528,7 @@ class Jacobi : public EnableLinOp<Jacobi<ValueType, IndexType>>,
         parameters_.block_pointers.set_executor(this->get_executor());
         parameters_.storage_optimization.block_wise.set_executor(
             this->get_executor());
-        this->generate(lend(system_matrix));
+        this->generate(lend(system_matrix), parameters_.skip_sorting);
     }
 
     /**
@@ -559,8 +578,11 @@ class Jacobi : public EnableLinOp<Jacobi<ValueType, IndexType>>,
      *
      * @param system_matrix  the source matrix used to generate the
      *                       preconditioner
+     * @param skip_sorting  determines if the sorting of system_matrix can be
+     *                      skipped (therefore, marking that it is already
+     *                      sorted)
      */
-    void generate(const LinOp *system_matrix);
+    void generate(const LinOp *system_matrix, bool skip_sorting);
 
     /**
      * Detects the diagonal blocks and allocates the memory needed to store the

From e18e75e441337627737efa36291013b7869cbf5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 20 Oct 2020 22:18:07 +0200
Subject: [PATCH 0110/1008] Add proper tests for unsorting

---
 core/test/utils/CMakeLists.txt         |   3 +-
 core/test/utils/shuffle_matrix.hpp     |  79 ------------
 core/test/utils/unsort_matrix.hpp      | 137 +++++++++++++++++++++
 core/test/utils/unsort_matrix_test.cpp | 163 +++++++++++++++++++++++++
 4 files changed, 302 insertions(+), 80 deletions(-)
 delete mode 100644 core/test/utils/shuffle_matrix.hpp
 create mode 100644 core/test/utils/unsort_matrix.hpp
 create mode 100644 core/test/utils/unsort_matrix_test.cpp

diff --git a/core/test/utils/CMakeLists.txt b/core/test/utils/CMakeLists.txt
index 223dd62e1f4..84e3d46958d 100644
--- a/core/test/utils/CMakeLists.txt
+++ b/core/test/utils/CMakeLists.txt
@@ -1,2 +1,3 @@
-ginkgo_create_test(matrix_generator_test)
 ginkgo_create_test(assertions_test)
+ginkgo_create_test(matrix_generator_test)
+ginkgo_create_test(unsort_matrix_test)
diff --git a/core/test/utils/shuffle_matrix.hpp b/core/test/utils/shuffle_matrix.hpp
deleted file mode 100644
index 44bcc45e502..00000000000
--- a/core/test/utils/shuffle_matrix.hpp
+++ /dev/null
@@ -1,79 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GKO_CORE_TEST_UTILS_MATRIX_SHUFFLE_HPP_
-#define GKO_CORE_TEST_UTILS_MATRIX_SHUFFLE_HPP_
-
-#include <algorithm>
-#include <random>
-
-
-#include <ginkgo/core/matrix/csr.hpp>
-
-
-namespace gko {
-namespace test {
-
-
-// Plan for now: shuffle values and column indices to unsort the given matrix
-// without changing the represented matrix.
-template <typename ValueType, typename IndexType, typename RandomEngine>
-void shuffle_nz_storage(matrix::Csr<ValueType, IndexType> *mtx,
-                        RandomEngine &&engine)
-{
-    using value_type = ValueType;
-    using index_type = IndexType;
-    auto size = mtx->get_size();
-    auto vals = mtx->get_values();
-    auto row_ptrs = mtx->get_row_ptrs();
-    auto cols = mtx->get_col_idxs();
-    for (index_type row = 0; row < size[0]; ++row) {
-        auto start = row_ptrs[row];
-        auto end = row_ptrs[row + 1] - 1;
-        std::uniform_int_distribution<index_type> dist{start, end};
-        for (index_type i = start; i < start + (start + end) / 2; ++i) {
-            auto a = dist(engine);
-            auto b = dist(engine);
-            using std::swap;
-            if (a != b) {
-                swap(vals[a], vals[b]);
-                swap(cols[a], cols[b]);
-            }
-        }
-    }
-}
-
-
-}  // namespace test
-}  // namespace gko
-
-#endif  // GKO_CORE_TEST_UTILS_MATRIX_SHUFFLE_HPP_
diff --git a/core/test/utils/unsort_matrix.hpp b/core/test/utils/unsort_matrix.hpp
new file mode 100644
index 00000000000..907fcfa5a41
--- /dev/null
+++ b/core/test/utils/unsort_matrix.hpp
@@ -0,0 +1,137 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_TEST_UTILS_UNSORT_MATRIX_HPP_
+#define GKO_CORE_TEST_UTILS_UNSORT_MATRIX_HPP_
+
+#include <algorithm>
+#include <random>
+
+
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+#include "core/base/iterator_factory.hpp"
+
+
+namespace gko {
+namespace test {
+
+
+// Plan for now: shuffle values and column indices to unsort the given matrix
+// without changing the represented matrix.
+template <typename ValueType, typename IndexType, typename RandomEngine>
+void unsort_matrix(matrix::Csr<ValueType, IndexType> *mtx,
+                   RandomEngine &&engine)
+{
+    using value_type = ValueType;
+    using index_type = IndexType;
+    auto size = mtx->get_size();
+    if (mtx->get_num_stored_elements() <= 0) {
+        return;
+    }
+    const auto &exec = mtx->get_executor();
+    const auto &master = mtx->get_executor()->get_master();
+
+    if (exec != master) {
+        auto h_mtx = mtx->clone(master);
+        unsort_matrix(lend(h_mtx), engine);
+        mtx->copy_from(lend(h_mtx));
+        return;
+    }
+
+    auto vals = mtx->get_values();
+    auto row_ptrs = mtx->get_row_ptrs();
+    auto cols = mtx->get_col_idxs();
+
+    for (index_type row = 0; row < size[0]; ++row) {
+        auto start = row_ptrs[row];
+        auto end = row_ptrs[row + 1];
+        auto iterator = detail::IteratorFactory<IndexType, ValueType>(
+            cols + start, vals + start, end - start);
+        std::shuffle(iterator.begin(), iterator.end(), engine);
+    }
+}
+
+
+// Plan for now: shuffle values and column indices to unsort the given matrix
+// without changing the represented matrix.
+// Note: This expects COO to be properly sorted by row index (which is required
+//       by the Ginkgo COO format)
+template <typename ValueType, typename IndexType, typename RandomEngine>
+void unsort_matrix(matrix::Coo<ValueType, IndexType> *mtx,
+                   RandomEngine &&engine)
+{
+    using value_type = ValueType;
+    using index_type = IndexType;
+    auto nnz = mtx->get_num_stored_elements();
+
+    if (nnz <= 0) {
+        return;
+    }
+
+    const auto &exec = mtx->get_executor();
+    const auto &master = mtx->get_executor()->get_master();
+
+    if (exec != master) {
+        auto h_mtx = mtx->clone(master);
+        unsort_matrix(lend(h_mtx), engine);
+        mtx->copy_from(lend(h_mtx));
+        return;
+    }
+
+    auto size = mtx->get_size();
+    auto vals = mtx->get_values();
+    auto rows = mtx->get_row_idxs();
+    auto cols = mtx->get_col_idxs();
+
+    auto current_row = rows[0];
+    for (IndexType i = 0; i < nnz;) {
+        auto start = i;
+        while (i < nnz && rows[i] == current_row) {
+            ++i;
+        }
+        current_row = rows[i];
+        auto end = i;
+        auto iterator = detail::IteratorFactory<IndexType, ValueType>(
+            cols + start, vals + start, end - start);
+        // since the row entries are supposed to be the same, there is no need
+        // to swap
+        std::shuffle(iterator.begin(), iterator.end(), engine);
+    }
+}
+
+
+}  // namespace test
+}  // namespace gko
+
+#endif  // GKO_CORE_TEST_UTILS_UNSORT_MATRIX_HPP_
diff --git a/core/test/utils/unsort_matrix_test.cpp b/core/test/utils/unsort_matrix_test.cpp
new file mode 100644
index 00000000000..9e80cdca83e
--- /dev/null
+++ b/core/test/utils/unsort_matrix_test.cpp
@@ -0,0 +1,163 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/test/utils/unsort_matrix.hpp"
+
+
+#include <cmath>
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/mtx_io.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class UnsortMatrix : public ::testing::Test {
+protected:
+    using value_type = double;
+    using index_type = gko::int32;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    using Coo = gko::matrix::Coo<value_type, index_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+    UnsortMatrix()
+        : exec(gko::ReferenceExecutor::create()),
+          rand_engine(42),
+          mtx(gko::initialize<Dense>({{1, 2, 0, 0, 0},
+                                      {0, 0, 0, 0, 0},
+                                      {3, 4, 5, 6, 0},
+                                      {0, 0, 7, 0, 0},
+                                      {0, 0, 8, 9, 10}},
+                                     exec)),
+          empty(Dense::create(exec, gko::dim<2>(0, 0)))
+    {}
+
+    bool is_coo_matrix_sorted(Coo *mtx)
+    {
+        auto size = mtx->get_size();
+        auto vals = mtx->get_values();
+        auto rows = mtx->get_row_idxs();
+        auto cols = mtx->get_col_idxs();
+        auto nnz = mtx->get_num_stored_elements();
+
+        if (nnz <= 0) {
+            return true;
+        }
+
+        auto prev_row = rows[0];
+        auto prev_col = cols[0];
+        for (index_type i = 0; i < nnz; ++i) {
+            auto cur_row = rows[i];
+            auto cur_col = cols[i];
+            if (prev_row == cur_row && prev_col > cur_col) {
+                return false;
+            }
+            prev_row = cur_row;
+            prev_col = cur_col;
+        }
+        return true;
+    }
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::ranlux48 rand_engine;
+    std::unique_ptr<Dense> mtx;
+    std::unique_ptr<Dense> empty;
+};
+
+
+TEST_F(UnsortMatrix, CsrWorks)
+{
+    auto csr = Csr::create(exec);
+    mtx->convert_to(gko::lend(csr));
+    bool was_sorted = csr->is_sorted_by_column_index();
+
+    gko::test::unsort_matrix(gko::lend(csr), rand_engine);
+
+    ASSERT_FALSE(csr->is_sorted_by_column_index());
+    ASSERT_TRUE(was_sorted);
+    GKO_ASSERT_MTX_NEAR(csr, mtx, 0.);
+}
+
+
+TEST_F(UnsortMatrix, CsrWorksWithEmpty)
+{
+    auto csr = Csr::create(exec);
+    empty->convert_to(gko::lend(csr));
+    bool was_sorted = csr->is_sorted_by_column_index();
+
+    gko::test::unsort_matrix(gko::lend(csr), rand_engine);
+
+    ASSERT_TRUE(was_sorted);
+    GKO_ASSERT_MTX_NEAR(csr, empty, 0.);
+}
+
+
+TEST_F(UnsortMatrix, CooWorks)
+{
+    auto coo = Coo::create(exec);
+    mtx->convert_to(gko::lend(coo));
+    const bool was_sorted = is_coo_matrix_sorted(gko::lend(coo));
+
+    gko::test::unsort_matrix(gko::lend(coo), rand_engine);
+
+    ASSERT_FALSE(is_coo_matrix_sorted(gko::lend(coo)));
+    ASSERT_TRUE(was_sorted);
+    GKO_ASSERT_MTX_NEAR(coo, mtx, 0.);
+}
+
+
+TEST_F(UnsortMatrix, CooWorksWithEmpty)
+{
+    auto coo = Coo::create(exec);
+    empty->convert_to(gko::lend(coo));
+    const bool was_sorted = is_coo_matrix_sorted(gko::lend(coo));
+
+    gko::test::unsort_matrix(gko::lend(coo), rand_engine);
+
+    ASSERT_TRUE(was_sorted);
+    GKO_ASSERT_MTX_NEAR(coo, empty, 0.);
+}
+
+
+}  // namespace

From dd60bb4ebe07173a849cf096620b09dbcd6a0bc5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 21 Oct 2020 09:15:12 +0200
Subject: [PATCH 0111/1008] Add CUDA tests for unsorted CSR

---
 cuda/test/matrix/csr_kernels.cpp | 66 ++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index f0a1416b6bd..e938d926680 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -52,6 +52,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/matrix/csr_kernels.hpp"
+#include "core/test/utils/unsort_matrix.hpp"
 #include "cuda/test/utils.hpp"
 
 
@@ -219,6 +220,18 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithLoadBalance)
 }
 
 
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithLoadBalanceUnsorted)
+{
+    set_up_apply_data(std::make_shared<Mtx::load_balance>(cuda));
+    gko::test::unsort_matrix(dmtx.get(), rand_engine);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithLoadBalance)
 {
     set_up_apply_data(std::make_shared<Mtx::load_balance>(cuda));
@@ -241,6 +254,18 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithCusparse)
 }
 
 
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithCusparseUnsorted)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    gko::test::unsort_matrix(dmtx.get(), rand_engine);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithCusparse)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
@@ -263,6 +288,18 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithMergePath)
 }
 
 
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithMergePathUnsorted)
+{
+    set_up_apply_data(std::make_shared<Mtx::merge_path>());
+    gko::test::unsort_matrix(dmtx.get(), rand_engine);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithMergePath)
 {
     set_up_apply_data(std::make_shared<Mtx::merge_path>());
@@ -285,6 +322,18 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithClassical)
 }
 
 
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithClassicalUnsorted)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+    gko::test::unsort_matrix(dmtx.get(), rand_engine);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithClassical)
 {
     set_up_apply_data(std::make_shared<Mtx::classical>());
@@ -403,6 +452,23 @@ TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRefUnsorted)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>());
+    auto trans = mtx->transpose();
+    auto d_trans = dmtx->transpose();
+    gko::test::unsort_matrix(static_cast<Mtx *>(dmtx.get()), rand_engine);
+    gko::test::unsort_matrix(static_cast<Mtx *>(d_trans.get()), rand_engine);
+
+    mtx->apply(trans.get(), square_mtx.get());
+    dmtx->apply(d_trans.get(), square_dmtx.get());
+
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
+    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
+}
+
+
 TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>());

From 62e36960cf6d4cfa014742dffcbdd54e3ca34f85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 26 Oct 2020 07:59:30 +0100
Subject: [PATCH 0112/1008] Add OMP and HIP tests

---
 core/test/utils/unsort_matrix.hpp     |   4 +-
 cuda/test/matrix/coo_kernels.cpp      |  18 +++++
 cuda/test/matrix/csr_kernels.cpp      |  56 ++++----------
 hip/test/matrix/coo_kernels.hip.cpp   |  19 +++++
 hip/test/matrix/csr_kernels.hip.cpp   | 105 +++++++++++++++++---------
 omp/test/matrix/coo_kernels.cpp       |  41 +++++++++-
 omp/test/matrix/csr_kernels.cpp       |  33 ++++----
 reference/test/matrix/coo_kernels.cpp |  58 ++++++++++++++
 8 files changed, 237 insertions(+), 97 deletions(-)

diff --git a/core/test/utils/unsort_matrix.hpp b/core/test/utils/unsort_matrix.hpp
index 907fcfa5a41..ff2ff500e0e 100644
--- a/core/test/utils/unsort_matrix.hpp
+++ b/core/test/utils/unsort_matrix.hpp
@@ -76,7 +76,7 @@ void unsort_matrix(matrix::Csr<ValueType, IndexType> *mtx,
     for (index_type row = 0; row < size[0]; ++row) {
         auto start = row_ptrs[row];
         auto end = row_ptrs[row + 1];
-        auto iterator = detail::IteratorFactory<IndexType, ValueType>(
+        auto iterator = gko::detail::IteratorFactory<IndexType, ValueType>(
             cols + start, vals + start, end - start);
         std::shuffle(iterator.begin(), iterator.end(), engine);
     }
@@ -122,7 +122,7 @@ void unsort_matrix(matrix::Coo<ValueType, IndexType> *mtx,
         }
         current_row = rows[i];
         auto end = i;
-        auto iterator = detail::IteratorFactory<IndexType, ValueType>(
+        auto iterator = gko::detail::IteratorFactory<IndexType, ValueType>(
             cols + start, vals + start, end - start);
         // since the row entries are supposed to be the same, there is no need
         // to swap
diff --git a/cuda/test/matrix/coo_kernels.cpp b/cuda/test/matrix/coo_kernels.cpp
index 1c65df1b692..d436cad06d5 100644
--- a/cuda/test/matrix/coo_kernels.cpp
+++ b/cuda/test/matrix/coo_kernels.cpp
@@ -48,6 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/matrix/coo_kernels.hpp"
+#include "core/test/utils/unsort_matrix.hpp"
 #include "cuda/test/utils.hpp"
 
 
@@ -104,6 +105,11 @@ class Coo : public ::testing::Test {
         dbeta->copy_from(beta.get());
     }
 
+    void unsort_mtx()
+    {
+        gko::test::unsort_matrix(mtx.get(), rand_engine);
+        dmtx->copy_from(mtx.get());
+    }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::CudaExecutor> cuda;
@@ -135,6 +141,18 @@ TEST_F(Coo, SimpleApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Coo, SimpleApplyIsEquivalentToRefUnsorted)
+{
+    set_up_apply_data();
+    unsort_mtx();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Coo, AdvancedApplyIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index e938d926680..29b7f04f292 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -140,36 +140,10 @@ class Csr : public ::testing::Test {
         complex_dmtx->copy_from(complex_mtx.get());
     }
 
-    struct matrix_pair {
-        std::unique_ptr<Mtx> ref;
-        std::unique_ptr<Mtx> cuda;
-    };
-
-    matrix_pair gen_unsorted_mtx()
+    void unsort_mtx()
     {
-        constexpr int min_nnz_per_row = 2;  // Must be at least 2
-        auto local_mtx_ref =
-            gen_mtx<Mtx>(mtx_size[0], mtx_size[1], min_nnz_per_row);
-        for (size_t row = 0; row < mtx_size[0]; ++row) {
-            const auto row_ptrs = local_mtx_ref->get_const_row_ptrs();
-            const auto start_row = row_ptrs[row];
-            auto col_idx = local_mtx_ref->get_col_idxs() + start_row;
-            auto vals = local_mtx_ref->get_values() + start_row;
-            const auto nnz_in_this_row = row_ptrs[row + 1] - row_ptrs[row];
-            auto swap_idx_dist =
-                std::uniform_int_distribution<>(0, nnz_in_this_row - 1);
-            // shuffle `nnz_in_this_row / 2` times
-            for (size_t perm = 0; perm < nnz_in_this_row; perm += 2) {
-                const auto idx1 = swap_idx_dist(rand_engine);
-                const auto idx2 = swap_idx_dist(rand_engine);
-                std::swap(col_idx[idx1], col_idx[idx2]);
-                std::swap(vals[idx1], vals[idx2]);
-            }
-        }
-        auto local_mtx_cuda = Mtx::create(cuda);
-        local_mtx_cuda->copy_from(local_mtx_ref.get());
-
-        return {std::move(local_mtx_ref), std::move(local_mtx_cuda)};
+        gko::test::unsort_matrix(mtx.get(), rand_engine);
+        dmtx->copy_from(mtx.get());
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -223,7 +197,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithLoadBalance)
 TEST_F(Csr, SimpleApplyIsEquivalentToRefWithLoadBalanceUnsorted)
 {
     set_up_apply_data(std::make_shared<Mtx::load_balance>(cuda));
-    gko::test::unsort_matrix(dmtx.get(), rand_engine);
+    unsort_mtx();
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -257,7 +231,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithCusparse)
 TEST_F(Csr, SimpleApplyIsEquivalentToRefWithCusparseUnsorted)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
-    gko::test::unsort_matrix(dmtx.get(), rand_engine);
+    unsort_mtx();
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -291,7 +265,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithMergePath)
 TEST_F(Csr, SimpleApplyIsEquivalentToRefWithMergePathUnsorted)
 {
     set_up_apply_data(std::make_shared<Mtx::merge_path>());
-    gko::test::unsort_matrix(dmtx.get(), rand_engine);
+    unsort_mtx();
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -325,7 +299,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithClassical)
 TEST_F(Csr, SimpleApplyIsEquivalentToRefWithClassicalUnsorted)
 {
     set_up_apply_data(std::make_shared<Mtx::classical>());
-    gko::test::unsort_matrix(dmtx.get(), rand_engine);
+    unsort_mtx();
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -837,12 +811,13 @@ TEST_F(Csr, RecognizeSortedMatrixIsEquivalentToRef)
 
 TEST_F(Csr, RecognizeUnsortedMatrixIsEquivalentToRef)
 {
-    auto uns_mtx = gen_unsorted_mtx();
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    unsort_mtx();
     bool is_sorted_cuda{};
     bool is_sorted_ref{};
 
-    is_sorted_ref = uns_mtx.ref->is_sorted_by_column_index();
-    is_sorted_cuda = uns_mtx.cuda->is_sorted_by_column_index();
+    is_sorted_ref = mtx->is_sorted_by_column_index();
+    is_sorted_cuda = dmtx->is_sorted_by_column_index();
 
     ASSERT_EQ(is_sorted_ref, is_sorted_cuda);
 }
@@ -862,13 +837,14 @@ TEST_F(Csr, SortSortedMatrixIsEquivalentToRef)
 
 TEST_F(Csr, SortUnsortedMatrixIsEquivalentToRef)
 {
-    auto uns_mtx = gen_unsorted_mtx();
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    unsort_mtx();
 
-    uns_mtx.ref->sort_by_column_index();
-    uns_mtx.cuda->sort_by_column_index();
+    mtx->sort_by_column_index();
+    dmtx->sort_by_column_index();
 
     // Values must be unchanged, therefore, tolerance is `0`
-    GKO_ASSERT_MTX_NEAR(uns_mtx.ref, uns_mtx.cuda, 0);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 0);
 }
 
 
diff --git a/hip/test/matrix/coo_kernels.hip.cpp b/hip/test/matrix/coo_kernels.hip.cpp
index b2dd2039926..e722c6cfdb7 100644
--- a/hip/test/matrix/coo_kernels.hip.cpp
+++ b/hip/test/matrix/coo_kernels.hip.cpp
@@ -48,6 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/matrix/coo_kernels.hpp"
+#include "core/test/utils/unsort_matrix.hpp"
 #include "hip/test/utils.hip.hpp"
 
 
@@ -104,6 +105,12 @@ class Coo : public ::testing::Test {
         dbeta->copy_from(beta.get());
     }
 
+    void unsort_mtx()
+    {
+        gko::test::unsort_matrix(mtx.get(), rand_engine);
+        dmtx->copy_from(mtx.get());
+    }
+
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::HipExecutor> hip;
@@ -135,6 +142,18 @@ TEST_F(Coo, SimpleApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Coo, SimpleApplyIsEquivalentToRefUnsorted)
+{
+    set_up_apply_data();
+    unsort_mtx();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Coo, AdvancedApplyIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index 1fb01640dfa..fea89e3ff6d 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -52,6 +52,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/matrix/csr_kernels.hpp"
+#include "core/test/utils/unsort_matrix.hpp"
 #include "hip/test/utils.hip.hpp"
 
 
@@ -139,36 +140,10 @@ class Csr : public ::testing::Test {
         complex_dmtx->copy_from(complex_mtx.get());
     }
 
-    struct matrix_pair {
-        std::unique_ptr<Mtx> ref;
-        std::unique_ptr<Mtx> hip;
-    };
-
-    matrix_pair gen_unsorted_mtx()
+    void unsort_mtx()
     {
-        constexpr int min_nnz_per_row = 2;  // Must be at least 2
-        auto local_mtx_ref =
-            gen_mtx<Mtx>(mtx_size[0], mtx_size[1], min_nnz_per_row);
-        for (size_t row = 0; row < mtx_size[0]; ++row) {
-            const auto row_ptrs = local_mtx_ref->get_const_row_ptrs();
-            const auto start_row = row_ptrs[row];
-            auto col_idx = local_mtx_ref->get_col_idxs() + start_row;
-            auto vals = local_mtx_ref->get_values() + start_row;
-            const auto nnz_in_this_row = row_ptrs[row + 1] - row_ptrs[row];
-            auto swap_idx_dist =
-                std::uniform_int_distribution<>(0, nnz_in_this_row - 1);
-            // shuffle `nnz_in_this_row / 2` times
-            for (size_t perm = 0; perm < nnz_in_this_row; perm += 2) {
-                const auto idx1 = swap_idx_dist(rand_engine);
-                const auto idx2 = swap_idx_dist(rand_engine);
-                std::swap(col_idx[idx1], col_idx[idx2]);
-                std::swap(vals[idx1], vals[idx2]);
-            }
-        }
-        auto local_mtx_hip = Mtx::create(hip);
-        local_mtx_hip->copy_from(local_mtx_ref.get());
-
-        return {std::move(local_mtx_ref), std::move(local_mtx_hip)};
+        gko::test::unsort_matrix(mtx.get(), rand_engine);
+        dmtx->copy_from(mtx.get());
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -219,6 +194,18 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithLoadBalance)
 }
 
 
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithLoadBalanceUnsorted)
+{
+    set_up_apply_data(std::make_shared<Mtx::load_balance>(hip));
+    unsort_mtx();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithLoadBalance)
 {
     set_up_apply_data(std::make_shared<Mtx::load_balance>(hip));
@@ -241,6 +228,18 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithHipsparse)
 }
 
 
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithHipsparseUnsorted)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    unsort_mtx();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithHipsparse)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
@@ -263,6 +262,18 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithMergePath)
 }
 
 
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithMergePathUnsorted)
+{
+    set_up_apply_data(std::make_shared<Mtx::merge_path>());
+    unsort_mtx();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithMergePath)
 {
     set_up_apply_data(std::make_shared<Mtx::merge_path>());
@@ -285,6 +296,18 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithClassical)
 }
 
 
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithClassicalUnsorted)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+    unsort_mtx();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithClassical)
 {
     set_up_apply_data(std::make_shared<Mtx::classical>());
@@ -307,6 +330,18 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithAutomatical)
 }
 
 
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithAutomaticalUnsorted)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+    unsort_mtx();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Csr, SimpleApplyToDenseMatrixIsEquivalentToRefWithLoadBalance)
 {
     set_up_apply_data(std::make_shared<Mtx::load_balance>(hip), 3);
@@ -770,7 +805,8 @@ TEST_F(Csr, RecognizeSortedMatrixIsEquivalentToRef)
 
 TEST_F(Csr, RecognizeUnsortedMatrixIsEquivalentToRef)
 {
-    auto uns_mtx = gen_unsorted_mtx();
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    unsort_mtx();
     bool is_sorted_hip{};
     bool is_sorted_ref{};
 
@@ -795,13 +831,14 @@ TEST_F(Csr, SortSortedMatrixIsEquivalentToRef)
 
 TEST_F(Csr, SortUnsortedMatrixIsEquivalentToRef)
 {
-    auto uns_mtx = gen_unsorted_mtx();
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    unsort_mtx();
 
-    uns_mtx.ref->sort_by_column_index();
-    uns_mtx.hip->sort_by_column_index();
+    mtx->sort_by_column_index();
+    dmtx->sort_by_column_index();
 
     // Values must be unchanged, therefore, tolerance is `0`
-    GKO_ASSERT_MTX_NEAR(uns_mtx.ref, uns_mtx.hip, 0);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 0);
 }
 
 
diff --git a/omp/test/matrix/coo_kernels.cpp b/omp/test/matrix/coo_kernels.cpp
index fae58704ab2..9f0dcfb7e73 100644
--- a/omp/test/matrix/coo_kernels.cpp
+++ b/omp/test/matrix/coo_kernels.cpp
@@ -49,6 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/matrix/coo_kernels.hpp"
 #include "core/test/utils.hpp"
+#include "core/test/utils/unsort_matrix.hpp"
 
 
 namespace {
@@ -60,7 +61,7 @@ class Coo : public ::testing::Test {
     using Vec = gko::matrix::Dense<>;
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
-    Coo() : rand_engine(42) {}
+    Coo() : mtx_size(532, 231), rand_engine(42) {}
 
     void SetUp()
     {
@@ -88,9 +89,9 @@ class Coo : public ::testing::Test {
     void set_up_apply_data(int num_vectors = 1)
     {
         mtx = Mtx::create(ref);
-        mtx->copy_from(gen_mtx(532, 231, 1));
-        expected = gen_mtx(532, num_vectors, 1);
-        y = gen_mtx(231, num_vectors, 1);
+        mtx->copy_from(gen_mtx(mtx_size[0], mtx_size[1], 1));
+        expected = gen_mtx(mtx_size[0], num_vectors, 1);
+        y = gen_mtx(mtx_size[1], num_vectors, 1);
         alpha = gko::initialize<Vec>({2.0}, ref);
         beta = gko::initialize<Vec>({-1.0}, ref);
         dmtx = Mtx::create(omp);
@@ -105,10 +106,30 @@ class Coo : public ::testing::Test {
         dbeta->copy_from(beta.get());
     }
 
+    struct matrix_pair {
+        std::unique_ptr<Mtx> ref;
+        std::unique_ptr<Mtx> omp;
+    };
+
+    matrix_pair gen_unsorted_mtx()
+    {
+        constexpr int min_nnz_per_row{2};
+        auto local_mtx_ref = Mtx::create(ref);
+        auto generated = gen_mtx(mtx_size[0], mtx_size[1], min_nnz_per_row);
+        local_mtx_ref->copy_from(generated.get());
+        gko::test::unsort_matrix(local_mtx_ref.get(), rand_engine);
+
+        auto local_mtx_omp = Mtx::create(omp);
+        local_mtx_omp->copy_from(local_mtx_ref.get());
+
+        return {std::move(local_mtx_ref), std::move(local_mtx_omp)};
+    }
+
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::OmpExecutor> omp;
 
+    const gko::dim<2> mtx_size;
     std::ranlux48 rand_engine;
 
     std::unique_ptr<Mtx> mtx;
@@ -202,6 +223,18 @@ TEST_F(Coo, SimpleApplyAddToDenseMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Coo, SimpleApplyAddToDenseMatrixIsEquivalentToRefUnsorted)
+{
+    set_up_apply_data(3);
+    auto pair = gen_unsorted_mtx();
+
+    pair.ref->apply2(y.get(), expected.get());
+    pair.omp->apply2(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Coo, AdvancedApplyAddToDenseMatrixIsEquivalentToRef)
 {
     set_up_apply_data(3);
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index 064a129b9eb..26af3397262 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -53,6 +53,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/matrix/csr_kernels.hpp"
 #include "core/test/utils.hpp"
+#include "core/test/utils/unsort_matrix.hpp"
 
 
 namespace {
@@ -139,25 +140,11 @@ class Csr : public ::testing::Test {
 
     matrix_pair gen_unsorted_mtx()
     {
-        constexpr int min_nnz_per_row = 2;  // Must be at least 2
+        constexpr int min_nnz_per_row{2};
         auto local_mtx_ref =
             gen_mtx<Mtx>(mtx_size[0], mtx_size[1], min_nnz_per_row);
-        for (size_t row = 0; row < mtx_size[0]; ++row) {
-            const auto row_ptrs = local_mtx_ref->get_const_row_ptrs();
-            const auto start_row = row_ptrs[row];
-            auto col_idx = local_mtx_ref->get_col_idxs() + start_row;
-            auto vals = local_mtx_ref->get_values() + start_row;
-            const auto nnz_in_this_row = row_ptrs[row + 1] - row_ptrs[row];
-            auto swap_idx_dist =
-                std::uniform_int_distribution<>(0, nnz_in_this_row - 1);
-            // shuffle `nnz_in_this_row / 2` times
-            for (size_t perm = 0; perm < nnz_in_this_row; perm += 2) {
-                const auto idx1 = swap_idx_dist(rand_engine);
-                const auto idx2 = swap_idx_dist(rand_engine);
-                std::swap(col_idx[idx1], col_idx[idx2]);
-                std::swap(vals[idx1], vals[idx2]);
-            }
-        }
+        gko::test::unsort_matrix(gko::lend(local_mtx_ref), rand_engine);
+
         auto local_mtx_omp = Mtx::create(omp);
         local_mtx_omp->copy_from(local_mtx_ref.get());
 
@@ -225,6 +212,18 @@ TEST_F(Csr, SimpleApplyToDenseMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, SimpleApplyToDenseMatrixIsEquivalentToRefUnsorted)
+{
+    set_up_apply_data(3);
+    auto pair = gen_unsorted_mtx();
+
+    pair.ref->apply(y.get(), expected.get());
+    pair.omp->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Csr, AdvancedApplyToCsrMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/reference/test/matrix/coo_kernels.cpp b/reference/test/matrix/coo_kernels.cpp
index 68657a8702e..2e7d7764c99 100644
--- a/reference/test/matrix/coo_kernels.cpp
+++ b/reference/test/matrix/coo_kernels.cpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/coo.hpp>
 
 
+#include <algorithm>
 #include <memory>
 
 
@@ -70,6 +71,11 @@ class Coo : public ::testing::Test {
         mtx = gko::initialize<Mtx>({{1.0, 3.0, 2.0},
                                      {0.0, 5.0, 0.0}}, exec);
         // clang-format on
+        uns_mtx = gko::clone(exec, mtx);
+        auto cols = uns_mtx->get_col_idxs();
+        auto vals = uns_mtx->get_values();
+        std::swap(cols[0], cols[1]);
+        std::swap(vals[0], vals[1]);
     }
 
     void assert_equal_to_mtx_in_csr_format(const Csr *m)
@@ -94,6 +100,7 @@ class Coo : public ::testing::Test {
 
     std::shared_ptr<const gko::Executor> exec;
     std::unique_ptr<Mtx> mtx;
+    std::unique_ptr<Mtx> uns_mtx;
 };
 
 TYPED_TEST_SUITE(Coo, gko::test::ValueIndexTypes);
@@ -199,6 +206,23 @@ TYPED_TEST(Coo, ConvertsToDense)
 }
 
 
+TYPED_TEST(Coo, ConvertsToDenseUnsorted)
+{
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using Dense = typename TestFixture::Vec;
+    auto dense_mtx = Dense::create(this->mtx->get_executor());
+
+    this->uns_mtx->convert_to(dense_mtx.get());
+
+    // clang-format off
+    GKO_ASSERT_MTX_NEAR(dense_mtx,
+                    l({{1.0, 3.0, 2.0},
+                       {0.0, 5.0, 0.0}}), 0.0);
+    // clang-format on
+}
+
+
 TYPED_TEST(Coo, MovesToDense)
 {
     using value_type = typename TestFixture::value_type;
@@ -325,6 +349,18 @@ TYPED_TEST(Coo, AppliesToDenseVector)
 }
 
 
+TYPED_TEST(Coo, AppliesToDenseVectorUnsorted)
+{
+    using Vec = typename TestFixture::Vec;
+    auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = Vec::create(this->exec, gko::dim<2>{2, 1});
+
+    this->uns_mtx->apply(x.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({13.0, 5.0}), 0.0);
+}
+
+
 TYPED_TEST(Coo, AppliesToDenseMatrix)
 {
     using Vec = typename TestFixture::Vec;
@@ -347,6 +383,28 @@ TYPED_TEST(Coo, AppliesToDenseMatrix)
 }
 
 
+TYPED_TEST(Coo, AppliesToDenseMatrixUnsorted)
+{
+    using Vec = typename TestFixture::Vec;
+    using T = typename TestFixture::value_type;
+    // clang-format off
+    auto x = gko::initialize<Vec>(
+        {I<T>{2.0, 3.0},
+         I<T>{1.0, -1.5},
+         I<T>{4.0, 2.5}}, this->exec);
+    // clang-format on
+    auto y = Vec::create(this->exec, gko::dim<2>{2, 2});
+
+    this->uns_mtx->apply(x.get(), y.get());
+
+    // clang-format off
+    GKO_ASSERT_MTX_NEAR(y,
+                        l({{13.0,  3.5},
+                           { 5.0, -7.5}}), 0.0);
+    // clang-format on
+}
+
+
 TYPED_TEST(Coo, AppliesLinearCombinationToDenseVector)
 {
     using Vec = typename TestFixture::Vec;

From 19dcb4bf1ffc7f1330837dffd9f9f4810062d8ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 29 Oct 2020 17:57:59 +0100
Subject: [PATCH 0113/1008] Add sorting to ILU factorization

---
 core/factorization/ilu.cpp                |  6 ++-
 core/test/utils/unsort_matrix.hpp         | 18 ++++++++-
 cuda/test/factorization/ilu_kernels.cpp   | 45 ++++++++++++++++++++---
 hip/test/factorization/ilu_kernels.cpp    | 44 +++++++++++++++++++---
 include/ginkgo/core/factorization/ilu.hpp | 27 +++++++++++++-
 reference/test/preconditioner/jacobi.cpp  | 25 ++++++++-----
 6 files changed, 140 insertions(+), 25 deletions(-)

diff --git a/core/factorization/ilu.cpp b/core/factorization/ilu.cpp
index c2f397151d3..22655b463eb 100644
--- a/core/factorization/ilu.cpp
+++ b/core/factorization/ilu.cpp
@@ -63,7 +63,7 @@ GKO_REGISTER_OPERATION(initialize_l_u, factorization::initialize_l_u);
 
 template <typename ValueType, typename IndexType>
 std::unique_ptr<Composition<ValueType>> Ilu<ValueType, IndexType>::generate_l_u(
-    const std::shared_ptr<const LinOp> &system_matrix) const
+    const std::shared_ptr<const LinOp> &system_matrix, bool skip_sorting) const
 {
     GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix);
 
@@ -75,6 +75,10 @@ std::unique_ptr<Composition<ValueType>> Ilu<ValueType, IndexType>::generate_l_u(
     as<ConvertibleTo<matrix_type>>(system_matrix.get())
         ->convert_to(local_system_matrix.get());
 
+    if (!skip_sorting) {
+        local_system_matrix->sort_by_column_index();
+    }
+
     // Add explicit diagonal zero elements if they are missing
     exec->run(ilu_factorization::make_add_diagonal_elements(
         local_system_matrix.get(), false));
diff --git a/core/test/utils/unsort_matrix.hpp b/core/test/utils/unsort_matrix.hpp
index ff2ff500e0e..9e0759c8097 100644
--- a/core/test/utils/unsort_matrix.hpp
+++ b/core/test/utils/unsort_matrix.hpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <random>
 
 
+#include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 
 
@@ -60,9 +61,24 @@ void unsort_matrix(matrix::Csr<ValueType, IndexType> *mtx,
         return;
     }
     const auto &exec = mtx->get_executor();
-    const auto &master = mtx->get_executor()->get_master();
+    auto master = mtx->get_executor()->get_master();
 
+    // If exec is not the master/host, extract the master and perform the
+    // unsorting there, followed by copying it back
     if (exec != master) {
+        constexpr int max_depth{10};  // Max depth that is searched for master
+        bool actual_master_found{false};
+        for (int i = 0; i < max_depth; ++i) {
+            const auto new_master = master->get_master();
+            if (new_master == master) {
+                actual_master_found = true;
+                break;
+            }
+            master = new_master;
+        }
+        if (!actual_master_found) {
+            GKO_NOT_SUPPORTED(exec);
+        }
         auto h_mtx = mtx->clone(master);
         unsort_matrix(lend(h_mtx), engine);
         mtx->copy_from(lend(h_mtx));
diff --git a/cuda/test/factorization/ilu_kernels.cpp b/cuda/test/factorization/ilu_kernels.cpp
index 4c1d356b0d0..f0025651a48 100644
--- a/cuda/test/factorization/ilu_kernels.cpp
+++ b/cuda/test/factorization/ilu_kernels.cpp
@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/factorization/par_ilu.hpp>
 
 
+#include "core/test/utils/unsort_matrix.hpp"
 #include "cuda/test/utils.hpp"
 #include "matrices/config.hpp"
 
@@ -61,12 +62,14 @@ class Ilu : public ::testing::Test {
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<gko::CudaExecutor> cuda;
+    std::ranlux48 rand_engine;
     std::shared_ptr<Csr> csr_ref;
     std::shared_ptr<Csr> csr_cuda;
 
     Ilu()
         : ref(gko::ReferenceExecutor::create()),
-          cuda(gko::CudaExecutor::create(0, ref))
+          cuda(gko::CudaExecutor::create(0, ref)),
+          rand_engine(1337)
     {}
 
     void SetUp() override
@@ -84,12 +87,42 @@ class Ilu : public ::testing::Test {
 };
 
 
-TEST_F(Ilu, ComputeILUIsEquivalentToRef)
+TEST_F(Ilu, ComputeILUIsEquivalentToRefSorted)
 {
-    auto ref_fact =
-        gko::factorization::ParIlu<>::build().on(ref)->generate(csr_ref);
-    auto cuda_fact =
-        gko::factorization::Ilu<>::build().on(cuda)->generate(csr_cuda);
+    auto ref_fact = gko::factorization::ParIlu<>::build()
+                        .with_skip_sorting(true)
+                        .on(ref)
+                        ->generate(csr_ref);
+    auto cuda_fact = gko::factorization::Ilu<>::build()
+                         .with_skip_sorting(true)
+                         .on(cuda)
+                         ->generate(csr_cuda);
+
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_l_factor(), cuda_fact->get_l_factor(),
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_u_factor(), cuda_fact->get_u_factor(),
+                        1e-14);
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_l_factor(),
+                               cuda_fact->get_l_factor());
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_u_factor(),
+                               cuda_fact->get_u_factor());
+}
+
+
+TEST_F(Ilu, ComputeILUIsEquivalentToRefUnsorted)
+{
+    gko::test::unsort_matrix(gko::lend(csr_ref), rand_engine);
+    csr_cuda->copy_from(gko::lend(csr_ref));
+
+    // TODO: Remove sorting, just for testing here!
+    auto ref_fact = gko::factorization::ParIlu<>::build()
+                        .with_skip_sorting(true)
+                        .on(ref)
+                        ->generate(csr_ref);
+    auto cuda_fact = gko::factorization::Ilu<>::build()
+                         .with_skip_sorting(true)
+                         .on(cuda)
+                         ->generate(csr_cuda);
 
     GKO_ASSERT_MTX_NEAR(ref_fact->get_l_factor(), cuda_fact->get_l_factor(),
                         1e-14);
diff --git a/hip/test/factorization/ilu_kernels.cpp b/hip/test/factorization/ilu_kernels.cpp
index b0bffcdd430..ce69b38602e 100644
--- a/hip/test/factorization/ilu_kernels.cpp
+++ b/hip/test/factorization/ilu_kernels.cpp
@@ -61,12 +61,14 @@ class Ilu : public ::testing::Test {
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<gko::HipExecutor> hip;
+    std::randlux48 rand_engine;
     std::shared_ptr<Csr> csr_ref;
     std::shared_ptr<Csr> csr_hip;
 
     Ilu()
         : ref(gko::ReferenceExecutor::create()),
-          hip(gko::HipExecutor::create(0, ref))
+          hip(gko::HipExecutor::create(0, ref)),
+          rand_engine(1337)
     {}
 
     void SetUp() override
@@ -84,12 +86,42 @@ class Ilu : public ::testing::Test {
 };
 
 
-TEST_F(Ilu, ComputeILUIsEquivalentToRef)
+TEST_F(Ilu, ComputeILUIsEquivalentToRefSorted)
 {
-    auto ref_fact =
-        gko::factorization::ParIlu<>::build().on(ref)->generate(csr_ref);
-    auto hip_fact =
-        gko::factorization::Ilu<>::build().on(hip)->generate(csr_hip);
+    auto ref_fact = gko::factorization::ParIlu<>::build()
+                        .with_skip_sorting(true)
+                        .on(ref)
+                        ->generate(csr_ref);
+    auto hip_fact = gko::factorization::Ilu<>::build()
+                        .with_skip_sorting(true)
+                        .on(hip)
+                        ->generate(csr_hip);
+
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_l_factor(), hip_fact->get_l_factor(),
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_u_factor(), hip_fact->get_u_factor(),
+                        1e-14);
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_l_factor(),
+                               hip_fact->get_l_factor());
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_u_factor(),
+                               hip_fact->get_u_factor());
+}
+
+
+TEST_F(Ilu, ComputeILUIsEquivalentToRefUnsorted)
+{
+    gko::test::unsort_matrix(gko::lend(csr_ref), rand_engine);
+    csr_hip->copy_from(gko::lend(csr_ref));
+
+    // TODO: Remove sorting, just for testing here!
+    auto ref_fact = gko::factorization::ParIlu<>::build()
+                        .with_skip_sorting(true)
+                        .on(ref)
+                        ->generate(csr_ref);
+    auto hip_fact = gko::factorization::Ilu<>::build()
+                        .with_skip_sorting(true)
+                        .on(hip)
+                        ->generate(csr_hip);
 
     GKO_ASSERT_MTX_NEAR(ref_fact->get_l_factor(), hip_fact->get_l_factor(),
                         1e-14);
diff --git a/include/ginkgo/core/factorization/ilu.hpp b/include/ginkgo/core/factorization/ilu.hpp
index ee71beb56af..ece4e2aeb87 100644
--- a/include/ginkgo/core/factorization/ilu.hpp
+++ b/include/ginkgo/core/factorization/ilu.hpp
@@ -109,6 +109,25 @@ class Ilu : public Composition<ValueType> {
          */
         std::shared_ptr<typename matrix_type::strategy_type>
             GKO_FACTORY_PARAMETER_SCALAR(u_strategy, nullptr);
+
+        /**
+         * @brief `true` means it is known that the matrix given to this
+         *        factory will be sorted first by row, then by column index,
+         *        `false` means it is unknown or not sorted, so an additional
+         *        sorting step will be performed during the factorization
+         *        (it will not change the matrix given).
+         *        The matrix must be sorted for this factorization to work.
+         *
+         * The `system_matrix`, which will be given to this factory, must be
+         * sorted (first by row, then by column) in order for the algorithm
+         * to work. If it is known that the matrix will be sorted, this
+         * parameter can be set to `true` to skip the sorting (therefore,
+         * shortening the runtime).
+         * However, if it is unknown or if the matrix is known to be not sorted,
+         * it must remain `false`, otherwise, this factorization might be
+         * incorrect.
+         */
+        bool GKO_FACTORY_PARAMETER_SCALAR(skip_sorting, false);
     };
     GKO_ENABLE_LIN_OP_FACTORY(Ilu, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);
@@ -126,7 +145,7 @@ class Ilu : public Composition<ValueType> {
             parameters_.u_strategy =
                 std::make_shared<typename matrix_type::classical>();
         }
-        generate_l_u(system_matrix)->move_to(this);
+        generate_l_u(system_matrix, parameters_.skip_sorting)->move_to(this);
     }
 
     /**
@@ -138,11 +157,15 @@ class Ilu : public Composition<ValueType> {
      * @param system_matrix  the source matrix used to generate the factors.
      *                       @note: system_matrix must be convertible to a Csr
      *                              Matrix, otherwise, an exception is thrown.
+     * @param skip_sorting  determines if the sorting of system_matrix can be
+     *                      skipped (therefore, marking that it is already
+     *                      sorted)
      * @return  A Composition, containing the incomplete LU factors for the
      *          given system_matrix (first element is L, then U)
      */
     std::unique_ptr<Composition<ValueType>> generate_l_u(
-        const std::shared_ptr<const LinOp> &system_matrix) const;
+        const std::shared_ptr<const LinOp> &system_matrix,
+        bool skip_sorting) const;
 };
 
 
diff --git a/reference/test/preconditioner/jacobi.cpp b/reference/test/preconditioner/jacobi.cpp
index 586825a2d3b..af0e9d9005b 100644
--- a/reference/test/preconditioner/jacobi.cpp
+++ b/reference/test/preconditioner/jacobi.cpp
@@ -314,16 +314,11 @@ TYPED_TEST(Jacobi, CanBeClearedWithAdaptivePrecision)
     GKO_EXPECT_NEAR(first.value, second.value, tol)
 
 
-TYPED_TEST(Jacobi, GeneratesCorrectMatrixData)
+template <typename ValueType, typename IndexType>
+void assert_matrix_data(gko::matrix_data<ValueType, IndexType> data)
 {
-    using value_type = typename TestFixture::value_type;
-    using index_type = typename TestFixture::index_type;
-    using tpl = typename gko::matrix_data<value_type, index_type>::nonzero_type;
-    auto tol = r<value_type>::value;
-    gko::matrix_data<value_type, index_type> data;
-
-    this->bj->write(data);
-
+    using tpl = typename decltype(data)::nonzero_type;
+    auto tol = r<ValueType>::value;
     ASSERT_EQ(data.size, gko::dim<2>{5});
     ASSERT_EQ(data.nonzeros.size(), 13);
     GKO_EXPECT_NONZERO_NEAR(data.nonzeros[0], tpl(0, 0, 4.0 / 14), tol);
@@ -342,6 +337,18 @@ TYPED_TEST(Jacobi, GeneratesCorrectMatrixData)
 }
 
 
+TYPED_TEST(Jacobi, GeneratesCorrectMatrixData)
+{
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    gko::matrix_data<value_type, index_type> data;
+
+    this->bj->write(data);
+
+    assert_matrix_data(data);
+}
+
+
 TYPED_TEST(Jacobi, GeneratesCorrectMatrixDataWithAdaptivePrecision)
 {
     using value_type = typename TestFixture::value_type;

From cfc82a0089edbd3f34fe43fffcf966658f57f35b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 30 Oct 2020 10:12:06 +0100
Subject: [PATCH 0114/1008] Add Jacobi preconditioner test for unsorted Mtx

---
 cuda/test/factorization/ilu_kernels.cpp     | 13 ++++--------
 cuda/test/preconditioner/jacobi_kernels.cpp | 22 +++++++++++++++++++--
 hip/test/factorization/ilu_kernels.cpp      | 16 ++++++---------
 hip/test/preconditioner/jacobi_kernels.cpp  | 20 +++++++++++++++++--
 omp/test/preconditioner/jacobi_kernels.cpp  | 22 +++++++++++++++++++--
 5 files changed, 68 insertions(+), 25 deletions(-)

diff --git a/cuda/test/factorization/ilu_kernels.cpp b/cuda/test/factorization/ilu_kernels.cpp
index f0025651a48..bcb18de9fae 100644
--- a/cuda/test/factorization/ilu_kernels.cpp
+++ b/cuda/test/factorization/ilu_kernels.cpp
@@ -114,15 +114,10 @@ TEST_F(Ilu, ComputeILUIsEquivalentToRefUnsorted)
     gko::test::unsort_matrix(gko::lend(csr_ref), rand_engine);
     csr_cuda->copy_from(gko::lend(csr_ref));
 
-    // TODO: Remove sorting, just for testing here!
-    auto ref_fact = gko::factorization::ParIlu<>::build()
-                        .with_skip_sorting(true)
-                        .on(ref)
-                        ->generate(csr_ref);
-    auto cuda_fact = gko::factorization::Ilu<>::build()
-                         .with_skip_sorting(true)
-                         .on(cuda)
-                         ->generate(csr_cuda);
+    auto ref_fact =
+        gko::factorization::ParIlu<>::build().on(ref)->generate(csr_ref);
+    auto cuda_fact =
+        gko::factorization::Ilu<>::build().on(cuda)->generate(csr_cuda);
 
     GKO_ASSERT_MTX_NEAR(ref_fact->get_l_factor(), cuda_fact->get_l_factor(),
                         1e-14);
diff --git a/cuda/test/preconditioner/jacobi_kernels.cpp b/cuda/test/preconditioner/jacobi_kernels.cpp
index 05ea7d766e8..2d90883a852 100644
--- a/cuda/test/preconditioner/jacobi_kernels.cpp
+++ b/cuda/test/preconditioner/jacobi_kernels.cpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/test/utils.hpp"
+#include "core/test/utils/unsort_matrix.hpp"
 
 
 namespace {
@@ -75,7 +76,7 @@ class Jacobi : public ::testing::Test {
         std::initializer_list<gko::precision_reduction> block_precisions,
         std::initializer_list<double> condition_numbers,
         gko::uint32 max_block_size, int min_nnz, int max_nnz, int num_rhs = 1,
-        double accuracy = 0.1)
+        double accuracy = 0.1, bool skip_sorting = true)
     {
         std::ranlux48 engine(42);
         const auto dim = *(end(block_pointers) - 1);
@@ -101,10 +102,12 @@ class Jacobi : public ::testing::Test {
             bj_factory = Bj::build()
                              .with_max_block_size(max_block_size)
                              .with_block_pointers(block_ptrs)
+                             .with_skip_sorting(skip_sorting)
                              .on(ref);
             d_bj_factory = Bj::build()
                                .with_max_block_size(max_block_size)
                                .with_block_pointers(block_ptrs)
+                               .with_skip_sorting(skip_sorting)
                                .on(cuda);
         } else {
             bj_factory = Bj::build()
@@ -112,12 +115,14 @@ class Jacobi : public ::testing::Test {
                              .with_block_pointers(block_ptrs)
                              .with_storage_optimization(block_prec)
                              .with_accuracy(accuracy)
+                             .with_skip_sorting(skip_sorting)
                              .on(ref);
             d_bj_factory = Bj::build()
                                .with_max_block_size(max_block_size)
                                .with_block_pointers(block_ptrs)
                                .with_storage_optimization(block_prec)
                                .with_accuracy(accuracy)
+                               .with_skip_sorting(skip_sorting)
                                .on(cuda);
         }
         b = gko::test::generate_random_matrix<Vec>(
@@ -290,7 +295,7 @@ TEST_F(Jacobi,
 }
 
 
-TEST_F(Jacobi, CudaPreconditionerEquivalentToRefWithBlockSize32)
+TEST_F(Jacobi, CudaPreconditionerEquivalentToRefWithBlockSize32Sorted)
 {
     initialize_data({0, 32, 64, 96, 128}, {}, {}, 32, 100, 110);
 
@@ -301,6 +306,19 @@ TEST_F(Jacobi, CudaPreconditionerEquivalentToRefWithBlockSize32)
 }
 
 
+TEST_F(Jacobi, CudaPreconditionerEquivalentToRefWithBlockSize32Unsorted)
+{
+    std::ranlux48 engine(42);
+    initialize_data({0, 32, 64, 96, 128}, {}, {}, 32, 100, 110, 1, 0.1, false);
+    gko::test::unsort_matrix(mtx.get(), engine);
+
+    auto bj = bj_factory->generate(mtx);
+    auto d_bj = d_bj_factory->generate(mtx);
+
+    GKO_ASSERT_MTX_NEAR(gko::as<Bj>(d_bj.get()), gko::as<Bj>(bj.get()), 1e-13);
+}
+
+
 TEST_F(Jacobi, CudaPreconditionerEquivalentToRefWithDifferentBlockSize)
 {
     initialize_data({0, 11, 24, 33, 45, 55, 67, 70, 80, 92, 100}, {}, {}, 32,
diff --git a/hip/test/factorization/ilu_kernels.cpp b/hip/test/factorization/ilu_kernels.cpp
index ce69b38602e..3cd8f2be763 100644
--- a/hip/test/factorization/ilu_kernels.cpp
+++ b/hip/test/factorization/ilu_kernels.cpp
@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/factorization/par_ilu.hpp>
 
 
+#include "core/test/utils/unsort_matrix.hpp"
 #include "hip/test/utils.hip.hpp"
 #include "matrices/config.hpp"
 
@@ -61,7 +62,7 @@ class Ilu : public ::testing::Test {
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<gko::HipExecutor> hip;
-    std::randlux48 rand_engine;
+    std::ranlux48 rand_engine;
     std::shared_ptr<Csr> csr_ref;
     std::shared_ptr<Csr> csr_hip;
 
@@ -113,15 +114,10 @@ TEST_F(Ilu, ComputeILUIsEquivalentToRefUnsorted)
     gko::test::unsort_matrix(gko::lend(csr_ref), rand_engine);
     csr_hip->copy_from(gko::lend(csr_ref));
 
-    // TODO: Remove sorting, just for testing here!
-    auto ref_fact = gko::factorization::ParIlu<>::build()
-                        .with_skip_sorting(true)
-                        .on(ref)
-                        ->generate(csr_ref);
-    auto hip_fact = gko::factorization::Ilu<>::build()
-                        .with_skip_sorting(true)
-                        .on(hip)
-                        ->generate(csr_hip);
+    auto ref_fact =
+        gko::factorization::ParIlu<>::build().on(ref)->generate(csr_ref);
+    auto hip_fact =
+        gko::factorization::Ilu<>::build().on(hip)->generate(csr_hip);
 
     GKO_ASSERT_MTX_NEAR(ref_fact->get_l_factor(), hip_fact->get_l_factor(),
                         1e-14);
diff --git a/hip/test/preconditioner/jacobi_kernels.cpp b/hip/test/preconditioner/jacobi_kernels.cpp
index 868e10fbbad..c6a48f2c0c3 100644
--- a/hip/test/preconditioner/jacobi_kernels.cpp
+++ b/hip/test/preconditioner/jacobi_kernels.cpp
@@ -75,7 +75,7 @@ class Jacobi : public ::testing::Test {
         std::initializer_list<gko::precision_reduction> block_precisions,
         std::initializer_list<double> condition_numbers,
         gko::uint32 max_block_size, int min_nnz, int max_nnz, int num_rhs = 1,
-        double accuracy = 0.1)
+        double accuracy = 0.1, bool skip_sorting = true)
     {
         std::ranlux48 engine(42);
         const auto dim = *(end(block_pointers) - 1);
@@ -103,6 +103,7 @@ class Jacobi : public ::testing::Test {
                     .with_max_block_size(max_block_size)
                     .with_block_pointers(block_ptrs)
                     .with_max_block_stride(gko::uint32(hip->get_warp_size()))
+                    .with_skip_sorting(skip_sorting)
                     .on(ref);
             d_bj_factory = Bj::build()
                                .with_max_block_size(max_block_size)
@@ -116,12 +117,14 @@ class Jacobi : public ::testing::Test {
                     .with_max_block_stride(gko::uint32(hip->get_warp_size()))
                     .with_storage_optimization(block_prec)
                     .with_accuracy(accuracy)
+                    .with_skip_sorting(skip_sorting)
                     .on(ref);
             d_bj_factory = Bj::build()
                                .with_max_block_size(max_block_size)
                                .with_block_pointers(block_ptrs)
                                .with_storage_optimization(block_prec)
                                .with_accuracy(accuracy)
+                               .with_skip_sorting(skip_sorting)
                                .on(hip);
         }
         b = gko::test::generate_random_matrix<Vec>(
@@ -294,7 +297,7 @@ TEST_F(Jacobi,
 }
 
 
-TEST_F(Jacobi, HipPreconditionerEquivalentToRefWithBlockSize32)
+TEST_F(Jacobi, HipPreconditionerEquivalentToRefWithBlockSize32Sorted)
 {
     initialize_data({0, 32, 64, 96, 128}, {}, {}, 32, 100, 110);
 
@@ -305,6 +308,19 @@ TEST_F(Jacobi, HipPreconditionerEquivalentToRefWithBlockSize32)
 }
 
 
+TEST_F(Jacobi, HipPreconditionerEquivalentToRefWithBlockSize32Unsorted)
+{
+    std::ranlux48 engine(43);
+    initialize_data({0, 32, 64, 96, 128}, {}, {}, 32, 100, 110, 0.1, false);
+    gko::test::unsort_matrix(mtx.get(), engine);
+
+    auto bj = bj_factory->generate(mtx);
+    auto d_bj = d_bj_factory->generate(mtx);
+
+    GKO_ASSERT_MTX_NEAR(gko::as<Bj>(d_bj.get()), gko::as<Bj>(bj.get()), 1e-13);
+}
+
+
 #if GINKGO_HIP_PLATFORM_HCC
 TEST_F(Jacobi, HipPreconditionerEquivalentToRefWithBlockSize64)
 {
diff --git a/omp/test/preconditioner/jacobi_kernels.cpp b/omp/test/preconditioner/jacobi_kernels.cpp
index 0753d22f008..c1d3a03ee32 100644
--- a/omp/test/preconditioner/jacobi_kernels.cpp
+++ b/omp/test/preconditioner/jacobi_kernels.cpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/test/utils.hpp"
+#include "core/test/utils/unsort_matrix.hpp"
 
 
 namespace {
@@ -74,7 +75,7 @@ class Jacobi : public ::testing::Test {
         std::initializer_list<gko::precision_reduction> block_precisions,
         std::initializer_list<double> condition_numbers,
         gko::uint32 max_block_size, int min_nnz, int max_nnz, int num_rhs = 1,
-        double accuracy = 0.1)
+        double accuracy = 0.1, bool skip_sorting = true)
     {
         std::ranlux48 engine(42);
         const auto dim = *(end(block_pointers) - 1);
@@ -100,10 +101,12 @@ class Jacobi : public ::testing::Test {
             bj_factory = Bj::build()
                              .with_max_block_size(max_block_size)
                              .with_block_pointers(block_ptrs)
+                             .with_skip_sorting(skip_sorting)
                              .on(ref);
             d_bj_factory = Bj::build()
                                .with_max_block_size(max_block_size)
                                .with_block_pointers(block_ptrs)
+                               .with_skip_sorting(skip_sorting)
                                .on(omp);
         } else {
             bj_factory = Bj::build()
@@ -111,12 +114,14 @@ class Jacobi : public ::testing::Test {
                              .with_block_pointers(block_ptrs)
                              .with_storage_optimization(block_prec)
                              .with_accuracy(accuracy)
+                             .with_skip_sorting(skip_sorting)
                              .on(ref);
             d_bj_factory = Bj::build()
                                .with_max_block_size(max_block_size)
                                .with_block_pointers(block_ptrs)
                                .with_storage_optimization(block_prec)
                                .with_accuracy(accuracy)
+                               .with_skip_sorting(skip_sorting)
                                .on(omp);
         }
         b = gko::test::generate_random_matrix<Vec>(
@@ -289,7 +294,7 @@ TEST_F(Jacobi,
 }
 
 
-TEST_F(Jacobi, OmpPreconditionerEquivalentToRefWithBlockSize32)
+TEST_F(Jacobi, OmpPreconditionerEquivalentToRefWithBlockSize32Sorted)
 {
     initialize_data({0, 32, 64, 96, 128}, {}, {}, 32, 100, 110);
 
@@ -300,6 +305,19 @@ TEST_F(Jacobi, OmpPreconditionerEquivalentToRefWithBlockSize32)
 }
 
 
+TEST_F(Jacobi, OmpPreconditionerEquivalentToRefWithBlockSize32Unsorted)
+{
+    std::ranlux48 engine(43);
+    initialize_data({0, 32, 64, 96, 128}, {}, {}, 32, 100, 110, 0.1, false);
+    gko::test::unsort_matrix(mtx.get(), engine);
+
+    auto bj = bj_factory->generate(mtx);
+    auto d_bj = d_bj_factory->generate(mtx);
+
+    GKO_ASSERT_MTX_NEAR(gko::as<Bj>(d_bj.get()), gko::as<Bj>(bj.get()), 1e-14);
+}
+
+
 TEST_F(Jacobi, OmpPreconditionerEquivalentToRefWithDifferentBlockSize)
 {
     initialize_data({0, 11, 24, 33, 45, 55, 67, 70, 80, 92, 100}, {}, {}, 32,

From 77f28af8171baac76c2b308e6460f2eab74b17c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 30 Oct 2020 12:34:41 +0100
Subject: [PATCH 0115/1008] Fix typo in HIP CSR test

---
 hip/test/matrix/csr_kernels.hip.cpp        | 4 ++--
 hip/test/preconditioner/jacobi_kernels.cpp | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index fea89e3ff6d..ea3b3a2395f 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -810,8 +810,8 @@ TEST_F(Csr, RecognizeUnsortedMatrixIsEquivalentToRef)
     bool is_sorted_hip{};
     bool is_sorted_ref{};
 
-    is_sorted_ref = uns_mtx.ref->is_sorted_by_column_index();
-    is_sorted_hip = uns_mtx.hip->is_sorted_by_column_index();
+    is_sorted_ref = mtx->is_sorted_by_column_index();
+    is_sorted_hip = dmtx->is_sorted_by_column_index();
 
     ASSERT_EQ(is_sorted_ref, is_sorted_hip);
 }
diff --git a/hip/test/preconditioner/jacobi_kernels.cpp b/hip/test/preconditioner/jacobi_kernels.cpp
index c6a48f2c0c3..d44716534f4 100644
--- a/hip/test/preconditioner/jacobi_kernels.cpp
+++ b/hip/test/preconditioner/jacobi_kernels.cpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "core/test/utils/unsort_matrix.hpp"
 #include "hip/test/utils.hip.hpp"
 
 
@@ -311,7 +312,7 @@ TEST_F(Jacobi, HipPreconditionerEquivalentToRefWithBlockSize32Sorted)
 TEST_F(Jacobi, HipPreconditionerEquivalentToRefWithBlockSize32Unsorted)
 {
     std::ranlux48 engine(43);
-    initialize_data({0, 32, 64, 96, 128}, {}, {}, 32, 100, 110, 0.1, false);
+    initialize_data({0, 32, 64, 96, 128}, {}, {}, 32, 100, 110, 1, 0.1, false);
     gko::test::unsort_matrix(mtx.get(), engine);
 
     auto bj = bj_factory->generate(mtx);

From 75ab209819bf082013fb77656ebfbd8cbd76e394 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 6 Nov 2020 13:00:20 +0100
Subject: [PATCH 0116/1008] Make core-tests independent of executor

Additionally, address review comments.

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 core/preconditioner/jacobi.cpp         |  1 -
 core/test/utils/unsort_matrix_test.cpp | 36 +++++++++++++++++++++-----
 2 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index a8a8ad7ba2f..6d52c5b4838 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -219,7 +219,6 @@ void Jacobi<ValueType, IndexType>::generate(const LinOp *system_matrix,
         auto editable_csr = csr_type::create(exec);
         as<ConvertibleTo<csr_type>>(system_matrix)
             ->convert_to(lend(editable_csr));
-        // Maybe check if it is sorted first
         editable_csr->sort_by_column_index();
         csr_mtx = decltype(csr_mtx){editable_csr.release(),
                                     std::default_delete<const csr_type>{}};
diff --git a/core/test/utils/unsort_matrix_test.cpp b/core/test/utils/unsort_matrix_test.cpp
index 9e80cdca83e..afeb22ae246 100644
--- a/core/test/utils/unsort_matrix_test.cpp
+++ b/core/test/utils/unsort_matrix_test.cpp
@@ -75,10 +75,8 @@ class UnsortMatrix : public ::testing::Test {
 
     bool is_coo_matrix_sorted(Coo *mtx)
     {
-        auto size = mtx->get_size();
-        auto vals = mtx->get_values();
-        auto rows = mtx->get_row_idxs();
-        auto cols = mtx->get_col_idxs();
+        auto rows = mtx->get_const_row_idxs();
+        auto cols = mtx->get_const_col_idxs();
         auto nnz = mtx->get_num_stored_elements();
 
         if (nnz <= 0) {
@@ -99,6 +97,30 @@ class UnsortMatrix : public ::testing::Test {
         return true;
     }
 
+    bool is_csr_matrix_sorted(Csr *mtx)
+    {
+        auto size = mtx->get_size();
+        auto rows = mtx->get_const_row_ptrs();
+        auto cols = mtx->get_const_col_idxs();
+        auto nnz = mtx->get_num_stored_elements();
+
+        if (nnz <= 0) {
+            return true;
+        }
+
+        for (index_type row = 0; row < size[1]; ++row) {
+            auto prev_col = cols[rows[row]];
+            for (index_type i = rows[row]; i < rows[row + 1]; ++i) {
+                auto cur_col = cols[i];
+                if (prev_col > cur_col) {
+                    return false;
+                }
+                prev_col = cur_col;
+            }
+        }
+        return true;
+    }
+
     std::shared_ptr<const gko::Executor> exec;
     std::ranlux48 rand_engine;
     std::unique_ptr<Dense> mtx;
@@ -110,11 +132,11 @@ TEST_F(UnsortMatrix, CsrWorks)
 {
     auto csr = Csr::create(exec);
     mtx->convert_to(gko::lend(csr));
-    bool was_sorted = csr->is_sorted_by_column_index();
+    bool was_sorted = is_csr_matrix_sorted(gko::lend(csr));
 
     gko::test::unsort_matrix(gko::lend(csr), rand_engine);
 
-    ASSERT_FALSE(csr->is_sorted_by_column_index());
+    ASSERT_FALSE(is_csr_matrix_sorted(gko::lend(csr)));
     ASSERT_TRUE(was_sorted);
     GKO_ASSERT_MTX_NEAR(csr, mtx, 0.);
 }
@@ -124,7 +146,7 @@ TEST_F(UnsortMatrix, CsrWorksWithEmpty)
 {
     auto csr = Csr::create(exec);
     empty->convert_to(gko::lend(csr));
-    bool was_sorted = csr->is_sorted_by_column_index();
+    bool was_sorted = is_csr_matrix_sorted(gko::lend(csr));
 
     gko::test::unsort_matrix(gko::lend(csr), rand_engine);
 

From f9ded77afbdee2c84f2ac795679bf502955a6d0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 6 Nov 2020 13:10:01 +0100
Subject: [PATCH 0117/1008] Search the proper master executor for COO matrices

Additionally, remove unnecessary variables in CSR and COO
`unsort_matrix`.
---
 core/test/utils/unsort_matrix.hpp | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/core/test/utils/unsort_matrix.hpp b/core/test/utils/unsort_matrix.hpp
index 9e0759c8097..1ad95a02c25 100644
--- a/core/test/utils/unsort_matrix.hpp
+++ b/core/test/utils/unsort_matrix.hpp
@@ -116,16 +116,31 @@ void unsort_matrix(matrix::Coo<ValueType, IndexType> *mtx,
     }
 
     const auto &exec = mtx->get_executor();
-    const auto &master = mtx->get_executor()->get_master();
+    auto master = mtx->get_executor()->get_master();
 
+    // If exec is not the master/host, extract the master and perform the
+    // unsorting there, followed by copying it back
     if (exec != master) {
+        constexpr int max_depth{10};  // Max depth that is searched for master
+        bool actual_master_found{false};
+        for (int i = 0; i < max_depth; ++i) {
+            const auto new_master = master->get_master();
+            if (new_master == master) {
+                actual_master_found = true;
+                break;
+            }
+            master = new_master;
+        }
+        if (!actual_master_found) {
+            GKO_NOT_SUPPORTED(exec);
+        }
         auto h_mtx = mtx->clone(master);
         unsort_matrix(lend(h_mtx), engine);
         mtx->copy_from(lend(h_mtx));
         return;
     }
 
-    auto size = mtx->get_size();
+
     auto vals = mtx->get_values();
     auto rows = mtx->get_row_idxs();
     auto cols = mtx->get_col_idxs();

From 00b083e7b80fd872d4f24ca5e1e48fe9ef92e08b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 6 Nov 2020 17:15:52 +0100
Subject: [PATCH 0118/1008] Remove recursive check for master in unsort_matrix

---
 core/test/utils/unsort_matrix.hpp | 32 ++-----------------------------
 1 file changed, 2 insertions(+), 30 deletions(-)

diff --git a/core/test/utils/unsort_matrix.hpp b/core/test/utils/unsort_matrix.hpp
index 1ad95a02c25..9256177d200 100644
--- a/core/test/utils/unsort_matrix.hpp
+++ b/core/test/utils/unsort_matrix.hpp
@@ -61,24 +61,11 @@ void unsort_matrix(matrix::Csr<ValueType, IndexType> *mtx,
         return;
     }
     const auto &exec = mtx->get_executor();
-    auto master = mtx->get_executor()->get_master();
+    const auto &master = exec->get_master();
 
     // If exec is not the master/host, extract the master and perform the
     // unsorting there, followed by copying it back
     if (exec != master) {
-        constexpr int max_depth{10};  // Max depth that is searched for master
-        bool actual_master_found{false};
-        for (int i = 0; i < max_depth; ++i) {
-            const auto new_master = master->get_master();
-            if (new_master == master) {
-                actual_master_found = true;
-                break;
-            }
-            master = new_master;
-        }
-        if (!actual_master_found) {
-            GKO_NOT_SUPPORTED(exec);
-        }
         auto h_mtx = mtx->clone(master);
         unsort_matrix(lend(h_mtx), engine);
         mtx->copy_from(lend(h_mtx));
@@ -110,37 +97,22 @@ void unsort_matrix(matrix::Coo<ValueType, IndexType> *mtx,
     using value_type = ValueType;
     using index_type = IndexType;
     auto nnz = mtx->get_num_stored_elements();
-
     if (nnz <= 0) {
         return;
     }
 
     const auto &exec = mtx->get_executor();
-    auto master = mtx->get_executor()->get_master();
+    const auto &master = exec->get_master();
 
     // If exec is not the master/host, extract the master and perform the
     // unsorting there, followed by copying it back
     if (exec != master) {
-        constexpr int max_depth{10};  // Max depth that is searched for master
-        bool actual_master_found{false};
-        for (int i = 0; i < max_depth; ++i) {
-            const auto new_master = master->get_master();
-            if (new_master == master) {
-                actual_master_found = true;
-                break;
-            }
-            master = new_master;
-        }
-        if (!actual_master_found) {
-            GKO_NOT_SUPPORTED(exec);
-        }
         auto h_mtx = mtx->clone(master);
         unsort_matrix(lend(h_mtx), engine);
         mtx->copy_from(lend(h_mtx));
         return;
     }
 
-
     auto vals = mtx->get_values();
     auto rows = mtx->get_row_idxs();
     auto cols = mtx->get_col_idxs();

From 19d6235f314cc4a9e6cfed898244c66c6688a8ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 11 Nov 2020 12:40:12 +0100
Subject: [PATCH 0119/1008] Fix unsort_matrix core test

by removing the kernel dependency with hand-initializing the matrices.
---
 core/test/utils/unsort_matrix.hpp      |  4 +-
 core/test/utils/unsort_matrix_test.cpp | 67 ++++++++++++++++----------
 2 files changed, 44 insertions(+), 27 deletions(-)

diff --git a/core/test/utils/unsort_matrix.hpp b/core/test/utils/unsort_matrix.hpp
index 9256177d200..9b2074086c5 100644
--- a/core/test/utils/unsort_matrix.hpp
+++ b/core/test/utils/unsort_matrix.hpp
@@ -79,9 +79,9 @@ void unsort_matrix(matrix::Csr<ValueType, IndexType> *mtx,
     for (index_type row = 0; row < size[0]; ++row) {
         auto start = row_ptrs[row];
         auto end = row_ptrs[row + 1];
-        auto iterator = gko::detail::IteratorFactory<IndexType, ValueType>(
+        auto sort_wrapper = gko::detail::IteratorFactory<IndexType, ValueType>(
             cols + start, vals + start, end - start);
-        std::shuffle(iterator.begin(), iterator.end(), engine);
+        std::shuffle(sort_wrapper.begin(), sort_wrapper.end(), engine);
     }
 }
 
diff --git a/core/test/utils/unsort_matrix_test.cpp b/core/test/utils/unsort_matrix_test.cpp
index afeb22ae246..c090686116d 100644
--- a/core/test/utils/unsort_matrix_test.cpp
+++ b/core/test/utils/unsort_matrix_test.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <cmath>
+#include <memory>
 #include <random>
 
 
@@ -64,14 +65,34 @@ class UnsortMatrix : public ::testing::Test {
     UnsortMatrix()
         : exec(gko::ReferenceExecutor::create()),
           rand_engine(42),
-          mtx(gko::initialize<Dense>({{1, 2, 0, 0, 0},
-                                      {0, 0, 0, 0, 0},
-                                      {3, 4, 5, 6, 0},
-                                      {0, 0, 7, 0, 0},
-                                      {0, 0, 8, 9, 10}},
-                                     exec)),
-          empty(Dense::create(exec, gko::dim<2>(0, 0)))
+          /*
+           */
+          csr_empty(Csr::create(exec, gko::dim<2>(0, 0))),
+          coo_empty(Coo::create(exec, gko::dim<2>(0, 0)))
     {}
+    /*
+     Matrix used for both CSR and COO:
+              1, 2, 0, 0, 0
+              0, 0, 0, 0, 0
+              3, 4, 5, 6, 0
+              0, 0, 7, 0, 0
+              0, 0, 8, 9, 10
+     */
+    std::unique_ptr<Csr> get_sorted_csr()
+    {
+        return Csr::create(exec, gko::dim<2>{5, 5},
+                           I<value_type>{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+                           I<index_type>{0, 1, 0, 1, 2, 3, 2, 2, 3, 4},
+                           I<index_type>{0, 2, 2, 6, 7, 10});
+    }
+
+    std::unique_ptr<Coo> get_sorted_coo()
+    {
+        return Coo::create(exec, gko::dim<2>{5, 5},
+                           I<value_type>{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+                           I<index_type>{0, 1, 0, 1, 2, 3, 2, 2, 3, 4},
+                           I<index_type>{0, 0, 2, 2, 2, 2, 3, 4, 4, 4});
+    }
 
     bool is_coo_matrix_sorted(Coo *mtx)
     {
@@ -123,62 +144,58 @@ class UnsortMatrix : public ::testing::Test {
 
     std::shared_ptr<const gko::Executor> exec;
     std::ranlux48 rand_engine;
-    std::unique_ptr<Dense> mtx;
-    std::unique_ptr<Dense> empty;
+    std::unique_ptr<Csr> csr_empty;
+    std::unique_ptr<Coo> coo_empty;
 };
 
 
 TEST_F(UnsortMatrix, CsrWorks)
 {
-    auto csr = Csr::create(exec);
-    mtx->convert_to(gko::lend(csr));
+    auto csr = get_sorted_csr();
+    const auto ref_mtx = get_sorted_csr();
     bool was_sorted = is_csr_matrix_sorted(gko::lend(csr));
 
     gko::test::unsort_matrix(gko::lend(csr), rand_engine);
 
     ASSERT_FALSE(is_csr_matrix_sorted(gko::lend(csr)));
     ASSERT_TRUE(was_sorted);
-    GKO_ASSERT_MTX_NEAR(csr, mtx, 0.);
+    GKO_ASSERT_MTX_NEAR(csr, ref_mtx, 0.);
 }
 
 
 TEST_F(UnsortMatrix, CsrWorksWithEmpty)
 {
-    auto csr = Csr::create(exec);
-    empty->convert_to(gko::lend(csr));
-    bool was_sorted = is_csr_matrix_sorted(gko::lend(csr));
+    const bool was_sorted = is_csr_matrix_sorted(gko::lend(csr_empty));
 
-    gko::test::unsort_matrix(gko::lend(csr), rand_engine);
+    gko::test::unsort_matrix(gko::lend(csr_empty), rand_engine);
 
     ASSERT_TRUE(was_sorted);
-    GKO_ASSERT_MTX_NEAR(csr, empty, 0.);
+    ASSERT_EQ(csr_empty->get_num_stored_elements(), 0);
 }
 
 
 TEST_F(UnsortMatrix, CooWorks)
 {
-    auto coo = Coo::create(exec);
-    mtx->convert_to(gko::lend(coo));
+    auto coo = get_sorted_coo();
+    const auto ref_mtx = get_sorted_coo();
     const bool was_sorted = is_coo_matrix_sorted(gko::lend(coo));
 
     gko::test::unsort_matrix(gko::lend(coo), rand_engine);
 
     ASSERT_FALSE(is_coo_matrix_sorted(gko::lend(coo)));
     ASSERT_TRUE(was_sorted);
-    GKO_ASSERT_MTX_NEAR(coo, mtx, 0.);
+    GKO_ASSERT_MTX_NEAR(coo, ref_mtx, 0.);
 }
 
 
 TEST_F(UnsortMatrix, CooWorksWithEmpty)
 {
-    auto coo = Coo::create(exec);
-    empty->convert_to(gko::lend(coo));
-    const bool was_sorted = is_coo_matrix_sorted(gko::lend(coo));
+    const bool was_sorted = is_coo_matrix_sorted(gko::lend(coo_empty));
 
-    gko::test::unsort_matrix(gko::lend(coo), rand_engine);
+    gko::test::unsort_matrix(gko::lend(coo_empty), rand_engine);
 
     ASSERT_TRUE(was_sorted);
-    GKO_ASSERT_MTX_NEAR(coo, empty, 0.);
+    ASSERT_EQ(coo_empty->get_num_stored_elements(), 0);
 }
 
 

From b3a344b4a36f95a97bff6f9488d6c060de7dff82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 12 Nov 2020 10:51:51 +0100
Subject: [PATCH 0120/1008] Add convert_to_with_sorting helper

Additionally, use this helper function everywhere a sorting with
conditional sorting is required.
---
 core/base/utils.hpp                | 171 +++++++++++++++++++++++++-
 core/factorization/par_ict.cpp     |  27 +----
 core/factorization/par_ilu.cpp     |  23 ++--
 core/factorization/par_ilut.cpp    |  29 ++---
 core/preconditioner/isai.cpp       |  45 +------
 core/preconditioner/jacobi.cpp     |  16 +--
 reference/test/base/CMakeLists.txt |   1 +
 reference/test/base/utils.cpp      | 187 +++++++++++++++++++++++++++++
 8 files changed, 383 insertions(+), 116 deletions(-)
 create mode 100644 reference/test/base/utils.cpp

diff --git a/core/base/utils.hpp b/core/base/utils.hpp
index 4e6fbc1dfce..ace930bd116 100644
--- a/core/base/utils.hpp
+++ b/core/base/utils.hpp
@@ -33,7 +33,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifndef GKO_INTERNAL_CORE_BASE_UTILS_HPP_
 #define GKO_INTERNAL_CORE_BASE_UTILS_HPP_
 
+
+#include <memory>
+#include <type_traits>
+
+
+#include <ginkgo/core/base/polymorphic_object.hpp>
 #include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/base/utils.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
 
 
 namespace gko {
@@ -50,7 +58,168 @@ GKO_ATTRIBUTES GKO_INLINE ValueType checked_load(const ValueType *p,
 
 
 }  // namespace kernels
+
+
+namespace detail {
+
+
+template <typename Dest>
+struct conversion_sort_helper {
+};
+
+template <typename ValueType, typename IndexType>
+struct conversion_sort_helper<matrix::Csr<ValueType, IndexType>> {
+    using mtx_type = matrix::Csr<ValueType, IndexType>;
+    template <typename Source>
+    static std::unique_ptr<mtx_type> get_sorted_conversion(
+        std::shared_ptr<const Executor> &exec, Source *source)
+    {
+        auto editable_mtx = mtx_type::create(exec);
+        as<ConvertibleTo<mtx_type>>(source)->convert_to(lend(editable_mtx));
+        editable_mtx->sort_by_column_index();
+        return editable_mtx;
+    }
+};
+
+
+template <typename Dest, typename Source>
+std::unique_ptr<Dest, std::function<void(Dest *)>> convert_to_with_sorting_impl(
+    std::shared_ptr<const Executor> &exec, Source *obj, bool skip_sorting)
+{
+    if (skip_sorting) {
+        return copy_and_convert_to<Dest>(exec, obj);
+    } else {
+        using decay_dest = std::decay_t<Dest>;
+        auto sorted_mtx =
+            detail::conversion_sort_helper<decay_dest>::get_sorted_conversion(
+                exec, obj);
+        return {sorted_mtx.release(), std::default_delete<Dest>()};
+    }
+}
+
+template <typename Dest, typename Source>
+std::shared_ptr<Dest> convert_to_with_sorting_impl(
+    std::shared_ptr<const Executor> &exec, std::shared_ptr<Source> obj,
+    bool skip_sorting)
+{
+    if (skip_sorting) {
+        return copy_and_convert_to<Dest>(exec, obj);
+    } else {
+        using decay_dest = std::decay_t<Dest>;
+        auto sorted_mtx =
+            detail::conversion_sort_helper<decay_dest>::get_sorted_conversion(
+                exec, obj.get());
+        return {std::move(sorted_mtx)};
+    }
+}
+
+
+}  // namespace detail
+
+
+/**
+ * @internal
+ *
+ * Helper function that converts the given matrix to the Dest format with
+ * additional sorting if requested.
+ *
+ * If the given matrix was already sorted, is on the same executor and with a
+ * dynamic type of `Dest`, the same pointer is returned with an empty
+ * deleter.
+ * In all other cases, a new matrix is created, which stores the converted
+ * matrix.
+ *
+ * @tparam Dest  the type to which the object should be converted
+ * @tparam Source  the type of the source object
+ *
+ * @param exec  the executor where the result should be placed
+ * @param obj  the source object that should be converted
+ * @param skip_sorting  indicator if the resulting matrix should be sorted or
+ *                      not
+ */
+template <typename Dest, typename Source>
+std::unique_ptr<Dest, std::function<void(Dest *)>> convert_to_with_sorting(
+    std::shared_ptr<const Executor> exec, Source *obj, bool skip_sorting)
+{
+    return detail::convert_to_with_sorting_impl<Dest>(exec, obj, skip_sorting);
+}
+
+/**
+ * @copydoc convert_to_with_sorting(std::shared_ptr<const Executor>,
+ * Source *, bool)
+ *
+ * @note This version adds the const qualifier for the result since the input is
+ *       also const
+ */
+template <typename Dest, typename Source>
+std::unique_ptr<const Dest, std::function<void(const Dest *)>>
+convert_to_with_sorting(std::shared_ptr<const Executor> exec, const Source *obj,
+                        bool skip_sorting)
+{
+    return detail::convert_to_with_sorting_impl<const Dest>(exec, obj,
+                                                            skip_sorting);
+}
+
+/**
+ * @copydoc convert_to_with_sorting(std::shared_ptr<const Executor>,
+ * Source *, bool)
+ *
+ * @note This version has a unique_ptr as the source instead of a plain pointer
+ */
+template <typename Dest, typename Source>
+std::unique_ptr<Dest, std::function<void(Dest *)>> convert_to_with_sorting(
+    std::shared_ptr<const Executor> exec, const std::unique_ptr<Source> &obj,
+    bool skip_sorting)
+{
+    return detail::convert_to_with_sorting_impl<Dest>(exec, obj.get(),
+                                                      skip_sorting);
+}
+
+/**
+ * @internal
+ *
+ * Helper function that converts the given matrix to the Dest format with
+ * additional sorting if requested.
+ *
+ * If the given matrix was already sorted, is on the same executor and with a
+ * dynamic type of `Dest`, the same pointer is returned.
+ * In all other cases, a new matrix is created, which stores the converted
+ * matrix.
+ *
+ * @tparam Dest  the type to which the object should be converted
+ * @tparam Source  the type of the source object
+ *
+ * @param exec  the executor where the result should be placed
+ * @param obj  the source object that should be converted
+ * @param skip_sorting  indicator if the resulting matrix should be sorted or
+ *                      not
+ */
+template <typename Dest, typename Source>
+std::shared_ptr<Dest> convert_to_with_sorting(
+    std::shared_ptr<const Executor> exec, std::shared_ptr<Source> obj,
+    bool skip_sorting)
+{
+    return detail::convert_to_with_sorting_impl<Dest>(exec, obj, skip_sorting);
+}
+
+/**
+ * @copydoc convert_to_with_sorting(std::shared_ptr<const Executor>,
+ * std::shared_ptr<Source>, bool)
+ *
+ * @note This version adds the const qualifier for the result since the input is
+ *       also const
+ */
+template <typename Dest, typename Source>
+std::shared_ptr<const Dest> convert_to_with_sorting(
+    std::shared_ptr<const Executor> exec, std::shared_ptr<const Source> obj,
+    bool skip_sorting)
+{
+    return detail::convert_to_with_sorting_impl<const Dest>(exec, obj,
+                                                            skip_sorting);
+}
+
+
 }  // namespace gko
 
 
-#endif  // GKO_INTERNAL_CORE_BASE_UTILS_HPP_
\ No newline at end of file
+#endif  // GKO_INTERNAL_CORE_BASE_UTILS_HPP_
diff --git a/core/factorization/par_ict.cpp b/core/factorization/par_ict.cpp
index a1a1408fb79..40d14a730d5 100644
--- a/core/factorization/par_ict.cpp
+++ b/core/factorization/par_ict.cpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/csr.hpp>
 
 
+#include "core/base/utils.hpp"
 #include "core/factorization/factorization_kernels.hpp"
 #include "core/factorization/par_ict_kernels.hpp"
 #include "core/factorization/par_ilu_kernels.hpp"
@@ -175,30 +176,14 @@ ParIct<ValueType, IndexType>::generate_l_lt(
     const auto exec = this->get_executor();
 
     // convert and/or sort the matrix if necessary
-    std::unique_ptr<CsrMatrix> csr_system_matrix_unique_ptr{};
-    auto csr_system_matrix =
-        dynamic_cast<const CsrMatrix *>(system_matrix.get());
-    if (csr_system_matrix == nullptr ||
-        csr_system_matrix->get_executor() != exec) {
-        csr_system_matrix_unique_ptr = CsrMatrix::create(exec);
-        as<ConvertibleTo<CsrMatrix>>(system_matrix.get())
-            ->convert_to(csr_system_matrix_unique_ptr.get());
-        csr_system_matrix = csr_system_matrix_unique_ptr.get();
-    }
-    if (!parameters_.skip_sorting) {
-        if (csr_system_matrix_unique_ptr == nullptr) {
-            csr_system_matrix_unique_ptr = CsrMatrix::create(exec);
-            csr_system_matrix_unique_ptr->copy_from(csr_system_matrix);
-        }
-        csr_system_matrix_unique_ptr->sort_by_column_index();
-        csr_system_matrix = csr_system_matrix_unique_ptr.get();
-    }
+    auto csr_system_matrix = convert_to_with_sorting<CsrMatrix>(
+        exec, system_matrix, parameters_.skip_sorting);
 
     // initialize the L matrix data structures
     const auto num_rows = csr_system_matrix->get_size()[0];
     Array<IndexType> l_row_ptrs_array{exec, num_rows + 1};
     auto l_row_ptrs = l_row_ptrs_array.get_data();
-    exec->run(make_initialize_row_ptrs_l(csr_system_matrix, l_row_ptrs));
+    exec->run(make_initialize_row_ptrs_l(csr_system_matrix.get(), l_row_ptrs));
 
     auto l_nnz =
         static_cast<size_type>(exec->copy_val_to_host(l_row_ptrs + num_rows));
@@ -209,14 +194,14 @@ ParIct<ValueType, IndexType>::generate_l_lt(
                                std::move(l_row_ptrs_array));
 
     // initialize L
-    exec->run(make_initialize_l(csr_system_matrix, l.get(), true));
+    exec->run(make_initialize_l(csr_system_matrix.get(), l.get(), true));
 
     // compute limit #nnz for L
     auto l_nnz_limit =
         static_cast<IndexType>(l_nnz * parameters_.fill_in_limit);
 
     ParIctState<ValueType, IndexType> state{exec,
-                                            csr_system_matrix,
+                                            csr_system_matrix.get(),
                                             std::move(l),
                                             l_nnz_limit,
                                             parameters_.approximate_select,
diff --git a/core/factorization/par_ilu.cpp b/core/factorization/par_ilu.cpp
index d61a27747af..fc984415c41 100644
--- a/core/factorization/par_ilu.cpp
+++ b/core/factorization/par_ilu.cpp
@@ -83,10 +83,9 @@ ParIlu<ValueType, IndexType>::generate_l_u(
 
     // Converts the system matrix to CSR.
     // Throws an exception if it is not convertible.
-    auto csr_system_matrix_unique_ptr = CsrMatrix::create(exec);
+    auto csr_system_matrix = CsrMatrix::create(exec);
     as<ConvertibleTo<CsrMatrix>>(system_matrix.get())
-        ->convert_to(csr_system_matrix_unique_ptr.get());
-    auto csr_system_matrix = csr_system_matrix_unique_ptr.get();
+        ->convert_to(csr_system_matrix.get());
     // If necessary, sort it
     if (!skip_sorting) {
         csr_system_matrix->sort_by_column_index();
@@ -94,14 +93,14 @@ ParIlu<ValueType, IndexType>::generate_l_u(
 
     // Add explicit diagonal zero elements if they are missing
     exec->run(par_ilu_factorization::make_add_diagonal_elements(
-        csr_system_matrix, true));
+        csr_system_matrix.get(), true));
 
     const auto matrix_size = csr_system_matrix->get_size();
     const auto number_rows = matrix_size[0];
     Array<IndexType> l_row_ptrs{exec, number_rows + 1};
     Array<IndexType> u_row_ptrs{exec, number_rows + 1};
     exec->run(par_ilu_factorization::make_initialize_row_ptrs_l_u(
-        csr_system_matrix, l_row_ptrs.get_data(), u_row_ptrs.get_data()));
+        csr_system_matrix.get(), l_row_ptrs.get_data(), u_row_ptrs.get_data()));
 
     // Get nnz from device memory
     auto l_nnz = static_cast<size_type>(
@@ -123,7 +122,7 @@ ParIlu<ValueType, IndexType>::generate_l_u(
         std::move(u_row_ptrs), u_strategy);
 
     exec->run(par_ilu_factorization::make_initialize_l_u(
-        csr_system_matrix, l_factor.get(), u_factor.get()));
+        csr_system_matrix.get(), l_factor.get(), u_factor.get()));
 
     // We use `transpose()` here to convert the Csr format to Csc.
     auto u_factor_transpose_lin_op = u_factor->transpose();
@@ -140,18 +139,10 @@ ParIlu<ValueType, IndexType>::generate_l_u(
 
     // If it was not, and we already own a CSR `system_matrix`,
     // we can move the Csr matrix to Coo, which has very little overhead.
-    // Otherwise, we convert from the Csr matrix, since it is the conversion
-    // with the least overhead.
-    // We also have to convert / move from the CSR matrix if it was not already
-    // sorted (in which case we definitively own a CSR `system_matrix`).
+    // We also have to move from the CSR matrix if it was not already sorted.
     if (!skip_sorting || coo_system_matrix_ptr == nullptr) {
         coo_system_matrix_unique_ptr = CooMatrix::create(exec);
-        if (csr_system_matrix_unique_ptr == nullptr) {
-            csr_system_matrix->convert_to(coo_system_matrix_unique_ptr.get());
-        } else {
-            csr_system_matrix_unique_ptr->move_to(
-                coo_system_matrix_unique_ptr.get());
-        }
+        csr_system_matrix->move_to(coo_system_matrix_unique_ptr.get());
         coo_system_matrix_ptr = coo_system_matrix_unique_ptr.get();
     }
 
diff --git a/core/factorization/par_ilut.cpp b/core/factorization/par_ilut.cpp
index 1eb3dfeb950..a68a7d11c91 100644
--- a/core/factorization/par_ilut.cpp
+++ b/core/factorization/par_ilut.cpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/csr.hpp>
 
 
+#include "core/base/utils.hpp"
 #include "core/factorization/factorization_kernels.hpp"
 #include "core/factorization/par_ilu_kernels.hpp"
 #include "core/factorization/par_ilut_kernels.hpp"
@@ -191,24 +192,8 @@ ParIlut<ValueType, IndexType>::generate_l_u(
     const auto exec = this->get_executor();
 
     // convert and/or sort the matrix if necessary
-    std::unique_ptr<CsrMatrix> csr_system_matrix_unique_ptr{};
-    auto csr_system_matrix =
-        dynamic_cast<const CsrMatrix *>(system_matrix.get());
-    if (csr_system_matrix == nullptr ||
-        csr_system_matrix->get_executor() != exec) {
-        csr_system_matrix_unique_ptr = CsrMatrix::create(exec);
-        as<ConvertibleTo<CsrMatrix>>(system_matrix.get())
-            ->convert_to(csr_system_matrix_unique_ptr.get());
-        csr_system_matrix = csr_system_matrix_unique_ptr.get();
-    }
-    if (!parameters_.skip_sorting) {
-        if (csr_system_matrix_unique_ptr == nullptr) {
-            csr_system_matrix_unique_ptr = CsrMatrix::create(exec);
-            csr_system_matrix_unique_ptr->copy_from(csr_system_matrix);
-        }
-        csr_system_matrix_unique_ptr->sort_by_column_index();
-        csr_system_matrix = csr_system_matrix_unique_ptr.get();
-    }
+    auto csr_system_matrix = convert_to_with_sorting<CsrMatrix>(
+        exec, system_matrix, parameters_.skip_sorting);
 
     // initialize the L and U matrix data structures
     const auto num_rows = csr_system_matrix->get_size()[0];
@@ -216,7 +201,7 @@ ParIlut<ValueType, IndexType>::generate_l_u(
     Array<IndexType> u_row_ptrs_array{exec, num_rows + 1};
     auto l_row_ptrs = l_row_ptrs_array.get_data();
     auto u_row_ptrs = u_row_ptrs_array.get_data();
-    exec->run(make_initialize_row_ptrs_l_u(csr_system_matrix, l_row_ptrs,
+    exec->run(make_initialize_row_ptrs_l_u(csr_system_matrix.get(), l_row_ptrs,
                                            u_row_ptrs));
 
     auto l_nnz =
@@ -233,7 +218,7 @@ ParIlut<ValueType, IndexType>::generate_l_u(
                                std::move(u_row_ptrs_array));
 
     // initialize L and U
-    exec->run(make_initialize_l_u(csr_system_matrix, l.get(), u.get()));
+    exec->run(make_initialize_l_u(csr_system_matrix.get(), l.get(), u.get()));
 
     // compute limit #nnz for L and U
     auto l_nnz_limit =
@@ -242,7 +227,7 @@ ParIlut<ValueType, IndexType>::generate_l_u(
         static_cast<IndexType>(u_nnz * parameters_.fill_in_limit);
 
     ParIlutState<ValueType, IndexType> state{exec,
-                                             csr_system_matrix,
+                                             csr_system_matrix.get(),
                                              std::move(l),
                                              std::move(u),
                                              l_nnz_limit,
@@ -352,4 +337,4 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_PAR_ILUT);
 
 
 }  // namespace factorization
-}  // namespace gko
\ No newline at end of file
+}  // namespace gko
diff --git a/core/preconditioner/isai.cpp b/core/preconditioner/isai.cpp
index 0b8738c5594..108ca300acc 100644
--- a/core/preconditioner/isai.cpp
+++ b/core/preconditioner/isai.cpp
@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/upper_trs.hpp>
 
 
+#include "core/base/utils.hpp"
 #include "core/preconditioner/isai_kernels.hpp"
 
 
@@ -62,48 +63,6 @@ GKO_REGISTER_OPERATION(scatter_excess_solution, isai::scatter_excess_solution);
 }  // namespace isai
 
 
-/**
- * @internal
- *
- * Helper function that converts the given matrix to the (const) CSR format with
- * additional sorting.
- *
- * If the given matrix was already sorted, is on the same executor and with a
- * dynamic type of `const Csr`, the same pointer is returned with an empty
- * deleter.
- * In all other cases, a new matrix is created, which stores the converted Csr
- * matrix.
- * If `skip_sorting` is false, the matrix will be sorted by column index,
- * otherwise, it will not be sorted.
- */
-template <typename Csr>
-std::shared_ptr<const Csr> convert_to_csr_and_sort(
-    std::shared_ptr<const Executor> &exec, std::shared_ptr<const LinOp> mtx,
-    bool skip_sorting)
-{
-    static_assert(
-        std::is_same<Csr, matrix::Csr<typename Csr::value_type,
-                                      typename Csr::index_type>>::value,
-        "The given `Csr` type must be of type `matrix::Csr`!");
-    if (skip_sorting && exec == mtx->get_executor()) {
-        auto csr_mtx = std::dynamic_pointer_cast<const Csr>(mtx);
-        if (csr_mtx) {
-            // Here, we can just forward the pointer with an empty deleter
-            // since it is already sorted and in the correct format
-            return csr_mtx;
-        }
-    }
-    auto copy = Csr::create(exec);
-    as<ConvertibleTo<Csr>>(mtx)->convert_to(lend(copy));
-    // Here, we assume that a sorted matrix converted to CSR will also be
-    // sorted
-    if (!skip_sorting) {
-        copy->sort_by_column_index();
-    }
-    return {std::move(copy)};
-}
-
-
 /**
  * @internal
  *
@@ -156,7 +115,7 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
     using UpperTrs = solver::UpperTrs<ValueType, IndexType>;
     GKO_ASSERT_IS_SQUARE_MATRIX(input);
     auto exec = this->get_executor();
-    auto to_invert = convert_to_csr_and_sort<Csr>(exec, input, skip_sorting);
+    auto to_invert = convert_to_with_sorting<Csr>(exec, input, skip_sorting);
     auto inverted = extend_sparsity(exec, to_invert, power);
     auto num_rows = inverted->get_size()[0];
     auto is_lower = IsaiType == isai_type::lower;
diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index 6d52c5b4838..b59a6289a46 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/base/extended_float.hpp"
+#include "core/base/utils.hpp"
 #include "core/preconditioner/jacobi_kernels.hpp"
 #include "core/preconditioner/jacobi_utils.hpp"
 
@@ -210,19 +211,8 @@ void Jacobi<ValueType, IndexType>::generate(const LinOp *system_matrix,
     GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix);
     using csr_type = matrix::Csr<ValueType, IndexType>;
     const auto exec = this->get_executor();
-    decltype(copy_and_convert_to<csr_type>(exec, system_matrix)) csr_mtx{};
-
-    if (skip_sorting) {
-        csr_mtx = copy_and_convert_to<matrix::Csr<ValueType, IndexType>>(
-            exec, system_matrix);
-    } else {
-        auto editable_csr = csr_type::create(exec);
-        as<ConvertibleTo<csr_type>>(system_matrix)
-            ->convert_to(lend(editable_csr));
-        editable_csr->sort_by_column_index();
-        csr_mtx = decltype(csr_mtx){editable_csr.release(),
-                                    std::default_delete<const csr_type>{}};
-    }
+    auto csr_mtx =
+        convert_to_with_sorting<csr_type>(exec, system_matrix, skip_sorting);
 
     if (parameters_.block_pointers.get_data() == nullptr) {
         this->detect_blocks(csr_mtx.get());
diff --git a/reference/test/base/CMakeLists.txt b/reference/test/base/CMakeLists.txt
index 3b86589507e..3386bb01e20 100644
--- a/reference/test/base/CMakeLists.txt
+++ b/reference/test/base/CMakeLists.txt
@@ -1,3 +1,4 @@
 ginkgo_create_test(combination)
 ginkgo_create_test(composition)
 ginkgo_create_test(perturbation)
+ginkgo_create_test(utils)
diff --git a/reference/test/base/utils.cpp b/reference/test/base/utils.cpp
new file mode 100644
index 00000000000..6a4b7d4b9bb
--- /dev/null
+++ b/reference/test/base/utils.cpp
@@ -0,0 +1,187 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/base/utils.hpp"
+
+
+#include <memory>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/polymorphic_object.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/test/utils.hpp"
+#include "core/test/utils/unsort_matrix.hpp"
+
+
+namespace {
+
+
+class ConvertToWithSorting : public ::testing::Test {
+protected:
+    using value_type = double;
+    using index_type = gko::int32;
+    using Dense = gko::matrix::Dense<value_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    using Coo = gko::matrix::Coo<value_type, index_type>;
+
+    ConvertToWithSorting()
+        : ref{gko::ReferenceExecutor::create()},
+          mtx{gko::initialize<Dense>({{1, 2, 3}, {6, 0, 7}, {-1, 8, 0}}, ref)},
+          unsorted_coo{Coo::create(ref, gko::dim<2>{3, 3},
+                                   I<value_type>{1, 3, 2, 7, 6, -1, 8},
+                                   I<index_type>{0, 2, 1, 2, 0, 0, 1},
+                                   I<index_type>{0, 0, 0, 1, 1, 2, 2})},
+          unsorted_csr{Csr::create(
+              ref, gko::dim<2>{3, 3}, I<value_type>{1, 3, 2, 7, 6, -1, 8},
+              I<index_type>{0, 2, 1, 2, 0, 0, 1}, I<index_type>{0, 3, 5, 7})}
+
+    {}
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::unique_ptr<Dense> mtx;
+    std::unique_ptr<Coo> unsorted_coo;
+    std::unique_ptr<Csr> unsorted_csr;
+};
+
+
+TEST_F(ConvertToWithSorting, SortWithUniquePtr)
+{
+    auto result = gko::convert_to_with_sorting<Csr>(ref, unsorted_coo, false);
+
+    ASSERT_TRUE(result->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_NEAR(result.get(), mtx.get(), 0.);
+}
+
+
+TEST_F(ConvertToWithSorting, DontSortWithUniquePtr)
+{
+    auto result = gko::convert_to_with_sorting<Csr>(ref, unsorted_csr, true);
+
+    ASSERT_EQ(result.get(), unsorted_csr.get());
+    GKO_ASSERT_MTX_NEAR(result.get(), mtx.get(), 0.);
+}
+
+
+TEST_F(ConvertToWithSorting, SortWithSharedPtr)
+{
+    std::shared_ptr<Csr> shared = gko::share(unsorted_csr);
+
+    auto result = gko::convert_to_with_sorting<Csr>(ref, shared, false);
+
+    ASSERT_TRUE(result->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_NEAR(result.get(), mtx.get(), 0.);
+}
+
+
+TEST_F(ConvertToWithSorting, DontSortWithSharedPtr)
+{
+    std::shared_ptr<Csr> shared = gko::share(unsorted_csr);
+
+    auto result = gko::convert_to_with_sorting<Csr>(ref, shared, true);
+
+    ASSERT_EQ(result.get(), shared.get());
+    GKO_ASSERT_MTX_NEAR(result.get(), mtx.get(), 0.);
+}
+
+
+TEST_F(ConvertToWithSorting, SortWithSharedConstPtr)
+{
+    std::shared_ptr<const Coo> shared = gko::share(unsorted_coo);
+
+    auto result = gko::convert_to_with_sorting<Csr>(ref, shared, false);
+
+    ASSERT_TRUE(result->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_NEAR(result.get(), mtx.get(), 0.);
+}
+
+
+TEST_F(ConvertToWithSorting, DontSortWithSharedConstPtr)
+{
+    std::shared_ptr<const Coo> shared = gko::share(unsorted_coo);
+
+    auto result = gko::convert_to_with_sorting<Csr>(ref, shared, true);
+
+    GKO_ASSERT_MTX_NEAR(result.get(), mtx.get(), 0.);
+}
+
+
+TEST_F(ConvertToWithSorting, SortWithRawPtr)
+{
+    auto result =
+        gko::convert_to_with_sorting<Csr>(ref, unsorted_coo.get(), false);
+
+    ASSERT_TRUE(result->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_NEAR(result.get(), mtx.get(), 0.);
+}
+
+
+TEST_F(ConvertToWithSorting, DontSortWithRawPtr)
+{
+    auto result =
+        gko::convert_to_with_sorting<Csr>(ref, unsorted_coo.get(), true);
+
+    GKO_ASSERT_MTX_NEAR(result.get(), mtx.get(), 0.);
+}
+
+
+TEST_F(ConvertToWithSorting, SortWithConstRawPtr)
+{
+    const Coo *cptr = unsorted_coo.get();
+
+    auto result = gko::convert_to_with_sorting<Csr>(ref, cptr, false);
+
+    ASSERT_TRUE(result->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_NEAR(result.get(), mtx.get(), 0.);
+}
+
+
+TEST_F(ConvertToWithSorting, DontSortWithConstRawPtr)
+{
+    const auto cptr = mtx.get();
+
+    auto result = gko::convert_to_with_sorting<Csr>(ref, cptr, true);
+
+    GKO_ASSERT_MTX_NEAR(result.get(), mtx.get(), 0.);
+}
+
+
+}  // namespace

From c088d3111476428509245d631eb81d42ac500e6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 12 Nov 2020 11:45:33 +0100
Subject: [PATCH 0121/1008] Change the unsorting of COO matrices

by using matrix_data:
1. shuffling all non-zeros
2. sorting by row
This takes longer, but does not rely on the COO sorting by row index
---
 core/test/utils/unsort_matrix.hpp | 31 ++++++++++---------------------
 1 file changed, 10 insertions(+), 21 deletions(-)

diff --git a/core/test/utils/unsort_matrix.hpp b/core/test/utils/unsort_matrix.hpp
index 9b2074086c5..3a4b83b6b4e 100644
--- a/core/test/utils/unsort_matrix.hpp
+++ b/core/test/utils/unsort_matrix.hpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/matrix_data.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 
 
@@ -88,8 +89,6 @@ void unsort_matrix(matrix::Csr<ValueType, IndexType> *mtx,
 
 // Plan for now: shuffle values and column indices to unsort the given matrix
 // without changing the represented matrix.
-// Note: This expects COO to be properly sorted by row index (which is required
-//       by the Ginkgo COO format)
 template <typename ValueType, typename IndexType, typename RandomEngine>
 void unsort_matrix(matrix::Coo<ValueType, IndexType> *mtx,
                    RandomEngine &&engine)
@@ -112,25 +111,15 @@ void unsort_matrix(matrix::Coo<ValueType, IndexType> *mtx,
         mtx->copy_from(lend(h_mtx));
         return;
     }
-
-    auto vals = mtx->get_values();
-    auto rows = mtx->get_row_idxs();
-    auto cols = mtx->get_col_idxs();
-
-    auto current_row = rows[0];
-    for (IndexType i = 0; i < nnz;) {
-        auto start = i;
-        while (i < nnz && rows[i] == current_row) {
-            ++i;
-        }
-        current_row = rows[i];
-        auto end = i;
-        auto iterator = gko::detail::IteratorFactory<IndexType, ValueType>(
-            cols + start, vals + start, end - start);
-        // since the row entries are supposed to be the same, there is no need
-        // to swap
-        std::shuffle(iterator.begin(), iterator.end(), engine);
-    }
+    matrix_data<value_type, index_type> data;
+    mtx->write(data);
+    auto &nonzeros = data.nonzeros;
+    using nz_type = typename decltype(data)::nonzero_type;
+
+    std::shuffle(nonzeros.begin(), nonzeros.end(), engine);
+    std::stable_sort(nonzeros.begin(), nonzeros.end(),
+                     [](nz_type a, nz_type b) { return a.row < b.row; });
+    mtx->read(data);
 }
 
 

From 44453dad41e1a463c3e62882f797326437e5a7e3 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 13 Nov 2020 18:06:59 +0100
Subject: [PATCH 0122/1008] add support for make_temporary_clone to Array

---
 core/test/base/array.cpp           | 32 ++++++++++++++++++
 include/ginkgo/core/base/array.hpp | 53 ++++++++++++++++++++++++++++++
 include/ginkgo/core/base/utils.hpp | 30 +++++++++++++----
 3 files changed, 109 insertions(+), 6 deletions(-)

diff --git a/core/test/base/array.cpp b/core/test/base/array.cpp
index ca6db900f94..da62a133162 100644
--- a/core/test/base/array.cpp
+++ b/core/test/base/array.cpp
@@ -263,6 +263,38 @@ TYPED_TEST(Array, CanBeMovedFromExecutorlessArray)
 }
 
 
+TYPED_TEST(Array, CanCreateTemporaryCloneOnSameExecutor)
+{
+    auto tmp_clone = make_temporary_clone(this->exec, &this->x);
+
+    ASSERT_EQ(tmp_clone.get(), &this->x);
+}
+
+
+TYPED_TEST(Array, CanCreateTemporaryCloneOnDifferentExecutor)
+{
+    auto omp = gko::OmpExecutor::create();
+
+    auto tmp_clone = make_temporary_clone(omp, &this->x);
+
+    this->assert_equal_to_original_x(*tmp_clone.get());
+    ASSERT_NE(tmp_clone.get(), &this->x);
+}
+
+
+TYPED_TEST(Array, CanCopyBackTemporaryCloneOnDifferentExecutor)
+{
+    auto omp = gko::OmpExecutor::create();
+
+    {
+        auto tmp_clone = make_temporary_clone(omp, &this->x);
+        this->x.get_data()[0] = 0;
+    }
+
+    this->assert_equal_to_original_x(this->x);
+}
+
+
 TYPED_TEST(Array, CanBeCleared)
 {
     this->x.clear();
diff --git a/include/ginkgo/core/base/array.hpp b/include/ginkgo/core/base/array.hpp
index 7f0df5c27f6..22f063433fe 100644
--- a/include/ginkgo/core/base/array.hpp
+++ b/include/ginkgo/core/base/array.hpp
@@ -563,6 +563,59 @@ class Array {
 };
 
 
+namespace detail {
+
+
+template <typename T>
+struct temporary_clone_helper<Array<T>> {
+    static std::unique_ptr<Array<T>> create(
+        std::shared_ptr<const Executor> exec, Array<T> *ptr)
+    {
+        return std::make_unique<Array<T>>(std::move(exec), *ptr);
+    }
+};
+
+template <typename T>
+struct temporary_clone_helper<const Array<T>> {
+    static std::unique_ptr<const Array<T>> create(
+        std::shared_ptr<const Executor> exec, const Array<T> *ptr)
+    {
+        return std::make_unique<const Array<T>>(std::move(exec), *ptr);
+    }
+};
+
+
+// specialization for non-constant arrays, copying back via assignment
+template <typename T>
+class copy_back_deleter<Array<T>> {
+public:
+    using pointer = Array<T> *;
+
+    /**
+     * Creates a new deleter object.
+     *
+     * @param original  the origin object where the data will be copied before
+     *                  deletion
+     */
+    copy_back_deleter(pointer original) : original_{original} {}
+
+    /**
+     * Deletes the object.
+     *
+     * @param ptr  pointer to the object being deleted
+     */
+    void operator()(pointer ptr) const
+    {
+        *original_ = *ptr;
+        delete ptr;
+    }
+
+private:
+    pointer original_;
+};
+
+
+}  // namespace detail
 }  // namespace gko
 
 
diff --git a/include/ginkgo/core/base/utils.hpp b/include/ginkgo/core/base/utils.hpp
index d3b81ffff23..12a2d0ef3e3 100644
--- a/include/ginkgo/core/base/utils.hpp
+++ b/include/ginkgo/core/base/utils.hpp
@@ -432,6 +432,9 @@ class null_deleter<T[]> {
 };
 
 
+namespace detail {
+
+
 /**
  * A copy_back_deleter is a type of deleter that copies the data to an
  * internally referenced object before performing the deletion.
@@ -490,6 +493,16 @@ class copy_back_deleter<const T> {
 };
 
 
+template <typename T>
+struct temporary_clone_helper {
+    static std::unique_ptr<T> create(std::shared_ptr<const Executor> exec,
+                                     T *ptr)
+    {
+        return gko::clone(std::move(exec), ptr);
+    }
+};
+
+
 /**
  * A temporary_clone is a special smart pointer-like object that is designed to
  * hold an object temporarily copied to another executor.
@@ -517,12 +530,14 @@ class temporary_clone {
     {
         if (ptr->get_executor() == exec) {
             // just use the object we already have
-            handle_ = handle_type(ptr, [](pointer) {});
+            handle_ = handle_type(ptr, null_deleter<T>());
         } else {
             // clone the object to the new executor and make sure it's copied
             // back before we delete it
-            handle_ = handle_type(gko::clone(std::move(exec), ptr).release(),
-                                  copy_back_deleter<T>(ptr));
+            handle_ = handle_type(
+                temporary_clone_helper<T>::create(std::move(exec), ptr)
+                    .release(),
+                copy_back_deleter<T>(ptr));
         }
     }
 
@@ -548,6 +563,9 @@ class temporary_clone {
 };
 
 
+}  // namespace detail
+
+
 /**
  * Creates a temporary_clone.
  *
@@ -559,10 +577,10 @@ class temporary_clone {
  * @param ptr  a pointer to the object of which the clone will be created
  */
 template <typename T>
-temporary_clone<T> make_temporary_clone(std::shared_ptr<const Executor> exec,
-                                        T *ptr)
+detail::temporary_clone<T> make_temporary_clone(
+    std::shared_ptr<const Executor> exec, T *ptr)
 {
-    return temporary_clone<T>(std::move(exec), ptr);
+    return detail::temporary_clone<T>(std::move(exec), ptr);
 }
 
 

From 0fe105ec3e1818c3f7678a729699b0713b2ccb4e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 13 Nov 2020 19:45:34 +0100
Subject: [PATCH 0123/1008] add dense row_gather and improve multiexec permute

Permutation now works when the permutation array
resides on another executor.
---
 common/matrix/dense_kernels.hpp.inc      |   2 +-
 core/device_hooks/common_kernels.inc.cpp |   4 +-
 core/matrix/csr.cpp                      |  18 ++--
 core/matrix/dense.cpp                    | 123 +++++++++++++++++------
 core/matrix/dense_kernels.hpp            |  12 +--
 cuda/matrix/dense_kernels.cu             |  21 ++--
 cuda/test/matrix/csr_kernels.cpp         |  12 +--
 cuda/test/matrix/dense_kernels.cpp       |  49 +++++++--
 dpcpp/matrix/dense_kernels.dp.cpp        |  10 +-
 hip/matrix/dense_kernels.hip.cpp         |  21 ++--
 hip/test/matrix/csr_kernels.hip.cpp      |  12 +--
 hip/test/matrix/dense_kernels.hip.cpp    |  49 +++++++--
 include/ginkgo/core/matrix/dense.hpp     |  10 ++
 omp/matrix/dense_kernels.cpp             |  16 +--
 omp/test/matrix/csr_kernels.cpp          |  12 +--
 omp/test/matrix/dense_kernels.cpp        |  53 ++++++++--
 reference/matrix/dense_kernels.cpp       |  16 +--
 reference/test/matrix/dense_kernels.cpp  |  43 ++++++++
 18 files changed, 344 insertions(+), 139 deletions(-)

diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index a9947ad891f..747356e4594 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -422,7 +422,7 @@ __global__ __launch_bounds__(default_block_size) void reduce_total_cols(
 
 
 template <size_type block_size, typename IndexType, typename ValueType>
-__global__ __launch_bounds__(block_size) void row_permute(
+__global__ __launch_bounds__(block_size) void row_gather(
     size_type num_rows, size_type num_cols,
     const IndexType *__restrict__ perm_idxs, const ValueType *__restrict__ orig,
     size_type stride_orig, ValueType *__restrict__ result,
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 34992284498..5d8e72e9160 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -218,10 +218,10 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 template <typename ValueType, typename IndexType>
-GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL(ValueType, IndexType)
+GKO_DECLARE_DENSE_ROW_GATHER_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_ROW_GATHER_KERNEL);
 
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)
diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp
index c6d410aee16..a9a40613e78 100644
--- a/core/matrix/csr.cpp
+++ b/core/matrix/csr.cpp
@@ -412,8 +412,9 @@ std::unique_ptr<LinOp> Csr<ValueType, IndexType>::row_permute(
         Csr::create(exec, this->get_size(), this->get_num_stored_elements(),
                     this->get_strategy());
 
-    exec->run(csr::make_row_permute(permutation_indices->get_const_data(), this,
-                                    permute_cpy.get()));
+    exec->run(csr::make_row_permute(
+        make_temporary_clone(exec, permutation_indices)->get_const_data(), this,
+        permute_cpy.get()));
     permute_cpy->make_srow();
     return std::move(permute_cpy);
 }
@@ -431,7 +432,8 @@ std::unique_ptr<LinOp> Csr<ValueType, IndexType>::column_permute(
     Array<IndexType> inv_permutation(exec, this->get_size()[1]);
 
     exec->run(csr::make_invert_permutation(
-        this->get_size()[1], permutation_indices->get_const_data(),
+        this->get_size()[1],
+        make_temporary_clone(exec, permutation_indices)->get_const_data(),
         inv_permutation.get_data()));
     exec->run(csr::make_inverse_column_permute(inv_permutation.get_const_data(),
                                                this, permute_cpy.get()));
@@ -453,8 +455,9 @@ std::unique_ptr<LinOp> Csr<ValueType, IndexType>::inverse_row_permute(
                     this->get_strategy());
 
     exec->run(csr::make_inverse_row_permute(
-        inverse_permutation_indices->get_const_data(), this,
-        inverse_permute_cpy.get()));
+        make_temporary_clone(exec, inverse_permutation_indices)
+            ->get_const_data(),
+        this, inverse_permute_cpy.get()));
     inverse_permute_cpy->make_srow();
     return std::move(inverse_permute_cpy);
 }
@@ -472,8 +475,9 @@ std::unique_ptr<LinOp> Csr<ValueType, IndexType>::inverse_column_permute(
                     this->get_strategy());
 
     exec->run(csr::make_inverse_column_permute(
-        inverse_permutation_indices->get_const_data(), this,
-        inverse_permute_cpy.get()));
+        make_temporary_clone(exec, inverse_permutation_indices)
+            ->get_const_data(),
+        this, inverse_permute_cpy.get()));
     inverse_permute_cpy->make_srow();
     inverse_permute_cpy->sort_by_column_index();
     return std::move(inverse_permute_cpy);
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index cb350c31628..a9e6a3a1515 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -75,7 +75,7 @@ GKO_REGISTER_OPERATION(calculate_nonzeros_per_row,
 GKO_REGISTER_OPERATION(calculate_total_cols, dense::calculate_total_cols);
 GKO_REGISTER_OPERATION(transpose, dense::transpose);
 GKO_REGISTER_OPERATION(conj_transpose, dense::conj_transpose);
-GKO_REGISTER_OPERATION(row_permute, dense::row_permute);
+GKO_REGISTER_OPERATION(row_gather, dense::row_gather);
 GKO_REGISTER_OPERATION(column_permute, dense::column_permute);
 GKO_REGISTER_OPERATION(inverse_row_permute, dense::inverse_row_permute);
 GKO_REGISTER_OPERATION(inverse_column_permute, dense::inverse_column_permute);
@@ -646,38 +646,100 @@ std::unique_ptr<LinOp> Dense<ValueType>::row_permute(
     auto exec = this->get_executor();
     auto permute_cpy = Dense::create(exec, this->get_size());
 
-    exec->run(
-        dense::make_row_permute(permutation_indices, this, permute_cpy.get()));
+    exec->run(dense::make_row_gather(
+        make_temporary_clone(exec, permutation_indices).get(), this,
+        permute_cpy.get()));
 
     return std::move(permute_cpy);
 }
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Dense<ValueType>::column_permute(
-    const Array<int32> *permutation_indices) const
+std::unique_ptr<LinOp> Dense<ValueType>::row_permute(
+    const Array<int64> *permutation_indices) const
 {
-    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
     auto exec = this->get_executor();
     auto permute_cpy = Dense::create(exec, this->get_size());
 
-    exec->run(dense::make_column_permute(permutation_indices, this,
-                                         permute_cpy.get()));
+    exec->run(dense::make_row_gather(
+        make_temporary_clone(exec, permutation_indices).get(), this,
+        permute_cpy.get()));
 
     return std::move(permute_cpy);
 }
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Dense<ValueType>::row_permute(
-    const Array<int64> *permutation_indices) const
+std::unique_ptr<Dense<ValueType>> Dense<ValueType>::row_gather(
+    const Array<int32> *row_indices) const
 {
-    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
     auto exec = this->get_executor();
-    auto permute_cpy = Dense::create(exec, this->get_size());
+    dim<2> out_dim{row_indices->get_num_elems(), this->get_size()[1]};
+    auto row_gathered = Dense::create(exec, out_dim);
+
+    exec->run(
+        dense::make_row_gather(make_temporary_clone(exec, row_indices).get(),
+                               this, row_gathered.get()));
+    return row_gathered;
+}
+
+
+template <typename ValueType>
+std::unique_ptr<Dense<ValueType>> Dense<ValueType>::row_gather(
+    const Array<int64> *row_indices) const
+{
+    auto exec = this->get_executor();
+    dim<2> out_dim{row_indices->get_num_elems(), this->get_size()[1]};
+    auto row_gathered = Dense::create(exec, out_dim);
 
     exec->run(
-        dense::make_row_permute(permutation_indices, this, permute_cpy.get()));
+        dense::make_row_gather(make_temporary_clone(exec, row_indices).get(),
+                               this, row_gathered.get()));
+    return row_gathered;
+}
+
+
+template <typename ValueType>
+void Dense<ValueType>::row_gather(const Array<int32> *row_indices,
+                                  Dense<ValueType> *row_gathered) const
+{
+    auto exec = this->get_executor();
+    dim<2> expected_dim{row_indices->get_num_elems(), this->get_size()[1]};
+    GKO_ASSERT_EQUAL_DIMENSIONS(expected_dim, row_gathered);
+
+    exec->run(dense::make_row_gather(
+        make_temporary_clone(exec, row_indices).get(), this,
+        make_temporary_clone(exec, row_gathered).get()));
+}
+
+
+template <typename ValueType>
+void Dense<ValueType>::row_gather(const Array<int64> *row_indices,
+                                  Dense<ValueType> *row_gathered) const
+{
+    dim<2> expected_dim{row_indices->get_num_elems(), this->get_size()[1]};
+    GKO_ASSERT_EQUAL_DIMENSIONS(expected_dim, row_gathered);
+
+    auto exec = this->get_executor();
+
+    this->get_executor()->run(dense::make_row_gather(
+        make_temporary_clone(exec, row_indices).get(), this,
+        make_temporary_clone(exec, row_gathered).get()));
+}
+
+
+template <typename ValueType>
+std::unique_ptr<LinOp> Dense<ValueType>::column_permute(
+    const Array<int32> *permutation_indices) const
+{
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
+    auto exec = this->get_executor();
+    auto permute_cpy = Dense::create(exec, this->get_size());
+
+    exec->run(dense::make_column_permute(
+        make_temporary_clone(exec, permutation_indices).get(), this,
+        permute_cpy.get()));
 
     return std::move(permute_cpy);
 }
@@ -691,8 +753,9 @@ std::unique_ptr<LinOp> Dense<ValueType>::column_permute(
     auto exec = this->get_executor();
     auto permute_cpy = Dense::create(exec, this->get_size());
 
-    exec->run(dense::make_column_permute(permutation_indices, this,
-                                         permute_cpy.get()));
+    exec->run(dense::make_column_permute(
+        make_temporary_clone(exec, permutation_indices).get(), this,
+        permute_cpy.get()));
 
     return std::move(permute_cpy);
 }
@@ -707,40 +770,43 @@ std::unique_ptr<LinOp> Dense<ValueType>::inverse_row_permute(
     auto exec = this->get_executor();
     auto inverse_permute_cpy = Dense::create(exec, this->get_size());
 
-    exec->run(dense::make_inverse_row_permute(inverse_permutation_indices, this,
-                                              inverse_permute_cpy.get()));
+    exec->run(dense::make_inverse_row_permute(
+        make_temporary_clone(exec, inverse_permutation_indices).get(), this,
+        inverse_permute_cpy.get()));
 
     return std::move(inverse_permute_cpy);
 }
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Dense<ValueType>::inverse_column_permute(
-    const Array<int32> *inverse_permutation_indices) const
+std::unique_ptr<LinOp> Dense<ValueType>::inverse_row_permute(
+    const Array<int64> *inverse_permutation_indices) const
 {
     GKO_ASSERT_EQ(inverse_permutation_indices->get_num_elems(),
-                  this->get_size()[1]);
+                  this->get_size()[0]);
     auto exec = this->get_executor();
     auto inverse_permute_cpy = Dense::create(exec, this->get_size());
 
-    exec->run(dense::make_inverse_column_permute(
-        inverse_permutation_indices, this, inverse_permute_cpy.get()));
+    exec->run(dense::make_inverse_row_permute(
+        make_temporary_clone(exec, inverse_permutation_indices).get(), this,
+        inverse_permute_cpy.get()));
 
     return std::move(inverse_permute_cpy);
 }
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Dense<ValueType>::inverse_row_permute(
-    const Array<int64> *inverse_permutation_indices) const
+std::unique_ptr<LinOp> Dense<ValueType>::inverse_column_permute(
+    const Array<int32> *inverse_permutation_indices) const
 {
     GKO_ASSERT_EQ(inverse_permutation_indices->get_num_elems(),
-                  this->get_size()[0]);
+                  this->get_size()[1]);
     auto exec = this->get_executor();
     auto inverse_permute_cpy = Dense::create(exec, this->get_size());
 
-    exec->run(dense::make_inverse_row_permute(inverse_permutation_indices, this,
-                                              inverse_permute_cpy.get()));
+    exec->run(dense::make_inverse_column_permute(
+        make_temporary_clone(exec, inverse_permutation_indices).get(), this,
+        inverse_permute_cpy.get()));
 
     return std::move(inverse_permute_cpy);
 }
@@ -756,7 +822,8 @@ std::unique_ptr<LinOp> Dense<ValueType>::inverse_column_permute(
     auto inverse_permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_inverse_column_permute(
-        inverse_permutation_indices, this, inverse_permute_cpy.get()));
+        make_temporary_clone(exec, inverse_permutation_indices).get(), this,
+        inverse_permute_cpy.get()));
 
     return std::move(inverse_permute_cpy);
 }
diff --git a/core/matrix/dense_kernels.hpp b/core/matrix/dense_kernels.hpp
index e99d455aa77..279c7001cc3 100644
--- a/core/matrix/dense_kernels.hpp
+++ b/core/matrix/dense_kernels.hpp
@@ -143,11 +143,11 @@ namespace kernels {
                         const matrix::Dense<_type> *orig,            \
                         matrix::Dense<_type> *trans)
 
-#define GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL(_vtype, _itype)      \
-    void row_permute(std::shared_ptr<const DefaultExecutor> exec, \
-                     const Array<_itype> *permutation_indices,    \
-                     const matrix::Dense<_vtype> *orig,           \
-                     matrix::Dense<_vtype> *row_permuted)
+#define GKO_DECLARE_DENSE_ROW_GATHER_KERNEL(_vtype, _itype)      \
+    void row_gather(std::shared_ptr<const DefaultExecutor> exec, \
+                    const Array<_itype> *gather_indices,         \
+                    const matrix::Dense<_vtype> *orig,           \
+                    matrix::Dense<_vtype> *row_gathered)
 
 #define GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL(_vtype, _itype)      \
     void column_permute(std::shared_ptr<const DefaultExecutor> exec, \
@@ -238,7 +238,7 @@ namespace kernels {
     template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL(ValueType);                     \
     template <typename ValueType, typename IndexType>                       \
-    GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL(ValueType, IndexType);             \
+    GKO_DECLARE_DENSE_ROW_GATHER_KERNEL(ValueType, IndexType);              \
     template <typename ValueType, typename IndexType>                       \
     GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType);          \
     template <typename ValueType, typename IndexType>                       \
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index 83804b43083..41405402b7a 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -603,24 +603,25 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void row_permute(std::shared_ptr<const CudaExecutor> exec,
-                 const Array<IndexType> *permutation_indices,
-                 const matrix::Dense<ValueType> *orig,
-                 matrix::Dense<ValueType> *row_permuted)
+void row_gather(std::shared_ptr<const CudaExecutor> exec,
+                const Array<IndexType> *row_indices,
+                const matrix::Dense<ValueType> *orig,
+                matrix::Dense<ValueType> *row_gathered)
 {
     constexpr auto block_size = default_block_size;
+    auto out_num_rows = row_indices->get_num_elems();
     const dim3 grid_dim =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
+        ceildiv(out_num_rows * orig->get_size()[1], block_size);
     const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
-    kernel::row_permute<block_size><<<grid_dim, block_dim>>>(
-        orig->get_size()[0], orig->get_size()[1],
-        as_cuda_type(permutation_indices->get_const_data()),
+    kernel::row_gather<block_size><<<grid_dim, block_dim>>>(
+        out_num_rows, orig->get_size()[1],
+        as_cuda_type(row_indices->get_const_data()),
         as_cuda_type(orig->get_const_values()), orig->get_stride(),
-        as_cuda_type(row_permuted->get_values()), row_permuted->get_stride());
+        as_cuda_type(row_gathered->get_values()), row_gathered->get_stride());
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_ROW_GATHER_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index 29b7f04f292..5cf33644c38 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -125,9 +125,7 @@ class Csr : public ::testing::Test {
         std::iota(tmp2.begin(), tmp2.end(), 0);
         std::shuffle(tmp2.begin(), tmp2.end(), rng);
         rpermute_idxs = std::make_unique<Arr>(ref, tmp.begin(), tmp.end());
-        drpermute_idxs = std::make_unique<Arr>(cuda, tmp.begin(), tmp.end());
         cpermute_idxs = std::make_unique<Arr>(ref, tmp2.begin(), tmp2.end());
-        dcpermute_idxs = std::make_unique<Arr>(cuda, tmp2.begin(), tmp2.end());
     }
 
     void set_up_apply_complex_data(
@@ -168,9 +166,7 @@ class Csr : public ::testing::Test {
     std::unique_ptr<Vec> dalpha;
     std::unique_ptr<Vec> dbeta;
     std::unique_ptr<Arr> rpermute_idxs;
-    std::unique_ptr<Arr> drpermute_idxs;
     std::unique_ptr<Arr> cpermute_idxs;
-    std::unique_ptr<Arr> dcpermute_idxs;
 };
 
 
@@ -747,7 +743,7 @@ TEST_F(Csr, IsRowPermutable)
     set_up_apply_data(std::make_shared<Mtx::classical>());
 
     auto r_permute = gko::as<Mtx>(mtx->row_permute(rpermute_idxs.get()));
-    auto dr_permute = gko::as<Mtx>(dmtx->row_permute(drpermute_idxs.get()));
+    auto dr_permute = gko::as<Mtx>(dmtx->row_permute(rpermute_idxs.get()));
 
     GKO_ASSERT_MTX_EQ_SPARSITY(r_permute, dr_permute);
     GKO_ASSERT_MTX_NEAR(r_permute, dr_permute, 0);
@@ -759,7 +755,7 @@ TEST_F(Csr, IsColPermutable)
     set_up_apply_data(std::make_shared<Mtx::classical>());
 
     auto c_permute = gko::as<Mtx>(mtx->column_permute(cpermute_idxs.get()));
-    auto dc_permute = gko::as<Mtx>(dmtx->column_permute(dcpermute_idxs.get()));
+    auto dc_permute = gko::as<Mtx>(dmtx->column_permute(cpermute_idxs.get()));
 
     ASSERT_TRUE(dc_permute->is_sorted_by_column_index());
     GKO_ASSERT_MTX_EQ_SPARSITY(c_permute, dc_permute);
@@ -774,7 +770,7 @@ TEST_F(Csr, IsInverseRowPermutable)
     auto inverse_r_permute =
         gko::as<Mtx>(mtx->inverse_row_permute(rpermute_idxs.get()));
     auto d_inverse_r_permute =
-        gko::as<Mtx>(dmtx->inverse_row_permute(drpermute_idxs.get()));
+        gko::as<Mtx>(dmtx->inverse_row_permute(rpermute_idxs.get()));
 
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse_r_permute, d_inverse_r_permute);
     GKO_ASSERT_MTX_NEAR(inverse_r_permute, d_inverse_r_permute, 0);
@@ -788,7 +784,7 @@ TEST_F(Csr, IsInverseColPermutable)
     auto inverse_c_permute =
         gko::as<Mtx>(mtx->inverse_column_permute(cpermute_idxs.get()));
     auto d_inverse_c_permute =
-        gko::as<Mtx>(dmtx->inverse_column_permute(dcpermute_idxs.get()));
+        gko::as<Mtx>(dmtx->inverse_column_permute(cpermute_idxs.get()));
 
     ASSERT_TRUE(d_inverse_c_permute->is_sorted_by_column_index());
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse_c_permute, d_inverse_c_permute);
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index 823aabe9b2b..9e48a5ea807 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -138,14 +138,17 @@ class Dense : public ::testing::Test {
         std::vector<itype> tmp2(x->get_size()[1], 0);
         std::iota(tmp2.begin(), tmp2.end(), 0);
         std::shuffle(tmp2.begin(), tmp2.end(), rng);
+        std::vector<itype> tmp3(x->get_size()[0] / 10);
+        std::uniform_int_distribution<itype> row_dist(0, x->get_size()[0] - 1);
+        for (auto &i : tmp3) {
+            i = row_dist(rng);
+        }
         rpermute_idxs =
             std::unique_ptr<Arr>(new Arr{ref, tmp.begin(), tmp.end()});
-        drpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{cuda, tmp.begin(), tmp.end()});
         cpermute_idxs =
             std::unique_ptr<Arr>(new Arr{ref, tmp2.begin(), tmp2.end()});
-        dcpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{cuda, tmp2.begin(), tmp2.end()});
+        rgather_idxs =
+            std::unique_ptr<Arr>(new Arr{ref, tmp3.begin(), tmp3.end()});
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -166,9 +169,8 @@ class Dense : public ::testing::Test {
     std::unique_ptr<Mtx> dalpha;
     std::unique_ptr<Mtx> dbeta;
     std::unique_ptr<Arr> rpermute_idxs;
-    std::unique_ptr<Arr> drpermute_idxs;
     std::unique_ptr<Arr> cpermute_idxs;
-    std::unique_ptr<Arr> dcpermute_idxs;
+    std::unique_ptr<Arr> rgather_idxs;
 };
 
 
@@ -562,12 +564,39 @@ TEST_F(Dense, CalculateTotalColsIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, CanGatherRows)
+{
+    set_up_apply_data();
+
+    auto r_gather = x->row_gather(rgather_idxs.get());
+    auto dr_gather = dx->row_gather(rgather_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
+}
+
+
+TEST_F(Dense, CanGatherRowsIntoDense)
+{
+    set_up_apply_data();
+    auto gather_size =
+        gko::dim<2>{rgather_idxs->get_num_elems(), x->get_size()[1]};
+    auto r_gather = Mtx::create(ref, gather_size);
+    // test make_temporary_clone and non-default stride
+    auto dr_gather = Mtx::create(ref, gather_size, x->get_size()[1] + 2);
+
+    x->row_gather(rgather_idxs.get(), r_gather.get());
+    dx->row_gather(rgather_idxs.get(), dr_gather.get());
+
+    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
+}
+
+
 TEST_F(Dense, IsRowPermutable)
 {
     set_up_apply_data();
 
     auto r_permute = x->row_permute(rpermute_idxs.get());
-    auto dr_permute = dx->row_permute(drpermute_idxs.get());
+    auto dr_permute = dx->row_permute(rpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(r_permute.get()),
                         static_cast<Mtx *>(dr_permute.get()), 0);
@@ -579,7 +608,7 @@ TEST_F(Dense, IsColPermutable)
     set_up_apply_data();
 
     auto c_permute = x->column_permute(cpermute_idxs.get());
-    auto dc_permute = dx->column_permute(dcpermute_idxs.get());
+    auto dc_permute = dx->column_permute(cpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(c_permute.get()),
                         static_cast<Mtx *>(dc_permute.get()), 0);
@@ -591,7 +620,7 @@ TEST_F(Dense, IsInverseRowPermutable)
     set_up_apply_data();
 
     auto inverse_r_permute = x->inverse_row_permute(rpermute_idxs.get());
-    auto d_inverse_r_permute = dx->inverse_row_permute(drpermute_idxs.get());
+    auto d_inverse_r_permute = dx->inverse_row_permute(rpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_r_permute.get()),
                         static_cast<Mtx *>(d_inverse_r_permute.get()), 0);
@@ -603,7 +632,7 @@ TEST_F(Dense, IsInverseColPermutable)
     set_up_apply_data();
 
     auto inverse_c_permute = x->inverse_column_permute(cpermute_idxs.get());
-    auto d_inverse_c_permute = dx->inverse_column_permute(dcpermute_idxs.get());
+    auto d_inverse_c_permute = dx->inverse_column_permute(cpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_c_permute.get()),
                         static_cast<Mtx *>(d_inverse_c_permute.get()), 0);
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index fac85ba3a14..714d4f588b2 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -241,13 +241,13 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void row_permute(std::shared_ptr<const DpcppExecutor> exec,
-                 const Array<IndexType> *permutation_indices,
-                 const matrix::Dense<ValueType> *orig,
-                 matrix::Dense<ValueType> *row_permuted) GKO_NOT_IMPLEMENTED;
+void row_gather(std::shared_ptr<const DpcppExecutor> exec,
+                const Array<IndexType> *gather_indices,
+                const matrix::Dense<ValueType> *orig,
+                matrix::Dense<ValueType> *row_gathered) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_ROW_GATHER_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index e5de9f7df83..b62b3736e65 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -626,25 +626,26 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void row_permute(std::shared_ptr<const HipExecutor> exec,
-                 const Array<IndexType> *permutation_indices,
-                 const matrix::Dense<ValueType> *orig,
-                 matrix::Dense<ValueType> *row_permuted)
+void row_gather(std::shared_ptr<const HipExecutor> exec,
+                const Array<IndexType> *row_indices,
+                const matrix::Dense<ValueType> *orig,
+                matrix::Dense<ValueType> *row_gathered)
 {
     constexpr auto block_size = default_block_size;
+    auto out_num_rows = row_indices->get_num_elems();
     const dim3 grid_dim =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
+        ceildiv(out_num_rows * orig->get_size()[1], block_size);
     const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
     hipLaunchKernelGGL(
-        kernel::row_permute<block_size>, dim3(grid_dim), dim3(block_dim), 0, 0,
-        orig->get_size()[0], orig->get_size()[1],
-        as_hip_type(permutation_indices->get_const_data()),
+        kernel::row_gather<block_size>, dim3(grid_dim), dim3(block_dim), 0, 0,
+        out_num_rows, orig->get_size()[1],
+        as_hip_type(row_indices->get_const_data()),
         as_hip_type(orig->get_const_values()), orig->get_stride(),
-        as_hip_type(row_permuted->get_values()), row_permuted->get_stride());
+        as_hip_type(row_gathered->get_values()), row_gathered->get_stride());
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_ROW_GATHER_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index ea3b3a2395f..21a2e751714 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -125,9 +125,7 @@ class Csr : public ::testing::Test {
         std::iota(tmp2.begin(), tmp2.end(), 0);
         std::shuffle(tmp2.begin(), tmp2.end(), rng);
         rpermute_idxs = std::make_unique<Arr>(ref, tmp.begin(), tmp.end());
-        drpermute_idxs = std::make_unique<Arr>(hip, tmp.begin(), tmp.end());
         cpermute_idxs = std::make_unique<Arr>(ref, tmp2.begin(), tmp2.end());
-        dcpermute_idxs = std::make_unique<Arr>(hip, tmp2.begin(), tmp2.end());
     }
 
     void set_up_apply_complex_data(
@@ -168,9 +166,7 @@ class Csr : public ::testing::Test {
     std::unique_ptr<Vec> dalpha;
     std::unique_ptr<Vec> dbeta;
     std::unique_ptr<Arr> rpermute_idxs;
-    std::unique_ptr<Arr> drpermute_idxs;
     std::unique_ptr<Arr> cpermute_idxs;
-    std::unique_ptr<Arr> dcpermute_idxs;
 };
 
 
@@ -741,7 +737,7 @@ TEST_F(Csr, IsRowPermutable)
     set_up_apply_data(std::make_shared<Mtx::classical>());
 
     auto r_permute = gko::as<Mtx>(mtx->row_permute(rpermute_idxs.get()));
-    auto dr_permute = gko::as<Mtx>(dmtx->row_permute(drpermute_idxs.get()));
+    auto dr_permute = gko::as<Mtx>(dmtx->row_permute(rpermute_idxs.get()));
 
     GKO_ASSERT_MTX_EQ_SPARSITY(r_permute, dr_permute);
     GKO_ASSERT_MTX_NEAR(r_permute, dr_permute, 0);
@@ -753,7 +749,7 @@ TEST_F(Csr, IsColPermutable)
     set_up_apply_data(std::make_shared<Mtx::classical>());
 
     auto c_permute = gko::as<Mtx>(mtx->column_permute(cpermute_idxs.get()));
-    auto dc_permute = gko::as<Mtx>(dmtx->column_permute(dcpermute_idxs.get()));
+    auto dc_permute = gko::as<Mtx>(dmtx->column_permute(cpermute_idxs.get()));
 
     ASSERT_TRUE(dc_permute->is_sorted_by_column_index());
     GKO_ASSERT_MTX_EQ_SPARSITY(c_permute, dc_permute);
@@ -768,7 +764,7 @@ TEST_F(Csr, IsInverseRowPermutable)
     auto inverse_r_permute =
         gko::as<Mtx>(mtx->inverse_row_permute(rpermute_idxs.get()));
     auto d_inverse_r_permute =
-        gko::as<Mtx>(dmtx->inverse_row_permute(drpermute_idxs.get()));
+        gko::as<Mtx>(dmtx->inverse_row_permute(rpermute_idxs.get()));
 
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse_r_permute, d_inverse_r_permute);
     GKO_ASSERT_MTX_NEAR(inverse_r_permute, d_inverse_r_permute, 0);
@@ -782,7 +778,7 @@ TEST_F(Csr, IsInverseColPermutable)
     auto inverse_c_permute =
         gko::as<Mtx>(mtx->inverse_column_permute(cpermute_idxs.get()));
     auto d_inverse_c_permute =
-        gko::as<Mtx>(dmtx->inverse_column_permute(dcpermute_idxs.get()));
+        gko::as<Mtx>(dmtx->inverse_column_permute(cpermute_idxs.get()));
 
     ASSERT_TRUE(d_inverse_c_permute->is_sorted_by_column_index());
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse_c_permute, d_inverse_c_permute);
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index a4f4b4e58f1..210f4a879e3 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -135,14 +135,17 @@ class Dense : public ::testing::Test {
         std::vector<itype> tmp2(x->get_size()[1], 0);
         std::iota(tmp2.begin(), tmp2.end(), 0);
         std::shuffle(tmp2.begin(), tmp2.end(), rng);
+        std::vector<itype> tmp3(x->get_size()[0] / 10);
+        std::uniform_int_distribution<itype> row_dist(0, x->get_size()[0] - 1);
+        for (auto &i : tmp3) {
+            i = row_dist(rng);
+        }
         rpermute_idxs =
             std::unique_ptr<Arr>(new Arr{ref, tmp.begin(), tmp.end()});
-        drpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{hip, tmp.begin(), tmp.end()});
         cpermute_idxs =
             std::unique_ptr<Arr>(new Arr{ref, tmp2.begin(), tmp2.end()});
-        dcpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{hip, tmp2.begin(), tmp2.end()});
+        rgather_idxs =
+            std::unique_ptr<Arr>(new Arr{ref, tmp3.begin(), tmp3.end()});
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -161,9 +164,8 @@ class Dense : public ::testing::Test {
     std::unique_ptr<Mtx> dalpha;
     std::unique_ptr<Mtx> dbeta;
     std::unique_ptr<Arr> rpermute_idxs;
-    std::unique_ptr<Arr> drpermute_idxs;
     std::unique_ptr<Arr> cpermute_idxs;
-    std::unique_ptr<Arr> dcpermute_idxs;
+    std::unique_ptr<Arr> rgather_idxs;
 };
 
 
@@ -545,12 +547,39 @@ TEST_F(Dense, CalculateTotalColsIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, CanGatherRows)
+{
+    set_up_apply_data();
+
+    auto r_gather = x->row_gather(rgather_idxs.get());
+    auto dr_gather = dx->row_gather(rgather_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
+}
+
+
+TEST_F(Dense, CanGatherRowsIntoDense)
+{
+    set_up_apply_data();
+    auto gather_size =
+        gko::dim<2>{rgather_idxs->get_num_elems(), x->get_size()[1]};
+    auto r_gather = Mtx::create(ref, gather_size);
+    // test make_temporary_clone and non-default stride
+    auto dr_gather = Mtx::create(ref, gather_size, x->get_size()[1] + 2);
+
+    x->row_gather(rgather_idxs.get(), r_gather.get());
+    dx->row_gather(rgather_idxs.get(), dr_gather.get());
+
+    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
+}
+
+
 TEST_F(Dense, IsRowPermutable)
 {
     set_up_apply_data();
 
     auto r_permute = x->row_permute(rpermute_idxs.get());
-    auto dr_permute = dx->row_permute(drpermute_idxs.get());
+    auto dr_permute = dx->row_permute(rpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(r_permute.get()),
                         static_cast<Mtx *>(dr_permute.get()), 0);
@@ -562,7 +591,7 @@ TEST_F(Dense, IsColPermutable)
     set_up_apply_data();
 
     auto c_permute = x->column_permute(cpermute_idxs.get());
-    auto dc_permute = dx->column_permute(dcpermute_idxs.get());
+    auto dc_permute = dx->column_permute(cpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(c_permute.get()),
                         static_cast<Mtx *>(dc_permute.get()), 0);
@@ -574,7 +603,7 @@ TEST_F(Dense, IsInverseRowPermutable)
     set_up_apply_data();
 
     auto inverse_r_permute = x->inverse_row_permute(rpermute_idxs.get());
-    auto d_inverse_r_permute = dx->inverse_row_permute(drpermute_idxs.get());
+    auto d_inverse_r_permute = dx->inverse_row_permute(rpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_r_permute.get()),
                         static_cast<Mtx *>(d_inverse_r_permute.get()), 0);
@@ -586,7 +615,7 @@ TEST_F(Dense, IsInverseColPermutable)
     set_up_apply_data();
 
     auto inverse_c_permute = x->inverse_column_permute(cpermute_idxs.get());
-    auto d_inverse_c_permute = dx->inverse_column_permute(dcpermute_idxs.get());
+    auto d_inverse_c_permute = dx->inverse_column_permute(cpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_c_permute.get()),
                         static_cast<Mtx *>(d_inverse_c_permute.get()), 0);
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 7bb439afc34..305c9bb666a 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -232,6 +232,16 @@ class Dense
     std::unique_ptr<LinOp> row_permute(
         const Array<int64> *permutation_indices) const override;
 
+    std::unique_ptr<Dense> row_gather(const Array<int32> *gather_indices) const;
+
+    std::unique_ptr<Dense> row_gather(const Array<int64> *gather_indices) const;
+
+    void row_gather(const Array<int32> *gather_indices,
+                    Dense *row_gathered) const;
+
+    void row_gather(const Array<int64> *gather_indices,
+                    Dense *row_gathered) const;
+
     std::unique_ptr<LinOp> column_permute(
         const Array<int32> *permutation_indices) const override;
 
diff --git a/omp/matrix/dense_kernels.cpp b/omp/matrix/dense_kernels.cpp
index 15a9d6e9a99..7f49b3d6c8b 100644
--- a/omp/matrix/dense_kernels.cpp
+++ b/omp/matrix/dense_kernels.cpp
@@ -680,22 +680,22 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void row_permute(std::shared_ptr<const OmpExecutor> exec,
-                 const Array<IndexType> *permutation_indices,
-                 const matrix::Dense<ValueType> *orig,
-                 matrix::Dense<ValueType> *row_permuted)
+void row_gather(std::shared_ptr<const OmpExecutor> exec,
+                const Array<IndexType> *row_indices,
+                const matrix::Dense<ValueType> *orig,
+                matrix::Dense<ValueType> *row_gathered)
 {
-    auto perm = permutation_indices->get_const_data();
+    auto rows = row_indices->get_const_data();
 #pragma omp parallel for
-    for (size_type i = 0; i < orig->get_size()[0]; ++i) {
+    for (size_type i = 0; i < row_indices->get_num_elems(); ++i) {
         for (size_type j = 0; j < orig->get_size()[1]; ++j) {
-            row_permuted->at(i, j) = orig->at(perm[i], j);
+            row_gathered->at(i, j) = orig->at(rows[i], j);
         }
     }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_ROW_GATHER_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index 26af3397262..439d8271a09 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -128,9 +128,7 @@ class Csr : public ::testing::Test {
         std::iota(tmp2.begin(), tmp2.end(), 0);
         std::shuffle(tmp2.begin(), tmp2.end(), rng);
         rpermute_idxs = std::make_unique<Arr>(ref, tmp.begin(), tmp.end());
-        drpermute_idxs = std::make_unique<Arr>(omp, tmp.begin(), tmp.end());
         cpermute_idxs = std::make_unique<Arr>(ref, tmp2.begin(), tmp2.end());
-        dcpermute_idxs = std::make_unique<Arr>(omp, tmp2.begin(), tmp2.end());
     }
 
     struct matrix_pair {
@@ -173,9 +171,7 @@ class Csr : public ::testing::Test {
     std::unique_ptr<Vec> dalpha;
     std::unique_ptr<Vec> dbeta;
     std::unique_ptr<Arr> rpermute_idxs;
-    std::unique_ptr<Arr> drpermute_idxs;
     std::unique_ptr<Arr> cpermute_idxs;
-    std::unique_ptr<Arr> dcpermute_idxs;
 };
 
 
@@ -474,7 +470,7 @@ TEST_F(Csr, IsRowPermutable)
     set_up_apply_data();
 
     auto r_permute = gko::as<Mtx>(mtx->row_permute(rpermute_idxs.get()));
-    auto dr_permute = gko::as<Mtx>(dmtx->row_permute(drpermute_idxs.get()));
+    auto dr_permute = gko::as<Mtx>(dmtx->row_permute(rpermute_idxs.get()));
 
     GKO_ASSERT_MTX_EQ_SPARSITY(r_permute, dr_permute);
     GKO_ASSERT_MTX_NEAR(r_permute, dr_permute, 0);
@@ -486,7 +482,7 @@ TEST_F(Csr, IsColPermutable)
     set_up_apply_data();
 
     auto c_permute = gko::as<Mtx>(mtx->column_permute(cpermute_idxs.get()));
-    auto dc_permute = gko::as<Mtx>(dmtx->column_permute(dcpermute_idxs.get()));
+    auto dc_permute = gko::as<Mtx>(dmtx->column_permute(cpermute_idxs.get()));
 
     ASSERT_TRUE(dc_permute->is_sorted_by_column_index());
     GKO_ASSERT_MTX_EQ_SPARSITY(c_permute, dc_permute);
@@ -501,7 +497,7 @@ TEST_F(Csr, IsInverseRowPermutable)
     auto inverse_r_permute =
         gko::as<Mtx>(mtx->inverse_row_permute(rpermute_idxs.get()));
     auto d_inverse_r_permute =
-        gko::as<Mtx>(dmtx->inverse_row_permute(drpermute_idxs.get()));
+        gko::as<Mtx>(dmtx->inverse_row_permute(rpermute_idxs.get()));
 
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse_r_permute, d_inverse_r_permute);
     GKO_ASSERT_MTX_NEAR(inverse_r_permute, d_inverse_r_permute, 0);
@@ -515,7 +511,7 @@ TEST_F(Csr, IsInverseColPermutable)
     auto inverse_c_permute =
         gko::as<Mtx>(mtx->inverse_column_permute(cpermute_idxs.get()));
     auto d_inverse_c_permute =
-        gko::as<Mtx>(dmtx->inverse_column_permute(dcpermute_idxs.get()));
+        gko::as<Mtx>(dmtx->inverse_column_permute(cpermute_idxs.get()));
 
     ASSERT_TRUE(d_inverse_c_permute->is_sorted_by_column_index());
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse_c_permute, d_inverse_c_permute);
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index cf2d975c2ef..2ee7ea96075 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -150,14 +150,17 @@ class Dense : public ::testing::Test {
         std::vector<int> tmp2(x->get_size()[1], 0);
         std::iota(tmp2.begin(), tmp2.end(), 0);
         std::shuffle(tmp2.begin(), tmp2.end(), rng);
+        std::vector<int> tmp3(x->get_size()[0] / 10);
+        std::uniform_int_distribution<int> row_dist(0, x->get_size()[0] - 1);
+        for (auto &i : tmp3) {
+            i = row_dist(rng);
+        }
         rpermute_idxs =
             std::unique_ptr<Arr>(new Arr{ref, tmp.begin(), tmp.end()});
-        drpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{omp, tmp.begin(), tmp.end()});
         cpermute_idxs =
             std::unique_ptr<Arr>(new Arr{ref, tmp2.begin(), tmp2.end()});
-        dcpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{omp, tmp2.begin(), tmp2.end()});
+        rgather_idxs =
+            std::unique_ptr<Arr>(new Arr{ref, tmp3.begin(), tmp3.end()});
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -178,9 +181,8 @@ class Dense : public ::testing::Test {
     std::unique_ptr<Mtx> dalpha;
     std::unique_ptr<Mtx> dbeta;
     std::unique_ptr<Arr> rpermute_idxs;
-    std::unique_ptr<Arr> drpermute_idxs;
     std::unique_ptr<Arr> cpermute_idxs;
-    std::unique_ptr<Arr> dcpermute_idxs;
+    std::unique_ptr<Arr> rgather_idxs;
 };
 
 
@@ -681,11 +683,39 @@ TEST_F(Dense, IsConjugateTransposable)
 }
 
 
+TEST_F(Dense, CanGatherRows)
+{
+    set_up_apply_data();
+
+    auto r_gather = x->row_gather(rgather_idxs.get());
+    auto dr_gather = dx->row_gather(rgather_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
+}
+
+
+TEST_F(Dense, CanGatherRowsIntoDense)
+{
+    set_up_apply_data();
+    auto gather_size =
+        gko::dim<2>{rgather_idxs->get_num_elems(), x->get_size()[1]};
+    auto r_gather = Mtx::create(ref, gather_size);
+    // test make_temporary_clone and non-default stride
+    auto dr_gather = Mtx::create(ref, gather_size, x->get_size()[1] + 2);
+
+    x->row_gather(rgather_idxs.get(), r_gather.get());
+    dx->row_gather(rgather_idxs.get(), dr_gather.get());
+
+    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
+}
+
+
 TEST_F(Dense, IsRowPermutable)
 {
     set_up_apply_data();
+
     auto r_permute = x->row_permute(rpermute_idxs.get());
-    auto dr_permute = dx->row_permute(drpermute_idxs.get());
+    auto dr_permute = dx->row_permute(rpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(r_permute.get()),
                         static_cast<Mtx *>(dr_permute.get()), 0);
@@ -695,8 +725,9 @@ TEST_F(Dense, IsRowPermutable)
 TEST_F(Dense, IsColPermutable)
 {
     set_up_apply_data();
+
     auto c_permute = x->column_permute(cpermute_idxs.get());
-    auto dc_permute = dx->column_permute(dcpermute_idxs.get());
+    auto dc_permute = dx->column_permute(cpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(c_permute.get()),
                         static_cast<Mtx *>(dc_permute.get()), 0);
@@ -706,8 +737,9 @@ TEST_F(Dense, IsColPermutable)
 TEST_F(Dense, IsInverseRowPermutable)
 {
     set_up_apply_data();
+
     auto inverse_r_permute = x->inverse_row_permute(rpermute_idxs.get());
-    auto d_inverse_r_permute = dx->inverse_row_permute(drpermute_idxs.get());
+    auto d_inverse_r_permute = dx->inverse_row_permute(rpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_r_permute.get()),
                         static_cast<Mtx *>(d_inverse_r_permute.get()), 0);
@@ -717,8 +749,9 @@ TEST_F(Dense, IsInverseRowPermutable)
 TEST_F(Dense, IsInverseColPermutable)
 {
     set_up_apply_data();
+
     auto inverse_c_permute = x->inverse_column_permute(cpermute_idxs.get());
-    auto d_inverse_c_permute = dx->inverse_column_permute(dcpermute_idxs.get());
+    auto d_inverse_c_permute = dx->inverse_column_permute(cpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_c_permute.get()),
                         static_cast<Mtx *>(d_inverse_c_permute.get()), 0);
diff --git a/reference/matrix/dense_kernels.cpp b/reference/matrix/dense_kernels.cpp
index 147204157e7..4cb84202605 100644
--- a/reference/matrix/dense_kernels.cpp
+++ b/reference/matrix/dense_kernels.cpp
@@ -600,21 +600,21 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void row_permute(std::shared_ptr<const ReferenceExecutor> exec,
-                 const Array<IndexType> *permutation_indices,
-                 const matrix::Dense<ValueType> *orig,
-                 matrix::Dense<ValueType> *row_permuted)
+void row_gather(std::shared_ptr<const ReferenceExecutor> exec,
+                const Array<IndexType> *row_indices,
+                const matrix::Dense<ValueType> *orig,
+                matrix::Dense<ValueType> *row_gathered)
 {
-    auto perm = permutation_indices->get_const_data();
-    for (size_type i = 0; i < orig->get_size()[0]; ++i) {
+    auto rows = row_indices->get_const_data();
+    for (size_type i = 0; i < row_indices->get_num_elems(); ++i) {
         for (size_type j = 0; j < orig->get_size()[1]; ++j) {
-            row_permuted->at(i, j) = orig->at(perm[i], j);
+            row_gathered->at(i, j) = orig->at(rows[i], j);
         }
     }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_ROW_PERMUTE_KERNEL);
+    GKO_DECLARE_DENSE_ROW_GATHER_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index d6618866de0..3a977bebb59 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -1718,6 +1718,49 @@ TYPED_TEST(Dense, NonSquareMatrixIsTransposable)
 }
 
 
+TYPED_TEST(Dense, SquareMatrixCanGatherRows)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
+
+    auto row_gathered = this->mtx5->row_gather(&permute_idxs);
+
+    // clang-format off
+    GKO_ASSERT_MTX_NEAR(row_gathered,
+                        l({{-2.0, 2.0, 4.5},
+                           {1.0, -1.0, -0.5}}), r<TypeParam>::value);
+    // clang-format on
+}
+
+
+TYPED_TEST(Dense, SquareMatrixCanGatherRowsIntoDense)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
+    auto row_gathered = Mtx::create(exec, gko::dim<2>{2, 3});
+
+    this->mtx5->row_gather(&permute_idxs, row_gathered.get());
+
+    // clang-format off
+    GKO_ASSERT_MTX_NEAR(row_gathered,
+                        l({{-2.0, 2.0, 4.5},
+                           {1.0, -1.0, -0.5}}), r<TypeParam>::value);
+    // clang-format on
+}
+
+
 TYPED_TEST(Dense, SquareMatrixIsRowPermutable)
 {
     // clang-format off

From 5ab0728eaab2fea69b574c957cd71bb4a933add5 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 13 Nov 2020 20:26:42 +0100
Subject: [PATCH 0124/1008] add missing license to preconditioners.hpp

---
 benchmark/utils/preconditioners.hpp | 32 +++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/benchmark/utils/preconditioners.hpp b/benchmark/utils/preconditioners.hpp
index f55b9b0e13a..0542a72da4b 100644
--- a/benchmark/utils/preconditioners.hpp
+++ b/benchmark/utils/preconditioners.hpp
@@ -1,3 +1,35 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
 
 #ifndef GKO_BENCHMARK_UTILS_PRECONDITIONERS_HPP_
 #define GKO_BENCHMARK_UTILS_PRECONDITIONERS_HPP_

From d06a47190ce9bd2e05048e60ce531a1ac4a2d317 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 13 Nov 2020 20:57:14 +0100
Subject: [PATCH 0125/1008] fix initializer list copy warnings

---
 core/test/utils/assertions.hpp | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/core/test/utils/assertions.hpp b/core/test/utils/assertions.hpp
index 0515b32879c..8eb90784d45 100644
--- a/core/test/utils/assertions.hpp
+++ b/core/test/utils/assertions.hpp
@@ -680,14 +680,14 @@ namespace detail {
 
 
 template <typename T>
-std::initializer_list<std::initializer_list<T>> l(
-    std::initializer_list<std::initializer_list<T>> list)
+const std::initializer_list<std::initializer_list<T>> &l(
+    const std::initializer_list<std::initializer_list<T>> &list)
 {
     return list;
 }
 
 template <typename T>
-std::initializer_list<T> l(std::initializer_list<T> list)
+const std::initializer_list<T> &l(const std::initializer_list<T> &list)
 {
     return list;
 }
@@ -711,7 +711,20 @@ T *plain_ptr(const std::unique_ptr<T> &ptr)
 }
 
 template <typename T>
-T plain_ptr(T ptr)
+const std::initializer_list<T> &plain_ptr(const std::initializer_list<T> &ptr)
+{
+    return ptr;
+}
+
+template <typename T>
+const std::initializer_list<std::initializer_list<T>> &plain_ptr(
+    const std::initializer_list<std::initializer_list<T>> &ptr)
+{
+    return ptr;
+}
+
+template <typename T>
+T *plain_ptr(T *ptr)
 {
     return ptr;
 }

From 8fe806a2bffe64ba2b011ccd9a0cbf1f1a8cd2b8 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 17 Nov 2020 12:37:39 +0100
Subject: [PATCH 0126/1008] improve documentation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 core/test/base/array.cpp             |  2 ++
 include/ginkgo/core/base/array.hpp   |  4 ++--
 include/ginkgo/core/matrix/dense.hpp | 33 +++++++++++++++++++++++++---
 3 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/core/test/base/array.cpp b/core/test/base/array.cpp
index da62a133162..959392e5c71 100644
--- a/core/test/base/array.cpp
+++ b/core/test/base/array.cpp
@@ -288,6 +288,8 @@ TYPED_TEST(Array, CanCopyBackTemporaryCloneOnDifferentExecutor)
 
     {
         auto tmp_clone = make_temporary_clone(omp, &this->x);
+        // change x, so it no longer matches the original x
+        // the copy-back will overwrite it again with the correct value
         this->x.get_data()[0] = 0;
     }
 
diff --git a/include/ginkgo/core/base/array.hpp b/include/ginkgo/core/base/array.hpp
index 22f063433fe..59960520edf 100644
--- a/include/ginkgo/core/base/array.hpp
+++ b/include/ginkgo/core/base/array.hpp
@@ -600,9 +600,9 @@ class copy_back_deleter<Array<T>> {
     copy_back_deleter(pointer original) : original_{original} {}
 
     /**
-     * Deletes the object.
+     * Copies back the pointed-to object to the original and deletes it.
      *
-     * @param ptr  pointer to the object being deleted
+     * @param ptr  pointer to the object to be copied back and deleted
      */
     void operator()(pointer ptr) const
     {
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 305c9bb666a..c7ff335b89e 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -232,13 +232,40 @@ class Dense
     std::unique_ptr<LinOp> row_permute(
         const Array<int64> *permutation_indices) const override;
 
+    /**
+     * Create a Dense matrix consisting of the given rows from this matrix.
+     *
+     * @param gather_indices  pointer to an array containing row indices
+     *                        from this matrix. It may contain duplicates.
+     * @return  Dense matrix on the same executor with the same number of
+     *          columns and `gather_indices->get_num_elems()` rows containing
+     *          the gathered rows from this matrix:
+     *          `output(i,j) = input(gather_indices(i), j)`
+     */
     std::unique_ptr<Dense> row_gather(const Array<int32> *gather_indices) const;
 
+    /**
+     * @copydoc row_gather(const Array<int32>*) const
+     */
     std::unique_ptr<Dense> row_gather(const Array<int64> *gather_indices) const;
 
+    /**
+     * Copies the given rows from this matrix into `row_gathered`
+     *
+     * @param gather_indices  pointer to an array containing row indices
+     *                        from this matrix. It may contain duplicates.
+     * @param row_gathered  pointer to a Dense matrix that will store the
+     *                      gathered rows:
+     *                      `output(i,j) = input(gather_indices(i), j)`
+     *                      It must have the same number of columns as this
+     *                      matrix and `gather_indices->get_num_elems()` rows.
+     */
     void row_gather(const Array<int32> *gather_indices,
                     Dense *row_gathered) const;
 
+    /**
+     * @copydoc row_gather(const Array<int32>*, Dense*) const
+     */
     void row_gather(const Array<int64> *gather_indices,
                     Dense *row_gathered) const;
 
@@ -478,7 +505,7 @@ class Dense
             stride);
     }
 
-    /*
+    /**
      * Create a submatrix from the original matrix.
      *
      * @param rows     row span
@@ -490,7 +517,7 @@ class Dense
         return create_submatrix(rows, columns, this->get_stride());
     }
 
-    /*
+    /**
      * Create a real view of the (potentially) complex original matrix.
      * If the original matrix is real, nothing changes. If the original matrix
      * is complex, the result is created by viewing the complex matrix with as
@@ -515,7 +542,7 @@ class Dense
             stride);
     }
 
-    /*
+    /**
      * @copydoc create_real_view()
      */
     std::unique_ptr<const Dense<remove_complex<ValueType>>> create_real_view()

From b663f48d167d132679a0b523b2bd29078d8e570d Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 18 Nov 2020 13:20:49 +0100
Subject: [PATCH 0127/1008] fix windows CI jobs

Replace set-env by per-step env variables
---
 .github/workflows/windows-build.yml | 67 ++++++-----------------------
 1 file changed, 12 insertions(+), 55 deletions(-)

diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml
index 126a029e942..89d15aefee5 100644
--- a/.github/workflows/windows-build.yml
+++ b/.github/workflows/windows-build.yml
@@ -22,18 +22,15 @@ jobs:
       if: matrix.config.version == 'latest'
       run: |
         choco install cuda -y
-    - name: bigobj_env
-      run: |
-        echo "::set-env name=CXXFLAGS::/bigobj"
     - name: configure
       run: |
         $env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.."
         Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
         refreshenv
+        $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
         mkdir build
         cd build
-        $env:PATH="$pwd\windows_shared_library;$env:PATH"
-        cmake -DGINKGO_BUILD_CUDA=ON -DGINKGO_BUILD_OMP=OFF ..
+        cmake -DCMAKE_CXX_FLAGS=/bigobj -DGINKGO_BUILD_CUDA=ON -DGINKGO_BUILD_OMP=OFF ..
         cmake --build . -j4 --config Release
 
   windows_ref:
@@ -49,27 +46,17 @@ jobs:
     runs-on: [windows-latest]
     steps:
     - uses: actions/checkout@v2
-    - name: shared_env
-      if: matrix.config.shared == 'ON'
-      run: |
-        echo "::set-env name=origin_path::$env:PATH"
-        echo "::add-path::$pwd\build\windows_shared_library"
-    - name: bigobj_env
-      run: |
-        echo "::set-env name=CXXFLAGS::/bigobj"
     - name: configure
       run: |
+        $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
         mkdir build
         cd build
-        cmake -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DGINKGO_BUILD_CUDA=OFF -DGINKGO_BUILD_OMP=OFF ..
+        cmake -DCMAKE_CXX_FLAGS=/bigobj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DGINKGO_BUILD_CUDA=OFF -DGINKGO_BUILD_OMP=OFF ..
         cmake --build . -j4 --config ${{ matrix.config.build_type }}
         ctest . -C ${{ matrix.config.build_type }} --output-on-failure
-    - name: install_shared_env
-      if: matrix.config.shared == 'ON'
-      run: |
-        echo "::set-env name=PATH::C:\Program Files (x86)\Ginkgo\lib;$env:origin_path"
     - name: install
       run: |
+        $env:PATH="$env:PATH;C:\Program Files (x86)\Ginkgo\lib"
         cd build
         cmake --install . --config ${{ matrix.config.build_type }}
         cmake --build . --target test_install --config ${{ matrix.config.build_type }}
@@ -84,36 +71,23 @@ jobs:
     runs-on: [windows-latest]
     steps:
     - uses: actions/checkout@v2
-    - name: shared_env
-      if: matrix.config.shared == 'ON'
-      run: |
-        echo "::set-env name=origin_path::$env:PATH"
-        echo "::add-path::$pwd\build\windows_shared_library"
-    - name: debug_env
-      if: matrix.config.build_type == 'Debug'
-      run: |
-        echo "::set-env name=CXXFLAGS::-Wa,-mbig-obj"
     - name: configure
     # Use cmd to remove the path easily
       run: |
-        set PATH=%PATH:C:\Program Files\Git\bin;=%
-        set PATH=%PATH:C:\Program Files\Git\usr\bin;=%
         bcdedit /set IncreaseUserVa 3072
         editbin /LARGEADDRESSAWARE "C:\Program Files\Git\mingw64\bin\cc1plus.exe"
+        path %PATH:C:\Program Files\Git\bin;=%
+        path %PATH:C:\Program Files\Git\usr\bin;=%;%GITHUB_WORKSPACE%\build\windows_shared_library
         mkdir build
         cd build
-        cmake -G "MinGW Makefiles" -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_COMPILER_FLAGS=${{ matrix.config.cflags }} ..
+        cmake -G "MinGW Makefiles" -DCMAKE_CXX_FLAGS=-Wa,-mbig-obj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_COMPILER_FLAGS=${{ matrix.config.cflags }} ..
         cmake --build . -j4
         ctest . --output-on-failure
       shell: cmd
-    - name: install_shared_env
-      if: matrix.config.shared == 'ON'
-      run: |
-        echo "::set-env name=PATH::C:\Program Files (x86)\Ginkgo\lib;$env:origin_path"
     - name: install
       run: |
         set PATH=%PATH:C:\Program Files\Git\bin;=%
-        set PATH=%PATH:C:\Program Files\Git\usr\bin;=%
+        set PATH=%PATH:C:\Program Files\Git\usr\bin;=%;C:\Program Files (x86)\Ginkgo\lib
         cd build
         cmake --install .
         cmake --build . --target test_install
@@ -136,36 +110,19 @@ jobs:
         choco install cygwin -y
         choco install cyg-get -y
         cyg-get cmake make gcc-g++ git
-    - name: shared_static_env
-      run: |
-        echo "::set-env name=shared_ON_path::;$pwd\build\windows_shared_library"
-        echo "::set-env name=shared_OFF_path::"
-    - name: debug_env
-      if: matrix.config.build_type == 'Debug'
-      run: |
-        echo "::set-env name=CXXFLAGS::-Wa,-mbig-obj"
     - name: configure
       run: |
-        path C:\tools\cygwin\bin%shared_${{ matrix.config.shared }}_path%
+        path C:\tools\cygwin\bin;%GITHUB_WORKSPACE%\build\windows_shared_library
         mkdir build
         cd build
-        bash -c "cmake -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_COMPILER_FLAGS=${{ matrix.config.cflags }} .."
+        bash -c "cmake -DCMAKE_CXX_FLAGS=-Wa,-mbig-obj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_COMPILER_FLAGS=${{ matrix.config.cflags }} .."
         bash -c "make -j4"
         bash -c "ctest . --output-on-failure"
       shell: cmd
-    - name: install_shared
-      if: matrix.config.shared == 'ON'
+    - name: install
       run: |
         path C:\tools\cygwin\bin
         cd build
         bash -c "make install"
         bash -c "export PATH=/usr/local/lib:$PATH && make test_install"
       shell: cmd
-    - name: install_static
-      if: matrix.config.shared == 'OFF'
-      run: |
-        path C:\tools\cygwin\bin
-        cd build
-        bash -c "make install"
-        bash -c "make test_install"
-      shell: cmd

From 47149626e2d85e919f46b4fca4a769109c30ac52 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 10 Nov 2020 16:18:14 +0100
Subject: [PATCH 0128/1008] Disable HIP with CUDA < 9.2.

---
 .gitlab-ci.yml     |  4 ----
 hip/CMakeLists.txt | 12 ++++++------
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8f5f3818c0d..9969d84177c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -158,7 +158,6 @@ build/cuda90/gcc/all/debug/shared:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
-    BUILD_HIP: "ON"
     BUILD_TYPE: "Debug"
     RUN_EXAMPLES: "ON"
     CUDA_ARCH: 35
@@ -179,7 +178,6 @@ build/cuda90/clang/all/release/static:
     CXX_COMPILER: "clang++"
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
-    BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
@@ -199,7 +197,6 @@ build/cuda91/gcc/all/debug/static:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
-    BUILD_HIP: "ON"
     BUILD_TYPE: "Debug"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
@@ -220,7 +217,6 @@ build/cuda91/clang/all/release/shared:
     CXX_COMPILER: "clang++"
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
-    BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
   only:
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 2bb1e6911ee..b536e56be3b 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -1,3 +1,7 @@
+if (GINKGO_HIP_PLATFORM MATCHES "nvcc" AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 9.2)
+    message(FATAL_ERROR "Ginkgo HIP backend requires CUDA >= 9.2.")
+endif()
+
 if(NOT DEFINED ROCM_PATH)
     if(DEFINED ENV{ROCM_PATH})
         set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCM has been installed")
@@ -195,11 +199,7 @@ if (GINKGO_HIP_PLATFORM MATCHES "nvcc")
     endif()
 
     # Remove false positive CUDA warnings when calling one<T>() and zero<T>()
-    # This creates a compilation bug on nvcc 9.0.102 *with* the new array_deleter
-    # merged at commit ed12b3df5d26
-    if(NOT CMAKE_CUDA_COMPILER_VERSION MATCHES "9.0")
-        set(GINKGO_HIP_NVCC_ADDITIONAL_FLAGS --expt-relaxed-constexpr)
-    endif()
+    set(GINKGO_HIP_NVCC_ADDITIONAL_FLAGS --expt-relaxed-constexpr)
 
     if (GINKGO_HIP_PLATFROM MATCHES "nvcc" AND CMAKE_CUDA_COMPILER_VERSION
             MATCHES "9.2" AND CMAKE_CUDA_HOST_COMPILER MATCHES ".*clang.*" )
@@ -267,7 +267,7 @@ if(GINKGO_HIP_PLATFORM MATCHES "hcc")
     endif()
     target_link_libraries(ginkgo_hip PRIVATE hip::device)
 elseif(GINKGO_HIP_PLATFORM MATCHES "nvcc")
-    find_package(CUDA 9.0 REQUIRED)
+    find_package(CUDA 9.2 REQUIRED)
     target_link_libraries(ginkgo_hip PUBLIC ${CUDA_LIBRARIES})
     set(HIP_CUDA_LIBRARIES ${CUDA_LIBRARIES} PARENT_SCOPE)
 endif()

From 3914b975038c6e9a944befad898816565611f053 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 18 Nov 2020 21:17:21 +0100
Subject: [PATCH 0129/1008] add 64bit tests for row_gather

---
 reference/test/matrix/dense_kernels.cpp | 43 +++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 3a977bebb59..6f64ab26289 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -1761,6 +1761,49 @@ TYPED_TEST(Dense, SquareMatrixCanGatherRowsIntoDense)
 }
 
 
+TYPED_TEST(Dense, SquareMatrixCanGatherRows64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
+
+    auto row_gathered = this->mtx5->row_gather(&permute_idxs);
+
+    // clang-format off
+    GKO_ASSERT_MTX_NEAR(row_gathered,
+                        l({{-2.0, 2.0, 4.5},
+                           {1.0, -1.0, -0.5}}), r<TypeParam>::value);
+    // clang-format on
+}
+
+
+TYPED_TEST(Dense, SquareMatrixCanGatherRowsIntoDense64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
+    auto row_gathered = Mtx::create(exec, gko::dim<2>{2, 3});
+
+    this->mtx5->row_gather(&permute_idxs, row_gathered.get());
+
+    // clang-format off
+    GKO_ASSERT_MTX_NEAR(row_gathered,
+                        l({{-2.0, 2.0, 4.5},
+                           {1.0, -1.0, -0.5}}), r<TypeParam>::value);
+    // clang-format on
+}
+
+
 TYPED_TEST(Dense, SquareMatrixIsRowPermutable)
 {
     // clang-format off

From 1db1c9aaec60963ff8bc2b2406f9237473130cf7 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 19 Nov 2020 18:29:48 +0100
Subject: [PATCH 0130/1008] ForwardIterator requirements in iterator_factory

---
 core/base/iterator_factory.hpp | 36 ++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/core/base/iterator_factory.hpp b/core/base/iterator_factory.hpp
index b7efd21dfe0..5e4bddeb178 100644
--- a/core/base/iterator_factory.hpp
+++ b/core/base/iterator_factory.hpp
@@ -110,7 +110,7 @@ class IteratorFactory {
 
         ~Reference() {}
 
-        Reference(IteratorFactory &parent, array_index_type array_index)
+        Reference(IteratorFactory *parent, array_index_type array_index)
             : parent_(parent), arr_index_(array_index)
         {}
 
@@ -143,10 +143,10 @@ class IteratorFactory {
         {
             // In C++11, it is legal for a nested class to access private
             // members of the parent class.
-            parent_.dominant_values_[arr_index_] =
-                std::move(other.parent_.dominant_values_[other.arr_index_]);
-            parent_.secondary_values_[arr_index_] =
-                std::move(other.parent_.secondary_values_[other.arr_index_]);
+            parent_->dominant_values_[arr_index_] =
+                std::move(other.parent_->dominant_values_[other.arr_index_]);
+            parent_->secondary_values_[arr_index_] =
+                std::move(other.parent_->secondary_values_[other.arr_index_]);
             return *this;
         }
 
@@ -174,25 +174,25 @@ class IteratorFactory {
             return left.dominant < right.dominant();
         }
 
-        ToSortType &dominant() { return parent_.dominant_values_[arr_index_]; }
+        ToSortType &dominant() { return parent_->dominant_values_[arr_index_]; }
 
         const ToSortType &dominant() const
         {
-            return parent_.dominant_values_[arr_index_];
+            return parent_->dominant_values_[arr_index_];
         }
 
         SecondaryType &secondary()
         {
-            return parent_.secondary_values_[arr_index_];
+            return parent_->secondary_values_[arr_index_];
         }
 
         const SecondaryType &secondary() const
         {
-            return parent_.secondary_values_[arr_index_];
+            return parent_->secondary_values_[arr_index_];
         }
 
     private:
-        IteratorFactory &parent_;
+        IteratorFactory *parent_;
         array_index_type arr_index_;
     };
 
@@ -214,9 +214,11 @@ class IteratorFactory {
         using reference = Reference;
         using iterator_category = std::random_access_iterator_tag;
 
+        Iterator() = default;
+
         ~Iterator() {}
 
-        Iterator(IteratorFactory &parent, difference_type array_index)
+        Iterator(IteratorFactory *parent, difference_type array_index)
             : parent_(parent), arr_index_(array_index)
         {}
 
@@ -298,12 +300,12 @@ class IteratorFactory {
         }
 
         // Comparable operators
-        bool operator==(const Iterator &other)
+        bool operator==(const Iterator &other) const
         {
             return arr_index_ == other.arr_index_;
         }
 
-        bool operator!=(const Iterator &other)
+        bool operator!=(const Iterator &other) const
         {
             return arr_index_ != other.arr_index_;
         }
@@ -329,8 +331,8 @@ class IteratorFactory {
         }
 
     private:
-        IteratorFactory &parent_;
-        difference_type arr_index_;
+        IteratorFactory *parent_{};
+        difference_type arr_index_{};
     };
 
 public:
@@ -363,7 +365,7 @@ class IteratorFactory {
      * Creates an iterator pointing to the beginning of both arrays
      * @returns  an iterator pointing to the beginning of both arrays
      */
-    Iterator begin() { return {*this, 0}; }
+    Iterator begin() { return {this, 0}; }
 
     /**
      * Creates an iterator pointing to the (excluding) end of both arrays
@@ -371,7 +373,7 @@ class IteratorFactory {
      */
     Iterator end()
     {
-        return {*this, static_cast<typename Iterator::difference_type>(size_)};
+        return {this, static_cast<typename Iterator::difference_type>(size_)};
     }
 
 private:

From 25b957d7a92a333ebb7fa7797bfa199f9ab13442 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 19 Nov 2020 09:50:25 +0100
Subject: [PATCH 0131/1008] add triangular solvers to benchmark

---
 BENCHMARKING.md             |  6 ++++--
 benchmark/solver/solver.cpp | 22 ++++++++++++++++++----
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/BENCHMARKING.md b/BENCHMARKING.md
index a26cfe5ee1a..b1c7e409b4b 100644
--- a/BENCHMARKING.md
+++ b/BENCHMARKING.md
@@ -293,8 +293,10 @@ The supported environment variables are described in the following list:
     library formats (cuSPARSE with `cusp_` prefix or hipSPARSE with `hipsp_`
     prefix) can be used as well. Multiple options can be passed. The default is
     `csr,coo,ell,hybrid,sellp`.
-* `SOLVERS={bicgstab,bicg,cg,cgs,fcg,gmres}` - the solvers which should be
-    benchmarked. Multiple options can be passed. The default is `cg`.
+* `SOLVERS={bicgstab,bicg,cg,cgs,fcg,gmres,lower_trs,upper_trs}` - the solvers
+    which should be benchmarked. Multiple options can be passed.
+    Note that `lower/upper_trs` by default don't use a preconditioner, as they
+    are exact direct solvers. The default is `cg`.
 * `SOLVERS_PRECISION=<precision>` - the minimal residual reduction before which
     the solver should stop. The default is `1e-6`.
 * `SOLVERS_MAX_ITERATION=<number>` - the maximum number of iterations with which
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index f5cfeae39ea..f4c88521dbe 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -65,10 +65,10 @@ DEFINE_bool(
     rel_residual, false,
     "Use relative residual instead of residual reduction stopping criterion");
 
-DEFINE_string(
-    solvers, "cg",
-    "A comma-separated list of solvers to run. "
-    "Supported values are: bicgstab, bicg, cg, cgs, fcg, gmres, overhead");
+DEFINE_string(solvers, "cg",
+              "A comma-separated list of solvers to run. "
+              "Supported values are: bicgstab, bicg, cg, cgs, fcg, gmres, "
+              "lower_trs, upper_trs, overhead");
 
 DEFINE_uint32(
     nrhs, 1,
@@ -167,6 +167,20 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                             .with_preconditioner(give(precond))
                             .on(exec);
                     }},
+                   {"lower_trs",
+                    [](std::shared_ptr<const gko::Executor> exec,
+                       std::shared_ptr<const gko::LinOpFactory> precond) {
+                        return gko::solver::LowerTrs<>::build()
+                            .with_num_rhs(FLAGS_nrhs)
+                            .on(exec);
+                    }},
+                   {"upper_trs",
+                    [](std::shared_ptr<const gko::Executor> exec,
+                       std::shared_ptr<const gko::LinOpFactory> precond) {
+                        return gko::solver::UpperTrs<>::build()
+                            .with_num_rhs(FLAGS_nrhs)
+                            .on(exec);
+                    }},
                    {"overhead", create_solver<gko::Overhead<>>}};
 
 

From f81c280db26be2165ed0ebdac6e7444b97631ea3 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 19 Nov 2020 17:26:23 +0100
Subject: [PATCH 0132/1008] review update

Co-authored-by: tcojean <terry.cojean@kit.edu>
Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
---
 BENCHMARKING.md             | 6 +++---
 benchmark/solver/solver.cpp | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/BENCHMARKING.md b/BENCHMARKING.md
index b1c7e409b4b..043cdd71cf7 100644
--- a/BENCHMARKING.md
+++ b/BENCHMARKING.md
@@ -294,9 +294,9 @@ The supported environment variables are described in the following list:
     prefix) can be used as well. Multiple options can be passed. The default is
     `csr,coo,ell,hybrid,sellp`.
 * `SOLVERS={bicgstab,bicg,cg,cgs,fcg,gmres,lower_trs,upper_trs}` - the solvers
-    which should be benchmarked. Multiple options can be passed.
-    Note that `lower/upper_trs` by default don't use a preconditioner, as they
-    are exact direct solvers. The default is `cg`.
+    which should be benchmarked. Multiple options can be passed. The default
+    is `cg`. Note that `lower/upper_trs` by default don't use a preconditioner,
+    as they are by default exact direct solvers.
 * `SOLVERS_PRECISION=<precision>` - the minimal residual reduction before which
     the solver should stop. The default is `1e-6`.
 * `SOLVERS_MAX_ITERATION=<number>` - the maximum number of iterations with which
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index f4c88521dbe..578c89a9473 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -169,14 +169,14 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                     }},
                    {"lower_trs",
                     [](std::shared_ptr<const gko::Executor> exec,
-                       std::shared_ptr<const gko::LinOpFactory> precond) {
+                       std::shared_ptr<const gko::LinOpFactory>) {
                         return gko::solver::LowerTrs<>::build()
                             .with_num_rhs(FLAGS_nrhs)
                             .on(exec);
                     }},
                    {"upper_trs",
                     [](std::shared_ptr<const gko::Executor> exec,
-                       std::shared_ptr<const gko::LinOpFactory> precond) {
+                       std::shared_ptr<const gko::LinOpFactory>) {
                         return gko::solver::UpperTrs<>::build()
                             .with_num_rhs(FLAGS_nrhs)
                             .on(exec);

From e02bceae2760aff583d452b84608fdc685c1ba28 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Fri, 7 Aug 2020 09:50:15 +0200
Subject: [PATCH 0133/1008] Add IDR skeleton

---
 core/CMakeLists.txt                      |  1 +
 core/device_hooks/common_kernels.inc.cpp | 34 ++++++++++++++++++++++++
 core/test/solver/CMakeLists.txt          |  1 +
 cuda/CMakeLists.txt                      |  5 ++--
 cuda/test/solver/CMakeLists.txt          |  5 ++--
 hip/CMakeLists.txt                       |  3 ++-
 hip/test/solver/CMakeLists.txt           |  1 +
 include/ginkgo/ginkgo.hpp                |  1 +
 omp/CMakeLists.txt                       |  3 ++-
 omp/test/solver/CMakeLists.txt           |  1 +
 reference/CMakeLists.txt                 |  3 ++-
 reference/test/solver/CMakeLists.txt     |  1 +
 12 files changed, 52 insertions(+), 7 deletions(-)

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 31639009e5a..399a779ace3 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -38,6 +38,7 @@ target_sources(ginkgo
     solver/cgs.cpp
     solver/fcg.cpp
     solver/gmres.cpp
+    solver/idr.cpp
     solver/ir.cpp
     solver/lower_trs.cpp
     solver/upper_trs.cpp
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 5d8e72e9160..4ce20c464af 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -59,6 +59,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/cgs_kernels.hpp"
 #include "core/solver/fcg_kernels.hpp"
 #include "core/solver/gmres_kernels.hpp"
+#include "core/solver/idr_kernels.hpp"
 #include "core/solver/ir_kernels.hpp"
 #include "core/solver/lower_trs_kernels.hpp"
 #include "core/solver/upper_trs_kernels.hpp"
@@ -464,6 +465,39 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_FINALIZE_KERNEL);
 }  // namespace bicgstab
 
 
+// TODO (script:idr): adapt this block as needed
+namespace idr {
+
+
+template <typename ValueType>
+GKO_DECLARE_IDR_INITIALIZE_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
+
+template <typename ValueType>
+GKO_DECLARE_IDR_STEP_1_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
+
+template <typename ValueType>
+GKO_DECLARE_IDR_STEP_2_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
+
+template <typename ValueType>
+GKO_DECLARE_IDR_STEP_3_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
+
+template <typename ValueType>
+GKO_DECLARE_IDR_FINALIZE_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_FINALIZE_KERNEL);
+
+
+}  // namespace idr
+
+
 namespace cgs {
 
 
diff --git a/core/test/solver/CMakeLists.txt b/core/test/solver/CMakeLists.txt
index e017edd6bee..7454c5a925a 100644
--- a/core/test/solver/CMakeLists.txt
+++ b/core/test/solver/CMakeLists.txt
@@ -4,6 +4,7 @@ ginkgo_create_test(cg)
 ginkgo_create_test(cgs)
 ginkgo_create_test(fcg)
 ginkgo_create_test(gmres)
+ginkgo_create_test(idr)
 ginkgo_create_test(ir)
 ginkgo_create_test(lower_trs)
 ginkgo_create_test(upper_trs)
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 021f17a081d..66d67fecb46 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -74,14 +74,14 @@ target_sources(ginkgo_cuda
     components/fill_array.cu
     components/precision_conversion.cu
     components/prefix_sum.cu
-    factorization/ilu_kernels.cu
     factorization/factorization_kernels.cu
+    factorization/ilu_kernels.cu
     factorization/par_ict_kernels.cu
     factorization/par_ilu_kernels.cu
     factorization/par_ilut_approx_filter_kernel.cu
     factorization/par_ilut_filter_kernel.cu
-    factorization/par_ilut_select_kernel.cu
     factorization/par_ilut_select_common.cu
+    factorization/par_ilut_select_kernel.cu
     factorization/par_ilut_spgeam_kernel.cu
     factorization/par_ilut_sweep_kernel.cu
     matrix/coo_kernels.cu
@@ -104,6 +104,7 @@ target_sources(ginkgo_cuda
     solver/cgs_kernels.cu
     solver/fcg_kernels.cu
     solver/gmres_kernels.cu
+    solver/idr_kernels.cu
     solver/ir_kernels.cu
     solver/lower_trs_kernels.cu
     solver/upper_trs_kernels.cu
diff --git a/cuda/test/solver/CMakeLists.txt b/cuda/test/solver/CMakeLists.txt
index 32dbb96fe61..56ad6f7d97d 100644
--- a/cuda/test/solver/CMakeLists.txt
+++ b/cuda/test/solver/CMakeLists.txt
@@ -2,8 +2,9 @@ ginkgo_create_test(bicg_kernels)
 ginkgo_create_test(bicgstab_kernels)
 ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
+ginkgo_create_test_cpp_cuda_header(lower_trs_kernels)
+ginkgo_create_test_cpp_cuda_header(upper_trs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
+ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
-ginkgo_create_test_cpp_cuda_header(lower_trs_kernels)
-ginkgo_create_test_cpp_cuda_header(upper_trs_kernels)
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index b536e56be3b..c1c08f72fff 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -151,8 +151,8 @@ set(GINKGO_HIP_SOURCES
     components/fill_array.hip.cpp
     components/precision_conversion.hip.cpp
     components/prefix_sum.hip.cpp
-    factorization/ilu_kernels.hip.cpp
     factorization/factorization_kernels.hip.cpp
+    factorization/ilu_kernels.hip.cpp
     factorization/par_ict_kernels.hip.cpp
     factorization/par_ilu_kernels.hip.cpp
     factorization/par_ilut_approx_filter_kernel.hip.cpp
@@ -181,6 +181,7 @@ set(GINKGO_HIP_SOURCES
     solver/cgs_kernels.hip.cpp
     solver/fcg_kernels.hip.cpp
     solver/gmres_kernels.hip.cpp
+    solver/idr_kernels.hip.cpp
     solver/ir_kernels.hip.cpp
     solver/lower_trs_kernels.hip.cpp
     solver/upper_trs_kernels.hip.cpp
diff --git a/hip/test/solver/CMakeLists.txt b/hip/test/solver/CMakeLists.txt
index 3ec7956cf65..e8bb9cc4768 100644
--- a/hip/test/solver/CMakeLists.txt
+++ b/hip/test/solver/CMakeLists.txt
@@ -4,6 +4,7 @@ ginkgo_create_hip_test_special_linkage(cg_kernels)
 ginkgo_create_hip_test_special_linkage(cgs_kernels)
 ginkgo_create_hip_test_special_linkage(fcg_kernels)
 ginkgo_create_hip_test_special_linkage(gmres_kernels)
+ginkgo_create_hip_test_special_linkage(idr_kernels)
 ginkgo_create_hip_test_special_linkage(ir_kernels)
 ginkgo_create_hip_test_special_linkage(lower_trs_kernels)
 ginkgo_create_hip_test_special_linkage(upper_trs_kernels)
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 755e7753379..40f415b2156 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -95,6 +95,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/cgs.hpp>
 #include <ginkgo/core/solver/fcg.hpp>
 #include <ginkgo/core/solver/gmres.hpp>
+#include <ginkgo/core/solver/idr.hpp>
 #include <ginkgo/core/solver/ir.hpp>
 #include <ginkgo/core/solver/lower_trs.hpp>
 #include <ginkgo/core/solver/upper_trs.hpp>
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index b0468af7cd2..b08c691410a 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -8,8 +8,8 @@ target_sources(ginkgo_omp
     components/fill_array.cpp
     components/precision_conversion.cpp
     components/prefix_sum.cpp
-    factorization/ilu_kernels.cpp
     factorization/factorization_kernels.cpp
+    factorization/ilu_kernels.cpp
     factorization/par_ict_kernels.cpp
     factorization/par_ilu_kernels.cpp
     factorization/par_ilut_kernels.cpp
@@ -30,6 +30,7 @@ target_sources(ginkgo_omp
     solver/cgs_kernels.cpp
     solver/fcg_kernels.cpp
     solver/gmres_kernels.cpp
+    solver/idr_kernels.cpp
     solver/ir_kernels.cpp
     solver/lower_trs_kernels.cpp
     solver/upper_trs_kernels.cpp
diff --git a/omp/test/solver/CMakeLists.txt b/omp/test/solver/CMakeLists.txt
index 44d37b6240d..58a1e19edd1 100644
--- a/omp/test/solver/CMakeLists.txt
+++ b/omp/test/solver/CMakeLists.txt
@@ -4,6 +4,7 @@ ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
+ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
 ginkgo_create_test(lower_trs_kernels)
 ginkgo_create_test(upper_trs_kernels)
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index ba3d22a9abf..b1bf9cfed55 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -6,8 +6,8 @@ target_sources(ginkgo_reference
     components/fill_array.cpp
     components/precision_conversion.cpp
     components/prefix_sum.cpp
-    factorization/ilu_kernels.cpp
     factorization/factorization_kernels.cpp
+    factorization/ilu_kernels.cpp
     factorization/par_ict_kernels.cpp
     factorization/par_ilu_kernels.cpp
     factorization/par_ilut_kernels.cpp
@@ -28,6 +28,7 @@ target_sources(ginkgo_reference
     solver/cgs_kernels.cpp
     solver/fcg_kernels.cpp
     solver/gmres_kernels.cpp
+    solver/idr_kernels.cpp
     solver/ir_kernels.cpp
     solver/lower_trs_kernels.cpp
     solver/upper_trs_kernels.cpp
diff --git a/reference/test/solver/CMakeLists.txt b/reference/test/solver/CMakeLists.txt
index c86beb38b5c..7c53f72d86e 100644
--- a/reference/test/solver/CMakeLists.txt
+++ b/reference/test/solver/CMakeLists.txt
@@ -4,6 +4,7 @@ ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
+ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
 ginkgo_create_test(lower_trs)
 ginkgo_create_test(lower_trs_kernels)

From 4967b4d1219cc99ddbdbc270e5dc4869aa1339b3 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Mon, 10 Aug 2020 18:35:25 +0200
Subject: [PATCH 0134/1008] Add core implementation

---
 core/device_hooks/common_kernels.inc.cpp |  10 +-
 core/solver/idr.cpp                      | 210 +++++++++++++
 core/solver/idr_kernels.hpp              | 141 +++++++++
 core/test/solver/idr.cpp                 | 319 +++++++++++++++++++
 cuda/solver/idr_kernels.cu               | 170 +++++++++++
 cuda/test/solver/idr_kernels.cpp         | 369 ++++++++++++++++++++++
 hip/solver/idr_kernels.hip.cpp           | 173 +++++++++++
 hip/test/solver/idr_kernels.cpp          | 369 ++++++++++++++++++++++
 include/ginkgo/core/solver/idr.hpp       | 214 +++++++++++++
 omp/solver/idr_kernels.cpp               | 169 ++++++++++
 omp/test/solver/idr_kernels.cpp          | 368 ++++++++++++++++++++++
 reference/solver/idr_kernels.cpp         | 167 ++++++++++
 reference/test/solver/idr_kernels.cpp    | 373 +++++++++++++++++++++++
 13 files changed, 3044 insertions(+), 8 deletions(-)
 create mode 100644 core/solver/idr.cpp
 create mode 100644 core/solver/idr_kernels.hpp
 create mode 100644 core/test/solver/idr.cpp
 create mode 100644 cuda/solver/idr_kernels.cu
 create mode 100644 cuda/test/solver/idr_kernels.cpp
 create mode 100644 hip/solver/idr_kernels.hip.cpp
 create mode 100644 hip/test/solver/idr_kernels.cpp
 create mode 100644 include/ginkgo/core/solver/idr.hpp
 create mode 100644 omp/solver/idr_kernels.cpp
 create mode 100644 omp/test/solver/idr_kernels.cpp
 create mode 100644 reference/solver/idr_kernels.cpp
 create mode 100644 reference/test/solver/idr_kernels.cpp

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 4ce20c464af..8321d9ea70b 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -468,12 +468,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_FINALIZE_KERNEL);
 // TODO (script:idr): adapt this block as needed
 namespace idr {
 
-
-template <typename ValueType>
-GKO_DECLARE_IDR_INITIALIZE_KERNEL(ValueType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
-
 template <typename ValueType>
 GKO_DECLARE_IDR_STEP_1_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
@@ -490,9 +484,9 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
 template <typename ValueType>
-GKO_DECLARE_IDR_FINALIZE_KERNEL(ValueType)
+GKO_DECLARE_IDR_STEP_4_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_FINALIZE_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_4_KERNEL);
 
 
 }  // namespace idr
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
new file mode 100644
index 00000000000..825cc2aa502
--- /dev/null
+++ b/core/solver/idr.cpp
@@ -0,0 +1,210 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/solver/idr.hpp>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/utils.hpp>
+#include "core/components/fill_array.hpp"
+
+#include "core/solver/idr_kernels.hpp"
+
+
+namespace gko {
+namespace solver {
+namespace idr {
+
+
+GKO_REGISTER_OPERATION(step_1, idr::step_1);
+GKO_REGISTER_OPERATION(step_2, idr::step_2);
+GKO_REGISTER_OPERATION(step_3, idr::step_3);
+GKO_REGISTER_OPERATION(step_4, idr::step_4);
+GKO_REGISTER_OPERATION(fill_array, components::fill_array);
+
+
+}  // namespace idr
+
+
+template <typename ValueType>
+std::unique_ptr<LinOp> Idr<ValueType>::transpose() const GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    return build()
+//        .with_generated_preconditioner(
+//            share(as<Transposable>(this->get_preconditioner())->transpose()))
+//        .with_criteria(this->stop_criterion_factory_)
+//        .on(this->get_executor())
+//        ->generate(
+//            share(as<Transposable>(this->get_system_matrix())->transpose()));
+//}
+
+
+template <typename ValueType>
+std::unique_ptr<LinOp> Idr<ValueType>::conj_transpose() const
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    return build()
+//        .with_generated_preconditioner(share(
+//            as<Transposable>(this->get_preconditioner())->conj_transpose()))
+//        .with_criteria(this->stop_criterion_factory_)
+//        .on(this->get_executor())
+//        ->generate(share(
+//            as<Transposable>(this->get_system_matrix())->conj_transpose()));
+//}
+
+
+template <typename ValueType>
+void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
+{
+    using std::swap;
+    using Vector = matrix::Dense<ValueType>;
+    using NormVector = matrix::Dense<remove_complex<ValueType>>;
+
+    auto exec = this->get_executor();
+
+    auto one_op = initialize<Vector>({one<ValueType>()}, exec);
+    auto neg_one_op = initialize<Vector>({-one<ValueType>()}, exec);
+
+    auto dense_b = as<Vector>(b);
+    auto dense_x = as<Vector>(x);
+
+    constexpr uint8 RelativeStoppingId{1};
+
+    const auto problem_size = system_matrix_->get_size()[0];
+    const auto nrhs = dense_b->get_size()[1];
+
+    auto residual = Vector::create_with_config_of(dense_b);
+    auto f = Vector::create_with_config_of(dense_b);
+    auto v = Vector::create_with_config_of(dense_b);
+    auto t = Vector::create_with_config_of(dense_b);
+    auto preconditioned_vector = Vector::create_with_config_of(dense_b);
+
+    auto m =
+        Vector::create(exec, gko::dim<2>{subspace_dim_, subspace_dim_ * nrhs});
+    auto g =
+        Vector::create(exec, gko::dim<2>{problem_size, subspace_dim_ * nrhs});
+    auto u =
+        Vector::create(exec, gko::dim<2>{problem_size, subspace_dim_ * nrhs});
+
+    auto c = Vector::create(exec, gko::dim<2>{subspace_dim_, nrhs});
+    auto omega = Vector::create(exec, gko::dim<2>{1, nrhs});
+    gko::fill_array(exec, omega->get_values(), nrhs, one<ValueType>());
+
+    auto residual_norm = NormVector::create(exec, dim<2>{1, nrhs});
+
+    int total_iter = -1;
+
+    bool one_changed{};
+    Array<stopping_status> stop_status(exec, nrhs);
+
+    residual->copy_from(dense_b);
+    system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(),
+                          residual.get());
+    v->copy_from(residual.get());
+
+    while (true) {
+        ++total_iter;
+        if (stop_criterion->update()
+                .num_iterations(total_iter)
+                .residual(residual.get())
+                .residual_norm(residual_norm.get())
+                .solution(dense_x)
+                .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
+            break;
+        }
+
+        subspace_vectors_->apply(residual.get(), f.get());
+
+        for (size_type k = 0; k < subspace_dim_; k++) {
+            exec->run(make_step_1(m.get(), f.get(), c.get(), v.get(), residual.get(), g.get());
+            // c = M \ f = (c_1, ..., c_s)^T
+            // v = residual - c_k * g_k - ... - c_s * g_s
+
+            get_preconditioner()->apply(v.get(), preconditioned_vector.get());
+
+            exec->run(make_step_2(u.get(), c.get(), preconditioned_vector.get()));
+            //u_k = omega * preconditioned_vector + c_k * u_k + ... + c_s * u_s
+
+            auto u_k = u->create_submatrix(span{0, problem_size}, span{k * subspace_dim_ * nrhs, (k + 1) * subspace_dim_ * nrhs});
+            auto g_k = g->create_submatrix(span{0, problem_size}, span{k * subspace_dim_ * nrhs, (k + 1) * subspace_dim_ * nrhs});
+
+            system_matrix_->apply(u_k.get(), g_k.get());
+
+            exec->run(make_step_3(subspace_vectors_.get(), g.get(), u.get(), m.get(), f.get(), c.get(), residual.get(), dense_x));
+            // for i = 1 to k - 1 do
+            //     alpha = p^H_i * g_k / m_i,i
+            //     g_k -= alpha * g_i
+            //     u_k -= alpha * u_i
+            // end for
+            // for i = k to s do
+            //     m_i,k = p^H_i * g_k
+            // end for
+            // beta = f_k / m_k,k
+            // residual -= beta * g_k
+            // dense_x += beta * u_k
+            // f = (0,...,0,f_k+1 - beta * m_k+1,k,...,f_s - beta * m_s,k)
+        }
+
+        get_preconditioner()->apply(residual.get(),
+                                    preconditioned_vector.get());
+        system_matrix->apply(preconditioned_vector.get(), t.get());
+
+        exec->run(make_step_4(kappa_, omega.get(), t.get(), residual.get(),
+                              residual_norm.get(), v.get(), dense_x));
+        // omega = (t^H * residual) / (t^H * t)
+        // rho = (t^H * residual) / (norm(t) * norm(residual))
+        // if abs(rho) < kappa then
+        //     omega *= kappa / abs(rho)
+        // end if
+        // residual -= omega * t
+        // dense_x += omega * v
+    }
+}
+
+
+template <typename ValueType>
+void Idr<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
+                                const LinOp *beta,
+                                LinOp *x) const GKO_NOT_IMPLEMENTED;
+
+
+#define GKO_DECLARE_IDR(_type) class Idr<_type>
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR);
+
+
+}  // namespace solver
+}  // namespace gko
diff --git a/core/solver/idr_kernels.hpp b/core/solver/idr_kernels.hpp
new file mode 100644
index 00000000000..f062ad589c7
--- /dev/null
+++ b/core/solver/idr_kernels.hpp
@@ -0,0 +1,141 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_SOLVER_IDR_KERNELS_HPP_
+#define GKO_CORE_SOLVER_IDR_KERNELS_HPP_
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/stopping_status.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace idr {
+
+
+#define GKO_DECLARE_IDR_STEP_1_KERNEL(_type)                                  \
+    void step_1(std::shared_ptr<const DefaultExecutor> exec,                  \
+                const matrix::Dense<_type> *m, matrix::Dense<_type> *f,       \
+                const matrix::Dense<_type> *c, const matrix::Dense<_type> *v, \
+                const matrix::Dense<_type> *residual,                         \
+                const matrix::Dense<_type> *g,                                \
+                const Array<stopping_status> *stop_status)
+
+
+#define GKO_DECLARE_IDR_STEP_2_KERNEL(_type)                            \
+    void step_2(std::shared_ptr<const DefaultExecutor> exec,            \
+                const matrix::Dense<_type> *u, matrix::Dense<_type> *c, \
+                const matrix::Dense<_type> *preconditioned_vector,      \
+                const Array<stopping_status> *stop_status)
+
+
+#define GKO_DECLARE_IDR_STEP_3_KERNEL(_type)                                  \
+    void step_3(std::shared_ptr<const DefaultExecutor> exec,                  \
+                matrix::Dense<_type> *p, matrix::Dense<_type> *g,             \
+                const matrix::Dense<_type> *u, const matrix::Dense<_type> *m, \
+                const matrix::Dense<_type> *f, const matrix::Dense<_type> *c, \
+                const matrix::Dense<_type> *residual,                         \
+                const matrix::Dense<_type> *x,                                \
+                const Array<stopping_status> *stop_status)
+
+
+#define GKO_DECLARE_IDR_STEP_4_KERNEL(_type)                                \
+    void step_4(                                                            \
+        std::shared_ptr<const DefaultExecutor> exec, const _type kappa,     \
+        matrix::Dense<_type> *omega, const matrix::Dense<_type> *t,         \
+        const matrix::Dense<_type> *residual,                               \
+        matrix::Dense<_type> *residual_norm, const matrix::Dense<_type> *v, \
+        matrix::Dense<_type> *x, Array<stopping_status> *stop_status)
+
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES          \
+    template <typename ValueType>             \
+    GKO_DECLARE_IDR_STEP_1_KERNEL(ValueType); \
+    template <typename ValueType>             \
+    GKO_DECLARE_IDR_STEP_2_KERNEL(ValueType); \
+    template <typename ValueType>             \
+    GKO_DECLARE_IDR_STEP_3_KERNEL(ValueType); \
+    template <typename ValueType>             \
+    GKO_DECLARE_IDR_STEP_4_KERNEL(ValueType)
+
+
+}  // namespace idr
+
+
+namespace omp {
+namespace idr {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace idr
+}  // namespace omp
+
+
+namespace cuda {
+namespace idr {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace idr
+}  // namespace cuda
+
+
+namespace reference {
+namespace idr {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace idr
+}  // namespace reference
+
+
+namespace hip {
+namespace idr {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace idr
+}  // namespace hip
+
+
+#undef GKO_DECLARE_ALL_AS_TEMPLATES
+
+
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_CORE_SOLVER_IDR_KERNELS_HPP_
diff --git a/core/test/solver/idr.cpp b/core/test/solver/idr.cpp
new file mode 100644
index 00000000000..3139662bb4b
--- /dev/null
+++ b/core/test/solver/idr.cpp
@@ -0,0 +1,319 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/solver/idr.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
+#include <ginkgo/core/stop/time.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+template <typename T>
+class Idr : public ::testing::Test {
+protected:
+    using value_type = T;
+    using Mtx = gko::matrix::Dense<value_type>;
+    using Solver = gko::solver::Idr<value_type>;
+
+    Idr()
+        : exec(gko::ReferenceExecutor::create()),
+          mtx(gko::initialize<Mtx>(
+              {{2, -1.0, 0.0}, {-1.0, 2, -1.0}, {0.0, -1.0, 2}}, exec)),
+          idr_factory(
+              Solver::build()
+                  .with_criteria(
+                      gko::stop::Iteration::build().with_max_iters(3u).on(exec),
+                      gko::stop::ResidualNormReduction<value_type>::build()
+                          .with_reduction_factor(gko::remove_complex<T>{1e-6})
+                          .on(exec))
+                  .on(exec)),
+          solver(idr_factory->generate(mtx))
+    {}
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<Mtx> mtx;
+    std::unique_ptr<typename Solver::Factory> idr_factory;
+    std::unique_ptr<gko::LinOp> solver;
+
+    static void assert_same_matrices(const Mtx *m1, const Mtx *m2)
+    {
+        ASSERT_EQ(m1->get_size()[0], m2->get_size()[0]);
+        ASSERT_EQ(m1->get_size()[1], m2->get_size()[1]);
+        for (gko::size_type i = 0; i < m1->get_size()[0]; ++i) {
+            for (gko::size_type j = 0; j < m2->get_size()[1]; ++j) {
+                EXPECT_EQ(m1->at(i, j), m2->at(i, j));
+            }
+        }
+    }
+};
+
+TYPED_TEST_CASE(Idr, gko::test::ValueTypes);
+
+
+TYPED_TEST(Idr, IdrFactoryKnowsItsExecutor)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    ASSERT_EQ(this->idr_factory->get_executor(), this->exec);
+//}
+
+
+TYPED_TEST(Idr, IdrFactoryCreatesCorrectSolver)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Solver = typename TestFixture::Solver;
+//    ASSERT_EQ(this->solver->get_size(), gko::dim<2>(3, 3));
+//    auto idr_solver = static_cast<Solver *>(this->solver.get());
+//    ASSERT_NE(idr_solver->get_system_matrix(), nullptr);
+//    ASSERT_EQ(idr_solver->get_system_matrix(), this->mtx);
+//}
+
+
+TYPED_TEST(Idr, CanBeCopied)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using Solver = typename TestFixture::Solver;
+//    auto copy = this->idr_factory->generate(Mtx::create(this->exec));
+//
+//    copy->copy_from(this->solver.get());
+//
+//    ASSERT_EQ(copy->get_size(), gko::dim<2>(3, 3));
+//    auto copy_mtx = static_cast<Solver *>(copy.get())->get_system_matrix();
+//    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
+//                               this->mtx.get());
+//}
+
+
+TYPED_TEST(Idr, CanBeMoved)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using Solver = typename TestFixture::Solver;
+//    auto copy = this->idr_factory->generate(Mtx::create(this->exec));
+//
+//    copy->copy_from(std::move(this->solver));
+//
+//    ASSERT_EQ(copy->get_size(), gko::dim<2>(3, 3));
+//    auto copy_mtx = static_cast<Solver *>(copy.get())->get_system_matrix();
+//    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
+//                               this->mtx.get());
+//}
+
+
+TYPED_TEST(Idr, CanBeCloned)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using Solver = typename TestFixture::Solver;
+//    auto clone = this->solver->clone();
+//
+//    ASSERT_EQ(clone->get_size(), gko::dim<2>(3, 3));
+//    auto clone_mtx = static_cast<Solver *>(clone.get())->get_system_matrix();
+//    this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
+//                               this->mtx.get());
+//}
+
+
+TYPED_TEST(Idr, CanBeCleared)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Solver = typename TestFixture::Solver;
+//    this->solver->clear();
+//
+//    ASSERT_EQ(this->solver->get_size(), gko::dim<2>(0, 0));
+//    auto solver_mtx =
+//        static_cast<Solver *>(this->solver.get())->get_system_matrix();
+//    ASSERT_EQ(solver_mtx, nullptr);
+//}
+
+
+TYPED_TEST(Idr, ApplyUsesInitialGuessReturnsTrue)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    ASSERT_TRUE(this->solver->apply_uses_initial_guess());
+//}
+
+
+TYPED_TEST(Idr, CanSetPreconditionerGenerator)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Solver = typename TestFixture::Solver;
+//    using value_type = typename TestFixture::value_type;
+//    auto idr_factory =
+//        Solver::build()
+//            .with_criteria(
+//                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+//            .with_preconditioner(
+//                Solver::build()
+//                    .with_criteria(
+//                        gko::stop::Iteration::build().with_max_iters(3u).on(
+//                            this->exec))
+//                    .on(this->exec))
+//            .on(this->exec);
+//
+//    auto solver = idr_factory->generate(this->mtx);
+//    auto precond = dynamic_cast<const gko::solver::Idr<value_type> *>(
+//        gko::lend(solver->get_preconditioner()));
+//
+//    ASSERT_NE(precond, nullptr);
+//    ASSERT_EQ(precond->get_size(), gko::dim<2>(3, 3));
+//    ASSERT_EQ(precond->get_system_matrix(), this->mtx);
+//}
+
+
+TYPED_TEST(Idr, CanSetCriteriaAgain)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Solver = typename TestFixture::Solver;
+//    std::shared_ptr<gko::stop::CriterionFactory> init_crit =
+//        gko::stop::Iteration::build().with_max_iters(3u).on(this->exec);
+//    auto idr_factory =
+//        Solver::build().with_criteria(init_crit).on(this->exec);
+//
+//    ASSERT_EQ((idr_factory->get_parameters().criteria).back(), init_crit);
+//
+//    auto solver = idr_factory->generate(this->mtx);
+//    std::shared_ptr<gko::stop::CriterionFactory> new_crit =
+//        gko::stop::Iteration::build().with_max_iters(5u).on(this->exec);
+//
+//    solver->set_stop_criterion_factory(new_crit);
+//    auto new_crit_fac = solver->get_stop_criterion_factory();
+//    auto niter =
+//        static_cast<const gko::stop::Iteration::Factory *>(new_crit_fac.get())
+//            ->get_parameters()
+//            .max_iters;
+//
+//    ASSERT_EQ(niter, 5);
+//}
+
+
+TYPED_TEST(Idr, CanSetPreconditionerInFactory)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Solver = typename TestFixture::Solver;
+//    std::shared_ptr<Solver> idr_precond =
+//        Solver::build()
+//            .with_criteria(
+//                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+//            .on(this->exec)
+//            ->generate(this->mtx);
+//
+//    auto idr_factory =
+//        Solver::build()
+//            .with_criteria(
+//                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+//            .with_generated_preconditioner(idr_precond)
+//            .on(this->exec);
+//    auto solver = idr_factory->generate(this->mtx);
+//    auto precond = solver->get_preconditioner();
+//
+//    ASSERT_NE(precond.get(), nullptr);
+//    ASSERT_EQ(precond.get(), idr_precond.get());
+//}
+
+
+TYPED_TEST(Idr, ThrowsOnWrongPreconditionerInFactory)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using Solver = typename TestFixture::Solver;
+//    std::shared_ptr<Mtx> wrong_sized_mtx =
+//        Mtx::create(this->exec, gko::dim<2>{1, 3});
+//    std::shared_ptr<Solver> idr_precond =
+//        Solver::build()
+//            .with_criteria(
+//                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+//            .on(this->exec)
+//            ->generate(wrong_sized_mtx);
+//
+//    auto idr_factory =
+//        Solver::build()
+//            .with_criteria(
+//                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+//            .with_generated_preconditioner(idr_precond)
+//            .on(this->exec);
+//
+//    ASSERT_THROW(idr_factory->generate(this->mtx), gko::DimensionMismatch);
+//}
+
+
+TYPED_TEST(Idr, CanSetPreconditioner)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Solver = typename TestFixture::Solver;
+//    std::shared_ptr<Solver> idr_precond =
+//        Solver::build()
+//            .with_criteria(
+//                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+//            .on(this->exec)
+//            ->generate(this->mtx);
+//
+//    auto idr_factory =
+//        Solver::build()
+//            .with_criteria(
+//                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+//            .on(this->exec);
+//    auto solver = idr_factory->generate(this->mtx);
+//    solver->set_preconditioner(idr_precond);
+//    auto precond = solver->get_preconditioner();
+//
+//    ASSERT_NE(precond.get(), nullptr);
+//    ASSERT_EQ(precond.get(), idr_precond.get());
+//}
+
+
+}  // namespace
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
new file mode 100644
index 00000000000..f63b10bd88f
--- /dev/null
+++ b/cuda/solver/idr_kernels.cu
@@ -0,0 +1,170 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/idr_kernels.hpp"
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+
+
+#include "cuda/base/math.hpp"
+#include "cuda/base/types.hpp"
+#include "cuda/components/thread_ids.cuh"
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+/**
+ * @brief The IDR solver namespace.
+ *
+ * @ingroup idr
+ */
+namespace idr {
+
+
+constexpr int default_block_size = 512;
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const CudaExecutor> exec,
+            const matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
+            const matrix::Dense<ValueType> *c,
+            const matrix::Dense<ValueType> *v,
+            const matrix::Dense<ValueType> *residual,
+            const matrix::Dense<ValueType> *g,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
+//
+//    step_1_kernel<<<grid_size, block_size, 0, 0>>>(
+//        r->get_size()[0], r->get_size()[1], r->get_stride(),
+//        as_cuda_type(r->get_const_values()), as_cuda_type(p->get_values()),
+//        as_cuda_type(v->get_const_values()),
+//        as_cuda_type(rho->get_const_values()),
+//        as_cuda_type(prev_rho->get_const_values()),
+//        as_cuda_type(alpha->get_const_values()),
+//        as_cuda_type(omega->get_const_values()),
+//        as_cuda_type(stop_status->get_const_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
+
+
+template <typename ValueType>
+void step_2(std::shared_ptr<const CudaExecutor> exec,
+            const matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *c,
+            const matrix::Dense<ValueType> *preconditioned_vector,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
+//
+//    step_2_kernel<<<grid_size, block_size, 0, 0>>>(
+//        r->get_size()[0], r->get_size()[1], r->get_stride(),
+//        as_cuda_type(r->get_const_values()), as_cuda_type(s->get_values()),
+//        as_cuda_type(v->get_const_values()),
+//        as_cuda_type(rho->get_const_values()),
+//        as_cuda_type(alpha->get_values()),
+//        as_cuda_type(beta->get_const_values()),
+//        as_cuda_type(stop_status->get_const_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
+
+
+template <typename ValueType>
+void step_3(std::shared_ptr<const CudaExecutor> exec,
+            matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
+            const matrix::Dense<ValueType> *u,
+            const matrix::Dense<ValueType> *m,
+            const matrix::Dense<ValueType> *f,
+            const matrix::Dense<ValueType> *c,
+            const matrix::Dense<ValueType> *residual,
+            const matrix::Dense<ValueType> *x,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
+//
+//    step_3_kernel<<<grid_size, block_size, 0, 0>>>(
+//        r->get_size()[0], r->get_size()[1], r->get_stride(), x->get_stride(),
+//        as_cuda_type(x->get_values()), as_cuda_type(r->get_values()),
+//        as_cuda_type(s->get_const_values()),
+//        as_cuda_type(t->get_const_values()),
+//        as_cuda_type(y->get_const_values()),
+//        as_cuda_type(z->get_const_values()),
+//        as_cuda_type(alpha->get_const_values()),
+//        as_cuda_type(beta->get_const_values()),
+//        as_cuda_type(gamma->get_const_values()),
+//        as_cuda_type(omega->get_values()),
+//        as_cuda_type(stop_status->get_const_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
+
+
+template <typename ValueType>
+void step_4(std::shared_ptr<const CudaExecutor> exec, const ValueType kappa,
+            matrix::Dense<ValueType> *omega, const matrix::Dense<ValueType> *t,
+            const matrix::Dense<ValueType> *residual,
+            matrix::Dense<ValueType> *residual_norm,
+            const matrix::Dense<ValueType> *v, matrix::Dense<ValueType> *x,
+            Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(y->get_size()[0] * y->get_stride(), block_size.x), 1, 1);
+//
+//    finalize_kernel<<<grid_size, block_size, 0, 0>>>(
+//        y->get_size()[0], y->get_size()[1], y->get_stride(), x->get_stride(),
+//        as_cuda_type(x->get_values()), as_cuda_type(y->get_const_values()),
+//        as_cuda_type(alpha->get_const_values()),
+//        as_cuda_type(stop_status->get_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_4_KERNEL);
+
+
+}  // namespace idr
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
new file mode 100644
index 00000000000..c1453fe405a
--- /dev/null
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -0,0 +1,369 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/solver/idr.hpp>
+
+
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
+
+
+#include "core/solver/idr_kernels.hpp"
+#include "cuda/test/utils.hpp"
+
+
+namespace {
+
+
+class Idr : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<>;
+    using Solver = gko::solver::Idr<>;
+
+    Idr() : rand_engine(30) {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        cuda = gko::CudaExecutor::create(0, ref);
+
+        mtx = gen_mtx(123, 123);
+        make_diag_dominant(mtx.get());
+        d_mtx = Mtx::create(cuda);
+        d_mtx->copy_from(mtx.get());
+
+        cuda_idr_factory =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(246u).on(cuda),
+                    gko::stop::ResidualNormReduction<>::build()
+                        .with_reduction_factor(1e-15)
+                        .on(cuda))
+                .on(cuda);
+        ref_idr_factory =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(246u).on(ref),
+                    gko::stop::ResidualNormReduction<>::build()
+                        .with_reduction_factor(1e-15)
+                        .on(ref))
+                .on(ref);
+    }
+
+    void TearDown()
+    {
+        if (cuda != nullptr) {
+            ASSERT_NO_THROW(cuda->synchronize());
+        }
+    }
+
+    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<Mtx>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(num_cols, num_cols),
+            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
+    }
+
+    void initialize_data()
+    {
+        int m = 597;
+        int n = 17;
+        x = gen_mtx(m, n);
+        b = gen_mtx(m, n);
+        r = gen_mtx(m, n);
+        z = gen_mtx(m, n);
+        p = gen_mtx(m, n);
+        rr = gen_mtx(m, n);
+        s = gen_mtx(m, n);
+        t = gen_mtx(m, n);
+        y = gen_mtx(m, n);
+        v = gen_mtx(m, n);
+        prev_rho = gen_mtx(1, n);
+        rho = gen_mtx(1, n);
+        alpha = gen_mtx(1, n);
+        beta = gen_mtx(1, n);
+        gamma = gen_mtx(1, n);
+        omega = gen_mtx(1, n);
+        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(ref, n));
+        for (size_t i = 0; i < n; ++i) {
+            stop_status->get_data()[i].reset();
+        }
+
+        d_x = Mtx::create(cuda);
+        d_b = Mtx::create(cuda);
+        d_r = Mtx::create(cuda);
+        d_z = Mtx::create(cuda);
+        d_p = Mtx::create(cuda);
+        d_t = Mtx::create(cuda);
+        d_s = Mtx::create(cuda);
+        d_y = Mtx::create(cuda);
+        d_v = Mtx::create(cuda);
+        d_rr = Mtx::create(cuda);
+        d_prev_rho = Mtx::create(cuda);
+        d_rho = Mtx::create(cuda);
+        d_alpha = Mtx::create(cuda);
+        d_beta = Mtx::create(cuda);
+        d_gamma = Mtx::create(cuda);
+        d_omega = Mtx::create(cuda);
+        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(cuda));
+
+        d_x->copy_from(x.get());
+        d_b->copy_from(b.get());
+        d_r->copy_from(r.get());
+        d_z->copy_from(z.get());
+        d_p->copy_from(p.get());
+        d_v->copy_from(v.get());
+        d_y->copy_from(y.get());
+        d_t->copy_from(t.get());
+        d_s->copy_from(s.get());
+        d_rr->copy_from(rr.get());
+        d_prev_rho->copy_from(prev_rho.get());
+        d_rho->copy_from(rho.get());
+        d_alpha->copy_from(alpha.get());
+        d_beta->copy_from(beta.get());
+        d_gamma->copy_from(gamma.get());
+        d_omega->copy_from(omega.get());
+        *d_stop_status =
+            *stop_status;  // copy_from is not a public member function of Array
+    }
+
+    void make_diag_dominant(Mtx *mtx)
+    {
+        using std::abs;
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            auto sum = gko::zero<Mtx::value_type>();
+            for (int j = 0; j < mtx->get_size()[1]; ++j) {
+                sum += abs(mtx->at(i, j));
+            }
+            mtx->at(i, i) = sum;
+        }
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::CudaExecutor> cuda;
+
+    std::ranlux48 rand_engine;
+
+    std::shared_ptr<Mtx> mtx;
+    std::shared_ptr<Mtx> d_mtx;
+    std::unique_ptr<Solver::Factory> cuda_idr_factory;
+    std::unique_ptr<Solver::Factory> ref_idr_factory;
+
+    std::unique_ptr<Mtx> x;
+    std::unique_ptr<Mtx> b;
+    std::unique_ptr<Mtx> r;
+    std::unique_ptr<Mtx> z;
+    std::unique_ptr<Mtx> p;
+    std::unique_ptr<Mtx> rr;
+    std::unique_ptr<Mtx> s;
+    std::unique_ptr<Mtx> t;
+    std::unique_ptr<Mtx> y;
+    std::unique_ptr<Mtx> v;
+    std::unique_ptr<Mtx> prev_rho;
+    std::unique_ptr<Mtx> rho;
+    std::unique_ptr<Mtx> alpha;
+    std::unique_ptr<Mtx> beta;
+    std::unique_ptr<Mtx> gamma;
+    std::unique_ptr<Mtx> omega;
+    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
+
+    std::unique_ptr<Mtx> d_x;
+    std::unique_ptr<Mtx> d_b;
+    std::unique_ptr<Mtx> d_r;
+    std::unique_ptr<Mtx> d_z;
+    std::unique_ptr<Mtx> d_p;
+    std::unique_ptr<Mtx> d_t;
+    std::unique_ptr<Mtx> d_s;
+    std::unique_ptr<Mtx> d_y;
+    std::unique_ptr<Mtx> d_v;
+    std::unique_ptr<Mtx> d_rr;
+    std::unique_ptr<Mtx> d_prev_rho;
+    std::unique_ptr<Mtx> d_rho;
+    std::unique_ptr<Mtx> d_alpha;
+    std::unique_ptr<Mtx> d_beta;
+    std::unique_ptr<Mtx> d_gamma;
+    std::unique_ptr<Mtx> d_omega;
+    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
+};
+
+
+TEST_F(Idr, CudaIdrInitializeIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    initialize_data();
+//
+//    gko::kernels::reference::idr::initialize(
+//        ref, b.get(), r.get(), rr.get(), y.get(), s.get(), t.get(), z.get(),
+//        v.get(), p.get(), prev_rho.get(), rho.get(), alpha.get(), beta.get(),
+//        gamma.get(), omega.get(), stop_status.get());
+//    gko::kernels::cuda::idr::initialize(
+//        cuda, d_b.get(), d_r.get(), d_rr.get(), d_y.get(), d_s.get(),
+//        d_t.get(), d_z.get(), d_v.get(), d_p.get(), d_prev_rho.get(),
+//        d_rho.get(), d_alpha.get(), d_beta.get(), d_gamma.get(),
+//        d_omega.get(), d_stop_status.get());
+//
+//    GKO_EXPECT_MTX_NEAR(d_r, r, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_z, z, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_p, p, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_y, y, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_t, t, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_s, s, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_rr, rr, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_v, v, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_rho, rho, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_alpha, alpha, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_beta, beta, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_gamma, gamma, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_omega, omega, 1e-14);
+//    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
+//}
+
+
+TEST_F(Idr, CudaIdrStep1IsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    initialize_data();
+//
+//    gko::kernels::reference::idr::step_1(
+//        ref, r.get(), p.get(), v.get(), rho.get(), prev_rho.get(),
+//        alpha.get(), omega.get(), stop_status.get());
+//    gko::kernels::cuda::idr::step_1(
+//        cuda, d_r.get(), d_p.get(), d_v.get(), d_rho.get(), d_prev_rho.get(),
+//        d_alpha.get(), d_omega.get(), d_stop_status.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
+//}
+
+
+TEST_F(Idr, CudaIdrStep2IsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    initialize_data();
+//
+//    gko::kernels::reference::idr::step_2(ref, r.get(), s.get(), v.get(),
+//                                              rho.get(), alpha.get(),
+//                                              beta.get(), stop_status.get());
+//    gko::kernels::cuda::idr::step_2(cuda, d_r.get(), d_s.get(), d_v.get(),
+//                                         d_rho.get(), d_alpha.get(),
+//                                         d_beta.get(), d_stop_status.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
+//    GKO_ASSERT_MTX_NEAR(d_s, s, 1e-14);
+//}
+
+
+TEST_F(Idr, CudaIdrStep3IsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    initialize_data();
+//
+//    gko::kernels::reference::idr::step_3(
+//        ref, x.get(), r.get(), s.get(), t.get(), y.get(), z.get(),
+//        alpha.get(), beta.get(), gamma.get(), omega.get(), stop_status.get());
+//    gko::kernels::cuda::idr::step_3(
+//        cuda, d_x.get(), d_r.get(), d_s.get(), d_t.get(), d_y.get(),
+//        d_z.get(), d_alpha.get(), d_beta.get(), d_gamma.get(), d_omega.get(),
+//        d_stop_status.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_omega, omega, 1e-14);
+//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+//    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
+//}
+
+
+TEST_F(Idr, CudaIdrApplyOneRHSIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    int m = 123;
+//    int n = 1;
+//    auto ref_solver = ref_idr_factory->generate(mtx);
+//    auto cuda_solver = cuda_idr_factory->generate(d_mtx);
+//    auto b = gen_mtx(m, n);
+//    auto x = gen_mtx(m, n);
+//    auto d_b = Mtx::create(cuda);
+//    auto d_x = Mtx::create(cuda);
+//    d_b->copy_from(b.get());
+//    d_x->copy_from(x.get());
+//
+//    ref_solver->apply(b.get(), x.get());
+//    cuda_solver->apply(d_b.get(), d_x.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+//}
+
+
+TEST_F(Idr, CudaIdrApplyMultipleRHSIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    int m = 123;
+//    int n = 16;
+//    auto cuda_solver = cuda_idr_factory->generate(d_mtx);
+//    auto ref_solver = ref_idr_factory->generate(mtx);
+//    auto b = gen_mtx(m, n);
+//    auto x = gen_mtx(m, n);
+//    auto d_b = Mtx::create(cuda);
+//    auto d_x = Mtx::create(cuda);
+//    d_b->copy_from(b.get());
+//    d_x->copy_from(x.get());
+//
+//    ref_solver->apply(b.get(), x.get());
+//    cuda_solver->apply(d_b.get(), d_x.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+//}
+
+
+}  // namespace
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
new file mode 100644
index 00000000000..da3ca49e590
--- /dev/null
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -0,0 +1,173 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/idr_kernels.hpp"
+
+
+#include <hip/hip_runtime.h>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+
+
+#include "hip/base/math.hip.hpp"
+#include "hip/base/types.hip.hpp"
+#include "hip/components/thread_ids.hip.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace hip {
+/**
+ * @brief The IDR solver namespace.
+ *
+ * @ingroup idr
+ */
+namespace idr {
+
+
+constexpr int default_block_size = 512;
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const HipExecutor> exec,
+            const matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
+            const matrix::Dense<ValueType> *c,
+            const matrix::Dense<ValueType> *v,
+            const matrix::Dense<ValueType> *residual,
+            const matrix::Dense<ValueType> *g,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
+//
+//    step_1_kernel<<<grid_size, block_size, 0, 0>>>(
+//        r->get_size()[0], r->get_size()[1], r->get_stride(),
+//        as_cuda_type(r->get_const_values()), as_cuda_type(p->get_values()),
+//        as_cuda_type(v->get_const_values()),
+//        as_cuda_type(rho->get_const_values()),
+//        as_cuda_type(prev_rho->get_const_values()),
+//        as_cuda_type(alpha->get_const_values()),
+//        as_cuda_type(omega->get_const_values()),
+//        as_cuda_type(stop_status->get_const_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
+
+
+template <typename ValueType>
+void step_2(std::shared_ptr<const HipExecutor> exec,
+            const matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *c,
+            const matrix::Dense<ValueType> *preconditioned_vector,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
+//
+//    step_2_kernel<<<grid_size, block_size, 0, 0>>>(
+//        r->get_size()[0], r->get_size()[1], r->get_stride(),
+//        as_cuda_type(r->get_const_values()), as_cuda_type(s->get_values()),
+//        as_cuda_type(v->get_const_values()),
+//        as_cuda_type(rho->get_const_values()),
+//        as_cuda_type(alpha->get_values()),
+//        as_cuda_type(beta->get_const_values()),
+//        as_cuda_type(stop_status->get_const_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
+
+
+template <typename ValueType>
+void step_3(std::shared_ptr<const HipExecutor> exec,
+            matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
+            const matrix::Dense<ValueType> *u,
+            const matrix::Dense<ValueType> *m,
+            const matrix::Dense<ValueType> *f,
+            const matrix::Dense<ValueType> *c,
+            const matrix::Dense<ValueType> *residual,
+            const matrix::Dense<ValueType> *x,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
+//
+//    step_3_kernel<<<grid_size, block_size, 0, 0>>>(
+//        r->get_size()[0], r->get_size()[1], r->get_stride(), x->get_stride(),
+//        as_cuda_type(x->get_values()), as_cuda_type(r->get_values()),
+//        as_cuda_type(s->get_const_values()),
+//        as_cuda_type(t->get_const_values()),
+//        as_cuda_type(y->get_const_values()),
+//        as_cuda_type(z->get_const_values()),
+//        as_cuda_type(alpha->get_const_values()),
+//        as_cuda_type(beta->get_const_values()),
+//        as_cuda_type(gamma->get_const_values()),
+//        as_cuda_type(omega->get_values()),
+//        as_cuda_type(stop_status->get_const_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
+
+
+template <typename ValueType>
+void step_4(std::shared_ptr<const HipExecutor> exec, const ValueType kappa,
+            matrix::Dense<ValueType> *omega, const matrix::Dense<ValueType> *t,
+            const matrix::Dense<ValueType> *residual,
+            matrix::Dense<ValueType> *residual_norm,
+            const matrix::Dense<ValueType> *v, matrix::Dense<ValueType> *x,
+            Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(y->get_size()[0] * y->get_stride(), block_size.x), 1, 1);
+//
+//    finalize_kernel<<<grid_size, block_size, 0, 0>>>(
+//        y->get_size()[0], y->get_size()[1], y->get_stride(), x->get_stride(),
+//        as_cuda_type(x->get_values()), as_cuda_type(y->get_const_values()),
+//        as_cuda_type(alpha->get_const_values()),
+//        as_cuda_type(stop_status->get_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_4_KERNEL);
+
+
+}  // namespace idr
+}  // namespace hip
+}  // namespace kernels
+}  // namespace gko
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
new file mode 100644
index 00000000000..74f17deda57
--- /dev/null
+++ b/hip/test/solver/idr_kernels.cpp
@@ -0,0 +1,369 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/solver/idr.hpp>
+
+
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
+
+
+#include "core/solver/idr_kernels.hpp"
+#include "hip/test/utils.hip.hpp"
+
+
+namespace {
+
+
+class Idr : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<>;
+    using Solver = gko::solver::Idr<>;
+
+    Idr() : rand_engine(30) {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::HipExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        hip = gko::HipExecutor::create(0, ref);
+
+        mtx = gen_mtx(123, 123);
+        make_diag_dominant(mtx.get());
+        d_mtx = Mtx::create(hip);
+        d_mtx->copy_from(mtx.get());
+
+        hip_idr_factory =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(246u).on(hip),
+                    gko::stop::ResidualNormReduction<>::build()
+                        .with_reduction_factor(1e-15)
+                        .on(hip))
+                .on(hip);
+        ref_idr_factory =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(246u).on(ref),
+                    gko::stop::ResidualNormReduction<>::build()
+                        .with_reduction_factor(1e-15)
+                        .on(ref))
+                .on(ref);
+    }
+
+    void TearDown()
+    {
+        if (hip != nullptr) {
+            ASSERT_NO_THROW(hip->synchronize());
+        }
+    }
+
+    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<Mtx>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(num_cols, num_cols),
+            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
+    }
+
+    void initialize_data()
+    {
+        int m = 597;
+        int n = 17;
+        x = gen_mtx(m, n);
+        b = gen_mtx(m, n);
+        r = gen_mtx(m, n);
+        z = gen_mtx(m, n);
+        p = gen_mtx(m, n);
+        rr = gen_mtx(m, n);
+        s = gen_mtx(m, n);
+        t = gen_mtx(m, n);
+        y = gen_mtx(m, n);
+        v = gen_mtx(m, n);
+        prev_rho = gen_mtx(1, n);
+        rho = gen_mtx(1, n);
+        alpha = gen_mtx(1, n);
+        beta = gen_mtx(1, n);
+        gamma = gen_mtx(1, n);
+        omega = gen_mtx(1, n);
+        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(ref, n));
+        for (size_t i = 0; i < n; ++i) {
+            stop_status->get_data()[i].reset();
+        }
+
+        d_x = Mtx::create(hip);
+        d_b = Mtx::create(hip);
+        d_r = Mtx::create(hip);
+        d_z = Mtx::create(hip);
+        d_p = Mtx::create(hip);
+        d_t = Mtx::create(hip);
+        d_s = Mtx::create(hip);
+        d_y = Mtx::create(hip);
+        d_v = Mtx::create(hip);
+        d_rr = Mtx::create(hip);
+        d_prev_rho = Mtx::create(hip);
+        d_rho = Mtx::create(hip);
+        d_alpha = Mtx::create(hip);
+        d_beta = Mtx::create(hip);
+        d_gamma = Mtx::create(hip);
+        d_omega = Mtx::create(hip);
+        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(hip));
+
+        d_x->copy_from(x.get());
+        d_b->copy_from(b.get());
+        d_r->copy_from(r.get());
+        d_z->copy_from(z.get());
+        d_p->copy_from(p.get());
+        d_v->copy_from(v.get());
+        d_y->copy_from(y.get());
+        d_t->copy_from(t.get());
+        d_s->copy_from(s.get());
+        d_rr->copy_from(rr.get());
+        d_prev_rho->copy_from(prev_rho.get());
+        d_rho->copy_from(rho.get());
+        d_alpha->copy_from(alpha.get());
+        d_beta->copy_from(beta.get());
+        d_gamma->copy_from(gamma.get());
+        d_omega->copy_from(omega.get());
+        *d_stop_status =
+            *stop_status;  // copy_from is not a public member function of Array
+    }
+
+    void make_diag_dominant(Mtx *mtx)
+    {
+        using std::abs;
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            auto sum = gko::zero<Mtx::value_type>();
+            for (int j = 0; j < mtx->get_size()[1]; ++j) {
+                sum += abs(mtx->at(i, j));
+            }
+            mtx->at(i, i) = sum;
+        }
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::HipExecutor> hip;
+
+    std::ranlux48 rand_engine;
+
+    std::shared_ptr<Mtx> mtx;
+    std::shared_ptr<Mtx> d_mtx;
+    std::unique_ptr<Solver::Factory> hip_idr_factory;
+    std::unique_ptr<Solver::Factory> ref_idr_factory;
+
+    std::unique_ptr<Mtx> x;
+    std::unique_ptr<Mtx> b;
+    std::unique_ptr<Mtx> r;
+    std::unique_ptr<Mtx> z;
+    std::unique_ptr<Mtx> p;
+    std::unique_ptr<Mtx> rr;
+    std::unique_ptr<Mtx> s;
+    std::unique_ptr<Mtx> t;
+    std::unique_ptr<Mtx> y;
+    std::unique_ptr<Mtx> v;
+    std::unique_ptr<Mtx> prev_rho;
+    std::unique_ptr<Mtx> rho;
+    std::unique_ptr<Mtx> alpha;
+    std::unique_ptr<Mtx> beta;
+    std::unique_ptr<Mtx> gamma;
+    std::unique_ptr<Mtx> omega;
+    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
+
+    std::unique_ptr<Mtx> d_x;
+    std::unique_ptr<Mtx> d_b;
+    std::unique_ptr<Mtx> d_r;
+    std::unique_ptr<Mtx> d_z;
+    std::unique_ptr<Mtx> d_p;
+    std::unique_ptr<Mtx> d_t;
+    std::unique_ptr<Mtx> d_s;
+    std::unique_ptr<Mtx> d_y;
+    std::unique_ptr<Mtx> d_v;
+    std::unique_ptr<Mtx> d_rr;
+    std::unique_ptr<Mtx> d_prev_rho;
+    std::unique_ptr<Mtx> d_rho;
+    std::unique_ptr<Mtx> d_alpha;
+    std::unique_ptr<Mtx> d_beta;
+    std::unique_ptr<Mtx> d_gamma;
+    std::unique_ptr<Mtx> d_omega;
+    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
+};
+
+
+TEST_F(Idr, HipIdrInitializeIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    initialize_data();
+//
+//    gko::kernels::reference::idr::initialize(
+//        ref, b.get(), r.get(), rr.get(), y.get(), s.get(), t.get(), z.get(),
+//        v.get(), p.get(), prev_rho.get(), rho.get(), alpha.get(), beta.get(),
+//        gamma.get(), omega.get(), stop_status.get());
+//    gko::kernels::hip::idr::initialize(
+//        hip, d_b.get(), d_r.get(), d_rr.get(), d_y.get(), d_s.get(),
+//        d_t.get(), d_z.get(), d_v.get(), d_p.get(), d_prev_rho.get(),
+//        d_rho.get(), d_alpha.get(), d_beta.get(), d_gamma.get(),
+//        d_omega.get(), d_stop_status.get());
+//
+//    GKO_EXPECT_MTX_NEAR(d_r, r, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_z, z, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_p, p, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_y, y, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_t, t, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_s, s, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_rr, rr, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_v, v, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_rho, rho, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_alpha, alpha, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_beta, beta, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_gamma, gamma, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_omega, omega, 1e-14);
+//    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
+//}
+
+
+TEST_F(Idr, HipIdrStep1IsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    initialize_data();
+//
+//    gko::kernels::reference::idr::step_1(
+//        ref, r.get(), p.get(), v.get(), rho.get(), prev_rho.get(),
+//        alpha.get(), omega.get(), stop_status.get());
+//    gko::kernels::hip::idr::step_1(
+//        hip, d_r.get(), d_p.get(), d_v.get(), d_rho.get(), d_prev_rho.get(),
+//        d_alpha.get(), d_omega.get(), d_stop_status.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
+//}
+
+
+TEST_F(Idr, HipIdrStep2IsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    initialize_data();
+//
+//    gko::kernels::reference::idr::step_2(ref, r.get(), s.get(), v.get(),
+//                                              rho.get(), alpha.get(),
+//                                              beta.get(), stop_status.get());
+//    gko::kernels::hip::idr::step_2(hip, d_r.get(), d_s.get(), d_v.get(),
+//                                        d_rho.get(), d_alpha.get(),
+//                                        d_beta.get(), d_stop_status.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
+//    GKO_ASSERT_MTX_NEAR(d_s, s, 1e-14);
+//}
+
+
+TEST_F(Idr, HipIdrStep3IsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    initialize_data();
+//
+//    gko::kernels::reference::idr::step_3(
+//        ref, x.get(), r.get(), s.get(), t.get(), y.get(), z.get(),
+//        alpha.get(), beta.get(), gamma.get(), omega.get(), stop_status.get());
+//    gko::kernels::hip::idr::step_3(
+//        hip, d_x.get(), d_r.get(), d_s.get(), d_t.get(), d_y.get(), d_z.get(),
+//        d_alpha.get(), d_beta.get(), d_gamma.get(), d_omega.get(),
+//        d_stop_status.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_omega, omega, 1e-14);
+//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+//    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
+//}
+
+
+TEST_F(Idr, HipIdrApplyOneRHSIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    int m = 123;
+//    int n = 1;
+//    auto ref_solver = ref_idr_factory->generate(mtx);
+//    auto hip_solver = hip_idr_factory->generate(d_mtx);
+//    auto b = gen_mtx(m, n);
+//    auto x = gen_mtx(m, n);
+//    auto d_b = Mtx::create(hip);
+//    auto d_x = Mtx::create(hip);
+//    d_b->copy_from(b.get());
+//    d_x->copy_from(x.get());
+//
+//    ref_solver->apply(b.get(), x.get());
+//    hip_solver->apply(d_b.get(), d_x.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+//}
+
+
+TEST_F(Idr, HipIdrApplyMultipleRHSIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    int m = 123;
+//    int n = 16;
+//    auto hip_solver = hip_idr_factory->generate(d_mtx);
+//    auto ref_solver = ref_idr_factory->generate(mtx);
+//    auto b = gen_mtx(m, n);
+//    auto x = gen_mtx(m, n);
+//    auto d_b = Mtx::create(hip);
+//    auto d_x = Mtx::create(hip);
+//    d_b->copy_from(b.get());
+//    d_x->copy_from(x.get());
+//
+//    ref_solver->apply(b.get(), x.get());
+//    hip_solver->apply(d_b.get(), d_x.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+//}
+
+
+}  // namespace
diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
new file mode 100644
index 00000000000..c7b9c5e910e
--- /dev/null
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -0,0 +1,214 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_SOLVER_IDR_HPP_
+#define GKO_CORE_SOLVER_IDR_HPP_
+
+
+#include <random>
+#include <vector>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/log/logger.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/identity.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/criterion.hpp>
+
+
+namespace gko {
+/**
+ * @brief The ginkgo Solve namespace.
+ *
+ * @ingroup solvers
+ */
+namespace solver {
+
+
+/**
+ * BiCGSTAB or the Bi-Conjugate Gradient-Stabilized is a Krylov subspace solver.
+ *
+ * Being a generic solver, it is capable of solving general matrices, including
+ * non-s.p.d matrices. Though, the memory and the computational requirement of
+ * the BiCGSTAB solver are higher than of its s.p.d solver counterpart, it has
+ * the capability to solve generic systems. It was developed by stabilizing the
+ * BiCG method.
+ *
+ * @tparam ValueType precision of the elements of the system matrix.
+ *
+ * @ingroup idr
+ * @ingroup solvers
+ * @ingroup LinOp
+ */
+template <typename ValueType = default_precision>
+class Idr : public EnableLinOp<Idr<ValueType>>,
+            public Preconditionable,
+            public Transposable {
+    friend class EnableLinOp<Idr>;
+    friend class EnablePolymorphicObject<Idr, LinOp>;
+
+public:
+    using value_type = ValueType;
+    using transposed_type = Idr<ValueType>;
+
+    /**
+     * Gets the system operator (matrix) of the linear system.
+     *
+     * @return the system operator (matrix)
+     */
+    std::shared_ptr<const LinOp> get_system_matrix() const
+    {
+        return system_matrix_;
+    }
+
+    std::unique_ptr<LinOp> transpose() const override;
+
+    std::unique_ptr<LinOp> conj_transpose() const override;
+
+    /**
+     * Return true as iterative solvers use the data in x as an initial guess.
+     *
+     * @return true as iterative solvers use the data in x as an initial guess.
+     */
+    bool apply_uses_initial_guess() const override { return true; }
+
+    /**
+     * Gets the stopping criterion factory of the solver.
+     *
+     * @return the stopping criterion factory
+     */
+    std::shared_ptr<const stop::CriterionFactory> get_stop_criterion_factory()
+        const
+    {
+        return stop_criterion_factory_;
+    }
+
+    /**
+     * Sets the stopping criterion of the solver.
+     *
+     * @param other  the new stopping criterion factory
+     */
+    void set_stop_criterion_factory(
+        std::shared_ptr<const stop::CriterionFactory> other)
+    {
+        stop_criterion_factory_ = std::move(other);
+    }
+
+    GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
+    {
+        /**
+         * Criterion factories.
+         */
+        std::vector<std::shared_ptr<const stop::CriterionFactory>>
+            GKO_FACTORY_PARAMETER(criteria, nullptr);
+
+        /**
+         * Preconditioner factory.
+         */
+        std::shared_ptr<const LinOpFactory> GKO_FACTORY_PARAMETER(
+            preconditioner, nullptr);
+
+        /**
+         * Already generated preconditioner. If one is provided, the factory
+         * `preconditioner` will be ignored.
+         */
+        std::shared_ptr<const LinOp> GKO_FACTORY_PARAMETER(
+            generated_preconditioner, nullptr);
+
+        size_type GKO_FACTORY_PARAMETER(subspace_dim, 1u);
+        remove_complex<ValueType> GKO_FACTORY_PARAMETER(kappa, 0.7);
+    };
+    GKO_ENABLE_LIN_OP_FACTORY(Idr, parameters, Factory);
+    GKO_ENABLE_BUILD_METHOD(Factory);
+
+protected:
+    void apply_impl(const LinOp *b, LinOp *x) const override;
+
+    void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
+                    LinOp *x) const override;
+
+    explicit Idr(std::shared_ptr<const Executor> exec)
+        : EnableLinOp<Idr>(std::move(exec))
+    {}
+
+    explicit Idr(const Factory *factory,
+                 std::shared_ptr<const LinOp> system_matrix)
+        : EnableLinOp<Idr>(factory->get_executor(),
+                           gko::transpose(system_matrix->get_size())),
+          parameters_{factory->get_parameters()},
+          system_matrix_{std::move(system_matrix)}
+    {
+        if (parameters_.generated_preconditioner) {
+            GKO_ASSERT_EQUAL_DIMENSIONS(parameters_.generated_preconditioner,
+                                        this);
+            set_preconditioner(parameters_.generated_preconditioner);
+        } else if (parameters_.preconditioner) {
+            set_preconditioner(
+                parameters_.preconditioner->generate(system_matrix_));
+        } else {
+            set_preconditioner(matrix::Identity<ValueType>::create(
+                this->get_executor(), this->get_size()[0]));
+        }
+        stop_criterion_factory_ =
+            stop::combine(std::move(parameters_.criteria));
+
+        subspace_dim_ = parameters_.subspace_dim;
+
+        auto subspace_vectors_data = matrix_data<ValueType>(
+            gko::dim<2>(subspace_dim_, system_matrix_->get_size()[0]),
+            std::normal_distribution<>(0.0, 1.0), std::ranlux48(15));
+
+        subspace_vectors_ =
+            matrix::Dense<ValueType>::create(factory->get_executor());
+        subspace_vectors_->read(subspace_vectors_data);
+        kappa_ = parameters_.kappa;
+    }
+
+private:
+    std::shared_ptr<const LinOp> system_matrix_{};
+    std::shared_ptr<matrix::Dense<ValueType>> subspace_vectors_{};
+    std::shared_ptr<const stop::CriterionFactory> stop_criterion_factory_{};
+    size_type subspace_dim_;
+    remove_complex<ValueType> kappa_;
+};
+
+
+}  // namespace solver
+}  // namespace gko
+
+
+#endif  // GKO_CORE_SOLVER_IDR_HPP_
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
new file mode 100644
index 00000000000..824a8f8894b
--- /dev/null
+++ b/omp/solver/idr_kernels.cpp
@@ -0,0 +1,169 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/idr_kernels.hpp"
+
+
+#include <algorithm>
+
+
+#include <omp.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+/**
+ * @brief The IDR solver namespace.
+ *
+ * @ingroup idr
+ */
+namespace idr {
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const OmpExecutor> exec,
+            const matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
+            const matrix::Dense<ValueType> *c,
+            const matrix::Dense<ValueType> *v,
+            const matrix::Dense<ValueType> *residual,
+            const matrix::Dense<ValueType> *g,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
+//
+//    step_1_kernel<<<grid_size, block_size, 0, 0>>>(
+//        r->get_size()[0], r->get_size()[1], r->get_stride(),
+//        as_cuda_type(r->get_const_values()), as_cuda_type(p->get_values()),
+//        as_cuda_type(v->get_const_values()),
+//        as_cuda_type(rho->get_const_values()),
+//        as_cuda_type(prev_rho->get_const_values()),
+//        as_cuda_type(alpha->get_const_values()),
+//        as_cuda_type(omega->get_const_values()),
+//        as_cuda_type(stop_status->get_const_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
+
+
+template <typename ValueType>
+void step_2(std::shared_ptr<const OmpExecutor> exec,
+            const matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *c,
+            const matrix::Dense<ValueType> *preconditioned_vector,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
+//
+//    step_2_kernel<<<grid_size, block_size, 0, 0>>>(
+//        r->get_size()[0], r->get_size()[1], r->get_stride(),
+//        as_cuda_type(r->get_const_values()), as_cuda_type(s->get_values()),
+//        as_cuda_type(v->get_const_values()),
+//        as_cuda_type(rho->get_const_values()),
+//        as_cuda_type(alpha->get_values()),
+//        as_cuda_type(beta->get_const_values()),
+//        as_cuda_type(stop_status->get_const_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
+
+
+template <typename ValueType>
+void step_3(std::shared_ptr<const OmpExecutor> exec,
+            matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
+            const matrix::Dense<ValueType> *u,
+            const matrix::Dense<ValueType> *m,
+            const matrix::Dense<ValueType> *f,
+            const matrix::Dense<ValueType> *c,
+            const matrix::Dense<ValueType> *residual,
+            const matrix::Dense<ValueType> *x,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
+//
+//    step_3_kernel<<<grid_size, block_size, 0, 0>>>(
+//        r->get_size()[0], r->get_size()[1], r->get_stride(), x->get_stride(),
+//        as_cuda_type(x->get_values()), as_cuda_type(r->get_values()),
+//        as_cuda_type(s->get_const_values()),
+//        as_cuda_type(t->get_const_values()),
+//        as_cuda_type(y->get_const_values()),
+//        as_cuda_type(z->get_const_values()),
+//        as_cuda_type(alpha->get_const_values()),
+//        as_cuda_type(beta->get_const_values()),
+//        as_cuda_type(gamma->get_const_values()),
+//        as_cuda_type(omega->get_values()),
+//        as_cuda_type(stop_status->get_const_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
+
+
+template <typename ValueType>
+void step_4(std::shared_ptr<const OmpExecutor> exec, const ValueType kappa,
+            matrix::Dense<ValueType> *omega, const matrix::Dense<ValueType> *t,
+            const matrix::Dense<ValueType> *residual,
+            matrix::Dense<ValueType> *residual_norm,
+            const matrix::Dense<ValueType> *v, matrix::Dense<ValueType> *x,
+            Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(y->get_size()[0] * y->get_stride(), block_size.x), 1, 1);
+//
+//    finalize_kernel<<<grid_size, block_size, 0, 0>>>(
+//        y->get_size()[0], y->get_size()[1], y->get_stride(), x->get_stride(),
+//        as_cuda_type(x->get_values()), as_cuda_type(y->get_const_values()),
+//        as_cuda_type(alpha->get_const_values()),
+//        as_cuda_type(stop_status->get_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_4_KERNEL);
+
+
+}  // namespace idr
+}  // namespace omp
+}  // namespace kernels
+}  // namespace gko
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
new file mode 100644
index 00000000000..d9de535dd5a
--- /dev/null
+++ b/omp/test/solver/idr_kernels.cpp
@@ -0,0 +1,368 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/solver/idr.hpp>
+
+
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
+
+
+#include "core/solver/idr_kernels.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class Idr : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<>;
+    using Solver = gko::solver::Idr<>;
+
+    Idr() : rand_engine(30) {}
+
+    void SetUp()
+    {
+        ref = gko::ReferenceExecutor::create();
+        omp = gko::OmpExecutor::create();
+
+        mtx = gen_mtx(123, 123);
+        make_diag_dominant(mtx.get());
+        d_mtx = Mtx::create(omp);
+        d_mtx->copy_from(mtx.get());
+        omp_idr_factory =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(246u).on(omp),
+                    gko::stop::ResidualNormReduction<>::build()
+                        .with_reduction_factor(1e-15)
+                        .on(omp))
+                .on(omp);
+
+        ref_idr_factory =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(246u).on(ref),
+                    gko::stop::ResidualNormReduction<>::build()
+                        .with_reduction_factor(1e-15)
+                        .on(ref))
+                .on(ref);
+    }
+
+    void TearDown()
+    {
+        if (omp != nullptr) {
+            ASSERT_NO_THROW(omp->synchronize());
+        }
+    }
+
+    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<Mtx>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(num_cols, num_cols),
+            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
+    }
+
+    void initialize_data()
+    {
+        int m = 597;
+        int n = 17;
+        x = gen_mtx(m, n);
+        b = gen_mtx(m, n);
+        r = gen_mtx(m, n);
+        z = gen_mtx(m, n);
+        p = gen_mtx(m, n);
+        rr = gen_mtx(m, n);
+        s = gen_mtx(m, n);
+        t = gen_mtx(m, n);
+        y = gen_mtx(m, n);
+        v = gen_mtx(m, n);
+        prev_rho = gen_mtx(1, n);
+        rho = gen_mtx(1, n);
+        alpha = gen_mtx(1, n);
+        beta = gen_mtx(1, n);
+        gamma = gen_mtx(1, n);
+        omega = gen_mtx(1, n);
+        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(ref, n));
+        for (size_t i = 0; i < n; ++i) {
+            stop_status->get_data()[i].reset();
+        }
+
+        d_x = Mtx::create(omp);
+        d_b = Mtx::create(omp);
+        d_r = Mtx::create(omp);
+        d_z = Mtx::create(omp);
+        d_p = Mtx::create(omp);
+        d_t = Mtx::create(omp);
+        d_s = Mtx::create(omp);
+        d_y = Mtx::create(omp);
+        d_v = Mtx::create(omp);
+        d_rr = Mtx::create(omp);
+        d_prev_rho = Mtx::create(omp);
+        d_rho = Mtx::create(omp);
+        d_alpha = Mtx::create(omp);
+        d_beta = Mtx::create(omp);
+        d_gamma = Mtx::create(omp);
+        d_omega = Mtx::create(omp);
+        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(omp));
+
+        d_x->copy_from(x.get());
+        d_b->copy_from(b.get());
+        d_r->copy_from(r.get());
+        d_z->copy_from(z.get());
+        d_p->copy_from(p.get());
+        d_v->copy_from(v.get());
+        d_y->copy_from(y.get());
+        d_t->copy_from(t.get());
+        d_s->copy_from(s.get());
+        d_rr->copy_from(rr.get());
+        d_prev_rho->copy_from(prev_rho.get());
+        d_rho->copy_from(rho.get());
+        d_alpha->copy_from(alpha.get());
+        d_beta->copy_from(beta.get());
+        d_gamma->copy_from(gamma.get());
+        d_omega->copy_from(omega.get());
+        *d_stop_status =
+            *stop_status;  // copy_from is not a public member function of Array
+    }
+
+    void make_diag_dominant(Mtx *mtx)
+    {
+        using std::abs;
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            auto sum = gko::zero<Mtx::value_type>();
+            for (int j = 0; j < mtx->get_size()[1]; ++j) {
+                sum += abs(mtx->at(i, j));
+            }
+            mtx->at(i, i) = sum;
+        }
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::OmpExecutor> omp;
+
+    std::ranlux48 rand_engine;
+
+    std::shared_ptr<Mtx> mtx;
+    std::shared_ptr<Mtx> d_mtx;
+    std::unique_ptr<Solver::Factory> omp_idr_factory;
+    std::unique_ptr<Solver::Factory> ref_idr_factory;
+
+    std::unique_ptr<Mtx> x;
+    std::unique_ptr<Mtx> b;
+    std::unique_ptr<Mtx> r;
+    std::unique_ptr<Mtx> z;
+    std::unique_ptr<Mtx> p;
+    std::unique_ptr<Mtx> rr;
+    std::unique_ptr<Mtx> s;
+    std::unique_ptr<Mtx> t;
+    std::unique_ptr<Mtx> y;
+    std::unique_ptr<Mtx> v;
+    std::unique_ptr<Mtx> prev_rho;
+    std::unique_ptr<Mtx> rho;
+    std::unique_ptr<Mtx> alpha;
+    std::unique_ptr<Mtx> beta;
+    std::unique_ptr<Mtx> gamma;
+    std::unique_ptr<Mtx> omega;
+    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
+
+    std::unique_ptr<Mtx> d_x;
+    std::unique_ptr<Mtx> d_b;
+    std::unique_ptr<Mtx> d_r;
+    std::unique_ptr<Mtx> d_z;
+    std::unique_ptr<Mtx> d_p;
+    std::unique_ptr<Mtx> d_t;
+    std::unique_ptr<Mtx> d_s;
+    std::unique_ptr<Mtx> d_y;
+    std::unique_ptr<Mtx> d_v;
+    std::unique_ptr<Mtx> d_rr;
+    std::unique_ptr<Mtx> d_prev_rho;
+    std::unique_ptr<Mtx> d_rho;
+    std::unique_ptr<Mtx> d_alpha;
+    std::unique_ptr<Mtx> d_beta;
+    std::unique_ptr<Mtx> d_gamma;
+    std::unique_ptr<Mtx> d_omega;
+    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
+};
+
+
+TEST_F(Idr, OmpIdrInitializeIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    initialize_data();
+//
+//    gko::kernels::reference::idr::initialize(
+//        ref, b.get(), r.get(), rr.get(), y.get(), s.get(), t.get(), z.get(),
+//        v.get(), p.get(), prev_rho.get(), rho.get(), alpha.get(), beta.get(),
+//        gamma.get(), omega.get(), stop_status.get());
+//    gko::kernels::omp::idr::initialize(
+//        omp, d_b.get(), d_r.get(), d_rr.get(), d_y.get(), d_s.get(),
+//        d_t.get(), d_z.get(), d_v.get(), d_p.get(), d_prev_rho.get(),
+//        d_rho.get(), d_alpha.get(), d_beta.get(), d_gamma.get(),
+//        d_omega.get(), d_stop_status.get());
+//
+//    GKO_EXPECT_MTX_NEAR(d_r, r, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_z, z, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_p, p, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_y, y, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_t, t, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_s, s, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_rr, rr, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_v, v, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_rho, rho, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_alpha, alpha, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_beta, beta, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_gamma, gamma, 1e-14);
+//    GKO_EXPECT_MTX_NEAR(d_omega, omega, 1e-14);
+//    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
+//}
+
+
+TEST_F(Idr, OmpIdrStep1IsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    initialize_data();
+//
+//    gko::kernels::reference::idr::step_1(
+//        ref, r.get(), p.get(), v.get(), rho.get(), prev_rho.get(),
+//        alpha.get(), omega.get(), stop_status.get());
+//    gko::kernels::omp::idr::step_1(
+//        omp, d_r.get(), d_p.get(), d_v.get(), d_rho.get(), d_prev_rho.get(),
+//        d_alpha.get(), d_omega.get(), d_stop_status.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
+//}
+
+
+TEST_F(Idr, OmpIdrStep2IsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    initialize_data();
+//
+//    gko::kernels::reference::idr::step_2(ref, r.get(), s.get(), v.get(),
+//                                              rho.get(), alpha.get(),
+//                                              beta.get(), stop_status.get());
+//    gko::kernels::omp::idr::step_2(omp, d_r.get(), d_s.get(), d_v.get(),
+//                                        d_rho.get(), d_alpha.get(),
+//                                        d_beta.get(), d_stop_status.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
+//    GKO_ASSERT_MTX_NEAR(d_s, s, 1e-14);
+//}
+
+
+TEST_F(Idr, OmpIdrStep3IsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    initialize_data();
+//
+//    gko::kernels::reference::idr::step_3(
+//        ref, x.get(), r.get(), s.get(), t.get(), y.get(), z.get(),
+//        alpha.get(), beta.get(), gamma.get(), omega.get(), stop_status.get());
+//    gko::kernels::omp::idr::step_3(
+//        omp, d_x.get(), d_r.get(), d_s.get(), d_t.get(), d_y.get(), d_z.get(),
+//        d_alpha.get(), d_beta.get(), d_gamma.get(), d_omega.get(),
+//        d_stop_status.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_omega, omega, 1e-14);
+//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+//    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
+//}
+
+
+TEST_F(Idr, OmpIdrApplyOneRHSIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    int m = 123;
+//    int n = 1;
+//    auto ref_solver = ref_idr_factory->generate(mtx);
+//    auto omp_solver = omp_idr_factory->generate(d_mtx);
+//    auto b = gen_mtx(m, n);
+//    auto x = gen_mtx(m, n);
+//    auto d_b = Mtx::create(omp);
+//    auto d_x = Mtx::create(omp);
+//    d_b->copy_from(b.get());
+//    d_x->copy_from(x.get());
+//
+//    ref_solver->apply(b.get(), x.get());
+//    omp_solver->apply(d_b.get(), d_x.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+//}
+
+
+TEST_F(Idr, OmpIdrApplyMultipleRHSIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    int m = 123;
+//    int n = 16;
+//    auto omp_solver = omp_idr_factory->generate(d_mtx);
+//    auto ref_solver = ref_idr_factory->generate(mtx);
+//    auto b = gen_mtx(m, n);
+//    auto x = gen_mtx(m, n);
+//    auto d_b = Mtx::create(omp);
+//    auto d_x = Mtx::create(omp);
+//    d_b->copy_from(b.get());
+//    d_x->copy_from(x.get());
+//
+//    ref_solver->apply(b.get(), x.get());
+//    omp_solver->apply(d_b.get(), d_x.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+//}
+
+
+}  // namespace
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
new file mode 100644
index 00000000000..fd72b7f814d
--- /dev/null
+++ b/reference/solver/idr_kernels.cpp
@@ -0,0 +1,167 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/idr_kernels.hpp"
+
+
+#include <algorithm>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace reference {
+/**
+ * @brief The BiCGSTAB solver namespace.
+ *
+ * @ingroup idr
+ */
+namespace idr {
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const ReferenceExecutor> exec,
+            const matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
+            const matrix::Dense<ValueType> *c,
+            const matrix::Dense<ValueType> *v,
+            const matrix::Dense<ValueType> *residual,
+            const matrix::Dense<ValueType> *g,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
+//
+//    step_1_kernel<<<grid_size, block_size, 0, 0>>>(
+//        r->get_size()[0], r->get_size()[1], r->get_stride(),
+//        as_cuda_type(r->get_const_values()), as_cuda_type(p->get_values()),
+//        as_cuda_type(v->get_const_values()),
+//        as_cuda_type(rho->get_const_values()),
+//        as_cuda_type(prev_rho->get_const_values()),
+//        as_cuda_type(alpha->get_const_values()),
+//        as_cuda_type(omega->get_const_values()),
+//        as_cuda_type(stop_status->get_const_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
+
+
+template <typename ValueType>
+void step_2(std::shared_ptr<const ReferenceExecutor> exec,
+            const matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *c,
+            const matrix::Dense<ValueType> *preconditioned_vector,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
+//
+//    step_2_kernel<<<grid_size, block_size, 0, 0>>>(
+//        r->get_size()[0], r->get_size()[1], r->get_stride(),
+//        as_cuda_type(r->get_const_values()), as_cuda_type(s->get_values()),
+//        as_cuda_type(v->get_const_values()),
+//        as_cuda_type(rho->get_const_values()),
+//        as_cuda_type(alpha->get_values()),
+//        as_cuda_type(beta->get_const_values()),
+//        as_cuda_type(stop_status->get_const_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
+
+
+template <typename ValueType>
+void step_3(std::shared_ptr<const ReferenceExecutor> exec,
+            matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
+            const matrix::Dense<ValueType> *u,
+            const matrix::Dense<ValueType> *m,
+            const matrix::Dense<ValueType> *f,
+            const matrix::Dense<ValueType> *c,
+            const matrix::Dense<ValueType> *residual,
+            const matrix::Dense<ValueType> *x,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
+//
+//    step_3_kernel<<<grid_size, block_size, 0, 0>>>(
+//        r->get_size()[0], r->get_size()[1], r->get_stride(), x->get_stride(),
+//        as_cuda_type(x->get_values()), as_cuda_type(r->get_values()),
+//        as_cuda_type(s->get_const_values()),
+//        as_cuda_type(t->get_const_values()),
+//        as_cuda_type(y->get_const_values()),
+//        as_cuda_type(z->get_const_values()),
+//        as_cuda_type(alpha->get_const_values()),
+//        as_cuda_type(beta->get_const_values()),
+//        as_cuda_type(gamma->get_const_values()),
+//        as_cuda_type(omega->get_values()),
+//        as_cuda_type(stop_status->get_const_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
+
+
+template <typename ValueType>
+void step_4(std::shared_ptr<const ReferenceExecutor> exec,
+            const ValueType kappa, matrix::Dense<ValueType> *omega,
+            const matrix::Dense<ValueType> *t,
+            const matrix::Dense<ValueType> *residual,
+            matrix::Dense<ValueType> *residual_norm,
+            const matrix::Dense<ValueType> *v, matrix::Dense<ValueType> *x,
+            Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    const dim3 block_size(default_block_size, 1, 1);
+//    const dim3 grid_size(
+//        ceildiv(y->get_size()[0] * y->get_stride(), block_size.x), 1, 1);
+//
+//    finalize_kernel<<<grid_size, block_size, 0, 0>>>(
+//        y->get_size()[0], y->get_size()[1], y->get_stride(), x->get_stride(),
+//        as_cuda_type(x->get_values()), as_cuda_type(y->get_const_values()),
+//        as_cuda_type(alpha->get_const_values()),
+//        as_cuda_type(stop_status->get_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_4_KERNEL);
+
+
+}  // namespace idr
+}  // namespace reference
+}  // namespace kernels
+}  // namespace gko
diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp
new file mode 100644
index 00000000000..3b5d800e54c
--- /dev/null
+++ b/reference/test/solver/idr_kernels.cpp
@@ -0,0 +1,373 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/solver/idr.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
+#include <ginkgo/core/stop/time.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+template <typename T>
+class Idr : public ::testing::Test {
+protected:
+    using value_type = T;
+    using Mtx = gko::matrix::Dense<value_type>;
+    using Solver = gko::solver::Idr<value_type>;
+
+    Idr()
+        : exec(gko::ReferenceExecutor::create()),
+          mtx(gko::initialize<Mtx>(
+              {{1.0, -3.0, 0.0}, {-4.0, 1.0, -3.0}, {2.0, -1.0, 2.0}}, exec)),
+          idr_factory(
+              Solver::build()
+                  .with_criteria(
+                      gko::stop::Iteration::build().with_max_iters(8u).on(exec),
+                      gko::stop::Time::build()
+                          .with_time_limit(std::chrono::seconds(6))
+                          .on(exec),
+                      gko::stop::ResidualNormReduction<value_type>::build()
+                          .with_reduction_factor(r<value_type>::value)
+                          .on(exec))
+                  .on(exec)),
+          idr_factory_precision(
+              Solver::build()
+                  .with_criteria(
+                      gko::stop::Iteration::build().with_max_iters(50u).on(
+                          exec),
+                      gko::stop::Time::build()
+                          .with_time_limit(std::chrono::seconds(6))
+                          .on(exec),
+                      gko::stop::ResidualNormReduction<value_type>::build()
+                          .with_reduction_factor(r<value_type>::value)
+                          .on(exec))
+                  .on(exec))
+    {}
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<Mtx> mtx;
+    std::unique_ptr<typename Solver::Factory> idr_factory;
+    std::unique_ptr<typename Solver::Factory> idr_factory_precision;
+};
+
+TYPED_TEST_CASE(Idr, gko::test::ValueTypes);
+
+
+TYPED_TEST(Idr, SolvesDenseSystem)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using value_type = typename TestFixture::value_type;
+//    auto half_tol = std::sqrt(r<value_type>::value);
+//    auto solver = this->idr_factory->generate(this->mtx);
+//    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+//    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+//
+//    solver->apply(b.get(), x.get());
+//
+//    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), half_tol);
+//}
+
+
+TYPED_TEST(Idr, SolvesMultipleDenseSystems)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using value_type = typename TestFixture::value_type;
+//    using T = value_type;
+//    auto half_tol = std::sqrt(r<value_type>::value);
+//    auto solver = this->idr_factory->generate(this->mtx);
+//    auto b = gko::initialize<Mtx>(
+//        {I<T>{-1.0, -5.0}, I<T>{3.0, 1.0}, I<T>{1.0, -2.0}}, this->exec);
+//    auto x = gko::initialize<Mtx>(
+//        {I<T>{0.0, 0.0}, I<T>{0.0, 0.0}, I<T>{0.0, 0.0}}, this->exec);
+//
+//    solver->apply(b.get(), x.get());
+//
+//    GKO_ASSERT_MTX_NEAR(x, l({{-4.0, 1.0}, {-1.0, 2.0}, {4.0, -1.0}}),
+//                        half_tol);
+//}
+
+
+TYPED_TEST(Idr, SolvesDenseSystemUsingAdvancedApply)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using value_type = typename TestFixture::value_type;
+//    auto half_tol = std::sqrt(r<value_type>::value);
+//    auto solver = this->idr_factory->generate(this->mtx);
+//    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+//    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+//    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+//    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
+//
+//    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+//
+//
+//    GKO_ASSERT_MTX_NEAR(x, l({-8.5, -3.0, 6.0}), half_tol);
+//}
+
+
+TYPED_TEST(Idr, SolvesMultipleDenseSystemsUsingAdvancedApply)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using value_type = typename TestFixture::value_type;
+//    using T = value_type;
+//    auto half_tol = std::sqrt(r<value_type>::value);
+//    auto solver = this->idr_factory->generate(this->mtx);
+//    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+//    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+//    auto b = gko::initialize<Mtx>(
+//        {I<T>{-1.0, -5.0}, I<T>{3.0, 1.0}, I<T>{1.0, -2.0}}, this->exec);
+//    auto x = gko::initialize<Mtx>(
+//        {I<T>{0.5, 1.0}, I<T>{1.0, 2.0}, I<T>{2.0, 3.0}}, this->exec);
+//
+//    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+//
+//
+//    GKO_ASSERT_MTX_NEAR(x, l({{-8.5, 1.0}, {-3.0, 2.0}, {6.0, -5.0}}),
+//                        half_tol);
+//}
+
+
+// The following test-data was generated and validated with MATLAB
+TYPED_TEST(Idr, SolvesBigDenseSystemForDivergenceCheck1)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using value_type = typename TestFixture::value_type;
+//    auto half_tol = std::sqrt(r<value_type>::value);
+//    std::shared_ptr<Mtx> locmtx =
+//        gko::initialize<Mtx>({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0},
+//                              {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0},
+//                              {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0},
+//                              {60.0, -86.0, 54.0, -40.0, -93.0, 56.0},
+//                              {53.0, 94.0, -54.0, 86.0, -61.0, 4.0},
+//                              {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}},
+//                             this->exec);
+//    auto solver = this->idr_factory_precision->generate(locmtx);
+//    auto b =
+//        gko::initialize<Mtx>({0.0, -9.0, -2.0, 8.0, -5.0, -6.0}, this->exec);
+//    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+//
+//    solver->apply(b.get(), x.get());
+//
+//    GKO_ASSERT_MTX_NEAR(
+//        x,
+//        l({0.13853406350816114, -0.08147485210505287, -0.0450299311807042,
+//           -0.0051264177562865719, 0.11609654300797841, 0.1018688746740561}),
+//        half_tol * 5e-1);
+//}
+
+
+TYPED_TEST(Idr, SolvesBigDenseSystemForDivergenceCheck2)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using value_type = typename TestFixture::value_type;
+//    auto half_tol = std::sqrt(r<value_type>::value);
+//    std::shared_ptr<Mtx> locmtx =
+//        gko::initialize<Mtx>({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0},
+//                              {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0},
+//                              {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0},
+//                              {60.0, -86.0, 54.0, -40.0, -93.0, 56.0},
+//                              {53.0, 94.0, -54.0, 86.0, -61.0, 4.0},
+//                              {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}},
+//                             this->exec);
+//    auto solver = this->idr_factory_precision->generate(locmtx);
+//    auto b =
+//        gko::initialize<Mtx>({9.0, -4.0, -6.0, -10.0, 1.0, 10.0}, this->exec);
+//    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+//
+//    solver->apply(b.get(), x.get());
+//
+//    GKO_ASSERT_MTX_NEAR(
+//        x,
+//        l({0.13517641417299162, 0.75117689075221139, 0.47572853185155239,
+//           -0.50927993095367852, 0.13463333820848167, 0.23126768306576015}),
+//        half_tol * 1e-1);
+//}
+
+
+template <typename T>
+gko::remove_complex<T> infNorm(gko::matrix::Dense<T> *mat,
+                               size_t col = 0) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using std::abs;
+//    using no_cpx_t = gko::remove_complex<T>;
+//    no_cpx_t norm = 0.0;
+//    for (size_t i = 0; i < mat->get_size()[0]; ++i) {
+//        no_cpx_t absEntry = abs(mat->at(i, col));
+//        if (norm < absEntry) norm = absEntry;
+//    }
+//    return norm;
+//}
+
+
+TYPED_TEST(Idr, SolvesMultipleDenseSystemsDivergenceCheck)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using value_type = typename TestFixture::value_type;
+//    using T = value_type;
+//    std::shared_ptr<Mtx> locmtx =
+//        gko::initialize<Mtx>({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0},
+//                              {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0},
+//                              {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0},
+//                              {60.0, -86.0, 54.0, -40.0, -93.0, 56.0},
+//                              {53.0, 94.0, -54.0, 86.0, -61.0, 4.0},
+//                              {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}},
+//                             this->exec);
+//    auto solver = this->idr_factory_precision->generate(locmtx);
+//    auto b1 =
+//        gko::initialize<Mtx>({0.0, -9.0, -2.0, 8.0, -5.0, -6.0}, this->exec);
+//    auto x1 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0},
+//    this->exec); auto b2 =
+//        gko::initialize<Mtx>({9.0, -4.0, -6.0, -10.0, 1.0, 10.0}, this->exec);
+//    auto x2 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0},
+//    this->exec); auto bc = gko::initialize<Mtx>({I<T>{0., 0.}, I<T>{0., 0.},
+//    I<T>{0., 0.},
+//                                    I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.}},
+//                                   this->exec);
+//    auto xc = gko::initialize<Mtx>({I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.},
+//                                    I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.}},
+//                                   this->exec);
+//    for (size_t i = 0; i < xc->get_size()[0]; ++i) {
+//        bc->at(i, 0) = b1->at(i);
+//        bc->at(i, 1) = b2->at(i);
+//        xc->at(i, 0) = x1->at(i);
+//        xc->at(i, 1) = x2->at(i);
+//    }
+//
+//    solver->apply(b1.get(), x1.get());
+//    solver->apply(b2.get(), x2.get());
+//    solver->apply(bc.get(), xc.get());
+//    auto testMtx =
+//        gko::initialize<Mtx>({I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.},
+//                              I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.}},
+//                             this->exec);
+//
+//    for (size_t i = 0; i < testMtx->get_size()[0]; ++i) {
+//        testMtx->at(i, 0) = x1->at(i);
+//        testMtx->at(i, 1) = x2->at(i);
+//    }
+//
+//    auto alpha = gko::initialize<Mtx>({1.0}, this->exec);
+//    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+//    auto residual1 = gko::initialize<Mtx>({0.}, this->exec);
+//    residual1->copy_from(b1->clone());
+//    auto residual2 = gko::initialize<Mtx>({0.}, this->exec);
+//    residual2->copy_from(b2->clone());
+//    auto residualC = gko::initialize<Mtx>({0.}, this->exec);
+//    residualC->copy_from(bc->clone());
+//
+//    locmtx->apply(alpha.get(), x1.get(), beta.get(), residual1.get());
+//    locmtx->apply(alpha.get(), x2.get(), beta.get(), residual2.get());
+//    locmtx->apply(alpha.get(), xc.get(), beta.get(), residualC.get());
+//
+//    auto normS1 = infNorm(residual1.get());
+//    auto normS2 = infNorm(residual2.get());
+//    auto normC1 = infNorm(residualC.get(), 0);
+//    auto normC2 = infNorm(residualC.get(), 1);
+//    auto normB1 = infNorm(bc.get(), 0);
+//    auto normB2 = infNorm(bc.get(), 1);
+//
+//    // make sure that all combined solutions are as good or better than the
+//    // single solutions
+//    ASSERT_LE(normC1 / normB1, normS1 / normB1 + r<value_type>::value * 1e2);
+//    ASSERT_LE(normC2 / normB2, normS2 / normB2 + r<value_type>::value * 1e2);
+//
+//    // Not sure if this is necessary, the assertions above should cover what
+//    is
+//    // needed.
+//    GKO_ASSERT_MTX_NEAR(xc, testMtx, r<value_type>::value);
+//}
+
+
+TYPED_TEST(Idr, SolvesTransposedDenseSystem)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using value_type = typename TestFixture::value_type;
+//    auto half_tol = std::sqrt(r<value_type>::value);
+//    auto solver = this->idr_factory->generate(this->mtx->transpose());
+//    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+//    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+//
+//    solver->transpose()->apply(b.get(), x.get());
+//
+//    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), half_tol);
+//}
+
+
+TYPED_TEST(Idr, SolvesConjTransposedDenseSystem)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:idr): change the code imported from solver/bicgstab if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using value_type = typename TestFixture::value_type;
+//    auto half_tol = std::sqrt(r<value_type>::value);
+//    auto solver = this->idr_factory->generate(this->mtx->conj_transpose());
+//    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+//    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+//
+//    solver->conj_transpose()->apply(b.get(), x.get());
+//
+//    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), half_tol);
+//}
+
+
+}  // namespace

From 6362b18cefb1a21650a5308653b01e2104d5582e Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Tue, 11 Aug 2020 18:27:51 +0200
Subject: [PATCH 0135/1008] Add reference kernels

---
 core/device_hooks/common_kernels.inc.cpp |   9 +-
 core/solver/idr.cpp                      | 119 +++++++++--
 core/solver/idr_kernels.hpp              |  75 ++++---
 cuda/solver/idr_kernels.cu               |  51 +++--
 hip/solver/idr_kernels.hip.cpp           |  51 +++--
 omp/solver/idr_kernels.cpp               |  51 +++--
 reference/solver/idr_kernels.cpp         | 246 +++++++++++++++--------
 reference/test/solver/idr_kernels.cpp    | 179 +++++++++--------
 8 files changed, 491 insertions(+), 290 deletions(-)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 8321d9ea70b..8525ec0a072 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -468,6 +468,11 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_FINALIZE_KERNEL);
 // TODO (script:idr): adapt this block as needed
 namespace idr {
 
+template <typename ValueType>
+GKO_DECLARE_IDR_INITIALIZE_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
+
 template <typename ValueType>
 GKO_DECLARE_IDR_STEP_1_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
@@ -484,9 +489,9 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
 template <typename ValueType>
-GKO_DECLARE_IDR_STEP_4_KERNEL(ValueType)
+GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_4_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
 
 }  // namespace idr
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index 825cc2aa502..f6751ba5113 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -38,20 +38,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/utils.hpp>
-#include "core/components/fill_array.hpp"
 
+#include "core/components/fill_array.hpp"
 #include "core/solver/idr_kernels.hpp"
 
+#include <iostream>
 
 namespace gko {
 namespace solver {
 namespace idr {
 
 
+GKO_REGISTER_OPERATION(initialize, idr::initialize);
 GKO_REGISTER_OPERATION(step_1, idr::step_1);
 GKO_REGISTER_OPERATION(step_2, idr::step_2);
 GKO_REGISTER_OPERATION(step_3, idr::step_3);
-GKO_REGISTER_OPERATION(step_4, idr::step_4);
+GKO_REGISTER_OPERATION(compute_omega, idr::compute_omega);
 GKO_REGISTER_OPERATION(fill_array, components::fill_array);
 
 
@@ -90,6 +92,7 @@ std::unique_ptr<LinOp> Idr<ValueType>::conj_transpose() const
 template <typename ValueType>
 void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 {
+    std::cout << "Starting apply\n";
     using std::swap;
     using Vector = matrix::Dense<ValueType>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
@@ -108,7 +111,6 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     const auto nrhs = dense_b->get_size()[1];
 
     auto residual = Vector::create_with_config_of(dense_b);
-    auto f = Vector::create_with_config_of(dense_b);
     auto v = Vector::create_with_config_of(dense_b);
     auto t = Vector::create_with_config_of(dense_b);
     auto preconditioned_vector = Vector::create_with_config_of(dense_b);
@@ -119,12 +121,17 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         Vector::create(exec, gko::dim<2>{problem_size, subspace_dim_ * nrhs});
     auto u =
         Vector::create(exec, gko::dim<2>{problem_size, subspace_dim_ * nrhs});
+    auto f = Vector::create(exec, gko::dim<2>{subspace_dim_, nrhs});
 
     auto c = Vector::create(exec, gko::dim<2>{subspace_dim_, nrhs});
     auto omega = Vector::create(exec, gko::dim<2>{1, nrhs});
-    gko::fill_array(exec, omega->get_values(), nrhs, one<ValueType>());
+    exec->run(
+        idr::make_fill_array(omega->get_values(), nrhs, one<ValueType>()));
+    auto rho = Vector::create_with_config_of(omega.get());
 
     auto residual_norm = NormVector::create(exec, dim<2>{1, nrhs});
+    auto tht = Vector::create(exec, dim<2>{1, nrhs});
+    auto t_norm = NormVector::create(exec, dim<2>{1, nrhs});
 
     int total_iter = -1;
 
@@ -136,35 +143,97 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                           residual.get());
     v->copy_from(residual.get());
 
+    auto stop_criterion = stop_criterion_factory_->generate(
+        system_matrix_, std::shared_ptr<const LinOp>(b, [](const LinOp *) {}),
+        x, residual.get());
+
+    exec->run(idr::make_initialize(m.get(), g.get(), &stop_status));
+
+    std::cout << "initialized M. M is: \n";
+    for (auto i = 0; i < m->get_size()[0]; i++) {
+        for (auto j = 0; j < m->get_size()[1]; j++) {
+            std::cout << m->at(i, j) << "  ";
+        }
+        std::cout << "\n";
+    }
+    residual->compute_norm2(residual_norm.get());
+    std::cout << "initial residual norm: " << residual_norm->at(0, 0) << "\n";
+
     while (true) {
         ++total_iter;
         if (stop_criterion->update()
                 .num_iterations(total_iter)
                 .residual(residual.get())
-                .residual_norm(residual_norm.get())
                 .solution(dense_x)
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
             break;
         }
 
+        std::cout << "starting iteration " << total_iter << "\n";
+
         subspace_vectors_->apply(residual.get(), f.get());
 
+        std::cout << "applied subspace matrix to residual. f is: \n";
+        for (auto i = 0; i < f->get_size()[0]; i++) {
+            std::cout << f->at(i, 0) << std::endl;
+        }
+
         for (size_type k = 0; k < subspace_dim_; k++) {
-            exec->run(make_step_1(m.get(), f.get(), c.get(), v.get(), residual.get(), g.get());
+            exec->run(idr::make_step_1(k, m.get(), f.get(), residual.get(),
+                                       g.get(), c.get(), v.get(),
+                                       &stop_status));
             // c = M \ f = (c_1, ..., c_s)^T
             // v = residual - c_k * g_k - ... - c_s * g_s
 
+            std::cout << "ran step 1. v is: \n";
+            for (auto i = 0; i < v->get_size()[0]; i++) {
+                std::cout << v->at(i, 0) << std::endl;
+            }
+
             get_preconditioner()->apply(v.get(), preconditioned_vector.get());
 
-            exec->run(make_step_2(u.get(), c.get(), preconditioned_vector.get()));
-            //u_k = omega * preconditioned_vector + c_k * u_k + ... + c_s * u_s
+            std::cout
+                << "applied preconditioner to v. preconditioned_vector is: \n";
+            for (auto i = 0; i < preconditioned_vector->get_size()[0]; i++) {
+                std::cout << preconditioned_vector->at(i, 0) << std::endl;
+            }
 
-            auto u_k = u->create_submatrix(span{0, problem_size}, span{k * subspace_dim_ * nrhs, (k + 1) * subspace_dim_ * nrhs});
-            auto g_k = g->create_submatrix(span{0, problem_size}, span{k * subspace_dim_ * nrhs, (k + 1) * subspace_dim_ * nrhs});
+            exec->run(idr::make_step_2(k, omega.get(),
+                                       preconditioned_vector.get(), c.get(),
+                                       u.get(), &stop_status));
+            // u_k = omega * preconditioned_vector + c_k * u_k + ... + c_s * u_s
+
+            std::cout << "ran step 2. u is: \n";
+            for (auto i = 0; i < u->get_size()[0]; i++) {
+                std::cout << u->at(i, k) << std::endl;
+            }
+
+            auto u_k = u->create_submatrix(span{0, problem_size},
+                                           span{k * nrhs, (k + 1) * nrhs});
+
+            std::cout << "created submatrix of u. it is: \n";
+            for (auto i = 0; i < u->get_size()[0]; i++) {
+                std::cout << u_k->at(i, k) << std::endl;
+            }
+
+            auto g_k = g->create_submatrix(span{0, problem_size},
+                                           span{k * nrhs, (k + 1) * nrhs});
+
+            std::cout << "created submatrix of g. it is: \n";
+            for (auto i = 0; i < g->get_size()[0]; i++) {
+                std::cout << g_k->at(i, k) << std::endl;
+            }
 
             system_matrix_->apply(u_k.get(), g_k.get());
 
-            exec->run(make_step_3(subspace_vectors_.get(), g.get(), u.get(), m.get(), f.get(), c.get(), residual.get(), dense_x));
+            std::cout << "applied system matrix to u_k. g_k is: \n";
+            for (auto i = 0; i < g->get_size()[0]; i++) {
+                std::cout << g_k->at(i, k) << std::endl;
+            }
+
+            exec->run(idr::make_step_3(k, subspace_vectors_.get(), g.get(),
+                                       u.get(), m.get(), f.get(),
+                                       residual.get(), dense_x, &stop_status));
             // for i = 1 to k - 1 do
             //     alpha = p^H_i * g_k / m_i,i
             //     g_k -= alpha * g_i
@@ -177,14 +246,35 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             // residual -= beta * g_k
             // dense_x += beta * u_k
             // f = (0,...,0,f_k+1 - beta * m_k+1,k,...,f_s - beta * m_s,k)
+
+            std::cout << "ran step 3. dense_x is: \n";
+            for (auto i = 0; i < dense_x->get_size()[0]; i++) {
+                std::cout << dense_x->at(i, 0) << std::endl;
+            }
+
+            std::cout << "residual is: \n";
+            for (auto i = 0; i < residual->get_size()[0]; i++) {
+                std::cout << residual->at(i, 0) << std::endl;
+            }
         }
 
         get_preconditioner()->apply(residual.get(),
                                     preconditioned_vector.get());
-        system_matrix->apply(preconditioned_vector.get(), t.get());
+        system_matrix_->apply(preconditioned_vector.get(), t.get());
+
+        t->compute_dot(residual.get(), omega.get());
+        t->compute_norm2(t_norm.get());
+        t->compute_dot(t.get(), tht.get());
+        residual->compute_norm2(residual_norm.get());
+
+        exec->run(idr::make_compute_omega(kappa_, tht.get(), t_norm.get(),
+                                          residual_norm.get(), rho.get(),
+                                          omega.get(), &stop_status));
+
+        t->scale(neg_one_op.get());
+        residual->add_scaled(omega.get(), t.get());
+        dense_x->add_scaled(omega.get(), v.get());
 
-        exec->run(make_step_4(kappa_, omega.get(), t.get(), residual.get(),
-                              residual_norm.get(), v.get(), dense_x));
         // omega = (t^H * residual) / (t^H * t)
         // rho = (t^H * residual) / (norm(t) * norm(residual))
         // if abs(rho) < kappa then
@@ -193,6 +283,7 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         // residual -= omega * t
         // dense_x += omega * v
     }
+    std::cout << "total iterations: " << total_iter << "\n";
 }
 
 
diff --git a/core/solver/idr_kernels.hpp b/core/solver/idr_kernels.hpp
index f062ad589c7..2bb0954c26a 100644
--- a/core/solver/idr_kernels.hpp
+++ b/core/solver/idr_kernels.hpp
@@ -46,50 +46,59 @@ namespace kernels {
 namespace idr {
 
 
-#define GKO_DECLARE_IDR_STEP_1_KERNEL(_type)                                  \
-    void step_1(std::shared_ptr<const DefaultExecutor> exec,                  \
-                const matrix::Dense<_type> *m, matrix::Dense<_type> *f,       \
-                const matrix::Dense<_type> *c, const matrix::Dense<_type> *v, \
-                const matrix::Dense<_type> *residual,                         \
-                const matrix::Dense<_type> *g,                                \
-                const Array<stopping_status> *stop_status)
+#define GKO_DECLARE_IDR_INITIALIZE_KERNEL(_type)                      \
+    void initialize(std::shared_ptr<const DefaultExecutor> exec,      \
+                    matrix::Dense<_type> *m, matrix::Dense<_type> *g, \
+                    Array<stopping_status> *stop_status)
+
+
+#define GKO_DECLARE_IDR_STEP_1_KERNEL(_type)                                 \
+    void step_1(                                                             \
+        std::shared_ptr<const DefaultExecutor> exec, const size_type k,      \
+        const matrix::Dense<_type> *m, const matrix::Dense<_type> *f,        \
+        const matrix::Dense<_type> *residual, const matrix::Dense<_type> *g, \
+        matrix::Dense<_type> *c, matrix::Dense<_type> *v,                    \
+        const Array<stopping_status> *stop_status)
 
 
 #define GKO_DECLARE_IDR_STEP_2_KERNEL(_type)                            \
     void step_2(std::shared_ptr<const DefaultExecutor> exec,            \
-                const matrix::Dense<_type> *u, matrix::Dense<_type> *c, \
+                const size_type k, const matrix::Dense<_type> *omega,   \
                 const matrix::Dense<_type> *preconditioned_vector,      \
+                const matrix::Dense<_type> *c, matrix::Dense<_type> *u, \
                 const Array<stopping_status> *stop_status)
 
 
-#define GKO_DECLARE_IDR_STEP_3_KERNEL(_type)                                  \
-    void step_3(std::shared_ptr<const DefaultExecutor> exec,                  \
-                matrix::Dense<_type> *p, matrix::Dense<_type> *g,             \
-                const matrix::Dense<_type> *u, const matrix::Dense<_type> *m, \
-                const matrix::Dense<_type> *f, const matrix::Dense<_type> *c, \
-                const matrix::Dense<_type> *residual,                         \
-                const matrix::Dense<_type> *x,                                \
+#define GKO_DECLARE_IDR_STEP_3_KERNEL(_type)                             \
+    void step_3(std::shared_ptr<const DefaultExecutor> exec,             \
+                const size_type k, const matrix::Dense<_type> *p,        \
+                matrix::Dense<_type> *g, matrix::Dense<_type> *u,        \
+                matrix::Dense<_type> *m, matrix::Dense<_type> *f,        \
+                matrix::Dense<_type> *residual, matrix::Dense<_type> *x, \
                 const Array<stopping_status> *stop_status)
 
 
-#define GKO_DECLARE_IDR_STEP_4_KERNEL(_type)                                \
-    void step_4(                                                            \
-        std::shared_ptr<const DefaultExecutor> exec, const _type kappa,     \
-        matrix::Dense<_type> *omega, const matrix::Dense<_type> *t,         \
-        const matrix::Dense<_type> *residual,                               \
-        matrix::Dense<_type> *residual_norm, const matrix::Dense<_type> *v, \
-        matrix::Dense<_type> *x, Array<stopping_status> *stop_status)
-
-
-#define GKO_DECLARE_ALL_AS_TEMPLATES          \
-    template <typename ValueType>             \
-    GKO_DECLARE_IDR_STEP_1_KERNEL(ValueType); \
-    template <typename ValueType>             \
-    GKO_DECLARE_IDR_STEP_2_KERNEL(ValueType); \
-    template <typename ValueType>             \
-    GKO_DECLARE_IDR_STEP_3_KERNEL(ValueType); \
-    template <typename ValueType>             \
-    GKO_DECLARE_IDR_STEP_4_KERNEL(ValueType)
+#define GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL(_type)                         \
+    void compute_omega(                                                     \
+        std::shared_ptr<const DefaultExecutor> exec,                        \
+        const remove_complex<_type> kappa, const matrix::Dense<_type> *tht, \
+        const matrix::Dense<remove_complex<_type>> *t_norm,                 \
+        const matrix::Dense<remove_complex<_type>> *residual_norm,          \
+        matrix::Dense<_type> *rho, matrix::Dense<_type> *omega,             \
+        const Array<stopping_status> *stop_status)
+
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES              \
+    template <typename ValueType>                 \
+    GKO_DECLARE_IDR_INITIALIZE_KERNEL(ValueType); \
+    template <typename ValueType>                 \
+    GKO_DECLARE_IDR_STEP_1_KERNEL(ValueType);     \
+    template <typename ValueType>                 \
+    GKO_DECLARE_IDR_STEP_2_KERNEL(ValueType);     \
+    template <typename ValueType>                 \
+    GKO_DECLARE_IDR_STEP_3_KERNEL(ValueType);     \
+    template <typename ValueType>                 \
+    GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL(ValueType)
 
 
 }  // namespace idr
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index f63b10bd88f..70e760bcac0 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -57,12 +57,20 @@ constexpr int default_block_size = 512;
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const CudaExecutor> exec,
-            const matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
-            const matrix::Dense<ValueType> *c,
-            const matrix::Dense<ValueType> *v,
+void initialize(std::shared_ptr<const CudaExecutor> exec,
+                matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *g,
+                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const CudaExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *m,
+            const matrix::Dense<ValueType> *f,
             const matrix::Dense<ValueType> *residual,
-            const matrix::Dense<ValueType> *g,
+            const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *c,
+            matrix::Dense<ValueType> *v,
             const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:idr): change the code imported from solver/bicgstab if needed
@@ -85,9 +93,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const CudaExecutor> exec,
-            const matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *c,
+void step_2(std::shared_ptr<const CudaExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *omega,
             const matrix::Dense<ValueType> *preconditioned_vector,
+            const matrix::Dense<ValueType> *c, matrix::Dense<ValueType> *u,
             const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:idr): change the code imported from solver/bicgstab if needed
@@ -109,14 +118,11 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 
 
 template <typename ValueType>
-void step_3(std::shared_ptr<const CudaExecutor> exec,
-            matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
-            const matrix::Dense<ValueType> *u,
-            const matrix::Dense<ValueType> *m,
-            const matrix::Dense<ValueType> *f,
-            const matrix::Dense<ValueType> *c,
-            const matrix::Dense<ValueType> *residual,
-            const matrix::Dense<ValueType> *x,
+void step_3(std::shared_ptr<const CudaExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
+            matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
+            matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *residual,
+            matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:idr): change the code imported from solver/bicgstab if needed
@@ -142,12 +148,13 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
 
 template <typename ValueType>
-void step_4(std::shared_ptr<const CudaExecutor> exec, const ValueType kappa,
-            matrix::Dense<ValueType> *omega, const matrix::Dense<ValueType> *t,
-            const matrix::Dense<ValueType> *residual,
-            matrix::Dense<ValueType> *residual_norm,
-            const matrix::Dense<ValueType> *v, matrix::Dense<ValueType> *x,
-            Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+void compute_omega(
+    std::shared_ptr<const CudaExecutor> exec,
+    const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
+    const matrix::Dense<remove_complex<ValueType>> *t_norm,
+    const matrix::Dense<remove_complex<ValueType>> *residual_norm,
+    matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *omega,
+    const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:idr): change the code imported from solver/bicgstab if needed
 //    const dim3 block_size(default_block_size, 1, 1);
@@ -161,7 +168,7 @@ void step_4(std::shared_ptr<const CudaExecutor> exec, const ValueType kappa,
 //        as_cuda_type(stop_status->get_data()));
 //}
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_4_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
 
 }  // namespace idr
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index da3ca49e590..f8b801142b7 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -60,12 +60,20 @@ constexpr int default_block_size = 512;
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const HipExecutor> exec,
-            const matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
-            const matrix::Dense<ValueType> *c,
-            const matrix::Dense<ValueType> *v,
+void initialize(std::shared_ptr<const HipExecutor> exec,
+                matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *g,
+                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const HipExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *m,
+            const matrix::Dense<ValueType> *f,
             const matrix::Dense<ValueType> *residual,
-            const matrix::Dense<ValueType> *g,
+            const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *c,
+            matrix::Dense<ValueType> *v,
             const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:idr): change the code imported from solver/bicgstab if needed
@@ -88,9 +96,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const HipExecutor> exec,
-            const matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *c,
+void step_2(std::shared_ptr<const HipExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *omega,
             const matrix::Dense<ValueType> *preconditioned_vector,
+            const matrix::Dense<ValueType> *c, matrix::Dense<ValueType> *u,
             const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:idr): change the code imported from solver/bicgstab if needed
@@ -112,14 +121,11 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 
 
 template <typename ValueType>
-void step_3(std::shared_ptr<const HipExecutor> exec,
-            matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
-            const matrix::Dense<ValueType> *u,
-            const matrix::Dense<ValueType> *m,
-            const matrix::Dense<ValueType> *f,
-            const matrix::Dense<ValueType> *c,
-            const matrix::Dense<ValueType> *residual,
-            const matrix::Dense<ValueType> *x,
+void step_3(std::shared_ptr<const HipExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
+            matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
+            matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *residual,
+            matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:idr): change the code imported from solver/bicgstab if needed
@@ -145,12 +151,13 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
 
 template <typename ValueType>
-void step_4(std::shared_ptr<const HipExecutor> exec, const ValueType kappa,
-            matrix::Dense<ValueType> *omega, const matrix::Dense<ValueType> *t,
-            const matrix::Dense<ValueType> *residual,
-            matrix::Dense<ValueType> *residual_norm,
-            const matrix::Dense<ValueType> *v, matrix::Dense<ValueType> *x,
-            Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+void compute_omega(
+    std::shared_ptr<const HipExecutor> exec,
+    const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
+    const matrix::Dense<remove_complex<ValueType>> *t_norm,
+    const matrix::Dense<remove_complex<ValueType>> *residual_norm,
+    matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *omega,
+    const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:idr): change the code imported from solver/bicgstab if needed
 //    const dim3 block_size(default_block_size, 1, 1);
@@ -164,7 +171,7 @@ void step_4(std::shared_ptr<const HipExecutor> exec, const ValueType kappa,
 //        as_cuda_type(stop_status->get_data()));
 //}
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_4_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
 
 }  // namespace idr
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index 824a8f8894b..ae83730fa1d 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -56,12 +56,20 @@ namespace idr {
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const OmpExecutor> exec,
-            const matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
-            const matrix::Dense<ValueType> *c,
-            const matrix::Dense<ValueType> *v,
+void initialize(std::shared_ptr<const OmpExecutor> exec,
+                matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *g,
+                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const OmpExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *m,
+            const matrix::Dense<ValueType> *f,
             const matrix::Dense<ValueType> *residual,
-            const matrix::Dense<ValueType> *g,
+            const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *c,
+            matrix::Dense<ValueType> *v,
             const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:idr): change the code imported from solver/bicgstab if needed
@@ -84,9 +92,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const OmpExecutor> exec,
-            const matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *c,
+void step_2(std::shared_ptr<const OmpExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *omega,
             const matrix::Dense<ValueType> *preconditioned_vector,
+            const matrix::Dense<ValueType> *c, matrix::Dense<ValueType> *u,
             const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:idr): change the code imported from solver/bicgstab if needed
@@ -108,14 +117,11 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 
 
 template <typename ValueType>
-void step_3(std::shared_ptr<const OmpExecutor> exec,
-            matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
-            const matrix::Dense<ValueType> *u,
-            const matrix::Dense<ValueType> *m,
-            const matrix::Dense<ValueType> *f,
-            const matrix::Dense<ValueType> *c,
-            const matrix::Dense<ValueType> *residual,
-            const matrix::Dense<ValueType> *x,
+void step_3(std::shared_ptr<const OmpExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
+            matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
+            matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *residual,
+            matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:idr): change the code imported from solver/bicgstab if needed
@@ -141,12 +147,13 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
 
 template <typename ValueType>
-void step_4(std::shared_ptr<const OmpExecutor> exec, const ValueType kappa,
-            matrix::Dense<ValueType> *omega, const matrix::Dense<ValueType> *t,
-            const matrix::Dense<ValueType> *residual,
-            matrix::Dense<ValueType> *residual_norm,
-            const matrix::Dense<ValueType> *v, matrix::Dense<ValueType> *x,
-            Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+void compute_omega(
+    std::shared_ptr<const OmpExecutor> exec,
+    const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
+    const matrix::Dense<remove_complex<ValueType>> *t_norm,
+    const matrix::Dense<remove_complex<ValueType>> *residual_norm,
+    matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *omega,
+    const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:idr): change the code imported from solver/bicgstab if needed
 //    const dim3 block_size(default_block_size, 1, 1);
@@ -160,7 +167,7 @@ void step_4(std::shared_ptr<const OmpExecutor> exec, const ValueType kappa,
 //        as_cuda_type(stop_status->get_data()));
 //}
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_4_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
 
 }  // namespace idr
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index fd72b7f814d..d36ce07d905 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -53,112 +53,180 @@ namespace idr {
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const ReferenceExecutor> exec,
-            const matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
-            const matrix::Dense<ValueType> *c,
-            const matrix::Dense<ValueType> *v,
+void initialize(std::shared_ptr<const ReferenceExecutor> exec,
+                matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *g,
+                Array<stopping_status> *stop_status)
+{
+    const auto nrhs = m->get_size()[1] / m->get_size()[0];
+    for (size_type i = 0; i < nrhs; i++) {
+        stop_status->get_data()[i].reset();
+    }
+
+    for (size_type row = 0; row < m->get_size()[0]; row++) {
+        for (size_type col = 0; col < m->get_size()[1]; col++) {
+            m->at(row, col) =
+                (row == col / nrhs) ? one<ValueType>() : zero<ValueType>();
+        }
+    }
+
+    for (auto row = 0; row < g->get_size()[0]; row++) {
+        for (auto col = 0; col < g->get_size()[1]; col++) {
+            g->at(row, col) = zero<ValueType>();
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const ReferenceExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *m,
+            const matrix::Dense<ValueType> *f,
             const matrix::Dense<ValueType> *residual,
-            const matrix::Dense<ValueType> *g,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-//
-//    step_1_kernel<<<grid_size, block_size, 0, 0>>>(
-//        r->get_size()[0], r->get_size()[1], r->get_stride(),
-//        as_cuda_type(r->get_const_values()), as_cuda_type(p->get_values()),
-//        as_cuda_type(v->get_const_values()),
-//        as_cuda_type(rho->get_const_values()),
-//        as_cuda_type(prev_rho->get_const_values()),
-//        as_cuda_type(alpha->get_const_values()),
-//        as_cuda_type(omega->get_const_values()),
-//        as_cuda_type(stop_status->get_const_data()));
-//}
+            const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *c,
+            matrix::Dense<ValueType> *v,
+            const Array<stopping_status> *stop_status)
+{
+    const auto m_size = m->get_size();
+    const auto nrhs = f->get_size()[1];
+
+    for (size_type i = 0; i < nrhs; i++) {
+        if (stop_status->get_const_data()[0].has_stopped()) {
+            continue;
+        }
+
+        // Compute c = M \ f
+        for (size_type row = 0; row < m_size[0]; row++) {
+            auto temp = f->at(row, i);
+            for (size_type col = 0; col < row; col++) {
+                temp -= m->at(row, col * nrhs + i) * c->at(col, i);
+            }
+            c->at(row, i) = temp / m->at(row, row * nrhs + i);
+        }
+
+        // v = residual - c_k * g_k - ... - c_s * g_s
+        for (size_type row = 0; row < v->get_size()[0]; row++) {
+            auto temp = residual->at(row, i);
+            for (size_type j = k; j < m->get_size()[0]; j++) {
+                temp -= c->at(j, i) * g->at(row, j * nrhs + i);
+            }
+            v->at(row, i) = temp;
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const ReferenceExecutor> exec,
-            const matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *c,
+void step_2(std::shared_ptr<const ReferenceExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *omega,
             const matrix::Dense<ValueType> *preconditioned_vector,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-//
-//    step_2_kernel<<<grid_size, block_size, 0, 0>>>(
-//        r->get_size()[0], r->get_size()[1], r->get_stride(),
-//        as_cuda_type(r->get_const_values()), as_cuda_type(s->get_values()),
-//        as_cuda_type(v->get_const_values()),
-//        as_cuda_type(rho->get_const_values()),
-//        as_cuda_type(alpha->get_values()),
-//        as_cuda_type(beta->get_const_values()),
-//        as_cuda_type(stop_status->get_const_data()));
-//}
+            const matrix::Dense<ValueType> *c, matrix::Dense<ValueType> *u,
+            const Array<stopping_status> *stop_status)
+{
+    const auto nrhs = omega->get_size()[1];
+    for (size_type i = 0; i < nrhs; i++) {
+        if (stop_status->get_const_data()[0].has_stopped()) {
+            continue;
+        }
+
+        for (size_type row = 0; row < u->get_size()[0]; row++) {
+            auto temp = omega->at(0, i) * preconditioned_vector->at(row, i);
+            for (size_type j = k; j < c->get_size()[0]; j++) {
+                temp += c->at(j, i) * u->at(row, j * nrhs + i);
+            }
+            u->at(row, k * nrhs + i) = temp;
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 
 
 template <typename ValueType>
-void step_3(std::shared_ptr<const ReferenceExecutor> exec,
-            matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
-            const matrix::Dense<ValueType> *u,
-            const matrix::Dense<ValueType> *m,
-            const matrix::Dense<ValueType> *f,
-            const matrix::Dense<ValueType> *c,
-            const matrix::Dense<ValueType> *residual,
-            const matrix::Dense<ValueType> *x,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-//
-//    step_3_kernel<<<grid_size, block_size, 0, 0>>>(
-//        r->get_size()[0], r->get_size()[1], r->get_stride(), x->get_stride(),
-//        as_cuda_type(x->get_values()), as_cuda_type(r->get_values()),
-//        as_cuda_type(s->get_const_values()),
-//        as_cuda_type(t->get_const_values()),
-//        as_cuda_type(y->get_const_values()),
-//        as_cuda_type(z->get_const_values()),
-//        as_cuda_type(alpha->get_const_values()),
-//        as_cuda_type(beta->get_const_values()),
-//        as_cuda_type(gamma->get_const_values()),
-//        as_cuda_type(omega->get_values()),
-//        as_cuda_type(stop_status->get_const_data()));
-//}
+void step_3(std::shared_ptr<const ReferenceExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
+            matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
+            matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *residual,
+            matrix::Dense<ValueType> *x,
+            const Array<stopping_status> *stop_status)
+{
+    const auto nrhs = x->get_size()[1];
+
+    for (size_type i = 0; i < nrhs; i++) {
+        if (stop_status->get_const_data()[0].has_stopped()) {
+            continue;
+        }
+
+        for (size_type j = 0; j < k; j++) {
+            auto alpha = zero<ValueType>();
+            for (size_type ind = 0; ind < p->get_size()[1]; ind++) {
+                alpha += p->at(j, ind) * g->at(ind, k * nrhs + i);
+            }
+            alpha /= m->at(j, j * nrhs + i);
+            for (size_type row = 0; row < g->get_size()[0]; row++) {
+                g->at(row, k * nrhs + i) -= alpha * g->at(row, j * nrhs + i);
+                u->at(row, k * nrhs + i) -= alpha * u->at(row, j * nrhs + i);
+            }
+        }
+
+        for (size_type j = k; j < m->get_size()[0]; j++) {
+            auto temp = zero<ValueType>();
+            for (size_type ind = 0; ind < p->get_size()[1]; ind++) {
+                temp += p->at(j, ind) * g->at(ind, k * nrhs + i);
+            }
+            m->at(j, k * nrhs + i) = temp;
+        }
+
+        auto beta = f->at(k, i) / m->at(k, k * nrhs + i);
+
+        for (size_type row = 0; row < g->get_size()[0]; row++) {
+            residual->at(row, i) -= beta * g->at(row, k * nrhs + i);
+            x->at(row, i) += beta * u->at(row, k * nrhs + i);
+        }
+
+        if (k + 1 < f->get_size()[0]) {
+            f->at(k, i) = zero<ValueType>();
+            for (size_type j = k + 1; j < f->get_size()[0]; j++) {
+                f->at(j, i) -= beta * m->at(j, k * nrhs + i);
+            }
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
 
 template <typename ValueType>
-void step_4(std::shared_ptr<const ReferenceExecutor> exec,
-            const ValueType kappa, matrix::Dense<ValueType> *omega,
-            const matrix::Dense<ValueType> *t,
-            const matrix::Dense<ValueType> *residual,
-            matrix::Dense<ValueType> *residual_norm,
-            const matrix::Dense<ValueType> *v, matrix::Dense<ValueType> *x,
-            Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(y->get_size()[0] * y->get_stride(), block_size.x), 1, 1);
-//
-//    finalize_kernel<<<grid_size, block_size, 0, 0>>>(
-//        y->get_size()[0], y->get_size()[1], y->get_stride(), x->get_stride(),
-//        as_cuda_type(x->get_values()), as_cuda_type(y->get_const_values()),
-//        as_cuda_type(alpha->get_const_values()),
-//        as_cuda_type(stop_status->get_data()));
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_4_KERNEL);
+void compute_omega(
+    std::shared_ptr<const ReferenceExecutor> exec,
+    const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
+    const matrix::Dense<remove_complex<ValueType>> *t_norm,
+    const matrix::Dense<remove_complex<ValueType>> *residual_norm,
+    matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *omega,
+    const Array<stopping_status> *stop_status)
+{
+    for (size_type i = 0; i < omega->get_size()[1]; i++) {
+        if (stop_status->get_const_data()[0].has_stopped()) {
+            continue;
+        }
+
+        auto thr = omega->at(0, i);
+        auto normt = t_norm->at(0, i);
+        omega->at(0, i) /= tht->at(0, i);
+        rho->at(0, i) = thr / (normt * residual_norm->at(0, i));
+
+        auto absrho = abs(rho->at(0, i));
+
+        if (absrho < kappa) {
+            omega->at(0, i) *= kappa / absrho;
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
 
 }  // namespace idr
diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp
index 3b5d800e54c..e89a590fd11 100644
--- a/reference/test/solver/idr_kernels.cpp
+++ b/reference/test/solver/idr_kernels.cpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <gtest/gtest.h>
 
+#include <iostream>
 
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
@@ -72,6 +73,7 @@ class Idr : public ::testing::Test {
                       gko::stop::ResidualNormReduction<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
+                  .with_subspace_dim(4u)
                   .on(exec)),
           idr_factory_precision(
               Solver::build()
@@ -84,6 +86,7 @@ class Idr : public ::testing::Test {
                       gko::stop::ResidualNormReduction<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
+                  .with_subspace_dim(4u)
                   .on(exec))
     {}
 
@@ -97,41 +100,37 @@ TYPED_TEST_CASE(Idr, gko::test::ValueTypes);
 
 
 TYPED_TEST(Idr, SolvesDenseSystem)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using value_type = typename TestFixture::value_type;
-//    auto half_tol = std::sqrt(r<value_type>::value);
-//    auto solver = this->idr_factory->generate(this->mtx);
-//    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
-//    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
-//
-//    solver->apply(b.get(), x.get());
-//
-//    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), half_tol);
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto half_tol = std::sqrt(r<value_type>::value);
+    auto solver = this->idr_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), half_tol);
+}
 
 
 TYPED_TEST(Idr, SolvesMultipleDenseSystems)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using value_type = typename TestFixture::value_type;
-//    using T = value_type;
-//    auto half_tol = std::sqrt(r<value_type>::value);
-//    auto solver = this->idr_factory->generate(this->mtx);
-//    auto b = gko::initialize<Mtx>(
-//        {I<T>{-1.0, -5.0}, I<T>{3.0, 1.0}, I<T>{1.0, -2.0}}, this->exec);
-//    auto x = gko::initialize<Mtx>(
-//        {I<T>{0.0, 0.0}, I<T>{0.0, 0.0}, I<T>{0.0, 0.0}}, this->exec);
-//
-//    solver->apply(b.get(), x.get());
-//
-//    GKO_ASSERT_MTX_NEAR(x, l({{-4.0, 1.0}, {-1.0, 2.0}, {4.0, -1.0}}),
-//                        half_tol);
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using T = value_type;
+    auto half_tol = std::sqrt(r<value_type>::value);
+    auto solver = this->idr_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {I<T>{-1.0, -5.0}, I<T>{3.0, 1.0}, I<T>{1.0, -2.0}}, this->exec);
+    auto x = gko::initialize<Mtx>(
+        {I<T>{0.0, 0.0}, I<T>{0.0, 0.0}, I<T>{0.0, 0.0}}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({{-4.0, 1.0}, {-1.0, 2.0}, {4.0, -1.0}}),
+                        half_tol);
+}
 
 
 TYPED_TEST(Idr, SolvesDenseSystemUsingAdvancedApply)
@@ -180,63 +179,71 @@ GKO_NOT_IMPLEMENTED;
 
 // The following test-data was generated and validated with MATLAB
 TYPED_TEST(Idr, SolvesBigDenseSystemForDivergenceCheck1)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using value_type = typename TestFixture::value_type;
-//    auto half_tol = std::sqrt(r<value_type>::value);
-//    std::shared_ptr<Mtx> locmtx =
-//        gko::initialize<Mtx>({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0},
-//                              {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0},
-//                              {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0},
-//                              {60.0, -86.0, 54.0, -40.0, -93.0, 56.0},
-//                              {53.0, 94.0, -54.0, 86.0, -61.0, 4.0},
-//                              {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}},
-//                             this->exec);
-//    auto solver = this->idr_factory_precision->generate(locmtx);
-//    auto b =
-//        gko::initialize<Mtx>({0.0, -9.0, -2.0, 8.0, -5.0, -6.0}, this->exec);
-//    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
-//
-//    solver->apply(b.get(), x.get());
-//
-//    GKO_ASSERT_MTX_NEAR(
-//        x,
-//        l({0.13853406350816114, -0.08147485210505287, -0.0450299311807042,
-//           -0.0051264177562865719, 0.11609654300797841, 0.1018688746740561}),
-//        half_tol * 5e-1);
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto half_tol = std::sqrt(r<value_type>::value);
+    std::shared_ptr<Mtx> locmtx =
+        gko::initialize<Mtx>({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0},
+                              {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0},
+                              {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0},
+                              {60.0, -86.0, 54.0, -40.0, -93.0, 56.0},
+                              {53.0, 94.0, -54.0, 86.0, -61.0, 4.0},
+                              {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}},
+                             this->exec);
+    auto solver = this->idr_factory_precision->generate(locmtx);
+    auto b =
+        gko::initialize<Mtx>({0.0, -9.0, -2.0, 8.0, -5.0, -6.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+
+    auto one_op = gko::initialize<gko::matrix::Dense<value_type>>(
+        {gko::one<value_type>()}, this->exec);
+    auto neg_one_op = gko::initialize<gko::matrix::Dense<value_type>>(
+        {-gko::one<value_type>()}, this->exec);
+    auto resnorm = gko::matrix::Dense<gko::remove_complex<value_type>>::create(
+        this->exec, gko::dim<2>{1, 1});
+    locmtx->apply(neg_one_op.get(), x.get(), one_op.get(), b.get());
+    b->compute_norm2(resnorm.get());
+
+    std::cout << "RESIDUAL NORM: " << resnorm->at(0, 0) << "\n";
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({0.13853406350816114, -0.08147485210505287, -0.0450299311807042,
+           -0.0051264177562865719, 0.11609654300797841, 0.1018688746740561}),
+        half_tol * 5e-1);
+}
 
 
 TYPED_TEST(Idr, SolvesBigDenseSystemForDivergenceCheck2)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using value_type = typename TestFixture::value_type;
-//    auto half_tol = std::sqrt(r<value_type>::value);
-//    std::shared_ptr<Mtx> locmtx =
-//        gko::initialize<Mtx>({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0},
-//                              {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0},
-//                              {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0},
-//                              {60.0, -86.0, 54.0, -40.0, -93.0, 56.0},
-//                              {53.0, 94.0, -54.0, 86.0, -61.0, 4.0},
-//                              {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}},
-//                             this->exec);
-//    auto solver = this->idr_factory_precision->generate(locmtx);
-//    auto b =
-//        gko::initialize<Mtx>({9.0, -4.0, -6.0, -10.0, 1.0, 10.0}, this->exec);
-//    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
-//
-//    solver->apply(b.get(), x.get());
-//
-//    GKO_ASSERT_MTX_NEAR(
-//        x,
-//        l({0.13517641417299162, 0.75117689075221139, 0.47572853185155239,
-//           -0.50927993095367852, 0.13463333820848167, 0.23126768306576015}),
-//        half_tol * 1e-1);
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto half_tol = std::sqrt(r<value_type>::value);
+    std::shared_ptr<Mtx> locmtx =
+        gko::initialize<Mtx>({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0},
+                              {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0},
+                              {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0},
+                              {60.0, -86.0, 54.0, -40.0, -93.0, 56.0},
+                              {53.0, 94.0, -54.0, 86.0, -61.0, 4.0},
+                              {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}},
+                             this->exec);
+    auto solver = this->idr_factory_precision->generate(locmtx);
+    auto b =
+        gko::initialize<Mtx>({9.0, -4.0, -6.0, -10.0, 1.0, 10.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({0.13517641417299162, 0.75117689075221139, 0.47572853185155239,
+           -0.50927993095367852, 0.13463333820848167, 0.23126768306576015}),
+        half_tol * 1e-1);
+}
 
 
 template <typename T>

From 8706341178c138d38ead58bc52f61d02cd82965d Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Fri, 14 Aug 2020 17:33:51 +0200
Subject: [PATCH 0136/1008] Finish reference implementation, add reference
 tests and omp implementation

---
 core/solver/idr.cpp                   | 120 +++-------
 core/solver/idr_kernels.hpp           |   6 +-
 cuda/solver/idr_kernels.cu            |   2 +-
 hip/solver/idr_kernels.hip.cpp        |   2 +-
 include/ginkgo/core/solver/idr.hpp    |   2 +-
 omp/solver/idr_kernels.cpp            | 236 ++++++++++++++------
 reference/solver/idr_kernels.cpp      |  86 +++++---
 reference/test/solver/idr_kernels.cpp | 304 ++++++++++++--------------
 8 files changed, 408 insertions(+), 350 deletions(-)

diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index f6751ba5113..e2d5cfc0fce 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -61,38 +61,34 @@ GKO_REGISTER_OPERATION(fill_array, components::fill_array);
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Idr<ValueType>::transpose() const GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    return build()
-//        .with_generated_preconditioner(
-//            share(as<Transposable>(this->get_preconditioner())->transpose()))
-//        .with_criteria(this->stop_criterion_factory_)
-//        .on(this->get_executor())
-//        ->generate(
-//            share(as<Transposable>(this->get_system_matrix())->transpose()));
-//}
+std::unique_ptr<LinOp> Idr<ValueType>::transpose() const
+{
+    return build()
+        .with_generated_preconditioner(
+            share(as<Transposable>(this->get_preconditioner())->transpose()))
+        .with_criteria(this->stop_criterion_factory_)
+        .on(this->get_executor())
+        ->generate(
+            share(as<Transposable>(this->get_system_matrix())->transpose()));
+}
 
 
 template <typename ValueType>
 std::unique_ptr<LinOp> Idr<ValueType>::conj_transpose() const
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    return build()
-//        .with_generated_preconditioner(share(
-//            as<Transposable>(this->get_preconditioner())->conj_transpose()))
-//        .with_criteria(this->stop_criterion_factory_)
-//        .on(this->get_executor())
-//        ->generate(share(
-//            as<Transposable>(this->get_system_matrix())->conj_transpose()));
-//}
+{
+    return build()
+        .with_generated_preconditioner(share(
+            as<Transposable>(this->get_preconditioner())->conj_transpose()))
+        .with_criteria(this->stop_criterion_factory_)
+        .on(this->get_executor())
+        ->generate(share(
+            as<Transposable>(this->get_system_matrix())->conj_transpose()));
+}
 
 
 template <typename ValueType>
 void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    std::cout << "Starting apply\n";
     using std::swap;
     using Vector = matrix::Dense<ValueType>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
@@ -119,8 +115,12 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         Vector::create(exec, gko::dim<2>{subspace_dim_, subspace_dim_ * nrhs});
     auto g =
         Vector::create(exec, gko::dim<2>{problem_size, subspace_dim_ * nrhs});
+    exec->run(idr::make_fill_array(
+        g->get_values(), problem_size * g->get_stride(), zero<ValueType>()));
     auto u =
         Vector::create(exec, gko::dim<2>{problem_size, subspace_dim_ * nrhs});
+    exec->run(idr::make_fill_array(
+        u->get_values(), problem_size * u->get_stride(), zero<ValueType>()));
     auto f = Vector::create(exec, gko::dim<2>{subspace_dim_, nrhs});
 
     auto c = Vector::create(exec, gko::dim<2>{subspace_dim_, nrhs});
@@ -128,6 +128,7 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     exec->run(
         idr::make_fill_array(omega->get_values(), nrhs, one<ValueType>()));
     auto rho = Vector::create_with_config_of(omega.get());
+    auto alpha = Vector::create(exec, gko::dim<2>{subspace_dim_, nrhs});
 
     auto residual_norm = NormVector::create(exec, dim<2>{1, nrhs});
     auto tht = Vector::create(exec, dim<2>{1, nrhs});
@@ -147,17 +148,8 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         system_matrix_, std::shared_ptr<const LinOp>(b, [](const LinOp *) {}),
         x, residual.get());
 
-    exec->run(idr::make_initialize(m.get(), g.get(), &stop_status));
-
-    std::cout << "initialized M. M is: \n";
-    for (auto i = 0; i < m->get_size()[0]; i++) {
-        for (auto j = 0; j < m->get_size()[1]; j++) {
-            std::cout << m->at(i, j) << "  ";
-        }
-        std::cout << "\n";
-    }
+    exec->run(idr::make_initialize(m.get(), &stop_status));
     residual->compute_norm2(residual_norm.get());
-    std::cout << "initial residual norm: " << residual_norm->at(0, 0) << "\n";
 
     while (true) {
         ++total_iter;
@@ -169,15 +161,8 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             break;
         }
 
-        std::cout << "starting iteration " << total_iter << "\n";
-
         subspace_vectors_->apply(residual.get(), f.get());
 
-        std::cout << "applied subspace matrix to residual. f is: \n";
-        for (auto i = 0; i < f->get_size()[0]; i++) {
-            std::cout << f->at(i, 0) << std::endl;
-        }
-
         for (size_type k = 0; k < subspace_dim_; k++) {
             exec->run(idr::make_step_1(k, m.get(), f.get(), residual.get(),
                                        g.get(), c.get(), v.get(),
@@ -185,52 +170,21 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             // c = M \ f = (c_1, ..., c_s)^T
             // v = residual - c_k * g_k - ... - c_s * g_s
 
-            std::cout << "ran step 1. v is: \n";
-            for (auto i = 0; i < v->get_size()[0]; i++) {
-                std::cout << v->at(i, 0) << std::endl;
-            }
-
             get_preconditioner()->apply(v.get(), preconditioned_vector.get());
 
-            std::cout
-                << "applied preconditioner to v. preconditioned_vector is: \n";
-            for (auto i = 0; i < preconditioned_vector->get_size()[0]; i++) {
-                std::cout << preconditioned_vector->at(i, 0) << std::endl;
-            }
-
             exec->run(idr::make_step_2(k, omega.get(),
                                        preconditioned_vector.get(), c.get(),
                                        u.get(), &stop_status));
             // u_k = omega * preconditioned_vector + c_k * u_k + ... + c_s * u_s
 
-            std::cout << "ran step 2. u is: \n";
-            for (auto i = 0; i < u->get_size()[0]; i++) {
-                std::cout << u->at(i, k) << std::endl;
-            }
-
             auto u_k = u->create_submatrix(span{0, problem_size},
                                            span{k * nrhs, (k + 1) * nrhs});
 
-            std::cout << "created submatrix of u. it is: \n";
-            for (auto i = 0; i < u->get_size()[0]; i++) {
-                std::cout << u_k->at(i, k) << std::endl;
-            }
-
             auto g_k = g->create_submatrix(span{0, problem_size},
                                            span{k * nrhs, (k + 1) * nrhs});
 
-            std::cout << "created submatrix of g. it is: \n";
-            for (auto i = 0; i < g->get_size()[0]; i++) {
-                std::cout << g_k->at(i, k) << std::endl;
-            }
-
             system_matrix_->apply(u_k.get(), g_k.get());
 
-            std::cout << "applied system matrix to u_k. g_k is: \n";
-            for (auto i = 0; i < g->get_size()[0]; i++) {
-                std::cout << g_k->at(i, k) << std::endl;
-            }
-
             exec->run(idr::make_step_3(k, subspace_vectors_.get(), g.get(),
                                        u.get(), m.get(), f.get(),
                                        residual.get(), dense_x, &stop_status));
@@ -246,16 +200,6 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             // residual -= beta * g_k
             // dense_x += beta * u_k
             // f = (0,...,0,f_k+1 - beta * m_k+1,k,...,f_s - beta * m_s,k)
-
-            std::cout << "ran step 3. dense_x is: \n";
-            for (auto i = 0; i < dense_x->get_size()[0]; i++) {
-                std::cout << dense_x->at(i, 0) << std::endl;
-            }
-
-            std::cout << "residual is: \n";
-            for (auto i = 0; i < residual->get_size()[0]; i++) {
-                std::cout << residual->at(i, 0) << std::endl;
-            }
         }
 
         get_preconditioner()->apply(residual.get(),
@@ -273,7 +217,7 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
         t->scale(neg_one_op.get());
         residual->add_scaled(omega.get(), t.get());
-        dense_x->add_scaled(omega.get(), v.get());
+        dense_x->add_scaled(omega.get(), preconditioned_vector.get());
 
         // omega = (t^H * residual) / (t^H * t)
         // rho = (t^H * residual) / (norm(t) * norm(residual))
@@ -283,14 +227,20 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         // residual -= omega * t
         // dense_x += omega * v
     }
-    std::cout << "total iterations: " << total_iter << "\n";
 }
 
 
 template <typename ValueType>
 void Idr<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
-                                const LinOp *beta,
-                                LinOp *x) const GKO_NOT_IMPLEMENTED;
+                                const LinOp *beta, LinOp *x) const
+{
+    auto dense_x = as<matrix::Dense<ValueType>>(x);
+
+    auto x_clone = dense_x->clone();
+    this->apply(b, x_clone.get());
+    dense_x->scale(beta);
+    dense_x->add_scaled(alpha, x_clone.get());
+}
 
 
 #define GKO_DECLARE_IDR(_type) class Idr<_type>
diff --git a/core/solver/idr_kernels.hpp b/core/solver/idr_kernels.hpp
index 2bb0954c26a..49b3e0a57a6 100644
--- a/core/solver/idr_kernels.hpp
+++ b/core/solver/idr_kernels.hpp
@@ -46,9 +46,9 @@ namespace kernels {
 namespace idr {
 
 
-#define GKO_DECLARE_IDR_INITIALIZE_KERNEL(_type)                      \
-    void initialize(std::shared_ptr<const DefaultExecutor> exec,      \
-                    matrix::Dense<_type> *m, matrix::Dense<_type> *g, \
+#define GKO_DECLARE_IDR_INITIALIZE_KERNEL(_type)                 \
+    void initialize(std::shared_ptr<const DefaultExecutor> exec, \
+                    matrix::Dense<_type> *m,                     \
                     Array<stopping_status> *stop_status)
 
 
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index 70e760bcac0..0f1f02e1c87 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -58,7 +58,7 @@ constexpr int default_block_size = 512;
 
 template <typename ValueType>
 void initialize(std::shared_ptr<const CudaExecutor> exec,
-                matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *g,
+                matrix::Dense<ValueType> *m,
                 Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index f8b801142b7..b3606471358 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -61,7 +61,7 @@ constexpr int default_block_size = 512;
 
 template <typename ValueType>
 void initialize(std::shared_ptr<const HipExecutor> exec,
-                matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *g,
+                matrix::Dense<ValueType> *m,
                 Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index c7b9c5e910e..247efcc8b0a 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -149,7 +149,7 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
         std::shared_ptr<const LinOp> GKO_FACTORY_PARAMETER(
             generated_preconditioner, nullptr);
 
-        size_type GKO_FACTORY_PARAMETER(subspace_dim, 1u);
+        size_type GKO_FACTORY_PARAMETER(subspace_dim, 2u);
         remove_complex<ValueType> GKO_FACTORY_PARAMETER(kappa, 0.7);
     };
     GKO_ENABLE_LIN_OP_FACTORY(Idr, parameters, Factory);
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index ae83730fa1d..3c7e953add3 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -55,10 +55,74 @@ namespace omp {
 namespace idr {
 
 
+namespace {
+
+
+template <typename ValueType>
+void solve_lower_triangular(const matrix::Dense<ValueType> *m,
+                            const matrix::Dense<ValueType> *f,
+                            matrix::Dense<ValueType> *c)
+{
+    const auto nrhs = m->get_size()[1] / m->get_size()[0];
+#pragma omp parallel for
+    for (size_type i = 0; i < f->get_size()[1]; i++) {
+        for (size_type row = 0; row < m->get_size()[0]; row++) {
+            auto temp = f->at(row, i);
+            for (size_type col = 0; col < row; col++) {
+                temp -= m->at(row, col * nrhs + i) * c->at(col, i);
+            }
+            c->at(row, i) = temp / m->at(row, row * nrhs + i);
+        }
+    }
+}
+
+
+template <typename ValueType>
+void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
+                    const matrix::Dense<ValueType> *m,
+                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *u)
+{
+    const auto nrhs = m->get_size()[1] / m->get_size()[0];
+#pragma omp parallel for
+    for (size_type i = 0; i < nrhs; i++) {
+        for (size_type j = 0; j < k; j++) {
+            auto alpha = zero<ValueType>();
+            for (size_type ind = 0; ind < p->get_size()[1]; ind++) {
+                alpha += p->at(j, ind) * g->at(ind, k * nrhs + i);
+            }
+            alpha /= m->at(j, j * nrhs + i);
+            for (size_type row = 0; row < g->get_size()[0]; row++) {
+                g->at(row, k * nrhs + i) -= alpha * g->at(row, j * nrhs + i);
+                u->at(row, k * nrhs + i) -= alpha * u->at(row, j * nrhs + i);
+            }
+        }
+    }
+}
+
+
+}  // namespace
+
+
 template <typename ValueType>
 void initialize(std::shared_ptr<const OmpExecutor> exec,
-                matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *g,
-                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+                matrix::Dense<ValueType> *m,
+                Array<stopping_status> *stop_status)
+{
+    const auto nrhs = m->get_size()[1] / m->get_size()[0];
+
+#pragma omp parallel for
+    for (size_type i = 0; i < nrhs; i++) {
+        stop_status->get_data()[i].reset();
+    }
+
+#pragma omp parallel for
+    for (size_type row = 0; row < m->get_size()[0]; row++) {
+        for (size_type col = 0; col < m->get_size()[1]; col++) {
+            m->at(row, col) =
+                (row == col / nrhs) ? one<ValueType>() : zero<ValueType>();
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
 
@@ -70,23 +134,32 @@ void step_1(std::shared_ptr<const OmpExecutor> exec, const size_type k,
             const matrix::Dense<ValueType> *residual,
             const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *c,
             matrix::Dense<ValueType> *v,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-//
-//    step_1_kernel<<<grid_size, block_size, 0, 0>>>(
-//        r->get_size()[0], r->get_size()[1], r->get_stride(),
-//        as_cuda_type(r->get_const_values()), as_cuda_type(p->get_values()),
-//        as_cuda_type(v->get_const_values()),
-//        as_cuda_type(rho->get_const_values()),
-//        as_cuda_type(prev_rho->get_const_values()),
-//        as_cuda_type(alpha->get_const_values()),
-//        as_cuda_type(omega->get_const_values()),
-//        as_cuda_type(stop_status->get_const_data()));
-//}
+            const Array<stopping_status> *stop_status)
+{
+    const auto m_size = m->get_size();
+    const auto nrhs = f->get_size()[1];
+
+    for (size_type i = 0; i < nrhs; i++) {
+        if (stop_status->get_const_data()[0].has_stopped()) {
+            continue;
+        }
+    }
+
+    // Compute c = M \ f
+    solve_lower_triangular(m, f, c);
+
+    for (size_type i = 0; i < nrhs; i++) {
+        // v = residual - c_k * g_k - ... - c_s * g_s
+#pragma omp parallel for
+        for (size_type row = 0; row < v->get_size()[0]; row++) {
+            auto temp = residual->at(row, i);
+            for (size_type j = k; j < m->get_size()[0]; j++) {
+                temp -= c->at(j, i) * g->at(row, j * nrhs + i);
+            }
+            v->at(row, i) = temp;
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
 
@@ -96,22 +169,24 @@ void step_2(std::shared_ptr<const OmpExecutor> exec, const size_type k,
             const matrix::Dense<ValueType> *omega,
             const matrix::Dense<ValueType> *preconditioned_vector,
             const matrix::Dense<ValueType> *c, matrix::Dense<ValueType> *u,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-//
-//    step_2_kernel<<<grid_size, block_size, 0, 0>>>(
-//        r->get_size()[0], r->get_size()[1], r->get_stride(),
-//        as_cuda_type(r->get_const_values()), as_cuda_type(s->get_values()),
-//        as_cuda_type(v->get_const_values()),
-//        as_cuda_type(rho->get_const_values()),
-//        as_cuda_type(alpha->get_values()),
-//        as_cuda_type(beta->get_const_values()),
-//        as_cuda_type(stop_status->get_const_data()));
-//}
+            const Array<stopping_status> *stop_status)
+{
+    const auto nrhs = omega->get_size()[1];
+    for (size_type i = 0; i < nrhs; i++) {
+        if (stop_status->get_const_data()[0].has_stopped()) {
+            continue;
+        }
+
+#pragma omp parallel for
+        for (size_type row = 0; row < u->get_size()[0]; row++) {
+            auto temp = omega->at(0, i) * preconditioned_vector->at(row, i);
+            for (size_type j = k; j < c->get_size()[0]; j++) {
+                temp += c->at(j, i) * u->at(row, j * nrhs + i);
+            }
+            u->at(row, k * nrhs + i) = temp;
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 
@@ -122,26 +197,45 @@ void step_3(std::shared_ptr<const OmpExecutor> exec, const size_type k,
             matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
             matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *residual,
             matrix::Dense<ValueType> *x,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-//
-//    step_3_kernel<<<grid_size, block_size, 0, 0>>>(
-//        r->get_size()[0], r->get_size()[1], r->get_stride(), x->get_stride(),
-//        as_cuda_type(x->get_values()), as_cuda_type(r->get_values()),
-//        as_cuda_type(s->get_const_values()),
-//        as_cuda_type(t->get_const_values()),
-//        as_cuda_type(y->get_const_values()),
-//        as_cuda_type(z->get_const_values()),
-//        as_cuda_type(alpha->get_const_values()),
-//        as_cuda_type(beta->get_const_values()),
-//        as_cuda_type(gamma->get_const_values()),
-//        as_cuda_type(omega->get_values()),
-//        as_cuda_type(stop_status->get_const_data()));
-//}
+            const Array<stopping_status> *stop_status)
+{
+    const auto nrhs = x->get_size()[1];
+
+    for (size_type i = 0; i < nrhs; i++) {
+        if (stop_status->get_const_data()[0].has_stopped()) {
+            continue;
+        }
+    }
+
+    update_g_and_u(k, p, m, g, u);
+
+    for (size_type i = 0; i < nrhs; i++) {
+#pragma omp parallel for
+        for (size_type j = k; j < m->get_size()[0]; j++) {
+            auto temp = zero<ValueType>();
+            for (size_type ind = 0; ind < p->get_size()[1]; ind++) {
+                temp += p->at(j, ind) * g->at(ind, k * nrhs + i);
+            }
+            m->at(j, k * nrhs + i) = temp;
+        }
+
+        auto beta = f->at(k, i) / m->at(k, k * nrhs + i);
+
+#pragma omp parallel for
+        for (size_type row = 0; row < g->get_size()[0]; row++) {
+            residual->at(row, i) -= beta * g->at(row, k * nrhs + i);
+            x->at(row, i) += beta * u->at(row, k * nrhs + i);
+        }
+
+        if (k + 1 < f->get_size()[0]) {
+            f->at(k, i) = zero<ValueType>();
+#pragma omp parallel for
+            for (size_type j = k + 1; j < f->get_size()[0]; j++) {
+                f->at(j, i) -= beta * m->at(j, k * nrhs + i);
+            }
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
@@ -153,19 +247,25 @@ void compute_omega(
     const matrix::Dense<remove_complex<ValueType>> *t_norm,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
     matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *omega,
-    const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(y->get_size()[0] * y->get_stride(), block_size.x), 1, 1);
-//
-//    finalize_kernel<<<grid_size, block_size, 0, 0>>>(
-//        y->get_size()[0], y->get_size()[1], y->get_stride(), x->get_stride(),
-//        as_cuda_type(x->get_values()), as_cuda_type(y->get_const_values()),
-//        as_cuda_type(alpha->get_const_values()),
-//        as_cuda_type(stop_status->get_data()));
-//}
+    const Array<stopping_status> *stop_status)
+{
+#pragma omp parallel for
+    for (size_type i = 0; i < omega->get_size()[1]; i++) {
+        if (stop_status->get_const_data()[0].has_stopped()) {
+            continue;
+        }
+
+        auto thr = omega->at(0, i);
+        auto normt = t_norm->at(0, i);
+        omega->at(0, i) /= tht->at(0, i);
+        auto absrho = abs(thr / (normt * residual_norm->at(0, i)));
+        rho->at(0, i) = absrho;
+
+        if (absrho < kappa) {
+            omega->at(0, i) *= kappa / absrho;
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index d36ce07d905..1972905801b 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -52,9 +52,55 @@ namespace reference {
 namespace idr {
 
 
+namespace {
+
+
+template <typename ValueType>
+void solve_lower_triangular(const matrix::Dense<ValueType> *m,
+                            const matrix::Dense<ValueType> *f,
+                            matrix::Dense<ValueType> *c)
+{
+    const auto nrhs = m->get_size()[1] / m->get_size()[0];
+    for (size_type i = 0; i < f->get_size()[1]; i++) {
+        for (size_type row = 0; row < m->get_size()[0]; row++) {
+            auto temp = f->at(row, i);
+            for (size_type col = 0; col < row; col++) {
+                temp -= m->at(row, col * nrhs + i) * c->at(col, i);
+            }
+            c->at(row, i) = temp / m->at(row, row * nrhs + i);
+        }
+    }
+}
+
+
+template <typename ValueType>
+void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
+                    const matrix::Dense<ValueType> *m,
+                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *u)
+{
+    const auto nrhs = m->get_size()[1] / m->get_size()[0];
+    for (size_type i = 0; i < nrhs; i++) {
+        for (size_type j = 0; j < k; j++) {
+            auto alpha = zero<ValueType>();
+            for (size_type ind = 0; ind < p->get_size()[1]; ind++) {
+                alpha += p->at(j, ind) * g->at(ind, k * nrhs + i);
+            }
+            alpha /= m->at(j, j * nrhs + i);
+            for (size_type row = 0; row < g->get_size()[0]; row++) {
+                g->at(row, k * nrhs + i) -= alpha * g->at(row, j * nrhs + i);
+                u->at(row, k * nrhs + i) -= alpha * u->at(row, j * nrhs + i);
+            }
+        }
+    }
+}
+
+
+}  // namespace
+
+
 template <typename ValueType>
 void initialize(std::shared_ptr<const ReferenceExecutor> exec,
-                matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *g,
+                matrix::Dense<ValueType> *m,
                 Array<stopping_status> *stop_status)
 {
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
@@ -68,12 +114,6 @@ void initialize(std::shared_ptr<const ReferenceExecutor> exec,
                 (row == col / nrhs) ? one<ValueType>() : zero<ValueType>();
         }
     }
-
-    for (auto row = 0; row < g->get_size()[0]; row++) {
-        for (auto col = 0; col < g->get_size()[1]; col++) {
-            g->at(row, col) = zero<ValueType>();
-        }
-    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
@@ -88,23 +128,18 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec, const size_type k,
             matrix::Dense<ValueType> *v,
             const Array<stopping_status> *stop_status)
 {
-    const auto m_size = m->get_size();
     const auto nrhs = f->get_size()[1];
 
     for (size_type i = 0; i < nrhs; i++) {
         if (stop_status->get_const_data()[0].has_stopped()) {
             continue;
         }
+    }
 
-        // Compute c = M \ f
-        for (size_type row = 0; row < m_size[0]; row++) {
-            auto temp = f->at(row, i);
-            for (size_type col = 0; col < row; col++) {
-                temp -= m->at(row, col * nrhs + i) * c->at(col, i);
-            }
-            c->at(row, i) = temp / m->at(row, row * nrhs + i);
-        }
+    // Compute c = M \ f
+    solve_lower_triangular(m, f, c);
 
+    for (size_type i = 0; i < nrhs; i++) {
         // v = residual - c_k * g_k - ... - c_s * g_s
         for (size_type row = 0; row < v->get_size()[0]; row++) {
             auto temp = residual->at(row, i);
@@ -159,19 +194,11 @@ void step_3(std::shared_ptr<const ReferenceExecutor> exec, const size_type k,
         if (stop_status->get_const_data()[0].has_stopped()) {
             continue;
         }
+    }
 
-        for (size_type j = 0; j < k; j++) {
-            auto alpha = zero<ValueType>();
-            for (size_type ind = 0; ind < p->get_size()[1]; ind++) {
-                alpha += p->at(j, ind) * g->at(ind, k * nrhs + i);
-            }
-            alpha /= m->at(j, j * nrhs + i);
-            for (size_type row = 0; row < g->get_size()[0]; row++) {
-                g->at(row, k * nrhs + i) -= alpha * g->at(row, j * nrhs + i);
-                u->at(row, k * nrhs + i) -= alpha * u->at(row, j * nrhs + i);
-            }
-        }
+    update_g_and_u(k, p, m, g, u);
 
+    for (size_type i = 0; i < nrhs; i++) {
         for (size_type j = k; j < m->get_size()[0]; j++) {
             auto temp = zero<ValueType>();
             for (size_type ind = 0; ind < p->get_size()[1]; ind++) {
@@ -216,9 +243,8 @@ void compute_omega(
         auto thr = omega->at(0, i);
         auto normt = t_norm->at(0, i);
         omega->at(0, i) /= tht->at(0, i);
-        rho->at(0, i) = thr / (normt * residual_norm->at(0, i));
-
-        auto absrho = abs(rho->at(0, i));
+        auto absrho = abs(thr / (normt * residual_norm->at(0, i)));
+        rho->at(0, i) = absrho;
 
         if (absrho < kappa) {
             omega->at(0, i) *= kappa / absrho;
diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp
index e89a590fd11..11a6bb0c264 100644
--- a/reference/test/solver/idr_kernels.cpp
+++ b/reference/test/solver/idr_kernels.cpp
@@ -73,7 +73,6 @@ class Idr : public ::testing::Test {
                       gko::stop::ResidualNormReduction<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
-                  .with_subspace_dim(4u)
                   .on(exec)),
           idr_factory_precision(
               Solver::build()
@@ -86,7 +85,6 @@ class Idr : public ::testing::Test {
                       gko::stop::ResidualNormReduction<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
-                  .with_subspace_dim(4u)
                   .on(exec))
     {}
 
@@ -134,47 +132,43 @@ TYPED_TEST(Idr, SolvesMultipleDenseSystems)
 
 
 TYPED_TEST(Idr, SolvesDenseSystemUsingAdvancedApply)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using value_type = typename TestFixture::value_type;
-//    auto half_tol = std::sqrt(r<value_type>::value);
-//    auto solver = this->idr_factory->generate(this->mtx);
-//    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
-//    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
-//    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
-//    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
-//
-//    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
-//
-//
-//    GKO_ASSERT_MTX_NEAR(x, l({-8.5, -3.0, 6.0}), half_tol);
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto half_tol = std::sqrt(r<value_type>::value);
+    auto solver = this->idr_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+
+    GKO_ASSERT_MTX_NEAR(x, l({-8.5, -3.0, 6.0}), half_tol);
+}
 
 
 TYPED_TEST(Idr, SolvesMultipleDenseSystemsUsingAdvancedApply)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using value_type = typename TestFixture::value_type;
-//    using T = value_type;
-//    auto half_tol = std::sqrt(r<value_type>::value);
-//    auto solver = this->idr_factory->generate(this->mtx);
-//    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
-//    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
-//    auto b = gko::initialize<Mtx>(
-//        {I<T>{-1.0, -5.0}, I<T>{3.0, 1.0}, I<T>{1.0, -2.0}}, this->exec);
-//    auto x = gko::initialize<Mtx>(
-//        {I<T>{0.5, 1.0}, I<T>{1.0, 2.0}, I<T>{2.0, 3.0}}, this->exec);
-//
-//    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
-//
-//
-//    GKO_ASSERT_MTX_NEAR(x, l({{-8.5, 1.0}, {-3.0, 2.0}, {6.0, -5.0}}),
-//                        half_tol);
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using T = value_type;
+    auto half_tol = std::sqrt(r<value_type>::value);
+    auto solver = this->idr_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {I<T>{-1.0, -5.0}, I<T>{3.0, 1.0}, I<T>{1.0, -2.0}}, this->exec);
+    auto x = gko::initialize<Mtx>(
+        {I<T>{0.5, 1.0}, I<T>{1.0, 2.0}, I<T>{2.0, 3.0}}, this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+
+    GKO_ASSERT_MTX_NEAR(x, l({{-8.5, 1.0}, {-3.0, 2.0}, {6.0, -5.0}}),
+                        half_tol);
+}
 
 
 // The following test-data was generated and validated with MATLAB
@@ -208,8 +202,6 @@ TYPED_TEST(Idr, SolvesBigDenseSystemForDivergenceCheck1)
     locmtx->apply(neg_one_op.get(), x.get(), one_op.get(), b.get());
     b->compute_norm2(resnorm.get());
 
-    std::cout << "RESIDUAL NORM: " << resnorm->at(0, 0) << "\n";
-
     GKO_ASSERT_MTX_NEAR(
         x,
         l({0.13853406350816114, -0.08147485210505287, -0.0450299311807042,
@@ -247,134 +239,124 @@ TYPED_TEST(Idr, SolvesBigDenseSystemForDivergenceCheck2)
 
 
 template <typename T>
-gko::remove_complex<T> infNorm(gko::matrix::Dense<T> *mat,
-                               size_t col = 0) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using std::abs;
-//    using no_cpx_t = gko::remove_complex<T>;
-//    no_cpx_t norm = 0.0;
-//    for (size_t i = 0; i < mat->get_size()[0]; ++i) {
-//        no_cpx_t absEntry = abs(mat->at(i, col));
-//        if (norm < absEntry) norm = absEntry;
-//    }
-//    return norm;
-//}
+gko::remove_complex<T> infNorm(gko::matrix::Dense<T> *mat, size_t col = 0)
+{
+    using std::abs;
+    using no_cpx_t = gko::remove_complex<T>;
+    no_cpx_t norm = 0.0;
+    for (size_t i = 0; i < mat->get_size()[0]; ++i) {
+        no_cpx_t absEntry = abs(mat->at(i, col));
+        if (norm < absEntry) norm = absEntry;
+    }
+    return norm;
+}
 
 
 TYPED_TEST(Idr, SolvesMultipleDenseSystemsDivergenceCheck)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using value_type = typename TestFixture::value_type;
-//    using T = value_type;
-//    std::shared_ptr<Mtx> locmtx =
-//        gko::initialize<Mtx>({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0},
-//                              {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0},
-//                              {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0},
-//                              {60.0, -86.0, 54.0, -40.0, -93.0, 56.0},
-//                              {53.0, 94.0, -54.0, 86.0, -61.0, 4.0},
-//                              {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}},
-//                             this->exec);
-//    auto solver = this->idr_factory_precision->generate(locmtx);
-//    auto b1 =
-//        gko::initialize<Mtx>({0.0, -9.0, -2.0, 8.0, -5.0, -6.0}, this->exec);
-//    auto x1 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0},
-//    this->exec); auto b2 =
-//        gko::initialize<Mtx>({9.0, -4.0, -6.0, -10.0, 1.0, 10.0}, this->exec);
-//    auto x2 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0},
-//    this->exec); auto bc = gko::initialize<Mtx>({I<T>{0., 0.}, I<T>{0., 0.},
-//    I<T>{0., 0.},
-//                                    I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.}},
-//                                   this->exec);
-//    auto xc = gko::initialize<Mtx>({I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.},
-//                                    I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.}},
-//                                   this->exec);
-//    for (size_t i = 0; i < xc->get_size()[0]; ++i) {
-//        bc->at(i, 0) = b1->at(i);
-//        bc->at(i, 1) = b2->at(i);
-//        xc->at(i, 0) = x1->at(i);
-//        xc->at(i, 1) = x2->at(i);
-//    }
-//
-//    solver->apply(b1.get(), x1.get());
-//    solver->apply(b2.get(), x2.get());
-//    solver->apply(bc.get(), xc.get());
-//    auto testMtx =
-//        gko::initialize<Mtx>({I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.},
-//                              I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.}},
-//                             this->exec);
-//
-//    for (size_t i = 0; i < testMtx->get_size()[0]; ++i) {
-//        testMtx->at(i, 0) = x1->at(i);
-//        testMtx->at(i, 1) = x2->at(i);
-//    }
-//
-//    auto alpha = gko::initialize<Mtx>({1.0}, this->exec);
-//    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
-//    auto residual1 = gko::initialize<Mtx>({0.}, this->exec);
-//    residual1->copy_from(b1->clone());
-//    auto residual2 = gko::initialize<Mtx>({0.}, this->exec);
-//    residual2->copy_from(b2->clone());
-//    auto residualC = gko::initialize<Mtx>({0.}, this->exec);
-//    residualC->copy_from(bc->clone());
-//
-//    locmtx->apply(alpha.get(), x1.get(), beta.get(), residual1.get());
-//    locmtx->apply(alpha.get(), x2.get(), beta.get(), residual2.get());
-//    locmtx->apply(alpha.get(), xc.get(), beta.get(), residualC.get());
-//
-//    auto normS1 = infNorm(residual1.get());
-//    auto normS2 = infNorm(residual2.get());
-//    auto normC1 = infNorm(residualC.get(), 0);
-//    auto normC2 = infNorm(residualC.get(), 1);
-//    auto normB1 = infNorm(bc.get(), 0);
-//    auto normB2 = infNorm(bc.get(), 1);
-//
-//    // make sure that all combined solutions are as good or better than the
-//    // single solutions
-//    ASSERT_LE(normC1 / normB1, normS1 / normB1 + r<value_type>::value * 1e2);
-//    ASSERT_LE(normC2 / normB2, normS2 / normB2 + r<value_type>::value * 1e2);
-//
-//    // Not sure if this is necessary, the assertions above should cover what
-//    is
-//    // needed.
-//    GKO_ASSERT_MTX_NEAR(xc, testMtx, r<value_type>::value);
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using T = value_type;
+    std::shared_ptr<Mtx> locmtx =
+        gko::initialize<Mtx>({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0},
+                              {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0},
+                              {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0},
+                              {60.0, -86.0, 54.0, -40.0, -93.0, 56.0},
+                              {53.0, 94.0, -54.0, 86.0, -61.0, 4.0},
+                              {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}},
+                             this->exec);
+    auto solver = this->idr_factory_precision->generate(locmtx);
+    auto b1 =
+        gko::initialize<Mtx>({0.0, -9.0, -2.0, 8.0, -5.0, -6.0}, this->exec);
+    auto x1 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+    auto b2 =
+        gko::initialize<Mtx>({9.0, -4.0, -6.0, -10.0, 1.0, 10.0}, this->exec);
+    auto x2 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+    auto bc = gko::initialize<Mtx>({I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.},
+                                    I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.}},
+                                   this->exec);
+    auto xc = gko::initialize<Mtx>({I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.},
+                                    I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.}},
+                                   this->exec);
+    for (size_t i = 0; i < xc->get_size()[0]; ++i) {
+        bc->at(i, 0) = b1->at(i);
+        bc->at(i, 1) = b2->at(i);
+        xc->at(i, 0) = x1->at(i);
+        xc->at(i, 1) = x2->at(i);
+    }
+
+    solver->apply(b1.get(), x1.get());
+    solver->apply(b2.get(), x2.get());
+    solver->apply(bc.get(), xc.get());
+    auto testMtx =
+        gko::initialize<Mtx>({I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.},
+                              I<T>{0., 0.}, I<T>{0., 0.}, I<T>{0., 0.}},
+                             this->exec);
+
+    for (size_t i = 0; i < testMtx->get_size()[0]; ++i) {
+        testMtx->at(i, 0) = x1->at(i);
+        testMtx->at(i, 1) = x2->at(i);
+    }
+
+    auto alpha = gko::initialize<Mtx>({1.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto residual1 = gko::initialize<Mtx>({0.}, this->exec);
+    residual1->copy_from(b1->clone());
+    auto residual2 = gko::initialize<Mtx>({0.}, this->exec);
+    residual2->copy_from(b2->clone());
+    auto residualC = gko::initialize<Mtx>({0.}, this->exec);
+    residualC->copy_from(bc->clone());
+
+    locmtx->apply(alpha.get(), x1.get(), beta.get(), residual1.get());
+    locmtx->apply(alpha.get(), x2.get(), beta.get(), residual2.get());
+    locmtx->apply(alpha.get(), xc.get(), beta.get(), residualC.get());
+
+    auto normS1 = infNorm(residual1.get());
+    auto normS2 = infNorm(residual2.get());
+    auto normC1 = infNorm(residualC.get(), 0);
+    auto normC2 = infNorm(residualC.get(), 1);
+    auto normB1 = infNorm(bc.get(), 0);
+    auto normB2 = infNorm(bc.get(), 1);
+
+    // make sure that all combined solutions are as good or better than the
+    // single solutions
+    ASSERT_LE(normC1 / normB1, normS1 / normB1 + r<value_type>::value * 1e2);
+    ASSERT_LE(normC2 / normB2, normS2 / normB2 + r<value_type>::value * 1e2);
+
+    // Not sure if this is necessary, the assertions above should cover what
+    // is needed.
+    GKO_ASSERT_MTX_NEAR(xc, testMtx, r<value_type>::value * 1e1);
+}
 
 
 TYPED_TEST(Idr, SolvesTransposedDenseSystem)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using value_type = typename TestFixture::value_type;
-//    auto half_tol = std::sqrt(r<value_type>::value);
-//    auto solver = this->idr_factory->generate(this->mtx->transpose());
-//    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
-//    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
-//
-//    solver->transpose()->apply(b.get(), x.get());
-//
-//    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), half_tol);
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto half_tol = std::sqrt(r<value_type>::value);
+    auto solver = this->idr_factory->generate(this->mtx->transpose());
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->transpose()->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), half_tol);
+}
 
 
 TYPED_TEST(Idr, SolvesConjTransposedDenseSystem)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using value_type = typename TestFixture::value_type;
-//    auto half_tol = std::sqrt(r<value_type>::value);
-//    auto solver = this->idr_factory->generate(this->mtx->conj_transpose());
-//    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
-//    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
-//
-//    solver->conj_transpose()->apply(b.get(), x.get());
-//
-//    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), half_tol);
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto half_tol = std::sqrt(r<value_type>::value);
+    auto solver = this->idr_factory->generate(this->mtx->conj_transpose());
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->conj_transpose()->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), half_tol);
+}
 
 
 }  // namespace

From 1fb03dc1fd9d709a572e53d2826147a4e16e94db Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Fri, 14 Aug 2020 18:38:00 +0200
Subject: [PATCH 0137/1008] Add some omp tests

---
 omp/test/solver/idr_kernels.cpp | 345 +++++++++++++++-----------------
 1 file changed, 164 insertions(+), 181 deletions(-)

diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index d9de535dd5a..c9c55fb7c0c 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/idr.hpp>
 
 
+#include <iostream>
 #include <random>
 
 
@@ -107,65 +108,63 @@ class Idr : public ::testing::Test {
 
     void initialize_data()
     {
-        int m = 597;
-        int n = 17;
-        x = gen_mtx(m, n);
-        b = gen_mtx(m, n);
-        r = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        rr = gen_mtx(m, n);
-        s = gen_mtx(m, n);
-        t = gen_mtx(m, n);
-        y = gen_mtx(m, n);
-        v = gen_mtx(m, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        alpha = gen_mtx(1, n);
-        beta = gen_mtx(1, n);
-        gamma = gen_mtx(1, n);
-        omega = gen_mtx(1, n);
+        int size = 597;
+        int nrhs = 17;
+        int s = 4;
+        x = gen_mtx(size, nrhs);
+        b = gen_mtx(size, nrhs);
+        r = gen_mtx(size, nrhs);
+        m = gen_mtx(s, nrhs * s);
+        f = gen_mtx(s, nrhs);
+        g = gen_mtx(size, nrhs * s);
+        u = gen_mtx(size, nrhs * s);
+        c = gen_mtx(s, nrhs);
+        v = gen_mtx(size, nrhs);
+        p = gen_mtx(s, size);
+        omega = gen_mtx(1, nrhs);
+        tht = gen_mtx(1, nrhs);
+        t_norm = gen_mtx(1, nrhs);
+        residual_norm = gen_mtx(1, nrhs);
+        rho = gen_mtx(1, nrhs);
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(ref, n));
-        for (size_t i = 0; i < n; ++i) {
+            new gko::Array<gko::stopping_status>(ref, nrhs));
+        for (size_t i = 0; i < nrhs; ++i) {
             stop_status->get_data()[i].reset();
         }
 
         d_x = Mtx::create(omp);
         d_b = Mtx::create(omp);
         d_r = Mtx::create(omp);
-        d_z = Mtx::create(omp);
-        d_p = Mtx::create(omp);
-        d_t = Mtx::create(omp);
-        d_s = Mtx::create(omp);
-        d_y = Mtx::create(omp);
+        d_m = Mtx::create(omp);
+        d_f = Mtx::create(omp);
+        d_g = Mtx::create(omp);
+        d_u = Mtx::create(omp);
+        d_c = Mtx::create(omp);
         d_v = Mtx::create(omp);
-        d_rr = Mtx::create(omp);
-        d_prev_rho = Mtx::create(omp);
-        d_rho = Mtx::create(omp);
-        d_alpha = Mtx::create(omp);
-        d_beta = Mtx::create(omp);
-        d_gamma = Mtx::create(omp);
+        d_p = Mtx::create(omp);
         d_omega = Mtx::create(omp);
+        d_tht = Mtx::create(omp);
+        d_t_norm = Mtx::create(omp);
+        d_residual_norm = Mtx::create(omp);
+        d_rho = Mtx::create(omp);
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(omp));
 
         d_x->copy_from(x.get());
         d_b->copy_from(b.get());
         d_r->copy_from(r.get());
-        d_z->copy_from(z.get());
-        d_p->copy_from(p.get());
+        d_m->copy_from(m.get());
+        d_f->copy_from(f.get());
+        d_g->copy_from(g.get());
+        d_u->copy_from(u.get());
+        d_c->copy_from(c.get());
         d_v->copy_from(v.get());
-        d_y->copy_from(y.get());
-        d_t->copy_from(t.get());
-        d_s->copy_from(s.get());
-        d_rr->copy_from(rr.get());
-        d_prev_rho->copy_from(prev_rho.get());
-        d_rho->copy_from(rho.get());
-        d_alpha->copy_from(alpha.get());
-        d_beta->copy_from(beta.get());
-        d_gamma->copy_from(gamma.get());
+        d_p->copy_from(p.get());
         d_omega->copy_from(omega.get());
+        d_tht->copy_from(tht.get());
+        d_t_norm->copy_from(t_norm.get());
+        d_residual_norm->copy_from(residual_norm.get());
+        d_rho->copy_from(rho.get());
         *d_stop_status =
             *stop_status;  // copy_from is not a public member function of Array
     }
@@ -195,174 +194,158 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
-    std::unique_ptr<Mtx> z;
-    std::unique_ptr<Mtx> p;
-    std::unique_ptr<Mtx> rr;
-    std::unique_ptr<Mtx> s;
-    std::unique_ptr<Mtx> t;
-    std::unique_ptr<Mtx> y;
+    std::unique_ptr<Mtx> m;
+    std::unique_ptr<Mtx> f;
+    std::unique_ptr<Mtx> g;
+    std::unique_ptr<Mtx> u;
+    std::unique_ptr<Mtx> c;
     std::unique_ptr<Mtx> v;
-    std::unique_ptr<Mtx> prev_rho;
-    std::unique_ptr<Mtx> rho;
-    std::unique_ptr<Mtx> alpha;
-    std::unique_ptr<Mtx> beta;
-    std::unique_ptr<Mtx> gamma;
+    std::unique_ptr<Mtx> p;
     std::unique_ptr<Mtx> omega;
+    std::unique_ptr<Mtx> tht;
+    std::unique_ptr<Mtx> t_norm;
+    std::unique_ptr<Mtx> residual_norm;
+    std::unique_ptr<Mtx> rho;
     std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
 
     std::unique_ptr<Mtx> d_x;
     std::unique_ptr<Mtx> d_b;
     std::unique_ptr<Mtx> d_r;
-    std::unique_ptr<Mtx> d_z;
-    std::unique_ptr<Mtx> d_p;
-    std::unique_ptr<Mtx> d_t;
-    std::unique_ptr<Mtx> d_s;
-    std::unique_ptr<Mtx> d_y;
+    std::unique_ptr<Mtx> d_m;
+    std::unique_ptr<Mtx> d_f;
+    std::unique_ptr<Mtx> d_g;
+    std::unique_ptr<Mtx> d_u;
+    std::unique_ptr<Mtx> d_c;
     std::unique_ptr<Mtx> d_v;
-    std::unique_ptr<Mtx> d_rr;
-    std::unique_ptr<Mtx> d_prev_rho;
-    std::unique_ptr<Mtx> d_rho;
-    std::unique_ptr<Mtx> d_alpha;
-    std::unique_ptr<Mtx> d_beta;
-    std::unique_ptr<Mtx> d_gamma;
+    std::unique_ptr<Mtx> d_p;
     std::unique_ptr<Mtx> d_omega;
+    std::unique_ptr<Mtx> d_tht;
+    std::unique_ptr<Mtx> d_t_norm;
+    std::unique_ptr<Mtx> d_residual_norm;
+    std::unique_ptr<Mtx> d_rho;
     std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
 };
 
 
 TEST_F(Idr, OmpIdrInitializeIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    initialize_data();
-//
-//    gko::kernels::reference::idr::initialize(
-//        ref, b.get(), r.get(), rr.get(), y.get(), s.get(), t.get(), z.get(),
-//        v.get(), p.get(), prev_rho.get(), rho.get(), alpha.get(), beta.get(),
-//        gamma.get(), omega.get(), stop_status.get());
-//    gko::kernels::omp::idr::initialize(
-//        omp, d_b.get(), d_r.get(), d_rr.get(), d_y.get(), d_s.get(),
-//        d_t.get(), d_z.get(), d_v.get(), d_p.get(), d_prev_rho.get(),
-//        d_rho.get(), d_alpha.get(), d_beta.get(), d_gamma.get(),
-//        d_omega.get(), d_stop_status.get());
-//
-//    GKO_EXPECT_MTX_NEAR(d_r, r, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_z, z, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_p, p, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_y, y, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_t, t, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_s, s, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_rr, rr, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_v, v, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_rho, rho, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_alpha, alpha, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_beta, beta, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_gamma, gamma, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_omega, omega, 1e-14);
-//    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
-//}
+{
+    initialize_data();
+
+    gko::kernels::reference::idr::initialize(ref, m.get(), stop_status.get());
+    gko::kernels::omp::idr::initialize(omp, d_m.get(), d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14)
+}
 
 
 TEST_F(Idr, OmpIdrStep1IsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    initialize_data();
-//
-//    gko::kernels::reference::idr::step_1(
-//        ref, r.get(), p.get(), v.get(), rho.get(), prev_rho.get(),
-//        alpha.get(), omega.get(), stop_status.get());
-//    gko::kernels::omp::idr::step_1(
-//        omp, d_r.get(), d_p.get(), d_v.get(), d_rho.get(), d_prev_rho.get(),
-//        d_alpha.get(), d_omega.get(), d_stop_status.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-//}
+{
+    initialize_data();
+
+    gko::size_type k = 2;
+    gko::kernels::reference::idr::step_1(ref, k, m.get(), f.get(), r.get(),
+                                         g.get(), c.get(), v.get(),
+                                         stop_status.get());
+    gko::kernels::omp::idr::step_1(omp, k, d_m.get(), d_f.get(), d_r.get(),
+                                   d_g.get(), d_c.get(), d_v.get(),
+                                   d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(c, d_c, 1e-14);
+    GKO_ASSERT_MTX_NEAR(v, d_v, 1e-14);
+}
 
 
 TEST_F(Idr, OmpIdrStep2IsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    initialize_data();
-//
-//    gko::kernels::reference::idr::step_2(ref, r.get(), s.get(), v.get(),
-//                                              rho.get(), alpha.get(),
-//                                              beta.get(), stop_status.get());
-//    gko::kernels::omp::idr::step_2(omp, d_r.get(), d_s.get(), d_v.get(),
-//                                        d_rho.get(), d_alpha.get(),
-//                                        d_beta.get(), d_stop_status.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
-//    GKO_ASSERT_MTX_NEAR(d_s, s, 1e-14);
-//}
+{
+    initialize_data();
+
+    gko::size_type k = 2;
+    gko::kernels::reference::idr::step_2(ref, k, omega.get(), v.get(), c.get(),
+                                         u.get(), stop_status.get());
+    gko::kernels::omp::idr::step_2(omp, k, d_omega.get(), d_v.get(), d_c.get(),
+                                   d_u.get(), d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
+}
 
 
 TEST_F(Idr, OmpIdrStep3IsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    initialize_data();
-//
-//    gko::kernels::reference::idr::step_3(
-//        ref, x.get(), r.get(), s.get(), t.get(), y.get(), z.get(),
-//        alpha.get(), beta.get(), gamma.get(), omega.get(), stop_status.get());
-//    gko::kernels::omp::idr::step_3(
-//        omp, d_x.get(), d_r.get(), d_s.get(), d_t.get(), d_y.get(), d_z.get(),
-//        d_alpha.get(), d_beta.get(), d_gamma.get(), d_omega.get(),
-//        d_stop_status.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_omega, omega, 1e-14);
-//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-//    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-//}
+{
+    initialize_data();
+
+    gko::size_type k = 2;
+    gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), u.get(),
+                                         m.get(), f.get(), r.get(), x.get(),
+                                         stop_status.get());
+    gko::kernels::omp::idr::step_3(omp, k, d_p.get(), d_g.get(), d_u.get(),
+                                   d_m.get(), d_f.get(), d_r.get(), d_x.get(),
+                                   d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(g, d_g, 1e-14);
+    GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
+    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
+    GKO_ASSERT_MTX_NEAR(f, d_f, 1e-14);
+    GKO_ASSERT_MTX_NEAR(r, d_r, 1e-14);
+    GKO_ASSERT_MTX_NEAR(x, d_x, 1e-14);
+}
+
+
+TEST_F(Idr, OmpIdrComputeOmegaIsEquivalentToRef)
+{
+    initialize_data();
+
+    double kappa = 0.7;
+    gko::kernels::reference::idr::compute_omega(
+        ref, kappa, tht.get(), t_norm.get(), residual_norm.get(), rho.get(),
+        omega.get(), stop_status.get());
+    gko::kernels::omp::idr::compute_omega(
+        omp, kappa, d_tht.get(), d_t_norm.get(), d_residual_norm.get(),
+        d_rho.get(), d_omega.get(), d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(omega, d_omega, 1e-14);
+}
 
 
 TEST_F(Idr, OmpIdrApplyOneRHSIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    int m = 123;
-//    int n = 1;
-//    auto ref_solver = ref_idr_factory->generate(mtx);
-//    auto omp_solver = omp_idr_factory->generate(d_mtx);
-//    auto b = gen_mtx(m, n);
-//    auto x = gen_mtx(m, n);
-//    auto d_b = Mtx::create(omp);
-//    auto d_x = Mtx::create(omp);
-//    d_b->copy_from(b.get());
-//    d_x->copy_from(x.get());
-//
-//    ref_solver->apply(b.get(), x.get());
-//    omp_solver->apply(d_b.get(), d_x.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-//}
+{
+    int m = 123;
+    int n = 1;
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto omp_solver = omp_idr_factory->generate(d_mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(omp);
+    auto d_x = Mtx::create(omp);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    omp_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+}
 
 
 TEST_F(Idr, OmpIdrApplyMultipleRHSIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    int m = 123;
-//    int n = 16;
-//    auto omp_solver = omp_idr_factory->generate(d_mtx);
-//    auto ref_solver = ref_idr_factory->generate(mtx);
-//    auto b = gen_mtx(m, n);
-//    auto x = gen_mtx(m, n);
-//    auto d_b = Mtx::create(omp);
-//    auto d_x = Mtx::create(omp);
-//    d_b->copy_from(b.get());
-//    d_x->copy_from(x.get());
-//
-//    ref_solver->apply(b.get(), x.get());
-//    omp_solver->apply(d_b.get(), d_x.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-//}
+{
+    int m = 123;
+    int n = 16;
+    auto omp_solver = omp_idr_factory->generate(d_mtx);
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(omp);
+    auto d_x = Mtx::create(omp);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    omp_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+}
 
 
 }  // namespace

From 86b052761d75ce937bcb242b5e9a07b6b387d075 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Mon, 17 Aug 2020 11:37:35 +0200
Subject: [PATCH 0138/1008] Add cuda implementation

---
 common/solver/idr_kernels.hpp.inc | 276 ++++++++++++++++++++++++
 core/solver/idr.cpp               |   6 +-
 core/solver/idr_kernels.hpp       |   2 +-
 cuda/solver/idr_kernels.cu        | 222 +++++++++++++------
 cuda/test/solver/idr_kernels.cpp  | 345 ++++++++++++++----------------
 hip/solver/idr_kernels.hip.cpp    |   2 +-
 omp/solver/idr_kernels.cpp        |   5 +-
 omp/test/solver/idr_kernels.cpp   |  15 +-
 reference/solver/idr_kernels.cpp  |   4 +-
 9 files changed, 603 insertions(+), 274 deletions(-)
 create mode 100644 common/solver/idr_kernels.hpp.inc

diff --git a/common/solver/idr_kernels.hpp.inc b/common/solver/idr_kernels.hpp.inc
new file mode 100644
index 00000000000..21af19ccc35
--- /dev/null
+++ b/common/solver/idr_kernels.hpp.inc
@@ -0,0 +1,276 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void initialize_kernel(
+    size_type subspace_dim, size_type nrhs, ValueType *__restrict__ m_values,
+    size_type m_stride, stopping_status *__restrict__ stop_status)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row = global_id / m_stride;
+    const auto col = global_id % m_stride;
+
+    if (global_id < nrhs) {
+        stop_status[global_id].reset();
+    }
+
+    if (row < subspace_dim && col < nrhs * subspace_dim) {
+        m_values[row * m_stride + col] =
+            (row == col / nrhs) ? one<ValueType>() : zero<ValueType>();
+    }
+}
+
+
+template <typename ValueType>
+__global__
+    __launch_bounds__(default_block_size) void solve_lower_triangular_kernel(
+        size_type subspace_dim, size_type nrhs,
+        const ValueType *__restrict__ m_values, size_type m_stride,
+        const ValueType *__restrict__ f_values, size_type f_stride,
+        ValueType *__restrict__ c_values, size_type c_stride,
+        const stopping_status *__restrict__ stop_status)
+{
+    const auto global_id = thread::get_thread_id_flat();
+
+    if (global_id >= nrhs) {
+        return;
+    }
+
+    if (!stop_status[global_id].has_stopped()) {
+        for (size_type row = 0; row < subspace_dim; row++) {
+            auto temp = f_values[row * f_stride + global_id];
+            for (size_type col = 0; col < row; col++) {
+                temp -= m_values[row * m_stride + col * nrhs + global_id] *
+                        c_values[col * c_stride + global_id];
+            }
+            c_values[row * c_stride + global_id] =
+                temp / m_values[row * m_stride + row * nrhs + global_id];
+        }
+    }
+}
+
+
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void step_1_kernel(
+    size_type k, size_type num_rows, size_type subspace_dim, size_type nrhs,
+    const ValueType *__restrict__ residual_values, size_type residual_stride,
+    const ValueType *__restrict__ c_values, size_type c_stride,
+    const ValueType *__restrict__ g_values, size_type g_stride,
+    ValueType *__restrict__ v_values, size_type v_stride,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row = global_id / v_stride;
+    const auto col = global_id % v_stride;
+
+    if (row >= num_rows || col >= nrhs) {
+        return;
+    }
+
+    if (!stop_status[col].has_stopped()) {
+        auto temp = residual_values[row * residual_stride + col];
+        for (size_type j = k; j < subspace_dim; j++) {
+            temp -= c_values[j * c_stride + col] *
+                    g_values[row * g_stride + j * nrhs + col];
+        }
+        v_values[row * v_stride + col] = temp;
+    }
+}
+
+
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void step_2_kernel(
+    size_type k, size_type num_rows, size_type subspace_dim, size_type nrhs,
+    const ValueType *__restrict__ omega_values,
+    const ValueType *__restrict__ v_values, size_type v_stride,
+    const ValueType *__restrict__ c_values, size_type c_stride,
+    ValueType *__restrict__ u_values, size_type u_stride,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row = global_id / u_stride;
+    const auto col = global_id % u_stride;
+
+    if (row >= num_rows || col >= nrhs) {
+        return;
+    }
+
+    if (!stop_status[col].has_stopped()) {
+        auto temp = omega_values[col] * v_values[row * v_stride + col];
+        for (size_type j = k; j < subspace_dim; j++) {
+            temp += c_values[j * c_stride + col] *
+                    u_values[row * u_stride + j * nrhs + col];
+        }
+        u_values[row * u_stride + k * nrhs + col] = temp;
+    }
+}
+
+
+template <size_type block_size, typename ValueType>
+__global__ __launch_bounds__(block_size) void update_g_and_u_kernel(
+    size_type k, size_type size, size_type nrhs,
+    const ValueType *__restrict__ p_values, size_type p_stride,
+    const ValueType *__restrict__ m_values, size_type m_stride,
+    ValueType *__restrict__ g_values, size_type g_stride,
+    ValueType *__restrict__ u_values, size_type u_stride,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto tidx = threadIdx.x;
+    const auto rhs = blockIdx.x;
+
+    __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
+
+    for (size_type i = 0; i < k; i++) {
+        __syncthreads();
+        ValueType alpha = zero<ValueType>();
+        if (!stop_status[rhs].has_stopped()) {
+            for (size_type j = tidx; j < size; j += block_size) {
+                alpha += p_values[i * p_stride + j] *
+                         g_values[j * g_stride + k * nrhs + rhs];
+            }
+        }
+        reduction_helper[tidx] = alpha;
+
+        __syncthreads();
+        reduce(group::this_thread_block(), reduction_helper,
+               [](const ValueType &a, const ValueType &b) { return a + b; });
+        __syncthreads();
+
+        if (!stop_status[rhs].has_stopped()) {
+            alpha =
+                reduction_helper[0] / m_values[i * m_stride + i * nrhs + rhs];
+
+            for (size_type j = tidx; j < size; j += block_size) {
+                g_values[j * g_stride + k * nrhs + rhs] -=
+                    alpha * g_values[j * g_stride + i * nrhs + rhs];
+                u_values[j * u_stride + k * nrhs + rhs] -=
+                    alpha * u_values[j * u_stride + i * nrhs + rhs];
+            }
+        }
+    }
+}
+
+
+template <size_type block_size, typename ValueType>
+__global__ __launch_bounds__(block_size) void update_m_kernel(
+    size_type k, size_type size, size_type subspace_dim, size_type nrhs,
+    const ValueType *__restrict__ p_values, size_type p_stride,
+    const ValueType *__restrict__ g_values, size_type g_stride,
+    ValueType *__restrict__ m_values, size_type m_stride,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto tidx = threadIdx.x;
+    const auto row = blockIdx.x + k;
+    const auto col = blockIdx.y;
+
+    __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
+
+    ValueType local_res = zero<ValueType>();
+    if (row < subspace_dim && col < nrhs && !stop_status[col].has_stopped()) {
+        for (size_type i = tidx; i < size; i += block_size) {
+            local_res += p_values[row * p_stride + i] *
+                         g_values[i * g_stride + k * nrhs + col];
+        }
+    }
+    reduction_helper[tidx] = local_res;
+    __syncthreads();
+    reduce(group::this_thread_block(), reduction_helper,
+           [](const ValueType &a, const ValueType &b) { return a + b; });
+    if (tidx == 0 && !stop_status[col].has_stopped()) {
+        m_values[row * m_stride + k * nrhs + col] = reduction_helper[0];
+    }
+}
+
+
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void update_x_r_and_f_kernel(
+    size_type k, size_type size, size_type subspace_dim, size_type nrhs,
+    const ValueType *__restrict__ m_values, size_type m_stride,
+    const ValueType *__restrict__ g_values, size_type g_stride,
+    const ValueType *__restrict__ u_values, size_type u_stride,
+    ValueType *__restrict__ f_values, size_type f_stride,
+    ValueType *__restrict__ r_values, size_type r_stride,
+    ValueType *__restrict__ x_values, size_type x_stride,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row = global_id / x_stride;
+    const auto col = global_id % x_stride;
+
+    if (row >= size || col >= nrhs) {
+        return;
+    }
+
+    if (!stop_status[col].has_stopped()) {
+        const auto beta = f_values[k * f_stride + col] /
+                          m_values[k * m_stride + k * nrhs + col];
+        r_values[row * r_stride + col] -=
+            beta * g_values[row * g_stride + k * nrhs + col];
+        x_values[row * x_stride + col] +=
+            beta * u_values[row * u_stride + k * nrhs + col];
+
+        if (k < row && k + 1 < subspace_dim && row < subspace_dim) {
+            f_values[row * f_stride + col] -=
+                beta * m_values[row * m_stride + k * nrhs + col];
+        }
+        if (k == row && row + 1 < subspace_dim) {
+            f_values[row * f_stride + col] = zero<ValueType>();
+        }
+    }
+}
+
+
+template <typename ValueType>
+__global__ __launch_bounds__(config::warp_size) void compute_omega_kernel(
+    size_type nrhs, const remove_complex<ValueType> kappa,
+    const ValueType *__restrict__ tht,
+    const remove_complex<ValueType> *__restrict__ t_norm,
+    const remove_complex<ValueType> *__restrict__ residual_norm,
+    ValueType *__restrict__ omega,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto global_id = thread::get_thread_id_flat();
+
+    if (global_id >= nrhs) {
+        return;
+    }
+
+    auto thr = omega[global_id];
+    omega[global_id] /= tht[global_id];
+    auto absrho = abs(thr / (t_norm[global_id] * residual_norm[global_id]));
+
+    if (absrho < kappa) {
+        omega[global_id] *= kappa / absrho;
+    }
+}
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index e2d5cfc0fce..a70023912fb 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -127,8 +127,6 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     auto omega = Vector::create(exec, gko::dim<2>{1, nrhs});
     exec->run(
         idr::make_fill_array(omega->get_values(), nrhs, one<ValueType>()));
-    auto rho = Vector::create_with_config_of(omega.get());
-    auto alpha = Vector::create(exec, gko::dim<2>{subspace_dim_, nrhs});
 
     auto residual_norm = NormVector::create(exec, dim<2>{1, nrhs});
     auto tht = Vector::create(exec, dim<2>{1, nrhs});
@@ -212,8 +210,8 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         residual->compute_norm2(residual_norm.get());
 
         exec->run(idr::make_compute_omega(kappa_, tht.get(), t_norm.get(),
-                                          residual_norm.get(), rho.get(),
-                                          omega.get(), &stop_status));
+                                          residual_norm.get(), omega.get(),
+                                          &stop_status));
 
         t->scale(neg_one_op.get());
         residual->add_scaled(omega.get(), t.get());
diff --git a/core/solver/idr_kernels.hpp b/core/solver/idr_kernels.hpp
index 49b3e0a57a6..b10b88064a3 100644
--- a/core/solver/idr_kernels.hpp
+++ b/core/solver/idr_kernels.hpp
@@ -84,7 +84,7 @@ namespace idr {
         const remove_complex<_type> kappa, const matrix::Dense<_type> *tht, \
         const matrix::Dense<remove_complex<_type>> *t_norm,                 \
         const matrix::Dense<remove_complex<_type>> *residual_norm,          \
-        matrix::Dense<_type> *rho, matrix::Dense<_type> *omega,             \
+        matrix::Dense<_type> *omega,                                        \
         const Array<stopping_status> *stop_status)
 
 
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index 0f1f02e1c87..a3c2c5bbdbe 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -39,6 +39,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "cuda/base/math.hpp"
 #include "cuda/base/types.hpp"
+#include "cuda/components/cooperative_groups.cuh"
+#include "cuda/components/reduction.cuh"
 #include "cuda/components/thread_ids.cuh"
 
 
@@ -56,10 +58,108 @@ namespace idr {
 constexpr int default_block_size = 512;
 
 
+#include "common/solver/idr_kernels.hpp.inc"
+
+
+namespace {
+
+
+template <typename ValueType>
+void solve_lower_triangular(const matrix::Dense<ValueType> *m,
+                            const matrix::Dense<ValueType> *f,
+                            matrix::Dense<ValueType> *c,
+                            const Array<stopping_status> *stop_status)
+{
+    const auto subspace_dim = m->get_size()[0];
+    const auto nrhs = m->get_size()[1] / subspace_dim;
+
+    const auto grid_dim = ceildiv(nrhs, default_block_size);
+    solve_lower_triangular_kernel<<<grid_dim, default_block_size>>>(
+        subspace_dim, nrhs, as_cuda_type(m->get_const_values()),
+        m->get_stride(), as_cuda_type(f->get_const_values()), f->get_stride(),
+        as_cuda_type(c->get_values()), c->get_stride(),
+        as_cuda_type(stop_status->get_const_data()));
+}
+
+
+template <typename ValueType>
+void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
+                    const matrix::Dense<ValueType> *m,
+                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *u,
+                    const Array<stopping_status> *stop_status)
+{
+    const auto size = g->get_size()[0];
+    const auto nrhs = m->get_size()[1] / m->get_size()[0];
+
+    const auto grid_dim = nrhs;
+    update_g_and_u_kernel<default_block_size><<<grid_dim, default_block_size>>>(
+        k, size, nrhs, as_cuda_type(p->get_const_values()), p->get_stride(),
+        as_cuda_type(m->get_const_values()), m->get_stride(),
+        as_cuda_type(g->get_values()), g->get_stride(),
+        as_cuda_type(u->get_values()), u->get_stride(),
+        as_cuda_type(stop_status->get_const_data()));
+}
+
+
+template <typename ValueType>
+void update_m(size_type k, const matrix::Dense<ValueType> *p,
+              const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *m,
+              const Array<stopping_status> *stop_status)
+{
+    const auto size = g->get_size()[0];
+    const auto subspace_dim = p->get_size()[0];
+    const auto nrhs = m->get_size()[1] / subspace_dim;
+
+    const auto grid_dim = dim3((subspace_dim - k), nrhs, 1);
+    update_m_kernel<default_block_size><<<grid_dim, default_block_size>>>(
+        k, size, subspace_dim, nrhs, as_cuda_type(p->get_const_values()),
+        p->get_stride(), as_cuda_type(g->get_const_values()), g->get_stride(),
+        as_cuda_type(m->get_values()), m->get_stride(),
+        as_cuda_type(stop_status->get_const_data()));
+}
+
+
+template <typename ValueType>
+void update_x_r_and_f(size_type k, const matrix::Dense<ValueType> *m,
+                      const matrix::Dense<ValueType> *g,
+                      const matrix::Dense<ValueType> *u,
+                      matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *r,
+                      matrix::Dense<ValueType> *x,
+                      const Array<stopping_status> *stop_status)
+{
+    const auto size = x->get_size()[0];
+    const auto subspace_dim = m->get_size()[0];
+    const auto nrhs = x->get_size()[1];
+
+    const auto grid_dim = ceildiv(size * x->get_stride(), default_block_size);
+    update_x_r_and_f_kernel<<<grid_dim, default_block_size>>>(
+        k, size, subspace_dim, nrhs, as_cuda_type(m->get_const_values()),
+        m->get_stride(), as_cuda_type(g->get_const_values()), g->get_stride(),
+        as_cuda_type(u->get_const_values()), u->get_stride(),
+        as_cuda_type(f->get_values()), f->get_stride(),
+        as_cuda_type(r->get_values()), r->get_stride(),
+        as_cuda_type(x->get_values()), x->get_stride(),
+        as_cuda_type(stop_status->get_const_data()));
+}
+
+
+}  // namespace
+
+
 template <typename ValueType>
 void initialize(std::shared_ptr<const CudaExecutor> exec,
                 matrix::Dense<ValueType> *m,
-                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+                Array<stopping_status> *stop_status)
+{
+    const auto subspace_dim = m->get_size()[0];
+    const auto nrhs = m->get_size()[1] / subspace_dim;
+    const auto m_stride = m->get_stride();
+
+    const auto grid_dim = ceildiv(m_stride * subspace_dim, default_block_size);
+    initialize_kernel<<<grid_dim, default_block_size>>>(
+        subspace_dim, nrhs, as_cuda_type(m->get_values()), m_stride,
+        as_cuda_type(stop_status->get_data()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
 
@@ -71,23 +171,24 @@ void step_1(std::shared_ptr<const CudaExecutor> exec, const size_type k,
             const matrix::Dense<ValueType> *residual,
             const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *c,
             matrix::Dense<ValueType> *v,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-//
-//    step_1_kernel<<<grid_size, block_size, 0, 0>>>(
-//        r->get_size()[0], r->get_size()[1], r->get_stride(),
-//        as_cuda_type(r->get_const_values()), as_cuda_type(p->get_values()),
-//        as_cuda_type(v->get_const_values()),
-//        as_cuda_type(rho->get_const_values()),
-//        as_cuda_type(prev_rho->get_const_values()),
-//        as_cuda_type(alpha->get_const_values()),
-//        as_cuda_type(omega->get_const_values()),
-//        as_cuda_type(stop_status->get_const_data()));
-//}
+            const Array<stopping_status> *stop_status)
+{
+    solve_lower_triangular(m, f, c, stop_status);
+
+    const auto num_rows = v->get_size()[0];
+    const auto subspace_dim = m->get_size()[0];
+    const auto nrhs = m->get_size()[1] / subspace_dim;
+
+    const auto grid_dim =
+        ceildiv(v->get_stride() * num_rows, default_block_size);
+    step_1_kernel<<<grid_dim, default_block_size>>>(
+        k, num_rows, subspace_dim, nrhs,
+        as_cuda_type(residual->get_const_values()), residual->get_stride(),
+        as_cuda_type(c->get_const_values()), c->get_stride(),
+        as_cuda_type(g->get_const_values()), g->get_stride(),
+        as_cuda_type(v->get_values()), v->get_stride(),
+        as_cuda_type(stop_status->get_const_data()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
 
@@ -97,22 +198,23 @@ void step_2(std::shared_ptr<const CudaExecutor> exec, const size_type k,
             const matrix::Dense<ValueType> *omega,
             const matrix::Dense<ValueType> *preconditioned_vector,
             const matrix::Dense<ValueType> *c, matrix::Dense<ValueType> *u,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-//
-//    step_2_kernel<<<grid_size, block_size, 0, 0>>>(
-//        r->get_size()[0], r->get_size()[1], r->get_stride(),
-//        as_cuda_type(r->get_const_values()), as_cuda_type(s->get_values()),
-//        as_cuda_type(v->get_const_values()),
-//        as_cuda_type(rho->get_const_values()),
-//        as_cuda_type(alpha->get_values()),
-//        as_cuda_type(beta->get_const_values()),
-//        as_cuda_type(stop_status->get_const_data()));
-//}
+            const Array<stopping_status> *stop_status)
+{
+    const auto num_rows = preconditioned_vector->get_size()[0];
+    const auto nrhs = preconditioned_vector->get_size()[1];
+    const auto subspace_dim = u->get_size()[1] / nrhs;
+
+    const auto grid_dim =
+        ceildiv(u->get_stride() * num_rows, default_block_size);
+    step_2_kernel<<<grid_dim, default_block_size>>>(
+        k, num_rows, subspace_dim, nrhs,
+        as_cuda_type(omega->get_const_values()),
+        as_cuda_type(preconditioned_vector->get_const_values()),
+        preconditioned_vector->get_stride(),
+        as_cuda_type(c->get_const_values()), c->get_stride(),
+        as_cuda_type(u->get_values()), u->get_stride(),
+        as_cuda_type(stop_status->get_const_data()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 
@@ -123,26 +225,12 @@ void step_3(std::shared_ptr<const CudaExecutor> exec, const size_type k,
             matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
             matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *residual,
             matrix::Dense<ValueType> *x,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-//
-//    step_3_kernel<<<grid_size, block_size, 0, 0>>>(
-//        r->get_size()[0], r->get_size()[1], r->get_stride(), x->get_stride(),
-//        as_cuda_type(x->get_values()), as_cuda_type(r->get_values()),
-//        as_cuda_type(s->get_const_values()),
-//        as_cuda_type(t->get_const_values()),
-//        as_cuda_type(y->get_const_values()),
-//        as_cuda_type(z->get_const_values()),
-//        as_cuda_type(alpha->get_const_values()),
-//        as_cuda_type(beta->get_const_values()),
-//        as_cuda_type(gamma->get_const_values()),
-//        as_cuda_type(omega->get_values()),
-//        as_cuda_type(stop_status->get_const_data()));
-//}
+            const Array<stopping_status> *stop_status)
+{
+    update_g_and_u(k, p, m, g, u, stop_status);
+    update_m(k, p, g, m, stop_status);
+    update_x_r_and_f(k, m, g, u, f, residual, x, stop_status);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
@@ -153,20 +241,18 @@ void compute_omega(
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
     const matrix::Dense<remove_complex<ValueType>> *t_norm,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
-    matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *omega,
-    const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(y->get_size()[0] * y->get_stride(), block_size.x), 1, 1);
-//
-//    finalize_kernel<<<grid_size, block_size, 0, 0>>>(
-//        y->get_size()[0], y->get_size()[1], y->get_stride(), x->get_stride(),
-//        as_cuda_type(x->get_values()), as_cuda_type(y->get_const_values()),
-//        as_cuda_type(alpha->get_const_values()),
-//        as_cuda_type(stop_status->get_data()));
-//}
+    matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
+{
+    const auto nrhs = omega->get_size()[1];
+
+    const auto grid_dim = ceildiv(nrhs, config::warp_size);
+    compute_omega_kernel<<<grid_dim, config::warp_size>>>(
+        nrhs, kappa, as_cuda_type(tht->get_const_values()),
+        as_cuda_type(t_norm->get_const_values()),
+        as_cuda_type(residual_norm->get_const_values()),
+        as_cuda_type(omega->get_values()),
+        as_cuda_type(stop_status->get_const_data()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index c1453fe405a..cb7166f4044 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/idr.hpp>
 
 
+#include <iostream>
 #include <random>
 
 
@@ -49,7 +50,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/solver/idr_kernels.hpp"
-#include "cuda/test/utils.hpp"
+#include "core/test/utils.hpp"
 
 
 namespace {
@@ -64,7 +65,6 @@ class Idr : public ::testing::Test {
 
     void SetUp()
     {
-        ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
         ref = gko::ReferenceExecutor::create();
         cuda = gko::CudaExecutor::create(0, ref);
 
@@ -72,7 +72,6 @@ class Idr : public ::testing::Test {
         make_diag_dominant(mtx.get());
         d_mtx = Mtx::create(cuda);
         d_mtx->copy_from(mtx.get());
-
         cuda_idr_factory =
             Solver::build()
                 .with_criteria(
@@ -81,6 +80,7 @@ class Idr : public ::testing::Test {
                         .with_reduction_factor(1e-15)
                         .on(cuda))
                 .on(cuda);
+
         ref_idr_factory =
             Solver::build()
                 .with_criteria(
@@ -108,65 +108,60 @@ class Idr : public ::testing::Test {
 
     void initialize_data()
     {
-        int m = 597;
-        int n = 17;
-        x = gen_mtx(m, n);
-        b = gen_mtx(m, n);
-        r = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        rr = gen_mtx(m, n);
-        s = gen_mtx(m, n);
-        t = gen_mtx(m, n);
-        y = gen_mtx(m, n);
-        v = gen_mtx(m, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        alpha = gen_mtx(1, n);
-        beta = gen_mtx(1, n);
-        gamma = gen_mtx(1, n);
-        omega = gen_mtx(1, n);
+        int size = 597;
+        int nrhs = 17;
+        int s = 4;
+        x = gen_mtx(size, nrhs);
+        b = gen_mtx(size, nrhs);
+        r = gen_mtx(size, nrhs);
+        m = gen_mtx(s, nrhs * s);
+        f = gen_mtx(s, nrhs);
+        g = gen_mtx(size, nrhs * s);
+        u = gen_mtx(size, nrhs * s);
+        c = gen_mtx(s, nrhs);
+        v = gen_mtx(size, nrhs);
+        p = gen_mtx(s, size);
+        omega = gen_mtx(1, nrhs);
+        tht = gen_mtx(1, nrhs);
+        t_norm = gen_mtx(1, nrhs);
+        residual_norm = gen_mtx(1, nrhs);
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(ref, n));
-        for (size_t i = 0; i < n; ++i) {
+            new gko::Array<gko::stopping_status>(ref, nrhs));
+        for (size_t i = 0; i < nrhs; ++i) {
             stop_status->get_data()[i].reset();
         }
 
         d_x = Mtx::create(cuda);
         d_b = Mtx::create(cuda);
         d_r = Mtx::create(cuda);
-        d_z = Mtx::create(cuda);
-        d_p = Mtx::create(cuda);
-        d_t = Mtx::create(cuda);
-        d_s = Mtx::create(cuda);
-        d_y = Mtx::create(cuda);
+        d_m = Mtx::create(cuda);
+        d_f = Mtx::create(cuda);
+        d_g = Mtx::create(cuda);
+        d_u = Mtx::create(cuda);
+        d_c = Mtx::create(cuda);
         d_v = Mtx::create(cuda);
-        d_rr = Mtx::create(cuda);
-        d_prev_rho = Mtx::create(cuda);
-        d_rho = Mtx::create(cuda);
-        d_alpha = Mtx::create(cuda);
-        d_beta = Mtx::create(cuda);
-        d_gamma = Mtx::create(cuda);
+        d_p = Mtx::create(cuda);
         d_omega = Mtx::create(cuda);
+        d_tht = Mtx::create(cuda);
+        d_t_norm = Mtx::create(cuda);
+        d_residual_norm = Mtx::create(cuda);
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(cuda));
 
         d_x->copy_from(x.get());
         d_b->copy_from(b.get());
         d_r->copy_from(r.get());
-        d_z->copy_from(z.get());
-        d_p->copy_from(p.get());
+        d_m->copy_from(m.get());
+        d_f->copy_from(f.get());
+        d_g->copy_from(g.get());
+        d_u->copy_from(u.get());
+        d_c->copy_from(c.get());
         d_v->copy_from(v.get());
-        d_y->copy_from(y.get());
-        d_t->copy_from(t.get());
-        d_s->copy_from(s.get());
-        d_rr->copy_from(rr.get());
-        d_prev_rho->copy_from(prev_rho.get());
-        d_rho->copy_from(rho.get());
-        d_alpha->copy_from(alpha.get());
-        d_beta->copy_from(beta.get());
-        d_gamma->copy_from(gamma.get());
+        d_p->copy_from(p.get());
         d_omega->copy_from(omega.get());
+        d_tht->copy_from(tht.get());
+        d_t_norm->copy_from(t_norm.get());
+        d_residual_norm->copy_from(residual_norm.get());
         *d_stop_status =
             *stop_status;  // copy_from is not a public member function of Array
     }
@@ -196,174 +191,156 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
-    std::unique_ptr<Mtx> z;
-    std::unique_ptr<Mtx> p;
-    std::unique_ptr<Mtx> rr;
-    std::unique_ptr<Mtx> s;
-    std::unique_ptr<Mtx> t;
-    std::unique_ptr<Mtx> y;
+    std::unique_ptr<Mtx> m;
+    std::unique_ptr<Mtx> f;
+    std::unique_ptr<Mtx> g;
+    std::unique_ptr<Mtx> u;
+    std::unique_ptr<Mtx> c;
     std::unique_ptr<Mtx> v;
-    std::unique_ptr<Mtx> prev_rho;
-    std::unique_ptr<Mtx> rho;
-    std::unique_ptr<Mtx> alpha;
-    std::unique_ptr<Mtx> beta;
-    std::unique_ptr<Mtx> gamma;
+    std::unique_ptr<Mtx> p;
     std::unique_ptr<Mtx> omega;
+    std::unique_ptr<Mtx> tht;
+    std::unique_ptr<Mtx> t_norm;
+    std::unique_ptr<Mtx> residual_norm;
     std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
 
     std::unique_ptr<Mtx> d_x;
     std::unique_ptr<Mtx> d_b;
     std::unique_ptr<Mtx> d_r;
-    std::unique_ptr<Mtx> d_z;
-    std::unique_ptr<Mtx> d_p;
-    std::unique_ptr<Mtx> d_t;
-    std::unique_ptr<Mtx> d_s;
-    std::unique_ptr<Mtx> d_y;
+    std::unique_ptr<Mtx> d_m;
+    std::unique_ptr<Mtx> d_f;
+    std::unique_ptr<Mtx> d_g;
+    std::unique_ptr<Mtx> d_u;
+    std::unique_ptr<Mtx> d_c;
     std::unique_ptr<Mtx> d_v;
-    std::unique_ptr<Mtx> d_rr;
-    std::unique_ptr<Mtx> d_prev_rho;
-    std::unique_ptr<Mtx> d_rho;
-    std::unique_ptr<Mtx> d_alpha;
-    std::unique_ptr<Mtx> d_beta;
-    std::unique_ptr<Mtx> d_gamma;
+    std::unique_ptr<Mtx> d_p;
     std::unique_ptr<Mtx> d_omega;
+    std::unique_ptr<Mtx> d_tht;
+    std::unique_ptr<Mtx> d_t_norm;
+    std::unique_ptr<Mtx> d_residual_norm;
     std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
 };
 
 
 TEST_F(Idr, CudaIdrInitializeIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    initialize_data();
-//
-//    gko::kernels::reference::idr::initialize(
-//        ref, b.get(), r.get(), rr.get(), y.get(), s.get(), t.get(), z.get(),
-//        v.get(), p.get(), prev_rho.get(), rho.get(), alpha.get(), beta.get(),
-//        gamma.get(), omega.get(), stop_status.get());
-//    gko::kernels::cuda::idr::initialize(
-//        cuda, d_b.get(), d_r.get(), d_rr.get(), d_y.get(), d_s.get(),
-//        d_t.get(), d_z.get(), d_v.get(), d_p.get(), d_prev_rho.get(),
-//        d_rho.get(), d_alpha.get(), d_beta.get(), d_gamma.get(),
-//        d_omega.get(), d_stop_status.get());
-//
-//    GKO_EXPECT_MTX_NEAR(d_r, r, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_z, z, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_p, p, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_y, y, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_t, t, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_s, s, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_rr, rr, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_v, v, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_rho, rho, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_alpha, alpha, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_beta, beta, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_gamma, gamma, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_omega, omega, 1e-14);
-//    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
-//}
+{
+    initialize_data();
+
+    gko::kernels::reference::idr::initialize(ref, m.get(), stop_status.get());
+    gko::kernels::cuda::idr::initialize(cuda, d_m.get(), d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14)
+}
 
 
 TEST_F(Idr, CudaIdrStep1IsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    initialize_data();
-//
-//    gko::kernels::reference::idr::step_1(
-//        ref, r.get(), p.get(), v.get(), rho.get(), prev_rho.get(),
-//        alpha.get(), omega.get(), stop_status.get());
-//    gko::kernels::cuda::idr::step_1(
-//        cuda, d_r.get(), d_p.get(), d_v.get(), d_rho.get(), d_prev_rho.get(),
-//        d_alpha.get(), d_omega.get(), d_stop_status.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-//}
+{
+    initialize_data();
+
+    gko::size_type k = 2;
+    gko::kernels::reference::idr::step_1(ref, k, m.get(), f.get(), r.get(),
+                                         g.get(), c.get(), v.get(),
+                                         stop_status.get());
+    gko::kernels::cuda::idr::step_1(cuda, k, d_m.get(), d_f.get(), d_r.get(),
+                                    d_g.get(), d_c.get(), d_v.get(),
+                                    d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(c, d_c, 1e-14);
+    GKO_ASSERT_MTX_NEAR(v, d_v, 1e-14);
+}
 
 
 TEST_F(Idr, CudaIdrStep2IsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    initialize_data();
-//
-//    gko::kernels::reference::idr::step_2(ref, r.get(), s.get(), v.get(),
-//                                              rho.get(), alpha.get(),
-//                                              beta.get(), stop_status.get());
-//    gko::kernels::cuda::idr::step_2(cuda, d_r.get(), d_s.get(), d_v.get(),
-//                                         d_rho.get(), d_alpha.get(),
-//                                         d_beta.get(), d_stop_status.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
-//    GKO_ASSERT_MTX_NEAR(d_s, s, 1e-14);
-//}
+{
+    initialize_data();
+
+    gko::size_type k = 2;
+    gko::kernels::reference::idr::step_2(ref, k, omega.get(), v.get(), c.get(),
+                                         u.get(), stop_status.get());
+    gko::kernels::cuda::idr::step_2(cuda, k, d_omega.get(), d_v.get(),
+                                    d_c.get(), d_u.get(), d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
+}
 
 
 TEST_F(Idr, CudaIdrStep3IsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    initialize_data();
-//
-//    gko::kernels::reference::idr::step_3(
-//        ref, x.get(), r.get(), s.get(), t.get(), y.get(), z.get(),
-//        alpha.get(), beta.get(), gamma.get(), omega.get(), stop_status.get());
-//    gko::kernels::cuda::idr::step_3(
-//        cuda, d_x.get(), d_r.get(), d_s.get(), d_t.get(), d_y.get(),
-//        d_z.get(), d_alpha.get(), d_beta.get(), d_gamma.get(), d_omega.get(),
-//        d_stop_status.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_omega, omega, 1e-14);
-//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-//    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-//}
+{
+    initialize_data();
+
+    gko::size_type k = 2;
+    gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), u.get(),
+                                         m.get(), f.get(), r.get(), x.get(),
+                                         stop_status.get());
+    gko::kernels::cuda::idr::step_3(cuda, k, d_p.get(), d_g.get(), d_u.get(),
+                                    d_m.get(), d_f.get(), d_r.get(), d_x.get(),
+                                    d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(g, d_g, 1e-14);
+    GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
+    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
+    GKO_ASSERT_MTX_NEAR(f, d_f, 1e-14);
+    GKO_ASSERT_MTX_NEAR(r, d_r, 1e-14);
+    GKO_ASSERT_MTX_NEAR(x, d_x, 1e-14);
+}
+
+
+TEST_F(Idr, CudaIdrComputeOmegaIsEquivalentToRef)
+{
+    initialize_data();
+
+    double kappa = 0.7;
+    gko::kernels::reference::idr::compute_omega(
+        ref, kappa, tht.get(), t_norm.get(), residual_norm.get(), omega.get(),
+        stop_status.get());
+    gko::kernels::cuda::idr::compute_omega(
+        cuda, kappa, d_tht.get(), d_t_norm.get(), d_residual_norm.get(),
+        d_omega.get(), d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(omega, d_omega, 1e-14);
+}
 
 
 TEST_F(Idr, CudaIdrApplyOneRHSIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    int m = 123;
-//    int n = 1;
-//    auto ref_solver = ref_idr_factory->generate(mtx);
-//    auto cuda_solver = cuda_idr_factory->generate(d_mtx);
-//    auto b = gen_mtx(m, n);
-//    auto x = gen_mtx(m, n);
-//    auto d_b = Mtx::create(cuda);
-//    auto d_x = Mtx::create(cuda);
-//    d_b->copy_from(b.get());
-//    d_x->copy_from(x.get());
-//
-//    ref_solver->apply(b.get(), x.get());
-//    cuda_solver->apply(d_b.get(), d_x.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-//}
+{
+    int m = 123;
+    int n = 1;
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto cuda_solver = cuda_idr_factory->generate(d_mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(cuda);
+    auto d_x = Mtx::create(cuda);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    cuda_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+}
 
 
 TEST_F(Idr, CudaIdrApplyMultipleRHSIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    int m = 123;
-//    int n = 16;
-//    auto cuda_solver = cuda_idr_factory->generate(d_mtx);
-//    auto ref_solver = ref_idr_factory->generate(mtx);
-//    auto b = gen_mtx(m, n);
-//    auto x = gen_mtx(m, n);
-//    auto d_b = Mtx::create(cuda);
-//    auto d_x = Mtx::create(cuda);
-//    d_b->copy_from(b.get());
-//    d_x->copy_from(x.get());
-//
-//    ref_solver->apply(b.get(), x.get());
-//    cuda_solver->apply(d_b.get(), d_x.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-//}
+{
+    int m = 123;
+    int n = 16;
+    auto cuda_solver = cuda_idr_factory->generate(d_mtx);
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(cuda);
+    auto d_x = Mtx::create(cuda);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    cuda_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-12);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
+}
 
 
 }  // namespace
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index b3606471358..3e8a25e651e 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -156,7 +156,7 @@ void compute_omega(
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
     const matrix::Dense<remove_complex<ValueType>> *t_norm,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
-    matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *omega,
+    matrix::Dense<ValueType> *omega,
     const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:idr): change the code imported from solver/bicgstab if needed
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index 3c7e953add3..97312b3531b 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -91,6 +91,7 @@ void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
                 alpha += p->at(j, ind) * g->at(ind, k * nrhs + i);
             }
             alpha /= m->at(j, j * nrhs + i);
+
             for (size_type row = 0; row < g->get_size()[0]; row++) {
                 g->at(row, k * nrhs + i) -= alpha * g->at(row, j * nrhs + i);
                 u->at(row, k * nrhs + i) -= alpha * u->at(row, j * nrhs + i);
@@ -246,8 +247,7 @@ void compute_omega(
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
     const matrix::Dense<remove_complex<ValueType>> *t_norm,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
-    matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *omega,
-    const Array<stopping_status> *stop_status)
+    matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
 {
 #pragma omp parallel for
     for (size_type i = 0; i < omega->get_size()[1]; i++) {
@@ -259,7 +259,6 @@ void compute_omega(
         auto normt = t_norm->at(0, i);
         omega->at(0, i) /= tht->at(0, i);
         auto absrho = abs(thr / (normt * residual_norm->at(0, i)));
-        rho->at(0, i) = absrho;
 
         if (absrho < kappa) {
             omega->at(0, i) *= kappa / absrho;
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index c9c55fb7c0c..0616b274686 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -125,7 +125,6 @@ class Idr : public ::testing::Test {
         tht = gen_mtx(1, nrhs);
         t_norm = gen_mtx(1, nrhs);
         residual_norm = gen_mtx(1, nrhs);
-        rho = gen_mtx(1, nrhs);
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(ref, nrhs));
         for (size_t i = 0; i < nrhs; ++i) {
@@ -146,7 +145,6 @@ class Idr : public ::testing::Test {
         d_tht = Mtx::create(omp);
         d_t_norm = Mtx::create(omp);
         d_residual_norm = Mtx::create(omp);
-        d_rho = Mtx::create(omp);
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(omp));
 
@@ -164,7 +162,6 @@ class Idr : public ::testing::Test {
         d_tht->copy_from(tht.get());
         d_t_norm->copy_from(t_norm.get());
         d_residual_norm->copy_from(residual_norm.get());
-        d_rho->copy_from(rho.get());
         *d_stop_status =
             *stop_status;  // copy_from is not a public member function of Array
     }
@@ -205,7 +202,6 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> tht;
     std::unique_ptr<Mtx> t_norm;
     std::unique_ptr<Mtx> residual_norm;
-    std::unique_ptr<Mtx> rho;
     std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
 
     std::unique_ptr<Mtx> d_x;
@@ -222,7 +218,6 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> d_tht;
     std::unique_ptr<Mtx> d_t_norm;
     std::unique_ptr<Mtx> d_residual_norm;
-    std::unique_ptr<Mtx> d_rho;
     std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
 };
 
@@ -296,11 +291,11 @@ TEST_F(Idr, OmpIdrComputeOmegaIsEquivalentToRef)
 
     double kappa = 0.7;
     gko::kernels::reference::idr::compute_omega(
-        ref, kappa, tht.get(), t_norm.get(), residual_norm.get(), rho.get(),
-        omega.get(), stop_status.get());
-    gko::kernels::omp::idr::compute_omega(
-        omp, kappa, d_tht.get(), d_t_norm.get(), d_residual_norm.get(),
-        d_rho.get(), d_omega.get(), d_stop_status.get());
+        ref, kappa, tht.get(), t_norm.get(), residual_norm.get(), omega.get(),
+        stop_status.get());
+    gko::kernels::omp::idr::compute_omega(omp, kappa, d_tht.get(),
+                                          d_t_norm.get(), d_residual_norm.get(),
+                                          d_omega.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(omega, d_omega, 1e-14);
 }
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index 1972905801b..5b694131ed8 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -232,8 +232,7 @@ void compute_omega(
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
     const matrix::Dense<remove_complex<ValueType>> *t_norm,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
-    matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *omega,
-    const Array<stopping_status> *stop_status)
+    matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
 {
     for (size_type i = 0; i < omega->get_size()[1]; i++) {
         if (stop_status->get_const_data()[0].has_stopped()) {
@@ -244,7 +243,6 @@ void compute_omega(
         auto normt = t_norm->at(0, i);
         omega->at(0, i) /= tht->at(0, i);
         auto absrho = abs(thr / (normt * residual_norm->at(0, i)));
-        rho->at(0, i) = absrho;
 
         if (absrho < kappa) {
             omega->at(0, i) *= kappa / absrho;

From a21904a62b238611aef94d33476f11b1c3403e71 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Tue, 18 Aug 2020 09:12:09 +0200
Subject: [PATCH 0139/1008] Add hip implementation and core / hip tests

---
 core/test/solver/idr.cpp        | 351 +++++++++++++++-----------------
 hip/solver/idr_kernels.hip.cpp  | 229 ++++++++++++++-------
 hip/test/solver/idr_kernels.cpp | 345 +++++++++++++++----------------
 3 files changed, 485 insertions(+), 440 deletions(-)

diff --git a/core/test/solver/idr.cpp b/core/test/solver/idr.cpp
index 3139662bb4b..c4f5af1ef8a 100644
--- a/core/test/solver/idr.cpp
+++ b/core/test/solver/idr.cpp
@@ -93,227 +93,202 @@ TYPED_TEST_CASE(Idr, gko::test::ValueTypes);
 
 
 TYPED_TEST(Idr, IdrFactoryKnowsItsExecutor)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    ASSERT_EQ(this->idr_factory->get_executor(), this->exec);
-//}
+{
+    ASSERT_EQ(this->idr_factory->get_executor(), this->exec);
+}
 
 
 TYPED_TEST(Idr, IdrFactoryCreatesCorrectSolver)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Solver = typename TestFixture::Solver;
-//    ASSERT_EQ(this->solver->get_size(), gko::dim<2>(3, 3));
-//    auto idr_solver = static_cast<Solver *>(this->solver.get());
-//    ASSERT_NE(idr_solver->get_system_matrix(), nullptr);
-//    ASSERT_EQ(idr_solver->get_system_matrix(), this->mtx);
-//}
+{
+    using Solver = typename TestFixture::Solver;
+    ASSERT_EQ(this->solver->get_size(), gko::dim<2>(3, 3));
+    auto idr_solver = static_cast<Solver *>(this->solver.get());
+    ASSERT_NE(idr_solver->get_system_matrix(), nullptr);
+    ASSERT_EQ(idr_solver->get_system_matrix(), this->mtx);
+}
 
 
 TYPED_TEST(Idr, CanBeCopied)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using Solver = typename TestFixture::Solver;
-//    auto copy = this->idr_factory->generate(Mtx::create(this->exec));
-//
-//    copy->copy_from(this->solver.get());
-//
-//    ASSERT_EQ(copy->get_size(), gko::dim<2>(3, 3));
-//    auto copy_mtx = static_cast<Solver *>(copy.get())->get_system_matrix();
-//    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
-//                               this->mtx.get());
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    auto copy = this->idr_factory->generate(Mtx::create(this->exec));
+
+    copy->copy_from(this->solver.get());
+
+    ASSERT_EQ(copy->get_size(), gko::dim<2>(3, 3));
+    auto copy_mtx = static_cast<Solver *>(copy.get())->get_system_matrix();
+    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
+                               this->mtx.get());
+}
 
 
 TYPED_TEST(Idr, CanBeMoved)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using Solver = typename TestFixture::Solver;
-//    auto copy = this->idr_factory->generate(Mtx::create(this->exec));
-//
-//    copy->copy_from(std::move(this->solver));
-//
-//    ASSERT_EQ(copy->get_size(), gko::dim<2>(3, 3));
-//    auto copy_mtx = static_cast<Solver *>(copy.get())->get_system_matrix();
-//    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
-//                               this->mtx.get());
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    auto copy = this->idr_factory->generate(Mtx::create(this->exec));
+
+    copy->copy_from(std::move(this->solver));
+
+    ASSERT_EQ(copy->get_size(), gko::dim<2>(3, 3));
+    auto copy_mtx = static_cast<Solver *>(copy.get())->get_system_matrix();
+    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
+                               this->mtx.get());
+}
 
 
 TYPED_TEST(Idr, CanBeCloned)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using Solver = typename TestFixture::Solver;
-//    auto clone = this->solver->clone();
-//
-//    ASSERT_EQ(clone->get_size(), gko::dim<2>(3, 3));
-//    auto clone_mtx = static_cast<Solver *>(clone.get())->get_system_matrix();
-//    this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
-//                               this->mtx.get());
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    auto clone = this->solver->clone();
+
+    ASSERT_EQ(clone->get_size(), gko::dim<2>(3, 3));
+    auto clone_mtx = static_cast<Solver *>(clone.get())->get_system_matrix();
+    this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
+                               this->mtx.get());
+}
 
 
 TYPED_TEST(Idr, CanBeCleared)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Solver = typename TestFixture::Solver;
-//    this->solver->clear();
-//
-//    ASSERT_EQ(this->solver->get_size(), gko::dim<2>(0, 0));
-//    auto solver_mtx =
-//        static_cast<Solver *>(this->solver.get())->get_system_matrix();
-//    ASSERT_EQ(solver_mtx, nullptr);
-//}
+{
+    using Solver = typename TestFixture::Solver;
+    this->solver->clear();
+
+    ASSERT_EQ(this->solver->get_size(), gko::dim<2>(0, 0));
+    auto solver_mtx =
+        static_cast<Solver *>(this->solver.get())->get_system_matrix();
+    ASSERT_EQ(solver_mtx, nullptr);
+}
 
 
 TYPED_TEST(Idr, ApplyUsesInitialGuessReturnsTrue)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    ASSERT_TRUE(this->solver->apply_uses_initial_guess());
-//}
+{
+    ASSERT_TRUE(this->solver->apply_uses_initial_guess());
+}
 
 
 TYPED_TEST(Idr, CanSetPreconditionerGenerator)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Solver = typename TestFixture::Solver;
-//    using value_type = typename TestFixture::value_type;
-//    auto idr_factory =
-//        Solver::build()
-//            .with_criteria(
-//                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
-//            .with_preconditioner(
-//                Solver::build()
-//                    .with_criteria(
-//                        gko::stop::Iteration::build().with_max_iters(3u).on(
-//                            this->exec))
-//                    .on(this->exec))
-//            .on(this->exec);
-//
-//    auto solver = idr_factory->generate(this->mtx);
-//    auto precond = dynamic_cast<const gko::solver::Idr<value_type> *>(
-//        gko::lend(solver->get_preconditioner()));
-//
-//    ASSERT_NE(precond, nullptr);
-//    ASSERT_EQ(precond->get_size(), gko::dim<2>(3, 3));
-//    ASSERT_EQ(precond->get_system_matrix(), this->mtx);
-//}
+{
+    using Solver = typename TestFixture::Solver;
+    using value_type = typename TestFixture::value_type;
+    auto idr_factory =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .with_preconditioner(
+                Solver::build()
+                    .with_criteria(
+                        gko::stop::Iteration::build().with_max_iters(3u).on(
+                            this->exec))
+                    .on(this->exec))
+            .on(this->exec);
+
+    auto solver = idr_factory->generate(this->mtx);
+    auto precond = dynamic_cast<const gko::solver::Idr<value_type> *>(
+        gko::lend(solver->get_preconditioner()));
+
+    ASSERT_NE(precond, nullptr);
+    ASSERT_EQ(precond->get_size(), gko::dim<2>(3, 3));
+    ASSERT_EQ(precond->get_system_matrix(), this->mtx);
+}
 
 
 TYPED_TEST(Idr, CanSetCriteriaAgain)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Solver = typename TestFixture::Solver;
-//    std::shared_ptr<gko::stop::CriterionFactory> init_crit =
-//        gko::stop::Iteration::build().with_max_iters(3u).on(this->exec);
-//    auto idr_factory =
-//        Solver::build().with_criteria(init_crit).on(this->exec);
-//
-//    ASSERT_EQ((idr_factory->get_parameters().criteria).back(), init_crit);
-//
-//    auto solver = idr_factory->generate(this->mtx);
-//    std::shared_ptr<gko::stop::CriterionFactory> new_crit =
-//        gko::stop::Iteration::build().with_max_iters(5u).on(this->exec);
-//
-//    solver->set_stop_criterion_factory(new_crit);
-//    auto new_crit_fac = solver->get_stop_criterion_factory();
-//    auto niter =
-//        static_cast<const gko::stop::Iteration::Factory *>(new_crit_fac.get())
-//            ->get_parameters()
-//            .max_iters;
-//
-//    ASSERT_EQ(niter, 5);
-//}
+{
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<gko::stop::CriterionFactory> init_crit =
+        gko::stop::Iteration::build().with_max_iters(3u).on(this->exec);
+    auto idr_factory = Solver::build().with_criteria(init_crit).on(this->exec);
+
+    ASSERT_EQ((idr_factory->get_parameters().criteria).back(), init_crit);
+
+    auto solver = idr_factory->generate(this->mtx);
+    std::shared_ptr<gko::stop::CriterionFactory> new_crit =
+        gko::stop::Iteration::build().with_max_iters(5u).on(this->exec);
+
+    solver->set_stop_criterion_factory(new_crit);
+    auto new_crit_fac = solver->get_stop_criterion_factory();
+    auto niter =
+        static_cast<const gko::stop::Iteration::Factory *>(new_crit_fac.get())
+            ->get_parameters()
+            .max_iters;
+
+    ASSERT_EQ(niter, 5);
+}
 
 
 TYPED_TEST(Idr, CanSetPreconditionerInFactory)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Solver = typename TestFixture::Solver;
-//    std::shared_ptr<Solver> idr_precond =
-//        Solver::build()
-//            .with_criteria(
-//                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
-//            .on(this->exec)
-//            ->generate(this->mtx);
-//
-//    auto idr_factory =
-//        Solver::build()
-//            .with_criteria(
-//                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
-//            .with_generated_preconditioner(idr_precond)
-//            .on(this->exec);
-//    auto solver = idr_factory->generate(this->mtx);
-//    auto precond = solver->get_preconditioner();
-//
-//    ASSERT_NE(precond.get(), nullptr);
-//    ASSERT_EQ(precond.get(), idr_precond.get());
-//}
+{
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<Solver> idr_precond =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .on(this->exec)
+            ->generate(this->mtx);
+
+    auto idr_factory =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .with_generated_preconditioner(idr_precond)
+            .on(this->exec);
+    auto solver = idr_factory->generate(this->mtx);
+    auto precond = solver->get_preconditioner();
+
+    ASSERT_NE(precond.get(), nullptr);
+    ASSERT_EQ(precond.get(), idr_precond.get());
+}
 
 
 TYPED_TEST(Idr, ThrowsOnWrongPreconditionerInFactory)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using Solver = typename TestFixture::Solver;
-//    std::shared_ptr<Mtx> wrong_sized_mtx =
-//        Mtx::create(this->exec, gko::dim<2>{1, 3});
-//    std::shared_ptr<Solver> idr_precond =
-//        Solver::build()
-//            .with_criteria(
-//                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
-//            .on(this->exec)
-//            ->generate(wrong_sized_mtx);
-//
-//    auto idr_factory =
-//        Solver::build()
-//            .with_criteria(
-//                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
-//            .with_generated_preconditioner(idr_precond)
-//            .on(this->exec);
-//
-//    ASSERT_THROW(idr_factory->generate(this->mtx), gko::DimensionMismatch);
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<Mtx> wrong_sized_mtx =
+        Mtx::create(this->exec, gko::dim<2>{1, 3});
+    std::shared_ptr<Solver> idr_precond =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .on(this->exec)
+            ->generate(wrong_sized_mtx);
+
+    auto idr_factory =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .with_generated_preconditioner(idr_precond)
+            .on(this->exec);
+
+    ASSERT_THROW(idr_factory->generate(this->mtx), gko::DimensionMismatch);
+}
 
 
 TYPED_TEST(Idr, CanSetPreconditioner)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    using Solver = typename TestFixture::Solver;
-//    std::shared_ptr<Solver> idr_precond =
-//        Solver::build()
-//            .with_criteria(
-//                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
-//            .on(this->exec)
-//            ->generate(this->mtx);
-//
-//    auto idr_factory =
-//        Solver::build()
-//            .with_criteria(
-//                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
-//            .on(this->exec);
-//    auto solver = idr_factory->generate(this->mtx);
-//    solver->set_preconditioner(idr_precond);
-//    auto precond = solver->get_preconditioner();
-//
-//    ASSERT_NE(precond.get(), nullptr);
-//    ASSERT_EQ(precond.get(), idr_precond.get());
-//}
+{
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<Solver> idr_precond =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .on(this->exec)
+            ->generate(this->mtx);
+
+    auto idr_factory =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .on(this->exec);
+    auto solver = idr_factory->generate(this->mtx);
+    solver->set_preconditioner(idr_precond);
+    auto precond = solver->get_preconditioner();
+
+    ASSERT_NE(precond.get(), nullptr);
+    ASSERT_EQ(precond.get(), idr_precond.get());
+}
 
 
 }  // namespace
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index 3e8a25e651e..bbca9594853 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -42,6 +42,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "hip/base/math.hip.hpp"
 #include "hip/base/types.hip.hpp"
+#include "hip/components/cooperative_groups.hip.hpp"
+#include "hip/components/reduction.hip.hpp"
 #include "hip/components/thread_ids.hip.hpp"
 
 
@@ -59,10 +61,115 @@ namespace idr {
 constexpr int default_block_size = 512;
 
 
+#include "common/solver/idr_kernels.hpp.inc"
+
+
+namespace {
+
+
+template <typename ValueType>
+void solve_lower_triangular(const matrix::Dense<ValueType> *m,
+                            const matrix::Dense<ValueType> *f,
+                            matrix::Dense<ValueType> *c,
+                            const Array<stopping_status> *stop_status)
+{
+    const auto subspace_dim = m->get_size()[0];
+    const auto nrhs = m->get_size()[1] / subspace_dim;
+
+    const auto grid_dim = ceildiv(nrhs, default_block_size);
+    hipLaunchKernelGGL(HIP_KERNEL_NAME(solve_lower_triangular_kernel), grid_dim,
+                       default_block_size, 0, 0, subspace_dim, nrhs,
+                       as_hip_type(m->get_const_values()), m->get_stride(),
+                       as_hip_type(f->get_const_values()), f->get_stride(),
+                       as_hip_type(c->get_values()), c->get_stride(),
+                       as_hip_type(stop_status->get_const_data()));
+}
+
+
+template <typename ValueType>
+void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
+                    const matrix::Dense<ValueType> *m,
+                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *u,
+                    const Array<stopping_status> *stop_status)
+{
+    const auto size = g->get_size()[0];
+    const auto nrhs = m->get_size()[1] / m->get_size()[0];
+
+    const auto grid_dim = nrhs;
+    hipLaunchKernelGGL(
+        HIP_KERNEL_NAME(update_g_and_u_kernel<default_block_size>), grid_dim,
+        default_block_size, 0, 0, k, size, nrhs,
+        as_hip_type(p->get_const_values()), p->get_stride(),
+        as_hip_type(m->get_const_values()), m->get_stride(),
+        as_hip_type(g->get_values()), g->get_stride(),
+        as_hip_type(u->get_values()), u->get_stride(),
+        as_hip_type(stop_status->get_const_data()));
+}
+
+
+template <typename ValueType>
+void update_m(size_type k, const matrix::Dense<ValueType> *p,
+              const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *m,
+              const Array<stopping_status> *stop_status)
+{
+    const auto size = g->get_size()[0];
+    const auto subspace_dim = p->get_size()[0];
+    const auto nrhs = m->get_size()[1] / subspace_dim;
+
+    const auto grid_dim = dim3((subspace_dim - k), nrhs, 1);
+    hipLaunchKernelGGL(HIP_KERNEL_NAME(update_m_kernel<default_block_size>),
+                       grid_dim, default_block_size, 0, 0, k, size,
+                       subspace_dim, nrhs, as_hip_type(p->get_const_values()),
+                       p->get_stride(), as_hip_type(g->get_const_values()),
+                       g->get_stride(), as_hip_type(m->get_values()),
+                       m->get_stride(),
+                       as_hip_type(stop_status->get_const_data()));
+}
+
+
+template <typename ValueType>
+void update_x_r_and_f(size_type k, const matrix::Dense<ValueType> *m,
+                      const matrix::Dense<ValueType> *g,
+                      const matrix::Dense<ValueType> *u,
+                      matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *r,
+                      matrix::Dense<ValueType> *x,
+                      const Array<stopping_status> *stop_status)
+{
+    const auto size = x->get_size()[0];
+    const auto subspace_dim = m->get_size()[0];
+    const auto nrhs = x->get_size()[1];
+
+    const auto grid_dim = ceildiv(size * x->get_stride(), default_block_size);
+    hipLaunchKernelGGL(HIP_KERNEL_NAME(update_x_r_and_f_kernel), grid_dim,
+                       default_block_size, 0, 0, k, size, subspace_dim, nrhs,
+                       as_hip_type(m->get_const_values()), m->get_stride(),
+                       as_hip_type(g->get_const_values()), g->get_stride(),
+                       as_hip_type(u->get_const_values()), u->get_stride(),
+                       as_hip_type(f->get_values()), f->get_stride(),
+                       as_hip_type(r->get_values()), r->get_stride(),
+                       as_hip_type(x->get_values()), x->get_stride(),
+                       as_hip_type(stop_status->get_const_data()));
+}
+
+
+}  // namespace
+
+
 template <typename ValueType>
 void initialize(std::shared_ptr<const HipExecutor> exec,
                 matrix::Dense<ValueType> *m,
-                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+                Array<stopping_status> *stop_status)
+{
+    const auto subspace_dim = m->get_size()[0];
+    const auto nrhs = m->get_size()[1] / subspace_dim;
+    const auto m_stride = m->get_stride();
+
+    const auto grid_dim = ceildiv(m_stride * subspace_dim, default_block_size);
+    hipLaunchKernelGGL(HIP_KERNEL_NAME(initialize_kernel), grid_dim,
+                       default_block_size, 0, 0, subspace_dim, nrhs,
+                       as_hip_type(m->get_values()), m_stride,
+                       as_hip_type(stop_status->get_data()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
 
@@ -74,23 +181,24 @@ void step_1(std::shared_ptr<const HipExecutor> exec, const size_type k,
             const matrix::Dense<ValueType> *residual,
             const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *c,
             matrix::Dense<ValueType> *v,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-//
-//    step_1_kernel<<<grid_size, block_size, 0, 0>>>(
-//        r->get_size()[0], r->get_size()[1], r->get_stride(),
-//        as_cuda_type(r->get_const_values()), as_cuda_type(p->get_values()),
-//        as_cuda_type(v->get_const_values()),
-//        as_cuda_type(rho->get_const_values()),
-//        as_cuda_type(prev_rho->get_const_values()),
-//        as_cuda_type(alpha->get_const_values()),
-//        as_cuda_type(omega->get_const_values()),
-//        as_cuda_type(stop_status->get_const_data()));
-//}
+            const Array<stopping_status> *stop_status)
+{
+    solve_lower_triangular(m, f, c, stop_status);
+
+    const auto num_rows = v->get_size()[0];
+    const auto subspace_dim = m->get_size()[0];
+    const auto nrhs = m->get_size()[1] / subspace_dim;
+
+    const auto grid_dim =
+        ceildiv(v->get_stride() * num_rows, default_block_size);
+    hipLaunchKernelGGL(
+        HIP_KERNEL_NAME(step_1_kernel), grid_dim, default_block_size, 0, 0, k,
+        num_rows, subspace_dim, nrhs, as_hip_type(residual->get_const_values()),
+        residual->get_stride(), as_hip_type(c->get_const_values()),
+        c->get_stride(), as_hip_type(g->get_const_values()), g->get_stride(),
+        as_hip_type(v->get_values()), v->get_stride(),
+        as_hip_type(stop_status->get_const_data()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
 
@@ -100,22 +208,22 @@ void step_2(std::shared_ptr<const HipExecutor> exec, const size_type k,
             const matrix::Dense<ValueType> *omega,
             const matrix::Dense<ValueType> *preconditioned_vector,
             const matrix::Dense<ValueType> *c, matrix::Dense<ValueType> *u,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-//
-//    step_2_kernel<<<grid_size, block_size, 0, 0>>>(
-//        r->get_size()[0], r->get_size()[1], r->get_stride(),
-//        as_cuda_type(r->get_const_values()), as_cuda_type(s->get_values()),
-//        as_cuda_type(v->get_const_values()),
-//        as_cuda_type(rho->get_const_values()),
-//        as_cuda_type(alpha->get_values()),
-//        as_cuda_type(beta->get_const_values()),
-//        as_cuda_type(stop_status->get_const_data()));
-//}
+            const Array<stopping_status> *stop_status)
+{
+    const auto num_rows = preconditioned_vector->get_size()[0];
+    const auto nrhs = preconditioned_vector->get_size()[1];
+    const auto subspace_dim = u->get_size()[1] / nrhs;
+
+    const auto grid_dim =
+        ceildiv(u->get_stride() * num_rows, default_block_size);
+    hipLaunchKernelGGL(
+        HIP_KERNEL_NAME(step_2_kernel), grid_dim, default_block_size, 0, 0, k,
+        num_rows, subspace_dim, nrhs, as_hip_type(omega->get_const_values()),
+        as_hip_type(preconditioned_vector->get_const_values()),
+        preconditioned_vector->get_stride(), as_hip_type(c->get_const_values()),
+        c->get_stride(), as_hip_type(u->get_values()), u->get_stride(),
+        as_hip_type(stop_status->get_const_data()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 
@@ -126,26 +234,12 @@ void step_3(std::shared_ptr<const HipExecutor> exec, const size_type k,
             matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
             matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *residual,
             matrix::Dense<ValueType> *x,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-//
-//    step_3_kernel<<<grid_size, block_size, 0, 0>>>(
-//        r->get_size()[0], r->get_size()[1], r->get_stride(), x->get_stride(),
-//        as_cuda_type(x->get_values()), as_cuda_type(r->get_values()),
-//        as_cuda_type(s->get_const_values()),
-//        as_cuda_type(t->get_const_values()),
-//        as_cuda_type(y->get_const_values()),
-//        as_cuda_type(z->get_const_values()),
-//        as_cuda_type(alpha->get_const_values()),
-//        as_cuda_type(beta->get_const_values()),
-//        as_cuda_type(gamma->get_const_values()),
-//        as_cuda_type(omega->get_values()),
-//        as_cuda_type(stop_status->get_const_data()));
-//}
+            const Array<stopping_status> *stop_status)
+{
+    update_g_and_u(k, p, m, g, u, stop_status);
+    update_m(k, p, g, m, stop_status);
+    update_x_r_and_f(k, m, g, u, f, residual, x, stop_status);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
@@ -156,20 +250,19 @@ void compute_omega(
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
     const matrix::Dense<remove_complex<ValueType>> *t_norm,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
-    matrix::Dense<ValueType> *omega,
-    const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    const dim3 block_size(default_block_size, 1, 1);
-//    const dim3 grid_size(
-//        ceildiv(y->get_size()[0] * y->get_stride(), block_size.x), 1, 1);
-//
-//    finalize_kernel<<<grid_size, block_size, 0, 0>>>(
-//        y->get_size()[0], y->get_size()[1], y->get_stride(), x->get_stride(),
-//        as_cuda_type(x->get_values()), as_cuda_type(y->get_const_values()),
-//        as_cuda_type(alpha->get_const_values()),
-//        as_cuda_type(stop_status->get_data()));
-//}
+    matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
+{
+    const auto nrhs = omega->get_size()[1];
+
+    const auto grid_dim = ceildiv(nrhs, config::warp_size);
+    hipLaunchKernelGGL(HIP_KERNEL_NAME(compute_omega_kernel), grid_dim,
+                       config::warp_size, 0, 0, nrhs, kappa,
+                       as_hip_type(tht->get_const_values()),
+                       as_hip_type(t_norm->get_const_values()),
+                       as_hip_type(residual_norm->get_const_values()),
+                       as_hip_type(omega->get_values()),
+                       as_hip_type(stop_status->get_const_data()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index 74f17deda57..52651178a76 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/idr.hpp>
 
 
+#include <iostream>
 #include <random>
 
 
@@ -49,7 +50,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/solver/idr_kernels.hpp"
-#include "hip/test/utils.hip.hpp"
+#include "core/test/utils.hpp"
 
 
 namespace {
@@ -64,7 +65,6 @@ class Idr : public ::testing::Test {
 
     void SetUp()
     {
-        ASSERT_GT(gko::HipExecutor::get_num_devices(), 0);
         ref = gko::ReferenceExecutor::create();
         hip = gko::HipExecutor::create(0, ref);
 
@@ -72,7 +72,6 @@ class Idr : public ::testing::Test {
         make_diag_dominant(mtx.get());
         d_mtx = Mtx::create(hip);
         d_mtx->copy_from(mtx.get());
-
         hip_idr_factory =
             Solver::build()
                 .with_criteria(
@@ -81,6 +80,7 @@ class Idr : public ::testing::Test {
                         .with_reduction_factor(1e-15)
                         .on(hip))
                 .on(hip);
+
         ref_idr_factory =
             Solver::build()
                 .with_criteria(
@@ -108,65 +108,60 @@ class Idr : public ::testing::Test {
 
     void initialize_data()
     {
-        int m = 597;
-        int n = 17;
-        x = gen_mtx(m, n);
-        b = gen_mtx(m, n);
-        r = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        rr = gen_mtx(m, n);
-        s = gen_mtx(m, n);
-        t = gen_mtx(m, n);
-        y = gen_mtx(m, n);
-        v = gen_mtx(m, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        alpha = gen_mtx(1, n);
-        beta = gen_mtx(1, n);
-        gamma = gen_mtx(1, n);
-        omega = gen_mtx(1, n);
+        int size = 597;
+        int nrhs = 17;
+        int s = 4;
+        x = gen_mtx(size, nrhs);
+        b = gen_mtx(size, nrhs);
+        r = gen_mtx(size, nrhs);
+        m = gen_mtx(s, nrhs * s);
+        f = gen_mtx(s, nrhs);
+        g = gen_mtx(size, nrhs * s);
+        u = gen_mtx(size, nrhs * s);
+        c = gen_mtx(s, nrhs);
+        v = gen_mtx(size, nrhs);
+        p = gen_mtx(s, size);
+        omega = gen_mtx(1, nrhs);
+        tht = gen_mtx(1, nrhs);
+        t_norm = gen_mtx(1, nrhs);
+        residual_norm = gen_mtx(1, nrhs);
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(ref, n));
-        for (size_t i = 0; i < n; ++i) {
+            new gko::Array<gko::stopping_status>(ref, nrhs));
+        for (size_t i = 0; i < nrhs; ++i) {
             stop_status->get_data()[i].reset();
         }
 
         d_x = Mtx::create(hip);
         d_b = Mtx::create(hip);
         d_r = Mtx::create(hip);
-        d_z = Mtx::create(hip);
-        d_p = Mtx::create(hip);
-        d_t = Mtx::create(hip);
-        d_s = Mtx::create(hip);
-        d_y = Mtx::create(hip);
+        d_m = Mtx::create(hip);
+        d_f = Mtx::create(hip);
+        d_g = Mtx::create(hip);
+        d_u = Mtx::create(hip);
+        d_c = Mtx::create(hip);
         d_v = Mtx::create(hip);
-        d_rr = Mtx::create(hip);
-        d_prev_rho = Mtx::create(hip);
-        d_rho = Mtx::create(hip);
-        d_alpha = Mtx::create(hip);
-        d_beta = Mtx::create(hip);
-        d_gamma = Mtx::create(hip);
+        d_p = Mtx::create(hip);
         d_omega = Mtx::create(hip);
+        d_tht = Mtx::create(hip);
+        d_t_norm = Mtx::create(hip);
+        d_residual_norm = Mtx::create(hip);
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(hip));
 
         d_x->copy_from(x.get());
         d_b->copy_from(b.get());
         d_r->copy_from(r.get());
-        d_z->copy_from(z.get());
-        d_p->copy_from(p.get());
+        d_m->copy_from(m.get());
+        d_f->copy_from(f.get());
+        d_g->copy_from(g.get());
+        d_u->copy_from(u.get());
+        d_c->copy_from(c.get());
         d_v->copy_from(v.get());
-        d_y->copy_from(y.get());
-        d_t->copy_from(t.get());
-        d_s->copy_from(s.get());
-        d_rr->copy_from(rr.get());
-        d_prev_rho->copy_from(prev_rho.get());
-        d_rho->copy_from(rho.get());
-        d_alpha->copy_from(alpha.get());
-        d_beta->copy_from(beta.get());
-        d_gamma->copy_from(gamma.get());
+        d_p->copy_from(p.get());
         d_omega->copy_from(omega.get());
+        d_tht->copy_from(tht.get());
+        d_t_norm->copy_from(t_norm.get());
+        d_residual_norm->copy_from(residual_norm.get());
         *d_stop_status =
             *stop_status;  // copy_from is not a public member function of Array
     }
@@ -196,174 +191,156 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
-    std::unique_ptr<Mtx> z;
-    std::unique_ptr<Mtx> p;
-    std::unique_ptr<Mtx> rr;
-    std::unique_ptr<Mtx> s;
-    std::unique_ptr<Mtx> t;
-    std::unique_ptr<Mtx> y;
+    std::unique_ptr<Mtx> m;
+    std::unique_ptr<Mtx> f;
+    std::unique_ptr<Mtx> g;
+    std::unique_ptr<Mtx> u;
+    std::unique_ptr<Mtx> c;
     std::unique_ptr<Mtx> v;
-    std::unique_ptr<Mtx> prev_rho;
-    std::unique_ptr<Mtx> rho;
-    std::unique_ptr<Mtx> alpha;
-    std::unique_ptr<Mtx> beta;
-    std::unique_ptr<Mtx> gamma;
+    std::unique_ptr<Mtx> p;
     std::unique_ptr<Mtx> omega;
+    std::unique_ptr<Mtx> tht;
+    std::unique_ptr<Mtx> t_norm;
+    std::unique_ptr<Mtx> residual_norm;
     std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
 
     std::unique_ptr<Mtx> d_x;
     std::unique_ptr<Mtx> d_b;
     std::unique_ptr<Mtx> d_r;
-    std::unique_ptr<Mtx> d_z;
-    std::unique_ptr<Mtx> d_p;
-    std::unique_ptr<Mtx> d_t;
-    std::unique_ptr<Mtx> d_s;
-    std::unique_ptr<Mtx> d_y;
+    std::unique_ptr<Mtx> d_m;
+    std::unique_ptr<Mtx> d_f;
+    std::unique_ptr<Mtx> d_g;
+    std::unique_ptr<Mtx> d_u;
+    std::unique_ptr<Mtx> d_c;
     std::unique_ptr<Mtx> d_v;
-    std::unique_ptr<Mtx> d_rr;
-    std::unique_ptr<Mtx> d_prev_rho;
-    std::unique_ptr<Mtx> d_rho;
-    std::unique_ptr<Mtx> d_alpha;
-    std::unique_ptr<Mtx> d_beta;
-    std::unique_ptr<Mtx> d_gamma;
+    std::unique_ptr<Mtx> d_p;
     std::unique_ptr<Mtx> d_omega;
+    std::unique_ptr<Mtx> d_tht;
+    std::unique_ptr<Mtx> d_t_norm;
+    std::unique_ptr<Mtx> d_residual_norm;
     std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
 };
 
 
 TEST_F(Idr, HipIdrInitializeIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    initialize_data();
-//
-//    gko::kernels::reference::idr::initialize(
-//        ref, b.get(), r.get(), rr.get(), y.get(), s.get(), t.get(), z.get(),
-//        v.get(), p.get(), prev_rho.get(), rho.get(), alpha.get(), beta.get(),
-//        gamma.get(), omega.get(), stop_status.get());
-//    gko::kernels::hip::idr::initialize(
-//        hip, d_b.get(), d_r.get(), d_rr.get(), d_y.get(), d_s.get(),
-//        d_t.get(), d_z.get(), d_v.get(), d_p.get(), d_prev_rho.get(),
-//        d_rho.get(), d_alpha.get(), d_beta.get(), d_gamma.get(),
-//        d_omega.get(), d_stop_status.get());
-//
-//    GKO_EXPECT_MTX_NEAR(d_r, r, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_z, z, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_p, p, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_y, y, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_t, t, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_s, s, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_rr, rr, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_v, v, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_rho, rho, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_alpha, alpha, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_beta, beta, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_gamma, gamma, 1e-14);
-//    GKO_EXPECT_MTX_NEAR(d_omega, omega, 1e-14);
-//    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
-//}
+{
+    initialize_data();
+
+    gko::kernels::reference::idr::initialize(ref, m.get(), stop_status.get());
+    gko::kernels::hip::idr::initialize(hip, d_m.get(), d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14)
+}
 
 
 TEST_F(Idr, HipIdrStep1IsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    initialize_data();
-//
-//    gko::kernels::reference::idr::step_1(
-//        ref, r.get(), p.get(), v.get(), rho.get(), prev_rho.get(),
-//        alpha.get(), omega.get(), stop_status.get());
-//    gko::kernels::hip::idr::step_1(
-//        hip, d_r.get(), d_p.get(), d_v.get(), d_rho.get(), d_prev_rho.get(),
-//        d_alpha.get(), d_omega.get(), d_stop_status.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-//}
+{
+    initialize_data();
+
+    gko::size_type k = 2;
+    gko::kernels::reference::idr::step_1(ref, k, m.get(), f.get(), r.get(),
+                                         g.get(), c.get(), v.get(),
+                                         stop_status.get());
+    gko::kernels::hip::idr::step_1(hip, k, d_m.get(), d_f.get(), d_r.get(),
+                                   d_g.get(), d_c.get(), d_v.get(),
+                                   d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(c, d_c, 1e-14);
+    GKO_ASSERT_MTX_NEAR(v, d_v, 1e-14);
+}
 
 
 TEST_F(Idr, HipIdrStep2IsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    initialize_data();
-//
-//    gko::kernels::reference::idr::step_2(ref, r.get(), s.get(), v.get(),
-//                                              rho.get(), alpha.get(),
-//                                              beta.get(), stop_status.get());
-//    gko::kernels::hip::idr::step_2(hip, d_r.get(), d_s.get(), d_v.get(),
-//                                        d_rho.get(), d_alpha.get(),
-//                                        d_beta.get(), d_stop_status.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
-//    GKO_ASSERT_MTX_NEAR(d_s, s, 1e-14);
-//}
+{
+    initialize_data();
+
+    gko::size_type k = 2;
+    gko::kernels::reference::idr::step_2(ref, k, omega.get(), v.get(), c.get(),
+                                         u.get(), stop_status.get());
+    gko::kernels::hip::idr::step_2(hip, k, d_omega.get(), d_v.get(), d_c.get(),
+                                   d_u.get(), d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
+}
 
 
 TEST_F(Idr, HipIdrStep3IsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    initialize_data();
-//
-//    gko::kernels::reference::idr::step_3(
-//        ref, x.get(), r.get(), s.get(), t.get(), y.get(), z.get(),
-//        alpha.get(), beta.get(), gamma.get(), omega.get(), stop_status.get());
-//    gko::kernels::hip::idr::step_3(
-//        hip, d_x.get(), d_r.get(), d_s.get(), d_t.get(), d_y.get(), d_z.get(),
-//        d_alpha.get(), d_beta.get(), d_gamma.get(), d_omega.get(),
-//        d_stop_status.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_omega, omega, 1e-14);
-//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-//    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-//}
+{
+    initialize_data();
+
+    gko::size_type k = 2;
+    gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), u.get(),
+                                         m.get(), f.get(), r.get(), x.get(),
+                                         stop_status.get());
+    gko::kernels::hip::idr::step_3(hip, k, d_p.get(), d_g.get(), d_u.get(),
+                                   d_m.get(), d_f.get(), d_r.get(), d_x.get(),
+                                   d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(g, d_g, 1e-14);
+    GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
+    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
+    GKO_ASSERT_MTX_NEAR(f, d_f, 1e-14);
+    GKO_ASSERT_MTX_NEAR(r, d_r, 1e-14);
+    GKO_ASSERT_MTX_NEAR(x, d_x, 1e-14);
+}
+
+
+TEST_F(Idr, HipIdrComputeOmegaIsEquivalentToRef)
+{
+    initialize_data();
+
+    double kappa = 0.7;
+    gko::kernels::reference::idr::compute_omega(
+        ref, kappa, tht.get(), t_norm.get(), residual_norm.get(), omega.get(),
+        stop_status.get());
+    gko::kernels::hip::idr::compute_omega(hip, kappa, d_tht.get(),
+                                          d_t_norm.get(), d_residual_norm.get(),
+                                          d_omega.get(), d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(omega, d_omega, 1e-14);
+}
 
 
 TEST_F(Idr, HipIdrApplyOneRHSIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    int m = 123;
-//    int n = 1;
-//    auto ref_solver = ref_idr_factory->generate(mtx);
-//    auto hip_solver = hip_idr_factory->generate(d_mtx);
-//    auto b = gen_mtx(m, n);
-//    auto x = gen_mtx(m, n);
-//    auto d_b = Mtx::create(hip);
-//    auto d_x = Mtx::create(hip);
-//    d_b->copy_from(b.get());
-//    d_x->copy_from(x.get());
-//
-//    ref_solver->apply(b.get(), x.get());
-//    hip_solver->apply(d_b.get(), d_x.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-//}
+{
+    int m = 123;
+    int n = 1;
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto hip_solver = hip_idr_factory->generate(d_mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(hip);
+    auto d_x = Mtx::create(hip);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    hip_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+}
 
 
 TEST_F(Idr, HipIdrApplyMultipleRHSIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:idr): change the code imported from solver/bicgstab if needed
-//    int m = 123;
-//    int n = 16;
-//    auto hip_solver = hip_idr_factory->generate(d_mtx);
-//    auto ref_solver = ref_idr_factory->generate(mtx);
-//    auto b = gen_mtx(m, n);
-//    auto x = gen_mtx(m, n);
-//    auto d_b = Mtx::create(hip);
-//    auto d_x = Mtx::create(hip);
-//    d_b->copy_from(b.get());
-//    d_x->copy_from(x.get());
-//
-//    ref_solver->apply(b.get(), x.get());
-//    hip_solver->apply(d_b.get(), d_x.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-//    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-//}
+{
+    int m = 123;
+    int n = 16;
+    auto hip_solver = hip_idr_factory->generate(d_mtx);
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(hip);
+    auto d_x = Mtx::create(hip);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    hip_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-12);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
+}
 
 
 }  // namespace

From 57bf39d5e67d4243bfd77e08db77049d2dbbad64 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 20 Aug 2020 11:46:44 +0200
Subject: [PATCH 0140/1008] Write documentation

---
 common/solver/idr_kernels.hpp.inc  |  4 +--
 core/solver/idr.cpp                | 42 ++++++++++++++++++------------
 core/solver/idr_kernels.hpp        |  1 -
 cuda/solver/idr_kernels.cu         |  2 --
 cuda/test/solver/idr_kernels.cpp   | 17 +++++-------
 hip/solver/idr_kernels.hip.cpp     |  2 --
 hip/test/solver/idr_kernels.cpp    | 15 ++++-------
 include/ginkgo/core/solver/idr.hpp | 11 ++++----
 omp/solver/idr_kernels.cpp         |  3 +--
 omp/test/solver/idr_kernels.cpp    | 15 ++++-------
 reference/solver/idr_kernels.cpp   |  3 +--
 11 files changed, 50 insertions(+), 65 deletions(-)

diff --git a/common/solver/idr_kernels.hpp.inc b/common/solver/idr_kernels.hpp.inc
index 21af19ccc35..e1fc536e130 100644
--- a/common/solver/idr_kernels.hpp.inc
+++ b/common/solver/idr_kernels.hpp.inc
@@ -255,7 +255,6 @@ template <typename ValueType>
 __global__ __launch_bounds__(config::warp_size) void compute_omega_kernel(
     size_type nrhs, const remove_complex<ValueType> kappa,
     const ValueType *__restrict__ tht,
-    const remove_complex<ValueType> *__restrict__ t_norm,
     const remove_complex<ValueType> *__restrict__ residual_norm,
     ValueType *__restrict__ omega,
     const stopping_status *__restrict__ stop_status)
@@ -268,7 +267,8 @@ __global__ __launch_bounds__(config::warp_size) void compute_omega_kernel(
 
     auto thr = omega[global_id];
     omega[global_id] /= tht[global_id];
-    auto absrho = abs(thr / (t_norm[global_id] * residual_norm[global_id]));
+    auto absrho =
+        abs(thr / (sqrt(real(tht[global_id])) * residual_norm[global_id]));
 
     if (absrho < kappa) {
         omega[global_id] *= kappa / absrho;
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index a70023912fb..011784a9e7f 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -113,41 +113,48 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     auto m =
         Vector::create(exec, gko::dim<2>{subspace_dim_, subspace_dim_ * nrhs});
+
     auto g =
         Vector::create(exec, gko::dim<2>{problem_size, subspace_dim_ * nrhs});
-    exec->run(idr::make_fill_array(
-        g->get_values(), problem_size * g->get_stride(), zero<ValueType>()));
     auto u =
         Vector::create(exec, gko::dim<2>{problem_size, subspace_dim_ * nrhs});
-    exec->run(idr::make_fill_array(
-        u->get_values(), problem_size * u->get_stride(), zero<ValueType>()));
-    auto f = Vector::create(exec, gko::dim<2>{subspace_dim_, nrhs});
 
+    auto f = Vector::create(exec, gko::dim<2>{subspace_dim_, nrhs});
     auto c = Vector::create(exec, gko::dim<2>{subspace_dim_, nrhs});
-    auto omega = Vector::create(exec, gko::dim<2>{1, nrhs});
-    exec->run(
-        idr::make_fill_array(omega->get_values(), nrhs, one<ValueType>()));
 
+    auto omega = Vector::create(exec, gko::dim<2>{1, nrhs});
     auto residual_norm = NormVector::create(exec, dim<2>{1, nrhs});
     auto tht = Vector::create(exec, dim<2>{1, nrhs});
     auto t_norm = NormVector::create(exec, dim<2>{1, nrhs});
 
-    int total_iter = -1;
-
     bool one_changed{};
     Array<stopping_status> stop_status(exec, nrhs);
 
+    // Initialization
+    // m = identity
+    exec->run(idr::make_initialize(m.get(), &stop_status));
+
+    // omega = 1
+    exec->run(
+        idr::make_fill_array(omega->get_values(), nrhs, one<ValueType>()));
+
+    // residual = b - Ax
     residual->copy_from(dense_b);
     system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(),
                           residual.get());
-    v->copy_from(residual.get());
+
+    // g = u = 0
+    exec->run(idr::make_fill_array(
+        g->get_values(), problem_size * g->get_stride(), zero<ValueType>()));
+    exec->run(idr::make_fill_array(
+        u->get_values(), problem_size * u->get_stride(), zero<ValueType>()));
+
 
     auto stop_criterion = stop_criterion_factory_->generate(
         system_matrix_, std::shared_ptr<const LinOp>(b, [](const LinOp *) {}),
         x, residual.get());
 
-    exec->run(idr::make_initialize(m.get(), &stop_status));
-    residual->compute_norm2(residual_norm.get());
+    int total_iter = -1;
 
     while (true) {
         ++total_iter;
@@ -160,6 +167,7 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         }
 
         subspace_vectors_->apply(residual.get(), f.get());
+        // f = P^H * residual
 
         for (size_type k = 0; k < subspace_dim_; k++) {
             exec->run(idr::make_step_1(k, m.get(), f.get(), residual.get(),
@@ -182,6 +190,7 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                                            span{k * nrhs, (k + 1) * nrhs});
 
             system_matrix_->apply(u_k.get(), g_k.get());
+            // g_k = Au_k
 
             exec->run(idr::make_step_3(k, subspace_vectors_.get(), g.get(),
                                        u.get(), m.get(), f.get(),
@@ -205,13 +214,12 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         system_matrix_->apply(preconditioned_vector.get(), t.get());
 
         t->compute_dot(residual.get(), omega.get());
-        t->compute_norm2(t_norm.get());
+        // t->compute_norm2(t_norm.get());
         t->compute_dot(t.get(), tht.get());
         residual->compute_norm2(residual_norm.get());
 
-        exec->run(idr::make_compute_omega(kappa_, tht.get(), t_norm.get(),
-                                          residual_norm.get(), omega.get(),
-                                          &stop_status));
+        exec->run(idr::make_compute_omega(
+            kappa_, tht.get(), residual_norm.get(), omega.get(), &stop_status));
 
         t->scale(neg_one_op.get());
         residual->add_scaled(omega.get(), t.get());
diff --git a/core/solver/idr_kernels.hpp b/core/solver/idr_kernels.hpp
index b10b88064a3..f26d58c0196 100644
--- a/core/solver/idr_kernels.hpp
+++ b/core/solver/idr_kernels.hpp
@@ -82,7 +82,6 @@ namespace idr {
     void compute_omega(                                                     \
         std::shared_ptr<const DefaultExecutor> exec,                        \
         const remove_complex<_type> kappa, const matrix::Dense<_type> *tht, \
-        const matrix::Dense<remove_complex<_type>> *t_norm,                 \
         const matrix::Dense<remove_complex<_type>> *residual_norm,          \
         matrix::Dense<_type> *omega,                                        \
         const Array<stopping_status> *stop_status)
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index a3c2c5bbdbe..480aff891f5 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -239,7 +239,6 @@ template <typename ValueType>
 void compute_omega(
     std::shared_ptr<const CudaExecutor> exec,
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
-    const matrix::Dense<remove_complex<ValueType>> *t_norm,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
     matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
 {
@@ -248,7 +247,6 @@ void compute_omega(
     const auto grid_dim = ceildiv(nrhs, config::warp_size);
     compute_omega_kernel<<<grid_dim, config::warp_size>>>(
         nrhs, kappa, as_cuda_type(tht->get_const_values()),
-        as_cuda_type(t_norm->get_const_values()),
         as_cuda_type(residual_norm->get_const_values()),
         as_cuda_type(omega->get_values()),
         as_cuda_type(stop_status->get_const_data()));
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index cb7166f4044..e74703380e9 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -123,7 +123,6 @@ class Idr : public ::testing::Test {
         p = gen_mtx(s, size);
         omega = gen_mtx(1, nrhs);
         tht = gen_mtx(1, nrhs);
-        t_norm = gen_mtx(1, nrhs);
         residual_norm = gen_mtx(1, nrhs);
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(ref, nrhs));
@@ -143,7 +142,6 @@ class Idr : public ::testing::Test {
         d_p = Mtx::create(cuda);
         d_omega = Mtx::create(cuda);
         d_tht = Mtx::create(cuda);
-        d_t_norm = Mtx::create(cuda);
         d_residual_norm = Mtx::create(cuda);
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(cuda));
@@ -160,7 +158,6 @@ class Idr : public ::testing::Test {
         d_p->copy_from(p.get());
         d_omega->copy_from(omega.get());
         d_tht->copy_from(tht.get());
-        d_t_norm->copy_from(t_norm.get());
         d_residual_norm->copy_from(residual_norm.get());
         *d_stop_status =
             *stop_status;  // copy_from is not a public member function of Array
@@ -200,7 +197,6 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> p;
     std::unique_ptr<Mtx> omega;
     std::unique_ptr<Mtx> tht;
-    std::unique_ptr<Mtx> t_norm;
     std::unique_ptr<Mtx> residual_norm;
     std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
 
@@ -216,7 +212,6 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> d_p;
     std::unique_ptr<Mtx> d_omega;
     std::unique_ptr<Mtx> d_tht;
-    std::unique_ptr<Mtx> d_t_norm;
     std::unique_ptr<Mtx> d_residual_norm;
     std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
 };
@@ -290,12 +285,12 @@ TEST_F(Idr, CudaIdrComputeOmegaIsEquivalentToRef)
     initialize_data();
 
     double kappa = 0.7;
-    gko::kernels::reference::idr::compute_omega(
-        ref, kappa, tht.get(), t_norm.get(), residual_norm.get(), omega.get(),
-        stop_status.get());
-    gko::kernels::cuda::idr::compute_omega(
-        cuda, kappa, d_tht.get(), d_t_norm.get(), d_residual_norm.get(),
-        d_omega.get(), d_stop_status.get());
+    gko::kernels::reference::idr::compute_omega(ref, kappa, tht.get(),
+                                                residual_norm.get(),
+                                                omega.get(), stop_status.get());
+    gko::kernels::cuda::idr::compute_omega(cuda, kappa, d_tht.get(),
+                                           d_residual_norm.get(), d_omega.get(),
+                                           d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(omega, d_omega, 1e-14);
 }
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index bbca9594853..8f9a3de1c92 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -248,7 +248,6 @@ template <typename ValueType>
 void compute_omega(
     std::shared_ptr<const HipExecutor> exec,
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
-    const matrix::Dense<remove_complex<ValueType>> *t_norm,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
     matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
 {
@@ -258,7 +257,6 @@ void compute_omega(
     hipLaunchKernelGGL(HIP_KERNEL_NAME(compute_omega_kernel), grid_dim,
                        config::warp_size, 0, 0, nrhs, kappa,
                        as_hip_type(tht->get_const_values()),
-                       as_hip_type(t_norm->get_const_values()),
                        as_hip_type(residual_norm->get_const_values()),
                        as_hip_type(omega->get_values()),
                        as_hip_type(stop_status->get_const_data()));
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index 52651178a76..27445562039 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -123,7 +123,6 @@ class Idr : public ::testing::Test {
         p = gen_mtx(s, size);
         omega = gen_mtx(1, nrhs);
         tht = gen_mtx(1, nrhs);
-        t_norm = gen_mtx(1, nrhs);
         residual_norm = gen_mtx(1, nrhs);
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(ref, nrhs));
@@ -143,7 +142,6 @@ class Idr : public ::testing::Test {
         d_p = Mtx::create(hip);
         d_omega = Mtx::create(hip);
         d_tht = Mtx::create(hip);
-        d_t_norm = Mtx::create(hip);
         d_residual_norm = Mtx::create(hip);
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(hip));
@@ -160,7 +158,6 @@ class Idr : public ::testing::Test {
         d_p->copy_from(p.get());
         d_omega->copy_from(omega.get());
         d_tht->copy_from(tht.get());
-        d_t_norm->copy_from(t_norm.get());
         d_residual_norm->copy_from(residual_norm.get());
         *d_stop_status =
             *stop_status;  // copy_from is not a public member function of Array
@@ -200,7 +197,6 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> p;
     std::unique_ptr<Mtx> omega;
     std::unique_ptr<Mtx> tht;
-    std::unique_ptr<Mtx> t_norm;
     std::unique_ptr<Mtx> residual_norm;
     std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
 
@@ -216,7 +212,6 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> d_p;
     std::unique_ptr<Mtx> d_omega;
     std::unique_ptr<Mtx> d_tht;
-    std::unique_ptr<Mtx> d_t_norm;
     std::unique_ptr<Mtx> d_residual_norm;
     std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
 };
@@ -290,12 +285,12 @@ TEST_F(Idr, HipIdrComputeOmegaIsEquivalentToRef)
     initialize_data();
 
     double kappa = 0.7;
-    gko::kernels::reference::idr::compute_omega(
-        ref, kappa, tht.get(), t_norm.get(), residual_norm.get(), omega.get(),
-        stop_status.get());
+    gko::kernels::reference::idr::compute_omega(ref, kappa, tht.get(),
+                                                residual_norm.get(),
+                                                omega.get(), stop_status.get());
     gko::kernels::hip::idr::compute_omega(hip, kappa, d_tht.get(),
-                                          d_t_norm.get(), d_residual_norm.get(),
-                                          d_omega.get(), d_stop_status.get());
+                                          d_residual_norm.get(), d_omega.get(),
+                                          d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(omega, d_omega, 1e-14);
 }
diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index 247efcc8b0a..fa63b464d2f 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -60,13 +60,12 @@ namespace solver {
 
 
 /**
- * BiCGSTAB or the Bi-Conjugate Gradient-Stabilized is a Krylov subspace solver.
+ * IDR(s) is an efficient method for solving large nonsymmetric systems of
+ * linear equations.
  *
- * Being a generic solver, it is capable of solving general matrices, including
- * non-s.p.d matrices. Though, the memory and the computational requirement of
- * the BiCGSTAB solver are higher than of its s.p.d solver counterpart, it has
- * the capability to solve generic systems. It was developed by stabilizing the
- * BiCG method.
+ * The method is based on the induced dimension reduction theorem which
+ * provides a way to construct subsequent residuals that lie in a sequence
+ * of shrinking subspaces.
  *
  * @tparam ValueType precision of the elements of the system matrix.
  *
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index 97312b3531b..64af234291d 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -245,7 +245,6 @@ template <typename ValueType>
 void compute_omega(
     std::shared_ptr<const OmpExecutor> exec,
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
-    const matrix::Dense<remove_complex<ValueType>> *t_norm,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
     matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
 {
@@ -256,7 +255,7 @@ void compute_omega(
         }
 
         auto thr = omega->at(0, i);
-        auto normt = t_norm->at(0, i);
+        auto normt = sqrt(real(tht->at(0, i)));
         omega->at(0, i) /= tht->at(0, i);
         auto absrho = abs(thr / (normt * residual_norm->at(0, i)));
 
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index 0616b274686..3a2ec1643f8 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -123,7 +123,6 @@ class Idr : public ::testing::Test {
         p = gen_mtx(s, size);
         omega = gen_mtx(1, nrhs);
         tht = gen_mtx(1, nrhs);
-        t_norm = gen_mtx(1, nrhs);
         residual_norm = gen_mtx(1, nrhs);
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(ref, nrhs));
@@ -143,7 +142,6 @@ class Idr : public ::testing::Test {
         d_p = Mtx::create(omp);
         d_omega = Mtx::create(omp);
         d_tht = Mtx::create(omp);
-        d_t_norm = Mtx::create(omp);
         d_residual_norm = Mtx::create(omp);
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(omp));
@@ -160,7 +158,6 @@ class Idr : public ::testing::Test {
         d_p->copy_from(p.get());
         d_omega->copy_from(omega.get());
         d_tht->copy_from(tht.get());
-        d_t_norm->copy_from(t_norm.get());
         d_residual_norm->copy_from(residual_norm.get());
         *d_stop_status =
             *stop_status;  // copy_from is not a public member function of Array
@@ -200,7 +197,6 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> p;
     std::unique_ptr<Mtx> omega;
     std::unique_ptr<Mtx> tht;
-    std::unique_ptr<Mtx> t_norm;
     std::unique_ptr<Mtx> residual_norm;
     std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
 
@@ -216,7 +212,6 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> d_p;
     std::unique_ptr<Mtx> d_omega;
     std::unique_ptr<Mtx> d_tht;
-    std::unique_ptr<Mtx> d_t_norm;
     std::unique_ptr<Mtx> d_residual_norm;
     std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
 };
@@ -290,12 +285,12 @@ TEST_F(Idr, OmpIdrComputeOmegaIsEquivalentToRef)
     initialize_data();
 
     double kappa = 0.7;
-    gko::kernels::reference::idr::compute_omega(
-        ref, kappa, tht.get(), t_norm.get(), residual_norm.get(), omega.get(),
-        stop_status.get());
+    gko::kernels::reference::idr::compute_omega(ref, kappa, tht.get(),
+                                                residual_norm.get(),
+                                                omega.get(), stop_status.get());
     gko::kernels::omp::idr::compute_omega(omp, kappa, d_tht.get(),
-                                          d_t_norm.get(), d_residual_norm.get(),
-                                          d_omega.get(), d_stop_status.get());
+                                          d_residual_norm.get(), d_omega.get(),
+                                          d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(omega, d_omega, 1e-14);
 }
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index 5b694131ed8..3d15589b317 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -230,7 +230,6 @@ template <typename ValueType>
 void compute_omega(
     std::shared_ptr<const ReferenceExecutor> exec,
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
-    const matrix::Dense<remove_complex<ValueType>> *t_norm,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
     matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
 {
@@ -240,7 +239,7 @@ void compute_omega(
         }
 
         auto thr = omega->at(0, i);
-        auto normt = t_norm->at(0, i);
+        auto normt = sqrt(real(tht->at(0, i)));
         omega->at(0, i) /= tht->at(0, i);
         auto absrho = abs(thr / (normt * residual_norm->at(0, i)));
 

From 86f47cdb58be2e669e42bac563627e5c59ffe89b Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 20 Aug 2020 15:15:41 +0200
Subject: [PATCH 0141/1008] Add comments for factory parameters

---
 include/ginkgo/core/solver/idr.hpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index fa63b464d2f..864d8f789e3 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -148,7 +148,16 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
         std::shared_ptr<const LinOp> GKO_FACTORY_PARAMETER(
             generated_preconditioner, nullptr);
 
+        /**
+         * Dimension of the subspace S. Determines how many intermediate
+         * residuals are computed in each iteration.
+         */
         size_type GKO_FACTORY_PARAMETER(subspace_dim, 2u);
+
+        /**
+         * Threshold to determine if Av_n and v_n are too close to being
+         * perpendicular.
+         */
         remove_complex<ValueType> GKO_FACTORY_PARAMETER(kappa, 0.7);
     };
     GKO_ENABLE_LIN_OP_FACTORY(Idr, parameters, Factory);

From 95e309f4ac5922abb24a71142cf993fe85997a93 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 20 Aug 2020 15:38:25 +0200
Subject: [PATCH 0142/1008] add IDR to benchmarks

---
 benchmark/run_all_benchmarks.sh | 4 ++--
 benchmark/solver/solver.cpp     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index 226fb4b2992..c965c5539df 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -36,8 +36,8 @@ if [ ! "${FORMATS}" ]; then
 fi
 
 if [ ! "${SOLVERS}" ]; then
-    echo "SOLVERS    environment variable not set - assuming \"bicgstab,cg,cgs,fcg,gmres\"" 1>&2
-    SOLVERS="bicgstab,cg,cgs,fcg,gmres"
+    echo "SOLVERS    environment variable not set - assuming \"bicgstab,cg,cgs,fcg,gmres,idr\"" 1>&2
+    SOLVERS="bicgstab,cg,cgs,fcg,gmres,idr"
 fi
 
 if [ ! "${SOLVERS_PRECISION}" ]; then
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 578c89a9473..937244e4c2a 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -67,7 +67,7 @@ DEFINE_bool(
 
 DEFINE_string(solvers, "cg",
               "A comma-separated list of solvers to run. "
-              "Supported values are: bicgstab, bicg, cg, cgs, fcg, gmres, "
+              "Supported values are: bicgstab, bicg, cg, cgs, fcg, gmres, idr, "
               "lower_trs, upper_trs, overhead");
 
 DEFINE_uint32(

From 0af6adcef38ee704d41396f080bb999add3752a2 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Fri, 21 Aug 2020 09:04:43 +0200
Subject: [PATCH 0143/1008] Add logging to IDR

---
 core/solver/idr.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index 011784a9e7f..a03e8dc67fb 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -158,6 +158,9 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     while (true) {
         ++total_iter;
+        this->template log<log::Logger::iteration_complete>(
+            this, total_iter, residual.get(), dense_x);
+
         if (stop_criterion->update()
                 .num_iterations(total_iter)
                 .residual(residual.get())

From 1bbad6b5d958d748d1571cfe6a5588ae49b5391d Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Mon, 24 Aug 2020 12:05:02 +0200
Subject: [PATCH 0144/1008] Add orthogonalization for P

---
 common/solver/idr_kernels.hpp.inc     | 109 ++++++++++++++++++++------
 core/solver/idr.cpp                   |   3 +-
 core/solver/idr_kernels.hpp           |   1 +
 cuda/solver/idr_kernels.cu            |  41 ++++++++--
 cuda/test/solver/idr_kernels.cpp      |   9 ++-
 hip/solver/idr_kernels.hip.cpp        |  47 ++++++++---
 hip/test/solver/idr_kernels.cpp       |   9 ++-
 include/ginkgo/core/solver/idr.hpp    |   2 +
 omp/solver/idr_kernels.cpp            |  72 +++++++++++++----
 omp/test/solver/idr_kernels.cpp       |   9 ++-
 reference/solver/idr_kernels.cpp      |  68 ++++++++++++----
 reference/test/solver/idr_kernels.cpp |   2 +-
 12 files changed, 290 insertions(+), 82 deletions(-)

diff --git a/common/solver/idr_kernels.hpp.inc b/common/solver/idr_kernels.hpp.inc
index e1fc536e130..d663afb43d1 100644
--- a/common/solver/idr_kernels.hpp.inc
+++ b/common/solver/idr_kernels.hpp.inc
@@ -31,7 +31,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void initialize_kernel(
+__global__ __launch_bounds__(default_block_size) void initialize_m_kernel(
     size_type subspace_dim, size_type nrhs, ValueType *__restrict__ m_values,
     size_type m_stride, stopping_status *__restrict__ stop_status)
 {
@@ -50,6 +50,64 @@ __global__ __launch_bounds__(default_block_size) void initialize_kernel(
 }
 
 
+template <size_type block_size, typename ValueType>
+__global__
+    __launch_bounds__(block_size) void orthonormalize_subspace_vectors_kernel(
+        size_type num_rows, size_type num_cols, ValueType *__restrict__ values,
+        size_type stride)
+{
+    const auto tidx = thread::get_thread_id_flat();
+
+    __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
+
+    __shared__ UninitializedArray<remove_complex<ValueType>, block_size>
+        reduction_helper_array_real;
+    remove_complex<ValueType> *__restrict__ reduction_helper_real =
+        reduction_helper_array_real;
+
+    for (size_type row = 0; row < num_rows; row++) {
+        for (size_type i = 0; i < row; i++) {
+            auto dot = zero<ValueType>();
+            for (size_type j = tidx; j < num_cols; j += block_size) {
+                dot += values[row * stride + j] * conj(values[i * stride + j]);
+            }
+
+            reduction_helper[tidx] = dot;
+
+            __syncthreads();
+            reduce(
+                group::this_thread_block(), reduction_helper,
+                [](const ValueType &a, const ValueType &b) { return a + b; });
+            __syncthreads();
+
+            dot = reduction_helper[0];
+            for (size_type j = tidx; j < num_cols; j += block_size) {
+                values[row * stride + j] -= dot * values[i * stride + j];
+            }
+        }
+
+        auto norm = zero<remove_complex<ValueType>>();
+        for (size_type j = tidx; j < num_cols; j += block_size) {
+            norm += squared_norm(values[row * stride + j]);
+        }
+
+        reduction_helper_real[tidx] = norm;
+
+        __syncthreads();
+        reduce(group::this_thread_block(), reduction_helper_real,
+               [](const remove_complex<ValueType> &a,
+                  const remove_complex<ValueType> &b) { return a + b; });
+        __syncthreads();
+
+        norm = sqrt(reduction_helper_real[0]);
+        for (size_type j = 0; j < num_cols; j += block_size) {
+            values[row * stride + j] /= norm;
+        }
+    }
+}
+
+
 template <typename ValueType>
 __global__
     __launch_bounds__(default_block_size) void solve_lower_triangular_kernel(
@@ -147,17 +205,19 @@ __global__ __launch_bounds__(block_size) void update_g_and_u_kernel(
     const auto tidx = threadIdx.x;
     const auto rhs = blockIdx.x;
 
+    if (stop_status[rhs].has_stopped()) {
+        return;
+    }
+
     __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
     ValueType *__restrict__ reduction_helper = reduction_helper_array;
 
     for (size_type i = 0; i < k; i++) {
         __syncthreads();
         ValueType alpha = zero<ValueType>();
-        if (!stop_status[rhs].has_stopped()) {
-            for (size_type j = tidx; j < size; j += block_size) {
-                alpha += p_values[i * p_stride + j] *
-                         g_values[j * g_stride + k * nrhs + rhs];
-            }
+        for (size_type j = tidx; j < size; j += block_size) {
+            alpha += p_values[i * p_stride + j] *
+                     g_values[j * g_stride + k * nrhs + rhs];
         }
         reduction_helper[tidx] = alpha;
 
@@ -166,16 +226,13 @@ __global__ __launch_bounds__(block_size) void update_g_and_u_kernel(
                [](const ValueType &a, const ValueType &b) { return a + b; });
         __syncthreads();
 
-        if (!stop_status[rhs].has_stopped()) {
-            alpha =
-                reduction_helper[0] / m_values[i * m_stride + i * nrhs + rhs];
+        alpha = reduction_helper[0] / m_values[i * m_stride + i * nrhs + rhs];
 
-            for (size_type j = tidx; j < size; j += block_size) {
-                g_values[j * g_stride + k * nrhs + rhs] -=
-                    alpha * g_values[j * g_stride + i * nrhs + rhs];
-                u_values[j * u_stride + k * nrhs + rhs] -=
-                    alpha * u_values[j * u_stride + i * nrhs + rhs];
-            }
+        for (size_type j = tidx; j < size; j += block_size) {
+            g_values[j * g_stride + k * nrhs + rhs] -=
+                alpha * g_values[j * g_stride + i * nrhs + rhs];
+            u_values[j * u_stride + k * nrhs + rhs] -=
+                alpha * u_values[j * u_stride + i * nrhs + rhs];
         }
     }
 }
@@ -193,11 +250,15 @@ __global__ __launch_bounds__(block_size) void update_m_kernel(
     const auto row = blockIdx.x + k;
     const auto col = blockIdx.y;
 
+    if (stop_status[col].has_stopped()) {
+        return;
+    }
+
     __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
     ValueType *__restrict__ reduction_helper = reduction_helper_array;
 
     ValueType local_res = zero<ValueType>();
-    if (row < subspace_dim && col < nrhs && !stop_status[col].has_stopped()) {
+    if (row < subspace_dim && col < nrhs) {
         for (size_type i = tidx; i < size; i += block_size) {
             local_res += p_values[row * p_stride + i] *
                          g_values[i * g_stride + k * nrhs + col];
@@ -207,7 +268,7 @@ __global__ __launch_bounds__(block_size) void update_m_kernel(
     __syncthreads();
     reduce(group::this_thread_block(), reduction_helper,
            [](const ValueType &a, const ValueType &b) { return a + b; });
-    if (tidx == 0 && !stop_status[col].has_stopped()) {
+    if (tidx == 0) {
         m_values[row * m_stride + k * nrhs + col] = reduction_helper[0];
     }
 }
@@ -265,12 +326,14 @@ __global__ __launch_bounds__(config::warp_size) void compute_omega_kernel(
         return;
     }
 
-    auto thr = omega[global_id];
-    omega[global_id] /= tht[global_id];
-    auto absrho =
-        abs(thr / (sqrt(real(tht[global_id])) * residual_norm[global_id]));
+    if (!stop_status[global_id].has_stopped()) {
+        auto thr = omega[global_id];
+        omega[global_id] /= tht[global_id];
+        auto absrho =
+            abs(thr / (sqrt(real(tht[global_id])) * residual_norm[global_id]));
 
-    if (absrho < kappa) {
-        omega[global_id] *= kappa / absrho;
+        if (absrho < kappa) {
+            omega[global_id] *= kappa / absrho;
+        }
     }
 }
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index a03e8dc67fb..c355189dcbc 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -132,7 +132,8 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     // Initialization
     // m = identity
-    exec->run(idr::make_initialize(m.get(), &stop_status));
+    exec->run(
+        idr::make_initialize(m.get(), subspace_vectors_.get(), &stop_status));
 
     // omega = 1
     exec->run(
diff --git a/core/solver/idr_kernels.hpp b/core/solver/idr_kernels.hpp
index f26d58c0196..f1d4e5dbb4e 100644
--- a/core/solver/idr_kernels.hpp
+++ b/core/solver/idr_kernels.hpp
@@ -49,6 +49,7 @@ namespace idr {
 #define GKO_DECLARE_IDR_INITIALIZE_KERNEL(_type)                 \
     void initialize(std::shared_ptr<const DefaultExecutor> exec, \
                     matrix::Dense<_type> *m,                     \
+                    matrix::Dense<_type> *subspace_vectors,      \
                     Array<stopping_status> *stop_status)
 
 
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index 480aff891f5..6cc8b7f76ef 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -33,6 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
+#include <random>
+
+
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 
@@ -64,6 +67,33 @@ constexpr int default_block_size = 512;
 namespace {
 
 
+template <typename ValueType>
+void initialize_m(matrix::Dense<ValueType> *m,
+                  Array<stopping_status> *stop_status)
+{
+    const auto subspace_dim = m->get_size()[0];
+    const auto nrhs = m->get_size()[1] / subspace_dim;
+    const auto m_stride = m->get_stride();
+
+    const auto grid_dim = ceildiv(m_stride * subspace_dim, default_block_size);
+    initialize_m_kernel<<<grid_dim, default_block_size>>>(
+        subspace_dim, nrhs, as_cuda_type(m->get_values()), m_stride,
+        as_cuda_type(stop_status->get_data()));
+}
+
+
+template <typename ValueType>
+void orthonormalize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors)
+{
+    orthonormalize_subspace_vectors_kernel<default_block_size>
+        <<<1, default_block_size>>>(
+            subspace_vectors->get_size()[0], subspace_vectors->get_size()[1],
+            as_cuda_type(subspace_vectors->get_values()),
+            subspace_vectors->get_stride());
+    )
+}
+
+
 template <typename ValueType>
 void solve_lower_triangular(const matrix::Dense<ValueType> *m,
                             const matrix::Dense<ValueType> *f,
@@ -149,16 +179,11 @@ void update_x_r_and_f(size_type k, const matrix::Dense<ValueType> *m,
 template <typename ValueType>
 void initialize(std::shared_ptr<const CudaExecutor> exec,
                 matrix::Dense<ValueType> *m,
+                matrix::Dense<ValueType> *subspace_vectors,
                 Array<stopping_status> *stop_status)
 {
-    const auto subspace_dim = m->get_size()[0];
-    const auto nrhs = m->get_size()[1] / subspace_dim;
-    const auto m_stride = m->get_stride();
-
-    const auto grid_dim = ceildiv(m_stride * subspace_dim, default_block_size);
-    initialize_kernel<<<grid_dim, default_block_size>>>(
-        subspace_dim, nrhs, as_cuda_type(m->get_values()), m_stride,
-        as_cuda_type(stop_status->get_data()));
+    initialize_m(m, stop_status);
+    orthonormalize_subspace_vectors(subspace_vectors);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index e74703380e9..c949ff02f16 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -221,10 +221,13 @@ TEST_F(Idr, CudaIdrInitializeIsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::idr::initialize(ref, m.get(), stop_status.get());
-    gko::kernels::cuda::idr::initialize(cuda, d_m.get(), d_stop_status.get());
+    gko::kernels::reference::idr::initialize(ref, m.get(), p.get(),
+                                             stop_status.get());
+    gko::kernels::cuda::idr::initialize(cuda, d_m.get(), d_p.get(),
+                                        d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14)
+    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
+    GKO_ASSERT_MTX_NEAR(p, d_p, 1e-14);
 }
 
 
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index 8f9a3de1c92..0f8eac7bc87 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -67,6 +67,41 @@ constexpr int default_block_size = 512;
 namespace {
 
 
+template <typename ValueType>
+void initialize_m(matrix::Dense<ValueType> *m,
+                  Array<stopping_status> *stop_status)
+{
+    const auto subspace_dim = m->get_size()[0];
+    const auto nrhs = m->get_size()[1] / subspace_dim;
+    const auto m_stride = m->get_stride();
+
+    const auto grid_dim = ceildiv(m_stride * subspace_dim, default_block_size);
+    hipLaunchKernelGGL(HIP_KERNEL_NAME(initialize_m_kernel), grid_dim,
+                       default_block_size, 0, 0 subspace_dim, nrhs,
+                       as_hip_type(m->get_values()), m_stride,
+                       as_hip_type(stop_status->get_data()));
+}
+
+
+template <typename ValueType>
+void orthonormalize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors)
+{
+    auto subspace_vectors_data = matrix_data<ValueType>(
+        subspace_vectors->get_size(), std::normal_distribution<>(0.0, 1.0),
+        std::ranlux48(15));
+    subspace_vectors->read(subspace_vectors_data);
+
+    hipLaunchKernelGGL(
+        HIP_KERNEL_NAME(
+            orthonormalize_subspace_vectors_kernel<default_block_size>),
+        1, default_block_size, 0, 0, subspace_vectors->get_size()[0],
+        subspace_vectors->get_size()[1],
+        as_hip_type(subspace_vectors->get_values()),
+        subspace_vectors->get_stride());
+    )
+}
+
+
 template <typename ValueType>
 void solve_lower_triangular(const matrix::Dense<ValueType> *m,
                             const matrix::Dense<ValueType> *f,
@@ -158,17 +193,11 @@ void update_x_r_and_f(size_type k, const matrix::Dense<ValueType> *m,
 template <typename ValueType>
 void initialize(std::shared_ptr<const HipExecutor> exec,
                 matrix::Dense<ValueType> *m,
+                matrix::Dense<ValueType> *subspace_vectors,
                 Array<stopping_status> *stop_status)
 {
-    const auto subspace_dim = m->get_size()[0];
-    const auto nrhs = m->get_size()[1] / subspace_dim;
-    const auto m_stride = m->get_stride();
-
-    const auto grid_dim = ceildiv(m_stride * subspace_dim, default_block_size);
-    hipLaunchKernelGGL(HIP_KERNEL_NAME(initialize_kernel), grid_dim,
-                       default_block_size, 0, 0, subspace_dim, nrhs,
-                       as_hip_type(m->get_values()), m_stride,
-                       as_hip_type(stop_status->get_data()));
+    initialize_m(m, stop_status);
+    initialize_subspace_vectors(subspace_vectors);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index 27445562039..9b94be3b297 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -221,10 +221,13 @@ TEST_F(Idr, HipIdrInitializeIsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::idr::initialize(ref, m.get(), stop_status.get());
-    gko::kernels::hip::idr::initialize(hip, d_m.get(), d_stop_status.get());
+    gko::kernels::reference::idr::initialize(ref, m.get(), p.get(),
+                                             stop_status.get());
+    gko::kernels::hip::idr::initialize(hip, d_m.get(), d_p.get(),
+                                       d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14)
+    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
+    GKO_ASSERT_MTX_NEAR(p, d_p, 1e-14);
 }
 
 
diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index 864d8f789e3..32bcd8240d9 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -202,7 +202,9 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
 
         subspace_vectors_ =
             matrix::Dense<ValueType>::create(factory->get_executor());
+
         subspace_vectors_->read(subspace_vectors_data);
+
         kappa_ = parameters_.kappa;
     }
 
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index 64af234291d..77d90c967ff 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
+#include <random>
 
 
 #include <omp.h>
@@ -61,11 +62,16 @@ namespace {
 template <typename ValueType>
 void solve_lower_triangular(const matrix::Dense<ValueType> *m,
                             const matrix::Dense<ValueType> *f,
-                            matrix::Dense<ValueType> *c)
+                            matrix::Dense<ValueType> *c,
+                            const Array<stopping_status> *stop_status)
 {
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
 #pragma omp parallel for
     for (size_type i = 0; i < f->get_size()[1]; i++) {
+        if (stop_status->get_const_data()[i].has_stopped()) {
+            continue;
+        }
+
         for (size_type row = 0; row < m->get_size()[0]; row++) {
             auto temp = f->at(row, i);
             for (size_type col = 0; col < row; col++) {
@@ -80,11 +86,16 @@ void solve_lower_triangular(const matrix::Dense<ValueType> *m,
 template <typename ValueType>
 void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
                     const matrix::Dense<ValueType> *m,
-                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *u)
+                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *u,
+                    const Array<stopping_status> *stop_status)
 {
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
 #pragma omp parallel for
     for (size_type i = 0; i < nrhs; i++) {
+        if (stop_status->get_const_data()[i].has_stopped()) {
+            continue;
+        }
+
         for (size_type j = 0; j < k; j++) {
             auto alpha = zero<ValueType>();
             for (size_type ind = 0; ind < p->get_size()[1]; ind++) {
@@ -107,10 +118,13 @@ void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
 template <typename ValueType>
 void initialize(std::shared_ptr<const OmpExecutor> exec,
                 matrix::Dense<ValueType> *m,
+                matrix::Dense<ValueType> *subspace_vectors,
                 Array<stopping_status> *stop_status)
 {
+#pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
 
+    // Initialize M
 #pragma omp parallel for
     for (size_type i = 0; i < nrhs; i++) {
         stop_status->get_data()[i].reset();
@@ -123,6 +137,38 @@ void initialize(std::shared_ptr<const OmpExecutor> exec,
                 (row == col / nrhs) ? one<ValueType>() : zero<ValueType>();
         }
     }
+
+    // Orthonormalize P
+    const auto num_rows = subspace_vectors->get_size()[0];
+    const auto num_cols = subspace_vectors->get_size()[1];
+    for (size_type row = 0; row < num_rows; row++) {
+        for (size_type i = 0; i < row; i++) {
+            auto dot = zero<ValueType>();
+#pragma omp parallel for reduction(add : dot)
+            for (size_type j = 0; j < num_cols; j++) {
+                dot += subspace_vectors->at(row, j) *
+                       conj(subspace_vectors->at(i, j));
+            }
+#pragma omp parallel for
+            for (size_type j = 0; j < num_cols; j++) {
+                subspace_vectors->at(row, j) -=
+                    dot * subspace_vectors->at(i, j);
+            }
+        }
+
+        auto norm = zero<remove_complex<ValueType>>();
+#pragma omp parallel for reduction(+ : norm)
+        for (size_type j = 0; j < num_cols; j++) {
+            norm += squared_norm(subspace_vectors->at(row, j));
+        }
+
+        norm = sqrt(norm);
+
+#pragma omp parallel for
+        for (size_type j = 0; j < num_cols; j++) {
+            subspace_vectors->at(row, j) /= norm;
+        }
+    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
@@ -140,16 +186,14 @@ void step_1(std::shared_ptr<const OmpExecutor> exec, const size_type k,
     const auto m_size = m->get_size();
     const auto nrhs = f->get_size()[1];
 
+    // Compute c = M \ f
+    solve_lower_triangular(m, f, c, stop_status);
+
     for (size_type i = 0; i < nrhs; i++) {
-        if (stop_status->get_const_data()[0].has_stopped()) {
+        if (stop_status->get_const_data()[i].has_stopped()) {
             continue;
         }
-    }
 
-    // Compute c = M \ f
-    solve_lower_triangular(m, f, c);
-
-    for (size_type i = 0; i < nrhs; i++) {
         // v = residual - c_k * g_k - ... - c_s * g_s
 #pragma omp parallel for
         for (size_type row = 0; row < v->get_size()[0]; row++) {
@@ -174,7 +218,7 @@ void step_2(std::shared_ptr<const OmpExecutor> exec, const size_type k,
 {
     const auto nrhs = omega->get_size()[1];
     for (size_type i = 0; i < nrhs; i++) {
-        if (stop_status->get_const_data()[0].has_stopped()) {
+        if (stop_status->get_const_data()[i].has_stopped()) {
             continue;
         }
 
@@ -202,15 +246,13 @@ void step_3(std::shared_ptr<const OmpExecutor> exec, const size_type k,
 {
     const auto nrhs = x->get_size()[1];
 
+    update_g_and_u(k, p, m, g, u, stop_status);
+
     for (size_type i = 0; i < nrhs; i++) {
-        if (stop_status->get_const_data()[0].has_stopped()) {
+        if (stop_status->get_const_data()[i].has_stopped()) {
             continue;
         }
-    }
-
-    update_g_and_u(k, p, m, g, u);
 
-    for (size_type i = 0; i < nrhs; i++) {
 #pragma omp parallel for
         for (size_type j = k; j < m->get_size()[0]; j++) {
             auto temp = zero<ValueType>();
@@ -250,7 +292,7 @@ void compute_omega(
 {
 #pragma omp parallel for
     for (size_type i = 0; i < omega->get_size()[1]; i++) {
-        if (stop_status->get_const_data()[0].has_stopped()) {
+        if (stop_status->get_const_data()[i].has_stopped()) {
             continue;
         }
 
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index 3a2ec1643f8..d91e1572877 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -221,10 +221,13 @@ TEST_F(Idr, OmpIdrInitializeIsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::idr::initialize(ref, m.get(), stop_status.get());
-    gko::kernels::omp::idr::initialize(omp, d_m.get(), d_stop_status.get());
+    gko::kernels::reference::idr::initialize(ref, m.get(), p.get(),
+                                             stop_status.get());
+    gko::kernels::omp::idr::initialize(omp, d_m.get(), d_p.get(),
+                                       d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14)
+    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
+    GKO_ASSERT_MTX_NEAR(p, d_p, 1e-14);
 }
 
 
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index 3d15589b317..8a308288af2 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
+#include <random>
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -58,10 +59,15 @@ namespace {
 template <typename ValueType>
 void solve_lower_triangular(const matrix::Dense<ValueType> *m,
                             const matrix::Dense<ValueType> *f,
-                            matrix::Dense<ValueType> *c)
+                            matrix::Dense<ValueType> *c,
+                            const Array<stopping_status> *stop_status)
 {
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
     for (size_type i = 0; i < f->get_size()[1]; i++) {
+        if (stop_status->get_const_data()[i].has_stopped()) {
+            continue;
+        }
+
         for (size_type row = 0; row < m->get_size()[0]; row++) {
             auto temp = f->at(row, i);
             for (size_type col = 0; col < row; col++) {
@@ -76,10 +82,15 @@ void solve_lower_triangular(const matrix::Dense<ValueType> *m,
 template <typename ValueType>
 void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
                     const matrix::Dense<ValueType> *m,
-                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *u)
+                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *u,
+                    const Array<stopping_status> *stop_status)
 {
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
     for (size_type i = 0; i < nrhs; i++) {
+        if (stop_status->get_const_data()[i].has_stopped()) {
+            continue;
+        }
+
         for (size_type j = 0; j < k; j++) {
             auto alpha = zero<ValueType>();
             for (size_type ind = 0; ind < p->get_size()[1]; ind++) {
@@ -101,8 +112,10 @@ void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
 template <typename ValueType>
 void initialize(std::shared_ptr<const ReferenceExecutor> exec,
                 matrix::Dense<ValueType> *m,
+                matrix::Dense<ValueType> *subspace_vectors,
                 Array<stopping_status> *stop_status)
 {
+    // Initialize M
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
     for (size_type i = 0; i < nrhs; i++) {
         stop_status->get_data()[i].reset();
@@ -114,6 +127,34 @@ void initialize(std::shared_ptr<const ReferenceExecutor> exec,
                 (row == col / nrhs) ? one<ValueType>() : zero<ValueType>();
         }
     }
+
+    // Orthonormalize P
+    const auto num_rows = subspace_vectors->get_size()[0];
+    const auto num_cols = subspace_vectors->get_size()[1];
+    for (size_type row = 0; row < num_rows; row++) {
+        for (size_type i = 0; i < row; i++) {
+            auto dot = zero<ValueType>();
+            for (size_type j = 0; j < num_cols; j++) {
+                dot += subspace_vectors->at(row, j) *
+                       conj(subspace_vectors->at(i, j));
+            }
+            for (size_type j = 0; j < num_cols; j++) {
+                subspace_vectors->at(row, j) -=
+                    dot * subspace_vectors->at(i, j);
+            }
+        }
+
+        auto norm = zero<remove_complex<ValueType>>();
+        for (size_type j = 0; j < num_cols; j++) {
+            norm += squared_norm(subspace_vectors->at(row, j));
+        }
+
+        norm = sqrt(norm);
+
+        for (size_type j = 0; j < num_cols; j++) {
+            subspace_vectors->at(row, j) /= norm;
+        }
+    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
@@ -130,16 +171,13 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec, const size_type k,
 {
     const auto nrhs = f->get_size()[1];
 
-    for (size_type i = 0; i < nrhs; i++) {
-        if (stop_status->get_const_data()[0].has_stopped()) {
-            continue;
-        }
-    }
-
     // Compute c = M \ f
-    solve_lower_triangular(m, f, c);
+    solve_lower_triangular(m, f, c, stop_status);
 
     for (size_type i = 0; i < nrhs; i++) {
+        if (stop_status->get_const_data()[i].has_stopped()) {
+            continue;
+        }
         // v = residual - c_k * g_k - ... - c_s * g_s
         for (size_type row = 0; row < v->get_size()[0]; row++) {
             auto temp = residual->at(row, i);
@@ -163,7 +201,7 @@ void step_2(std::shared_ptr<const ReferenceExecutor> exec, const size_type k,
 {
     const auto nrhs = omega->get_size()[1];
     for (size_type i = 0; i < nrhs; i++) {
-        if (stop_status->get_const_data()[0].has_stopped()) {
+        if (stop_status->get_const_data()[i].has_stopped()) {
             continue;
         }
 
@@ -190,15 +228,13 @@ void step_3(std::shared_ptr<const ReferenceExecutor> exec, const size_type k,
 {
     const auto nrhs = x->get_size()[1];
 
+    update_g_and_u(k, p, m, g, u, stop_status);
+
     for (size_type i = 0; i < nrhs; i++) {
-        if (stop_status->get_const_data()[0].has_stopped()) {
+        if (stop_status->get_const_data()[i].has_stopped()) {
             continue;
         }
-    }
-
-    update_g_and_u(k, p, m, g, u);
 
-    for (size_type i = 0; i < nrhs; i++) {
         for (size_type j = k; j < m->get_size()[0]; j++) {
             auto temp = zero<ValueType>();
             for (size_type ind = 0; ind < p->get_size()[1]; ind++) {
@@ -234,7 +270,7 @@ void compute_omega(
     matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
 {
     for (size_type i = 0; i < omega->get_size()[1]; i++) {
-        if (stop_status->get_const_data()[0].has_stopped()) {
+        if (stop_status->get_const_data()[i].has_stopped()) {
             continue;
         }
 
diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp
index 11a6bb0c264..b21527a4622 100644
--- a/reference/test/solver/idr_kernels.cpp
+++ b/reference/test/solver/idr_kernels.cpp
@@ -325,7 +325,7 @@ TYPED_TEST(Idr, SolvesMultipleDenseSystemsDivergenceCheck)
 
     // Not sure if this is necessary, the assertions above should cover what
     // is needed.
-    GKO_ASSERT_MTX_NEAR(xc, testMtx, r<value_type>::value * 1e1);
+    GKO_ASSERT_MTX_NEAR(xc, testMtx, r<value_type>::value * 1e2);
 }
 
 

From 1ef3be234c576ec4bf7ec9cd74f79ab39faee034 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Tue, 25 Aug 2020 15:36:19 +0200
Subject: [PATCH 0145/1008] Move g_k to separate vector instead of using it as
 a submatrix of g

---
 common/solver/idr_kernels.hpp.inc | 31 ++++++++++++++++++++++++-------
 core/solver/idr.cpp               | 21 ++++++++-------------
 core/solver/idr_kernels.hpp       |  7 ++++---
 cuda/solver/idr_kernels.cu        | 15 +++++++++------
 cuda/test/solver/idr_kernels.cpp  | 12 ++++++------
 hip/solver/idr_kernels.hip.cpp    | 22 +++++++++++-----------
 hip/test/solver/idr_kernels.cpp   | 12 ++++++------
 omp/solver/idr_kernels.cpp        | 19 ++++++++++++-------
 omp/test/solver/idr_kernels.cpp   | 12 ++++++------
 reference/solver/idr_kernels.cpp  | 19 ++++++++++++-------
 10 files changed, 98 insertions(+), 72 deletions(-)

diff --git a/common/solver/idr_kernels.hpp.inc b/common/solver/idr_kernels.hpp.inc
index d663afb43d1..45190c03b82 100644
--- a/common/solver/idr_kernels.hpp.inc
+++ b/common/solver/idr_kernels.hpp.inc
@@ -101,7 +101,7 @@ __global__
         __syncthreads();
 
         norm = sqrt(reduction_helper_real[0]);
-        for (size_type j = 0; j < num_cols; j += block_size) {
+        for (size_type j = tidx; j < num_cols; j += block_size) {
             values[row * stride + j] /= norm;
         }
     }
@@ -199,6 +199,7 @@ __global__ __launch_bounds__(block_size) void update_g_and_u_kernel(
     const ValueType *__restrict__ p_values, size_type p_stride,
     const ValueType *__restrict__ m_values, size_type m_stride,
     ValueType *__restrict__ g_values, size_type g_stride,
+    ValueType *__restrict__ g_k_values, size_type g_k_stride,
     ValueType *__restrict__ u_values, size_type u_stride,
     const stopping_status *__restrict__ stop_status)
 {
@@ -216,8 +217,8 @@ __global__ __launch_bounds__(block_size) void update_g_and_u_kernel(
         __syncthreads();
         ValueType alpha = zero<ValueType>();
         for (size_type j = tidx; j < size; j += block_size) {
-            alpha += p_values[i * p_stride + j] *
-                     g_values[j * g_stride + k * nrhs + rhs];
+            alpha +=
+                p_values[i * p_stride + j] * g_k_values[j * g_k_stride + rhs];
         }
         reduction_helper[tidx] = alpha;
 
@@ -229,12 +230,17 @@ __global__ __launch_bounds__(block_size) void update_g_and_u_kernel(
         alpha = reduction_helper[0] / m_values[i * m_stride + i * nrhs + rhs];
 
         for (size_type j = tidx; j < size; j += block_size) {
-            g_values[j * g_stride + k * nrhs + rhs] -=
+            g_k_values[j * g_k_stride + rhs] -=
                 alpha * g_values[j * g_stride + i * nrhs + rhs];
             u_values[j * u_stride + k * nrhs + rhs] -=
                 alpha * u_values[j * u_stride + i * nrhs + rhs];
         }
     }
+
+    for (size_type j = tidx; j < size; j += block_size) {
+        g_values[j * g_stride + k * nrhs + rhs] =
+            g_k_values[j * g_k_stride + rhs];
+    }
 }
 
 
@@ -268,6 +274,8 @@ __global__ __launch_bounds__(block_size) void update_m_kernel(
     __syncthreads();
     reduce(group::this_thread_block(), reduction_helper,
            [](const ValueType &a, const ValueType &b) { return a + b; });
+    __syncthreads();
+
     if (tidx == 0) {
         m_values[row * m_stride + k * nrhs + col] = reduction_helper[0];
     }
@@ -305,9 +313,18 @@ __global__ __launch_bounds__(default_block_size) void update_x_r_and_f_kernel(
             f_values[row * f_stride + col] -=
                 beta * m_values[row * m_stride + k * nrhs + col];
         }
-        if (k == row && row + 1 < subspace_dim) {
-            f_values[row * f_stride + col] = zero<ValueType>();
-        }
+    }
+}
+
+
+template <typename ValueType>
+__global__ __launch_bounds__(config::warp_size) void set_f_zeros(
+    size_type k, size_type nrhs, ValueType *__restrict__ f_values,
+    size_type f_stride)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    if (tidx < nrhs) {
+        f_values[k * f_stride + k * nrhs + tidx] = zero<ValueType>();
     }
 }
 
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index c355189dcbc..cc565886954 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -109,7 +109,7 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     auto residual = Vector::create_with_config_of(dense_b);
     auto v = Vector::create_with_config_of(dense_b);
     auto t = Vector::create_with_config_of(dense_b);
-    auto preconditioned_vector = Vector::create_with_config_of(dense_b);
+    auto helper = Vector::create_with_config_of(dense_b);
 
     auto m =
         Vector::create(exec, gko::dim<2>{subspace_dim_, subspace_dim_ * nrhs});
@@ -180,24 +180,20 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             // c = M \ f = (c_1, ..., c_s)^T
             // v = residual - c_k * g_k - ... - c_s * g_s
 
-            get_preconditioner()->apply(v.get(), preconditioned_vector.get());
+            get_preconditioner()->apply(v.get(), helper.get());
 
-            exec->run(idr::make_step_2(k, omega.get(),
-                                       preconditioned_vector.get(), c.get(),
+            exec->run(idr::make_step_2(k, omega.get(), helper.get(), c.get(),
                                        u.get(), &stop_status));
             // u_k = omega * preconditioned_vector + c_k * u_k + ... + c_s * u_s
 
             auto u_k = u->create_submatrix(span{0, problem_size},
                                            span{k * nrhs, (k + 1) * nrhs});
 
-            auto g_k = g->create_submatrix(span{0, problem_size},
-                                           span{k * nrhs, (k + 1) * nrhs});
-
-            system_matrix_->apply(u_k.get(), g_k.get());
+            system_matrix_->apply(u_k.get(), helper.get());
             // g_k = Au_k
 
             exec->run(idr::make_step_3(k, subspace_vectors_.get(), g.get(),
-                                       u.get(), m.get(), f.get(),
+                                       helper.get(), u.get(), m.get(), f.get(),
                                        residual.get(), dense_x, &stop_status));
             // for i = 1 to k - 1 do
             //     alpha = p^H_i * g_k / m_i,i
@@ -213,9 +209,8 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             // f = (0,...,0,f_k+1 - beta * m_k+1,k,...,f_s - beta * m_s,k)
         }
 
-        get_preconditioner()->apply(residual.get(),
-                                    preconditioned_vector.get());
-        system_matrix_->apply(preconditioned_vector.get(), t.get());
+        get_preconditioner()->apply(residual.get(), helper.get());
+        system_matrix_->apply(helper.get(), t.get());
 
         t->compute_dot(residual.get(), omega.get());
         // t->compute_norm2(t_norm.get());
@@ -227,7 +222,7 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
         t->scale(neg_one_op.get());
         residual->add_scaled(omega.get(), t.get());
-        dense_x->add_scaled(omega.get(), preconditioned_vector.get());
+        dense_x->add_scaled(omega.get(), helper.get());
 
         // omega = (t^H * residual) / (t^H * t)
         // rho = (t^H * residual) / (norm(t) * norm(residual))
diff --git a/core/solver/idr_kernels.hpp b/core/solver/idr_kernels.hpp
index f1d4e5dbb4e..0cf01700c61 100644
--- a/core/solver/idr_kernels.hpp
+++ b/core/solver/idr_kernels.hpp
@@ -73,9 +73,10 @@ namespace idr {
 #define GKO_DECLARE_IDR_STEP_3_KERNEL(_type)                             \
     void step_3(std::shared_ptr<const DefaultExecutor> exec,             \
                 const size_type k, const matrix::Dense<_type> *p,        \
-                matrix::Dense<_type> *g, matrix::Dense<_type> *u,        \
-                matrix::Dense<_type> *m, matrix::Dense<_type> *f,        \
-                matrix::Dense<_type> *residual, matrix::Dense<_type> *x, \
+                matrix::Dense<_type> *g, matrix::Dense<_type> *g_k,      \
+                matrix::Dense<_type> *u, matrix::Dense<_type> *m,        \
+                matrix::Dense<_type> *f, matrix::Dense<_type> *residual, \
+                matrix::Dense<_type> *x,                                 \
                 const Array<stopping_status> *stop_status)
 
 
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index 6cc8b7f76ef..a2654a67c5f 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -90,7 +90,6 @@ void orthonormalize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors)
             subspace_vectors->get_size()[0], subspace_vectors->get_size()[1],
             as_cuda_type(subspace_vectors->get_values()),
             subspace_vectors->get_stride());
-    )
 }
 
 
@@ -115,7 +114,8 @@ void solve_lower_triangular(const matrix::Dense<ValueType> *m,
 template <typename ValueType>
 void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
                     const matrix::Dense<ValueType> *m,
-                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *u,
+                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *g_k,
+                    matrix::Dense<ValueType> *u,
                     const Array<stopping_status> *stop_status)
 {
     const auto size = g->get_size()[0];
@@ -126,6 +126,7 @@ void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
         k, size, nrhs, as_cuda_type(p->get_const_values()), p->get_stride(),
         as_cuda_type(m->get_const_values()), m->get_stride(),
         as_cuda_type(g->get_values()), g->get_stride(),
+        as_cuda_type(g_k->get_values()), g_k->get_stride(),
         as_cuda_type(u->get_values()), u->get_stride(),
         as_cuda_type(stop_status->get_const_data()));
 }
@@ -170,6 +171,8 @@ void update_x_r_and_f(size_type k, const matrix::Dense<ValueType> *m,
         as_cuda_type(r->get_values()), r->get_stride(),
         as_cuda_type(x->get_values()), x->get_stride(),
         as_cuda_type(stop_status->get_const_data()));
+    set_f_zeros<<<ceildiv(nrhs, config::warp_size), config::warp_size>>>(
+        k, nrhs, as_cuda_type(f->get_values()), f->get_stride());
 }
 
 
@@ -247,12 +250,12 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 template <typename ValueType>
 void step_3(std::shared_ptr<const CudaExecutor> exec, const size_type k,
             const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
-            matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
-            matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *residual,
-            matrix::Dense<ValueType> *x,
+            matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *u,
+            matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
+            matrix::Dense<ValueType> *residual, matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status)
 {
-    update_g_and_u(k, p, m, g, u, stop_status);
+    update_g_and_u(k, p, m, g, g_k, u, stop_status);
     update_m(k, p, g, m, stop_status);
     update_x_r_and_f(k, m, g, u, f, residual, x, stop_status);
 }
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index c949ff02f16..898d55ea6f0 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -267,12 +267,12 @@ TEST_F(Idr, CudaIdrStep3IsEquivalentToRef)
     initialize_data();
 
     gko::size_type k = 2;
-    gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), u.get(),
-                                         m.get(), f.get(), r.get(), x.get(),
-                                         stop_status.get());
-    gko::kernels::cuda::idr::step_3(cuda, k, d_p.get(), d_g.get(), d_u.get(),
-                                    d_m.get(), d_f.get(), d_r.get(), d_x.get(),
-                                    d_stop_status.get());
+    gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), v.get(),
+                                         u.get(), m.get(), f.get(), r.get(),
+                                         x.get(), stop_status.get());
+    gko::kernels::cuda::idr::step_3(cuda, k, d_p.get(), d_g.get(), d_v.get(),
+                                    d_u.get(), d_m.get(), d_f.get(), d_r.get(),
+                                    d_x.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(g, d_g, 1e-14);
     GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index 0f8eac7bc87..a8985720404 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -86,11 +86,6 @@ void initialize_m(matrix::Dense<ValueType> *m,
 template <typename ValueType>
 void orthonormalize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors)
 {
-    auto subspace_vectors_data = matrix_data<ValueType>(
-        subspace_vectors->get_size(), std::normal_distribution<>(0.0, 1.0),
-        std::ranlux48(15));
-    subspace_vectors->read(subspace_vectors_data);
-
     hipLaunchKernelGGL(
         HIP_KERNEL_NAME(
             orthonormalize_subspace_vectors_kernel<default_block_size>),
@@ -98,7 +93,6 @@ void orthonormalize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors)
         subspace_vectors->get_size()[1],
         as_hip_type(subspace_vectors->get_values()),
         subspace_vectors->get_stride());
-    )
 }
 
 
@@ -124,7 +118,8 @@ void solve_lower_triangular(const matrix::Dense<ValueType> *m,
 template <typename ValueType>
 void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
                     const matrix::Dense<ValueType> *m,
-                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *u,
+                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *g_k,
+                    matrix::Dense<ValueType> *u,
                     const Array<stopping_status> *stop_status)
 {
     const auto size = g->get_size()[0];
@@ -137,6 +132,7 @@ void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
         as_hip_type(p->get_const_values()), p->get_stride(),
         as_hip_type(m->get_const_values()), m->get_stride(),
         as_hip_type(g->get_values()), g->get_stride(),
+        as_hip_type(g_k->get_values()), g_k->get_stride(),
         as_hip_type(u->get_values()), u->get_stride(),
         as_hip_type(stop_status->get_const_data()));
 }
@@ -184,6 +180,10 @@ void update_x_r_and_f(size_type k, const matrix::Dense<ValueType> *m,
                        as_hip_type(r->get_values()), r->get_stride(),
                        as_hip_type(x->get_values()), x->get_stride(),
                        as_hip_type(stop_status->get_const_data()));
+    hipLaunchKernelGGL(HIP_KERNEL_NAME(set_f_zeros),
+                       ceildiv(nrhs, config::warp_size), config::warp_size, 0,
+                       0, k, nrhs, as_hip_type(f->get_values()),
+                       f->get_stride());
 }
 
 
@@ -260,12 +260,12 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 template <typename ValueType>
 void step_3(std::shared_ptr<const HipExecutor> exec, const size_type k,
             const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
-            matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
-            matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *residual,
-            matrix::Dense<ValueType> *x,
+            matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *u,
+            matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
+            matrix::Dense<ValueType> *residual, matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status)
 {
-    update_g_and_u(k, p, m, g, u, stop_status);
+    update_g_and_u(k, p, m, g, g_k, u, stop_status);
     update_m(k, p, g, m, stop_status);
     update_x_r_and_f(k, m, g, u, f, residual, x, stop_status);
 }
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index 9b94be3b297..0ce13008973 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -267,12 +267,12 @@ TEST_F(Idr, HipIdrStep3IsEquivalentToRef)
     initialize_data();
 
     gko::size_type k = 2;
-    gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), u.get(),
-                                         m.get(), f.get(), r.get(), x.get(),
-                                         stop_status.get());
-    gko::kernels::hip::idr::step_3(hip, k, d_p.get(), d_g.get(), d_u.get(),
-                                   d_m.get(), d_f.get(), d_r.get(), d_x.get(),
-                                   d_stop_status.get());
+    gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), v.get(),
+                                         u.get(), m.get(), f.get(), r.get(),
+                                         x.get(), stop_status.get());
+    gko::kernels::hip::idr::step_3(hip, k, d_p.get(), d_g.get(), d_v.get(),
+                                   d_u.get(), d_m.get(), d_f.get(), d_r.get(),
+                                   d_x.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(g, d_g, 1e-14);
     GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index 77d90c967ff..171405fd198 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -86,7 +86,8 @@ void solve_lower_triangular(const matrix::Dense<ValueType> *m,
 template <typename ValueType>
 void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
                     const matrix::Dense<ValueType> *m,
-                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *u,
+                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *g_k,
+                    matrix::Dense<ValueType> *u,
                     const Array<stopping_status> *stop_status)
 {
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
@@ -99,15 +100,19 @@ void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
         for (size_type j = 0; j < k; j++) {
             auto alpha = zero<ValueType>();
             for (size_type ind = 0; ind < p->get_size()[1]; ind++) {
-                alpha += p->at(j, ind) * g->at(ind, k * nrhs + i);
+                alpha += p->at(j, ind) * g_k->at(ind, i);
             }
             alpha /= m->at(j, j * nrhs + i);
 
             for (size_type row = 0; row < g->get_size()[0]; row++) {
-                g->at(row, k * nrhs + i) -= alpha * g->at(row, j * nrhs + i);
+                g_k->at(row, i) -= alpha * g->at(row, j * nrhs + i);
                 u->at(row, k * nrhs + i) -= alpha * u->at(row, j * nrhs + i);
             }
         }
+
+        for (size_type row = 0; row < g->get_size()[0]; row++) {
+            g->at(row, k * nrhs + i) = g_k->at(row, i);
+        }
     }
 }
 
@@ -239,14 +244,14 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 template <typename ValueType>
 void step_3(std::shared_ptr<const OmpExecutor> exec, const size_type k,
             const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
-            matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
-            matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *residual,
-            matrix::Dense<ValueType> *x,
+            matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *u,
+            matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
+            matrix::Dense<ValueType> *residual, matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status)
 {
     const auto nrhs = x->get_size()[1];
 
-    update_g_and_u(k, p, m, g, u, stop_status);
+    update_g_and_u(k, p, m, g, g_k, u, stop_status);
 
     for (size_type i = 0; i < nrhs; i++) {
         if (stop_status->get_const_data()[i].has_stopped()) {
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index d91e1572877..f93e994389f 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -267,12 +267,12 @@ TEST_F(Idr, OmpIdrStep3IsEquivalentToRef)
     initialize_data();
 
     gko::size_type k = 2;
-    gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), u.get(),
-                                         m.get(), f.get(), r.get(), x.get(),
-                                         stop_status.get());
-    gko::kernels::omp::idr::step_3(omp, k, d_p.get(), d_g.get(), d_u.get(),
-                                   d_m.get(), d_f.get(), d_r.get(), d_x.get(),
-                                   d_stop_status.get());
+    gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), v.get(),
+                                         u.get(), m.get(), f.get(), r.get(),
+                                         x.get(), stop_status.get());
+    gko::kernels::omp::idr::step_3(omp, k, d_p.get(), d_g.get(), d_v.get(),
+                                   d_u.get(), d_m.get(), d_f.get(), d_r.get(),
+                                   d_x.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(g, d_g, 1e-14);
     GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index 8a308288af2..22b7371ccc8 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -82,7 +82,8 @@ void solve_lower_triangular(const matrix::Dense<ValueType> *m,
 template <typename ValueType>
 void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
                     const matrix::Dense<ValueType> *m,
-                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *u,
+                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *g_k,
+                    matrix::Dense<ValueType> *u,
                     const Array<stopping_status> *stop_status)
 {
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
@@ -94,14 +95,18 @@ void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
         for (size_type j = 0; j < k; j++) {
             auto alpha = zero<ValueType>();
             for (size_type ind = 0; ind < p->get_size()[1]; ind++) {
-                alpha += p->at(j, ind) * g->at(ind, k * nrhs + i);
+                alpha += p->at(j, ind) * g_k->at(ind, i);
             }
             alpha /= m->at(j, j * nrhs + i);
             for (size_type row = 0; row < g->get_size()[0]; row++) {
-                g->at(row, k * nrhs + i) -= alpha * g->at(row, j * nrhs + i);
+                g_k->at(row, i) -= alpha * g->at(row, j * nrhs + i);
                 u->at(row, k * nrhs + i) -= alpha * u->at(row, j * nrhs + i);
             }
         }
+
+        for (size_type row = 0; row < g->get_size()[0]; row++) {
+            g->at(row, k * nrhs + i) = g_k->at(row, i);
+        }
     }
 }
 
@@ -221,14 +226,14 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 template <typename ValueType>
 void step_3(std::shared_ptr<const ReferenceExecutor> exec, const size_type k,
             const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
-            matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
-            matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *residual,
-            matrix::Dense<ValueType> *x,
+            matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *u,
+            matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
+            matrix::Dense<ValueType> *residual, matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status)
 {
     const auto nrhs = x->get_size()[1];
 
-    update_g_and_u(k, p, m, g, u, stop_status);
+    update_g_and_u(k, p, m, g, g_k, u, stop_status);
 
     for (size_type i = 0; i < nrhs; i++) {
         if (stop_status->get_const_data()[i].has_stopped()) {

From a527e45e5f10ae8369e8e7d2e9aef9dded562cef Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Wed, 26 Aug 2020 16:14:02 +0200
Subject: [PATCH 0146/1008] Add multidot to optimize update_g_and_u

---
 common/solver/idr_kernels.hpp.inc | 114 ++++++++++++++++++++----------
 core/solver/idr.cpp               |   4 +-
 core/solver/idr_kernels.hpp       |   4 +-
 cuda/solver/idr_kernels.cu        |  60 ++++++++++++----
 cuda/test/solver/idr_kernels.cpp  |  10 +--
 hip/solver/idr_kernels.hip.cpp    |  59 ++++++++++++----
 hip/test/solver/idr_kernels.cpp   |  10 +--
 omp/solver/idr_kernels.cpp        |   3 +-
 omp/test/solver/idr_kernels.cpp   |  10 +--
 reference/solver/idr_kernels.cpp  |   3 +-
 10 files changed, 195 insertions(+), 82 deletions(-)

diff --git a/common/solver/idr_kernels.hpp.inc b/common/solver/idr_kernels.hpp.inc
index 45190c03b82..f033d8d4446 100644
--- a/common/solver/idr_kernels.hpp.inc
+++ b/common/solver/idr_kernels.hpp.inc
@@ -193,53 +193,95 @@ __global__ __launch_bounds__(default_block_size) void step_2_kernel(
 }
 
 
-template <size_type block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void update_g_and_u_kernel(
-    size_type k, size_type size, size_type nrhs,
-    const ValueType *__restrict__ p_values, size_type p_stride,
-    const ValueType *__restrict__ m_values, size_type m_stride,
-    ValueType *__restrict__ g_values, size_type g_stride,
-    ValueType *__restrict__ g_k_values, size_type g_k_stride,
-    ValueType *__restrict__ u_values, size_type u_stride,
+template <typename ValueType>
+__global__ __launch_bounds__(default_dot_size) void multidot_kernel(
+    size_type i, size_type num_rows, size_type nrhs,
+    const ValueType *__restrict__ p_i, const ValueType *__restrict__ g_k,
+    size_type g_k_stride, ValueType *__restrict__ alpha,
     const stopping_status *__restrict__ stop_status)
 {
     const auto tidx = threadIdx.x;
-    const auto rhs = blockIdx.x;
+    const auto tidy = threadIdx.y;
+    const auto rhs = blockIdx.x * default_dot_dim + tidx;
+    const auto num = ceildiv(num_rows, gridDim.y);
+    const auto start_row = blockIdx.y * num;
+    const auto end_row =
+        ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+    __shared__
+        UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
+            reduction_helper_array;
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
 
-    if (stop_status[rhs].has_stopped()) {
-        return;
+    ValueType local_res = zero<ValueType>();
+    if (rhs < nrhs && !stop_status[rhs].has_stopped()) {
+        for (size_type i = start_row + tidy; i < end_row;
+             i += default_dot_dim) {
+            const auto g_idx = i * g_k_stride + rhs;
+            local_res += p_i[i] * g_k[g_idx];
+        }
     }
+    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
+    __syncthreads();
+    local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
+    const auto tile_block =
+        group::tiled_partition<default_dot_dim>(group::this_thread_block());
+    const auto sum =
+        reduce(tile_block, local_res,
+               [](const ValueType &a, const ValueType &b) { return a + b; });
+    const auto new_rhs = blockIdx.x * default_dot_dim + tidy;
+    if (tidx == 0 && new_rhs < nrhs && !stop_status[new_rhs].has_stopped()) {
+        atomic_add(alpha + new_rhs, sum);
+    }
+}
 
-    __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
-    ValueType *__restrict__ reduction_helper = reduction_helper_array;
 
-    for (size_type i = 0; i < k; i++) {
-        __syncthreads();
-        ValueType alpha = zero<ValueType>();
-        for (size_type j = tidx; j < size; j += block_size) {
-            alpha +=
-                p_values[i * p_stride + j] * g_k_values[j * g_k_stride + rhs];
-        }
-        reduction_helper[tidx] = alpha;
+template <size_type block_size, typename ValueType>
+__global__ __launch_bounds__(block_size) void update_g_k_and_u_kernel(
+    size_type k, size_type i, size_type size, size_type nrhs,
+    const ValueType *__restrict__ alpha, const ValueType *__restrict__ m_values,
+    size_type m_stride, const ValueType *__restrict__ g_values,
+    size_type g_stride, ValueType *__restrict__ g_k_values,
+    size_type g_k_stride, ValueType *__restrict__ u_values, size_type u_stride,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    const auto row = tidx / g_k_stride;
+    const auto rhs = tidx % g_k_stride;
 
-        __syncthreads();
-        reduce(group::this_thread_block(), reduction_helper,
-               [](const ValueType &a, const ValueType &b) { return a + b; });
-        __syncthreads();
+    if (row >= size || rhs >= nrhs) {
+        return;
+    }
+
+    if (!stop_status[rhs].has_stopped()) {
+        const auto fact = alpha[rhs] / m_values[i * m_stride + i * nrhs + rhs];
+        g_k_values[row * g_k_stride + rhs] -=
+            fact * g_values[row * g_stride + i * nrhs + rhs];
+        u_values[row * u_stride + k * nrhs + rhs] -=
+            fact * u_values[row * u_stride + i * nrhs + rhs];
+    }
+}
 
-        alpha = reduction_helper[0] / m_values[i * m_stride + i * nrhs + rhs];
 
-        for (size_type j = tidx; j < size; j += block_size) {
-            g_k_values[j * g_k_stride + rhs] -=
-                alpha * g_values[j * g_stride + i * nrhs + rhs];
-            u_values[j * u_stride + k * nrhs + rhs] -=
-                alpha * u_values[j * u_stride + i * nrhs + rhs];
-        }
+template <size_type block_size, typename ValueType>
+__global__ __launch_bounds__(block_size) void update_g_kernel(
+    size_type k, size_type size, size_type nrhs,
+    const ValueType *__restrict__ g_k_values, size_type g_k_stride,
+    ValueType *__restrict__ g_values, size_type g_stride,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    const auto row = tidx / g_k_stride;
+    const auto rhs = tidx % nrhs;
+
+    if (row >= size || rhs >= nrhs) {
+        return;
     }
 
-    for (size_type j = tidx; j < size; j += block_size) {
-        g_values[j * g_stride + k * nrhs + rhs] =
-            g_k_values[j * g_k_stride + rhs];
+    if (!stop_status[rhs].has_stopped()) {
+        g_values[row * g_stride + k * nrhs + rhs] =
+            g_k_values[row * g_k_stride + rhs];
     }
 }
 
@@ -324,7 +366,7 @@ __global__ __launch_bounds__(config::warp_size) void set_f_zeros(
 {
     const auto tidx = thread::get_thread_id_flat();
     if (tidx < nrhs) {
-        f_values[k * f_stride + k * nrhs + tidx] = zero<ValueType>();
+        f_values[k * f_stride + tidx] = zero<ValueType>();
     }
 }
 
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index cc565886954..6a80dad0f4b 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -126,6 +126,7 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     auto residual_norm = NormVector::create(exec, dim<2>{1, nrhs});
     auto tht = Vector::create(exec, dim<2>{1, nrhs});
     auto t_norm = NormVector::create(exec, dim<2>{1, nrhs});
+    auto alpha = Vector::create(exec, gko::dim<2>{1, nrhs});
 
     bool one_changed{};
     Array<stopping_status> stop_status(exec, nrhs);
@@ -194,7 +195,8 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
             exec->run(idr::make_step_3(k, subspace_vectors_.get(), g.get(),
                                        helper.get(), u.get(), m.get(), f.get(),
-                                       residual.get(), dense_x, &stop_status));
+                                       alpha.get(), residual.get(), dense_x,
+                                       &stop_status));
             // for i = 1 to k - 1 do
             //     alpha = p^H_i * g_k / m_i,i
             //     g_k -= alpha * g_i
diff --git a/core/solver/idr_kernels.hpp b/core/solver/idr_kernels.hpp
index 0cf01700c61..ed12a49f0b0 100644
--- a/core/solver/idr_kernels.hpp
+++ b/core/solver/idr_kernels.hpp
@@ -75,8 +75,8 @@ namespace idr {
                 const size_type k, const matrix::Dense<_type> *p,        \
                 matrix::Dense<_type> *g, matrix::Dense<_type> *g_k,      \
                 matrix::Dense<_type> *u, matrix::Dense<_type> *m,        \
-                matrix::Dense<_type> *f, matrix::Dense<_type> *residual, \
-                matrix::Dense<_type> *x,                                 \
+                matrix::Dense<_type> *f, matrix::Dense<_type> *alpha,    \
+                matrix::Dense<_type> *residual, matrix::Dense<_type> *x, \
                 const Array<stopping_status> *stop_status)
 
 
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index a2654a67c5f..97c1ca777b2 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -40,8 +40,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
+#include "core/components/fill_array.hpp"
+#include "cuda/base/config.hpp"
+#include "cuda/base/cublas_bindings.hpp"
 #include "cuda/base/math.hpp"
 #include "cuda/base/types.hpp"
+#include "cuda/components/atomic.cuh"
 #include "cuda/components/cooperative_groups.cuh"
 #include "cuda/components/reduction.cuh"
 #include "cuda/components/thread_ids.cuh"
@@ -59,6 +63,8 @@ namespace idr {
 
 
 constexpr int default_block_size = 512;
+constexpr int default_dot_dim = 32;
+constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
 #include "common/solver/idr_kernels.hpp.inc"
@@ -112,23 +118,52 @@ void solve_lower_triangular(const matrix::Dense<ValueType> *m,
 
 
 template <typename ValueType>
-void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
+void update_g_and_u(std::shared_ptr<const CudaExecutor> exec, size_type k,
+                    const matrix::Dense<ValueType> *p,
                     const matrix::Dense<ValueType> *m,
+                    matrix::Dense<ValueType> *alpha,
                     matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *g_k,
                     matrix::Dense<ValueType> *u,
                     const Array<stopping_status> *stop_status)
 {
     const auto size = g->get_size()[0];
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
-
-    const auto grid_dim = nrhs;
-    update_g_and_u_kernel<default_block_size><<<grid_dim, default_block_size>>>(
-        k, size, nrhs, as_cuda_type(p->get_const_values()), p->get_stride(),
-        as_cuda_type(m->get_const_values()), m->get_stride(),
-        as_cuda_type(g->get_values()), g->get_stride(),
-        as_cuda_type(g_k->get_values()), g_k->get_stride(),
-        as_cuda_type(u->get_values()), u->get_stride(),
-        as_cuda_type(stop_status->get_const_data()));
+    const auto p_stride = p->get_stride();
+
+    const dim3 grid_dim(ceildiv(nrhs, default_dot_dim),
+                        exec->get_num_multiprocessor() * 2);
+    const dim3 block_dim(default_dot_dim, default_dot_dim);
+
+    for (size_type i = 0; i < k; i++) {
+        const auto p_i = p->get_const_values() + i * p_stride;
+        if (nrhs > 1) {
+            components::fill_array(exec, alpha->get_values(), nrhs,
+                                   zero<ValueType>());
+            multidot_kernel<<<grid_dim, block_dim>>>(
+                i, size, nrhs, as_cuda_type(p_i),
+                as_cuda_type(g_k->get_values()), g_k->get_stride(),
+                as_cuda_type(alpha->get_values()),
+                as_cuda_type(stop_status->get_const_data()));
+        } else {
+            cublas::dot(exec->get_cublas_handle(), size, p_i, p_stride,
+                        g_k->get_values(), g_k->get_stride(),
+                        alpha->get_values());
+        }
+        update_g_k_and_u_kernel<default_block_size>
+            <<<ceildiv(size * g_k->get_stride(), default_block_size),
+               default_block_size>>>(
+                k, i, size, nrhs, as_cuda_type(alpha->get_const_values()),
+                as_cuda_type(m->get_const_values()), m->get_stride(),
+                as_cuda_type(g->get_const_values()), g->get_stride(),
+                as_cuda_type(g_k->get_values()), g_k->get_stride(),
+                as_cuda_type(u->get_values()), u->get_stride(),
+                as_cuda_type(stop_status->get_const_data()));
+    }
+    update_g_kernel<default_block_size>
+        <<<ceildiv(size * g_k->get_stride()), default_block_size>>>(
+            k, size, nrhs, as_cuda_type(g_k->get_const_values()),
+            g_k->get_stride(), as_cuda_type(g->get_values()), g->get_stride(),
+            as_cuda_type(stop_status->get_const_data()));
 }
 
 
@@ -252,10 +287,11 @@ void step_3(std::shared_ptr<const CudaExecutor> exec, const size_type k,
             const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
             matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *u,
             matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
-            matrix::Dense<ValueType> *residual, matrix::Dense<ValueType> *x,
+            matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *residual,
+            matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status)
 {
-    update_g_and_u(k, p, m, g, g_k, u, stop_status);
+    update_g_and_u(exec, k, p, m, alpha, g, g_k, u, stop_status);
     update_m(k, p, g, m, stop_status);
     update_x_r_and_f(k, m, g, u, f, residual, x, stop_status);
 }
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index 898d55ea6f0..df9346f0051 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -268,11 +268,11 @@ TEST_F(Idr, CudaIdrStep3IsEquivalentToRef)
 
     gko::size_type k = 2;
     gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), v.get(),
-                                         u.get(), m.get(), f.get(), r.get(),
-                                         x.get(), stop_status.get());
-    gko::kernels::cuda::idr::step_3(cuda, k, d_p.get(), d_g.get(), d_v.get(),
-                                    d_u.get(), d_m.get(), d_f.get(), d_r.get(),
-                                    d_x.get(), d_stop_status.get());
+                                         u.get(), m.get(), f.get(), omega.get(),
+                                         r.get(), x.get(), stop_status.get());
+    gko::kernels::cuda::idr::step_3(
+        cuda, k, d_p.get(), d_g.get(), d_v.get(), d_u.get(), d_m.get(),
+        d_f.get(), d_omega.get(), d_r.get(), d_x.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(g, d_g, 1e-14);
     GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index a8985720404..08f852161ab 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "hip/base/math.hip.hpp"
 #include "hip/base/types.hip.hpp"
+#include "hip/components/atomic.hip.hpp"
 #include "hip/components/cooperative_groups.hip.hpp"
 #include "hip/components/reduction.hip.hpp"
 #include "hip/components/thread_ids.hip.hpp"
@@ -116,25 +117,54 @@ void solve_lower_triangular(const matrix::Dense<ValueType> *m,
 
 
 template <typename ValueType>
-void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
+void update_g_and_u(std::shared_ptr<const HipExecutor> exec, size_type k,
+                    const matrix::Dense<ValueType> *p,
                     const matrix::Dense<ValueType> *m,
+                    matrix::Dense<ValueType> *alpha,
                     matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *g_k,
                     matrix::Dense<ValueType> *u,
                     const Array<stopping_status> *stop_status)
 {
     const auto size = g->get_size()[0];
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
-
-    const auto grid_dim = nrhs;
-    hipLaunchKernelGGL(
-        HIP_KERNEL_NAME(update_g_and_u_kernel<default_block_size>), grid_dim,
-        default_block_size, 0, 0, k, size, nrhs,
-        as_hip_type(p->get_const_values()), p->get_stride(),
-        as_hip_type(m->get_const_values()), m->get_stride(),
-        as_hip_type(g->get_values()), g->get_stride(),
-        as_hip_type(g_k->get_values()), g_k->get_stride(),
-        as_hip_type(u->get_values()), u->get_stride(),
-        as_hip_type(stop_status->get_const_data()));
+    const auto p_stride = p->get_stride();
+
+    const dim3 grid_dim(ceildiv(nrhs, default_dot_dim),
+                        exec->get_num_multiprocessor() * 2);
+    const dim3 block_dim(default_dot_dim, default_dot_dim);
+
+    for (size_type i = 0; i < k; i++) {
+        const auto p_i = p->get_const_values() + i * p_stride;
+        if (nrhs > 1) {
+            components::fill_array(exec, alpha->get_values(), nrhs,
+                                   zero<ValueType>());
+            hipLaunchKernelGGL(
+                multidot_kernel, grid_dim, block_dim, 0, 0, i, size, nrhs,
+                as_hip_type(p_i), as_hip_type(g_k->get_values()),
+                g_k->get_stride(), as_hip_type(alpha->get_values()),
+                as_hip_type(stop_status->get_const_data()));
+        } else {
+            hipblas::dot(exec->get_hiplas_handle(), size, p_i, 1,
+                         g_k->get_values(), g_k->get_stride(),
+                         alpha->get_values());
+        }
+        hipLaunchKernelGGL(
+            update_g_k_and_u_kernel<default_block_size>,
+            ceildiv(size * g_k->get_stride(), default_block_size),
+            default_block_size, 0, 0, k, i, size, nrhs,
+            as_hip_type(alpha->get_const_values()),
+            as_hip_type(m->get_const_values()), m->get_stride(),
+            as_hip_type(g->get_const_values()), g->get_stride(),
+            as_hip_type(g_k->get_values()), g_k->get_stride(),
+            as_hip_type(u->get_values()), u->get_stride(),
+            as_hip_type(stop_status->get_const_data()));
+    }
+    hipLaunchKernelGGL(update_g_kernel<default_block_size>,
+                       ceildiv(size * g_k->get_stride(), default_block_size),
+                       default_block_size, 0, 0, k, size, nrhs,
+                       as_hip_type(g_k->get_const_values()), g_k->get_stride(),
+                       as_hip_type(g->get_values()), g->get_stride(),
+                       as_hip_type(stop_status->get_const_data()));
 }
 
 
@@ -262,10 +292,11 @@ void step_3(std::shared_ptr<const HipExecutor> exec, const size_type k,
             const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
             matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *u,
             matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
-            matrix::Dense<ValueType> *residual, matrix::Dense<ValueType> *x,
+            matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *residual,
+            matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status)
 {
-    update_g_and_u(k, p, m, g, g_k, u, stop_status);
+    update_g_and_u(exec, k, p, m, alpha, g, g_k, u, stop_status);
     update_m(k, p, g, m, stop_status);
     update_x_r_and_f(k, m, g, u, f, residual, x, stop_status);
 }
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index 0ce13008973..9f17ff00007 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -268,11 +268,11 @@ TEST_F(Idr, HipIdrStep3IsEquivalentToRef)
 
     gko::size_type k = 2;
     gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), v.get(),
-                                         u.get(), m.get(), f.get(), r.get(),
-                                         x.get(), stop_status.get());
-    gko::kernels::hip::idr::step_3(hip, k, d_p.get(), d_g.get(), d_v.get(),
-                                   d_u.get(), d_m.get(), d_f.get(), d_r.get(),
-                                   d_x.get(), d_stop_status.get());
+                                         u.get(), m.get(), f.get(), omega.get(),
+                                         r.get(), x.get(), stop_status.get());
+    gko::kernels::hip::idr::step_3(
+        hip, k, d_p.get(), d_g.get(), d_v.get(), d_u.get(), d_m.get(),
+        d_f.get(), d_omega.get(), d_r.get(), d_x.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(g, d_g, 1e-14);
     GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index 171405fd198..355d50abd96 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -246,7 +246,8 @@ void step_3(std::shared_ptr<const OmpExecutor> exec, const size_type k,
             const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
             matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *u,
             matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
-            matrix::Dense<ValueType> *residual, matrix::Dense<ValueType> *x,
+            matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *residual,
+            matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status)
 {
     const auto nrhs = x->get_size()[1];
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index f93e994389f..bee822fba96 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -268,11 +268,11 @@ TEST_F(Idr, OmpIdrStep3IsEquivalentToRef)
 
     gko::size_type k = 2;
     gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), v.get(),
-                                         u.get(), m.get(), f.get(), r.get(),
-                                         x.get(), stop_status.get());
-    gko::kernels::omp::idr::step_3(omp, k, d_p.get(), d_g.get(), d_v.get(),
-                                   d_u.get(), d_m.get(), d_f.get(), d_r.get(),
-                                   d_x.get(), d_stop_status.get());
+                                         u.get(), m.get(), f.get(), omega.get(),
+                                         r.get(), x.get(), stop_status.get());
+    gko::kernels::omp::idr::step_3(
+        omp, k, d_p.get(), d_g.get(), d_v.get(), d_u.get(), d_m.get(),
+        d_f.get(), d_omega.get(), d_r.get(), d_x.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(g, d_g, 1e-14);
     GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index 22b7371ccc8..f481f279609 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -228,7 +228,8 @@ void step_3(std::shared_ptr<const ReferenceExecutor> exec, const size_type k,
             const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
             matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *u,
             matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
-            matrix::Dense<ValueType> *residual, matrix::Dense<ValueType> *x,
+            matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *residual,
+            matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status)
 {
     const auto nrhs = x->get_size()[1];

From edbf9403a8dee20bad960e63d868829f5c295f46 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Wed, 26 Aug 2020 18:37:50 +0200
Subject: [PATCH 0147/1008] Use multidot for m update

---
 common/solver/idr_kernels.hpp.inc | 44 ++-------------------------
 cuda/solver/idr_kernels.cu        | 49 ++++++++++++++++++++-----------
 hip/solver/idr_kernels.hip.cpp    | 42 +++++++++++++++++---------
 3 files changed, 63 insertions(+), 72 deletions(-)

diff --git a/common/solver/idr_kernels.hpp.inc b/common/solver/idr_kernels.hpp.inc
index f033d8d4446..094915b37f5 100644
--- a/common/solver/idr_kernels.hpp.inc
+++ b/common/solver/idr_kernels.hpp.inc
@@ -195,9 +195,9 @@ __global__ __launch_bounds__(default_block_size) void step_2_kernel(
 
 template <typename ValueType>
 __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
-    size_type i, size_type num_rows, size_type nrhs,
-    const ValueType *__restrict__ p_i, const ValueType *__restrict__ g_k,
-    size_type g_k_stride, ValueType *__restrict__ alpha,
+    size_type num_rows, size_type nrhs, const ValueType *__restrict__ p_i,
+    const ValueType *__restrict__ g_k, size_type g_k_stride,
+    ValueType *__restrict__ alpha,
     const stopping_status *__restrict__ stop_status)
 {
     const auto tidx = threadIdx.x;
@@ -286,44 +286,6 @@ __global__ __launch_bounds__(block_size) void update_g_kernel(
 }
 
 
-template <size_type block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void update_m_kernel(
-    size_type k, size_type size, size_type subspace_dim, size_type nrhs,
-    const ValueType *__restrict__ p_values, size_type p_stride,
-    const ValueType *__restrict__ g_values, size_type g_stride,
-    ValueType *__restrict__ m_values, size_type m_stride,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = threadIdx.x;
-    const auto row = blockIdx.x + k;
-    const auto col = blockIdx.y;
-
-    if (stop_status[col].has_stopped()) {
-        return;
-    }
-
-    __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
-    ValueType *__restrict__ reduction_helper = reduction_helper_array;
-
-    ValueType local_res = zero<ValueType>();
-    if (row < subspace_dim && col < nrhs) {
-        for (size_type i = tidx; i < size; i += block_size) {
-            local_res += p_values[row * p_stride + i] *
-                         g_values[i * g_stride + k * nrhs + col];
-        }
-    }
-    reduction_helper[tidx] = local_res;
-    __syncthreads();
-    reduce(group::this_thread_block(), reduction_helper,
-           [](const ValueType &a, const ValueType &b) { return a + b; });
-    __syncthreads();
-
-    if (tidx == 0) {
-        m_values[row * m_stride + k * nrhs + col] = reduction_helper[0];
-    }
-}
-
-
 template <typename ValueType>
 __global__ __launch_bounds__(default_block_size) void update_x_r_and_f_kernel(
     size_type k, size_type size, size_type subspace_dim, size_type nrhs,
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index 97c1ca777b2..7e5fcc1a56c 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -140,12 +140,11 @@ void update_g_and_u(std::shared_ptr<const CudaExecutor> exec, size_type k,
             components::fill_array(exec, alpha->get_values(), nrhs,
                                    zero<ValueType>());
             multidot_kernel<<<grid_dim, block_dim>>>(
-                i, size, nrhs, as_cuda_type(p_i),
-                as_cuda_type(g_k->get_values()), g_k->get_stride(),
-                as_cuda_type(alpha->get_values()),
+                size, nrhs, as_cuda_type(p_i), as_cuda_type(g_k->get_values()),
+                g_k->get_stride(), as_cuda_type(alpha->get_values()),
                 as_cuda_type(stop_status->get_const_data()));
         } else {
-            cublas::dot(exec->get_cublas_handle(), size, p_i, p_stride,
+            cublas::dot(exec->get_cublas_handle(), size, p_i, 1,
                         g_k->get_values(), g_k->get_stride(),
                         alpha->get_values());
         }
@@ -160,7 +159,8 @@ void update_g_and_u(std::shared_ptr<const CudaExecutor> exec, size_type k,
                 as_cuda_type(stop_status->get_const_data()));
     }
     update_g_kernel<default_block_size>
-        <<<ceildiv(size * g_k->get_stride()), default_block_size>>>(
+        <<<ceildiv(size * g_k->get_stride(), default_block_size),
+           default_block_size>>>(
             k, size, nrhs, as_cuda_type(g_k->get_const_values()),
             g_k->get_stride(), as_cuda_type(g->get_values()), g->get_stride(),
             as_cuda_type(stop_status->get_const_data()));
@@ -168,20 +168,35 @@ void update_g_and_u(std::shared_ptr<const CudaExecutor> exec, size_type k,
 
 
 template <typename ValueType>
-void update_m(size_type k, const matrix::Dense<ValueType> *p,
-              const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *m,
+void update_m(std::shared_ptr<const CudaExecutor> exec, size_type k,
+              const matrix::Dense<ValueType> *p,
+              const matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *m,
               const Array<stopping_status> *stop_status)
 {
-    const auto size = g->get_size()[0];
-    const auto subspace_dim = p->get_size()[0];
-    const auto nrhs = m->get_size()[1] / subspace_dim;
+    const auto size = g_k->get_size()[0];
+    const auto subspace_dim = m->get_size()[0];
+    const auto nrhs = m->get_size()[1] / m->get_size()[0];
+    const auto p_stride = p->get_stride();
+    const auto m_stride = m->get_stride();
 
-    const auto grid_dim = dim3((subspace_dim - k), nrhs, 1);
-    update_m_kernel<default_block_size><<<grid_dim, default_block_size>>>(
-        k, size, subspace_dim, nrhs, as_cuda_type(p->get_const_values()),
-        p->get_stride(), as_cuda_type(g->get_const_values()), g->get_stride(),
-        as_cuda_type(m->get_values()), m->get_stride(),
-        as_cuda_type(stop_status->get_const_data()));
+    const dim3 grid_dim(ceildiv(nrhs, default_dot_dim),
+                        exec->get_num_multiprocessor() * 2);
+    const dim3 block_dim(default_dot_dim, default_dot_dim);
+
+    for (size_type i = k; i < subspace_dim; i++) {
+        const auto p_i = p->get_const_values() + i * p_stride;
+        auto m_i = m->get_values() + i * m_stride + k * nrhs;
+        if (nrhs > 1) {
+            components::fill_array(exec, m_i, nrhs, zero<ValueType>());
+            multidot_kernel<<<grid_dim, block_dim>>>(
+                size, nrhs, as_cuda_type(p_i),
+                as_cuda_type(g_k->get_const_values()), g_k->get_stride(),
+                as_cuda_type(m_i), as_cuda_type(stop_status->get_const_data()));
+        } else {
+            cublas::dot(exec->get_cublas_handle(), size, p_i, 1,
+                        g_k->get_const_values(), g_k->get_stride(), m_i);
+        }
+    }
 }
 
 
@@ -292,7 +307,7 @@ void step_3(std::shared_ptr<const CudaExecutor> exec, const size_type k,
             const Array<stopping_status> *stop_status)
 {
     update_g_and_u(exec, k, p, m, alpha, g, g_k, u, stop_status);
-    update_m(k, p, g, m, stop_status);
+    update_m(exec, k, p, g_k, m, stop_status);
     update_x_r_and_f(k, m, g, u, f, residual, x, stop_status);
 }
 
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index 08f852161ab..4afa43f7de9 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -139,7 +139,7 @@ void update_g_and_u(std::shared_ptr<const HipExecutor> exec, size_type k,
             components::fill_array(exec, alpha->get_values(), nrhs,
                                    zero<ValueType>());
             hipLaunchKernelGGL(
-                multidot_kernel, grid_dim, block_dim, 0, 0, i, size, nrhs,
+                multidot_kernel, grid_dim, block_dim, 0, 0, size, nrhs,
                 as_hip_type(p_i), as_hip_type(g_k->get_values()),
                 g_k->get_stride(), as_hip_type(alpha->get_values()),
                 as_hip_type(stop_status->get_const_data()));
@@ -169,22 +169,36 @@ void update_g_and_u(std::shared_ptr<const HipExecutor> exec, size_type k,
 
 
 template <typename ValueType>
-void update_m(size_type k, const matrix::Dense<ValueType> *p,
+void update_m(std::shared_ptr<const HipExecutor> exec, size_type k,
+              const matrix::Dense<ValueType> *p,
               const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *m,
               const Array<stopping_status> *stop_status)
 {
-    const auto size = g->get_size()[0];
-    const auto subspace_dim = p->get_size()[0];
-    const auto nrhs = m->get_size()[1] / subspace_dim;
+    const auto size = g_k->get_size()[0];
+    const auto subspace_dim = m->get_size()[0];
+    const auto nrhs = m->get_size()[1] / m->get_size()[0];
+    const auto p_stride = p->get_stride();
+    const auto m_stride = m->get_stride();
 
-    const auto grid_dim = dim3((subspace_dim - k), nrhs, 1);
-    hipLaunchKernelGGL(HIP_KERNEL_NAME(update_m_kernel<default_block_size>),
-                       grid_dim, default_block_size, 0, 0, k, size,
-                       subspace_dim, nrhs, as_hip_type(p->get_const_values()),
-                       p->get_stride(), as_hip_type(g->get_const_values()),
-                       g->get_stride(), as_hip_type(m->get_values()),
-                       m->get_stride(),
-                       as_hip_type(stop_status->get_const_data()));
+    const dim3 grid_dim(ceildiv(nrhs, default_dot_dim),
+                        exec->get_num_multiprocessor() * 2);
+    const dim3 block_dim(default_dot_dim, default_dot_dim);
+
+    for (size_type i = k; i < subspace_dim; i++) {
+        const auto p_i = p->get_const_values() + i * p_stride;
+        auto m_i = m->get_values() + i * m_stride + k * nrhs;
+        if (nrhs > 1) {
+            components::fill_array(exec, m_i, nrhs, zero<ValueType>());
+            hipLaunchKernelGGL(multidot_kernel, grid_dim, block_dim, 0, 0, size,
+                               nrhs, as_hip_type(p_i),
+                               as_hip_type(g_k->get_const_values()),
+                               g_k->get_stride(), as_hip_type(m_i),
+                               as_hip_type(stop_status->get_const_data()));
+        } else {
+            hipblas::dot(exec->get_hipblas_handle(), size, p_i, 1,
+                         g_k->get_const_values(), g_k->get_stride(), m_i);
+        }
+    }
 }
 
 
@@ -297,7 +311,7 @@ void step_3(std::shared_ptr<const HipExecutor> exec, const size_type k,
             const Array<stopping_status> *stop_status)
 {
     update_g_and_u(exec, k, p, m, alpha, g, g_k, u, stop_status);
-    update_m(k, p, g, m, stop_status);
+    update_m(exec, k, p, g_k, m, stop_status);
     update_x_r_and_f(k, m, g, u, f, residual, x, stop_status);
 }
 

From ca222d67dffd6d42dbb24bdb75c14c8b2a25c69d Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Tue, 8 Sep 2020 11:33:41 +0200
Subject: [PATCH 0148/1008] improve random initialization of P

---
 core/device_hooks/cuda_hooks.cpp              |   6 +
 core/solver/idr.cpp                           |   3 +
 core/test/base/exception.cpp                  |   8 ++
 core/test/base/exception_helpers.cpp          |   8 ++
 cuda/CMakeLists.txt                           |   4 +-
 cuda/base/curand_bindings.hpp                 | 110 ++++++++++++++++++
 cuda/base/exception.cpp                       |  24 ++++
 cuda/solver/idr_kernels.cu                    |   5 +
 cuda/test/base/exception_helpers.cu           |  12 ++
 include/ginkgo/core/base/exception.hpp        |  23 ++++
 .../ginkgo/core/base/exception_helpers.hpp    |  23 ++++
 include/ginkgo/core/solver/idr.hpp            |  10 --
 omp/solver/idr_kernels.cpp                    |  25 +++-
 reference/solver/idr_kernels.cpp              |  25 +++-
 14 files changed, 273 insertions(+), 13 deletions(-)
 create mode 100644 cuda/base/curand_bindings.hpp

diff --git a/core/device_hooks/cuda_hooks.cpp b/core/device_hooks/cuda_hooks.cpp
index 5bd7f5cb09f..46e342d9c0a 100644
--- a/core/device_hooks/cuda_hooks.cpp
+++ b/core/device_hooks/cuda_hooks.cpp
@@ -120,6 +120,12 @@ std::string CublasError::get_error(int64)
 }
 
 
+std::string CurandError::get_error(int64)
+{
+    return "ginkgo CUDA module is not compiled";
+}
+
+
 std::string CusparseError::get_error(int64)
 {
     return "ginkgo CUDA module is not compiled";
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index 6a80dad0f4b..7fa631913f1 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -131,6 +131,9 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     bool one_changed{};
     Array<stopping_status> stop_status(exec, nrhs);
 
+    auto subspace_vectors_ =
+        Vector::create(exec, gko::dim<2>(subspace_dim_, problem_size));
+
     // Initialization
     // m = identity
     exec->run(
diff --git a/core/test/base/exception.cpp b/core/test/base/exception.cpp
index b04d7553103..f1fb831f871 100644
--- a/core/test/base/exception.cpp
+++ b/core/test/base/exception.cpp
@@ -90,6 +90,14 @@ TEST(ExceptionClasses, CublasErrorReturnsCorrectWhatMessage)
 }
 
 
+TEST(ExceptionClasses, CurandErrorReturnsCorrectWhatMessage)
+{
+    gko::CurandError error("test_file.cpp", 123, "test_func", 1);
+    std::string expected = "test_file.cpp:123: test_func: ";
+    ASSERT_EQ(expected, std::string(error.what()).substr(0, expected.size()));
+}
+
+
 TEST(ExceptionClasses, CusparseErrorReturnsCorrectWhatMessage)
 {
     gko::CusparseError error("test_file.cpp", 123, "test_func", 1);
diff --git a/core/test/base/exception_helpers.cpp b/core/test/base/exception_helpers.cpp
index cad8a3d5684..719f7d5d119 100644
--- a/core/test/base/exception_helpers.cpp
+++ b/core/test/base/exception_helpers.cpp
@@ -116,6 +116,14 @@ TEST(CudaError, ReturnsCublasError)
 }
 
 
+void throws_curand_error() { throw GKO_CURAND_ERROR(0); }
+
+TEST(CudaError, ReturnsCurandError)
+{
+    ASSERT_THROW(throws_curand_error(), gko::CurandError);
+}
+
+
 void throws_cusparse_error() { throw GKO_CUSPARSE_ERROR(0); }
 
 TEST(CudaError, ReturnsCusparseError)
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 66d67fecb46..6b091a61263 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -63,6 +63,8 @@ find_library(CUBLAS cublas
     HINT ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES} NO_DEFAULT_PATH)
 find_library(CUSPARSE cusparse
     HINT ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
+find_library(CURAND curand
+    HINT ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
 
 add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_cuda_device> "")
 target_sources(ginkgo_cuda
@@ -150,7 +152,7 @@ target_compile_options(ginkgo_cuda PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${GINKGO_CO
 ginkgo_compile_features(ginkgo_cuda)
 target_include_directories(ginkgo_cuda
     SYSTEM PRIVATE ${CUDA_INCLUDE_DIRS})
-target_link_libraries(ginkgo_cuda PRIVATE ${CUDA_RUNTIME_LIBS} ${CUBLAS} ${CUSPARSE})
+target_link_libraries(ginkgo_cuda PRIVATE ${CUDA_RUNTIME_LIBS} ${CUBLAS} ${CUSPARSE} ${CURAND})
 
 cas_target_cuda_architectures(ginkgo_cuda
     ARCHITECTURES ${GINKGO_CUDA_ARCHITECTURES}
diff --git a/cuda/base/curand_bindings.hpp b/cuda/base/curand_bindings.hpp
new file mode 100644
index 00000000000..8e57c8025e3
--- /dev/null
+++ b/cuda/base/curand_bindings.hpp
@@ -0,0 +1,110 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CUDA_BASE_CURAND_BINDINGS_HPP_
+#define GKO_CUDA_BASE_CURAND_BINDINGS_HPP_
+
+
+#include <curand.h>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
+#include "cuda/base/types.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+/**
+ * @brief The CUSPARSE namespace.
+ *
+ * @ingroup cusparse
+ */
+namespace curand {
+/**
+ * @brief The detail namespace.
+ *
+ * @ingroup detail
+ */
+
+
+template <typename ValueType>
+struct is_supported : std::false_type {};
+
+template <>
+struct is_supported<float> : std::true_type {};
+
+template <>
+struct is_supported<double> : std::true_type {};
+
+template <>
+struct is_supported<std::complex<float>> : std::true_type {};
+
+template <>
+struct is_supported<std::complex<double>> : std::true_type {};
+
+
+#define GKO_BIND_CURAND_RANDOM_VECTOR(ValueType, CurandName)                 \
+    inline void rand_vector(int seed, int n, remove_complex<ValueType> mean, \
+                            remove_complex<ValueType> stddev,                \
+                            ValueType *values)                               \
+    {                                                                        \
+        curandGenerator_t gen;                                               \
+        curandCreateGenerator(&gen, CURAND_RNG_PSEUDO_DEFAULT);              \
+        curandSetPseudoRandomGeneratorSeed(gen, seed);                       \
+        n = is_complex<ValueType>() ? 2 * n : n;                             \
+        GKO_ASSERT_NO_CURAND_ERRORS(CurandName(                              \
+            gen, reinterpret_cast<remove_complex<ValueType> *>(values), n,   \
+            mean, stddev));                                                  \
+    }                                                                        \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
+                  "semi-colon warnings")
+
+GKO_BIND_CURAND_RANDOM_VECTOR(float, curandGenerateNormal);
+GKO_BIND_CURAND_RANDOM_VECTOR(double, curandGenerateNormalDouble);
+GKO_BIND_CURAND_RANDOM_VECTOR(std::complex<float>, curandGenerateNormal);
+GKO_BIND_CURAND_RANDOM_VECTOR(std::complex<double>, curandGenerateNormalDouble);
+
+
+#undef GKO_BIND_CURAND_RANDOM_VECTOR
+
+
+}  // namespace curand
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_CUDA_BASE_CUSPARSE_BINDINGS_HPP_
diff --git a/cuda/base/exception.cpp b/cuda/base/exception.cpp
index 93fcd5e7cfd..0e7ffaad754 100644
--- a/cuda/base/exception.cpp
+++ b/cuda/base/exception.cpp
@@ -78,6 +78,30 @@ std::string CublasError::get_error(int64 error_code)
 }
 
 
+std::string CurandError::get_error(int64 error_code)
+{
+#define GKO_REGISTER_CURAND_ERROR(error_name)           \
+    if (error_code == static_cast<int64>(error_name)) { \
+        return #error_name;                             \
+    }
+    GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_SUCCESS);
+    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_VERSION_MISMATCH);
+    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_NOT_INITIALIZED);
+    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_ALLOCATION_FAILED);
+    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_TYPE_ERROR);
+    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_OUT_OF_RANGE);
+    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_LENGTH_NOT_MULTIPLE);
+    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_LAUNCH_FAILURE);
+    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_PREEXISTING_FAILURE);
+    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_INITIALIZATION_FAILED);
+    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_ARCH_MISMATCH);
+    GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_INTERNAL_ERROR);
+    return "Unknown error";
+
+#undef GKO_REGISTER_CURAND_ERROR
+}
+
+
 std::string CusparseError::get_error(int64 error_code)
 {
 #define GKO_REGISTER_CUSPARSE_ERROR(error_name) \
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index 7e5fcc1a56c..cf39b1a3cd7 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/fill_array.hpp"
 #include "cuda/base/config.hpp"
 #include "cuda/base/cublas_bindings.hpp"
+#include "cuda/base/curand_bindings.hpp"
 #include "cuda/base/math.hpp"
 #include "cuda/base/types.hpp"
 #include "cuda/components/atomic.cuh"
@@ -91,6 +92,10 @@ void initialize_m(matrix::Dense<ValueType> *m,
 template <typename ValueType>
 void orthonormalize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors)
 {
+    curand::rand_vector(
+        1234ULL,
+        subspace_vectors->get_size()[0] * subspace_vectors->get_stride(), 0.5,
+        0.5, subspace_vectors->get_values());
     orthonormalize_subspace_vectors_kernel<default_block_size>
         <<<1, default_block_size>>>(
             subspace_vectors->get_size()[0], subspace_vectors->get_size()[1],
diff --git a/cuda/test/base/exception_helpers.cu b/cuda/test/base/exception_helpers.cu
index 1652594803a..58136d2d157 100644
--- a/cuda/test/base/exception_helpers.cu
+++ b/cuda/test/base/exception_helpers.cu
@@ -68,6 +68,18 @@ TEST(AssertNoCublasErrors, DoesNotThrowOnSuccess)
 }
 
 
+TEST(AssertNoCurandErrors, ThrowsOnError)
+{
+    ASSERT_THROW(GKO_ASSERT_NO_CURAND_ERRORS(1), gko::CurandError);
+}
+
+
+TEST(AssertNoCurandErrors, DoesNotThrowOnSuccess)
+{
+    ASSERT_NO_THROW(GKO_ASSERT_NO_CURAND_ERRORS(CURAND_STATUS_SUCCESS));
+}
+
+
 TEST(AssertNoCusparseErrors, ThrowsOnError)
 {
     ASSERT_THROW(GKO_ASSERT_NO_CUSPARSE_ERRORS(1), gko::CusparseError);
diff --git a/include/ginkgo/core/base/exception.hpp b/include/ginkgo/core/base/exception.hpp
index 78fe81a617e..516efdab90a 100644
--- a/include/ginkgo/core/base/exception.hpp
+++ b/include/ginkgo/core/base/exception.hpp
@@ -219,6 +219,29 @@ class CublasError : public Error {
 };
 
 
+/**
+ * CurandError is thrown when a cuRAND routine throws a non-zero error code.
+ */
+class CurandError : public Error {
+public:
+    /**
+     * Initializes a cuBLAS error.
+     *
+     * @param file  The name of the offending source file
+     * @param line  The source code line number where the error occurred
+     * @param func  The name of the cuBLAS routine that failed
+     * @param error_code  The resulting cuBLAS error code
+     */
+    CurandError(const std::string &file, int line, const std::string &func,
+                int64 error_code)
+        : Error(file, line, func + ": " + get_error(error_code))
+    {}
+
+private:
+    static std::string get_error(int64 error_code);
+};
+
+
 /**
  * CusparseError is thrown when a cuSPARSE routine throws a non-zero error code.
  */
diff --git a/include/ginkgo/core/base/exception_helpers.hpp b/include/ginkgo/core/base/exception_helpers.hpp
index 287b59380f8..e482f142d2d 100644
--- a/include/ginkgo/core/base/exception_helpers.hpp
+++ b/include/ginkgo/core/base/exception_helpers.hpp
@@ -300,6 +300,15 @@ inline dim<2> get_size(const dim<2> &size) { return size; }
     ::gko::CublasError(__FILE__, __LINE__, __func__, _errcode)
 
 
+/**
+ * Instantiates a CurandError.
+ *
+ * @param errcode  The error code returned from the cuRAND routine.
+ */
+#define GKO_CURAND_ERROR(_errcode) \
+    ::gko::CurandError(__FILE__, __LINE__, __func__, _errcode)
+
+
 /**
  * Instantiates a CusparseError.
  *
@@ -337,6 +346,20 @@ inline dim<2> get_size(const dim<2> &size) { return size; }
     } while (false)
 
 
+/**
+ * Asserts that a cuRAND library call completed without errors.
+ *
+ * @param _curand_call  a library call expression
+ */
+#define GKO_ASSERT_NO_CURAND_ERRORS(_curand_call) \
+    do {                                          \
+        auto _errcode = _curand_call;             \
+        if (_errcode != CURAND_STATUS_SUCCESS) {  \
+            throw GKO_CURAND_ERROR(_errcode);     \
+        }                                         \
+    } while (false)
+
+
 /**
  * Asserts that a cuSPARSE library call completed without errors.
  *
diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index 32bcd8240d9..1c6c825a99b 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -196,21 +196,11 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
 
         subspace_dim_ = parameters_.subspace_dim;
 
-        auto subspace_vectors_data = matrix_data<ValueType>(
-            gko::dim<2>(subspace_dim_, system_matrix_->get_size()[0]),
-            std::normal_distribution<>(0.0, 1.0), std::ranlux48(15));
-
-        subspace_vectors_ =
-            matrix::Dense<ValueType>::create(factory->get_executor());
-
-        subspace_vectors_->read(subspace_vectors_data);
-
         kappa_ = parameters_.kappa;
     }
 
 private:
     std::shared_ptr<const LinOp> system_matrix_{};
-    std::shared_ptr<matrix::Dense<ValueType>> subspace_vectors_{};
     std::shared_ptr<const stop::CriterionFactory> stop_criterion_factory_{};
     size_type subspace_dim_;
     remove_complex<ValueType> kappa_;
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index 355d50abd96..e088ba3f64f 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -117,6 +117,22 @@ void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
 }
 
 
+template <typename ValueType, typename Distribution, typename Generator>
+typename std::enable_if<!is_complex_s<ValueType>::value, ValueType>::type
+get_rand_value(Distribution &&dist, Generator &&gen)
+{
+    return dist(gen);
+}
+
+
+template <typename ValueType, typename Distribution, typename Generator>
+typename std::enable_if<is_complex_s<ValueType>::value, ValueType>::type
+get_rand_value(Distribution &&dist, Generator &&gen)
+{
+    return ValueType(dist(gen), dist(gen));
+}
+
+
 }  // namespace
 
 
@@ -143,10 +159,17 @@ void initialize(std::shared_ptr<const OmpExecutor> exec,
         }
     }
 
-    // Orthonormalize P
+    // Initialize and Orthonormalize P
     const auto num_rows = subspace_vectors->get_size()[0];
     const auto num_cols = subspace_vectors->get_size()[1];
+    auto dist = std::normal_distribution<remove_complex<ValueType>>(0.0, 1.0);
+    auto gen = std::ranlux48(15);
     for (size_type row = 0; row < num_rows; row++) {
+        for (size_type col = 0; col < num_cols; col++) {
+            subspace_vectors->at(row, col) =
+                get_rand_value<ValueType>(dist, gen);
+        }
+
         for (size_type i = 0; i < row; i++) {
             auto dot = zero<ValueType>();
 #pragma omp parallel for reduction(add : dot)
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index f481f279609..99e6d793dad 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -111,6 +111,22 @@ void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
 }
 
 
+template <typename ValueType, typename Distribution, typename Generator>
+typename std::enable_if<!is_complex_s<ValueType>::value, ValueType>::type
+get_rand_value(Distribution &&dist, Generator &&gen)
+{
+    return dist(gen);
+}
+
+
+template <typename ValueType, typename Distribution, typename Generator>
+typename std::enable_if<is_complex_s<ValueType>::value, ValueType>::type
+get_rand_value(Distribution &&dist, Generator &&gen)
+{
+    return ValueType(dist(gen), dist(gen));
+}
+
+
 }  // namespace
 
 
@@ -133,10 +149,17 @@ void initialize(std::shared_ptr<const ReferenceExecutor> exec,
         }
     }
 
-    // Orthonormalize P
+    // Initialize and Orthonormalize P
     const auto num_rows = subspace_vectors->get_size()[0];
     const auto num_cols = subspace_vectors->get_size()[1];
+    auto dist = std::normal_distribution<remove_complex<ValueType>>(0.0, 1.0);
+    auto gen = std::ranlux48(15);
     for (size_type row = 0; row < num_rows; row++) {
+        for (size_type col = 0; col < num_cols; col++) {
+            subspace_vectors->at(row, col) =
+                get_rand_value<ValueType>(dist, gen);
+        }
+
         for (size_type i = 0; i < row; i++) {
             auto dot = zero<ValueType>();
             for (size_type j = 0; j < num_cols; j++) {

From bed953cab0a1b2e40ee68e39087ba09c5abc754e Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Tue, 8 Sep 2020 17:31:53 +0200
Subject: [PATCH 0149/1008] Add hiprand binding

---
 cmake/GinkgoConfig.cmake.in                   |   1 +
 cmake/build_helpers.cmake                     |   3 +-
 cmake/create_test.cmake                       |   1 +
 core/test/base/exception_helpers.cpp          |   8 ++
 cuda/base/exception.cpp                       |  21 ++--
 cuda/test/base/exception_helpers.cu           |   1 +
 hip/CMakeLists.txt                            |  23 +++-
 hip/base/exception.hip.cpp                    |  25 ++++
 hip/base/hiprand_bindings.hip.hpp             | 111 ++++++++++++++++++
 hip/get_info.cmake                            |   2 +-
 hip/solver/idr_kernels.hip.cpp                |  17 ++-
 hip/test/base/CMakeLists.txt                  |   2 +-
 hip/test/base/exception_helpers.hip.cpp       |  13 ++
 include/ginkgo/core/base/exception.hpp        |  23 ++++
 .../ginkgo/core/base/exception_helpers.hpp    |  23 ++++
 15 files changed, 256 insertions(+), 18 deletions(-)
 create mode 100644 hip/base/hiprand_bindings.hip.hpp

diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 354158bf07a..d58bbf9aac0 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -164,6 +164,7 @@ endif()
 if((NOT GINKGO_BUILD_SHARED_LIBS) AND GINKGO_BUILD_HIP)
     find_package(HIP REQUIRED)
     find_package(hipblas REQUIRED)
+    find_package(hiprand REQUIRED)
     find_package(hipsparse REQUIRED)
     if(GINKGO_HIP_PLATFORM MATCHES "hcc")
         ginkgo_hip_ban_link_hcflag(hcc::hccrt)
diff --git a/cmake/build_helpers.cmake b/cmake/build_helpers.cmake
index 0323b47d309..06e77eca733 100644
--- a/cmake/build_helpers.cmake
+++ b/cmake/build_helpers.cmake
@@ -86,12 +86,13 @@ function(ginkgo_check_headers target)
     if (HIP_SOURCES)
         set_source_files_properties(${HIP_SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
         hip_add_library(${target}_headers_hip ${HIP_SOURCES}) # the compiler options get set by linking to ginkgo_hip
-        target_link_libraries(${target}_headers_hip PRIVATE ${target} roc::hipblas roc::hipsparse)
+        target_link_libraries(${target}_headers_hip PRIVATE ${target} roc::hipblas roc::hipsparse hip::hiprand roc::rocrand)
         target_include_directories(${target}_headers_hip
             PRIVATE
             "${CMAKE_CURRENT_SOURCE_DIR}"
             "${GINKGO_HIP_THRUST_PATH}"
             "${HIPBLAS_INCLUDE_DIRS}"
+            "${HIPRAND_INCLUDE_DIRS}"
             "${HIPSPARSE_INCLUDE_DIRS}"
             "${ROCPRIM_INCLUDE_DIRS}")
     endif()
diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index a2d2ef5d978..9a7ece55f2f 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -160,6 +160,7 @@ function(ginkgo_create_hip_test test_name)
         # Only `exception_helpers` requires thess so far, but it's much easier
         # to put these this way.
         ${HIPBLAS_INCLUDE_DIRS}
+        ${HIPRAND_INCLUDE_DIRS}
         ${HIPSPARSE_INCLUDE_DIRS}
         )
     set_target_properties(${TEST_TARGET_NAME} PROPERTIES
diff --git a/core/test/base/exception_helpers.cpp b/core/test/base/exception_helpers.cpp
index 719f7d5d119..57b28b02d2a 100644
--- a/core/test/base/exception_helpers.cpp
+++ b/core/test/base/exception_helpers.cpp
@@ -148,6 +148,14 @@ TEST(HipError, ReturnsHipblasError)
 }
 
 
+void throws_hiprand_error() { throw GKO_HIPRAND_ERROR(0); }
+
+TEST(HipError, ReturnsHiprandError)
+{
+    ASSERT_THROW(throws_hiprand_error(), gko::HiprandError);
+}
+
+
 void throws_hipsparse_error() { throw GKO_HIPSPARSE_ERROR(0); }
 
 TEST(HipError, ReturnsHipsparseError)
diff --git a/cuda/base/exception.cpp b/cuda/base/exception.cpp
index 0e7ffaad754..e32f6a9eebf 100644
--- a/cuda/base/exception.cpp
+++ b/cuda/base/exception.cpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <cublas_v2.h>
 #include <cuda_runtime.h>
+#include <curand.h>
 #include <cusparse.h>
 
 
@@ -85,16 +86,16 @@ std::string CurandError::get_error(int64 error_code)
         return #error_name;                             \
     }
     GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_SUCCESS);
-    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_VERSION_MISMATCH);
-    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_NOT_INITIALIZED);
-    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_ALLOCATION_FAILED);
-    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_TYPE_ERROR);
-    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_OUT_OF_RANGE);
-    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_LENGTH_NOT_MULTIPLE);
-    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_LAUNCH_FAILURE);
-    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_PREEXISTING_FAILURE);
-    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_INITIALIZATION_FAILED);
-    GKO_REGISTER_CUBLAS_ERROR(CURAND_STATUS_ARCH_MISMATCH);
+    GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_VERSION_MISMATCH);
+    GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_NOT_INITIALIZED);
+    GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_ALLOCATION_FAILED);
+    GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_TYPE_ERROR);
+    GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_OUT_OF_RANGE);
+    GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_LENGTH_NOT_MULTIPLE);
+    GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_LAUNCH_FAILURE);
+    GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_PREEXISTING_FAILURE);
+    GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_INITIALIZATION_FAILED);
+    GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_ARCH_MISMATCH);
     GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_INTERNAL_ERROR);
     return "Unknown error";
 
diff --git a/cuda/test/base/exception_helpers.cu b/cuda/test/base/exception_helpers.cu
index 58136d2d157..6b05cfaaff7 100644
--- a/cuda/test/base/exception_helpers.cu
+++ b/cuda/test/base/exception_helpers.cu
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <cublas_v2.h>
 #include <cuda_runtime.h>
+#include <curand.h>
 #include <cusparse.h>
 
 
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index c1c08f72fff..58d5a074199 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -20,6 +20,22 @@ if(NOT DEFINED HIPBLAS_PATH)
     endif()
 endif()
 
+if(NOT DEFINED HIPRAND_PATH)
+    if(DEFINED ENV{HIPRAND_PATH})
+        set(HIPRAND_PATH $ENV{HIPRAND_PATH} CACHE PATH "Path to which HIPRAND has been installed")
+    else()
+        set(HIPRAND_PATH "${ROCM_PATH}/hiprand" CACHE PATH "Path to which HIPRAND has been installed")
+    endif()
+endif()
+
+if(NOT DEFINED ROCRAND_PATH)
+    if(DEFINED ENV{ROCRAND_PATH})
+        set(ROCRAND_PATH $ENV{ROCRAND_PATH} CACHE PATH "Path to which ROCRAND has been installed")
+    else()
+        set(ROCRAND_PATH "${ROCM_PATH}/rocrand" CACHE PATH "Path to which ROCRAND has been installed")
+    endif()
+endif()
+
 if(NOT DEFINED HIPSPARSE_PATH)
     if(DEFINED ENV{HIPSPARSE_PATH})
         set(HIPSPARSE_PATH $ENV{HIPSPARSE_PATH} CACHE PATH "Path to which HIPSPARSE has been installed")
@@ -120,7 +136,9 @@ endif()
 list(APPEND CMAKE_MODULE_PATH "${HIP_PATH}/cmake")
 list(APPEND CMAKE_PREFIX_PATH
     "${HIPBLAS_PATH}/lib/cmake"
+    "${HIPRAND_PATH}/lib/cmake"
     "${HIPSPARSE_PATH}/lib/cmake"
+    "${ROCRAND_PATH}/lib/cmake"
 )
 # Set CMAKE_MODULE_PATH and CMAKE_PREFIX_PATH as PARENT_SCOPE to easily find HIP again
 set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" PARENT_SCOPE)
@@ -135,7 +153,9 @@ set(HIP_HIPCC_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}" CACHE STR
 
 find_package(HIP REQUIRED)
 find_package(hipblas REQUIRED)
+find_package(hiprand REQUIRED)
 find_package(hipsparse REQUIRED)
+find_package(rocrand REQUIRED)
 find_path(GINKGO_HIP_THRUST_PATH "thrust/complex.h"
     PATHS "${HIP_PATH}/../include"
     ENV HIP_THRUST_PATH)
@@ -273,7 +293,7 @@ elseif(GINKGO_HIP_PLATFORM MATCHES "nvcc")
     set(HIP_CUDA_LIBRARIES ${CUDA_LIBRARIES} PARENT_SCOPE)
 endif()
 
-target_link_libraries(ginkgo_hip PRIVATE roc::hipblas roc::hipsparse)
+target_link_libraries(ginkgo_hip PRIVATE roc::hipblas roc::hipsparse hip::hiprand roc::rocrand)
 
 target_include_directories(ginkgo_hip
     PUBLIC
@@ -281,6 +301,7 @@ target_include_directories(ginkgo_hip
     PRIVATE
         ${GINKGO_HIP_THRUST_PATH}
         ${HIPBLAS_INCLUDE_DIRS}
+        ${HIPRAND_INCLUDE_DIRS}
         ${HIPSPARSE_INCLUDE_DIRS}
         $<BUILD_INTERFACE:${ROCPRIM_INCLUDE_DIRS}>)
 
diff --git a/hip/base/exception.hip.cpp b/hip/base/exception.hip.cpp
index 9e6f2ff7a00..66418c58c08 100644
--- a/hip/base/exception.hip.cpp
+++ b/hip/base/exception.hip.cpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <hip/hip_runtime.h>
 #include <hipblas.h>
+#include <hiprand.h>
 #include <hipsparse.h>
 
 
@@ -77,6 +78,30 @@ std::string HipblasError::get_error(int64 error_code)
 }
 
 
+std::string HiprandError::get_error(int64 error_code)
+{
+#define GKO_REGISTER_HIPRAND_ERROR(error_name)          \
+    if (error_code == static_cast<int64>(error_name)) { \
+        return #error_name;                             \
+    }
+    GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_SUCCESS);
+    GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_VERSION_MISMATCH);
+    GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_NOT_INITIALIZED);
+    GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_ALLOCATION_FAILED);
+    GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_TYPE_ERROR);
+    GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_OUT_OF_RANGE);
+    GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_LENGTH_NOT_MULTIPLE);
+    GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_LAUNCH_FAILURE);
+    GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_PREEXISTING_FAILURE);
+    GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_INITIALIZATION_FAILED);
+    GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_ARCH_MISMATCH);
+    GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_INTERNAL_ERROR);
+    return "Unknown error";
+
+#undef GKO_REGISTER_HIPRAND_ERROR
+}
+
+
 std::string HipsparseError::get_error(int64 error_code)
 {
 #define GKO_REGISTER_HIPSPARSE_ERROR(error_name) \
diff --git a/hip/base/hiprand_bindings.hip.hpp b/hip/base/hiprand_bindings.hip.hpp
new file mode 100644
index 00000000000..6bb56b3a635
--- /dev/null
+++ b/hip/base/hiprand_bindings.hip.hpp
@@ -0,0 +1,111 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CUDA_BASE_HIPRAND_BINDINGS_HPP_
+#define GKO_CUDA_BASE_HIPRAND_BINDINGS_HPP_
+
+
+#include <hiprand.h>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
+#include "hip/base/types.hip.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace hip {
+/**
+ * @brief The HIPSPARSE namespace.
+ *
+ * @ingroup hiprand
+ */
+namespace hiprand {
+/**
+ * @brief The detail namespace.
+ *
+ * @ingroup detail
+ */
+
+
+template <typename ValueType>
+struct is_supported : std::false_type {};
+
+template <>
+struct is_supported<float> : std::true_type {};
+
+template <>
+struct is_supported<double> : std::true_type {};
+
+template <>
+struct is_supported<std::complex<float>> : std::true_type {};
+
+template <>
+struct is_supported<std::complex<double>> : std::true_type {};
+
+
+#define GKO_BIND_HIPRAND_RANDOM_VECTOR(ValueType, HiprandName)               \
+    inline void rand_vector(int seed, int n, remove_complex<ValueType> mean, \
+                            remove_complex<ValueType> stddev,                \
+                            ValueType *values)                               \
+    {                                                                        \
+        hiprandGenerator_t gen;                                              \
+        hiprandCreateGenerator(&gen, HIPRAND_RNG_PSEUDO_DEFAULT);            \
+        hiprandSetPseudoRandomGeneratorSeed(gen, seed);                      \
+        n = is_complex<ValueType>() ? 2 * n : n;                             \
+        GKO_ASSERT_NO_HIPRAND_ERRORS(HiprandName(                            \
+            gen, reinterpret_cast<remove_complex<ValueType> *>(values), n,   \
+            mean, stddev));                                                  \
+    }                                                                        \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
+                  "semi-colon warnings")
+
+GKO_BIND_HIPRAND_RANDOM_VECTOR(float, hiprandGenerateNormal);
+GKO_BIND_HIPRAND_RANDOM_VECTOR(double, hiprandGenerateNormalDouble);
+GKO_BIND_HIPRAND_RANDOM_VECTOR(std::complex<float>, hiprandGenerateNormal);
+GKO_BIND_HIPRAND_RANDOM_VECTOR(std::complex<double>,
+                               hiprandGenerateNormalDouble);
+
+
+#undef GKO_BIND_HIPRAND_RANDOM_VECTOR
+
+
+}  // namespace hiprand
+}  // namespace hip
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_CUDA_BASE_HIPSPARSE_BINDINGS_HPP_
diff --git a/hip/get_info.cmake b/hip/get_info.cmake
index f98083dc468..20fe5056e90 100644
--- a/hip/get_info.cmake
+++ b/hip/get_info.cmake
@@ -9,7 +9,7 @@ foreach(log_type ${log_types})
 endforeach()
 foreach(log_type ${log_types})
     ginkgo_print_module_footer(${${log_type}} "HIP variables:")
-    set(print_var "HIP_VERSION;HIP_COMPILER;HIP_PATH;ROCM_PATH;HIP_PLATFORM;HIP_ROOT_DIR;HCC_PATH;HIP_RUNTIME;HIPBLAS_PATH;HIPSPARSE_PATH;HIP_CLANG_INCLUDE_PATH;HIP_CLANG_PATH;HIP_HIPCC_EXECUTABLE;HIP_HIPCONFIG_EXECUTABLE;HIP_HOST_COMPILATION_CPP"
+    set(print_var "HIP_VERSION;HIP_COMPILER;HIP_PATH;ROCM_PATH;HIP_PLATFORM;HIP_ROOT_DIR;HCC_PATH;HIP_RUNTIME;HIPBLAS_PATH;HIPSPARSE_PATH;HIPRAND_PATH;ROCRAND_PATH;HIP_CLANG_INCLUDE_PATH;HIP_CLANG_PATH;HIP_HIPCC_EXECUTABLE;HIP_HIPCONFIG_EXECUTABLE;HIP_HOST_COMPILATION_CPP"
         )
     foreach(var ${print_var})
         ginkgo_print_variable(${${log_type}} ${var} )
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index 4afa43f7de9..876d5a4c429 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -40,6 +40,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
+#include "core/components/fill_array.hpp"
+#include "hip/base/hipblas_bindings.hip.hpp"
+#include "hip/base/hiprand_bindings.hip.hpp"
 #include "hip/base/math.hip.hpp"
 #include "hip/base/types.hip.hpp"
 #include "hip/components/atomic.hip.hpp"
@@ -60,6 +63,8 @@ namespace idr {
 
 
 constexpr int default_block_size = 512;
+constexpr int default_dot_dim = 32;
+constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
 #include "common/solver/idr_kernels.hpp.inc"
@@ -78,7 +83,7 @@ void initialize_m(matrix::Dense<ValueType> *m,
 
     const auto grid_dim = ceildiv(m_stride * subspace_dim, default_block_size);
     hipLaunchKernelGGL(HIP_KERNEL_NAME(initialize_m_kernel), grid_dim,
-                       default_block_size, 0, 0 subspace_dim, nrhs,
+                       default_block_size, 0, 0, subspace_dim, nrhs,
                        as_hip_type(m->get_values()), m_stride,
                        as_hip_type(stop_status->get_data()));
 }
@@ -87,6 +92,10 @@ void initialize_m(matrix::Dense<ValueType> *m,
 template <typename ValueType>
 void orthonormalize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors)
 {
+    hiprand::rand_vector(
+        1234ULL,
+        subspace_vectors->get_size()[0] * subspace_vectors->get_stride(), 0.5,
+        0.5, subspace_vectors->get_values());
     hipLaunchKernelGGL(
         HIP_KERNEL_NAME(
             orthonormalize_subspace_vectors_kernel<default_block_size>),
@@ -144,7 +153,7 @@ void update_g_and_u(std::shared_ptr<const HipExecutor> exec, size_type k,
                 g_k->get_stride(), as_hip_type(alpha->get_values()),
                 as_hip_type(stop_status->get_const_data()));
         } else {
-            hipblas::dot(exec->get_hiplas_handle(), size, p_i, 1,
+            hipblas::dot(exec->get_hipblas_handle(), size, p_i, 1,
                          g_k->get_values(), g_k->get_stride(),
                          alpha->get_values());
         }
@@ -171,7 +180,7 @@ void update_g_and_u(std::shared_ptr<const HipExecutor> exec, size_type k,
 template <typename ValueType>
 void update_m(std::shared_ptr<const HipExecutor> exec, size_type k,
               const matrix::Dense<ValueType> *p,
-              const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *m,
+              const matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *m,
               const Array<stopping_status> *stop_status)
 {
     const auto size = g_k->get_size()[0];
@@ -241,7 +250,7 @@ void initialize(std::shared_ptr<const HipExecutor> exec,
                 Array<stopping_status> *stop_status)
 {
     initialize_m(m, stop_status);
-    initialize_subspace_vectors(subspace_vectors);
+    orthonormalize_subspace_vectors(subspace_vectors);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
diff --git a/hip/test/base/CMakeLists.txt b/hip/test/base/CMakeLists.txt
index 795ededc410..f7938076c9d 100644
--- a/hip/test/base/CMakeLists.txt
+++ b/hip/test/base/CMakeLists.txt
@@ -3,7 +3,7 @@ ginkgo_create_hip_test(lin_op)
 ginkgo_create_hip_test(math)
 # Only hcc needs the libraries. nvcc only requires the headers.
 if (GINKGO_HIP_PLATFORM MATCHES "hcc")
-    ginkgo_create_hip_test(exception_helpers roc::hipblas roc::hipsparse)
+    ginkgo_create_hip_test(exception_helpers roc::hipblas roc::hipsparse hip::hiprand roc::rocrand)
 else()
     ginkgo_create_hip_test(exception_helpers)
 endif()
diff --git a/hip/test/base/exception_helpers.hip.cpp b/hip/test/base/exception_helpers.hip.cpp
index 8261cc24f0e..521643ba81f 100644
--- a/hip/test/base/exception_helpers.hip.cpp
+++ b/hip/test/base/exception_helpers.hip.cpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <hip/hip_runtime.h>
 #include <hipblas.h>
+#include <hiprand.h>
 #include <hipsparse.h>
 
 
@@ -68,6 +69,18 @@ TEST(AssertNoHipblasErrors, DoesNotThrowOnSuccess)
 }
 
 
+TEST(AssertNoHiprandErrors, ThrowsOnError)
+{
+    ASSERT_THROW(GKO_ASSERT_NO_HIPRAND_ERRORS(1), gko::HiprandError);
+}
+
+
+TEST(AssertNoHiprandErrors, DoesNotThrowOnSuccess)
+{
+    ASSERT_NO_THROW(GKO_ASSERT_NO_HIPRAND_ERRORS(HIPRAND_STATUS_SUCCESS));
+}
+
+
 TEST(AssertNoHipsparseErrors, ThrowsOnError)
 {
     ASSERT_THROW(GKO_ASSERT_NO_HIPSPARSE_ERRORS(1), gko::HipsparseError);
diff --git a/include/ginkgo/core/base/exception.hpp b/include/ginkgo/core/base/exception.hpp
index 516efdab90a..1ce91cee4f5 100644
--- a/include/ginkgo/core/base/exception.hpp
+++ b/include/ginkgo/core/base/exception.hpp
@@ -311,6 +311,29 @@ class HipblasError : public Error {
 };
 
 
+/**
+ * HiprandError is thrown when a hipRAND routine throws a non-zero error code.
+ */
+class HiprandError : public Error {
+public:
+    /**
+     * Initializes a hipRAND error.
+     *
+     * @param file  The name of the offending source file
+     * @param line  The source code line number where the error occurred
+     * @param func  The name of the hipRAND routine that failed
+     * @param error_code  The resulting hipRAND error code
+     */
+    HiprandError(const std::string &file, int line, const std::string &func,
+                 int64 error_code)
+        : Error(file, line, func + ": " + get_error(error_code))
+    {}
+
+private:
+    static std::string get_error(int64 error_code);
+};
+
+
 /**
  * HipsparseError is thrown when a hipSPARSE routine throws a non-zero error
  * code.
diff --git a/include/ginkgo/core/base/exception_helpers.hpp b/include/ginkgo/core/base/exception_helpers.hpp
index e482f142d2d..10df01005ab 100644
--- a/include/ginkgo/core/base/exception_helpers.hpp
+++ b/include/ginkgo/core/base/exception_helpers.hpp
@@ -392,6 +392,15 @@ inline dim<2> get_size(const dim<2> &size) { return size; }
     ::gko::HipblasError(__FILE__, __LINE__, __func__, _errcode)
 
 
+/**
+ * Instantiates a HiprandError.
+ *
+ * @param errcode  The error code returned from the HIPRAND routine.
+ */
+#define GKO_HIPRAND_ERROR(_errcode) \
+    ::gko::HiprandError(__FILE__, __LINE__, __func__, _errcode)
+
+
 /**
  * Instantiates a HipsparseError.
  *
@@ -429,6 +438,20 @@ inline dim<2> get_size(const dim<2> &size) { return size; }
     } while (false)
 
 
+/**
+ * Asserts that a HIPRAND library call completed without errors.
+ *
+ * @param _hiprand_call  a library call expression
+ */
+#define GKO_ASSERT_NO_HIPRAND_ERRORS(_hiprand_call) \
+    do {                                            \
+        auto _errcode = _hiprand_call;              \
+        if (_errcode != HIPRAND_STATUS_SUCCESS) {   \
+            throw GKO_HIPRAND_ERROR(_errcode);      \
+        }                                           \
+    } while (false)
+
+
 /**
  * Asserts that a HIPSPARSE library call completed without errors.
  *

From 6dda07a4c4e2c29f3b18e048fd563df522be04f1 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 10 Sep 2020 15:42:52 +0200
Subject: [PATCH 0150/1008] Add option to deterministically generate subspace
 vectors for testing

---
 core/device_hooks/hip_hooks.cpp    |  6 ++++++
 core/solver/idr.cpp                |  4 ++--
 core/solver/idr_kernels.hpp        |  2 +-
 cuda/solver/idr_kernels.cu         | 28 +++++++++++++++++++++++-----
 cuda/test/solver/idr_kernels.cpp   |  6 ++++--
 hip/solver/idr_kernels.hip.cpp     | 29 ++++++++++++++++++++++++-----
 hip/test/solver/idr_kernels.cpp    |  6 ++++--
 include/ginkgo/core/solver/idr.hpp | 14 ++++++++++++++
 omp/solver/idr_kernels.cpp         |  7 ++++---
 omp/test/solver/idr_kernels.cpp    |  6 ++++--
 reference/solver/idr_kernels.cpp   |  6 ++++--
 11 files changed, 90 insertions(+), 24 deletions(-)

diff --git a/core/device_hooks/hip_hooks.cpp b/core/device_hooks/hip_hooks.cpp
index a60b657bac7..b98b2fcb8a4 100644
--- a/core/device_hooks/hip_hooks.cpp
+++ b/core/device_hooks/hip_hooks.cpp
@@ -117,6 +117,12 @@ std::string HipblasError::get_error(int64)
 }
 
 
+std::string HiprandError::get_error(int64)
+{
+    return "ginkgo HIP module is not compiled";
+}
+
+
 std::string HipsparseError::get_error(int64)
 {
     return "ginkgo HIP module is not compiled";
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index 7fa631913f1..b52728462f5 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -136,8 +136,8 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     // Initialization
     // m = identity
-    exec->run(
-        idr::make_initialize(m.get(), subspace_vectors_.get(), &stop_status));
+    exec->run(idr::make_initialize(m.get(), subspace_vectors_.get(),
+                                   deterministic_, &stop_status));
 
     // omega = 1
     exec->run(
diff --git a/core/solver/idr_kernels.hpp b/core/solver/idr_kernels.hpp
index ed12a49f0b0..29080c36fe6 100644
--- a/core/solver/idr_kernels.hpp
+++ b/core/solver/idr_kernels.hpp
@@ -50,7 +50,7 @@ namespace idr {
     void initialize(std::shared_ptr<const DefaultExecutor> exec, \
                     matrix::Dense<_type> *m,                     \
                     matrix::Dense<_type> *subspace_vectors,      \
-                    Array<stopping_status> *stop_status)
+                    bool deterministic, Array<stopping_status> *stop_status)
 
 
 #define GKO_DECLARE_IDR_STEP_1_KERNEL(_type)                                 \
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index cf39b1a3cd7..883e7992501 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -36,6 +36,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <random>
 
 
+#include <time.h>
+
+
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 
@@ -89,13 +92,27 @@ void initialize_m(matrix::Dense<ValueType> *m,
 }
 
 
+template <typename ValueType>
+void initialize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors,
+                                 bool deterministic)
+{
+    if (deterministic) {
+        auto subspace_vectors_data = matrix_data<ValueType>(
+            subspace_vectors->get_size(), std::normal_distribution<>(0.0, 1.0),
+            std::ranlux48(15));
+        subspace_vectors->read(subspace_vectors_data);
+    } else {
+        curand::rand_vector(
+            time(NULL),
+            subspace_vectors->get_size()[0] * subspace_vectors->get_stride(),
+            0.0, 1.0, subspace_vectors->get_values());
+    }
+}
+
+
 template <typename ValueType>
 void orthonormalize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors)
 {
-    curand::rand_vector(
-        1234ULL,
-        subspace_vectors->get_size()[0] * subspace_vectors->get_stride(), 0.5,
-        0.5, subspace_vectors->get_values());
     orthonormalize_subspace_vectors_kernel<default_block_size>
         <<<1, default_block_size>>>(
             subspace_vectors->get_size()[0], subspace_vectors->get_size()[1],
@@ -237,10 +254,11 @@ void update_x_r_and_f(size_type k, const matrix::Dense<ValueType> *m,
 template <typename ValueType>
 void initialize(std::shared_ptr<const CudaExecutor> exec,
                 matrix::Dense<ValueType> *m,
-                matrix::Dense<ValueType> *subspace_vectors,
+                matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
                 Array<stopping_status> *stop_status)
 {
     initialize_m(m, stop_status);
+    initialize_subspace_vectors(subspace_vectors, deterministic);
     orthonormalize_subspace_vectors(subspace_vectors);
 }
 
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index df9346f0051..ca51a708c82 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -74,6 +74,7 @@ class Idr : public ::testing::Test {
         d_mtx->copy_from(mtx.get());
         cuda_idr_factory =
             Solver::build()
+                .with_deterministic(true)
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(cuda),
                     gko::stop::ResidualNormReduction<>::build()
@@ -83,6 +84,7 @@ class Idr : public ::testing::Test {
 
         ref_idr_factory =
             Solver::build()
+                .with_deterministic(true)
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(ref),
                     gko::stop::ResidualNormReduction<>::build()
@@ -221,9 +223,9 @@ TEST_F(Idr, CudaIdrInitializeIsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::idr::initialize(ref, m.get(), p.get(),
+    gko::kernels::reference::idr::initialize(ref, m.get(), p.get(), true,
                                              stop_status.get());
-    gko::kernels::cuda::idr::initialize(cuda, d_m.get(), d_p.get(),
+    gko::kernels::cuda::idr::initialize(cuda, d_m.get(), d_p.get(), true,
                                         d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index 876d5a4c429..f930e191f80 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -33,7 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
+#include <random>
+
+
 #include <hip/hip_runtime.h>
+#include <time.h>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
@@ -89,13 +93,27 @@ void initialize_m(matrix::Dense<ValueType> *m,
 }
 
 
+template <typename ValueType>
+void initialize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors,
+                                 bool deterministic)
+{
+    if (deterministic) {
+        auto subspace_vectors_data = matrix_data<ValueType>(
+            subspace_vectors->get_size(), std::normal_distribution<>(0.0, 1.0),
+            std::ranlux48(15));
+        subspace_vectors->read(subspace_vectors_data);
+    } else {
+        hiprand::rand_vector(
+            time(NULL),
+            subspace_vectors->get_size()[0] * subspace_vectors->get_stride(),
+            0.0, 1.0, subspace_vectors->get_values());
+    }
+}
+
+
 template <typename ValueType>
 void orthonormalize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors)
 {
-    hiprand::rand_vector(
-        1234ULL,
-        subspace_vectors->get_size()[0] * subspace_vectors->get_stride(), 0.5,
-        0.5, subspace_vectors->get_values());
     hipLaunchKernelGGL(
         HIP_KERNEL_NAME(
             orthonormalize_subspace_vectors_kernel<default_block_size>),
@@ -246,10 +264,11 @@ void update_x_r_and_f(size_type k, const matrix::Dense<ValueType> *m,
 template <typename ValueType>
 void initialize(std::shared_ptr<const HipExecutor> exec,
                 matrix::Dense<ValueType> *m,
-                matrix::Dense<ValueType> *subspace_vectors,
+                matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
                 Array<stopping_status> *stop_status)
 {
     initialize_m(m, stop_status);
+    initialize_subspace_vectors(subspace_vectors, deterministic);
     orthonormalize_subspace_vectors(subspace_vectors);
 }
 
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index 9f17ff00007..5d5e6c7e74c 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -74,6 +74,7 @@ class Idr : public ::testing::Test {
         d_mtx->copy_from(mtx.get());
         hip_idr_factory =
             Solver::build()
+                .with_deterministic(true)
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(hip),
                     gko::stop::ResidualNormReduction<>::build()
@@ -83,6 +84,7 @@ class Idr : public ::testing::Test {
 
         ref_idr_factory =
             Solver::build()
+                .with_deterministic(true)
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(ref),
                     gko::stop::ResidualNormReduction<>::build()
@@ -221,9 +223,9 @@ TEST_F(Idr, HipIdrInitializeIsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::idr::initialize(ref, m.get(), p.get(),
+    gko::kernels::reference::idr::initialize(ref, m.get(), p.get(), true,
                                              stop_status.get());
-    gko::kernels::hip::idr::initialize(hip, d_m.get(), d_p.get(),
+    gko::kernels::hip::idr::initialize(hip, d_m.get(), d_p.get(), true,
                                        d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index 1c6c825a99b..4690c87cbf9 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -159,6 +159,17 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
          * perpendicular.
          */
         remove_complex<ValueType> GKO_FACTORY_PARAMETER(kappa, 0.7);
+
+        /**
+         * If set to true, the vectors spanning the subspace S are chosen
+         * deterministically. This is mostly needed for testing purposes.
+         *
+         * Note: If 'deterministic' is set to true, the subspace vectors are
+         * generated in serial on the CPU, which can be very slow.
+         *
+         * The default behaviour is to choose the subspace vectors randomly.
+         */
+        bool GKO_FACTORY_PARAMETER(deterministic, false);
     };
     GKO_ENABLE_LIN_OP_FACTORY(Idr, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);
@@ -197,6 +208,8 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
         subspace_dim_ = parameters_.subspace_dim;
 
         kappa_ = parameters_.kappa;
+
+        deterministic_ = parameters_.deterministic;
     }
 
 private:
@@ -204,6 +217,7 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
     std::shared_ptr<const stop::CriterionFactory> stop_criterion_factory_{};
     size_type subspace_dim_;
     remove_complex<ValueType> kappa_;
+    bool deterministic_;
 };
 
 
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index e088ba3f64f..ee39c8f723e 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -36,8 +36,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <algorithm>
 #include <random>
 
-
 #include <omp.h>
+#include <time.h>
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -139,7 +139,7 @@ get_rand_value(Distribution &&dist, Generator &&gen)
 template <typename ValueType>
 void initialize(std::shared_ptr<const OmpExecutor> exec,
                 matrix::Dense<ValueType> *m,
-                matrix::Dense<ValueType> *subspace_vectors,
+                matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
                 Array<stopping_status> *stop_status)
 {
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
@@ -163,7 +163,8 @@ void initialize(std::shared_ptr<const OmpExecutor> exec,
     const auto num_rows = subspace_vectors->get_size()[0];
     const auto num_cols = subspace_vectors->get_size()[1];
     auto dist = std::normal_distribution<remove_complex<ValueType>>(0.0, 1.0);
-    auto gen = std::ranlux48(15);
+    auto seed = deterministic ? 15 : time(NULL);
+    auto gen = std::ranlux48(seed);
     for (size_type row = 0; row < num_rows; row++) {
         for (size_type col = 0; col < num_cols; col++) {
             subspace_vectors->at(row, col) =
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index bee822fba96..f1b97d2ae07 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -74,6 +74,7 @@ class Idr : public ::testing::Test {
         d_mtx->copy_from(mtx.get());
         omp_idr_factory =
             Solver::build()
+                .with_deterministic(true)
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(omp),
                     gko::stop::ResidualNormReduction<>::build()
@@ -83,6 +84,7 @@ class Idr : public ::testing::Test {
 
         ref_idr_factory =
             Solver::build()
+                .with_deterministic(true)
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(ref),
                     gko::stop::ResidualNormReduction<>::build()
@@ -221,9 +223,9 @@ TEST_F(Idr, OmpIdrInitializeIsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::idr::initialize(ref, m.get(), p.get(),
+    gko::kernels::reference::idr::initialize(ref, m.get(), p.get(), true,
                                              stop_status.get());
-    gko::kernels::omp::idr::initialize(omp, d_m.get(), d_p.get(),
+    gko::kernels::omp::idr::initialize(omp, d_m.get(), d_p.get(), true,
                                        d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index 99e6d793dad..b06c204d6da 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
+#include <time.h>
 #include <algorithm>
 #include <random>
 
@@ -133,7 +134,7 @@ get_rand_value(Distribution &&dist, Generator &&gen)
 template <typename ValueType>
 void initialize(std::shared_ptr<const ReferenceExecutor> exec,
                 matrix::Dense<ValueType> *m,
-                matrix::Dense<ValueType> *subspace_vectors,
+                matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
                 Array<stopping_status> *stop_status)
 {
     // Initialize M
@@ -153,7 +154,8 @@ void initialize(std::shared_ptr<const ReferenceExecutor> exec,
     const auto num_rows = subspace_vectors->get_size()[0];
     const auto num_cols = subspace_vectors->get_size()[1];
     auto dist = std::normal_distribution<remove_complex<ValueType>>(0.0, 1.0);
-    auto gen = std::ranlux48(15);
+    auto seed = deterministic ? 15 : time(NULL);
+    auto gen = std::ranlux48(seed);
     for (size_type row = 0; row < num_rows; row++) {
         for (size_type col = 0; col < num_cols; col++) {
             subspace_vectors->at(row, col) =

From c2e2bebddce3e1c7db4a98aaad430cb5eabafacb Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 10 Sep 2020 16:54:24 +0200
Subject: [PATCH 0151/1008] Suggestions from code review

---
 core/device_hooks/common_kernels.inc.cpp |  1 -
 include/ginkgo/core/solver/idr.hpp       | 14 ++++++++------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 8525ec0a072..d43a49fa192 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -465,7 +465,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_FINALIZE_KERNEL);
 }  // namespace bicgstab
 
 
-// TODO (script:idr): adapt this block as needed
 namespace idr {
 
 template <typename ValueType>
diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index 4690c87cbf9..ac453e97a61 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -133,32 +133,34 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
          * Criterion factories.
          */
         std::vector<std::shared_ptr<const stop::CriterionFactory>>
-            GKO_FACTORY_PARAMETER(criteria, nullptr);
+            GKO_FACTORY_PARAMETER_VECTOR(criteria, nullptr);
 
         /**
          * Preconditioner factory.
          */
-        std::shared_ptr<const LinOpFactory> GKO_FACTORY_PARAMETER(
+        std::shared_ptr<const LinOpFactory> GKO_FACTORY_PARAMETER_SCALAR(
             preconditioner, nullptr);
 
         /**
          * Already generated preconditioner. If one is provided, the factory
          * `preconditioner` will be ignored.
          */
-        std::shared_ptr<const LinOp> GKO_FACTORY_PARAMETER(
+        std::shared_ptr<const LinOp> GKO_FACTORY_PARAMETER_SCALAR(
             generated_preconditioner, nullptr);
 
         /**
          * Dimension of the subspace S. Determines how many intermediate
          * residuals are computed in each iteration.
          */
-        size_type GKO_FACTORY_PARAMETER(subspace_dim, 2u);
+        size_type GKO_FACTORY_PARAMETER_SCALAR(subspace_dim, 2u);
 
         /**
          * Threshold to determine if Av_n and v_n are too close to being
          * perpendicular.
+         * This is considered to be the case if
+         * |(Av_n)^H * v_n / (norm(Av_n) * norm(v_n))| < kappa
          */
-        remove_complex<ValueType> GKO_FACTORY_PARAMETER(kappa, 0.7);
+        remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(kappa, 0.7);
 
         /**
          * If set to true, the vectors spanning the subspace S are chosen
@@ -169,7 +171,7 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
          *
          * The default behaviour is to choose the subspace vectors randomly.
          */
-        bool GKO_FACTORY_PARAMETER(deterministic, false);
+        bool GKO_FACTORY_PARAMETER_SCALAR(deterministic, false);
     };
     GKO_ENABLE_LIN_OP_FACTORY(Idr, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);

From 88fbe64949747c58aba66957cebfde7c754c84ea Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Tue, 29 Sep 2020 10:24:41 +0200
Subject: [PATCH 0152/1008] change images to hiprand ones

---
 .gitlab-ci.yml | 86 +++++++++++++++++++++++++-------------------------
 1 file changed, 43 insertions(+), 43 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9969d84177c..6bcf4e5d278 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -127,7 +127,7 @@ stages:
 
 sync:
   stage: sync
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
   variables:
     GIT_STRATEGY: none
     PRIVATE_REPO: git@gitlab.com:ginkgo-project/ginkgo.git
@@ -153,7 +153,7 @@ sync:
 # Job with example runs.
 build/cuda90/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda90-gnu5-llvm39
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda90-gnu5-llvm39
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -171,7 +171,7 @@ build/cuda90/gcc/all/debug/shared:
 
 build/cuda90/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda90-gnu5-llvm39
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda90-gnu5-llvm39
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -192,7 +192,7 @@ build/cuda90/clang/all/release/static:
 # cuda 9.1 and friends
 build/cuda91/gcc/all/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda91-gnu6-llvm40
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda91-gnu6-llvm40
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -210,7 +210,7 @@ build/cuda91/gcc/all/debug/static:
 
 build/cuda91/clang/all/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda91-gnu6-llvm40
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda91-gnu6-llvm40
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -231,7 +231,7 @@ build/cuda91/clang/all/release/shared:
 # cuda 9.2 and friends
 build/cuda92/gcc/all/release/debug:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda92-gnu7-llvm50-intel2017
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -249,7 +249,7 @@ build/cuda92/gcc/all/release/debug:
 
 build/cuda92/intel/cuda/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda92-gnu7-llvm50-intel2017
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -270,7 +270,7 @@ build/cuda92/intel/cuda/release/static:
 # cuda 10.0 and friends
 build/cuda100/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -288,7 +288,7 @@ build/cuda100/gcc/all/debug/shared:
 
 build/cuda100/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -309,7 +309,7 @@ build/cuda100/clang/all/release/static:
 
 build/cuda100/intel/cuda/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -329,7 +329,7 @@ build/cuda100/intel/cuda/release/shared:
 # cuda 10.1 and friends
 build/cuda101/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -347,7 +347,7 @@ build/cuda101/gcc/all/debug/shared:
 
 build/cuda101/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -369,7 +369,7 @@ build/cuda101/clang/all/release/static:
 # clang-cuda with cuda 10.1 and friends
 build/clang-cuda101/gcc/all/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm10-intel2019
   variables:
     <<: *default_variables
     CUDA_COMPILER: "clang++"
@@ -388,7 +388,7 @@ build/clang-cuda101/gcc/all/release/shared:
 
 build/clang-cuda101/clang/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm10-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -410,7 +410,7 @@ build/clang-cuda101/clang/cuda/debug/static:
 # cuda 10.2 and friends
 build/cuda102/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -428,7 +428,7 @@ build/cuda102/gcc/all/debug/shared:
 
 build/cuda102/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -449,7 +449,7 @@ build/cuda102/clang/all/release/static:
 
 build/cuda102/intel/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -470,7 +470,7 @@ build/cuda102/intel/cuda/debug/static:
 # cuda 11.0 and friends
 build/cuda110/gcc/cuda/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -487,7 +487,7 @@ build/cuda110/gcc/cuda/debug/shared:
 
 build/cuda110/clang/cuda/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -507,7 +507,7 @@ build/cuda110/clang/cuda/release/static:
 
 build/cuda110/intel/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -528,7 +528,7 @@ build/cuda110/intel/cuda/debug/static:
 # HIP AMD
 build/amd/gcc/hip/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-amd-gnu8-llvm7
+  image: localhost:5000/add_hiprand_cdb07838_gko-amd-gnu8-llvm7
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -545,7 +545,7 @@ build/amd/gcc/hip/debug/shared:
 
 build/amd/clang/hip/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-amd-gnu8-llvm7
+  image: localhost:5000/add_hiprand_cdb07838_gko-amd-gnu8-llvm7
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -565,7 +565,7 @@ build/amd/clang/hip/release/static:
 # no cuda but latest gcc and clang
 build/nocuda/gcc/core/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_REFERENCE: "OFF"
@@ -580,7 +580,7 @@ build/nocuda/gcc/core/debug/static:
 
 build/nocuda/clang/core/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -596,7 +596,7 @@ build/nocuda/clang/core/release/shared:
 
 build/nocuda/intel/core/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -613,7 +613,7 @@ build/nocuda/intel/core/debug/shared:
 
 build/nocuda/gcc/omp/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -627,7 +627,7 @@ build/nocuda/gcc/omp/release/shared:
 
 build/nocuda/clang/omp/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -644,7 +644,7 @@ build/nocuda/clang/omp/debug/static:
 
 build/nocuda/intel/omp/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -683,7 +683,7 @@ build/dpcpp/cpu/release/static:
 warnings:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -704,7 +704,7 @@ warnings:
 no-circular-deps:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -746,7 +746,7 @@ subdir-build:
 clang-tidy:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -766,7 +766,7 @@ clang-tidy:
 iwyu:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -787,7 +787,7 @@ iwyu:
 # For short living branches or PRs, try to detect an open PR
 sonarqube_cov_:
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
   before_script: *default_before_script
   script:
     - PR_ID=$(curl "https://api.github.com/search/issues?q=sha:${CI_COMMIT_SHA}"
@@ -828,7 +828,7 @@ sonarqube_cov_:
 # (the one that was merged).
 sonarqube_cov:
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
   before_script: *default_before_script
   script:
     - ctest -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=COVERAGE
@@ -855,7 +855,7 @@ sonarqube_cov:
 # Deploy documentation to github-pages
 gh-pages:
   stage: deploy
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     PUBLIC_REPO: git@github.com:ginkgo-project/ginkgo.git
@@ -902,7 +902,7 @@ gh-pages:
 
 threadsanitizer:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - LD_PRELOAD=/usr/local/lib/libomp.so
@@ -926,7 +926,7 @@ threadsanitizer:
 
 leaksanitizer:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=LSAN
@@ -946,7 +946,7 @@ leaksanitizer:
 
 addresssanitizer:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=ASAN
@@ -966,7 +966,7 @@ addresssanitizer:
 
 undefinedsanitizer:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     # the Gold linker is required because of a linker flag issues given by UBsan
@@ -988,7 +988,7 @@ undefinedsanitizer:
 
 cudamemcheck:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=RelWithDebInfo
@@ -1027,7 +1027,7 @@ cudamemcheck:
 
 fineci-benchmark-build:
   stage: benchmark-build
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BENCHMARK_SERVER: FINECI
@@ -1108,7 +1108,7 @@ fineci-benchmark-build:
 
 fineci-benchmark-cuda:
   stage: benchmark-cuda
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BENCHMARK_SERVER: FINECI
@@ -1146,7 +1146,7 @@ fineci-benchmark-cuda:
 
 new-issue-on-failure:
   stage: on-failure
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
   script: curl --request POST "https://gitlab.com/api/v4/projects/${PROJECT_ID}/issues?private_token=${BOT_ACCESS_TOKEN}&title=Error%20in%20${CI_PROJECT_NAME}%20with%20pipeline%20${CI_PIPELINE_ID}%20for%20commit%20${CI_COMMIT_SHA}&labels&description=${CI_PIPELINE_URL}"
   when: on_failure
   only:

From 9a00a0b5f93ac8d4605f99e75c61e1157b4c3448 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Mon, 5 Oct 2020 15:21:47 +0200
Subject: [PATCH 0153/1008] Suggestions from Code Review

---
 common/solver/idr_kernels.hpp.inc        |  24 ++---
 core/device_hooks/common_kernels.inc.cpp |   6 ++
 core/matrix/dense.cpp                    |  33 +++++++
 core/solver/idr.cpp                      | 110 ++++++++++++++-------
 core/solver/idr_kernels.hpp              |  42 +++++---
 cuda/base/curand_bindings.hpp            |  30 +++---
 cuda/solver/idr_kernels.cu               |  23 +++--
 cuda/test/solver/idr_kernels.cpp         |  14 +--
 hip/base/hiprand_bindings.hip.hpp        |  32 +++---
 hip/solver/idr_kernels.hip.cpp           |  17 ++--
 hip/test/solver/idr_kernels.cpp          |  14 +--
 include/ginkgo/core/base/array.hpp       |  19 ++++
 include/ginkgo/core/base/exception.hpp   |   6 +-
 include/ginkgo/core/solver/idr.hpp       |  15 +--
 include/ginkgo/ginkgo.hpp                |   1 +
 omp/solver/idr_kernels.cpp               |  54 +++++++++--
 omp/stop/residual_norm_kernels.cpp       |   5 +-
 omp/test/matrix/csr_kernels.cpp          |   2 +-
 omp/test/solver/gmres_kernels.cpp        |  62 +++++++++++-
 omp/test/solver/idr_kernels.cpp          | 118 +++++++++++++++++++----
 reference/solver/idr_kernels.cpp         |  52 ++++++++--
 reference/stop/residual_norm_kernels.cpp |   5 +-
 reference/test/solver/idr_kernels.cpp    |   7 +-
 23 files changed, 504 insertions(+), 187 deletions(-)

diff --git a/common/solver/idr_kernels.hpp.inc b/common/solver/idr_kernels.hpp.inc
index 094915b37f5..c4ef95ccf2a 100644
--- a/common/solver/idr_kernels.hpp.inc
+++ b/common/solver/idr_kernels.hpp.inc
@@ -147,10 +147,10 @@ __global__ __launch_bounds__(default_block_size) void step_1_kernel(
     const stopping_status *__restrict__ stop_status)
 {
     const auto global_id = thread::get_thread_id_flat();
-    const auto row = global_id / v_stride;
-    const auto col = global_id % v_stride;
+    const auto row = global_id / nrhs;
+    const auto col = global_id % nrhs;
 
-    if (row >= num_rows || col >= nrhs) {
+    if (row >= num_rows) {
         return;
     }
 
@@ -175,10 +175,10 @@ __global__ __launch_bounds__(default_block_size) void step_2_kernel(
     const stopping_status *__restrict__ stop_status)
 {
     const auto global_id = thread::get_thread_id_flat();
-    const auto row = global_id / u_stride;
-    const auto col = global_id % u_stride;
+    const auto row = global_id / nrhs;
+    const auto col = global_id % nrhs;
 
-    if (row >= num_rows || col >= nrhs) {
+    if (row >= num_rows) {
         return;
     }
 
@@ -321,18 +321,6 @@ __global__ __launch_bounds__(default_block_size) void update_x_r_and_f_kernel(
 }
 
 
-template <typename ValueType>
-__global__ __launch_bounds__(config::warp_size) void set_f_zeros(
-    size_type k, size_type nrhs, ValueType *__restrict__ f_values,
-    size_type f_stride)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    if (tidx < nrhs) {
-        f_values[k * f_stride + tidx] = zero<ValueType>();
-    }
-}
-
-
 template <typename ValueType>
 __global__ __launch_bounds__(config::warp_size) void compute_omega_kernel(
     size_type nrhs, const remove_complex<ValueType> kappa,
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index d43a49fa192..d5299e96e75 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -467,6 +467,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_FINALIZE_KERNEL);
 
 namespace idr {
 
+
 template <typename ValueType>
 GKO_DECLARE_IDR_INITIALIZE_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
@@ -492,6 +493,11 @@ GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
+template <typename ValueType>
+GKO_DECLARE_IDR_FINALIZE_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_FINALIZE_KERNEL);
+
 
 }  // namespace idr
 
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index a9e6a3a1515..06f680a1857 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -215,6 +215,39 @@ inline void conversion_helper(SparsityCsr<ValueType, IndexType> *result,
 
 }  // namespace
 
+template <typename ValueType>
+std::unique_ptr<matrix::Dense<ValueType>> interpret_as_real(
+    matrix::Dense<to_complex<ValueType>> *source)
+{
+    auto exec = source->get_executor();
+    auto rows = source->get_size()[0];
+    auto cols = 2 * source->get_size()[1];
+    auto stride = 2 * source->get_stride();
+    return matrix::Dense<ValueType>::create(
+        exec, gko::dim<2>{rows, cols},
+        gko::Array<ValueType>::view(
+            exec, rows * stride,
+            reinterpret_cast<ValueType *>(source->get_values())),
+        stride);
+}
+
+template <typename ValueType>
+std::unique_ptr<matrix::Dense<ValueType>> interpret_as_real(
+    const matrix::Dense<to_complex<ValueType>> *source)
+{
+    auto exec = source->get_executor();
+    auto rows = source->get_size()[0];
+    auto cols = 2 * source->get_size()[1];
+    auto stride = 2 * source->get_stride();
+    return matrix::Dense<ValueType>::create(
+        exec, gko::dim<2>{rows, cols},
+        gko::Array<ValueType>::view(
+            exec, rows * stride,
+            const_cast<ValueType *>(
+                reinterpret_cast<ValueType *>(source->get_const_values()))),
+        stride);
+}
+
 
 template <typename ValueType>
 void Dense<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index b52728462f5..baaf6586e08 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -39,11 +39,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/utils.hpp>
 
+
 #include "core/components/fill_array.hpp"
 #include "core/solver/idr_kernels.hpp"
 
-#include <iostream>
-
 namespace gko {
 namespace solver {
 namespace idr {
@@ -54,6 +53,7 @@ GKO_REGISTER_OPERATION(step_1, idr::step_1);
 GKO_REGISTER_OPERATION(step_2, idr::step_2);
 GKO_REGISTER_OPERATION(step_3, idr::step_3);
 GKO_REGISTER_OPERATION(compute_omega, idr::compute_omega);
+GKO_REGISTER_OPERATION(finalize, idr::finalize);
 GKO_REGISTER_OPERATION(fill_array, components::fill_array);
 
 
@@ -85,31 +85,59 @@ std::unique_ptr<LinOp> Idr<ValueType>::conj_transpose() const
             as<Transposable>(this->get_system_matrix())->conj_transpose()));
 }
 
+template <typename ValueType>
+std::unique_ptr<matrix::Dense<ValueType>> interpret_as_value_type(
+    matrix::Dense<to_complex<ValueType>> *source)
+{
+    auto exec = source->get_executor();
+    auto rows = source->get_size()[0];
+    auto cols = is_complex<ValueType>() ? source->get_size()[1]
+                                        : 2 * source->get_size()[1];
+    auto stride = is_complex<ValueType>() ? source->get_stride()
+                                          : 2 * source->get_stride();
+    return matrix::Dense<ValueType>::create(
+        exec, gko::dim<2>{rows, cols},
+        gko::Array<ValueType>::view(
+            exec, rows * stride,
+            reinterpret_cast<ValueType *>(source->get_values())),
+        stride);
+}
+
 
 template <typename ValueType>
 void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 {
     using std::swap;
-    using Vector = matrix::Dense<ValueType>;
+    using ComplexType = to_complex<ValueType>;
+    using Vector = matrix::Dense<ComplexType>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
+    bool complex = true;
 
     auto exec = this->get_executor();
 
-    auto one_op = initialize<Vector>({one<ValueType>()}, exec);
-    auto neg_one_op = initialize<Vector>({-one<ValueType>()}, exec);
+    auto one_op =
+        initialize<matrix::Dense<ValueType>>({one<ValueType>()}, exec);
+    auto neg_one_op =
+        initialize<matrix::Dense<ValueType>>({-one<ValueType>()}, exec);
+    auto complex_neg_one_op =
+        initialize<Vector>({-one<to_complex<ValueType>>()}, exec);
+
+    auto dense_b = as<matrix::Dense<ValueType>>(b);
+    auto dense_x = as<matrix::Dense<ValueType>>(x);
+
+    auto complex_b = Vector::create(exec, dense_b->get_size());
+    auto complex_x = Vector::create(exec, dense_x->get_size());
 
-    auto dense_b = as<Vector>(b);
-    auto dense_x = as<Vector>(x);
 
     constexpr uint8 RelativeStoppingId{1};
 
     const auto problem_size = system_matrix_->get_size()[0];
-    const auto nrhs = dense_b->get_size()[1];
+    const auto nrhs = complex_b->get_size()[1];
 
-    auto residual = Vector::create_with_config_of(dense_b);
-    auto v = Vector::create_with_config_of(dense_b);
-    auto t = Vector::create_with_config_of(dense_b);
-    auto helper = Vector::create_with_config_of(dense_b);
+    auto residual = Vector::create_with_config_of(complex_b.get());
+    auto v = Vector::create_with_config_of(complex_b.get());
+    auto t = Vector::create_with_config_of(complex_b.get());
+    auto helper = Vector::create_with_config_of(complex_b.get());
 
     auto m =
         Vector::create(exec, gko::dim<2>{subspace_dim_, subspace_dim_ * nrhs});
@@ -131,50 +159,55 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     bool one_changed{};
     Array<stopping_status> stop_status(exec, nrhs);
 
-    auto subspace_vectors_ =
+    // The dense matrix containing the randomly generated subspace vectors.
+    // Stored in column major order and complex conjugated. So, if the
+    // matrix containing the subspace vectors in row major order is called P,
+    // subspace_vectors actually contains P^H.
+    auto subspace_vectors =
         Vector::create(exec, gko::dim<2>(subspace_dim_, problem_size));
 
     // Initialization
     // m = identity
-    exec->run(idr::make_initialize(m.get(), subspace_vectors_.get(),
-                                   deterministic_, &stop_status));
+    exec->run(idr::make_initialize(
+        dense_b, dense_x, complex_b.get(), complex_x.get(), m.get(),
+        subspace_vectors.get(), deterministic_, &stop_status));
 
     // omega = 1
     exec->run(
-        idr::make_fill_array(omega->get_values(), nrhs, one<ValueType>()));
+        idr::make_fill_array(omega->get_values(), nrhs, one<ComplexType>()));
 
     // residual = b - Ax
-    residual->copy_from(dense_b);
-    system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(),
+    residual->copy_from(complex_b.get());
+    system_matrix_->apply(neg_one_op.get(), complex_x.get(), one_op.get(),
                           residual.get());
 
     // g = u = 0
     exec->run(idr::make_fill_array(
-        g->get_values(), problem_size * g->get_stride(), zero<ValueType>()));
+        g->get_values(), problem_size * g->get_stride(), zero<ComplexType>()));
     exec->run(idr::make_fill_array(
-        u->get_values(), problem_size * u->get_stride(), zero<ValueType>()));
+        u->get_values(), problem_size * u->get_stride(), zero<ComplexType>()));
 
 
-    auto stop_criterion = stop_criterion_factory_->generate(
-        system_matrix_, std::shared_ptr<const LinOp>(b, [](const LinOp *) {}),
-        x, residual.get());
+    /*auto stop_criterion = stop_criterion_factory_->generate(
+        system_matrix_, std::shared_ptr<const LinOp>(complex_b.get(), [](const
+       LinOp *) {}), complex_x.get(), residual.get());*/
 
     int total_iter = -1;
 
-    while (true) {
+    while (total_iter < 3) {
         ++total_iter;
-        this->template log<log::Logger::iteration_complete>(
-            this, total_iter, residual.get(), dense_x);
+        /*this->template log<log::Logger::iteration_complete>(
+            this, total_iter, residual.get(), complex_x.get());
 
         if (stop_criterion->update()
                 .num_iterations(total_iter)
                 .residual(residual.get())
-                .solution(dense_x)
+                .solution(complex_x.get())
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
             break;
-        }
+        }*/
 
-        subspace_vectors_->apply(residual.get(), f.get());
+        subspace_vectors->apply(residual.get(), f.get());
         // f = P^H * residual
 
         for (size_type k = 0; k < subspace_dim_; k++) {
@@ -184,7 +217,9 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             // c = M \ f = (c_1, ..., c_s)^T
             // v = residual - c_k * g_k - ... - c_s * g_s
 
-            get_preconditioner()->apply(v.get(), helper.get());
+            get_preconditioner()->apply(
+                interpret_as_value_type<ValueType>(v.get()).get(),
+                interpret_as_value_type<ValueType>(helper.get()).get());
 
             exec->run(idr::make_step_2(k, omega.get(), helper.get(), c.get(),
                                        u.get(), &stop_status));
@@ -196,10 +231,10 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             system_matrix_->apply(u_k.get(), helper.get());
             // g_k = Au_k
 
-            exec->run(idr::make_step_3(k, subspace_vectors_.get(), g.get(),
+            exec->run(idr::make_step_3(k, subspace_vectors.get(), g.get(),
                                        helper.get(), u.get(), m.get(), f.get(),
-                                       alpha.get(), residual.get(), dense_x,
-                                       &stop_status));
+                                       alpha.get(), residual.get(),
+                                       complex_x.get(), &stop_status));
             // for i = 1 to k - 1 do
             //     alpha = p^H_i * g_k / m_i,i
             //     g_k -= alpha * g_i
@@ -214,7 +249,9 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             // f = (0,...,0,f_k+1 - beta * m_k+1,k,...,f_s - beta * m_s,k)
         }
 
-        get_preconditioner()->apply(residual.get(), helper.get());
+        get_preconditioner()->apply(
+            interpret_as_value_type<ValueType>(residual.get()).get(),
+            interpret_as_value_type<ValueType>(helper.get()).get());
         system_matrix_->apply(helper.get(), t.get());
 
         t->compute_dot(residual.get(), omega.get());
@@ -225,9 +262,9 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         exec->run(idr::make_compute_omega(
             kappa_, tht.get(), residual_norm.get(), omega.get(), &stop_status));
 
-        t->scale(neg_one_op.get());
+        t->scale(complex_neg_one_op.get());
         residual->add_scaled(omega.get(), t.get());
-        dense_x->add_scaled(omega.get(), helper.get());
+        complex_x.get()->add_scaled(omega.get(), helper.get());
 
         // omega = (t^H * residual) / (t^H * t)
         // rho = (t^H * residual) / (norm(t) * norm(residual))
@@ -237,6 +274,7 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         // residual -= omega * t
         // dense_x += omega * v
     }
+    exec->run(idr::make_finalize(complex_x.get(), dense_x));
 }
 
 
diff --git a/core/solver/idr_kernels.hpp b/core/solver/idr_kernels.hpp
index 29080c36fe6..1dcc7c7ea10 100644
--- a/core/solver/idr_kernels.hpp
+++ b/core/solver/idr_kernels.hpp
@@ -46,10 +46,14 @@ namespace kernels {
 namespace idr {
 
 
-#define GKO_DECLARE_IDR_INITIALIZE_KERNEL(_type)                 \
-    void initialize(std::shared_ptr<const DefaultExecutor> exec, \
-                    matrix::Dense<_type> *m,                     \
-                    matrix::Dense<_type> *subspace_vectors,      \
+#define GKO_DECLARE_IDR_INITIALIZE_KERNEL(_type)                        \
+    void initialize(std::shared_ptr<const DefaultExecutor> exec,        \
+                    const matrix::Dense<_type> *dense_b,                \
+                    const matrix::Dense<_type> *dense_x,                \
+                    matrix::Dense<to_complex<_type>> *complex_b,        \
+                    matrix::Dense<to_complex<_type>> *complex_x,        \
+                    matrix::Dense<to_complex<_type>> *m,                \
+                    matrix::Dense<to_complex<_type>> *subspace_vectors, \
                     bool deterministic, Array<stopping_status> *stop_status)
 
 
@@ -89,17 +93,25 @@ namespace idr {
         const Array<stopping_status> *stop_status)
 
 
-#define GKO_DECLARE_ALL_AS_TEMPLATES              \
-    template <typename ValueType>                 \
-    GKO_DECLARE_IDR_INITIALIZE_KERNEL(ValueType); \
-    template <typename ValueType>                 \
-    GKO_DECLARE_IDR_STEP_1_KERNEL(ValueType);     \
-    template <typename ValueType>                 \
-    GKO_DECLARE_IDR_STEP_2_KERNEL(ValueType);     \
-    template <typename ValueType>                 \
-    GKO_DECLARE_IDR_STEP_3_KERNEL(ValueType);     \
-    template <typename ValueType>                 \
-    GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL(ValueType)
+#define GKO_DECLARE_IDR_FINALIZE_KERNEL(_type)                       \
+    void finalize(std::shared_ptr<const DefaultExecutor> exec,       \
+                  const matrix::Dense<to_complex<_type>> *complex_x, \
+                  matrix::Dense<_type> *dense_x)
+
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES                 \
+    template <typename ValueType>                    \
+    GKO_DECLARE_IDR_INITIALIZE_KERNEL(ValueType);    \
+    template <typename ValueType>                    \
+    GKO_DECLARE_IDR_STEP_1_KERNEL(ValueType);        \
+    template <typename ValueType>                    \
+    GKO_DECLARE_IDR_STEP_2_KERNEL(ValueType);        \
+    template <typename ValueType>                    \
+    GKO_DECLARE_IDR_STEP_3_KERNEL(ValueType);        \
+    template <typename ValueType>                    \
+    GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL(ValueType); \
+    template <typename ValueType>                    \
+    GKO_DECLARE_IDR_FINALIZE_KERNEL(ValueType)
 
 
 }  // namespace idr
diff --git a/cuda/base/curand_bindings.hpp b/cuda/base/curand_bindings.hpp
index 8e57c8025e3..219994f2a5b 100644
--- a/cuda/base/curand_bindings.hpp
+++ b/cuda/base/curand_bindings.hpp
@@ -47,16 +47,11 @@ namespace gko {
 namespace kernels {
 namespace cuda {
 /**
- * @brief The CUSPARSE namespace.
+ * @brief The CURAND namespace.
  *
- * @ingroup cusparse
+ * @ingroup curand
  */
 namespace curand {
-/**
- * @brief The detail namespace.
- *
- * @ingroup detail
- */
 
 
 template <typename ValueType>
@@ -75,14 +70,21 @@ template <>
 struct is_supported<std::complex<double>> : std::true_type {};
 
 
+inline curandGenerator_t rand_generator(int64 seed,
+                                        curandRngType generator_type)
+{
+    curandGenerator_t gen;
+    curandCreateGenerator(&gen, generator_type);
+    curandSetPseudoRandomGeneratorSeed(gen, seed);
+    return gen;
+}
+
+
 #define GKO_BIND_CURAND_RANDOM_VECTOR(ValueType, CurandName)                 \
-    inline void rand_vector(int seed, int n, remove_complex<ValueType> mean, \
-                            remove_complex<ValueType> stddev,                \
-                            ValueType *values)                               \
+    inline void rand_vector(                                                 \
+        curandGenerator_t &gen, int n, remove_complex<ValueType> mean,       \
+        remove_complex<ValueType> stddev, ValueType *values)                 \
     {                                                                        \
-        curandGenerator_t gen;                                               \
-        curandCreateGenerator(&gen, CURAND_RNG_PSEUDO_DEFAULT);              \
-        curandSetPseudoRandomGeneratorSeed(gen, seed);                       \
         n = is_complex<ValueType>() ? 2 * n : n;                             \
         GKO_ASSERT_NO_CURAND_ERRORS(CurandName(                              \
             gen, reinterpret_cast<remove_complex<ValueType> *>(values), n,   \
@@ -107,4 +109,4 @@ GKO_BIND_CURAND_RANDOM_VECTOR(std::complex<double>, curandGenerateNormalDouble);
 }  // namespace gko
 
 
-#endif  // GKO_CUDA_BASE_CUSPARSE_BINDINGS_HPP_
+#endif  // GKO_CUDA_BASE_CURAND_BINDINGS_HPP_
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index 883e7992501..158653f5ade 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -33,10 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
-#include <random>
-
-
 #include <time.h>
+#include <random>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
@@ -102,8 +100,10 @@ void initialize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors,
             std::ranlux48(15));
         subspace_vectors->read(subspace_vectors_data);
     } else {
+        auto gen =
+            curand::rand_generator(time(NULL), CURAND_RNG_PSEUDO_DEFAULT);
         curand::rand_vector(
-            time(NULL),
+            gen,
             subspace_vectors->get_size()[0] * subspace_vectors->get_stride(),
             0.0, 1.0, subspace_vectors->get_values());
     }
@@ -223,7 +223,8 @@ void update_m(std::shared_ptr<const CudaExecutor> exec, size_type k,
 
 
 template <typename ValueType>
-void update_x_r_and_f(size_type k, const matrix::Dense<ValueType> *m,
+void update_x_r_and_f(std::shared_ptr<const CudaExecutor> exec, size_type k,
+                      const matrix::Dense<ValueType> *m,
                       const matrix::Dense<ValueType> *g,
                       const matrix::Dense<ValueType> *u,
                       matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *r,
@@ -243,8 +244,8 @@ void update_x_r_and_f(size_type k, const matrix::Dense<ValueType> *m,
         as_cuda_type(r->get_values()), r->get_stride(),
         as_cuda_type(x->get_values()), x->get_stride(),
         as_cuda_type(stop_status->get_const_data()));
-    set_f_zeros<<<ceildiv(nrhs, config::warp_size), config::warp_size>>>(
-        k, nrhs, as_cuda_type(f->get_values()), f->get_stride());
+    components::fill_array(exec, f->get_values() + k * f->get_stride(), nrhs,
+                           zero<ValueType>());
 }
 
 
@@ -280,8 +281,7 @@ void step_1(std::shared_ptr<const CudaExecutor> exec, const size_type k,
     const auto subspace_dim = m->get_size()[0];
     const auto nrhs = m->get_size()[1] / subspace_dim;
 
-    const auto grid_dim =
-        ceildiv(v->get_stride() * num_rows, default_block_size);
+    const auto grid_dim = ceildiv(nrhs * num_rows, default_block_size);
     step_1_kernel<<<grid_dim, default_block_size>>>(
         k, num_rows, subspace_dim, nrhs,
         as_cuda_type(residual->get_const_values()), residual->get_stride(),
@@ -305,8 +305,7 @@ void step_2(std::shared_ptr<const CudaExecutor> exec, const size_type k,
     const auto nrhs = preconditioned_vector->get_size()[1];
     const auto subspace_dim = u->get_size()[1] / nrhs;
 
-    const auto grid_dim =
-        ceildiv(u->get_stride() * num_rows, default_block_size);
+    const auto grid_dim = ceildiv(nrhs * num_rows, default_block_size);
     step_2_kernel<<<grid_dim, default_block_size>>>(
         k, num_rows, subspace_dim, nrhs,
         as_cuda_type(omega->get_const_values()),
@@ -331,7 +330,7 @@ void step_3(std::shared_ptr<const CudaExecutor> exec, const size_type k,
 {
     update_g_and_u(exec, k, p, m, alpha, g, g_k, u, stop_status);
     update_m(exec, k, p, g_k, m, stop_status);
-    update_x_r_and_f(k, m, g, u, f, residual, x, stop_status);
+    update_x_r_and_f(exec, k, m, g, u, f, residual, x, stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index ca51a708c82..e1457ca55a1 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -219,7 +219,7 @@ class Idr : public ::testing::Test {
 };
 
 
-TEST_F(Idr, CudaIdrInitializeIsEquivalentToRef)
+TEST_F(Idr, IdrInitializeIsEquivalentToRef)
 {
     initialize_data();
 
@@ -233,7 +233,7 @@ TEST_F(Idr, CudaIdrInitializeIsEquivalentToRef)
 }
 
 
-TEST_F(Idr, CudaIdrStep1IsEquivalentToRef)
+TEST_F(Idr, IdrStep1IsEquivalentToRef)
 {
     initialize_data();
 
@@ -250,7 +250,7 @@ TEST_F(Idr, CudaIdrStep1IsEquivalentToRef)
 }
 
 
-TEST_F(Idr, CudaIdrStep2IsEquivalentToRef)
+TEST_F(Idr, IdrStep2IsEquivalentToRef)
 {
     initialize_data();
 
@@ -264,7 +264,7 @@ TEST_F(Idr, CudaIdrStep2IsEquivalentToRef)
 }
 
 
-TEST_F(Idr, CudaIdrStep3IsEquivalentToRef)
+TEST_F(Idr, IdrStep3IsEquivalentToRef)
 {
     initialize_data();
 
@@ -285,7 +285,7 @@ TEST_F(Idr, CudaIdrStep3IsEquivalentToRef)
 }
 
 
-TEST_F(Idr, CudaIdrComputeOmegaIsEquivalentToRef)
+TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
 {
     initialize_data();
 
@@ -301,7 +301,7 @@ TEST_F(Idr, CudaIdrComputeOmegaIsEquivalentToRef)
 }
 
 
-TEST_F(Idr, CudaIdrApplyOneRHSIsEquivalentToRef)
+TEST_F(Idr, IdrApplyOneRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 1;
@@ -322,7 +322,7 @@ TEST_F(Idr, CudaIdrApplyOneRHSIsEquivalentToRef)
 }
 
 
-TEST_F(Idr, CudaIdrApplyMultipleRHSIsEquivalentToRef)
+TEST_F(Idr, IdrApplyMultipleRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 16;
diff --git a/hip/base/hiprand_bindings.hip.hpp b/hip/base/hiprand_bindings.hip.hpp
index 6bb56b3a635..86550f6aecd 100644
--- a/hip/base/hiprand_bindings.hip.hpp
+++ b/hip/base/hiprand_bindings.hip.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CUDA_BASE_HIPRAND_BINDINGS_HPP_
-#define GKO_CUDA_BASE_HIPRAND_BINDINGS_HPP_
+#ifndef GKO_HIP_BASE_HIPRAND_BINDINGS_HIP_HPP_
+#define GKO_HIP_BASE_HIPRAND_BINDINGS_HIP_HPP_
 
 
 #include <hiprand.h>
@@ -47,16 +47,11 @@ namespace gko {
 namespace kernels {
 namespace hip {
 /**
- * @brief The HIPSPARSE namespace.
+ * @brief The HIPRAND namespace.
  *
  * @ingroup hiprand
  */
 namespace hiprand {
-/**
- * @brief The detail namespace.
- *
- * @ingroup detail
- */
 
 
 template <typename ValueType>
@@ -75,14 +70,21 @@ template <>
 struct is_supported<std::complex<double>> : std::true_type {};
 
 
+inline hiprandGenerator_t rand_generator(int64 seed,
+                                         hiprandRngType generator_type)
+{
+    hiprandGenerator_t gen;
+    hiprandCreateGenerator(&gen, generator_type);
+    hiprandSetPseudoRandomGeneratorSeed(gen, seed);
+    return gen;
+}
+
+
 #define GKO_BIND_HIPRAND_RANDOM_VECTOR(ValueType, HiprandName)               \
-    inline void rand_vector(int seed, int n, remove_complex<ValueType> mean, \
-                            remove_complex<ValueType> stddev,                \
-                            ValueType *values)                               \
+    inline void rand_vector(                                                 \
+        hiprandGenerator_t &gen, int n, remove_complex<ValueType> mean,      \
+        remove_complex<ValueType> stddev, ValueType *values)                 \
     {                                                                        \
-        hiprandGenerator_t gen;                                              \
-        hiprandCreateGenerator(&gen, HIPRAND_RNG_PSEUDO_DEFAULT);            \
-        hiprandSetPseudoRandomGeneratorSeed(gen, seed);                      \
         n = is_complex<ValueType>() ? 2 * n : n;                             \
         GKO_ASSERT_NO_HIPRAND_ERRORS(HiprandName(                            \
             gen, reinterpret_cast<remove_complex<ValueType> *>(values), n,   \
@@ -108,4 +110,4 @@ GKO_BIND_HIPRAND_RANDOM_VECTOR(std::complex<double>,
 }  // namespace gko
 
 
-#endif  // GKO_CUDA_BASE_HIPSPARSE_BINDINGS_HPP_
+#endif  // GKO_HIP_BASE_HIPRAND_BINDINGS_HIP_HPP_
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index f930e191f80..dcf8eb86dc7 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -33,11 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
+#include <time.h>
 #include <random>
 
 
 #include <hip/hip_runtime.h>
-#include <time.h>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
@@ -103,8 +103,10 @@ void initialize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors,
             std::ranlux48(15));
         subspace_vectors->read(subspace_vectors_data);
     } else {
+        auto gen =
+            hiprand::rand_generator(time(NULL), HIPRAND_RNG_PSEUDO_DEFAULT);
         hiprand::rand_vector(
-            time(NULL),
+            gen,
             subspace_vectors->get_size()[0] * subspace_vectors->get_stride(),
             0.0, 1.0, subspace_vectors->get_values());
     }
@@ -230,7 +232,8 @@ void update_m(std::shared_ptr<const HipExecutor> exec, size_type k,
 
 
 template <typename ValueType>
-void update_x_r_and_f(size_type k, const matrix::Dense<ValueType> *m,
+void update_x_r_and_f(std::shared_ptr<const HipExecutor> exec, size_type k,
+                      const matrix::Dense<ValueType> *m,
                       const matrix::Dense<ValueType> *g,
                       const matrix::Dense<ValueType> *u,
                       matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *r,
@@ -251,10 +254,8 @@ void update_x_r_and_f(size_type k, const matrix::Dense<ValueType> *m,
                        as_hip_type(r->get_values()), r->get_stride(),
                        as_hip_type(x->get_values()), x->get_stride(),
                        as_hip_type(stop_status->get_const_data()));
-    hipLaunchKernelGGL(HIP_KERNEL_NAME(set_f_zeros),
-                       ceildiv(nrhs, config::warp_size), config::warp_size, 0,
-                       0, k, nrhs, as_hip_type(f->get_values()),
-                       f->get_stride());
+    components::fill_array(exec, f->get_values() + k * f->get_stride(), nrhs,
+                           zero<ValueType>());
 }
 
 
@@ -340,7 +341,7 @@ void step_3(std::shared_ptr<const HipExecutor> exec, const size_type k,
 {
     update_g_and_u(exec, k, p, m, alpha, g, g_k, u, stop_status);
     update_m(exec, k, p, g_k, m, stop_status);
-    update_x_r_and_f(k, m, g, u, f, residual, x, stop_status);
+    update_x_r_and_f(exec, k, m, g, u, f, residual, x, stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index 5d5e6c7e74c..c5ac1d00894 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -219,7 +219,7 @@ class Idr : public ::testing::Test {
 };
 
 
-TEST_F(Idr, HipIdrInitializeIsEquivalentToRef)
+TEST_F(Idr, IdrInitializeIsEquivalentToRef)
 {
     initialize_data();
 
@@ -233,7 +233,7 @@ TEST_F(Idr, HipIdrInitializeIsEquivalentToRef)
 }
 
 
-TEST_F(Idr, HipIdrStep1IsEquivalentToRef)
+TEST_F(Idr, IdrStep1IsEquivalentToRef)
 {
     initialize_data();
 
@@ -250,7 +250,7 @@ TEST_F(Idr, HipIdrStep1IsEquivalentToRef)
 }
 
 
-TEST_F(Idr, HipIdrStep2IsEquivalentToRef)
+TEST_F(Idr, IdrStep2IsEquivalentToRef)
 {
     initialize_data();
 
@@ -264,7 +264,7 @@ TEST_F(Idr, HipIdrStep2IsEquivalentToRef)
 }
 
 
-TEST_F(Idr, HipIdrStep3IsEquivalentToRef)
+TEST_F(Idr, IdrStep3IsEquivalentToRef)
 {
     initialize_data();
 
@@ -285,7 +285,7 @@ TEST_F(Idr, HipIdrStep3IsEquivalentToRef)
 }
 
 
-TEST_F(Idr, HipIdrComputeOmegaIsEquivalentToRef)
+TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
 {
     initialize_data();
 
@@ -301,7 +301,7 @@ TEST_F(Idr, HipIdrComputeOmegaIsEquivalentToRef)
 }
 
 
-TEST_F(Idr, HipIdrApplyOneRHSIsEquivalentToRef)
+TEST_F(Idr, IdrApplyOneRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 1;
@@ -322,7 +322,7 @@ TEST_F(Idr, HipIdrApplyOneRHSIsEquivalentToRef)
 }
 
 
-TEST_F(Idr, HipIdrApplyMultipleRHSIsEquivalentToRef)
+TEST_F(Idr, IdrApplyMultipleRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 16;
diff --git a/include/ginkgo/core/base/array.hpp b/include/ginkgo/core/base/array.hpp
index 59960520edf..8d02bfd030c 100644
--- a/include/ginkgo/core/base/array.hpp
+++ b/include/ginkgo/core/base/array.hpp
@@ -288,6 +288,25 @@ class Array {
         return Array{exec, num_elems, data, view_deleter{}};
     }
 
+    /**
+     * Creates an Array from existing memory.
+     *
+     * The Array does not take ownership of the memory, and will not deallocate
+     * it once it goes out of scope. This array type cannot use the function
+     * `resize_and_reset` since it does not own the data it should resize.
+     *
+     * @param exec  executor where `data` is located
+     * @param num_elems  number of elements in `data`
+     * @param data  chunk of memory used to create the array
+     *
+     * @return an Array constructed from `data`
+     */
+    static Array view(std::shared_ptr<const Executor> exec, size_type num_elems,
+                      const value_type *data)
+    {
+        return Array{exec, num_elems, data, view_deleter{}};
+    }
+
     /**
      * Copies data from another array or view. In the case of an array target,
      * the array is resized to match the source's size. In the case of a view
diff --git a/include/ginkgo/core/base/exception.hpp b/include/ginkgo/core/base/exception.hpp
index 1ce91cee4f5..20ac5ba8958 100644
--- a/include/ginkgo/core/base/exception.hpp
+++ b/include/ginkgo/core/base/exception.hpp
@@ -225,12 +225,12 @@ class CublasError : public Error {
 class CurandError : public Error {
 public:
     /**
-     * Initializes a cuBLAS error.
+     * Initializes a cuRAND error.
      *
      * @param file  The name of the offending source file
      * @param line  The source code line number where the error occurred
-     * @param func  The name of the cuBLAS routine that failed
-     * @param error_code  The resulting cuBLAS error code
+     * @param func  The name of the cuRAND routine that failed
+     * @param error_code  The resulting cuRAND error code
      */
     CurandError(const std::string &file, int line, const std::string &func,
                 int64 error_code)
diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index ac453e97a61..46703a365c2 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -52,7 +52,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace gko {
 /**
- * @brief The ginkgo Solve namespace.
+ * @brief The ginkgo Solver namespace.
  *
  * @ingroup solvers
  */
@@ -61,13 +61,17 @@ namespace solver {
 
 /**
  * IDR(s) is an efficient method for solving large nonsymmetric systems of
- * linear equations.
+ * linear equations. The implemented version is the one presented in the
+ * paper "Algorithm 913: An elegant IDR(s) variant that efficiently exploits
+ * biorthogonality properties" by M. B. Van Gijzen and P. Sonneveld.
  *
  * The method is based on the induced dimension reduction theorem which
  * provides a way to construct subsequent residuals that lie in a sequence
- * of shrinking subspaces.
+ * of shrinking subspaces. These subspaces are spanned by s vectors which are
+ * first generated randomly and then orthonormalized. They are stored in
+ * a dense matrix.
  *
- * @tparam ValueType precision of the elements of the system matrix.
+ * @tparam ValueType  precision of the elements of the system matrix.
  *
  * @ingroup idr
  * @ingroup solvers
@@ -206,11 +210,8 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
         }
         stop_criterion_factory_ =
             stop::combine(std::move(parameters_.criteria));
-
         subspace_dim_ = parameters_.subspace_dim;
-
         kappa_ = parameters_.kappa;
-
         deterministic_ = parameters_.deterministic;
     }
 
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 40f415b2156..1052dcdf50b 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -55,6 +55,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/polymorphic_object.hpp>
 #include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/base/range_accessors.hpp>
+#include <ginkgo/core/base/readable_data.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/base/utils.hpp>
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index ee39c8f723e..f1d9aba566b 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -33,11 +33,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
+#include <time.h>
 #include <algorithm>
+#include <iostream>
 #include <random>
 
+
 #include <omp.h>
-#include <time.h>
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -138,13 +140,28 @@ get_rand_value(Distribution &&dist, Generator &&gen)
 
 template <typename ValueType>
 void initialize(std::shared_ptr<const OmpExecutor> exec,
-                matrix::Dense<ValueType> *m,
-                matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
-                Array<stopping_status> *stop_status)
+                const matrix::Dense<ValueType> *dense_b,
+                const matrix::Dense<ValueType> *dense_x,
+                matrix::Dense<to_complex<ValueType>> *complex_b,
+                matrix::Dense<to_complex<ValueType>> *complex_x,
+                matrix::Dense<to_complex<ValueType>> *m,
+                matrix::Dense<to_complex<ValueType>> *subspace_vectors,
+                bool deterministic, Array<stopping_status> *stop_status)
 {
-#pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
+#pragma omp declare reduction(add:to_complex<ValueType> : omp_out = omp_out + omp_in)
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
 
+    // Initialize complex b and x
+#pragma omp parallel for
+    for (size_type row = 0; row < dense_b->get_size()[0]; row++) {
+        for (size_type rhs = 0; rhs < nrhs; rhs++) {
+            complex_x->at(row, rhs) =
+                to_complex<ValueType>{dense_x->at(row, rhs)};
+            complex_b->at(row, rhs) =
+                to_complex<ValueType>{dense_b->at(row, rhs)};
+        }
+    }
+
     // Initialize M
 #pragma omp parallel for
     for (size_type i = 0; i < nrhs; i++) {
@@ -154,8 +171,9 @@ void initialize(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for
     for (size_type row = 0; row < m->get_size()[0]; row++) {
         for (size_type col = 0; col < m->get_size()[1]; col++) {
-            m->at(row, col) =
-                (row == col / nrhs) ? one<ValueType>() : zero<ValueType>();
+            m->at(row, col) = (row == col / nrhs)
+                                  ? one<to_complex<ValueType>>()
+                                  : zero<to_complex<ValueType>>();
         }
     }
 
@@ -172,7 +190,7 @@ void initialize(std::shared_ptr<const OmpExecutor> exec,
         }
 
         for (size_type i = 0; i < row; i++) {
-            auto dot = zero<ValueType>();
+            auto dot = zero<to_complex<ValueType>>();
 #pragma omp parallel for reduction(add : dot)
             for (size_type j = 0; j < num_cols; j++) {
                 dot += subspace_vectors->at(row, j) *
@@ -340,6 +358,26 @@ void compute_omega(
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
 
+template <typename ValueType>
+void finalize(std::shared_ptr<const OmpExecutor> exec,
+              const matrix::Dense<to_complex<ValueType>> *complex_x,
+              matrix::Dense<ValueType> *dense_x)
+{
+    if (is_complex<ValueType>()) {
+        dense_x->copy_from(complex_x);
+    } else {
+#pragma omp parallel for
+        for (size_type row = 0; row < dense_x->get_size()[0]; row++) {
+            for (size_type rhs = 0; rhs < dense_x->get_size()[1]; rhs++) {
+                dense_x->at(row, rhs) = real(complex_x->at(row, rhs));
+            }
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_FINALIZE_KERNEL);
+
+
 }  // namespace idr
 }  // namespace omp
 }  // namespace kernels
diff --git a/omp/stop/residual_norm_kernels.cpp b/omp/stop/residual_norm_kernels.cpp
index 1fd3a14cf85..88f92ad4ba2 100644
--- a/omp/stop/residual_norm_kernels.cpp
+++ b/omp/stop/residual_norm_kernels.cpp
@@ -34,7 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <omp.h>
-
+#include <iostream>
 
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
@@ -66,6 +66,9 @@ void residual_norm(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for reduction(|| : local_one_changed)
     for (size_type i = 0; i < tau->get_size()[1]; ++i) {
         if (tau->at(i) < rel_residual_goal * orig_tau->at(i)) {
+            std::cout << "ORIG TAU: " << orig_tau->at(i)
+                      << ", TAU: " << tau->at(i)
+                      << ", TOLERANCE: " << rel_residual_goal << "\n";
             stop_status->get_data()[i].converge(stoppingId, setFinalized);
             local_one_changed = true;
         }
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index 439d8271a09..e145d16bfca 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -67,7 +67,7 @@ class Csr : public ::testing::Test {
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
     using ComplexMtx = gko::matrix::Csr<std::complex<double>>;
 
-    Csr() : mtx_size(532, 231), rand_engine(42) {}
+    Csr() : mtx_size(53, 23), rand_engine(42) {}
 
     void SetUp()
     {
diff --git a/omp/test/solver/gmres_kernels.cpp b/omp/test/solver/gmres_kernels.cpp
index aa845b21ff0..31d9a68f8bb 100644
--- a/omp/test/solver/gmres_kernels.cpp
+++ b/omp/test/solver/gmres_kernels.cpp
@@ -60,6 +60,7 @@ class Gmres : public ::testing::Test {
     using value_type = gko::default_precision;
     using index_type = gko::int32;
     using Mtx = gko::matrix::Dense<value_type>;
+    using Solver = gko::solver::Gmres<value_type>;
     using norm_type = gko::remove_complex<value_type>;
     using NormVector = gko::matrix::Dense<norm_type>;
     template <typename T>
@@ -71,6 +72,27 @@ class Gmres : public ::testing::Test {
     {
         ref = gko::ReferenceExecutor::create();
         omp = gko::OmpExecutor::create();
+
+        mtx = gen_mtx(123, 123);
+        d_mtx = Mtx::create(omp);
+        d_mtx->copy_from(mtx.get());
+        omp_gmres_factory =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(246u).on(omp),
+                    gko::stop::ResidualNormReduction<>::build()
+                        .with_reduction_factor(1e-15)
+                        .on(omp))
+                .on(omp);
+
+        ref_gmres_factory =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(246u).on(ref),
+                    gko::stop::ResidualNormReduction<>::build()
+                        .with_reduction_factor(1e-15)
+                        .on(ref))
+                .on(ref);
     }
 
     void TearDown()
@@ -89,9 +111,21 @@ class Gmres : public ::testing::Test {
             std::normal_distribution<ValueType>(-1.0, 1.0), rand_engine, ref);
     }
 
+    void make_diag_dominant(Mtx *mtx)
+    {
+        using std::abs;
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            auto sum = gko::zero<Mtx::value_type>();
+            for (int j = 0; j < mtx->get_size()[1]; ++j) {
+                sum += abs(mtx->at(i, j));
+            }
+            mtx->at(i, i) = sum;
+        }
+    }
+
     void initialize_data()
     {
-        int m = 597;
+        int m = 5;  // 97;
         int n = 43;
         x = gen_mtx(m, n);
         y = gen_mtx(gko::solver::default_krylov_dim, n);
@@ -154,6 +188,11 @@ class Gmres : public ::testing::Test {
 
     std::ranlux48 rand_engine;
 
+    std::unique_ptr<Mtx> mtx;
+    std::unique_ptr<Mtx> d_mtx;
+    std::unique_ptr<Solver::Factory> omp_gmres_factory;
+    std::unique_ptr<Solver::Factory> ref_gmres_factory;
+
     std::unique_ptr<Mtx> before_preconditioner;
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> y;
@@ -270,4 +309,25 @@ TEST_F(Gmres, OmpGmresStep2IsEquivalentToRef)
 }
 
 
+TEST_F(Gmres, GmresApplyOneRHSIsEquivalentToRef)
+{
+    int m = 123;
+    int n = 1;
+    auto ref_solver = ref_gmres_factory->generate(gko::share(mtx));
+    auto omp_solver = omp_gmres_factory->generate(gko::share(d_mtx));
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(omp);
+    auto d_x = Mtx::create(omp);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    omp_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+}
+
+
 }  // namespace
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index f1b97d2ae07..3916c0d40de 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -30,10 +30,12 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include <ginkgo/core/solver/bicgstab.hpp>
+#include <ginkgo/core/solver/gmres.hpp>
 #include <ginkgo/core/solver/idr.hpp>
 
 
-#include <iostream>
+#include <fstream>
 #include <random>
 
 
@@ -43,6 +45,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/mtx_io.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
@@ -60,6 +64,7 @@ class Idr : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
     using Solver = gko::solver::Idr<>;
+    using Csr = gko::matrix::Csr<>;
 
     Idr() : rand_engine(30) {}
 
@@ -69,16 +74,25 @@ class Idr : public ::testing::Test {
         omp = gko::OmpExecutor::create();
 
         mtx = gen_mtx(123, 123);
-        make_diag_dominant(mtx.get());
+
+        std::ofstream outfile;
+        outfile.open("test.mtx");
+        outfile << std::setprecision(20);
+
+        gko::write(outfile, mtx.get(), gko::layout_type::coordinate);
+
+        outfile.close();
+
         d_mtx = Mtx::create(omp);
+        // make_diag_dominant(mtx.get());
         d_mtx->copy_from(mtx.get());
         omp_idr_factory =
             Solver::build()
                 .with_deterministic(true)
                 .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(omp),
-                    gko::stop::ResidualNormReduction<>::build()
-                        .with_reduction_factor(1e-15)
+                    gko::stop::Iteration::build().with_max_iters(10u).on(omp),
+                    gko::stop::RelativeResidualNorm<>::build()
+                        .with_tolerance(1e-13)
                         .on(omp))
                 .on(omp);
 
@@ -86,9 +100,9 @@ class Idr : public ::testing::Test {
             Solver::build()
                 .with_deterministic(true)
                 .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNormReduction<>::build()
-                        .with_reduction_factor(1e-15)
+                    gko::stop::Iteration::build().with_max_iters(10u).on(ref),
+                    gko::stop::RelativeResidualNorm<>::build()
+                        .with_tolerance(1e-13)
                         .on(ref))
                 .on(ref);
     }
@@ -173,11 +187,11 @@ class Idr : public ::testing::Test {
             for (int j = 0; j < mtx->get_size()[1]; ++j) {
                 sum += abs(mtx->at(i, j));
             }
-            mtx->at(i, i) = sum;
+            mtx->at(i, i) = sum / 4;
         }
     }
 
-    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::OmpExecutor> omp;
 
     std::ranlux48 rand_engine;
@@ -219,7 +233,7 @@ class Idr : public ::testing::Test {
 };
 
 
-TEST_F(Idr, OmpIdrInitializeIsEquivalentToRef)
+/*TEST_F(Idr, IdrInitializeIsEquivalentToRef)
 {
     initialize_data();
 
@@ -230,10 +244,10 @@ TEST_F(Idr, OmpIdrInitializeIsEquivalentToRef)
 
     GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
     GKO_ASSERT_MTX_NEAR(p, d_p, 1e-14);
-}
+}*/
 
 
-TEST_F(Idr, OmpIdrStep1IsEquivalentToRef)
+TEST_F(Idr, IdrStep1IsEquivalentToRef)
 {
     initialize_data();
 
@@ -250,7 +264,7 @@ TEST_F(Idr, OmpIdrStep1IsEquivalentToRef)
 }
 
 
-TEST_F(Idr, OmpIdrStep2IsEquivalentToRef)
+TEST_F(Idr, IdrStep2IsEquivalentToRef)
 {
     initialize_data();
 
@@ -264,7 +278,7 @@ TEST_F(Idr, OmpIdrStep2IsEquivalentToRef)
 }
 
 
-TEST_F(Idr, OmpIdrStep3IsEquivalentToRef)
+TEST_F(Idr, IdrStep3IsEquivalentToRef)
 {
     initialize_data();
 
@@ -285,7 +299,7 @@ TEST_F(Idr, OmpIdrStep3IsEquivalentToRef)
 }
 
 
-TEST_F(Idr, OmpIdrComputeOmegaIsEquivalentToRef)
+TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
 {
     initialize_data();
 
@@ -301,12 +315,20 @@ TEST_F(Idr, OmpIdrComputeOmegaIsEquivalentToRef)
 }
 
 
-TEST_F(Idr, OmpIdrApplyOneRHSIsEquivalentToRef)
+TEST_F(Idr, IdrApplyOneRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 1;
-    auto ref_solver = ref_idr_factory->generate(mtx);
-    auto omp_solver = omp_idr_factory->generate(d_mtx);
+
+    auto bus = gko::read<Csr>(std::ifstream("1138_bus.mtx"), ref,
+                              std::make_shared<Csr::classical>());
+    auto d_bus = Csr::create(omp, std::make_shared<Csr::classical>());
+    d_bus->copy_from(bus.get());
+
+    m = bus->get_size()[0];
+
+    auto ref_solver = ref_idr_factory->generate(gko::share(bus));
+    auto omp_solver = omp_idr_factory->generate(gko::share(d_bus));
     auto b = gen_mtx(m, n);
     auto x = gen_mtx(m, n);
     auto d_b = Mtx::create(omp);
@@ -317,12 +339,66 @@ TEST_F(Idr, OmpIdrApplyOneRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     omp_solver->apply(d_b.get(), d_x.get());
 
+    /*auto res = Mtx::create(omp, gko::dim<2>{123,1});
+    auto one_op = gko::initialize<Mtx>({1.}, omp);
+    auto neg_one_op = gko::initialize<Mtx>({-1.}, omp);
+
+    res->copy_from(d_b.get());
+    d_mtx->apply(neg_one_op.get(), d_x.get(), one_op.get(), res.get());
+
+    auto d_resnorm = Mtx::create(omp, gko::dim<2>{1, 1});
+    res->compute_norm2(d_resnorm.get());
+
+    res->copy_from(b.get());
+    mtx->apply(neg_one_op.get(), x.get(), one_op.get(), res.get());
+
+    auto resnorm = Mtx::create(ref, gko::dim<2>{1, 1});
+    res->compute_norm2(resnorm.get());
+
+    std::cout << "INITIAL RESIDUAL NORM REF: " << resnorm->at(0,0) << ", INITIAL
+    RESIDUAL NORM OMP: " << d_resnorm->at(0, 0) << "\n";
+
+    b->compute_norm2(resnorm.get());
+    d_b->compute_norm2(d_resnorm.get());
+
+    std::cout << "B NORM REF: " << resnorm->at(0,0) << ", B NORM OMP: " <<
+    d_resnorm->at(0,0) << "\n";
+
+    ref_solver->apply(b.get(), x.get());
+    omp_solver->apply(d_b.get(), d_x.get());
+
+    res->copy_from(d_b.get());
+    d_mtx->apply(neg_one_op.get(), d_x.get(), one_op.get(), res.get());
+    res->compute_norm2(d_resnorm.get());
+
+    res->copy_from(b.get());
+    mtx->apply(neg_one_op.get(), x.get(), one_op.get(), res.get());
+    res->compute_norm2(resnorm.get());
+
+    std::cout << "RESIDUAL NORM REF: " << resnorm->at(0,0) << ", RESIDUAL NORM
+    OMP: " << d_resnorm->at(0, 0) << "\n";
+
+    std::ofstream outfile;
+    outfile.open("refresult.mtx");
+    outfile << std::setprecision(20);
+
+    gko::write(outfile, x.get(), gko::layout_type::coordinate);
+
+    outfile.close();
+
+    outfile.open("rhs.mtx");
+    outfile << std::setprecision(20);
+
+    gko::write(outfile, b.get(), gko::layout_type::coordinate);
+
+    outfile.close();*/
+
     GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
 }
 
 
-TEST_F(Idr, OmpIdrApplyMultipleRHSIsEquivalentToRef)
+/*TEST_F(Idr, IdrApplyMultipleRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 16;
@@ -340,7 +416,7 @@ TEST_F(Idr, OmpIdrApplyMultipleRHSIsEquivalentToRef)
 
     GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-}
+}*/
 
 
 }  // namespace
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index b06c204d6da..a4fa5d453da 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <time.h>
 #include <algorithm>
+#include <iostream>
 #include <random>
 
 
@@ -47,7 +48,7 @@ namespace gko {
 namespace kernels {
 namespace reference {
 /**
- * @brief The BiCGSTAB solver namespace.
+ * @brief The IDR solver namespace.
  *
  * @ingroup idr
  */
@@ -133,20 +134,36 @@ get_rand_value(Distribution &&dist, Generator &&gen)
 
 template <typename ValueType>
 void initialize(std::shared_ptr<const ReferenceExecutor> exec,
-                matrix::Dense<ValueType> *m,
-                matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
-                Array<stopping_status> *stop_status)
+                const matrix::Dense<ValueType> *dense_b,
+                const matrix::Dense<ValueType> *dense_x,
+                matrix::Dense<to_complex<ValueType>> *complex_b,
+                matrix::Dense<to_complex<ValueType>> *complex_x,
+                matrix::Dense<to_complex<ValueType>> *m,
+                matrix::Dense<to_complex<ValueType>> *subspace_vectors,
+                bool deterministic, Array<stopping_status> *stop_status)
 {
-    // Initialize M
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
+
+    // Initialize complex b and x
+    for (size_type row = 0; row < dense_b->get_size()[0]; row++) {
+        for (size_type rhs = 0; rhs < nrhs; rhs++) {
+            complex_x->at(row, rhs) =
+                to_complex<ValueType>{dense_x->at(row, rhs)};
+            complex_b->at(row, rhs) =
+                to_complex<ValueType>{dense_b->at(row, rhs)};
+        }
+    }
+
+    // Initialize M
     for (size_type i = 0; i < nrhs; i++) {
         stop_status->get_data()[i].reset();
     }
 
     for (size_type row = 0; row < m->get_size()[0]; row++) {
         for (size_type col = 0; col < m->get_size()[1]; col++) {
-            m->at(row, col) =
-                (row == col / nrhs) ? one<ValueType>() : zero<ValueType>();
+            m->at(row, col) = (row == col / nrhs)
+                                  ? one<to_complex<ValueType>>()
+                                  : zero<to_complex<ValueType>>();
         }
     }
 
@@ -163,7 +180,7 @@ void initialize(std::shared_ptr<const ReferenceExecutor> exec,
         }
 
         for (size_type i = 0; i < row; i++) {
-            auto dot = zero<ValueType>();
+            auto dot = zero<to_complex<ValueType>>();
             for (size_type j = 0; j < num_cols; j++) {
                 dot += subspace_vectors->at(row, j) *
                        conj(subspace_vectors->at(i, j));
@@ -319,6 +336,25 @@ void compute_omega(
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
 
+template <typename ValueType>
+void finalize(std::shared_ptr<const ReferenceExecutor> exec,
+              const matrix::Dense<to_complex<ValueType>> *complex_x,
+              matrix::Dense<ValueType> *dense_x)
+{
+    if (is_complex<ValueType>()) {
+        dense_x->copy_from(complex_x);
+    } else {
+        for (size_type row = 0; row < dense_x->get_size()[0]; row++) {
+            for (size_type rhs = 0; rhs < dense_x->get_size()[1]; rhs++) {
+                dense_x->at(row, rhs) = real(complex_x->at(row, rhs));
+            }
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_FINALIZE_KERNEL);
+
+
 }  // namespace idr
 }  // namespace reference
 }  // namespace kernels
diff --git a/reference/stop/residual_norm_kernels.cpp b/reference/stop/residual_norm_kernels.cpp
index fb968e0eae3..a3446414d25 100644
--- a/reference/stop/residual_norm_kernels.cpp
+++ b/reference/stop/residual_norm_kernels.cpp
@@ -34,7 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
-
+#include <iostream>
 
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
@@ -68,6 +68,9 @@ void residual_norm(std::shared_ptr<const ReferenceExecutor> exec,
     *one_changed = false;
     for (size_type i = 0; i < tau->get_size()[1]; ++i) {
         if (tau->at(i) < rel_residual_goal * orig_tau->at(i)) {
+            std::cout << "ORIG TAU: " << orig_tau->at(i)
+                      << ", TAU: " << tau->at(i)
+                      << ", TOLERANCE: " << rel_residual_goal << "\n";
             stop_status->get_data()[i].converge(stoppingId, setFinalized);
             *one_changed = true;
         }
diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp
index b21527a4622..566721d2aba 100644
--- a/reference/test/solver/idr_kernels.cpp
+++ b/reference/test/solver/idr_kernels.cpp
@@ -33,9 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/idr.hpp>
 
 
+#include <iostream>
+
+
 #include <gtest/gtest.h>
 
-#include <iostream>
 
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
@@ -144,7 +146,6 @@ TYPED_TEST(Idr, SolvesDenseSystemUsingAdvancedApply)
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
-
     GKO_ASSERT_MTX_NEAR(x, l({-8.5, -3.0, 6.0}), half_tol);
 }
 
@@ -165,7 +166,6 @@ TYPED_TEST(Idr, SolvesMultipleDenseSystemsUsingAdvancedApply)
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
-
     GKO_ASSERT_MTX_NEAR(x, l({{-8.5, 1.0}, {-3.0, 2.0}, {6.0, -5.0}}),
                         half_tol);
 }
@@ -200,7 +200,6 @@ TYPED_TEST(Idr, SolvesBigDenseSystemForDivergenceCheck1)
     auto resnorm = gko::matrix::Dense<gko::remove_complex<value_type>>::create(
         this->exec, gko::dim<2>{1, 1});
     locmtx->apply(neg_one_op.get(), x.get(), one_op.get(), b.get());
-    b->compute_norm2(resnorm.get());
 
     GKO_ASSERT_MTX_NEAR(
         x,

From d3df5daf7f216b44990a232d75f0347f4a2604c1 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 29 Oct 2020 12:22:15 +0100
Subject: [PATCH 0154/1008] Add option to use complex subspace for real
 problems

---
 .gitlab-ci.yml                             |  86 ++++++------
 core/device_hooks/common_kernels.inc.cpp   |   5 -
 core/solver/idr.cpp                        | 115 ++++++++--------
 core/solver/idr_kernels.hpp                |  42 +++---
 core/stop/residual_norm.cpp                |  11 +-
 cuda/test/solver/idr_kernels.cpp           |  97 +++++++++++++-
 hip/test/solver/idr_kernels.cpp            |  97 +++++++++++++-
 include/ginkgo/core/solver/idr.hpp         |  16 +++
 include/ginkgo/core/stop/residual_norm.hpp |  21 ++-
 include/ginkgo/ginkgo.hpp                  |   1 -
 omp/solver/idr_kernels.cpp                 |  50 +------
 omp/stop/residual_norm_kernels.cpp         |   4 -
 omp/test/matrix/csr_kernels.cpp            |   2 +-
 omp/test/solver/idr_kernels.cpp            | 148 ++++++++++++---------
 reference/solver/idr_kernels.cpp           |  48 +------
 reference/stop/residual_norm_kernels.cpp   |   4 -
 reference/test/solver/idr_kernels.cpp      |  64 +++++++++
 17 files changed, 500 insertions(+), 311 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6bcf4e5d278..51ab318a3c9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -127,7 +127,7 @@ stages:
 
 sync:
   stage: sync
-  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
   variables:
     GIT_STRATEGY: none
     PRIVATE_REPO: git@gitlab.com:ginkgo-project/ginkgo.git
@@ -153,7 +153,7 @@ sync:
 # Job with example runs.
 build/cuda90/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda90-gnu5-llvm39
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda90-gnu5-llvm39
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -171,7 +171,7 @@ build/cuda90/gcc/all/debug/shared:
 
 build/cuda90/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda90-gnu5-llvm39
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda90-gnu5-llvm39
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -192,7 +192,7 @@ build/cuda90/clang/all/release/static:
 # cuda 9.1 and friends
 build/cuda91/gcc/all/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda91-gnu6-llvm40
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda91-gnu6-llvm40
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -210,7 +210,7 @@ build/cuda91/gcc/all/debug/static:
 
 build/cuda91/clang/all/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda91-gnu6-llvm40
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda91-gnu6-llvm40
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -231,7 +231,7 @@ build/cuda91/clang/all/release/shared:
 # cuda 9.2 and friends
 build/cuda92/gcc/all/release/debug:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda92-gnu7-llvm50-intel2017
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -249,7 +249,7 @@ build/cuda92/gcc/all/release/debug:
 
 build/cuda92/intel/cuda/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda92-gnu7-llvm50-intel2017
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -270,7 +270,7 @@ build/cuda92/intel/cuda/release/static:
 # cuda 10.0 and friends
 build/cuda100/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda100-gnu7-llvm60-intel2018
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -288,7 +288,7 @@ build/cuda100/gcc/all/debug/shared:
 
 build/cuda100/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda100-gnu7-llvm60-intel2018
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -309,7 +309,7 @@ build/cuda100/clang/all/release/static:
 
 build/cuda100/intel/cuda/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda100-gnu7-llvm60-intel2018
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -329,7 +329,7 @@ build/cuda100/intel/cuda/release/shared:
 # cuda 10.1 and friends
 build/cuda101/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -347,7 +347,7 @@ build/cuda101/gcc/all/debug/shared:
 
 build/cuda101/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -369,7 +369,7 @@ build/cuda101/clang/all/release/static:
 # clang-cuda with cuda 10.1 and friends
 build/clang-cuda101/gcc/all/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm10-intel2019
   variables:
     <<: *default_variables
     CUDA_COMPILER: "clang++"
@@ -388,7 +388,7 @@ build/clang-cuda101/gcc/all/release/shared:
 
 build/clang-cuda101/clang/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm10-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -410,7 +410,7 @@ build/clang-cuda101/clang/cuda/debug/static:
 # cuda 10.2 and friends
 build/cuda102/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda102-gnu8-llvm8-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -428,7 +428,7 @@ build/cuda102/gcc/all/debug/shared:
 
 build/cuda102/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda102-gnu8-llvm8-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -449,7 +449,7 @@ build/cuda102/clang/all/release/static:
 
 build/cuda102/intel/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda102-gnu8-llvm8-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -470,7 +470,7 @@ build/cuda102/intel/cuda/debug/static:
 # cuda 11.0 and friends
 build/cuda110/gcc/cuda/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda110-gnu9-llvm9-intel2020
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -487,7 +487,7 @@ build/cuda110/gcc/cuda/debug/shared:
 
 build/cuda110/clang/cuda/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda110-gnu9-llvm9-intel2020
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -507,7 +507,7 @@ build/cuda110/clang/cuda/release/static:
 
 build/cuda110/intel/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda110-gnu9-llvm9-intel2020
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -528,7 +528,7 @@ build/cuda110/intel/cuda/debug/static:
 # HIP AMD
 build/amd/gcc/hip/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-amd-gnu8-llvm7
+  image: localhost:5000/add_hiprand_fec6ada9_gko-amd-gnu8-llvm7
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -545,7 +545,7 @@ build/amd/gcc/hip/debug/shared:
 
 build/amd/clang/hip/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-amd-gnu8-llvm7
+  image: localhost:5000/add_hiprand_fec6ada9_gko-amd-gnu8-llvm7
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -565,7 +565,7 @@ build/amd/clang/hip/release/static:
 # no cuda but latest gcc and clang
 build/nocuda/gcc/core/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_REFERENCE: "OFF"
@@ -580,7 +580,7 @@ build/nocuda/gcc/core/debug/static:
 
 build/nocuda/clang/core/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -596,7 +596,7 @@ build/nocuda/clang/core/release/shared:
 
 build/nocuda/intel/core/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -613,7 +613,7 @@ build/nocuda/intel/core/debug/shared:
 
 build/nocuda/gcc/omp/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -627,7 +627,7 @@ build/nocuda/gcc/omp/release/shared:
 
 build/nocuda/clang/omp/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -644,7 +644,7 @@ build/nocuda/clang/omp/debug/static:
 
 build/nocuda/intel/omp/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -683,7 +683,7 @@ build/dpcpp/cpu/release/static:
 warnings:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -704,7 +704,7 @@ warnings:
 no-circular-deps:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -746,7 +746,7 @@ subdir-build:
 clang-tidy:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -766,7 +766,7 @@ clang-tidy:
 iwyu:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -787,7 +787,7 @@ iwyu:
 # For short living branches or PRs, try to detect an open PR
 sonarqube_cov_:
   stage: code_quality
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
   before_script: *default_before_script
   script:
     - PR_ID=$(curl "https://api.github.com/search/issues?q=sha:${CI_COMMIT_SHA}"
@@ -828,7 +828,7 @@ sonarqube_cov_:
 # (the one that was merged).
 sonarqube_cov:
   stage: code_quality
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
   before_script: *default_before_script
   script:
     - ctest -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=COVERAGE
@@ -855,7 +855,7 @@ sonarqube_cov:
 # Deploy documentation to github-pages
 gh-pages:
   stage: deploy
-  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     PUBLIC_REPO: git@github.com:ginkgo-project/ginkgo.git
@@ -902,7 +902,7 @@ gh-pages:
 
 threadsanitizer:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - LD_PRELOAD=/usr/local/lib/libomp.so
@@ -926,7 +926,7 @@ threadsanitizer:
 
 leaksanitizer:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=LSAN
@@ -946,7 +946,7 @@ leaksanitizer:
 
 addresssanitizer:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=ASAN
@@ -966,7 +966,7 @@ addresssanitizer:
 
 undefinedsanitizer:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     # the Gold linker is required because of a linker flag issues given by UBsan
@@ -988,7 +988,7 @@ undefinedsanitizer:
 
 cudamemcheck:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_cdb07838_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=RelWithDebInfo
@@ -1027,7 +1027,7 @@ cudamemcheck:
 
 fineci-benchmark-build:
   stage: benchmark-build
-  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BENCHMARK_SERVER: FINECI
@@ -1108,7 +1108,7 @@ fineci-benchmark-build:
 
 fineci-benchmark-cuda:
   stage: benchmark-cuda
-  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BENCHMARK_SERVER: FINECI
@@ -1146,7 +1146,7 @@ fineci-benchmark-cuda:
 
 new-issue-on-failure:
   stage: on-failure
-  image: localhost:5000/add_hiprand_cdb07838_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
   script: curl --request POST "https://gitlab.com/api/v4/projects/${PROJECT_ID}/issues?private_token=${BOT_ACCESS_TOKEN}&title=Error%20in%20${CI_PROJECT_NAME}%20with%20pipeline%20${CI_PIPELINE_ID}%20for%20commit%20${CI_COMMIT_SHA}&labels&description=${CI_PIPELINE_URL}"
   when: on_failure
   only:
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index d5299e96e75..da34e4bd11a 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -493,11 +493,6 @@ GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
-template <typename ValueType>
-GKO_DECLARE_IDR_FINALIZE_KERNEL(ValueType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_FINALIZE_KERNEL);
-
 
 }  // namespace idr
 
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index baaf6586e08..f7300f00ea6 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/utils.hpp>
 
 
+#include <variant>
 #include "core/components/fill_array.hpp"
 #include "core/solver/idr_kernels.hpp"
 
@@ -53,7 +54,6 @@ GKO_REGISTER_OPERATION(step_1, idr::step_1);
 GKO_REGISTER_OPERATION(step_2, idr::step_2);
 GKO_REGISTER_OPERATION(step_3, idr::step_3);
 GKO_REGISTER_OPERATION(compute_omega, idr::compute_omega);
-GKO_REGISTER_OPERATION(finalize, idr::finalize);
 GKO_REGISTER_OPERATION(fill_array, components::fill_array);
 
 
@@ -85,33 +85,14 @@ std::unique_ptr<LinOp> Idr<ValueType>::conj_transpose() const
             as<Transposable>(this->get_system_matrix())->conj_transpose()));
 }
 
-template <typename ValueType>
-std::unique_ptr<matrix::Dense<ValueType>> interpret_as_value_type(
-    matrix::Dense<to_complex<ValueType>> *source)
-{
-    auto exec = source->get_executor();
-    auto rows = source->get_size()[0];
-    auto cols = is_complex<ValueType>() ? source->get_size()[1]
-                                        : 2 * source->get_size()[1];
-    auto stride = is_complex<ValueType>() ? source->get_stride()
-                                          : 2 * source->get_stride();
-    return matrix::Dense<ValueType>::create(
-        exec, gko::dim<2>{rows, cols},
-        gko::Array<ValueType>::view(
-            exec, rows * stride,
-            reinterpret_cast<ValueType *>(source->get_values())),
-        stride);
-}
-
 
 template <typename ValueType>
-void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
+template <typename SubspaceType>
+void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
 {
     using std::swap;
-    using ComplexType = to_complex<ValueType>;
-    using Vector = matrix::Dense<ComplexType>;
+    using Vector = matrix::Dense<SubspaceType>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
-    bool complex = true;
 
     auto exec = this->get_executor();
 
@@ -119,25 +100,20 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         initialize<matrix::Dense<ValueType>>({one<ValueType>()}, exec);
     auto neg_one_op =
         initialize<matrix::Dense<ValueType>>({-one<ValueType>()}, exec);
-    auto complex_neg_one_op =
-        initialize<Vector>({-one<to_complex<ValueType>>()}, exec);
-
-    auto dense_b = as<matrix::Dense<ValueType>>(b);
-    auto dense_x = as<matrix::Dense<ValueType>>(x);
-
-    auto complex_b = Vector::create(exec, dense_b->get_size());
-    auto complex_x = Vector::create(exec, dense_x->get_size());
+    auto subspace_neg_one_op = initialize<Vector>({-one<SubspaceType>()}, exec);
 
+    auto dense_b = as<Vector>(b);
+    auto dense_x = as<Vector>(x);
 
     constexpr uint8 RelativeStoppingId{1};
 
     const auto problem_size = system_matrix_->get_size()[0];
-    const auto nrhs = complex_b->get_size()[1];
+    const auto nrhs = dense_b->get_size()[1];
 
-    auto residual = Vector::create_with_config_of(complex_b.get());
-    auto v = Vector::create_with_config_of(complex_b.get());
-    auto t = Vector::create_with_config_of(complex_b.get());
-    auto helper = Vector::create_with_config_of(complex_b.get());
+    auto residual = Vector::create_with_config_of(dense_b);
+    auto v = Vector::create_with_config_of(dense_b);
+    auto t = Vector::create_with_config_of(dense_b);
+    auto helper = Vector::create_with_config_of(dense_b);
 
     auto m =
         Vector::create(exec, gko::dim<2>{subspace_dim_, subspace_dim_ * nrhs});
@@ -168,44 +144,44 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     // Initialization
     // m = identity
-    exec->run(idr::make_initialize(
-        dense_b, dense_x, complex_b.get(), complex_x.get(), m.get(),
-        subspace_vectors.get(), deterministic_, &stop_status));
+    exec->run(idr::make_initialize(m.get(), subspace_vectors.get(),
+                                   deterministic_, &stop_status));
 
     // omega = 1
     exec->run(
-        idr::make_fill_array(omega->get_values(), nrhs, one<ComplexType>()));
+        idr::make_fill_array(omega->get_values(), nrhs, one<SubspaceType>()));
 
     // residual = b - Ax
-    residual->copy_from(complex_b.get());
-    system_matrix_->apply(neg_one_op.get(), complex_x.get(), one_op.get(),
+    residual->copy_from(dense_b);
+    system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(),
                           residual.get());
 
     // g = u = 0
     exec->run(idr::make_fill_array(
-        g->get_values(), problem_size * g->get_stride(), zero<ComplexType>()));
+        g->get_values(), problem_size * g->get_stride(), zero<SubspaceType>()));
     exec->run(idr::make_fill_array(
-        u->get_values(), problem_size * u->get_stride(), zero<ComplexType>()));
+        u->get_values(), problem_size * u->get_stride(), zero<SubspaceType>()));
 
 
-    /*auto stop_criterion = stop_criterion_factory_->generate(
-        system_matrix_, std::shared_ptr<const LinOp>(complex_b.get(), [](const
-       LinOp *) {}), complex_x.get(), residual.get());*/
+    auto stop_criterion = stop_criterion_factory_->generate(
+        system_matrix_,
+        std::shared_ptr<const LinOp>(dense_b, [](const LinOp *) {}), dense_x,
+        residual.get());
 
     int total_iter = -1;
 
-    while (total_iter < 3) {
+    while (true) {
         ++total_iter;
-        /*this->template log<log::Logger::iteration_complete>(
-            this, total_iter, residual.get(), complex_x.get());
+        this->template log<log::Logger::iteration_complete>(
+            this, total_iter, residual.get(), dense_x);
 
         if (stop_criterion->update()
                 .num_iterations(total_iter)
                 .residual(residual.get())
-                .solution(complex_x.get())
+                .solution(dense_x)
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
             break;
-        }*/
+        }
 
         subspace_vectors->apply(residual.get(), f.get());
         // f = P^H * residual
@@ -217,9 +193,7 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             // c = M \ f = (c_1, ..., c_s)^T
             // v = residual - c_k * g_k - ... - c_s * g_s
 
-            get_preconditioner()->apply(
-                interpret_as_value_type<ValueType>(v.get()).get(),
-                interpret_as_value_type<ValueType>(helper.get()).get());
+            get_preconditioner()->apply(v.get(), helper.get());
 
             exec->run(idr::make_step_2(k, omega.get(), helper.get(), c.get(),
                                        u.get(), &stop_status));
@@ -233,8 +207,8 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
             exec->run(idr::make_step_3(k, subspace_vectors.get(), g.get(),
                                        helper.get(), u.get(), m.get(), f.get(),
-                                       alpha.get(), residual.get(),
-                                       complex_x.get(), &stop_status));
+                                       alpha.get(), residual.get(), dense_x,
+                                       &stop_status));
             // for i = 1 to k - 1 do
             //     alpha = p^H_i * g_k / m_i,i
             //     g_k -= alpha * g_i
@@ -249,9 +223,7 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             // f = (0,...,0,f_k+1 - beta * m_k+1,k,...,f_s - beta * m_s,k)
         }
 
-        get_preconditioner()->apply(
-            interpret_as_value_type<ValueType>(residual.get()).get(),
-            interpret_as_value_type<ValueType>(helper.get()).get());
+        get_preconditioner()->apply(residual.get(), helper.get());
         system_matrix_->apply(helper.get(), t.get());
 
         t->compute_dot(residual.get(), omega.get());
@@ -262,9 +234,9 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         exec->run(idr::make_compute_omega(
             kappa_, tht.get(), residual_norm.get(), omega.get(), &stop_status));
 
-        t->scale(complex_neg_one_op.get());
+        t->scale(subspace_neg_one_op.get());
         residual->add_scaled(omega.get(), t.get());
-        complex_x.get()->add_scaled(omega.get(), helper.get());
+        dense_x->add_scaled(omega.get(), helper.get());
 
         // omega = (t^H * residual) / (t^H * t)
         // rho = (t^H * residual) / (norm(t) * norm(residual))
@@ -274,7 +246,24 @@ void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         // residual -= omega * t
         // dense_x += omega * v
     }
-    exec->run(idr::make_finalize(complex_x.get(), dense_x));
+}
+
+
+template <typename ValueType>
+void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
+{
+    // If ValueType is complex, the subspace matrix P will be complex anyway.
+    if (!is_complex<ValueType>() && complex_subspace_) {
+        auto dense_b = as<matrix::Dense<ValueType>>(b);
+        auto dense_x = as<matrix::Dense<ValueType>>(x);
+        auto complex_b = dense_b->make_complex();
+        auto complex_x = dense_x->make_complex();
+        this->iterate<to_complex<ValueType>>(complex_b.get(), complex_x.get());
+        complex_x->get_real(
+            dynamic_cast<matrix::Dense<remove_complex<ValueType>> *>(dense_x));
+    } else {
+        this->iterate<ValueType>(b, x);
+    }
 }
 
 
diff --git a/core/solver/idr_kernels.hpp b/core/solver/idr_kernels.hpp
index 1dcc7c7ea10..29080c36fe6 100644
--- a/core/solver/idr_kernels.hpp
+++ b/core/solver/idr_kernels.hpp
@@ -46,14 +46,10 @@ namespace kernels {
 namespace idr {
 
 
-#define GKO_DECLARE_IDR_INITIALIZE_KERNEL(_type)                        \
-    void initialize(std::shared_ptr<const DefaultExecutor> exec,        \
-                    const matrix::Dense<_type> *dense_b,                \
-                    const matrix::Dense<_type> *dense_x,                \
-                    matrix::Dense<to_complex<_type>> *complex_b,        \
-                    matrix::Dense<to_complex<_type>> *complex_x,        \
-                    matrix::Dense<to_complex<_type>> *m,                \
-                    matrix::Dense<to_complex<_type>> *subspace_vectors, \
+#define GKO_DECLARE_IDR_INITIALIZE_KERNEL(_type)                 \
+    void initialize(std::shared_ptr<const DefaultExecutor> exec, \
+                    matrix::Dense<_type> *m,                     \
+                    matrix::Dense<_type> *subspace_vectors,      \
                     bool deterministic, Array<stopping_status> *stop_status)
 
 
@@ -93,25 +89,17 @@ namespace idr {
         const Array<stopping_status> *stop_status)
 
 
-#define GKO_DECLARE_IDR_FINALIZE_KERNEL(_type)                       \
-    void finalize(std::shared_ptr<const DefaultExecutor> exec,       \
-                  const matrix::Dense<to_complex<_type>> *complex_x, \
-                  matrix::Dense<_type> *dense_x)
-
-
-#define GKO_DECLARE_ALL_AS_TEMPLATES                 \
-    template <typename ValueType>                    \
-    GKO_DECLARE_IDR_INITIALIZE_KERNEL(ValueType);    \
-    template <typename ValueType>                    \
-    GKO_DECLARE_IDR_STEP_1_KERNEL(ValueType);        \
-    template <typename ValueType>                    \
-    GKO_DECLARE_IDR_STEP_2_KERNEL(ValueType);        \
-    template <typename ValueType>                    \
-    GKO_DECLARE_IDR_STEP_3_KERNEL(ValueType);        \
-    template <typename ValueType>                    \
-    GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL(ValueType); \
-    template <typename ValueType>                    \
-    GKO_DECLARE_IDR_FINALIZE_KERNEL(ValueType)
+#define GKO_DECLARE_ALL_AS_TEMPLATES              \
+    template <typename ValueType>                 \
+    GKO_DECLARE_IDR_INITIALIZE_KERNEL(ValueType); \
+    template <typename ValueType>                 \
+    GKO_DECLARE_IDR_STEP_1_KERNEL(ValueType);     \
+    template <typename ValueType>                 \
+    GKO_DECLARE_IDR_STEP_2_KERNEL(ValueType);     \
+    template <typename ValueType>                 \
+    GKO_DECLARE_IDR_STEP_3_KERNEL(ValueType);     \
+    template <typename ValueType>                 \
+    GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL(ValueType)
 
 
 }  // namespace idr
diff --git a/core/stop/residual_norm.cpp b/core/stop/residual_norm.cpp
index 5c928bbf48d..877a2eb238c 100644
--- a/core/stop/residual_norm.cpp
+++ b/core/stop/residual_norm.cpp
@@ -59,8 +59,15 @@ bool ResidualNorm<ValueType>::check_impl(uint8 stoppingId, bool setFinalized,
     if (updater.residual_norm_ != nullptr) {
         dense_tau = as<NormVector>(updater.residual_norm_);
     } else if (updater.residual_ != nullptr) {
-        auto *dense_r = as<Vector>(updater.residual_);
-        dense_r->compute_norm2(u_dense_tau_.get());
+        if (dynamic_cast<const matrix::Dense<to_complex<ValueType>> *>(
+                updater.residual_)) {
+            auto *dense_r =
+                as<matrix::Dense<to_complex<ValueType>>>(updater.residual_);
+            dense_r->compute_norm2(u_dense_tau_.get());
+        } else {
+            auto *dense_r = as<Vector>(updater.residual_);
+            dense_r->compute_norm2(u_dense_tau_.get());
+        }
         dense_tau = u_dense_tau_.get();
     } else {
         GKO_NOT_SUPPORTED(nullptr);
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index e1457ca55a1..31019fc0d13 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -301,10 +301,55 @@ TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
 }
 
 
-TEST_F(Idr, IdrApplyOneRHSIsEquivalentToRef)
+TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 1;
+
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto cuda_solver = cuda_idr_factory->generate(d_mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(cuda);
+    auto d_x = Mtx::create(cuda);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    cuda_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+}
+
+
+TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
+{
+    int m = 123;
+    int n = 1;
+
+    cuda_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(cuda),
+                gko::stop::RelativeResidualNorm<>::build()
+                    .with_tolerance(1e-13)
+                    .on(cuda))
+            .on(cuda);
+
+    ref_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(ref),
+                gko::stop::RelativeResidualNorm<>::build()
+                    .with_tolerance(1e-13)
+                    .on(ref))
+            .on(ref);
+
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto cuda_solver = cuda_idr_factory->generate(d_mtx);
     auto b = gen_mtx(m, n);
@@ -322,7 +367,7 @@ TEST_F(Idr, IdrApplyOneRHSIsEquivalentToRef)
 }
 
 
-TEST_F(Idr, IdrApplyMultipleRHSIsEquivalentToRef)
+TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 16;
@@ -338,8 +383,52 @@ TEST_F(Idr, IdrApplyMultipleRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     cuda_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-12);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+}
+
+
+TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
+{
+    int m = 123;
+    int n = 16;
+
+    cuda_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(cuda),
+                gko::stop::RelativeResidualNorm<>::build()
+                    .with_tolerance(1e-13)
+                    .on(cuda))
+            .on(cuda);
+
+    ref_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(ref),
+                gko::stop::RelativeResidualNorm<>::build()
+                    .with_tolerance(1e-13)
+                    .on(ref))
+            .on(ref);
+
+    auto cuda_solver = cuda_idr_factory->generate(d_mtx);
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(cuda);
+    auto d_x = Mtx::create(cuda);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    cuda_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
 }
 
 
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index c5ac1d00894..f8e84cc448f 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -301,10 +301,55 @@ TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
 }
 
 
-TEST_F(Idr, IdrApplyOneRHSIsEquivalentToRef)
+TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 1;
+
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto hip_solver = hip_idr_factory->generate(d_mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(hip);
+    auto d_x = Mtx::create(hip);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    hip_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+}
+
+
+TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
+{
+    int m = 123;
+    int n = 1;
+
+    hip_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(hip),
+                gko::stop::RelativeResidualNorm<>::build()
+                    .with_tolerance(1e-13)
+                    .on(hip))
+            .on(hip);
+
+    ref_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(ref),
+                gko::stop::RelativeResidualNorm<>::build()
+                    .with_tolerance(1e-13)
+                    .on(ref))
+            .on(ref);
+
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto hip_solver = hip_idr_factory->generate(d_mtx);
     auto b = gen_mtx(m, n);
@@ -322,7 +367,7 @@ TEST_F(Idr, IdrApplyOneRHSIsEquivalentToRef)
 }
 
 
-TEST_F(Idr, IdrApplyMultipleRHSIsEquivalentToRef)
+TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 16;
@@ -338,8 +383,52 @@ TEST_F(Idr, IdrApplyMultipleRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     hip_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-12);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+}
+
+
+TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
+{
+    int m = 123;
+    int n = 16;
+
+    hip_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(hip),
+                gko::stop::RelativeResidualNorm<>::build()
+                    .with_tolerance(1e-13)
+                    .on(hip))
+            .on(hip);
+
+    ref_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(ref),
+                gko::stop::RelativeResidualNorm<>::build()
+                    .with_tolerance(1e-13)
+                    .on(ref))
+            .on(ref);
+
+    auto hip_solver = hip_idr_factory->generate(d_mtx);
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(hip);
+    auto d_x = Mtx::create(hip);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    hip_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
 }
 
 
diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index 46703a365c2..6ed8ac964a9 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -34,7 +34,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_SOLVER_IDR_HPP_
 
 
+#include <iostream>
 #include <random>
+#include <typeinfo>
 #include <vector>
 
 
@@ -176,6 +178,15 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
          * The default behaviour is to choose the subspace vectors randomly.
          */
         bool GKO_FACTORY_PARAMETER_SCALAR(deterministic, false);
+
+        /**
+         * If set to true, IDR will use a complex subspace S also for real
+         * problems, allowing for faster convergence and better results by
+         * acknowledging the influence of complex eigenvectors.
+         *
+         * The default is false.
+         */
+        bool GKO_FACTORY_PARAMETER_SCALAR(complex_subspace, false);
     };
     GKO_ENABLE_LIN_OP_FACTORY(Idr, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);
@@ -186,6 +197,9 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override;
 
+    template <typename SubspaceType>
+    void iterate(const LinOp *b, LinOp *x) const;
+
     explicit Idr(std::shared_ptr<const Executor> exec)
         : EnableLinOp<Idr>(std::move(exec))
     {}
@@ -213,6 +227,7 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
         subspace_dim_ = parameters_.subspace_dim;
         kappa_ = parameters_.kappa;
         deterministic_ = parameters_.deterministic;
+        complex_subspace_ = parameters_.complex_subspace;
     }
 
 private:
@@ -221,6 +236,7 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
     size_type subspace_dim_;
     remove_complex<ValueType> kappa_;
     bool deterministic_;
+    bool complex_subspace_;
 };
 
 
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index f0c4a94b377..20d4dffd38d 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -143,12 +143,19 @@ class ResidualNormReduction : public ResidualNorm<ValueType> {
 
         auto exec = factory->get_executor();
 
-        auto dense_r = as<Vector>(args.initial_residual);
         this->starting_tau_ = NormVector::create(
             exec, dim<2>{1, args.initial_residual->get_size()[1]});
         this->u_dense_tau_ =
             NormVector::create_with_config_of(this->starting_tau_.get());
-        dense_r->compute_norm2(this->starting_tau_.get());
+        if (dynamic_cast<const matrix::Dense<to_complex<ValueType>> *>(
+                args.initial_residual)) {
+            auto dense_r =
+                as<matrix::Dense<to_complex<ValueType>>>(args.initial_residual);
+            dense_r->compute_norm2(this->starting_tau_.get());
+        } else {
+            auto dense_r = as<Vector>(args.initial_residual);
+            dense_r->compute_norm2(this->starting_tau_.get());
+        }
     }
 };
 
@@ -203,12 +210,18 @@ class RelativeResidualNorm : public ResidualNorm<ValueType> {
 
         auto exec = factory->get_executor();
 
-        auto dense_rhs = as<Vector>(args.b);
         this->starting_tau_ =
             NormVector::create(exec, dim<2>{1, args.b->get_size()[1]});
         this->u_dense_tau_ =
             NormVector::create_with_config_of(this->starting_tau_.get());
-        dense_rhs->compute_norm2(this->starting_tau_.get());
+        if (dynamic_cast<const matrix::Dense<to_complex<ValueType>> *>(
+                args.b.get())) {
+            auto dense_rhs = as<matrix::Dense<to_complex<ValueType>>>(args.b);
+            dense_rhs->compute_norm2(this->starting_tau_.get());
+        } else {
+            auto dense_rhs = as<Vector>(args.b);
+            dense_rhs->compute_norm2(this->starting_tau_.get());
+        }
     }
 };
 
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 1052dcdf50b..40f415b2156 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -55,7 +55,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/polymorphic_object.hpp>
 #include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/base/range_accessors.hpp>
-#include <ginkgo/core/base/readable_data.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/base/utils.hpp>
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index f1d9aba566b..3fb5a64cd2d 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -140,28 +140,13 @@ get_rand_value(Distribution &&dist, Generator &&gen)
 
 template <typename ValueType>
 void initialize(std::shared_ptr<const OmpExecutor> exec,
-                const matrix::Dense<ValueType> *dense_b,
-                const matrix::Dense<ValueType> *dense_x,
-                matrix::Dense<to_complex<ValueType>> *complex_b,
-                matrix::Dense<to_complex<ValueType>> *complex_x,
-                matrix::Dense<to_complex<ValueType>> *m,
-                matrix::Dense<to_complex<ValueType>> *subspace_vectors,
-                bool deterministic, Array<stopping_status> *stop_status)
+                matrix::Dense<ValueType> *m,
+                matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
+                Array<stopping_status> *stop_status)
 {
-#pragma omp declare reduction(add:to_complex<ValueType> : omp_out = omp_out + omp_in)
+#pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
 
-    // Initialize complex b and x
-#pragma omp parallel for
-    for (size_type row = 0; row < dense_b->get_size()[0]; row++) {
-        for (size_type rhs = 0; rhs < nrhs; rhs++) {
-            complex_x->at(row, rhs) =
-                to_complex<ValueType>{dense_x->at(row, rhs)};
-            complex_b->at(row, rhs) =
-                to_complex<ValueType>{dense_b->at(row, rhs)};
-        }
-    }
-
     // Initialize M
 #pragma omp parallel for
     for (size_type i = 0; i < nrhs; i++) {
@@ -171,9 +156,8 @@ void initialize(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for
     for (size_type row = 0; row < m->get_size()[0]; row++) {
         for (size_type col = 0; col < m->get_size()[1]; col++) {
-            m->at(row, col) = (row == col / nrhs)
-                                  ? one<to_complex<ValueType>>()
-                                  : zero<to_complex<ValueType>>();
+            m->at(row, col) =
+                (row == col / nrhs) ? one<ValueType>() : zero<ValueType>();
         }
     }
 
@@ -190,7 +174,7 @@ void initialize(std::shared_ptr<const OmpExecutor> exec,
         }
 
         for (size_type i = 0; i < row; i++) {
-            auto dot = zero<to_complex<ValueType>>();
+            auto dot = zero<ValueType>();
 #pragma omp parallel for reduction(add : dot)
             for (size_type j = 0; j < num_cols; j++) {
                 dot += subspace_vectors->at(row, j) *
@@ -358,26 +342,6 @@ void compute_omega(
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
 
-template <typename ValueType>
-void finalize(std::shared_ptr<const OmpExecutor> exec,
-              const matrix::Dense<to_complex<ValueType>> *complex_x,
-              matrix::Dense<ValueType> *dense_x)
-{
-    if (is_complex<ValueType>()) {
-        dense_x->copy_from(complex_x);
-    } else {
-#pragma omp parallel for
-        for (size_type row = 0; row < dense_x->get_size()[0]; row++) {
-            for (size_type rhs = 0; rhs < dense_x->get_size()[1]; rhs++) {
-                dense_x->at(row, rhs) = real(complex_x->at(row, rhs));
-            }
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_FINALIZE_KERNEL);
-
-
 }  // namespace idr
 }  // namespace omp
 }  // namespace kernels
diff --git a/omp/stop/residual_norm_kernels.cpp b/omp/stop/residual_norm_kernels.cpp
index 88f92ad4ba2..3abe68d58f6 100644
--- a/omp/stop/residual_norm_kernels.cpp
+++ b/omp/stop/residual_norm_kernels.cpp
@@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <omp.h>
-#include <iostream>
 
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
@@ -66,9 +65,6 @@ void residual_norm(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for reduction(|| : local_one_changed)
     for (size_type i = 0; i < tau->get_size()[1]; ++i) {
         if (tau->at(i) < rel_residual_goal * orig_tau->at(i)) {
-            std::cout << "ORIG TAU: " << orig_tau->at(i)
-                      << ", TAU: " << tau->at(i)
-                      << ", TOLERANCE: " << rel_residual_goal << "\n";
             stop_status->get_data()[i].converge(stoppingId, setFinalized);
             local_one_changed = true;
         }
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index e145d16bfca..439d8271a09 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -67,7 +67,7 @@ class Csr : public ::testing::Test {
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
     using ComplexMtx = gko::matrix::Csr<std::complex<double>>;
 
-    Csr() : mtx_size(53, 23), rand_engine(42) {}
+    Csr() : mtx_size(532, 231), rand_engine(42) {}
 
     void SetUp()
     {
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index 3916c0d40de..60291ea2bc8 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -75,22 +75,13 @@ class Idr : public ::testing::Test {
 
         mtx = gen_mtx(123, 123);
 
-        std::ofstream outfile;
-        outfile.open("test.mtx");
-        outfile << std::setprecision(20);
-
-        gko::write(outfile, mtx.get(), gko::layout_type::coordinate);
-
-        outfile.close();
-
         d_mtx = Mtx::create(omp);
-        // make_diag_dominant(mtx.get());
         d_mtx->copy_from(mtx.get());
         omp_idr_factory =
             Solver::build()
                 .with_deterministic(true)
                 .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(10u).on(omp),
+                    gko::stop::Iteration::build().with_max_iters(1u).on(omp),
                     gko::stop::RelativeResidualNorm<>::build()
                         .with_tolerance(1e-13)
                         .on(omp))
@@ -100,7 +91,7 @@ class Idr : public ::testing::Test {
             Solver::build()
                 .with_deterministic(true)
                 .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(10u).on(ref),
+                    gko::stop::Iteration::build().with_max_iters(1u).on(ref),
                     gko::stop::RelativeResidualNorm<>::build()
                         .with_tolerance(1e-13)
                         .on(ref))
@@ -315,20 +306,13 @@ TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
 }
 
 
-TEST_F(Idr, IdrApplyOneRHSIsEquivalentToRef)
+TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 1;
 
-    auto bus = gko::read<Csr>(std::ifstream("1138_bus.mtx"), ref,
-                              std::make_shared<Csr::classical>());
-    auto d_bus = Csr::create(omp, std::make_shared<Csr::classical>());
-    d_bus->copy_from(bus.get());
-
-    m = bus->get_size()[0];
-
-    auto ref_solver = ref_idr_factory->generate(gko::share(bus));
-    auto omp_solver = omp_idr_factory->generate(gko::share(d_bus));
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto omp_solver = omp_idr_factory->generate(d_mtx);
     auto b = gen_mtx(m, n);
     auto x = gen_mtx(m, n);
     auto d_b = Mtx::create(omp);
@@ -339,69 +323,103 @@ TEST_F(Idr, IdrApplyOneRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     omp_solver->apply(d_b.get(), d_x.get());
 
-    /*auto res = Mtx::create(omp, gko::dim<2>{123,1});
-    auto one_op = gko::initialize<Mtx>({1.}, omp);
-    auto neg_one_op = gko::initialize<Mtx>({-1.}, omp);
-
-    res->copy_from(d_b.get());
-    d_mtx->apply(neg_one_op.get(), d_x.get(), one_op.get(), res.get());
-
-    auto d_resnorm = Mtx::create(omp, gko::dim<2>{1, 1});
-    res->compute_norm2(d_resnorm.get());
-
-    res->copy_from(b.get());
-    mtx->apply(neg_one_op.get(), x.get(), one_op.get(), res.get());
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+}
 
-    auto resnorm = Mtx::create(ref, gko::dim<2>{1, 1});
-    res->compute_norm2(resnorm.get());
 
-    std::cout << "INITIAL RESIDUAL NORM REF: " << resnorm->at(0,0) << ", INITIAL
-    RESIDUAL NORM OMP: " << d_resnorm->at(0, 0) << "\n";
+TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
+{
+    int m = 123;
+    int n = 1;
 
-    b->compute_norm2(resnorm.get());
-    d_b->compute_norm2(d_resnorm.get());
+    omp_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(omp),
+                gko::stop::RelativeResidualNorm<>::build()
+                    .with_tolerance(1e-13)
+                    .on(omp))
+            .on(omp);
+
+    ref_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(ref),
+                gko::stop::RelativeResidualNorm<>::build()
+                    .with_tolerance(1e-13)
+                    .on(ref))
+            .on(ref);
 
-    std::cout << "B NORM REF: " << resnorm->at(0,0) << ", B NORM OMP: " <<
-    d_resnorm->at(0,0) << "\n";
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto omp_solver = omp_idr_factory->generate(d_mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(omp);
+    auto d_x = Mtx::create(omp);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     omp_solver->apply(d_b.get(), d_x.get());
 
-    res->copy_from(d_b.get());
-    d_mtx->apply(neg_one_op.get(), d_x.get(), one_op.get(), res.get());
-    res->compute_norm2(d_resnorm.get());
-
-    res->copy_from(b.get());
-    mtx->apply(neg_one_op.get(), x.get(), one_op.get(), res.get());
-    res->compute_norm2(resnorm.get());
-
-    std::cout << "RESIDUAL NORM REF: " << resnorm->at(0,0) << ", RESIDUAL NORM
-    OMP: " << d_resnorm->at(0, 0) << "\n";
-
-    std::ofstream outfile;
-    outfile.open("refresult.mtx");
-    outfile << std::setprecision(20);
-
-    gko::write(outfile, x.get(), gko::layout_type::coordinate);
-
-    outfile.close();
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+}
 
-    outfile.open("rhs.mtx");
-    outfile << std::setprecision(20);
 
-    gko::write(outfile, b.get(), gko::layout_type::coordinate);
+TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
+{
+    int m = 123;
+    int n = 16;
+    auto omp_solver = omp_idr_factory->generate(d_mtx);
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(omp);
+    auto d_x = Mtx::create(omp);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
 
-    outfile.close();*/
+    ref_solver->apply(b.get(), x.get());
+    omp_solver->apply(d_b.get(), d_x.get());
 
     GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
 }
 
 
-/*TEST_F(Idr, IdrApplyMultipleRHSIsEquivalentToRef)
+TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 16;
+
+    omp_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(omp),
+                gko::stop::RelativeResidualNorm<>::build()
+                    .with_tolerance(1e-13)
+                    .on(omp))
+            .on(omp);
+
+    ref_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(ref),
+                gko::stop::RelativeResidualNorm<>::build()
+                    .with_tolerance(1e-13)
+                    .on(ref))
+            .on(ref);
+
     auto omp_solver = omp_idr_factory->generate(d_mtx);
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto b = gen_mtx(m, n);
@@ -416,7 +434,7 @@ TEST_F(Idr, IdrApplyOneRHSIsEquivalentToRef)
 
     GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-}*/
+}
 
 
 }  // namespace
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index a4fa5d453da..23d32aa0cbb 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -134,26 +134,12 @@ get_rand_value(Distribution &&dist, Generator &&gen)
 
 template <typename ValueType>
 void initialize(std::shared_ptr<const ReferenceExecutor> exec,
-                const matrix::Dense<ValueType> *dense_b,
-                const matrix::Dense<ValueType> *dense_x,
-                matrix::Dense<to_complex<ValueType>> *complex_b,
-                matrix::Dense<to_complex<ValueType>> *complex_x,
-                matrix::Dense<to_complex<ValueType>> *m,
-                matrix::Dense<to_complex<ValueType>> *subspace_vectors,
-                bool deterministic, Array<stopping_status> *stop_status)
+                matrix::Dense<ValueType> *m,
+                matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
+                Array<stopping_status> *stop_status)
 {
     const auto nrhs = m->get_size()[1] / m->get_size()[0];
 
-    // Initialize complex b and x
-    for (size_type row = 0; row < dense_b->get_size()[0]; row++) {
-        for (size_type rhs = 0; rhs < nrhs; rhs++) {
-            complex_x->at(row, rhs) =
-                to_complex<ValueType>{dense_x->at(row, rhs)};
-            complex_b->at(row, rhs) =
-                to_complex<ValueType>{dense_b->at(row, rhs)};
-        }
-    }
-
     // Initialize M
     for (size_type i = 0; i < nrhs; i++) {
         stop_status->get_data()[i].reset();
@@ -161,9 +147,8 @@ void initialize(std::shared_ptr<const ReferenceExecutor> exec,
 
     for (size_type row = 0; row < m->get_size()[0]; row++) {
         for (size_type col = 0; col < m->get_size()[1]; col++) {
-            m->at(row, col) = (row == col / nrhs)
-                                  ? one<to_complex<ValueType>>()
-                                  : zero<to_complex<ValueType>>();
+            m->at(row, col) =
+                (row == col / nrhs) ? one<ValueType>() : zero<ValueType>();
         }
     }
 
@@ -180,7 +165,7 @@ void initialize(std::shared_ptr<const ReferenceExecutor> exec,
         }
 
         for (size_type i = 0; i < row; i++) {
-            auto dot = zero<to_complex<ValueType>>();
+            auto dot = zero<ValueType>();
             for (size_type j = 0; j < num_cols; j++) {
                 dot += subspace_vectors->at(row, j) *
                        conj(subspace_vectors->at(i, j));
@@ -191,7 +176,7 @@ void initialize(std::shared_ptr<const ReferenceExecutor> exec,
             }
         }
 
-        auto norm = zero<remove_complex<ValueType>>();
+        auto norm = zero<ValueType>();
         for (size_type j = 0; j < num_cols; j++) {
             norm += squared_norm(subspace_vectors->at(row, j));
         }
@@ -336,25 +321,6 @@ void compute_omega(
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
 
-template <typename ValueType>
-void finalize(std::shared_ptr<const ReferenceExecutor> exec,
-              const matrix::Dense<to_complex<ValueType>> *complex_x,
-              matrix::Dense<ValueType> *dense_x)
-{
-    if (is_complex<ValueType>()) {
-        dense_x->copy_from(complex_x);
-    } else {
-        for (size_type row = 0; row < dense_x->get_size()[0]; row++) {
-            for (size_type rhs = 0; rhs < dense_x->get_size()[1]; rhs++) {
-                dense_x->at(row, rhs) = real(complex_x->at(row, rhs));
-            }
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_FINALIZE_KERNEL);
-
-
 }  // namespace idr
 }  // namespace reference
 }  // namespace kernels
diff --git a/reference/stop/residual_norm_kernels.cpp b/reference/stop/residual_norm_kernels.cpp
index a3446414d25..0777276e31c 100644
--- a/reference/stop/residual_norm_kernels.cpp
+++ b/reference/stop/residual_norm_kernels.cpp
@@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
-#include <iostream>
 
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
@@ -68,9 +67,6 @@ void residual_norm(std::shared_ptr<const ReferenceExecutor> exec,
     *one_changed = false;
     for (size_type i = 0; i < tau->get_size()[1]; ++i) {
         if (tau->at(i) < rel_residual_goal * orig_tau->at(i)) {
-            std::cout << "ORIG TAU: " << orig_tau->at(i)
-                      << ", TAU: " << tau->at(i)
-                      << ", TOLERANCE: " << rel_residual_goal << "\n";
             stop_status->get_data()[i].converge(stoppingId, setFinalized);
             *one_changed = true;
         }
diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp
index 566721d2aba..1c34aa8f06b 100644
--- a/reference/test/solver/idr_kernels.cpp
+++ b/reference/test/solver/idr_kernels.cpp
@@ -67,6 +67,7 @@ class Idr : public ::testing::Test {
               {{1.0, -3.0, 0.0}, {-4.0, 1.0, -3.0}, {2.0, -1.0, 2.0}}, exec)),
           idr_factory(
               Solver::build()
+                  .with_deterministic(true)
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(8u).on(exec),
                       gko::stop::Time::build()
@@ -78,6 +79,7 @@ class Idr : public ::testing::Test {
                   .on(exec)),
           idr_factory_precision(
               Solver::build()
+                  .with_deterministic(true)
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(50u).on(
                           exec),
@@ -114,6 +116,35 @@ TYPED_TEST(Idr, SolvesDenseSystem)
 }
 
 
+TYPED_TEST(Idr, SolvesDenseSystemWithComplexSubSpace)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using Solver = typename TestFixture::Solver;
+    auto half_tol = std::sqrt(r<value_type>::value);
+    auto solver_factory =
+        Solver::build()
+            .with_complex_subspace(true)
+            .with_deterministic(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(8u).on(this->exec),
+                gko::stop::Time::build()
+                    .with_time_limit(std::chrono::seconds(6))
+                    .on(this->exec),
+                gko::stop::ResidualNormReduction<value_type>::build()
+                    .with_reduction_factor(r<value_type>::value)
+                    .on(this->exec))
+            .on(this->exec);
+    auto solver = solver_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), half_tol);
+}
+
+
 TYPED_TEST(Idr, SolvesMultipleDenseSystems)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -133,6 +164,39 @@ TYPED_TEST(Idr, SolvesMultipleDenseSystems)
 }
 
 
+TYPED_TEST(Idr, SolvesMultipleDenseSystemsWithComplexSubspace)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using T = value_type;
+    using Solver = typename TestFixture::Solver;
+    auto half_tol = std::sqrt(r<value_type>::value);
+    auto solver_factory =
+        Solver::build()
+            .with_complex_subspace(true)
+            .with_deterministic(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(8u).on(this->exec),
+                gko::stop::Time::build()
+                    .with_time_limit(std::chrono::seconds(6))
+                    .on(this->exec),
+                gko::stop::ResidualNormReduction<value_type>::build()
+                    .with_reduction_factor(r<value_type>::value)
+                    .on(this->exec))
+            .on(this->exec);
+    auto solver = solver_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {I<T>{-1.0, -5.0}, I<T>{3.0, 1.0}, I<T>{1.0, -2.0}}, this->exec);
+    auto x = gko::initialize<Mtx>(
+        {I<T>{0.0, 0.0}, I<T>{0.0, 0.0}, I<T>{0.0, 0.0}}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({{-4.0, 1.0}, {-1.0, 2.0}, {4.0, -1.0}}),
+                        half_tol);
+}
+
+
 TYPED_TEST(Idr, SolvesDenseSystemUsingAdvancedApply)
 {
     using Mtx = typename TestFixture::Mtx;

From a1366e9f862726ab02f5bd53fbe709ec6613c61f Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 5 Nov 2020 13:01:44 +0100
Subject: [PATCH 0155/1008] Add hip support for complex blas functions

---
 cuda/solver/idr_kernels.cu        |  2 +-
 cuda/test/solver/idr_kernels.cpp  | 32 +++-------------
 hip/base/hipblas_bindings.hip.hpp | 43 ++++++++-------------
 hip/base/types.hip.hpp            | 64 +++++++++++++++++++++++++++++++
 hip/solver/idr_kernels.hip.cpp    |  2 +-
 hip/test/solver/idr_kernels.cpp   | 31 +++------------
 omp/test/solver/idr_kernels.cpp   | 35 ++++-------------
 7 files changed, 103 insertions(+), 106 deletions(-)

diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index 158653f5ade..47a278ffab3 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -158,7 +158,7 @@ void update_g_and_u(std::shared_ptr<const CudaExecutor> exec, size_type k,
 
     for (size_type i = 0; i < k; i++) {
         const auto p_i = p->get_const_values() + i * p_stride;
-        if (nrhs > 1) {
+        if (nrhs > 1 || is_complex<ValueType>()) {
             components::fill_array(exec, alpha->get_values(), nrhs,
                                    zero<ValueType>());
             multidot_kernel<<<grid_dim, block_dim>>>(
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index 31019fc0d13..7254d09fbcc 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -76,20 +76,14 @@ class Idr : public ::testing::Test {
             Solver::build()
                 .with_deterministic(true)
                 .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(cuda),
-                    gko::stop::ResidualNormReduction<>::build()
-                        .with_reduction_factor(1e-15)
-                        .on(cuda))
+                    gko::stop::Iteration::build().with_max_iters(1u).on(cuda))
                 .on(cuda);
 
         ref_idr_factory =
             Solver::build()
                 .with_deterministic(true)
                 .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNormReduction<>::build()
-                        .with_reduction_factor(1e-15)
-                        .on(ref))
+                    gko::stop::Iteration::build().with_max_iters(1u).on(ref))
                 .on(ref);
     }
 
@@ -333,10 +327,7 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .with_deterministic(true)
             .with_complex_subspace(true)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(cuda),
-                gko::stop::RelativeResidualNorm<>::build()
-                    .with_tolerance(1e-13)
-                    .on(cuda))
+                gko::stop::Iteration::build().with_max_iters(1u).on(cuda))
             .on(cuda);
 
     ref_idr_factory =
@@ -344,10 +335,7 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .with_deterministic(true)
             .with_complex_subspace(true)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(ref),
-                gko::stop::RelativeResidualNorm<>::build()
-                    .with_tolerance(1e-13)
-                    .on(ref))
+                gko::stop::Iteration::build().with_max_iters(1u).on(ref))
             .on(ref);
 
     auto ref_solver = ref_idr_factory->generate(mtx);
@@ -397,22 +385,14 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
         Solver::build()
             .with_deterministic(true)
             .with_complex_subspace(true)
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(cuda),
-                gko::stop::RelativeResidualNorm<>::build()
-                    .with_tolerance(1e-13)
-                    .on(cuda))
+            .with_criteria(gko::stop::Iteration::build().with_max_iters(1u))
             .on(cuda);
 
     ref_idr_factory =
         Solver::build()
             .with_deterministic(true)
             .with_complex_subspace(true)
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(ref),
-                gko::stop::RelativeResidualNorm<>::build()
-                    .with_tolerance(1e-13)
-                    .on(ref))
+            .with_criteria(gko::stop::Iteration::build().with_max_iters(1u))
             .on(ref);
 
     auto cuda_solver = cuda_idr_factory->generate(d_mtx);
diff --git a/hip/base/hipblas_bindings.hip.hpp b/hip/base/hipblas_bindings.hip.hpp
index 7bef3278f79..d80908cebc4 100644
--- a/hip/base/hipblas_bindings.hip.hpp
+++ b/hip/base/hipblas_bindings.hip.hpp
@@ -91,15 +91,11 @@ struct is_supported<float> : std::true_type {};
 template <>
 struct is_supported<double> : std::true_type {};
 
-// hipblas supports part of complex function version is >= 0.19, but the version
-// is not set now.
-/* not implemented
 template <>
 struct is_supported<std::complex<float>> : std::true_type {};
 
 template <>
 struct is_supported<std::complex<double>> : std::true_type {};
-*/
 
 
 #define GKO_BIND_HIPBLAS_GEMM(ValueType, HipblasName)                        \
@@ -110,9 +106,9 @@ struct is_supported<std::complex<double>> : std::true_type {};
                      ValueType *c, int ldc)                                  \
     {                                                                        \
         GKO_ASSERT_NO_HIPBLAS_ERRORS(HipblasName(                            \
-            handle, transa, transb, m, n, k, as_hiplibs_type(alpha),         \
-            as_hiplibs_type(a), lda, as_hiplibs_type(b), ldb,                \
-            as_hiplibs_type(beta), as_hiplibs_type(c), ldc));                \
+            handle, transa, transb, m, n, k, as_hipblas_type(alpha),         \
+            as_hipblas_type(a), lda, as_hipblas_type(b), ldb,                \
+            as_hipblas_type(beta), as_hipblas_type(c), ldc));                \
     }                                                                        \
     static_assert(true,                                                      \
                   "This assert is used to counter the false positive extra " \
@@ -120,10 +116,9 @@ struct is_supported<std::complex<double>> : std::true_type {};
 
 GKO_BIND_HIPBLAS_GEMM(float, hipblasSgemm);
 GKO_BIND_HIPBLAS_GEMM(double, hipblasDgemm);
-/* not implemented
 GKO_BIND_HIPBLAS_GEMM(std::complex<float>, hipblasCgemm);
 GKO_BIND_HIPBLAS_GEMM(std::complex<double>, hipblasZgemm);
-*/
+
 template <typename ValueType>
 GKO_BIND_HIPBLAS_GEMM(ValueType, detail::not_implemented);
 
@@ -138,9 +133,9 @@ GKO_BIND_HIPBLAS_GEMM(ValueType, detail::not_implemented);
                      ValueType *c, int ldc)                                   \
     {                                                                         \
         GKO_ASSERT_NO_HIPBLAS_ERRORS(                                         \
-            HipblasName(handle, transa, transb, m, n, as_hiplibs_type(alpha), \
-                        as_hiplibs_type(a), lda, as_hiplibs_type(beta),       \
-                        as_hiplibs_type(b), ldb, as_hiplibs_type(c), ldc));   \
+            HipblasName(handle, transa, transb, m, n, as_hipblas_type(alpha), \
+                        as_hipblas_type(a), lda, as_hipblas_type(beta),       \
+                        as_hipblas_type(b), ldb, as_hipblas_type(c), ldc));   \
     }                                                                         \
     static_assert(true,                                                       \
                   "This assert is used to counter the false positive extra "  \
@@ -160,7 +155,7 @@ GKO_BIND_HIPBLAS_GEAM(ValueType, detail::not_implemented);
                      ValueType *x, int incx)                                 \
     {                                                                        \
         GKO_ASSERT_NO_HIPBLAS_ERRORS(HipblasName(                            \
-            handle, n, as_hiplibs_type(alpha), as_hiplibs_type(x), incx));   \
+            handle, n, as_hipblas_type(alpha), as_hipblas_type(x), incx));   \
     }                                                                        \
     static_assert(true,                                                      \
                   "This assert is used to counter the false positive extra " \
@@ -168,10 +163,9 @@ GKO_BIND_HIPBLAS_GEAM(ValueType, detail::not_implemented);
 
 GKO_BIND_HIPBLAS_SCAL(float, hipblasSscal);
 GKO_BIND_HIPBLAS_SCAL(double, hipblasDscal);
-/* not implemented
 GKO_BIND_HIPBLAS_SCAL(std::complex<float>, hipblasCscal);
 GKO_BIND_HIPBLAS_SCAL(std::complex<double>, hipblasZscal);
-*/
+
 template <typename ValueType>
 GKO_BIND_HIPBLAS_SCAL(ValueType, detail::not_implemented);
 
@@ -183,8 +177,8 @@ GKO_BIND_HIPBLAS_SCAL(ValueType, detail::not_implemented);
                      const ValueType *x, int incx, ValueType *y, int incy)     \
     {                                                                          \
         GKO_ASSERT_NO_HIPBLAS_ERRORS(                                          \
-            HipblasName(handle, n, as_hiplibs_type(alpha), as_hiplibs_type(x), \
-                        incx, as_hiplibs_type(y), incy));                      \
+            HipblasName(handle, n, as_hipblas_type(alpha), as_hipblas_type(x), \
+                        incx, as_hipblas_type(y), incy));                      \
     }                                                                          \
     static_assert(true,                                                        \
                   "This assert is used to counter the false positive extra "   \
@@ -192,10 +186,9 @@ GKO_BIND_HIPBLAS_SCAL(ValueType, detail::not_implemented);
 
 GKO_BIND_HIPBLAS_AXPY(float, hipblasSaxpy);
 GKO_BIND_HIPBLAS_AXPY(double, hipblasDaxpy);
-/* not implemented
 GKO_BIND_HIPBLAS_AXPY(std::complex<float>, hipblasCaxpy);
 GKO_BIND_HIPBLAS_AXPY(std::complex<double>, hipblasZaxpy);
-*/
+
 template <typename ValueType>
 GKO_BIND_HIPBLAS_AXPY(ValueType, detail::not_implemented);
 
@@ -207,8 +200,8 @@ GKO_BIND_HIPBLAS_AXPY(ValueType, detail::not_implemented);
                     int incx, const ValueType *y, int incy, ValueType *result) \
     {                                                                          \
         GKO_ASSERT_NO_HIPBLAS_ERRORS(                                          \
-            HipblasName(handle, n, as_hiplibs_type(x), incx,                   \
-                        as_hiplibs_type(y), incy, as_hiplibs_type(result)));   \
+            HipblasName(handle, n, as_hipblas_type(x), incx,                   \
+                        as_hipblas_type(y), incy, as_hipblas_type(result)));   \
     }                                                                          \
     static_assert(true,                                                        \
                   "This assert is used to counter the false positive extra "   \
@@ -216,10 +209,9 @@ GKO_BIND_HIPBLAS_AXPY(ValueType, detail::not_implemented);
 
 GKO_BIND_HIPBLAS_DOT(float, hipblasSdot);
 GKO_BIND_HIPBLAS_DOT(double, hipblasDdot);
-/* not implemented
 GKO_BIND_HIPBLAS_DOT(std::complex<float>, hipblasCdotc);
 GKO_BIND_HIPBLAS_DOT(std::complex<double>, hipblasZdotc);
-*/
+
 template <typename ValueType>
 GKO_BIND_HIPBLAS_DOT(ValueType, detail::not_implemented);
 
@@ -231,7 +223,7 @@ GKO_BIND_HIPBLAS_DOT(ValueType, detail::not_implemented);
                       int incx, remove_complex<ValueType> *result)           \
     {                                                                        \
         GKO_ASSERT_NO_HIPBLAS_ERRORS(HipblasName(                            \
-            handle, n, as_hiplibs_type(x), incx, as_hiplibs_type(result)));  \
+            handle, n, as_hipblas_type(x), incx, as_hipblas_type(result)));  \
     }                                                                        \
     static_assert(true,                                                      \
                   "This assert is used to counter the false positive extra " \
@@ -239,10 +231,9 @@ GKO_BIND_HIPBLAS_DOT(ValueType, detail::not_implemented);
 
 GKO_BIND_HIPBLAS_NORM2(float, hipblasSnrm2);
 GKO_BIND_HIPBLAS_NORM2(double, hipblasDnrm2);
-/* not implemented
 GKO_BIND_HIPBLAS_NORM2(std::complex<float>, hipblasScnrm2);
 GKO_BIND_HIPBLAS_NORM2(std::complex<double>, hipblasDznrm2);
-*/
+
 template <typename ValueType>
 GKO_BIND_HIPBLAS_NORM2(ValueType, detail::not_implemented);
 
diff --git a/hip/base/types.hip.hpp b/hip/base/types.hip.hpp
index add1b10b40b..e3f43e8e67c 100644
--- a/hip/base/types.hip.hpp
+++ b/hip/base/types.hip.hpp
@@ -95,6 +95,46 @@ struct hiplibs_type_impl<thrust::complex<T>> {
     using type = typename hiplibs_type_impl<std::complex<T>>::type;
 };
 
+template <typename T>
+struct hipblas_type_impl {
+    using type = T;
+};
+
+template <typename T>
+struct hipblas_type_impl<T *> {
+    using type = typename hipblas_type_impl<T>::type *;
+};
+
+template <typename T>
+struct hipblas_type_impl<T &> {
+    using type = typename hipblas_type_impl<T>::type &;
+};
+
+template <typename T>
+struct hipblas_type_impl<const T> {
+    using type = const typename hipblas_type_impl<T>::type;
+};
+
+template <typename T>
+struct hipblas_type_impl<volatile T> {
+    using type = volatile typename hipblas_type_impl<T>::type;
+};
+
+template <>
+struct hipblas_type_impl<std::complex<float>> {
+    using type = hipblasComplex;
+};
+
+template <>
+struct hipblas_type_impl<std::complex<double>> {
+    using type = hipblasDoubleComplex;
+};
+
+template <typename T>
+struct hipblas_type_impl<thrust::complex<T>> {
+    using type = typename hipblas_type_impl<std::complex<T>>::type;
+};
+
 template <typename T>
 struct hip_type_impl {
     using type = T;
@@ -252,6 +292,30 @@ inline hiplibs_type<T> as_hiplibs_type(T val)
 }
 
 
+/**
+ * This is an alias for equivalent of type T used in the HIPBLAS library.
+ *
+ * @tparam T  a type
+ */
+template <typename T>
+using hipblas_type = typename detail::hipblas_type_impl<T>::type;
+
+
+/**
+ * Reinterprets the passed in value as an equivalent type used by the HIPBLAS
+ * library.
+ *
+ * @param val  the value to reinterpret
+ *
+ * @return `val` reinterpreted to type used by HIP libraries
+ */
+template <typename T>
+inline hipblas_type<T> as_hipblas_type(T val)
+{
+    return reinterpret_cast<hipblas_type<T>>(val);
+}
+
+
 }  // namespace hip
 }  // namespace kernels
 }  // namespace gko
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index dcf8eb86dc7..b6fc531c0a6 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -164,7 +164,7 @@ void update_g_and_u(std::shared_ptr<const HipExecutor> exec, size_type k,
 
     for (size_type i = 0; i < k; i++) {
         const auto p_i = p->get_const_values() + i * p_stride;
-        if (nrhs > 1) {
+        if (nrhs > 1 || is_complex<ValueType>()) {
             components::fill_array(exec, alpha->get_values(), nrhs,
                                    zero<ValueType>());
             hipLaunchKernelGGL(
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index f8e84cc448f..b9779f8772c 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -69,27 +69,20 @@ class Idr : public ::testing::Test {
         hip = gko::HipExecutor::create(0, ref);
 
         mtx = gen_mtx(123, 123);
-        make_diag_dominant(mtx.get());
         d_mtx = Mtx::create(hip);
         d_mtx->copy_from(mtx.get());
         hip_idr_factory =
             Solver::build()
                 .with_deterministic(true)
                 .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(hip),
-                    gko::stop::ResidualNormReduction<>::build()
-                        .with_reduction_factor(1e-15)
-                        .on(hip))
+                    gko::stop::Iteration::build().with_max_iters(1u).on(hip))
                 .on(hip);
 
         ref_idr_factory =
             Solver::build()
                 .with_deterministic(true)
                 .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNormReduction<>::build()
-                        .with_reduction_factor(1e-15)
-                        .on(ref))
+                    gko::stop::Iteration::build().with_max_iters(1u).on(ref))
                 .on(ref);
     }
 
@@ -333,10 +326,7 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .with_deterministic(true)
             .with_complex_subspace(true)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(hip),
-                gko::stop::RelativeResidualNorm<>::build()
-                    .with_tolerance(1e-13)
-                    .on(hip))
+                gko::stop::Iteration::build().with_max_iters(1u).on(hip))
             .on(hip);
 
     ref_idr_factory =
@@ -344,10 +334,7 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .with_deterministic(true)
             .with_complex_subspace(true)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(ref),
-                gko::stop::RelativeResidualNorm<>::build()
-                    .with_tolerance(1e-13)
-                    .on(ref))
+                gko::stop::Iteration::build().with_max_iters(1u).on(ref))
             .on(ref);
 
     auto ref_solver = ref_idr_factory->generate(mtx);
@@ -398,10 +385,7 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
             .with_deterministic(true)
             .with_complex_subspace(true)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(hip),
-                gko::stop::RelativeResidualNorm<>::build()
-                    .with_tolerance(1e-13)
-                    .on(hip))
+                gko::stop::Iteration::build().with_max_iters(1u).on(hip))
             .on(hip);
 
     ref_idr_factory =
@@ -409,10 +393,7 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
             .with_deterministic(true)
             .with_complex_subspace(true)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(ref),
-                gko::stop::RelativeResidualNorm<>::build()
-                    .with_tolerance(1e-13)
-                    .on(ref))
+                gko::stop::Iteration::build().with_max_iters(1u).on(ref))
             .on(ref);
 
     auto hip_solver = hip_idr_factory->generate(d_mtx);
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index 60291ea2bc8..b6cdc94ef15 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -74,27 +74,20 @@ class Idr : public ::testing::Test {
         omp = gko::OmpExecutor::create();
 
         mtx = gen_mtx(123, 123);
-
         d_mtx = Mtx::create(omp);
         d_mtx->copy_from(mtx.get());
         omp_idr_factory =
             Solver::build()
                 .with_deterministic(true)
                 .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(1u).on(omp),
-                    gko::stop::RelativeResidualNorm<>::build()
-                        .with_tolerance(1e-13)
-                        .on(omp))
+                    gko::stop::Iteration::build().with_max_iters(1u).on(omp))
                 .on(omp);
 
         ref_idr_factory =
             Solver::build()
                 .with_deterministic(true)
                 .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(1u).on(ref),
-                    gko::stop::RelativeResidualNorm<>::build()
-                        .with_tolerance(1e-13)
-                        .on(ref))
+                    gko::stop::Iteration::build().with_max_iters(1u).on(ref))
                 .on(ref);
     }
 
@@ -224,7 +217,7 @@ class Idr : public ::testing::Test {
 };
 
 
-/*TEST_F(Idr, IdrInitializeIsEquivalentToRef)
+TEST_F(Idr, IdrInitializeIsEquivalentToRef)
 {
     initialize_data();
 
@@ -235,7 +228,7 @@ class Idr : public ::testing::Test {
 
     GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
     GKO_ASSERT_MTX_NEAR(p, d_p, 1e-14);
-}*/
+}
 
 
 TEST_F(Idr, IdrStep1IsEquivalentToRef)
@@ -338,10 +331,7 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .with_deterministic(true)
             .with_complex_subspace(true)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(omp),
-                gko::stop::RelativeResidualNorm<>::build()
-                    .with_tolerance(1e-13)
-                    .on(omp))
+                gko::stop::Iteration::build().with_max_iters(1u).on(omp))
             .on(omp);
 
     ref_idr_factory =
@@ -349,10 +339,7 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .with_deterministic(true)
             .with_complex_subspace(true)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(ref),
-                gko::stop::RelativeResidualNorm<>::build()
-                    .with_tolerance(1e-13)
-                    .on(ref))
+                gko::stop::Iteration::build().with_max_iters(1u).on(ref))
             .on(ref);
 
     auto ref_solver = ref_idr_factory->generate(mtx);
@@ -403,10 +390,7 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
             .with_deterministic(true)
             .with_complex_subspace(true)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(omp),
-                gko::stop::RelativeResidualNorm<>::build()
-                    .with_tolerance(1e-13)
-                    .on(omp))
+                gko::stop::Iteration::build().with_max_iters(1u).on(omp))
             .on(omp);
 
     ref_idr_factory =
@@ -414,10 +398,7 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
             .with_deterministic(true)
             .with_complex_subspace(true)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(ref),
-                gko::stop::RelativeResidualNorm<>::build()
-                    .with_tolerance(1e-13)
-                    .on(ref))
+                gko::stop::Iteration::build().with_max_iters(1u).on(ref))
             .on(ref);
 
     auto omp_solver = omp_idr_factory->generate(d_mtx);

From be736a9de5d55cf3d631dba7259381a269719688 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 5 Nov 2020 14:12:37 +0100
Subject: [PATCH 0156/1008] Add missing check for complex in hip and cuda

---
 benchmark/CMakeLists.txt         | 2 ++
 cmake/GinkgoConfig.cmake.in      | 1 +
 cmake/build_helpers.cmake        | 2 +-
 cmake/create_test.cmake          | 2 +-
 cuda/solver/idr_kernels.cu       | 2 +-
 cuda/test/solver/idr_kernels.cpp | 6 ++++--
 hip/CMakeLists.txt               | 2 +-
 hip/solver/idr_kernels.hip.cpp   | 2 +-
 8 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
index 62cfb1b2538..ad63598fe70 100644
--- a/benchmark/CMakeLists.txt
+++ b/benchmark/CMakeLists.txt
@@ -32,7 +32,9 @@ function(ginkgo_benchmark_hipsp_linops name)
     find_package(Threads REQUIRED)
     find_package(HIP REQUIRED)
     find_package(hipblas REQUIRED)
+    find_package(hiprand REQUIRED)
     find_package(hipsparse REQUIRED)
+    find_package(rocrand REQUIRED)
     target_include_directories("${name}" SYSTEM PRIVATE
         ${HSA_HEADER} ${HIP_INCLUDE_DIRS}
         ${HIPBLAS_INCLUDE_DIRS} ${HIPSPARSE_INCLUDE_DIRS})
diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index d58bbf9aac0..3f8160daeff 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -166,6 +166,7 @@ if((NOT GINKGO_BUILD_SHARED_LIBS) AND GINKGO_BUILD_HIP)
     find_package(hipblas REQUIRED)
     find_package(hiprand REQUIRED)
     find_package(hipsparse REQUIRED)
+    find_package(rocrand REQUIRED)
     if(GINKGO_HIP_PLATFORM MATCHES "hcc")
         ginkgo_hip_ban_link_hcflag(hcc::hccrt)
         ginkgo_hip_ban_link_hcflag(hcc::hc_am)
diff --git a/cmake/build_helpers.cmake b/cmake/build_helpers.cmake
index 06e77eca733..78c3d810021 100644
--- a/cmake/build_helpers.cmake
+++ b/cmake/build_helpers.cmake
@@ -92,7 +92,7 @@ function(ginkgo_check_headers target)
             "${CMAKE_CURRENT_SOURCE_DIR}"
             "${GINKGO_HIP_THRUST_PATH}"
             "${HIPBLAS_INCLUDE_DIRS}"
-            "${HIPRAND_INCLUDE_DIRS}"
+            "${hiprand_INCLUDE_DIRS}"
             "${HIPSPARSE_INCLUDE_DIRS}"
             "${ROCPRIM_INCLUDE_DIRS}")
     endif()
diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index 9a7ece55f2f..51d264dd367 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -160,7 +160,7 @@ function(ginkgo_create_hip_test test_name)
         # Only `exception_helpers` requires thess so far, but it's much easier
         # to put these this way.
         ${HIPBLAS_INCLUDE_DIRS}
-        ${HIPRAND_INCLUDE_DIRS}
+        ${hiprand_INCLUDE_DIRS}
         ${HIPSPARSE_INCLUDE_DIRS}
         )
     set_target_properties(${TEST_TARGET_NAME} PROPERTIES
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index 47a278ffab3..dc72bc0eb5b 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -208,7 +208,7 @@ void update_m(std::shared_ptr<const CudaExecutor> exec, size_type k,
     for (size_type i = k; i < subspace_dim; i++) {
         const auto p_i = p->get_const_values() + i * p_stride;
         auto m_i = m->get_values() + i * m_stride + k * nrhs;
-        if (nrhs > 1) {
+        if (nrhs > 1 || is_complex<ValueType>()) {
             components::fill_array(exec, m_i, nrhs, zero<ValueType>());
             multidot_kernel<<<grid_dim, block_dim>>>(
                 size, nrhs, as_cuda_type(p_i),
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index 7254d09fbcc..67cbd886f66 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -385,14 +385,16 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
         Solver::build()
             .with_deterministic(true)
             .with_complex_subspace(true)
-            .with_criteria(gko::stop::Iteration::build().with_max_iters(1u))
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(cuda))
             .on(cuda);
 
     ref_idr_factory =
         Solver::build()
             .with_deterministic(true)
             .with_complex_subspace(true)
-            .with_criteria(gko::stop::Iteration::build().with_max_iters(1u))
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(ref))
             .on(ref);
 
     auto cuda_solver = cuda_idr_factory->generate(d_mtx);
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 58d5a074199..486f550ea27 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -301,7 +301,7 @@ target_include_directories(ginkgo_hip
     PRIVATE
         ${GINKGO_HIP_THRUST_PATH}
         ${HIPBLAS_INCLUDE_DIRS}
-        ${HIPRAND_INCLUDE_DIRS}
+        ${hiprand_INCLUDE_DIRS}
         ${HIPSPARSE_INCLUDE_DIRS}
         $<BUILD_INTERFACE:${ROCPRIM_INCLUDE_DIRS}>)
 
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index b6fc531c0a6..78d4ad6aae2 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -216,7 +216,7 @@ void update_m(std::shared_ptr<const HipExecutor> exec, size_type k,
     for (size_type i = k; i < subspace_dim; i++) {
         const auto p_i = p->get_const_values() + i * p_stride;
         auto m_i = m->get_values() + i * m_stride + k * nrhs;
-        if (nrhs > 1) {
+        if (nrhs > 1 || is_complex<ValueType>()) {
             components::fill_array(exec, m_i, nrhs, zero<ValueType>());
             hipLaunchKernelGGL(multidot_kernel, grid_dim, block_dim, 0, 0, size,
                                nrhs, as_hip_type(p_i),

From aac73574c2e8475092502f55412c3a1ec7b366fb Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 10 Nov 2020 16:18:14 +0100
Subject: [PATCH 0157/1008] Disable HIP with CUDA < 9.2.

---
 .gitlab-ci.yml | 86 +++++++++++++++++++++++++-------------------------
 1 file changed, 43 insertions(+), 43 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 51ab318a3c9..b1ae57ba935 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -127,7 +127,7 @@ stages:
 
 sync:
   stage: sync
-  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
   variables:
     GIT_STRATEGY: none
     PRIVATE_REPO: git@gitlab.com:ginkgo-project/ginkgo.git
@@ -153,7 +153,7 @@ sync:
 # Job with example runs.
 build/cuda90/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda90-gnu5-llvm39
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda90-gnu5-llvm39
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -171,7 +171,7 @@ build/cuda90/gcc/all/debug/shared:
 
 build/cuda90/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda90-gnu5-llvm39
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda90-gnu5-llvm39
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -192,7 +192,7 @@ build/cuda90/clang/all/release/static:
 # cuda 9.1 and friends
 build/cuda91/gcc/all/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda91-gnu6-llvm40
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda91-gnu6-llvm40
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -210,7 +210,7 @@ build/cuda91/gcc/all/debug/static:
 
 build/cuda91/clang/all/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda91-gnu6-llvm40
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda91-gnu6-llvm40
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -231,7 +231,7 @@ build/cuda91/clang/all/release/shared:
 # cuda 9.2 and friends
 build/cuda92/gcc/all/release/debug:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda92-gnu7-llvm50-intel2017
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -249,7 +249,7 @@ build/cuda92/gcc/all/release/debug:
 
 build/cuda92/intel/cuda/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda92-gnu7-llvm50-intel2017
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -270,7 +270,7 @@ build/cuda92/intel/cuda/release/static:
 # cuda 10.0 and friends
 build/cuda100/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda100-gnu7-llvm60-intel2018
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -288,7 +288,7 @@ build/cuda100/gcc/all/debug/shared:
 
 build/cuda100/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda100-gnu7-llvm60-intel2018
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -309,7 +309,7 @@ build/cuda100/clang/all/release/static:
 
 build/cuda100/intel/cuda/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda100-gnu7-llvm60-intel2018
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -329,7 +329,7 @@ build/cuda100/intel/cuda/release/shared:
 # cuda 10.1 and friends
 build/cuda101/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -347,7 +347,7 @@ build/cuda101/gcc/all/debug/shared:
 
 build/cuda101/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -369,7 +369,7 @@ build/cuda101/clang/all/release/static:
 # clang-cuda with cuda 10.1 and friends
 build/clang-cuda101/gcc/all/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm10-intel2019
   variables:
     <<: *default_variables
     CUDA_COMPILER: "clang++"
@@ -388,7 +388,7 @@ build/clang-cuda101/gcc/all/release/shared:
 
 build/clang-cuda101/clang/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm10-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -410,7 +410,7 @@ build/clang-cuda101/clang/cuda/debug/static:
 # cuda 10.2 and friends
 build/cuda102/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda102-gnu8-llvm8-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -428,7 +428,7 @@ build/cuda102/gcc/all/debug/shared:
 
 build/cuda102/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda102-gnu8-llvm8-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -449,7 +449,7 @@ build/cuda102/clang/all/release/static:
 
 build/cuda102/intel/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda102-gnu8-llvm8-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -470,7 +470,7 @@ build/cuda102/intel/cuda/debug/static:
 # cuda 11.0 and friends
 build/cuda110/gcc/cuda/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda110-gnu9-llvm9-intel2020
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -487,7 +487,7 @@ build/cuda110/gcc/cuda/debug/shared:
 
 build/cuda110/clang/cuda/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda110-gnu9-llvm9-intel2020
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -507,7 +507,7 @@ build/cuda110/clang/cuda/release/static:
 
 build/cuda110/intel/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda110-gnu9-llvm9-intel2020
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -528,7 +528,7 @@ build/cuda110/intel/cuda/debug/static:
 # HIP AMD
 build/amd/gcc/hip/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-amd-gnu8-llvm7
+  image: localhost:5000/add_hiprand_8d042b48_gko-amd-gnu8-llvm7
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -545,7 +545,7 @@ build/amd/gcc/hip/debug/shared:
 
 build/amd/clang/hip/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-amd-gnu8-llvm7
+  image: localhost:5000/add_hiprand_8d042b48_gko-amd-gnu8-llvm7
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -565,7 +565,7 @@ build/amd/clang/hip/release/static:
 # no cuda but latest gcc and clang
 build/nocuda/gcc/core/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_REFERENCE: "OFF"
@@ -580,7 +580,7 @@ build/nocuda/gcc/core/debug/static:
 
 build/nocuda/clang/core/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -596,7 +596,7 @@ build/nocuda/clang/core/release/shared:
 
 build/nocuda/intel/core/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -613,7 +613,7 @@ build/nocuda/intel/core/debug/shared:
 
 build/nocuda/gcc/omp/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -627,7 +627,7 @@ build/nocuda/gcc/omp/release/shared:
 
 build/nocuda/clang/omp/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -644,7 +644,7 @@ build/nocuda/clang/omp/debug/static:
 
 build/nocuda/intel/omp/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -683,7 +683,7 @@ build/dpcpp/cpu/release/static:
 warnings:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -704,7 +704,7 @@ warnings:
 no-circular-deps:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -746,7 +746,7 @@ subdir-build:
 clang-tidy:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -766,7 +766,7 @@ clang-tidy:
 iwyu:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -787,7 +787,7 @@ iwyu:
 # For short living branches or PRs, try to detect an open PR
 sonarqube_cov_:
   stage: code_quality
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
   before_script: *default_before_script
   script:
     - PR_ID=$(curl "https://api.github.com/search/issues?q=sha:${CI_COMMIT_SHA}"
@@ -828,7 +828,7 @@ sonarqube_cov_:
 # (the one that was merged).
 sonarqube_cov:
   stage: code_quality
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
   before_script: *default_before_script
   script:
     - ctest -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=COVERAGE
@@ -855,7 +855,7 @@ sonarqube_cov:
 # Deploy documentation to github-pages
 gh-pages:
   stage: deploy
-  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     PUBLIC_REPO: git@github.com:ginkgo-project/ginkgo.git
@@ -902,7 +902,7 @@ gh-pages:
 
 threadsanitizer:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - LD_PRELOAD=/usr/local/lib/libomp.so
@@ -926,7 +926,7 @@ threadsanitizer:
 
 leaksanitizer:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=LSAN
@@ -946,7 +946,7 @@ leaksanitizer:
 
 addresssanitizer:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=ASAN
@@ -966,7 +966,7 @@ addresssanitizer:
 
 undefinedsanitizer:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     # the Gold linker is required because of a linker flag issues given by UBsan
@@ -988,7 +988,7 @@ undefinedsanitizer:
 
 cudamemcheck:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_fec6ada9_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=RelWithDebInfo
@@ -1027,7 +1027,7 @@ cudamemcheck:
 
 fineci-benchmark-build:
   stage: benchmark-build
-  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BENCHMARK_SERVER: FINECI
@@ -1108,7 +1108,7 @@ fineci-benchmark-build:
 
 fineci-benchmark-cuda:
   stage: benchmark-cuda
-  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BENCHMARK_SERVER: FINECI
@@ -1146,7 +1146,7 @@ fineci-benchmark-cuda:
 
 new-issue-on-failure:
   stage: on-failure
-  image: localhost:5000/add_hiprand_fec6ada9_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
   script: curl --request POST "https://gitlab.com/api/v4/projects/${PROJECT_ID}/issues?private_token=${BOT_ACCESS_TOKEN}&title=Error%20in%20${CI_PROJECT_NAME}%20with%20pipeline%20${CI_PIPELINE_ID}%20for%20commit%20${CI_COMMIT_SHA}&labels&description=${CI_PIPELINE_URL}"
   when: on_failure
   only:

From 19f41d94323d2bcb9feed4c71d58b0f935a9fd3b Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 11 Nov 2020 10:41:28 +0100
Subject: [PATCH 0158/1008] Add DPC++ NOT_IMPLEMENTED IDR kernels.

---
 core/solver/idr.cpp             |   1 -
 core/solver/idr_kernels.hpp     |   9 +++
 dpcpp/CMakeLists.txt            |   1 +
 dpcpp/solver/idr_kernels.dp.cpp | 115 ++++++++++++++++++++++++++++++++
 4 files changed, 125 insertions(+), 1 deletion(-)
 create mode 100644 dpcpp/solver/idr_kernels.dp.cpp

diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index f7300f00ea6..ae583f5ee60 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -40,7 +40,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/utils.hpp>
 
 
-#include <variant>
 #include "core/components/fill_array.hpp"
 #include "core/solver/idr_kernels.hpp"
 
diff --git a/core/solver/idr_kernels.hpp b/core/solver/idr_kernels.hpp
index 29080c36fe6..fa1cca81b08 100644
--- a/core/solver/idr_kernels.hpp
+++ b/core/solver/idr_kernels.hpp
@@ -141,6 +141,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace idr {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace idr
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 10f8668dbe2..d3fe25ceb10 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -37,6 +37,7 @@ target_sources(ginkgo_dpcpp
     solver/cgs_kernels.dp.cpp
     solver/fcg_kernels.dp.cpp
     solver/gmres_kernels.dp.cpp
+    solver/idr_kernels.dp.cpp
     solver/ir_kernels.dp.cpp
     solver/lower_trs_kernels.dp.cpp
     solver/upper_trs_kernels.dp.cpp
diff --git a/dpcpp/solver/idr_kernels.dp.cpp b/dpcpp/solver/idr_kernels.dp.cpp
new file mode 100644
index 00000000000..d585de0cc9c
--- /dev/null
+++ b/dpcpp/solver/idr_kernels.dp.cpp
@@ -0,0 +1,115 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/idr_kernels.hpp"
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/stopping_status.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The IDR solver namespace.
+ *
+ * @ingroup idr
+ */
+namespace idr {
+
+
+template <typename ValueType>
+void initialize(std::shared_ptr<const DpcppExecutor> exec,
+                matrix::Dense<ValueType> *m,
+                matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
+                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
+
+
+template <typename ValueType>
+void step_1(std::shared_ptr<const DpcppExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *m,
+            const matrix::Dense<ValueType> *f,
+            const matrix::Dense<ValueType> *residual,
+            const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *c,
+            matrix::Dense<ValueType> *v,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
+
+
+template <typename ValueType>
+void step_2(std::shared_ptr<const DpcppExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *omega,
+            const matrix::Dense<ValueType> *preconditioned_vector,
+            const matrix::Dense<ValueType> *c, matrix::Dense<ValueType> *u,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
+
+
+template <typename ValueType>
+void step_3(std::shared_ptr<const DpcppExecutor> exec, const size_type k,
+            const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
+            matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *u,
+            matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
+            matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *residual,
+            matrix::Dense<ValueType> *x,
+            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
+
+
+template <typename ValueType>
+void compute_omega(
+    std::shared_ptr<const DpcppExecutor> exec,
+    const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
+    const matrix::Dense<remove_complex<ValueType>> *residual_norm,
+    matrix::Dense<ValueType> *omega,
+    const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
+
+
+}  // namespace idr
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko

From 97bc0cfda8b10b3185d3896a655355f77b3e8463 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 12 Nov 2020 11:21:05 +0100
Subject: [PATCH 0159/1008] Remove make_diag_dominant from CUDA test

---
 cuda/test/solver/idr_kernels.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index 67cbd886f66..24c25d174ce 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -69,7 +69,6 @@ class Idr : public ::testing::Test {
         cuda = gko::CudaExecutor::create(0, ref);
 
         mtx = gen_mtx(123, 123);
-        make_diag_dominant(mtx.get());
         d_mtx = Mtx::create(cuda);
         d_mtx->copy_from(mtx.get());
         cuda_idr_factory =

From 5f8427c832fc450ebcee0070fb9c3c36ff1634a8 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Fri, 13 Nov 2020 12:41:06 +0100
Subject: [PATCH 0160/1008] Add math header to curand and hiprand bindings, use
 hiprand image for subdir_build

---
 .gitlab-ci.yml                    | 2 +-
 cuda/base/curand_bindings.hpp     | 1 +
 hip/base/hiprand_bindings.hip.hpp | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b1ae57ba935..c2fe2fba806 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -725,7 +725,7 @@ no-circular-deps:
 subdir-build:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
diff --git a/cuda/base/curand_bindings.hpp b/cuda/base/curand_bindings.hpp
index 219994f2a5b..033f0bd8e4c 100644
--- a/cuda/base/curand_bindings.hpp
+++ b/cuda/base/curand_bindings.hpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 
 
+#include "cuda/base/math.hpp"
 #include "cuda/base/types.hpp"
 
 
diff --git a/hip/base/hiprand_bindings.hip.hpp b/hip/base/hiprand_bindings.hip.hpp
index 86550f6aecd..83534627ca5 100644
--- a/hip/base/hiprand_bindings.hip.hpp
+++ b/hip/base/hiprand_bindings.hip.hpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 
 
+#include "hip/base/math.hip.hpp"
 #include "hip/base/types.hip.hpp"
 
 

From 1f5161d70ce3da73f55882f8f5f776c1aefd474e Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 19 Nov 2020 15:58:57 +0100
Subject: [PATCH 0161/1008] Suggestions from code review

---
 core/matrix/dense.cpp                      |  33 -----
 core/solver/idr.cpp                        |  21 +--
 core/solver/idr_kernels.hpp                |  38 +++---
 core/stop/residual_norm.cpp                |   6 +-
 core/test/base/exception.cpp               |   8 ++
 core/test/solver/idr.cpp                   | 152 +++++++++++++++++++++
 cuda/base/exception.cpp                    |   1 +
 cuda/solver/idr_kernels.cu                 |  58 ++++----
 cuda/test/solver/CMakeLists.txt            |   4 +-
 cuda/test/solver/idr_kernels.cpp           |  51 +++----
 hip/CMakeLists.txt                         |   1 +
 hip/base/exception.hip.cpp                 |   1 +
 hip/base/types.hip.hpp                     |   2 +
 hip/solver/idr_kernels.hip.cpp             |  56 ++++----
 hip/test/solver/idr_kernels.cpp            |  51 +++----
 include/ginkgo/core/base/array.hpp         |  19 ---
 include/ginkgo/core/solver/idr.hpp         |  56 +++++++-
 include/ginkgo/core/stop/residual_norm.hpp |  14 +-
 omp/solver/idr_kernels.cpp                 |  43 +++---
 omp/stop/residual_norm_kernels.cpp         |   1 +
 omp/test/solver/gmres_kernels.cpp          |   2 +-
 omp/test/solver/idr_kernels.cpp            |  51 +++----
 reference/solver/idr_kernels.cpp           |  45 +++---
 reference/stop/residual_norm_kernels.cpp   |   1 +
 24 files changed, 432 insertions(+), 283 deletions(-)

diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 06f680a1857..a9e6a3a1515 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -215,39 +215,6 @@ inline void conversion_helper(SparsityCsr<ValueType, IndexType> *result,
 
 }  // namespace
 
-template <typename ValueType>
-std::unique_ptr<matrix::Dense<ValueType>> interpret_as_real(
-    matrix::Dense<to_complex<ValueType>> *source)
-{
-    auto exec = source->get_executor();
-    auto rows = source->get_size()[0];
-    auto cols = 2 * source->get_size()[1];
-    auto stride = 2 * source->get_stride();
-    return matrix::Dense<ValueType>::create(
-        exec, gko::dim<2>{rows, cols},
-        gko::Array<ValueType>::view(
-            exec, rows * stride,
-            reinterpret_cast<ValueType *>(source->get_values())),
-        stride);
-}
-
-template <typename ValueType>
-std::unique_ptr<matrix::Dense<ValueType>> interpret_as_real(
-    const matrix::Dense<to_complex<ValueType>> *source)
-{
-    auto exec = source->get_executor();
-    auto rows = source->get_size()[0];
-    auto cols = 2 * source->get_size()[1];
-    auto stride = 2 * source->get_stride();
-    return matrix::Dense<ValueType>::create(
-        exec, gko::dim<2>{rows, cols},
-        gko::Array<ValueType>::view(
-            exec, rows * stride,
-            const_cast<ValueType *>(
-                reinterpret_cast<ValueType *>(source->get_const_values()))),
-        stride);
-}
-
 
 template <typename ValueType>
 void Dense<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index ae583f5ee60..43c6faaeff4 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/fill_array.hpp"
 #include "core/solver/idr_kernels.hpp"
 
+
 namespace gko {
 namespace solver {
 namespace idr {
@@ -143,7 +144,7 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
 
     // Initialization
     // m = identity
-    exec->run(idr::make_initialize(m.get(), subspace_vectors.get(),
+    exec->run(idr::make_initialize(nrhs, m.get(), subspace_vectors.get(),
                                    deterministic_, &stop_status));
 
     // omega = 1
@@ -186,16 +187,16 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
         // f = P^H * residual
 
         for (size_type k = 0; k < subspace_dim_; k++) {
-            exec->run(idr::make_step_1(k, m.get(), f.get(), residual.get(),
-                                       g.get(), c.get(), v.get(),
-                                       &stop_status));
+            exec->run(idr::make_step_1(nrhs, k, m.get(), f.get(),
+                                       residual.get(), g.get(), c.get(),
+                                       v.get(), &stop_status));
             // c = M \ f = (c_1, ..., c_s)^T
             // v = residual - c_k * g_k - ... - c_s * g_s
 
             get_preconditioner()->apply(v.get(), helper.get());
 
-            exec->run(idr::make_step_2(k, omega.get(), helper.get(), c.get(),
-                                       u.get(), &stop_status));
+            exec->run(idr::make_step_2(nrhs, k, omega.get(), helper.get(),
+                                       c.get(), u.get(), &stop_status));
             // u_k = omega * preconditioned_vector + c_k * u_k + ... + c_s * u_s
 
             auto u_k = u->create_submatrix(span{0, problem_size},
@@ -204,7 +205,7 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
             system_matrix_->apply(u_k.get(), helper.get());
             // g_k = Au_k
 
-            exec->run(idr::make_step_3(k, subspace_vectors.get(), g.get(),
+            exec->run(idr::make_step_3(nrhs, k, subspace_vectors.get(), g.get(),
                                        helper.get(), u.get(), m.get(), f.get(),
                                        alpha.get(), residual.get(), dense_x,
                                        &stop_status));
@@ -226,12 +227,12 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
         system_matrix_->apply(helper.get(), t.get());
 
         t->compute_dot(residual.get(), omega.get());
-        // t->compute_norm2(t_norm.get());
         t->compute_dot(t.get(), tht.get());
         residual->compute_norm2(residual_norm.get());
 
-        exec->run(idr::make_compute_omega(
-            kappa_, tht.get(), residual_norm.get(), omega.get(), &stop_status));
+        exec->run(idr::make_compute_omega(nrhs, kappa_, tht.get(),
+                                          residual_norm.get(), omega.get(),
+                                          &stop_status));
 
         t->scale(subspace_neg_one_op.get());
         residual->add_scaled(omega.get(), t.get());
diff --git a/core/solver/idr_kernels.hpp b/core/solver/idr_kernels.hpp
index fa1cca81b08..7f2e00bae45 100644
--- a/core/solver/idr_kernels.hpp
+++ b/core/solver/idr_kernels.hpp
@@ -46,43 +46,45 @@ namespace kernels {
 namespace idr {
 
 
-#define GKO_DECLARE_IDR_INITIALIZE_KERNEL(_type)                 \
-    void initialize(std::shared_ptr<const DefaultExecutor> exec, \
-                    matrix::Dense<_type> *m,                     \
-                    matrix::Dense<_type> *subspace_vectors,      \
+#define GKO_DECLARE_IDR_INITIALIZE_KERNEL(_type)                   \
+    void initialize(std::shared_ptr<const DefaultExecutor> exec,   \
+                    const size_type nrhs, matrix::Dense<_type> *m, \
+                    matrix::Dense<_type> *subspace_vectors,        \
                     bool deterministic, Array<stopping_status> *stop_status)
 
 
 #define GKO_DECLARE_IDR_STEP_1_KERNEL(_type)                                 \
     void step_1(                                                             \
-        std::shared_ptr<const DefaultExecutor> exec, const size_type k,      \
-        const matrix::Dense<_type> *m, const matrix::Dense<_type> *f,        \
-        const matrix::Dense<_type> *residual, const matrix::Dense<_type> *g, \
-        matrix::Dense<_type> *c, matrix::Dense<_type> *v,                    \
-        const Array<stopping_status> *stop_status)
+        std::shared_ptr<const DefaultExecutor> exec, const size_type nrhs,   \
+        const size_type k, const matrix::Dense<_type> *m,                    \
+        const matrix::Dense<_type> *f, const matrix::Dense<_type> *residual, \
+        const matrix::Dense<_type> *g, matrix::Dense<_type> *c,              \
+        matrix::Dense<_type> *v, const Array<stopping_status> *stop_status)
 
 
 #define GKO_DECLARE_IDR_STEP_2_KERNEL(_type)                            \
     void step_2(std::shared_ptr<const DefaultExecutor> exec,            \
-                const size_type k, const matrix::Dense<_type> *omega,   \
+                const size_type nrhs, const size_type k,                \
+                const matrix::Dense<_type> *omega,                      \
                 const matrix::Dense<_type> *preconditioned_vector,      \
                 const matrix::Dense<_type> *c, matrix::Dense<_type> *u, \
                 const Array<stopping_status> *stop_status)
 
 
-#define GKO_DECLARE_IDR_STEP_3_KERNEL(_type)                             \
-    void step_3(std::shared_ptr<const DefaultExecutor> exec,             \
-                const size_type k, const matrix::Dense<_type> *p,        \
-                matrix::Dense<_type> *g, matrix::Dense<_type> *g_k,      \
-                matrix::Dense<_type> *u, matrix::Dense<_type> *m,        \
-                matrix::Dense<_type> *f, matrix::Dense<_type> *alpha,    \
-                matrix::Dense<_type> *residual, matrix::Dense<_type> *x, \
+#define GKO_DECLARE_IDR_STEP_3_KERNEL(_type)                                 \
+    void step_3(std::shared_ptr<const DefaultExecutor> exec,                 \
+                const size_type nrhs, const size_type k,                     \
+                const matrix::Dense<_type> *p, matrix::Dense<_type> *g,      \
+                matrix::Dense<_type> *g_k, matrix::Dense<_type> *u,          \
+                matrix::Dense<_type> *m, matrix::Dense<_type> *f,            \
+                matrix::Dense<_type> *alpha, matrix::Dense<_type> *residual, \
+                matrix::Dense<_type> *x,                                     \
                 const Array<stopping_status> *stop_status)
 
 
 #define GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL(_type)                         \
     void compute_omega(                                                     \
-        std::shared_ptr<const DefaultExecutor> exec,                        \
+        std::shared_ptr<const DefaultExecutor> exec, const size_type nrhs,  \
         const remove_complex<_type> kappa, const matrix::Dense<_type> *tht, \
         const matrix::Dense<remove_complex<_type>> *residual_norm,          \
         matrix::Dense<_type> *omega,                                        \
diff --git a/core/stop/residual_norm.cpp b/core/stop/residual_norm.cpp
index 877a2eb238c..a678226a257 100644
--- a/core/stop/residual_norm.cpp
+++ b/core/stop/residual_norm.cpp
@@ -59,10 +59,8 @@ bool ResidualNorm<ValueType>::check_impl(uint8 stoppingId, bool setFinalized,
     if (updater.residual_norm_ != nullptr) {
         dense_tau = as<NormVector>(updater.residual_norm_);
     } else if (updater.residual_ != nullptr) {
-        if (dynamic_cast<const matrix::Dense<to_complex<ValueType>> *>(
-                updater.residual_)) {
-            auto *dense_r =
-                as<matrix::Dense<to_complex<ValueType>>>(updater.residual_);
+        if (dynamic_cast<const ComplexVector *>(updater.residual_)) {
+            auto *dense_r = as<ComplexVector>(updater.residual_);
             dense_r->compute_norm2(u_dense_tau_.get());
         } else {
             auto *dense_r = as<Vector>(updater.residual_);
diff --git a/core/test/base/exception.cpp b/core/test/base/exception.cpp
index f1fb831f871..de0fbb1bdd7 100644
--- a/core/test/base/exception.cpp
+++ b/core/test/base/exception.cpp
@@ -122,6 +122,14 @@ TEST(ExceptionClasses, HipblasErrorReturnsCorrectWhatMessage)
 }
 
 
+TEST(ExceptionClasses, HiprandErrorReturnsCorrectWhatMessage)
+{
+    gko::HipblasError error("test_file.cpp", 123, "test_func", 1);
+    std::string expected = "test_file.cpp:123: test_func: ";
+    ASSERT_EQ(expected, std::string(error.what()).substr(0, expected.size()));
+}
+
+
 TEST(ExceptionClasses, HipsparseErrorReturnsCorrectWhatMessage)
 {
     gko::HipsparseError error("test_file.cpp", 123, "test_func", 1);
diff --git a/core/test/solver/idr.cpp b/core/test/solver/idr.cpp
index c4f5af1ef8a..1eb60aad03a 100644
--- a/core/test/solver/idr.cpp
+++ b/core/test/solver/idr.cpp
@@ -142,6 +142,7 @@ TYPED_TEST(Idr, CanBeCloned)
 {
     using Mtx = typename TestFixture::Mtx;
     using Solver = typename TestFixture::Solver;
+
     auto clone = this->solver->clone();
 
     ASSERT_EQ(clone->get_size(), gko::dim<2>(3, 3));
@@ -154,6 +155,7 @@ TYPED_TEST(Idr, CanBeCloned)
 TYPED_TEST(Idr, CanBeCleared)
 {
     using Solver = typename TestFixture::Solver;
+
     this->solver->clear();
 
     ASSERT_EQ(this->solver->get_size(), gko::dim<2>(0, 0));
@@ -291,4 +293,154 @@ TYPED_TEST(Idr, CanSetPreconditioner)
 }
 
 
+TYPED_TEST(Idr, CanSetSubspaceDim)
+{
+    using Solver = typename TestFixture::Solver;
+    using value_type = typename TestFixture::value_type;
+    auto idr_factory =
+        Solver::build()
+            .with_subspace_dim(8u)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(4u).on(this->exec))
+            .on(this->exec);
+    auto solver = idr_factory->generate(this->mtx);
+    auto subspace_dim = solver->get_subspace_dim();
+
+    ASSERT_EQ(subspace_dim, 8u);
+}
+
+
+TYPED_TEST(Idr, CanSetSubspaceDimAgain)
+{
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<gko::stop::CriterionFactory> init_crit =
+        gko::stop::Iteration::build().with_max_iters(3u).on(this->exec);
+    auto idr_factory =
+        Solver::build().with_criteria(init_crit).with_subspace_dim(10u).on(
+            this->exec);
+
+    ASSERT_EQ(idr_factory->get_parameters().subspace_dim, 10);
+
+    auto solver = idr_factory->generate(this->mtx);
+
+    solver->set_subspace_dim(20);
+
+    ASSERT_EQ(solver->get_subspace_dim(), 20);
+}
+
+
+TYPED_TEST(Idr, CanSetKappa)
+{
+    using Solver = typename TestFixture::Solver;
+    using value_type = typename TestFixture::value_type;
+    using real_type = gko::remove_complex<value_type>;
+    auto idr_factory =
+        Solver::build()
+            .with_kappa(real_type{0.05})
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(4u).on(this->exec))
+            .on(this->exec);
+    auto solver = idr_factory->generate(this->mtx);
+    auto kappa = solver->get_kappa();
+
+    ASSERT_EQ(kappa, real_type{0.05});
+}
+
+
+TYPED_TEST(Idr, CanSetKappaAgain)
+{
+    using Solver = typename TestFixture::Solver;
+    using value_type = typename TestFixture::value_type;
+    using real_type = gko::remove_complex<value_type>;
+    std::shared_ptr<gko::stop::CriterionFactory> init_crit =
+        gko::stop::Iteration::build().with_max_iters(3u).on(this->exec);
+    auto idr_factory = Solver::build()
+                           .with_criteria(init_crit)
+                           .with_kappa(real_type{0.05})
+                           .on(this->exec);
+
+    ASSERT_EQ(idr_factory->get_parameters().kappa, real_type{0.05});
+
+    auto solver = idr_factory->generate(this->mtx);
+
+    solver->set_kappa(real_type{0.3});
+
+    ASSERT_EQ(solver->get_kappa(), real_type{0.3});
+}
+
+
+TYPED_TEST(Idr, CanSetDeterministic)
+{
+    using Solver = typename TestFixture::Solver;
+    using value_type = typename TestFixture::value_type;
+    auto idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(4u).on(this->exec))
+            .on(this->exec);
+    auto solver = idr_factory->generate(this->mtx);
+    auto deterministic = solver->get_deterministic();
+
+    ASSERT_EQ(deterministic, true);
+}
+
+
+TYPED_TEST(Idr, CanSetDeterministicAgain)
+{
+    using Solver = typename TestFixture::Solver;
+    using value_type = typename TestFixture::value_type;
+    std::shared_ptr<gko::stop::CriterionFactory> init_crit =
+        gko::stop::Iteration::build().with_max_iters(3u).on(this->exec);
+    auto idr_factory =
+        Solver::build().with_criteria(init_crit).with_deterministic(true).on(
+            this->exec);
+
+    ASSERT_EQ(idr_factory->get_parameters().deterministic, true);
+
+    auto solver = idr_factory->generate(this->mtx);
+
+    solver->set_deterministic(false);
+
+    ASSERT_EQ(solver->get_deterministic(), false);
+}
+
+
+TYPED_TEST(Idr, CanSetComplexSubspace)
+{
+    using Solver = typename TestFixture::Solver;
+    using value_type = typename TestFixture::value_type;
+    auto idr_factory =
+        Solver::build()
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(4u).on(this->exec))
+            .on(this->exec);
+    auto solver = idr_factory->generate(this->mtx);
+    auto complex_subspace = solver->get_complex_subspace();
+
+    ASSERT_EQ(complex_subspace, true);
+}
+
+
+TYPED_TEST(Idr, CanSetComplexSubspaceAgain)
+{
+    using Solver = typename TestFixture::Solver;
+    using value_type = typename TestFixture::value_type;
+    std::shared_ptr<gko::stop::CriterionFactory> init_crit =
+        gko::stop::Iteration::build().with_max_iters(3u).on(this->exec);
+    auto idr_factory =
+        Solver::build().with_criteria(init_crit).with_complex_subspace(true).on(
+            this->exec);
+
+    ASSERT_EQ(idr_factory->get_parameters().complex_subspace, true);
+
+    auto solver = idr_factory->generate(this->mtx);
+
+    solver->set_complex_subpsace(false);
+
+    ASSERT_EQ(solver->get_complex_subspace(), false);
+}
+
+
 }  // namespace
diff --git a/cuda/base/exception.cpp b/cuda/base/exception.cpp
index e32f6a9eebf..e4acaa451a6 100644
--- a/cuda/base/exception.cpp
+++ b/cuda/base/exception.cpp
@@ -92,6 +92,7 @@ std::string CurandError::get_error(int64 error_code)
     GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_TYPE_ERROR);
     GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_OUT_OF_RANGE);
     GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_LENGTH_NOT_MULTIPLE);
+    GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_DOUBLE_PRECISION_REQUIRED);
     GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_LAUNCH_FAILURE);
     GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_PREEXISTING_FAILURE);
     GKO_REGISTER_CURAND_ERROR(CURAND_STATUS_INITIALIZATION_FAILED);
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index dc72bc0eb5b..83f018c4330 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -76,11 +76,10 @@ namespace {
 
 
 template <typename ValueType>
-void initialize_m(matrix::Dense<ValueType> *m,
+void initialize_m(const size_type nrhs, matrix::Dense<ValueType> *m,
                   Array<stopping_status> *stop_status)
 {
     const auto subspace_dim = m->get_size()[0];
-    const auto nrhs = m->get_size()[1] / subspace_dim;
     const auto m_stride = m->get_stride();
 
     const auto grid_dim = ceildiv(m_stride * subspace_dim, default_block_size);
@@ -122,13 +121,13 @@ void orthonormalize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors)
 
 
 template <typename ValueType>
-void solve_lower_triangular(const matrix::Dense<ValueType> *m,
+void solve_lower_triangular(const size_type nrhs,
+                            const matrix::Dense<ValueType> *m,
                             const matrix::Dense<ValueType> *f,
                             matrix::Dense<ValueType> *c,
                             const Array<stopping_status> *stop_status)
 {
     const auto subspace_dim = m->get_size()[0];
-    const auto nrhs = m->get_size()[1] / subspace_dim;
 
     const auto grid_dim = ceildiv(nrhs, default_block_size);
     solve_lower_triangular_kernel<<<grid_dim, default_block_size>>>(
@@ -140,7 +139,8 @@ void solve_lower_triangular(const matrix::Dense<ValueType> *m,
 
 
 template <typename ValueType>
-void update_g_and_u(std::shared_ptr<const CudaExecutor> exec, size_type k,
+void update_g_and_u(std::shared_ptr<const CudaExecutor> exec,
+                    const size_type nrhs, const size_type k,
                     const matrix::Dense<ValueType> *p,
                     const matrix::Dense<ValueType> *m,
                     matrix::Dense<ValueType> *alpha,
@@ -149,7 +149,6 @@ void update_g_and_u(std::shared_ptr<const CudaExecutor> exec, size_type k,
                     const Array<stopping_status> *stop_status)
 {
     const auto size = g->get_size()[0];
-    const auto nrhs = m->get_size()[1] / m->get_size()[0];
     const auto p_stride = p->get_stride();
 
     const dim3 grid_dim(ceildiv(nrhs, default_dot_dim),
@@ -190,14 +189,13 @@ void update_g_and_u(std::shared_ptr<const CudaExecutor> exec, size_type k,
 
 
 template <typename ValueType>
-void update_m(std::shared_ptr<const CudaExecutor> exec, size_type k,
-              const matrix::Dense<ValueType> *p,
+void update_m(std::shared_ptr<const CudaExecutor> exec, const size_type nrhs,
+              const size_type k, const matrix::Dense<ValueType> *p,
               const matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *m,
               const Array<stopping_status> *stop_status)
 {
     const auto size = g_k->get_size()[0];
     const auto subspace_dim = m->get_size()[0];
-    const auto nrhs = m->get_size()[1] / m->get_size()[0];
     const auto p_stride = p->get_stride();
     const auto m_stride = m->get_stride();
 
@@ -223,7 +221,8 @@ void update_m(std::shared_ptr<const CudaExecutor> exec, size_type k,
 
 
 template <typename ValueType>
-void update_x_r_and_f(std::shared_ptr<const CudaExecutor> exec, size_type k,
+void update_x_r_and_f(std::shared_ptr<const CudaExecutor> exec,
+                      const size_type nrhs, const size_type k,
                       const matrix::Dense<ValueType> *m,
                       const matrix::Dense<ValueType> *g,
                       const matrix::Dense<ValueType> *u,
@@ -233,7 +232,6 @@ void update_x_r_and_f(std::shared_ptr<const CudaExecutor> exec, size_type k,
 {
     const auto size = x->get_size()[0];
     const auto subspace_dim = m->get_size()[0];
-    const auto nrhs = x->get_size()[1];
 
     const auto grid_dim = ceildiv(size * x->get_stride(), default_block_size);
     update_x_r_and_f_kernel<<<grid_dim, default_block_size>>>(
@@ -253,12 +251,12 @@ void update_x_r_and_f(std::shared_ptr<const CudaExecutor> exec, size_type k,
 
 
 template <typename ValueType>
-void initialize(std::shared_ptr<const CudaExecutor> exec,
+void initialize(std::shared_ptr<const CudaExecutor> exec, const size_type nrhs,
                 matrix::Dense<ValueType> *m,
                 matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
                 Array<stopping_status> *stop_status)
 {
-    initialize_m(m, stop_status);
+    initialize_m(nrhs, m, stop_status);
     initialize_subspace_vectors(subspace_vectors, deterministic);
     orthonormalize_subspace_vectors(subspace_vectors);
 }
@@ -267,19 +265,18 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const CudaExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *m,
+void step_1(std::shared_ptr<const CudaExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *m,
             const matrix::Dense<ValueType> *f,
             const matrix::Dense<ValueType> *residual,
             const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *c,
             matrix::Dense<ValueType> *v,
             const Array<stopping_status> *stop_status)
 {
-    solve_lower_triangular(m, f, c, stop_status);
+    solve_lower_triangular(nrhs, m, f, c, stop_status);
 
     const auto num_rows = v->get_size()[0];
     const auto subspace_dim = m->get_size()[0];
-    const auto nrhs = m->get_size()[1] / subspace_dim;
 
     const auto grid_dim = ceildiv(nrhs * num_rows, default_block_size);
     step_1_kernel<<<grid_dim, default_block_size>>>(
@@ -295,14 +292,13 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const CudaExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *omega,
+void step_2(std::shared_ptr<const CudaExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *omega,
             const matrix::Dense<ValueType> *preconditioned_vector,
             const matrix::Dense<ValueType> *c, matrix::Dense<ValueType> *u,
             const Array<stopping_status> *stop_status)
 {
     const auto num_rows = preconditioned_vector->get_size()[0];
-    const auto nrhs = preconditioned_vector->get_size()[1];
     const auto subspace_dim = u->get_size()[1] / nrhs;
 
     const auto grid_dim = ceildiv(nrhs * num_rows, default_block_size);
@@ -320,17 +316,17 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 
 
 template <typename ValueType>
-void step_3(std::shared_ptr<const CudaExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
-            matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *u,
-            matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
-            matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *residual,
-            matrix::Dense<ValueType> *x,
+void step_3(std::shared_ptr<const CudaExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *p,
+            matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *g_k,
+            matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
+            matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *alpha,
+            matrix::Dense<ValueType> *residual, matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status)
 {
-    update_g_and_u(exec, k, p, m, alpha, g, g_k, u, stop_status);
-    update_m(exec, k, p, g_k, m, stop_status);
-    update_x_r_and_f(exec, k, m, g, u, f, residual, x, stop_status);
+    update_g_and_u(exec, nrhs, k, p, m, alpha, g, g_k, u, stop_status);
+    update_m(exec, nrhs, k, p, g_k, m, stop_status);
+    update_x_r_and_f(exec, nrhs, k, m, g, u, f, residual, x, stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
@@ -338,13 +334,11 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
 template <typename ValueType>
 void compute_omega(
-    std::shared_ptr<const CudaExecutor> exec,
+    std::shared_ptr<const CudaExecutor> exec, const size_type nrhs,
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
     matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
 {
-    const auto nrhs = omega->get_size()[1];
-
     const auto grid_dim = ceildiv(nrhs, config::warp_size);
     compute_omega_kernel<<<grid_dim, config::warp_size>>>(
         nrhs, kappa, as_cuda_type(tht->get_const_values()),
diff --git a/cuda/test/solver/CMakeLists.txt b/cuda/test/solver/CMakeLists.txt
index 56ad6f7d97d..f4f671add8f 100644
--- a/cuda/test/solver/CMakeLists.txt
+++ b/cuda/test/solver/CMakeLists.txt
@@ -2,9 +2,9 @@ ginkgo_create_test(bicg_kernels)
 ginkgo_create_test(bicgstab_kernels)
 ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
-ginkgo_create_test_cpp_cuda_header(lower_trs_kernels)
-ginkgo_create_test_cpp_cuda_header(upper_trs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
 ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
+ginkgo_create_test_cpp_cuda_header(lower_trs_kernels)
+ginkgo_create_test_cpp_cuda_header(upper_trs_kernels)
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index 24c25d174ce..1fd6aa11c06 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -104,7 +104,7 @@ class Idr : public ::testing::Test {
     void initialize_data()
     {
         int size = 597;
-        int nrhs = 17;
+        nrhs = 17;
         int s = 4;
         x = gen_mtx(size, nrhs);
         b = gen_mtx(size, nrhs);
@@ -116,6 +116,7 @@ class Idr : public ::testing::Test {
         c = gen_mtx(s, nrhs);
         v = gen_mtx(size, nrhs);
         p = gen_mtx(s, size);
+        alpha = gen_mtx(1, nrhs);
         omega = gen_mtx(1, nrhs);
         tht = gen_mtx(1, nrhs);
         residual_norm = gen_mtx(1, nrhs);
@@ -135,6 +136,7 @@ class Idr : public ::testing::Test {
         d_c = Mtx::create(cuda);
         d_v = Mtx::create(cuda);
         d_p = Mtx::create(cuda);
+        d_alpha = Mtx::create(cuda);
         d_omega = Mtx::create(cuda);
         d_tht = Mtx::create(cuda);
         d_residual_norm = Mtx::create(cuda);
@@ -151,6 +153,7 @@ class Idr : public ::testing::Test {
         d_c->copy_from(c.get());
         d_v->copy_from(v.get());
         d_p->copy_from(p.get());
+        d_alpha->copy_from(alpha.get());
         d_omega->copy_from(omega.get());
         d_tht->copy_from(tht.get());
         d_residual_norm->copy_from(residual_norm.get());
@@ -180,7 +183,9 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Solver::Factory> cuda_idr_factory;
     std::unique_ptr<Solver::Factory> ref_idr_factory;
 
-    std::unique_ptr<Mtx> x;
+    gko::size_type nrhs,
+
+        std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
     std::unique_ptr<Mtx> m;
@@ -190,6 +195,7 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> c;
     std::unique_ptr<Mtx> v;
     std::unique_ptr<Mtx> p;
+    std::unique_ptr<Mtx> alpha;
     std::unique_ptr<Mtx> omega;
     std::unique_ptr<Mtx> tht;
     std::unique_ptr<Mtx> residual_norm;
@@ -205,6 +211,7 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> d_c;
     std::unique_ptr<Mtx> d_v;
     std::unique_ptr<Mtx> d_p;
+    std::unique_ptr<Mtx> d_alpha;
     std::unique_ptr<Mtx> d_omega;
     std::unique_ptr<Mtx> d_tht;
     std::unique_ptr<Mtx> d_residual_norm;
@@ -216,9 +223,9 @@ TEST_F(Idr, IdrInitializeIsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::idr::initialize(ref, m.get(), p.get(), true,
+    gko::kernels::reference::idr::initialize(ref, nrhs, m.get(), p.get(), true,
                                              stop_status.get());
-    gko::kernels::cuda::idr::initialize(cuda, d_m.get(), d_p.get(), true,
+    gko::kernels::cuda::idr::initialize(cuda, nrhs, d_m.get(), d_p.get(), true,
                                         d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
@@ -231,11 +238,11 @@ TEST_F(Idr, IdrStep1IsEquivalentToRef)
     initialize_data();
 
     gko::size_type k = 2;
-    gko::kernels::reference::idr::step_1(ref, k, m.get(), f.get(), r.get(),
-                                         g.get(), c.get(), v.get(),
+    gko::kernels::reference::idr::step_1(ref, nrhs, k, m.get(), f.get(),
+                                         r.get(), g.get(), c.get(), v.get(),
                                          stop_status.get());
-    gko::kernels::cuda::idr::step_1(cuda, k, d_m.get(), d_f.get(), d_r.get(),
-                                    d_g.get(), d_c.get(), d_v.get(),
+    gko::kernels::cuda::idr::step_1(cuda, nrhs, k, d_m.get(), d_f.get(),
+                                    d_r.get(), d_g.get(), d_c.get(), d_v.get(),
                                     d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(c, d_c, 1e-14);
@@ -248,9 +255,9 @@ TEST_F(Idr, IdrStep2IsEquivalentToRef)
     initialize_data();
 
     gko::size_type k = 2;
-    gko::kernels::reference::idr::step_2(ref, k, omega.get(), v.get(), c.get(),
-                                         u.get(), stop_status.get());
-    gko::kernels::cuda::idr::step_2(cuda, k, d_omega.get(), d_v.get(),
+    gko::kernels::reference::idr::step_2(ref, nrhs, k, omega.get(), v.get(),
+                                         c.get(), u.get(), stop_status.get());
+    gko::kernels::cuda::idr::step_2(cuda, nrhs, k, d_omega.get(), d_v.get(),
                                     d_c.get(), d_u.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
@@ -262,14 +269,15 @@ TEST_F(Idr, IdrStep3IsEquivalentToRef)
     initialize_data();
 
     gko::size_type k = 2;
-    gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), v.get(),
-                                         u.get(), m.get(), f.get(), omega.get(),
-                                         r.get(), x.get(), stop_status.get());
+    gko::kernels::reference::idr::step_3(
+        ref, nrhs, k, p.get(), g.get(), v.get(), u.get(), m.get(), f.get(),
+        alpha.get(), r.get(), x.get(), stop_status.get());
     gko::kernels::cuda::idr::step_3(
-        cuda, k, d_p.get(), d_g.get(), d_v.get(), d_u.get(), d_m.get(),
-        d_f.get(), d_omega.get(), d_r.get(), d_x.get(), d_stop_status.get());
+        cuda, nrhs, k, d_p.get(), d_g.get(), d_v.get(), d_u.get(), d_m.get(),
+        d_f.get(), d_alpha.get(), d_r.get(), d_x.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(g, d_g, 1e-14);
+    GKO_ASSERT_MTX_NEAR(v, d_v, 1e-14);
     GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
     GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
     GKO_ASSERT_MTX_NEAR(f, d_f, 1e-14);
@@ -283,10 +291,10 @@ TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
     initialize_data();
 
     double kappa = 0.7;
-    gko::kernels::reference::idr::compute_omega(ref, kappa, tht.get(),
+    gko::kernels::reference::idr::compute_omega(ref, nrhs, kappa, tht.get(),
                                                 residual_norm.get(),
                                                 omega.get(), stop_status.get());
-    gko::kernels::cuda::idr::compute_omega(cuda, kappa, d_tht.get(),
+    gko::kernels::cuda::idr::compute_omega(cuda, nrhs, kappa, d_tht.get(),
                                            d_residual_norm.get(), d_omega.get(),
                                            d_stop_status.get());
 
@@ -298,7 +306,6 @@ TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 1;
-
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto cuda_solver = cuda_idr_factory->generate(d_mtx);
     auto b = gen_mtx(m, n);
@@ -320,7 +327,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 1;
-
     cuda_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -328,7 +334,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(1u).on(cuda))
             .on(cuda);
-
     ref_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -336,7 +341,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(1u).on(ref))
             .on(ref);
-
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto cuda_solver = cuda_idr_factory->generate(d_mtx);
     auto b = gen_mtx(m, n);
@@ -379,7 +383,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 16;
-
     cuda_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -387,7 +390,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(1u).on(cuda))
             .on(cuda);
-
     ref_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -395,7 +397,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(1u).on(ref))
             .on(ref);
-
     auto cuda_solver = cuda_idr_factory->generate(d_mtx);
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto b = gen_mtx(m, n);
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 486f550ea27..634c7eb7dee 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -155,6 +155,7 @@ find_package(HIP REQUIRED)
 find_package(hipblas REQUIRED)
 find_package(hiprand REQUIRED)
 find_package(hipsparse REQUIRED)
+# At the moment, for hiprand to work also rocrand is required.
 find_package(rocrand REQUIRED)
 find_path(GINKGO_HIP_THRUST_PATH "thrust/complex.h"
     PATHS "${HIP_PATH}/../include"
diff --git a/hip/base/exception.hip.cpp b/hip/base/exception.hip.cpp
index 66418c58c08..ca73a88c6dc 100644
--- a/hip/base/exception.hip.cpp
+++ b/hip/base/exception.hip.cpp
@@ -91,6 +91,7 @@ std::string HiprandError::get_error(int64 error_code)
     GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_TYPE_ERROR);
     GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_OUT_OF_RANGE);
     GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_LENGTH_NOT_MULTIPLE);
+    GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_DOUBLE_PRECISION_REQUIRED);
     GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_LAUNCH_FAILURE);
     GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_PREEXISTING_FAILURE);
     GKO_REGISTER_HIPRAND_ERROR(HIPRAND_STATUS_INITIALIZATION_FAILED);
diff --git a/hip/base/types.hip.hpp b/hip/base/types.hip.hpp
index e3f43e8e67c..2a7395fac72 100644
--- a/hip/base/types.hip.hpp
+++ b/hip/base/types.hip.hpp
@@ -95,6 +95,7 @@ struct hiplibs_type_impl<thrust::complex<T>> {
     using type = typename hiplibs_type_impl<std::complex<T>>::type;
 };
 
+
 template <typename T>
 struct hipblas_type_impl {
     using type = T;
@@ -135,6 +136,7 @@ struct hipblas_type_impl<thrust::complex<T>> {
     using type = typename hipblas_type_impl<std::complex<T>>::type;
 };
 
+
 template <typename T>
 struct hip_type_impl {
     using type = T;
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index 78d4ad6aae2..465b17c960e 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -78,11 +78,10 @@ namespace {
 
 
 template <typename ValueType>
-void initialize_m(matrix::Dense<ValueType> *m,
+void initialize_m(const size_type nrhs, matrix::Dense<ValueType> *m,
                   Array<stopping_status> *stop_status)
 {
     const auto subspace_dim = m->get_size()[0];
-    const auto nrhs = m->get_size()[1] / subspace_dim;
     const auto m_stride = m->get_stride();
 
     const auto grid_dim = ceildiv(m_stride * subspace_dim, default_block_size);
@@ -127,13 +126,13 @@ void orthonormalize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors)
 
 
 template <typename ValueType>
-void solve_lower_triangular(const matrix::Dense<ValueType> *m,
+void solve_lower_triangular(const size_type nrhs,
+                            const matrix::Dense<ValueType> *m,
                             const matrix::Dense<ValueType> *f,
                             matrix::Dense<ValueType> *c,
                             const Array<stopping_status> *stop_status)
 {
     const auto subspace_dim = m->get_size()[0];
-    const auto nrhs = m->get_size()[1] / subspace_dim;
 
     const auto grid_dim = ceildiv(nrhs, default_block_size);
     hipLaunchKernelGGL(HIP_KERNEL_NAME(solve_lower_triangular_kernel), grid_dim,
@@ -146,7 +145,8 @@ void solve_lower_triangular(const matrix::Dense<ValueType> *m,
 
 
 template <typename ValueType>
-void update_g_and_u(std::shared_ptr<const HipExecutor> exec, size_type k,
+void update_g_and_u(std::shared_ptr<const HipExecutor> exec,
+                    const size_type nrhs, const size_type k,
                     const matrix::Dense<ValueType> *p,
                     const matrix::Dense<ValueType> *m,
                     matrix::Dense<ValueType> *alpha,
@@ -155,7 +155,6 @@ void update_g_and_u(std::shared_ptr<const HipExecutor> exec, size_type k,
                     const Array<stopping_status> *stop_status)
 {
     const auto size = g->get_size()[0];
-    const auto nrhs = m->get_size()[1] / m->get_size()[0];
     const auto p_stride = p->get_stride();
 
     const dim3 grid_dim(ceildiv(nrhs, default_dot_dim),
@@ -198,14 +197,13 @@ void update_g_and_u(std::shared_ptr<const HipExecutor> exec, size_type k,
 
 
 template <typename ValueType>
-void update_m(std::shared_ptr<const HipExecutor> exec, size_type k,
-              const matrix::Dense<ValueType> *p,
+void update_m(std::shared_ptr<const HipExecutor> exec, const size_type nrhs,
+              const size_type k, const matrix::Dense<ValueType> *p,
               const matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *m,
               const Array<stopping_status> *stop_status)
 {
     const auto size = g_k->get_size()[0];
     const auto subspace_dim = m->get_size()[0];
-    const auto nrhs = m->get_size()[1] / m->get_size()[0];
     const auto p_stride = p->get_stride();
     const auto m_stride = m->get_stride();
 
@@ -232,7 +230,8 @@ void update_m(std::shared_ptr<const HipExecutor> exec, size_type k,
 
 
 template <typename ValueType>
-void update_x_r_and_f(std::shared_ptr<const HipExecutor> exec, size_type k,
+void update_x_r_and_f(std::shared_ptr<const HipExecutor> exec,
+                      const size_type nrhs, const size_type k,
                       const matrix::Dense<ValueType> *m,
                       const matrix::Dense<ValueType> *g,
                       const matrix::Dense<ValueType> *u,
@@ -242,7 +241,6 @@ void update_x_r_and_f(std::shared_ptr<const HipExecutor> exec, size_type k,
 {
     const auto size = x->get_size()[0];
     const auto subspace_dim = m->get_size()[0];
-    const auto nrhs = x->get_size()[1];
 
     const auto grid_dim = ceildiv(size * x->get_stride(), default_block_size);
     hipLaunchKernelGGL(HIP_KERNEL_NAME(update_x_r_and_f_kernel), grid_dim,
@@ -268,7 +266,7 @@ void initialize(std::shared_ptr<const HipExecutor> exec,
                 matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
                 Array<stopping_status> *stop_status)
 {
-    initialize_m(m, stop_status);
+    initialize_m(nrhs, m, stop_status);
     initialize_subspace_vectors(subspace_vectors, deterministic);
     orthonormalize_subspace_vectors(subspace_vectors);
 }
@@ -277,19 +275,18 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const HipExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *m,
+void step_1(std::shared_ptr<const HipExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *m,
             const matrix::Dense<ValueType> *f,
             const matrix::Dense<ValueType> *residual,
             const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *c,
             matrix::Dense<ValueType> *v,
             const Array<stopping_status> *stop_status)
 {
-    solve_lower_triangular(m, f, c, stop_status);
+    solve_lower_triangular(nrhs, m, f, c, stop_status);
 
     const auto num_rows = v->get_size()[0];
     const auto subspace_dim = m->get_size()[0];
-    const auto nrhs = m->get_size()[1] / subspace_dim;
 
     const auto grid_dim =
         ceildiv(v->get_stride() * num_rows, default_block_size);
@@ -306,14 +303,13 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const HipExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *omega,
+void step_2(std::shared_ptr<const HipExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *omega,
             const matrix::Dense<ValueType> *preconditioned_vector,
             const matrix::Dense<ValueType> *c, matrix::Dense<ValueType> *u,
             const Array<stopping_status> *stop_status)
 {
     const auto num_rows = preconditioned_vector->get_size()[0];
-    const auto nrhs = preconditioned_vector->get_size()[1];
     const auto subspace_dim = u->get_size()[1] / nrhs;
 
     const auto grid_dim =
@@ -331,17 +327,17 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 
 
 template <typename ValueType>
-void step_3(std::shared_ptr<const HipExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
-            matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *u,
-            matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
-            matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *residual,
-            matrix::Dense<ValueType> *x,
+void step_3(std::shared_ptr<const HipExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *p,
+            matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *g_k,
+            matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
+            matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *alpha,
+            matrix::Dense<ValueType> *residual, matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status)
 {
-    update_g_and_u(exec, k, p, m, alpha, g, g_k, u, stop_status);
-    update_m(exec, k, p, g_k, m, stop_status);
-    update_x_r_and_f(exec, k, m, g, u, f, residual, x, stop_status);
+    update_g_and_u(exec, nrhs, k, p, m, alpha, g, g_k, u, stop_status);
+    update_m(exec, nrhs, k, p, g_k, m, stop_status);
+    update_x_r_and_f(exec, nrhs, k, m, g, u, f, residual, x, stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
@@ -349,13 +345,11 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
 template <typename ValueType>
 void compute_omega(
-    std::shared_ptr<const HipExecutor> exec,
+    std::shared_ptr<const HipExecutor> exec, const size_type nrhs,
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
     matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
 {
-    const auto nrhs = omega->get_size()[1];
-
     const auto grid_dim = ceildiv(nrhs, config::warp_size);
     hipLaunchKernelGGL(HIP_KERNEL_NAME(compute_omega_kernel), grid_dim,
                        config::warp_size, 0, 0, nrhs, kappa,
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index b9779f8772c..e27e2c6b9fe 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -104,7 +104,7 @@ class Idr : public ::testing::Test {
     void initialize_data()
     {
         int size = 597;
-        int nrhs = 17;
+        nrhs = 17;
         int s = 4;
         x = gen_mtx(size, nrhs);
         b = gen_mtx(size, nrhs);
@@ -116,6 +116,7 @@ class Idr : public ::testing::Test {
         c = gen_mtx(s, nrhs);
         v = gen_mtx(size, nrhs);
         p = gen_mtx(s, size);
+        alpha = gen_mtx(1, nrhs);
         omega = gen_mtx(1, nrhs);
         tht = gen_mtx(1, nrhs);
         residual_norm = gen_mtx(1, nrhs);
@@ -135,6 +136,7 @@ class Idr : public ::testing::Test {
         d_c = Mtx::create(hip);
         d_v = Mtx::create(hip);
         d_p = Mtx::create(hip);
+        d_alpha = Mtx::create(hip);
         d_omega = Mtx::create(hip);
         d_tht = Mtx::create(hip);
         d_residual_norm = Mtx::create(hip);
@@ -151,6 +153,7 @@ class Idr : public ::testing::Test {
         d_c->copy_from(c.get());
         d_v->copy_from(v.get());
         d_p->copy_from(p.get());
+        d_alpha->copy_from(alpha.get());
         d_omega->copy_from(omega.get());
         d_tht->copy_from(tht.get());
         d_residual_norm->copy_from(residual_norm.get());
@@ -180,6 +183,8 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Solver::Factory> hip_idr_factory;
     std::unique_ptr<Solver::Factory> ref_idr_factory;
 
+    gko::size_type nrhs;
+
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
@@ -190,6 +195,7 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> c;
     std::unique_ptr<Mtx> v;
     std::unique_ptr<Mtx> p;
+    std::unique_ptr<Mtx> alpha;
     std::unique_ptr<Mtx> omega;
     std::unique_ptr<Mtx> tht;
     std::unique_ptr<Mtx> residual_norm;
@@ -205,6 +211,7 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> d_c;
     std::unique_ptr<Mtx> d_v;
     std::unique_ptr<Mtx> d_p;
+    std::unique_ptr<Mtx> d_alpha;
     std::unique_ptr<Mtx> d_omega;
     std::unique_ptr<Mtx> d_tht;
     std::unique_ptr<Mtx> d_residual_norm;
@@ -216,9 +223,9 @@ TEST_F(Idr, IdrInitializeIsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::idr::initialize(ref, m.get(), p.get(), true,
+    gko::kernels::reference::idr::initialize(ref, nrhs, m.get(), p.get(), true,
                                              stop_status.get());
-    gko::kernels::hip::idr::initialize(hip, d_m.get(), d_p.get(), true,
+    gko::kernels::hip::idr::initialize(hip, nrhs, d_m.get(), d_p.get(), true,
                                        d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
@@ -231,11 +238,11 @@ TEST_F(Idr, IdrStep1IsEquivalentToRef)
     initialize_data();
 
     gko::size_type k = 2;
-    gko::kernels::reference::idr::step_1(ref, k, m.get(), f.get(), r.get(),
-                                         g.get(), c.get(), v.get(),
+    gko::kernels::reference::idr::step_1(ref, nrhs, k, m.get(), f.get(),
+                                         r.get(), g.get(), c.get(), v.get(),
                                          stop_status.get());
-    gko::kernels::hip::idr::step_1(hip, k, d_m.get(), d_f.get(), d_r.get(),
-                                   d_g.get(), d_c.get(), d_v.get(),
+    gko::kernels::hip::idr::step_1(hip, nrhs, k, d_m.get(), d_f.get(),
+                                   d_r.get(), d_g.get(), d_c.get(), d_v.get(),
                                    d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(c, d_c, 1e-14);
@@ -248,10 +255,10 @@ TEST_F(Idr, IdrStep2IsEquivalentToRef)
     initialize_data();
 
     gko::size_type k = 2;
-    gko::kernels::reference::idr::step_2(ref, k, omega.get(), v.get(), c.get(),
-                                         u.get(), stop_status.get());
-    gko::kernels::hip::idr::step_2(hip, k, d_omega.get(), d_v.get(), d_c.get(),
-                                   d_u.get(), d_stop_status.get());
+    gko::kernels::reference::idr::step_2(ref, nrhs, k, omega.get(), v.get(),
+                                         c.get(), u.get(), stop_status.get());
+    gko::kernels::hip::idr::step_2(hip, nrhs, k, d_omega.get(), d_v.get(),
+                                   d_c.get(), d_u.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
 }
@@ -262,14 +269,15 @@ TEST_F(Idr, IdrStep3IsEquivalentToRef)
     initialize_data();
 
     gko::size_type k = 2;
-    gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), v.get(),
-                                         u.get(), m.get(), f.get(), omega.get(),
-                                         r.get(), x.get(), stop_status.get());
+    gko::kernels::reference::idr::step_3(
+        ref, nrhs, k, p.get(), g.get(), v.get(), u.get(), m.get(), f.get(),
+        alpha.get(), r.get(), x.get(), stop_status.get());
     gko::kernels::hip::idr::step_3(
-        hip, k, d_p.get(), d_g.get(), d_v.get(), d_u.get(), d_m.get(),
-        d_f.get(), d_omega.get(), d_r.get(), d_x.get(), d_stop_status.get());
+        hip, nrhs, k, d_p.get(), d_g.get(), d_v.get(), d_u.get(), d_m.get(),
+        d_f.get(), d_alpha.get(), d_r.get(), d_x.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(g, d_g, 1e-14);
+    GKO_ASSERT_MTX_NEAR(v, d_v, 1e-14);
     GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
     GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
     GKO_ASSERT_MTX_NEAR(f, d_f, 1e-14);
@@ -283,10 +291,10 @@ TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
     initialize_data();
 
     double kappa = 0.7;
-    gko::kernels::reference::idr::compute_omega(ref, kappa, tht.get(),
+    gko::kernels::reference::idr::compute_omega(ref, nrhs, kappa, tht.get(),
                                                 residual_norm.get(),
                                                 omega.get(), stop_status.get());
-    gko::kernels::hip::idr::compute_omega(hip, kappa, d_tht.get(),
+    gko::kernels::hip::idr::compute_omega(hip, nrhs, kappa, d_tht.get(),
                                           d_residual_norm.get(), d_omega.get(),
                                           d_stop_status.get());
 
@@ -298,7 +306,6 @@ TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 1;
-
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto hip_solver = hip_idr_factory->generate(d_mtx);
     auto b = gen_mtx(m, n);
@@ -320,7 +327,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 1;
-
     hip_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -328,7 +334,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(1u).on(hip))
             .on(hip);
-
     ref_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -336,7 +341,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(1u).on(ref))
             .on(ref);
-
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto hip_solver = hip_idr_factory->generate(d_mtx);
     auto b = gen_mtx(m, n);
@@ -379,7 +383,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 16;
-
     hip_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -387,7 +390,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(1u).on(hip))
             .on(hip);
-
     ref_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -395,7 +397,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(1u).on(ref))
             .on(ref);
-
     auto hip_solver = hip_idr_factory->generate(d_mtx);
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto b = gen_mtx(m, n);
diff --git a/include/ginkgo/core/base/array.hpp b/include/ginkgo/core/base/array.hpp
index 8d02bfd030c..59960520edf 100644
--- a/include/ginkgo/core/base/array.hpp
+++ b/include/ginkgo/core/base/array.hpp
@@ -288,25 +288,6 @@ class Array {
         return Array{exec, num_elems, data, view_deleter{}};
     }
 
-    /**
-     * Creates an Array from existing memory.
-     *
-     * The Array does not take ownership of the memory, and will not deallocate
-     * it once it goes out of scope. This array type cannot use the function
-     * `resize_and_reset` since it does not own the data it should resize.
-     *
-     * @param exec  executor where `data` is located
-     * @param num_elems  number of elements in `data`
-     * @param data  chunk of memory used to create the array
-     *
-     * @return an Array constructed from `data`
-     */
-    static Array view(std::shared_ptr<const Executor> exec, size_type num_elems,
-                      const value_type *data)
-    {
-        return Array{exec, num_elems, data, view_deleter{}};
-    }
-
     /**
      * Copies data from another array or view. In the case of an array target,
      * the array is resized to match the source's size. In the case of a view
diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index 6ed8ac964a9..fb3d6a860a2 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -133,6 +133,60 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
         stop_criterion_factory_ = std::move(other);
     }
 
+    /**
+     * Gets the subspace dimension of the solver.
+     *
+     * @return the subspace Dimension*/
+    size_type get_subspace_dim() const { return subspace_dim_; }
+
+    /**
+     * Sets the subspace dimension of the solver.
+     *
+     * @param other  the new subspace Dimension*/
+    void set_subspace_dim(const size_type other) { subspace_dim_ = other; }
+
+    /**
+     * Gets the kappa parameter of the solver.
+     *
+     * @return the kappa parameter
+     */
+    remove_complex<ValueType> get_kappa() const { return kappa_; }
+
+    /**
+     * Sets the kappa parameter of the solver.
+     *
+     * @param other  the new kappa parameter
+     */
+    void set_kappa(const remove_complex<ValueType> other) { kappa_ = other; }
+
+    /**
+     * Gets the deterministic parameter of the solver.
+     *
+     * @return the deterministic parameter
+     */
+    bool get_deterministic() const { return deterministic_; }
+
+    /**
+     * Sets the deterministic parameter of the solver.
+     *
+     * @param other  the new deterministic parameter
+     */
+    void set_deterministic(const bool other) { deterministic_ = other; }
+
+    /**
+     * Gets the complex_subspace parameter of the solver.
+     *
+     * @return the complex_subspace parameter
+     */
+    bool get_complex_subspace() const { return complex_subspace_; }
+
+    /**
+     * Sets the complex_subspace parameter of the solver.
+     *
+     * @param other  the new complex_subspace parameter
+     */
+    void set_complex_subpsace(const bool other) { complex_subspace_ = other; }
+
     GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
     {
         /**
@@ -164,7 +218,7 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
          * Threshold to determine if Av_n and v_n are too close to being
          * perpendicular.
          * This is considered to be the case if
-         * |(Av_n)^H * v_n / (norm(Av_n) * norm(v_n))| < kappa
+         * $|(Av_n)^H * v_n / (norm(Av_n) * norm(v_n))| < kappa$
          */
         remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(kappa, 0.7);
 
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index 20d4dffd38d..c1aa45525e3 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -62,6 +62,7 @@ class ResidualNorm
     friend class EnablePolymorphicObject<ResidualNorm<ValueType>, Criterion>;
 
 public:
+    using ComplexVector = matrix::Dense<to_complex<ValueType>>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
     using Vector = matrix::Dense<ValueType>;
 
@@ -111,6 +112,7 @@ class ResidualNorm
 template <typename ValueType = default_precision>
 class ResidualNormReduction : public ResidualNorm<ValueType> {
 public:
+    using ComplexVector = matrix::Dense<to_complex<ValueType>>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
     using Vector = matrix::Dense<ValueType>;
 
@@ -147,10 +149,8 @@ class ResidualNormReduction : public ResidualNorm<ValueType> {
             exec, dim<2>{1, args.initial_residual->get_size()[1]});
         this->u_dense_tau_ =
             NormVector::create_with_config_of(this->starting_tau_.get());
-        if (dynamic_cast<const matrix::Dense<to_complex<ValueType>> *>(
-                args.initial_residual)) {
-            auto dense_r =
-                as<matrix::Dense<to_complex<ValueType>>>(args.initial_residual);
+        if (dynamic_cast<const ComplexVector *>(args.initial_residual)) {
+            auto dense_r = as<ComplexVector>(args.initial_residual);
             dense_r->compute_norm2(this->starting_tau_.get());
         } else {
             auto dense_r = as<Vector>(args.initial_residual);
@@ -178,6 +178,7 @@ class ResidualNormReduction : public ResidualNorm<ValueType> {
 template <typename ValueType = default_precision>
 class RelativeResidualNorm : public ResidualNorm<ValueType> {
 public:
+    using ComplexVector = matrix::Dense<to_complex<ValueType>>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
     using Vector = matrix::Dense<ValueType>;
 
@@ -214,9 +215,8 @@ class RelativeResidualNorm : public ResidualNorm<ValueType> {
             NormVector::create(exec, dim<2>{1, args.b->get_size()[1]});
         this->u_dense_tau_ =
             NormVector::create_with_config_of(this->starting_tau_.get());
-        if (dynamic_cast<const matrix::Dense<to_complex<ValueType>> *>(
-                args.b.get())) {
-            auto dense_rhs = as<matrix::Dense<to_complex<ValueType>>>(args.b);
+        if (dynamic_cast<const ComplexVector *>(args.b.get())) {
+            auto dense_rhs = as<ComplexVector>(args.b);
             dense_rhs->compute_norm2(this->starting_tau_.get());
         } else {
             auto dense_rhs = as<Vector>(args.b);
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index 3fb5a64cd2d..d6aaeddee3b 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -62,12 +62,12 @@ namespace {
 
 
 template <typename ValueType>
-void solve_lower_triangular(const matrix::Dense<ValueType> *m,
+void solve_lower_triangular(const size_type nrhs,
+                            const matrix::Dense<ValueType> *m,
                             const matrix::Dense<ValueType> *f,
                             matrix::Dense<ValueType> *c,
                             const Array<stopping_status> *stop_status)
 {
-    const auto nrhs = m->get_size()[1] / m->get_size()[0];
 #pragma omp parallel for
     for (size_type i = 0; i < f->get_size()[1]; i++) {
         if (stop_status->get_const_data()[i].has_stopped()) {
@@ -86,13 +86,13 @@ void solve_lower_triangular(const matrix::Dense<ValueType> *m,
 
 
 template <typename ValueType>
-void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
+void update_g_and_u(const size_type nrhs, const size_type k,
+                    const matrix::Dense<ValueType> *p,
                     const matrix::Dense<ValueType> *m,
                     matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *g_k,
                     matrix::Dense<ValueType> *u,
                     const Array<stopping_status> *stop_status)
 {
-    const auto nrhs = m->get_size()[1] / m->get_size()[0];
 #pragma omp parallel for
     for (size_type i = 0; i < nrhs; i++) {
         if (stop_status->get_const_data()[i].has_stopped()) {
@@ -139,13 +139,12 @@ get_rand_value(Distribution &&dist, Generator &&gen)
 
 
 template <typename ValueType>
-void initialize(std::shared_ptr<const OmpExecutor> exec,
+void initialize(std::shared_ptr<const OmpExecutor> exec, const size_type nrhs,
                 matrix::Dense<ValueType> *m,
                 matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
                 Array<stopping_status> *stop_status)
 {
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
-    const auto nrhs = m->get_size()[1] / m->get_size()[0];
 
     // Initialize M
 #pragma omp parallel for
@@ -206,8 +205,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const OmpExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *m,
+void step_1(std::shared_ptr<const OmpExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *m,
             const matrix::Dense<ValueType> *f,
             const matrix::Dense<ValueType> *residual,
             const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *c,
@@ -215,10 +214,9 @@ void step_1(std::shared_ptr<const OmpExecutor> exec, const size_type k,
             const Array<stopping_status> *stop_status)
 {
     const auto m_size = m->get_size();
-    const auto nrhs = f->get_size()[1];
 
     // Compute c = M \ f
-    solve_lower_triangular(m, f, c, stop_status);
+    solve_lower_triangular(nrhs, m, f, c, stop_status);
 
     for (size_type i = 0; i < nrhs; i++) {
         if (stop_status->get_const_data()[i].has_stopped()) {
@@ -241,13 +239,12 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const OmpExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *omega,
+void step_2(std::shared_ptr<const OmpExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *omega,
             const matrix::Dense<ValueType> *preconditioned_vector,
             const matrix::Dense<ValueType> *c, matrix::Dense<ValueType> *u,
             const Array<stopping_status> *stop_status)
 {
-    const auto nrhs = omega->get_size()[1];
     for (size_type i = 0; i < nrhs; i++) {
         if (stop_status->get_const_data()[i].has_stopped()) {
             continue;
@@ -268,17 +265,15 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 
 
 template <typename ValueType>
-void step_3(std::shared_ptr<const OmpExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
-            matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *u,
-            matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
-            matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *residual,
-            matrix::Dense<ValueType> *x,
+void step_3(std::shared_ptr<const OmpExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *p,
+            matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *g_k,
+            matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
+            matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *,
+            matrix::Dense<ValueType> *residual, matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status)
 {
-    const auto nrhs = x->get_size()[1];
-
-    update_g_and_u(k, p, m, g, g_k, u, stop_status);
+    update_g_and_u(nrhs, k, p, m, g, g_k, u, stop_status);
 
     for (size_type i = 0; i < nrhs; i++) {
         if (stop_status->get_const_data()[i].has_stopped()) {
@@ -317,13 +312,13 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
 template <typename ValueType>
 void compute_omega(
-    std::shared_ptr<const OmpExecutor> exec,
+    std::shared_ptr<const OmpExecutor> exec, const size_type nrhs,
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
     matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
 {
 #pragma omp parallel for
-    for (size_type i = 0; i < omega->get_size()[1]; i++) {
+    for (size_type i = 0; i < nrhs; i++) {
         if (stop_status->get_const_data()[i].has_stopped()) {
             continue;
         }
diff --git a/omp/stop/residual_norm_kernels.cpp b/omp/stop/residual_norm_kernels.cpp
index 3abe68d58f6..1fd3a14cf85 100644
--- a/omp/stop/residual_norm_kernels.cpp
+++ b/omp/stop/residual_norm_kernels.cpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <omp.h>
 
+
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 
diff --git a/omp/test/solver/gmres_kernels.cpp b/omp/test/solver/gmres_kernels.cpp
index 31d9a68f8bb..da04a15c44b 100644
--- a/omp/test/solver/gmres_kernels.cpp
+++ b/omp/test/solver/gmres_kernels.cpp
@@ -125,7 +125,7 @@ class Gmres : public ::testing::Test {
 
     void initialize_data()
     {
-        int m = 5;  // 97;
+        int m = 597;
         int n = 43;
         x = gen_mtx(m, n);
         y = gen_mtx(gko::solver::default_krylov_dim, n);
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index b6cdc94ef15..2a4f50e9772 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -109,7 +109,7 @@ class Idr : public ::testing::Test {
     void initialize_data()
     {
         int size = 597;
-        int nrhs = 17;
+        nrhs = 17;
         int s = 4;
         x = gen_mtx(size, nrhs);
         b = gen_mtx(size, nrhs);
@@ -121,6 +121,7 @@ class Idr : public ::testing::Test {
         c = gen_mtx(s, nrhs);
         v = gen_mtx(size, nrhs);
         p = gen_mtx(s, size);
+        alpha = gen_mtx(1, nrhs);
         omega = gen_mtx(1, nrhs);
         tht = gen_mtx(1, nrhs);
         residual_norm = gen_mtx(1, nrhs);
@@ -140,6 +141,7 @@ class Idr : public ::testing::Test {
         d_c = Mtx::create(omp);
         d_v = Mtx::create(omp);
         d_p = Mtx::create(omp);
+        d_alpha = Mtx::create(omp);
         d_omega = Mtx::create(omp);
         d_tht = Mtx::create(omp);
         d_residual_norm = Mtx::create(omp);
@@ -156,6 +158,7 @@ class Idr : public ::testing::Test {
         d_c->copy_from(c.get());
         d_v->copy_from(v.get());
         d_p->copy_from(p.get());
+        d_alpha->copy_from(alpha.get());
         d_omega->copy_from(omega.get());
         d_tht->copy_from(tht.get());
         d_residual_norm->copy_from(residual_norm.get());
@@ -185,6 +188,8 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Solver::Factory> omp_idr_factory;
     std::unique_ptr<Solver::Factory> ref_idr_factory;
 
+    gko::size_type nrhs;
+
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
@@ -195,6 +200,7 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> c;
     std::unique_ptr<Mtx> v;
     std::unique_ptr<Mtx> p;
+    std::unique_ptr<Mtx> alpha;
     std::unique_ptr<Mtx> omega;
     std::unique_ptr<Mtx> tht;
     std::unique_ptr<Mtx> residual_norm;
@@ -210,6 +216,7 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Mtx> d_c;
     std::unique_ptr<Mtx> d_v;
     std::unique_ptr<Mtx> d_p;
+    std::unique_ptr<Mtx> d_alpha;
     std::unique_ptr<Mtx> d_omega;
     std::unique_ptr<Mtx> d_tht;
     std::unique_ptr<Mtx> d_residual_norm;
@@ -221,9 +228,9 @@ TEST_F(Idr, IdrInitializeIsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::idr::initialize(ref, m.get(), p.get(), true,
+    gko::kernels::reference::idr::initialize(ref, nrhs, m.get(), p.get(), true,
                                              stop_status.get());
-    gko::kernels::omp::idr::initialize(omp, d_m.get(), d_p.get(), true,
+    gko::kernels::omp::idr::initialize(omp, nrhs, d_m.get(), d_p.get(), true,
                                        d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
@@ -236,11 +243,11 @@ TEST_F(Idr, IdrStep1IsEquivalentToRef)
     initialize_data();
 
     gko::size_type k = 2;
-    gko::kernels::reference::idr::step_1(ref, k, m.get(), f.get(), r.get(),
-                                         g.get(), c.get(), v.get(),
+    gko::kernels::reference::idr::step_1(ref, nrhs, k, m.get(), f.get(),
+                                         r.get(), g.get(), c.get(), v.get(),
                                          stop_status.get());
-    gko::kernels::omp::idr::step_1(omp, k, d_m.get(), d_f.get(), d_r.get(),
-                                   d_g.get(), d_c.get(), d_v.get(),
+    gko::kernels::omp::idr::step_1(omp, nrhs, k, d_m.get(), d_f.get(),
+                                   d_r.get(), d_g.get(), d_c.get(), d_v.get(),
                                    d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(c, d_c, 1e-14);
@@ -253,10 +260,10 @@ TEST_F(Idr, IdrStep2IsEquivalentToRef)
     initialize_data();
 
     gko::size_type k = 2;
-    gko::kernels::reference::idr::step_2(ref, k, omega.get(), v.get(), c.get(),
-                                         u.get(), stop_status.get());
-    gko::kernels::omp::idr::step_2(omp, k, d_omega.get(), d_v.get(), d_c.get(),
-                                   d_u.get(), d_stop_status.get());
+    gko::kernels::reference::idr::step_2(ref, nrhs, k, omega.get(), v.get(),
+                                         c.get(), u.get(), stop_status.get());
+    gko::kernels::omp::idr::step_2(omp, nrhs, k, d_omega.get(), d_v.get(),
+                                   d_c.get(), d_u.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
 }
@@ -267,14 +274,15 @@ TEST_F(Idr, IdrStep3IsEquivalentToRef)
     initialize_data();
 
     gko::size_type k = 2;
-    gko::kernels::reference::idr::step_3(ref, k, p.get(), g.get(), v.get(),
-                                         u.get(), m.get(), f.get(), omega.get(),
-                                         r.get(), x.get(), stop_status.get());
+    gko::kernels::reference::idr::step_3(
+        ref, nrhs, k, p.get(), g.get(), v.get(), u.get(), m.get(), f.get(),
+        alpha.get(), r.get(), x.get(), stop_status.get());
     gko::kernels::omp::idr::step_3(
-        omp, k, d_p.get(), d_g.get(), d_v.get(), d_u.get(), d_m.get(),
-        d_f.get(), d_omega.get(), d_r.get(), d_x.get(), d_stop_status.get());
+        omp, nrhs, k, d_p.get(), d_g.get(), d_v.get(), d_u.get(), d_m.get(),
+        d_f.get(), d_alpha.get(), d_r.get(), d_x.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(g, d_g, 1e-14);
+    GKO_ASSERT_MTX_NEAR(v, d_v, 1e-14);
     GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
     GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
     GKO_ASSERT_MTX_NEAR(f, d_f, 1e-14);
@@ -288,10 +296,10 @@ TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
     initialize_data();
 
     double kappa = 0.7;
-    gko::kernels::reference::idr::compute_omega(ref, kappa, tht.get(),
+    gko::kernels::reference::idr::compute_omega(ref, nrhs, kappa, tht.get(),
                                                 residual_norm.get(),
                                                 omega.get(), stop_status.get());
-    gko::kernels::omp::idr::compute_omega(omp, kappa, d_tht.get(),
+    gko::kernels::omp::idr::compute_omega(omp, nrhs, kappa, d_tht.get(),
                                           d_residual_norm.get(), d_omega.get(),
                                           d_stop_status.get());
 
@@ -303,7 +311,6 @@ TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 1;
-
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto omp_solver = omp_idr_factory->generate(d_mtx);
     auto b = gen_mtx(m, n);
@@ -325,7 +332,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 1;
-
     omp_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -333,7 +339,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(1u).on(omp))
             .on(omp);
-
     ref_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -341,7 +346,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(1u).on(ref))
             .on(ref);
-
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto omp_solver = omp_idr_factory->generate(d_mtx);
     auto b = gen_mtx(m, n);
@@ -384,7 +388,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 16;
-
     omp_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -392,7 +395,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(1u).on(omp))
             .on(omp);
-
     ref_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -400,7 +402,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(1u).on(ref))
             .on(ref);
-
     auto omp_solver = omp_idr_factory->generate(d_mtx);
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto b = gen_mtx(m, n);
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index 23d32aa0cbb..f9d743a9077 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -59,12 +59,12 @@ namespace {
 
 
 template <typename ValueType>
-void solve_lower_triangular(const matrix::Dense<ValueType> *m,
+void solve_lower_triangular(const size_type nrhs,
+                            const matrix::Dense<ValueType> *m,
                             const matrix::Dense<ValueType> *f,
                             matrix::Dense<ValueType> *c,
                             const Array<stopping_status> *stop_status)
 {
-    const auto nrhs = m->get_size()[1] / m->get_size()[0];
     for (size_type i = 0; i < f->get_size()[1]; i++) {
         if (stop_status->get_const_data()[i].has_stopped()) {
             continue;
@@ -82,13 +82,13 @@ void solve_lower_triangular(const matrix::Dense<ValueType> *m,
 
 
 template <typename ValueType>
-void update_g_and_u(size_type k, const matrix::Dense<ValueType> *p,
+void update_g_and_u(const size_type nrhs, const size_type k,
+                    const matrix::Dense<ValueType> *p,
                     const matrix::Dense<ValueType> *m,
                     matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *g_k,
                     matrix::Dense<ValueType> *u,
                     const Array<stopping_status> *stop_status)
 {
-    const auto nrhs = m->get_size()[1] / m->get_size()[0];
     for (size_type i = 0; i < nrhs; i++) {
         if (stop_status->get_const_data()[i].has_stopped()) {
             continue;
@@ -134,12 +134,10 @@ get_rand_value(Distribution &&dist, Generator &&gen)
 
 template <typename ValueType>
 void initialize(std::shared_ptr<const ReferenceExecutor> exec,
-                matrix::Dense<ValueType> *m,
+                const size_type nrhs, matrix::Dense<ValueType> *m,
                 matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
                 Array<stopping_status> *stop_status)
 {
-    const auto nrhs = m->get_size()[1] / m->get_size()[0];
-
     // Initialize M
     for (size_type i = 0; i < nrhs; i++) {
         stop_status->get_data()[i].reset();
@@ -193,18 +191,16 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const ReferenceExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *m,
+void step_1(std::shared_ptr<const ReferenceExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *m,
             const matrix::Dense<ValueType> *f,
             const matrix::Dense<ValueType> *residual,
             const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *c,
             matrix::Dense<ValueType> *v,
             const Array<stopping_status> *stop_status)
 {
-    const auto nrhs = f->get_size()[1];
-
     // Compute c = M \ f
-    solve_lower_triangular(m, f, c, stop_status);
+    solve_lower_triangular(nrhs, m, f, c, stop_status);
 
     for (size_type i = 0; i < nrhs; i++) {
         if (stop_status->get_const_data()[i].has_stopped()) {
@@ -225,13 +221,12 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const ReferenceExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *omega,
+void step_2(std::shared_ptr<const ReferenceExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *omega,
             const matrix::Dense<ValueType> *preconditioned_vector,
             const matrix::Dense<ValueType> *c, matrix::Dense<ValueType> *u,
             const Array<stopping_status> *stop_status)
 {
-    const auto nrhs = omega->get_size()[1];
     for (size_type i = 0; i < nrhs; i++) {
         if (stop_status->get_const_data()[i].has_stopped()) {
             continue;
@@ -251,17 +246,15 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 
 
 template <typename ValueType>
-void step_3(std::shared_ptr<const ReferenceExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
-            matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *u,
-            matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
-            matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *residual,
-            matrix::Dense<ValueType> *x,
+void step_3(std::shared_ptr<const ReferenceExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *p,
+            matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *g_k,
+            matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
+            matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *,
+            matrix::Dense<ValueType> *residual, matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status)
 {
-    const auto nrhs = x->get_size()[1];
-
-    update_g_and_u(k, p, m, g, g_k, u, stop_status);
+    update_g_and_u(nrhs, k, p, m, g, g_k, u, stop_status);
 
     for (size_type i = 0; i < nrhs; i++) {
         if (stop_status->get_const_data()[i].has_stopped()) {
@@ -297,12 +290,12 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
 template <typename ValueType>
 void compute_omega(
-    std::shared_ptr<const ReferenceExecutor> exec,
+    std::shared_ptr<const ReferenceExecutor> exec, const size_type nrhs,
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
     matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
 {
-    for (size_type i = 0; i < omega->get_size()[1]; i++) {
+    for (size_type i = 0; i < nrhs; i++) {
         if (stop_status->get_const_data()[i].has_stopped()) {
             continue;
         }
diff --git a/reference/stop/residual_norm_kernels.cpp b/reference/stop/residual_norm_kernels.cpp
index 0777276e31c..fb968e0eae3 100644
--- a/reference/stop/residual_norm_kernels.cpp
+++ b/reference/stop/residual_norm_kernels.cpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <algorithm>
 
+
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>

From bb0082bfb8846a271884aef02d6a352390d92e29 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 19 Nov 2020 16:47:34 +0100
Subject: [PATCH 0162/1008] use default containers

---
 .gitlab-ci.yml                   | 88 ++++++++++++++++----------------
 cuda/test/solver/idr_kernels.cpp |  4 +-
 hip/solver/idr_kernels.hip.cpp   |  2 +-
 3 files changed, 47 insertions(+), 47 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c2fe2fba806..9969d84177c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -127,7 +127,7 @@ stages:
 
 sync:
   stage: sync
-  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
   variables:
     GIT_STRATEGY: none
     PRIVATE_REPO: git@gitlab.com:ginkgo-project/ginkgo.git
@@ -153,7 +153,7 @@ sync:
 # Job with example runs.
 build/cuda90/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda90-gnu5-llvm39
+  image: localhost:5000/gko-cuda90-gnu5-llvm39
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -171,7 +171,7 @@ build/cuda90/gcc/all/debug/shared:
 
 build/cuda90/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda90-gnu5-llvm39
+  image: localhost:5000/gko-cuda90-gnu5-llvm39
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -192,7 +192,7 @@ build/cuda90/clang/all/release/static:
 # cuda 9.1 and friends
 build/cuda91/gcc/all/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda91-gnu6-llvm40
+  image: localhost:5000/gko-cuda91-gnu6-llvm40
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -210,7 +210,7 @@ build/cuda91/gcc/all/debug/static:
 
 build/cuda91/clang/all/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda91-gnu6-llvm40
+  image: localhost:5000/gko-cuda91-gnu6-llvm40
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -231,7 +231,7 @@ build/cuda91/clang/all/release/shared:
 # cuda 9.2 and friends
 build/cuda92/gcc/all/release/debug:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda92-gnu7-llvm50-intel2017
+  image: localhost:5000/gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -249,7 +249,7 @@ build/cuda92/gcc/all/release/debug:
 
 build/cuda92/intel/cuda/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda92-gnu7-llvm50-intel2017
+  image: localhost:5000/gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -270,7 +270,7 @@ build/cuda92/intel/cuda/release/static:
 # cuda 10.0 and friends
 build/cuda100/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda100-gnu7-llvm60-intel2018
+  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -288,7 +288,7 @@ build/cuda100/gcc/all/debug/shared:
 
 build/cuda100/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda100-gnu7-llvm60-intel2018
+  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -309,7 +309,7 @@ build/cuda100/clang/all/release/static:
 
 build/cuda100/intel/cuda/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda100-gnu7-llvm60-intel2018
+  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -329,7 +329,7 @@ build/cuda100/intel/cuda/release/shared:
 # cuda 10.1 and friends
 build/cuda101/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -347,7 +347,7 @@ build/cuda101/gcc/all/debug/shared:
 
 build/cuda101/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -369,7 +369,7 @@ build/cuda101/clang/all/release/static:
 # clang-cuda with cuda 10.1 and friends
 build/clang-cuda101/gcc/all/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
   variables:
     <<: *default_variables
     CUDA_COMPILER: "clang++"
@@ -388,7 +388,7 @@ build/clang-cuda101/gcc/all/release/shared:
 
 build/clang-cuda101/clang/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -410,7 +410,7 @@ build/clang-cuda101/clang/cuda/debug/static:
 # cuda 10.2 and friends
 build/cuda102/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda102-gnu8-llvm8-intel2019
+  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -428,7 +428,7 @@ build/cuda102/gcc/all/debug/shared:
 
 build/cuda102/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda102-gnu8-llvm8-intel2019
+  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -449,7 +449,7 @@ build/cuda102/clang/all/release/static:
 
 build/cuda102/intel/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda102-gnu8-llvm8-intel2019
+  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -470,7 +470,7 @@ build/cuda102/intel/cuda/debug/static:
 # cuda 11.0 and friends
 build/cuda110/gcc/cuda/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda110-gnu9-llvm9-intel2020
+  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -487,7 +487,7 @@ build/cuda110/gcc/cuda/debug/shared:
 
 build/cuda110/clang/cuda/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda110-gnu9-llvm9-intel2020
+  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -507,7 +507,7 @@ build/cuda110/clang/cuda/release/static:
 
 build/cuda110/intel/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda110-gnu9-llvm9-intel2020
+  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -528,7 +528,7 @@ build/cuda110/intel/cuda/debug/static:
 # HIP AMD
 build/amd/gcc/hip/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-amd-gnu8-llvm7
+  image: localhost:5000/gko-amd-gnu8-llvm7
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -545,7 +545,7 @@ build/amd/gcc/hip/debug/shared:
 
 build/amd/clang/hip/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-amd-gnu8-llvm7
+  image: localhost:5000/gko-amd-gnu8-llvm7
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -565,7 +565,7 @@ build/amd/clang/hip/release/static:
 # no cuda but latest gcc and clang
 build/nocuda/gcc/core/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_REFERENCE: "OFF"
@@ -580,7 +580,7 @@ build/nocuda/gcc/core/debug/static:
 
 build/nocuda/clang/core/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -596,7 +596,7 @@ build/nocuda/clang/core/release/shared:
 
 build/nocuda/intel/core/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -613,7 +613,7 @@ build/nocuda/intel/core/debug/shared:
 
 build/nocuda/gcc/omp/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -627,7 +627,7 @@ build/nocuda/gcc/omp/release/shared:
 
 build/nocuda/clang/omp/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -644,7 +644,7 @@ build/nocuda/clang/omp/debug/static:
 
 build/nocuda/intel/omp/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -683,7 +683,7 @@ build/dpcpp/cpu/release/static:
 warnings:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -704,7 +704,7 @@ warnings:
 no-circular-deps:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -725,7 +725,7 @@ no-circular-deps:
 subdir-build:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -746,7 +746,7 @@ subdir-build:
 clang-tidy:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -766,7 +766,7 @@ clang-tidy:
 iwyu:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -787,7 +787,7 @@ iwyu:
 # For short living branches or PRs, try to detect an open PR
 sonarqube_cov_:
   stage: code_quality
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
   before_script: *default_before_script
   script:
     - PR_ID=$(curl "https://api.github.com/search/issues?q=sha:${CI_COMMIT_SHA}"
@@ -828,7 +828,7 @@ sonarqube_cov_:
 # (the one that was merged).
 sonarqube_cov:
   stage: code_quality
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
   before_script: *default_before_script
   script:
     - ctest -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=COVERAGE
@@ -855,7 +855,7 @@ sonarqube_cov:
 # Deploy documentation to github-pages
 gh-pages:
   stage: deploy
-  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     PUBLIC_REPO: git@github.com:ginkgo-project/ginkgo.git
@@ -902,7 +902,7 @@ gh-pages:
 
 threadsanitizer:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - LD_PRELOAD=/usr/local/lib/libomp.so
@@ -926,7 +926,7 @@ threadsanitizer:
 
 leaksanitizer:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=LSAN
@@ -946,7 +946,7 @@ leaksanitizer:
 
 addresssanitizer:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=ASAN
@@ -966,7 +966,7 @@ addresssanitizer:
 
 undefinedsanitizer:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     # the Gold linker is required because of a linker flag issues given by UBsan
@@ -988,7 +988,7 @@ undefinedsanitizer:
 
 cudamemcheck:
   stage: QoS_tools
-  image: localhost:5000/add_hiprand_8d042b48_gko-cuda101-gnu8-llvm10-intel2019
+  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=RelWithDebInfo
@@ -1027,7 +1027,7 @@ cudamemcheck:
 
 fineci-benchmark-build:
   stage: benchmark-build
-  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BENCHMARK_SERVER: FINECI
@@ -1108,7 +1108,7 @@ fineci-benchmark-build:
 
 fineci-benchmark-cuda:
   stage: benchmark-cuda
-  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BENCHMARK_SERVER: FINECI
@@ -1146,7 +1146,7 @@ fineci-benchmark-cuda:
 
 new-issue-on-failure:
   stage: on-failure
-  image: localhost:5000/add_hiprand_8d042b48_gko-nocuda-gnu9-llvm8
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
   script: curl --request POST "https://gitlab.com/api/v4/projects/${PROJECT_ID}/issues?private_token=${BOT_ACCESS_TOKEN}&title=Error%20in%20${CI_PROJECT_NAME}%20with%20pipeline%20${CI_PIPELINE_ID}%20for%20commit%20${CI_COMMIT_SHA}&labels&description=${CI_PIPELINE_URL}"
   when: on_failure
   only:
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index 1fd6aa11c06..769da2019c5 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -183,9 +183,9 @@ class Idr : public ::testing::Test {
     std::unique_ptr<Solver::Factory> cuda_idr_factory;
     std::unique_ptr<Solver::Factory> ref_idr_factory;
 
-    gko::size_type nrhs,
+    gko::size_type nrhs;
 
-        std::unique_ptr<Mtx> x;
+    std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
     std::unique_ptr<Mtx> m;
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index 465b17c960e..a2c3025a136 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -261,7 +261,7 @@ void update_x_r_and_f(std::shared_ptr<const HipExecutor> exec,
 
 
 template <typename ValueType>
-void initialize(std::shared_ptr<const HipExecutor> exec,
+void initialize(std::shared_ptr<const HipExecutor> exec, const size_type nrhs,
                 matrix::Dense<ValueType> *m,
                 matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
                 Array<stopping_status> *stop_status)

From 584efe04cbcf0655a5853c8fe3362283bf778be5 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Fri, 20 Nov 2020 11:12:07 +0100
Subject: [PATCH 0163/1008] fix dpcpp kernels, suggestions from code review

---
 common/solver/idr_kernels.hpp.inc     |  5 +----
 dpcpp/solver/idr_kernels.dp.cpp       | 24 ++++++++++++------------
 reference/test/solver/idr_kernels.cpp |  1 -
 3 files changed, 13 insertions(+), 17 deletions(-)

diff --git a/common/solver/idr_kernels.hpp.inc b/common/solver/idr_kernels.hpp.inc
index c4ef95ccf2a..5c323dee44e 100644
--- a/common/solver/idr_kernels.hpp.inc
+++ b/common/solver/idr_kernels.hpp.inc
@@ -61,10 +61,7 @@ __global__
     __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
     ValueType *__restrict__ reduction_helper = reduction_helper_array;
 
-    __shared__ UninitializedArray<remove_complex<ValueType>, block_size>
-        reduction_helper_array_real;
-    remove_complex<ValueType> *__restrict__ reduction_helper_real =
-        reduction_helper_array_real;
+    __shared__ __restrict__ reduction_helper_real[block_size];
 
     for (size_type row = 0; row < num_rows; row++) {
         for (size_type i = 0; i < row; i++) {
diff --git a/dpcpp/solver/idr_kernels.dp.cpp b/dpcpp/solver/idr_kernels.dp.cpp
index d585de0cc9c..016df356b10 100644
--- a/dpcpp/solver/idr_kernels.dp.cpp
+++ b/dpcpp/solver/idr_kernels.dp.cpp
@@ -56,7 +56,7 @@ namespace idr {
 
 
 template <typename ValueType>
-void initialize(std::shared_ptr<const DpcppExecutor> exec,
+void initialize(std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
                 matrix::Dense<ValueType> *m,
                 matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
                 Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
@@ -65,8 +65,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const DpcppExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *m,
+void step_1(std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *m,
             const matrix::Dense<ValueType> *f,
             const matrix::Dense<ValueType> *residual,
             const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *c,
@@ -77,8 +77,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const DpcppExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *omega,
+void step_2(std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *omega,
             const matrix::Dense<ValueType> *preconditioned_vector,
             const matrix::Dense<ValueType> *c, matrix::Dense<ValueType> *u,
             const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
@@ -87,12 +87,12 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 
 
 template <typename ValueType>
-void step_3(std::shared_ptr<const DpcppExecutor> exec, const size_type k,
-            const matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *g,
-            matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *u,
-            matrix::Dense<ValueType> *m, matrix::Dense<ValueType> *f,
-            matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *residual,
-            matrix::Dense<ValueType> *x,
+void step_3(std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
+            const size_type k, const matrix::Dense<ValueType> *p,
+            matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *g_k,
+            matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
+            matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *alpha,
+            matrix::Dense<ValueType> *residual, matrix::Dense<ValueType> *x,
             const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
@@ -100,7 +100,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
 template <typename ValueType>
 void compute_omega(
-    std::shared_ptr<const DpcppExecutor> exec,
+    std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
     matrix::Dense<ValueType> *omega,
diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp
index 1c34aa8f06b..055731174e3 100644
--- a/reference/test/solver/idr_kernels.cpp
+++ b/reference/test/solver/idr_kernels.cpp
@@ -256,7 +256,6 @@ TYPED_TEST(Idr, SolvesBigDenseSystemForDivergenceCheck1)
 
     solver->apply(b.get(), x.get());
 
-
     auto one_op = gko::initialize<gko::matrix::Dense<value_type>>(
         {gko::one<value_type>()}, this->exec);
     auto neg_one_op = gko::initialize<gko::matrix::Dense<value_type>>(

From bb6c679172970ccbfc1a4027f9abef1afb5bfe33 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Fri, 20 Nov 2020 18:43:33 +0100
Subject: [PATCH 0164/1008] update windows build

---
 .github/workflows/windows-build.yml | 2 +-
 common/solver/idr_kernels.hpp.inc   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml
index 89d15aefee5..6cdc7d82462 100644
--- a/.github/workflows/windows-build.yml
+++ b/.github/workflows/windows-build.yml
@@ -51,7 +51,7 @@ jobs:
         $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
         mkdir build
         cd build
-        cmake -DCMAKE_CXX_FLAGS=/bigobj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DGINKGO_BUILD_CUDA=OFF -DGINKGO_BUILD_OMP=OFF ..
+        cmake  -DCMAKE_CXX_FLAGS=/bigobj -DCMAKE_CXX_FLAGS_DEBUG="/MDd /Zi /Ob1 /Od /RTC1" -DGINKGO_BUILD_CUDA=OFF -DGINKGO_BUILD_OMP=OFF ..
         cmake --build . -j4 --config ${{ matrix.config.build_type }}
         ctest . -C ${{ matrix.config.build_type }} --output-on-failure
     - name: install
diff --git a/common/solver/idr_kernels.hpp.inc b/common/solver/idr_kernels.hpp.inc
index 5c323dee44e..9067af54e49 100644
--- a/common/solver/idr_kernels.hpp.inc
+++ b/common/solver/idr_kernels.hpp.inc
@@ -61,7 +61,7 @@ __global__
     __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
     ValueType *__restrict__ reduction_helper = reduction_helper_array;
 
-    __shared__ __restrict__ reduction_helper_real[block_size];
+    __shared__ remove_complex<ValueType> reduction_helper_real[block_size];
 
     for (size_type row = 0; row < num_rows; row++) {
         for (size_type i = 0; i < row; i++) {

From efcabde98a8fa7382f94f7bd6e1a9cc3f2f65dc6 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Wed, 25 Nov 2020 09:55:25 +0100
Subject: [PATCH 0165/1008] slightly larger tolerance for multiple rhs tests

---
 cuda/test/solver/idr_kernels.cpp | 4 ++--
 hip/test/solver/idr_kernels.cpp  | 4 ++--
 omp/test/solver/idr_kernels.cpp  | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index 769da2019c5..4e44aa66520 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -374,8 +374,8 @@ TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     cuda_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-12);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
 }
 
 
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index e27e2c6b9fe..6f128a9691d 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -374,8 +374,8 @@ TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     hip_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-12);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
 }
 
 
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index 2a4f50e9772..6bc8e264997 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -379,8 +379,8 @@ TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     omp_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-12);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
 }
 
 

From b364f973decce9509ebf75720500458478991192 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 20 Nov 2020 15:38:18 +0100
Subject: [PATCH 0166/1008] add github bot

---
 .github/bot-base.sh                  | 22 ++++++++++
 .github/bot-pr-comment-base.sh       | 57 ++++++++++++++++++++++++++
 .github/check-format.sh              | 61 ++++++++++++++++++++++++++++
 .github/format.sh                    | 52 ++++++++++++++++++++++++
 .github/label.sh                     | 58 ++++++++++++++++++++++++++
 .github/rebase.sh                    | 27 ++++++++++++
 .github/workflows/bot-pr-comment.yml | 60 +++++++++++++++++++++++++++
 .github/workflows/bot-pr-created.yml | 19 +++++++++
 .github/workflows/bot-pr-updated.yml | 19 +++++++++
 9 files changed, 375 insertions(+)
 create mode 100644 .github/bot-base.sh
 create mode 100644 .github/bot-pr-comment-base.sh
 create mode 100755 .github/check-format.sh
 create mode 100755 .github/format.sh
 create mode 100755 .github/label.sh
 create mode 100755 .github/rebase.sh
 create mode 100644 .github/workflows/bot-pr-comment.yml
 create mode 100644 .github/workflows/bot-pr-created.yml
 create mode 100644 .github/workflows/bot-pr-updated.yml

diff --git a/.github/bot-base.sh b/.github/bot-base.sh
new file mode 100644
index 00000000000..608cd6d6308
--- /dev/null
+++ b/.github/bot-base.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+API_HEADER="Accept: application/vnd.github.v3+json"
+AUTH_HEADER="Authorization: token $GITHUB_TOKEN"
+
+api_get() {
+  curl -X GET -s -H "${AUTH_HEADER}" -H "${API_HEADER}" "$1"
+}
+
+api_post() {
+  curl -X POST -s -H "${AUTH_HEADER}" -H "${API_HEADER}" "$1" -d "$2"
+}
+
+api_patch() {
+  curl -X PATCH -s -H "${AUTH_HEADER}" -H "${API_HEADER}" "$1" -d "$2"
+}
+
+api_delete() {
+  curl -X DELETE -s -H "${AUTH_HEADER}" -H "${API_HEADER}" "$1"
+}
diff --git a/.github/bot-pr-comment-base.sh b/.github/bot-pr-comment-base.sh
new file mode 100644
index 00000000000..b37f31ce834
--- /dev/null
+++ b/.github/bot-pr-comment-base.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+source $(dirname "${BASH_SOURCE[0]}")/bot-base.sh
+
+echo -n "Collecting information on triggering event"
+ISSUE_URL=$(jq -er ".issue.url" "$GITHUB_EVENT_PATH")
+echo -n .
+USER_LOGIN=$(jq -er ".comment.user.login" "$GITHUB_EVENT_PATH")
+echo -n .
+USER_URL=$(jq -er ".comment.user.url" "$GITHUB_EVENT_PATH")
+echo -n .
+PR_URL=$(jq -er ".issue.pull_request.url" "$GITHUB_EVENT_PATH")
+echo .
+
+bot_comment() {
+  api_post $ISSUE_URL/comments "{\"body\":\"$1\"}" > /dev/null
+}
+
+bot_error() {
+  echo "$1"
+  bot_comment "Error: $1"
+  exit 1
+}
+
+# collect info on the PR we are checking
+echo -n "Collecting information on pull request"
+PR_JSON=$(api_get $PR_URL)
+echo -n .
+
+PR_MERGED=$(echo "$PR_JSON" | jq -r .merged)
+echo -n .
+BASE_REPO=$(echo "$PR_JSON" | jq -er .base.repo.full_name)
+echo -n .
+BASE_BRANCH=$(echo "$PR_JSON" | jq -er .base.ref)
+echo -n .
+HEAD_REPO=$(echo "$PR_JSON" | jq -er .head.repo.full_name)
+echo -n .
+HEAD_BRANCH=$(echo "$PR_JSON" | jq -er .head.ref)
+echo .
+
+BASE_URL="https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/$BASE_REPO"
+HEAD_URL="https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/$HEAD_REPO"
+
+# collect info on the user that invoked the bot
+echo -n "Collecting information on triggering user"
+USER_JSON=$(api_get $USER_URL)
+echo .
+
+USER_NAME=$(echo "$USER_JSON" | jq -r ".name")
+if [[ "$USER_NAME" == "null" ]]; then
+	USER_NAME=$USER_LOGIN
+fi
+USER_EMAIL=$(echo "$USER_JSON" | jq -r ".email")
+if [[ "$USER_EMAIL" == "null" ]]; then
+	USER_EMAIL="$USER_LOGIN@users.noreply.github.com"
+fi
+USER_COMBINED="$USER_NAME <$USER_EMAIL>"
diff --git a/.github/check-format.sh b/.github/check-format.sh
new file mode 100755
index 00000000000..27a97676869
--- /dev/null
+++ b/.github/check-format.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+source $(dirname "${BASH_SOURCE[0]}")/bot-base.sh
+
+echo -n "Collecting information on triggering PR"
+PR_URL=$(jq -r .pull_request.url "$GITHUB_EVENT_PATH")
+if [[ "$PR_URL" == "null" ]]; then
+  echo -n ........
+  PR_URL=$(jq -er .issue.pull_request.url "$GITHUB_EVENT_PATH")
+  echo -n .
+fi
+echo -n .
+PR_JSON=$(api_get $PR_URL)
+echo -n .
+ISSUE_URL=$(echo "$PR_JSON" | jq -er ".issue_url")
+echo -n .
+HEAD_REPO=$(echo "$PR_JSON" | jq -er .head.repo.full_name)
+echo -n .
+HEAD_BRANCH=$(echo "$PR_JSON" | jq -er .head.ref)
+echo .
+HEAD_URL="https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/$HEAD_REPO"
+
+bot_comment() {
+  api_post "$ISSUE_URL/comments" "{\"body\":\"$1\"}" > /dev/null
+}
+
+bot_error() {
+  echo "$1"
+  bot_comment "Error: $1"
+  exit 1
+}
+
+# save scripts from develop
+cd dev_tools/scripts
+cp add_license.sh update_ginkgo_header.sh format_header.sh ../../../
+cd ../../
+
+git remote add fork "${HEAD_URL}"
+git fetch fork "$HEAD_BRANCH"
+git checkout -b format-tmp-$HEAD_BRANCH "fork/$HEAD_BRANCH"
+
+# restore files from develop
+cp ../add_license.sh dev_tools/scripts/
+cp ../update_ginkgo_header.sh dev_tools/scripts/
+cp ../format_header.sh dev_tools/scripts/
+
+# format files
+dev_tools/scripts/update_ginkgo_header.sh
+find . -type f \( -name '*.cuh' -o -name '*.hpp' -o -name '*.hpp.in' -o -name '*.cpp' -o -name '*.cu' -o -name '*.hpp.inc' \) -exec clang-format-8 -i "{}" \;
+dev_tools/scripts/add_license.sh
+  
+# restore formatting scripts so they don't appear in the diff
+git checkout -- dev_tools/scripts/*.sh
+
+# replace newlines by \n
+LIST_FILES=$(git diff --name-only | sed '$!s/$/\\n/' | tr -d '\n')
+
+if [[ "$LIST_FILES" != "" ]]; then
+  bot_error "The following files need to be formatted:\n"'```'"\n$LIST_FILES\n"'```'
+  git diff
+fi
diff --git a/.github/format.sh b/.github/format.sh
new file mode 100755
index 00000000000..b703719fe49
--- /dev/null
+++ b/.github/format.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+source $(dirname "${BASH_SOURCE[0]}")/bot-pr-comment-base.sh
+
+if [[ "$PR_MERGED" == "true" ]]; then
+  bot_error "PR already merged!"
+fi
+
+git remote set-url origin "$BASE_URL"
+git remote add fork "$HEAD_URL"
+
+# make sure branches are up-to-date
+git fetch origin $BASE_BRANCH
+git fetch fork $HEAD_BRANCH
+
+git config user.email "ginkgo.library@gmail.com"
+git config user.name "ginkgo-bot"
+
+# save scripts from develop
+cd dev_tools/scripts
+cp add_license.sh update_ginkgo_header.sh format_header.sh ../../../
+cd ../../
+
+LOCAL_BRANCH=format-tmp-$HEAD_BRANCH
+git checkout -b $LOCAL_BRANCH fork/$HEAD_BRANCH
+
+# restore files from develop
+cp ../add_license.sh dev_tools/scripts/
+cp ../update_ginkgo_header.sh dev_tools/scripts/
+cp ../format_header.sh dev_tools/scripts/
+
+# format files
+dev_tools/scripts/update_ginkgo_header.sh
+find . -type f \( -name '*.cuh' -o -name '*.hpp' -o -name '*.hpp.in' -o -name '*.cpp' -o -name '*.cu' -o -name '*.hpp.inc' \) -exec clang-format-8 -i "{}" \;
+dev_tools/scripts/add_license.sh
+  
+# restore formatting scripts so they don't appear in the diff
+git checkout -- dev_tools/scripts/*.sh
+
+# check for changed files, replace newlines by \n
+LIST_FILES=$(git diff --name-only | sed '$!s/$/\\n/' | tr -d '\n')
+
+# commit changes if necessary
+if [[ "$LIST_FILES" != "" ]]; then
+  git commit -a -m "Format files
+
+Co-authored-by: $USER_COMBINED"
+  git push fork $LOCAL_BRANCH:$HEAD_BRANCH 2>&1 || bot_error "Cannot push formatted branch, are edits for maintainers allowed?"
+  bot_comment "Formatted the following files:\n"'```'"\n$LIST_FILES\n"'```'
+else
+  bot_comment "Nothing to format"
+fi
diff --git a/.github/label.sh b/.github/label.sh
new file mode 100755
index 00000000000..d6294b28239
--- /dev/null
+++ b/.github/label.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+source $(dirname "${BASH_SOURCE[0]}")/bot-base.sh
+
+echo -n "Collecting information on triggering PR"
+PR_URL=$(jq -r .pull_request.url "$GITHUB_EVENT_PATH")
+if [[ "$PR_URL" == "null" ]]; then
+  echo -n ...
+  PR_URL=$(jq -er .issue.pull_request.url "$GITHUB_EVENT_PATH")
+  echo -n .
+  ISSUE_URL=$(jq -er .issue.url "$GITHUB_EVENT_PATH")
+  echo .
+else
+  echo -n .
+  ISSUE_URL=$(jq -er .pull_request.issue_url "$GITHUB_EVENT_PATH")
+  echo .
+fi
+
+echo "Retrieving PR file list"
+PR_FILES=$(api_get "$PR_URL/files?&per_page=1000" | jq -er '.[] | .filename')
+
+echo "Retrieving PR label list"
+OLD_LABELS=$(api_get "$ISSUE_URL" | jq -er '[.labels | .[] | .name]')
+
+
+label_match() {
+  if echo "$PR_FILES" | grep -qE "$2"; then
+    echo "+[\"$1\"]"
+  fi
+}
+
+LABELS="[]"
+LABELS=$LABELS$(label_match mod:core '(^core/|^include/)')
+LABELS=$LABELS$(label_match mod:reference ^reference/)
+LABELS=$LABELS$(label_match mod:openmp ^omp/)
+LABELS=$LABELS$(label_match mod:cuda '(^cuda/|^common/)')
+LABELS=$LABELS$(label_match mod:hip '(^hip/|^common/)')
+LABELS=$LABELS$(label_match mod:dpcpp ^dpcpp/)
+LABELS=$LABELS$(label_match reg:benchmarking ^benchmark/)
+LABELS=$LABELS$(label_match reg:example ^examples/)
+LABELS=$LABELS$(label_match reg:build '(cm|CM)ake')
+LABELS=$LABELS$(label_match reg:ci-cd '.yml$')
+LABELS=$LABELS$(label_match reg:documentation ^doc/)
+LABELS=$LABELS$(label_match reg:testing /test/)
+LABELS=$LABELS$(label_match reg:helper-scripts ^dev_tools/)
+LABELS=$LABELS$(label_match type:factorization /factorization/)
+LABELS=$LABELS$(label_match type:matrix-format /matrix/)
+LABELS=$LABELS$(label_match type:multigrid /multigrid/)
+LABELS=$LABELS$(label_match type:preconditioner /preconditioner/)
+LABELS=$LABELS$(label_match type:reordering /reorder/)
+LABELS=$LABELS$(label_match type:solver /solver/)
+LABELS=$LABELS$(label_match type:stopping-criteria /stop/)
+
+# if all mod: labels present: replace by mod:all
+LABELS=$(echo "$LABELS" | sed 's/.*mod:.*mod:.*mod:.*mod:.*mod:[^"]*"\]/[]+["mod:all"]/')
+
+PATCH_BODY=$(jq -rn "{labels:($OLD_LABELS + $LABELS | unique)}")
+api_patch "$ISSUE_URL" "$PATCH_BODY" > /dev/null
diff --git a/.github/rebase.sh b/.github/rebase.sh
new file mode 100755
index 00000000000..9bbcd04feb6
--- /dev/null
+++ b/.github/rebase.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+source $(dirname "${BASH_SOURCE[0]}")/bot-pr-comment-base.sh
+
+if [[ "$PR_MERGED" == "true" ]]; then
+  bot_error "PR already merged!"
+fi
+
+git remote add base "$BASE_URL"
+git remote add fork "$HEAD_URL"
+
+git fetch base $BASE_BRANCH
+git fetch fork $HEAD_BRANCH
+
+git config user.email "$USER_EMAIL"
+git config user.name "$USER_NAME"
+
+LOCAL_BRANCH=rebase-tmp-$HEAD_BRANCH
+git checkout -b $LOCAL_BRANCH fork/$HEAD_BRANCH
+
+# do the rebase
+git rebase base/$BASE_BRANCH 2>&1 || bot_error "Rebasing failed"
+
+# push back
+git push --force-with-lease fork $LOCAL_BRANCH:$HEAD_BRANCH 2>&1 || bot_error "Cannot push rebased branch, are edits for maintainers allowed?"
+
+bot_comment "Rebasing succeeded"
diff --git a/.github/workflows/bot-pr-comment.yml b/.github/workflows/bot-pr-comment.yml
new file mode 100644
index 00000000000..668be587240
--- /dev/null
+++ b/.github/workflows/bot-pr-comment.yml
@@ -0,0 +1,60 @@
+on: 
+  issue_comment:
+    types: [created]
+name: OnCommentPR
+jobs:
+  label:
+    runs-on: ubuntu-latest
+    if: github.event.issue.pull_request != '' && github.event.comment.body == 'label!' && (github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER')
+    steps:
+    - name: Checkout the latest code
+      uses: actions/checkout@v2
+      with:
+        ref: develop
+        fetch-depth: 0
+    - name: Add appropriate labels
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: .github/label.sh
+  check_format:
+    name: check-format
+    runs-on: ubuntu-18.04
+    if: github.event.issue.pull_request != '' && github.event.comment.body == 'check-format!' && (github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER')
+    steps:
+    - name: Checkout the latest code
+      uses: actions/checkout@v2
+      with:
+        ref: develop
+        fetch-depth: 0
+    - name: Check the code formatting
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: .github/check-format.sh
+  format:
+    name: format
+    runs-on: ubuntu-18.04
+    if: github.event.issue.pull_request != '' && github.event.comment.body == 'format!' && (github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER')
+    steps:
+    - name: Checkout the latest code
+      uses: actions/checkout@v2
+      with:
+        ref: develop
+        fetch-depth: 0
+    - name: Format the code
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: .github/format.sh
+  rebase:
+    name: rebase
+    if: github.event.issue.pull_request != '' && github.event.comment.body == 'rebase!' && (github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER')
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout the latest code
+      uses: actions/checkout@v2
+      with:
+        ref: develop
+        fetch-depth: 0
+    - name: Automatic Rebase
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: .github/rebase.sh
diff --git a/.github/workflows/bot-pr-created.yml b/.github/workflows/bot-pr-created.yml
new file mode 100644
index 00000000000..75a6a7e4564
--- /dev/null
+++ b/.github/workflows/bot-pr-created.yml
@@ -0,0 +1,19 @@
+on:
+  pull_request_target:
+    types: [opened]
+
+name: OnNewPR
+jobs:
+  label:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.author_association == 'COLLABORATOR' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'OWNER'
+    steps:
+      - name: Checkout the latest code
+        uses: actions/checkout@v2
+        with:
+          ref: develop
+          fetch-depth: 0
+      - name: Add appropriate labels
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: .github/label.sh
diff --git a/.github/workflows/bot-pr-updated.yml b/.github/workflows/bot-pr-updated.yml
new file mode 100644
index 00000000000..e127ce5c53d
--- /dev/null
+++ b/.github/workflows/bot-pr-updated.yml
@@ -0,0 +1,19 @@
+on:
+  pull_request_target:
+    types: [opened,synchronize]
+
+name: OnSyncPR
+jobs:
+  check-format:
+    runs-on: ubuntu-18.04
+    if: github.event.pull_request.author_association == 'COLLABORATOR' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'OWNER'
+    steps:
+      - name: Checkout the latest code
+        uses: actions/checkout@v2
+        with:
+          ref: develop
+          fetch-depth: 0
+      - name: Check for formatting changes
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: .github/check-format.sh

From 78f2823d4e3e8c0e35fa2f184bf623595f04863e Mon Sep 17 00:00:00 2001
From: Aditya <aditya.kashi@yahoo.com>
Date: Tue, 24 Nov 2020 10:20:23 +0100
Subject: [PATCH 0167/1008] Fix number of modules for mod:all

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 .github/label.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/label.sh b/.github/label.sh
index d6294b28239..e6f2c79029d 100755
--- a/.github/label.sh
+++ b/.github/label.sh
@@ -52,7 +52,7 @@ LABELS=$LABELS$(label_match type:solver /solver/)
 LABELS=$LABELS$(label_match type:stopping-criteria /stop/)
 
 # if all mod: labels present: replace by mod:all
-LABELS=$(echo "$LABELS" | sed 's/.*mod:.*mod:.*mod:.*mod:.*mod:[^"]*"\]/[]+["mod:all"]/')
+LABELS=$(echo "$LABELS" | sed 's/.*mod:.*mod:.*mod:.*mod:.*mod:.*mod:[^"]*"\]/[]+["mod:all"]/')
 
 PATCH_BODY=$(jq -rn "{labels:($OLD_LABELS + $LABELS | unique)}")
 api_patch "$ISSUE_URL" "$PATCH_BODY" > /dev/null

From 2614fd60192266b497636aa8e88e7d87f3f699c2 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 27 Nov 2020 14:28:32 +0100
Subject: [PATCH 0168/1008] improve security and structure

---
 ...{bot-pr-comment-base.sh => bot-pr-base.sh} | 60 ++++++++++-------
 .github/bot-pr-format-base.sh                 | 41 ++++++++++++
 .github/check-format.sh                       | 66 ++++---------------
 .github/format.sh                             | 41 +-----------
 .github/label.sh                              | 16 +----
 .github/rebase.sh                             | 10 +--
 .github/workflows/bot-pr-comment.yml          | 20 ++++--
 .github/workflows/bot-pr-updated.yml          |  8 ++-
 dev_tools/scripts/format_header.sh            | 14 ++--
 9 files changed, 124 insertions(+), 152 deletions(-)
 rename .github/{bot-pr-comment-base.sh => bot-pr-base.sh} (51%)
 create mode 100644 .github/bot-pr-format-base.sh

diff --git a/.github/bot-pr-comment-base.sh b/.github/bot-pr-base.sh
similarity index 51%
rename from .github/bot-pr-comment-base.sh
rename to .github/bot-pr-base.sh
index b37f31ce834..421ef773e2d 100644
--- a/.github/bot-pr-comment-base.sh
+++ b/.github/bot-pr-base.sh
@@ -1,34 +1,32 @@
 #!/bin/bash
 
-source $(dirname "${BASH_SOURCE[0]}")/bot-base.sh
+source .github/bot-base.sh
 
-echo -n "Collecting information on triggering event"
-ISSUE_URL=$(jq -er ".issue.url" "$GITHUB_EVENT_PATH")
-echo -n .
-USER_LOGIN=$(jq -er ".comment.user.login" "$GITHUB_EVENT_PATH")
-echo -n .
-USER_URL=$(jq -er ".comment.user.url" "$GITHUB_EVENT_PATH")
+echo -n "Collecting information on triggering PR"
+PR_URL=$(jq -r .pull_request.url "$GITHUB_EVENT_PATH")
+if [[ "$PR_URL" == "null" ]]; then
+  # if this was triggered by an issue comment: get PR and commenter
+  echo -n .............
+  PR_URL=$(jq -er .issue.pull_request.url "$GITHUB_EVENT_PATH")
+  echo -n .
+  USER_LOGIN=$(jq -er ".comment.user.login" "$GITHUB_EVENT_PATH")
+  echo -n .
+  USER_URL=$(jq -er ".comment.user.url" "$GITHUB_EVENT_PATH")
+  echo -n .
+else
+  # else it was triggered by a PR sync: get PR creator
+  USER_LOGIN=$(jq -er ".pull_request.user.login" "$GITHUB_EVENT_PATH")
+  echo -n .
+  USER_URL=$(jq -er ".pull_request.user.url" "$GITHUB_EVENT_PATH")
+  echo -n .
+fi
 echo -n .
-PR_URL=$(jq -er ".issue.pull_request.url" "$GITHUB_EVENT_PATH")
-echo .
-
-bot_comment() {
-  api_post $ISSUE_URL/comments "{\"body\":\"$1\"}" > /dev/null
-}
-
-bot_error() {
-  echo "$1"
-  bot_comment "Error: $1"
-  exit 1
-}
-
-# collect info on the PR we are checking
-echo -n "Collecting information on pull request"
 PR_JSON=$(api_get $PR_URL)
 echo -n .
-
 PR_MERGED=$(echo "$PR_JSON" | jq -r .merged)
 echo -n .
+ISSUE_URL=$(echo "$PR_JSON" | jq -er ".issue_url")
+echo -n .
 BASE_REPO=$(echo "$PR_JSON" | jq -er .base.repo.full_name)
 echo -n .
 BASE_BRANCH=$(echo "$PR_JSON" | jq -er .base.ref)
@@ -41,6 +39,18 @@ echo .
 BASE_URL="https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/$BASE_REPO"
 HEAD_URL="https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/$HEAD_REPO"
 
+JOB_URL="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+
+bot_comment() {
+  api_post "$ISSUE_URL/comments" "{\"body\":\"$1\"}" > /dev/null
+}
+
+bot_error() {
+  echo "$1"
+  bot_comment "Error: $1"
+  exit 1
+}
+
 # collect info on the user that invoked the bot
 echo -n "Collecting information on triggering user"
 USER_JSON=$(api_get $USER_URL)
@@ -55,3 +65,7 @@ if [[ "$USER_EMAIL" == "null" ]]; then
 	USER_EMAIL="$USER_LOGIN@users.noreply.github.com"
 fi
 USER_COMBINED="$USER_NAME <$USER_EMAIL>"
+
+if [[ "$PR_MERGED" == "true" ]]; then
+  bot_error "PR already merged!"
+fi
diff --git a/.github/bot-pr-format-base.sh b/.github/bot-pr-format-base.sh
new file mode 100644
index 00000000000..d89d3f2ee2a
--- /dev/null
+++ b/.github/bot-pr-format-base.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+source .github/bot-pr-base.sh
+
+EXTENSION_REGEX='\.(cuh?|hpp|hpp\.inc?|cpp)$'
+FORMAT_HEADER_REGEX='^(benchmark|core|cuda|hip|include/ginkgo/core|omp|reference)/'
+FORMAT_REGEX='^(common|examples|test_install)/'
+
+echo "Retrieving PR file list"
+PR_FILES="$(api_get "$PR_URL/files?&per_page=1000" | jq -er '.[] | .filename')"
+TO_FORMAT="$(echo "$PR_FILES" | grep -E $EXTENSION_REGEX || true)"
+
+git remote add fork "$HEAD_URL"
+git fetch fork "$HEAD_BRANCH"
+
+git config user.email "ginkgo.library@gmail.com"
+git config user.name "ginkgo-bot"
+
+# save scripts from develop
+pushd dev_tools/scripts
+cp add_license.sh format_header.sh update_ginkgo_header.sh /tmp
+popd
+
+# checkout current PR head
+LOCAL_BRANCH=format-tmp-$HEAD_BRANCH
+git checkout -b $LOCAL_BRANCH fork/$HEAD_BRANCH
+
+# restore files from develop
+cp /tmp/add_license.sh dev_tools/scripts/
+cp /tmp/format_header.sh dev_tools/scripts/
+cp /tmp/update_ginkgo_header.sh dev_tools/scripts/
+
+# format files
+CLANG_FORMAT=clang-format-8
+dev_tools/scripts/add_license.sh
+dev_tools/scripts/update_ginkgo_header.sh
+for f in $(echo "$TO_FORMAT" | grep -E $FORMAT_HEADER_REGEX); do dev_tools/scripts/format_header.sh "$f"; done
+for f in $(echo "$TO_FORMAT" | grep -E $FORMAT_REGEX); do "$CLANG_FORMAT" -i -style=file "$f"; done
+
+# restore formatting scripts so they don't appear in the diff
+git checkout -- dev_tools/scripts/*.sh
diff --git a/.github/check-format.sh b/.github/check-format.sh
index 27a97676869..704a78690b6 100755
--- a/.github/check-format.sh
+++ b/.github/check-format.sh
@@ -1,61 +1,19 @@
 #!/bin/bash
 
-source $(dirname "${BASH_SOURCE[0]}")/bot-base.sh
+cp .github/bot-pr-format-base.sh /tmp
+source /tmp/bot-pr-format-base.sh
 
-echo -n "Collecting information on triggering PR"
-PR_URL=$(jq -r .pull_request.url "$GITHUB_EVENT_PATH")
-if [[ "$PR_URL" == "null" ]]; then
-  echo -n ........
-  PR_URL=$(jq -er .issue.pull_request.url "$GITHUB_EVENT_PATH")
-  echo -n .
-fi
-echo -n .
-PR_JSON=$(api_get $PR_URL)
-echo -n .
-ISSUE_URL=$(echo "$PR_JSON" | jq -er ".issue_url")
-echo -n .
-HEAD_REPO=$(echo "$PR_JSON" | jq -er .head.repo.full_name)
-echo -n .
-HEAD_BRANCH=$(echo "$PR_JSON" | jq -er .head.ref)
-echo .
-HEAD_URL="https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/$HEAD_REPO"
-
-bot_comment() {
-  api_post "$ISSUE_URL/comments" "{\"body\":\"$1\"}" > /dev/null
-}
-
-bot_error() {
-  echo "$1"
-  bot_comment "Error: $1"
-  exit 1
-}
-
-# save scripts from develop
-cd dev_tools/scripts
-cp add_license.sh update_ginkgo_header.sh format_header.sh ../../../
-cd ../../
-
-git remote add fork "${HEAD_URL}"
-git fetch fork "$HEAD_BRANCH"
-git checkout -b format-tmp-$HEAD_BRANCH "fork/$HEAD_BRANCH"
-
-# restore files from develop
-cp ../add_license.sh dev_tools/scripts/
-cp ../update_ginkgo_header.sh dev_tools/scripts/
-cp ../format_header.sh dev_tools/scripts/
-
-# format files
-dev_tools/scripts/update_ginkgo_header.sh
-find . -type f \( -name '*.cuh' -o -name '*.hpp' -o -name '*.hpp.in' -o -name '*.cpp' -o -name '*.cu' -o -name '*.hpp.inc' \) -exec clang-format-8 -i "{}" \;
-dev_tools/scripts/add_license.sh
-  
-# restore formatting scripts so they don't appear in the diff
-git checkout -- dev_tools/scripts/*.sh
-
-# replace newlines by \n
+# check for changed files, replace newlines by \n
 LIST_FILES=$(git diff --name-only | sed '$!s/$/\\n/' | tr -d '\n')
 
+git diff > /tmp/format.patch
+mv /tmp/format.patch .
+
 if [[ "$LIST_FILES" != "" ]]; then
-  bot_error "The following files need to be formatted:\n"'```'"\n$LIST_FILES\n"'```'
-  git diff
+  MESSAGE="The following files need to be formatted:\n"'```'"\n$LIST_FILES\n"'```'
+  MESSAGE="$MESSAGE\nYou can find a formatting patch under **Artifacts** [here]"
+  MESSAGE="$MESSAGE($JOB_URL) or run "'`format!` if you have write access to Ginkgo'
+  bot_error "$MESSAGE"
+else
+  bot_comment "No formatting necessary"
 fi
diff --git a/.github/format.sh b/.github/format.sh
index b703719fe49..581bd1bdb38 100755
--- a/.github/format.sh
+++ b/.github/format.sh
@@ -1,41 +1,7 @@
 #!/bin/bash
 
-source $(dirname "${BASH_SOURCE[0]}")/bot-pr-comment-base.sh
-
-if [[ "$PR_MERGED" == "true" ]]; then
-  bot_error "PR already merged!"
-fi
-
-git remote set-url origin "$BASE_URL"
-git remote add fork "$HEAD_URL"
-
-# make sure branches are up-to-date
-git fetch origin $BASE_BRANCH
-git fetch fork $HEAD_BRANCH
-
-git config user.email "ginkgo.library@gmail.com"
-git config user.name "ginkgo-bot"
-
-# save scripts from develop
-cd dev_tools/scripts
-cp add_license.sh update_ginkgo_header.sh format_header.sh ../../../
-cd ../../
-
-LOCAL_BRANCH=format-tmp-$HEAD_BRANCH
-git checkout -b $LOCAL_BRANCH fork/$HEAD_BRANCH
-
-# restore files from develop
-cp ../add_license.sh dev_tools/scripts/
-cp ../update_ginkgo_header.sh dev_tools/scripts/
-cp ../format_header.sh dev_tools/scripts/
-
-# format files
-dev_tools/scripts/update_ginkgo_header.sh
-find . -type f \( -name '*.cuh' -o -name '*.hpp' -o -name '*.hpp.in' -o -name '*.cpp' -o -name '*.cu' -o -name '*.hpp.inc' \) -exec clang-format-8 -i "{}" \;
-dev_tools/scripts/add_license.sh
-  
-# restore formatting scripts so they don't appear in the diff
-git checkout -- dev_tools/scripts/*.sh
+cp .github/bot-pr-format-base.sh /tmp
+source /tmp/bot-pr-format-base.sh
 
 # check for changed files, replace newlines by \n
 LIST_FILES=$(git diff --name-only | sed '$!s/$/\\n/' | tr -d '\n')
@@ -46,7 +12,6 @@ if [[ "$LIST_FILES" != "" ]]; then
 
 Co-authored-by: $USER_COMBINED"
   git push fork $LOCAL_BRANCH:$HEAD_BRANCH 2>&1 || bot_error "Cannot push formatted branch, are edits for maintainers allowed?"
-  bot_comment "Formatted the following files:\n"'```'"\n$LIST_FILES\n"'```'
 else
-  bot_comment "Nothing to format"
+  bot_comment "No formatting necessary"
 fi
diff --git a/.github/label.sh b/.github/label.sh
index e6f2c79029d..3cd5023829b 100755
--- a/.github/label.sh
+++ b/.github/label.sh
@@ -1,20 +1,6 @@
 #!/bin/bash
 
-source $(dirname "${BASH_SOURCE[0]}")/bot-base.sh
-
-echo -n "Collecting information on triggering PR"
-PR_URL=$(jq -r .pull_request.url "$GITHUB_EVENT_PATH")
-if [[ "$PR_URL" == "null" ]]; then
-  echo -n ...
-  PR_URL=$(jq -er .issue.pull_request.url "$GITHUB_EVENT_PATH")
-  echo -n .
-  ISSUE_URL=$(jq -er .issue.url "$GITHUB_EVENT_PATH")
-  echo .
-else
-  echo -n .
-  ISSUE_URL=$(jq -er .pull_request.issue_url "$GITHUB_EVENT_PATH")
-  echo .
-fi
+source .github/bot-pr-base.sh
 
 echo "Retrieving PR file list"
 PR_FILES=$(api_get "$PR_URL/files?&per_page=1000" | jq -er '.[] | .filename')
diff --git a/.github/rebase.sh b/.github/rebase.sh
index 9bbcd04feb6..6377ae4030e 100755
--- a/.github/rebase.sh
+++ b/.github/rebase.sh
@@ -1,10 +1,6 @@
 #!/bin/bash
 
-source $(dirname "${BASH_SOURCE[0]}")/bot-pr-comment-base.sh
-
-if [[ "$PR_MERGED" == "true" ]]; then
-  bot_error "PR already merged!"
-fi
+source .github/bot-pr-base.sh
 
 git remote add base "$BASE_URL"
 git remote add fork "$HEAD_URL"
@@ -19,9 +15,7 @@ LOCAL_BRANCH=rebase-tmp-$HEAD_BRANCH
 git checkout -b $LOCAL_BRANCH fork/$HEAD_BRANCH
 
 # do the rebase
-git rebase base/$BASE_BRANCH 2>&1 || bot_error "Rebasing failed"
+git rebase base/$BASE_BRANCH 2>&1 || bot_error "Rebase failed, see the related [Action]($JOB_URL) for details"
 
 # push back
 git push --force-with-lease fork $LOCAL_BRANCH:$HEAD_BRANCH 2>&1 || bot_error "Cannot push rebased branch, are edits for maintainers allowed?"
-
-bot_comment "Rebasing succeeded"
diff --git a/.github/workflows/bot-pr-comment.yml b/.github/workflows/bot-pr-comment.yml
index 668be587240..da198121140 100644
--- a/.github/workflows/bot-pr-comment.yml
+++ b/.github/workflows/bot-pr-comment.yml
@@ -1,4 +1,4 @@
-on: 
+on:
   issue_comment:
     types: [created]
 name: OnCommentPR
@@ -15,7 +15,7 @@ jobs:
     - name: Add appropriate labels
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: .github/label.sh
+      run: cp .github/label.sh /tmp && /tmp/label.sh
   check_format:
     name: check-format
     runs-on: ubuntu-18.04
@@ -26,10 +26,16 @@ jobs:
       with:
         ref: develop
         fetch-depth: 0
-    - name: Check the code formatting
+    - name: Check for formatting changes
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: .github/check-format.sh
+      run: cp .github/check-format.sh /tmp && /tmp/check-format.sh
+    - name: Upload code formatting patch
+      if: failure()
+      uses: actions/upload-artifact@v2
+      with:
+          name: patch
+          path: format.patch
   format:
     name: format
     runs-on: ubuntu-18.04
@@ -40,10 +46,10 @@ jobs:
       with:
         ref: develop
         fetch-depth: 0
-    - name: Format the code
+    - name: Commit formatting changes
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: .github/format.sh
+      run: cp .github/format.sh /tmp && /tmp/format.sh
   rebase:
     name: rebase
     if: github.event.issue.pull_request != '' && github.event.comment.body == 'rebase!' && (github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER')
@@ -57,4 +63,4 @@ jobs:
     - name: Automatic Rebase
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: .github/rebase.sh
+      run: cp .github/rebase.sh /tmp && /tmp/rebase.sh
diff --git a/.github/workflows/bot-pr-updated.yml b/.github/workflows/bot-pr-updated.yml
index e127ce5c53d..44d78473179 100644
--- a/.github/workflows/bot-pr-updated.yml
+++ b/.github/workflows/bot-pr-updated.yml
@@ -16,4 +16,10 @@ jobs:
       - name: Check for formatting changes
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: .github/check-format.sh
+        run: cp .github/check-format.sh /tmp && /tmp/check-format.sh
+      - name: Upload code formatting patch
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: patch
+          path: format.patch
diff --git a/dev_tools/scripts/format_header.sh b/dev_tools/scripts/format_header.sh
index c250af38a85..1b2b6f76966 100755
--- a/dev_tools/scripts/format_header.sh
+++ b/dev_tools/scripts/format_header.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+CLANG_FORMAT=${CLANG_FORMAT:="clang-format"}
+
 convert_header () {
     local regex="^(#include )(<|\")(.*)(\"|>)$"
     if [[ $@ =~ ${regex} ]]; then
@@ -10,7 +12,7 @@ convert_header () {
             else
                 echo "#include \"${header_file}\""
             fi
-        elif [ "${header_file}" = "matrices/config.hpp" ]; then 
+        elif [ "${header_file}" = "matrices/config.hpp" ]; then
             echo "#include \"${header_file}\""
         else
             echo "#include <${header_file}>"
@@ -23,7 +25,7 @@ convert_header () {
 get_header_def () {
     local regex="\.(hpp|cuh)"
     if [[ $@ =~ $regex ]]; then
-        local def=$(echo "$@" | sed -E "s~include/ginkgo/~~g;s~/|\.~_~g")
+        local def=$(echo "$@" | sed -E "s~include/ginkgo/~PUBLIC_~g;s~/|\.~_~g")
         def=$(echo GKO_${def^^}_)
         echo $def
     else
@@ -249,7 +251,7 @@ fi
 # Write the main header and give warnning if there are multiple matches
 if [ -f "${BEFORE}" ]; then
     # sort or remove the duplication
-    clang-format -i -style=file ${BEFORE}
+    "${CLANG_FORMAT}" -i -style=file ${BEFORE}
     if [ $(wc -l < ${BEFORE}) -gt "1" ]; then
         echo "Warning $1: there are multiple main header matchings"
     fi
@@ -261,7 +263,7 @@ if [ -f "${BEFORE}" ]; then
     rm "${BEFORE}"
 fi
 
-# Arrange the remain files and give 
+# Arrange the remain files and give
 if [ -f "${CONTENT}" ]; then
     add_regroup
     if [ "${HAS_HIP_RUNTIME}" = "true" ]; then
@@ -273,14 +275,14 @@ if [ -f "${CONTENT}" ]; then
         if [[ "${LAST_NONEMPTY}" =~ $ENDIF_REX ]]; then
             head -n -1 temp > ${CONTENT}
             echo "#endif  // $HEADER_DEF" >> ${CONTENT}
-        else 
+        else
             echo "Warning $1: Found the begin header_def but did not find the end of header_def"
             cat temp > ${CONTENT}
         fi
     else
         cat temp > "${CONTENT}"
     fi
-    clang-format -i -style=file "${CONTENT}"
+    "${CLANG_FORMAT}" -i -style=file "${CONTENT}"
     rm temp
     remove_regroup
     PREV_INC=0

From cabfbe84cb37268a09b480e487b216b880d7a386 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 2 Dec 2020 11:49:42 +0100
Subject: [PATCH 0169/1008] format everything

---
 benchmark/solver/solver.cpp                      |  1 -
 benchmark/utils/preconditioners.hpp              |  1 -
 core/base/allocator.hpp                          |  2 +-
 core/base/utils.hpp                              | 13 +++++++------
 core/device_hooks/dpcpp_hooks.cpp                |  1 -
 core/preconditioner/isai_kernels.hpp             |  4 +++-
 core/test/utils/unsort_matrix.hpp                |  1 +
 cuda/base/types.hpp                              |  6 +++---
 cuda/factorization/factorization_kernels.cu      |  1 +
 .../par_ilut_approx_filter_kernel.cu             |  1 -
 cuda/factorization/par_ilut_filter_kernel.cu     |  2 +-
 cuda/solver/idr_kernels.cu                       |  2 +-
 cuda/test/reorder/rcm_kernels.cpp                |  1 -
 dev_tools/scripts/config                         | 16 ++++++++++++++++
 dev_tools/scripts/regroup                        |  2 +-
 examples/ginkgo-overhead/ginkgo-overhead.cpp     |  3 ++-
 hip/base/types.hip.hpp                           |  4 ++--
 .../par_ilut_approx_filter_kernel.hip.cpp        |  5 ++---
 hip/factorization/par_ilut_filter_kernel.hip.cpp |  2 +-
 hip/factorization/par_ilut_select_kernel.hip.cpp |  4 ++--
 hip/solver/idr_kernels.hip.cpp                   |  2 +-
 include/ginkgo/core/base/abstract_factory.hpp    |  6 +++---
 include/ginkgo/core/base/array.hpp               |  6 +++---
 include/ginkgo/core/base/combination.hpp         |  6 +++---
 include/ginkgo/core/base/composition.hpp         |  6 +++---
 include/ginkgo/core/base/dim.hpp                 |  6 +++---
 include/ginkgo/core/base/exception.hpp           |  6 +++---
 include/ginkgo/core/base/exception_helpers.hpp   |  6 +++---
 include/ginkgo/core/base/executor.hpp            |  6 +++---
 include/ginkgo/core/base/lin_op.hpp              |  6 +++---
 include/ginkgo/core/base/math.hpp                |  6 +++---
 .../ginkgo/core/base/matrix_assembly_data.hpp    |  6 +++---
 include/ginkgo/core/base/matrix_data.hpp         |  6 +++---
 include/ginkgo/core/base/mtx_io.hpp              |  6 +++---
 include/ginkgo/core/base/name_demangling.hpp     |  6 +++---
 include/ginkgo/core/base/perturbation.hpp        |  6 +++---
 include/ginkgo/core/base/polymorphic_object.hpp  |  6 +++---
 include/ginkgo/core/base/range.hpp               |  6 +++---
 include/ginkgo/core/base/range_accessors.hpp     |  6 +++---
 include/ginkgo/core/base/std_extensions.hpp      |  6 +++---
 include/ginkgo/core/base/types.hpp               |  6 +++---
 include/ginkgo/core/base/utils.hpp               |  6 +++---
 include/ginkgo/core/base/version.hpp             |  6 +++---
 include/ginkgo/core/factorization/ilu.hpp        |  6 +++---
 include/ginkgo/core/factorization/par_ict.hpp    |  6 +++---
 include/ginkgo/core/factorization/par_ilu.hpp    |  6 +++---
 include/ginkgo/core/factorization/par_ilut.hpp   |  6 +++---
 include/ginkgo/core/log/convergence.hpp          |  6 +++---
 include/ginkgo/core/log/logger.hpp               |  6 +++---
 include/ginkgo/core/log/papi.hpp                 |  6 +++---
 include/ginkgo/core/log/record.hpp               |  6 +++---
 include/ginkgo/core/log/stream.hpp               |  6 +++---
 include/ginkgo/core/matrix/coo.hpp               |  6 +++---
 include/ginkgo/core/matrix/csr.hpp               |  6 +++---
 include/ginkgo/core/matrix/dense.hpp             |  6 +++---
 include/ginkgo/core/matrix/diagonal.hpp          |  6 +++---
 include/ginkgo/core/matrix/ell.hpp               |  6 +++---
 include/ginkgo/core/matrix/hybrid.hpp            |  6 +++---
 include/ginkgo/core/matrix/identity.hpp          |  6 +++---
 include/ginkgo/core/matrix/permutation.hpp       |  6 +++---
 include/ginkgo/core/matrix/sellp.hpp             |  6 +++---
 include/ginkgo/core/matrix/sparsity_csr.hpp      |  6 +++---
 include/ginkgo/core/preconditioner/ilu.hpp       |  6 +++---
 include/ginkgo/core/preconditioner/isai.hpp      |  6 +++---
 include/ginkgo/core/preconditioner/jacobi.hpp    | 14 +++++++-------
 include/ginkgo/core/reorder/rcm.hpp              |  6 +++---
 include/ginkgo/core/reorder/reordering_base.hpp  |  6 +++---
 include/ginkgo/core/solver/bicg.hpp              | 11 ++++++-----
 include/ginkgo/core/solver/bicgstab.hpp          |  6 +++---
 include/ginkgo/core/solver/cg.hpp                |  6 +++---
 include/ginkgo/core/solver/cgs.hpp               |  6 +++---
 include/ginkgo/core/solver/fcg.hpp               |  6 +++---
 include/ginkgo/core/solver/gmres.hpp             |  6 +++---
 include/ginkgo/core/solver/idr.hpp               |  6 +++---
 include/ginkgo/core/solver/ir.hpp                |  6 +++---
 include/ginkgo/core/solver/lower_trs.hpp         |  6 +++---
 include/ginkgo/core/solver/upper_trs.hpp         |  6 +++---
 include/ginkgo/core/stop/combined.hpp            |  6 +++---
 include/ginkgo/core/stop/criterion.hpp           |  6 +++---
 include/ginkgo/core/stop/iteration.hpp           |  6 +++---
 include/ginkgo/core/stop/residual_norm.hpp       |  6 +++---
 .../ginkgo/core/stop/residual_norm_reduction.hpp |  6 +++---
 include/ginkgo/core/stop/stopping_status.hpp     |  6 +++---
 include/ginkgo/core/stop/time.hpp                |  6 +++---
 include/ginkgo/core/synthesizer/containers.hpp   |  6 +++---
 omp/solver/idr_kernels.cpp                       |  2 +-
 omp/test/solver/idr_kernels.cpp                  |  4 ++--
 reference/components/csr_spgeam.hpp              |  6 +++---
 reference/reorder/rcm_kernels.cpp                |  1 -
 reference/solver/idr_kernels.cpp                 |  2 +-
 reference/test/preconditioner/isai_kernels.cpp   |  4 ++--
 reference/test/reorder/rcm_kernels.cpp           |  2 +-
 test_install/test_install.cpp                    | 12 ++++++------
 test_install/test_install_cuda.cu                | 10 ++++------
 94 files changed, 264 insertions(+), 250 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 937244e4c2a..11169effae8 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -48,7 +48,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/loggers.hpp"
 #include "benchmark/utils/overhead_linop.hpp"
 #include "benchmark/utils/preconditioners.hpp"
-#include "ginkgo/core/preconditioner/isai.hpp"
 
 
 // some Ginkgo shortcuts
diff --git a/benchmark/utils/preconditioners.hpp b/benchmark/utils/preconditioners.hpp
index 0542a72da4b..4ec84d9acd5 100644
--- a/benchmark/utils/preconditioners.hpp
+++ b/benchmark/utils/preconditioners.hpp
@@ -30,7 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-
 #ifndef GKO_BENCHMARK_UTILS_PRECONDITIONERS_HPP_
 #define GKO_BENCHMARK_UTILS_PRECONDITIONERS_HPP_
 
diff --git a/core/base/allocator.hpp b/core/base/allocator.hpp
index 3f618974b46..6a654fc56d6 100644
--- a/core/base/allocator.hpp
+++ b/core/base/allocator.hpp
@@ -172,4 +172,4 @@ using unordered_map =
 
 }  // namespace gko
 
-#endif  // GKO_CORE_BASE_ALLOCATOR_HPP_
\ No newline at end of file
+#endif  // GKO_CORE_BASE_ALLOCATOR_HPP_
diff --git a/core/base/utils.hpp b/core/base/utils.hpp
index ace930bd116..9842917b19d 100644
--- a/core/base/utils.hpp
+++ b/core/base/utils.hpp
@@ -30,8 +30,11 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_INTERNAL_CORE_BASE_UTILS_HPP_
-#define GKO_INTERNAL_CORE_BASE_UTILS_HPP_
+#ifndef GKO_CORE_BASE_UTILS_HPP_
+#define GKO_CORE_BASE_UTILS_HPP_
+
+
+#include <ginkgo/core/base/utils.hpp>
 
 
 #include <memory>
@@ -40,7 +43,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/polymorphic_object.hpp>
 #include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 
 
@@ -64,8 +66,7 @@ namespace detail {
 
 
 template <typename Dest>
-struct conversion_sort_helper {
-};
+struct conversion_sort_helper {};
 
 template <typename ValueType, typename IndexType>
 struct conversion_sort_helper<matrix::Csr<ValueType, IndexType>> {
@@ -222,4 +223,4 @@ std::shared_ptr<const Dest> convert_to_with_sorting(
 }  // namespace gko
 
 
-#endif  // GKO_INTERNAL_CORE_BASE_UTILS_HPP_
+#endif  // GKO_CORE_BASE_UTILS_HPP_
diff --git a/core/device_hooks/dpcpp_hooks.cpp b/core/device_hooks/dpcpp_hooks.cpp
index cce8fdec27e..9d6c57f11a4 100644
--- a/core/device_hooks/dpcpp_hooks.cpp
+++ b/core/device_hooks/dpcpp_hooks.cpp
@@ -30,7 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-
 #include <memory>
 #include <string>
 
diff --git a/core/preconditioner/isai_kernels.hpp b/core/preconditioner/isai_kernels.hpp
index 8093f454255..c8e1dcf4b91 100644
--- a/core/preconditioner/isai_kernels.hpp
+++ b/core/preconditioner/isai_kernels.hpp
@@ -34,10 +34,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_PRECONDITIONER_ISAI_KERNELS_HPP_
 
 
-#include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/preconditioner/isai.hpp>
 
 
+#include <ginkgo/core/matrix/csr.hpp>
+
+
 namespace gko {
 namespace kernels {
 
diff --git a/core/test/utils/unsort_matrix.hpp b/core/test/utils/unsort_matrix.hpp
index 3a4b83b6b4e..8d1f7d7d498 100644
--- a/core/test/utils/unsort_matrix.hpp
+++ b/core/test/utils/unsort_matrix.hpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifndef GKO_CORE_TEST_UTILS_UNSORT_MATRIX_HPP_
 #define GKO_CORE_TEST_UTILS_UNSORT_MATRIX_HPP_
 
+
 #include <algorithm>
 #include <random>
 
diff --git a/cuda/base/types.hpp b/cuda/base/types.hpp
index 580976b265f..43130c12246 100644
--- a/cuda/base/types.hpp
+++ b/cuda/base/types.hpp
@@ -34,6 +34,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CUDA_BASE_TYPES_HPP_
 
 
+#include <ginkgo/core/base/types.hpp>
+
+
 #include <type_traits>
 
 
@@ -43,9 +46,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <thrust/complex.h>
 
 
-#include <ginkgo/core/base/types.hpp>
-
-
 namespace gko {
 
 
diff --git a/cuda/factorization/factorization_kernels.cu b/cuda/factorization/factorization_kernels.cu
index bcf861a1acf..8bbd5ec3448 100644
--- a/cuda/factorization/factorization_kernels.cu
+++ b/cuda/factorization/factorization_kernels.cu
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/array.hpp>
 
+
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/csr_builder.hpp"
 #include "cuda/base/config.hpp"
diff --git a/cuda/factorization/par_ilut_approx_filter_kernel.cu b/cuda/factorization/par_ilut_approx_filter_kernel.cu
index 1a17572d9bd..6ad16f99b5b 100644
--- a/cuda/factorization/par_ilut_approx_filter_kernel.cu
+++ b/cuda/factorization/par_ilut_approx_filter_kernel.cu
@@ -44,7 +44,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/components/prefix_sum.hpp"
-#include "core/factorization/par_ilut_kernels.hpp"
 #include "core/matrix/coo_builder.hpp"
 #include "core/matrix/csr_builder.hpp"
 #include "core/matrix/csr_kernels.hpp"
diff --git a/cuda/factorization/par_ilut_filter_kernel.cu b/cuda/factorization/par_ilut_filter_kernel.cu
index 1b2e6e921f8..e1f0afd4ad4 100644
--- a/cuda/factorization/par_ilut_filter_kernel.cu
+++ b/cuda/factorization/par_ilut_filter_kernel.cu
@@ -159,4 +159,4 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 }  // namespace par_ilut_factorization
 }  // namespace cuda
 }  // namespace kernels
-}  // namespace gko
\ No newline at end of file
+}  // namespace gko
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index 83f018c4330..be430bac4e4 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -33,8 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
-#include <time.h>
 #include <random>
+#include <time.h>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
diff --git a/cuda/test/reorder/rcm_kernels.cpp b/cuda/test/reorder/rcm_kernels.cpp
index d4822b36859..9e72e604cf2 100644
--- a/cuda/test/reorder/rcm_kernels.cpp
+++ b/cuda/test/reorder/rcm_kernels.cpp
@@ -30,7 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-
 #include <ginkgo/core/reorder/rcm.hpp>
 
 
diff --git a/dev_tools/scripts/config b/dev_tools/scripts/config
index ba29fed413b..49ced95b64d 100644
--- a/dev_tools/scripts/config
+++ b/dev_tools/scripts/config
@@ -6,10 +6,26 @@
     - FixInclude: "hip/hip_runtime.h"
 - "(cuda|hip|omp|dpcpp)/test/factorization/par_ilu_kernels"
     - FixInclude: "core/factorization/par_ilu_kernels.hpp"
+- "(cuda|hip|omp|dpcpp)/test/factorization/par_ilut_kernels"
+    - FixInclude: "core/factorization/par_ilut_kernels.hpp"
+- "(cuda|hip|omp|dpcpp)/test/factorization/par_ict_kernels"
+    - FixInclude: "core/factorization/par_ict_kernels.hpp"
+- "cuda/factorization/par_ilut_select_common"
+    - FixInclude: "cuda/factorization/par_ilut_select_common.cuh"
+- "hip/factorization/par_ilut_select_common"
+    - FixInclude: "hip/factorization/par_ilut_select_common.hip.hpp"
+- "(cuda|hip|dpcpp)/factorization/par_ilut_"
+    - FixInclude: "core/factorization/par_ilut_kernels.hpp"
+- "(cuda|hip|dpcpp)/factorization/par_ict_"
+    - FixInclude: "core/factorization/par_ict_kernels.hpp"
 - "(cuda|hip|dpcpp)/preconditioner/jacobi_"
     - FixInclude: "core/preconditioner/jacobi_kernels.hpp"
 - "core/test/base/(extended_float|iterator_factory)"
     - RemoveTest: "true"
+- "core/test/base/allocator"
+    - FixInclude: "core/base/allocator.hpp"
+- "reference/test/base/utils"
+    - FixInclude: "core/base/utils.hpp"
 - "_builder\.cpp"
     - RemoveTest: "true"
 - "_builder\.hpp"
diff --git a/dev_tools/scripts/regroup b/dev_tools/scripts/regroup
index 5351dc7009e..9e8a5172d05 100644
--- a/dev_tools/scripts/regroup
+++ b/dev_tools/scripts/regroup
@@ -2,7 +2,7 @@ IncludeBlocks: Regroup
 IncludeCategories:
   - Regex: '^<(rapidjson|gflags|gtest|papi).*'
     Priority: 3
-  - Regex: '^<(omp|cu|hip|thrust|CL).*'
+  - Regex: '^<(omp|cu|hip|thrust|CL/|cooperative).*'
     Priority: 2
   - Regex: '^<ginkgo.*'
     Priority: 5
diff --git a/examples/ginkgo-overhead/ginkgo-overhead.cpp b/examples/ginkgo-overhead/ginkgo-overhead.cpp
index 17ce96d2900..b8bc7acc2b1 100644
--- a/examples/ginkgo-overhead/ginkgo-overhead.cpp
+++ b/examples/ginkgo-overhead/ginkgo-overhead.cpp
@@ -38,7 +38,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <iostream>
 
 
-[[noreturn]] void print_usage_and_exit(const char *name) {
+[[noreturn]] void print_usage_and_exit(const char *name)
+{
     std::cerr << "Usage: " << name << " [NUM_ITERS]" << std::endl;
     std::exit(-1);
 }
diff --git a/hip/base/types.hip.hpp b/hip/base/types.hip.hpp
index 2a7395fac72..4db75f4024e 100644
--- a/hip/base/types.hip.hpp
+++ b/hip/base/types.hip.hpp
@@ -34,10 +34,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_HIP_BASE_TYPES_HIP_HPP_
 
 
-#include <type_traits>
+#include <ginkgo/core/base/types.hpp>
 
 
-#include <ginkgo/core/base/types.hpp>
+#include <type_traits>
 
 
 #include <hip/hip_complex.h>
diff --git a/hip/factorization/par_ilut_approx_filter_kernel.hip.cpp b/hip/factorization/par_ilut_approx_filter_kernel.hip.cpp
index 1d3e1eba6f7..7955898a291 100644
--- a/hip/factorization/par_ilut_approx_filter_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_approx_filter_kernel.hip.cpp
@@ -33,10 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/factorization/par_ilut_kernels.hpp"
 
 
-#include <hip/hip_runtime.h>
+#include <algorithm>
 
 
-#include <algorithm>
+#include <hip/hip_runtime.h>
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -47,7 +47,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/components/prefix_sum.hpp"
-#include "core/factorization/par_ilut_kernels.hpp"
 #include "core/matrix/coo_builder.hpp"
 #include "core/matrix/csr_builder.hpp"
 #include "core/matrix/csr_kernels.hpp"
diff --git a/hip/factorization/par_ilut_filter_kernel.hip.cpp b/hip/factorization/par_ilut_filter_kernel.hip.cpp
index f1b57bd9f32..c08b7ef767e 100644
--- a/hip/factorization/par_ilut_filter_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_filter_kernel.hip.cpp
@@ -163,4 +163,4 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 }  // namespace par_ilut_factorization
 }  // namespace hip
 }  // namespace kernels
-}  // namespace gko
\ No newline at end of file
+}  // namespace gko
diff --git a/hip/factorization/par_ilut_select_kernel.hip.cpp b/hip/factorization/par_ilut_select_kernel.hip.cpp
index 9d8c73562e9..3f92051989b 100644
--- a/hip/factorization/par_ilut_select_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_select_kernel.hip.cpp
@@ -33,10 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/factorization/par_ilut_kernels.hpp"
 
 
-#include <hip/hip_runtime.h>
+#include <algorithm>
 
 
-#include <algorithm>
+#include <hip/hip_runtime.h>
 
 
 #include <ginkgo/core/base/array.hpp>
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index a2c3025a136..5bb26f25399 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -33,8 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
-#include <time.h>
 #include <random>
+#include <time.h>
 
 
 #include <hip/hip_runtime.h>
diff --git a/include/ginkgo/core/base/abstract_factory.hpp b/include/ginkgo/core/base/abstract_factory.hpp
index 2db193027a1..ff185509224 100644
--- a/include/ginkgo/core/base/abstract_factory.hpp
+++ b/include/ginkgo/core/base/abstract_factory.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_ABSTRACT_FACTORY_HPP_
-#define GKO_CORE_BASE_ABSTRACT_FACTORY_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_ABSTRACT_FACTORY_HPP_
+#define GKO_PUBLIC_CORE_BASE_ABSTRACT_FACTORY_HPP_
 
 
 #include <ginkgo/core/base/polymorphic_object.hpp>
@@ -289,4 +289,4 @@ struct enable_parameters_type {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_ABSTRACT_FACTORY_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_ABSTRACT_FACTORY_HPP_
diff --git a/include/ginkgo/core/base/array.hpp b/include/ginkgo/core/base/array.hpp
index 59960520edf..7c04ff0bfb2 100644
--- a/include/ginkgo/core/base/array.hpp
+++ b/include/ginkgo/core/base/array.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_ARRAY_HPP_
-#define GKO_CORE_BASE_ARRAY_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_ARRAY_HPP_
+#define GKO_PUBLIC_CORE_BASE_ARRAY_HPP_
 
 
 #include <algorithm>
@@ -619,4 +619,4 @@ class copy_back_deleter<Array<T>> {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_ARRAY_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_ARRAY_HPP_
diff --git a/include/ginkgo/core/base/combination.hpp b/include/ginkgo/core/base/combination.hpp
index 908013a3e57..5ca773bbc3e 100644
--- a/include/ginkgo/core/base/combination.hpp
+++ b/include/ginkgo/core/base/combination.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_COMBINATION_HPP_
-#define GKO_CORE_BASE_COMBINATION_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_COMBINATION_HPP_
+#define GKO_PUBLIC_CORE_BASE_COMBINATION_HPP_
 
 
 #include <vector>
@@ -207,4 +207,4 @@ class Combination : public EnableLinOp<Combination<ValueType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_COMBINATION_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_COMBINATION_HPP_
diff --git a/include/ginkgo/core/base/composition.hpp b/include/ginkgo/core/base/composition.hpp
index 4a7ecc7874f..dfcfb13c5ad 100644
--- a/include/ginkgo/core/base/composition.hpp
+++ b/include/ginkgo/core/base/composition.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_COMPOSITION_HPP_
-#define GKO_CORE_BASE_COMPOSITION_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_COMPOSITION_HPP_
+#define GKO_PUBLIC_CORE_BASE_COMPOSITION_HPP_
 
 
 #include <vector>
@@ -170,4 +170,4 @@ class Composition : public EnableLinOp<Composition<ValueType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_COMPOSITION_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_COMPOSITION_HPP_
diff --git a/include/ginkgo/core/base/dim.hpp b/include/ginkgo/core/base/dim.hpp
index e0aaf25666c..3030f0d1442 100644
--- a/include/ginkgo/core/base/dim.hpp
+++ b/include/ginkgo/core/base/dim.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_DIM_HPP_
-#define GKO_CORE_BASE_DIM_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_DIM_HPP_
+#define GKO_PUBLIC_CORE_BASE_DIM_HPP_
 
 
 #include <ginkgo/core/base/types.hpp>
@@ -246,4 +246,4 @@ constexpr GKO_ATTRIBUTES GKO_INLINE dim<2, DimensionType> transpose(
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_DIM_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_DIM_HPP_
diff --git a/include/ginkgo/core/base/exception.hpp b/include/ginkgo/core/base/exception.hpp
index 20ac5ba8958..6ede978ffc3 100644
--- a/include/ginkgo/core/base/exception.hpp
+++ b/include/ginkgo/core/base/exception.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_EXCEPTION_HPP_
-#define GKO_CORE_BASE_EXCEPTION_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_EXCEPTION_HPP_
+#define GKO_PUBLIC_CORE_BASE_EXCEPTION_HPP_
 
 
 #include <exception>
@@ -534,4 +534,4 @@ class KernelNotFound : public Error {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_EXCEPTION_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_EXCEPTION_HPP_
diff --git a/include/ginkgo/core/base/exception_helpers.hpp b/include/ginkgo/core/base/exception_helpers.hpp
index 10df01005ab..e14bb1c0c61 100644
--- a/include/ginkgo/core/base/exception_helpers.hpp
+++ b/include/ginkgo/core/base/exception_helpers.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_EXCEPTION_HELPERS_HPP_
-#define GKO_CORE_BASE_EXCEPTION_HELPERS_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_EXCEPTION_HELPERS_HPP_
+#define GKO_PUBLIC_CORE_BASE_EXCEPTION_HELPERS_HPP_
 
 
 #include <typeinfo>
@@ -569,4 +569,4 @@ inline T ensure_allocated_impl(T ptr, const std::string &file, int line,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_EXCEPTION_HELPERS_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_EXCEPTION_HELPERS_HPP_
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 950e7e1b2ed..2cc18bf3502 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_EXECUTOR_HPP_
-#define GKO_CORE_BASE_EXECUTOR_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_EXECUTOR_HPP_
+#define GKO_PUBLIC_CORE_BASE_EXECUTOR_HPP_
 
 
 #include <memory>
@@ -1421,4 +1421,4 @@ using DefaultExecutor = DpcppExecutor;
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_EXECUTOR_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_EXECUTOR_HPP_
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index a1530ff2880..1512802edb6 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_LIN_OP_HPP_
-#define GKO_CORE_BASE_LIN_OP_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_LIN_OP_HPP_
+#define GKO_PUBLIC_CORE_BASE_LIN_OP_HPP_
 
 
 #include <memory>
@@ -1074,4 +1074,4 @@ public:                                                                      \
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_LIN_OP_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_LIN_OP_HPP_
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 077830602a3..1cf729e2d2e 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_MATH_HPP_
-#define GKO_CORE_BASE_MATH_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_MATH_HPP_
+#define GKO_PUBLIC_CORE_BASE_MATH_HPP_
 
 
 #include <cmath>
@@ -1026,4 +1026,4 @@ using cl::sycl::sqrt;
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_MATH_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_MATH_HPP_
diff --git a/include/ginkgo/core/base/matrix_assembly_data.hpp b/include/ginkgo/core/base/matrix_assembly_data.hpp
index b54d10f4db1..5456a75e6e9 100644
--- a/include/ginkgo/core/base/matrix_assembly_data.hpp
+++ b/include/ginkgo/core/base/matrix_assembly_data.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_MATRIX_ASSEMBLY_DATA_HPP_
-#define GKO_CORE_BASE_MATRIX_ASSEMBLY_DATA_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_MATRIX_ASSEMBLY_DATA_HPP_
+#define GKO_PUBLIC_CORE_BASE_MATRIX_ASSEMBLY_DATA_HPP_
 
 
 #include <algorithm>
@@ -204,4 +204,4 @@ class matrix_assembly_data {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_MATRIX_ASSEMBLY_DATA_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_MATRIX_ASSEMBLY_DATA_HPP_
diff --git a/include/ginkgo/core/base/matrix_data.hpp b/include/ginkgo/core/base/matrix_data.hpp
index 94c01461079..8597b31df68 100644
--- a/include/ginkgo/core/base/matrix_data.hpp
+++ b/include/ginkgo/core/base/matrix_data.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_MATRIX_DATA_HPP_
-#define GKO_CORE_BASE_MATRIX_DATA_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_MATRIX_DATA_HPP_
+#define GKO_PUBLIC_CORE_BASE_MATRIX_DATA_HPP_
 
 
 #include <algorithm>
@@ -532,4 +532,4 @@ struct matrix_data {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_MATRIX_DATA_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_MATRIX_DATA_HPP_
diff --git a/include/ginkgo/core/base/mtx_io.hpp b/include/ginkgo/core/base/mtx_io.hpp
index 8ebe65e973a..38977fe04c0 100644
--- a/include/ginkgo/core/base/mtx_io.hpp
+++ b/include/ginkgo/core/base/mtx_io.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_MTX_IO_HPP_
-#define GKO_CORE_BASE_MTX_IO_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_MTX_IO_HPP_
+#define GKO_PUBLIC_CORE_BASE_MTX_IO_HPP_
 
 
 #include <istream>
@@ -145,4 +145,4 @@ inline void write(StreamType &&os, MatrixType *matrix,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_MTX_IO_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_MTX_IO_HPP_
diff --git a/include/ginkgo/core/base/name_demangling.hpp b/include/ginkgo/core/base/name_demangling.hpp
index adfb7392412..fb41561fa9f 100644
--- a/include/ginkgo/core/base/name_demangling.hpp
+++ b/include/ginkgo/core/base/name_demangling.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_NAME_DEMANGLING_HPP_
-#define GKO_CORE_BASE_NAME_DEMANGLING_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_NAME_DEMANGLING_HPP_
+#define GKO_PUBLIC_CORE_BASE_NAME_DEMANGLING_HPP_
 
 
 #include <ginkgo/config.hpp>
@@ -141,4 +141,4 @@ std::string get_enclosing_scope(const T &)
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_NAME_DEMANGLING_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_NAME_DEMANGLING_HPP_
diff --git a/include/ginkgo/core/base/perturbation.hpp b/include/ginkgo/core/base/perturbation.hpp
index 16e4406605b..d410856b00b 100644
--- a/include/ginkgo/core/base/perturbation.hpp
+++ b/include/ginkgo/core/base/perturbation.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_PERTURBATION_HPP_
-#define GKO_CORE_BASE_PERTURBATION_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_PERTURBATION_HPP_
+#define GKO_PUBLIC_CORE_BASE_PERTURBATION_HPP_
 
 
 #include <memory>
@@ -205,4 +205,4 @@ class Perturbation : public EnableLinOp<Perturbation<ValueType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_PERTURBATION_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_PERTURBATION_HPP_
diff --git a/include/ginkgo/core/base/polymorphic_object.hpp b/include/ginkgo/core/base/polymorphic_object.hpp
index c3554ea2455..ad712c2cd08 100644
--- a/include/ginkgo/core/base/polymorphic_object.hpp
+++ b/include/ginkgo/core/base/polymorphic_object.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_POLYMORPHIC_OBJECT_HPP_
-#define GKO_CORE_BASE_POLYMORPHIC_OBJECT_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_POLYMORPHIC_OBJECT_HPP_
+#define GKO_PUBLIC_CORE_BASE_POLYMORPHIC_OBJECT_HPP_
 
 
 #include <memory>
@@ -652,4 +652,4 @@ class EnableCreateMethod {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_POLYMORPHIC_OBJECT_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_POLYMORPHIC_OBJECT_HPP_
diff --git a/include/ginkgo/core/base/range.hpp b/include/ginkgo/core/base/range.hpp
index 10b77ad8a29..6e59ee3c637 100644
--- a/include/ginkgo/core/base/range.hpp
+++ b/include/ginkgo/core/base/range.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_RANGE_HPP_
-#define GKO_CORE_BASE_RANGE_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_RANGE_HPP_
+#define GKO_PUBLIC_CORE_BASE_RANGE_HPP_
 
 
 #include <type_traits>
@@ -1007,4 +1007,4 @@ GKO_BIND_RANGE_OPERATION_TO_OPERATOR(mmul_operation, mmul);
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_RANGE_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_RANGE_HPP_
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 4040301f287..28b9ac6b51f 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_RANGE_ACCESSORS_HPP_
-#define GKO_CORE_BASE_RANGE_ACCESSORS_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_RANGE_ACCESSORS_HPP_
+#define GKO_PUBLIC_CORE_BASE_RANGE_ACCESSORS_HPP_
 
 
 #include <array>
@@ -190,4 +190,4 @@ class row_major {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_RANGE_ACCESSORS_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_RANGE_ACCESSORS_HPP_
diff --git a/include/ginkgo/core/base/std_extensions.hpp b/include/ginkgo/core/base/std_extensions.hpp
index af52f8b33d7..c1cec39b6e4 100644
--- a/include/ginkgo/core/base/std_extensions.hpp
+++ b/include/ginkgo/core/base/std_extensions.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_STD_EXTENSIONS_HPP_
-#define GKO_CORE_BASE_STD_EXTENSIONS_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_STD_EXTENSIONS_HPP_
+#define GKO_PUBLIC_CORE_BASE_STD_EXTENSIONS_HPP_
 
 
 #include <functional>
@@ -120,4 +120,4 @@ constexpr bool less_equal(const T &&lhs, const T &&rhs)
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_STD_EXTENSIONS_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_STD_EXTENSIONS_HPP_
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index d63623fea40..3cca7cbfd8a 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_TYPES_HPP_
-#define GKO_CORE_BASE_TYPES_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_TYPES_HPP_
+#define GKO_PUBLIC_CORE_BASE_TYPES_HPP_
 
 
 #include <cassert>
@@ -474,4 +474,4 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_TYPES_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_TYPES_HPP_
diff --git a/include/ginkgo/core/base/utils.hpp b/include/ginkgo/core/base/utils.hpp
index 12a2d0ef3e3..5270d619445 100644
--- a/include/ginkgo/core/base/utils.hpp
+++ b/include/ginkgo/core/base/utils.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_UTILS_HPP_
-#define GKO_CORE_BASE_UTILS_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_UTILS_HPP_
+#define GKO_PUBLIC_CORE_BASE_UTILS_HPP_
 
 
 #include <functional>
@@ -587,4 +587,4 @@ detail::temporary_clone<T> make_temporary_clone(
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_UTILS_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_UTILS_HPP_
diff --git a/include/ginkgo/core/base/version.hpp b/include/ginkgo/core/base/version.hpp
index 33ed03a8872..d56526a8613 100644
--- a/include/ginkgo/core/base/version.hpp
+++ b/include/ginkgo/core/base/version.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_VERSION_HPP_
-#define GKO_CORE_BASE_VERSION_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_VERSION_HPP_
+#define GKO_PUBLIC_CORE_BASE_VERSION_HPP_
 
 
 #include <ostream>
@@ -264,4 +264,4 @@ std::ostream &operator<<(std::ostream &os, const version_info &ver_info);
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_VERSION_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_VERSION_HPP_
diff --git a/include/ginkgo/core/factorization/ilu.hpp b/include/ginkgo/core/factorization/ilu.hpp
index ece4e2aeb87..a1322116250 100644
--- a/include/ginkgo/core/factorization/ilu.hpp
+++ b/include/ginkgo/core/factorization/ilu.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_FACTORIZATION_ILU_HPP_
-#define GKO_CORE_FACTORIZATION_ILU_HPP_
+#ifndef GKO_PUBLIC_CORE_FACTORIZATION_ILU_HPP_
+#define GKO_PUBLIC_CORE_FACTORIZATION_ILU_HPP_
 
 
 #include <memory>
@@ -173,4 +173,4 @@ class Ilu : public Composition<ValueType> {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_FACTORIZATION_ILU_HPP_
+#endif  // GKO_PUBLIC_CORE_FACTORIZATION_ILU_HPP_
diff --git a/include/ginkgo/core/factorization/par_ict.hpp b/include/ginkgo/core/factorization/par_ict.hpp
index 0686093686d..fd89af6aa47 100644
--- a/include/ginkgo/core/factorization/par_ict.hpp
+++ b/include/ginkgo/core/factorization/par_ict.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_FACTORIZATION_PAR_ICT_HPP_
-#define GKO_CORE_FACTORIZATION_PAR_ICT_HPP_
+#ifndef GKO_PUBLIC_CORE_FACTORIZATION_PAR_ICT_HPP_
+#define GKO_PUBLIC_CORE_FACTORIZATION_PAR_ICT_HPP_
 
 
 #include <memory>
@@ -250,4 +250,4 @@ class ParIct : public Composition<ValueType> {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_FACTORIZATION_PAR_ICT_HPP_
+#endif  // GKO_PUBLIC_CORE_FACTORIZATION_PAR_ICT_HPP_
diff --git a/include/ginkgo/core/factorization/par_ilu.hpp b/include/ginkgo/core/factorization/par_ilu.hpp
index 63cf7468a60..c92abd88ba1 100644
--- a/include/ginkgo/core/factorization/par_ilu.hpp
+++ b/include/ginkgo/core/factorization/par_ilu.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_FACTORIZATION_PAR_ILU_HPP_
-#define GKO_CORE_FACTORIZATION_PAR_ILU_HPP_
+#ifndef GKO_PUBLIC_CORE_FACTORIZATION_PAR_ILU_HPP_
+#define GKO_PUBLIC_CORE_FACTORIZATION_PAR_ILU_HPP_
 
 
 #include <memory>
@@ -213,4 +213,4 @@ class ParIlu : public Composition<ValueType> {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_FACTORIZATION_PAR_ILU_HPP_
+#endif  // GKO_PUBLIC_CORE_FACTORIZATION_PAR_ILU_HPP_
diff --git a/include/ginkgo/core/factorization/par_ilut.hpp b/include/ginkgo/core/factorization/par_ilut.hpp
index 87a8f866061..4ee0acf2612 100644
--- a/include/ginkgo/core/factorization/par_ilut.hpp
+++ b/include/ginkgo/core/factorization/par_ilut.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_FACTORIZATION_PAR_ILUT_HPP_
-#define GKO_CORE_FACTORIZATION_PAR_ILUT_HPP_
+#ifndef GKO_PUBLIC_CORE_FACTORIZATION_PAR_ILUT_HPP_
+#define GKO_PUBLIC_CORE_FACTORIZATION_PAR_ILUT_HPP_
 
 
 #include <memory>
@@ -255,4 +255,4 @@ class ParIlut : public Composition<ValueType> {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_FACTORIZATION_PAR_ILUT_HPP_
+#endif  // GKO_PUBLIC_CORE_FACTORIZATION_PAR_ILUT_HPP_
diff --git a/include/ginkgo/core/log/convergence.hpp b/include/ginkgo/core/log/convergence.hpp
index 64cd16ea1aa..e8a5da27eb7 100644
--- a/include/ginkgo/core/log/convergence.hpp
+++ b/include/ginkgo/core/log/convergence.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_LOG_CONVERGENCE_HPP_
-#define GKO_CORE_LOG_CONVERGENCE_HPP_
+#ifndef GKO_PUBLIC_CORE_LOG_CONVERGENCE_HPP_
+#define GKO_PUBLIC_CORE_LOG_CONVERGENCE_HPP_
 
 
 #include <memory>
@@ -146,4 +146,4 @@ class Convergence : public Logger {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_LOG_CONVERGENCE_HPP_
+#endif  // GKO_PUBLIC_CORE_LOG_CONVERGENCE_HPP_
diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp
index 3d6ae9b4309..d84e51c8ed6 100644
--- a/include/ginkgo/core/log/logger.hpp
+++ b/include/ginkgo/core/log/logger.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_LOG_LOGGER_HPP_
-#define GKO_CORE_LOG_LOGGER_HPP_
+#ifndef GKO_PUBLIC_CORE_LOG_LOGGER_HPP_
+#define GKO_PUBLIC_CORE_LOG_LOGGER_HPP_
 
 
 #include <algorithm>
@@ -556,4 +556,4 @@ class EnableLogging : public Loggable {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_LOG_LOGGER_HPP_
+#endif  // GKO_PUBLIC_CORE_LOG_LOGGER_HPP_
diff --git a/include/ginkgo/core/log/papi.hpp b/include/ginkgo/core/log/papi.hpp
index 7b54e478f6e..00687f88719 100644
--- a/include/ginkgo/core/log/papi.hpp
+++ b/include/ginkgo/core/log/papi.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_LOG_PAPI_HPP_
-#define GKO_CORE_LOG_PAPI_HPP_
+#ifndef GKO_PUBLIC_CORE_LOG_PAPI_HPP_
+#define GKO_PUBLIC_CORE_LOG_PAPI_HPP_
 
 
 #include <ginkgo/config.hpp>
@@ -318,4 +318,4 @@ class Papi : public Logger {
 
 
 #endif  // GKO_HAVE_PAPI_SDE
-#endif  // GKO_CORE_LOG_PAPI_HPP_
+#endif  // GKO_PUBLIC_CORE_LOG_PAPI_HPP_
diff --git a/include/ginkgo/core/log/record.hpp b/include/ginkgo/core/log/record.hpp
index 0c791e5e278..1e30990cc1f 100644
--- a/include/ginkgo/core/log/record.hpp
+++ b/include/ginkgo/core/log/record.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_LOG_RECORD_HPP_
-#define GKO_CORE_LOG_RECORD_HPP_
+#ifndef GKO_PUBLIC_CORE_LOG_RECORD_HPP_
+#define GKO_PUBLIC_CORE_LOG_RECORD_HPP_
 
 
 #include <deque>
@@ -454,4 +454,4 @@ class Record : public Logger {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_LOG_RECORD_HPP_
+#endif  // GKO_PUBLIC_CORE_LOG_RECORD_HPP_
diff --git a/include/ginkgo/core/log/stream.hpp b/include/ginkgo/core/log/stream.hpp
index d46a0d07be0..30b1beabf2b 100644
--- a/include/ginkgo/core/log/stream.hpp
+++ b/include/ginkgo/core/log/stream.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_LOG_STREAM_HPP_
-#define GKO_CORE_LOG_STREAM_HPP_
+#ifndef GKO_PUBLIC_CORE_LOG_STREAM_HPP_
+#define GKO_PUBLIC_CORE_LOG_STREAM_HPP_
 
 
 #include <fstream>
@@ -211,4 +211,4 @@ class Stream : public Logger {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_LOG_STREAM_HPP_
+#endif  // GKO_PUBLIC_CORE_LOG_STREAM_HPP_
diff --git a/include/ginkgo/core/matrix/coo.hpp b/include/ginkgo/core/matrix/coo.hpp
index 7fe9fe02533..ce8cf678eb1 100644
--- a/include/ginkgo/core/matrix/coo.hpp
+++ b/include/ginkgo/core/matrix/coo.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_MATRIX_COO_HPP_
-#define GKO_CORE_MATRIX_COO_HPP_
+#ifndef GKO_PUBLIC_CORE_MATRIX_COO_HPP_
+#define GKO_PUBLIC_CORE_MATRIX_COO_HPP_
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -327,4 +327,4 @@ class Coo : public EnableLinOp<Coo<ValueType, IndexType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_MATRIX_COO_HPP_
+#endif  // GKO_PUBLIC_CORE_MATRIX_COO_HPP_
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index f6e486e9f5d..9ba5ed4d343 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_MATRIX_CSR_HPP_
-#define GKO_CORE_MATRIX_CSR_HPP_
+#ifndef GKO_PUBLIC_CORE_MATRIX_CSR_HPP_
+#define GKO_PUBLIC_CORE_MATRIX_CSR_HPP_
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -1057,4 +1057,4 @@ void strategy_rebuild_helper(Csr<ValueType, IndexType> *result)
 }  // namespace gko
 
 
-#endif  // GKO_CORE_MATRIX_CSR_HPP_
+#endif  // GKO_PUBLIC_CORE_MATRIX_CSR_HPP_
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index c7ff335b89e..ad689c8ad1f 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_MATRIX_DENSE_HPP_
-#define GKO_CORE_MATRIX_DENSE_HPP_
+#ifndef GKO_PUBLIC_CORE_MATRIX_DENSE_HPP_
+#define GKO_PUBLIC_CORE_MATRIX_DENSE_HPP_
 
 
 #include <initializer_list>
@@ -842,4 +842,4 @@ std::unique_ptr<Matrix> initialize(
 }  // namespace gko
 
 
-#endif  // GKO_CORE_MATRIX_DENSE_HPP_
+#endif  // GKO_PUBLIC_CORE_MATRIX_DENSE_HPP_
diff --git a/include/ginkgo/core/matrix/diagonal.hpp b/include/ginkgo/core/matrix/diagonal.hpp
index b268421c163..58463b8f838 100644
--- a/include/ginkgo/core/matrix/diagonal.hpp
+++ b/include/ginkgo/core/matrix/diagonal.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_MATRIX_DIAGONAL_HPP_
-#define GKO_CORE_MATRIX_DIAGONAL_HPP_
+#ifndef GKO_PUBLIC_CORE_MATRIX_DIAGONAL_HPP_
+#define GKO_PUBLIC_CORE_MATRIX_DIAGONAL_HPP_
 
 
 #include <ginkgo/core/base/lin_op.hpp>
@@ -211,4 +211,4 @@ class Diagonal
 }  // namespace gko
 
 
-#endif  // GKO_CORE_MATRIX_DIAGONAL_HPP_
+#endif  // GKO_PUBLIC_CORE_MATRIX_DIAGONAL_HPP_
diff --git a/include/ginkgo/core/matrix/ell.hpp b/include/ginkgo/core/matrix/ell.hpp
index d8970d2b1c3..932bfc7fda9 100644
--- a/include/ginkgo/core/matrix/ell.hpp
+++ b/include/ginkgo/core/matrix/ell.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_MATRIX_ELL_HPP_
-#define GKO_CORE_MATRIX_ELL_HPP_
+#ifndef GKO_PUBLIC_CORE_MATRIX_ELL_HPP_
+#define GKO_PUBLIC_CORE_MATRIX_ELL_HPP_
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -335,4 +335,4 @@ class Ell : public EnableLinOp<Ell<ValueType, IndexType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_MATRIX_ELL_HPP_
+#endif  // GKO_PUBLIC_CORE_MATRIX_ELL_HPP_
diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp
index babff97efbc..4641a0443ec 100644
--- a/include/ginkgo/core/matrix/hybrid.hpp
+++ b/include/ginkgo/core/matrix/hybrid.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_MATRIX_HYBRID_HPP_
-#define GKO_CORE_MATRIX_HYBRID_HPP_
+#ifndef GKO_PUBLIC_CORE_MATRIX_HYBRID_HPP_
+#define GKO_PUBLIC_CORE_MATRIX_HYBRID_HPP_
 
 
 #include <algorithm>
@@ -791,4 +791,4 @@ Hybrid<ValueType, IndexType>::get_strategy() const
 }  // namespace gko
 
 
-#endif  // GKO_CORE_MATRIX_HYBRID_HPP_
+#endif  // GKO_PUBLIC_CORE_MATRIX_HYBRID_HPP_
diff --git a/include/ginkgo/core/matrix/identity.hpp b/include/ginkgo/core/matrix/identity.hpp
index 06682c46abb..d00ffec9b9e 100644
--- a/include/ginkgo/core/matrix/identity.hpp
+++ b/include/ginkgo/core/matrix/identity.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_MATRIX_IDENTITY_HPP_
-#define GKO_CORE_MATRIX_IDENTITY_HPP_
+#ifndef GKO_PUBLIC_CORE_MATRIX_IDENTITY_HPP_
+#define GKO_PUBLIC_CORE_MATRIX_IDENTITY_HPP_
 
 
 #include <ginkgo/core/base/lin_op.hpp>
@@ -163,4 +163,4 @@ class IdentityFactory
 }  // namespace gko
 
 
-#endif  // GKO_CORE_MATRIX_IDENTITY_HPP_
+#endif  // GKO_PUBLIC_CORE_MATRIX_IDENTITY_HPP_
diff --git a/include/ginkgo/core/matrix/permutation.hpp b/include/ginkgo/core/matrix/permutation.hpp
index 656cee4ae3d..cb81544c835 100644
--- a/include/ginkgo/core/matrix/permutation.hpp
+++ b/include/ginkgo/core/matrix/permutation.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_MATRIX_PERMUTATION_HPP_
-#define GKO_CORE_MATRIX_PERMUTATION_HPP_
+#ifndef GKO_PUBLIC_CORE_MATRIX_PERMUTATION_HPP_
+#define GKO_PUBLIC_CORE_MATRIX_PERMUTATION_HPP_
 
 
 #include <algorithm>
@@ -246,4 +246,4 @@ class Permutation : public EnableLinOp<Permutation<IndexType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_MATRIX_PERMUTATION_HPP_
+#endif  // GKO_PUBLIC_CORE_MATRIX_PERMUTATION_HPP_
diff --git a/include/ginkgo/core/matrix/sellp.hpp b/include/ginkgo/core/matrix/sellp.hpp
index 8540b7f1c5d..b99746ec3b6 100644
--- a/include/ginkgo/core/matrix/sellp.hpp
+++ b/include/ginkgo/core/matrix/sellp.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_MATRIX_SELLP_HPP_
-#define GKO_CORE_MATRIX_SELLP_HPP_
+#ifndef GKO_PUBLIC_CORE_MATRIX_SELLP_HPP_
+#define GKO_PUBLIC_CORE_MATRIX_SELLP_HPP_
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -358,4 +358,4 @@ class Sellp : public EnableLinOp<Sellp<ValueType, IndexType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_MATRIX_SELLP_HPP_
+#endif  // GKO_PUBLIC_CORE_MATRIX_SELLP_HPP_
diff --git a/include/ginkgo/core/matrix/sparsity_csr.hpp b/include/ginkgo/core/matrix/sparsity_csr.hpp
index f55da0bfa83..f736d35f840 100644
--- a/include/ginkgo/core/matrix/sparsity_csr.hpp
+++ b/include/ginkgo/core/matrix/sparsity_csr.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_MATRIX_SPARSITY_CSR_HPP_
-#define GKO_CORE_MATRIX_SPARSITY_CSR_HPP_
+#ifndef GKO_PUBLIC_CORE_MATRIX_SPARSITY_CSR_HPP_
+#define GKO_PUBLIC_CORE_MATRIX_SPARSITY_CSR_HPP_
 
 
 #include <vector>
@@ -265,4 +265,4 @@ class SparsityCsr
 }  // namespace gko
 
 
-#endif  // GKO_CORE_MATRIX_SPARSITY_CSR_HPP_
+#endif  // GKO_PUBLIC_CORE_MATRIX_SPARSITY_CSR_HPP_
diff --git a/include/ginkgo/core/preconditioner/ilu.hpp b/include/ginkgo/core/preconditioner/ilu.hpp
index e312f6a5391..72a478ac405 100644
--- a/include/ginkgo/core/preconditioner/ilu.hpp
+++ b/include/ginkgo/core/preconditioner/ilu.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_PRECONDITIONER_ILU_HPP_
-#define GKO_CORE_PRECONDITIONER_ILU_HPP_
+#ifndef GKO_PUBLIC_CORE_PRECONDITIONER_ILU_HPP_
+#define GKO_PUBLIC_CORE_PRECONDITIONER_ILU_HPP_
 
 
 #include <memory>
@@ -414,4 +414,4 @@ class Ilu : public EnableLinOp<
 }  // namespace gko
 
 
-#endif  // GKO_CORE_PRECONDITIONER_ILU_HPP_
+#endif  // GKO_PUBLIC_CORE_PRECONDITIONER_ILU_HPP_
diff --git a/include/ginkgo/core/preconditioner/isai.hpp b/include/ginkgo/core/preconditioner/isai.hpp
index 7abb186b16b..420b376c0a0 100644
--- a/include/ginkgo/core/preconditioner/isai.hpp
+++ b/include/ginkgo/core/preconditioner/isai.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_PRECONDITIONER_ISAI_HPP_
-#define GKO_CORE_PRECONDITIONER_ISAI_HPP_
+#ifndef GKO_PUBLIC_CORE_PRECONDITIONER_ISAI_HPP_
+#define GKO_PUBLIC_CORE_PRECONDITIONER_ISAI_HPP_
 
 
 #include <memory>
@@ -215,4 +215,4 @@ using UpperIsai = Isai<isai_type::upper, ValueType, IndexType>;
 }  // namespace gko
 
 
-#endif  // GKO_CORE_PRECONDITIONER_ISAI_HPP_
+#endif  // GKO_PUBLIC_CORE_PRECONDITIONER_ISAI_HPP_
diff --git a/include/ginkgo/core/preconditioner/jacobi.hpp b/include/ginkgo/core/preconditioner/jacobi.hpp
index 60433b2f2cc..4828609d077 100644
--- a/include/ginkgo/core/preconditioner/jacobi.hpp
+++ b/include/ginkgo/core/preconditioner/jacobi.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_PRECONDITIONER_JACOBI_HPP_
-#define GKO_CORE_PRECONDITIONER_JACOBI_HPP_
+#ifndef GKO_PUBLIC_CORE_PRECONDITIONER_JACOBI_HPP_
+#define GKO_PUBLIC_CORE_PRECONDITIONER_JACOBI_HPP_
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -107,8 +107,8 @@ struct block_interleaved_storage_scheme {
      *        blocks is not known, for a special input `size_type{} - 1`
      *        the method returns `0` to avoid overallocation of memory.
      */
-    GKO_ATTRIBUTES size_type
-    compute_storage_space(size_type num_blocks) const noexcept
+    GKO_ATTRIBUTES size_type compute_storage_space(size_type num_blocks) const
+        noexcept
     {
         return (num_blocks + 1 == size_type{0})
                    ? size_type{0}
@@ -146,8 +146,8 @@ struct block_interleaved_storage_scheme {
      *
      * @return the offset of the block with ID `block_id`
      */
-    GKO_ATTRIBUTES IndexType
-    get_global_block_offset(IndexType block_id) const noexcept
+    GKO_ATTRIBUTES IndexType get_global_block_offset(IndexType block_id) const
+        noexcept
     {
         return this->get_group_offset(block_id) +
                this->get_block_offset(block_id);
@@ -610,4 +610,4 @@ class Jacobi : public EnableLinOp<Jacobi<ValueType, IndexType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_PRECONDITIONER_JACOBI_HPP_
+#endif  // GKO_PUBLIC_CORE_PRECONDITIONER_JACOBI_HPP_
diff --git a/include/ginkgo/core/reorder/rcm.hpp b/include/ginkgo/core/reorder/rcm.hpp
index 3a36eccf465..2f95a95c39a 100644
--- a/include/ginkgo/core/reorder/rcm.hpp
+++ b/include/ginkgo/core/reorder/rcm.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_REORDER_RCM_HPP_
-#define GKO_CORE_REORDER_RCM_HPP_
+#ifndef GKO_PUBLIC_CORE_REORDER_RCM_HPP_
+#define GKO_PUBLIC_CORE_REORDER_RCM_HPP_
 
 
 #include <memory>
@@ -215,4 +215,4 @@ class Rcm
 }  // namespace gko
 
 
-#endif  // GKO_CORE_REORDER_RCM_HPP_
+#endif  // GKO_PUBLIC_CORE_REORDER_RCM_HPP_
diff --git a/include/ginkgo/core/reorder/reordering_base.hpp b/include/ginkgo/core/reorder/reordering_base.hpp
index 7b93d575813..ea6a2d2e5b0 100644
--- a/include/ginkgo/core/reorder/reordering_base.hpp
+++ b/include/ginkgo/core/reorder/reordering_base.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_REORDER_REORDERING_BASE_HPP_
-#define GKO_CORE_REORDER_REORDERING_BASE_HPP_
+#ifndef GKO_PUBLIC_CORE_REORDER_REORDERING_BASE_HPP_
+#define GKO_PUBLIC_CORE_REORDER_REORDERING_BASE_HPP_
 
 
 #include <memory>
@@ -172,4 +172,4 @@ public:                                                                        \
 }  // namespace gko
 
 
-#endif  // GKO_CORE_REORDER_REORDERING_BASE_HPP_
+#endif  // GKO_PUBLIC_CORE_REORDER_REORDERING_BASE_HPP_
diff --git a/include/ginkgo/core/solver/bicg.hpp b/include/ginkgo/core/solver/bicg.hpp
index 19a92365050..570e968e468 100644
--- a/include/ginkgo/core/solver/bicg.hpp
+++ b/include/ginkgo/core/solver/bicg.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_BICG_HPP_
-#define GKO_CORE_SOLVER_BICG_HPP_
+#ifndef GKO_PUBLIC_CORE_SOLVER_BICG_HPP_
+#define GKO_PUBLIC_CORE_SOLVER_BICG_HPP_
 
 
 #include <vector>
@@ -62,8 +62,9 @@ namespace solver {
  * the capability to solve generic systems.
  *
  * BiCG is based on the bi-Lanczos tridiagonalization method and in exact
- * arithmetic should terminate in at most N iterations (2N MV's, with A and A^H).
- * It forms the basis of many of the cheaper methods such as BiCGSTAB and CGS.
+ * arithmetic should terminate in at most N iterations (2N MV's, with A and
+ * A^H). It forms the basis of many of the cheaper methods such as BiCGSTAB and
+ * CGS.
  *
  * Reference: R.Fletcher, Conjugate gradient methods for indefinite systems,
  * doi: 10.1007/BFb0080116
@@ -194,4 +195,4 @@ class Bicg : public EnableLinOp<Bicg<ValueType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_BICG_HPP_
+#endif  // GKO_PUBLIC_CORE_SOLVER_BICG_HPP_
diff --git a/include/ginkgo/core/solver/bicgstab.hpp b/include/ginkgo/core/solver/bicgstab.hpp
index e6fce0d9f53..5a3aad0db06 100644
--- a/include/ginkgo/core/solver/bicgstab.hpp
+++ b/include/ginkgo/core/solver/bicgstab.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_BICGSTAB_HPP_
-#define GKO_CORE_SOLVER_BICGSTAB_HPP_
+#ifndef GKO_PUBLIC_CORE_SOLVER_BICGSTAB_HPP_
+#define GKO_PUBLIC_CORE_SOLVER_BICGSTAB_HPP_
 
 
 #include <vector>
@@ -193,4 +193,4 @@ class Bicgstab : public EnableLinOp<Bicgstab<ValueType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_BICGSTAB_HPP_
+#endif  // GKO_PUBLIC_CORE_SOLVER_BICGSTAB_HPP_
diff --git a/include/ginkgo/core/solver/cg.hpp b/include/ginkgo/core/solver/cg.hpp
index fbe94677933..88aa379a972 100644
--- a/include/ginkgo/core/solver/cg.hpp
+++ b/include/ginkgo/core/solver/cg.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_CG_HPP_
-#define GKO_CORE_SOLVER_CG_HPP_
+#ifndef GKO_PUBLIC_CORE_SOLVER_CG_HPP_
+#define GKO_PUBLIC_CORE_SOLVER_CG_HPP_
 
 
 #include <vector>
@@ -189,4 +189,4 @@ class Cg : public EnableLinOp<Cg<ValueType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_CG_HPP_
+#endif  // GKO_PUBLIC_CORE_SOLVER_CG_HPP_
diff --git a/include/ginkgo/core/solver/cgs.hpp b/include/ginkgo/core/solver/cgs.hpp
index 3d7c8c0a854..ab3d8bee0bb 100644
--- a/include/ginkgo/core/solver/cgs.hpp
+++ b/include/ginkgo/core/solver/cgs.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_CGS_HPP_
-#define GKO_CORE_SOLVER_CGS_HPP_
+#ifndef GKO_PUBLIC_CORE_SOLVER_CGS_HPP_
+#define GKO_PUBLIC_CORE_SOLVER_CGS_HPP_
 
 
 #include <vector>
@@ -186,4 +186,4 @@ class Cgs : public EnableLinOp<Cgs<ValueType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_CGS_HPP_
+#endif  // GKO_PUBLIC_CORE_SOLVER_CGS_HPP_
diff --git a/include/ginkgo/core/solver/fcg.hpp b/include/ginkgo/core/solver/fcg.hpp
index 3d25ff057bf..b1c1eaf1329 100644
--- a/include/ginkgo/core/solver/fcg.hpp
+++ b/include/ginkgo/core/solver/fcg.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_FCG_HPP_
-#define GKO_CORE_SOLVER_FCG_HPP_
+#ifndef GKO_PUBLIC_CORE_SOLVER_FCG_HPP_
+#define GKO_PUBLIC_CORE_SOLVER_FCG_HPP_
 
 
 #include <vector>
@@ -194,4 +194,4 @@ class Fcg : public EnableLinOp<Fcg<ValueType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_FCG_HPP_
+#endif  // GKO_PUBLIC_CORE_SOLVER_FCG_HPP_
diff --git a/include/ginkgo/core/solver/gmres.hpp b/include/ginkgo/core/solver/gmres.hpp
index 36142219a7f..327dc196d94 100644
--- a/include/ginkgo/core/solver/gmres.hpp
+++ b/include/ginkgo/core/solver/gmres.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_GMRES_HPP_
-#define GKO_CORE_SOLVER_GMRES_HPP_
+#ifndef GKO_PUBLIC_CORE_SOLVER_GMRES_HPP_
+#define GKO_PUBLIC_CORE_SOLVER_GMRES_HPP_
 
 
 #include <vector>
@@ -214,4 +214,4 @@ class Gmres : public EnableLinOp<Gmres<ValueType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_GMRES_HPP_
+#endif  // GKO_PUBLIC_CORE_SOLVER_GMRES_HPP_
diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index fb3d6a860a2..1d1e0b120f1 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_IDR_HPP_
-#define GKO_CORE_SOLVER_IDR_HPP_
+#ifndef GKO_PUBLIC_CORE_SOLVER_IDR_HPP_
+#define GKO_PUBLIC_CORE_SOLVER_IDR_HPP_
 
 
 #include <iostream>
@@ -298,4 +298,4 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_IDR_HPP_
+#endif  // GKO_PUBLIC_CORE_SOLVER_IDR_HPP_
diff --git a/include/ginkgo/core/solver/ir.hpp b/include/ginkgo/core/solver/ir.hpp
index 1059c733614..a34d6ea6d41 100644
--- a/include/ginkgo/core/solver/ir.hpp
+++ b/include/ginkgo/core/solver/ir.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_IR_HPP_
-#define GKO_CORE_SOLVER_IR_HPP_
+#ifndef GKO_PUBLIC_CORE_SOLVER_IR_HPP_
+#define GKO_PUBLIC_CORE_SOLVER_IR_HPP_
 
 
 #include <vector>
@@ -251,4 +251,4 @@ using Richardson = Ir<ValueType>;
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_IR_HPP_
+#endif  // GKO_PUBLIC_CORE_SOLVER_IR_HPP_
diff --git a/include/ginkgo/core/solver/lower_trs.hpp b/include/ginkgo/core/solver/lower_trs.hpp
index 8681870ab16..b87c6528d00 100644
--- a/include/ginkgo/core/solver/lower_trs.hpp
+++ b/include/ginkgo/core/solver/lower_trs.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_LOWER_TRS_HPP_
-#define GKO_CORE_SOLVER_LOWER_TRS_HPP_
+#ifndef GKO_PUBLIC_CORE_SOLVER_LOWER_TRS_HPP_
+#define GKO_PUBLIC_CORE_SOLVER_LOWER_TRS_HPP_
 
 
 #include <memory>
@@ -173,4 +173,4 @@ class LowerTrs : public EnableLinOp<LowerTrs<ValueType, IndexType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_LOWER_TRS_HPP_
+#endif  // GKO_PUBLIC_CORE_SOLVER_LOWER_TRS_HPP_
diff --git a/include/ginkgo/core/solver/upper_trs.hpp b/include/ginkgo/core/solver/upper_trs.hpp
index b3c3a12d8fa..5d41637dc2c 100644
--- a/include/ginkgo/core/solver/upper_trs.hpp
+++ b/include/ginkgo/core/solver/upper_trs.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_UPPER_TRS_HPP_
-#define GKO_CORE_SOLVER_UPPER_TRS_HPP_
+#ifndef GKO_PUBLIC_CORE_SOLVER_UPPER_TRS_HPP_
+#define GKO_PUBLIC_CORE_SOLVER_UPPER_TRS_HPP_
 
 
 #include <memory>
@@ -173,4 +173,4 @@ class UpperTrs : public EnableLinOp<UpperTrs<ValueType, IndexType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_UPPER_TRS_HPP_
+#endif  // GKO_PUBLIC_CORE_SOLVER_UPPER_TRS_HPP_
diff --git a/include/ginkgo/core/stop/combined.hpp b/include/ginkgo/core/stop/combined.hpp
index 647c919af19..33bad75df69 100644
--- a/include/ginkgo/core/stop/combined.hpp
+++ b/include/ginkgo/core/stop/combined.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_STOP_COMBINED_HPP_
-#define GKO_CORE_STOP_COMBINED_HPP_
+#ifndef GKO_PUBLIC_CORE_STOP_COMBINED_HPP_
+#define GKO_PUBLIC_CORE_STOP_COMBINED_HPP_
 
 
 #include <vector>
@@ -150,4 +150,4 @@ std::shared_ptr<const CriterionFactory> combine(FactoryContainer &&factories)
 }  // namespace gko
 
 
-#endif  // GKO_CORE_STOP_COMBINED_HPP_
+#endif  // GKO_PUBLIC_CORE_STOP_COMBINED_HPP_
diff --git a/include/ginkgo/core/stop/criterion.hpp b/include/ginkgo/core/stop/criterion.hpp
index 41d48b2d9ae..e309df9bc42 100644
--- a/include/ginkgo/core/stop/criterion.hpp
+++ b/include/ginkgo/core/stop/criterion.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_STOP_CRITERION_HPP_
-#define GKO_CORE_STOP_CRITERION_HPP_
+#ifndef GKO_PUBLIC_CORE_STOP_CRITERION_HPP_
+#define GKO_PUBLIC_CORE_STOP_CRITERION_HPP_
 
 
 #include <ginkgo/core/base/abstract_factory.hpp>
@@ -327,4 +327,4 @@ public:                                                                      \
 }  // namespace gko
 
 
-#endif  // GKO_CORE_STOP_CRITERION_HPP_
+#endif  // GKO_PUBLIC_CORE_STOP_CRITERION_HPP_
diff --git a/include/ginkgo/core/stop/iteration.hpp b/include/ginkgo/core/stop/iteration.hpp
index 8bab8724291..55100a4c121 100644
--- a/include/ginkgo/core/stop/iteration.hpp
+++ b/include/ginkgo/core/stop/iteration.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_STOP_ITERATION_HPP_
-#define GKO_CORE_STOP_ITERATION_HPP_
+#ifndef GKO_PUBLIC_CORE_STOP_ITERATION_HPP_
+#define GKO_PUBLIC_CORE_STOP_ITERATION_HPP_
 
 
 #include <ginkgo/core/stop/criterion.hpp>
@@ -84,4 +84,4 @@ class Iteration : public EnablePolymorphicObject<Iteration, Criterion> {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_STOP_ITERATION_HPP_
+#endif  // GKO_PUBLIC_CORE_STOP_ITERATION_HPP_
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index c1aa45525e3..f786a84056b 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_STOP_RESIDUAL_NORM_HPP_
-#define GKO_CORE_STOP_RESIDUAL_NORM_HPP_
+#ifndef GKO_PUBLIC_CORE_STOP_RESIDUAL_NORM_HPP_
+#define GKO_PUBLIC_CORE_STOP_RESIDUAL_NORM_HPP_
 
 
 #include <type_traits>
@@ -290,4 +290,4 @@ class AbsoluteResidualNorm : public ResidualNorm<ValueType> {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_STOP_RESIDUAL_NORM_HPP_
+#endif  // GKO_PUBLIC_CORE_STOP_RESIDUAL_NORM_HPP_
diff --git a/include/ginkgo/core/stop/residual_norm_reduction.hpp b/include/ginkgo/core/stop/residual_norm_reduction.hpp
index 6872b7be5c2..f9c9ff7c61f 100644
--- a/include/ginkgo/core/stop/residual_norm_reduction.hpp
+++ b/include/ginkgo/core/stop/residual_norm_reduction.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_STOP_RESIDUAL_NORM_REDUCTION_HPP_
-#define GKO_CORE_STOP_RESIDUAL_NORM_REDUCTION_HPP_
+#ifndef GKO_PUBLIC_CORE_STOP_RESIDUAL_NORM_REDUCTION_HPP_
+#define GKO_PUBLIC_CORE_STOP_RESIDUAL_NORM_REDUCTION_HPP_
 
 
 #ifdef __GNUC__
@@ -44,4 +44,4 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/residual_norm.hpp>
 
 
-#endif  // GKO_CORE_STOP_RESIDUAL_NORM_REDUCTION_HPP_
+#endif  // GKO_PUBLIC_CORE_STOP_RESIDUAL_NORM_REDUCTION_HPP_
diff --git a/include/ginkgo/core/stop/stopping_status.hpp b/include/ginkgo/core/stop/stopping_status.hpp
index 679d78be4de..415c6e3cffe 100644
--- a/include/ginkgo/core/stop/stopping_status.hpp
+++ b/include/ginkgo/core/stop/stopping_status.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_STOP_STOPPING_STATUS_HPP_
-#define GKO_CORE_STOP_STOPPING_STATUS_HPP_
+#ifndef GKO_PUBLIC_CORE_STOP_STOPPING_STATUS_HPP_
+#define GKO_PUBLIC_CORE_STOP_STOPPING_STATUS_HPP_
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -184,4 +184,4 @@ GKO_ATTRIBUTES GKO_INLINE bool operator!=(const stopping_status &x,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_STOP_STOPPING_STATUS_HPP_
+#endif  // GKO_PUBLIC_CORE_STOP_STOPPING_STATUS_HPP_
diff --git a/include/ginkgo/core/stop/time.hpp b/include/ginkgo/core/stop/time.hpp
index 0b4e90baf7a..e7cf7621d30 100644
--- a/include/ginkgo/core/stop/time.hpp
+++ b/include/ginkgo/core/stop/time.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_STOP_TIME_HPP_
-#define GKO_CORE_STOP_TIME_HPP_
+#ifndef GKO_PUBLIC_CORE_STOP_TIME_HPP_
+#define GKO_PUBLIC_CORE_STOP_TIME_HPP_
 
 
 #include <chrono>
@@ -100,4 +100,4 @@ class Time : public EnablePolymorphicObject<Time, Criterion> {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_STOP_TIME_HPP_
+#endif  // GKO_PUBLIC_CORE_STOP_TIME_HPP_
diff --git a/include/ginkgo/core/synthesizer/containers.hpp b/include/ginkgo/core/synthesizer/containers.hpp
index b75936a9adc..1f3901c22c1 100644
--- a/include/ginkgo/core/synthesizer/containers.hpp
+++ b/include/ginkgo/core/synthesizer/containers.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SYNTHESIZER_CONTAINERS_HPP_
-#define GKO_CORE_SYNTHESIZER_CONTAINERS_HPP_
+#ifndef GKO_PUBLIC_CORE_SYNTHESIZER_CONTAINERS_HPP_
+#define GKO_PUBLIC_CORE_SYNTHESIZER_CONTAINERS_HPP_
 
 
 #include <type_traits>
@@ -117,4 +117,4 @@ using as_list = typename detail::as_list_impl<T>::type;
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SYNTHESIZER_CONTAINERS_HPP_
+#endif  // GKO_PUBLIC_CORE_SYNTHESIZER_CONTAINERS_HPP_
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index d6aaeddee3b..ce91497b1ee 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -33,10 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
-#include <time.h>
 #include <algorithm>
 #include <iostream>
 #include <random>
+#include <time.h>
 
 
 #include <omp.h>
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index 6bc8e264997..70695b7762f 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -30,8 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <ginkgo/core/solver/bicgstab.hpp>
-#include <ginkgo/core/solver/gmres.hpp>
 #include <ginkgo/core/solver/idr.hpp>
 
 
@@ -48,6 +46,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/mtx_io.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/solver/bicgstab.hpp>
+#include <ginkgo/core/solver/gmres.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm.hpp>
diff --git a/reference/components/csr_spgeam.hpp b/reference/components/csr_spgeam.hpp
index f09b34d5926..3f313df5756 100644
--- a/reference/components/csr_spgeam.hpp
+++ b/reference/components/csr_spgeam.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_OMP_COMPONENTS_CSR_SPGEAM_HPP_
-#define GKO_OMP_COMPONENTS_CSR_SPGEAM_HPP_
+#ifndef GKO_REFERENCE_COMPONENTS_CSR_SPGEAM_HPP_
+#define GKO_REFERENCE_COMPONENTS_CSR_SPGEAM_HPP_
 
 
 #include <limits>
@@ -109,4 +109,4 @@ void abstract_spgeam(const matrix::Csr<ValueType, IndexType> *a,
 }  // namespace gko
 
 
-#endif  // GKO_OMP_COMPONENTS_CSR_SPGEAM_HPP_
+#endif  // GKO_REFERENCE_COMPONENTS_CSR_SPGEAM_HPP_
diff --git a/reference/reorder/rcm_kernels.cpp b/reference/reorder/rcm_kernels.cpp
index 452b8c877ac..74b02fff26b 100644
--- a/reference/reorder/rcm_kernels.cpp
+++ b/reference/reorder/rcm_kernels.cpp
@@ -30,7 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-
 #include "core/reorder/rcm_kernels.hpp"
 
 
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index f9d743a9077..eff6751fc85 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -33,10 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
-#include <time.h>
 #include <algorithm>
 #include <iostream>
 #include <random>
+#include <time.h>
 
 
 #include <ginkgo/core/base/array.hpp>
diff --git a/reference/test/preconditioner/isai_kernels.cpp b/reference/test/preconditioner/isai_kernels.cpp
index 6fc099189df..ff578096391 100644
--- a/reference/test/preconditioner/isai_kernels.cpp
+++ b/reference/test/preconditioner/isai_kernels.cpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "reference/preconditioner/isai_kernels.cpp"
+#include <ginkgo/core/preconditioner/isai.hpp>
 
 
 #include <algorithm>
@@ -47,11 +47,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/preconditioner/ilu.hpp>
-#include <ginkgo/core/preconditioner/isai.hpp>
 
 
 #include "core/test/utils.hpp"
 #include "matrices/config.hpp"
+#include "reference/preconditioner/isai_kernels.cpp"
 
 
 namespace {
diff --git a/reference/test/reorder/rcm_kernels.cpp b/reference/test/reorder/rcm_kernels.cpp
index a12c94ec7ae..3905918e3d7 100644
--- a/reference/test/reorder/rcm_kernels.cpp
+++ b/reference/test/reorder/rcm_kernels.cpp
@@ -113,7 +113,7 @@ class Rcm : public ::testing::Test {
 TEST_F(Rcm, CreatesAPermutation)
 {
     auto p = reorder_op_0->get_permutation();
-    
+
     ASSERT_PRED1(is_permutation, p.get());
 }
 
diff --git a/test_install/test_install.cpp b/test_install/test_install.cpp
index 5ea59794440..c80b0976b34 100644
--- a/test_install/test_install.cpp
+++ b/test_install/test_install.cpp
@@ -357,13 +357,13 @@ int main(int, char **)
                            .with_reduction_factor(1e-10)
                            .on(refExec);
 
-        auto rel_res = gko::stop::RelativeResidualNorm<>::build()
-                           .with_tolerance(1e-10)
-                           .on(refExec);
+        auto rel_res =
+            gko::stop::RelativeResidualNorm<>::build().with_tolerance(1e-10).on(
+                refExec);
 
-        auto abs_res = gko::stop::AbsoluteResidualNorm<>::build()
-                           .with_tolerance(1e-10)
-                           .on(refExec);
+        auto abs_res =
+            gko::stop::AbsoluteResidualNorm<>::build().with_tolerance(1e-10).on(
+                refExec);
 
         // stopping_status.hpp
         auto stop_status = gko::stopping_status{};
diff --git a/test_install/test_install_cuda.cu b/test_install/test_install_cuda.cu
index ed2e18c307d..a4f46b371f8 100644
--- a/test_install/test_install_cuda.cu
+++ b/test_install/test_install_cuda.cu
@@ -349,13 +349,11 @@ int main(int, char **)
             .with_reduction_factor(1e-10)
             .on(cudaExec);
 
-        gko::stop::RelativeResidualNorm<>::build()
-            .with_tolerance(1e-10)
-            .on(cudaExec);
+        gko::stop::RelativeResidualNorm<>::build().with_tolerance(1e-10).on(
+            cudaExec);
 
-        gko::stop::AbsoluteResidualNorm<>::build()
-            .with_tolerance(1e-10)
-            .on(cudaExec);
+        gko::stop::AbsoluteResidualNorm<>::build().with_tolerance(1e-10).on(
+            cudaExec);
 
         // stopping_status.hpp
         gko::stopping_status{};

From af3441fc47207a4304eef722b7d24b9ff3745603 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 2 Dec 2020 13:21:13 +0100
Subject: [PATCH 0170/1008] update bot to delete old comments

---
 .github/bot-pr-base.sh               | 8 ++++++++
 .github/check-format.sh              | 4 ++--
 .github/format.sh                    | 2 --
 .github/rebase.sh                    | 2 ++
 .github/workflows/bot-pr-comment.yml | 8 ++++----
 .github/workflows/bot-pr-created.yml | 2 +-
 .github/workflows/bot-pr-updated.yml | 2 +-
 7 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/.github/bot-pr-base.sh b/.github/bot-pr-base.sh
index 421ef773e2d..cdb0787c410 100644
--- a/.github/bot-pr-base.sh
+++ b/.github/bot-pr-base.sh
@@ -41,6 +41,14 @@ HEAD_URL="https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/$HEAD_REPO"
 
 JOB_URL="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
 
+bot_delete_comments_matching() {
+  local search_matching="$1"
+  COMMENTS=$(api_get "$ISSUE_URL/comments" | jq -r '.[] | select((.user.login == "ginkgo-bot") and (.body | startswith('"\"$search_matching\""'))) | .url')
+  for URL in $COMMENTS; do
+    api_delete "$URL" > /dev/null
+  done
+}
+
 bot_comment() {
   api_post "$ISSUE_URL/comments" "{\"body\":\"$1\"}" > /dev/null
 }
diff --git a/.github/check-format.sh b/.github/check-format.sh
index 704a78690b6..635b64c943f 100755
--- a/.github/check-format.sh
+++ b/.github/check-format.sh
@@ -9,11 +9,11 @@ LIST_FILES=$(git diff --name-only | sed '$!s/$/\\n/' | tr -d '\n')
 git diff > /tmp/format.patch
 mv /tmp/format.patch .
 
+bot_delete_comments_matching "Error: The following files need to be formatted"
+
 if [[ "$LIST_FILES" != "" ]]; then
   MESSAGE="The following files need to be formatted:\n"'```'"\n$LIST_FILES\n"'```'
   MESSAGE="$MESSAGE\nYou can find a formatting patch under **Artifacts** [here]"
   MESSAGE="$MESSAGE($JOB_URL) or run "'`format!` if you have write access to Ginkgo'
   bot_error "$MESSAGE"
-else
-  bot_comment "No formatting necessary"
 fi
diff --git a/.github/format.sh b/.github/format.sh
index 581bd1bdb38..8829b719b8f 100755
--- a/.github/format.sh
+++ b/.github/format.sh
@@ -12,6 +12,4 @@ if [[ "$LIST_FILES" != "" ]]; then
 
 Co-authored-by: $USER_COMBINED"
   git push fork $LOCAL_BRANCH:$HEAD_BRANCH 2>&1 || bot_error "Cannot push formatted branch, are edits for maintainers allowed?"
-else
-  bot_comment "No formatting necessary"
 fi
diff --git a/.github/rebase.sh b/.github/rebase.sh
index 6377ae4030e..7367ed04fb1 100755
--- a/.github/rebase.sh
+++ b/.github/rebase.sh
@@ -14,6 +14,8 @@ git config user.name "$USER_NAME"
 LOCAL_BRANCH=rebase-tmp-$HEAD_BRANCH
 git checkout -b $LOCAL_BRANCH fork/$HEAD_BRANCH
 
+bot_delete_comments_matching "Error: Rebase failed"
+
 # do the rebase
 git rebase base/$BASE_BRANCH 2>&1 || bot_error "Rebase failed, see the related [Action]($JOB_URL) for details"
 
diff --git a/.github/workflows/bot-pr-comment.yml b/.github/workflows/bot-pr-comment.yml
index da198121140..61952428f3e 100644
--- a/.github/workflows/bot-pr-comment.yml
+++ b/.github/workflows/bot-pr-comment.yml
@@ -14,7 +14,7 @@ jobs:
         fetch-depth: 0
     - name: Add appropriate labels
       env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
       run: cp .github/label.sh /tmp && /tmp/label.sh
   check_format:
     name: check-format
@@ -28,7 +28,7 @@ jobs:
         fetch-depth: 0
     - name: Check for formatting changes
       env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
       run: cp .github/check-format.sh /tmp && /tmp/check-format.sh
     - name: Upload code formatting patch
       if: failure()
@@ -48,7 +48,7 @@ jobs:
         fetch-depth: 0
     - name: Commit formatting changes
       env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
       run: cp .github/format.sh /tmp && /tmp/format.sh
   rebase:
     name: rebase
@@ -62,5 +62,5 @@ jobs:
         fetch-depth: 0
     - name: Automatic Rebase
       env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
       run: cp .github/rebase.sh /tmp && /tmp/rebase.sh
diff --git a/.github/workflows/bot-pr-created.yml b/.github/workflows/bot-pr-created.yml
index 75a6a7e4564..d86d6f5c869 100644
--- a/.github/workflows/bot-pr-created.yml
+++ b/.github/workflows/bot-pr-created.yml
@@ -15,5 +15,5 @@ jobs:
           fetch-depth: 0
       - name: Add appropriate labels
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
         run: .github/label.sh
diff --git a/.github/workflows/bot-pr-updated.yml b/.github/workflows/bot-pr-updated.yml
index 44d78473179..b0f0c2e5570 100644
--- a/.github/workflows/bot-pr-updated.yml
+++ b/.github/workflows/bot-pr-updated.yml
@@ -15,7 +15,7 @@ jobs:
           fetch-depth: 0
       - name: Check for formatting changes
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
         run: cp .github/check-format.sh /tmp && /tmp/check-format.sh
       - name: Upload code formatting patch
         if: failure()

From 42b45b4a6561edf63788cd26630478131d66972b Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 2 Dec 2020 19:52:38 +0100
Subject: [PATCH 0171/1008] fix ISAI test include

---
 reference/test/preconditioner/isai_kernels.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reference/test/preconditioner/isai_kernels.cpp b/reference/test/preconditioner/isai_kernels.cpp
index ff578096391..5ed332f7e95 100644
--- a/reference/test/preconditioner/isai_kernels.cpp
+++ b/reference/test/preconditioner/isai_kernels.cpp
@@ -49,9 +49,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/preconditioner/ilu.hpp>
 
 
+#include "core/preconditioner/isai_kernels.hpp"
 #include "core/test/utils.hpp"
 #include "matrices/config.hpp"
-#include "reference/preconditioner/isai_kernels.cpp"
 
 
 namespace {

From 4a1e89025ef009682c043659b246438547156a87 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 1 Dec 2020 15:16:27 +0100
Subject: [PATCH 0172/1008] add idr in solver_factory

---
 benchmark/solver/solver.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 11169effae8..05b9a5744ba 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -151,6 +151,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                    {"cg", create_solver<gko::solver::Cg<>>},
                    {"cgs", create_solver<gko::solver::Cgs<>>},
                    {"fcg", create_solver<gko::solver::Fcg<>>},
+                   {"idr", create_solver<gko::solver::Idr<>>},
                    {"gmres",
                     [](std::shared_ptr<const gko::Executor> exec,
                        std::shared_ptr<const gko::LinOpFactory> precond) {

From e1a4d2e461630aaac28a4b5ecee67757ca05bc24 Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Tue, 1 Dec 2020 14:24:42 +0000
Subject: [PATCH 0173/1008] Format files

Co-authored-by: Yuhsiang M. Tsai <yhmtsai@gmail.com>
---
 benchmark/solver/solver.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 05b9a5744ba..5e893b7c9e4 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -43,6 +43,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <iostream>
 
 
+#include <ginkgo/core/preconditioner/isai.hpp>
+
+
 #include "benchmark/utils/formats.hpp"
 #include "benchmark/utils/general.hpp"
 #include "benchmark/utils/loggers.hpp"

From bc42199d5bf866a4e557da1df45e563ca263f434 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 2 Dec 2020 14:53:49 +0100
Subject: [PATCH 0174/1008] review update and add create_criterion - add
 subspace_dim and kappa for idr - remove unnecessary header - add
 create_critrion to keep the consistent criteria
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 benchmark/solver/solver.cpp | 55 ++++++++++++++++++++++++-------------
 1 file changed, 36 insertions(+), 19 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 5e893b7c9e4..a02c8d1a30c 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -41,9 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <fstream>
 #include <iomanip>
 #include <iostream>
-
-
-#include <ginkgo/core/preconditioner/isai.hpp>
+#include <vector>
 
 
 #include "benchmark/utils/formats.hpp"
@@ -79,6 +77,13 @@ DEFINE_uint32(
 DEFINE_uint32(gmres_restart, 100,
               "What maximum dimension of the Krylov space to use in GMRES");
 
+DEFINE_uint32(idr_subspace_dim, 2,
+              "What dimension of the subspace to use in IDR");
+
+DEFINE_double(
+    idr_kappa, 0.7,
+    "the number to check whether Av_n and v_n are too close or not in IDR");
+
 DEFINE_bool(random_rhs, false,
             "Use a random vector for the rhs (otherwise use all ones)");
 
@@ -120,11 +125,8 @@ void validate_option_object(const rapidjson::Value &value)
 }
 
 
-// solver mapping
-template <typename SolverType>
-std::unique_ptr<gko::LinOpFactory> create_solver(
-    std::shared_ptr<const gko::Executor> exec,
-    std::shared_ptr<const gko::LinOpFactory> precond)
+std::shared_ptr<const gko::stop::CriterionFactory> create_criterion(
+    std::shared_ptr<const gko::Executor> exec)
 {
     std::shared_ptr<const gko::stop::CriterionFactory> residual_stop;
     if (FLAGS_rel_residual) {
@@ -137,10 +139,22 @@ std::unique_ptr<gko::LinOpFactory> create_solver(
                            .with_reduction_factor(FLAGS_rel_res_goal)
                            .on(exec));
     }
+    auto iteration_stop = gko::share(
+        gko::stop::Iteration::build().with_max_iters(FLAGS_max_iters).on(exec));
+    std::vector<std::shared_ptr<const gko::stop::CriterionFactory>>
+        criterion_vector{residual_stop, iteration_stop};
+    return gko::stop::combine(criterion_vector);
+}
+
+
+// solver mapping
+template <typename SolverType>
+std::unique_ptr<gko::LinOpFactory> create_solver(
+    std::shared_ptr<const gko::Executor> exec,
+    std::shared_ptr<const gko::LinOpFactory> precond)
+{
     return SolverType::build()
-        .with_criteria(residual_stop, gko::stop::Iteration::build()
-                                          .with_max_iters(FLAGS_max_iters)
-                                          .on(exec))
+        .with_criteria(create_criterion(exec))
         .with_preconditioner(give(precond))
         .on(exec);
 }
@@ -154,18 +168,21 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                    {"cg", create_solver<gko::solver::Cg<>>},
                    {"cgs", create_solver<gko::solver::Cgs<>>},
                    {"fcg", create_solver<gko::solver::Fcg<>>},
-                   {"idr", create_solver<gko::solver::Idr<>>},
+                   {"idr",
+                    [](std::shared_ptr<const gko::Executor> exec,
+                       std::shared_ptr<const gko::LinOpFactory> precond) {
+                        return gko::solver::Idr<>::build()
+                            .with_criteria(create_criterion(exec))
+                            .with_subspace_dim(FLAGS_idr_subspace_dim)
+                            .with_kappa(FLAGS_idr_kappa)
+                            .with_preconditioner(give(precond))
+                            .on(exec);
+                    }},
                    {"gmres",
                     [](std::shared_ptr<const gko::Executor> exec,
                        std::shared_ptr<const gko::LinOpFactory> precond) {
                         return gko::solver::Gmres<>::build()
-                            .with_criteria(
-                                gko::stop::ResidualNormReduction<>::build()
-                                    .with_reduction_factor(FLAGS_rel_res_goal)
-                                    .on(exec),
-                                gko::stop::Iteration::build()
-                                    .with_max_iters(FLAGS_max_iters)
-                                    .on(exec))
+                            .with_criteria(create_criterion(exec))
                             .with_krylov_dim(FLAGS_gmres_restart)
                             .with_preconditioner(give(precond))
                             .on(exec);

From d6ed482e6afe10bb2b932fb8e5fdc58094c62189 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 11 Sep 2020 17:28:35 +0200
Subject: [PATCH 0175/1008] Add executor operator== for memory compatibility

Co-authored-with: Terry Cojean <terry.cojean@kit.edu>
---
 core/device_hooks/cuda_hooks.cpp              |   3 +
 core/device_hooks/dpcpp_hooks.cpp             |  28 +
 core/device_hooks/hip_hooks.cpp               |   3 +
 core/test/base/array.cpp                      |  20 +-
 core/test/base/executor.cpp                   |  59 ++
 core/test/base/lin_op.cpp                     |  35 +-
 core/test/base/utils.cpp                      |   4 +-
 core/test/utils.hpp                           |   2 +
 cuda/base/executor.cpp                        |  10 +
 cuda/test/base/CMakeLists.txt                 |   1 +
 cuda/test/base/array.cu                       |  92 +++
 cuda/test/base/lin_op.cu                      | 104 ++++
 dpcpp/base/executor.dp.cpp                    |  39 +-
 hip/base/executor.hip.cpp                     |  10 +
 include/ginkgo/core/base/dim.hpp              |   1 -
 .../ginkgo/core/base/exception_helpers.hpp    |   1 +
 include/ginkgo/core/base/executor.hpp         | 119 ++++
 include/ginkgo/core/base/temporary_clone.hpp  | 202 +++++++
 include/ginkgo/core/base/utils.hpp            | 553 +-----------------
 include/ginkgo/core/base/utils_helper.hpp     | 438 ++++++++++++++
 include/ginkgo/core/log/logger.hpp            |   2 +-
 21 files changed, 1143 insertions(+), 583 deletions(-)
 create mode 100644 cuda/test/base/array.cu
 create mode 100644 include/ginkgo/core/base/temporary_clone.hpp
 create mode 100644 include/ginkgo/core/base/utils_helper.hpp

diff --git a/core/device_hooks/cuda_hooks.cpp b/core/device_hooks/cuda_hooks.cpp
index 46e342d9c0a..a5c33c6541d 100644
--- a/core/device_hooks/cuda_hooks.cpp
+++ b/core/device_hooks/cuda_hooks.cpp
@@ -98,6 +98,9 @@ void CudaExecutor::raw_copy_to(const DpcppExecutor *, size_type num_bytes,
     GKO_NOT_COMPILED(cuda);
 
 
+bool CudaExecutor::verify_memory_to(const HipExecutor *) const { return false; }
+
+
 void CudaExecutor::synchronize() const GKO_NOT_COMPILED(cuda);
 
 
diff --git a/core/device_hooks/dpcpp_hooks.cpp b/core/device_hooks/dpcpp_hooks.cpp
index 9d6c57f11a4..472451c6513 100644
--- a/core/device_hooks/dpcpp_hooks.cpp
+++ b/core/device_hooks/dpcpp_hooks.cpp
@@ -64,6 +64,14 @@ void OmpExecutor::raw_copy_to(const DpcppExecutor *, size_type num_bytes,
     GKO_NOT_COMPILED(dpcpp);
 
 
+bool OmpExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
+{
+    // Dummy check
+    auto dev_type = dest_exec->get_device_type();
+    return dev_type == "cpu" || dev_type == "host";
+}
+
+
 void DpcppExecutor::raw_free(void *ptr) const noexcept
 {
     // Free must never fail, as it can be called in destructors.
@@ -112,6 +120,26 @@ int DpcppExecutor::get_num_devices(std::string) { return 0; }
 void DpcppExecutor::set_device_property() {}
 
 
+bool DpcppExecutor::verify_memory_to(const OmpExecutor *dest_exec) const
+{
+    // Dummy check
+    return device_type_ == "cpu" || device_type_ == "host";
+}
+
+bool DpcppExecutor::verify_memory_to(const ReferenceExecutor *dest_exec) const
+{
+    // Dummy check
+    return device_type_ == "cpu" || device_type_ == "host";
+}
+
+bool DpcppExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
+{
+    // Dummy check
+    return dest_exec->get_device_type() == device_type_ &&
+           dest_exec->get_device_id() == device_id_;
+}
+
+
 }  // namespace gko
 
 
diff --git a/core/device_hooks/hip_hooks.cpp b/core/device_hooks/hip_hooks.cpp
index b98b2fcb8a4..45180980159 100644
--- a/core/device_hooks/hip_hooks.cpp
+++ b/core/device_hooks/hip_hooks.cpp
@@ -95,6 +95,9 @@ void HipExecutor::raw_copy_to(const DpcppExecutor *, size_type num_bytes,
     GKO_NOT_COMPILED(hip);
 
 
+bool HipExecutor::verify_memory_to(const CudaExecutor *) const { return false; }
+
+
 void HipExecutor::synchronize() const GKO_NOT_COMPILED(hip);
 
 
diff --git a/core/test/base/array.cpp b/core/test/base/array.cpp
index 959392e5c71..c20782651f7 100644
--- a/core/test/base/array.cpp
+++ b/core/test/base/array.cpp
@@ -57,12 +57,13 @@ class Array : public ::testing::Test {
         x.get_data()[1] = 2;
     }
 
-    static void assert_equal_to_original_x(gko::Array<T> &a)
+    static void assert_equal_to_original_x(gko::Array<T> &a,
+                                           bool check_zero = true)
     {
         ASSERT_EQ(a.get_num_elems(), 2);
-        EXPECT_EQ(a.get_data()[0], T{5});
+        if (check_zero) EXPECT_EQ(a.get_data()[0], T{5});
         EXPECT_EQ(a.get_data()[1], T{2});
-        EXPECT_EQ(a.get_const_data()[0], T{5});
+        if (check_zero) EXPECT_EQ(a.get_const_data()[0], T{5});
         EXPECT_EQ(a.get_const_data()[1], T{2});
     }
 
@@ -271,29 +272,30 @@ TYPED_TEST(Array, CanCreateTemporaryCloneOnSameExecutor)
 }
 
 
-TYPED_TEST(Array, CanCreateTemporaryCloneOnDifferentExecutor)
+// For tests between different memory, check cuda/test/base/array.cu
+TYPED_TEST(Array, DoesNotCreateATemporaryCloneBetweenSameMemory)
 {
     auto omp = gko::OmpExecutor::create();
 
     auto tmp_clone = make_temporary_clone(omp, &this->x);
 
     this->assert_equal_to_original_x(*tmp_clone.get());
-    ASSERT_NE(tmp_clone.get(), &this->x);
+    ASSERT_EQ(tmp_clone.get(), &this->x);
 }
 
 
-TYPED_TEST(Array, CanCopyBackTemporaryCloneOnDifferentExecutor)
+TYPED_TEST(Array, CanDoesNotCopyBackTemporaryCloneBetweenSameMemory)
 {
     auto omp = gko::OmpExecutor::create();
 
     {
         auto tmp_clone = make_temporary_clone(omp, &this->x);
-        // change x, so it no longer matches the original x
-        // the copy-back will overwrite it again with the correct value
+        // change x, and check that there is no copy-back to overwrite it again
         this->x.get_data()[0] = 0;
     }
 
-    this->assert_equal_to_original_x(this->x);
+    this->assert_equal_to_original_x(this->x, false);
+    EXPECT_EQ(this->x.get_data()[0], TypeParam{0});
 }
 
 
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index b968bbe210a..73c82a06fa8 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -474,6 +474,65 @@ TEST(DpcppExecutor, KnowsItsDeviceId)
 }
 
 
+TEST(Executor, CanVerifyMemory)
+{
+    auto ref = gko::ReferenceExecutor::create();
+    auto omp = gko::OmpExecutor::create();
+    auto hip = gko::HipExecutor::create(0, omp);
+    auto cuda = gko::CudaExecutor::create(0, omp);
+    auto cpu_dpcpp = gko::DpcppExecutor::create(0, omp, "cpu");
+    auto host_dpcpp = gko::DpcppExecutor::create(0, omp, "host");
+    auto gpu_dpcpp = gko::DpcppExecutor::create(0, omp, "gpu");
+    auto omp2 = gko::OmpExecutor::create();
+    auto hip2 = gko::HipExecutor::create(0, omp);
+    auto cuda2 = gko::CudaExecutor::create(0, omp);
+    std::shared_ptr<gko::HipExecutor> hip_1 = gko::HipExecutor::create(1, omp);
+    std::shared_ptr<gko::CudaExecutor> cuda_1 =
+        gko::CudaExecutor::create(1, omp);
+
+    ASSERT_EQ(true, *ref == *omp);
+    ASSERT_EQ(true, *omp == *ref);
+    ASSERT_EQ(false, *ref == *hip);
+    ASSERT_EQ(false, *hip == *ref);
+    ASSERT_EQ(false, *omp == *hip);
+    ASSERT_EQ(false, *hip == *omp);
+    ASSERT_EQ(false, *ref == *cuda);
+    ASSERT_EQ(false, *cuda == *ref);
+    ASSERT_EQ(false, *omp == *cuda);
+    ASSERT_EQ(false, *cuda == *omp);
+    ASSERT_EQ(true, *cpu_dpcpp == *ref);
+    ASSERT_EQ(true, *host_dpcpp == *ref);
+    ASSERT_EQ(false, *gpu_dpcpp == *ref);
+    ASSERT_EQ(true, *ref == *cpu_dpcpp);
+    ASSERT_EQ(true, *ref == *host_dpcpp);
+    ASSERT_EQ(false, *ref == *gpu_dpcpp);
+    ASSERT_EQ(true, *cpu_dpcpp == *omp);
+    ASSERT_EQ(true, *host_dpcpp == *omp);
+    ASSERT_EQ(false, *gpu_dpcpp == *omp);
+    ASSERT_EQ(true, *omp == *cpu_dpcpp);
+    ASSERT_EQ(true, *omp == *host_dpcpp);
+    ASSERT_EQ(false, *omp == *gpu_dpcpp);
+#if GINKGO_HIP_PLATFORM_NVCC
+    ASSERT_EQ(true, *hip == *cuda);
+    ASSERT_EQ(true, *cuda == *hip);
+    ASSERT_EQ(true, *hip_1 == *cuda_1);
+    ASSERT_EQ(true, *cuda_1 == *hip_1);
+#else
+    ASSERT_EQ(false, *hip == *cuda);
+    ASSERT_EQ(false, *cuda == *hip);
+    ASSERT_EQ(false, *hip_1 == *cuda_1);
+    ASSERT_EQ(false, *cuda_1 == *hip_1);
+#endif
+    ASSERT_EQ(true, *omp == *omp2);
+    ASSERT_EQ(true, *hip == *hip2);
+    ASSERT_EQ(true, *cuda == *cuda2);
+    ASSERT_EQ(false, *hip == *hip_1);
+    ASSERT_EQ(false, *cuda == *hip_1);
+    ASSERT_EQ(false, *cuda == *cuda_1);
+    ASSERT_EQ(false, *hip == *cuda_1);
+}
+
+
 template <typename T>
 struct mock_free : T {
     /**
diff --git a/core/test/base/lin_op.cpp b/core/test/base/lin_op.cpp
index 303e8bd6e89..3da8eda55f0 100644
--- a/core/test/base/lin_op.cpp
+++ b/core/test/base/lin_op.cpp
@@ -204,43 +204,44 @@ TEST_F(EnableLinOp, ExtendedApplyFailsOnWrongBetaDimension)
 }
 
 
-TEST_F(EnableLinOp, ApplyCopiesDataToCorrectExecutor)
+// For tests between different memory, check cuda/test/base/lin_op.cu
+TEST_F(EnableLinOp, ApplyDoesNotCopyBetweenSameMemory)
 {
     op->apply(gko::lend(b), gko::lend(x));
 
-    ASSERT_EQ(op->last_b_access, omp);
-    ASSERT_EQ(op->last_x_access, omp);
+    ASSERT_EQ(op->last_b_access, ref);
+    ASSERT_EQ(op->last_x_access, ref);
 }
 
 
-TEST_F(EnableLinOp, ApplyCopiesBackOnlyX)
+TEST_F(EnableLinOp, ApplyNoCopyBackBetweenSameMemory)
 {
     op->apply(gko::lend(b), gko::lend(x));
 
-    ASSERT_EQ(b->last_access, nullptr);
-    ASSERT_EQ(x->last_access, omp);
+    ASSERT_EQ(b->last_access, ref);
+    ASSERT_EQ(x->last_access, ref);
 }
 
 
-TEST_F(EnableLinOp, ExtendedApplyCopiesDataToCorrectExecutor)
+TEST_F(EnableLinOp, ExtendedApplyDoesNotCopyBetweenSameMemory)
 {
     op->apply(gko::lend(alpha), gko::lend(b), gko::lend(beta), gko::lend(x));
 
-    ASSERT_EQ(op->last_alpha_access, omp);
-    ASSERT_EQ(op->last_b_access, omp);
-    ASSERT_EQ(op->last_beta_access, omp);
-    ASSERT_EQ(op->last_x_access, omp);
+    ASSERT_EQ(op->last_alpha_access, ref);
+    ASSERT_EQ(op->last_b_access, ref);
+    ASSERT_EQ(op->last_beta_access, ref);
+    ASSERT_EQ(op->last_x_access, ref);
 }
 
 
-TEST_F(EnableLinOp, ExtendedApplyCopiesBackOnlyX)
+TEST_F(EnableLinOp, ExtendedApplyNoCopyBackBetweenSameMemory)
 {
-    op->apply(gko::lend(b), gko::lend(x));
+    op->apply(gko::lend(alpha), gko::lend(b), gko::lend(beta), gko::lend(x));
 
-    ASSERT_EQ(alpha->last_access, nullptr);
-    ASSERT_EQ(b->last_access, nullptr);
-    ASSERT_EQ(beta->last_access, nullptr);
-    ASSERT_EQ(x->last_access, omp);
+    ASSERT_EQ(alpha->last_access, ref);
+    ASSERT_EQ(b->last_access, ref);
+    ASSERT_EQ(beta->last_access, ref);
+    ASSERT_EQ(x->last_access, ref);
 }
 
 
diff --git a/core/test/base/utils.cpp b/core/test/base/utils.cpp
index df187cfb81f..822449e8f24 100644
--- a/core/test/base/utils.cpp
+++ b/core/test/base/utils.cpp
@@ -400,11 +400,11 @@ class TemporaryClone : public ::testing::Test {
 };
 
 
-TEST_F(TemporaryClone, CopiesToAnotherExecutor)
+TEST_F(TemporaryClone, DoesNotCopyToSameMemory)
 {
     auto clone = make_temporary_clone(omp, gko::lend(obj));
 
-    ASSERT_EQ(clone.get()->get_executor(), omp);
+    ASSERT_NE(clone.get()->get_executor(), omp);
     ASSERT_EQ(obj->get_executor(), ref);
 }
 
diff --git a/core/test/utils.hpp b/core/test/utils.hpp
index d6ae92bb951..5838379c5f0 100644
--- a/core/test/utils.hpp
+++ b/core/test/utils.hpp
@@ -69,6 +69,8 @@ using ValueAndIndexTypes =
     ::testing::Types<float, double, std::complex<float>, std::complex<double>,
                      gko::int32, gko::int64, gko::size_type>;
 
+using RealValueAndIndexTypes =
+    ::testing::Types<float, double, gko::int32, gko::int64, gko::size_type>;
 
 using ValueIndexTypes = ::testing::Types<
     std::tuple<float, gko::int32>, std::tuple<double, gko::int32>,
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 7926ae98a21..5284daa2dd1 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -162,6 +162,16 @@ void CudaExecutor::raw_copy_to(const CudaExecutor *dest, size_type num_bytes,
 }
 
 
+bool CudaExecutor::verify_memory_to(const HipExecutor *dest_exec) const
+{
+#if GINKGO_HIP_PLATFORM_NVCC
+    return device_id_ == dest_exec->get_device_id();
+#else
+    return false;
+#endif
+}
+
+
 void CudaExecutor::synchronize() const
 {
     cuda::device_guard g(this->get_device_id());
diff --git a/cuda/test/base/CMakeLists.txt b/cuda/test/base/CMakeLists.txt
index d9ba808fa21..acbe3244fd4 100644
--- a/cuda/test/base/CMakeLists.txt
+++ b/cuda/test/base/CMakeLists.txt
@@ -1,3 +1,4 @@
+ginkgo_create_cuda_test(array)
 ginkgo_create_cuda_test(cuda_executor)
 ginkgo_create_cuda_test(exception_helpers)
 ginkgo_create_cuda_test(lin_op)
diff --git a/cuda/test/base/array.cu b/cuda/test/base/array.cu
new file mode 100644
index 00000000000..c4757c42b44
--- /dev/null
+++ b/cuda/test/base/array.cu
@@ -0,0 +1,92 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/base/array.hpp>
+
+
+#include <gtest/gtest.h>
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+template <typename T>
+class Array : public ::testing::Test {
+protected:
+    Array() : exec(gko::ReferenceExecutor::create()), x(exec, 2)
+    {
+        x.get_data()[0] = 5;
+        x.get_data()[1] = 2;
+    }
+
+    static void assert_equal_to_original_x(gko::Array<T> &a)
+    {
+        ASSERT_EQ(a.get_num_elems(), 2);
+        EXPECT_EQ(a.get_data()[0], T{5});
+        EXPECT_EQ(a.get_data()[1], T{2});
+        EXPECT_EQ(a.get_const_data()[0], T{5});
+        EXPECT_EQ(a.get_const_data()[1], T{2});
+    }
+
+    std::shared_ptr<gko::Executor> exec;
+    gko::Array<T> x;
+};
+
+TYPED_TEST_SUITE(Array, gko::test::ValueAndIndexTypes);
+
+
+TYPED_TEST(Array, CanCreateTemporaryCloneOnDifferentExecutor)
+{
+    auto cuda = gko::CudaExecutor::create(0, this->exec);
+
+    auto tmp_clone = make_temporary_clone(cuda, &this->x);
+
+    ASSERT_NE(tmp_clone.get(), &this->x);
+    tmp_clone->set_executor(this->exec);
+    this->assert_equal_to_original_x(*tmp_clone.get());
+}
+
+
+TYPED_TEST(Array, CanCopyBackTemporaryCloneOnDifferentExecutor)
+{
+    auto cuda = gko::CudaExecutor::create(0, this->exec);
+
+    {
+        auto tmp_clone = make_temporary_clone(cuda, &this->x);
+        // change x, so it no longer matches the original x
+        // the copy-back will overwrite it again with the correct value
+        this->x.get_data()[0] = 0;
+    }
+
+    this->assert_equal_to_original_x(this->x);
+}
diff --git a/cuda/test/base/lin_op.cu b/cuda/test/base/lin_op.cu
index 82016f87816..668c2df339c 100644
--- a/cuda/test/base/lin_op.cu
+++ b/cuda/test/base/lin_op.cu
@@ -39,6 +39,110 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
+class DummyLinOp : public gko::EnableLinOp<DummyLinOp>,
+                   public gko::EnableCreateMethod<DummyLinOp> {
+public:
+    DummyLinOp(std::shared_ptr<const gko::Executor> exec,
+               gko::dim<2> size = gko::dim<2>{})
+        : EnableLinOp<DummyLinOp>(exec, size)
+    {}
+
+    void access() const { last_access = this->get_executor(); }
+
+    mutable std::shared_ptr<const gko::Executor> last_access;
+    mutable std::shared_ptr<const gko::Executor> last_b_access;
+    mutable std::shared_ptr<const gko::Executor> last_x_access;
+    mutable std::shared_ptr<const gko::Executor> last_alpha_access;
+    mutable std::shared_ptr<const gko::Executor> last_beta_access;
+
+protected:
+    void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
+    {
+        this->access();
+        static_cast<const DummyLinOp *>(b)->access();
+        static_cast<const DummyLinOp *>(x)->access();
+        last_b_access = b->get_executor();
+        last_x_access = x->get_executor();
+    }
+
+    void apply_impl(const gko::LinOp *alpha, const gko::LinOp *b,
+                    const gko::LinOp *beta, gko::LinOp *x) const override
+    {
+        this->access();
+        static_cast<const DummyLinOp *>(alpha)->access();
+        static_cast<const DummyLinOp *>(b)->access();
+        static_cast<const DummyLinOp *>(beta)->access();
+        static_cast<const DummyLinOp *>(x)->access();
+        last_alpha_access = alpha->get_executor();
+        last_b_access = b->get_executor();
+        last_beta_access = beta->get_executor();
+        last_x_access = x->get_executor();
+    }
+};
+
+
+class EnableLinOp : public ::testing::Test {
+protected:
+    EnableLinOp()
+        : ref{gko::ReferenceExecutor::create()},
+          cuda{gko::CudaExecutor::create(0, ref)},
+          op{DummyLinOp::create(cuda, gko::dim<2>{3, 5})},
+          alpha{DummyLinOp::create(ref, gko::dim<2>{1})},
+          beta{DummyLinOp::create(ref, gko::dim<2>{1})},
+          b{DummyLinOp::create(ref, gko::dim<2>{5, 4})},
+          x{DummyLinOp::create(ref, gko::dim<2>{3, 4})}
+    {}
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::CudaExecutor> cuda;
+    std::unique_ptr<DummyLinOp> op;
+    std::unique_ptr<DummyLinOp> alpha;
+    std::unique_ptr<DummyLinOp> beta;
+    std::unique_ptr<DummyLinOp> b;
+    std::unique_ptr<DummyLinOp> x;
+};
+
+
+TEST_F(EnableLinOp, ApplyCopiesDataToCorrectExecutor)
+{
+    op->apply(gko::lend(b), gko::lend(x));
+
+    ASSERT_EQ(op->last_b_access, cuda);
+    ASSERT_EQ(op->last_x_access, cuda);
+}
+
+
+TEST_F(EnableLinOp, ApplyCopiesBackOnlyX)
+{
+    op->apply(gko::lend(b), gko::lend(x));
+
+    ASSERT_EQ(b->last_access, nullptr);
+    ASSERT_EQ(x->last_access, cuda);
+}
+
+
+TEST_F(EnableLinOp, ExtendedApplyCopiesDataToCorrectExecutor)
+{
+    op->apply(gko::lend(alpha), gko::lend(b), gko::lend(beta), gko::lend(x));
+
+    ASSERT_EQ(op->last_alpha_access, cuda);
+    ASSERT_EQ(op->last_b_access, cuda);
+    ASSERT_EQ(op->last_beta_access, cuda);
+    ASSERT_EQ(op->last_x_access, cuda);
+}
+
+
+TEST_F(EnableLinOp, ExtendedApplyCopiesBackOnlyX)
+{
+    op->apply(gko::lend(alpha), gko::lend(b), gko::lend(beta), gko::lend(x));
+
+    ASSERT_EQ(alpha->last_access, nullptr);
+    ASSERT_EQ(b->last_access, nullptr);
+    ASSERT_EQ(beta->last_access, nullptr);
+    ASSERT_EQ(x->last_access, cuda);
+}
+
+
 class FactoryParameter : public ::testing::Test {
 protected:
     FactoryParameter() {}
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 7e311d489c7..1381f079945 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -57,6 +57,7 @@ const std::vector<sycl::device> get_devices(std::string device_type)
         {"accelerator", sycl::info::device_type::accelerator},
         {"all", sycl::info::device_type::all},
         {"cpu", sycl::info::device_type::cpu},
+        {"host", sycl::info::device_type::host},
         {"gpu", sycl::info::device_type::gpu}};
     std::for_each(device_type.begin(), device_type.end(),
                   [](char &c) { c = std::tolower(c); });
@@ -76,6 +77,14 @@ void OmpExecutor::raw_copy_to(const DpcppExecutor *dest, size_type num_bytes,
 }
 
 
+bool OmpExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
+{
+    auto device = detail::get_devices(
+        dest_exec->getdevice_type())[dest_exec->get_device_id()];
+    return device.is_host() || device.is_cpu();
+}
+
+
 std::shared_ptr<DpcppExecutor> DpcppExecutor::create(
     int device_id, std::shared_ptr<Executor> master, std::string device_type)
 {
@@ -150,6 +159,30 @@ int DpcppExecutor::get_num_devices(std::string device_type)
 }
 
 
+bool DpcppExecutor::verify_memory_to(const OmpExecutor *dest_exec) const
+{
+    auto device = detail::get_devices(device_type_)[device_id_];
+
+    return device.is_host() || device.is_cpu();
+}
+
+bool DpcppExecutor::verify_memory_to(const ReferenceExecutor *dest_exec) const
+{
+    auto device = detail::get_devices(device_type_)[device_id_];
+    return device.is_host() || device.is_cpu();
+}
+
+bool DpcppExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
+{
+    auto device = detail::get_devices(device_type_)[device_id_];
+    auto other_device = detail::get_devices(
+        dest_exec->get_device_type())[dest_exec->get_device_id()];
+    return device.get_info<cl::sycl::info::device::device_type>() ==
+               other_device.get_info<cl::sycl::info::device::device_type>() &&
+           device.get() == other_device.get();
+}
+
+
 namespace detail {
 
 
@@ -181,8 +214,10 @@ void DpcppExecutor::set_device_property()
     for (std::size_t i = 0; i < 3; i++) {
         max_workitem_sizes_.push_back(max_workitem_sizes[i]);
     }
-    max_workgroup_size_ =
-        device.get_info<sycl::info::device::max_work_group_size>();
+    if (!device.is_host()) {
+        max_workgroup_size_ =
+            device.get_info<sycl::info::device::max_work_group_size>();
+    }
     // Here we declare the queue with the property `in_order` which ensures the
     // kernels are executed in the submission order. Otherwise, calls to
     // `wait()` would be needed after every call to a DPC++ function or kernel.
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 3e400986e2b..8f8bbbf7d9d 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -162,6 +162,16 @@ void HipExecutor::raw_copy_to(const HipExecutor *dest, size_type num_bytes,
 }
 
 
+bool HipExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
+{
+#if GINKGO_HIP_PLATFORM_NVCC
+    return device_id_ == dest_exec->get_device_id();
+#else
+    return false;
+#endif
+}
+
+
 void HipExecutor::synchronize() const
 {
     hip::device_guard g(this->get_device_id());
diff --git a/include/ginkgo/core/base/dim.hpp b/include/ginkgo/core/base/dim.hpp
index 3030f0d1442..052c50e13b8 100644
--- a/include/ginkgo/core/base/dim.hpp
+++ b/include/ginkgo/core/base/dim.hpp
@@ -35,7 +35,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/base/utils.hpp>
 
 
 namespace gko {
diff --git a/include/ginkgo/core/base/exception_helpers.hpp b/include/ginkgo/core/base/exception_helpers.hpp
index e14bb1c0c61..af3e1efb598 100644
--- a/include/ginkgo/core/base/exception_helpers.hpp
+++ b/include/ginkgo/core/base/exception_helpers.hpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/dim.hpp>
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/name_demangling.hpp>
+#include <ginkgo/core/base/utils_helper.hpp>
 
 
 namespace gko {
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 2cc18bf3502..dd0293d30ee 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -633,6 +633,17 @@ class Executor : public log::EnableLogging<Executor> {
      */
     virtual void synchronize() const = 0;
 
+    /**
+     * Overload the equal-to operator which verifies whether the executors share
+     * the same memory.
+     *
+     * @param other  the other Executor to compare against
+     */
+    bool operator==(const Executor &other) const
+    {
+        return this->verify_memory_from(other);
+    }
+
 protected:
     /**
      * Allocates raw memory in this Executor.
@@ -684,6 +695,33 @@ class Executor : public log::EnableLogging<Executor> {
 
 #undef GKO_ENABLE_RAW_COPY_TO
 
+    /**
+     * Verify the memory from another Executor.
+     *
+     * @param src_exec  Executor from which to verify the memory.
+     *
+     * @return whether this executor and src_exec share the same memory.
+     */
+    virtual bool verify_memory_from(const Executor &src_exec) const = 0;
+
+/**
+ * @internal
+ * Declares a verify_memory_to() overload for a specified Executor subclass.
+ *
+ * This is the second stage of the double dispatch emulation required to
+ * implement verify_memory_from().
+ *
+ * @param _exec_type  the Executor subclass
+ */
+#define GKO_ENABLE_VERIFY_MEMORY_TO(_exec_type, ...) \
+    virtual bool verify_memory_to(const _exec_type *dest_exec) const = 0
+
+    GKO_ENABLE_FOR_ALL_EXECUTORS(GKO_ENABLE_VERIFY_MEMORY_TO);
+
+    GKO_ENABLE_VERIFY_MEMORY_TO(ReferenceExecutor);
+
+#undef GKO_ENABLE_VERIFY_MEMORY_TO
+
 private:
     /**
      * The LambdaOperation class wraps three functor objects into an
@@ -832,6 +870,11 @@ class ExecutorBase : public Executor {
         src_exec->raw_copy_to(self(), n_bytes, src_ptr, dest_ptr);
     }
 
+    bool verify_memory_from(const Executor &src_exec) const override
+    {
+        return src_exec.verify_memory_to(self());
+    }
+
 private:
     ConcreteExecutor *self() noexcept
     {
@@ -891,6 +934,13 @@ class EnableDeviceReset {
                      const void *src_ptr, void *dest_ptr) const override
 
 
+#define GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(dest_, bool_)                       \
+    bool verify_memory_to(const dest_ *other) const override { return bool_; } \
+    static_assert(true,                                                        \
+                  "This assert is used to counter the false positive extra "   \
+                  "semi-colon warnings")
+
+
 /**
  * This is the Executor subclass which represents the OpenMP device
  * (typically CPU).
@@ -925,6 +975,16 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
     void raw_free(void *ptr) const noexcept override;
 
     GKO_ENABLE_FOR_ALL_EXECUTORS(GKO_OVERRIDE_RAW_COPY_TO);
+
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(OmpExecutor, true);
+
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(ReferenceExecutor, true);
+
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(HipExecutor, false);
+
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(CudaExecutor, false);
+
+    bool verify_memory_to(const DpcppExecutor *dest_exec) const override;
 };
 
 
@@ -1090,6 +1150,19 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
     GKO_ENABLE_FOR_ALL_EXECUTORS(GKO_OVERRIDE_RAW_COPY_TO);
 
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(OmpExecutor, false);
+
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(ReferenceExecutor, false);
+
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(DpcppExecutor, false);
+
+    bool verify_memory_to(const HipExecutor *dest_exec) const override;
+
+    bool verify_memory_to(const CudaExecutor *dest_exec) const override
+    {
+        return device_id_ == dest_exec->get_device_id();
+    }
+
     static void increase_num_execs(unsigned device_id)
     {
         std::lock_guard<std::mutex> guard(mutex[device_id]);
@@ -1256,6 +1329,19 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
     GKO_ENABLE_FOR_ALL_EXECUTORS(GKO_OVERRIDE_RAW_COPY_TO);
 
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(OmpExecutor, false);
+
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(ReferenceExecutor, false);
+
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(DpcppExecutor, false);
+
+    bool verify_memory_to(const CudaExecutor *dest_exec) const override;
+
+    bool verify_memory_to(const HipExecutor *dest_exec) const override
+    {
+        return device_id_ == dest_exec->get_device_id();
+    }
+
     static void increase_num_execs(int device_id)
     {
         std::lock_guard<std::mutex> guard(mutex[device_id]);
@@ -1335,6 +1421,8 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
 
     /**
      * Get the DPCPP device id of the device associated to this executor.
+     *
+     * @return the DPCPP device id of the device associated to this executor
      */
     int get_device_id() const noexcept { return device_id_; }
 
@@ -1342,11 +1430,17 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
 
     /**
      * Get the number of devices present on the system.
+     *
+     * @param device_type  a string representing the device type
+     *
+     * @return the number of devices present on the system
      */
     static int get_num_devices(std::string device_type);
 
     /**
      * Get the available subgroup sizes for this device.
+     *
+     * @return the available subgroup sizes for this device
      */
     const std::vector<size_type> &get_subgroup_sizes() const noexcept
     {
@@ -1355,6 +1449,8 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
 
     /**
      * Get the number of Computing Units of this executor.
+     *
+     * @return the number of Computing Units of this executor
      */
     size_type get_num_computing_units() const noexcept
     {
@@ -1363,6 +1459,8 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
 
     /**
      * Get the maximum work item sizes.
+     *
+     * @return the maximum work item sizes
      */
     const std::vector<size_type> &get_max_workitem_sizes() const noexcept
     {
@@ -1371,12 +1469,21 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
 
     /**
      * Get the maximum workgroup size.
+     *
+     * @return the maximum workgroup size
      */
     size_type get_max_workgroup_size() const noexcept
     {
         return max_workgroup_size_;
     }
 
+    /**
+     * Get a string representing the device type.
+     *
+     * @return a string representing the device type
+     */
+    std::string get_device_type() const noexcept { return device_type_; }
+
 protected:
     void set_device_property();
 
@@ -1384,6 +1491,8 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
                   std::string device_type = "all")
         : device_id_(device_id), master_(master), device_type_(device_type)
     {
+        std::for_each(device_type_.begin(), device_type_.end(),
+                      [](char &c) { c = std::tolower(c); });
         this->set_device_property();
     }
 
@@ -1393,6 +1502,16 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
 
     GKO_ENABLE_FOR_ALL_EXECUTORS(GKO_OVERRIDE_RAW_COPY_TO);
 
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(CudaExecutor, false);
+
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(HipExecutor, false);
+
+    bool verify_memory_to(const OmpExecutor *dest_exec) const override;
+
+    bool verify_memory_to(const ReferenceExecutor *dest_exec) const override;
+
+    bool verify_memory_to(const DpcppExecutor *dest_exec) const override;
+
 private:
     int device_id_;
     std::shared_ptr<Executor> master_;
diff --git a/include/ginkgo/core/base/temporary_clone.hpp b/include/ginkgo/core/base/temporary_clone.hpp
new file mode 100644
index 00000000000..a0f59f73919
--- /dev/null
+++ b/include/ginkgo/core/base/temporary_clone.hpp
@@ -0,0 +1,202 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_BASE_TEMPORARY_CLONE_HPP_
+#define GKO_CORE_BASE_TEMPORARY_CLONE_HPP_
+
+
+#include <functional>
+#include <memory>
+#include <type_traits>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/utils_helper.hpp>
+
+
+namespace gko {
+namespace detail {
+
+
+/**
+ * A copy_back_deleter is a type of deleter that copies the data to an
+ * internally referenced object before performing the deletion.
+ *
+ * The deleter will use the `copy_from` method to perform the copy, and then
+ * delete the passed object using the `delete` keyword. This kind of deleter is
+ * useful when temporarily copying an object with the intent of copying it back
+ * once it goes out of scope.
+ *
+ * There is also a specialization for constant objects that does not perform the
+ * copy, since a constant object couldn't have been changed.
+ *
+ * @tparam T  the type of object being deleted
+ */
+template <typename T>
+class copy_back_deleter {
+public:
+    using pointer = T *;
+
+    /**
+     * Creates a new deleter object.
+     *
+     * @param original  the origin object where the data will be copied before
+     *                  deletion
+     */
+    copy_back_deleter(pointer original) : original_{original} {}
+
+    /**
+     * Deletes the object.
+     *
+     * @param ptr  pointer to the object being deleted
+     */
+    void operator()(pointer ptr) const
+    {
+        original_->copy_from(ptr);
+        delete ptr;
+    }
+
+private:
+    pointer original_;
+};
+
+// specialization for constant objects, no need to copy back something that
+// cannot change
+template <typename T>
+class copy_back_deleter<const T> {
+public:
+    using pointer = const T *;
+    copy_back_deleter(pointer original) : original_{original} {}
+
+    void operator()(pointer ptr) const { delete ptr; }
+
+private:
+    pointer original_;
+};
+
+
+template <typename T>
+struct temporary_clone_helper {
+    static std::unique_ptr<T> create(std::shared_ptr<const Executor> exec,
+                                     T *ptr)
+    {
+        return gko::clone(std::move(exec), ptr);
+    }
+};
+
+
+/**
+ * A temporary_clone is a special smart pointer-like object that is designed to
+ * hold an object temporarily copied to another executor.
+ *
+ * After the temporary_clone goes out of scope, the stored object will be copied
+ * back to its original location. This class is optimized to avoid copies if the
+ * object is already on the correct executor, in which case it will just hold a
+ * reference to that object, without performing the copy.
+ *
+ * @tparam T  the type of object held in the temporary_clone
+ */
+template <typename T>
+class temporary_clone {
+public:
+    using value_type = T;
+    using pointer = T *;
+
+    /**
+     * Creates a temporary_clone.
+     *
+     * @param exec  the executor where the clone will be created
+     * @param ptr  a pointer to the object of which the clone will be created
+     */
+    explicit temporary_clone(std::shared_ptr<const Executor> exec, pointer ptr)
+    {
+        if (*ptr->get_executor() == *exec) {
+            // just use the object we already have
+            handle_ = handle_type(ptr, null_deleter<T>());
+        } else {
+            // clone the object to the new executor and make sure it's copied
+            // back before we delete it
+            handle_ = handle_type(
+                temporary_clone_helper<T>::create(std::move(exec), ptr)
+                    .release(),
+                copy_back_deleter<T>(ptr));
+        }
+    }
+
+    /**
+     * Returns the object held by temporary_clone.
+     *
+     * @return the object held by temporary_clone
+     */
+    T *get() const { return handle_.get(); }
+
+    /**
+     * Calls a method on the underlying object.
+     *
+     * @return the underlying object
+     */
+    T *operator->() const { return handle_.get(); }
+
+private:
+    // std::function deleter allows to decide the (type of) deleter at runtime
+    using handle_type = std::unique_ptr<T, std::function<void(T *)>>;
+
+    handle_type handle_;
+};
+
+
+}  // namespace detail
+
+
+/**
+ * Creates a temporary_clone.
+ *
+ * This is a helper function which avoids the need to explicitly specify the
+ * type of the object, as would be the case if using the constructor of
+ * temporary_clone.
+ *
+ * @param exec  the executor where the clone will be created
+ * @param ptr  a pointer to the object of which the clone will be created
+ */
+template <typename T>
+detail::temporary_clone<T> make_temporary_clone(
+    std::shared_ptr<const Executor> exec, T *ptr)
+{
+    return detail::temporary_clone<T>(std::move(exec), ptr);
+}
+
+
+}  // namespace gko
+
+
+#endif  // GKO_CORE_BASE_TEMPORARY_CLONE_HPP_
diff --git a/include/ginkgo/core/base/utils.hpp b/include/ginkgo/core/base/utils.hpp
index 5270d619445..509213e8476 100644
--- a/include/ginkgo/core/base/utils.hpp
+++ b/include/ginkgo/core/base/utils.hpp
@@ -34,557 +34,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_PUBLIC_CORE_BASE_UTILS_HPP_
 
 
-#include <functional>
-#include <memory>
-#include <type_traits>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/name_demangling.hpp>
-#include <ginkgo/core/base/std_extensions.hpp>
-#include <ginkgo/core/base/types.hpp>
-
-
-#ifndef NDEBUG
-#include <cstdio>
-#endif  // NDEBUG
-
-
-namespace gko {
-
-
-class Executor;
-
-
-namespace detail {
-
-
-template <typename T>
-struct pointee_impl {};
-
-template <typename T>
-struct pointee_impl<T *> {
-    using type = T;
-};
-
-template <typename T, typename Deleter>
-struct pointee_impl<std::unique_ptr<T, Deleter>> {
-    using type = T;
-};
-
-template <typename T>
-struct pointee_impl<std::shared_ptr<T>> {
-    using type = T;
-};
-
-template <typename T>
-using pointee = typename pointee_impl<typename std::decay<T>::type>::type;
-
-
-template <typename T, typename = void>
-struct is_clonable_impl : std::false_type {};
-
-template <typename T>
-struct is_clonable_impl<T, xstd::void_t<decltype(std::declval<T>().clone())>>
-    : std::true_type {};
-
-template <typename T>
-constexpr bool is_clonable()
-{
-    return is_clonable_impl<typename std::decay<T>::type>::value;
-}
-
-
-template <typename T, typename = void>
-struct is_clonable_to_impl : std::false_type {};
-
-template <typename T>
-struct is_clonable_to_impl<
-    T, xstd::void_t<decltype(std::declval<T>().clone(
-           std::declval<std::shared_ptr<const Executor>>()))>>
-    : std::true_type {};
-
-template <typename T>
-constexpr bool is_clonable_to()
-{
-    return is_clonable_to_impl<typename std::decay<T>::type>::value;
-}
-
-
-template <typename T>
-struct have_ownership_impl : std::false_type {};
-
-template <typename T, typename Deleter>
-struct have_ownership_impl<std::unique_ptr<T, Deleter>> : std::true_type {};
-
-template <typename T>
-struct have_ownership_impl<std::shared_ptr<T>> : std::true_type {};
-
-template <typename T>
-using have_ownership_s = have_ownership_impl<typename std::decay<T>::type>;
-
-template <typename T>
-constexpr bool have_ownership()
-{
-    return have_ownership_s<T>::value;
-}
-
-
-template <typename Pointer>
-using cloned_type =
-    std::unique_ptr<typename std::remove_cv<pointee<Pointer>>::type>;
-
-
-template <typename Pointer>
-using shared_type = std::shared_ptr<pointee<Pointer>>;
-
-
-}  // namespace detail
-
-
-/**
- * Creates a unique clone of the object pointed to by `p`.
- *
- * The pointee (i.e. `*p`) needs to have a clone method that returns a
- * std::unique_ptr in order for this method to work.
- *
- * @tparam Pointer  type of pointer to the object (plain or smart pointer)
- *
- * @param p  a pointer to the object
- *
- * @note The difference between this function and directly calling
- *       LinOp::clone() is that this one preserves the static type of the
- *       object.
- */
-template <typename Pointer>
-inline detail::cloned_type<Pointer> clone(const Pointer &p)
-{
-    static_assert(detail::is_clonable<detail::pointee<Pointer>>(),
-                  "Object is not clonable");
-    return detail::cloned_type<Pointer>(
-        static_cast<typename std::remove_cv<detail::pointee<Pointer>>::type *>(
-            p->clone().release()));
-}
-
-
-/**
- * Creates a unique clone of the object pointed to by `p` on Executor `exec`.
- *
- * The pointee (i.e. `*p`) needs to have a clone method that takes an
- * executor and returns a std::unique_ptr in order for this method to work.
- *
- * @tparam Pointer  type of pointer to the object (plain or smart pointer)
- *
- * @param exec  the executor where the cloned object should be stored
- * @param p  a pointer to the object
- *
- * @note The difference between this function and directly calling
- *       LinOp::clone() is that this one preserves the static type of the
- *       object.
- */
-template <typename Pointer>
-inline detail::cloned_type<Pointer> clone(std::shared_ptr<const Executor> exec,
-                                          const Pointer &p)
-{
-    static_assert(detail::is_clonable_to<detail::pointee<Pointer>>(),
-                  "Object is not clonable");
-    return detail::cloned_type<Pointer>(
-        static_cast<typename std::remove_cv<detail::pointee<Pointer>>::type *>(
-            p->clone(std::move(exec)).release()));
-}
-
-
-/**
- * Marks the object pointed to by `p` as shared.
- *
- * Effectively converts a pointer with ownership to std::shared_ptr.
- *
- * @tparam OwningPointer  type of pointer with ownership to the object
- *                        (has to be a smart pointer)
- *
- * @param p  a pointer to the object
- *
- * @note The original pointer `p` becomes invalid after this call.
- */
-template <typename OwningPointer>
-inline detail::shared_type<OwningPointer> share(OwningPointer &&p)
-{
-    static_assert(detail::have_ownership<OwningPointer>(),
-                  "OwningPointer does not have ownership of the object");
-    return detail::shared_type<OwningPointer>(std::move(p));
-}
-
-
-/**
- * Marks that the object pointed to by `p` can be given to the callee.
- *
- * Effectively calls `std::move(p)`.
- *
- * @tparam OwningPointer  type of pointer with ownership to the object
- *                        (has to be a smart pointer)
- *
- * @param p  a pointer to the object
- *
- * @note The original pointer `p` becomes invalid after this call.
- */
-template <typename OwningPointer>
-inline typename std::remove_reference<OwningPointer>::type &&give(
-    OwningPointer &&p)
-{
-    static_assert(detail::have_ownership<OwningPointer>(),
-                  "OwningPointer does not have ownership of the object");
-    return std::move(p);
-}
-
-
-/**
- * Returns a non-owning (plain) pointer to the object pointed to by `p`.
- *
- * @tparam Pointer  type of pointer to the object (plain or smart pointer)
- *
- * @param p  a pointer to the object
- *
- * @note This is the overload for owning (smart) pointers, that behaves the
- *       same as calling .get() on the smart pointer.
- */
-template <typename Pointer>
-inline typename std::enable_if<detail::have_ownership_s<Pointer>::value,
-                               detail::pointee<Pointer> *>::type
-lend(const Pointer &p)
-{
-    return p.get();
-}
-
-/**
- * Returns a non-owning (plain) pointer to the object pointed to by `p`.
- *
- * @tparam Pointer  type of pointer to the object (plain or smart pointer)
- *
- * @param p  a pointer to the object
- *
- * @note This is the overload for non-owning (plain) pointers, that just
- *       returns `p`.
- */
-template <typename Pointer>
-inline typename std::enable_if<!detail::have_ownership_s<Pointer>::value,
-                               detail::pointee<Pointer> *>::type
-lend(const Pointer &p)
-{
-    return p;
-}
-
-
-/**
- * Performs polymorphic type conversion.
- *
- * @tparam T  requested result type
- * @tparam U  static type of the passed object
- *
- * @param obj  the object which should be converted
- *
- * @return If successful, returns a pointer to the subtype, otherwise throws
- *         NotSupported.
- */
-template <typename T, typename U>
-inline typename std::decay<T>::type *as(U *obj)
-{
-    if (auto p = dynamic_cast<typename std::decay<T>::type *>(obj)) {
-        return p;
-    } else {
-        throw NotSupported(__FILE__, __LINE__,
-                           std::string{"gko::as<"} +
-                               name_demangling::get_type_name(typeid(T)) + ">",
-                           name_demangling::get_type_name(typeid(*obj)));
-    }
-}
-
-/**
- * Performs polymorphic type conversion.
- *
- * This is the constant version of the function.
- *
- * @tparam T  requested result type
- * @tparam U  static type of the passed object
- *
- * @param obj  the object which should be converted
- *
- * @return If successful, returns a pointer to the subtype, otherwise throws
- *         NotSupported.
- */
-template <typename T, typename U>
-inline const typename std::decay<T>::type *as(const U *obj)
-{
-    if (auto p = dynamic_cast<const typename std::decay<T>::type *>(obj)) {
-        return p;
-    } else {
-        throw NotSupported(__FILE__, __LINE__,
-                           std::string{"gko::as<"} +
-                               name_demangling::get_type_name(typeid(T)) + ">",
-                           name_demangling::get_type_name(typeid(*obj)));
-    }
-}
-
-
-/**
- * Performs polymorphic type conversion of a unique_ptr.
- *
- * @tparam T  requested result type
- * @tparam U  static type of the passed object
- *
- * @param obj  the unique_ptr to the object which should be converted.
- *             If successful, it will be reset to a nullptr.
- *
- * @return If successful, returns a unique_ptr to the subtype, otherwise throws
- *         NotSupported.
- */
-template <typename T, typename U>
-inline std::unique_ptr<typename std::decay<T>::type> as(
-    std::unique_ptr<U> &&obj)
-{
-    if (auto p = dynamic_cast<typename std::decay<T>::type *>(obj.get())) {
-        obj.release();
-        return std::unique_ptr<typename std::decay<T>::type>{p};
-    } else {
-        throw NotSupported(__FILE__, __LINE__, __func__,
-                           name_demangling::get_type_name(typeid(*obj)));
-    }
-}
-
-
-/**
- * Performs polymorphic type conversion of a shared_ptr.
- *
- * @tparam T  requested result type
- * @tparam U  static type of the passed object
- *
- * @param obj  the shared_ptr to the object which should be converted.
- *
- * @return If successful, returns a shared_ptr to the subtype, otherwise throws
- *         NotSupported. This pointer shares ownership with the input pointer.
- */
-template <typename T, typename U>
-inline std::shared_ptr<typename std::decay<T>::type> as(std::shared_ptr<U> obj)
-{
-    auto ptr = std::dynamic_pointer_cast<typename std::decay<T>::type>(obj);
-    if (ptr) {
-        return ptr;
-    } else {
-        throw NotSupported(__FILE__, __LINE__, __func__,
-                           name_demangling::get_type_name(typeid(*obj)));
-    }
-}
-
-
-/**
- * Performs polymorphic type conversion of a shared_ptr.
- *
- * This is the constant version of the function.
- *
- * @tparam T  requested result type
- * @tparam U  static type of the passed object
- *
- * @param obj  the shared_ptr to the object which should be converted.
- *
- * @return If successful, returns a shared_ptr to the subtype, otherwise throws
- *         NotSupported. This pointer shares ownership with the input pointer.
- */
-template <typename T, typename U>
-inline std::shared_ptr<const typename std::decay<T>::type> as(
-    std::shared_ptr<const U> obj)
-{
-    auto ptr =
-        std::dynamic_pointer_cast<const typename std::decay<T>::type>(obj);
-    if (ptr) {
-        return ptr;
-    } else {
-        throw NotSupported(__FILE__, __LINE__, __func__,
-                           name_demangling::get_type_name(typeid(*obj)));
-    }
-}
-
-
-/**
- * This is a deleter that does not delete the object.
- *
- * It is useful where the object has been allocated elsewhere and will be
- * deleted manually.
- */
-template <typename T>
-class null_deleter {
-public:
-    using pointer = T *;
-
-    /**
-     * Deletes the object.
-     *
-     * @param ptr  pointer to the object being deleted
-     */
-    void operator()(pointer) const noexcept {}
-};
-
-// a specialization for arrays
-template <typename T>
-class null_deleter<T[]> {
-public:
-    using pointer = T[];
-
-    void operator()(pointer) const noexcept {}
-};
-
-
-namespace detail {
-
-
-/**
- * A copy_back_deleter is a type of deleter that copies the data to an
- * internally referenced object before performing the deletion.
- *
- * The deleter will use the `copy_from` method to perform the copy, and then
- * delete the passed object using the `delete` keyword. This kind of deleter is
- * useful when temporarily copying an object with the intent of copying it back
- * once it goes out of scope.
- *
- * There is also a specialization for constant objects that does not perform the
- * copy, since a constant object couldn't have been changed.
- *
- * @tparam T  the type of object being deleted
- */
-template <typename T>
-class copy_back_deleter {
-public:
-    using pointer = T *;
-
-    /**
-     * Creates a new deleter object.
-     *
-     * @param original  the origin object where the data will be copied before
-     *                  deletion
-     */
-    copy_back_deleter(pointer original) : original_{original} {}
-
-    /**
-     * Deletes the object.
-     *
-     * @param ptr  pointer to the object being deleted
-     */
-    void operator()(pointer ptr) const
-    {
-        original_->copy_from(ptr);
-        delete ptr;
-    }
-
-private:
-    pointer original_;
-};
-
-
-// specialization for constant objects, no need to copy back something that
-// cannot change
-template <typename T>
-class copy_back_deleter<const T> {
-public:
-    using pointer = const T *;
-    copy_back_deleter(pointer original) : original_{original} {}
-
-    void operator()(pointer ptr) const { delete ptr; }
-
-private:
-    pointer original_;
-};
-
-
-template <typename T>
-struct temporary_clone_helper {
-    static std::unique_ptr<T> create(std::shared_ptr<const Executor> exec,
-                                     T *ptr)
-    {
-        return gko::clone(std::move(exec), ptr);
-    }
-};
-
-
-/**
- * A temporary_clone is a special smart pointer-like object that is designed to
- * hold an object temporarily copied to another executor.
- *
- * After the temporary_clone goes out of scope, the stored object will be copied
- * back to its original location. This class is optimized to avoid copies if the
- * object is already on the correct executor, in which case it will just hold a
- * reference to that object, without performing the copy.
- *
- * @tparam T  the type of object held in the temporary_clone
- */
-template <typename T>
-class temporary_clone {
-public:
-    using value_type = T;
-    using pointer = T *;
-
-    /**
-     * Creates a temporary_clone.
-     *
-     * @param exec  the executor where the clone will be created
-     * @param ptr  a pointer to the object of which the clone will be created
-     */
-    explicit temporary_clone(std::shared_ptr<const Executor> exec, pointer ptr)
-    {
-        if (ptr->get_executor() == exec) {
-            // just use the object we already have
-            handle_ = handle_type(ptr, null_deleter<T>());
-        } else {
-            // clone the object to the new executor and make sure it's copied
-            // back before we delete it
-            handle_ = handle_type(
-                temporary_clone_helper<T>::create(std::move(exec), ptr)
-                    .release(),
-                copy_back_deleter<T>(ptr));
-        }
-    }
-
-    /**
-     * Returns the object held by temporary_clone.
-     *
-     * @return the object held by temporary_clone
-     */
-    T *get() const { return handle_.get(); }
-
-    /**
-     * Calls a method on the underlying object.
-     *
-     * @return the underlying object
-     */
-    T *operator->() const { return handle_.get(); }
-
-private:
-    // std::function deleter allows to decide the (type of) deleter at runtime
-    using handle_type = std::unique_ptr<T, std::function<void(T *)>>;
-
-    handle_type handle_;
-};
-
-
-}  // namespace detail
-
-
-/**
- * Creates a temporary_clone.
- *
- * This is a helper function which avoids the need to explicitly specify the
- * type of the object, as would be the case if using the constructor of
- * temporary_clone.
- *
- * @param exec  the executor where the clone will be created
- * @param ptr  a pointer to the object of which the clone will be created
- */
-template <typename T>
-detail::temporary_clone<T> make_temporary_clone(
-    std::shared_ptr<const Executor> exec, T *ptr)
-{
-    return detail::temporary_clone<T>(std::move(exec), ptr);
-}
-
-
-}  // namespace gko
+#include <ginkgo/core/base/temporary_clone.hpp>
+#include <ginkgo/core/base/utils_helper.hpp>
 
 
 #endif  // GKO_PUBLIC_CORE_BASE_UTILS_HPP_
diff --git a/include/ginkgo/core/base/utils_helper.hpp b/include/ginkgo/core/base/utils_helper.hpp
new file mode 100644
index 00000000000..01c6b5cf5f5
--- /dev/null
+++ b/include/ginkgo/core/base/utils_helper.hpp
@@ -0,0 +1,438 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_BASE_UTILS_HELPER_HPP_
+#define GKO_CORE_BASE_UTILS_HELPER_HPP_
+
+
+#include <functional>
+#include <memory>
+#include <type_traits>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/name_demangling.hpp>
+#include <ginkgo/core/base/std_extensions.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+#ifndef NDEBUG
+#include <cstdio>
+#endif  // NDEBUG
+
+
+namespace gko {
+
+
+class Executor;
+
+
+namespace detail {
+
+
+template <typename T>
+struct pointee_impl {};
+
+template <typename T>
+struct pointee_impl<T *> {
+    using type = T;
+};
+
+template <typename T, typename Deleter>
+struct pointee_impl<std::unique_ptr<T, Deleter>> {
+    using type = T;
+};
+
+template <typename T>
+struct pointee_impl<std::shared_ptr<T>> {
+    using type = T;
+};
+
+template <typename T>
+using pointee = typename pointee_impl<typename std::decay<T>::type>::type;
+
+
+template <typename T, typename = void>
+struct is_clonable_impl : std::false_type {};
+
+template <typename T>
+struct is_clonable_impl<T, xstd::void_t<decltype(std::declval<T>().clone())>>
+    : std::true_type {};
+
+template <typename T>
+constexpr bool is_clonable()
+{
+    return is_clonable_impl<typename std::decay<T>::type>::value;
+}
+
+
+template <typename T, typename = void>
+struct is_clonable_to_impl : std::false_type {};
+
+template <typename T>
+struct is_clonable_to_impl<
+    T, xstd::void_t<decltype(std::declval<T>().clone(
+           std::declval<std::shared_ptr<const Executor>>()))>>
+    : std::true_type {};
+
+template <typename T>
+constexpr bool is_clonable_to()
+{
+    return is_clonable_to_impl<typename std::decay<T>::type>::value;
+}
+
+
+template <typename T>
+struct have_ownership_impl : std::false_type {};
+
+template <typename T, typename Deleter>
+struct have_ownership_impl<std::unique_ptr<T, Deleter>> : std::true_type {};
+
+template <typename T>
+struct have_ownership_impl<std::shared_ptr<T>> : std::true_type {};
+
+template <typename T>
+using have_ownership_s = have_ownership_impl<typename std::decay<T>::type>;
+
+template <typename T>
+constexpr bool have_ownership()
+{
+    return have_ownership_s<T>::value;
+}
+
+
+template <typename Pointer>
+using cloned_type =
+    std::unique_ptr<typename std::remove_cv<pointee<Pointer>>::type>;
+
+
+template <typename Pointer>
+using shared_type = std::shared_ptr<pointee<Pointer>>;
+
+
+}  // namespace detail
+
+
+/**
+ * Creates a unique clone of the object pointed to by `p`.
+ *
+ * The pointee (i.e. `*p`) needs to have a clone method that returns a
+ * std::unique_ptr in order for this method to work.
+ *
+ * @tparam Pointer  type of pointer to the object (plain or smart pointer)
+ *
+ * @param p  a pointer to the object
+ *
+ * @note The difference between this function and directly calling
+ *       LinOp::clone() is that this one preserves the static type of the
+ *       object.
+ */
+template <typename Pointer>
+inline detail::cloned_type<Pointer> clone(const Pointer &p)
+{
+    static_assert(detail::is_clonable<detail::pointee<Pointer>>(),
+                  "Object is not clonable");
+    return detail::cloned_type<Pointer>(
+        static_cast<typename std::remove_cv<detail::pointee<Pointer>>::type *>(
+            p->clone().release()));
+}
+
+
+/**
+ * Creates a unique clone of the object pointed to by `p` on Executor `exec`.
+ *
+ * The pointee (i.e. `*p`) needs to have a clone method that takes an
+ * executor and returns a std::unique_ptr in order for this method to work.
+ *
+ * @tparam Pointer  type of pointer to the object (plain or smart pointer)
+ *
+ * @param exec  the executor where the cloned object should be stored
+ * @param p  a pointer to the object
+ *
+ * @note The difference between this function and directly calling
+ *       LinOp::clone() is that this one preserves the static type of the
+ *       object.
+ */
+template <typename Pointer>
+inline detail::cloned_type<Pointer> clone(std::shared_ptr<const Executor> exec,
+                                          const Pointer &p)
+{
+    static_assert(detail::is_clonable_to<detail::pointee<Pointer>>(),
+                  "Object is not clonable");
+    return detail::cloned_type<Pointer>(
+        static_cast<typename std::remove_cv<detail::pointee<Pointer>>::type *>(
+            p->clone(std::move(exec)).release()));
+}
+
+
+/**
+ * Marks the object pointed to by `p` as shared.
+ *
+ * Effectively converts a pointer with ownership to std::shared_ptr.
+ *
+ * @tparam OwningPointer  type of pointer with ownership to the object
+ *                        (has to be a smart pointer)
+ *
+ * @param p  a pointer to the object
+ *
+ * @note The original pointer `p` becomes invalid after this call.
+ */
+template <typename OwningPointer>
+inline detail::shared_type<OwningPointer> share(OwningPointer &&p)
+{
+    static_assert(detail::have_ownership<OwningPointer>(),
+                  "OwningPointer does not have ownership of the object");
+    return detail::shared_type<OwningPointer>(std::move(p));
+}
+
+
+/**
+ * Marks that the object pointed to by `p` can be given to the callee.
+ *
+ * Effectively calls `std::move(p)`.
+ *
+ * @tparam OwningPointer  type of pointer with ownership to the object
+ *                        (has to be a smart pointer)
+ *
+ * @param p  a pointer to the object
+ *
+ * @note The original pointer `p` becomes invalid after this call.
+ */
+template <typename OwningPointer>
+inline typename std::remove_reference<OwningPointer>::type &&give(
+    OwningPointer &&p)
+{
+    static_assert(detail::have_ownership<OwningPointer>(),
+                  "OwningPointer does not have ownership of the object");
+    return std::move(p);
+}
+
+
+/**
+ * Returns a non-owning (plain) pointer to the object pointed to by `p`.
+ *
+ * @tparam Pointer  type of pointer to the object (plain or smart pointer)
+ *
+ * @param p  a pointer to the object
+ *
+ * @note This is the overload for owning (smart) pointers, that behaves the
+ *       same as calling .get() on the smart pointer.
+ */
+template <typename Pointer>
+inline typename std::enable_if<detail::have_ownership_s<Pointer>::value,
+                               detail::pointee<Pointer> *>::type
+lend(const Pointer &p)
+{
+    return p.get();
+}
+
+/**
+ * Returns a non-owning (plain) pointer to the object pointed to by `p`.
+ *
+ * @tparam Pointer  type of pointer to the object (plain or smart pointer)
+ *
+ * @param p  a pointer to the object
+ *
+ * @note This is the overload for non-owning (plain) pointers, that just
+ *       returns `p`.
+ */
+template <typename Pointer>
+inline typename std::enable_if<!detail::have_ownership_s<Pointer>::value,
+                               detail::pointee<Pointer> *>::type
+lend(const Pointer &p)
+{
+    return p;
+}
+
+
+/**
+ * Performs polymorphic type conversion.
+ *
+ * @tparam T  requested result type
+ * @tparam U  static type of the passed object
+ *
+ * @param obj  the object which should be converted
+ *
+ * @return If successful, returns a pointer to the subtype, otherwise throws
+ *         NotSupported.
+ */
+template <typename T, typename U>
+inline typename std::decay<T>::type *as(U *obj)
+{
+    if (auto p = dynamic_cast<typename std::decay<T>::type *>(obj)) {
+        return p;
+    } else {
+        throw NotSupported(__FILE__, __LINE__,
+                           std::string{"gko::as<"} +
+                               name_demangling::get_type_name(typeid(T)) + ">",
+                           name_demangling::get_type_name(typeid(*obj)));
+    }
+}
+
+/**
+ * Performs polymorphic type conversion.
+ *
+ * This is the constant version of the function.
+ *
+ * @tparam T  requested result type
+ * @tparam U  static type of the passed object
+ *
+ * @param obj  the object which should be converted
+ *
+ * @return If successful, returns a pointer to the subtype, otherwise throws
+ *         NotSupported.
+ */
+template <typename T, typename U>
+inline const typename std::decay<T>::type *as(const U *obj)
+{
+    if (auto p = dynamic_cast<const typename std::decay<T>::type *>(obj)) {
+        return p;
+    } else {
+        throw NotSupported(__FILE__, __LINE__,
+                           std::string{"gko::as<"} +
+                               name_demangling::get_type_name(typeid(T)) + ">",
+                           name_demangling::get_type_name(typeid(*obj)));
+    }
+}
+
+
+/**
+ * Performs polymorphic type conversion of a unique_ptr.
+ *
+ * @tparam T  requested result type
+ * @tparam U  static type of the passed object
+ *
+ * @param obj  the unique_ptr to the object which should be converted.
+ *             If successful, it will be reset to a nullptr.
+ *
+ * @return If successful, returns a unique_ptr to the subtype, otherwise throws
+ *         NotSupported.
+ */
+template <typename T, typename U>
+inline std::unique_ptr<typename std::decay<T>::type> as(
+    std::unique_ptr<U> &&obj)
+{
+    if (auto p = dynamic_cast<typename std::decay<T>::type *>(obj.get())) {
+        obj.release();
+        return std::unique_ptr<typename std::decay<T>::type>{p};
+    } else {
+        throw NotSupported(__FILE__, __LINE__, __func__,
+                           name_demangling::get_type_name(typeid(*obj)));
+    }
+}
+
+
+/**
+ * Performs polymorphic type conversion of a shared_ptr.
+ *
+ * @tparam T  requested result type
+ * @tparam U  static type of the passed object
+ *
+ * @param obj  the shared_ptr to the object which should be converted.
+ *
+ * @return If successful, returns a shared_ptr to the subtype, otherwise throws
+ *         NotSupported. This pointer shares ownership with the input pointer.
+ */
+template <typename T, typename U>
+inline std::shared_ptr<typename std::decay<T>::type> as(std::shared_ptr<U> obj)
+{
+    auto ptr = std::dynamic_pointer_cast<typename std::decay<T>::type>(obj);
+    if (ptr) {
+        return ptr;
+    } else {
+        throw NotSupported(__FILE__, __LINE__, __func__,
+                           name_demangling::get_type_name(typeid(*obj)));
+    }
+}
+
+
+/**
+ * Performs polymorphic type conversion of a shared_ptr.
+ *
+ * This is the constant version of the function.
+ *
+ * @tparam T  requested result type
+ * @tparam U  static type of the passed object
+ *
+ * @param obj  the shared_ptr to the object which should be converted.
+ *
+ * @return If successful, returns a shared_ptr to the subtype, otherwise throws
+ *         NotSupported. This pointer shares ownership with the input pointer.
+ */
+template <typename T, typename U>
+inline std::shared_ptr<const typename std::decay<T>::type> as(
+    std::shared_ptr<const U> obj)
+{
+    auto ptr =
+        std::dynamic_pointer_cast<const typename std::decay<T>::type>(obj);
+    if (ptr) {
+        return ptr;
+    } else {
+        throw NotSupported(__FILE__, __LINE__, __func__,
+                           name_demangling::get_type_name(typeid(*obj)));
+    }
+}
+
+
+/**
+ * This is a deleter that does not delete the object.
+ *
+ * It is useful where the object has been allocated elsewhere and will be
+ * deleted manually.
+ */
+template <typename T>
+class null_deleter {
+public:
+    using pointer = T *;
+
+    /**
+     * Deletes the object.
+     *
+     * @param ptr  pointer to the object being deleted
+     */
+    void operator()(pointer) const noexcept {}
+};
+
+// a specialization for arrays
+template <typename T>
+class null_deleter<T[]> {
+public:
+    using pointer = T[];
+
+    void operator()(pointer) const noexcept {}
+};
+
+
+}  // namespace gko
+
+
+#endif  // GKO_CORE_BASE_UTILS_HELPER_HPP_
diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp
index d84e51c8ed6..f9df5660fd2 100644
--- a/include/ginkgo/core/log/logger.hpp
+++ b/include/ginkgo/core/log/logger.hpp
@@ -42,7 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/base/utils.hpp>
+#include <ginkgo/core/base/utils_helper.hpp>
 
 
 namespace gko {

From 9bde43cfdd4ac4a622bee5bb8cd3b91efe387318 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 25 Nov 2020 18:39:00 +0100
Subject: [PATCH 0176/1008] Review updates.

+ Do not use `operator==`, but a funciton `memory_accessible` instead.
+ Make DPC++ host and CPU be memory compatible.
+ Use pointers for the interface instead of references.
+ Ensure DPC++ tests always work.
+ Fix some typos.

Co-authored-by: Yuhsiang M. Tsai <yhmtsai@gmail.com>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
---
 core/test/base/array.cpp                     |  2 +-
 core/test/base/executor.cpp                  | 99 +++++++++++---------
 dpcpp/base/executor.dp.cpp                   | 28 +++---
 include/ginkgo/core/base/executor.hpp        | 17 ++--
 include/ginkgo/core/base/temporary_clone.hpp |  2 +-
 5 files changed, 81 insertions(+), 67 deletions(-)

diff --git a/core/test/base/array.cpp b/core/test/base/array.cpp
index c20782651f7..54f7cfea053 100644
--- a/core/test/base/array.cpp
+++ b/core/test/base/array.cpp
@@ -284,7 +284,7 @@ TYPED_TEST(Array, DoesNotCreateATemporaryCloneBetweenSameMemory)
 }
 
 
-TYPED_TEST(Array, CanDoesNotCopyBackTemporaryCloneBetweenSameMemory)
+TYPED_TEST(Array, DoesNotCopyBackTemporaryCloneBetweenSameMemory)
 {
     auto omp = gko::OmpExecutor::create();
 
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 73c82a06fa8..783e20f1008 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -480,56 +480,67 @@ TEST(Executor, CanVerifyMemory)
     auto omp = gko::OmpExecutor::create();
     auto hip = gko::HipExecutor::create(0, omp);
     auto cuda = gko::CudaExecutor::create(0, omp);
-    auto cpu_dpcpp = gko::DpcppExecutor::create(0, omp, "cpu");
-    auto host_dpcpp = gko::DpcppExecutor::create(0, omp, "host");
-    auto gpu_dpcpp = gko::DpcppExecutor::create(0, omp, "gpu");
     auto omp2 = gko::OmpExecutor::create();
     auto hip2 = gko::HipExecutor::create(0, omp);
     auto cuda2 = gko::CudaExecutor::create(0, omp);
-    std::shared_ptr<gko::HipExecutor> hip_1 = gko::HipExecutor::create(1, omp);
-    std::shared_ptr<gko::CudaExecutor> cuda_1 =
-        gko::CudaExecutor::create(1, omp);
-
-    ASSERT_EQ(true, *ref == *omp);
-    ASSERT_EQ(true, *omp == *ref);
-    ASSERT_EQ(false, *ref == *hip);
-    ASSERT_EQ(false, *hip == *ref);
-    ASSERT_EQ(false, *omp == *hip);
-    ASSERT_EQ(false, *hip == *omp);
-    ASSERT_EQ(false, *ref == *cuda);
-    ASSERT_EQ(false, *cuda == *ref);
-    ASSERT_EQ(false, *omp == *cuda);
-    ASSERT_EQ(false, *cuda == *omp);
-    ASSERT_EQ(true, *cpu_dpcpp == *ref);
-    ASSERT_EQ(true, *host_dpcpp == *ref);
-    ASSERT_EQ(false, *gpu_dpcpp == *ref);
-    ASSERT_EQ(true, *ref == *cpu_dpcpp);
-    ASSERT_EQ(true, *ref == *host_dpcpp);
-    ASSERT_EQ(false, *ref == *gpu_dpcpp);
-    ASSERT_EQ(true, *cpu_dpcpp == *omp);
-    ASSERT_EQ(true, *host_dpcpp == *omp);
-    ASSERT_EQ(false, *gpu_dpcpp == *omp);
-    ASSERT_EQ(true, *omp == *cpu_dpcpp);
-    ASSERT_EQ(true, *omp == *host_dpcpp);
-    ASSERT_EQ(false, *omp == *gpu_dpcpp);
+    auto hip_1 = gko::HipExecutor::create(1, omp);
+    auto cuda_1 = gko::CudaExecutor::create(1, omp);
+    std::shared_ptr<gko::DpcppExecutor> host_dpcpp;
+    std::shared_ptr<gko::DpcppExecutor> cpu_dpcpp;
+    std::shared_ptr<gko::DpcppExecutor> gpu_dpcpp;
+    if (gko::DpcppExecutor::get_num_devices("host"))
+        host_dpcpp = gko::DpcppExecutor::create(0, omp, "host");
+    if (gko::DpcppExecutor::get_num_devices("cpu"))
+        cpu_dpcpp = gko::DpcppExecutor::create(0, omp, "cpu");
+    if (gko::DpcppExecutor::get_num_devices("gpu"))
+        gpu_dpcpp = gko::DpcppExecutor::create(0, omp, "gpu");
+
+    ASSERT_EQ(true, ref->memory_accessible(omp));
+    ASSERT_EQ(true, omp->memory_accessible(ref));
+    ASSERT_EQ(false, ref->memory_accessible(hip));
+    ASSERT_EQ(false, hip->memory_accessible(ref));
+    ASSERT_EQ(false, omp->memory_accessible(hip));
+    ASSERT_EQ(false, hip->memory_accessible(omp));
+    ASSERT_EQ(false, ref->memory_accessible(cuda));
+    ASSERT_EQ(false, cuda->memory_accessible(ref));
+    ASSERT_EQ(false, omp->memory_accessible(cuda));
+    ASSERT_EQ(false, cuda->memory_accessible(omp));
+    if (gko::DpcppExecutor::get_num_devices("host")) {
+        ASSERT_EQ(true, host_dpcpp->memory_accessible(ref));
+        ASSERT_EQ(true, ref->memory_accessible(host_dpcpp));
+        ASSERT_EQ(true, host_dpcpp->memory_accessible(omp));
+        ASSERT_EQ(true, omp->memory_accessible(host_dpcpp));
+    }
+    if (gko::DpcppExecutor::get_num_devices("cpu")) {
+        ASSERT_EQ(true, ref->memory_accessible(cpu_dpcpp));
+        ASSERT_EQ(true, cpu_dpcpp->memory_accessible(ref));
+        ASSERT_EQ(true, cpu_dpcpp->memory_accessible(omp));
+        ASSERT_EQ(true, omp->memory_accessible(cpu_dpcpp));
+    }
+    if (gko::DpcppExecutor::get_num_devices("gpu")) {
+        ASSERT_EQ(false, gpu_dpcpp->memory_accessible(ref));
+        ASSERT_EQ(false, ref->memory_accessible(gpu_dpcpp));
+        ASSERT_EQ(false, gpu_dpcpp->memory_accessible(omp));
+        ASSERT_EQ(false, omp->memory_accessible(gpu_dpcpp));
+    }
 #if GINKGO_HIP_PLATFORM_NVCC
-    ASSERT_EQ(true, *hip == *cuda);
-    ASSERT_EQ(true, *cuda == *hip);
-    ASSERT_EQ(true, *hip_1 == *cuda_1);
-    ASSERT_EQ(true, *cuda_1 == *hip_1);
+    ASSERT_EQ(true, hip->memory_accessible(cuda));
+    ASSERT_EQ(true, cuda->memory_accessible(hip));
+    ASSERT_EQ(true, hip_1->memory_accessible(cuda_1));
+    ASSERT_EQ(true, cuda_1->memory_accessible(hip_1));
 #else
-    ASSERT_EQ(false, *hip == *cuda);
-    ASSERT_EQ(false, *cuda == *hip);
-    ASSERT_EQ(false, *hip_1 == *cuda_1);
-    ASSERT_EQ(false, *cuda_1 == *hip_1);
+    ASSERT_EQ(false, hip->memory_accessible(cuda));
+    ASSERT_EQ(false, cuda->memory_accessible(hip));
+    ASSERT_EQ(false, hip_1->memory_accessible(cuda_1));
+    ASSERT_EQ(false, cuda_1->memory_accessible(hip_1));
 #endif
-    ASSERT_EQ(true, *omp == *omp2);
-    ASSERT_EQ(true, *hip == *hip2);
-    ASSERT_EQ(true, *cuda == *cuda2);
-    ASSERT_EQ(false, *hip == *hip_1);
-    ASSERT_EQ(false, *cuda == *hip_1);
-    ASSERT_EQ(false, *cuda == *cuda_1);
-    ASSERT_EQ(false, *hip == *cuda_1);
+    ASSERT_EQ(true, omp->memory_accessible(omp2));
+    ASSERT_EQ(true, hip->memory_accessible(hip2));
+    ASSERT_EQ(true, cuda->memory_accessible(cuda2));
+    ASSERT_EQ(false, hip->memory_accessible(hip_1));
+    ASSERT_EQ(false, cuda->memory_accessible(hip_1));
+    ASSERT_EQ(false, cuda->memory_accessible(cuda_1));
+    ASSERT_EQ(false, hip->memory_accessible(cuda_1));
 }
 
 
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 1381f079945..25e106a2546 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -80,7 +80,7 @@ void OmpExecutor::raw_copy_to(const DpcppExecutor *dest, size_type num_bytes,
 bool OmpExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
 {
     auto device = detail::get_devices(
-        dest_exec->getdevice_type())[dest_exec->get_device_id()];
+        dest_exec->get_device_type())[dest_exec->get_device_id()];
     return device.is_host() || device.is_cpu();
 }
 
@@ -177,9 +177,11 @@ bool DpcppExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
     auto device = detail::get_devices(device_type_)[device_id_];
     auto other_device = detail::get_devices(
         dest_exec->get_device_type())[dest_exec->get_device_id()];
-    return device.get_info<cl::sycl::info::device::device_type>() ==
-               other_device.get_info<cl::sycl::info::device::device_type>() &&
-           device.get() == other_device.get();
+    return ((device.is_host() || device.is_cpu()) &&
+            (other_device.is_host() || other_device.is_cpu())) ||
+           (device.get_info<cl::sycl::info::device::device_type>() ==
+                other_device.get_info<cl::sycl::info::device::device_type>() &&
+            device.get() == other_device.get());
 }
 
 
@@ -200,11 +202,13 @@ void DpcppExecutor::set_device_property()
 {
     assert(device_id_ < DpcppExecutor::get_num_devices(device_type_));
     auto device = detail::get_devices(device_type_)[device_id_];
-    try {
-        subgroup_sizes_ =
-            device.get_info<cl::sycl::info::device::sub_group_sizes>();
-    } catch (cl::sycl::runtime_error &err) {
-        GKO_NOT_SUPPORTED(device);
+    if (!device.is_host()) {
+        try {
+            subgroup_sizes_ =
+                device.get_info<cl::sycl::info::device::sub_group_sizes>();
+        } catch (cl::sycl::runtime_error &err) {
+            GKO_NOT_SUPPORTED(device);
+        }
     }
     num_computing_units_ =
         device.get_info<sycl::info::device::max_compute_units>();
@@ -214,10 +218,8 @@ void DpcppExecutor::set_device_property()
     for (std::size_t i = 0; i < 3; i++) {
         max_workitem_sizes_.push_back(max_workitem_sizes[i]);
     }
-    if (!device.is_host()) {
-        max_workgroup_size_ =
-            device.get_info<sycl::info::device::max_work_group_size>();
-    }
+    max_workgroup_size_ =
+        device.get_info<sycl::info::device::max_work_group_size>();
     // Here we declare the queue with the property `in_order` which ensures the
     // kernels are executed in the submission order. Otherwise, calls to
     // `wait()` would be needed after every call to a DPC++ function or kernel.
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index dd0293d30ee..2ff58f6fe82 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -634,14 +634,15 @@ class Executor : public log::EnableLogging<Executor> {
     virtual void synchronize() const = 0;
 
     /**
-     * Overload the equal-to operator which verifies whether the executors share
-     * the same memory.
+     * Verifies whether the executors share the same memory.
      *
      * @param other  the other Executor to compare against
+     *
+     * @return whether the executors this and other share the same memory.
      */
-    bool operator==(const Executor &other) const
+    bool memory_accessible(const std::shared_ptr<const Executor> &other) const
     {
-        return this->verify_memory_from(other);
+        return this->verify_memory_from(other.get());
     }
 
 protected:
@@ -702,7 +703,7 @@ class Executor : public log::EnableLogging<Executor> {
      *
      * @return whether this executor and src_exec share the same memory.
      */
-    virtual bool verify_memory_from(const Executor &src_exec) const = 0;
+    virtual bool verify_memory_from(const Executor *src_exec) const = 0;
 
 /**
  * @internal
@@ -718,7 +719,7 @@ class Executor : public log::EnableLogging<Executor> {
 
     GKO_ENABLE_FOR_ALL_EXECUTORS(GKO_ENABLE_VERIFY_MEMORY_TO);
 
-    GKO_ENABLE_VERIFY_MEMORY_TO(ReferenceExecutor);
+    GKO_ENABLE_VERIFY_MEMORY_TO(ReferenceExecutor, ref);
 
 #undef GKO_ENABLE_VERIFY_MEMORY_TO
 
@@ -870,9 +871,9 @@ class ExecutorBase : public Executor {
         src_exec->raw_copy_to(self(), n_bytes, src_ptr, dest_ptr);
     }
 
-    bool verify_memory_from(const Executor &src_exec) const override
+    bool verify_memory_from(const Executor *src_exec) const override
     {
-        return src_exec.verify_memory_to(self());
+        return src_exec->verify_memory_to(self());
     }
 
 private:
diff --git a/include/ginkgo/core/base/temporary_clone.hpp b/include/ginkgo/core/base/temporary_clone.hpp
index a0f59f73919..33f80848336 100644
--- a/include/ginkgo/core/base/temporary_clone.hpp
+++ b/include/ginkgo/core/base/temporary_clone.hpp
@@ -140,7 +140,7 @@ class temporary_clone {
      */
     explicit temporary_clone(std::shared_ptr<const Executor> exec, pointer ptr)
     {
-        if (*ptr->get_executor() == *exec) {
+        if (ptr->get_executor()->memory_accessible(exec)) {
             // just use the object we already have
             handle_ = handle_type(ptr, null_deleter<T>());
         } else {

From bbe5bf01cfc4ce30a1e754270b405b52667bd976 Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Wed, 2 Dec 2020 11:48:39 +0000
Subject: [PATCH 0177/1008] Format files

Co-authored-by: tcojean <tcojean@users.noreply.github.com>
---
 cuda/test/base/array.cu                      | 2 ++
 include/ginkgo/core/base/temporary_clone.hpp | 6 +++---
 include/ginkgo/core/base/utils_helper.hpp    | 6 +++---
 include/ginkgo/ginkgo.hpp                    | 2 ++
 4 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/cuda/test/base/array.cu b/cuda/test/base/array.cu
index c4757c42b44..73770b36bb5 100644
--- a/cuda/test/base/array.cu
+++ b/cuda/test/base/array.cu
@@ -34,6 +34,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <gtest/gtest.h>
+
+
 #include <ginkgo/core/base/executor.hpp>
 
 
diff --git a/include/ginkgo/core/base/temporary_clone.hpp b/include/ginkgo/core/base/temporary_clone.hpp
index 33f80848336..fa0c2e32b13 100644
--- a/include/ginkgo/core/base/temporary_clone.hpp
+++ b/include/ginkgo/core/base/temporary_clone.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_TEMPORARY_CLONE_HPP_
-#define GKO_CORE_BASE_TEMPORARY_CLONE_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_TEMPORARY_CLONE_HPP_
+#define GKO_PUBLIC_CORE_BASE_TEMPORARY_CLONE_HPP_
 
 
 #include <functional>
@@ -199,4 +199,4 @@ detail::temporary_clone<T> make_temporary_clone(
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_TEMPORARY_CLONE_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_TEMPORARY_CLONE_HPP_
diff --git a/include/ginkgo/core/base/utils_helper.hpp b/include/ginkgo/core/base/utils_helper.hpp
index 01c6b5cf5f5..d277d20cb4c 100644
--- a/include/ginkgo/core/base/utils_helper.hpp
+++ b/include/ginkgo/core/base/utils_helper.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_UTILS_HELPER_HPP_
-#define GKO_CORE_BASE_UTILS_HELPER_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_UTILS_HELPER_HPP_
+#define GKO_PUBLIC_CORE_BASE_UTILS_HELPER_HPP_
 
 
 #include <functional>
@@ -435,4 +435,4 @@ class null_deleter<T[]> {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_UTILS_HELPER_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_UTILS_HELPER_HPP_
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 40f415b2156..0602875c4dd 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -56,8 +56,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/base/range_accessors.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
+#include <ginkgo/core/base/temporary_clone.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/base/utils.hpp>
+#include <ginkgo/core/base/utils_helper.hpp>
 #include <ginkgo/core/base/version.hpp>
 
 #include <ginkgo/core/factorization/ilu.hpp>

From e50ca9a9458a304376e824d52b5fe82ac726a63d Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 8 Dec 2020 09:15:34 +0100
Subject: [PATCH 0178/1008] Review updates.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Some code style issues.

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 core/test/base/executor.cpp | 9 ++++++---
 dpcpp/base/executor.dp.cpp  | 1 -
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 783e20f1008..7e2f4ac23b4 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -488,12 +488,15 @@ TEST(Executor, CanVerifyMemory)
     std::shared_ptr<gko::DpcppExecutor> host_dpcpp;
     std::shared_ptr<gko::DpcppExecutor> cpu_dpcpp;
     std::shared_ptr<gko::DpcppExecutor> gpu_dpcpp;
-    if (gko::DpcppExecutor::get_num_devices("host"))
+    if (gko::DpcppExecutor::get_num_devices("host")) {
         host_dpcpp = gko::DpcppExecutor::create(0, omp, "host");
-    if (gko::DpcppExecutor::get_num_devices("cpu"))
+    }
+    if (gko::DpcppExecutor::get_num_devices("cpu")) {
         cpu_dpcpp = gko::DpcppExecutor::create(0, omp, "cpu");
-    if (gko::DpcppExecutor::get_num_devices("gpu"))
+    }
+    if (gko::DpcppExecutor::get_num_devices("gpu")) {
         gpu_dpcpp = gko::DpcppExecutor::create(0, omp, "gpu");
+    }
 
     ASSERT_EQ(true, ref->memory_accessible(omp));
     ASSERT_EQ(true, omp->memory_accessible(ref));
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 25e106a2546..39f9120701a 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -162,7 +162,6 @@ int DpcppExecutor::get_num_devices(std::string device_type)
 bool DpcppExecutor::verify_memory_to(const OmpExecutor *dest_exec) const
 {
     auto device = detail::get_devices(device_type_)[device_id_];
-
     return device.is_host() || device.is_cpu();
 }
 

From 4ced09a26d4e72cc15c560388eb8e0e3bfaa7f14 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 8 Dec 2020 19:11:41 +0100
Subject: [PATCH 0179/1008] Isolate Reference from other executors in memory.

---
 core/device_hooks/cuda_hooks.cpp      |  3 --
 core/device_hooks/dpcpp_hooks.cpp     |  6 ----
 core/device_hooks/hip_hooks.cpp       |  3 --
 core/devices/cuda/executor.cpp        | 16 ++++++++++
 core/devices/hip/executor.cpp         | 16 ++++++++++
 core/test/base/array.cpp              |  8 ++---
 core/test/base/executor.cpp           | 12 +++----
 core/test/base/lin_op.cpp             | 10 +++---
 core/test/base/utils.cpp              |  5 +--
 cuda/base/executor.cpp                | 10 ------
 dpcpp/base/executor.dp.cpp            |  6 ----
 hip/base/executor.hip.cpp             | 10 ------
 include/ginkgo/core/base/executor.hpp | 46 +++++++++++++++++----------
 13 files changed, 80 insertions(+), 71 deletions(-)

diff --git a/core/device_hooks/cuda_hooks.cpp b/core/device_hooks/cuda_hooks.cpp
index a5c33c6541d..46e342d9c0a 100644
--- a/core/device_hooks/cuda_hooks.cpp
+++ b/core/device_hooks/cuda_hooks.cpp
@@ -98,9 +98,6 @@ void CudaExecutor::raw_copy_to(const DpcppExecutor *, size_type num_bytes,
     GKO_NOT_COMPILED(cuda);
 
 
-bool CudaExecutor::verify_memory_to(const HipExecutor *) const { return false; }
-
-
 void CudaExecutor::synchronize() const GKO_NOT_COMPILED(cuda);
 
 
diff --git a/core/device_hooks/dpcpp_hooks.cpp b/core/device_hooks/dpcpp_hooks.cpp
index 472451c6513..88caac1ccd3 100644
--- a/core/device_hooks/dpcpp_hooks.cpp
+++ b/core/device_hooks/dpcpp_hooks.cpp
@@ -126,12 +126,6 @@ bool DpcppExecutor::verify_memory_to(const OmpExecutor *dest_exec) const
     return device_type_ == "cpu" || device_type_ == "host";
 }
 
-bool DpcppExecutor::verify_memory_to(const ReferenceExecutor *dest_exec) const
-{
-    // Dummy check
-    return device_type_ == "cpu" || device_type_ == "host";
-}
-
 bool DpcppExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
 {
     // Dummy check
diff --git a/core/device_hooks/hip_hooks.cpp b/core/device_hooks/hip_hooks.cpp
index 45180980159..b98b2fcb8a4 100644
--- a/core/device_hooks/hip_hooks.cpp
+++ b/core/device_hooks/hip_hooks.cpp
@@ -95,9 +95,6 @@ void HipExecutor::raw_copy_to(const DpcppExecutor *, size_type num_bytes,
     GKO_NOT_COMPILED(hip);
 
 
-bool HipExecutor::verify_memory_to(const CudaExecutor *) const { return false; }
-
-
 void HipExecutor::synchronize() const GKO_NOT_COMPILED(hip);
 
 
diff --git a/core/devices/cuda/executor.cpp b/core/devices/cuda/executor.cpp
index 3566578a681..2d6af2b91ef 100644
--- a/core/devices/cuda/executor.cpp
+++ b/core/devices/cuda/executor.cpp
@@ -48,6 +48,22 @@ std::shared_ptr<const Executor> CudaExecutor::get_master() const noexcept
 }
 
 
+bool CudaExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
+{
+    return device_id_ == dest_exec->get_device_id();
+}
+
+
+bool CudaExecutor::verify_memory_to(const HipExecutor *dest_exec) const
+{
+#if GINKGO_HIP_PLATFORM_NVCC
+    return device_id_ == dest_exec->get_device_id();
+#else
+    return false;
+#endif
+}
+
+
 unsigned CudaExecutor::num_execs[max_devices];
 
 
diff --git a/core/devices/hip/executor.cpp b/core/devices/hip/executor.cpp
index f4787523290..b9a643eb7e2 100644
--- a/core/devices/hip/executor.cpp
+++ b/core/devices/hip/executor.cpp
@@ -45,6 +45,22 @@ std::shared_ptr<const Executor> HipExecutor::get_master() const noexcept
 }
 
 
+bool HipExecutor::verify_memory_to(const HipExecutor *dest_exec) const
+{
+    return device_id_ == dest_exec->get_device_id();
+}
+
+
+bool HipExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
+{
+#if GINKGO_HIP_PLATFORM_NVCC
+    return device_id_ == dest_exec->get_device_id();
+#else
+    return false;
+#endif
+}
+
+
 int HipExecutor::num_execs[max_devices];
 
 
diff --git a/core/test/base/array.cpp b/core/test/base/array.cpp
index 54f7cfea053..c20f5ca72a9 100644
--- a/core/test/base/array.cpp
+++ b/core/test/base/array.cpp
@@ -275,9 +275,9 @@ TYPED_TEST(Array, CanCreateTemporaryCloneOnSameExecutor)
 // For tests between different memory, check cuda/test/base/array.cu
 TYPED_TEST(Array, DoesNotCreateATemporaryCloneBetweenSameMemory)
 {
-    auto omp = gko::OmpExecutor::create();
+    auto other = gko::ReferenceExecutor::create();
 
-    auto tmp_clone = make_temporary_clone(omp, &this->x);
+    auto tmp_clone = make_temporary_clone(other, &this->x);
 
     this->assert_equal_to_original_x(*tmp_clone.get());
     ASSERT_EQ(tmp_clone.get(), &this->x);
@@ -286,10 +286,10 @@ TYPED_TEST(Array, DoesNotCreateATemporaryCloneBetweenSameMemory)
 
 TYPED_TEST(Array, DoesNotCopyBackTemporaryCloneBetweenSameMemory)
 {
-    auto omp = gko::OmpExecutor::create();
+    auto other = gko::ReferenceExecutor::create();
 
     {
-        auto tmp_clone = make_temporary_clone(omp, &this->x);
+        auto tmp_clone = make_temporary_clone(other, &this->x);
         // change x, and check that there is no copy-back to overwrite it again
         this->x.get_data()[0] = 0;
     }
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 7e2f4ac23b4..488334c9ab9 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -498,8 +498,8 @@ TEST(Executor, CanVerifyMemory)
         gpu_dpcpp = gko::DpcppExecutor::create(0, omp, "gpu");
     }
 
-    ASSERT_EQ(true, ref->memory_accessible(omp));
-    ASSERT_EQ(true, omp->memory_accessible(ref));
+    ASSERT_EQ(false, ref->memory_accessible(omp));
+    ASSERT_EQ(false, omp->memory_accessible(ref));
     ASSERT_EQ(false, ref->memory_accessible(hip));
     ASSERT_EQ(false, hip->memory_accessible(ref));
     ASSERT_EQ(false, omp->memory_accessible(hip));
@@ -509,14 +509,14 @@ TEST(Executor, CanVerifyMemory)
     ASSERT_EQ(false, omp->memory_accessible(cuda));
     ASSERT_EQ(false, cuda->memory_accessible(omp));
     if (gko::DpcppExecutor::get_num_devices("host")) {
-        ASSERT_EQ(true, host_dpcpp->memory_accessible(ref));
-        ASSERT_EQ(true, ref->memory_accessible(host_dpcpp));
+        ASSERT_EQ(false, host_dpcpp->memory_accessible(ref));
+        ASSERT_EQ(false, ref->memory_accessible(host_dpcpp));
         ASSERT_EQ(true, host_dpcpp->memory_accessible(omp));
         ASSERT_EQ(true, omp->memory_accessible(host_dpcpp));
     }
     if (gko::DpcppExecutor::get_num_devices("cpu")) {
-        ASSERT_EQ(true, ref->memory_accessible(cpu_dpcpp));
-        ASSERT_EQ(true, cpu_dpcpp->memory_accessible(ref));
+        ASSERT_EQ(false, ref->memory_accessible(cpu_dpcpp));
+        ASSERT_EQ(false, cpu_dpcpp->memory_accessible(ref));
         ASSERT_EQ(true, cpu_dpcpp->memory_accessible(omp));
         ASSERT_EQ(true, omp->memory_accessible(cpu_dpcpp));
     }
diff --git a/core/test/base/lin_op.cpp b/core/test/base/lin_op.cpp
index 3da8eda55f0..e608654ca1f 100644
--- a/core/test/base/lin_op.cpp
+++ b/core/test/base/lin_op.cpp
@@ -93,8 +93,8 @@ class EnableLinOp : public ::testing::Test {
 protected:
     EnableLinOp()
         : ref{gko::ReferenceExecutor::create()},
-          omp{gko::OmpExecutor::create()},
-          op{DummyLinOp::create(omp, gko::dim<2>{3, 5})},
+          ref2{gko::ReferenceExecutor::create()},
+          op{DummyLinOp::create(ref2, gko::dim<2>{3, 5})},
           alpha{DummyLinOp::create(ref, gko::dim<2>{1})},
           beta{DummyLinOp::create(ref, gko::dim<2>{1})},
           b{DummyLinOp::create(ref, gko::dim<2>{5, 4})},
@@ -102,7 +102,7 @@ class EnableLinOp : public ::testing::Test {
     {}
 
     std::shared_ptr<const gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::OmpExecutor> omp;
+    std::shared_ptr<const gko::ReferenceExecutor> ref2;
     std::unique_ptr<DummyLinOp> op;
     std::unique_ptr<DummyLinOp> alpha;
     std::unique_ptr<DummyLinOp> beta;
@@ -115,7 +115,7 @@ TEST_F(EnableLinOp, CallsApplyImpl)
 {
     op->apply(gko::lend(b), gko::lend(x));
 
-    ASSERT_EQ(op->last_access, omp);
+    ASSERT_EQ(op->last_access, ref2);
 }
 
 
@@ -123,7 +123,7 @@ TEST_F(EnableLinOp, CallsExtendedApplyImpl)
 {
     op->apply(gko::lend(alpha), gko::lend(b), gko::lend(beta), gko::lend(x));
 
-    ASSERT_EQ(op->last_access, omp);
+    ASSERT_EQ(op->last_access, ref2);
 }
 
 
diff --git a/core/test/base/utils.cpp b/core/test/base/utils.cpp
index 822449e8f24..a3bafcb5b69 100644
--- a/core/test/base/utils.cpp
+++ b/core/test/base/utils.cpp
@@ -402,9 +402,10 @@ class TemporaryClone : public ::testing::Test {
 
 TEST_F(TemporaryClone, DoesNotCopyToSameMemory)
 {
-    auto clone = make_temporary_clone(omp, gko::lend(obj));
+    auto other = gko::ReferenceExecutor::create();
+    auto clone = make_temporary_clone(other, gko::lend(obj));
 
-    ASSERT_NE(clone.get()->get_executor(), omp);
+    ASSERT_NE(clone.get()->get_executor(), other);
     ASSERT_EQ(obj->get_executor(), ref);
 }
 
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 5284daa2dd1..7926ae98a21 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -162,16 +162,6 @@ void CudaExecutor::raw_copy_to(const CudaExecutor *dest, size_type num_bytes,
 }
 
 
-bool CudaExecutor::verify_memory_to(const HipExecutor *dest_exec) const
-{
-#if GINKGO_HIP_PLATFORM_NVCC
-    return device_id_ == dest_exec->get_device_id();
-#else
-    return false;
-#endif
-}
-
-
 void CudaExecutor::synchronize() const
 {
     cuda::device_guard g(this->get_device_id());
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 39f9120701a..92089616c02 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -165,12 +165,6 @@ bool DpcppExecutor::verify_memory_to(const OmpExecutor *dest_exec) const
     return device.is_host() || device.is_cpu();
 }
 
-bool DpcppExecutor::verify_memory_to(const ReferenceExecutor *dest_exec) const
-{
-    auto device = detail::get_devices(device_type_)[device_id_];
-    return device.is_host() || device.is_cpu();
-}
-
 bool DpcppExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
 {
     auto device = detail::get_devices(device_type_)[device_id_];
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 8f8bbbf7d9d..3e400986e2b 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -162,16 +162,6 @@ void HipExecutor::raw_copy_to(const HipExecutor *dest, size_type num_bytes,
 }
 
 
-bool HipExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
-{
-#if GINKGO_HIP_PLATFORM_NVCC
-    return device_id_ == dest_exec->get_device_id();
-#else
-    return false;
-#endif
-}
-
-
 void HipExecutor::synchronize() const
 {
     hip::device_guard g(this->get_device_id());
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 2ff58f6fe82..f02c5834c73 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -468,6 +468,7 @@ class Executor : public log::EnableLogging<Executor> {
     friend class detail::ExecutorBase;
 
     GKO_ENABLE_FOR_ALL_EXECUTORS(GKO_DECLARE_EXECUTOR_FRIEND);
+    friend class ReferenceExecutor;
 
 public:
     virtual ~Executor() = default;
@@ -855,6 +856,7 @@ namespace detail {
 template <typename ConcreteExecutor>
 class ExecutorBase : public Executor {
     GKO_ENABLE_FOR_ALL_EXECUTORS(GKO_DECLARE_EXECUTOR_FRIEND);
+    friend class ReferenceExecutor;
 
 public:
     void run(const Operation &op) const override
@@ -871,7 +873,7 @@ class ExecutorBase : public Executor {
         src_exec->raw_copy_to(self(), n_bytes, src_ptr, dest_ptr);
     }
 
-    bool verify_memory_from(const Executor *src_exec) const override
+    virtual bool verify_memory_from(const Executor *src_exec) const override
     {
         return src_exec->verify_memory_to(self());
     }
@@ -935,10 +937,13 @@ class EnableDeviceReset {
                      const void *src_ptr, void *dest_ptr) const override
 
 
-#define GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(dest_, bool_)                       \
-    bool verify_memory_to(const dest_ *other) const override { return bool_; } \
-    static_assert(true,                                                        \
-                  "This assert is used to counter the false positive extra "   \
+#define GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(dest_, bool_)                     \
+    virtual bool verify_memory_to(const dest_ *other) const override         \
+    {                                                                        \
+        return bool_;                                                        \
+    }                                                                        \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
                   "semi-colon warnings")
 
 
@@ -979,7 +984,7 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
 
     GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(OmpExecutor, true);
 
-    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(ReferenceExecutor, true);
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(ReferenceExecutor, false);
 
     GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(HipExecutor, false);
 
@@ -1020,6 +1025,21 @@ class ReferenceExecutor : public OmpExecutor {
 
 protected:
     ReferenceExecutor() = default;
+
+    bool verify_memory_from(const Executor *src_exec) const override
+    {
+        return src_exec->verify_memory_to(this);
+    }
+
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(ReferenceExecutor, true);
+
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(OmpExecutor, false);
+
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(DpcppExecutor, false);
+
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(CudaExecutor, false);
+
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(HipExecutor, false);
 };
 
 
@@ -1159,10 +1179,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
     bool verify_memory_to(const HipExecutor *dest_exec) const override;
 
-    bool verify_memory_to(const CudaExecutor *dest_exec) const override
-    {
-        return device_id_ == dest_exec->get_device_id();
-    }
+    bool verify_memory_to(const CudaExecutor *dest_exec) const override;
 
     static void increase_num_execs(unsigned device_id)
     {
@@ -1338,10 +1355,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
     bool verify_memory_to(const CudaExecutor *dest_exec) const override;
 
-    bool verify_memory_to(const HipExecutor *dest_exec) const override
-    {
-        return device_id_ == dest_exec->get_device_id();
-    }
+    bool verify_memory_to(const HipExecutor *dest_exec) const override;
 
     static void increase_num_execs(int device_id)
     {
@@ -1507,9 +1521,9 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
 
     GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(HipExecutor, false);
 
-    bool verify_memory_to(const OmpExecutor *dest_exec) const override;
+    GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(ReferenceExecutor, false);
 
-    bool verify_memory_to(const ReferenceExecutor *dest_exec) const override;
+    bool verify_memory_to(const OmpExecutor *dest_exec) const override;
 
     bool verify_memory_to(const DpcppExecutor *dest_exec) const override;
 

From 57e5b45e53dde57739b67033603f00319af1fb61 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 23 Nov 2020 11:58:37 +0100
Subject: [PATCH 0180/1008] timer for benchmark

---
 benchmark/spmv/spmv.cpp   |  20 +--
 benchmark/utils/timer.hpp | 266 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 278 insertions(+), 8 deletions(-)
 create mode 100644 benchmark/utils/timer.hpp

diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp
index 07debcf9426..c8a5586aa57 100644
--- a/benchmark/spmv/spmv.cpp
+++ b/benchmark/spmv/spmv.cpp
@@ -47,6 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/general.hpp"
 #include "benchmark/utils/loggers.hpp"
 #include "benchmark/utils/spmv_common.hpp"
+#include "benchmark/utils/timer.hpp"
 
 
 using etype = double;
@@ -95,21 +96,24 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
             system_matrix->apply(lend(b), lend(x_clone));
             exec->synchronize();
         }
-        std::chrono::nanoseconds time(0);
+        // std::chrono::nanoseconds time(0);
         // timed run
+        auto timer = get_timer(exec, false);
         for (unsigned int i = 0; i < FLAGS_repetitions; i++) {
             auto x_clone = clone(x);
-            exec->synchronize();
-            auto tic = std::chrono::steady_clock::now();
+            // exec->synchronize();
+            // auto tic = std::chrono::steady_clock::now();
+            timer->tic();
             system_matrix->apply(lend(b), lend(x_clone));
 
-            exec->synchronize();
-            auto toc = std::chrono::steady_clock::now();
-            time +=
-                std::chrono::duration_cast<std::chrono::nanoseconds>(toc - tic);
+            // exec->synchronize();
+            // auto toc = std::chrono::steady_clock::now();
+            // time +=
+            //     std::chrono::duration_cast<std::chrono::nanoseconds>(toc - tic);
+            timer->toc();
         }
         add_or_set_member(spmv_case[format_name], "time",
-                          static_cast<double>(time.count()) / FLAGS_repetitions,
+                          timer->get_average_time(),
                           allocator);
 
         // compute and write benchmark data
diff --git a/benchmark/utils/timer.hpp b/benchmark/utils/timer.hpp
new file mode 100644
index 00000000000..f197bf414e8
--- /dev/null
+++ b/benchmark/utils/timer.hpp
@@ -0,0 +1,266 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/ginkgo.hpp>
+
+
+#include <chrono>
+#include <memory>
+
+
+#ifdef HAS_CUDA
+
+
+#include <cuda.h>
+#include <cuda_runtime.h>
+
+
+#include "cuda/base/device_guard.hpp"
+
+
+#endif  // HAS_CUDA
+
+
+#ifdef HAS_HIP
+
+
+#include <hip/hip_runtime.h>
+
+
+#include "hip/base/device_guard.hip.hpp"
+
+
+#endif  // HAS_CUDA
+
+
+class Timer {
+public:
+    void tic()
+    {
+        assert(tic_called_ == false);
+        this->tic_impl();
+        tic_called_ = true;
+    }
+
+    std::size_t toc()
+    {
+        assert(tic_called_ == true);
+        auto ns = this->toc_impl();
+        tic_called_ = false;
+        this->add_record(ns);
+        return ns;
+    }
+
+    std::size_t get_total_time() { return total_duration_ns_; }
+
+    std::size_t get_tictoc_num() { return duration_ns_.size(); }
+
+    double get_average_time()
+    {
+        return static_cast<double>(this->get_total_time()) /
+               this->get_tictoc_num();
+    }
+
+    Timer() : tic_called_(false), total_duration_ns_(0) {}
+
+protected:
+    void add_record(std::size_t ns)
+    {
+        // add the result;
+        duration_ns_.emplace_back(ns);
+        total_duration_ns_ += ns;
+    }
+
+    virtual void tic_impl() = 0;
+
+    virtual std::size_t toc_impl() = 0;
+
+private:
+    std::vector<std::size_t> duration_ns_;
+    bool tic_called_;
+    std::size_t total_duration_ns_;
+};
+
+
+class CpuTimer : public Timer {
+public:
+    CpuTimer(std::shared_ptr<const gko::Executor> exec) : Timer(), exec_(exec)
+    {}
+
+protected:
+    void tic_impl() override
+    {
+        exec_->synchronize();
+        start_ = std::chrono::steady_clock::now();
+    }
+
+    std::size_t toc_impl() override
+    {
+        exec_->synchronize();
+        auto stop = std::chrono::steady_clock::now();
+        auto duration_time =
+            std::chrono::duration_cast<std::chrono::nanoseconds>(stop - start_)
+                .count();
+        return static_cast<std::size_t>(duration_time);
+    }
+
+private:
+    std::shared_ptr<const gko::Executor> exec_;
+    std::chrono::time_point<std::chrono::steady_clock> start_;
+};
+
+
+#ifdef HAS_CUDA
+
+
+class CudaTimer : public Timer {
+public:
+    CudaTimer(std::shared_ptr<const gko::Executor> exec)
+        : CudaTimer(std::dynamic_pointer_cast<const gko::CudaExecutor>(exec))
+    {}
+
+    CudaTimer(std::shared_ptr<const gko::CudaExecutor> exec) : Timer()
+    {
+        assert(exec != nullptr);
+        exec_ = exec;
+        id_ = exec_->get_device_id();
+        gko::cuda::device_guard g{id_};
+        GKO_ASSERT_NO_CUDA_ERRORS(cudaEventCreate(&start_));
+        GKO_ASSERT_NO_CUDA_ERRORS(cudaEventCreate(&stop_));
+    }
+
+protected:
+    void tic_impl() override
+    {
+        gko::cuda::device_guard g{id_};
+        // Currently, gko::CudaExecutor always use default stream.
+        GKO_ASSERT_NO_CUDA_ERRORS(cudaEventRecord(start_));
+    }
+
+    std::size_t toc_impl() override
+    {
+        gko::cuda::device_guard g{id_};
+        // Currently, gko::CudaExecutor always use default stream.
+        GKO_ASSERT_NO_CUDA_ERRORS(cudaEventRecord(stop_));
+        GKO_ASSERT_NO_CUDA_ERRORS(cudaEventSynchronize(stop_));
+        float duration_time = 0;
+        // cudaEventElapsedTime gives the duration_time in milliseconds with a
+        // resolution of around 0.5 microseconds
+        GKO_ASSERT_NO_CUDA_ERRORS(
+            cudaEventElapsedTime(&duration_time, start_, stop_));
+        return static_cast<std::size_t>(duration_time * 1e6);
+    }
+
+private:
+    std::shared_ptr<const gko::CudaExecutor> exec_;
+    cudaEvent_t start_;
+    cudaEvent_t stop_;
+    int id_;
+};
+
+
+#endif  // HAS_CUDA
+
+
+#ifdef HAS_HIP
+
+
+class HipTimer : public Timer {
+public:
+    HipTimer(std::shared_ptr<const gko::Executor> exec)
+        : HipTimer(std::dynamic_pointer_cast<const gko::HipExecutor>(exec))
+    {}
+
+    HipTimer(std::shared_ptr<const gko::HipExecutor> exec) : Timer()
+    {
+        assert(exec != nullptr);
+        exec_ = exec;
+        id_ = exec_->get_device_id();
+        gko::hip::device_guard g{id_};
+        GKO_ASSERT_NO_HIP_ERRORS(hipEventCreate(&start_));
+        GKO_ASSERT_NO_HIP_ERRORS(hipEventCreate(&stop_));
+    }
+
+protected:
+    void tic_impl() override
+    {
+        gko::hip::device_guard g{id_};
+        // Currently, gko::HipExecutor always use default stream.
+        GKO_ASSERT_NO_HIP_ERRORS(hipEventRecord(start_));
+    }
+
+    std::size_t toc_impl() override
+    {
+        gko::hip::device_guard g{id_};
+        // Currently, gko::HipExecutor always use default stream.
+        GKO_ASSERT_NO_HIP_ERRORS(hipEventRecord(stop_));
+        GKO_ASSERT_NO_HIP_ERRORS(hipEventSynchronize(stop_));
+        float duration_time = 0;
+        // hipEventElapsedTime gives the duration_time in milliseconds with a
+        // resolution of around 0.5 microseconds
+        GKO_ASSERT_NO_HIP_ERRORS(
+            hipEventElapsedTime(&duration_time, start_, stop_));
+        return static_cast<std::size_t>(duration_time * 1e6);
+    }
+
+private:
+    std::shared_ptr<const gko::HipExecutor> exec_;
+    hipEvent_t start_;
+    hipEvent_t stop_;
+    int id_;
+};
+
+
+#endif  // HAS_HIP
+
+
+std::shared_ptr<Timer> get_timer(std::shared_ptr<const gko::Executor> exec,
+                                 bool use_gpu_timer)
+{
+    if (use_gpu_timer) {
+#ifdef HAS_CUDA
+        if (auto cuda =
+                std::dynamic_pointer_cast<const gko::CudaExecutor>(exec)) {
+            return std::make_shared<CudaTimer>(cuda);
+        }
+#endif  // HAS_CUDA
+
+#ifdef HAS_HIP
+        if (auto hip =
+                std::dynamic_pointer_cast<const gko::HipExecutor>(exec)) {
+            return std::make_shared<HipTimer>(hip);
+        }
+#endif  // HAS_HIP
+    }
+    // Not use gpu_timer or not cuda/hip executor
+    return std::make_shared<CpuTimer>(exec);
+}

From 81a1a6942e78ae7daf67d549333b5051634e8fc2 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 23 Nov 2020 23:59:44 +0100
Subject: [PATCH 0181/1008] add timer gflags and use timer in benchmark

---
 benchmark/conversions/conversions.cpp       | 13 +++-----
 benchmark/preconditioner/preconditioner.cpp | 32 +++++++-----------
 benchmark/run_all_benchmarks.sh             | 13 +++++---
 benchmark/solver/solver.cpp                 | 36 +++++++--------------
 benchmark/spmv/spmv.cpp                     | 14 ++------
 benchmark/utils/timer.hpp                   | 15 +++++++++
 6 files changed, 55 insertions(+), 68 deletions(-)

diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index d2cc6c147e9..3c614aa897a 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -47,6 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/general.hpp"
 #include "benchmark/utils/loggers.hpp"
 #include "benchmark/utils/spmv_common.hpp"
+#include "benchmark/utils/timer.hpp"
 
 
 using etype = double;
@@ -75,21 +76,17 @@ void convert_matrix(const gko::LinOp *matrix_from, const char *format_to,
             exec->synchronize();
             matrix_to->clear();
         }
-        std::chrono::nanoseconds time(0);
+        auto timer = get_timer(exec, FLAGS_gpu_timer);
         // timed run
         for (unsigned int i = 0; i < FLAGS_repetitions; i++) {
             exec->synchronize();
-            auto tic = std::chrono::steady_clock::now();
+            timer->tic();
             matrix_to->copy_from(matrix_from);
-            exec->synchronize();
-            auto toc = std::chrono::steady_clock::now();
-            time +=
-                std::chrono::duration_cast<std::chrono::nanoseconds>(toc - tic);
+            timer->toc();
             matrix_to->clear();
         }
         add_or_set_member(conversion_case[conversion_name], "time",
-                          static_cast<double>(time.count()) / FLAGS_repetitions,
-                          allocator);
+                          timer->get_average_time(), allocator);
 
         // compute and write benchmark data
         add_or_set_member(conversion_case[conversion_name], "completed", true,
diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index 0745ebc6037..9bf3663e3d2 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -47,6 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/loggers.hpp"
 #include "benchmark/utils/preconditioners.hpp"
 #include "benchmark/utils/spmv_common.hpp"
+#include "benchmark/utils/timer.hpp"
 
 
 // some shortcuts
@@ -157,41 +158,32 @@ void run_preconditioner(const char *precond_name,
             for (auto i = 0u; i < FLAGS_warmup; ++i) {
                 precond->generate(system_matrix)->apply(lend(b), lend(x_clone));
             }
+            auto generate_timer = get_timer(exec, FLAGS_gpu_timer);
+            auto apply_timer = get_timer(exec, FLAGS_gpu_timer);
 
             exec->synchronize();
-            auto g_tic = std::chrono::steady_clock::now();
-
+            generate_timer->tic();
             std::unique_ptr<gko::LinOp> precond_op;
             for (auto i = 0u; i < FLAGS_repetitions; ++i) {
                 precond_op = precond->generate(system_matrix);
             }
-
-            exec->synchronize();
-            auto g_tac = std::chrono::steady_clock::now();
+            generate_timer->toc();
 
             auto generate_time =
-                std::chrono::duration_cast<std::chrono::nanoseconds>(g_tac -
-                                                                     g_tic) /
-                FLAGS_repetitions;
+                generate_timer->get_total_time() / FLAGS_repetitions;
             add_or_set_member(this_precond_data["generate"], "time",
-                              generate_time.count(), allocator);
+                              generate_time, allocator);
 
             exec->synchronize();
-            auto a_tic = std::chrono::steady_clock::now();
-
+            apply_timer->tic();
             for (auto i = 0u; i < FLAGS_repetitions; ++i) {
                 precond_op->apply(lend(b), lend(x_clone));
             }
+            apply_timer->toc();
 
-            exec->synchronize();
-            auto a_tac = std::chrono::steady_clock::now();
-
-            auto apply_time =
-                std::chrono::duration_cast<std::chrono::nanoseconds>(a_tac -
-                                                                     a_tic) /
-                FLAGS_repetitions;
-            add_or_set_member(this_precond_data["apply"], "time",
-                              apply_time.count(), allocator);
+            auto apply_time = apply_timer->get_total_time() / FLAGS_repetitions;
+            add_or_set_member(this_precond_data["apply"], "time", apply_time,
+                              allocator);
         }
 
         if (FLAGS_detailed) {
diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index c965c5539df..24bea8efbe2 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -71,6 +71,11 @@ else
     SOLVERS_RHS_FLAG="--random_rhs=false"
 fi
 
+if [ ! "${GPU_TIMER}" ]; then
+    echo "GPU_TIMER    environment variable not set - assuming \"false\"" 1>&2
+    GPU_TIMER="false"
+fi
+
 # Control whether to run detailed benchmarks or not.
 # Default setting is detailed=false. To activate, set DETAILED=1.
 if  [ ! "${DETAILED}" ] || [ "${DETAILED}" -eq 0 ]; then
@@ -144,7 +149,7 @@ run_conversion_benchmarks() {
     cp "$1" "$1.imd" # make sure we're not loosing the original input
     ./conversions/conversions --backup="$1.bkp" --double_buffer="$1.bkp2" \
                 --executor="${EXECUTOR}" --formats="${FORMATS}" \
-                --device_id="${DEVICE_ID}" \
+                --device_id="${DEVICE_ID}" --gpu_time=${GPU_TIMER} \
                 <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }
@@ -160,7 +165,7 @@ run_spmv_benchmarks() {
     cp "$1" "$1.imd" # make sure we're not loosing the original input
     ./spmv/spmv --backup="$1.bkp" --double_buffer="$1.bkp2" \
                 --executor="${EXECUTOR}" --formats="${FORMATS}" \
-                --device_id="${DEVICE_ID}" \
+                --device_id="${DEVICE_ID}" --gpu_time=${GPU_TIMER} \
                 <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }
@@ -178,7 +183,7 @@ run_solver_benchmarks() {
                     --executor="${EXECUTOR}" --solvers="${SOLVERS}" \
                     --preconditioners="${PRECONDS}" \
                     --max_iters=${SOLVERS_MAX_ITERATIONS} --rel_res_goal=${SOLVERS_PRECISION} \
-                    ${SOLVERS_RHS_FLAG} ${DETAILED_STR} --device_id="${DEVICE_ID}" \
+                    ${SOLVERS_RHS_FLAG} ${DETAILED_STR} --device_id="${DEVICE_ID}" --gpu_time=${GPU_TIMER} \
                     <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }
@@ -204,7 +209,7 @@ run_preconditioner_benchmarks() {
                 --executor="${EXECUTOR}" --preconditioners="jacobi" \
                 --jacobi_max_block_size="${bsize}" \
                 --jacobi_storage="${prec}" \
-                --device_id="${DEVICE_ID}" \
+                --device_id="${DEVICE_ID}" --gpu_time=${GPU_TIMER} \
                 <"$1.imd" 2>&1 >"$1"
             keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
         done
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index a02c8d1a30c..5df124dac4b 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -49,6 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/loggers.hpp"
 #include "benchmark/utils/overhead_linop.hpp"
 #include "benchmark/utils/preconditioners.hpp"
+#include "benchmark/utils/timer.hpp"
 
 
 // some Ginkgo shortcuts
@@ -355,33 +356,22 @@ void solve_system(const std::string &solver_name,
         }
 
         // timed run
-        std::chrono::nanoseconds apply_time(0);
-        std::chrono::nanoseconds generate_time(0);
+        auto generate_timer = get_timer(exec, FLAGS_gpu_timer);
+        auto apply_timer = get_timer(exec, FLAGS_gpu_timer);
         for (unsigned int i = 0; i < FLAGS_repetitions; i++) {
             auto x_clone = clone(x);
 
             exec->synchronize();
-            auto g_tic = std::chrono::steady_clock::now();
-
+            generate_timer->tic();
             auto precond = precond_factory.at(precond_name)(exec);
             auto solver = solver_factory.at(solver_name)(exec, give(precond))
                               ->generate(system_matrix);
+            generate_timer->toc();
 
             exec->synchronize();
-            auto g_tac = std::chrono::steady_clock::now();
-            generate_time +=
-                std::chrono::duration_cast<std::chrono::nanoseconds>(g_tac -
-                                                                     g_tic);
-
-            exec->synchronize();
-            auto a_tic = std::chrono::steady_clock::now();
-
+            apply_timer->tic();
             solver->apply(lend(b), lend(x_clone));
-
-            exec->synchronize();
-            auto a_tac = std::chrono::steady_clock::now();
-            apply_time += std::chrono::duration_cast<std::chrono::nanoseconds>(
-                a_tac - a_tic);
+            apply_timer->toc();
 
             if (b->get_size()[1] == 1 && i == FLAGS_repetitions - 1 &&
                 !FLAGS_overhead) {
@@ -391,14 +381,10 @@ void solve_system(const std::string &solver_name,
                                   allocator);
             }
         }
-        add_or_set_member(
-            solver_json["generate"], "time",
-            static_cast<double>(generate_time.count()) / FLAGS_repetitions,
-            allocator);
-        add_or_set_member(
-            solver_json["apply"], "time",
-            static_cast<double>(apply_time.count()) / FLAGS_repetitions,
-            allocator);
+        add_or_set_member(solver_json["generate"], "time",
+                          generate_timer->get_average_time(), allocator);
+        add_or_set_member(solver_json["apply"], "time",
+                          apply_timer->get_average_time(), allocator);
 
         // compute and write benchmark data
         add_or_set_member(solver_json, "completed", true, allocator);
diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp
index c8a5586aa57..3dd3381ab7f 100644
--- a/benchmark/spmv/spmv.cpp
+++ b/benchmark/spmv/spmv.cpp
@@ -96,25 +96,17 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
             system_matrix->apply(lend(b), lend(x_clone));
             exec->synchronize();
         }
-        // std::chrono::nanoseconds time(0);
         // timed run
-        auto timer = get_timer(exec, false);
+        auto timer = get_timer(exec, FLAGS_gpu_timer);
         for (unsigned int i = 0; i < FLAGS_repetitions; i++) {
             auto x_clone = clone(x);
-            // exec->synchronize();
-            // auto tic = std::chrono::steady_clock::now();
+            exec->synchronize();
             timer->tic();
             system_matrix->apply(lend(b), lend(x_clone));
-
-            // exec->synchronize();
-            // auto toc = std::chrono::steady_clock::now();
-            // time +=
-            //     std::chrono::duration_cast<std::chrono::nanoseconds>(toc - tic);
             timer->toc();
         }
         add_or_set_member(spmv_case[format_name], "time",
-                          timer->get_average_time(),
-                          allocator);
+                          timer->get_average_time(), allocator);
 
         // compute and write benchmark data
         add_or_set_member(spmv_case[format_name], "completed", true, allocator);
diff --git a/benchmark/utils/timer.hpp b/benchmark/utils/timer.hpp
index f197bf414e8..ffd39919337 100644
--- a/benchmark/utils/timer.hpp
+++ b/benchmark/utils/timer.hpp
@@ -37,6 +37,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <memory>
 
 
+#include <gflags/gflags.h>
+
+
 #ifdef HAS_CUDA
 
 
@@ -62,6 +65,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif  // HAS_CUDA
 
 
+DEFINE_bool(gpu_timer, false,
+            "use gpu timer based on event. It is valid only when "
+            "executor is cuda or hip");
+
+
 class Timer {
 public:
     void tic()
@@ -90,6 +98,13 @@ class Timer {
                this->get_tictoc_num();
     }
 
+    void clear()
+    {
+        duration_ns_.clear();
+        tic_called_ = false;
+        total_duration_ns_ = 0;
+    }
+
     Timer() : tic_called_(false), total_duration_ns_(0) {}
 
 protected:

From aa7aaf92eb291242ac0ef72c2c86daeb9ae763d0 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 27 Nov 2020 21:50:42 +0100
Subject: [PATCH 0182/1008] Review update and rename - fix gpu_timer in script
 - use int64_t - add const to function - update documentation - add
 get_latest_time - rename get_average_time -> compute_average_time
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 BENCHMARKING.md                             |   2 +
 benchmark/conversions/conversions.cpp       |   2 +-
 benchmark/preconditioner/preconditioner.cpp |  15 +-
 benchmark/run_all_benchmarks.sh             |   8 +-
 benchmark/solver/solver.cpp                 |   4 +-
 benchmark/spmv/spmv.cpp                     |   2 +-
 benchmark/utils/timer.hpp                   | 147 +++++++++++++++++---
 7 files changed, 150 insertions(+), 30 deletions(-)

diff --git a/BENCHMARKING.md b/BENCHMARKING.md
index 043cdd71cf7..883100d0f02 100644
--- a/BENCHMARKING.md
+++ b/BENCHMARKING.md
@@ -305,3 +305,5 @@ The supported environment variables are described in the following list:
     values as the right-hand side in solver benchmarks. Default is `unit`.
 * `DETAILED={0,1}` - selects whether detailed benchmarks should be ran for the
     solver benchmarks, can be either `0` (off) or `1` (on). The default is `0`.
+* `GPU_TIMER={true, false}` - If set to `true`, use the gpu timer, which is
+    valid for cuda/hip executor, to measure the timing. Default is `false`.
diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index 3c614aa897a..c7e87703102 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -86,7 +86,7 @@ void convert_matrix(const gko::LinOp *matrix_from, const char *format_to,
             matrix_to->clear();
         }
         add_or_set_member(conversion_case[conversion_name], "time",
-                          timer->get_average_time(), allocator);
+                          timer->compute_average_time(), allocator);
 
         // compute and write benchmark data
         add_or_set_member(conversion_case[conversion_name], "completed", true,
diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index 9bf3663e3d2..81bcc2f78b4 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -169,8 +169,13 @@ void run_preconditioner(const char *precond_name,
             }
             generate_timer->toc();
 
+            // the timer is out of the loops to reduce calling synchronize
+            // overhead, so the timer does not know the number of repetitions.
+            // get_total_time() and compute_average_time() have the same result
+            // but in different type. We use compute_average_time to eliminate
+            // the type-casting here.
             auto generate_time =
-                generate_timer->get_total_time() / FLAGS_repetitions;
+                generate_timer->compute_average_time() / FLAGS_repetitions;
             add_or_set_member(this_precond_data["generate"], "time",
                               generate_time, allocator);
 
@@ -181,7 +186,13 @@ void run_preconditioner(const char *precond_name,
             }
             apply_timer->toc();
 
-            auto apply_time = apply_timer->get_total_time() / FLAGS_repetitions;
+            // the timer is out of the loops to reduce calling synchronize
+            // overhead, so the timer does not know the number of repetitions.
+            // get_total_time() and compute_average_time() have the same result
+            // but in different type. We use compute_average_time to eliminate
+            // the type-casting here.
+            auto apply_time =
+                apply_timer->compute_average_time() / FLAGS_repetitions;
             add_or_set_member(this_precond_data["apply"], "time", apply_time,
                               allocator);
         }
diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index 24bea8efbe2..d8bb6c6a88b 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -149,7 +149,7 @@ run_conversion_benchmarks() {
     cp "$1" "$1.imd" # make sure we're not loosing the original input
     ./conversions/conversions --backup="$1.bkp" --double_buffer="$1.bkp2" \
                 --executor="${EXECUTOR}" --formats="${FORMATS}" \
-                --device_id="${DEVICE_ID}" --gpu_time=${GPU_TIMER} \
+                --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
                 <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }
@@ -165,7 +165,7 @@ run_spmv_benchmarks() {
     cp "$1" "$1.imd" # make sure we're not loosing the original input
     ./spmv/spmv --backup="$1.bkp" --double_buffer="$1.bkp2" \
                 --executor="${EXECUTOR}" --formats="${FORMATS}" \
-                --device_id="${DEVICE_ID}" --gpu_time=${GPU_TIMER} \
+                --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
                 <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }
@@ -183,7 +183,7 @@ run_solver_benchmarks() {
                     --executor="${EXECUTOR}" --solvers="${SOLVERS}" \
                     --preconditioners="${PRECONDS}" \
                     --max_iters=${SOLVERS_MAX_ITERATIONS} --rel_res_goal=${SOLVERS_PRECISION} \
-                    ${SOLVERS_RHS_FLAG} ${DETAILED_STR} --device_id="${DEVICE_ID}" --gpu_time=${GPU_TIMER} \
+                    ${SOLVERS_RHS_FLAG} ${DETAILED_STR} --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
                     <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }
@@ -209,7 +209,7 @@ run_preconditioner_benchmarks() {
                 --executor="${EXECUTOR}" --preconditioners="jacobi" \
                 --jacobi_max_block_size="${bsize}" \
                 --jacobi_storage="${prec}" \
-                --device_id="${DEVICE_ID}" --gpu_time=${GPU_TIMER} \
+                --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
                 <"$1.imd" 2>&1 >"$1"
             keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
         done
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 5df124dac4b..5ec3da9de08 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -382,9 +382,9 @@ void solve_system(const std::string &solver_name,
             }
         }
         add_or_set_member(solver_json["generate"], "time",
-                          generate_timer->get_average_time(), allocator);
+                          generate_timer->compute_average_time(), allocator);
         add_or_set_member(solver_json["apply"], "time",
-                          apply_timer->get_average_time(), allocator);
+                          apply_timer->compute_average_time(), allocator);
 
         // compute and write benchmark data
         add_or_set_member(solver_json, "completed", true, allocator);
diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp
index 3dd3381ab7f..36d9b19651f 100644
--- a/benchmark/spmv/spmv.cpp
+++ b/benchmark/spmv/spmv.cpp
@@ -106,7 +106,7 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
             timer->toc();
         }
         add_or_set_member(spmv_case[format_name], "time",
-                          timer->get_average_time(), allocator);
+                          timer->compute_average_time(), allocator);
 
         // compute and write benchmark data
         add_or_set_member(spmv_case[format_name], "completed", true, allocator);
diff --git a/benchmark/utils/timer.hpp b/benchmark/utils/timer.hpp
index ffd39919337..4480b6f19cd 100644
--- a/benchmark/utils/timer.hpp
+++ b/benchmark/utils/timer.hpp
@@ -62,16 +62,23 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "hip/base/device_guard.hip.hpp"
 
 
-#endif  // HAS_CUDA
+#endif  // HAS_HIP
 
 
+// Command-line arguments
 DEFINE_bool(gpu_timer, false,
             "use gpu timer based on event. It is valid only when "
             "executor is cuda or hip");
 
 
+/**
+ * Timer stores the timing information
+ */
 class Timer {
 public:
+    /**
+     * Start the timer
+     */
     void tic()
     {
         assert(tic_called_ == false);
@@ -79,25 +86,67 @@ class Timer {
         tic_called_ = true;
     }
 
-    std::size_t toc()
+    /**
+     * Finish the timer
+     */
+    void toc()
     {
         assert(tic_called_ == true);
         auto ns = this->toc_impl();
         tic_called_ = false;
         this->add_record(ns);
-        return ns;
     }
 
-    std::size_t get_total_time() { return total_duration_ns_; }
-
-    std::size_t get_tictoc_num() { return duration_ns_.size(); }
-
-    double get_average_time()
+    /**
+     * Get the summation of each time in nanoseconds.
+     *
+     * @return the nanoseconds of total time
+     */
+    std::int64_t get_total_time() const { return total_duration_ns_; }
+
+    /**
+     * Get the number of repetitions.
+     *
+     * @return the number of repetitions
+     */
+    std::int64_t get_num_repetitions() const { return duration_ns_.size(); }
+
+    /**
+     * Compute the average time of repetitions in nanoseconds
+     *
+     * @return the average time in nanoseconds
+     */
+    double compute_average_time() const
     {
         return static_cast<double>(this->get_total_time()) /
-               this->get_tictoc_num();
+               this->get_num_repetitions();
+    }
+
+    /**
+     * Get the vector containing the time of each repetition in nanoseconds.
+     *
+     * @return the vector of time for each repetition in nanoseconds
+     */
+    std::vector<std::int64_t> get_time_detail() const { return duration_ns_; }
+
+    /**
+     * Get the latest result in nanoseconds. If there is no result yet, return
+     * 0.
+     *
+     * @return the latest result in nanoseconds
+     */
+    std::int64_t get_latest_time() const
+    {
+        if (duration_ns_.size() >= 1) {
+            return duration_ns_.back();
+        } else {
+            return 0;
+        }
     }
 
+    /**
+     * Clear the results of timer
+     */
     void clear()
     {
         duration_ns_.clear();
@@ -105,29 +154,54 @@ class Timer {
         total_duration_ns_ = 0;
     }
 
+    /**
+     * Create a timer
+     */
     Timer() : tic_called_(false), total_duration_ns_(0) {}
 
 protected:
-    void add_record(std::size_t ns)
+    /**
+     * Put the nanosecond result into vector
+     *
+     * @param ns  the nanosecond result to insert
+     */
+    void add_record(std::int64_t ns)
     {
         // add the result;
         duration_ns_.emplace_back(ns);
         total_duration_ns_ += ns;
     }
 
+    /**
+     * The implementation of tic.
+     */
     virtual void tic_impl() = 0;
 
-    virtual std::size_t toc_impl() = 0;
+    /**
+     * The implementation of toc. Return the nanoseconds result.
+     *
+     * @return the nanoseconds result
+     */
+    virtual std::int64_t toc_impl() = 0;
 
 private:
-    std::vector<std::size_t> duration_ns_;
+    std::vector<std::int64_t> duration_ns_;
     bool tic_called_;
-    std::size_t total_duration_ns_;
+    std::int64_t total_duration_ns_;
 };
 
 
+/**
+ * CpuTimer uses the synchronize of the executor and std::chrono to measure the
+ * timing.
+ */
 class CpuTimer : public Timer {
 public:
+    /**
+     * Create a CpuTimer
+     *
+     * @param exec  Executor associated to the timer
+     */
     CpuTimer(std::shared_ptr<const gko::Executor> exec) : Timer(), exec_(exec)
     {}
 
@@ -138,14 +212,14 @@ class CpuTimer : public Timer {
         start_ = std::chrono::steady_clock::now();
     }
 
-    std::size_t toc_impl() override
+    std::int64_t toc_impl() override
     {
         exec_->synchronize();
         auto stop = std::chrono::steady_clock::now();
         auto duration_time =
             std::chrono::duration_cast<std::chrono::nanoseconds>(stop - start_)
                 .count();
-        return static_cast<std::size_t>(duration_time);
+        return duration_time;
     }
 
 private:
@@ -157,12 +231,25 @@ class CpuTimer : public Timer {
 #ifdef HAS_CUDA
 
 
+/**
+ * CudaTimer uses cuda executor and cudaEvent to measure the timing.
+ */
 class CudaTimer : public Timer {
 public:
+    /**
+     * Create a CudaTimer.
+     *
+     * @param exec  Executor which should be a CudaExecutor
+     */
     CudaTimer(std::shared_ptr<const gko::Executor> exec)
         : CudaTimer(std::dynamic_pointer_cast<const gko::CudaExecutor>(exec))
     {}
 
+    /**
+     * Create a CudaTimer.
+     *
+     * @param exec  CudaExecutor associated to the timer
+     */
     CudaTimer(std::shared_ptr<const gko::CudaExecutor> exec) : Timer()
     {
         assert(exec != nullptr);
@@ -181,7 +268,7 @@ class CudaTimer : public Timer {
         GKO_ASSERT_NO_CUDA_ERRORS(cudaEventRecord(start_));
     }
 
-    std::size_t toc_impl() override
+    std::int64_t toc_impl() override
     {
         gko::cuda::device_guard g{id_};
         // Currently, gko::CudaExecutor always use default stream.
@@ -192,7 +279,7 @@ class CudaTimer : public Timer {
         // resolution of around 0.5 microseconds
         GKO_ASSERT_NO_CUDA_ERRORS(
             cudaEventElapsedTime(&duration_time, start_, stop_));
-        return static_cast<std::size_t>(duration_time * 1e6);
+        return static_cast<std::int64_t>(duration_time * 1e6);
     }
 
 private:
@@ -209,12 +296,25 @@ class CudaTimer : public Timer {
 #ifdef HAS_HIP
 
 
+/**
+ * HipTimer uses hip executor and hipEvent to measure the timing.
+ */
 class HipTimer : public Timer {
 public:
+    /**
+     * Create a HipTimer.
+     *
+     * @param exec  Executor which should be a HipExecutor
+     */
     HipTimer(std::shared_ptr<const gko::Executor> exec)
         : HipTimer(std::dynamic_pointer_cast<const gko::HipExecutor>(exec))
     {}
 
+    /**
+     * Create a HipTimer.
+     *
+     * @param exec  HipExecutor associated to the timer
+     */
     HipTimer(std::shared_ptr<const gko::HipExecutor> exec) : Timer()
     {
         assert(exec != nullptr);
@@ -233,7 +333,7 @@ class HipTimer : public Timer {
         GKO_ASSERT_NO_HIP_ERRORS(hipEventRecord(start_));
     }
 
-    std::size_t toc_impl() override
+    std::int64_t toc_impl() override
     {
         gko::hip::device_guard g{id_};
         // Currently, gko::HipExecutor always use default stream.
@@ -244,7 +344,7 @@ class HipTimer : public Timer {
         // resolution of around 0.5 microseconds
         GKO_ASSERT_NO_HIP_ERRORS(
             hipEventElapsedTime(&duration_time, start_, stop_));
-        return static_cast<std::size_t>(duration_time * 1e6);
+        return static_cast<std::int64_t>(duration_time * 1e6);
     }
 
 private:
@@ -258,6 +358,13 @@ class HipTimer : public Timer {
 #endif  // HAS_HIP
 
 
+/**
+ * Get the timer. If the executor does not support gpu timer, still return the
+ * cpu timer.
+ *
+ * @param exec  Executor associated to the timer
+ * @param use_gpu_timer  whether to use the gpu timer
+ */
 std::shared_ptr<Timer> get_timer(std::shared_ptr<const gko::Executor> exec,
                                  bool use_gpu_timer)
 {
@@ -276,6 +383,6 @@ std::shared_ptr<Timer> get_timer(std::shared_ptr<const gko::Executor> exec,
         }
 #endif  // HAS_HIP
     }
-    // Not use gpu_timer or not cuda/hip executor
+    // No cuda/hip executor available or no gpu_timer used
     return std::make_shared<CpuTimer>(exec);
 }

From e85ae4035e2a2692921fc0b02d72ba8dfa0273f4 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 14 Dec 2020 21:01:14 +0100
Subject: [PATCH 0183/1008] use second and double precision of timing result

---
 benchmark/preconditioner/preconditioner.cpp | 11 +--
 benchmark/utils/chrono_utils.hpp            | 62 +++++++++++++++++
 benchmark/utils/loggers.hpp                 | 11 +--
 benchmark/utils/timer.hpp                   | 76 +++++++++++----------
 4 files changed, 106 insertions(+), 54 deletions(-)
 create mode 100644 benchmark/utils/chrono_utils.hpp

diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index 81bcc2f78b4..9bef693fd51 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -171,11 +171,8 @@ void run_preconditioner(const char *precond_name,
 
             // the timer is out of the loops to reduce calling synchronize
             // overhead, so the timer does not know the number of repetitions.
-            // get_total_time() and compute_average_time() have the same result
-            // but in different type. We use compute_average_time to eliminate
-            // the type-casting here.
             auto generate_time =
-                generate_timer->compute_average_time() / FLAGS_repetitions;
+                generate_timer->get_total_time() / FLAGS_repetitions;
             add_or_set_member(this_precond_data["generate"], "time",
                               generate_time, allocator);
 
@@ -188,11 +185,7 @@ void run_preconditioner(const char *precond_name,
 
             // the timer is out of the loops to reduce calling synchronize
             // overhead, so the timer does not know the number of repetitions.
-            // get_total_time() and compute_average_time() have the same result
-            // but in different type. We use compute_average_time to eliminate
-            // the type-casting here.
-            auto apply_time =
-                apply_timer->compute_average_time() / FLAGS_repetitions;
+            auto apply_time = apply_timer->get_total_time() / FLAGS_repetitions;
             add_or_set_member(this_precond_data["apply"], "time", apply_time,
                               allocator);
         }
diff --git a/benchmark/utils/chrono_utils.hpp b/benchmark/utils/chrono_utils.hpp
new file mode 100644
index 00000000000..a2ade3f4a7f
--- /dev/null
+++ b/benchmark/utils/chrono_utils.hpp
@@ -0,0 +1,62 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_BENCHMARK_UTILS_CHRONO_UTILS_HPP_
+#define GKO_BENCHMARK_UTILS_CHRONO_UTILS_HPP_
+
+
+#include <chrono>
+
+
+namespace {
+
+
+/**
+ * Get the seconds of duration in double precision.
+ *
+ * @param duration  the chrono::steady_clock::duration
+ *
+ * @return the seconds of duration in double precision.
+ */
+double get_duration_in_seconds(std::chrono::steady_clock::duration time)
+{
+    auto counts =
+        std::chrono::duration_cast<std::chrono::nanoseconds>(time).count();
+    constexpr int sec_in_ns = 1e9;
+    return static_cast<double>(counts) / sec_in_ns;
+}
+
+
+}  // namespace
+
+
+#endif  // GKO_BENCHMARK_UTILS_CHRONO_UTILS_HPP_
diff --git a/benchmark/utils/loggers.hpp b/benchmark/utils/loggers.hpp
index ea6bbea2797..3c05bac4ce7 100644
--- a/benchmark/utils/loggers.hpp
+++ b/benchmark/utils/loggers.hpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <unordered_map>
 
 
+#include "benchmark/utils/chrono_utils.hpp"
 #include "benchmark/utils/general.hpp"
 
 
@@ -109,11 +110,7 @@ struct OperationLogger : gko::log::Logger {
         for (const auto &entry : total) {
             add_or_set_member(
                 object, entry.first.c_str(),
-                std::chrono::duration_cast<std::chrono::nanoseconds>(
-                    entry.second)
-                        .count() /
-                    repetitions,
-                alloc);
+                get_duration_in_seconds(entry.second) / repetitions, alloc);
         }
     }
 
@@ -208,9 +205,7 @@ struct ResidualLogger : gko::log::Logger {
                                const gko::LinOp *residual_norm) const override
     {
         timestamps.PushBack(
-            std::chrono::duration_cast<std::chrono::nanoseconds>(
-                std::chrono::steady_clock::now() - start)
-                .count(),
+            get_duration_in_seconds(std::chrono::steady_clock::now() - start),
             alloc);
         if (residual_norm) {
             rec_res_norms.PushBack(
diff --git a/benchmark/utils/timer.hpp b/benchmark/utils/timer.hpp
index 4480b6f19cd..3d0b47b1c6e 100644
--- a/benchmark/utils/timer.hpp
+++ b/benchmark/utils/timer.hpp
@@ -40,6 +40,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gflags/gflags.h>
 
 
+#include "benchmark/utils/chrono_utils.hpp"
+
+
 #ifdef HAS_CUDA
 
 
@@ -98,47 +101,46 @@ class Timer {
     }
 
     /**
-     * Get the summation of each time in nanoseconds.
+     * Get the summation of each time in seconds.
      *
-     * @return the nanoseconds of total time
+     * @return the seconds of total time
      */
-    std::int64_t get_total_time() const { return total_duration_ns_; }
+    double get_total_time() const { return total_duration_sec_; }
 
     /**
      * Get the number of repetitions.
      *
      * @return the number of repetitions
      */
-    std::int64_t get_num_repetitions() const { return duration_ns_.size(); }
+    std::int64_t get_num_repetitions() const { return duration_sec_.size(); }
 
     /**
-     * Compute the average time of repetitions in nanoseconds
+     * Compute the average time of repetitions in seconds
      *
-     * @return the average time in nanoseconds
+     * @return the average time in seconds
      */
     double compute_average_time() const
     {
-        return static_cast<double>(this->get_total_time()) /
-               this->get_num_repetitions();
+        return this->get_total_time() / this->get_num_repetitions();
     }
 
     /**
-     * Get the vector containing the time of each repetition in nanoseconds.
+     * Get the vector containing the time of each repetition in seconds.
      *
-     * @return the vector of time for each repetition in nanoseconds
+     * @return the vector of time for each repetition in seconds
      */
-    std::vector<std::int64_t> get_time_detail() const { return duration_ns_; }
+    std::vector<double> get_time_detail() const { return duration_sec_; }
 
     /**
-     * Get the latest result in nanoseconds. If there is no result yet, return
+     * Get the latest result in seconds. If there is no result yet, return
      * 0.
      *
-     * @return the latest result in nanoseconds
+     * @return the latest result in seconds
      */
-    std::int64_t get_latest_time() const
+    double get_latest_time() const
     {
-        if (duration_ns_.size() >= 1) {
-            return duration_ns_.back();
+        if (duration_sec_.size() >= 1) {
+            return duration_sec_.back();
         } else {
             return 0;
         }
@@ -149,27 +151,27 @@ class Timer {
      */
     void clear()
     {
-        duration_ns_.clear();
+        duration_sec_.clear();
         tic_called_ = false;
-        total_duration_ns_ = 0;
+        total_duration_sec_ = 0;
     }
 
     /**
      * Create a timer
      */
-    Timer() : tic_called_(false), total_duration_ns_(0) {}
+    Timer() : tic_called_(false), total_duration_sec_(0) {}
 
 protected:
     /**
-     * Put the nanosecond result into vector
+     * Put the second result into vector
      *
-     * @param ns  the nanosecond result to insert
+     * @param ns  the second result to insert
      */
-    void add_record(std::int64_t ns)
+    void add_record(double ns)
     {
         // add the result;
-        duration_ns_.emplace_back(ns);
-        total_duration_ns_ += ns;
+        duration_sec_.emplace_back(ns);
+        total_duration_sec_ += ns;
     }
 
     /**
@@ -178,16 +180,16 @@ class Timer {
     virtual void tic_impl() = 0;
 
     /**
-     * The implementation of toc. Return the nanoseconds result.
+     * The implementation of toc. Return the seconds result.
      *
-     * @return the nanoseconds result
+     * @return the seconds result
      */
-    virtual std::int64_t toc_impl() = 0;
+    virtual double toc_impl() = 0;
 
 private:
-    std::vector<std::int64_t> duration_ns_;
+    std::vector<double> duration_sec_;
     bool tic_called_;
-    std::int64_t total_duration_ns_;
+    double total_duration_sec_;
 };
 
 
@@ -212,13 +214,11 @@ class CpuTimer : public Timer {
         start_ = std::chrono::steady_clock::now();
     }
 
-    std::int64_t toc_impl() override
+    double toc_impl() override
     {
         exec_->synchronize();
         auto stop = std::chrono::steady_clock::now();
-        auto duration_time =
-            std::chrono::duration_cast<std::chrono::nanoseconds>(stop - start_)
-                .count();
+        auto duration_time = get_duration_in_seconds(stop - start_);
         return duration_time;
     }
 
@@ -268,7 +268,7 @@ class CudaTimer : public Timer {
         GKO_ASSERT_NO_CUDA_ERRORS(cudaEventRecord(start_));
     }
 
-    std::int64_t toc_impl() override
+    double toc_impl() override
     {
         gko::cuda::device_guard g{id_};
         // Currently, gko::CudaExecutor always use default stream.
@@ -279,7 +279,8 @@ class CudaTimer : public Timer {
         // resolution of around 0.5 microseconds
         GKO_ASSERT_NO_CUDA_ERRORS(
             cudaEventElapsedTime(&duration_time, start_, stop_));
-        return static_cast<std::int64_t>(duration_time * 1e6);
+        constexpr int sec_in_ms = 1e3;
+        return static_cast<double>(duration_time) / sec_in_ms;
     }
 
 private:
@@ -333,7 +334,7 @@ class HipTimer : public Timer {
         GKO_ASSERT_NO_HIP_ERRORS(hipEventRecord(start_));
     }
 
-    std::int64_t toc_impl() override
+    double toc_impl() override
     {
         gko::hip::device_guard g{id_};
         // Currently, gko::HipExecutor always use default stream.
@@ -344,7 +345,8 @@ class HipTimer : public Timer {
         // resolution of around 0.5 microseconds
         GKO_ASSERT_NO_HIP_ERRORS(
             hipEventElapsedTime(&duration_time, start_, stop_));
-        return static_cast<std::int64_t>(duration_time * 1e6);
+        constexpr int sec_in_ms = 1e3;
+        return static_cast<double>(duration_time) / sec_in_ms;
     }
 
 private:

From 93b3e4acd63cb969446f41ef0dabaaa31165353e Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 17 Dec 2020 14:42:36 +0100
Subject: [PATCH 0184/1008] use standard way to output duration in seconds

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
---
 benchmark/utils/chrono_utils.hpp | 62 --------------------------------
 benchmark/utils/loggers.hpp      | 12 ++++---
 benchmark/utils/timer.hpp        | 19 +++++-----
 3 files changed, 15 insertions(+), 78 deletions(-)
 delete mode 100644 benchmark/utils/chrono_utils.hpp

diff --git a/benchmark/utils/chrono_utils.hpp b/benchmark/utils/chrono_utils.hpp
deleted file mode 100644
index a2ade3f4a7f..00000000000
--- a/benchmark/utils/chrono_utils.hpp
+++ /dev/null
@@ -1,62 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GKO_BENCHMARK_UTILS_CHRONO_UTILS_HPP_
-#define GKO_BENCHMARK_UTILS_CHRONO_UTILS_HPP_
-
-
-#include <chrono>
-
-
-namespace {
-
-
-/**
- * Get the seconds of duration in double precision.
- *
- * @param duration  the chrono::steady_clock::duration
- *
- * @return the seconds of duration in double precision.
- */
-double get_duration_in_seconds(std::chrono::steady_clock::duration time)
-{
-    auto counts =
-        std::chrono::duration_cast<std::chrono::nanoseconds>(time).count();
-    constexpr int sec_in_ns = 1e9;
-    return static_cast<double>(counts) / sec_in_ns;
-}
-
-
-}  // namespace
-
-
-#endif  // GKO_BENCHMARK_UTILS_CHRONO_UTILS_HPP_
diff --git a/benchmark/utils/loggers.hpp b/benchmark/utils/loggers.hpp
index 3c05bac4ce7..72892891f0e 100644
--- a/benchmark/utils/loggers.hpp
+++ b/benchmark/utils/loggers.hpp
@@ -43,7 +43,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <unordered_map>
 
 
-#include "benchmark/utils/chrono_utils.hpp"
 #include "benchmark/utils/general.hpp"
 
 
@@ -110,7 +109,9 @@ struct OperationLogger : gko::log::Logger {
         for (const auto &entry : total) {
             add_or_set_member(
                 object, entry.first.c_str(),
-                get_duration_in_seconds(entry.second) / repetitions, alloc);
+                std::chrono::duration<double>(entry.second).count() /
+                    repetitions,
+                alloc);
         }
     }
 
@@ -204,9 +205,10 @@ struct ResidualLogger : gko::log::Logger {
                                const gko::LinOp *solution,
                                const gko::LinOp *residual_norm) const override
     {
-        timestamps.PushBack(
-            get_duration_in_seconds(std::chrono::steady_clock::now() - start),
-            alloc);
+        timestamps.PushBack(std::chrono::duration<double>(
+                                std::chrono::steady_clock::now() - start)
+                                .count(),
+                            alloc);
         if (residual_norm) {
             rec_res_norms.PushBack(
                 get_norm(gko::as<vec<ValueType>>(residual_norm)), alloc);
diff --git a/benchmark/utils/timer.hpp b/benchmark/utils/timer.hpp
index 3d0b47b1c6e..c94f19d92a3 100644
--- a/benchmark/utils/timer.hpp
+++ b/benchmark/utils/timer.hpp
@@ -40,9 +40,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gflags/gflags.h>
 
 
-#include "benchmark/utils/chrono_utils.hpp"
-
-
 #ifdef HAS_CUDA
 
 
@@ -95,9 +92,9 @@ class Timer {
     void toc()
     {
         assert(tic_called_ == true);
-        auto ns = this->toc_impl();
+        auto sec = this->toc_impl();
         tic_called_ = false;
-        this->add_record(ns);
+        this->add_record(sec);
     }
 
     /**
@@ -165,13 +162,13 @@ class Timer {
     /**
      * Put the second result into vector
      *
-     * @param ns  the second result to insert
+     * @param sec  the second result to insert
      */
-    void add_record(double ns)
+    void add_record(double sec)
     {
         // add the result;
-        duration_sec_.emplace_back(ns);
-        total_duration_sec_ += ns;
+        duration_sec_.emplace_back(sec);
+        total_duration_sec_ += sec;
     }
 
     /**
@@ -218,8 +215,8 @@ class CpuTimer : public Timer {
     {
         exec_->synchronize();
         auto stop = std::chrono::steady_clock::now();
-        auto duration_time = get_duration_in_seconds(stop - start_);
-        return duration_time;
+        std::chrono::duration<double> duration_time = stop - start_;
+        return duration_time.count();
     }
 
 private:

From 29b6d0216befc61c7ae6513a9e191509eb030928 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 19 Nov 2020 20:59:01 +0100
Subject: [PATCH 0185/1008] fix some overflows and windows warnings

---
 core/test/matrix/hybrid.cpp                       | 12 ++++++------
 include/ginkgo/core/base/abstract_factory.hpp     |  3 ++-
 include/ginkgo/core/base/lin_op.hpp               |  4 ++--
 include/ginkgo/core/matrix/csr.hpp                | 10 +++++-----
 include/ginkgo/core/matrix/hybrid.hpp             | 12 ++++++------
 include/ginkgo/core/preconditioner/jacobi.hpp     |  3 ++-
 include/ginkgo/core/stop/criterion.hpp            |  4 ++--
 include/ginkgo/core/stop/residual_norm.hpp        | 12 ++++++------
 omp/components/prefix_sum.cpp                     |  2 +-
 omp/solver/lower_trs_kernels.cpp                  |  2 +-
 omp/solver/upper_trs_kernels.cpp                  |  8 +++++---
 reference/components/csr_spgeam.hpp               |  2 +-
 reference/components/prefix_sum.cpp               |  2 +-
 reference/factorization/factorization_kernels.cpp | 10 +++++-----
 reference/factorization/par_ict_kernels.cpp       |  3 +--
 reference/matrix/csr_kernels.cpp                  |  6 +++---
 reference/matrix/dense_kernels.cpp                |  2 +-
 reference/preconditioner/jacobi_kernels.cpp       | 10 +++++-----
 reference/solver/lower_trs_kernels.cpp            |  2 +-
 reference/solver/upper_trs_kernels.cpp            |  8 +++++---
 20 files changed, 61 insertions(+), 56 deletions(-)

diff --git a/core/test/matrix/hybrid.cpp b/core/test/matrix/hybrid.cpp
index 8d2fbe8efaa..6d65078a3ed 100644
--- a/core/test/matrix/hybrid.cpp
+++ b/core/test/matrix/hybrid.cpp
@@ -443,8 +443,8 @@ TYPED_TEST(Hybrid, GetCorrectImbalanceLimit)
     auto mtx_stra = gko::as<strategy>(mtx->get_strategy());
     auto mtx2_stra = gko::as<strategy2>(mtx->template get_strategy<Mtx2>());
 
-    EXPECT_EQ(mtx_stra->get_percentage(), 0.4f);
-    EXPECT_EQ(mtx2_stra->get_percentage(), 0.4f);
+    EXPECT_EQ(mtx_stra->get_percentage(), 0.4);
+    EXPECT_EQ(mtx2_stra->get_percentage(), 0.4);
 }
 
 
@@ -459,10 +459,10 @@ TYPED_TEST(Hybrid, GetCorrectImbalanceBoundedLimit)
     auto mtx_stra = gko::as<strategy>(mtx->get_strategy());
     auto mtx2_stra = gko::as<strategy2>(mtx->template get_strategy<Mtx2>());
 
-    EXPECT_EQ(mtx_stra->get_percentage(), 0.4f);
-    EXPECT_EQ(mtx_stra->get_ratio(), 0.1f);
-    EXPECT_EQ(mtx2_stra->get_percentage(), 0.4f);
-    EXPECT_EQ(mtx2_stra->get_ratio(), 0.1f);
+    EXPECT_EQ(mtx_stra->get_percentage(), 0.4);
+    EXPECT_EQ(mtx_stra->get_ratio(), 0.1);
+    EXPECT_EQ(mtx2_stra->get_percentage(), 0.4);
+    EXPECT_EQ(mtx2_stra->get_ratio(), 0.1);
 }
 
 
diff --git a/include/ginkgo/core/base/abstract_factory.hpp b/include/ginkgo/core/base/abstract_factory.hpp
index ff185509224..3e89c770128 100644
--- a/include/ginkgo/core/base/abstract_factory.hpp
+++ b/include/ginkgo/core/base/abstract_factory.hpp
@@ -266,7 +266,8 @@ class EnableDefaultFactory
  *                  used
  */
 template <typename ConcreteParametersType, typename Factory>
-struct enable_parameters_type {
+class enable_parameters_type {
+public:
     using factory = Factory;
 
     /**
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index 1512802edb6..5c2c9a34fa1 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -916,8 +916,8 @@ public:                                                                      \
                                                   _parameters_name##_type> { \
         friend class ::gko::EnablePolymorphicObject<_factory_name,           \
                                                     ::gko::LinOpFactory>;    \
-        friend struct ::gko::enable_parameters_type<_parameters_name##_type, \
-                                                    _factory_name>;          \
+        friend class ::gko::enable_parameters_type<_parameters_name##_type,  \
+                                                   _factory_name>;           \
         explicit _factory_name(std::shared_ptr<const ::gko::Executor> exec)  \
             : ::gko::EnableDefaultLinOpFactory<_factory_name, _lin_op,       \
                                                _parameters_name##_type>(     \
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index 9ba5ed4d343..bb54cabdd7a 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -239,9 +239,9 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
             }
             auto num_rows = mtx_row_ptrs.get_num_elems() - 1;
             max_length_per_row_ = 0;
-            for (index_type i = 1; i < num_rows + 1; i++) {
+            for (size_type i = 0; i < num_rows; i++) {
                 max_length_per_row_ = std::max(max_length_per_row_,
-                                               row_ptrs[i] - row_ptrs[i - 1]);
+                                               row_ptrs[i + 1] - row_ptrs[i]);
             }
         }
 
@@ -461,7 +461,7 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
 #endif  // GINKGO_HIP_PLATFORM_HCC
 
                 auto nwarps = nwarps_ * multiple;
-                return min(ceildiv(nnz, warp_size_), int64_t(nwarps));
+                return min(ceildiv(nnz, warp_size_), nwarps);
             } else {
                 return 0;
             }
@@ -577,8 +577,8 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
                 this->set_name(actual_strategy.get_name());
             } else {
                 index_type maxnum = 0;
-                for (index_type i = 1; i < num_rows + 1; i++) {
-                    maxnum = std::max(maxnum, row_ptrs[i] - row_ptrs[i - 1]);
+                for (size_type i = 0; i < num_rows; i++) {
+                    maxnum = std::max(maxnum, row_ptrs[i + 1] - row_ptrs[i]);
                 }
                 if (maxnum > row_len_limit) {
                     load_balance actual_strategy(nwarps_, warp_size_,
diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp
index 4641a0443ec..93174e45652 100644
--- a/include/ginkgo/core/matrix/hybrid.hpp
+++ b/include/ginkgo/core/matrix/hybrid.hpp
@@ -245,10 +245,10 @@ class Hybrid
          * @param percent  the row_nnz[floor(num_rows*percent)] is the number of
          *                 stored elements per row of the ell part
          */
-        explicit imbalance_limit(float percent = 0.8) : percent_(percent)
+        explicit imbalance_limit(double percent = 0.8) : percent_(percent)
         {
-            percent_ = std::min(percent_, 1.0f);
-            percent_ = std::max(percent_, 0.0f);
+            percent_ = std::min(percent_, 1.0);
+            percent_ = std::max(percent_, 0.0);
         }
 
         size_type compute_ell_num_stored_elements_per_row(
@@ -276,7 +276,7 @@ class Hybrid
         auto get_percentage() const { return percent_; }
 
     private:
-        float percent_;
+        double percent_;
     };
 
     /**
@@ -289,7 +289,7 @@ class Hybrid
         /**
          * Creates a imbalance_bounded_limit strategy.
          */
-        imbalance_bounded_limit(float percent = 0.8, float ratio = 0.0001)
+        imbalance_bounded_limit(double percent = 0.8, double ratio = 0.0001)
             : strategy_(imbalance_limit(percent)), ratio_(ratio)
         {}
 
@@ -319,7 +319,7 @@ class Hybrid
 
     private:
         imbalance_limit strategy_;
-        float ratio_;
+        double ratio_;
     };
 
 
diff --git a/include/ginkgo/core/preconditioner/jacobi.hpp b/include/ginkgo/core/preconditioner/jacobi.hpp
index 4828609d077..44d1ca4a1a8 100644
--- a/include/ginkgo/core/preconditioner/jacobi.hpp
+++ b/include/ginkgo/core/preconditioner/jacobi.hpp
@@ -484,7 +484,8 @@ class Jacobi : public EnableLinOp<Jacobi<ValueType, IndexType>>,
          * accuracy to a value as close as possible to `dropout` will result in
          * optimal memory savings, while not degrading the quality of solution.
          */
-        remove_complex<value_type> GKO_FACTORY_PARAMETER_SCALAR(accuracy, 1e-1);
+        remove_complex<value_type> GKO_FACTORY_PARAMETER_SCALAR(
+            accuracy, static_cast<remove_complex<value_type>>(1e-1));
     };
     GKO_ENABLE_LIN_OP_FACTORY(Jacobi, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);
diff --git a/include/ginkgo/core/stop/criterion.hpp b/include/ginkgo/core/stop/criterion.hpp
index e309df9bc42..69b92693848 100644
--- a/include/ginkgo/core/stop/criterion.hpp
+++ b/include/ginkgo/core/stop/criterion.hpp
@@ -297,8 +297,8 @@ public:                                                                      \
               _factory_name, _criterion, _parameters_name##_type> {          \
         friend class ::gko::EnablePolymorphicObject<                         \
             _factory_name, ::gko::stop::CriterionFactory>;                   \
-        friend struct ::gko::enable_parameters_type<_parameters_name##_type, \
-                                                    _factory_name>;          \
+        friend class ::gko::enable_parameters_type<_parameters_name##_type,  \
+                                                   _factory_name>;           \
         explicit _factory_name(std::shared_ptr<const ::gko::Executor> exec)  \
             : ::gko::stop::EnableDefaultCriterionFactory<                    \
                   _factory_name, _criterion, _parameters_name##_type>(       \
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index f786a84056b..c145210e89d 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -121,8 +121,8 @@ class ResidualNormReduction : public ResidualNorm<ValueType> {
         /**
          * Factor by which the residual norm will be reduced
          */
-        remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(reduction_factor,
-                                                               1e-15);
+        remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(
+            reduction_factor, static_cast<remove_complex<ValueType>>(1e-15));
     };
     GKO_ENABLE_CRITERION_FACTORY(ResidualNormReduction<ValueType>, parameters,
                                  Factory);
@@ -187,8 +187,8 @@ class RelativeResidualNorm : public ResidualNorm<ValueType> {
         /**
          * Relative residual norm goal
          */
-        remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(tolerance,
-                                                               1e-15);
+        remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(
+            tolerance, static_cast<remove_complex<ValueType>>(1e-15));
     };
     GKO_ENABLE_CRITERION_FACTORY(RelativeResidualNorm<ValueType>, parameters,
                                  Factory);
@@ -251,8 +251,8 @@ class AbsoluteResidualNorm : public ResidualNorm<ValueType> {
         /**
          * Absolute residual norm goal
          */
-        remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(tolerance,
-                                                               1e-15);
+        remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(
+            tolerance, static_cast<remove_complex<ValueType>>(1e-15));
     };
     GKO_ENABLE_CRITERION_FACTORY(AbsoluteResidualNorm<ValueType>, parameters,
                                  Factory);
diff --git a/omp/components/prefix_sum.cpp b/omp/components/prefix_sum.cpp
index 1375c1643dc..df307abd09c 100644
--- a/omp/components/prefix_sum.cpp
+++ b/omp/components/prefix_sum.cpp
@@ -44,7 +44,7 @@ void prefix_sum(std::shared_ptr<const OmpExecutor> exec, IndexType *counts,
                 size_type num_entries)
 {
     IndexType partial_sum{};
-    for (IndexType i = 0; i < num_entries; ++i) {
+    for (size_type i = 0; i < num_entries; ++i) {
         auto nnz = counts[i];
         counts[i] = partial_sum;
         partial_sum += nnz;
diff --git a/omp/solver/lower_trs_kernels.cpp b/omp/solver/lower_trs_kernels.cpp
index af6a0670ea0..b30a23a69d6 100644
--- a/omp/solver/lower_trs_kernels.cpp
+++ b/omp/solver/lower_trs_kernels.cpp
@@ -107,7 +107,7 @@ void solve(std::shared_ptr<const OmpExecutor> exec,
     for (size_type j = 0; j < b->get_size()[1]; ++j) {
         for (size_type row = 0; row < matrix->get_size()[0]; ++row) {
             x->at(row, j) = b->at(row, j) / vals[row_ptrs[row + 1] - 1];
-            for (size_type k = row_ptrs[row]; k < row_ptrs[row + 1]; ++k) {
+            for (auto k = row_ptrs[row]; k < row_ptrs[row + 1]; ++k) {
                 auto col = col_idxs[k];
                 if (col < row) {
                     x->at(row, j) +=
diff --git a/omp/solver/upper_trs_kernels.cpp b/omp/solver/upper_trs_kernels.cpp
index 2fa0b6a3db9..15d982998e0 100644
--- a/omp/solver/upper_trs_kernels.cpp
+++ b/omp/solver/upper_trs_kernels.cpp
@@ -104,10 +104,12 @@ void solve(std::shared_ptr<const OmpExecutor> exec,
     auto vals = matrix->get_const_values();
 
 #pragma omp parallel for
-    for (int j = 0; j < b->get_size()[1]; ++j) {
-        for (int row = matrix->get_size()[0] - 1; row >= 0; --row) {
+    for (size_type j = 0; j < b->get_size()[1]; ++j) {
+        for (size_type inv_row = 0; inv_row < matrix->get_size()[0];
+             ++inv_row) {
+            auto row = matrix->get_size()[0] - 1 - inv_row;
             x->at(row, j) = b->at(row, j) / vals[row_ptrs[row]];
-            for (int k = row_ptrs[row]; k < row_ptrs[row + 1]; ++k) {
+            for (auto k = row_ptrs[row]; k < row_ptrs[row + 1]; ++k) {
                 auto col = col_idxs[k];
                 if (col > row) {
                     x->at(row, j) +=
diff --git a/reference/components/csr_spgeam.hpp b/reference/components/csr_spgeam.hpp
index 3f313df5756..84ebbaecccc 100644
--- a/reference/components/csr_spgeam.hpp
+++ b/reference/components/csr_spgeam.hpp
@@ -77,7 +77,7 @@ void abstract_spgeam(const matrix::Csr<ValueType, IndexType> *a,
         auto b_end = b_row_ptrs[row + 1];
         auto total_size = (a_end - a_begin) + (b_end - b_begin);
         bool skip{};
-        auto local_data = begin_cb(row);
+        auto local_data = begin_cb(static_cast<IndexType>(row));
         for (IndexType i = 0; i < total_size; ++i) {
             if (skip) {
                 skip = false;
diff --git a/reference/components/prefix_sum.cpp b/reference/components/prefix_sum.cpp
index 2530e960a68..003491ad5d9 100644
--- a/reference/components/prefix_sum.cpp
+++ b/reference/components/prefix_sum.cpp
@@ -44,7 +44,7 @@ void prefix_sum(std::shared_ptr<const ReferenceExecutor> exec,
                 IndexType *counts, size_type num_entries)
 {
     IndexType partial_sum{};
-    for (IndexType i = 0; i < num_entries; ++i) {
+    for (size_type i = 0; i < num_entries; ++i) {
         auto nnz = counts[i];
         counts[i] = partial_sum;
         partial_sum += nnz;
diff --git a/reference/factorization/factorization_kernels.cpp b/reference/factorization/factorization_kernels.cpp
index feb64a56b40..6155208f41b 100644
--- a/reference/factorization/factorization_kernels.cpp
+++ b/reference/factorization/factorization_kernels.cpp
@@ -150,7 +150,7 @@ void add_diagonal_elements(std::shared_ptr<const ReferenceExecutor> exec,
             ++added_elements;
         }
     }
-    row_ptrs[num_rows] = new_nnz;
+    row_ptrs[num_rows] = static_cast<IndexType>(new_nnz);
 
     matrix::CsrBuilder<ValueType, IndexType> mtx_builder{mtx};
     mtx_builder.get_value_array() = std::move(new_values_array);
@@ -169,14 +169,14 @@ void initialize_row_ptrs_l_u(
 {
     auto row_ptrs = system_matrix->get_const_row_ptrs();
     auto col_idxs = system_matrix->get_const_col_idxs();
-    size_type l_nnz{};
-    size_type u_nnz{};
+    IndexType l_nnz{};
+    IndexType u_nnz{};
 
     l_row_ptrs[0] = 0;
     u_row_ptrs[0] = 0;
     for (size_type row = 0; row < system_matrix->get_size()[0]; ++row) {
-        for (size_type el = row_ptrs[row]; el < row_ptrs[row + 1]; ++el) {
-            size_type col = col_idxs[el];
+        for (auto el = row_ptrs[row]; el < row_ptrs[row + 1]; ++el) {
+            auto col = col_idxs[el];
             // don't count diagonal
             l_nnz += col < row;
             u_nnz += col > row;
diff --git a/reference/factorization/par_ict_kernels.cpp b/reference/factorization/par_ict_kernels.cpp
index 8114d22b493..b360709a628 100644
--- a/reference/factorization/par_ict_kernels.cpp
+++ b/reference/factorization/par_ict_kernels.cpp
@@ -77,8 +77,7 @@ void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
     auto a_vals = a->get_const_values();
 
     for (size_type row = 0; row < num_rows; ++row) {
-        for (size_type l_nz = l_row_ptrs[row]; l_nz < l_row_ptrs[row + 1];
-             ++l_nz) {
+        for (auto l_nz = l_row_ptrs[row]; l_nz < l_row_ptrs[row + 1]; ++l_nz) {
             auto col = l_col_idxs[l_nz];
             // find value from A
             auto a_begin = a_row_ptrs[row];
diff --git a/reference/matrix/csr_kernels.cpp b/reference/matrix/csr_kernels.cpp
index b22d7e3d66e..9cb739a82b9 100644
--- a/reference/matrix/csr_kernels.cpp
+++ b/reference/matrix/csr_kernels.cpp
@@ -218,7 +218,7 @@ void spgemm(std::shared_ptr<const ReferenceExecutor> exec,
     for (size_type a_row = 0; a_row < num_rows; ++a_row) {
         local_col_idxs.clear();
         spgemm_insert_row2(local_col_idxs, a, b, a_row);
-        c_row_ptrs[a_row] = local_col_idxs.size();
+        c_row_ptrs[a_row] = static_cast<IndexType>(local_col_idxs.size());
     }
 
     // build row pointers
@@ -272,7 +272,7 @@ void advanced_spgemm(std::shared_ptr<const ReferenceExecutor> exec,
         local_col_idxs.clear();
         spgemm_insert_row(local_col_idxs, d, a_row);
         spgemm_insert_row2(local_col_idxs, a, b, a_row);
-        c_row_ptrs[a_row] = local_col_idxs.size();
+        c_row_ptrs[a_row] = static_cast<IndexType>(local_col_idxs.size());
     }
 
     // build row pointers
@@ -429,7 +429,7 @@ void convert_to_sellp(std::shared_ptr<const ReferenceExecutor> exec,
     const auto source_col_idxs = source->get_const_col_idxs();
     const auto source_values = source->get_const_values();
 
-    int slice_num = ceildiv(num_rows, slice_size);
+    auto slice_num = ceildiv(num_rows, slice_size);
     slice_sets[0] = 0;
     for (size_type slice = 0; slice < slice_num; slice++) {
         if (slice > 0) {
diff --git a/reference/matrix/dense_kernels.cpp b/reference/matrix/dense_kernels.cpp
index 4cb84202605..ab020a84a3d 100644
--- a/reference/matrix/dense_kernels.cpp
+++ b/reference/matrix/dense_kernels.cpp
@@ -383,7 +383,7 @@ void convert_to_sellp(std::shared_ptr<const ReferenceExecutor> exec,
     auto stride_factor = (result->get_stride_factor() == 0)
                              ? matrix::default_stride_factor
                              : result->get_stride_factor();
-    int slice_num = ceildiv(num_rows, slice_size);
+    auto slice_num = ceildiv(num_rows, slice_size);
     slice_sets[0] = 0;
     for (size_type slice = 0; slice < slice_num; slice++) {
         if (slice > 0) {
diff --git a/reference/preconditioner/jacobi_kernels.cpp b/reference/preconditioner/jacobi_kernels.cpp
index d72065452e0..a547e6405f6 100644
--- a/reference/preconditioner/jacobi_kernels.cpp
+++ b/reference/preconditioner/jacobi_kernels.cpp
@@ -116,9 +116,9 @@ inline size_type agglomerate_supervariables(uint32 max_block_size,
         return 0;
     }
     size_type num_blocks = 1;
-    int32 current_block_size = block_ptrs[1] - block_ptrs[0];
+    auto current_block_size = block_ptrs[1] - block_ptrs[0];
     for (size_type i = 1; i < num_natural_blocks; ++i) {
-        const int32 block_size = block_ptrs[i + 1] - block_ptrs[i];
+        const auto block_size = block_ptrs[i + 1] - block_ptrs[i];
         if (current_block_size + block_size <= max_block_size) {
             current_block_size += block_size;
         } else {
@@ -170,7 +170,7 @@ inline void extract_block(const matrix::Csr<ValueType, IndexType> *mtx,
     for (int row = 0; row < block_size; ++row) {
         const auto start = row_ptrs[block_start + row];
         const auto end = row_ptrs[block_start + row + 1];
-        for (int i = start; i < end; ++i) {
+        for (auto i = start; i < end; ++i) {
             const auto col = col_idxs[i] - block_start;
             if (0 <= col && col < block_size) {
                 block[row * stride + col] = vals[i];
@@ -359,7 +359,7 @@ void generate(std::shared_ptr<const ReferenceExecutor> exec,
         vector<Array<IndexType>> perm(group_size, {}, exec);
         vector<uint32> pr_descriptors(group_size, uint32{} - 1, exec);
         // extract group of blocks, invert them, figure out storage precision
-        for (size_type b = 0; b < group_size; ++b) {
+        for (IndexType b = 0; b < group_size; ++b) {
             if (b + g >= num_blocks) {
                 break;
             }
@@ -413,7 +413,7 @@ void generate(std::shared_ptr<const ReferenceExecutor> exec,
                             [](uint32 x, uint32 y) { return x & y; }));
 
         // store the blocks
-        for (size_type b = 0; b < group_size; ++b) {
+        for (IndexType b = 0; b < group_size; ++b) {
             if (b + g >= num_blocks) {
                 break;
             }
diff --git a/reference/solver/lower_trs_kernels.cpp b/reference/solver/lower_trs_kernels.cpp
index b1678d06f83..daa62716f1b 100644
--- a/reference/solver/lower_trs_kernels.cpp
+++ b/reference/solver/lower_trs_kernels.cpp
@@ -103,7 +103,7 @@ void solve(std::shared_ptr<const ReferenceExecutor> exec,
     for (size_type j = 0; j < b->get_size()[1]; ++j) {
         for (size_type row = 0; row < matrix->get_size()[0]; ++row) {
             x->at(row, j) = b->at(row, j) / vals[row_ptrs[row + 1] - 1];
-            for (size_type k = row_ptrs[row]; k < row_ptrs[row + 1]; ++k) {
+            for (auto k = row_ptrs[row]; k < row_ptrs[row + 1]; ++k) {
                 auto col = col_idxs[k];
                 if (col < row) {
                     x->at(row, j) +=
diff --git a/reference/solver/upper_trs_kernels.cpp b/reference/solver/upper_trs_kernels.cpp
index 02f3666ecf4..2cb41ed397f 100644
--- a/reference/solver/upper_trs_kernels.cpp
+++ b/reference/solver/upper_trs_kernels.cpp
@@ -100,10 +100,12 @@ void solve(std::shared_ptr<const ReferenceExecutor> exec,
     auto col_idxs = matrix->get_const_col_idxs();
     auto vals = matrix->get_const_values();
 
-    for (int j = 0; j < b->get_size()[1]; ++j) {
-        for (int row = matrix->get_size()[0] - 1; row >= 0; --row) {
+    for (size_type j = 0; j < b->get_size()[1]; ++j) {
+        for (size_type inv_row = 0; inv_row < matrix->get_size()[0];
+             ++inv_row) {
+            auto row = matrix->get_size()[0] - 1 - inv_row;
             x->at(row, j) = b->at(row, j) / vals[row_ptrs[row]];
-            for (int k = row_ptrs[row]; k < row_ptrs[row + 1]; ++k) {
+            for (auto k = row_ptrs[row]; k < row_ptrs[row + 1]; ++k) {
                 auto col = col_idxs[k];
                 if (col > row) {
                     x->at(row, j) +=

From fc7c5f7e970c3c5e6985cfd7b929d34fa74cb9fa Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 30 Dec 2020 23:51:04 +0100
Subject: [PATCH 0186/1008] add symmetric permutation kernels

---
 common/matrix/csr_kernels.hpp.inc        | 26 +++++++
 common/matrix/dense_kernels.hpp.inc      | 66 +++++++++++-----
 core/device_hooks/common_kernels.inc.cpp | 18 +++++
 core/matrix/csr.cpp                      | 63 ++++++++++++---
 core/matrix/csr_kernels.hpp              |  8 ++
 core/matrix/dense.cpp                    | 98 ++++++++++++++++++++----
 core/matrix/dense_kernels.hpp            | 16 ++++
 cuda/matrix/csr_kernels.cu               | 25 ++++++
 cuda/matrix/dense_kernels.cu             | 79 +++++++++++++------
 cuda/test/matrix/csr_kernels.cpp         | 26 +++++++
 cuda/test/matrix/dense_kernels.cpp       | 29 +++++++
 dpcpp/matrix/csr_kernels.dp.cpp          | 10 +++
 dpcpp/matrix/dense_kernels.dp.cpp        | 20 +++++
 hip/matrix/csr_kernels.hip.cpp           | 27 +++++++
 hip/matrix/dense_kernels.hip.cpp         | 92 +++++++++++++++-------
 hip/test/matrix/csr_kernels.hip.cpp      | 26 +++++++
 hip/test/matrix/dense_kernels.hip.cpp    | 29 +++++++
 include/ginkgo/core/base/lin_op.hpp      | 48 ++++++++++--
 include/ginkgo/core/matrix/csr.hpp       |  6 ++
 include/ginkgo/core/matrix/dense.hpp     | 20 ++++-
 omp/matrix/csr_kernels.cpp               | 39 ++++++++++
 omp/matrix/dense_kernels.cpp             | 40 ++++++++++
 omp/test/matrix/csr_kernels.cpp          | 26 +++++++
 omp/test/matrix/dense_kernels.cpp        | 29 +++++++
 reference/matrix/csr_kernels.cpp         | 37 +++++++++
 reference/matrix/dense_kernels.cpp       | 38 +++++++++
 reference/test/matrix/csr_kernels.cpp    | 40 ++++++++++
 reference/test/matrix/dense_kernels.cpp  | 80 +++++++++++++++++++
 28 files changed, 948 insertions(+), 113 deletions(-)

diff --git a/common/matrix/csr_kernels.hpp.inc b/common/matrix/csr_kernels.hpp.inc
index 8f7e1a3f5a2..edd6014154b 100644
--- a/common/matrix/csr_kernels.hpp.inc
+++ b/common/matrix/csr_kernels.hpp.inc
@@ -1062,4 +1062,30 @@ __global__ __launch_bounds__(default_block_size) void inv_row_permute_kernel(
         out_cols[out_begin + i] = in_cols[in_begin + i];
         out_vals[out_begin + i] = in_vals[in_begin + i];
     }
+}
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+__global__ __launch_bounds__(default_block_size) void inv_symm_permute_kernel(
+    size_type num_rows, const IndexType *__restrict__ permutation,
+    const IndexType *__restrict__ in_row_ptrs,
+    const IndexType *__restrict__ in_cols,
+    const ValueType *__restrict__ in_vals,
+    const IndexType *__restrict__ out_row_ptrs,
+    IndexType *__restrict__ out_cols, ValueType *__restrict__ out_vals)
+{
+    auto tid = thread::get_subwarp_id_flat<subwarp_size>();
+    if (tid >= num_rows) {
+        return;
+    }
+    auto lane = threadIdx.x % subwarp_size;
+    auto in_row = tid;
+    auto out_row = permutation[tid];
+    auto in_begin = in_row_ptrs[in_row];
+    auto in_size = in_row_ptrs[in_row + 1] - in_begin;
+    auto out_begin = out_row_ptrs[out_row];
+    for (IndexType i = lane; i < in_size; i += subwarp_size) {
+        out_cols[out_begin + i] = permutation[in_cols[in_begin + i]];
+        out_vals[out_begin + i] = in_vals[in_begin + i];
+    }
 }
\ No newline at end of file
diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index 747356e4594..9df9caf0353 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -421,16 +421,48 @@ __global__ __launch_bounds__(default_block_size) void reduce_total_cols(
 }
 
 
-template <size_type block_size, typename IndexType, typename ValueType>
-__global__ __launch_bounds__(block_size) void row_gather(
+template <typename IndexType, typename ValueType>
+__global__ __launch_bounds__(default_block_size) void symm_permute(
     size_type num_rows, size_type num_cols,
     const IndexType *__restrict__ perm_idxs, const ValueType *__restrict__ orig,
     size_type stride_orig, ValueType *__restrict__ result,
     size_type stride_result)
 {
-    constexpr auto warps_per_block = block_size / config::warp_size;
-    const auto global_id =
-        thread::get_thread_id<config::warp_size, warps_per_block>();
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row_id = global_id / num_cols;
+    const auto col_id = global_id % num_cols;
+    if (row_id < num_rows) {
+        result[row_id * stride_result + col_id] =
+            orig[perm_idxs[row_id] * stride_orig + perm_idxs[col_id]];
+    }
+}
+
+
+template <typename IndexType, typename ValueType>
+__global__ __launch_bounds__(default_block_size) void inv_symm_permute(
+    size_type num_rows, size_type num_cols,
+    const IndexType *__restrict__ perm_idxs, const ValueType *__restrict__ orig,
+    size_type stride_orig, ValueType *__restrict__ result,
+    size_type stride_result)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row_id = global_id / num_cols;
+    const auto col_id = global_id % num_cols;
+    if (row_id < num_rows) {
+        result[perm_idxs[row_id] * stride_result + perm_idxs[col_id]] =
+            orig[row_id * stride_orig + col_id];
+    }
+}
+
+
+template <typename IndexType, typename ValueType>
+__global__ __launch_bounds__(default_block_size) void row_gather(
+    size_type num_rows, size_type num_cols,
+    const IndexType *__restrict__ perm_idxs, const ValueType *__restrict__ orig,
+    size_type stride_orig, ValueType *__restrict__ result,
+    size_type stride_result)
+{
+    const auto global_id = thread::get_thread_id_flat();
     const auto row_id = global_id / num_cols;
     const auto col_id = global_id % num_cols;
     if (row_id < num_rows) {
@@ -440,16 +472,14 @@ __global__ __launch_bounds__(block_size) void row_gather(
 }
 
 
-template <size_type block_size, typename IndexType, typename ValueType>
-__global__ __launch_bounds__(block_size) void column_permute(
+template <typename IndexType, typename ValueType>
+__global__ __launch_bounds__(default_block_size) void column_permute(
     size_type num_rows, size_type num_cols,
     const IndexType *__restrict__ perm_idxs, const ValueType *__restrict__ orig,
     size_type stride_orig, ValueType *__restrict__ result,
     size_type stride_result)
 {
-    constexpr auto warps_per_block = block_size / config::warp_size;
-    const auto global_id =
-        thread::get_thread_id<config::warp_size, warps_per_block>();
+    const auto global_id = thread::get_thread_id_flat();
     const auto row_id = global_id / num_cols;
     const auto col_id = global_id % num_cols;
     if (row_id < num_rows) {
@@ -459,16 +489,14 @@ __global__ __launch_bounds__(block_size) void column_permute(
 }
 
 
-template <size_type block_size, typename IndexType, typename ValueType>
-__global__ __launch_bounds__(block_size) void inverse_row_permute(
+template <typename IndexType, typename ValueType>
+__global__ __launch_bounds__(default_block_size) void inverse_row_permute(
     size_type num_rows, size_type num_cols,
     const IndexType *__restrict__ perm_idxs, const ValueType *__restrict__ orig,
     size_type stride_orig, ValueType *__restrict__ result,
     size_type stride_result)
 {
-    constexpr auto warps_per_block = block_size / config::warp_size;
-    const auto global_id =
-        thread::get_thread_id<config::warp_size, warps_per_block>();
+    const auto global_id = thread::get_thread_id_flat();
     const auto row_id = global_id / num_cols;
     const auto col_id = global_id % num_cols;
     if (row_id < num_rows) {
@@ -478,16 +506,14 @@ __global__ __launch_bounds__(block_size) void inverse_row_permute(
 }
 
 
-template <size_type block_size, typename IndexType, typename ValueType>
-__global__ __launch_bounds__(block_size) void inverse_column_permute(
+template <typename IndexType, typename ValueType>
+__global__ __launch_bounds__(default_block_size) void inverse_column_permute(
     size_type num_rows, size_type num_cols,
     const IndexType *__restrict__ perm_idxs, const ValueType *__restrict__ orig,
     size_type stride_orig, ValueType *__restrict__ result,
     size_type stride_result)
 {
-    constexpr auto warps_per_block = block_size / config::warp_size;
-    const auto global_id =
-        thread::get_thread_id<config::warp_size, warps_per_block>();
+    const auto global_id = thread::get_thread_id_flat();
     const auto row_id = global_id / num_cols;
     const auto col_id = global_id % num_cols;
     if (row_id < num_rows) {
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index da34e4bd11a..60ca0316f5e 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -218,6 +218,18 @@ GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL);
+
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_DENSE_ROW_GATHER_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
@@ -688,6 +700,12 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONJ_TRANSPOSE_KERNEL);
 
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_CSR_INV_SYMM_PERMUTE_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_INV_SYMM_PERMUTE_KERNEL);
+
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp
index a9a40613e78..fc87cdaf561 100644
--- a/core/matrix/csr.cpp
+++ b/core/matrix/csr.cpp
@@ -69,6 +69,7 @@ GKO_REGISTER_OPERATION(convert_to_ell, csr::convert_to_ell);
 GKO_REGISTER_OPERATION(convert_to_hybrid, csr::convert_to_hybrid);
 GKO_REGISTER_OPERATION(transpose, csr::transpose);
 GKO_REGISTER_OPERATION(conj_transpose, csr::conj_transpose);
+GKO_REGISTER_OPERATION(inv_symm_permute, csr::inv_symm_permute);
 GKO_REGISTER_OPERATION(row_permute, csr::row_permute);
 GKO_REGISTER_OPERATION(inverse_row_permute, csr::inverse_row_permute);
 GKO_REGISTER_OPERATION(inverse_column_permute, csr::inverse_column_permute);
@@ -402,6 +403,48 @@ std::unique_ptr<LinOp> Csr<ValueType, IndexType>::conj_transpose() const
 }
 
 
+template <typename ValueType, typename IndexType>
+std::unique_ptr<LinOp> Csr<ValueType, IndexType>::permute(
+    const Array<IndexType> *permutation_indices) const
+{
+    GKO_ASSERT_IS_SQUARE_MATRIX(this);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+    auto exec = this->get_executor();
+    auto permute_cpy =
+        Csr::create(exec, this->get_size(), this->get_num_stored_elements(),
+                    this->get_strategy());
+    Array<IndexType> inv_permutation(exec, this->get_size()[1]);
+
+    exec->run(csr::make_invert_permutation(
+        this->get_size()[1],
+        make_temporary_clone(exec, permutation_indices)->get_const_data(),
+        inv_permutation.get_data()));
+    exec->run(csr::make_inv_symm_permute(inv_permutation.get_const_data(), this,
+                                         permute_cpy.get()));
+    permute_cpy->make_srow();
+    return std::move(permute_cpy);
+}
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<LinOp> Csr<ValueType, IndexType>::inverse_permute(
+    const Array<IndexType> *permutation_indices) const
+{
+    GKO_ASSERT_IS_SQUARE_MATRIX(this);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+    auto exec = this->get_executor();
+    auto permute_cpy =
+        Csr::create(exec, this->get_size(), this->get_num_stored_elements(),
+                    this->get_strategy());
+
+    exec->run(csr::make_inv_symm_permute(
+        make_temporary_clone(exec, permutation_indices)->get_const_data(), this,
+        permute_cpy.get()));
+    permute_cpy->make_srow();
+    return std::move(permute_cpy);
+}
+
+
 template <typename ValueType, typename IndexType>
 std::unique_ptr<LinOp> Csr<ValueType, IndexType>::row_permute(
     const Array<IndexType> *permutation_indices) const
@@ -445,19 +488,17 @@ std::unique_ptr<LinOp> Csr<ValueType, IndexType>::column_permute(
 
 template <typename ValueType, typename IndexType>
 std::unique_ptr<LinOp> Csr<ValueType, IndexType>::inverse_row_permute(
-    const Array<IndexType> *inverse_permutation_indices) const
+    const Array<IndexType> *permutation_indices) const
 {
-    GKO_ASSERT_EQ(inverse_permutation_indices->get_num_elems(),
-                  this->get_size()[0]);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
     auto exec = this->get_executor();
     auto inverse_permute_cpy =
         Csr::create(exec, this->get_size(), this->get_num_stored_elements(),
                     this->get_strategy());
 
     exec->run(csr::make_inverse_row_permute(
-        make_temporary_clone(exec, inverse_permutation_indices)
-            ->get_const_data(),
-        this, inverse_permute_cpy.get()));
+        make_temporary_clone(exec, permutation_indices)->get_const_data(), this,
+        inverse_permute_cpy.get()));
     inverse_permute_cpy->make_srow();
     return std::move(inverse_permute_cpy);
 }
@@ -465,19 +506,17 @@ std::unique_ptr<LinOp> Csr<ValueType, IndexType>::inverse_row_permute(
 
 template <typename ValueType, typename IndexType>
 std::unique_ptr<LinOp> Csr<ValueType, IndexType>::inverse_column_permute(
-    const Array<IndexType> *inverse_permutation_indices) const
+    const Array<IndexType> *permutation_indices) const
 {
-    GKO_ASSERT_EQ(inverse_permutation_indices->get_num_elems(),
-                  this->get_size()[1]);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
     auto exec = this->get_executor();
     auto inverse_permute_cpy =
         Csr::create(exec, this->get_size(), this->get_num_stored_elements(),
                     this->get_strategy());
 
     exec->run(csr::make_inverse_column_permute(
-        make_temporary_clone(exec, inverse_permutation_indices)
-            ->get_const_data(),
-        this, inverse_permute_cpy.get()));
+        make_temporary_clone(exec, permutation_indices)->get_const_data(), this,
+        inverse_permute_cpy.get()));
     inverse_permute_cpy->make_srow();
     inverse_permute_cpy->sort_by_column_index();
     return std::move(inverse_permute_cpy);
diff --git a/core/matrix/csr_kernels.hpp b/core/matrix/csr_kernels.hpp
index 6f7b72312d9..3679c353fb0 100644
--- a/core/matrix/csr_kernels.hpp
+++ b/core/matrix/csr_kernels.hpp
@@ -129,6 +129,12 @@ namespace kernels {
                         const matrix::Csr<ValueType, IndexType> *orig, \
                         matrix::Csr<ValueType, IndexType> *trans)
 
+#define GKO_DECLARE_CSR_INV_SYMM_PERMUTE_KERNEL(ValueType, IndexType)    \
+    void inv_symm_permute(std::shared_ptr<const DefaultExecutor> exec,   \
+                          const IndexType *permutation_indices,          \
+                          const matrix::Csr<ValueType, IndexType> *orig, \
+                          matrix::Csr<ValueType, IndexType> *permuted)
+
 #define GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL(ValueType, IndexType)    \
     void row_permute(std::shared_ptr<const DefaultExecutor> exec,   \
                      const IndexType *permutation_indices,          \
@@ -207,6 +213,8 @@ namespace kernels {
     template <typename ValueType, typename IndexType>                        \
     GKO_DECLARE_CSR_CONJ_TRANSPOSE_KERNEL(ValueType, IndexType);             \
     template <typename ValueType, typename IndexType>                        \
+    GKO_DECLARE_CSR_INV_SYMM_PERMUTE_KERNEL(ValueType, IndexType);           \
+    template <typename ValueType, typename IndexType>                        \
     GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL(ValueType, IndexType);                \
     template <typename ValueType, typename IndexType>                        \
     GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType);        \
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index a9e6a3a1515..0d70016ab6d 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -75,6 +75,8 @@ GKO_REGISTER_OPERATION(calculate_nonzeros_per_row,
 GKO_REGISTER_OPERATION(calculate_total_cols, dense::calculate_total_cols);
 GKO_REGISTER_OPERATION(transpose, dense::transpose);
 GKO_REGISTER_OPERATION(conj_transpose, dense::conj_transpose);
+GKO_REGISTER_OPERATION(symm_permute, dense::symm_permute);
+GKO_REGISTER_OPERATION(inv_symm_permute, dense::inv_symm_permute);
 GKO_REGISTER_OPERATION(row_gather, dense::row_gather);
 GKO_REGISTER_OPERATION(column_permute, dense::column_permute);
 GKO_REGISTER_OPERATION(inverse_row_permute, dense::inverse_row_permute);
@@ -638,6 +640,74 @@ std::unique_ptr<LinOp> Dense<ValueType>::conj_transpose() const
 }
 
 
+template <typename ValueType>
+std::unique_ptr<LinOp> Dense<ValueType>::permute(
+    const Array<int32> *permutation_indices) const
+{
+    GKO_ASSERT_IS_SQUARE_MATRIX(this);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+    auto exec = this->get_executor();
+    auto permute_cpy = Dense::create(exec, this->get_size());
+
+    exec->run(dense::make_symm_permute(
+        make_temporary_clone(exec, permutation_indices).get(), this,
+        permute_cpy.get()));
+
+    return std::move(permute_cpy);
+}
+
+
+template <typename ValueType>
+std::unique_ptr<LinOp> Dense<ValueType>::permute(
+    const Array<int64> *permutation_indices) const
+{
+    GKO_ASSERT_IS_SQUARE_MATRIX(this);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+    auto exec = this->get_executor();
+    auto permute_cpy = Dense::create(exec, this->get_size());
+
+    exec->run(dense::make_symm_permute(
+        make_temporary_clone(exec, permutation_indices).get(), this,
+        permute_cpy.get()));
+
+    return std::move(permute_cpy);
+}
+
+
+template <typename ValueType>
+std::unique_ptr<LinOp> Dense<ValueType>::inverse_permute(
+    const Array<int32> *permutation_indices) const
+{
+    GKO_ASSERT_IS_SQUARE_MATRIX(this);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+    auto exec = this->get_executor();
+    auto permute_cpy = Dense::create(exec, this->get_size());
+
+    exec->run(dense::make_inv_symm_permute(
+        make_temporary_clone(exec, permutation_indices).get(), this,
+        permute_cpy.get()));
+
+    return std::move(permute_cpy);
+}
+
+
+template <typename ValueType>
+std::unique_ptr<LinOp> Dense<ValueType>::inverse_permute(
+    const Array<int64> *permutation_indices) const
+{
+    GKO_ASSERT_IS_SQUARE_MATRIX(this);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+    auto exec = this->get_executor();
+    auto permute_cpy = Dense::create(exec, this->get_size());
+
+    exec->run(dense::make_inv_symm_permute(
+        make_temporary_clone(exec, permutation_indices).get(), this,
+        permute_cpy.get()));
+
+    return std::move(permute_cpy);
+}
+
+
 template <typename ValueType>
 std::unique_ptr<LinOp> Dense<ValueType>::row_permute(
     const Array<int32> *permutation_indices) const
@@ -763,15 +833,14 @@ std::unique_ptr<LinOp> Dense<ValueType>::column_permute(
 
 template <typename ValueType>
 std::unique_ptr<LinOp> Dense<ValueType>::inverse_row_permute(
-    const Array<int32> *inverse_permutation_indices) const
+    const Array<int32> *permutation_indices) const
 {
-    GKO_ASSERT_EQ(inverse_permutation_indices->get_num_elems(),
-                  this->get_size()[0]);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
     auto exec = this->get_executor();
     auto inverse_permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_inverse_row_permute(
-        make_temporary_clone(exec, inverse_permutation_indices).get(), this,
+        make_temporary_clone(exec, permutation_indices).get(), this,
         inverse_permute_cpy.get()));
 
     return std::move(inverse_permute_cpy);
@@ -780,15 +849,14 @@ std::unique_ptr<LinOp> Dense<ValueType>::inverse_row_permute(
 
 template <typename ValueType>
 std::unique_ptr<LinOp> Dense<ValueType>::inverse_row_permute(
-    const Array<int64> *inverse_permutation_indices) const
+    const Array<int64> *permutation_indices) const
 {
-    GKO_ASSERT_EQ(inverse_permutation_indices->get_num_elems(),
-                  this->get_size()[0]);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
     auto exec = this->get_executor();
     auto inverse_permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_inverse_row_permute(
-        make_temporary_clone(exec, inverse_permutation_indices).get(), this,
+        make_temporary_clone(exec, permutation_indices).get(), this,
         inverse_permute_cpy.get()));
 
     return std::move(inverse_permute_cpy);
@@ -797,15 +865,14 @@ std::unique_ptr<LinOp> Dense<ValueType>::inverse_row_permute(
 
 template <typename ValueType>
 std::unique_ptr<LinOp> Dense<ValueType>::inverse_column_permute(
-    const Array<int32> *inverse_permutation_indices) const
+    const Array<int32> *permutation_indices) const
 {
-    GKO_ASSERT_EQ(inverse_permutation_indices->get_num_elems(),
-                  this->get_size()[1]);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
     auto exec = this->get_executor();
     auto inverse_permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_inverse_column_permute(
-        make_temporary_clone(exec, inverse_permutation_indices).get(), this,
+        make_temporary_clone(exec, permutation_indices).get(), this,
         inverse_permute_cpy.get()));
 
     return std::move(inverse_permute_cpy);
@@ -814,15 +881,14 @@ std::unique_ptr<LinOp> Dense<ValueType>::inverse_column_permute(
 
 template <typename ValueType>
 std::unique_ptr<LinOp> Dense<ValueType>::inverse_column_permute(
-    const Array<int64> *inverse_permutation_indices) const
+    const Array<int64> *permutation_indices) const
 {
-    GKO_ASSERT_EQ(inverse_permutation_indices->get_num_elems(),
-                  this->get_size()[1]);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
     auto exec = this->get_executor();
     auto inverse_permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_inverse_column_permute(
-        make_temporary_clone(exec, inverse_permutation_indices).get(), this,
+        make_temporary_clone(exec, permutation_indices).get(), this,
         inverse_permute_cpy.get()));
 
     return std::move(inverse_permute_cpy);
diff --git a/core/matrix/dense_kernels.hpp b/core/matrix/dense_kernels.hpp
index 279c7001cc3..1799147d36e 100644
--- a/core/matrix/dense_kernels.hpp
+++ b/core/matrix/dense_kernels.hpp
@@ -143,6 +143,18 @@ namespace kernels {
                         const matrix::Dense<_type> *orig,            \
                         matrix::Dense<_type> *trans)
 
+#define GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL(_vtype, _itype)      \
+    void symm_permute(std::shared_ptr<const DefaultExecutor> exec, \
+                      const Array<_itype> *permutation_indices,    \
+                      const matrix::Dense<_vtype> *orig,           \
+                      matrix::Dense<_vtype> *permuted)
+
+#define GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL(_vtype, _itype)      \
+    void inv_symm_permute(std::shared_ptr<const DefaultExecutor> exec, \
+                          const Array<_itype> *permutation_indices,    \
+                          const matrix::Dense<_vtype> *orig,           \
+                          matrix::Dense<_vtype> *permuted)
+
 #define GKO_DECLARE_DENSE_ROW_GATHER_KERNEL(_vtype, _itype)      \
     void row_gather(std::shared_ptr<const DefaultExecutor> exec, \
                     const Array<_itype> *gather_indices,         \
@@ -238,6 +250,10 @@ namespace kernels {
     template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL(ValueType);                     \
     template <typename ValueType, typename IndexType>                       \
+    GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL(ValueType, IndexType);            \
+    template <typename ValueType, typename IndexType>                       \
+    GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL(ValueType, IndexType);        \
+    template <typename ValueType, typename IndexType>                       \
     GKO_DECLARE_DENSE_ROW_GATHER_KERNEL(ValueType, IndexType);              \
     template <typename ValueType, typename IndexType>                       \
     GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType);          \
diff --git a/cuda/matrix/csr_kernels.cu b/cuda/matrix/csr_kernels.cu
index b90745fc8d9..e2ab3e23f95 100644
--- a/cuda/matrix/csr_kernels.cu
+++ b/cuda/matrix/csr_kernels.cu
@@ -1156,6 +1156,31 @@ void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void inv_symm_permute(std::shared_ptr<const CudaExecutor> exec,
+                      const IndexType *perm,
+                      const matrix::Csr<ValueType, IndexType> *orig,
+                      matrix::Csr<ValueType, IndexType> *permuted)
+{
+    auto num_rows = orig->get_size()[0];
+    auto count_num_blocks = ceildiv(num_rows, default_block_size);
+    inv_row_ptr_permute_kernel<<<count_num_blocks, default_block_size>>>(
+        num_rows, perm, orig->get_const_row_ptrs(), permuted->get_row_ptrs());
+    components::prefix_sum(exec, permuted->get_row_ptrs(), num_rows + 1);
+    auto copy_num_blocks =
+        ceildiv(num_rows, default_block_size / config::warp_size);
+    inv_symm_permute_kernel<config::warp_size>
+        <<<copy_num_blocks, default_block_size>>>(
+            num_rows, perm, orig->get_const_row_ptrs(),
+            orig->get_const_col_idxs(), as_cuda_type(orig->get_const_values()),
+            permuted->get_row_ptrs(), permuted->get_col_idxs(),
+            as_cuda_type(permuted->get_values()));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_INV_SYMM_PERMUTE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void row_permute(std::shared_ptr<const CudaExecutor> exec,
                  const IndexType *perm,
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index 41405402b7a..ae5be6b21ec 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -602,20 +602,55 @@ void conj_transpose(std::shared_ptr<const CudaExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void symm_permute(std::shared_ptr<const CudaExecutor> exec,
+                  const Array<IndexType> *permutation_indices,
+                  const matrix::Dense<ValueType> *orig,
+                  matrix::Dense<ValueType> *permuted)
+{
+    const auto num_blocks =
+        ceildiv(orig->get_size()[0] * orig->get_size()[1], default_block_size);
+    kernel::symm_permute<<<num_blocks, default_block_size>>>(
+        orig->get_size()[0], orig->get_size()[1],
+        permutation_indices->get_const_data(),
+        as_cuda_type(orig->get_const_values()), orig->get_stride(),
+        as_cuda_type(permuted->get_values()), permuted->get_stride());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inv_symm_permute(std::shared_ptr<const CudaExecutor> exec,
+                      const Array<IndexType> *permutation_indices,
+                      const matrix::Dense<ValueType> *orig,
+                      matrix::Dense<ValueType> *permuted)
+{
+    const auto num_blocks =
+        ceildiv(orig->get_size()[0] * orig->get_size()[1], default_block_size);
+    kernel::inv_symm_permute<<<num_blocks, default_block_size>>>(
+        orig->get_size()[0], orig->get_size()[1],
+        permutation_indices->get_const_data(),
+        as_cuda_type(orig->get_const_values()), orig->get_stride(),
+        as_cuda_type(permuted->get_values()), permuted->get_stride());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void row_gather(std::shared_ptr<const CudaExecutor> exec,
                 const Array<IndexType> *row_indices,
                 const matrix::Dense<ValueType> *orig,
                 matrix::Dense<ValueType> *row_gathered)
 {
-    constexpr auto block_size = default_block_size;
     auto out_num_rows = row_indices->get_num_elems();
-    const dim3 grid_dim =
-        ceildiv(out_num_rows * orig->get_size()[1], block_size);
-    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
-    kernel::row_gather<block_size><<<grid_dim, block_dim>>>(
-        out_num_rows, orig->get_size()[1],
-        as_cuda_type(row_indices->get_const_data()),
+    const auto num_blocks =
+        ceildiv(out_num_rows * orig->get_size()[1], default_block_size);
+    kernel::row_gather<<<num_blocks, default_block_size>>>(
+        out_num_rows, orig->get_size()[1], row_indices->get_const_data(),
         as_cuda_type(orig->get_const_values()), orig->get_stride(),
         as_cuda_type(row_gathered->get_values()), row_gathered->get_stride());
 }
@@ -630,13 +665,11 @@ void column_permute(std::shared_ptr<const CudaExecutor> exec,
                     const matrix::Dense<ValueType> *orig,
                     matrix::Dense<ValueType> *column_permuted)
 {
-    constexpr auto block_size = default_block_size;
-    const dim3 grid_dim =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
-    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
-    kernel::column_permute<block_size><<<grid_dim, block_dim>>>(
+    const auto num_blocks =
+        ceildiv(orig->get_size()[0] * orig->get_size()[1], default_block_size);
+    kernel::column_permute<<<num_blocks, default_block_size>>>(
         orig->get_size()[0], orig->get_size()[1],
-        as_cuda_type(permutation_indices->get_const_data()),
+        permutation_indices->get_const_data(),
         as_cuda_type(orig->get_const_values()), orig->get_stride(),
         as_cuda_type(column_permuted->get_values()),
         column_permuted->get_stride());
@@ -652,13 +685,11 @@ void inverse_row_permute(std::shared_ptr<const CudaExecutor> exec,
                          const matrix::Dense<ValueType> *orig,
                          matrix::Dense<ValueType> *row_permuted)
 {
-    constexpr auto block_size = default_block_size;
-    const dim3 grid_dim =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
-    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
-    kernel::inverse_row_permute<block_size><<<grid_dim, block_dim>>>(
+    const auto num_blocks =
+        ceildiv(orig->get_size()[0] * orig->get_size()[1], default_block_size);
+    kernel::inverse_row_permute<<<num_blocks, default_block_size>>>(
         orig->get_size()[0], orig->get_size()[1],
-        as_cuda_type(permutation_indices->get_const_data()),
+        permutation_indices->get_const_data(),
         as_cuda_type(orig->get_const_values()), orig->get_stride(),
         as_cuda_type(row_permuted->get_values()), row_permuted->get_stride());
 }
@@ -673,13 +704,11 @@ void inverse_column_permute(std::shared_ptr<const CudaExecutor> exec,
                             const matrix::Dense<ValueType> *orig,
                             matrix::Dense<ValueType> *column_permuted)
 {
-    constexpr auto block_size = default_block_size;
-    const dim3 grid_dim =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
-    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
-    kernel::inverse_column_permute<block_size><<<grid_dim, block_dim>>>(
+    const auto num_blocks =
+        ceildiv(orig->get_size()[0] * orig->get_size()[1], default_block_size);
+    kernel::inverse_column_permute<<<num_blocks, default_block_size>>>(
         orig->get_size()[0], orig->get_size()[1],
-        as_cuda_type(permutation_indices->get_const_data()),
+        permutation_indices->get_const_data(),
         as_cuda_type(orig->get_const_values()), orig->get_stride(),
         as_cuda_type(column_permuted->get_values()),
         column_permuted->get_stride());
diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index 5cf33644c38..dd8dca752ef 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -738,6 +738,32 @@ TEST_F(Csr, MoveToHybridIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, IsPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+
+    auto permuted = gko::as<Mtx>(square_mtx->permute(rpermute_idxs.get()));
+    auto dpermuted = gko::as<Mtx>(square_dmtx->permute(rpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(permuted, dpermuted);
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
+TEST_F(Csr, IsInversePermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+
+    auto permuted =
+        gko::as<Mtx>(square_mtx->inverse_permute(rpermute_idxs.get()));
+    auto dpermuted =
+        gko::as<Mtx>(square_dmtx->inverse_permute(rpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(permuted, dpermuted);
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Csr, IsRowPermutable)
 {
     set_up_apply_data(std::make_shared<Mtx::classical>());
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index 9e48a5ea807..a21af6a2a67 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -118,6 +118,7 @@ class Dense : public ::testing::Test {
         expected = gen_mtx<Mtx>(65, 35);
         alpha = gko::initialize<Mtx>({2.0}, ref);
         beta = gko::initialize<Mtx>({-1.0}, ref);
+        square = gen_mtx<Mtx>(x->get_size()[0], x->get_size()[0]);
         dx = Mtx::create(cuda);
         dx->copy_from(x.get());
         dc_x = ComplexMtx::create(cuda);
@@ -130,6 +131,8 @@ class Dense : public ::testing::Test {
         dalpha->copy_from(alpha.get());
         dbeta = Mtx::create(cuda);
         dbeta->copy_from(beta.get());
+        dsquare = Mtx::create(cuda);
+        dsquare->copy_from(square.get());
 
         std::vector<itype> tmp(x->get_size()[0], 0);
         auto rng = std::default_random_engine{};
@@ -162,12 +165,14 @@ class Dense : public ::testing::Test {
     std::unique_ptr<Mtx> alpha;
     std::unique_ptr<Mtx> beta;
     std::unique_ptr<Mtx> expected;
+    std::unique_ptr<Mtx> square;
     std::unique_ptr<Mtx> dresult;
     std::unique_ptr<Mtx> dx;
     std::unique_ptr<ComplexMtx> dc_x;
     std::unique_ptr<Mtx> dy;
     std::unique_ptr<Mtx> dalpha;
     std::unique_ptr<Mtx> dbeta;
+    std::unique_ptr<Mtx> dsquare;
     std::unique_ptr<Arr> rpermute_idxs;
     std::unique_ptr<Arr> cpermute_idxs;
     std::unique_ptr<Arr> rgather_idxs;
@@ -591,6 +596,30 @@ TEST_F(Dense, CanGatherRowsIntoDense)
 }
 
 
+TEST_F(Dense, IsPermutable)
+{
+    set_up_apply_data();
+
+    auto permuted = square->permute(rpermute_idxs.get());
+    auto dpermuted = dsquare->permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
+                        static_cast<Mtx *>(dpermuted.get()), 0);
+}
+
+
+TEST_F(Dense, IsInversePermutable)
+{
+    set_up_apply_data();
+
+    auto permuted = square->inverse_permute(rpermute_idxs.get());
+    auto dpermuted = dsquare->inverse_permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
+                        static_cast<Mtx *>(dpermuted.get()), 0);
+}
+
+
 TEST_F(Dense, IsRowPermutable)
 {
     set_up_apply_data();
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index 7863539f7a2..9220939777f 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -262,6 +262,16 @@ void invert_permutation(std::shared_ptr<const DpcppExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void inv_symm_permute(
+    std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
+    const matrix::Csr<ValueType, IndexType> *orig,
+    matrix::Csr<ValueType, IndexType> *permuted) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_INV_SYMM_PERMUTE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void row_permute(
     std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 714d4f588b2..b9a759a1123 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -240,6 +240,26 @@ void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void symm_permute(std::shared_ptr<const DpcppExecutor> exec,
+                  const Array<IndexType> *permutation_indices,
+                  const matrix::Dense<ValueType> *orig,
+                  matrix::Dense<ValueType> *permuted) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inv_symm_permute(std::shared_ptr<const DpcppExecutor> exec,
+                      const Array<IndexType> *permutation_indices,
+                      const matrix::Dense<ValueType> *orig,
+                      matrix::Dense<ValueType> *permuted) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void row_gather(std::shared_ptr<const DpcppExecutor> exec,
                 const Array<IndexType> *gather_indices,
diff --git a/hip/matrix/csr_kernels.hip.cpp b/hip/matrix/csr_kernels.hip.cpp
index 6effd739123..8adf3f98a80 100644
--- a/hip/matrix/csr_kernels.hip.cpp
+++ b/hip/matrix/csr_kernels.hip.cpp
@@ -981,6 +981,33 @@ void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void inv_symm_permute(std::shared_ptr<const HipExecutor> exec,
+                      const IndexType *perm,
+                      const matrix::Csr<ValueType, IndexType> *orig,
+                      matrix::Csr<ValueType, IndexType> *permuted)
+{
+    auto num_rows = orig->get_size()[0];
+    auto count_num_blocks = ceildiv(num_rows, default_block_size);
+    hipLaunchKernelGGL(HIP_KERNEL_NAME(inv_row_ptr_permute_kernel),
+                       count_num_blocks, default_block_size, 0, 0, num_rows,
+                       perm, orig->get_const_row_ptrs(),
+                       permuted->get_row_ptrs());
+    components::prefix_sum(exec, permuted->get_row_ptrs(), num_rows + 1);
+    auto copy_num_blocks =
+        ceildiv(num_rows, default_block_size / config::warp_size);
+    hipLaunchKernelGGL(
+        HIP_KERNEL_NAME(inv_symm_permute_kernel<config::warp_size>),
+        copy_num_blocks, default_block_size, 0, 0, num_rows, perm,
+        orig->get_const_row_ptrs(), orig->get_const_col_idxs(),
+        as_hip_type(orig->get_const_values()), permuted->get_row_ptrs(),
+        permuted->get_col_idxs(), as_hip_type(permuted->get_values()));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_INV_SYMM_PERMUTE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void row_permute(std::shared_ptr<const HipExecutor> exec, const IndexType *perm,
                  const matrix::Csr<ValueType, IndexType> *orig,
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index b62b3736e65..3f7f4942dc8 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -625,6 +625,47 @@ void conj_transpose(std::shared_ptr<const HipExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void symm_permute(std::shared_ptr<const HipExecutor> exec,
+                  const Array<IndexType> *permutation_indices,
+                  const matrix::Dense<ValueType> *orig,
+                  matrix::Dense<ValueType> *permuted)
+{
+    constexpr auto block_size = default_block_size;
+    const auto num_blocks =
+        ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
+    hipLaunchKernelGGL(
+        kernel::symm_permute, num_blocks, block_size, 0, 0, orig->get_size()[0],
+        orig->get_size()[1], permutation_indices->get_const_data(),
+        as_hip_type(orig->get_const_values()), orig->get_stride(),
+        as_hip_type(permuted->get_values()), permuted->get_stride());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inv_symm_permute(std::shared_ptr<const HipExecutor> exec,
+                      const Array<IndexType> *permutation_indices,
+                      const matrix::Dense<ValueType> *orig,
+                      matrix::Dense<ValueType> *permuted)
+{
+    constexpr auto block_size = default_block_size;
+    const auto num_blocks =
+        ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
+    hipLaunchKernelGGL(kernel::inv_symm_permute, num_blocks, block_size, 0, 0,
+                       orig->get_size()[0], orig->get_size()[1],
+                       permutation_indices->get_const_data(),
+                       as_hip_type(orig->get_const_values()),
+                       orig->get_stride(), as_hip_type(permuted->get_values()),
+                       permuted->get_stride());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void row_gather(std::shared_ptr<const HipExecutor> exec,
                 const Array<IndexType> *row_indices,
@@ -633,13 +674,11 @@ void row_gather(std::shared_ptr<const HipExecutor> exec,
 {
     constexpr auto block_size = default_block_size;
     auto out_num_rows = row_indices->get_num_elems();
-    const dim3 grid_dim =
+    const auto num_blocks =
         ceildiv(out_num_rows * orig->get_size()[1], block_size);
-    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
     hipLaunchKernelGGL(
-        kernel::row_gather<block_size>, dim3(grid_dim), dim3(block_dim), 0, 0,
-        out_num_rows, orig->get_size()[1],
-        as_hip_type(row_indices->get_const_data()),
+        kernel::row_gather, num_blocks, block_size, 0, 0, out_num_rows,
+        orig->get_size()[1], row_indices->get_const_data(),
         as_hip_type(orig->get_const_values()), orig->get_stride(),
         as_hip_type(row_gathered->get_values()), row_gathered->get_stride());
 }
@@ -655,16 +694,15 @@ void column_permute(std::shared_ptr<const HipExecutor> exec,
                     matrix::Dense<ValueType> *column_permuted)
 {
     constexpr auto block_size = default_block_size;
-    const dim3 grid_dim =
+    const auto num_blocks =
         ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
-    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
-    hipLaunchKernelGGL(
-        kernel::column_permute<block_size>, dim3(grid_dim), dim3(block_dim), 0,
-        0, orig->get_size()[0], orig->get_size()[1],
-        as_hip_type(permutation_indices->get_const_data()),
-        as_hip_type(orig->get_const_values()), orig->get_stride(),
-        as_hip_type(column_permuted->get_values()),
-        column_permuted->get_stride());
+    hipLaunchKernelGGL(kernel::column_permute, num_blocks, block_size, 0, 0,
+                       orig->get_size()[0], orig->get_size()[1],
+                       permutation_indices->get_const_data(),
+                       as_hip_type(orig->get_const_values()),
+                       orig->get_stride(),
+                       as_hip_type(column_permuted->get_values()),
+                       column_permuted->get_stride());
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -678,13 +716,12 @@ void inverse_row_permute(std::shared_ptr<const HipExecutor> exec,
                          matrix::Dense<ValueType> *row_permuted)
 {
     constexpr auto block_size = default_block_size;
-    const dim3 grid_dim =
+    const auto num_blocks =
         ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
-    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
     hipLaunchKernelGGL(
-        kernel::inverse_row_permute<block_size>, dim3(grid_dim),
-        dim3(block_dim), 0, 0, orig->get_size()[0], orig->get_size()[1],
-        as_hip_type(permutation_indices->get_const_data()),
+        kernel::inverse_row_permute, num_blocks, block_size, 0, 0,
+        orig->get_size()[0], orig->get_size()[1],
+        permutation_indices->get_const_data(),
         as_hip_type(orig->get_const_values()), orig->get_stride(),
         as_hip_type(row_permuted->get_values()), row_permuted->get_stride());
 }
@@ -700,16 +737,15 @@ void inverse_column_permute(std::shared_ptr<const HipExecutor> exec,
                             matrix::Dense<ValueType> *column_permuted)
 {
     constexpr auto block_size = default_block_size;
-    const dim3 grid_dim =
+    const auto num_blocks =
         ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
-    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
-    hipLaunchKernelGGL(
-        kernel::inverse_column_permute<block_size>, dim3(grid_dim),
-        dim3(block_dim), 0, 0, orig->get_size()[0], orig->get_size()[1],
-        as_hip_type(permutation_indices->get_const_data()),
-        as_hip_type(orig->get_const_values()), orig->get_stride(),
-        as_hip_type(column_permuted->get_values()),
-        column_permuted->get_stride());
+    hipLaunchKernelGGL(kernel::inverse_column_permute, num_blocks, block_size,
+                       0, 0, orig->get_size()[0], orig->get_size()[1],
+                       permutation_indices->get_const_data(),
+                       as_hip_type(orig->get_const_values()),
+                       orig->get_stride(),
+                       as_hip_type(column_permuted->get_values()),
+                       column_permuted->get_stride());
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index 21a2e751714..36ad284ad71 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -732,6 +732,32 @@ TEST_F(Csr, MoveToHybridIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, IsPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+
+    auto permuted = gko::as<Mtx>(square_mtx->permute(rpermute_idxs.get()));
+    auto dpermuted = gko::as<Mtx>(square_dmtx->permute(rpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(permuted, dpermuted);
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
+TEST_F(Csr, IsInversePermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+
+    auto permuted =
+        gko::as<Mtx>(square_mtx->inverse_permute(rpermute_idxs.get()));
+    auto dpermuted =
+        gko::as<Mtx>(square_dmtx->inverse_permute(rpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(permuted, dpermuted);
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Csr, IsRowPermutable)
 {
     set_up_apply_data(std::make_shared<Mtx::classical>());
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index 210f4a879e3..dcd3ace06b9 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -117,6 +117,7 @@ class Dense : public ::testing::Test {
         expected = gen_mtx<Mtx>(65, 35);
         alpha = gko::initialize<Mtx>({2.0}, ref);
         beta = gko::initialize<Mtx>({-1.0}, ref);
+        square = gen_mtx<Mtx>(x->get_size()[0], x->get_size()[0]);
         dx = Mtx::create(hip);
         dx->copy_from(x.get());
         dy = Mtx::create(hip);
@@ -127,6 +128,8 @@ class Dense : public ::testing::Test {
         dalpha->copy_from(alpha.get());
         dbeta = Mtx::create(hip);
         dbeta->copy_from(beta.get());
+        dsquare = Mtx::create(hip);
+        dsquare->copy_from(square.get());
 
         std::vector<itype> tmp(x->get_size()[0], 0);
         auto rng = std::default_random_engine{};
@@ -158,11 +161,13 @@ class Dense : public ::testing::Test {
     std::unique_ptr<Mtx> alpha;
     std::unique_ptr<Mtx> beta;
     std::unique_ptr<Mtx> expected;
+    std::unique_ptr<Mtx> square;
     std::unique_ptr<Mtx> dresult;
     std::unique_ptr<Mtx> dx;
     std::unique_ptr<Mtx> dy;
     std::unique_ptr<Mtx> dalpha;
     std::unique_ptr<Mtx> dbeta;
+    std::unique_ptr<Mtx> dsquare;
     std::unique_ptr<Arr> rpermute_idxs;
     std::unique_ptr<Arr> cpermute_idxs;
     std::unique_ptr<Arr> rgather_idxs;
@@ -574,6 +579,30 @@ TEST_F(Dense, CanGatherRowsIntoDense)
 }
 
 
+TEST_F(Dense, IsPermutable)
+{
+    set_up_apply_data();
+
+    auto permuted = square->permute(rpermute_idxs.get());
+    auto dpermuted = dsquare->permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
+                        static_cast<Mtx *>(dpermuted.get()), 0);
+}
+
+
+TEST_F(Dense, IsInversePermutable)
+{
+    set_up_apply_data();
+
+    auto permuted = square->inverse_permute(rpermute_idxs.get());
+    auto dpermuted = dsquare->inverse_permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
+                        static_cast<Mtx *>(dpermuted.get()), 0);
+}
+
+
 TEST_F(Dense, IsRowPermutable)
 {
     set_up_apply_data();
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index 5c2c9a34fa1..b831e71fc3b 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -469,12 +469,44 @@ class Permutable {
 public:
     virtual ~Permutable() = default;
 
+    /**
+     * Returns a LinOp representing the symmetric row and column permutation of
+     * the Permutable object.
+     *
+     * @param permutation_indices  the array of indices contaning the
+     *                             permutation order.
+     *
+     * @return a pointer to the new permuted object
+     */
+    virtual std::unique_ptr<LinOp> permute(
+        const Array<IndexType> *permutation_indices) const
+    {
+        return as<Permutable>(this->row_permute(permutation_indices))
+            ->column_permute(permutation_indices);
+    };
+
+    /**
+     * Returns a LinOp representing the symmetric inverse row and column
+     * permutation of the Permutable object.
+     *
+     * @param permutation_indices  the array of indices contaning the
+     *                             permutation order.
+     *
+     * @return a pointer to the new permuted object
+     */
+    virtual std::unique_ptr<LinOp> inverse_permute(
+        const Array<IndexType> *permutation_indices) const
+    {
+        return as<Permutable>(this->inverse_row_permute(permutation_indices))
+            ->inverse_column_permute(permutation_indices);
+    };
+
     /**
      * Returns a LinOp representing the row permutation of the Permutable
      * object.
      *
      * @param permutation_indices  the array of indices contaning the
-     * permutation order.
+     *                             permutation order.
      *
      * @return a pointer to the new permuted object
      */
@@ -486,7 +518,7 @@ class Permutable {
      * object.
      *
      * @param permutation_indices  the array of indices contaning the
-     * permutation order.
+     *                             permutation order.
      *
      * @return a pointer to the new column permuted object
      */
@@ -497,25 +529,25 @@ class Permutable {
      * Returns a LinOp representing the row permutation of the inverse permuted
      * object.
      *
-     * @param inverse_permutation_indices  the array of indices contaning the
-     * inverse permutation order.
+     * @param permutation_indices  the array of indices contaning the
+     *                             permutation order.
      *
      * @return a pointer to the new inverse permuted object
      */
     virtual std::unique_ptr<LinOp> inverse_row_permute(
-        const Array<IndexType> *inverse_permutation_indices) const = 0;
+        const Array<IndexType> *permutation_indices) const = 0;
 
     /**
      * Returns a LinOp representing the row permutation of the inverse permuted
      * object.
      *
-     * @param inverse_permutation_indices  the array of indices contaning the
-     * inverse permutation order.
+     * @param permutation_indices  the array of indices contaning the
+     *                             permutation order.
      *
      * @return a pointer to the new inverse permuted object
      */
     virtual std::unique_ptr<LinOp> inverse_column_permute(
-        const Array<IndexType> *inverse_permutation_indices) const = 0;
+        const Array<IndexType> *permutation_indices) const = 0;
 };
 
 
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index bb54cabdd7a..f75cc80e1a1 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -694,6 +694,12 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
 
     std::unique_ptr<LinOp> conj_transpose() const override;
 
+    std::unique_ptr<LinOp> permute(
+        const Array<IndexType> *permutation_indices) const override;
+
+    std::unique_ptr<LinOp> inverse_permute(
+        const Array<IndexType> *inverse_permutation_indices) const override;
+
     std::unique_ptr<LinOp> row_permute(
         const Array<IndexType> *permutation_indices) const override;
 
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index ad689c8ad1f..a7fc245cd71 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -226,6 +226,18 @@ class Dense
 
     std::unique_ptr<LinOp> conj_transpose() const override;
 
+    std::unique_ptr<LinOp> permute(
+        const Array<int32> *permutation_indices) const override;
+
+    std::unique_ptr<LinOp> permute(
+        const Array<int64> *permutation_indices) const override;
+
+    std::unique_ptr<LinOp> inverse_permute(
+        const Array<int32> *permutation_indices) const override;
+
+    std::unique_ptr<LinOp> inverse_permute(
+        const Array<int64> *permutation_indices) const override;
+
     std::unique_ptr<LinOp> row_permute(
         const Array<int32> *permutation_indices) const override;
 
@@ -276,16 +288,16 @@ class Dense
         const Array<int64> *permutation_indices) const override;
 
     std::unique_ptr<LinOp> inverse_row_permute(
-        const Array<int32> *inverse_permutation_indices) const override;
+        const Array<int32> *permutation_indices) const override;
 
     std::unique_ptr<LinOp> inverse_row_permute(
-        const Array<int64> *inverse_permutation_indices) const override;
+        const Array<int64> *permutation_indices) const override;
 
     std::unique_ptr<LinOp> inverse_column_permute(
-        const Array<int32> *inverse_permutation_indices) const override;
+        const Array<int32> *permutation_indices) const override;
 
     std::unique_ptr<LinOp> inverse_column_permute(
-        const Array<int64> *inverse_permutation_indices) const override;
+        const Array<int64> *permutation_indices) const override;
 
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
diff --git a/omp/matrix/csr_kernels.cpp b/omp/matrix/csr_kernels.cpp
index 0f681fa409b..284bc380544 100644
--- a/omp/matrix/csr_kernels.cpp
+++ b/omp/matrix/csr_kernels.cpp
@@ -612,6 +612,45 @@ void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void inv_symm_permute(std::shared_ptr<const DefaultExecutor> exec,
+                      const IndexType *perm,
+                      const matrix::Csr<ValueType, IndexType> *orig,
+                      matrix::Csr<ValueType, IndexType> *permuted)
+{
+    auto in_row_ptrs = orig->get_const_row_ptrs();
+    auto in_col_idxs = orig->get_const_col_idxs();
+    auto in_vals = orig->get_const_values();
+    auto p_row_ptrs = permuted->get_row_ptrs();
+    auto p_col_idxs = permuted->get_col_idxs();
+    auto p_vals = permuted->get_values();
+    size_type num_rows = orig->get_size()[0];
+
+#pragma omp parallel for
+    for (size_type row = 0; row < num_rows; ++row) {
+        auto src_row = row;
+        auto dst_row = perm[row];
+        p_row_ptrs[dst_row] = in_row_ptrs[src_row + 1] - in_row_ptrs[src_row];
+    }
+    components::prefix_sum(exec, p_row_ptrs, num_rows + 1);
+#pragma omp parallel for
+    for (size_type row = 0; row < num_rows; ++row) {
+        auto src_row = row;
+        auto dst_row = perm[row];
+        auto src_begin = in_row_ptrs[src_row];
+        auto dst_begin = p_row_ptrs[dst_row];
+        auto row_size = in_row_ptrs[src_row + 1] - src_begin;
+        for (IndexType i = 0; i < row_size; ++i) {
+            p_col_idxs[dst_begin + i] = perm[in_col_idxs[src_begin + i]];
+            p_vals[dst_begin + i] = in_vals[src_begin + i];
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_INV_SYMM_PERMUTE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void row_permute(std::shared_ptr<const OmpExecutor> exec, const IndexType *perm,
                  const matrix::Csr<ValueType, IndexType> *orig,
diff --git a/omp/matrix/dense_kernels.cpp b/omp/matrix/dense_kernels.cpp
index 7f49b3d6c8b..a14f45ba033 100644
--- a/omp/matrix/dense_kernels.cpp
+++ b/omp/matrix/dense_kernels.cpp
@@ -679,6 +679,46 @@ void conj_transpose(std::shared_ptr<const OmpExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void symm_permute(std::shared_ptr<const OmpExecutor> exec,
+                  const Array<IndexType> *permutation_indices,
+                  const matrix::Dense<ValueType> *orig,
+                  matrix::Dense<ValueType> *permuted)
+{
+    auto perm = permutation_indices->get_const_data();
+    auto size = orig->get_size()[0];
+#pragma omp parallel for
+    for (size_type i = 0; i < size; ++i) {
+        for (size_type j = 0; j < size; ++j) {
+            permuted->at(i, j) = orig->at(perm[i], perm[j]);
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inv_symm_permute(std::shared_ptr<const OmpExecutor> exec,
+                      const Array<IndexType> *permutation_indices,
+                      const matrix::Dense<ValueType> *orig,
+                      matrix::Dense<ValueType> *permuted)
+{
+    auto perm = permutation_indices->get_const_data();
+    auto size = orig->get_size()[0];
+#pragma omp parallel for
+    for (size_type i = 0; i < size; ++i) {
+        for (size_type j = 0; j < size; ++j) {
+            permuted->at(perm[i], perm[j]) = orig->at(i, j);
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void row_gather(std::shared_ptr<const OmpExecutor> exec,
                 const Array<IndexType> *row_indices,
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index 439d8271a09..2419b5c1a41 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -465,6 +465,32 @@ TEST_F(Csr, MoveToHybridIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, IsPermutable)
+{
+    set_up_apply_data();
+
+    auto permuted = gko::as<Mtx>(square_mtx->permute(rpermute_idxs.get()));
+    auto dpermuted = gko::as<Mtx>(square_dmtx->permute(rpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(permuted, dpermuted);
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
+TEST_F(Csr, IsInversePermutable)
+{
+    set_up_apply_data();
+
+    auto permuted =
+        gko::as<Mtx>(square_mtx->inverse_permute(rpermute_idxs.get()));
+    auto dpermuted =
+        gko::as<Mtx>(square_dmtx->inverse_permute(rpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(permuted, dpermuted);
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Csr, IsRowPermutable)
 {
     set_up_apply_data();
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index 2ee7ea96075..4f36313c8ad 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -130,6 +130,7 @@ class Dense : public ::testing::Test {
         expected = gen_mtx<Mtx>(40, 35);
         alpha = gko::initialize<Mtx>({2.0}, ref);
         beta = gko::initialize<Mtx>({-1.0}, ref);
+        square = gen_mtx<Mtx>(x->get_size()[0], x->get_size()[0]);
         dx = Mtx::create(omp);
         dx->copy_from(x.get());
         dc_x = ComplexMtx::create(omp);
@@ -142,6 +143,8 @@ class Dense : public ::testing::Test {
         dalpha->copy_from(alpha.get());
         dbeta = Mtx::create(omp);
         dbeta->copy_from(beta.get());
+        dsquare = Mtx::create(omp);
+        dsquare->copy_from(square.get());
 
         std::vector<int> tmp(x->get_size()[0], 0);
         auto rng = std::default_random_engine{};
@@ -174,12 +177,14 @@ class Dense : public ::testing::Test {
     std::unique_ptr<Mtx> alpha;
     std::unique_ptr<Mtx> beta;
     std::unique_ptr<Mtx> expected;
+    std::unique_ptr<Mtx> square;
     std::unique_ptr<Mtx> dresult;
     std::unique_ptr<Mtx> dx;
     std::unique_ptr<ComplexMtx> dc_x;
     std::unique_ptr<Mtx> dy;
     std::unique_ptr<Mtx> dalpha;
     std::unique_ptr<Mtx> dbeta;
+    std::unique_ptr<Mtx> dsquare;
     std::unique_ptr<Arr> rpermute_idxs;
     std::unique_ptr<Arr> cpermute_idxs;
     std::unique_ptr<Arr> rgather_idxs;
@@ -710,6 +715,30 @@ TEST_F(Dense, CanGatherRowsIntoDense)
 }
 
 
+TEST_F(Dense, IsPermutable)
+{
+    set_up_apply_data();
+
+    auto permuted = square->permute(rpermute_idxs.get());
+    auto dpermuted = dsquare->permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
+                        static_cast<Mtx *>(dpermuted.get()), 0);
+}
+
+
+TEST_F(Dense, IsInversePermutable)
+{
+    set_up_apply_data();
+
+    auto permuted = square->inverse_permute(rpermute_idxs.get());
+    auto dpermuted = dsquare->inverse_permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
+                        static_cast<Mtx *>(dpermuted.get()), 0);
+}
+
+
 TEST_F(Dense, IsRowPermutable)
 {
     set_up_apply_data();
diff --git a/reference/matrix/csr_kernels.cpp b/reference/matrix/csr_kernels.cpp
index 9cb739a82b9..d4d5a02e473 100644
--- a/reference/matrix/csr_kernels.cpp
+++ b/reference/matrix/csr_kernels.cpp
@@ -702,6 +702,43 @@ void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void inv_symm_permute(std::shared_ptr<const ReferenceExecutor> exec,
+                      const IndexType *perm,
+                      const matrix::Csr<ValueType, IndexType> *orig,
+                      matrix::Csr<ValueType, IndexType> *permuted)
+{
+    auto in_row_ptrs = orig->get_const_row_ptrs();
+    auto in_col_idxs = orig->get_const_col_idxs();
+    auto in_vals = orig->get_const_values();
+    auto p_row_ptrs = permuted->get_row_ptrs();
+    auto p_col_idxs = permuted->get_col_idxs();
+    auto p_vals = permuted->get_values();
+    size_type num_rows = orig->get_size()[0];
+
+    for (size_type row = 0; row < num_rows; ++row) {
+        auto src_row = row;
+        auto dst_row = perm[row];
+        p_row_ptrs[dst_row] = in_row_ptrs[src_row + 1] - in_row_ptrs[src_row];
+    }
+    components::prefix_sum(exec, p_row_ptrs, num_rows + 1);
+    for (size_type row = 0; row < num_rows; ++row) {
+        auto src_row = row;
+        auto dst_row = perm[row];
+        auto src_begin = in_row_ptrs[src_row];
+        auto dst_begin = p_row_ptrs[dst_row];
+        auto row_size = in_row_ptrs[src_row + 1] - src_begin;
+        for (IndexType i = 0; i < row_size; ++i) {
+            p_col_idxs[dst_begin + i] = perm[in_col_idxs[src_begin + i]];
+            p_vals[dst_begin + i] = in_vals[src_begin + i];
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_INV_SYMM_PERMUTE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void row_permute(std::shared_ptr<const ReferenceExecutor> exec,
                  const IndexType *perm,
diff --git a/reference/matrix/dense_kernels.cpp b/reference/matrix/dense_kernels.cpp
index ab020a84a3d..16443c0bbb5 100644
--- a/reference/matrix/dense_kernels.cpp
+++ b/reference/matrix/dense_kernels.cpp
@@ -599,6 +599,44 @@ void conj_transpose(std::shared_ptr<const ReferenceExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void symm_permute(std::shared_ptr<const ReferenceExecutor> exec,
+                  const Array<IndexType> *permutation_indices,
+                  const matrix::Dense<ValueType> *orig,
+                  matrix::Dense<ValueType> *permuted)
+{
+    auto perm = permutation_indices->get_const_data();
+    auto size = orig->get_size()[0];
+    for (size_type i = 0; i < size; ++i) {
+        for (size_type j = 0; j < size; ++j) {
+            permuted->at(i, j) = orig->at(perm[i], perm[j]);
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inv_symm_permute(std::shared_ptr<const ReferenceExecutor> exec,
+                      const Array<IndexType> *permutation_indices,
+                      const matrix::Dense<ValueType> *orig,
+                      matrix::Dense<ValueType> *permuted)
+{
+    auto perm = permutation_indices->get_const_data();
+    auto size = orig->get_size()[0];
+    for (size_type i = 0; i < size; ++i) {
+        for (size_type j = 0; j < size; ++j) {
+            permuted->at(perm[i], perm[j]) = orig->at(i, j);
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void row_gather(std::shared_ptr<const ReferenceExecutor> exec,
                 const Array<IndexType> *row_indices,
diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp
index 14f153ff8f3..b963508eb45 100644
--- a/reference/test/matrix/csr_kernels.cpp
+++ b/reference/test/matrix/csr_kernels.cpp
@@ -1067,6 +1067,46 @@ TYPED_TEST(Csr, NonSquareMtxIsTransposable)
 }
 
 
+TYPED_TEST(Csr, SquareMatrixIsPermutable)
+{
+    using Csr = typename TestFixture::Mtx;
+    using index_type = typename TestFixture::index_type;
+    // clang-format off
+    auto p_mtx = gko::initialize<Csr>({{1.0, 3.0, 2.0},
+                                       {0.0, 5.0, 0.0},
+                                       {0.0, 1.5, 2.0}}, this->exec);
+    // clang-format on
+    gko::Array<index_type> permute_idxs{this->exec, {1, 2, 0}};
+
+    auto ref_permute_csr =
+        gko::as<Csr>(gko::as<Csr>(p_mtx->row_permute(&permute_idxs))
+                         ->column_permute(&permute_idxs));
+    auto permute_csr = gko::as<Csr>(p_mtx->permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(ref_permute_csr, permute_csr, 0.0);
+}
+
+
+TYPED_TEST(Csr, SquareMatrixIsInversePermutable)
+{
+    using Csr = typename TestFixture::Mtx;
+    using index_type = typename TestFixture::index_type;
+    // clang-format off
+    auto p_mtx = gko::initialize<Csr>({{1.0, 3.0, 2.0},
+                                       {0.0, 5.0, 0.0},
+                                       {0.0, 1.5, 2.0}}, this->exec);
+    // clang-format on
+    gko::Array<index_type> permute_idxs{this->exec, {1, 2, 0}};
+
+    auto ref_permute_csr =
+        gko::as<Csr>(gko::as<Csr>(p_mtx->inverse_row_permute(&permute_idxs))
+                         ->inverse_column_permute(&permute_idxs));
+    auto permute_csr = gko::as<Csr>(p_mtx->inverse_permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(ref_permute_csr, permute_csr, 0.0);
+}
+
+
 TYPED_TEST(Csr, SquareMatrixIsRowPermutable)
 {
     using Csr = typename TestFixture::Mtx;
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 6f64ab26289..3022ece5124 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -1804,6 +1804,86 @@ TYPED_TEST(Dense, SquareMatrixCanGatherRowsIntoDense64)
 }
 
 
+TYPED_TEST(Dense, SquareMatrixIsPermutable)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+
+    auto ref_permuted =
+        gko::as<Mtx>(gko::as<Mtx>(this->mtx5->row_permute(&permute_idxs))
+                         ->column_permute(&permute_idxs));
+    auto permuted = gko::as<Mtx>(this->mtx5->permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(ref_permuted, ref_permuted, r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsInversePermutable)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+
+    auto ref_permuted = gko::as<Mtx>(
+        gko::as<Mtx>(this->mtx5->inverse_row_permute(&permute_idxs))
+            ->inverse_column_permute(&permute_idxs));
+    auto permuted = gko::as<Mtx>(this->mtx5->inverse_permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(ref_permuted, ref_permuted, r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsPermutable64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+
+    auto ref_permuted =
+        gko::as<Mtx>(gko::as<Mtx>(this->mtx5->row_permute(&permute_idxs))
+                         ->column_permute(&permute_idxs));
+    auto permuted = gko::as<Mtx>(this->mtx5->permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(ref_permuted, ref_permuted, r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsInversePermutable64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+
+    auto ref_permuted = gko::as<Mtx>(
+        gko::as<Mtx>(this->mtx5->inverse_row_permute(&permute_idxs))
+            ->inverse_column_permute(&permute_idxs));
+    auto permuted = gko::as<Mtx>(this->mtx5->inverse_permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(ref_permuted, ref_permuted, r<TypeParam>::value);
+}
+
+
 TYPED_TEST(Dense, SquareMatrixIsRowPermutable)
 {
     // clang-format off

From 05424e29c0b044b9307c046d3f78bf99e777f1e4 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 6 Jan 2021 13:38:06 +0100
Subject: [PATCH 0187/1008] improve Permutable documentation

---
 include/ginkgo/core/base/lin_op.hpp | 53 +++++++++++++++++------------
 1 file changed, 32 insertions(+), 21 deletions(-)

diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index b831e71fc3b..59c7a02586d 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -440,18 +440,19 @@ class Transposable {
  * Linear operators which support permutation should implement the
  * Permutable interface.
  *
- * It provides four functionalities, the row permute, the
- * column permute, the inverse row permute and the inverse column permute.
- *
- * The row permute returns the permutation of the linear operator after
- * permuting the rows of the linear operator. For example, if for a matrix A,
- * the permuted matrix A' and the permutation array perm, the row i of the
- * matrix A is the row perm[i] in the matrix A'. And similarly, for the inverse
- * permutation, the row i in the matrix A' is the row perm[i] in the matrix A.
- *
- * The column permute returns the permutation of the linear operator after
- * permuting the columns of the linear operator. The definitions of permute and
- * inverse permute for the row_permute hold here as well.
+ * It provides functions to permute the rows and columns of a LinOp,
+ * independently or symmetrically, and with a regular or inverted permutation.
+ *
+ * After a regular row permutation with permutation array `perm` the row `i` in
+ * the output LinOp contains the row `perm[i]` from the input LinOp.
+ * After an inverse row permutation, the row `perm[i]` in the output LinOp
+ * contains the row `i` from the input LinOp.
+ * Equivalently, after a column permutation, the output stores in column `i`
+ * the column `perm[i]` from the input, and an inverse column permutation
+ * stores in column `perm[i]` the column `i` from the input.
+ * A symmetric permutation is functionally equivalent to calling
+ * `as<Permutable>(A->row_permute(perm))->column_permute(perm)`, but the
+ * implementation can provide better performance due to kernel fusion.
  *
  * Example: Permuting a Csr matrix:
  * ------------------------------------
@@ -472,8 +473,10 @@ class Permutable {
     /**
      * Returns a LinOp representing the symmetric row and column permutation of
      * the Permutable object.
+     * In the resulting LinOp, the entry at location `(i,j)` contains the input
+     * value `(perm[i],perm[j])`.
      *
-     * @param permutation_indices  the array of indices contaning the
+     * @param permutation_indices  the array of indices containing the
      *                             permutation order.
      *
      * @return a pointer to the new permuted object
@@ -488,8 +491,10 @@ class Permutable {
     /**
      * Returns a LinOp representing the symmetric inverse row and column
      * permutation of the Permutable object.
+     * In the resulting LinOp, the entry at location `(perm[i],perm[j])`
+     * contains the input value `(i,j)`.
      *
-     * @param permutation_indices  the array of indices contaning the
+     * @param permutation_indices  the array of indices containing the
      *                             permutation order.
      *
      * @return a pointer to the new permuted object
@@ -504,8 +509,9 @@ class Permutable {
     /**
      * Returns a LinOp representing the row permutation of the Permutable
      * object.
+     * In the resulting LinOp, the row `i` contains the input row `perm[i]`.
      *
-     * @param permutation_indices  the array of indices contaning the
+     * @param permutation_indices  the array of indices containing the
      *                             permutation order.
      *
      * @return a pointer to the new permuted object
@@ -516,9 +522,11 @@ class Permutable {
     /**
      * Returns a LinOp representing the column permutation of the Permutable
      * object.
+     * In the resulting LinOp, the column `i` contains the input column
+     * `perm[i]`.
      *
-     * @param permutation_indices  the array of indices contaning the
-     *                             permutation order.
+     * @param permutation_indices  the array of indices containing the
+     *                             permutation order `perm`.
      *
      * @return a pointer to the new column permuted object
      */
@@ -528,9 +536,10 @@ class Permutable {
     /**
      * Returns a LinOp representing the row permutation of the inverse permuted
      * object.
+     * In the resulting LinOp, the row `perm[i]` contains the input row `i`.
      *
-     * @param permutation_indices  the array of indices contaning the
-     *                             permutation order.
+     * @param permutation_indices  the array of indices containing the
+     *                             permutation order `perm`.
      *
      * @return a pointer to the new inverse permuted object
      */
@@ -540,9 +549,11 @@ class Permutable {
     /**
      * Returns a LinOp representing the row permutation of the inverse permuted
      * object.
+     * In the resulting LinOp, the column `perm[i]` contains the input column
+     * `i`.
      *
-     * @param permutation_indices  the array of indices contaning the
-     *                             permutation order.
+     * @param permutation_indices  the array of indices containing the
+     *                             permutation order `perm`.
      *
      * @return a pointer to the new inverse permuted object
      */

From e7c392dd8c1bc8ce73e9ccd4d9066ef11c94fe46 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 5 Jan 2021 12:10:02 +0100
Subject: [PATCH 0188/1008] allow bot to run all pages and update label regex

- if label use regex, use quote
- add .github match for reg:ci-cd

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 .github/bot-pr-format-base.sh | 21 +++++++++++++++++++-
 .github/label.sh              | 36 ++++++++++++++++++++++++++---------
 2 files changed, 47 insertions(+), 10 deletions(-)

diff --git a/.github/bot-pr-format-base.sh b/.github/bot-pr-format-base.sh
index d89d3f2ee2a..8d213dcfe69 100644
--- a/.github/bot-pr-format-base.sh
+++ b/.github/bot-pr-format-base.sh
@@ -7,7 +7,26 @@ FORMAT_HEADER_REGEX='^(benchmark|core|cuda|hip|include/ginkgo/core|omp|reference
 FORMAT_REGEX='^(common|examples|test_install)/'
 
 echo "Retrieving PR file list"
-PR_FILES="$(api_get "$PR_URL/files?&per_page=1000" | jq -er '.[] | .filename')"
+PR_FILES=""
+PAGE="1"
+while true; do
+  # this api allows 100 items per page
+  # github action uses `bash -e`. The last empty page will leads jq error, use `|| :` to ignore the error.
+  PR_PAGE_FILES=$(api_get "$PR_URL/files?&per_page=100&page=${PAGE}" | jq -er '.[] | .filename' || :)
+  if [ "${PR_PAGE_FILES}" = "" ]; then
+    break
+  fi
+  echo "Retrieving PR file list - ${PAGE} pages"
+  if [ ! "${PR_FILES}" = "" ]; then
+    # add the same new line format as jq output
+    PR_FILES="${PR_FILES}"$'\n'
+  fi
+  PR_FILES="${PR_FILES}${PR_PAGE_FILES}"
+  PAGE=$(( PAGE + 1 ))
+done
+NUM=$(echo "${PR_FILES}" | wc -l)
+echo "PR has ${NUM} changed files"
+
 TO_FORMAT="$(echo "$PR_FILES" | grep -E $EXTENSION_REGEX || true)"
 
 git remote add fork "$HEAD_URL"
diff --git a/.github/label.sh b/.github/label.sh
index 3cd5023829b..860e520720c 100755
--- a/.github/label.sh
+++ b/.github/label.sh
@@ -3,7 +3,25 @@
 source .github/bot-pr-base.sh
 
 echo "Retrieving PR file list"
-PR_FILES=$(api_get "$PR_URL/files?&per_page=1000" | jq -er '.[] | .filename')
+PR_FILES=""
+PAGE="1"
+while true; do
+  # this api allows 100 items per page
+  # github action uses `bash -e`. The last empty page will leads jq error, use `|| :` to ignore the error.
+  PR_PAGE_FILES=$(api_get "$PR_URL/files?&per_page=100&page=${PAGE}" | jq -er '.[] | .filename' || :)
+  if [ "${PR_PAGE_FILES}" = "" ]; then
+    break
+  fi
+  echo "Retrieving PR file list - ${PAGE} pages"
+  if [ ! "${PR_FILES}" = "" ]; then
+    # add the same new line format as jq output
+    PR_FILES="${PR_FILES}"$'\n'
+  fi
+  PR_FILES="${PR_FILES}${PR_PAGE_FILES}"
+  PAGE=$(( PAGE + 1 ))
+done
+NUM=$(echo "${PR_FILES}" | wc -l)
+echo "PR has ${NUM} changed files"
 
 echo "Retrieving PR label list"
 OLD_LABELS=$(api_get "$ISSUE_URL" | jq -er '[.labels | .[] | .name]')
@@ -17,18 +35,18 @@ label_match() {
 
 LABELS="[]"
 LABELS=$LABELS$(label_match mod:core '(^core/|^include/)')
-LABELS=$LABELS$(label_match mod:reference ^reference/)
-LABELS=$LABELS$(label_match mod:openmp ^omp/)
+LABELS=$LABELS$(label_match mod:reference '^reference/')
+LABELS=$LABELS$(label_match mod:openmp '^omp/')
 LABELS=$LABELS$(label_match mod:cuda '(^cuda/|^common/)')
 LABELS=$LABELS$(label_match mod:hip '(^hip/|^common/)')
-LABELS=$LABELS$(label_match mod:dpcpp ^dpcpp/)
-LABELS=$LABELS$(label_match reg:benchmarking ^benchmark/)
-LABELS=$LABELS$(label_match reg:example ^examples/)
+LABELS=$LABELS$(label_match mod:dpcpp '^dpcpp/')
+LABELS=$LABELS$(label_match reg:benchmarking '^benchmark/')
+LABELS=$LABELS$(label_match reg:example '^examples/')
 LABELS=$LABELS$(label_match reg:build '(cm|CM)ake')
-LABELS=$LABELS$(label_match reg:ci-cd '.yml$')
-LABELS=$LABELS$(label_match reg:documentation ^doc/)
+LABELS=$LABELS$(label_match reg:ci-cd '(^\.github/|\.yml$)')
+LABELS=$LABELS$(label_match reg:documentation '^doc/')
 LABELS=$LABELS$(label_match reg:testing /test/)
-LABELS=$LABELS$(label_match reg:helper-scripts ^dev_tools/)
+LABELS=$LABELS$(label_match reg:helper-scripts '^dev_tools/')
 LABELS=$LABELS$(label_match type:factorization /factorization/)
 LABELS=$LABELS$(label_match type:matrix-format /matrix/)
 LABELS=$LABELS$(label_match type:multigrid /multigrid/)

From e528cc4cad3dc7ded411a62d4593b703e76220c6 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 6 Jan 2021 20:32:08 +0100
Subject: [PATCH 0189/1008] move get_all_changed to bot-pr-base

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 .github/bot-pr-base.sh        | 21 +++++++++++++++++++++
 .github/bot-pr-format-base.sh | 18 +-----------------
 .github/label.sh              | 18 +-----------------
 3 files changed, 23 insertions(+), 34 deletions(-)

diff --git a/.github/bot-pr-base.sh b/.github/bot-pr-base.sh
index cdb0787c410..b261bf74324 100644
--- a/.github/bot-pr-base.sh
+++ b/.github/bot-pr-base.sh
@@ -59,6 +59,27 @@ bot_error() {
   exit 1
 }
 
+bot_get_all_changed_files() {
+  local pr_url="$1"
+  local pr_files=""
+  local page="1"
+  while true; do
+    # this api allows 100 items per page
+    # github action uses `bash -e`. The last empty page will leads jq error, use `|| :` to ignore the error.
+    local pr_page_files=$(api_get "$pr_url/files?&per_page=100&page=${page}" | jq -er '.[] | .filename' || :)
+    if [ "${pr_page_files}" = "" ]; then
+      break
+    fi
+    if [ ! "${pr_files}" = "" ]; then
+      # add the same new line format as jq output
+      pr_files="${pr_files}"$'\n'
+    fi
+    pr_files="${pr_files}${pr_page_files}"
+    page=$(( page + 1 ))
+  done
+  echo "${pr_files}"
+}
+
 # collect info on the user that invoked the bot
 echo -n "Collecting information on triggering user"
 USER_JSON=$(api_get $USER_URL)
diff --git a/.github/bot-pr-format-base.sh b/.github/bot-pr-format-base.sh
index 8d213dcfe69..298f061cc87 100644
--- a/.github/bot-pr-format-base.sh
+++ b/.github/bot-pr-format-base.sh
@@ -7,23 +7,7 @@ FORMAT_HEADER_REGEX='^(benchmark|core|cuda|hip|include/ginkgo/core|omp|reference
 FORMAT_REGEX='^(common|examples|test_install)/'
 
 echo "Retrieving PR file list"
-PR_FILES=""
-PAGE="1"
-while true; do
-  # this api allows 100 items per page
-  # github action uses `bash -e`. The last empty page will leads jq error, use `|| :` to ignore the error.
-  PR_PAGE_FILES=$(api_get "$PR_URL/files?&per_page=100&page=${PAGE}" | jq -er '.[] | .filename' || :)
-  if [ "${PR_PAGE_FILES}" = "" ]; then
-    break
-  fi
-  echo "Retrieving PR file list - ${PAGE} pages"
-  if [ ! "${PR_FILES}" = "" ]; then
-    # add the same new line format as jq output
-    PR_FILES="${PR_FILES}"$'\n'
-  fi
-  PR_FILES="${PR_FILES}${PR_PAGE_FILES}"
-  PAGE=$(( PAGE + 1 ))
-done
+PR_FILES=$(bot_get_all_changed_files ${PR_URL})
 NUM=$(echo "${PR_FILES}" | wc -l)
 echo "PR has ${NUM} changed files"
 
diff --git a/.github/label.sh b/.github/label.sh
index 860e520720c..8849f53d01c 100755
--- a/.github/label.sh
+++ b/.github/label.sh
@@ -3,23 +3,7 @@
 source .github/bot-pr-base.sh
 
 echo "Retrieving PR file list"
-PR_FILES=""
-PAGE="1"
-while true; do
-  # this api allows 100 items per page
-  # github action uses `bash -e`. The last empty page will leads jq error, use `|| :` to ignore the error.
-  PR_PAGE_FILES=$(api_get "$PR_URL/files?&per_page=100&page=${PAGE}" | jq -er '.[] | .filename' || :)
-  if [ "${PR_PAGE_FILES}" = "" ]; then
-    break
-  fi
-  echo "Retrieving PR file list - ${PAGE} pages"
-  if [ ! "${PR_FILES}" = "" ]; then
-    # add the same new line format as jq output
-    PR_FILES="${PR_FILES}"$'\n'
-  fi
-  PR_FILES="${PR_FILES}${PR_PAGE_FILES}"
-  PAGE=$(( PAGE + 1 ))
-done
+PR_FILES=$(bot_get_all_changed_files ${PR_URL})
 NUM=$(echo "${PR_FILES}" | wc -l)
 echo "PR has ${NUM} changed files"
 

From a659a1be19e52c77244b26d3b4b6c9089411e2ba Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 4 Jan 2021 17:44:48 +0100
Subject: [PATCH 0190/1008] bump copyright year to 2021

---
 LICENSE                                                       | 2 +-
 benchmark/conversions/conversions.cpp                         | 2 +-
 benchmark/matrix_generator/matrix_generator.cpp               | 2 +-
 benchmark/matrix_statistics/matrix_statistics.cpp             | 2 +-
 benchmark/preconditioner/preconditioner.cpp                   | 2 +-
 benchmark/solver/solver.cpp                                   | 2 +-
 benchmark/spmv/spmv.cpp                                       | 2 +-
 benchmark/utils/cuda_linops.hpp                               | 2 +-
 benchmark/utils/formats.hpp                                   | 2 +-
 benchmark/utils/general.hpp                                   | 2 +-
 benchmark/utils/hip_linops.hip.hpp                            | 2 +-
 benchmark/utils/loggers.hpp                                   | 2 +-
 benchmark/utils/overhead_linop.hpp                            | 2 +-
 benchmark/utils/preconditioners.hpp                           | 2 +-
 benchmark/utils/spmv_common.hpp                               | 2 +-
 benchmark/utils/timer.hpp                                     | 2 +-
 common/base/executor.hpp.inc                                  | 2 +-
 common/base/math.hpp.inc                                      | 2 +-
 common/components/absolute_array.hpp.inc                      | 2 +-
 common/components/atomic.hpp.inc                              | 2 +-
 common/components/diagonal_block_manipulation.hpp.inc         | 2 +-
 common/components/fill_array.hpp.inc                          | 2 +-
 common/components/intrinsics.hpp.inc                          | 2 +-
 common/components/merging.hpp.inc                             | 2 +-
 common/components/precision_conversion.hpp.inc                | 2 +-
 common/components/prefix_sum.hpp.inc                          | 2 +-
 common/components/reduction.hpp.inc                           | 2 +-
 common/components/searching.hpp.inc                           | 2 +-
 common/components/segment_scan.hpp.inc                        | 2 +-
 common/components/sorting.hpp.inc                             | 2 +-
 common/components/thread_ids.hpp.inc                          | 2 +-
 common/components/uninitialized_array.hpp.inc                 | 2 +-
 common/components/warp_blas.hpp.inc                           | 2 +-
 common/factorization/factorization_kernels.hpp.inc            | 2 +-
 common/factorization/par_ict_spgeam_kernels.hpp.inc           | 2 +-
 common/factorization/par_ict_sweep_kernels.hpp.inc            | 2 +-
 common/factorization/par_ilu_kernels.hpp.inc                  | 2 +-
 common/factorization/par_ilut_filter_kernels.hpp.inc          | 2 +-
 common/factorization/par_ilut_select_kernels.hpp.inc          | 2 +-
 common/factorization/par_ilut_spgeam_kernels.hpp.inc          | 2 +-
 common/factorization/par_ilut_sweep_kernels.hpp.inc           | 2 +-
 common/matrix/coo_kernels.hpp.inc                             | 2 +-
 common/matrix/csr_kernels.hpp.inc                             | 2 +-
 common/matrix/dense_kernels.hpp.inc                           | 2 +-
 common/matrix/diagonal_kernels.hpp.inc                        | 2 +-
 common/matrix/ell_kernels.hpp.inc                             | 2 +-
 common/matrix/hybrid_kernels.hpp.inc                          | 2 +-
 common/matrix/sellp_kernels.hpp.inc                           | 2 +-
 common/preconditioner/isai_kernels.hpp.inc                    | 2 +-
 common/preconditioner/jacobi_advanced_apply_kernel.hpp.inc    | 2 +-
 common/preconditioner/jacobi_generate_kernel.hpp.inc          | 2 +-
 common/preconditioner/jacobi_kernels.hpp.inc                  | 2 +-
 common/preconditioner/jacobi_simple_apply_kernel.hpp.inc      | 2 +-
 common/solver/bicg_kernels.hpp.inc                            | 2 +-
 common/solver/bicgstab_kernels.hpp.inc                        | 2 +-
 common/solver/cg_kernels.hpp.inc                              | 2 +-
 common/solver/cgs_kernels.hpp.inc                             | 2 +-
 common/solver/fcg_kernels.hpp.inc                             | 2 +-
 common/solver/gmres_kernels.hpp.inc                           | 2 +-
 common/solver/idr_kernels.hpp.inc                             | 2 +-
 common/solver/ir_kernels.hpp.inc                              | 2 +-
 core/base/allocator.hpp                                       | 2 +-
 core/base/array.cpp                                           | 2 +-
 core/base/combination.cpp                                     | 2 +-
 core/base/composition.cpp                                     | 2 +-
 core/base/executor.cpp                                        | 2 +-
 core/base/extended_float.hpp                                  | 2 +-
 core/base/iterator_factory.hpp                                | 2 +-
 core/base/mtx_io.cpp                                          | 2 +-
 core/base/perturbation.cpp                                    | 2 +-
 core/base/utils.hpp                                           | 2 +-
 core/base/version.cpp                                         | 2 +-
 core/components/absolute_array.hpp                            | 2 +-
 core/components/fill_array.hpp                                | 2 +-
 core/components/precision_conversion.hpp                      | 2 +-
 core/components/prefix_sum.hpp                                | 2 +-
 core/device_hooks/common_kernels.inc.cpp                      | 2 +-
 core/device_hooks/cuda_hooks.cpp                              | 2 +-
 core/device_hooks/dpcpp_hooks.cpp                             | 2 +-
 core/device_hooks/hip_hooks.cpp                               | 2 +-
 core/device_hooks/omp_hooks.cpp                               | 2 +-
 core/device_hooks/reference_hooks.cpp                         | 2 +-
 core/devices/cuda/executor.cpp                                | 2 +-
 core/devices/dpcpp/executor.cpp                               | 2 +-
 core/devices/hip/executor.cpp                                 | 2 +-
 core/devices/omp/executor.cpp                                 | 2 +-
 core/devices/reference/dummy.cpp                              | 2 +-
 core/factorization/factorization_kernels.hpp                  | 2 +-
 core/factorization/ilu.cpp                                    | 2 +-
 core/factorization/ilu_kernels.hpp                            | 2 +-
 core/factorization/par_ict.cpp                                | 2 +-
 core/factorization/par_ict_kernels.hpp                        | 2 +-
 core/factorization/par_ilu.cpp                                | 2 +-
 core/factorization/par_ilu_kernels.hpp                        | 2 +-
 core/factorization/par_ilut.cpp                               | 2 +-
 core/factorization/par_ilut_kernels.hpp                       | 2 +-
 core/log/convergence.cpp                                      | 2 +-
 core/log/logger.cpp                                           | 2 +-
 core/log/papi.cpp                                             | 2 +-
 core/log/record.cpp                                           | 2 +-
 core/log/stream.cpp                                           | 2 +-
 core/matrix/coo.cpp                                           | 2 +-
 core/matrix/coo_builder.hpp                                   | 2 +-
 core/matrix/coo_kernels.hpp                                   | 2 +-
 core/matrix/csr.cpp                                           | 2 +-
 core/matrix/csr_builder.hpp                                   | 2 +-
 core/matrix/csr_kernels.hpp                                   | 2 +-
 core/matrix/dense.cpp                                         | 2 +-
 core/matrix/dense_kernels.hpp                                 | 2 +-
 core/matrix/diagonal.cpp                                      | 2 +-
 core/matrix/diagonal_kernels.hpp                              | 2 +-
 core/matrix/ell.cpp                                           | 2 +-
 core/matrix/ell_kernels.hpp                                   | 2 +-
 core/matrix/hybrid.cpp                                        | 2 +-
 core/matrix/hybrid_kernels.hpp                                | 2 +-
 core/matrix/identity.cpp                                      | 2 +-
 core/matrix/permutation.cpp                                   | 2 +-
 core/matrix/sellp.cpp                                         | 2 +-
 core/matrix/sellp_kernels.hpp                                 | 2 +-
 core/matrix/sparsity_csr.cpp                                  | 2 +-
 core/matrix/sparsity_csr_kernels.hpp                          | 2 +-
 core/preconditioner/isai.cpp                                  | 2 +-
 core/preconditioner/isai_kernels.hpp                          | 2 +-
 core/preconditioner/jacobi.cpp                                | 2 +-
 core/preconditioner/jacobi_kernels.hpp                        | 2 +-
 core/preconditioner/jacobi_utils.hpp                          | 2 +-
 core/reorder/rcm.cpp                                          | 2 +-
 core/reorder/rcm_kernels.hpp                                  | 2 +-
 core/solver/bicg.cpp                                          | 2 +-
 core/solver/bicg_kernels.hpp                                  | 2 +-
 core/solver/bicgstab.cpp                                      | 2 +-
 core/solver/bicgstab_kernels.hpp                              | 2 +-
 core/solver/cg.cpp                                            | 2 +-
 core/solver/cg_kernels.hpp                                    | 2 +-
 core/solver/cgs.cpp                                           | 2 +-
 core/solver/cgs_kernels.hpp                                   | 2 +-
 core/solver/fcg.cpp                                           | 2 +-
 core/solver/fcg_kernels.hpp                                   | 2 +-
 core/solver/gmres.cpp                                         | 2 +-
 core/solver/gmres_kernels.hpp                                 | 2 +-
 core/solver/idr.cpp                                           | 2 +-
 core/solver/idr_kernels.hpp                                   | 2 +-
 core/solver/ir.cpp                                            | 2 +-
 core/solver/ir_kernels.hpp                                    | 2 +-
 core/solver/lower_trs.cpp                                     | 2 +-
 core/solver/lower_trs_kernels.hpp                             | 2 +-
 core/solver/upper_trs.cpp                                     | 2 +-
 core/solver/upper_trs_kernels.hpp                             | 2 +-
 core/stop/combined.cpp                                        | 2 +-
 core/stop/criterion.cpp                                       | 2 +-
 core/stop/criterion_kernels.hpp                               | 2 +-
 core/stop/iteration.cpp                                       | 2 +-
 core/stop/residual_norm.cpp                                   | 2 +-
 core/stop/residual_norm_kernels.hpp                           | 2 +-
 core/stop/time.cpp                                            | 2 +-
 core/synthesizer/implementation_selection.hpp                 | 2 +-
 core/test/base/abstract_factory.cpp                           | 2 +-
 core/test/base/allocator.cpp                                  | 2 +-
 core/test/base/array.cpp                                      | 2 +-
 core/test/base/combination.cpp                                | 2 +-
 core/test/base/composition.cpp                                | 2 +-
 core/test/base/dim.cpp                                        | 2 +-
 core/test/base/exception.cpp                                  | 2 +-
 core/test/base/exception_helpers.cpp                          | 2 +-
 core/test/base/executor.cpp                                   | 2 +-
 core/test/base/extended_float.cpp                             | 2 +-
 core/test/base/iterator_factory.cpp                           | 2 +-
 core/test/base/lin_op.cpp                                     | 2 +-
 core/test/base/math.cpp                                       | 2 +-
 core/test/base/matrix_assembly_data.cpp                       | 2 +-
 core/test/base/matrix_data.cpp                                | 2 +-
 core/test/base/mtx_io.cpp                                     | 2 +-
 core/test/base/perturbation.cpp                               | 2 +-
 core/test/base/polymorphic_object.cpp                         | 2 +-
 core/test/base/range.cpp                                      | 2 +-
 core/test/base/range_accessors.cpp                            | 2 +-
 core/test/base/sanitizers.cpp                                 | 2 +-
 core/test/base/types.cpp                                      | 2 +-
 core/test/base/utils.cpp                                      | 2 +-
 core/test/base/version.cpp                                    | 2 +-
 core/test/factorization/par_ilu.cpp                           | 2 +-
 core/test/factorization/par_ilut.cpp                          | 2 +-
 core/test/log/convergence.cpp                                 | 2 +-
 core/test/log/logger.cpp                                      | 2 +-
 core/test/log/papi.cpp                                        | 2 +-
 core/test/log/record.cpp                                      | 2 +-
 core/test/log/stream.cpp                                      | 2 +-
 core/test/matrix/coo.cpp                                      | 2 +-
 core/test/matrix/coo_builder.cpp                              | 2 +-
 core/test/matrix/csr.cpp                                      | 2 +-
 core/test/matrix/csr_builder.cpp                              | 2 +-
 core/test/matrix/dense.cpp                                    | 2 +-
 core/test/matrix/diagonal.cpp                                 | 2 +-
 core/test/matrix/ell.cpp                                      | 2 +-
 core/test/matrix/hybrid.cpp                                   | 2 +-
 core/test/matrix/identity.cpp                                 | 2 +-
 core/test/matrix/permutation.cpp                              | 2 +-
 core/test/matrix/sellp.cpp                                    | 2 +-
 core/test/matrix/sparsity_csr.cpp                             | 2 +-
 core/test/preconditioner/ilu.cpp                              | 2 +-
 core/test/preconditioner/isai.cpp                             | 2 +-
 core/test/preconditioner/jacobi.cpp                           | 2 +-
 core/test/reorder/rcm.cpp                                     | 2 +-
 core/test/solver/bicg.cpp                                     | 2 +-
 core/test/solver/bicgstab.cpp                                 | 2 +-
 core/test/solver/cg.cpp                                       | 2 +-
 core/test/solver/cgs.cpp                                      | 2 +-
 core/test/solver/fcg.cpp                                      | 2 +-
 core/test/solver/gmres.cpp                                    | 2 +-
 core/test/solver/idr.cpp                                      | 2 +-
 core/test/solver/ir.cpp                                       | 2 +-
 core/test/solver/lower_trs.cpp                                | 2 +-
 core/test/solver/upper_trs.cpp                                | 2 +-
 core/test/stop/combined.cpp                                   | 2 +-
 core/test/stop/iteration.cpp                                  | 2 +-
 core/test/stop/stopping_status.cpp                            | 2 +-
 core/test/stop/time.cpp                                       | 2 +-
 core/test/utils.hpp                                           | 2 +-
 core/test/utils/assertions.hpp                                | 2 +-
 core/test/utils/assertions_test.cpp                           | 2 +-
 core/test/utils/matrix_generator.hpp                          | 2 +-
 core/test/utils/matrix_generator_test.cpp                     | 2 +-
 core/test/utils/unsort_matrix.hpp                             | 2 +-
 core/test/utils/unsort_matrix_test.cpp                        | 2 +-
 cuda/base/config.hpp                                          | 2 +-
 cuda/base/cublas_bindings.hpp                                 | 2 +-
 cuda/base/curand_bindings.hpp                                 | 2 +-
 cuda/base/cusparse_bindings.hpp                               | 2 +-
 cuda/base/cusparse_handle.hpp                                 | 2 +-
 cuda/base/device_guard.hpp                                    | 2 +-
 cuda/base/exception.cpp                                       | 2 +-
 cuda/base/executor.cpp                                        | 2 +-
 cuda/base/math.hpp                                            | 2 +-
 cuda/base/pointer_mode_guard.hpp                              | 2 +-
 cuda/base/types.hpp                                           | 2 +-
 cuda/base/version.cpp                                         | 2 +-
 cuda/components/absolute_array.cu                             | 2 +-
 cuda/components/atomic.cuh                                    | 2 +-
 cuda/components/cooperative_groups.cuh                        | 2 +-
 cuda/components/diagonal_block_manipulation.cuh               | 2 +-
 cuda/components/fill_array.cu                                 | 2 +-
 cuda/components/format_conversion.cuh                         | 2 +-
 cuda/components/intrinsics.cuh                                | 2 +-
 cuda/components/merging.cuh                                   | 2 +-
 cuda/components/precision_conversion.cu                       | 2 +-
 cuda/components/prefix_sum.cu                                 | 2 +-
 cuda/components/prefix_sum.cuh                                | 2 +-
 cuda/components/reduction.cuh                                 | 2 +-
 cuda/components/searching.cuh                                 | 2 +-
 cuda/components/segment_scan.cuh                              | 2 +-
 cuda/components/sorting.cuh                                   | 2 +-
 cuda/components/thread_ids.cuh                                | 2 +-
 cuda/components/uninitialized_array.hpp                       | 2 +-
 cuda/components/warp_blas.cuh                                 | 2 +-
 cuda/factorization/factorization_kernels.cu                   | 2 +-
 cuda/factorization/ilu_kernels.cu                             | 2 +-
 cuda/factorization/par_ict_kernels.cu                         | 2 +-
 cuda/factorization/par_ilu_kernels.cu                         | 2 +-
 cuda/factorization/par_ilut_approx_filter_kernel.cu           | 2 +-
 cuda/factorization/par_ilut_filter_kernel.cu                  | 2 +-
 cuda/factorization/par_ilut_select_common.cu                  | 2 +-
 cuda/factorization/par_ilut_select_common.cuh                 | 2 +-
 cuda/factorization/par_ilut_select_kernel.cu                  | 2 +-
 cuda/factorization/par_ilut_spgeam_kernel.cu                  | 2 +-
 cuda/factorization/par_ilut_sweep_kernel.cu                   | 2 +-
 cuda/matrix/coo_kernels.cu                                    | 2 +-
 cuda/matrix/csr_kernels.cu                                    | 2 +-
 cuda/matrix/dense_kernels.cu                                  | 2 +-
 cuda/matrix/diagonal_kernels.cu                               | 2 +-
 cuda/matrix/ell_kernels.cu                                    | 2 +-
 cuda/matrix/hybrid_kernels.cu                                 | 2 +-
 cuda/matrix/sellp_kernels.cu                                  | 2 +-
 cuda/matrix/sparsity_csr_kernels.cu                           | 2 +-
 cuda/preconditioner/isai_kernels.cu                           | 2 +-
 cuda/preconditioner/jacobi_advanced_apply_kernel.cu           | 2 +-
 cuda/preconditioner/jacobi_common.hpp                         | 2 +-
 cuda/preconditioner/jacobi_generate_kernel.cu                 | 2 +-
 cuda/preconditioner/jacobi_kernels.cu                         | 2 +-
 cuda/preconditioner/jacobi_simple_apply_kernel.cu             | 2 +-
 cuda/reorder/rcm_kernels.cu                                   | 2 +-
 cuda/solver/bicg_kernels.cu                                   | 2 +-
 cuda/solver/bicgstab_kernels.cu                               | 2 +-
 cuda/solver/cg_kernels.cu                                     | 2 +-
 cuda/solver/cgs_kernels.cu                                    | 2 +-
 cuda/solver/common_trs_kernels.cuh                            | 2 +-
 cuda/solver/fcg_kernels.cu                                    | 2 +-
 cuda/solver/gmres_kernels.cu                                  | 2 +-
 cuda/solver/idr_kernels.cu                                    | 4 ++--
 cuda/solver/ir_kernels.cu                                     | 2 +-
 cuda/solver/lower_trs_kernels.cu                              | 2 +-
 cuda/solver/upper_trs_kernels.cu                              | 2 +-
 cuda/stop/criterion_kernels.cu                                | 2 +-
 cuda/stop/residual_norm_kernels.cu                            | 2 +-
 cuda/test/base/array.cu                                       | 2 +-
 cuda/test/base/cuda_executor.cu                               | 2 +-
 cuda/test/base/exception_helpers.cu                           | 2 +-
 cuda/test/base/lin_op.cu                                      | 2 +-
 cuda/test/base/math.cu                                        | 2 +-
 cuda/test/components/absolute_array.cpp                       | 2 +-
 cuda/test/components/cooperative_groups_kernels.cu            | 2 +-
 cuda/test/components/fill_array.cpp                           | 2 +-
 cuda/test/components/merging_kernels.cu                       | 2 +-
 cuda/test/components/precision_conversion.cpp                 | 2 +-
 cuda/test/components/prefix_sum.cpp                           | 2 +-
 cuda/test/components/searching_kernels.cu                     | 2 +-
 cuda/test/components/sorting_kernels.cu                       | 2 +-
 cuda/test/factorization/ilu_kernels.cpp                       | 2 +-
 cuda/test/factorization/par_ict_kernels.cpp                   | 2 +-
 cuda/test/factorization/par_ilu_kernels.cpp                   | 2 +-
 cuda/test/factorization/par_ilut_kernels.cpp                  | 2 +-
 cuda/test/matrix/coo_kernels.cpp                              | 2 +-
 cuda/test/matrix/csr_kernels.cpp                              | 2 +-
 cuda/test/matrix/dense_kernels.cpp                            | 2 +-
 cuda/test/matrix/diagonal_kernels.cpp                         | 2 +-
 cuda/test/matrix/ell_kernels.cpp                              | 2 +-
 cuda/test/matrix/hybrid_kernels.cpp                           | 2 +-
 cuda/test/matrix/sellp_kernels.cpp                            | 2 +-
 cuda/test/preconditioner/isai_kernels.cpp                     | 2 +-
 cuda/test/preconditioner/jacobi_kernels.cpp                   | 2 +-
 cuda/test/reorder/rcm_kernels.cpp                             | 2 +-
 cuda/test/solver/bicg_kernels.cpp                             | 2 +-
 cuda/test/solver/bicgstab_kernels.cpp                         | 2 +-
 cuda/test/solver/cg_kernels.cpp                               | 2 +-
 cuda/test/solver/cgs_kernels.cpp                              | 2 +-
 cuda/test/solver/fcg_kernels.cpp                              | 2 +-
 cuda/test/solver/gmres_kernels.cpp                            | 2 +-
 cuda/test/solver/idr_kernels.cpp                              | 2 +-
 cuda/test/solver/ir_kernels.cpp                               | 2 +-
 cuda/test/solver/lower_trs_kernels.cpp                        | 2 +-
 cuda/test/solver/upper_trs_kernels.cpp                        | 2 +-
 cuda/test/stop/criterion_kernels.cpp                          | 2 +-
 cuda/test/stop/residual_norm_kernels.cpp                      | 2 +-
 cuda/test/utils.hpp                                           | 2 +-
 cuda/test/utils/assertions_test.cpp                           | 2 +-
 doc/examples/examples.hpp.in                                  | 2 +-
 doc/headers/cuda_executor.hpp                                 | 2 +-
 doc/headers/dpcpp_executor.hpp                                | 2 +-
 doc/headers/executors.hpp                                     | 2 +-
 doc/headers/factor.hpp                                        | 2 +-
 doc/headers/hip_executor.hpp                                  | 2 +-
 doc/headers/jacobi.hpp                                        | 2 +-
 doc/headers/linop.hpp                                         | 2 +-
 doc/headers/logging.hpp                                       | 2 +-
 doc/headers/matrix_formats.hpp                                | 2 +-
 doc/headers/omp_executor.hpp                                  | 2 +-
 doc/headers/preconditioners.hpp                               | 2 +-
 doc/headers/ref_executor.hpp                                  | 2 +-
 doc/headers/solvers.hpp                                       | 2 +-
 doc/headers/stop.hpp                                          | 2 +-
 dpcpp/base/executor.dp.cpp                                    | 2 +-
 dpcpp/base/version.dp.cpp                                     | 2 +-
 dpcpp/components/absolute_array.dp.cpp                        | 2 +-
 dpcpp/components/fill_array.dp.cpp                            | 2 +-
 dpcpp/components/format_conversion.dp.hpp                     | 2 +-
 dpcpp/components/matrix_operations.dp.hpp                     | 2 +-
 dpcpp/components/precision_conversion.dp.cpp                  | 2 +-
 dpcpp/components/prefix_sum.dp.cpp                            | 2 +-
 dpcpp/factorization/factorization_kernels.dp.cpp              | 2 +-
 dpcpp/factorization/ilu_kernels.dp.cpp                        | 2 +-
 dpcpp/factorization/par_ict_kernels.dp.cpp                    | 2 +-
 dpcpp/factorization/par_ilu_kernels.dp.cpp                    | 2 +-
 dpcpp/factorization/par_ilut_kernels.dp.cpp                   | 2 +-
 dpcpp/matrix/coo_kernels.dp.cpp                               | 2 +-
 dpcpp/matrix/csr_kernels.dp.cpp                               | 2 +-
 dpcpp/matrix/dense_kernels.dp.cpp                             | 2 +-
 dpcpp/matrix/diagonal_kernels.dp.cpp                          | 2 +-
 dpcpp/matrix/ell_kernels.dp.cpp                               | 2 +-
 dpcpp/matrix/hybrid_kernels.dp.cpp                            | 2 +-
 dpcpp/matrix/sellp_kernels.dp.cpp                             | 2 +-
 dpcpp/matrix/sparsity_csr_kernels.dp.cpp                      | 2 +-
 dpcpp/preconditioner/isai_kernels.dp.cpp                      | 2 +-
 dpcpp/preconditioner/jacobi_kernels.dp.cpp                    | 2 +-
 dpcpp/reorder/rcm_kernels.dp.cpp                              | 2 +-
 dpcpp/solver/bicg_kernels.dp.cpp                              | 2 +-
 dpcpp/solver/bicgstab_kernels.dp.cpp                          | 2 +-
 dpcpp/solver/cg_kernels.dp.cpp                                | 2 +-
 dpcpp/solver/cgs_kernels.dp.cpp                               | 2 +-
 dpcpp/solver/fcg_kernels.dp.cpp                               | 2 +-
 dpcpp/solver/gmres_kernels.dp.cpp                             | 2 +-
 dpcpp/solver/idr_kernels.dp.cpp                               | 2 +-
 dpcpp/solver/ir_kernels.dp.cpp                                | 2 +-
 dpcpp/solver/lower_trs_kernels.dp.cpp                         | 2 +-
 dpcpp/solver/upper_trs_kernels.dp.cpp                         | 2 +-
 dpcpp/stop/criterion_kernels.dp.cpp                           | 2 +-
 dpcpp/stop/residual_norm_kernels.dp.cpp                       | 2 +-
 dpcpp/test/base/executor.dp.cpp                               | 2 +-
 dpcpp/test/components/absolute_array.cpp                      | 2 +-
 dpcpp/test/components/fill_array.cpp                          | 2 +-
 dpcpp/test/components/precision_conversion.cpp                | 2 +-
 dpcpp/test/stop/criterion_kernels.cpp                         | 2 +-
 dpcpp/test_dpcpp.dp.cpp                                       | 2 +-
 .../adaptiveprecision-blockjacobi.cpp                         | 2 +-
 examples/custom-logger/custom-logger.cpp                      | 2 +-
 examples/custom-matrix-format/custom-matrix-format.cpp        | 2 +-
 examples/custom-matrix-format/stencil_kernel.cu               | 2 +-
 .../custom-stopping-criterion/custom-stopping-criterion.cpp   | 2 +-
 examples/ginkgo-overhead/ginkgo-overhead.cpp                  | 2 +-
 examples/ginkgo-ranges/ginkgo-ranges.cpp                      | 2 +-
 .../ilu-preconditioned-solver/ilu-preconditioned-solver.cpp   | 2 +-
 examples/inverse-iteration/inverse-iteration.cpp              | 2 +-
 .../ir-ilu-preconditioned-solver.cpp                          | 2 +-
 examples/iterative-refinement/iterative-refinement.cpp        | 2 +-
 examples/minimal-cuda-solver/minimal-cuda-solver.cpp          | 2 +-
 examples/mixed-precision-ir/mixed-precision-ir.cpp            | 2 +-
 examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp    | 2 +-
 examples/papi-logging/papi-logging.cpp                        | 2 +-
 examples/performance-debugging/performance-debugging.cpp      | 2 +-
 examples/poisson-solver/poisson-solver.cpp                    | 2 +-
 examples/preconditioned-solver/preconditioned-solver.cpp      | 2 +-
 examples/preconditioner-export/preconditioner-export.cpp      | 2 +-
 examples/simple-solver-logging/simple-solver-logging.cpp      | 2 +-
 examples/simple-solver/simple-solver.cpp                      | 2 +-
 examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp  | 2 +-
 hip/base/config.hip.hpp                                       | 2 +-
 hip/base/device_guard.hip.hpp                                 | 2 +-
 hip/base/exception.hip.cpp                                    | 2 +-
 hip/base/executor.hip.cpp                                     | 2 +-
 hip/base/hipblas_bindings.hip.hpp                             | 2 +-
 hip/base/hiprand_bindings.hip.hpp                             | 2 +-
 hip/base/hipsparse_bindings.hip.hpp                           | 2 +-
 hip/base/math.hip.hpp                                         | 2 +-
 hip/base/pointer_mode_guard.hip.hpp                           | 2 +-
 hip/base/types.hip.hpp                                        | 2 +-
 hip/base/version.hip.cpp                                      | 2 +-
 hip/components/absolute_array.hip.cpp                         | 2 +-
 hip/components/atomic.hip.hpp                                 | 2 +-
 hip/components/cooperative_groups.hip.hpp                     | 2 +-
 hip/components/diagonal_block_manipulation.hip.hpp            | 2 +-
 hip/components/fill_array.hip.cpp                             | 2 +-
 hip/components/format_conversion.hip.hpp                      | 2 +-
 hip/components/intrinsics.hip.hpp                             | 2 +-
 hip/components/merging.hip.hpp                                | 2 +-
 hip/components/precision_conversion.hip.cpp                   | 2 +-
 hip/components/prefix_sum.hip.cpp                             | 2 +-
 hip/components/prefix_sum.hip.hpp                             | 2 +-
 hip/components/reduction.hip.hpp                              | 2 +-
 hip/components/searching.hip.hpp                              | 2 +-
 hip/components/segment_scan.hip.hpp                           | 2 +-
 hip/components/sorting.hip.hpp                                | 2 +-
 hip/components/thread_ids.hip.hpp                             | 2 +-
 hip/components/uninitialized_array.hip.hpp                    | 2 +-
 hip/components/warp_blas.hip.hpp                              | 2 +-
 hip/factorization/factorization_kernels.hip.cpp               | 2 +-
 hip/factorization/ilu_kernels.hip.cpp                         | 2 +-
 hip/factorization/par_ict_kernels.hip.cpp                     | 2 +-
 hip/factorization/par_ilu_kernels.hip.cpp                     | 2 +-
 hip/factorization/par_ilut_approx_filter_kernel.hip.cpp       | 2 +-
 hip/factorization/par_ilut_filter_kernel.hip.cpp              | 2 +-
 hip/factorization/par_ilut_select_common.hip.cpp              | 2 +-
 hip/factorization/par_ilut_select_common.hip.hpp              | 2 +-
 hip/factorization/par_ilut_select_kernel.hip.cpp              | 2 +-
 hip/factorization/par_ilut_spgeam_kernel.hip.cpp              | 2 +-
 hip/factorization/par_ilut_sweep_kernel.hip.cpp               | 2 +-
 hip/matrix/coo_kernels.hip.cpp                                | 2 +-
 hip/matrix/csr_kernels.hip.cpp                                | 2 +-
 hip/matrix/dense_kernels.hip.cpp                              | 2 +-
 hip/matrix/diagonal_kernels.hip.cpp                           | 2 +-
 hip/matrix/ell_kernels.hip.cpp                                | 2 +-
 hip/matrix/hybrid_kernels.hip.cpp                             | 2 +-
 hip/matrix/sellp_kernels.hip.cpp                              | 2 +-
 hip/matrix/sparsity_csr_kernels.hip.cpp                       | 2 +-
 hip/preconditioner/isai_kernels.hip.cpp                       | 2 +-
 hip/preconditioner/jacobi_advanced_apply_kernel.hip.cpp       | 2 +-
 hip/preconditioner/jacobi_common.hip.hpp                      | 2 +-
 hip/preconditioner/jacobi_generate_kernel.hip.cpp             | 2 +-
 hip/preconditioner/jacobi_kernels.hip.cpp                     | 2 +-
 hip/preconditioner/jacobi_simple_apply_kernel.hip.cpp         | 2 +-
 hip/reorder/rcm_kernels.hip.cpp                               | 2 +-
 hip/solver/bicg_kernels.hip.cpp                               | 2 +-
 hip/solver/bicgstab_kernels.hip.cpp                           | 2 +-
 hip/solver/cg_kernels.hip.cpp                                 | 2 +-
 hip/solver/cgs_kernels.hip.cpp                                | 2 +-
 hip/solver/common_trs_kernels.hip.hpp                         | 2 +-
 hip/solver/fcg_kernels.hip.cpp                                | 2 +-
 hip/solver/gmres_kernels.hip.cpp                              | 2 +-
 hip/solver/idr_kernels.hip.cpp                                | 4 ++--
 hip/solver/ir_kernels.hip.cpp                                 | 2 +-
 hip/solver/lower_trs_kernels.hip.cpp                          | 2 +-
 hip/solver/upper_trs_kernels.hip.cpp                          | 2 +-
 hip/stop/criterion_kernels.hip.cpp                            | 2 +-
 hip/stop/residual_norm_kernels.hip.cpp                        | 2 +-
 hip/test/base/exception_helpers.hip.cpp                       | 2 +-
 hip/test/base/hip_executor.hip.cpp                            | 2 +-
 hip/test/base/lin_op.hip.cpp                                  | 2 +-
 hip/test/base/math.hip.cpp                                    | 2 +-
 hip/test/components/absolute_array.hip.cpp                    | 2 +-
 hip/test/components/cooperative_groups_kernels.hip.cpp        | 2 +-
 hip/test/components/fill_array.hip.cpp                        | 2 +-
 hip/test/components/merging_kernels.hip.cpp                   | 2 +-
 hip/test/components/precision_conversion.hip.cpp              | 2 +-
 hip/test/components/prefix_sum.hip.cpp                        | 2 +-
 hip/test/components/searching_kernels.hip.cpp                 | 2 +-
 hip/test/components/sorting_kernels.hip.cpp                   | 2 +-
 hip/test/factorization/ilu_kernels.cpp                        | 2 +-
 hip/test/factorization/par_ict_kernels.hip.cpp                | 2 +-
 hip/test/factorization/par_ilu_kernels.hip.cpp                | 2 +-
 hip/test/factorization/par_ilut_kernels.hip.cpp               | 2 +-
 hip/test/matrix/coo_kernels.hip.cpp                           | 2 +-
 hip/test/matrix/csr_kernels.hip.cpp                           | 2 +-
 hip/test/matrix/dense_kernels.hip.cpp                         | 2 +-
 hip/test/matrix/diagonal_kernels.hip.cpp                      | 2 +-
 hip/test/matrix/ell_kernels.hip.cpp                           | 2 +-
 hip/test/matrix/hybrid_kernels.hip.cpp                        | 2 +-
 hip/test/matrix/sellp_kernels.hip.cpp                         | 2 +-
 hip/test/preconditioner/isai_kernels.hip.cpp                  | 2 +-
 hip/test/preconditioner/jacobi_kernels.cpp                    | 2 +-
 hip/test/solver/bicg_kernels.cpp                              | 2 +-
 hip/test/solver/bicgstab_kernels.cpp                          | 2 +-
 hip/test/solver/cg_kernels.cpp                                | 2 +-
 hip/test/solver/cgs_kernels.cpp                               | 2 +-
 hip/test/solver/fcg_kernels.cpp                               | 2 +-
 hip/test/solver/gmres_kernels.cpp                             | 2 +-
 hip/test/solver/idr_kernels.cpp                               | 2 +-
 hip/test/solver/ir_kernels.cpp                                | 2 +-
 hip/test/solver/lower_trs_kernels.cpp                         | 2 +-
 hip/test/solver/upper_trs_kernels.cpp                         | 2 +-
 hip/test/stop/criterion_kernels.hip.cpp                       | 2 +-
 hip/test/stop/residual_norm_kernels.cpp                       | 2 +-
 hip/test/utils.hip.hpp                                        | 2 +-
 hip/test/utils/assertions_test.hip.cpp                        | 2 +-
 include/ginkgo/config.hpp.in                                  | 2 +-
 include/ginkgo/core/base/abstract_factory.hpp                 | 2 +-
 include/ginkgo/core/base/array.hpp                            | 2 +-
 include/ginkgo/core/base/combination.hpp                      | 2 +-
 include/ginkgo/core/base/composition.hpp                      | 2 +-
 include/ginkgo/core/base/dim.hpp                              | 2 +-
 include/ginkgo/core/base/exception.hpp                        | 2 +-
 include/ginkgo/core/base/exception_helpers.hpp                | 2 +-
 include/ginkgo/core/base/executor.hpp                         | 2 +-
 include/ginkgo/core/base/lin_op.hpp                           | 2 +-
 include/ginkgo/core/base/math.hpp                             | 2 +-
 include/ginkgo/core/base/matrix_assembly_data.hpp             | 2 +-
 include/ginkgo/core/base/matrix_data.hpp                      | 2 +-
 include/ginkgo/core/base/mtx_io.hpp                           | 2 +-
 include/ginkgo/core/base/name_demangling.hpp                  | 2 +-
 include/ginkgo/core/base/perturbation.hpp                     | 2 +-
 include/ginkgo/core/base/polymorphic_object.hpp               | 2 +-
 include/ginkgo/core/base/range.hpp                            | 2 +-
 include/ginkgo/core/base/range_accessors.hpp                  | 2 +-
 include/ginkgo/core/base/std_extensions.hpp                   | 2 +-
 include/ginkgo/core/base/temporary_clone.hpp                  | 2 +-
 include/ginkgo/core/base/types.hpp                            | 2 +-
 include/ginkgo/core/base/utils.hpp                            | 2 +-
 include/ginkgo/core/base/utils_helper.hpp                     | 2 +-
 include/ginkgo/core/base/version.hpp                          | 2 +-
 include/ginkgo/core/factorization/ilu.hpp                     | 2 +-
 include/ginkgo/core/factorization/par_ict.hpp                 | 2 +-
 include/ginkgo/core/factorization/par_ilu.hpp                 | 2 +-
 include/ginkgo/core/factorization/par_ilut.hpp                | 2 +-
 include/ginkgo/core/log/convergence.hpp                       | 2 +-
 include/ginkgo/core/log/logger.hpp                            | 2 +-
 include/ginkgo/core/log/papi.hpp                              | 2 +-
 include/ginkgo/core/log/record.hpp                            | 2 +-
 include/ginkgo/core/log/stream.hpp                            | 2 +-
 include/ginkgo/core/matrix/coo.hpp                            | 2 +-
 include/ginkgo/core/matrix/csr.hpp                            | 2 +-
 include/ginkgo/core/matrix/dense.hpp                          | 2 +-
 include/ginkgo/core/matrix/diagonal.hpp                       | 2 +-
 include/ginkgo/core/matrix/ell.hpp                            | 2 +-
 include/ginkgo/core/matrix/hybrid.hpp                         | 2 +-
 include/ginkgo/core/matrix/identity.hpp                       | 2 +-
 include/ginkgo/core/matrix/permutation.hpp                    | 2 +-
 include/ginkgo/core/matrix/sellp.hpp                          | 2 +-
 include/ginkgo/core/matrix/sparsity_csr.hpp                   | 2 +-
 include/ginkgo/core/preconditioner/ilu.hpp                    | 2 +-
 include/ginkgo/core/preconditioner/isai.hpp                   | 2 +-
 include/ginkgo/core/preconditioner/jacobi.hpp                 | 2 +-
 include/ginkgo/core/reorder/rcm.hpp                           | 2 +-
 include/ginkgo/core/reorder/reordering_base.hpp               | 2 +-
 include/ginkgo/core/solver/bicg.hpp                           | 2 +-
 include/ginkgo/core/solver/bicgstab.hpp                       | 2 +-
 include/ginkgo/core/solver/cg.hpp                             | 2 +-
 include/ginkgo/core/solver/cgs.hpp                            | 2 +-
 include/ginkgo/core/solver/fcg.hpp                            | 2 +-
 include/ginkgo/core/solver/gmres.hpp                          | 2 +-
 include/ginkgo/core/solver/idr.hpp                            | 2 +-
 include/ginkgo/core/solver/ir.hpp                             | 2 +-
 include/ginkgo/core/solver/lower_trs.hpp                      | 2 +-
 include/ginkgo/core/solver/upper_trs.hpp                      | 2 +-
 include/ginkgo/core/stop/combined.hpp                         | 2 +-
 include/ginkgo/core/stop/criterion.hpp                        | 2 +-
 include/ginkgo/core/stop/iteration.hpp                        | 2 +-
 include/ginkgo/core/stop/residual_norm.hpp                    | 2 +-
 include/ginkgo/core/stop/residual_norm_reduction.hpp          | 2 +-
 include/ginkgo/core/stop/stopping_status.hpp                  | 2 +-
 include/ginkgo/core/stop/time.hpp                             | 2 +-
 include/ginkgo/core/synthesizer/containers.hpp                | 2 +-
 include/ginkgo/ginkgo.hpp                                     | 2 +-
 include/ginkgo/ginkgo.hpp.in                                  | 2 +-
 matrices/config.hpp.in                                        | 2 +-
 omp/base/version.cpp                                          | 2 +-
 omp/components/absolute_array.cpp                             | 2 +-
 omp/components/csr_spgeam.hpp                                 | 2 +-
 omp/components/fill_array.cpp                                 | 2 +-
 omp/components/format_conversion.hpp                          | 2 +-
 omp/components/matrix_operations.hpp                          | 2 +-
 omp/components/omp_mutex.hpp                                  | 2 +-
 omp/components/precision_conversion.cpp                       | 2 +-
 omp/components/prefix_sum.cpp                                 | 2 +-
 omp/components/sort_small.hpp                                 | 2 +-
 omp/factorization/factorization_kernels.cpp                   | 2 +-
 omp/factorization/ilu_kernels.cpp                             | 2 +-
 omp/factorization/par_ict_kernels.cpp                         | 2 +-
 omp/factorization/par_ilu_kernels.cpp                         | 2 +-
 omp/factorization/par_ilut_kernels.cpp                        | 2 +-
 omp/matrix/coo_kernels.cpp                                    | 2 +-
 omp/matrix/csr_kernels.cpp                                    | 2 +-
 omp/matrix/dense_kernels.cpp                                  | 2 +-
 omp/matrix/diagonal_kernels.cpp                               | 2 +-
 omp/matrix/ell_kernels.cpp                                    | 2 +-
 omp/matrix/hybrid_kernels.cpp                                 | 2 +-
 omp/matrix/sellp_kernels.cpp                                  | 2 +-
 omp/matrix/sparsity_csr_kernels.cpp                           | 2 +-
 omp/preconditioner/isai_kernels.cpp                           | 2 +-
 omp/preconditioner/jacobi_kernels.cpp                         | 2 +-
 omp/reorder/rcm_kernels.cpp                                   | 2 +-
 omp/solver/bicg_kernels.cpp                                   | 2 +-
 omp/solver/bicgstab_kernels.cpp                               | 2 +-
 omp/solver/cg_kernels.cpp                                     | 2 +-
 omp/solver/cgs_kernels.cpp                                    | 2 +-
 omp/solver/fcg_kernels.cpp                                    | 2 +-
 omp/solver/gmres_kernels.cpp                                  | 2 +-
 omp/solver/idr_kernels.cpp                                    | 4 ++--
 omp/solver/ir_kernels.cpp                                     | 2 +-
 omp/solver/lower_trs_kernels.cpp                              | 2 +-
 omp/solver/upper_trs_kernels.cpp                              | 2 +-
 omp/stop/criterion_kernels.cpp                                | 2 +-
 omp/stop/residual_norm_kernels.cpp                            | 2 +-
 omp/test/components/absolute_array.cpp                        | 2 +-
 omp/test/components/fill_array.cpp                            | 2 +-
 omp/test/components/precision_conversion.cpp                  | 2 +-
 omp/test/components/prefix_sum.cpp                            | 2 +-
 omp/test/factorization/par_ict_kernels.cpp                    | 2 +-
 omp/test/factorization/par_ilu_kernels.cpp                    | 2 +-
 omp/test/factorization/par_ilut_kernels.cpp                   | 2 +-
 omp/test/matrix/coo_kernels.cpp                               | 2 +-
 omp/test/matrix/csr_kernels.cpp                               | 2 +-
 omp/test/matrix/dense_kernels.cpp                             | 2 +-
 omp/test/matrix/diagonal_kernels.cpp                          | 2 +-
 omp/test/matrix/ell_kernels.cpp                               | 2 +-
 omp/test/matrix/hybrid_kernels.cpp                            | 2 +-
 omp/test/matrix/sellp_kernels.cpp                             | 2 +-
 omp/test/matrix/sparsity_csr_kernels.cpp                      | 2 +-
 omp/test/preconditioner/isai_kernels.cpp                      | 2 +-
 omp/test/preconditioner/jacobi_kernels.cpp                    | 2 +-
 omp/test/reorder/rcm_kernels.cpp                              | 2 +-
 omp/test/solver/bicg_kernels.cpp                              | 2 +-
 omp/test/solver/bicgstab_kernels.cpp                          | 2 +-
 omp/test/solver/cg_kernels.cpp                                | 2 +-
 omp/test/solver/cgs_kernels.cpp                               | 2 +-
 omp/test/solver/fcg_kernels.cpp                               | 2 +-
 omp/test/solver/gmres_kernels.cpp                             | 2 +-
 omp/test/solver/idr_kernels.cpp                               | 2 +-
 omp/test/solver/ir_kernels.cpp                                | 2 +-
 omp/test/solver/lower_trs_kernels.cpp                         | 2 +-
 omp/test/solver/upper_trs_kernels.cpp                         | 2 +-
 omp/test/stop/criterion_kernels.cpp                           | 2 +-
 omp/test/stop/residual_norm_kernels.cpp                       | 2 +-
 reference/base/version.cpp                                    | 2 +-
 reference/components/absolute_array.cpp                       | 2 +-
 reference/components/convert_ptrs.hpp                         | 2 +-
 reference/components/csr_spgeam.hpp                           | 2 +-
 reference/components/fill_array.cpp                           | 2 +-
 reference/components/format_conversion.hpp                    | 2 +-
 reference/components/matrix_operations.hpp                    | 2 +-
 reference/components/precision_conversion.cpp                 | 2 +-
 reference/components/prefix_sum.cpp                           | 2 +-
 reference/factorization/factorization_kernels.cpp             | 2 +-
 reference/factorization/ilu_kernels.cpp                       | 2 +-
 reference/factorization/par_ict_kernels.cpp                   | 2 +-
 reference/factorization/par_ilu_kernels.cpp                   | 2 +-
 reference/factorization/par_ilut_kernels.cpp                  | 2 +-
 reference/matrix/coo_kernels.cpp                              | 2 +-
 reference/matrix/csr_kernels.cpp                              | 2 +-
 reference/matrix/dense_kernels.cpp                            | 2 +-
 reference/matrix/diagonal_kernels.cpp                         | 2 +-
 reference/matrix/ell_kernels.cpp                              | 2 +-
 reference/matrix/hybrid_kernels.cpp                           | 2 +-
 reference/matrix/sellp_kernels.cpp                            | 2 +-
 reference/matrix/sparsity_csr_kernels.cpp                     | 2 +-
 reference/preconditioner/isai_kernels.cpp                     | 2 +-
 reference/preconditioner/jacobi_kernels.cpp                   | 2 +-
 reference/reorder/rcm_kernels.cpp                             | 2 +-
 reference/solver/bicg_kernels.cpp                             | 2 +-
 reference/solver/bicgstab_kernels.cpp                         | 2 +-
 reference/solver/cg_kernels.cpp                               | 2 +-
 reference/solver/cgs_kernels.cpp                              | 2 +-
 reference/solver/fcg_kernels.cpp                              | 2 +-
 reference/solver/gmres_kernels.cpp                            | 2 +-
 reference/solver/idr_kernels.cpp                              | 4 ++--
 reference/solver/ir_kernels.cpp                               | 2 +-
 reference/solver/lower_trs_kernels.cpp                        | 2 +-
 reference/solver/upper_trs_kernels.cpp                        | 2 +-
 reference/stop/criterion_kernels.cpp                          | 2 +-
 reference/stop/residual_norm_kernels.cpp                      | 2 +-
 reference/test/base/combination.cpp                           | 2 +-
 reference/test/base/composition.cpp                           | 2 +-
 reference/test/base/perturbation.cpp                          | 2 +-
 reference/test/base/utils.cpp                                 | 2 +-
 reference/test/components/absolute_array.cpp                  | 2 +-
 reference/test/components/fill_array.cpp                      | 2 +-
 reference/test/components/precision_conversion.cpp            | 2 +-
 reference/test/components/prefix_sum.cpp                      | 2 +-
 reference/test/factorization/par_ict_kernels.cpp              | 2 +-
 reference/test/factorization/par_ilu_kernels.cpp              | 2 +-
 reference/test/factorization/par_ilut_kernels.cpp             | 2 +-
 reference/test/log/convergence.cpp                            | 2 +-
 reference/test/log/papi.cpp                                   | 2 +-
 reference/test/matrix/coo_kernels.cpp                         | 2 +-
 reference/test/matrix/csr_kernels.cpp                         | 2 +-
 reference/test/matrix/dense_kernels.cpp                       | 2 +-
 reference/test/matrix/diagonal_kernels.cpp                    | 2 +-
 reference/test/matrix/ell_kernels.cpp                         | 2 +-
 reference/test/matrix/hybrid_kernels.cpp                      | 2 +-
 reference/test/matrix/identity.cpp                            | 2 +-
 reference/test/matrix/permutation.cpp                         | 2 +-
 reference/test/matrix/sellp_kernels.cpp                       | 2 +-
 reference/test/matrix/sparsity_csr.cpp                        | 2 +-
 reference/test/matrix/sparsity_csr_kernels.cpp                | 2 +-
 reference/test/preconditioner/ilu.cpp                         | 2 +-
 reference/test/preconditioner/isai_kernels.cpp                | 2 +-
 reference/test/preconditioner/jacobi.cpp                      | 2 +-
 reference/test/preconditioner/jacobi_kernels.cpp              | 2 +-
 reference/test/reorder/rcm.cpp                                | 2 +-
 reference/test/reorder/rcm_kernels.cpp                        | 2 +-
 reference/test/solver/bicg_kernels.cpp                        | 2 +-
 reference/test/solver/bicgstab_kernels.cpp                    | 2 +-
 reference/test/solver/cg_kernels.cpp                          | 2 +-
 reference/test/solver/cgs_kernels.cpp                         | 2 +-
 reference/test/solver/fcg_kernels.cpp                         | 2 +-
 reference/test/solver/gmres_kernels.cpp                       | 2 +-
 reference/test/solver/idr_kernels.cpp                         | 2 +-
 reference/test/solver/ir_kernels.cpp                          | 2 +-
 reference/test/solver/lower_trs.cpp                           | 2 +-
 reference/test/solver/lower_trs_kernels.cpp                   | 2 +-
 reference/test/solver/upper_trs.cpp                           | 2 +-
 reference/test/solver/upper_trs_kernels.cpp                   | 2 +-
 reference/test/stop/combined.cpp                              | 2 +-
 reference/test/stop/criterion_kernels.cpp                     | 2 +-
 reference/test/stop/iteration.cpp                             | 2 +-
 reference/test/stop/residual_norm_kernels.cpp                 | 2 +-
 reference/test/stop/time.cpp                                  | 2 +-
 reference/test/utils/assertions_test.cpp                      | 2 +-
 test_install/test_install.cpp                                 | 2 +-
 test_install/test_install_cuda.cu                             | 2 +-
 745 files changed, 749 insertions(+), 749 deletions(-)

diff --git a/LICENSE b/LICENSE
index 48867b57a87..b193691a64e 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index c7e87703102..70df82ccba4 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/matrix_generator/matrix_generator.cpp b/benchmark/matrix_generator/matrix_generator.cpp
index 7622d2cd4ed..b32810b96a5 100644
--- a/benchmark/matrix_generator/matrix_generator.cpp
+++ b/benchmark/matrix_generator/matrix_generator.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/matrix_statistics/matrix_statistics.cpp b/benchmark/matrix_statistics/matrix_statistics.cpp
index 72e899407d7..2602a16a967 100644
--- a/benchmark/matrix_statistics/matrix_statistics.cpp
+++ b/benchmark/matrix_statistics/matrix_statistics.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index 9bef693fd51..b4a1acaa741 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 5ec3da9de08..3ec32280935 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp
index 36d9b19651f..0304026c0ea 100644
--- a/benchmark/spmv/spmv.cpp
+++ b/benchmark/spmv/spmv.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/utils/cuda_linops.hpp b/benchmark/utils/cuda_linops.hpp
index 17f386016df..d51fa512f92 100644
--- a/benchmark/utils/cuda_linops.hpp
+++ b/benchmark/utils/cuda_linops.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/utils/formats.hpp b/benchmark/utils/formats.hpp
index ca5ad5f58be..259cd2bda5f 100644
--- a/benchmark/utils/formats.hpp
+++ b/benchmark/utils/formats.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index d4e6607ff5f..a9321ca2c44 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/utils/hip_linops.hip.hpp b/benchmark/utils/hip_linops.hip.hpp
index 5d62d605d24..d4af5c58b49 100644
--- a/benchmark/utils/hip_linops.hip.hpp
+++ b/benchmark/utils/hip_linops.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/utils/loggers.hpp b/benchmark/utils/loggers.hpp
index 72892891f0e..1a04bd98644 100644
--- a/benchmark/utils/loggers.hpp
+++ b/benchmark/utils/loggers.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/utils/overhead_linop.hpp b/benchmark/utils/overhead_linop.hpp
index 5bddad88325..8ade2e74f83 100644
--- a/benchmark/utils/overhead_linop.hpp
+++ b/benchmark/utils/overhead_linop.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/utils/preconditioners.hpp b/benchmark/utils/preconditioners.hpp
index 4ec84d9acd5..fd7294226df 100644
--- a/benchmark/utils/preconditioners.hpp
+++ b/benchmark/utils/preconditioners.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/utils/spmv_common.hpp b/benchmark/utils/spmv_common.hpp
index 34cd51067ae..0603d3706fb 100644
--- a/benchmark/utils/spmv_common.hpp
+++ b/benchmark/utils/spmv_common.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/benchmark/utils/timer.hpp b/benchmark/utils/timer.hpp
index c94f19d92a3..60e45367d5a 100644
--- a/benchmark/utils/timer.hpp
+++ b/benchmark/utils/timer.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/base/executor.hpp.inc b/common/base/executor.hpp.inc
index 705404b314e..0903f6dad90 100644
--- a/common/base/executor.hpp.inc
+++ b/common/base/executor.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/base/math.hpp.inc b/common/base/math.hpp.inc
index 576ab32859c..b56a11ddfef 100644
--- a/common/base/math.hpp.inc
+++ b/common/base/math.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/absolute_array.hpp.inc b/common/components/absolute_array.hpp.inc
index ee6079fb8fc..8c82324175d 100644
--- a/common/components/absolute_array.hpp.inc
+++ b/common/components/absolute_array.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/atomic.hpp.inc b/common/components/atomic.hpp.inc
index 6ae0ac5e751..40f87255284 100644
--- a/common/components/atomic.hpp.inc
+++ b/common/components/atomic.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/diagonal_block_manipulation.hpp.inc b/common/components/diagonal_block_manipulation.hpp.inc
index a9a199e33c5..baf991550e8 100644
--- a/common/components/diagonal_block_manipulation.hpp.inc
+++ b/common/components/diagonal_block_manipulation.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/fill_array.hpp.inc b/common/components/fill_array.hpp.inc
index 04e6fe67b79..1d219c7c634 100644
--- a/common/components/fill_array.hpp.inc
+++ b/common/components/fill_array.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/intrinsics.hpp.inc b/common/components/intrinsics.hpp.inc
index f89fa434eb4..fa1581c4e84 100644
--- a/common/components/intrinsics.hpp.inc
+++ b/common/components/intrinsics.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/merging.hpp.inc b/common/components/merging.hpp.inc
index c91f76e1fd4..4ddd68b5beb 100644
--- a/common/components/merging.hpp.inc
+++ b/common/components/merging.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/precision_conversion.hpp.inc b/common/components/precision_conversion.hpp.inc
index c486354f156..af59f35d058 100644
--- a/common/components/precision_conversion.hpp.inc
+++ b/common/components/precision_conversion.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/prefix_sum.hpp.inc b/common/components/prefix_sum.hpp.inc
index 01612295379..8f759b1dc95 100644
--- a/common/components/prefix_sum.hpp.inc
+++ b/common/components/prefix_sum.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/reduction.hpp.inc b/common/components/reduction.hpp.inc
index ce35337a7ff..ef56a05504f 100644
--- a/common/components/reduction.hpp.inc
+++ b/common/components/reduction.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/searching.hpp.inc b/common/components/searching.hpp.inc
index e7e558508f0..d731aae319c 100644
--- a/common/components/searching.hpp.inc
+++ b/common/components/searching.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/segment_scan.hpp.inc b/common/components/segment_scan.hpp.inc
index 9de87b6c702..f16b3794d4d 100644
--- a/common/components/segment_scan.hpp.inc
+++ b/common/components/segment_scan.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/sorting.hpp.inc b/common/components/sorting.hpp.inc
index 611b99c5538..ef5bd690937 100644
--- a/common/components/sorting.hpp.inc
+++ b/common/components/sorting.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/thread_ids.hpp.inc b/common/components/thread_ids.hpp.inc
index 3a28dad5326..c148d11b2d4 100644
--- a/common/components/thread_ids.hpp.inc
+++ b/common/components/thread_ids.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/uninitialized_array.hpp.inc b/common/components/uninitialized_array.hpp.inc
index ced072c40f4..3a8b3796c12 100644
--- a/common/components/uninitialized_array.hpp.inc
+++ b/common/components/uninitialized_array.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/components/warp_blas.hpp.inc b/common/components/warp_blas.hpp.inc
index 279ada4ae2f..128fa124d77 100644
--- a/common/components/warp_blas.hpp.inc
+++ b/common/components/warp_blas.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/factorization/factorization_kernels.hpp.inc b/common/factorization/factorization_kernels.hpp.inc
index 7050c5ce116..80178bc76a7 100644
--- a/common/factorization/factorization_kernels.hpp.inc
+++ b/common/factorization/factorization_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/factorization/par_ict_spgeam_kernels.hpp.inc b/common/factorization/par_ict_spgeam_kernels.hpp.inc
index 7a9febf3f03..6f76a14323c 100644
--- a/common/factorization/par_ict_spgeam_kernels.hpp.inc
+++ b/common/factorization/par_ict_spgeam_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/factorization/par_ict_sweep_kernels.hpp.inc b/common/factorization/par_ict_sweep_kernels.hpp.inc
index 060bacb2144..325ea25061a 100644
--- a/common/factorization/par_ict_sweep_kernels.hpp.inc
+++ b/common/factorization/par_ict_sweep_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/factorization/par_ilu_kernels.hpp.inc b/common/factorization/par_ilu_kernels.hpp.inc
index af28012cf81..f2fdc1468e1 100644
--- a/common/factorization/par_ilu_kernels.hpp.inc
+++ b/common/factorization/par_ilu_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/factorization/par_ilut_filter_kernels.hpp.inc b/common/factorization/par_ilut_filter_kernels.hpp.inc
index 2d520f33019..25b43e789ee 100644
--- a/common/factorization/par_ilut_filter_kernels.hpp.inc
+++ b/common/factorization/par_ilut_filter_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/factorization/par_ilut_select_kernels.hpp.inc b/common/factorization/par_ilut_select_kernels.hpp.inc
index 338588d5695..059069faf41 100644
--- a/common/factorization/par_ilut_select_kernels.hpp.inc
+++ b/common/factorization/par_ilut_select_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/factorization/par_ilut_spgeam_kernels.hpp.inc b/common/factorization/par_ilut_spgeam_kernels.hpp.inc
index 903968bf4a6..b8ba3b0171f 100644
--- a/common/factorization/par_ilut_spgeam_kernels.hpp.inc
+++ b/common/factorization/par_ilut_spgeam_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/factorization/par_ilut_sweep_kernels.hpp.inc b/common/factorization/par_ilut_sweep_kernels.hpp.inc
index 96cfc951b64..5b78d07f28e 100644
--- a/common/factorization/par_ilut_sweep_kernels.hpp.inc
+++ b/common/factorization/par_ilut_sweep_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/matrix/coo_kernels.hpp.inc b/common/matrix/coo_kernels.hpp.inc
index c6d550a4149..489a65729f3 100644
--- a/common/matrix/coo_kernels.hpp.inc
+++ b/common/matrix/coo_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/matrix/csr_kernels.hpp.inc b/common/matrix/csr_kernels.hpp.inc
index edd6014154b..fb88d4f9ab2 100644
--- a/common/matrix/csr_kernels.hpp.inc
+++ b/common/matrix/csr_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index 9df9caf0353..8722726da1b 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/matrix/diagonal_kernels.hpp.inc b/common/matrix/diagonal_kernels.hpp.inc
index 6e29ca2cd5e..fc1d288df8e 100644
--- a/common/matrix/diagonal_kernels.hpp.inc
+++ b/common/matrix/diagonal_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/matrix/ell_kernels.hpp.inc b/common/matrix/ell_kernels.hpp.inc
index 57991f2e06d..00eb2d6871d 100644
--- a/common/matrix/ell_kernels.hpp.inc
+++ b/common/matrix/ell_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/matrix/hybrid_kernels.hpp.inc b/common/matrix/hybrid_kernels.hpp.inc
index 3a45bd38e79..b6af7c2be36 100644
--- a/common/matrix/hybrid_kernels.hpp.inc
+++ b/common/matrix/hybrid_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/matrix/sellp_kernels.hpp.inc b/common/matrix/sellp_kernels.hpp.inc
index b15d96e6e91..b0e7ab9dc93 100644
--- a/common/matrix/sellp_kernels.hpp.inc
+++ b/common/matrix/sellp_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/preconditioner/isai_kernels.hpp.inc b/common/preconditioner/isai_kernels.hpp.inc
index 9eec6afaa04..1a71a6fbebf 100644
--- a/common/preconditioner/isai_kernels.hpp.inc
+++ b/common/preconditioner/isai_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/preconditioner/jacobi_advanced_apply_kernel.hpp.inc b/common/preconditioner/jacobi_advanced_apply_kernel.hpp.inc
index 2426728d402..abf100be7e5 100644
--- a/common/preconditioner/jacobi_advanced_apply_kernel.hpp.inc
+++ b/common/preconditioner/jacobi_advanced_apply_kernel.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/preconditioner/jacobi_generate_kernel.hpp.inc b/common/preconditioner/jacobi_generate_kernel.hpp.inc
index b402c94e5db..713dcc3dedc 100644
--- a/common/preconditioner/jacobi_generate_kernel.hpp.inc
+++ b/common/preconditioner/jacobi_generate_kernel.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/preconditioner/jacobi_kernels.hpp.inc b/common/preconditioner/jacobi_kernels.hpp.inc
index d480a0a154a..c3fa889b210 100644
--- a/common/preconditioner/jacobi_kernels.hpp.inc
+++ b/common/preconditioner/jacobi_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/preconditioner/jacobi_simple_apply_kernel.hpp.inc b/common/preconditioner/jacobi_simple_apply_kernel.hpp.inc
index c7a472bd409..db73880732d 100644
--- a/common/preconditioner/jacobi_simple_apply_kernel.hpp.inc
+++ b/common/preconditioner/jacobi_simple_apply_kernel.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/solver/bicg_kernels.hpp.inc b/common/solver/bicg_kernels.hpp.inc
index fdb8ee8f3f9..3402fa219b5 100644
--- a/common/solver/bicg_kernels.hpp.inc
+++ b/common/solver/bicg_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/solver/bicgstab_kernels.hpp.inc b/common/solver/bicgstab_kernels.hpp.inc
index 03071970fcc..dce6c3651cd 100644
--- a/common/solver/bicgstab_kernels.hpp.inc
+++ b/common/solver/bicgstab_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/solver/cg_kernels.hpp.inc b/common/solver/cg_kernels.hpp.inc
index d318c30f338..7a87fc9a9ba 100644
--- a/common/solver/cg_kernels.hpp.inc
+++ b/common/solver/cg_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/solver/cgs_kernels.hpp.inc b/common/solver/cgs_kernels.hpp.inc
index d6c3e64cd4c..0aba6af73f3 100644
--- a/common/solver/cgs_kernels.hpp.inc
+++ b/common/solver/cgs_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/solver/fcg_kernels.hpp.inc b/common/solver/fcg_kernels.hpp.inc
index 2b5b72029a2..312b0bad50d 100644
--- a/common/solver/fcg_kernels.hpp.inc
+++ b/common/solver/fcg_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/solver/gmres_kernels.hpp.inc b/common/solver/gmres_kernels.hpp.inc
index 7b991879571..9956c38a622 100644
--- a/common/solver/gmres_kernels.hpp.inc
+++ b/common/solver/gmres_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/solver/idr_kernels.hpp.inc b/common/solver/idr_kernels.hpp.inc
index 9067af54e49..fd57a31afc7 100644
--- a/common/solver/idr_kernels.hpp.inc
+++ b/common/solver/idr_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/common/solver/ir_kernels.hpp.inc b/common/solver/ir_kernels.hpp.inc
index 24a66f2795b..39a8d04f7fc 100644
--- a/common/solver/ir_kernels.hpp.inc
+++ b/common/solver/ir_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/base/allocator.hpp b/core/base/allocator.hpp
index 6a654fc56d6..791e525e037 100644
--- a/core/base/allocator.hpp
+++ b/core/base/allocator.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/base/array.cpp b/core/base/array.cpp
index 21d8b5f3326..b74da73c803 100644
--- a/core/base/array.cpp
+++ b/core/base/array.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/base/combination.cpp b/core/base/combination.cpp
index dd95298858e..61bd892af55 100644
--- a/core/base/combination.cpp
+++ b/core/base/combination.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/base/composition.cpp b/core/base/composition.cpp
index 6fb0171b56e..f5826abb396 100644
--- a/core/base/composition.cpp
+++ b/core/base/composition.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/base/executor.cpp b/core/base/executor.cpp
index 90342b30e43..109dc6601ed 100644
--- a/core/base/executor.cpp
+++ b/core/base/executor.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/base/extended_float.hpp b/core/base/extended_float.hpp
index c83e7b81f20..418a91dd9dc 100644
--- a/core/base/extended_float.hpp
+++ b/core/base/extended_float.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/base/iterator_factory.hpp b/core/base/iterator_factory.hpp
index 5e4bddeb178..0f5450fb907 100644
--- a/core/base/iterator_factory.hpp
+++ b/core/base/iterator_factory.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/base/mtx_io.cpp b/core/base/mtx_io.cpp
index b6f4d6b35c5..fea768d8c88 100644
--- a/core/base/mtx_io.cpp
+++ b/core/base/mtx_io.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/base/perturbation.cpp b/core/base/perturbation.cpp
index a7a6a0b004b..19ab053d403 100644
--- a/core/base/perturbation.cpp
+++ b/core/base/perturbation.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/base/utils.hpp b/core/base/utils.hpp
index 9842917b19d..bf09996ed3b 100644
--- a/core/base/utils.hpp
+++ b/core/base/utils.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/base/version.cpp b/core/base/version.cpp
index 7993cee5cae..37db8e1dab8 100644
--- a/core/base/version.cpp
+++ b/core/base/version.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/components/absolute_array.hpp b/core/components/absolute_array.hpp
index 746c1de733a..5f66c89254f 100644
--- a/core/components/absolute_array.hpp
+++ b/core/components/absolute_array.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/components/fill_array.hpp b/core/components/fill_array.hpp
index 452661bed84..e8b39ff8465 100644
--- a/core/components/fill_array.hpp
+++ b/core/components/fill_array.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/components/precision_conversion.hpp b/core/components/precision_conversion.hpp
index 82b684d4be1..9f21b59d2dd 100644
--- a/core/components/precision_conversion.hpp
+++ b/core/components/precision_conversion.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/components/prefix_sum.hpp b/core/components/prefix_sum.hpp
index f226ce4c282..90848050590 100644
--- a/core/components/prefix_sum.hpp
+++ b/core/components/prefix_sum.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 60ca0316f5e..cf46b6e4cae 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/device_hooks/cuda_hooks.cpp b/core/device_hooks/cuda_hooks.cpp
index 46e342d9c0a..6a0e09486a8 100644
--- a/core/device_hooks/cuda_hooks.cpp
+++ b/core/device_hooks/cuda_hooks.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/device_hooks/dpcpp_hooks.cpp b/core/device_hooks/dpcpp_hooks.cpp
index 88caac1ccd3..df17a4959e4 100644
--- a/core/device_hooks/dpcpp_hooks.cpp
+++ b/core/device_hooks/dpcpp_hooks.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/device_hooks/hip_hooks.cpp b/core/device_hooks/hip_hooks.cpp
index b98b2fcb8a4..fbc4b77cdfc 100644
--- a/core/device_hooks/hip_hooks.cpp
+++ b/core/device_hooks/hip_hooks.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/device_hooks/omp_hooks.cpp b/core/device_hooks/omp_hooks.cpp
index 131fa51a4d8..d4589755f01 100644
--- a/core/device_hooks/omp_hooks.cpp
+++ b/core/device_hooks/omp_hooks.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/device_hooks/reference_hooks.cpp b/core/device_hooks/reference_hooks.cpp
index ea7742776c8..33713edbed8 100644
--- a/core/device_hooks/reference_hooks.cpp
+++ b/core/device_hooks/reference_hooks.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/devices/cuda/executor.cpp b/core/devices/cuda/executor.cpp
index 2d6af2b91ef..61ba73ca89c 100644
--- a/core/devices/cuda/executor.cpp
+++ b/core/devices/cuda/executor.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/devices/dpcpp/executor.cpp b/core/devices/dpcpp/executor.cpp
index 90b7eb67616..7a32d5e49a3 100644
--- a/core/devices/dpcpp/executor.cpp
+++ b/core/devices/dpcpp/executor.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/devices/hip/executor.cpp b/core/devices/hip/executor.cpp
index b9a643eb7e2..565bd50279b 100644
--- a/core/devices/hip/executor.cpp
+++ b/core/devices/hip/executor.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/devices/omp/executor.cpp b/core/devices/omp/executor.cpp
index e53a1b53c43..f13101f092e 100644
--- a/core/devices/omp/executor.cpp
+++ b/core/devices/omp/executor.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/devices/reference/dummy.cpp b/core/devices/reference/dummy.cpp
index a2f3f380cbe..1c2a9b39c48 100644
--- a/core/devices/reference/dummy.cpp
+++ b/core/devices/reference/dummy.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/factorization/factorization_kernels.hpp b/core/factorization/factorization_kernels.hpp
index 96e87ed4362..02af1ed270d 100644
--- a/core/factorization/factorization_kernels.hpp
+++ b/core/factorization/factorization_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/factorization/ilu.cpp b/core/factorization/ilu.cpp
index 22655b463eb..1518853b9b8 100644
--- a/core/factorization/ilu.cpp
+++ b/core/factorization/ilu.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/factorization/ilu_kernels.hpp b/core/factorization/ilu_kernels.hpp
index 1c71ceba897..50ef5fa831b 100644
--- a/core/factorization/ilu_kernels.hpp
+++ b/core/factorization/ilu_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/factorization/par_ict.cpp b/core/factorization/par_ict.cpp
index 40d14a730d5..905c350468b 100644
--- a/core/factorization/par_ict.cpp
+++ b/core/factorization/par_ict.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/factorization/par_ict_kernels.hpp b/core/factorization/par_ict_kernels.hpp
index 5cde5fa14e2..b7bf4554238 100644
--- a/core/factorization/par_ict_kernels.hpp
+++ b/core/factorization/par_ict_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/factorization/par_ilu.cpp b/core/factorization/par_ilu.cpp
index fc984415c41..3b5bf605a4c 100644
--- a/core/factorization/par_ilu.cpp
+++ b/core/factorization/par_ilu.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/factorization/par_ilu_kernels.hpp b/core/factorization/par_ilu_kernels.hpp
index 6d8451b32e7..ea612d66ad9 100644
--- a/core/factorization/par_ilu_kernels.hpp
+++ b/core/factorization/par_ilu_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/factorization/par_ilut.cpp b/core/factorization/par_ilut.cpp
index a68a7d11c91..69370659cc3 100644
--- a/core/factorization/par_ilut.cpp
+++ b/core/factorization/par_ilut.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/factorization/par_ilut_kernels.hpp b/core/factorization/par_ilut_kernels.hpp
index 714e797e553..c484caaf81d 100644
--- a/core/factorization/par_ilut_kernels.hpp
+++ b/core/factorization/par_ilut_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/log/convergence.cpp b/core/log/convergence.cpp
index 9947e40fc60..09db4d30523 100644
--- a/core/log/convergence.cpp
+++ b/core/log/convergence.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/log/logger.cpp b/core/log/logger.cpp
index 46ee98b2895..0dc36f98948 100644
--- a/core/log/logger.cpp
+++ b/core/log/logger.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/log/papi.cpp b/core/log/papi.cpp
index 1c8a17419fa..0ae7d334333 100644
--- a/core/log/papi.cpp
+++ b/core/log/papi.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/log/record.cpp b/core/log/record.cpp
index 48026c1563b..c4543e3e4fc 100644
--- a/core/log/record.cpp
+++ b/core/log/record.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/log/stream.cpp b/core/log/stream.cpp
index 3cad7421aee..92e3f51da64 100644
--- a/core/log/stream.cpp
+++ b/core/log/stream.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/coo.cpp b/core/matrix/coo.cpp
index b256f4e4190..55ec75371aa 100644
--- a/core/matrix/coo.cpp
+++ b/core/matrix/coo.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/coo_builder.hpp b/core/matrix/coo_builder.hpp
index de323ad42b6..7c4ce38cb89 100644
--- a/core/matrix/coo_builder.hpp
+++ b/core/matrix/coo_builder.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/coo_kernels.hpp b/core/matrix/coo_kernels.hpp
index 8ae0466db26..2f2134cf183 100644
--- a/core/matrix/coo_kernels.hpp
+++ b/core/matrix/coo_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp
index fc87cdaf561..64ed24d49be 100644
--- a/core/matrix/csr.cpp
+++ b/core/matrix/csr.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/csr_builder.hpp b/core/matrix/csr_builder.hpp
index 73f892dc3a8..37b4ac76f24 100644
--- a/core/matrix/csr_builder.hpp
+++ b/core/matrix/csr_builder.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/csr_kernels.hpp b/core/matrix/csr_kernels.hpp
index 3679c353fb0..89cb7fd7e7c 100644
--- a/core/matrix/csr_kernels.hpp
+++ b/core/matrix/csr_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 0d70016ab6d..feb67558d44 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/dense_kernels.hpp b/core/matrix/dense_kernels.hpp
index 1799147d36e..cad52a11686 100644
--- a/core/matrix/dense_kernels.hpp
+++ b/core/matrix/dense_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/diagonal.cpp b/core/matrix/diagonal.cpp
index 8df8af6a801..7bca7555ae7 100644
--- a/core/matrix/diagonal.cpp
+++ b/core/matrix/diagonal.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/diagonal_kernels.hpp b/core/matrix/diagonal_kernels.hpp
index 556ab66d8a3..3db4c42478b 100644
--- a/core/matrix/diagonal_kernels.hpp
+++ b/core/matrix/diagonal_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/ell.cpp b/core/matrix/ell.cpp
index 04eb5eab482..f2855383e98 100644
--- a/core/matrix/ell.cpp
+++ b/core/matrix/ell.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/ell_kernels.hpp b/core/matrix/ell_kernels.hpp
index aa78e17881e..7a1ec261d15 100644
--- a/core/matrix/ell_kernels.hpp
+++ b/core/matrix/ell_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/hybrid.cpp b/core/matrix/hybrid.cpp
index 92768a09f89..cf5cd017ad2 100644
--- a/core/matrix/hybrid.cpp
+++ b/core/matrix/hybrid.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/hybrid_kernels.hpp b/core/matrix/hybrid_kernels.hpp
index fb813a0e428..b0222cba0bd 100644
--- a/core/matrix/hybrid_kernels.hpp
+++ b/core/matrix/hybrid_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/identity.cpp b/core/matrix/identity.cpp
index 884e5781ee8..95451e2f553 100644
--- a/core/matrix/identity.cpp
+++ b/core/matrix/identity.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/permutation.cpp b/core/matrix/permutation.cpp
index a8b6e5ff139..5e29d88ddd6 100644
--- a/core/matrix/permutation.cpp
+++ b/core/matrix/permutation.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/sellp.cpp b/core/matrix/sellp.cpp
index b34e6119e59..b5f1d9b50e8 100644
--- a/core/matrix/sellp.cpp
+++ b/core/matrix/sellp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/sellp_kernels.hpp b/core/matrix/sellp_kernels.hpp
index f57cf7087a8..a2ea6f87682 100644
--- a/core/matrix/sellp_kernels.hpp
+++ b/core/matrix/sellp_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/sparsity_csr.cpp b/core/matrix/sparsity_csr.cpp
index 851dcd946a5..b8a8bf5d087 100644
--- a/core/matrix/sparsity_csr.cpp
+++ b/core/matrix/sparsity_csr.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/sparsity_csr_kernels.hpp b/core/matrix/sparsity_csr_kernels.hpp
index 8d5e3570699..a9d17cdd133 100644
--- a/core/matrix/sparsity_csr_kernels.hpp
+++ b/core/matrix/sparsity_csr_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/preconditioner/isai.cpp b/core/preconditioner/isai.cpp
index 108ca300acc..222eaf28835 100644
--- a/core/preconditioner/isai.cpp
+++ b/core/preconditioner/isai.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/preconditioner/isai_kernels.hpp b/core/preconditioner/isai_kernels.hpp
index c8e1dcf4b91..44d25b50b7a 100644
--- a/core/preconditioner/isai_kernels.hpp
+++ b/core/preconditioner/isai_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index b59a6289a46..93abaf6a7fe 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/preconditioner/jacobi_kernels.hpp b/core/preconditioner/jacobi_kernels.hpp
index 96ae123df12..e5738e9f51f 100644
--- a/core/preconditioner/jacobi_kernels.hpp
+++ b/core/preconditioner/jacobi_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/preconditioner/jacobi_utils.hpp b/core/preconditioner/jacobi_utils.hpp
index 904820cbce2..baa7fda04b6 100644
--- a/core/preconditioner/jacobi_utils.hpp
+++ b/core/preconditioner/jacobi_utils.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/reorder/rcm.cpp b/core/reorder/rcm.cpp
index 885c2982878..f2ec8f0fd63 100644
--- a/core/reorder/rcm.cpp
+++ b/core/reorder/rcm.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/reorder/rcm_kernels.hpp b/core/reorder/rcm_kernels.hpp
index 2eeaa63a3ab..d72e8c0160a 100644
--- a/core/reorder/rcm_kernels.hpp
+++ b/core/reorder/rcm_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp
index 37568284703..d4fa8f96921 100644
--- a/core/solver/bicg.cpp
+++ b/core/solver/bicg.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/bicg_kernels.hpp b/core/solver/bicg_kernels.hpp
index c11c7f4e4a1..e715694f6e3 100644
--- a/core/solver/bicg_kernels.hpp
+++ b/core/solver/bicg_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/bicgstab.cpp b/core/solver/bicgstab.cpp
index 570c9daee6f..2a452031fcc 100644
--- a/core/solver/bicgstab.cpp
+++ b/core/solver/bicgstab.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/bicgstab_kernels.hpp b/core/solver/bicgstab_kernels.hpp
index 0ebd591a1fe..b6b5072be26 100644
--- a/core/solver/bicgstab_kernels.hpp
+++ b/core/solver/bicgstab_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/cg.cpp b/core/solver/cg.cpp
index 838ede4a882..66b8fee43df 100644
--- a/core/solver/cg.cpp
+++ b/core/solver/cg.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/cg_kernels.hpp b/core/solver/cg_kernels.hpp
index 1042b24cccd..fed7d5b97bc 100644
--- a/core/solver/cg_kernels.hpp
+++ b/core/solver/cg_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index f92f9afc30f..6532835cd75 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/cgs_kernels.hpp b/core/solver/cgs_kernels.hpp
index e2622419573..210aacbc260 100644
--- a/core/solver/cgs_kernels.hpp
+++ b/core/solver/cgs_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/fcg.cpp b/core/solver/fcg.cpp
index 595476f1637..a7f13060983 100644
--- a/core/solver/fcg.cpp
+++ b/core/solver/fcg.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/fcg_kernels.hpp b/core/solver/fcg_kernels.hpp
index 89c3e1c3504..9baa62e9e91 100644
--- a/core/solver/fcg_kernels.hpp
+++ b/core/solver/fcg_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/gmres.cpp b/core/solver/gmres.cpp
index b33d85db0de..3ee99cd3ed3 100644
--- a/core/solver/gmres.cpp
+++ b/core/solver/gmres.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/gmres_kernels.hpp b/core/solver/gmres_kernels.hpp
index 2aaa0fb509e..81ca349a99e 100644
--- a/core/solver/gmres_kernels.hpp
+++ b/core/solver/gmres_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index 43c6faaeff4..a76649b0fc7 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/idr_kernels.hpp b/core/solver/idr_kernels.hpp
index 7f2e00bae45..6a6e43f45ef 100644
--- a/core/solver/idr_kernels.hpp
+++ b/core/solver/idr_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/ir.cpp b/core/solver/ir.cpp
index 63e80f86c04..0526a701661 100644
--- a/core/solver/ir.cpp
+++ b/core/solver/ir.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/ir_kernels.hpp b/core/solver/ir_kernels.hpp
index ac76627755a..da3fc5c0997 100644
--- a/core/solver/ir_kernels.hpp
+++ b/core/solver/ir_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/lower_trs.cpp b/core/solver/lower_trs.cpp
index bb4bb19c25b..8f5b979a3b9 100644
--- a/core/solver/lower_trs.cpp
+++ b/core/solver/lower_trs.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/lower_trs_kernels.hpp b/core/solver/lower_trs_kernels.hpp
index 4430de5b468..5f9f272417a 100644
--- a/core/solver/lower_trs_kernels.hpp
+++ b/core/solver/lower_trs_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/upper_trs.cpp b/core/solver/upper_trs.cpp
index 236de82a27b..7b0fede44ea 100644
--- a/core/solver/upper_trs.cpp
+++ b/core/solver/upper_trs.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/solver/upper_trs_kernels.hpp b/core/solver/upper_trs_kernels.hpp
index e49fe8cc8e7..bdbc4a9b1d7 100644
--- a/core/solver/upper_trs_kernels.hpp
+++ b/core/solver/upper_trs_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/stop/combined.cpp b/core/stop/combined.cpp
index f80df54b90b..898c6ea56d9 100644
--- a/core/stop/combined.cpp
+++ b/core/stop/combined.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/stop/criterion.cpp b/core/stop/criterion.cpp
index 25019d7d0d7..59d1db2839e 100644
--- a/core/stop/criterion.cpp
+++ b/core/stop/criterion.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/stop/criterion_kernels.hpp b/core/stop/criterion_kernels.hpp
index c894e6337ca..0844caba099 100644
--- a/core/stop/criterion_kernels.hpp
+++ b/core/stop/criterion_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/stop/iteration.cpp b/core/stop/iteration.cpp
index 8c1a6bc5a7d..dabe099d112 100644
--- a/core/stop/iteration.cpp
+++ b/core/stop/iteration.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/stop/residual_norm.cpp b/core/stop/residual_norm.cpp
index a678226a257..d6adfdf31a0 100644
--- a/core/stop/residual_norm.cpp
+++ b/core/stop/residual_norm.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/stop/residual_norm_kernels.hpp b/core/stop/residual_norm_kernels.hpp
index 94c9b7001ad..9acbf9bd3fc 100644
--- a/core/stop/residual_norm_kernels.hpp
+++ b/core/stop/residual_norm_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/stop/time.cpp b/core/stop/time.cpp
index 8ec4ad4948a..f9f28b5365d 100644
--- a/core/stop/time.cpp
+++ b/core/stop/time.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/synthesizer/implementation_selection.hpp b/core/synthesizer/implementation_selection.hpp
index a1c1a493029..b8be20f5043 100644
--- a/core/synthesizer/implementation_selection.hpp
+++ b/core/synthesizer/implementation_selection.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/abstract_factory.cpp b/core/test/base/abstract_factory.cpp
index cf57531e7fe..15ee287d742 100644
--- a/core/test/base/abstract_factory.cpp
+++ b/core/test/base/abstract_factory.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/allocator.cpp b/core/test/base/allocator.cpp
index 32e4c9db85d..3d09052487f 100644
--- a/core/test/base/allocator.cpp
+++ b/core/test/base/allocator.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/array.cpp b/core/test/base/array.cpp
index c20f5ca72a9..91c5e8867b9 100644
--- a/core/test/base/array.cpp
+++ b/core/test/base/array.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/combination.cpp b/core/test/base/combination.cpp
index 22d41d5a0cd..de24b3eba99 100644
--- a/core/test/base/combination.cpp
+++ b/core/test/base/combination.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/composition.cpp b/core/test/base/composition.cpp
index aca88922238..628be53b08c 100644
--- a/core/test/base/composition.cpp
+++ b/core/test/base/composition.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/dim.cpp b/core/test/base/dim.cpp
index b94ef6672c1..6d48bcab794 100644
--- a/core/test/base/dim.cpp
+++ b/core/test/base/dim.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/exception.cpp b/core/test/base/exception.cpp
index de0fbb1bdd7..2c1b672f1db 100644
--- a/core/test/base/exception.cpp
+++ b/core/test/base/exception.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/exception_helpers.cpp b/core/test/base/exception_helpers.cpp
index 57b28b02d2a..c9f961f807b 100644
--- a/core/test/base/exception_helpers.cpp
+++ b/core/test/base/exception_helpers.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 488334c9ab9..5ff74d2b68e 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/extended_float.cpp b/core/test/base/extended_float.cpp
index f2f7da597bf..f1c38f8b983 100644
--- a/core/test/base/extended_float.cpp
+++ b/core/test/base/extended_float.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/iterator_factory.cpp b/core/test/base/iterator_factory.cpp
index 058a52ae58a..340398d64c9 100644
--- a/core/test/base/iterator_factory.cpp
+++ b/core/test/base/iterator_factory.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/lin_op.cpp b/core/test/base/lin_op.cpp
index e608654ca1f..1c72b536778 100644
--- a/core/test/base/lin_op.cpp
+++ b/core/test/base/lin_op.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/math.cpp b/core/test/base/math.cpp
index 288bea41576..c8f6e8725f5 100644
--- a/core/test/base/math.cpp
+++ b/core/test/base/math.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/matrix_assembly_data.cpp b/core/test/base/matrix_assembly_data.cpp
index 8ae072e39d6..d696a008410 100644
--- a/core/test/base/matrix_assembly_data.cpp
+++ b/core/test/base/matrix_assembly_data.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/matrix_data.cpp b/core/test/base/matrix_data.cpp
index fcb2f48f29b..23181ce77bd 100644
--- a/core/test/base/matrix_data.cpp
+++ b/core/test/base/matrix_data.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/mtx_io.cpp b/core/test/base/mtx_io.cpp
index e8f26d73f19..83da9f28b21 100644
--- a/core/test/base/mtx_io.cpp
+++ b/core/test/base/mtx_io.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/perturbation.cpp b/core/test/base/perturbation.cpp
index c0ddbd73cdc..c9a6f1a1440 100644
--- a/core/test/base/perturbation.cpp
+++ b/core/test/base/perturbation.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/polymorphic_object.cpp b/core/test/base/polymorphic_object.cpp
index 4fcb52c7039..62c4445d7fb 100644
--- a/core/test/base/polymorphic_object.cpp
+++ b/core/test/base/polymorphic_object.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/range.cpp b/core/test/base/range.cpp
index 6be342ca380..fbcc55b4f09 100644
--- a/core/test/base/range.cpp
+++ b/core/test/base/range.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 9066566a33a..967397e011b 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/sanitizers.cpp b/core/test/base/sanitizers.cpp
index 724b7f38871..e3b7a23b628 100644
--- a/core/test/base/sanitizers.cpp
+++ b/core/test/base/sanitizers.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/types.cpp b/core/test/base/types.cpp
index 50979cd8a67..5446d2865b1 100644
--- a/core/test/base/types.cpp
+++ b/core/test/base/types.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/utils.cpp b/core/test/base/utils.cpp
index a3bafcb5b69..5e6c1a1a956 100644
--- a/core/test/base/utils.cpp
+++ b/core/test/base/utils.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/version.cpp b/core/test/base/version.cpp
index 0fc86a03b7f..13418277cf9 100644
--- a/core/test/base/version.cpp
+++ b/core/test/base/version.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/factorization/par_ilu.cpp b/core/test/factorization/par_ilu.cpp
index 7d7226e7ca5..eeb5e1646c0 100644
--- a/core/test/factorization/par_ilu.cpp
+++ b/core/test/factorization/par_ilu.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/factorization/par_ilut.cpp b/core/test/factorization/par_ilut.cpp
index e33f0bc35b5..945dcae152b 100644
--- a/core/test/factorization/par_ilut.cpp
+++ b/core/test/factorization/par_ilut.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/log/convergence.cpp b/core/test/log/convergence.cpp
index 413c6dcd2a0..a77b9ad0db0 100644
--- a/core/test/log/convergence.cpp
+++ b/core/test/log/convergence.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/log/logger.cpp b/core/test/log/logger.cpp
index e051e16f692..483cc47093d 100644
--- a/core/test/log/logger.cpp
+++ b/core/test/log/logger.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/log/papi.cpp b/core/test/log/papi.cpp
index 1b8dba4bfd7..9c72b8233e7 100644
--- a/core/test/log/papi.cpp
+++ b/core/test/log/papi.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/log/record.cpp b/core/test/log/record.cpp
index dd829d39a8c..647691d517d 100644
--- a/core/test/log/record.cpp
+++ b/core/test/log/record.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/log/stream.cpp b/core/test/log/stream.cpp
index 3e11f1f9669..3ea1fc8b5b9 100644
--- a/core/test/log/stream.cpp
+++ b/core/test/log/stream.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/coo.cpp b/core/test/matrix/coo.cpp
index 7035b61ea74..3500dc77edc 100644
--- a/core/test/matrix/coo.cpp
+++ b/core/test/matrix/coo.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/coo_builder.cpp b/core/test/matrix/coo_builder.cpp
index 1027d84067f..67f847681d1 100644
--- a/core/test/matrix/coo_builder.cpp
+++ b/core/test/matrix/coo_builder.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/csr.cpp b/core/test/matrix/csr.cpp
index 56e88708b08..a9bcb92a7c8 100644
--- a/core/test/matrix/csr.cpp
+++ b/core/test/matrix/csr.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/csr_builder.cpp b/core/test/matrix/csr_builder.cpp
index 30336778eb3..d5a0187ce93 100644
--- a/core/test/matrix/csr_builder.cpp
+++ b/core/test/matrix/csr_builder.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/dense.cpp b/core/test/matrix/dense.cpp
index 2226f8faeec..f352648fadd 100644
--- a/core/test/matrix/dense.cpp
+++ b/core/test/matrix/dense.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/diagonal.cpp b/core/test/matrix/diagonal.cpp
index 023d319a71e..9e35a51c31e 100644
--- a/core/test/matrix/diagonal.cpp
+++ b/core/test/matrix/diagonal.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/ell.cpp b/core/test/matrix/ell.cpp
index 37cfe11e6d5..f62e17e3958 100644
--- a/core/test/matrix/ell.cpp
+++ b/core/test/matrix/ell.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/hybrid.cpp b/core/test/matrix/hybrid.cpp
index 6d65078a3ed..57bf3e24c47 100644
--- a/core/test/matrix/hybrid.cpp
+++ b/core/test/matrix/hybrid.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/identity.cpp b/core/test/matrix/identity.cpp
index a224e373fce..f2d9ad58084 100644
--- a/core/test/matrix/identity.cpp
+++ b/core/test/matrix/identity.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/permutation.cpp b/core/test/matrix/permutation.cpp
index 2c4f5a33214..c5c5fe81db1 100644
--- a/core/test/matrix/permutation.cpp
+++ b/core/test/matrix/permutation.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/sellp.cpp b/core/test/matrix/sellp.cpp
index 276f93dc370..65e3d532b20 100644
--- a/core/test/matrix/sellp.cpp
+++ b/core/test/matrix/sellp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/sparsity_csr.cpp b/core/test/matrix/sparsity_csr.cpp
index 754e159f8cd..4faf44dbccd 100644
--- a/core/test/matrix/sparsity_csr.cpp
+++ b/core/test/matrix/sparsity_csr.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/preconditioner/ilu.cpp b/core/test/preconditioner/ilu.cpp
index 66f858d1d89..c5c9e9363f2 100644
--- a/core/test/preconditioner/ilu.cpp
+++ b/core/test/preconditioner/ilu.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/preconditioner/isai.cpp b/core/test/preconditioner/isai.cpp
index f72e4f5ade5..350055c6b79 100644
--- a/core/test/preconditioner/isai.cpp
+++ b/core/test/preconditioner/isai.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/preconditioner/jacobi.cpp b/core/test/preconditioner/jacobi.cpp
index 36308f882a9..7c13ca68749 100644
--- a/core/test/preconditioner/jacobi.cpp
+++ b/core/test/preconditioner/jacobi.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/reorder/rcm.cpp b/core/test/reorder/rcm.cpp
index 4d05bb0574f..95c48c34635 100644
--- a/core/test/reorder/rcm.cpp
+++ b/core/test/reorder/rcm.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/solver/bicg.cpp b/core/test/solver/bicg.cpp
index f0e5af4bc4d..1e8de3473eb 100644
--- a/core/test/solver/bicg.cpp
+++ b/core/test/solver/bicg.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/solver/bicgstab.cpp b/core/test/solver/bicgstab.cpp
index f871e33ff0c..455d05d8d29 100644
--- a/core/test/solver/bicgstab.cpp
+++ b/core/test/solver/bicgstab.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/solver/cg.cpp b/core/test/solver/cg.cpp
index 150625b0385..b4bca2fc830 100644
--- a/core/test/solver/cg.cpp
+++ b/core/test/solver/cg.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/solver/cgs.cpp b/core/test/solver/cgs.cpp
index d6726bdc94b..4adfa5264fd 100644
--- a/core/test/solver/cgs.cpp
+++ b/core/test/solver/cgs.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/solver/fcg.cpp b/core/test/solver/fcg.cpp
index bd588d4b594..0f5663ca99c 100644
--- a/core/test/solver/fcg.cpp
+++ b/core/test/solver/fcg.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/solver/gmres.cpp b/core/test/solver/gmres.cpp
index 1d3cf385407..ceb9d1314da 100644
--- a/core/test/solver/gmres.cpp
+++ b/core/test/solver/gmres.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/solver/idr.cpp b/core/test/solver/idr.cpp
index 1eb60aad03a..a1393e62675 100644
--- a/core/test/solver/idr.cpp
+++ b/core/test/solver/idr.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/solver/ir.cpp b/core/test/solver/ir.cpp
index 6bd864b2a4a..e2069153a59 100644
--- a/core/test/solver/ir.cpp
+++ b/core/test/solver/ir.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/solver/lower_trs.cpp b/core/test/solver/lower_trs.cpp
index 8effe6c6f3d..42c6342f1d9 100644
--- a/core/test/solver/lower_trs.cpp
+++ b/core/test/solver/lower_trs.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/solver/upper_trs.cpp b/core/test/solver/upper_trs.cpp
index ae41a2b65cc..be542597a74 100644
--- a/core/test/solver/upper_trs.cpp
+++ b/core/test/solver/upper_trs.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/stop/combined.cpp b/core/test/stop/combined.cpp
index 8a443790429..28cfc9dc9df 100644
--- a/core/test/stop/combined.cpp
+++ b/core/test/stop/combined.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/stop/iteration.cpp b/core/test/stop/iteration.cpp
index aedc443eb76..f0540b419e9 100644
--- a/core/test/stop/iteration.cpp
+++ b/core/test/stop/iteration.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/stop/stopping_status.cpp b/core/test/stop/stopping_status.cpp
index d9cdebc165e..45684a3d0d6 100644
--- a/core/test/stop/stopping_status.cpp
+++ b/core/test/stop/stopping_status.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/stop/time.cpp b/core/test/stop/time.cpp
index 53966fbacad..13e8e482d87 100644
--- a/core/test/stop/time.cpp
+++ b/core/test/stop/time.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/utils.hpp b/core/test/utils.hpp
index 5838379c5f0..c4af1f9a7ba 100644
--- a/core/test/utils.hpp
+++ b/core/test/utils.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/utils/assertions.hpp b/core/test/utils/assertions.hpp
index 8eb90784d45..57e885e569c 100644
--- a/core/test/utils/assertions.hpp
+++ b/core/test/utils/assertions.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/utils/assertions_test.cpp b/core/test/utils/assertions_test.cpp
index 89d15ca585a..c5ebccb10ec 100644
--- a/core/test/utils/assertions_test.cpp
+++ b/core/test/utils/assertions_test.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/utils/matrix_generator.hpp b/core/test/utils/matrix_generator.hpp
index 171e4b2dd69..6d235a4f5eb 100644
--- a/core/test/utils/matrix_generator.hpp
+++ b/core/test/utils/matrix_generator.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/utils/matrix_generator_test.cpp b/core/test/utils/matrix_generator_test.cpp
index 8a585994dc0..1a85604a788 100644
--- a/core/test/utils/matrix_generator_test.cpp
+++ b/core/test/utils/matrix_generator_test.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/utils/unsort_matrix.hpp b/core/test/utils/unsort_matrix.hpp
index 8d1f7d7d498..b8064dfbade 100644
--- a/core/test/utils/unsort_matrix.hpp
+++ b/core/test/utils/unsort_matrix.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/utils/unsort_matrix_test.cpp b/core/test/utils/unsort_matrix_test.cpp
index c090686116d..0a242c7dc2c 100644
--- a/core/test/utils/unsort_matrix_test.cpp
+++ b/core/test/utils/unsort_matrix_test.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/base/config.hpp b/cuda/base/config.hpp
index cd69b6a2c56..3aabff70d38 100644
--- a/cuda/base/config.hpp
+++ b/cuda/base/config.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/base/cublas_bindings.hpp b/cuda/base/cublas_bindings.hpp
index 72a67d958e9..1850c5238d7 100644
--- a/cuda/base/cublas_bindings.hpp
+++ b/cuda/base/cublas_bindings.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/base/curand_bindings.hpp b/cuda/base/curand_bindings.hpp
index 033f0bd8e4c..9df87b230ef 100644
--- a/cuda/base/curand_bindings.hpp
+++ b/cuda/base/curand_bindings.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/base/cusparse_bindings.hpp b/cuda/base/cusparse_bindings.hpp
index 0cbb962bb15..efc35b7774e 100644
--- a/cuda/base/cusparse_bindings.hpp
+++ b/cuda/base/cusparse_bindings.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/base/cusparse_handle.hpp b/cuda/base/cusparse_handle.hpp
index 0f2733c0ed4..14858a2487d 100644
--- a/cuda/base/cusparse_handle.hpp
+++ b/cuda/base/cusparse_handle.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/base/device_guard.hpp b/cuda/base/device_guard.hpp
index aa347994327..63928671953 100644
--- a/cuda/base/device_guard.hpp
+++ b/cuda/base/device_guard.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/base/exception.cpp b/cuda/base/exception.cpp
index e4acaa451a6..1a3bd3c7413 100644
--- a/cuda/base/exception.cpp
+++ b/cuda/base/exception.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 7926ae98a21..fb266e36069 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/base/math.hpp b/cuda/base/math.hpp
index 7e970486a1e..1a894d3bdc2 100644
--- a/cuda/base/math.hpp
+++ b/cuda/base/math.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/base/pointer_mode_guard.hpp b/cuda/base/pointer_mode_guard.hpp
index 89bd724bedf..72ac4f372d7 100644
--- a/cuda/base/pointer_mode_guard.hpp
+++ b/cuda/base/pointer_mode_guard.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/base/types.hpp b/cuda/base/types.hpp
index 43130c12246..ab6fa968f51 100644
--- a/cuda/base/types.hpp
+++ b/cuda/base/types.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/base/version.cpp b/cuda/base/version.cpp
index 41785e5fc1f..ce043aca833 100644
--- a/cuda/base/version.cpp
+++ b/cuda/base/version.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/absolute_array.cu b/cuda/components/absolute_array.cu
index 8ef4901bf2c..09ca759cb1d 100644
--- a/cuda/components/absolute_array.cu
+++ b/cuda/components/absolute_array.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/atomic.cuh b/cuda/components/atomic.cuh
index a2b3abda458..56f757f0dba 100644
--- a/cuda/components/atomic.cuh
+++ b/cuda/components/atomic.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/cooperative_groups.cuh b/cuda/components/cooperative_groups.cuh
index d48815c0c6f..4e2305966b0 100644
--- a/cuda/components/cooperative_groups.cuh
+++ b/cuda/components/cooperative_groups.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/diagonal_block_manipulation.cuh b/cuda/components/diagonal_block_manipulation.cuh
index 9fb76112de4..c4ee79be397 100644
--- a/cuda/components/diagonal_block_manipulation.cuh
+++ b/cuda/components/diagonal_block_manipulation.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/fill_array.cu b/cuda/components/fill_array.cu
index 63344b7f94b..be4876b9aa4 100644
--- a/cuda/components/fill_array.cu
+++ b/cuda/components/fill_array.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/format_conversion.cuh b/cuda/components/format_conversion.cuh
index 17b20438524..fa411773574 100644
--- a/cuda/components/format_conversion.cuh
+++ b/cuda/components/format_conversion.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/intrinsics.cuh b/cuda/components/intrinsics.cuh
index 7726062cfa7..b2bd0643271 100644
--- a/cuda/components/intrinsics.cuh
+++ b/cuda/components/intrinsics.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/merging.cuh b/cuda/components/merging.cuh
index 80b300a4daf..a4cbd3e87e4 100644
--- a/cuda/components/merging.cuh
+++ b/cuda/components/merging.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/precision_conversion.cu b/cuda/components/precision_conversion.cu
index f98ef2cba32..335158ff439 100644
--- a/cuda/components/precision_conversion.cu
+++ b/cuda/components/precision_conversion.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/prefix_sum.cu b/cuda/components/prefix_sum.cu
index 08412bfbe60..54739c783c8 100644
--- a/cuda/components/prefix_sum.cu
+++ b/cuda/components/prefix_sum.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/prefix_sum.cuh b/cuda/components/prefix_sum.cuh
index a4f699ffee7..7f7827c79dc 100644
--- a/cuda/components/prefix_sum.cuh
+++ b/cuda/components/prefix_sum.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/reduction.cuh b/cuda/components/reduction.cuh
index 3badd97ddcc..49394cf60ba 100644
--- a/cuda/components/reduction.cuh
+++ b/cuda/components/reduction.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/searching.cuh b/cuda/components/searching.cuh
index 186123e04f3..d6c5fbf6140 100644
--- a/cuda/components/searching.cuh
+++ b/cuda/components/searching.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/segment_scan.cuh b/cuda/components/segment_scan.cuh
index b9283f8bfe9..1aa8c5b56a6 100644
--- a/cuda/components/segment_scan.cuh
+++ b/cuda/components/segment_scan.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/sorting.cuh b/cuda/components/sorting.cuh
index 9a5525f7a94..da9cfac8c94 100644
--- a/cuda/components/sorting.cuh
+++ b/cuda/components/sorting.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/thread_ids.cuh b/cuda/components/thread_ids.cuh
index 31ebe0a28a6..3133e4cbe59 100644
--- a/cuda/components/thread_ids.cuh
+++ b/cuda/components/thread_ids.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/uninitialized_array.hpp b/cuda/components/uninitialized_array.hpp
index b3d9096f0c9..84bd021b85b 100644
--- a/cuda/components/uninitialized_array.hpp
+++ b/cuda/components/uninitialized_array.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/components/warp_blas.cuh b/cuda/components/warp_blas.cuh
index 7d77ca6b354..f3e1287b850 100644
--- a/cuda/components/warp_blas.cuh
+++ b/cuda/components/warp_blas.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/factorization/factorization_kernels.cu b/cuda/factorization/factorization_kernels.cu
index 8bbd5ec3448..d98aeaf544f 100644
--- a/cuda/factorization/factorization_kernels.cu
+++ b/cuda/factorization/factorization_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/factorization/ilu_kernels.cu b/cuda/factorization/ilu_kernels.cu
index b7debb21bc3..fa1b222e26a 100644
--- a/cuda/factorization/ilu_kernels.cu
+++ b/cuda/factorization/ilu_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/factorization/par_ict_kernels.cu b/cuda/factorization/par_ict_kernels.cu
index 98aa1c04831..bd682c44a28 100644
--- a/cuda/factorization/par_ict_kernels.cu
+++ b/cuda/factorization/par_ict_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/factorization/par_ilu_kernels.cu b/cuda/factorization/par_ilu_kernels.cu
index 4c47f5e5438..8113543a90a 100644
--- a/cuda/factorization/par_ilu_kernels.cu
+++ b/cuda/factorization/par_ilu_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/factorization/par_ilut_approx_filter_kernel.cu b/cuda/factorization/par_ilut_approx_filter_kernel.cu
index 6ad16f99b5b..1e909c6d584 100644
--- a/cuda/factorization/par_ilut_approx_filter_kernel.cu
+++ b/cuda/factorization/par_ilut_approx_filter_kernel.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/factorization/par_ilut_filter_kernel.cu b/cuda/factorization/par_ilut_filter_kernel.cu
index e1f0afd4ad4..661eaf7680d 100644
--- a/cuda/factorization/par_ilut_filter_kernel.cu
+++ b/cuda/factorization/par_ilut_filter_kernel.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/factorization/par_ilut_select_common.cu b/cuda/factorization/par_ilut_select_common.cu
index e4d9a4c1e93..1373015325d 100644
--- a/cuda/factorization/par_ilut_select_common.cu
+++ b/cuda/factorization/par_ilut_select_common.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/factorization/par_ilut_select_common.cuh b/cuda/factorization/par_ilut_select_common.cuh
index 54b2a348c59..f664d41193f 100644
--- a/cuda/factorization/par_ilut_select_common.cuh
+++ b/cuda/factorization/par_ilut_select_common.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/factorization/par_ilut_select_kernel.cu b/cuda/factorization/par_ilut_select_kernel.cu
index 4fc5315e182..a2278a6c3ab 100644
--- a/cuda/factorization/par_ilut_select_kernel.cu
+++ b/cuda/factorization/par_ilut_select_kernel.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/factorization/par_ilut_spgeam_kernel.cu b/cuda/factorization/par_ilut_spgeam_kernel.cu
index 1efb704e272..29071e3c798 100644
--- a/cuda/factorization/par_ilut_spgeam_kernel.cu
+++ b/cuda/factorization/par_ilut_spgeam_kernel.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/factorization/par_ilut_sweep_kernel.cu b/cuda/factorization/par_ilut_sweep_kernel.cu
index 91b68b723da..118903a2a3c 100644
--- a/cuda/factorization/par_ilut_sweep_kernel.cu
+++ b/cuda/factorization/par_ilut_sweep_kernel.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/matrix/coo_kernels.cu b/cuda/matrix/coo_kernels.cu
index 056925c8bdd..c9c1328038b 100644
--- a/cuda/matrix/coo_kernels.cu
+++ b/cuda/matrix/coo_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/matrix/csr_kernels.cu b/cuda/matrix/csr_kernels.cu
index e2ab3e23f95..da53054727c 100644
--- a/cuda/matrix/csr_kernels.cu
+++ b/cuda/matrix/csr_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index ae5be6b21ec..0864198fec8 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/matrix/diagonal_kernels.cu b/cuda/matrix/diagonal_kernels.cu
index 3b969fb8c92..9b0e174b982 100644
--- a/cuda/matrix/diagonal_kernels.cu
+++ b/cuda/matrix/diagonal_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/matrix/ell_kernels.cu b/cuda/matrix/ell_kernels.cu
index 937c0d3b010..9bc47f71e63 100644
--- a/cuda/matrix/ell_kernels.cu
+++ b/cuda/matrix/ell_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/matrix/hybrid_kernels.cu b/cuda/matrix/hybrid_kernels.cu
index 7b731559672..153146aa62d 100644
--- a/cuda/matrix/hybrid_kernels.cu
+++ b/cuda/matrix/hybrid_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/matrix/sellp_kernels.cu b/cuda/matrix/sellp_kernels.cu
index 1f1a978734f..997b9a5ed9a 100644
--- a/cuda/matrix/sellp_kernels.cu
+++ b/cuda/matrix/sellp_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/matrix/sparsity_csr_kernels.cu b/cuda/matrix/sparsity_csr_kernels.cu
index 69d2e53fe37..b312e2a7872 100644
--- a/cuda/matrix/sparsity_csr_kernels.cu
+++ b/cuda/matrix/sparsity_csr_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/preconditioner/isai_kernels.cu b/cuda/preconditioner/isai_kernels.cu
index 858c82584d5..e3e75b01eb5 100644
--- a/cuda/preconditioner/isai_kernels.cu
+++ b/cuda/preconditioner/isai_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/preconditioner/jacobi_advanced_apply_kernel.cu b/cuda/preconditioner/jacobi_advanced_apply_kernel.cu
index 2dc9aeaf23b..5bb602e7b95 100644
--- a/cuda/preconditioner/jacobi_advanced_apply_kernel.cu
+++ b/cuda/preconditioner/jacobi_advanced_apply_kernel.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/preconditioner/jacobi_common.hpp b/cuda/preconditioner/jacobi_common.hpp
index 3c76bb78388..21920f4fe63 100644
--- a/cuda/preconditioner/jacobi_common.hpp
+++ b/cuda/preconditioner/jacobi_common.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/preconditioner/jacobi_generate_kernel.cu b/cuda/preconditioner/jacobi_generate_kernel.cu
index 0f1c52e9621..7b7113bef4d 100644
--- a/cuda/preconditioner/jacobi_generate_kernel.cu
+++ b/cuda/preconditioner/jacobi_generate_kernel.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/preconditioner/jacobi_kernels.cu b/cuda/preconditioner/jacobi_kernels.cu
index e0662499762..889fd1b0db6 100644
--- a/cuda/preconditioner/jacobi_kernels.cu
+++ b/cuda/preconditioner/jacobi_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/preconditioner/jacobi_simple_apply_kernel.cu b/cuda/preconditioner/jacobi_simple_apply_kernel.cu
index fb6721bbdca..d55ab89070e 100644
--- a/cuda/preconditioner/jacobi_simple_apply_kernel.cu
+++ b/cuda/preconditioner/jacobi_simple_apply_kernel.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/reorder/rcm_kernels.cu b/cuda/reorder/rcm_kernels.cu
index e1f35511178..a970b6e86b2 100644
--- a/cuda/reorder/rcm_kernels.cu
+++ b/cuda/reorder/rcm_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/solver/bicg_kernels.cu b/cuda/solver/bicg_kernels.cu
index 175198d26d9..414d637f01c 100644
--- a/cuda/solver/bicg_kernels.cu
+++ b/cuda/solver/bicg_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/solver/bicgstab_kernels.cu b/cuda/solver/bicgstab_kernels.cu
index a0e5376cf69..959ddc15283 100644
--- a/cuda/solver/bicgstab_kernels.cu
+++ b/cuda/solver/bicgstab_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/solver/cg_kernels.cu b/cuda/solver/cg_kernels.cu
index 9adb589a9ea..0c7df9e06d8 100644
--- a/cuda/solver/cg_kernels.cu
+++ b/cuda/solver/cg_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/solver/cgs_kernels.cu b/cuda/solver/cgs_kernels.cu
index 1c1b1af6b48..ef80b20b1e3 100644
--- a/cuda/solver/cgs_kernels.cu
+++ b/cuda/solver/cgs_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/solver/common_trs_kernels.cuh b/cuda/solver/common_trs_kernels.cuh
index f16be5ee0e1..53d6661c45f 100644
--- a/cuda/solver/common_trs_kernels.cuh
+++ b/cuda/solver/common_trs_kernels.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/solver/fcg_kernels.cu b/cuda/solver/fcg_kernels.cu
index ed92ca19120..214fd59bddb 100644
--- a/cuda/solver/fcg_kernels.cu
+++ b/cuda/solver/fcg_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/solver/gmres_kernels.cu b/cuda/solver/gmres_kernels.cu
index 0ddddfc74f7..a5bb25f06d5 100644
--- a/cuda/solver/gmres_kernels.cu
+++ b/cuda/solver/gmres_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index be430bac4e4..ea207a99408 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -33,8 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
-#include <random>
 #include <time.h>
+#include <random>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
diff --git a/cuda/solver/ir_kernels.cu b/cuda/solver/ir_kernels.cu
index 7b26ab3527f..02d6c5bf561 100644
--- a/cuda/solver/ir_kernels.cu
+++ b/cuda/solver/ir_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/solver/lower_trs_kernels.cu b/cuda/solver/lower_trs_kernels.cu
index 1cd2764d481..95da4d221a9 100644
--- a/cuda/solver/lower_trs_kernels.cu
+++ b/cuda/solver/lower_trs_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/solver/upper_trs_kernels.cu b/cuda/solver/upper_trs_kernels.cu
index 0518b11bed7..361a0738d0d 100644
--- a/cuda/solver/upper_trs_kernels.cu
+++ b/cuda/solver/upper_trs_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/stop/criterion_kernels.cu b/cuda/stop/criterion_kernels.cu
index 390f96cb2f2..2bc18045715 100644
--- a/cuda/stop/criterion_kernels.cu
+++ b/cuda/stop/criterion_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/stop/residual_norm_kernels.cu b/cuda/stop/residual_norm_kernels.cu
index 45f2c2336d5..dbb17c515fb 100644
--- a/cuda/stop/residual_norm_kernels.cu
+++ b/cuda/stop/residual_norm_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/base/array.cu b/cuda/test/base/array.cu
index 73770b36bb5..f35ac061972 100644
--- a/cuda/test/base/array.cu
+++ b/cuda/test/base/array.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/base/cuda_executor.cu b/cuda/test/base/cuda_executor.cu
index 7b944fd13a1..99db90458ee 100644
--- a/cuda/test/base/cuda_executor.cu
+++ b/cuda/test/base/cuda_executor.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/base/exception_helpers.cu b/cuda/test/base/exception_helpers.cu
index 6b05cfaaff7..f49df30bcc5 100644
--- a/cuda/test/base/exception_helpers.cu
+++ b/cuda/test/base/exception_helpers.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/base/lin_op.cu b/cuda/test/base/lin_op.cu
index 668c2df339c..cc4ca8099b7 100644
--- a/cuda/test/base/lin_op.cu
+++ b/cuda/test/base/lin_op.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/base/math.cu b/cuda/test/base/math.cu
index 08deb9a29d9..19ee73ea3eb 100644
--- a/cuda/test/base/math.cu
+++ b/cuda/test/base/math.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/components/absolute_array.cpp b/cuda/test/components/absolute_array.cpp
index 919c042cff0..4cc92ec0b17 100644
--- a/cuda/test/components/absolute_array.cpp
+++ b/cuda/test/components/absolute_array.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/components/cooperative_groups_kernels.cu b/cuda/test/components/cooperative_groups_kernels.cu
index e565a6c9952..47f1e6446cd 100644
--- a/cuda/test/components/cooperative_groups_kernels.cu
+++ b/cuda/test/components/cooperative_groups_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/components/fill_array.cpp b/cuda/test/components/fill_array.cpp
index f5a1f8734f1..1a774a0ddac 100644
--- a/cuda/test/components/fill_array.cpp
+++ b/cuda/test/components/fill_array.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/components/merging_kernels.cu b/cuda/test/components/merging_kernels.cu
index abd135b4d65..60e1605da0c 100644
--- a/cuda/test/components/merging_kernels.cu
+++ b/cuda/test/components/merging_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/components/precision_conversion.cpp b/cuda/test/components/precision_conversion.cpp
index 73751dbc1d9..87c41a9b34e 100644
--- a/cuda/test/components/precision_conversion.cpp
+++ b/cuda/test/components/precision_conversion.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/components/prefix_sum.cpp b/cuda/test/components/prefix_sum.cpp
index 6c3ad82f21e..5d3fb835156 100644
--- a/cuda/test/components/prefix_sum.cpp
+++ b/cuda/test/components/prefix_sum.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/components/searching_kernels.cu b/cuda/test/components/searching_kernels.cu
index d4f92099f4a..faadf2cf377 100644
--- a/cuda/test/components/searching_kernels.cu
+++ b/cuda/test/components/searching_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/components/sorting_kernels.cu b/cuda/test/components/sorting_kernels.cu
index cc50281177b..e973cc0f650 100644
--- a/cuda/test/components/sorting_kernels.cu
+++ b/cuda/test/components/sorting_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/factorization/ilu_kernels.cpp b/cuda/test/factorization/ilu_kernels.cpp
index bcb18de9fae..85ed7ffc05f 100644
--- a/cuda/test/factorization/ilu_kernels.cpp
+++ b/cuda/test/factorization/ilu_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/factorization/par_ict_kernels.cpp b/cuda/test/factorization/par_ict_kernels.cpp
index f052ac4bc85..2088cf5cb60 100644
--- a/cuda/test/factorization/par_ict_kernels.cpp
+++ b/cuda/test/factorization/par_ict_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/factorization/par_ilu_kernels.cpp b/cuda/test/factorization/par_ilu_kernels.cpp
index f3ae4150924..47c95061834 100644
--- a/cuda/test/factorization/par_ilu_kernels.cpp
+++ b/cuda/test/factorization/par_ilu_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/factorization/par_ilut_kernels.cpp b/cuda/test/factorization/par_ilut_kernels.cpp
index 7a66ffe2ec0..3004c2cbfd5 100644
--- a/cuda/test/factorization/par_ilut_kernels.cpp
+++ b/cuda/test/factorization/par_ilut_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/matrix/coo_kernels.cpp b/cuda/test/matrix/coo_kernels.cpp
index d436cad06d5..57593d27f9f 100644
--- a/cuda/test/matrix/coo_kernels.cpp
+++ b/cuda/test/matrix/coo_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index dd8dca752ef..2b34a817f6d 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index a21af6a2a67..f64a4cc9c39 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/matrix/diagonal_kernels.cpp b/cuda/test/matrix/diagonal_kernels.cpp
index 695eee1a880..da16185704f 100644
--- a/cuda/test/matrix/diagonal_kernels.cpp
+++ b/cuda/test/matrix/diagonal_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/matrix/ell_kernels.cpp b/cuda/test/matrix/ell_kernels.cpp
index 11ec8dbbea5..437066a84bf 100644
--- a/cuda/test/matrix/ell_kernels.cpp
+++ b/cuda/test/matrix/ell_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/matrix/hybrid_kernels.cpp b/cuda/test/matrix/hybrid_kernels.cpp
index 75ebab55443..4970e149aa3 100644
--- a/cuda/test/matrix/hybrid_kernels.cpp
+++ b/cuda/test/matrix/hybrid_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/matrix/sellp_kernels.cpp b/cuda/test/matrix/sellp_kernels.cpp
index 390c0de211b..4bf5a43cce8 100644
--- a/cuda/test/matrix/sellp_kernels.cpp
+++ b/cuda/test/matrix/sellp_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/preconditioner/isai_kernels.cpp b/cuda/test/preconditioner/isai_kernels.cpp
index fb8947e9ae6..e5673bf4bbd 100644
--- a/cuda/test/preconditioner/isai_kernels.cpp
+++ b/cuda/test/preconditioner/isai_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/preconditioner/jacobi_kernels.cpp b/cuda/test/preconditioner/jacobi_kernels.cpp
index 2d90883a852..9bbeebe1652 100644
--- a/cuda/test/preconditioner/jacobi_kernels.cpp
+++ b/cuda/test/preconditioner/jacobi_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/reorder/rcm_kernels.cpp b/cuda/test/reorder/rcm_kernels.cpp
index 9e72e604cf2..0a066918b95 100644
--- a/cuda/test/reorder/rcm_kernels.cpp
+++ b/cuda/test/reorder/rcm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/solver/bicg_kernels.cpp b/cuda/test/solver/bicg_kernels.cpp
index e58eef7e68f..c5277173ef2 100644
--- a/cuda/test/solver/bicg_kernels.cpp
+++ b/cuda/test/solver/bicg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/solver/bicgstab_kernels.cpp b/cuda/test/solver/bicgstab_kernels.cpp
index c809ad6a17b..8a742847ab7 100644
--- a/cuda/test/solver/bicgstab_kernels.cpp
+++ b/cuda/test/solver/bicgstab_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/solver/cg_kernels.cpp b/cuda/test/solver/cg_kernels.cpp
index 65f8d78781f..3f9f1b94e8f 100644
--- a/cuda/test/solver/cg_kernels.cpp
+++ b/cuda/test/solver/cg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/solver/cgs_kernels.cpp b/cuda/test/solver/cgs_kernels.cpp
index 3e49804ddab..ca530ed5a5a 100644
--- a/cuda/test/solver/cgs_kernels.cpp
+++ b/cuda/test/solver/cgs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/solver/fcg_kernels.cpp b/cuda/test/solver/fcg_kernels.cpp
index 2b5f3ac5441..0edddbcc606 100644
--- a/cuda/test/solver/fcg_kernels.cpp
+++ b/cuda/test/solver/fcg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/solver/gmres_kernels.cpp b/cuda/test/solver/gmres_kernels.cpp
index 2dcd4d2653c..103b3f9edfb 100644
--- a/cuda/test/solver/gmres_kernels.cpp
+++ b/cuda/test/solver/gmres_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index 4e44aa66520..ef80acf5965 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/solver/ir_kernels.cpp b/cuda/test/solver/ir_kernels.cpp
index 35b844274b9..88e89456128 100644
--- a/cuda/test/solver/ir_kernels.cpp
+++ b/cuda/test/solver/ir_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/solver/lower_trs_kernels.cpp b/cuda/test/solver/lower_trs_kernels.cpp
index b677b9eb10d..ea63bbf0f44 100644
--- a/cuda/test/solver/lower_trs_kernels.cpp
+++ b/cuda/test/solver/lower_trs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/solver/upper_trs_kernels.cpp b/cuda/test/solver/upper_trs_kernels.cpp
index 9da9d33930c..fbafd9f3d23 100644
--- a/cuda/test/solver/upper_trs_kernels.cpp
+++ b/cuda/test/solver/upper_trs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/stop/criterion_kernels.cpp b/cuda/test/stop/criterion_kernels.cpp
index 8265ffea284..882c6730424 100644
--- a/cuda/test/stop/criterion_kernels.cpp
+++ b/cuda/test/stop/criterion_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/stop/residual_norm_kernels.cpp b/cuda/test/stop/residual_norm_kernels.cpp
index ec5dc3bf511..f962c243d3d 100644
--- a/cuda/test/stop/residual_norm_kernels.cpp
+++ b/cuda/test/stop/residual_norm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/utils.hpp b/cuda/test/utils.hpp
index 903ed6a77c3..4b6e0e0a667 100644
--- a/cuda/test/utils.hpp
+++ b/cuda/test/utils.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/utils/assertions_test.cpp b/cuda/test/utils/assertions_test.cpp
index 71a2fb0109b..6dccae4b02a 100644
--- a/cuda/test/utils/assertions_test.cpp
+++ b/cuda/test/utils/assertions_test.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/examples/examples.hpp.in b/doc/examples/examples.hpp.in
index 5290b4ecaf6..b511b71e48f 100644
--- a/doc/examples/examples.hpp.in
+++ b/doc/examples/examples.hpp.in
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/headers/cuda_executor.hpp b/doc/headers/cuda_executor.hpp
index 52b9307357a..75aea04914c 100644
--- a/doc/headers/cuda_executor.hpp
+++ b/doc/headers/cuda_executor.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/headers/dpcpp_executor.hpp b/doc/headers/dpcpp_executor.hpp
index bef39d461db..f41571209fe 100644
--- a/doc/headers/dpcpp_executor.hpp
+++ b/doc/headers/dpcpp_executor.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/headers/executors.hpp b/doc/headers/executors.hpp
index 1a07f985395..4b6fae3b025 100644
--- a/doc/headers/executors.hpp
+++ b/doc/headers/executors.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/headers/factor.hpp b/doc/headers/factor.hpp
index 320668cabae..4f2b9260e53 100644
--- a/doc/headers/factor.hpp
+++ b/doc/headers/factor.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/headers/hip_executor.hpp b/doc/headers/hip_executor.hpp
index 4805cfb4b87..da4b8303cb1 100644
--- a/doc/headers/hip_executor.hpp
+++ b/doc/headers/hip_executor.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/headers/jacobi.hpp b/doc/headers/jacobi.hpp
index 875efa2c4d0..8eb87bc0a50 100644
--- a/doc/headers/jacobi.hpp
+++ b/doc/headers/jacobi.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/headers/linop.hpp b/doc/headers/linop.hpp
index 12fc582eb9d..89f1811d1f5 100644
--- a/doc/headers/linop.hpp
+++ b/doc/headers/linop.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/headers/logging.hpp b/doc/headers/logging.hpp
index e9563469b87..0259932e407 100644
--- a/doc/headers/logging.hpp
+++ b/doc/headers/logging.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/headers/matrix_formats.hpp b/doc/headers/matrix_formats.hpp
index 641cb98bc13..1114bec1df2 100644
--- a/doc/headers/matrix_formats.hpp
+++ b/doc/headers/matrix_formats.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/headers/omp_executor.hpp b/doc/headers/omp_executor.hpp
index 83df1f5b292..f0b0783afba 100644
--- a/doc/headers/omp_executor.hpp
+++ b/doc/headers/omp_executor.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/headers/preconditioners.hpp b/doc/headers/preconditioners.hpp
index b7797b92d1b..de71c54804d 100644
--- a/doc/headers/preconditioners.hpp
+++ b/doc/headers/preconditioners.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/headers/ref_executor.hpp b/doc/headers/ref_executor.hpp
index c4faf61e2c4..0d69b4dc798 100644
--- a/doc/headers/ref_executor.hpp
+++ b/doc/headers/ref_executor.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/headers/solvers.hpp b/doc/headers/solvers.hpp
index ac0f797cf02..bca38ef834e 100644
--- a/doc/headers/solvers.hpp
+++ b/doc/headers/solvers.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/doc/headers/stop.hpp b/doc/headers/stop.hpp
index 16ce4487601..cb5b412839c 100644
--- a/doc/headers/stop.hpp
+++ b/doc/headers/stop.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 92089616c02..0807c0b59ce 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/base/version.dp.cpp b/dpcpp/base/version.dp.cpp
index 229a72ca72c..e6875535486 100644
--- a/dpcpp/base/version.dp.cpp
+++ b/dpcpp/base/version.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/components/absolute_array.dp.cpp b/dpcpp/components/absolute_array.dp.cpp
index 830752de343..6562ac8d2c8 100644
--- a/dpcpp/components/absolute_array.dp.cpp
+++ b/dpcpp/components/absolute_array.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/components/fill_array.dp.cpp b/dpcpp/components/fill_array.dp.cpp
index 98328484be2..8006ddd16c9 100644
--- a/dpcpp/components/fill_array.dp.cpp
+++ b/dpcpp/components/fill_array.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/components/format_conversion.dp.hpp b/dpcpp/components/format_conversion.dp.hpp
index 0b603e7bcf2..404637daed2 100644
--- a/dpcpp/components/format_conversion.dp.hpp
+++ b/dpcpp/components/format_conversion.dp.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/components/matrix_operations.dp.hpp b/dpcpp/components/matrix_operations.dp.hpp
index 7c39735f485..0768242ce90 100644
--- a/dpcpp/components/matrix_operations.dp.hpp
+++ b/dpcpp/components/matrix_operations.dp.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/components/precision_conversion.dp.cpp b/dpcpp/components/precision_conversion.dp.cpp
index 41f9c66833a..27c8098bd4b 100644
--- a/dpcpp/components/precision_conversion.dp.cpp
+++ b/dpcpp/components/precision_conversion.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/components/prefix_sum.dp.cpp b/dpcpp/components/prefix_sum.dp.cpp
index 6740a7e966c..7b1a5a2bf6d 100644
--- a/dpcpp/components/prefix_sum.dp.cpp
+++ b/dpcpp/components/prefix_sum.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/factorization/factorization_kernels.dp.cpp b/dpcpp/factorization/factorization_kernels.dp.cpp
index 5e33d368733..ebb45716c45 100644
--- a/dpcpp/factorization/factorization_kernels.dp.cpp
+++ b/dpcpp/factorization/factorization_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/factorization/ilu_kernels.dp.cpp b/dpcpp/factorization/ilu_kernels.dp.cpp
index 1d854a35a3d..53a4ee2c636 100644
--- a/dpcpp/factorization/ilu_kernels.dp.cpp
+++ b/dpcpp/factorization/ilu_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/factorization/par_ict_kernels.dp.cpp b/dpcpp/factorization/par_ict_kernels.dp.cpp
index c99d4e813cd..cbf12277050 100644
--- a/dpcpp/factorization/par_ict_kernels.dp.cpp
+++ b/dpcpp/factorization/par_ict_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/factorization/par_ilu_kernels.dp.cpp b/dpcpp/factorization/par_ilu_kernels.dp.cpp
index cd1ae44ce37..4087eed7717 100644
--- a/dpcpp/factorization/par_ilu_kernels.dp.cpp
+++ b/dpcpp/factorization/par_ilu_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/factorization/par_ilut_kernels.dp.cpp b/dpcpp/factorization/par_ilut_kernels.dp.cpp
index 80b4e11c219..5b224275827 100644
--- a/dpcpp/factorization/par_ilut_kernels.dp.cpp
+++ b/dpcpp/factorization/par_ilut_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/matrix/coo_kernels.dp.cpp b/dpcpp/matrix/coo_kernels.dp.cpp
index 57f70edf499..2bf6987213e 100644
--- a/dpcpp/matrix/coo_kernels.dp.cpp
+++ b/dpcpp/matrix/coo_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index 9220939777f..9148acb4743 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index b9a759a1123..251ffbf18d6 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/matrix/diagonal_kernels.dp.cpp b/dpcpp/matrix/diagonal_kernels.dp.cpp
index 8825daf65c7..c2910c60f0a 100644
--- a/dpcpp/matrix/diagonal_kernels.dp.cpp
+++ b/dpcpp/matrix/diagonal_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/matrix/ell_kernels.dp.cpp b/dpcpp/matrix/ell_kernels.dp.cpp
index f0bb1dbf6f6..01a40b5d9b5 100644
--- a/dpcpp/matrix/ell_kernels.dp.cpp
+++ b/dpcpp/matrix/ell_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/matrix/hybrid_kernels.dp.cpp b/dpcpp/matrix/hybrid_kernels.dp.cpp
index 327903c51ab..b5d9377f990 100644
--- a/dpcpp/matrix/hybrid_kernels.dp.cpp
+++ b/dpcpp/matrix/hybrid_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/matrix/sellp_kernels.dp.cpp b/dpcpp/matrix/sellp_kernels.dp.cpp
index 310c3706e88..4547accd805 100644
--- a/dpcpp/matrix/sellp_kernels.dp.cpp
+++ b/dpcpp/matrix/sellp_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/matrix/sparsity_csr_kernels.dp.cpp b/dpcpp/matrix/sparsity_csr_kernels.dp.cpp
index cfc21f27727..39203fcd3b6 100644
--- a/dpcpp/matrix/sparsity_csr_kernels.dp.cpp
+++ b/dpcpp/matrix/sparsity_csr_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/preconditioner/isai_kernels.dp.cpp b/dpcpp/preconditioner/isai_kernels.dp.cpp
index a8aa955174c..bbad36ea9ae 100644
--- a/dpcpp/preconditioner/isai_kernels.dp.cpp
+++ b/dpcpp/preconditioner/isai_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/preconditioner/jacobi_kernels.dp.cpp b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
index 635f08ca87e..623e2c121ec 100644
--- a/dpcpp/preconditioner/jacobi_kernels.dp.cpp
+++ b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/reorder/rcm_kernels.dp.cpp b/dpcpp/reorder/rcm_kernels.dp.cpp
index ef05c1f07be..baf60231bf5 100644
--- a/dpcpp/reorder/rcm_kernels.dp.cpp
+++ b/dpcpp/reorder/rcm_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/solver/bicg_kernels.dp.cpp b/dpcpp/solver/bicg_kernels.dp.cpp
index bb1b6e0f49d..db558863934 100644
--- a/dpcpp/solver/bicg_kernels.dp.cpp
+++ b/dpcpp/solver/bicg_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/solver/bicgstab_kernels.dp.cpp b/dpcpp/solver/bicgstab_kernels.dp.cpp
index 25b133f087e..70ecd523193 100644
--- a/dpcpp/solver/bicgstab_kernels.dp.cpp
+++ b/dpcpp/solver/bicgstab_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/solver/cg_kernels.dp.cpp b/dpcpp/solver/cg_kernels.dp.cpp
index 2e5bf4bc47b..d7f85000be1 100644
--- a/dpcpp/solver/cg_kernels.dp.cpp
+++ b/dpcpp/solver/cg_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/solver/cgs_kernels.dp.cpp b/dpcpp/solver/cgs_kernels.dp.cpp
index 42e1ff9b37e..ade102c1ac2 100644
--- a/dpcpp/solver/cgs_kernels.dp.cpp
+++ b/dpcpp/solver/cgs_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/solver/fcg_kernels.dp.cpp b/dpcpp/solver/fcg_kernels.dp.cpp
index 70bff10d329..752beaa1884 100644
--- a/dpcpp/solver/fcg_kernels.dp.cpp
+++ b/dpcpp/solver/fcg_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/solver/gmres_kernels.dp.cpp b/dpcpp/solver/gmres_kernels.dp.cpp
index 8a174cb22f8..0853eeb1804 100644
--- a/dpcpp/solver/gmres_kernels.dp.cpp
+++ b/dpcpp/solver/gmres_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/solver/idr_kernels.dp.cpp b/dpcpp/solver/idr_kernels.dp.cpp
index 016df356b10..9bb956f3807 100644
--- a/dpcpp/solver/idr_kernels.dp.cpp
+++ b/dpcpp/solver/idr_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/solver/ir_kernels.dp.cpp b/dpcpp/solver/ir_kernels.dp.cpp
index 03f1150e004..a60d9b5273d 100644
--- a/dpcpp/solver/ir_kernels.dp.cpp
+++ b/dpcpp/solver/ir_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/solver/lower_trs_kernels.dp.cpp b/dpcpp/solver/lower_trs_kernels.dp.cpp
index 6b6d71b30fd..7144108593f 100644
--- a/dpcpp/solver/lower_trs_kernels.dp.cpp
+++ b/dpcpp/solver/lower_trs_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/solver/upper_trs_kernels.dp.cpp b/dpcpp/solver/upper_trs_kernels.dp.cpp
index 2d7afa5dbc8..cc1d40f711d 100644
--- a/dpcpp/solver/upper_trs_kernels.dp.cpp
+++ b/dpcpp/solver/upper_trs_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/stop/criterion_kernels.dp.cpp b/dpcpp/stop/criterion_kernels.dp.cpp
index bc59cc1cae8..46dc6243ad8 100644
--- a/dpcpp/stop/criterion_kernels.dp.cpp
+++ b/dpcpp/stop/criterion_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/stop/residual_norm_kernels.dp.cpp b/dpcpp/stop/residual_norm_kernels.dp.cpp
index e42861c38bc..0e1c6bf9642 100644
--- a/dpcpp/stop/residual_norm_kernels.dp.cpp
+++ b/dpcpp/stop/residual_norm_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/test/base/executor.dp.cpp b/dpcpp/test/base/executor.dp.cpp
index 090c9bf671f..a83db327226 100644
--- a/dpcpp/test/base/executor.dp.cpp
+++ b/dpcpp/test/base/executor.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/test/components/absolute_array.cpp b/dpcpp/test/components/absolute_array.cpp
index 7dfe989de4b..71156c15fe9 100644
--- a/dpcpp/test/components/absolute_array.cpp
+++ b/dpcpp/test/components/absolute_array.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/test/components/fill_array.cpp b/dpcpp/test/components/fill_array.cpp
index 5b2be7d06f7..1f8f6e04858 100644
--- a/dpcpp/test/components/fill_array.cpp
+++ b/dpcpp/test/components/fill_array.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/test/components/precision_conversion.cpp b/dpcpp/test/components/precision_conversion.cpp
index 660042f1afb..dc8e331d8fa 100644
--- a/dpcpp/test/components/precision_conversion.cpp
+++ b/dpcpp/test/components/precision_conversion.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/test/stop/criterion_kernels.cpp b/dpcpp/test/stop/criterion_kernels.cpp
index 5bdfc7a9d3e..cdb7f00cb46 100644
--- a/dpcpp/test/stop/criterion_kernels.cpp
+++ b/dpcpp/test/stop/criterion_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/test_dpcpp.dp.cpp b/dpcpp/test_dpcpp.dp.cpp
index 1e4d05fde18..33771901780 100644
--- a/dpcpp/test_dpcpp.dp.cpp
+++ b/dpcpp/test_dpcpp.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp b/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
index 2e5cd58ba3a..261c9428824 100644
--- a/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
+++ b/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/custom-logger/custom-logger.cpp b/examples/custom-logger/custom-logger.cpp
index 7944acfeb6e..94bbd065c5b 100644
--- a/examples/custom-logger/custom-logger.cpp
+++ b/examples/custom-logger/custom-logger.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/custom-matrix-format/custom-matrix-format.cpp b/examples/custom-matrix-format/custom-matrix-format.cpp
index 96e9235773c..c586284e6f8 100644
--- a/examples/custom-matrix-format/custom-matrix-format.cpp
+++ b/examples/custom-matrix-format/custom-matrix-format.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/custom-matrix-format/stencil_kernel.cu b/examples/custom-matrix-format/stencil_kernel.cu
index fa8b9deb9d1..3f926e31ecf 100644
--- a/examples/custom-matrix-format/stencil_kernel.cu
+++ b/examples/custom-matrix-format/stencil_kernel.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/custom-stopping-criterion/custom-stopping-criterion.cpp b/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
index fe1086a1bfe..7c5074009bc 100644
--- a/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
+++ b/examples/custom-stopping-criterion/custom-stopping-criterion.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/ginkgo-overhead/ginkgo-overhead.cpp b/examples/ginkgo-overhead/ginkgo-overhead.cpp
index b8bc7acc2b1..ba41b01acf4 100644
--- a/examples/ginkgo-overhead/ginkgo-overhead.cpp
+++ b/examples/ginkgo-overhead/ginkgo-overhead.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/ginkgo-ranges/ginkgo-ranges.cpp b/examples/ginkgo-ranges/ginkgo-ranges.cpp
index c471f967d60..0853e74f3fc 100644
--- a/examples/ginkgo-ranges/ginkgo-ranges.cpp
+++ b/examples/ginkgo-ranges/ginkgo-ranges.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp b/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
index 7df73d3694b..511689a61f9 100644
--- a/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
+++ b/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/inverse-iteration/inverse-iteration.cpp b/examples/inverse-iteration/inverse-iteration.cpp
index 80575f02d9e..56afa33442a 100644
--- a/examples/inverse-iteration/inverse-iteration.cpp
+++ b/examples/inverse-iteration/inverse-iteration.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp b/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
index f5f215a87a8..38b690090e1 100644
--- a/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
+++ b/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/iterative-refinement/iterative-refinement.cpp b/examples/iterative-refinement/iterative-refinement.cpp
index 14bafe34ec5..71587eba5f7 100644
--- a/examples/iterative-refinement/iterative-refinement.cpp
+++ b/examples/iterative-refinement/iterative-refinement.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/minimal-cuda-solver/minimal-cuda-solver.cpp b/examples/minimal-cuda-solver/minimal-cuda-solver.cpp
index 1b47f712766..e099a9cc7e8 100644
--- a/examples/minimal-cuda-solver/minimal-cuda-solver.cpp
+++ b/examples/minimal-cuda-solver/minimal-cuda-solver.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/mixed-precision-ir/mixed-precision-ir.cpp b/examples/mixed-precision-ir/mixed-precision-ir.cpp
index 9cd21696253..f7f67724e2c 100644
--- a/examples/mixed-precision-ir/mixed-precision-ir.cpp
+++ b/examples/mixed-precision-ir/mixed-precision-ir.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp b/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
index fe194e99ae4..1f469bccd4e 100644
--- a/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
+++ b/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/papi-logging/papi-logging.cpp b/examples/papi-logging/papi-logging.cpp
index e50fe72c210..4e2990bc3fc 100644
--- a/examples/papi-logging/papi-logging.cpp
+++ b/examples/papi-logging/papi-logging.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/performance-debugging/performance-debugging.cpp b/examples/performance-debugging/performance-debugging.cpp
index eb88bd28fee..2378bf7864d 100644
--- a/examples/performance-debugging/performance-debugging.cpp
+++ b/examples/performance-debugging/performance-debugging.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/poisson-solver/poisson-solver.cpp b/examples/poisson-solver/poisson-solver.cpp
index 5079d9f5645..b83c7825c22 100644
--- a/examples/poisson-solver/poisson-solver.cpp
+++ b/examples/poisson-solver/poisson-solver.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/preconditioned-solver/preconditioned-solver.cpp b/examples/preconditioned-solver/preconditioned-solver.cpp
index 00ff326c5e1..73c460e019b 100644
--- a/examples/preconditioned-solver/preconditioned-solver.cpp
+++ b/examples/preconditioned-solver/preconditioned-solver.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/preconditioner-export/preconditioner-export.cpp b/examples/preconditioner-export/preconditioner-export.cpp
index 0cb2eb73384..6d048e0e9d9 100644
--- a/examples/preconditioner-export/preconditioner-export.cpp
+++ b/examples/preconditioner-export/preconditioner-export.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/simple-solver-logging/simple-solver-logging.cpp b/examples/simple-solver-logging/simple-solver-logging.cpp
index 8fc26d4a65b..01fcf93d622 100644
--- a/examples/simple-solver-logging/simple-solver-logging.cpp
+++ b/examples/simple-solver-logging/simple-solver-logging.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/simple-solver/simple-solver.cpp b/examples/simple-solver/simple-solver.cpp
index fec576052e8..09bfb476a8f 100644
--- a/examples/simple-solver/simple-solver.cpp
+++ b/examples/simple-solver/simple-solver.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp b/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
index e413556d909..8d695803a9a 100644
--- a/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
+++ b/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/base/config.hip.hpp b/hip/base/config.hip.hpp
index d698a6a8d83..95323dd7e96 100644
--- a/hip/base/config.hip.hpp
+++ b/hip/base/config.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/base/device_guard.hip.hpp b/hip/base/device_guard.hip.hpp
index b7d63ebc152..a0e02c78658 100644
--- a/hip/base/device_guard.hip.hpp
+++ b/hip/base/device_guard.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/base/exception.hip.cpp b/hip/base/exception.hip.cpp
index ca73a88c6dc..1f39ff9890f 100644
--- a/hip/base/exception.hip.cpp
+++ b/hip/base/exception.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 3e400986e2b..b9f63a15337 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/base/hipblas_bindings.hip.hpp b/hip/base/hipblas_bindings.hip.hpp
index d80908cebc4..019206f8ea9 100644
--- a/hip/base/hipblas_bindings.hip.hpp
+++ b/hip/base/hipblas_bindings.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/base/hiprand_bindings.hip.hpp b/hip/base/hiprand_bindings.hip.hpp
index 83534627ca5..f2f54313bad 100644
--- a/hip/base/hiprand_bindings.hip.hpp
+++ b/hip/base/hiprand_bindings.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/base/hipsparse_bindings.hip.hpp b/hip/base/hipsparse_bindings.hip.hpp
index 95be1bfe129..d1efeb55ef1 100644
--- a/hip/base/hipsparse_bindings.hip.hpp
+++ b/hip/base/hipsparse_bindings.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/base/math.hip.hpp b/hip/base/math.hip.hpp
index a80cc24f989..2dae1f6b67f 100644
--- a/hip/base/math.hip.hpp
+++ b/hip/base/math.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/base/pointer_mode_guard.hip.hpp b/hip/base/pointer_mode_guard.hip.hpp
index f5601c5003a..f231909e6a5 100644
--- a/hip/base/pointer_mode_guard.hip.hpp
+++ b/hip/base/pointer_mode_guard.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/base/types.hip.hpp b/hip/base/types.hip.hpp
index 4db75f4024e..c0c7e87e333 100644
--- a/hip/base/types.hip.hpp
+++ b/hip/base/types.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/base/version.hip.cpp b/hip/base/version.hip.cpp
index 5c5473cbd55..e947640306a 100644
--- a/hip/base/version.hip.cpp
+++ b/hip/base/version.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/absolute_array.hip.cpp b/hip/components/absolute_array.hip.cpp
index 3f7ee4c012c..3a4409444ef 100644
--- a/hip/components/absolute_array.hip.cpp
+++ b/hip/components/absolute_array.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/atomic.hip.hpp b/hip/components/atomic.hip.hpp
index 14c664dd876..a3e3e324f79 100644
--- a/hip/components/atomic.hip.hpp
+++ b/hip/components/atomic.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/cooperative_groups.hip.hpp b/hip/components/cooperative_groups.hip.hpp
index 5f818dca9be..10c1217da11 100644
--- a/hip/components/cooperative_groups.hip.hpp
+++ b/hip/components/cooperative_groups.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/diagonal_block_manipulation.hip.hpp b/hip/components/diagonal_block_manipulation.hip.hpp
index db26e35db1e..73f9715a487 100644
--- a/hip/components/diagonal_block_manipulation.hip.hpp
+++ b/hip/components/diagonal_block_manipulation.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/fill_array.hip.cpp b/hip/components/fill_array.hip.cpp
index e738a68811e..dc4e24720ba 100644
--- a/hip/components/fill_array.hip.cpp
+++ b/hip/components/fill_array.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/format_conversion.hip.hpp b/hip/components/format_conversion.hip.hpp
index f97e70ef438..4a28cfae17b 100644
--- a/hip/components/format_conversion.hip.hpp
+++ b/hip/components/format_conversion.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/intrinsics.hip.hpp b/hip/components/intrinsics.hip.hpp
index 8d9d0579013..4a5f3751a4c 100644
--- a/hip/components/intrinsics.hip.hpp
+++ b/hip/components/intrinsics.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/merging.hip.hpp b/hip/components/merging.hip.hpp
index 30289d41ed2..42e5b32ec71 100644
--- a/hip/components/merging.hip.hpp
+++ b/hip/components/merging.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/precision_conversion.hip.cpp b/hip/components/precision_conversion.hip.cpp
index 6720cf8c92b..bef1727e901 100644
--- a/hip/components/precision_conversion.hip.cpp
+++ b/hip/components/precision_conversion.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/prefix_sum.hip.cpp b/hip/components/prefix_sum.hip.cpp
index 9947622c565..28cd01b4fb5 100644
--- a/hip/components/prefix_sum.hip.cpp
+++ b/hip/components/prefix_sum.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/prefix_sum.hip.hpp b/hip/components/prefix_sum.hip.hpp
index de5aeb168e5..295311bbbb4 100644
--- a/hip/components/prefix_sum.hip.hpp
+++ b/hip/components/prefix_sum.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/reduction.hip.hpp b/hip/components/reduction.hip.hpp
index 425dc5b5023..e5fcb5745a3 100644
--- a/hip/components/reduction.hip.hpp
+++ b/hip/components/reduction.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/searching.hip.hpp b/hip/components/searching.hip.hpp
index 7611b23fdee..fd26e759794 100644
--- a/hip/components/searching.hip.hpp
+++ b/hip/components/searching.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/segment_scan.hip.hpp b/hip/components/segment_scan.hip.hpp
index eae5953813a..397a78b4556 100644
--- a/hip/components/segment_scan.hip.hpp
+++ b/hip/components/segment_scan.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/sorting.hip.hpp b/hip/components/sorting.hip.hpp
index 704c8f9dd07..5ebd4c495a1 100644
--- a/hip/components/sorting.hip.hpp
+++ b/hip/components/sorting.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/thread_ids.hip.hpp b/hip/components/thread_ids.hip.hpp
index 6016c26cf68..59eaeb7abbd 100644
--- a/hip/components/thread_ids.hip.hpp
+++ b/hip/components/thread_ids.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/uninitialized_array.hip.hpp b/hip/components/uninitialized_array.hip.hpp
index 7780ebb10f5..45d55d7af63 100644
--- a/hip/components/uninitialized_array.hip.hpp
+++ b/hip/components/uninitialized_array.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/components/warp_blas.hip.hpp b/hip/components/warp_blas.hip.hpp
index 52aa2acaf07..0a341d8a98c 100644
--- a/hip/components/warp_blas.hip.hpp
+++ b/hip/components/warp_blas.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/factorization/factorization_kernels.hip.cpp b/hip/factorization/factorization_kernels.hip.cpp
index bfe00c9b0b8..9e53d54e16d 100644
--- a/hip/factorization/factorization_kernels.hip.cpp
+++ b/hip/factorization/factorization_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/factorization/ilu_kernels.hip.cpp b/hip/factorization/ilu_kernels.hip.cpp
index 8888856e898..e22a8140ea9 100644
--- a/hip/factorization/ilu_kernels.hip.cpp
+++ b/hip/factorization/ilu_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/factorization/par_ict_kernels.hip.cpp b/hip/factorization/par_ict_kernels.hip.cpp
index d987ff36856..2abfeff94de 100644
--- a/hip/factorization/par_ict_kernels.hip.cpp
+++ b/hip/factorization/par_ict_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/factorization/par_ilu_kernels.hip.cpp b/hip/factorization/par_ilu_kernels.hip.cpp
index d8caeb90d16..5dcc174ebc0 100644
--- a/hip/factorization/par_ilu_kernels.hip.cpp
+++ b/hip/factorization/par_ilu_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/factorization/par_ilut_approx_filter_kernel.hip.cpp b/hip/factorization/par_ilut_approx_filter_kernel.hip.cpp
index 7955898a291..dde375f41c6 100644
--- a/hip/factorization/par_ilut_approx_filter_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_approx_filter_kernel.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/factorization/par_ilut_filter_kernel.hip.cpp b/hip/factorization/par_ilut_filter_kernel.hip.cpp
index c08b7ef767e..bb9d784c6d1 100644
--- a/hip/factorization/par_ilut_filter_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_filter_kernel.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/factorization/par_ilut_select_common.hip.cpp b/hip/factorization/par_ilut_select_common.hip.cpp
index 92431c6b0a3..d9df5ce853c 100644
--- a/hip/factorization/par_ilut_select_common.hip.cpp
+++ b/hip/factorization/par_ilut_select_common.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/factorization/par_ilut_select_common.hip.hpp b/hip/factorization/par_ilut_select_common.hip.hpp
index ce77d3a6921..fc3a5b9bb88 100644
--- a/hip/factorization/par_ilut_select_common.hip.hpp
+++ b/hip/factorization/par_ilut_select_common.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/factorization/par_ilut_select_kernel.hip.cpp b/hip/factorization/par_ilut_select_kernel.hip.cpp
index 3f92051989b..d75d08063d5 100644
--- a/hip/factorization/par_ilut_select_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_select_kernel.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/factorization/par_ilut_spgeam_kernel.hip.cpp b/hip/factorization/par_ilut_spgeam_kernel.hip.cpp
index 3d00ce153ba..b4ccb7f2904 100644
--- a/hip/factorization/par_ilut_spgeam_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_spgeam_kernel.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/factorization/par_ilut_sweep_kernel.hip.cpp b/hip/factorization/par_ilut_sweep_kernel.hip.cpp
index 15fb33ec34e..9c803cbfb5b 100644
--- a/hip/factorization/par_ilut_sweep_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_sweep_kernel.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/matrix/coo_kernels.hip.cpp b/hip/matrix/coo_kernels.hip.cpp
index 3e9e4f6b190..8f68bfa0867 100644
--- a/hip/matrix/coo_kernels.hip.cpp
+++ b/hip/matrix/coo_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/matrix/csr_kernels.hip.cpp b/hip/matrix/csr_kernels.hip.cpp
index 8adf3f98a80..d296ece80e5 100644
--- a/hip/matrix/csr_kernels.hip.cpp
+++ b/hip/matrix/csr_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index 3f7f4942dc8..c254e9a31a4 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/matrix/diagonal_kernels.hip.cpp b/hip/matrix/diagonal_kernels.hip.cpp
index 3dc510c0195..ea965ef3e46 100644
--- a/hip/matrix/diagonal_kernels.hip.cpp
+++ b/hip/matrix/diagonal_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/matrix/ell_kernels.hip.cpp b/hip/matrix/ell_kernels.hip.cpp
index 91c9a3678af..2e7a4eeeb50 100644
--- a/hip/matrix/ell_kernels.hip.cpp
+++ b/hip/matrix/ell_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/matrix/hybrid_kernels.hip.cpp b/hip/matrix/hybrid_kernels.hip.cpp
index e9efb0eb8ee..20c4a540a1b 100644
--- a/hip/matrix/hybrid_kernels.hip.cpp
+++ b/hip/matrix/hybrid_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/matrix/sellp_kernels.hip.cpp b/hip/matrix/sellp_kernels.hip.cpp
index 7d0a6ad353b..10178606a2a 100644
--- a/hip/matrix/sellp_kernels.hip.cpp
+++ b/hip/matrix/sellp_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/matrix/sparsity_csr_kernels.hip.cpp b/hip/matrix/sparsity_csr_kernels.hip.cpp
index 8ab3066f1ff..6747806eca4 100644
--- a/hip/matrix/sparsity_csr_kernels.hip.cpp
+++ b/hip/matrix/sparsity_csr_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/preconditioner/isai_kernels.hip.cpp b/hip/preconditioner/isai_kernels.hip.cpp
index f2289eba530..840d5e3c1ed 100644
--- a/hip/preconditioner/isai_kernels.hip.cpp
+++ b/hip/preconditioner/isai_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/preconditioner/jacobi_advanced_apply_kernel.hip.cpp b/hip/preconditioner/jacobi_advanced_apply_kernel.hip.cpp
index d7d3e87970c..67d98906ea9 100644
--- a/hip/preconditioner/jacobi_advanced_apply_kernel.hip.cpp
+++ b/hip/preconditioner/jacobi_advanced_apply_kernel.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/preconditioner/jacobi_common.hip.hpp b/hip/preconditioner/jacobi_common.hip.hpp
index d81dd3f9e97..2c94e7ce3b4 100644
--- a/hip/preconditioner/jacobi_common.hip.hpp
+++ b/hip/preconditioner/jacobi_common.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/preconditioner/jacobi_generate_kernel.hip.cpp b/hip/preconditioner/jacobi_generate_kernel.hip.cpp
index 6f8def4af6e..facf1916de3 100644
--- a/hip/preconditioner/jacobi_generate_kernel.hip.cpp
+++ b/hip/preconditioner/jacobi_generate_kernel.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/preconditioner/jacobi_kernels.hip.cpp b/hip/preconditioner/jacobi_kernels.hip.cpp
index b2d249f12b7..fc020837a22 100644
--- a/hip/preconditioner/jacobi_kernels.hip.cpp
+++ b/hip/preconditioner/jacobi_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/preconditioner/jacobi_simple_apply_kernel.hip.cpp b/hip/preconditioner/jacobi_simple_apply_kernel.hip.cpp
index 0c2fefb1afc..608e33d5f26 100644
--- a/hip/preconditioner/jacobi_simple_apply_kernel.hip.cpp
+++ b/hip/preconditioner/jacobi_simple_apply_kernel.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/reorder/rcm_kernels.hip.cpp b/hip/reorder/rcm_kernels.hip.cpp
index fe07008d2c9..2f5f8c32ef3 100644
--- a/hip/reorder/rcm_kernels.hip.cpp
+++ b/hip/reorder/rcm_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/solver/bicg_kernels.hip.cpp b/hip/solver/bicg_kernels.hip.cpp
index e773520b258..62bfd05fb14 100644
--- a/hip/solver/bicg_kernels.hip.cpp
+++ b/hip/solver/bicg_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/solver/bicgstab_kernels.hip.cpp b/hip/solver/bicgstab_kernels.hip.cpp
index a8776876f6f..8b6a43ac70f 100644
--- a/hip/solver/bicgstab_kernels.hip.cpp
+++ b/hip/solver/bicgstab_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/solver/cg_kernels.hip.cpp b/hip/solver/cg_kernels.hip.cpp
index 688a6ab7f49..72b3de4261b 100644
--- a/hip/solver/cg_kernels.hip.cpp
+++ b/hip/solver/cg_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/solver/cgs_kernels.hip.cpp b/hip/solver/cgs_kernels.hip.cpp
index b5597777790..8c5314648cc 100644
--- a/hip/solver/cgs_kernels.hip.cpp
+++ b/hip/solver/cgs_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/solver/common_trs_kernels.hip.hpp b/hip/solver/common_trs_kernels.hip.hpp
index 3bf0e56c7fa..8698b23383f 100644
--- a/hip/solver/common_trs_kernels.hip.hpp
+++ b/hip/solver/common_trs_kernels.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/solver/fcg_kernels.hip.cpp b/hip/solver/fcg_kernels.hip.cpp
index 750aa5743d7..c41bdeafc5b 100644
--- a/hip/solver/fcg_kernels.hip.cpp
+++ b/hip/solver/fcg_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/solver/gmres_kernels.hip.cpp b/hip/solver/gmres_kernels.hip.cpp
index 2780b149660..cd430ffa916 100644
--- a/hip/solver/gmres_kernels.hip.cpp
+++ b/hip/solver/gmres_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index 5bb26f25399..37378e00cf9 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -33,8 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
-#include <random>
 #include <time.h>
+#include <random>
 
 
 #include <hip/hip_runtime.h>
diff --git a/hip/solver/ir_kernels.hip.cpp b/hip/solver/ir_kernels.hip.cpp
index 5993c4b120b..32c3fc6df02 100644
--- a/hip/solver/ir_kernels.hip.cpp
+++ b/hip/solver/ir_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/solver/lower_trs_kernels.hip.cpp b/hip/solver/lower_trs_kernels.hip.cpp
index d4e66513ebe..3eeb50185ac 100644
--- a/hip/solver/lower_trs_kernels.hip.cpp
+++ b/hip/solver/lower_trs_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/solver/upper_trs_kernels.hip.cpp b/hip/solver/upper_trs_kernels.hip.cpp
index 0f27b6ceb28..835e2f3803c 100644
--- a/hip/solver/upper_trs_kernels.hip.cpp
+++ b/hip/solver/upper_trs_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/stop/criterion_kernels.hip.cpp b/hip/stop/criterion_kernels.hip.cpp
index 0c2cf4da378..0443930660e 100644
--- a/hip/stop/criterion_kernels.hip.cpp
+++ b/hip/stop/criterion_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/stop/residual_norm_kernels.hip.cpp b/hip/stop/residual_norm_kernels.hip.cpp
index d104a29d8a8..eb0060801dd 100644
--- a/hip/stop/residual_norm_kernels.hip.cpp
+++ b/hip/stop/residual_norm_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/base/exception_helpers.hip.cpp b/hip/test/base/exception_helpers.hip.cpp
index 521643ba81f..88d0e14de94 100644
--- a/hip/test/base/exception_helpers.hip.cpp
+++ b/hip/test/base/exception_helpers.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/base/hip_executor.hip.cpp b/hip/test/base/hip_executor.hip.cpp
index e0003e91c4b..c4f4757e397 100644
--- a/hip/test/base/hip_executor.hip.cpp
+++ b/hip/test/base/hip_executor.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/base/lin_op.hip.cpp b/hip/test/base/lin_op.hip.cpp
index 1977912be9e..4e540d80081 100644
--- a/hip/test/base/lin_op.hip.cpp
+++ b/hip/test/base/lin_op.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/base/math.hip.cpp b/hip/test/base/math.hip.cpp
index 818506a8d25..95d626225de 100644
--- a/hip/test/base/math.hip.cpp
+++ b/hip/test/base/math.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/components/absolute_array.hip.cpp b/hip/test/components/absolute_array.hip.cpp
index f718805d5eb..69e567ff8a6 100644
--- a/hip/test/components/absolute_array.hip.cpp
+++ b/hip/test/components/absolute_array.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/components/cooperative_groups_kernels.hip.cpp b/hip/test/components/cooperative_groups_kernels.hip.cpp
index 823dcef0df1..ec1ce79e080 100644
--- a/hip/test/components/cooperative_groups_kernels.hip.cpp
+++ b/hip/test/components/cooperative_groups_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/components/fill_array.hip.cpp b/hip/test/components/fill_array.hip.cpp
index 1c7bfda89d0..c139829596a 100644
--- a/hip/test/components/fill_array.hip.cpp
+++ b/hip/test/components/fill_array.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/components/merging_kernels.hip.cpp b/hip/test/components/merging_kernels.hip.cpp
index 466c31a48b3..d5acbf2b8c4 100644
--- a/hip/test/components/merging_kernels.hip.cpp
+++ b/hip/test/components/merging_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/components/precision_conversion.hip.cpp b/hip/test/components/precision_conversion.hip.cpp
index a7b9713b871..f6ef67fb76c 100644
--- a/hip/test/components/precision_conversion.hip.cpp
+++ b/hip/test/components/precision_conversion.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/components/prefix_sum.hip.cpp b/hip/test/components/prefix_sum.hip.cpp
index 96f91522d06..69577df5a7b 100644
--- a/hip/test/components/prefix_sum.hip.cpp
+++ b/hip/test/components/prefix_sum.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/components/searching_kernels.hip.cpp b/hip/test/components/searching_kernels.hip.cpp
index e55855e40c3..d22c4a125e0 100644
--- a/hip/test/components/searching_kernels.hip.cpp
+++ b/hip/test/components/searching_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/components/sorting_kernels.hip.cpp b/hip/test/components/sorting_kernels.hip.cpp
index ca30186096c..40e1aa473b1 100644
--- a/hip/test/components/sorting_kernels.hip.cpp
+++ b/hip/test/components/sorting_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/factorization/ilu_kernels.cpp b/hip/test/factorization/ilu_kernels.cpp
index 3cd8f2be763..e8c7ec41d6f 100644
--- a/hip/test/factorization/ilu_kernels.cpp
+++ b/hip/test/factorization/ilu_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/factorization/par_ict_kernels.hip.cpp b/hip/test/factorization/par_ict_kernels.hip.cpp
index b8858dadaa4..7d3376f277a 100644
--- a/hip/test/factorization/par_ict_kernels.hip.cpp
+++ b/hip/test/factorization/par_ict_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/factorization/par_ilu_kernels.hip.cpp b/hip/test/factorization/par_ilu_kernels.hip.cpp
index 96dffed19e1..98811f97af3 100644
--- a/hip/test/factorization/par_ilu_kernels.hip.cpp
+++ b/hip/test/factorization/par_ilu_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/factorization/par_ilut_kernels.hip.cpp b/hip/test/factorization/par_ilut_kernels.hip.cpp
index 38fb5eb205f..9fe1b758f2f 100644
--- a/hip/test/factorization/par_ilut_kernels.hip.cpp
+++ b/hip/test/factorization/par_ilut_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/matrix/coo_kernels.hip.cpp b/hip/test/matrix/coo_kernels.hip.cpp
index e722c6cfdb7..f40fc9436e4 100644
--- a/hip/test/matrix/coo_kernels.hip.cpp
+++ b/hip/test/matrix/coo_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index 36ad284ad71..4f67cd74421 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index dcd3ace06b9..d68c2041b20 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/matrix/diagonal_kernels.hip.cpp b/hip/test/matrix/diagonal_kernels.hip.cpp
index 1bd1b3f0b8a..b7554565cc5 100644
--- a/hip/test/matrix/diagonal_kernels.hip.cpp
+++ b/hip/test/matrix/diagonal_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/matrix/ell_kernels.hip.cpp b/hip/test/matrix/ell_kernels.hip.cpp
index d1eb395aec1..28dc3c8c38f 100644
--- a/hip/test/matrix/ell_kernels.hip.cpp
+++ b/hip/test/matrix/ell_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/matrix/hybrid_kernels.hip.cpp b/hip/test/matrix/hybrid_kernels.hip.cpp
index 31b35e1906c..a113ced6361 100644
--- a/hip/test/matrix/hybrid_kernels.hip.cpp
+++ b/hip/test/matrix/hybrid_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/matrix/sellp_kernels.hip.cpp b/hip/test/matrix/sellp_kernels.hip.cpp
index 36547a221da..869930a539e 100644
--- a/hip/test/matrix/sellp_kernels.hip.cpp
+++ b/hip/test/matrix/sellp_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/preconditioner/isai_kernels.hip.cpp b/hip/test/preconditioner/isai_kernels.hip.cpp
index 88f67c0adb3..7db20b3e22c 100644
--- a/hip/test/preconditioner/isai_kernels.hip.cpp
+++ b/hip/test/preconditioner/isai_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/preconditioner/jacobi_kernels.cpp b/hip/test/preconditioner/jacobi_kernels.cpp
index d44716534f4..3931f666f12 100644
--- a/hip/test/preconditioner/jacobi_kernels.cpp
+++ b/hip/test/preconditioner/jacobi_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/solver/bicg_kernels.cpp b/hip/test/solver/bicg_kernels.cpp
index 67fda77f84b..3eb3359f01d 100644
--- a/hip/test/solver/bicg_kernels.cpp
+++ b/hip/test/solver/bicg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/solver/bicgstab_kernels.cpp b/hip/test/solver/bicgstab_kernels.cpp
index 999b40bebaa..ee9810e005a 100644
--- a/hip/test/solver/bicgstab_kernels.cpp
+++ b/hip/test/solver/bicgstab_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/solver/cg_kernels.cpp b/hip/test/solver/cg_kernels.cpp
index db472f22000..3b7e4ccb078 100644
--- a/hip/test/solver/cg_kernels.cpp
+++ b/hip/test/solver/cg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/solver/cgs_kernels.cpp b/hip/test/solver/cgs_kernels.cpp
index ff676c2dffc..7069d1fe0a2 100644
--- a/hip/test/solver/cgs_kernels.cpp
+++ b/hip/test/solver/cgs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/solver/fcg_kernels.cpp b/hip/test/solver/fcg_kernels.cpp
index 7771cf9b03c..53ff0021e19 100644
--- a/hip/test/solver/fcg_kernels.cpp
+++ b/hip/test/solver/fcg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/solver/gmres_kernels.cpp b/hip/test/solver/gmres_kernels.cpp
index d16c781cb1e..04c2097bea5 100644
--- a/hip/test/solver/gmres_kernels.cpp
+++ b/hip/test/solver/gmres_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index 6f128a9691d..18ed225d095 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/solver/ir_kernels.cpp b/hip/test/solver/ir_kernels.cpp
index 0e5791cd7cf..48dd88b89f8 100644
--- a/hip/test/solver/ir_kernels.cpp
+++ b/hip/test/solver/ir_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/solver/lower_trs_kernels.cpp b/hip/test/solver/lower_trs_kernels.cpp
index b497b525020..3328984a901 100644
--- a/hip/test/solver/lower_trs_kernels.cpp
+++ b/hip/test/solver/lower_trs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/solver/upper_trs_kernels.cpp b/hip/test/solver/upper_trs_kernels.cpp
index ba55bc6325c..fef330d9046 100644
--- a/hip/test/solver/upper_trs_kernels.cpp
+++ b/hip/test/solver/upper_trs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/stop/criterion_kernels.hip.cpp b/hip/test/stop/criterion_kernels.hip.cpp
index 92935ea4867..60bff1bed99 100644
--- a/hip/test/stop/criterion_kernels.hip.cpp
+++ b/hip/test/stop/criterion_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/stop/residual_norm_kernels.cpp b/hip/test/stop/residual_norm_kernels.cpp
index 42c505da601..361b6545509 100644
--- a/hip/test/stop/residual_norm_kernels.cpp
+++ b/hip/test/stop/residual_norm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/utils.hip.hpp b/hip/test/utils.hip.hpp
index 8d524b0b615..dda2c068524 100644
--- a/hip/test/utils.hip.hpp
+++ b/hip/test/utils.hip.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/utils/assertions_test.hip.cpp b/hip/test/utils/assertions_test.hip.cpp
index 2d5c67addc1..dfe95d6e186 100644
--- a/hip/test/utils/assertions_test.hip.cpp
+++ b/hip/test/utils/assertions_test.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/config.hpp.in b/include/ginkgo/config.hpp.in
index fc5bae0b225..d8d14e5cef2 100644
--- a/include/ginkgo/config.hpp.in
+++ b/include/ginkgo/config.hpp.in
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/abstract_factory.hpp b/include/ginkgo/core/base/abstract_factory.hpp
index 3e89c770128..3a6535884ce 100644
--- a/include/ginkgo/core/base/abstract_factory.hpp
+++ b/include/ginkgo/core/base/abstract_factory.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/array.hpp b/include/ginkgo/core/base/array.hpp
index 7c04ff0bfb2..04dea7f309b 100644
--- a/include/ginkgo/core/base/array.hpp
+++ b/include/ginkgo/core/base/array.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/combination.hpp b/include/ginkgo/core/base/combination.hpp
index 5ca773bbc3e..d7cc9f71cde 100644
--- a/include/ginkgo/core/base/combination.hpp
+++ b/include/ginkgo/core/base/combination.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/composition.hpp b/include/ginkgo/core/base/composition.hpp
index dfcfb13c5ad..d7ba65bb436 100644
--- a/include/ginkgo/core/base/composition.hpp
+++ b/include/ginkgo/core/base/composition.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/dim.hpp b/include/ginkgo/core/base/dim.hpp
index 052c50e13b8..0881d954aa0 100644
--- a/include/ginkgo/core/base/dim.hpp
+++ b/include/ginkgo/core/base/dim.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/exception.hpp b/include/ginkgo/core/base/exception.hpp
index 6ede978ffc3..46fc9db93af 100644
--- a/include/ginkgo/core/base/exception.hpp
+++ b/include/ginkgo/core/base/exception.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/exception_helpers.hpp b/include/ginkgo/core/base/exception_helpers.hpp
index af3e1efb598..a10f05896fe 100644
--- a/include/ginkgo/core/base/exception_helpers.hpp
+++ b/include/ginkgo/core/base/exception_helpers.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index f02c5834c73..4a64a812b08 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index 59c7a02586d..cde3840c30f 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 1cf729e2d2e..91058522b3b 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/matrix_assembly_data.hpp b/include/ginkgo/core/base/matrix_assembly_data.hpp
index 5456a75e6e9..547f841edaf 100644
--- a/include/ginkgo/core/base/matrix_assembly_data.hpp
+++ b/include/ginkgo/core/base/matrix_assembly_data.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/matrix_data.hpp b/include/ginkgo/core/base/matrix_data.hpp
index 8597b31df68..74b85e4140e 100644
--- a/include/ginkgo/core/base/matrix_data.hpp
+++ b/include/ginkgo/core/base/matrix_data.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/mtx_io.hpp b/include/ginkgo/core/base/mtx_io.hpp
index 38977fe04c0..3fde2de108d 100644
--- a/include/ginkgo/core/base/mtx_io.hpp
+++ b/include/ginkgo/core/base/mtx_io.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/name_demangling.hpp b/include/ginkgo/core/base/name_demangling.hpp
index fb41561fa9f..35394ede2b1 100644
--- a/include/ginkgo/core/base/name_demangling.hpp
+++ b/include/ginkgo/core/base/name_demangling.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/perturbation.hpp b/include/ginkgo/core/base/perturbation.hpp
index d410856b00b..219c29d9ad1 100644
--- a/include/ginkgo/core/base/perturbation.hpp
+++ b/include/ginkgo/core/base/perturbation.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/polymorphic_object.hpp b/include/ginkgo/core/base/polymorphic_object.hpp
index ad712c2cd08..e9d09e5d636 100644
--- a/include/ginkgo/core/base/polymorphic_object.hpp
+++ b/include/ginkgo/core/base/polymorphic_object.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/range.hpp b/include/ginkgo/core/base/range.hpp
index 6e59ee3c637..2fe87a364a7 100644
--- a/include/ginkgo/core/base/range.hpp
+++ b/include/ginkgo/core/base/range.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 28b9ac6b51f..4435c51f1b3 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/std_extensions.hpp b/include/ginkgo/core/base/std_extensions.hpp
index c1cec39b6e4..166d8c68bf7 100644
--- a/include/ginkgo/core/base/std_extensions.hpp
+++ b/include/ginkgo/core/base/std_extensions.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/temporary_clone.hpp b/include/ginkgo/core/base/temporary_clone.hpp
index fa0c2e32b13..c3309d8736a 100644
--- a/include/ginkgo/core/base/temporary_clone.hpp
+++ b/include/ginkgo/core/base/temporary_clone.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 3cca7cbfd8a..0c908b677e8 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/utils.hpp b/include/ginkgo/core/base/utils.hpp
index 509213e8476..06c722e9f78 100644
--- a/include/ginkgo/core/base/utils.hpp
+++ b/include/ginkgo/core/base/utils.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/utils_helper.hpp b/include/ginkgo/core/base/utils_helper.hpp
index d277d20cb4c..51efb3d5ca7 100644
--- a/include/ginkgo/core/base/utils_helper.hpp
+++ b/include/ginkgo/core/base/utils_helper.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/version.hpp b/include/ginkgo/core/base/version.hpp
index d56526a8613..b5a712ed1d1 100644
--- a/include/ginkgo/core/base/version.hpp
+++ b/include/ginkgo/core/base/version.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/factorization/ilu.hpp b/include/ginkgo/core/factorization/ilu.hpp
index a1322116250..ccc0ff6a072 100644
--- a/include/ginkgo/core/factorization/ilu.hpp
+++ b/include/ginkgo/core/factorization/ilu.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/factorization/par_ict.hpp b/include/ginkgo/core/factorization/par_ict.hpp
index fd89af6aa47..cc113b8c807 100644
--- a/include/ginkgo/core/factorization/par_ict.hpp
+++ b/include/ginkgo/core/factorization/par_ict.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/factorization/par_ilu.hpp b/include/ginkgo/core/factorization/par_ilu.hpp
index c92abd88ba1..e503874345d 100644
--- a/include/ginkgo/core/factorization/par_ilu.hpp
+++ b/include/ginkgo/core/factorization/par_ilu.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/factorization/par_ilut.hpp b/include/ginkgo/core/factorization/par_ilut.hpp
index 4ee0acf2612..e8caa43c45a 100644
--- a/include/ginkgo/core/factorization/par_ilut.hpp
+++ b/include/ginkgo/core/factorization/par_ilut.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/log/convergence.hpp b/include/ginkgo/core/log/convergence.hpp
index e8a5da27eb7..8addf0d2747 100644
--- a/include/ginkgo/core/log/convergence.hpp
+++ b/include/ginkgo/core/log/convergence.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp
index f9df5660fd2..b276fbcfc9c 100644
--- a/include/ginkgo/core/log/logger.hpp
+++ b/include/ginkgo/core/log/logger.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/log/papi.hpp b/include/ginkgo/core/log/papi.hpp
index 00687f88719..1f470dd1bb4 100644
--- a/include/ginkgo/core/log/papi.hpp
+++ b/include/ginkgo/core/log/papi.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/log/record.hpp b/include/ginkgo/core/log/record.hpp
index 1e30990cc1f..2fe1d4295c8 100644
--- a/include/ginkgo/core/log/record.hpp
+++ b/include/ginkgo/core/log/record.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/log/stream.hpp b/include/ginkgo/core/log/stream.hpp
index 30b1beabf2b..634ba7e2a9b 100644
--- a/include/ginkgo/core/log/stream.hpp
+++ b/include/ginkgo/core/log/stream.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/matrix/coo.hpp b/include/ginkgo/core/matrix/coo.hpp
index ce8cf678eb1..f0872f32787 100644
--- a/include/ginkgo/core/matrix/coo.hpp
+++ b/include/ginkgo/core/matrix/coo.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index f75cc80e1a1..7e01c1293f6 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index a7fc245cd71..22bd0314392 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/matrix/diagonal.hpp b/include/ginkgo/core/matrix/diagonal.hpp
index 58463b8f838..70dfc0bd352 100644
--- a/include/ginkgo/core/matrix/diagonal.hpp
+++ b/include/ginkgo/core/matrix/diagonal.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/matrix/ell.hpp b/include/ginkgo/core/matrix/ell.hpp
index 932bfc7fda9..ea24770fbea 100644
--- a/include/ginkgo/core/matrix/ell.hpp
+++ b/include/ginkgo/core/matrix/ell.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp
index 93174e45652..7511000fade 100644
--- a/include/ginkgo/core/matrix/hybrid.hpp
+++ b/include/ginkgo/core/matrix/hybrid.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/matrix/identity.hpp b/include/ginkgo/core/matrix/identity.hpp
index d00ffec9b9e..5c5315eedce 100644
--- a/include/ginkgo/core/matrix/identity.hpp
+++ b/include/ginkgo/core/matrix/identity.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/matrix/permutation.hpp b/include/ginkgo/core/matrix/permutation.hpp
index cb81544c835..fe0b5c5b3f7 100644
--- a/include/ginkgo/core/matrix/permutation.hpp
+++ b/include/ginkgo/core/matrix/permutation.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/matrix/sellp.hpp b/include/ginkgo/core/matrix/sellp.hpp
index b99746ec3b6..d2dff8199bb 100644
--- a/include/ginkgo/core/matrix/sellp.hpp
+++ b/include/ginkgo/core/matrix/sellp.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/matrix/sparsity_csr.hpp b/include/ginkgo/core/matrix/sparsity_csr.hpp
index f736d35f840..087c0538b7d 100644
--- a/include/ginkgo/core/matrix/sparsity_csr.hpp
+++ b/include/ginkgo/core/matrix/sparsity_csr.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/preconditioner/ilu.hpp b/include/ginkgo/core/preconditioner/ilu.hpp
index 72a478ac405..e80978158bb 100644
--- a/include/ginkgo/core/preconditioner/ilu.hpp
+++ b/include/ginkgo/core/preconditioner/ilu.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/preconditioner/isai.hpp b/include/ginkgo/core/preconditioner/isai.hpp
index 420b376c0a0..d88e1487514 100644
--- a/include/ginkgo/core/preconditioner/isai.hpp
+++ b/include/ginkgo/core/preconditioner/isai.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/preconditioner/jacobi.hpp b/include/ginkgo/core/preconditioner/jacobi.hpp
index 44d1ca4a1a8..85924ac3772 100644
--- a/include/ginkgo/core/preconditioner/jacobi.hpp
+++ b/include/ginkgo/core/preconditioner/jacobi.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/reorder/rcm.hpp b/include/ginkgo/core/reorder/rcm.hpp
index 2f95a95c39a..922df382f66 100644
--- a/include/ginkgo/core/reorder/rcm.hpp
+++ b/include/ginkgo/core/reorder/rcm.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/reorder/reordering_base.hpp b/include/ginkgo/core/reorder/reordering_base.hpp
index ea6a2d2e5b0..ac0efc8e446 100644
--- a/include/ginkgo/core/reorder/reordering_base.hpp
+++ b/include/ginkgo/core/reorder/reordering_base.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/solver/bicg.hpp b/include/ginkgo/core/solver/bicg.hpp
index 570e968e468..85a6cbcf6c6 100644
--- a/include/ginkgo/core/solver/bicg.hpp
+++ b/include/ginkgo/core/solver/bicg.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/solver/bicgstab.hpp b/include/ginkgo/core/solver/bicgstab.hpp
index 5a3aad0db06..a7027356a8d 100644
--- a/include/ginkgo/core/solver/bicgstab.hpp
+++ b/include/ginkgo/core/solver/bicgstab.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/solver/cg.hpp b/include/ginkgo/core/solver/cg.hpp
index 88aa379a972..b3c6469ed8c 100644
--- a/include/ginkgo/core/solver/cg.hpp
+++ b/include/ginkgo/core/solver/cg.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/solver/cgs.hpp b/include/ginkgo/core/solver/cgs.hpp
index ab3d8bee0bb..781032f5081 100644
--- a/include/ginkgo/core/solver/cgs.hpp
+++ b/include/ginkgo/core/solver/cgs.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/solver/fcg.hpp b/include/ginkgo/core/solver/fcg.hpp
index b1c1eaf1329..570e4ac5033 100644
--- a/include/ginkgo/core/solver/fcg.hpp
+++ b/include/ginkgo/core/solver/fcg.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/solver/gmres.hpp b/include/ginkgo/core/solver/gmres.hpp
index 327dc196d94..2e426be55f8 100644
--- a/include/ginkgo/core/solver/gmres.hpp
+++ b/include/ginkgo/core/solver/gmres.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index 1d1e0b120f1..110ebf8c686 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/solver/ir.hpp b/include/ginkgo/core/solver/ir.hpp
index a34d6ea6d41..9d572339e3a 100644
--- a/include/ginkgo/core/solver/ir.hpp
+++ b/include/ginkgo/core/solver/ir.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/solver/lower_trs.hpp b/include/ginkgo/core/solver/lower_trs.hpp
index b87c6528d00..c3b95702fd0 100644
--- a/include/ginkgo/core/solver/lower_trs.hpp
+++ b/include/ginkgo/core/solver/lower_trs.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/solver/upper_trs.hpp b/include/ginkgo/core/solver/upper_trs.hpp
index 5d41637dc2c..dd82b5df6e4 100644
--- a/include/ginkgo/core/solver/upper_trs.hpp
+++ b/include/ginkgo/core/solver/upper_trs.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/stop/combined.hpp b/include/ginkgo/core/stop/combined.hpp
index 33bad75df69..d993e0ffbc7 100644
--- a/include/ginkgo/core/stop/combined.hpp
+++ b/include/ginkgo/core/stop/combined.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/stop/criterion.hpp b/include/ginkgo/core/stop/criterion.hpp
index 69b92693848..22cc74da9fd 100644
--- a/include/ginkgo/core/stop/criterion.hpp
+++ b/include/ginkgo/core/stop/criterion.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/stop/iteration.hpp b/include/ginkgo/core/stop/iteration.hpp
index 55100a4c121..1efc1c5301e 100644
--- a/include/ginkgo/core/stop/iteration.hpp
+++ b/include/ginkgo/core/stop/iteration.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index c145210e89d..6abc109d126 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/stop/residual_norm_reduction.hpp b/include/ginkgo/core/stop/residual_norm_reduction.hpp
index f9c9ff7c61f..99ebd364c31 100644
--- a/include/ginkgo/core/stop/residual_norm_reduction.hpp
+++ b/include/ginkgo/core/stop/residual_norm_reduction.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/stop/stopping_status.hpp b/include/ginkgo/core/stop/stopping_status.hpp
index 415c6e3cffe..665ab90c331 100644
--- a/include/ginkgo/core/stop/stopping_status.hpp
+++ b/include/ginkgo/core/stop/stopping_status.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/stop/time.hpp b/include/ginkgo/core/stop/time.hpp
index e7cf7621d30..bba6bf2b9d4 100644
--- a/include/ginkgo/core/stop/time.hpp
+++ b/include/ginkgo/core/stop/time.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/synthesizer/containers.hpp b/include/ginkgo/core/synthesizer/containers.hpp
index 1f3901c22c1..ebd5b441f6c 100644
--- a/include/ginkgo/core/synthesizer/containers.hpp
+++ b/include/ginkgo/core/synthesizer/containers.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 0602875c4dd..42c535cf4e4 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/ginkgo.hpp.in b/include/ginkgo/ginkgo.hpp.in
index d3e83f82ccd..147e7d66ee0 100644
--- a/include/ginkgo/ginkgo.hpp.in
+++ b/include/ginkgo/ginkgo.hpp.in
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/matrices/config.hpp.in b/matrices/config.hpp.in
index ff7b586ac3c..54f01a55e8c 100644
--- a/matrices/config.hpp.in
+++ b/matrices/config.hpp.in
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/base/version.cpp b/omp/base/version.cpp
index d429e2cbcf1..3395f71c0ca 100644
--- a/omp/base/version.cpp
+++ b/omp/base/version.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/components/absolute_array.cpp b/omp/components/absolute_array.cpp
index 36d94ec8471..5a1a5df503e 100644
--- a/omp/components/absolute_array.cpp
+++ b/omp/components/absolute_array.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/components/csr_spgeam.hpp b/omp/components/csr_spgeam.hpp
index d88e612a0ae..0429aed8a85 100644
--- a/omp/components/csr_spgeam.hpp
+++ b/omp/components/csr_spgeam.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/components/fill_array.cpp b/omp/components/fill_array.cpp
index 60844522cbe..c528c8fe7d3 100644
--- a/omp/components/fill_array.cpp
+++ b/omp/components/fill_array.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/components/format_conversion.hpp b/omp/components/format_conversion.hpp
index 1d2e9da9e46..706c686ac83 100644
--- a/omp/components/format_conversion.hpp
+++ b/omp/components/format_conversion.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/components/matrix_operations.hpp b/omp/components/matrix_operations.hpp
index 7fa629811ee..86c1c152ff8 100644
--- a/omp/components/matrix_operations.hpp
+++ b/omp/components/matrix_operations.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/components/omp_mutex.hpp b/omp/components/omp_mutex.hpp
index 2f1c410b9a5..10b35c50d75 100644
--- a/omp/components/omp_mutex.hpp
+++ b/omp/components/omp_mutex.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/components/precision_conversion.cpp b/omp/components/precision_conversion.cpp
index 4c4553470a8..75f55e39b46 100644
--- a/omp/components/precision_conversion.cpp
+++ b/omp/components/precision_conversion.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/components/prefix_sum.cpp b/omp/components/prefix_sum.cpp
index df307abd09c..8e2e964934b 100644
--- a/omp/components/prefix_sum.cpp
+++ b/omp/components/prefix_sum.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/components/sort_small.hpp b/omp/components/sort_small.hpp
index cb93b55e001..6cb5c905837 100644
--- a/omp/components/sort_small.hpp
+++ b/omp/components/sort_small.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/factorization/factorization_kernels.cpp b/omp/factorization/factorization_kernels.cpp
index f4b1f616444..09b2d5fd24a 100644
--- a/omp/factorization/factorization_kernels.cpp
+++ b/omp/factorization/factorization_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/factorization/ilu_kernels.cpp b/omp/factorization/ilu_kernels.cpp
index 77f30a9a753..976c8129395 100644
--- a/omp/factorization/ilu_kernels.cpp
+++ b/omp/factorization/ilu_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/factorization/par_ict_kernels.cpp b/omp/factorization/par_ict_kernels.cpp
index 15d80ab8755..331c384b069 100644
--- a/omp/factorization/par_ict_kernels.cpp
+++ b/omp/factorization/par_ict_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/factorization/par_ilu_kernels.cpp b/omp/factorization/par_ilu_kernels.cpp
index d658c0da579..bbff6d84d7d 100644
--- a/omp/factorization/par_ilu_kernels.cpp
+++ b/omp/factorization/par_ilu_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/factorization/par_ilut_kernels.cpp b/omp/factorization/par_ilut_kernels.cpp
index 1e7d4988c5c..bd1dd027173 100644
--- a/omp/factorization/par_ilut_kernels.cpp
+++ b/omp/factorization/par_ilut_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/matrix/coo_kernels.cpp b/omp/matrix/coo_kernels.cpp
index f3cf968a356..676051af076 100644
--- a/omp/matrix/coo_kernels.cpp
+++ b/omp/matrix/coo_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/matrix/csr_kernels.cpp b/omp/matrix/csr_kernels.cpp
index 284bc380544..44e8f18a74e 100644
--- a/omp/matrix/csr_kernels.cpp
+++ b/omp/matrix/csr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/matrix/dense_kernels.cpp b/omp/matrix/dense_kernels.cpp
index a14f45ba033..6ca39cccd22 100644
--- a/omp/matrix/dense_kernels.cpp
+++ b/omp/matrix/dense_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/matrix/diagonal_kernels.cpp b/omp/matrix/diagonal_kernels.cpp
index ac7c5c4fe67..35985dc0e18 100644
--- a/omp/matrix/diagonal_kernels.cpp
+++ b/omp/matrix/diagonal_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/matrix/ell_kernels.cpp b/omp/matrix/ell_kernels.cpp
index 19d32988042..a590357a1c2 100644
--- a/omp/matrix/ell_kernels.cpp
+++ b/omp/matrix/ell_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/matrix/hybrid_kernels.cpp b/omp/matrix/hybrid_kernels.cpp
index 8282d7c7ab8..204da570121 100644
--- a/omp/matrix/hybrid_kernels.cpp
+++ b/omp/matrix/hybrid_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/matrix/sellp_kernels.cpp b/omp/matrix/sellp_kernels.cpp
index 95cd6551315..28ae80ff92a 100644
--- a/omp/matrix/sellp_kernels.cpp
+++ b/omp/matrix/sellp_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/matrix/sparsity_csr_kernels.cpp b/omp/matrix/sparsity_csr_kernels.cpp
index 22987b55287..903e4c43414 100644
--- a/omp/matrix/sparsity_csr_kernels.cpp
+++ b/omp/matrix/sparsity_csr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/preconditioner/isai_kernels.cpp b/omp/preconditioner/isai_kernels.cpp
index 98acbf68446..a64724b260e 100644
--- a/omp/preconditioner/isai_kernels.cpp
+++ b/omp/preconditioner/isai_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/preconditioner/jacobi_kernels.cpp b/omp/preconditioner/jacobi_kernels.cpp
index cf3e6fb4ec8..df97082319d 100644
--- a/omp/preconditioner/jacobi_kernels.cpp
+++ b/omp/preconditioner/jacobi_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/reorder/rcm_kernels.cpp b/omp/reorder/rcm_kernels.cpp
index a51c57cef41..9b3a454f334 100644
--- a/omp/reorder/rcm_kernels.cpp
+++ b/omp/reorder/rcm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/solver/bicg_kernels.cpp b/omp/solver/bicg_kernels.cpp
index d9e2864eedf..9e98a000699 100644
--- a/omp/solver/bicg_kernels.cpp
+++ b/omp/solver/bicg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/solver/bicgstab_kernels.cpp b/omp/solver/bicgstab_kernels.cpp
index d761fc044cf..228f7c50e27 100644
--- a/omp/solver/bicgstab_kernels.cpp
+++ b/omp/solver/bicgstab_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/solver/cg_kernels.cpp b/omp/solver/cg_kernels.cpp
index b9a88f25761..b012c2cb260 100644
--- a/omp/solver/cg_kernels.cpp
+++ b/omp/solver/cg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/solver/cgs_kernels.cpp b/omp/solver/cgs_kernels.cpp
index a0678788565..34323bc7216 100644
--- a/omp/solver/cgs_kernels.cpp
+++ b/omp/solver/cgs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/solver/fcg_kernels.cpp b/omp/solver/fcg_kernels.cpp
index b8b69ee6d91..3bba54d5b0f 100644
--- a/omp/solver/fcg_kernels.cpp
+++ b/omp/solver/fcg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/solver/gmres_kernels.cpp b/omp/solver/gmres_kernels.cpp
index 6c44263bc75..80bfbd6f54e 100644
--- a/omp/solver/gmres_kernels.cpp
+++ b/omp/solver/gmres_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index ce91497b1ee..f03272ec8e8 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -33,10 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
+#include <time.h>
 #include <algorithm>
 #include <iostream>
 #include <random>
-#include <time.h>
 
 
 #include <omp.h>
diff --git a/omp/solver/ir_kernels.cpp b/omp/solver/ir_kernels.cpp
index ba68c407e95..4b9c8b21a61 100644
--- a/omp/solver/ir_kernels.cpp
+++ b/omp/solver/ir_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/solver/lower_trs_kernels.cpp b/omp/solver/lower_trs_kernels.cpp
index b30a23a69d6..b313f762d2f 100644
--- a/omp/solver/lower_trs_kernels.cpp
+++ b/omp/solver/lower_trs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/solver/upper_trs_kernels.cpp b/omp/solver/upper_trs_kernels.cpp
index 15d982998e0..29ecfda6c8c 100644
--- a/omp/solver/upper_trs_kernels.cpp
+++ b/omp/solver/upper_trs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/stop/criterion_kernels.cpp b/omp/stop/criterion_kernels.cpp
index ef8ff9f1221..d48b5cc0165 100644
--- a/omp/stop/criterion_kernels.cpp
+++ b/omp/stop/criterion_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/stop/residual_norm_kernels.cpp b/omp/stop/residual_norm_kernels.cpp
index 1fd3a14cf85..630fd9b05c6 100644
--- a/omp/stop/residual_norm_kernels.cpp
+++ b/omp/stop/residual_norm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/components/absolute_array.cpp b/omp/test/components/absolute_array.cpp
index 92203777d9c..c6d80de903b 100644
--- a/omp/test/components/absolute_array.cpp
+++ b/omp/test/components/absolute_array.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/components/fill_array.cpp b/omp/test/components/fill_array.cpp
index a9bafc74ad5..eddf3e9eb51 100644
--- a/omp/test/components/fill_array.cpp
+++ b/omp/test/components/fill_array.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/components/precision_conversion.cpp b/omp/test/components/precision_conversion.cpp
index ffd9c25df3e..5290a5f258e 100644
--- a/omp/test/components/precision_conversion.cpp
+++ b/omp/test/components/precision_conversion.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/components/prefix_sum.cpp b/omp/test/components/prefix_sum.cpp
index af00ed9977a..1322f8a7188 100644
--- a/omp/test/components/prefix_sum.cpp
+++ b/omp/test/components/prefix_sum.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/factorization/par_ict_kernels.cpp b/omp/test/factorization/par_ict_kernels.cpp
index a6afed077a6..0e3c50e3ea7 100644
--- a/omp/test/factorization/par_ict_kernels.cpp
+++ b/omp/test/factorization/par_ict_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/factorization/par_ilu_kernels.cpp b/omp/test/factorization/par_ilu_kernels.cpp
index 583a68c26e9..ea3e399b686 100644
--- a/omp/test/factorization/par_ilu_kernels.cpp
+++ b/omp/test/factorization/par_ilu_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/factorization/par_ilut_kernels.cpp b/omp/test/factorization/par_ilut_kernels.cpp
index a30ffba8e45..eaea0c2d41e 100644
--- a/omp/test/factorization/par_ilut_kernels.cpp
+++ b/omp/test/factorization/par_ilut_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/matrix/coo_kernels.cpp b/omp/test/matrix/coo_kernels.cpp
index 9f0dcfb7e73..8f90fe7336c 100644
--- a/omp/test/matrix/coo_kernels.cpp
+++ b/omp/test/matrix/coo_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index 2419b5c1a41..a27f6638a04 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index 4f36313c8ad..bb860e7c820 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/matrix/diagonal_kernels.cpp b/omp/test/matrix/diagonal_kernels.cpp
index 02ddd354241..9fcd618ad38 100644
--- a/omp/test/matrix/diagonal_kernels.cpp
+++ b/omp/test/matrix/diagonal_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/matrix/ell_kernels.cpp b/omp/test/matrix/ell_kernels.cpp
index 2d11a61f68b..281292e9684 100644
--- a/omp/test/matrix/ell_kernels.cpp
+++ b/omp/test/matrix/ell_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/matrix/hybrid_kernels.cpp b/omp/test/matrix/hybrid_kernels.cpp
index a0dd242e1f7..39f56d8442c 100644
--- a/omp/test/matrix/hybrid_kernels.cpp
+++ b/omp/test/matrix/hybrid_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/matrix/sellp_kernels.cpp b/omp/test/matrix/sellp_kernels.cpp
index 802abdbdfdf..a48ea35d9b9 100644
--- a/omp/test/matrix/sellp_kernels.cpp
+++ b/omp/test/matrix/sellp_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/matrix/sparsity_csr_kernels.cpp b/omp/test/matrix/sparsity_csr_kernels.cpp
index dea4844885a..fd9d65d3860 100644
--- a/omp/test/matrix/sparsity_csr_kernels.cpp
+++ b/omp/test/matrix/sparsity_csr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/preconditioner/isai_kernels.cpp b/omp/test/preconditioner/isai_kernels.cpp
index ea3c52755a1..f0fa382a9d1 100644
--- a/omp/test/preconditioner/isai_kernels.cpp
+++ b/omp/test/preconditioner/isai_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/preconditioner/jacobi_kernels.cpp b/omp/test/preconditioner/jacobi_kernels.cpp
index c1d3a03ee32..73f8a216f9e 100644
--- a/omp/test/preconditioner/jacobi_kernels.cpp
+++ b/omp/test/preconditioner/jacobi_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/reorder/rcm_kernels.cpp b/omp/test/reorder/rcm_kernels.cpp
index 9f7a891cb89..18cd5850451 100644
--- a/omp/test/reorder/rcm_kernels.cpp
+++ b/omp/test/reorder/rcm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/solver/bicg_kernels.cpp b/omp/test/solver/bicg_kernels.cpp
index 2766f1eb910..6a18e176119 100644
--- a/omp/test/solver/bicg_kernels.cpp
+++ b/omp/test/solver/bicg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/solver/bicgstab_kernels.cpp b/omp/test/solver/bicgstab_kernels.cpp
index 5a81532f787..83b691b6455 100644
--- a/omp/test/solver/bicgstab_kernels.cpp
+++ b/omp/test/solver/bicgstab_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/solver/cg_kernels.cpp b/omp/test/solver/cg_kernels.cpp
index 695789f0205..35d09fc0f6d 100644
--- a/omp/test/solver/cg_kernels.cpp
+++ b/omp/test/solver/cg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/solver/cgs_kernels.cpp b/omp/test/solver/cgs_kernels.cpp
index 7fabfe22e93..1c64f8e22fb 100644
--- a/omp/test/solver/cgs_kernels.cpp
+++ b/omp/test/solver/cgs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/solver/fcg_kernels.cpp b/omp/test/solver/fcg_kernels.cpp
index af7fe606413..9c55c762a78 100644
--- a/omp/test/solver/fcg_kernels.cpp
+++ b/omp/test/solver/fcg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/solver/gmres_kernels.cpp b/omp/test/solver/gmres_kernels.cpp
index da04a15c44b..8e9e8a5a762 100644
--- a/omp/test/solver/gmres_kernels.cpp
+++ b/omp/test/solver/gmres_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index 70695b7762f..f26bf4dcf4b 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/solver/ir_kernels.cpp b/omp/test/solver/ir_kernels.cpp
index a1b7b55f448..86312d491d6 100644
--- a/omp/test/solver/ir_kernels.cpp
+++ b/omp/test/solver/ir_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/solver/lower_trs_kernels.cpp b/omp/test/solver/lower_trs_kernels.cpp
index 52c21bfa356..7065c7135e1 100644
--- a/omp/test/solver/lower_trs_kernels.cpp
+++ b/omp/test/solver/lower_trs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/solver/upper_trs_kernels.cpp b/omp/test/solver/upper_trs_kernels.cpp
index db6097f6623..bc434aca064 100644
--- a/omp/test/solver/upper_trs_kernels.cpp
+++ b/omp/test/solver/upper_trs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/stop/criterion_kernels.cpp b/omp/test/stop/criterion_kernels.cpp
index 8ab87f2b6f2..8734c7a2392 100644
--- a/omp/test/stop/criterion_kernels.cpp
+++ b/omp/test/stop/criterion_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/stop/residual_norm_kernels.cpp b/omp/test/stop/residual_norm_kernels.cpp
index 082dfd7bc57..4d960f7b442 100644
--- a/omp/test/stop/residual_norm_kernels.cpp
+++ b/omp/test/stop/residual_norm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/base/version.cpp b/reference/base/version.cpp
index aac3a23180e..662c9e1b85a 100644
--- a/reference/base/version.cpp
+++ b/reference/base/version.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/components/absolute_array.cpp b/reference/components/absolute_array.cpp
index 688945886f9..287e4cfeb95 100644
--- a/reference/components/absolute_array.cpp
+++ b/reference/components/absolute_array.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/components/convert_ptrs.hpp b/reference/components/convert_ptrs.hpp
index bc89f9f2df0..ad59da923cc 100644
--- a/reference/components/convert_ptrs.hpp
+++ b/reference/components/convert_ptrs.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/components/csr_spgeam.hpp b/reference/components/csr_spgeam.hpp
index 84ebbaecccc..c8149f9186f 100644
--- a/reference/components/csr_spgeam.hpp
+++ b/reference/components/csr_spgeam.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/components/fill_array.cpp b/reference/components/fill_array.cpp
index 23499dabd3b..43e586c1092 100644
--- a/reference/components/fill_array.cpp
+++ b/reference/components/fill_array.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/components/format_conversion.hpp b/reference/components/format_conversion.hpp
index 38520dd8b66..ebf24d3acd5 100644
--- a/reference/components/format_conversion.hpp
+++ b/reference/components/format_conversion.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/components/matrix_operations.hpp b/reference/components/matrix_operations.hpp
index b62a3a84ec8..97a639bf907 100644
--- a/reference/components/matrix_operations.hpp
+++ b/reference/components/matrix_operations.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/components/precision_conversion.cpp b/reference/components/precision_conversion.cpp
index 6bc37efe940..7fcffeb8300 100644
--- a/reference/components/precision_conversion.cpp
+++ b/reference/components/precision_conversion.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/components/prefix_sum.cpp b/reference/components/prefix_sum.cpp
index 003491ad5d9..9cdf95bdd30 100644
--- a/reference/components/prefix_sum.cpp
+++ b/reference/components/prefix_sum.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/factorization/factorization_kernels.cpp b/reference/factorization/factorization_kernels.cpp
index 6155208f41b..dfae5dfe08e 100644
--- a/reference/factorization/factorization_kernels.cpp
+++ b/reference/factorization/factorization_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/factorization/ilu_kernels.cpp b/reference/factorization/ilu_kernels.cpp
index 279bfda5c18..73f4cb8579e 100644
--- a/reference/factorization/ilu_kernels.cpp
+++ b/reference/factorization/ilu_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/factorization/par_ict_kernels.cpp b/reference/factorization/par_ict_kernels.cpp
index b360709a628..038da5238da 100644
--- a/reference/factorization/par_ict_kernels.cpp
+++ b/reference/factorization/par_ict_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/factorization/par_ilu_kernels.cpp b/reference/factorization/par_ilu_kernels.cpp
index e2234d52fdf..11d6341fb5f 100644
--- a/reference/factorization/par_ilu_kernels.cpp
+++ b/reference/factorization/par_ilu_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/factorization/par_ilut_kernels.cpp b/reference/factorization/par_ilut_kernels.cpp
index 63df1c5634e..82cb34579df 100644
--- a/reference/factorization/par_ilut_kernels.cpp
+++ b/reference/factorization/par_ilut_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/matrix/coo_kernels.cpp b/reference/matrix/coo_kernels.cpp
index 3d875bcda95..77a5de52913 100644
--- a/reference/matrix/coo_kernels.cpp
+++ b/reference/matrix/coo_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/matrix/csr_kernels.cpp b/reference/matrix/csr_kernels.cpp
index d4d5a02e473..ec7ac790c7b 100644
--- a/reference/matrix/csr_kernels.cpp
+++ b/reference/matrix/csr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/matrix/dense_kernels.cpp b/reference/matrix/dense_kernels.cpp
index 16443c0bbb5..df0b8595afb 100644
--- a/reference/matrix/dense_kernels.cpp
+++ b/reference/matrix/dense_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/matrix/diagonal_kernels.cpp b/reference/matrix/diagonal_kernels.cpp
index 187b4ef25e6..1a832c68468 100644
--- a/reference/matrix/diagonal_kernels.cpp
+++ b/reference/matrix/diagonal_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/matrix/ell_kernels.cpp b/reference/matrix/ell_kernels.cpp
index fb46c0fe085..fdbaff5dfc5 100644
--- a/reference/matrix/ell_kernels.cpp
+++ b/reference/matrix/ell_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/matrix/hybrid_kernels.cpp b/reference/matrix/hybrid_kernels.cpp
index 74e126334e2..7bb5b4d512d 100644
--- a/reference/matrix/hybrid_kernels.cpp
+++ b/reference/matrix/hybrid_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/matrix/sellp_kernels.cpp b/reference/matrix/sellp_kernels.cpp
index f91be5a0573..094d29a0ec6 100644
--- a/reference/matrix/sellp_kernels.cpp
+++ b/reference/matrix/sellp_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/matrix/sparsity_csr_kernels.cpp b/reference/matrix/sparsity_csr_kernels.cpp
index 70ab3b15aff..0150fec8fcf 100644
--- a/reference/matrix/sparsity_csr_kernels.cpp
+++ b/reference/matrix/sparsity_csr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/preconditioner/isai_kernels.cpp b/reference/preconditioner/isai_kernels.cpp
index 6a3a682e395..89f52a4bf16 100644
--- a/reference/preconditioner/isai_kernels.cpp
+++ b/reference/preconditioner/isai_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/preconditioner/jacobi_kernels.cpp b/reference/preconditioner/jacobi_kernels.cpp
index a547e6405f6..b96af16dca2 100644
--- a/reference/preconditioner/jacobi_kernels.cpp
+++ b/reference/preconditioner/jacobi_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/reorder/rcm_kernels.cpp b/reference/reorder/rcm_kernels.cpp
index 74b02fff26b..06142e7435b 100644
--- a/reference/reorder/rcm_kernels.cpp
+++ b/reference/reorder/rcm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/solver/bicg_kernels.cpp b/reference/solver/bicg_kernels.cpp
index 5142b9461fd..165b6bb4f5c 100644
--- a/reference/solver/bicg_kernels.cpp
+++ b/reference/solver/bicg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/solver/bicgstab_kernels.cpp b/reference/solver/bicgstab_kernels.cpp
index 29927d18953..4c6bc038344 100644
--- a/reference/solver/bicgstab_kernels.cpp
+++ b/reference/solver/bicgstab_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/solver/cg_kernels.cpp b/reference/solver/cg_kernels.cpp
index bf4625de9de..4202d41a2e3 100644
--- a/reference/solver/cg_kernels.cpp
+++ b/reference/solver/cg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/solver/cgs_kernels.cpp b/reference/solver/cgs_kernels.cpp
index f393bb4ecd5..c2cfb0f0cda 100644
--- a/reference/solver/cgs_kernels.cpp
+++ b/reference/solver/cgs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/solver/fcg_kernels.cpp b/reference/solver/fcg_kernels.cpp
index 24c758d1140..b8af3445e5d 100644
--- a/reference/solver/fcg_kernels.cpp
+++ b/reference/solver/fcg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/solver/gmres_kernels.cpp b/reference/solver/gmres_kernels.cpp
index fd79e7d8574..e70cb8e0dc7 100644
--- a/reference/solver/gmres_kernels.cpp
+++ b/reference/solver/gmres_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index eff6751fc85..17c34f00d04 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -33,10 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
+#include <time.h>
 #include <algorithm>
 #include <iostream>
 #include <random>
-#include <time.h>
 
 
 #include <ginkgo/core/base/array.hpp>
diff --git a/reference/solver/ir_kernels.cpp b/reference/solver/ir_kernels.cpp
index 1febced33ad..b3f4b85bc7f 100644
--- a/reference/solver/ir_kernels.cpp
+++ b/reference/solver/ir_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/solver/lower_trs_kernels.cpp b/reference/solver/lower_trs_kernels.cpp
index daa62716f1b..8247397c4ba 100644
--- a/reference/solver/lower_trs_kernels.cpp
+++ b/reference/solver/lower_trs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/solver/upper_trs_kernels.cpp b/reference/solver/upper_trs_kernels.cpp
index 2cb41ed397f..7c938d505ab 100644
--- a/reference/solver/upper_trs_kernels.cpp
+++ b/reference/solver/upper_trs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/stop/criterion_kernels.cpp b/reference/stop/criterion_kernels.cpp
index 050ebab01af..e54f5f96f9f 100644
--- a/reference/stop/criterion_kernels.cpp
+++ b/reference/stop/criterion_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/stop/residual_norm_kernels.cpp b/reference/stop/residual_norm_kernels.cpp
index fb968e0eae3..c6c4393a4eb 100644
--- a/reference/stop/residual_norm_kernels.cpp
+++ b/reference/stop/residual_norm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/base/combination.cpp b/reference/test/base/combination.cpp
index f490433ef8b..b6dba6f7386 100644
--- a/reference/test/base/combination.cpp
+++ b/reference/test/base/combination.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/base/composition.cpp b/reference/test/base/composition.cpp
index 5e381a8a48e..f48ff60ef4d 100644
--- a/reference/test/base/composition.cpp
+++ b/reference/test/base/composition.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/base/perturbation.cpp b/reference/test/base/perturbation.cpp
index d2bc6a617f5..7f847e825c2 100644
--- a/reference/test/base/perturbation.cpp
+++ b/reference/test/base/perturbation.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/base/utils.cpp b/reference/test/base/utils.cpp
index 6a4b7d4b9bb..4a904cc3f5a 100644
--- a/reference/test/base/utils.cpp
+++ b/reference/test/base/utils.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/components/absolute_array.cpp b/reference/test/components/absolute_array.cpp
index 8d08b6d8b95..a28d8f18cb3 100644
--- a/reference/test/components/absolute_array.cpp
+++ b/reference/test/components/absolute_array.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/components/fill_array.cpp b/reference/test/components/fill_array.cpp
index 7ad30a78aff..c58b2d27995 100644
--- a/reference/test/components/fill_array.cpp
+++ b/reference/test/components/fill_array.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/components/precision_conversion.cpp b/reference/test/components/precision_conversion.cpp
index 10c96e82f23..1566aee59bc 100644
--- a/reference/test/components/precision_conversion.cpp
+++ b/reference/test/components/precision_conversion.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/components/prefix_sum.cpp b/reference/test/components/prefix_sum.cpp
index 58d40606bb2..53dc25c6a1a 100644
--- a/reference/test/components/prefix_sum.cpp
+++ b/reference/test/components/prefix_sum.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/factorization/par_ict_kernels.cpp b/reference/test/factorization/par_ict_kernels.cpp
index 3cde2a85cab..0466278a102 100644
--- a/reference/test/factorization/par_ict_kernels.cpp
+++ b/reference/test/factorization/par_ict_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/factorization/par_ilu_kernels.cpp b/reference/test/factorization/par_ilu_kernels.cpp
index 551f591b53d..388d54232ad 100644
--- a/reference/test/factorization/par_ilu_kernels.cpp
+++ b/reference/test/factorization/par_ilu_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/factorization/par_ilut_kernels.cpp b/reference/test/factorization/par_ilut_kernels.cpp
index 0a5aa7ae049..33dd875ab92 100644
--- a/reference/test/factorization/par_ilut_kernels.cpp
+++ b/reference/test/factorization/par_ilut_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/log/convergence.cpp b/reference/test/log/convergence.cpp
index 484e605864c..1e10f370996 100644
--- a/reference/test/log/convergence.cpp
+++ b/reference/test/log/convergence.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/log/papi.cpp b/reference/test/log/papi.cpp
index f1e98dd0a51..fd84c507125 100644
--- a/reference/test/log/papi.cpp
+++ b/reference/test/log/papi.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/matrix/coo_kernels.cpp b/reference/test/matrix/coo_kernels.cpp
index 2e7d7764c99..c728a7ccbda 100644
--- a/reference/test/matrix/coo_kernels.cpp
+++ b/reference/test/matrix/coo_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp
index b963508eb45..186aa700598 100644
--- a/reference/test/matrix/csr_kernels.cpp
+++ b/reference/test/matrix/csr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 3022ece5124..d8d8758af21 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/matrix/diagonal_kernels.cpp b/reference/test/matrix/diagonal_kernels.cpp
index 440719f33bc..01db0c6e800 100644
--- a/reference/test/matrix/diagonal_kernels.cpp
+++ b/reference/test/matrix/diagonal_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/matrix/ell_kernels.cpp b/reference/test/matrix/ell_kernels.cpp
index 03e0b6417d4..91b29752be9 100644
--- a/reference/test/matrix/ell_kernels.cpp
+++ b/reference/test/matrix/ell_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/matrix/hybrid_kernels.cpp b/reference/test/matrix/hybrid_kernels.cpp
index da8a5dded83..59064eeceaa 100644
--- a/reference/test/matrix/hybrid_kernels.cpp
+++ b/reference/test/matrix/hybrid_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/matrix/identity.cpp b/reference/test/matrix/identity.cpp
index 586d8ed232b..690c7b811ce 100644
--- a/reference/test/matrix/identity.cpp
+++ b/reference/test/matrix/identity.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/matrix/permutation.cpp b/reference/test/matrix/permutation.cpp
index 4535cd6a292..0e941a530cf 100644
--- a/reference/test/matrix/permutation.cpp
+++ b/reference/test/matrix/permutation.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/matrix/sellp_kernels.cpp b/reference/test/matrix/sellp_kernels.cpp
index 736264d873d..f4ea24287ea 100644
--- a/reference/test/matrix/sellp_kernels.cpp
+++ b/reference/test/matrix/sellp_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/matrix/sparsity_csr.cpp b/reference/test/matrix/sparsity_csr.cpp
index 8569072711a..e87eeacbc5a 100644
--- a/reference/test/matrix/sparsity_csr.cpp
+++ b/reference/test/matrix/sparsity_csr.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/matrix/sparsity_csr_kernels.cpp b/reference/test/matrix/sparsity_csr_kernels.cpp
index 141013061c0..ae5d5a532c7 100644
--- a/reference/test/matrix/sparsity_csr_kernels.cpp
+++ b/reference/test/matrix/sparsity_csr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/preconditioner/ilu.cpp b/reference/test/preconditioner/ilu.cpp
index 373906be719..10bc527d1b1 100644
--- a/reference/test/preconditioner/ilu.cpp
+++ b/reference/test/preconditioner/ilu.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/preconditioner/isai_kernels.cpp b/reference/test/preconditioner/isai_kernels.cpp
index 5ed332f7e95..fe139517bfc 100644
--- a/reference/test/preconditioner/isai_kernels.cpp
+++ b/reference/test/preconditioner/isai_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/preconditioner/jacobi.cpp b/reference/test/preconditioner/jacobi.cpp
index af0e9d9005b..1dfc17c5569 100644
--- a/reference/test/preconditioner/jacobi.cpp
+++ b/reference/test/preconditioner/jacobi.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/preconditioner/jacobi_kernels.cpp b/reference/test/preconditioner/jacobi_kernels.cpp
index 48f66a19723..778d0dd467d 100644
--- a/reference/test/preconditioner/jacobi_kernels.cpp
+++ b/reference/test/preconditioner/jacobi_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/reorder/rcm.cpp b/reference/test/reorder/rcm.cpp
index c81045c8fd6..1730258231f 100644
--- a/reference/test/reorder/rcm.cpp
+++ b/reference/test/reorder/rcm.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/reorder/rcm_kernels.cpp b/reference/test/reorder/rcm_kernels.cpp
index 3905918e3d7..a5b2b3069a3 100644
--- a/reference/test/reorder/rcm_kernels.cpp
+++ b/reference/test/reorder/rcm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/solver/bicg_kernels.cpp b/reference/test/solver/bicg_kernels.cpp
index 9ff64123176..d00fde64493 100644
--- a/reference/test/solver/bicg_kernels.cpp
+++ b/reference/test/solver/bicg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/solver/bicgstab_kernels.cpp b/reference/test/solver/bicgstab_kernels.cpp
index 3e01f6d12aa..0779821d159 100644
--- a/reference/test/solver/bicgstab_kernels.cpp
+++ b/reference/test/solver/bicgstab_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/solver/cg_kernels.cpp b/reference/test/solver/cg_kernels.cpp
index 7de370c020e..a2884779e03 100644
--- a/reference/test/solver/cg_kernels.cpp
+++ b/reference/test/solver/cg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/solver/cgs_kernels.cpp b/reference/test/solver/cgs_kernels.cpp
index 105062ff187..efaa0d28176 100644
--- a/reference/test/solver/cgs_kernels.cpp
+++ b/reference/test/solver/cgs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/solver/fcg_kernels.cpp b/reference/test/solver/fcg_kernels.cpp
index eb21327256e..947ec1d325a 100644
--- a/reference/test/solver/fcg_kernels.cpp
+++ b/reference/test/solver/fcg_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/solver/gmres_kernels.cpp b/reference/test/solver/gmres_kernels.cpp
index 9232971c98f..8574af14dc3 100644
--- a/reference/test/solver/gmres_kernels.cpp
+++ b/reference/test/solver/gmres_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp
index 055731174e3..882c5565c5b 100644
--- a/reference/test/solver/idr_kernels.cpp
+++ b/reference/test/solver/idr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/solver/ir_kernels.cpp b/reference/test/solver/ir_kernels.cpp
index 76c66873d00..48bed351e6d 100644
--- a/reference/test/solver/ir_kernels.cpp
+++ b/reference/test/solver/ir_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/solver/lower_trs.cpp b/reference/test/solver/lower_trs.cpp
index fe0deeca70c..3049a74f7b6 100644
--- a/reference/test/solver/lower_trs.cpp
+++ b/reference/test/solver/lower_trs.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/solver/lower_trs_kernels.cpp b/reference/test/solver/lower_trs_kernels.cpp
index 88158a9b288..4b7ff6b94a7 100644
--- a/reference/test/solver/lower_trs_kernels.cpp
+++ b/reference/test/solver/lower_trs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/solver/upper_trs.cpp b/reference/test/solver/upper_trs.cpp
index 5476897c6ce..c186d147358 100644
--- a/reference/test/solver/upper_trs.cpp
+++ b/reference/test/solver/upper_trs.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/solver/upper_trs_kernels.cpp b/reference/test/solver/upper_trs_kernels.cpp
index d0077e63398..1fdab317316 100644
--- a/reference/test/solver/upper_trs_kernels.cpp
+++ b/reference/test/solver/upper_trs_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/stop/combined.cpp b/reference/test/stop/combined.cpp
index 00eed7107bd..28e613ff3d4 100644
--- a/reference/test/stop/combined.cpp
+++ b/reference/test/stop/combined.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/stop/criterion_kernels.cpp b/reference/test/stop/criterion_kernels.cpp
index b2fa160f8e3..89b9b9665d8 100644
--- a/reference/test/stop/criterion_kernels.cpp
+++ b/reference/test/stop/criterion_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/stop/iteration.cpp b/reference/test/stop/iteration.cpp
index fd9d34114b5..5a10542ccea 100644
--- a/reference/test/stop/iteration.cpp
+++ b/reference/test/stop/iteration.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/stop/residual_norm_kernels.cpp b/reference/test/stop/residual_norm_kernels.cpp
index 8a2b31d8b38..6a25f249186 100644
--- a/reference/test/stop/residual_norm_kernels.cpp
+++ b/reference/test/stop/residual_norm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/stop/time.cpp b/reference/test/stop/time.cpp
index 70da8138ae7..60fb6a157a9 100644
--- a/reference/test/stop/time.cpp
+++ b/reference/test/stop/time.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/utils/assertions_test.cpp b/reference/test/utils/assertions_test.cpp
index ffc467e06aa..737557fc52a 100644
--- a/reference/test/utils/assertions_test.cpp
+++ b/reference/test/utils/assertions_test.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/test_install/test_install.cpp b/test_install/test_install.cpp
index c80b0976b34..589a52bf8c2 100644
--- a/test_install/test_install.cpp
+++ b/test_install/test_install.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/test_install/test_install_cuda.cu b/test_install/test_install_cuda.cu
index a4f46b371f8..f11b545c309 100644
--- a/test_install/test_install_cuda.cu
+++ b/test_install/test_install_cuda.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

From f15b8b3439a052ba4f2fd3136fab2de0c6f2fb28 Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Thu, 7 Jan 2021 11:08:26 +0000
Subject: [PATCH 0191/1008] Format files

Co-authored-by: Yuhsiang M. Tsai <yhmtsai@gmail.com>
---
 cuda/solver/idr_kernels.cu       | 2 +-
 hip/solver/idr_kernels.hip.cpp   | 2 +-
 omp/solver/idr_kernels.cpp       | 2 +-
 reference/solver/idr_kernels.cpp | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index ea207a99408..a493b2c8105 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -33,8 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
-#include <time.h>
 #include <random>
+#include <time.h>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index 37378e00cf9..5c6b3bd6e92 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -33,8 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
-#include <time.h>
 #include <random>
+#include <time.h>
 
 
 #include <hip/hip_runtime.h>
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index f03272ec8e8..59edde11e76 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -33,10 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
-#include <time.h>
 #include <algorithm>
 #include <iostream>
 #include <random>
+#include <time.h>
 
 
 #include <omp.h>
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index 17c34f00d04..e419b2b1556 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -33,10 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
-#include <time.h>
 #include <algorithm>
 #include <iostream>
 #include <random>
+#include <time.h>
 
 
 #include <ginkgo/core/base/array.hpp>

From 2e6b38fa9dd142c4e4d9d186350ff9adda37b524 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Thu, 31 Dec 2020 09:29:15 +0100
Subject: [PATCH 0192/1008] Fix GinkgoConfig.cmake. Test linking build dir.

Issue found by: https://github.com/ginkgo-project/ginkgo/pull/680
---
 .gitlab-ci.yml                  | 27 +++++++++++++++++++++++++++
 CMakeLists.txt                  | 16 ++++++++++++++++
 cmake/install_helpers.cmake     |  5 +++++
 test_exportbuild/CMakeLists.txt | 33 +++++++++++++++++++++++++++++++++
 test_install/CMakeLists.txt     |  7 +++----
 5 files changed, 84 insertions(+), 4 deletions(-)
 create mode 100644 test_exportbuild/CMakeLists.txt

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9969d84177c..668bbe9c0d6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,6 +26,7 @@ stages:
   CONFIG_LOG: "ON"
   CXX_FLAGS: ""
   EXTRA_CMAKE_FLAGS: ""
+  EXPORT_BUILD_DIR: "OFF"
   CI_PROJECT_DIR_SUFFIX: ""
 
 .before_script_template: &default_before_script
@@ -65,8 +66,10 @@ stages:
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
+        -DGINKGO_EXPORT_BUILD_DIR=${EXPORT_BUILD_DIR}
     - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT}
     - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then unset SYCL_DEVICE_TYPE; fi
+    - if [ "${EXPORT_BUILD_DIR}" == "ON" ]; then ninja test_exportbuild; fi
   dependencies: []
   except:
       - schedules
@@ -94,6 +97,7 @@ stages:
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
+        -DGINKGO_EXPORT_BUILD_DIR=${EXPORT_BUILD_DIR}
     - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT} install
     - |
         (( $(ctest -N | tail -1 | sed 's/Total Tests: //') != 0 )) || exit 1
@@ -121,6 +125,7 @@ stages:
         fi
       fi
     - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then unset SYCL_DEVICE_TYPE; fi
+    - if [ "${EXPORT_BUILD_DIR}" == "ON" ]; then ninja test_exportbuild; fi
   dependencies: []
   except:
       - schedules
@@ -742,6 +747,28 @@ subdir-build:
     - cuda
     - gpu
 
+# Ensure Ginkgo can be used when exporting the build directory
+export-build:
+  <<: *default_build
+  stage: code_quality
+  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
+  variables:
+    <<: *default_variables
+    BUILD_OMP: "ON"
+    BUILD_CUDA: "ON"
+    BUILD_HIP: "ON"
+    EXPORT_BUILD_DIR: "ON"
+  only:
+    variables:
+      - $RUN_CI_TAG
+  dependencies: []
+  allow_failure: no
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+
 # Run clang-tidy and iwyu
 clang-tidy:
   <<: *default_build
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5c2016d90bf..b1144e771e9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -290,11 +290,13 @@ ginkgo_install()
 if(MSVC)
     # Set path/command with $<CONFIG>
     set(GINKGO_TEST_INSTALL_COMMAND "${Ginkgo_BINARY_DIR}/test_install/$<CONFIG>/test_install")
+    set(GINKGO_TEST_EXPORTBUILD_COMMAND "${Ginkgo_BINARY_DIR}/test_exportbuild/$<CONFIG>/test_exportbuild")
     if(GINKGO_BUILD_CUDA)
         set(GINKGO_TEST_INSTALL_COMMAND "${GINKGO_TEST_INSTALL_COMMAND}" "${Ginkgo_BINARY_DIR}/test_install/$<CONFIG>/test_install_cuda")
     endif()
 else()
     set(GINKGO_TEST_INSTALL_COMMAND "${Ginkgo_BINARY_DIR}/test_install/test_install")
+    set(GINKGO_TEST_EXPORTBUILD_COMMAND "${Ginkgo_BINARY_DIR}/test_exportbuild/test_exportbuild")
     if(GINKGO_BUILD_CUDA)
         set(GINKGO_TEST_INSTALL_COMMAND "${GINKGO_TEST_INSTALL_COMMAND}" "${Ginkgo_BINARY_DIR}/test_install/test_install_cuda")
     endif()
@@ -312,6 +314,20 @@ add_custom_target(test_install
     COMMAND ${GINKGO_TEST_INSTALL_COMMAND}
     COMMENT "Running a test on the installed binaries. This requires running `(sudo) make install` first.")
 
+add_custom_target(test_exportbuild
+    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} -H${Ginkgo_SOURCE_DIR}/test_exportbuild
+    -B${Ginkgo_BINARY_DIR}/test_exportbuild
+    -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+    -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
+    # `--config cfg` is ignored by single-configuration generator.
+    # `$<CONFIG>` is always be the same as `CMAKE_BUILD_TYPE` in
+    # single-configuration generator.
+    COMMAND ${CMAKE_COMMAND} --build ${Ginkgo_BINARY_DIR}/test_exportbuild --config $<CONFIG>
+    COMMAND ${GINKGO_TEST_EXPORTBUILD_COMMAND}
+    COMMENT "Running a test on Ginkgo's exported build directory. "
+    "This requires compiling Ginkgo with `-DGINKGO_EXPORT_BUILD_DIR=ON` first.")
+
+
 # Setup CPack
 set(CPACK_PACKAGE_DESCRIPTION_FILE "${Ginkgo_SOURCE_DIR}/README.md")
 set(CPACK_RESOURCE_FILE_LICENSE "${Ginkgo_SOURCE_DIR}/LICENSE")
diff --git a/cmake/install_helpers.cmake b/cmake/install_helpers.cmake
index ba7ea3fd468..68a75a5f9a5 100644
--- a/cmake/install_helpers.cmake
+++ b/cmake/install_helpers.cmake
@@ -67,6 +67,11 @@ function(ginkgo_install)
         "${Ginkgo_BINARY_DIR}/GinkgoConfig.cmake"
         INSTALL_DESTINATION "${GINKGO_INSTALL_CONFIG_DIR}"
         )
+    set(HELPERS "hip_helpers.cmake" "windows_helpers.cmake")
+    foreach (helper ${HELPERS})
+        configure_file(${Ginkgo_SOURCE_DIR}/cmake/${helper}
+            ${Ginkgo_BINARY_DIR}/${helper} COPYONLY)
+    endforeach()
     install(FILES
         "${Ginkgo_BINARY_DIR}/GinkgoConfig.cmake"
         "${Ginkgo_BINARY_DIR}/GinkgoConfigVersion.cmake"
diff --git a/test_exportbuild/CMakeLists.txt b/test_exportbuild/CMakeLists.txt
new file mode 100644
index 00000000000..8cfb100263e
--- /dev/null
+++ b/test_exportbuild/CMakeLists.txt
@@ -0,0 +1,33 @@
+cmake_minimum_required(VERSION 3.9)
+project(GinkgoExportBuildTest LANGUAGES CXX)
+
+find_package(Ginkgo REQUIRED)
+
+if(MSVC)
+    if(GINKGO_BUILD_SHARED_LIBS)
+        ginkgo_switch_to_windows_dynamic("CXX")
+        ginkgo_switch_to_windows_dynamic("C")
+    else()
+        ginkgo_switch_to_windows_static("CXX")
+        ginkgo_switch_to_windows_static("C")
+    endif()
+endif()
+
+add_executable(test_exportbuild ../test_install/test_install.cpp)
+target_compile_features(test_exportbuild PUBLIC cxx_std_14)
+target_link_libraries(test_exportbuild PRIVATE Ginkgo::ginkgo)
+
+if(GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc"
+        AND GINKGO_HIP_VERSION VERSION_GREATER_EQUAL "3.5"
+        AND NOT GINKGO_BUILD_SHARED_LIBS)
+    # Compile options somehow add hip-clang specific flags. Wipe them.
+    # Currently, the flags wiped out should be:
+    # -x;hip;--hip-device-lib-path=/opt/rocm/lib;--cuda-gpu-arch=gfx900;
+    # --cuda-gpu-arch=gfx906
+    set_target_properties(hip::device PROPERTIES INTERFACE_COMPILE_OPTIONS "")
+    # In addition, link libraries have a similar problem. We only keep
+    # `hip::host`. Currently, the flags should be:
+    # hip::host;--hip-device-lib-path=/opt/rocm/lib;--hip-link;
+    # --cuda-gpu-arch=gfx900;--cuda-gpu-arch=gfx906
+    set_target_properties(hip::device PROPERTIES INTERFACE_LINK_LIBRARIES "hip::host")
+endif()
diff --git a/test_install/CMakeLists.txt b/test_install/CMakeLists.txt
index 49dc0ba2bf8..53a588f3359 100644
--- a/test_install/CMakeLists.txt
+++ b/test_install/CMakeLists.txt
@@ -37,10 +37,9 @@ if(GINKGO_BUILD_CUDA)
     target_link_libraries(test_install_cuda PRIVATE Ginkgo::ginkgo)
 endif()
 
-if(GINKGO_BUILD_HIP
-   AND GINKGO_HIP_PLATFORM MATCHES "hcc"
-   AND GINKGO_HIP_VERSION VERSION_GREATER_EQUAL "3.5"
-   AND NOT GINKGO_BUILD_SHARED_LIBS)
+if(GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc"
+        AND GINKGO_HIP_VERSION VERSION_GREATER_EQUAL "3.5"
+        AND NOT GINKGO_BUILD_SHARED_LIBS)
     # Compile options somehow add hip-clang specific flags. Wipe them.
     # Currently, the flags wiped out should be:
     # -x;hip;--hip-device-lib-path=/opt/rocm/lib;--cuda-gpu-arch=gfx900;

From 9e1a9048c721a875fda40115a18e5b0d6fad85a8 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 5 Jan 2021 09:33:06 +0100
Subject: [PATCH 0193/1008] Disable building CUDA and HIP for external build.

When testing Ginkgo as an external for the subdir and export build
tests, there should be no need to build CUDA and HIP as well since these
tests only check the usability from a standard C++ executable.
---
 .gitlab-ci.yml | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 668bbe9c0d6..ce90238f31f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -730,12 +730,10 @@ no-circular-deps:
 subdir-build:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
-    BUILD_CUDA: "ON"
-    BUILD_HIP: "ON"
     CI_PROJECT_PATH_SUFFIX: "/test_subdir"
   only:
     variables:
@@ -744,19 +742,16 @@ subdir-build:
   allow_failure: no
   tags:
     - private_ci
-    - cuda
-    - gpu
+    - cpu
 
 # Ensure Ginkgo can be used when exporting the build directory
 export-build:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
-    BUILD_CUDA: "ON"
-    BUILD_HIP: "ON"
     EXPORT_BUILD_DIR: "ON"
   only:
     variables:
@@ -765,9 +760,7 @@ export-build:
   allow_failure: no
   tags:
     - private_ci
-    - cuda
-    - gpu
-
+    - cpu
 
 # Run clang-tidy and iwyu
 clang-tidy:

From c56c2d9473ce1cb0197a9e87cea3f5f467174fb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 11 Sep 2020 18:00:36 +0200
Subject: [PATCH 0194/1008] Create new accessor in range_accessors.hpp

---
 include/ginkgo/core/base/range_accessors.hpp | 516 +++++++++++++++++++
 include/ginkgo/core/base/types.hpp           |   2 +
 2 files changed, 518 insertions(+)

diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 4435c51f1b3..0ad6743c506 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -35,9 +35,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <array>
+#include <memory>
+#include <type_traits>
 
 
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/range.hpp>
+#include <ginkgo/core/base/types.hpp>
 
 
 namespace gko {
@@ -186,6 +192,516 @@ class row_major {
 };
 
 
+namespace detail {
+
+
+/**
+ * This is a mixin which defines the binary operators for *, /, +, - for the
+ * Reference class, the unary operator -, and the assignment operators
+ * *=, /=, +=, -=
+ * Additionally, it prevents the default generation of copy and move constructor
+ * and copy and move assignment.
+ *
+ * @warning  This struct should only be used by reference classes.
+ */
+template <typename Reference, typename ArithmeticType>
+struct enable_reference {
+    using arithmetic_type = ArithmeticType;
+
+    /*GKO_ATTRIBUTES*/ enable_reference() =
+        default;  // Needs to exist, so creation is not probibited
+    enable_reference(enable_reference &&) = delete;
+    enable_reference(const enable_reference &) = delete;
+    enable_reference &operator=(const enable_reference &) = delete;
+    enable_reference &operator=(enable_reference &&) = delete;
+
+#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_oper, _op)            \
+    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper( \
+        const Reference &ref1, const Reference &ref2)                 \
+    {                                                                 \
+        return static_cast<arithmetic_type>(ref1)                     \
+            _op static_cast<arithmetic_type>(ref2);                   \
+    }                                                                 \
+    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper( \
+        const Reference &ref, const arithmetic_type &a)               \
+    {                                                                 \
+        return static_cast<arithmetic_type>(ref) _op a;               \
+    }                                                                 \
+    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper( \
+        const arithmetic_type &a, const Reference &ref)               \
+    {                                                                 \
+        return a _op static_cast<arithmetic_type>(ref);               \
+    }
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator*, *)
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator/, /)
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator+, +)
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator-, -)
+#undef GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD
+
+#define GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(_oper, _op)           \
+    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper(    \
+        Reference &&ref1, const Reference &ref2)                         \
+    {                                                                    \
+        return std::move(ref1) = static_cast<arithmetic_type>(ref1)      \
+                   _op static_cast<arithmetic_type>(ref2);               \
+    }                                                                    \
+    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper(    \
+        Reference &&ref, const arithmetic_type &a)                       \
+    {                                                                    \
+        return std::move(ref) = static_cast<arithmetic_type>(ref) _op a; \
+    }
+
+    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator*=, *)
+    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator/=, /)
+    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator+=, +)
+    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator-=, -)
+#undef GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD
+
+    // TODO test if comparison operators need to be overloaded as well
+
+    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type
+    operator-(const Reference &ref)
+    {
+        return -static_cast<arithmetic_type>(ref);
+    }
+};
+
+
+/**
+ * This namespace contains reference classes used inside accessors.
+ *
+ * @warning  This class should not be used by anything else but accessors.
+ */
+namespace reference {
+
+
+/**
+ * Reference class for a different storage than arithmetic type. The conversion
+ * between both formats is done with a simple static_cast.
+ */
+template <typename ArithmeticType, typename StorageType>
+class ReducedStorageReference
+    : public detail::enable_reference<
+          ReducedStorageReference<ArithmeticType, StorageType>,
+          ArithmeticType> {
+public:
+    using arithmetic_type = ArithmeticType;
+    using storage_type = StorageType;
+    ReducedStorageReference() = delete;
+    GKO_ATTRIBUTES ReducedStorageReference(storage_type *const GKO_RESTRICT ptr)
+        : ptr_{ptr}
+    {}
+    GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
+    {
+        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return static_cast<arithmetic_type>(*r_ptr);
+    }
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator=(arithmetic_type val) &&
+    {
+        storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return *r_ptr = static_cast<storage_type>(val);
+    }
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type
+    operator=(const ReducedStorageReference &ref) &&
+    {
+        return std::move(*this) = static_cast<arithmetic_type>(ref);
+    }
+
+private:
+    storage_type *const GKO_RESTRICT ptr_;
+};
+
+/**
+ * Reference class for a different storage than arithmetic type. The conversion
+ * between both formats is done with a simple static_cast followed by a
+ * multiplication with a scale.
+ */
+template <typename ArithmeticType, typename StorageType>
+class ScaledReducedStorageReference
+    : public detail::enable_reference<
+          ScaledReducedStorageReference<ArithmeticType, StorageType>,
+          ArithmeticType> {
+public:
+    using arithmetic_type = ArithmeticType;
+    using storage_type = StorageType;
+
+    ScaledReducedStorageReference() = delete;
+    GKO_ATTRIBUTES ScaledReducedStorageReference(
+        storage_type *const GKO_RESTRICT ptr, arithmetic_type scale)
+        : ptr_{ptr}, scale_{scale}
+    {}
+    GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
+    {
+        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return static_cast<arithmetic_type>(*r_ptr) * scale_;
+    }
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator=(arithmetic_type val) &&
+    {
+        storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return *r_ptr = static_cast<storage_type>(val / scale_), val;
+    }
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type
+    operator=(const ScaledReducedStorageReference &ref) &&
+    {
+        return std::move(*this) = static_cast<arithmetic_type>(ref);
+    }
+
+private:
+    storage_type *const GKO_RESTRICT ptr_;
+    const arithmetic_type scale_;
+};
+
+
+}  // namespace reference
+}  // namespace detail
+
+
+/**
+ * The ReducedStorage3d class allows a storage format that is different from
+ * the arithmetic format (which is returned from the brace operator).
+ * As storage, the storage_type is used.
+ * This accessor uses row-major access.
+ *
+ * @note  This class only manages the accesses and not the memory itself.
+ */
+template <typename ArithmeticType, typename StorageType>
+class ConstReducedStorage3d {
+public:
+    using arithmetic_type = ArithmeticType;
+    using storage_type = StorageType;
+    using const_accessor = ConstReducedStorage3d;
+    static_assert(!std::is_const<storage_type>::value,
+                  "StorageType must not be const!");
+    static constexpr size_type dimensionality{3};
+
+protected:
+    using reference =
+        detail::reference::ReducedStorageReference<ArithmeticType, StorageType>;
+
+public:
+    /**
+     * Creates the accessor with an already allocated storage space with a
+     * stride.
+     *
+     * @internal
+     * const_cast needed, so ReducedStorage3d(ac.get_storage(), ...)
+     * works. Also, storage_ is never accessed in a non-const fashion in this
+     * class, so it is not invalid or UB code.
+     */
+    GKO_ATTRIBUTES ConstReducedStorage3d(const storage_type *storage,
+                                         size_type stride0, size_type stride1)
+        : storage_{const_cast<storage_type *>(storage)},
+          stride_{stride0, stride1}
+    {}
+
+    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const { return *this; }
+
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator()(IndexType x,
+                                                         IndexType y,
+                                                         IndexType z) const
+    {
+        return reference{storage_ + compute_index(x, y, z)};
+    }
+
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type at(IndexType x, IndexType y,
+                                                 IndexType z) const
+    {
+        return reference{storage_ + compute_index(x, y, z)};
+    }
+
+    GKO_ATTRIBUTES GKO_INLINE size_type get_stride0() const
+    {
+        return stride_[0];
+    }
+    GKO_ATTRIBUTES GKO_INLINE size_type get_stride1() const
+    {
+        return stride_[1];
+    }
+
+    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_storage() const
+    {
+        return storage_;
+    }
+
+    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_const_storage() const
+    {
+        return storage_;
+    }
+
+protected:
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE constexpr size_type compute_index(
+        IndexType x, IndexType y, IndexType z) const
+    {
+        return x * stride_[0] + y * stride_[1] + z;
+    }
+
+    storage_type *storage_;
+    size_type stride_[2];  // std::array leads to conflicts on devices
+};
+
+template <typename ArithmeticType, typename StorageType>
+class ReducedStorage3d
+    : public ConstReducedStorage3d<ArithmeticType, StorageType> {
+public:
+    using arithmetic_type = ArithmeticType;
+    using storage_type = StorageType;
+    using const_accessor = ConstReducedStorage3d<arithmetic_type, storage_type>;
+
+protected:
+    using reference = typename const_accessor::reference;
+
+public:
+    GKO_ATTRIBUTES ReducedStorage3d() : const_accessor(nullptr, 0, 0) {}
+
+    GKO_ATTRIBUTES ReducedStorage3d(storage_type *storage, size_type stride0,
+                                    size_type stride1)
+        : const_accessor(storage, stride0, stride1)
+    {}
+
+    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const
+    {
+        return {this->storage_, this->stride_[0], this->stride_[1]};
+    }
+
+    using const_accessor::get_storage;
+    GKO_ATTRIBUTES GKO_INLINE storage_type *get_storage()
+    {
+        return this->storage_;
+    }
+
+    // Makes sure the operator() const function is visible
+    using const_accessor::operator();
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE reference operator()(IndexType x, IndexType y,
+                                                   IndexType z)
+    {
+        return {this->storage_ + const_accessor::compute_index(x, y, z)};
+    }
+
+    using const_accessor::at;  // Makes sure the at() const function is visible
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE reference at(IndexType x, IndexType y,
+                                           IndexType z)
+    {
+        return {this->storage_ + const_accessor::compute_index(x, y, z)};
+    }
+};
+
+
+/**
+ * @internal
+ *
+ * The ScaledReducedStorage3d class hides the underlying storage_ format and
+ * provides a simple interface for accessing a one dimensional storage.
+ * Additionally, this accessor posesses a scale array, which is used for each
+ * read and write operation to do a proper conversion.
+ *
+ * This class only manages the accesses, however, and not the memory itself.
+ *
+ * TODO move this documentation to the core since this is a general purpose
+ * accessor
+ *
+ * The dimensions {x, y, z} explained for the krylov_bases:
+ * - x: selects the krylov vector (usually has krylov_dim + 1 vectors)
+ * - y: selects the (row-)element of said krylov vector
+ * - z: selects which column-element of said krylov vector should be used
+ *
+ * The accessor uses row-major access.
+ */
+template <typename ArithmeticType, typename StorageType>
+class ConstScaledReducedStorage3d {
+public:
+    using arithmetic_type = ArithmeticType;
+    using storage_type = StorageType;
+    using const_accessor = ConstScaledReducedStorage3d;
+    static_assert(!std::is_const<storage_type>::value,
+                  "StorageType must not be const!");
+    static constexpr size_type dimensionality{3};
+
+protected:
+    using reference =
+        detail::reference::ScaledReducedStorageReference<ArithmeticType,
+                                                         StorageType>;
+
+public:
+    /**
+     * Creates the accessor with an already allocated storage space with a
+     * stride.
+     *
+     * @internal
+     * const_cast needed, so ScaledReducedStorage3d(ac.get_storage(), ...)
+     * works. Also, storage_ is never accessed in a non-const fashion (from
+     * ConstScaledReducedStorage3d), so it is not invalid or UB code.
+     */
+    GKO_ATTRIBUTES ConstScaledReducedStorage3d(const storage_type *storage,
+                                               size_type stride0,
+                                               size_type stride1,
+                                               const arithmetic_type *scale)
+        : storage_{const_cast<storage_type *>(storage)},
+          stride_{stride0, stride1},
+          scale_{const_cast<arithmetic_type *>(scale)}
+    {}
+
+    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const { return *this; }
+
+    /**
+     * Reads the scale value at the given indices.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type read_scale(IndexType x,
+                                                         IndexType z) const
+    {
+        const arithmetic_type *GKO_RESTRICT rest_scale = scale_;
+        return rest_scale[x * stride_[1] + z];
+    }
+
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator()(IndexType x,
+                                                         IndexType y,
+                                                         IndexType z) const
+    {
+        return reference{storage_ + compute_index(x, y, z), read_scale(x, z)};
+    }
+
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type at(IndexType x, IndexType y,
+                                                 IndexType z) const
+    {
+        return reference{storage_ + compute_index(x, y, z), read_scale(x, z)};
+    }
+
+    GKO_ATTRIBUTES GKO_INLINE size_type get_stride0() const
+    {
+        return stride_[0];
+    }
+    GKO_ATTRIBUTES GKO_INLINE size_type get_stride1() const
+    {
+        return stride_[1];
+    }
+
+    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_storage() const
+    {
+        return storage_;
+    }
+
+    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_const_storage() const
+    {
+        return storage_;
+    }
+
+    GKO_ATTRIBUTES GKO_INLINE const arithmetic_type *get_scale() const
+    {
+        return this->scale_;
+    }
+
+    GKO_ATTRIBUTES GKO_INLINE const arithmetic_type *get_const_scale() const
+    {
+        return this->scale_;
+    }
+
+protected:
+    template <typename IndexType>
+    GKO_ATTRIBUTES constexpr GKO_INLINE size_type
+    compute_index(IndexType x, IndexType y, IndexType z) const
+    {
+        return x * stride_[0] + y * stride_[1] + z;
+    }
+
+    storage_type *storage_;
+    size_type stride_[2];  // std::array leads to conflicts on devices
+    arithmetic_type *scale_;
+};
+
+
+template <typename ArithmeticType, typename StorageType>
+class ScaledReducedStorage3d
+    : public ConstScaledReducedStorage3d<ArithmeticType, StorageType> {
+public:
+    using arithmetic_type = ArithmeticType;
+    using storage_type = StorageType;
+    using const_accessor =
+        ConstScaledReducedStorage3d<arithmetic_type, storage_type>;
+
+protected:
+    using reference = typename const_accessor::reference;
+
+public:
+    GKO_ATTRIBUTES ScaledReducedStorage3d()
+        : const_accessor(nullptr, {}, {}, nullptr)
+    {}
+
+    GKO_ATTRIBUTES ScaledReducedStorage3d(storage_type *storage,
+                                          size_type stride0, size_type stride1,
+                                          arithmetic_type *scale)
+        : const_accessor(storage, stride0, stride1, scale)
+    {}
+
+    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const
+    {
+        return {this->storage_, this->stride_[0], this->stride_[1],
+                this->scale_};
+    }
+
+    // Makes sure the operator() const function is visible
+    using const_accessor::operator();
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE reference operator()(IndexType x, IndexType y,
+                                                   IndexType z)
+    {
+        return {this->storage_ + const_accessor::compute_index(x, y, z),
+                this->read_scale(x, z)};
+    }
+
+
+    using const_accessor::at;
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE reference at(IndexType x, IndexType y,
+                                           IndexType z)
+    {
+        return {this->storage_ + const_accessor::compute_index(x, y, z),
+                this->read_scale(x, z)};
+    }
+
+    /**
+     * Writes the given value at the given index.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE void write(IndexType x, IndexType y, IndexType z,
+                                         arithmetic_type value)
+    {
+        storage_type *GKO_RESTRICT rest_storage = this->storage_;
+        rest_storage[const_accessor::compute_index(x, y, z)] =
+            static_cast<storage_type>(value / this->read_scale(x, z));
+    }
+
+    /**
+     * Writes the given value at the given index for a scale.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE void set_scale(IndexType x, IndexType z,
+                                             arithmetic_type val)
+    {
+        arithmetic_type *GKO_RESTRICT rest_scale = this->scale_;
+        rest_scale[x * this->stride_[1] + z] = val;
+    }
+
+    using const_accessor::get_storage;
+    GKO_ATTRIBUTES GKO_INLINE storage_type *get_storage()
+    {
+        return this->storage_;
+    }
+
+    using const_accessor::get_scale;
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type *get_scale()
+    {
+        return this->scale_;
+    }
+};
+
+
 }  // namespace accessor
 }  // namespace gko
 
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 0c908b677e8..1a23ade2b0c 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -52,9 +52,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #if defined(__CUDACC__) || defined(__HIPCC__)
 #define GKO_ATTRIBUTES __host__ __device__
 #define GKO_INLINE __forceinline__
+#define GKO_RESTRICT __restrict__
 #else
 #define GKO_ATTRIBUTES
 #define GKO_INLINE inline
+#define GKO_RESTRICT
 #endif  // defined(__CUDACC__) || defined(__HIPCC__)
 
 

From 492320f439c52c6a132c5dcf05ca400e807b5e6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 14 Sep 2020 07:53:37 +0200
Subject: [PATCH 0195/1008] Make accessor range compatible

---
 core/test/base/range_accessors.cpp           |  88 ++++++
 include/ginkgo/core/base/range.hpp           |  11 +-
 include/ginkgo/core/base/range_accessors.hpp | 302 ++++++++++++++-----
 include/ginkgo/core/base/types.hpp           |   8 +
 4 files changed, 325 insertions(+), 84 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 967397e011b..5f4a69deb9f 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -67,6 +67,23 @@ TEST_F(RowMajorAccessor, CanAccessData)
 }
 
 
+TEST_F(RowMajorAccessor, CanWriteData)
+{
+    r(0, 0) = 4;
+
+    EXPECT_EQ(r(0, 0), 4);
+}
+
+
+TEST_F(RowMajorAccessor, CanWriteDataToConst)
+{
+    const row_major_int_range cr = r;
+    cr(0, 0) = 4;
+
+    EXPECT_EQ(cr(0, 0), 4);
+}
+
+
 TEST_F(RowMajorAccessor, CanCreateSubrange)
 {
     auto subr = r(span{1, 3}, span{0, 2});
@@ -121,4 +138,75 @@ TEST_F(RowMajorAccessor, CanAssignSubranges)
 }
 
 
+class ReducedStorage3d : public ::testing::Test {
+protected:
+    using span = gko::span;
+    using ar_type = double;
+    using st_type = double;
+
+    using accessor = gko::accessor::ReducedStorage3d<ar_type, st_type>;
+
+    using reduced_storage = gko::range<accessor>;
+
+    // clang-format off
+    double data[8]{
+        1.0, 2.1,
+        -1.2, 3.3,
+        4.4, -2.5,
+        5.6, 6.7
+    };
+    //clang-format on
+    reduced_storage r{data, gko::dim<3>{2u, 2u, 2u}};
+};
+
+
+TEST_F(ReducedStorage3d, CanReadData)
+{
+    EXPECT_EQ(static_cast<ar_type>(r(0, 0, 0)), 1.0);
+    EXPECT_EQ(static_cast<ar_type>(r(0, 0, 1)), 2.1);
+    EXPECT_EQ(static_cast<ar_type>(r(0, 1, 0)), -1.2);
+    EXPECT_EQ(static_cast<ar_type>(r(0, 1, 1)), 3.3);
+    EXPECT_EQ(static_cast<ar_type>(r(1, 0, 0)), 4.4);
+    EXPECT_EQ(static_cast<ar_type>(r(1, 0, 1)), -2.5);
+    EXPECT_EQ(static_cast<ar_type>(r(1, 1, 0)), 5.6);
+    EXPECT_EQ(static_cast<ar_type>(r(1, 1, 1)), 6.7);
+}
+
+
+TEST_F(ReducedStorage3d, CanWriteData)
+{
+    r(0, 0, 0) = 2.0;
+
+    EXPECT_EQ(r(0, 0, 0), 2.0);
+}
+
+
+TEST_F(ReducedStorage3d, CanCreateSubrange)
+{
+    auto subr = r(span{1, 2}, span{0, 2}, span{0, 1});
+
+    EXPECT_EQ(static_cast<ar_type>(subr(0, 0, 0)), 4.4);
+    EXPECT_EQ(static_cast<ar_type>(subr(0, 1, 0)), 5.6);
+}
+
+
+TEST_F(ReducedStorage3d, CanCreateRowVector)
+{
+    auto subr = r(0, 0, span{0, 2});
+
+    EXPECT_EQ(subr(0, 0, 0), 1.0);
+    EXPECT_EQ(subr(0, 0, 1), 2.1);
+}
+
+
+TEST_F(ReducedStorage3d, CanCreateColumnVector)
+{
+    auto subr = r(span{0, 2}, 0, 0);
+
+    EXPECT_EQ(subr(0, 0, 0), 1.0);
+    EXPECT_EQ(subr(1, 0, 0), 4.4);
+}
+
+
+
 }  // namespace
diff --git a/include/ginkgo/core/base/range.hpp b/include/ginkgo/core/base/range.hpp
index 2fe87a364a7..c0d275d1307 100644
--- a/include/ginkgo/core/base/range.hpp
+++ b/include/ginkgo/core/base/range.hpp
@@ -339,7 +339,7 @@ class range {
      */
     template <typename... DimensionTypes>
     GKO_ATTRIBUTES constexpr auto operator()(DimensionTypes &&... dimensions)
-        const -> decltype(std::declval<accessor>()(
+        const -> decltype(std::declval<const accessor>()(
             std::forward<DimensionTypes>(dimensions)...))
     {
         static_assert(sizeof...(dimensions) <= dimensionality,
@@ -347,6 +347,15 @@ class range {
         return accessor_(std::forward<DimensionTypes>(dimensions)...);
     }
 
+    template <typename... DimensionTypes>
+    GKO_ATTRIBUTES auto operator()(DimensionTypes &&... dimensions) -> decltype(
+        std::declval<accessor>()(std::forward<DimensionTypes>(dimensions)...))
+    {
+        static_assert(sizeof...(dimensions) <= dimensionality,
+                      "Too many dimensions in range call");
+        return accessor_(std::forward<DimensionTypes>(dimensions)...);
+    }
+
     /**
      * @copydoc operator=(const range &other)
      *
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 0ad6743c506..b412b05e130 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -208,13 +208,6 @@ template <typename Reference, typename ArithmeticType>
 struct enable_reference {
     using arithmetic_type = ArithmeticType;
 
-    /*GKO_ATTRIBUTES*/ enable_reference() =
-        default;  // Needs to exist, so creation is not probibited
-    enable_reference(enable_reference &&) = delete;
-    enable_reference(const enable_reference &) = delete;
-    enable_reference &operator=(const enable_reference &) = delete;
-    enable_reference &operator=(enable_reference &&) = delete;
-
 #define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_oper, _op)            \
     friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper( \
         const Reference &ref1, const Reference &ref2)                 \
@@ -287,7 +280,14 @@ class ReducedStorageReference
 public:
     using arithmetic_type = ArithmeticType;
     using storage_type = StorageType;
+    // Allow move construction, so perfect forwarding is possible
+    ReducedStorageReference(ReducedStorageReference &&) = default;
+
     ReducedStorageReference() = delete;
+    // Forbid copy construction and move assignment
+    ReducedStorageReference(const ReducedStorageReference &) = delete;
+    ReducedStorageReference &operator=(ReducedStorageReference &&) = delete;
+
     GKO_ATTRIBUTES ReducedStorageReference(storage_type *const GKO_RESTRICT ptr)
         : ptr_{ptr}
     {}
@@ -325,7 +325,16 @@ class ScaledReducedStorageReference
     using arithmetic_type = ArithmeticType;
     using storage_type = StorageType;
 
+    // Allow move construction, so perfect forwarding is possible
+    ScaledReducedStorageReference(ScaledReducedStorageReference &&) = default;
+
     ScaledReducedStorageReference() = delete;
+    // Forbid copy construction and move assignment
+    ScaledReducedStorageReference(const ScaledReducedStorageReference &) =
+        delete;
+    ScaledReducedStorageReference &operator=(ScaledReducedStorageReference &&) =
+        delete;
+
     GKO_ATTRIBUTES ScaledReducedStorageReference(
         storage_type *const GKO_RESTRICT ptr, arithmetic_type scale)
         : ptr_{ptr}, scale_{scale}
@@ -363,6 +372,8 @@ class ScaledReducedStorageReference
  * This accessor uses row-major access.
  *
  * @note  This class only manages the accesses and not the memory itself.
+ *
+ * TODO Rename, so it is in lower_case only (since no virtual functions!)
  */
 template <typename ArithmeticType, typename StorageType>
 class ConstReducedStorage3d {
@@ -389,28 +400,69 @@ class ConstReducedStorage3d {
      * class, so it is not invalid or UB code.
      */
     GKO_ATTRIBUTES ConstReducedStorage3d(const storage_type *storage,
+                                         gko::dim<dimensionality> size,
                                          size_type stride0, size_type stride1)
         : storage_{const_cast<storage_type *>(storage)},
+          size_{size},
           stride_{stride0, stride1}
     {}
 
+    GKO_ATTRIBUTES ConstReducedStorage3d(const storage_type *storage,
+                                         dim<dimensionality> size)
+        : storage_{const_cast<storage_type *>(storage)},
+          size_{size},
+          stride_{size_[1] * size[2], size[2]}
+    {}
+
     GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const { return *this; }
 
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator()(IndexType x,
-                                                         IndexType y,
-                                                         IndexType z) const
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator()(size_type x,
+                                                         size_type y,
+                                                         size_type z) const
     {
         return reference{storage_ + compute_index(x, y, z)};
     }
 
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type at(IndexType x, IndexType y,
-                                                 IndexType z) const
+    // Functions required by the `range` interface:
+    GKO_ATTRIBUTES GKO_INLINE dim<dimensionality> length(
+        size_type dimension) const
     {
-        return reference{storage_ + compute_index(x, y, z)};
+        return dimension < dimensionality ? size_[dimension] : 1;
+    }
+    template <typename OtherAccessor>
+    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other)
+    {
+        // TODO figure out how to do this properly in an const accessor...
+        //      Additionally, this will always be inefficient!
+        for (size_type i = 0; i < size_[0]; ++i) {
+            for (size_type j = 0; j < size_[1]; ++j) {
+                for (size_type k = 0; k < size_[2]; ++k) {
+                    //(*this)(i, j, k) = other(i, j, k);
+                }
+            }
+        }
+    }
+    GKO_ATTRIBUTES constexpr range<ConstReducedStorage3d> operator()(
+        const span &x_span, const span &y_span, const span &z_span) const
+    {
+        return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
+               GKO_ASSERT(z_span.is_valid()),
+               GKO_ASSERT(x_span <= span{size_[0]}),
+               GKO_ASSERT(y_span <= span{size_[1]}),
+               GKO_ASSERT(z_span <= span{size_[2]}),
+               range<ConstReducedStorage3d>(
+                   storage_ +
+                       compute_index(x_span.begin, y_span.begin, z_span.begin),
+                   dim<dimensionality>{x_span.end - x_span.begin,
+                                       y_span.end - y_span.begin,
+                                       z_span.end - z_span.begin},
+                   stride_[0], stride_[1]);
     }
 
+    GKO_ATTRIBUTES GKO_INLINE dim<dimensionality> get_size() const
+    {
+        return size_;
+    }
     GKO_ATTRIBUTES GKO_INLINE size_type get_stride0() const
     {
         return stride_[0];
@@ -431,14 +483,15 @@ class ConstReducedStorage3d {
     }
 
 protected:
-    template <typename IndexType>
     GKO_ATTRIBUTES GKO_INLINE constexpr size_type compute_index(
-        IndexType x, IndexType y, IndexType z) const
+        size_type x, size_type y, size_type z) const
     {
-        return x * stride_[0] + y * stride_[1] + z;
+        return GKO_ASSERT(x < size_[0]), GKO_ASSERT(y < size_[1]),
+               GKO_ASSERT(z < size_[2]), x * stride_[0] + y * stride_[1] + z;
     }
 
     storage_type *storage_;
+    dim<dimensionality> size_;
     size_type stride_[2];  // std::array leads to conflicts on devices
 };
 
@@ -449,21 +502,58 @@ class ReducedStorage3d
     using arithmetic_type = ArithmeticType;
     using storage_type = StorageType;
     using const_accessor = ConstReducedStorage3d<arithmetic_type, storage_type>;
+    static constexpr size_type dimensionality = const_accessor::dimensionality;
 
 protected:
     using reference = typename const_accessor::reference;
 
 public:
-    GKO_ATTRIBUTES ReducedStorage3d() : const_accessor(nullptr, 0, 0) {}
+    GKO_ATTRIBUTES ReducedStorage3d() : const_accessor(nullptr, {0, 0, 0}) {}
 
-    GKO_ATTRIBUTES ReducedStorage3d(storage_type *storage, size_type stride0,
+    GKO_ATTRIBUTES ReducedStorage3d(storage_type *storage,
+                                    dim<dimensionality> size, size_type stride0,
                                     size_type stride1)
-        : const_accessor(storage, stride0, stride1)
+        : const_accessor(storage, size, stride0, stride1)
+    {}
+
+    GKO_ATTRIBUTES ReducedStorage3d(storage_type *storage,
+                                    dim<dimensionality> size)
+        : const_accessor(storage, size)
     {}
 
     GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const
     {
-        return {this->storage_, this->stride_[0], this->stride_[1]};
+        return {this->storage_, this->size_, this->stride_[0],
+                this->stride_[1]};
+    }
+
+    template <typename OtherAccessor>
+    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other)
+    {
+        for (size_type i = 0; i < this->size_[0]; ++i) {
+            for (size_type j = 0; j < this->size_[1]; ++j) {
+                for (size_type k = 0; k < this->size_[2]; ++k) {
+                    (*this)(i, j, k) = other(i, j, k);
+                }
+            }
+        }
+    }
+    GKO_ATTRIBUTES constexpr range<ReducedStorage3d> operator()(
+        const span &x_span, const span &y_span, const span &z_span) const
+    {
+        return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
+               GKO_ASSERT(z_span.is_valid()),
+               GKO_ASSERT(x_span <= span{this->size_[0]}),
+               GKO_ASSERT(y_span <= span{this->size_[1]}),
+               GKO_ASSERT(z_span <= span{this->size_[2]}),
+               range<ReducedStorage3d>(
+                   this->storage_ + this->compute_index(x_span.begin,
+                                                        y_span.begin,
+                                                        z_span.begin),
+                   dim<dimensionality>{x_span.end - x_span.begin,
+                                       y_span.end - y_span.begin,
+                                       z_span.end - z_span.begin},
+                   this->stride_[0], this->stride_[1]);
     }
 
     using const_accessor::get_storage;
@@ -474,17 +564,8 @@ class ReducedStorage3d
 
     // Makes sure the operator() const function is visible
     using const_accessor::operator();
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE reference operator()(IndexType x, IndexType y,
-                                                   IndexType z)
-    {
-        return {this->storage_ + const_accessor::compute_index(x, y, z)};
-    }
-
-    using const_accessor::at;  // Makes sure the at() const function is visible
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE reference at(IndexType x, IndexType y,
-                                           IndexType z)
+    GKO_ATTRIBUTES GKO_INLINE reference operator()(size_type x, size_type y,
+                                                   size_type z)
     {
         return {this->storage_ + const_accessor::compute_index(x, y, z)};
     }
@@ -501,14 +582,6 @@ class ReducedStorage3d
  *
  * This class only manages the accesses, however, and not the memory itself.
  *
- * TODO move this documentation to the core since this is a general purpose
- * accessor
- *
- * The dimensions {x, y, z} explained for the krylov_bases:
- * - x: selects the krylov vector (usually has krylov_dim + 1 vectors)
- * - y: selects the (row-)element of said krylov vector
- * - z: selects which column-element of said krylov vector should be used
- *
  * The accessor uses row-major access.
  */
 template <typename ArithmeticType, typename StorageType>
@@ -537,42 +610,84 @@ class ConstScaledReducedStorage3d {
      * ConstScaledReducedStorage3d), so it is not invalid or UB code.
      */
     GKO_ATTRIBUTES ConstScaledReducedStorage3d(const storage_type *storage,
+                                               dim<dimensionality> size,
                                                size_type stride0,
                                                size_type stride1,
                                                const arithmetic_type *scale)
         : storage_{const_cast<storage_type *>(storage)},
+          size_{size},
           stride_{stride0, stride1},
           scale_{const_cast<arithmetic_type *>(scale)}
     {}
+    GKO_ATTRIBUTES ConstScaledReducedStorage3d(const storage_type *storage,
+                                               dim<dimensionality> size,
+                                               const arithmetic_type *scale)
+        : storage_{const_cast<storage_type *>(storage)},
+          size_{size},
+          stride_{size_[1] * size_[2], size_[2]},
+          scale_{const_cast<arithmetic_type *>(scale)}
+    {}
 
     GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const { return *this; }
 
     /**
      * Reads the scale value at the given indices.
      */
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type read_scale(IndexType x,
-                                                         IndexType z) const
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type read_scale(size_type x,
+                                                         size_type z) const
     {
         const arithmetic_type *GKO_RESTRICT rest_scale = scale_;
-        return rest_scale[x * stride_[1] + z];
+        return GKO_ASSERT(x < size_[0]), GKO_ASSERT(z < size_[2]),
+               rest_scale[x * stride_[1] + z];
     }
 
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator()(IndexType x,
-                                                         IndexType y,
-                                                         IndexType z) const
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator()(size_type x,
+                                                         size_type y,
+                                                         size_type z) const
     {
         return reference{storage_ + compute_index(x, y, z), read_scale(x, z)};
     }
 
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type at(IndexType x, IndexType y,
-                                                 IndexType z) const
+    // Functions required by the `range` interface:
+    GKO_ATTRIBUTES GKO_INLINE dim<dimensionality> length(
+        size_type dimension) const
     {
-        return reference{storage_ + compute_index(x, y, z), read_scale(x, z)};
+        return dimension < dimensionality ? size_[dimension] : 1;
+    }
+    template <typename OtherAccessor>
+    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other)
+    {
+        // TODO figure out how to do this properly in an const accessor...
+        //      Additionally, this will always be inefficient!
+        for (size_type i = 0; i < size_[0]; ++i) {
+            for (size_type j = 0; j < size_[1]; ++j) {
+                for (size_type k = 0; k < size_[2]; ++k) {
+                    //(*this)(i, j, k) = other(i, j, k);
+                }
+            }
+        }
+    }
+    GKO_ATTRIBUTES constexpr range<ConstScaledReducedStorage3d> operator()(
+        const span &x_span, const span &y_span, const span &z_span) const
+    {
+        return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
+               GKO_ASSERT(z_span.is_valid()),
+               GKO_ASSERT(x_span <= span{size_[0]}),
+               GKO_ASSERT(y_span <= span{size_[1]}),
+               GKO_ASSERT(z_span <= span{size_[2]}),
+               range<ConstScaledReducedStorage3d>(
+                   storage_ +
+                       compute_index(x_span.begin, y_span.begin, z_span.begin),
+                   dim<dimensionality>{x_span.end - x_span.begin,
+                                       y_span.end - y_span.begin,
+                                       z_span.end - z_span.begin},
+                   stride_[0], stride_[1], scale_);
     }
 
+    GKO_ATTRIBUTES GKO_INLINE dim<dimensionality> get_size() const
+    {
+        return size_;
+    }
     GKO_ATTRIBUTES GKO_INLINE size_type get_stride0() const
     {
         return stride_[0];
@@ -603,14 +718,15 @@ class ConstScaledReducedStorage3d {
     }
 
 protected:
-    template <typename IndexType>
     GKO_ATTRIBUTES constexpr GKO_INLINE size_type
-    compute_index(IndexType x, IndexType y, IndexType z) const
+    compute_index(size_type x, size_type y, size_type z) const
     {
-        return x * stride_[0] + y * stride_[1] + z;
+        return GKO_ASSERT(x < size_[0]), GKO_ASSERT(y < size_[1]),
+               GKO_ASSERT(z < size_[2]), x * stride_[0] + y * stride_[1] + z;
     }
 
     storage_type *storage_;
+    dim<dimensionality> size_;
     size_type stride_[2];  // std::array leads to conflicts on devices
     arithmetic_type *scale_;
 };
@@ -624,66 +740,86 @@ class ScaledReducedStorage3d
     using storage_type = StorageType;
     using const_accessor =
         ConstScaledReducedStorage3d<arithmetic_type, storage_type>;
+    static constexpr size_type dimensionality = const_accessor::dimensionality;
 
 protected:
     using reference = typename const_accessor::reference;
 
 public:
     GKO_ATTRIBUTES ScaledReducedStorage3d()
-        : const_accessor(nullptr, {}, {}, nullptr)
+        : const_accessor(nullptr, {0, 0, 0}, nullptr)
     {}
 
     GKO_ATTRIBUTES ScaledReducedStorage3d(storage_type *storage,
+                                          dim<dimensionality> size,
                                           size_type stride0, size_type stride1,
                                           arithmetic_type *scale)
-        : const_accessor(storage, stride0, stride1, scale)
+        : const_accessor(storage, size, stride0, stride1, scale)
+    {}
+    GKO_ATTRIBUTES ScaledReducedStorage3d(storage_type *storage,
+                                          dim<dimensionality> size,
+                                          arithmetic_type *scale)
+        : const_accessor(storage, size, scale)
     {}
 
     GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const
     {
-        return {this->storage_, this->stride_[0], this->stride_[1],
+        return {this->storage_, this->size_, this->stride_[0], this->stride_[1],
                 this->scale_};
     }
 
     // Makes sure the operator() const function is visible
     using const_accessor::operator();
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE reference operator()(IndexType x, IndexType y,
-                                                   IndexType z)
+    GKO_ATTRIBUTES GKO_INLINE reference operator()(size_type x, size_type y,
+                                                   size_type z)
     {
-        return {this->storage_ + const_accessor::compute_index(x, y, z),
+        return {this->storage_ + this->compute_index(x, y, z),
                 this->read_scale(x, z)};
     }
 
-
-    using const_accessor::at;
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE reference at(IndexType x, IndexType y,
-                                           IndexType z)
-    {
-        return {this->storage_ + const_accessor::compute_index(x, y, z),
-                this->read_scale(x, z)};
+    template <typename OtherAccessor>
+    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other)
+    {
+        // TODO figure out how to do this properly in an const accessor...
+        //      Additionally, this will always be inefficient!
+        for (size_type i = 0; i < this->size_[0]; ++i) {
+            for (size_type j = 0; j < this->size_[1]; ++j) {
+                for (size_type k = 0; k < this->size_[2]; ++k) {
+                    (*this)(i, j, k) = other(i, j, k);
+                }
+            }
+        }
+        for (size_type i = 0; i < this->size_[0]; ++i) {
+            for (size_type k = 0; k < this->size_[2]; ++k) {
+                this->set_scale(i, k, other.read_scale(i, k));
+            }
+        }
     }
-
-    /**
-     * Writes the given value at the given index.
-     */
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE void write(IndexType x, IndexType y, IndexType z,
-                                         arithmetic_type value)
+    GKO_ATTRIBUTES constexpr range<ScaledReducedStorage3d> operator()(
+        const span &x_span, const span &y_span, const span &z_span) const
     {
-        storage_type *GKO_RESTRICT rest_storage = this->storage_;
-        rest_storage[const_accessor::compute_index(x, y, z)] =
-            static_cast<storage_type>(value / this->read_scale(x, z));
+        return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
+               GKO_ASSERT(z_span.is_valid()),
+               GKO_ASSERT(x_span <= span{this->size_[0]}),
+               GKO_ASSERT(y_span <= span{this->size_[1]}),
+               GKO_ASSERT(z_span <= span{this->size_[2]}),
+               range<ScaledReducedStorage3d>(
+                   this->storage_ + this->compute_index(x_span.begin,
+                                                        y_span.begin,
+                                                        z_span.begin),
+                   dim<dimensionality>{x_span.end - x_span.begin,
+                                       y_span.end - y_span.begin,
+                                       z_span.end - z_span.begin},
+                   this->stride_[0], this->stride_[1], this->scale_);
     }
 
     /**
      * Writes the given value at the given index for a scale.
      */
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE void set_scale(IndexType x, IndexType z,
+    GKO_ATTRIBUTES GKO_INLINE void set_scale(size_type x, size_type z,
                                              arithmetic_type val)
     {
+        GKO_ASSERT(x < this->size_[0] && z < this->size_[2]);
         arithmetic_type *GKO_RESTRICT rest_scale = this->scale_;
         rest_scale[x * this->stride_[1] + z] = val;
     }
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 1a23ade2b0c..9a97f59e285 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -93,6 +93,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif  // defined(_WIN32) || defined(__CYGWIN__)
 
 
+/**
+ * Encapsules all arguments in one macro, so a comma inside an argument (e.g.
+ * std::tuple<int, float>) still works as an argument and is not considered as
+ * multiple arguments.
+ */
+#define GKO_UNPACK(...) __VA_ARGS__
+
+
 namespace gko {
 
 

From aa102e3835f4191d9a42a3253498a4e97ee17044 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 15 Sep 2020 08:23:04 +0200
Subject: [PATCH 0196/1008] Half-way of integrating proper const support

---
 core/test/base/range_accessors.cpp           | 100 ++++++++--
 include/ginkgo/core/base/range_accessors.hpp | 189 +++++++++++++++----
 2 files changed, 245 insertions(+), 44 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 5f4a69deb9f..8ce44f5c4cc 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -36,6 +36,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
+#include <iostream>
+
+
 namespace {
 
 
@@ -145,11 +148,13 @@ class ReducedStorage3d : public ::testing::Test {
     using st_type = double;
 
     using accessor = gko::accessor::ReducedStorage3d<ar_type, st_type>;
+    using const_accessor = gko::accessor::ReducedStorage3d<ar_type, const st_type>;
 
     using reduced_storage = gko::range<accessor>;
+    using const_reduced_storage = gko::range<const_accessor>;
 
     // clang-format off
-    double data[8]{
+    st_type data[8]{
         1.0, 2.1,
         -1.2, 3.3,
         4.4, -2.5,
@@ -157,19 +162,42 @@ class ReducedStorage3d : public ::testing::Test {
     };
     //clang-format on
     reduced_storage r{data, gko::dim<3>{2u, 2u, 2u}};
+    const_reduced_storage cr{data, gko::dim<3>{2u, 2u, 2u}};
 };
 
 
+TEST_F(ReducedStorage3d, CanUseConst)
+{
+    EXPECT_EQ(cr(0, 0, 0), 1.0);
+    EXPECT_EQ(cr(0, 0, 1), 2.1);
+    EXPECT_EQ(cr(0, 1, 0), -1.2);
+    EXPECT_EQ(cr(0, 1, 1), 3.3);
+    EXPECT_EQ(cr(1, 0, 0), 4.4);
+    EXPECT_EQ(cr(1, 0, 1), -2.5);
+    EXPECT_EQ(cr(1, 1, 0), 5.6);
+    EXPECT_EQ(cr(1, 1, 1), 6.7);
+
+    r(0, 1, 0) = cr(0, 0, 0);
+    EXPECT_EQ(r(0, 1, 0), 1.0);
+
+    auto subr = cr(span{0, 2}, 0, 0);
+    //cr(0, 0, 0) = 2.0;
+
+    EXPECT_EQ(subr(0, 0, 0), 1.0);
+    EXPECT_EQ(subr(1, 0, 0), 4.4);
+}
+
+
 TEST_F(ReducedStorage3d, CanReadData)
 {
-    EXPECT_EQ(static_cast<ar_type>(r(0, 0, 0)), 1.0);
-    EXPECT_EQ(static_cast<ar_type>(r(0, 0, 1)), 2.1);
-    EXPECT_EQ(static_cast<ar_type>(r(0, 1, 0)), -1.2);
-    EXPECT_EQ(static_cast<ar_type>(r(0, 1, 1)), 3.3);
-    EXPECT_EQ(static_cast<ar_type>(r(1, 0, 0)), 4.4);
-    EXPECT_EQ(static_cast<ar_type>(r(1, 0, 1)), -2.5);
-    EXPECT_EQ(static_cast<ar_type>(r(1, 1, 0)), 5.6);
-    EXPECT_EQ(static_cast<ar_type>(r(1, 1, 1)), 6.7);
+    EXPECT_EQ(r(0, 0, 0), 1.0);
+    EXPECT_EQ(r(0, 0, 1), 2.1);
+    EXPECT_EQ(r(0, 1, 0), -1.2);
+    EXPECT_EQ(r(0, 1, 1), 3.3);
+    EXPECT_EQ(r(1, 0, 0), 4.4);
+    EXPECT_EQ(r(1, 0, 1), -2.5);
+    EXPECT_EQ(r(1, 1, 0), 5.6);
+    EXPECT_EQ(r(1, 1, 1), 6.7);
 }
 
 
@@ -185,8 +213,8 @@ TEST_F(ReducedStorage3d, CanCreateSubrange)
 {
     auto subr = r(span{1, 2}, span{0, 2}, span{0, 1});
 
-    EXPECT_EQ(static_cast<ar_type>(subr(0, 0, 0)), 4.4);
-    EXPECT_EQ(static_cast<ar_type>(subr(0, 1, 0)), 5.6);
+    EXPECT_EQ(subr(0, 0, 0), 4.4);
+    EXPECT_EQ(subr(0, 1, 0), 5.6);
 }
 
 
@@ -208,5 +236,55 @@ TEST_F(ReducedStorage3d, CanCreateColumnVector)
 }
 
 
+class ScaledReducedStorage3d : public ::testing::Test {
+protected:
+    using span = gko::span;
+    using ar_type = double;
+    using st_type = gko::int32;
+
+    using accessor = gko::accessor::ScaledReducedStorage3d<ar_type, st_type>;
+    using const_accessor = gko::accessor::ScaledReducedStorage3d<ar_type, const st_type>;
+
+    using reduced_storage = gko::range<accessor>;
+    using const_reduced_storage = gko::range<const_accessor>;
+
+    // clang-format off
+    st_type data[8]{
+        1, 2,
+        -3, 4,
+        55, 6,
+        -777, 8
+    };
+    ar_type scale[4]{
+        1., 1.,
+        1., 1.
+    };
+    //clang-format on
+    reduced_storage r{data, gko::dim<3>{2u, 2u, 2u}, scale};
+    const_reduced_storage cr{data, gko::dim<3>{2u, 2u, 2u}, scale};
+};
+
+
+TEST_F(ScaledReducedStorage3d, CanUseConst)
+{
+    EXPECT_EQ(cr(0, 0, 0), 1.);
+    EXPECT_EQ(cr(0, 0, 1), 2.);
+    EXPECT_EQ(cr(0, 1, 0), -3.);
+    EXPECT_EQ(cr(0, 1, 1), 4.);
+    EXPECT_EQ(cr(1, 0, 0), 55.);
+    EXPECT_EQ(cr(1, 0, 1), 6.);
+    EXPECT_EQ(cr(1, 1, 0), -777.);
+    EXPECT_EQ(cr(1, 1, 1), 8.);
+
+    auto subr = cr(span{0, 2}, 0, 0);
+
+    EXPECT_EQ(subr(0, 0, 0), 1.0);
+    EXPECT_EQ(subr(1, 0, 0), 55.);
+
+    //cr(0, 0, 0) = 2.0;
+    r->set_scale(0, 0, 2.);
+    EXPECT_EQ(r(0, 0, 0), 2.);
+}
+
 
 }  // namespace
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index b412b05e130..73e02c8ba9c 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -161,6 +161,9 @@ class row_major {
     /**
      * Copies data from another accessor
      *
+     * @warning Do not use this function since it is not optimized for a
+     * specific executor. It will always be performed sequentially.
+     *
      * @tparam OtherAccessor  type of the other accessor
      *
      * @param other  other accessor
@@ -311,6 +314,37 @@ class ReducedStorageReference
     storage_type *const GKO_RESTRICT ptr_;
 };
 
+// Specialization for const storage_type to prevent `operator=`
+template <typename ArithmeticType, typename StorageType>
+class ReducedStorageReference<ArithmeticType, const StorageType>
+    : public detail::enable_reference<
+          ReducedStorageReference<ArithmeticType, const StorageType>,
+          ArithmeticType> {
+public:
+    using arithmetic_type = ArithmeticType;
+    using storage_type = const StorageType;
+    // Allow move construction, so perfect forwarding is possible
+    ReducedStorageReference(ReducedStorageReference &&) = default;
+
+    ReducedStorageReference() = delete;
+    // Forbid copy construction and move assignment
+    ReducedStorageReference(const ReducedStorageReference &) = delete;
+    ReducedStorageReference &operator=(ReducedStorageReference &&) = delete;
+
+    GKO_ATTRIBUTES ReducedStorageReference(storage_type *const GKO_RESTRICT ptr)
+        : ptr_{ptr}
+    {}
+    GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
+    {
+        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return static_cast<arithmetic_type>(*r_ptr);
+    }
+
+private:
+    storage_type *const GKO_RESTRICT ptr_;
+};
+
+
 /**
  * Reference class for a different storage than arithmetic type. The conversion
  * between both formats is done with a simple static_cast followed by a
@@ -360,6 +394,41 @@ class ScaledReducedStorageReference
     const arithmetic_type scale_;
 };
 
+// Specialization for constant storage_type (no `operator=`)
+template <typename ArithmeticType, typename StorageType>
+class ScaledReducedStorageReference<ArithmeticType, const StorageType>
+    : public detail::enable_reference<
+          ScaledReducedStorageReference<ArithmeticType, const StorageType>,
+          ArithmeticType> {
+public:
+    using arithmetic_type = ArithmeticType;
+    using storage_type = const StorageType;
+
+    // Allow move construction, so perfect forwarding is possible
+    ScaledReducedStorageReference(ScaledReducedStorageReference &&) = default;
+
+    ScaledReducedStorageReference() = delete;
+    // Forbid copy construction and move assignment
+    ScaledReducedStorageReference(const ScaledReducedStorageReference &) =
+        delete;
+    ScaledReducedStorageReference &operator=(ScaledReducedStorageReference &&) =
+        delete;
+
+    GKO_ATTRIBUTES ScaledReducedStorageReference(
+        storage_type *const GKO_RESTRICT ptr, arithmetic_type scale)
+        : ptr_{ptr}, scale_{scale}
+    {}
+    GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
+    {
+        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return static_cast<arithmetic_type>(*r_ptr) * scale_;
+    }
+
+private:
+    storage_type *const GKO_RESTRICT ptr_;
+    const arithmetic_type scale_;
+};
+
 
 }  // namespace reference
 }  // namespace detail
@@ -381,13 +450,14 @@ class ConstReducedStorage3d {
     using arithmetic_type = ArithmeticType;
     using storage_type = StorageType;
     using const_accessor = ConstReducedStorage3d;
-    static_assert(!std::is_const<storage_type>::value,
-                  "StorageType must not be const!");
+    //    static_assert(!std::is_const<storage_type>::value,
+    //                  "StorageType must not be const!");
     static constexpr size_type dimensionality{3};
 
 protected:
     using reference =
         detail::reference::ReducedStorageReference<ArithmeticType, StorageType>;
+    // using j;
 
 public:
     /**
@@ -503,6 +573,7 @@ class ReducedStorage3d
     using storage_type = StorageType;
     using const_accessor = ConstReducedStorage3d<arithmetic_type, storage_type>;
     static constexpr size_type dimensionality = const_accessor::dimensionality;
+    static constexpr bool is_const = std::is_const<storage_type>::value;
 
 protected:
     using reference = typename const_accessor::reference;
@@ -527,6 +598,10 @@ class ReducedStorage3d
                 this->stride_[1]};
     }
 
+    /**
+     * @warning Do not use this function since it is not optimized for a
+     *          specific executor. It will always be performed sequentially.
+     */
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other)
     {
@@ -564,14 +639,50 @@ class ReducedStorage3d
 
     // Makes sure the operator() const function is visible
     using const_accessor::operator();
-    GKO_ATTRIBUTES GKO_INLINE reference operator()(size_type x, size_type y,
-                                                   size_type z)
+    GKO_ATTRIBUTES GKO_INLINE
+        std::conditional_t<is_const, arithmetic_type, reference>
+        operator()(size_type x, size_type y, size_type z)
+    {
+        return reference{this->storage_ +
+                         const_accessor::compute_index(x, y, z)};
+    }
+};
+
+
+namespace detail {
+
+
+template <typename Accessor, typename ScaleType, bool is_const>
+struct enable_scale_write {
+    using scale_type = ScaleType;
+
+    /**
+     * Reads the scale value at the given indices.
+     */
+    GKO_ATTRIBUTES GKO_INLINE constexpr scale_type set_scale(
+        size_type x, size_type z, scale_type value) const
     {
-        return {this->storage_ + const_accessor::compute_index(x, y, z)};
+        scale_type *GKO_RESTRICT rest_scale = self()->scale_;
+        return rest_scale[self()->compute_scale_index(x, z)] = value;
     }
+
+private:
+    GKO_ATTRIBUTES GKO_INLINE constexpr const Accessor *self() const
+    {
+        return static_cast<const Accessor *>(this);
+    }
+};
+
+// In case of a const type, do not provide a write function
+template <typename Accessor, typename ScaleType>
+struct enable_scale_write<Accessor, ScaleType, true> {
+    using scale_type = ScaleType;
 };
 
 
+}  // namespace detail
+
+
 /**
  * @internal
  *
@@ -585,13 +696,20 @@ class ReducedStorage3d
  * The accessor uses row-major access.
  */
 template <typename ArithmeticType, typename StorageType>
-class ConstScaledReducedStorage3d {
+class ConstScaledReducedStorage3d
+    : public detail::enable_scale_write<
+          ConstScaledReducedStorage3d<ArithmeticType, StorageType>,
+          ArithmeticType, std::is_const<StorageType>::value> {
 public:
     using arithmetic_type = ArithmeticType;
     using storage_type = StorageType;
+    using scale_type = arithmetic_type;
+    // Allow access to both `scale_` and `compute_scale_index()`
+    friend detail::enable_scale_write<ConstScaledReducedStorage3d, scale_type,
+                                      std::is_const<storage_type>::value>;
     using const_accessor = ConstScaledReducedStorage3d;
-    static_assert(!std::is_const<storage_type>::value,
-                  "StorageType must not be const!");
+    //    static_assert(!std::is_const<storage_type>::value,
+    //                  "StorageType must not be const!");
     static constexpr size_type dimensionality{3};
 
 protected:
@@ -628,17 +746,19 @@ class ConstScaledReducedStorage3d {
           scale_{const_cast<arithmetic_type *>(scale)}
     {}
 
-    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const { return *this; }
+    GKO_ATTRIBUTES GKO_INLINE constexpr const_accessor to_const() const
+    {
+        return *this;
+    }
 
     /**
      * Reads the scale value at the given indices.
      */
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type read_scale(size_type x,
-                                                         size_type z) const
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type read_scale(
+        size_type x, size_type z) const
     {
         const arithmetic_type *GKO_RESTRICT rest_scale = scale_;
-        return GKO_ASSERT(x < size_[0]), GKO_ASSERT(z < size_[2]),
-               rest_scale[x * stride_[1] + z];
+        return rest_scale[compute_scale_index(x, z)];
     }
 
     GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator()(size_type x,
@@ -654,6 +774,11 @@ class ConstScaledReducedStorage3d {
     {
         return dimension < dimensionality ? size_[dimension] : 1;
     }
+
+    /**
+     * @warning Do not use this function since it is not optimized for a
+     * specific executor. It will always be performed sequentially.
+     */
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other)
     {
@@ -707,7 +832,7 @@ class ConstScaledReducedStorage3d {
         return storage_;
     }
 
-    GKO_ATTRIBUTES GKO_INLINE const arithmetic_type *get_scale() const
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type *get_scale() const
     {
         return this->scale_;
     }
@@ -725,6 +850,13 @@ class ConstScaledReducedStorage3d {
                GKO_ASSERT(z < size_[2]), x * stride_[0] + y * stride_[1] + z;
     }
 
+    GKO_ATTRIBUTES constexpr GKO_INLINE size_type
+    compute_scale_index(size_type x, size_type z) const
+    {
+        return GKO_ASSERT(x < size_[0]), GKO_ASSERT(z < size_[2]),
+               x * stride_[1] + z;
+    }
+
     storage_type *storage_;
     dim<dimensionality> size_;
     size_type stride_[2];  // std::array leads to conflicts on devices
@@ -741,6 +873,7 @@ class ScaledReducedStorage3d
     using const_accessor =
         ConstScaledReducedStorage3d<arithmetic_type, storage_type>;
     static constexpr size_type dimensionality = const_accessor::dimensionality;
+    static constexpr bool is_const = std::is_const<storage_type>::value;
 
 protected:
     using reference = typename const_accessor::reference;
@@ -770,13 +903,18 @@ class ScaledReducedStorage3d
 
     // Makes sure the operator() const function is visible
     using const_accessor::operator();
-    GKO_ATTRIBUTES GKO_INLINE reference operator()(size_type x, size_type y,
-                                                   size_type z)
+    GKO_ATTRIBUTES GKO_INLINE
+        std::conditional_t<is_const, arithmetic_type, reference>
+        operator()(size_type x, size_type y, size_type z)
     {
-        return {this->storage_ + this->compute_index(x, y, z),
-                this->read_scale(x, z)};
+        return reference{this->storage_ + this->compute_index(x, y, z),
+                         this->read_scale(x, z)};
     }
 
+    /**
+     * @warning Do not use this function since it is not optimized for a
+     * specific executor. It will always be performed sequentially.
+     */
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other)
     {
@@ -813,17 +951,6 @@ class ScaledReducedStorage3d
                    this->stride_[0], this->stride_[1], this->scale_);
     }
 
-    /**
-     * Writes the given value at the given index for a scale.
-     */
-    GKO_ATTRIBUTES GKO_INLINE void set_scale(size_type x, size_type z,
-                                             arithmetic_type val)
-    {
-        GKO_ASSERT(x < this->size_[0] && z < this->size_[2]);
-        arithmetic_type *GKO_RESTRICT rest_scale = this->scale_;
-        rest_scale[x * this->stride_[1] + z] = val;
-    }
-
     using const_accessor::get_storage;
     GKO_ATTRIBUTES GKO_INLINE storage_type *get_storage()
     {
@@ -831,10 +958,6 @@ class ScaledReducedStorage3d
     }
 
     using const_accessor::get_scale;
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type *get_scale()
-    {
-        return this->scale_;
-    }
 };
 
 

From 2b51ca18a4f7c8493ed7338ca02cdfb38cec45f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 15 Sep 2020 09:38:03 +0200
Subject: [PATCH 0197/1008] Finish proper const-type support

---
 core/test/base/range_accessors.cpp           |  58 +--
 include/ginkgo/core/base/range_accessors.hpp | 376 ++++++-------------
 2 files changed, 139 insertions(+), 295 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 8ce44f5c4cc..b91c2ed67a9 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -54,7 +54,7 @@ class RowMajorAccessor : public ::testing::Test {
         3, 4, -2,
         5, 6, -3
     };
-    //clang-format on
+    // clang-format on
     row_major_int_range r{data, 3u, 2u, 3u};
 };
 
@@ -148,7 +148,8 @@ class ReducedStorage3d : public ::testing::Test {
     using st_type = double;
 
     using accessor = gko::accessor::ReducedStorage3d<ar_type, st_type>;
-    using const_accessor = gko::accessor::ReducedStorage3d<ar_type, const st_type>;
+    using const_accessor =
+        gko::accessor::ReducedStorage3d<ar_type, const st_type>;
 
     using reduced_storage = gko::range<accessor>;
     using const_reduced_storage = gko::range<const_accessor>;
@@ -160,14 +161,23 @@ class ReducedStorage3d : public ::testing::Test {
         4.4, -2.5,
         5.6, 6.7
     };
-    //clang-format on
+    // clang-format on
     reduced_storage r{data, gko::dim<3>{2u, 2u, 2u}};
     const_reduced_storage cr{data, gko::dim<3>{2u, 2u, 2u}};
 };
 
 
-TEST_F(ReducedStorage3d, CanUseConst)
+TEST_F(ReducedStorage3d, CanReadData)
 {
+    EXPECT_EQ(r(0, 0, 0), 1.0);
+    EXPECT_EQ(r(0, 0, 1), 2.1);
+    EXPECT_EQ(r(0, 1, 0), -1.2);
+    EXPECT_EQ(r(0, 1, 1), 3.3);
+    EXPECT_EQ(r(1, 0, 0), 4.4);
+    EXPECT_EQ(r(1, 0, 1), -2.5);
+    EXPECT_EQ(r(1, 1, 0), 5.6);
+    EXPECT_EQ(r(1, 1, 1), 6.7);
+    // Const should read the exact same values!
     EXPECT_EQ(cr(0, 0, 0), 1.0);
     EXPECT_EQ(cr(0, 0, 1), 2.1);
     EXPECT_EQ(cr(0, 1, 0), -1.2);
@@ -176,36 +186,33 @@ TEST_F(ReducedStorage3d, CanUseConst)
     EXPECT_EQ(cr(1, 0, 1), -2.5);
     EXPECT_EQ(cr(1, 1, 0), 5.6);
     EXPECT_EQ(cr(1, 1, 1), 6.7);
-
-    r(0, 1, 0) = cr(0, 0, 0);
-    EXPECT_EQ(r(0, 1, 0), 1.0);
-
-    auto subr = cr(span{0, 2}, 0, 0);
-    //cr(0, 0, 0) = 2.0;
-
-    EXPECT_EQ(subr(0, 0, 0), 1.0);
-    EXPECT_EQ(subr(1, 0, 0), 4.4);
 }
 
 
-TEST_F(ReducedStorage3d, CanReadData)
+TEST_F(ReducedStorage3d, ToConstWorking)
 {
-    EXPECT_EQ(r(0, 0, 0), 1.0);
-    EXPECT_EQ(r(0, 0, 1), 2.1);
-    EXPECT_EQ(r(0, 1, 0), -1.2);
-    EXPECT_EQ(r(0, 1, 1), 3.3);
-    EXPECT_EQ(r(1, 0, 0), 4.4);
-    EXPECT_EQ(r(1, 0, 1), -2.5);
-    EXPECT_EQ(r(1, 1, 0), 5.6);
-    EXPECT_EQ(r(1, 1, 1), 6.7);
+    auto cr2 = r->to_const();
+
+    // static_assert(std::is_same<decltype(cr2), const_reduced_storage>::value,
+    // "Types must be equal!");
+    EXPECT_EQ(cr2(0, 0, 0), 1.0);
+    EXPECT_EQ(cr2(0, 0, 1), 2.1);
+    EXPECT_EQ(cr2(0, 1, 0), -1.2);
+    EXPECT_EQ(cr2(0, 1, 1), 3.3);
+    EXPECT_EQ(cr2(1, 0, 0), 4.4);
+    EXPECT_EQ(cr2(1, 0, 1), -2.5);
+    EXPECT_EQ(cr2(1, 1, 0), 5.6);
+    EXPECT_EQ(cr2(1, 1, 1), 6.7);
 }
 
 
 TEST_F(ReducedStorage3d, CanWriteData)
 {
     r(0, 0, 0) = 2.0;
+    r(0, 1, 0) = 100.0;
 
     EXPECT_EQ(r(0, 0, 0), 2.0);
+    EXPECT_EQ(r(0, 1, 0), 100.0);
 }
 
 
@@ -243,7 +250,8 @@ class ScaledReducedStorage3d : public ::testing::Test {
     using st_type = gko::int32;
 
     using accessor = gko::accessor::ScaledReducedStorage3d<ar_type, st_type>;
-    using const_accessor = gko::accessor::ScaledReducedStorage3d<ar_type, const st_type>;
+    using const_accessor =
+        gko::accessor::ScaledReducedStorage3d<ar_type, const st_type>;
 
     using reduced_storage = gko::range<accessor>;
     using const_reduced_storage = gko::range<const_accessor>;
@@ -259,7 +267,7 @@ class ScaledReducedStorage3d : public ::testing::Test {
         1., 1.,
         1., 1.
     };
-    //clang-format on
+    // clang-format on
     reduced_storage r{data, gko::dim<3>{2u, 2u, 2u}, scale};
     const_reduced_storage cr{data, gko::dim<3>{2u, 2u, 2u}, scale};
 };
@@ -281,7 +289,7 @@ TEST_F(ScaledReducedStorage3d, CanUseConst)
     EXPECT_EQ(subr(0, 0, 0), 1.0);
     EXPECT_EQ(subr(1, 0, 0), 55.);
 
-    //cr(0, 0, 0) = 2.0;
+    // cr(0, 0, 0) = 2.0;
     r->set_scale(0, 0, 2.);
     EXPECT_EQ(r(0, 0, 0), 2.);
 }
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 73e02c8ba9c..fc604544cf4 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -445,19 +445,19 @@ class ScaledReducedStorageReference<ArithmeticType, const StorageType>
  * TODO Rename, so it is in lower_case only (since no virtual functions!)
  */
 template <typename ArithmeticType, typename StorageType>
-class ConstReducedStorage3d {
+class ReducedStorage3d {
 public:
     using arithmetic_type = ArithmeticType;
     using storage_type = StorageType;
-    using const_accessor = ConstReducedStorage3d;
-    //    static_assert(!std::is_const<storage_type>::value,
-    //                  "StorageType must not be const!");
+    using const_accessor =
+        ReducedStorage3d<arithmetic_type, const storage_type>;
     static constexpr size_type dimensionality{3};
+    static constexpr bool is_const{std::is_const<StorageType>::value};
 
 protected:
     using reference =
-        detail::reference::ReducedStorageReference<ArithmeticType, StorageType>;
-    // using j;
+        detail::reference::ReducedStorageReference<arithmetic_type,
+                                                   storage_type>;
 
 public:
     /**
@@ -469,28 +469,24 @@ class ConstReducedStorage3d {
      * works. Also, storage_ is never accessed in a non-const fashion in this
      * class, so it is not invalid or UB code.
      */
-    GKO_ATTRIBUTES ConstReducedStorage3d(const storage_type *storage,
-                                         gko::dim<dimensionality> size,
-                                         size_type stride0, size_type stride1)
-        : storage_{const_cast<storage_type *>(storage)},
-          size_{size},
-          stride_{stride0, stride1}
+    GKO_ATTRIBUTES constexpr ReducedStorage3d(storage_type *storage,
+                                              gko::dim<dimensionality> size,
+                                              size_type stride0,
+                                              size_type stride1)
+        : storage_{storage}, size_{size}, stride_{stride0, stride1}
     {}
 
-    GKO_ATTRIBUTES ConstReducedStorage3d(const storage_type *storage,
-                                         dim<dimensionality> size)
-        : storage_{const_cast<storage_type *>(storage)},
-          size_{size},
-          stride_{size_[1] * size[2], size[2]}
+    GKO_ATTRIBUTES constexpr ReducedStorage3d(storage_type *storage,
+                                              dim<dimensionality> size)
+        : ReducedStorage3d{storage, size, size[1] * size[2], size[2]}
+    {}
+    GKO_ATTRIBUTES constexpr ReducedStorage3d()
+        : ReducedStorage3d{nullptr, {0, 0, 0}}
     {}
 
-    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const { return *this; }
-
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator()(size_type x,
-                                                         size_type y,
-                                                         size_type z) const
+    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const
     {
-        return reference{storage_ + compute_index(x, y, z)};
+        return {storage_, size_, stride_[0], stride_[1]};
     }
 
     // Functions required by the `range` interface:
@@ -499,6 +495,10 @@ class ConstReducedStorage3d {
     {
         return dimension < dimensionality ? size_[dimension] : 1;
     }
+    /**
+     * @warning Do not use this function since it is not optimized for a
+     *          specific executor. It will always be performed sequentially.
+     */
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other)
     {
@@ -507,12 +507,19 @@ class ConstReducedStorage3d {
         for (size_type i = 0; i < size_[0]; ++i) {
             for (size_type j = 0; j < size_[1]; ++j) {
                 for (size_type k = 0; k < size_[2]; ++k) {
-                    //(*this)(i, j, k) = other(i, j, k);
+                    (*this)(i, j, k) = other(i, j, k);
                 }
             }
         }
     }
-    GKO_ATTRIBUTES constexpr range<ConstReducedStorage3d> operator()(
+    GKO_ATTRIBUTES GKO_INLINE
+        std::conditional_t<is_const, arithmetic_type, reference>
+        operator()(size_type x, size_type y, size_type z) const
+    {
+        return reference{this->storage_ + compute_index(x, y, z)};
+    }
+
+    GKO_ATTRIBUTES constexpr range<ReducedStorage3d> operator()(
         const span &x_span, const span &y_span, const span &z_span) const
     {
         return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
@@ -520,7 +527,7 @@ class ConstReducedStorage3d {
                GKO_ASSERT(x_span <= span{size_[0]}),
                GKO_ASSERT(y_span <= span{size_[1]}),
                GKO_ASSERT(z_span <= span{size_[2]}),
-               range<ConstReducedStorage3d>(
+               range<ReducedStorage3d>(
                    storage_ +
                        compute_index(x_span.begin, y_span.begin, z_span.begin),
                    dim<dimensionality>{x_span.end - x_span.begin,
@@ -542,7 +549,7 @@ class ConstReducedStorage3d {
         return stride_[1];
     }
 
-    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_storage() const
+    GKO_ATTRIBUTES GKO_INLINE storage_type *get_storage() const
     {
         return storage_;
     }
@@ -565,95 +572,24 @@ class ConstReducedStorage3d {
     size_type stride_[2];  // std::array leads to conflicts on devices
 };
 
-template <typename ArithmeticType, typename StorageType>
-class ReducedStorage3d
-    : public ConstReducedStorage3d<ArithmeticType, StorageType> {
-public:
-    using arithmetic_type = ArithmeticType;
-    using storage_type = StorageType;
-    using const_accessor = ConstReducedStorage3d<arithmetic_type, storage_type>;
-    static constexpr size_type dimensionality = const_accessor::dimensionality;
-    static constexpr bool is_const = std::is_const<storage_type>::value;
-
-protected:
-    using reference = typename const_accessor::reference;
-
-public:
-    GKO_ATTRIBUTES ReducedStorage3d() : const_accessor(nullptr, {0, 0, 0}) {}
-
-    GKO_ATTRIBUTES ReducedStorage3d(storage_type *storage,
-                                    dim<dimensionality> size, size_type stride0,
-                                    size_type stride1)
-        : const_accessor(storage, size, stride0, stride1)
-    {}
-
-    GKO_ATTRIBUTES ReducedStorage3d(storage_type *storage,
-                                    dim<dimensionality> size)
-        : const_accessor(storage, size)
-    {}
-
-    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const
-    {
-        return {this->storage_, this->size_, this->stride_[0],
-                this->stride_[1]};
-    }
-
-    /**
-     * @warning Do not use this function since it is not optimized for a
-     *          specific executor. It will always be performed sequentially.
-     */
-    template <typename OtherAccessor>
-    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other)
-    {
-        for (size_type i = 0; i < this->size_[0]; ++i) {
-            for (size_type j = 0; j < this->size_[1]; ++j) {
-                for (size_type k = 0; k < this->size_[2]; ++k) {
-                    (*this)(i, j, k) = other(i, j, k);
-                }
-            }
-        }
-    }
-    GKO_ATTRIBUTES constexpr range<ReducedStorage3d> operator()(
-        const span &x_span, const span &y_span, const span &z_span) const
-    {
-        return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
-               GKO_ASSERT(z_span.is_valid()),
-               GKO_ASSERT(x_span <= span{this->size_[0]}),
-               GKO_ASSERT(y_span <= span{this->size_[1]}),
-               GKO_ASSERT(z_span <= span{this->size_[2]}),
-               range<ReducedStorage3d>(
-                   this->storage_ + this->compute_index(x_span.begin,
-                                                        y_span.begin,
-                                                        z_span.begin),
-                   dim<dimensionality>{x_span.end - x_span.begin,
-                                       y_span.end - y_span.begin,
-                                       z_span.end - z_span.begin},
-                   this->stride_[0], this->stride_[1]);
-    }
-
-    using const_accessor::get_storage;
-    GKO_ATTRIBUTES GKO_INLINE storage_type *get_storage()
-    {
-        return this->storage_;
-    }
-
-    // Makes sure the operator() const function is visible
-    using const_accessor::operator();
-    GKO_ATTRIBUTES GKO_INLINE
-        std::conditional_t<is_const, arithmetic_type, reference>
-        operator()(size_type x, size_type y, size_type z)
-    {
-        return reference{this->storage_ +
-                         const_accessor::compute_index(x, y, z)};
-    }
-};
-
 
 namespace detail {
 
 
-template <typename Accessor, typename ScaleType, bool is_const>
+// In case of a const type, do not provide a write function
+template <typename Accessor, typename ScaleType,
+          bool = std::is_const<ScaleType>::value>
 struct enable_scale_write {
+    static_assert(std::is_const<ScaleType>::value,
+                  "This class must have a constant ScaleType!");
+    using scale_type = ScaleType;
+};
+
+// In case of a non-const type, enable the write function
+template <typename Accessor, typename ScaleType>
+struct enable_scale_write<Accessor, ScaleType, false> {
+    static_assert(!std::is_const<ScaleType>::value,
+                  "This class must NOT have a constant ScaleType!");
     using scale_type = ScaleType;
 
     /**
@@ -673,12 +609,6 @@ struct enable_scale_write {
     }
 };
 
-// In case of a const type, do not provide a write function
-template <typename Accessor, typename ScaleType>
-struct enable_scale_write<Accessor, ScaleType, true> {
-    using scale_type = ScaleType;
-};
-
 
 }  // namespace detail
 
@@ -686,7 +616,7 @@ struct enable_scale_write<Accessor, ScaleType, true> {
 /**
  * @internal
  *
- * The ScaledReducedStorage3d class hides the underlying storage_ format and
+ * The ScaledReducedStorage3d class hides the underlying storage format and
  * provides a simple interface for accessing a one dimensional storage.
  * Additionally, this accessor posesses a scale array, which is used for each
  * read and write operation to do a proper conversion.
@@ -696,21 +626,21 @@ struct enable_scale_write<Accessor, ScaleType, true> {
  * The accessor uses row-major access.
  */
 template <typename ArithmeticType, typename StorageType>
-class ConstScaledReducedStorage3d
+class ScaledReducedStorage3d
     : public detail::enable_scale_write<
-          ConstScaledReducedStorage3d<ArithmeticType, StorageType>,
-          ArithmeticType, std::is_const<StorageType>::value> {
+          ScaledReducedStorage3d<ArithmeticType, StorageType>, ArithmeticType> {
 public:
     using arithmetic_type = ArithmeticType;
     using storage_type = StorageType;
-    using scale_type = arithmetic_type;
-    // Allow access to both `scale_` and `compute_scale_index()`
-    friend detail::enable_scale_write<ConstScaledReducedStorage3d, scale_type,
-                                      std::is_const<storage_type>::value>;
-    using const_accessor = ConstScaledReducedStorage3d;
-    //    static_assert(!std::is_const<storage_type>::value,
-    //                  "StorageType must not be const!");
     static constexpr size_type dimensionality{3};
+    static constexpr bool is_const{std::is_const<StorageType>::value};
+    using scale_type =
+        std::conditional_t<is_const, const arithmetic_type, arithmetic_type>;
+
+    // Allow access to both `scale_` and `compute_scale_index()`
+    friend detail::enable_scale_write<ScaledReducedStorage3d, scale_type>;
+    using const_accessor =
+        ScaledReducedStorage3d<arithmetic_type, const storage_type>;
 
 protected:
     using reference =
@@ -721,53 +651,42 @@ class ConstScaledReducedStorage3d
     /**
      * Creates the accessor with an already allocated storage space with a
      * stride.
-     *
-     * @internal
-     * const_cast needed, so ScaledReducedStorage3d(ac.get_storage(), ...)
-     * works. Also, storage_ is never accessed in a non-const fashion (from
-     * ConstScaledReducedStorage3d), so it is not invalid or UB code.
      */
-    GKO_ATTRIBUTES ConstScaledReducedStorage3d(const storage_type *storage,
-                                               dim<dimensionality> size,
-                                               size_type stride0,
-                                               size_type stride1,
-                                               const arithmetic_type *scale)
-        : storage_{const_cast<storage_type *>(storage)},
+    GKO_ATTRIBUTES constexpr ScaledReducedStorage3d(storage_type *storage,
+                                                    dim<dimensionality> size,
+                                                    size_type stride0,
+                                                    size_type stride1,
+                                                    scale_type *scale)
+        : storage_{storage},
           size_{size},
           stride_{stride0, stride1},
-          scale_{const_cast<arithmetic_type *>(scale)}
+          scale_{scale}
     {}
-    GKO_ATTRIBUTES ConstScaledReducedStorage3d(const storage_type *storage,
-                                               dim<dimensionality> size,
-                                               const arithmetic_type *scale)
-        : storage_{const_cast<storage_type *>(storage)},
-          size_{size},
-          stride_{size_[1] * size_[2], size_[2]},
-          scale_{const_cast<arithmetic_type *>(scale)}
+    GKO_ATTRIBUTES constexpr ScaledReducedStorage3d(storage_type *storage,
+                                                    dim<dimensionality> size,
+                                                    scale_type *scale)
+        : ScaledReducedStorage3d{storage, size, size[1] * size[2], size[2],
+                                 scale}
+    {}
+    GKO_ATTRIBUTES constexpr ScaledReducedStorage3d()
+        : ScaledReducedStorage3d{nullptr, {0, 0, 0}, nullptr}
     {}
 
     GKO_ATTRIBUTES GKO_INLINE constexpr const_accessor to_const() const
     {
-        return *this;
+        return {storage_, size_, stride_[0], stride_[1], scale_};
     }
 
     /**
      * Reads the scale value at the given indices.
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type read_scale(
-        size_type x, size_type z) const
+    GKO_ATTRIBUTES GKO_INLINE constexpr scale_type read_scale(size_type x,
+                                                              size_type z) const
     {
         const arithmetic_type *GKO_RESTRICT rest_scale = scale_;
         return rest_scale[compute_scale_index(x, z)];
     }
 
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator()(size_type x,
-                                                         size_type y,
-                                                         size_type z) const
-    {
-        return reference{storage_ + compute_index(x, y, z), read_scale(x, z)};
-    }
-
     // Functions required by the `range` interface:
     GKO_ATTRIBUTES GKO_INLINE dim<dimensionality> length(
         size_type dimension) const
@@ -784,29 +703,43 @@ class ConstScaledReducedStorage3d
     {
         // TODO figure out how to do this properly in an const accessor...
         //      Additionally, this will always be inefficient!
-        for (size_type i = 0; i < size_[0]; ++i) {
-            for (size_type j = 0; j < size_[1]; ++j) {
-                for (size_type k = 0; k < size_[2]; ++k) {
-                    //(*this)(i, j, k) = other(i, j, k);
+        for (size_type i = 0; i < this->size_[0]; ++i) {
+            for (size_type j = 0; j < this->size_[1]; ++j) {
+                for (size_type k = 0; k < this->size_[2]; ++k) {
+                    (*this)(i, j, k) = other(i, j, k);
                 }
             }
         }
+        for (size_type i = 0; i < this->size_[0]; ++i) {
+            for (size_type k = 0; k < this->size_[2]; ++k) {
+                this->set_scale(i, k, other.read_scale(i, k));
+            }
+        }
+    }
+    GKO_ATTRIBUTES GKO_INLINE
+        std::conditional_t<is_const, arithmetic_type, reference>
+        operator()(size_type x, size_type y, size_type z) const
+    {
+        return reference{this->storage_ + this->compute_index(x, y, z),
+                         this->read_scale(x, z)};
     }
-    GKO_ATTRIBUTES constexpr range<ConstScaledReducedStorage3d> operator()(
+
+    GKO_ATTRIBUTES constexpr range<ScaledReducedStorage3d> operator()(
         const span &x_span, const span &y_span, const span &z_span) const
     {
         return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
                GKO_ASSERT(z_span.is_valid()),
-               GKO_ASSERT(x_span <= span{size_[0]}),
-               GKO_ASSERT(y_span <= span{size_[1]}),
-               GKO_ASSERT(z_span <= span{size_[2]}),
-               range<ConstScaledReducedStorage3d>(
-                   storage_ +
-                       compute_index(x_span.begin, y_span.begin, z_span.begin),
+               GKO_ASSERT(x_span <= span{this->size_[0]}),
+               GKO_ASSERT(y_span <= span{this->size_[1]}),
+               GKO_ASSERT(z_span <= span{this->size_[2]}),
+               range<ScaledReducedStorage3d>(
+                   this->storage_ + this->compute_index(x_span.begin,
+                                                        y_span.begin,
+                                                        z_span.begin),
                    dim<dimensionality>{x_span.end - x_span.begin,
                                        y_span.end - y_span.begin,
                                        z_span.end - z_span.begin},
-                   stride_[0], stride_[1], scale_);
+                   this->stride_[0], this->stride_[1], this->scale_);
     }
 
     GKO_ATTRIBUTES GKO_INLINE dim<dimensionality> get_size() const
@@ -822,7 +755,7 @@ class ConstScaledReducedStorage3d
         return stride_[1];
     }
 
-    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_storage() const
+    GKO_ATTRIBUTES GKO_INLINE storage_type *get_storage() const
     {
         return storage_;
     }
@@ -832,12 +765,12 @@ class ConstScaledReducedStorage3d
         return storage_;
     }
 
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type *get_scale() const
+    GKO_ATTRIBUTES GKO_INLINE scale_type *get_scale() const
     {
         return this->scale_;
     }
 
-    GKO_ATTRIBUTES GKO_INLINE const arithmetic_type *get_const_scale() const
+    GKO_ATTRIBUTES GKO_INLINE const scale_type *get_const_scale() const
     {
         return this->scale_;
     }
@@ -860,104 +793,7 @@ class ConstScaledReducedStorage3d
     storage_type *storage_;
     dim<dimensionality> size_;
     size_type stride_[2];  // std::array leads to conflicts on devices
-    arithmetic_type *scale_;
-};
-
-
-template <typename ArithmeticType, typename StorageType>
-class ScaledReducedStorage3d
-    : public ConstScaledReducedStorage3d<ArithmeticType, StorageType> {
-public:
-    using arithmetic_type = ArithmeticType;
-    using storage_type = StorageType;
-    using const_accessor =
-        ConstScaledReducedStorage3d<arithmetic_type, storage_type>;
-    static constexpr size_type dimensionality = const_accessor::dimensionality;
-    static constexpr bool is_const = std::is_const<storage_type>::value;
-
-protected:
-    using reference = typename const_accessor::reference;
-
-public:
-    GKO_ATTRIBUTES ScaledReducedStorage3d()
-        : const_accessor(nullptr, {0, 0, 0}, nullptr)
-    {}
-
-    GKO_ATTRIBUTES ScaledReducedStorage3d(storage_type *storage,
-                                          dim<dimensionality> size,
-                                          size_type stride0, size_type stride1,
-                                          arithmetic_type *scale)
-        : const_accessor(storage, size, stride0, stride1, scale)
-    {}
-    GKO_ATTRIBUTES ScaledReducedStorage3d(storage_type *storage,
-                                          dim<dimensionality> size,
-                                          arithmetic_type *scale)
-        : const_accessor(storage, size, scale)
-    {}
-
-    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const
-    {
-        return {this->storage_, this->size_, this->stride_[0], this->stride_[1],
-                this->scale_};
-    }
-
-    // Makes sure the operator() const function is visible
-    using const_accessor::operator();
-    GKO_ATTRIBUTES GKO_INLINE
-        std::conditional_t<is_const, arithmetic_type, reference>
-        operator()(size_type x, size_type y, size_type z)
-    {
-        return reference{this->storage_ + this->compute_index(x, y, z),
-                         this->read_scale(x, z)};
-    }
-
-    /**
-     * @warning Do not use this function since it is not optimized for a
-     * specific executor. It will always be performed sequentially.
-     */
-    template <typename OtherAccessor>
-    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other)
-    {
-        // TODO figure out how to do this properly in an const accessor...
-        //      Additionally, this will always be inefficient!
-        for (size_type i = 0; i < this->size_[0]; ++i) {
-            for (size_type j = 0; j < this->size_[1]; ++j) {
-                for (size_type k = 0; k < this->size_[2]; ++k) {
-                    (*this)(i, j, k) = other(i, j, k);
-                }
-            }
-        }
-        for (size_type i = 0; i < this->size_[0]; ++i) {
-            for (size_type k = 0; k < this->size_[2]; ++k) {
-                this->set_scale(i, k, other.read_scale(i, k));
-            }
-        }
-    }
-    GKO_ATTRIBUTES constexpr range<ScaledReducedStorage3d> operator()(
-        const span &x_span, const span &y_span, const span &z_span) const
-    {
-        return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
-               GKO_ASSERT(z_span.is_valid()),
-               GKO_ASSERT(x_span <= span{this->size_[0]}),
-               GKO_ASSERT(y_span <= span{this->size_[1]}),
-               GKO_ASSERT(z_span <= span{this->size_[2]}),
-               range<ScaledReducedStorage3d>(
-                   this->storage_ + this->compute_index(x_span.begin,
-                                                        y_span.begin,
-                                                        z_span.begin),
-                   dim<dimensionality>{x_span.end - x_span.begin,
-                                       y_span.end - y_span.begin,
-                                       z_span.end - z_span.begin},
-                   this->stride_[0], this->stride_[1], this->scale_);
-    }
-
-    using const_accessor::get_storage;
-    GKO_ATTRIBUTES GKO_INLINE storage_type *get_storage()
-    {
-        return this->storage_;
-    }
-
-    using const_accessor::get_scale;
+    scale_type *scale_;
 };
 
 

From 5aeac5b27ccbb3b2854005ad3bdb509ddad75c0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 15 Sep 2020 12:38:12 +0200
Subject: [PATCH 0198/1008] Add constexpr everywhere in accessor

---
 include/ginkgo/core/base/range.hpp           | 13 +--
 include/ginkgo/core/base/range_accessors.hpp | 85 ++++++++++++--------
 2 files changed, 52 insertions(+), 46 deletions(-)

diff --git a/include/ginkgo/core/base/range.hpp b/include/ginkgo/core/base/range.hpp
index c0d275d1307..b426f2729bd 100644
--- a/include/ginkgo/core/base/range.hpp
+++ b/include/ginkgo/core/base/range.hpp
@@ -338,8 +338,8 @@ class range {
      * @return a value on position `(dimensions...)`.
      */
     template <typename... DimensionTypes>
-    GKO_ATTRIBUTES constexpr auto operator()(DimensionTypes &&... dimensions)
-        const -> decltype(std::declval<const accessor>()(
+    GKO_ATTRIBUTES auto operator()(DimensionTypes &&... dimensions) const
+        -> decltype(std::declval<accessor>()(
             std::forward<DimensionTypes>(dimensions)...))
     {
         static_assert(sizeof...(dimensions) <= dimensionality,
@@ -347,15 +347,6 @@ class range {
         return accessor_(std::forward<DimensionTypes>(dimensions)...);
     }
 
-    template <typename... DimensionTypes>
-    GKO_ATTRIBUTES auto operator()(DimensionTypes &&... dimensions) -> decltype(
-        std::declval<accessor>()(std::forward<DimensionTypes>(dimensions)...))
-    {
-        static_assert(sizeof...(dimensions) <= dimensionality,
-                      "Too many dimensions in range call");
-        return accessor_(std::forward<DimensionTypes>(dimensions)...);
-    }
-
     /**
      * @copydoc operator=(const range &other)
      *
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index fc604544cf4..15d01b1e244 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -291,21 +291,25 @@ class ReducedStorageReference
     ReducedStorageReference(const ReducedStorageReference &) = delete;
     ReducedStorageReference &operator=(ReducedStorageReference &&) = delete;
 
-    GKO_ATTRIBUTES ReducedStorageReference(storage_type *const GKO_RESTRICT ptr)
+    GKO_ATTRIBUTES constexpr ReducedStorageReference(
+        storage_type *const GKO_RESTRICT ptr)
         : ptr_{ptr}
     {}
-    GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
+    ~ReducedStorageReference() = default;
+
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr);
     }
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator=(arithmetic_type val) &&
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
+        arithmetic_type val) &&
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return *r_ptr = static_cast<storage_type>(val);
     }
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type
-    operator=(const ReducedStorageReference &ref) &&
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
+        const ReducedStorageReference &ref) &&
     {
         return std::move(*this) = static_cast<arithmetic_type>(ref);
     }
@@ -331,10 +335,13 @@ class ReducedStorageReference<ArithmeticType, const StorageType>
     ReducedStorageReference(const ReducedStorageReference &) = delete;
     ReducedStorageReference &operator=(ReducedStorageReference &&) = delete;
 
-    GKO_ATTRIBUTES ReducedStorageReference(storage_type *const GKO_RESTRICT ptr)
+    GKO_ATTRIBUTES constexpr ReducedStorageReference(
+        storage_type *const GKO_RESTRICT ptr)
         : ptr_{ptr}
     {}
-    GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
+    ~ReducedStorageReference() = default;
+
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr);
@@ -369,22 +376,25 @@ class ScaledReducedStorageReference
     ScaledReducedStorageReference &operator=(ScaledReducedStorageReference &&) =
         delete;
 
-    GKO_ATTRIBUTES ScaledReducedStorageReference(
+    GKO_ATTRIBUTES constexpr ScaledReducedStorageReference(
         storage_type *const GKO_RESTRICT ptr, arithmetic_type scale)
         : ptr_{ptr}, scale_{scale}
     {}
-    GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
+    ~ScaledReducedStorageReference() = default;
+
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr) * scale_;
     }
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator=(arithmetic_type val) &&
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
+        arithmetic_type val) &&
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return *r_ptr = static_cast<storage_type>(val / scale_), val;
     }
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type
-    operator=(const ScaledReducedStorageReference &ref) &&
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
+        const ScaledReducedStorageReference &ref) &&
     {
         return std::move(*this) = static_cast<arithmetic_type>(ref);
     }
@@ -414,11 +424,13 @@ class ScaledReducedStorageReference<ArithmeticType, const StorageType>
     ScaledReducedStorageReference &operator=(ScaledReducedStorageReference &&) =
         delete;
 
-    GKO_ATTRIBUTES ScaledReducedStorageReference(
+    GKO_ATTRIBUTES constexpr ScaledReducedStorageReference(
         storage_type *const GKO_RESTRICT ptr, arithmetic_type scale)
         : ptr_{ptr}, scale_{scale}
     {}
-    GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
+    ~ScaledReducedStorageReference() = default;
+
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr) * scale_;
@@ -484,13 +496,13 @@ class ReducedStorage3d {
         : ReducedStorage3d{nullptr, {0, 0, 0}}
     {}
 
-    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr const_accessor to_const() const
     {
         return {storage_, size_, stride_[0], stride_[1]};
     }
 
     // Functions required by the `range` interface:
-    GKO_ATTRIBUTES GKO_INLINE dim<dimensionality> length(
+    GKO_ATTRIBUTES GKO_INLINE constexpr dim<dimensionality> length(
         size_type dimension) const
     {
         return dimension < dimensionality ? size_[dimension] : 1;
@@ -512,9 +524,9 @@ class ReducedStorage3d {
             }
         }
     }
-    GKO_ATTRIBUTES GKO_INLINE
-        std::conditional_t<is_const, arithmetic_type, reference>
-        operator()(size_type x, size_type y, size_type z) const
+    GKO_ATTRIBUTES GKO_INLINE constexpr std::conditional_t<
+        is_const, arithmetic_type, reference>
+    operator()(size_type x, size_type y, size_type z) const
     {
         return reference{this->storage_ + compute_index(x, y, z)};
     }
@@ -536,25 +548,26 @@ class ReducedStorage3d {
                    stride_[0], stride_[1]);
     }
 
-    GKO_ATTRIBUTES GKO_INLINE dim<dimensionality> get_size() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr dim<dimensionality> get_size() const
     {
         return size_;
     }
-    GKO_ATTRIBUTES GKO_INLINE size_type get_stride0() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr size_type get_stride0() const
     {
         return stride_[0];
     }
-    GKO_ATTRIBUTES GKO_INLINE size_type get_stride1() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr size_type get_stride1() const
     {
         return stride_[1];
     }
 
-    GKO_ATTRIBUTES GKO_INLINE storage_type *get_storage() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr storage_type *get_storage() const
     {
         return storage_;
     }
 
-    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_const_storage() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr const storage_type *get_const_storage()
+        const
     {
         return storage_;
     }
@@ -688,7 +701,7 @@ class ScaledReducedStorage3d
     }
 
     // Functions required by the `range` interface:
-    GKO_ATTRIBUTES GKO_INLINE dim<dimensionality> length(
+    GKO_ATTRIBUTES GKO_INLINE constexpr dim<dimensionality> length(
         size_type dimension) const
     {
         return dimension < dimensionality ? size_[dimension] : 1;
@@ -716,9 +729,9 @@ class ScaledReducedStorage3d
             }
         }
     }
-    GKO_ATTRIBUTES GKO_INLINE
-        std::conditional_t<is_const, arithmetic_type, reference>
-        operator()(size_type x, size_type y, size_type z) const
+    GKO_ATTRIBUTES GKO_INLINE constexpr std::conditional_t<
+        is_const, arithmetic_type, reference>
+    operator()(size_type x, size_type y, size_type z) const
     {
         return reference{this->storage_ + this->compute_index(x, y, z),
                          this->read_scale(x, z)};
@@ -742,35 +755,37 @@ class ScaledReducedStorage3d
                    this->stride_[0], this->stride_[1], this->scale_);
     }
 
-    GKO_ATTRIBUTES GKO_INLINE dim<dimensionality> get_size() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr dim<dimensionality> get_size() const
     {
         return size_;
     }
-    GKO_ATTRIBUTES GKO_INLINE size_type get_stride0() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr size_type get_stride0() const
     {
         return stride_[0];
     }
-    GKO_ATTRIBUTES GKO_INLINE size_type get_stride1() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr size_type get_stride1() const
     {
         return stride_[1];
     }
 
-    GKO_ATTRIBUTES GKO_INLINE storage_type *get_storage() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr storage_type *get_storage() const
     {
         return storage_;
     }
 
-    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_const_storage() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr const storage_type *get_const_storage()
+        const
     {
         return storage_;
     }
 
-    GKO_ATTRIBUTES GKO_INLINE scale_type *get_scale() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr scale_type *get_scale() const
     {
         return this->scale_;
     }
 
-    GKO_ATTRIBUTES GKO_INLINE const scale_type *get_const_scale() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr const scale_type *get_const_scale()
+        const
     {
         return this->scale_;
     }

From 0ee84c77f7b57213321c98d1d3581aa450e29fd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 15 Sep 2020 19:04:06 +0200
Subject: [PATCH 0199/1008] Attempt to fix thrust::complex conversion issue

---
 include/ginkgo/core/base/range_accessors.hpp | 121 +++++++++++++------
 1 file changed, 81 insertions(+), 40 deletions(-)

diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 15d01b1e244..f4e88888f4c 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/dim.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/range.hpp>
+#include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/types.hpp>
 
 
@@ -198,6 +199,45 @@ class row_major {
 namespace detail {
 
 
+// tests if the cast operator to `ValueType` is present
+template <typename Ref, typename ValueType, typename = xstd::void_t<>>
+struct has_cast_operator : std::false_type {};
+
+template <typename Ref, typename ValueType>
+struct has_cast_operator<
+    Ref, ValueType,
+    xstd::void_t<decltype(std::declval<Ref>().operator ValueType())>>
+    : std::true_type {};
+
+/**
+ * @internal
+ * converts `ref` to ValueType while prefering the operator overload from
+ * class `Ref`
+ * This function is only needed for CUDA TOOLKIT < 11 because thrust::complex
+ * has a constructor call
+ * `template<T> complex(const T &other) : real(other), imag()`, which is always
+ * preferred over the overloaded `operator value_type()`.
+ * This function uses the overloaded operator when available, otherwise simply
+ * calls `static_cast`.
+ */
+template <typename ValueType, typename Ref>
+GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
+    has_cast_operator<Ref, ValueType>::value, ValueType>
+to_value_type(const Ref &ref)
+{
+    return ref.operator ValueType();
+}
+
+template <typename ValueType, typename Ref>
+GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
+    !has_cast_operator<Ref, ValueType>::value, ValueType>
+to_value_type(const Ref &ref)
+{
+    static_assert(std::is_same<ValueType, void>::value,
+                  "This should not be used here!");
+    return static_cast<ValueType>(ref);
+}
+
 /**
  * This is a mixin which defines the binary operators for *, /, +, - for the
  * Reference class, the unary operator -, and the assignment operators
@@ -209,24 +249,25 @@ namespace detail {
  */
 template <typename Reference, typename ArithmeticType>
 struct enable_reference {
-    using arithmetic_type = ArithmeticType;
-
-#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_oper, _op)            \
-    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper( \
-        const Reference &ref1, const Reference &ref2)                 \
-    {                                                                 \
-        return static_cast<arithmetic_type>(ref1)                     \
-            _op static_cast<arithmetic_type>(ref2);                   \
-    }                                                                 \
-    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper( \
-        const Reference &ref, const arithmetic_type &a)               \
-    {                                                                 \
-        return static_cast<arithmetic_type>(ref) _op a;               \
-    }                                                                 \
-    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper( \
-        const arithmetic_type &a, const Reference &ref)               \
-    {                                                                 \
-        return a _op static_cast<arithmetic_type>(ref);               \
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
+
+
+#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_oper, _op)  \
+    friend GKO_ATTRIBUTES GKO_INLINE arithmetic_type _oper( \
+        const Reference &ref1, const Reference &ref2)       \
+    {                                                       \
+        return to_value_type<arithmetic_type>(ref1)         \
+            _op to_value_type<arithmetic_type>(ref2);       \
+    }                                                       \
+    friend GKO_ATTRIBUTES GKO_INLINE arithmetic_type _oper( \
+        const Reference &ref, const arithmetic_type &a)     \
+    {                                                       \
+        return to_value_type<arithmetic_type>(ref) _op a;   \
+    }                                                       \
+    friend GKO_ATTRIBUTES GKO_INLINE arithmetic_type _oper( \
+        const arithmetic_type &a, const Reference &ref)     \
+    {                                                       \
+        return a _op to_value_type<arithmetic_type>(ref);   \
     }
     GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator*, *)
     GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator/, /)
@@ -234,17 +275,17 @@ struct enable_reference {
     GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator-, -)
 #undef GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD
 
-#define GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(_oper, _op)           \
-    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper(    \
-        Reference &&ref1, const Reference &ref2)                         \
-    {                                                                    \
-        return std::move(ref1) = static_cast<arithmetic_type>(ref1)      \
-                   _op static_cast<arithmetic_type>(ref2);               \
-    }                                                                    \
-    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper(    \
-        Reference &&ref, const arithmetic_type &a)                       \
-    {                                                                    \
-        return std::move(ref) = static_cast<arithmetic_type>(ref) _op a; \
+#define GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(_oper, _op)             \
+    friend GKO_ATTRIBUTES GKO_INLINE arithmetic_type _oper(                \
+        Reference &&ref1, const Reference &ref2)                           \
+    {                                                                      \
+        return std::move(ref1) = to_value_type<arithmetic_type>(ref1)      \
+                   _op to_value_type<arithmetic_type>(ref2);               \
+    }                                                                      \
+    friend GKO_ATTRIBUTES GKO_INLINE arithmetic_type _oper(                \
+        Reference &&ref, const arithmetic_type &a)                         \
+    {                                                                      \
+        return std::move(ref) = to_value_type<arithmetic_type>(ref) _op a; \
     }
 
     GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator*=, *)
@@ -255,10 +296,10 @@ struct enable_reference {
 
     // TODO test if comparison operators need to be overloaded as well
 
-    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type
+    friend GKO_ATTRIBUTES GKO_INLINE arithmetic_type
     operator-(const Reference &ref)
     {
-        return -static_cast<arithmetic_type>(ref);
+        return -to_value_type<arithmetic_type>(ref);
     }
 };
 
@@ -281,7 +322,7 @@ class ReducedStorageReference
           ReducedStorageReference<ArithmeticType, StorageType>,
           ArithmeticType> {
 public:
-    using arithmetic_type = ArithmeticType;
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = StorageType;
     // Allow move construction, so perfect forwarding is possible
     ReducedStorageReference(ReducedStorageReference &&) = default;
@@ -325,7 +366,7 @@ class ReducedStorageReference<ArithmeticType, const StorageType>
           ReducedStorageReference<ArithmeticType, const StorageType>,
           ArithmeticType> {
 public:
-    using arithmetic_type = ArithmeticType;
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = const StorageType;
     // Allow move construction, so perfect forwarding is possible
     ReducedStorageReference(ReducedStorageReference &&) = default;
@@ -363,7 +404,7 @@ class ScaledReducedStorageReference
           ScaledReducedStorageReference<ArithmeticType, StorageType>,
           ArithmeticType> {
 public:
-    using arithmetic_type = ArithmeticType;
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = StorageType;
 
     // Allow move construction, so perfect forwarding is possible
@@ -411,7 +452,7 @@ class ScaledReducedStorageReference<ArithmeticType, const StorageType>
           ScaledReducedStorageReference<ArithmeticType, const StorageType>,
           ArithmeticType> {
 public:
-    using arithmetic_type = ArithmeticType;
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = const StorageType;
 
     // Allow move construction, so perfect forwarding is possible
@@ -459,12 +500,12 @@ class ScaledReducedStorageReference<ArithmeticType, const StorageType>
 template <typename ArithmeticType, typename StorageType>
 class ReducedStorage3d {
 public:
-    using arithmetic_type = ArithmeticType;
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = StorageType;
     using const_accessor =
         ReducedStorage3d<arithmetic_type, const storage_type>;
     static constexpr size_type dimensionality{3};
-    static constexpr bool is_const{std::is_const<StorageType>::value};
+    static constexpr bool is_const{std::is_const<storage_type>::value};
 
 protected:
     using reference =
@@ -643,10 +684,10 @@ class ScaledReducedStorage3d
     : public detail::enable_scale_write<
           ScaledReducedStorage3d<ArithmeticType, StorageType>, ArithmeticType> {
 public:
-    using arithmetic_type = ArithmeticType;
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = StorageType;
     static constexpr size_type dimensionality{3};
-    static constexpr bool is_const{std::is_const<StorageType>::value};
+    static constexpr bool is_const{std::is_const<storage_type>::value};
     using scale_type =
         std::conditional_t<is_const, const arithmetic_type, arithmetic_type>;
 
@@ -657,7 +698,7 @@ class ScaledReducedStorage3d
 
 protected:
     using reference =
-        detail::reference::ScaledReducedStorageReference<ArithmeticType,
+        detail::reference::ScaledReducedStorageReference<arithmetic_type,
                                                          StorageType>;
 
 public:

From 598f6954d59328bf8d089d2418c1e6716d90f519 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 17 Sep 2020 02:47:27 +0200
Subject: [PATCH 0200/1008] Add workaround for CUDA for reference casting

---
 include/ginkgo/core/base/range_accessors.hpp | 90 +++++++++++++++++---
 1 file changed, 77 insertions(+), 13 deletions(-)

diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index f4e88888f4c..740db1a0477 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -201,22 +201,38 @@ namespace detail {
 
 // tests if the cast operator to `ValueType` is present
 template <typename Ref, typename ValueType, typename = xstd::void_t<>>
-struct has_cast_operator : std::false_type {};
+struct has_cast_operator : std::false_type {
+};
 
 template <typename Ref, typename ValueType>
 struct has_cast_operator<
     Ref, ValueType,
     xstd::void_t<decltype(std::declval<Ref>().operator ValueType())>>
-    : std::true_type {};
+    : std::true_type {
+};
+
+template <typename Ref, typename ValueType, typename = xstd::void_t<>>
+struct has_to_arithmetic_member : std::false_type {
+};
+
+template <typename Ref, typename ValueType>
+struct has_to_arithmetic_member<
+    Ref, ValueType, xstd::void_t<decltype(std::declval<Ref>().to_arithmetic())>>
+    : std::true_type {
+};
 
 /**
  * @internal
- * converts `ref` to ValueType while prefering the operator overload from
- * class `Ref`
+ * converts `ref` to ValueType while preferring the operator overload, followed
+ * by the function call `to_arithmetic()` from class `Ref` before falling back
+ * to a simple `static_cast<ValueType>`.
+ *
  * This function is only needed for CUDA TOOLKIT < 11 because thrust::complex
- * has a constructor call
+ * has a constructor call:
  * `template<T> complex(const T &other) : real(other), imag()`, which is always
  * preferred over the overloaded `operator value_type()`.
+ * Additionally, these Toolkit versions seem to not recognize the cast operator
+ * overload, meaning a function `to_arithmetic`
  * This function uses the overloaded operator when available, otherwise simply
  * calls `static_cast`.
  */
@@ -224,13 +240,25 @@ template <typename ValueType, typename Ref>
 GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
     has_cast_operator<Ref, ValueType>::value, ValueType>
 to_value_type(const Ref &ref)
+{
+    return ref.to_arithmetic();
+}
+
+template <typename ValueType, typename Ref>
+GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
+    !has_cast_operator<Ref, ValueType>::value &&
+        has_to_arithmetic_member<Ref, ValueType>::value,
+    ValueType>
+to_value_type(const Ref &ref)
 {
     return ref.operator ValueType();
 }
 
 template <typename ValueType, typename Ref>
 GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
-    !has_cast_operator<Ref, ValueType>::value, ValueType>
+    !has_cast_operator<Ref, ValueType>::value &&
+        !has_to_arithmetic_member<Ref, ValueType>::value,
+    ValueType>
 to_value_type(const Ref &ref)
 {
     static_assert(std::is_same<ValueType, void>::value,
@@ -242,8 +270,21 @@ to_value_type(const Ref &ref)
  * This is a mixin which defines the binary operators for *, /, +, - for the
  * Reference class, the unary operator -, and the assignment operators
  * *=, /=, +=, -=
- * Additionally, it prevents the default generation of copy and move constructor
- * and copy and move assignment.
+ * All assignment operators expect an rvalue reference (Reference &&) for the
+ * Reference class in order to prevent copying the Reference object.
+ *
+ * @tparam Reference  The reference class this mixin provides operator
+ * overloads for. The reference class needs to overload either the cast operator
+ * to ValueType, or a `to_arithmetic()` function, returning the translated
+ * ValueType value (recommended is to provide both!).
+ *
+ * @tparam ArithmeticType  arithmetic type the Reference class is supposed to
+ *                         represent.
+ *
+ * @note  For CUDA Toolkit < 11, the cast operator is not recognized properly,
+ *        which is why a function `to_arithmetic()` needs to be provided by the
+ *        Reference class, which does the exact same (returning the referenced
+ *        value).
  *
  * @warning  This struct should only be used by reference classes.
  */
@@ -315,6 +356,10 @@ namespace reference {
 /**
  * Reference class for a different storage than arithmetic type. The conversion
  * between both formats is done with a simple static_cast.
+ *
+ * Copying this reference is disabled, but move construction is possible to
+ * allow for an additional layer (like gko::range).
+ * The assignment operator only works for an rvalue reference (&&)
  */
 template <typename ArithmeticType, typename StorageType>
 class ReducedStorageReference
@@ -324,7 +369,8 @@ class ReducedStorageReference
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = StorageType;
-    // Allow move construction, so perfect forwarding is possible
+    // Allow move construction, so perfect forwarding is possible (required for
+    // `range` support)
     ReducedStorageReference(ReducedStorageReference &&) = default;
 
     ReducedStorageReference() = delete;
@@ -338,11 +384,15 @@ class ReducedStorageReference
     {}
     ~ReducedStorageReference() = default;
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type to_arithmetic() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr);
     }
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    {
+        return to_arithmetic();
+    }
     GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
         arithmetic_type val) &&
     {
@@ -382,11 +432,15 @@ class ReducedStorageReference<ArithmeticType, const StorageType>
     {}
     ~ReducedStorageReference() = default;
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type to_arithmetic() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr);
     }
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    {
+        return to_arithmetic();
+    }
 
 private:
     storage_type *const GKO_RESTRICT ptr_;
@@ -423,11 +477,16 @@ class ScaledReducedStorageReference
     {}
     ~ScaledReducedStorageReference() = default;
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type to_arithmetic() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr) * scale_;
     }
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    {
+        return to_arithmetic();
+    }
+
     GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
         arithmetic_type val) &&
     {
@@ -471,11 +530,16 @@ class ScaledReducedStorageReference<ArithmeticType, const StorageType>
     {}
     ~ScaledReducedStorageReference() = default;
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type to_arithmetic() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr) * scale_;
     }
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    {
+        return to_arithmetic();
+    }
+
 
 private:
     storage_type *const GKO_RESTRICT ptr_;

From 4a1d1535213c52937840d4db57290e0978249ba4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 17 Sep 2020 06:13:26 +0200
Subject: [PATCH 0201/1008] Use better workaround for CUDA references

Make accessor references work with older CUDA versions by having a
conditional constexpr qualifier and by forcing it to use the overloaded
cast operation (when present).
---
 include/ginkgo/core/base/range_accessors.hpp | 210 +++++++++----------
 1 file changed, 101 insertions(+), 109 deletions(-)

diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 740db1a0477..686bd59bb3d 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -47,6 +47,20 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
+// CUDA TOOLKIT < 11 does not support constexpr in combination with
+// thrust::complex, which is why constexpr is only present in later versions
+#if defined(__CUDA_ARCH__) && defined(__CUDACC_VER_MAJOR__) && \
+    (__CUDACC_VER_MAJOR__ < 11)
+
+#define GKO_ENABLE_REFERENCE_CONSTEXPR
+
+#else
+
+#define GKO_ENABLE_REFERENCE_CONSTEXPR constexpr
+
+#endif
+
+
 namespace gko {
 /**
  * @brief The accessor namespace.
@@ -69,7 +83,8 @@ namespace accessor {
  *          2-dimensional ranges.
  *
  * @tparam ValueType  type of values this accessor returns
- * @tparam Dimensionality  number of dimensions of this accessor (has to be 2)
+ * @tparam Dimensionality  number of dimensions of this accessor (has to be
+ * 2)
  */
 template <typename ValueType, size_type Dimensionality>
 class row_major {
@@ -102,8 +117,8 @@ class row_major {
      * @param num_row  number of rows of the accessor
      * @param num_cols  number of columns of the accessor
      * @param stride  distance (in elements) between starting positions of
-     *                consecutive rows (i.e. `data + i * stride` points to the
-     *                `i`-th row)
+     *                consecutive rows (i.e. `data + i * stride` points to
+     * the `i`-th row)
      */
     GKO_ATTRIBUTES constexpr explicit row_major(data_type data,
                                                 size_type num_rows,
@@ -201,66 +216,39 @@ namespace detail {
 
 // tests if the cast operator to `ValueType` is present
 template <typename Ref, typename ValueType, typename = xstd::void_t<>>
-struct has_cast_operator : std::false_type {
-};
+struct has_cast_operator : std::false_type {};
 
 template <typename Ref, typename ValueType>
 struct has_cast_operator<
     Ref, ValueType,
-    xstd::void_t<decltype(std::declval<Ref>().operator ValueType())>>
-    : std::true_type {
-};
-
-template <typename Ref, typename ValueType, typename = xstd::void_t<>>
-struct has_to_arithmetic_member : std::false_type {
-};
-
-template <typename Ref, typename ValueType>
-struct has_to_arithmetic_member<
-    Ref, ValueType, xstd::void_t<decltype(std::declval<Ref>().to_arithmetic())>>
-    : std::true_type {
-};
+    xstd::void_t<decltype(std::declval<Ref>().Ref::operator ValueType())>>
+    : std::true_type {};
 
 /**
  * @internal
- * converts `ref` to ValueType while preferring the operator overload, followed
- * by the function call `to_arithmetic()` from class `Ref` before falling back
- * to a simple `static_cast<ValueType>`.
+ * converts `ref` to ValueType while preferring the cast operator overload
+ * from class `Ref` before falling back to a simple
+ * `static_cast<ValueType>`.
  *
- * This function is only needed for CUDA TOOLKIT < 11 because thrust::complex
- * has a constructor call:
- * `template<T> complex(const T &other) : real(other), imag()`, which is always
- * preferred over the overloaded `operator value_type()`.
- * Additionally, these Toolkit versions seem to not recognize the cast operator
- * overload, meaning a function `to_arithmetic`
- * This function uses the overloaded operator when available, otherwise simply
- * calls `static_cast`.
+ * This function is only needed for CUDA TOOLKIT < 11 because
+ * thrust::complex has a constructor call: `template<T> complex(const T
+ * &other) : real(other), imag()`, which is always preferred over the
+ * overloaded `operator value_type()`.
  */
 template <typename ValueType, typename Ref>
 GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
     has_cast_operator<Ref, ValueType>::value, ValueType>
 to_value_type(const Ref &ref)
 {
-    return ref.to_arithmetic();
+    return ref.Ref::operator ValueType();
 }
 
 template <typename ValueType, typename Ref>
 GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
-    !has_cast_operator<Ref, ValueType>::value &&
-        has_to_arithmetic_member<Ref, ValueType>::value,
-    ValueType>
-to_value_type(const Ref &ref)
-{
-    return ref.operator ValueType();
-}
-
-template <typename ValueType, typename Ref>
-GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
-    !has_cast_operator<Ref, ValueType>::value &&
-        !has_to_arithmetic_member<Ref, ValueType>::value,
-    ValueType>
+    !has_cast_operator<Ref, ValueType>::value, ValueType>
 to_value_type(const Ref &ref)
 {
+    // TODO  remove
     static_assert(std::is_same<ValueType, void>::value,
                   "This should not be used here!");
     return static_cast<ValueType>(ref);
@@ -270,21 +258,15 @@ to_value_type(const Ref &ref)
  * This is a mixin which defines the binary operators for *, /, +, - for the
  * Reference class, the unary operator -, and the assignment operators
  * *=, /=, +=, -=
- * All assignment operators expect an rvalue reference (Reference &&) for the
- * Reference class in order to prevent copying the Reference object.
+ * All assignment operators expect an rvalue reference (Reference &&) for
+ * the Reference class in order to prevent copying the Reference object.
  *
  * @tparam Reference  The reference class this mixin provides operator
- * overloads for. The reference class needs to overload either the cast operator
- * to ValueType, or a `to_arithmetic()` function, returning the translated
- * ValueType value (recommended is to provide both!).
+ * overloads for. The reference class needs to overload the cast operator
+ * to ValueType
  *
- * @tparam ArithmeticType  arithmetic type the Reference class is supposed to
- *                         represent.
- *
- * @note  For CUDA Toolkit < 11, the cast operator is not recognized properly,
- *        which is why a function `to_arithmetic()` needs to be provided by the
- *        Reference class, which does the exact same (returning the referenced
- *        value).
+ * @tparam ArithmeticType  arithmetic type the Reference class is supposed
+ * to represent.
  *
  * @warning  This struct should only be used by reference classes.
  */
@@ -293,22 +275,25 @@ struct enable_reference {
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
 
 
-#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_oper, _op)  \
-    friend GKO_ATTRIBUTES GKO_INLINE arithmetic_type _oper( \
-        const Reference &ref1, const Reference &ref2)       \
-    {                                                       \
-        return to_value_type<arithmetic_type>(ref1)         \
-            _op to_value_type<arithmetic_type>(ref2);       \
-    }                                                       \
-    friend GKO_ATTRIBUTES GKO_INLINE arithmetic_type _oper( \
-        const Reference &ref, const arithmetic_type &a)     \
-    {                                                       \
-        return to_value_type<arithmetic_type>(ref) _op a;   \
-    }                                                       \
-    friend GKO_ATTRIBUTES GKO_INLINE arithmetic_type _oper( \
-        const arithmetic_type &a, const Reference &ref)     \
-    {                                                       \
-        return a _op to_value_type<arithmetic_type>(ref);   \
+#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_oper, _op)          \
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR \
+        arithmetic_type                                             \
+        _oper(const Reference &ref1, const Reference &ref2)         \
+    {                                                               \
+        return to_value_type<arithmetic_type>(ref1)                 \
+            _op to_value_type<arithmetic_type>(ref2);               \
+    }                                                               \
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR \
+        arithmetic_type                                             \
+        _oper(const Reference &ref, const arithmetic_type &a)       \
+    {                                                               \
+        return to_value_type<arithmetic_type>(ref) _op a;           \
+    }                                                               \
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR \
+        arithmetic_type                                             \
+        _oper(const arithmetic_type &a, const Reference &ref)       \
+    {                                                               \
+        return a _op to_value_type<arithmetic_type>(ref);           \
     }
     GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator*, *)
     GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator/, /)
@@ -317,14 +302,16 @@ struct enable_reference {
 #undef GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD
 
 #define GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(_oper, _op)             \
-    friend GKO_ATTRIBUTES GKO_INLINE arithmetic_type _oper(                \
-        Reference &&ref1, const Reference &ref2)                           \
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR        \
+        arithmetic_type                                                    \
+        _oper(Reference &&ref1, const Reference &ref2)                     \
     {                                                                      \
         return std::move(ref1) = to_value_type<arithmetic_type>(ref1)      \
                    _op to_value_type<arithmetic_type>(ref2);               \
     }                                                                      \
-    friend GKO_ATTRIBUTES GKO_INLINE arithmetic_type _oper(                \
-        Reference &&ref, const arithmetic_type &a)                         \
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR        \
+        arithmetic_type                                                    \
+        _oper(Reference &&ref, const arithmetic_type &a)                   \
     {                                                                      \
         return std::move(ref) = to_value_type<arithmetic_type>(ref) _op a; \
     }
@@ -337,8 +324,9 @@ struct enable_reference {
 
     // TODO test if comparison operators need to be overloaded as well
 
-    friend GKO_ATTRIBUTES GKO_INLINE arithmetic_type
-    operator-(const Reference &ref)
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR
+        arithmetic_type
+        operator-(const Reference &ref)
     {
         return -to_value_type<arithmetic_type>(ref);
     }
@@ -354,12 +342,19 @@ namespace reference {
 
 
 /**
- * Reference class for a different storage than arithmetic type. The conversion
- * between both formats is done with a simple static_cast.
+ * Reference class for a different storage than arithmetic type. The
+ * conversion between both formats is done with a simple static_cast.
  *
  * Copying this reference is disabled, but move construction is possible to
  * allow for an additional layer (like gko::range).
- * The assignment operator only works for an rvalue reference (&&)
+ * The assignment operator only works for an rvalue reference (&&) to
+ * prevent accidental copying the reference and working on a reference.
+ *
+ * @tparam ArithmeticType  Type used for arithmetic operations, therefore,
+ * the type which is used for input and output of this class.
+ *
+ * @tparam StorageType  Type actually used as a storage, which is converted
+ * to ArithmeticType before usage
  */
 template <typename ArithmeticType, typename StorageType>
 class ReducedStorageReference
@@ -369,8 +364,8 @@ class ReducedStorageReference
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = StorageType;
-    // Allow move construction, so perfect forwarding is possible (required for
-    // `range` support)
+    // Allow move construction, so perfect forwarding is possible (required
+    // for `range` support)
     ReducedStorageReference(ReducedStorageReference &&) = default;
 
     ReducedStorageReference() = delete;
@@ -384,15 +379,11 @@ class ReducedStorageReference
     {}
     ~ReducedStorageReference() = default;
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type to_arithmetic() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr);
     }
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
-    {
-        return to_arithmetic();
-    }
     GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
         arithmetic_type val) &&
     {
@@ -432,15 +423,11 @@ class ReducedStorageReference<ArithmeticType, const StorageType>
     {}
     ~ReducedStorageReference() = default;
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type to_arithmetic() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr);
     }
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
-    {
-        return to_arithmetic();
-    }
 
 private:
     storage_type *const GKO_RESTRICT ptr_;
@@ -448,9 +435,22 @@ class ReducedStorageReference<ArithmeticType, const StorageType>
 
 
 /**
- * Reference class for a different storage than arithmetic type. The conversion
- * between both formats is done with a simple static_cast followed by a
- * multiplication with a scale.
+ * Reference class for a different storage than arithmetic type with the
+ * addition of a scaling factor. The conversion between both formats is done
+ * with a static_cast to the ArithmeticType, followed by a multiplication
+ * of the scale (when reading; for writing, the new value is divided by the
+ * scale before casting to the StorageType).
+ *
+ * Copying this reference is disabled, but move construction is possible to
+ * allow for an additional layer (like gko::range).
+ * The assignment operator only works for an rvalue reference (&&) to
+ * prevent accidental copying the reference and working on a reference.
+ *
+ * @tparam ArithmeticType  Type used for arithmetic operations, therefore,
+ * the type which is used for input and output of this class.
+ *
+ * @tparam StorageType  Type actually used as a storage, which is converted
+ * to ArithmeticType before usage
  */
 template <typename ArithmeticType, typename StorageType>
 class ScaledReducedStorageReference
@@ -477,15 +477,11 @@ class ScaledReducedStorageReference
     {}
     ~ScaledReducedStorageReference() = default;
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type to_arithmetic() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr) * scale_;
     }
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
-    {
-        return to_arithmetic();
-    }
 
     GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
         arithmetic_type val) &&
@@ -530,15 +526,11 @@ class ScaledReducedStorageReference<ArithmeticType, const StorageType>
     {}
     ~ScaledReducedStorageReference() = default;
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type to_arithmetic() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr) * scale_;
     }
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
-    {
-        return to_arithmetic();
-    }
 
 
 private:
@@ -583,8 +575,8 @@ class ReducedStorage3d {
      *
      * @internal
      * const_cast needed, so ReducedStorage3d(ac.get_storage(), ...)
-     * works. Also, storage_ is never accessed in a non-const fashion in this
-     * class, so it is not invalid or UB code.
+     * works. Also, storage_ is never accessed in a non-const fashion in
+     * this class, so it is not invalid or UB code.
      */
     GKO_ATTRIBUTES constexpr ReducedStorage3d(storage_type *storage,
                                               gko::dim<dimensionality> size,
@@ -736,8 +728,8 @@ struct enable_scale_write<Accessor, ScaleType, false> {
  *
  * The ScaledReducedStorage3d class hides the underlying storage format and
  * provides a simple interface for accessing a one dimensional storage.
- * Additionally, this accessor posesses a scale array, which is used for each
- * read and write operation to do a proper conversion.
+ * Additionally, this accessor posesses a scale array, which is used for
+ * each read and write operation to do a proper conversion.
  *
  * This class only manages the accesses, however, and not the memory itself.
  *

From c24afd526dba91df9fbc83725c6fe2312392ed41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 17 Sep 2020 20:07:54 +0200
Subject: [PATCH 0202/1008] Rename and document new accessors properly

---
 core/test/base/range_accessors.cpp           |  15 +-
 include/ginkgo/core/base/range_accessors.hpp | 478 ++++++++++++++-----
 2 files changed, 359 insertions(+), 134 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index b91c2ed67a9..c57a082d1cf 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -147,9 +147,9 @@ class ReducedStorage3d : public ::testing::Test {
     using ar_type = double;
     using st_type = double;
 
-    using accessor = gko::accessor::ReducedStorage3d<ar_type, st_type>;
+    using accessor = gko::accessor::reduced_row_major<3, ar_type, st_type>;
     using const_accessor =
-        gko::accessor::ReducedStorage3d<ar_type, const st_type>;
+        gko::accessor::reduced_row_major<3, ar_type, const st_type>;
 
     using reduced_storage = gko::range<accessor>;
     using const_reduced_storage = gko::range<const_accessor>;
@@ -249,9 +249,10 @@ class ScaledReducedStorage3d : public ::testing::Test {
     using ar_type = double;
     using st_type = gko::int32;
 
-    using accessor = gko::accessor::ScaledReducedStorage3d<ar_type, st_type>;
+    using accessor =
+        gko::accessor::scaled_reduced_row_major<3, ar_type, st_type>;
     using const_accessor =
-        gko::accessor::ScaledReducedStorage3d<ar_type, const st_type>;
+        gko::accessor::scaled_reduced_row_major<3, ar_type, const st_type>;
 
     using reduced_storage = gko::range<accessor>;
     using const_reduced_storage = gko::range<const_accessor>;
@@ -268,8 +269,8 @@ class ScaledReducedStorage3d : public ::testing::Test {
         1., 1.
     };
     // clang-format on
-    reduced_storage r{data, gko::dim<3>{2u, 2u, 2u}, scale};
-    const_reduced_storage cr{data, gko::dim<3>{2u, 2u, 2u}, scale};
+    reduced_storage r{data, scale, gko::dim<3>{2u, 2u, 2u}};
+    const_reduced_storage cr{data, scale, gko::dim<3>{2u, 2u, 2u}};
 };
 
 
@@ -290,7 +291,7 @@ TEST_F(ScaledReducedStorage3d, CanUseConst)
     EXPECT_EQ(subr(1, 0, 0), 55.);
 
     // cr(0, 0, 0) = 2.0;
-    r->set_scale(0, 0, 2.);
+    r->write_scale(0, 0, 2.);
     EXPECT_EQ(r(0, 0, 0), 2.);
 }
 
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 686bd59bb3d..5f412e19925 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -109,6 +109,8 @@ class row_major {
      */
     static constexpr size_type dimensionality = 2;
 
+    using const_accessor = row_major<const ValueType, Dimensionality>;
+
 protected:
     /**
      * Creates a row_major accessor.
@@ -128,6 +130,17 @@ class row_major {
     {}
 
 public:
+    /**
+     * Creates a row_major range which contains a read-only version of the
+     * current accessor.
+     *
+     * @returns  a row major range which is read-only.
+     */
+    GKO_ATTRIBUTES constexpr range<const_accessor> to_const() const
+    {
+        return range<const_accessor>{data, lengths[0], lengths[1], stride};
+    }
+
     /**
      * Returns the data element at position (row, col)
      *
@@ -248,9 +261,6 @@ GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
     !has_cast_operator<Ref, ValueType>::value, ValueType>
 to_value_type(const Ref &ref)
 {
-    // TODO  remove
-    static_assert(std::is_same<ValueType, void>::value,
-                  "This should not be used here!");
     return static_cast<ValueType>(ref);
 }
 
@@ -357,27 +367,26 @@ namespace reference {
  * to ArithmeticType before usage
  */
 template <typename ArithmeticType, typename StorageType>
-class ReducedStorageReference
+class reduced_storage
     : public detail::enable_reference<
-          ReducedStorageReference<ArithmeticType, StorageType>,
-          ArithmeticType> {
+          reduced_storage<ArithmeticType, StorageType>, ArithmeticType> {
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = StorageType;
     // Allow move construction, so perfect forwarding is possible (required
     // for `range` support)
-    ReducedStorageReference(ReducedStorageReference &&) = default;
+    reduced_storage(reduced_storage &&) = default;
 
-    ReducedStorageReference() = delete;
+    reduced_storage() = delete;
+    ~reduced_storage() = default;
     // Forbid copy construction and move assignment
-    ReducedStorageReference(const ReducedStorageReference &) = delete;
-    ReducedStorageReference &operator=(ReducedStorageReference &&) = delete;
+    reduced_storage(const reduced_storage &) = delete;
+    reduced_storage &operator=(reduced_storage &&) = delete;
 
-    GKO_ATTRIBUTES constexpr ReducedStorageReference(
+    GKO_ATTRIBUTES constexpr reduced_storage(
         storage_type *const GKO_RESTRICT ptr)
         : ptr_{ptr}
     {}
-    ~ReducedStorageReference() = default;
 
     GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
     {
@@ -391,7 +400,7 @@ class ReducedStorageReference
         return *r_ptr = static_cast<storage_type>(val);
     }
     GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
-        const ReducedStorageReference &ref) &&
+        const reduced_storage &ref) &&
     {
         return std::move(*this) = static_cast<arithmetic_type>(ref);
     }
@@ -402,26 +411,25 @@ class ReducedStorageReference
 
 // Specialization for const storage_type to prevent `operator=`
 template <typename ArithmeticType, typename StorageType>
-class ReducedStorageReference<ArithmeticType, const StorageType>
+class reduced_storage<ArithmeticType, const StorageType>
     : public detail::enable_reference<
-          ReducedStorageReference<ArithmeticType, const StorageType>,
-          ArithmeticType> {
+          reduced_storage<ArithmeticType, const StorageType>, ArithmeticType> {
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = const StorageType;
     // Allow move construction, so perfect forwarding is possible
-    ReducedStorageReference(ReducedStorageReference &&) = default;
+    reduced_storage(reduced_storage &&) = default;
 
-    ReducedStorageReference() = delete;
+    reduced_storage() = delete;
+    ~reduced_storage() = default;
     // Forbid copy construction and move assignment
-    ReducedStorageReference(const ReducedStorageReference &) = delete;
-    ReducedStorageReference &operator=(ReducedStorageReference &&) = delete;
+    reduced_storage(const reduced_storage &) = delete;
+    reduced_storage &operator=(reduced_storage &&) = delete;
 
-    GKO_ATTRIBUTES constexpr ReducedStorageReference(
+    GKO_ATTRIBUTES constexpr reduced_storage(
         storage_type *const GKO_RESTRICT ptr)
         : ptr_{ptr}
     {}
-    ~ReducedStorageReference() = default;
 
     GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
     {
@@ -453,29 +461,26 @@ class ReducedStorageReference<ArithmeticType, const StorageType>
  * to ArithmeticType before usage
  */
 template <typename ArithmeticType, typename StorageType>
-class ScaledReducedStorageReference
+class scaled_reduced_storage
     : public detail::enable_reference<
-          ScaledReducedStorageReference<ArithmeticType, StorageType>,
-          ArithmeticType> {
+          scaled_reduced_storage<ArithmeticType, StorageType>, ArithmeticType> {
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = StorageType;
 
     // Allow move construction, so perfect forwarding is possible
-    ScaledReducedStorageReference(ScaledReducedStorageReference &&) = default;
+    scaled_reduced_storage(scaled_reduced_storage &&) = default;
 
-    ScaledReducedStorageReference() = delete;
+    scaled_reduced_storage() = delete;
+    ~scaled_reduced_storage() = default;
     // Forbid copy construction and move assignment
-    ScaledReducedStorageReference(const ScaledReducedStorageReference &) =
-        delete;
-    ScaledReducedStorageReference &operator=(ScaledReducedStorageReference &&) =
-        delete;
+    scaled_reduced_storage(const scaled_reduced_storage &) = delete;
+    scaled_reduced_storage &operator=(scaled_reduced_storage &&) = delete;
 
-    GKO_ATTRIBUTES constexpr ScaledReducedStorageReference(
+    GKO_ATTRIBUTES constexpr scaled_reduced_storage(
         storage_type *const GKO_RESTRICT ptr, arithmetic_type scale)
         : ptr_{ptr}, scale_{scale}
     {}
-    ~ScaledReducedStorageReference() = default;
 
     GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
     {
@@ -490,7 +495,7 @@ class ScaledReducedStorageReference
         return *r_ptr = static_cast<storage_type>(val / scale_), val;
     }
     GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
-        const ScaledReducedStorageReference &ref) &&
+        const scaled_reduced_storage &ref) &&
     {
         return std::move(*this) = static_cast<arithmetic_type>(ref);
     }
@@ -502,29 +507,27 @@ class ScaledReducedStorageReference
 
 // Specialization for constant storage_type (no `operator=`)
 template <typename ArithmeticType, typename StorageType>
-class ScaledReducedStorageReference<ArithmeticType, const StorageType>
+class scaled_reduced_storage<ArithmeticType, const StorageType>
     : public detail::enable_reference<
-          ScaledReducedStorageReference<ArithmeticType, const StorageType>,
+          scaled_reduced_storage<ArithmeticType, const StorageType>,
           ArithmeticType> {
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = const StorageType;
 
     // Allow move construction, so perfect forwarding is possible
-    ScaledReducedStorageReference(ScaledReducedStorageReference &&) = default;
+    scaled_reduced_storage(scaled_reduced_storage &&) = default;
 
-    ScaledReducedStorageReference() = delete;
+    scaled_reduced_storage() = delete;
+    ~scaled_reduced_storage() = default;
     // Forbid copy construction and move assignment
-    ScaledReducedStorageReference(const ScaledReducedStorageReference &) =
-        delete;
-    ScaledReducedStorageReference &operator=(ScaledReducedStorageReference &&) =
-        delete;
+    scaled_reduced_storage(const scaled_reduced_storage &) = delete;
+    scaled_reduced_storage &operator=(scaled_reduced_storage &&) = delete;
 
-    GKO_ATTRIBUTES constexpr ScaledReducedStorageReference(
+    GKO_ATTRIBUTES constexpr scaled_reduced_storage(
         storage_type *const GKO_RESTRICT ptr, arithmetic_type scale)
         : ptr_{ptr}, scale_{scale}
     {}
-    ~ScaledReducedStorageReference() = default;
 
     GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
     {
@@ -544,75 +547,119 @@ class ScaledReducedStorageReference<ArithmeticType, const StorageType>
 
 
 /**
- * The ReducedStorage3d class allows a storage format that is different from
+ * The reduced_row_major class allows a storage format that is different from
  * the arithmetic format (which is returned from the brace operator).
  * As storage, the storage_type is used.
- * This accessor uses row-major access.
  *
- * @note  This class only manages the accesses and not the memory itself.
+ * This accessor uses row-major access, meaning neighboring z coordinates are
+ * next to each other in memory, followed by y coordinates and then x
+ * coordinates.
+ *
+ * @tparam Dimensionality  The number of dimensions managed by this accessor
+ *
+ * @tparam ArithmeticType  Value type used for arithmetic operations and
+ *                         for in- and output
  *
- * TODO Rename, so it is in lower_case only (since no virtual functions!)
+ * @tparam StorageType  Value type used for storing the actual value to memory
+ *
+ * @note  This class only manages the accesses and not the memory itself.
+ * @note  Currently, only Dimensionality = 3 is supported.
  */
-template <typename ArithmeticType, typename StorageType>
-class ReducedStorage3d {
+template <int Dimensionality, typename ArithmeticType, typename StorageType>
+class reduced_row_major {
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = StorageType;
-    using const_accessor =
-        ReducedStorage3d<arithmetic_type, const storage_type>;
-    static constexpr size_type dimensionality{3};
+    static constexpr size_type dimensionality{Dimensionality};
     static constexpr bool is_const{std::is_const<storage_type>::value};
+    using const_accessor =
+        reduced_row_major<dimensionality, arithmetic_type, const storage_type>;
+
+    static_assert(dimensionality == 3,
+                  "Only Dimensionality == 3 is currently supported");
+
+    friend class range<reduced_row_major>;
 
 protected:
     using reference =
-        detail::reference::ReducedStorageReference<arithmetic_type,
-                                                   storage_type>;
+        detail::reference::reduced_storage<arithmetic_type, storage_type>;
 
-public:
     /**
      * Creates the accessor with an already allocated storage space with a
-     * stride.
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
      *
-     * @internal
-     * const_cast needed, so ReducedStorage3d(ac.get_storage(), ...)
-     * works. Also, storage_ is never accessed in a non-const fashion in
-     * this class, so it is not invalid or UB code.
+     * @param storage  pointer to the block of memory containing the storage
+     * @param size  multidimensional size of the memory
+     * @param stride0  stride used for the x-indices
+     * @param stride1  stride used for the y-indices
      */
-    GKO_ATTRIBUTES constexpr ReducedStorage3d(storage_type *storage,
-                                              gko::dim<dimensionality> size,
-                                              size_type stride0,
-                                              size_type stride1)
+    GKO_ATTRIBUTES constexpr reduced_row_major(storage_type *storage,
+                                               gko::dim<dimensionality> size,
+                                               size_type stride0,
+                                               size_type stride1)
         : storage_{storage}, size_{size}, stride_{stride0, stride1}
     {}
 
-    GKO_ATTRIBUTES constexpr ReducedStorage3d(storage_type *storage,
-                                              dim<dimensionality> size)
-        : ReducedStorage3d{storage, size, size[1] * size[2], size[2]}
+    /**
+     * Creates the accessor with an already allocated storage space with a
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
+     * It is assumed that accesses are without a stride.
+     *
+     * @param storage  pointer to the block of memory containing the storage
+     * @param size  multidimensional size of the memory
+     */
+    GKO_ATTRIBUTES constexpr reduced_row_major(storage_type *storage,
+                                               dim<dimensionality> size)
+        : reduced_row_major{storage, size, size[1] * size[2], size[2]}
     {}
-    GKO_ATTRIBUTES constexpr ReducedStorage3d()
-        : ReducedStorage3d{nullptr, {0, 0, 0}}
+
+    /**
+     * Creates an empty accessor (pointing nowhere with an empty size)
+     */
+    GKO_ATTRIBUTES constexpr reduced_row_major()
+        : reduced_row_major{nullptr, {0, 0, 0}}
     {}
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr const_accessor to_const() const
+public:
+    /**
+     * Creates a reduced_row_major range which contains a read-only version of
+     * the current accessor.
+     *
+     * @returns  a reduced_row_major major range which is read-only.
+     */
+    GKO_ATTRIBUTES GKO_INLINE constexpr range<const_accessor> to_const() const
     {
-        return {storage_, size_, stride_[0], stride_[1]};
+        return range<const_accessor>{storage_, size_, stride_[0], stride_[1]};
     }
 
-    // Functions required by the `range` interface:
+    /**
+     * Returns the length in dimension `dimension`.
+     *
+     * @param dimension  a dimension index
+     *
+     * @returns  length in dimension `dimension`
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr dim<dimensionality> length(
         size_type dimension) const
     {
         return dimension < dimensionality ? size_[dimension] : 1;
     }
+
     /**
+     * Copies data from another accessor
+     *
      * @warning Do not use this function since it is not optimized for a
-     *          specific executor. It will always be performed sequentially.
+     * specific executor. It will always be performed sequentially.
+     *
+     * @tparam OtherAccessor  type of the other accessor
+     *
+     * @param other  other accessor
      */
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other)
     {
-        // TODO figure out how to do this properly in an const accessor...
-        //      Additionally, this will always be inefficient!
         for (size_type i = 0; i < size_[0]; ++i) {
             for (size_type j = 0; j < size_[1]; ++j) {
                 for (size_type k = 0; k < size_[2]; ++k) {
@@ -621,6 +668,18 @@ class ReducedStorage3d {
             }
         }
     }
+
+    /**
+     * Returns the stored value for the given indices. If the storage is const,
+     * a value is returned, otherwise, a reference is returned.
+     *
+     * @param x  x index
+     * @param y  y index
+     * @param z  z index
+     *
+     * @returns  the stored value if the accessor is const (if the storage type
+     * is const), or a reference if the accessor is non-const
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr std::conditional_t<
         is_const, arithmetic_type, reference>
     operator()(size_type x, size_type y, size_type z) const
@@ -628,7 +687,16 @@ class ReducedStorage3d {
         return reference{this->storage_ + compute_index(x, y, z)};
     }
 
-    GKO_ATTRIBUTES constexpr range<ReducedStorage3d> operator()(
+    /**
+     * Returns a sub-range spinning the range (x_span, y_span, z_span)
+     *
+     * @param x  span for the x indices
+     * @param y  span for the y indices
+     * @param z  span for the z indices
+     *
+     * @returns a sub-range for the given spans.
+     */
+    GKO_ATTRIBUTES constexpr range<reduced_row_major> operator()(
         const span &x_span, const span &y_span, const span &z_span) const
     {
         return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
@@ -636,7 +704,7 @@ class ReducedStorage3d {
                GKO_ASSERT(x_span <= span{size_[0]}),
                GKO_ASSERT(y_span <= span{size_[1]}),
                GKO_ASSERT(z_span <= span{size_[2]}),
-               range<ReducedStorage3d>(
+               range<reduced_row_major>(
                    storage_ +
                        compute_index(x_span.begin, y_span.begin, z_span.begin),
                    dim<dimensionality>{x_span.end - x_span.begin,
@@ -645,24 +713,51 @@ class ReducedStorage3d {
                    stride_[0], stride_[1]);
     }
 
+    /**
+     * Returns the size of the accessor
+     *
+     * @returns the size of the accessor
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr dim<dimensionality> get_size() const
     {
         return size_;
     }
+
+    /**
+     * Returns the stride for the first index
+     *
+     * @returns the stride for the first index
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr size_type get_stride0() const
     {
         return stride_[0];
     }
+
+    /**
+     * Returns the stride for the second index
+     *
+     * @returns the stride for the second index
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr size_type get_stride1() const
     {
         return stride_[1];
     }
 
+    /**
+     * Returns the pointer to the storage data
+     *
+     * @returns the pointer to the storage data
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr storage_type *get_storage() const
     {
         return storage_;
     }
 
+    /**
+     * Returns a const pointer to the storage data
+     *
+     * @returns a const pointer to the storage data
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr const storage_type *get_const_storage()
         const
     {
@@ -687,25 +782,33 @@ namespace detail {
 
 
 // In case of a const type, do not provide a write function
-template <typename Accessor, typename ScaleType,
+template <int Dimensionality, typename Accessor, typename ScaleType,
           bool = std::is_const<ScaleType>::value>
-struct enable_scale_write {
+struct enable_write_scale {
     static_assert(std::is_const<ScaleType>::value,
                   "This class must have a constant ScaleType!");
     using scale_type = ScaleType;
 };
 
 // In case of a non-const type, enable the write function
-template <typename Accessor, typename ScaleType>
-struct enable_scale_write<Accessor, ScaleType, false> {
+template <int Dimensionality, typename Accessor, typename ScaleType>
+struct enable_write_scale<Dimensionality, Accessor, ScaleType, false> {
     static_assert(!std::is_const<ScaleType>::value,
                   "This class must NOT have a constant ScaleType!");
+    static_assert(Dimensionality == 3, "Only Dimensionality 3 supported!");
+
     using scale_type = ScaleType;
 
     /**
-     * Reads the scale value at the given indices.
+     * Writes the scale value at the given indices.
+     *
+     * @param x  x index
+     * @param z  z index
+     * @param value  value to write.
+     *
+     * @returns the written value.
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr scale_type set_scale(
+    GKO_ATTRIBUTES GKO_INLINE constexpr scale_type write_scale(
         size_type x, size_type z, scale_type value) const
     {
         scale_type *GKO_RESTRICT rest_scale = self()->scale_;
@@ -724,71 +827,121 @@ struct enable_scale_write<Accessor, ScaleType, false> {
 
 
 /**
- * @internal
+ * The reduced_row_major class allows a storage format that is different from
+ * the arithmetic format (which is returned from the brace operator).
+ * As storage, the storage_type is used.
  *
- * The ScaledReducedStorage3d class hides the underlying storage format and
- * provides a simple interface for accessing a one dimensional storage.
- * Additionally, this accessor posesses a scale array, which is used for
- * each read and write operation to do a proper conversion.
+ * This accessor uses row-major access, meaning neighboring z coordinates are
+ * next to each other in memory, followed by y coordinates and then x
+ * coordinates.
  *
- * This class only manages the accesses, however, and not the memory itself.
+ * @tparam Dimensionality  The number of dimensions managed by this accessor
  *
- * The accessor uses row-major access.
+ * @tparam ArithmeticType  Value type used for arithmetic operations and
+ *                         for in- and output
+ *
+ * @tparam StorageType  Value type used for storing the actual value to memory
+ *
+ * @note  This class only manages the accesses and not the memory itself.
+ * @note  Currently, only Dimensionality = 3 is supported.
  */
-template <typename ArithmeticType, typename StorageType>
-class ScaledReducedStorage3d
-    : public detail::enable_scale_write<
-          ScaledReducedStorage3d<ArithmeticType, StorageType>, ArithmeticType> {
+template <int Dimensionality, typename ArithmeticType, typename StorageType>
+class scaled_reduced_row_major
+    : public detail::enable_write_scale<
+          Dimensionality,
+          scaled_reduced_row_major<Dimensionality, ArithmeticType, StorageType>,
+          ArithmeticType> {
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = StorageType;
-    static constexpr size_type dimensionality{3};
+    static constexpr size_type dimensionality{Dimensionality};
     static constexpr bool is_const{std::is_const<storage_type>::value};
     using scale_type =
         std::conditional_t<is_const, const arithmetic_type, arithmetic_type>;
 
-    // Allow access to both `scale_` and `compute_scale_index()`
-    friend detail::enable_scale_write<ScaledReducedStorage3d, scale_type>;
     using const_accessor =
-        ScaledReducedStorage3d<arithmetic_type, const storage_type>;
+        scaled_reduced_row_major<dimensionality, arithmetic_type,
+                                 const storage_type>;
+
+    static_assert(dimensionality == 3,
+                  "Only Dimensionality == 3 is currently supported");
+
+    // Allow access to both `scale_` and `compute_scale_index()`
+    friend class detail::enable_write_scale<
+        dimensionality, scaled_reduced_row_major, scale_type>;
+    friend class range<scaled_reduced_row_major>;
 
 protected:
     using reference =
-        detail::reference::ScaledReducedStorageReference<arithmetic_type,
-                                                         StorageType>;
+        detail::reference::scaled_reduced_storage<arithmetic_type, StorageType>;
 
 public:
     /**
      * Creates the accessor with an already allocated storage space with a
-     * stride.
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
+     *
+     * @param storage  pointer to the block of memory containing the storage
+     * @param scale  pointer to the block of memory containing the scale values.
+     *               Memory required is size[0] * stride1.
+     * @param size  multidimensional size of the memory
+     * @param stride0  stride used for the x-indices
+     * @param stride1  stride used for the y-indices
      */
-    GKO_ATTRIBUTES constexpr ScaledReducedStorage3d(storage_type *storage,
-                                                    dim<dimensionality> size,
-                                                    size_type stride0,
-                                                    size_type stride1,
-                                                    scale_type *scale)
+    GKO_ATTRIBUTES constexpr scaled_reduced_row_major(storage_type *storage,
+                                                      scale_type *scale,
+                                                      dim<dimensionality> size,
+                                                      size_type stride0,
+                                                      size_type stride1)
         : storage_{storage},
+          scale_{scale},
           size_{size},
-          stride_{stride0, stride1},
-          scale_{scale}
+          stride_{stride0, stride1}
     {}
-    GKO_ATTRIBUTES constexpr ScaledReducedStorage3d(storage_type *storage,
-                                                    dim<dimensionality> size,
-                                                    scale_type *scale)
-        : ScaledReducedStorage3d{storage, size, size[1] * size[2], size[2],
-                                 scale}
+    /**
+     * Creates the accessor with an already allocated storage space with a
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
+     * It is assumed that accesses are without a stride.
+     *
+     * @param storage  pointer to the block of memory containing the storage
+     * @param scale  pointer to the block of memory containing the scale values.
+     *               Memory required is size[0] * size[1].
+     * @param size  multidimensional size of the memory
+     */
+    GKO_ATTRIBUTES constexpr scaled_reduced_row_major(storage_type *storage,
+                                                      scale_type *scale,
+                                                      dim<dimensionality> size)
+        : scaled_reduced_row_major{storage, scale, size, size[1] * size[2],
+                                   size[2]}
     {}
-    GKO_ATTRIBUTES constexpr ScaledReducedStorage3d()
-        : ScaledReducedStorage3d{nullptr, {0, 0, 0}, nullptr}
+
+    /**
+     * Creates an empty accessor (pointing nowhere with an empty size)
+     */
+    GKO_ATTRIBUTES constexpr scaled_reduced_row_major()
+        : scaled_reduced_row_major{nullptr, nullptr, {0, 0, 0}}
     {}
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr const_accessor to_const() const
+    /**
+     * Creates a reduced_row_major range which contains a read-only version of
+     * the current accessor.
+     *
+     * @returns  a reduced_row_major major range which is read-only.
+     */
+    GKO_ATTRIBUTES GKO_INLINE constexpr range<const_accessor> to_const() const
     {
-        return {storage_, size_, stride_[0], stride_[1], scale_};
+        return range<const_accessor>{storage_, scale_, size_, stride_[0],
+                                     stride_[1]};
     }
 
     /**
      * Reads the scale value at the given indices.
+     *
+     * @param x  x index
+     * @param z  z index
+     *
+     * @returns the scale value at the given indices.
      */
     GKO_ATTRIBUTES GKO_INLINE constexpr scale_type read_scale(size_type x,
                                                               size_type z) const
@@ -797,7 +950,13 @@ class ScaledReducedStorage3d
         return rest_scale[compute_scale_index(x, z)];
     }
 
-    // Functions required by the `range` interface:
+    /**
+     * Returns the length in dimension `dimension`.
+     *
+     * @param dimension  a dimension index
+     *
+     * @returns length in dimension `dimension`
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr dim<dimensionality> length(
         size_type dimension) const
     {
@@ -805,14 +964,18 @@ class ScaledReducedStorage3d
     }
 
     /**
+     * Copies data from another accessor
+     *
      * @warning Do not use this function since it is not optimized for a
      * specific executor. It will always be performed sequentially.
+     *
+     * @tparam OtherAccessor  type of the other accessor
+     *
+     * @param other  other accessor
      */
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other)
     {
-        // TODO figure out how to do this properly in an const accessor...
-        //      Additionally, this will always be inefficient!
         for (size_type i = 0; i < this->size_[0]; ++i) {
             for (size_type j = 0; j < this->size_[1]; ++j) {
                 for (size_type k = 0; k < this->size_[2]; ++k) {
@@ -822,10 +985,22 @@ class ScaledReducedStorage3d
         }
         for (size_type i = 0; i < this->size_[0]; ++i) {
             for (size_type k = 0; k < this->size_[2]; ++k) {
-                this->set_scale(i, k, other.read_scale(i, k));
+                this->write_scale(i, k, other.read_scale(i, k));
             }
         }
     }
+
+    /**
+     * Returns the stored value for the given indices. If the storage is const,
+     * a value is returned, otherwise, a reference is returned.
+     *
+     * @param x  x index
+     * @param y  y index
+     * @param z  z index
+     *
+     * @returns  the stored value if the accessor is const (if the storage type
+     * is const), or a reference if the accessor is non-const
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr std::conditional_t<
         is_const, arithmetic_type, reference>
     operator()(size_type x, size_type y, size_type z) const
@@ -834,7 +1009,16 @@ class ScaledReducedStorage3d
                          this->read_scale(x, z)};
     }
 
-    GKO_ATTRIBUTES constexpr range<ScaledReducedStorage3d> operator()(
+    /**
+     * Returns a sub-range spinning the range (x_span, y_span, z_span)
+     *
+     * @param x  span for the x indices
+     * @param y  span for the y indices
+     * @param z  span for the z indices
+     *
+     * @returns a sub-range for the given spans.
+     */
+    GKO_ATTRIBUTES constexpr range<scaled_reduced_row_major> operator()(
         const span &x_span, const span &y_span, const span &z_span) const
     {
         return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
@@ -842,45 +1026,84 @@ class ScaledReducedStorage3d
                GKO_ASSERT(x_span <= span{this->size_[0]}),
                GKO_ASSERT(y_span <= span{this->size_[1]}),
                GKO_ASSERT(z_span <= span{this->size_[2]}),
-               range<ScaledReducedStorage3d>(
+               range<scaled_reduced_row_major>(
                    this->storage_ + this->compute_index(x_span.begin,
                                                         y_span.begin,
                                                         z_span.begin),
+                   this->scale_ +
+                       this->compute_scale_index(x_span.begin, z_span.begin),
                    dim<dimensionality>{x_span.end - x_span.begin,
                                        y_span.end - y_span.begin,
                                        z_span.end - z_span.begin},
-                   this->stride_[0], this->stride_[1], this->scale_);
+                   this->stride_[0], this->stride_[1]);
     }
 
+    /**
+     * Returns the size of the accessor
+     *
+     * @returns the size of the accessor
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr dim<dimensionality> get_size() const
     {
         return size_;
     }
+
+    /**
+     * Returns the stride for the first index
+     *
+     * @returns the stride for the first index
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr size_type get_stride0() const
     {
         return stride_[0];
     }
+
+    /**
+     * Returns the stride for the second index
+     *
+     * @returns the stride for the second index
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr size_type get_stride1() const
     {
         return stride_[1];
     }
 
+    /**
+     * Returns the pointer to the storage data
+     *
+     * @returns the pointer to the storage data
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr storage_type *get_storage() const
     {
         return storage_;
     }
 
+    /**
+     * Returns a const pointer to the storage data
+     *
+     * @returns a const pointer to the storage data
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr const storage_type *get_const_storage()
         const
     {
         return storage_;
     }
 
+    /**
+     * Returns the pointer to the scale data
+     *
+     * @returns the pointer to the scale data
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr scale_type *get_scale() const
     {
         return this->scale_;
     }
 
+    /**
+     * Returns a const pointer to the scale data
+     *
+     * @returns a const pointer to the scale data
+     */
     GKO_ATTRIBUTES GKO_INLINE constexpr const scale_type *get_const_scale()
         const
     {
@@ -903,9 +1126,10 @@ class ScaledReducedStorage3d
     }
 
     storage_type *storage_;
-    dim<dimensionality> size_;
-    size_type stride_[2];  // std::array leads to conflicts on devices
     scale_type *scale_;
+    dim<dimensionality> size_;
+    // std::array leads to conflicts on devices
+    size_type stride_[dimensionality - 1];
 };
 
 

From f78a288b73ff0bad02c5812239eb5cf425135fa6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 18 Sep 2020 02:32:28 +0200
Subject: [PATCH 0203/1008] Make reduce_storage test more detailed

---
 core/test/base/range_accessors.cpp | 137 ++++++++++++++++-------------
 include/ginkgo/core/base/range.hpp |   7 +-
 2 files changed, 79 insertions(+), 65 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index c57a082d1cf..1876f5de3eb 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -36,7 +36,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
-#include <iostream>
+#include <tuple>
+
+
+#include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/types.hpp>
 
 
 namespace {
@@ -78,15 +82,6 @@ TEST_F(RowMajorAccessor, CanWriteData)
 }
 
 
-TEST_F(RowMajorAccessor, CanWriteDataToConst)
-{
-    const row_major_int_range cr = r;
-    cr(0, 0) = 4;
-
-    EXPECT_EQ(cr(0, 0), 4);
-}
-
-
 TEST_F(RowMajorAccessor, CanCreateSubrange)
 {
     auto subr = r(span{1, 3}, span{0, 2});
@@ -155,37 +150,68 @@ class ReducedStorage3d : public ::testing::Test {
     using const_reduced_storage = gko::range<const_accessor>;
 
     // clang-format off
-    st_type data[8]{
-        1.0, 2.1,
-        -1.2, 3.3,
-        4.4, -2.5,
-        5.6, 6.7
+    st_type data[4 * 3 * 2]{
+        // 0, y, z
+        1.0, 2.01,
+        -1.02, 3.03,
+        4.04, -2.05,
+        // 1, y, z
+        5.06, 6.07,
+        2.08, 3.09,
+        -1.1, -9.11,
+        // 2, y, z
+        -2.12, 2.13,
+        0.14, 15.15,
+        -9.16, 8.17,
+        // 3, y, z
+        7.18, -6.19,
+        5.2, -4.21,
+        3.22, -2.23
     };
     // clang-format on
-    reduced_storage r{data, gko::dim<3>{2u, 2u, 2u}};
-    const_reduced_storage cr{data, gko::dim<3>{2u, 2u, 2u}};
+    reduced_storage r{data, gko::dim<3>{4u, 3u, 2u}};
+    const_reduced_storage cr{data, gko::dim<3>{4u, 3u, 2u}};
+
+    template <typename Accessor>
+    static void check_accessor_correctness(const Accessor &a,
+                                           std::tuple<int, int, int> ignore = {
+                                               99, 99, 99})
+    {
+        using t = std::tuple<int, int, int>;
+        // clang-format off
+        if (ignore != t{0, 0, 0}) { EXPECT_EQ(a(0, 0, 0), 1.0);     }
+        if (ignore != t{0, 0, 1}) { EXPECT_EQ(a(0, 0, 1), 2.01);    }
+        if (ignore != t{0, 1, 0}) { EXPECT_EQ(a(0, 1, 0), -1.02);   }
+        if (ignore != t{0, 1, 1}) { EXPECT_EQ(a(0, 1, 1), 3.03);    }
+        if (ignore != t{0, 2, 0}) { EXPECT_EQ(a(0, 2, 0), 4.04);    }
+        if (ignore != t{0, 2, 1}) { EXPECT_EQ(a(0, 2, 1), -2.05);   }
+        if (ignore != t{1, 0, 0}) { EXPECT_EQ(a(1, 0, 0), 5.06);    }
+        if (ignore != t{1, 0, 1}) { EXPECT_EQ(a(1, 0, 1), 6.07);    }
+        if (ignore != t{1, 1, 0}) { EXPECT_EQ(a(1, 1, 0), 2.08);    }
+        if (ignore != t{1, 1, 1}) { EXPECT_EQ(a(1, 1, 1), 3.09);    }
+        if (ignore != t{1, 2, 0}) { EXPECT_EQ(a(1, 2, 0), -1.1);    }
+        if (ignore != t{1, 2, 1}) { EXPECT_EQ(a(1, 2, 1), -9.11);   }
+        if (ignore != t{2, 0, 0}) { EXPECT_EQ(a(2, 0, 0), -2.12);   }
+        if (ignore != t{2, 0, 1}) { EXPECT_EQ(a(2, 0, 1), 2.13);    }
+        if (ignore != t{2, 1, 0}) { EXPECT_EQ(a(2, 1, 0), 0.14);    }
+        if (ignore != t{2, 1, 1}) { EXPECT_EQ(a(2, 1, 1), 15.15);   }
+        if (ignore != t{2, 2, 0}) { EXPECT_EQ(a(2, 2, 0), -9.16);   }
+        if (ignore != t{2, 2, 1}) { EXPECT_EQ(a(2, 2, 1), 8.17);    }
+        if (ignore != t{3, 0, 0}) { EXPECT_EQ(a(3, 0, 0), 7.18);    }
+        if (ignore != t{3, 0, 1}) { EXPECT_EQ(a(3, 0, 1), -6.19);   }
+        if (ignore != t{3, 1, 0}) { EXPECT_EQ(a(3, 1, 0), 5.2);     }
+        if (ignore != t{3, 1, 1}) { EXPECT_EQ(a(3, 1, 1), -4.21);   }
+        if (ignore != t{3, 2, 0}) { EXPECT_EQ(a(3, 2, 0), 3.22);    }
+        if (ignore != t{3, 2, 1}) { EXPECT_EQ(a(3, 2, 1), -2.23);   }
+        // clang-format on
+    }
 };
 
 
 TEST_F(ReducedStorage3d, CanReadData)
 {
-    EXPECT_EQ(r(0, 0, 0), 1.0);
-    EXPECT_EQ(r(0, 0, 1), 2.1);
-    EXPECT_EQ(r(0, 1, 0), -1.2);
-    EXPECT_EQ(r(0, 1, 1), 3.3);
-    EXPECT_EQ(r(1, 0, 0), 4.4);
-    EXPECT_EQ(r(1, 0, 1), -2.5);
-    EXPECT_EQ(r(1, 1, 0), 5.6);
-    EXPECT_EQ(r(1, 1, 1), 6.7);
-    // Const should read the exact same values!
-    EXPECT_EQ(cr(0, 0, 0), 1.0);
-    EXPECT_EQ(cr(0, 0, 1), 2.1);
-    EXPECT_EQ(cr(0, 1, 0), -1.2);
-    EXPECT_EQ(cr(0, 1, 1), 3.3);
-    EXPECT_EQ(cr(1, 0, 0), 4.4);
-    EXPECT_EQ(cr(1, 0, 1), -2.5);
-    EXPECT_EQ(cr(1, 1, 0), 5.6);
-    EXPECT_EQ(cr(1, 1, 1), 6.7);
+    check_accessor_correctness(r);
+    check_accessor_correctness(cr);
 }
 
 
@@ -193,53 +219,40 @@ TEST_F(ReducedStorage3d, ToConstWorking)
 {
     auto cr2 = r->to_const();
 
-    // static_assert(std::is_same<decltype(cr2), const_reduced_storage>::value,
-    // "Types must be equal!");
-    EXPECT_EQ(cr2(0, 0, 0), 1.0);
-    EXPECT_EQ(cr2(0, 0, 1), 2.1);
-    EXPECT_EQ(cr2(0, 1, 0), -1.2);
-    EXPECT_EQ(cr2(0, 1, 1), 3.3);
-    EXPECT_EQ(cr2(1, 0, 0), 4.4);
-    EXPECT_EQ(cr2(1, 0, 1), -2.5);
-    EXPECT_EQ(cr2(1, 1, 0), 5.6);
-    EXPECT_EQ(cr2(1, 1, 1), 6.7);
+    static_assert(std::is_same<decltype(cr2), const_reduced_storage>::value,
+                  "Types must be equal!");
+    check_accessor_correctness(cr2);
 }
 
 
 TEST_F(ReducedStorage3d, CanWriteData)
 {
-    r(0, 0, 0) = 2.0;
     r(0, 1, 0) = 100.0;
 
-    EXPECT_EQ(r(0, 0, 0), 2.0);
+    check_accessor_correctness(r, {0, 1, 0});
     EXPECT_EQ(r(0, 1, 0), 100.0);
 }
 
 
 TEST_F(ReducedStorage3d, CanCreateSubrange)
 {
-    auto subr = r(span{1, 2}, span{0, 2}, span{0, 1});
+    auto subr = r(span{1, 3}, span{0, 2}, 0);
 
-    EXPECT_EQ(subr(0, 0, 0), 4.4);
-    EXPECT_EQ(subr(0, 1, 0), 5.6);
+    EXPECT_EQ(subr(0, 0, 0), 5.06);
+    EXPECT_EQ(subr(0, 1, 0), 2.08);
+    EXPECT_EQ(subr(1, 0, 0), -2.12);
+    EXPECT_EQ(subr(1, 1, 0), 0.14);
 }
 
 
-TEST_F(ReducedStorage3d, CanCreateRowVector)
+TEST_F(ReducedStorage3d, CanCreateSubrange2)
 {
-    auto subr = r(0, 0, span{0, 2});
+    auto subr = r(span{1, 3}, span{0, 2}, span{0, 1});
 
-    EXPECT_EQ(subr(0, 0, 0), 1.0);
-    EXPECT_EQ(subr(0, 0, 1), 2.1);
-}
-
-
-TEST_F(ReducedStorage3d, CanCreateColumnVector)
-{
-    auto subr = r(span{0, 2}, 0, 0);
-
-    EXPECT_EQ(subr(0, 0, 0), 1.0);
-    EXPECT_EQ(subr(1, 0, 0), 4.4);
+    EXPECT_EQ(subr(0, 0, 0), 5.06);
+    EXPECT_EQ(subr(0, 1, 0), 2.08);
+    EXPECT_EQ(subr(1, 0, 0), -2.12);
+    EXPECT_EQ(subr(1, 1, 0), 0.14);
 }
 
 
diff --git a/include/ginkgo/core/base/range.hpp b/include/ginkgo/core/base/range.hpp
index b426f2729bd..c08e1f2f305 100644
--- a/include/ginkgo/core/base/range.hpp
+++ b/include/ginkgo/core/base/range.hpp
@@ -338,8 +338,8 @@ class range {
      * @return a value on position `(dimensions...)`.
      */
     template <typename... DimensionTypes>
-    GKO_ATTRIBUTES auto operator()(DimensionTypes &&... dimensions) const
-        -> decltype(std::declval<accessor>()(
+    GKO_ATTRIBUTES constexpr auto operator()(DimensionTypes &&... dimensions)
+        const -> decltype(std::declval<accessor>()(
             std::forward<DimensionTypes>(dimensions)...))
     {
         static_assert(sizeof...(dimensions) <= dimensionality,
@@ -469,7 +469,8 @@ struct implement_unary_operation {
 
 template <operation_kind Kind, typename FirstOperand, typename SecondOperand,
           typename Operation>
-struct implement_binary_operation {};
+struct implement_binary_operation {
+};
 
 template <typename FirstAccessor, typename SecondAccessor, typename Operation>
 struct implement_binary_operation<operation_kind::range_by_range, FirstAccessor,

From 2c674f38dbb624c6162f79b3e80d516563ac5f25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 21 Sep 2020 19:34:57 +0200
Subject: [PATCH 0204/1008] Add more range tests

---
 core/test/base/range_accessors.cpp | 310 ++++++++++++++++++++++++++---
 1 file changed, 280 insertions(+), 30 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 1876f5de3eb..f6856d0ecff 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -43,6 +43,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
+#include "core/test/utils.hpp"
+
+
 namespace {
 
 
@@ -141,6 +144,7 @@ class ReducedStorage3d : public ::testing::Test {
     using span = gko::span;
     using ar_type = double;
     using st_type = double;
+    static constexpr ar_type delta{::r<st_type>::value};
 
     using accessor = gko::accessor::reduced_row_major<3, ar_type, st_type>;
     using const_accessor =
@@ -173,10 +177,12 @@ class ReducedStorage3d : public ::testing::Test {
     const_reduced_storage cr{data, gko::dim<3>{4u, 3u, 2u}};
 
     template <typename Accessor>
-    static void check_accessor_correctness(const Accessor &a,
-                                           std::tuple<int, int, int> ignore = {
-                                               99, 99, 99})
+    static void check_accessor_correctness(
+        const Accessor &a,
+        std::tuple<int, int, int> ignore = std::tuple<int, int, int>(99, 99,
+                                                                     99))
     {
+        // Test for equality is fine here since they should not be modified
         using t = std::tuple<int, int, int>;
         // clang-format off
         if (ignore != t{0, 0, 0}) { EXPECT_EQ(a(0, 0, 0), 1.0);     }
@@ -227,10 +233,114 @@ TEST_F(ReducedStorage3d, ToConstWorking)
 
 TEST_F(ReducedStorage3d, CanWriteData)
 {
-    r(0, 1, 0) = 100.0;
+    r(0, 1, 0) = 100.2;
 
     check_accessor_correctness(r, {0, 1, 0});
-    EXPECT_EQ(r(0, 1, 0), 100.0);
+    EXPECT_EQ(r(0, 1, 0), 100.2);
+}
+
+
+TEST_F(ReducedStorage3d, Addition)
+{
+    const ar_type expected = 10.2 + 2.01;
+
+    auto result = r(0, 0, 1) + 10.2;
+    r(0, 0, 1) += 10.2;
+
+    check_accessor_correctness(r, {0, 0, 1});
+    EXPECT_NEAR(r(0, 0, 1), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
+}
+
+
+TEST_F(ReducedStorage3d, Addition2)
+{
+    const ar_type expected = 2.01 + -1.02;
+
+    auto result = r(0, 0, 1) + r(0, 1, 0);
+    r(0, 0, 1) += r(0, 1, 0);
+
+    check_accessor_correctness(r, {0, 0, 1});
+    EXPECT_NEAR(r(0, 0, 1), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
+}
+
+
+TEST_F(ReducedStorage3d, Subtraction)
+{
+    const ar_type expected = -2.23 - 1;
+
+    auto result = r(3, 2, 1) - 1.;
+    r(3, 2, 1) -= 1;
+
+    check_accessor_correctness(r, {3, 2, 1});
+    EXPECT_NEAR(r(3, 2, 1), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
+}
+
+
+TEST_F(ReducedStorage3d, Subtraction2)
+{
+    const ar_type expected = 3.22 - -2.23;
+
+    auto result = cr(3, 2, 0) - r(3, 2, 1);
+    r(3, 2, 0) -= r(3, 2, 1);
+
+    check_accessor_correctness(r, {3, 2, 0});
+    EXPECT_NEAR(r(3, 2, 0), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
+}
+
+
+TEST_F(ReducedStorage3d, Multiplication)
+{
+    const ar_type expected = 1 * 2;
+
+    auto result = r(0, 0, 0) * 2.;
+    r(0, 0, 0) *= 2;
+
+    check_accessor_correctness(r, {0, 0, 0});
+    EXPECT_NEAR(r(0, 0, 0), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
+}
+
+
+TEST_F(ReducedStorage3d, Multiplication2)
+{
+    const ar_type expected = 2.01 * 3.03;
+
+    auto result = r(0, 0, 1) * cr(0, 1, 1);
+    r(0, 0, 1) *= r(0, 1, 1);
+
+    check_accessor_correctness(r, {0, 0, 1});
+    EXPECT_NEAR(r(0, 0, 1), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
+}
+
+
+TEST_F(ReducedStorage3d, Division)
+{
+    const ar_type expected = 2.01 / 2.0;
+
+    auto result = cr(0, 0, 1) / 2.;
+    r(0, 0, 1) /= 2.;
+
+    check_accessor_correctness(r, {0, 0, 1});
+    EXPECT_NEAR(r(0, 0, 1), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
+}
+
+
+TEST_F(ReducedStorage3d, Division2)
+{
+    const ar_type expected = 5.06 / 4.04;
+
+    auto result = r(1, 0, 0) / cr(0, 2, 0);
+    r(1, 0, 0) /= r(0, 2, 0);
+
+    check_accessor_correctness(r, {1, 0, 0});
+    EXPECT_NEAR(r(1, 0, 0), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
 }
 
 
@@ -247,7 +357,7 @@ TEST_F(ReducedStorage3d, CanCreateSubrange)
 
 TEST_F(ReducedStorage3d, CanCreateSubrange2)
 {
-    auto subr = r(span{1, 3}, span{0, 2}, span{0, 1});
+    auto subr = cr(span{1, 3}, span{0, 2}, span{0, 1});
 
     EXPECT_EQ(subr(0, 0, 0), 5.06);
     EXPECT_EQ(subr(0, 1, 0), 2.08);
@@ -262,6 +372,8 @@ class ScaledReducedStorage3d : public ::testing::Test {
     using ar_type = double;
     using st_type = gko::int32;
 
+    static constexpr ar_type delta{::r<ar_type>::value};
+
     using accessor =
         gko::accessor::scaled_reduced_row_major<3, ar_type, st_type>;
     using const_accessor =
@@ -272,40 +384,178 @@ class ScaledReducedStorage3d : public ::testing::Test {
 
     // clang-format off
     st_type data[8]{
-        1, 2,
-        -3, 4,
-        55, 6,
-        -777, 8
+        10, 11,
+        -12, 13,
+        14, -115,
+        6, 77
     };
-    ar_type scale[4]{
-        1., 1.,
-        1., 1.
+    ar_type scale[2]{
+        1., 2.,
     };
     // clang-format on
-    reduced_storage r{data, scale, gko::dim<3>{2u, 2u, 2u}};
-    const_reduced_storage cr{data, scale, gko::dim<3>{2u, 2u, 2u}};
+    reduced_storage r{data, scale, gko::dim<3>{1u, 4u, 2u}};
+    const_reduced_storage cr{data, scale, gko::dim<3>{1u, 4u, 2u}};
+
+    template <typename Accessor>
+    static void check_accessor_correctness(
+        const Accessor &a,
+        std::tuple<int, int, int> ignore = std::tuple<int, int, int>(99, 99,
+                                                                     99))
+    {
+        // Test for equality is fine here since they should not be modified
+        using t = std::tuple<int, int, int>;
+        // clang-format off
+        if (ignore != t{0, 0, 0}) { EXPECT_EQ(a(0, 0, 0), 10.);   }
+        if (ignore != t{0, 0, 1}) { EXPECT_EQ(a(0, 0, 1), 22.);   }
+        if (ignore != t{0, 1, 0}) { EXPECT_EQ(a(0, 1, 0), -12.);  }
+        if (ignore != t{0, 1, 1}) { EXPECT_EQ(a(0, 1, 1), 26.);   }
+        if (ignore != t{0, 2, 0}) { EXPECT_EQ(a(0, 2, 0), 14.);   }
+        if (ignore != t{0, 2, 1}) { EXPECT_EQ(a(0, 2, 1), -230.); }
+        if (ignore != t{0, 3, 0}) { EXPECT_EQ(a(0, 3, 0), 6.);    }
+        if (ignore != t{0, 3, 1}) { EXPECT_EQ(a(0, 3, 1), 154.);  }
+        // clang-format on
+    }
 };
 
 
-TEST_F(ScaledReducedStorage3d, CanUseConst)
+TEST_F(ScaledReducedStorage3d, CanRead)
 {
-    EXPECT_EQ(cr(0, 0, 0), 1.);
-    EXPECT_EQ(cr(0, 0, 1), 2.);
-    EXPECT_EQ(cr(0, 1, 0), -3.);
-    EXPECT_EQ(cr(0, 1, 1), 4.);
-    EXPECT_EQ(cr(1, 0, 0), 55.);
-    EXPECT_EQ(cr(1, 0, 1), 6.);
-    EXPECT_EQ(cr(1, 1, 0), -777.);
-    EXPECT_EQ(cr(1, 1, 1), 8.);
+    check_accessor_correctness(cr);
+    check_accessor_correctness(r);
+}
+
+TEST_F(ScaledReducedStorage3d, Subrange)
+{
+    auto subr = cr(0, gko::span{0, 2}, 1);
+
+    EXPECT_EQ(subr(0, 0, 0), 22.);
+    EXPECT_EQ(subr(0, 1, 0), 26.);
+}
+
+
+TEST_F(ScaledReducedStorage3d, CanWriteScale)
+{
+    r->write_scale(0, 0, 10.);
+
+    EXPECT_EQ(r(0, 0, 0), 100.);
+    EXPECT_EQ(r(0, 0, 1), 22.);
+    EXPECT_EQ(r(0, 1, 0), -120.);
+    EXPECT_EQ(r(0, 1, 1), 26.);
+    EXPECT_EQ(r(0, 2, 0), 140.);
+    EXPECT_EQ(r(0, 2, 1), -230.);
+    EXPECT_EQ(r(0, 3, 0), 60.);
+    EXPECT_EQ(r(0, 3, 1), 154.);
+}
+
+
+TEST_F(ScaledReducedStorage3d, CanReadScale)
+{
+    EXPECT_EQ(r->read_scale(0, 0), 1.);
+    EXPECT_EQ(r->read_scale(0, 1), 2.);
+}
+
+
+TEST_F(ScaledReducedStorage3d, Addition)
+{
+    const ar_type expected = 10. + 3.;
+
+    const auto result = cr(0, 0, 0) + 3.;
+    r(0, 0, 0) += 3.;
+
+    check_accessor_correctness(r, {0, 0, 0});
+    EXPECT_NEAR(r(0, 0, 0), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
+}
+
+
+TEST_F(ScaledReducedStorage3d, Addition2)
+{
+    const ar_type expected = 10. + 22.;
+
+    const auto result = cr(0, 0, 0) + r(0, 0, 1);
+    r(0, 0, 0) += cr(0, 0, 1);
 
-    auto subr = cr(span{0, 2}, 0, 0);
+    check_accessor_correctness(r, {0, 0, 0});
+    EXPECT_NEAR(r(0, 0, 0), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
+}
+
+
+TEST_F(ScaledReducedStorage3d, Subtraction)
+{
+    const ar_type expected = 22. - 2.;
+
+    const auto result = cr(0, 0, 1) - 2.;
+    r(0, 0, 1) -= 2.;
+
+    check_accessor_correctness(r, {0, 0, 1});
+    EXPECT_NEAR(r(0, 0, 1), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
+}
 
-    EXPECT_EQ(subr(0, 0, 0), 1.0);
-    EXPECT_EQ(subr(1, 0, 0), 55.);
 
-    // cr(0, 0, 0) = 2.0;
-    r->write_scale(0, 0, 2.);
-    EXPECT_EQ(r(0, 0, 0), 2.);
+TEST_F(ScaledReducedStorage3d, Subtraction2)
+{
+    const ar_type expected = -12. - 26.;
+
+    const auto result = cr(0, 1, 0) - r(0, 1, 1);
+    r(0, 1, 0) -= r(0, 1, 1);
+
+    check_accessor_correctness(r, {0, 1, 0});
+    EXPECT_NEAR(r(0, 1, 0), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
+}
+
+
+TEST_F(ScaledReducedStorage3d, Multiplication)
+{
+    const ar_type expected = 26. * 3.;
+
+    const auto result = cr(0, 1, 1) * 3.;
+    r(0, 1, 1) *= 3.;
+
+    check_accessor_correctness(r, {0, 1, 1});
+    EXPECT_NEAR(r(0, 1, 1), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
+}
+
+
+TEST_F(ScaledReducedStorage3d, Multiplication2)
+{
+    const ar_type expected = 14. * 10.;
+
+    const auto result = r(0, 2, 0) * r(0, 0, 0);
+    r(0, 2, 0) *= r(0, 0, 0);
+
+    check_accessor_correctness(r, {0, 2, 0});
+    EXPECT_NEAR(r(0, 2, 0), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
+}
+
+
+TEST_F(ScaledReducedStorage3d, Division)
+{
+    const ar_type expected = 10. / 2.;
+
+    const auto result = cr(0, 0, 0) / 2.;
+    r(0, 0, 0) /= 2.;
+
+    check_accessor_correctness(r, {0, 0, 0});
+    EXPECT_NEAR(r(0, 0, 0), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
+}
+
+
+TEST_F(ScaledReducedStorage3d, Division2)
+{
+    const ar_type expected = -12. / 6.;
+
+    const auto result = r(0, 1, 0) / r(0, 3, 0);
+    r(0, 1, 0) /= r(0, 3, 0);
+
+    check_accessor_correctness(r, {0, 1, 0});
+    EXPECT_NEAR(r(0, 1, 0), expected, delta);
+    EXPECT_NEAR(result, expected, delta);
 }
 
 

From c6321103c2dddaeaf528bb1ad96af7ecaef524c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 22 Sep 2020 10:25:12 +0200
Subject: [PATCH 0205/1008] Update std::tuple initialization in range test

---
 core/test/base/range_accessors.cpp | 109 +++++++++++++++--------------
 1 file changed, 55 insertions(+), 54 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index f6856d0ecff..045b2f49e50 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -146,6 +146,9 @@ class ReducedStorage3d : public ::testing::Test {
     using st_type = double;
     static constexpr ar_type delta{::r<st_type>::value};
 
+    // Type for `check_accessor_correctness` to forward the indices
+    using t = std::tuple<int, int, int>;
+
     using accessor = gko::accessor::reduced_row_major<3, ar_type, st_type>;
     using const_accessor =
         gko::accessor::reduced_row_major<3, ar_type, const st_type>;
@@ -178,37 +181,34 @@ class ReducedStorage3d : public ::testing::Test {
 
     template <typename Accessor>
     static void check_accessor_correctness(
-        const Accessor &a,
-        std::tuple<int, int, int> ignore = std::tuple<int, int, int>(99, 99,
-                                                                     99))
+        const Accessor &a, std::tuple<int, int, int> ignore = t(99, 99, 99))
     {
         // Test for equality is fine here since they should not be modified
-        using t = std::tuple<int, int, int>;
         // clang-format off
-        if (ignore != t{0, 0, 0}) { EXPECT_EQ(a(0, 0, 0), 1.0);     }
-        if (ignore != t{0, 0, 1}) { EXPECT_EQ(a(0, 0, 1), 2.01);    }
-        if (ignore != t{0, 1, 0}) { EXPECT_EQ(a(0, 1, 0), -1.02);   }
-        if (ignore != t{0, 1, 1}) { EXPECT_EQ(a(0, 1, 1), 3.03);    }
-        if (ignore != t{0, 2, 0}) { EXPECT_EQ(a(0, 2, 0), 4.04);    }
-        if (ignore != t{0, 2, 1}) { EXPECT_EQ(a(0, 2, 1), -2.05);   }
-        if (ignore != t{1, 0, 0}) { EXPECT_EQ(a(1, 0, 0), 5.06);    }
-        if (ignore != t{1, 0, 1}) { EXPECT_EQ(a(1, 0, 1), 6.07);    }
-        if (ignore != t{1, 1, 0}) { EXPECT_EQ(a(1, 1, 0), 2.08);    }
-        if (ignore != t{1, 1, 1}) { EXPECT_EQ(a(1, 1, 1), 3.09);    }
-        if (ignore != t{1, 2, 0}) { EXPECT_EQ(a(1, 2, 0), -1.1);    }
-        if (ignore != t{1, 2, 1}) { EXPECT_EQ(a(1, 2, 1), -9.11);   }
-        if (ignore != t{2, 0, 0}) { EXPECT_EQ(a(2, 0, 0), -2.12);   }
-        if (ignore != t{2, 0, 1}) { EXPECT_EQ(a(2, 0, 1), 2.13);    }
-        if (ignore != t{2, 1, 0}) { EXPECT_EQ(a(2, 1, 0), 0.14);    }
-        if (ignore != t{2, 1, 1}) { EXPECT_EQ(a(2, 1, 1), 15.15);   }
-        if (ignore != t{2, 2, 0}) { EXPECT_EQ(a(2, 2, 0), -9.16);   }
-        if (ignore != t{2, 2, 1}) { EXPECT_EQ(a(2, 2, 1), 8.17);    }
-        if (ignore != t{3, 0, 0}) { EXPECT_EQ(a(3, 0, 0), 7.18);    }
-        if (ignore != t{3, 0, 1}) { EXPECT_EQ(a(3, 0, 1), -6.19);   }
-        if (ignore != t{3, 1, 0}) { EXPECT_EQ(a(3, 1, 0), 5.2);     }
-        if (ignore != t{3, 1, 1}) { EXPECT_EQ(a(3, 1, 1), -4.21);   }
-        if (ignore != t{3, 2, 0}) { EXPECT_EQ(a(3, 2, 0), 3.22);    }
-        if (ignore != t{3, 2, 1}) { EXPECT_EQ(a(3, 2, 1), -2.23);   }
+        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), 1.0);     }
+        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), 2.01);    }
+        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), -1.02);   }
+        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), 3.03);    }
+        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), 4.04);    }
+        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), -2.05);   }
+        if (ignore != t(1, 0, 0)) { EXPECT_EQ(a(1, 0, 0), 5.06);    }
+        if (ignore != t(1, 0, 1)) { EXPECT_EQ(a(1, 0, 1), 6.07);    }
+        if (ignore != t(1, 1, 0)) { EXPECT_EQ(a(1, 1, 0), 2.08);    }
+        if (ignore != t(1, 1, 1)) { EXPECT_EQ(a(1, 1, 1), 3.09);    }
+        if (ignore != t(1, 2, 0)) { EXPECT_EQ(a(1, 2, 0), -1.1);    }
+        if (ignore != t(1, 2, 1)) { EXPECT_EQ(a(1, 2, 1), -9.11);   }
+        if (ignore != t(2, 0, 0)) { EXPECT_EQ(a(2, 0, 0), -2.12);   }
+        if (ignore != t(2, 0, 1)) { EXPECT_EQ(a(2, 0, 1), 2.13);    }
+        if (ignore != t(2, 1, 0)) { EXPECT_EQ(a(2, 1, 0), 0.14);    }
+        if (ignore != t(2, 1, 1)) { EXPECT_EQ(a(2, 1, 1), 15.15);   }
+        if (ignore != t(2, 2, 0)) { EXPECT_EQ(a(2, 2, 0), -9.16);   }
+        if (ignore != t(2, 2, 1)) { EXPECT_EQ(a(2, 2, 1), 8.17);    }
+        if (ignore != t(3, 0, 0)) { EXPECT_EQ(a(3, 0, 0), 7.18);    }
+        if (ignore != t(3, 0, 1)) { EXPECT_EQ(a(3, 0, 1), -6.19);   }
+        if (ignore != t(3, 1, 0)) { EXPECT_EQ(a(3, 1, 0), 5.2);     }
+        if (ignore != t(3, 1, 1)) { EXPECT_EQ(a(3, 1, 1), -4.21);   }
+        if (ignore != t(3, 2, 0)) { EXPECT_EQ(a(3, 2, 0), 3.22);    }
+        if (ignore != t(3, 2, 1)) { EXPECT_EQ(a(3, 2, 1), -2.23);   }
         // clang-format on
     }
 };
@@ -235,7 +235,7 @@ TEST_F(ReducedStorage3d, CanWriteData)
 {
     r(0, 1, 0) = 100.2;
 
-    check_accessor_correctness(r, {0, 1, 0});
+    check_accessor_correctness(r, t(0, 1, 0));
     EXPECT_EQ(r(0, 1, 0), 100.2);
 }
 
@@ -247,7 +247,7 @@ TEST_F(ReducedStorage3d, Addition)
     auto result = r(0, 0, 1) + 10.2;
     r(0, 0, 1) += 10.2;
 
-    check_accessor_correctness(r, {0, 0, 1});
+    check_accessor_correctness(r, t(0, 0, 1));
     EXPECT_NEAR(r(0, 0, 1), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -260,7 +260,7 @@ TEST_F(ReducedStorage3d, Addition2)
     auto result = r(0, 0, 1) + r(0, 1, 0);
     r(0, 0, 1) += r(0, 1, 0);
 
-    check_accessor_correctness(r, {0, 0, 1});
+    check_accessor_correctness(r, t(0, 0, 1));
     EXPECT_NEAR(r(0, 0, 1), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -273,7 +273,7 @@ TEST_F(ReducedStorage3d, Subtraction)
     auto result = r(3, 2, 1) - 1.;
     r(3, 2, 1) -= 1;
 
-    check_accessor_correctness(r, {3, 2, 1});
+    check_accessor_correctness(r, t(3, 2, 1));
     EXPECT_NEAR(r(3, 2, 1), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -286,7 +286,7 @@ TEST_F(ReducedStorage3d, Subtraction2)
     auto result = cr(3, 2, 0) - r(3, 2, 1);
     r(3, 2, 0) -= r(3, 2, 1);
 
-    check_accessor_correctness(r, {3, 2, 0});
+    check_accessor_correctness(r, t(3, 2, 0));
     EXPECT_NEAR(r(3, 2, 0), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -299,7 +299,7 @@ TEST_F(ReducedStorage3d, Multiplication)
     auto result = r(0, 0, 0) * 2.;
     r(0, 0, 0) *= 2;
 
-    check_accessor_correctness(r, {0, 0, 0});
+    check_accessor_correctness(r, t(0, 0, 0));
     EXPECT_NEAR(r(0, 0, 0), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -312,7 +312,7 @@ TEST_F(ReducedStorage3d, Multiplication2)
     auto result = r(0, 0, 1) * cr(0, 1, 1);
     r(0, 0, 1) *= r(0, 1, 1);
 
-    check_accessor_correctness(r, {0, 0, 1});
+    check_accessor_correctness(r, t(0, 0, 1));
     EXPECT_NEAR(r(0, 0, 1), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -325,7 +325,7 @@ TEST_F(ReducedStorage3d, Division)
     auto result = cr(0, 0, 1) / 2.;
     r(0, 0, 1) /= 2.;
 
-    check_accessor_correctness(r, {0, 0, 1});
+    check_accessor_correctness(r, t(0, 0, 1));
     EXPECT_NEAR(r(0, 0, 1), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -338,7 +338,7 @@ TEST_F(ReducedStorage3d, Division2)
     auto result = r(1, 0, 0) / cr(0, 2, 0);
     r(1, 0, 0) /= r(0, 2, 0);
 
-    check_accessor_correctness(r, {1, 0, 0});
+    check_accessor_correctness(r, t(1, 0, 0));
     EXPECT_NEAR(r(1, 0, 0), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -371,6 +371,8 @@ class ScaledReducedStorage3d : public ::testing::Test {
     using span = gko::span;
     using ar_type = double;
     using st_type = gko::int32;
+    // Type for `check_accessor_correctness` to forward the indices
+    using t = std::tuple<int, int, int>;
 
     static constexpr ar_type delta{::r<ar_type>::value};
 
@@ -403,16 +405,15 @@ class ScaledReducedStorage3d : public ::testing::Test {
                                                                      99))
     {
         // Test for equality is fine here since they should not be modified
-        using t = std::tuple<int, int, int>;
         // clang-format off
-        if (ignore != t{0, 0, 0}) { EXPECT_EQ(a(0, 0, 0), 10.);   }
-        if (ignore != t{0, 0, 1}) { EXPECT_EQ(a(0, 0, 1), 22.);   }
-        if (ignore != t{0, 1, 0}) { EXPECT_EQ(a(0, 1, 0), -12.);  }
-        if (ignore != t{0, 1, 1}) { EXPECT_EQ(a(0, 1, 1), 26.);   }
-        if (ignore != t{0, 2, 0}) { EXPECT_EQ(a(0, 2, 0), 14.);   }
-        if (ignore != t{0, 2, 1}) { EXPECT_EQ(a(0, 2, 1), -230.); }
-        if (ignore != t{0, 3, 0}) { EXPECT_EQ(a(0, 3, 0), 6.);    }
-        if (ignore != t{0, 3, 1}) { EXPECT_EQ(a(0, 3, 1), 154.);  }
+        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), 10.);   }
+        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), 22.);   }
+        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), -12.);  }
+        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), 26.);   }
+        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), 14.);   }
+        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), -230.); }
+        if (ignore != t(0, 3, 0)) { EXPECT_EQ(a(0, 3, 0), 6.);    }
+        if (ignore != t(0, 3, 1)) { EXPECT_EQ(a(0, 3, 1), 154.);  }
         // clang-format on
     }
 };
@@ -462,7 +463,7 @@ TEST_F(ScaledReducedStorage3d, Addition)
     const auto result = cr(0, 0, 0) + 3.;
     r(0, 0, 0) += 3.;
 
-    check_accessor_correctness(r, {0, 0, 0});
+    check_accessor_correctness(r, t(0, 0, 0));
     EXPECT_NEAR(r(0, 0, 0), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -475,7 +476,7 @@ TEST_F(ScaledReducedStorage3d, Addition2)
     const auto result = cr(0, 0, 0) + r(0, 0, 1);
     r(0, 0, 0) += cr(0, 0, 1);
 
-    check_accessor_correctness(r, {0, 0, 0});
+    check_accessor_correctness(r, t(0, 0, 0));
     EXPECT_NEAR(r(0, 0, 0), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -488,7 +489,7 @@ TEST_F(ScaledReducedStorage3d, Subtraction)
     const auto result = cr(0, 0, 1) - 2.;
     r(0, 0, 1) -= 2.;
 
-    check_accessor_correctness(r, {0, 0, 1});
+    check_accessor_correctness(r, t(0, 0, 1));
     EXPECT_NEAR(r(0, 0, 1), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -501,7 +502,7 @@ TEST_F(ScaledReducedStorage3d, Subtraction2)
     const auto result = cr(0, 1, 0) - r(0, 1, 1);
     r(0, 1, 0) -= r(0, 1, 1);
 
-    check_accessor_correctness(r, {0, 1, 0});
+    check_accessor_correctness(r, t(0, 1, 0));
     EXPECT_NEAR(r(0, 1, 0), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -514,7 +515,7 @@ TEST_F(ScaledReducedStorage3d, Multiplication)
     const auto result = cr(0, 1, 1) * 3.;
     r(0, 1, 1) *= 3.;
 
-    check_accessor_correctness(r, {0, 1, 1});
+    check_accessor_correctness(r, t(0, 1, 1));
     EXPECT_NEAR(r(0, 1, 1), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -527,7 +528,7 @@ TEST_F(ScaledReducedStorage3d, Multiplication2)
     const auto result = r(0, 2, 0) * r(0, 0, 0);
     r(0, 2, 0) *= r(0, 0, 0);
 
-    check_accessor_correctness(r, {0, 2, 0});
+    check_accessor_correctness(r, t(0, 2, 0));
     EXPECT_NEAR(r(0, 2, 0), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -540,7 +541,7 @@ TEST_F(ScaledReducedStorage3d, Division)
     const auto result = cr(0, 0, 0) / 2.;
     r(0, 0, 0) /= 2.;
 
-    check_accessor_correctness(r, {0, 0, 0});
+    check_accessor_correctness(r, t(0, 0, 0));
     EXPECT_NEAR(r(0, 0, 0), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }
@@ -553,7 +554,7 @@ TEST_F(ScaledReducedStorage3d, Division2)
     const auto result = r(0, 1, 0) / r(0, 3, 0);
     r(0, 1, 0) /= r(0, 3, 0);
 
-    check_accessor_correctness(r, {0, 1, 0});
+    check_accessor_correctness(r, t(0, 1, 0));
     EXPECT_NEAR(r(0, 1, 0), expected, delta);
     EXPECT_NEAR(result, expected, delta);
 }

From 55963c02fa3d4efe4a1861a3bb5cb93e324a220f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 22 Sep 2020 16:25:12 +0200
Subject: [PATCH 0206/1008] Add more tests; Make scale_ a std::array

---
 core/test/base/range_accessors.cpp           | 156 ++++++++++++++++++-
 include/ginkgo/core/base/range_accessors.hpp |  83 +++++-----
 2 files changed, 187 insertions(+), 52 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 045b2f49e50..6110daca24a 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -156,8 +156,10 @@ class ReducedStorage3d : public ::testing::Test {
     using reduced_storage = gko::range<accessor>;
     using const_reduced_storage = gko::range<const_accessor>;
 
+    const gko::dim<3> size{4u, 3u, 2u};
+    static constexpr gko::size_type data_elements{4 * 3 * 2};
     // clang-format off
-    st_type data[4 * 3 * 2]{
+    st_type data[data_elements] {
         // 0, y, z
         1.0, 2.01,
         -1.02, 3.03,
@@ -176,8 +178,8 @@ class ReducedStorage3d : public ::testing::Test {
         3.22, -2.23
     };
     // clang-format on
-    reduced_storage r{data, gko::dim<3>{4u, 3u, 2u}};
-    const_reduced_storage cr{data, gko::dim<3>{4u, 3u, 2u}};
+    reduced_storage r{data, size};
+    const_reduced_storage cr{data, size};
 
     template <typename Accessor>
     static void check_accessor_correctness(
@@ -214,6 +216,32 @@ class ReducedStorage3d : public ::testing::Test {
 };
 
 
+TEST_F(ReducedStorage3d, CorrectLengths)
+{
+    EXPECT_EQ(r.length(0), size[0]);
+    EXPECT_EQ(r.length(1), size[1]);
+    EXPECT_EQ(r.length(2), size[2]);
+    EXPECT_EQ(r.length(3), 1);
+    EXPECT_EQ(r->get_size(), size);
+}
+
+
+TEST_F(ReducedStorage3d, CorrectStride)
+{
+    EXPECT_EQ(r->get_stride()[0], size[1] * size[2]);
+    EXPECT_EQ(r->get_stride().at(0), size[1] * size[2]);
+    EXPECT_EQ(r->get_stride()[1], size[2]);
+    EXPECT_EQ(r->get_stride().at(1), size[2]);
+}
+
+
+TEST_F(ReducedStorage3d, CorrectStorage)
+{
+    EXPECT_EQ(r->get_storage(), data);
+    EXPECT_EQ(r->get_const_storage(), data);
+}
+
+
 TEST_F(ReducedStorage3d, CanReadData)
 {
     check_accessor_correctness(r);
@@ -221,6 +249,28 @@ TEST_F(ReducedStorage3d, CanReadData)
 }
 
 
+TEST_F(ReducedStorage3d, CopyFrom)
+{
+    st_type data2[data_elements];
+    reduced_storage cpy(data2, size);
+
+    // Do not use this in regular code since the implementation is slow
+    cpy = r;
+
+    check_accessor_correctness(cpy);
+}
+
+
+TEST_F(ReducedStorage3d, CanImplicitlyConvertToConst)
+{
+    const_reduced_storage const_rs = r->to_const();
+    const_reduced_storage const_rs2 = cr;
+
+    check_accessor_correctness(const_rs);
+    check_accessor_correctness(const_rs2);
+}
+
+
 TEST_F(ReducedStorage3d, ToConstWorking)
 {
     auto cr2 = r->to_const();
@@ -240,6 +290,24 @@ TEST_F(ReducedStorage3d, CanWriteData)
 }
 
 
+TEST_F(ReducedStorage3d, Assignment)
+{
+    r(0, 0, 1) = 10.2;
+
+    check_accessor_correctness(r, t(0, 0, 1));
+    EXPECT_NEAR(r(0, 0, 1), 10.2, delta);
+}
+
+
+TEST_F(ReducedStorage3d, Assignment2)
+{
+    r(0, 0, 1) = r(0, 1, 0);
+
+    check_accessor_correctness(r, t(0, 0, 1));
+    EXPECT_NEAR(r(0, 0, 1), -1.02, delta);
+}
+
+
 TEST_F(ReducedStorage3d, Addition)
 {
     const ar_type expected = 10.2 + 2.01;
@@ -384,6 +452,9 @@ class ScaledReducedStorage3d : public ::testing::Test {
     using reduced_storage = gko::range<accessor>;
     using const_reduced_storage = gko::range<const_accessor>;
 
+    const gko::dim<3> size{1u, 4u, 2u};
+    static constexpr gko::size_type data_elements{8};
+    static constexpr gko::size_type scale_elements{8};
     // clang-format off
     st_type data[8]{
         10, 11,
@@ -391,12 +462,12 @@ class ScaledReducedStorage3d : public ::testing::Test {
         14, -115,
         6, 77
     };
-    ar_type scale[2]{
+    ar_type scale[scale_elements]{
         1., 2.,
     };
     // clang-format on
-    reduced_storage r{data, scale, gko::dim<3>{1u, 4u, 2u}};
-    const_reduced_storage cr{data, scale, gko::dim<3>{1u, 4u, 2u}};
+    reduced_storage r{data, scale, size};
+    const_reduced_storage cr{data, scale, size};
 
     template <typename Accessor>
     static void check_accessor_correctness(
@@ -419,6 +490,79 @@ class ScaledReducedStorage3d : public ::testing::Test {
 };
 
 
+TEST_F(ScaledReducedStorage3d, CorrectLengths)
+{
+    EXPECT_EQ(r.length(0), size[0]);
+    EXPECT_EQ(r.length(1), size[1]);
+    EXPECT_EQ(r.length(2), size[2]);
+    EXPECT_EQ(r.length(3), 1);
+    EXPECT_EQ(r->get_size(), size);
+}
+
+
+TEST_F(ScaledReducedStorage3d, CorrectStride)
+{
+    EXPECT_EQ(r->get_stride()[0], size[1] * size[2]);
+    EXPECT_EQ(r->get_stride().at(0), size[1] * size[2]);
+    EXPECT_EQ(r->get_stride()[1], size[2]);
+    EXPECT_EQ(r->get_stride().at(1), size[2]);
+}
+
+
+TEST_F(ScaledReducedStorage3d, CorrectStorage)
+{
+    EXPECT_EQ(r->get_storage(), data);
+    EXPECT_EQ(r->get_const_storage(), data);
+}
+
+
+TEST_F(ScaledReducedStorage3d, CorrectScale)
+{
+    EXPECT_EQ(r->get_scale(), scale);
+    EXPECT_EQ(r->get_const_scale(), scale);
+}
+
+
+TEST_F(ScaledReducedStorage3d, CanReadData)
+{
+    check_accessor_correctness(r);
+    check_accessor_correctness(cr);
+}
+
+
+TEST_F(ScaledReducedStorage3d, CopyFrom)
+{
+    st_type data2[data_elements];
+    ar_type scale2[scale_elements];
+    reduced_storage cpy(data2, scale2, size);
+
+    // Do not use this in regular code since the implementation is slow
+    cpy = r;
+
+    check_accessor_correctness(cpy);
+}
+
+
+TEST_F(ScaledReducedStorage3d, CanImplicitlyConvertToConst)
+{
+    const_reduced_storage const_rs = r->to_const();
+    const_reduced_storage const_rs2 = cr;
+
+    check_accessor_correctness(const_rs);
+    check_accessor_correctness(const_rs2);
+}
+
+
+TEST_F(ScaledReducedStorage3d, ToConstWorking)
+{
+    auto cr2 = r->to_const();
+
+    static_assert(std::is_same<decltype(cr2), const_reduced_storage>::value,
+                  "Types must be equal!");
+    check_accessor_correctness(cr2);
+}
+
+
 TEST_F(ScaledReducedStorage3d, CanRead)
 {
     check_accessor_correctness(cr);
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 5f412e19925..a4fc3e50b22 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -379,9 +379,8 @@ class reduced_storage
 
     reduced_storage() = delete;
     ~reduced_storage() = default;
-    // Forbid copy construction and move assignment
+    // Forbid copy construction
     reduced_storage(const reduced_storage &) = delete;
-    reduced_storage &operator=(reduced_storage &&) = delete;
 
     GKO_ATTRIBUTES constexpr reduced_storage(
         storage_type *const GKO_RESTRICT ptr)
@@ -404,6 +403,11 @@ class reduced_storage
     {
         return std::move(*this) = static_cast<arithmetic_type>(ref);
     }
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
+        reduced_storage &&ref) &&
+    {
+        return std::move(*this) = static_cast<arithmetic_type>(ref);
+    }
 
 private:
     storage_type *const GKO_RESTRICT ptr_;
@@ -473,9 +477,8 @@ class scaled_reduced_storage
 
     scaled_reduced_storage() = delete;
     ~scaled_reduced_storage() = default;
-    // Forbid copy construction and move assignment
+    // Forbid copy construction
     scaled_reduced_storage(const scaled_reduced_storage &) = delete;
-    scaled_reduced_storage &operator=(scaled_reduced_storage &&) = delete;
 
     GKO_ATTRIBUTES constexpr scaled_reduced_storage(
         storage_type *const GKO_RESTRICT ptr, arithmetic_type scale)
@@ -499,6 +502,11 @@ class scaled_reduced_storage
     {
         return std::move(*this) = static_cast<arithmetic_type>(ref);
     }
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
+        scaled_reduced_storage &&ref) &&
+    {
+        return std::move(*this) = static_cast<arithmetic_type>(ref);
+    }
 
 private:
     storage_type *const GKO_RESTRICT ptr_;
@@ -641,7 +649,7 @@ class reduced_row_major {
      *
      * @returns  length in dimension `dimension`
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr dim<dimensionality> length(
+    GKO_ATTRIBUTES GKO_INLINE constexpr size_type length(
         size_type dimension) const
     {
         return dimension < dimensionality ? size_[dimension] : 1;
@@ -658,7 +666,7 @@ class reduced_row_major {
      * @param other  other accessor
      */
     template <typename OtherAccessor>
-    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other)
+    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
         for (size_type i = 0; i < size_[0]; ++i) {
             for (size_type j = 0; j < size_[1]; ++j) {
@@ -724,23 +732,15 @@ class reduced_row_major {
     }
 
     /**
-     * Returns the stride for the first index
-     *
-     * @returns the stride for the first index
-     */
-    GKO_ATTRIBUTES GKO_INLINE constexpr size_type get_stride0() const
-    {
-        return stride_[0];
-    }
-
-    /**
-     * Returns the stride for the second index
+     * Returns a pointer to a stride array of size dimensionality - 1
      *
-     * @returns the stride for the second index
+     * @returns returns a pointer to a stride array of size dimensionality - 1
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr size_type get_stride1() const
+    GKO_ATTRIBUTES
+    GKO_INLINE constexpr const std::array<const size_type, dimensionality - 1>
+        &get_stride() const
     {
-        return stride_[1];
+        return stride_;
     }
 
     /**
@@ -773,8 +773,8 @@ class reduced_row_major {
     }
 
     storage_type *storage_;
-    dim<dimensionality> size_;
-    size_type stride_[2];  // std::array leads to conflicts on devices
+    const dim<dimensionality> size_;
+    const std::array<const size_type, dimensionality - 1> stride_;
 };
 
 
@@ -957,7 +957,7 @@ class scaled_reduced_row_major
      *
      * @returns length in dimension `dimension`
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr dim<dimensionality> length(
+    GKO_ATTRIBUTES GKO_INLINE constexpr size_type length(
         size_type dimension) const
     {
         return dimension < dimensionality ? size_[dimension] : 1;
@@ -974,8 +974,13 @@ class scaled_reduced_row_major
      * @param other  other accessor
      */
     template <typename OtherAccessor>
-    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other)
+    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
+        for (size_type i = 0; i < this->size_[0]; ++i) {
+            for (size_type k = 0; k < this->size_[2]; ++k) {
+                this->write_scale(i, k, other.read_scale(i, k));
+            }
+        }
         for (size_type i = 0; i < this->size_[0]; ++i) {
             for (size_type j = 0; j < this->size_[1]; ++j) {
                 for (size_type k = 0; k < this->size_[2]; ++k) {
@@ -983,11 +988,6 @@ class scaled_reduced_row_major
                 }
             }
         }
-        for (size_type i = 0; i < this->size_[0]; ++i) {
-            for (size_type k = 0; k < this->size_[2]; ++k) {
-                this->write_scale(i, k, other.read_scale(i, k));
-            }
-        }
     }
 
     /**
@@ -1049,23 +1049,15 @@ class scaled_reduced_row_major
     }
 
     /**
-     * Returns the stride for the first index
-     *
-     * @returns the stride for the first index
-     */
-    GKO_ATTRIBUTES GKO_INLINE constexpr size_type get_stride0() const
-    {
-        return stride_[0];
-    }
-
-    /**
-     * Returns the stride for the second index
+     * Returns a pointer to a stride array of size dimensionality - 1
      *
-     * @returns the stride for the second index
+     * @returns returns a pointer to a stride array of size dimensionality - 1
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr size_type get_stride1() const
+    GKO_ATTRIBUTES
+    GKO_INLINE constexpr const std::array<const size_type, dimensionality - 1>
+        &get_stride() const
     {
-        return stride_[1];
+        return stride_;
     }
 
     /**
@@ -1127,9 +1119,8 @@ class scaled_reduced_row_major
 
     storage_type *storage_;
     scale_type *scale_;
-    dim<dimensionality> size_;
-    // std::array leads to conflicts on devices
-    size_type stride_[dimensionality - 1];
+    const dim<dimensionality> size_;
+    const std::array<const size_type, dimensionality - 1> stride_;
 };
 
 

From 83c2da62f60147aa15dc0173c3b351242c04e9af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 25 Sep 2020 12:14:00 +0200
Subject: [PATCH 0207/1008] Apply newer clang-format to range.hpp

---
 include/ginkgo/core/base/range.hpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/include/ginkgo/core/base/range.hpp b/include/ginkgo/core/base/range.hpp
index c08e1f2f305..2fe87a364a7 100644
--- a/include/ginkgo/core/base/range.hpp
+++ b/include/ginkgo/core/base/range.hpp
@@ -469,8 +469,7 @@ struct implement_unary_operation {
 
 template <operation_kind Kind, typename FirstOperand, typename SecondOperand,
           typename Operation>
-struct implement_binary_operation {
-};
+struct implement_binary_operation {};
 
 template <typename FirstAccessor, typename SecondAccessor, typename Operation>
 struct implement_binary_operation<operation_kind::range_by_range, FirstAccessor,

From 697a9e85c83201d1ddd5f5329ca3602fa7f11a8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 30 Sep 2020 07:43:09 +0200
Subject: [PATCH 0208/1008] Add TODO

---
 include/ginkgo/core/base/range_accessors.hpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index a4fc3e50b22..37534564fb5 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -229,13 +229,15 @@ namespace detail {
 
 // tests if the cast operator to `ValueType` is present
 template <typename Ref, typename ValueType, typename = xstd::void_t<>>
-struct has_cast_operator : std::false_type {};
+struct has_cast_operator : std::false_type {
+};
 
 template <typename Ref, typename ValueType>
 struct has_cast_operator<
     Ref, ValueType,
     xstd::void_t<decltype(std::declval<Ref>().Ref::operator ValueType())>>
-    : std::true_type {};
+    : std::true_type {
+};
 
 /**
  * @internal
@@ -428,6 +430,7 @@ class reduced_storage<ArithmeticType, const StorageType>
     ~reduced_storage() = default;
     // Forbid copy construction and move assignment
     reduced_storage(const reduced_storage &) = delete;
+    // TODO  Implement this operator and call `operator=(const &)`
     reduced_storage &operator=(reduced_storage &&) = delete;
 
     GKO_ATTRIBUTES constexpr reduced_storage(
@@ -530,6 +533,7 @@ class scaled_reduced_storage<ArithmeticType, const StorageType>
     ~scaled_reduced_storage() = default;
     // Forbid copy construction and move assignment
     scaled_reduced_storage(const scaled_reduced_storage &) = delete;
+    // TODO  Implement this operator and call `operator=(const &)`
     scaled_reduced_storage &operator=(scaled_reduced_storage &&) = delete;
 
     GKO_ATTRIBUTES constexpr scaled_reduced_storage(

From c021f0353b47c666262d7e2102d2e732a6cb3f36 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 19 Oct 2020 14:55:00 +0200
Subject: [PATCH 0209/1008] Apply review suggestions

Additionally, added test for unary operator.

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 core/test/base/range_accessors.cpp           |  53 ++++--
 include/ginkgo/core/base/range_accessors.hpp | 177 +++++++++++--------
 2 files changed, 138 insertions(+), 92 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 6110daca24a..b5b353991b3 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -237,7 +237,7 @@ TEST_F(ReducedStorage3d, CorrectStride)
 
 TEST_F(ReducedStorage3d, CorrectStorage)
 {
-    EXPECT_EQ(r->get_storage(), data);
+    EXPECT_EQ(r->get_stored_data(), data);
     EXPECT_EQ(r->get_const_storage(), data);
 }
 
@@ -271,7 +271,7 @@ TEST_F(ReducedStorage3d, CanImplicitlyConvertToConst)
 }
 
 
-TEST_F(ReducedStorage3d, ToConstWorking)
+TEST_F(ReducedStorage3d, ToConstWorks)
 {
     auto cr2 = r->to_const();
 
@@ -412,6 +412,18 @@ TEST_F(ReducedStorage3d, Division2)
 }
 
 
+TEST_F(ReducedStorage3d, UnaryMinus)
+{
+    const ar_type neg_expected = r(2, 0, 0);
+    const ar_type expected = -neg_expected;
+
+    auto result = -r(2, 0, 0);
+
+    check_accessor_correctness(r);
+    EXPECT_EQ(result, expected);
+}
+
+
 TEST_F(ReducedStorage3d, CanCreateSubrange)
 {
     auto subr = r(span{1, 3}, span{0, 2}, 0);
@@ -454,7 +466,7 @@ class ScaledReducedStorage3d : public ::testing::Test {
 
     const gko::dim<3> size{1u, 4u, 2u};
     static constexpr gko::size_type data_elements{8};
-    static constexpr gko::size_type scale_elements{8};
+    static constexpr gko::size_type scalar_elements{8};
     // clang-format off
     st_type data[8]{
         10, 11,
@@ -462,12 +474,12 @@ class ScaledReducedStorage3d : public ::testing::Test {
         14, -115,
         6, 77
     };
-    ar_type scale[scale_elements]{
+    ar_type scalar[scalar_elements]{
         1., 2.,
     };
     // clang-format on
-    reduced_storage r{data, scale, size};
-    const_reduced_storage cr{data, scale, size};
+    reduced_storage r{data, scalar, size};
+    const_reduced_storage cr{data, scalar, size};
 
     template <typename Accessor>
     static void check_accessor_correctness(
@@ -511,15 +523,15 @@ TEST_F(ScaledReducedStorage3d, CorrectStride)
 
 TEST_F(ScaledReducedStorage3d, CorrectStorage)
 {
-    EXPECT_EQ(r->get_storage(), data);
+    EXPECT_EQ(r->get_stored_data(), data);
     EXPECT_EQ(r->get_const_storage(), data);
 }
 
 
 TEST_F(ScaledReducedStorage3d, CorrectScale)
 {
-    EXPECT_EQ(r->get_scale(), scale);
-    EXPECT_EQ(r->get_const_scale(), scale);
+    EXPECT_EQ(r->get_scalar(), scalar);
+    EXPECT_EQ(r->get_const_scalar(), scalar);
 }
 
 
@@ -533,7 +545,7 @@ TEST_F(ScaledReducedStorage3d, CanReadData)
 TEST_F(ScaledReducedStorage3d, CopyFrom)
 {
     st_type data2[data_elements];
-    ar_type scale2[scale_elements];
+    ar_type scale2[scalar_elements];
     reduced_storage cpy(data2, scale2, size);
 
     // Do not use this in regular code since the implementation is slow
@@ -553,7 +565,7 @@ TEST_F(ScaledReducedStorage3d, CanImplicitlyConvertToConst)
 }
 
 
-TEST_F(ScaledReducedStorage3d, ToConstWorking)
+TEST_F(ScaledReducedStorage3d, ToConstWorks)
 {
     auto cr2 = r->to_const();
 
@@ -569,6 +581,7 @@ TEST_F(ScaledReducedStorage3d, CanRead)
     check_accessor_correctness(r);
 }
 
+
 TEST_F(ScaledReducedStorage3d, Subrange)
 {
     auto subr = cr(0, gko::span{0, 2}, 1);
@@ -580,7 +593,7 @@ TEST_F(ScaledReducedStorage3d, Subrange)
 
 TEST_F(ScaledReducedStorage3d, CanWriteScale)
 {
-    r->write_scale(0, 0, 10.);
+    r->write_scalar(0, 0, 10.);
 
     EXPECT_EQ(r(0, 0, 0), 100.);
     EXPECT_EQ(r(0, 0, 1), 22.);
@@ -595,8 +608,8 @@ TEST_F(ScaledReducedStorage3d, CanWriteScale)
 
 TEST_F(ScaledReducedStorage3d, CanReadScale)
 {
-    EXPECT_EQ(r->read_scale(0, 0), 1.);
-    EXPECT_EQ(r->read_scale(0, 1), 2.);
+    EXPECT_EQ(r->read_scalar(0, 0), 1.);
+    EXPECT_EQ(r->read_scalar(0, 1), 2.);
 }
 
 
@@ -704,4 +717,16 @@ TEST_F(ScaledReducedStorage3d, Division2)
 }
 
 
+TEST_F(ScaledReducedStorage3d, UnaryMinus)
+{
+    const ar_type neg_expected = r(0, 1, 1);
+    const ar_type expected = -neg_expected;
+
+    auto result = -r(0, 1, 1);
+
+    check_accessor_correctness(r);
+    EXPECT_EQ(result, expected);
+}
+
+
 }  // namespace
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 37534564fb5..c2a9695efd0 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -120,7 +120,7 @@ class row_major {
      * @param num_cols  number of columns of the accessor
      * @param stride  distance (in elements) between starting positions of
      *                consecutive rows (i.e. `data + i * stride` points to
-     * the `i`-th row)
+     *                the `i`-th row)
      */
     GKO_ATTRIBUTES constexpr explicit row_major(data_type data,
                                                 size_type num_rows,
@@ -191,7 +191,9 @@ class row_major {
      * Copies data from another accessor
      *
      * @warning Do not use this function since it is not optimized for a
-     * specific executor. It will always be performed sequentially.
+     *          specific executor. It will always be performed sequentially.
+     *          Please write an optimized version (adjusted to the architecture)
+     *          by iterating through the values yourself.
      *
      * @tparam OtherAccessor  type of the other accessor
      *
@@ -229,15 +231,14 @@ namespace detail {
 
 // tests if the cast operator to `ValueType` is present
 template <typename Ref, typename ValueType, typename = xstd::void_t<>>
-struct has_cast_operator : std::false_type {
-};
+struct has_cast_operator : std::false_type {};
 
 template <typename Ref, typename ValueType>
 struct has_cast_operator<
     Ref, ValueType,
     xstd::void_t<decltype(std::declval<Ref>().Ref::operator ValueType())>>
-    : std::true_type {
-};
+    : std::true_type {};
+
 
 /**
  * @internal
@@ -266,6 +267,7 @@ to_value_type(const Ref &ref)
     return static_cast<ValueType>(ref);
 }
 
+
 /**
  * This is a mixin which defines the binary operators for *, /, +, - for the
  * Reference class, the unary operator -, and the assignment operators
@@ -286,7 +288,6 @@ template <typename Reference, typename ArithmeticType>
 struct enable_reference {
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
 
-
 #define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_oper, _op)          \
     friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR \
         arithmetic_type                                             \
@@ -348,9 +349,9 @@ struct enable_reference {
 /**
  * This namespace contains reference classes used inside accessors.
  *
- * @warning  This class should not be used by anything else but accessors.
+ * @warning  These classes should not be used by anything else but accessors.
  */
-namespace reference {
+namespace reference_class {
 
 
 /**
@@ -375,12 +376,15 @@ class reduced_storage
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = StorageType;
+
     // Allow move construction, so perfect forwarding is possible (required
     // for `range` support)
     reduced_storage(reduced_storage &&) = default;
 
     reduced_storage() = delete;
+
     ~reduced_storage() = default;
+
     // Forbid copy construction
     reduced_storage(const reduced_storage &) = delete;
 
@@ -394,17 +398,20 @@ class reduced_storage
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr);
     }
+
     GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
         arithmetic_type val) &&
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return *r_ptr = static_cast<storage_type>(val);
     }
+
     GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
         const reduced_storage &ref) &&
     {
         return std::move(*this) = static_cast<arithmetic_type>(ref);
     }
+
     GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
         reduced_storage &&ref) &&
     {
@@ -423,14 +430,17 @@ class reduced_storage<ArithmeticType, const StorageType>
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = const StorageType;
+
     // Allow move construction, so perfect forwarding is possible
     reduced_storage(reduced_storage &&) = default;
 
     reduced_storage() = delete;
+
     ~reduced_storage() = default;
+
     // Forbid copy construction and move assignment
     reduced_storage(const reduced_storage &) = delete;
-    // TODO  Implement this operator and call `operator=(const &)`
+
     reduced_storage &operator=(reduced_storage &&) = delete;
 
     GKO_ATTRIBUTES constexpr reduced_storage(
@@ -453,8 +463,8 @@ class reduced_storage<ArithmeticType, const StorageType>
  * Reference class for a different storage than arithmetic type with the
  * addition of a scaling factor. The conversion between both formats is done
  * with a static_cast to the ArithmeticType, followed by a multiplication
- * of the scale (when reading; for writing, the new value is divided by the
- * scale before casting to the StorageType).
+ * of the scalar (when reading; for writing, the new value is divided by the
+ * scalar before casting to the StorageType).
  *
  * Copying this reference is disabled, but move construction is possible to
  * allow for an additional layer (like gko::range).
@@ -479,32 +489,36 @@ class scaled_reduced_storage
     scaled_reduced_storage(scaled_reduced_storage &&) = default;
 
     scaled_reduced_storage() = delete;
+
     ~scaled_reduced_storage() = default;
+
     // Forbid copy construction
     scaled_reduced_storage(const scaled_reduced_storage &) = delete;
 
     GKO_ATTRIBUTES constexpr scaled_reduced_storage(
-        storage_type *const GKO_RESTRICT ptr, arithmetic_type scale)
-        : ptr_{ptr}, scale_{scale}
+        storage_type *const GKO_RESTRICT ptr, arithmetic_type scalar)
+        : ptr_{ptr}, scalar_{scalar}
     {}
 
     GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return static_cast<arithmetic_type>(*r_ptr) * scale_;
+        return static_cast<arithmetic_type>(*r_ptr) * scalar_;
     }
 
     GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
         arithmetic_type val) &&
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return *r_ptr = static_cast<storage_type>(val / scale_), val;
+        return *r_ptr = static_cast<storage_type>(val / scalar_), val;
     }
+
     GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
         const scaled_reduced_storage &ref) &&
     {
         return std::move(*this) = static_cast<arithmetic_type>(ref);
     }
+
     GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
         scaled_reduced_storage &&ref) &&
     {
@@ -513,7 +527,7 @@ class scaled_reduced_storage
 
 private:
     storage_type *const GKO_RESTRICT ptr_;
-    const arithmetic_type scale_;
+    const arithmetic_type scalar_;
 };
 
 // Specialization for constant storage_type (no `operator=`)
@@ -530,31 +544,33 @@ class scaled_reduced_storage<ArithmeticType, const StorageType>
     scaled_reduced_storage(scaled_reduced_storage &&) = default;
 
     scaled_reduced_storage() = delete;
+
     ~scaled_reduced_storage() = default;
+
     // Forbid copy construction and move assignment
     scaled_reduced_storage(const scaled_reduced_storage &) = delete;
-    // TODO  Implement this operator and call `operator=(const &)`
+
     scaled_reduced_storage &operator=(scaled_reduced_storage &&) = delete;
 
     GKO_ATTRIBUTES constexpr scaled_reduced_storage(
-        storage_type *const GKO_RESTRICT ptr, arithmetic_type scale)
-        : ptr_{ptr}, scale_{scale}
+        storage_type *const GKO_RESTRICT ptr, arithmetic_type scalar)
+        : ptr_{ptr}, scalar_{scalar}
     {}
 
     GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return static_cast<arithmetic_type>(*r_ptr) * scale_;
+        return static_cast<arithmetic_type>(*r_ptr) * scalar_;
     }
 
 
 private:
     storage_type *const GKO_RESTRICT ptr_;
-    const arithmetic_type scale_;
+    const arithmetic_type scalar_;
 };
 
 
-}  // namespace reference
+}  // namespace reference_class
 }  // namespace detail
 
 
@@ -593,8 +609,8 @@ class reduced_row_major {
     friend class range<reduced_row_major>;
 
 protected:
-    using reference =
-        detail::reference::reduced_storage<arithmetic_type, storage_type>;
+    using reference_type =
+        detail::reference_class::reduced_storage<arithmetic_type, storage_type>;
 
     /**
      * Creates the accessor with an already allocated storage space with a
@@ -663,7 +679,9 @@ class reduced_row_major {
      * Copies data from another accessor
      *
      * @warning Do not use this function since it is not optimized for a
-     * specific executor. It will always be performed sequentially.
+     *          specific executor. It will always be performed sequentially.
+     *          Please write an optimized version (adjusted to the architecture)
+     *          by iterating through the values yourself.
      *
      * @tparam OtherAccessor  type of the other accessor
      *
@@ -693,10 +711,10 @@ class reduced_row_major {
      * is const), or a reference if the accessor is non-const
      */
     GKO_ATTRIBUTES GKO_INLINE constexpr std::conditional_t<
-        is_const, arithmetic_type, reference>
+        is_const, arithmetic_type, reference_type>
     operator()(size_type x, size_type y, size_type z) const
     {
-        return reference{this->storage_ + compute_index(x, y, z)};
+        return reference_type{this->storage_ + compute_index(x, y, z)};
     }
 
     /**
@@ -752,7 +770,7 @@ class reduced_row_major {
      *
      * @returns the pointer to the storage data
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr storage_type *get_storage() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr storage_type *get_stored_data() const
     {
         return storage_;
     }
@@ -788,23 +806,23 @@ namespace detail {
 // In case of a const type, do not provide a write function
 template <int Dimensionality, typename Accessor, typename ScaleType,
           bool = std::is_const<ScaleType>::value>
-struct enable_write_scale {
+struct enable_write_scalar {
     static_assert(std::is_const<ScaleType>::value,
                   "This class must have a constant ScaleType!");
-    using scale_type = ScaleType;
+    using scalar_type = ScaleType;
 };
 
 // In case of a non-const type, enable the write function
 template <int Dimensionality, typename Accessor, typename ScaleType>
-struct enable_write_scale<Dimensionality, Accessor, ScaleType, false> {
+struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
     static_assert(!std::is_const<ScaleType>::value,
                   "This class must NOT have a constant ScaleType!");
     static_assert(Dimensionality == 3, "Only Dimensionality 3 supported!");
 
-    using scale_type = ScaleType;
+    using scalar_type = ScaleType;
 
     /**
-     * Writes the scale value at the given indices.
+     * Writes the scalar value at the given indices.
      *
      * @param x  x index
      * @param z  z index
@@ -812,11 +830,11 @@ struct enable_write_scale<Dimensionality, Accessor, ScaleType, false> {
      *
      * @returns the written value.
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr scale_type write_scale(
-        size_type x, size_type z, scale_type value) const
+    GKO_ATTRIBUTES GKO_INLINE constexpr scalar_type write_scalar(
+        size_type x, size_type z, scalar_type value) const
     {
-        scale_type *GKO_RESTRICT rest_scale = self()->scale_;
-        return rest_scale[self()->compute_scale_index(x, z)] = value;
+        scalar_type *GKO_RESTRICT rest_scalar = self()->scalar_;
+        return rest_scalar[self()->compute_scalar_index(x, z)] = value;
     }
 
 private:
@@ -851,7 +869,7 @@ struct enable_write_scale<Dimensionality, Accessor, ScaleType, false> {
  */
 template <int Dimensionality, typename ArithmeticType, typename StorageType>
 class scaled_reduced_row_major
-    : public detail::enable_write_scale<
+    : public detail::enable_write_scalar<
           Dimensionality,
           scaled_reduced_row_major<Dimensionality, ArithmeticType, StorageType>,
           ArithmeticType> {
@@ -860,7 +878,7 @@ class scaled_reduced_row_major
     using storage_type = StorageType;
     static constexpr size_type dimensionality{Dimensionality};
     static constexpr bool is_const{std::is_const<storage_type>::value};
-    using scale_type =
+    using scalar_type =
         std::conditional_t<is_const, const arithmetic_type, arithmetic_type>;
 
     using const_accessor =
@@ -870,14 +888,15 @@ class scaled_reduced_row_major
     static_assert(dimensionality == 3,
                   "Only Dimensionality == 3 is currently supported");
 
-    // Allow access to both `scale_` and `compute_scale_index()`
-    friend class detail::enable_write_scale<
-        dimensionality, scaled_reduced_row_major, scale_type>;
+    // Allow access to both `scalar_` and `compute_scalar_index()`
+    friend class detail::enable_write_scalar<
+        dimensionality, scaled_reduced_row_major, scalar_type>;
     friend class range<scaled_reduced_row_major>;
 
 protected:
-    using reference =
-        detail::reference::scaled_reduced_storage<arithmetic_type, StorageType>;
+    using reference_type =
+        detail::reference_class::scaled_reduced_storage<arithmetic_type,
+                                                        StorageType>;
 
 public:
     /**
@@ -886,19 +905,19 @@ class scaled_reduced_row_major
      * index, the second stride for the second index, and so on.
      *
      * @param storage  pointer to the block of memory containing the storage
-     * @param scale  pointer to the block of memory containing the scale values.
-     *               Memory required is size[0] * stride1.
+     * @param scalar  pointer to the block of memory containing the scalar
+     * values. Memory required is size[0] * stride1.
      * @param size  multidimensional size of the memory
      * @param stride0  stride used for the x-indices
      * @param stride1  stride used for the y-indices
      */
     GKO_ATTRIBUTES constexpr scaled_reduced_row_major(storage_type *storage,
-                                                      scale_type *scale,
+                                                      scalar_type *scalar,
                                                       dim<dimensionality> size,
                                                       size_type stride0,
                                                       size_type stride1)
         : storage_{storage},
-          scale_{scale},
+          scalar_{scalar},
           size_{size},
           stride_{stride0, stride1}
     {}
@@ -909,14 +928,14 @@ class scaled_reduced_row_major
      * It is assumed that accesses are without a stride.
      *
      * @param storage  pointer to the block of memory containing the storage
-     * @param scale  pointer to the block of memory containing the scale values.
-     *               Memory required is size[0] * size[1].
+     * @param scalar  pointer to the block of memory containing the scalar
+     * values. Memory required is size[0] * size[1].
      * @param size  multidimensional size of the memory
      */
     GKO_ATTRIBUTES constexpr scaled_reduced_row_major(storage_type *storage,
-                                                      scale_type *scale,
+                                                      scalar_type *scalar,
                                                       dim<dimensionality> size)
-        : scaled_reduced_row_major{storage, scale, size, size[1] * size[2],
+        : scaled_reduced_row_major{storage, scalar, size, size[1] * size[2],
                                    size[2]}
     {}
 
@@ -935,23 +954,23 @@ class scaled_reduced_row_major
      */
     GKO_ATTRIBUTES GKO_INLINE constexpr range<const_accessor> to_const() const
     {
-        return range<const_accessor>{storage_, scale_, size_, stride_[0],
+        return range<const_accessor>{storage_, scalar_, size_, stride_[0],
                                      stride_[1]};
     }
 
     /**
-     * Reads the scale value at the given indices.
+     * Reads the scalar value at the given indices.
      *
      * @param x  x index
      * @param z  z index
      *
-     * @returns the scale value at the given indices.
+     * @returns the scalar value at the given indices.
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr scale_type read_scale(size_type x,
-                                                              size_type z) const
+    GKO_ATTRIBUTES GKO_INLINE constexpr scalar_type read_scalar(
+        size_type x, size_type z) const
     {
-        const arithmetic_type *GKO_RESTRICT rest_scale = scale_;
-        return rest_scale[compute_scale_index(x, z)];
+        const arithmetic_type *GKO_RESTRICT rest_scalar = scalar_;
+        return rest_scalar[compute_scalar_index(x, z)];
     }
 
     /**
@@ -971,7 +990,9 @@ class scaled_reduced_row_major
      * Copies data from another accessor
      *
      * @warning Do not use this function since it is not optimized for a
-     * specific executor. It will always be performed sequentially.
+     *          specific executor. It will always be performed sequentially.
+     *          Please write an optimized version (adjusted to the architecture)
+     *          by iterating through the values yourself.
      *
      * @tparam OtherAccessor  type of the other accessor
      *
@@ -982,7 +1003,7 @@ class scaled_reduced_row_major
     {
         for (size_type i = 0; i < this->size_[0]; ++i) {
             for (size_type k = 0; k < this->size_[2]; ++k) {
-                this->write_scale(i, k, other.read_scale(i, k));
+                this->write_scalar(i, k, other.read_scalar(i, k));
             }
         }
         for (size_type i = 0; i < this->size_[0]; ++i) {
@@ -1006,11 +1027,11 @@ class scaled_reduced_row_major
      * is const), or a reference if the accessor is non-const
      */
     GKO_ATTRIBUTES GKO_INLINE constexpr std::conditional_t<
-        is_const, arithmetic_type, reference>
+        is_const, arithmetic_type, reference_type>
     operator()(size_type x, size_type y, size_type z) const
     {
-        return reference{this->storage_ + this->compute_index(x, y, z),
-                         this->read_scale(x, z)};
+        return reference_type{this->storage_ + this->compute_index(x, y, z),
+                              this->read_scalar(x, z)};
     }
 
     /**
@@ -1034,8 +1055,8 @@ class scaled_reduced_row_major
                    this->storage_ + this->compute_index(x_span.begin,
                                                         y_span.begin,
                                                         z_span.begin),
-                   this->scale_ +
-                       this->compute_scale_index(x_span.begin, z_span.begin),
+                   this->scalar_ +
+                       this->compute_scalar_index(x_span.begin, z_span.begin),
                    dim<dimensionality>{x_span.end - x_span.begin,
                                        y_span.end - y_span.begin,
                                        z_span.end - z_span.begin},
@@ -1069,7 +1090,7 @@ class scaled_reduced_row_major
      *
      * @returns the pointer to the storage data
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr storage_type *get_storage() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr storage_type *get_stored_data() const
     {
         return storage_;
     }
@@ -1086,24 +1107,24 @@ class scaled_reduced_row_major
     }
 
     /**
-     * Returns the pointer to the scale data
+     * Returns the pointer to the scalar data
      *
-     * @returns the pointer to the scale data
+     * @returns the pointer to the scalar data
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr scale_type *get_scale() const
+    GKO_ATTRIBUTES GKO_INLINE constexpr scalar_type *get_scalar() const
     {
-        return this->scale_;
+        return this->scalar_;
     }
 
     /**
-     * Returns a const pointer to the scale data
+     * Returns a const pointer to the scalar data
      *
-     * @returns a const pointer to the scale data
+     * @returns a const pointer to the scalar data
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr const scale_type *get_const_scale()
+    GKO_ATTRIBUTES GKO_INLINE constexpr const scalar_type *get_const_scalar()
         const
     {
-        return this->scale_;
+        return this->scalar_;
     }
 
 protected:
@@ -1115,14 +1136,14 @@ class scaled_reduced_row_major
     }
 
     GKO_ATTRIBUTES constexpr GKO_INLINE size_type
-    compute_scale_index(size_type x, size_type z) const
+    compute_scalar_index(size_type x, size_type z) const
     {
         return GKO_ASSERT(x < size_[0]), GKO_ASSERT(z < size_[2]),
                x * stride_[1] + z;
     }
 
     storage_type *storage_;
-    scale_type *scale_;
+    scalar_type *scalar_;
     const dim<dimensionality> size_;
     const std::array<const size_type, dimensionality - 1> stride_;
 };

From 5b3e6b0d1c7b0ae50230ff26a57cfde8b53d7437 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Sun, 25 Oct 2020 21:57:32 +0100
Subject: [PATCH 0210/1008] Add better documentation; add stride mask template

---
 include/ginkgo/core/base/range_accessors.hpp | 25 +++++++++++++++-----
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index c2a9695efd0..cfd9fbc6893 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -849,8 +849,10 @@ struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
 
 
 /**
- * The reduced_row_major class allows a storage format that is different from
- * the arithmetic format (which is returned from the brace operator).
+ * The scaled_reduced_row_major class allows a storage format that is different
+ * from the arithmetic format (which is returned from the brace operator).
+ * Additionally, a scalar is used when reading and writing data to allow for
+ * a shift in range.
  * As storage, the storage_type is used.
  *
  * This accessor uses row-major access, meaning neighboring z coordinates are
@@ -864,10 +866,18 @@ struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
  *
  * @tparam StorageType  Value type used for storing the actual value to memory
  *
+ * @tparam ScalarMask  Binary mask that marks which indices matter for the
+ *                     scalar selection (set bit means the corresponding index
+ *                     needs to be considered, 0 means it is not). The least
+ *                     significand bit corresponds to the first index dimension,
+ *                     the second least to the second index dimension, and so
+ *                     on.
+ *
  * @note  This class only manages the accesses and not the memory itself.
  * @note  Currently, only Dimensionality = 3 is supported.
  */
-template <int Dimensionality, typename ArithmeticType, typename StorageType>
+template <int Dimensionality, typename ArithmeticType, typename StorageType,
+          int32 ScalarMask = 0b0101>
 class scaled_reduced_row_major
     : public detail::enable_write_scalar<
           Dimensionality,
@@ -877,6 +887,7 @@ class scaled_reduced_row_major
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = StorageType;
     static constexpr size_type dimensionality{Dimensionality};
+    static constexpr int32 scalar_mask{ScalarMask};
     static constexpr bool is_const{std::is_const<storage_type>::value};
     using scalar_type =
         std::conditional_t<is_const, const arithmetic_type, arithmetic_type>;
@@ -887,6 +898,8 @@ class scaled_reduced_row_major
 
     static_assert(dimensionality == 3,
                   "Only Dimensionality == 3 is currently supported");
+    static_assert(scalar_mask == 0b0101,
+                  "Only StorageMask == 0b0101 is currently supported");
 
     // Allow access to both `scalar_` and `compute_scalar_index()`
     friend class detail::enable_write_scalar<
@@ -962,15 +975,15 @@ class scaled_reduced_row_major
      * Reads the scalar value at the given indices.
      *
      * @param x  x index
-     * @param z  z index
+     * @param y  y index
      *
      * @returns the scalar value at the given indices.
      */
     GKO_ATTRIBUTES GKO_INLINE constexpr scalar_type read_scalar(
-        size_type x, size_type z) const
+        size_type x, size_type y) const
     {
         const arithmetic_type *GKO_RESTRICT rest_scalar = scalar_;
-        return rest_scalar[compute_scalar_index(x, z)];
+        return rest_scalar[compute_scalar_index(x, y)];
     }
 
     /**

From 3f5cf9a22d394a06e11ffc433767f3b4bd1a3373 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 9 Nov 2020 17:27:09 +0100
Subject: [PATCH 0211/1008] Add multidimension support to new accessors

Additionally, split `range_accessors.hpp` into 3 different headers to
increase readability.
---
 core/test/base/range_accessors.cpp            |  65 +-
 .../core/base/range_accessor_helper.hpp       | 205 ++++++
 .../core/base/range_accessor_references.hpp   | 418 ++++++++++++
 include/ginkgo/core/base/range_accessors.hpp  | 602 +++++-------------
 include/ginkgo/ginkgo.hpp                     |   2 +
 5 files changed, 842 insertions(+), 450 deletions(-)
 create mode 100644 include/ginkgo/core/base/range_accessor_helper.hpp
 create mode 100644 include/ginkgo/core/base/range_accessor_references.hpp

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index b5b353991b3..e5d2ee38029 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -446,6 +446,60 @@ TEST_F(ReducedStorage3d, CanCreateSubrange2)
 }
 
 
+class ReducedStorageXd : public ::testing::Test {
+protected:
+    using span = gko::span;
+    using ar_type = double;
+    using st_type = float;
+    using size_type = gko::size_type;
+    static constexpr ar_type delta{::r<st_type>::value};
+
+    using accessor1d = gko::accessor::reduced_row_major<1, ar_type, st_type>;
+    using accessor2d = gko::accessor::reduced_row_major<2, ar_type, st_type>;
+    using const_accessor1d =
+        gko::accessor::reduced_row_major<1, ar_type, const st_type>;
+    using const_accessor2d =
+        gko::accessor::reduced_row_major<2, ar_type, const st_type>;
+
+    using reduced_storage1d = gko::range<accessor1d>;
+    using reduced_storage2d = gko::range<accessor2d>;
+    using const_reduced_storage2d = gko::range<const_accessor2d>;
+    using const_reduced_storage1d = gko::range<const_accessor1d>;
+
+    const std::array<const size_type, 0> stride0{};
+    const std::array<const size_type, 1> stride1{4};
+    const gko::dim<1> size_1d{8u};
+    const gko::dim<2> size_2d{2u, 4u};
+    static constexpr gko::size_type data_elements{8};
+    st_type data[data_elements]{1.1f, 2.2f, 3.3f, 4.4f,
+                                5.5f, 6.6f, 7.7f, -8.8f};
+    reduced_storage1d r1{data, size_1d, stride0};
+    reduced_storage2d r2{data, size_2d, stride1[0]};
+    const_reduced_storage1d cr1{data, size_1d, stride0};
+    const_reduced_storage2d cr2{data, size_2d, stride1};
+
+    void data_equal_except_for(int idx)
+    {
+        // clang-format off
+        if (idx != 0) { EXPECT_EQ(data[0], 1.1f); }
+        if (idx != 1) { EXPECT_EQ(data[1], 2.2f); }
+        if (idx != 2) { EXPECT_EQ(data[2], 3.3f); }
+        if (idx != 3) { EXPECT_EQ(data[3], 4.4f); }
+        if (idx != 4) { EXPECT_EQ(data[4], 5.5f); }
+        if (idx != 5) { EXPECT_EQ(data[5], 6.6f); }
+        if (idx != 6) { EXPECT_EQ(data[6], 7.7f); }
+        if (idx != 7) { EXPECT_EQ(data[7], -8.8f); }
+        // clang-format on
+    }
+};
+
+TEST_F(ReducedStorageXd, CanRead)
+{
+    EXPECT_EQ(cr1(1), 2.2f);
+    EXPECT_EQ(cr2(0, 1), 2.2f);
+}
+
+
 class ScaledReducedStorage3d : public ::testing::Test {
 protected:
     using span = gko::span;
@@ -457,9 +511,10 @@ class ScaledReducedStorage3d : public ::testing::Test {
     static constexpr ar_type delta{::r<ar_type>::value};
 
     using accessor =
-        gko::accessor::scaled_reduced_row_major<3, ar_type, st_type>;
+        gko::accessor::scaled_reduced_row_major<3, ar_type, st_type, 0b0101>;
     using const_accessor =
-        gko::accessor::scaled_reduced_row_major<3, ar_type, const st_type>;
+        gko::accessor::scaled_reduced_row_major<3, ar_type, const st_type,
+                                                0b0101>;
 
     using reduced_storage = gko::range<accessor>;
     using const_reduced_storage = gko::range<const_accessor>;
@@ -593,7 +648,7 @@ TEST_F(ScaledReducedStorage3d, Subrange)
 
 TEST_F(ScaledReducedStorage3d, CanWriteScale)
 {
-    r->write_scalar(0, 0, 10.);
+    r->write_scalar(10., 0, 0, 0);
 
     EXPECT_EQ(r(0, 0, 0), 100.);
     EXPECT_EQ(r(0, 0, 1), 22.);
@@ -608,8 +663,8 @@ TEST_F(ScaledReducedStorage3d, CanWriteScale)
 
 TEST_F(ScaledReducedStorage3d, CanReadScale)
 {
-    EXPECT_EQ(r->read_scalar(0, 0), 1.);
-    EXPECT_EQ(r->read_scalar(0, 1), 2.);
+    EXPECT_EQ(r->read_scalar(0, 0, 0), 1.);
+    EXPECT_EQ(r->read_scalar(0, 0, 1), 2.);
 }
 
 
diff --git a/include/ginkgo/core/base/range_accessor_helper.hpp b/include/ginkgo/core/base/range_accessor_helper.hpp
new file mode 100644
index 00000000000..33db18f9534
--- /dev/null
+++ b/include/ginkgo/core/base/range_accessor_helper.hpp
@@ -0,0 +1,205 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_BASE_RANGE_ACCESSOR_HELPER_HPP_
+#define GKO_CORE_BASE_RANGE_ACCESSOR_HELPER_HPP_
+
+#include <array>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+
+#include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/range.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+namespace gko {
+namespace accessor {
+/**
+ * This namespace contains helper functionality for the accessors.
+ *
+ * @note This namespace is not part of the public interface and can change
+ * without notice.
+ */
+namespace helper {
+
+
+/**
+ * This helper runs from first to last dimension in order to compute the index.
+ * The index is computed like this:
+ * indices: x1, x2, x3, ...
+ * compute(stride, x1, x2, x3) -> x1 * stride[0] + x2 * stride[1] + x3
+ */
+template <typename IndexType, int32 total_dim, int32 current_iter = total_dim>
+struct row_major_index {
+    static constexpr int32 current_dim{total_dim - current_iter};
+    template <typename FirstType, typename... Indices>
+    static constexpr IndexType compute(
+        const dim<total_dim> &size,
+        const std::array<IndexType, (total_dim > 1 ? total_dim - 1 : 0)>
+            &stride,
+        FirstType &&first, Indices &&... idxs)
+    {
+        return GKO_ASSERT(first < size[current_dim]),
+               first * stride[current_dim] +
+                   row_major_index<IndexType, total_dim, current_iter - 1>::
+                       compute(size, stride, std::forward<Indices>(idxs)...);
+    }
+};
+
+template <typename IndexType, int32 total_dim>
+struct row_major_index<IndexType, total_dim, 1> {
+    static constexpr int32 current_dim{total_dim - 1};
+    template <typename FirstType>
+    static constexpr IndexType compute(
+        const dim<total_dim> &size,
+        const std::array<IndexType, (total_dim > 1 ? total_dim - 1 : 0)>,
+        FirstType &&first)
+    {
+        return GKO_ASSERT(first < size[current_dim]), first;
+    }
+};
+
+
+/**
+ * This helper must be walked through in reverse (highest to lowest dimension)
+ * in order to properly use the stride for the scalar indices. The mask
+ * indicates which indices are actually used. The least significant bit set
+ * means using the last index, second bit corresponds to the second last
+ * dimension, and so on.
+ *
+ * Basically, this computes indices in a similar fashion as `row_major_index`
+ * while having a mask signaling which indices to skip.
+ *
+ * Example: Mask = 0b01101
+ * compute(stride, tuple(x1, x2, x3, x4))
+ * -> x4 + x2 * stride[2] + x1 * stride[1]
+ */
+template <typename IndexType, int32 total_dim, int32 mask,
+          int32 current_dim = total_dim - 1,
+          int32 current_scalar = total_dim - 1>
+struct row_major_mask_index {
+    static constexpr int32 msb_mask{static_cast<int32>(1)};
+    static constexpr int32 stride_size = total_dim > 1 ? total_dim - 1 : 0;
+    static constexpr int32 use_current_stride{mask & static_cast<int32>(1)};
+    static constexpr bool is_last_stride_dim{stride_size == 0 ||
+                                             current_scalar == total_dim - 1};
+
+    template <typename... Args>
+    static constexpr IndexType compute(
+        const dim<total_dim> &size,
+        const std::array<IndexType, stride_size> &stride,
+        std::tuple<Args...> indices)
+    {
+        return GKO_ASSERT(std::get<current_dim>(indices) < size[current_dim]),
+               (use_current_stride
+                    ? std::get<current_dim>(indices) *
+                          (is_last_stride_dim ? 1 : stride[current_scalar])
+                    : 0) +
+                   row_major_mask_index<
+                       IndexType, total_dim, (mask >> 1), current_dim - 1,
+                       current_scalar - use_current_stride>::compute(size,
+                                                                     stride,
+                                                                     indices);
+    }
+};
+
+template <typename IndexType, int32 total_dim, int32 mask, int32 current_scalar>
+struct row_major_mask_index<IndexType, total_dim, mask, 0, current_scalar> {
+    static constexpr int32 stride_size = total_dim > 1 ? total_dim - 1 : 0;
+    static constexpr int32 use_current_stride{mask & static_cast<int32>(1)};
+    static constexpr int32 current_dim{0};
+    static constexpr bool is_last_stride_dim{stride_size == 0 ||
+                                             current_scalar == total_dim - 1};
+
+    template <typename... Args>
+    static constexpr IndexType compute(
+        const dim<total_dim> &size,
+        const std::array<IndexType, stride_size> &stride,
+        std::tuple<Args...> indices)
+    {
+        return GKO_ASSERT(std::get<current_dim>(indices) < size[current_dim]),
+               (use_current_stride
+                    ? std::get<current_dim>(indices) *
+                          (is_last_stride_dim ? 1 : stride[current_scalar])
+                    : 0);
+    }
+};
+
+
+template <typename... Args>
+struct are_all_integral : public std::true_type {
+};
+
+template <typename First, typename... Args>
+struct are_all_integral<First, Args...>
+    : public std::conditional<std::is_integral<std::decay_t<First>>::value,
+                              are_all_integral<Args...>,
+                              std::false_type>::type {
+};
+
+
+namespace detail {
+
+
+template <bool has_span, typename... Args>
+struct are_span_compatible_impl
+    : public std::integral_constant<bool, has_span> {
+};
+
+template <bool has_span, typename First, typename... Args>
+struct are_span_compatible_impl<has_span, First, Args...>
+    : public std::conditional<
+          std::is_integral<std::decay_t<First>>::value ||
+              std::is_same<std::decay_t<First>, span>::value,
+          are_span_compatible_impl<
+              has_span || std::is_same<std::decay_t<First>, span>::value,
+              Args...>,
+          std::false_type>::type {
+};
+
+
+}  // namespace detail
+
+
+template <typename... Args>
+using are_span_compatible = detail::are_span_compatible_impl<false, Args...>;
+
+
+}  // namespace helper
+}  // namespace accessor
+}  // namespace gko
+
+
+#endif  // GKO_CORE_BASE_RANGE_ACCESSOR_HELPER_HPP_
diff --git a/include/ginkgo/core/base/range_accessor_references.hpp b/include/ginkgo/core/base/range_accessor_references.hpp
new file mode 100644
index 00000000000..1228e309089
--- /dev/null
+++ b/include/ginkgo/core/base/range_accessor_references.hpp
@@ -0,0 +1,418 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_BASE_RANGE_ACCESSOR_REFERENCES_HPP_
+#define GKO_CORE_BASE_RANGE_ACCESSOR_REFERENCES_HPP_
+
+
+#include <type_traits>
+#include <utility>
+
+
+#include <ginkgo/core/base/std_extensions.hpp>
+
+
+// CUDA TOOLKIT < 11 does not support constexpr in combination with
+// thrust::complex, which is why constexpr is only present in later versions
+#if defined(__CUDA_ARCH__) && defined(__CUDACC_VER_MAJOR__) && \
+    (__CUDACC_VER_MAJOR__ < 11)
+
+#define GKO_ENABLE_REFERENCE_CONSTEXPR
+
+#else
+
+#define GKO_ENABLE_REFERENCE_CONSTEXPR constexpr
+
+#endif
+
+
+namespace gko {
+namespace accessor {
+/**
+ * This namespace is not part of the public interface and can change without
+ * notice.
+ */
+namespace detail {
+
+
+// tests if the cast operator to `ValueType` is present
+template <typename Ref, typename ValueType, typename = xstd::void_t<>>
+struct has_cast_operator : std::false_type {
+};
+
+template <typename Ref, typename ValueType>
+struct has_cast_operator<
+    Ref, ValueType,
+    xstd::void_t<decltype(std::declval<Ref>().Ref::operator ValueType())>>
+    : std::true_type {
+};
+
+
+/**
+ * @internal
+ * converts `ref` to ValueType while preferring the cast operator overload
+ * from class `Ref` before falling back to a simple
+ * `static_cast<ValueType>`.
+ *
+ * This function is only needed for CUDA TOOLKIT < 11 because
+ * thrust::complex has a constructor call: `template<T> complex(const T
+ * &other) : real(other), imag()`, which is always preferred over the
+ * overloaded `operator value_type()`.
+ */
+template <typename ValueType, typename Ref>
+GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
+    has_cast_operator<Ref, ValueType>::value, ValueType>
+to_value_type(const Ref &ref)
+{
+    return ref.Ref::operator ValueType();
+}
+
+template <typename ValueType, typename Ref>
+GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
+    !has_cast_operator<Ref, ValueType>::value, ValueType>
+to_value_type(const Ref &ref)
+{
+    return static_cast<ValueType>(ref);
+}
+
+
+/**
+ * This is a mixin which defines the binary operators for *, /, +, - for the
+ * Reference class, the unary operator -, and the assignment operators
+ * *=, /=, +=, -=
+ * All assignment operators expect an rvalue reference (Reference &&) for
+ * the Reference class in order to prevent copying the Reference object.
+ *
+ * @tparam Reference  The reference class this mixin provides operator
+ * overloads for. The reference class needs to overload the cast operator
+ * to ValueType
+ *
+ * @tparam ArithmeticType  arithmetic type the Reference class is supposed
+ * to represent.
+ *
+ * @warning  This struct should only be used by reference classes.
+ */
+template <typename Reference, typename ArithmeticType>
+struct enable_reference {
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
+
+#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_oper, _op)          \
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR \
+        arithmetic_type                                             \
+        _oper(const Reference &ref1, const Reference &ref2)         \
+    {                                                               \
+        return to_value_type<arithmetic_type>(ref1)                 \
+            _op to_value_type<arithmetic_type>(ref2);               \
+    }                                                               \
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR \
+        arithmetic_type                                             \
+        _oper(const Reference &ref, const arithmetic_type &a)       \
+    {                                                               \
+        return to_value_type<arithmetic_type>(ref) _op a;           \
+    }                                                               \
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR \
+        arithmetic_type                                             \
+        _oper(const arithmetic_type &a, const Reference &ref)       \
+    {                                                               \
+        return a _op to_value_type<arithmetic_type>(ref);           \
+    }
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator*, *)
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator/, /)
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator+, +)
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator-, -)
+#undef GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD
+
+#define GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(_oper, _op)             \
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR        \
+        arithmetic_type                                                    \
+        _oper(Reference &&ref1, const Reference &ref2)                     \
+    {                                                                      \
+        return std::move(ref1) = to_value_type<arithmetic_type>(ref1)      \
+                   _op to_value_type<arithmetic_type>(ref2);               \
+    }                                                                      \
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR        \
+        arithmetic_type                                                    \
+        _oper(Reference &&ref, const arithmetic_type &a)                   \
+    {                                                                      \
+        return std::move(ref) = to_value_type<arithmetic_type>(ref) _op a; \
+    }
+
+    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator*=, *)
+    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator/=, /)
+    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator+=, +)
+    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator-=, -)
+#undef GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD
+
+    // TODO test if comparison operators need to be overloaded as well
+
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR
+        arithmetic_type
+        operator-(const Reference &ref)
+    {
+        return -to_value_type<arithmetic_type>(ref);
+    }
+};
+
+
+}  // namespace detail
+
+
+/**
+ * This namespace contains reference classes used inside accessors.
+ *
+ * @warning  These classes should not be used by anything else but accessors.
+ */
+namespace reference_class {
+
+
+/**
+ * Reference class for a different storage than arithmetic type. The
+ * conversion between both formats is done with a simple static_cast.
+ *
+ * Copying this reference is disabled, but move construction is possible to
+ * allow for an additional layer (like gko::range).
+ * The assignment operator only works for an rvalue reference (&&) to
+ * prevent accidental copying the reference and working on a reference.
+ *
+ * @tparam ArithmeticType  Type used for arithmetic operations, therefore,
+ * the type which is used for input and output of this class.
+ *
+ * @tparam StorageType  Type actually used as a storage, which is converted
+ * to ArithmeticType before usage
+ */
+template <typename ArithmeticType, typename StorageType>
+class reduced_storage
+    : public detail::enable_reference<
+          reduced_storage<ArithmeticType, StorageType>, ArithmeticType> {
+public:
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
+    using storage_type = StorageType;
+
+    // Allow move construction, so perfect forwarding is possible (required
+    // for `range` support)
+    reduced_storage(reduced_storage &&) = default;
+
+    reduced_storage() = delete;
+
+    ~reduced_storage() = default;
+
+    // Forbid copy construction
+    reduced_storage(const reduced_storage &) = delete;
+
+    GKO_ATTRIBUTES constexpr reduced_storage(
+        storage_type *const GKO_RESTRICT ptr)
+        : ptr_{ptr}
+    {}
+
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    {
+        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return static_cast<arithmetic_type>(*r_ptr);
+    }
+
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
+        arithmetic_type val) &&
+    {
+        storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return *r_ptr = static_cast<storage_type>(val);
+    }
+
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
+        const reduced_storage &ref) &&
+    {
+        return std::move(*this) = static_cast<arithmetic_type>(ref);
+    }
+
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
+        reduced_storage &&ref) &&
+    {
+        return std::move(*this) = static_cast<arithmetic_type>(ref);
+    }
+
+private:
+    storage_type *const GKO_RESTRICT ptr_;
+};
+
+// Specialization for const storage_type to prevent `operator=`
+template <typename ArithmeticType, typename StorageType>
+class reduced_storage<ArithmeticType, const StorageType>
+    : public detail::enable_reference<
+          reduced_storage<ArithmeticType, const StorageType>, ArithmeticType> {
+public:
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
+    using storage_type = const StorageType;
+
+    // Allow move construction, so perfect forwarding is possible
+    reduced_storage(reduced_storage &&) = default;
+
+    reduced_storage() = delete;
+
+    ~reduced_storage() = default;
+
+    // Forbid copy construction and move assignment
+    reduced_storage(const reduced_storage &) = delete;
+
+    reduced_storage &operator=(reduced_storage &&) = delete;
+
+    GKO_ATTRIBUTES constexpr reduced_storage(
+        storage_type *const GKO_RESTRICT ptr)
+        : ptr_{ptr}
+    {}
+
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    {
+        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return static_cast<arithmetic_type>(*r_ptr);
+    }
+
+private:
+    storage_type *const GKO_RESTRICT ptr_;
+};
+
+
+/**
+ * Reference class for a different storage than arithmetic type with the
+ * addition of a scaling factor. The conversion between both formats is done
+ * with a static_cast to the ArithmeticType, followed by a multiplication
+ * of the scalar (when reading; for writing, the new value is divided by the
+ * scalar before casting to the StorageType).
+ *
+ * Copying this reference is disabled, but move construction is possible to
+ * allow for an additional layer (like gko::range).
+ * The assignment operator only works for an rvalue reference (&&) to
+ * prevent accidental copying the reference and working on a reference.
+ *
+ * @tparam ArithmeticType  Type used for arithmetic operations, therefore,
+ * the type which is used for input and output of this class.
+ *
+ * @tparam StorageType  Type actually used as a storage, which is converted
+ * to ArithmeticType before usage
+ */
+template <typename ArithmeticType, typename StorageType>
+class scaled_reduced_storage
+    : public detail::enable_reference<
+          scaled_reduced_storage<ArithmeticType, StorageType>, ArithmeticType> {
+public:
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
+    using storage_type = StorageType;
+
+    // Allow move construction, so perfect forwarding is possible
+    scaled_reduced_storage(scaled_reduced_storage &&) = default;
+
+    scaled_reduced_storage() = delete;
+
+    ~scaled_reduced_storage() = default;
+
+    // Forbid copy construction
+    scaled_reduced_storage(const scaled_reduced_storage &) = delete;
+
+    GKO_ATTRIBUTES constexpr scaled_reduced_storage(
+        storage_type *const GKO_RESTRICT ptr, arithmetic_type scalar)
+        : ptr_{ptr}, scalar_{scalar}
+    {}
+
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    {
+        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return static_cast<arithmetic_type>(*r_ptr) * scalar_;
+    }
+
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
+        arithmetic_type val) &&
+    {
+        storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return *r_ptr = static_cast<storage_type>(val / scalar_), val;
+    }
+
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
+        const scaled_reduced_storage &ref) &&
+    {
+        return std::move(*this) = static_cast<arithmetic_type>(ref);
+    }
+
+    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
+        scaled_reduced_storage &&ref) &&
+    {
+        return std::move(*this) = static_cast<arithmetic_type>(ref);
+    }
+
+private:
+    storage_type *const GKO_RESTRICT ptr_;
+    const arithmetic_type scalar_;
+};
+
+// Specialization for constant storage_type (no `operator=`)
+template <typename ArithmeticType, typename StorageType>
+class scaled_reduced_storage<ArithmeticType, const StorageType>
+    : public detail::enable_reference<
+          scaled_reduced_storage<ArithmeticType, const StorageType>,
+          ArithmeticType> {
+public:
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
+    using storage_type = const StorageType;
+
+    // Allow move construction, so perfect forwarding is possible
+    scaled_reduced_storage(scaled_reduced_storage &&) = default;
+
+    scaled_reduced_storage() = delete;
+
+    ~scaled_reduced_storage() = default;
+
+    // Forbid copy construction and move assignment
+    scaled_reduced_storage(const scaled_reduced_storage &) = delete;
+
+    scaled_reduced_storage &operator=(scaled_reduced_storage &&) = delete;
+
+    GKO_ATTRIBUTES constexpr scaled_reduced_storage(
+        storage_type *const GKO_RESTRICT ptr, arithmetic_type scalar)
+        : ptr_{ptr}, scalar_{scalar}
+    {}
+
+    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    {
+        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return static_cast<arithmetic_type>(*r_ptr) * scalar_;
+    }
+
+
+private:
+    storage_type *const GKO_RESTRICT ptr_;
+    const arithmetic_type scalar_;
+};
+
+
+}  // namespace reference_class
+}  // namespace accessor
+}  // namespace gko
+
+
+#endif  // GKO_CORE_BASE_RANGE_ACCESSOR_REFERENCES_HPP_
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index cfd9fbc6893..454d55925dd 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -36,31 +36,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <array>
 #include <memory>
+#include <tuple>
 #include <type_traits>
+#include <utility>
 
 
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/dim.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/range.hpp>
+#include <ginkgo/core/base/range_accessor_helper.hpp>
+#include <ginkgo/core/base/range_accessor_references.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/types.hpp>
 
 
-// CUDA TOOLKIT < 11 does not support constexpr in combination with
-// thrust::complex, which is why constexpr is only present in later versions
-#if defined(__CUDA_ARCH__) && defined(__CUDACC_VER_MAJOR__) && \
-    (__CUDACC_VER_MAJOR__ < 11)
-
-#define GKO_ENABLE_REFERENCE_CONSTEXPR
-
-#else
-
-#define GKO_ENABLE_REFERENCE_CONSTEXPR constexpr
-
-#endif
-
-
 namespace gko {
 /**
  * @brief The accessor namespace.
@@ -226,354 +216,6 @@ class row_major {
 };
 
 
-namespace detail {
-
-
-// tests if the cast operator to `ValueType` is present
-template <typename Ref, typename ValueType, typename = xstd::void_t<>>
-struct has_cast_operator : std::false_type {};
-
-template <typename Ref, typename ValueType>
-struct has_cast_operator<
-    Ref, ValueType,
-    xstd::void_t<decltype(std::declval<Ref>().Ref::operator ValueType())>>
-    : std::true_type {};
-
-
-/**
- * @internal
- * converts `ref` to ValueType while preferring the cast operator overload
- * from class `Ref` before falling back to a simple
- * `static_cast<ValueType>`.
- *
- * This function is only needed for CUDA TOOLKIT < 11 because
- * thrust::complex has a constructor call: `template<T> complex(const T
- * &other) : real(other), imag()`, which is always preferred over the
- * overloaded `operator value_type()`.
- */
-template <typename ValueType, typename Ref>
-GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
-    has_cast_operator<Ref, ValueType>::value, ValueType>
-to_value_type(const Ref &ref)
-{
-    return ref.Ref::operator ValueType();
-}
-
-template <typename ValueType, typename Ref>
-GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
-    !has_cast_operator<Ref, ValueType>::value, ValueType>
-to_value_type(const Ref &ref)
-{
-    return static_cast<ValueType>(ref);
-}
-
-
-/**
- * This is a mixin which defines the binary operators for *, /, +, - for the
- * Reference class, the unary operator -, and the assignment operators
- * *=, /=, +=, -=
- * All assignment operators expect an rvalue reference (Reference &&) for
- * the Reference class in order to prevent copying the Reference object.
- *
- * @tparam Reference  The reference class this mixin provides operator
- * overloads for. The reference class needs to overload the cast operator
- * to ValueType
- *
- * @tparam ArithmeticType  arithmetic type the Reference class is supposed
- * to represent.
- *
- * @warning  This struct should only be used by reference classes.
- */
-template <typename Reference, typename ArithmeticType>
-struct enable_reference {
-    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
-
-#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_oper, _op)          \
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR \
-        arithmetic_type                                             \
-        _oper(const Reference &ref1, const Reference &ref2)         \
-    {                                                               \
-        return to_value_type<arithmetic_type>(ref1)                 \
-            _op to_value_type<arithmetic_type>(ref2);               \
-    }                                                               \
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR \
-        arithmetic_type                                             \
-        _oper(const Reference &ref, const arithmetic_type &a)       \
-    {                                                               \
-        return to_value_type<arithmetic_type>(ref) _op a;           \
-    }                                                               \
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR \
-        arithmetic_type                                             \
-        _oper(const arithmetic_type &a, const Reference &ref)       \
-    {                                                               \
-        return a _op to_value_type<arithmetic_type>(ref);           \
-    }
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator*, *)
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator/, /)
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator+, +)
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator-, -)
-#undef GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD
-
-#define GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(_oper, _op)             \
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR        \
-        arithmetic_type                                                    \
-        _oper(Reference &&ref1, const Reference &ref2)                     \
-    {                                                                      \
-        return std::move(ref1) = to_value_type<arithmetic_type>(ref1)      \
-                   _op to_value_type<arithmetic_type>(ref2);               \
-    }                                                                      \
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR        \
-        arithmetic_type                                                    \
-        _oper(Reference &&ref, const arithmetic_type &a)                   \
-    {                                                                      \
-        return std::move(ref) = to_value_type<arithmetic_type>(ref) _op a; \
-    }
-
-    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator*=, *)
-    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator/=, /)
-    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator+=, +)
-    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator-=, -)
-#undef GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD
-
-    // TODO test if comparison operators need to be overloaded as well
-
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR
-        arithmetic_type
-        operator-(const Reference &ref)
-    {
-        return -to_value_type<arithmetic_type>(ref);
-    }
-};
-
-
-/**
- * This namespace contains reference classes used inside accessors.
- *
- * @warning  These classes should not be used by anything else but accessors.
- */
-namespace reference_class {
-
-
-/**
- * Reference class for a different storage than arithmetic type. The
- * conversion between both formats is done with a simple static_cast.
- *
- * Copying this reference is disabled, but move construction is possible to
- * allow for an additional layer (like gko::range).
- * The assignment operator only works for an rvalue reference (&&) to
- * prevent accidental copying the reference and working on a reference.
- *
- * @tparam ArithmeticType  Type used for arithmetic operations, therefore,
- * the type which is used for input and output of this class.
- *
- * @tparam StorageType  Type actually used as a storage, which is converted
- * to ArithmeticType before usage
- */
-template <typename ArithmeticType, typename StorageType>
-class reduced_storage
-    : public detail::enable_reference<
-          reduced_storage<ArithmeticType, StorageType>, ArithmeticType> {
-public:
-    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
-    using storage_type = StorageType;
-
-    // Allow move construction, so perfect forwarding is possible (required
-    // for `range` support)
-    reduced_storage(reduced_storage &&) = default;
-
-    reduced_storage() = delete;
-
-    ~reduced_storage() = default;
-
-    // Forbid copy construction
-    reduced_storage(const reduced_storage &) = delete;
-
-    GKO_ATTRIBUTES constexpr reduced_storage(
-        storage_type *const GKO_RESTRICT ptr)
-        : ptr_{ptr}
-    {}
-
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
-    {
-        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return static_cast<arithmetic_type>(*r_ptr);
-    }
-
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
-        arithmetic_type val) &&
-    {
-        storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return *r_ptr = static_cast<storage_type>(val);
-    }
-
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
-        const reduced_storage &ref) &&
-    {
-        return std::move(*this) = static_cast<arithmetic_type>(ref);
-    }
-
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
-        reduced_storage &&ref) &&
-    {
-        return std::move(*this) = static_cast<arithmetic_type>(ref);
-    }
-
-private:
-    storage_type *const GKO_RESTRICT ptr_;
-};
-
-// Specialization for const storage_type to prevent `operator=`
-template <typename ArithmeticType, typename StorageType>
-class reduced_storage<ArithmeticType, const StorageType>
-    : public detail::enable_reference<
-          reduced_storage<ArithmeticType, const StorageType>, ArithmeticType> {
-public:
-    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
-    using storage_type = const StorageType;
-
-    // Allow move construction, so perfect forwarding is possible
-    reduced_storage(reduced_storage &&) = default;
-
-    reduced_storage() = delete;
-
-    ~reduced_storage() = default;
-
-    // Forbid copy construction and move assignment
-    reduced_storage(const reduced_storage &) = delete;
-
-    reduced_storage &operator=(reduced_storage &&) = delete;
-
-    GKO_ATTRIBUTES constexpr reduced_storage(
-        storage_type *const GKO_RESTRICT ptr)
-        : ptr_{ptr}
-    {}
-
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
-    {
-        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return static_cast<arithmetic_type>(*r_ptr);
-    }
-
-private:
-    storage_type *const GKO_RESTRICT ptr_;
-};
-
-
-/**
- * Reference class for a different storage than arithmetic type with the
- * addition of a scaling factor. The conversion between both formats is done
- * with a static_cast to the ArithmeticType, followed by a multiplication
- * of the scalar (when reading; for writing, the new value is divided by the
- * scalar before casting to the StorageType).
- *
- * Copying this reference is disabled, but move construction is possible to
- * allow for an additional layer (like gko::range).
- * The assignment operator only works for an rvalue reference (&&) to
- * prevent accidental copying the reference and working on a reference.
- *
- * @tparam ArithmeticType  Type used for arithmetic operations, therefore,
- * the type which is used for input and output of this class.
- *
- * @tparam StorageType  Type actually used as a storage, which is converted
- * to ArithmeticType before usage
- */
-template <typename ArithmeticType, typename StorageType>
-class scaled_reduced_storage
-    : public detail::enable_reference<
-          scaled_reduced_storage<ArithmeticType, StorageType>, ArithmeticType> {
-public:
-    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
-    using storage_type = StorageType;
-
-    // Allow move construction, so perfect forwarding is possible
-    scaled_reduced_storage(scaled_reduced_storage &&) = default;
-
-    scaled_reduced_storage() = delete;
-
-    ~scaled_reduced_storage() = default;
-
-    // Forbid copy construction
-    scaled_reduced_storage(const scaled_reduced_storage &) = delete;
-
-    GKO_ATTRIBUTES constexpr scaled_reduced_storage(
-        storage_type *const GKO_RESTRICT ptr, arithmetic_type scalar)
-        : ptr_{ptr}, scalar_{scalar}
-    {}
-
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
-    {
-        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return static_cast<arithmetic_type>(*r_ptr) * scalar_;
-    }
-
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
-        arithmetic_type val) &&
-    {
-        storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return *r_ptr = static_cast<storage_type>(val / scalar_), val;
-    }
-
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
-        const scaled_reduced_storage &ref) &&
-    {
-        return std::move(*this) = static_cast<arithmetic_type>(ref);
-    }
-
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
-        scaled_reduced_storage &&ref) &&
-    {
-        return std::move(*this) = static_cast<arithmetic_type>(ref);
-    }
-
-private:
-    storage_type *const GKO_RESTRICT ptr_;
-    const arithmetic_type scalar_;
-};
-
-// Specialization for constant storage_type (no `operator=`)
-template <typename ArithmeticType, typename StorageType>
-class scaled_reduced_storage<ArithmeticType, const StorageType>
-    : public detail::enable_reference<
-          scaled_reduced_storage<ArithmeticType, const StorageType>,
-          ArithmeticType> {
-public:
-    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
-    using storage_type = const StorageType;
-
-    // Allow move construction, so perfect forwarding is possible
-    scaled_reduced_storage(scaled_reduced_storage &&) = default;
-
-    scaled_reduced_storage() = delete;
-
-    ~scaled_reduced_storage() = default;
-
-    // Forbid copy construction and move assignment
-    scaled_reduced_storage(const scaled_reduced_storage &) = delete;
-
-    scaled_reduced_storage &operator=(scaled_reduced_storage &&) = delete;
-
-    GKO_ATTRIBUTES constexpr scaled_reduced_storage(
-        storage_type *const GKO_RESTRICT ptr, arithmetic_type scalar)
-        : ptr_{ptr}, scalar_{scalar}
-    {}
-
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
-    {
-        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return static_cast<arithmetic_type>(*r_ptr) * scalar_;
-    }
-
-
-private:
-    storage_type *const GKO_RESTRICT ptr_;
-    const arithmetic_type scalar_;
-};
-
-
-}  // namespace reference_class
-}  // namespace detail
-
-
 /**
  * The reduced_row_major class allows a storage format that is different from
  * the arithmetic format (which is returned from the brace operator).
@@ -591,7 +233,6 @@ class scaled_reduced_storage<ArithmeticType, const StorageType>
  * @tparam StorageType  Value type used for storing the actual value to memory
  *
  * @note  This class only manages the accesses and not the memory itself.
- * @note  Currently, only Dimensionality = 3 is supported.
  */
 template <int Dimensionality, typename ArithmeticType, typename StorageType>
 class reduced_row_major {
@@ -603,14 +244,11 @@ class reduced_row_major {
     using const_accessor =
         reduced_row_major<dimensionality, arithmetic_type, const storage_type>;
 
-    static_assert(dimensionality == 3,
-                  "Only Dimensionality == 3 is currently supported");
-
     friend class range<reduced_row_major>;
 
 protected:
     using reference_type =
-        detail::reference_class::reduced_storage<arithmetic_type, storage_type>;
+        reference_class::reduced_storage<arithmetic_type, storage_type>;
 
     /**
      * Creates the accessor with an already allocated storage space with a
@@ -619,15 +257,34 @@ class reduced_row_major {
      *
      * @param storage  pointer to the block of memory containing the storage
      * @param size  multidimensional size of the memory
-     * @param stride0  stride used for the x-indices
-     * @param stride1  stride used for the y-indices
+     * @param stride  stride array used for the x-indices
      */
+    GKO_ATTRIBUTES constexpr reduced_row_major(
+        storage_type *storage, gko::dim<dimensionality> size,
+        const std::array<const size_type, dimensionality - 1> &stride)
+        : storage_{storage}, size_{size}, stride_{stride}
+    {}
+
+    /**
+     * Creates the accessor with an already allocated storage space with a
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
+     *
+     * @param storage  pointer to the block of memory containing the storage
+     * @param size  multidimensional size of the memory
+     * @param stride  stride array used for the x-indices
+     */
+    template <typename... Strides>
     GKO_ATTRIBUTES constexpr reduced_row_major(storage_type *storage,
                                                gko::dim<dimensionality> size,
-                                               size_type stride0,
-                                               size_type stride1)
-        : storage_{storage}, size_{size}, stride_{stride0, stride1}
-    {}
+                                               Strides &&... stride)
+        : storage_{storage},
+          size_{size},
+          stride_{std::forward<Strides>(stride)...}
+    {
+        static_assert(sizeof...(Strides) == dimensionality - 1,
+                      "Number of provided Strides must be dimensionality - 1!");
+    }
 
     /**
      * Creates the accessor with an already allocated storage space with a
@@ -640,7 +297,7 @@ class reduced_row_major {
      */
     GKO_ATTRIBUTES constexpr reduced_row_major(storage_type *storage,
                                                dim<dimensionality> size)
-        : reduced_row_major{storage, size, size[1] * size[2], size[2]}
+        : reduced_row_major{storage, size, {{size[1] * size[2], size[2]}}}
     {}
 
     /**
@@ -659,7 +316,7 @@ class reduced_row_major {
      */
     GKO_ATTRIBUTES GKO_INLINE constexpr range<const_accessor> to_const() const
     {
-        return range<const_accessor>{storage_, size_, stride_[0], stride_[1]};
+        return range<const_accessor>{storage_, size_, stride_};
     }
 
     /**
@@ -703,32 +360,41 @@ class reduced_row_major {
      * Returns the stored value for the given indices. If the storage is const,
      * a value is returned, otherwise, a reference is returned.
      *
-     * @param x  x index
-     * @param y  y index
-     * @param z  z index
+     * @param indices  indices which value is supposed to access
      *
      * @returns  the stored value if the accessor is const (if the storage type
-     * is const), or a reference if the accessor is non-const
+     *           is const), or a reference if the accessor is non-const
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr std::conditional_t<
-        is_const, arithmetic_type, reference_type>
-    operator()(size_type x, size_type y, size_type z) const
+    template <typename... Indices>
+    GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
+        helper::are_all_integral<Indices...>::value,
+        std::conditional_t<is_const, arithmetic_type, reference_type>>
+    operator()(Indices &&... indices) const
     {
-        return reference_type{this->storage_ + compute_index(x, y, z)};
+        return reference_type{this->storage_ +
+                              compute_index(std::forward<Indices>(indices)...)};
     }
 
     /**
-     * Returns a sub-range spinning the range (x_span, y_span, z_span)
+     * Returns a sub-range spinning the current range (x1_span, x2_span, ...)
      *
-     * @param x  span for the x indices
-     * @param y  span for the y indices
-     * @param z  span for the z indices
+     * @param spans  span for the indices
      *
      * @returns a sub-range for the given spans.
      */
-    GKO_ATTRIBUTES constexpr range<reduced_row_major> operator()(
-        const span &x_span, const span &y_span, const span &z_span) const
-    {
+    template <typename... SpanTypes>
+    GKO_ATTRIBUTES constexpr std::enable_if_t<
+        helper::are_span_compatible<SpanTypes...>::value,
+        range<reduced_row_major>>
+    operator()(SpanTypes &&... spans) const
+    {
+        // TODO: Add validation check for spans!
+        return range<reduced_row_major>{
+            storage_ + compute_index((static_cast<span>(spans).begin)...),
+            dim<dimensionality>{(static_cast<span>(spans).end -
+                                 static_cast<span>(spans).begin)...},
+            stride_};
+        /*
         return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
                GKO_ASSERT(z_span.is_valid()),
                GKO_ASSERT(x_span <= span{size_[0]}),
@@ -741,6 +407,7 @@ class reduced_row_major {
                                        y_span.end - y_span.begin,
                                        z_span.end - z_span.begin},
                    stride_[0], stride_[1]);
+        */
     }
 
     /**
@@ -787,11 +454,16 @@ class reduced_row_major {
     }
 
 protected:
+    template <typename... Indices>
     GKO_ATTRIBUTES GKO_INLINE constexpr size_type compute_index(
-        size_type x, size_type y, size_type z) const
+        Indices &&... indices) const
     {
-        return GKO_ASSERT(x < size_[0]), GKO_ASSERT(y < size_[1]),
-               GKO_ASSERT(z < size_[2]), x * stride_[0] + y * stride_[1] + z;
+        static_assert(sizeof...(Indices) == dimensionality,
+                      "Number of indices must match dimensionality!");
+        return helper::row_major_index<
+            const size_type, dimensionality>::compute(size_, stride_,
+                                                      std::forward<Indices>(
+                                                          indices)...);
     }
 
     storage_type *storage_;
@@ -824,17 +496,20 @@ struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
     /**
      * Writes the scalar value at the given indices.
      *
-     * @param x  x index
-     * @param z  z index
-     * @param value  value to write.
+     * @param value  value to write
+     * @param indices  indices where to write the value
      *
      * @returns the written value.
      */
+    template <typename... Indices>
     GKO_ATTRIBUTES GKO_INLINE constexpr scalar_type write_scalar(
-        size_type x, size_type z, scalar_type value) const
+        scalar_type value, Indices &&... indices) const
     {
+        static_assert(sizeof...(Indices) == Dimensionality,
+                      "Number of indices must match dimensionality!");
         scalar_type *GKO_RESTRICT rest_scalar = self()->scalar_;
-        return rest_scalar[self()->compute_scalar_index(x, z)] = value;
+        return rest_scalar[self()->compute_scalar_index(
+                   std::forward<Indices>(indices)...)] = value;
     }
 
 private:
@@ -877,11 +552,12 @@ struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
  * @note  Currently, only Dimensionality = 3 is supported.
  */
 template <int Dimensionality, typename ArithmeticType, typename StorageType,
-          int32 ScalarMask = 0b0101>
+          int32 ScalarMask>
 class scaled_reduced_row_major
     : public detail::enable_write_scalar<
           Dimensionality,
-          scaled_reduced_row_major<Dimensionality, ArithmeticType, StorageType>,
+          scaled_reduced_row_major<Dimensionality, ArithmeticType, StorageType,
+                                   ScalarMask>,
           ArithmeticType> {
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
@@ -894,12 +570,10 @@ class scaled_reduced_row_major
 
     using const_accessor =
         scaled_reduced_row_major<dimensionality, arithmetic_type,
-                                 const storage_type>;
+                                 const storage_type, ScalarMask>;
 
-    static_assert(dimensionality == 3,
-                  "Only Dimensionality == 3 is currently supported");
-    static_assert(scalar_mask == 0b0101,
-                  "Only StorageMask == 0b0101 is currently supported");
+    static_assert(dimensionality <= 32,
+                  "Only Dimensionality <= 32 is currently supported");
 
     // Allow access to both `scalar_` and `compute_scalar_index()`
     friend class detail::enable_write_scalar<
@@ -908,8 +582,7 @@ class scaled_reduced_row_major
 
 protected:
     using reference_type =
-        detail::reference_class::scaled_reduced_storage<arithmetic_type,
-                                                        StorageType>;
+        reference_class::scaled_reduced_storage<arithmetic_type, StorageType>;
 
 public:
     /**
@@ -934,6 +607,24 @@ class scaled_reduced_row_major
           size_{size},
           stride_{stride0, stride1}
     {}
+    /**
+     * Creates the accessor with an already allocated storage space with a
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
+     *
+     * @param storage  pointer to the block of memory containing the storage
+     * @param scalar  pointer to the block of memory containing the scalar
+     * values. Memory required is size[0] * stride1.
+     * @param size  multidimensional size of the memory
+     * @param stride  stride array of size dimensionality - 1, used for the
+     * x-indices
+     */
+    GKO_ATTRIBUTES constexpr scaled_reduced_row_major(
+        storage_type *storage, scalar_type *scalar, dim<dimensionality> size,
+        std::array<const size_type, dimensionality - 1> stride)
+        : storage_{storage}, scalar_{scalar}, size_{size}, stride_{stride}
+    {}
+
     /**
      * Creates the accessor with an already allocated storage space with a
      * stride. The first stride is used for computing the index for the first
@@ -979,11 +670,13 @@ class scaled_reduced_row_major
      *
      * @returns the scalar value at the given indices.
      */
+    template <typename... Indices>
     GKO_ATTRIBUTES GKO_INLINE constexpr scalar_type read_scalar(
-        size_type x, size_type y) const
+        Indices &&... indices) const
     {
         const arithmetic_type *GKO_RESTRICT rest_scalar = scalar_;
-        return rest_scalar[compute_scalar_index(x, y)];
+        return rest_scalar[compute_scalar_index(
+            std::forward<Indices>(indices)...)];
     }
 
     /**
@@ -1016,7 +709,7 @@ class scaled_reduced_row_major
     {
         for (size_type i = 0; i < this->size_[0]; ++i) {
             for (size_type k = 0; k < this->size_[2]; ++k) {
-                this->write_scalar(i, k, other.read_scalar(i, k));
+                this->write_scalar(other.read_scalar(i, 0, k), i, 0, k);
             }
         }
         for (size_type i = 0; i < this->size_[0]; ++i) {
@@ -1032,48 +725,60 @@ class scaled_reduced_row_major
      * Returns the stored value for the given indices. If the storage is const,
      * a value is returned, otherwise, a reference is returned.
      *
-     * @param x  x index
-     * @param y  y index
-     * @param z  z index
+     * @param indices  indices which value is supposed to access
      *
      * @returns  the stored value if the accessor is const (if the storage type
-     * is const), or a reference if the accessor is non-const
+     *           is const), or a reference if the accessor is non-const
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr std::conditional_t<
-        is_const, arithmetic_type, reference_type>
-    operator()(size_type x, size_type y, size_type z) const
+    template <typename... Indices>
+    GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
+        helper::are_all_integral<Indices...>::value,
+        std::conditional_t<is_const, arithmetic_type, reference_type>>
+    operator()(Indices &&... indices) const
     {
-        return reference_type{this->storage_ + this->compute_index(x, y, z),
-                              this->read_scalar(x, z)};
+        return reference_type{
+            this->storage_ + compute_index(std::forward<Indices>(indices)...),
+            read_scalar(std::forward<Indices>(indices)...)};
     }
 
     /**
-     * Returns a sub-range spinning the range (x_span, y_span, z_span)
+     * Returns a sub-range spinning the current range (x1_span, x2_span, ...)
      *
-     * @param x  span for the x indices
-     * @param y  span for the y indices
-     * @param z  span for the z indices
+     * @param spans  span for the indices
      *
      * @returns a sub-range for the given spans.
      */
-    GKO_ATTRIBUTES constexpr range<scaled_reduced_row_major> operator()(
-        const span &x_span, const span &y_span, const span &z_span) const
-    {
-        return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
-               GKO_ASSERT(z_span.is_valid()),
-               GKO_ASSERT(x_span <= span{this->size_[0]}),
-               GKO_ASSERT(y_span <= span{this->size_[1]}),
-               GKO_ASSERT(z_span <= span{this->size_[2]}),
-               range<scaled_reduced_row_major>(
-                   this->storage_ + this->compute_index(x_span.begin,
-                                                        y_span.begin,
-                                                        z_span.begin),
-                   this->scalar_ +
-                       this->compute_scalar_index(x_span.begin, z_span.begin),
-                   dim<dimensionality>{x_span.end - x_span.begin,
-                                       y_span.end - y_span.begin,
-                                       z_span.end - z_span.begin},
-                   this->stride_[0], this->stride_[1]);
+    template <typename... SpanTypes>
+    GKO_ATTRIBUTES constexpr std::enable_if_t<
+        helper::are_span_compatible<SpanTypes...>::value,
+        range<scaled_reduced_row_major>>
+    operator()(SpanTypes &&... spans) const
+    {
+        // TODO: Add validation check for spans!
+        return range<scaled_reduced_row_major>{
+            storage_ + compute_index((static_cast<span>(spans).begin)...),
+            scalar_ + compute_scalar_index(static_cast<span>(spans).begin...),
+            dim<dimensionality>{(static_cast<span>(spans).end -
+                                 static_cast<span>(spans).begin)...},
+            stride_};
+        /*
+            return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
+                   GKO_ASSERT(z_span.is_valid()),
+                   GKO_ASSERT(x_span <= span{this->size_[0]}),
+                   GKO_ASSERT(y_span <= span{this->size_[1]}),
+                   GKO_ASSERT(z_span <= span{this->size_[2]}),
+                   range<scaled_reduced_row_major>(
+                       this->storage_ + this->compute_index(x_span.begin,
+                                                            y_span.begin,
+                                                            z_span.begin),
+                       this->scalar_ + this->compute_scalar_index(x_span.begin,
+                                                                  y_span.begin,
+                                                                  z_span.begin),
+                       dim<dimensionality>{x_span.end - x_span.begin,
+                                           y_span.end - y_span.begin,
+                                           z_span.end - z_span.begin},
+                       this->stride_[0], this->stride_[1]);
+        */
     }
 
     /**
@@ -1148,11 +853,18 @@ class scaled_reduced_row_major
                GKO_ASSERT(z < size_[2]), x * stride_[0] + y * stride_[1] + z;
     }
 
+    template <typename... Indices>
     GKO_ATTRIBUTES constexpr GKO_INLINE size_type
-    compute_scalar_index(size_type x, size_type z) const
-    {
-        return GKO_ASSERT(x < size_[0]), GKO_ASSERT(z < size_[2]),
-               x * stride_[1] + z;
+    compute_scalar_index(Indices &&... indices) const
+    {
+        static_assert(sizeof...(Indices) == dimensionality,
+                      "Number of indices must match dimensionality!");
+        return helper::row_major_mask_index<
+            const size_type, dimensionality,
+            scalar_mask>::compute(size_, stride_,
+                                  std::forward_as_tuple(indices...));
+        // return GKO_ASSERT(x < size_[0]), GKO_ASSERT(z < size_[2]),
+        //       x * stride_[1] + z;
     }
 
     storage_type *storage_;
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 42c535cf4e4..f1a467e1ae0 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -54,6 +54,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/perturbation.hpp>
 #include <ginkgo/core/base/polymorphic_object.hpp>
 #include <ginkgo/core/base/range.hpp>
+#include <ginkgo/core/base/range_accessor_helper.hpp>
+#include <ginkgo/core/base/range_accessor_references.hpp>
 #include <ginkgo/core/base/range_accessors.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/temporary_clone.hpp>

From 64d488810a345e129666b9407a58061b6b7ea0c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 9 Nov 2020 18:56:55 +0100
Subject: [PATCH 0212/1008] Decrease number constructors of reduced_row_major

---
 core/test/base/range_accessors.cpp            |  6 ++-
 .../core/base/range_accessor_helper.hpp       | 45 +++++++++++++++++++
 include/ginkgo/core/base/range_accessors.hpp  | 40 ++++++++++-------
 3 files changed, 74 insertions(+), 17 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index e5d2ee38029..eaac6bf9a41 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -473,8 +473,8 @@ class ReducedStorageXd : public ::testing::Test {
     static constexpr gko::size_type data_elements{8};
     st_type data[data_elements]{1.1f, 2.2f, 3.3f, 4.4f,
                                 5.5f, 6.6f, 7.7f, -8.8f};
-    reduced_storage1d r1{data, size_1d, stride0};
-    reduced_storage2d r2{data, size_2d, stride1[0]};
+    reduced_storage1d r1{data, size_1d /*, stride0*/};
+    reduced_storage2d r2{data, size_2d /*, stride1*/};
     const_reduced_storage1d cr1{data, size_1d, stride0};
     const_reduced_storage2d cr2{data, size_2d, stride1};
 
@@ -497,6 +497,8 @@ TEST_F(ReducedStorageXd, CanRead)
 {
     EXPECT_EQ(cr1(1), 2.2f);
     EXPECT_EQ(cr2(0, 1), 2.2f);
+    EXPECT_EQ(r1(1), 2.2f);
+    EXPECT_EQ(r2(0, 1), 2.2f);
 }
 
 
diff --git a/include/ginkgo/core/base/range_accessor_helper.hpp b/include/ginkgo/core/base/range_accessor_helper.hpp
index 33db18f9534..9f82a7bb0b4 100644
--- a/include/ginkgo/core/base/range_accessor_helper.hpp
+++ b/include/ginkgo/core/base/range_accessor_helper.hpp
@@ -197,6 +197,51 @@ template <typename... Args>
 using are_span_compatible = detail::are_span_compatible_impl<false, Args...>;
 
 
+namespace detail {
+
+
+template <typename ValueType, size_type Iter, size_type N>
+constexpr std::enable_if_t<Iter == N, ValueType> mult_array(const dim<N> &size)
+{
+    return 1;
+}
+template <typename ValueType, size_type Iter, size_type N>
+constexpr std::enable_if_t<Iter <= N - 1, ValueType> mult_array(
+    const dim<N> &size)
+{
+    return size[Iter] * mult_array<ValueType, Iter + 1>(size);
+}
+
+
+template <typename ValueType, int Iter = 1, size_type N, typename... Args>
+constexpr std::enable_if_t<N == 0 || (Iter == N && Iter == sizeof...(Args) + 1),
+                           std::array<ValueType, N == 0 ? 0 : N - 1>>
+extract_factorization(const dim<N> &size, Args... args)
+{
+    return {{args...}};
+}
+
+
+template <typename ValueType, int Iter = 1, size_type N, typename... Args>
+constexpr std::enable_if_t<(Iter < N) && (Iter == sizeof...(Args) + 1),
+                           std::array<ValueType, N - 1>>
+extract_factorization(const dim<N> &size, Args... args)
+{
+    return extract_factorization<ValueType, Iter + 1>(
+        size, args..., mult_array<ValueType, Iter>(size));
+}
+
+
+}  // namespace detail
+
+
+template <typename ValueType, size_type N>
+constexpr auto compute_stride_array(const dim<N> &size)
+{
+    return detail::extract_factorization<ValueType>(size);
+}
+
+
 }  // namespace helper
 }  // namespace accessor
 }  // namespace gko
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 454d55925dd..53fe23d076f 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -259,46 +259,56 @@ class reduced_row_major {
      * @param size  multidimensional size of the memory
      * @param stride  stride array used for the x-indices
      */
+    /*
     GKO_ATTRIBUTES constexpr reduced_row_major(
         storage_type *storage, gko::dim<dimensionality> size,
         const std::array<const size_type, dimensionality - 1> &stride)
         : storage_{storage}, size_{size}, stride_{stride}
     {}
+    */
 
     /**
      * Creates the accessor with an already allocated storage space with a
      * stride. The first stride is used for computing the index for the first
      * index, the second stride for the second index, and so on.
+     * It is assumed that accesses are without a stride.
      *
      * @param storage  pointer to the block of memory containing the storage
      * @param size  multidimensional size of the memory
-     * @param stride  stride array used for the x-indices
      */
-    template <typename... Strides>
     GKO_ATTRIBUTES constexpr reduced_row_major(storage_type *storage,
-                                               gko::dim<dimensionality> size,
-                                               Strides &&... stride)
-        : storage_{storage},
-          size_{size},
-          stride_{std::forward<Strides>(stride)...}
-    {
-        static_assert(sizeof...(Strides) == dimensionality - 1,
-                      "Number of provided Strides must be dimensionality - 1!");
-    }
+                                               dim<dimensionality> size)
+        : reduced_row_major{storage, size,
+                            helper::compute_stride_array<const size_type>(size)}
+    {}
 
     /**
      * Creates the accessor with an already allocated storage space with a
      * stride. The first stride is used for computing the index for the first
      * index, the second stride for the second index, and so on.
-     * It is assumed that accesses are without a stride.
      *
      * @param storage  pointer to the block of memory containing the storage
      * @param size  multidimensional size of the memory
+     * @param stride  stride array used for the x-indices
      */
+    template <typename First, typename... Strides>
     GKO_ATTRIBUTES constexpr reduced_row_major(storage_type *storage,
-                                               dim<dimensionality> size)
-        : reduced_row_major{storage, size, {{size[1] * size[2], size[2]}}}
-    {}
+                                               gko::dim<dimensionality> size,
+                                               First &&first_stride,
+                                               Strides &&... strides)
+        : storage_{storage},
+          size_{size},
+          stride_{std::forward<First>(first_stride),
+                  std::forward<Strides>(strides)...}
+    {
+        static_assert(
+            sizeof...(Strides) + 2 == dimensionality ||
+                sizeof...(Strides) == 0 &&
+                    std::is_same<
+                        std::decay_t<First>,
+                        std::array<const size_type, dimensionality - 1>>::value,
+            "Number of provided Strides must be dimensionality - 1!");
+    }
 
     /**
      * Creates an empty accessor (pointing nowhere with an empty size)

From 81ca3ed472c119482bcb2df98f438da95b0cfa33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 9 Nov 2020 19:44:01 +0100
Subject: [PATCH 0213/1008] Make intent of constructors clear

Add more constructor specializations to reduce_row_major in order to
make each purpose and potential use-case clear.
---
 core/test/base/range_accessors.cpp           |  2 +-
 include/ginkgo/core/base/range_accessors.hpp | 34 ++++++++++++--------
 2 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index eaac6bf9a41..f3324c52413 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -474,7 +474,7 @@ class ReducedStorageXd : public ::testing::Test {
     st_type data[data_elements]{1.1f, 2.2f, 3.3f, 4.4f,
                                 5.5f, 6.6f, 7.7f, -8.8f};
     reduced_storage1d r1{data, size_1d /*, stride0*/};
-    reduced_storage2d r2{data, size_2d /*, stride1*/};
+    reduced_storage2d r2{data, size_2d, stride1[0]};
     const_reduced_storage1d cr1{data, size_1d, stride0};
     const_reduced_storage2d cr2{data, size_2d, stride1};
 
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 53fe23d076f..ba36ecc52e5 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -259,13 +259,26 @@ class reduced_row_major {
      * @param size  multidimensional size of the memory
      * @param stride  stride array used for the x-indices
      */
-    /*
     GKO_ATTRIBUTES constexpr reduced_row_major(
         storage_type *storage, gko::dim<dimensionality> size,
         const std::array<const size_type, dimensionality - 1> &stride)
         : storage_{storage}, size_{size}, stride_{stride}
     {}
-    */
+
+    /**
+     * Creates the accessor with an already allocated storage space with a
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
+     *
+     * @param storage  pointer to the block of memory containing the storage
+     * @param size  multidimensional size of the memory
+     * @param stride  stride array used for the x-indices
+     */
+    GKO_ATTRIBUTES constexpr reduced_row_major(
+        storage_type *storage, gko::dim<dimensionality> size,
+        std::array<const size_type, dimensionality - 1> &&stride)
+        : storage_{storage}, size_{size}, stride_{stride}
+    {}
 
     /**
      * Creates the accessor with an already allocated storage space with a
@@ -291,23 +304,16 @@ class reduced_row_major {
      * @param size  multidimensional size of the memory
      * @param stride  stride array used for the x-indices
      */
-    template <typename First, typename... Strides>
+    template <typename... Strides>
     GKO_ATTRIBUTES constexpr reduced_row_major(storage_type *storage,
                                                gko::dim<dimensionality> size,
-                                               First &&first_stride,
                                                Strides &&... strides)
         : storage_{storage},
           size_{size},
-          stride_{std::forward<First>(first_stride),
-                  std::forward<Strides>(strides)...}
-    {
-        static_assert(
-            sizeof...(Strides) + 2 == dimensionality ||
-                sizeof...(Strides) == 0 &&
-                    std::is_same<
-                        std::decay_t<First>,
-                        std::array<const size_type, dimensionality - 1>>::value,
-            "Number of provided Strides must be dimensionality - 1!");
+          stride_{std::forward<Strides>(strides)...}
+    {
+        static_assert(sizeof...(Strides) + 1 == dimensionality,
+                      "Number of provided Strides must be dimensionality - 1!");
     }
 
     /**

From 22947006aed4315bd3fdeb9bcb502f53a73d1ece Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 10 Nov 2020 06:13:16 +0100
Subject: [PATCH 0214/1008] Add multidim support for scaled_reduced_row_major

---
 core/test/base/range_accessors.cpp           | 129 ++++++++++++++++++-
 include/ginkgo/core/base/range_accessors.hpp | 101 +++++++++------
 2 files changed, 193 insertions(+), 37 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index f3324c52413..a5c32035b90 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <tuple>
+#include <type_traits>
 
 
 #include <ginkgo/core/base/dim.hpp>
@@ -460,6 +461,12 @@ class ReducedStorageXd : public ::testing::Test {
         gko::accessor::reduced_row_major<1, ar_type, const st_type>;
     using const_accessor2d =
         gko::accessor::reduced_row_major<2, ar_type, const st_type>;
+    static_assert(std::is_same<const_accessor1d,
+                               typename accessor1d::const_accessor>::value,
+                  "Const accessors must be the same!");
+    static_assert(std::is_same<const_accessor2d,
+                               typename accessor2d::const_accessor>::value,
+                  "Const accessors must be the same!");
 
     using reduced_storage1d = gko::range<accessor1d>;
     using reduced_storage2d = gko::range<accessor2d>;
@@ -473,7 +480,7 @@ class ReducedStorageXd : public ::testing::Test {
     static constexpr gko::size_type data_elements{8};
     st_type data[data_elements]{1.1f, 2.2f, 3.3f, 4.4f,
                                 5.5f, 6.6f, 7.7f, -8.8f};
-    reduced_storage1d r1{data, size_1d /*, stride0*/};
+    reduced_storage1d r1{data, size_1d};
     reduced_storage2d r2{data, size_2d, stride1[0]};
     const_reduced_storage1d cr1{data, size_1d, stride0};
     const_reduced_storage2d cr2{data, size_2d, stride1};
@@ -502,6 +509,24 @@ TEST_F(ReducedStorageXd, CanRead)
 }
 
 
+TEST_F(ReducedStorageXd, CanWrite1)
+{
+    r1(2) = 0.25;
+
+    data_equal_except_for(2);
+    EXPECT_EQ(r1(2), 0.25);  // expect exact since easy to store
+}
+
+
+TEST_F(ReducedStorageXd, CanWrite2)
+{
+    r2(1, 1) = 0.75;
+
+    data_equal_except_for(5);
+    EXPECT_EQ(r2(1, 1), 0.75);  // expect exact since easy to store
+}
+
+
 class ScaledReducedStorage3d : public ::testing::Test {
 protected:
     using span = gko::span;
@@ -786,4 +811,106 @@ TEST_F(ScaledReducedStorage3d, UnaryMinus)
 }
 
 
+class ScaledReducedStorageXd : public ::testing::Test {
+protected:
+    using span = gko::span;
+    using ar_type = double;
+    using st_type = int;
+    using size_type = gko::size_type;
+    static constexpr ar_type delta{0.1};
+
+    using accessor1d =
+        gko::accessor::scaled_reduced_row_major<1, ar_type, st_type, 1>;
+    using accessor2d =
+        gko::accessor::scaled_reduced_row_major<2, ar_type, st_type, 2>;
+    using const_accessor1d =
+        gko::accessor::scaled_reduced_row_major<1, ar_type, const st_type, 1>;
+    using const_accessor2d =
+        gko::accessor::scaled_reduced_row_major<2, ar_type, const st_type, 2>;
+    static_assert(std::is_same<const_accessor1d,
+                               typename accessor1d::const_accessor>::value,
+                  "Const accessors must be the same!");
+    static_assert(std::is_same<const_accessor2d,
+                               typename accessor2d::const_accessor>::value,
+                  "Const accessors must be the same!");
+
+    using reduced_storage1d = gko::range<accessor1d>;
+    using reduced_storage2d = gko::range<accessor2d>;
+    using const_reduced_storage2d = gko::range<const_accessor2d>;
+    using const_reduced_storage1d = gko::range<const_accessor1d>;
+
+    const std::array<const size_type, 0> stride0{};
+    const std::array<const size_type, 1> stride1{4};
+    const gko::dim<1> size_1d{8u};
+    const gko::dim<2> size_2d{2u, 4u};
+
+    static constexpr gko::size_type data_elements{8};
+    st_type data[data_elements]{11, 22, 33, 44, 55, 66, 77, -88};
+    const double default_scalar{.1};
+    ar_type scalar[data_elements]{
+        default_scalar, default_scalar, default_scalar, default_scalar,
+        default_scalar, default_scalar, default_scalar, default_scalar};
+
+    reduced_storage1d r1{data, scalar, size_1d};
+    reduced_storage2d r2{data, scalar, size_2d, stride1[0]};
+    const_reduced_storage1d cr1{data, scalar, size_1d, stride0};
+    const_reduced_storage2d cr2{data, scalar, size_2d, stride1};
+
+    void data_equal_except_for(int idx)
+    {
+        // clang-format off
+        if (idx != 0) { EXPECT_EQ(data[0], 11); }
+        if (idx != 1) { EXPECT_EQ(data[1], 22); }
+        if (idx != 2) { EXPECT_EQ(data[2], 33); }
+        if (idx != 3) { EXPECT_EQ(data[3], 44); }
+        if (idx != 4) { EXPECT_EQ(data[4], 55); }
+        if (idx != 5) { EXPECT_EQ(data[5], 66); }
+        if (idx != 6) { EXPECT_EQ(data[6], 77); }
+        if (idx != 7) { EXPECT_EQ(data[7], -88); }
+        // clang-format on
+    }
+    void scalar_equal_except_for(int idx)
+    {
+        // clang-format off
+        if (idx != 0) { EXPECT_EQ(scalar[0], default_scalar); }
+        if (idx != 1) { EXPECT_EQ(scalar[1], default_scalar); }
+        if (idx != 2) { EXPECT_EQ(scalar[2], default_scalar); }
+        if (idx != 3) { EXPECT_EQ(scalar[3], default_scalar); }
+        if (idx != 4) { EXPECT_EQ(scalar[4], default_scalar); }
+        if (idx != 5) { EXPECT_EQ(scalar[5], default_scalar); }
+        if (idx != 6) { EXPECT_EQ(scalar[6], default_scalar); }
+        if (idx != 7) { EXPECT_EQ(scalar[7], default_scalar); }
+        // clang-format on
+    }
+};
+
+TEST_F(ScaledReducedStorageXd, CanRead)
+{
+    EXPECT_NEAR(cr1(1), 2.2, delta);
+    EXPECT_NEAR(cr2(0, 1), 2.2, delta);
+    EXPECT_NEAR(r1(1), 2.2, delta);
+    EXPECT_NEAR(r2(0, 1), 2.2, delta);
+}
+
+
+TEST_F(ScaledReducedStorageXd, CanWrite1)
+{
+    r1(2) = 0.2;
+
+    data_equal_except_for(2);
+    scalar_equal_except_for(99);
+    EXPECT_NEAR(r1(2), 0.2, delta);
+}
+
+
+TEST_F(ScaledReducedStorageXd, CanWrite2)
+{
+    r2(1, 1) = 0.7;
+
+    data_equal_except_for(5);
+    scalar_equal_except_for(99);
+    EXPECT_NEAR(r2(1, 1), 0.7, delta);
+}
+
+
 }  // namespace
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index ba36ecc52e5..885251d53e5 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -244,6 +244,9 @@ class reduced_row_major {
     using const_accessor =
         reduced_row_major<dimensionality, arithmetic_type, const storage_type>;
 
+    static_assert(Dimensionality >= 1,
+                  "Dimensionality must be a positive number!");
+
     friend class range<reduced_row_major>;
 
 protected:
@@ -257,7 +260,7 @@ class reduced_row_major {
      *
      * @param storage  pointer to the block of memory containing the storage
      * @param size  multidimensional size of the memory
-     * @param stride  stride array used for the x-indices
+     * @param stride  stride array used for memory accesses
      */
     GKO_ATTRIBUTES constexpr reduced_row_major(
         storage_type *storage, gko::dim<dimensionality> size,
@@ -272,7 +275,7 @@ class reduced_row_major {
      *
      * @param storage  pointer to the block of memory containing the storage
      * @param size  multidimensional size of the memory
-     * @param stride  stride array used for the x-indices
+     * @param stride  stride array used for memory accesses
      */
     GKO_ATTRIBUTES constexpr reduced_row_major(
         storage_type *storage, gko::dim<dimensionality> size,
@@ -284,7 +287,7 @@ class reduced_row_major {
      * Creates the accessor with an already allocated storage space with a
      * stride. The first stride is used for computing the index for the first
      * index, the second stride for the second index, and so on.
-     * It is assumed that accesses are without a stride.
+     * It is assumed that accesses are without padding.
      *
      * @param storage  pointer to the block of memory containing the storage
      * @param size  multidimensional size of the memory
@@ -302,7 +305,7 @@ class reduced_row_major {
      *
      * @param storage  pointer to the block of memory containing the storage
      * @param size  multidimensional size of the memory
-     * @param stride  stride array used for the x-indices
+     * @param strides  strides used for memory accesses
      */
     template <typename... Strides>
     GKO_ATTRIBUTES constexpr reduced_row_major(storage_type *storage,
@@ -363,6 +366,7 @@ class reduced_row_major {
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
+        // TODO add proper copy_from implementation independent of size!!!
         for (size_type i = 0; i < size_[0]; ++i) {
             for (size_type j = 0; j < size_[1]; ++j) {
                 for (size_type k = 0; k < size_[2]; ++k) {
@@ -505,7 +509,8 @@ template <int Dimensionality, typename Accessor, typename ScaleType>
 struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
     static_assert(!std::is_const<ScaleType>::value,
                   "This class must NOT have a constant ScaleType!");
-    static_assert(Dimensionality == 3, "Only Dimensionality 3 supported!");
+    static_assert(Dimensionality >= 1,
+                  "Dimensionality must be a positive number!");
 
     using scalar_type = ScaleType;
 
@@ -588,6 +593,8 @@ class scaled_reduced_row_major
         scaled_reduced_row_major<dimensionality, arithmetic_type,
                                  const storage_type, ScalarMask>;
 
+    static_assert(Dimensionality >= 1,
+                  "Dimensionality must be a positive number!");
     static_assert(dimensionality <= 32,
                   "Only Dimensionality <= 32 is currently supported");
 
@@ -608,21 +615,16 @@ class scaled_reduced_row_major
      *
      * @param storage  pointer to the block of memory containing the storage
      * @param scalar  pointer to the block of memory containing the scalar
-     * values. Memory required is size[0] * stride1.
+     *                values.
      * @param size  multidimensional size of the memory
-     * @param stride0  stride used for the x-indices
-     * @param stride1  stride used for the y-indices
+     * @param stride  strides used for memory accesses
      */
-    GKO_ATTRIBUTES constexpr scaled_reduced_row_major(storage_type *storage,
-                                                      scalar_type *scalar,
-                                                      dim<dimensionality> size,
-                                                      size_type stride0,
-                                                      size_type stride1)
-        : storage_{storage},
-          scalar_{scalar},
-          size_{size},
-          stride_{stride0, stride1}
+    GKO_ATTRIBUTES constexpr scaled_reduced_row_major(
+        storage_type *storage, scalar_type *scalar, dim<dimensionality> size,
+        const std::array<const size_type, dimensionality - 1> &stride)
+        : storage_{storage}, scalar_{scalar}, size_{size}, stride_{stride}
     {}
+
     /**
      * Creates the accessor with an already allocated storage space with a
      * stride. The first stride is used for computing the index for the first
@@ -630,14 +632,13 @@ class scaled_reduced_row_major
      *
      * @param storage  pointer to the block of memory containing the storage
      * @param scalar  pointer to the block of memory containing the scalar
-     * values. Memory required is size[0] * stride1.
+     *                values.
      * @param size  multidimensional size of the memory
-     * @param stride  stride array of size dimensionality - 1, used for the
-     * x-indices
+     * @param stride  strides used for memory accesses
      */
     GKO_ATTRIBUTES constexpr scaled_reduced_row_major(
         storage_type *storage, scalar_type *scalar, dim<dimensionality> size,
-        std::array<const size_type, dimensionality - 1> stride)
+        std::array<const size_type, dimensionality - 1> &&stride)
         : storage_{storage}, scalar_{scalar}, size_{size}, stride_{stride}
     {}
 
@@ -645,20 +646,46 @@ class scaled_reduced_row_major
      * Creates the accessor with an already allocated storage space with a
      * stride. The first stride is used for computing the index for the first
      * index, the second stride for the second index, and so on.
-     * It is assumed that accesses are without a stride.
+     * It is assumed that accesses are without padding.
      *
      * @param storage  pointer to the block of memory containing the storage
      * @param scalar  pointer to the block of memory containing the scalar
-     * values. Memory required is size[0] * size[1].
+     *                values.
      * @param size  multidimensional size of the memory
      */
     GKO_ATTRIBUTES constexpr scaled_reduced_row_major(storage_type *storage,
                                                       scalar_type *scalar,
                                                       dim<dimensionality> size)
-        : scaled_reduced_row_major{storage, scalar, size, size[1] * size[2],
-                                   size[2]}
+        : scaled_reduced_row_major{
+              storage, scalar, size,
+              helper::compute_stride_array<const size_type>(size)}
     {}
 
+    /**
+     * Creates the accessor with an already allocated storage space with a
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
+     *
+     * @param storage  pointer to the block of memory containing the storage
+     * @param scalar  pointer to the block of memory containing the scalar
+     *                values.
+     * @param size  multidimensional size of the memory
+     * @param strides  strides used for memory accesses
+     */
+    template <typename... Strides>
+    GKO_ATTRIBUTES constexpr scaled_reduced_row_major(storage_type *storage,
+                                                      scalar_type *scalar,
+                                                      dim<dimensionality> size,
+                                                      Strides &&... strides)
+        : storage_{storage},
+          scalar_{scalar},
+          size_{size},
+          stride_{std::forward<Strides>(strides)...}
+    {
+        static_assert(sizeof...(Strides) + 1 == dimensionality,
+                      "Number of provided Strides must be dimensionality - 1!");
+    }
+
     /**
      * Creates an empty accessor (pointing nowhere with an empty size)
      */
@@ -674,15 +701,13 @@ class scaled_reduced_row_major
      */
     GKO_ATTRIBUTES GKO_INLINE constexpr range<const_accessor> to_const() const
     {
-        return range<const_accessor>{storage_, scalar_, size_, stride_[0],
-                                     stride_[1]};
+        return range<const_accessor>{storage_, scalar_, size_, stride_};
     }
 
     /**
      * Reads the scalar value at the given indices.
      *
-     * @param x  x index
-     * @param y  y index
+     * @param indices  indices which data to access
      *
      * @returns the scalar value at the given indices.
      */
@@ -723,6 +748,7 @@ class scaled_reduced_row_major
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
+        // TODO add proper copy_from implementation independent of size!!!
         for (size_type i = 0; i < this->size_[0]; ++i) {
             for (size_type k = 0; k < this->size_[2]; ++k) {
                 this->write_scalar(other.read_scalar(i, 0, k), i, 0, k);
@@ -750,7 +776,7 @@ class scaled_reduced_row_major
     GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
         helper::are_all_integral<Indices...>::value,
         std::conditional_t<is_const, arithmetic_type, reference_type>>
-    operator()(Indices &&... indices) const
+    operator()(Indices... indices) const
     {
         return reference_type{
             this->storage_ + compute_index(std::forward<Indices>(indices)...),
@@ -862,11 +888,16 @@ class scaled_reduced_row_major
     }
 
 protected:
-    GKO_ATTRIBUTES constexpr GKO_INLINE size_type
-    compute_index(size_type x, size_type y, size_type z) const
+    template <typename... Indices>
+    GKO_ATTRIBUTES GKO_INLINE constexpr size_type compute_index(
+        Indices &&... indices) const
     {
-        return GKO_ASSERT(x < size_[0]), GKO_ASSERT(y < size_[1]),
-               GKO_ASSERT(z < size_[2]), x * stride_[0] + y * stride_[1] + z;
+        static_assert(sizeof...(Indices) == dimensionality,
+                      "Number of indices must match dimensionality!");
+        return helper::row_major_index<
+            const size_type, dimensionality>::compute(size_, stride_,
+                                                      std::forward<Indices>(
+                                                          indices)...);
     }
 
     template <typename... Indices>
@@ -879,8 +910,6 @@ class scaled_reduced_row_major
             const size_type, dimensionality,
             scalar_mask>::compute(size_, stride_,
                                   std::forward_as_tuple(indices...));
-        // return GKO_ASSERT(x < size_[0]), GKO_ASSERT(z < size_[2]),
-        //       x * stride_[1] + z;
     }
 
     storage_type *storage_;

From 3b93f40bd22ef6757d71d27d3fae8e133b7985f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 10 Nov 2020 07:57:03 +0100
Subject: [PATCH 0215/1008] Make copy_from and Span test multidim compatible

---
 core/test/base/range_accessors.cpp            |   6 +-
 .../core/base/range_accessor_helper.hpp       | 142 ++++++++++++++----
 include/ginkgo/core/base/range_accessors.hpp  |  87 +++--------
 3 files changed, 137 insertions(+), 98 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index a5c32035b90..c0dfeb8c20a 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -427,7 +427,7 @@ TEST_F(ReducedStorage3d, UnaryMinus)
 
 TEST_F(ReducedStorage3d, CanCreateSubrange)
 {
-    auto subr = r(span{1, 3}, span{0, 2}, 0);
+    auto subr = r(span{1u, 3u}, span{0u, 2u}, 0u);
 
     EXPECT_EQ(subr(0, 0, 0), 5.06);
     EXPECT_EQ(subr(0, 1, 0), 2.08);
@@ -438,7 +438,7 @@ TEST_F(ReducedStorage3d, CanCreateSubrange)
 
 TEST_F(ReducedStorage3d, CanCreateSubrange2)
 {
-    auto subr = cr(span{1, 3}, span{0, 2}, span{0, 1});
+    auto subr = cr(span{1u, 3u}, span{0u, 2u}, span{0u, 1u});
 
     EXPECT_EQ(subr(0, 0, 0), 5.06);
     EXPECT_EQ(subr(0, 1, 0), 2.08);
@@ -666,7 +666,7 @@ TEST_F(ScaledReducedStorage3d, CanRead)
 
 TEST_F(ScaledReducedStorage3d, Subrange)
 {
-    auto subr = cr(0, gko::span{0, 2}, 1);
+    auto subr = cr(0u, gko::span{0u, 2u}, 1u);
 
     EXPECT_EQ(subr(0, 0, 0), 22.);
     EXPECT_EQ(subr(0, 1, 0), 26.);
diff --git a/include/ginkgo/core/base/range_accessor_helper.hpp b/include/ginkgo/core/base/range_accessor_helper.hpp
index 9f82a7bb0b4..1aace2f7675 100644
--- a/include/ginkgo/core/base/range_accessor_helper.hpp
+++ b/include/ginkgo/core/base/range_accessor_helper.hpp
@@ -65,11 +65,11 @@ template <typename IndexType, int32 total_dim, int32 current_iter = total_dim>
 struct row_major_index {
     static constexpr int32 current_dim{total_dim - current_iter};
     template <typename FirstType, typename... Indices>
-    static constexpr IndexType compute(
-        const dim<total_dim> &size,
-        const std::array<IndexType, (total_dim > 1 ? total_dim - 1 : 0)>
-            &stride,
-        FirstType &&first, Indices &&... idxs)
+    static constexpr GKO_ATTRIBUTES GKO_INLINE IndexType
+    compute(const dim<total_dim> &size,
+            const std::array<IndexType, (total_dim > 1 ? total_dim - 1 : 0)>
+                &stride,
+            FirstType &&first, Indices &&... idxs)
     {
         return GKO_ASSERT(first < size[current_dim]),
                first * stride[current_dim] +
@@ -82,10 +82,10 @@ template <typename IndexType, int32 total_dim>
 struct row_major_index<IndexType, total_dim, 1> {
     static constexpr int32 current_dim{total_dim - 1};
     template <typename FirstType>
-    static constexpr IndexType compute(
-        const dim<total_dim> &size,
-        const std::array<IndexType, (total_dim > 1 ? total_dim - 1 : 0)>,
-        FirstType &&first)
+    static constexpr GKO_ATTRIBUTES GKO_INLINE IndexType
+    compute(const dim<total_dim> &size,
+            const std::array<IndexType, (total_dim > 1 ? total_dim - 1 : 0)>,
+            FirstType &&first)
     {
         return GKO_ASSERT(first < size[current_dim]), first;
     }
@@ -117,10 +117,10 @@ struct row_major_mask_index {
                                              current_scalar == total_dim - 1};
 
     template <typename... Args>
-    static constexpr IndexType compute(
-        const dim<total_dim> &size,
-        const std::array<IndexType, stride_size> &stride,
-        std::tuple<Args...> indices)
+    static constexpr GKO_ATTRIBUTES GKO_INLINE IndexType
+    compute(const dim<total_dim> &size,
+            const std::array<IndexType, stride_size> &stride,
+            std::tuple<Args...> indices)
     {
         return GKO_ASSERT(std::get<current_dim>(indices) < size[current_dim]),
                (use_current_stride
@@ -144,10 +144,10 @@ struct row_major_mask_index<IndexType, total_dim, mask, 0, current_scalar> {
                                              current_scalar == total_dim - 1};
 
     template <typename... Args>
-    static constexpr IndexType compute(
-        const dim<total_dim> &size,
-        const std::array<IndexType, stride_size> &stride,
-        std::tuple<Args...> indices)
+    static constexpr GKO_ATTRIBUTES GKO_INLINE IndexType
+    compute(const dim<total_dim> &size,
+            const std::array<IndexType, stride_size> &stride,
+            std::tuple<Args...> indices)
     {
         return GKO_ASSERT(std::get<current_dim>(indices) < size[current_dim]),
                (use_current_stride
@@ -158,6 +158,9 @@ struct row_major_mask_index<IndexType, total_dim, mask, 0, current_scalar> {
 };
 
 
+/**
+ * Evaluates if all Args fulfill std::is_integral
+ */
 template <typename... Args>
 struct are_all_integral : public std::true_type {
 };
@@ -193,6 +196,10 @@ struct are_span_compatible_impl<has_span, First, Args...>
 }  // namespace detail
 
 
+/**
+ * Evaluates if at least one type of Args is a gko::span and the others either
+ * also gko::span or fulfill std::is_integral
+ */
 template <typename... Args>
 using are_span_compatible = detail::are_span_compatible_impl<false, Args...>;
 
@@ -201,30 +208,32 @@ namespace detail {
 
 
 template <typename ValueType, size_type Iter, size_type N>
-constexpr std::enable_if_t<Iter == N, ValueType> mult_array(const dim<N> &size)
+constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<Iter == N, ValueType>
+mult_array(const dim<N> &size)
 {
     return 1;
 }
 template <typename ValueType, size_type Iter, size_type N>
-constexpr std::enable_if_t<Iter <= N - 1, ValueType> mult_array(
-    const dim<N> &size)
+constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<Iter + 1 <= N, ValueType>
+mult_array(const dim<N> &size)
 {
     return size[Iter] * mult_array<ValueType, Iter + 1>(size);
 }
 
 
-template <typename ValueType, int Iter = 1, size_type N, typename... Args>
-constexpr std::enable_if_t<N == 0 || (Iter == N && Iter == sizeof...(Args) + 1),
-                           std::array<ValueType, N == 0 ? 0 : N - 1>>
-extract_factorization(const dim<N> &size, Args... args)
+template <typename ValueType, size_type Iter = 1, size_type N, typename... Args>
+constexpr GKO_ATTRIBUTES GKO_INLINE
+    std::enable_if_t<N == 0 || (Iter == N && Iter == sizeof...(Args) + 1),
+                     std::array<ValueType, N == 0 ? 0 : N - 1>>
+    extract_factorization(const dim<N> &size, Args... args)
 {
     return {{args...}};
 }
 
 
-template <typename ValueType, int Iter = 1, size_type N, typename... Args>
-constexpr std::enable_if_t<(Iter < N) && (Iter == sizeof...(Args) + 1),
-                           std::array<ValueType, N - 1>>
+template <typename ValueType, size_type Iter = 1, size_type N, typename... Args>
+constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<
+    (Iter < N) && (Iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
 extract_factorization(const dim<N> &size, Args... args)
 {
     return extract_factorization<ValueType, Iter + 1>(
@@ -235,13 +244,90 @@ extract_factorization(const dim<N> &size, Args... args)
 }  // namespace detail
 
 
+/**
+ * Computes the default stride array from a size, assuming there is no padding
+ */
 template <typename ValueType, size_type N>
-constexpr auto compute_stride_array(const dim<N> &size)
+constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_stride_array(
+    const dim<N> &size)
 {
     return detail::extract_factorization<ValueType>(size);
 }
 
 
+namespace detail {
+
+
+template <size_type Iter, size_type N, typename Callable, typename... Indices>
+GKO_ATTRIBUTES std::enable_if_t<Iter == N> multidim_for_each_impl(
+    const dim<N> &size, Callable &&callable, Indices... indices)
+{
+    static_assert(Iter == sizeof...(Indices),
+                  "Number arguments must match current iteration!");
+    callable(indices...);
+}
+
+template <size_type Iter, size_type N, typename Callable, typename... Indices>
+GKO_ATTRIBUTES std::enable_if_t<(Iter < N)> multidim_for_each_impl(
+    const dim<N> &size, Callable &&callable, Indices... indices)
+{
+    static_assert(Iter == sizeof...(Indices),
+                  "Number arguments must match current iteration!");
+    for (size_type i = 0; i < size[Iter]; ++i) {
+        multidim_for_each_impl<Iter + 1>(size, std::forward<Callable>(callable),
+                                         indices..., i);
+    }
+}
+
+
+}  // namespace detail
+
+
+/**
+ * Creates a recursive for-loop for each dimension and calls dest(indices...) =
+ * source(indices...)
+ */
+template <size_type N, typename Callable>
+GKO_ATTRIBUTES void multidim_for_each(const dim<N> &size, Callable &&callable)
+{
+    detail::multidim_for_each_impl<0>(size, std::forward<Callable>(callable));
+}
+
+
+namespace detail {
+
+
+template <size_type Iter, size_type N>
+GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<Iter == N, int>
+spans_in_size(const dim<N> &size)
+{
+    return 0;
+}
+
+
+template <size_type Iter, size_type N, typename First, typename... Remaining>
+GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<(Iter < N), int>
+spans_in_size(const dim<N> &size, First &&first, Remaining &&... remaining)
+{
+    static_assert(sizeof...(Remaining) + 1 == N - Iter,
+                  "Number of remaining spans must be equal to N - Iter");
+    return GKO_ASSERT(span{first}.is_valid()),
+           GKO_ASSERT(span{first} <= span{size[Iter]}),
+           spans_in_size<Iter + 1>(size, std::forward<Remaining>(remaining)...);
+}
+
+
+}  // namespace detail
+
+
+template <size_type N, typename... Spans>
+GKO_ATTRIBUTES GKO_INLINE constexpr int validate_spans(const dim<N> &size,
+                                                       Spans... spans)
+{
+    return detail::spans_in_size<0>(size, std::forward<Spans>(spans)...);
+}
+
+
 }  // namespace helper
 }  // namespace accessor
 }  // namespace gko
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 885251d53e5..b4b6503f8f9 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -366,14 +366,9 @@ class reduced_row_major {
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
-        // TODO add proper copy_from implementation independent of size!!!
-        for (size_type i = 0; i < size_[0]; ++i) {
-            for (size_type j = 0; j < size_[1]; ++j) {
-                for (size_type k = 0; k < size_[2]; ++k) {
-                    (*this)(i, j, k) = other(i, j, k);
-                }
-            }
-        }
+        helper::multidim_for_each(size_, [&](auto... indices) {
+            (*this)(indices...) = other(indices...);
+        });
     }
 
     /**
@@ -408,26 +403,12 @@ class reduced_row_major {
         range<reduced_row_major>>
     operator()(SpanTypes &&... spans) const
     {
-        // TODO: Add validation check for spans!
-        return range<reduced_row_major>{
-            storage_ + compute_index((static_cast<span>(spans).begin)...),
-            dim<dimensionality>{(static_cast<span>(spans).end -
-                                 static_cast<span>(spans).begin)...},
-            stride_};
-        /*
-        return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
-               GKO_ASSERT(z_span.is_valid()),
-               GKO_ASSERT(x_span <= span{size_[0]}),
-               GKO_ASSERT(y_span <= span{size_[1]}),
-               GKO_ASSERT(z_span <= span{size_[2]}),
-               range<reduced_row_major>(
-                   storage_ +
-                       compute_index(x_span.begin, y_span.begin, z_span.begin),
-                   dim<dimensionality>{x_span.end - x_span.begin,
-                                       y_span.end - y_span.begin,
-                                       z_span.end - z_span.begin},
-                   stride_[0], stride_[1]);
-        */
+        return helper::validate_spans(size_, spans...),
+               range<reduced_row_major>{
+                   storage_ + compute_index((span{spans}.begin)...),
+                   dim<dimensionality>{
+                       (span{spans}.end - span{spans}.begin)...},
+                   stride_};
     }
 
     /**
@@ -570,7 +551,6 @@ struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
  *                     on.
  *
  * @note  This class only manages the accesses and not the memory itself.
- * @note  Currently, only Dimensionality = 3 is supported.
  */
 template <int Dimensionality, typename ArithmeticType, typename StorageType,
           int32 ScalarMask>
@@ -748,19 +728,10 @@ class scaled_reduced_row_major
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
-        // TODO add proper copy_from implementation independent of size!!!
-        for (size_type i = 0; i < this->size_[0]; ++i) {
-            for (size_type k = 0; k < this->size_[2]; ++k) {
-                this->write_scalar(other.read_scalar(i, 0, k), i, 0, k);
-            }
-        }
-        for (size_type i = 0; i < this->size_[0]; ++i) {
-            for (size_type j = 0; j < this->size_[1]; ++j) {
-                for (size_type k = 0; k < this->size_[2]; ++k) {
-                    (*this)(i, j, k) = other(i, j, k);
-                }
-            }
-        }
+        helper::multidim_for_each(size_, [&](auto... indices) {
+            this->write_scalar(other.read_scalar(indices...), indices...);
+            (*this)(indices...) = other(indices...);
+        });
     }
 
     /**
@@ -796,31 +767,13 @@ class scaled_reduced_row_major
         range<scaled_reduced_row_major>>
     operator()(SpanTypes &&... spans) const
     {
-        // TODO: Add validation check for spans!
-        return range<scaled_reduced_row_major>{
-            storage_ + compute_index((static_cast<span>(spans).begin)...),
-            scalar_ + compute_scalar_index(static_cast<span>(spans).begin...),
-            dim<dimensionality>{(static_cast<span>(spans).end -
-                                 static_cast<span>(spans).begin)...},
-            stride_};
-        /*
-            return GKO_ASSERT(x_span.is_valid()), GKO_ASSERT(y_span.is_valid()),
-                   GKO_ASSERT(z_span.is_valid()),
-                   GKO_ASSERT(x_span <= span{this->size_[0]}),
-                   GKO_ASSERT(y_span <= span{this->size_[1]}),
-                   GKO_ASSERT(z_span <= span{this->size_[2]}),
-                   range<scaled_reduced_row_major>(
-                       this->storage_ + this->compute_index(x_span.begin,
-                                                            y_span.begin,
-                                                            z_span.begin),
-                       this->scalar_ + this->compute_scalar_index(x_span.begin,
-                                                                  y_span.begin,
-                                                                  z_span.begin),
-                       dim<dimensionality>{x_span.end - x_span.begin,
-                                           y_span.end - y_span.begin,
-                                           z_span.end - z_span.begin},
-                       this->stride_[0], this->stride_[1]);
-        */
+        return helper::validate_spans(size_, spans...),
+               range<scaled_reduced_row_major>{
+                   storage_ + compute_index((span{spans}.begin)...),
+                   scalar_ + compute_scalar_index(span{spans}.begin...),
+                   dim<dimensionality>{
+                       (span{spans}.end - span{spans}.begin)...},
+                   stride_};
     }
 
     /**

From fadec4d7e0b71b18d6e1b7ae6fdc2fe757fa369b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 10 Nov 2020 09:14:12 +0100
Subject: [PATCH 0216/1008] Make Accessor tests typed

---
 core/test/base/range_accessors.cpp | 580 ++++++++++++++++-------------
 include/ginkgo/core/base/types.hpp |   2 +-
 2 files changed, 332 insertions(+), 250 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index c0dfeb8c20a..31b71793b50 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -140,12 +140,14 @@ TEST_F(RowMajorAccessor, CanAssignSubranges)
 }
 
 
+template <typename ArithmeticStorageType>
 class ReducedStorage3d : public ::testing::Test {
 protected:
-    using span = gko::span;
-    using ar_type = double;
-    using st_type = double;
-    static constexpr ar_type delta{::r<st_type>::value};
+    using ar_type =
+        typename std::tuple_element<0, decltype(ArithmeticStorageType{})>::type;
+    using st_type =
+        typename std::tuple_element<1, decltype(ArithmeticStorageType{})>::type;
+    static constexpr ar_type delta{::r<st_type>::value * 2};
 
     // Type for `check_accessor_correctness` to forward the indices
     using t = std::tuple<int, int, int>;
@@ -188,262 +190,309 @@ class ReducedStorage3d : public ::testing::Test {
     {
         // Test for equality is fine here since they should not be modified
         // clang-format off
-        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), 1.0);     }
-        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), 2.01);    }
-        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), -1.02);   }
-        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), 3.03);    }
-        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), 4.04);    }
-        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), -2.05);   }
-        if (ignore != t(1, 0, 0)) { EXPECT_EQ(a(1, 0, 0), 5.06);    }
-        if (ignore != t(1, 0, 1)) { EXPECT_EQ(a(1, 0, 1), 6.07);    }
-        if (ignore != t(1, 1, 0)) { EXPECT_EQ(a(1, 1, 0), 2.08);    }
-        if (ignore != t(1, 1, 1)) { EXPECT_EQ(a(1, 1, 1), 3.09);    }
-        if (ignore != t(1, 2, 0)) { EXPECT_EQ(a(1, 2, 0), -1.1);    }
-        if (ignore != t(1, 2, 1)) { EXPECT_EQ(a(1, 2, 1), -9.11);   }
-        if (ignore != t(2, 0, 0)) { EXPECT_EQ(a(2, 0, 0), -2.12);   }
-        if (ignore != t(2, 0, 1)) { EXPECT_EQ(a(2, 0, 1), 2.13);    }
-        if (ignore != t(2, 1, 0)) { EXPECT_EQ(a(2, 1, 0), 0.14);    }
-        if (ignore != t(2, 1, 1)) { EXPECT_EQ(a(2, 1, 1), 15.15);   }
-        if (ignore != t(2, 2, 0)) { EXPECT_EQ(a(2, 2, 0), -9.16);   }
-        if (ignore != t(2, 2, 1)) { EXPECT_EQ(a(2, 2, 1), 8.17);    }
-        if (ignore != t(3, 0, 0)) { EXPECT_EQ(a(3, 0, 0), 7.18);    }
-        if (ignore != t(3, 0, 1)) { EXPECT_EQ(a(3, 0, 1), -6.19);   }
-        if (ignore != t(3, 1, 0)) { EXPECT_EQ(a(3, 1, 0), 5.2);     }
-        if (ignore != t(3, 1, 1)) { EXPECT_EQ(a(3, 1, 1), -4.21);   }
-        if (ignore != t(3, 2, 0)) { EXPECT_EQ(a(3, 2, 0), 3.22);    }
-        if (ignore != t(3, 2, 1)) { EXPECT_EQ(a(3, 2, 1), -2.23);   }
+        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), st_type{1.0});     }
+        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), st_type{2.01});    }
+        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), st_type{-1.02});   }
+        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), st_type{3.03});    }
+        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), st_type{4.04});    }
+        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), st_type{-2.05});   }
+        if (ignore != t(1, 0, 0)) { EXPECT_EQ(a(1, 0, 0), st_type{5.06});    }
+        if (ignore != t(1, 0, 1)) { EXPECT_EQ(a(1, 0, 1), st_type{6.07});    }
+        if (ignore != t(1, 1, 0)) { EXPECT_EQ(a(1, 1, 0), st_type{2.08});    }
+        if (ignore != t(1, 1, 1)) { EXPECT_EQ(a(1, 1, 1), st_type{3.09});    }
+        if (ignore != t(1, 2, 0)) { EXPECT_EQ(a(1, 2, 0), st_type{-1.1});    }
+        if (ignore != t(1, 2, 1)) { EXPECT_EQ(a(1, 2, 1), st_type{-9.11});   }
+        if (ignore != t(2, 0, 0)) { EXPECT_EQ(a(2, 0, 0), st_type{-2.12});   }
+        if (ignore != t(2, 0, 1)) { EXPECT_EQ(a(2, 0, 1), st_type{2.13});    }
+        if (ignore != t(2, 1, 0)) { EXPECT_EQ(a(2, 1, 0), st_type{0.14});    }
+        if (ignore != t(2, 1, 1)) { EXPECT_EQ(a(2, 1, 1), st_type{15.15});   }
+        if (ignore != t(2, 2, 0)) { EXPECT_EQ(a(2, 2, 0), st_type{-9.16});   }
+        if (ignore != t(2, 2, 1)) { EXPECT_EQ(a(2, 2, 1), st_type{8.17});    }
+        if (ignore != t(3, 0, 0)) { EXPECT_EQ(a(3, 0, 0), st_type{7.18});    }
+        if (ignore != t(3, 0, 1)) { EXPECT_EQ(a(3, 0, 1), st_type{-6.19});   }
+        if (ignore != t(3, 1, 0)) { EXPECT_EQ(a(3, 1, 0), st_type{5.2});     }
+        if (ignore != t(3, 1, 1)) { EXPECT_EQ(a(3, 1, 1), st_type{-4.21});   }
+        if (ignore != t(3, 2, 0)) { EXPECT_EQ(a(3, 2, 0), st_type{3.22});    }
+        if (ignore != t(3, 2, 1)) { EXPECT_EQ(a(3, 2, 1), st_type{-2.23});   }
         // clang-format on
     }
 };
 
+using ReducedStorage3dTypes =
+    ::testing::Types<std::tuple<double, double>, std::tuple<double, float>,
+                     std::tuple<float, float>
+                     /*,
+                     std::tuple<std::complex<double>, std::complex<double>>,
+                     std::tuple<std::complex<double>, std::complex<float>>,
+                     std::tuple<std::complex<float>, std::complex<float>>
+                     */
+                     >;
+
+TYPED_TEST_SUITE(ReducedStorage3d, ReducedStorage3dTypes);
+
 
-TEST_F(ReducedStorage3d, CorrectLengths)
+TYPED_TEST(ReducedStorage3d, CorrectLengths)
 {
-    EXPECT_EQ(r.length(0), size[0]);
-    EXPECT_EQ(r.length(1), size[1]);
-    EXPECT_EQ(r.length(2), size[2]);
-    EXPECT_EQ(r.length(3), 1);
-    EXPECT_EQ(r->get_size(), size);
+    EXPECT_EQ(this->r.length(0), this->size[0]);
+    EXPECT_EQ(this->r.length(1), this->size[1]);
+    EXPECT_EQ(this->r.length(2), this->size[2]);
+    EXPECT_EQ(this->r.length(3), 1);
+    EXPECT_EQ(this->r->get_size(), this->size);
 }
 
 
-TEST_F(ReducedStorage3d, CorrectStride)
+TYPED_TEST(ReducedStorage3d, CorrectStride)
 {
-    EXPECT_EQ(r->get_stride()[0], size[1] * size[2]);
-    EXPECT_EQ(r->get_stride().at(0), size[1] * size[2]);
-    EXPECT_EQ(r->get_stride()[1], size[2]);
-    EXPECT_EQ(r->get_stride().at(1), size[2]);
+    EXPECT_EQ(this->r->get_stride()[0], this->size[1] * this->size[2]);
+    EXPECT_EQ(this->r->get_stride().at(0), this->size[1] * this->size[2]);
+    EXPECT_EQ(this->r->get_stride()[1], this->size[2]);
+    EXPECT_EQ(this->r->get_stride().at(1), this->size[2]);
 }
 
 
-TEST_F(ReducedStorage3d, CorrectStorage)
+TYPED_TEST(ReducedStorage3d, CorrectStorage)
 {
-    EXPECT_EQ(r->get_stored_data(), data);
-    EXPECT_EQ(r->get_const_storage(), data);
+    EXPECT_EQ(this->r->get_stored_data(), this->data);
+    EXPECT_EQ(this->r->get_const_storage(), this->data);
 }
 
 
-TEST_F(ReducedStorage3d, CanReadData)
+TYPED_TEST(ReducedStorage3d, CanReadData)
 {
-    check_accessor_correctness(r);
-    check_accessor_correctness(cr);
+    this->check_accessor_correctness(this->r);
+    this->check_accessor_correctness(this->cr);
 }
 
 
-TEST_F(ReducedStorage3d, CopyFrom)
+TYPED_TEST(ReducedStorage3d, CopyFrom)
 {
-    st_type data2[data_elements];
-    reduced_storage cpy(data2, size);
+    using st_type = typename TestFixture::st_type;
+    using reduced_storage = typename TestFixture::reduced_storage;
+    st_type data2[this->data_elements];
+    reduced_storage cpy(data2, this->size);
 
     // Do not use this in regular code since the implementation is slow
-    cpy = r;
+    cpy = this->r;
 
-    check_accessor_correctness(cpy);
+    this->check_accessor_correctness(cpy);
 }
 
 
-TEST_F(ReducedStorage3d, CanImplicitlyConvertToConst)
+TYPED_TEST(ReducedStorage3d, CanImplicitlyConvertToConst)
 {
-    const_reduced_storage const_rs = r->to_const();
-    const_reduced_storage const_rs2 = cr;
+    using const_reduced_storage = typename TestFixture::const_reduced_storage;
 
-    check_accessor_correctness(const_rs);
-    check_accessor_correctness(const_rs2);
+    const_reduced_storage const_rs = this->r->to_const();
+    const_reduced_storage const_rs2 = this->cr;
+
+    this->check_accessor_correctness(const_rs);
+    this->check_accessor_correctness(const_rs2);
 }
 
 
-TEST_F(ReducedStorage3d, ToConstWorks)
+TYPED_TEST(ReducedStorage3d, ToConstWorks)
 {
-    auto cr2 = r->to_const();
+    using const_reduced_storage = typename TestFixture::const_reduced_storage;
+
+    auto cr2 = this->r->to_const();
 
     static_assert(std::is_same<decltype(cr2), const_reduced_storage>::value,
                   "Types must be equal!");
-    check_accessor_correctness(cr2);
+    this->check_accessor_correctness(cr2);
 }
 
 
-TEST_F(ReducedStorage3d, CanWriteData)
+TYPED_TEST(ReducedStorage3d, CanWriteData)
 {
-    r(0, 1, 0) = 100.2;
+    using t = typename TestFixture::t;
 
-    check_accessor_correctness(r, t(0, 1, 0));
-    EXPECT_EQ(r(0, 1, 0), 100.2);
+    this->r(0, 1, 0) = 100.25;
+
+    this->check_accessor_correctness(this->r, t(0, 1, 0));
+    EXPECT_NEAR(this->r(0, 1, 0), 100.25, this->delta);
 }
 
 
-TEST_F(ReducedStorage3d, Assignment)
+TYPED_TEST(ReducedStorage3d, Assignment)
 {
-    r(0, 0, 1) = 10.2;
+    using t = typename TestFixture::t;
+
+    this->r(0, 0, 1) = 10.2;
 
-    check_accessor_correctness(r, t(0, 0, 1));
-    EXPECT_NEAR(r(0, 0, 1), 10.2, delta);
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), 10.2, this->delta);
 }
 
 
-TEST_F(ReducedStorage3d, Assignment2)
+TYPED_TEST(ReducedStorage3d, Assignment2)
 {
-    r(0, 0, 1) = r(0, 1, 0);
+    using t = typename TestFixture::t;
 
-    check_accessor_correctness(r, t(0, 0, 1));
-    EXPECT_NEAR(r(0, 0, 1), -1.02, delta);
+    this->r(0, 0, 1) = this->r(0, 1, 0);
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), -1.02, this->delta);
 }
 
 
-TEST_F(ReducedStorage3d, Addition)
+TYPED_TEST(ReducedStorage3d, Addition)
 {
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
     const ar_type expected = 10.2 + 2.01;
 
-    auto result = r(0, 0, 1) + 10.2;
-    r(0, 0, 1) += 10.2;
+    auto result = this->r(0, 0, 1) + ar_type{10.2};
+    this->r(0, 0, 1) += 10.2;
 
-    check_accessor_correctness(r, t(0, 0, 1));
-    EXPECT_NEAR(r(0, 0, 1), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ReducedStorage3d, Addition2)
+TYPED_TEST(ReducedStorage3d, Addition2)
 {
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
     const ar_type expected = 2.01 + -1.02;
 
-    auto result = r(0, 0, 1) + r(0, 1, 0);
-    r(0, 0, 1) += r(0, 1, 0);
+    auto result = this->r(0, 0, 1) + this->r(0, 1, 0);
+    this->r(0, 0, 1) += this->r(0, 1, 0);
 
-    check_accessor_correctness(r, t(0, 0, 1));
-    EXPECT_NEAR(r(0, 0, 1), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ReducedStorage3d, Subtraction)
+TYPED_TEST(ReducedStorage3d, Subtraction)
 {
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
     const ar_type expected = -2.23 - 1;
 
-    auto result = r(3, 2, 1) - 1.;
-    r(3, 2, 1) -= 1;
+    auto result = this->r(3, 2, 1) - ar_type{1.};
+    this->r(3, 2, 1) -= 1;
 
-    check_accessor_correctness(r, t(3, 2, 1));
-    EXPECT_NEAR(r(3, 2, 1), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(3, 2, 1));
+    EXPECT_NEAR(this->r(3, 2, 1), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ReducedStorage3d, Subtraction2)
+TYPED_TEST(ReducedStorage3d, Subtraction2)
 {
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
     const ar_type expected = 3.22 - -2.23;
 
-    auto result = cr(3, 2, 0) - r(3, 2, 1);
-    r(3, 2, 0) -= r(3, 2, 1);
+    auto result = this->cr(3, 2, 0) - this->r(3, 2, 1);
+    this->r(3, 2, 0) -= this->r(3, 2, 1);
 
-    check_accessor_correctness(r, t(3, 2, 0));
-    EXPECT_NEAR(r(3, 2, 0), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(3, 2, 0));
+    EXPECT_NEAR(this->r(3, 2, 0), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ReducedStorage3d, Multiplication)
+TYPED_TEST(ReducedStorage3d, Multiplication)
 {
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
     const ar_type expected = 1 * 2;
 
-    auto result = r(0, 0, 0) * 2.;
-    r(0, 0, 0) *= 2;
+    auto result = this->r(0, 0, 0) * ar_type{2.};
+    this->r(0, 0, 0) *= 2;
 
-    check_accessor_correctness(r, t(0, 0, 0));
-    EXPECT_NEAR(r(0, 0, 0), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(0, 0, 0));
+    EXPECT_NEAR(this->r(0, 0, 0), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ReducedStorage3d, Multiplication2)
+TYPED_TEST(ReducedStorage3d, Multiplication2)
 {
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
     const ar_type expected = 2.01 * 3.03;
 
-    auto result = r(0, 0, 1) * cr(0, 1, 1);
-    r(0, 0, 1) *= r(0, 1, 1);
+    auto result = this->r(0, 0, 1) * this->cr(0, 1, 1);
+    this->r(0, 0, 1) *= this->r(0, 1, 1);
 
-    check_accessor_correctness(r, t(0, 0, 1));
-    EXPECT_NEAR(r(0, 0, 1), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ReducedStorage3d, Division)
+TYPED_TEST(ReducedStorage3d, Division)
 {
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
     const ar_type expected = 2.01 / 2.0;
 
-    auto result = cr(0, 0, 1) / 2.;
-    r(0, 0, 1) /= 2.;
+    auto result = this->cr(0, 0, 1) / ar_type{2.};
+    this->r(0, 0, 1) /= 2.;
 
-    check_accessor_correctness(r, t(0, 0, 1));
-    EXPECT_NEAR(r(0, 0, 1), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ReducedStorage3d, Division2)
+TYPED_TEST(ReducedStorage3d, Division2)
 {
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
     const ar_type expected = 5.06 / 4.04;
 
-    auto result = r(1, 0, 0) / cr(0, 2, 0);
-    r(1, 0, 0) /= r(0, 2, 0);
+    auto result = this->r(1, 0, 0) / this->cr(0, 2, 0);
+    this->r(1, 0, 0) /= this->r(0, 2, 0);
 
-    check_accessor_correctness(r, t(1, 0, 0));
-    EXPECT_NEAR(r(1, 0, 0), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(1, 0, 0));
+    EXPECT_NEAR(this->r(1, 0, 0), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ReducedStorage3d, UnaryMinus)
+TYPED_TEST(ReducedStorage3d, UnaryMinus)
 {
-    const ar_type neg_expected = r(2, 0, 0);
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type neg_expected = this->r(2, 0, 0);
     const ar_type expected = -neg_expected;
 
-    auto result = -r(2, 0, 0);
+    auto result = -this->r(2, 0, 0);
 
-    check_accessor_correctness(r);
+    this->check_accessor_correctness(this->r);
     EXPECT_EQ(result, expected);
 }
 
 
-TEST_F(ReducedStorage3d, CanCreateSubrange)
+TYPED_TEST(ReducedStorage3d, CanCreateSubrange)
 {
-    auto subr = r(span{1u, 3u}, span{0u, 2u}, 0u);
+    using st_type = typename TestFixture::st_type;
+
+    auto subr = this->r(gko::span{1u, 3u}, gko::span{0u, 2u}, 0u);
 
-    EXPECT_EQ(subr(0, 0, 0), 5.06);
-    EXPECT_EQ(subr(0, 1, 0), 2.08);
-    EXPECT_EQ(subr(1, 0, 0), -2.12);
-    EXPECT_EQ(subr(1, 1, 0), 0.14);
+    EXPECT_EQ(subr(0, 0, 0), st_type{5.06});
+    EXPECT_EQ(subr(0, 1, 0), st_type{2.08});
+    EXPECT_EQ(subr(1, 0, 0), st_type{-2.12});
+    EXPECT_EQ(subr(1, 1, 0), st_type{0.14});
 }
 
 
-TEST_F(ReducedStorage3d, CanCreateSubrange2)
+TYPED_TEST(ReducedStorage3d, CanCreateSubrange2)
 {
-    auto subr = cr(span{1u, 3u}, span{0u, 2u}, span{0u, 1u});
+    using st_type = typename TestFixture::st_type;
 
-    EXPECT_EQ(subr(0, 0, 0), 5.06);
-    EXPECT_EQ(subr(0, 1, 0), 2.08);
-    EXPECT_EQ(subr(1, 0, 0), -2.12);
-    EXPECT_EQ(subr(1, 1, 0), 0.14);
+    auto subr =
+        this->cr(gko::span{1u, 3u}, gko::span{0u, 2u}, gko::span{0u, 1u});
+
+    EXPECT_EQ(subr(0, 0, 0), st_type{5.06});
+    EXPECT_EQ(subr(0, 1, 0), st_type{2.08});
+    EXPECT_EQ(subr(1, 0, 0), st_type{-2.12});
+    EXPECT_EQ(subr(1, 1, 0), st_type{0.14});
 }
 
 
@@ -527,11 +576,13 @@ TEST_F(ReducedStorageXd, CanWrite2)
 }
 
 
+template <typename ArithmeticStorageType>
 class ScaledReducedStorage3d : public ::testing::Test {
 protected:
-    using span = gko::span;
-    using ar_type = double;
-    using st_type = gko::int32;
+    using ar_type =
+        typename std::tuple_element<0, decltype(ArithmeticStorageType{})>::type;
+    using st_type =
+        typename std::tuple_element<1, decltype(ArithmeticStorageType{})>::type;
     // Type for `check_accessor_correctness` to forward the indices
     using t = std::tuple<int, int, int>;
 
@@ -571,242 +622,273 @@ class ScaledReducedStorage3d : public ::testing::Test {
     {
         // Test for equality is fine here since they should not be modified
         // clang-format off
-        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), 10.);   }
-        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), 22.);   }
-        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), -12.);  }
-        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), 26.);   }
-        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), 14.);   }
-        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), -230.); }
-        if (ignore != t(0, 3, 0)) { EXPECT_EQ(a(0, 3, 0), 6.);    }
-        if (ignore != t(0, 3, 1)) { EXPECT_EQ(a(0, 3, 1), 154.);  }
+        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), ar_type{10.});   }
+        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), ar_type{22.});   }
+        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), ar_type{-12.});  }
+        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), ar_type{26.});   }
+        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), ar_type{14.});   }
+        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), ar_type{-230.}); }
+        if (ignore != t(0, 3, 0)) { EXPECT_EQ(a(0, 3, 0), ar_type{6.});    }
+        if (ignore != t(0, 3, 1)) { EXPECT_EQ(a(0, 3, 1), ar_type{154.});  }
         // clang-format on
     }
 };
 
+using ScaledReducedStorage3dTypes = ::testing::Types<
+    std::tuple<double, gko::int64>, std::tuple<double, gko::int32>,
+    std::tuple<double, gko::int16>, std::tuple<float, gko::int32>,
+    std::tuple<float, gko::int16>>;
+
+TYPED_TEST_SUITE(ScaledReducedStorage3d, ScaledReducedStorage3dTypes);
 
-TEST_F(ScaledReducedStorage3d, CorrectLengths)
+
+TYPED_TEST(ScaledReducedStorage3d, CorrectLengths)
 {
-    EXPECT_EQ(r.length(0), size[0]);
-    EXPECT_EQ(r.length(1), size[1]);
-    EXPECT_EQ(r.length(2), size[2]);
-    EXPECT_EQ(r.length(3), 1);
-    EXPECT_EQ(r->get_size(), size);
+    EXPECT_EQ(this->r.length(0), this->size[0]);
+    EXPECT_EQ(this->r.length(1), this->size[1]);
+    EXPECT_EQ(this->r.length(2), this->size[2]);
+    EXPECT_EQ(this->r.length(3), 1);
+    EXPECT_EQ(this->r->get_size(), this->size);
 }
 
 
-TEST_F(ScaledReducedStorage3d, CorrectStride)
+TYPED_TEST(ScaledReducedStorage3d, CorrectStride)
 {
-    EXPECT_EQ(r->get_stride()[0], size[1] * size[2]);
-    EXPECT_EQ(r->get_stride().at(0), size[1] * size[2]);
-    EXPECT_EQ(r->get_stride()[1], size[2]);
-    EXPECT_EQ(r->get_stride().at(1), size[2]);
+    EXPECT_EQ(this->r->get_stride()[0], this->size[1] * this->size[2]);
+    EXPECT_EQ(this->r->get_stride().at(0), this->size[1] * this->size[2]);
+    EXPECT_EQ(this->r->get_stride()[1], this->size[2]);
+    EXPECT_EQ(this->r->get_stride().at(1), this->size[2]);
 }
 
 
-TEST_F(ScaledReducedStorage3d, CorrectStorage)
+TYPED_TEST(ScaledReducedStorage3d, CorrectStorage)
 {
-    EXPECT_EQ(r->get_stored_data(), data);
-    EXPECT_EQ(r->get_const_storage(), data);
+    EXPECT_EQ(this->r->get_stored_data(), this->data);
+    EXPECT_EQ(this->r->get_const_storage(), this->data);
 }
 
 
-TEST_F(ScaledReducedStorage3d, CorrectScale)
+TYPED_TEST(ScaledReducedStorage3d, CorrectScale)
 {
-    EXPECT_EQ(r->get_scalar(), scalar);
-    EXPECT_EQ(r->get_const_scalar(), scalar);
+    EXPECT_EQ(this->r->get_scalar(), this->scalar);
+    EXPECT_EQ(this->r->get_const_scalar(), this->scalar);
 }
 
 
-TEST_F(ScaledReducedStorage3d, CanReadData)
+TYPED_TEST(ScaledReducedStorage3d, CanReadData)
 {
-    check_accessor_correctness(r);
-    check_accessor_correctness(cr);
+    this->check_accessor_correctness(this->r);
+    this->check_accessor_correctness(this->cr);
 }
 
 
-TEST_F(ScaledReducedStorage3d, CopyFrom)
+TYPED_TEST(ScaledReducedStorage3d, CopyFrom)
 {
-    st_type data2[data_elements];
-    ar_type scale2[scalar_elements];
-    reduced_storage cpy(data2, scale2, size);
+    using ar_type = typename TestFixture::ar_type;
+    using st_type = typename TestFixture::st_type;
+    using reduced_storage = typename TestFixture::reduced_storage;
+    st_type data2[this->data_elements];
+    ar_type scale2[this->scalar_elements];
+    reduced_storage cpy(data2, scale2, this->size);
 
     // Do not use this in regular code since the implementation is slow
-    cpy = r;
+    cpy = this->r;
 
-    check_accessor_correctness(cpy);
+    this->check_accessor_correctness(cpy);
 }
 
 
-TEST_F(ScaledReducedStorage3d, CanImplicitlyConvertToConst)
+TYPED_TEST(ScaledReducedStorage3d, CanImplicitlyConvertToConst)
 {
-    const_reduced_storage const_rs = r->to_const();
-    const_reduced_storage const_rs2 = cr;
+    using const_reduced_storage = typename TestFixture::const_reduced_storage;
+
+    const_reduced_storage const_rs = this->r->to_const();
+    const_reduced_storage const_rs2 = this->cr;
 
-    check_accessor_correctness(const_rs);
-    check_accessor_correctness(const_rs2);
+    this->check_accessor_correctness(const_rs);
+    this->check_accessor_correctness(const_rs2);
 }
 
 
-TEST_F(ScaledReducedStorage3d, ToConstWorks)
+TYPED_TEST(ScaledReducedStorage3d, ToConstWorks)
 {
-    auto cr2 = r->to_const();
+    using const_reduced_storage = typename TestFixture::const_reduced_storage;
+    auto cr2 = this->r->to_const();
 
     static_assert(std::is_same<decltype(cr2), const_reduced_storage>::value,
                   "Types must be equal!");
-    check_accessor_correctness(cr2);
+    this->check_accessor_correctness(cr2);
 }
 
 
-TEST_F(ScaledReducedStorage3d, CanRead)
+TYPED_TEST(ScaledReducedStorage3d, CanRead)
 {
-    check_accessor_correctness(cr);
-    check_accessor_correctness(r);
+    this->check_accessor_correctness(this->cr);
+    this->check_accessor_correctness(this->r);
 }
 
 
-TEST_F(ScaledReducedStorage3d, Subrange)
+TYPED_TEST(ScaledReducedStorage3d, Subrange)
 {
-    auto subr = cr(0u, gko::span{0u, 2u}, 1u);
+    auto subr = this->cr(0u, gko::span{0u, 2u}, 1u);
 
     EXPECT_EQ(subr(0, 0, 0), 22.);
     EXPECT_EQ(subr(0, 1, 0), 26.);
 }
 
 
-TEST_F(ScaledReducedStorage3d, CanWriteScale)
+TYPED_TEST(ScaledReducedStorage3d, CanWriteScale)
 {
-    r->write_scalar(10., 0, 0, 0);
+    this->r->write_scalar(10., 0, 0, 0);
 
-    EXPECT_EQ(r(0, 0, 0), 100.);
-    EXPECT_EQ(r(0, 0, 1), 22.);
-    EXPECT_EQ(r(0, 1, 0), -120.);
-    EXPECT_EQ(r(0, 1, 1), 26.);
-    EXPECT_EQ(r(0, 2, 0), 140.);
-    EXPECT_EQ(r(0, 2, 1), -230.);
-    EXPECT_EQ(r(0, 3, 0), 60.);
-    EXPECT_EQ(r(0, 3, 1), 154.);
+    EXPECT_EQ(this->r(0, 0, 0), 100.);
+    EXPECT_EQ(this->r(0, 0, 1), 22.);
+    EXPECT_EQ(this->r(0, 1, 0), -120.);
+    EXPECT_EQ(this->r(0, 1, 1), 26.);
+    EXPECT_EQ(this->r(0, 2, 0), 140.);
+    EXPECT_EQ(this->r(0, 2, 1), -230.);
+    EXPECT_EQ(this->r(0, 3, 0), 60.);
+    EXPECT_EQ(this->r(0, 3, 1), 154.);
 }
 
 
-TEST_F(ScaledReducedStorage3d, CanReadScale)
+TYPED_TEST(ScaledReducedStorage3d, CanReadScale)
 {
-    EXPECT_EQ(r->read_scalar(0, 0, 0), 1.);
-    EXPECT_EQ(r->read_scalar(0, 0, 1), 2.);
+    EXPECT_EQ(this->r->read_scalar(0, 0, 0), 1.);
+    EXPECT_EQ(this->r->read_scalar(0, 0, 1), 2.);
 }
 
 
-TEST_F(ScaledReducedStorage3d, Addition)
+TYPED_TEST(ScaledReducedStorage3d, Addition)
 {
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
     const ar_type expected = 10. + 3.;
 
-    const auto result = cr(0, 0, 0) + 3.;
-    r(0, 0, 0) += 3.;
+    const auto result = this->cr(0, 0, 0) + 3.;
+    this->r(0, 0, 0) += 3.;
 
-    check_accessor_correctness(r, t(0, 0, 0));
-    EXPECT_NEAR(r(0, 0, 0), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(0, 0, 0));
+    EXPECT_NEAR(this->r(0, 0, 0), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ScaledReducedStorage3d, Addition2)
+TYPED_TEST(ScaledReducedStorage3d, Addition2)
 {
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
     const ar_type expected = 10. + 22.;
 
-    const auto result = cr(0, 0, 0) + r(0, 0, 1);
-    r(0, 0, 0) += cr(0, 0, 1);
+    const auto result = this->cr(0, 0, 0) + this->r(0, 0, 1);
+    this->r(0, 0, 0) += this->cr(0, 0, 1);
 
-    check_accessor_correctness(r, t(0, 0, 0));
-    EXPECT_NEAR(r(0, 0, 0), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(0, 0, 0));
+    EXPECT_NEAR(this->r(0, 0, 0), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ScaledReducedStorage3d, Subtraction)
+TYPED_TEST(ScaledReducedStorage3d, Subtraction)
 {
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
     const ar_type expected = 22. - 2.;
 
-    const auto result = cr(0, 0, 1) - 2.;
-    r(0, 0, 1) -= 2.;
+    const auto result = this->cr(0, 0, 1) - 2.;
+    this->r(0, 0, 1) -= 2.;
 
-    check_accessor_correctness(r, t(0, 0, 1));
-    EXPECT_NEAR(r(0, 0, 1), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ScaledReducedStorage3d, Subtraction2)
+TYPED_TEST(ScaledReducedStorage3d, Subtraction2)
 {
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
     const ar_type expected = -12. - 26.;
 
-    const auto result = cr(0, 1, 0) - r(0, 1, 1);
-    r(0, 1, 0) -= r(0, 1, 1);
+    const auto result = this->cr(0, 1, 0) - this->r(0, 1, 1);
+    this->r(0, 1, 0) -= this->r(0, 1, 1);
 
-    check_accessor_correctness(r, t(0, 1, 0));
-    EXPECT_NEAR(r(0, 1, 0), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(0, 1, 0));
+    EXPECT_NEAR(this->r(0, 1, 0), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ScaledReducedStorage3d, Multiplication)
+TYPED_TEST(ScaledReducedStorage3d, Multiplication)
 {
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
     const ar_type expected = 26. * 3.;
 
-    const auto result = cr(0, 1, 1) * 3.;
-    r(0, 1, 1) *= 3.;
+    const auto result = this->cr(0, 1, 1) * 3.;
+    this->r(0, 1, 1) *= 3.;
 
-    check_accessor_correctness(r, t(0, 1, 1));
-    EXPECT_NEAR(r(0, 1, 1), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(0, 1, 1));
+    EXPECT_NEAR(this->r(0, 1, 1), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ScaledReducedStorage3d, Multiplication2)
+TYPED_TEST(ScaledReducedStorage3d, Multiplication2)
 {
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
     const ar_type expected = 14. * 10.;
 
-    const auto result = r(0, 2, 0) * r(0, 0, 0);
-    r(0, 2, 0) *= r(0, 0, 0);
+    const auto result = this->r(0, 2, 0) * this->r(0, 0, 0);
+    this->r(0, 2, 0) *= this->r(0, 0, 0);
 
-    check_accessor_correctness(r, t(0, 2, 0));
-    EXPECT_NEAR(r(0, 2, 0), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(0, 2, 0));
+    EXPECT_NEAR(this->r(0, 2, 0), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ScaledReducedStorage3d, Division)
+TYPED_TEST(ScaledReducedStorage3d, Division)
 {
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
     const ar_type expected = 10. / 2.;
 
-    const auto result = cr(0, 0, 0) / 2.;
-    r(0, 0, 0) /= 2.;
+    const auto result = this->cr(0, 0, 0) / 2.;
+    this->r(0, 0, 0) /= 2.;
 
-    check_accessor_correctness(r, t(0, 0, 0));
-    EXPECT_NEAR(r(0, 0, 0), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(0, 0, 0));
+    EXPECT_NEAR(this->r(0, 0, 0), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ScaledReducedStorage3d, Division2)
+TYPED_TEST(ScaledReducedStorage3d, Division2)
 {
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
     const ar_type expected = -12. / 6.;
 
-    const auto result = r(0, 1, 0) / r(0, 3, 0);
-    r(0, 1, 0) /= r(0, 3, 0);
+    const auto result = this->r(0, 1, 0) / this->r(0, 3, 0);
+    this->r(0, 1, 0) /= this->r(0, 3, 0);
 
-    check_accessor_correctness(r, t(0, 1, 0));
-    EXPECT_NEAR(r(0, 1, 0), expected, delta);
-    EXPECT_NEAR(result, expected, delta);
+    this->check_accessor_correctness(this->r, t(0, 1, 0));
+    EXPECT_NEAR(this->r(0, 1, 0), expected, this->delta);
+    EXPECT_NEAR(result, expected, this->delta);
 }
 
 
-TEST_F(ScaledReducedStorage3d, UnaryMinus)
+TYPED_TEST(ScaledReducedStorage3d, UnaryMinus)
 {
-    const ar_type neg_expected = r(0, 1, 1);
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type neg_expected = this->r(0, 1, 1);
     const ar_type expected = -neg_expected;
 
-    auto result = -r(0, 1, 1);
+    auto result = -this->r(0, 1, 1);
 
-    check_accessor_correctness(r);
+    this->check_accessor_correctness(this->r);
     EXPECT_EQ(result, expected);
 }
 
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 9a97f59e285..f2d9ccb6310 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -94,7 +94,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 /**
- * Encapsules all arguments in one macro, so a comma inside an argument (e.g.
+ * Encapsulates all arguments in one macro, so a comma inside an argument (e.g.
  * std::tuple<int, float>) still works as an argument and is not considered as
  * multiple arguments.
  */

From fd634c985f5f84d4e1b2edc0c24cd4d77dbe52c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 10 Nov 2020 10:21:30 +0100
Subject: [PATCH 0217/1008] Review update

Update documentation and correct spacing.

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 core/test/base/range_accessors.cpp            | 10 ++--
 .../core/base/range_accessor_helper.hpp       |  2 +-
 .../core/base/range_accessor_references.hpp   |  1 -
 include/ginkgo/core/base/range_accessors.hpp  | 52 +++++++++----------
 4 files changed, 31 insertions(+), 34 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 31b71793b50..93b1954a75b 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -522,8 +522,8 @@ class ReducedStorageXd : public ::testing::Test {
     using const_reduced_storage2d = gko::range<const_accessor2d>;
     using const_reduced_storage1d = gko::range<const_accessor1d>;
 
-    const std::array<const size_type, 0> stride0{};
-    const std::array<const size_type, 1> stride1{4};
+    const std::array<const size_type, 0> stride0{{}};
+    const std::array<const size_type, 1> stride1{{4}};
     const gko::dim<1> size_1d{8u};
     const gko::dim<2> size_2d{2u, 4u};
     static constexpr gko::size_type data_elements{8};
@@ -549,6 +549,7 @@ class ReducedStorageXd : public ::testing::Test {
     }
 };
 
+
 TEST_F(ReducedStorageXd, CanRead)
 {
     EXPECT_EQ(cr1(1), 2.2f);
@@ -921,8 +922,8 @@ class ScaledReducedStorageXd : public ::testing::Test {
     using const_reduced_storage2d = gko::range<const_accessor2d>;
     using const_reduced_storage1d = gko::range<const_accessor1d>;
 
-    const std::array<const size_type, 0> stride0{};
-    const std::array<const size_type, 1> stride1{4};
+    const std::array<const size_type, 0> stride0{{}};
+    const std::array<const size_type, 1> stride1{{4}};
     const gko::dim<1> size_1d{8u};
     const gko::dim<2> size_2d{2u, 4u};
 
@@ -966,6 +967,7 @@ class ScaledReducedStorageXd : public ::testing::Test {
     }
 };
 
+
 TEST_F(ScaledReducedStorageXd, CanRead)
 {
     EXPECT_NEAR(cr1(1), 2.2, delta);
diff --git a/include/ginkgo/core/base/range_accessor_helper.hpp b/include/ginkgo/core/base/range_accessor_helper.hpp
index 1aace2f7675..464088a96ef 100644
--- a/include/ginkgo/core/base/range_accessor_helper.hpp
+++ b/include/ginkgo/core/base/range_accessor_helper.hpp
@@ -213,6 +213,7 @@ mult_array(const dim<N> &size)
 {
     return 1;
 }
+
 template <typename ValueType, size_type Iter, size_type N>
 constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<Iter + 1 <= N, ValueType>
 mult_array(const dim<N> &size)
@@ -304,7 +305,6 @@ spans_in_size(const dim<N> &size)
     return 0;
 }
 
-
 template <size_type Iter, size_type N, typename First, typename... Remaining>
 GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<(Iter < N), int>
 spans_in_size(const dim<N> &size, First &&first, Remaining &&... remaining)
diff --git a/include/ginkgo/core/base/range_accessor_references.hpp b/include/ginkgo/core/base/range_accessor_references.hpp
index 1228e309089..680730d55f2 100644
--- a/include/ginkgo/core/base/range_accessor_references.hpp
+++ b/include/ginkgo/core/base/range_accessor_references.hpp
@@ -403,7 +403,6 @@ class scaled_reduced_storage<ArithmeticType, const StorageType>
         return static_cast<arithmetic_type>(*r_ptr) * scalar_;
     }
 
-
 private:
     storage_type *const GKO_RESTRICT ptr_;
     const arithmetic_type scalar_;
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index b4b6503f8f9..2ed352a8952 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -219,11 +219,11 @@ class row_major {
 /**
  * The reduced_row_major class allows a storage format that is different from
  * the arithmetic format (which is returned from the brace operator).
- * As storage, the storage_type is used.
+ * As storage, the StorageType is used.
  *
- * This accessor uses row-major access, meaning neighboring z coordinates are
- * next to each other in memory, followed by y coordinates and then x
- * coordinates.
+ * This accessor uses row-major access. For example for three dimensions,
+ * neighboring z coordinates are next to each other in memory, followed by y
+ * coordinates and then x coordinates.
  *
  * @tparam Dimensionality  The number of dimensions managed by this accessor
  *
@@ -254,7 +254,7 @@ class reduced_row_major {
         reference_class::reduced_storage<arithmetic_type, storage_type>;
 
     /**
-     * Creates the accessor with an already allocated storage space with a
+     * Creates the accessor for an already allocated storage space with a
      * stride. The first stride is used for computing the index for the first
      * index, the second stride for the second index, and so on.
      *
@@ -269,7 +269,7 @@ class reduced_row_major {
     {}
 
     /**
-     * Creates the accessor with an already allocated storage space with a
+     * Creates the accessor for an already allocated storage space with a
      * stride. The first stride is used for computing the index for the first
      * index, the second stride for the second index, and so on.
      *
@@ -284,10 +284,8 @@ class reduced_row_major {
     {}
 
     /**
-     * Creates the accessor with an already allocated storage space with a
-     * stride. The first stride is used for computing the index for the first
-     * index, the second stride for the second index, and so on.
-     * It is assumed that accesses are without padding.
+     * Creates the accessor for an already allocated storage space.
+     * It is assumed that all accesses are without padding.
      *
      * @param storage  pointer to the block of memory containing the storage
      * @param size  multidimensional size of the memory
@@ -299,7 +297,7 @@ class reduced_row_major {
     {}
 
     /**
-     * Creates the accessor with an already allocated storage space with a
+     * Creates the accessor for an already allocated storage space with a
      * stride. The first stride is used for computing the index for the first
      * index, the second stride for the second index, and so on.
      *
@@ -530,11 +528,11 @@ struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
  * from the arithmetic format (which is returned from the brace operator).
  * Additionally, a scalar is used when reading and writing data to allow for
  * a shift in range.
- * As storage, the storage_type is used.
+ * As storage, the StorageType is used.
  *
- * This accessor uses row-major access, meaning neighboring z coordinates are
- * next to each other in memory, followed by y coordinates and then x
- * coordinates.
+ * This accessor uses row-major access. For example, for three dimensions,
+ * neighboring z coordinates are next to each other in memory, followed by y
+ * coordinates and then x coordinates.
  *
  * @tparam Dimensionality  The number of dimensions managed by this accessor
  *
@@ -589,7 +587,7 @@ class scaled_reduced_row_major
 
 public:
     /**
-     * Creates the accessor with an already allocated storage space with a
+     * Creates the accessor for an already allocated storage space with a
      * stride. The first stride is used for computing the index for the first
      * index, the second stride for the second index, and so on.
      *
@@ -597,7 +595,7 @@ class scaled_reduced_row_major
      * @param scalar  pointer to the block of memory containing the scalar
      *                values.
      * @param size  multidimensional size of the memory
-     * @param stride  strides used for memory accesses
+     * @param stride  stride array used for memory accesses
      */
     GKO_ATTRIBUTES constexpr scaled_reduced_row_major(
         storage_type *storage, scalar_type *scalar, dim<dimensionality> size,
@@ -606,7 +604,7 @@ class scaled_reduced_row_major
     {}
 
     /**
-     * Creates the accessor with an already allocated storage space with a
+     * Creates the accessor for an already allocated storage space with a
      * stride. The first stride is used for computing the index for the first
      * index, the second stride for the second index, and so on.
      *
@@ -614,7 +612,7 @@ class scaled_reduced_row_major
      * @param scalar  pointer to the block of memory containing the scalar
      *                values.
      * @param size  multidimensional size of the memory
-     * @param stride  strides used for memory accesses
+     * @param stride  stride array used for memory accesses
      */
     GKO_ATTRIBUTES constexpr scaled_reduced_row_major(
         storage_type *storage, scalar_type *scalar, dim<dimensionality> size,
@@ -623,10 +621,8 @@ class scaled_reduced_row_major
     {}
 
     /**
-     * Creates the accessor with an already allocated storage space with a
-     * stride. The first stride is used for computing the index for the first
-     * index, the second stride for the second index, and so on.
-     * It is assumed that accesses are without padding.
+     * Creates the accessor for an already allocated storage space.
+     * It is assumed that all accesses are without padding.
      *
      * @param storage  pointer to the block of memory containing the storage
      * @param scalar  pointer to the block of memory containing the scalar
@@ -642,7 +638,7 @@ class scaled_reduced_row_major
     {}
 
     /**
-     * Creates the accessor with an already allocated storage space with a
+     * Creates the accessor for an already allocated storage space with a
      * stride. The first stride is used for computing the index for the first
      * index, the second stride for the second index, and so on.
      *
@@ -650,7 +646,7 @@ class scaled_reduced_row_major
      * @param scalar  pointer to the block of memory containing the scalar
      *                values.
      * @param size  multidimensional size of the memory
-     * @param strides  strides used for memory accesses
+     * @param strides  stride array used for memory accesses
      */
     template <typename... Strides>
     GKO_ATTRIBUTES constexpr scaled_reduced_row_major(storage_type *storage,
@@ -674,10 +670,10 @@ class scaled_reduced_row_major
     {}
 
     /**
-     * Creates a reduced_row_major range which contains a read-only version of
-     * the current accessor.
+     * Creates a scaled_reduced_row_major range which contains a read-only
+     * version of the current accessor.
      *
-     * @returns  a reduced_row_major major range which is read-only.
+     * @returns  a scaled_reduced_row_major major range which is read-only.
      */
     GKO_ATTRIBUTES GKO_INLINE constexpr range<const_accessor> to_const() const
     {

From b2e06078c035e3878d1df9b0388eedec650abe2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 12 Nov 2020 13:40:47 +0100
Subject: [PATCH 0218/1008] Fix compiler error in accessor test

Also remove `this->` from accessor implementation
---
 core/test/base/range_accessors.cpp           | 76 ++++++++++----------
 include/ginkgo/core/base/range_accessors.hpp | 32 +++++----
 2 files changed, 57 insertions(+), 51 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 93b1954a75b..402dad50037 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -268,7 +268,7 @@ TYPED_TEST(ReducedStorage3d, CopyFrom)
 {
     using st_type = typename TestFixture::st_type;
     using reduced_storage = typename TestFixture::reduced_storage;
-    st_type data2[this->data_elements];
+    st_type data2[TestFixture::data_elements];
     reduced_storage cpy(data2, this->size);
 
     // Do not use this in regular code since the implementation is slow
@@ -309,7 +309,7 @@ TYPED_TEST(ReducedStorage3d, CanWriteData)
     this->r(0, 1, 0) = 100.25;
 
     this->check_accessor_correctness(this->r, t(0, 1, 0));
-    EXPECT_NEAR(this->r(0, 1, 0), 100.25, this->delta);
+    EXPECT_NEAR(this->r(0, 1, 0), 100.25, TestFixture::delta);
 }
 
 
@@ -320,7 +320,7 @@ TYPED_TEST(ReducedStorage3d, Assignment)
     this->r(0, 0, 1) = 10.2;
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), 10.2, this->delta);
+    EXPECT_NEAR(this->r(0, 0, 1), 10.2, TestFixture::delta);
 }
 
 
@@ -331,7 +331,7 @@ TYPED_TEST(ReducedStorage3d, Assignment2)
     this->r(0, 0, 1) = this->r(0, 1, 0);
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), -1.02, this->delta);
+    EXPECT_NEAR(this->r(0, 0, 1), -1.02, TestFixture::delta);
 }
 
 
@@ -345,8 +345,8 @@ TYPED_TEST(ReducedStorage3d, Addition)
     this->r(0, 0, 1) += 10.2;
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -360,8 +360,8 @@ TYPED_TEST(ReducedStorage3d, Addition2)
     this->r(0, 0, 1) += this->r(0, 1, 0);
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -375,8 +375,8 @@ TYPED_TEST(ReducedStorage3d, Subtraction)
     this->r(3, 2, 1) -= 1;
 
     this->check_accessor_correctness(this->r, t(3, 2, 1));
-    EXPECT_NEAR(this->r(3, 2, 1), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(3, 2, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -390,8 +390,8 @@ TYPED_TEST(ReducedStorage3d, Subtraction2)
     this->r(3, 2, 0) -= this->r(3, 2, 1);
 
     this->check_accessor_correctness(this->r, t(3, 2, 0));
-    EXPECT_NEAR(this->r(3, 2, 0), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(3, 2, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -405,8 +405,8 @@ TYPED_TEST(ReducedStorage3d, Multiplication)
     this->r(0, 0, 0) *= 2;
 
     this->check_accessor_correctness(this->r, t(0, 0, 0));
-    EXPECT_NEAR(this->r(0, 0, 0), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -420,8 +420,8 @@ TYPED_TEST(ReducedStorage3d, Multiplication2)
     this->r(0, 0, 1) *= this->r(0, 1, 1);
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -435,8 +435,8 @@ TYPED_TEST(ReducedStorage3d, Division)
     this->r(0, 0, 1) /= 2.;
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -450,8 +450,8 @@ TYPED_TEST(ReducedStorage3d, Division2)
     this->r(1, 0, 0) /= this->r(0, 2, 0);
 
     this->check_accessor_correctness(this->r, t(1, 0, 0));
-    EXPECT_NEAR(this->r(1, 0, 0), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(1, 0, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -688,8 +688,8 @@ TYPED_TEST(ScaledReducedStorage3d, CopyFrom)
     using ar_type = typename TestFixture::ar_type;
     using st_type = typename TestFixture::st_type;
     using reduced_storage = typename TestFixture::reduced_storage;
-    st_type data2[this->data_elements];
-    ar_type scale2[this->scalar_elements];
+    st_type data2[TestFixture::data_elements];
+    ar_type scale2[TestFixture::scalar_elements];
     reduced_storage cpy(data2, scale2, this->size);
 
     // Do not use this in regular code since the implementation is slow
@@ -770,8 +770,8 @@ TYPED_TEST(ScaledReducedStorage3d, Addition)
     this->r(0, 0, 0) += 3.;
 
     this->check_accessor_correctness(this->r, t(0, 0, 0));
-    EXPECT_NEAR(this->r(0, 0, 0), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -785,8 +785,8 @@ TYPED_TEST(ScaledReducedStorage3d, Addition2)
     this->r(0, 0, 0) += this->cr(0, 0, 1);
 
     this->check_accessor_correctness(this->r, t(0, 0, 0));
-    EXPECT_NEAR(this->r(0, 0, 0), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -800,8 +800,8 @@ TYPED_TEST(ScaledReducedStorage3d, Subtraction)
     this->r(0, 0, 1) -= 2.;
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -815,8 +815,8 @@ TYPED_TEST(ScaledReducedStorage3d, Subtraction2)
     this->r(0, 1, 0) -= this->r(0, 1, 1);
 
     this->check_accessor_correctness(this->r, t(0, 1, 0));
-    EXPECT_NEAR(this->r(0, 1, 0), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(0, 1, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -830,8 +830,8 @@ TYPED_TEST(ScaledReducedStorage3d, Multiplication)
     this->r(0, 1, 1) *= 3.;
 
     this->check_accessor_correctness(this->r, t(0, 1, 1));
-    EXPECT_NEAR(this->r(0, 1, 1), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(0, 1, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -845,8 +845,8 @@ TYPED_TEST(ScaledReducedStorage3d, Multiplication2)
     this->r(0, 2, 0) *= this->r(0, 0, 0);
 
     this->check_accessor_correctness(this->r, t(0, 2, 0));
-    EXPECT_NEAR(this->r(0, 2, 0), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(0, 2, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -860,8 +860,8 @@ TYPED_TEST(ScaledReducedStorage3d, Division)
     this->r(0, 0, 0) /= 2.;
 
     this->check_accessor_correctness(this->r, t(0, 0, 0));
-    EXPECT_NEAR(this->r(0, 0, 0), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
@@ -875,8 +875,8 @@ TYPED_TEST(ScaledReducedStorage3d, Division2)
     this->r(0, 1, 0) /= this->r(0, 3, 0);
 
     this->check_accessor_correctness(this->r, t(0, 1, 0));
-    EXPECT_NEAR(this->r(0, 1, 0), expected, this->delta);
-    EXPECT_NEAR(result, expected, this->delta);
+    EXPECT_NEAR(this->r(0, 1, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
 }
 
 
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 2ed352a8952..d6598125026 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -384,7 +384,7 @@ class reduced_row_major {
         std::conditional_t<is_const, arithmetic_type, reference_type>>
     operator()(Indices &&... indices) const
     {
-        return reference_type{this->storage_ +
+        return reference_type{storage_ +
                               compute_index(std::forward<Indices>(indices)...)};
     }
 
@@ -600,7 +600,10 @@ class scaled_reduced_row_major
     GKO_ATTRIBUTES constexpr scaled_reduced_row_major(
         storage_type *storage, scalar_type *scalar, dim<dimensionality> size,
         const std::array<const size_type, dimensionality - 1> &stride)
-        : storage_{storage}, scalar_{scalar}, size_{size}, stride_{stride}
+        : storage_{storage},
+          scalar_{scalar},
+          size_{size},
+          storage_stride_{stride}
     {}
 
     /**
@@ -617,7 +620,10 @@ class scaled_reduced_row_major
     GKO_ATTRIBUTES constexpr scaled_reduced_row_major(
         storage_type *storage, scalar_type *scalar, dim<dimensionality> size,
         std::array<const size_type, dimensionality - 1> &&stride)
-        : storage_{storage}, scalar_{scalar}, size_{size}, stride_{stride}
+        : storage_{storage},
+          scalar_{scalar},
+          size_{size},
+          storage_stride_{stride}
     {}
 
     /**
@@ -656,7 +662,7 @@ class scaled_reduced_row_major
         : storage_{storage},
           scalar_{scalar},
           size_{size},
-          stride_{std::forward<Strides>(strides)...}
+          storage_stride_{std::forward<Strides>(strides)...}
     {
         static_assert(sizeof...(Strides) + 1 == dimensionality,
                       "Number of provided Strides must be dimensionality - 1!");
@@ -677,7 +683,7 @@ class scaled_reduced_row_major
      */
     GKO_ATTRIBUTES GKO_INLINE constexpr range<const_accessor> to_const() const
     {
-        return range<const_accessor>{storage_, scalar_, size_, stride_};
+        return range<const_accessor>{storage_, scalar_, size_, storage_stride_};
     }
 
     /**
@@ -746,7 +752,7 @@ class scaled_reduced_row_major
     operator()(Indices... indices) const
     {
         return reference_type{
-            this->storage_ + compute_index(std::forward<Indices>(indices)...),
+            storage_ + compute_index(std::forward<Indices>(indices)...),
             read_scalar(std::forward<Indices>(indices)...)};
     }
 
@@ -769,7 +775,7 @@ class scaled_reduced_row_major
                    scalar_ + compute_scalar_index(span{spans}.begin...),
                    dim<dimensionality>{
                        (span{spans}.end - span{spans}.begin)...},
-                   stride_};
+                   storage_stride_};
     }
 
     /**
@@ -791,7 +797,7 @@ class scaled_reduced_row_major
     GKO_INLINE constexpr const std::array<const size_type, dimensionality - 1>
         &get_stride() const
     {
-        return stride_;
+        return storage_stride_;
     }
 
     /**
@@ -822,7 +828,7 @@ class scaled_reduced_row_major
      */
     GKO_ATTRIBUTES GKO_INLINE constexpr scalar_type *get_scalar() const
     {
-        return this->scalar_;
+        return scalar_;
     }
 
     /**
@@ -833,7 +839,7 @@ class scaled_reduced_row_major
     GKO_ATTRIBUTES GKO_INLINE constexpr const scalar_type *get_const_scalar()
         const
     {
-        return this->scalar_;
+        return scalar_;
     }
 
 protected:
@@ -844,7 +850,7 @@ class scaled_reduced_row_major
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
         return helper::row_major_index<
-            const size_type, dimensionality>::compute(size_, stride_,
+            const size_type, dimensionality>::compute(size_, storage_stride_,
                                                       std::forward<Indices>(
                                                           indices)...);
     }
@@ -857,14 +863,14 @@ class scaled_reduced_row_major
                       "Number of indices must match dimensionality!");
         return helper::row_major_mask_index<
             const size_type, dimensionality,
-            scalar_mask>::compute(size_, stride_,
+            scalar_mask>::compute(size_, storage_stride_,
                                   std::forward_as_tuple(indices...));
     }
 
     storage_type *storage_;
     scalar_type *scalar_;
     const dim<dimensionality> size_;
-    const std::array<const size_type, dimensionality - 1> stride_;
+    const std::array<const size_type, dimensionality - 1> storage_stride_;
 };
 
 

From 100b86b728b7cb8ccaa9cef177219e6800db4be9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 12 Nov 2020 21:34:51 +0100
Subject: [PATCH 0219/1008] Add separate stride for scalar

---
 core/test/base/range_accessors.cpp            |  39 ++--
 .../core/base/range_accessor_helper.hpp       | 107 ++++++++++-
 include/ginkgo/core/base/range_accessors.hpp  | 178 ++++++------------
 3 files changed, 182 insertions(+), 142 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 402dad50037..095762ab188 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -39,6 +39,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <tuple>
 #include <type_traits>
 
+#include <iostream>
+
 
 #include <ginkgo/core/base/dim.hpp>
 #include <ginkgo/core/base/types.hpp>
@@ -181,8 +183,8 @@ class ReducedStorage3d : public ::testing::Test {
         3.22, -2.23
     };
     // clang-format on
-    reduced_storage r{data, size};
-    const_reduced_storage cr{data, size};
+    reduced_storage r{size, data};
+    const_reduced_storage cr{size, data};
 
     template <typename Accessor>
     static void check_accessor_correctness(
@@ -269,7 +271,7 @@ TYPED_TEST(ReducedStorage3d, CopyFrom)
     using st_type = typename TestFixture::st_type;
     using reduced_storage = typename TestFixture::reduced_storage;
     st_type data2[TestFixture::data_elements];
-    reduced_storage cpy(data2, this->size);
+    reduced_storage cpy(this->size, data2);
 
     // Do not use this in regular code since the implementation is slow
     cpy = this->r;
@@ -522,17 +524,17 @@ class ReducedStorageXd : public ::testing::Test {
     using const_reduced_storage2d = gko::range<const_accessor2d>;
     using const_reduced_storage1d = gko::range<const_accessor1d>;
 
-    const std::array<const size_type, 0> stride0{{}};
-    const std::array<const size_type, 1> stride1{{4}};
+    const std::array<size_type, 0> stride0{{}};
+    const std::array<size_type, 1> stride1{{4}};
     const gko::dim<1> size_1d{8u};
     const gko::dim<2> size_2d{2u, 4u};
     static constexpr gko::size_type data_elements{8};
     st_type data[data_elements]{1.1f, 2.2f, 3.3f, 4.4f,
                                 5.5f, 6.6f, 7.7f, -8.8f};
-    reduced_storage1d r1{data, size_1d};
-    reduced_storage2d r2{data, size_2d, stride1[0]};
-    const_reduced_storage1d cr1{data, size_1d, stride0};
-    const_reduced_storage2d cr2{data, size_2d, stride1};
+    reduced_storage1d r1{size_1d, data};
+    reduced_storage2d r2{size_2d, data, stride1[0]};
+    const_reduced_storage1d cr1{size_1d, data, stride0};
+    const_reduced_storage2d cr2{size_2d, data, stride1};
 
     void data_equal_except_for(int idx)
     {
@@ -612,8 +614,8 @@ class ScaledReducedStorage3d : public ::testing::Test {
         1., 2.,
     };
     // clang-format on
-    reduced_storage r{data, scalar, size};
-    const_reduced_storage cr{data, scalar, size};
+    reduced_storage r{size, data, scalar};
+    const_reduced_storage cr{size, data, scalar};
 
     template <typename Accessor>
     static void check_accessor_correctness(
@@ -690,7 +692,7 @@ TYPED_TEST(ScaledReducedStorage3d, CopyFrom)
     using reduced_storage = typename TestFixture::reduced_storage;
     st_type data2[TestFixture::data_elements];
     ar_type scale2[TestFixture::scalar_elements];
-    reduced_storage cpy(data2, scale2, this->size);
+    reduced_storage cpy(this->size, data2, scale2);
 
     // Do not use this in regular code since the implementation is slow
     cpy = this->r;
@@ -922,8 +924,9 @@ class ScaledReducedStorageXd : public ::testing::Test {
     using const_reduced_storage2d = gko::range<const_accessor2d>;
     using const_reduced_storage1d = gko::range<const_accessor1d>;
 
-    const std::array<const size_type, 0> stride0{{}};
-    const std::array<const size_type, 1> stride1{{4}};
+    const std::array<size_type, 0> stride0{{}};
+    const std::array<size_type, 1> stride1{{4}};
+    const size_type stride1nc[1] = {4};
     const gko::dim<1> size_1d{8u};
     const gko::dim<2> size_2d{2u, 4u};
 
@@ -934,10 +937,10 @@ class ScaledReducedStorageXd : public ::testing::Test {
         default_scalar, default_scalar, default_scalar, default_scalar,
         default_scalar, default_scalar, default_scalar, default_scalar};
 
-    reduced_storage1d r1{data, scalar, size_1d};
-    reduced_storage2d r2{data, scalar, size_2d, stride1[0]};
-    const_reduced_storage1d cr1{data, scalar, size_1d, stride0};
-    const_reduced_storage2d cr2{data, scalar, size_2d, stride1};
+    reduced_storage1d r1{size_1d, data, scalar};
+    reduced_storage2d r2{size_2d, data, stride1, scalar};
+    const_reduced_storage1d cr1{size_1d, data, stride0, scalar};
+    const_reduced_storage2d cr2{size_2d, data, stride1, scalar};
 
     void data_equal_except_for(int idx)
     {
diff --git a/include/ginkgo/core/base/range_accessor_helper.hpp b/include/ginkgo/core/base/range_accessor_helper.hpp
index 464088a96ef..84146ca83e6 100644
--- a/include/ginkgo/core/base/range_accessor_helper.hpp
+++ b/include/ginkgo/core/base/range_accessor_helper.hpp
@@ -106,13 +106,13 @@ struct row_major_index<IndexType, total_dim, 1> {
  * compute(stride, tuple(x1, x2, x3, x4))
  * -> x4 + x2 * stride[2] + x1 * stride[1]
  */
-template <typename IndexType, int32 total_dim, int32 mask,
+template <typename IndexType, int32 total_dim, size_type mask,
           int32 current_dim = total_dim - 1,
           int32 current_scalar = total_dim - 1>
 struct row_major_mask_index {
-    static constexpr int32 msb_mask{static_cast<int32>(1)};
+    static constexpr size_type msb_mask{size_type{1} << current_dim};
     static constexpr int32 stride_size = total_dim > 1 ? total_dim - 1 : 0;
-    static constexpr int32 use_current_stride{mask & static_cast<int32>(1)};
+    static constexpr bool use_current_stride = mask & msb_mask;
     static constexpr bool is_last_stride_dim{stride_size == 0 ||
                                              current_scalar == total_dim - 1};
 
@@ -128,18 +128,20 @@ struct row_major_mask_index {
                           (is_last_stride_dim ? 1 : stride[current_scalar])
                     : 0) +
                    row_major_mask_index<
-                       IndexType, total_dim, (mask >> 1), current_dim - 1,
+                       IndexType, total_dim, mask, current_dim - 1,
                        current_scalar - use_current_stride>::compute(size,
                                                                      stride,
                                                                      indices);
     }
 };
 
-template <typename IndexType, int32 total_dim, int32 mask, int32 current_scalar>
+template <typename IndexType, int32 total_dim, size_type mask,
+          int32 current_scalar>
 struct row_major_mask_index<IndexType, total_dim, mask, 0, current_scalar> {
-    static constexpr int32 stride_size = total_dim > 1 ? total_dim - 1 : 0;
-    static constexpr int32 use_current_stride{mask & static_cast<int32>(1)};
     static constexpr int32 current_dim{0};
+    static constexpr size_type msb_mask{size_type{1}};
+    static constexpr int32 stride_size = total_dim > 1 ? total_dim - 1 : 0;
+    static constexpr bool use_current_stride{mask & msb_mask};
     static constexpr bool is_last_stride_dim{stride_size == 0 ||
                                              current_scalar == total_dim - 1};
 
@@ -222,6 +224,32 @@ mult_array(const dim<N> &size)
 }
 
 
+template <typename ValueType, size_type Iter, size_type mask, size_type N>
+constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<Iter == N, ValueType>
+mult_masked_array(const dim<N> &size)
+{
+    return 1;
+}
+
+// in case corresponding bit is set
+template <typename ValueType, size_type Iter, size_type mask, size_type N>
+constexpr GKO_ATTRIBUTES GKO_INLINE
+    std::enable_if_t<Iter + 1 <= N && (mask & size_type{1}), ValueType>
+    mult_masked_array(const dim<N> &size)
+{
+    return size[Iter] * mult_array<ValueType, Iter + 1>(size);
+}
+
+// in case corresponding bit is NOT set
+template <typename ValueType, size_type Iter, size_type mask, size_type N>
+constexpr GKO_ATTRIBUTES GKO_INLINE
+    std::enable_if_t<Iter + 1 <= N && ((mask & size_type{1}) == 0), ValueType>
+    mult_masked_array(const dim<N> &size)
+{
+    return mult_array<ValueType, Iter + 1, (mask >> 1)>(size);
+}
+
+
 template <typename ValueType, size_type Iter = 1, size_type N, typename... Args>
 constexpr GKO_ATTRIBUTES GKO_INLINE
     std::enable_if_t<N == 0 || (Iter == N && Iter == sizeof...(Args) + 1),
@@ -231,7 +259,6 @@ constexpr GKO_ATTRIBUTES GKO_INLINE
     return {{args...}};
 }
 
-
 template <typename ValueType, size_type Iter = 1, size_type N, typename... Args>
 constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<
     (Iter < N) && (Iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
@@ -242,6 +269,41 @@ extract_factorization(const dim<N> &size, Args... args)
 }
 
 
+template <typename ValueType, size_type Iter = 1, size_type mask,
+          size_type res_size, size_type N, typename... Args>
+constexpr GKO_ATTRIBUTES GKO_INLINE
+    std::enable_if_t<N == 0 || (Iter == N && res_size == sizeof...(Args) + 1),
+                     std::array<ValueType, res_size>>
+    extract_masked_factorization(const dim<N> &size, Args... args)
+{
+    return {{args...}};
+}
+
+// In case the corresponding bit is set
+template <typename ValueType, size_type Iter = 1, size_type mask,
+          size_type res_size, size_type N, typename... Args>
+constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<
+    (Iter < N) && (Iter == sizeof...(Args) + 1) && (mask & size_type{1}),
+    std::array<ValueType, res_size>>
+extract_factorization(const dim<N> &size, Args... args)
+{
+    return extract_masked_factorization<ValueType, Iter + 1>(
+        size, args..., mult_array<ValueType, Iter>(size));
+}
+
+// In case the corresponding bit is NOT set
+template <typename ValueType, size_type Iter = 1, size_type mask,
+          size_type res_size, size_type N, typename... Args>
+constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<
+    (Iter < N) && (Iter == sizeof...(Args) + 1) && ((mask & size_type{1}) == 0),
+    std::array<ValueType, res_size>>
+extract_factorization(const dim<N> &size, Args... args)
+{
+    return extract_masked_factorization<ValueType, Iter + 1>(
+        size, args..., mult_array<ValueType, Iter>(size));
+}
+
+
 }  // namespace detail
 
 
@@ -328,6 +390,35 @@ GKO_ATTRIBUTES GKO_INLINE constexpr int validate_spans(const dim<N> &size,
 }
 
 
+namespace detail {
+
+
+template <int32 mask, size_type N, size_type Iter = 0>
+constexpr std::enable_if_t<Iter == N, size_type>
+count_mask_dimensionality_impl()
+{
+    return 0;
+}
+
+template <int32 mask, size_type N, size_type Iter = 0>
+constexpr std::enable_if_t<(Iter < N), size_type>
+count_mask_dimensionality_impl()
+{
+    return (mask & int32{1}) +
+           count_mask_dimensionality_impl<(mask >> 1), N, Iter + 1>();
+}
+
+
+}  // namespace detail
+
+
+template <int32 mask, size_type N>
+constexpr size_type count_mask_dimensionality()
+{
+    return detail::count_mask_dimensionality_impl<mask, N>();
+}
+
+
 }  // namespace helper
 }  // namespace accessor
 }  // namespace gko
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index d6598125026..fa3719d1a4e 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -250,6 +250,7 @@ class reduced_row_major {
     friend class range<reduced_row_major>;
 
 protected:
+    using storage_stride_type = std::array<size_type, dimensionality - 1>;
     using reference_type =
         reference_class::reduced_storage<arithmetic_type, storage_type>;
 
@@ -258,29 +259,14 @@ class reduced_row_major {
      * stride. The first stride is used for computing the index for the first
      * index, the second stride for the second index, and so on.
      *
-     * @param storage  pointer to the block of memory containing the storage
      * @param size  multidimensional size of the memory
-     * @param stride  stride array used for memory accesses
-     */
-    GKO_ATTRIBUTES constexpr reduced_row_major(
-        storage_type *storage, gko::dim<dimensionality> size,
-        const std::array<const size_type, dimensionality - 1> &stride)
-        : storage_{storage}, size_{size}, stride_{stride}
-    {}
-
-    /**
-     * Creates the accessor for an already allocated storage space with a
-     * stride. The first stride is used for computing the index for the first
-     * index, the second stride for the second index, and so on.
-     *
      * @param storage  pointer to the block of memory containing the storage
-     * @param size  multidimensional size of the memory
      * @param stride  stride array used for memory accesses
      */
-    GKO_ATTRIBUTES constexpr reduced_row_major(
-        storage_type *storage, gko::dim<dimensionality> size,
-        std::array<const size_type, dimensionality - 1> &&stride)
-        : storage_{storage}, size_{size}, stride_{stride}
+    GKO_ATTRIBUTES constexpr reduced_row_major(gko::dim<dimensionality> size,
+                                               storage_type *storage,
+                                               storage_stride_type stride)
+        : size_{size}, storage_{storage}, stride_{stride}
     {}
 
     /**
@@ -290,10 +276,10 @@ class reduced_row_major {
      * @param storage  pointer to the block of memory containing the storage
      * @param size  multidimensional size of the memory
      */
-    GKO_ATTRIBUTES constexpr reduced_row_major(storage_type *storage,
-                                               dim<dimensionality> size)
-        : reduced_row_major{storage, size,
-                            helper::compute_stride_array<const size_type>(size)}
+    GKO_ATTRIBUTES constexpr reduced_row_major(dim<dimensionality> size,
+                                               storage_type *storage)
+        : reduced_row_major{size, storage,
+                            helper::compute_stride_array<size_type>(size)}
     {}
 
     /**
@@ -306,11 +292,11 @@ class reduced_row_major {
      * @param strides  strides used for memory accesses
      */
     template <typename... Strides>
-    GKO_ATTRIBUTES constexpr reduced_row_major(storage_type *storage,
-                                               gko::dim<dimensionality> size,
+    GKO_ATTRIBUTES constexpr reduced_row_major(gko::dim<dimensionality> size,
+                                               storage_type *storage,
                                                Strides &&... strides)
-        : storage_{storage},
-          size_{size},
+        : size_{size},
+          storage_{storage},
           stride_{std::forward<Strides>(strides)...}
     {
         static_assert(sizeof...(Strides) + 1 == dimensionality,
@@ -321,7 +307,7 @@ class reduced_row_major {
      * Creates an empty accessor (pointing nowhere with an empty size)
      */
     GKO_ATTRIBUTES constexpr reduced_row_major()
-        : reduced_row_major{nullptr, {0, 0, 0}}
+        : reduced_row_major{{0, 0, 0}, nullptr}
     {}
 
 public:
@@ -333,7 +319,7 @@ class reduced_row_major {
      */
     GKO_ATTRIBUTES GKO_INLINE constexpr range<const_accessor> to_const() const
     {
-        return range<const_accessor>{storage_, size_, stride_};
+        return range<const_accessor>{size_, storage_, stride_};
     }
 
     /**
@@ -403,10 +389,9 @@ class reduced_row_major {
     {
         return helper::validate_spans(size_, spans...),
                range<reduced_row_major>{
-                   storage_ + compute_index((span{spans}.begin)...),
                    dim<dimensionality>{
                        (span{spans}.end - span{spans}.begin)...},
-                   stride_};
+                   storage_ + compute_index((span{spans}.begin)...), stride_};
     }
 
     /**
@@ -425,8 +410,7 @@ class reduced_row_major {
      * @returns returns a pointer to a stride array of size dimensionality - 1
      */
     GKO_ATTRIBUTES
-    GKO_INLINE constexpr const std::array<const size_type, dimensionality - 1>
-        &get_stride() const
+    GKO_INLINE constexpr const storage_stride_type &get_stride() const
     {
         return stride_;
     }
@@ -459,15 +443,13 @@ class reduced_row_major {
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
-        return helper::row_major_index<
-            const size_type, dimensionality>::compute(size_, stride_,
-                                                      std::forward<Indices>(
-                                                          indices)...);
+        return helper::row_major_index<size_type, dimensionality>::compute(
+            size_, stride_, std::forward<Indices>(indices)...);
     }
 
-    storage_type *storage_;
     const dim<dimensionality> size_;
-    const std::array<const size_type, dimensionality - 1> stride_;
+    storage_type *storage_;
+    const storage_stride_type stride_;
 };
 
 
@@ -544,14 +526,17 @@ struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
  * @tparam ScalarMask  Binary mask that marks which indices matter for the
  *                     scalar selection (set bit means the corresponding index
  *                     needs to be considered, 0 means it is not). The least
- *                     significand bit corresponds to the first index dimension,
- *                     the second least to the second index dimension, and so
- *                     on.
+ *                     significand bit corresponds to the last index dimension,
+ *                     the second least to the second last index dimension, and
+ *                     so on.
+ *                     For example, the mask = 0b011101 means that for the 5d
+ *                     indices (x1, x2, x3, x4, x5), (x1, x2, x3, x5) are
+ *                     considered for the scalar, making the scalar itself 4d.
  *
  * @note  This class only manages the accesses and not the memory itself.
  */
 template <int Dimensionality, typename ArithmeticType, typename StorageType,
-          int32 ScalarMask>
+          size_type ScalarMask>
 class scaled_reduced_row_major
     : public detail::enable_write_scalar<
           Dimensionality,
@@ -562,7 +547,7 @@ class scaled_reduced_row_major
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = StorageType;
     static constexpr size_type dimensionality{Dimensionality};
-    static constexpr int32 scalar_mask{ScalarMask};
+    static constexpr size_type scalar_mask{ScalarMask};
     static constexpr bool is_const{std::is_const<storage_type>::value};
     using scalar_type =
         std::conditional_t<is_const, const arithmetic_type, arithmetic_type>;
@@ -582,99 +567,62 @@ class scaled_reduced_row_major
     friend class range<scaled_reduced_row_major>;
 
 protected:
+    static constexpr size_type scalar_dim{
+        helper::count_mask_dimensionality<scalar_mask, dimensionality>()};
+    static constexpr size_type scalar_stride_dim{
+        scalar_dim == 0 ? 0 : (scalar_dim - 1)};
+
+    using storage_stride_type = std::array<size_type, dimensionality - 1>;
+    using scalar_stride_type = std::array<size_type, scalar_stride_dim>;
     using reference_type =
         reference_class::scaled_reduced_storage<arithmetic_type, StorageType>;
 
-public:
     /**
      * Creates the accessor for an already allocated storage space with a
      * stride. The first stride is used for computing the index for the first
      * index, the second stride for the second index, and so on.
      *
-     * @param storage  pointer to the block of memory containing the storage
-     * @param scalar  pointer to the block of memory containing the scalar
-     *                values.
      * @param size  multidimensional size of the memory
-     * @param stride  stride array used for memory accesses
-     */
-    GKO_ATTRIBUTES constexpr scaled_reduced_row_major(
-        storage_type *storage, scalar_type *scalar, dim<dimensionality> size,
-        const std::array<const size_type, dimensionality - 1> &stride)
-        : storage_{storage},
-          scalar_{scalar},
-          size_{size},
-          storage_stride_{stride}
-    {}
-
-    /**
-     * Creates the accessor for an already allocated storage space with a
-     * stride. The first stride is used for computing the index for the first
-     * index, the second stride for the second index, and so on.
-     *
      * @param storage  pointer to the block of memory containing the storage
+     * @param stride  stride array used for memory accesses to storage
      * @param scalar  pointer to the block of memory containing the scalar
      *                values.
-     * @param size  multidimensional size of the memory
-     * @param stride  stride array used for memory accesses
      */
     GKO_ATTRIBUTES constexpr scaled_reduced_row_major(
-        storage_type *storage, scalar_type *scalar, dim<dimensionality> size,
-        std::array<const size_type, dimensionality - 1> &&stride)
-        : storage_{storage},
+        dim<dimensionality> size, storage_type *storage,
+        storage_stride_type stride, scalar_type *scalar)
+        : size_{size},
+          storage_{storage},
+          storage_stride_{stride},
           scalar_{scalar},
-          size_{size},
-          storage_stride_{stride}
+          scalar_stride_{}  // TODO
     {}
 
     /**
      * Creates the accessor for an already allocated storage space.
      * It is assumed that all accesses are without padding.
      *
+     * @param size  multidimensional size of the memory
      * @param storage  pointer to the block of memory containing the storage
      * @param scalar  pointer to the block of memory containing the scalar
      *                values.
-     * @param size  multidimensional size of the memory
      */
-    GKO_ATTRIBUTES constexpr scaled_reduced_row_major(storage_type *storage,
-                                                      scalar_type *scalar,
-                                                      dim<dimensionality> size)
+    GKO_ATTRIBUTES constexpr scaled_reduced_row_major(dim<dimensionality> size,
+                                                      storage_type *storage,
+                                                      scalar_type *scalar)
         : scaled_reduced_row_major{
-              storage, scalar, size,
-              helper::compute_stride_array<const size_type>(size)}
+              size, storage, helper::compute_stride_array<size_type>(size),
+              scalar}
     {}
 
-    /**
-     * Creates the accessor for an already allocated storage space with a
-     * stride. The first stride is used for computing the index for the first
-     * index, the second stride for the second index, and so on.
-     *
-     * @param storage  pointer to the block of memory containing the storage
-     * @param scalar  pointer to the block of memory containing the scalar
-     *                values.
-     * @param size  multidimensional size of the memory
-     * @param strides  stride array used for memory accesses
-     */
-    template <typename... Strides>
-    GKO_ATTRIBUTES constexpr scaled_reduced_row_major(storage_type *storage,
-                                                      scalar_type *scalar,
-                                                      dim<dimensionality> size,
-                                                      Strides &&... strides)
-        : storage_{storage},
-          scalar_{scalar},
-          size_{size},
-          storage_stride_{std::forward<Strides>(strides)...}
-    {
-        static_assert(sizeof...(Strides) + 1 == dimensionality,
-                      "Number of provided Strides must be dimensionality - 1!");
-    }
-
     /**
      * Creates an empty accessor (pointing nowhere with an empty size)
      */
     GKO_ATTRIBUTES constexpr scaled_reduced_row_major()
-        : scaled_reduced_row_major{nullptr, nullptr, {0, 0, 0}}
+        : scaled_reduced_row_major{{0, 0, 0}, nullptr, nullptr}
     {}
 
+public:
     /**
      * Creates a scaled_reduced_row_major range which contains a read-only
      * version of the current accessor.
@@ -683,7 +631,7 @@ class scaled_reduced_row_major
      */
     GKO_ATTRIBUTES GKO_INLINE constexpr range<const_accessor> to_const() const
     {
-        return range<const_accessor>{storage_, scalar_, size_, storage_stride_};
+        return range<const_accessor>{size_, storage_, storage_stride_, scalar_};
     }
 
     /**
@@ -771,11 +719,11 @@ class scaled_reduced_row_major
     {
         return helper::validate_spans(size_, spans...),
                range<scaled_reduced_row_major>{
-                   storage_ + compute_index((span{spans}.begin)...),
-                   scalar_ + compute_scalar_index(span{spans}.begin...),
                    dim<dimensionality>{
                        (span{spans}.end - span{spans}.begin)...},
-                   storage_stride_};
+                   storage_ + compute_index((span{spans}.begin)...),
+                   storage_stride_,
+                   scalar_ + compute_scalar_index(span{spans}.begin...)};
     }
 
     /**
@@ -794,8 +742,7 @@ class scaled_reduced_row_major
      * @returns returns a pointer to a stride array of size dimensionality - 1
      */
     GKO_ATTRIBUTES
-    GKO_INLINE constexpr const std::array<const size_type, dimensionality - 1>
-        &get_stride() const
+    GKO_INLINE constexpr const storage_stride_type &get_stride() const
     {
         return storage_stride_;
     }
@@ -849,10 +796,8 @@ class scaled_reduced_row_major
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
-        return helper::row_major_index<
-            const size_type, dimensionality>::compute(size_, storage_stride_,
-                                                      std::forward<Indices>(
-                                                          indices)...);
+        return helper::row_major_index<size_type, dimensionality>::compute(
+            size_, storage_stride_, std::forward<Indices>(indices)...);
     }
 
     template <typename... Indices>
@@ -862,15 +807,16 @@ class scaled_reduced_row_major
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
         return helper::row_major_mask_index<
-            const size_type, dimensionality,
+            size_type, dimensionality,
             scalar_mask>::compute(size_, storage_stride_,
                                   std::forward_as_tuple(indices...));
     }
 
+    const dim<dimensionality> size_;
     storage_type *storage_;
+    const storage_stride_type storage_stride_;
     scalar_type *scalar_;
-    const dim<dimensionality> size_;
-    const std::array<const size_type, dimensionality - 1> storage_stride_;
+    const scalar_stride_type scalar_stride_;
 };
 
 

From 4795ecb838ea37d90dd6565fe89d020f38f1c234 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 16 Nov 2020 09:29:43 +0100
Subject: [PATCH 0220/1008] Add scalar_stride for scaled_reduced_row_major

Including default scalar_stride computation while adapting the index
computation.
Additionally, tests for strides were added.
---
 core/test/base/range_accessors.cpp            | 115 ++--
 .../core/base/range_accessor_helper.hpp       | 523 ++++++++++++------
 include/ginkgo/core/base/range_accessors.hpp  |  62 ++-
 3 files changed, 468 insertions(+), 232 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 095762ab188..e8c67ca0f92 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -304,6 +304,28 @@ TYPED_TEST(ReducedStorage3d, ToConstWorks)
 }
 
 
+TYPED_TEST(ReducedStorage3d, CanCreateWithStride)
+{
+    using reduced_storage = typename TestFixture::reduced_storage;
+    using ar_type = typename TestFixture::ar_type;
+    using st_type = typename TestFixture::st_type;
+    auto size = gko::dim<3>{2, 2, 2};
+    auto stride = std::array<gko::size_type, 2>{12, 2};
+
+    auto range = reduced_storage{size, this->data, stride};
+    range(1, 1, 0) = ar_type{2.};
+
+    EXPECT_EQ(range(0, 0, 0), st_type{1.0});
+    EXPECT_EQ(range(0, 0, 1), st_type{2.01});
+    EXPECT_EQ(range(0, 1, 0), st_type{-1.02});
+    EXPECT_EQ(range(0, 1, 1), st_type{3.03});
+    EXPECT_EQ(range(1, 0, 0), st_type{-2.12});
+    EXPECT_EQ(range(1, 0, 1), st_type{2.13});
+    EXPECT_EQ(range(1, 1, 0), st_type{2.});
+    EXPECT_EQ(range(1, 1, 1), st_type{15.15});
+}
+
+
 TYPED_TEST(ReducedStorage3d, CanWriteData)
 {
     using t = typename TestFixture::t;
@@ -611,10 +633,12 @@ class ScaledReducedStorage3d : public ::testing::Test {
         6, 77
     };
     ar_type scalar[scalar_elements]{
-        1., 2.,
+        1., 2., 3., 4., 5., 6., 7., 8.
     };
     // clang-format on
-    reduced_storage r{size, data, scalar};
+    const std::array<gko::size_type, 2> storage_stride{8, 2};
+    const std::array<gko::size_type, 1> scalar_stride{2};
+    reduced_storage r{size, data, storage_stride, scalar, scalar_stride};
     const_reduced_storage cr{size, data, scalar};
 
     template <typename Accessor>
@@ -657,10 +681,13 @@ TYPED_TEST(ScaledReducedStorage3d, CorrectLengths)
 
 TYPED_TEST(ScaledReducedStorage3d, CorrectStride)
 {
-    EXPECT_EQ(this->r->get_stride()[0], this->size[1] * this->size[2]);
-    EXPECT_EQ(this->r->get_stride().at(0), this->size[1] * this->size[2]);
-    EXPECT_EQ(this->r->get_stride()[1], this->size[2]);
-    EXPECT_EQ(this->r->get_stride().at(1), this->size[2]);
+    EXPECT_EQ(this->r->get_storage_stride()[0], this->size[1] * this->size[2]);
+    EXPECT_EQ(this->r->get_storage_stride().at(0),
+              this->size[1] * this->size[2]);
+    EXPECT_EQ(this->r->get_storage_stride()[1], this->size[2]);
+    EXPECT_EQ(this->r->get_storage_stride().at(1), this->size[2]);
+    EXPECT_EQ(this->r->get_scalar_stride(), this->scalar_stride);
+    EXPECT_EQ(this->r->get_scalar_stride(), this->scalar_stride);
 }
 
 
@@ -724,10 +751,22 @@ TYPED_TEST(ScaledReducedStorage3d, ToConstWorks)
 }
 
 
-TYPED_TEST(ScaledReducedStorage3d, CanRead)
+TYPED_TEST(ScaledReducedStorage3d, CanCreateWithStride)
 {
-    this->check_accessor_correctness(this->cr);
-    this->check_accessor_correctness(this->r);
+    using reduced_storage = typename TestFixture::reduced_storage;
+    using ar_type = typename TestFixture::ar_type;
+    gko::dim<3> size{2, 1, 2};
+    std::array<gko::size_type, 2> stride_storage{5, 2};
+    std::array<gko::size_type, 1> stride_scalar{4};
+
+    reduced_storage range{size, this->data, stride_storage, this->scalar,
+                          stride_scalar};
+    range(1, 0, 0) = ar_type{15};
+
+    EXPECT_EQ(range(0, 0, 0), ar_type{10});
+    EXPECT_EQ(range(0, 0, 1), ar_type{22});
+    EXPECT_EQ(range(1, 0, 0), ar_type{15});
+    EXPECT_EQ(range(1, 0, 1), ar_type{36});
 }
 
 
@@ -907,11 +946,11 @@ class ScaledReducedStorageXd : public ::testing::Test {
     using accessor1d =
         gko::accessor::scaled_reduced_row_major<1, ar_type, st_type, 1>;
     using accessor2d =
-        gko::accessor::scaled_reduced_row_major<2, ar_type, st_type, 2>;
+        gko::accessor::scaled_reduced_row_major<2, ar_type, st_type, 3>;
     using const_accessor1d =
         gko::accessor::scaled_reduced_row_major<1, ar_type, const st_type, 1>;
     using const_accessor2d =
-        gko::accessor::scaled_reduced_row_major<2, ar_type, const st_type, 2>;
+        gko::accessor::scaled_reduced_row_major<2, ar_type, const st_type, 3>;
     static_assert(std::is_same<const_accessor1d,
                                typename accessor1d::const_accessor>::value,
                   "Const accessors must be the same!");
@@ -926,46 +965,44 @@ class ScaledReducedStorageXd : public ::testing::Test {
 
     const std::array<size_type, 0> stride0{{}};
     const std::array<size_type, 1> stride1{{4}};
-    const size_type stride1nc[1] = {4};
+    const std::array<size_type, 1> stride_sc{{5}};
     const gko::dim<1> size_1d{8u};
-    const gko::dim<2> size_2d{2u, 4u};
+    const gko::dim<2> size_2d{2u, 2u};
 
     static constexpr gko::size_type data_elements{8};
-    st_type data[data_elements]{11, 22, 33, 44, 55, 66, 77, -88};
-    const double default_scalar{.1};
-    ar_type scalar[data_elements]{
-        default_scalar, default_scalar, default_scalar, default_scalar,
-        default_scalar, default_scalar, default_scalar, default_scalar};
+    st_type data[data_elements]{10, 22, 32, 44, 54, 66, 76, -88};
+    ar_type scalar[data_elements]{1e0,  5e-1, 1e-1, 5e-2,
+                                  1e-2, 5e-3, 1e-3, 5e-4};
 
     reduced_storage1d r1{size_1d, data, scalar};
-    reduced_storage2d r2{size_2d, data, stride1, scalar};
+    reduced_storage2d r2{size_2d, data, stride1, scalar, stride_sc};
     const_reduced_storage1d cr1{size_1d, data, stride0, scalar};
-    const_reduced_storage2d cr2{size_2d, data, stride1, scalar};
+    const_reduced_storage2d cr2{size_2d, data, stride1, scalar, stride_sc};
 
     void data_equal_except_for(int idx)
     {
         // clang-format off
-        if (idx != 0) { EXPECT_EQ(data[0], 11); }
+        if (idx != 0) { EXPECT_EQ(data[0], 10); }
         if (idx != 1) { EXPECT_EQ(data[1], 22); }
-        if (idx != 2) { EXPECT_EQ(data[2], 33); }
+        if (idx != 2) { EXPECT_EQ(data[2], 32); }
         if (idx != 3) { EXPECT_EQ(data[3], 44); }
-        if (idx != 4) { EXPECT_EQ(data[4], 55); }
+        if (idx != 4) { EXPECT_EQ(data[4], 54); }
         if (idx != 5) { EXPECT_EQ(data[5], 66); }
-        if (idx != 6) { EXPECT_EQ(data[6], 77); }
+        if (idx != 6) { EXPECT_EQ(data[6], 76); }
         if (idx != 7) { EXPECT_EQ(data[7], -88); }
         // clang-format on
     }
     void scalar_equal_except_for(int idx)
     {
         // clang-format off
-        if (idx != 0) { EXPECT_EQ(scalar[0], default_scalar); }
-        if (idx != 1) { EXPECT_EQ(scalar[1], default_scalar); }
-        if (idx != 2) { EXPECT_EQ(scalar[2], default_scalar); }
-        if (idx != 3) { EXPECT_EQ(scalar[3], default_scalar); }
-        if (idx != 4) { EXPECT_EQ(scalar[4], default_scalar); }
-        if (idx != 5) { EXPECT_EQ(scalar[5], default_scalar); }
-        if (idx != 6) { EXPECT_EQ(scalar[6], default_scalar); }
-        if (idx != 7) { EXPECT_EQ(scalar[7], default_scalar); }
+        if (idx != 0) { EXPECT_EQ(scalar[0], ar_type{1e0}); }
+        if (idx != 1) { EXPECT_EQ(scalar[1], ar_type{5e-1}); }
+        if (idx != 2) { EXPECT_EQ(scalar[2], ar_type{1e-1}); }
+        if (idx != 3) { EXPECT_EQ(scalar[3], ar_type{5e-2}); }
+        if (idx != 4) { EXPECT_EQ(scalar[4], ar_type{1e-2}); }
+        if (idx != 5) { EXPECT_EQ(scalar[5], ar_type{5e-3}); }
+        if (idx != 6) { EXPECT_EQ(scalar[6], ar_type{1e-3}); }
+        if (idx != 7) { EXPECT_EQ(scalar[7], ar_type{5e-4}); }
         // clang-format on
     }
 };
@@ -973,10 +1010,12 @@ class ScaledReducedStorageXd : public ::testing::Test {
 
 TEST_F(ScaledReducedStorageXd, CanRead)
 {
-    EXPECT_NEAR(cr1(1), 2.2, delta);
-    EXPECT_NEAR(cr2(0, 1), 2.2, delta);
-    EXPECT_NEAR(r1(1), 2.2, delta);
-    EXPECT_NEAR(r2(0, 1), 2.2, delta);
+    EXPECT_NEAR(cr1(1), 11., delta);
+    EXPECT_NEAR(cr2(0, 1), 11., delta);
+    EXPECT_NEAR(cr2(1, 1), 66e-3, delta);
+    EXPECT_NEAR(r1(1), 11., delta);
+    EXPECT_NEAR(r2(0, 1), 11., delta);
+    EXPECT_NEAR(r2(1, 1), 66e-3, delta);
 }
 
 
@@ -992,11 +1031,11 @@ TEST_F(ScaledReducedStorageXd, CanWrite1)
 
 TEST_F(ScaledReducedStorageXd, CanWrite2)
 {
-    r2(1, 1) = 0.7;
+    r2(1, 1) = 0.5;
 
     data_equal_except_for(5);
     scalar_equal_except_for(99);
-    EXPECT_NEAR(r2(1, 1), 0.7, delta);
+    EXPECT_NEAR(r2(1, 1), 0.5, delta);
 }
 
 
diff --git a/include/ginkgo/core/base/range_accessor_helper.hpp b/include/ginkgo/core/base/range_accessor_helper.hpp
index 84146ca83e6..e90f325d99f 100644
--- a/include/ginkgo/core/base/range_accessor_helper.hpp
+++ b/include/ginkgo/core/base/range_accessor_helper.hpp
@@ -55,43 +55,121 @@ namespace accessor {
 namespace helper {
 
 
+namespace detail {
+
+
 /**
  * This helper runs from first to last dimension in order to compute the index.
  * The index is computed like this:
  * indices: x1, x2, x3, ...
  * compute(stride, x1, x2, x3) -> x1 * stride[0] + x2 * stride[1] + x3
  */
-template <typename IndexType, int32 total_dim, int32 current_iter = total_dim>
-struct row_major_index {
-    static constexpr int32 current_dim{total_dim - current_iter};
+template <typename ValueType, size_type total_dim, size_type current_iter = 1>
+struct row_major_helper_s {
+    static_assert(total_dim >= 1, "Dimensionality must be >= 1");
+    static_assert(current_iter < total_dim, "Iteration must be < total_dim!");
+
+    static constexpr size_type dim_idx{current_iter - 1};
+
     template <typename FirstType, typename... Indices>
-    static constexpr GKO_ATTRIBUTES GKO_INLINE IndexType
+    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
     compute(const dim<total_dim> &size,
-            const std::array<IndexType, (total_dim > 1 ? total_dim - 1 : 0)>
+            const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
                 &stride,
             FirstType &&first, Indices &&... idxs)
     {
-        return GKO_ASSERT(first < size[current_dim]),
-               first * stride[current_dim] +
-                   row_major_index<IndexType, total_dim, current_iter - 1>::
+        // The ASSERT size check must NOT be indexed with `dim_idx` directy,
+        // otherwise, it leads to a linker error. The reason is likely that
+        // `dim<N>::operator[](const size_type &)` uses a reference. Since
+        // `dim_idx` is constexpr (and not defined in a namespace scope), it
+        // can't be odr-used.
+        return GKO_ASSERT(first < size[static_cast<size_type>(dim_idx)]),
+               first * stride[dim_idx] +
+                   row_major_helper_s<ValueType, total_dim, current_iter + 1>::
                        compute(size, stride, std::forward<Indices>(idxs)...);
     }
 };
 
-template <typename IndexType, int32 total_dim>
-struct row_major_index<IndexType, total_dim, 1> {
-    static constexpr int32 current_dim{total_dim - 1};
+template <typename ValueType, size_type total_dim>
+struct row_major_helper_s<ValueType, total_dim, total_dim> {
     template <typename FirstType>
-    static constexpr GKO_ATTRIBUTES GKO_INLINE IndexType
+    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
     compute(const dim<total_dim> &size,
-            const std::array<IndexType, (total_dim > 1 ? total_dim - 1 : 0)>,
+            const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>,
             FirstType &&first)
     {
-        return GKO_ASSERT(first < size[current_dim]), first;
+        return GKO_ASSERT(first < size[total_dim - 1]), first;
     }
 };
 
 
+template <typename ValueType, size_type iter, size_type N>
+constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<iter == N, ValueType>
+mult_array(const dim<N> &size)
+{
+    return 1;
+}
+
+template <typename ValueType, size_type iter, size_type N>
+constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<(iter < N), ValueType>
+mult_array(const dim<N> &size)
+{
+    return size[iter] * mult_array<ValueType, iter + 1>(size);
+}
+
+
+template <typename ValueType, size_type iter = 1, size_type N, typename... Args>
+constexpr GKO_ATTRIBUTES GKO_INLINE
+    std::enable_if_t<N == 0 || (iter == N && iter == sizeof...(Args) + 1),
+                     std::array<ValueType, N == 0 ? 0 : N - 1>>
+    extract_factorization(const dim<N> &size, Args... args)
+{
+    return {{args...}};
+}
+
+template <typename ValueType, size_type iter = 1, size_type N, typename... Args>
+constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<
+    (iter < N) && (iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
+extract_factorization(const dim<N> &size, Args... args)
+{
+    return extract_factorization<ValueType, iter + 1>(
+        size, args..., mult_array<ValueType, iter>(size));
+}
+
+
+}  // namespace detail
+
+
+/**
+ * Computes the storage index for the given indices with respect to the given
+ * stride array
+ */
+template <typename ValueType, size_type total_dim, typename... Indices>
+constexpr GKO_ATTRIBUTES GKO_INLINE ValueType compute_storage_index(
+    const dim<total_dim> &size,
+    const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)> &stride,
+    Indices &&... idxs)
+{
+    return detail::row_major_helper_s<ValueType, total_dim>::compute(
+        size, stride, std::forward<Indices>(idxs)...);
+}
+
+
+/**
+ * Computes the default stride array from a size, assuming there is no
+ * padding
+ */
+template <typename ValueType, size_type N>
+constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_stride_array(
+    const dim<N> &size)
+{
+    return detail::extract_factorization<ValueType>(size);
+}
+
+
+namespace detail {
+
+
 /**
  * This helper must be walked through in reverse (highest to lowest dimension)
  * in order to properly use the stride for the scalar indices. The mask
@@ -99,66 +177,245 @@ struct row_major_index<IndexType, total_dim, 1> {
  * means using the last index, second bit corresponds to the second last
  * dimension, and so on.
  *
- * Basically, this computes indices in a similar fashion as `row_major_index`
+ * Basically, this computes indices in a similar fashion as `row_major_helper_s`
  * while having a mask signaling which indices to skip.
  *
  * Example: Mask = 0b01101
  * compute(stride, tuple(x1, x2, x3, x4))
  * -> x4 + x2 * stride[2] + x1 * stride[1]
  */
-template <typename IndexType, int32 total_dim, size_type mask,
-          int32 current_dim = total_dim - 1,
-          int32 current_scalar = total_dim - 1>
-struct row_major_mask_index {
-    static constexpr size_type msb_mask{size_type{1} << current_dim};
-    static constexpr int32 stride_size = total_dim > 1 ? total_dim - 1 : 0;
-    static constexpr bool use_current_stride = mask & msb_mask;
-    static constexpr bool is_last_stride_dim{stride_size == 0 ||
-                                             current_scalar == total_dim - 1};
+template <
+    typename ValueType, size_type mask, size_type set_bits_processed,
+    size_type stride_size, size_type dim_idx, size_type total_dim,
+    // Determine if the bit in the mask is set for dim_idx
+    // If the end is reached (dim_idx == total_dim), set it to false in order to
+    // only need one specialization
+    bool mask_set = (dim_idx < total_dim)
+                        ? static_cast<bool>(
+                              mask &(size_type{1} << (total_dim - 1 - dim_idx)))
+                        : false>
+struct row_major_masked_helper_s {
+};
+
+
+// bit for current dimensionality is set
+template <typename ValueType, size_type mask, size_type set_bits_processed,
+          size_type stride_size, size_type dim_idx, size_type total_dim>
+struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
+                                 stride_size, dim_idx, total_dim, true> {
+    static_assert(mask &
+                      (size_type{1}
+                       << (dim_idx < total_dim ? total_dim - 1 - dim_idx : 0)),
+                  "Do not touch the `mask_set` template parameter!");
+    static_assert(dim_idx < total_dim,
+                  "The iteration count must be smaller than total_dim here!!!");
+    static_assert(set_bits_processed <= stride_size,
+                  "The processed bits must be < total number of set bits!");
 
     template <typename... Args>
-    static constexpr GKO_ATTRIBUTES GKO_INLINE IndexType
-    compute(const dim<total_dim> &size,
-            const std::array<IndexType, stride_size> &stride,
-            std::tuple<Args...> indices)
+    static constexpr GKO_ATTRIBUTES GKO_INLINE
+        std::array<ValueType, stride_size>
+        build_stride(const dim<total_dim> &size, Args... args)
+    {
+        return row_major_masked_helper_s<
+            ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
+            total_dim>::build_stride(size, std::forward<Args>(args)...,
+                                     mult_array(size));
+    }
+
+    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    mult_array(const dim<total_dim> &size)
     {
-        return GKO_ASSERT(std::get<current_dim>(indices) < size[current_dim]),
-               (use_current_stride
-                    ? std::get<current_dim>(indices) *
-                          (is_last_stride_dim ? 1 : stride[current_scalar])
-                    : 0) +
-                   row_major_mask_index<
-                       IndexType, total_dim, mask, current_dim - 1,
-                       current_scalar - use_current_stride>::compute(size,
-                                                                     stride,
-                                                                     indices);
+        return size[dim_idx] *
+               row_major_masked_helper_s<
+                   ValueType, mask, set_bits_processed + 1, stride_size,
+                   dim_idx + 1, total_dim>::mult_array(size);
+    }
+
+    template <typename First, typename... Indices>
+    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    compute_idx(const dim<total_dim> &size,
+                const std::array<ValueType, stride_size> &stride, First &&first,
+                Indices &&... idxs)
+    {
+        static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
+                      "Mismatching number of Idxs!");
+        // If it is the last set dimension, there is no need for a stride
+        return GKO_ASSERT(first < size[dim_idx]),
+               first * (set_bits_processed == stride_size
+                            ? 1
+                            : stride[set_bits_processed]) +
+                   row_major_masked_helper_s<
+                       ValueType, mask, set_bits_processed + 1, stride_size,
+                       dim_idx + 1,
+                       total_dim>::compute_idx(size, stride,
+                                               std::forward<Indices>(idxs)...);
     }
 };
 
-template <typename IndexType, int32 total_dim, size_type mask,
-          int32 current_scalar>
-struct row_major_mask_index<IndexType, total_dim, mask, 0, current_scalar> {
-    static constexpr int32 current_dim{0};
-    static constexpr size_type msb_mask{size_type{1}};
-    static constexpr int32 stride_size = total_dim > 1 ? total_dim - 1 : 0;
-    static constexpr bool use_current_stride{mask & msb_mask};
-    static constexpr bool is_last_stride_dim{stride_size == 0 ||
-                                             current_scalar == total_dim - 1};
+// first set bit (from the left) for current dimensionality encountered:
+//     Do not calculate stride value for it (since no lower dimension needs it)!
+template <typename ValueType, size_type mask, size_type stride_size,
+          size_type dim_idx, size_type total_dim>
+struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
+                                 total_dim, true> {
+    static constexpr size_type set_bits_processed{0};
+    static_assert(mask &
+                      (size_type{1}
+                       << (dim_idx < total_dim ? total_dim - 1 - dim_idx : 0)),
+                  "Do not touch the `mask_set` template parameter!");
+    static_assert(dim_idx < total_dim,
+                  "The iteration count must be smaller than total_dim here!!!");
 
     template <typename... Args>
-    static constexpr GKO_ATTRIBUTES GKO_INLINE IndexType
-    compute(const dim<total_dim> &size,
-            const std::array<IndexType, stride_size> &stride,
-            std::tuple<Args...> indices)
+    static constexpr GKO_ATTRIBUTES GKO_INLINE
+        std::array<ValueType, stride_size>
+        build_stride(const dim<total_dim> &size, Args... args)
     {
-        return GKO_ASSERT(std::get<current_dim>(indices) < size[current_dim]),
-               (use_current_stride
-                    ? std::get<current_dim>(indices) *
-                          (is_last_stride_dim ? 1 : stride[current_scalar])
-                    : 0);
+        return row_major_masked_helper_s<
+            ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
+            total_dim>::build_stride(size, std::forward<Args>(args)...);
+    }
+
+    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    mult_array(const dim<total_dim> &size)
+    {
+        return row_major_masked_helper_s<
+            ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
+            total_dim>::mult_array(size);
+    }
+
+    template <typename First, typename... Indices>
+    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    compute_idx(const dim<total_dim> &size,
+                const std::array<ValueType, stride_size> &stride, First &&first,
+                Indices &&... idxs)
+    {
+        static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
+                      "Mismatching number of Idxs!");
+        // If it is the last set dimension, there is no need for a stride
+        return GKO_ASSERT(first < size[dim_idx]),
+               first * (set_bits_processed == stride_size
+                            ? 1
+                            : stride[set_bits_processed]) +
+                   row_major_masked_helper_s<
+                       ValueType, mask, set_bits_processed + 1, stride_size,
+                       dim_idx + 1,
+                       total_dim>::compute_idx(size, stride,
+                                               std::forward<Indices>(idxs)...);
     }
 };
 
+// bit for current dimensionality is not set
+template <typename ValueType, size_type mask, size_type set_bits_processed,
+          size_type stride_size, size_type dim_idx, size_type total_dim>
+struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
+                                 stride_size, dim_idx, total_dim, false> {
+    static_assert((mask & (size_type{1}
+                           << (dim_idx < total_dim ? total_dim - 1 - dim_idx
+                                                   : 0))) == 0,
+                  "Do not touch the `mask_set` template parameter!");
+    static_assert(dim_idx < total_dim,
+                  "The iteration count must be smaller than total_dim here!!!");
+    static_assert(set_bits_processed <= stride_size + 1,
+                  "The processed bits must be < total number of set bits!");
+    template <typename... Args>
+    static constexpr GKO_ATTRIBUTES GKO_INLINE
+        std::array<ValueType, stride_size>
+        build_stride(const dim<total_dim> &size, Args... args)
+    {
+        return row_major_masked_helper_s<
+            ValueType, mask, set_bits_processed, stride_size, dim_idx + 1,
+            total_dim>::build_stride(size, std::forward<Args>(args)...);
+    }
+
+    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    mult_array(const dim<total_dim> &size)
+    {
+        return row_major_masked_helper_s<ValueType, mask, set_bits_processed,
+                                         stride_size, dim_idx + 1,
+                                         total_dim>::mult_array(size);
+    }
+
+    template <typename First, typename... Indices>
+    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    compute_idx(const dim<total_dim> &size,
+                const std::array<ValueType, stride_size> &stride, First &&,
+                Indices &&... idxs)
+    {
+        static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
+                      "Mismatching number of Idxs!");
+        return row_major_masked_helper_s<
+            ValueType, mask, set_bits_processed, stride_size, dim_idx + 1,
+            total_dim>::compute_idx(size, stride,
+                                    std::forward<Indices>(idxs)...);
+    }
+};
+
+// Specialization for the end of recursion: build_stride array from created
+// arguments
+template <typename ValueType, size_type mask, size_type set_bits_processed,
+          size_type stride_size, size_type total_dim>
+struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
+                                 stride_size, total_dim, total_dim, false> {
+    static_assert(set_bits_processed <= stride_size + 1,
+                  "The processed bits must be smaller than the total number of "
+                  "set bits!");
+    template <typename... Args>
+    static constexpr GKO_ATTRIBUTES GKO_INLINE
+        std::array<ValueType, stride_size>
+        build_stride(const dim<total_dim> &size, Args... args)
+    {
+        return {{std::forward<Args>(args)...}};
+    }
+    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    mult_array(const dim<total_dim> &size)
+    {
+        return 1;
+    }
+
+    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    compute_idx(const dim<total_dim> &size,
+                const std::array<ValueType, stride_size> &stride)
+    {
+        return 0;
+    }
+};
+
+
+}  // namespace detail
+
+
+/**
+ * Computes the memory index for the given indices considering the stride.
+ * Only indices are considered where the corresponding mask bit is set.
+ */
+template <typename ValueType, size_type mask, size_type stride_size,
+          size_type total_dim, typename... Indices>
+constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_masked_index(
+    const dim<total_dim> &size,
+    const std::array<ValueType, stride_size> &stride, Indices &&... idxs)
+{
+    return detail::row_major_masked_helper_s<
+        ValueType, mask, 0, stride_size, 0,
+        total_dim>::compute_idx(size, stride, std::forward<Indices>(idxs)...);
+}
+
+
+/**
+ * Computes the default stride array from a size and a given mask which
+ * indicates which array indices to consider. It is assumed that there is no
+ * padding
+ */
+template <typename ValueType, size_type mask, size_type stride_size,
+          size_type total_dim>
+constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_masked_stride_array(
+    const dim<total_dim> &size)
+{
+    return detail::row_major_masked_helper_s<ValueType, mask, 0, stride_size, 0,
+                                             total_dim>::build_stride(size);
+}
+
 
 /**
  * Evaluates if all Args fulfill std::is_integral
@@ -206,138 +463,40 @@ template <typename... Args>
 using are_span_compatible = detail::are_span_compatible_impl<false, Args...>;
 
 
-namespace detail {
-
-
-template <typename ValueType, size_type Iter, size_type N>
-constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<Iter == N, ValueType>
-mult_array(const dim<N> &size)
-{
-    return 1;
-}
-
-template <typename ValueType, size_type Iter, size_type N>
-constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<Iter + 1 <= N, ValueType>
-mult_array(const dim<N> &size)
-{
-    return size[Iter] * mult_array<ValueType, Iter + 1>(size);
-}
-
-
-template <typename ValueType, size_type Iter, size_type mask, size_type N>
-constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<Iter == N, ValueType>
-mult_masked_array(const dim<N> &size)
-{
-    return 1;
-}
-
-// in case corresponding bit is set
-template <typename ValueType, size_type Iter, size_type mask, size_type N>
-constexpr GKO_ATTRIBUTES GKO_INLINE
-    std::enable_if_t<Iter + 1 <= N && (mask & size_type{1}), ValueType>
-    mult_masked_array(const dim<N> &size)
-{
-    return size[Iter] * mult_array<ValueType, Iter + 1>(size);
-}
-
-// in case corresponding bit is NOT set
-template <typename ValueType, size_type Iter, size_type mask, size_type N>
-constexpr GKO_ATTRIBUTES GKO_INLINE
-    std::enable_if_t<Iter + 1 <= N && ((mask & size_type{1}) == 0), ValueType>
-    mult_masked_array(const dim<N> &size)
-{
-    return mult_array<ValueType, Iter + 1, (mask >> 1)>(size);
-}
-
-
-template <typename ValueType, size_type Iter = 1, size_type N, typename... Args>
-constexpr GKO_ATTRIBUTES GKO_INLINE
-    std::enable_if_t<N == 0 || (Iter == N && Iter == sizeof...(Args) + 1),
-                     std::array<ValueType, N == 0 ? 0 : N - 1>>
-    extract_factorization(const dim<N> &size, Args... args)
-{
-    return {{args...}};
-}
-
-template <typename ValueType, size_type Iter = 1, size_type N, typename... Args>
-constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<
-    (Iter < N) && (Iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
-extract_factorization(const dim<N> &size, Args... args)
-{
-    return extract_factorization<ValueType, Iter + 1>(
-        size, args..., mult_array<ValueType, Iter>(size));
-}
-
-
-template <typename ValueType, size_type Iter = 1, size_type mask,
-          size_type res_size, size_type N, typename... Args>
-constexpr GKO_ATTRIBUTES GKO_INLINE
-    std::enable_if_t<N == 0 || (Iter == N && res_size == sizeof...(Args) + 1),
-                     std::array<ValueType, res_size>>
-    extract_masked_factorization(const dim<N> &size, Args... args)
-{
-    return {{args...}};
-}
-
-// In case the corresponding bit is set
-template <typename ValueType, size_type Iter = 1, size_type mask,
-          size_type res_size, size_type N, typename... Args>
-constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<
-    (Iter < N) && (Iter == sizeof...(Args) + 1) && (mask & size_type{1}),
-    std::array<ValueType, res_size>>
-extract_factorization(const dim<N> &size, Args... args)
-{
-    return extract_masked_factorization<ValueType, Iter + 1>(
-        size, args..., mult_array<ValueType, Iter>(size));
-}
-
-// In case the corresponding bit is NOT set
-template <typename ValueType, size_type Iter = 1, size_type mask,
-          size_type res_size, size_type N, typename... Args>
-constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<
-    (Iter < N) && (Iter == sizeof...(Args) + 1) && ((mask & size_type{1}) == 0),
-    std::array<ValueType, res_size>>
-extract_factorization(const dim<N> &size, Args... args)
-{
-    return extract_masked_factorization<ValueType, Iter + 1>(
-        size, args..., mult_array<ValueType, Iter>(size));
-}
-
-
-}  // namespace detail
-
-
 /**
- * Computes the default stride array from a size, assuming there is no padding
+ * Computes the default stride array from a size and a given mask which
+ * indicates which array indices to consider. It is assumed that there is no
+ * padding
  */
-template <typename ValueType, size_type N>
+template <typename ValueType, size_type mask, size_type res_size, size_type N>
 constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_stride_array(
     const dim<N> &size)
 {
-    return detail::extract_factorization<ValueType>(size);
+    // TODO fill in!!!
+    return std::array<ValueType, res_size>{{}};
 }
 
 
 namespace detail {
 
 
-template <size_type Iter, size_type N, typename Callable, typename... Indices>
-GKO_ATTRIBUTES std::enable_if_t<Iter == N> multidim_for_each_impl(
+template <size_type iter, size_type N, typename Callable, typename... Indices>
+GKO_ATTRIBUTES std::enable_if_t<iter == N> multidim_for_each_impl(
     const dim<N> &size, Callable &&callable, Indices... indices)
 {
-    static_assert(Iter == sizeof...(Indices),
+    static_assert(iter == sizeof...(Indices),
                   "Number arguments must match current iteration!");
     callable(indices...);
 }
 
-template <size_type Iter, size_type N, typename Callable, typename... Indices>
-GKO_ATTRIBUTES std::enable_if_t<(Iter < N)> multidim_for_each_impl(
+template <size_type iter, size_type N, typename Callable, typename... Indices>
+GKO_ATTRIBUTES std::enable_if_t<(iter < N)> multidim_for_each_impl(
     const dim<N> &size, Callable &&callable, Indices... indices)
 {
-    static_assert(Iter == sizeof...(Indices),
+    static_assert(iter == sizeof...(Indices),
                   "Number arguments must match current iteration!");
-    for (size_type i = 0; i < size[Iter]; ++i) {
-        multidim_for_each_impl<Iter + 1>(size, std::forward<Callable>(callable),
+    for (size_type i = 0; i < size[iter]; ++i) {
+        multidim_for_each_impl<iter + 1>(size, std::forward<Callable>(callable),
                                          indices..., i);
     }
 }
@@ -360,22 +519,22 @@ GKO_ATTRIBUTES void multidim_for_each(const dim<N> &size, Callable &&callable)
 namespace detail {
 
 
-template <size_type Iter, size_type N>
-GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<Iter == N, int>
+template <size_type iter, size_type N>
+GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<iter == N, int>
 spans_in_size(const dim<N> &size)
 {
     return 0;
 }
 
-template <size_type Iter, size_type N, typename First, typename... Remaining>
-GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<(Iter < N), int>
+template <size_type iter, size_type N, typename First, typename... Remaining>
+GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<(iter < N), int>
 spans_in_size(const dim<N> &size, First &&first, Remaining &&... remaining)
 {
-    static_assert(sizeof...(Remaining) + 1 == N - Iter,
-                  "Number of remaining spans must be equal to N - Iter");
+    static_assert(sizeof...(Remaining) + 1 == N - iter,
+                  "Number of remaining spans must be equal to N - iter");
     return GKO_ASSERT(span{first}.is_valid()),
-           GKO_ASSERT(span{first} <= span{size[Iter]}),
-           spans_in_size<Iter + 1>(size, std::forward<Remaining>(remaining)...);
+           GKO_ASSERT(span{first} <= span{size[iter]}),
+           spans_in_size<iter + 1>(size, std::forward<Remaining>(remaining)...);
 }
 
 
@@ -393,19 +552,19 @@ GKO_ATTRIBUTES GKO_INLINE constexpr int validate_spans(const dim<N> &size,
 namespace detail {
 
 
-template <int32 mask, size_type N, size_type Iter = 0>
-constexpr std::enable_if_t<Iter == N, size_type>
+template <int32 mask, size_type N, size_type iter = 0>
+constexpr std::enable_if_t<iter == N, size_type>
 count_mask_dimensionality_impl()
 {
     return 0;
 }
 
-template <int32 mask, size_type N, size_type Iter = 0>
-constexpr std::enable_if_t<(Iter < N), size_type>
+template <int32 mask, size_type N, size_type iter = 0>
+constexpr std::enable_if_t<(iter < N), size_type>
 count_mask_dimensionality_impl()
 {
-    return (mask & int32{1}) +
-           count_mask_dimensionality_impl<(mask >> 1), N, Iter + 1>();
+    return (mask & size_type{1}) +
+           count_mask_dimensionality_impl<(mask >> 1), N, iter + 1>();
 }
 
 
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index fa3719d1a4e..82f5fac74a3 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -443,7 +443,7 @@ class reduced_row_major {
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
-        return helper::row_major_index<size_type, dimensionality>::compute(
+        return helper::compute_storage_index<size_type, dimensionality>(
             size_, stride_, std::forward<Indices>(indices)...);
     }
 
@@ -577,6 +577,29 @@ class scaled_reduced_row_major
     using reference_type =
         reference_class::scaled_reduced_storage<arithmetic_type, StorageType>;
 
+    /**
+     * Creates the accessor for an already allocated storage space with a
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
+     *
+     * @param size  multidimensional size of the memory
+     * @param storage  pointer to the block of memory containing the storage
+     * @param storage_stride  stride array used for memory accesses to storage
+     * @param scalar  pointer to the block of memory containing the scalar
+     *                values.
+     * @param scalar_stride  stride array used for memory accesses to scalar
+     */
+    GKO_ATTRIBUTES constexpr scaled_reduced_row_major(
+        dim<dimensionality> size, storage_type *storage,
+        storage_stride_type storage_stride, scalar_type *scalar,
+        scalar_stride_type scalar_stride)
+        : size_{size},
+          storage_{storage},
+          storage_stride_{storage_stride},
+          scalar_{scalar},
+          scalar_stride_{scalar_stride}
+    {}
+
     /**
      * Creates the accessor for an already allocated storage space with a
      * stride. The first stride is used for computing the index for the first
@@ -595,7 +618,8 @@ class scaled_reduced_row_major
           storage_{storage},
           storage_stride_{stride},
           scalar_{scalar},
-          scalar_stride_{}  // TODO
+          scalar_stride_{helper::compute_masked_stride_array<
+              size_type, scalar_mask, scalar_stride_dim, dimensionality>(size)}
     {}
 
     /**
@@ -631,7 +655,8 @@ class scaled_reduced_row_major
      */
     GKO_ATTRIBUTES GKO_INLINE constexpr range<const_accessor> to_const() const
     {
-        return range<const_accessor>{size_, storage_, storage_stride_, scalar_};
+        return range<const_accessor>{size_, storage_, storage_stride_, scalar_,
+                                     scalar_stride_};
     }
 
     /**
@@ -723,7 +748,8 @@ class scaled_reduced_row_major
                        (span{spans}.end - span{spans}.begin)...},
                    storage_ + compute_index((span{spans}.begin)...),
                    storage_stride_,
-                   scalar_ + compute_scalar_index(span{spans}.begin...)};
+                   scalar_ + compute_scalar_index(span{spans}.begin...),
+                   scalar_stride_};
     }
 
     /**
@@ -737,16 +763,29 @@ class scaled_reduced_row_major
     }
 
     /**
-     * Returns a pointer to a stride array of size dimensionality - 1
+     * Returns a const reference to the storage stride array of size
+     * dimensionality - 1
      *
-     * @returns returns a pointer to a stride array of size dimensionality - 1
+     * @returns a const reference to the storage stride array of size
+     * dimensionality - 1
      */
     GKO_ATTRIBUTES
-    GKO_INLINE constexpr const storage_stride_type &get_stride() const
+    GKO_INLINE constexpr const storage_stride_type &get_storage_stride() const
     {
         return storage_stride_;
     }
 
+    /**
+     * Returns a const reference to the scalar stride array
+     *
+     * @returns a const reference to the scalar stride array
+     */
+    GKO_ATTRIBUTES
+    GKO_INLINE constexpr const scalar_stride_type &get_scalar_stride() const
+    {
+        return scalar_stride_;
+    }
+
     /**
      * Returns the pointer to the storage data
      *
@@ -796,7 +835,7 @@ class scaled_reduced_row_major
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
-        return helper::row_major_index<size_type, dimensionality>::compute(
+        return helper::compute_storage_index<size_type, dimensionality>(
             size_, storage_stride_, std::forward<Indices>(indices)...);
     }
 
@@ -806,10 +845,9 @@ class scaled_reduced_row_major
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
-        return helper::row_major_mask_index<
-            size_type, dimensionality,
-            scalar_mask>::compute(size_, storage_stride_,
-                                  std::forward_as_tuple(indices...));
+        return helper::compute_masked_index<size_type, scalar_mask,
+                                            scalar_stride_dim>(
+            size_, scalar_stride_, std::forward<Indices>(indices)...);
     }
 
     const dim<dimensionality> size_;

From c2c4a2df4b13970f536268515658efe5a0f67760 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 16 Nov 2020 12:01:18 +0100
Subject: [PATCH 0221/1008] Add additional braces around array initialization

---
 core/test/base/range_accessors.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index e8c67ca0f92..03e078d3ab6 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -310,7 +310,7 @@ TYPED_TEST(ReducedStorage3d, CanCreateWithStride)
     using ar_type = typename TestFixture::ar_type;
     using st_type = typename TestFixture::st_type;
     auto size = gko::dim<3>{2, 2, 2};
-    auto stride = std::array<gko::size_type, 2>{12, 2};
+    auto stride = std::array<gko::size_type, 2>{{12, 2}};
 
     auto range = reduced_storage{size, this->data, stride};
     range(1, 1, 0) = ar_type{2.};
@@ -636,8 +636,8 @@ class ScaledReducedStorage3d : public ::testing::Test {
         1., 2., 3., 4., 5., 6., 7., 8.
     };
     // clang-format on
-    const std::array<gko::size_type, 2> storage_stride{8, 2};
-    const std::array<gko::size_type, 1> scalar_stride{2};
+    const std::array<gko::size_type, 2> storage_stride{{8, 2}};
+    const std::array<gko::size_type, 1> scalar_stride{{2}};
     reduced_storage r{size, data, storage_stride, scalar, scalar_stride};
     const_reduced_storage cr{size, data, scalar};
 
@@ -756,8 +756,8 @@ TYPED_TEST(ScaledReducedStorage3d, CanCreateWithStride)
     using reduced_storage = typename TestFixture::reduced_storage;
     using ar_type = typename TestFixture::ar_type;
     gko::dim<3> size{2, 1, 2};
-    std::array<gko::size_type, 2> stride_storage{5, 2};
-    std::array<gko::size_type, 1> stride_scalar{4};
+    std::array<gko::size_type, 2> stride_storage{{5, 2}};
+    std::array<gko::size_type, 1> stride_scalar{{4}};
 
     reduced_storage range{size, this->data, stride_storage, this->scalar,
                           stride_scalar};

From edc1383c497c5fa14025be039c8ea9d7a4d1551f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 16 Nov 2020 20:56:01 +0100
Subject: [PATCH 0222/1008] Fix intel compiler error

By using the constructor of std::array to initialize it inside the
accessors instead of list initialization (which apparently tried to
convert `std::array<size_type, N>` to `size_type` unsuccessfully).
---
 include/ginkgo/core/base/range_accessors.hpp | 49 ++++++++++----------
 1 file changed, 24 insertions(+), 25 deletions(-)

diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 82f5fac74a3..1bd3c5be21d 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -266,20 +266,7 @@ class reduced_row_major {
     GKO_ATTRIBUTES constexpr reduced_row_major(gko::dim<dimensionality> size,
                                                storage_type *storage,
                                                storage_stride_type stride)
-        : size_{size}, storage_{storage}, stride_{stride}
-    {}
-
-    /**
-     * Creates the accessor for an already allocated storage space.
-     * It is assumed that all accesses are without padding.
-     *
-     * @param storage  pointer to the block of memory containing the storage
-     * @param size  multidimensional size of the memory
-     */
-    GKO_ATTRIBUTES constexpr reduced_row_major(dim<dimensionality> size,
-                                               storage_type *storage)
-        : reduced_row_major{size, storage,
-                            helper::compute_stride_array<size_type>(size)}
+        : size_{size}, storage_{storage}, stride_(stride)
     {}
 
     /**
@@ -295,14 +282,27 @@ class reduced_row_major {
     GKO_ATTRIBUTES constexpr reduced_row_major(gko::dim<dimensionality> size,
                                                storage_type *storage,
                                                Strides &&... strides)
-        : size_{size},
-          storage_{storage},
-          stride_{std::forward<Strides>(strides)...}
+        : reduced_row_major{
+              size, storage,
+              storage_stride_type{{std::forward<Strides>(strides)...}}}
     {
         static_assert(sizeof...(Strides) + 1 == dimensionality,
                       "Number of provided Strides must be dimensionality - 1!");
     }
 
+    /**
+     * Creates the accessor for an already allocated storage space.
+     * It is assumed that all accesses are without padding.
+     *
+     * @param storage  pointer to the block of memory containing the storage
+     * @param size  multidimensional size of the memory
+     */
+    GKO_ATTRIBUTES constexpr reduced_row_major(dim<dimensionality> size,
+                                               storage_type *storage)
+        : reduced_row_major{size, storage,
+                            helper::compute_stride_array<size_type>(size)}
+    {}
+
     /**
      * Creates an empty accessor (pointing nowhere with an empty size)
      */
@@ -595,9 +595,9 @@ class scaled_reduced_row_major
         scalar_stride_type scalar_stride)
         : size_{size},
           storage_{storage},
-          storage_stride_{storage_stride},
+          storage_stride_(storage_stride),
           scalar_{scalar},
-          scalar_stride_{scalar_stride}
+          scalar_stride_(scalar_stride)
     {}
 
     /**
@@ -614,12 +614,11 @@ class scaled_reduced_row_major
     GKO_ATTRIBUTES constexpr scaled_reduced_row_major(
         dim<dimensionality> size, storage_type *storage,
         storage_stride_type stride, scalar_type *scalar)
-        : size_{size},
-          storage_{storage},
-          storage_stride_{stride},
-          scalar_{scalar},
-          scalar_stride_{helper::compute_masked_stride_array<
-              size_type, scalar_mask, scalar_stride_dim, dimensionality>(size)}
+        : scaled_reduced_row_major{
+              size, storage, stride, scalar,
+              helper::compute_masked_stride_array<
+                  size_type, scalar_mask, scalar_stride_dim, dimensionality>(
+                  size)}
     {}
 
     /**

From 52ee142b18663c3c3b994aefcf1f3abc49411ffb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 17 Nov 2020 04:19:33 +0100
Subject: [PATCH 0223/1008] Add missing include in
 range_accessor_references.hpp

---
 include/ginkgo/core/base/range_accessor_references.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/include/ginkgo/core/base/range_accessor_references.hpp b/include/ginkgo/core/base/range_accessor_references.hpp
index 680730d55f2..063598d7b7b 100644
--- a/include/ginkgo/core/base/range_accessor_references.hpp
+++ b/include/ginkgo/core/base/range_accessor_references.hpp
@@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/std_extensions.hpp>
+#include <ginkgo/core/base/types.hpp>
 
 
 // CUDA TOOLKIT < 11 does not support constexpr in combination with

From 2f4051102612477efa3f9df7745b82e1a7d2a070 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 17 Nov 2020 11:55:00 +0100
Subject: [PATCH 0224/1008] Code quality improvement

- Use perfect forwarding properly
- Remove the name of unused parameters
---
 .../core/base/range_accessor_helper.hpp       | 103 +++++++-----------
 include/ginkgo/core/base/range_accessors.hpp  |  29 +++--
 2 files changed, 55 insertions(+), 77 deletions(-)

diff --git a/include/ginkgo/core/base/range_accessor_helper.hpp b/include/ginkgo/core/base/range_accessor_helper.hpp
index e90f325d99f..4a1d50ceb8f 100644
--- a/include/ginkgo/core/base/range_accessor_helper.hpp
+++ b/include/ginkgo/core/base/range_accessor_helper.hpp
@@ -76,7 +76,7 @@ struct row_major_helper_s {
     compute(const dim<total_dim> &size,
             const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
                 &stride,
-            FirstType &&first, Indices &&... idxs)
+            FirstType &&first, Indices &&...idxs)
     {
         // The ASSERT size check must NOT be indexed with `dim_idx` directy,
         // otherwise, it leads to a linker error. The reason is likely that
@@ -105,7 +105,7 @@ struct row_major_helper_s<ValueType, total_dim, total_dim> {
 
 template <typename ValueType, size_type iter, size_type N>
 constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<iter == N, ValueType>
-mult_array(const dim<N> &size)
+mult_array(const dim<N> &)
 {
     return 1;
 }
@@ -122,18 +122,18 @@ template <typename ValueType, size_type iter = 1, size_type N, typename... Args>
 constexpr GKO_ATTRIBUTES GKO_INLINE
     std::enable_if_t<N == 0 || (iter == N && iter == sizeof...(Args) + 1),
                      std::array<ValueType, N == 0 ? 0 : N - 1>>
-    extract_factorization(const dim<N> &size, Args... args)
+    extract_factorization(const dim<N> &, Args &&...args)
 {
-    return {{args...}};
+    return {{std::forward<Args>(args)...}};
 }
 
 template <typename ValueType, size_type iter = 1, size_type N, typename... Args>
 constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<
     (iter < N) && (iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
-extract_factorization(const dim<N> &size, Args... args)
+extract_factorization(const dim<N> &size, Args &&...args)
 {
     return extract_factorization<ValueType, iter + 1>(
-        size, args..., mult_array<ValueType, iter>(size));
+        size, std::forward<Args>(args)..., mult_array<ValueType, iter>(size));
 }
 
 
@@ -148,7 +148,7 @@ template <typename ValueType, size_type total_dim, typename... Indices>
 constexpr GKO_ATTRIBUTES GKO_INLINE ValueType compute_storage_index(
     const dim<total_dim> &size,
     const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)> &stride,
-    Indices &&... idxs)
+    Indices &&...idxs)
 {
     return detail::row_major_helper_s<ValueType, total_dim>::compute(
         size, stride, std::forward<Indices>(idxs)...);
@@ -184,18 +184,16 @@ namespace detail {
  * compute(stride, tuple(x1, x2, x3, x4))
  * -> x4 + x2 * stride[2] + x1 * stride[1]
  */
-template <
-    typename ValueType, size_type mask, size_type set_bits_processed,
-    size_type stride_size, size_type dim_idx, size_type total_dim,
-    // Determine if the bit in the mask is set for dim_idx
-    // If the end is reached (dim_idx == total_dim), set it to false in order to
-    // only need one specialization
-    bool mask_set = (dim_idx < total_dim)
-                        ? static_cast<bool>(
-                              mask &(size_type{1} << (total_dim - 1 - dim_idx)))
-                        : false>
-struct row_major_masked_helper_s {
-};
+template <typename ValueType, size_type mask, size_type set_bits_processed,
+          size_type stride_size, size_type dim_idx, size_type total_dim,
+          // Determine if the bit in the mask is set for dim_idx
+          // If the end is reached (dim_idx == total_dim), set it to false in
+          // order to only need one specialization
+          bool mask_set = (dim_idx < total_dim)
+                              ? static_cast<bool>(mask &(
+                                    size_type{1} << (total_dim - 1 - dim_idx)))
+                              : false>
+struct row_major_masked_helper_s {};
 
 
 // bit for current dimensionality is set
@@ -215,7 +213,7 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     template <typename... Args>
     static constexpr GKO_ATTRIBUTES GKO_INLINE
         std::array<ValueType, stride_size>
-        build_stride(const dim<total_dim> &size, Args... args)
+        build_stride(const dim<total_dim> &size, Args &&...args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
@@ -236,7 +234,7 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
     compute_idx(const dim<total_dim> &size,
                 const std::array<ValueType, stride_size> &stride, First &&first,
-                Indices &&... idxs)
+                Indices &&...idxs)
     {
         static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
                       "Mismatching number of Idxs!");
@@ -270,7 +268,7 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
     template <typename... Args>
     static constexpr GKO_ATTRIBUTES GKO_INLINE
         std::array<ValueType, stride_size>
-        build_stride(const dim<total_dim> &size, Args... args)
+        build_stride(const dim<total_dim> &size, Args &&...args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
@@ -289,7 +287,7 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
     static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
     compute_idx(const dim<total_dim> &size,
                 const std::array<ValueType, stride_size> &stride, First &&first,
-                Indices &&... idxs)
+                Indices &&...idxs)
     {
         static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
                       "Mismatching number of Idxs!");
@@ -322,7 +320,7 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     template <typename... Args>
     static constexpr GKO_ATTRIBUTES GKO_INLINE
         std::array<ValueType, stride_size>
-        build_stride(const dim<total_dim> &size, Args... args)
+        build_stride(const dim<total_dim> &size, Args &&...args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed, stride_size, dim_idx + 1,
@@ -341,7 +339,7 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
     compute_idx(const dim<total_dim> &size,
                 const std::array<ValueType, stride_size> &stride, First &&,
-                Indices &&... idxs)
+                Indices &&...idxs)
     {
         static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
                       "Mismatching number of Idxs!");
@@ -364,19 +362,18 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     template <typename... Args>
     static constexpr GKO_ATTRIBUTES GKO_INLINE
         std::array<ValueType, stride_size>
-        build_stride(const dim<total_dim> &size, Args... args)
+        build_stride(const dim<total_dim> &, Args &&...args)
     {
         return {{std::forward<Args>(args)...}};
     }
     static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
-    mult_array(const dim<total_dim> &size)
+    mult_array(const dim<total_dim> &)
     {
         return 1;
     }
 
-    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
-    compute_idx(const dim<total_dim> &size,
-                const std::array<ValueType, stride_size> &stride)
+    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType compute_idx(
+        const dim<total_dim> &, const std::array<ValueType, stride_size> &)
     {
         return 0;
     }
@@ -394,7 +391,7 @@ template <typename ValueType, size_type mask, size_type stride_size,
           size_type total_dim, typename... Indices>
 constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_masked_index(
     const dim<total_dim> &size,
-    const std::array<ValueType, stride_size> &stride, Indices &&... idxs)
+    const std::array<ValueType, stride_size> &stride, Indices &&...idxs)
 {
     return detail::row_major_masked_helper_s<
         ValueType, mask, 0, stride_size, 0,
@@ -421,15 +418,13 @@ constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_masked_stride_array(
  * Evaluates if all Args fulfill std::is_integral
  */
 template <typename... Args>
-struct are_all_integral : public std::true_type {
-};
+struct are_all_integral : public std::true_type {};
 
 template <typename First, typename... Args>
 struct are_all_integral<First, Args...>
     : public std::conditional<std::is_integral<std::decay_t<First>>::value,
                               are_all_integral<Args...>,
-                              std::false_type>::type {
-};
+                              std::false_type>::type {};
 
 
 namespace detail {
@@ -437,8 +432,7 @@ namespace detail {
 
 template <bool has_span, typename... Args>
 struct are_span_compatible_impl
-    : public std::integral_constant<bool, has_span> {
-};
+    : public std::integral_constant<bool, has_span> {};
 
 template <bool has_span, typename First, typename... Args>
 struct are_span_compatible_impl<has_span, First, Args...>
@@ -448,8 +442,7 @@ struct are_span_compatible_impl<has_span, First, Args...>
           are_span_compatible_impl<
               has_span || std::is_same<std::decay_t<First>, span>::value,
               Args...>,
-          std::false_type>::type {
-};
+          std::false_type>::type {};
 
 
 }  // namespace detail
@@ -463,30 +456,16 @@ template <typename... Args>
 using are_span_compatible = detail::are_span_compatible_impl<false, Args...>;
 
 
-/**
- * Computes the default stride array from a size and a given mask which
- * indicates which array indices to consider. It is assumed that there is no
- * padding
- */
-template <typename ValueType, size_type mask, size_type res_size, size_type N>
-constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_stride_array(
-    const dim<N> &size)
-{
-    // TODO fill in!!!
-    return std::array<ValueType, res_size>{{}};
-}
-
-
 namespace detail {
 
 
 template <size_type iter, size_type N, typename Callable, typename... Indices>
 GKO_ATTRIBUTES std::enable_if_t<iter == N> multidim_for_each_impl(
-    const dim<N> &size, Callable &&callable, Indices... indices)
+    const dim<N> &, Callable &&callable, Indices &&...indices)
 {
     static_assert(iter == sizeof...(Indices),
                   "Number arguments must match current iteration!");
-    callable(indices...);
+    callable(std::forward<Indices>(indices)...);
 }
 
 template <size_type iter, size_type N, typename Callable, typename... Indices>
@@ -497,7 +476,7 @@ GKO_ATTRIBUTES std::enable_if_t<(iter < N)> multidim_for_each_impl(
                   "Number arguments must match current iteration!");
     for (size_type i = 0; i < size[iter]; ++i) {
         multidim_for_each_impl<iter + 1>(size, std::forward<Callable>(callable),
-                                         indices..., i);
+                                         std::forward<Indices>(indices)..., i);
     }
 }
 
@@ -521,14 +500,14 @@ namespace detail {
 
 template <size_type iter, size_type N>
 GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<iter == N, int>
-spans_in_size(const dim<N> &size)
+spans_in_size(const dim<N> &)
 {
     return 0;
 }
 
 template <size_type iter, size_type N, typename First, typename... Remaining>
 GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<(iter < N), int>
-spans_in_size(const dim<N> &size, First &&first, Remaining &&... remaining)
+spans_in_size(const dim<N> &size, First &&first, Remaining &&...remaining)
 {
     static_assert(sizeof...(Remaining) + 1 == N - iter,
                   "Number of remaining spans must be equal to N - iter");
@@ -543,7 +522,7 @@ spans_in_size(const dim<N> &size, First &&first, Remaining &&... remaining)
 
 template <size_type N, typename... Spans>
 GKO_ATTRIBUTES GKO_INLINE constexpr int validate_spans(const dim<N> &size,
-                                                       Spans... spans)
+                                                       Spans &&...spans)
 {
     return detail::spans_in_size<0>(size, std::forward<Spans>(spans)...);
 }
@@ -552,14 +531,14 @@ GKO_ATTRIBUTES GKO_INLINE constexpr int validate_spans(const dim<N> &size,
 namespace detail {
 
 
-template <int32 mask, size_type N, size_type iter = 0>
+template <size_type, size_type N, size_type iter = 0>
 constexpr std::enable_if_t<iter == N, size_type>
 count_mask_dimensionality_impl()
 {
     return 0;
 }
 
-template <int32 mask, size_type N, size_type iter = 0>
+template <size_type mask, size_type N, size_type iter = 0>
 constexpr std::enable_if_t<(iter < N), size_type>
 count_mask_dimensionality_impl()
 {
@@ -571,7 +550,7 @@ count_mask_dimensionality_impl()
 }  // namespace detail
 
 
-template <int32 mask, size_type N>
+template <size_type mask, size_type N>
 constexpr size_type count_mask_dimensionality()
 {
     return detail::count_mask_dimensionality_impl<mask, N>();
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 1bd3c5be21d..c4b8ff5416e 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -281,7 +281,7 @@ class reduced_row_major {
     template <typename... Strides>
     GKO_ATTRIBUTES constexpr reduced_row_major(gko::dim<dimensionality> size,
                                                storage_type *storage,
-                                               Strides &&... strides)
+                                               Strides &&...strides)
         : reduced_row_major{
               size, storage,
               storage_stride_type{{std::forward<Strides>(strides)...}}}
@@ -350,7 +350,7 @@ class reduced_row_major {
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
-        helper::multidim_for_each(size_, [&](auto... indices) {
+        helper::multidim_for_each(size_, [&](auto &&...indices) {
             (*this)(indices...) = other(indices...);
         });
     }
@@ -368,7 +368,7 @@ class reduced_row_major {
     GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
         helper::are_all_integral<Indices...>::value,
         std::conditional_t<is_const, arithmetic_type, reference_type>>
-    operator()(Indices &&... indices) const
+    operator()(Indices &&...indices) const
     {
         return reference_type{storage_ +
                               compute_index(std::forward<Indices>(indices)...)};
@@ -385,7 +385,7 @@ class reduced_row_major {
     GKO_ATTRIBUTES constexpr std::enable_if_t<
         helper::are_span_compatible<SpanTypes...>::value,
         range<reduced_row_major>>
-    operator()(SpanTypes &&... spans) const
+    operator()(SpanTypes &&...spans) const
     {
         return helper::validate_spans(size_, spans...),
                range<reduced_row_major>{
@@ -439,7 +439,7 @@ class reduced_row_major {
 protected:
     template <typename... Indices>
     GKO_ATTRIBUTES GKO_INLINE constexpr size_type compute_index(
-        Indices &&... indices) const
+        Indices &&...indices) const
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
@@ -485,7 +485,7 @@ struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
      */
     template <typename... Indices>
     GKO_ATTRIBUTES GKO_INLINE constexpr scalar_type write_scalar(
-        scalar_type value, Indices &&... indices) const
+        scalar_type value, Indices &&...indices) const
     {
         static_assert(sizeof...(Indices) == Dimensionality,
                       "Number of indices must match dimensionality!");
@@ -667,7 +667,7 @@ class scaled_reduced_row_major
      */
     template <typename... Indices>
     GKO_ATTRIBUTES GKO_INLINE constexpr scalar_type read_scalar(
-        Indices &&... indices) const
+        Indices &&...indices) const
     {
         const arithmetic_type *GKO_RESTRICT rest_scalar = scalar_;
         return rest_scalar[compute_scalar_index(
@@ -702,7 +702,7 @@ class scaled_reduced_row_major
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
-        helper::multidim_for_each(size_, [&](auto... indices) {
+        helper::multidim_for_each(size_, [&](auto &&...indices) {
             this->write_scalar(other.read_scalar(indices...), indices...);
             (*this)(indices...) = other(indices...);
         });
@@ -721,11 +721,10 @@ class scaled_reduced_row_major
     GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
         helper::are_all_integral<Indices...>::value,
         std::conditional_t<is_const, arithmetic_type, reference_type>>
-    operator()(Indices... indices) const
+    operator()(Indices &&...indices) const
     {
-        return reference_type{
-            storage_ + compute_index(std::forward<Indices>(indices)...),
-            read_scalar(std::forward<Indices>(indices)...)};
+        return reference_type{storage_ + compute_index(indices...),
+                              read_scalar(indices...)};
     }
 
     /**
@@ -739,7 +738,7 @@ class scaled_reduced_row_major
     GKO_ATTRIBUTES constexpr std::enable_if_t<
         helper::are_span_compatible<SpanTypes...>::value,
         range<scaled_reduced_row_major>>
-    operator()(SpanTypes &&... spans) const
+    operator()(SpanTypes &&...spans) const
     {
         return helper::validate_spans(size_, spans...),
                range<scaled_reduced_row_major>{
@@ -830,7 +829,7 @@ class scaled_reduced_row_major
 protected:
     template <typename... Indices>
     GKO_ATTRIBUTES GKO_INLINE constexpr size_type compute_index(
-        Indices &&... indices) const
+        Indices &&...indices) const
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
@@ -840,7 +839,7 @@ class scaled_reduced_row_major
 
     template <typename... Indices>
     GKO_ATTRIBUTES constexpr GKO_INLINE size_type
-    compute_scalar_index(Indices &&... indices) const
+    compute_scalar_index(Indices &&...indices) const
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");

From f485f547fa8c4a87deffcf6bea9021c4270e3c1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 17 Nov 2020 15:42:21 +0100
Subject: [PATCH 0225/1008] Review update

Improve naming and documentation.

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 core/test/base/range_accessors.cpp            |  7 +-
 .../core/base/range_accessor_helper.hpp       | 78 +++++++++--------
 .../core/base/range_accessor_references.hpp   | 86 ++++++++++---------
 include/ginkgo/core/base/range_accessors.hpp  | 42 ++++-----
 4 files changed, 109 insertions(+), 104 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 03e078d3ab6..c008c2fb982 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -681,13 +681,8 @@ TYPED_TEST(ScaledReducedStorage3d, CorrectLengths)
 
 TYPED_TEST(ScaledReducedStorage3d, CorrectStride)
 {
-    EXPECT_EQ(this->r->get_storage_stride()[0], this->size[1] * this->size[2]);
-    EXPECT_EQ(this->r->get_storage_stride().at(0),
-              this->size[1] * this->size[2]);
-    EXPECT_EQ(this->r->get_storage_stride()[1], this->size[2]);
-    EXPECT_EQ(this->r->get_storage_stride().at(1), this->size[2]);
-    EXPECT_EQ(this->r->get_scalar_stride(), this->scalar_stride);
     EXPECT_EQ(this->r->get_scalar_stride(), this->scalar_stride);
+    EXPECT_EQ(this->r->get_storage_stride(), this->storage_stride);
 }
 
 
diff --git a/include/ginkgo/core/base/range_accessor_helper.hpp b/include/ginkgo/core/base/range_accessor_helper.hpp
index 4a1d50ceb8f..089d727b7bb 100644
--- a/include/ginkgo/core/base/range_accessor_helper.hpp
+++ b/include/ginkgo/core/base/range_accessor_helper.hpp
@@ -76,7 +76,7 @@ struct row_major_helper_s {
     compute(const dim<total_dim> &size,
             const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
                 &stride,
-            FirstType &&first, Indices &&...idxs)
+            FirstType &&first, Indices &&... idxs)
     {
         // The ASSERT size check must NOT be indexed with `dim_idx` directy,
         // otherwise, it leads to a linker error. The reason is likely that
@@ -105,16 +105,16 @@ struct row_major_helper_s<ValueType, total_dim, total_dim> {
 
 template <typename ValueType, size_type iter, size_type N>
 constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<iter == N, ValueType>
-mult_array(const dim<N> &)
+mult_size_upwards(const dim<N> &)
 {
     return 1;
 }
 
 template <typename ValueType, size_type iter, size_type N>
 constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<(iter < N), ValueType>
-mult_array(const dim<N> &size)
+mult_size_upwards(const dim<N> &size)
 {
-    return size[iter] * mult_array<ValueType, iter + 1>(size);
+    return size[iter] * mult_size_upwards<ValueType, iter + 1>(size);
 }
 
 
@@ -122,7 +122,7 @@ template <typename ValueType, size_type iter = 1, size_type N, typename... Args>
 constexpr GKO_ATTRIBUTES GKO_INLINE
     std::enable_if_t<N == 0 || (iter == N && iter == sizeof...(Args) + 1),
                      std::array<ValueType, N == 0 ? 0 : N - 1>>
-    extract_factorization(const dim<N> &, Args &&...args)
+    compute_default_stride_array(const dim<N> &, Args &&... args)
 {
     return {{std::forward<Args>(args)...}};
 }
@@ -130,10 +130,11 @@ constexpr GKO_ATTRIBUTES GKO_INLINE
 template <typename ValueType, size_type iter = 1, size_type N, typename... Args>
 constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<
     (iter < N) && (iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
-extract_factorization(const dim<N> &size, Args &&...args)
+compute_default_stride_array(const dim<N> &size, Args &&... args)
 {
-    return extract_factorization<ValueType, iter + 1>(
-        size, std::forward<Args>(args)..., mult_array<ValueType, iter>(size));
+    return compute_default_stride_array<ValueType, iter + 1>(
+        size, std::forward<Args>(args)...,
+        mult_size_upwards<ValueType, iter>(size));
 }
 
 
@@ -148,7 +149,7 @@ template <typename ValueType, size_type total_dim, typename... Indices>
 constexpr GKO_ATTRIBUTES GKO_INLINE ValueType compute_storage_index(
     const dim<total_dim> &size,
     const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)> &stride,
-    Indices &&...idxs)
+    Indices &&... idxs)
 {
     return detail::row_major_helper_s<ValueType, total_dim>::compute(
         size, stride, std::forward<Indices>(idxs)...);
@@ -160,10 +161,10 @@ constexpr GKO_ATTRIBUTES GKO_INLINE ValueType compute_storage_index(
  * padding
  */
 template <typename ValueType, size_type N>
-constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_stride_array(
+constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_default_stride_array(
     const dim<N> &size)
 {
-    return detail::extract_factorization<ValueType>(size);
+    return detail::compute_default_stride_array<ValueType>(size);
 }
 
 
@@ -193,7 +194,8 @@ template <typename ValueType, size_type mask, size_type set_bits_processed,
                               ? static_cast<bool>(mask &(
                                     size_type{1} << (total_dim - 1 - dim_idx)))
                               : false>
-struct row_major_masked_helper_s {};
+struct row_major_masked_helper_s {
+};
 
 
 // bit for current dimensionality is set
@@ -213,28 +215,28 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     template <typename... Args>
     static constexpr GKO_ATTRIBUTES GKO_INLINE
         std::array<ValueType, stride_size>
-        build_stride(const dim<total_dim> &size, Args &&...args)
+        build_stride(const dim<total_dim> &size, Args &&... args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
             total_dim>::build_stride(size, std::forward<Args>(args)...,
-                                     mult_array(size));
+                                     mult_size_upwards(size));
     }
 
     static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
-    mult_array(const dim<total_dim> &size)
+    mult_size_upwards(const dim<total_dim> &size)
     {
         return size[dim_idx] *
                row_major_masked_helper_s<
                    ValueType, mask, set_bits_processed + 1, stride_size,
-                   dim_idx + 1, total_dim>::mult_array(size);
+                   dim_idx + 1, total_dim>::mult_size_upwards(size);
     }
 
     template <typename First, typename... Indices>
     static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
     compute_idx(const dim<total_dim> &size,
                 const std::array<ValueType, stride_size> &stride, First &&first,
-                Indices &&...idxs)
+                Indices &&... idxs)
     {
         static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
                       "Mismatching number of Idxs!");
@@ -268,7 +270,7 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
     template <typename... Args>
     static constexpr GKO_ATTRIBUTES GKO_INLINE
         std::array<ValueType, stride_size>
-        build_stride(const dim<total_dim> &size, Args &&...args)
+        build_stride(const dim<total_dim> &size, Args &&... args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
@@ -276,18 +278,18 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
     }
 
     static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
-    mult_array(const dim<total_dim> &size)
+    mult_size_upwards(const dim<total_dim> &size)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
-            total_dim>::mult_array(size);
+            total_dim>::mult_size_upwards(size);
     }
 
     template <typename First, typename... Indices>
     static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
     compute_idx(const dim<total_dim> &size,
                 const std::array<ValueType, stride_size> &stride, First &&first,
-                Indices &&...idxs)
+                Indices &&... idxs)
     {
         static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
                       "Mismatching number of Idxs!");
@@ -320,7 +322,7 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     template <typename... Args>
     static constexpr GKO_ATTRIBUTES GKO_INLINE
         std::array<ValueType, stride_size>
-        build_stride(const dim<total_dim> &size, Args &&...args)
+        build_stride(const dim<total_dim> &size, Args &&... args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed, stride_size, dim_idx + 1,
@@ -328,18 +330,18 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     }
 
     static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
-    mult_array(const dim<total_dim> &size)
+    mult_size_upwards(const dim<total_dim> &size)
     {
         return row_major_masked_helper_s<ValueType, mask, set_bits_processed,
                                          stride_size, dim_idx + 1,
-                                         total_dim>::mult_array(size);
+                                         total_dim>::mult_size_upwards(size);
     }
 
     template <typename First, typename... Indices>
     static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
     compute_idx(const dim<total_dim> &size,
                 const std::array<ValueType, stride_size> &stride, First &&,
-                Indices &&...idxs)
+                Indices &&... idxs)
     {
         static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
                       "Mismatching number of Idxs!");
@@ -362,12 +364,12 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     template <typename... Args>
     static constexpr GKO_ATTRIBUTES GKO_INLINE
         std::array<ValueType, stride_size>
-        build_stride(const dim<total_dim> &, Args &&...args)
+        build_stride(const dim<total_dim> &, Args &&... args)
     {
         return {{std::forward<Args>(args)...}};
     }
     static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
-    mult_array(const dim<total_dim> &)
+    mult_size_upwards(const dim<total_dim> &)
     {
         return 1;
     }
@@ -391,7 +393,7 @@ template <typename ValueType, size_type mask, size_type stride_size,
           size_type total_dim, typename... Indices>
 constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_masked_index(
     const dim<total_dim> &size,
-    const std::array<ValueType, stride_size> &stride, Indices &&...idxs)
+    const std::array<ValueType, stride_size> &stride, Indices &&... idxs)
 {
     return detail::row_major_masked_helper_s<
         ValueType, mask, 0, stride_size, 0,
@@ -406,7 +408,7 @@ constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_masked_index(
  */
 template <typename ValueType, size_type mask, size_type stride_size,
           size_type total_dim>
-constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_masked_stride_array(
+constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_default_masked_stride_array(
     const dim<total_dim> &size)
 {
     return detail::row_major_masked_helper_s<ValueType, mask, 0, stride_size, 0,
@@ -418,13 +420,15 @@ constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_masked_stride_array(
  * Evaluates if all Args fulfill std::is_integral
  */
 template <typename... Args>
-struct are_all_integral : public std::true_type {};
+struct are_all_integral : public std::true_type {
+};
 
 template <typename First, typename... Args>
 struct are_all_integral<First, Args...>
     : public std::conditional<std::is_integral<std::decay_t<First>>::value,
                               are_all_integral<Args...>,
-                              std::false_type>::type {};
+                              std::false_type>::type {
+};
 
 
 namespace detail {
@@ -432,7 +436,8 @@ namespace detail {
 
 template <bool has_span, typename... Args>
 struct are_span_compatible_impl
-    : public std::integral_constant<bool, has_span> {};
+    : public std::integral_constant<bool, has_span> {
+};
 
 template <bool has_span, typename First, typename... Args>
 struct are_span_compatible_impl<has_span, First, Args...>
@@ -442,7 +447,8 @@ struct are_span_compatible_impl<has_span, First, Args...>
           are_span_compatible_impl<
               has_span || std::is_same<std::decay_t<First>, span>::value,
               Args...>,
-          std::false_type>::type {};
+          std::false_type>::type {
+};
 
 
 }  // namespace detail
@@ -461,7 +467,7 @@ namespace detail {
 
 template <size_type iter, size_type N, typename Callable, typename... Indices>
 GKO_ATTRIBUTES std::enable_if_t<iter == N> multidim_for_each_impl(
-    const dim<N> &, Callable &&callable, Indices &&...indices)
+    const dim<N> &, Callable &&callable, Indices &&... indices)
 {
     static_assert(iter == sizeof...(Indices),
                   "Number arguments must match current iteration!");
@@ -507,7 +513,7 @@ spans_in_size(const dim<N> &)
 
 template <size_type iter, size_type N, typename First, typename... Remaining>
 GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<(iter < N), int>
-spans_in_size(const dim<N> &size, First &&first, Remaining &&...remaining)
+spans_in_size(const dim<N> &size, First &&first, Remaining &&... remaining)
 {
     static_assert(sizeof...(Remaining) + 1 == N - iter,
                   "Number of remaining spans must be equal to N - iter");
@@ -522,7 +528,7 @@ spans_in_size(const dim<N> &size, First &&first, Remaining &&...remaining)
 
 template <size_type N, typename... Spans>
 GKO_ATTRIBUTES GKO_INLINE constexpr int validate_spans(const dim<N> &size,
-                                                       Spans &&...spans)
+                                                       Spans &&... spans)
 {
     return detail::spans_in_size<0>(size, std::forward<Spans>(spans)...);
 }
diff --git a/include/ginkgo/core/base/range_accessor_references.hpp b/include/ginkgo/core/base/range_accessor_references.hpp
index 063598d7b7b..a6062b7dd42 100644
--- a/include/ginkgo/core/base/range_accessor_references.hpp
+++ b/include/ginkgo/core/base/range_accessor_references.hpp
@@ -53,7 +53,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #define GKO_ENABLE_REFERENCE_CONSTEXPR constexpr
 
-#endif
+#endif  // __CUDA_ARCH__ && __CUDACC_VER_MAJOR__ && __CUDACC_VER_MAJOR__ < 11
 
 
 namespace gko {
@@ -113,43 +113,43 @@ to_value_type(const Ref &ref)
  * All assignment operators expect an rvalue reference (Reference &&) for
  * the Reference class in order to prevent copying the Reference object.
  *
- * @tparam Reference  The reference class this mixin provides operator
- * overloads for. The reference class needs to overload the cast operator
- * to ValueType
+ * @tparam Reference  The reference class this mixin provides operator overloads
+ *                    for. The reference class needs to overload the cast
+ *                    operatorto ValueType
  *
  * @tparam ArithmeticType  arithmetic type the Reference class is supposed
- * to represent.
+ *         to represent.
  *
  * @warning  This struct should only be used by reference classes.
  */
 template <typename Reference, typename ArithmeticType>
-struct enable_reference {
+struct enable_reference_operators {
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
 
-#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_oper, _op)          \
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR \
-        arithmetic_type                                             \
-        _oper(const Reference &ref1, const Reference &ref2)         \
-    {                                                               \
-        return to_value_type<arithmetic_type>(ref1)                 \
-            _op to_value_type<arithmetic_type>(ref2);               \
-    }                                                               \
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR \
-        arithmetic_type                                             \
-        _oper(const Reference &ref, const arithmetic_type &a)       \
-    {                                                               \
-        return to_value_type<arithmetic_type>(ref) _op a;           \
-    }                                                               \
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR \
-        arithmetic_type                                             \
-        _oper(const arithmetic_type &a, const Reference &ref)       \
-    {                                                               \
-        return a _op to_value_type<arithmetic_type>(ref);           \
+#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_op)                  \
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR  \
+        arithmetic_type                                              \
+        operator _op(const Reference &ref1, const Reference &ref2)   \
+    {                                                                \
+        return to_value_type<arithmetic_type>(ref1)                  \
+            _op to_value_type<arithmetic_type>(ref2);                \
+    }                                                                \
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR  \
+        arithmetic_type                                              \
+        operator _op(const Reference &ref, const arithmetic_type &a) \
+    {                                                                \
+        return to_value_type<arithmetic_type>(ref) _op a;            \
+    }                                                                \
+    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR  \
+        arithmetic_type                                              \
+        operator _op(const arithmetic_type &a, const Reference &ref) \
+    {                                                                \
+        return a _op to_value_type<arithmetic_type>(ref);            \
     }
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator*, *)
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator/, /)
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator+, +)
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator-, -)
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(*)
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(/)
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(+)
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(-)
 #undef GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD
 
 #define GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(_oper, _op)             \
@@ -190,7 +190,7 @@ struct enable_reference {
 /**
  * This namespace contains reference classes used inside accessors.
  *
- * @warning  These classes should not be used by anything else but accessors.
+ * @warning These classes should only be used by accessors.
  */
 namespace reference_class {
 
@@ -202,17 +202,18 @@ namespace reference_class {
  * Copying this reference is disabled, but move construction is possible to
  * allow for an additional layer (like gko::range).
  * The assignment operator only works for an rvalue reference (&&) to
- * prevent accidental copying the reference and working on a reference.
+ * prevent accidentally copying the reference and working on a reference.
  *
  * @tparam ArithmeticType  Type used for arithmetic operations, therefore,
- * the type which is used for input and output of this class.
+ *                         the type which is used for input and output of this
+ *                         class.
  *
  * @tparam StorageType  Type actually used as a storage, which is converted
- * to ArithmeticType before usage
+ *                      to ArithmeticType before usage
  */
 template <typename ArithmeticType, typename StorageType>
 class reduced_storage
-    : public detail::enable_reference<
+    : public detail::enable_reference_operators<
           reduced_storage<ArithmeticType, StorageType>, ArithmeticType> {
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
@@ -244,7 +245,8 @@ class reduced_storage
         arithmetic_type val) &&
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return *r_ptr = static_cast<storage_type>(val);
+        *r_ptr = static_cast<storage_type>(val);
+        return val;
     }
 
     GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
@@ -266,7 +268,7 @@ class reduced_storage
 // Specialization for const storage_type to prevent `operator=`
 template <typename ArithmeticType, typename StorageType>
 class reduced_storage<ArithmeticType, const StorageType>
-    : public detail::enable_reference<
+    : public detail::enable_reference_operators<
           reduced_storage<ArithmeticType, const StorageType>, ArithmeticType> {
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
@@ -313,14 +315,15 @@ class reduced_storage<ArithmeticType, const StorageType>
  * prevent accidental copying the reference and working on a reference.
  *
  * @tparam ArithmeticType  Type used for arithmetic operations, therefore,
- * the type which is used for input and output of this class.
+ *                         the type which is used for input and output of this
+ *                         class.
  *
  * @tparam StorageType  Type actually used as a storage, which is converted
- * to ArithmeticType before usage
+ *                      to ArithmeticType before usage
  */
 template <typename ArithmeticType, typename StorageType>
 class scaled_reduced_storage
-    : public detail::enable_reference<
+    : public detail::enable_reference_operators<
           scaled_reduced_storage<ArithmeticType, StorageType>, ArithmeticType> {
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
@@ -351,7 +354,8 @@ class scaled_reduced_storage
         arithmetic_type val) &&
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return *r_ptr = static_cast<storage_type>(val / scalar_), val;
+        *r_ptr = static_cast<storage_type>(val / scalar_);
+        return val;
     }
 
     GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
@@ -374,7 +378,7 @@ class scaled_reduced_storage
 // Specialization for constant storage_type (no `operator=`)
 template <typename ArithmeticType, typename StorageType>
 class scaled_reduced_storage<ArithmeticType, const StorageType>
-    : public detail::enable_reference<
+    : public detail::enable_reference_operators<
           scaled_reduced_storage<ArithmeticType, const StorageType>,
           ArithmeticType> {
 public:
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index c4b8ff5416e..b4f7d3a1405 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -73,8 +73,7 @@ namespace accessor {
  *          2-dimensional ranges.
  *
  * @tparam ValueType  type of values this accessor returns
- * @tparam Dimensionality  number of dimensions of this accessor (has to be
- * 2)
+ * @tparam Dimensionality  number of dimensions of this accessor (has to be 2)
  */
 template <typename ValueType, size_type Dimensionality>
 class row_major {
@@ -281,7 +280,7 @@ class reduced_row_major {
     template <typename... Strides>
     GKO_ATTRIBUTES constexpr reduced_row_major(gko::dim<dimensionality> size,
                                                storage_type *storage,
-                                               Strides &&...strides)
+                                               Strides &&... strides)
         : reduced_row_major{
               size, storage,
               storage_stride_type{{std::forward<Strides>(strides)...}}}
@@ -299,8 +298,9 @@ class reduced_row_major {
      */
     GKO_ATTRIBUTES constexpr reduced_row_major(dim<dimensionality> size,
                                                storage_type *storage)
-        : reduced_row_major{size, storage,
-                            helper::compute_stride_array<size_type>(size)}
+        : reduced_row_major{
+              size, storage,
+              helper::compute_default_stride_array<size_type>(size)}
     {}
 
     /**
@@ -350,7 +350,7 @@ class reduced_row_major {
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
-        helper::multidim_for_each(size_, [&](auto &&...indices) {
+        helper::multidim_for_each(size_, [&](auto &&... indices) {
             (*this)(indices...) = other(indices...);
         });
     }
@@ -368,14 +368,14 @@ class reduced_row_major {
     GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
         helper::are_all_integral<Indices...>::value,
         std::conditional_t<is_const, arithmetic_type, reference_type>>
-    operator()(Indices &&...indices) const
+    operator()(Indices &&... indices) const
     {
         return reference_type{storage_ +
                               compute_index(std::forward<Indices>(indices)...)};
     }
 
     /**
-     * Returns a sub-range spinning the current range (x1_span, x2_span, ...)
+     * Returns a sub-range spanning the current range (x1_span, x2_span, ...)
      *
      * @param spans  span for the indices
      *
@@ -385,7 +385,7 @@ class reduced_row_major {
     GKO_ATTRIBUTES constexpr std::enable_if_t<
         helper::are_span_compatible<SpanTypes...>::value,
         range<reduced_row_major>>
-    operator()(SpanTypes &&...spans) const
+    operator()(SpanTypes &&... spans) const
     {
         return helper::validate_spans(size_, spans...),
                range<reduced_row_major>{
@@ -439,7 +439,7 @@ class reduced_row_major {
 protected:
     template <typename... Indices>
     GKO_ATTRIBUTES GKO_INLINE constexpr size_type compute_index(
-        Indices &&...indices) const
+        Indices &&... indices) const
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
@@ -485,7 +485,7 @@ struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
      */
     template <typename... Indices>
     GKO_ATTRIBUTES GKO_INLINE constexpr scalar_type write_scalar(
-        scalar_type value, Indices &&...indices) const
+        scalar_type value, Indices &&... indices) const
     {
         static_assert(sizeof...(Indices) == Dimensionality,
                       "Number of indices must match dimensionality!");
@@ -616,7 +616,7 @@ class scaled_reduced_row_major
         storage_stride_type stride, scalar_type *scalar)
         : scaled_reduced_row_major{
               size, storage, stride, scalar,
-              helper::compute_masked_stride_array<
+              helper::compute_default_masked_stride_array<
                   size_type, scalar_mask, scalar_stride_dim, dimensionality>(
                   size)}
     {}
@@ -634,8 +634,8 @@ class scaled_reduced_row_major
                                                       storage_type *storage,
                                                       scalar_type *scalar)
         : scaled_reduced_row_major{
-              size, storage, helper::compute_stride_array<size_type>(size),
-              scalar}
+              size, storage,
+              helper::compute_default_stride_array<size_type>(size), scalar}
     {}
 
     /**
@@ -667,7 +667,7 @@ class scaled_reduced_row_major
      */
     template <typename... Indices>
     GKO_ATTRIBUTES GKO_INLINE constexpr scalar_type read_scalar(
-        Indices &&...indices) const
+        Indices &&... indices) const
     {
         const arithmetic_type *GKO_RESTRICT rest_scalar = scalar_;
         return rest_scalar[compute_scalar_index(
@@ -702,7 +702,7 @@ class scaled_reduced_row_major
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
-        helper::multidim_for_each(size_, [&](auto &&...indices) {
+        helper::multidim_for_each(size_, [&](auto &&... indices) {
             this->write_scalar(other.read_scalar(indices...), indices...);
             (*this)(indices...) = other(indices...);
         });
@@ -721,7 +721,7 @@ class scaled_reduced_row_major
     GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
         helper::are_all_integral<Indices...>::value,
         std::conditional_t<is_const, arithmetic_type, reference_type>>
-    operator()(Indices &&...indices) const
+    operator()(Indices &&... indices) const
     {
         return reference_type{storage_ + compute_index(indices...),
                               read_scalar(indices...)};
@@ -738,7 +738,7 @@ class scaled_reduced_row_major
     GKO_ATTRIBUTES constexpr std::enable_if_t<
         helper::are_span_compatible<SpanTypes...>::value,
         range<scaled_reduced_row_major>>
-    operator()(SpanTypes &&...spans) const
+    operator()(SpanTypes &&... spans) const
     {
         return helper::validate_spans(size_, spans...),
                range<scaled_reduced_row_major>{
@@ -765,7 +765,7 @@ class scaled_reduced_row_major
      * dimensionality - 1
      *
      * @returns a const reference to the storage stride array of size
-     * dimensionality - 1
+     *          dimensionality - 1
      */
     GKO_ATTRIBUTES
     GKO_INLINE constexpr const storage_stride_type &get_storage_stride() const
@@ -829,7 +829,7 @@ class scaled_reduced_row_major
 protected:
     template <typename... Indices>
     GKO_ATTRIBUTES GKO_INLINE constexpr size_type compute_index(
-        Indices &&...indices) const
+        Indices &&... indices) const
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
@@ -839,7 +839,7 @@ class scaled_reduced_row_major
 
     template <typename... Indices>
     GKO_ATTRIBUTES constexpr GKO_INLINE size_type
-    compute_scalar_index(Indices &&...indices) const
+    compute_scalar_index(Indices &&... indices) const
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");

From 2c47bb58798ee68c779ed7dc0122a32740efa749 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 17 Nov 2020 20:53:54 +0100
Subject: [PATCH 0226/1008] Remove inlining for constexpr

Additionally, sort the function specifier to follow roughly:
[static,friend] [constexpr] [ATTRIBUTES] <return-type> <function name>
---
 .../core/base/range_accessor_helper.hpp       |  69 +++++------
 .../core/base/range_accessor_references.hpp   |  64 +++++-----
 include/ginkgo/core/base/range_accessors.hpp  | 110 ++++++++----------
 3 files changed, 113 insertions(+), 130 deletions(-)

diff --git a/include/ginkgo/core/base/range_accessor_helper.hpp b/include/ginkgo/core/base/range_accessor_helper.hpp
index 089d727b7bb..847f9d09c5a 100644
--- a/include/ginkgo/core/base/range_accessor_helper.hpp
+++ b/include/ginkgo/core/base/range_accessor_helper.hpp
@@ -72,7 +72,7 @@ struct row_major_helper_s {
     static constexpr size_type dim_idx{current_iter - 1};
 
     template <typename FirstType, typename... Indices>
-    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    static constexpr GKO_ATTRIBUTES ValueType
     compute(const dim<total_dim> &size,
             const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
                 &stride,
@@ -93,7 +93,7 @@ struct row_major_helper_s {
 template <typename ValueType, size_type total_dim>
 struct row_major_helper_s<ValueType, total_dim, total_dim> {
     template <typename FirstType>
-    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    static constexpr GKO_ATTRIBUTES ValueType
     compute(const dim<total_dim> &size,
             const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>,
             FirstType &&first)
@@ -104,14 +104,14 @@ struct row_major_helper_s<ValueType, total_dim, total_dim> {
 
 
 template <typename ValueType, size_type iter, size_type N>
-constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<iter == N, ValueType>
+constexpr GKO_ATTRIBUTES std::enable_if_t<iter == N, ValueType>
 mult_size_upwards(const dim<N> &)
 {
     return 1;
 }
 
 template <typename ValueType, size_type iter, size_type N>
-constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<(iter < N), ValueType>
+constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), ValueType>
 mult_size_upwards(const dim<N> &size)
 {
     return size[iter] * mult_size_upwards<ValueType, iter + 1>(size);
@@ -119,7 +119,7 @@ mult_size_upwards(const dim<N> &size)
 
 
 template <typename ValueType, size_type iter = 1, size_type N, typename... Args>
-constexpr GKO_ATTRIBUTES GKO_INLINE
+constexpr GKO_ATTRIBUTES
     std::enable_if_t<N == 0 || (iter == N && iter == sizeof...(Args) + 1),
                      std::array<ValueType, N == 0 ? 0 : N - 1>>
     compute_default_stride_array(const dim<N> &, Args &&... args)
@@ -128,7 +128,7 @@ constexpr GKO_ATTRIBUTES GKO_INLINE
 }
 
 template <typename ValueType, size_type iter = 1, size_type N, typename... Args>
-constexpr GKO_ATTRIBUTES GKO_INLINE std::enable_if_t<
+constexpr GKO_ATTRIBUTES std::enable_if_t<
     (iter < N) && (iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
 compute_default_stride_array(const dim<N> &size, Args &&... args)
 {
@@ -146,7 +146,7 @@ compute_default_stride_array(const dim<N> &size, Args &&... args)
  * stride array
  */
 template <typename ValueType, size_type total_dim, typename... Indices>
-constexpr GKO_ATTRIBUTES GKO_INLINE ValueType compute_storage_index(
+constexpr GKO_ATTRIBUTES ValueType compute_storage_index(
     const dim<total_dim> &size,
     const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)> &stride,
     Indices &&... idxs)
@@ -161,8 +161,7 @@ constexpr GKO_ATTRIBUTES GKO_INLINE ValueType compute_storage_index(
  * padding
  */
 template <typename ValueType, size_type N>
-constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_default_stride_array(
-    const dim<N> &size)
+constexpr GKO_ATTRIBUTES auto compute_default_stride_array(const dim<N> &size)
 {
     return detail::compute_default_stride_array<ValueType>(size);
 }
@@ -213,9 +212,8 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
                   "The processed bits must be < total number of set bits!");
 
     template <typename... Args>
-    static constexpr GKO_ATTRIBUTES GKO_INLINE
-        std::array<ValueType, stride_size>
-        build_stride(const dim<total_dim> &size, Args &&... args)
+    static constexpr GKO_ATTRIBUTES std::array<ValueType, stride_size>
+    build_stride(const dim<total_dim> &size, Args &&... args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
@@ -223,7 +221,7 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
                                      mult_size_upwards(size));
     }
 
-    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    static constexpr GKO_ATTRIBUTES ValueType
     mult_size_upwards(const dim<total_dim> &size)
     {
         return size[dim_idx] *
@@ -233,7 +231,7 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     }
 
     template <typename First, typename... Indices>
-    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    static constexpr GKO_ATTRIBUTES ValueType
     compute_idx(const dim<total_dim> &size,
                 const std::array<ValueType, stride_size> &stride, First &&first,
                 Indices &&... idxs)
@@ -268,16 +266,15 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
                   "The iteration count must be smaller than total_dim here!!!");
 
     template <typename... Args>
-    static constexpr GKO_ATTRIBUTES GKO_INLINE
-        std::array<ValueType, stride_size>
-        build_stride(const dim<total_dim> &size, Args &&... args)
+    static constexpr GKO_ATTRIBUTES std::array<ValueType, stride_size>
+    build_stride(const dim<total_dim> &size, Args &&... args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
             total_dim>::build_stride(size, std::forward<Args>(args)...);
     }
 
-    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    static constexpr GKO_ATTRIBUTES ValueType
     mult_size_upwards(const dim<total_dim> &size)
     {
         return row_major_masked_helper_s<
@@ -286,7 +283,7 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
     }
 
     template <typename First, typename... Indices>
-    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    static constexpr GKO_ATTRIBUTES ValueType
     compute_idx(const dim<total_dim> &size,
                 const std::array<ValueType, stride_size> &stride, First &&first,
                 Indices &&... idxs)
@@ -320,16 +317,15 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     static_assert(set_bits_processed <= stride_size + 1,
                   "The processed bits must be < total number of set bits!");
     template <typename... Args>
-    static constexpr GKO_ATTRIBUTES GKO_INLINE
-        std::array<ValueType, stride_size>
-        build_stride(const dim<total_dim> &size, Args &&... args)
+    static constexpr GKO_ATTRIBUTES std::array<ValueType, stride_size>
+    build_stride(const dim<total_dim> &size, Args &&... args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed, stride_size, dim_idx + 1,
             total_dim>::build_stride(size, std::forward<Args>(args)...);
     }
 
-    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    static constexpr GKO_ATTRIBUTES ValueType
     mult_size_upwards(const dim<total_dim> &size)
     {
         return row_major_masked_helper_s<ValueType, mask, set_bits_processed,
@@ -338,7 +334,7 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     }
 
     template <typename First, typename... Indices>
-    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    static constexpr GKO_ATTRIBUTES ValueType
     compute_idx(const dim<total_dim> &size,
                 const std::array<ValueType, stride_size> &stride, First &&,
                 Indices &&... idxs)
@@ -362,19 +358,18 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
                   "The processed bits must be smaller than the total number of "
                   "set bits!");
     template <typename... Args>
-    static constexpr GKO_ATTRIBUTES GKO_INLINE
-        std::array<ValueType, stride_size>
-        build_stride(const dim<total_dim> &, Args &&... args)
+    static constexpr GKO_ATTRIBUTES std::array<ValueType, stride_size>
+    build_stride(const dim<total_dim> &, Args &&... args)
     {
         return {{std::forward<Args>(args)...}};
     }
-    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType
+    static constexpr GKO_ATTRIBUTES ValueType
     mult_size_upwards(const dim<total_dim> &)
     {
         return 1;
     }
 
-    static constexpr GKO_ATTRIBUTES GKO_INLINE ValueType compute_idx(
+    static constexpr GKO_ATTRIBUTES ValueType compute_idx(
         const dim<total_dim> &, const std::array<ValueType, stride_size> &)
     {
         return 0;
@@ -391,7 +386,7 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
  */
 template <typename ValueType, size_type mask, size_type stride_size,
           size_type total_dim, typename... Indices>
-constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_masked_index(
+constexpr GKO_ATTRIBUTES auto compute_masked_index(
     const dim<total_dim> &size,
     const std::array<ValueType, stride_size> &stride, Indices &&... idxs)
 {
@@ -408,7 +403,7 @@ constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_masked_index(
  */
 template <typename ValueType, size_type mask, size_type stride_size,
           size_type total_dim>
-constexpr GKO_ATTRIBUTES GKO_INLINE auto compute_default_masked_stride_array(
+constexpr GKO_ATTRIBUTES auto compute_default_masked_stride_array(
     const dim<total_dim> &size)
 {
     return detail::row_major_masked_helper_s<ValueType, mask, 0, stride_size, 0,
@@ -505,15 +500,15 @@ namespace detail {
 
 
 template <size_type iter, size_type N>
-GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<iter == N, int>
-spans_in_size(const dim<N> &)
+constexpr GKO_ATTRIBUTES std::enable_if_t<iter == N, int> spans_in_size(
+    const dim<N> &)
 {
     return 0;
 }
 
 template <size_type iter, size_type N, typename First, typename... Remaining>
-GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<(iter < N), int>
-spans_in_size(const dim<N> &size, First &&first, Remaining &&... remaining)
+constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), int> spans_in_size(
+    const dim<N> &size, First &&first, Remaining &&... remaining)
 {
     static_assert(sizeof...(Remaining) + 1 == N - iter,
                   "Number of remaining spans must be equal to N - iter");
@@ -527,8 +522,8 @@ spans_in_size(const dim<N> &size, First &&first, Remaining &&... remaining)
 
 
 template <size_type N, typename... Spans>
-GKO_ATTRIBUTES GKO_INLINE constexpr int validate_spans(const dim<N> &size,
-                                                       Spans &&... spans)
+constexpr GKO_ATTRIBUTES int validate_spans(const dim<N> &size,
+                                            Spans &&... spans)
 {
     return detail::spans_in_size<0>(size, std::forward<Spans>(spans)...);
 }
diff --git a/include/ginkgo/core/base/range_accessor_references.hpp b/include/ginkgo/core/base/range_accessor_references.hpp
index a6062b7dd42..97d398ef89c 100644
--- a/include/ginkgo/core/base/range_accessor_references.hpp
+++ b/include/ginkgo/core/base/range_accessor_references.hpp
@@ -90,17 +90,17 @@ struct has_cast_operator<
  * overloaded `operator value_type()`.
  */
 template <typename ValueType, typename Ref>
-GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
-    has_cast_operator<Ref, ValueType>::value, ValueType>
-to_value_type(const Ref &ref)
+constexpr GKO_ATTRIBUTES
+    std::enable_if_t<has_cast_operator<Ref, ValueType>::value, ValueType>
+    to_value_type(const Ref &ref)
 {
     return ref.Ref::operator ValueType();
 }
 
 template <typename ValueType, typename Ref>
-GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
-    !has_cast_operator<Ref, ValueType>::value, ValueType>
-to_value_type(const Ref &ref)
+constexpr GKO_ATTRIBUTES
+    std::enable_if_t<!has_cast_operator<Ref, ValueType>::value, ValueType>
+    to_value_type(const Ref &ref)
 {
     return static_cast<ValueType>(ref);
 }
@@ -127,20 +127,20 @@ struct enable_reference_operators {
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
 
 #define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_op)                  \
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR  \
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES  \
         arithmetic_type                                              \
         operator _op(const Reference &ref1, const Reference &ref2)   \
     {                                                                \
         return to_value_type<arithmetic_type>(ref1)                  \
             _op to_value_type<arithmetic_type>(ref2);                \
     }                                                                \
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR  \
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES  \
         arithmetic_type                                              \
         operator _op(const Reference &ref, const arithmetic_type &a) \
     {                                                                \
         return to_value_type<arithmetic_type>(ref) _op a;            \
     }                                                                \
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR  \
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES  \
         arithmetic_type                                              \
         operator _op(const arithmetic_type &a, const Reference &ref) \
     {                                                                \
@@ -153,14 +153,14 @@ struct enable_reference_operators {
 #undef GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD
 
 #define GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(_oper, _op)             \
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR        \
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES        \
         arithmetic_type                                                    \
         _oper(Reference &&ref1, const Reference &ref2)                     \
     {                                                                      \
         return std::move(ref1) = to_value_type<arithmetic_type>(ref1)      \
                    _op to_value_type<arithmetic_type>(ref2);               \
     }                                                                      \
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR        \
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES        \
         arithmetic_type                                                    \
         _oper(Reference &&ref, const arithmetic_type &a)                   \
     {                                                                      \
@@ -175,7 +175,7 @@ struct enable_reference_operators {
 
     // TODO test if comparison operators need to be overloaded as well
 
-    friend GKO_ATTRIBUTES GKO_INLINE GKO_ENABLE_REFERENCE_CONSTEXPR
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES
         arithmetic_type
         operator-(const Reference &ref)
     {
@@ -183,6 +183,9 @@ struct enable_reference_operators {
     }
 };
 
+// There is no more need for this macro in this file
+#undef GKO_ENABLE_REFERENCE_CONSTEXPR
+
 
 }  // namespace detail
 
@@ -230,33 +233,31 @@ class reduced_storage
     // Forbid copy construction
     reduced_storage(const reduced_storage &) = delete;
 
-    GKO_ATTRIBUTES constexpr reduced_storage(
+    constexpr GKO_ATTRIBUTES reduced_storage(
         storage_type *const GKO_RESTRICT ptr)
         : ptr_{ptr}
     {}
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    constexpr GKO_ATTRIBUTES operator arithmetic_type() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr);
     }
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
-        arithmetic_type val) &&
+    constexpr GKO_ATTRIBUTES arithmetic_type operator=(arithmetic_type val) &&
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         *r_ptr = static_cast<storage_type>(val);
         return val;
     }
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
-        const reduced_storage &ref) &&
+    constexpr GKO_ATTRIBUTES arithmetic_type
+    operator=(const reduced_storage &ref) &&
     {
         return std::move(*this) = static_cast<arithmetic_type>(ref);
     }
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
-        reduced_storage &&ref) &&
+    constexpr GKO_ATTRIBUTES arithmetic_type operator=(reduced_storage &&ref) &&
     {
         return std::move(*this) = static_cast<arithmetic_type>(ref);
     }
@@ -286,12 +287,12 @@ class reduced_storage<ArithmeticType, const StorageType>
 
     reduced_storage &operator=(reduced_storage &&) = delete;
 
-    GKO_ATTRIBUTES constexpr reduced_storage(
+    constexpr GKO_ATTRIBUTES reduced_storage(
         storage_type *const GKO_RESTRICT ptr)
         : ptr_{ptr}
     {}
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    constexpr GKO_ATTRIBUTES operator arithmetic_type() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr);
@@ -339,33 +340,32 @@ class scaled_reduced_storage
     // Forbid copy construction
     scaled_reduced_storage(const scaled_reduced_storage &) = delete;
 
-    GKO_ATTRIBUTES constexpr scaled_reduced_storage(
+    constexpr GKO_ATTRIBUTES scaled_reduced_storage(
         storage_type *const GKO_RESTRICT ptr, arithmetic_type scalar)
         : ptr_{ptr}, scalar_{scalar}
     {}
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    constexpr GKO_ATTRIBUTES operator arithmetic_type() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr) * scalar_;
     }
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
-        arithmetic_type val) &&
+    constexpr GKO_ATTRIBUTES arithmetic_type operator=(arithmetic_type val) &&
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         *r_ptr = static_cast<storage_type>(val / scalar_);
         return val;
     }
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
-        const scaled_reduced_storage &ref) &&
+    constexpr GKO_ATTRIBUTES arithmetic_type
+    operator=(const scaled_reduced_storage &ref) &&
     {
         return std::move(*this) = static_cast<arithmetic_type>(ref);
     }
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr arithmetic_type operator=(
-        scaled_reduced_storage &&ref) &&
+    constexpr GKO_ATTRIBUTES arithmetic_type
+    operator=(scaled_reduced_storage &&ref) &&
     {
         return std::move(*this) = static_cast<arithmetic_type>(ref);
     }
@@ -397,12 +397,12 @@ class scaled_reduced_storage<ArithmeticType, const StorageType>
 
     scaled_reduced_storage &operator=(scaled_reduced_storage &&) = delete;
 
-    GKO_ATTRIBUTES constexpr scaled_reduced_storage(
+    constexpr GKO_ATTRIBUTES scaled_reduced_storage(
         storage_type *const GKO_RESTRICT ptr, arithmetic_type scalar)
         : ptr_{ptr}, scalar_{scalar}
     {}
 
-    GKO_ATTRIBUTES GKO_INLINE constexpr operator arithmetic_type() const
+    constexpr GKO_ATTRIBUTES operator arithmetic_type() const
     {
         const storage_type *const GKO_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr) * scalar_;
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index b4f7d3a1405..717b1ba6868 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -111,7 +111,7 @@ class row_major {
      *                consecutive rows (i.e. `data + i * stride` points to
      *                the `i`-th row)
      */
-    GKO_ATTRIBUTES constexpr explicit row_major(data_type data,
+    constexpr GKO_ATTRIBUTES explicit row_major(data_type data,
                                                 size_type num_rows,
                                                 size_type num_cols,
                                                 size_type stride)
@@ -125,7 +125,7 @@ class row_major {
      *
      * @returns  a row major range which is read-only.
      */
-    GKO_ATTRIBUTES constexpr range<const_accessor> to_const() const
+    constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
     {
         return range<const_accessor>{data, lengths[0], lengths[1], stride};
     }
@@ -138,7 +138,7 @@ class row_major {
      *
      * @return data element at (row, col)
      */
-    GKO_ATTRIBUTES constexpr value_type &operator()(size_type row,
+    constexpr GKO_ATTRIBUTES value_type &operator()(size_type row,
                                                     size_type col) const
     {
         return GKO_ASSERT(row < lengths[0]), GKO_ASSERT(col < lengths[1]),
@@ -153,7 +153,7 @@ class row_major {
      *
      * @return sub-range spanning the range (rows, cols)
      */
-    GKO_ATTRIBUTES constexpr range<row_major> operator()(const span &rows,
+    constexpr GKO_ATTRIBUTES range<row_major> operator()(const span &rows,
                                                          const span &cols) const
     {
         return GKO_ASSERT(rows.is_valid()), GKO_ASSERT(cols.is_valid()),
@@ -171,7 +171,7 @@ class row_major {
      *
      * @return length in dimension `dimension`
      */
-    GKO_ATTRIBUTES constexpr size_type length(size_type dimension) const
+    constexpr GKO_ATTRIBUTES size_type length(size_type dimension) const
     {
         return dimension < 2 ? lengths[dimension] : 1;
     }
@@ -262,7 +262,7 @@ class reduced_row_major {
      * @param storage  pointer to the block of memory containing the storage
      * @param stride  stride array used for memory accesses
      */
-    GKO_ATTRIBUTES constexpr reduced_row_major(gko::dim<dimensionality> size,
+    constexpr GKO_ATTRIBUTES reduced_row_major(gko::dim<dimensionality> size,
                                                storage_type *storage,
                                                storage_stride_type stride)
         : size_{size}, storage_{storage}, stride_(stride)
@@ -278,7 +278,7 @@ class reduced_row_major {
      * @param strides  strides used for memory accesses
      */
     template <typename... Strides>
-    GKO_ATTRIBUTES constexpr reduced_row_major(gko::dim<dimensionality> size,
+    constexpr GKO_ATTRIBUTES reduced_row_major(gko::dim<dimensionality> size,
                                                storage_type *storage,
                                                Strides &&... strides)
         : reduced_row_major{
@@ -296,7 +296,7 @@ class reduced_row_major {
      * @param storage  pointer to the block of memory containing the storage
      * @param size  multidimensional size of the memory
      */
-    GKO_ATTRIBUTES constexpr reduced_row_major(dim<dimensionality> size,
+    constexpr GKO_ATTRIBUTES reduced_row_major(dim<dimensionality> size,
                                                storage_type *storage)
         : reduced_row_major{
               size, storage,
@@ -306,7 +306,7 @@ class reduced_row_major {
     /**
      * Creates an empty accessor (pointing nowhere with an empty size)
      */
-    GKO_ATTRIBUTES constexpr reduced_row_major()
+    constexpr GKO_ATTRIBUTES reduced_row_major()
         : reduced_row_major{{0, 0, 0}, nullptr}
     {}
 
@@ -317,7 +317,7 @@ class reduced_row_major {
      *
      * @returns  a reduced_row_major major range which is read-only.
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr range<const_accessor> to_const() const
+    constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
     {
         return range<const_accessor>{size_, storage_, stride_};
     }
@@ -329,8 +329,7 @@ class reduced_row_major {
      *
      * @returns  length in dimension `dimension`
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr size_type length(
-        size_type dimension) const
+    constexpr GKO_ATTRIBUTES size_type length(size_type dimension) const
     {
         return dimension < dimensionality ? size_[dimension] : 1;
     }
@@ -365,7 +364,7 @@ class reduced_row_major {
      *           is const), or a reference if the accessor is non-const
      */
     template <typename... Indices>
-    GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
+    constexpr GKO_ATTRIBUTES std::enable_if_t<
         helper::are_all_integral<Indices...>::value,
         std::conditional_t<is_const, arithmetic_type, reference_type>>
     operator()(Indices &&... indices) const
@@ -382,10 +381,10 @@ class reduced_row_major {
      * @returns a sub-range for the given spans.
      */
     template <typename... SpanTypes>
-    GKO_ATTRIBUTES constexpr std::enable_if_t<
-        helper::are_span_compatible<SpanTypes...>::value,
-        range<reduced_row_major>>
-    operator()(SpanTypes &&... spans) const
+    constexpr GKO_ATTRIBUTES
+        std::enable_if_t<helper::are_span_compatible<SpanTypes...>::value,
+                         range<reduced_row_major>>
+        operator()(SpanTypes &&... spans) const
     {
         return helper::validate_spans(size_, spans...),
                range<reduced_row_major>{
@@ -399,7 +398,7 @@ class reduced_row_major {
      *
      * @returns the size of the accessor
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr dim<dimensionality> get_size() const
+    constexpr GKO_ATTRIBUTES dim<dimensionality> get_size() const
     {
         return size_;
     }
@@ -410,17 +409,14 @@ class reduced_row_major {
      * @returns returns a pointer to a stride array of size dimensionality - 1
      */
     GKO_ATTRIBUTES
-    GKO_INLINE constexpr const storage_stride_type &get_stride() const
-    {
-        return stride_;
-    }
+    constexpr const storage_stride_type &get_stride() const { return stride_; }
 
     /**
      * Returns the pointer to the storage data
      *
      * @returns the pointer to the storage data
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr storage_type *get_stored_data() const
+    constexpr GKO_ATTRIBUTES storage_type *get_stored_data() const
     {
         return storage_;
     }
@@ -430,16 +426,15 @@ class reduced_row_major {
      *
      * @returns a const pointer to the storage data
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr const storage_type *get_const_storage()
-        const
+    constexpr GKO_ATTRIBUTES const storage_type *get_const_storage() const
     {
         return storage_;
     }
 
 protected:
     template <typename... Indices>
-    GKO_ATTRIBUTES GKO_INLINE constexpr size_type compute_index(
-        Indices &&... indices) const
+    constexpr GKO_ATTRIBUTES size_type
+    compute_index(Indices &&... indices) const
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
@@ -484,8 +479,8 @@ struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
      * @returns the written value.
      */
     template <typename... Indices>
-    GKO_ATTRIBUTES GKO_INLINE constexpr scalar_type write_scalar(
-        scalar_type value, Indices &&... indices) const
+    constexpr GKO_ATTRIBUTES scalar_type
+    write_scalar(scalar_type value, Indices &&... indices) const
     {
         static_assert(sizeof...(Indices) == Dimensionality,
                       "Number of indices must match dimensionality!");
@@ -495,7 +490,7 @@ struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
     }
 
 private:
-    GKO_ATTRIBUTES GKO_INLINE constexpr const Accessor *self() const
+    constexpr GKO_ATTRIBUTES const Accessor *self() const
     {
         return static_cast<const Accessor *>(this);
     }
@@ -589,7 +584,7 @@ class scaled_reduced_row_major
      *                values.
      * @param scalar_stride  stride array used for memory accesses to scalar
      */
-    GKO_ATTRIBUTES constexpr scaled_reduced_row_major(
+    constexpr GKO_ATTRIBUTES scaled_reduced_row_major(
         dim<dimensionality> size, storage_type *storage,
         storage_stride_type storage_stride, scalar_type *scalar,
         scalar_stride_type scalar_stride)
@@ -611,7 +606,7 @@ class scaled_reduced_row_major
      * @param scalar  pointer to the block of memory containing the scalar
      *                values.
      */
-    GKO_ATTRIBUTES constexpr scaled_reduced_row_major(
+    constexpr GKO_ATTRIBUTES scaled_reduced_row_major(
         dim<dimensionality> size, storage_type *storage,
         storage_stride_type stride, scalar_type *scalar)
         : scaled_reduced_row_major{
@@ -630,7 +625,7 @@ class scaled_reduced_row_major
      * @param scalar  pointer to the block of memory containing the scalar
      *                values.
      */
-    GKO_ATTRIBUTES constexpr scaled_reduced_row_major(dim<dimensionality> size,
+    constexpr GKO_ATTRIBUTES scaled_reduced_row_major(dim<dimensionality> size,
                                                       storage_type *storage,
                                                       scalar_type *scalar)
         : scaled_reduced_row_major{
@@ -641,7 +636,7 @@ class scaled_reduced_row_major
     /**
      * Creates an empty accessor (pointing nowhere with an empty size)
      */
-    GKO_ATTRIBUTES constexpr scaled_reduced_row_major()
+    constexpr GKO_ATTRIBUTES scaled_reduced_row_major()
         : scaled_reduced_row_major{{0, 0, 0}, nullptr, nullptr}
     {}
 
@@ -652,7 +647,7 @@ class scaled_reduced_row_major
      *
      * @returns  a scaled_reduced_row_major major range which is read-only.
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr range<const_accessor> to_const() const
+    constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
     {
         return range<const_accessor>{size_, storage_, storage_stride_, scalar_,
                                      scalar_stride_};
@@ -666,8 +661,8 @@ class scaled_reduced_row_major
      * @returns the scalar value at the given indices.
      */
     template <typename... Indices>
-    GKO_ATTRIBUTES GKO_INLINE constexpr scalar_type read_scalar(
-        Indices &&... indices) const
+    constexpr GKO_ATTRIBUTES scalar_type
+    read_scalar(Indices &&... indices) const
     {
         const arithmetic_type *GKO_RESTRICT rest_scalar = scalar_;
         return rest_scalar[compute_scalar_index(
@@ -681,8 +676,7 @@ class scaled_reduced_row_major
      *
      * @returns length in dimension `dimension`
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr size_type length(
-        size_type dimension) const
+    constexpr GKO_ATTRIBUTES size_type length(size_type dimension) const
     {
         return dimension < dimensionality ? size_[dimension] : 1;
     }
@@ -718,7 +712,7 @@ class scaled_reduced_row_major
      *           is const), or a reference if the accessor is non-const
      */
     template <typename... Indices>
-    GKO_ATTRIBUTES GKO_INLINE constexpr std::enable_if_t<
+    constexpr GKO_ATTRIBUTES std::enable_if_t<
         helper::are_all_integral<Indices...>::value,
         std::conditional_t<is_const, arithmetic_type, reference_type>>
     operator()(Indices &&... indices) const
@@ -735,10 +729,10 @@ class scaled_reduced_row_major
      * @returns a sub-range for the given spans.
      */
     template <typename... SpanTypes>
-    GKO_ATTRIBUTES constexpr std::enable_if_t<
-        helper::are_span_compatible<SpanTypes...>::value,
-        range<scaled_reduced_row_major>>
-    operator()(SpanTypes &&... spans) const
+    constexpr GKO_ATTRIBUTES
+        std::enable_if_t<helper::are_span_compatible<SpanTypes...>::value,
+                         range<scaled_reduced_row_major>>
+        operator()(SpanTypes &&... spans) const
     {
         return helper::validate_spans(size_, spans...),
                range<scaled_reduced_row_major>{
@@ -755,7 +749,7 @@ class scaled_reduced_row_major
      *
      * @returns the size of the accessor
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr dim<dimensionality> get_size() const
+    constexpr GKO_ATTRIBUTES dim<dimensionality> get_size() const
     {
         return size_;
     }
@@ -767,8 +761,8 @@ class scaled_reduced_row_major
      * @returns a const reference to the storage stride array of size
      *          dimensionality - 1
      */
-    GKO_ATTRIBUTES
-    GKO_INLINE constexpr const storage_stride_type &get_storage_stride() const
+    constexpr GKO_ATTRIBUTES const storage_stride_type &get_storage_stride()
+        const
     {
         return storage_stride_;
     }
@@ -778,8 +772,7 @@ class scaled_reduced_row_major
      *
      * @returns a const reference to the scalar stride array
      */
-    GKO_ATTRIBUTES
-    GKO_INLINE constexpr const scalar_stride_type &get_scalar_stride() const
+    constexpr GKO_ATTRIBUTES const scalar_stride_type &get_scalar_stride() const
     {
         return scalar_stride_;
     }
@@ -789,7 +782,7 @@ class scaled_reduced_row_major
      *
      * @returns the pointer to the storage data
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr storage_type *get_stored_data() const
+    constexpr GKO_ATTRIBUTES storage_type *get_stored_data() const
     {
         return storage_;
     }
@@ -799,8 +792,7 @@ class scaled_reduced_row_major
      *
      * @returns a const pointer to the storage data
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr const storage_type *get_const_storage()
-        const
+    constexpr GKO_ATTRIBUTES const storage_type *get_const_storage() const
     {
         return storage_;
     }
@@ -810,26 +802,22 @@ class scaled_reduced_row_major
      *
      * @returns the pointer to the scalar data
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr scalar_type *get_scalar() const
-    {
-        return scalar_;
-    }
+    constexpr GKO_ATTRIBUTES scalar_type *get_scalar() const { return scalar_; }
 
     /**
      * Returns a const pointer to the scalar data
      *
      * @returns a const pointer to the scalar data
      */
-    GKO_ATTRIBUTES GKO_INLINE constexpr const scalar_type *get_const_scalar()
-        const
+    constexpr GKO_ATTRIBUTES const scalar_type *get_const_scalar() const
     {
         return scalar_;
     }
 
 protected:
     template <typename... Indices>
-    GKO_ATTRIBUTES GKO_INLINE constexpr size_type compute_index(
-        Indices &&... indices) const
+    constexpr GKO_ATTRIBUTES size_type
+    compute_index(Indices &&... indices) const
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
@@ -838,7 +826,7 @@ class scaled_reduced_row_major
     }
 
     template <typename... Indices>
-    GKO_ATTRIBUTES constexpr GKO_INLINE size_type
+    constexpr GKO_ATTRIBUTES size_type
     compute_scalar_index(Indices &&... indices) const
     {
         static_assert(sizeof...(Indices) == dimensionality,

From 0f61edb19940c7c1910208756638bdb46d5b953d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 17 Nov 2020 21:11:14 +0100
Subject: [PATCH 0227/1008] Only use forwarding references when forwarding

---
 include/ginkgo/core/base/range_accessor_helper.hpp | 14 +++++++-------
 include/ginkgo/core/base/range_accessors.hpp       | 11 ++++++-----
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/include/ginkgo/core/base/range_accessor_helper.hpp b/include/ginkgo/core/base/range_accessor_helper.hpp
index 847f9d09c5a..094638cbe8f 100644
--- a/include/ginkgo/core/base/range_accessor_helper.hpp
+++ b/include/ginkgo/core/base/range_accessor_helper.hpp
@@ -76,7 +76,7 @@ struct row_major_helper_s {
     compute(const dim<total_dim> &size,
             const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
                 &stride,
-            FirstType &&first, Indices &&... idxs)
+            FirstType first, Indices &&... idxs)
     {
         // The ASSERT size check must NOT be indexed with `dim_idx` directy,
         // otherwise, it leads to a linker error. The reason is likely that
@@ -233,7 +233,7 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     template <typename First, typename... Indices>
     static constexpr GKO_ATTRIBUTES ValueType
     compute_idx(const dim<total_dim> &size,
-                const std::array<ValueType, stride_size> &stride, First &&first,
+                const std::array<ValueType, stride_size> &stride, First first,
                 Indices &&... idxs)
     {
         static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
@@ -285,7 +285,7 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
     template <typename First, typename... Indices>
     static constexpr GKO_ATTRIBUTES ValueType
     compute_idx(const dim<total_dim> &size,
-                const std::array<ValueType, stride_size> &stride, First &&first,
+                const std::array<ValueType, stride_size> &stride, First first,
                 Indices &&... idxs)
     {
         static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
@@ -336,7 +336,7 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     template <typename First, typename... Indices>
     static constexpr GKO_ATTRIBUTES ValueType
     compute_idx(const dim<total_dim> &size,
-                const std::array<ValueType, stride_size> &stride, First &&,
+                const std::array<ValueType, stride_size> &stride, First,
                 Indices &&... idxs)
     {
         static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
@@ -462,7 +462,7 @@ namespace detail {
 
 template <size_type iter, size_type N, typename Callable, typename... Indices>
 GKO_ATTRIBUTES std::enable_if_t<iter == N> multidim_for_each_impl(
-    const dim<N> &, Callable &&callable, Indices &&... indices)
+    const dim<N> &, Callable callable, Indices &&... indices)
 {
     static_assert(iter == sizeof...(Indices),
                   "Number arguments must match current iteration!");
@@ -471,7 +471,7 @@ GKO_ATTRIBUTES std::enable_if_t<iter == N> multidim_for_each_impl(
 
 template <size_type iter, size_type N, typename Callable, typename... Indices>
 GKO_ATTRIBUTES std::enable_if_t<(iter < N)> multidim_for_each_impl(
-    const dim<N> &size, Callable &&callable, Indices... indices)
+    const dim<N> &size, Callable &&callable, Indices &&... indices)
 {
     static_assert(iter == sizeof...(Indices),
                   "Number arguments must match current iteration!");
@@ -508,7 +508,7 @@ constexpr GKO_ATTRIBUTES std::enable_if_t<iter == N, int> spans_in_size(
 
 template <size_type iter, size_type N, typename First, typename... Remaining>
 constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), int> spans_in_size(
-    const dim<N> &size, First &&first, Remaining &&... remaining)
+    const dim<N> &size, First first, Remaining &&... remaining)
 {
     static_assert(sizeof...(Remaining) + 1 == N - iter,
                   "Number of remaining spans must be equal to N - iter");
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 717b1ba6868..7b3d357979c 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -349,7 +349,7 @@ class reduced_row_major {
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
-        helper::multidim_for_each(size_, [&](auto &&... indices) {
+        helper::multidim_for_each(size_, [&](auto... indices) {
             (*this)(indices...) = other(indices...);
         });
     }
@@ -384,7 +384,7 @@ class reduced_row_major {
     constexpr GKO_ATTRIBUTES
         std::enable_if_t<helper::are_span_compatible<SpanTypes...>::value,
                          range<reduced_row_major>>
-        operator()(SpanTypes &&... spans) const
+        operator()(SpanTypes... spans) const
     {
         return helper::validate_spans(size_, spans...),
                range<reduced_row_major>{
@@ -696,7 +696,8 @@ class scaled_reduced_row_major
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
-        helper::multidim_for_each(size_, [&](auto &&... indices) {
+        helper::multidim_for_each(size_, [&](auto... indices) {
+            // especially inefficient if mask has not all bits set
             this->write_scalar(other.read_scalar(indices...), indices...);
             (*this)(indices...) = other(indices...);
         });
@@ -715,7 +716,7 @@ class scaled_reduced_row_major
     constexpr GKO_ATTRIBUTES std::enable_if_t<
         helper::are_all_integral<Indices...>::value,
         std::conditional_t<is_const, arithmetic_type, reference_type>>
-    operator()(Indices &&... indices) const
+    operator()(Indices... indices) const
     {
         return reference_type{storage_ + compute_index(indices...),
                               read_scalar(indices...)};
@@ -732,7 +733,7 @@ class scaled_reduced_row_major
     constexpr GKO_ATTRIBUTES
         std::enable_if_t<helper::are_span_compatible<SpanTypes...>::value,
                          range<scaled_reduced_row_major>>
-        operator()(SpanTypes &&... spans) const
+        operator()(SpanTypes... spans) const
     {
         return helper::validate_spans(size_, spans...),
                range<scaled_reduced_row_major>{

From b6a07e677f69c74b239c5ae71bc69afcfc2a23e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 17 Nov 2020 21:56:25 +0100
Subject: [PATCH 0228/1008] Fix more sonarcloud code smells

- explicitly state what variables to capture in a lambda
- add `explicit` to reference_class constructors
- add `noexcept` for reference_class copy move assignment
---
 .../core/base/range_accessor_references.hpp   | 28 ++++++++++++-------
 include/ginkgo/core/base/range_accessors.hpp  |  6 ++--
 2 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/include/ginkgo/core/base/range_accessor_references.hpp b/include/ginkgo/core/base/range_accessor_references.hpp
index 97d398ef89c..9f41351f4a8 100644
--- a/include/ginkgo/core/base/range_accessor_references.hpp
+++ b/include/ginkgo/core/base/range_accessor_references.hpp
@@ -173,7 +173,7 @@ struct enable_reference_operators {
     GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator-=, -)
 #undef GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD
 
-    // TODO test if comparison operators need to be overloaded as well
+    // TODO Add overloads for comparison operators if needed
 
     friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES
         arithmetic_type
@@ -233,7 +233,7 @@ class reduced_storage
     // Forbid copy construction
     reduced_storage(const reduced_storage &) = delete;
 
-    constexpr GKO_ATTRIBUTES reduced_storage(
+    constexpr explicit GKO_ATTRIBUTES reduced_storage(
         storage_type *const GKO_RESTRICT ptr)
         : ptr_{ptr}
     {}
@@ -245,6 +245,7 @@ class reduced_storage
     }
 
     constexpr GKO_ATTRIBUTES arithmetic_type operator=(arithmetic_type val) &&
+        noexcept
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         *r_ptr = static_cast<storage_type>(val);
@@ -254,12 +255,15 @@ class reduced_storage
     constexpr GKO_ATTRIBUTES arithmetic_type
     operator=(const reduced_storage &ref) &&
     {
-        return std::move(*this) = static_cast<arithmetic_type>(ref);
+        std::move(*this) = static_cast<arithmetic_type>(ref);
+        return static_cast<arithmetic_type>(*this);
     }
 
     constexpr GKO_ATTRIBUTES arithmetic_type operator=(reduced_storage &&ref) &&
+        noexcept
     {
-        return std::move(*this) = static_cast<arithmetic_type>(ref);
+        std::move(*this) = static_cast<arithmetic_type>(ref);
+        return static_cast<arithmetic_type>(*this);
     }
 
 private:
@@ -287,7 +291,7 @@ class reduced_storage<ArithmeticType, const StorageType>
 
     reduced_storage &operator=(reduced_storage &&) = delete;
 
-    constexpr GKO_ATTRIBUTES reduced_storage(
+    constexpr explicit GKO_ATTRIBUTES reduced_storage(
         storage_type *const GKO_RESTRICT ptr)
         : ptr_{ptr}
     {}
@@ -340,7 +344,7 @@ class scaled_reduced_storage
     // Forbid copy construction
     scaled_reduced_storage(const scaled_reduced_storage &) = delete;
 
-    constexpr GKO_ATTRIBUTES scaled_reduced_storage(
+    constexpr explicit GKO_ATTRIBUTES scaled_reduced_storage(
         storage_type *const GKO_RESTRICT ptr, arithmetic_type scalar)
         : ptr_{ptr}, scalar_{scalar}
     {}
@@ -352,6 +356,7 @@ class scaled_reduced_storage
     }
 
     constexpr GKO_ATTRIBUTES arithmetic_type operator=(arithmetic_type val) &&
+        noexcept
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         *r_ptr = static_cast<storage_type>(val / scalar_);
@@ -361,13 +366,16 @@ class scaled_reduced_storage
     constexpr GKO_ATTRIBUTES arithmetic_type
     operator=(const scaled_reduced_storage &ref) &&
     {
-        return std::move(*this) = static_cast<arithmetic_type>(ref);
+        std::move(*this) = static_cast<arithmetic_type>(ref);
+        return static_cast<arithmetic_type>(*this);
     }
 
     constexpr GKO_ATTRIBUTES arithmetic_type
-    operator=(scaled_reduced_storage &&ref) &&
+        operator=(scaled_reduced_storage &&ref) &&
+        noexcept
     {
-        return std::move(*this) = static_cast<arithmetic_type>(ref);
+        std::move(*this) = static_cast<arithmetic_type>(ref);
+        return static_cast<arithmetic_type>(*this);
     }
 
 private:
@@ -397,7 +405,7 @@ class scaled_reduced_storage<ArithmeticType, const StorageType>
 
     scaled_reduced_storage &operator=(scaled_reduced_storage &&) = delete;
 
-    constexpr GKO_ATTRIBUTES scaled_reduced_storage(
+    constexpr explicit GKO_ATTRIBUTES scaled_reduced_storage(
         storage_type *const GKO_RESTRICT ptr, arithmetic_type scalar)
         : ptr_{ptr}, scalar_{scalar}
     {}
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 7b3d357979c..5269bbe79d7 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -349,7 +349,7 @@ class reduced_row_major {
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
-        helper::multidim_for_each(size_, [&](auto... indices) {
+        helper::multidim_for_each(size_, [this, &other](auto... indices) {
             (*this)(indices...) = other(indices...);
         });
     }
@@ -442,6 +442,7 @@ class reduced_row_major {
             size_, stride_, std::forward<Indices>(indices)...);
     }
 
+private:
     const dim<dimensionality> size_;
     storage_type *storage_;
     const storage_stride_type stride_;
@@ -696,7 +697,7 @@ class scaled_reduced_row_major
     template <typename OtherAccessor>
     GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
-        helper::multidim_for_each(size_, [&](auto... indices) {
+        helper::multidim_for_each(size_, [this, &other](auto... indices) {
             // especially inefficient if mask has not all bits set
             this->write_scalar(other.read_scalar(indices...), indices...);
             (*this)(indices...) = other(indices...);
@@ -837,6 +838,7 @@ class scaled_reduced_row_major
             size_, scalar_stride_, std::forward<Indices>(indices)...);
     }
 
+private:
     const dim<dimensionality> size_;
     storage_type *storage_;
     const storage_stride_type storage_stride_;

From d7c97c7e23dab31588cfb392cb8e08b685b91ffb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 18 Nov 2020 15:30:54 +0100
Subject: [PATCH 0229/1008] Fix OMP Jacobi test error

---
 omp/test/preconditioner/jacobi_kernels.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/omp/test/preconditioner/jacobi_kernels.cpp b/omp/test/preconditioner/jacobi_kernels.cpp
index 73f8a216f9e..78663913f71 100644
--- a/omp/test/preconditioner/jacobi_kernels.cpp
+++ b/omp/test/preconditioner/jacobi_kernels.cpp
@@ -308,7 +308,7 @@ TEST_F(Jacobi, OmpPreconditionerEquivalentToRefWithBlockSize32Sorted)
 TEST_F(Jacobi, OmpPreconditionerEquivalentToRefWithBlockSize32Unsorted)
 {
     std::ranlux48 engine(43);
-    initialize_data({0, 32, 64, 96, 128}, {}, {}, 32, 100, 110, 0.1, false);
+    initialize_data({0, 32, 64, 96, 128}, {}, {}, 32, 100, 110, 1, 0.1, false);
     gko::test::unsort_matrix(mtx.get(), engine);
 
     auto bj = bj_factory->generate(mtx);

From dc825cbb0f5cc9bc902a1e88ab0a761e862b8d6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 19 Nov 2020 00:13:03 +0100
Subject: [PATCH 0230/1008] Add gko::half to the accessor testing

Additionally, fix minor code smells
---
 core/base/extended_float.hpp                  |  11 +-
 core/test/base/range_accessors.cpp            | 170 ++++++++++--------
 core/test/utils.hpp                           |  20 ++-
 .../core/base/range_accessor_helper.hpp       |   2 +-
 .../core/base/range_accessor_references.hpp   |  35 ++--
 5 files changed, 134 insertions(+), 104 deletions(-)

diff --git a/core/base/extended_float.hpp b/core/base/extended_float.hpp
index 418a91dd9dc..fb58734e594 100644
--- a/core/base/extended_float.hpp
+++ b/core/base/extended_float.hpp
@@ -67,7 +67,8 @@ namespace detail {
 
 
 template <std::size_t, typename = void>
-struct uint_of_impl {};
+struct uint_of_impl {
+};
 
 template <std::size_t Bits>
 struct uint_of_impl<Bits, std::enable_if_t<(Bits <= 16)>> {
@@ -89,7 +90,8 @@ using uint_of = typename uint_of_impl<Bits>::type;
 
 
 template <typename T>
-struct basic_float_traits {};
+struct basic_float_traits {
+};
 
 template <>
 struct basic_float_traits<float16> {
@@ -550,6 +552,11 @@ class complex<gko::truncated<T, NumComponents>> {
 };
 
 
+template <>
+struct is_scalar<gko::half> : std::true_type {
+};
+
+
 }  // namespace std
 
 
diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index c008c2fb982..ad3f122c503 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
+#include "core/base/extended_float.hpp"
 #include "core/test/utils.hpp"
 
 
@@ -149,7 +150,9 @@ class ReducedStorage3d : public ::testing::Test {
         typename std::tuple_element<0, decltype(ArithmeticStorageType{})>::type;
     using st_type =
         typename std::tuple_element<1, decltype(ArithmeticStorageType{})>::type;
-    static constexpr ar_type delta{::r<st_type>::value * 2};
+    static constexpr ar_type delta{std::is_same<ar_type, st_type>::value
+                                       ? 0
+                                       : ::r<st_type, ar_type>::value};
 
     // Type for `check_accessor_correctness` to forward the indices
     using t = std::tuple<int, int, int>;
@@ -186,43 +189,53 @@ class ReducedStorage3d : public ::testing::Test {
     reduced_storage r{size, data};
     const_reduced_storage cr{size, data};
 
+    // Casts val first to `st_type`, then to `ar_type` in order to be allowed
+    // to test for equality
+    template <typename T>
+    static ar_type c_st_ar(T val)
+    {
+        return static_cast<ar_type>(static_cast<st_type>(val));
+    }
+
     template <typename Accessor>
-    static void check_accessor_correctness(
-        const Accessor &a, std::tuple<int, int, int> ignore = t(99, 99, 99))
+    void check_accessor_correctness(const Accessor &a,
+                                    std::tuple<int, int, int> ignore = t(99, 99,
+                                                                         99))
     {
         // Test for equality is fine here since they should not be modified
         // clang-format off
-        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), st_type{1.0});     }
-        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), st_type{2.01});    }
-        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), st_type{-1.02});   }
-        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), st_type{3.03});    }
-        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), st_type{4.04});    }
-        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), st_type{-2.05});   }
-        if (ignore != t(1, 0, 0)) { EXPECT_EQ(a(1, 0, 0), st_type{5.06});    }
-        if (ignore != t(1, 0, 1)) { EXPECT_EQ(a(1, 0, 1), st_type{6.07});    }
-        if (ignore != t(1, 1, 0)) { EXPECT_EQ(a(1, 1, 0), st_type{2.08});    }
-        if (ignore != t(1, 1, 1)) { EXPECT_EQ(a(1, 1, 1), st_type{3.09});    }
-        if (ignore != t(1, 2, 0)) { EXPECT_EQ(a(1, 2, 0), st_type{-1.1});    }
-        if (ignore != t(1, 2, 1)) { EXPECT_EQ(a(1, 2, 1), st_type{-9.11});   }
-        if (ignore != t(2, 0, 0)) { EXPECT_EQ(a(2, 0, 0), st_type{-2.12});   }
-        if (ignore != t(2, 0, 1)) { EXPECT_EQ(a(2, 0, 1), st_type{2.13});    }
-        if (ignore != t(2, 1, 0)) { EXPECT_EQ(a(2, 1, 0), st_type{0.14});    }
-        if (ignore != t(2, 1, 1)) { EXPECT_EQ(a(2, 1, 1), st_type{15.15});   }
-        if (ignore != t(2, 2, 0)) { EXPECT_EQ(a(2, 2, 0), st_type{-9.16});   }
-        if (ignore != t(2, 2, 1)) { EXPECT_EQ(a(2, 2, 1), st_type{8.17});    }
-        if (ignore != t(3, 0, 0)) { EXPECT_EQ(a(3, 0, 0), st_type{7.18});    }
-        if (ignore != t(3, 0, 1)) { EXPECT_EQ(a(3, 0, 1), st_type{-6.19});   }
-        if (ignore != t(3, 1, 0)) { EXPECT_EQ(a(3, 1, 0), st_type{5.2});     }
-        if (ignore != t(3, 1, 1)) { EXPECT_EQ(a(3, 1, 1), st_type{-4.21});   }
-        if (ignore != t(3, 2, 0)) { EXPECT_EQ(a(3, 2, 0), st_type{3.22});    }
-        if (ignore != t(3, 2, 1)) { EXPECT_EQ(a(3, 2, 1), st_type{-2.23});   }
+        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), c_st_ar(1.0));     }
+        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), c_st_ar(2.01));    }
+        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), c_st_ar(-1.02));   }
+        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), c_st_ar(3.03));    }
+        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), c_st_ar(4.04));    }
+        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), c_st_ar(-2.05));   }
+        if (ignore != t(1, 0, 0)) { EXPECT_EQ(a(1, 0, 0), c_st_ar(5.06));    }
+        if (ignore != t(1, 0, 1)) { EXPECT_EQ(a(1, 0, 1), c_st_ar(6.07));    }
+        if (ignore != t(1, 1, 0)) { EXPECT_EQ(a(1, 1, 0), c_st_ar(2.08));    }
+        if (ignore != t(1, 1, 1)) { EXPECT_EQ(a(1, 1, 1), c_st_ar(3.09));    }
+        if (ignore != t(1, 2, 0)) { EXPECT_EQ(a(1, 2, 0), c_st_ar(-1.1));    }
+        if (ignore != t(1, 2, 1)) { EXPECT_EQ(a(1, 2, 1), c_st_ar(-9.11));   }
+        if (ignore != t(2, 0, 0)) { EXPECT_EQ(a(2, 0, 0), c_st_ar(-2.12));   }
+        if (ignore != t(2, 0, 1)) { EXPECT_EQ(a(2, 0, 1), c_st_ar(2.13));    }
+        if (ignore != t(2, 1, 0)) { EXPECT_EQ(a(2, 1, 0), c_st_ar(0.14));    }
+        if (ignore != t(2, 1, 1)) { EXPECT_EQ(a(2, 1, 1), c_st_ar(15.15));   }
+        if (ignore != t(2, 2, 0)) { EXPECT_EQ(a(2, 2, 0), c_st_ar(-9.16));   }
+        if (ignore != t(2, 2, 1)) { EXPECT_EQ(a(2, 2, 1), c_st_ar(8.17));    }
+        if (ignore != t(3, 0, 0)) { EXPECT_EQ(a(3, 0, 0), c_st_ar(7.18));    }
+        if (ignore != t(3, 0, 1)) { EXPECT_EQ(a(3, 0, 1), c_st_ar(-6.19));   }
+        if (ignore != t(3, 1, 0)) { EXPECT_EQ(a(3, 1, 0), c_st_ar(5.2));     }
+        if (ignore != t(3, 1, 1)) { EXPECT_EQ(a(3, 1, 1), c_st_ar(-4.21));   }
+        if (ignore != t(3, 2, 0)) { EXPECT_EQ(a(3, 2, 0), c_st_ar(3.22));    }
+        if (ignore != t(3, 2, 1)) { EXPECT_EQ(a(3, 2, 1), c_st_ar(-2.23));   }
         // clang-format on
     }
 };
 
 using ReducedStorage3dTypes =
     ::testing::Types<std::tuple<double, double>, std::tuple<double, float>,
-                     std::tuple<float, float>
+                     std::tuple<float, float>, std::tuple<double, gko::half>,
+                     std::tuple<float, gko::half>
                      /*,
                      std::tuple<std::complex<double>, std::complex<double>>,
                      std::tuple<std::complex<double>, std::complex<float>>,
@@ -308,21 +321,20 @@ TYPED_TEST(ReducedStorage3d, CanCreateWithStride)
 {
     using reduced_storage = typename TestFixture::reduced_storage;
     using ar_type = typename TestFixture::ar_type;
-    using st_type = typename TestFixture::st_type;
     auto size = gko::dim<3>{2, 2, 2};
     auto stride = std::array<gko::size_type, 2>{{12, 2}};
 
     auto range = reduced_storage{size, this->data, stride};
     range(1, 1, 0) = ar_type{2.};
 
-    EXPECT_EQ(range(0, 0, 0), st_type{1.0});
-    EXPECT_EQ(range(0, 0, 1), st_type{2.01});
-    EXPECT_EQ(range(0, 1, 0), st_type{-1.02});
-    EXPECT_EQ(range(0, 1, 1), st_type{3.03});
-    EXPECT_EQ(range(1, 0, 0), st_type{-2.12});
-    EXPECT_EQ(range(1, 0, 1), st_type{2.13});
-    EXPECT_EQ(range(1, 1, 0), st_type{2.});
-    EXPECT_EQ(range(1, 1, 1), st_type{15.15});
+    EXPECT_EQ(range(0, 0, 0), this->c_st_ar(1.0));
+    EXPECT_EQ(range(0, 0, 1), this->c_st_ar(2.01));
+    EXPECT_EQ(range(0, 1, 0), this->c_st_ar(-1.02));
+    EXPECT_EQ(range(0, 1, 1), this->c_st_ar(3.03));
+    EXPECT_EQ(range(1, 0, 0), this->c_st_ar(-2.12));
+    EXPECT_EQ(range(1, 0, 1), this->c_st_ar(2.13));
+    EXPECT_EQ(range(1, 1, 0), this->c_st_ar(2.));
+    EXPECT_EQ(range(1, 1, 1), this->c_st_ar(15.15));
 }
 
 
@@ -340,22 +352,26 @@ TYPED_TEST(ReducedStorage3d, CanWriteData)
 TYPED_TEST(ReducedStorage3d, Assignment)
 {
     using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = 1.2;
 
-    this->r(0, 0, 1) = 10.2;
+    this->r(0, 0, 1) = expected;
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), 10.2, TestFixture::delta);
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
 }
 
 
 TYPED_TEST(ReducedStorage3d, Assignment2)
 {
     using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = -1.02;
 
     this->r(0, 0, 1) = this->r(0, 1, 0);
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), -1.02, TestFixture::delta);
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
 }
 
 
@@ -363,10 +379,10 @@ TYPED_TEST(ReducedStorage3d, Addition)
 {
     using t = typename TestFixture::t;
     using ar_type = typename TestFixture::ar_type;
-    const ar_type expected = 10.2 + 2.01;
+    const ar_type expected = 1.2 + 2.01;
 
-    auto result = this->r(0, 0, 1) + ar_type{10.2};
-    this->r(0, 0, 1) += 10.2;
+    auto result = this->r(0, 0, 1) + ar_type{1.2};
+    this->r(0, 0, 1) += 1.2;
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
     EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
@@ -495,28 +511,24 @@ TYPED_TEST(ReducedStorage3d, UnaryMinus)
 
 TYPED_TEST(ReducedStorage3d, CanCreateSubrange)
 {
-    using st_type = typename TestFixture::st_type;
-
     auto subr = this->r(gko::span{1u, 3u}, gko::span{0u, 2u}, 0u);
 
-    EXPECT_EQ(subr(0, 0, 0), st_type{5.06});
-    EXPECT_EQ(subr(0, 1, 0), st_type{2.08});
-    EXPECT_EQ(subr(1, 0, 0), st_type{-2.12});
-    EXPECT_EQ(subr(1, 1, 0), st_type{0.14});
+    EXPECT_EQ(subr(0, 0, 0), this->c_st_ar(5.06));
+    EXPECT_EQ(subr(0, 1, 0), this->c_st_ar(2.08));
+    EXPECT_EQ(subr(1, 0, 0), this->c_st_ar(-2.12));
+    EXPECT_EQ(subr(1, 1, 0), this->c_st_ar(0.14));
 }
 
 
 TYPED_TEST(ReducedStorage3d, CanCreateSubrange2)
 {
-    using st_type = typename TestFixture::st_type;
-
     auto subr =
         this->cr(gko::span{1u, 3u}, gko::span{0u, 2u}, gko::span{0u, 1u});
 
-    EXPECT_EQ(subr(0, 0, 0), st_type{5.06});
-    EXPECT_EQ(subr(0, 1, 0), st_type{2.08});
-    EXPECT_EQ(subr(1, 0, 0), st_type{-2.12});
-    EXPECT_EQ(subr(1, 1, 0), st_type{0.14});
+    EXPECT_EQ(subr(0, 0, 0), this->c_st_ar(5.06));
+    EXPECT_EQ(subr(0, 1, 0), this->c_st_ar(2.08));
+    EXPECT_EQ(subr(1, 0, 0), this->c_st_ar(-2.12));
+    EXPECT_EQ(subr(1, 1, 0), this->c_st_ar(0.14));
 }
 
 
@@ -558,17 +570,23 @@ class ReducedStorageXd : public ::testing::Test {
     const_reduced_storage1d cr1{size_1d, data, stride0};
     const_reduced_storage2d cr2{size_2d, data, stride1};
 
+    template <typename T>
+    static ar_type c_st_ar(T val)
+    {
+        return static_cast<ar_type>(static_cast<st_type>(val));
+    }
+
     void data_equal_except_for(int idx)
     {
         // clang-format off
-        if (idx != 0) { EXPECT_EQ(data[0], 1.1f); }
-        if (idx != 1) { EXPECT_EQ(data[1], 2.2f); }
-        if (idx != 2) { EXPECT_EQ(data[2], 3.3f); }
-        if (idx != 3) { EXPECT_EQ(data[3], 4.4f); }
-        if (idx != 4) { EXPECT_EQ(data[4], 5.5f); }
-        if (idx != 5) { EXPECT_EQ(data[5], 6.6f); }
-        if (idx != 6) { EXPECT_EQ(data[6], 7.7f); }
-        if (idx != 7) { EXPECT_EQ(data[7], -8.8f); }
+        if (idx != 0) { EXPECT_EQ(data[0], c_st_ar(1.1)); }
+        if (idx != 1) { EXPECT_EQ(data[1], c_st_ar(2.2)); }
+        if (idx != 2) { EXPECT_EQ(data[2], c_st_ar(3.3)); }
+        if (idx != 3) { EXPECT_EQ(data[3], c_st_ar(4.4)); }
+        if (idx != 4) { EXPECT_EQ(data[4], c_st_ar(5.5)); }
+        if (idx != 5) { EXPECT_EQ(data[5], c_st_ar(6.6)); }
+        if (idx != 6) { EXPECT_EQ(data[6], c_st_ar(7.7)); }
+        if (idx != 7) { EXPECT_EQ(data[7], c_st_ar(-8.8)); }
         // clang-format on
     }
 };
@@ -576,10 +594,10 @@ class ReducedStorageXd : public ::testing::Test {
 
 TEST_F(ReducedStorageXd, CanRead)
 {
-    EXPECT_EQ(cr1(1), 2.2f);
-    EXPECT_EQ(cr2(0, 1), 2.2f);
-    EXPECT_EQ(r1(1), 2.2f);
-    EXPECT_EQ(r2(0, 1), 2.2f);
+    EXPECT_EQ(cr1(1), this->c_st_ar(2.2));
+    EXPECT_EQ(cr2(0, 1), this->c_st_ar(2.2));
+    EXPECT_EQ(r1(1), this->c_st_ar(2.2));
+    EXPECT_EQ(r2(0, 1), this->c_st_ar(2.2));
 }
 
 
@@ -776,16 +794,18 @@ TYPED_TEST(ScaledReducedStorage3d, Subrange)
 
 TYPED_TEST(ScaledReducedStorage3d, CanWriteScale)
 {
+    using ar_type = typename TestFixture::ar_type;
+
     this->r->write_scalar(10., 0, 0, 0);
 
-    EXPECT_EQ(this->r(0, 0, 0), 100.);
-    EXPECT_EQ(this->r(0, 0, 1), 22.);
-    EXPECT_EQ(this->r(0, 1, 0), -120.);
-    EXPECT_EQ(this->r(0, 1, 1), 26.);
-    EXPECT_EQ(this->r(0, 2, 0), 140.);
-    EXPECT_EQ(this->r(0, 2, 1), -230.);
-    EXPECT_EQ(this->r(0, 3, 0), 60.);
-    EXPECT_EQ(this->r(0, 3, 1), 154.);
+    EXPECT_EQ(this->r(0, 0, 0), ar_type{100.});
+    EXPECT_EQ(this->r(0, 0, 1), ar_type{22.});
+    EXPECT_EQ(this->r(0, 1, 0), ar_type{-120.});
+    EXPECT_EQ(this->r(0, 1, 1), ar_type{26.});
+    EXPECT_EQ(this->r(0, 2, 0), ar_type{140.});
+    EXPECT_EQ(this->r(0, 2, 1), ar_type{-230.});
+    EXPECT_EQ(this->r(0, 3, 0), ar_type{60.});
+    EXPECT_EQ(this->r(0, 3, 1), ar_type{154.});
 }
 
 
diff --git a/core/test/utils.hpp b/core/test/utils.hpp
index c4af1f9a7ba..0b314a59463 100644
--- a/core/test/utils.hpp
+++ b/core/test/utils.hpp
@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
+#include "core/base/extended_float.hpp"
 #include "core/test/utils/assertions.hpp"
 #include "core/test/utils/matrix_generator.hpp"
 
@@ -92,23 +93,28 @@ using ComplexValueIndexTypes =
                      std::tuple<std::complex<double>, gko::int64>>;
 
 
-template <typename T>
+template <typename Precision, typename OutputType>
 struct reduction_factor {
-    static constexpr gko::remove_complex<T> value =
-        std::is_same<gko::remove_complex<T>, float>::value ? 1.0e-7 : 1.0e-14;
+    static constexpr remove_complex<OutputType> value =
+        std::is_same<gko::remove_complex<Precision>, gko::half>::value
+            ? 1.0e-2
+            : std::is_same<gko::remove_complex<Precision>, float>::value
+                  ? 1.0e-6
+                  : 1.0e-14;
 };
 
 
-template <typename T>
-constexpr gko::remove_complex<T> reduction_factor<T>::value;
+template <typename Precision, typename OutputType>
+constexpr remove_complex<OutputType>
+    reduction_factor<Precision, OutputType>::value;
 
 
 }  // namespace test
 }  // namespace gko
 
 
-template <typename T>
-using r = typename gko::test::reduction_factor<T>;
+template <typename Precision, typename OutputType = Precision>
+using r = typename gko::test::reduction_factor<Precision, OutputType>;
 
 
 template <typename T>
diff --git a/include/ginkgo/core/base/range_accessor_helper.hpp b/include/ginkgo/core/base/range_accessor_helper.hpp
index 094638cbe8f..030203cd962 100644
--- a/include/ginkgo/core/base/range_accessor_helper.hpp
+++ b/include/ginkgo/core/base/range_accessor_helper.hpp
@@ -96,7 +96,7 @@ struct row_major_helper_s<ValueType, total_dim, total_dim> {
     static constexpr GKO_ATTRIBUTES ValueType
     compute(const dim<total_dim> &size,
             const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>,
-            FirstType &&first)
+            FirstType first)
     {
         return GKO_ASSERT(first < size[total_dim - 1]), first;
     }
diff --git a/include/ginkgo/core/base/range_accessor_references.hpp b/include/ginkgo/core/base/range_accessor_references.hpp
index 9f41351f4a8..324a24c17ae 100644
--- a/include/ginkgo/core/base/range_accessor_references.hpp
+++ b/include/ginkgo/core/base/range_accessor_references.hpp
@@ -126,25 +126,22 @@ template <typename Reference, typename ArithmeticType>
 struct enable_reference_operators {
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
 
-#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_op)                  \
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES  \
-        arithmetic_type                                              \
-        operator _op(const Reference &ref1, const Reference &ref2)   \
-    {                                                                \
-        return to_value_type<arithmetic_type>(ref1)                  \
-            _op to_value_type<arithmetic_type>(ref2);                \
-    }                                                                \
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES  \
-        arithmetic_type                                              \
-        operator _op(const Reference &ref, const arithmetic_type &a) \
-    {                                                                \
-        return to_value_type<arithmetic_type>(ref) _op a;            \
-    }                                                                \
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES  \
-        arithmetic_type                                              \
-        operator _op(const arithmetic_type &a, const Reference &ref) \
-    {                                                                \
-        return a _op to_value_type<arithmetic_type>(ref);            \
+#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_op)                      \
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES auto \
+    operator _op(const Reference &ref1, const Reference &ref2)           \
+    {                                                                    \
+        return to_value_type<arithmetic_type>(ref1)                      \
+            _op to_value_type<arithmetic_type>(ref2);                    \
+    }                                                                    \
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES auto \
+    operator _op(const Reference &ref, const arithmetic_type &a)         \
+    {                                                                    \
+        return to_value_type<arithmetic_type>(ref) _op a;                \
+    }                                                                    \
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES auto \
+    operator _op(const arithmetic_type &a, const Reference &ref)         \
+    {                                                                    \
+        return a _op to_value_type<arithmetic_type>(ref);                \
     }
     GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(*)
     GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(/)

From dfd271c3659a68b00230804e09829db287767ae4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 19 Nov 2020 04:06:01 +0100
Subject: [PATCH 0231/1008] Specify return type of reference operators again

---
 .../core/base/range_accessor_references.hpp   | 38 ++++++++++---------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/include/ginkgo/core/base/range_accessor_references.hpp b/include/ginkgo/core/base/range_accessor_references.hpp
index 324a24c17ae..3b23c742e35 100644
--- a/include/ginkgo/core/base/range_accessor_references.hpp
+++ b/include/ginkgo/core/base/range_accessor_references.hpp
@@ -126,23 +126,27 @@ template <typename Reference, typename ArithmeticType>
 struct enable_reference_operators {
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
 
-#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_op)                      \
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES auto \
-    operator _op(const Reference &ref1, const Reference &ref2)           \
-    {                                                                    \
-        return to_value_type<arithmetic_type>(ref1)                      \
-            _op to_value_type<arithmetic_type>(ref2);                    \
-    }                                                                    \
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES auto \
-    operator _op(const Reference &ref, const arithmetic_type &a)         \
-    {                                                                    \
-        return to_value_type<arithmetic_type>(ref) _op a;                \
-    }                                                                    \
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES auto \
-    operator _op(const arithmetic_type &a, const Reference &ref)         \
-    {                                                                    \
-        return a _op to_value_type<arithmetic_type>(ref);                \
+#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_op)                  \
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES  \
+        arithmetic_type                                              \
+        operator _op(const Reference &ref1, const Reference &ref2)   \
+    {                                                                \
+        return to_value_type<arithmetic_type>(ref1)                  \
+            _op to_value_type<arithmetic_type>(ref2);                \
+    }                                                                \
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES  \
+        arithmetic_type                                              \
+        operator _op(const Reference &ref, const arithmetic_type &a) \
+    {                                                                \
+        return to_value_type<arithmetic_type>(ref) _op a;            \
+    }                                                                \
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES  \
+        arithmetic_type                                              \
+        operator _op(const arithmetic_type &a, const Reference &ref) \
+    {                                                                \
+        return a _op to_value_type<arithmetic_type>(ref);            \
     }
+
     GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(*)
     GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(/)
     GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(+)
@@ -170,8 +174,6 @@ struct enable_reference_operators {
     GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator-=, -)
 #undef GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD
 
-    // TODO Add overloads for comparison operators if needed
-
     friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES
         arithmetic_type
         operator-(const Reference &ref)

From 113150c909cdedff94928e014574bc802be45a80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 19 Nov 2020 17:36:02 +0100
Subject: [PATCH 0232/1008] Move new accessor to private headers

---
 .../base/accessor_helper.hpp                  |  23 +-
 .../base/accessor_references.hpp              |  27 +-
 core/base/accessors.hpp                       | 698 +++++++++++++
 core/test/base/CMakeLists.txt                 |   1 +
 core/test/base/accessors_reduced_storage.cpp  | 965 ++++++++++++++++++
 core/test/base/range_accessors.cpp            | 919 -----------------
 include/ginkgo/core/base/range_accessors.hpp  | 642 ------------
 include/ginkgo/ginkgo.hpp                     |   2 -
 8 files changed, 1684 insertions(+), 1593 deletions(-)
 rename include/ginkgo/core/base/range_accessor_helper.hpp => core/base/accessor_helper.hpp (98%)
 rename include/ginkgo/core/base/range_accessor_references.hpp => core/base/accessor_references.hpp (96%)
 create mode 100644 core/base/accessors.hpp
 create mode 100644 core/test/base/accessors_reduced_storage.cpp

diff --git a/include/ginkgo/core/base/range_accessor_helper.hpp b/core/base/accessor_helper.hpp
similarity index 98%
rename from include/ginkgo/core/base/range_accessor_helper.hpp
rename to core/base/accessor_helper.hpp
index 030203cd962..d837812700c 100644
--- a/include/ginkgo/core/base/range_accessor_helper.hpp
+++ b/core/base/accessor_helper.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_RANGE_ACCESSOR_HELPER_HPP_
-#define GKO_CORE_BASE_RANGE_ACCESSOR_HELPER_HPP_
+#ifndef GKO_CORE_BASE_ACCESSOR_HELPER_HPP_
+#define GKO_CORE_BASE_ACCESSOR_HELPER_HPP_
 
 #include <array>
 #include <tuple>
@@ -53,8 +53,6 @@ namespace accessor {
  * without notice.
  */
 namespace helper {
-
-
 namespace detail {
 
 
@@ -193,8 +191,7 @@ template <typename ValueType, size_type mask, size_type set_bits_processed,
                               ? static_cast<bool>(mask &(
                                     size_type{1} << (total_dim - 1 - dim_idx)))
                               : false>
-struct row_major_masked_helper_s {
-};
+struct row_major_masked_helper_s {};
 
 
 // bit for current dimensionality is set
@@ -415,15 +412,13 @@ constexpr GKO_ATTRIBUTES auto compute_default_masked_stride_array(
  * Evaluates if all Args fulfill std::is_integral
  */
 template <typename... Args>
-struct are_all_integral : public std::true_type {
-};
+struct are_all_integral : public std::true_type {};
 
 template <typename First, typename... Args>
 struct are_all_integral<First, Args...>
     : public std::conditional<std::is_integral<std::decay_t<First>>::value,
                               are_all_integral<Args...>,
-                              std::false_type>::type {
-};
+                              std::false_type>::type {};
 
 
 namespace detail {
@@ -431,8 +426,7 @@ namespace detail {
 
 template <bool has_span, typename... Args>
 struct are_span_compatible_impl
-    : public std::integral_constant<bool, has_span> {
-};
+    : public std::integral_constant<bool, has_span> {};
 
 template <bool has_span, typename First, typename... Args>
 struct are_span_compatible_impl<has_span, First, Args...>
@@ -442,8 +436,7 @@ struct are_span_compatible_impl<has_span, First, Args...>
           are_span_compatible_impl<
               has_span || std::is_same<std::decay_t<First>, span>::value,
               Args...>,
-          std::false_type>::type {
-};
+          std::false_type>::type {};
 
 
 }  // namespace detail
@@ -563,4 +556,4 @@ constexpr size_type count_mask_dimensionality()
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_RANGE_ACCESSOR_HELPER_HPP_
+#endif  // GKO_CORE_BASE_ACCESSOR_HELPER_HPP_
diff --git a/include/ginkgo/core/base/range_accessor_references.hpp b/core/base/accessor_references.hpp
similarity index 96%
rename from include/ginkgo/core/base/range_accessor_references.hpp
rename to core/base/accessor_references.hpp
index 3b23c742e35..00825a5e618 100644
--- a/include/ginkgo/core/base/range_accessor_references.hpp
+++ b/core/base/accessor_references.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_RANGE_ACCESSOR_REFERENCES_HPP_
-#define GKO_CORE_BASE_RANGE_ACCESSOR_REFERENCES_HPP_
+#ifndef GKO_CORE_BASE_ACCESSOR_REFERENCES_HPP_
+#define GKO_CORE_BASE_ACCESSOR_REFERENCES_HPP_
 
 
 #include <type_traits>
@@ -67,15 +67,13 @@ namespace detail {
 
 // tests if the cast operator to `ValueType` is present
 template <typename Ref, typename ValueType, typename = xstd::void_t<>>
-struct has_cast_operator : std::false_type {
-};
+struct has_cast_operator : std::false_type {};
 
 template <typename Ref, typename ValueType>
 struct has_cast_operator<
     Ref, ValueType,
     xstd::void_t<decltype(std::declval<Ref>().Ref::operator ValueType())>>
-    : std::true_type {
-};
+    : std::true_type {};
 
 
 /**
@@ -243,8 +241,8 @@ class reduced_storage
         return static_cast<arithmetic_type>(*r_ptr);
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type operator=(arithmetic_type val) &&
-        noexcept
+    constexpr GKO_ATTRIBUTES arithmetic_type
+    operator=(arithmetic_type val) &&noexcept
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         *r_ptr = static_cast<storage_type>(val);
@@ -258,8 +256,8 @@ class reduced_storage
         return static_cast<arithmetic_type>(*this);
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type operator=(reduced_storage &&ref) &&
-        noexcept
+    constexpr GKO_ATTRIBUTES arithmetic_type
+    operator=(reduced_storage &&ref) &&noexcept
     {
         std::move(*this) = static_cast<arithmetic_type>(ref);
         return static_cast<arithmetic_type>(*this);
@@ -354,8 +352,8 @@ class scaled_reduced_storage
         return static_cast<arithmetic_type>(*r_ptr) * scalar_;
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type operator=(arithmetic_type val) &&
-        noexcept
+    constexpr GKO_ATTRIBUTES arithmetic_type
+    operator=(arithmetic_type val) &&noexcept
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         *r_ptr = static_cast<storage_type>(val / scalar_);
@@ -370,8 +368,7 @@ class scaled_reduced_storage
     }
 
     constexpr GKO_ATTRIBUTES arithmetic_type
-        operator=(scaled_reduced_storage &&ref) &&
-        noexcept
+    operator=(scaled_reduced_storage &&ref) &&noexcept
     {
         std::move(*this) = static_cast<arithmetic_type>(ref);
         return static_cast<arithmetic_type>(*this);
@@ -426,4 +423,4 @@ class scaled_reduced_storage<ArithmeticType, const StorageType>
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_RANGE_ACCESSOR_REFERENCES_HPP_
+#endif  // GKO_CORE_BASE_ACCESSOR_REFERENCES_HPP_
diff --git a/core/base/accessors.hpp b/core/base/accessors.hpp
new file mode 100644
index 00000000000..808d210fa45
--- /dev/null
+++ b/core/base/accessors.hpp
@@ -0,0 +1,698 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_BASE_ACCESSORS_HPP_
+#define GKO_CORE_BASE_ACCESSORS_HPP_
+
+#include <array>
+#include <memory>
+#include <type_traits>
+#include <utility>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/range.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+#include "core/base/accessor_helper.hpp"
+#include "core/base/accessor_references.hpp"
+
+
+namespace gko {
+/**
+ * @brief The accessor namespace.
+ *
+ * @ingroup accessor
+ */
+namespace accessor {
+
+
+/**
+ * The reduced_row_major class allows a storage format that is different from
+ * the arithmetic format (which is returned from the brace operator).
+ * As storage, the StorageType is used.
+ *
+ * This accessor uses row-major access. For example for three dimensions,
+ * neighboring z coordinates are next to each other in memory, followed by y
+ * coordinates and then x coordinates.
+ *
+ * @tparam Dimensionality  The number of dimensions managed by this accessor
+ *
+ * @tparam ArithmeticType  Value type used for arithmetic operations and
+ *                         for in- and output
+ *
+ * @tparam StorageType  Value type used for storing the actual value to memory
+ *
+ * @note  This class only manages the accesses and not the memory itself.
+ */
+template <int Dimensionality, typename ArithmeticType, typename StorageType>
+class reduced_row_major {
+public:
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
+    using storage_type = StorageType;
+    static constexpr size_type dimensionality{Dimensionality};
+    static constexpr bool is_const{std::is_const<storage_type>::value};
+    using const_accessor =
+        reduced_row_major<dimensionality, arithmetic_type, const storage_type>;
+
+    static_assert(Dimensionality >= 1,
+                  "Dimensionality must be a positive number!");
+
+    friend class range<reduced_row_major>;
+
+protected:
+    using storage_stride_type = std::array<size_type, dimensionality - 1>;
+    using reference_type =
+        reference_class::reduced_storage<arithmetic_type, storage_type>;
+
+    /**
+     * Creates the accessor for an already allocated storage space with a
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
+     *
+     * @param size  multidimensional size of the memory
+     * @param storage  pointer to the block of memory containing the storage
+     * @param stride  stride array used for memory accesses
+     */
+    constexpr GKO_ATTRIBUTES reduced_row_major(gko::dim<dimensionality> size,
+                                               storage_type *storage,
+                                               storage_stride_type stride)
+        : size_{size}, storage_{storage}, stride_(stride)
+    {}
+
+    /**
+     * Creates the accessor for an already allocated storage space with a
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
+     *
+     * @param storage  pointer to the block of memory containing the storage
+     * @param size  multidimensional size of the memory
+     * @param strides  strides used for memory accesses
+     */
+    template <typename... Strides>
+    constexpr GKO_ATTRIBUTES reduced_row_major(gko::dim<dimensionality> size,
+                                               storage_type *storage,
+                                               Strides &&... strides)
+        : reduced_row_major{
+              size, storage,
+              storage_stride_type{{std::forward<Strides>(strides)...}}}
+    {
+        static_assert(sizeof...(Strides) + 1 == dimensionality,
+                      "Number of provided Strides must be dimensionality - 1!");
+    }
+
+    /**
+     * Creates the accessor for an already allocated storage space.
+     * It is assumed that all accesses are without padding.
+     *
+     * @param storage  pointer to the block of memory containing the storage
+     * @param size  multidimensional size of the memory
+     */
+    constexpr GKO_ATTRIBUTES reduced_row_major(dim<dimensionality> size,
+                                               storage_type *storage)
+        : reduced_row_major{
+              size, storage,
+              helper::compute_default_stride_array<size_type>(size)}
+    {}
+
+    /**
+     * Creates an empty accessor (pointing nowhere with an empty size)
+     */
+    constexpr GKO_ATTRIBUTES reduced_row_major()
+        : reduced_row_major{{0, 0, 0}, nullptr}
+    {}
+
+public:
+    /**
+     * Creates a reduced_row_major range which contains a read-only version of
+     * the current accessor.
+     *
+     * @returns  a reduced_row_major major range which is read-only.
+     */
+    constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
+    {
+        return range<const_accessor>{size_, storage_, stride_};
+    }
+
+    /**
+     * Returns the length in dimension `dimension`.
+     *
+     * @param dimension  a dimension index
+     *
+     * @returns  length in dimension `dimension`
+     */
+    constexpr GKO_ATTRIBUTES size_type length(size_type dimension) const
+    {
+        return dimension < dimensionality ? size_[dimension] : 1;
+    }
+
+    /**
+     * Copies data from another accessor
+     *
+     * @warning Do not use this function since it is not optimized for a
+     *          specific executor. It will always be performed sequentially.
+     *          Please write an optimized version (adjusted to the architecture)
+     *          by iterating through the values yourself.
+     *
+     * @tparam OtherAccessor  type of the other accessor
+     *
+     * @param other  other accessor
+     */
+    template <typename OtherAccessor>
+    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
+    {
+        helper::multidim_for_each(size_, [this, &other](auto... indices) {
+            (*this)(indices...) = other(indices...);
+        });
+    }
+
+    /**
+     * Returns the stored value for the given indices. If the storage is const,
+     * a value is returned, otherwise, a reference is returned.
+     *
+     * @param indices  indices which value is supposed to access
+     *
+     * @returns  the stored value if the accessor is const (if the storage type
+     *           is const), or a reference if the accessor is non-const
+     */
+    template <typename... Indices>
+    constexpr GKO_ATTRIBUTES std::enable_if_t<
+        helper::are_all_integral<Indices...>::value,
+        std::conditional_t<is_const, arithmetic_type, reference_type>>
+    operator()(Indices &&... indices) const
+    {
+        return reference_type{storage_ +
+                              compute_index(std::forward<Indices>(indices)...)};
+    }
+
+    /**
+     * Returns a sub-range spanning the current range (x1_span, x2_span, ...)
+     *
+     * @param spans  span for the indices
+     *
+     * @returns a sub-range for the given spans.
+     */
+    template <typename... SpanTypes>
+    constexpr GKO_ATTRIBUTES
+        std::enable_if_t<helper::are_span_compatible<SpanTypes...>::value,
+                         range<reduced_row_major>>
+        operator()(SpanTypes... spans) const
+    {
+        return helper::validate_spans(size_, spans...),
+               range<reduced_row_major>{
+                   dim<dimensionality>{
+                       (span{spans}.end - span{spans}.begin)...},
+                   storage_ + compute_index((span{spans}.begin)...), stride_};
+    }
+
+    /**
+     * Returns the size of the accessor
+     *
+     * @returns the size of the accessor
+     */
+    constexpr GKO_ATTRIBUTES dim<dimensionality> get_size() const
+    {
+        return size_;
+    }
+
+    /**
+     * Returns a pointer to a stride array of size dimensionality - 1
+     *
+     * @returns returns a pointer to a stride array of size dimensionality - 1
+     */
+    GKO_ATTRIBUTES
+    constexpr const storage_stride_type &get_stride() const { return stride_; }
+
+    /**
+     * Returns the pointer to the storage data
+     *
+     * @returns the pointer to the storage data
+     */
+    constexpr GKO_ATTRIBUTES storage_type *get_stored_data() const
+    {
+        return storage_;
+    }
+
+    /**
+     * Returns a const pointer to the storage data
+     *
+     * @returns a const pointer to the storage data
+     */
+    constexpr GKO_ATTRIBUTES const storage_type *get_const_storage() const
+    {
+        return storage_;
+    }
+
+protected:
+    template <typename... Indices>
+    constexpr GKO_ATTRIBUTES size_type
+    compute_index(Indices &&... indices) const
+    {
+        static_assert(sizeof...(Indices) == dimensionality,
+                      "Number of indices must match dimensionality!");
+        return helper::compute_storage_index<size_type, dimensionality>(
+            size_, stride_, std::forward<Indices>(indices)...);
+    }
+
+private:
+    const dim<dimensionality> size_;
+    storage_type *storage_;
+    const storage_stride_type stride_;
+};
+
+
+namespace detail {
+
+
+// In case of a const type, do not provide a write function
+template <int Dimensionality, typename Accessor, typename ScaleType,
+          bool = std::is_const<ScaleType>::value>
+struct enable_write_scalar {
+    static_assert(std::is_const<ScaleType>::value,
+                  "This class must have a constant ScaleType!");
+    using scalar_type = ScaleType;
+};
+
+// In case of a non-const type, enable the write function
+template <int Dimensionality, typename Accessor, typename ScaleType>
+struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
+    static_assert(!std::is_const<ScaleType>::value,
+                  "This class must NOT have a constant ScaleType!");
+    static_assert(Dimensionality >= 1,
+                  "Dimensionality must be a positive number!");
+
+    using scalar_type = ScaleType;
+
+    /**
+     * Writes the scalar value at the given indices.
+     *
+     * @param value  value to write
+     * @param indices  indices where to write the value
+     *
+     * @returns the written value.
+     */
+    template <typename... Indices>
+    constexpr GKO_ATTRIBUTES scalar_type
+    write_scalar(scalar_type value, Indices &&... indices) const
+    {
+        static_assert(sizeof...(Indices) == Dimensionality,
+                      "Number of indices must match dimensionality!");
+        scalar_type *GKO_RESTRICT rest_scalar = self()->scalar_;
+        return rest_scalar[self()->compute_scalar_index(
+                   std::forward<Indices>(indices)...)] = value;
+    }
+
+private:
+    constexpr GKO_ATTRIBUTES const Accessor *self() const
+    {
+        return static_cast<const Accessor *>(this);
+    }
+};
+
+
+}  // namespace detail
+
+
+/**
+ * The scaled_reduced_row_major class allows a storage format that is different
+ * from the arithmetic format (which is returned from the brace operator).
+ * Additionally, a scalar is used when reading and writing data to allow for
+ * a shift in range.
+ * As storage, the StorageType is used.
+ *
+ * This accessor uses row-major access. For example, for three dimensions,
+ * neighboring z coordinates are next to each other in memory, followed by y
+ * coordinates and then x coordinates.
+ *
+ * @tparam Dimensionality  The number of dimensions managed by this accessor
+ *
+ * @tparam ArithmeticType  Value type used for arithmetic operations and
+ *                         for in- and output
+ *
+ * @tparam StorageType  Value type used for storing the actual value to memory
+ *
+ * @tparam ScalarMask  Binary mask that marks which indices matter for the
+ *                     scalar selection (set bit means the corresponding index
+ *                     needs to be considered, 0 means it is not). The least
+ *                     significand bit corresponds to the last index dimension,
+ *                     the second least to the second last index dimension, and
+ *                     so on.
+ *                     For example, the mask = 0b011101 means that for the 5d
+ *                     indices (x1, x2, x3, x4, x5), (x1, x2, x3, x5) are
+ *                     considered for the scalar, making the scalar itself 4d.
+ *
+ * @note  This class only manages the accesses and not the memory itself.
+ */
+template <int Dimensionality, typename ArithmeticType, typename StorageType,
+          size_type ScalarMask>
+class scaled_reduced_row_major
+    : public detail::enable_write_scalar<
+          Dimensionality,
+          scaled_reduced_row_major<Dimensionality, ArithmeticType, StorageType,
+                                   ScalarMask>,
+          ArithmeticType> {
+public:
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
+    using storage_type = StorageType;
+    static constexpr size_type dimensionality{Dimensionality};
+    static constexpr size_type scalar_mask{ScalarMask};
+    static constexpr bool is_const{std::is_const<storage_type>::value};
+    using scalar_type =
+        std::conditional_t<is_const, const arithmetic_type, arithmetic_type>;
+
+    using const_accessor =
+        scaled_reduced_row_major<dimensionality, arithmetic_type,
+                                 const storage_type, ScalarMask>;
+
+    static_assert(Dimensionality >= 1,
+                  "Dimensionality must be a positive number!");
+    static_assert(dimensionality <= 32,
+                  "Only Dimensionality <= 32 is currently supported");
+
+    // Allow access to both `scalar_` and `compute_scalar_index()`
+    friend class detail::enable_write_scalar<
+        dimensionality, scaled_reduced_row_major, scalar_type>;
+    friend class range<scaled_reduced_row_major>;
+
+protected:
+    static constexpr size_type scalar_dim{
+        helper::count_mask_dimensionality<scalar_mask, dimensionality>()};
+    static constexpr size_type scalar_stride_dim{
+        scalar_dim == 0 ? 0 : (scalar_dim - 1)};
+
+    using storage_stride_type = std::array<size_type, dimensionality - 1>;
+    using scalar_stride_type = std::array<size_type, scalar_stride_dim>;
+    using reference_type =
+        reference_class::scaled_reduced_storage<arithmetic_type, StorageType>;
+
+    /**
+     * Creates the accessor for an already allocated storage space with a
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
+     *
+     * @param size  multidimensional size of the memory
+     * @param storage  pointer to the block of memory containing the storage
+     * @param storage_stride  stride array used for memory accesses to storage
+     * @param scalar  pointer to the block of memory containing the scalar
+     *                values.
+     * @param scalar_stride  stride array used for memory accesses to scalar
+     */
+    constexpr GKO_ATTRIBUTES scaled_reduced_row_major(
+        dim<dimensionality> size, storage_type *storage,
+        storage_stride_type storage_stride, scalar_type *scalar,
+        scalar_stride_type scalar_stride)
+        : size_{size},
+          storage_{storage},
+          storage_stride_(storage_stride),
+          scalar_{scalar},
+          scalar_stride_(scalar_stride)
+    {}
+
+    /**
+     * Creates the accessor for an already allocated storage space with a
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
+     *
+     * @param size  multidimensional size of the memory
+     * @param storage  pointer to the block of memory containing the storage
+     * @param stride  stride array used for memory accesses to storage
+     * @param scalar  pointer to the block of memory containing the scalar
+     *                values.
+     */
+    constexpr GKO_ATTRIBUTES scaled_reduced_row_major(
+        dim<dimensionality> size, storage_type *storage,
+        storage_stride_type stride, scalar_type *scalar)
+        : scaled_reduced_row_major{
+              size, storage, stride, scalar,
+              helper::compute_default_masked_stride_array<
+                  size_type, scalar_mask, scalar_stride_dim, dimensionality>(
+                  size)}
+    {}
+
+    /**
+     * Creates the accessor for an already allocated storage space.
+     * It is assumed that all accesses are without padding.
+     *
+     * @param size  multidimensional size of the memory
+     * @param storage  pointer to the block of memory containing the storage
+     * @param scalar  pointer to the block of memory containing the scalar
+     *                values.
+     */
+    constexpr GKO_ATTRIBUTES scaled_reduced_row_major(dim<dimensionality> size,
+                                                      storage_type *storage,
+                                                      scalar_type *scalar)
+        : scaled_reduced_row_major{
+              size, storage,
+              helper::compute_default_stride_array<size_type>(size), scalar}
+    {}
+
+    /**
+     * Creates an empty accessor (pointing nowhere with an empty size)
+     */
+    constexpr GKO_ATTRIBUTES scaled_reduced_row_major()
+        : scaled_reduced_row_major{{0, 0, 0}, nullptr, nullptr}
+    {}
+
+public:
+    /**
+     * Creates a scaled_reduced_row_major range which contains a read-only
+     * version of the current accessor.
+     *
+     * @returns  a scaled_reduced_row_major major range which is read-only.
+     */
+    constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
+    {
+        return range<const_accessor>{size_, storage_, storage_stride_, scalar_,
+                                     scalar_stride_};
+    }
+
+    /**
+     * Reads the scalar value at the given indices.
+     *
+     * @param indices  indices which data to access
+     *
+     * @returns the scalar value at the given indices.
+     */
+    template <typename... Indices>
+    constexpr GKO_ATTRIBUTES scalar_type
+    read_scalar(Indices &&... indices) const
+    {
+        const arithmetic_type *GKO_RESTRICT rest_scalar = scalar_;
+        return rest_scalar[compute_scalar_index(
+            std::forward<Indices>(indices)...)];
+    }
+
+    /**
+     * Returns the length in dimension `dimension`.
+     *
+     * @param dimension  a dimension index
+     *
+     * @returns length in dimension `dimension`
+     */
+    constexpr GKO_ATTRIBUTES size_type length(size_type dimension) const
+    {
+        return dimension < dimensionality ? size_[dimension] : 1;
+    }
+
+    /**
+     * Copies data from another accessor
+     *
+     * @warning Do not use this function since it is not optimized for a
+     *          specific executor. It will always be performed sequentially.
+     *          Please write an optimized version (adjusted to the architecture)
+     *          by iterating through the values yourself.
+     *
+     * @tparam OtherAccessor  type of the other accessor
+     *
+     * @param other  other accessor
+     */
+    template <typename OtherAccessor>
+    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
+    {
+        helper::multidim_for_each(size_, [this, &other](auto... indices) {
+            // especially inefficient if mask has not all bits set
+            this->write_scalar(other.read_scalar(indices...), indices...);
+            (*this)(indices...) = other(indices...);
+        });
+    }
+
+    /**
+     * Returns the stored value for the given indices. If the storage is const,
+     * a value is returned, otherwise, a reference is returned.
+     *
+     * @param indices  indices which value is supposed to access
+     *
+     * @returns  the stored value if the accessor is const (if the storage type
+     *           is const), or a reference if the accessor is non-const
+     */
+    template <typename... Indices>
+    constexpr GKO_ATTRIBUTES std::enable_if_t<
+        helper::are_all_integral<Indices...>::value,
+        std::conditional_t<is_const, arithmetic_type, reference_type>>
+    operator()(Indices... indices) const
+    {
+        return reference_type{storage_ + compute_index(indices...),
+                              read_scalar(indices...)};
+    }
+
+    /**
+     * Returns a sub-range spinning the current range (x1_span, x2_span, ...)
+     *
+     * @param spans  span for the indices
+     *
+     * @returns a sub-range for the given spans.
+     */
+    template <typename... SpanTypes>
+    constexpr GKO_ATTRIBUTES
+        std::enable_if_t<helper::are_span_compatible<SpanTypes...>::value,
+                         range<scaled_reduced_row_major>>
+        operator()(SpanTypes... spans) const
+    {
+        return helper::validate_spans(size_, spans...),
+               range<scaled_reduced_row_major>{
+                   dim<dimensionality>{
+                       (span{spans}.end - span{spans}.begin)...},
+                   storage_ + compute_index((span{spans}.begin)...),
+                   storage_stride_,
+                   scalar_ + compute_scalar_index(span{spans}.begin...),
+                   scalar_stride_};
+    }
+
+    /**
+     * Returns the size of the accessor
+     *
+     * @returns the size of the accessor
+     */
+    constexpr GKO_ATTRIBUTES dim<dimensionality> get_size() const
+    {
+        return size_;
+    }
+
+    /**
+     * Returns a const reference to the storage stride array of size
+     * dimensionality - 1
+     *
+     * @returns a const reference to the storage stride array of size
+     *          dimensionality - 1
+     */
+    constexpr GKO_ATTRIBUTES const storage_stride_type &get_storage_stride()
+        const
+    {
+        return storage_stride_;
+    }
+
+    /**
+     * Returns a const reference to the scalar stride array
+     *
+     * @returns a const reference to the scalar stride array
+     */
+    constexpr GKO_ATTRIBUTES const scalar_stride_type &get_scalar_stride() const
+    {
+        return scalar_stride_;
+    }
+
+    /**
+     * Returns the pointer to the storage data
+     *
+     * @returns the pointer to the storage data
+     */
+    constexpr GKO_ATTRIBUTES storage_type *get_stored_data() const
+    {
+        return storage_;
+    }
+
+    /**
+     * Returns a const pointer to the storage data
+     *
+     * @returns a const pointer to the storage data
+     */
+    constexpr GKO_ATTRIBUTES const storage_type *get_const_storage() const
+    {
+        return storage_;
+    }
+
+    /**
+     * Returns the pointer to the scalar data
+     *
+     * @returns the pointer to the scalar data
+     */
+    constexpr GKO_ATTRIBUTES scalar_type *get_scalar() const { return scalar_; }
+
+    /**
+     * Returns a const pointer to the scalar data
+     *
+     * @returns a const pointer to the scalar data
+     */
+    constexpr GKO_ATTRIBUTES const scalar_type *get_const_scalar() const
+    {
+        return scalar_;
+    }
+
+protected:
+    template <typename... Indices>
+    constexpr GKO_ATTRIBUTES size_type
+    compute_index(Indices &&... indices) const
+    {
+        static_assert(sizeof...(Indices) == dimensionality,
+                      "Number of indices must match dimensionality!");
+        return helper::compute_storage_index<size_type, dimensionality>(
+            size_, storage_stride_, std::forward<Indices>(indices)...);
+    }
+
+    template <typename... Indices>
+    constexpr GKO_ATTRIBUTES size_type
+    compute_scalar_index(Indices &&... indices) const
+    {
+        static_assert(sizeof...(Indices) == dimensionality,
+                      "Number of indices must match dimensionality!");
+        return helper::compute_masked_index<size_type, scalar_mask,
+                                            scalar_stride_dim>(
+            size_, scalar_stride_, std::forward<Indices>(indices)...);
+    }
+
+private:
+    const dim<dimensionality> size_;
+    storage_type *storage_;
+    const storage_stride_type storage_stride_;
+    scalar_type *scalar_;
+    const scalar_stride_type scalar_stride_;
+};
+
+
+}  // namespace accessor
+}  // namespace gko
+
+
+#endif  // GKO_CORE_BASE_ACCESSORS_HPP_
diff --git a/core/test/base/CMakeLists.txt b/core/test/base/CMakeLists.txt
index 6f8c7291165..7b1b2cefddf 100644
--- a/core/test/base/CMakeLists.txt
+++ b/core/test/base/CMakeLists.txt
@@ -1,4 +1,5 @@
 ginkgo_create_test(abstract_factory)
+ginkgo_create_test(accessors_reduced_storage)
 ginkgo_create_test(allocator)
 ginkgo_create_test(array)
 ginkgo_create_test(combination)
diff --git a/core/test/base/accessors_reduced_storage.cpp b/core/test/base/accessors_reduced_storage.cpp
new file mode 100644
index 00000000000..b8fd30b46e7
--- /dev/null
+++ b/core/test/base/accessors_reduced_storage.cpp
@@ -0,0 +1,965 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/base/accessors.hpp"
+
+
+#include <gtest/gtest.h>
+
+
+#include <tuple>
+#include <type_traits>
+
+
+#include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+#include "core/base/extended_float.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+template <typename ArithmeticStorageType>
+class ReducedStorage3d : public ::testing::Test {
+protected:
+    using ar_type =
+        typename std::tuple_element<0, decltype(ArithmeticStorageType{})>::type;
+    using st_type =
+        typename std::tuple_element<1, decltype(ArithmeticStorageType{})>::type;
+    static constexpr ar_type delta{std::is_same<ar_type, st_type>::value
+                                       ? 0
+                                       : ::r<st_type, ar_type>::value};
+
+    // Type for `check_accessor_correctness` to forward the indices
+    using t = std::tuple<int, int, int>;
+
+    using accessor = gko::accessor::reduced_row_major<3, ar_type, st_type>;
+    using const_accessor =
+        gko::accessor::reduced_row_major<3, ar_type, const st_type>;
+
+    using reduced_storage = gko::range<accessor>;
+    using const_reduced_storage = gko::range<const_accessor>;
+
+    const gko::dim<3> size{4u, 3u, 2u};
+    static constexpr gko::size_type data_elements{4 * 3 * 2};
+    // clang-format off
+    st_type data[data_elements] {
+        // 0, y, z
+        1.0, 2.01,
+        -1.02, 3.03,
+        4.04, -2.05,
+        // 1, y, z
+        5.06, 6.07,
+        2.08, 3.09,
+        -1.1, -9.11,
+        // 2, y, z
+        -2.12, 2.13,
+        0.14, 15.15,
+        -9.16, 8.17,
+        // 3, y, z
+        7.18, -6.19,
+        5.2, -4.21,
+        3.22, -2.23
+    };
+    // clang-format on
+    reduced_storage r{size, data};
+    const_reduced_storage cr{size, data};
+
+    // Casts val first to `st_type`, then to `ar_type` in order to be allowed
+    // to test for equality
+    template <typename T>
+    static ar_type c_st_ar(T val)
+    {
+        return static_cast<ar_type>(static_cast<st_type>(val));
+    }
+
+    template <typename Accessor>
+    void check_accessor_correctness(const Accessor &a,
+                                    std::tuple<int, int, int> ignore = t(99, 99,
+                                                                         99))
+    {
+        // Test for equality is fine here since they should not be modified
+        // clang-format off
+        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), c_st_ar(1.0));     }
+        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), c_st_ar(2.01));    }
+        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), c_st_ar(-1.02));   }
+        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), c_st_ar(3.03));    }
+        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), c_st_ar(4.04));    }
+        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), c_st_ar(-2.05));   }
+        if (ignore != t(1, 0, 0)) { EXPECT_EQ(a(1, 0, 0), c_st_ar(5.06));    }
+        if (ignore != t(1, 0, 1)) { EXPECT_EQ(a(1, 0, 1), c_st_ar(6.07));    }
+        if (ignore != t(1, 1, 0)) { EXPECT_EQ(a(1, 1, 0), c_st_ar(2.08));    }
+        if (ignore != t(1, 1, 1)) { EXPECT_EQ(a(1, 1, 1), c_st_ar(3.09));    }
+        if (ignore != t(1, 2, 0)) { EXPECT_EQ(a(1, 2, 0), c_st_ar(-1.1));    }
+        if (ignore != t(1, 2, 1)) { EXPECT_EQ(a(1, 2, 1), c_st_ar(-9.11));   }
+        if (ignore != t(2, 0, 0)) { EXPECT_EQ(a(2, 0, 0), c_st_ar(-2.12));   }
+        if (ignore != t(2, 0, 1)) { EXPECT_EQ(a(2, 0, 1), c_st_ar(2.13));    }
+        if (ignore != t(2, 1, 0)) { EXPECT_EQ(a(2, 1, 0), c_st_ar(0.14));    }
+        if (ignore != t(2, 1, 1)) { EXPECT_EQ(a(2, 1, 1), c_st_ar(15.15));   }
+        if (ignore != t(2, 2, 0)) { EXPECT_EQ(a(2, 2, 0), c_st_ar(-9.16));   }
+        if (ignore != t(2, 2, 1)) { EXPECT_EQ(a(2, 2, 1), c_st_ar(8.17));    }
+        if (ignore != t(3, 0, 0)) { EXPECT_EQ(a(3, 0, 0), c_st_ar(7.18));    }
+        if (ignore != t(3, 0, 1)) { EXPECT_EQ(a(3, 0, 1), c_st_ar(-6.19));   }
+        if (ignore != t(3, 1, 0)) { EXPECT_EQ(a(3, 1, 0), c_st_ar(5.2));     }
+        if (ignore != t(3, 1, 1)) { EXPECT_EQ(a(3, 1, 1), c_st_ar(-4.21));   }
+        if (ignore != t(3, 2, 0)) { EXPECT_EQ(a(3, 2, 0), c_st_ar(3.22));    }
+        if (ignore != t(3, 2, 1)) { EXPECT_EQ(a(3, 2, 1), c_st_ar(-2.23));   }
+        // clang-format on
+    }
+};
+
+using ReducedStorage3dTypes =
+    ::testing::Types<std::tuple<double, double>, std::tuple<double, float>,
+                     std::tuple<float, float>, std::tuple<double, gko::half>,
+                     std::tuple<float, gko::half>
+                     /*,
+                     std::tuple<std::complex<double>, std::complex<double>>,
+                     std::tuple<std::complex<double>, std::complex<float>>,
+                     std::tuple<std::complex<float>, std::complex<float>>
+                     */
+                     >;
+
+TYPED_TEST_SUITE(ReducedStorage3d, ReducedStorage3dTypes);
+
+
+TYPED_TEST(ReducedStorage3d, CorrectLengths)
+{
+    EXPECT_EQ(this->r.length(0), this->size[0]);
+    EXPECT_EQ(this->r.length(1), this->size[1]);
+    EXPECT_EQ(this->r.length(2), this->size[2]);
+    EXPECT_EQ(this->r.length(3), 1);
+    EXPECT_EQ(this->r->get_size(), this->size);
+}
+
+
+TYPED_TEST(ReducedStorage3d, CorrectStride)
+{
+    EXPECT_EQ(this->r->get_stride()[0], this->size[1] * this->size[2]);
+    EXPECT_EQ(this->r->get_stride().at(0), this->size[1] * this->size[2]);
+    EXPECT_EQ(this->r->get_stride()[1], this->size[2]);
+    EXPECT_EQ(this->r->get_stride().at(1), this->size[2]);
+}
+
+
+TYPED_TEST(ReducedStorage3d, CorrectStorage)
+{
+    EXPECT_EQ(this->r->get_stored_data(), this->data);
+    EXPECT_EQ(this->r->get_const_storage(), this->data);
+}
+
+
+TYPED_TEST(ReducedStorage3d, CanReadData)
+{
+    this->check_accessor_correctness(this->r);
+    this->check_accessor_correctness(this->cr);
+}
+
+
+TYPED_TEST(ReducedStorage3d, CopyFrom)
+{
+    using st_type = typename TestFixture::st_type;
+    using reduced_storage = typename TestFixture::reduced_storage;
+    st_type data2[TestFixture::data_elements];
+    reduced_storage cpy(this->size, data2);
+
+    // Do not use this in regular code since the implementation is slow
+    cpy = this->r;
+
+    this->check_accessor_correctness(cpy);
+}
+
+
+TYPED_TEST(ReducedStorage3d, CanImplicitlyConvertToConst)
+{
+    using const_reduced_storage = typename TestFixture::const_reduced_storage;
+
+    const_reduced_storage const_rs = this->r->to_const();
+    const_reduced_storage const_rs2 = this->cr;
+
+    this->check_accessor_correctness(const_rs);
+    this->check_accessor_correctness(const_rs2);
+}
+
+
+TYPED_TEST(ReducedStorage3d, ToConstWorks)
+{
+    using const_reduced_storage = typename TestFixture::const_reduced_storage;
+
+    auto cr2 = this->r->to_const();
+
+    static_assert(std::is_same<decltype(cr2), const_reduced_storage>::value,
+                  "Types must be equal!");
+    this->check_accessor_correctness(cr2);
+}
+
+
+TYPED_TEST(ReducedStorage3d, CanCreateWithStride)
+{
+    using reduced_storage = typename TestFixture::reduced_storage;
+    using ar_type = typename TestFixture::ar_type;
+    auto size = gko::dim<3>{2, 2, 2};
+    auto stride = std::array<gko::size_type, 2>{{12, 2}};
+
+    auto range = reduced_storage{size, this->data, stride};
+    range(1, 1, 0) = ar_type{2.};
+
+    EXPECT_EQ(range(0, 0, 0), this->c_st_ar(1.0));
+    EXPECT_EQ(range(0, 0, 1), this->c_st_ar(2.01));
+    EXPECT_EQ(range(0, 1, 0), this->c_st_ar(-1.02));
+    EXPECT_EQ(range(0, 1, 1), this->c_st_ar(3.03));
+    EXPECT_EQ(range(1, 0, 0), this->c_st_ar(-2.12));
+    EXPECT_EQ(range(1, 0, 1), this->c_st_ar(2.13));
+    EXPECT_EQ(range(1, 1, 0), this->c_st_ar(2.));
+    EXPECT_EQ(range(1, 1, 1), this->c_st_ar(15.15));
+}
+
+
+TYPED_TEST(ReducedStorage3d, CanWriteData)
+{
+    using t = typename TestFixture::t;
+
+    this->r(0, 1, 0) = 100.25;
+
+    this->check_accessor_correctness(this->r, t(0, 1, 0));
+    EXPECT_NEAR(this->r(0, 1, 0), 100.25, TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Assignment)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = 1.2;
+
+    this->r(0, 0, 1) = expected;
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Assignment2)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = -1.02;
+
+    this->r(0, 0, 1) = this->r(0, 1, 0);
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Addition)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = 1.2 + 2.01;
+
+    auto result = this->r(0, 0, 1) + ar_type{1.2};
+    this->r(0, 0, 1) += 1.2;
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Addition2)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = 2.01 + -1.02;
+
+    auto result = this->r(0, 0, 1) + this->r(0, 1, 0);
+    this->r(0, 0, 1) += this->r(0, 1, 0);
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Subtraction)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = -2.23 - 1;
+
+    auto result = this->r(3, 2, 1) - ar_type{1.};
+    this->r(3, 2, 1) -= 1;
+
+    this->check_accessor_correctness(this->r, t(3, 2, 1));
+    EXPECT_NEAR(this->r(3, 2, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Subtraction2)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = 3.22 - -2.23;
+
+    auto result = this->cr(3, 2, 0) - this->r(3, 2, 1);
+    this->r(3, 2, 0) -= this->r(3, 2, 1);
+
+    this->check_accessor_correctness(this->r, t(3, 2, 0));
+    EXPECT_NEAR(this->r(3, 2, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Multiplication)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = 1 * 2;
+
+    auto result = this->r(0, 0, 0) * ar_type{2.};
+    this->r(0, 0, 0) *= 2;
+
+    this->check_accessor_correctness(this->r, t(0, 0, 0));
+    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Multiplication2)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = 2.01 * 3.03;
+
+    auto result = this->r(0, 0, 1) * this->cr(0, 1, 1);
+    this->r(0, 0, 1) *= this->r(0, 1, 1);
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Division)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = 2.01 / 2.0;
+
+    auto result = this->cr(0, 0, 1) / ar_type{2.};
+    this->r(0, 0, 1) /= 2.;
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Division2)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = 5.06 / 4.04;
+
+    auto result = this->r(1, 0, 0) / this->cr(0, 2, 0);
+    this->r(1, 0, 0) /= this->r(0, 2, 0);
+
+    this->check_accessor_correctness(this->r, t(1, 0, 0));
+    EXPECT_NEAR(this->r(1, 0, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, UnaryMinus)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type neg_expected = this->r(2, 0, 0);
+    const ar_type expected = -neg_expected;
+
+    auto result = -this->r(2, 0, 0);
+
+    this->check_accessor_correctness(this->r);
+    EXPECT_EQ(result, expected);
+}
+
+
+TYPED_TEST(ReducedStorage3d, CanCreateSubrange)
+{
+    auto subr = this->r(gko::span{1u, 3u}, gko::span{0u, 2u}, 0u);
+
+    EXPECT_EQ(subr(0, 0, 0), this->c_st_ar(5.06));
+    EXPECT_EQ(subr(0, 1, 0), this->c_st_ar(2.08));
+    EXPECT_EQ(subr(1, 0, 0), this->c_st_ar(-2.12));
+    EXPECT_EQ(subr(1, 1, 0), this->c_st_ar(0.14));
+}
+
+
+TYPED_TEST(ReducedStorage3d, CanCreateSubrange2)
+{
+    auto subr =
+        this->cr(gko::span{1u, 3u}, gko::span{0u, 2u}, gko::span{0u, 1u});
+
+    EXPECT_EQ(subr(0, 0, 0), this->c_st_ar(5.06));
+    EXPECT_EQ(subr(0, 1, 0), this->c_st_ar(2.08));
+    EXPECT_EQ(subr(1, 0, 0), this->c_st_ar(-2.12));
+    EXPECT_EQ(subr(1, 1, 0), this->c_st_ar(0.14));
+}
+
+
+class ReducedStorageXd : public ::testing::Test {
+protected:
+    using span = gko::span;
+    using ar_type = double;
+    using st_type = float;
+    using size_type = gko::size_type;
+    static constexpr ar_type delta{::r<st_type>::value};
+
+    using accessor1d = gko::accessor::reduced_row_major<1, ar_type, st_type>;
+    using accessor2d = gko::accessor::reduced_row_major<2, ar_type, st_type>;
+    using const_accessor1d =
+        gko::accessor::reduced_row_major<1, ar_type, const st_type>;
+    using const_accessor2d =
+        gko::accessor::reduced_row_major<2, ar_type, const st_type>;
+    static_assert(std::is_same<const_accessor1d,
+                               typename accessor1d::const_accessor>::value,
+                  "Const accessors must be the same!");
+    static_assert(std::is_same<const_accessor2d,
+                               typename accessor2d::const_accessor>::value,
+                  "Const accessors must be the same!");
+
+    using reduced_storage1d = gko::range<accessor1d>;
+    using reduced_storage2d = gko::range<accessor2d>;
+    using const_reduced_storage2d = gko::range<const_accessor2d>;
+    using const_reduced_storage1d = gko::range<const_accessor1d>;
+
+    const std::array<size_type, 0> stride0{{}};
+    const std::array<size_type, 1> stride1{{4}};
+    const gko::dim<1> size_1d{8u};
+    const gko::dim<2> size_2d{2u, 4u};
+    static constexpr gko::size_type data_elements{8};
+    st_type data[data_elements]{1.1f, 2.2f, 3.3f, 4.4f,
+                                5.5f, 6.6f, 7.7f, -8.8f};
+    reduced_storage1d r1{size_1d, data};
+    reduced_storage2d r2{size_2d, data, stride1[0]};
+    const_reduced_storage1d cr1{size_1d, data, stride0};
+    const_reduced_storage2d cr2{size_2d, data, stride1};
+
+    template <typename T>
+    static ar_type c_st_ar(T val)
+    {
+        return static_cast<ar_type>(static_cast<st_type>(val));
+    }
+
+    void data_equal_except_for(int idx)
+    {
+        // clang-format off
+        if (idx != 0) { EXPECT_EQ(data[0], c_st_ar(1.1)); }
+        if (idx != 1) { EXPECT_EQ(data[1], c_st_ar(2.2)); }
+        if (idx != 2) { EXPECT_EQ(data[2], c_st_ar(3.3)); }
+        if (idx != 3) { EXPECT_EQ(data[3], c_st_ar(4.4)); }
+        if (idx != 4) { EXPECT_EQ(data[4], c_st_ar(5.5)); }
+        if (idx != 5) { EXPECT_EQ(data[5], c_st_ar(6.6)); }
+        if (idx != 6) { EXPECT_EQ(data[6], c_st_ar(7.7)); }
+        if (idx != 7) { EXPECT_EQ(data[7], c_st_ar(-8.8)); }
+        // clang-format on
+    }
+};
+
+
+TEST_F(ReducedStorageXd, CanRead)
+{
+    EXPECT_EQ(cr1(1), this->c_st_ar(2.2));
+    EXPECT_EQ(cr2(0, 1), this->c_st_ar(2.2));
+    EXPECT_EQ(r1(1), this->c_st_ar(2.2));
+    EXPECT_EQ(r2(0, 1), this->c_st_ar(2.2));
+}
+
+
+TEST_F(ReducedStorageXd, CanWrite1)
+{
+    r1(2) = 0.25;
+
+    data_equal_except_for(2);
+    EXPECT_EQ(r1(2), 0.25);  // expect exact since easy to store
+}
+
+
+TEST_F(ReducedStorageXd, CanWrite2)
+{
+    r2(1, 1) = 0.75;
+
+    data_equal_except_for(5);
+    EXPECT_EQ(r2(1, 1), 0.75);  // expect exact since easy to store
+}
+
+
+template <typename ArithmeticStorageType>
+class ScaledReducedStorage3d : public ::testing::Test {
+protected:
+    using ar_type =
+        typename std::tuple_element<0, decltype(ArithmeticStorageType{})>::type;
+    using st_type =
+        typename std::tuple_element<1, decltype(ArithmeticStorageType{})>::type;
+    // Type for `check_accessor_correctness` to forward the indices
+    using t = std::tuple<int, int, int>;
+
+    static constexpr ar_type delta{::r<ar_type>::value};
+
+    using accessor =
+        gko::accessor::scaled_reduced_row_major<3, ar_type, st_type, 0b0101>;
+    using const_accessor =
+        gko::accessor::scaled_reduced_row_major<3, ar_type, const st_type,
+                                                0b0101>;
+
+    using reduced_storage = gko::range<accessor>;
+    using const_reduced_storage = gko::range<const_accessor>;
+
+    const gko::dim<3> size{1u, 4u, 2u};
+    static constexpr gko::size_type data_elements{8};
+    static constexpr gko::size_type scalar_elements{8};
+    // clang-format off
+    st_type data[8]{
+        10, 11,
+        -12, 13,
+        14, -115,
+        6, 77
+    };
+    ar_type scalar[scalar_elements]{
+        1., 2., 3., 4., 5., 6., 7., 8.
+    };
+    // clang-format on
+    const std::array<gko::size_type, 2> storage_stride{{8, 2}};
+    const std::array<gko::size_type, 1> scalar_stride{{2}};
+    reduced_storage r{size, data, storage_stride, scalar, scalar_stride};
+    const_reduced_storage cr{size, data, scalar};
+
+    template <typename Accessor>
+    static void check_accessor_correctness(
+        const Accessor &a,
+        std::tuple<int, int, int> ignore = std::tuple<int, int, int>(99, 99,
+                                                                     99))
+    {
+        // Test for equality is fine here since they should not be modified
+        // clang-format off
+        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), ar_type{10.});   }
+        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), ar_type{22.});   }
+        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), ar_type{-12.});  }
+        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), ar_type{26.});   }
+        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), ar_type{14.});   }
+        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), ar_type{-230.}); }
+        if (ignore != t(0, 3, 0)) { EXPECT_EQ(a(0, 3, 0), ar_type{6.});    }
+        if (ignore != t(0, 3, 1)) { EXPECT_EQ(a(0, 3, 1), ar_type{154.});  }
+        // clang-format on
+    }
+};
+
+using ScaledReducedStorage3dTypes = ::testing::Types<
+    std::tuple<double, gko::int64>, std::tuple<double, gko::int32>,
+    std::tuple<double, gko::int16>, std::tuple<float, gko::int32>,
+    std::tuple<float, gko::int16>>;
+
+TYPED_TEST_SUITE(ScaledReducedStorage3d, ScaledReducedStorage3dTypes);
+
+
+TYPED_TEST(ScaledReducedStorage3d, CorrectLengths)
+{
+    EXPECT_EQ(this->r.length(0), this->size[0]);
+    EXPECT_EQ(this->r.length(1), this->size[1]);
+    EXPECT_EQ(this->r.length(2), this->size[2]);
+    EXPECT_EQ(this->r.length(3), 1);
+    EXPECT_EQ(this->r->get_size(), this->size);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CorrectStride)
+{
+    EXPECT_EQ(this->r->get_scalar_stride(), this->scalar_stride);
+    EXPECT_EQ(this->r->get_storage_stride(), this->storage_stride);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CorrectStorage)
+{
+    EXPECT_EQ(this->r->get_stored_data(), this->data);
+    EXPECT_EQ(this->r->get_const_storage(), this->data);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CorrectScale)
+{
+    EXPECT_EQ(this->r->get_scalar(), this->scalar);
+    EXPECT_EQ(this->r->get_const_scalar(), this->scalar);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CanReadData)
+{
+    this->check_accessor_correctness(this->r);
+    this->check_accessor_correctness(this->cr);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CopyFrom)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using st_type = typename TestFixture::st_type;
+    using reduced_storage = typename TestFixture::reduced_storage;
+    st_type data2[TestFixture::data_elements];
+    ar_type scale2[TestFixture::scalar_elements];
+    reduced_storage cpy(this->size, data2, scale2);
+
+    // Do not use this in regular code since the implementation is slow
+    cpy = this->r;
+
+    this->check_accessor_correctness(cpy);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CanImplicitlyConvertToConst)
+{
+    using const_reduced_storage = typename TestFixture::const_reduced_storage;
+
+    const_reduced_storage const_rs = this->r->to_const();
+    const_reduced_storage const_rs2 = this->cr;
+
+    this->check_accessor_correctness(const_rs);
+    this->check_accessor_correctness(const_rs2);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, ToConstWorks)
+{
+    using const_reduced_storage = typename TestFixture::const_reduced_storage;
+    auto cr2 = this->r->to_const();
+
+    static_assert(std::is_same<decltype(cr2), const_reduced_storage>::value,
+                  "Types must be equal!");
+    this->check_accessor_correctness(cr2);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CanCreateWithStride)
+{
+    using reduced_storage = typename TestFixture::reduced_storage;
+    using ar_type = typename TestFixture::ar_type;
+    gko::dim<3> size{2, 1, 2};
+    std::array<gko::size_type, 2> stride_storage{{5, 2}};
+    std::array<gko::size_type, 1> stride_scalar{{4}};
+
+    reduced_storage range{size, this->data, stride_storage, this->scalar,
+                          stride_scalar};
+    range(1, 0, 0) = ar_type{15};
+
+    EXPECT_EQ(range(0, 0, 0), ar_type{10});
+    EXPECT_EQ(range(0, 0, 1), ar_type{22});
+    EXPECT_EQ(range(1, 0, 0), ar_type{15});
+    EXPECT_EQ(range(1, 0, 1), ar_type{36});
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Subrange)
+{
+    auto subr = this->cr(0u, gko::span{0u, 2u}, 1u);
+
+    EXPECT_EQ(subr(0, 0, 0), 22.);
+    EXPECT_EQ(subr(0, 1, 0), 26.);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CanWriteScale)
+{
+    using ar_type = typename TestFixture::ar_type;
+
+    this->r->write_scalar(10., 0, 0, 0);
+
+    EXPECT_EQ(this->r(0, 0, 0), ar_type{100.});
+    EXPECT_EQ(this->r(0, 0, 1), ar_type{22.});
+    EXPECT_EQ(this->r(0, 1, 0), ar_type{-120.});
+    EXPECT_EQ(this->r(0, 1, 1), ar_type{26.});
+    EXPECT_EQ(this->r(0, 2, 0), ar_type{140.});
+    EXPECT_EQ(this->r(0, 2, 1), ar_type{-230.});
+    EXPECT_EQ(this->r(0, 3, 0), ar_type{60.});
+    EXPECT_EQ(this->r(0, 3, 1), ar_type{154.});
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CanReadScale)
+{
+    EXPECT_EQ(this->r->read_scalar(0, 0, 0), 1.);
+    EXPECT_EQ(this->r->read_scalar(0, 0, 1), 2.);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Addition)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = 10. + 3.;
+
+    const auto result = this->cr(0, 0, 0) + 3.;
+    this->r(0, 0, 0) += 3.;
+
+    this->check_accessor_correctness(this->r, t(0, 0, 0));
+    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Addition2)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = 10. + 22.;
+
+    const auto result = this->cr(0, 0, 0) + this->r(0, 0, 1);
+    this->r(0, 0, 0) += this->cr(0, 0, 1);
+
+    this->check_accessor_correctness(this->r, t(0, 0, 0));
+    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Subtraction)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = 22. - 2.;
+
+    const auto result = this->cr(0, 0, 1) - 2.;
+    this->r(0, 0, 1) -= 2.;
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Subtraction2)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = -12. - 26.;
+
+    const auto result = this->cr(0, 1, 0) - this->r(0, 1, 1);
+    this->r(0, 1, 0) -= this->r(0, 1, 1);
+
+    this->check_accessor_correctness(this->r, t(0, 1, 0));
+    EXPECT_NEAR(this->r(0, 1, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Multiplication)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = 26. * 3.;
+
+    const auto result = this->cr(0, 1, 1) * 3.;
+    this->r(0, 1, 1) *= 3.;
+
+    this->check_accessor_correctness(this->r, t(0, 1, 1));
+    EXPECT_NEAR(this->r(0, 1, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Multiplication2)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = 14. * 10.;
+
+    const auto result = this->r(0, 2, 0) * this->r(0, 0, 0);
+    this->r(0, 2, 0) *= this->r(0, 0, 0);
+
+    this->check_accessor_correctness(this->r, t(0, 2, 0));
+    EXPECT_NEAR(this->r(0, 2, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Division)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = 10. / 2.;
+
+    const auto result = this->cr(0, 0, 0) / 2.;
+    this->r(0, 0, 0) /= 2.;
+
+    this->check_accessor_correctness(this->r, t(0, 0, 0));
+    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Division2)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = -12. / 6.;
+
+    const auto result = this->r(0, 1, 0) / this->r(0, 3, 0);
+    this->r(0, 1, 0) /= this->r(0, 3, 0);
+
+    this->check_accessor_correctness(this->r, t(0, 1, 0));
+    EXPECT_NEAR(this->r(0, 1, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, UnaryMinus)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type neg_expected = this->r(0, 1, 1);
+    const ar_type expected = -neg_expected;
+
+    auto result = -this->r(0, 1, 1);
+
+    this->check_accessor_correctness(this->r);
+    EXPECT_EQ(result, expected);
+}
+
+
+class ScaledReducedStorageXd : public ::testing::Test {
+protected:
+    using span = gko::span;
+    using ar_type = double;
+    using st_type = int;
+    using size_type = gko::size_type;
+    static constexpr ar_type delta{0.1};
+
+    using accessor1d =
+        gko::accessor::scaled_reduced_row_major<1, ar_type, st_type, 1>;
+    using accessor2d =
+        gko::accessor::scaled_reduced_row_major<2, ar_type, st_type, 3>;
+    using const_accessor1d =
+        gko::accessor::scaled_reduced_row_major<1, ar_type, const st_type, 1>;
+    using const_accessor2d =
+        gko::accessor::scaled_reduced_row_major<2, ar_type, const st_type, 3>;
+    static_assert(std::is_same<const_accessor1d,
+                               typename accessor1d::const_accessor>::value,
+                  "Const accessors must be the same!");
+    static_assert(std::is_same<const_accessor2d,
+                               typename accessor2d::const_accessor>::value,
+                  "Const accessors must be the same!");
+
+    using reduced_storage1d = gko::range<accessor1d>;
+    using reduced_storage2d = gko::range<accessor2d>;
+    using const_reduced_storage2d = gko::range<const_accessor2d>;
+    using const_reduced_storage1d = gko::range<const_accessor1d>;
+
+    const std::array<size_type, 0> stride0{{}};
+    const std::array<size_type, 1> stride1{{4}};
+    const std::array<size_type, 1> stride_sc{{5}};
+    const gko::dim<1> size_1d{8u};
+    const gko::dim<2> size_2d{2u, 2u};
+
+    static constexpr gko::size_type data_elements{8};
+    st_type data[data_elements]{10, 22, 32, 44, 54, 66, 76, -88};
+    ar_type scalar[data_elements]{1e0,  5e-1, 1e-1, 5e-2,
+                                  1e-2, 5e-3, 1e-3, 5e-4};
+
+    reduced_storage1d r1{size_1d, data, scalar};
+    reduced_storage2d r2{size_2d, data, stride1, scalar, stride_sc};
+    const_reduced_storage1d cr1{size_1d, data, stride0, scalar};
+    const_reduced_storage2d cr2{size_2d, data, stride1, scalar, stride_sc};
+
+    void data_equal_except_for(int idx)
+    {
+        // clang-format off
+        if (idx != 0) { EXPECT_EQ(data[0], 10); }
+        if (idx != 1) { EXPECT_EQ(data[1], 22); }
+        if (idx != 2) { EXPECT_EQ(data[2], 32); }
+        if (idx != 3) { EXPECT_EQ(data[3], 44); }
+        if (idx != 4) { EXPECT_EQ(data[4], 54); }
+        if (idx != 5) { EXPECT_EQ(data[5], 66); }
+        if (idx != 6) { EXPECT_EQ(data[6], 76); }
+        if (idx != 7) { EXPECT_EQ(data[7], -88); }
+        // clang-format on
+    }
+    void scalar_equal_except_for(int idx)
+    {
+        // clang-format off
+        if (idx != 0) { EXPECT_EQ(scalar[0], ar_type{1e0}); }
+        if (idx != 1) { EXPECT_EQ(scalar[1], ar_type{5e-1}); }
+        if (idx != 2) { EXPECT_EQ(scalar[2], ar_type{1e-1}); }
+        if (idx != 3) { EXPECT_EQ(scalar[3], ar_type{5e-2}); }
+        if (idx != 4) { EXPECT_EQ(scalar[4], ar_type{1e-2}); }
+        if (idx != 5) { EXPECT_EQ(scalar[5], ar_type{5e-3}); }
+        if (idx != 6) { EXPECT_EQ(scalar[6], ar_type{1e-3}); }
+        if (idx != 7) { EXPECT_EQ(scalar[7], ar_type{5e-4}); }
+        // clang-format on
+    }
+};
+
+
+TEST_F(ScaledReducedStorageXd, CanRead)
+{
+    EXPECT_NEAR(cr1(1), 11., delta);
+    EXPECT_NEAR(cr2(0, 1), 11., delta);
+    EXPECT_NEAR(cr2(1, 1), 66e-3, delta);
+    EXPECT_NEAR(r1(1), 11., delta);
+    EXPECT_NEAR(r2(0, 1), 11., delta);
+    EXPECT_NEAR(r2(1, 1), 66e-3, delta);
+}
+
+
+TEST_F(ScaledReducedStorageXd, CanWrite1)
+{
+    r1(2) = 0.2;
+
+    data_equal_except_for(2);
+    scalar_equal_except_for(99);
+    EXPECT_NEAR(r1(2), 0.2, delta);
+}
+
+
+TEST_F(ScaledReducedStorageXd, CanWrite2)
+{
+    r2(1, 1) = 0.5;
+
+    data_equal_except_for(5);
+    scalar_equal_except_for(99);
+    EXPECT_NEAR(r2(1, 1), 0.5, delta);
+}
+
+
+}  // namespace
diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index ad3f122c503..eecac4ae60f 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -36,17 +36,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
-#include <tuple>
-#include <type_traits>
-
-#include <iostream>
-
-
-#include <ginkgo/core/base/dim.hpp>
 #include <ginkgo/core/base/types.hpp>
 
 
-#include "core/base/extended_float.hpp"
 #include "core/test/utils.hpp"
 
 
@@ -143,915 +135,4 @@ TEST_F(RowMajorAccessor, CanAssignSubranges)
 }
 
 
-template <typename ArithmeticStorageType>
-class ReducedStorage3d : public ::testing::Test {
-protected:
-    using ar_type =
-        typename std::tuple_element<0, decltype(ArithmeticStorageType{})>::type;
-    using st_type =
-        typename std::tuple_element<1, decltype(ArithmeticStorageType{})>::type;
-    static constexpr ar_type delta{std::is_same<ar_type, st_type>::value
-                                       ? 0
-                                       : ::r<st_type, ar_type>::value};
-
-    // Type for `check_accessor_correctness` to forward the indices
-    using t = std::tuple<int, int, int>;
-
-    using accessor = gko::accessor::reduced_row_major<3, ar_type, st_type>;
-    using const_accessor =
-        gko::accessor::reduced_row_major<3, ar_type, const st_type>;
-
-    using reduced_storage = gko::range<accessor>;
-    using const_reduced_storage = gko::range<const_accessor>;
-
-    const gko::dim<3> size{4u, 3u, 2u};
-    static constexpr gko::size_type data_elements{4 * 3 * 2};
-    // clang-format off
-    st_type data[data_elements] {
-        // 0, y, z
-        1.0, 2.01,
-        -1.02, 3.03,
-        4.04, -2.05,
-        // 1, y, z
-        5.06, 6.07,
-        2.08, 3.09,
-        -1.1, -9.11,
-        // 2, y, z
-        -2.12, 2.13,
-        0.14, 15.15,
-        -9.16, 8.17,
-        // 3, y, z
-        7.18, -6.19,
-        5.2, -4.21,
-        3.22, -2.23
-    };
-    // clang-format on
-    reduced_storage r{size, data};
-    const_reduced_storage cr{size, data};
-
-    // Casts val first to `st_type`, then to `ar_type` in order to be allowed
-    // to test for equality
-    template <typename T>
-    static ar_type c_st_ar(T val)
-    {
-        return static_cast<ar_type>(static_cast<st_type>(val));
-    }
-
-    template <typename Accessor>
-    void check_accessor_correctness(const Accessor &a,
-                                    std::tuple<int, int, int> ignore = t(99, 99,
-                                                                         99))
-    {
-        // Test for equality is fine here since they should not be modified
-        // clang-format off
-        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), c_st_ar(1.0));     }
-        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), c_st_ar(2.01));    }
-        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), c_st_ar(-1.02));   }
-        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), c_st_ar(3.03));    }
-        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), c_st_ar(4.04));    }
-        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), c_st_ar(-2.05));   }
-        if (ignore != t(1, 0, 0)) { EXPECT_EQ(a(1, 0, 0), c_st_ar(5.06));    }
-        if (ignore != t(1, 0, 1)) { EXPECT_EQ(a(1, 0, 1), c_st_ar(6.07));    }
-        if (ignore != t(1, 1, 0)) { EXPECT_EQ(a(1, 1, 0), c_st_ar(2.08));    }
-        if (ignore != t(1, 1, 1)) { EXPECT_EQ(a(1, 1, 1), c_st_ar(3.09));    }
-        if (ignore != t(1, 2, 0)) { EXPECT_EQ(a(1, 2, 0), c_st_ar(-1.1));    }
-        if (ignore != t(1, 2, 1)) { EXPECT_EQ(a(1, 2, 1), c_st_ar(-9.11));   }
-        if (ignore != t(2, 0, 0)) { EXPECT_EQ(a(2, 0, 0), c_st_ar(-2.12));   }
-        if (ignore != t(2, 0, 1)) { EXPECT_EQ(a(2, 0, 1), c_st_ar(2.13));    }
-        if (ignore != t(2, 1, 0)) { EXPECT_EQ(a(2, 1, 0), c_st_ar(0.14));    }
-        if (ignore != t(2, 1, 1)) { EXPECT_EQ(a(2, 1, 1), c_st_ar(15.15));   }
-        if (ignore != t(2, 2, 0)) { EXPECT_EQ(a(2, 2, 0), c_st_ar(-9.16));   }
-        if (ignore != t(2, 2, 1)) { EXPECT_EQ(a(2, 2, 1), c_st_ar(8.17));    }
-        if (ignore != t(3, 0, 0)) { EXPECT_EQ(a(3, 0, 0), c_st_ar(7.18));    }
-        if (ignore != t(3, 0, 1)) { EXPECT_EQ(a(3, 0, 1), c_st_ar(-6.19));   }
-        if (ignore != t(3, 1, 0)) { EXPECT_EQ(a(3, 1, 0), c_st_ar(5.2));     }
-        if (ignore != t(3, 1, 1)) { EXPECT_EQ(a(3, 1, 1), c_st_ar(-4.21));   }
-        if (ignore != t(3, 2, 0)) { EXPECT_EQ(a(3, 2, 0), c_st_ar(3.22));    }
-        if (ignore != t(3, 2, 1)) { EXPECT_EQ(a(3, 2, 1), c_st_ar(-2.23));   }
-        // clang-format on
-    }
-};
-
-using ReducedStorage3dTypes =
-    ::testing::Types<std::tuple<double, double>, std::tuple<double, float>,
-                     std::tuple<float, float>, std::tuple<double, gko::half>,
-                     std::tuple<float, gko::half>
-                     /*,
-                     std::tuple<std::complex<double>, std::complex<double>>,
-                     std::tuple<std::complex<double>, std::complex<float>>,
-                     std::tuple<std::complex<float>, std::complex<float>>
-                     */
-                     >;
-
-TYPED_TEST_SUITE(ReducedStorage3d, ReducedStorage3dTypes);
-
-
-TYPED_TEST(ReducedStorage3d, CorrectLengths)
-{
-    EXPECT_EQ(this->r.length(0), this->size[0]);
-    EXPECT_EQ(this->r.length(1), this->size[1]);
-    EXPECT_EQ(this->r.length(2), this->size[2]);
-    EXPECT_EQ(this->r.length(3), 1);
-    EXPECT_EQ(this->r->get_size(), this->size);
-}
-
-
-TYPED_TEST(ReducedStorage3d, CorrectStride)
-{
-    EXPECT_EQ(this->r->get_stride()[0], this->size[1] * this->size[2]);
-    EXPECT_EQ(this->r->get_stride().at(0), this->size[1] * this->size[2]);
-    EXPECT_EQ(this->r->get_stride()[1], this->size[2]);
-    EXPECT_EQ(this->r->get_stride().at(1), this->size[2]);
-}
-
-
-TYPED_TEST(ReducedStorage3d, CorrectStorage)
-{
-    EXPECT_EQ(this->r->get_stored_data(), this->data);
-    EXPECT_EQ(this->r->get_const_storage(), this->data);
-}
-
-
-TYPED_TEST(ReducedStorage3d, CanReadData)
-{
-    this->check_accessor_correctness(this->r);
-    this->check_accessor_correctness(this->cr);
-}
-
-
-TYPED_TEST(ReducedStorage3d, CopyFrom)
-{
-    using st_type = typename TestFixture::st_type;
-    using reduced_storage = typename TestFixture::reduced_storage;
-    st_type data2[TestFixture::data_elements];
-    reduced_storage cpy(this->size, data2);
-
-    // Do not use this in regular code since the implementation is slow
-    cpy = this->r;
-
-    this->check_accessor_correctness(cpy);
-}
-
-
-TYPED_TEST(ReducedStorage3d, CanImplicitlyConvertToConst)
-{
-    using const_reduced_storage = typename TestFixture::const_reduced_storage;
-
-    const_reduced_storage const_rs = this->r->to_const();
-    const_reduced_storage const_rs2 = this->cr;
-
-    this->check_accessor_correctness(const_rs);
-    this->check_accessor_correctness(const_rs2);
-}
-
-
-TYPED_TEST(ReducedStorage3d, ToConstWorks)
-{
-    using const_reduced_storage = typename TestFixture::const_reduced_storage;
-
-    auto cr2 = this->r->to_const();
-
-    static_assert(std::is_same<decltype(cr2), const_reduced_storage>::value,
-                  "Types must be equal!");
-    this->check_accessor_correctness(cr2);
-}
-
-
-TYPED_TEST(ReducedStorage3d, CanCreateWithStride)
-{
-    using reduced_storage = typename TestFixture::reduced_storage;
-    using ar_type = typename TestFixture::ar_type;
-    auto size = gko::dim<3>{2, 2, 2};
-    auto stride = std::array<gko::size_type, 2>{{12, 2}};
-
-    auto range = reduced_storage{size, this->data, stride};
-    range(1, 1, 0) = ar_type{2.};
-
-    EXPECT_EQ(range(0, 0, 0), this->c_st_ar(1.0));
-    EXPECT_EQ(range(0, 0, 1), this->c_st_ar(2.01));
-    EXPECT_EQ(range(0, 1, 0), this->c_st_ar(-1.02));
-    EXPECT_EQ(range(0, 1, 1), this->c_st_ar(3.03));
-    EXPECT_EQ(range(1, 0, 0), this->c_st_ar(-2.12));
-    EXPECT_EQ(range(1, 0, 1), this->c_st_ar(2.13));
-    EXPECT_EQ(range(1, 1, 0), this->c_st_ar(2.));
-    EXPECT_EQ(range(1, 1, 1), this->c_st_ar(15.15));
-}
-
-
-TYPED_TEST(ReducedStorage3d, CanWriteData)
-{
-    using t = typename TestFixture::t;
-
-    this->r(0, 1, 0) = 100.25;
-
-    this->check_accessor_correctness(this->r, t(0, 1, 0));
-    EXPECT_NEAR(this->r(0, 1, 0), 100.25, TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Assignment)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type expected = 1.2;
-
-    this->r(0, 0, 1) = expected;
-
-    this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Assignment2)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type expected = -1.02;
-
-    this->r(0, 0, 1) = this->r(0, 1, 0);
-
-    this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Addition)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type expected = 1.2 + 2.01;
-
-    auto result = this->r(0, 0, 1) + ar_type{1.2};
-    this->r(0, 0, 1) += 1.2;
-
-    this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Addition2)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type expected = 2.01 + -1.02;
-
-    auto result = this->r(0, 0, 1) + this->r(0, 1, 0);
-    this->r(0, 0, 1) += this->r(0, 1, 0);
-
-    this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Subtraction)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type expected = -2.23 - 1;
-
-    auto result = this->r(3, 2, 1) - ar_type{1.};
-    this->r(3, 2, 1) -= 1;
-
-    this->check_accessor_correctness(this->r, t(3, 2, 1));
-    EXPECT_NEAR(this->r(3, 2, 1), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Subtraction2)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type expected = 3.22 - -2.23;
-
-    auto result = this->cr(3, 2, 0) - this->r(3, 2, 1);
-    this->r(3, 2, 0) -= this->r(3, 2, 1);
-
-    this->check_accessor_correctness(this->r, t(3, 2, 0));
-    EXPECT_NEAR(this->r(3, 2, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Multiplication)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type expected = 1 * 2;
-
-    auto result = this->r(0, 0, 0) * ar_type{2.};
-    this->r(0, 0, 0) *= 2;
-
-    this->check_accessor_correctness(this->r, t(0, 0, 0));
-    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Multiplication2)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type expected = 2.01 * 3.03;
-
-    auto result = this->r(0, 0, 1) * this->cr(0, 1, 1);
-    this->r(0, 0, 1) *= this->r(0, 1, 1);
-
-    this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Division)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type expected = 2.01 / 2.0;
-
-    auto result = this->cr(0, 0, 1) / ar_type{2.};
-    this->r(0, 0, 1) /= 2.;
-
-    this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Division2)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type expected = 5.06 / 4.04;
-
-    auto result = this->r(1, 0, 0) / this->cr(0, 2, 0);
-    this->r(1, 0, 0) /= this->r(0, 2, 0);
-
-    this->check_accessor_correctness(this->r, t(1, 0, 0));
-    EXPECT_NEAR(this->r(1, 0, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, UnaryMinus)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type neg_expected = this->r(2, 0, 0);
-    const ar_type expected = -neg_expected;
-
-    auto result = -this->r(2, 0, 0);
-
-    this->check_accessor_correctness(this->r);
-    EXPECT_EQ(result, expected);
-}
-
-
-TYPED_TEST(ReducedStorage3d, CanCreateSubrange)
-{
-    auto subr = this->r(gko::span{1u, 3u}, gko::span{0u, 2u}, 0u);
-
-    EXPECT_EQ(subr(0, 0, 0), this->c_st_ar(5.06));
-    EXPECT_EQ(subr(0, 1, 0), this->c_st_ar(2.08));
-    EXPECT_EQ(subr(1, 0, 0), this->c_st_ar(-2.12));
-    EXPECT_EQ(subr(1, 1, 0), this->c_st_ar(0.14));
-}
-
-
-TYPED_TEST(ReducedStorage3d, CanCreateSubrange2)
-{
-    auto subr =
-        this->cr(gko::span{1u, 3u}, gko::span{0u, 2u}, gko::span{0u, 1u});
-
-    EXPECT_EQ(subr(0, 0, 0), this->c_st_ar(5.06));
-    EXPECT_EQ(subr(0, 1, 0), this->c_st_ar(2.08));
-    EXPECT_EQ(subr(1, 0, 0), this->c_st_ar(-2.12));
-    EXPECT_EQ(subr(1, 1, 0), this->c_st_ar(0.14));
-}
-
-
-class ReducedStorageXd : public ::testing::Test {
-protected:
-    using span = gko::span;
-    using ar_type = double;
-    using st_type = float;
-    using size_type = gko::size_type;
-    static constexpr ar_type delta{::r<st_type>::value};
-
-    using accessor1d = gko::accessor::reduced_row_major<1, ar_type, st_type>;
-    using accessor2d = gko::accessor::reduced_row_major<2, ar_type, st_type>;
-    using const_accessor1d =
-        gko::accessor::reduced_row_major<1, ar_type, const st_type>;
-    using const_accessor2d =
-        gko::accessor::reduced_row_major<2, ar_type, const st_type>;
-    static_assert(std::is_same<const_accessor1d,
-                               typename accessor1d::const_accessor>::value,
-                  "Const accessors must be the same!");
-    static_assert(std::is_same<const_accessor2d,
-                               typename accessor2d::const_accessor>::value,
-                  "Const accessors must be the same!");
-
-    using reduced_storage1d = gko::range<accessor1d>;
-    using reduced_storage2d = gko::range<accessor2d>;
-    using const_reduced_storage2d = gko::range<const_accessor2d>;
-    using const_reduced_storage1d = gko::range<const_accessor1d>;
-
-    const std::array<size_type, 0> stride0{{}};
-    const std::array<size_type, 1> stride1{{4}};
-    const gko::dim<1> size_1d{8u};
-    const gko::dim<2> size_2d{2u, 4u};
-    static constexpr gko::size_type data_elements{8};
-    st_type data[data_elements]{1.1f, 2.2f, 3.3f, 4.4f,
-                                5.5f, 6.6f, 7.7f, -8.8f};
-    reduced_storage1d r1{size_1d, data};
-    reduced_storage2d r2{size_2d, data, stride1[0]};
-    const_reduced_storage1d cr1{size_1d, data, stride0};
-    const_reduced_storage2d cr2{size_2d, data, stride1};
-
-    template <typename T>
-    static ar_type c_st_ar(T val)
-    {
-        return static_cast<ar_type>(static_cast<st_type>(val));
-    }
-
-    void data_equal_except_for(int idx)
-    {
-        // clang-format off
-        if (idx != 0) { EXPECT_EQ(data[0], c_st_ar(1.1)); }
-        if (idx != 1) { EXPECT_EQ(data[1], c_st_ar(2.2)); }
-        if (idx != 2) { EXPECT_EQ(data[2], c_st_ar(3.3)); }
-        if (idx != 3) { EXPECT_EQ(data[3], c_st_ar(4.4)); }
-        if (idx != 4) { EXPECT_EQ(data[4], c_st_ar(5.5)); }
-        if (idx != 5) { EXPECT_EQ(data[5], c_st_ar(6.6)); }
-        if (idx != 6) { EXPECT_EQ(data[6], c_st_ar(7.7)); }
-        if (idx != 7) { EXPECT_EQ(data[7], c_st_ar(-8.8)); }
-        // clang-format on
-    }
-};
-
-
-TEST_F(ReducedStorageXd, CanRead)
-{
-    EXPECT_EQ(cr1(1), this->c_st_ar(2.2));
-    EXPECT_EQ(cr2(0, 1), this->c_st_ar(2.2));
-    EXPECT_EQ(r1(1), this->c_st_ar(2.2));
-    EXPECT_EQ(r2(0, 1), this->c_st_ar(2.2));
-}
-
-
-TEST_F(ReducedStorageXd, CanWrite1)
-{
-    r1(2) = 0.25;
-
-    data_equal_except_for(2);
-    EXPECT_EQ(r1(2), 0.25);  // expect exact since easy to store
-}
-
-
-TEST_F(ReducedStorageXd, CanWrite2)
-{
-    r2(1, 1) = 0.75;
-
-    data_equal_except_for(5);
-    EXPECT_EQ(r2(1, 1), 0.75);  // expect exact since easy to store
-}
-
-
-template <typename ArithmeticStorageType>
-class ScaledReducedStorage3d : public ::testing::Test {
-protected:
-    using ar_type =
-        typename std::tuple_element<0, decltype(ArithmeticStorageType{})>::type;
-    using st_type =
-        typename std::tuple_element<1, decltype(ArithmeticStorageType{})>::type;
-    // Type for `check_accessor_correctness` to forward the indices
-    using t = std::tuple<int, int, int>;
-
-    static constexpr ar_type delta{::r<ar_type>::value};
-
-    using accessor =
-        gko::accessor::scaled_reduced_row_major<3, ar_type, st_type, 0b0101>;
-    using const_accessor =
-        gko::accessor::scaled_reduced_row_major<3, ar_type, const st_type,
-                                                0b0101>;
-
-    using reduced_storage = gko::range<accessor>;
-    using const_reduced_storage = gko::range<const_accessor>;
-
-    const gko::dim<3> size{1u, 4u, 2u};
-    static constexpr gko::size_type data_elements{8};
-    static constexpr gko::size_type scalar_elements{8};
-    // clang-format off
-    st_type data[8]{
-        10, 11,
-        -12, 13,
-        14, -115,
-        6, 77
-    };
-    ar_type scalar[scalar_elements]{
-        1., 2., 3., 4., 5., 6., 7., 8.
-    };
-    // clang-format on
-    const std::array<gko::size_type, 2> storage_stride{{8, 2}};
-    const std::array<gko::size_type, 1> scalar_stride{{2}};
-    reduced_storage r{size, data, storage_stride, scalar, scalar_stride};
-    const_reduced_storage cr{size, data, scalar};
-
-    template <typename Accessor>
-    static void check_accessor_correctness(
-        const Accessor &a,
-        std::tuple<int, int, int> ignore = std::tuple<int, int, int>(99, 99,
-                                                                     99))
-    {
-        // Test for equality is fine here since they should not be modified
-        // clang-format off
-        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), ar_type{10.});   }
-        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), ar_type{22.});   }
-        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), ar_type{-12.});  }
-        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), ar_type{26.});   }
-        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), ar_type{14.});   }
-        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), ar_type{-230.}); }
-        if (ignore != t(0, 3, 0)) { EXPECT_EQ(a(0, 3, 0), ar_type{6.});    }
-        if (ignore != t(0, 3, 1)) { EXPECT_EQ(a(0, 3, 1), ar_type{154.});  }
-        // clang-format on
-    }
-};
-
-using ScaledReducedStorage3dTypes = ::testing::Types<
-    std::tuple<double, gko::int64>, std::tuple<double, gko::int32>,
-    std::tuple<double, gko::int16>, std::tuple<float, gko::int32>,
-    std::tuple<float, gko::int16>>;
-
-TYPED_TEST_SUITE(ScaledReducedStorage3d, ScaledReducedStorage3dTypes);
-
-
-TYPED_TEST(ScaledReducedStorage3d, CorrectLengths)
-{
-    EXPECT_EQ(this->r.length(0), this->size[0]);
-    EXPECT_EQ(this->r.length(1), this->size[1]);
-    EXPECT_EQ(this->r.length(2), this->size[2]);
-    EXPECT_EQ(this->r.length(3), 1);
-    EXPECT_EQ(this->r->get_size(), this->size);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CorrectStride)
-{
-    EXPECT_EQ(this->r->get_scalar_stride(), this->scalar_stride);
-    EXPECT_EQ(this->r->get_storage_stride(), this->storage_stride);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CorrectStorage)
-{
-    EXPECT_EQ(this->r->get_stored_data(), this->data);
-    EXPECT_EQ(this->r->get_const_storage(), this->data);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CorrectScale)
-{
-    EXPECT_EQ(this->r->get_scalar(), this->scalar);
-    EXPECT_EQ(this->r->get_const_scalar(), this->scalar);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CanReadData)
-{
-    this->check_accessor_correctness(this->r);
-    this->check_accessor_correctness(this->cr);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CopyFrom)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using st_type = typename TestFixture::st_type;
-    using reduced_storage = typename TestFixture::reduced_storage;
-    st_type data2[TestFixture::data_elements];
-    ar_type scale2[TestFixture::scalar_elements];
-    reduced_storage cpy(this->size, data2, scale2);
-
-    // Do not use this in regular code since the implementation is slow
-    cpy = this->r;
-
-    this->check_accessor_correctness(cpy);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CanImplicitlyConvertToConst)
-{
-    using const_reduced_storage = typename TestFixture::const_reduced_storage;
-
-    const_reduced_storage const_rs = this->r->to_const();
-    const_reduced_storage const_rs2 = this->cr;
-
-    this->check_accessor_correctness(const_rs);
-    this->check_accessor_correctness(const_rs2);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, ToConstWorks)
-{
-    using const_reduced_storage = typename TestFixture::const_reduced_storage;
-    auto cr2 = this->r->to_const();
-
-    static_assert(std::is_same<decltype(cr2), const_reduced_storage>::value,
-                  "Types must be equal!");
-    this->check_accessor_correctness(cr2);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CanCreateWithStride)
-{
-    using reduced_storage = typename TestFixture::reduced_storage;
-    using ar_type = typename TestFixture::ar_type;
-    gko::dim<3> size{2, 1, 2};
-    std::array<gko::size_type, 2> stride_storage{{5, 2}};
-    std::array<gko::size_type, 1> stride_scalar{{4}};
-
-    reduced_storage range{size, this->data, stride_storage, this->scalar,
-                          stride_scalar};
-    range(1, 0, 0) = ar_type{15};
-
-    EXPECT_EQ(range(0, 0, 0), ar_type{10});
-    EXPECT_EQ(range(0, 0, 1), ar_type{22});
-    EXPECT_EQ(range(1, 0, 0), ar_type{15});
-    EXPECT_EQ(range(1, 0, 1), ar_type{36});
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Subrange)
-{
-    auto subr = this->cr(0u, gko::span{0u, 2u}, 1u);
-
-    EXPECT_EQ(subr(0, 0, 0), 22.);
-    EXPECT_EQ(subr(0, 1, 0), 26.);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CanWriteScale)
-{
-    using ar_type = typename TestFixture::ar_type;
-
-    this->r->write_scalar(10., 0, 0, 0);
-
-    EXPECT_EQ(this->r(0, 0, 0), ar_type{100.});
-    EXPECT_EQ(this->r(0, 0, 1), ar_type{22.});
-    EXPECT_EQ(this->r(0, 1, 0), ar_type{-120.});
-    EXPECT_EQ(this->r(0, 1, 1), ar_type{26.});
-    EXPECT_EQ(this->r(0, 2, 0), ar_type{140.});
-    EXPECT_EQ(this->r(0, 2, 1), ar_type{-230.});
-    EXPECT_EQ(this->r(0, 3, 0), ar_type{60.});
-    EXPECT_EQ(this->r(0, 3, 1), ar_type{154.});
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CanReadScale)
-{
-    EXPECT_EQ(this->r->read_scalar(0, 0, 0), 1.);
-    EXPECT_EQ(this->r->read_scalar(0, 0, 1), 2.);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Addition)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = 10. + 3.;
-
-    const auto result = this->cr(0, 0, 0) + 3.;
-    this->r(0, 0, 0) += 3.;
-
-    this->check_accessor_correctness(this->r, t(0, 0, 0));
-    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Addition2)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = 10. + 22.;
-
-    const auto result = this->cr(0, 0, 0) + this->r(0, 0, 1);
-    this->r(0, 0, 0) += this->cr(0, 0, 1);
-
-    this->check_accessor_correctness(this->r, t(0, 0, 0));
-    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Subtraction)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = 22. - 2.;
-
-    const auto result = this->cr(0, 0, 1) - 2.;
-    this->r(0, 0, 1) -= 2.;
-
-    this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Subtraction2)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = -12. - 26.;
-
-    const auto result = this->cr(0, 1, 0) - this->r(0, 1, 1);
-    this->r(0, 1, 0) -= this->r(0, 1, 1);
-
-    this->check_accessor_correctness(this->r, t(0, 1, 0));
-    EXPECT_NEAR(this->r(0, 1, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Multiplication)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = 26. * 3.;
-
-    const auto result = this->cr(0, 1, 1) * 3.;
-    this->r(0, 1, 1) *= 3.;
-
-    this->check_accessor_correctness(this->r, t(0, 1, 1));
-    EXPECT_NEAR(this->r(0, 1, 1), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Multiplication2)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = 14. * 10.;
-
-    const auto result = this->r(0, 2, 0) * this->r(0, 0, 0);
-    this->r(0, 2, 0) *= this->r(0, 0, 0);
-
-    this->check_accessor_correctness(this->r, t(0, 2, 0));
-    EXPECT_NEAR(this->r(0, 2, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Division)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = 10. / 2.;
-
-    const auto result = this->cr(0, 0, 0) / 2.;
-    this->r(0, 0, 0) /= 2.;
-
-    this->check_accessor_correctness(this->r, t(0, 0, 0));
-    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Division2)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = -12. / 6.;
-
-    const auto result = this->r(0, 1, 0) / this->r(0, 3, 0);
-    this->r(0, 1, 0) /= this->r(0, 3, 0);
-
-    this->check_accessor_correctness(this->r, t(0, 1, 0));
-    EXPECT_NEAR(this->r(0, 1, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, UnaryMinus)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type neg_expected = this->r(0, 1, 1);
-    const ar_type expected = -neg_expected;
-
-    auto result = -this->r(0, 1, 1);
-
-    this->check_accessor_correctness(this->r);
-    EXPECT_EQ(result, expected);
-}
-
-
-class ScaledReducedStorageXd : public ::testing::Test {
-protected:
-    using span = gko::span;
-    using ar_type = double;
-    using st_type = int;
-    using size_type = gko::size_type;
-    static constexpr ar_type delta{0.1};
-
-    using accessor1d =
-        gko::accessor::scaled_reduced_row_major<1, ar_type, st_type, 1>;
-    using accessor2d =
-        gko::accessor::scaled_reduced_row_major<2, ar_type, st_type, 3>;
-    using const_accessor1d =
-        gko::accessor::scaled_reduced_row_major<1, ar_type, const st_type, 1>;
-    using const_accessor2d =
-        gko::accessor::scaled_reduced_row_major<2, ar_type, const st_type, 3>;
-    static_assert(std::is_same<const_accessor1d,
-                               typename accessor1d::const_accessor>::value,
-                  "Const accessors must be the same!");
-    static_assert(std::is_same<const_accessor2d,
-                               typename accessor2d::const_accessor>::value,
-                  "Const accessors must be the same!");
-
-    using reduced_storage1d = gko::range<accessor1d>;
-    using reduced_storage2d = gko::range<accessor2d>;
-    using const_reduced_storage2d = gko::range<const_accessor2d>;
-    using const_reduced_storage1d = gko::range<const_accessor1d>;
-
-    const std::array<size_type, 0> stride0{{}};
-    const std::array<size_type, 1> stride1{{4}};
-    const std::array<size_type, 1> stride_sc{{5}};
-    const gko::dim<1> size_1d{8u};
-    const gko::dim<2> size_2d{2u, 2u};
-
-    static constexpr gko::size_type data_elements{8};
-    st_type data[data_elements]{10, 22, 32, 44, 54, 66, 76, -88};
-    ar_type scalar[data_elements]{1e0,  5e-1, 1e-1, 5e-2,
-                                  1e-2, 5e-3, 1e-3, 5e-4};
-
-    reduced_storage1d r1{size_1d, data, scalar};
-    reduced_storage2d r2{size_2d, data, stride1, scalar, stride_sc};
-    const_reduced_storage1d cr1{size_1d, data, stride0, scalar};
-    const_reduced_storage2d cr2{size_2d, data, stride1, scalar, stride_sc};
-
-    void data_equal_except_for(int idx)
-    {
-        // clang-format off
-        if (idx != 0) { EXPECT_EQ(data[0], 10); }
-        if (idx != 1) { EXPECT_EQ(data[1], 22); }
-        if (idx != 2) { EXPECT_EQ(data[2], 32); }
-        if (idx != 3) { EXPECT_EQ(data[3], 44); }
-        if (idx != 4) { EXPECT_EQ(data[4], 54); }
-        if (idx != 5) { EXPECT_EQ(data[5], 66); }
-        if (idx != 6) { EXPECT_EQ(data[6], 76); }
-        if (idx != 7) { EXPECT_EQ(data[7], -88); }
-        // clang-format on
-    }
-    void scalar_equal_except_for(int idx)
-    {
-        // clang-format off
-        if (idx != 0) { EXPECT_EQ(scalar[0], ar_type{1e0}); }
-        if (idx != 1) { EXPECT_EQ(scalar[1], ar_type{5e-1}); }
-        if (idx != 2) { EXPECT_EQ(scalar[2], ar_type{1e-1}); }
-        if (idx != 3) { EXPECT_EQ(scalar[3], ar_type{5e-2}); }
-        if (idx != 4) { EXPECT_EQ(scalar[4], ar_type{1e-2}); }
-        if (idx != 5) { EXPECT_EQ(scalar[5], ar_type{5e-3}); }
-        if (idx != 6) { EXPECT_EQ(scalar[6], ar_type{1e-3}); }
-        if (idx != 7) { EXPECT_EQ(scalar[7], ar_type{5e-4}); }
-        // clang-format on
-    }
-};
-
-
-TEST_F(ScaledReducedStorageXd, CanRead)
-{
-    EXPECT_NEAR(cr1(1), 11., delta);
-    EXPECT_NEAR(cr2(0, 1), 11., delta);
-    EXPECT_NEAR(cr2(1, 1), 66e-3, delta);
-    EXPECT_NEAR(r1(1), 11., delta);
-    EXPECT_NEAR(r2(0, 1), 11., delta);
-    EXPECT_NEAR(r2(1, 1), 66e-3, delta);
-}
-
-
-TEST_F(ScaledReducedStorageXd, CanWrite1)
-{
-    r1(2) = 0.2;
-
-    data_equal_except_for(2);
-    scalar_equal_except_for(99);
-    EXPECT_NEAR(r1(2), 0.2, delta);
-}
-
-
-TEST_F(ScaledReducedStorageXd, CanWrite2)
-{
-    r2(1, 1) = 0.5;
-
-    data_equal_except_for(5);
-    scalar_equal_except_for(99);
-    EXPECT_NEAR(r2(1, 1), 0.5, delta);
-}
-
-
 }  // namespace
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 5269bbe79d7..f4fd01247d4 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -35,19 +35,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <array>
-#include <memory>
-#include <tuple>
-#include <type_traits>
-#include <utility>
 
 
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/dim.hpp>
-#include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/range.hpp>
-#include <ginkgo/core/base/range_accessor_helper.hpp>
-#include <ginkgo/core/base/range_accessor_references.hpp>
-#include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/types.hpp>
 
 
@@ -215,638 +205,6 @@ class row_major {
 };
 
 
-/**
- * The reduced_row_major class allows a storage format that is different from
- * the arithmetic format (which is returned from the brace operator).
- * As storage, the StorageType is used.
- *
- * This accessor uses row-major access. For example for three dimensions,
- * neighboring z coordinates are next to each other in memory, followed by y
- * coordinates and then x coordinates.
- *
- * @tparam Dimensionality  The number of dimensions managed by this accessor
- *
- * @tparam ArithmeticType  Value type used for arithmetic operations and
- *                         for in- and output
- *
- * @tparam StorageType  Value type used for storing the actual value to memory
- *
- * @note  This class only manages the accesses and not the memory itself.
- */
-template <int Dimensionality, typename ArithmeticType, typename StorageType>
-class reduced_row_major {
-public:
-    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
-    using storage_type = StorageType;
-    static constexpr size_type dimensionality{Dimensionality};
-    static constexpr bool is_const{std::is_const<storage_type>::value};
-    using const_accessor =
-        reduced_row_major<dimensionality, arithmetic_type, const storage_type>;
-
-    static_assert(Dimensionality >= 1,
-                  "Dimensionality must be a positive number!");
-
-    friend class range<reduced_row_major>;
-
-protected:
-    using storage_stride_type = std::array<size_type, dimensionality - 1>;
-    using reference_type =
-        reference_class::reduced_storage<arithmetic_type, storage_type>;
-
-    /**
-     * Creates the accessor for an already allocated storage space with a
-     * stride. The first stride is used for computing the index for the first
-     * index, the second stride for the second index, and so on.
-     *
-     * @param size  multidimensional size of the memory
-     * @param storage  pointer to the block of memory containing the storage
-     * @param stride  stride array used for memory accesses
-     */
-    constexpr GKO_ATTRIBUTES reduced_row_major(gko::dim<dimensionality> size,
-                                               storage_type *storage,
-                                               storage_stride_type stride)
-        : size_{size}, storage_{storage}, stride_(stride)
-    {}
-
-    /**
-     * Creates the accessor for an already allocated storage space with a
-     * stride. The first stride is used for computing the index for the first
-     * index, the second stride for the second index, and so on.
-     *
-     * @param storage  pointer to the block of memory containing the storage
-     * @param size  multidimensional size of the memory
-     * @param strides  strides used for memory accesses
-     */
-    template <typename... Strides>
-    constexpr GKO_ATTRIBUTES reduced_row_major(gko::dim<dimensionality> size,
-                                               storage_type *storage,
-                                               Strides &&... strides)
-        : reduced_row_major{
-              size, storage,
-              storage_stride_type{{std::forward<Strides>(strides)...}}}
-    {
-        static_assert(sizeof...(Strides) + 1 == dimensionality,
-                      "Number of provided Strides must be dimensionality - 1!");
-    }
-
-    /**
-     * Creates the accessor for an already allocated storage space.
-     * It is assumed that all accesses are without padding.
-     *
-     * @param storage  pointer to the block of memory containing the storage
-     * @param size  multidimensional size of the memory
-     */
-    constexpr GKO_ATTRIBUTES reduced_row_major(dim<dimensionality> size,
-                                               storage_type *storage)
-        : reduced_row_major{
-              size, storage,
-              helper::compute_default_stride_array<size_type>(size)}
-    {}
-
-    /**
-     * Creates an empty accessor (pointing nowhere with an empty size)
-     */
-    constexpr GKO_ATTRIBUTES reduced_row_major()
-        : reduced_row_major{{0, 0, 0}, nullptr}
-    {}
-
-public:
-    /**
-     * Creates a reduced_row_major range which contains a read-only version of
-     * the current accessor.
-     *
-     * @returns  a reduced_row_major major range which is read-only.
-     */
-    constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
-    {
-        return range<const_accessor>{size_, storage_, stride_};
-    }
-
-    /**
-     * Returns the length in dimension `dimension`.
-     *
-     * @param dimension  a dimension index
-     *
-     * @returns  length in dimension `dimension`
-     */
-    constexpr GKO_ATTRIBUTES size_type length(size_type dimension) const
-    {
-        return dimension < dimensionality ? size_[dimension] : 1;
-    }
-
-    /**
-     * Copies data from another accessor
-     *
-     * @warning Do not use this function since it is not optimized for a
-     *          specific executor. It will always be performed sequentially.
-     *          Please write an optimized version (adjusted to the architecture)
-     *          by iterating through the values yourself.
-     *
-     * @tparam OtherAccessor  type of the other accessor
-     *
-     * @param other  other accessor
-     */
-    template <typename OtherAccessor>
-    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
-    {
-        helper::multidim_for_each(size_, [this, &other](auto... indices) {
-            (*this)(indices...) = other(indices...);
-        });
-    }
-
-    /**
-     * Returns the stored value for the given indices. If the storage is const,
-     * a value is returned, otherwise, a reference is returned.
-     *
-     * @param indices  indices which value is supposed to access
-     *
-     * @returns  the stored value if the accessor is const (if the storage type
-     *           is const), or a reference if the accessor is non-const
-     */
-    template <typename... Indices>
-    constexpr GKO_ATTRIBUTES std::enable_if_t<
-        helper::are_all_integral<Indices...>::value,
-        std::conditional_t<is_const, arithmetic_type, reference_type>>
-    operator()(Indices &&... indices) const
-    {
-        return reference_type{storage_ +
-                              compute_index(std::forward<Indices>(indices)...)};
-    }
-
-    /**
-     * Returns a sub-range spanning the current range (x1_span, x2_span, ...)
-     *
-     * @param spans  span for the indices
-     *
-     * @returns a sub-range for the given spans.
-     */
-    template <typename... SpanTypes>
-    constexpr GKO_ATTRIBUTES
-        std::enable_if_t<helper::are_span_compatible<SpanTypes...>::value,
-                         range<reduced_row_major>>
-        operator()(SpanTypes... spans) const
-    {
-        return helper::validate_spans(size_, spans...),
-               range<reduced_row_major>{
-                   dim<dimensionality>{
-                       (span{spans}.end - span{spans}.begin)...},
-                   storage_ + compute_index((span{spans}.begin)...), stride_};
-    }
-
-    /**
-     * Returns the size of the accessor
-     *
-     * @returns the size of the accessor
-     */
-    constexpr GKO_ATTRIBUTES dim<dimensionality> get_size() const
-    {
-        return size_;
-    }
-
-    /**
-     * Returns a pointer to a stride array of size dimensionality - 1
-     *
-     * @returns returns a pointer to a stride array of size dimensionality - 1
-     */
-    GKO_ATTRIBUTES
-    constexpr const storage_stride_type &get_stride() const { return stride_; }
-
-    /**
-     * Returns the pointer to the storage data
-     *
-     * @returns the pointer to the storage data
-     */
-    constexpr GKO_ATTRIBUTES storage_type *get_stored_data() const
-    {
-        return storage_;
-    }
-
-    /**
-     * Returns a const pointer to the storage data
-     *
-     * @returns a const pointer to the storage data
-     */
-    constexpr GKO_ATTRIBUTES const storage_type *get_const_storage() const
-    {
-        return storage_;
-    }
-
-protected:
-    template <typename... Indices>
-    constexpr GKO_ATTRIBUTES size_type
-    compute_index(Indices &&... indices) const
-    {
-        static_assert(sizeof...(Indices) == dimensionality,
-                      "Number of indices must match dimensionality!");
-        return helper::compute_storage_index<size_type, dimensionality>(
-            size_, stride_, std::forward<Indices>(indices)...);
-    }
-
-private:
-    const dim<dimensionality> size_;
-    storage_type *storage_;
-    const storage_stride_type stride_;
-};
-
-
-namespace detail {
-
-
-// In case of a const type, do not provide a write function
-template <int Dimensionality, typename Accessor, typename ScaleType,
-          bool = std::is_const<ScaleType>::value>
-struct enable_write_scalar {
-    static_assert(std::is_const<ScaleType>::value,
-                  "This class must have a constant ScaleType!");
-    using scalar_type = ScaleType;
-};
-
-// In case of a non-const type, enable the write function
-template <int Dimensionality, typename Accessor, typename ScaleType>
-struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
-    static_assert(!std::is_const<ScaleType>::value,
-                  "This class must NOT have a constant ScaleType!");
-    static_assert(Dimensionality >= 1,
-                  "Dimensionality must be a positive number!");
-
-    using scalar_type = ScaleType;
-
-    /**
-     * Writes the scalar value at the given indices.
-     *
-     * @param value  value to write
-     * @param indices  indices where to write the value
-     *
-     * @returns the written value.
-     */
-    template <typename... Indices>
-    constexpr GKO_ATTRIBUTES scalar_type
-    write_scalar(scalar_type value, Indices &&... indices) const
-    {
-        static_assert(sizeof...(Indices) == Dimensionality,
-                      "Number of indices must match dimensionality!");
-        scalar_type *GKO_RESTRICT rest_scalar = self()->scalar_;
-        return rest_scalar[self()->compute_scalar_index(
-                   std::forward<Indices>(indices)...)] = value;
-    }
-
-private:
-    constexpr GKO_ATTRIBUTES const Accessor *self() const
-    {
-        return static_cast<const Accessor *>(this);
-    }
-};
-
-
-}  // namespace detail
-
-
-/**
- * The scaled_reduced_row_major class allows a storage format that is different
- * from the arithmetic format (which is returned from the brace operator).
- * Additionally, a scalar is used when reading and writing data to allow for
- * a shift in range.
- * As storage, the StorageType is used.
- *
- * This accessor uses row-major access. For example, for three dimensions,
- * neighboring z coordinates are next to each other in memory, followed by y
- * coordinates and then x coordinates.
- *
- * @tparam Dimensionality  The number of dimensions managed by this accessor
- *
- * @tparam ArithmeticType  Value type used for arithmetic operations and
- *                         for in- and output
- *
- * @tparam StorageType  Value type used for storing the actual value to memory
- *
- * @tparam ScalarMask  Binary mask that marks which indices matter for the
- *                     scalar selection (set bit means the corresponding index
- *                     needs to be considered, 0 means it is not). The least
- *                     significand bit corresponds to the last index dimension,
- *                     the second least to the second last index dimension, and
- *                     so on.
- *                     For example, the mask = 0b011101 means that for the 5d
- *                     indices (x1, x2, x3, x4, x5), (x1, x2, x3, x5) are
- *                     considered for the scalar, making the scalar itself 4d.
- *
- * @note  This class only manages the accesses and not the memory itself.
- */
-template <int Dimensionality, typename ArithmeticType, typename StorageType,
-          size_type ScalarMask>
-class scaled_reduced_row_major
-    : public detail::enable_write_scalar<
-          Dimensionality,
-          scaled_reduced_row_major<Dimensionality, ArithmeticType, StorageType,
-                                   ScalarMask>,
-          ArithmeticType> {
-public:
-    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
-    using storage_type = StorageType;
-    static constexpr size_type dimensionality{Dimensionality};
-    static constexpr size_type scalar_mask{ScalarMask};
-    static constexpr bool is_const{std::is_const<storage_type>::value};
-    using scalar_type =
-        std::conditional_t<is_const, const arithmetic_type, arithmetic_type>;
-
-    using const_accessor =
-        scaled_reduced_row_major<dimensionality, arithmetic_type,
-                                 const storage_type, ScalarMask>;
-
-    static_assert(Dimensionality >= 1,
-                  "Dimensionality must be a positive number!");
-    static_assert(dimensionality <= 32,
-                  "Only Dimensionality <= 32 is currently supported");
-
-    // Allow access to both `scalar_` and `compute_scalar_index()`
-    friend class detail::enable_write_scalar<
-        dimensionality, scaled_reduced_row_major, scalar_type>;
-    friend class range<scaled_reduced_row_major>;
-
-protected:
-    static constexpr size_type scalar_dim{
-        helper::count_mask_dimensionality<scalar_mask, dimensionality>()};
-    static constexpr size_type scalar_stride_dim{
-        scalar_dim == 0 ? 0 : (scalar_dim - 1)};
-
-    using storage_stride_type = std::array<size_type, dimensionality - 1>;
-    using scalar_stride_type = std::array<size_type, scalar_stride_dim>;
-    using reference_type =
-        reference_class::scaled_reduced_storage<arithmetic_type, StorageType>;
-
-    /**
-     * Creates the accessor for an already allocated storage space with a
-     * stride. The first stride is used for computing the index for the first
-     * index, the second stride for the second index, and so on.
-     *
-     * @param size  multidimensional size of the memory
-     * @param storage  pointer to the block of memory containing the storage
-     * @param storage_stride  stride array used for memory accesses to storage
-     * @param scalar  pointer to the block of memory containing the scalar
-     *                values.
-     * @param scalar_stride  stride array used for memory accesses to scalar
-     */
-    constexpr GKO_ATTRIBUTES scaled_reduced_row_major(
-        dim<dimensionality> size, storage_type *storage,
-        storage_stride_type storage_stride, scalar_type *scalar,
-        scalar_stride_type scalar_stride)
-        : size_{size},
-          storage_{storage},
-          storage_stride_(storage_stride),
-          scalar_{scalar},
-          scalar_stride_(scalar_stride)
-    {}
-
-    /**
-     * Creates the accessor for an already allocated storage space with a
-     * stride. The first stride is used for computing the index for the first
-     * index, the second stride for the second index, and so on.
-     *
-     * @param size  multidimensional size of the memory
-     * @param storage  pointer to the block of memory containing the storage
-     * @param stride  stride array used for memory accesses to storage
-     * @param scalar  pointer to the block of memory containing the scalar
-     *                values.
-     */
-    constexpr GKO_ATTRIBUTES scaled_reduced_row_major(
-        dim<dimensionality> size, storage_type *storage,
-        storage_stride_type stride, scalar_type *scalar)
-        : scaled_reduced_row_major{
-              size, storage, stride, scalar,
-              helper::compute_default_masked_stride_array<
-                  size_type, scalar_mask, scalar_stride_dim, dimensionality>(
-                  size)}
-    {}
-
-    /**
-     * Creates the accessor for an already allocated storage space.
-     * It is assumed that all accesses are without padding.
-     *
-     * @param size  multidimensional size of the memory
-     * @param storage  pointer to the block of memory containing the storage
-     * @param scalar  pointer to the block of memory containing the scalar
-     *                values.
-     */
-    constexpr GKO_ATTRIBUTES scaled_reduced_row_major(dim<dimensionality> size,
-                                                      storage_type *storage,
-                                                      scalar_type *scalar)
-        : scaled_reduced_row_major{
-              size, storage,
-              helper::compute_default_stride_array<size_type>(size), scalar}
-    {}
-
-    /**
-     * Creates an empty accessor (pointing nowhere with an empty size)
-     */
-    constexpr GKO_ATTRIBUTES scaled_reduced_row_major()
-        : scaled_reduced_row_major{{0, 0, 0}, nullptr, nullptr}
-    {}
-
-public:
-    /**
-     * Creates a scaled_reduced_row_major range which contains a read-only
-     * version of the current accessor.
-     *
-     * @returns  a scaled_reduced_row_major major range which is read-only.
-     */
-    constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
-    {
-        return range<const_accessor>{size_, storage_, storage_stride_, scalar_,
-                                     scalar_stride_};
-    }
-
-    /**
-     * Reads the scalar value at the given indices.
-     *
-     * @param indices  indices which data to access
-     *
-     * @returns the scalar value at the given indices.
-     */
-    template <typename... Indices>
-    constexpr GKO_ATTRIBUTES scalar_type
-    read_scalar(Indices &&... indices) const
-    {
-        const arithmetic_type *GKO_RESTRICT rest_scalar = scalar_;
-        return rest_scalar[compute_scalar_index(
-            std::forward<Indices>(indices)...)];
-    }
-
-    /**
-     * Returns the length in dimension `dimension`.
-     *
-     * @param dimension  a dimension index
-     *
-     * @returns length in dimension `dimension`
-     */
-    constexpr GKO_ATTRIBUTES size_type length(size_type dimension) const
-    {
-        return dimension < dimensionality ? size_[dimension] : 1;
-    }
-
-    /**
-     * Copies data from another accessor
-     *
-     * @warning Do not use this function since it is not optimized for a
-     *          specific executor. It will always be performed sequentially.
-     *          Please write an optimized version (adjusted to the architecture)
-     *          by iterating through the values yourself.
-     *
-     * @tparam OtherAccessor  type of the other accessor
-     *
-     * @param other  other accessor
-     */
-    template <typename OtherAccessor>
-    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
-    {
-        helper::multidim_for_each(size_, [this, &other](auto... indices) {
-            // especially inefficient if mask has not all bits set
-            this->write_scalar(other.read_scalar(indices...), indices...);
-            (*this)(indices...) = other(indices...);
-        });
-    }
-
-    /**
-     * Returns the stored value for the given indices. If the storage is const,
-     * a value is returned, otherwise, a reference is returned.
-     *
-     * @param indices  indices which value is supposed to access
-     *
-     * @returns  the stored value if the accessor is const (if the storage type
-     *           is const), or a reference if the accessor is non-const
-     */
-    template <typename... Indices>
-    constexpr GKO_ATTRIBUTES std::enable_if_t<
-        helper::are_all_integral<Indices...>::value,
-        std::conditional_t<is_const, arithmetic_type, reference_type>>
-    operator()(Indices... indices) const
-    {
-        return reference_type{storage_ + compute_index(indices...),
-                              read_scalar(indices...)};
-    }
-
-    /**
-     * Returns a sub-range spinning the current range (x1_span, x2_span, ...)
-     *
-     * @param spans  span for the indices
-     *
-     * @returns a sub-range for the given spans.
-     */
-    template <typename... SpanTypes>
-    constexpr GKO_ATTRIBUTES
-        std::enable_if_t<helper::are_span_compatible<SpanTypes...>::value,
-                         range<scaled_reduced_row_major>>
-        operator()(SpanTypes... spans) const
-    {
-        return helper::validate_spans(size_, spans...),
-               range<scaled_reduced_row_major>{
-                   dim<dimensionality>{
-                       (span{spans}.end - span{spans}.begin)...},
-                   storage_ + compute_index((span{spans}.begin)...),
-                   storage_stride_,
-                   scalar_ + compute_scalar_index(span{spans}.begin...),
-                   scalar_stride_};
-    }
-
-    /**
-     * Returns the size of the accessor
-     *
-     * @returns the size of the accessor
-     */
-    constexpr GKO_ATTRIBUTES dim<dimensionality> get_size() const
-    {
-        return size_;
-    }
-
-    /**
-     * Returns a const reference to the storage stride array of size
-     * dimensionality - 1
-     *
-     * @returns a const reference to the storage stride array of size
-     *          dimensionality - 1
-     */
-    constexpr GKO_ATTRIBUTES const storage_stride_type &get_storage_stride()
-        const
-    {
-        return storage_stride_;
-    }
-
-    /**
-     * Returns a const reference to the scalar stride array
-     *
-     * @returns a const reference to the scalar stride array
-     */
-    constexpr GKO_ATTRIBUTES const scalar_stride_type &get_scalar_stride() const
-    {
-        return scalar_stride_;
-    }
-
-    /**
-     * Returns the pointer to the storage data
-     *
-     * @returns the pointer to the storage data
-     */
-    constexpr GKO_ATTRIBUTES storage_type *get_stored_data() const
-    {
-        return storage_;
-    }
-
-    /**
-     * Returns a const pointer to the storage data
-     *
-     * @returns a const pointer to the storage data
-     */
-    constexpr GKO_ATTRIBUTES const storage_type *get_const_storage() const
-    {
-        return storage_;
-    }
-
-    /**
-     * Returns the pointer to the scalar data
-     *
-     * @returns the pointer to the scalar data
-     */
-    constexpr GKO_ATTRIBUTES scalar_type *get_scalar() const { return scalar_; }
-
-    /**
-     * Returns a const pointer to the scalar data
-     *
-     * @returns a const pointer to the scalar data
-     */
-    constexpr GKO_ATTRIBUTES const scalar_type *get_const_scalar() const
-    {
-        return scalar_;
-    }
-
-protected:
-    template <typename... Indices>
-    constexpr GKO_ATTRIBUTES size_type
-    compute_index(Indices &&... indices) const
-    {
-        static_assert(sizeof...(Indices) == dimensionality,
-                      "Number of indices must match dimensionality!");
-        return helper::compute_storage_index<size_type, dimensionality>(
-            size_, storage_stride_, std::forward<Indices>(indices)...);
-    }
-
-    template <typename... Indices>
-    constexpr GKO_ATTRIBUTES size_type
-    compute_scalar_index(Indices &&... indices) const
-    {
-        static_assert(sizeof...(Indices) == dimensionality,
-                      "Number of indices must match dimensionality!");
-        return helper::compute_masked_index<size_type, scalar_mask,
-                                            scalar_stride_dim>(
-            size_, scalar_stride_, std::forward<Indices>(indices)...);
-    }
-
-private:
-    const dim<dimensionality> size_;
-    storage_type *storage_;
-    const storage_stride_type storage_stride_;
-    scalar_type *scalar_;
-    const scalar_stride_type scalar_stride_;
-};
-
-
 }  // namespace accessor
 }  // namespace gko
 
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index f1a467e1ae0..42c535cf4e4 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -54,8 +54,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/perturbation.hpp>
 #include <ginkgo/core/base/polymorphic_object.hpp>
 #include <ginkgo/core/base/range.hpp>
-#include <ginkgo/core/base/range_accessor_helper.hpp>
-#include <ginkgo/core/base/range_accessor_references.hpp>
 #include <ginkgo/core/base/range_accessors.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/temporary_clone.hpp>

From 8b6107e319f94ef189a650885c96bdade3f4d89b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 27 Nov 2020 04:11:53 +0100
Subject: [PATCH 0233/1008] Add numeric_limits to gko::half

Use `std::numeric_limits<>::epsilon()` for the test reduction factor
---
 core/base/extended_float.hpp       | 31 ++++++++++++++++++++++++------
 core/test/utils.hpp                | 11 +++++------
 include/ginkgo/core/base/types.hpp |  8 --------
 3 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/core/base/extended_float.hpp b/core/base/extended_float.hpp
index fb58734e594..3961f47c3bd 100644
--- a/core/base/extended_float.hpp
+++ b/core/base/extended_float.hpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_BASE_EXTENDED_FLOAT_HPP_
 
 
+#include <limits>
 #include <type_traits>
 
 
@@ -67,8 +68,7 @@ namespace detail {
 
 
 template <std::size_t, typename = void>
-struct uint_of_impl {
-};
+struct uint_of_impl {};
 
 template <std::size_t Bits>
 struct uint_of_impl<Bits, std::enable_if_t<(Bits <= 16)>> {
@@ -90,8 +90,7 @@ using uint_of = typename uint_of_impl<Bits>::type;
 
 
 template <typename T>
-struct basic_float_traits {
-};
+struct basic_float_traits {};
 
 template <>
 struct basic_float_traits<float16> {
@@ -553,9 +552,29 @@ class complex<gko::truncated<T, NumComponents>> {
 
 
 template <>
-struct is_scalar<gko::half> : std::true_type {
-};
+struct is_scalar<gko::half> : std::true_type {};
+
 
+template <>
+struct numeric_limits<gko::half> {
+    static constexpr bool is_specializec{true};
+    static constexpr bool is_signed{true};
+    static constexpr bool is_integer{false};
+    static constexpr bool is_exact{false};
+    static constexpr bool is_bounded{true};
+    static constexpr bool is_modulo{false};
+    static constexpr int digits{
+        gko::detail::float_traits<gko::half>::significand_bits + 1};
+    // 3/10 is approx. log_10(2)
+    static constexpr int digits10{digits * 3 / 10};
+
+    // Note: gko::half can't return gko::half here because it does not have
+    //       a constexpr constructor.
+    static constexpr float epsilon()
+    {
+        return gko::detail::float_traits<gko::half>::eps;
+    }
+};
 
 }  // namespace std
 
diff --git a/core/test/utils.hpp b/core/test/utils.hpp
index 0b314a59463..843955d219e 100644
--- a/core/test/utils.hpp
+++ b/core/test/utils.hpp
@@ -36,6 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <complex>
 #include <initializer_list>
+#include <limits>
 #include <type_traits>
 
 
@@ -95,12 +96,10 @@ using ComplexValueIndexTypes =
 
 template <typename Precision, typename OutputType>
 struct reduction_factor {
-    static constexpr remove_complex<OutputType> value =
-        std::is_same<gko::remove_complex<Precision>, gko::half>::value
-            ? 1.0e-2
-            : std::is_same<gko::remove_complex<Precision>, float>::value
-                  ? 1.0e-6
-                  : 1.0e-14;
+    using nc_output = remove_complex<OutputType>;
+    using nc_precision = remove_complex<Precision>;
+    static constexpr nc_output value{
+        std::numeric_limits<nc_precision>::epsilon() * nc_output{10}};
 };
 
 
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index f2d9ccb6310..1a23ade2b0c 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -93,14 +93,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif  // defined(_WIN32) || defined(__CYGWIN__)
 
 
-/**
- * Encapsulates all arguments in one macro, so a comma inside an argument (e.g.
- * std::tuple<int, float>) still works as an argument and is not considered as
- * multiple arguments.
- */
-#define GKO_UNPACK(...) __VA_ARGS__
-
-
 namespace gko {
 
 

From cd7816577d9bd0cd36bcdb34ae85b79c76f2c875 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 27 Nov 2020 05:03:18 +0100
Subject: [PATCH 0234/1008] Move compute_default_stride_array to dim header

---
 core/base/accessor_helper.hpp    | 46 --------------------
 core/base/accessors.hpp          |  9 ++--
 core/test/base/dim.cpp           | 61 ++++++++++++++++++++++++++
 include/ginkgo/core/base/dim.hpp | 73 ++++++++++++++++++++++++++++++++
 4 files changed, 138 insertions(+), 51 deletions(-)

diff --git a/core/base/accessor_helper.hpp b/core/base/accessor_helper.hpp
index d837812700c..3e62f0a69d8 100644
--- a/core/base/accessor_helper.hpp
+++ b/core/base/accessor_helper.hpp
@@ -101,41 +101,6 @@ struct row_major_helper_s<ValueType, total_dim, total_dim> {
 };
 
 
-template <typename ValueType, size_type iter, size_type N>
-constexpr GKO_ATTRIBUTES std::enable_if_t<iter == N, ValueType>
-mult_size_upwards(const dim<N> &)
-{
-    return 1;
-}
-
-template <typename ValueType, size_type iter, size_type N>
-constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), ValueType>
-mult_size_upwards(const dim<N> &size)
-{
-    return size[iter] * mult_size_upwards<ValueType, iter + 1>(size);
-}
-
-
-template <typename ValueType, size_type iter = 1, size_type N, typename... Args>
-constexpr GKO_ATTRIBUTES
-    std::enable_if_t<N == 0 || (iter == N && iter == sizeof...(Args) + 1),
-                     std::array<ValueType, N == 0 ? 0 : N - 1>>
-    compute_default_stride_array(const dim<N> &, Args &&... args)
-{
-    return {{std::forward<Args>(args)...}};
-}
-
-template <typename ValueType, size_type iter = 1, size_type N, typename... Args>
-constexpr GKO_ATTRIBUTES std::enable_if_t<
-    (iter < N) && (iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
-compute_default_stride_array(const dim<N> &size, Args &&... args)
-{
-    return compute_default_stride_array<ValueType, iter + 1>(
-        size, std::forward<Args>(args)...,
-        mult_size_upwards<ValueType, iter>(size));
-}
-
-
 }  // namespace detail
 
 
@@ -154,17 +119,6 @@ constexpr GKO_ATTRIBUTES ValueType compute_storage_index(
 }
 
 
-/**
- * Computes the default stride array from a size, assuming there is no
- * padding
- */
-template <typename ValueType, size_type N>
-constexpr GKO_ATTRIBUTES auto compute_default_stride_array(const dim<N> &size)
-{
-    return detail::compute_default_stride_array<ValueType>(size);
-}
-
-
 namespace detail {
 
 
diff --git a/core/base/accessors.hpp b/core/base/accessors.hpp
index 808d210fa45..bf2e7a950e7 100644
--- a/core/base/accessors.hpp
+++ b/core/base/accessors.hpp
@@ -142,9 +142,8 @@ class reduced_row_major {
      */
     constexpr GKO_ATTRIBUTES reduced_row_major(dim<dimensionality> size,
                                                storage_type *storage)
-        : reduced_row_major{
-              size, storage,
-              helper::compute_default_stride_array<size_type>(size)}
+        : reduced_row_major{size, storage,
+                            compute_default_stride_array<size_type>(size)}
     {}
 
     /**
@@ -474,8 +473,8 @@ class scaled_reduced_row_major
                                                       storage_type *storage,
                                                       scalar_type *scalar)
         : scaled_reduced_row_major{
-              size, storage,
-              helper::compute_default_stride_array<size_type>(size), scalar}
+              size, storage, compute_default_stride_array<size_type>(size),
+              scalar}
     {}
 
     /**
diff --git a/core/test/base/dim.cpp b/core/test/base/dim.cpp
index 6d48bcab794..6e60a975b7e 100644
--- a/core/test/base/dim.cpp
+++ b/core/test/base/dim.cpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/dim.hpp>
 
 
+#include <array>
 #include <memory>
 
 
@@ -79,6 +80,66 @@ TEST(Dim, ConstructsNullObject)
 }
 
 
+TEST(Dim, ComputeDefaultStrideArrayConstexpr)
+{
+    using value_type = unsigned int;
+    constexpr int dimens{4};
+    constexpr gko::dim<dimens, int> d{2, 3, 5, 7};
+    constexpr std::array<value_type, dimens - 1> expected{3 * 5 * 7, 5 * 7, 7};
+
+    constexpr auto res = gko::compute_default_stride_array<value_type>(d);
+
+    ::testing::StaticAssertTypeEq<decltype(res),
+                                  const std::array<value_type, dimens - 1>>();
+    ASSERT_EQ(res, expected);
+}
+
+
+TEST(Dim, ComputeDefaultStrideArray)
+{
+    using value_type = gko::size_type;
+    constexpr int dimens{4};
+    gko::dim<dimens, int> d{2, 3, 5, 7};
+    std::array<value_type, dimens - 1> expected{3 * 5 * 7, 5 * 7, 7};
+
+    auto res = gko::compute_default_stride_array<value_type>(d);
+
+    ::testing::StaticAssertTypeEq<decltype(res),
+                                  std::array<value_type, dimens - 1>>();
+    ASSERT_EQ(res, expected);
+}
+
+
+TEST(Dim, ComputeDefaultStrideArrayEmpty)
+{
+    using value_type = gko::size_type;
+    constexpr int dimens{1};
+    gko::dim<dimens> d{2};
+    std::array<value_type, 0> expected{};
+
+    auto res = gko::compute_default_stride_array<value_type>(d);
+
+    ::testing::StaticAssertTypeEq<decltype(res),
+                                  std::array<value_type, dimens - 1>>();
+    ASSERT_EQ(res, expected);
+}
+
+
+TEST(Dim, ComputeDefaultStrideArrayEmptyConstexpr)
+{
+    using value_type = gko::size_type;
+    constexpr int dimens{1};
+    constexpr gko::dim<dimens> d{2};
+    constexpr std::array<value_type, 0> expected{};
+
+    constexpr auto res = gko::compute_default_stride_array<value_type>(d);
+
+    ::testing::StaticAssertTypeEq<decltype(res),
+                                  const std::array<value_type, dimens - 1>>();
+    ASSERT_EQ(res, expected);
+}
+
+
 class dim_manager {
 public:
     using dim = gko::dim<3>;
diff --git a/include/ginkgo/core/base/dim.hpp b/include/ginkgo/core/base/dim.hpp
index 0881d954aa0..64645fe3f8d 100644
--- a/include/ginkgo/core/base/dim.hpp
+++ b/include/ginkgo/core/base/dim.hpp
@@ -34,6 +34,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_PUBLIC_CORE_BASE_DIM_HPP_
 
 
+#include <array>
+
+
 #include <ginkgo/core/base/types.hpp>
 
 
@@ -242,6 +245,76 @@ constexpr GKO_ATTRIBUTES GKO_INLINE dim<2, DimensionType> transpose(
 }
 
 
+namespace detail {
+
+
+template <typename ValueType, size_type iter, size_type N,
+          typename DimensionType>
+constexpr GKO_ATTRIBUTES std::enable_if_t<iter == N, ValueType>
+mult_dim_upwards(const dim<N, DimensionType> &)
+{
+    return 1;
+}
+
+template <typename ValueType, size_type iter, size_type N,
+          typename DimensionType>
+constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), ValueType>
+mult_dim_upwards(const dim<N, DimensionType> &size)
+{
+    return size[iter] * mult_dim_upwards<ValueType, iter + 1>(size);
+}
+
+
+template <typename ValueType, size_type iter = 1, size_type N,
+          typename DimensionType, typename... Args>
+constexpr GKO_ATTRIBUTES
+    std::enable_if_t<N == 0 || (iter == N && iter == sizeof...(Args) + 1),
+                     std::array<ValueType, N == 0 ? 0 : N - 1>>
+    compute_default_stride_array(const dim<N, DimensionType> &, Args &&... args)
+{
+    return {{std::forward<Args>(args)...}};
+}
+
+template <typename ValueType, size_type iter = 1, size_type N,
+          typename DimensionType, typename... Args>
+constexpr GKO_ATTRIBUTES std::enable_if_t<
+    (iter < N) && (iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
+compute_default_stride_array(const dim<N, DimensionType> &size, Args &&... args)
+{
+    return compute_default_stride_array<ValueType, iter + 1>(
+        size, std::forward<Args>(args)...,
+        mult_dim_upwards<ValueType, iter>(size));
+}
+
+
+}  // namespace detail
+
+
+/**
+ * Computes the default stride array from a given size, assuming there is no
+ * padding.
+ *
+ * Example: dim<4> size={2, 3, 5, 7} results in a return value of:
+ * std::array<ValueType, 3> = {3*5*7, 5*7, 7}
+ *
+ * @tparam ValueType  value type of the values in the returned array
+ *
+ * @tparam dimensions  number of dimensions in the given size
+ *
+ * @tparam DimensionType  value type of the stored size
+ *
+ * @returns an std::array<ValueType, dimensions - 1> with the stride
+ *          information.
+ */
+template <typename ValueType, size_type dimensions, typename DimensionType>
+constexpr GKO_ATTRIBUTES
+    std::array<ValueType, (dimensions > 0 ? dimensions - 1 : 0)>
+    compute_default_stride_array(const dim<dimensions, DimensionType> &size)
+{
+    return detail::compute_default_stride_array<ValueType>(size);
+}
+
+
 }  // namespace gko
 
 

From 215f03b13e4950bf4c1173fdf9a76570e59e857a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 27 Nov 2020 10:39:22 +0100
Subject: [PATCH 0235/1008] Move are_all_integral to types.hpp

---
 core/base/accessor_helper.hpp      | 13 -------------
 core/base/accessors.hpp            |  4 ++--
 include/ginkgo/core/base/types.hpp | 18 ++++++++++++++++++
 3 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/core/base/accessor_helper.hpp b/core/base/accessor_helper.hpp
index 3e62f0a69d8..c8b704adf7b 100644
--- a/core/base/accessor_helper.hpp
+++ b/core/base/accessor_helper.hpp
@@ -362,19 +362,6 @@ constexpr GKO_ATTRIBUTES auto compute_default_masked_stride_array(
 }
 
 
-/**
- * Evaluates if all Args fulfill std::is_integral
- */
-template <typename... Args>
-struct are_all_integral : public std::true_type {};
-
-template <typename First, typename... Args>
-struct are_all_integral<First, Args...>
-    : public std::conditional<std::is_integral<std::decay_t<First>>::value,
-                              are_all_integral<Args...>,
-                              std::false_type>::type {};
-
-
 namespace detail {
 
 
diff --git a/core/base/accessors.hpp b/core/base/accessors.hpp
index bf2e7a950e7..a9005671481 100644
--- a/core/base/accessors.hpp
+++ b/core/base/accessors.hpp
@@ -208,7 +208,7 @@ class reduced_row_major {
      */
     template <typename... Indices>
     constexpr GKO_ATTRIBUTES std::enable_if_t<
-        helper::are_all_integral<Indices...>::value,
+        are_all_integral<Indices...>::value,
         std::conditional_t<is_const, arithmetic_type, reference_type>>
     operator()(Indices &&... indices) const
     {
@@ -558,7 +558,7 @@ class scaled_reduced_row_major
      */
     template <typename... Indices>
     constexpr GKO_ATTRIBUTES std::enable_if_t<
-        helper::are_all_integral<Indices...>::value,
+        are_all_integral<Indices...>::value,
         std::conditional_t<is_const, arithmetic_type, reference_type>>
     operator()(Indices... indices) const
     {
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 1a23ade2b0c..2b665371285 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -193,6 +193,24 @@ using default_precision = double;
 constexpr size_type byte_size = CHAR_BIT;
 
 
+/**
+ * Evaluates if all template arguments Args fulfill std::is_integral. If that is
+ * the case, this class inherits from `std::true_type`, otherwise, it inherits
+ * from `std::false_type`.
+ * If no values are passed in, `std::true_type` is inherited from.
+ *
+ * @tparam Args...  Arguments to test for std::is_integral
+ */
+template <typename... Args>
+struct are_all_integral : public std::true_type {};
+
+template <typename First, typename... Args>
+struct are_all_integral<First, Args...>
+    : public std::conditional<std::is_integral<std::decay_t<First>>::value,
+                              are_all_integral<Args...>,
+                              std::false_type>::type {};
+
+
 /**
  * This class is used to encode storage precisions of low precision algorithms.
  *

From e77857de371bc57ba4bc1da49976b4556c39410d Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Mon, 7 Dec 2020 12:10:22 +0000
Subject: [PATCH 0236/1008] Format files
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thoasm@users.noreply.github.com>
---
 core/base/accessor_helper.hpp                |  1 +
 core/base/accessor_references.hpp            | 15 ++++++++-------
 core/base/accessors.hpp                      |  1 +
 core/test/base/accessors_reduced_storage.cpp |  8 +++-----
 4 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/core/base/accessor_helper.hpp b/core/base/accessor_helper.hpp
index c8b704adf7b..b8d4f1f28f6 100644
--- a/core/base/accessor_helper.hpp
+++ b/core/base/accessor_helper.hpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifndef GKO_CORE_BASE_ACCESSOR_HELPER_HPP_
 #define GKO_CORE_BASE_ACCESSOR_HELPER_HPP_
 
+
 #include <array>
 #include <tuple>
 #include <type_traits>
diff --git a/core/base/accessor_references.hpp b/core/base/accessor_references.hpp
index 00825a5e618..712d4ea7a5a 100644
--- a/core/base/accessor_references.hpp
+++ b/core/base/accessor_references.hpp
@@ -241,8 +241,8 @@ class reduced_storage
         return static_cast<arithmetic_type>(*r_ptr);
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type
-    operator=(arithmetic_type val) &&noexcept
+    constexpr GKO_ATTRIBUTES arithmetic_type operator=(arithmetic_type val) &&
+        noexcept
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         *r_ptr = static_cast<storage_type>(val);
@@ -256,8 +256,8 @@ class reduced_storage
         return static_cast<arithmetic_type>(*this);
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type
-    operator=(reduced_storage &&ref) &&noexcept
+    constexpr GKO_ATTRIBUTES arithmetic_type operator=(reduced_storage &&ref) &&
+        noexcept
     {
         std::move(*this) = static_cast<arithmetic_type>(ref);
         return static_cast<arithmetic_type>(*this);
@@ -352,8 +352,8 @@ class scaled_reduced_storage
         return static_cast<arithmetic_type>(*r_ptr) * scalar_;
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type
-    operator=(arithmetic_type val) &&noexcept
+    constexpr GKO_ATTRIBUTES arithmetic_type operator=(arithmetic_type val) &&
+        noexcept
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         *r_ptr = static_cast<storage_type>(val / scalar_);
@@ -368,7 +368,8 @@ class scaled_reduced_storage
     }
 
     constexpr GKO_ATTRIBUTES arithmetic_type
-    operator=(scaled_reduced_storage &&ref) &&noexcept
+        operator=(scaled_reduced_storage &&ref) &&
+        noexcept
     {
         std::move(*this) = static_cast<arithmetic_type>(ref);
         return static_cast<arithmetic_type>(*this);
diff --git a/core/base/accessors.hpp b/core/base/accessors.hpp
index a9005671481..9345700afcd 100644
--- a/core/base/accessors.hpp
+++ b/core/base/accessors.hpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifndef GKO_CORE_BASE_ACCESSORS_HPP_
 #define GKO_CORE_BASE_ACCESSORS_HPP_
 
+
 #include <array>
 #include <memory>
 #include <type_traits>
diff --git a/core/test/base/accessors_reduced_storage.cpp b/core/test/base/accessors_reduced_storage.cpp
index b8fd30b46e7..3694cb347e0 100644
--- a/core/test/base/accessors_reduced_storage.cpp
+++ b/core/test/base/accessors_reduced_storage.cpp
@@ -30,20 +30,18 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/base/accessors.hpp"
+#include <tuple>
+#include <type_traits>
 
 
 #include <gtest/gtest.h>
 
 
-#include <tuple>
-#include <type_traits>
-
-
 #include <ginkgo/core/base/dim.hpp>
 #include <ginkgo/core/base/types.hpp>
 
 
+#include "core/base/accessors.hpp"
 #include "core/base/extended_float.hpp"
 #include "core/test/utils.hpp"
 

From 218c176dba3bd2fed47d35efdb760a1268de2944 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 11 Jan 2021 12:18:48 +0100
Subject: [PATCH 0237/1008] Only add write scalar option when non-const

Previously, a write_scalar function always existed, even though the
scalar value was const, therefore not writable.
Now, the write_scalar function only exists for the proper non-const
accessor.
---
 core/base/accessors.hpp | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/core/base/accessors.hpp b/core/base/accessors.hpp
index 9345700afcd..52ccf5c5ecb 100644
--- a/core/base/accessors.hpp
+++ b/core/base/accessors.hpp
@@ -297,23 +297,19 @@ namespace detail {
 
 
 // In case of a const type, do not provide a write function
-template <int Dimensionality, typename Accessor, typename ScaleType,
-          bool = std::is_const<ScaleType>::value>
+template <int Dimensionality, typename Accessor, typename ScalarType,
+          bool = std::is_const<ScalarType>::value>
 struct enable_write_scalar {
-    static_assert(std::is_const<ScaleType>::value,
-                  "This class must have a constant ScaleType!");
-    using scalar_type = ScaleType;
+    using scalar_type = ScalarType;
 };
 
 // In case of a non-const type, enable the write function
-template <int Dimensionality, typename Accessor, typename ScaleType>
-struct enable_write_scalar<Dimensionality, Accessor, ScaleType, false> {
-    static_assert(!std::is_const<ScaleType>::value,
-                  "This class must NOT have a constant ScaleType!");
+template <int Dimensionality, typename Accessor, typename ScalarType>
+struct enable_write_scalar<Dimensionality, Accessor, ScalarType, false> {
     static_assert(Dimensionality >= 1,
                   "Dimensionality must be a positive number!");
 
-    using scalar_type = ScaleType;
+    using scalar_type = ScalarType;
 
     /**
      * Writes the scalar value at the given indices.
@@ -382,7 +378,7 @@ class scaled_reduced_row_major
           Dimensionality,
           scaled_reduced_row_major<Dimensionality, ArithmeticType, StorageType,
                                    ScalarMask>,
-          ArithmeticType> {
+          ArithmeticType, std::is_const<StorageType>::value> {
 public:
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
     using storage_type = StorageType;

From f592f35fd96fb51bbef27bc02db75ff246c67544 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 13 Jan 2021 04:49:00 +0100
Subject: [PATCH 0238/1008] Include complex numbers in  accessor tests

---
 core/base/accessor_references.hpp            | 57 ++++++++++++++---
 core/test/base/accessors_reduced_storage.cpp | 65 +++++++++++---------
 2 files changed, 86 insertions(+), 36 deletions(-)

diff --git a/core/base/accessor_references.hpp b/core/base/accessor_references.hpp
index 712d4ea7a5a..358078ebd32 100644
--- a/core/base/accessor_references.hpp
+++ b/core/base/accessor_references.hpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <utility>
 
 
+#include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/types.hpp>
 
@@ -172,6 +173,27 @@ struct enable_reference_operators {
     GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator-=, -)
 #undef GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD
 
+#define GKO_REFERENCE_COMPARISON_OPERATOR_OVERLOAD(_op)                  \
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES bool \
+    operator _op(const Reference &ref1, const Reference &ref2)           \
+    {                                                                    \
+        return to_value_type<arithmetic_type>(ref1)                      \
+            _op to_value_type<arithmetic_type>(ref2);                    \
+    }                                                                    \
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES bool \
+    operator _op(const Reference &ref, const arithmetic_type &a)         \
+    {                                                                    \
+        return to_value_type<arithmetic_type>(ref) _op a;                \
+    }                                                                    \
+    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES bool \
+    operator _op(const arithmetic_type &a, const Reference &ref)         \
+    {                                                                    \
+        return a _op to_value_type<arithmetic_type>(ref);                \
+    }
+
+    GKO_REFERENCE_COMPARISON_OPERATOR_OVERLOAD(==)
+#undef GKO_REFERENCE_COMPARISON_OPERATOR_OVERLOAD
+
     friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES
         arithmetic_type
         operator-(const Reference &ref)
@@ -241,8 +263,8 @@ class reduced_storage
         return static_cast<arithmetic_type>(*r_ptr);
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type operator=(arithmetic_type val) &&
-        noexcept
+    constexpr GKO_ATTRIBUTES arithmetic_type
+    operator=(arithmetic_type val) &&noexcept
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         *r_ptr = static_cast<storage_type>(val);
@@ -256,8 +278,8 @@ class reduced_storage
         return static_cast<arithmetic_type>(*this);
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type operator=(reduced_storage &&ref) &&
-        noexcept
+    constexpr GKO_ATTRIBUTES arithmetic_type
+    operator=(reduced_storage &&ref) &&noexcept
     {
         std::move(*this) = static_cast<arithmetic_type>(ref);
         return static_cast<arithmetic_type>(*this);
@@ -304,6 +326,16 @@ class reduced_storage<ArithmeticType, const StorageType>
 };
 
 
+template <typename ArithmeticType, typename StorageType>
+constexpr remove_complex<ArithmeticType> abs(
+    const reduced_storage<ArithmeticType, StorageType> &ref)
+{
+    // For some reason necessary to get the correct abs function
+    using gko::abs;
+    return abs(static_cast<ArithmeticType>(ref));
+}
+
+
 /**
  * Reference class for a different storage than arithmetic type with the
  * addition of a scaling factor. The conversion between both formats is done
@@ -352,8 +384,8 @@ class scaled_reduced_storage
         return static_cast<arithmetic_type>(*r_ptr) * scalar_;
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type operator=(arithmetic_type val) &&
-        noexcept
+    constexpr GKO_ATTRIBUTES arithmetic_type
+    operator=(arithmetic_type val) &&noexcept
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         *r_ptr = static_cast<storage_type>(val / scalar_);
@@ -368,8 +400,7 @@ class scaled_reduced_storage
     }
 
     constexpr GKO_ATTRIBUTES arithmetic_type
-        operator=(scaled_reduced_storage &&ref) &&
-        noexcept
+    operator=(scaled_reduced_storage &&ref) &&noexcept
     {
         std::move(*this) = static_cast<arithmetic_type>(ref);
         return static_cast<arithmetic_type>(*this);
@@ -419,6 +450,16 @@ class scaled_reduced_storage<ArithmeticType, const StorageType>
 };
 
 
+template <typename ArithmeticType, typename StorageType>
+constexpr remove_complex<ArithmeticType> abs(
+    const scaled_reduced_storage<ArithmeticType, StorageType> &ref)
+{
+    // For some reason necessary to get the correct abs function
+    using gko::abs;
+    return abs(static_cast<ArithmeticType>(ref));
+}
+
+
 }  // namespace reference_class
 }  // namespace accessor
 }  // namespace gko
diff --git a/core/test/base/accessors_reduced_storage.cpp b/core/test/base/accessors_reduced_storage.cpp
index 3694cb347e0..3bfe18c88e0 100644
--- a/core/test/base/accessors_reduced_storage.cpp
+++ b/core/test/base/accessors_reduced_storage.cpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
 
 
@@ -56,9 +57,10 @@ class ReducedStorage3d : public ::testing::Test {
         typename std::tuple_element<0, decltype(ArithmeticStorageType{})>::type;
     using st_type =
         typename std::tuple_element<1, decltype(ArithmeticStorageType{})>::type;
-    static constexpr ar_type delta{std::is_same<ar_type, st_type>::value
-                                       ? 0
-                                       : ::r<st_type, ar_type>::value};
+    using rcar_type = gko::remove_complex<ar_type>;
+    static constexpr rcar_type delta{std::is_same<ar_type, st_type>::value
+                                         ? 0
+                                         : ::r<st_type, ar_type>::value};
 
     // Type for `check_accessor_correctness` to forward the indices
     using t = std::tuple<int, int, int>;
@@ -141,13 +143,10 @@ class ReducedStorage3d : public ::testing::Test {
 using ReducedStorage3dTypes =
     ::testing::Types<std::tuple<double, double>, std::tuple<double, float>,
                      std::tuple<float, float>, std::tuple<double, gko::half>,
-                     std::tuple<float, gko::half>
-                     /*,
+                     std::tuple<float, gko::half>,
                      std::tuple<std::complex<double>, std::complex<double>>,
                      std::tuple<std::complex<double>, std::complex<float>>,
-                     std::tuple<std::complex<float>, std::complex<float>>
-                     */
-                     >;
+                     std::tuple<std::complex<float>, std::complex<float>>>;
 
 TYPED_TEST_SUITE(ReducedStorage3d, ReducedStorage3dTypes);
 
@@ -251,7 +250,7 @@ TYPED_TEST(ReducedStorage3d, CanWriteData)
     this->r(0, 1, 0) = 100.25;
 
     this->check_accessor_correctness(this->r, t(0, 1, 0));
-    EXPECT_NEAR(this->r(0, 1, 0), 100.25, TestFixture::delta);
+    EXPECT_EQ(this->r(0, 1, 0), this->c_st_ar(100.25));
 }
 
 
@@ -264,7 +263,7 @@ TYPED_TEST(ReducedStorage3d, Assignment)
     this->r(0, 0, 1) = expected;
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+    EXPECT_EQ(this->r(0, 0, 1), this->c_st_ar(expected));
 }
 
 
@@ -277,7 +276,7 @@ TYPED_TEST(ReducedStorage3d, Assignment2)
     this->r(0, 0, 1) = this->r(0, 1, 0);
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+    EXPECT_EQ(this->r(0, 0, 1), this->c_st_ar(expected));
 }
 
 
@@ -285,14 +284,15 @@ TYPED_TEST(ReducedStorage3d, Addition)
 {
     using t = typename TestFixture::t;
     using ar_type = typename TestFixture::ar_type;
+    using gko::abs;
     const ar_type expected = 1.2 + 2.01;
 
-    auto result = this->r(0, 0, 1) + ar_type{1.2};
+    ar_type result = this->r(0, 0, 1) + ar_type{1.2};
     this->r(0, 0, 1) += 1.2;
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
+    EXPECT_NEAR(abs(this->r(0, 0, 1)), abs(expected), TestFixture::delta);
+    EXPECT_NEAR(abs(result), abs(expected), TestFixture::delta);
 }
 
 
@@ -300,14 +300,16 @@ TYPED_TEST(ReducedStorage3d, Addition2)
 {
     using t = typename TestFixture::t;
     using ar_type = typename TestFixture::ar_type;
+    using gko::abs;
     const ar_type expected = 2.01 + -1.02;
 
     auto result = this->r(0, 0, 1) + this->r(0, 1, 0);
     this->r(0, 0, 1) += this->r(0, 1, 0);
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
+    EXPECT_NEAR(abs(this->r(0, 0, 1)), abs(this->c_st_ar(expected)),
+                TestFixture::delta);
+    EXPECT_NEAR(abs(result), abs(this->c_st_ar(expected)), TestFixture::delta);
 }
 
 
@@ -315,14 +317,16 @@ TYPED_TEST(ReducedStorage3d, Subtraction)
 {
     using t = typename TestFixture::t;
     using ar_type = typename TestFixture::ar_type;
+    using gko::abs;
     const ar_type expected = -2.23 - 1;
 
     auto result = this->r(3, 2, 1) - ar_type{1.};
     this->r(3, 2, 1) -= 1;
 
     this->check_accessor_correctness(this->r, t(3, 2, 1));
-    EXPECT_NEAR(this->r(3, 2, 1), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
+    EXPECT_NEAR(abs(this->r(3, 2, 1)), abs(this->c_st_ar(expected)),
+                TestFixture::delta);
+    EXPECT_NEAR(abs(result), abs(this->c_st_ar(expected)), TestFixture::delta);
 }
 
 
@@ -330,14 +334,16 @@ TYPED_TEST(ReducedStorage3d, Subtraction2)
 {
     using t = typename TestFixture::t;
     using ar_type = typename TestFixture::ar_type;
+    using gko::abs;
     const ar_type expected = 3.22 - -2.23;
 
     auto result = this->cr(3, 2, 0) - this->r(3, 2, 1);
     this->r(3, 2, 0) -= this->r(3, 2, 1);
 
     this->check_accessor_correctness(this->r, t(3, 2, 0));
-    EXPECT_NEAR(this->r(3, 2, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
+    EXPECT_NEAR(abs(this->r(3, 2, 0)), abs(this->c_st_ar(expected)),
+                TestFixture::delta);
+    EXPECT_NEAR(abs(result), abs(this->c_st_ar(expected)), TestFixture::delta);
 }
 
 
@@ -351,8 +357,8 @@ TYPED_TEST(ReducedStorage3d, Multiplication)
     this->r(0, 0, 0) *= 2;
 
     this->check_accessor_correctness(this->r, t(0, 0, 0));
-    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
+    EXPECT_EQ(this->r(0, 0, 0), this->c_st_ar(expected));
+    EXPECT_EQ(result, this->c_st_ar(expected));
 }
 
 
@@ -360,14 +366,15 @@ TYPED_TEST(ReducedStorage3d, Multiplication2)
 {
     using t = typename TestFixture::t;
     using ar_type = typename TestFixture::ar_type;
+    using gko::abs;
     const ar_type expected = 2.01 * 3.03;
 
     auto result = this->r(0, 0, 1) * this->cr(0, 1, 1);
     this->r(0, 0, 1) *= this->r(0, 1, 1);
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
+    EXPECT_NEAR(abs(this->r(0, 0, 1)), abs(expected), TestFixture::delta);
+    EXPECT_NEAR(abs(result), abs(expected), TestFixture::delta);
 }
 
 
@@ -375,14 +382,15 @@ TYPED_TEST(ReducedStorage3d, Division)
 {
     using t = typename TestFixture::t;
     using ar_type = typename TestFixture::ar_type;
+    using gko::abs;
     const ar_type expected = 2.01 / 2.0;
 
     auto result = this->cr(0, 0, 1) / ar_type{2.};
     this->r(0, 0, 1) /= 2.;
 
     this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
+    EXPECT_NEAR(abs(this->r(0, 0, 1)), abs(expected), TestFixture::delta);
+    EXPECT_NEAR(abs(result), abs(expected), TestFixture::delta);
 }
 
 
@@ -390,14 +398,15 @@ TYPED_TEST(ReducedStorage3d, Division2)
 {
     using t = typename TestFixture::t;
     using ar_type = typename TestFixture::ar_type;
+    using gko::abs;
     const ar_type expected = 5.06 / 4.04;
 
     auto result = this->r(1, 0, 0) / this->cr(0, 2, 0);
     this->r(1, 0, 0) /= this->r(0, 2, 0);
 
     this->check_accessor_correctness(this->r, t(1, 0, 0));
-    EXPECT_NEAR(this->r(1, 0, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
+    EXPECT_NEAR(abs(this->r(1, 0, 0)), abs(expected), TestFixture::delta);
+    EXPECT_NEAR(abs(result), abs(expected), TestFixture::delta);
 }
 
 

From 4193824bb573cb9f8620da53ed25f90de93771b6 Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Wed, 13 Jan 2021 04:20:05 +0000
Subject: [PATCH 0239/1008] Format files
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thoasm@users.noreply.github.com>
---
 core/base/accessor_helper.hpp                |  2 +-
 core/base/accessor_references.hpp            | 17 +++++++++--------
 core/base/accessors.hpp                      |  2 +-
 core/test/base/accessors_reduced_storage.cpp |  2 +-
 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/core/base/accessor_helper.hpp b/core/base/accessor_helper.hpp
index b8d4f1f28f6..48a3a635656 100644
--- a/core/base/accessor_helper.hpp
+++ b/core/base/accessor_helper.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/base/accessor_references.hpp b/core/base/accessor_references.hpp
index 358078ebd32..be3e13a28a7 100644
--- a/core/base/accessor_references.hpp
+++ b/core/base/accessor_references.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -263,8 +263,8 @@ class reduced_storage
         return static_cast<arithmetic_type>(*r_ptr);
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type
-    operator=(arithmetic_type val) &&noexcept
+    constexpr GKO_ATTRIBUTES arithmetic_type operator=(arithmetic_type val) &&
+        noexcept
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         *r_ptr = static_cast<storage_type>(val);
@@ -278,8 +278,8 @@ class reduced_storage
         return static_cast<arithmetic_type>(*this);
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type
-    operator=(reduced_storage &&ref) &&noexcept
+    constexpr GKO_ATTRIBUTES arithmetic_type operator=(reduced_storage &&ref) &&
+        noexcept
     {
         std::move(*this) = static_cast<arithmetic_type>(ref);
         return static_cast<arithmetic_type>(*this);
@@ -384,8 +384,8 @@ class scaled_reduced_storage
         return static_cast<arithmetic_type>(*r_ptr) * scalar_;
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type
-    operator=(arithmetic_type val) &&noexcept
+    constexpr GKO_ATTRIBUTES arithmetic_type operator=(arithmetic_type val) &&
+        noexcept
     {
         storage_type *const GKO_RESTRICT r_ptr = ptr_;
         *r_ptr = static_cast<storage_type>(val / scalar_);
@@ -400,7 +400,8 @@ class scaled_reduced_storage
     }
 
     constexpr GKO_ATTRIBUTES arithmetic_type
-    operator=(scaled_reduced_storage &&ref) &&noexcept
+        operator=(scaled_reduced_storage &&ref) &&
+        noexcept
     {
         std::move(*this) = static_cast<arithmetic_type>(ref);
         return static_cast<arithmetic_type>(*this);
diff --git a/core/base/accessors.hpp b/core/base/accessors.hpp
index 52ccf5c5ecb..d7c465742f5 100644
--- a/core/base/accessors.hpp
+++ b/core/base/accessors.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/base/accessors_reduced_storage.cpp b/core/test/base/accessors_reduced_storage.cpp
index 3bfe18c88e0..1ae27d9c242 100644
--- a/core/test/base/accessors_reduced_storage.cpp
+++ b/core/test/base/accessors_reduced_storage.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

From ad1f32e2ad2076f0e355e13732fd94a5c11f09ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 13 Jan 2021 08:27:12 +0100
Subject: [PATCH 0240/1008] Update interface for scalar handling

Make a distinction between addressing the scalar value with the direct
indices (specifying the actual indices without the ignored ones)
and with masked indices (specifying all dimensions while ignoring all
where the bit of the scalar mask is not set).
Read and write functions are available for both variants.
---
 core/base/accessor_helper.hpp                | 125 +++++++++++++++----
 core/base/accessors.hpp                      |  76 +++++++++--
 core/test/base/accessors_reduced_storage.cpp |  25 +++-
 3 files changed, 189 insertions(+), 37 deletions(-)

diff --git a/core/base/accessor_helper.hpp b/core/base/accessor_helper.hpp
index 48a3a635656..fcfbd6a0d44 100644
--- a/core/base/accessor_helper.hpp
+++ b/core/base/accessor_helper.hpp
@@ -124,18 +124,18 @@ namespace detail {
 
 
 /**
- * This helper must be walked through in reverse (highest to lowest dimension)
- * in order to properly use the stride for the scalar indices. The mask
- * indicates which indices are actually used. The least significant bit set
- * means using the last index, second bit corresponds to the second last
- * dimension, and so on.
+ * This helper walkes through the index arguments from left to right (lowest to
+ * highest dimension) in order to properly use the stride for the scalar
+ * indices. The mask indicates which indices are actually used. The least
+ * significant bit set means using the last index, second bit corresponds to the
+ * second last dimension, and so on.
  *
  * Basically, this computes indices in a similar fashion as `row_major_helper_s`
  * while having a mask signaling which indices to skip.
  *
- * Example: Mask = 0b01101
+ * Example: Mask = 0b1101
  * compute(stride, tuple(x1, x2, x3, x4))
- * -> x4 + x2 * stride[2] + x1 * stride[1]
+ * -> x1 * stride[1] + x2 * stride[2] + x4    (x3 skipped since bit not set)
  */
 template <typename ValueType, size_type mask, size_type set_bits_processed,
           size_type stride_size, size_type dim_idx, size_type total_dim,
@@ -184,9 +184,9 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
 
     template <typename First, typename... Indices>
     static constexpr GKO_ATTRIBUTES ValueType
-    compute_idx(const dim<total_dim> &size,
-                const std::array<ValueType, stride_size> &stride, First first,
-                Indices &&... idxs)
+    compute_mask_idx(const dim<total_dim> &size,
+                     const std::array<ValueType, stride_size> &stride,
+                     First first, Indices &&... idxs)
     {
         static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
                       "Mismatching number of Idxs!");
@@ -198,8 +198,30 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
                    row_major_masked_helper_s<
                        ValueType, mask, set_bits_processed + 1, stride_size,
                        dim_idx + 1,
-                       total_dim>::compute_idx(size, stride,
-                                               std::forward<Indices>(idxs)...);
+                       total_dim>::compute_mask_idx(size, stride,
+                                                    std::forward<Indices>(
+                                                        idxs)...);
+    }
+
+    template <typename First, typename... Indices>
+    static constexpr GKO_ATTRIBUTES ValueType
+    compute_direct_idx(const dim<total_dim> &size,
+                       const std::array<ValueType, stride_size> &stride,
+                       First first, Indices &&... idxs)
+    {
+        static_assert(sizeof...(Indices) == stride_size - set_bits_processed,
+                      "Mismatching number of Idxs!");
+        // If it is the last set dimension, there is no need for a stride
+        return GKO_ASSERT(first < size[dim_idx]),
+               first * (set_bits_processed == stride_size
+                            ? 1
+                            : stride[set_bits_processed]) +
+                   row_major_masked_helper_s<
+                       ValueType, mask, set_bits_processed + 1, stride_size,
+                       dim_idx + 1,
+                       total_dim>::compute_direct_idx(size, stride,
+                                                      std::forward<Indices>(
+                                                          idxs)...);
     }
 };
 
@@ -236,9 +258,9 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
 
     template <typename First, typename... Indices>
     static constexpr GKO_ATTRIBUTES ValueType
-    compute_idx(const dim<total_dim> &size,
-                const std::array<ValueType, stride_size> &stride, First first,
-                Indices &&... idxs)
+    compute_mask_idx(const dim<total_dim> &size,
+                     const std::array<ValueType, stride_size> &stride,
+                     First first, Indices &&... idxs)
     {
         static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
                       "Mismatching number of Idxs!");
@@ -250,8 +272,30 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
                    row_major_masked_helper_s<
                        ValueType, mask, set_bits_processed + 1, stride_size,
                        dim_idx + 1,
-                       total_dim>::compute_idx(size, stride,
-                                               std::forward<Indices>(idxs)...);
+                       total_dim>::compute_mask_idx(size, stride,
+                                                    std::forward<Indices>(
+                                                        idxs)...);
+    }
+
+    template <typename First, typename... Indices>
+    static constexpr GKO_ATTRIBUTES ValueType
+    compute_direct_idx(const dim<total_dim> &size,
+                       const std::array<ValueType, stride_size> &stride,
+                       First first, Indices &&... idxs)
+    {
+        static_assert(sizeof...(Indices) == stride_size - set_bits_processed,
+                      "Mismatching number of Idxs!");
+        // If it is the last set dimension, there is no need for a stride
+        return GKO_ASSERT(first < size[dim_idx]),
+               first * (set_bits_processed == stride_size
+                            ? 1
+                            : stride[set_bits_processed]) +
+                   row_major_masked_helper_s<
+                       ValueType, mask, set_bits_processed + 1, stride_size,
+                       dim_idx + 1,
+                       total_dim>::compute_direct_idx(size, stride,
+                                                      std::forward<Indices>(
+                                                          idxs)...);
     }
 };
 
@@ -287,16 +331,27 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
 
     template <typename First, typename... Indices>
     static constexpr GKO_ATTRIBUTES ValueType
-    compute_idx(const dim<total_dim> &size,
-                const std::array<ValueType, stride_size> &stride, First,
-                Indices &&... idxs)
+    compute_mask_idx(const dim<total_dim> &size,
+                     const std::array<ValueType, stride_size> &stride, First,
+                     Indices &&... idxs)
     {
         static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
                       "Mismatching number of Idxs!");
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed, stride_size, dim_idx + 1,
-            total_dim>::compute_idx(size, stride,
-                                    std::forward<Indices>(idxs)...);
+            total_dim>::compute_mask_idx(size, stride,
+                                         std::forward<Indices>(idxs)...);
+    }
+
+    template <typename... Indices>
+    static constexpr GKO_ATTRIBUTES ValueType compute_direct_idx(
+        const dim<total_dim> &size,
+        const std::array<ValueType, stride_size> &stride, Indices &&... idxs)
+    {
+        return row_major_masked_helper_s<
+            ValueType, mask, set_bits_processed, stride_size, dim_idx + 1,
+            total_dim>::compute_direct_idx(size, stride,
+                                           std::forward<Indices>(idxs)...);
     }
 };
 
@@ -321,7 +376,12 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
         return 1;
     }
 
-    static constexpr GKO_ATTRIBUTES ValueType compute_idx(
+    static constexpr GKO_ATTRIBUTES ValueType compute_mask_idx(
+        const dim<total_dim> &, const std::array<ValueType, stride_size> &)
+    {
+        return 0;
+    }
+    static constexpr GKO_ATTRIBUTES ValueType compute_direct_idx(
         const dim<total_dim> &, const std::array<ValueType, stride_size> &)
     {
         return 0;
@@ -344,7 +404,24 @@ constexpr GKO_ATTRIBUTES auto compute_masked_index(
 {
     return detail::row_major_masked_helper_s<
         ValueType, mask, 0, stride_size, 0,
-        total_dim>::compute_idx(size, stride, std::forward<Indices>(idxs)...);
+        total_dim>::compute_mask_idx(size, stride,
+                                     std::forward<Indices>(idxs)...);
+}
+
+
+/**
+ * Computes the memory index for the given indices considering the stride.
+ */
+template <typename ValueType, size_type mask, size_type stride_size,
+          size_type total_dim, typename... Indices>
+constexpr GKO_ATTRIBUTES auto compute_masked_index_direct(
+    const dim<total_dim> &size,
+    const std::array<ValueType, stride_size> &stride, Indices &&... idxs)
+{
+    return detail::row_major_masked_helper_s<
+        ValueType, mask, 0, stride_size, 0,
+        total_dim>::compute_direct_idx(size, stride,
+                                       std::forward<Indices>(idxs)...);
 }
 
 
diff --git a/core/base/accessors.hpp b/core/base/accessors.hpp
index d7c465742f5..fb1b1700c0b 100644
--- a/core/base/accessors.hpp
+++ b/core/base/accessors.hpp
@@ -313,6 +313,8 @@ struct enable_write_scalar<Dimensionality, Accessor, ScalarType, false> {
 
     /**
      * Writes the scalar value at the given indices.
+     * The number of indices must be equal to the number of dimensions, even
+     * if some of the indices are ignored (depending on the scalar mask).
      *
      * @param value  value to write
      * @param indices  indices where to write the value
@@ -321,15 +323,36 @@ struct enable_write_scalar<Dimensionality, Accessor, ScalarType, false> {
      */
     template <typename... Indices>
     constexpr GKO_ATTRIBUTES scalar_type
-    write_scalar(scalar_type value, Indices &&... indices) const
+    write_scalar_masked(scalar_type value, Indices &&... indices) const
     {
         static_assert(sizeof...(Indices) == Dimensionality,
                       "Number of indices must match dimensionality!");
         scalar_type *GKO_RESTRICT rest_scalar = self()->scalar_;
-        return rest_scalar[self()->compute_scalar_index(
+        return rest_scalar[self()->compute_mask_scalar_index(
                    std::forward<Indices>(indices)...)] = value;
     }
 
+    /**
+     * Writes the scalar value at the given indices.
+     * Only the actually used indices must be provided, meaning the number of
+     * specified indices must be equal to the number of set bits in the
+     * scalar mask.
+     *
+     * @param value  value to write
+     * @param indices  indices where to write the value
+     *
+     * @returns the written value.
+     */
+    template <typename... Indices>
+    constexpr GKO_ATTRIBUTES scalar_type
+    write_scalar_direct(scalar_type value, Indices &&... indices) const
+    {
+        scalar_type *GKO_RESTRICT rest_scalar = self()->scalar_;
+        return rest_scalar[self()->compute_direct_scalar_index(
+                   std::forward<Indices>(indices)...)] = value;
+    }
+
+
 private:
     constexpr GKO_ATTRIBUTES const Accessor *self() const
     {
@@ -397,7 +420,7 @@ class scaled_reduced_row_major
     static_assert(dimensionality <= 32,
                   "Only Dimensionality <= 32 is currently supported");
 
-    // Allow access to both `scalar_` and `compute_scalar_index()`
+    // Allow access to both `scalar_` and `compute_mask_scalar_index()`
     friend class detail::enable_write_scalar<
         dimensionality, scaled_reduced_row_major, scalar_type>;
     friend class range<scaled_reduced_row_major>;
@@ -495,7 +518,26 @@ class scaled_reduced_row_major
     }
 
     /**
-     * Reads the scalar value at the given indices.
+     * Reads the scalar value at the given indices. Only indices where the
+     * scalar mask bit is set are considered, the others are ignored.
+     *
+     * @param indices  indices which data to access
+     *
+     * @returns the scalar value at the given indices.
+     */
+    template <typename... Indices>
+    constexpr GKO_ATTRIBUTES scalar_type
+    read_scalar_masked(Indices &&... indices) const
+    {
+        const arithmetic_type *GKO_RESTRICT rest_scalar = scalar_;
+        return rest_scalar[compute_mask_scalar_index(
+            std::forward<Indices>(indices)...)];
+    }
+
+    /**
+     * Reads the scalar value at the given indices. Only the actually used
+     * indices must be provided, meaning the number of specified indices must
+     * be equal to the number of set bits in the scalar mask.
      *
      * @param indices  indices which data to access
      *
@@ -503,10 +545,10 @@ class scaled_reduced_row_major
      */
     template <typename... Indices>
     constexpr GKO_ATTRIBUTES scalar_type
-    read_scalar(Indices &&... indices) const
+    read_scalar_direct(Indices &&... indices) const
     {
         const arithmetic_type *GKO_RESTRICT rest_scalar = scalar_;
-        return rest_scalar[compute_scalar_index(
+        return rest_scalar[compute_direct_scalar_index(
             std::forward<Indices>(indices)...)];
     }
 
@@ -539,7 +581,8 @@ class scaled_reduced_row_major
     {
         helper::multidim_for_each(size_, [this, &other](auto... indices) {
             // especially inefficient if mask has not all bits set
-            this->write_scalar(other.read_scalar(indices...), indices...);
+            this->write_scalar_masked(other.read_scalar_masked(indices...),
+                                      indices...);
             (*this)(indices...) = other(indices...);
         });
     }
@@ -560,7 +603,7 @@ class scaled_reduced_row_major
     operator()(Indices... indices) const
     {
         return reference_type{storage_ + compute_index(indices...),
-                              read_scalar(indices...)};
+                              read_scalar_masked(indices...)};
     }
 
     /**
@@ -582,7 +625,7 @@ class scaled_reduced_row_major
                        (span{spans}.end - span{spans}.begin)...},
                    storage_ + compute_index((span{spans}.begin)...),
                    storage_stride_,
-                   scalar_ + compute_scalar_index(span{spans}.begin...),
+                   scalar_ + compute_mask_scalar_index(span{spans}.begin...),
                    scalar_stride_};
     }
 
@@ -669,7 +712,7 @@ class scaled_reduced_row_major
 
     template <typename... Indices>
     constexpr GKO_ATTRIBUTES size_type
-    compute_scalar_index(Indices &&... indices) const
+    compute_mask_scalar_index(Indices &&... indices) const
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
@@ -678,6 +721,19 @@ class scaled_reduced_row_major
             size_, scalar_stride_, std::forward<Indices>(indices)...);
     }
 
+    template <typename... Indices>
+    constexpr GKO_ATTRIBUTES size_type
+    compute_direct_scalar_index(Indices &&... indices) const
+    {
+        static_assert(
+            sizeof...(Indices) == scalar_dim,
+            "Number of indices must match number of set bits in scalar mask!");
+        return helper::compute_masked_index_direct<size_type, scalar_mask,
+                                                   scalar_stride_dim>(
+            size_, scalar_stride_, std::forward<Indices>(indices)...);
+    }
+
+
 private:
     const dim<dimensionality> size_;
     storage_type *storage_;
diff --git a/core/test/base/accessors_reduced_storage.cpp b/core/test/base/accessors_reduced_storage.cpp
index 1ae27d9c242..a59f85dd1f6 100644
--- a/core/test/base/accessors_reduced_storage.cpp
+++ b/core/test/base/accessors_reduced_storage.cpp
@@ -711,7 +711,24 @@ TYPED_TEST(ScaledReducedStorage3d, CanWriteScale)
 {
     using ar_type = typename TestFixture::ar_type;
 
-    this->r->write_scalar(10., 0, 0, 0);
+    this->r->write_scalar_masked(10., 0, 0, 0);
+
+    EXPECT_EQ(this->r(0, 0, 0), ar_type{100.});
+    EXPECT_EQ(this->r(0, 0, 1), ar_type{22.});
+    EXPECT_EQ(this->r(0, 1, 0), ar_type{-120.});
+    EXPECT_EQ(this->r(0, 1, 1), ar_type{26.});
+    EXPECT_EQ(this->r(0, 2, 0), ar_type{140.});
+    EXPECT_EQ(this->r(0, 2, 1), ar_type{-230.});
+    EXPECT_EQ(this->r(0, 3, 0), ar_type{60.});
+    EXPECT_EQ(this->r(0, 3, 1), ar_type{154.});
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CanWriteMaskedScale)
+{
+    using ar_type = typename TestFixture::ar_type;
+
+    this->r->write_scalar_direct(10., 0, 0);
 
     EXPECT_EQ(this->r(0, 0, 0), ar_type{100.});
     EXPECT_EQ(this->r(0, 0, 1), ar_type{22.});
@@ -726,8 +743,10 @@ TYPED_TEST(ScaledReducedStorage3d, CanWriteScale)
 
 TYPED_TEST(ScaledReducedStorage3d, CanReadScale)
 {
-    EXPECT_EQ(this->r->read_scalar(0, 0, 0), 1.);
-    EXPECT_EQ(this->r->read_scalar(0, 0, 1), 2.);
+    EXPECT_EQ(this->r->read_scalar_masked(0, 0, 0), 1.);
+    EXPECT_EQ(this->r->read_scalar_masked(0, 0, 1), 2.);
+    EXPECT_EQ(this->r->read_scalar_direct(0, 0), 1.);
+    EXPECT_EQ(this->r->read_scalar_direct(0, 1), 2.);
 }
 
 

From 3b57e9cde32f66659ad50782d636d4eb239d17dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 15 Jan 2021 06:07:22 +0100
Subject: [PATCH 0241/1008] Review update

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 core/base/accessor_helper.hpp     | 2 +-
 core/base/accessor_references.hpp | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/base/accessor_helper.hpp b/core/base/accessor_helper.hpp
index fcfbd6a0d44..ac5246f5625 100644
--- a/core/base/accessor_helper.hpp
+++ b/core/base/accessor_helper.hpp
@@ -124,7 +124,7 @@ namespace detail {
 
 
 /**
- * This helper walkes through the index arguments from left to right (lowest to
+ * This helper walks through the index arguments from left to right (lowest to
  * highest dimension) in order to properly use the stride for the scalar
  * indices. The mask indicates which indices are actually used. The least
  * significant bit set means using the last index, second bit corresponds to the
diff --git a/core/base/accessor_references.hpp b/core/base/accessor_references.hpp
index be3e13a28a7..e392275e1eb 100644
--- a/core/base/accessor_references.hpp
+++ b/core/base/accessor_references.hpp
@@ -114,7 +114,7 @@ constexpr GKO_ATTRIBUTES
  *
  * @tparam Reference  The reference class this mixin provides operator overloads
  *                    for. The reference class needs to overload the cast
- *                    operatorto ValueType
+ *                    operator to ValueType
  *
  * @tparam ArithmeticType  arithmetic type the Reference class is supposed
  *         to represent.
@@ -346,7 +346,7 @@ constexpr remove_complex<ArithmeticType> abs(
  * Copying this reference is disabled, but move construction is possible to
  * allow for an additional layer (like gko::range).
  * The assignment operator only works for an rvalue reference (&&) to
- * prevent accidental copying the reference and working on a reference.
+ * prevent accidentally copying and working on the reference.
  *
  * @tparam ArithmeticType  Type used for arithmetic operations, therefore,
  *                         the type which is used for input and output of this

From ef8d7b697d442b6ae7426746a33c492f0f35be8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 21 Jan 2021 16:18:17 +0100
Subject: [PATCH 0242/1008] Review update

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/base/extended_float.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/base/extended_float.hpp b/core/base/extended_float.hpp
index 3961f47c3bd..34b3eed0d59 100644
--- a/core/base/extended_float.hpp
+++ b/core/base/extended_float.hpp
@@ -557,7 +557,7 @@ struct is_scalar<gko::half> : std::true_type {};
 
 template <>
 struct numeric_limits<gko::half> {
-    static constexpr bool is_specializec{true};
+    static constexpr bool is_specialized{true};
     static constexpr bool is_signed{true};
     static constexpr bool is_integer{false};
     static constexpr bool is_exact{false};

From ee7fe2394ca011475294dfefb9b231dd01fe2fda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 7 Dec 2020 13:00:55 +0100
Subject: [PATCH 0243/1008] Enable benchmark to run with different precisions

Note: benchmark is used with `float` in this commit to test it on CI
---
 benchmark/conversions/conversions.cpp         |   8 +-
 .../matrix_generator/matrix_generator.cpp     |   9 +-
 .../matrix_statistics/matrix_statistics.cpp   |   5 +-
 benchmark/preconditioner/preconditioner.cpp   |   5 +-
 benchmark/solver/solver.cpp                   |  48 +++----
 benchmark/spmv/spmv.cpp                       |   6 +-
 benchmark/utils/cuda_linops.hpp               |  23 ++--
 benchmark/utils/formats.hpp                   |  30 +++--
 benchmark/utils/general.hpp                   |  18 +--
 benchmark/utils/hip_linops.hip.hpp            |  11 +-
 benchmark/utils/preconditioners.hpp           | 117 +++++++++++-------
 benchmark/utils/spmv_common.hpp               |   4 -
 benchmark/utils/types.hpp                     |  40 ++++++
 13 files changed, 188 insertions(+), 136 deletions(-)
 create mode 100644 benchmark/utils/types.hpp

diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index 70df82ccba4..7a3d9c61a0c 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -48,9 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/loggers.hpp"
 #include "benchmark/utils/spmv_common.hpp"
 #include "benchmark/utils/timer.hpp"
-
-
-using etype = double;
+#include "benchmark/utils/types.hpp"
 
 
 // This function supposes that management of `FLAGS_overwrite` is done before
@@ -66,7 +64,7 @@ void convert_matrix(const gko::LinOp *matrix_from, const char *format_to,
         add_or_set_member(conversion_case, conversion_name,
                           rapidjson::Value(rapidjson::kObjectType), allocator);
 
-        gko::matrix_data<> data{gko::dim<2>{1, 1}, 1};
+        gko::matrix_data<etype> data{gko::dim<2>{1, 1}, 1};
         auto matrix_to =
             share(formats::matrix_factory.at(format_to)(exec, data));
         // warm run
@@ -138,7 +136,7 @@ int main(int argc, char *argv[])
 
         std::clog << "Running test case: " << test_case << std::endl;
         std::ifstream mtx_fd(test_case["filename"].GetString());
-        gko::matrix_data<> data;
+        gko::matrix_data<etype> data;
         try {
             data = gko::read_raw<etype>(mtx_fd);
         } catch (std::exception &e) {
diff --git a/benchmark/matrix_generator/matrix_generator.cpp b/benchmark/matrix_generator/matrix_generator.cpp
index b32810b96a5..90a63702e86 100644
--- a/benchmark/matrix_generator/matrix_generator.cpp
+++ b/benchmark/matrix_generator/matrix_generator.cpp
@@ -40,10 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "benchmark/utils/general.hpp"
-
-
-// some Ginkgo shortcuts
-using etype = double;
+#include "benchmark/utils/types.hpp"
 
 
 namespace {
@@ -109,8 +106,8 @@ gko::matrix_data<etype> generate_block_diagonal(rapidjson::Value &config,
     auto num_blocks = config["num_blocks"].GetUint();
     auto block_size = config["block_size"].GetUint();
     auto block = gko::matrix_data<etype>(
-        gko::dim<2>(block_size), std::uniform_real_distribution<>(-1.0, 1.0),
-        engine);
+        gko::dim<2>(block_size),
+        std::uniform_real_distribution<etype>(-1.0, 1.0), engine);
     return gko::matrix_data<etype>::diag(num_blocks, block);
 }
 
diff --git a/benchmark/matrix_statistics/matrix_statistics.cpp b/benchmark/matrix_statistics/matrix_statistics.cpp
index 2602a16a967..bc273acf70d 100644
--- a/benchmark/matrix_statistics/matrix_statistics.cpp
+++ b/benchmark/matrix_statistics/matrix_statistics.cpp
@@ -42,10 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "benchmark/utils/general.hpp"
 #include "benchmark/utils/spmv_common.hpp"
-
-
-// some Ginkgo shortcuts
-using etype = double;
+#include "benchmark/utils/types.hpp"
 
 
 // See en.wikipedia.org/wiki/Five-number_summary
diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index b4a1acaa741..00d928a9f09 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -48,10 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/preconditioners.hpp"
 #include "benchmark/utils/spmv_common.hpp"
 #include "benchmark/utils/timer.hpp"
-
-
-// some shortcuts
-using etype = double;
+#include "benchmark/utils/types.hpp"
 
 
 // preconditioner generation and application
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 3ec32280935..09454e763bd 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <fstream>
 #include <iomanip>
 #include <iostream>
+#include <limits>
 #include <vector>
 
 
@@ -50,10 +51,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/overhead_linop.hpp"
 #include "benchmark/utils/preconditioners.hpp"
 #include "benchmark/utils/timer.hpp"
-
-
-// some Ginkgo shortcuts
-using etype = double;
+#include "benchmark/utils/types.hpp"
 
 
 // Command-line arguments
@@ -131,14 +129,15 @@ std::shared_ptr<const gko::stop::CriterionFactory> create_criterion(
 {
     std::shared_ptr<const gko::stop::CriterionFactory> residual_stop;
     if (FLAGS_rel_residual) {
-        residual_stop = gko::share(gko::stop::RelativeResidualNorm<>::build()
-                                       .with_tolerance(FLAGS_rel_res_goal)
-                                       .on(exec));
+        residual_stop = gko::share(
+            gko::stop::RelativeResidualNorm<etype>::build()
+                .with_tolerance(static_cast<etype>(FLAGS_rel_res_goal))
+                .on(exec));
     } else {
-        residual_stop =
-            gko::share(gko::stop::ResidualNormReduction<>::build()
-                           .with_reduction_factor(FLAGS_rel_res_goal)
-                           .on(exec));
+        residual_stop = gko::share(
+            gko::stop::ResidualNormReduction<etype>::build()
+                .with_reduction_factor(static_cast<etype>(FLAGS_rel_res_goal))
+                .on(exec));
     }
     auto iteration_stop = gko::share(
         gko::stop::Iteration::build().with_max_iters(FLAGS_max_iters).on(exec));
@@ -164,25 +163,25 @@ std::unique_ptr<gko::LinOpFactory> create_solver(
 const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                                 std::shared_ptr<const gko::Executor>,
                                 std::shared_ptr<const gko::LinOpFactory>)>>
-    solver_factory{{"bicgstab", create_solver<gko::solver::Bicgstab<>>},
-                   {"bicg", create_solver<gko::solver::Bicg<>>},
-                   {"cg", create_solver<gko::solver::Cg<>>},
-                   {"cgs", create_solver<gko::solver::Cgs<>>},
-                   {"fcg", create_solver<gko::solver::Fcg<>>},
+    solver_factory{{"bicgstab", create_solver<gko::solver::Bicgstab<etype>>},
+                   {"bicg", create_solver<gko::solver::Bicg<etype>>},
+                   {"cg", create_solver<gko::solver::Cg<etype>>},
+                   {"cgs", create_solver<gko::solver::Cgs<etype>>},
+                   {"fcg", create_solver<gko::solver::Fcg<etype>>},
                    {"idr",
                     [](std::shared_ptr<const gko::Executor> exec,
                        std::shared_ptr<const gko::LinOpFactory> precond) {
-                        return gko::solver::Idr<>::build()
+                        return gko::solver::Idr<etype>::build()
                             .with_criteria(create_criterion(exec))
                             .with_subspace_dim(FLAGS_idr_subspace_dim)
-                            .with_kappa(FLAGS_idr_kappa)
+                            .with_kappa(static_cast<etype>(FLAGS_idr_kappa))
                             .with_preconditioner(give(precond))
                             .on(exec);
                     }},
                    {"gmres",
                     [](std::shared_ptr<const gko::Executor> exec,
                        std::shared_ptr<const gko::LinOpFactory> precond) {
-                        return gko::solver::Gmres<>::build()
+                        return gko::solver::Gmres<etype>::build()
                             .with_criteria(create_criterion(exec))
                             .with_krylov_dim(FLAGS_gmres_restart)
                             .with_preconditioner(give(precond))
@@ -191,18 +190,18 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                    {"lower_trs",
                     [](std::shared_ptr<const gko::Executor> exec,
                        std::shared_ptr<const gko::LinOpFactory>) {
-                        return gko::solver::LowerTrs<>::build()
+                        return gko::solver::LowerTrs<etype>::build()
                             .with_num_rhs(FLAGS_nrhs)
                             .on(exec);
                     }},
                    {"upper_trs",
                     [](std::shared_ptr<const gko::Executor> exec,
                        std::shared_ptr<const gko::LinOpFactory>) {
-                        return gko::solver::UpperTrs<>::build()
+                        return gko::solver::UpperTrs<etype>::build()
                             .with_num_rhs(FLAGS_nrhs)
                             .on(exec);
                     }},
-                   {"overhead", create_solver<gko::Overhead<>>}};
+                   {"overhead", create_solver<gko::Overhead<etype>>}};
 
 
 void write_precond_info(const gko::LinOp *precond,
@@ -472,13 +471,14 @@ int main(int argc, char *argv[])
             std::clog << "Running test case: " << test_case << std::endl;
             std::ifstream mtx_fd(test_case["filename"].GetString());
 
-            using Vec = gko::matrix::Dense<>;
+            using Vec = gko::matrix::Dense<etype>;
             std::shared_ptr<gko::LinOp> system_matrix;
             std::unique_ptr<Vec> b;
             std::unique_ptr<Vec> x;
             if (FLAGS_overhead) {
                 system_matrix = gko::initialize<Vec>({1.0}, exec);
-                b = gko::initialize<Vec>({std::nan("")}, exec);
+                b = gko::initialize<Vec>(
+                    {std::numeric_limits<etype>::quiet_NaN()}, exec);
                 x = gko::initialize<Vec>({0.0}, exec);
             } else {
                 auto data = gko::read_raw<etype>(mtx_fd);
diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp
index 0304026c0ea..b00dbff152a 100644
--- a/benchmark/spmv/spmv.cpp
+++ b/benchmark/spmv/spmv.cpp
@@ -48,9 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/loggers.hpp"
 #include "benchmark/utils/spmv_common.hpp"
 #include "benchmark/utils/timer.hpp"
-
-
-using etype = double;
+#include "benchmark/utils/types.hpp"
 
 
 // Command-line arguments
@@ -84,7 +82,7 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
             exec->synchronize();
             system_matrix->apply(lend(b), lend(x_clone));
             exec->synchronize();
-            double max_relative_norm2 =
+            auto max_relative_norm2 =
                 compute_max_relative_norm2(lend(x_clone), lend(answer));
             add_or_set_member(spmv_case[format_name], "max_relative_norm2",
                               max_relative_norm2, allocator);
diff --git a/benchmark/utils/cuda_linops.hpp b/benchmark/utils/cuda_linops.hpp
index d51fa512f92..30866aeb285 100644
--- a/benchmark/utils/cuda_linops.hpp
+++ b/benchmark/utils/cuda_linops.hpp
@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <cusparse.h>
 
 
+#include "benchmark/utils/types.hpp"
 #include "cuda/base/cusparse_bindings.hpp"
 #include "cuda/base/device_guard.hpp"
 #include "cuda/base/pointer_mode_guard.hpp"
@@ -147,7 +148,7 @@ class CuspCsrmp
 protected:
     void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override
     {
-        auto dense_b = gko::as<gko::matrix::Dense<double>>(b);
+        auto dense_b = gko::as<gko::matrix::Dense<ValueType>>(b);
         auto dense_x = gko::as<gko::matrix::Dense<ValueType>>(x);
         auto db = dense_b->get_const_values();
         auto dx = dense_x->get_values();
@@ -684,11 +685,11 @@ class CuspGenericCoo
 
 
 // Some shortcuts
-using cusp_csrex = detail::CuspCsrEx<>;
+using cusp_csrex = detail::CuspCsrEx<etype>;
 #if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
-using cusp_csr = detail::CuspCsr<>;
-using cusp_csrmp = detail::CuspCsrmp<>;
-using cusp_csrmm = detail::CuspCsrmm<>;
+using cusp_csr = detail::CuspCsr<etype>;
+using cusp_csrmp = detail::CuspCsrmp<etype>;
+using cusp_csrmm = detail::CuspCsrmm<etype>;
 #endif  // defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
 
 
@@ -697,10 +698,10 @@ using cusp_csrmm = detail::CuspCsrmm<>;
      ((CUDA_VERSION >= 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
 
 
-using cusp_gcsr = detail::CuspGenericCsr<>;
+using cusp_gcsr = detail::CuspGenericCsr<etype>;
 using cusp_gcsr2 =
-    detail::CuspGenericCsr<double, gko::int32, CUSPARSE_CSRMV_ALG2>;
-using cusp_gcoo = detail::CuspGenericCoo<>;
+    detail::CuspGenericCsr<etype, gko::int32, CUSPARSE_CSRMV_ALG2>;
+using cusp_gcoo = detail::CuspGenericCoo<etype>;
 
 
 #endif  // defined(CUDA_VERSION) && (CUDA_VERSION >= 11000 || ((CUDA_VERSION >=
@@ -709,10 +710,10 @@ using cusp_gcoo = detail::CuspGenericCoo<>;
 
 #if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
 using cusp_coo =
-    detail::CuspHybrid<double, gko::int32, CUSPARSE_HYB_PARTITION_USER, 0>;
+    detail::CuspHybrid<etype, gko::int32, CUSPARSE_HYB_PARTITION_USER, 0>;
 using cusp_ell =
-    detail::CuspHybrid<double, gko::int32, CUSPARSE_HYB_PARTITION_MAX, 0>;
-using cusp_hybrid = detail::CuspHybrid<>;
+    detail::CuspHybrid<etype, gko::int32, CUSPARSE_HYB_PARTITION_MAX, 0>;
+using cusp_hybrid = detail::CuspHybrid<etype>;
 #endif  // defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
 
 
diff --git a/benchmark/utils/formats.hpp b/benchmark/utils/formats.hpp
index 259cd2bda5f..095b44da218 100644
--- a/benchmark/utils/formats.hpp
+++ b/benchmark/utils/formats.hpp
@@ -52,6 +52,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif  // HAS_HIP
 
 
+#include "benchmark/utils/types.hpp"
+
+
 namespace formats {
 
 
@@ -151,8 +154,8 @@ namespace formats {
 
 
 // some shortcuts
-using hybrid = gko::matrix::Hybrid<>;
-using csr = gko::matrix::Csr<>;
+using hybrid = gko::matrix::Hybrid<etype>;
+using csr = gko::matrix::Csr<etype>;
 
 /**
  * Creates a Ginkgo matrix from the intermediate data representation format
@@ -167,7 +170,8 @@ using csr = gko::matrix::Csr<>;
  */
 template <typename MatrixType>
 std::unique_ptr<MatrixType> read_matrix_from_data(
-    std::shared_ptr<const gko::Executor> exec, const gko::matrix_data<> &data)
+    std::shared_ptr<const gko::Executor> exec,
+    const gko::matrix_data<etype> &data)
 {
     auto mat = MatrixType::create(std::move(exec));
     mat->read(data);
@@ -180,26 +184,26 @@ std::unique_ptr<MatrixType> read_matrix_from_data(
  *
  * @param MATRIX_TYPE  the Ginkgo matrix type (such as `gko::matrix::Csr<>`)
  */
-#define READ_MATRIX(MATRIX_TYPE, ...)                                    \
-    [](std::shared_ptr<const gko::Executor> exec,                        \
-       const gko::matrix_data<> &data) -> std::unique_ptr<MATRIX_TYPE> { \
-        auto mat = MATRIX_TYPE::create(std::move(exec), __VA_ARGS__);    \
-        mat->read(data);                                                 \
-        return mat;                                                      \
+#define READ_MATRIX(MATRIX_TYPE, ...)                                         \
+    [](std::shared_ptr<const gko::Executor> exec,                             \
+       const gko::matrix_data<etype> &data) -> std::unique_ptr<MATRIX_TYPE> { \
+        auto mat = MATRIX_TYPE::create(std::move(exec), __VA_ARGS__);         \
+        mat->read(data);                                                      \
+        return mat;                                                           \
     }
 
 
 // clang-format off
 const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
                                 std::shared_ptr<const gko::Executor>,
-                                const gko::matrix_data<> &)>>
+                                const gko::matrix_data<etype> &)>>
     matrix_factory{
         {"csr", READ_MATRIX(csr, std::make_shared<csr::automatical>())},
         {"csri", READ_MATRIX(csr, std::make_shared<csr::load_balance>())},
         {"csrm", READ_MATRIX(csr, std::make_shared<csr::merge_path>())},
         {"csrc", READ_MATRIX(csr, std::make_shared<csr::classical>())},
-        {"coo", read_matrix_from_data<gko::matrix::Coo<>>},
-        {"ell", read_matrix_from_data<gko::matrix::Ell<>>},
+        {"coo", read_matrix_from_data<gko::matrix::Coo<etype>>},
+        {"ell", read_matrix_from_data<gko::matrix::Ell<etype>>},
 #ifdef HAS_CUDA
 #if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
         {"cusp_csr", read_matrix_from_data<cusp_csr>},
@@ -256,7 +260,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
         {"hybridminstorage",
          READ_MATRIX(hybrid,
                      std::make_shared<hybrid::minimal_storage_limit>())},
-        {"sellp", read_matrix_from_data<gko::matrix::Sellp<>>}};
+        {"sellp", read_matrix_from_data<gko::matrix::Sellp<etype>>}};
 // clang-format on
 
 
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index a9321ca2c44..fc449b69c16 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -316,7 +316,8 @@ std::unique_ptr<vec<ValueType>> create_matrix(
     auto res = vec<ValueType>::create(exec);
     if (random) {
         res->read(gko::matrix_data<ValueType>(
-            size, std::uniform_real_distribution<>(-1.0, 1.0), engine));
+            size, std::uniform_real_distribution<ValueType>(-1.0, 1.0),
+            engine));
     } else {
         res->read(gko::matrix_data<ValueType>(size, gko::one<ValueType>()));
     }
@@ -336,14 +337,14 @@ std::unique_ptr<vec<ValueType>> create_vector(
 
 // utilities for computing norms and residuals
 template <typename ValueType>
-double get_norm(const vec<ValueType> *norm)
+ValueType get_norm(const vec<ValueType> *norm)
 {
     return clone(norm->get_executor()->get_master(), norm)->at(0, 0);
 }
 
 
 template <typename ValueType>
-double compute_norm2(const vec<ValueType> *b)
+ValueType compute_norm2(const vec<ValueType> *b)
 {
     auto exec = b->get_executor();
     auto b_norm = gko::initialize<vec<ValueType>>({0.0}, exec);
@@ -353,8 +354,9 @@ double compute_norm2(const vec<ValueType> *b)
 
 
 template <typename ValueType>
-double compute_residual_norm(const gko::LinOp *system_matrix,
-                             const vec<ValueType> *b, const vec<ValueType> *x)
+ValueType compute_residual_norm(const gko::LinOp *system_matrix,
+                                const vec<ValueType> *b,
+                                const vec<ValueType> *x)
 {
     auto exec = system_matrix->get_executor();
     auto one = gko::initialize<vec<ValueType>>({1.0}, exec);
@@ -366,8 +368,8 @@ double compute_residual_norm(const gko::LinOp *system_matrix,
 
 
 template <typename ValueType>
-double compute_max_relative_norm2(vec<ValueType> *result,
-                                  const vec<ValueType> *answer)
+ValueType compute_max_relative_norm2(vec<ValueType> *result,
+                                     const vec<ValueType> *answer)
 {
     auto exec = answer->get_executor();
     auto answer_norm =
@@ -382,7 +384,7 @@ double compute_max_relative_norm2(vec<ValueType> *result,
         clone(answer_norm->get_executor()->get_master(), answer_norm);
     auto host_absolute_norm =
         clone(absolute_norm->get_executor()->get_master(), absolute_norm);
-    double max_relative_norm2 = 0;
+    ValueType max_relative_norm2 = 0;
     for (gko::size_type i = 0; i < host_answer_norm->get_size()[1]; i++) {
         max_relative_norm2 =
             std::max(host_absolute_norm->at(0, i) / host_answer_norm->at(0, i),
diff --git a/benchmark/utils/hip_linops.hip.hpp b/benchmark/utils/hip_linops.hip.hpp
index d4af5c58b49..b46d85646f4 100644
--- a/benchmark/utils/hip_linops.hip.hpp
+++ b/benchmark/utils/hip_linops.hip.hpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <hipsparse.h>
 
 
+#include "benchmark/utils/types.hpp"
 #include "hip/base/device_guard.hip.hpp"
 #include "hip/base/hipsparse_bindings.hip.hpp"
 
@@ -321,14 +322,14 @@ class HipspHybrid
 
 
 // Some shortcuts
-using hipsp_csr = detail::HipspCsr<>;
-using hipsp_csrmm = detail::HipspCsrmm<>;
+using hipsp_csr = detail::HipspCsr<etype>;
+using hipsp_csrmm = detail::HipspCsrmm<etype>;
 
 
 using hipsp_coo =
-    detail::HipspHybrid<double, gko::int32, HIPSPARSE_HYB_PARTITION_USER, 0>;
+    detail::HipspHybrid<etype, gko::int32, HIPSPARSE_HYB_PARTITION_USER, 0>;
 using hipsp_ell =
-    detail::HipspHybrid<double, gko::int32, HIPSPARSE_HYB_PARTITION_MAX, 0>;
-using hipsp_hybrid = detail::HipspHybrid<>;
+    detail::HipspHybrid<etype, gko::int32, HIPSPARSE_HYB_PARTITION_MAX, 0>;
+using hipsp_hybrid = detail::HipspHybrid<etype>;
 
 #endif  // GKO_BENCHMARK_UTILS_HIP_LINOPS_HIP_HPP_
diff --git a/benchmark/utils/preconditioners.hpp b/benchmark/utils/preconditioners.hpp
index fd7294226df..8dd10bfddf3 100644
--- a/benchmark/utils/preconditioners.hpp
+++ b/benchmark/utils/preconditioners.hpp
@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "benchmark/utils/general.hpp"
 #include "benchmark/utils/overhead_linop.hpp"
+#include "benchmark/utils/types.hpp"
 
 
 DEFINE_string(
@@ -100,81 +101,91 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
     precond_factory{
         {"none",
          [](std::shared_ptr<const gko::Executor> exec) {
-             return gko::matrix::IdentityFactory<>::create(exec);
+             return gko::matrix::IdentityFactory<etype>::create(exec);
          }},
         {"jacobi",
          [](std::shared_ptr<const gko::Executor> exec) {
-             return gko::preconditioner::Jacobi<>::build()
+             return gko::preconditioner::Jacobi<etype>::build()
                  .with_max_block_size(FLAGS_jacobi_max_block_size)
                  .with_storage_optimization(
                      parse_storage_optimization(FLAGS_jacobi_storage))
-                 .with_accuracy(FLAGS_jacobi_accuracy)
+                 .with_accuracy(static_cast<etype>(FLAGS_jacobi_accuracy))
                  .with_skip_sorting(true)
                  .on(exec);
          }},
         {"parict",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact = gko::share(
-                 gko::factorization::ParIct<>::build()
+                 gko::factorization::ParIct<etype>::build()
                      .with_iterations(FLAGS_parilu_iterations)
                      .with_approximate_select(FLAGS_parilut_approx_select)
                      .with_fill_in_limit(FLAGS_parilut_limit)
                      .with_skip_sorting(true)
                      .on(exec));
-             return gko::preconditioner::Ilu<>::build()
+             return gko::preconditioner::Ilu<
+                        gko::solver::LowerTrs<etype>,
+                        gko::solver::UpperTrs<etype>>::build()
                  .with_factorization_factory(fact)
                  .on(exec);
          }},
         {"parilu",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact =
-                 gko::share(gko::factorization::ParIlu<>::build()
+                 gko::share(gko::factorization::ParIlu<etype>::build()
                                 .with_iterations(FLAGS_parilu_iterations)
                                 .with_skip_sorting(true)
                                 .on(exec));
-             return gko::preconditioner::Ilu<>::build()
+             return gko::preconditioner::Ilu<
+                        gko::solver::LowerTrs<etype>,
+                        gko::solver::UpperTrs<etype>>::build()
                  .with_factorization_factory(fact)
                  .on(exec);
          }},
         {"parilut",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact = gko::share(
-                 gko::factorization::ParIlut<>::build()
+                 gko::factorization::ParIlut<etype>::build()
                      .with_iterations(FLAGS_parilu_iterations)
                      .with_approximate_select(FLAGS_parilut_approx_select)
                      .with_fill_in_limit(FLAGS_parilut_limit)
                      .with_skip_sorting(true)
                      .on(exec));
-             return gko::preconditioner::Ilu<>::build()
+             return gko::preconditioner::Ilu<
+                        gko::solver::LowerTrs<etype>,
+                        gko::solver::UpperTrs<etype>>::build()
                  .with_factorization_factory(fact)
                  .on(exec);
          }},
         {"ilu",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact =
-                 gko::share(gko::factorization::Ilu<>::build().on(exec));
-             return gko::preconditioner::Ilu<>::build()
+                 gko::share(gko::factorization::Ilu<etype>::build().on(exec));
+             return gko::preconditioner::Ilu<
+                        gko::solver::LowerTrs<etype>,
+                        gko::solver::UpperTrs<etype>>::build()
                  .with_factorization_factory(fact)
                  .on(exec);
          }},
         {"parict-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact = gko::share(
-                 gko::factorization::ParIct<>::build()
+                 gko::factorization::ParIct<etype>::build()
                      .with_iterations(FLAGS_parilu_iterations)
                      .with_approximate_select(FLAGS_parilut_approx_select)
                      .with_fill_in_limit(FLAGS_parilut_limit)
                      .with_skip_sorting(true)
                      .on(exec));
-             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
-             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
+             auto lisai =
+                 gko::share(gko::preconditioner::LowerIsai<etype>::build()
+                                .with_sparsity_power(FLAGS_isai_power)
+                                .on(exec));
+             auto uisai =
+                 gko::share(gko::preconditioner::UpperIsai<etype>::build()
+                                .with_sparsity_power(FLAGS_isai_power)
+                                .on(exec));
              return gko::preconditioner::Ilu<
-                        gko::preconditioner::LowerIsai<>,
-                        gko::preconditioner::UpperIsai<>>::build()
+                        gko::preconditioner::LowerIsai<etype>,
+                        gko::preconditioner::UpperIsai<etype>>::build()
                  .with_factorization_factory(fact)
                  .with_l_solver_factory(lisai)
                  .with_u_solver_factory(uisai)
@@ -183,19 +194,21 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"parilu-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact =
-                 gko::share(gko::factorization::ParIlu<>::build()
+                 gko::share(gko::factorization::ParIlu<etype>::build()
                                 .with_iterations(FLAGS_parilu_iterations)
                                 .with_skip_sorting(true)
                                 .on(exec));
-             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
-             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
+             auto lisai =
+                 gko::share(gko::preconditioner::LowerIsai<etype>::build()
+                                .with_sparsity_power(FLAGS_isai_power)
+                                .on(exec));
+             auto uisai =
+                 gko::share(gko::preconditioner::UpperIsai<etype>::build()
+                                .with_sparsity_power(FLAGS_isai_power)
+                                .on(exec));
              return gko::preconditioner::Ilu<
-                        gko::preconditioner::LowerIsai<>,
-                        gko::preconditioner::UpperIsai<>>::build()
+                        gko::preconditioner::LowerIsai<etype>,
+                        gko::preconditioner::UpperIsai<etype>>::build()
                  .with_factorization_factory(fact)
                  .with_l_solver_factory(lisai)
                  .with_u_solver_factory(uisai)
@@ -204,21 +217,23 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"parilut-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact = gko::share(
-                 gko::factorization::ParIlut<>::build()
+                 gko::factorization::ParIlut<etype>::build()
                      .with_iterations(FLAGS_parilu_iterations)
                      .with_approximate_select(FLAGS_parilut_approx_select)
                      .with_fill_in_limit(FLAGS_parilut_limit)
                      .with_skip_sorting(true)
                      .on(exec));
-             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
-             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
+             auto lisai =
+                 gko::share(gko::preconditioner::LowerIsai<etype>::build()
+                                .with_sparsity_power(FLAGS_isai_power)
+                                .on(exec));
+             auto uisai =
+                 gko::share(gko::preconditioner::UpperIsai<etype>::build()
+                                .with_sparsity_power(FLAGS_isai_power)
+                                .on(exec));
              return gko::preconditioner::Ilu<
-                        gko::preconditioner::LowerIsai<>,
-                        gko::preconditioner::UpperIsai<>>::build()
+                        gko::preconditioner::LowerIsai<etype>,
+                        gko::preconditioner::UpperIsai<etype>>::build()
                  .with_factorization_factory(fact)
                  .with_l_solver_factory(lisai)
                  .with_u_solver_factory(uisai)
@@ -227,23 +242,29 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"ilu-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact =
-                 gko::share(gko::factorization::Ilu<>::build().on(exec));
-             auto lisai = gko::share(gko::preconditioner::LowerIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
-             auto uisai = gko::share(gko::preconditioner::UpperIsai<>::build()
-                                         .with_sparsity_power(FLAGS_isai_power)
-                                         .on(exec));
+                 gko::share(gko::factorization::Ilu<etype>::build().on(exec));
+             auto lisai =
+                 gko::share(gko::preconditioner::LowerIsai<etype>::build()
+                                .with_sparsity_power(FLAGS_isai_power)
+                                .on(exec));
+             auto uisai =
+                 gko::share(gko::preconditioner::UpperIsai<etype>::build()
+                                .with_sparsity_power(FLAGS_isai_power)
+                                .on(exec));
              return gko::preconditioner::Ilu<
-                        gko::preconditioner::LowerIsai<>,
-                        gko::preconditioner::UpperIsai<>>::build()
+                        gko::preconditioner::LowerIsai<etype>,
+                        gko::preconditioner::UpperIsai<etype>>::build()
                  .with_factorization_factory(fact)
                  .with_l_solver_factory(lisai)
                  .with_u_solver_factory(uisai)
                  .on(exec);
          }},
         {"overhead", [](std::shared_ptr<const gko::Executor> exec) {
-             return gko::Overhead<>::build().on(exec);
+             return gko::Overhead<etype>::build()
+                 .with_criteria(gko::stop::ResidualNormReduction<etype>::build()
+                                    .with_reduction_factor(etype{})
+                                    .on(exec))
+                 .on(exec);
          }}};
 
 
diff --git a/benchmark/utils/spmv_common.hpp b/benchmark/utils/spmv_common.hpp
index 0603d3706fb..4fd4ff21d95 100644
--- a/benchmark/utils/spmv_common.hpp
+++ b/benchmark/utils/spmv_common.hpp
@@ -44,10 +44,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <rapidjson/document.h>
 
 
-// some shortcuts
-using hybrid = gko::matrix::Hybrid<>;
-using csr = gko::matrix::Csr<>;
-
 /**
  * Function which outputs the input format for benchmarks similar to the spmv.
  */
diff --git a/benchmark/utils/types.hpp b/benchmark/utils/types.hpp
new file mode 100644
index 00000000000..05e3223613b
--- /dev/null
+++ b/benchmark/utils/types.hpp
@@ -0,0 +1,40 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_BENCHMARK_UTILS_TYPES_HPP_
+#define GKO_BENCHMARK_UTILS_TYPES_HPP_
+
+
+using etype = float;
+
+
+#endif  // GKO_BENCHMARK_UTILS_TYPES_HPP_

From 85041ef3b3bef956ded5e4d6e4fcb94cbe556d30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 7 Dec 2020 14:33:55 +0100
Subject: [PATCH 0244/1008] Change benchmark precision to `double` again

---
 benchmark/utils/types.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmark/utils/types.hpp b/benchmark/utils/types.hpp
index 05e3223613b..2bef33cd76a 100644
--- a/benchmark/utils/types.hpp
+++ b/benchmark/utils/types.hpp
@@ -34,7 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_BENCHMARK_UTILS_TYPES_HPP_
 
 
-using etype = float;
+using etype = double;
 
 
 #endif  // GKO_BENCHMARK_UTILS_TYPES_HPP_

From d2fe5e2a6cac849a23add2d3dad6f89615c25210 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 10 Dec 2020 04:56:35 +0100
Subject: [PATCH 0245/1008] Add support for multiple precisions in benchmark

Every benchmark is build with multiple precisions (currently
double and single precision), each generating its own binary.
The "run_all_benchmarks.sh" script additionally got a variable deciding
which precision to choose (which binary to execute).
---
 BENCHMARKING.md                               |  2 ++
 benchmark/CMakeLists.txt                      | 30 +++++++++++++++++++
 benchmark/conversions/CMakeLists.txt          |  3 +-
 benchmark/matrix_generator/CMakeLists.txt     |  3 +-
 .../matrix_generator/matrix_generator.cpp     |  2 +-
 benchmark/matrix_statistics/CMakeLists.txt    |  3 +-
 benchmark/preconditioner/CMakeLists.txt       |  3 +-
 benchmark/run_all_benchmarks.sh               | 17 +++++++++--
 benchmark/solver/CMakeLists.txt               |  9 +-----
 benchmark/solver/solver.cpp                   | 19 ++++++------
 benchmark/spmv/CMakeLists.txt                 |  9 +-----
 benchmark/utils/types.hpp                     | 11 ++++++-
 12 files changed, 74 insertions(+), 37 deletions(-)

diff --git a/BENCHMARKING.md b/BENCHMARKING.md
index 883100d0f02..84eae796cc7 100644
--- a/BENCHMARKING.md
+++ b/BENCHMARKING.md
@@ -156,6 +156,8 @@ benchmark options). Here are the most important options:
 * `SEGMENT_ID=<I>` - used in combination with the `SEGMENTS` variable. `<I>`
     should be an integer between 1 and `<N>`, the number of `SEGMENTS`. If
     specified, only the `<I>`-th segment of the benchmark suite will be run.
+* `BENCHMARK_PRECISION` - defines the precision the benchmarks are run in.
+    Supported values are: "double" (default) and "single"
 * `MATRIX_LIST_FILE=/path/to/matrix_list.file` - allows to list SuiteSparse
     matrix id or name to benchmark. As an example, a matrix list file containing
     the following will ensure that benchmarks are ran for only those three
diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
index ad63598fe70..f900be86100 100644
--- a/benchmark/CMakeLists.txt
+++ b/benchmark/CMakeLists.txt
@@ -54,6 +54,36 @@ function(ginkgo_benchmark_hipsp_linops name)
     target_link_libraries("${name}" ${HIPSPARSE_LIBRARIES})
 endfunction()
 
+
+# Generates an executable for each supported precision. Each executable will be
+# linked to `ginkgo`, `gflags` and `rapidjson`.
+#
+# \param name            base-name for the executable to create
+# \param use_lib_linops  Boolean indicating if linking against hipsparse/cusparse
+#                        is necessary
+# All remaining arguments will be treated as source files
+function(ginkgo_add_benchmark_executable name use_lib_linops)
+    set(SINGLE_PREC_SUFFIX "_single")
+    set(SINGLE_PREC_MACRO "GKO_BENCHMARK_USE_SINGLE_PRECISION")
+    add_executable("${name}" ${ARGN})
+    target_link_libraries("${name}" ginkgo gflags rapidjson)
+
+    add_executable("${name}${SINGLE_PREC_SUFFIX}" ${ARGN})
+    target_link_libraries("${name}${SINGLE_PREC_SUFFIX}" ginkgo gflags rapidjson)
+    target_compile_definitions("${name}${SINGLE_PREC_SUFFIX}" PRIVATE "${SINGLE_PREC_MACRO}")
+    if("${use_lib_linops}")
+        if (GINKGO_BUILD_CUDA)
+            ginkgo_benchmark_cusp_linops("${name}")
+            ginkgo_benchmark_cusp_linops("${name}${SINGLE_PREC_SUFFIX}")
+        endif()
+        if (GINKGO_BUILD_HIP)
+            ginkgo_benchmark_hipsp_linops("${name}")
+            ginkgo_benchmark_hipsp_linops("${name}${SINGLE_PREC_SUFFIX}")
+        endif()
+    endif()
+endfunction(ginkgo_add_benchmark_executable)
+
+
 add_subdirectory(conversions)
 add_subdirectory(matrix_generator)
 add_subdirectory(matrix_statistics)
diff --git a/benchmark/conversions/CMakeLists.txt b/benchmark/conversions/CMakeLists.txt
index 7f4a43f73c1..508c8298e87 100644
--- a/benchmark/conversions/CMakeLists.txt
+++ b/benchmark/conversions/CMakeLists.txt
@@ -1,2 +1 @@
-add_executable(conversions conversions.cpp)
-target_link_libraries(conversions ginkgo gflags rapidjson)
+ginkgo_add_benchmark_executable(conversions "NO" conversions.cpp)
diff --git a/benchmark/matrix_generator/CMakeLists.txt b/benchmark/matrix_generator/CMakeLists.txt
index 95d4e00ac51..e41bc47f840 100644
--- a/benchmark/matrix_generator/CMakeLists.txt
+++ b/benchmark/matrix_generator/CMakeLists.txt
@@ -1,2 +1 @@
-add_executable(matrix_generator matrix_generator.cpp)
-target_link_libraries(matrix_generator ginkgo gflags rapidjson)
+ginkgo_add_benchmark_executable(matrix_generator "NO" matrix_generator.cpp)
diff --git a/benchmark/matrix_generator/matrix_generator.cpp b/benchmark/matrix_generator/matrix_generator.cpp
index 90a63702e86..ab98a20d20e 100644
--- a/benchmark/matrix_generator/matrix_generator.cpp
+++ b/benchmark/matrix_generator/matrix_generator.cpp
@@ -107,7 +107,7 @@ gko::matrix_data<etype> generate_block_diagonal(rapidjson::Value &config,
     auto block_size = config["block_size"].GetUint();
     auto block = gko::matrix_data<etype>(
         gko::dim<2>(block_size),
-        std::uniform_real_distribution<etype>(-1.0, 1.0), engine);
+        std::uniform_real_distribution<rc_etype>(-1.0, 1.0), engine);
     return gko::matrix_data<etype>::diag(num_blocks, block);
 }
 
diff --git a/benchmark/matrix_statistics/CMakeLists.txt b/benchmark/matrix_statistics/CMakeLists.txt
index 2de10097f36..7ca4c531e5d 100644
--- a/benchmark/matrix_statistics/CMakeLists.txt
+++ b/benchmark/matrix_statistics/CMakeLists.txt
@@ -1,2 +1 @@
-add_executable(matrix_statistics matrix_statistics.cpp)
-target_link_libraries(matrix_statistics ginkgo gflags rapidjson)
+ginkgo_add_benchmark_executable(matrix_statistics "NO" matrix_statistics.cpp)
diff --git a/benchmark/preconditioner/CMakeLists.txt b/benchmark/preconditioner/CMakeLists.txt
index 962831fc1d4..666e75c2fd2 100644
--- a/benchmark/preconditioner/CMakeLists.txt
+++ b/benchmark/preconditioner/CMakeLists.txt
@@ -1,2 +1 @@
-add_executable(preconditioner preconditioner.cpp)
-target_link_libraries(preconditioner ginkgo gflags rapidjson)
+ginkgo_add_benchmark_executable(preconditioner "NO" preconditioner.cpp)
diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index d8bb6c6a88b..aea0f09dbde 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -65,6 +65,19 @@ if [ ! "${SOLVERS_RHS}" ]; then
     SOLVERS_RHS="unit"
 fi
 
+if [ ! "${BENCHMARK_PRECISION}" ]; then
+    echo "BENCHMARK_PRECISION not set - assuming \"double\"" 1>&2
+    BENCH_SUFFIX=""
+elif [ "${BENCHMARK_PRECISION}" == "double" ]; then
+    BENCH_SUFFIX=""
+elif [ "${BENCHMARK_PRECISION}" == "single" ]; then
+    BENCH_SUFFIX="_single"
+else
+    echo "BENCHMARK_PRECISION is set to the not supported \"${BENCHMARK_PRECISION}\"." 1>&2
+    echo "Currently supported values: \"double\" and \"single\"" 1>&2
+    exit 1
+fi
+
 if [ "${SOLVERS_RHS}" == "random" ]; then
     SOLVERS_RHS_FLAG="--random_rhs=true"
 else
@@ -132,7 +145,7 @@ keep_latest() {
 compute_matrix_statistics() {
     [ "${DRY_RUN}" == "true" ] && return
     cp "$1" "$1.imd" # make sure we're not loosing the original input
-    ./matrix_statistics/matrix_statistics \
+    ./matrix_statistics/matrix_statistics${BENCH_SUFFIX} \
         --backup="$1.bkp" --double_buffer="$1.bkp2" \
         <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
@@ -204,7 +217,7 @@ run_preconditioner_benchmarks() {
         for prec in ${PRECISIONS}; do
             echo -e "\t\t running jacobi ($prec) for block size ${bsize}" 1>&2
             cp "$1" "$1.imd" # make sure we're not loosing the original input
-            ./preconditioner/preconditioner \
+            ./preconditioner/preconditioner${BENCH_SUFFIX} \
                 --backup="$1.bkp" --double_buffer="$1.bkp2" \
                 --executor="${EXECUTOR}" --preconditioners="jacobi" \
                 --jacobi_max_block_size="${bsize}" \
diff --git a/benchmark/solver/CMakeLists.txt b/benchmark/solver/CMakeLists.txt
index 1faae042b24..b45fa9c6757 100644
--- a/benchmark/solver/CMakeLists.txt
+++ b/benchmark/solver/CMakeLists.txt
@@ -1,8 +1 @@
-add_executable(solver solver.cpp)
-target_link_libraries(solver ginkgo gflags rapidjson)
-if (GINKGO_BUILD_CUDA)
-    ginkgo_benchmark_cusp_linops(solver)
-endif()
-if (GINKGO_BUILD_HIP)
-    ginkgo_benchmark_hipsp_linops(solver)
-endif()
\ No newline at end of file
+ginkgo_add_benchmark_executable(solver "YES" solver.cpp)
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 09454e763bd..3ac5e331de5 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -130,14 +130,15 @@ std::shared_ptr<const gko::stop::CriterionFactory> create_criterion(
     std::shared_ptr<const gko::stop::CriterionFactory> residual_stop;
     if (FLAGS_rel_residual) {
         residual_stop = gko::share(
-            gko::stop::RelativeResidualNorm<etype>::build()
-                .with_tolerance(static_cast<etype>(FLAGS_rel_res_goal))
+            gko::stop::RelativeResidualNorm<rc_etype>::build()
+                .with_tolerance(static_cast<rc_etype>(FLAGS_rel_res_goal))
                 .on(exec));
     } else {
-        residual_stop = gko::share(
-            gko::stop::ResidualNormReduction<etype>::build()
-                .with_reduction_factor(static_cast<etype>(FLAGS_rel_res_goal))
-                .on(exec));
+        residual_stop =
+            gko::share(gko::stop::ResidualNormReduction<rc_etype>::build()
+                           .with_reduction_factor(
+                               static_cast<rc_etype>(FLAGS_rel_res_goal))
+                           .on(exec));
     }
     auto iteration_stop = gko::share(
         gko::stop::Iteration::build().with_max_iters(FLAGS_max_iters).on(exec));
@@ -174,7 +175,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                         return gko::solver::Idr<etype>::build()
                             .with_criteria(create_criterion(exec))
                             .with_subspace_dim(FLAGS_idr_subspace_dim)
-                            .with_kappa(static_cast<etype>(FLAGS_idr_kappa))
+                            .with_kappa(static_cast<rc_etype>(FLAGS_idr_kappa))
                             .with_preconditioner(give(precond))
                             .on(exec);
                     }},
@@ -343,7 +344,7 @@ void solve_system(const std::string &solver_name,
             // slow run, gets the recurrent and true residuals of each iteration
             if (b->get_size()[1] == 1) {
                 x_clone = clone(x);
-                auto res_logger = std::make_shared<ResidualLogger<etype>>(
+                auto res_logger = std::make_shared<ResidualLogger<rc_etype>>(
                     exec, lend(system_matrix), b,
                     solver_json["recurrent_residuals"],
                     solver_json["true_residuals"],
@@ -478,7 +479,7 @@ int main(int argc, char *argv[])
             if (FLAGS_overhead) {
                 system_matrix = gko::initialize<Vec>({1.0}, exec);
                 b = gko::initialize<Vec>(
-                    {std::numeric_limits<etype>::quiet_NaN()}, exec);
+                    {std::numeric_limits<rc_etype>::quiet_NaN()}, exec);
                 x = gko::initialize<Vec>({0.0}, exec);
             } else {
                 auto data = gko::read_raw<etype>(mtx_fd);
diff --git a/benchmark/spmv/CMakeLists.txt b/benchmark/spmv/CMakeLists.txt
index 222d3f750b4..3b06d3d84ca 100644
--- a/benchmark/spmv/CMakeLists.txt
+++ b/benchmark/spmv/CMakeLists.txt
@@ -1,8 +1 @@
-add_executable(spmv spmv.cpp)
-target_link_libraries(spmv ginkgo gflags rapidjson)
-if (GINKGO_BUILD_CUDA)
-    ginkgo_benchmark_cusp_linops(spmv)
-endif()
-if (GINKGO_BUILD_HIP)
-    ginkgo_benchmark_hipsp_linops(spmv)
-endif()
+ginkgo_add_benchmark_executable(spmv "YES" spmv.cpp)
diff --git a/benchmark/utils/types.hpp b/benchmark/utils/types.hpp
index 2bef33cd76a..ca8f57dae08 100644
--- a/benchmark/utils/types.hpp
+++ b/benchmark/utils/types.hpp
@@ -34,7 +34,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_BENCHMARK_UTILS_TYPES_HPP_
 
 
-using etype = double;
+#include <ginkgo/core/base/math.hpp>
+
+
+#ifdef GKO_BENCHMARK_USE_SINGLE_PRECISION
+using etype = float;
+#else
+using etype = double;  // std::complex<double>;
+#endif
+
+using rc_etype = gko::remove_complex<etype>;
 
 
 #endif  // GKO_BENCHMARK_UTILS_TYPES_HPP_

From 88d6f35828ceb47375889c20cf1c4fd3d36da274 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 11 Dec 2020 09:12:23 +0100
Subject: [PATCH 0246/1008] Make benchmark compatible with complex numbers

---
 benchmark/solver/solver.cpp         |  2 +-
 benchmark/utils/general.hpp         | 26 +++++++++++++++-----------
 benchmark/utils/loggers.hpp         |  4 +++-
 benchmark/utils/preconditioners.hpp |  4 ++--
 benchmark/utils/types.hpp           |  5 ++++-
 5 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 3ac5e331de5..bd0c6422e5f 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -344,7 +344,7 @@ void solve_system(const std::string &solver_name,
             // slow run, gets the recurrent and true residuals of each iteration
             if (b->get_size()[1] == 1) {
                 x_clone = clone(x);
-                auto res_logger = std::make_shared<ResidualLogger<rc_etype>>(
+                auto res_logger = std::make_shared<ResidualLogger<etype>>(
                     exec, lend(system_matrix), b,
                     solver_json["recurrent_residuals"],
                     solver_json["true_residuals"],
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index fc449b69c16..3094083c26d 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -316,7 +316,9 @@ std::unique_ptr<vec<ValueType>> create_matrix(
     auto res = vec<ValueType>::create(exec);
     if (random) {
         res->read(gko::matrix_data<ValueType>(
-            size, std::uniform_real_distribution<ValueType>(-1.0, 1.0),
+            size,
+            std::uniform_real_distribution<gko::remove_complex<ValueType>>(-1.0,
+                                                                           1.0),
             engine));
     } else {
         res->read(gko::matrix_data<ValueType>(size, gko::one<ValueType>()));
@@ -344,19 +346,20 @@ ValueType get_norm(const vec<ValueType> *norm)
 
 
 template <typename ValueType>
-ValueType compute_norm2(const vec<ValueType> *b)
+gko::remove_complex<ValueType> compute_norm2(const vec<ValueType> *b)
 {
     auto exec = b->get_executor();
-    auto b_norm = gko::initialize<vec<ValueType>>({0.0}, exec);
+    auto b_norm =
+        gko::initialize<vec<gko::remove_complex<ValueType>>>({0.0}, exec);
     b->compute_norm2(lend(b_norm));
     return get_norm(lend(b_norm));
 }
 
 
 template <typename ValueType>
-ValueType compute_residual_norm(const gko::LinOp *system_matrix,
-                                const vec<ValueType> *b,
-                                const vec<ValueType> *x)
+gko::remove_complex<ValueType> compute_residual_norm(
+    const gko::LinOp *system_matrix, const vec<ValueType> *b,
+    const vec<ValueType> *x)
 {
     auto exec = system_matrix->get_executor();
     auto one = gko::initialize<vec<ValueType>>({1.0}, exec);
@@ -368,23 +371,24 @@ ValueType compute_residual_norm(const gko::LinOp *system_matrix,
 
 
 template <typename ValueType>
-ValueType compute_max_relative_norm2(vec<ValueType> *result,
-                                     const vec<ValueType> *answer)
+gko::remove_complex<ValueType> compute_max_relative_norm2(
+    vec<ValueType> *result, const vec<ValueType> *answer)
 {
+    using rc_vtype = gko::remove_complex<ValueType>;
     auto exec = answer->get_executor();
     auto answer_norm =
-        vec<ValueType>::create(exec, gko::dim<2>{1, answer->get_size()[1]});
+        vec<rc_vtype>::create(exec, gko::dim<2>{1, answer->get_size()[1]});
     answer->compute_norm2(lend(answer_norm));
     auto neg_one = gko::initialize<vec<ValueType>>({-1.0}, exec);
     result->add_scaled(lend(neg_one), lend(answer));
     auto absolute_norm =
-        vec<ValueType>::create(exec, gko::dim<2>{1, answer->get_size()[1]});
+        vec<rc_vtype>::create(exec, gko::dim<2>{1, answer->get_size()[1]});
     result->compute_norm2(lend(absolute_norm));
     auto host_answer_norm =
         clone(answer_norm->get_executor()->get_master(), answer_norm);
     auto host_absolute_norm =
         clone(absolute_norm->get_executor()->get_master(), absolute_norm);
-    ValueType max_relative_norm2 = 0;
+    rc_vtype max_relative_norm2 = 0;
     for (gko::size_type i = 0; i < host_answer_norm->get_size()[1]; i++) {
         max_relative_norm2 =
             std::max(host_absolute_norm->at(0, i) / host_answer_norm->at(0, i),
diff --git a/benchmark/utils/loggers.hpp b/benchmark/utils/loggers.hpp
index 1a04bd98644..5edf8f96b69 100644
--- a/benchmark/utils/loggers.hpp
+++ b/benchmark/utils/loggers.hpp
@@ -200,6 +200,8 @@ struct StorageLogger : gko::log::Logger {
 // Logs true and recurrent residuals of the solver
 template <typename ValueType>
 struct ResidualLogger : gko::log::Logger {
+    using rc_vtype = gko::remove_complex<ValueType>;
+
     void on_iteration_complete(const gko::LinOp *, const gko::size_type &,
                                const gko::LinOp *residual,
                                const gko::LinOp *solution,
@@ -211,7 +213,7 @@ struct ResidualLogger : gko::log::Logger {
                             alloc);
         if (residual_norm) {
             rec_res_norms.PushBack(
-                get_norm(gko::as<vec<ValueType>>(residual_norm)), alloc);
+                get_norm(gko::as<vec<rc_vtype>>(residual_norm)), alloc);
         } else {
             rec_res_norms.PushBack(
                 compute_norm2(gko::as<vec<ValueType>>(residual)), alloc);
diff --git a/benchmark/utils/preconditioners.hpp b/benchmark/utils/preconditioners.hpp
index 8dd10bfddf3..4c7dc733e34 100644
--- a/benchmark/utils/preconditioners.hpp
+++ b/benchmark/utils/preconditioners.hpp
@@ -109,7 +109,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_max_block_size(FLAGS_jacobi_max_block_size)
                  .with_storage_optimization(
                      parse_storage_optimization(FLAGS_jacobi_storage))
-                 .with_accuracy(static_cast<etype>(FLAGS_jacobi_accuracy))
+                 .with_accuracy(static_cast<rc_etype>(FLAGS_jacobi_accuracy))
                  .with_skip_sorting(true)
                  .on(exec);
          }},
@@ -262,7 +262,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"overhead", [](std::shared_ptr<const gko::Executor> exec) {
              return gko::Overhead<etype>::build()
                  .with_criteria(gko::stop::ResidualNormReduction<etype>::build()
-                                    .with_reduction_factor(etype{})
+                                    .with_reduction_factor(rc_etype{})
                                     .on(exec))
                  .on(exec);
          }}};
diff --git a/benchmark/utils/types.hpp b/benchmark/utils/types.hpp
index ca8f57dae08..1e9cce1ca02 100644
--- a/benchmark/utils/types.hpp
+++ b/benchmark/utils/types.hpp
@@ -34,13 +34,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_BENCHMARK_UTILS_TYPES_HPP_
 
 
+#include <complex>
+
+
 #include <ginkgo/core/base/math.hpp>
 
 
 #ifdef GKO_BENCHMARK_USE_SINGLE_PRECISION
 using etype = float;
 #else
-using etype = double;  // std::complex<double>;
+using etype = double;
 #endif
 
 using rc_etype = gko::remove_complex<etype>;

From a80036a67bec3949935d8c64ed7d8b5166381e8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 11 Dec 2020 11:10:14 +0100
Subject: [PATCH 0247/1008] Add support for complex types in benchmark script

Benchmark binaries are now always generated for single, double,
single complex and double complex.
---
 BENCHMARKING.md                            |  2 +-
 benchmark/CMakeLists.txt                   | 43 +++++++++++++++-------
 benchmark/conversions/CMakeLists.txt       |  2 +-
 benchmark/matrix_generator/CMakeLists.txt  |  2 +-
 benchmark/matrix_statistics/CMakeLists.txt |  2 +-
 benchmark/preconditioner/CMakeLists.txt    |  2 +-
 benchmark/run_all_benchmarks.sh            |  4 ++
 benchmark/solver/CMakeLists.txt            |  2 +-
 benchmark/spmv/CMakeLists.txt              |  2 +-
 benchmark/utils/types.hpp                  | 10 ++++-
 10 files changed, 49 insertions(+), 22 deletions(-)

diff --git a/BENCHMARKING.md b/BENCHMARKING.md
index 84eae796cc7..31a5546f7e8 100644
--- a/BENCHMARKING.md
+++ b/BENCHMARKING.md
@@ -157,7 +157,7 @@ benchmark options). Here are the most important options:
     should be an integer between 1 and `<N>`, the number of `SEGMENTS`. If
     specified, only the `<I>`-th segment of the benchmark suite will be run.
 * `BENCHMARK_PRECISION` - defines the precision the benchmarks are run in.
-    Supported values are: "double" (default) and "single"
+    Supported values are: "double" (default), "single", "dcomplex" and "scomplex"
 * `MATRIX_LIST_FILE=/path/to/matrix_list.file` - allows to list SuiteSparse
     matrix id or name to benchmark. As an example, a matrix list file containing
     the following will ensure that benchmarks are ran for only those three
diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
index f900be86100..b55ba2326c1 100644
--- a/benchmark/CMakeLists.txt
+++ b/benchmark/CMakeLists.txt
@@ -55,33 +55,50 @@ function(ginkgo_benchmark_hipsp_linops name)
 endfunction()
 
 
-# Generates an executable for each supported precision. Each executable will be
-# linked to `ginkgo`, `gflags` and `rapidjson`.
+# Generates an executable for one precision. Each executable will be linked to
+# `ginkgo`, `gflags` and `rapidjson`.
+# Note: This should only be used by `ginkgo_add_typed_benchmark_executables`
 #
-# \param name            base-name for the executable to create
+# \param name            name for the executable to create (including type suffix)
 # \param use_lib_linops  Boolean indicating if linking against hipsparse/cusparse
 #                        is necessary
+# \param macro_def       preprocessor macro name that will be defined during
+#                        building (to compile for a specific type)
 # All remaining arguments will be treated as source files
-function(ginkgo_add_benchmark_executable name use_lib_linops)
-    set(SINGLE_PREC_SUFFIX "_single")
-    set(SINGLE_PREC_MACRO "GKO_BENCHMARK_USE_SINGLE_PRECISION")
+function(ginkgo_add_single_benchmark_executable name use_lib_linops macro_def)
     add_executable("${name}" ${ARGN})
     target_link_libraries("${name}" ginkgo gflags rapidjson)
-
-    add_executable("${name}${SINGLE_PREC_SUFFIX}" ${ARGN})
-    target_link_libraries("${name}${SINGLE_PREC_SUFFIX}" ginkgo gflags rapidjson)
-    target_compile_definitions("${name}${SINGLE_PREC_SUFFIX}" PRIVATE "${SINGLE_PREC_MACRO}")
+    target_compile_definitions("${name}" PRIVATE "${macro_def}")
     if("${use_lib_linops}")
         if (GINKGO_BUILD_CUDA)
             ginkgo_benchmark_cusp_linops("${name}")
-            ginkgo_benchmark_cusp_linops("${name}${SINGLE_PREC_SUFFIX}")
         endif()
         if (GINKGO_BUILD_HIP)
             ginkgo_benchmark_hipsp_linops("${name}")
-            ginkgo_benchmark_hipsp_linops("${name}${SINGLE_PREC_SUFFIX}")
         endif()
     endif()
-endfunction(ginkgo_add_benchmark_executable)
+
+
+endfunction(ginkgo_add_single_benchmark_executable)
+
+
+# Generates an executable for each supported precision. Each executable will be
+# linked to `ginkgo`, `gflags` and `rapidjson`.
+#
+# \param name            base-name for the executable to create
+# \param use_lib_linops  Boolean indicating if linking against hipsparse/cusparse
+#                        is necessary
+# All remaining arguments will be treated as source files
+function(ginkgo_add_typed_benchmark_executables name use_lib_linops)
+    ginkgo_add_single_benchmark_executable(
+        "${name}" use_lib_linops "GKO_BENCHMARK_USE_DOUBLE_PRECISION" ${ARGN})
+    ginkgo_add_single_benchmark_executable(
+        "${name}_single" use_lib_linops "GKO_BENCHMARK_USE_SINGLE_PRECISION" ${ARGN})
+    ginkgo_add_single_benchmark_executable(
+        "${name}_dcomplex" use_lib_linops "GKO_BENCHMARK_USE_DOUBLE_COMPLEX_PRECISION" ${ARGN})
+    ginkgo_add_single_benchmark_executable(
+        "${name}_scomplex" use_lib_linops "GKO_BENCHMARK_USE_SINGLE_COMPLEX_PRECISION" ${ARGN})
+endfunction(ginkgo_add_typed_benchmark_executables)
 
 
 add_subdirectory(conversions)
diff --git a/benchmark/conversions/CMakeLists.txt b/benchmark/conversions/CMakeLists.txt
index 508c8298e87..0e0893c3aec 100644
--- a/benchmark/conversions/CMakeLists.txt
+++ b/benchmark/conversions/CMakeLists.txt
@@ -1 +1 @@
-ginkgo_add_benchmark_executable(conversions "NO" conversions.cpp)
+ginkgo_add_typed_benchmark_executables(conversions "NO" conversions.cpp)
diff --git a/benchmark/matrix_generator/CMakeLists.txt b/benchmark/matrix_generator/CMakeLists.txt
index e41bc47f840..9ae32e71fae 100644
--- a/benchmark/matrix_generator/CMakeLists.txt
+++ b/benchmark/matrix_generator/CMakeLists.txt
@@ -1 +1 @@
-ginkgo_add_benchmark_executable(matrix_generator "NO" matrix_generator.cpp)
+ginkgo_add_typed_benchmark_executables(matrix_generator "NO" matrix_generator.cpp)
diff --git a/benchmark/matrix_statistics/CMakeLists.txt b/benchmark/matrix_statistics/CMakeLists.txt
index 7ca4c531e5d..e347ede0e16 100644
--- a/benchmark/matrix_statistics/CMakeLists.txt
+++ b/benchmark/matrix_statistics/CMakeLists.txt
@@ -1 +1 @@
-ginkgo_add_benchmark_executable(matrix_statistics "NO" matrix_statistics.cpp)
+ginkgo_add_typed_benchmark_executables(matrix_statistics "NO" matrix_statistics.cpp)
diff --git a/benchmark/preconditioner/CMakeLists.txt b/benchmark/preconditioner/CMakeLists.txt
index 666e75c2fd2..7b666000b04 100644
--- a/benchmark/preconditioner/CMakeLists.txt
+++ b/benchmark/preconditioner/CMakeLists.txt
@@ -1 +1 @@
-ginkgo_add_benchmark_executable(preconditioner "NO" preconditioner.cpp)
+ginkgo_add_typed_benchmark_executables(preconditioner "NO" preconditioner.cpp)
diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index aea0f09dbde..f3662b88568 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -72,6 +72,10 @@ elif [ "${BENCHMARK_PRECISION}" == "double" ]; then
     BENCH_SUFFIX=""
 elif [ "${BENCHMARK_PRECISION}" == "single" ]; then
     BENCH_SUFFIX="_single"
+elif [ "${BENCHMARK_PRECISION}" == "dcomplex" ]; then
+    BENCH_SUFFIX="_dcomplex"
+elif [ "${BENCHMARK_PRECISION}" == "scomplex" ]; then
+    BENCH_SUFFIX="_scomplex"
 else
     echo "BENCHMARK_PRECISION is set to the not supported \"${BENCHMARK_PRECISION}\"." 1>&2
     echo "Currently supported values: \"double\" and \"single\"" 1>&2
diff --git a/benchmark/solver/CMakeLists.txt b/benchmark/solver/CMakeLists.txt
index b45fa9c6757..11c08bdf6d2 100644
--- a/benchmark/solver/CMakeLists.txt
+++ b/benchmark/solver/CMakeLists.txt
@@ -1 +1 @@
-ginkgo_add_benchmark_executable(solver "YES" solver.cpp)
+ginkgo_add_typed_benchmark_executables(solver "YES" solver.cpp)
diff --git a/benchmark/spmv/CMakeLists.txt b/benchmark/spmv/CMakeLists.txt
index 3b06d3d84ca..6c5d10517a8 100644
--- a/benchmark/spmv/CMakeLists.txt
+++ b/benchmark/spmv/CMakeLists.txt
@@ -1 +1 @@
-ginkgo_add_benchmark_executable(spmv "YES" spmv.cpp)
+ginkgo_add_typed_benchmark_executables(spmv "YES" spmv.cpp)
diff --git a/benchmark/utils/types.hpp b/benchmark/utils/types.hpp
index 1e9cce1ca02..bc03218777e 100644
--- a/benchmark/utils/types.hpp
+++ b/benchmark/utils/types.hpp
@@ -40,9 +40,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#ifdef GKO_BENCHMARK_USE_SINGLE_PRECISION
+#if defined(GKO_BENCHMARK_USE_DOUBLE_PRECISION)
+using etype = double;
+#elif defined(GKO_BENCHMARK_USE_SINGLE_PRECISION)
 using etype = float;
-#else
+#elif defined(GKO_BENCHMARK_USE_DOUBLE_COMPLEX_PRECISION)
+using etype = std::complex<double>;
+#elif defined(GKO_BENCHMARK_USE_SINGLE_COMPLEX_PRECISION)
+using etype = std::complex<float>;
+#else  // default to double precision
 using etype = double;
 #endif
 

From 420c401f6573c89f026cef9292484959d399c71c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 15 Dec 2020 11:02:15 +0100
Subject: [PATCH 0248/1008] Review update

Update message for supported precisions.
Additionally, improve coding for BENCHMARK_PRECISION default case

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
---
 benchmark/run_all_benchmarks.sh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index f3662b88568..fb30eaac17b 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -66,9 +66,11 @@ if [ ! "${SOLVERS_RHS}" ]; then
 fi
 
 if [ ! "${BENCHMARK_PRECISION}" ]; then
-    echo "BENCHMARK_PRECISION not set - assuming \"double\"" 1>&2
-    BENCH_SUFFIX=""
-elif [ "${BENCHMARK_PRECISION}" == "double" ]; then
+    BENCHMARK_PRECISION="double"
+    echo "BENCHMARK_PRECISION not set - assuming \"${BENCHMARK_PRECISION}\"" 1>&2
+fi
+
+if [ "${BENCHMARK_PRECISION}" == "double" ]; then
     BENCH_SUFFIX=""
 elif [ "${BENCHMARK_PRECISION}" == "single" ]; then
     BENCH_SUFFIX="_single"
@@ -78,7 +80,7 @@ elif [ "${BENCHMARK_PRECISION}" == "scomplex" ]; then
     BENCH_SUFFIX="_scomplex"
 else
     echo "BENCHMARK_PRECISION is set to the not supported \"${BENCHMARK_PRECISION}\"." 1>&2
-    echo "Currently supported values: \"double\" and \"single\"" 1>&2
+    echo "Currently supported values: \"double\", \"single\", \"dcomplex\" and \"scomplex\"" 1>&2
     exit 1
 fi
 

From 07c2c2e9acd813480b57cc1e8c6fdb1e193667a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 15 Dec 2020 12:21:08 +0100
Subject: [PATCH 0249/1008] Benchmark: print res_goal in scientific notation

---
 benchmark/solver/solver.cpp | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index bd0c6422e5f..f75ae9216b1 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <iomanip>
 #include <iostream>
 #include <limits>
+#include <sstream>
 #include <vector>
 
 
@@ -415,12 +416,14 @@ int main(int argc, char *argv[])
         "format\n\n";
     initialize_argument_parsing(&argc, &argv, header, format);
 
-    std::string extra_information = "Running " + FLAGS_solvers + " with " +
-                                    std::to_string(FLAGS_max_iters) +
-                                    " iterations and residual goal of " +
-                                    std::to_string(FLAGS_rel_res_goal) +
-                                    "\nThe number of right hand sides is " +
-                                    std::to_string(FLAGS_nrhs) + "\n";
+    std::stringstream ss_rel_res_goal;
+    ss_rel_res_goal << std::scientific << FLAGS_rel_res_goal;
+
+    std::string extra_information =
+        "Running " + FLAGS_solvers + " with " +
+        std::to_string(FLAGS_max_iters) + " iterations and residual goal of " +
+        ss_rel_res_goal.str() + "\nThe number of right hand sides is " +
+        std::to_string(FLAGS_nrhs) + "\n";
     print_general_information(extra_information);
 
     auto exec = get_executor();

From 9cf7874c478696df3713f7d8fb5ec2ce08a86d8e Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Fri, 15 Jan 2021 04:52:31 +0000
Subject: [PATCH 0250/1008] Format files
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thoasm@users.noreply.github.com>
---
 benchmark/utils/types.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmark/utils/types.hpp b/benchmark/utils/types.hpp
index bc03218777e..c819ba30f92 100644
--- a/benchmark/utils/types.hpp
+++ b/benchmark/utils/types.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

From 329e86883f65779e8e29f319d1478170250c15eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 21 Jan 2021 14:15:38 +0100
Subject: [PATCH 0251/1008] Review update

Co-authored-by: tcojean <terry.cojean@kit.edu>
---
 benchmark/CMakeLists.txt | 2 --
 1 file changed, 2 deletions(-)

diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
index b55ba2326c1..9323039f6ea 100644
--- a/benchmark/CMakeLists.txt
+++ b/benchmark/CMakeLists.txt
@@ -77,8 +77,6 @@ function(ginkgo_add_single_benchmark_executable name use_lib_linops macro_def)
             ginkgo_benchmark_hipsp_linops("${name}")
         endif()
     endif()
-
-
 endfunction(ginkgo_add_single_benchmark_executable)
 
 

From 39a8dc046871a9eef3f78a6e55e004c358f73ede Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 21 Jan 2021 14:46:36 +0100
Subject: [PATCH 0252/1008] Update benchmark script to run correct precision

---
 benchmark/run_all_benchmarks.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index fb30eaac17b..9105bd85e10 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -166,7 +166,7 @@ compute_matrix_statistics() {
 run_conversion_benchmarks() {
     [ "${DRY_RUN}" == "true" ] && return
     cp "$1" "$1.imd" # make sure we're not loosing the original input
-    ./conversions/conversions --backup="$1.bkp" --double_buffer="$1.bkp2" \
+    ./conversions/conversions${BENCH_SUFFIX} --backup="$1.bkp" --double_buffer="$1.bkp2" \
                 --executor="${EXECUTOR}" --formats="${FORMATS}" \
                 --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
                 <"$1.imd" 2>&1 >"$1"
@@ -182,7 +182,7 @@ run_conversion_benchmarks() {
 run_spmv_benchmarks() {
     [ "${DRY_RUN}" == "true" ] && return
     cp "$1" "$1.imd" # make sure we're not loosing the original input
-    ./spmv/spmv --backup="$1.bkp" --double_buffer="$1.bkp2" \
+    ./spmv/spmv${BENCH_SUFFIX} --backup="$1.bkp" --double_buffer="$1.bkp2" \
                 --executor="${EXECUTOR}" --formats="${FORMATS}" \
                 --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
                 <"$1.imd" 2>&1 >"$1"
@@ -198,7 +198,7 @@ run_spmv_benchmarks() {
 run_solver_benchmarks() {
     [ "${DRY_RUN}" == "true" ] && return
     cp "$1" "$1.imd" # make sure we're not loosing the original input
-    ./solver/solver --backup="$1.bkp" --double_buffer="$1.bkp2" \
+    ./solver/solver${BENCH_SUFFIX} --backup="$1.bkp" --double_buffer="$1.bkp2" \
                     --executor="${EXECUTOR}" --solvers="${SOLVERS}" \
                     --preconditioners="${PRECONDS}" \
                     --max_iters=${SOLVERS_MAX_ITERATIONS} --rel_res_goal=${SOLVERS_PRECISION} \
@@ -377,7 +377,7 @@ EOT
 generate_problem() {
     [ "${DRY_RUN}" == "true" ] && return
     cp "$1" "$1.tmp"
-    ./matrix_generator/matrix_generator <"$1.tmp" 2>&1 >"$1"
+    ./matrix_generator/matrix_generator${BENCH_SUFFIX} <"$1.tmp" 2>&1 >"$1"
     keep_latest "$1" "$1.tmp"
 }
 

From 0751c5a41c30c01698ab70ee89dd3c945ed49048 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 26 Nov 2020 16:12:27 +0100
Subject: [PATCH 0253/1008] incomplete cholesky

---
 common/factorization/par_ic_kernels.hpp.inc   | 113 +++++
 .../par_ict_spgeam_kernels.hpp.inc            |  94 ++---
 .../par_ict_sweep_kernels.hpp.inc             |  28 +-
 core/CMakeLists.txt                           |   2 +
 core/device_hooks/common_kernels.inc.cpp      |  33 ++
 core/factorization/ic.cpp                     | 126 ++++++
 core/factorization/ic_kernels.hpp             | 114 +++++
 core/factorization/par_ic.cpp                 | 156 +++++++
 core/factorization/par_ic_kernels.hpp         | 121 ++++++
 core/factorization/par_ict.cpp                |  28 +-
 core/factorization/par_ict_kernels.hpp        |   2 +-
 core/test/factorization/CMakeLists.txt        |   2 +
 core/test/factorization/par_ic.cpp            | 125 ++++++
 core/test/factorization/par_ict.cpp           | 147 +++++++
 core/test/factorization/par_ilu.cpp           |  72 +++-
 core/test/factorization/par_ilut.cpp          |  38 +-
 core/test/preconditioner/CMakeLists.txt       |   1 +
 core/test/preconditioner/ic.cpp               |  99 +++++
 core/test/preconditioner/ilu.cpp              |  35 +-
 cuda/CMakeLists.txt                           |   2 +
 cuda/base/cusparse_bindings.hpp               | 112 +++++
 cuda/factorization/ic_kernels.cu              |  94 +++++
 cuda/factorization/par_ic_kernels.cu          | 141 +++++++
 cuda/factorization/par_ict_kernels.cu         |  20 +-
 cuda/test/factorization/CMakeLists.txt        |   2 +
 cuda/test/factorization/ic_kernels.cpp        | 147 +++++++
 cuda/test/factorization/par_ic_kernels.cpp    | 167 ++++++++
 cuda/test/factorization/par_ict_kernels.cpp   |  12 +-
 dev_tools/scripts/config                      |   2 +
 dpcpp/CMakeLists.txt                          |   2 +
 dpcpp/factorization/ic_kernels.dp.cpp         |  57 +++
 dpcpp/factorization/par_ic_kernels.dp.cpp     |  75 ++++
 dpcpp/factorization/par_ict_kernels.dp.cpp    |   2 +-
 hip/CMakeLists.txt                            |   2 +
 hip/base/hipsparse_bindings.hip.hpp           | 126 ++++++
 hip/factorization/ic_kernels.hip.cpp          |  97 +++++
 hip/factorization/par_ic_kernels.hip.cpp      | 144 +++++++
 hip/factorization/par_ict_kernels.hip.cpp     |  24 +-
 hip/test/factorization/CMakeLists.txt         |   2 +
 hip/test/factorization/ic_kernels.cpp         | 147 +++++++
 hip/test/factorization/par_ic_kernels.hip.cpp | 167 ++++++++
 .../factorization/par_ict_kernels.hip.cpp     |  12 +-
 include/ginkgo/core/factorization/ic.hpp      | 164 ++++++++
 include/ginkgo/core/factorization/par_ic.hpp  | 196 +++++++++
 include/ginkgo/core/factorization/par_ict.hpp |   2 +-
 include/ginkgo/core/factorization/par_ilu.hpp |  26 +-
 .../ginkgo/core/factorization/par_ilut.hpp    |  21 +-
 include/ginkgo/core/preconditioner/ic.hpp     | 395 ++++++++++++++++++
 include/ginkgo/ginkgo.hpp                     |   3 +
 omp/CMakeLists.txt                            |   2 +
 omp/factorization/ic_kernels.cpp              |  57 +++
 omp/factorization/par_ic_kernels.cpp          | 140 +++++++
 omp/factorization/par_ict_kernels.cpp         |  32 +-
 omp/test/factorization/CMakeLists.txt         |   1 +
 omp/test/factorization/par_ic_kernels.cpp     | 177 ++++++++
 omp/test/factorization/par_ict_kernels.cpp    |  12 +-
 reference/CMakeLists.txt                      |   2 +
 reference/factorization/ic_kernels.cpp        |  57 +++
 reference/factorization/par_ic_kernels.cpp    | 140 +++++++
 reference/factorization/par_ict_kernels.cpp   |  32 +-
 reference/test/factorization/CMakeLists.txt   |   1 +
 .../test/factorization/par_ic_kernels.cpp     | 278 ++++++++++++
 .../test/factorization/par_ict_kernels.cpp    |  22 +-
 reference/test/preconditioner/CMakeLists.txt  |   1 +
 reference/test/preconditioner/ic.cpp          | 297 +++++++++++++
 65 files changed, 4728 insertions(+), 222 deletions(-)
 create mode 100644 common/factorization/par_ic_kernels.hpp.inc
 create mode 100644 core/factorization/ic.cpp
 create mode 100644 core/factorization/ic_kernels.hpp
 create mode 100644 core/factorization/par_ic.cpp
 create mode 100644 core/factorization/par_ic_kernels.hpp
 create mode 100644 core/test/factorization/par_ic.cpp
 create mode 100644 core/test/factorization/par_ict.cpp
 create mode 100644 core/test/preconditioner/ic.cpp
 create mode 100644 cuda/factorization/ic_kernels.cu
 create mode 100644 cuda/factorization/par_ic_kernels.cu
 create mode 100644 cuda/test/factorization/ic_kernels.cpp
 create mode 100644 cuda/test/factorization/par_ic_kernels.cpp
 create mode 100644 dpcpp/factorization/ic_kernels.dp.cpp
 create mode 100644 dpcpp/factorization/par_ic_kernels.dp.cpp
 create mode 100644 hip/factorization/ic_kernels.hip.cpp
 create mode 100644 hip/factorization/par_ic_kernels.hip.cpp
 create mode 100644 hip/test/factorization/ic_kernels.cpp
 create mode 100644 hip/test/factorization/par_ic_kernels.hip.cpp
 create mode 100644 include/ginkgo/core/factorization/ic.hpp
 create mode 100644 include/ginkgo/core/factorization/par_ic.hpp
 create mode 100644 include/ginkgo/core/preconditioner/ic.hpp
 create mode 100644 omp/factorization/ic_kernels.cpp
 create mode 100644 omp/factorization/par_ic_kernels.cpp
 create mode 100644 omp/test/factorization/par_ic_kernels.cpp
 create mode 100644 reference/factorization/ic_kernels.cpp
 create mode 100644 reference/factorization/par_ic_kernels.cpp
 create mode 100644 reference/test/factorization/par_ic_kernels.cpp
 create mode 100644 reference/test/preconditioner/ic.cpp

diff --git a/common/factorization/par_ic_kernels.hpp.inc b/common/factorization/par_ic_kernels.hpp.inc
new file mode 100644
index 00000000000..4c3da30ae48
--- /dev/null
+++ b/common/factorization/par_ic_kernels.hpp.inc
@@ -0,0 +1,113 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+namespace kernel {
+
+
+template <typename ValueType, typename IndexType>
+__global__ __launch_bounds__(default_block_size) void ic_init(
+    const IndexType *__restrict__ l_row_ptrs, ValueType *__restrict__ l_vals,
+    size_type num_rows)
+{
+    auto row = thread::get_thread_id_flat();
+    if (row >= num_rows) {
+        return;
+    }
+    auto l_nz = l_row_ptrs[row + 1] - 1;
+    auto diag = sqrt(l_vals[l_nz]);
+    if (is_finite(diag)) {
+        l_vals[l_nz] = diag;
+    } else {
+        l_vals[l_nz] = one<ValueType>();
+    }
+}
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+__global__ __launch_bounds__(default_block_size) void ic_sweep(
+    const IndexType *__restrict__ a_row_idxs,
+    const IndexType *__restrict__ a_col_idxs,
+    const ValueType *__restrict__ a_vals,
+    const IndexType *__restrict__ l_row_ptrs,
+    const IndexType *__restrict__ l_col_idxs, ValueType *__restrict__ l_vals,
+    IndexType l_nnz)
+{
+    auto l_nz = thread::get_subwarp_id_flat<subwarp_size, IndexType>();
+    if (l_nz >= l_nnz) {
+        return;
+    }
+    auto row = a_row_idxs[l_nz];
+    auto col = l_col_idxs[l_nz];
+    auto subwarp =
+        group::tiled_partition<subwarp_size>(group::this_thread_block());
+    auto a_val = a_vals[l_nz];
+    auto l_row_begin = l_row_ptrs[row];
+    auto l_row_size = l_row_ptrs[row + 1] - l_row_begin;
+    auto lh_col_begin = l_row_ptrs[col];
+    auto lh_col_size = l_row_ptrs[col + 1] - lh_col_begin;
+    ValueType sum{};
+    IndexType lh_nz{};
+    auto last_entry = col;
+    group_merge<subwarp_size>(
+        l_col_idxs + l_row_begin, l_row_size, l_col_idxs + lh_col_begin,
+        lh_col_size, subwarp,
+        [&](IndexType l_idx, IndexType l_col, IndexType lh_idx,
+            IndexType lh_row, IndexType, bool) {
+            // we don't need to use the `bool valid` because last_entry is
+            // already a smaller sentinel value than the one used in group_merge
+            if (l_col == lh_row && l_col < last_entry) {
+                sum += l_vals[l_idx + l_row_begin] *
+                       conj(l_vals[lh_idx + lh_col_begin]);
+            }
+            // remember the transposed element
+            auto found_transp = subwarp.ballot(lh_row == row);
+            if (found_transp) {
+                lh_nz =
+                    subwarp.shfl(lh_idx + lh_col_begin, ffs(found_transp) - 1);
+            }
+            return true;
+        });
+    // accumulate result from all threads
+    sum = reduce(subwarp, sum, [](ValueType a, ValueType b) { return a + b; });
+
+    if (subwarp.thread_rank() == 0) {
+        auto to_write = row == col
+                            ? sqrt(a_val - sum)
+                            : (a_val - sum) / l_vals[l_row_ptrs[col + 1] - 1];
+        if (is_finite(to_write)) {
+            l_vals[l_nz] = to_write;
+        }
+    }
+}
+
+
+}  // namespace kernel
diff --git a/common/factorization/par_ict_spgeam_kernels.hpp.inc b/common/factorization/par_ict_spgeam_kernels.hpp.inc
index 6f76a14323c..5e1dea39544 100644
--- a/common/factorization/par_ict_spgeam_kernels.hpp.inc
+++ b/common/factorization/par_ict_spgeam_kernels.hpp.inc
@@ -35,8 +35,8 @@ namespace kernel {
 
 template <int subwarp_size, typename IndexType>
 __global__ __launch_bounds__(default_block_size) void ict_tri_spgeam_nnz(
-    const IndexType *__restrict__ llt_row_ptrs,
-    const IndexType *__restrict__ llt_col_idxs,
+    const IndexType *__restrict__ llh_row_ptrs,
+    const IndexType *__restrict__ llh_col_idxs,
     const IndexType *__restrict__ a_row_ptrs,
     const IndexType *__restrict__ a_col_idxs,
     IndexType *__restrict__ l_new_row_ptrs, IndexType num_rows)
@@ -48,20 +48,20 @@ __global__ __launch_bounds__(default_block_size) void ict_tri_spgeam_nnz(
         return;
     }
 
-    auto llt_begin = llt_row_ptrs[row];
-    auto llt_size = llt_row_ptrs[row + 1] - llt_begin;
+    auto llh_begin = llh_row_ptrs[row];
+    auto llh_size = llh_row_ptrs[row + 1] - llh_begin;
     auto a_begin = a_row_ptrs[row];
     auto a_size = a_row_ptrs[row + 1] - a_begin;
     IndexType count{};
     group_merge<subwarp_size>(
-        a_col_idxs + a_begin, a_size, llt_col_idxs + llt_begin, llt_size,
+        a_col_idxs + a_begin, a_size, llh_col_idxs + llh_begin, llh_size,
         subwarp,
-        [&](IndexType a_nz, IndexType a_col, IndexType llt_nz,
-            IndexType llt_col, IndexType out_nz, bool valid) {
-            auto col = min(a_col, llt_col);
+        [&](IndexType a_nz, IndexType a_col, IndexType llh_nz,
+            IndexType llh_col, IndexType out_nz, bool valid) {
+            auto col = min(a_col, llh_col);
             // count the number of unique elements being merged
             count +=
-                popcnt(subwarp.ballot(col <= row && a_col != llt_col && valid));
+                popcnt(subwarp.ballot(col <= row && a_col != llh_col && valid));
             return true;
         });
     if (subwarp.thread_rank() == 0) {
@@ -72,9 +72,9 @@ __global__ __launch_bounds__(default_block_size) void ict_tri_spgeam_nnz(
 
 template <int subwarp_size, typename ValueType, typename IndexType>
 __global__ __launch_bounds__(default_block_size) void ict_tri_spgeam_init(
-    const IndexType *__restrict__ llt_row_ptrs,
-    const IndexType *__restrict__ llt_col_idxs,
-    const ValueType *__restrict__ llt_vals,
+    const IndexType *__restrict__ llh_row_ptrs,
+    const IndexType *__restrict__ llh_col_idxs,
+    const ValueType *__restrict__ llh_vals,
     const IndexType *__restrict__ a_row_ptrs,
     const IndexType *__restrict__ a_col_idxs,
     const ValueType *__restrict__ a_vals,
@@ -100,16 +100,16 @@ __global__ __launch_bounds__(default_block_size) void ict_tri_spgeam_init(
     auto l_begin = l_row_ptrs[row];
     auto l_end = l_row_ptrs[row + 1];
 
-    auto llt_begin = llt_row_ptrs[row];
-    auto llt_end = llt_row_ptrs[row + 1];
-    auto llt_size = llt_end - llt_begin;
+    auto llh_begin = llh_row_ptrs[row];
+    auto llh_end = llh_row_ptrs[row + 1];
+    auto llh_size = llh_end - llh_begin;
 
     auto a_begin = a_row_ptrs[row];
     auto a_end = a_row_ptrs[row + 1];
     auto a_size = a_end - a_begin;
 
     IndexType out_begin{};
-    auto out_size = llt_size + a_size;
+    auto out_size = llh_size + a_size;
 
     IndexType l_new_begin = l_new_row_ptrs[row];
 
@@ -117,44 +117,44 @@ __global__ __launch_bounds__(default_block_size) void ict_tri_spgeam_init(
     // load column indices and values for the first merge step
     auto a_col = checked_load(a_col_idxs, a_begin + lane, a_end, sentinel);
     auto a_val = checked_load(a_vals, a_begin + lane, a_end, zero<ValueType>());
-    auto llt_col =
-        checked_load(llt_col_idxs, llt_begin + lane, llt_end, sentinel);
-    auto llt_val =
-        checked_load(llt_vals, llt_begin + lane, llt_end, zero<ValueType>());
+    auto llh_col =
+        checked_load(llh_col_idxs, llh_begin + lane, llh_end, sentinel);
+    auto llh_val =
+        checked_load(llh_vals, llh_begin + lane, llh_end, zero<ValueType>());
     auto l_col = checked_load(l_col_idxs, l_begin + lane, l_end, sentinel);
     auto l_val = checked_load(l_vals, l_begin + lane, l_end, zero<ValueType>());
     bool skip_first{};
     while (out_begin < out_size) {
-        // merge subwarp.size() elements from A and L*L^T
+        // merge subwarp.size() elements from A and L*L^H
         auto merge_result =
-            group_merge_step<subwarp_size>(a_col, llt_col, subwarp);
+            group_merge_step<subwarp_size>(a_col, llh_col, subwarp);
         auto a_cur_col = merge_result.a_val;
-        auto llt_cur_col = merge_result.b_val;
+        auto llh_cur_col = merge_result.b_val;
         auto a_cur_val = subwarp.shfl(a_val, merge_result.a_idx);
-        auto llt_cur_val = subwarp.shfl(llt_val, merge_result.b_idx);
+        auto llh_cur_val = subwarp.shfl(llh_val, merge_result.b_idx);
         auto valid = out_begin + lane < out_size;
         // check if the previous thread has matching columns
-        auto equal_mask = subwarp.ballot(a_cur_col == llt_cur_col && valid);
+        auto equal_mask = subwarp.ballot(a_cur_col == llh_cur_col && valid);
         auto prev_equal_mask = equal_mask << 1 | skip_first;
         skip_first = bool(equal_mask >> (subwarp_size - 1));
         auto prev_equal = bool(prev_equal_mask & lanemask_eq);
 
-        auto r_col = min(a_cur_col, llt_cur_col);
+        auto r_col = min(a_cur_col, llh_cur_col);
         // find matching entry of L
-        // S(L) is a subset of S(A - L * L^T) since L has a diagonal
+        // S(L) is a subset of S(A - L * L^H) since L has a diagonal
         auto l_source = synchronous_fixed_binary_search<subwarp_size>(
             [&](int i) { return subwarp.shfl(l_col, i) >= r_col; });
         auto l_cur_col = subwarp.shfl(l_col, l_source);
         auto l_cur_val = subwarp.shfl(l_val, l_source);
 
-        // determine actual values of A and L*L^T at r_col
+        // determine actual values of A and L*L^H at r_col
         if (r_col != a_cur_col) {
             a_cur_val = zero<ValueType>();
         }
-        if (r_col != llt_cur_col) {
-            llt_cur_val = zero<ValueType>();
+        if (r_col != llh_cur_col) {
+            llh_cur_val = zero<ValueType>();
         }
-        auto r_val = a_cur_val - llt_cur_val;
+        auto r_val = a_cur_val - llh_cur_val;
 
         // early return when reaching the upper diagonal
         if (subwarp.all(r_col > row)) {
@@ -176,11 +176,11 @@ __global__ __launch_bounds__(default_block_size) void ict_tri_spgeam_init(
 
         // advance *_begin offsets
         auto a_advance = merge_result.a_advance;
-        auto llt_advance = merge_result.b_advance;
+        auto llh_advance = merge_result.b_advance;
         auto l_advance = popcnt(subwarp.ballot(do_write && use_l));
         auto l_new_advance = popcnt(l_new_advance_mask);
         a_begin += a_advance;
-        llt_begin += llt_advance;
+        llh_begin += llh_advance;
         l_begin += l_advance;
         l_new_begin += l_new_advance;
         out_begin += subwarp_size;
@@ -188,25 +188,25 @@ __global__ __launch_bounds__(default_block_size) void ict_tri_spgeam_init(
         // shuffle the unmerged elements to the front
         a_col = subwarp.shfl_down(a_col, a_advance);
         a_val = subwarp.shfl_down(a_val, a_advance);
-        llt_col = subwarp.shfl_down(llt_col, llt_advance);
-        llt_val = subwarp.shfl_down(llt_val, llt_advance);
+        llh_col = subwarp.shfl_down(llh_col, llh_advance);
+        llh_val = subwarp.shfl_down(llh_val, llh_advance);
         l_col = subwarp.shfl_down(l_col, l_advance);
         l_val = subwarp.shfl_down(l_val, l_advance);
         /*
-         * To optimize memory access, we load the new elements for `a` and `llt`
+         * To optimize memory access, we load the new elements for `a` and `llh`
          * with a single load instruction:
          * the lower part of the group loads new elements for `a`
-         * the upper part of the group loads new elements for `llt`
+         * the upper part of the group loads new elements for `llh`
          * `load_lane` is the part-local lane idx
          * The elements for `a` have to be shuffled up afterwards.
          */
         auto load_a = lane < a_advance;
         auto load_lane = load_a ? lane : lane - a_advance;
-        auto load_source_col = load_a ? a_col_idxs : llt_col_idxs;
-        auto load_source_val = load_a ? a_vals : llt_vals;
+        auto load_source_col = load_a ? a_col_idxs : llh_col_idxs;
+        auto load_source_val = load_a ? a_vals : llh_vals;
         auto load_begin =
-            load_a ? a_begin + llt_advance : llt_begin + a_advance;
-        auto load_end = load_a ? a_end : llt_end;
+            load_a ? a_begin + llh_advance : llh_begin + a_advance;
+        auto load_end = load_a ? a_end : llh_end;
 
         auto load_idx = load_begin + load_lane;
         auto loaded_col =
@@ -214,15 +214,15 @@ __global__ __launch_bounds__(default_block_size) void ict_tri_spgeam_init(
         auto loaded_val = checked_load(load_source_val, load_idx, load_end,
                                        zero<ValueType>());
         // shuffle the `a` values to the end of the warp
-        auto lower_loaded_col = subwarp.shfl_up(loaded_col, llt_advance);
-        auto lower_loaded_val = subwarp.shfl_up(loaded_val, llt_advance);
-        if (lane >= llt_advance) {
+        auto lower_loaded_col = subwarp.shfl_up(loaded_col, llh_advance);
+        auto lower_loaded_val = subwarp.shfl_up(loaded_val, llh_advance);
+        if (lane >= llh_advance) {
             a_col = lower_loaded_col;
             a_val = lower_loaded_val;
         }
         if (lane >= a_advance) {
-            llt_col = loaded_col;
-            llt_val = loaded_val;
+            llh_col = loaded_col;
+            llh_val = loaded_val;
         }
         // load the new values for l
         if (lane >= subwarp_size - l_advance) {
@@ -234,4 +234,4 @@ __global__ __launch_bounds__(default_block_size) void ict_tri_spgeam_init(
 }
 
 
-}  // namespace kernel
\ No newline at end of file
+}  // namespace kernel
diff --git a/common/factorization/par_ict_sweep_kernels.hpp.inc b/common/factorization/par_ict_sweep_kernels.hpp.inc
index 325ea25061a..418efef9c5c 100644
--- a/common/factorization/par_ict_sweep_kernels.hpp.inc
+++ b/common/factorization/par_ict_sweep_kernels.hpp.inc
@@ -62,27 +62,27 @@ __global__ __launch_bounds__(default_block_size) void ict_sweep(
     auto a_val = has_a ? a_vals[a_idx] : zero<ValueType>();
     auto l_row_begin = l_row_ptrs[row];
     auto l_row_size = l_row_ptrs[row + 1] - l_row_begin;
-    auto lt_col_begin = l_row_ptrs[col];
-    auto lt_col_size = l_row_ptrs[col + 1] - lt_col_begin;
+    auto lh_col_begin = l_row_ptrs[col];
+    auto lh_col_size = l_row_ptrs[col + 1] - lh_col_begin;
     ValueType sum{};
-    IndexType lt_nz{};
+    IndexType lh_nz{};
     auto last_entry = col;
     group_merge<subwarp_size>(
-        l_col_idxs + l_row_begin, l_row_size, l_col_idxs + lt_col_begin,
-        lt_col_size, subwarp,
-        [&](IndexType l_idx, IndexType l_col, IndexType lt_idx,
-            IndexType lt_row, IndexType, bool) {
+        l_col_idxs + l_row_begin, l_row_size, l_col_idxs + lh_col_begin,
+        lh_col_size, subwarp,
+        [&](IndexType l_idx, IndexType l_col, IndexType lh_idx,
+            IndexType lh_row, IndexType, bool) {
             // we don't need to use the `bool valid` because last_entry is
             // already a smaller sentinel value than the one used in group_merge
-            if (l_col == lt_row && l_col < last_entry) {
-                sum +=
-                    l_vals[l_idx + l_row_begin] * l_vals[lt_idx + lt_col_begin];
+            if (l_col == lh_row && l_col < last_entry) {
+                sum += l_vals[l_idx + l_row_begin] *
+                       conj(l_vals[lh_idx + lh_col_begin]);
             }
             // remember the transposed element
-            auto found_transp = subwarp.ballot(lt_row == row);
+            auto found_transp = subwarp.ballot(lh_row == row);
             if (found_transp) {
-                lt_nz =
-                    subwarp.shfl(lt_idx + lt_col_begin, ffs(found_transp) - 1);
+                lh_nz =
+                    subwarp.shfl(lh_idx + lh_col_begin, ffs(found_transp) - 1);
             }
             return true;
         });
@@ -100,4 +100,4 @@ __global__ __launch_bounds__(default_block_size) void ict_sweep(
 }
 
 
-}  // namespace kernel
\ No newline at end of file
+}  // namespace kernel
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 399a779ace3..a63e108ad57 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -11,7 +11,9 @@ target_sources(ginkgo
     base/mtx_io.cpp
     base/perturbation.cpp
     base/version.cpp
+    factorization/ic.cpp
     factorization/ilu.cpp
+    factorization/par_ic.cpp
     factorization/par_ict.cpp
     factorization/par_ilu.cpp
     factorization/par_ilut.cpp
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index cf46b6e4cae..6fd03ce9877 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -38,7 +38,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/precision_conversion.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/factorization/factorization_kernels.hpp"
+#include "core/factorization/ic_kernels.hpp"
 #include "core/factorization/ilu_kernels.hpp"
+#include "core/factorization/par_ic_kernels.hpp"
 #include "core/factorization/par_ict_kernels.hpp"
 #include "core/factorization/par_ilu_kernels.hpp"
 #include "core/factorization/par_ilut_kernels.hpp"
@@ -1037,6 +1039,18 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 }  // namespace factorization
 
 
+namespace ic_factorization {
+
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_IC_COMPUTE_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_IC_COMPUTE_KERNEL);
+
+
+}  // namespace ic_factorization
+
+
 namespace ilu_factorization {
 
 
@@ -1050,6 +1064,25 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 }  // namespace ilu_factorization
 
 
+namespace par_ic_factorization {
+
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_PAR_IC_INIT_FACTOR_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_IC_INIT_FACTOR_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_PAR_IC_COMPUTE_FACTOR_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_IC_COMPUTE_FACTOR_KERNEL);
+
+
+}  // namespace par_ic_factorization
+
+
 namespace par_ict_factorization {
 
 
diff --git a/core/factorization/ic.cpp b/core/factorization/ic.cpp
new file mode 100644
index 00000000000..e89754d29e7
--- /dev/null
+++ b/core/factorization/ic.cpp
@@ -0,0 +1,126 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/factorization/ic.hpp>
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
+#include "core/factorization/factorization_kernels.hpp"
+#include "core/factorization/ic_kernels.hpp"
+
+
+namespace gko {
+namespace factorization {
+namespace ic_factorization {
+
+
+GKO_REGISTER_OPERATION(compute, ic_factorization::compute);
+GKO_REGISTER_OPERATION(add_diagonal_elements,
+                       factorization::add_diagonal_elements);
+GKO_REGISTER_OPERATION(initialize_row_ptrs_l,
+                       factorization::initialize_row_ptrs_l);
+GKO_REGISTER_OPERATION(initialize_l, factorization::initialize_l);
+
+
+}  // namespace ic_factorization
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<Composition<ValueType>> Ic<ValueType, IndexType>::generate(
+    const std::shared_ptr<const LinOp> &system_matrix, bool skip_sorting,
+    bool both_factors) const
+{
+    GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix);
+
+    const auto exec = this->get_executor();
+
+    // Converts the system matrix to CSR.
+    // Throws an exception if it is not convertible.
+    auto local_system_matrix = matrix_type::create(exec);
+    as<ConvertibleTo<matrix_type>>(system_matrix.get())
+        ->convert_to(local_system_matrix.get());
+
+    if (!skip_sorting) {
+        local_system_matrix->sort_by_column_index();
+    }
+
+    // Add explicit diagonal zero elements if they are missing
+    exec->run(ic_factorization::make_add_diagonal_elements(
+        local_system_matrix.get(), false));
+
+    // Compute LC factorization
+    exec->run(ic_factorization::make_compute(local_system_matrix.get()));
+
+    // Extract lower factor: compute non-zeros
+    const auto matrix_size = local_system_matrix->get_size();
+    const auto num_rows = matrix_size[0];
+    Array<IndexType> l_row_ptrs{exec, num_rows + 1};
+    exec->run(ic_factorization::make_initialize_row_ptrs_l(
+        local_system_matrix.get(), l_row_ptrs.get_data()));
+
+    // Get nnz from device memory
+    auto l_nnz = static_cast<size_type>(
+        exec->copy_val_to_host(l_row_ptrs.get_data() + num_rows));
+
+    // Init arrays
+    Array<IndexType> l_col_idxs{exec, l_nnz};
+    Array<ValueType> l_vals{exec, l_nnz};
+    std::shared_ptr<matrix_type> l_factor = matrix_type::create(
+        exec, matrix_size, std::move(l_vals), std::move(l_col_idxs),
+        std::move(l_row_ptrs), parameters_.l_strategy);
+
+    // Extract lower factor: columns and values
+    exec->run(ic_factorization::make_initialize_l(local_system_matrix.get(),
+                                                  l_factor.get(), false));
+
+    if (both_factors) {
+        auto lh_factor = l_factor->conj_transpose();
+        return Composition<ValueType>::create(std::move(l_factor),
+                                              std::move(lh_factor));
+    } else {
+        return Composition<ValueType>::create(std::move(l_factor));
+    }
+}
+
+
+#define GKO_DECLARE_IC(ValueType, IndexType) class Ic<ValueType, IndexType>
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_IC);
+
+
+}  // namespace factorization
+}  // namespace gko
diff --git a/core/factorization/ic_kernels.hpp b/core/factorization/ic_kernels.hpp
new file mode 100644
index 00000000000..2160a444a4c
--- /dev/null
+++ b/core/factorization/ic_kernels.hpp
@@ -0,0 +1,114 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_FACTORIZATION_IC_KERNELS_HPP_
+#define GKO_CORE_FACTORIZATION_IC_KERNELS_HPP_
+
+
+#include <ginkgo/core/factorization/ic.hpp>
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+namespace gko {
+namespace kernels {
+
+
+#define GKO_DECLARE_IC_COMPUTE_KERNEL(ValueType, IndexType)   \
+    void compute(std::shared_ptr<const DefaultExecutor> exec, \
+                 matrix::Csr<ValueType, IndexType> *system_matrix)
+
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES                  \
+    template <typename ValueType, typename IndexType> \
+    GKO_DECLARE_IC_COMPUTE_KERNEL(ValueType, IndexType)
+
+
+namespace omp {
+namespace ic_factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace ic_factorization
+}  // namespace omp
+
+
+namespace cuda {
+namespace ic_factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace ic_factorization
+}  // namespace cuda
+
+
+namespace reference {
+namespace ic_factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace ic_factorization
+}  // namespace reference
+
+
+namespace hip {
+namespace ic_factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace ic_factorization
+}  // namespace hip
+
+
+namespace dpcpp {
+namespace ic_factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace ic_factorization
+}  // namespace dpcpp
+
+
+#undef GKO_DECLARE_ALL_AS_TEMPLATES
+
+
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_CORE_FACTORIZATION_IC_KERNELS_HPP_
diff --git a/core/factorization/par_ic.cpp b/core/factorization/par_ic.cpp
new file mode 100644
index 00000000000..ddbe1424f6e
--- /dev/null
+++ b/core/factorization/par_ic.cpp
@@ -0,0 +1,156 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/factorization/par_ic.hpp>
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/polymorphic_object.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+#include "core/factorization/factorization_kernels.hpp"
+#include "core/factorization/par_ic_kernels.hpp"
+#include "core/matrix/csr_kernels.hpp"
+
+
+namespace gko {
+namespace factorization {
+namespace par_ic_factorization {
+
+
+GKO_REGISTER_OPERATION(add_diagonal_elements,
+                       factorization::add_diagonal_elements);
+GKO_REGISTER_OPERATION(initialize_row_ptrs_l,
+                       factorization::initialize_row_ptrs_l);
+GKO_REGISTER_OPERATION(initialize_l, factorization::initialize_l);
+GKO_REGISTER_OPERATION(init_factor, par_ic_factorization::init_factor);
+GKO_REGISTER_OPERATION(compute_factor, par_ic_factorization::compute_factor);
+GKO_REGISTER_OPERATION(csr_transpose, csr::transpose);
+GKO_REGISTER_OPERATION(convert_to_coo, csr::convert_to_coo);
+
+
+}  // namespace par_ic_factorization
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<Composition<ValueType>> ParIc<ValueType, IndexType>::generate(
+    const std::shared_ptr<const LinOp> &system_matrix, bool skip_sorting,
+    bool both_factors) const
+{
+    using CsrMatrix = matrix::Csr<ValueType, IndexType>;
+    using CooMatrix = matrix::Coo<ValueType, IndexType>;
+
+    GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix);
+
+    const auto exec = this->get_executor();
+
+    // Converts the system matrix to CSR.
+    // Throws an exception if it is not convertible.
+    auto csr_system_matrix = CsrMatrix::create(exec);
+    as<ConvertibleTo<CsrMatrix>>(system_matrix.get())
+        ->convert_to(csr_system_matrix.get());
+    // If necessary, sort it
+    if (!skip_sorting) {
+        csr_system_matrix->sort_by_column_index();
+    }
+
+    // Add explicit diagonal zero elements if they are missing
+    exec->run(par_ic_factorization::make_add_diagonal_elements(
+        csr_system_matrix.get(), true));
+
+    const auto matrix_size = csr_system_matrix->get_size();
+    const auto number_rows = matrix_size[0];
+    Array<IndexType> l_row_ptrs{exec, number_rows + 1};
+    exec->run(par_ic_factorization::make_initialize_row_ptrs_l(
+        csr_system_matrix.get(), l_row_ptrs.get_data()));
+
+    // Get nnz from device memory
+    auto l_nnz = static_cast<size_type>(
+        exec->copy_val_to_host(l_row_ptrs.get_data() + number_rows));
+
+    // Since `row_ptrs` of L is already created, the matrix can be
+    // directly created with it
+    Array<IndexType> l_col_idxs{exec, l_nnz};
+    Array<ValueType> l_vals{exec, l_nnz};
+    std::shared_ptr<CsrMatrix> l_factor = matrix_type::create(
+        exec, matrix_size, std::move(l_vals), std::move(l_col_idxs),
+        std::move(l_row_ptrs), parameters_.l_strategy);
+
+    exec->run(par_ic_factorization::make_initialize_l(csr_system_matrix.get(),
+                                                      l_factor.get(), false));
+
+    // build COO representation of lower factor
+    Array<IndexType> l_row_idxs{exec, l_nnz};
+    // copy values from l_factor, which are the lower triangular values of A
+    auto l_vals_view =
+        Array<ValueType>::view(exec, l_nnz, l_factor->get_values());
+    auto a_vals = Array<ValueType>{exec, l_vals_view};
+    auto a_row_idxs =
+        Array<IndexType>::view(exec, l_nnz, l_factor->get_col_idxs());
+    auto a_col_idxs = Array<IndexType>{exec, l_nnz};
+    auto a_lower_coo =
+        CooMatrix::create(exec, matrix_size, std::move(a_vals),
+                          std::move(a_row_idxs), std::move(a_col_idxs));
+    exec->run(par_ic_factorization::make_convert_to_coo(l_factor.get(),
+                                                        a_lower_coo.get()));
+
+    // compute sqrt of diagonal entries
+    exec->run(par_ic_factorization::make_init_factor(l_factor.get()));
+
+    // execute sweeps
+    exec->run(par_ic_factorization::make_compute_factor(
+        parameters_.iterations, a_lower_coo.get(), l_factor.get()));
+
+    if (both_factors) {
+        auto lh_factor = l_factor->conj_transpose();
+        return Composition<ValueType>::create(std::move(l_factor),
+                                              std::move(lh_factor));
+    } else {
+        return Composition<ValueType>::create(std::move(l_factor));
+    }
+}
+
+
+#define GKO_DECLARE_PAR_IC(ValueType, IndexType) \
+    class ParIc<ValueType, IndexType>
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_PAR_IC);
+
+
+}  // namespace factorization
+}  // namespace gko
diff --git a/core/factorization/par_ic_kernels.hpp b/core/factorization/par_ic_kernels.hpp
new file mode 100644
index 00000000000..5a64861f853
--- /dev/null
+++ b/core/factorization/par_ic_kernels.hpp
@@ -0,0 +1,121 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_FACTORIZATION_PAR_IC_KERNELS_HPP_
+#define GKO_CORE_FACTORIZATION_PAR_IC_KERNELS_HPP_
+
+
+#include <ginkgo/core/factorization/par_ic.hpp>
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+namespace gko {
+namespace kernels {
+
+#define GKO_DECLARE_PAR_IC_INIT_FACTOR_KERNEL(ValueType, IndexType) \
+    void init_factor(std::shared_ptr<const DefaultExecutor> exec,   \
+                     matrix::Csr<ValueType, IndexType> *l_factor)
+
+#define GKO_DECLARE_PAR_IC_COMPUTE_FACTOR_KERNEL(ValueType, IndexType)     \
+    void compute_factor(                                                   \
+        std::shared_ptr<const DefaultExecutor> exec, size_type iterations, \
+        const matrix::Coo<ValueType, IndexType> *lower_system_matrix,      \
+        matrix::Csr<ValueType, IndexType> *l_factor)
+
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES                             \
+    template <typename ValueType, typename IndexType>            \
+    GKO_DECLARE_PAR_IC_INIT_FACTOR_KERNEL(ValueType, IndexType); \
+    template <typename ValueType, typename IndexType>            \
+    GKO_DECLARE_PAR_IC_COMPUTE_FACTOR_KERNEL(ValueType, IndexType)
+
+
+namespace omp {
+namespace par_ic_factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace par_ic_factorization
+}  // namespace omp
+
+
+namespace cuda {
+namespace par_ic_factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace par_ic_factorization
+}  // namespace cuda
+
+
+namespace reference {
+namespace par_ic_factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace par_ic_factorization
+}  // namespace reference
+
+
+namespace hip {
+namespace par_ic_factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace par_ic_factorization
+}  // namespace hip
+
+
+namespace dpcpp {
+namespace par_ic_factorization {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace par_ic_factorization
+}  // namespace dpcpp
+
+
+#undef GKO_DECLARE_ALL_AS_TEMPLATES
+
+
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_CORE_FACTORIZATION_PAR_IC_KERNELS_HPP_
diff --git a/core/factorization/par_ict.cpp b/core/factorization/par_ict.cpp
index 905c350468b..69025ecd305 100644
--- a/core/factorization/par_ict.cpp
+++ b/core/factorization/par_ict.cpp
@@ -110,9 +110,9 @@ struct ParIctState {
     // current lower factor L
     std::unique_ptr<CsrMatrix> l;
     // current upper factor L^H
-    std::unique_ptr<CsrMatrix> lt;
+    std::unique_ptr<CsrMatrix> lh;
     // current product L * L^H
-    std::unique_ptr<CsrMatrix> llt;
+    std::unique_ptr<CsrMatrix> llh;
     // temporary lower factor L' before filtering
     std::unique_ptr<CsrMatrix> l_new;
     // lower factor L currently being updated with asynchronous iterations
@@ -124,14 +124,14 @@ struct ParIctState {
     // strategy to be used by the lower factor
     std::shared_ptr<typename CsrMatrix::strategy_type> l_strategy;
     // strategy to be used by the upper factor
-    std::shared_ptr<typename CsrMatrix::strategy_type> lt_strategy;
+    std::shared_ptr<typename CsrMatrix::strategy_type> lh_strategy;
 
     ParIctState(std::shared_ptr<const Executor> exec_in,
                 const CsrMatrix *system_matrix_in,
                 std::unique_ptr<CsrMatrix> l_in, IndexType l_nnz_limit,
                 bool use_approx_select,
                 std::shared_ptr<typename CsrMatrix::strategy_type> l_strategy_,
-                std::shared_ptr<typename CsrMatrix::strategy_type> lt_strategy_)
+                std::shared_ptr<typename CsrMatrix::strategy_type> lh_strategy_)
         : exec{std::move(exec_in)},
           l_nnz_limit{l_nnz_limit},
           use_approx_select{use_approx_select},
@@ -140,22 +140,22 @@ struct ParIctState {
           selection_tmp{exec},
           selection_tmp2{exec},
           l_strategy{std::move(l_strategy_)},
-          lt_strategy{std::move(lt_strategy_)}
+          lh_strategy{std::move(lh_strategy_)}
     {
         auto mtx_size = system_matrix->get_size();
         auto l_nnz = l->get_num_stored_elements();
-        lt = CsrMatrix::create(exec, mtx_size, l_nnz);
-        llt = CsrMatrix::create(exec, mtx_size);
+        lh = CsrMatrix::create(exec, mtx_size, l_nnz);
+        llh = CsrMatrix::create(exec, mtx_size);
         l_new = CsrMatrix::create(exec, mtx_size);
         l_coo = CooMatrix::create(exec, mtx_size);
-        exec->run(make_csr_conj_transpose(l.get(), lt.get()));
+        exec->run(make_csr_conj_transpose(l.get(), lh.get()));
     }
 
     std::unique_ptr<Composition<ValueType>> to_factors() &&
     {
         l->set_strategy(l_strategy);
-        lt->set_strategy(lt_strategy);
-        return Composition<ValueType>::create(std::move(l), std::move(lt));
+        lh->set_strategy(lh_strategy);
+        return Composition<ValueType>::create(std::move(l), std::move(lh));
     }
 
     void iterate();
@@ -220,11 +220,11 @@ template <typename ValueType, typename IndexType>
 void ParIctState<ValueType, IndexType>::iterate()
 {
     // compute L * L^H
-    exec->run(make_spgemm(l.get(), lt.get(), llt.get()));
+    exec->run(make_spgemm(l.get(), lh.get(), llh.get()));
 
     // add new candidates to L' factor
     exec->run(
-        make_add_candidates(llt.get(), system_matrix, l.get(), l_new.get()));
+        make_add_candidates(llh.get(), system_matrix, l.get(), l_new.get()));
 
     // update L(COO), L'^H sizes and pointers
     {
@@ -273,11 +273,11 @@ void ParIctState<ValueType, IndexType>::iterate()
     // convert L to L^H
     {
         auto l_nnz = l->get_num_stored_elements();
-        CsrBuilder lt_builder{lt.get()};
+        CsrBuilder lt_builder{lh.get()};
         lt_builder.get_col_idx_array().resize_and_reset(l_nnz);
         lt_builder.get_value_array().resize_and_reset(l_nnz);
     }
-    exec->run(make_csr_conj_transpose(l.get(), lt.get()));
+    exec->run(make_csr_conj_transpose(l.get(), lh.get()));
 }
 
 
diff --git a/core/factorization/par_ict_kernels.hpp b/core/factorization/par_ict_kernels.hpp
index b7bf4554238..04d0af80e0c 100644
--- a/core/factorization/par_ict_kernels.hpp
+++ b/core/factorization/par_ict_kernels.hpp
@@ -52,7 +52,7 @@ namespace kernels {
 
 #define GKO_DECLARE_PAR_ICT_ADD_CANDIDATES_KERNEL(ValueType, IndexType) \
     void add_candidates(std::shared_ptr<const DefaultExecutor> exec,    \
-                        const matrix::Csr<ValueType, IndexType> *llt,   \
+                        const matrix::Csr<ValueType, IndexType> *llh,   \
                         const matrix::Csr<ValueType, IndexType> *a,     \
                         const matrix::Csr<ValueType, IndexType> *l,     \
                         matrix::Csr<ValueType, IndexType> *l_new)
diff --git a/core/test/factorization/CMakeLists.txt b/core/test/factorization/CMakeLists.txt
index 9b2e3082e51..8d9529eec9d 100644
--- a/core/test/factorization/CMakeLists.txt
+++ b/core/test/factorization/CMakeLists.txt
@@ -1,2 +1,4 @@
+ginkgo_create_test(par_ic)
+ginkgo_create_test(par_ict)
 ginkgo_create_test(par_ilu)
 ginkgo_create_test(par_ilut)
diff --git a/core/test/factorization/par_ic.cpp b/core/test/factorization/par_ic.cpp
new file mode 100644
index 00000000000..b99f60cc77d
--- /dev/null
+++ b/core/test/factorization/par_ic.cpp
@@ -0,0 +1,125 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/factorization/par_ic.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+namespace {
+
+
+class ParIc : public ::testing::Test {
+public:
+    using value_type = double;
+    using index_type = gko::int32;
+    using ic_factory_type = gko::factorization::ParIc<value_type, index_type>;
+    using strategy_type = ic_factory_type::matrix_type::classical;
+
+protected:
+    ParIc() : ref(gko::ReferenceExecutor::create()) {}
+
+    std::shared_ptr<const gko::ReferenceExecutor> ref;
+};
+
+
+TEST_F(ParIc, SetIterations)
+{
+    auto factory = ic_factory_type::build().with_iterations(5u).on(this->ref);
+
+    ASSERT_EQ(factory->get_parameters().iterations, 5u);
+}
+
+
+TEST_F(ParIc, SetSkip)
+{
+    auto factory =
+        ic_factory_type::build().with_skip_sorting(true).on(this->ref);
+
+    ASSERT_EQ(factory->get_parameters().skip_sorting, true);
+}
+
+
+TEST_F(ParIc, SetLStrategy)
+{
+    auto strategy = std::make_shared<strategy_type>();
+
+    auto factory =
+        ic_factory_type::build().with_l_strategy(strategy).on(this->ref);
+
+    ASSERT_EQ(factory->get_parameters().l_strategy, strategy);
+}
+
+
+TEST_F(ParIc, SetBothFactors)
+{
+    auto factory =
+        ic_factory_type::build().with_both_factors(false).on(this->ref);
+
+    ASSERT_FALSE(factory->get_parameters().both_factors);
+}
+
+
+TEST_F(ParIc, SetDefaults)
+{
+    auto factory = ic_factory_type::build().on(this->ref);
+
+    ASSERT_EQ(factory->get_parameters().iterations, 0u);
+    ASSERT_EQ(factory->get_parameters().skip_sorting, false);
+    ASSERT_EQ(factory->get_parameters().l_strategy, nullptr);
+    ASSERT_TRUE(factory->get_parameters().both_factors);
+}
+
+
+TEST_F(ParIc, SetEverything)
+{
+    auto strategy = std::make_shared<strategy_type>();
+
+    auto factory = ic_factory_type::build()
+                       .with_iterations(7u)
+                       .with_skip_sorting(false)
+                       .with_l_strategy(strategy)
+                       .with_both_factors(false)
+                       .on(this->ref);
+
+    ASSERT_EQ(factory->get_parameters().iterations, 7u);
+    ASSERT_EQ(factory->get_parameters().skip_sorting, false);
+    ASSERT_EQ(factory->get_parameters().l_strategy, strategy);
+    ASSERT_FALSE(factory->get_parameters().both_factors);
+}
+
+
+}  // namespace
diff --git a/core/test/factorization/par_ict.cpp b/core/test/factorization/par_ict.cpp
new file mode 100644
index 00000000000..6f11c33e25e
--- /dev/null
+++ b/core/test/factorization/par_ict.cpp
@@ -0,0 +1,147 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/factorization/par_ict.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+namespace {
+
+
+class ParIct : public ::testing::Test {
+public:
+    using value_type = gko::default_precision;
+    using index_type = gko::int32;
+    using ict_factory_type = gko::factorization::ParIct<value_type, index_type>;
+    using strategy_type = ict_factory_type::matrix_type::classical;
+
+protected:
+    ParIct() : ref(gko::ReferenceExecutor::create()) {}
+
+    std::shared_ptr<const gko::ReferenceExecutor> ref;
+};
+
+
+TEST_F(ParIct, SetIterations)
+{
+    auto factory = ict_factory_type::build().with_iterations(6u).on(ref);
+
+    ASSERT_EQ(factory->get_parameters().iterations, 6u);
+}
+
+
+TEST_F(ParIct, SetSkip)
+{
+    auto factory = ict_factory_type::build().with_skip_sorting(true).on(ref);
+
+    ASSERT_EQ(factory->get_parameters().skip_sorting, true);
+}
+
+
+TEST_F(ParIct, SetApprox)
+{
+    auto factory =
+        ict_factory_type::build().with_approximate_select(false).on(ref);
+
+    ASSERT_EQ(factory->get_parameters().approximate_select, false);
+}
+
+
+TEST_F(ParIct, SetDeterministic)
+{
+    auto factory =
+        ict_factory_type::build().with_deterministic_sample(true).on(ref);
+
+    ASSERT_EQ(factory->get_parameters().deterministic_sample, true);
+}
+
+
+TEST_F(ParIct, SetFillIn)
+{
+    auto factory = ict_factory_type::build().with_fill_in_limit(1.2).on(ref);
+
+    ASSERT_EQ(factory->get_parameters().fill_in_limit, 1.2);
+}
+
+
+TEST_F(ParIct, SetLStrategy)
+{
+    auto strategy = std::make_shared<strategy_type>();
+
+    auto factory =
+        ict_factory_type::build().with_l_strategy(strategy).on(this->ref);
+
+    ASSERT_EQ(factory->get_parameters().l_strategy, strategy);
+}
+
+
+TEST_F(ParIct, SetDefaults)
+{
+    auto factory = ict_factory_type::build().on(ref);
+
+    ASSERT_EQ(factory->get_parameters().iterations, 5u);
+    ASSERT_EQ(factory->get_parameters().skip_sorting, false);
+    ASSERT_EQ(factory->get_parameters().approximate_select, true);
+    ASSERT_EQ(factory->get_parameters().deterministic_sample, false);
+    ASSERT_EQ(factory->get_parameters().fill_in_limit, 2.0);
+    ASSERT_EQ(factory->get_parameters().l_strategy, nullptr);
+}
+
+
+TEST_F(ParIct, SetEverything)
+{
+    auto strategy = std::make_shared<strategy_type>();
+
+    auto factory = ict_factory_type::build()
+                       .with_iterations(7u)
+                       .with_skip_sorting(true)
+                       .with_approximate_select(false)
+                       .with_deterministic_sample(true)
+                       .with_fill_in_limit(1.2)
+                       .with_l_strategy(strategy)
+                       .on(ref);
+
+    ASSERT_EQ(factory->get_parameters().iterations, 7u);
+    ASSERT_EQ(factory->get_parameters().skip_sorting, true);
+    ASSERT_EQ(factory->get_parameters().approximate_select, false);
+    ASSERT_EQ(factory->get_parameters().deterministic_sample, true);
+    ASSERT_EQ(factory->get_parameters().fill_in_limit, 1.2);
+    ASSERT_EQ(factory->get_parameters().l_strategy, strategy);
+}
+
+
+}  // namespace
diff --git a/core/test/factorization/par_ilu.cpp b/core/test/factorization/par_ilu.cpp
index eeb5e1646c0..b0253c3f083 100644
--- a/core/test/factorization/par_ilu.cpp
+++ b/core/test/factorization/par_ilu.cpp
@@ -39,20 +39,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
-#include "core/test/utils.hpp"
-
-
 namespace {
 
 
-template <typename ValueIndexType>
 class ParIlu : public ::testing::Test {
 public:
-    using value_type =
-        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
-    using index_type =
-        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using value_type = double;
+    using index_type = gko::int32;
     using ilu_factory_type = gko::factorization::ParIlu<value_type, index_type>;
+    using strategy_type = ilu_factory_type::matrix_type::classical;
 
 protected:
     ParIlu() : ref(gko::ReferenceExecutor::create()) {}
@@ -60,38 +55,73 @@ class ParIlu : public ::testing::Test {
     std::shared_ptr<const gko::ReferenceExecutor> ref;
 };
 
-TYPED_TEST_SUITE(ParIlu, gko::test::ValueIndexTypes);
 
-
-TYPED_TEST(ParIlu, SetIterations)
+TEST_F(ParIlu, SetIterations)
 {
-    auto factory =
-        TestFixture::ilu_factory_type::build().with_iterations(5u).on(
-            this->ref);
+    auto factory = ilu_factory_type::build().with_iterations(5u).on(this->ref);
 
     ASSERT_EQ(factory->get_parameters().iterations, 5u);
 }
 
 
-TYPED_TEST(ParIlu, SetSkip)
+TEST_F(ParIlu, SetSkip)
 {
     auto factory =
-        TestFixture::ilu_factory_type::build().with_skip_sorting(true).on(
-            this->ref);
+        ilu_factory_type::build().with_skip_sorting(true).on(this->ref);
 
     ASSERT_EQ(factory->get_parameters().skip_sorting, true);
 }
 
 
-TYPED_TEST(ParIlu, SetEverything)
+TEST_F(ParIlu, SetLStrategy)
 {
-    auto factory = TestFixture::ilu_factory_type::build()
-                       .with_skip_sorting(false)
+    auto strategy = std::make_shared<strategy_type>();
+
+    auto factory =
+        ilu_factory_type::build().with_l_strategy(strategy).on(this->ref);
+
+    ASSERT_EQ(factory->get_parameters().l_strategy, strategy);
+}
+
+
+TEST_F(ParIlu, SetUStrategy)
+{
+    auto strategy = std::make_shared<strategy_type>();
+
+    auto factory =
+        ilu_factory_type::build().with_u_strategy(strategy).on(this->ref);
+
+    ASSERT_EQ(factory->get_parameters().u_strategy, strategy);
+}
+
+
+TEST_F(ParIlu, SetDefaults)
+{
+    auto factory = ilu_factory_type::build().on(this->ref);
+
+    ASSERT_EQ(factory->get_parameters().iterations, 0u);
+    ASSERT_EQ(factory->get_parameters().skip_sorting, false);
+    ASSERT_EQ(factory->get_parameters().l_strategy, nullptr);
+    ASSERT_EQ(factory->get_parameters().u_strategy, nullptr);
+}
+
+
+TEST_F(ParIlu, SetEverything)
+{
+    auto strategy = std::make_shared<strategy_type>();
+    auto strategy2 = std::make_shared<strategy_type>();
+
+    auto factory = ilu_factory_type::build()
                        .with_iterations(7u)
+                       .with_skip_sorting(false)
+                       .with_l_strategy(strategy)
+                       .with_u_strategy(strategy2)
                        .on(this->ref);
 
-    ASSERT_EQ(factory->get_parameters().skip_sorting, false);
     ASSERT_EQ(factory->get_parameters().iterations, 7u);
+    ASSERT_EQ(factory->get_parameters().skip_sorting, false);
+    ASSERT_EQ(factory->get_parameters().l_strategy, strategy);
+    ASSERT_EQ(factory->get_parameters().u_strategy, strategy2);
 }
 
 
diff --git a/core/test/factorization/par_ilut.cpp b/core/test/factorization/par_ilut.cpp
index 945dcae152b..7f659481106 100644
--- a/core/test/factorization/par_ilut.cpp
+++ b/core/test/factorization/par_ilut.cpp
@@ -48,6 +48,7 @@ class ParIlut : public ::testing::Test {
     using index_type = gko::int32;
     using ilut_factory_type =
         gko::factorization::ParIlut<value_type, index_type>;
+    using strategy_type = ilut_factory_type::matrix_type::classical;
 
 protected:
     ParIlut() : ref(gko::ReferenceExecutor::create()) {}
@@ -98,33 +99,64 @@ TEST_F(ParIlut, SetFillIn)
 }
 
 
+TEST_F(ParIlut, SetLStrategy)
+{
+    auto strategy = std::make_shared<strategy_type>();
+
+    auto factory =
+        ilut_factory_type::build().with_l_strategy(strategy).on(this->ref);
+
+    ASSERT_EQ(factory->get_parameters().l_strategy, strategy);
+}
+
+
+TEST_F(ParIlut, SetUStrategy)
+{
+    auto strategy = std::make_shared<strategy_type>();
+
+    auto factory =
+        ilut_factory_type::build().with_u_strategy(strategy).on(this->ref);
+
+    ASSERT_EQ(factory->get_parameters().u_strategy, strategy);
+}
+
+
 TEST_F(ParIlut, SetDefaults)
 {
     auto factory = ilut_factory_type::build().on(ref);
 
-    ASSERT_EQ(factory->get_parameters().skip_sorting, false);
     ASSERT_EQ(factory->get_parameters().iterations, 5u);
+    ASSERT_EQ(factory->get_parameters().skip_sorting, false);
     ASSERT_EQ(factory->get_parameters().approximate_select, true);
     ASSERT_EQ(factory->get_parameters().deterministic_sample, false);
     ASSERT_EQ(factory->get_parameters().fill_in_limit, 2.0);
+    ASSERT_EQ(factory->get_parameters().l_strategy, nullptr);
+    ASSERT_EQ(factory->get_parameters().u_strategy, nullptr);
 }
 
 
 TEST_F(ParIlut, SetEverything)
 {
+    auto strategy = std::make_shared<strategy_type>();
+    auto strategy2 = std::make_shared<strategy_type>();
+
     auto factory = ilut_factory_type::build()
-                       .with_skip_sorting(true)
                        .with_iterations(7u)
+                       .with_skip_sorting(true)
                        .with_approximate_select(false)
                        .with_deterministic_sample(true)
                        .with_fill_in_limit(1.2)
+                       .with_l_strategy(strategy)
+                       .with_u_strategy(strategy2)
                        .on(ref);
 
-    ASSERT_EQ(factory->get_parameters().skip_sorting, true);
     ASSERT_EQ(factory->get_parameters().iterations, 7u);
+    ASSERT_EQ(factory->get_parameters().skip_sorting, true);
     ASSERT_EQ(factory->get_parameters().approximate_select, false);
     ASSERT_EQ(factory->get_parameters().deterministic_sample, true);
     ASSERT_EQ(factory->get_parameters().fill_in_limit, 1.2);
+    ASSERT_EQ(factory->get_parameters().l_strategy, strategy);
+    ASSERT_EQ(factory->get_parameters().u_strategy, strategy2);
 }
 
 
diff --git a/core/test/preconditioner/CMakeLists.txt b/core/test/preconditioner/CMakeLists.txt
index efbeed1af2e..f75cd6b3baf 100644
--- a/core/test/preconditioner/CMakeLists.txt
+++ b/core/test/preconditioner/CMakeLists.txt
@@ -1,3 +1,4 @@
+ginkgo_create_test(ic)
 ginkgo_create_test(ilu)
 ginkgo_create_test(isai)
 ginkgo_create_test(jacobi)
diff --git a/core/test/preconditioner/ic.cpp b/core/test/preconditioner/ic.cpp
new file mode 100644
index 00000000000..a2e6345d2b5
--- /dev/null
+++ b/core/test/preconditioner/ic.cpp
@@ -0,0 +1,99 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/preconditioner/ic.hpp>
+
+
+#include <memory>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/factorization/par_ic.hpp>
+#include <ginkgo/core/solver/bicgstab.hpp>
+
+
+namespace {
+
+
+class IcFactory : public ::testing::Test {
+protected:
+    using value_type = double;
+    using index_type = gko::int32;
+    using solver_type = gko::solver::Bicgstab<value_type>;
+    using ic_prec_type = gko::preconditioner::Ic<solver_type>;
+    using ic_type = gko::factorization::ParIc<value_type, index_type>;
+
+    IcFactory()
+        : exec(gko::ReferenceExecutor::create()),
+          l_factory(solver_type::build().on(exec)),
+          fact_factory(ic_type::build().on(exec))
+    {}
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<typename solver_type::Factory> l_factory;
+    std::shared_ptr<typename ic_type::Factory> fact_factory;
+};
+
+
+TEST_F(IcFactory, KnowsItsExecutor)
+{
+    auto ic_factory = ic_prec_type::build().on(this->exec);
+
+    ASSERT_EQ(ic_factory->get_executor(), this->exec);
+}
+
+
+TEST_F(IcFactory, CanSetLSolverFactory)
+{
+    auto ic_factory = ic_prec_type::build()
+                          .with_l_solver_factory(this->l_factory)
+                          .on(this->exec);
+
+    ASSERT_EQ(ic_factory->get_parameters().l_solver_factory, this->l_factory);
+}
+
+
+TEST_F(IcFactory, CanSetFactorizationFactory)
+{
+    auto ic_factory = ic_prec_type::build()
+                          .with_factorization_factory(this->fact_factory)
+                          .on(this->exec);
+
+    ASSERT_EQ(ic_factory->get_parameters().factorization_factory,
+              this->fact_factory);
+}
+
+
+}  // namespace
diff --git a/core/test/preconditioner/ilu.cpp b/core/test/preconditioner/ilu.cpp
index c5c9e9363f2..3d93fc1411b 100644
--- a/core/test/preconditioner/ilu.cpp
+++ b/core/test/preconditioner/ilu.cpp
@@ -40,50 +40,47 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/factorization/par_ilu.hpp>
 #include <ginkgo/core/solver/bicgstab.hpp>
 
 
-#include "core/test/utils.hpp"
-
-
 namespace {
 
 
-template <typename T>
 class IluFactory : public ::testing::Test {
 protected:
-    using value_type = T;
+    using value_type = double;
+    using index_type = gko::int32;
     using l_solver_type = gko::solver::Bicgstab<value_type>;
     using u_solver_type = gko::solver::Bicgstab<value_type>;
     using ilu_prec_type =
         gko::preconditioner::Ilu<l_solver_type, u_solver_type, false>;
+    using ilu_type = gko::factorization::ParIlu<value_type, index_type>;
 
     IluFactory()
         : exec(gko::ReferenceExecutor::create()),
           l_factory(l_solver_type::build().on(exec)),
-          u_factory(u_solver_type::build().on(exec))
+          u_factory(u_solver_type::build().on(exec)),
+          fact_factory(ilu_type::build().on(exec))
     {}
 
     std::shared_ptr<const gko::Executor> exec;
     std::shared_ptr<typename l_solver_type::Factory> l_factory;
     std::shared_ptr<typename u_solver_type::Factory> u_factory;
+    std::shared_ptr<typename ilu_type::Factory> fact_factory;
 };
 
-TYPED_TEST_SUITE(IluFactory, gko::test::ValueTypes);
-
 
-TYPED_TEST(IluFactory, KnowsItsExecutor)
+TEST_F(IluFactory, KnowsItsExecutor)
 {
-    using ilu_prec_type = typename TestFixture::ilu_prec_type;
     auto ilu_factory = ilu_prec_type::build().on(this->exec);
 
     ASSERT_EQ(ilu_factory->get_executor(), this->exec);
 }
 
 
-TYPED_TEST(IluFactory, CanSetLSolverFactory)
+TEST_F(IluFactory, CanSetLSolverFactory)
 {
-    using ilu_prec_type = typename TestFixture::ilu_prec_type;
     auto ilu_factory = ilu_prec_type::build()
                            .with_l_solver_factory(this->l_factory)
                            .on(this->exec);
@@ -92,9 +89,8 @@ TYPED_TEST(IluFactory, CanSetLSolverFactory)
 }
 
 
-TYPED_TEST(IluFactory, CanSetUSolverFactory)
+TEST_F(IluFactory, CanSetUSolverFactory)
 {
-    using ilu_prec_type = typename TestFixture::ilu_prec_type;
     auto ilu_factory = ilu_prec_type::build()
                            .with_u_solver_factory(this->u_factory)
                            .on(this->exec);
@@ -103,4 +99,15 @@ TYPED_TEST(IluFactory, CanSetUSolverFactory)
 }
 
 
+TEST_F(IluFactory, CanSetFactorizationFactory)
+{
+    auto ilu_factory = ilu_prec_type::build()
+                           .with_factorization_factory(this->fact_factory)
+                           .on(this->exec);
+
+    ASSERT_EQ(ilu_factory->get_parameters().factorization_factory,
+              this->fact_factory);
+}
+
+
 }  // namespace
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 6b091a61263..708af4e931a 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -77,7 +77,9 @@ target_sources(ginkgo_cuda
     components/precision_conversion.cu
     components/prefix_sum.cu
     factorization/factorization_kernels.cu
+    factorization/ic_kernels.cu
     factorization/ilu_kernels.cu
+    factorization/par_ic_kernels.cu
     factorization/par_ict_kernels.cu
     factorization/par_ilu_kernels.cu
     factorization/par_ilut_approx_filter_kernel.cu
diff --git a/cuda/base/cusparse_bindings.hpp b/cuda/base/cusparse_bindings.hpp
index efc35b7774e..8a3de85293b 100644
--- a/cuda/base/cusparse_bindings.hpp
+++ b/cuda/base/cusparse_bindings.hpp
@@ -903,6 +903,20 @@ inline void destroy(csrilu02Info_t info)
 }
 
 
+inline csric02Info_t create_ic0_info()
+{
+    csric02Info_t info{};
+    GKO_ASSERT_NO_CUSPARSE_ERRORS(cusparseCreateCsric02Info(&info));
+    return info;
+}
+
+
+inline void destroy(csric02Info_t info)
+{
+    GKO_ASSERT_NO_CUSPARSE_ERRORS(cusparseDestroyCsric02Info(info));
+}
+
+
 // CUDA versions 9.2 and above have csrsm2.
 #if (defined(CUDA_VERSION) && (CUDA_VERSION >= 9020))
 
@@ -1352,6 +1366,104 @@ GKO_BIND_CUSPARSE_ILU0(std::complex<double>, cusparseZcsrilu02);
 #undef GKO_BIND_CUSPARSE_ILU0
 
 
+template <typename ValueType, typename IndexType>
+void ic0_buffer_size(cusparseHandle_t handle, IndexType m, IndexType nnz,
+                     const cusparseMatDescr_t descr, const ValueType *vals,
+                     const IndexType *row_ptrs, const IndexType *col_idxs,
+                     csric02Info_t info,
+                     size_type &buffer_size) GKO_NOT_IMPLEMENTED;
+
+#define GKO_BIND_CUSPARSE_IC0_BUFFER_SIZE(ValueType, CusparseName)           \
+    template <>                                                              \
+    inline void ic0_buffer_size<ValueType, int32>(                           \
+        cusparseHandle_t handle, int32 m, int32 nnz,                         \
+        const cusparseMatDescr_t descr, const ValueType *vals,               \
+        const int32 *row_ptrs, const int32 *col_idxs, csric02Info_t info,    \
+        size_type &buffer_size)                                              \
+    {                                                                        \
+        int tmp_buffer_size{};                                               \
+        GKO_ASSERT_NO_CUSPARSE_ERRORS(                                       \
+            CusparseName(handle, m, nnz, descr,                              \
+                         as_culibs_type(const_cast<ValueType *>(vals)),      \
+                         row_ptrs, col_idxs, info, &tmp_buffer_size));       \
+        buffer_size = tmp_buffer_size;                                       \
+    }                                                                        \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
+                  "semi-colon warnings")
+
+GKO_BIND_CUSPARSE_IC0_BUFFER_SIZE(float, cusparseScsric02_bufferSize);
+GKO_BIND_CUSPARSE_IC0_BUFFER_SIZE(double, cusparseDcsric02_bufferSize);
+GKO_BIND_CUSPARSE_IC0_BUFFER_SIZE(std::complex<float>,
+                                  cusparseCcsric02_bufferSize);
+GKO_BIND_CUSPARSE_IC0_BUFFER_SIZE(std::complex<double>,
+                                  cusparseZcsric02_bufferSize);
+
+#undef GKO_BIND_CUSPARSE_IC0_BUFFER_SIZE
+
+
+template <typename ValueType, typename IndexType>
+void ic0_analysis(cusparseHandle_t handle, IndexType m, IndexType nnz,
+                  const cusparseMatDescr_t descr, const ValueType *vals,
+                  const IndexType *row_ptrs, const IndexType *col_idxs,
+                  csric02Info_t info, cusparseSolvePolicy_t policy,
+                  void *buffer) GKO_NOT_IMPLEMENTED;
+
+#define GKO_BIND_CUSPARSE_IC0_ANALYSIS(ValueType, CusparseName)              \
+    template <>                                                              \
+    inline void ic0_analysis<ValueType, int32>(                              \
+        cusparseHandle_t handle, int32 m, int32 nnz,                         \
+        const cusparseMatDescr_t descr, const ValueType *vals,               \
+        const int32 *row_ptrs, const int32 *col_idxs, csric02Info_t info,    \
+        cusparseSolvePolicy_t policy, void *buffer)                          \
+    {                                                                        \
+        GKO_ASSERT_NO_CUSPARSE_ERRORS(                                       \
+            CusparseName(handle, m, nnz, descr, as_culibs_type(vals),        \
+                         row_ptrs, col_idxs, info, policy, buffer));         \
+    }                                                                        \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
+                  "semi-colon warnings")
+
+GKO_BIND_CUSPARSE_IC0_ANALYSIS(float, cusparseScsric02_analysis);
+GKO_BIND_CUSPARSE_IC0_ANALYSIS(double, cusparseDcsric02_analysis);
+GKO_BIND_CUSPARSE_IC0_ANALYSIS(std::complex<float>, cusparseCcsric02_analysis);
+GKO_BIND_CUSPARSE_IC0_ANALYSIS(std::complex<double>, cusparseZcsric02_analysis);
+
+#undef GKO_BIND_CUSPARSE_ILU0_ANALYSIS
+
+
+template <typename ValueType, typename IndexType>
+void ic0(cusparseHandle_t handle, IndexType m, IndexType nnz,
+         const cusparseMatDescr_t descr, ValueType *vals,
+         const IndexType *row_ptrs, const IndexType *col_idxs,
+         csric02Info_t info, cusparseSolvePolicy_t policy,
+         void *buffer) GKO_NOT_IMPLEMENTED;
+
+#define GKO_BIND_CUSPARSE_IC0(ValueType, CusparseName)                       \
+    template <>                                                              \
+    inline void ic0<ValueType, int32>(                                       \
+        cusparseHandle_t handle, int32 m, int32 nnz,                         \
+        const cusparseMatDescr_t descr, ValueType *vals,                     \
+        const int32 *row_ptrs, const int32 *col_idxs, csric02Info_t info,    \
+        cusparseSolvePolicy_t policy, void *buffer)                          \
+    {                                                                        \
+        GKO_ASSERT_NO_CUSPARSE_ERRORS(                                       \
+            CusparseName(handle, m, nnz, descr, as_culibs_type(vals),        \
+                         row_ptrs, col_idxs, info, policy, buffer));         \
+    }                                                                        \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
+                  "semi-colon warnings")
+
+GKO_BIND_CUSPARSE_IC0(float, cusparseScsric02);
+GKO_BIND_CUSPARSE_IC0(double, cusparseDcsric02);
+GKO_BIND_CUSPARSE_IC0(std::complex<float>, cusparseCcsric02);
+GKO_BIND_CUSPARSE_IC0(std::complex<double>, cusparseZcsric02);
+
+#undef GKO_BIND_CUSPARSE_IC0
+
+
 }  // namespace cusparse
 }  // namespace cuda
 }  // namespace kernels
diff --git a/cuda/factorization/ic_kernels.cu b/cuda/factorization/ic_kernels.cu
new file mode 100644
index 00000000000..de30263008f
--- /dev/null
+++ b/cuda/factorization/ic_kernels.cu
@@ -0,0 +1,94 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/ic_kernels.hpp"
+
+
+#include <ginkgo/core/base/array.hpp>
+
+
+#include "cuda/base/cusparse_bindings.hpp"
+#include "cuda/base/device_guard.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+/**
+ * @brief The ic factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace ic_factorization {
+
+
+template <typename ValueType, typename IndexType>
+void compute(std::shared_ptr<const DefaultExecutor> exec,
+             matrix::Csr<ValueType, IndexType> *m)
+{
+    const auto id = exec->get_device_id();
+    auto handle = exec->get_cusparse_handle();
+    gko::cuda::device_guard g{id};
+    auto desc = cusparse::create_mat_descr();
+    auto info = cusparse::create_ic0_info();
+
+    // get buffer size for IC
+    IndexType num_rows = m->get_size()[0];
+    IndexType nnz = m->get_num_stored_elements();
+    size_type buffer_size{};
+    cusparse::ic0_buffer_size(handle, num_rows, nnz, desc,
+                              m->get_const_values(), m->get_const_row_ptrs(),
+                              m->get_const_col_idxs(), info, buffer_size);
+
+    Array<char> buffer{exec, buffer_size};
+
+    // set up IC(0)
+    cusparse::ic0_analysis(handle, num_rows, nnz, desc, m->get_const_values(),
+                           m->get_const_row_ptrs(), m->get_const_col_idxs(),
+                           info, CUSPARSE_SOLVE_POLICY_USE_LEVEL,
+                           buffer.get_data());
+
+    cusparse::ic0(handle, num_rows, nnz, desc, m->get_values(),
+                  m->get_const_row_ptrs(), m->get_const_col_idxs(), info,
+                  CUSPARSE_SOLVE_POLICY_USE_LEVEL, buffer.get_data());
+
+    cusparse::destroy(info);
+    cusparse::destroy(desc);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_IC_COMPUTE_KERNEL);
+
+
+}  // namespace ic_factorization
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
diff --git a/cuda/factorization/par_ic_kernels.cu b/cuda/factorization/par_ic_kernels.cu
new file mode 100644
index 00000000000..56b00eb0fb1
--- /dev/null
+++ b/cuda/factorization/par_ic_kernels.cu
@@ -0,0 +1,141 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/par_ic_kernels.hpp"
+
+
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+#include "core/synthesizer/implementation_selection.hpp"
+#include "cuda/base/math.hpp"
+#include "cuda/components/merging.cuh"
+#include "cuda/components/reduction.cuh"
+#include "cuda/components/thread_ids.cuh"
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+/**
+ * @brief The parallel IC factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace par_ic_factorization {
+
+
+constexpr auto default_block_size = 512;
+
+
+// subwarp sizes for all warp-parallel kernels (sweep)
+using compiled_kernels =
+    syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
+
+
+#include "common/factorization/par_ic_kernels.hpp.inc"
+
+
+template <typename ValueType, typename IndexType>
+void init_factor(std::shared_ptr<const DefaultExecutor> exec,
+                 matrix::Csr<ValueType, IndexType> *l)
+{
+    auto num_rows = l->get_size()[0];
+    auto num_blocks = ceildiv(num_rows, default_block_size);
+    auto l_row_ptrs = l->get_const_row_ptrs();
+    auto l_vals = l->get_values();
+    kernel::ic_init<<<num_blocks, default_block_size>>>(
+        l_row_ptrs, as_cuda_type(l_vals), num_rows);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_IC_INIT_FACTOR_KERNEL);
+
+
+namespace {
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+void compute_factor(syn::value_list<int, subwarp_size>,
+                    std::shared_ptr<const DefaultExecutor> exec,
+                    size_type iterations,
+                    const matrix::Coo<ValueType, IndexType> *a_lower,
+                    matrix::Csr<ValueType, IndexType> *l)
+{
+    auto total_nnz = static_cast<IndexType>(l->get_num_stored_elements());
+    auto block_size = default_block_size / subwarp_size;
+    auto num_blocks = ceildiv(total_nnz, block_size);
+    for (size_type i = 0; i < iterations; ++i) {
+        kernel::ic_sweep<subwarp_size><<<num_blocks, default_block_size>>>(
+            a_lower->get_const_row_idxs(), a_lower->get_const_col_idxs(),
+            as_cuda_type(a_lower->get_const_values()), l->get_const_row_ptrs(),
+            l->get_const_col_idxs(), as_cuda_type(l->get_values()),
+            static_cast<IndexType>(l->get_num_stored_elements()));
+    }
+}
+
+
+GKO_ENABLE_IMPLEMENTATION_SELECTION(select_compute_factor, compute_factor);
+
+
+}  // namespace
+
+
+template <typename ValueType, typename IndexType>
+void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
+                    size_type iterations,
+                    const matrix::Coo<ValueType, IndexType> *a_lower,
+                    matrix::Csr<ValueType, IndexType> *l)
+{
+    auto num_rows = l->get_size()[0];
+    auto total_nnz = 2 * l->get_num_stored_elements();
+    auto total_nnz_per_row = total_nnz / num_rows;
+    select_compute_factor(
+        compiled_kernels(),
+        [&](int compiled_subwarp_size) {
+            return total_nnz_per_row <= compiled_subwarp_size ||
+                   compiled_subwarp_size == config::warp_size;
+        },
+        syn::value_list<int>(), syn::type_list<>(), exec, iterations, a_lower,
+        l);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_IC_COMPUTE_FACTOR_KERNEL);
+
+
+}  // namespace par_ic_factorization
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
diff --git a/cuda/factorization/par_ict_kernels.cu b/cuda/factorization/par_ict_kernels.cu
index bd682c44a28..512b34dfee7 100644
--- a/cuda/factorization/par_ict_kernels.cu
+++ b/cuda/factorization/par_ict_kernels.cu
@@ -83,18 +83,18 @@ namespace {
 template <int subwarp_size, typename ValueType, typename IndexType>
 void add_candidates(syn::value_list<int, subwarp_size>,
                     std::shared_ptr<const DefaultExecutor> exec,
-                    const matrix::Csr<ValueType, IndexType> *llt,
+                    const matrix::Csr<ValueType, IndexType> *llh,
                     const matrix::Csr<ValueType, IndexType> *a,
                     const matrix::Csr<ValueType, IndexType> *l,
                     matrix::Csr<ValueType, IndexType> *l_new)
 {
-    auto num_rows = static_cast<IndexType>(llt->get_size()[0]);
+    auto num_rows = static_cast<IndexType>(llh->get_size()[0]);
     auto subwarps_per_block = default_block_size / subwarp_size;
     auto num_blocks = ceildiv(num_rows, subwarps_per_block);
     matrix::CsrBuilder<ValueType, IndexType> l_new_builder(l_new);
-    auto llt_row_ptrs = llt->get_const_row_ptrs();
-    auto llt_col_idxs = llt->get_const_col_idxs();
-    auto llt_vals = llt->get_const_values();
+    auto llh_row_ptrs = llh->get_const_row_ptrs();
+    auto llh_col_idxs = llh->get_const_col_idxs();
+    auto llh_vals = llh->get_const_values();
     auto a_row_ptrs = a->get_const_row_ptrs();
     auto a_col_idxs = a->get_const_col_idxs();
     auto a_vals = a->get_const_values();
@@ -104,7 +104,7 @@ void add_candidates(syn::value_list<int, subwarp_size>,
     auto l_new_row_ptrs = l_new->get_row_ptrs();
     // count non-zeros per row
     kernel::ict_tri_spgeam_nnz<subwarp_size>
-        <<<num_blocks, default_block_size>>>(llt_row_ptrs, llt_col_idxs,
+        <<<num_blocks, default_block_size>>>(llh_row_ptrs, llh_col_idxs,
                                              a_row_ptrs, a_col_idxs,
                                              l_new_row_ptrs, num_rows);
 
@@ -122,7 +122,7 @@ void add_candidates(syn::value_list<int, subwarp_size>,
     // fill columns and values
     kernel::ict_tri_spgeam_init<subwarp_size>
         <<<num_blocks, default_block_size>>>(
-            llt_row_ptrs, llt_col_idxs, as_cuda_type(llt_vals), a_row_ptrs,
+            llh_row_ptrs, llh_col_idxs, as_cuda_type(llh_vals), a_row_ptrs,
             a_col_idxs, as_cuda_type(a_vals), l_row_ptrs, l_col_idxs,
             as_cuda_type(l_vals), l_new_row_ptrs, l_new_col_idxs,
             as_cuda_type(l_new_vals), num_rows);
@@ -159,14 +159,14 @@ GKO_ENABLE_IMPLEMENTATION_SELECTION(select_compute_factor, compute_factor);
 
 template <typename ValueType, typename IndexType>
 void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
-                    const matrix::Csr<ValueType, IndexType> *llt,
+                    const matrix::Csr<ValueType, IndexType> *llh,
                     const matrix::Csr<ValueType, IndexType> *a,
                     const matrix::Csr<ValueType, IndexType> *l,
                     matrix::Csr<ValueType, IndexType> *l_new)
 {
     auto num_rows = a->get_size()[0];
     auto total_nnz =
-        llt->get_num_stored_elements() + a->get_num_stored_elements();
+        llh->get_num_stored_elements() + a->get_num_stored_elements();
     auto total_nnz_per_row = total_nnz / num_rows;
     select_add_candidates(
         compiled_kernels(),
@@ -174,7 +174,7 @@ void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
             return total_nnz_per_row <= compiled_subwarp_size ||
                    compiled_subwarp_size == config::warp_size;
         },
-        syn::value_list<int>(), syn::type_list<>(), exec, llt, a, l, l_new);
+        syn::value_list<int>(), syn::type_list<>(), exec, llh, a, l, l_new);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
diff --git a/cuda/test/factorization/CMakeLists.txt b/cuda/test/factorization/CMakeLists.txt
index 5b494bf99b9..6fb7f5fea43 100644
--- a/cuda/test/factorization/CMakeLists.txt
+++ b/cuda/test/factorization/CMakeLists.txt
@@ -1,4 +1,6 @@
+ginkgo_create_test(ic_kernels)
 ginkgo_create_test(ilu_kernels)
+ginkgo_create_test(par_ic_kernels)
 ginkgo_create_test(par_ict_kernels)
 ginkgo_create_test(par_ilu_kernels)
 ginkgo_create_test(par_ilut_kernels)
diff --git a/cuda/test/factorization/ic_kernels.cpp b/cuda/test/factorization/ic_kernels.cpp
new file mode 100644
index 00000000000..dd7ecea6c65
--- /dev/null
+++ b/cuda/test/factorization/ic_kernels.cpp
@@ -0,0 +1,147 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/factorization/ic.hpp>
+
+
+#include <fstream>
+#include <memory>
+#include <random>
+#include <string>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/factorization/par_ic.hpp>
+
+
+#include "core/test/utils/unsort_matrix.hpp"
+#include "cuda/test/utils.hpp"
+#include "matrices/config.hpp"
+
+
+namespace {
+
+
+class Ic : public ::testing::Test {
+protected:
+    using value_type = gko::default_precision;
+    using index_type = gko::int32;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<gko::CudaExecutor> cuda;
+    std::ranlux48 rand_engine;
+    std::shared_ptr<Csr> csr_ref;
+    std::shared_ptr<Csr> csr_cuda;
+
+    Ic()
+        : ref(gko::ReferenceExecutor::create()),
+          cuda(gko::CudaExecutor::create(0, ref)),
+          rand_engine(6794)
+    {}
+
+    void SetUp() override
+    {
+        std::string file_name(gko::matrices::location_ani4_mtx);
+        auto input_file = std::ifstream(file_name, std::ios::in);
+        if (!input_file) {
+            FAIL() << "Could not find the file \"" << file_name
+                   << "\", which is required for this test.\n";
+        }
+        csr_ref = gko::read<Csr>(input_file, ref);
+        csr_cuda = Csr::create(cuda);
+        csr_cuda->copy_from(gko::lend(csr_ref));
+    }
+};
+
+
+TEST_F(Ic, ComputeICIsEquivalentToRefSorted)
+{
+    auto ref_fact = gko::factorization::ParIc<>::build()
+                        .with_skip_sorting(true)
+                        .on(ref)
+                        ->generate(csr_ref);
+    auto cuda_fact = gko::factorization::Ic<>::build()
+                         .with_skip_sorting(true)
+                         .on(cuda)
+                         ->generate(csr_cuda);
+
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_l_factor(), cuda_fact->get_l_factor(),
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_lh_factor(), cuda_fact->get_lh_factor(),
+                        1e-14);
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_l_factor(),
+                               cuda_fact->get_l_factor());
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_lh_factor(),
+                               cuda_fact->get_lh_factor());
+}
+
+
+TEST_F(Ic, ComputeICIsEquivalentToRefUnsorted)
+{
+    gko::test::unsort_matrix(gko::lend(csr_ref), rand_engine);
+    csr_cuda->copy_from(gko::lend(csr_ref));
+
+    auto ref_fact =
+        gko::factorization::ParIc<>::build().on(ref)->generate(csr_ref);
+    auto cuda_fact =
+        gko::factorization::Ic<>::build().on(cuda)->generate(csr_cuda);
+
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_l_factor(), cuda_fact->get_l_factor(),
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_lh_factor(), cuda_fact->get_lh_factor(),
+                        1e-14);
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_l_factor(),
+                               cuda_fact->get_l_factor());
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_lh_factor(),
+                               cuda_fact->get_lh_factor());
+}
+
+
+TEST_F(Ic, SetsCorrectStrategy)
+{
+    auto cuda_fact = gko::factorization::Ic<>::build()
+                         .with_l_strategy(std::make_shared<Csr::merge_path>())
+                         .on(cuda)
+                         ->generate(csr_cuda);
+
+    ASSERT_EQ(cuda_fact->get_l_factor()->get_strategy()->get_name(),
+              "merge_path");
+    ASSERT_EQ(cuda_fact->get_lh_factor()->get_strategy()->get_name(),
+              "merge_path");
+}
+
+
+}  // namespace
diff --git a/cuda/test/factorization/par_ic_kernels.cpp b/cuda/test/factorization/par_ic_kernels.cpp
new file mode 100644
index 00000000000..6577f2c798a
--- /dev/null
+++ b/cuda/test/factorization/par_ic_kernels.cpp
@@ -0,0 +1,167 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/par_ic_kernels.hpp"
+
+
+#include <algorithm>
+#include <fstream>
+#include <memory>
+#include <random>
+#include <string>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/factorization/factorization_kernels.hpp"
+#include "core/matrix/csr_builder.hpp"
+#include "core/matrix/csr_kernels.hpp"
+#include "cuda/test/utils.hpp"
+#include "matrices/config.hpp"
+
+
+namespace {
+
+
+class ParIc : public ::testing::Test {
+protected:
+    using value_type = double;
+    using index_type = gko::int32;
+    using Coo = gko::matrix::Coo<value_type, index_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+
+    ParIc()
+        : mtx_size(624, 624),
+          rand_engine(43456),
+          ref(gko::ReferenceExecutor::create()),
+          cuda(gko::CudaExecutor::create(0, gko::OmpExecutor::create()))
+    {
+        mtx_l = gko::test::generate_random_lower_triangular_matrix<Csr>(
+            mtx_size[0], mtx_size[0], false,
+            std::uniform_int_distribution<index_type>(10, mtx_size[0]),
+            std::normal_distribution<gko::remove_complex<value_type>>(0, 10.0),
+            rand_engine, ref);
+
+        dmtx_ani = Csr::create(cuda);
+        dmtx_l_ani = Csr::create(cuda);
+        dmtx_l_ani_init = Csr::create(cuda);
+        dmtx_l = Csr::create(cuda);
+        dmtx_l->copy_from(lend(mtx_l));
+    }
+
+    void SetUp()
+    {
+        std::string file_name(gko::matrices::location_ani4_mtx);
+        auto input_file = std::ifstream(file_name, std::ios::in);
+        if (!input_file) {
+            FAIL() << "Could not find the file \"" << file_name
+                   << "\", which is required for this test.\n";
+        }
+        mtx_ani = gko::read<Csr>(input_file, ref);
+        mtx_ani->sort_by_column_index();
+
+        {
+            mtx_l_ani = Csr::create(ref, mtx_ani->get_size());
+            gko::matrix::CsrBuilder<value_type, index_type> l_builder(
+                lend(mtx_l_ani));
+            gko::kernels::reference::factorization::initialize_row_ptrs_l(
+                ref, lend(mtx_ani), mtx_l_ani->get_row_ptrs());
+            auto l_nnz =
+                mtx_l_ani->get_const_row_ptrs()[mtx_ani->get_size()[0]];
+            l_builder.get_col_idx_array().resize_and_reset(l_nnz);
+            l_builder.get_value_array().resize_and_reset(l_nnz);
+            gko::kernels::reference::factorization::initialize_l(
+                ref, lend(mtx_ani), lend(mtx_l_ani), false);
+            mtx_l_ani_init = Csr::create(ref);
+            mtx_l_ani_init->copy_from(lend(mtx_l_ani));
+            gko::kernels::reference::par_ic_factorization::init_factor(
+                ref, lend(mtx_l_ani_init));
+        }
+        dmtx_ani->copy_from(lend(mtx_ani));
+        dmtx_l_ani->copy_from(lend(mtx_l_ani));
+        dmtx_l_ani_init->copy_from(lend(mtx_l_ani_init));
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<gko::CudaExecutor> cuda;
+
+    const gko::dim<2> mtx_size;
+    std::default_random_engine rand_engine;
+
+    std::unique_ptr<Csr> mtx_l;
+    std::unique_ptr<Csr> mtx_ani;
+    std::unique_ptr<Csr> mtx_l_ani;
+    std::unique_ptr<Csr> mtx_l_ani_init;
+
+    std::unique_ptr<Csr> dmtx_l;
+    std::unique_ptr<Csr> dmtx_ani;
+    std::unique_ptr<Csr> dmtx_l_ani;
+    std::unique_ptr<Csr> dmtx_l_ani_init;
+};
+
+
+TEST_F(ParIc, KernelInitFactorIsEquivalentToRef)
+{
+    gko::kernels::reference::par_ic_factorization::init_factor(ref,
+                                                               lend(mtx_l));
+    gko::kernels::cuda::par_ic_factorization::init_factor(cuda, lend(dmtx_l));
+
+    GKO_ASSERT_MTX_NEAR(mtx_l, dmtx_l, r<value_type>::value);
+}
+
+
+TEST_F(ParIc, KernelComputeFactorIsEquivalentToRef)
+{
+    auto square_size = mtx_ani->get_size();
+    auto mtx_l_coo = Coo::create(ref, square_size);
+    mtx_l_ani->convert_to(lend(mtx_l_coo));
+    auto dmtx_l_coo = Coo::create(cuda, square_size);
+    dmtx_l_coo->copy_from(lend(mtx_l_coo));
+
+    gko::kernels::reference::par_ic_factorization::compute_factor(
+        ref, 1, lend(mtx_l_coo), lend(mtx_l_ani_init));
+    gko::kernels::cuda::par_ic_factorization::compute_factor(
+        cuda, 20, lend(dmtx_l_coo), lend(dmtx_l_ani_init));
+
+    GKO_ASSERT_MTX_NEAR(mtx_l_ani, dmtx_l_ani, 1e-2);
+}
+
+
+}  // namespace
diff --git a/cuda/test/factorization/par_ict_kernels.cpp b/cuda/test/factorization/par_ict_kernels.cpp
index 2088cf5cb60..60360fce033 100644
--- a/cuda/test/factorization/par_ict_kernels.cpp
+++ b/cuda/test/factorization/par_ict_kernels.cpp
@@ -138,17 +138,17 @@ class ParIct : public ::testing::Test {
 
 TEST_F(ParIct, KernelAddCandidatesIsEquivalentToRef)
 {
-    auto mtx_llt = Csr::create(ref, mtx_size);
-    mtx_l->apply(lend(mtx_l->transpose()), lend(mtx_llt));
-    auto dmtx_llt = Csr::create(cuda, mtx_size);
-    dmtx_llt->copy_from(lend(mtx_llt));
+    auto mtx_llh = Csr::create(ref, mtx_size);
+    mtx_l->apply(lend(mtx_l->conj_transpose()), lend(mtx_llh));
+    auto dmtx_llh = Csr::create(cuda, mtx_size);
+    dmtx_llh->copy_from(lend(mtx_llh));
     auto res_mtx_l = Csr::create(ref, mtx_size);
     auto dres_mtx_l = Csr::create(cuda, mtx_size);
 
     gko::kernels::reference::par_ict_factorization::add_candidates(
-        ref, lend(mtx_llt), lend(mtx), lend(mtx_l), lend(res_mtx_l));
+        ref, lend(mtx_llh), lend(mtx), lend(mtx_l), lend(res_mtx_l));
     gko::kernels::cuda::par_ict_factorization::add_candidates(
-        cuda, lend(dmtx_llt), lend(dmtx), lend(dmtx_l), lend(dres_mtx_l));
+        cuda, lend(dmtx_llh), lend(dmtx), lend(dmtx_l), lend(dres_mtx_l));
 
     GKO_ASSERT_MTX_EQ_SPARSITY(res_mtx_l, dres_mtx_l);
     GKO_ASSERT_MTX_NEAR(res_mtx_l, dres_mtx_l, 1e-14);
diff --git a/dev_tools/scripts/config b/dev_tools/scripts/config
index 49ced95b64d..00cb06893d8 100644
--- a/dev_tools/scripts/config
+++ b/dev_tools/scripts/config
@@ -10,6 +10,8 @@
     - FixInclude: "core/factorization/par_ilut_kernels.hpp"
 - "(cuda|hip|omp|dpcpp)/test/factorization/par_ict_kernels"
     - FixInclude: "core/factorization/par_ict_kernels.hpp"
+- "(cuda|hip|omp|dpcpp)/test/factorization/par_ic_kernels"
+    - FixInclude: "core/factorization/par_ic_kernels.hpp"
 - "cuda/factorization/par_ilut_select_common"
     - FixInclude: "cuda/factorization/par_ilut_select_common.cuh"
 - "hip/factorization/par_ilut_select_common"
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index d3fe25ceb10..abbc0e65e63 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -15,8 +15,10 @@ target_sources(ginkgo_dpcpp
     components/fill_array.dp.cpp
     components/precision_conversion.dp.cpp
     components/prefix_sum.dp.cpp
+    factorization/ic_kernels.dp.cpp
     factorization/ilu_kernels.dp.cpp
     factorization/factorization_kernels.dp.cpp
+    factorization/par_c_kernels.dp.cpp
     factorization/par_ict_kernels.dp.cpp
     factorization/par_ilu_kernels.dp.cpp
     factorization/par_ilut_kernels.dp.cpp
diff --git a/dpcpp/factorization/ic_kernels.dp.cpp b/dpcpp/factorization/ic_kernels.dp.cpp
new file mode 100644
index 00000000000..f6c6d8ffadb
--- /dev/null
+++ b/dpcpp/factorization/ic_kernels.dp.cpp
@@ -0,0 +1,57 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/ic_kernels.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The ic factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace ic_factorization {
+
+
+template <typename ValueType, typename IndexType>
+void compute(std::shared_ptr<const DefaultExecutor> exec,
+             matrix::Csr<ValueType, IndexType> *m) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_IC_COMPUTE_KERNEL);
+
+
+}  // namespace ic_factorization
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/factorization/par_ic_kernels.dp.cpp b/dpcpp/factorization/par_ic_kernels.dp.cpp
new file mode 100644
index 00000000000..91b8a3c4862
--- /dev/null
+++ b/dpcpp/factorization/par_ic_kernels.dp.cpp
@@ -0,0 +1,75 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/par_ic_kernels.hpp"
+
+
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+#include "core/base/utils.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The parallel IC factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace par_ic_factorization {
+
+
+template <typename ValueType, typename IndexType>
+void init_factor(std::shared_ptr<const DefaultExecutor> exec,
+                 matrix::Csr<ValueType, IndexType> *l) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_IC_INIT_FACTOR_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
+                    size_type iterations,
+                    const matrix::Coo<ValueType, IndexType> *a_lower,
+                    matrix::Csr<ValueType, IndexType> *l) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_IC_COMPUTE_FACTOR_KERNEL);
+
+
+}  // namespace par_ic_factorization
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/factorization/par_ict_kernels.dp.cpp b/dpcpp/factorization/par_ict_kernels.dp.cpp
index cbf12277050..a782dba4af1 100644
--- a/dpcpp/factorization/par_ict_kernels.dp.cpp
+++ b/dpcpp/factorization/par_ict_kernels.dp.cpp
@@ -74,7 +74,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
-                    const matrix::Csr<ValueType, IndexType> *llt,
+                    const matrix::Csr<ValueType, IndexType> *llh,
                     const matrix::Csr<ValueType, IndexType> *a,
                     const matrix::Csr<ValueType, IndexType> *l,
                     matrix::Csr<ValueType, IndexType> *l_new)
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 634c7eb7dee..04df7e06506 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -173,7 +173,9 @@ set(GINKGO_HIP_SOURCES
     components/precision_conversion.hip.cpp
     components/prefix_sum.hip.cpp
     factorization/factorization_kernels.hip.cpp
+    factorization/ic_kernels.hip.cpp
     factorization/ilu_kernels.hip.cpp
+    factorization/par_ic_kernels.hip.cpp
     factorization/par_ict_kernels.hip.cpp
     factorization/par_ilu_kernels.hip.cpp
     factorization/par_ilut_approx_filter_kernel.hip.cpp
diff --git a/hip/base/hipsparse_bindings.hip.hpp b/hip/base/hipsparse_bindings.hip.hpp
index d1efeb55ef1..8a275631fc6 100644
--- a/hip/base/hipsparse_bindings.hip.hpp
+++ b/hip/base/hipsparse_bindings.hip.hpp
@@ -617,6 +617,20 @@ inline void destroy_ilu0_info(csrilu02Info_t info)
 }
 
 
+inline csric02Info_t create_ic0_info()
+{
+    csric02Info_t info{};
+    GKO_ASSERT_NO_HIPSPARSE_ERRORS(hipsparseCreateCsric02Info(&info));
+    return info;
+}
+
+
+inline void destroy_ic0_info(csric02Info_t info)
+{
+    GKO_ASSERT_NO_HIPSPARSE_ERRORS(hipsparseDestroyCsric02Info(info));
+}
+
+
 template <typename IndexType>
 void create_identity_permutation(hipsparseHandle_t handle, IndexType size,
                                  IndexType *permutation) GKO_NOT_IMPLEMENTED;
@@ -807,6 +821,118 @@ GKO_BIND_HIPSPARSE_ILU0(std::complex<double>, hipsparseZcsrilu02);
 #undef GKO_BIND_HIPSPARSE_ILU0
 
 
+template <typename ValueType, typename IndexType>
+void ic0_buffer_size(hipsparseHandle_t handle, IndexType m, IndexType nnz,
+                     const hipsparseMatDescr_t descr, const ValueType *vals,
+                     const IndexType *row_ptrs, const IndexType *col_idxs,
+                     csric02Info_t info,
+                     size_type &buffer_size) GKO_NOT_IMPLEMENTED;
+
+#define GKO_BIND_HIPSPARSE_IC0_BUFFER_SIZE(ValueType, HipsparseName)         \
+    template <>                                                              \
+    inline void ic0_buffer_size<ValueType, int32>(                           \
+        hipsparseHandle_t handle, int32 m, int32 nnz,                        \
+        const hipsparseMatDescr_t descr, const ValueType *vals,              \
+        const int32 *row_ptrs, const int32 *col_idxs, csric02Info_t info,    \
+        size_type &buffer_size)                                              \
+    {                                                                        \
+        int tmp_buffer_size{};                                               \
+        GKO_ASSERT_NO_HIPSPARSE_ERRORS(                                      \
+            HipsparseName(handle, m, nnz, descr,                             \
+                          as_hiplibs_type(const_cast<ValueType *>(vals)),    \
+                          row_ptrs, col_idxs, info, &tmp_buffer_size));      \
+        buffer_size = tmp_buffer_size;                                       \
+    }                                                                        \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
+                  "semi-colon warnings")
+
+GKO_BIND_HIPSPARSE_IC0_BUFFER_SIZE(float, hipsparseScsric02_bufferSize);
+GKO_BIND_HIPSPARSE_IC0_BUFFER_SIZE(double, hipsparseDcsric02_bufferSize);
+#if defined(hipsparseVersionMajor) && defined(hipsparseVersionMinor) && \
+    ((hipsparseVersionMajor > 1) ||                                     \
+     (hipsparseVersionMajor == 1 && hipsparseVersionMinor >= 4))
+GKO_BIND_HIPSPARSE_IC0_BUFFER_SIZE(std::complex<float>,
+                                   hipsparseCcsric02_bufferSize);
+GKO_BIND_HIPSPARSE_IC0_BUFFER_SIZE(std::complex<double>,
+                                   hipsparseZcsric02_bufferSize);
+#endif  // hipsparse version >= 1.4
+
+#undef GKO_BIND_HIPSPARSE_ILU0_BUFFER_SIZE
+
+
+template <typename ValueType, typename IndexType>
+void ic0_analysis(hipsparseHandle_t handle, IndexType m, IndexType nnz,
+                  const hipsparseMatDescr_t descr, const ValueType *vals,
+                  const IndexType *row_ptrs, const IndexType *col_idxs,
+                  csric02Info_t info, hipsparseSolvePolicy_t policy,
+                  void *buffer) GKO_NOT_IMPLEMENTED;
+
+#define GKO_BIND_HIPSPARSE_IC0_ANALYSIS(ValueType, HipsparseName)            \
+    template <>                                                              \
+    inline void ic0_analysis<ValueType, int32>(                              \
+        hipsparseHandle_t handle, int32 m, int32 nnz,                        \
+        const hipsparseMatDescr_t descr, const ValueType *vals,              \
+        const int32 *row_ptrs, const int32 *col_idxs, csric02Info_t info,    \
+        hipsparseSolvePolicy_t policy, void *buffer)                         \
+    {                                                                        \
+        GKO_ASSERT_NO_HIPSPARSE_ERRORS(                                      \
+            HipsparseName(handle, m, nnz, descr, as_hiplibs_type(vals),      \
+                          row_ptrs, col_idxs, info, policy, buffer));        \
+    }                                                                        \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
+                  "semi-colon warnings")
+
+GKO_BIND_HIPSPARSE_IC0_ANALYSIS(float, hipsparseScsric02_analysis);
+GKO_BIND_HIPSPARSE_IC0_ANALYSIS(double, hipsparseDcsric02_analysis);
+#if defined(hipsparseVersionMajor) && defined(hipsparseVersionMinor) && \
+    ((hipsparseVersionMajor > 1) ||                                     \
+     (hipsparseVersionMajor == 1 && hipsparseVersionMinor >= 4))
+GKO_BIND_HIPSPARSE_IC0_ANALYSIS(std::complex<float>,
+                                hipsparseCcsric02_analysis);
+GKO_BIND_HIPSPARSE_IC0_ANALYSIS(std::complex<double>,
+                                hipsparseZcsric02_analysis);
+#endif  // hipsparse version >= 1.4
+
+#undef GKO_BIND_HIPSPARSE_IC0_ANALYSIS
+
+
+template <typename ValueType, typename IndexType>
+void ic0(hipsparseHandle_t handle, IndexType m, IndexType nnz,
+         const hipsparseMatDescr_t descr, ValueType *vals,
+         const IndexType *row_ptrs, const IndexType *col_idxs,
+         csric02Info_t info, hipsparseSolvePolicy_t policy,
+         void *buffer) GKO_NOT_IMPLEMENTED;
+
+#define GKO_BIND_HIPSPARSE_IC0(ValueType, HipsparseName)                     \
+    template <>                                                              \
+    inline void ic0<ValueType, int32>(                                       \
+        hipsparseHandle_t handle, int32 m, int32 nnz,                        \
+        const hipsparseMatDescr_t descr, ValueType *vals,                    \
+        const int32 *row_ptrs, const int32 *col_idxs, csric02Info_t info,    \
+        hipsparseSolvePolicy_t policy, void *buffer)                         \
+    {                                                                        \
+        GKO_ASSERT_NO_HIPSPARSE_ERRORS(                                      \
+            HipsparseName(handle, m, nnz, descr, as_hiplibs_type(vals),      \
+                          row_ptrs, col_idxs, info, policy, buffer));        \
+    }                                                                        \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
+                  "semi-colon warnings")
+
+GKO_BIND_HIPSPARSE_IC0(float, hipsparseScsric02);
+GKO_BIND_HIPSPARSE_IC0(double, hipsparseDcsric02);
+#if defined(hipsparseVersionMajor) && defined(hipsparseVersionMinor) && \
+    ((hipsparseVersionMajor > 1) ||                                     \
+     (hipsparseVersionMajor == 1 && hipsparseVersionMinor >= 4))
+GKO_BIND_HIPSPARSE_IC0(std::complex<float>, hipsparseCcsric02);
+GKO_BIND_HIPSPARSE_IC0(std::complex<double>, hipsparseZcsric02);
+#endif  // hipsparse version >= 1.4
+
+#undef GKO_BIND_HIPSPARSE_IC0
+
+
 }  // namespace hipsparse
 }  // namespace hip
 }  // namespace kernels
diff --git a/hip/factorization/ic_kernels.hip.cpp b/hip/factorization/ic_kernels.hip.cpp
new file mode 100644
index 00000000000..8924c0521ac
--- /dev/null
+++ b/hip/factorization/ic_kernels.hip.cpp
@@ -0,0 +1,97 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/ic_kernels.hpp"
+
+
+#include <hip/hip_runtime.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+
+
+#include "hip/base/device_guard.hip.hpp"
+#include "hip/base/hipsparse_bindings.hip.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace hip {
+/**
+ * @brief The ic factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace ic_factorization {
+
+
+template <typename ValueType, typename IndexType>
+void compute(std::shared_ptr<const DefaultExecutor> exec,
+             matrix::Csr<ValueType, IndexType> *m)
+{
+    const auto id = exec->get_device_id();
+    auto handle = exec->get_hipsparse_handle();
+    gko::hip::device_guard g{id};
+    auto desc = hipsparse::create_mat_descr();
+    auto info = hipsparse::create_ic0_info();
+
+    // get buffer size for IC
+    IndexType num_rows = m->get_size()[0];
+    IndexType nnz = m->get_num_stored_elements();
+    size_type buffer_size{};
+    hipsparse::ic0_buffer_size(handle, num_rows, nnz, desc,
+                               m->get_const_values(), m->get_const_row_ptrs(),
+                               m->get_const_col_idxs(), info, buffer_size);
+
+    Array<char> buffer{exec, buffer_size};
+
+    // set up IC(0)
+    hipsparse::ic0_analysis(handle, num_rows, nnz, desc, m->get_const_values(),
+                            m->get_const_row_ptrs(), m->get_const_col_idxs(),
+                            info, HIPSPARSE_SOLVE_POLICY_USE_LEVEL,
+                            buffer.get_data());
+
+    hipsparse::ic0(handle, num_rows, nnz, desc, m->get_values(),
+                   m->get_const_row_ptrs(), m->get_const_col_idxs(), info,
+                   HIPSPARSE_SOLVE_POLICY_USE_LEVEL, buffer.get_data());
+
+    hipsparse::destroy_ic0_info(info);
+    hipsparse::destroy(desc);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_IC_COMPUTE_KERNEL);
+
+
+}  // namespace ic_factorization
+}  // namespace hip
+}  // namespace kernels
+}  // namespace gko
diff --git a/hip/factorization/par_ic_kernels.hip.cpp b/hip/factorization/par_ic_kernels.hip.cpp
new file mode 100644
index 00000000000..c4c11fda4b5
--- /dev/null
+++ b/hip/factorization/par_ic_kernels.hip.cpp
@@ -0,0 +1,144 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/par_ic_kernels.hpp"
+
+
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+#include "core/synthesizer/implementation_selection.hpp"
+#include "hip/base/math.hip.hpp"
+#include "hip/components/merging.hip.hpp"
+#include "hip/components/reduction.hip.hpp"
+#include "hip/components/thread_ids.hip.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace hip {
+/**
+ * @brief The parallel IC factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace par_ic_factorization {
+
+
+constexpr auto default_block_size = 512;
+
+
+// subwarp sizes for all warp-parallel kernels (sweep)
+using compiled_kernels =
+    syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
+
+
+#include "common/factorization/par_ic_kernels.hpp.inc"
+
+
+template <typename ValueType, typename IndexType>
+void init_factor(std::shared_ptr<const DefaultExecutor> exec,
+                 matrix::Csr<ValueType, IndexType> *l)
+{
+    auto num_rows = l->get_size()[0];
+    auto num_blocks = ceildiv(num_rows, default_block_size);
+    auto l_row_ptrs = l->get_const_row_ptrs();
+    auto l_vals = l->get_values();
+    hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel::ic_init), num_blocks,
+                       default_block_size, 0, 0, l_row_ptrs,
+                       as_hip_type(l_vals), num_rows);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_IC_INIT_FACTOR_KERNEL);
+
+
+namespace {
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+void compute_factor(syn::value_list<int, subwarp_size>,
+                    std::shared_ptr<const DefaultExecutor> exec,
+                    size_type iterations,
+                    const matrix::Coo<ValueType, IndexType> *a_lower,
+                    matrix::Csr<ValueType, IndexType> *l)
+{
+    auto total_nnz = static_cast<IndexType>(l->get_num_stored_elements());
+    auto block_size = default_block_size / subwarp_size;
+    auto num_blocks = ceildiv(total_nnz, block_size);
+    for (size_type i = 0; i < iterations; ++i) {
+        hipLaunchKernelGGL(
+            HIP_KERNEL_NAME(kernel::ic_sweep<subwarp_size>), num_blocks,
+            default_block_size, 0, 0, a_lower->get_const_row_idxs(),
+            a_lower->get_const_col_idxs(),
+            as_hip_type(a_lower->get_const_values()), l->get_const_row_ptrs(),
+            l->get_const_col_idxs(), as_hip_type(l->get_values()),
+            static_cast<IndexType>(l->get_num_stored_elements()));
+    }
+}
+
+
+GKO_ENABLE_IMPLEMENTATION_SELECTION(select_compute_factor, compute_factor);
+
+
+}  // namespace
+
+
+template <typename ValueType, typename IndexType>
+void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
+                    size_type iterations,
+                    const matrix::Coo<ValueType, IndexType> *a_lower,
+                    matrix::Csr<ValueType, IndexType> *l)
+{
+    auto num_rows = l->get_size()[0];
+    auto total_nnz = 2 * l->get_num_stored_elements();
+    auto total_nnz_per_row = total_nnz / num_rows;
+    select_compute_factor(
+        compiled_kernels(),
+        [&](int compiled_subwarp_size) {
+            return total_nnz_per_row <= compiled_subwarp_size ||
+                   compiled_subwarp_size == config::warp_size;
+        },
+        syn::value_list<int>(), syn::type_list<>(), exec, iterations, a_lower,
+        l);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_IC_COMPUTE_FACTOR_KERNEL);
+
+
+}  // namespace par_ic_factorization
+}  // namespace hip
+}  // namespace kernels
+}  // namespace gko
diff --git a/hip/factorization/par_ict_kernels.hip.cpp b/hip/factorization/par_ict_kernels.hip.cpp
index 2abfeff94de..456f0677e37 100644
--- a/hip/factorization/par_ict_kernels.hip.cpp
+++ b/hip/factorization/par_ict_kernels.hip.cpp
@@ -86,18 +86,18 @@ namespace {
 template <int subwarp_size, typename ValueType, typename IndexType>
 void add_candidates(syn::value_list<int, subwarp_size>,
                     std::shared_ptr<const DefaultExecutor> exec,
-                    const matrix::Csr<ValueType, IndexType> *llt,
+                    const matrix::Csr<ValueType, IndexType> *llh,
                     const matrix::Csr<ValueType, IndexType> *a,
                     const matrix::Csr<ValueType, IndexType> *l,
                     matrix::Csr<ValueType, IndexType> *l_new)
 {
-    auto num_rows = static_cast<IndexType>(llt->get_size()[0]);
+    auto num_rows = static_cast<IndexType>(llh->get_size()[0]);
     auto subwarps_per_block = default_block_size / subwarp_size;
     auto num_blocks = ceildiv(num_rows, subwarps_per_block);
     matrix::CsrBuilder<ValueType, IndexType> l_new_builder(l_new);
-    auto llt_row_ptrs = llt->get_const_row_ptrs();
-    auto llt_col_idxs = llt->get_const_col_idxs();
-    auto llt_vals = llt->get_const_values();
+    auto llh_row_ptrs = llh->get_const_row_ptrs();
+    auto llh_col_idxs = llh->get_const_col_idxs();
+    auto llh_vals = llh->get_const_values();
     auto a_row_ptrs = a->get_const_row_ptrs();
     auto a_col_idxs = a->get_const_col_idxs();
     auto a_vals = a->get_const_values();
@@ -108,8 +108,8 @@ void add_candidates(syn::value_list<int, subwarp_size>,
     // count non-zeros per row
     hipLaunchKernelGGL(
         HIP_KERNEL_NAME(kernel::ict_tri_spgeam_nnz<subwarp_size>),
-        dim3(num_blocks), dim3(default_block_size), 0, 0, llt_row_ptrs,
-        llt_col_idxs, a_row_ptrs, a_col_idxs, l_new_row_ptrs, num_rows);
+        dim3(num_blocks), dim3(default_block_size), 0, 0, llh_row_ptrs,
+        llh_col_idxs, a_row_ptrs, a_col_idxs, l_new_row_ptrs, num_rows);
 
     // build row ptrs
     components::prefix_sum(exec, l_new_row_ptrs, num_rows + 1);
@@ -125,8 +125,8 @@ void add_candidates(syn::value_list<int, subwarp_size>,
     // fill columns and values
     hipLaunchKernelGGL(
         HIP_KERNEL_NAME(kernel::ict_tri_spgeam_init<subwarp_size>),
-        dim3(num_blocks), dim3(default_block_size), 0, 0, llt_row_ptrs,
-        llt_col_idxs, as_hip_type(llt_vals), a_row_ptrs, a_col_idxs,
+        dim3(num_blocks), dim3(default_block_size), 0, 0, llh_row_ptrs,
+        llh_col_idxs, as_hip_type(llh_vals), a_row_ptrs, a_col_idxs,
         as_hip_type(a_vals), l_row_ptrs, l_col_idxs, as_hip_type(l_vals),
         l_new_row_ptrs, l_new_col_idxs, as_hip_type(l_new_vals), num_rows);
 }
@@ -163,14 +163,14 @@ GKO_ENABLE_IMPLEMENTATION_SELECTION(select_compute_factor, compute_factor);
 
 template <typename ValueType, typename IndexType>
 void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
-                    const matrix::Csr<ValueType, IndexType> *llt,
+                    const matrix::Csr<ValueType, IndexType> *llh,
                     const matrix::Csr<ValueType, IndexType> *a,
                     const matrix::Csr<ValueType, IndexType> *l,
                     matrix::Csr<ValueType, IndexType> *l_new)
 {
     auto num_rows = a->get_size()[0];
     auto total_nnz =
-        llt->get_num_stored_elements() + a->get_num_stored_elements();
+        llh->get_num_stored_elements() + a->get_num_stored_elements();
     auto total_nnz_per_row = total_nnz / num_rows;
     select_add_candidates(
         compiled_kernels(),
@@ -178,7 +178,7 @@ void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
             return total_nnz_per_row <= compiled_subwarp_size ||
                    compiled_subwarp_size == config::warp_size;
         },
-        syn::value_list<int>(), syn::type_list<>(), exec, llt, a, l, l_new);
+        syn::value_list<int>(), syn::type_list<>(), exec, llh, a, l, l_new);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
diff --git a/hip/test/factorization/CMakeLists.txt b/hip/test/factorization/CMakeLists.txt
index da6c40ca680..5eb375638c3 100644
--- a/hip/test/factorization/CMakeLists.txt
+++ b/hip/test/factorization/CMakeLists.txt
@@ -1,4 +1,6 @@
+ginkgo_create_hip_test_special_linkage(ic_kernels)
 ginkgo_create_hip_test_special_linkage(ilu_kernels)
+ginkgo_create_hip_test(par_ic_kernels)
 ginkgo_create_hip_test(par_ict_kernels)
 ginkgo_create_hip_test(par_ilu_kernels)
 ginkgo_create_hip_test(par_ilut_kernels)
diff --git a/hip/test/factorization/ic_kernels.cpp b/hip/test/factorization/ic_kernels.cpp
new file mode 100644
index 00000000000..6b39dcf8636
--- /dev/null
+++ b/hip/test/factorization/ic_kernels.cpp
@@ -0,0 +1,147 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/factorization/ic.hpp>
+
+
+#include <fstream>
+#include <memory>
+#include <random>
+#include <string>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/factorization/par_ic.hpp>
+
+
+#include "core/test/utils/unsort_matrix.hpp"
+#include "hip/test/utils.hip.hpp"
+#include "matrices/config.hpp"
+
+
+namespace {
+
+
+class Ic : public ::testing::Test {
+protected:
+    using value_type = gko::default_precision;
+    using index_type = gko::int32;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<gko::HipExecutor> hip;
+    std::ranlux48 rand_engine;
+    std::shared_ptr<Csr> csr_ref;
+    std::shared_ptr<Csr> csr_hip;
+
+    Ic()
+        : ref(gko::ReferenceExecutor::create()),
+          hip(gko::HipExecutor::create(0, ref)),
+          rand_engine(6794)
+    {}
+
+    void SetUp() override
+    {
+        std::string file_name(gko::matrices::location_ani4_mtx);
+        auto input_file = std::ifstream(file_name, std::ios::in);
+        if (!input_file) {
+            FAIL() << "Could not find the file \"" << file_name
+                   << "\", which is required for this test.\n";
+        }
+        csr_ref = gko::read<Csr>(input_file, ref);
+        csr_hip = Csr::create(hip);
+        csr_hip->copy_from(gko::lend(csr_ref));
+    }
+};
+
+
+TEST_F(Ic, ComputeICIsEquivalentToRefSorted)
+{
+    auto ref_fact = gko::factorization::ParIc<>::build()
+                        .with_skip_sorting(true)
+                        .on(ref)
+                        ->generate(csr_ref);
+    auto hip_fact = gko::factorization::Ic<>::build()
+                        .with_skip_sorting(true)
+                        .on(hip)
+                        ->generate(csr_hip);
+
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_l_factor(), hip_fact->get_l_factor(),
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_lh_factor(), hip_fact->get_lh_factor(),
+                        1e-14);
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_l_factor(),
+                               hip_fact->get_l_factor());
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_lh_factor(),
+                               hip_fact->get_lh_factor());
+}
+
+
+TEST_F(Ic, ComputeICIsEquivalentToRefUnsorted)
+{
+    gko::test::unsort_matrix(gko::lend(csr_ref), rand_engine);
+    csr_hip->copy_from(gko::lend(csr_ref));
+
+    auto ref_fact =
+        gko::factorization::ParIc<>::build().on(ref)->generate(csr_ref);
+    auto hip_fact =
+        gko::factorization::Ic<>::build().on(hip)->generate(csr_hip);
+
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_l_factor(), hip_fact->get_l_factor(),
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_lh_factor(), hip_fact->get_lh_factor(),
+                        1e-14);
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_l_factor(),
+                               hip_fact->get_l_factor());
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_lh_factor(),
+                               hip_fact->get_lh_factor());
+}
+
+
+TEST_F(Ic, SetsCorrectStrategy)
+{
+    auto hip_fact = gko::factorization::Ic<>::build()
+                        .with_l_strategy(std::make_shared<Csr::merge_path>())
+                        .on(hip)
+                        ->generate(csr_hip);
+
+    ASSERT_EQ(hip_fact->get_l_factor()->get_strategy()->get_name(),
+              "merge_path");
+    ASSERT_EQ(hip_fact->get_lh_factor()->get_strategy()->get_name(),
+              "merge_path");
+}
+
+
+}  // namespace
diff --git a/hip/test/factorization/par_ic_kernels.hip.cpp b/hip/test/factorization/par_ic_kernels.hip.cpp
new file mode 100644
index 00000000000..8a2b06fe0b4
--- /dev/null
+++ b/hip/test/factorization/par_ic_kernels.hip.cpp
@@ -0,0 +1,167 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/par_ic_kernels.hpp"
+
+
+#include <algorithm>
+#include <fstream>
+#include <memory>
+#include <random>
+#include <string>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/factorization/factorization_kernels.hpp"
+#include "core/matrix/csr_builder.hpp"
+#include "core/matrix/csr_kernels.hpp"
+#include "hip/test/utils.hip.hpp"
+#include "matrices/config.hpp"
+
+
+namespace {
+
+
+class ParIc : public ::testing::Test {
+protected:
+    using value_type = double;
+    using index_type = gko::int32;
+    using Coo = gko::matrix::Coo<value_type, index_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+
+    ParIc()
+        : mtx_size(585, 585),
+          rand_engine(10667),
+          ref(gko::ReferenceExecutor::create()),
+          hip(gko::HipExecutor::create(0, gko::OmpExecutor::create()))
+    {
+        mtx_l = gko::test::generate_random_lower_triangular_matrix<Csr>(
+            mtx_size[0], mtx_size[0], false,
+            std::uniform_int_distribution<index_type>(10, mtx_size[0]),
+            std::normal_distribution<gko::remove_complex<value_type>>(0, 10.0),
+            rand_engine, ref);
+
+        dmtx_ani = Csr::create(hip);
+        dmtx_l_ani = Csr::create(hip);
+        dmtx_l_ani_init = Csr::create(hip);
+        dmtx_l = Csr::create(hip);
+        dmtx_l->copy_from(lend(mtx_l));
+    }
+
+    void SetUp()
+    {
+        std::string file_name(gko::matrices::location_ani4_mtx);
+        auto input_file = std::ifstream(file_name, std::ios::in);
+        if (!input_file) {
+            FAIL() << "Could not find the file \"" << file_name
+                   << "\", which is required for this test.\n";
+        }
+        mtx_ani = gko::read<Csr>(input_file, ref);
+        mtx_ani->sort_by_column_index();
+
+        {
+            mtx_l_ani = Csr::create(ref, mtx_ani->get_size());
+            gko::matrix::CsrBuilder<value_type, index_type> l_builder(
+                lend(mtx_l_ani));
+            gko::kernels::reference::factorization::initialize_row_ptrs_l(
+                ref, lend(mtx_ani), mtx_l_ani->get_row_ptrs());
+            auto l_nnz =
+                mtx_l_ani->get_const_row_ptrs()[mtx_ani->get_size()[0]];
+            l_builder.get_col_idx_array().resize_and_reset(l_nnz);
+            l_builder.get_value_array().resize_and_reset(l_nnz);
+            gko::kernels::reference::factorization::initialize_l(
+                ref, lend(mtx_ani), lend(mtx_l_ani), false);
+            mtx_l_ani_init = Csr::create(ref);
+            mtx_l_ani_init->copy_from(lend(mtx_l_ani));
+            gko::kernels::reference::par_ic_factorization::init_factor(
+                ref, lend(mtx_l_ani_init));
+        }
+        dmtx_ani->copy_from(lend(mtx_ani));
+        dmtx_l_ani->copy_from(lend(mtx_l_ani));
+        dmtx_l_ani_init->copy_from(lend(mtx_l_ani_init));
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<gko::HipExecutor> hip;
+
+    const gko::dim<2> mtx_size;
+    std::default_random_engine rand_engine;
+
+    std::unique_ptr<Csr> mtx_l;
+    std::unique_ptr<Csr> mtx_ani;
+    std::unique_ptr<Csr> mtx_l_ani;
+    std::unique_ptr<Csr> mtx_l_ani_init;
+
+    std::unique_ptr<Csr> dmtx_l;
+    std::unique_ptr<Csr> dmtx_ani;
+    std::unique_ptr<Csr> dmtx_l_ani;
+    std::unique_ptr<Csr> dmtx_l_ani_init;
+};
+
+
+TEST_F(ParIc, KernelInitFactorIsEquivalentToRef)
+{
+    gko::kernels::reference::par_ic_factorization::init_factor(ref,
+                                                               lend(mtx_l));
+    gko::kernels::hip::par_ic_factorization::init_factor(hip, lend(dmtx_l));
+
+    GKO_ASSERT_MTX_NEAR(mtx_l, dmtx_l, r<value_type>::value);
+}
+
+
+TEST_F(ParIc, KernelComputeFactorIsEquivalentToRef)
+{
+    auto square_size = mtx_ani->get_size();
+    auto mtx_l_coo = Coo::create(ref, square_size);
+    mtx_l_ani->convert_to(lend(mtx_l_coo));
+    auto dmtx_l_coo = Coo::create(hip, square_size);
+    dmtx_l_coo->copy_from(lend(mtx_l_coo));
+
+    gko::kernels::reference::par_ic_factorization::compute_factor(
+        ref, 1, lend(mtx_l_coo), lend(mtx_l_ani_init));
+    gko::kernels::hip::par_ic_factorization::compute_factor(
+        hip, 20, lend(dmtx_l_coo), lend(dmtx_l_ani_init));
+
+    GKO_ASSERT_MTX_NEAR(mtx_l_ani, dmtx_l_ani, 1e-2);
+}
+
+
+}  // namespace
diff --git a/hip/test/factorization/par_ict_kernels.hip.cpp b/hip/test/factorization/par_ict_kernels.hip.cpp
index 7d3376f277a..6e334da4939 100644
--- a/hip/test/factorization/par_ict_kernels.hip.cpp
+++ b/hip/test/factorization/par_ict_kernels.hip.cpp
@@ -138,17 +138,17 @@ class ParIct : public ::testing::Test {
 
 TEST_F(ParIct, KernelAddCandidatesIsEquivalentToRef)
 {
-    auto mtx_llt = Csr::create(ref, mtx_size);
-    mtx_l->apply(lend(mtx_l->transpose()), lend(mtx_llt));
-    auto dmtx_llt = Csr::create(hip, mtx_size);
-    dmtx_llt->copy_from(lend(mtx_llt));
+    auto mtx_llh = Csr::create(ref, mtx_size);
+    mtx_l->apply(lend(mtx_l->transpose()), lend(mtx_llh));
+    auto dmtx_llh = Csr::create(hip, mtx_size);
+    dmtx_llh->copy_from(lend(mtx_llh));
     auto res_mtx_l = Csr::create(ref, mtx_size);
     auto dres_mtx_l = Csr::create(hip, mtx_size);
 
     gko::kernels::reference::par_ict_factorization::add_candidates(
-        ref, lend(mtx_llt), lend(mtx), lend(mtx_l), lend(res_mtx_l));
+        ref, lend(mtx_llh), lend(mtx), lend(mtx_l), lend(res_mtx_l));
     gko::kernels::hip::par_ict_factorization::add_candidates(
-        hip, lend(dmtx_llt), lend(dmtx), lend(dmtx_l), lend(dres_mtx_l));
+        hip, lend(dmtx_llh), lend(dmtx), lend(dmtx_l), lend(dres_mtx_l));
 
     GKO_ASSERT_MTX_EQ_SPARSITY(res_mtx_l, dres_mtx_l);
     GKO_ASSERT_MTX_NEAR(res_mtx_l, dres_mtx_l, 1e-14);
diff --git a/include/ginkgo/core/factorization/ic.hpp b/include/ginkgo/core/factorization/ic.hpp
new file mode 100644
index 00000000000..585d926b0d4
--- /dev/null
+++ b/include/ginkgo/core/factorization/ic.hpp
@@ -0,0 +1,164 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_PUBLIC_CORE_FACTORIZATION_IC_HPP_
+#define GKO_PUBLIC_CORE_FACTORIZATION_IC_HPP_
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/composition.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+namespace gko {
+/**
+ * @brief The Factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace factorization {
+
+
+/**
+ * Represents an incomplete Cholesky factorization (IC(0)) of a sparse matrix.
+ *
+ * More specifically, it consists of a lower triangular factor $L$ and
+ * its conjugate transpose $L^H$ with sparsity pattern
+ * $\mathcal S(L + L^H)$ = $\mathcal S(A)$
+ * fulfilling $LL^H = A$ at every non-zero location of $A$.
+ *
+ * @tparam ValueType  Type of the values of all matrices used in this class
+ * @tparam IndexType  Type of the indices of all matrices used in this class
+ *
+ * @ingroup factor
+ * @ingroup LinOp
+ */
+template <typename ValueType = gko::default_precision,
+          typename IndexType = gko::int32>
+class Ic : public Composition<ValueType> {
+public:
+    using value_type = ValueType;
+    using index_type = IndexType;
+    using matrix_type = matrix::Csr<ValueType, IndexType>;
+
+    std::shared_ptr<const matrix_type> get_l_factor() const
+    {
+        // Can be `static_cast` since the type is guaranteed in this class
+        return std::static_pointer_cast<const matrix_type>(
+            this->get_operators()[0]);
+    }
+
+    std::shared_ptr<const matrix_type> get_lh_factor() const
+    {
+        if (this->get_operators().size() == 2) {
+            // Can be `static_cast` since the type is guaranteed in this class
+            return std::static_pointer_cast<const matrix_type>(
+                this->get_operators()[1]);
+        } else {
+            return std::static_pointer_cast<const matrix_type>(
+                share(get_l_factor()->conj_transpose()));
+        }
+    }
+
+    // Remove the possibility of calling `create`, which was enabled by
+    // `Composition`
+    template <typename... Args>
+    static std::unique_ptr<Composition<ValueType>> create(Args &&... args) =
+        delete;
+
+    GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
+    {
+        /**
+         * Strategy which will be used by the L matrix. The default value
+         * `nullptr` will result in the strategy `classical`.
+         */
+        std::shared_ptr<typename matrix_type::strategy_type>
+            GKO_FACTORY_PARAMETER_SCALAR(l_strategy, nullptr);
+
+        /**
+         * @brief `true` means it is known that the matrix given to this
+         *        factory will be sorted first by row, then by column index,
+         *        `false` means it is unknown or not sorted, so an additional
+         *        sorting step will be performed during the factorization
+         *        (it will not change the matrix given).
+         *        The matrix must be sorted for this factorization to work.
+         *
+         * The `system_matrix`, which will be given to this factory, must be
+         * sorted (first by row, then by column) in order for the algorithm
+         * to work. If it is known that the matrix will be sorted, this
+         * parameter can be set to `true` to skip the sorting (therefore,
+         * shortening the runtime).
+         * However, if it is unknown or if the matrix is known to be not sorted,
+         * it must remain `false`, otherwise, this factorization might be
+         * incorrect.
+         */
+        bool GKO_FACTORY_PARAMETER_SCALAR(skip_sorting, false);
+
+        /**
+         * `true` will generate both L and L^H, `false` will only generate the L
+         * factor, resulting in a Composition of only a single LinOp. This can
+         * be used to avoid the transposition operation.
+         */
+        bool GKO_FACTORY_PARAMETER_SCALAR(both_factors, true);
+    };
+    GKO_ENABLE_LIN_OP_FACTORY(Ic, parameters, Factory);
+    GKO_ENABLE_BUILD_METHOD(Factory);
+
+protected:
+    Ic(const Factory *factory, std::shared_ptr<const gko::LinOp> system_matrix)
+        : Composition<ValueType>{factory->get_executor()},
+          parameters_{factory->get_parameters()}
+    {
+        if (parameters_.l_strategy == nullptr) {
+            parameters_.l_strategy =
+                std::make_shared<typename matrix_type::classical>();
+        }
+        generate(system_matrix, parameters_.skip_sorting,
+                 parameters_.both_factors)
+            ->move_to(this);
+    }
+
+    std::unique_ptr<Composition<ValueType>> generate(
+        const std::shared_ptr<const LinOp> &system_matrix, bool skip_sorting,
+        bool both_factors) const;
+};
+
+
+}  // namespace factorization
+}  // namespace gko
+
+
+#endif  // GKO_PUBLIC_CORE_FACTORIZATION_IC_HPP_
diff --git a/include/ginkgo/core/factorization/par_ic.hpp b/include/ginkgo/core/factorization/par_ic.hpp
new file mode 100644
index 00000000000..12bae4cfc37
--- /dev/null
+++ b/include/ginkgo/core/factorization/par_ic.hpp
@@ -0,0 +1,196 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_PUBLIC_CORE_FACTORIZATION_PAR_IC_HPP_
+#define GKO_PUBLIC_CORE_FACTORIZATION_PAR_IC_HPP_
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/composition.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+namespace gko {
+/**
+ * @brief The Factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace factorization {
+
+
+/**
+ * ParIC is an incomplete Cholesky factorization which is computed in parallel.
+ *
+ * $L$ is a lower triangular matrix, which approximates a given matrix $A$ with
+ * $A \approx LL^H$. Here, $L + L^H$ has the same sparsity pattern as $A$, which
+ * is also called IC(0).
+ *
+ * The ParIC algorithm generates the incomplete factors iteratively, using a
+ * fixed-point iteration of the form
+ *
+ * $
+ * F(L) =
+ * \begin{cases}
+ *     \sqrt{a_{ii}-\sum_{k=1}^{i-1}|l_{ik}|^2}, \quad & i == j \\
+ *     a_{ij}-\sum_{k=1}^{i-1}l_{ik}u_{kj}, \quad & i < j
+ * \end{cases}
+ * $
+ *
+ * In general, the entries of $L$ can be iterated in parallel and in
+ * asynchronous fashion, the algorithm asymptotically converges to the
+ * incomplete factors $L$ and $L^H$ fulfilling $\left(R = A - L \cdot
+ * L^H\right)\vert_\mathcal{S} = 0\vert_\mathcal{S}$ where $\mathcal{S}$ is the
+ * pre-defined sparsity pattern (in case of IC(0) the sparsity pattern of the
+ * system matrix $A$). The number of ParIC sweeps needed for convergence
+ * depends on the parallelism level: For sequential execution, a single sweep
+ * is sufficient, for fine-grained parallelism, 3 sweeps are typically
+ * generating a good approximation.
+ *
+ * The ParIC algorithm in Ginkgo follows the design of E. Chow and A. Patel,
+ * Fine-grained Parallel Incomplete LU Factorization, SIAM Journal on Scientific
+ * Computing, 37, C169-C193 (2015).
+ *
+ * @tparam ValueType  Type of the values of all matrices used in this class
+ * @tparam IndexType  Type of the indices of all matrices used in this class
+ *
+ * @ingroup factor
+ * @ingroup LinOp
+ */
+template <typename ValueType = default_precision, typename IndexType = int32>
+class ParIc : public Composition<ValueType> {
+public:
+    using value_type = ValueType;
+    using index_type = IndexType;
+    using matrix_type = matrix::Csr<ValueType, IndexType>;
+
+    std::shared_ptr<const matrix_type> get_l_factor() const
+    {
+        // Can be `static_cast` since the type is guaranteed in this class
+        return std::static_pointer_cast<const matrix_type>(
+            this->get_operators()[0]);
+    }
+
+    std::shared_ptr<const matrix_type> get_lh_factor() const
+    {
+        if (this->get_operators().size() == 2) {
+            // Can be `static_cast` since the type is guaranteed in this class
+            return std::static_pointer_cast<const matrix_type>(
+                this->get_operators()[1]);
+        } else {
+            return std::static_pointer_cast<const matrix_type>(
+                share(get_l_factor()->conj_transpose()));
+        }
+    }
+
+    // Remove the possibility of calling `create`, which was enabled by
+    // `Composition`
+    template <typename... Args>
+    static std::unique_ptr<Composition<ValueType>> create(Args &&... args) =
+        delete;
+
+    GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
+    {
+        /**
+         * The number of iterations the `compute` kernel will use when doing
+         * the factorization. The default value `0` means `Auto`, so the
+         * implementation decides on the actual value depending on the
+         * ressources that are available.
+         */
+        size_type GKO_FACTORY_PARAMETER_SCALAR(iterations, 0);
+
+        /**
+         * @brief `true` means it is known that the matrix given to this
+         *        factory will be sorted first by row, then by column index,
+         *        `false` means it is unknown or not sorted, so an additional
+         *        sorting step will be performed during the factorization
+         *        (it will not change the matrix given).
+         *        The matrix must be sorted for this factorization to work.
+         *
+         * The `system_matrix`, which will be given to this factory, must be
+         * sorted (first by row, then by column) in order for the algorithm
+         * to work. If it is known that the matrix will be sorted, this
+         * parameter can be set to `true` to skip the sorting (therefore,
+         * shortening the runtime).
+         * However, if it is unknown or if the matrix is known to be not sorted,
+         * it must remain `false`, otherwise, the factorization might be
+         * incorrect.
+         */
+        bool GKO_FACTORY_PARAMETER_SCALAR(skip_sorting, false);
+
+        /**
+         * Strategy which will be used by the L matrix. The default value
+         * `nullptr` will result in the strategy `classical`.
+         */
+        std::shared_ptr<typename matrix_type::strategy_type>
+            GKO_FACTORY_PARAMETER_SCALAR(l_strategy, nullptr);
+
+        /**
+         * `true` will generate both L and L^H, `false` will only generate the L
+         * factor, resulting in a Composition of only a single LinOp. This can
+         * be used to avoid the transposition operation.
+         */
+        bool GKO_FACTORY_PARAMETER_SCALAR(both_factors, true);
+    };
+    GKO_ENABLE_LIN_OP_FACTORY(ParIc, parameters, Factory);
+    GKO_ENABLE_BUILD_METHOD(Factory);
+
+protected:
+    explicit ParIc(const Factory *factory,
+                   std::shared_ptr<const LinOp> system_matrix)
+        : Composition<ValueType>(factory->get_executor()),
+          parameters_{factory->get_parameters()}
+    {
+        if (parameters_.l_strategy == nullptr) {
+            parameters_.l_strategy =
+                std::make_shared<typename matrix_type::classical>();
+        }
+        generate(system_matrix, parameters_.skip_sorting,
+                 parameters_.both_factors)
+            ->move_to(this);
+    }
+
+    std::unique_ptr<Composition<ValueType>> generate(
+        const std::shared_ptr<const LinOp> &system_matrix, bool skip_sorting,
+        bool both_factors) const;
+};
+
+
+}  // namespace factorization
+}  // namespace gko
+
+
+#endif  // GKO_PUBLIC_CORE_FACTORIZATION_PAR_IC_HPP_
diff --git a/include/ginkgo/core/factorization/par_ict.hpp b/include/ginkgo/core/factorization/par_ict.hpp
index cc113b8c807..292c9d54ef2 100644
--- a/include/ginkgo/core/factorization/par_ict.hpp
+++ b/include/ginkgo/core/factorization/par_ict.hpp
@@ -106,7 +106,7 @@ class ParIct : public Composition<ValueType> {
             this->get_operators()[0]);
     }
 
-    std::shared_ptr<const matrix_type> get_lt_factor() const
+    std::shared_ptr<const matrix_type> get_lh_factor() const
     {
         // Can be `static_cast` since the type is guaranteed in this class
         return std::static_pointer_cast<const matrix_type>(
diff --git a/include/ginkgo/core/factorization/par_ilu.hpp b/include/ginkgo/core/factorization/par_ilu.hpp
index e503874345d..eaafb9b3971 100644
--- a/include/ginkgo/core/factorization/par_ilu.hpp
+++ b/include/ginkgo/core/factorization/par_ilu.hpp
@@ -96,20 +96,21 @@ class ParIlu : public Composition<ValueType> {
 public:
     using value_type = ValueType;
     using index_type = IndexType;
-    using l_matrix_type = matrix::Csr<ValueType, IndexType>;
-    using u_matrix_type = matrix::Csr<ValueType, IndexType>;
+    using matrix_type = matrix::Csr<ValueType, IndexType>;
+    using l_matrix_type = matrix_type;
+    using u_matrix_type = matrix_type;
 
-    std::shared_ptr<const l_matrix_type> get_l_factor() const
+    std::shared_ptr<const matrix_type> get_l_factor() const
     {
         // Can be `static_cast` since the type is guaranteed in this class
-        return std::static_pointer_cast<const l_matrix_type>(
+        return std::static_pointer_cast<const matrix_type>(
             this->get_operators()[0]);
     }
 
-    std::shared_ptr<const u_matrix_type> get_u_factor() const
+    std::shared_ptr<const matrix_type> get_u_factor() const
     {
         // Can be `static_cast` since the type is guaranteed in this class
-        return std::static_pointer_cast<const u_matrix_type>(
+        return std::static_pointer_cast<const matrix_type>(
             this->get_operators()[1]);
     }
 
@@ -152,14 +153,14 @@ class ParIlu : public Composition<ValueType> {
          * Strategy which will be used by the L matrix. The default value
          * `nullptr` will result in the strategy `classical`.
          */
-        std::shared_ptr<typename l_matrix_type::strategy_type>
+        std::shared_ptr<typename matrix_type::strategy_type>
             GKO_FACTORY_PARAMETER_SCALAR(l_strategy, nullptr);
 
         /**
          * Strategy which will be used by the U matrix. The default value
          * `nullptr` will result in the strategy `classical`.
          */
-        std::shared_ptr<typename u_matrix_type::strategy_type>
+        std::shared_ptr<typename matrix_type::strategy_type>
             GKO_FACTORY_PARAMETER_SCALAR(u_strategy, nullptr);
     };
     GKO_ENABLE_LIN_OP_FACTORY(ParIlu, parameters, Factory);
@@ -173,11 +174,11 @@ class ParIlu : public Composition<ValueType> {
     {
         if (parameters_.l_strategy == nullptr) {
             parameters_.l_strategy =
-                std::make_shared<typename l_matrix_type::classical>();
+                std::make_shared<typename matrix_type::classical>();
         }
         if (parameters_.u_strategy == nullptr) {
             parameters_.u_strategy =
-                std::make_shared<typename u_matrix_type::classical>();
+                std::make_shared<typename matrix_type::classical>();
         }
         generate_l_u(system_matrix, parameters_.skip_sorting,
                      parameters_.l_strategy, parameters_.u_strategy)
@@ -203,9 +204,8 @@ class ParIlu : public Composition<ValueType> {
      */
     std::unique_ptr<Composition<ValueType>> generate_l_u(
         const std::shared_ptr<const LinOp> &system_matrix, bool skip_sorting,
-        std::shared_ptr<typename l_matrix_type::strategy_type> l_strategy,
-        std::shared_ptr<typename u_matrix_type::strategy_type> u_strategy)
-        const;
+        std::shared_ptr<typename matrix_type::strategy_type> l_strategy,
+        std::shared_ptr<typename matrix_type::strategy_type> u_strategy) const;
 };
 
 
diff --git a/include/ginkgo/core/factorization/par_ilut.hpp b/include/ginkgo/core/factorization/par_ilut.hpp
index e8caa43c45a..0cb54dfa73d 100644
--- a/include/ginkgo/core/factorization/par_ilut.hpp
+++ b/include/ginkgo/core/factorization/par_ilut.hpp
@@ -100,20 +100,21 @@ class ParIlut : public Composition<ValueType> {
 public:
     using value_type = ValueType;
     using index_type = IndexType;
-    using l_matrix_type = matrix::Csr<ValueType, IndexType>;
-    using u_matrix_type = matrix::Csr<ValueType, IndexType>;
+    using matrix_type = matrix::Csr<ValueType, IndexType>;
+    using l_matrix_type = matrix_type;
+    using u_matrix_type = matrix_type;
 
-    std::shared_ptr<const l_matrix_type> get_l_factor() const
+    std::shared_ptr<const matrix_type> get_l_factor() const
     {
         // Can be `static_cast` since the type is guaranteed in this class
-        return std::static_pointer_cast<const l_matrix_type>(
+        return std::static_pointer_cast<const matrix_type>(
             this->get_operators()[0]);
     }
 
-    std::shared_ptr<const u_matrix_type> get_u_factor() const
+    std::shared_ptr<const matrix_type> get_u_factor() const
     {
         // Can be `static_cast` since the type is guaranteed in this class
-        return std::static_pointer_cast<const u_matrix_type>(
+        return std::static_pointer_cast<const matrix_type>(
             this->get_operators()[1]);
     }
 
@@ -204,14 +205,14 @@ class ParIlut : public Composition<ValueType> {
          * Strategy which will be used by the L matrix. The default value
          * `nullptr` will result in the strategy `classical`.
          */
-        std::shared_ptr<typename l_matrix_type::strategy_type>
+        std::shared_ptr<typename matrix_type::strategy_type>
             GKO_FACTORY_PARAMETER_SCALAR(l_strategy, nullptr);
 
         /**
          * Strategy which will be used by the U matrix. The default value
          * `nullptr` will result in the strategy `classical`.
          */
-        std::shared_ptr<typename u_matrix_type::strategy_type>
+        std::shared_ptr<typename matrix_type::strategy_type>
             GKO_FACTORY_PARAMETER_SCALAR(u_strategy, nullptr);
     };
     GKO_ENABLE_LIN_OP_FACTORY(ParIlut, parameters, Factory);
@@ -225,11 +226,11 @@ class ParIlut : public Composition<ValueType> {
     {
         if (parameters_.l_strategy == nullptr) {
             parameters_.l_strategy =
-                std::make_shared<typename l_matrix_type::classical>();
+                std::make_shared<typename matrix_type::classical>();
         }
         if (parameters_.u_strategy == nullptr) {
             parameters_.u_strategy =
-                std::make_shared<typename u_matrix_type::classical>();
+                std::make_shared<typename matrix_type::classical>();
         }
         generate_l_u(std::move(system_matrix))->move_to(this);
     }
diff --git a/include/ginkgo/core/preconditioner/ic.hpp b/include/ginkgo/core/preconditioner/ic.hpp
new file mode 100644
index 00000000000..70fc233dfd5
--- /dev/null
+++ b/include/ginkgo/core/preconditioner/ic.hpp
@@ -0,0 +1,395 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_PUBLIC_CORE_PRECONDITIONER_IC_HPP_
+#define GKO_PUBLIC_CORE_PRECONDITIONER_IC_HPP_
+
+
+#include <memory>
+#include <type_traits>
+
+
+#include <ginkgo/core/base/abstract_factory.hpp>
+#include <ginkgo/core/base/composition.hpp>
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/std_extensions.hpp>
+#include <ginkgo/core/factorization/par_ic.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/solver/lower_trs.hpp>
+#include <ginkgo/core/solver/upper_trs.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
+
+
+namespace gko {
+namespace preconditioner {
+
+
+/**
+ * The Incomplete LU (ILU) preconditioner solves the equation $LUx = b$ for a
+ * given lower triangular matrix L, an upper triangular matrix U and the right
+ * hand side b (can contain multiple right hand sides).
+ *
+ * It allows to set both the solver for L and the solver for U independently,
+ * while providing the defaults solver::LowerTrs and solver::UpperTrs, which
+ * are direct triangular solvers.
+ * For these solvers, a factory can be provided (with `with_l_solver_factory`
+ * and `with_u_solver_factory`) to have more control over their behavior.
+ * In particular, it is possible to use an iterative method for solving the
+ * triangular systems. The default parameters for an iterative triangluar
+ * solver are:
+ * - reduction factor = 1e-4
+ * - max iteration = <number of rows of the matrix given to the solver>
+ * Solvers without such criteria can also be used, in which case none are set.
+ *
+ * An object of this class can be created with a matrix or a gko::Composition
+ * containing two matrices. If created with a matrix, it is factorized before
+ * creating the solver. If a gko::Composition (containing two matrices) is
+ * used, the first operand will be taken as the L matrix, the second will be
+ * considered the U matrix. ParIlu can be directly used, since it orders the
+ * factors in the correct way.
+ *
+ * @note When providing a gko::Composition, the first matrix must be the lower
+ *       matrix ($L$), and the second matrix must be the upper matrix ($U$).
+ *       If they are swapped, solving might crash or return the wrong result.
+ *
+ * @note Do not use symmetric solvers (like CG) for L or U solvers since both
+ *       matrices (L and U) are, by design, not symmetric.
+ *
+ * @note This class is not thread safe (even a const object is not) because it
+ *       uses an internal cache to accelerate multiple (sequential) applies.
+ *       Using it in parallel can lead to segmentation faults, wrong results
+ *       and other unwanted behavior.
+ *
+ * @tparam LSolverType  type of the solver used for the L matrix.
+ *                      Defaults to solver::LowerTrs
+ * @tparam USolverType  type of the solver used for the U matrix
+ *                      Defaults to solver::UpperTrs
+ * @tparam ReverseApply  default behavior (ReverseApply = false) is first to
+ *                       solve with L (Ly = b) and then with U (Ux = y).
+ *                       When set to true, it will solve first with U, and then
+ *                       with L.
+ * @tparam IndexTypeParIlu  Type of the indices when ParIlu is used to generate
+ *                          both L and U factors. Irrelevant otherwise.
+ *
+ * @ingroup precond
+ * @ingroup LinOp
+ */
+template <typename LSolverType = solver::LowerTrs<>, typename IndexType = int32>
+class Ic : public EnableLinOp<Ic<LSolverType, IndexType>>, public Transposable {
+    friend class EnableLinOp<Ic>;
+    friend class EnablePolymorphicObject<Ic, LinOp>;
+
+public:
+    static_assert(
+        std::is_same<typename LSolverType::transposed_type::transposed_type,
+                     LSolverType>::value,
+        "LSolverType::transposed_type must be symmetric");
+    using value_type = typename LSolverType::value_type;
+    using l_solver_type = LSolverType;
+    using lh_solver_type = typename LSolverType::transposed_type;
+    using index_type = IndexType;
+    using transposed_type = Ic<LSolverType, IndexType>;
+
+    GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
+    {
+        /**
+         * Factory for the L solver
+         */
+        std::shared_ptr<typename l_solver_type::Factory>
+            GKO_FACTORY_PARAMETER_SCALAR(l_solver_factory, nullptr);
+
+        /**
+         * Factory for the factorization
+         */
+        std::shared_ptr<LinOpFactory> GKO_FACTORY_PARAMETER_SCALAR(
+            factorization_factory, nullptr);
+    };
+
+    GKO_ENABLE_LIN_OP_FACTORY(Ic, parameters, Factory);
+    GKO_ENABLE_BUILD_METHOD(Factory);
+
+    /**
+     * Returns the solver which is used for the provided L matrix.
+     *
+     * @returns  the solver which is used for the provided L matrix
+     */
+    std::shared_ptr<const l_solver_type> get_l_solver() const
+    {
+        return l_solver_;
+    }
+
+    /**
+     * Returns the solver which is used for the L^H matrix.
+     *
+     * @returns  the solver which is used for the L^H matrix
+     */
+    std::shared_ptr<const lh_solver_type> get_lh_solver() const
+    {
+        return lh_solver_;
+    }
+
+    std::unique_ptr<LinOp> transpose() const override
+    {
+        std::unique_ptr<transposed_type> transposed{
+            new transposed_type{this->get_executor()}};
+        transposed->set_size(gko::transpose(this->get_size()));
+        transposed->l_solver_ =
+            share(as<typename lh_solver_type::transposed_type>(
+                this->get_lh_solver()->transpose()));
+        transposed->lh_solver_ =
+            share(as<typename l_solver_type::transposed_type>(
+                this->get_l_solver()->transpose()));
+
+        return std::move(transposed);
+    }
+
+    std::unique_ptr<LinOp> conj_transpose() const override
+    {
+        std::unique_ptr<transposed_type> transposed{
+            new transposed_type{this->get_executor()}};
+        transposed->set_size(gko::transpose(this->get_size()));
+        transposed->l_solver_ =
+            share(as<typename lh_solver_type::transposed_type>(
+                this->get_lh_solver()->conj_transpose()));
+        transposed->lh_solver_ =
+            share(as<typename l_solver_type::transposed_type>(
+                this->get_l_solver()->conj_transpose()));
+
+        return std::move(transposed);
+    }
+
+protected:
+    void apply_impl(const LinOp *b, LinOp *x) const override
+    {
+        set_cache_to(b);
+        l_solver_->apply(b, cache_.intermediate.get());
+        x->copy_from(cache_.intermediate.get());
+        lh_solver_->apply(cache_.intermediate.get(), x);
+    }
+
+    void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
+                    LinOp *x) const override
+    {
+        set_cache_to(b);
+        l_solver_->apply(b, cache_.intermediate.get());
+        lh_solver_->apply(alpha, cache_.intermediate.get(), beta, x);
+    }
+
+    explicit Ic(std::shared_ptr<const Executor> exec)
+        : EnableLinOp<Ic>(std::move(exec))
+    {}
+
+    explicit Ic(const Factory *factory, std::shared_ptr<const LinOp> lin_op)
+        : EnableLinOp<Ic>(factory->get_executor(), lin_op->get_size()),
+          parameters_{factory->get_parameters()}
+    {
+        auto comp =
+            std::dynamic_pointer_cast<const Composition<value_type>>(lin_op);
+        std::shared_ptr<const LinOp> l_factor;
+
+        // build factorization if we weren't passed a composition
+        if (!comp) {
+            auto exec = lin_op->get_executor();
+            if (!parameters_.factorization_factory) {
+                parameters_.factorization_factory =
+                    factorization::ParIc<value_type, index_type>::build()
+                        .with_both_factors(false)
+                        .on(exec);
+            }
+            auto fact = std::shared_ptr<const LinOp>(
+                parameters_.factorization_factory->generate(lin_op));
+            // ensure that the result is a composition
+            comp =
+                std::dynamic_pointer_cast<const Composition<value_type>>(fact);
+            if (!comp) {
+                GKO_NOT_SUPPORTED(comp);
+            }
+        }
+        // comp must contain one or two factors
+        if (comp->get_operators().size() > 2 || comp->get_operators().empty()) {
+            GKO_NOT_SUPPORTED(comp);
+        }
+        l_factor = comp->get_operators()[0];
+        GKO_ASSERT_IS_SQUARE_MATRIX(l_factor);
+
+        auto exec = this->get_executor();
+
+        // If no factories are provided, generate default ones
+        if (!parameters_.l_solver_factory) {
+            l_solver_ = generate_default_solver<l_solver_type>(exec, l_factor);
+            // If comp contains both factors: use the transposed factor to avoid
+            // transposing twice
+            if (comp->get_operators().size() == 2) {
+                auto lh_factor = comp->get_operators()[1];
+                GKO_ASSERT_EQUAL_DIMENSIONS(l_factor, lh_factor);
+                lh_solver_ = as<lh_solver_type>(l_solver_->conj_transpose());
+            } else {
+                lh_solver_ = as<lh_solver_type>(l_solver_->conj_transpose());
+            }
+        } else {
+            l_solver_ = parameters_.l_solver_factory->generate(l_factor);
+        }
+    }
+
+    /**
+     * Prepares the intermediate vector for the solve by creating it and
+     * by copying the values from `b`, so `b` acts as the initial guess.
+     *
+     * @param b  Right hand side of the first solve. Also acts as the
+     * initial guess, meaning the intermediate value will be a copy of b
+     */
+    void set_cache_to(const LinOp *b) const
+    {
+        dim<2> expected_size =
+            dim<2>{l_solver_->get_size()[0], b->get_size()[1]};
+        if (cache_.intermediate == nullptr) {
+            cache_.intermediate =
+                matrix::Dense<value_type>::create(this->get_executor());
+        }
+        // Use b as the initial guess for the first triangular solve
+        cache_.intermediate->copy_from(b);
+    }
+
+    /**
+     * @internal  Looks at the build() method to determine the type of the
+     *            factory.
+     */
+    template <typename T>
+    using factory_type_t = decltype(T::build());
+
+    // Parameter type of function `with_criteria`.
+    using with_criteria_param_type =
+        std::shared_ptr<const stop::CriterionFactory>;
+
+    /**
+     * Helper structure to test if the Factory of SolverType has a function
+     * `with_criteria`.
+     *
+     * Contains a constexpr boolean `value`, which is true if the Factory
+     * class of SolverType has a `with_criteria`, and false otherwise.
+     *
+     * @tparam SolverType   Solver to test if its factory has a
+     * with_criteria function.
+     *
+     */
+    template <typename SolverType, typename = void>
+    struct has_with_criteria : std::false_type {};
+
+    /**
+     * @copydoc has_with_criteria
+     *
+     * @internal  The second template parameter (which uses SFINAE) must
+     * match the default value of the general case in order to be accepted
+     *            as a specialization, which is why `xstd::void_t` is used.
+     */
+    template <typename SolverType>
+    struct has_with_criteria<
+        SolverType,
+        xstd::void_t<decltype(std::declval<factory_type_t<SolverType>>()
+                                  .with_criteria(with_criteria_param_type()))>>
+        : std::true_type {};
+
+
+    /**
+     * Generates a default solver of type SolverType.
+     *
+     * Also checks whether SolverType can be assigned a criteria, and if it
+     * can, it is assigned default values which should be well suited for a
+     * preconditioner.
+     */
+    template <typename SolverType>
+    static std::enable_if_t<has_with_criteria<SolverType>::value,
+                            std::unique_ptr<SolverType>>
+    generate_default_solver(const std::shared_ptr<const Executor> &exec,
+                            const std::shared_ptr<const LinOp> &mtx)
+    {
+        constexpr gko::remove_complex<value_type> default_reduce_residual{1e-4};
+        const unsigned int default_max_iters{
+            static_cast<unsigned int>(mtx->get_size()[0])};
+
+        return SolverType::build()
+            .with_criteria(gko::stop::Iteration::build()
+                               .with_max_iters(default_max_iters)
+                               .on(exec),
+                           gko::stop::ResidualNormReduction<value_type>::build()
+                               .with_reduction_factor(default_reduce_residual)
+                               .on(exec))
+            .on(exec)
+            ->generate(mtx);
+    }
+
+    /**
+     * @copydoc generate_default_solver
+     */
+    template <typename SolverType>
+    static std::enable_if_t<!has_with_criteria<SolverType>::value,
+                            std::unique_ptr<SolverType>>
+    generate_default_solver(const std::shared_ptr<const Executor> &exec,
+                            const std::shared_ptr<const LinOp> &mtx)
+    {
+        return SolverType::build().on(exec)->generate(mtx);
+    }
+
+private:
+    std::shared_ptr<const l_solver_type> l_solver_{};
+    std::shared_ptr<const lh_solver_type> lh_solver_{};
+    /**
+     * Manages a vector as a cache, so there is no need to allocate one
+     * every time an intermediate vector is required. Copying an instance
+     * will only yield an empty object since copying the cached vector would
+     * not make sense.
+     *
+     * @internal  The struct is present so the whole class can be copyable
+     *            (could also be done with writing `operator=` and copy
+     *            constructor of the enclosing class by hand)
+     */
+    mutable struct cache_struct {
+        cache_struct() = default;
+        ~cache_struct() = default;
+        cache_struct(const cache_struct &) {}
+        cache_struct(cache_struct &&) {}
+        cache_struct &operator=(const cache_struct &) { return *this; }
+        cache_struct &operator=(cache_struct &&) { return *this; }
+        std::unique_ptr<LinOp> intermediate{};
+    } cache_;
+};  // namespace preconditioner
+
+
+}  // namespace preconditioner
+}  // namespace gko
+
+
+#endif  // GKO_PUBLIC_CORE_PRECONDITIONER_IC_HPP_
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 42c535cf4e4..7fc929dfa93 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -62,7 +62,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/utils_helper.hpp>
 #include <ginkgo/core/base/version.hpp>
 
+#include <ginkgo/core/factorization/ic.hpp>
 #include <ginkgo/core/factorization/ilu.hpp>
+#include <ginkgo/core/factorization/par_ic.hpp>
 #include <ginkgo/core/factorization/par_ict.hpp>
 #include <ginkgo/core/factorization/par_ilu.hpp>
 #include <ginkgo/core/factorization/par_ilut.hpp>
@@ -84,6 +86,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sellp.hpp>
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
+#include <ginkgo/core/preconditioner/ic.hpp>
 #include <ginkgo/core/preconditioner/ilu.hpp>
 #include <ginkgo/core/preconditioner/isai.hpp>
 #include <ginkgo/core/preconditioner/jacobi.hpp>
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index b08c691410a..bedcfaa4890 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -9,7 +9,9 @@ target_sources(ginkgo_omp
     components/precision_conversion.cpp
     components/prefix_sum.cpp
     factorization/factorization_kernels.cpp
+    factorization/ic_kernels.cpp
     factorization/ilu_kernels.cpp
+    factorization/par_ic_kernels.cpp
     factorization/par_ict_kernels.cpp
     factorization/par_ilu_kernels.cpp
     factorization/par_ilut_kernels.cpp
diff --git a/omp/factorization/ic_kernels.cpp b/omp/factorization/ic_kernels.cpp
new file mode 100644
index 00000000000..04120436bda
--- /dev/null
+++ b/omp/factorization/ic_kernels.cpp
@@ -0,0 +1,57 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/ic_kernels.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+/**
+ * @brief The ic factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace ic_factorization {
+
+
+template <typename ValueType, typename IndexType>
+void compute(std::shared_ptr<const DefaultExecutor> exec,
+             matrix::Csr<ValueType, IndexType> *m) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_IC_COMPUTE_KERNEL);
+
+
+}  // namespace ic_factorization
+}  // namespace omp
+}  // namespace kernels
+}  // namespace gko
diff --git a/omp/factorization/par_ic_kernels.cpp b/omp/factorization/par_ic_kernels.cpp
new file mode 100644
index 00000000000..78f6d8fd634
--- /dev/null
+++ b/omp/factorization/par_ic_kernels.cpp
@@ -0,0 +1,140 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/par_ic_kernels.hpp"
+
+
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+#include "core/base/utils.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+/**
+ * @brief The parallel IC factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace par_ic_factorization {
+
+
+template <typename ValueType, typename IndexType>
+void init_factor(std::shared_ptr<const DefaultExecutor> exec,
+                 matrix::Csr<ValueType, IndexType> *l)
+{
+    auto num_rows = l->get_size()[0];
+    auto l_row_ptrs = l->get_const_row_ptrs();
+    auto l_vals = l->get_values();
+
+#pragma omp parallel for
+    for (size_type row = 0; row < num_rows; ++row) {
+        auto l_nz = l_row_ptrs[row + 1] - 1;
+        auto diag = sqrt(l_vals[l_nz]);
+        if (is_finite(diag)) {
+            l_vals[l_nz] = diag;
+        } else {
+            l_vals[l_nz] = one<ValueType>();
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_IC_INIT_FACTOR_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
+                    size_type iterations,
+                    const matrix::Coo<ValueType, IndexType> *a_lower,
+                    matrix::Csr<ValueType, IndexType> *l)
+{
+    auto num_rows = a_lower->get_size()[0];
+    auto l_row_ptrs = l->get_const_row_ptrs();
+    auto l_col_idxs = l->get_const_col_idxs();
+    auto l_vals = l->get_values();
+    auto a_vals = a_lower->get_const_values();
+
+    for (size_type i = 0; i < iterations; ++i) {
+#pragma omp parallel for
+        for (size_type row = 0; row < num_rows; ++row) {
+            for (size_type l_nz = l_row_ptrs[row]; l_nz < l_row_ptrs[row + 1];
+                 ++l_nz) {
+                auto col = l_col_idxs[l_nz];
+                auto a_val = a_vals[l_nz];
+                // accumulate l(row,:) * l(col,:) without the last entry l(col,
+                // col)
+                ValueType sum{};
+                IndexType lh_nz{};
+                auto l_begin = l_row_ptrs[row];
+                auto l_end = l_row_ptrs[row + 1];
+                auto lh_begin = l_row_ptrs[col];
+                auto lh_end = l_row_ptrs[col + 1];
+                while (l_begin < l_end && lh_begin < lh_end) {
+                    auto l_col = l_col_idxs[l_begin];
+                    auto lh_row = l_col_idxs[lh_begin];
+                    if (l_col == lh_row && l_col < col) {
+                        sum += l_vals[l_begin] * conj(l_vals[lh_begin]);
+                    }
+                    if (lh_row == row) {
+                        lh_nz = lh_begin;
+                    }
+                    l_begin += (l_col <= lh_row);
+                    lh_begin += (lh_row <= l_col);
+                }
+                auto new_val = a_val - sum;
+                if (row == col) {
+                    new_val = sqrt(new_val);
+                } else {
+                    auto diag = l_vals[l_row_ptrs[col + 1] - 1];
+                    new_val = new_val / diag;
+                }
+                if (is_finite(new_val)) {
+                    l_vals[l_nz] = new_val;
+                }
+            }
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_IC_COMPUTE_FACTOR_KERNEL);
+
+
+}  // namespace par_ic_factorization
+}  // namespace omp
+}  // namespace kernels
+}  // namespace gko
diff --git a/omp/factorization/par_ict_kernels.cpp b/omp/factorization/par_ict_kernels.cpp
index 331c384b069..fed5243aab7 100644
--- a/omp/factorization/par_ict_kernels.cpp
+++ b/omp/factorization/par_ict_kernels.cpp
@@ -91,22 +91,22 @@ void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
             auto a_val = has_a ? a_vals[a_nz] : zero<ValueType>();
             // accumulate l(row,:) * l(col,:) without the last entry l(col, col)
             ValueType sum{};
-            IndexType lt_nz{};
+            IndexType lh_nz{};
             auto l_begin = l_row_ptrs[row];
             auto l_end = l_row_ptrs[row + 1];
-            auto lt_begin = l_row_ptrs[col];
-            auto lt_end = l_row_ptrs[col + 1];
-            while (l_begin < l_end && lt_begin < lt_end) {
+            auto lh_begin = l_row_ptrs[col];
+            auto lh_end = l_row_ptrs[col + 1];
+            while (l_begin < l_end && lh_begin < lh_end) {
                 auto l_col = l_col_idxs[l_begin];
-                auto lt_row = l_col_idxs[lt_begin];
-                if (l_col == lt_row && l_col < col) {
-                    sum += l_vals[l_begin] * l_vals[lt_begin];
+                auto lh_row = l_col_idxs[lh_begin];
+                if (l_col == lh_row && l_col < col) {
+                    sum += l_vals[l_begin] * conj(l_vals[lh_begin]);
                 }
-                if (lt_row == row) {
-                    lt_nz = lt_begin;
+                if (lh_row == row) {
+                    lh_nz = lh_begin;
                 }
-                l_begin += (l_col <= lt_row);
-                lt_begin += (lt_row <= l_col);
+                l_begin += (l_col <= lh_row);
+                lh_begin += (lh_row <= l_col);
             }
             auto new_val = a_val - sum;
             if (row == col) {
@@ -128,7 +128,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
-                    const matrix::Csr<ValueType, IndexType> *llt,
+                    const matrix::Csr<ValueType, IndexType> *llh,
                     const matrix::Csr<ValueType, IndexType> *a,
                     const matrix::Csr<ValueType, IndexType> *l,
                     matrix::Csr<ValueType, IndexType> *l_new)
@@ -141,7 +141,7 @@ void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
     constexpr auto sentinel = std::numeric_limits<IndexType>::max();
     // count nnz
     abstract_spgeam(
-        a, llt, [](IndexType) { return IndexType{}; },
+        a, llh, [](IndexType) { return IndexType{}; },
         [](IndexType row, IndexType col, ValueType, ValueType, IndexType &nnz) {
             nnz += col <= row;
         },
@@ -164,7 +164,7 @@ void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
         IndexType l_old_end;
     };
     abstract_spgeam(
-        a, llt,
+        a, llh,
         [&](IndexType row) {
             row_state state{};
             state.l_new_nz = l_new_row_ptrs[row];
@@ -172,9 +172,9 @@ void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
             state.l_old_end = l_row_ptrs[row + 1];
             return state;
         },
-        [&](IndexType row, IndexType col, ValueType a_val, ValueType llt_val,
+        [&](IndexType row, IndexType col, ValueType a_val, ValueType llh_val,
             row_state &state) {
-            auto r_val = a_val - llt_val;
+            auto r_val = a_val - llh_val;
             // load matching entry of L
             auto l_col = checked_load(l_col_idxs, state.l_old_begin,
                                       state.l_old_end, sentinel);
diff --git a/omp/test/factorization/CMakeLists.txt b/omp/test/factorization/CMakeLists.txt
index b52c2d938d7..d3d66296318 100644
--- a/omp/test/factorization/CMakeLists.txt
+++ b/omp/test/factorization/CMakeLists.txt
@@ -1,3 +1,4 @@
+ginkgo_create_test(par_ic_kernels)
 ginkgo_create_test(par_ict_kernels)
 ginkgo_create_test(par_ilu_kernels)
 ginkgo_create_test(par_ilut_kernels)
diff --git a/omp/test/factorization/par_ic_kernels.cpp b/omp/test/factorization/par_ic_kernels.cpp
new file mode 100644
index 00000000000..4192d5d2d42
--- /dev/null
+++ b/omp/test/factorization/par_ic_kernels.cpp
@@ -0,0 +1,177 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/par_ic_kernels.hpp"
+
+
+#include <algorithm>
+#include <fstream>
+#include <memory>
+#include <random>
+#include <string>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/factorization/factorization_kernels.hpp"
+#include "core/matrix/csr_builder.hpp"
+#include "core/matrix/csr_kernels.hpp"
+#include "core/test/utils.hpp"
+#include "matrices/config.hpp"
+
+
+namespace {
+
+
+template <typename ValueIndexType>
+class ParIc : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using Coo = gko::matrix::Coo<value_type, index_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+
+    ParIc()
+        : mtx_size(532, 532),
+          rand_engine(35461),
+          ref(gko::ReferenceExecutor::create()),
+          omp(gko::OmpExecutor::create())
+    {
+        mtx_l = gko::test::generate_random_lower_triangular_matrix<Csr>(
+            mtx_size[0], mtx_size[0], false,
+            std::uniform_int_distribution<index_type>(10, mtx_size[0]),
+            std::normal_distribution<gko::remove_complex<value_type>>(0, 10.0),
+            rand_engine, ref);
+
+        dmtx_ani = Csr::create(omp);
+        dmtx_l_ani = Csr::create(omp);
+        dmtx_l_ani_init = Csr::create(omp);
+        dmtx_l = Csr::create(omp);
+        dmtx_l->copy_from(lend(mtx_l));
+    }
+
+    void SetUp()
+    {
+        std::string file_name(gko::matrices::location_ani4_mtx);
+        auto input_file = std::ifstream(file_name, std::ios::in);
+        if (!input_file) {
+            FAIL() << "Could not find the file \"" << file_name
+                   << "\", which is required for this test.\n";
+        }
+        mtx_ani = gko::read<Csr>(input_file, ref);
+        mtx_ani->sort_by_column_index();
+
+        {
+            mtx_l_ani = Csr::create(ref, mtx_ani->get_size());
+            gko::matrix::CsrBuilder<value_type, index_type> l_builder(
+                lend(mtx_l_ani));
+            gko::kernels::reference::factorization::initialize_row_ptrs_l(
+                ref, lend(mtx_ani), mtx_l_ani->get_row_ptrs());
+            auto l_nnz =
+                mtx_l_ani->get_const_row_ptrs()[mtx_ani->get_size()[0]];
+            l_builder.get_col_idx_array().resize_and_reset(l_nnz);
+            l_builder.get_value_array().resize_and_reset(l_nnz);
+            gko::kernels::reference::factorization::initialize_l(
+                ref, lend(mtx_ani), lend(mtx_l_ani), false);
+            mtx_l_ani_init = Csr::create(ref);
+            mtx_l_ani_init->copy_from(lend(mtx_l_ani));
+            gko::kernels::reference::par_ic_factorization::init_factor(
+                ref, lend(mtx_l_ani_init));
+        }
+        dmtx_ani->copy_from(lend(mtx_ani));
+        dmtx_l_ani->copy_from(lend(mtx_l_ani));
+        dmtx_l_ani_init->copy_from(lend(mtx_l_ani_init));
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<gko::OmpExecutor> omp;
+
+    const gko::dim<2> mtx_size;
+    std::default_random_engine rand_engine;
+
+    std::unique_ptr<Csr> mtx_l;
+    std::unique_ptr<Csr> mtx_ani;
+    std::unique_ptr<Csr> mtx_l_ani;
+    std::unique_ptr<Csr> mtx_l_ani_init;
+
+    std::unique_ptr<Csr> dmtx_l;
+    std::unique_ptr<Csr> dmtx_ani;
+    std::unique_ptr<Csr> dmtx_l_ani;
+    std::unique_ptr<Csr> dmtx_l_ani_init;
+};
+
+TYPED_TEST_SUITE(ParIc, gko::test::ValueIndexTypes);
+
+
+TYPED_TEST(ParIc, KernelInitFactorIsEquivalentToRef)
+{
+    using value_type = typename TestFixture::value_type;
+
+    gko::kernels::reference::par_ic_factorization::init_factor(
+        this->ref, lend(this->mtx_l));
+    gko::kernels::omp::par_ic_factorization::init_factor(this->omp,
+                                                         lend(this->dmtx_l));
+
+    GKO_ASSERT_MTX_NEAR(this->mtx_l, this->dmtx_l, r<value_type>::value);
+}
+
+
+TYPED_TEST(ParIc, KernelComputeFactorIsEquivalentToRef)
+{
+    using Csr = typename TestFixture::Csr;
+    using Coo = typename TestFixture::Coo;
+    auto square_size = this->mtx_ani->get_size();
+    auto mtx_l_coo = Coo::create(this->ref, square_size);
+    this->mtx_l_ani->convert_to(lend(mtx_l_coo));
+    auto dmtx_l_coo = Coo::create(this->omp, square_size);
+    dmtx_l_coo->copy_from(lend(mtx_l_coo));
+
+    gko::kernels::reference::par_ic_factorization::compute_factor(
+        this->ref, 1, lend(mtx_l_coo), lend(this->mtx_l_ani_init));
+    gko::kernels::omp::par_ic_factorization::compute_factor(
+        this->omp, 20, lend(dmtx_l_coo), lend(this->dmtx_l_ani_init));
+
+    GKO_ASSERT_MTX_NEAR(this->mtx_l_ani, this->dmtx_l_ani, 1e-2);
+}
+
+
+}  // namespace
diff --git a/omp/test/factorization/par_ict_kernels.cpp b/omp/test/factorization/par_ict_kernels.cpp
index 0e3c50e3ea7..43e13fee7c9 100644
--- a/omp/test/factorization/par_ict_kernels.cpp
+++ b/omp/test/factorization/par_ict_kernels.cpp
@@ -149,18 +149,18 @@ TYPED_TEST(ParIct, KernelAddCandidatesIsEquivalentToRef)
 {
     using Csr = typename TestFixture::Csr;
     using value_type = typename TestFixture::value_type;
-    auto mtx_llt = Csr::create(this->ref, this->mtx_size);
-    this->mtx_l->apply(lend(this->mtx_l->transpose()), lend(mtx_llt));
-    auto dmtx_llt = Csr::create(this->omp, this->mtx_size);
-    dmtx_llt->copy_from(lend(mtx_llt));
+    auto mtx_llh = Csr::create(this->ref, this->mtx_size);
+    this->mtx_l->apply(lend(this->mtx_l->transpose()), lend(mtx_llh));
+    auto dmtx_llh = Csr::create(this->omp, this->mtx_size);
+    dmtx_llh->copy_from(lend(mtx_llh));
     auto res_mtx_l = Csr::create(this->ref, this->mtx_size);
     auto dres_mtx_l = Csr::create(this->omp, this->mtx_size);
 
     gko::kernels::reference::par_ict_factorization::add_candidates(
-        this->ref, lend(mtx_llt), lend(this->mtx), lend(this->mtx_l),
+        this->ref, lend(mtx_llh), lend(this->mtx), lend(this->mtx_l),
         lend(res_mtx_l));
     gko::kernels::omp::par_ict_factorization::add_candidates(
-        this->omp, lend(dmtx_llt), lend(this->dmtx), lend(this->dmtx_l),
+        this->omp, lend(dmtx_llh), lend(this->dmtx), lend(this->dmtx_l),
         lend(dres_mtx_l));
 
     GKO_ASSERT_MTX_EQ_SPARSITY(res_mtx_l, dres_mtx_l);
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index b1bf9cfed55..2328e5b5763 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -7,7 +7,9 @@ target_sources(ginkgo_reference
     components/precision_conversion.cpp
     components/prefix_sum.cpp
     factorization/factorization_kernels.cpp
+    factorization/ic_kernels.cpp
     factorization/ilu_kernels.cpp
+    factorization/par_ic_kernels.cpp
     factorization/par_ict_kernels.cpp
     factorization/par_ilu_kernels.cpp
     factorization/par_ilut_kernels.cpp
diff --git a/reference/factorization/ic_kernels.cpp b/reference/factorization/ic_kernels.cpp
new file mode 100644
index 00000000000..28b433bf331
--- /dev/null
+++ b/reference/factorization/ic_kernels.cpp
@@ -0,0 +1,57 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/ic_kernels.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace reference {
+/**
+ * @brief The ic factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace ic_factorization {
+
+
+template <typename ValueType, typename IndexType>
+void compute(std::shared_ptr<const DefaultExecutor> exec,
+             matrix::Csr<ValueType, IndexType> *m) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_IC_COMPUTE_KERNEL);
+
+
+}  // namespace ic_factorization
+}  // namespace reference
+}  // namespace kernels
+}  // namespace gko
diff --git a/reference/factorization/par_ic_kernels.cpp b/reference/factorization/par_ic_kernels.cpp
new file mode 100644
index 00000000000..eeee9e46b8d
--- /dev/null
+++ b/reference/factorization/par_ic_kernels.cpp
@@ -0,0 +1,140 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/factorization/par_ic_kernels.hpp"
+
+
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+#include "core/base/utils.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace reference {
+/**
+ * @brief The parallel ic factorization namespace.
+ *
+ * @ingroup factor
+ */
+namespace par_ic_factorization {
+
+
+template <typename ValueType, typename IndexType>
+void init_factor(std::shared_ptr<const DefaultExecutor> exec,
+                 matrix::Csr<ValueType, IndexType> *l)
+{
+    auto num_rows = l->get_size()[0];
+    auto l_row_ptrs = l->get_const_row_ptrs();
+    auto l_col_idxs = l->get_const_col_idxs();
+    auto l_vals = l->get_values();
+
+    for (size_type row = 0; row < num_rows; ++row) {
+        for (size_type l_nz = l_row_ptrs[row]; l_nz < l_row_ptrs[row + 1];
+             ++l_nz) {
+            if (l_col_idxs[l_nz] == row) {
+                auto diag = sqrt(l_vals[l_nz]);
+                if (is_finite(diag)) {
+                    l_vals[l_nz] = diag;
+                } else {
+                    l_vals[l_nz] = one<ValueType>();
+                }
+            }
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_IC_INIT_FACTOR_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
+                    size_type /* num_iterations */,
+                    const matrix::Coo<ValueType, IndexType> *a_lower,
+                    matrix::Csr<ValueType, IndexType> *l)
+{
+    auto num_rows = a_lower->get_size()[0];
+    auto l_row_ptrs = l->get_const_row_ptrs();
+    auto l_col_idxs = l->get_const_col_idxs();
+    auto l_vals = l->get_values();
+    auto a_vals = a_lower->get_const_values();
+
+    for (size_type row = 0; row < num_rows; ++row) {
+        for (size_type l_nz = l_row_ptrs[row]; l_nz < l_row_ptrs[row + 1];
+             ++l_nz) {
+            auto col = l_col_idxs[l_nz];
+            auto a_val = a_vals[l_nz];
+            // accumulate l(row,:) * l(col,:) without the last entry l(col, col)
+            ValueType sum{};
+            IndexType lh_nz{};
+            auto l_begin = l_row_ptrs[row];
+            auto l_end = l_row_ptrs[row + 1];
+            auto lh_begin = l_row_ptrs[col];
+            auto lh_end = l_row_ptrs[col + 1];
+            while (l_begin < l_end && lh_begin < lh_end) {
+                auto l_col = l_col_idxs[l_begin];
+                auto lh_row = l_col_idxs[lh_begin];
+                if (l_col == lh_row && l_col < col) {
+                    sum += l_vals[l_begin] * conj(l_vals[lh_begin]);
+                }
+                if (lh_row == row) {
+                    lh_nz = lh_begin;
+                }
+                l_begin += (l_col <= lh_row);
+                lh_begin += (lh_row <= l_col);
+            }
+            auto new_val = a_val - sum;
+            if (row == col) {
+                new_val = sqrt(new_val);
+            } else {
+                auto diag = l_vals[l_row_ptrs[col + 1] - 1];
+                new_val = new_val / diag;
+            }
+            if (is_finite(new_val)) {
+                l_vals[l_nz] = new_val;
+            }
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_PAR_IC_COMPUTE_FACTOR_KERNEL);
+
+
+}  // namespace par_ic_factorization
+}  // namespace reference
+}  // namespace kernels
+}  // namespace gko
diff --git a/reference/factorization/par_ict_kernels.cpp b/reference/factorization/par_ict_kernels.cpp
index 038da5238da..399bd264b7a 100644
--- a/reference/factorization/par_ict_kernels.cpp
+++ b/reference/factorization/par_ict_kernels.cpp
@@ -89,22 +89,22 @@ void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
             auto a_val = has_a ? a_vals[a_nz] : zero<ValueType>();
             // accumulate l(row,:) * l(col,:) without the last entry l(col, col)
             ValueType sum{};
-            IndexType lt_nz{};
+            IndexType lh_nz{};
             auto l_begin = l_row_ptrs[row];
             auto l_end = l_row_ptrs[row + 1];
-            auto lt_begin = l_row_ptrs[col];
-            auto lt_end = l_row_ptrs[col + 1];
-            while (l_begin < l_end && lt_begin < lt_end) {
+            auto lh_begin = l_row_ptrs[col];
+            auto lh_end = l_row_ptrs[col + 1];
+            while (l_begin < l_end && lh_begin < lh_end) {
                 auto l_col = l_col_idxs[l_begin];
-                auto lt_row = l_col_idxs[lt_begin];
-                if (l_col == lt_row && l_col < col) {
-                    sum += l_vals[l_begin] * l_vals[lt_begin];
+                auto lh_row = l_col_idxs[lh_begin];
+                if (l_col == lh_row && l_col < col) {
+                    sum += l_vals[l_begin] * conj(l_vals[lh_begin]);
                 }
-                if (lt_row == row) {
-                    lt_nz = lt_begin;
+                if (lh_row == row) {
+                    lh_nz = lh_begin;
                 }
-                l_begin += (l_col <= lt_row);
-                lt_begin += (lt_row <= l_col);
+                l_begin += (l_col <= lh_row);
+                lh_begin += (lh_row <= l_col);
             }
             auto new_val = a_val - sum;
             if (row == col) {
@@ -126,7 +126,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
-                    const matrix::Csr<ValueType, IndexType> *llt,
+                    const matrix::Csr<ValueType, IndexType> *llh,
                     const matrix::Csr<ValueType, IndexType> *a,
                     const matrix::Csr<ValueType, IndexType> *l,
                     matrix::Csr<ValueType, IndexType> *l_new)
@@ -140,7 +140,7 @@ void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
     // count nnz
     IndexType l_nnz{};
     abstract_spgeam(
-        a, llt,
+        a, llh,
         [&](IndexType row) {
             l_new_row_ptrs[row] = l_nnz;
             return 0;
@@ -165,7 +165,7 @@ void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
         IndexType l_old_end;
     };
     abstract_spgeam(
-        a, llt,
+        a, llh,
         [&](IndexType row) {
             row_state state{};
             state.l_new_nz = l_new_row_ptrs[row];
@@ -173,9 +173,9 @@ void add_candidates(std::shared_ptr<const DefaultExecutor> exec,
             state.l_old_end = l_row_ptrs[row + 1];
             return state;
         },
-        [&](IndexType row, IndexType col, ValueType a_val, ValueType llt_val,
+        [&](IndexType row, IndexType col, ValueType a_val, ValueType llh_val,
             row_state &state) {
-            auto r_val = a_val - llt_val;
+            auto r_val = a_val - llh_val;
             // load matching entry of L
             auto l_col = checked_load(l_col_idxs, state.l_old_begin,
                                       state.l_old_end, sentinel);
diff --git a/reference/test/factorization/CMakeLists.txt b/reference/test/factorization/CMakeLists.txt
index b52c2d938d7..d3d66296318 100644
--- a/reference/test/factorization/CMakeLists.txt
+++ b/reference/test/factorization/CMakeLists.txt
@@ -1,3 +1,4 @@
+ginkgo_create_test(par_ic_kernels)
 ginkgo_create_test(par_ict_kernels)
 ginkgo_create_test(par_ilu_kernels)
 ginkgo_create_test(par_ilut_kernels)
diff --git a/reference/test/factorization/par_ic_kernels.cpp b/reference/test/factorization/par_ic_kernels.cpp
new file mode 100644
index 00000000000..1548a8b0a5f
--- /dev/null
+++ b/reference/test/factorization/par_ic_kernels.cpp
@@ -0,0 +1,278 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/factorization/par_ic.hpp>
+
+
+#include <algorithm>
+#include <memory>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+#include "core/factorization/factorization_kernels.hpp"
+#include "core/factorization/par_ic_kernels.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class DummyLinOp : public gko::EnableLinOp<DummyLinOp>,
+                   public gko::EnableCreateMethod<DummyLinOp> {
+public:
+    DummyLinOp(std::shared_ptr<const gko::Executor> exec,
+               gko::dim<2> size = gko::dim<2>{})
+        : EnableLinOp<DummyLinOp>(exec, size)
+    {}
+
+protected:
+    void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override {}
+
+    void apply_impl(const gko::LinOp *alpha, const gko::LinOp *b,
+                    const gko::LinOp *beta, gko::LinOp *x) const override
+    {}
+};
+
+
+template <typename ValueIndexType>
+class ParIc : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using factorization_type =
+        gko::factorization::ParIc<value_type, index_type>;
+    using Coo = gko::matrix::Coo<value_type, index_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+
+    ParIc()
+        : ref(gko::ReferenceExecutor::create()),
+          exec(std::static_pointer_cast<const gko::Executor>(ref)),
+          identity(gko::initialize<Csr>(
+              {{1., 0., 0.}, {0., 1., 0.}, {0., 0., 1.}}, ref)),
+          banded(gko::initialize<Csr>(
+              {{1., 1., 0.}, {1., 2., 1.}, {0., 1., 2.}}, ref)),
+          banded_l_expect(gko::initialize<Csr>(
+              {{1., 0., 0.}, {1., 1., 0.}, {0., 1., 1.}}, ref)),
+          mtx_system(gko::initialize<Csr>({{9., 0., -6., 3.},
+                                           {0., 36., 18., 24.},
+                                           {-6., 18., 17., 14.},
+                                           {-3., 24., 14., 18.}},
+                                          ref)),
+          mtx_l_system(gko::initialize<Csr>({{9., 0., 0., 0.},
+                                             {0., 36., 0., 0.},
+                                             {-6., 18., 17., 0.},
+                                             {-3., 24., 14., 18.}},
+                                            ref)),
+          mtx_l_system_coo(Coo::create(exec)),
+          mtx_l_init_expect(gko::initialize<Csr>(
+              {{3., 0., 0., 0.},
+               {0., 6., 0., 0.},
+               {-6., 18., static_cast<value_type>(sqrt(17.)), 0.},
+               {-3., 24., 14., static_cast<value_type>(sqrt(18.))}},
+              ref)),
+          mtx_l_it_expect(gko::initialize<Csr>({{3., 0., 0., 0.},
+                                                {0., 6., 0., 0.},
+                                                {-2., 3., 2., 0.},
+                                                {-1., 4., 0., 1.}},
+                                               ref)),
+          fact_fact(factorization_type::build().on(exec)),
+          tol{r<value_type>::value}
+    {
+        mtx_l_system->convert_to(gko::lend(mtx_l_system_coo));
+    }
+
+    std::shared_ptr<const gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<Csr> identity;
+    std::shared_ptr<Csr> banded;
+    std::shared_ptr<Csr> banded_l_expect;
+    std::shared_ptr<Csr> mtx_system;
+    std::unique_ptr<Csr> mtx_l_system;
+    std::unique_ptr<Coo> mtx_l_system_coo;
+    std::unique_ptr<Csr> mtx_l_init_expect;
+    std::unique_ptr<Csr> mtx_l_it_expect;
+    std::unique_ptr<typename factorization_type::Factory> fact_fact;
+    gko::remove_complex<value_type> tol;
+};
+
+TYPED_TEST_SUITE(ParIc, gko::test::ValueIndexTypes);
+
+
+TYPED_TEST(ParIc, KernelCompute)
+{
+    gko::kernels::reference::par_ic_factorization::compute_factor(
+        this->ref, 1, this->mtx_l_system_coo.get(), this->mtx_l_system.get());
+
+    GKO_ASSERT_MTX_NEAR(this->mtx_l_system, this->mtx_l_it_expect, this->tol);
+}
+
+
+TYPED_TEST(ParIc, KernelInit)
+{
+    gko::kernels::reference::par_ic_factorization::init_factor(
+        this->ref, this->mtx_l_system.get());
+
+    GKO_ASSERT_MTX_NEAR(this->mtx_l_system, this->mtx_l_init_expect, this->tol);
+}
+
+
+TYPED_TEST(ParIc, ThrowNotSupportedForWrongLinOp)
+{
+    auto lin_op = DummyLinOp::create(this->ref);
+
+    ASSERT_THROW(this->fact_fact->generate(gko::share(lin_op)),
+                 gko::NotSupported);
+}
+
+
+TYPED_TEST(ParIc, ThrowDimensionMismatch)
+{
+    using Csr = typename TestFixture::Csr;
+    auto matrix = Csr::create(this->ref, gko::dim<2>{2, 3}, 4);
+
+    ASSERT_THROW(this->fact_fact->generate(gko::share(matrix)),
+                 gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(ParIc, SetStrategy)
+{
+    using Csr = typename TestFixture::Csr;
+    using factorization_type = typename TestFixture::factorization_type;
+    auto l_strategy = std::make_shared<typename Csr::merge_path>();
+
+    auto factory =
+        factorization_type::build().with_l_strategy(l_strategy).on(this->ref);
+    auto fact = factory->generate(this->mtx_system);
+
+    ASSERT_EQ(factory->get_parameters().l_strategy, l_strategy);
+    ASSERT_EQ(fact->get_l_factor()->get_strategy()->get_name(),
+              l_strategy->get_name());
+    ASSERT_EQ(fact->get_lh_factor()->get_strategy()->get_name(),
+              l_strategy->get_name());
+}
+
+
+TYPED_TEST(ParIc, IsConsistentWithComposition)
+{
+    auto fact = this->fact_fact->generate(this->mtx_system);
+
+    auto lin_op_l_factor = gko::as<gko::LinOp>(fact->get_l_factor());
+    auto lin_op_lt_factor = gko::as<gko::LinOp>(fact->get_lh_factor());
+    auto first_operator = fact->get_operators()[0];
+    auto second_operator = fact->get_operators()[1];
+
+    ASSERT_EQ(lin_op_l_factor, first_operator);
+    ASSERT_EQ(lin_op_lt_factor, second_operator);
+}
+
+
+TYPED_TEST(ParIc, GenerateSingleFactor)
+{
+    using factorization_type = typename TestFixture::factorization_type;
+    using Csr = typename TestFixture::Csr;
+    auto factory =
+        factorization_type::build().with_both_factors(false).on(this->ref);
+    auto fact = factory->generate(this->mtx_system);
+
+    auto lin_op_l_factor = fact->get_l_factor();
+    auto lin_op_lt_factor = fact->get_lh_factor();
+    auto first_operator = gko::as<Csr>(fact->get_operators()[0]);
+    auto first_operator_h = gko::as<Csr>(first_operator->conj_transpose());
+
+    ASSERT_EQ(fact->get_operators().size(), 1);
+    ASSERT_EQ(lin_op_l_factor, first_operator);
+    GKO_ASSERT_MTX_NEAR(lin_op_lt_factor, first_operator_h, this->tol);
+}
+
+
+TYPED_TEST(ParIc, GenerateIdentity)
+{
+    auto fact = this->fact_fact->generate(this->identity);
+
+    GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->identity, this->tol);
+    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(), this->identity, this->tol);
+}
+
+
+TYPED_TEST(ParIc, GenerateDenseIdentity)
+{
+    using Dense = typename TestFixture::Dense;
+    auto dense_id = Dense::create(this->exec, this->identity->get_size());
+    this->identity->convert_to(dense_id.get());
+    auto fact = this->fact_fact->generate(gko::share(dense_id));
+
+    GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->identity, this->tol);
+    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(), this->identity, this->tol);
+}
+
+
+TYPED_TEST(ParIc, GenerateBanded)
+{
+    using factorization_type = typename TestFixture::factorization_type;
+    using Csr = typename TestFixture::Csr;
+    auto fact =
+        factorization_type::build().on(this->exec)->generate(this->banded);
+
+    GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->banded_l_expect, this->tol);
+    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(),
+                        gko::as<Csr>(this->banded_l_expect->conj_transpose()),
+                        this->tol);
+}
+
+
+TYPED_TEST(ParIc, GenerateGeneral)
+{
+    using factorization_type = typename TestFixture::factorization_type;
+    using Csr = typename TestFixture::Csr;
+    auto fact =
+        factorization_type::build().on(this->exec)->generate(this->mtx_system);
+
+    GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->mtx_l_it_expect, this->tol);
+    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(),
+                        gko::as<Csr>(this->mtx_l_it_expect->conj_transpose()),
+                        this->tol);
+}
+
+
+}  // namespace
diff --git a/reference/test/factorization/par_ict_kernels.cpp b/reference/test/factorization/par_ict_kernels.cpp
index 0466278a102..2858e77b4cf 100644
--- a/reference/test/factorization/par_ict_kernels.cpp
+++ b/reference/test/factorization/par_ict_kernels.cpp
@@ -113,7 +113,7 @@ class ParIct : public ::testing::Test {
                                       {0., 0., 3., 0.},
                                       {-2., 0., -3., 4.}},
                                      ref)),
-          mtx_llt(gko::initialize<Csr>({{1., 1., 0., -2.},
+          mtx_llh(gko::initialize<Csr>({{1., 1., 0., -2.},
                                         {1., 5., 0., -2.},
                                         {0., 0., 9., -9.},
                                         {-2., -2., -9., 29.}},
@@ -158,7 +158,7 @@ class ParIct : public ::testing::Test {
     std::unique_ptr<Csr> mtx_l_system;
     std::unique_ptr<Csr> mtx_init;
     std::unique_ptr<Csr> mtx_l;
-    std::unique_ptr<Csr> mtx_llt;
+    std::unique_ptr<Csr> mtx_llh;
     std::unique_ptr<Csr> mtx_l_init_expect;
     std::unique_ptr<Csr> mtx_l_add_expect;
     std::unique_ptr<Csr> mtx_l_it_expect;
@@ -213,7 +213,7 @@ TYPED_TEST(ParIct, KernelAddCandidates)
     auto res_mtx_l = Csr::create(this->exec, this->mtx_system->get_size());
 
     gko::kernels::reference::par_ict_factorization::add_candidates(
-        this->ref, this->mtx_llt.get(), this->mtx_system.get(),
+        this->ref, this->mtx_llh.get(), this->mtx_system.get(),
         this->mtx_l.get(), res_mtx_l.get());
 
     GKO_ASSERT_MTX_EQ_SPARSITY(res_mtx_l, this->mtx_l_add_expect);
@@ -273,7 +273,7 @@ TYPED_TEST(ParIct, SetStrategies)
     ASSERT_EQ(fact->get_l_factor()->get_strategy()->get_name(),
               l_strategy->get_name());
     ASSERT_EQ(factory->get_parameters().lt_strategy, lt_strategy);
-    ASSERT_EQ(fact->get_lt_factor()->get_strategy()->get_name(),
+    ASSERT_EQ(fact->get_lh_factor()->get_strategy()->get_name(),
               lt_strategy->get_name());
 }
 
@@ -285,7 +285,7 @@ TYPED_TEST(ParIct, IsConsistentWithComposition)
     auto lin_op_l_factor =
         static_cast<const gko::LinOp *>(gko::lend(fact->get_l_factor()));
     auto lin_op_lt_factor =
-        static_cast<const gko::LinOp *>(gko::lend(fact->get_lt_factor()));
+        static_cast<const gko::LinOp *>(gko::lend(fact->get_lh_factor()));
     auto first_operator = gko::lend(fact->get_operators()[0]);
     auto second_operator = gko::lend(fact->get_operators()[1]);
 
@@ -299,7 +299,7 @@ TYPED_TEST(ParIct, GenerateIdentity)
     auto fact = this->fact_fact->generate(this->identity);
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->identity, this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(), this->identity, this->tol);
+    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(), this->identity, this->tol);
 }
 
 
@@ -311,7 +311,7 @@ TYPED_TEST(ParIct, GenerateDenseIdentity)
     auto fact = this->fact_fact->generate(gko::share(dense_id));
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->identity, this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(), this->identity, this->tol);
+    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(), this->identity, this->tol);
 }
 
 
@@ -327,7 +327,7 @@ TYPED_TEST(ParIct, GenerateWithExactSmallLimit)
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->mtx_l_small_expect,
                         this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(),
+    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(),
                         gko::as<Csr>(this->mtx_l_small_expect->transpose()),
                         this->tol);
 }
@@ -345,7 +345,7 @@ TYPED_TEST(ParIct, GenerateWithApproxSmallLimit)
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->mtx_l_small_expect,
                         this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(),
+    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(),
                         gko::as<Csr>(this->mtx_l_small_expect->transpose()),
                         this->tol);
 }
@@ -363,7 +363,7 @@ TYPED_TEST(ParIct, GenerateWithExactLargeLimit)
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->mtx_l_large_expect,
                         this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(),
+    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(),
                         gko::as<Csr>(this->mtx_l_large_expect->transpose()),
                         this->tol);
 }
@@ -381,7 +381,7 @@ TYPED_TEST(ParIct, GenerateWithApproxLargeLimit)
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->mtx_l_large_expect,
                         this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(),
+    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(),
                         gko::as<Csr>(this->mtx_l_large_expect->transpose()),
                         this->tol);
 }
diff --git a/reference/test/preconditioner/CMakeLists.txt b/reference/test/preconditioner/CMakeLists.txt
index 908ac83533e..dc6f56cfc49 100644
--- a/reference/test/preconditioner/CMakeLists.txt
+++ b/reference/test/preconditioner/CMakeLists.txt
@@ -1,4 +1,5 @@
 ginkgo_create_test(ilu)
+ginkgo_create_test(ic)
 ginkgo_create_test(isai_kernels)
 ginkgo_create_test(jacobi)
 ginkgo_create_test(jacobi_kernels)
diff --git a/reference/test/preconditioner/ic.cpp b/reference/test/preconditioner/ic.cpp
new file mode 100644
index 00000000000..15884f1f01d
--- /dev/null
+++ b/reference/test/preconditioner/ic.cpp
@@ -0,0 +1,297 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/preconditioner/ic.hpp>
+
+
+#include <memory>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/composition.hpp>
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/factorization/par_ic.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/preconditioner/isai.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+template <typename ValueIndexType>
+class Ic : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using Mtx = gko::matrix::Csr<value_type, index_type>;
+    using Vec = gko::matrix::Dense<value_type>;
+    using ic_prec_type =
+        gko::preconditioner::Ic<gko::solver::LowerTrs<value_type, index_type>,
+                                index_type>;
+    using ic_isai_prec_type = gko::preconditioner::Ic<
+        gko::preconditioner::LowerIsai<value_type, index_type>, index_type>;
+    using Composition = gko::Composition<value_type>;
+
+    Ic()
+        : exec(gko::ReferenceExecutor::create()),
+          mtx(gko::initialize<Mtx>({{1., 1., 0.}, {1., 2., 1.}, {0., 1., 2.}},
+                                   exec)),
+          l_factor(gko::initialize<Mtx>(
+              {{1., 0., 0.}, {1., 1., 0.}, {0., 1., 1.}}, exec)),
+          lh_factor(gko::as<Mtx>(l_factor->conj_transpose())),
+          l_isai_factor(gko::initialize<Mtx>(
+              {{1., 0., 0.}, {-1., 1., 0.}, {0., -1., 1.}}, exec)),
+          lh_isai_factor(gko::as<Mtx>(l_isai_factor->conj_transpose())),
+          l_composition(Composition::create(l_factor)),
+          l_lh_composition(Composition::create(l_factor, lh_factor)),
+          ic_pre_factory(ic_prec_type::build().on(exec)),
+          ic_isai_pre_factory(ic_isai_prec_type::build().on(exec)),
+          tol{r<value_type>::value}
+    {}
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<Mtx> mtx;
+    std::shared_ptr<Mtx> l_factor;
+    std::shared_ptr<Mtx> lh_factor;
+    std::shared_ptr<Mtx> l_isai_factor;
+    std::shared_ptr<Mtx> lh_isai_factor;
+    std::shared_ptr<Composition> l_composition;
+    std::shared_ptr<Composition> l_lh_composition;
+    std::shared_ptr<typename ic_prec_type::Factory> ic_pre_factory;
+    std::shared_ptr<typename ic_isai_prec_type::Factory> ic_isai_pre_factory;
+    gko::remove_complex<value_type> tol;
+};
+
+TYPED_TEST_SUITE(Ic, gko::test::ValueIndexTypes);
+
+
+TYPED_TEST(Ic, BuildsTwoFactorComposition)
+{
+    auto precond = this->ic_pre_factory->generate(this->l_lh_composition);
+
+    GKO_ASSERT_MTX_NEAR(precond->get_l_solver()->get_system_matrix(),
+                        this->l_factor, this->tol);
+    GKO_ASSERT_MTX_NEAR(precond->get_lh_solver()->get_system_matrix(),
+                        this->lh_factor, this->tol);
+}
+
+
+TYPED_TEST(Ic, BuildsOneFactorComposition)
+{
+    auto precond = this->ic_pre_factory->generate(this->l_composition);
+
+    GKO_ASSERT_MTX_NEAR(precond->get_l_solver()->get_system_matrix(),
+                        this->l_factor, this->tol);
+    GKO_ASSERT_MTX_NEAR(precond->get_lh_solver()->get_system_matrix(),
+                        this->lh_factor, this->tol);
+}
+
+
+TYPED_TEST(Ic, BuildsSymmetricTwoFactor)
+{
+    auto precond = this->ic_pre_factory->generate(this->l_lh_composition);
+
+    // the first factor should be identical, the second not as it was transposed
+    ASSERT_EQ(precond->get_l_solver()->get_system_matrix().get(),
+              this->l_factor.get());
+    ASSERT_NE(precond->get_lh_solver()->get_system_matrix().get(),
+              this->lh_factor.get());
+}
+
+
+TYPED_TEST(Ic, BuildsFromMatrix)
+{
+    auto precond = this->ic_pre_factory->generate(this->mtx);
+
+    GKO_ASSERT_MTX_NEAR(precond->get_l_solver()->get_system_matrix(),
+                        this->l_factor, this->tol);
+    GKO_ASSERT_MTX_NEAR(precond->get_lh_solver()->get_system_matrix(),
+                        this->lh_factor, this->tol);
+}
+
+
+TYPED_TEST(Ic, BuildsIsaiFromMatrix)
+{
+    auto precond = this->ic_isai_pre_factory->generate(this->mtx);
+
+    GKO_ASSERT_MTX_NEAR(precond->get_l_solver()->get_approximate_inverse(),
+                        this->l_isai_factor, this->tol);
+    GKO_ASSERT_MTX_NEAR(precond->get_lh_solver()->get_approximate_inverse(),
+                        this->lh_isai_factor, this->tol);
+}
+
+
+TYPED_TEST(Ic, ThrowOnWrongCompositionInput)
+{
+    using Composition = typename TestFixture::Composition;
+    std::shared_ptr<Composition> composition =
+        Composition::create(this->l_factor, this->l_factor, this->l_factor);
+
+    ASSERT_THROW(this->ic_pre_factory->generate(composition),
+                 gko::NotSupported);
+}
+
+
+TYPED_TEST(Ic, CanBeCopied)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using ic_prec_type = typename TestFixture::ic_prec_type;
+    using Composition = typename TestFixture::Composition;
+    auto ic = this->ic_pre_factory->generate(this->l_lh_composition);
+    auto before_l_solver = ic->get_l_solver();
+    auto before_lh_solver = ic->get_lh_solver();
+    // The switch up of matrices is intentional, to make sure they are distinct!
+    auto lh_l_composition =
+        Composition::create(this->lh_factor, this->l_factor);
+    auto copied = ic_prec_type::build()
+                      .on(this->exec)
+                      ->generate(gko::share(lh_l_composition));
+
+    copied->copy_from(ic.get());
+
+    ASSERT_EQ(before_l_solver, copied->get_l_solver());
+    ASSERT_EQ(before_lh_solver, copied->get_lh_solver());
+}
+
+
+TYPED_TEST(Ic, CanBeMoved)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using ic_prec_type = typename TestFixture::ic_prec_type;
+    using Composition = typename TestFixture::Composition;
+    auto ic = this->ic_pre_factory->generate(this->l_lh_composition);
+    auto before_l_solver = ic->get_l_solver();
+    auto before_lh_solver = ic->get_lh_solver();
+    // The switch up of matrices is intentional, to make sure they are distinct!
+    auto lh_l_composition =
+        Composition::create(this->lh_factor, this->l_factor);
+    auto moved = ic_prec_type::build()
+                     .on(this->exec)
+                     ->generate(gko::share(lh_l_composition));
+
+    moved->copy_from(std::move(ic));
+
+    ASSERT_EQ(before_l_solver, moved->get_l_solver());
+    ASSERT_EQ(before_lh_solver, moved->get_lh_solver());
+}
+
+
+TYPED_TEST(Ic, CanBeCloned)
+{
+    auto ic = this->ic_pre_factory->generate(this->l_lh_composition);
+    auto before_l_solver = ic->get_l_solver();
+    auto before_lh_solver = ic->get_lh_solver();
+
+    auto clone = ic->clone();
+
+    ASSERT_EQ(before_l_solver, clone->get_l_solver());
+    ASSERT_EQ(before_lh_solver, clone->get_lh_solver());
+}
+
+
+TYPED_TEST(Ic, CanBeTransposed)
+{
+    using Ic = typename TestFixture::ic_prec_type;
+    using Mtx = typename TestFixture::Mtx;
+    auto ic = this->ic_pre_factory->generate(this->l_lh_composition);
+    auto l_ref = gko::as<Mtx>(ic->get_l_solver()->get_system_matrix());
+    auto lh_ref = gko::as<Mtx>(ic->get_lh_solver()->get_system_matrix());
+
+    auto transp = gko::as<Ic>(ic->transpose());
+
+    auto l_transp = gko::as<Mtx>(transp->get_l_solver()->get_system_matrix());
+    auto lh_transp = gko::as<Mtx>(transp->get_lh_solver()->get_system_matrix());
+    GKO_ASSERT_MTX_NEAR(l_transp, l_ref, 0);
+    GKO_ASSERT_MTX_NEAR(lh_transp, lh_ref, 0);
+}
+
+
+TYPED_TEST(Ic, CanBeConjTransposed)
+{
+    using Ic = typename TestFixture::ic_prec_type;
+    using Mtx = typename TestFixture::Mtx;
+    auto ic = this->ic_pre_factory->generate(this->l_lh_composition);
+    auto l_ref = gko::as<Mtx>(ic->get_l_solver()->get_system_matrix());
+    auto lh_ref = gko::as<Mtx>(ic->get_lh_solver()->get_system_matrix());
+
+    auto transp = gko::as<Ic>(ic->conj_transpose());
+
+    auto l_transp = gko::as<Mtx>(transp->get_l_solver()->get_system_matrix());
+    auto lh_transp = gko::as<Mtx>(transp->get_lh_solver()->get_system_matrix());
+    GKO_ASSERT_MTX_NEAR(l_transp, l_ref, 0);
+    GKO_ASSERT_MTX_NEAR(lh_transp, lh_ref, 0);
+}
+
+
+TYPED_TEST(Ic, SolvesSingleRhs)
+{
+    using ic_prec_type = typename TestFixture::ic_prec_type;
+    using Vec = typename TestFixture::Vec;
+    const auto b = gko::initialize<Vec>({1.0, 3.0, 6.0}, this->exec);
+    auto x = Vec::create(this->exec, gko::dim<2>{3, 1});
+    auto preconditioner =
+        ic_prec_type::build().on(this->exec)->generate(this->mtx);
+
+    preconditioner->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({3.0, -2.0, 4.0}), this->tol);
+}
+
+
+TYPED_TEST(Ic, SolvesMultipleRhs)
+{
+    using ic_prec_type = typename TestFixture::ic_prec_type;
+    using Vec = typename TestFixture::Vec;
+    const auto b = gko::initialize<Vec>(
+        {{1.0, 2.0, 3.0}, {3.0, 6.0, 9.0}, {6.0, 12.0, 18.0}}, this->exec);
+    auto x = Vec::create(this->exec, gko::dim<2>{3, 3});
+    auto preconditioner =
+        ic_prec_type::build().on(this->exec)->generate(this->mtx);
+
+    preconditioner->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x, l({{3.0, 6.0, 9.0}, {-2.0, -4.0, -6.0}, {4.0, 8.0, 12.0}}),
+        this->tol);
+}
+
+
+}  // namespace

From de04a125b309d779c3719a8c843c1347dcf9b9e6 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 3 Dec 2020 23:56:45 +0100
Subject: [PATCH 0254/1008] fix gpu test device config

---
 cuda/test/matrix/diagonal_kernels.cpp       | 2 +-
 cuda/test/matrix/ell_kernels.cpp            | 1 -
 cuda/test/matrix/sellp_kernels.cpp          | 1 -
 cuda/test/preconditioner/jacobi_kernels.cpp | 2 +-
 hip/test/matrix/diagonal_kernels.hip.cpp    | 2 +-
 hip/test/matrix/ell_kernels.hip.cpp         | 1 -
 6 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/cuda/test/matrix/diagonal_kernels.cpp b/cuda/test/matrix/diagonal_kernels.cpp
index da16185704f..c9a6580076b 100644
--- a/cuda/test/matrix/diagonal_kernels.cpp
+++ b/cuda/test/matrix/diagonal_kernels.cpp
@@ -47,7 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/matrix/diagonal_kernels.hpp"
-#include "core/test/utils.hpp"
+#include "cuda/test/utils.hpp"
 
 
 namespace {
diff --git a/cuda/test/matrix/ell_kernels.cpp b/cuda/test/matrix/ell_kernels.cpp
index 437066a84bf..dff573a284f 100644
--- a/cuda/test/matrix/ell_kernels.cpp
+++ b/cuda/test/matrix/ell_kernels.cpp
@@ -48,7 +48,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/matrix/ell_kernels.hpp"
-#include "core/test/utils.hpp"
 #include "cuda/test/utils.hpp"
 
 
diff --git a/cuda/test/matrix/sellp_kernels.cpp b/cuda/test/matrix/sellp_kernels.cpp
index 4bf5a43cce8..2935377014a 100644
--- a/cuda/test/matrix/sellp_kernels.cpp
+++ b/cuda/test/matrix/sellp_kernels.cpp
@@ -48,7 +48,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/matrix/sellp_kernels.hpp"
-#include "core/test/utils.hpp"
 #include "cuda/test/utils.hpp"
 
 
diff --git a/cuda/test/preconditioner/jacobi_kernels.cpp b/cuda/test/preconditioner/jacobi_kernels.cpp
index 9bbeebe1652..aafd6546996 100644
--- a/cuda/test/preconditioner/jacobi_kernels.cpp
+++ b/cuda/test/preconditioner/jacobi_kernels.cpp
@@ -43,8 +43,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
-#include "core/test/utils.hpp"
 #include "core/test/utils/unsort_matrix.hpp"
+#include "cuda/test/utils.hpp"
 
 
 namespace {
diff --git a/hip/test/matrix/diagonal_kernels.hip.cpp b/hip/test/matrix/diagonal_kernels.hip.cpp
index b7554565cc5..3b62ad8316e 100644
--- a/hip/test/matrix/diagonal_kernels.hip.cpp
+++ b/hip/test/matrix/diagonal_kernels.hip.cpp
@@ -47,7 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/matrix/diagonal_kernels.hpp"
-#include "core/test/utils.hpp"
+#include "hip/test/utils.hip.hpp"
 
 
 namespace {
diff --git a/hip/test/matrix/ell_kernels.hip.cpp b/hip/test/matrix/ell_kernels.hip.cpp
index 28dc3c8c38f..e9a875eb544 100644
--- a/hip/test/matrix/ell_kernels.hip.cpp
+++ b/hip/test/matrix/ell_kernels.hip.cpp
@@ -48,7 +48,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/matrix/ell_kernels.hpp"
-#include "core/test/utils.hpp"
 #include "hip/test/utils.hip.hpp"
 
 

From 5e8b7a7fa36b83edbb05d8ca1db03cbeca103583 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 4 Dec 2020 00:10:35 +0100
Subject: [PATCH 0255/1008] add dpcpp to formatting

---
 .github/bot-pr-format-base.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/bot-pr-format-base.sh b/.github/bot-pr-format-base.sh
index 298f061cc87..c4171aac779 100644
--- a/.github/bot-pr-format-base.sh
+++ b/.github/bot-pr-format-base.sh
@@ -3,7 +3,7 @@
 source .github/bot-pr-base.sh
 
 EXTENSION_REGEX='\.(cuh?|hpp|hpp\.inc?|cpp)$'
-FORMAT_HEADER_REGEX='^(benchmark|core|cuda|hip|include/ginkgo/core|omp|reference)/'
+FORMAT_HEADER_REGEX='^(benchmark|core|cuda|hip|include/ginkgo/core|omp|reference|dpcpp)/'
 FORMAT_REGEX='^(common|examples|test_install)/'
 
 echo "Retrieving PR file list"

From 7c5565e8159bf3604f72fdf76e22ecec9cc4b1c7 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 7 Dec 2020 13:54:31 +0100
Subject: [PATCH 0256/1008] fix IC preconditioner setup

---
 include/ginkgo/core/preconditioner/ic.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/include/ginkgo/core/preconditioner/ic.hpp b/include/ginkgo/core/preconditioner/ic.hpp
index 70fc233dfd5..27e9d5bc582 100644
--- a/include/ginkgo/core/preconditioner/ic.hpp
+++ b/include/ginkgo/core/preconditioner/ic.hpp
@@ -261,6 +261,7 @@ class Ic : public EnableLinOp<Ic<LSolverType, IndexType>>, public Transposable {
             }
         } else {
             l_solver_ = parameters_.l_solver_factory->generate(l_factor);
+            lh_solver_ = as<lh_solver_type>(l_solver_->conj_transpose());
         }
     }
 

From 88d71fc1799b250041624a733aa8a02297fdc44c Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 7 Dec 2020 13:56:22 +0100
Subject: [PATCH 0257/1008] add IC preconditioners to benchmark

---
 BENCHMARKING.md                     |  2 +-
 benchmark/utils/preconditioners.hpp | 74 +++++++++++++++++++++++------
 2 files changed, 61 insertions(+), 15 deletions(-)

diff --git a/BENCHMARKING.md b/BENCHMARKING.md
index 31a5546f7e8..33e4505c0d6 100644
--- a/BENCHMARKING.md
+++ b/BENCHMARKING.md
@@ -284,7 +284,7 @@ The supported environment variables are described in the following list:
     benchmark runs (downloads the collections, creates the result structure,
     etc.) and outputs the list of commands that would normally be run, but does
     not run the benchmarks themselves. Default is `false`.
-* `PRECONDS={jacobi,ilu,parict,parilu,parilut,ilu-isai,parict-isai,parilu-isai,parilut-isai,none}`
+* `PRECONDS={jacobi,ic,ilu,paric,parict,parilu,parilut,ic-isai,ilu-isai,paric-isai,parict-isai,parilu-isai,parilut-isai,none}`
     the preconditioners to use for either `solver` or `preconditioner` benchmarks.
     Multiple options can be passed to this variable. Default is `none`.
 * `FORMATS={csr,coo,ell,hybrid,sellp,hybridxx,cusp_xx,hipsp_xx}` the matrix
diff --git a/benchmark/utils/preconditioners.hpp b/benchmark/utils/preconditioners.hpp
index 4c7dc733e34..17f20c40ebe 100644
--- a/benchmark/utils/preconditioners.hpp
+++ b/benchmark/utils/preconditioners.hpp
@@ -49,14 +49,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/types.hpp"
 
 
-DEFINE_string(
-    preconditioners, "none",
-    "A comma-separated list of preconditioners to use. "
-    "Supported values are: none, jacobi, parict, parilu, parilut, ilu, "
-    "parict-isai, parilu-isai, parilut-isai, ilu-isai, overhead");
+DEFINE_string(preconditioners, "none",
+              "A comma-separated list of preconditioners to use. "
+              "Supported values are: none, jacobi, paric, parict, parilu, "
+              "parilut, ic, ilu, paric-isai, parict-isai, parilu-isai, "
+              "parilut-isai, ic-isai, ilu-isai, overhead");
 
 DEFINE_uint32(parilu_iterations, 5,
-              "The number of iterations for ParICT/ParILU(T)");
+              "The number of iterations for ParIC(T)/ParILU(T)");
 
 DEFINE_bool(parilut_approx_select, true,
             "Use approximate selection for ParICT/ParILUT");
@@ -113,6 +113,18 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_skip_sorting(true)
                  .on(exec);
          }},
+        {"paric",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact =
+                 gko::share(gko::factorization::ParIc<etype>::build()
+                                .with_iterations(FLAGS_parilu_iterations)
+                                .with_skip_sorting(true)
+                                .on(exec));
+             return gko::preconditioner::Ic<
+                        gko::solver::LowerTrs<etype>>::build()
+                 .with_factorization_factory(fact)
+                 .on(exec);
+         }},
         {"parict",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact = gko::share(
@@ -156,6 +168,15 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_factorization_factory(fact)
                  .on(exec);
          }},
+        {"ic",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact =
+                 gko::share(gko::factorization::Ic<etype>::build().on(exec));
+             return gko::preconditioner::Ic<
+                        gko::solver::LowerTrs<etype>>::build()
+                 .with_factorization_factory(fact)
+                 .on(exec);
+         }},
         {"ilu",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact =
@@ -166,6 +187,23 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_factorization_factory(fact)
                  .on(exec);
          }},
+        {"paric-isai",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact =
+                 gko::share(gko::factorization::ParIc<etype>::build()
+                                .with_iterations(FLAGS_parilu_iterations)
+                                .with_skip_sorting(true)
+                                .on(exec));
+             auto lisai =
+                 gko::share(gko::preconditioner::LowerIsai<etype>::build()
+                                .with_sparsity_power(FLAGS_isai_power)
+                                .on(exec));
+             return gko::preconditioner::Ic<
+                        gko::preconditioner::LowerIsai<etype>>::build()
+                 .with_factorization_factory(fact)
+                 .with_l_solver_factory(lisai)
+                 .on(exec);
+         }},
         {"parict-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact = gko::share(
@@ -179,16 +217,10 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  gko::share(gko::preconditioner::LowerIsai<etype>::build()
                                 .with_sparsity_power(FLAGS_isai_power)
                                 .on(exec));
-             auto uisai =
-                 gko::share(gko::preconditioner::UpperIsai<etype>::build()
-                                .with_sparsity_power(FLAGS_isai_power)
-                                .on(exec));
-             return gko::preconditioner::Ilu<
-                        gko::preconditioner::LowerIsai<etype>,
-                        gko::preconditioner::UpperIsai<etype>>::build()
+             return gko::preconditioner::Ic<
+                        gko::preconditioner::LowerIsai<etype>>::build()
                  .with_factorization_factory(fact)
                  .with_l_solver_factory(lisai)
-                 .with_u_solver_factory(uisai)
                  .on(exec);
          }},
         {"parilu-isai",
@@ -239,6 +271,20 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_u_solver_factory(uisai)
                  .on(exec);
          }},
+        {"ic-isai",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             auto fact =
+                 gko::share(gko::factorization::Ic<etype>::build().on(exec));
+             auto lisai =
+                 gko::share(gko::preconditioner::LowerIsai<etype>::build()
+                                .with_sparsity_power(FLAGS_isai_power)
+                                .on(exec));
+             return gko::preconditioner::Ic<
+                        gko::preconditioner::LowerIsai<etype>>::build()
+                 .with_factorization_factory(fact)
+                 .with_l_solver_factory(lisai)
+                 .on(exec);
+         }},
         {"ilu-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact =

From a3abb75e4ebd2660b05bd29773521e39503806fe Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 7 Dec 2020 13:56:49 +0100
Subject: [PATCH 0258/1008] add ParILU convergence example

---
 examples/CMakeLists.txt                       |   1 +
 examples/par-ilu-convergence/CMakeLists.txt   |   5 +
 examples/par-ilu-convergence/build.sh         |  17 ++
 examples/par-ilu-convergence/data/A.mtx       | 114 ++++++++++
 examples/par-ilu-convergence/doc/builds-on    |   1 +
 examples/par-ilu-convergence/doc/intro.dox    |   7 +
 examples/par-ilu-convergence/doc/kind         |   1 +
 examples/par-ilu-convergence/doc/results.dox  |  28 +++
 examples/par-ilu-convergence/doc/short-intro  |   1 +
 examples/par-ilu-convergence/doc/tooltip      |   1 +
 .../par-ilu-convergence.cpp                   | 203 ++++++++++++++++++
 11 files changed, 379 insertions(+)
 create mode 100644 examples/par-ilu-convergence/CMakeLists.txt
 create mode 100755 examples/par-ilu-convergence/build.sh
 create mode 100644 examples/par-ilu-convergence/data/A.mtx
 create mode 100644 examples/par-ilu-convergence/doc/builds-on
 create mode 100644 examples/par-ilu-convergence/doc/intro.dox
 create mode 100644 examples/par-ilu-convergence/doc/kind
 create mode 100644 examples/par-ilu-convergence/doc/results.dox
 create mode 100644 examples/par-ilu-convergence/doc/short-intro
 create mode 100644 examples/par-ilu-convergence/doc/tooltip
 create mode 100644 examples/par-ilu-convergence/par-ilu-convergence.cpp

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index b36429cfcf1..c023940383b 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -23,6 +23,7 @@ set(EXAMPLES_LIST
     ginkgo-overhead
     minimal-cuda-solver
     papi-logging
+    par-ilu-convergence
     performance-debugging
     preconditioner-export
     simple-solver-logging)
diff --git a/examples/par-ilu-convergence/CMakeLists.txt b/examples/par-ilu-convergence/CMakeLists.txt
new file mode 100644
index 00000000000..a81f0520a9a
--- /dev/null
+++ b/examples/par-ilu-convergence/CMakeLists.txt
@@ -0,0 +1,5 @@
+set(target_name "par-ilu-convergence")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/par-ilu-convergence/build.sh b/examples/par-ilu-convergence/build.sh
new file mode 100755
index 00000000000..6af53f00387
--- /dev/null
+++ b/examples/par-ilu-convergence/build.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# set up script
+if [ $# -ne 1 ]; then
+    echo -e "Usage: $0 GINKGO_BUILD_DIRECTORY"
+    exit 1
+fi
+BUILD_DIR=$1
+THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" &>/dev/null && pwd )
+
+source ${THIS_DIR}/../build-setup.sh
+
+# build
+${CXX} -std=c++14 -o ${THIS_DIR}/par-ilu-convergence \
+    ${THIS_DIR}/par-ilu-convergence.cpp \
+    -I${THIS_DIR}/../../include -I${BUILD_DIR}/include \
+    -L${THIS_DIR} ${LINK_FLAGS}
diff --git a/examples/par-ilu-convergence/data/A.mtx b/examples/par-ilu-convergence/data/A.mtx
new file mode 100644
index 00000000000..c67437da567
--- /dev/null
+++ b/examples/par-ilu-convergence/data/A.mtx
@@ -0,0 +1,114 @@
+%%MatrixMarket matrix coordinate integer symmetric
+%-------------------------------------------------------------------------------
+% UF Sparse Matrix Collection, Tim Davis
+% http://www.cise.ufl.edu/research/sparse/matrices/JGD_Trefethen/Trefethen_20b
+% name: JGD_Trefethen/Trefethen_20b
+% [Diagonal matrices with primes, Nick Trefethen, Oxford Univ.]
+% id: 2203
+% date: 2008
+% author: N. Trefethen
+% ed: J.-G. Dumas
+% fields: name title A id date author ed kind notes
+% kind: combinatorial problem
+%-------------------------------------------------------------------------------
+% notes:
+% Diagonal matrices with primes, Nick Trefethen, Oxford Univ.          
+% From Jean-Guillaume Dumas' Sparse Integer Matrix Collection,         
+% http://ljk.imag.fr/membres/Jean-Guillaume.Dumas/simc.html            
+%                                                                      
+% Problem 7 of the Hundred-dollar, Hundred-digit Challenge Problems,   
+% SIAM News, vol 35, no. 1.                                            
+%                                                                      
+% 7. Let A be the 20,000 x 20,000 matrix whose entries are zero        
+% everywhere except for the primes 2, 3, 5, 7, . . . , 224737 along the
+% main diagonal and the number 1 in all the positions A(i,j) with      
+% |i-j| = 1,2,4,8, . . . ,16384.  What is the (1,1) entry of inv(A)?   
+%                                                                      
+% http://www.siam.org/news/news.php?id=388                             
+%                                                                      
+% Filename in JGD collection: Trefethen/trefethen_20__19_minor.sms     
+%-------------------------------------------------------------------------------
+19 19 83
+1 1 3
+2 1 1
+3 1 1
+5 1 1
+9 1 1
+17 1 1
+2 2 5
+3 2 1
+4 2 1
+6 2 1
+10 2 1
+18 2 1
+3 3 7
+4 3 1
+5 3 1
+7 3 1
+11 3 1
+19 3 1
+4 4 11
+5 4 1
+6 4 1
+8 4 1
+12 4 1
+5 5 13
+6 5 1
+7 5 1
+9 5 1
+13 5 1
+6 6 17
+7 6 1
+8 6 1
+10 6 1
+14 6 1
+7 7 19
+8 7 1
+9 7 1
+11 7 1
+15 7 1
+8 8 23
+9 8 1
+10 8 1
+12 8 1
+16 8 1
+9 9 29
+10 9 1
+11 9 1
+13 9 1
+17 9 1
+10 10 31
+11 10 1
+12 10 1
+14 10 1
+18 10 1
+11 11 37
+12 11 1
+13 11 1
+15 11 1
+19 11 1
+12 12 41
+13 12 1
+14 12 1
+16 12 1
+13 13 43
+14 13 1
+15 13 1
+17 13 1
+14 14 47
+15 14 1
+16 14 1
+18 14 1
+15 15 53
+16 15 1
+17 15 1
+19 15 1
+16 16 59
+17 16 1
+18 16 1
+17 17 61
+18 17 1
+19 17 1
+18 18 67
+19 18 1
+19 19 71
diff --git a/examples/par-ilu-convergence/doc/builds-on b/examples/par-ilu-convergence/doc/builds-on
new file mode 100644
index 00000000000..369aa997770
--- /dev/null
+++ b/examples/par-ilu-convergence/doc/builds-on
@@ -0,0 +1 @@
+simple-solver
diff --git a/examples/par-ilu-convergence/doc/intro.dox b/examples/par-ilu-convergence/doc/intro.dox
new file mode 100644
index 00000000000..adc036224d4
--- /dev/null
+++ b/examples/par-ilu-convergence/doc/intro.dox
@@ -0,0 +1,7 @@
+<a name="Intro"></a>
+<h1>Introduction</h1>
+
+<h3> About the example </h3>
+
+This example can be used to inspect the convergence behavior of parallel
+incomplete factorizations.
\ No newline at end of file
diff --git a/examples/par-ilu-convergence/doc/kind b/examples/par-ilu-convergence/doc/kind
new file mode 100644
index 00000000000..53a96d5771f
--- /dev/null
+++ b/examples/par-ilu-convergence/doc/kind
@@ -0,0 +1 @@
+preconditioners
diff --git a/examples/par-ilu-convergence/doc/results.dox b/examples/par-ilu-convergence/doc/results.dox
new file mode 100644
index 00000000000..e0456cc9d2a
--- /dev/null
+++ b/examples/par-ilu-convergence/doc/results.dox
@@ -0,0 +1,28 @@
+<h1>Results</h1>
+This is the expected output:
+
+@code{.cpp}
+
+Usage: executable <reference|omp|cuda|hip|dpcpp> [<matrix-file>] [<parilu|parilut|paric|parict] [<max-iterations>] [<num-repetitions>] [fill-in-limit]
+
+@endcode
+
+When specifying an executor:
+
+@code{.cpp}
+
+Reading data/A.mtx
+1;71800;10300;8800;8200;8000;7700;7500;7500;7500;7400;1.0331e-14;1.0331e-14;1.0331e-14;1.0331e-14;1.0331e-14;1.0331e-14;1.0331e-14;1.0331e-14;1.0331e-14;1.0331e-14;
+2;15500;9100;13500;9000;8600;8800;8700;8600;8600;8500;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;
+3;16500;10200;10100;10100;9900;10000;9800;9800;9900;9900;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;
+4;17500;11500;11200;15600;11300;11200;11400;11200;11200;11100;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;
+5;18800;12800;12700;12600;12500;12400;12400;12400;12400;14100;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;
+6;19200;13400;23100;15400;13200;13000;13000;13000;13100;13000;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;
+7;20500;14500;14400;14200;14200;14300;14200;14100;14300;14200;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;
+8;21600;15700;86200;16300;15700;15600;15500;15400;15500;15600;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;
+9;22700;17000;16700;16600;16700;16800;20400;17400;17500;17400;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;
+10;25500;19000;18800;18700;18700;18800;18600;18700;18600;18700;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;4.15407e-16;
+
+@endcode
+
+<h3> Comments about programming and debugging </h3>
diff --git a/examples/par-ilu-convergence/doc/short-intro b/examples/par-ilu-convergence/doc/short-intro
new file mode 100644
index 00000000000..c8f8281379e
--- /dev/null
+++ b/examples/par-ilu-convergence/doc/short-intro
@@ -0,0 +1 @@
+The ParILU convergence example.
diff --git a/examples/par-ilu-convergence/doc/tooltip b/examples/par-ilu-convergence/doc/tooltip
new file mode 100644
index 00000000000..0f788aaa603
--- /dev/null
+++ b/examples/par-ilu-convergence/doc/tooltip
@@ -0,0 +1 @@
+Compute runtimes and residuals for different ParILU(T)/IC(T) iteration counts.
diff --git a/examples/par-ilu-convergence/par-ilu-convergence.cpp b/examples/par-ilu-convergence/par-ilu-convergence.cpp
new file mode 100644
index 00000000000..23f9111e4c3
--- /dev/null
+++ b/examples/par-ilu-convergence/par-ilu-convergence.cpp
@@ -0,0 +1,203 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+
+#include <ginkgo/ginkgo.hpp>
+
+
+#include <fstream>
+#include <functional>
+#include <iostream>
+#include <map>
+#include <memory>
+#include <string>
+
+
+const std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+    executors{
+        {"reference", [] { return gko::ReferenceExecutor::create(); }},
+        {"omp", [] { return gko::OmpExecutor::create(); }},
+        {"cuda",
+         [] {
+             return gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+         }},
+        {"hip",
+         [] {
+             return gko::HipExecutor::create(0, gko::OmpExecutor::create());
+         }},
+        {"dpcpp", [] {
+             return gko::DpcppExecutor::create(0, gko::OmpExecutor::create());
+         }}};
+
+
+void output(const gko::WritableToMatrixData<double, int> *mtx, std::string name)
+{
+    std::ofstream stream{name};
+    std::cerr << "Writing " << name << std::endl;
+    gko::write(stream, mtx, gko::layout_type::coordinate);
+}
+
+
+template <typename Function>
+auto try_generate(Function fun) -> decltype(fun())
+{
+    decltype(fun()) result;
+    try {
+        result = fun();
+    } catch (const gko::Error &err) {
+        std::cerr << "Error: " << err.what() << '\n';
+        std::exit(-1);
+    }
+    return result;
+}
+
+
+double compute_ilu_residual_norm(const gko::matrix::Csr<> *residual,
+                                 const gko::matrix::Csr<> *mtx)
+{
+    gko::matrix_data<> residual_data;
+    gko::matrix_data<> mtx_data;
+    residual->write(residual_data);
+    mtx->write(mtx_data);
+    residual_data.ensure_row_major_order();
+    mtx_data.ensure_row_major_order();
+    auto it = mtx_data.nonzeros.begin();
+    double residual_norm{};
+    for (auto entry : residual_data.nonzeros) {
+        auto ref_row = it->row;
+        auto ref_col = it->column;
+        if (entry.row == ref_row && entry.column == ref_col) {
+            residual_norm += gko::squared_norm(entry.value);
+            ++it;
+        }
+    }
+    return std::sqrt(residual_norm);
+}
+
+
+int main(int argc, char *argv[])
+{
+    // print usage message
+    if (argc < 2 || executors.find(argv[1]) == executors.end()) {
+        std::cerr << "Usage: executable"
+                  << " <reference|omp|cuda|hip|dpcpp> [<matrix-file>] "
+                     "[<parilu|parilut|paric|parict] [<max-iterations>] "
+                     "[<num-repetitions>] [fill-in-limit]\n";
+        return -1;
+    }
+
+    // generate executor based on first argument
+    auto exec = try_generate([&] { return executors.at(argv[1])(); });
+
+    // set matrix and preconditioner name with default values
+    std::string matrix = argc < 3 ? "data/A.mtx" : argv[2];
+    std::string precond = argc < 4 ? "parilu" : argv[3];
+    int max_iterations = argc < 5 ? 10 : std::stoi(argv[4]);
+    int num_repetitions = argc < 6 ? 10 : std::stoi(argv[5]);
+    double limit = argc < 7 ? 2 : std::stod(argv[6]);
+
+    // load matrix file into Csr format
+    auto mtx = gko::share(try_generate([&] {
+        std::ifstream mtx_stream{matrix};
+        if (!mtx_stream) {
+            throw GKO_STREAM_ERROR("Unable to open matrix file");
+        }
+        std::cerr << "Reading " << matrix << std::endl;
+        return gko::read<gko::matrix::Csr<>>(mtx_stream, exec);
+    }));
+
+    std::shared_ptr<gko::LinOpFactory> factory;
+    std::function<void(int)> set_iterations;
+    if (precond == "parilu") {
+        factory = gko::factorization::ParIlu<>::build().on(exec);
+        set_iterations = [&](int it) {
+            gko::as<gko::factorization::ParIlu<>::Factory>(factory)
+                ->get_parameters()
+                .iterations = it;
+        };
+    } else if (precond == "paric") {
+        factory = gko::factorization::ParIc<>::build().on(exec);
+        set_iterations = [&](int it) {
+            gko::as<gko::factorization::ParIc<>::Factory>(factory)
+                ->get_parameters()
+                .iterations = it;
+        };
+    } else if (precond == "parilut") {
+        factory =
+            gko::factorization::ParIlut<>::build().with_fill_in_limit(limit).on(
+                exec);
+        set_iterations = [&](int it) {
+            gko::as<gko::factorization::ParIlut<>::Factory>(factory)
+                ->get_parameters()
+                .iterations = it;
+        };
+    } else if (precond == "parict") {
+        factory =
+            gko::factorization::ParIct<>::build().with_fill_in_limit(limit).on(
+                exec);
+        set_iterations = [&](int it) {
+            gko::as<gko::factorization::ParIct<>::Factory>(factory)
+                ->get_parameters()
+                .iterations = it;
+        };
+    }
+    auto one = gko::initialize<gko::matrix::Dense<>>({1.0}, exec);
+    auto minus_one = gko::initialize<gko::matrix::Dense<>>({-1.0}, exec);
+    for (int it = 1; it <= max_iterations; ++it) {
+        set_iterations(it);
+        std::cout << it << ';';
+        std::vector<long> times;
+        std::vector<double> residuals;
+        for (int rep = 0; rep < num_repetitions; ++rep) {
+            auto tic = std::chrono::high_resolution_clock::now();
+            auto result = gko::as<gko::Composition<>>(factory->generate(mtx));
+            exec->synchronize();
+            auto toc = std::chrono::high_resolution_clock::now();
+            auto residual = gko::clone(exec, mtx);
+            result->get_operators()[0]->apply(lend(one),
+                                              lend(result->get_operators()[1]),
+                                              lend(minus_one), lend(residual));
+            times.push_back(
+                std::chrono::duration_cast<std::chrono::nanoseconds>(toc - tic)
+                    .count());
+            residuals.push_back(
+                compute_ilu_residual_norm(lend(residual), lend(mtx)));
+        }
+        for (auto el : times) {
+            std::cout << el << ';';
+        }
+        for (auto el : residuals) {
+            std::cout << el << ';';
+        }
+        std::cout << '\n';
+    }
+}

From 75b5abadcf38bba2190dce6ca24b301628ce87f4 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 7 Dec 2020 13:57:45 +0100
Subject: [PATCH 0259/1008] add subwarp-parallel and thread-parallel ParIC

---
 common/factorization/par_ic_kernels.hpp.inc   | 49 ++++++++++++++++---
 cuda/factorization/par_ic_kernels.cu          | 45 +++++++++++------
 cuda/test/factorization/par_ic_kernels.cpp    | 21 +++++++-
 hip/factorization/par_ic_kernels.hip.cpp      | 36 +++++++++-----
 hip/test/factorization/par_ic_kernels.hip.cpp | 21 +++++++-
 omp/test/factorization/par_ic_kernels.cpp     |  2 +-
 6 files changed, 136 insertions(+), 38 deletions(-)

diff --git a/common/factorization/par_ic_kernels.hpp.inc b/common/factorization/par_ic_kernels.hpp.inc
index 4c3da30ae48..70d9fd9e70e 100644
--- a/common/factorization/par_ic_kernels.hpp.inc
+++ b/common/factorization/par_ic_kernels.hpp.inc
@@ -52,7 +52,7 @@ __global__ __launch_bounds__(default_block_size) void ic_init(
 }
 
 
-template <int subwarp_size, typename ValueType, typename IndexType>
+template <typename ValueType, typename IndexType>
 __global__ __launch_bounds__(default_block_size) void ic_sweep(
     const IndexType *__restrict__ a_row_idxs,
     const IndexType *__restrict__ a_col_idxs,
@@ -60,6 +60,46 @@ __global__ __launch_bounds__(default_block_size) void ic_sweep(
     const IndexType *__restrict__ l_row_ptrs,
     const IndexType *__restrict__ l_col_idxs, ValueType *__restrict__ l_vals,
     IndexType l_nnz)
+{
+    auto l_nz = thread::get_thread_id_flat<IndexType>();
+    if (l_nz >= l_nnz) {
+        return;
+    }
+    auto row = a_row_idxs[l_nz];
+    auto col = l_col_idxs[l_nz];
+    auto a_val = a_vals[l_nz];
+    auto l_row_begin = l_row_ptrs[row];
+    auto l_row_end = l_row_ptrs[row + 1];
+    auto lh_col_begin = l_row_ptrs[col];
+    auto lh_col_end = l_row_ptrs[col + 1];
+    ValueType sum{};
+    auto last_entry = col;
+    while (l_row_begin < l_row_end && lh_col_begin < lh_col_end) {
+        auto l_col = l_col_idxs[l_row_begin];
+        auto lh_row = l_col_idxs[lh_col_begin];
+        if (l_col == lh_row && l_col < last_entry) {
+            sum += l_vals[l_row_begin] * conj(l_vals[lh_col_begin]);
+        }
+        l_row_begin += l_col <= lh_row;
+        lh_col_begin += l_col >= lh_row;
+    }
+    auto to_write = row == col
+                        ? sqrt(a_val - sum)
+                        : (a_val - sum) / l_vals[l_row_ptrs[col + 1] - 1];
+    if (is_finite(to_write)) {
+        l_vals[l_nz] = to_write;
+    }
+}
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+__global__ __launch_bounds__(default_block_size) void ic_sweep_subwarp(
+    const IndexType *__restrict__ a_row_idxs,
+    const IndexType *__restrict__ a_col_idxs,
+    const ValueType *__restrict__ a_vals,
+    const IndexType *__restrict__ l_row_ptrs,
+    const IndexType *__restrict__ l_col_idxs, ValueType *__restrict__ l_vals,
+    IndexType l_nnz)
 {
     auto l_nz = thread::get_subwarp_id_flat<subwarp_size, IndexType>();
     if (l_nz >= l_nnz) {
@@ -75,7 +115,6 @@ __global__ __launch_bounds__(default_block_size) void ic_sweep(
     auto lh_col_begin = l_row_ptrs[col];
     auto lh_col_size = l_row_ptrs[col + 1] - lh_col_begin;
     ValueType sum{};
-    IndexType lh_nz{};
     auto last_entry = col;
     group_merge<subwarp_size>(
         l_col_idxs + l_row_begin, l_row_size, l_col_idxs + lh_col_begin,
@@ -88,12 +127,6 @@ __global__ __launch_bounds__(default_block_size) void ic_sweep(
                 sum += l_vals[l_idx + l_row_begin] *
                        conj(l_vals[lh_idx + lh_col_begin]);
             }
-            // remember the transposed element
-            auto found_transp = subwarp.ballot(lh_row == row);
-            if (found_transp) {
-                lh_nz =
-                    subwarp.shfl(lh_idx + lh_col_begin, ffs(found_transp) - 1);
-            }
             return true;
         });
     // accumulate result from all threads
diff --git a/cuda/factorization/par_ic_kernels.cu b/cuda/factorization/par_ic_kernels.cu
index 56b00eb0fb1..6f99558ba5b 100644
--- a/cuda/factorization/par_ic_kernels.cu
+++ b/cuda/factorization/par_ic_kernels.cu
@@ -97,11 +97,13 @@ void compute_factor(syn::value_list<int, subwarp_size>,
     auto block_size = default_block_size / subwarp_size;
     auto num_blocks = ceildiv(total_nnz, block_size);
     for (size_type i = 0; i < iterations; ++i) {
-        kernel::ic_sweep<subwarp_size><<<num_blocks, default_block_size>>>(
-            a_lower->get_const_row_idxs(), a_lower->get_const_col_idxs(),
-            as_cuda_type(a_lower->get_const_values()), l->get_const_row_ptrs(),
-            l->get_const_col_idxs(), as_cuda_type(l->get_values()),
-            static_cast<IndexType>(l->get_num_stored_elements()));
+        kernel::ic_sweep_subwarp<subwarp_size>
+            <<<num_blocks, default_block_size>>>(
+                a_lower->get_const_row_idxs(), a_lower->get_const_col_idxs(),
+                as_cuda_type(a_lower->get_const_values()),
+                l->get_const_row_ptrs(), l->get_const_col_idxs(),
+                as_cuda_type(l->get_values()),
+                static_cast<IndexType>(l->get_num_stored_elements()));
     }
 }
 
@@ -118,17 +120,28 @@ void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
                     const matrix::Coo<ValueType, IndexType> *a_lower,
                     matrix::Csr<ValueType, IndexType> *l)
 {
-    auto num_rows = l->get_size()[0];
-    auto total_nnz = 2 * l->get_num_stored_elements();
-    auto total_nnz_per_row = total_nnz / num_rows;
-    select_compute_factor(
-        compiled_kernels(),
-        [&](int compiled_subwarp_size) {
-            return total_nnz_per_row <= compiled_subwarp_size ||
-                   compiled_subwarp_size == config::warp_size;
-        },
-        syn::value_list<int>(), syn::type_list<>(), exec, iterations, a_lower,
-        l);
+    auto nnz = l->get_num_stored_elements();
+    auto work_per_row = 2 * nnz / l->get_size()[0];
+    if (l->get_strategy()->get_name() == "classical") {
+        auto num_blocks = ceildiv(nnz, default_block_size);
+        for (size_type i = 0; i < iterations; ++i) {
+            kernel::ic_sweep<<<num_blocks, default_block_size>>>(
+                a_lower->get_const_row_idxs(), a_lower->get_const_col_idxs(),
+                as_cuda_type(a_lower->get_const_values()),
+                l->get_const_row_ptrs(), l->get_const_col_idxs(),
+                as_cuda_type(l->get_values()),
+                static_cast<IndexType>(l->get_num_stored_elements()));
+        }
+    } else {
+        select_compute_factor(
+            compiled_kernels(),
+            [&](int compiled_subwarp_size) {
+                return work_per_row <= compiled_subwarp_size ||
+                       compiled_subwarp_size == config::warp_size;
+            },
+            syn::value_list<int>(), syn::type_list<>(), exec, iterations,
+            a_lower, l);
+    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
diff --git a/cuda/test/factorization/par_ic_kernels.cpp b/cuda/test/factorization/par_ic_kernels.cpp
index 6577f2c798a..bd504f0119e 100644
--- a/cuda/test/factorization/par_ic_kernels.cpp
+++ b/cuda/test/factorization/par_ic_kernels.cpp
@@ -154,13 +154,32 @@ TEST_F(ParIc, KernelComputeFactorIsEquivalentToRef)
     mtx_l_ani->convert_to(lend(mtx_l_coo));
     auto dmtx_l_coo = Coo::create(cuda, square_size);
     dmtx_l_coo->copy_from(lend(mtx_l_coo));
+    dmtx_l_ani_init->set_strategy(std::make_shared<Csr::classical>());
 
     gko::kernels::reference::par_ic_factorization::compute_factor(
         ref, 1, lend(mtx_l_coo), lend(mtx_l_ani_init));
     gko::kernels::cuda::par_ic_factorization::compute_factor(
         cuda, 20, lend(dmtx_l_coo), lend(dmtx_l_ani_init));
 
-    GKO_ASSERT_MTX_NEAR(mtx_l_ani, dmtx_l_ani, 1e-2);
+    GKO_ASSERT_MTX_NEAR(mtx_l_ani_init, dmtx_l_ani_init, 1e-2);
+}
+
+
+TEST_F(ParIc, KernelComputeFactorSubwarpIsEquivalentToRef)
+{
+    auto square_size = mtx_ani->get_size();
+    auto mtx_l_coo = Coo::create(ref, square_size);
+    mtx_l_ani->convert_to(lend(mtx_l_coo));
+    auto dmtx_l_coo = Coo::create(cuda, square_size);
+    dmtx_l_coo->copy_from(lend(mtx_l_coo));
+    dmtx_l_ani_init->set_strategy(std::make_shared<Csr::merge_path>());
+
+    gko::kernels::reference::par_ic_factorization::compute_factor(
+        ref, 1, lend(mtx_l_coo), lend(mtx_l_ani_init));
+    gko::kernels::cuda::par_ic_factorization::compute_factor(
+        cuda, 20, lend(dmtx_l_coo), lend(dmtx_l_ani_init));
+
+    GKO_ASSERT_MTX_NEAR(mtx_l_ani_init, dmtx_l_ani_init, 1e-2);
 }
 
 
diff --git a/hip/factorization/par_ic_kernels.hip.cpp b/hip/factorization/par_ic_kernels.hip.cpp
index c4c11fda4b5..71aaaf7948e 100644
--- a/hip/factorization/par_ic_kernels.hip.cpp
+++ b/hip/factorization/par_ic_kernels.hip.cpp
@@ -121,17 +121,31 @@ void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
                     const matrix::Coo<ValueType, IndexType> *a_lower,
                     matrix::Csr<ValueType, IndexType> *l)
 {
-    auto num_rows = l->get_size()[0];
-    auto total_nnz = 2 * l->get_num_stored_elements();
-    auto total_nnz_per_row = total_nnz / num_rows;
-    select_compute_factor(
-        compiled_kernels(),
-        [&](int compiled_subwarp_size) {
-            return total_nnz_per_row <= compiled_subwarp_size ||
-                   compiled_subwarp_size == config::warp_size;
-        },
-        syn::value_list<int>(), syn::type_list<>(), exec, iterations, a_lower,
-        l);
+    auto nnz = l->get_num_stored_elements();
+    auto work_per_row = 2 * nnz / l->get_size()[0];
+    if (l->get_strategy()->get_name() == "classical") {
+        auto num_blocks = ceildiv(nnz, default_block_size);
+
+        for (size_type i = 0; i < iterations; ++i) {
+            hipLaunchKernelGGL(
+                HIP_KERNEL_NAME(kernel::ic_sweep), num_blocks,
+                default_block_size, 0, 0, a_lower->get_const_row_idxs(),
+                a_lower->get_const_col_idxs(),
+                as_hip_type(a_lower->get_const_values()),
+                l->get_const_row_ptrs(), l->get_const_col_idxs(),
+                as_hip_type(l->get_values()),
+                static_cast<IndexType>(l->get_num_stored_elements()));
+        }
+    } else {
+        select_compute_factor(
+            compiled_kernels(),
+            [&](int compiled_subwarp_size) {
+                return work_per_row <= compiled_subwarp_size ||
+                       compiled_subwarp_size == config::warp_size;
+            },
+            syn::value_list<int>(), syn::type_list<>(), exec, iterations,
+            a_lower, l);
+    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
diff --git a/hip/test/factorization/par_ic_kernels.hip.cpp b/hip/test/factorization/par_ic_kernels.hip.cpp
index 8a2b06fe0b4..c222346b2b1 100644
--- a/hip/test/factorization/par_ic_kernels.hip.cpp
+++ b/hip/test/factorization/par_ic_kernels.hip.cpp
@@ -154,13 +154,32 @@ TEST_F(ParIc, KernelComputeFactorIsEquivalentToRef)
     mtx_l_ani->convert_to(lend(mtx_l_coo));
     auto dmtx_l_coo = Coo::create(hip, square_size);
     dmtx_l_coo->copy_from(lend(mtx_l_coo));
+    dmtx_l_ani_init->set_strategy(std::make_shared<Csr::classical>());
 
     gko::kernels::reference::par_ic_factorization::compute_factor(
         ref, 1, lend(mtx_l_coo), lend(mtx_l_ani_init));
     gko::kernels::hip::par_ic_factorization::compute_factor(
         hip, 20, lend(dmtx_l_coo), lend(dmtx_l_ani_init));
 
-    GKO_ASSERT_MTX_NEAR(mtx_l_ani, dmtx_l_ani, 1e-2);
+    GKO_ASSERT_MTX_NEAR(mtx_l_ani_init, dmtx_l_ani_init, 1e-2);
+}
+
+
+TEST_F(ParIc, KernelComputeFactorSubwarpIsEquivalentToRef)
+{
+    auto square_size = mtx_ani->get_size();
+    auto mtx_l_coo = Coo::create(ref, square_size);
+    mtx_l_ani->convert_to(lend(mtx_l_coo));
+    auto dmtx_l_coo = Coo::create(hip, square_size);
+    dmtx_l_coo->copy_from(lend(mtx_l_coo));
+    dmtx_l_ani_init->set_strategy(std::make_shared<Csr::merge_path>());
+
+    gko::kernels::reference::par_ic_factorization::compute_factor(
+        ref, 1, lend(mtx_l_coo), lend(mtx_l_ani_init));
+    gko::kernels::hip::par_ic_factorization::compute_factor(
+        hip, 20, lend(dmtx_l_coo), lend(dmtx_l_ani_init));
+
+    GKO_ASSERT_MTX_NEAR(mtx_l_ani_init, dmtx_l_ani_init, 1e-2);
 }
 
 
diff --git a/omp/test/factorization/par_ic_kernels.cpp b/omp/test/factorization/par_ic_kernels.cpp
index 4192d5d2d42..928e6d97e0b 100644
--- a/omp/test/factorization/par_ic_kernels.cpp
+++ b/omp/test/factorization/par_ic_kernels.cpp
@@ -170,7 +170,7 @@ TYPED_TEST(ParIc, KernelComputeFactorIsEquivalentToRef)
     gko::kernels::omp::par_ic_factorization::compute_factor(
         this->omp, 20, lend(dmtx_l_coo), lend(this->dmtx_l_ani_init));
 
-    GKO_ASSERT_MTX_NEAR(this->mtx_l_ani, this->dmtx_l_ani, 1e-2);
+    GKO_ASSERT_MTX_NEAR(this->mtx_l_ani_init, this->dmtx_l_ani_init, 1e-2);
 }
 
 

From 6be3e8ead61e8e8a55fee3b94c76f9c1948d76cd Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 7 Dec 2020 16:34:58 +0100
Subject: [PATCH 0260/1008] fix HIP build

---
 hip/factorization/par_ic_kernels.hip.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hip/factorization/par_ic_kernels.hip.cpp b/hip/factorization/par_ic_kernels.hip.cpp
index 71aaaf7948e..5f35acfccca 100644
--- a/hip/factorization/par_ic_kernels.hip.cpp
+++ b/hip/factorization/par_ic_kernels.hip.cpp
@@ -99,7 +99,7 @@ void compute_factor(syn::value_list<int, subwarp_size>,
     auto num_blocks = ceildiv(total_nnz, block_size);
     for (size_type i = 0; i < iterations; ++i) {
         hipLaunchKernelGGL(
-            HIP_KERNEL_NAME(kernel::ic_sweep<subwarp_size>), num_blocks,
+            HIP_KERNEL_NAME(kernel::ic_sweep_subwarp<subwarp_size>), num_blocks,
             default_block_size, 0, 0, a_lower->get_const_row_idxs(),
             a_lower->get_const_col_idxs(),
             as_hip_type(a_lower->get_const_values()), l->get_const_row_ptrs(),

From 2f9ef8f0edce71e573d9ff9df5b8d0e35bad22d3 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 9 Dec 2020 15:02:24 +0100
Subject: [PATCH 0261/1008] add subwarp-parallel ParILU kernel

---
 common/factorization/par_ilu_kernels.hpp.inc  | 70 ++++++++++++++
 cuda/factorization/par_ilu_kernels.cu         | 91 ++++++++++++++----
 cuda/test/factorization/par_ilu_kernels.cpp   | 24 ++++-
 hip/factorization/par_ilu_kernels.hip.cpp     | 94 +++++++++++++++----
 .../factorization/par_ilu_kernels.hip.cpp     | 24 ++++-
 5 files changed, 264 insertions(+), 39 deletions(-)

diff --git a/common/factorization/par_ilu_kernels.hpp.inc b/common/factorization/par_ilu_kernels.hpp.inc
index f2fdc1468e1..5968225f21f 100644
--- a/common/factorization/par_ilu_kernels.hpp.inc
+++ b/common/factorization/par_ilu_kernels.hpp.inc
@@ -79,4 +79,74 @@ __global__ __launch_bounds__(default_block_size) void compute_l_u_factors(
 }
 
 
+template <int subwarp_size, typename ValueType, typename IndexType>
+__global__
+    __launch_bounds__(default_block_size) void compute_l_u_factors_subwarp(
+        size_type num_elements, const IndexType *__restrict__ row_idxs,
+        const IndexType *__restrict__ col_idxs,
+        const ValueType *__restrict__ values,
+        const IndexType *__restrict__ l_row_ptrs,
+        const IndexType *__restrict__ l_col_idxs,
+        ValueType *__restrict__ l_vals,
+        const IndexType *__restrict__ ut_col_ptrs,
+        const IndexType *__restrict__ ut_row_idxs,
+        ValueType *__restrict__ ut_vals)
+{
+    auto tidx = thread::get_subwarp_id_flat<subwarp_size, IndexType>();
+    if (tidx >= num_elements) {
+        return;
+    }
+    auto row = row_idxs[tidx];
+    auto col = col_idxs[tidx];
+    auto subwarp =
+        group::tiled_partition<subwarp_size>(group::this_thread_block());
+    auto a_val = values[tidx];
+    auto l_row_begin = l_row_ptrs[row];
+    auto l_row_size = l_row_ptrs[row + 1] - l_row_begin;
+    auto ut_col_begin = ut_col_ptrs[col];
+    auto ut_col_size = ut_col_ptrs[col + 1] - ut_col_begin;
+    ValueType sum{};
+    IndexType out_nz{};
+    bool lower = col < row;
+    auto last_entry = min(row, col);
+    group_merge<subwarp_size>(
+        l_col_idxs + l_row_begin, l_row_size, ut_row_idxs + ut_col_begin,
+        ut_col_size, subwarp,
+        [&](IndexType l_idx, IndexType l_col, IndexType ut_idx,
+            IndexType ut_row, IndexType, bool) {
+            // we don't need to use the `bool valid` because last_entry is
+            // already a smaller sentinel value than the one used in group_merge
+            if (l_col == ut_row && l_col < last_entry) {
+                sum += l_vals[l_idx + l_row_begin] *
+                       ut_vals[ut_idx + ut_col_begin];
+            }
+            // remember the output element
+            bool found = lower ? (l_col == col) : (ut_row == row);
+            auto found_mask = subwarp.ballot(found);
+            if (found_mask) {
+                auto out_local =
+                    lower ? (l_idx + l_row_begin) : (ut_idx + ut_col_begin);
+                out_nz = subwarp.shfl(out_local, ffs(found_mask) - 1);
+            }
+            return true;
+        });
+    // accumulate result from all threads
+    sum = reduce(subwarp, sum, [](ValueType a, ValueType b) { return a + b; });
+
+    if (subwarp.thread_rank() == 0) {
+        if (lower) {
+            auto to_write = (a_val - sum) / ut_vals[ut_col_ptrs[col + 1] - 1];
+            if (is_finite(to_write)) {
+                l_vals[out_nz] = to_write;
+            }
+        } else {
+            auto to_write = a_val - sum;
+            if (is_finite(to_write)) {
+                ut_vals[out_nz] = to_write;
+            }
+        }
+    }
+}
+
+
 }  // namespace kernel
diff --git a/cuda/factorization/par_ilu_kernels.cu b/cuda/factorization/par_ilu_kernels.cu
index 8113543a90a..316c34d1ec7 100644
--- a/cuda/factorization/par_ilu_kernels.cu
+++ b/cuda/factorization/par_ilu_kernels.cu
@@ -36,8 +36,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/coo.hpp>
 
 
+#include "core/synthesizer/implementation_selection.hpp"
 #include "cuda/base/math.hpp"
 #include "cuda/base/types.hpp"
+#include "cuda/components/merging.cuh"
+#include "cuda/components/reduction.cuh"
 #include "cuda/components/thread_ids.cuh"
 
 
@@ -55,9 +58,51 @@ namespace par_ilu_factorization {
 constexpr int default_block_size{512};
 
 
+// subwarp sizes for all warp-parallel kernels (sweep)
+using compiled_kernels =
+    syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
+
+
 #include "common/factorization/par_ilu_kernels.hpp.inc"
 
 
+namespace {
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+void compute_l_u_factors(syn::value_list<int, subwarp_size>,
+                         std::shared_ptr<const CudaExecutor> exec,
+                         size_type iterations,
+                         const matrix::Coo<ValueType, IndexType> *system_matrix,
+                         matrix::Csr<ValueType, IndexType> *l_factor,
+                         matrix::Csr<ValueType, IndexType> *u_factor)
+{
+    auto total_nnz =
+        static_cast<IndexType>(system_matrix->get_num_stored_elements());
+    auto block_size = default_block_size / subwarp_size;
+    auto num_blocks = ceildiv(total_nnz, block_size);
+    for (size_type i = 0; i < iterations; ++i) {
+        kernel::compute_l_u_factors_subwarp<subwarp_size>
+            <<<num_blocks, default_block_size, 0, 0>>>(
+                system_matrix->get_num_stored_elements(),
+                system_matrix->get_const_row_idxs(),
+                system_matrix->get_const_col_idxs(),
+                as_cuda_type(system_matrix->get_const_values()),
+                l_factor->get_const_row_ptrs(), l_factor->get_const_col_idxs(),
+                as_cuda_type(l_factor->get_values()),
+                u_factor->get_const_row_ptrs(), u_factor->get_const_col_idxs(),
+                as_cuda_type(u_factor->get_values()));
+    }
+}
+
+
+GKO_ENABLE_IMPLEMENTATION_SELECTION(select_compute_l_u_factors,
+                                    compute_l_u_factors);
+
+
+}  // namespace
+
+
 template <typename ValueType, typename IndexType>
 void compute_l_u_factors(std::shared_ptr<const CudaExecutor> exec,
                          size_type iterations,
@@ -66,23 +111,35 @@ void compute_l_u_factors(std::shared_ptr<const CudaExecutor> exec,
                          matrix::Csr<ValueType, IndexType> *u_factor)
 {
     iterations = (iterations == 0) ? 10 : iterations;
-    const auto num_elements = system_matrix->get_num_stored_elements();
-    const dim3 block_size{default_block_size, 1, 1};
-    const dim3 grid_dim{
-        static_cast<uint32>(
-            ceildiv(num_elements, static_cast<size_type>(block_size.x))),
-        1, 1};
-    for (size_type i = 0; i < iterations; ++i) {
-        kernel::compute_l_u_factors<<<grid_dim, block_size, 0, 0>>>(
-            num_elements, as_cuda_type(system_matrix->get_const_row_idxs()),
-            as_cuda_type(system_matrix->get_const_col_idxs()),
-            as_cuda_type(system_matrix->get_const_values()),
-            as_cuda_type(l_factor->get_const_row_ptrs()),
-            as_cuda_type(l_factor->get_const_col_idxs()),
-            as_cuda_type(l_factor->get_values()),
-            as_cuda_type(u_factor->get_const_row_ptrs()),
-            as_cuda_type(u_factor->get_const_col_idxs()),
-            as_cuda_type(u_factor->get_values()));
+    if (l_factor->get_strategy()->get_name() == "classical") {
+        const auto num_elements = system_matrix->get_num_stored_elements();
+        const dim3 block_size{default_block_size, 1, 1};
+        const dim3 grid_dim{
+            static_cast<uint32>(
+                ceildiv(num_elements, static_cast<size_type>(block_size.x))),
+            1, 1};
+        for (size_type i = 0; i < iterations; ++i) {
+            kernel::compute_l_u_factors<<<grid_dim, block_size, 0, 0>>>(
+                num_elements, system_matrix->get_const_row_idxs(),
+                system_matrix->get_const_col_idxs(),
+                as_cuda_type(system_matrix->get_const_values()),
+                l_factor->get_const_row_ptrs(), l_factor->get_const_col_idxs(),
+                as_cuda_type(l_factor->get_values()),
+                u_factor->get_const_row_ptrs(), u_factor->get_const_col_idxs(),
+                as_cuda_type(u_factor->get_values()));
+        }
+    } else {
+        auto work = l_factor->get_num_stored_elements() +
+                    u_factor->get_num_stored_elements();
+        auto work_per_row = work / system_matrix->get_size()[0];
+        select_compute_l_u_factors(
+            compiled_kernels(),
+            [&](int compiled_subwarp_size) {
+                return work_per_row <= compiled_subwarp_size ||
+                       compiled_subwarp_size == config::warp_size;
+            },
+            syn::value_list<int>(), syn::type_list<>(), exec, iterations,
+            system_matrix, l_factor, u_factor);
     }
 }
 
diff --git a/cuda/test/factorization/par_ilu_kernels.cpp b/cuda/test/factorization/par_ilu_kernels.cpp
index 47c95061834..303ca5d0bdb 100644
--- a/cuda/test/factorization/par_ilu_kernels.cpp
+++ b/cuda/test/factorization/par_ilu_kernels.cpp
@@ -193,7 +193,9 @@ class ParIlu : public ::testing::Test {
 
     void compute_lu(std::unique_ptr<Csr> *l_ref, std::unique_ptr<Csr> *u_ref,
                     std::unique_ptr<Csr> *l_cuda, std::unique_ptr<Csr> *u_cuda,
-                    gko::size_type iterations = 0)
+                    gko::size_type iterations = 0,
+                    std::shared_ptr<Csr::strategy_type> strat =
+                        std::make_shared<Csr::sparselib>())
     {
         auto coo_ref = Coo::create(ref);
         csr_ref->convert_to(gko::lend(coo_ref));
@@ -206,6 +208,7 @@ class ParIlu : public ::testing::Test {
         auto u_transpose_lin_op_cuda = (*u_cuda)->transpose();
         auto u_transpose_csr_cuda =
             static_unique_ptr_cast<Csr>(std::move(u_transpose_lin_op_cuda));
+        (*l_cuda)->set_strategy(strat);
 
         gko::kernels::reference::par_ilu_factorization::compute_l_u_factors(
             ref, iterations, gko::lend(coo_ref), gko::lend(*l_ref),
@@ -311,7 +314,7 @@ TEST_F(ParIlu, KernelInitializeParILUIsEquivalentToRef)
 }
 
 
-TEST_F(ParIlu, KernelComputeParILUIsEquivalentToRef)
+TEST_F(ParIlu, KernelComputeParILUSubwarpIsEquivalentToRef)
 {
     std::unique_ptr<Csr> l_ref{};
     std::unique_ptr<Csr> u_ref{};
@@ -327,6 +330,23 @@ TEST_F(ParIlu, KernelComputeParILUIsEquivalentToRef)
 }
 
 
+TEST_F(ParIlu, KernelComputeParILUIsEquivalentToRef)
+{
+    std::unique_ptr<Csr> l_ref{};
+    std::unique_ptr<Csr> u_ref{};
+    std::unique_ptr<Csr> l_cuda{};
+    std::unique_ptr<Csr> u_cuda{};
+
+    compute_lu(&l_ref, &u_ref, &l_cuda, &u_cuda, 0,
+               std::make_shared<Csr::classical>());
+
+    GKO_ASSERT_MTX_NEAR(l_ref, l_cuda, 5e-2);
+    GKO_ASSERT_MTX_NEAR(u_ref, u_cuda, 5e-2);
+    GKO_ASSERT_MTX_EQ_SPARSITY(l_ref, l_cuda);
+    GKO_ASSERT_MTX_EQ_SPARSITY(u_ref, u_cuda);
+}
+
+
 TEST_F(ParIlu, KernelComputeParILUWithMoreIterationsIsEquivalentToRef)
 {
     std::unique_ptr<Csr> l_ref{};
diff --git a/hip/factorization/par_ilu_kernels.hip.cpp b/hip/factorization/par_ilu_kernels.hip.cpp
index 5dcc174ebc0..1504ca76590 100644
--- a/hip/factorization/par_ilu_kernels.hip.cpp
+++ b/hip/factorization/par_ilu_kernels.hip.cpp
@@ -42,6 +42,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "hip/base/math.hip.hpp"
 #include "hip/base/types.hip.hpp"
+#include "hip/components/merging.hip.hpp"
+#include "hip/components/reduction.hip.hpp"
 #include "hip/components/thread_ids.hip.hpp"
 
 
@@ -59,9 +61,51 @@ namespace par_ilu_factorization {
 constexpr int default_block_size{512};
 
 
+// subwarp sizes for all warp-parallel kernels (sweep)
+using compiled_kernels =
+    syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
+
+
 #include "common/factorization/par_ilu_kernels.hpp.inc"
 
 
+namespace {
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+void compute_l_u_factors(syn::value_list<int, subwarp_size>,
+                         std::shared_ptr<const CudaExecutor> exec,
+                         size_type iterations,
+                         const matrix::Coo<ValueType, IndexType> *system_matrix,
+                         matrix::Csr<ValueType, IndexType> *l_factor,
+                         matrix::Csr<ValueType, IndexType> *u_factor)
+{
+    auto total_nnz =
+        static_cast<IndexType>(system_matrix->get_num_stored_elements());
+    auto block_size = default_block_size / subwarp_size;
+    auto num_blocks = ceildiv(total_nnz, block_size);
+    for (size_type i = 0; i < iterations; ++i) {
+        kernel::compute_l_u_factors_subwarp<subwarp_size>
+            <<<num_blocks, default_block_size, 0, 0>>>(
+                system_matrix->get_num_stored_elements(),
+                system_matrix->get_const_row_idxs(),
+                system_matrix->get_const_col_idxs(),
+                as_hip_type(system_matrix->get_const_values()),
+                l_factor->get_const_row_ptrs(), l_factor->get_const_col_idxs(),
+                as_hip_type(l_factor->get_values()),
+                u_factor->get_const_row_ptrs(), u_factor->get_const_col_idxs(),
+                as_hip_type(u_factor->get_values()));
+    }
+}
+
+
+GKO_ENABLE_IMPLEMENTATION_SELECTION(select_compute_l_u_factors,
+                                    compute_l_u_factors);
+
+
+}  // namespace
+
+
 template <typename ValueType, typename IndexType>
 void compute_l_u_factors(std::shared_ptr<const HipExecutor> exec,
                          size_type iterations,
@@ -70,24 +114,38 @@ void compute_l_u_factors(std::shared_ptr<const HipExecutor> exec,
                          matrix::Csr<ValueType, IndexType> *u_factor)
 {
     iterations = (iterations == 0) ? 10 : iterations;
-    const auto num_elements = system_matrix->get_num_stored_elements();
-    const dim3 block_size{default_block_size, 1, 1};
-    const dim3 grid_dim{
-        static_cast<uint32>(
-            ceildiv(num_elements, static_cast<size_type>(block_size.x))),
-        1, 1};
-    for (size_type i = 0; i < iterations; ++i) {
-        hipLaunchKernelGGL(kernel::compute_l_u_factors, dim3(grid_dim),
-                           dim3(block_size), 0, 0, num_elements,
-                           as_hip_type(system_matrix->get_const_row_idxs()),
-                           as_hip_type(system_matrix->get_const_col_idxs()),
-                           as_hip_type(system_matrix->get_const_values()),
-                           as_hip_type(l_factor->get_const_row_ptrs()),
-                           as_hip_type(l_factor->get_const_col_idxs()),
-                           as_hip_type(l_factor->get_values()),
-                           as_hip_type(u_factor->get_const_row_ptrs()),
-                           as_hip_type(u_factor->get_const_col_idxs()),
-                           as_hip_type(u_factor->get_values()));
+    if (l_factor->get_strategy()->get_name() == "classical") {
+        const auto num_elements = system_matrix->get_num_stored_elements();
+        const dim3 block_size{default_block_size, 1, 1};
+        const dim3 grid_dim{
+            static_cast<uint32>(
+                ceildiv(num_elements, static_cast<size_type>(block_size.x))),
+            1, 1};
+        for (size_type i = 0; i < iterations; ++i) {
+            hipLaunchKernelGGL(
+                kernel::compute_l_u_factors, dim3(grid_dim), dim3(block_size),
+                0, 0, num_elements, system_matrix->get_const_row_idxs(),
+                system_matrix->get_const_col_idxs(),
+                as_hip_type(system_matrix->get_const_values()),
+                l_factor->get_const_row_ptrs(), l_factor->get_const_col_idxs(),
+                as_hip_type(l_factor->get_values()),
+                u_factor->get_const_row_ptrs(), u_factor->get_const_col_idxs(),
+                as_hip_type(u_factor->get_values()));
+        }
+        else
+        {
+            auto work = l_factor->get_num_stored_elements() +
+                        u_factor->get_num_stored_elements();
+            auto work_per_row = work / system_matrix->get_size()[0];
+            select_compute_l_u_factors(
+                compiled_kernels(),
+                [&](int compiled_subwarp_size) {
+                    return work_per_row <= compiled_subwarp_size ||
+                           compiled_subwarp_size == config::warp_size;
+                },
+                syn::value_list<int>(), syn::type_list<>(), exec, iterations,
+                system_matrix, l_factor, u_factor);
+        }
     }
 }
 
diff --git a/hip/test/factorization/par_ilu_kernels.hip.cpp b/hip/test/factorization/par_ilu_kernels.hip.cpp
index 98811f97af3..a4b98c8b068 100644
--- a/hip/test/factorization/par_ilu_kernels.hip.cpp
+++ b/hip/test/factorization/par_ilu_kernels.hip.cpp
@@ -192,7 +192,9 @@ class ParIlu : public ::testing::Test {
 
     void compute_lu(std::unique_ptr<Csr> *l_ref, std::unique_ptr<Csr> *u_ref,
                     std::unique_ptr<Csr> *l_hip, std::unique_ptr<Csr> *u_hip,
-                    gko::size_type iterations = 0)
+                    gko::size_type iterations = 0,
+                    std::shared_ptr<Csr::strategy_type> strat =
+                        std::make_shared<Csr::sparselib>())
     {
         auto coo_ref = Coo::create(ref);
         csr_ref->convert_to(gko::lend(coo_ref));
@@ -205,6 +207,7 @@ class ParIlu : public ::testing::Test {
         auto u_transpose_lin_op_hip = (*u_hip)->transpose();
         auto u_transpose_csr_hip =
             static_unique_ptr_cast<Csr>(std::move(u_transpose_lin_op_hip));
+        (*l_hip)->set_strategy(strat);
 
         gko::kernels::reference::par_ilu_factorization::compute_l_u_factors(
             ref, iterations, gko::lend(coo_ref), gko::lend(*l_ref),
@@ -313,7 +316,7 @@ TEST_F(ParIlu, KernelInitializeParILUIsEquivalentToRef)
 }
 
 
-TEST_F(ParIlu, KernelComputeParILUIsEquivalentToRef)
+TEST_F(ParIlu, KernelComputeParILUSubwarpIsEquivalentToRef)
 {
     std::unique_ptr<Csr> l_ref{};
     std::unique_ptr<Csr> u_ref{};
@@ -329,6 +332,23 @@ TEST_F(ParIlu, KernelComputeParILUIsEquivalentToRef)
 }
 
 
+TEST_F(ParIlu, KernelComputeParILUIsEquivalentToRef)
+{
+    std::unique_ptr<Csr> l_ref{};
+    std::unique_ptr<Csr> u_ref{};
+    std::unique_ptr<Csr> l_hip{};
+    std::unique_ptr<Csr> u_hip{};
+
+    compute_lu(&l_ref, &u_ref, &l_hip, &u_hip, 0,
+               std::make_shared<Csr::classical>());
+
+    GKO_ASSERT_MTX_NEAR(l_ref, l_hip, 5e-2);
+    GKO_ASSERT_MTX_NEAR(u_ref, u_hip, 5e-2);
+    GKO_ASSERT_MTX_EQ_SPARSITY(l_ref, l_hip);
+    GKO_ASSERT_MTX_EQ_SPARSITY(u_ref, u_hip);
+}
+
+
 TEST_F(ParIlu, KernelComputeParILUWithMoreIterationsIsEquivalentToRef)
 {
     std::unique_ptr<Csr> l_ref{};

From 7b0a741fef5dda1e31f9bc13aa6de77375db11a0 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 10 Dec 2020 16:38:31 +0100
Subject: [PATCH 0262/1008] add l_strategy to par_ilu convergence ex

---
 .../par-ilu-convergence.cpp                   | 33 +++++++++++--------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/examples/par-ilu-convergence/par-ilu-convergence.cpp b/examples/par-ilu-convergence/par-ilu-convergence.cpp
index 23f9111e4c3..55e83bb0ba3 100644
--- a/examples/par-ilu-convergence/par-ilu-convergence.cpp
+++ b/examples/par-ilu-convergence/par-ilu-convergence.cpp
@@ -59,14 +59,6 @@ const std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
          }}};
 
 
-void output(const gko::WritableToMatrixData<double, int> *mtx, std::string name)
-{
-    std::ofstream stream{name};
-    std::cerr << "Writing " << name << std::endl;
-    gko::write(stream, mtx, gko::layout_type::coordinate);
-}
-
-
 template <typename Function>
 auto try_generate(Function fun) -> decltype(fun())
 {
@@ -108,10 +100,11 @@ int main(int argc, char *argv[])
 {
     // print usage message
     if (argc < 2 || executors.find(argv[1]) == executors.end()) {
-        std::cerr << "Usage: executable"
-                  << " <reference|omp|cuda|hip|dpcpp> [<matrix-file>] "
-                     "[<parilu|parilut|paric|parict] [<max-iterations>] "
-                     "[<num-repetitions>] [fill-in-limit]\n";
+        std::cerr
+            << "Usage: executable"
+            << " <reference|omp|cuda|hip|dpcpp> [<matrix-file>] "
+               "[<parilu|parilut|paric|parict] [<max-iterations>] "
+               "[<num-repetitions>] [<fill-in-limit>] [<sequential-kernels>]\n";
         return -1;
     }
 
@@ -124,6 +117,7 @@ int main(int argc, char *argv[])
     int max_iterations = argc < 5 ? 10 : std::stoi(argv[4]);
     int num_repetitions = argc < 6 ? 10 : std::stoi(argv[5]);
     double limit = argc < 7 ? 2 : std::stod(argv[6]);
+    bool sequential = argc < 8 ? false : std::stoi(argv[7]);
 
     // load matrix file into Csr format
     auto mtx = gko::share(try_generate([&] {
@@ -135,17 +129,28 @@ int main(int argc, char *argv[])
         return gko::read<gko::matrix::Csr<>>(mtx_stream, exec);
     }));
 
+    std::shared_ptr<gko::matrix::Csr<>::strategy_type> strategy;
+    if (sequential) {
+        strategy = std::make_shared<gko::matrix::Csr<>::classical>();
+    } else {
+        strategy = std::make_shared<gko::matrix::Csr<>::sparselib>();
+    }
+
     std::shared_ptr<gko::LinOpFactory> factory;
     std::function<void(int)> set_iterations;
     if (precond == "parilu") {
-        factory = gko::factorization::ParIlu<>::build().on(exec);
+        factory =
+            gko::factorization::ParIlu<>::build().with_l_strategy(strategy).on(
+                exec);
         set_iterations = [&](int it) {
             gko::as<gko::factorization::ParIlu<>::Factory>(factory)
                 ->get_parameters()
                 .iterations = it;
         };
     } else if (precond == "paric") {
-        factory = gko::factorization::ParIc<>::build().on(exec);
+        factory =
+            gko::factorization::ParIc<>::build().with_l_strategy(strategy).on(
+                exec);
         set_iterations = [&](int it) {
             gko::as<gko::factorization::ParIc<>::Factory>(factory)
                 ->get_parameters()

From 6c9d23073cc17f38b24bea77f641adc6cd503551 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 10 Dec 2020 17:59:39 +0100
Subject: [PATCH 0263/1008] Revert "add l_strategy to par_ilu convergence ex"

---
 .../par-ilu-convergence.cpp                   | 25 +++++--------------
 1 file changed, 6 insertions(+), 19 deletions(-)

diff --git a/examples/par-ilu-convergence/par-ilu-convergence.cpp b/examples/par-ilu-convergence/par-ilu-convergence.cpp
index 55e83bb0ba3..77395a159b8 100644
--- a/examples/par-ilu-convergence/par-ilu-convergence.cpp
+++ b/examples/par-ilu-convergence/par-ilu-convergence.cpp
@@ -100,11 +100,10 @@ int main(int argc, char *argv[])
 {
     // print usage message
     if (argc < 2 || executors.find(argv[1]) == executors.end()) {
-        std::cerr
-            << "Usage: executable"
-            << " <reference|omp|cuda|hip|dpcpp> [<matrix-file>] "
-               "[<parilu|parilut|paric|parict] [<max-iterations>] "
-               "[<num-repetitions>] [<fill-in-limit>] [<sequential-kernels>]\n";
+        std::cerr << "Usage: executable"
+                  << " <reference|omp|cuda|hip|dpcpp> [<matrix-file>] "
+                     "[<parilu|parilut|paric|parict] [<max-iterations>] "
+                     "[<num-repetitions>] [<fill-in-limit>]\n";
         return -1;
     }
 
@@ -117,7 +116,6 @@ int main(int argc, char *argv[])
     int max_iterations = argc < 5 ? 10 : std::stoi(argv[4]);
     int num_repetitions = argc < 6 ? 10 : std::stoi(argv[5]);
     double limit = argc < 7 ? 2 : std::stod(argv[6]);
-    bool sequential = argc < 8 ? false : std::stoi(argv[7]);
 
     // load matrix file into Csr format
     auto mtx = gko::share(try_generate([&] {
@@ -129,28 +127,17 @@ int main(int argc, char *argv[])
         return gko::read<gko::matrix::Csr<>>(mtx_stream, exec);
     }));
 
-    std::shared_ptr<gko::matrix::Csr<>::strategy_type> strategy;
-    if (sequential) {
-        strategy = std::make_shared<gko::matrix::Csr<>::classical>();
-    } else {
-        strategy = std::make_shared<gko::matrix::Csr<>::sparselib>();
-    }
-
     std::shared_ptr<gko::LinOpFactory> factory;
     std::function<void(int)> set_iterations;
     if (precond == "parilu") {
-        factory =
-            gko::factorization::ParIlu<>::build().with_l_strategy(strategy).on(
-                exec);
+        factory = gko::factorization::ParIlu<>::build().on(exec);
         set_iterations = [&](int it) {
             gko::as<gko::factorization::ParIlu<>::Factory>(factory)
                 ->get_parameters()
                 .iterations = it;
         };
     } else if (precond == "paric") {
-        factory =
-            gko::factorization::ParIc<>::build().with_l_strategy(strategy).on(
-                exec);
+        factory = gko::factorization::ParIc<>::build().on(exec);
         set_iterations = [&](int it) {
             gko::as<gko::factorization::ParIc<>::Factory>(factory)
                 ->get_parameters()

From 12188b66b308f7f2221c53fefe763298ec01f281 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 10 Dec 2020 18:20:40 +0100
Subject: [PATCH 0264/1008] remove subwarp-parallel incomplete fact kernels

---
 common/factorization/par_ic_kernels.hpp.inc   | 51 ----------
 common/factorization/par_ilu_kernels.hpp.inc  | 70 --------------
 cuda/factorization/par_ic_kernels.cu          | 62 ++-----------
 cuda/factorization/par_ilu_kernels.cu         | 89 +++---------------
 cuda/test/factorization/par_ic_kernels.cpp    | 19 ----
 cuda/test/factorization/par_ilu_kernels.cpp   | 24 +----
 hip/factorization/par_ic_kernels.hip.cpp      | 62 ++-----------
 hip/factorization/par_ilu_kernels.hip.cpp     | 92 ++++---------------
 hip/test/factorization/par_ic_kernels.hip.cpp | 19 ----
 .../factorization/par_ilu_kernels.hip.cpp     | 24 +----
 10 files changed, 50 insertions(+), 462 deletions(-)

diff --git a/common/factorization/par_ic_kernels.hpp.inc b/common/factorization/par_ic_kernels.hpp.inc
index 70d9fd9e70e..fc5ddfdb703 100644
--- a/common/factorization/par_ic_kernels.hpp.inc
+++ b/common/factorization/par_ic_kernels.hpp.inc
@@ -92,55 +92,4 @@ __global__ __launch_bounds__(default_block_size) void ic_sweep(
 }
 
 
-template <int subwarp_size, typename ValueType, typename IndexType>
-__global__ __launch_bounds__(default_block_size) void ic_sweep_subwarp(
-    const IndexType *__restrict__ a_row_idxs,
-    const IndexType *__restrict__ a_col_idxs,
-    const ValueType *__restrict__ a_vals,
-    const IndexType *__restrict__ l_row_ptrs,
-    const IndexType *__restrict__ l_col_idxs, ValueType *__restrict__ l_vals,
-    IndexType l_nnz)
-{
-    auto l_nz = thread::get_subwarp_id_flat<subwarp_size, IndexType>();
-    if (l_nz >= l_nnz) {
-        return;
-    }
-    auto row = a_row_idxs[l_nz];
-    auto col = l_col_idxs[l_nz];
-    auto subwarp =
-        group::tiled_partition<subwarp_size>(group::this_thread_block());
-    auto a_val = a_vals[l_nz];
-    auto l_row_begin = l_row_ptrs[row];
-    auto l_row_size = l_row_ptrs[row + 1] - l_row_begin;
-    auto lh_col_begin = l_row_ptrs[col];
-    auto lh_col_size = l_row_ptrs[col + 1] - lh_col_begin;
-    ValueType sum{};
-    auto last_entry = col;
-    group_merge<subwarp_size>(
-        l_col_idxs + l_row_begin, l_row_size, l_col_idxs + lh_col_begin,
-        lh_col_size, subwarp,
-        [&](IndexType l_idx, IndexType l_col, IndexType lh_idx,
-            IndexType lh_row, IndexType, bool) {
-            // we don't need to use the `bool valid` because last_entry is
-            // already a smaller sentinel value than the one used in group_merge
-            if (l_col == lh_row && l_col < last_entry) {
-                sum += l_vals[l_idx + l_row_begin] *
-                       conj(l_vals[lh_idx + lh_col_begin]);
-            }
-            return true;
-        });
-    // accumulate result from all threads
-    sum = reduce(subwarp, sum, [](ValueType a, ValueType b) { return a + b; });
-
-    if (subwarp.thread_rank() == 0) {
-        auto to_write = row == col
-                            ? sqrt(a_val - sum)
-                            : (a_val - sum) / l_vals[l_row_ptrs[col + 1] - 1];
-        if (is_finite(to_write)) {
-            l_vals[l_nz] = to_write;
-        }
-    }
-}
-
-
 }  // namespace kernel
diff --git a/common/factorization/par_ilu_kernels.hpp.inc b/common/factorization/par_ilu_kernels.hpp.inc
index 5968225f21f..f2fdc1468e1 100644
--- a/common/factorization/par_ilu_kernels.hpp.inc
+++ b/common/factorization/par_ilu_kernels.hpp.inc
@@ -79,74 +79,4 @@ __global__ __launch_bounds__(default_block_size) void compute_l_u_factors(
 }
 
 
-template <int subwarp_size, typename ValueType, typename IndexType>
-__global__
-    __launch_bounds__(default_block_size) void compute_l_u_factors_subwarp(
-        size_type num_elements, const IndexType *__restrict__ row_idxs,
-        const IndexType *__restrict__ col_idxs,
-        const ValueType *__restrict__ values,
-        const IndexType *__restrict__ l_row_ptrs,
-        const IndexType *__restrict__ l_col_idxs,
-        ValueType *__restrict__ l_vals,
-        const IndexType *__restrict__ ut_col_ptrs,
-        const IndexType *__restrict__ ut_row_idxs,
-        ValueType *__restrict__ ut_vals)
-{
-    auto tidx = thread::get_subwarp_id_flat<subwarp_size, IndexType>();
-    if (tidx >= num_elements) {
-        return;
-    }
-    auto row = row_idxs[tidx];
-    auto col = col_idxs[tidx];
-    auto subwarp =
-        group::tiled_partition<subwarp_size>(group::this_thread_block());
-    auto a_val = values[tidx];
-    auto l_row_begin = l_row_ptrs[row];
-    auto l_row_size = l_row_ptrs[row + 1] - l_row_begin;
-    auto ut_col_begin = ut_col_ptrs[col];
-    auto ut_col_size = ut_col_ptrs[col + 1] - ut_col_begin;
-    ValueType sum{};
-    IndexType out_nz{};
-    bool lower = col < row;
-    auto last_entry = min(row, col);
-    group_merge<subwarp_size>(
-        l_col_idxs + l_row_begin, l_row_size, ut_row_idxs + ut_col_begin,
-        ut_col_size, subwarp,
-        [&](IndexType l_idx, IndexType l_col, IndexType ut_idx,
-            IndexType ut_row, IndexType, bool) {
-            // we don't need to use the `bool valid` because last_entry is
-            // already a smaller sentinel value than the one used in group_merge
-            if (l_col == ut_row && l_col < last_entry) {
-                sum += l_vals[l_idx + l_row_begin] *
-                       ut_vals[ut_idx + ut_col_begin];
-            }
-            // remember the output element
-            bool found = lower ? (l_col == col) : (ut_row == row);
-            auto found_mask = subwarp.ballot(found);
-            if (found_mask) {
-                auto out_local =
-                    lower ? (l_idx + l_row_begin) : (ut_idx + ut_col_begin);
-                out_nz = subwarp.shfl(out_local, ffs(found_mask) - 1);
-            }
-            return true;
-        });
-    // accumulate result from all threads
-    sum = reduce(subwarp, sum, [](ValueType a, ValueType b) { return a + b; });
-
-    if (subwarp.thread_rank() == 0) {
-        if (lower) {
-            auto to_write = (a_val - sum) / ut_vals[ut_col_ptrs[col + 1] - 1];
-            if (is_finite(to_write)) {
-                l_vals[out_nz] = to_write;
-            }
-        } else {
-            auto to_write = a_val - sum;
-            if (is_finite(to_write)) {
-                ut_vals[out_nz] = to_write;
-            }
-        }
-    }
-}
-
-
 }  // namespace kernel
diff --git a/cuda/factorization/par_ic_kernels.cu b/cuda/factorization/par_ic_kernels.cu
index 6f99558ba5b..9c42b27ad09 100644
--- a/cuda/factorization/par_ic_kernels.cu
+++ b/cuda/factorization/par_ic_kernels.cu
@@ -38,10 +38,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/csr.hpp>
 
 
-#include "core/synthesizer/implementation_selection.hpp"
 #include "cuda/base/math.hpp"
-#include "cuda/components/merging.cuh"
-#include "cuda/components/reduction.cuh"
+#include "cuda/base/types.hpp"
 #include "cuda/components/thread_ids.cuh"
 
 
@@ -83,37 +81,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_PAR_IC_INIT_FACTOR_KERNEL);
 
 
-namespace {
-
-
-template <int subwarp_size, typename ValueType, typename IndexType>
-void compute_factor(syn::value_list<int, subwarp_size>,
-                    std::shared_ptr<const DefaultExecutor> exec,
-                    size_type iterations,
-                    const matrix::Coo<ValueType, IndexType> *a_lower,
-                    matrix::Csr<ValueType, IndexType> *l)
-{
-    auto total_nnz = static_cast<IndexType>(l->get_num_stored_elements());
-    auto block_size = default_block_size / subwarp_size;
-    auto num_blocks = ceildiv(total_nnz, block_size);
-    for (size_type i = 0; i < iterations; ++i) {
-        kernel::ic_sweep_subwarp<subwarp_size>
-            <<<num_blocks, default_block_size>>>(
-                a_lower->get_const_row_idxs(), a_lower->get_const_col_idxs(),
-                as_cuda_type(a_lower->get_const_values()),
-                l->get_const_row_ptrs(), l->get_const_col_idxs(),
-                as_cuda_type(l->get_values()),
-                static_cast<IndexType>(l->get_num_stored_elements()));
-    }
-}
-
-
-GKO_ENABLE_IMPLEMENTATION_SELECTION(select_compute_factor, compute_factor);
-
-
-}  // namespace
-
-
 template <typename ValueType, typename IndexType>
 void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
                     size_type iterations,
@@ -121,26 +88,13 @@ void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
                     matrix::Csr<ValueType, IndexType> *l)
 {
     auto nnz = l->get_num_stored_elements();
-    auto work_per_row = 2 * nnz / l->get_size()[0];
-    if (l->get_strategy()->get_name() == "classical") {
-        auto num_blocks = ceildiv(nnz, default_block_size);
-        for (size_type i = 0; i < iterations; ++i) {
-            kernel::ic_sweep<<<num_blocks, default_block_size>>>(
-                a_lower->get_const_row_idxs(), a_lower->get_const_col_idxs(),
-                as_cuda_type(a_lower->get_const_values()),
-                l->get_const_row_ptrs(), l->get_const_col_idxs(),
-                as_cuda_type(l->get_values()),
-                static_cast<IndexType>(l->get_num_stored_elements()));
-        }
-    } else {
-        select_compute_factor(
-            compiled_kernels(),
-            [&](int compiled_subwarp_size) {
-                return work_per_row <= compiled_subwarp_size ||
-                       compiled_subwarp_size == config::warp_size;
-            },
-            syn::value_list<int>(), syn::type_list<>(), exec, iterations,
-            a_lower, l);
+    auto num_blocks = ceildiv(nnz, default_block_size);
+    for (size_type i = 0; i < iterations; ++i) {
+        kernel::ic_sweep<<<num_blocks, default_block_size>>>(
+            a_lower->get_const_row_idxs(), a_lower->get_const_col_idxs(),
+            as_cuda_type(a_lower->get_const_values()), l->get_const_row_ptrs(),
+            l->get_const_col_idxs(), as_cuda_type(l->get_values()),
+            static_cast<IndexType>(l->get_num_stored_elements()));
     }
 }
 
diff --git a/cuda/factorization/par_ilu_kernels.cu b/cuda/factorization/par_ilu_kernels.cu
index 316c34d1ec7..9d26f587a7d 100644
--- a/cuda/factorization/par_ilu_kernels.cu
+++ b/cuda/factorization/par_ilu_kernels.cu
@@ -36,11 +36,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/coo.hpp>
 
 
-#include "core/synthesizer/implementation_selection.hpp"
 #include "cuda/base/math.hpp"
 #include "cuda/base/types.hpp"
-#include "cuda/components/merging.cuh"
-#include "cuda/components/reduction.cuh"
 #include "cuda/components/thread_ids.cuh"
 
 
@@ -58,51 +55,9 @@ namespace par_ilu_factorization {
 constexpr int default_block_size{512};
 
 
-// subwarp sizes for all warp-parallel kernels (sweep)
-using compiled_kernels =
-    syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
-
-
 #include "common/factorization/par_ilu_kernels.hpp.inc"
 
 
-namespace {
-
-
-template <int subwarp_size, typename ValueType, typename IndexType>
-void compute_l_u_factors(syn::value_list<int, subwarp_size>,
-                         std::shared_ptr<const CudaExecutor> exec,
-                         size_type iterations,
-                         const matrix::Coo<ValueType, IndexType> *system_matrix,
-                         matrix::Csr<ValueType, IndexType> *l_factor,
-                         matrix::Csr<ValueType, IndexType> *u_factor)
-{
-    auto total_nnz =
-        static_cast<IndexType>(system_matrix->get_num_stored_elements());
-    auto block_size = default_block_size / subwarp_size;
-    auto num_blocks = ceildiv(total_nnz, block_size);
-    for (size_type i = 0; i < iterations; ++i) {
-        kernel::compute_l_u_factors_subwarp<subwarp_size>
-            <<<num_blocks, default_block_size, 0, 0>>>(
-                system_matrix->get_num_stored_elements(),
-                system_matrix->get_const_row_idxs(),
-                system_matrix->get_const_col_idxs(),
-                as_cuda_type(system_matrix->get_const_values()),
-                l_factor->get_const_row_ptrs(), l_factor->get_const_col_idxs(),
-                as_cuda_type(l_factor->get_values()),
-                u_factor->get_const_row_ptrs(), u_factor->get_const_col_idxs(),
-                as_cuda_type(u_factor->get_values()));
-    }
-}
-
-
-GKO_ENABLE_IMPLEMENTATION_SELECTION(select_compute_l_u_factors,
-                                    compute_l_u_factors);
-
-
-}  // namespace
-
-
 template <typename ValueType, typename IndexType>
 void compute_l_u_factors(std::shared_ptr<const CudaExecutor> exec,
                          size_type iterations,
@@ -111,35 +66,21 @@ void compute_l_u_factors(std::shared_ptr<const CudaExecutor> exec,
                          matrix::Csr<ValueType, IndexType> *u_factor)
 {
     iterations = (iterations == 0) ? 10 : iterations;
-    if (l_factor->get_strategy()->get_name() == "classical") {
-        const auto num_elements = system_matrix->get_num_stored_elements();
-        const dim3 block_size{default_block_size, 1, 1};
-        const dim3 grid_dim{
-            static_cast<uint32>(
-                ceildiv(num_elements, static_cast<size_type>(block_size.x))),
-            1, 1};
-        for (size_type i = 0; i < iterations; ++i) {
-            kernel::compute_l_u_factors<<<grid_dim, block_size, 0, 0>>>(
-                num_elements, system_matrix->get_const_row_idxs(),
-                system_matrix->get_const_col_idxs(),
-                as_cuda_type(system_matrix->get_const_values()),
-                l_factor->get_const_row_ptrs(), l_factor->get_const_col_idxs(),
-                as_cuda_type(l_factor->get_values()),
-                u_factor->get_const_row_ptrs(), u_factor->get_const_col_idxs(),
-                as_cuda_type(u_factor->get_values()));
-        }
-    } else {
-        auto work = l_factor->get_num_stored_elements() +
-                    u_factor->get_num_stored_elements();
-        auto work_per_row = work / system_matrix->get_size()[0];
-        select_compute_l_u_factors(
-            compiled_kernels(),
-            [&](int compiled_subwarp_size) {
-                return work_per_row <= compiled_subwarp_size ||
-                       compiled_subwarp_size == config::warp_size;
-            },
-            syn::value_list<int>(), syn::type_list<>(), exec, iterations,
-            system_matrix, l_factor, u_factor);
+    const auto num_elements = system_matrix->get_num_stored_elements();
+    const dim3 block_size{default_block_size, 1, 1};
+    const dim3 grid_dim{
+        static_cast<uint32>(
+            ceildiv(num_elements, static_cast<size_type>(block_size.x))),
+        1, 1};
+    for (size_type i = 0; i < iterations; ++i) {
+        kernel::compute_l_u_factors<<<grid_dim, block_size, 0, 0>>>(
+            num_elements, system_matrix->get_const_row_idxs(),
+            system_matrix->get_const_col_idxs(),
+            as_cuda_type(system_matrix->get_const_values()),
+            l_factor->get_const_row_ptrs(), l_factor->get_const_col_idxs(),
+            as_cuda_type(l_factor->get_values()),
+            u_factor->get_const_row_ptrs(), u_factor->get_const_col_idxs(),
+            as_cuda_type(u_factor->get_values()));
     }
 }
 
diff --git a/cuda/test/factorization/par_ic_kernels.cpp b/cuda/test/factorization/par_ic_kernels.cpp
index bd504f0119e..606e58dac70 100644
--- a/cuda/test/factorization/par_ic_kernels.cpp
+++ b/cuda/test/factorization/par_ic_kernels.cpp
@@ -154,25 +154,6 @@ TEST_F(ParIc, KernelComputeFactorIsEquivalentToRef)
     mtx_l_ani->convert_to(lend(mtx_l_coo));
     auto dmtx_l_coo = Coo::create(cuda, square_size);
     dmtx_l_coo->copy_from(lend(mtx_l_coo));
-    dmtx_l_ani_init->set_strategy(std::make_shared<Csr::classical>());
-
-    gko::kernels::reference::par_ic_factorization::compute_factor(
-        ref, 1, lend(mtx_l_coo), lend(mtx_l_ani_init));
-    gko::kernels::cuda::par_ic_factorization::compute_factor(
-        cuda, 20, lend(dmtx_l_coo), lend(dmtx_l_ani_init));
-
-    GKO_ASSERT_MTX_NEAR(mtx_l_ani_init, dmtx_l_ani_init, 1e-2);
-}
-
-
-TEST_F(ParIc, KernelComputeFactorSubwarpIsEquivalentToRef)
-{
-    auto square_size = mtx_ani->get_size();
-    auto mtx_l_coo = Coo::create(ref, square_size);
-    mtx_l_ani->convert_to(lend(mtx_l_coo));
-    auto dmtx_l_coo = Coo::create(cuda, square_size);
-    dmtx_l_coo->copy_from(lend(mtx_l_coo));
-    dmtx_l_ani_init->set_strategy(std::make_shared<Csr::merge_path>());
 
     gko::kernels::reference::par_ic_factorization::compute_factor(
         ref, 1, lend(mtx_l_coo), lend(mtx_l_ani_init));
diff --git a/cuda/test/factorization/par_ilu_kernels.cpp b/cuda/test/factorization/par_ilu_kernels.cpp
index 303ca5d0bdb..47c95061834 100644
--- a/cuda/test/factorization/par_ilu_kernels.cpp
+++ b/cuda/test/factorization/par_ilu_kernels.cpp
@@ -193,9 +193,7 @@ class ParIlu : public ::testing::Test {
 
     void compute_lu(std::unique_ptr<Csr> *l_ref, std::unique_ptr<Csr> *u_ref,
                     std::unique_ptr<Csr> *l_cuda, std::unique_ptr<Csr> *u_cuda,
-                    gko::size_type iterations = 0,
-                    std::shared_ptr<Csr::strategy_type> strat =
-                        std::make_shared<Csr::sparselib>())
+                    gko::size_type iterations = 0)
     {
         auto coo_ref = Coo::create(ref);
         csr_ref->convert_to(gko::lend(coo_ref));
@@ -208,7 +206,6 @@ class ParIlu : public ::testing::Test {
         auto u_transpose_lin_op_cuda = (*u_cuda)->transpose();
         auto u_transpose_csr_cuda =
             static_unique_ptr_cast<Csr>(std::move(u_transpose_lin_op_cuda));
-        (*l_cuda)->set_strategy(strat);
 
         gko::kernels::reference::par_ilu_factorization::compute_l_u_factors(
             ref, iterations, gko::lend(coo_ref), gko::lend(*l_ref),
@@ -314,22 +311,6 @@ TEST_F(ParIlu, KernelInitializeParILUIsEquivalentToRef)
 }
 
 
-TEST_F(ParIlu, KernelComputeParILUSubwarpIsEquivalentToRef)
-{
-    std::unique_ptr<Csr> l_ref{};
-    std::unique_ptr<Csr> u_ref{};
-    std::unique_ptr<Csr> l_cuda{};
-    std::unique_ptr<Csr> u_cuda{};
-
-    compute_lu(&l_ref, &u_ref, &l_cuda, &u_cuda);
-
-    GKO_ASSERT_MTX_NEAR(l_ref, l_cuda, 5e-2);
-    GKO_ASSERT_MTX_NEAR(u_ref, u_cuda, 5e-2);
-    GKO_ASSERT_MTX_EQ_SPARSITY(l_ref, l_cuda);
-    GKO_ASSERT_MTX_EQ_SPARSITY(u_ref, u_cuda);
-}
-
-
 TEST_F(ParIlu, KernelComputeParILUIsEquivalentToRef)
 {
     std::unique_ptr<Csr> l_ref{};
@@ -337,8 +318,7 @@ TEST_F(ParIlu, KernelComputeParILUIsEquivalentToRef)
     std::unique_ptr<Csr> l_cuda{};
     std::unique_ptr<Csr> u_cuda{};
 
-    compute_lu(&l_ref, &u_ref, &l_cuda, &u_cuda, 0,
-               std::make_shared<Csr::classical>());
+    compute_lu(&l_ref, &u_ref, &l_cuda, &u_cuda);
 
     GKO_ASSERT_MTX_NEAR(l_ref, l_cuda, 5e-2);
     GKO_ASSERT_MTX_NEAR(u_ref, u_cuda, 5e-2);
diff --git a/hip/factorization/par_ic_kernels.hip.cpp b/hip/factorization/par_ic_kernels.hip.cpp
index 5f35acfccca..39c80c57650 100644
--- a/hip/factorization/par_ic_kernels.hip.cpp
+++ b/hip/factorization/par_ic_kernels.hip.cpp
@@ -38,10 +38,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/csr.hpp>
 
 
-#include "core/synthesizer/implementation_selection.hpp"
 #include "hip/base/math.hip.hpp"
-#include "hip/components/merging.hip.hpp"
-#include "hip/components/reduction.hip.hpp"
+#include "hip/base/types.hip.hpp"
 #include "hip/components/thread_ids.hip.hpp"
 
 
@@ -84,70 +82,24 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_PAR_IC_INIT_FACTOR_KERNEL);
 
 
-namespace {
-
-
-template <int subwarp_size, typename ValueType, typename IndexType>
-void compute_factor(syn::value_list<int, subwarp_size>,
-                    std::shared_ptr<const DefaultExecutor> exec,
+template <typename ValueType, typename IndexType>
+void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
                     size_type iterations,
                     const matrix::Coo<ValueType, IndexType> *a_lower,
                     matrix::Csr<ValueType, IndexType> *l)
 {
-    auto total_nnz = static_cast<IndexType>(l->get_num_stored_elements());
-    auto block_size = default_block_size / subwarp_size;
-    auto num_blocks = ceildiv(total_nnz, block_size);
+    auto nnz = l->get_num_stored_elements();
+    auto num_blocks = ceildiv(nnz, default_block_size);
     for (size_type i = 0; i < iterations; ++i) {
         hipLaunchKernelGGL(
-            HIP_KERNEL_NAME(kernel::ic_sweep_subwarp<subwarp_size>), num_blocks,
-            default_block_size, 0, 0, a_lower->get_const_row_idxs(),
-            a_lower->get_const_col_idxs(),
+            HIP_KERNEL_NAME(kernel::ic_sweep), num_blocks, default_block_size,
+            0, 0, a_lower->get_const_row_idxs(), a_lower->get_const_col_idxs(),
             as_hip_type(a_lower->get_const_values()), l->get_const_row_ptrs(),
             l->get_const_col_idxs(), as_hip_type(l->get_values()),
             static_cast<IndexType>(l->get_num_stored_elements()));
     }
 }
 
-
-GKO_ENABLE_IMPLEMENTATION_SELECTION(select_compute_factor, compute_factor);
-
-
-}  // namespace
-
-
-template <typename ValueType, typename IndexType>
-void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
-                    size_type iterations,
-                    const matrix::Coo<ValueType, IndexType> *a_lower,
-                    matrix::Csr<ValueType, IndexType> *l)
-{
-    auto nnz = l->get_num_stored_elements();
-    auto work_per_row = 2 * nnz / l->get_size()[0];
-    if (l->get_strategy()->get_name() == "classical") {
-        auto num_blocks = ceildiv(nnz, default_block_size);
-
-        for (size_type i = 0; i < iterations; ++i) {
-            hipLaunchKernelGGL(
-                HIP_KERNEL_NAME(kernel::ic_sweep), num_blocks,
-                default_block_size, 0, 0, a_lower->get_const_row_idxs(),
-                a_lower->get_const_col_idxs(),
-                as_hip_type(a_lower->get_const_values()),
-                l->get_const_row_ptrs(), l->get_const_col_idxs(),
-                as_hip_type(l->get_values()),
-                static_cast<IndexType>(l->get_num_stored_elements()));
-        }
-    } else {
-        select_compute_factor(
-            compiled_kernels(),
-            [&](int compiled_subwarp_size) {
-                return work_per_row <= compiled_subwarp_size ||
-                       compiled_subwarp_size == config::warp_size;
-            },
-            syn::value_list<int>(), syn::type_list<>(), exec, iterations,
-            a_lower, l);
-    }
-}
-
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_PAR_IC_COMPUTE_FACTOR_KERNEL);
 
diff --git a/hip/factorization/par_ilu_kernels.hip.cpp b/hip/factorization/par_ilu_kernels.hip.cpp
index 1504ca76590..081ac0fe317 100644
--- a/hip/factorization/par_ilu_kernels.hip.cpp
+++ b/hip/factorization/par_ilu_kernels.hip.cpp
@@ -42,8 +42,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "hip/base/math.hip.hpp"
 #include "hip/base/types.hip.hpp"
-#include "hip/components/merging.hip.hpp"
-#include "hip/components/reduction.hip.hpp"
 #include "hip/components/thread_ids.hip.hpp"
 
 
@@ -61,51 +59,9 @@ namespace par_ilu_factorization {
 constexpr int default_block_size{512};
 
 
-// subwarp sizes for all warp-parallel kernels (sweep)
-using compiled_kernels =
-    syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
-
-
 #include "common/factorization/par_ilu_kernels.hpp.inc"
 
 
-namespace {
-
-
-template <int subwarp_size, typename ValueType, typename IndexType>
-void compute_l_u_factors(syn::value_list<int, subwarp_size>,
-                         std::shared_ptr<const CudaExecutor> exec,
-                         size_type iterations,
-                         const matrix::Coo<ValueType, IndexType> *system_matrix,
-                         matrix::Csr<ValueType, IndexType> *l_factor,
-                         matrix::Csr<ValueType, IndexType> *u_factor)
-{
-    auto total_nnz =
-        static_cast<IndexType>(system_matrix->get_num_stored_elements());
-    auto block_size = default_block_size / subwarp_size;
-    auto num_blocks = ceildiv(total_nnz, block_size);
-    for (size_type i = 0; i < iterations; ++i) {
-        kernel::compute_l_u_factors_subwarp<subwarp_size>
-            <<<num_blocks, default_block_size, 0, 0>>>(
-                system_matrix->get_num_stored_elements(),
-                system_matrix->get_const_row_idxs(),
-                system_matrix->get_const_col_idxs(),
-                as_hip_type(system_matrix->get_const_values()),
-                l_factor->get_const_row_ptrs(), l_factor->get_const_col_idxs(),
-                as_hip_type(l_factor->get_values()),
-                u_factor->get_const_row_ptrs(), u_factor->get_const_col_idxs(),
-                as_hip_type(u_factor->get_values()));
-    }
-}
-
-
-GKO_ENABLE_IMPLEMENTATION_SELECTION(select_compute_l_u_factors,
-                                    compute_l_u_factors);
-
-
-}  // namespace
-
-
 template <typename ValueType, typename IndexType>
 void compute_l_u_factors(std::shared_ptr<const HipExecutor> exec,
                          size_type iterations,
@@ -114,38 +70,22 @@ void compute_l_u_factors(std::shared_ptr<const HipExecutor> exec,
                          matrix::Csr<ValueType, IndexType> *u_factor)
 {
     iterations = (iterations == 0) ? 10 : iterations;
-    if (l_factor->get_strategy()->get_name() == "classical") {
-        const auto num_elements = system_matrix->get_num_stored_elements();
-        const dim3 block_size{default_block_size, 1, 1};
-        const dim3 grid_dim{
-            static_cast<uint32>(
-                ceildiv(num_elements, static_cast<size_type>(block_size.x))),
-            1, 1};
-        for (size_type i = 0; i < iterations; ++i) {
-            hipLaunchKernelGGL(
-                kernel::compute_l_u_factors, dim3(grid_dim), dim3(block_size),
-                0, 0, num_elements, system_matrix->get_const_row_idxs(),
-                system_matrix->get_const_col_idxs(),
-                as_hip_type(system_matrix->get_const_values()),
-                l_factor->get_const_row_ptrs(), l_factor->get_const_col_idxs(),
-                as_hip_type(l_factor->get_values()),
-                u_factor->get_const_row_ptrs(), u_factor->get_const_col_idxs(),
-                as_hip_type(u_factor->get_values()));
-        }
-        else
-        {
-            auto work = l_factor->get_num_stored_elements() +
-                        u_factor->get_num_stored_elements();
-            auto work_per_row = work / system_matrix->get_size()[0];
-            select_compute_l_u_factors(
-                compiled_kernels(),
-                [&](int compiled_subwarp_size) {
-                    return work_per_row <= compiled_subwarp_size ||
-                           compiled_subwarp_size == config::warp_size;
-                },
-                syn::value_list<int>(), syn::type_list<>(), exec, iterations,
-                system_matrix, l_factor, u_factor);
-        }
+    const auto num_elements = system_matrix->get_num_stored_elements();
+    const dim3 block_size{default_block_size, 1, 1};
+    const dim3 grid_dim{
+        static_cast<uint32>(
+            ceildiv(num_elements, static_cast<size_type>(block_size.x))),
+        1, 1};
+    for (size_type i = 0; i < iterations; ++i) {
+        hipLaunchKernelGGL(
+            kernel::compute_l_u_factors, dim3(grid_dim), dim3(block_size), 0, 0,
+            num_elements, system_matrix->get_const_row_idxs(),
+            system_matrix->get_const_col_idxs(),
+            as_hip_type(system_matrix->get_const_values()),
+            l_factor->get_const_row_ptrs(), l_factor->get_const_col_idxs(),
+            as_hip_type(l_factor->get_values()), u_factor->get_const_row_ptrs(),
+            u_factor->get_const_col_idxs(),
+            as_hip_type(u_factor->get_values()));
     }
 }
 
diff --git a/hip/test/factorization/par_ic_kernels.hip.cpp b/hip/test/factorization/par_ic_kernels.hip.cpp
index c222346b2b1..d7bc838121a 100644
--- a/hip/test/factorization/par_ic_kernels.hip.cpp
+++ b/hip/test/factorization/par_ic_kernels.hip.cpp
@@ -154,25 +154,6 @@ TEST_F(ParIc, KernelComputeFactorIsEquivalentToRef)
     mtx_l_ani->convert_to(lend(mtx_l_coo));
     auto dmtx_l_coo = Coo::create(hip, square_size);
     dmtx_l_coo->copy_from(lend(mtx_l_coo));
-    dmtx_l_ani_init->set_strategy(std::make_shared<Csr::classical>());
-
-    gko::kernels::reference::par_ic_factorization::compute_factor(
-        ref, 1, lend(mtx_l_coo), lend(mtx_l_ani_init));
-    gko::kernels::hip::par_ic_factorization::compute_factor(
-        hip, 20, lend(dmtx_l_coo), lend(dmtx_l_ani_init));
-
-    GKO_ASSERT_MTX_NEAR(mtx_l_ani_init, dmtx_l_ani_init, 1e-2);
-}
-
-
-TEST_F(ParIc, KernelComputeFactorSubwarpIsEquivalentToRef)
-{
-    auto square_size = mtx_ani->get_size();
-    auto mtx_l_coo = Coo::create(ref, square_size);
-    mtx_l_ani->convert_to(lend(mtx_l_coo));
-    auto dmtx_l_coo = Coo::create(hip, square_size);
-    dmtx_l_coo->copy_from(lend(mtx_l_coo));
-    dmtx_l_ani_init->set_strategy(std::make_shared<Csr::merge_path>());
 
     gko::kernels::reference::par_ic_factorization::compute_factor(
         ref, 1, lend(mtx_l_coo), lend(mtx_l_ani_init));
diff --git a/hip/test/factorization/par_ilu_kernels.hip.cpp b/hip/test/factorization/par_ilu_kernels.hip.cpp
index a4b98c8b068..98811f97af3 100644
--- a/hip/test/factorization/par_ilu_kernels.hip.cpp
+++ b/hip/test/factorization/par_ilu_kernels.hip.cpp
@@ -192,9 +192,7 @@ class ParIlu : public ::testing::Test {
 
     void compute_lu(std::unique_ptr<Csr> *l_ref, std::unique_ptr<Csr> *u_ref,
                     std::unique_ptr<Csr> *l_hip, std::unique_ptr<Csr> *u_hip,
-                    gko::size_type iterations = 0,
-                    std::shared_ptr<Csr::strategy_type> strat =
-                        std::make_shared<Csr::sparselib>())
+                    gko::size_type iterations = 0)
     {
         auto coo_ref = Coo::create(ref);
         csr_ref->convert_to(gko::lend(coo_ref));
@@ -207,7 +205,6 @@ class ParIlu : public ::testing::Test {
         auto u_transpose_lin_op_hip = (*u_hip)->transpose();
         auto u_transpose_csr_hip =
             static_unique_ptr_cast<Csr>(std::move(u_transpose_lin_op_hip));
-        (*l_hip)->set_strategy(strat);
 
         gko::kernels::reference::par_ilu_factorization::compute_l_u_factors(
             ref, iterations, gko::lend(coo_ref), gko::lend(*l_ref),
@@ -316,22 +313,6 @@ TEST_F(ParIlu, KernelInitializeParILUIsEquivalentToRef)
 }
 
 
-TEST_F(ParIlu, KernelComputeParILUSubwarpIsEquivalentToRef)
-{
-    std::unique_ptr<Csr> l_ref{};
-    std::unique_ptr<Csr> u_ref{};
-    std::unique_ptr<Csr> l_hip{};
-    std::unique_ptr<Csr> u_hip{};
-
-    compute_lu(&l_ref, &u_ref, &l_hip, &u_hip);
-
-    GKO_ASSERT_MTX_NEAR(l_ref, l_hip, 5e-2);
-    GKO_ASSERT_MTX_NEAR(u_ref, u_hip, 5e-2);
-    GKO_ASSERT_MTX_EQ_SPARSITY(l_ref, l_hip);
-    GKO_ASSERT_MTX_EQ_SPARSITY(u_ref, u_hip);
-}
-
-
 TEST_F(ParIlu, KernelComputeParILUIsEquivalentToRef)
 {
     std::unique_ptr<Csr> l_ref{};
@@ -339,8 +320,7 @@ TEST_F(ParIlu, KernelComputeParILUIsEquivalentToRef)
     std::unique_ptr<Csr> l_hip{};
     std::unique_ptr<Csr> u_hip{};
 
-    compute_lu(&l_ref, &u_ref, &l_hip, &u_hip, 0,
-               std::make_shared<Csr::classical>());
+    compute_lu(&l_ref, &u_ref, &l_hip, &u_hip);
 
     GKO_ASSERT_MTX_NEAR(l_ref, l_hip, 5e-2);
     GKO_ASSERT_MTX_NEAR(u_ref, u_hip, 5e-2);

From 062121fb4802ee4eb135249007d66a5a825d71d4 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 21 Jan 2021 15:30:58 +0100
Subject: [PATCH 0265/1008] Review updates

* rename get_lh_factor -> get_lt_factor
* add missing and remove superfluous includes
* update Doxygen documentation
* factor out has_with_criteria trait for solvers
* higher iteration count and accuracy for GPU sweeps
* fix dpcpp build failure
* remove unused code in ParIC sweep
* add type aliases to ParILU convergence example

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Pratik Nayak <pratik.nayak@kit.edu>
Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
---
 common/factorization/par_ic_kernels.hpp.inc   | 12 +--
 core/factorization/ic.cpp                     |  1 +
 core/factorization/ic_kernels.hpp             |  2 -
 core/factorization/par_ic.cpp                 |  1 +
 core/factorization/par_ic_kernels.hpp         |  3 +-
 core/test/factorization/par_ic.cpp            | 46 +++++----
 core/test/factorization/par_ict.cpp           | 65 ++++++++-----
 core/test/factorization/par_ilu.cpp           | 51 ++++++----
 core/test/factorization/par_ilut.cpp          | 74 ++++++++------
 cuda/test/factorization/ic_kernels.cpp        | 14 +--
 cuda/test/factorization/par_ic_kernels.cpp    |  4 +-
 dpcpp/CMakeLists.txt                          |  2 +-
 .../par-ilu-convergence.cpp                   | 54 +++++++----
 hip/test/factorization/ic_kernels.cpp         | 14 +--
 hip/test/factorization/par_ic_kernels.hip.cpp |  4 +-
 include/ginkgo/core/factorization/ic.hpp      |  9 +-
 include/ginkgo/core/factorization/ilu.hpp     |  7 --
 include/ginkgo/core/factorization/par_ic.hpp  | 15 +--
 include/ginkgo/core/factorization/par_ict.hpp |  2 +-
 include/ginkgo/core/factorization/par_ilu.hpp | 13 +--
 include/ginkgo/core/preconditioner/ic.hpp     | 97 ++++++-------------
 include/ginkgo/core/preconditioner/ilu.hpp    | 55 ++---------
 include/ginkgo/core/solver/solver_traits.hpp  | 87 +++++++++++++++++
 include/ginkgo/ginkgo.hpp                     |  1 +
 omp/factorization/par_ic_kernels.cpp          |  4 -
 omp/test/factorization/par_ic_kernels.cpp     |  4 +-
 reference/factorization/par_ic_kernels.cpp    |  4 -
 .../test/factorization/par_ic_kernels.cpp     | 15 +--
 .../test/factorization/par_ict_kernels.cpp    | 17 ++--
 29 files changed, 365 insertions(+), 312 deletions(-)
 create mode 100644 include/ginkgo/core/solver/solver_traits.hpp

diff --git a/common/factorization/par_ic_kernels.hpp.inc b/common/factorization/par_ic_kernels.hpp.inc
index fc5ddfdb703..f18e6ca5d71 100644
--- a/common/factorization/par_ic_kernels.hpp.inc
+++ b/common/factorization/par_ic_kernels.hpp.inc
@@ -61,17 +61,17 @@ __global__ __launch_bounds__(default_block_size) void ic_sweep(
     const IndexType *__restrict__ l_col_idxs, ValueType *__restrict__ l_vals,
     IndexType l_nnz)
 {
-    auto l_nz = thread::get_thread_id_flat<IndexType>();
+    const auto l_nz = thread::get_thread_id_flat<IndexType>();
     if (l_nz >= l_nnz) {
         return;
     }
-    auto row = a_row_idxs[l_nz];
-    auto col = l_col_idxs[l_nz];
-    auto a_val = a_vals[l_nz];
+    const auto row = a_row_idxs[l_nz];
+    const auto col = l_col_idxs[l_nz];
+    const auto a_val = a_vals[l_nz];
     auto l_row_begin = l_row_ptrs[row];
-    auto l_row_end = l_row_ptrs[row + 1];
+    const auto l_row_end = l_row_ptrs[row + 1];
     auto lh_col_begin = l_row_ptrs[col];
-    auto lh_col_end = l_row_ptrs[col + 1];
+    const auto lh_col_end = l_row_ptrs[col + 1];
     ValueType sum{};
     auto last_entry = col;
     while (l_row_begin < l_row_end && lh_col_begin < lh_col_end) {
diff --git a/core/factorization/ic.cpp b/core/factorization/ic.cpp
index e89754d29e7..56b65d2be2a 100644
--- a/core/factorization/ic.cpp
+++ b/core/factorization/ic.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/composition.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 
 
diff --git a/core/factorization/ic_kernels.hpp b/core/factorization/ic_kernels.hpp
index 2160a444a4c..8962e3a9110 100644
--- a/core/factorization/ic_kernels.hpp
+++ b/core/factorization/ic_kernels.hpp
@@ -41,7 +41,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 
 
@@ -53,7 +52,6 @@ namespace kernels {
     void compute(std::shared_ptr<const DefaultExecutor> exec, \
                  matrix::Csr<ValueType, IndexType> *system_matrix)
 
-
 #define GKO_DECLARE_ALL_AS_TEMPLATES                  \
     template <typename ValueType, typename IndexType> \
     GKO_DECLARE_IC_COMPUTE_KERNEL(ValueType, IndexType)
diff --git a/core/factorization/par_ic.cpp b/core/factorization/par_ic.cpp
index ddbe1424f6e..f4c3217889c 100644
--- a/core/factorization/par_ic.cpp
+++ b/core/factorization/par_ic.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/composition.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/polymorphic_object.hpp>
 #include <ginkgo/core/base/types.hpp>
diff --git a/core/factorization/par_ic_kernels.hpp b/core/factorization/par_ic_kernels.hpp
index 5a64861f853..02815276e4f 100644
--- a/core/factorization/par_ic_kernels.hpp
+++ b/core/factorization/par_ic_kernels.hpp
@@ -42,12 +42,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 
 
 namespace gko {
 namespace kernels {
 
+
 #define GKO_DECLARE_PAR_IC_INIT_FACTOR_KERNEL(ValueType, IndexType) \
     void init_factor(std::shared_ptr<const DefaultExecutor> exec,   \
                      matrix::Csr<ValueType, IndexType> *l_factor)
@@ -58,7 +60,6 @@ namespace kernels {
         const matrix::Coo<ValueType, IndexType> *lower_system_matrix,      \
         matrix::Csr<ValueType, IndexType> *l_factor)
 
-
 #define GKO_DECLARE_ALL_AS_TEMPLATES                             \
     template <typename ValueType, typename IndexType>            \
     GKO_DECLARE_PAR_IC_INIT_FACTOR_KERNEL(ValueType, IndexType); \
diff --git a/core/test/factorization/par_ic.cpp b/core/test/factorization/par_ic.cpp
index b99f60cc77d..e78cfaa9a15 100644
--- a/core/test/factorization/par_ic.cpp
+++ b/core/test/factorization/par_ic.cpp
@@ -39,15 +39,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
+#include "core/test/utils.hpp"
+
+
 namespace {
 
 
+template <typename ValueIndexType>
 class ParIc : public ::testing::Test {
 public:
-    using value_type = double;
-    using index_type = gko::int32;
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using ic_factory_type = gko::factorization::ParIc<value_type, index_type>;
-    using strategy_type = ic_factory_type::matrix_type::classical;
+    using strategy_type = typename ic_factory_type::matrix_type::classical;
 
 protected:
     ParIc() : ref(gko::ReferenceExecutor::create()) {}
@@ -55,47 +61,53 @@ class ParIc : public ::testing::Test {
     std::shared_ptr<const gko::ReferenceExecutor> ref;
 };
 
+TYPED_TEST_SUITE(ParIc, gko::test::ValueIndexTypes);
 
-TEST_F(ParIc, SetIterations)
+
+TYPED_TEST(ParIc, SetIterations)
 {
-    auto factory = ic_factory_type::build().with_iterations(5u).on(this->ref);
+    auto factory =
+        TestFixture::ic_factory_type::build().with_iterations(5u).on(this->ref);
 
     ASSERT_EQ(factory->get_parameters().iterations, 5u);
 }
 
 
-TEST_F(ParIc, SetSkip)
+TYPED_TEST(ParIc, SetSkip)
 {
     auto factory =
-        ic_factory_type::build().with_skip_sorting(true).on(this->ref);
+        TestFixture::ic_factory_type::build().with_skip_sorting(true).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().skip_sorting, true);
 }
 
 
-TEST_F(ParIc, SetLStrategy)
+TYPED_TEST(ParIc, SetLStrategy)
 {
-    auto strategy = std::make_shared<strategy_type>();
+    auto strategy = std::make_shared<typename TestFixture::strategy_type>();
 
     auto factory =
-        ic_factory_type::build().with_l_strategy(strategy).on(this->ref);
+        TestFixture::ic_factory_type::build().with_l_strategy(strategy).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().l_strategy, strategy);
 }
 
 
-TEST_F(ParIc, SetBothFactors)
+TYPED_TEST(ParIc, SetBothFactors)
 {
     auto factory =
-        ic_factory_type::build().with_both_factors(false).on(this->ref);
+        TestFixture::ic_factory_type::build().with_both_factors(false).on(
+            this->ref);
 
     ASSERT_FALSE(factory->get_parameters().both_factors);
 }
 
 
-TEST_F(ParIc, SetDefaults)
+TYPED_TEST(ParIc, SetDefaults)
 {
-    auto factory = ic_factory_type::build().on(this->ref);
+    auto factory = TestFixture::ic_factory_type::build().on(this->ref);
 
     ASSERT_EQ(factory->get_parameters().iterations, 0u);
     ASSERT_EQ(factory->get_parameters().skip_sorting, false);
@@ -104,11 +116,11 @@ TEST_F(ParIc, SetDefaults)
 }
 
 
-TEST_F(ParIc, SetEverything)
+TYPED_TEST(ParIc, SetEverything)
 {
-    auto strategy = std::make_shared<strategy_type>();
+    auto strategy = std::make_shared<typename TestFixture::strategy_type>();
 
-    auto factory = ic_factory_type::build()
+    auto factory = TestFixture::ic_factory_type::build()
                        .with_iterations(7u)
                        .with_skip_sorting(false)
                        .with_l_strategy(strategy)
diff --git a/core/test/factorization/par_ict.cpp b/core/test/factorization/par_ict.cpp
index 6f11c33e25e..296f62bb9ec 100644
--- a/core/test/factorization/par_ict.cpp
+++ b/core/test/factorization/par_ict.cpp
@@ -39,15 +39,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
+#include "core/test/utils.hpp"
+
+
 namespace {
 
 
+template <typename ValueIndexType>
 class ParIct : public ::testing::Test {
 public:
-    using value_type = gko::default_precision;
-    using index_type = gko::int32;
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using ict_factory_type = gko::factorization::ParIct<value_type, index_type>;
-    using strategy_type = ict_factory_type::matrix_type::classical;
+    using strategy_type = typename ict_factory_type::matrix_type::classical;
 
 protected:
     ParIct() : ref(gko::ReferenceExecutor::create()) {}
@@ -55,63 +61,74 @@ class ParIct : public ::testing::Test {
     std::shared_ptr<const gko::ReferenceExecutor> ref;
 };
 
+TYPED_TEST_SUITE(ParIct, gko::test::ValueIndexTypes);
+
 
-TEST_F(ParIct, SetIterations)
+TYPED_TEST(ParIct, SetIterations)
 {
-    auto factory = ict_factory_type::build().with_iterations(6u).on(ref);
+    auto factory =
+        TestFixture::ict_factory_type::build().with_iterations(6u).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().iterations, 6u);
 }
 
 
-TEST_F(ParIct, SetSkip)
+TYPED_TEST(ParIct, SetSkip)
 {
-    auto factory = ict_factory_type::build().with_skip_sorting(true).on(ref);
+    auto factory =
+        TestFixture::ict_factory_type::build().with_skip_sorting(true).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().skip_sorting, true);
 }
 
 
-TEST_F(ParIct, SetApprox)
+TYPED_TEST(ParIct, SetApprox)
 {
-    auto factory =
-        ict_factory_type::build().with_approximate_select(false).on(ref);
+    auto factory = TestFixture::ict_factory_type::build()
+                       .with_approximate_select(false)
+                       .on(this->ref);
 
     ASSERT_EQ(factory->get_parameters().approximate_select, false);
 }
 
 
-TEST_F(ParIct, SetDeterministic)
+TYPED_TEST(ParIct, SetDeterministic)
 {
-    auto factory =
-        ict_factory_type::build().with_deterministic_sample(true).on(ref);
+    auto factory = TestFixture::ict_factory_type::build()
+                       .with_deterministic_sample(true)
+                       .on(this->ref);
 
     ASSERT_EQ(factory->get_parameters().deterministic_sample, true);
 }
 
 
-TEST_F(ParIct, SetFillIn)
+TYPED_TEST(ParIct, SetFillIn)
 {
-    auto factory = ict_factory_type::build().with_fill_in_limit(1.2).on(ref);
+    auto factory =
+        TestFixture::ict_factory_type::build().with_fill_in_limit(1.2).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().fill_in_limit, 1.2);
 }
 
 
-TEST_F(ParIct, SetLStrategy)
+TYPED_TEST(ParIct, SetLStrategy)
 {
-    auto strategy = std::make_shared<strategy_type>();
+    auto strategy = std::make_shared<typename TestFixture::strategy_type>();
 
     auto factory =
-        ict_factory_type::build().with_l_strategy(strategy).on(this->ref);
+        TestFixture::ict_factory_type::build().with_l_strategy(strategy).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().l_strategy, strategy);
 }
 
 
-TEST_F(ParIct, SetDefaults)
+TYPED_TEST(ParIct, SetDefaults)
 {
-    auto factory = ict_factory_type::build().on(ref);
+    auto factory = TestFixture::ict_factory_type::build().on(this->ref);
 
     ASSERT_EQ(factory->get_parameters().iterations, 5u);
     ASSERT_EQ(factory->get_parameters().skip_sorting, false);
@@ -122,18 +139,18 @@ TEST_F(ParIct, SetDefaults)
 }
 
 
-TEST_F(ParIct, SetEverything)
+TYPED_TEST(ParIct, SetEverything)
 {
-    auto strategy = std::make_shared<strategy_type>();
+    auto strategy = std::make_shared<typename TestFixture::strategy_type>();
 
-    auto factory = ict_factory_type::build()
+    auto factory = TestFixture::ict_factory_type::build()
                        .with_iterations(7u)
                        .with_skip_sorting(true)
                        .with_approximate_select(false)
                        .with_deterministic_sample(true)
                        .with_fill_in_limit(1.2)
                        .with_l_strategy(strategy)
-                       .on(ref);
+                       .on(this->ref);
 
     ASSERT_EQ(factory->get_parameters().iterations, 7u);
     ASSERT_EQ(factory->get_parameters().skip_sorting, true);
diff --git a/core/test/factorization/par_ilu.cpp b/core/test/factorization/par_ilu.cpp
index b0253c3f083..6a1022d49db 100644
--- a/core/test/factorization/par_ilu.cpp
+++ b/core/test/factorization/par_ilu.cpp
@@ -39,15 +39,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
+#include "core/test/utils.hpp"
+
+
 namespace {
 
 
+template <typename ValueIndexType>
 class ParIlu : public ::testing::Test {
 public:
-    using value_type = double;
-    using index_type = gko::int32;
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using ilu_factory_type = gko::factorization::ParIlu<value_type, index_type>;
-    using strategy_type = ilu_factory_type::matrix_type::classical;
+    using strategy_type = typename ilu_factory_type::matrix_type::classical;
 
 protected:
     ParIlu() : ref(gko::ReferenceExecutor::create()) {}
@@ -55,49 +61,56 @@ class ParIlu : public ::testing::Test {
     std::shared_ptr<const gko::ReferenceExecutor> ref;
 };
 
+TYPED_TEST_SUITE(ParIlu, gko::test::ValueIndexTypes);
 
-TEST_F(ParIlu, SetIterations)
+
+TYPED_TEST(ParIlu, SetIterations)
 {
-    auto factory = ilu_factory_type::build().with_iterations(5u).on(this->ref);
+    auto factory =
+        TestFixture::ilu_factory_type::build().with_iterations(5u).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().iterations, 5u);
 }
 
 
-TEST_F(ParIlu, SetSkip)
+TYPED_TEST(ParIlu, SetSkip)
 {
     auto factory =
-        ilu_factory_type::build().with_skip_sorting(true).on(this->ref);
+        TestFixture::ilu_factory_type::build().with_skip_sorting(true).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().skip_sorting, true);
 }
 
 
-TEST_F(ParIlu, SetLStrategy)
+TYPED_TEST(ParIlu, SetLStrategy)
 {
-    auto strategy = std::make_shared<strategy_type>();
+    auto strategy = std::make_shared<typename TestFixture::strategy_type>();
 
     auto factory =
-        ilu_factory_type::build().with_l_strategy(strategy).on(this->ref);
+        TestFixture::ilu_factory_type::build().with_l_strategy(strategy).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().l_strategy, strategy);
 }
 
 
-TEST_F(ParIlu, SetUStrategy)
+TYPED_TEST(ParIlu, SetUStrategy)
 {
-    auto strategy = std::make_shared<strategy_type>();
+    auto strategy = std::make_shared<typename TestFixture::strategy_type>();
 
     auto factory =
-        ilu_factory_type::build().with_u_strategy(strategy).on(this->ref);
+        TestFixture::ilu_factory_type::build().with_u_strategy(strategy).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().u_strategy, strategy);
 }
 
 
-TEST_F(ParIlu, SetDefaults)
+TYPED_TEST(ParIlu, SetDefaults)
 {
-    auto factory = ilu_factory_type::build().on(this->ref);
+    auto factory = TestFixture::ilu_factory_type::build().on(this->ref);
 
     ASSERT_EQ(factory->get_parameters().iterations, 0u);
     ASSERT_EQ(factory->get_parameters().skip_sorting, false);
@@ -106,12 +119,12 @@ TEST_F(ParIlu, SetDefaults)
 }
 
 
-TEST_F(ParIlu, SetEverything)
+TYPED_TEST(ParIlu, SetEverything)
 {
-    auto strategy = std::make_shared<strategy_type>();
-    auto strategy2 = std::make_shared<strategy_type>();
+    auto strategy = std::make_shared<typename TestFixture::strategy_type>();
+    auto strategy2 = std::make_shared<typename TestFixture::strategy_type>();
 
-    auto factory = ilu_factory_type::build()
+    auto factory = TestFixture::ilu_factory_type::build()
                        .with_iterations(7u)
                        .with_skip_sorting(false)
                        .with_l_strategy(strategy)
diff --git a/core/test/factorization/par_ilut.cpp b/core/test/factorization/par_ilut.cpp
index 7f659481106..293763cf0af 100644
--- a/core/test/factorization/par_ilut.cpp
+++ b/core/test/factorization/par_ilut.cpp
@@ -39,16 +39,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
+#include "core/test/utils.hpp"
+
+
 namespace {
 
 
+template <typename ValueIndexType>
 class ParIlut : public ::testing::Test {
 public:
-    using value_type = gko::default_precision;
-    using index_type = gko::int32;
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using ilut_factory_type =
         gko::factorization::ParIlut<value_type, index_type>;
-    using strategy_type = ilut_factory_type::matrix_type::classical;
+    using strategy_type = typename ilut_factory_type::matrix_type::classical;
 
 protected:
     ParIlut() : ref(gko::ReferenceExecutor::create()) {}
@@ -56,74 +62,86 @@ class ParIlut : public ::testing::Test {
     std::shared_ptr<const gko::ReferenceExecutor> ref;
 };
 
+TYPED_TEST_SUITE(ParIlut, gko::test::ValueIndexTypes);
+
 
-TEST_F(ParIlut, SetIterations)
+TYPED_TEST(ParIlut, SetIterations)
 {
-    auto factory = ilut_factory_type::build().with_iterations(6u).on(ref);
+    auto factory =
+        TestFixture::ilut_factory_type::build().with_iterations(6u).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().iterations, 6u);
 }
 
 
-TEST_F(ParIlut, SetSkip)
+TYPED_TEST(ParIlut, SetSkip)
 {
-    auto factory = ilut_factory_type::build().with_skip_sorting(true).on(ref);
+    auto factory =
+        TestFixture::ilut_factory_type::build().with_skip_sorting(true).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().skip_sorting, true);
 }
 
 
-TEST_F(ParIlut, SetApprox)
+TYPED_TEST(ParIlut, SetApprox)
 {
-    auto factory =
-        ilut_factory_type::build().with_approximate_select(false).on(ref);
+    auto factory = TestFixture::ilut_factory_type::build()
+                       .with_approximate_select(false)
+                       .on(this->ref);
 
     ASSERT_EQ(factory->get_parameters().approximate_select, false);
 }
 
 
-TEST_F(ParIlut, SetDeterministic)
+TYPED_TEST(ParIlut, SetDeterministic)
 {
-    auto factory =
-        ilut_factory_type::build().with_deterministic_sample(true).on(ref);
+    auto factory = TestFixture::ilut_factory_type::build()
+                       .with_deterministic_sample(true)
+                       .on(this->ref);
 
     ASSERT_EQ(factory->get_parameters().deterministic_sample, true);
 }
 
 
-TEST_F(ParIlut, SetFillIn)
+TYPED_TEST(ParIlut, SetFillIn)
 {
-    auto factory = ilut_factory_type::build().with_fill_in_limit(1.2).on(ref);
+    auto factory =
+        TestFixture::ilut_factory_type::build().with_fill_in_limit(1.2).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().fill_in_limit, 1.2);
 }
 
 
-TEST_F(ParIlut, SetLStrategy)
+TYPED_TEST(ParIlut, SetLStrategy)
 {
-    auto strategy = std::make_shared<strategy_type>();
+    auto strategy = std::make_shared<typename TestFixture::strategy_type>();
 
     auto factory =
-        ilut_factory_type::build().with_l_strategy(strategy).on(this->ref);
+        TestFixture::ilut_factory_type::build().with_l_strategy(strategy).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().l_strategy, strategy);
 }
 
 
-TEST_F(ParIlut, SetUStrategy)
+TYPED_TEST(ParIlut, SetUStrategy)
 {
-    auto strategy = std::make_shared<strategy_type>();
+    auto strategy = std::make_shared<typename TestFixture::strategy_type>();
 
     auto factory =
-        ilut_factory_type::build().with_u_strategy(strategy).on(this->ref);
+        TestFixture::ilut_factory_type::build().with_u_strategy(strategy).on(
+            this->ref);
 
     ASSERT_EQ(factory->get_parameters().u_strategy, strategy);
 }
 
 
-TEST_F(ParIlut, SetDefaults)
+TYPED_TEST(ParIlut, SetDefaults)
 {
-    auto factory = ilut_factory_type::build().on(ref);
+    auto factory = TestFixture::ilut_factory_type::build().on(this->ref);
 
     ASSERT_EQ(factory->get_parameters().iterations, 5u);
     ASSERT_EQ(factory->get_parameters().skip_sorting, false);
@@ -135,12 +153,12 @@ TEST_F(ParIlut, SetDefaults)
 }
 
 
-TEST_F(ParIlut, SetEverything)
+TYPED_TEST(ParIlut, SetEverything)
 {
-    auto strategy = std::make_shared<strategy_type>();
-    auto strategy2 = std::make_shared<strategy_type>();
+    auto strategy = std::make_shared<typename TestFixture::strategy_type>();
+    auto strategy2 = std::make_shared<typename TestFixture::strategy_type>();
 
-    auto factory = ilut_factory_type::build()
+    auto factory = TestFixture::ilut_factory_type::build()
                        .with_iterations(7u)
                        .with_skip_sorting(true)
                        .with_approximate_select(false)
@@ -148,7 +166,7 @@ TEST_F(ParIlut, SetEverything)
                        .with_fill_in_limit(1.2)
                        .with_l_strategy(strategy)
                        .with_u_strategy(strategy2)
-                       .on(ref);
+                       .on(this->ref);
 
     ASSERT_EQ(factory->get_parameters().iterations, 7u);
     ASSERT_EQ(factory->get_parameters().skip_sorting, true);
diff --git a/cuda/test/factorization/ic_kernels.cpp b/cuda/test/factorization/ic_kernels.cpp
index dd7ecea6c65..e3ee5fe588d 100644
--- a/cuda/test/factorization/ic_kernels.cpp
+++ b/cuda/test/factorization/ic_kernels.cpp
@@ -100,12 +100,12 @@ TEST_F(Ic, ComputeICIsEquivalentToRefSorted)
 
     GKO_ASSERT_MTX_NEAR(ref_fact->get_l_factor(), cuda_fact->get_l_factor(),
                         1e-14);
-    GKO_ASSERT_MTX_NEAR(ref_fact->get_lh_factor(), cuda_fact->get_lh_factor(),
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_lt_factor(), cuda_fact->get_lt_factor(),
                         1e-14);
     GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_l_factor(),
                                cuda_fact->get_l_factor());
-    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_lh_factor(),
-                               cuda_fact->get_lh_factor());
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_lt_factor(),
+                               cuda_fact->get_lt_factor());
 }
 
 
@@ -121,12 +121,12 @@ TEST_F(Ic, ComputeICIsEquivalentToRefUnsorted)
 
     GKO_ASSERT_MTX_NEAR(ref_fact->get_l_factor(), cuda_fact->get_l_factor(),
                         1e-14);
-    GKO_ASSERT_MTX_NEAR(ref_fact->get_lh_factor(), cuda_fact->get_lh_factor(),
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_lt_factor(), cuda_fact->get_lt_factor(),
                         1e-14);
     GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_l_factor(),
                                cuda_fact->get_l_factor());
-    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_lh_factor(),
-                               cuda_fact->get_lh_factor());
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_lt_factor(),
+                               cuda_fact->get_lt_factor());
 }
 
 
@@ -139,7 +139,7 @@ TEST_F(Ic, SetsCorrectStrategy)
 
     ASSERT_EQ(cuda_fact->get_l_factor()->get_strategy()->get_name(),
               "merge_path");
-    ASSERT_EQ(cuda_fact->get_lh_factor()->get_strategy()->get_name(),
+    ASSERT_EQ(cuda_fact->get_lt_factor()->get_strategy()->get_name(),
               "merge_path");
 }
 
diff --git a/cuda/test/factorization/par_ic_kernels.cpp b/cuda/test/factorization/par_ic_kernels.cpp
index 606e58dac70..dec3b9617d9 100644
--- a/cuda/test/factorization/par_ic_kernels.cpp
+++ b/cuda/test/factorization/par_ic_kernels.cpp
@@ -158,9 +158,9 @@ TEST_F(ParIc, KernelComputeFactorIsEquivalentToRef)
     gko::kernels::reference::par_ic_factorization::compute_factor(
         ref, 1, lend(mtx_l_coo), lend(mtx_l_ani_init));
     gko::kernels::cuda::par_ic_factorization::compute_factor(
-        cuda, 20, lend(dmtx_l_coo), lend(dmtx_l_ani_init));
+        cuda, 100, lend(dmtx_l_coo), lend(dmtx_l_ani_init));
 
-    GKO_ASSERT_MTX_NEAR(mtx_l_ani_init, dmtx_l_ani_init, 1e-2);
+    GKO_ASSERT_MTX_NEAR(mtx_l_ani_init, dmtx_l_ani_init, 1e-4);
 }
 
 
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index abbc0e65e63..9a61ade6052 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -18,7 +18,7 @@ target_sources(ginkgo_dpcpp
     factorization/ic_kernels.dp.cpp
     factorization/ilu_kernels.dp.cpp
     factorization/factorization_kernels.dp.cpp
-    factorization/par_c_kernels.dp.cpp
+    factorization/par_ic_kernels.dp.cpp
     factorization/par_ict_kernels.dp.cpp
     factorization/par_ilu_kernels.dp.cpp
     factorization/par_ilut_kernels.dp.cpp
diff --git a/examples/par-ilu-convergence/par-ilu-convergence.cpp b/examples/par-ilu-convergence/par-ilu-convergence.cpp
index 77395a159b8..b2b4303049e 100644
--- a/examples/par-ilu-convergence/par-ilu-convergence.cpp
+++ b/examples/par-ilu-convergence/par-ilu-convergence.cpp
@@ -73,11 +73,13 @@ auto try_generate(Function fun) -> decltype(fun())
 }
 
 
-double compute_ilu_residual_norm(const gko::matrix::Csr<> *residual,
-                                 const gko::matrix::Csr<> *mtx)
+template <typename ValueType, typename IndexType>
+double compute_ilu_residual_norm(
+    const gko::matrix::Csr<ValueType, IndexType> *residual,
+    const gko::matrix::Csr<ValueType, IndexType> *mtx)
 {
-    gko::matrix_data<> residual_data;
-    gko::matrix_data<> mtx_data;
+    gko::matrix_data<ValueType, IndexType> residual_data;
+    gko::matrix_data<ValueType, IndexType> mtx_data;
     residual->write(residual_data);
     mtx->write(mtx_data);
     residual_data.ensure_row_major_order();
@@ -98,6 +100,9 @@ double compute_ilu_residual_norm(const gko::matrix::Csr<> *residual,
 
 int main(int argc, char *argv[])
 {
+    using ValueType = double;
+    using IndexType = int;
+
     // print usage message
     if (argc < 2 || executors.find(argv[1]) == executors.end()) {
         std::cerr << "Usage: executable"
@@ -124,46 +129,54 @@ int main(int argc, char *argv[])
             throw GKO_STREAM_ERROR("Unable to open matrix file");
         }
         std::cerr << "Reading " << matrix << std::endl;
-        return gko::read<gko::matrix::Csr<>>(mtx_stream, exec);
+        return gko::read<gko::matrix::Csr<ValueType, IndexType>>(mtx_stream,
+                                                                 exec);
     }));
 
     std::shared_ptr<gko::LinOpFactory> factory;
     std::function<void(int)> set_iterations;
     if (precond == "parilu") {
-        factory = gko::factorization::ParIlu<>::build().on(exec);
+        factory =
+            gko::factorization::ParIlu<ValueType, IndexType>::build().on(exec);
         set_iterations = [&](int it) {
-            gko::as<gko::factorization::ParIlu<>::Factory>(factory)
+            gko::as<gko::factorization::ParIlu<ValueType, IndexType>::Factory>(
+                factory)
                 ->get_parameters()
                 .iterations = it;
         };
     } else if (precond == "paric") {
-        factory = gko::factorization::ParIc<>::build().on(exec);
+        factory =
+            gko::factorization::ParIc<ValueType, IndexType>::build().on(exec);
         set_iterations = [&](int it) {
-            gko::as<gko::factorization::ParIc<>::Factory>(factory)
+            gko::as<gko::factorization::ParIc<ValueType, IndexType>::Factory>(
+                factory)
                 ->get_parameters()
                 .iterations = it;
         };
     } else if (precond == "parilut") {
-        factory =
-            gko::factorization::ParIlut<>::build().with_fill_in_limit(limit).on(
-                exec);
+        factory = gko::factorization::ParIlut<ValueType, IndexType>::build()
+                      .with_fill_in_limit(limit)
+                      .on(exec);
         set_iterations = [&](int it) {
-            gko::as<gko::factorization::ParIlut<>::Factory>(factory)
+            gko::as<gko::factorization::ParIlut<ValueType, IndexType>::Factory>(
+                factory)
                 ->get_parameters()
                 .iterations = it;
         };
     } else if (precond == "parict") {
-        factory =
-            gko::factorization::ParIct<>::build().with_fill_in_limit(limit).on(
-                exec);
+        factory = gko::factorization::ParIct<ValueType, IndexType>::build()
+                      .with_fill_in_limit(limit)
+                      .on(exec);
         set_iterations = [&](int it) {
-            gko::as<gko::factorization::ParIct<>::Factory>(factory)
+            gko::as<gko::factorization::ParIct<ValueType, IndexType>::Factory>(
+                factory)
                 ->get_parameters()
                 .iterations = it;
         };
     }
-    auto one = gko::initialize<gko::matrix::Dense<>>({1.0}, exec);
-    auto minus_one = gko::initialize<gko::matrix::Dense<>>({-1.0}, exec);
+    auto one = gko::initialize<gko::matrix::Dense<ValueType>>({1.0}, exec);
+    auto minus_one =
+        gko::initialize<gko::matrix::Dense<ValueType>>({-1.0}, exec);
     for (int it = 1; it <= max_iterations; ++it) {
         set_iterations(it);
         std::cout << it << ';';
@@ -171,7 +184,8 @@ int main(int argc, char *argv[])
         std::vector<double> residuals;
         for (int rep = 0; rep < num_repetitions; ++rep) {
             auto tic = std::chrono::high_resolution_clock::now();
-            auto result = gko::as<gko::Composition<>>(factory->generate(mtx));
+            auto result =
+                gko::as<gko::Composition<ValueType>>(factory->generate(mtx));
             exec->synchronize();
             auto toc = std::chrono::high_resolution_clock::now();
             auto residual = gko::clone(exec, mtx);
diff --git a/hip/test/factorization/ic_kernels.cpp b/hip/test/factorization/ic_kernels.cpp
index 6b39dcf8636..a630f831895 100644
--- a/hip/test/factorization/ic_kernels.cpp
+++ b/hip/test/factorization/ic_kernels.cpp
@@ -100,12 +100,12 @@ TEST_F(Ic, ComputeICIsEquivalentToRefSorted)
 
     GKO_ASSERT_MTX_NEAR(ref_fact->get_l_factor(), hip_fact->get_l_factor(),
                         1e-14);
-    GKO_ASSERT_MTX_NEAR(ref_fact->get_lh_factor(), hip_fact->get_lh_factor(),
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_lt_factor(), hip_fact->get_lt_factor(),
                         1e-14);
     GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_l_factor(),
                                hip_fact->get_l_factor());
-    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_lh_factor(),
-                               hip_fact->get_lh_factor());
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_lt_factor(),
+                               hip_fact->get_lt_factor());
 }
 
 
@@ -121,12 +121,12 @@ TEST_F(Ic, ComputeICIsEquivalentToRefUnsorted)
 
     GKO_ASSERT_MTX_NEAR(ref_fact->get_l_factor(), hip_fact->get_l_factor(),
                         1e-14);
-    GKO_ASSERT_MTX_NEAR(ref_fact->get_lh_factor(), hip_fact->get_lh_factor(),
+    GKO_ASSERT_MTX_NEAR(ref_fact->get_lt_factor(), hip_fact->get_lt_factor(),
                         1e-14);
     GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_l_factor(),
                                hip_fact->get_l_factor());
-    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_lh_factor(),
-                               hip_fact->get_lh_factor());
+    GKO_ASSERT_MTX_EQ_SPARSITY(ref_fact->get_lt_factor(),
+                               hip_fact->get_lt_factor());
 }
 
 
@@ -139,7 +139,7 @@ TEST_F(Ic, SetsCorrectStrategy)
 
     ASSERT_EQ(hip_fact->get_l_factor()->get_strategy()->get_name(),
               "merge_path");
-    ASSERT_EQ(hip_fact->get_lh_factor()->get_strategy()->get_name(),
+    ASSERT_EQ(hip_fact->get_lt_factor()->get_strategy()->get_name(),
               "merge_path");
 }
 
diff --git a/hip/test/factorization/par_ic_kernels.hip.cpp b/hip/test/factorization/par_ic_kernels.hip.cpp
index d7bc838121a..eb51157269c 100644
--- a/hip/test/factorization/par_ic_kernels.hip.cpp
+++ b/hip/test/factorization/par_ic_kernels.hip.cpp
@@ -158,9 +158,9 @@ TEST_F(ParIc, KernelComputeFactorIsEquivalentToRef)
     gko::kernels::reference::par_ic_factorization::compute_factor(
         ref, 1, lend(mtx_l_coo), lend(mtx_l_ani_init));
     gko::kernels::hip::par_ic_factorization::compute_factor(
-        hip, 20, lend(dmtx_l_coo), lend(dmtx_l_ani_init));
+        hip, 100, lend(dmtx_l_coo), lend(dmtx_l_ani_init));
 
-    GKO_ASSERT_MTX_NEAR(mtx_l_ani_init, dmtx_l_ani_init, 1e-2);
+    GKO_ASSERT_MTX_NEAR(mtx_l_ani_init, dmtx_l_ani_init, 1e-4);
 }
 
 
diff --git a/include/ginkgo/core/factorization/ic.hpp b/include/ginkgo/core/factorization/ic.hpp
index 585d926b0d4..8990ba3a15f 100644
--- a/include/ginkgo/core/factorization/ic.hpp
+++ b/include/ginkgo/core/factorization/ic.hpp
@@ -81,7 +81,7 @@ class Ic : public Composition<ValueType> {
             this->get_operators()[0]);
     }
 
-    std::shared_ptr<const matrix_type> get_lh_factor() const
+    std::shared_ptr<const matrix_type> get_lt_factor() const
     {
         if (this->get_operators().size() == 2) {
             // Can be `static_cast` since the type is guaranteed in this class
@@ -109,13 +109,6 @@ class Ic : public Composition<ValueType> {
             GKO_FACTORY_PARAMETER_SCALAR(l_strategy, nullptr);
 
         /**
-         * @brief `true` means it is known that the matrix given to this
-         *        factory will be sorted first by row, then by column index,
-         *        `false` means it is unknown or not sorted, so an additional
-         *        sorting step will be performed during the factorization
-         *        (it will not change the matrix given).
-         *        The matrix must be sorted for this factorization to work.
-         *
          * The `system_matrix`, which will be given to this factory, must be
          * sorted (first by row, then by column) in order for the algorithm
          * to work. If it is known that the matrix will be sorted, this
diff --git a/include/ginkgo/core/factorization/ilu.hpp b/include/ginkgo/core/factorization/ilu.hpp
index ccc0ff6a072..4e0bbef4008 100644
--- a/include/ginkgo/core/factorization/ilu.hpp
+++ b/include/ginkgo/core/factorization/ilu.hpp
@@ -111,13 +111,6 @@ class Ilu : public Composition<ValueType> {
             GKO_FACTORY_PARAMETER_SCALAR(u_strategy, nullptr);
 
         /**
-         * @brief `true` means it is known that the matrix given to this
-         *        factory will be sorted first by row, then by column index,
-         *        `false` means it is unknown or not sorted, so an additional
-         *        sorting step will be performed during the factorization
-         *        (it will not change the matrix given).
-         *        The matrix must be sorted for this factorization to work.
-         *
          * The `system_matrix`, which will be given to this factory, must be
          * sorted (first by row, then by column) in order for the algorithm
          * to work. If it is known that the matrix will be sorted, this
diff --git a/include/ginkgo/core/factorization/par_ic.hpp b/include/ginkgo/core/factorization/par_ic.hpp
index 12bae4cfc37..220aa6a7b83 100644
--- a/include/ginkgo/core/factorization/par_ic.hpp
+++ b/include/ginkgo/core/factorization/par_ic.hpp
@@ -77,8 +77,10 @@ namespace factorization {
  * pre-defined sparsity pattern (in case of IC(0) the sparsity pattern of the
  * system matrix $A$). The number of ParIC sweeps needed for convergence
  * depends on the parallelism level: For sequential execution, a single sweep
- * is sufficient, for fine-grained parallelism, 3 sweeps are typically
- * generating a good approximation.
+ * is sufficient, for fine-grained parallelism, the number of sweeps necessary
+ * to get a good approximation of the incomplete factors depends heavily on the
+ * problem. On the OpenMP executor, 3 sweeps usually give a decent approximation
+ * in our experiments, while GPU executors can take 10 or more iterations.
  *
  * The ParIC algorithm in Ginkgo follows the design of E. Chow and A. Patel,
  * Fine-grained Parallel Incomplete LU Factorization, SIAM Journal on Scientific
@@ -104,7 +106,7 @@ class ParIc : public Composition<ValueType> {
             this->get_operators()[0]);
     }
 
-    std::shared_ptr<const matrix_type> get_lh_factor() const
+    std::shared_ptr<const matrix_type> get_lt_factor() const
     {
         if (this->get_operators().size() == 2) {
             // Can be `static_cast` since the type is guaranteed in this class
@@ -133,13 +135,6 @@ class ParIc : public Composition<ValueType> {
         size_type GKO_FACTORY_PARAMETER_SCALAR(iterations, 0);
 
         /**
-         * @brief `true` means it is known that the matrix given to this
-         *        factory will be sorted first by row, then by column index,
-         *        `false` means it is unknown or not sorted, so an additional
-         *        sorting step will be performed during the factorization
-         *        (it will not change the matrix given).
-         *        The matrix must be sorted for this factorization to work.
-         *
          * The `system_matrix`, which will be given to this factory, must be
          * sorted (first by row, then by column) in order for the algorithm
          * to work. If it is known that the matrix will be sorted, this
diff --git a/include/ginkgo/core/factorization/par_ict.hpp b/include/ginkgo/core/factorization/par_ict.hpp
index 292c9d54ef2..cc113b8c807 100644
--- a/include/ginkgo/core/factorization/par_ict.hpp
+++ b/include/ginkgo/core/factorization/par_ict.hpp
@@ -106,7 +106,7 @@ class ParIct : public Composition<ValueType> {
             this->get_operators()[0]);
     }
 
-    std::shared_ptr<const matrix_type> get_lh_factor() const
+    std::shared_ptr<const matrix_type> get_lt_factor() const
     {
         // Can be `static_cast` since the type is guaranteed in this class
         return std::static_pointer_cast<const matrix_type>(
diff --git a/include/ginkgo/core/factorization/par_ilu.hpp b/include/ginkgo/core/factorization/par_ilu.hpp
index eaafb9b3971..bfd6884ebbf 100644
--- a/include/ginkgo/core/factorization/par_ilu.hpp
+++ b/include/ginkgo/core/factorization/par_ilu.hpp
@@ -78,8 +78,10 @@ namespace factorization {
  * pre-defined sparsity pattern (in case of ILU(0) the sparsity pattern of the
  * system matrix $A$). The number of ParILU sweeps needed for convergence
  * depends on the parallelism level: For sequential execution, a single sweep
- * is sufficient, for fine-grained parallelism, 3 sweeps are typically
- * generating a good approximation.
+ * is sufficient, for fine-grained parallelism, the number of sweeps necessary
+ * to get a good approximation of the incomplete factors depends heavily on the
+ * problem. On the OpenMP executor, 3 sweeps usually give a decent approximation
+ * in our experiments, while GPU executors can take 10 or more iterations.
  *
  * The ParILU algorithm in Ginkgo follows the design of E. Chow and A. Patel,
  * Fine-grained Parallel Incomplete LU Factorization, SIAM Journal on Scientific
@@ -131,13 +133,6 @@ class ParIlu : public Composition<ValueType> {
         size_type GKO_FACTORY_PARAMETER_SCALAR(iterations, 0);
 
         /**
-         * @brief `true` means it is known that the matrix given to this
-         *        factory will be sorted first by row, then by column index,
-         *        `false` means it is unknown or not sorted, so an additional
-         *        sorting step will be performed during the factorization
-         *        (it will not change the matrix given).
-         *        The matrix must be sorted for this factorization to work.
-         *
          * The `system_matrix`, which will be given to this factory, must be
          * sorted (first by row, then by column) in order for the algorithm
          * to work. If it is known that the matrix will be sorted, this
diff --git a/include/ginkgo/core/preconditioner/ic.hpp b/include/ginkgo/core/preconditioner/ic.hpp
index 27e9d5bc582..e08894b836c 100644
--- a/include/ginkgo/core/preconditioner/ic.hpp
+++ b/include/ginkgo/core/preconditioner/ic.hpp
@@ -47,6 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/factorization/par_ic.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/solver/lower_trs.hpp>
+#include <ginkgo/core/solver/solver_traits.hpp>
 #include <ginkgo/core/solver/upper_trs.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
@@ -58,18 +59,18 @@ namespace preconditioner {
 
 
 /**
- * The Incomplete LU (ILU) preconditioner solves the equation $LUx = b$ for a
- * given lower triangular matrix L, an upper triangular matrix U and the right
- * hand side b (can contain multiple right hand sides).
+ * The Incomplete Cholesky (IC) preconditioner solves the equation $LL^H*x = b$
+ * for a given lower triangular matrix L and the right hand side b (can contain
+ * multiple right hand sides).
  *
- * It allows to set both the solver for L and the solver for U independently,
- * while providing the defaults solver::LowerTrs and solver::UpperTrs, which
- * are direct triangular solvers.
- * For these solvers, a factory can be provided (with `with_l_solver_factory`
- * and `with_u_solver_factory`) to have more control over their behavior.
- * In particular, it is possible to use an iterative method for solving the
- * triangular systems. The default parameters for an iterative triangluar
- * solver are:
+ * It allows to set both the solver for L defaulting to solver::LowerTrs, which
+ * is a direct triangular solvers. The solver for L^H is the
+ * conjugate-transposed solver for L, ensuring that the preconditioner is
+ * symmetric and positive-definite. For this L solver, a factory can be provided
+ * (using `with_l_solver_factory`) to have more control over their behavior. In
+ * particular, it is possible to use an iterative method for solving the
+ * triangular systems. The default parameters for an iterative triangluar solver
+ * are:
  * - reduction factor = 1e-4
  * - max iteration = <number of rows of the matrix given to the solver>
  * Solvers without such criteria can also be used, in which case none are set.
@@ -78,15 +79,16 @@ namespace preconditioner {
  * containing two matrices. If created with a matrix, it is factorized before
  * creating the solver. If a gko::Composition (containing two matrices) is
  * used, the first operand will be taken as the L matrix, the second will be
- * considered the U matrix. ParIlu can be directly used, since it orders the
- * factors in the correct way.
+ * considered the L^H matrix, which helps to avoid the otherwise necessary
+ * transposition of L inside the solver. ParIc can be directly used, since it
+ * orders the factors in the correct way.
  *
  * @note When providing a gko::Composition, the first matrix must be the lower
- *       matrix ($L$), and the second matrix must be the upper matrix ($U$).
- *       If they are swapped, solving might crash or return the wrong result.
+ *       matrix ($L$), and the second matrix must be its conjugate-transpose
+ * ($L^H$). If they are swapped, solving might crash or return the wrong result.
  *
- * @note Do not use symmetric solvers (like CG) for L or U solvers since both
- *       matrices (L and U) are, by design, not symmetric.
+ * @note Do not use symmetric solvers (like CG) for the L solver since both
+ *       matrices (L and L^H) are, by design, not symmetric.
  *
  * @note This class is not thread safe (even a const object is not) because it
  *       uses an internal cache to accelerate multiple (sequential) applies.
@@ -95,14 +97,8 @@ namespace preconditioner {
  *
  * @tparam LSolverType  type of the solver used for the L matrix.
  *                      Defaults to solver::LowerTrs
- * @tparam USolverType  type of the solver used for the U matrix
- *                      Defaults to solver::UpperTrs
- * @tparam ReverseApply  default behavior (ReverseApply = false) is first to
- *                       solve with L (Ly = b) and then with U (Ux = y).
- *                       When set to true, it will solve first with U, and then
- *                       with L.
- * @tparam IndexTypeParIlu  Type of the indices when ParIlu is used to generate
- *                          both L and U factors. Irrelevant otherwise.
+ * @tparam IndexType  type of the indices when ParIc is used to generate
+ *                    the L and L^H factors. Irrelevant otherwise.
  *
  * @ingroup precond
  * @ingroup LinOp
@@ -196,7 +192,9 @@ class Ic : public EnableLinOp<Ic<LSolverType, IndexType>>, public Transposable {
     {
         set_cache_to(b);
         l_solver_->apply(b, cache_.intermediate.get());
-        x->copy_from(cache_.intermediate.get());
+        if (lh_solver_->apply_uses_initial_guess()) {
+            x->copy_from(cache_.intermediate.get());
+        }
         lh_solver_->apply(cache_.intermediate.get(), x);
     }
 
@@ -274,8 +272,6 @@ class Ic : public EnableLinOp<Ic<LSolverType, IndexType>>, public Transposable {
      */
     void set_cache_to(const LinOp *b) const
     {
-        dim<2> expected_size =
-            dim<2>{l_solver_->get_size()[0], b->get_size()[1]};
         if (cache_.intermediate == nullptr) {
             cache_.intermediate =
                 matrix::Dense<value_type>::create(this->get_executor());
@@ -284,45 +280,6 @@ class Ic : public EnableLinOp<Ic<LSolverType, IndexType>>, public Transposable {
         cache_.intermediate->copy_from(b);
     }
 
-    /**
-     * @internal  Looks at the build() method to determine the type of the
-     *            factory.
-     */
-    template <typename T>
-    using factory_type_t = decltype(T::build());
-
-    // Parameter type of function `with_criteria`.
-    using with_criteria_param_type =
-        std::shared_ptr<const stop::CriterionFactory>;
-
-    /**
-     * Helper structure to test if the Factory of SolverType has a function
-     * `with_criteria`.
-     *
-     * Contains a constexpr boolean `value`, which is true if the Factory
-     * class of SolverType has a `with_criteria`, and false otherwise.
-     *
-     * @tparam SolverType   Solver to test if its factory has a
-     * with_criteria function.
-     *
-     */
-    template <typename SolverType, typename = void>
-    struct has_with_criteria : std::false_type {};
-
-    /**
-     * @copydoc has_with_criteria
-     *
-     * @internal  The second template parameter (which uses SFINAE) must
-     * match the default value of the general case in order to be accepted
-     *            as a specialization, which is why `xstd::void_t` is used.
-     */
-    template <typename SolverType>
-    struct has_with_criteria<
-        SolverType,
-        xstd::void_t<decltype(std::declval<factory_type_t<SolverType>>()
-                                  .with_criteria(with_criteria_param_type()))>>
-        : std::true_type {};
-
 
     /**
      * Generates a default solver of type SolverType.
@@ -332,7 +289,7 @@ class Ic : public EnableLinOp<Ic<LSolverType, IndexType>>, public Transposable {
      * preconditioner.
      */
     template <typename SolverType>
-    static std::enable_if_t<has_with_criteria<SolverType>::value,
+    static std::enable_if_t<solver::has_with_criteria<SolverType>(),
                             std::unique_ptr<SolverType>>
     generate_default_solver(const std::shared_ptr<const Executor> &exec,
                             const std::shared_ptr<const LinOp> &mtx)
@@ -356,7 +313,7 @@ class Ic : public EnableLinOp<Ic<LSolverType, IndexType>>, public Transposable {
      * @copydoc generate_default_solver
      */
     template <typename SolverType>
-    static std::enable_if_t<!has_with_criteria<SolverType>::value,
+    static std::enable_if_t<!solver::has_with_criteria<SolverType>(),
                             std::unique_ptr<SolverType>>
     generate_default_solver(const std::shared_ptr<const Executor> &exec,
                             const std::shared_ptr<const LinOp> &mtx)
@@ -386,7 +343,7 @@ class Ic : public EnableLinOp<Ic<LSolverType, IndexType>>, public Transposable {
         cache_struct &operator=(cache_struct &&) { return *this; }
         std::unique_ptr<LinOp> intermediate{};
     } cache_;
-};  // namespace preconditioner
+};
 
 
 }  // namespace preconditioner
diff --git a/include/ginkgo/core/preconditioner/ilu.hpp b/include/ginkgo/core/preconditioner/ilu.hpp
index e80978158bb..554f599f688 100644
--- a/include/ginkgo/core/preconditioner/ilu.hpp
+++ b/include/ginkgo/core/preconditioner/ilu.hpp
@@ -47,6 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/factorization/par_ilu.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/solver/lower_trs.hpp>
+#include <ginkgo/core/solver/solver_traits.hpp>
 #include <ginkgo/core/solver/upper_trs.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
@@ -210,11 +211,15 @@ class Ilu : public EnableLinOp<
         set_cache_to(b);
         if (!ReverseApply) {
             l_solver_->apply(b, cache_.intermediate.get());
-            x->copy_from(cache_.intermediate.get());
+            if (u_solver_->apply_uses_initial_guess()) {
+                x->copy_from(cache_.intermediate.get());
+            }
             u_solver_->apply(cache_.intermediate.get(), x);
         } else {
             u_solver_->apply(b, cache_.intermediate.get());
-            x->copy_from(cache_.intermediate.get());
+            if (l_solver_->apply_uses_initial_guess()) {
+                x->copy_from(cache_.intermediate.get());
+            }
             l_solver_->apply(cache_.intermediate.get(), x);
         }
     }
@@ -294,9 +299,6 @@ class Ilu : public EnableLinOp<
      */
     void set_cache_to(const LinOp *b) const
     {
-        dim<2> expected_size =
-            ReverseApply ? dim<2>{u_solver_->get_size()[0], b->get_size()[1]}
-                         : dim<2>{l_solver_->get_size()[0], b->get_size()[1]};
         if (cache_.intermediate == nullptr) {
             cache_.intermediate =
                 matrix::Dense<value_type>::create(this->get_executor());
@@ -305,45 +307,6 @@ class Ilu : public EnableLinOp<
         cache_.intermediate->copy_from(b);
     }
 
-    /**
-     * @internal  Looks at the build() method to determine the type of the
-     *            factory.
-     */
-    template <typename T>
-    using factory_type_t = decltype(T::build());
-
-    // Parameter type of function `with_criteria`.
-    using with_criteria_param_type =
-        std::shared_ptr<const stop::CriterionFactory>;
-
-    /**
-     * Helper structure to test if the Factory of SolverType has a function
-     * `with_criteria`.
-     *
-     * Contains a constexpr boolean `value`, which is true if the Factory class
-     * of SolverType has a `with_criteria`, and false otherwise.
-     *
-     * @tparam SolverType   Solver to test if its factory has a with_criteria
-     *                      function.
-     *
-     */
-    template <typename SolverType, typename = void>
-    struct has_with_criteria : std::false_type {};
-
-    /**
-     * @copydoc has_with_criteria
-     *
-     * @internal  The second template parameter (which uses SFINAE) must match
-     *            the default value of the general case in order to be accepted
-     *            as a specialization, which is why `xstd::void_t` is used.
-     */
-    template <typename SolverType>
-    struct has_with_criteria<
-        SolverType,
-        xstd::void_t<decltype(std::declval<factory_type_t<SolverType>>()
-                                  .with_criteria(with_criteria_param_type()))>>
-        : std::true_type {};
-
 
     /**
      * Generates a default solver of type SolverType.
@@ -353,7 +316,7 @@ class Ilu : public EnableLinOp<
      * preconditioner.
      */
     template <typename SolverType>
-    static std::enable_if_t<has_with_criteria<SolverType>::value,
+    static std::enable_if_t<solver::has_with_criteria<SolverType>(),
                             std::unique_ptr<SolverType>>
     generate_default_solver(const std::shared_ptr<const Executor> &exec,
                             const std::shared_ptr<const LinOp> &mtx)
@@ -377,7 +340,7 @@ class Ilu : public EnableLinOp<
      * @copydoc generate_default_solver
      */
     template <typename SolverType>
-    static std::enable_if_t<!has_with_criteria<SolverType>::value,
+    static std::enable_if_t<!solver::has_with_criteria<SolverType>(),
                             std::unique_ptr<SolverType>>
     generate_default_solver(const std::shared_ptr<const Executor> &exec,
                             const std::shared_ptr<const LinOp> &mtx)
diff --git a/include/ginkgo/core/solver/solver_traits.hpp b/include/ginkgo/core/solver/solver_traits.hpp
new file mode 100644
index 00000000000..0ce101d9aef
--- /dev/null
+++ b/include/ginkgo/core/solver/solver_traits.hpp
@@ -0,0 +1,87 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_PUBLIC_CORE_SOLVER_SOLVER_TRAITS_HPP_
+#define GKO_PUBLIC_CORE_SOLVER_SOLVER_TRAITS_HPP_
+
+
+#include <ginkgo/core/base/std_extensions.hpp>
+#include <ginkgo/core/stop/criterion.hpp>
+
+
+namespace gko {
+namespace solver {
+namespace detail {
+
+
+/**
+ * Helper structure to test if the Factory of SolverType has a function
+ * `with_criteria`.
+ *
+ * Contains a constexpr boolean `value`, which is true if the Factory class
+ * of SolverType has a `with_criteria`, and false otherwise.
+ *
+ * @tparam SolverType   Solver to test if its factory has a with_criteria
+ *                      function.
+ *
+ */
+template <typename SolverType, typename = void>
+struct has_with_criteria_impl : std::false_type {};
+
+/**
+ * @copydoc has_with_criteria
+ *
+ * @internal  The second template parameter (which uses SFINAE) must match
+ *            the default value of the general case in order to be accepted
+ *            as a specialization, which is why `xstd::void_t` is used.
+ */
+template <typename SolverType>
+struct has_with_criteria_impl<
+    SolverType, xstd::void_t<decltype(SolverType::build().with_criteria(
+                    std::shared_ptr<const stop::CriterionFactory>()))>>
+    : std::true_type {};
+
+
+}  // namespace detail
+
+
+template <typename SolverType>
+constexpr bool has_with_criteria()
+{
+    return detail::has_with_criteria_impl<SolverType>::value;
+}
+
+
+}  // namespace solver
+}  // namespace gko
+
+#endif  // GKO_PUBLIC_CORE_SOLVER_SOLVER_TRAITS_HPP_
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 7fc929dfa93..49686250cc6 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -103,6 +103,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/idr.hpp>
 #include <ginkgo/core/solver/ir.hpp>
 #include <ginkgo/core/solver/lower_trs.hpp>
+#include <ginkgo/core/solver/solver_traits.hpp>
 #include <ginkgo/core/solver/upper_trs.hpp>
 
 #include <ginkgo/core/stop/combined.hpp>
diff --git a/omp/factorization/par_ic_kernels.cpp b/omp/factorization/par_ic_kernels.cpp
index 78f6d8fd634..478d8363ea6 100644
--- a/omp/factorization/par_ic_kernels.cpp
+++ b/omp/factorization/par_ic_kernels.cpp
@@ -98,7 +98,6 @@ void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
                 // accumulate l(row,:) * l(col,:) without the last entry l(col,
                 // col)
                 ValueType sum{};
-                IndexType lh_nz{};
                 auto l_begin = l_row_ptrs[row];
                 auto l_end = l_row_ptrs[row + 1];
                 auto lh_begin = l_row_ptrs[col];
@@ -109,9 +108,6 @@ void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
                     if (l_col == lh_row && l_col < col) {
                         sum += l_vals[l_begin] * conj(l_vals[lh_begin]);
                     }
-                    if (lh_row == row) {
-                        lh_nz = lh_begin;
-                    }
                     l_begin += (l_col <= lh_row);
                     lh_begin += (lh_row <= l_col);
                 }
diff --git a/omp/test/factorization/par_ic_kernels.cpp b/omp/test/factorization/par_ic_kernels.cpp
index 928e6d97e0b..34d37095454 100644
--- a/omp/test/factorization/par_ic_kernels.cpp
+++ b/omp/test/factorization/par_ic_kernels.cpp
@@ -168,9 +168,9 @@ TYPED_TEST(ParIc, KernelComputeFactorIsEquivalentToRef)
     gko::kernels::reference::par_ic_factorization::compute_factor(
         this->ref, 1, lend(mtx_l_coo), lend(this->mtx_l_ani_init));
     gko::kernels::omp::par_ic_factorization::compute_factor(
-        this->omp, 20, lend(dmtx_l_coo), lend(this->dmtx_l_ani_init));
+        this->omp, 100, lend(dmtx_l_coo), lend(this->dmtx_l_ani_init));
 
-    GKO_ASSERT_MTX_NEAR(this->mtx_l_ani_init, this->dmtx_l_ani_init, 1e-2);
+    GKO_ASSERT_MTX_NEAR(this->mtx_l_ani_init, this->dmtx_l_ani_init, 1e-4);
 }
 
 
diff --git a/reference/factorization/par_ic_kernels.cpp b/reference/factorization/par_ic_kernels.cpp
index eeee9e46b8d..f71e28d2e00 100644
--- a/reference/factorization/par_ic_kernels.cpp
+++ b/reference/factorization/par_ic_kernels.cpp
@@ -99,7 +99,6 @@ void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
             auto a_val = a_vals[l_nz];
             // accumulate l(row,:) * l(col,:) without the last entry l(col, col)
             ValueType sum{};
-            IndexType lh_nz{};
             auto l_begin = l_row_ptrs[row];
             auto l_end = l_row_ptrs[row + 1];
             auto lh_begin = l_row_ptrs[col];
@@ -110,9 +109,6 @@ void compute_factor(std::shared_ptr<const DefaultExecutor> exec,
                 if (l_col == lh_row && l_col < col) {
                     sum += l_vals[l_begin] * conj(l_vals[lh_begin]);
                 }
-                if (lh_row == row) {
-                    lh_nz = lh_begin;
-                }
                 l_begin += (l_col <= lh_row);
                 lh_begin += (lh_row <= l_col);
             }
diff --git a/reference/test/factorization/par_ic_kernels.cpp b/reference/test/factorization/par_ic_kernels.cpp
index 1548a8b0a5f..015af22a491 100644
--- a/reference/test/factorization/par_ic_kernels.cpp
+++ b/reference/test/factorization/par_ic_kernels.cpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
 
 
 #include "core/factorization/factorization_kernels.hpp"
@@ -188,7 +189,7 @@ TYPED_TEST(ParIc, SetStrategy)
     ASSERT_EQ(factory->get_parameters().l_strategy, l_strategy);
     ASSERT_EQ(fact->get_l_factor()->get_strategy()->get_name(),
               l_strategy->get_name());
-    ASSERT_EQ(fact->get_lh_factor()->get_strategy()->get_name(),
+    ASSERT_EQ(fact->get_lt_factor()->get_strategy()->get_name(),
               l_strategy->get_name());
 }
 
@@ -198,7 +199,7 @@ TYPED_TEST(ParIc, IsConsistentWithComposition)
     auto fact = this->fact_fact->generate(this->mtx_system);
 
     auto lin_op_l_factor = gko::as<gko::LinOp>(fact->get_l_factor());
-    auto lin_op_lt_factor = gko::as<gko::LinOp>(fact->get_lh_factor());
+    auto lin_op_lt_factor = gko::as<gko::LinOp>(fact->get_lt_factor());
     auto first_operator = fact->get_operators()[0];
     auto second_operator = fact->get_operators()[1];
 
@@ -216,7 +217,7 @@ TYPED_TEST(ParIc, GenerateSingleFactor)
     auto fact = factory->generate(this->mtx_system);
 
     auto lin_op_l_factor = fact->get_l_factor();
-    auto lin_op_lt_factor = fact->get_lh_factor();
+    auto lin_op_lt_factor = fact->get_lt_factor();
     auto first_operator = gko::as<Csr>(fact->get_operators()[0]);
     auto first_operator_h = gko::as<Csr>(first_operator->conj_transpose());
 
@@ -231,7 +232,7 @@ TYPED_TEST(ParIc, GenerateIdentity)
     auto fact = this->fact_fact->generate(this->identity);
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->identity, this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(), this->identity, this->tol);
+    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(), this->identity, this->tol);
 }
 
 
@@ -243,7 +244,7 @@ TYPED_TEST(ParIc, GenerateDenseIdentity)
     auto fact = this->fact_fact->generate(gko::share(dense_id));
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->identity, this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(), this->identity, this->tol);
+    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(), this->identity, this->tol);
 }
 
 
@@ -255,7 +256,7 @@ TYPED_TEST(ParIc, GenerateBanded)
         factorization_type::build().on(this->exec)->generate(this->banded);
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->banded_l_expect, this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(),
+    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(),
                         gko::as<Csr>(this->banded_l_expect->conj_transpose()),
                         this->tol);
 }
@@ -269,7 +270,7 @@ TYPED_TEST(ParIc, GenerateGeneral)
         factorization_type::build().on(this->exec)->generate(this->mtx_system);
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->mtx_l_it_expect, this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(),
+    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(),
                         gko::as<Csr>(this->mtx_l_it_expect->conj_transpose()),
                         this->tol);
 }
diff --git a/reference/test/factorization/par_ict_kernels.cpp b/reference/test/factorization/par_ict_kernels.cpp
index 2858e77b4cf..5f0fc25c021 100644
--- a/reference/test/factorization/par_ict_kernels.cpp
+++ b/reference/test/factorization/par_ict_kernels.cpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
 
 
 #include "core/factorization/factorization_kernels.hpp"
@@ -273,7 +274,7 @@ TYPED_TEST(ParIct, SetStrategies)
     ASSERT_EQ(fact->get_l_factor()->get_strategy()->get_name(),
               l_strategy->get_name());
     ASSERT_EQ(factory->get_parameters().lt_strategy, lt_strategy);
-    ASSERT_EQ(fact->get_lh_factor()->get_strategy()->get_name(),
+    ASSERT_EQ(fact->get_lt_factor()->get_strategy()->get_name(),
               lt_strategy->get_name());
 }
 
@@ -285,7 +286,7 @@ TYPED_TEST(ParIct, IsConsistentWithComposition)
     auto lin_op_l_factor =
         static_cast<const gko::LinOp *>(gko::lend(fact->get_l_factor()));
     auto lin_op_lt_factor =
-        static_cast<const gko::LinOp *>(gko::lend(fact->get_lh_factor()));
+        static_cast<const gko::LinOp *>(gko::lend(fact->get_lt_factor()));
     auto first_operator = gko::lend(fact->get_operators()[0]);
     auto second_operator = gko::lend(fact->get_operators()[1]);
 
@@ -299,7 +300,7 @@ TYPED_TEST(ParIct, GenerateIdentity)
     auto fact = this->fact_fact->generate(this->identity);
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->identity, this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(), this->identity, this->tol);
+    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(), this->identity, this->tol);
 }
 
 
@@ -311,7 +312,7 @@ TYPED_TEST(ParIct, GenerateDenseIdentity)
     auto fact = this->fact_fact->generate(gko::share(dense_id));
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->identity, this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(), this->identity, this->tol);
+    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(), this->identity, this->tol);
 }
 
 
@@ -327,7 +328,7 @@ TYPED_TEST(ParIct, GenerateWithExactSmallLimit)
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->mtx_l_small_expect,
                         this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(),
+    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(),
                         gko::as<Csr>(this->mtx_l_small_expect->transpose()),
                         this->tol);
 }
@@ -345,7 +346,7 @@ TYPED_TEST(ParIct, GenerateWithApproxSmallLimit)
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->mtx_l_small_expect,
                         this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(),
+    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(),
                         gko::as<Csr>(this->mtx_l_small_expect->transpose()),
                         this->tol);
 }
@@ -363,7 +364,7 @@ TYPED_TEST(ParIct, GenerateWithExactLargeLimit)
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->mtx_l_large_expect,
                         this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(),
+    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(),
                         gko::as<Csr>(this->mtx_l_large_expect->transpose()),
                         this->tol);
 }
@@ -381,7 +382,7 @@ TYPED_TEST(ParIct, GenerateWithApproxLargeLimit)
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->mtx_l_large_expect,
                         this->tol);
-    GKO_ASSERT_MTX_NEAR(fact->get_lh_factor(),
+    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(),
                         gko::as<Csr>(this->mtx_l_large_expect->transpose()),
                         this->tol);
 }

From 3808c92f10ecb56861cfab066381cb372d7ae8df Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 21 Jan 2021 15:32:44 +0100
Subject: [PATCH 0266/1008] update copyright year

---
 common/factorization/par_ic_kernels.hpp.inc          | 2 +-
 core/factorization/ic.cpp                            | 2 +-
 core/factorization/ic_kernels.hpp                    | 2 +-
 core/factorization/par_ic.cpp                        | 2 +-
 core/factorization/par_ic_kernels.hpp                | 2 +-
 core/test/factorization/par_ic.cpp                   | 2 +-
 core/test/factorization/par_ict.cpp                  | 2 +-
 core/test/preconditioner/ic.cpp                      | 2 +-
 cuda/factorization/ic_kernels.cu                     | 2 +-
 cuda/factorization/par_ic_kernels.cu                 | 2 +-
 cuda/test/factorization/ic_kernels.cpp               | 2 +-
 cuda/test/factorization/par_ic_kernels.cpp           | 2 +-
 dpcpp/factorization/ic_kernels.dp.cpp                | 2 +-
 dpcpp/factorization/par_ic_kernels.dp.cpp            | 2 +-
 examples/par-ilu-convergence/par-ilu-convergence.cpp | 2 +-
 hip/factorization/ic_kernels.hip.cpp                 | 2 +-
 hip/factorization/par_ic_kernels.hip.cpp             | 2 +-
 hip/test/factorization/ic_kernels.cpp                | 2 +-
 hip/test/factorization/par_ic_kernels.hip.cpp        | 2 +-
 include/ginkgo/core/factorization/ic.hpp             | 2 +-
 include/ginkgo/core/factorization/par_ic.hpp         | 2 +-
 include/ginkgo/core/preconditioner/ic.hpp            | 2 +-
 include/ginkgo/core/solver/solver_traits.hpp         | 2 +-
 omp/factorization/ic_kernels.cpp                     | 2 +-
 omp/factorization/par_ic_kernels.cpp                 | 2 +-
 omp/test/factorization/par_ic_kernels.cpp            | 2 +-
 reference/factorization/ic_kernels.cpp               | 2 +-
 reference/factorization/par_ic_kernels.cpp           | 2 +-
 reference/test/factorization/par_ic_kernels.cpp      | 2 +-
 reference/test/preconditioner/ic.cpp                 | 2 +-
 30 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/common/factorization/par_ic_kernels.hpp.inc b/common/factorization/par_ic_kernels.hpp.inc
index f18e6ca5d71..f83ab19c8ba 100644
--- a/common/factorization/par_ic_kernels.hpp.inc
+++ b/common/factorization/par_ic_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/factorization/ic.cpp b/core/factorization/ic.cpp
index 56b65d2be2a..3c8b6dd29ba 100644
--- a/core/factorization/ic.cpp
+++ b/core/factorization/ic.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/factorization/ic_kernels.hpp b/core/factorization/ic_kernels.hpp
index 8962e3a9110..4efcfe1565a 100644
--- a/core/factorization/ic_kernels.hpp
+++ b/core/factorization/ic_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/factorization/par_ic.cpp b/core/factorization/par_ic.cpp
index f4c3217889c..33e40d85bb0 100644
--- a/core/factorization/par_ic.cpp
+++ b/core/factorization/par_ic.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/factorization/par_ic_kernels.hpp b/core/factorization/par_ic_kernels.hpp
index 02815276e4f..4611cf88fa3 100644
--- a/core/factorization/par_ic_kernels.hpp
+++ b/core/factorization/par_ic_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/factorization/par_ic.cpp b/core/test/factorization/par_ic.cpp
index e78cfaa9a15..2bd4bc4fab4 100644
--- a/core/test/factorization/par_ic.cpp
+++ b/core/test/factorization/par_ic.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/factorization/par_ict.cpp b/core/test/factorization/par_ict.cpp
index 296f62bb9ec..b06281a8a30 100644
--- a/core/test/factorization/par_ict.cpp
+++ b/core/test/factorization/par_ict.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/preconditioner/ic.cpp b/core/test/preconditioner/ic.cpp
index a2e6345d2b5..c287887ced3 100644
--- a/core/test/preconditioner/ic.cpp
+++ b/core/test/preconditioner/ic.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/factorization/ic_kernels.cu b/cuda/factorization/ic_kernels.cu
index de30263008f..eb87b79c633 100644
--- a/cuda/factorization/ic_kernels.cu
+++ b/cuda/factorization/ic_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/factorization/par_ic_kernels.cu b/cuda/factorization/par_ic_kernels.cu
index 9c42b27ad09..35f49c40436 100644
--- a/cuda/factorization/par_ic_kernels.cu
+++ b/cuda/factorization/par_ic_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/factorization/ic_kernels.cpp b/cuda/test/factorization/ic_kernels.cpp
index e3ee5fe588d..d99121aa320 100644
--- a/cuda/test/factorization/ic_kernels.cpp
+++ b/cuda/test/factorization/ic_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/test/factorization/par_ic_kernels.cpp b/cuda/test/factorization/par_ic_kernels.cpp
index dec3b9617d9..ac454eb6b80 100644
--- a/cuda/test/factorization/par_ic_kernels.cpp
+++ b/cuda/test/factorization/par_ic_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/factorization/ic_kernels.dp.cpp b/dpcpp/factorization/ic_kernels.dp.cpp
index f6c6d8ffadb..4ea40141287 100644
--- a/dpcpp/factorization/ic_kernels.dp.cpp
+++ b/dpcpp/factorization/ic_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/factorization/par_ic_kernels.dp.cpp b/dpcpp/factorization/par_ic_kernels.dp.cpp
index 91b8a3c4862..9a1279927a5 100644
--- a/dpcpp/factorization/par_ic_kernels.dp.cpp
+++ b/dpcpp/factorization/par_ic_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/examples/par-ilu-convergence/par-ilu-convergence.cpp b/examples/par-ilu-convergence/par-ilu-convergence.cpp
index b2b4303049e..b72ea5e667a 100644
--- a/examples/par-ilu-convergence/par-ilu-convergence.cpp
+++ b/examples/par-ilu-convergence/par-ilu-convergence.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/factorization/ic_kernels.hip.cpp b/hip/factorization/ic_kernels.hip.cpp
index 8924c0521ac..4510a18781c 100644
--- a/hip/factorization/ic_kernels.hip.cpp
+++ b/hip/factorization/ic_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/factorization/par_ic_kernels.hip.cpp b/hip/factorization/par_ic_kernels.hip.cpp
index 39c80c57650..5365083d63c 100644
--- a/hip/factorization/par_ic_kernels.hip.cpp
+++ b/hip/factorization/par_ic_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/factorization/ic_kernels.cpp b/hip/test/factorization/ic_kernels.cpp
index a630f831895..34899790c7b 100644
--- a/hip/test/factorization/ic_kernels.cpp
+++ b/hip/test/factorization/ic_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/factorization/par_ic_kernels.hip.cpp b/hip/test/factorization/par_ic_kernels.hip.cpp
index eb51157269c..230ade961e5 100644
--- a/hip/test/factorization/par_ic_kernels.hip.cpp
+++ b/hip/test/factorization/par_ic_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/factorization/ic.hpp b/include/ginkgo/core/factorization/ic.hpp
index 8990ba3a15f..fb6da2d9764 100644
--- a/include/ginkgo/core/factorization/ic.hpp
+++ b/include/ginkgo/core/factorization/ic.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/factorization/par_ic.hpp b/include/ginkgo/core/factorization/par_ic.hpp
index 220aa6a7b83..ce94d3d32ae 100644
--- a/include/ginkgo/core/factorization/par_ic.hpp
+++ b/include/ginkgo/core/factorization/par_ic.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/preconditioner/ic.hpp b/include/ginkgo/core/preconditioner/ic.hpp
index e08894b836c..3e5ea6439e0 100644
--- a/include/ginkgo/core/preconditioner/ic.hpp
+++ b/include/ginkgo/core/preconditioner/ic.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/solver/solver_traits.hpp b/include/ginkgo/core/solver/solver_traits.hpp
index 0ce101d9aef..78757d06975 100644
--- a/include/ginkgo/core/solver/solver_traits.hpp
+++ b/include/ginkgo/core/solver/solver_traits.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/factorization/ic_kernels.cpp b/omp/factorization/ic_kernels.cpp
index 04120436bda..b20f9c1da2a 100644
--- a/omp/factorization/ic_kernels.cpp
+++ b/omp/factorization/ic_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/factorization/par_ic_kernels.cpp b/omp/factorization/par_ic_kernels.cpp
index 478d8363ea6..67205efbaf7 100644
--- a/omp/factorization/par_ic_kernels.cpp
+++ b/omp/factorization/par_ic_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/factorization/par_ic_kernels.cpp b/omp/test/factorization/par_ic_kernels.cpp
index 34d37095454..b44980449ec 100644
--- a/omp/test/factorization/par_ic_kernels.cpp
+++ b/omp/test/factorization/par_ic_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/factorization/ic_kernels.cpp b/reference/factorization/ic_kernels.cpp
index 28b433bf331..b1d075044ad 100644
--- a/reference/factorization/ic_kernels.cpp
+++ b/reference/factorization/ic_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/factorization/par_ic_kernels.cpp b/reference/factorization/par_ic_kernels.cpp
index f71e28d2e00..95cc8aa1cda 100644
--- a/reference/factorization/par_ic_kernels.cpp
+++ b/reference/factorization/par_ic_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/factorization/par_ic_kernels.cpp b/reference/test/factorization/par_ic_kernels.cpp
index 015af22a491..899f5e1a34e 100644
--- a/reference/test/factorization/par_ic_kernels.cpp
+++ b/reference/test/factorization/par_ic_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/preconditioner/ic.cpp b/reference/test/preconditioner/ic.cpp
index 15884f1f01d..24d49d0b5fb 100644
--- a/reference/test/preconditioner/ic.cpp
+++ b/reference/test/preconditioner/ic.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

From fccf3d5bf044e1b89bb3d94e071e1fee4c84b909 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 27 Jan 2021 21:39:12 +0100
Subject: [PATCH 0267/1008] fix MSVC build for solver_traits

---
 include/ginkgo/core/preconditioner/ic.hpp    |  4 ++--
 include/ginkgo/core/preconditioner/ilu.hpp   |  4 ++--
 include/ginkgo/core/solver/solver_traits.hpp | 15 ++-------------
 3 files changed, 6 insertions(+), 17 deletions(-)

diff --git a/include/ginkgo/core/preconditioner/ic.hpp b/include/ginkgo/core/preconditioner/ic.hpp
index 3e5ea6439e0..ec1f80607b6 100644
--- a/include/ginkgo/core/preconditioner/ic.hpp
+++ b/include/ginkgo/core/preconditioner/ic.hpp
@@ -289,7 +289,7 @@ class Ic : public EnableLinOp<Ic<LSolverType, IndexType>>, public Transposable {
      * preconditioner.
      */
     template <typename SolverType>
-    static std::enable_if_t<solver::has_with_criteria<SolverType>(),
+    static std::enable_if_t<solver::has_with_criteria<SolverType>::value,
                             std::unique_ptr<SolverType>>
     generate_default_solver(const std::shared_ptr<const Executor> &exec,
                             const std::shared_ptr<const LinOp> &mtx)
@@ -313,7 +313,7 @@ class Ic : public EnableLinOp<Ic<LSolverType, IndexType>>, public Transposable {
      * @copydoc generate_default_solver
      */
     template <typename SolverType>
-    static std::enable_if_t<!solver::has_with_criteria<SolverType>(),
+    static std::enable_if_t<!solver::has_with_criteria<SolverType>::value,
                             std::unique_ptr<SolverType>>
     generate_default_solver(const std::shared_ptr<const Executor> &exec,
                             const std::shared_ptr<const LinOp> &mtx)
diff --git a/include/ginkgo/core/preconditioner/ilu.hpp b/include/ginkgo/core/preconditioner/ilu.hpp
index 554f599f688..f138212bb4a 100644
--- a/include/ginkgo/core/preconditioner/ilu.hpp
+++ b/include/ginkgo/core/preconditioner/ilu.hpp
@@ -316,7 +316,7 @@ class Ilu : public EnableLinOp<
      * preconditioner.
      */
     template <typename SolverType>
-    static std::enable_if_t<solver::has_with_criteria<SolverType>(),
+    static std::enable_if_t<solver::has_with_criteria<SolverType>::value,
                             std::unique_ptr<SolverType>>
     generate_default_solver(const std::shared_ptr<const Executor> &exec,
                             const std::shared_ptr<const LinOp> &mtx)
@@ -340,7 +340,7 @@ class Ilu : public EnableLinOp<
      * @copydoc generate_default_solver
      */
     template <typename SolverType>
-    static std::enable_if_t<!solver::has_with_criteria<SolverType>(),
+    static std::enable_if_t<!solver::has_with_criteria<SolverType>::value,
                             std::unique_ptr<SolverType>>
     generate_default_solver(const std::shared_ptr<const Executor> &exec,
                             const std::shared_ptr<const LinOp> &mtx)
diff --git a/include/ginkgo/core/solver/solver_traits.hpp b/include/ginkgo/core/solver/solver_traits.hpp
index 78757d06975..e303e4b1739 100644
--- a/include/ginkgo/core/solver/solver_traits.hpp
+++ b/include/ginkgo/core/solver/solver_traits.hpp
@@ -40,7 +40,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace gko {
 namespace solver {
-namespace detail {
 
 
 /**
@@ -55,7 +54,7 @@ namespace detail {
  *
  */
 template <typename SolverType, typename = void>
-struct has_with_criteria_impl : std::false_type {};
+struct has_with_criteria : std::false_type {};
 
 /**
  * @copydoc has_with_criteria
@@ -65,22 +64,12 @@ struct has_with_criteria_impl : std::false_type {};
  *            as a specialization, which is why `xstd::void_t` is used.
  */
 template <typename SolverType>
-struct has_with_criteria_impl<
+struct has_with_criteria<
     SolverType, xstd::void_t<decltype(SolverType::build().with_criteria(
                     std::shared_ptr<const stop::CriterionFactory>()))>>
     : std::true_type {};
 
 
-}  // namespace detail
-
-
-template <typename SolverType>
-constexpr bool has_with_criteria()
-{
-    return detail::has_with_criteria_impl<SolverType>::value;
-}
-
-
 }  // namespace solver
 }  // namespace gko
 

From 29acef9670fcd49aa0aabd446ccae55bf61513fe Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 28 Jan 2021 10:51:45 +0100
Subject: [PATCH 0268/1008] disable MinGW and Cygwin debug builds

They fail due to file size limits/string table overflows,
probably since we create too many symbols in debug.
---
 .github/workflows/windows-build.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml
index 6cdc7d82462..25d9987b38a 100644
--- a/.github/workflows/windows-build.yml
+++ b/.github/workflows/windows-build.yml
@@ -65,7 +65,6 @@ jobs:
       fail-fast: false
       matrix:
         config:
-        - {shared: "ON", build_type: "Debug", name: "omp/debug/shared", cflags: "-O1"}
         - {shared: "OFF", build_type: "Release", name: "omp/release/static", cflags: ""}
     name: mingw/${{ matrix.config.name }}
     runs-on: [windows-latest]
@@ -98,7 +97,6 @@ jobs:
       fail-fast: false
       matrix:
         config:
-        - {shared: "ON", build_type: "Debug", name: "omp/debug/shared", cflags: "-O1"}
         - {shared: "OFF", build_type: "Release", name: "omp/release/static", cflags: ""}
     name: cygwin/${{ matrix.config.name }}
     runs-on: [windows-latest]

From c9ac0464c3a2d7662115964791b486c18e84cb32 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 2 Jun 2020 15:08:33 +0200
Subject: [PATCH 0269/1008] Add cmake HWLOC detection.

---
 CMakeLists.txt                                |  18 +-
 cmake/Modules/FindHWLOC.cmake                 | 331 ++++++++++++++++++
 ... => autodetect_library_dependencies.cmake} |   9 +
 core/CMakeLists.txt                           |   4 +
 include/ginkgo/config.hpp.in                  |   9 +
 5 files changed, 369 insertions(+), 2 deletions(-)
 create mode 100644 cmake/Modules/FindHWLOC.cmake
 rename cmake/{autodetect_executors.cmake => autodetect_library_dependencies.cmake} (83%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b1144e771e9..64a241a9d66 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -15,9 +15,11 @@ project(Ginkgo LANGUAGES C CXX VERSION 1.3.0 DESCRIPTION "A numerical linear alg
 set(Ginkgo_VERSION_TAG "master")
 set(PROJECT_VERSION_TAG ${Ginkgo_VERSION_TAG})
 
-# Determine which executors can be compiled
+# Add custom modules search path for the find_package functions.
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules/")
+# Determine which modules can be compiled
 include(cmake/hip_path.cmake)
-include(cmake/autodetect_executors.cmake)
+include(cmake/autodetect_library_dependencies.cmake)
 include(cmake/build_type_helpers.cmake)
 
 # Ginkgo configuration options
@@ -31,6 +33,7 @@ option(GINKGO_BUILD_DPCPP
     "Compile DPC++ kernels for Intel GPUs or other DPC++ enabled hardware" ${GINKGO_HAS_DPCPP})
 option(GINKGO_BUILD_CUDA "Compile kernels for NVIDIA GPUs" ${GINKGO_HAS_CUDA})
 option(GINKGO_BUILD_HIP "Compile kernels for AMD or NVIDIA GPUs" ${GINKGO_HAS_HIP})
+option(GINKGO_WITH_HWLOC "Build Ginkgo with HWLOC support" ${GINKGO_HAS_HWLOC})
 option(GINKGO_BUILD_DOC "Generate documentation" OFF)
 option(GINKGO_SKIP_DEPENDENCY_UPDATE
     "Do not update dependencies each time the project is rebuilt" ON)
@@ -143,6 +146,17 @@ if(PAPI_sde_FOUND)
     set(GINKGO_HAVE_PAPI_SDE 1)
 endif()
 
+# Find HWLOC if needed
+set(GINKGO_HAVE_HWLOC 0)
+if(GINKGO_WITH_HWLOC)
+  find_package(HWLOC REQUIRED)
+  set(GKO_HWLOC_XMLFILE "${HWLOC_XMLFILE}")
+  message(STATUS "HWLOC XML file set to: ${GKO_HWLOC_XMLFILE}")
+  if(HWLOC_FOUND)
+    set(GINKGO_HAVE_HWLOC 1)
+  endif()
+endif()
+
 set(GINKGO_HIP_PLATFORM_NVCC 0)
 set(GINKGO_HIP_PLATFORM_HCC 0)
 
diff --git a/cmake/Modules/FindHWLOC.cmake b/cmake/Modules/FindHWLOC.cmake
new file mode 100644
index 00000000000..a831b5c725a
--- /dev/null
+++ b/cmake/Modules/FindHWLOC.cmake
@@ -0,0 +1,331 @@
+###
+#
+# @copyright (c) 2009-2014 The University of Tennessee and The University
+#                          of Tennessee Research Foundation.
+#                          All rights reserved.
+# @copyright (c) 2012-2014 Inria. All rights reserved.
+# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+#
+###
+#
+# - Find HWLOC include dirs and libraries
+# Use this module by invoking find_package with the form:
+#  find_package(HWLOC
+#               [REQUIRED]) # Fail with error if hwloc is not found
+#
+# This module finds headers and hwloc library.
+# Results are reported in variables:
+#  HWLOC_FOUND           - True if headers and requested libraries were found
+#  HWLOC_INCLUDE_DIRS    - hwloc include directories
+#  HWLOC_LIBRARY_DIRS    - Link directories for hwloc libraries
+#  HWLOC_LIBRARIES       - hwloc component libraries to be linked
+#
+# The user can give specific paths where to find the libraries adding cmake
+# options at configure (ex: cmake path/to/project -DHWLOC_DIR=path/to/hwloc):
+#  HWLOC_DIR             - Where to find the base directory of hwloc
+#  HWLOC_INCDIR          - Where to find the header files
+#  HWLOC_LIBDIR          - Where to find the library files
+# The module can also look for the following environment variables if paths
+# are not given as cmake variable: HWLOC_DIR, HWLOC_INCDIR, HWLOC_LIBDIR
+
+#=============================================================================
+# Copyright 2012-2013 Inria
+# Copyright 2012-2013 Emmanuel Agullo
+# Copyright 2012-2013 Mathieu Faverge
+# Copyright 2012      Cedric Castagnede
+# Copyright 2013      Florent Pruvost
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file MORSE-Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of Morse, substitute the full
+#  License text for the above reference.)
+
+include(CheckStructHasMember)
+include(CheckCSourceCompiles)
+
+if (NOT HWLOC_FOUND)
+  set(HWLOC_DIR "" CACHE PATH "Installation directory of HWLOC library")
+  if (NOT HWLOC_FIND_QUIETLY)
+    message(STATUS "A cache variable, namely HWLOC_DIR, has been set to specify the install directory of HWLOC")
+  endif()
+endif()
+
+set(ENV_HWLOC_DIR "$ENV{HWLOC_DIR}")
+set(ENV_HWLOC_INCDIR "$ENV{HWLOC_INCDIR}")
+set(ENV_HWLOC_LIBDIR "$ENV{HWLOC_LIBDIR}")
+set(HWLOC_GIVEN_BY_USER "FALSE")
+if ( HWLOC_DIR OR ( HWLOC_INCDIR AND HWLOC_LIBDIR) OR ENV_HWLOC_DIR OR (ENV_HWLOC_INCDIR AND ENV_HWLOC_LIBDIR) )
+  set(HWLOC_GIVEN_BY_USER "TRUE")
+endif()
+
+# Optionally use pkg-config to detect include/library dirs (if pkg-config is available)
+# -------------------------------------------------------------------------------------
+include(FindPkgConfig)
+find_package(PkgConfig QUIET)
+if( PKG_CONFIG_EXECUTABLE AND NOT HWLOC_GIVEN_BY_USER )
+
+  pkg_search_module(HWLOC hwloc)
+  if (NOT HWLOC_FIND_QUIETLY)
+    if (HWLOC_FOUND AND HWLOC_LIBRARIES)
+      message(STATUS "Looking for HWLOC - found using PkgConfig")
+      #if(NOT HWLOC_INCLUDE_DIRS)
+      #    message("${Magenta}HWLOC_INCLUDE_DIRS is empty using PkgConfig."
+      #        "Perhaps the path to hwloc headers is already present in your"
+      #        "C(PLUS)_INCLUDE_PATH environment variable.${ColourReset}")
+      #endif()
+    else()
+      message(STATUS "${Magenta}Looking for HWLOC - not found using PkgConfig."
+	"\n   Perhaps you should add the directory containing hwloc.pc to"
+	"\n   the PKG_CONFIG_PATH environment variable.${ColourReset}")
+    endif()
+  endif()
+
+endif( PKG_CONFIG_EXECUTABLE AND NOT HWLOC_GIVEN_BY_USER )
+
+if( (NOT PKG_CONFIG_EXECUTABLE) OR (PKG_CONFIG_EXECUTABLE AND NOT HWLOC_FOUND) OR (HWLOC_GIVEN_BY_USER) )
+
+  if (NOT HWLOC_FIND_QUIETLY)
+    message(STATUS "Looking for HWLOC - PkgConfig not used")
+  endif()
+
+  # Looking for include
+  # -------------------
+
+  # Add system include paths to search include
+  # ------------------------------------------
+  unset(_inc_env)
+  if(ENV_HWLOC_INCDIR)
+    list(APPEND _inc_env "${ENV_HWLOC_INCDIR}")
+  elseif(ENV_HWLOC_DIR)
+    list(APPEND _inc_env "${ENV_HWLOC_DIR}")
+    list(APPEND _inc_env "${ENV_HWLOC_DIR}/include")
+    list(APPEND _inc_env "${ENV_HWLOC_DIR}/include/hwloc")
+  else()
+    if(WIN32)
+      string(REPLACE ":" ";" _inc_env "$ENV{INCLUDE}")
+    else()
+      string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}")
+      list(APPEND _inc_env "${_path_env}")
+      string(REPLACE ":" ";" _path_env "$ENV{C_INCLUDE_PATH}")
+      list(APPEND _inc_env "${_path_env}")
+      string(REPLACE ":" ";" _path_env "$ENV{CPATH}")
+      list(APPEND _inc_env "${_path_env}")
+      string(REPLACE ":" ";" _path_env "$ENV{INCLUDE_PATH}")
+      list(APPEND _inc_env "${_path_env}")
+    endif()
+  endif()
+  list(APPEND _inc_env "${CMAKE_PLATFORM_IMPLICIT_INCLUDE_DIRECTORIES}")
+  list(APPEND _inc_env "${CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES}")
+  list(REMOVE_DUPLICATES _inc_env)
+
+  # set paths where to look for
+  set(PATH_TO_LOOK_FOR "${_inc_env}")
+
+  # Try to find the hwloc header in the given paths
+  # -------------------------------------------------
+  # call cmake macro to find the header path
+  if(HWLOC_INCDIR)
+    set(HWLOC_hwloc.h_DIRS "HWLOC_hwloc.h_DIRS-NOTFOUND")
+    find_path(HWLOC_hwloc.h_DIRS
+      NAMES hwloc.h
+      HINTS ${HWLOC_INCDIR})
+  else()
+    if(HWLOC_DIR)
+      set(HWLOC_hwloc.h_DIRS "HWLOC_hwloc.h_DIRS-NOTFOUND")
+      find_path(HWLOC_hwloc.h_DIRS
+	NAMES hwloc.h
+	HINTS ${HWLOC_DIR}
+	PATH_SUFFIXES "include" "include/hwloc")
+    else()
+      set(HWLOC_hwloc.h_DIRS "HWLOC_hwloc.h_DIRS-NOTFOUND")
+      find_path(HWLOC_hwloc.h_DIRS
+	NAMES hwloc.h
+	HINTS ${PATH_TO_LOOK_FOR}
+	PATH_SUFFIXES "hwloc")
+    endif()
+  endif()
+  mark_as_advanced(HWLOC_hwloc.h_DIRS)
+
+  # Add path to cmake variable
+  # ------------------------------------
+  if (HWLOC_hwloc.h_DIRS)
+    set(HWLOC_INCLUDE_DIRS "${HWLOC_hwloc.h_DIRS}")
+  else ()
+    set(HWLOC_INCLUDE_DIRS "HWLOC_INCLUDE_DIRS-NOTFOUND")
+    if(NOT HWLOC_FIND_QUIETLY)
+      message(STATUS "Looking for hwloc -- hwloc.h not found")
+    endif()
+  endif ()
+
+  if (HWLOC_INCLUDE_DIRS)
+    list(REMOVE_DUPLICATES HWLOC_INCLUDE_DIRS)
+  endif ()
+
+
+  # Looking for lib
+  # ---------------
+
+  # Add system library paths to search lib
+  # --------------------------------------
+  unset(_lib_env)
+  if(ENV_HWLOC_LIBDIR)
+    list(APPEND _lib_env "${ENV_HWLOC_LIBDIR}")
+  elseif(ENV_HWLOC_DIR)
+    list(APPEND _lib_env "${ENV_HWLOC_DIR}")
+    list(APPEND _lib_env "${ENV_HWLOC_DIR}/lib")
+  else()
+    if(WIN32)
+      string(REPLACE ":" ";" _lib_env "$ENV{LIB}")
+    else()
+      if(APPLE)
+	string(REPLACE ":" ";" _lib_env "$ENV{DYLD_LIBRARY_PATH}")
+      else()
+	string(REPLACE ":" ";" _lib_env "$ENV{LD_LIBRARY_PATH}")
+      endif()
+      list(APPEND _lib_env "${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES}")
+      list(APPEND _lib_env "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}")
+    endif()
+  endif()
+  list(REMOVE_DUPLICATES _lib_env)
+
+  # set paths where to look for
+  set(PATH_TO_LOOK_FOR "${_lib_env}")
+
+  # Try to find the hwloc lib in the given paths
+  # ----------------------------------------------
+
+  # call cmake macro to find the lib path
+  if(HWLOC_LIBDIR)
+    set(HWLOC_hwloc_LIBRARY "HWLOC_hwloc_LIBRARY-NOTFOUND")
+    find_library(HWLOC_hwloc_LIBRARY
+      NAMES hwloc
+      HINTS ${HWLOC_LIBDIR})
+  else()
+    if(HWLOC_DIR)
+      set(HWLOC_hwloc_LIBRARY "HWLOC_hwloc_LIBRARY-NOTFOUND")
+      find_library(HWLOC_hwloc_LIBRARY
+	NAMES hwloc
+	HINTS ${HWLOC_DIR}
+	PATH_SUFFIXES lib lib32 lib64)
+    else()
+      set(HWLOC_hwloc_LIBRARY "HWLOC_hwloc_LIBRARY-NOTFOUND")
+      find_library(HWLOC_hwloc_LIBRARY
+	NAMES hwloc
+	HINTS ${PATH_TO_LOOK_FOR})
+    endif()
+  endif()
+  mark_as_advanced(HWLOC_hwloc_LIBRARY)
+
+  # If found, add path to cmake variable
+  # ------------------------------------
+  if (HWLOC_hwloc_LIBRARY)
+    get_filename_component(hwloc_lib_path ${HWLOC_hwloc_LIBRARY} PATH)
+    # set cmake variables (respects naming convention)
+    set(HWLOC_LIBRARIES    "${HWLOC_hwloc_LIBRARY}")
+    set(HWLOC_LIBRARY_DIRS "${hwloc_lib_path}")
+  else ()
+    set(HWLOC_LIBRARIES    "HWLOC_LIBRARIES-NOTFOUND")
+    set(HWLOC_LIBRARY_DIRS "HWLOC_LIBRARY_DIRS-NOTFOUND")
+    if(NOT HWLOC_FIND_QUIETLY)
+      message(STATUS "Looking for hwloc -- lib hwloc not found")
+    endif()
+  endif ()
+
+  if (HWLOC_LIBRARY_DIRS)
+    list(REMOVE_DUPLICATES HWLOC_LIBRARY_DIRS)
+  endif ()
+
+  # check a function to validate the find
+  if(HWLOC_LIBRARIES)
+
+    set(REQUIRED_INCDIRS)
+    set(REQUIRED_LIBDIRS)
+    set(REQUIRED_LIBS)
+
+    # HWLOC
+    if (HWLOC_INCLUDE_DIRS)
+      set(REQUIRED_INCDIRS "${HWLOC_INCLUDE_DIRS}")
+    endif()
+    if (HWLOC_LIBRARY_DIRS)
+      set(REQUIRED_LIBDIRS "${HWLOC_LIBRARY_DIRS}")
+    endif()
+    set(REQUIRED_LIBS "${HWLOC_LIBRARIES}")
+
+    # set required libraries for link
+    set(CMAKE_REQUIRED_INCLUDES "${REQUIRED_INCDIRS}")
+    set(CMAKE_REQUIRED_LIBRARIES)
+    foreach(lib_dir ${REQUIRED_LIBDIRS})
+      list(APPEND CMAKE_REQUIRED_LIBRARIES "-L${lib_dir}")
+    endforeach()
+    list(APPEND CMAKE_REQUIRED_LIBRARIES "${REQUIRED_LIBS}")
+    string(REGEX REPLACE "^ -" "-" CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}")
+
+    # test link
+    unset(HWLOC_WORKS CACHE)
+    include(CheckFunctionExists)
+    check_function_exists(hwloc_topology_init HWLOC_WORKS)
+    mark_as_advanced(HWLOC_WORKS)
+
+    if(NOT HWLOC_WORKS)
+      if(NOT HWLOC_FIND_QUIETLY)
+	message(STATUS "Looking for hwloc : test of hwloc_topology_init with hwloc library fails")
+	message(STATUS "CMAKE_REQUIRED_LIBRARIES: ${CMAKE_REQUIRED_LIBRARIES}")
+	message(STATUS "CMAKE_REQUIRED_INCLUDES: ${CMAKE_REQUIRED_INCLUDES}")
+	message(STATUS "Check in CMakeFiles/CMakeError.log to figure out why it fails")
+      endif()
+    endif()
+    set(CMAKE_REQUIRED_INCLUDES)
+    set(CMAKE_REQUIRED_FLAGS)
+    set(CMAKE_REQUIRED_LIBRARIES)
+  endif(HWLOC_LIBRARIES)
+
+endif( (NOT PKG_CONFIG_EXECUTABLE) OR (PKG_CONFIG_EXECUTABLE AND NOT HWLOC_FOUND) OR (HWLOC_GIVEN_BY_USER) )
+
+if (HWLOC_LIBRARIES)
+  if (HWLOC_LIBRARY_DIRS)
+    list(GET HWLOC_LIBRARY_DIRS 0 first_lib_path)
+  else()
+    list(GET HWLOC_LIBRARIES 0 first_lib)
+    get_filename_component(first_lib_path "${first_lib}" PATH)
+  endif()
+  if (${first_lib_path} MATCHES "/lib(32|64)?$")
+    string(REGEX REPLACE "/lib(32|64)?$" "" not_cached_dir "${first_lib_path}")
+    set(HWLOC_DIR_FOUND "${not_cached_dir}" CACHE PATH "Installation directory of HWLOC library" FORCE)
+  else()
+    set(HWLOC_DIR_FOUND "${first_lib_path}" CACHE PATH "Installation directory of HWLOC library" FORCE)
+  endif()
+endif()
+mark_as_advanced(HWLOC_DIR)
+mark_as_advanced(HWLOC_DIR_FOUND)
+
+# check that HWLOC has been found
+# -------------------------------
+include(FindPackageHandleStandardArgs)
+if (PKG_CONFIG_EXECUTABLE AND HWLOC_FOUND)
+  find_package_handle_standard_args(HWLOC DEFAULT_MSG
+    HWLOC_LIBRARIES)
+else()
+  find_package_handle_standard_args(HWLOC DEFAULT_MSG
+    HWLOC_LIBRARIES
+    HWLOC_WORKS)
+endif()
+
+if (HWLOC_FOUND)
+  set(HWLOC_SAVE_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
+  list(APPEND CMAKE_REQUIRED_INCLUDES ${HWLOC_INCLUDE_DIRS})
+
+  # test headers to guess the version
+  check_struct_has_member( "struct hwloc_obj" parent hwloc.h HAVE_HWLOC_PARENT_MEMBER )
+  check_struct_has_member( "struct hwloc_cache_attr_s" size hwloc.h HAVE_HWLOC_CACHE_ATTR )
+  check_c_source_compiles( "#include <hwloc.h>
+	    int main(void) { hwloc_obj_t o; o->type = HWLOC_OBJ_PU; return 0;}" HAVE_HWLOC_OBJ_PU)
+  include(CheckLibraryExists)
+  check_library_exists(${HWLOC_LIBRARIES} hwloc_bitmap_free "" HAVE_HWLOC_BITMAP)
+
+  set(CMAKE_REQUIRED_INCLUDES ${HWLOC_SAVE_CMAKE_REQUIRED_INCLUDES})
+endif()
diff --git a/cmake/autodetect_executors.cmake b/cmake/autodetect_library_dependencies.cmake
similarity index 83%
rename from cmake/autodetect_executors.cmake
rename to cmake/autodetect_library_dependencies.cmake
index c432a92d593..0a108d14006 100644
--- a/cmake/autodetect_executors.cmake
+++ b/cmake/autodetect_library_dependencies.cmake
@@ -2,12 +2,14 @@ set(GINKGO_HAS_OMP OFF)
 set(GINKGO_HAS_CUDA OFF)
 set(GINKGO_HAS_DPCPP OFF)
 set(GINKGO_HAS_HIP OFF)
+set(GINKGO_HAS_HWLOC OFF)
 find_package(OpenMP)
 include(CheckLanguage)
 check_language(CUDA)
 try_compile(GKO_CAN_COMPILE_DPCPP ${PROJECT_BINARY_DIR}/dpcpp
     SOURCES ${PROJECT_SOURCE_DIR}/dpcpp/test_dpcpp.dp.cpp
     CXX_STANDARD 17)
+find_package(HWLOC)
 
 if(OpenMP_CXX_FOUND)
     if(NOT DEFINED GINKGO_BUILD_OMP)
@@ -16,6 +18,13 @@ if(OpenMP_CXX_FOUND)
     set(GINKGO_HAS_OMP ON)
 endif()
 
+if(HWLOC_FOUND)
+  if(NOT DEFINED GINKGO_WITH_HWLOC)
+    message(STATUS "Enabling HWLOC support")
+  endif()
+  set(GINKGO_HAS_HWLOC ON)
+endif()
+
 if(CMAKE_CUDA_COMPILER)
     if(NOT DEFINED GINKGO_BUILD_CUDA)
         message(STATUS "Enabling CUDA executor")
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index a63e108ad57..07b70d32c78 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -66,6 +66,10 @@ target_link_libraries(ginkgo
 if (GINKGO_HAVE_PAPI_SDE)
     target_link_libraries(ginkgo PUBLIC PAPI::PAPI)
 endif()
+if (GINKGO_HAVE_HWLOC)
+  target_include_directories(ginkgo PRIVATE ${HWLOC_INCLUDE_DIRS})
+  target_link_libraries(ginkgo PRIVATE ${HWLOC_LIBRARIES})
+endif()
 ginkgo_default_includes(ginkgo)
 ginkgo_install_library(ginkgo core)
 
diff --git a/include/ginkgo/config.hpp.in b/include/ginkgo/config.hpp.in
index d8d14e5cef2..55042286815 100644
--- a/include/ginkgo/config.hpp.in
+++ b/include/ginkgo/config.hpp.in
@@ -73,5 +73,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_HAVE_PAPI_SDE @GINKGO_HAVE_PAPI_SDE@
 // clang-format on
 
+/* Is HWLOC available ? */
+// clang-format off
+#define GKO_HAVE_HWLOC @GINKGO_HAVE_HWLOC@
+// clang-format on
+
+/* Set the system provided XML config file*/
+// clang-format off
+#define GKO_HWLOC_XMLFILE "@GKO_HWLOC_XMLFILE@"
+// clang-format on
 
 #endif  // GKO_INCLUDE_CONFIG_H

From e79713f45b923690edd6c1b8aa31da12f926b13f Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 2 Jun 2020 23:01:48 +0200
Subject: [PATCH 0270/1008] Add a Topology mixin dependent on Executors.

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 include/ginkgo/core/base/executor.hpp       |   1 +
 include/ginkgo/core/base/machine_config.hpp | 253 ++++++++++++++++++++
 2 files changed, 254 insertions(+)
 create mode 100644 include/ginkgo/core/base/machine_config.hpp

diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 4a64a812b08..34f96da8ae6 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <type_traits>
 
 
+#include <ginkgo/core/base/machine_config.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/log/logger.hpp>
 #include <ginkgo/core/synthesizer/containers.hpp>
diff --git a/include/ginkgo/core/base/machine_config.hpp b/include/ginkgo/core/base/machine_config.hpp
new file mode 100644
index 00000000000..d91a4925919
--- /dev/null
+++ b/include/ginkgo/core/base/machine_config.hpp
@@ -0,0 +1,253 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_BASE_MACHINE_CONFIG_HPP_
+#define GKO_CORE_BASE_MACHINE_CONFIG_HPP_
+
+
+#include <cassert>
+#include <cstddef>
+#include <functional>
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+
+#include <ginkgo/config.hpp>
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
+#if GKO_HAVE_HWLOC
+#include <hwloc.h>
+#endif
+
+
+#if GKO_HAVE_HWLOC == 0
+struct hwloc_obj_type_t {};
+struct hwloc_obj_t {};
+#endif
+
+
+struct hwloc_topology;
+
+
+namespace gko {
+namespace machine_config {
+
+
+struct topology_obj_info {
+    hwloc_obj_t obj;
+    int numa;
+    std::size_t logical_id;
+    std::size_t physical_id;  // for GPUs, this is their number in the numa
+};
+
+
+struct machine_info {
+    virtual const topology_obj_info &get_pu(std::size_t id) = 0;
+    virtual const topology_obj_info &get_core(std::size_t id) = 0;
+    virtual const topology_obj_info &get_gpu(std::size_t id) = 0;
+    virtual std::size_t get_num_pus() = 0;
+    virtual std::size_t get_num_cores() = 0;
+    virtual std::size_t get_num_gpus() = 0;
+    virtual std::size_t get_num_numas() = 0;
+};
+
+
+struct binder {
+    virtual void bind_to_core(std::size_t id) = 0;
+    virtual void bind_to_pu(std::size_t id) = 0;
+};
+
+
+template <class Executor>
+class Topology : public machine_info, public binder {
+public:
+    Topology(Topology &) = delete;
+    Topology(Topology &&) = delete;
+
+    static std::unique_ptr<Topology> create()
+    {
+        return std::unique_ptr<Topology>(new Topology());
+    }
+
+
+    void bind_to_core(std::size_t id) override
+    {
+        hwloc_binding_helper(this->cores_, id);
+    }
+
+    void bind_to_pu(std::size_t id) override
+    {
+        hwloc_binding_helper(this->pus_, id);
+    }
+
+    const topology_obj_info &get_pu(std::size_t id) override
+    {
+        GKO_ENSURE_IN_BOUNDS(id, pus_.size());
+        return pus_[id];
+    }
+
+    const topology_obj_info &get_core(std::size_t id) override
+    {
+        GKO_ENSURE_IN_BOUNDS(id, cores_.size());
+        return cores_[id];
+    }
+
+    const topology_obj_info &get_gpu(std::size_t id) override
+    {
+        GKO_ENSURE_IN_BOUNDS(id, gpus_.size());
+        return gpus_[id];
+    }
+
+
+    std::size_t get_num_pus() override { return pus_.size(); }
+    std::size_t get_num_cores() override { return cores_.size(); }
+    std::size_t get_num_gpus() override { return gpus_.size(); }
+    std::size_t get_num_numas() override { return num_numas_; }
+
+    virtual void load_gpus() {}
+
+    void hwloc_binding_helper(std::vector<topology_obj_info> &obj,
+                              std::size_t id)
+    {
+#if GKO_HAVE_HWLOC
+        auto bitmap = hwloc_bitmap_alloc();
+        hwloc_bitmap_set(bitmap, obj[id].physical_id);
+        hwloc_bitmap_singlify(bitmap);
+        hwloc_set_cpubind(topo_.get(), bitmap, 0);
+        hwloc_bitmap_free(bitmap);
+#endif
+    }
+
+
+protected:
+    Topology()
+    {
+#if GKO_HAVE_HWLOC
+
+
+        this->topo_ = topo_manager<hwloc_topology>(init_topology(),
+                                                   hwloc_topology_destroy);
+
+        load_objects(HWLOC_OBJ_CORE, this->cores_);
+        load_objects(HWLOC_OBJ_PU, this->pus_);
+        this->load_gpus();
+
+        num_numas_ = hwloc_get_nbobjs_by_type(topo_.get(), HWLOC_OBJ_PACKAGE);
+#else
+
+        this->topo_ = topo_manager<hwloc_topology>();
+
+#endif
+    }
+
+
+#if GKO_HAVE_HWLOC
+
+
+    // The objects should be sorted by logical index since hwloc uses logical
+    // index with these functions
+    void load_objects(hwloc_obj_type_t type,
+                      std::vector<topology_obj_info> &vector)
+    {
+        unsigned nbcores = hwloc_get_nbobjs_by_type(topo_.get(), type);
+        for (unsigned i = 0; i < nbcores; i++) {
+            hwloc_obj_t obj = hwloc_get_obj_by_type(topo_.get(), type, i);
+            vector.push_back(
+                topology_obj_info{obj, hwloc_bitmap_first(obj->nodeset),
+                                  obj->logical_index, obj->os_index});
+        }
+    }
+
+
+    hwloc_topology *init_topology()
+    {
+        hwloc_topology_t tmp;
+        hwloc_topology_init(&tmp);
+
+#if HWLOC_API_VERSION >= 0x00020000
+        hwloc_topology_set_io_types_filter(tmp,
+                                           HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
+#else
+        hwloc_topology_set_flags(tmp, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
+#endif
+        hwloc_topology_set_xml(tmp, GKO_HWLOC_XMLFILE);
+        hwloc_topology_load(tmp);
+
+        return tmp;
+    }
+
+
+    static void hwloc_print_children(hwloc_topology *topology, hwloc_obj_t obj,
+                                     int depth)
+    {
+        char type[32], attr[1024];
+        unsigned i;
+        hwloc_obj_type_snprintf(type, sizeof(type), obj, 0);
+        std::cout << std::string(2 * depth, ' ') << type;
+        if (obj->os_index != (unsigned)-1) {
+            std::cout << "#" << obj->os_index;
+        }
+        hwloc_obj_attr_snprintf(attr, sizeof(attr), obj, " ", 0);
+        if (*attr) {
+            std::cout << "(" << attr << ")";
+        }
+        std::cout << std::endl;
+        for (i = 0; i < obj->arity; i++) {
+            hwloc_print_children(topology, obj->children[i], depth + 1);
+        }
+    }
+
+
+#endif
+
+private:
+    std::vector<topology_obj_info> gpus_;
+    std::vector<topology_obj_info> pus_;
+    std::vector<topology_obj_info> cores_;
+    std::size_t num_numas_;
+
+    template <typename T>
+    using topo_manager = std::unique_ptr<T, std::function<void(T *)>>;
+    topo_manager<hwloc_topology> topo_;
+};
+
+
+}  // namespace machine_config
+}  // namespace gko
+
+
+#endif  // GKO_CORE_BASE_MACHINE_CONFIG_HPP_

From a866d8a0a9eee05a22362f3ee56b8c2ca2fb36c3 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 4 Jun 2020 15:24:42 +0200
Subject: [PATCH 0271/1008] Add a test and make the linking to HWLOC PUBLIC

---
 cmake/get_info.cmake                        |  2 +-
 core/CMakeLists.txt                         |  4 +-
 core/device_hooks/CMakeLists.txt            | 17 ++++++
 core/test/base/executor.cpp                 | 21 ++++++++
 cuda/base/executor.cpp                      | 48 ++++++++++++++++-
 hip/CMakeLists.txt                          |  5 ++
 include/ginkgo/core/base/executor.hpp       | 59 +++++++++++++++++++--
 include/ginkgo/core/base/machine_config.hpp | 45 ++++++++--------
 include/ginkgo/ginkgo.hpp                   |  1 +
 9 files changed, 172 insertions(+), 30 deletions(-)

diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index dcaf07f334b..bbca15acbbd 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -127,7 +127,7 @@ foreach(log_type ${log_types})
     ginkgo_print_module_footer(${${log_type}} "User configuration:")
     ginkgo_print_module_footer(${${log_type}} "  Enabled modules:")
     ginkgo_print_foreach_variable(
-        "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP")
+        "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP;GINKGO_WITH_HWLOC")
     ginkgo_print_module_footer(${${log_type}} "  Tests, benchmarks and examples:")
     ginkgo_print_foreach_variable(
         "GINKGO_BUILD_TESTS;GINKGO_BUILD_EXAMPLES;GINKGO_EXTLIB_EXAMPLE;GINKGO_BUILD_BENCHMARKS")
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 07b70d32c78..0f6db2ab032 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -67,8 +67,8 @@ if (GINKGO_HAVE_PAPI_SDE)
     target_link_libraries(ginkgo PUBLIC PAPI::PAPI)
 endif()
 if (GINKGO_HAVE_HWLOC)
-  target_include_directories(ginkgo PRIVATE ${HWLOC_INCLUDE_DIRS})
-  target_link_libraries(ginkgo PRIVATE ${HWLOC_LIBRARIES})
+  target_include_directories(ginkgo PUBLIC ${HWLOC_INCLUDE_DIRS})
+  target_link_libraries(ginkgo PUBLIC ${HWLOC_LIBRARIES})
 endif()
 ginkgo_default_includes(ginkgo)
 ginkgo_install_library(ginkgo core)
diff --git a/core/device_hooks/CMakeLists.txt b/core/device_hooks/CMakeLists.txt
index 5b8f8bffb22..f247452f7d4 100644
--- a/core/device_hooks/CMakeLists.txt
+++ b/core/device_hooks/CMakeLists.txt
@@ -45,3 +45,20 @@ if (NOT GINKGO_BUILD_REFERENCE)
     ginkgo_default_includes(ginkgo_reference)
     ginkgo_install_library(ginkgo_reference reference)
 endif()
+<<<<<<< HEAD
+=======
+
+if(NOT GINKGO_BUILD_HIP)
+    add_library(ginkgo_hip
+        $<TARGET_OBJECTS:ginkgo_hip_device>
+        hip_hooks.cpp)
+    ginkgo_compile_features(ginkgo_hip)
+    ginkgo_default_includes(ginkgo_hip)
+    ginkgo_install_library(ginkgo_hip hip)
+
+    if (GINKGO_HAVE_HWLOC)
+        target_include_directories(ginkgo_hip PUBLIC ${HWLOC_INCLUDE_DIRS})
+        target_link_libraries(ginkgo_hip PUBLIC ${HWLOC_LIBRARIES})
+    endif()
+endif()
+>>>>>>> Add a test and make the linking to HWLOC PUBLIC
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 5ff74d2b68e..079e353dcb0 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -101,6 +101,27 @@ TEST(OmpExecutor, RunsCorrectLambdaOperation)
     ASSERT_EQ(1, value);
 }
 
+#if GKO_HAVE_HWLOC
+
+
+TEST(OmpExecutor, GetsExecInfo)
+{
+    int num_pus = 0;
+    int num_cores = 0;
+    int num_numas = 0;
+    auto omp = gko::OmpExecutor::create();
+    auto omp_info = omp->get_exec_info();
+    num_pus = omp_info->get_num_pus();
+    num_cores = omp_info->get_num_cores();
+    num_numas = omp_info->get_num_numas();
+    EXPECT_GT(num_numas, 0);
+    EXPECT_GT(num_pus, 0);
+    EXPECT_GT(num_cores, 0);
+}
+
+
+#endif
+
 
 TEST(OmpExecutor, AllocatesAndFreesMemory)
 {
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index fb266e36069..f2c40466c8b 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -39,6 +39,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <cuda_runtime.h>
 
 
+#if GKO_HAVE_HWLOC
+#include "hwloc/cuda.h"
+#endif
+
+
 #include <ginkgo/config.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 
@@ -48,13 +53,54 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "cuda/base/cusparse_handle.hpp"
 #include "cuda/base/device_guard.hpp"
 
-
 namespace gko {
 
 
 #include "common/base/executor.hpp.inc"
 
 
+namespace machine_config {
+
+
+template <>
+void Topology<CudaExecutor>::load_gpus()
+{
+#if GKO_HAVE_HWLOC
+    std::size_t num_in_numa = 0;
+    int last_numa = 0;
+    auto topology = this->topo_.get();
+    int ngpus = 0;
+    GKO_ASSERT_NO_CUDA_ERRORS(cudaGetDeviceCount(&ngpus));
+    auto n_objs = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_OS_DEVICE);
+    for (std::size_t i = 1; i < n_objs; i++, num_in_numa++) {
+        hwloc_obj_t obj = NULL;
+        while ((obj = hwloc_get_next_osdev(topology, obj)) != NULL) {
+            if (HWLOC_OBJ_OSDEV_COPROC == obj->attr->osdev.type && obj->name &&
+                !strncmp("cuda", obj->name, 4) &&
+                atoi(obj->name + 4) == (int)i) {
+                while (obj &&
+                       (!obj->nodeset || hwloc_bitmap_iszero(obj->nodeset)))
+                    obj = obj->parent;
+                if (obj && obj->nodeset) {
+                    auto this_numa = hwloc_bitmap_first(obj->nodeset);
+                    if (this_numa != last_numa) {
+                        num_in_numa = 0;
+                    }
+                    this->gpus_.push_back(
+                        topology_obj_info{obj, this_numa, i, num_in_numa});
+                    last_numa = this_numa;
+                }
+            }
+        }
+    }
+
+#endif
+}
+
+
+}  // namespace machine_config
+
+
 std::shared_ptr<CudaExecutor> CudaExecutor::create(
     int device_id, std::shared_ptr<Executor> master, bool device_reset)
 {
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 04df7e06506..25e51e0bc2b 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -316,6 +316,11 @@ if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo_hip)
 endif()
 
+if (GINKGO_HAVE_HWLOC)
+    target_include_directories(ginkgo_hip PUBLIC ${HWLOC_INCLUDE_DIRS})
+    target_link_libraries(ginkgo_hip PUBLIC ${HWLOC_LIBRARIES})
+endif()
+
 if(GINKGO_BUILD_TESTS)
   # Here, we go through all of Ginkgo's dependencies to build a `-Wl,-rpath` string since for
   # some reason `hipcc` through CMake does not have rpath settings unlike the other compilers.
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 34f96da8ae6..5ea6cde6b42 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -956,10 +956,13 @@ class EnableDeviceReset {
  * @ingroup Executor
  */
 class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
-                    public std::enable_shared_from_this<OmpExecutor> {
+                    public std::enable_shared_from_this<OmpExecutor>,
+                    public machine_config::Topology<OmpExecutor> {
     friend class detail::ExecutorBase<OmpExecutor>;
 
 public:
+    using omp_exec_info = machine_config::Topology<OmpExecutor>;
+
     /**
      * Creates a new OmpExecutor.
      */
@@ -974,8 +977,15 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
 
     void synchronize() const override;
 
+    /**
+     * Get the Executor information for this executor
+     *
+     * @return the executor info (omp_exec_info*) for this executor
+     */
+    omp_exec_info *get_exec_info() const { return exec_info_.get(); }
+
 protected:
-    OmpExecutor() = default;
+    OmpExecutor() : exec_info_(omp_exec_info::create()) {}
 
     void *raw_alloc(size_type size) const override;
 
@@ -992,6 +1002,9 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
     GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(CudaExecutor, false);
 
     bool verify_memory_to(const DpcppExecutor *dest_exec) const override;
+
+private:
+    std::unique_ptr<omp_exec_info> exec_info_;
 };
 
 
@@ -1011,6 +1024,8 @@ using DefaultExecutor = OmpExecutor;
  */
 class ReferenceExecutor : public OmpExecutor {
 public:
+    using ref_exec_info = machine_config::Topology<OmpExecutor>;
+
     static std::shared_ptr<ReferenceExecutor> create()
     {
         return std::shared_ptr<ReferenceExecutor>(new ReferenceExecutor());
@@ -1024,8 +1039,15 @@ class ReferenceExecutor : public OmpExecutor {
         this->template log<log::Logger::operation_completed>(this, &op);
     }
 
+    /**
+     * Get the Executor information for this executor
+     *
+     * @return the executor info (ref_exec_info*) for this executor
+     */
+    ref_exec_info *get_exec_info() const { return exec_info_.get(); }
+
 protected:
-    ReferenceExecutor() = default;
+    ReferenceExecutor() : exec_info_(ref_exec_info::create()) {}
 
     bool verify_memory_from(const Executor *src_exec) const override
     {
@@ -1041,6 +1063,9 @@ class ReferenceExecutor : public OmpExecutor {
     GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(CudaExecutor, false);
 
     GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(HipExecutor, false);
+
+private:
+    std::unique_ptr<ref_exec_info> exec_info_;
 };
 
 
@@ -1059,10 +1084,13 @@ using DefaultExecutor = ReferenceExecutor;
  */
 class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
                      public std::enable_shared_from_this<CudaExecutor>,
-                     public detail::EnableDeviceReset {
+                     public detail::EnableDeviceReset,
+                     public machine_config::Topology<CudaExecutor> {
     friend class detail::ExecutorBase<CudaExecutor>;
 
 public:
+    using cuda_exec_info = machine_config::Topology<CudaExecutor>;
+
     /**
      * Creates a new CudaExecutor.
      *
@@ -1144,6 +1172,13 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         return cusparse_handle_.get();
     }
 
+    /**
+     * Get the Executor information for this executor
+     *
+     * @return the executor info (cuda_exec_info*) for this executor
+     */
+    cuda_exec_info *get_exec_info() const { return exec_info_.get(); }
+
 protected:
     void set_gpu_property();
 
@@ -1164,6 +1199,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         this->set_gpu_property();
         this->init_handles();
         increase_num_execs(device_id);
+        exec_info_ = cuda_exec_info::create();
     }
 
     void *raw_alloc(size_type size) const override;
@@ -1217,6 +1253,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     static constexpr int max_devices = 64;
     static unsigned num_execs[max_devices];
     static std::mutex mutex[max_devices];
+    std::unique_ptr<cuda_exec_info> exec_info_;
 };
 
 
@@ -1235,10 +1272,13 @@ using DefaultExecutor = CudaExecutor;
  */
 class HipExecutor : public detail::ExecutorBase<HipExecutor>,
                     public std::enable_shared_from_this<HipExecutor>,
-                    public detail::EnableDeviceReset {
+                    public detail::EnableDeviceReset,
+                    public machine_config::Topology<HipExecutor> {
     friend class detail::ExecutorBase<HipExecutor>;
 
 public:
+    using hip_exec_info = machine_config::Topology<HipExecutor>;
+
     /**
      * Creates a new HipExecutor.
      *
@@ -1320,6 +1360,13 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         return hipsparse_handle_.get();
     }
 
+    /**
+     * Get the Executor information for this executor
+     *
+     * @return the executor info (hip_exec_info*) for this executor
+     */
+    hip_exec_info *get_exec_info() const { return exec_info_.get(); }
+
 protected:
     void set_gpu_property();
 
@@ -1340,6 +1387,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         this->set_gpu_property();
         this->init_handles();
         increase_num_execs(device_id);
+        exec_info_ = hip_exec_info::create();
     }
 
     void *raw_alloc(size_type size) const override;
@@ -1393,6 +1441,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     static constexpr int max_devices = 64;
     static int num_execs[max_devices];
     static std::mutex mutex[max_devices];
+    std::unique_ptr<hip_exec_info> exec_info_;
 };
 
 
diff --git a/include/ginkgo/core/base/machine_config.hpp b/include/ginkgo/core/base/machine_config.hpp
index d91a4925919..69dfdfe0f36 100644
--- a/include/ginkgo/core/base/machine_config.hpp
+++ b/include/ginkgo/core/base/machine_config.hpp
@@ -103,6 +103,7 @@ class Topology : public machine_info, public binder {
         return std::unique_ptr<Topology>(new Topology());
     }
 
+    hwloc_topology *get_topology() const { return this->topo_.get(); }
 
     void bind_to_core(std::size_t id) override
     {
@@ -152,6 +153,29 @@ class Topology : public machine_info, public binder {
 #endif
     }
 
+#if GKO_HAVE_HWLOC
+
+    static void hwloc_print_children(hwloc_topology *topology, hwloc_obj_t obj,
+                                     int depth)
+    {
+        char type[32], attr[1024];
+        unsigned i;
+        hwloc_obj_type_snprintf(type, sizeof(type), obj, 0);
+        std::cout << std::string(2 * depth, ' ') << type;
+        if (obj->os_index != (unsigned)-1) {
+            std::cout << "#" << obj->os_index;
+        }
+        hwloc_obj_attr_snprintf(attr, sizeof(attr), obj, " ", 0);
+        if (*attr) {
+            std::cout << "(" << attr << ")";
+        }
+        std::cout << std::endl;
+        for (i = 0; i < obj->arity; i++) {
+            hwloc_print_children(topology, obj->children[i], depth + 1);
+        }
+    }
+
+#endif
 
 protected:
     Topology()
@@ -211,27 +235,6 @@ class Topology : public machine_info, public binder {
     }
 
 
-    static void hwloc_print_children(hwloc_topology *topology, hwloc_obj_t obj,
-                                     int depth)
-    {
-        char type[32], attr[1024];
-        unsigned i;
-        hwloc_obj_type_snprintf(type, sizeof(type), obj, 0);
-        std::cout << std::string(2 * depth, ' ') << type;
-        if (obj->os_index != (unsigned)-1) {
-            std::cout << "#" << obj->os_index;
-        }
-        hwloc_obj_attr_snprintf(attr, sizeof(attr), obj, " ", 0);
-        if (*attr) {
-            std::cout << "(" << attr << ")";
-        }
-        std::cout << std::endl;
-        for (i = 0; i < obj->arity; i++) {
-            hwloc_print_children(topology, obj->children[i], depth + 1);
-        }
-    }
-
-
 #endif
 
 private:
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 49686250cc6..f2c8c27bb6d 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/machine_config.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/matrix_assembly_data.hpp>
 #include <ginkgo/core/base/matrix_data.hpp>

From 9691640b68deca39d1b6fc952df3c8340c70fbe7 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Sat, 17 Oct 2020 22:36:07 +0200
Subject: [PATCH 0272/1008] Update FindHWLOC and require min version 2.1

---
 CMakeLists.txt                              |   6 +-
 cmake/Modules/FindHWLOC.cmake               | 430 ++++++--------------
 cmake/autodetect_library_dependencies.cmake |   2 +-
 3 files changed, 124 insertions(+), 314 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 64a241a9d66..38e5b38b511 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -149,12 +149,10 @@ endif()
 # Find HWLOC if needed
 set(GINKGO_HAVE_HWLOC 0)
 if(GINKGO_WITH_HWLOC)
-  find_package(HWLOC REQUIRED)
+  find_package(HWLOC 2.1 REQUIRED)
   set(GKO_HWLOC_XMLFILE "${HWLOC_XMLFILE}")
   message(STATUS "HWLOC XML file set to: ${GKO_HWLOC_XMLFILE}")
-  if(HWLOC_FOUND)
-    set(GINKGO_HAVE_HWLOC 1)
-  endif()
+  set(GINKGO_HAVE_HWLOC 1)
 endif()
 
 set(GINKGO_HIP_PLATFORM_NVCC 0)
diff --git a/cmake/Modules/FindHWLOC.cmake b/cmake/Modules/FindHWLOC.cmake
index a831b5c725a..a36a47518ec 100644
--- a/cmake/Modules/FindHWLOC.cmake
+++ b/cmake/Modules/FindHWLOC.cmake
@@ -1,331 +1,143 @@
-###
 #
-# @copyright (c) 2009-2014 The University of Tennessee and The University
-#                          of Tennessee Research Foundation.
-#                          All rights reserved.
-# @copyright (c) 2012-2014 Inria. All rights reserved.
-# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+# This file is a modified version of the file in the GROMACS molecular simulation
+# package.
 #
-###
+# Copyright (c) 2015,2016,2017,2018, by the GROMACS development team, led by
+# Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
+# and including many others, as listed in the AUTHORS file in the
+# top-level source directory and at http://www.gromacs.org.
 #
-# - Find HWLOC include dirs and libraries
-# Use this module by invoking find_package with the form:
-#  find_package(HWLOC
-#               [REQUIRED]) # Fail with error if hwloc is not found
+# GROMACS is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public License
+# as published by the Free Software Foundation; either version 2.1
+# of the License, or (at your option) any later version.
 #
-# This module finds headers and hwloc library.
-# Results are reported in variables:
-#  HWLOC_FOUND           - True if headers and requested libraries were found
-#  HWLOC_INCLUDE_DIRS    - hwloc include directories
-#  HWLOC_LIBRARY_DIRS    - Link directories for hwloc libraries
-#  HWLOC_LIBRARIES       - hwloc component libraries to be linked
+# GROMACS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
 #
-# The user can give specific paths where to find the libraries adding cmake
-# options at configure (ex: cmake path/to/project -DHWLOC_DIR=path/to/hwloc):
-#  HWLOC_DIR             - Where to find the base directory of hwloc
-#  HWLOC_INCDIR          - Where to find the header files
-#  HWLOC_LIBDIR          - Where to find the library files
-# The module can also look for the following environment variables if paths
-# are not given as cmake variable: HWLOC_DIR, HWLOC_INCDIR, HWLOC_LIBDIR
-
-#=============================================================================
-# Copyright 2012-2013 Inria
-# Copyright 2012-2013 Emmanuel Agullo
-# Copyright 2012-2013 Mathieu Faverge
-# Copyright 2012      Cedric Castagnede
-# Copyright 2013      Florent Pruvost
+# You should have received a copy of the GNU Lesser General Public
+# License along with GROMACS; if not, see
+# http://www.gnu.org/licenses, or write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
 #
-# Distributed under the OSI-approved BSD License (the "License");
-# see accompanying file MORSE-Copyright.txt for details.
+# If you want to redistribute modifications to GROMACS, please
+# consider that scientific software is very special. Version
+# control is crucial - bugs must be traceable. We will be happy to
+# consider code for inclusion in the official distribution, but
+# derived work must not be called official GROMACS. Details are found
+# in the README & COPYING files - if they are missing, get the
+# official version at http://www.gromacs.org.
 #
-# This software is distributed WITHOUT ANY WARRANTY; without even the
-# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the License for more information.
-#=============================================================================
-# (To distribute this file outside of Morse, substitute the full
-#  License text for the above reference.)
-
-include(CheckStructHasMember)
-include(CheckCSourceCompiles)
-
-if (NOT HWLOC_FOUND)
-  set(HWLOC_DIR "" CACHE PATH "Installation directory of HWLOC library")
-  if (NOT HWLOC_FIND_QUIETLY)
-    message(STATUS "A cache variable, namely HWLOC_DIR, has been set to specify the install directory of HWLOC")
-  endif()
-endif()
-
-set(ENV_HWLOC_DIR "$ENV{HWLOC_DIR}")
-set(ENV_HWLOC_INCDIR "$ENV{HWLOC_INCDIR}")
-set(ENV_HWLOC_LIBDIR "$ENV{HWLOC_LIBDIR}")
-set(HWLOC_GIVEN_BY_USER "FALSE")
-if ( HWLOC_DIR OR ( HWLOC_INCDIR AND HWLOC_LIBDIR) OR ENV_HWLOC_DIR OR (ENV_HWLOC_INCDIR AND ENV_HWLOC_LIBDIR) )
-  set(HWLOC_GIVEN_BY_USER "TRUE")
-endif()
-
-# Optionally use pkg-config to detect include/library dirs (if pkg-config is available)
-# -------------------------------------------------------------------------------------
-include(FindPkgConfig)
-find_package(PkgConfig QUIET)
-if( PKG_CONFIG_EXECUTABLE AND NOT HWLOC_GIVEN_BY_USER )
-
-  pkg_search_module(HWLOC hwloc)
-  if (NOT HWLOC_FIND_QUIETLY)
-    if (HWLOC_FOUND AND HWLOC_LIBRARIES)
-      message(STATUS "Looking for HWLOC - found using PkgConfig")
-      #if(NOT HWLOC_INCLUDE_DIRS)
-      #    message("${Magenta}HWLOC_INCLUDE_DIRS is empty using PkgConfig."
-      #        "Perhaps the path to hwloc headers is already present in your"
-      #        "C(PLUS)_INCLUDE_PATH environment variable.${ColourReset}")
-      #endif()
-    else()
-      message(STATUS "${Magenta}Looking for HWLOC - not found using PkgConfig."
-	"\n   Perhaps you should add the directory containing hwloc.pc to"
-	"\n   the PKG_CONFIG_PATH environment variable.${ColourReset}")
-    endif()
-  endif()
-
-endif( PKG_CONFIG_EXECUTABLE AND NOT HWLOC_GIVEN_BY_USER )
-
-if( (NOT PKG_CONFIG_EXECUTABLE) OR (PKG_CONFIG_EXECUTABLE AND NOT HWLOC_FOUND) OR (HWLOC_GIVEN_BY_USER) )
-
-  if (NOT HWLOC_FIND_QUIETLY)
-    message(STATUS "Looking for HWLOC - PkgConfig not used")
-  endif()
-
-  # Looking for include
-  # -------------------
-
-  # Add system include paths to search include
-  # ------------------------------------------
-  unset(_inc_env)
-  if(ENV_HWLOC_INCDIR)
-    list(APPEND _inc_env "${ENV_HWLOC_INCDIR}")
-  elseif(ENV_HWLOC_DIR)
-    list(APPEND _inc_env "${ENV_HWLOC_DIR}")
-    list(APPEND _inc_env "${ENV_HWLOC_DIR}/include")
-    list(APPEND _inc_env "${ENV_HWLOC_DIR}/include/hwloc")
-  else()
-    if(WIN32)
-      string(REPLACE ":" ";" _inc_env "$ENV{INCLUDE}")
-    else()
-      string(REPLACE ":" ";" _path_env "$ENV{INCLUDE}")
-      list(APPEND _inc_env "${_path_env}")
-      string(REPLACE ":" ";" _path_env "$ENV{C_INCLUDE_PATH}")
-      list(APPEND _inc_env "${_path_env}")
-      string(REPLACE ":" ";" _path_env "$ENV{CPATH}")
-      list(APPEND _inc_env "${_path_env}")
-      string(REPLACE ":" ";" _path_env "$ENV{INCLUDE_PATH}")
-      list(APPEND _inc_env "${_path_env}")
-    endif()
-  endif()
-  list(APPEND _inc_env "${CMAKE_PLATFORM_IMPLICIT_INCLUDE_DIRECTORIES}")
-  list(APPEND _inc_env "${CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES}")
-  list(REMOVE_DUPLICATES _inc_env)
-
-  # set paths where to look for
-  set(PATH_TO_LOOK_FOR "${_inc_env}")
-
-  # Try to find the hwloc header in the given paths
-  # -------------------------------------------------
-  # call cmake macro to find the header path
-  if(HWLOC_INCDIR)
-    set(HWLOC_hwloc.h_DIRS "HWLOC_hwloc.h_DIRS-NOTFOUND")
-    find_path(HWLOC_hwloc.h_DIRS
-      NAMES hwloc.h
-      HINTS ${HWLOC_INCDIR})
-  else()
-    if(HWLOC_DIR)
-      set(HWLOC_hwloc.h_DIRS "HWLOC_hwloc.h_DIRS-NOTFOUND")
-      find_path(HWLOC_hwloc.h_DIRS
-	NAMES hwloc.h
-	HINTS ${HWLOC_DIR}
-	PATH_SUFFIXES "include" "include/hwloc")
-    else()
-      set(HWLOC_hwloc.h_DIRS "HWLOC_hwloc.h_DIRS-NOTFOUND")
-      find_path(HWLOC_hwloc.h_DIRS
-	NAMES hwloc.h
-	HINTS ${PATH_TO_LOOK_FOR}
-	PATH_SUFFIXES "hwloc")
-    endif()
-  endif()
-  mark_as_advanced(HWLOC_hwloc.h_DIRS)
-
-  # Add path to cmake variable
-  # ------------------------------------
-  if (HWLOC_hwloc.h_DIRS)
-    set(HWLOC_INCLUDE_DIRS "${HWLOC_hwloc.h_DIRS}")
-  else ()
-    set(HWLOC_INCLUDE_DIRS "HWLOC_INCLUDE_DIRS-NOTFOUND")
-    if(NOT HWLOC_FIND_QUIETLY)
-      message(STATUS "Looking for hwloc -- hwloc.h not found")
-    endif()
-  endif ()
+# To help us fund GROMACS development, we humbly ask that you cite
+# the research papers on the package. Check out http://www.gromacs.org.
 
-  if (HWLOC_INCLUDE_DIRS)
-    list(REMOVE_DUPLICATES HWLOC_INCLUDE_DIRS)
-  endif ()
-
-
-  # Looking for lib
-  # ---------------
+# FindHwloc
+#
+# - Locate headers and libraries for the Portable Hardware Locality (hwloc)
+#   library.
+#
+# Usage: find_package(HWLOC VERSION)
+#        find_package(HWLOC VERSION REQUIRED)
+#
+# FindHwloc defines the following variables:
+#
+# HWLOC_FOUND          - True if both headers and libraries were located
+# HWLOC_INCLUDE_DIRS   - Where to find hwloc headers
+# HWLOC_LIBRARIES      - Libraries to link with to enable hwloc usage
+# HWLOC_VERSION        - Version (string) of the hwloc library
+#
 
-  # Add system library paths to search lib
-  # --------------------------------------
-  unset(_lib_env)
-  if(ENV_HWLOC_LIBDIR)
-    list(APPEND _lib_env "${ENV_HWLOC_LIBDIR}")
-  elseif(ENV_HWLOC_DIR)
-    list(APPEND _lib_env "${ENV_HWLOC_DIR}")
-    list(APPEND _lib_env "${ENV_HWLOC_DIR}/lib")
-  else()
-    if(WIN32)
-      string(REPLACE ":" ";" _lib_env "$ENV{LIB}")
+# Function for converting hex version numbers from HWLOC_API_VERSION if necessary
+function(HEX2DEC str res)
+    string(LENGTH "${str}" len)
+    if("${len}" EQUAL 1)
+        if("${str}" MATCHES "[0-9]")
+            set(${res} "${str}" PARENT_SCOPE)
+        elseif( "${str}" MATCHES "[aA]")
+            set(${res} 10 PARENT_SCOPE)
+        elseif( "${str}" MATCHES "[bB]")
+            set(${res} 11 PARENT_SCOPE)
+        elseif( "${str}" MATCHES "[cC]")
+            set(${res} 12 PARENT_SCOPE)
+        elseif( "${str}" MATCHES "[dD]")
+            set(${res} 13 PARENT_SCOPE)
+        elseif( "${str}" MATCHES "[eE]")
+            set(${res} 14 PARENT_SCOPE)
+        elseif( "${str}" MATCHES "[fF]")
+            set(${res} 15 PARENT_SCOPE)
+        else()
+            return()
+        endif()
     else()
-      if(APPLE)
-	string(REPLACE ":" ";" _lib_env "$ENV{DYLD_LIBRARY_PATH}")
-      else()
-	string(REPLACE ":" ";" _lib_env "$ENV{LD_LIBRARY_PATH}")
-      endif()
-      list(APPEND _lib_env "${CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES}")
-      list(APPEND _lib_env "${CMAKE_C_IMPLICIT_LINK_DIRECTORIES}")
+        string(SUBSTRING "${str}" 0 1 str1)
+        string(SUBSTRING "${str}" 1 -1 str2)
+        hex2dec(${str1} res1)
+        hex2dec(${str2} res2)
+        math(EXPR val "16 * ${res1} + ${res2}")
+        set(${res} "${val}" PARENT_SCOPE)
     endif()
-  endif()
-  list(REMOVE_DUPLICATES _lib_env)
-
-  # set paths where to look for
-  set(PATH_TO_LOOK_FOR "${_lib_env}")
-
-  # Try to find the hwloc lib in the given paths
-  # ----------------------------------------------
-
-  # call cmake macro to find the lib path
-  if(HWLOC_LIBDIR)
-    set(HWLOC_hwloc_LIBRARY "HWLOC_hwloc_LIBRARY-NOTFOUND")
-    find_library(HWLOC_hwloc_LIBRARY
-      NAMES hwloc
-      HINTS ${HWLOC_LIBDIR})
-  else()
-    if(HWLOC_DIR)
-      set(HWLOC_hwloc_LIBRARY "HWLOC_hwloc_LIBRARY-NOTFOUND")
-      find_library(HWLOC_hwloc_LIBRARY
-	NAMES hwloc
-	HINTS ${HWLOC_DIR}
-	PATH_SUFFIXES lib lib32 lib64)
-    else()
-      set(HWLOC_hwloc_LIBRARY "HWLOC_hwloc_LIBRARY-NOTFOUND")
-      find_library(HWLOC_hwloc_LIBRARY
-	NAMES hwloc
-	HINTS ${PATH_TO_LOOK_FOR})
+endfunction()
+
+find_path(HWLOC_INCLUDE_DIRS "hwloc.h")
+find_library(HWLOC_LIBRARIES "hwloc")
+
+if(HWLOC_INCLUDE_DIRS)
+    # If we are not cross-compiling we try to use the hwloc-info program
+    if(NOT CMAKE_CROSSCOMPILING)
+        find_program(HWLOC_INFO "hwloc-info")
+        mark_as_advanced(HWLOC_INFO)
+
+        if(HWLOC_INFO)
+            execute_process(COMMAND ${HWLOC_INFO} "--version"
+                            RESULT_VARIABLE HWLOC_INFO_RES
+                            OUTPUT_VARIABLE HWLOC_INFO_OUT
+                            ERROR_VARIABLE  HWLOC_INFO_ERR)
+
+            if(HWLOC_INFO_ERR)
+	        message(STATUS "Error executing hwloc-info: ${HWLOC_INFO_ERR}")
+            endif()
+            string(REGEX MATCH "[0-9]+.*[0-9]+" HWLOC_INFO_OUT "${HWLOC_INFO_OUT}")
+            set(HWLOC_VERSION ${HWLOC_INFO_OUT} CACHE STRING "Hwloc library version")
+        endif()
     endif()
-  endif()
-  mark_as_advanced(HWLOC_hwloc_LIBRARY)
 
-  # If found, add path to cmake variable
-  # ------------------------------------
-  if (HWLOC_hwloc_LIBRARY)
-    get_filename_component(hwloc_lib_path ${HWLOC_hwloc_LIBRARY} PATH)
-    # set cmake variables (respects naming convention)
-    set(HWLOC_LIBRARIES    "${HWLOC_hwloc_LIBRARY}")
-    set(HWLOC_LIBRARY_DIRS "${hwloc_lib_path}")
-  else ()
-    set(HWLOC_LIBRARIES    "HWLOC_LIBRARIES-NOTFOUND")
-    set(HWLOC_LIBRARY_DIRS "HWLOC_LIBRARY_DIRS-NOTFOUND")
-    if(NOT HWLOC_FIND_QUIETLY)
-      message(STATUS "Looking for hwloc -- lib hwloc not found")
+    if (NOT Hwloc_FIND_QUIETLY)
+        message(STATUS "hwloc version: ${HWLOC_VERSION}")
     endif()
-  endif ()
-
-  if (HWLOC_LIBRARY_DIRS)
-    list(REMOVE_DUPLICATES HWLOC_LIBRARY_DIRS)
-  endif ()
 
-  # check a function to validate the find
-  if(HWLOC_LIBRARIES)
-
-    set(REQUIRED_INCDIRS)
-    set(REQUIRED_LIBDIRS)
-    set(REQUIRED_LIBS)
-
-    # HWLOC
-    if (HWLOC_INCLUDE_DIRS)
-      set(REQUIRED_INCDIRS "${HWLOC_INCLUDE_DIRS}")
-    endif()
-    if (HWLOC_LIBRARY_DIRS)
-      set(REQUIRED_LIBDIRS "${HWLOC_LIBRARY_DIRS}")
+    # Parse header if cross-compiling, or if hwloc-info was not found
+    if(NOT HWLOC_VERSION)
+        # Hwloc is never installed as a framework on OS X, so this should always work.
+        file(READ "${HWLOC_INCLUDE_DIRS}/hwloc.h"
+             HEADER_CONTENTS LIMIT 16384)
+        string(REGEX REPLACE ".*#define HWLOC_API_VERSION (0[xX][0-9a-fA-F]+).*" "\\1"
+               HWLOC_API_VERSION "${HEADER_CONTENTS}")
+        string(SUBSTRING "${HWLOC_API_VERSION}" 4 2 HEX_MAJOR)
+        string(SUBSTRING "${HWLOC_API_VERSION}" 6 2 HEX_MINOR)
+        string(SUBSTRING "${HWLOC_API_VERSION}" 8 2 HEX_PATCH)
+        hex2dec(${HEX_MAJOR} DEC_MAJOR)
+        hex2dec(${HEX_MINOR} DEC_MINOR)
+        hex2dec(${HEX_PATCH} DEC_PATCH)
+        set(HWLOC_VERSION "${DEC_MAJOR}.${DEC_MINOR}.${DEC_PATCH}" CACHE STRING "Hwloc library version")
     endif()
-    set(REQUIRED_LIBS "${HWLOC_LIBRARIES}")
-
-    # set required libraries for link
-    set(CMAKE_REQUIRED_INCLUDES "${REQUIRED_INCDIRS}")
-    set(CMAKE_REQUIRED_LIBRARIES)
-    foreach(lib_dir ${REQUIRED_LIBDIRS})
-      list(APPEND CMAKE_REQUIRED_LIBRARIES "-L${lib_dir}")
-    endforeach()
-    list(APPEND CMAKE_REQUIRED_LIBRARIES "${REQUIRED_LIBS}")
-    string(REGEX REPLACE "^ -" "-" CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}")
-
-    # test link
-    unset(HWLOC_WORKS CACHE)
-    include(CheckFunctionExists)
-    check_function_exists(hwloc_topology_init HWLOC_WORKS)
-    mark_as_advanced(HWLOC_WORKS)
-
-    if(NOT HWLOC_WORKS)
-      if(NOT HWLOC_FIND_QUIETLY)
-	message(STATUS "Looking for hwloc : test of hwloc_topology_init with hwloc library fails")
-	message(STATUS "CMAKE_REQUIRED_LIBRARIES: ${CMAKE_REQUIRED_LIBRARIES}")
-	message(STATUS "CMAKE_REQUIRED_INCLUDES: ${CMAKE_REQUIRED_INCLUDES}")
-	message(STATUS "Check in CMakeFiles/CMakeError.log to figure out why it fails")
-      endif()
-    endif()
-    set(CMAKE_REQUIRED_INCLUDES)
-    set(CMAKE_REQUIRED_FLAGS)
-    set(CMAKE_REQUIRED_LIBRARIES)
-  endif(HWLOC_LIBRARIES)
-
-endif( (NOT PKG_CONFIG_EXECUTABLE) OR (PKG_CONFIG_EXECUTABLE AND NOT HWLOC_FOUND) OR (HWLOC_GIVEN_BY_USER) )
-
-if (HWLOC_LIBRARIES)
-  if (HWLOC_LIBRARY_DIRS)
-    list(GET HWLOC_LIBRARY_DIRS 0 first_lib_path)
-  else()
-    list(GET HWLOC_LIBRARIES 0 first_lib)
-    get_filename_component(first_lib_path "${first_lib}" PATH)
-  endif()
-  if (${first_lib_path} MATCHES "/lib(32|64)?$")
-    string(REGEX REPLACE "/lib(32|64)?$" "" not_cached_dir "${first_lib_path}")
-    set(HWLOC_DIR_FOUND "${not_cached_dir}" CACHE PATH "Installation directory of HWLOC library" FORCE)
-  else()
-    set(HWLOC_DIR_FOUND "${first_lib_path}" CACHE PATH "Installation directory of HWLOC library" FORCE)
-  endif()
 endif()
-mark_as_advanced(HWLOC_DIR)
-mark_as_advanced(HWLOC_DIR_FOUND)
 
-# check that HWLOC has been found
-# -------------------------------
-include(FindPackageHandleStandardArgs)
-if (PKG_CONFIG_EXECUTABLE AND HWLOC_FOUND)
-  find_package_handle_standard_args(HWLOC DEFAULT_MSG
-    HWLOC_LIBRARIES)
+string(SUBSTRING "${HWLOC_VERSION}" 0 3 HWLOC_VERSION)
+if(HWLOC_VERSION LESS_EQUAL HWLOC_FIND_VERSION)
+    message(STATUS "Required version ${HWLOC_FIND_VERSION}, but found version ${HWLOC_VERSION}")
+    set(HWLOC_FOUND 0)
+    unset(HWLOC_LIBRARIES)
+    unset(HWLOC_INCLUDE_DIRS)
 else()
-  find_package_handle_standard_args(HWLOC DEFAULT_MSG
-    HWLOC_LIBRARIES
-    HWLOC_WORKS)
+    include(FindPackageHandleStandardArgs)
+    find_package_handle_standard_args(Hwloc
+        REQUIRED_VARS HWLOC_LIBRARIES HWLOC_INCLUDE_DIRS
+        VERSION_VAR HWLOC_VERSION)
+    mark_as_advanced(HWLOC_INCLUDE_DIRS HWLOC_LIBRARIES HWLOC_VERSION)
 endif()
 
-if (HWLOC_FOUND)
-  set(HWLOC_SAVE_CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES})
-  list(APPEND CMAKE_REQUIRED_INCLUDES ${HWLOC_INCLUDE_DIRS})
-
-  # test headers to guess the version
-  check_struct_has_member( "struct hwloc_obj" parent hwloc.h HAVE_HWLOC_PARENT_MEMBER )
-  check_struct_has_member( "struct hwloc_cache_attr_s" size hwloc.h HAVE_HWLOC_CACHE_ATTR )
-  check_c_source_compiles( "#include <hwloc.h>
-	    int main(void) { hwloc_obj_t o; o->type = HWLOC_OBJ_PU; return 0;}" HAVE_HWLOC_OBJ_PU)
-  include(CheckLibraryExists)
-  check_library_exists(${HWLOC_LIBRARIES} hwloc_bitmap_free "" HAVE_HWLOC_BITMAP)
-
-  set(CMAKE_REQUIRED_INCLUDES ${HWLOC_SAVE_CMAKE_REQUIRED_INCLUDES})
+if(HWLOC_FIND_REQUIRED AND NOT HWLOC_FOUND)
+    message(SEND_ERROR "HWLOC could not be found. A version mismatch could have occured. The version found was ${HWLOC_VERSION}")
 endif()
diff --git a/cmake/autodetect_library_dependencies.cmake b/cmake/autodetect_library_dependencies.cmake
index 0a108d14006..f2095599ea5 100644
--- a/cmake/autodetect_library_dependencies.cmake
+++ b/cmake/autodetect_library_dependencies.cmake
@@ -9,7 +9,7 @@ check_language(CUDA)
 try_compile(GKO_CAN_COMPILE_DPCPP ${PROJECT_BINARY_DIR}/dpcpp
     SOURCES ${PROJECT_SOURCE_DIR}/dpcpp/test_dpcpp.dp.cpp
     CXX_STANDARD 17)
-find_package(HWLOC)
+find_package(HWLOC 2.1)
 
 if(OpenMP_CXX_FOUND)
     if(NOT DEFINED GINKGO_BUILD_OMP)

From 71378f1a589e4cfdecba674f111fe6fdf6ebc712 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Sat, 17 Oct 2020 22:36:28 +0200
Subject: [PATCH 0273/1008] Add AMD GPU detection and tests for exec_info.

---
 cuda/base/executor.cpp                      | 18 ++----
 cuda/test/base/cuda_executor.cu             | 24 +++++++
 hip/base/executor.hip.cpp                   | 44 +++++++++++++
 hip/test/base/hip_executor.hip.cpp          | 25 ++++++++
 include/ginkgo/core/base/machine_config.hpp | 70 +++++++--------------
 5 files changed, 122 insertions(+), 59 deletions(-)

diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index f2c40466c8b..48094a16830 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -39,11 +39,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <cuda_runtime.h>
 
 
-#if GKO_HAVE_HWLOC
-#include "hwloc/cuda.h"
-#endif
-
-
 #include <ginkgo/config.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 
@@ -66,18 +61,17 @@ template <>
 void Topology<CudaExecutor>::load_gpus()
 {
 #if GKO_HAVE_HWLOC
-    std::size_t num_in_numa = 0;
+    size_type num_in_numa = 0;
     int last_numa = 0;
     auto topology = this->topo_.get();
-    int ngpus = 0;
-    GKO_ASSERT_NO_CUDA_ERRORS(cudaGetDeviceCount(&ngpus));
     auto n_objs = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_OS_DEVICE);
-    for (std::size_t i = 1; i < n_objs; i++, num_in_numa++) {
+    for (size_type i = 1; i < n_objs; i++, num_in_numa++) {
         hwloc_obj_t obj = NULL;
         while ((obj = hwloc_get_next_osdev(topology, obj)) != NULL) {
-            if (HWLOC_OBJ_OSDEV_COPROC == obj->attr->osdev.type && obj->name &&
-                !strncmp("cuda", obj->name, 4) &&
-                atoi(obj->name + 4) == (int)i) {
+            bool is_gpu = HWLOC_OBJ_OSDEV_COPROC == obj->attr->osdev.type &&
+                          obj->name && !strncmp("cuda", obj->name, 4) &&
+                          atoi(obj->name + 4) == (int)i;
+            if (is_gpu) {
                 while (obj &&
                        (!obj->nodeset || hwloc_bitmap_iszero(obj->nodeset)))
                     obj = obj->parent;
diff --git a/cuda/test/base/cuda_executor.cu b/cuda/test/base/cuda_executor.cu
index 99db90458ee..7c6bbe0ebfa 100644
--- a/cuda/test/base/cuda_executor.cu
+++ b/cuda/test/base/cuda_executor.cu
@@ -131,6 +131,30 @@ TEST_F(CudaExecutor, MasterKnowsNumberOfDevices)
 }
 
 
+#if GKO_HAVE_HWLOC
+
+
+TEST_F(CudaExecutor, GetsExecInfo)
+{
+    int num_pus = 0;
+    int num_gpus = 0;
+    int num_cores = 0;
+    int num_numas = 0;
+    auto cuda = gko::CudaExecutor::create(0, omp);
+    auto cuda_info = cuda->get_exec_info();
+    num_pus = cuda_info->get_num_pus();
+    num_gpus = cuda_info->get_num_gpus();
+    num_cores = cuda_info->get_num_cores();
+    num_numas = cuda_info->get_num_numas();
+    EXPECT_GT(num_numas, 0);
+    EXPECT_GT(num_gpus, 0);
+    EXPECT_GT(num_pus, 0);
+    EXPECT_GT(num_cores, 0);
+}
+
+
+#endif
+
 TEST_F(CudaExecutor, AllocatesAndFreesMemory)
 {
     int *ptr = nullptr;
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index b9f63a15337..865fdec4a6c 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -55,6 +55,50 @@ namespace gko {
 #include "common/base/executor.hpp.inc"
 
 
+namespace machine_config {
+
+
+template <>
+void Topology<HipExecutor>::load_gpus()
+{
+#if GKO_HAVE_HWLOC
+    size_type num_in_numa = 0;
+    int last_numa = 0;
+    auto topology = this->topo_.get();
+    auto n_objs = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_OS_DEVICE);
+    for (size_type i = 0; i < n_objs; i++, num_in_numa++) {
+        hwloc_obj_t obj = NULL;
+        while ((obj = hwloc_get_next_osdev(topology, obj)) != NULL) {
+            bool is_gpu = (HWLOC_OBJ_OSDEV_GPU == obj->attr->osdev.type ||
+                           HWLOC_OBJ_OSDEV_COPROC == obj->attr->osdev.type) &&
+                          obj->name &&
+                          (!strncmp("rsmi", obj->name, 4) ||
+                           !strncmp("cuda", obj->name, 4)) &&
+                          atoi(obj->name + 4) == (int)i;
+            if (is_gpu) {
+                while (obj &&
+                       (!obj->nodeset || hwloc_bitmap_iszero(obj->nodeset)))
+                    obj = obj->parent;
+                if (obj && obj->nodeset) {
+                    auto this_numa = hwloc_bitmap_first(obj->nodeset);
+                    if (this_numa != last_numa) {
+                        num_in_numa = 0;
+                    }
+                    this->gpus_.push_back(
+                        topology_obj_info{obj, this_numa, i, num_in_numa});
+                    last_numa = this_numa;
+                }
+            }
+        }
+    }
+
+#endif
+}
+
+
+}  // namespace machine_config
+
+
 std::shared_ptr<HipExecutor> HipExecutor::create(
     int device_id, std::shared_ptr<Executor> master, bool device_reset)
 {
diff --git a/hip/test/base/hip_executor.hip.cpp b/hip/test/base/hip_executor.hip.cpp
index c4f4757e397..5ca1b9915e0 100644
--- a/hip/test/base/hip_executor.hip.cpp
+++ b/hip/test/base/hip_executor.hip.cpp
@@ -136,6 +136,31 @@ TEST_F(HipExecutor, MasterKnowsNumberOfDevices)
 }
 
 
+#if GKO_HAVE_HWLOC
+
+
+TEST_F(HipExecutor, GetsExecInfo)
+{
+    int num_pus = 0;
+    int num_gpus = 0;
+    int num_cores = 0;
+    int num_numas = 0;
+    auto hip = gko::HipExecutor::create(0, omp);
+    auto hip_info = hip->get_exec_info();
+    num_pus = hip_info->get_num_pus();
+    num_gpus = hip_info->get_num_gpus();
+    num_cores = hip_info->get_num_cores();
+    num_numas = hip_info->get_num_numas();
+    EXPECT_GT(num_numas, 0);
+    EXPECT_GT(num_gpus, 0);
+    EXPECT_GT(num_pus, 0);
+    EXPECT_GT(num_cores, 0);
+}
+
+
+#endif
+
+
 TEST_F(HipExecutor, AllocatesAndFreesMemory)
 {
     int *ptr = nullptr;
diff --git a/include/ginkgo/core/base/machine_config.hpp b/include/ginkgo/core/base/machine_config.hpp
index 69dfdfe0f36..fb17e22dc4f 100644
--- a/include/ginkgo/core/base/machine_config.hpp
+++ b/include/ginkgo/core/base/machine_config.hpp
@@ -67,37 +67,17 @@ namespace gko {
 namespace machine_config {
 
 
-struct topology_obj_info {
-    hwloc_obj_t obj;
-    int numa;
-    std::size_t logical_id;
-    std::size_t physical_id;  // for GPUs, this is their number in the numa
-};
-
-
-struct machine_info {
-    virtual const topology_obj_info &get_pu(std::size_t id) = 0;
-    virtual const topology_obj_info &get_core(std::size_t id) = 0;
-    virtual const topology_obj_info &get_gpu(std::size_t id) = 0;
-    virtual std::size_t get_num_pus() = 0;
-    virtual std::size_t get_num_cores() = 0;
-    virtual std::size_t get_num_gpus() = 0;
-    virtual std::size_t get_num_numas() = 0;
-};
-
-
-struct binder {
-    virtual void bind_to_core(std::size_t id) = 0;
-    virtual void bind_to_pu(std::size_t id) = 0;
-};
-
-
 template <class Executor>
-class Topology : public machine_info, public binder {
-public:
-    Topology(Topology &) = delete;
-    Topology(Topology &&) = delete;
+class Topology {
+private:
+    struct topology_obj_info {
+        hwloc_obj_t obj;
+        int numa;
+        size_type logical_id;
+        size_type physical_id;  // for GPUs, this is their number in the numa
+    };
 
+public:
     static std::unique_ptr<Topology> create()
     {
         return std::unique_ptr<Topology>(new Topology());
@@ -105,44 +85,37 @@ class Topology : public machine_info, public binder {
 
     hwloc_topology *get_topology() const { return this->topo_.get(); }
 
-    void bind_to_core(std::size_t id) override
-    {
-        hwloc_binding_helper(this->cores_, id);
-    }
+    void bind_to_core(size_type id) { hwloc_binding_helper(this->cores_, id); }
 
-    void bind_to_pu(std::size_t id) override
-    {
-        hwloc_binding_helper(this->pus_, id);
-    }
+    void bind_to_pu(size_type id) { hwloc_binding_helper(this->pus_, id); }
 
-    const topology_obj_info &get_pu(std::size_t id) override
+    const topology_obj_info &get_pu(size_type id)
     {
         GKO_ENSURE_IN_BOUNDS(id, pus_.size());
         return pus_[id];
     }
 
-    const topology_obj_info &get_core(std::size_t id) override
+    const topology_obj_info &get_core(size_type id)
     {
         GKO_ENSURE_IN_BOUNDS(id, cores_.size());
         return cores_[id];
     }
 
-    const topology_obj_info &get_gpu(std::size_t id) override
+    const topology_obj_info &get_gpu(size_type id)
     {
         GKO_ENSURE_IN_BOUNDS(id, gpus_.size());
         return gpus_[id];
     }
 
 
-    std::size_t get_num_pus() override { return pus_.size(); }
-    std::size_t get_num_cores() override { return cores_.size(); }
-    std::size_t get_num_gpus() override { return gpus_.size(); }
-    std::size_t get_num_numas() override { return num_numas_; }
+    size_type get_num_pus() { return pus_.size(); }
+    size_type get_num_cores() { return cores_.size(); }
+    size_type get_num_gpus() { return gpus_.size(); }
+    size_type get_num_numas() { return num_numas_; }
 
     virtual void load_gpus() {}
 
-    void hwloc_binding_helper(std::vector<topology_obj_info> &obj,
-                              std::size_t id)
+    void hwloc_binding_helper(std::vector<topology_obj_info> &obj, size_type id)
     {
 #if GKO_HAVE_HWLOC
         auto bitmap = hwloc_bitmap_alloc();
@@ -178,6 +151,9 @@ class Topology : public machine_info, public binder {
 #endif
 
 protected:
+    Topology(Topology &) = delete;
+    Topology(Topology &&) = delete;
+
     Topology()
     {
 #if GKO_HAVE_HWLOC
@@ -241,7 +217,7 @@ class Topology : public machine_info, public binder {
     std::vector<topology_obj_info> gpus_;
     std::vector<topology_obj_info> pus_;
     std::vector<topology_obj_info> cores_;
-    std::size_t num_numas_;
+    size_type num_numas_;
 
     template <typename T>
     using topo_manager = std::unique_ptr<T, std::function<void(T *)>>;

From 1ac0052ed656901c5b981d05199b57665e17824f Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Mon, 19 Oct 2020 09:06:24 +0200
Subject: [PATCH 0274/1008] Add some documentation and rearrange class.

---
 .gitlab-ci.yml                              |   4 +
 core/device_hooks/CMakeLists.txt            |  22 +--
 include/ginkgo/core/base/machine_config.hpp | 167 +++++++++++++++-----
 3 files changed, 140 insertions(+), 53 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ce90238f31f..8f38f81d93c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,6 +22,7 @@ stages:
   BUILD_OMP: "OFF"
   BUILD_CUDA: "OFF"
   BUILD_HIP: "OFF"
+  BUILD_HWLOC: "ON"
   RUN_EXAMPLES: "OFF"
   CONFIG_LOG: "ON"
   CXX_FLAGS: ""
@@ -63,6 +64,7 @@ stages:
         -DGINKGO_DEVEL_TOOLS=OFF -DGINKGO_BUILD_REFERENCE=${BUILD_REFERENCE}
         -DGINKGO_BUILD_OMP=${BUILD_OMP} -DGINKGO_BUILD_CUDA=${BUILD_CUDA}
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
+        -DGINKGO_WITH_HWLOC=${BUILD_HWLOC}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
@@ -94,6 +96,7 @@ stages:
         -DGINKGO_DEVEL_TOOLS=OFF -DGINKGO_BUILD_REFERENCE=${BUILD_REFERENCE}
         -DGINKGO_BUILD_OMP=${BUILD_OMP} -DGINKGO_BUILD_CUDA=${BUILD_CUDA}
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
+        -DGINKGO_WITH_HWLOC=${BUILD_HWLOC}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
@@ -321,6 +324,7 @@ build/cuda100/intel/cuda/release/shared:
     CXX_COMPILER: "icpc"
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
+    BUILD_HWLOC: "OFF"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
   only:
diff --git a/core/device_hooks/CMakeLists.txt b/core/device_hooks/CMakeLists.txt
index f247452f7d4..7e94cd6ab49 100644
--- a/core/device_hooks/CMakeLists.txt
+++ b/core/device_hooks/CMakeLists.txt
@@ -23,6 +23,11 @@ if(NOT GINKGO_BUILD_HIP)
     ginkgo_compile_features(ginkgo_hip)
     ginkgo_default_includes(ginkgo_hip)
     ginkgo_install_library(ginkgo_hip hip)
+
+    if (GINKGO_HAVE_HWLOC)
+        target_include_directories(ginkgo_hip PUBLIC ${HWLOC_INCLUDE_DIRS})
+        target_link_libraries(ginkgo_hip PUBLIC ${HWLOC_LIBRARIES})
+    endif()
 endif()
 
 if (NOT GINKGO_BUILD_OMP)
@@ -45,20 +50,3 @@ if (NOT GINKGO_BUILD_REFERENCE)
     ginkgo_default_includes(ginkgo_reference)
     ginkgo_install_library(ginkgo_reference reference)
 endif()
-<<<<<<< HEAD
-=======
-
-if(NOT GINKGO_BUILD_HIP)
-    add_library(ginkgo_hip
-        $<TARGET_OBJECTS:ginkgo_hip_device>
-        hip_hooks.cpp)
-    ginkgo_compile_features(ginkgo_hip)
-    ginkgo_default_includes(ginkgo_hip)
-    ginkgo_install_library(ginkgo_hip hip)
-
-    if (GINKGO_HAVE_HWLOC)
-        target_include_directories(ginkgo_hip PUBLIC ${HWLOC_INCLUDE_DIRS})
-        target_link_libraries(ginkgo_hip PUBLIC ${HWLOC_LIBRARIES})
-    endif()
-endif()
->>>>>>> Add a test and make the linking to HWLOC PUBLIC
diff --git a/include/ginkgo/core/base/machine_config.hpp b/include/ginkgo/core/base/machine_config.hpp
index fb17e22dc4f..360ce06643d 100644
--- a/include/ginkgo/core/base/machine_config.hpp
+++ b/include/ginkgo/core/base/machine_config.hpp
@@ -50,13 +50,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #if GKO_HAVE_HWLOC
+
 #include <hwloc.h>
-#endif
 
+#else
 
-#if GKO_HAVE_HWLOC == 0
 struct hwloc_obj_type_t {};
 struct hwloc_obj_t {};
+
 #endif
 
 
@@ -64,43 +65,112 @@ struct hwloc_topology;
 
 
 namespace gko {
+/**
+ * @brief The machine configuration namespace.
+ *
+ * @ingroup machine_config
+ */
 namespace machine_config {
 
 
+/**
+ * The topology class represents the heirarchical topology of a machine,
+ * including NUMA nodes, cores and GPUs. Various infomation of the machine are
+ * gathered with the help of the Hardware Locality library (hwloc).
+ *
+ * This class also provides functionalities to bind objects in the topology to
+ * the execution objects. Binding can enhance performance by allowing data to be
+ * closer to the executing object.
+ */
 template <class Executor>
 class Topology {
 private:
+    /**
+     * This struct holds the attributes for an object.
+     */
     struct topology_obj_info {
+        /**
+         * The hwloc object.
+         */
         hwloc_obj_t obj;
+
+        /**
+         * The numa number of the object.
+         */
         int numa;
+
+        /**
+         * The logical_id assigned by the OS.
+         */
         size_type logical_id;
-        size_type physical_id;  // for GPUs, this is their number in the numa
+
+        /**
+         * The physical_id assigned to the object.
+         * For GPUs, this is their number in the numa
+         */
+        size_type physical_id;
     };
 
 public:
+    /**
+     * Creates a new Topology object.
+     */
     static std::unique_ptr<Topology> create()
     {
         return std::unique_ptr<Topology>(new Topology());
     }
 
+    /**
+     * Get the Topology object.
+     *
+     * @return  the topology object.
+     */
     hwloc_topology *get_topology() const { return this->topo_.get(); }
 
+
+    /**
+     * Bind the object associated with the id to a core.
+     *
+     * @param id  The id of the object to be bound.
+     */
     void bind_to_core(size_type id) { hwloc_binding_helper(this->cores_, id); }
 
+    /**
+     * Bind the object associated with the id to a Processing unit(PU).
+     *
+     * @param id  The id of the object to be bound.
+     */
     void bind_to_pu(size_type id) { hwloc_binding_helper(this->pus_, id); }
 
+    /**
+     * Get the object of type PU associated with the id.
+     *
+     * @param id  The id of the PU
+     */
     const topology_obj_info &get_pu(size_type id)
     {
         GKO_ENSURE_IN_BOUNDS(id, pus_.size());
         return pus_[id];
     }
 
+
+    /**
+     * Get the object of type core associated with the id.
+     *
+     * @param id  The id of the core
+     */
     const topology_obj_info &get_core(size_type id)
     {
         GKO_ENSURE_IN_BOUNDS(id, cores_.size());
         return cores_[id];
     }
 
+
+    /**
+     * Get the object of type gpu associated with the id.
+     *
+     * @param id  The id of the gpu
+     */
     const topology_obj_info &get_gpu(size_type id)
     {
         GKO_ENSURE_IN_BOUNDS(id, gpus_.size());
@@ -108,13 +178,64 @@ class Topology {
     }
 
 
+    /**
+     * Get the number of PU objects stored in this Topology tree.
+     */
     size_type get_num_pus() { return pus_.size(); }
+
+
+    /**
+     * Get the number of core objects stored in this Topology tree.
+     */
     size_type get_num_cores() { return cores_.size(); }
+
+
+    /**
+     * Get the number of GPU objects stored in this Topology tree.
+     */
     size_type get_num_gpus() { return gpus_.size(); }
+
+
+    /**
+     * Get the number of NUMA objects stored in this Topology tree.
+     */
     size_type get_num_numas() { return num_numas_; }
 
+
+    /**
+     * Load the gpu objects. These functions are implemened by the respective
+     * GPU Executor classes.
+     */
     virtual void load_gpus() {}
 
+protected:
+    Topology(Topology &) = delete;
+    Topology(Topology &&) = delete;
+
+    Topology()
+    {
+#if GKO_HAVE_HWLOC
+
+
+        this->topo_ = topo_manager<hwloc_topology>(init_topology(),
+                                                   hwloc_topology_destroy);
+
+        load_objects(HWLOC_OBJ_CORE, this->cores_);
+        load_objects(HWLOC_OBJ_PU, this->pus_);
+        this->load_gpus();
+
+        num_numas_ = hwloc_get_nbobjs_by_type(topo_.get(), HWLOC_OBJ_PACKAGE);
+#else
+
+        this->topo_ = topo_manager<hwloc_topology>();
+
+#endif
+    }
+
+
+    /**
+     * A helper function that binds the object with an id.
+     */
     void hwloc_binding_helper(std::vector<topology_obj_info> &obj, size_type id)
     {
 #if GKO_HAVE_HWLOC
@@ -126,8 +247,14 @@ class Topology {
 #endif
     }
 
+
 #if GKO_HAVE_HWLOC
 
+
+    /**
+     * A helper function that prints the topology tree of an object to a given
+     * depth. Provided from the hwloc library.
+     */
     static void hwloc_print_children(hwloc_topology *topology, hwloc_obj_t obj,
                                      int depth)
     {
@@ -148,35 +275,6 @@ class Topology {
         }
     }
 
-#endif
-
-protected:
-    Topology(Topology &) = delete;
-    Topology(Topology &&) = delete;
-
-    Topology()
-    {
-#if GKO_HAVE_HWLOC
-
-
-        this->topo_ = topo_manager<hwloc_topology>(init_topology(),
-                                                   hwloc_topology_destroy);
-
-        load_objects(HWLOC_OBJ_CORE, this->cores_);
-        load_objects(HWLOC_OBJ_PU, this->pus_);
-        this->load_gpus();
-
-        num_numas_ = hwloc_get_nbobjs_by_type(topo_.get(), HWLOC_OBJ_PACKAGE);
-#else
-
-        this->topo_ = topo_manager<hwloc_topology>();
-
-#endif
-    }
-
-
-#if GKO_HAVE_HWLOC
-
 
     // The objects should be sorted by logical index since hwloc uses logical
     // index with these functions
@@ -198,12 +296,8 @@ class Topology {
         hwloc_topology_t tmp;
         hwloc_topology_init(&tmp);
 
-#if HWLOC_API_VERSION >= 0x00020000
         hwloc_topology_set_io_types_filter(tmp,
                                            HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
-#else
-        hwloc_topology_set_flags(tmp, HWLOC_TOPOLOGY_FLAG_IO_DEVICES);
-#endif
         hwloc_topology_set_xml(tmp, GKO_HWLOC_XMLFILE);
         hwloc_topology_load(tmp);
 
@@ -213,6 +307,7 @@ class Topology {
 
 #endif
 
+
 private:
     std::vector<topology_obj_info> gpus_;
     std::vector<topology_obj_info> pus_;

From c0b856be74ef655c01c721cae3ff7c25159af496 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Sat, 28 Nov 2020 23:59:38 +0100
Subject: [PATCH 0275/1008] Update MachineTopology framework.

+ Create a exec info struct to store executor properties.
+ Make get_machine_topology() a global function with MachineTopology
  object a global object created initially and deleted last.
+ All executors can query information from this global object as and
  when needed.

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 core/CMakeLists.txt                           |   1 +
 core/base/machine_topology.cpp                |  78 ++++++
 core/device_hooks/cuda_hooks.cpp              |   3 +
 core/device_hooks/dpcpp_hooks.cpp             |   3 +
 core/device_hooks/hip_hooks.cpp               |   3 +
 core/devices/omp/executor.cpp                 |   3 +
 core/test/base/executor.cpp                   |  40 +--
 cuda/base/executor.cpp                        | 111 ++++----
 dpcpp/base/executor.dp.cpp                    |  17 +-
 hip/base/executor.hip.cpp                     | 121 +++++----
 hip/test/base/hip_executor.hip.cpp            |  46 ++--
 include/ginkgo/core/base/executor.hpp         | 245 ++++++++++--------
 ...achine_config.hpp => machine_topology.hpp} |  77 +++---
 include/ginkgo/ginkgo.hpp                     |   2 +-
 14 files changed, 445 insertions(+), 305 deletions(-)
 create mode 100644 core/base/machine_topology.cpp
 rename include/ginkgo/core/base/{machine_config.hpp => machine_topology.hpp} (82%)

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 0f6db2ab032..6d4524b7c32 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -9,6 +9,7 @@ target_sources(ginkgo
     base/composition.cpp
     base/executor.cpp
     base/mtx_io.cpp
+    base/machine_topology.cpp
     base/perturbation.cpp
     base/version.cpp
     factorization/ic.cpp
diff --git a/core/base/machine_topology.cpp b/core/base/machine_topology.cpp
new file mode 100644
index 00000000000..b424f6f0dcb
--- /dev/null
+++ b/core/base/machine_topology.cpp
@@ -0,0 +1,78 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+
+#include <ginkgo/core/base/machine_topology.hpp>
+
+
+#include <atomic>
+#include <memory>
+#include <mutex>
+
+
+namespace gko {
+
+
+namespace detail {
+
+
+std::shared_ptr<const MachineTopology> machine_topology{};
+std::mutex machine_topology_mutex{};
+std::atomic<bool> initialized_machine_topology{};
+
+
+}  // namespace detail
+
+
+const MachineTopology *get_machine_topology()
+{
+    {
+        if (!detail::initialized_machine_topology.load()) {
+            std::lock_guard<std::mutex> guard(detail::machine_topology_mutex);
+            if (!detail::machine_topology) {
+                detail::machine_topology = MachineTopology::create();
+                detail::initialized_machine_topology.store(true);
+            }
+        }
+    }
+    return detail::machine_topology.get();
+}
+
+
+std::shared_ptr<const MachineTopology> get_shared_machine_topology()
+{
+    get_machine_topology();
+    return detail::machine_topology;
+}
+
+
+}  // namespace gko
diff --git a/core/device_hooks/cuda_hooks.cpp b/core/device_hooks/cuda_hooks.cpp
index 6a0e09486a8..3c88599f897 100644
--- a/core/device_hooks/cuda_hooks.cpp
+++ b/core/device_hooks/cuda_hooks.cpp
@@ -62,6 +62,9 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
 }
 
 
+void CudaExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+
+
 void OmpExecutor::raw_copy_to(const CudaExecutor *, size_type num_bytes,
                               const void *src_ptr, void *dest_ptr) const
     GKO_NOT_COMPILED(cuda);
diff --git a/core/device_hooks/dpcpp_hooks.cpp b/core/device_hooks/dpcpp_hooks.cpp
index df17a4959e4..e336ee7340f 100644
--- a/core/device_hooks/dpcpp_hooks.cpp
+++ b/core/device_hooks/dpcpp_hooks.cpp
@@ -59,6 +59,9 @@ std::shared_ptr<DpcppExecutor> DpcppExecutor::create(
 }
 
 
+void DpcppExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+
+
 void OmpExecutor::raw_copy_to(const DpcppExecutor *, size_type num_bytes,
                               const void *src_ptr, void *dest_ptr) const
     GKO_NOT_COMPILED(dpcpp);
diff --git a/core/device_hooks/hip_hooks.cpp b/core/device_hooks/hip_hooks.cpp
index fbc4b77cdfc..2616b9b847d 100644
--- a/core/device_hooks/hip_hooks.cpp
+++ b/core/device_hooks/hip_hooks.cpp
@@ -59,6 +59,9 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
 }
 
 
+void OmpExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+
+
 void OmpExecutor::raw_copy_to(const HipExecutor *, size_type num_bytes,
                               const void *src_ptr, void *dest_ptr) const
     GKO_NOT_COMPILED(hip);
diff --git a/core/devices/omp/executor.cpp b/core/devices/omp/executor.cpp
index f13101f092e..ba9a7e768cc 100644
--- a/core/devices/omp/executor.cpp
+++ b/core/devices/omp/executor.cpp
@@ -44,6 +44,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace gko {
 
 
+void OmpExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+
+
 void OmpExecutor::raw_free(void *ptr) const noexcept { std::free(ptr); }
 
 
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 079e353dcb0..ceeafebb5ec 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -101,26 +101,26 @@ TEST(OmpExecutor, RunsCorrectLambdaOperation)
     ASSERT_EQ(1, value);
 }
 
-#if GKO_HAVE_HWLOC
-
-
-TEST(OmpExecutor, GetsExecInfo)
-{
-    int num_pus = 0;
-    int num_cores = 0;
-    int num_numas = 0;
-    auto omp = gko::OmpExecutor::create();
-    auto omp_info = omp->get_exec_info();
-    num_pus = omp_info->get_num_pus();
-    num_cores = omp_info->get_num_cores();
-    num_numas = omp_info->get_num_numas();
-    EXPECT_GT(num_numas, 0);
-    EXPECT_GT(num_pus, 0);
-    EXPECT_GT(num_cores, 0);
-}
-
-
-#endif
+// #if GKO_HAVE_HWLOC
+
+
+// TEST(OmpExecutor, GetsExecInfo)
+// {
+//     int num_pus = 0;
+//     int num_cores = 0;
+//     int num_numas = 0;
+//     auto omp = gko::OmpExecutor::create();
+//     auto omp_info = omp->get_exec_info();
+//     num_pus = omp_info->get_num_pus();
+//     num_cores = omp_info->get_num_cores();
+//     num_numas = omp_info->get_num_numas();
+//     EXPECT_GT(num_numas, 0);
+//     EXPECT_GT(num_pus, 0);
+//     EXPECT_GT(num_cores, 0);
+// }
+
+
+// #endif
 
 
 TEST(OmpExecutor, AllocatesAndFreesMemory)
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 48094a16830..a4cfa396d0d 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -54,45 +54,45 @@ namespace gko {
 #include "common/base/executor.hpp.inc"
 
 
-namespace machine_config {
-
-
-template <>
-void Topology<CudaExecutor>::load_gpus()
-{
-#if GKO_HAVE_HWLOC
-    size_type num_in_numa = 0;
-    int last_numa = 0;
-    auto topology = this->topo_.get();
-    auto n_objs = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_OS_DEVICE);
-    for (size_type i = 1; i < n_objs; i++, num_in_numa++) {
-        hwloc_obj_t obj = NULL;
-        while ((obj = hwloc_get_next_osdev(topology, obj)) != NULL) {
-            bool is_gpu = HWLOC_OBJ_OSDEV_COPROC == obj->attr->osdev.type &&
-                          obj->name && !strncmp("cuda", obj->name, 4) &&
-                          atoi(obj->name + 4) == (int)i;
-            if (is_gpu) {
-                while (obj &&
-                       (!obj->nodeset || hwloc_bitmap_iszero(obj->nodeset)))
-                    obj = obj->parent;
-                if (obj && obj->nodeset) {
-                    auto this_numa = hwloc_bitmap_first(obj->nodeset);
-                    if (this_numa != last_numa) {
-                        num_in_numa = 0;
-                    }
-                    this->gpus_.push_back(
-                        topology_obj_info{obj, this_numa, i, num_in_numa});
-                    last_numa = this_numa;
-                }
-            }
-        }
-    }
-
-#endif
-}
-
-
-}  // namespace machine_config
+// namespace machine_config {
+
+
+// template <>
+// void Topology<CudaExecutor>::load_gpus()
+// {
+// #if GKO_HAVE_HWLOC
+//     size_type num_in_numa = 0;
+//     int last_numa = 0;
+//     auto topology = this->topo_.get();
+//     auto n_objs = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_OS_DEVICE);
+//     for (size_type i = 1; i < n_objs; i++, num_in_numa++) {
+//         hwloc_obj_t obj = NULL;
+//         while ((obj = hwloc_get_next_osdev(topology, obj)) != NULL) {
+//             bool is_gpu = HWLOC_OBJ_OSDEV_COPROC == obj->attr->osdev.type &&
+//                           obj->name && !strncmp("cuda", obj->name, 4) &&
+//                           atoi(obj->name + 4) == (int)i;
+//             if (is_gpu) {
+//                 while (obj &&
+//                        (!obj->nodeset || hwloc_bitmap_iszero(obj->nodeset)))
+//                     obj = obj->parent;
+//                 if (obj && obj->nodeset) {
+//                     auto this_numa = hwloc_bitmap_first(obj->nodeset);
+//                     if (this_numa != last_numa) {
+//                         num_in_numa = 0;
+//                     }
+//                     this->gpus_.push_back(
+//                         topology_obj_info{obj, this_numa, i, num_in_numa});
+//                     last_numa = this_numa;
+//                 }
+//             }
+//         }
+//     }
+
+// #endif
+// }
+
+
+// }  // namespace machine_config
 
 
 std::shared_ptr<CudaExecutor> CudaExecutor::create(
@@ -111,6 +111,9 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
 }
 
 
+void CudaExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+
+
 void OmpExecutor::raw_copy_to(const CudaExecutor *dest, size_type num_bytes,
                               const void *src_ptr, void *dest_ptr) const
 {
@@ -129,9 +132,10 @@ void CudaExecutor::raw_free(void *ptr) const noexcept
     if (error_code != cudaSuccess) {
 #if GKO_VERBOSE_LEVEL >= 1
         // Unfortunately, if memory free fails, there's not much we can do
-        std::cerr << "Unrecoverable CUDA error on device " << this->device_id_
-                  << " in " << __func__ << ": " << cudaGetErrorName(error_code)
-                  << ": " << cudaGetErrorString(error_code) << std::endl
+        std::cerr << "Unrecoverable CUDA error on device "
+                  << this->get_device_id() << " in " << __func__ << ": "
+                  << cudaGetErrorName(error_code) << ": "
+                  << cudaGetErrorString(error_code) << std::endl
                   << "Exiting program" << std::endl;
 #endif  // GKO_VERBOSE_LEVEL >= 1
         std::exit(error_code);
@@ -233,24 +237,31 @@ int CudaExecutor::get_num_devices()
 
 void CudaExecutor::set_gpu_property()
 {
-    if (device_id_ < this->get_num_devices() && device_id_ >= 0) {
+    if (this->cuda_exec_info_.device_id < this->get_num_devices() &&
+        this->cuda_exec_info_.device_id >= 0) {
         cuda::device_guard g(this->get_device_id());
         GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
-            &major_, cudaDevAttrComputeCapabilityMajor, device_id_));
+            &this->cuda_exec_info_.major, cudaDevAttrComputeCapabilityMajor,
+            this->cuda_exec_info_.device_id));
         GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
-            &minor_, cudaDevAttrComputeCapabilityMinor, device_id_));
+            &this->cuda_exec_info_.minor, cudaDevAttrComputeCapabilityMinor,
+            this->cuda_exec_info_.device_id));
         GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
-            &num_multiprocessor_, cudaDevAttrMultiProcessorCount, device_id_));
-        num_warps_per_sm_ = convert_sm_ver_to_cores(major_, minor_) /
-                            kernels::cuda::config::warp_size;
-        warp_size_ = kernels::cuda::config::warp_size;
+            &this->cuda_exec_info_.num_cores, cudaDevAttrMultiProcessorCount,
+            this->cuda_exec_info_.device_id));
+        this->cuda_exec_info_.num_work_groups_per_core =
+            convert_sm_ver_to_cores(this->cuda_exec_info_.major,
+                                    this->cuda_exec_info_.minor) /
+            kernels::cuda::config::warp_size;
+        this->cuda_exec_info_.warp_size = kernels::cuda::config::warp_size;
     }
 }
 
 
 void CudaExecutor::init_handles()
 {
-    if (device_id_ < this->get_num_devices() && device_id_ >= 0) {
+    if (this->cuda_exec_info_.device_id < this->get_num_devices() &&
+        this->cuda_exec_info_.device_id >= 0) {
         const auto id = this->get_device_id();
         cuda::device_guard g(id);
         this->cublas_handle_ = handle_manager<cublasContext>(
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 0807c0b59ce..38abcd359d7 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -93,6 +93,9 @@ std::shared_ptr<DpcppExecutor> DpcppExecutor::create(
 }
 
 
+void DpcppExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+
+
 void DpcppExecutor::raw_free(void *ptr) const noexcept
 {
     sycl::free(ptr, queue_->get_context());
@@ -193,25 +196,27 @@ void delete_queue(sycl::queue *queue)
 
 void DpcppExecutor::set_device_property()
 {
-    assert(device_id_ < DpcppExecutor::get_num_devices(device_type_));
-    auto device = detail::get_devices(device_type_)[device_id_];
+    assert(dpcpp_exec_info_.device_id <
+           DpcppExecutor::get_num_devices(dpcpp_exec_info_.device_type));
+    auto device = detail::get_devices(
+        dpcpp_exec_info_.device_type)[dpcpp_exec_info_.device_id];
     if (!device.is_host()) {
         try {
-            subgroup_sizes_ =
+            dpcpp_exec_info_.subgroup_sizes =
                 device.get_info<cl::sycl::info::device::sub_group_sizes>();
         } catch (cl::sycl::runtime_error &err) {
             GKO_NOT_SUPPORTED(device);
         }
     }
-    num_computing_units_ =
+    dpcpp_exec_info_.num_computing_units =
         device.get_info<sycl::info::device::max_compute_units>();
     auto max_workitem_sizes =
         device.get_info<sycl::info::device::max_work_item_sizes>();
     // There is no way to get the dimension of a sycl::id object
     for (std::size_t i = 0; i < 3; i++) {
-        max_workitem_sizes_.push_back(max_workitem_sizes[i]);
+        dpcpp_exec_info_.max_workitem_sizes.push_back(max_workitem_sizes[i]);
     }
-    max_workgroup_size_ =
+    dpcpp_exec_info_.num_work_groups_per_core =
         device.get_info<sycl::info::device::max_work_group_size>();
     // Here we declare the queue with the property `in_order` which ensures the
     // kernels are executed in the submission order. Otherwise, calls to
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 865fdec4a6c..654d500a25b 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -55,48 +55,49 @@ namespace gko {
 #include "common/base/executor.hpp.inc"
 
 
-namespace machine_config {
-
-
-template <>
-void Topology<HipExecutor>::load_gpus()
-{
-#if GKO_HAVE_HWLOC
-    size_type num_in_numa = 0;
-    int last_numa = 0;
-    auto topology = this->topo_.get();
-    auto n_objs = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_OS_DEVICE);
-    for (size_type i = 0; i < n_objs; i++, num_in_numa++) {
-        hwloc_obj_t obj = NULL;
-        while ((obj = hwloc_get_next_osdev(topology, obj)) != NULL) {
-            bool is_gpu = (HWLOC_OBJ_OSDEV_GPU == obj->attr->osdev.type ||
-                           HWLOC_OBJ_OSDEV_COPROC == obj->attr->osdev.type) &&
-                          obj->name &&
-                          (!strncmp("rsmi", obj->name, 4) ||
-                           !strncmp("cuda", obj->name, 4)) &&
-                          atoi(obj->name + 4) == (int)i;
-            if (is_gpu) {
-                while (obj &&
-                       (!obj->nodeset || hwloc_bitmap_iszero(obj->nodeset)))
-                    obj = obj->parent;
-                if (obj && obj->nodeset) {
-                    auto this_numa = hwloc_bitmap_first(obj->nodeset);
-                    if (this_numa != last_numa) {
-                        num_in_numa = 0;
-                    }
-                    this->gpus_.push_back(
-                        topology_obj_info{obj, this_numa, i, num_in_numa});
-                    last_numa = this_numa;
-                }
-            }
-        }
-    }
-
-#endif
-}
-
-
-}  // namespace machine_config
+// namespace machine_config {
+
+
+// template <>
+// void Topology<HipExecutor>::load_gpus()
+// {
+// #if GKO_HAVE_HWLOC
+//     size_type num_in_numa = 0;
+//     int last_numa = 0;
+//     auto topology = this->topo_.get();
+//     auto n_objs = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_OS_DEVICE);
+//     for (size_type i = 0; i < n_objs; i++, num_in_numa++) {
+//         hwloc_obj_t obj = NULL;
+//         while ((obj = hwloc_get_next_osdev(topology, obj)) != NULL) {
+//             bool is_gpu = (HWLOC_OBJ_OSDEV_GPU == obj->attr->osdev.type ||
+//                            HWLOC_OBJ_OSDEV_COPROC == obj->attr->osdev.type)
+//                            &&
+//                           obj->name &&
+//                           (!strncmp("rsmi", obj->name, 4) ||
+//                            !strncmp("cuda", obj->name, 4)) &&
+//                           atoi(obj->name + 4) == (int)i;
+//             if (is_gpu) {
+//                 while (obj &&
+//                        (!obj->nodeset || hwloc_bitmap_iszero(obj->nodeset)))
+//                     obj = obj->parent;
+//                 if (obj && obj->nodeset) {
+//                     auto this_numa = hwloc_bitmap_first(obj->nodeset);
+//                     if (this_numa != last_numa) {
+//                         num_in_numa = 0;
+//                     }
+//                     this->gpus_.push_back(
+//                         topology_obj_info{obj, this_numa, i, num_in_numa});
+//                     last_numa = this_numa;
+//                 }
+//             }
+//         }
+//     }
+
+// #endif
+// }
+
+
+// }  // namespace machine_config
 
 
 std::shared_ptr<HipExecutor> HipExecutor::create(
@@ -115,6 +116,9 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
 }
 
 
+void HipExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+
+
 void OmpExecutor::raw_copy_to(const HipExecutor *dest, size_type num_bytes,
                               const void *src_ptr, void *dest_ptr) const
 {
@@ -133,9 +137,10 @@ void HipExecutor::raw_free(void *ptr) const noexcept
     if (error_code != hipSuccess) {
 #if GKO_VERBOSE_LEVEL >= 1
         // Unfortunately, if memory free fails, there's not much we can do
-        std::cerr << "Unrecoverable HIP error on device " << this->device_id_
-                  << " in " << __func__ << ": " << hipGetErrorName(error_code)
-                  << ": " << hipGetErrorString(error_code) << std::endl
+        std::cerr << "Unrecoverable HIP error on device "
+                  << this->get_device_id() << " in " << __func__ << ": "
+                  << hipGetErrorName(error_code) << ": "
+                  << hipGetErrorString(error_code) << std::endl
                   << "Exiting program" << std::endl;
 #endif  // GKO_VERBOSE_LEVEL >= 1
         std::exit(error_code);
@@ -237,31 +242,37 @@ int HipExecutor::get_num_devices()
 
 void HipExecutor::set_gpu_property()
 {
-    if (device_id_ < this->get_num_devices() && device_id_ >= 0) {
+    if (this->get_device_id() < this->get_num_devices() &&
+        this->get_device_id() >= 0) {
         hip::device_guard g(this->get_device_id());
         GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
-            &num_multiprocessor_, hipDeviceAttributeMultiprocessorCount,
-            device_id_));
+            &this->hip_exec_info_.num_cores,
+            hipDeviceAttributeMultiprocessorCount, this->get_device_id()));
         GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
-            &major_, hipDeviceAttributeComputeCapabilityMajor, device_id_));
+            &this->hip_exec_info_.major,
+            hipDeviceAttributeComputeCapabilityMajor, this->get_device_id()));
         GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
-            &minor_, hipDeviceAttributeComputeCapabilityMinor, device_id_));
+            &this->hip_exec_info_.minor,
+            hipDeviceAttributeComputeCapabilityMinor, this->get_device_id()));
 #if GINKGO_HIP_PLATFORM_NVCC
-        num_warps_per_sm_ = convert_sm_ver_to_cores(major_, minor_) /
-                            kernels::hip::config::warp_size;
+        this->hip_exec_info_.num_work_groups_per_core =
+            convert_sm_ver_to_cores(this->hip_exec_info_.major,
+                                    this->hip_exec_info_.minor) /
+            kernels::hip::config::warp_size;
 #else
         // In GCN (Graphics Core Next), each multiprocessor has 4 SIMD
         // Reference: https://en.wikipedia.org/wiki/Graphics_Core_Next
-        num_warps_per_sm_ = 4;
+        this->hip_exec_info_.num_work_groups_per_core = 4;
 #endif  // GINKGO_HIP_PLATFORM_NVCC
-        warp_size_ = kernels::hip::config::warp_size;
+        this->hip_exec_info_.warp_size = kernels::hip::config::warp_size;
     }
 }
 
 
 void HipExecutor::init_handles()
 {
-    if (device_id_ < this->get_num_devices() && device_id_ >= 0) {
+    if (this->get_device_id() < this->get_num_devices() &&
+        this->get_device_id() >= 0) {
         const auto id = this->get_device_id();
         hip::device_guard g(id);
         this->hipblas_handle_ = handle_manager<hipblasContext>(
diff --git a/hip/test/base/hip_executor.hip.cpp b/hip/test/base/hip_executor.hip.cpp
index 5ca1b9915e0..1eb55530ec5 100644
--- a/hip/test/base/hip_executor.hip.cpp
+++ b/hip/test/base/hip_executor.hip.cpp
@@ -136,29 +136,29 @@ TEST_F(HipExecutor, MasterKnowsNumberOfDevices)
 }
 
 
-#if GKO_HAVE_HWLOC
-
-
-TEST_F(HipExecutor, GetsExecInfo)
-{
-    int num_pus = 0;
-    int num_gpus = 0;
-    int num_cores = 0;
-    int num_numas = 0;
-    auto hip = gko::HipExecutor::create(0, omp);
-    auto hip_info = hip->get_exec_info();
-    num_pus = hip_info->get_num_pus();
-    num_gpus = hip_info->get_num_gpus();
-    num_cores = hip_info->get_num_cores();
-    num_numas = hip_info->get_num_numas();
-    EXPECT_GT(num_numas, 0);
-    EXPECT_GT(num_gpus, 0);
-    EXPECT_GT(num_pus, 0);
-    EXPECT_GT(num_cores, 0);
-}
-
-
-#endif
+// #if GKO_HAVE_HWLOC
+
+
+// TEST_F(HipExecutor, GetsExecInfo)
+// {
+//     int num_pus = 0;
+//     int num_gpus = 0;
+//     int num_cores = 0;
+//     int num_numas = 0;
+//     auto hip = gko::HipExecutor::create(0, omp);
+//     auto hip_info = hip->get_exec_info();
+//     num_pus = hip_info->get_num_pus();
+//     num_gpus = hip_info->get_num_gpus();
+//     num_cores = hip_info->get_num_cores();
+//     num_numas = hip_info->get_num_numas();
+//     EXPECT_GT(num_numas, 0);
+//     EXPECT_GT(num_gpus, 0);
+//     EXPECT_GT(num_pus, 0);
+//     EXPECT_GT(num_cores, 0);
+// }
+
+
+// #endif
 
 
 TEST_F(HipExecutor, AllocatesAndFreesMemory)
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 5ea6cde6b42..701a7ad9384 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -42,7 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <type_traits>
 
 
-#include <ginkgo/core/base/machine_config.hpp>
+#include <ginkgo/core/base/machine_topology.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/log/logger.hpp>
 #include <ginkgo/core/synthesizer/containers.hpp>
@@ -725,6 +725,20 @@ class Executor : public log::EnableLogging<Executor> {
 
 #undef GKO_ENABLE_VERIFY_MEMORY_TO
 
+    /**
+     * Populates the executor specific info from the global machine topology
+     * object.
+     *
+     * @param mach_topo the machine topology object.
+     */
+    virtual void populate_exec_info(const MachineTopology *mach_topo) = 0;
+
+    struct exec_info {
+        int device_id = -1;
+        int num_cores = 0;
+        int num_work_groups_per_core = 0;
+    };
+
 private:
     /**
      * The LambdaOperation class wraps three functor objects into an
@@ -956,13 +970,10 @@ class EnableDeviceReset {
  * @ingroup Executor
  */
 class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
-                    public std::enable_shared_from_this<OmpExecutor>,
-                    public machine_config::Topology<OmpExecutor> {
+                    public std::enable_shared_from_this<OmpExecutor> {
     friend class detail::ExecutorBase<OmpExecutor>;
 
 public:
-    using omp_exec_info = machine_config::Topology<OmpExecutor>;
-
     /**
      * Creates a new OmpExecutor.
      */
@@ -977,15 +988,10 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
 
     void synchronize() const override;
 
-    /**
-     * Get the Executor information for this executor
-     *
-     * @return the executor info (omp_exec_info*) for this executor
-     */
-    omp_exec_info *get_exec_info() const { return exec_info_.get(); }
-
 protected:
-    OmpExecutor() : exec_info_(omp_exec_info::create()) {}
+    OmpExecutor() = default;
+
+    void populate_exec_info(const MachineTopology *mach_topo) override;
 
     void *raw_alloc(size_type size) const override;
 
@@ -1004,7 +1010,7 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
     bool verify_memory_to(const DpcppExecutor *dest_exec) const override;
 
 private:
-    std::unique_ptr<omp_exec_info> exec_info_;
+    exec_info omp_exec_info_;
 };
 
 
@@ -1024,8 +1030,6 @@ using DefaultExecutor = OmpExecutor;
  */
 class ReferenceExecutor : public OmpExecutor {
 public:
-    using ref_exec_info = machine_config::Topology<OmpExecutor>;
-
     static std::shared_ptr<ReferenceExecutor> create()
     {
         return std::shared_ptr<ReferenceExecutor>(new ReferenceExecutor());
@@ -1039,15 +1043,15 @@ class ReferenceExecutor : public OmpExecutor {
         this->template log<log::Logger::operation_completed>(this, &op);
     }
 
-    /**
-     * Get the Executor information for this executor
-     *
-     * @return the executor info (ref_exec_info*) for this executor
-     */
-    ref_exec_info *get_exec_info() const { return exec_info_.get(); }
-
 protected:
-    ReferenceExecutor() : exec_info_(ref_exec_info::create()) {}
+    ReferenceExecutor() = default;
+
+    void populate_exec_info(const MachineTopology *) override
+    {
+        this->ref_exec_info_.device_id = -1;
+        this->ref_exec_info_.num_cores = 1;
+        this->ref_exec_info_.num_work_groups_per_core = 1;
+    }
 
     bool verify_memory_from(const Executor *src_exec) const override
     {
@@ -1065,7 +1069,7 @@ class ReferenceExecutor : public OmpExecutor {
     GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(HipExecutor, false);
 
 private:
-    std::unique_ptr<ref_exec_info> exec_info_;
+    exec_info ref_exec_info_;
 };
 
 
@@ -1084,13 +1088,10 @@ using DefaultExecutor = ReferenceExecutor;
  */
 class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
                      public std::enable_shared_from_this<CudaExecutor>,
-                     public detail::EnableDeviceReset,
-                     public machine_config::Topology<CudaExecutor> {
+                     public detail::EnableDeviceReset {
     friend class detail::ExecutorBase<CudaExecutor>;
 
 public:
-    using cuda_exec_info = machine_config::Topology<CudaExecutor>;
-
     /**
      * Creates a new CudaExecutor.
      *
@@ -1102,7 +1103,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         int device_id, std::shared_ptr<Executor> master,
         bool device_reset = false);
 
-    ~CudaExecutor() { decrease_num_execs(this->device_id_); }
+    ~CudaExecutor() { decrease_num_execs(this->cuda_exec_info_.device_id); }
 
     std::shared_ptr<Executor> get_master() noexcept override;
 
@@ -1115,7 +1116,10 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     /**
      * Get the CUDA device id of the device associated to this executor.
      */
-    int get_device_id() const noexcept { return device_id_; }
+    int get_device_id() const noexcept
+    {
+        return this->cuda_exec_info_.device_id;
+    }
 
     /**
      * Get the number of devices present on the system.
@@ -1125,35 +1129,51 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     /**
      * Get the number of warps per SM of this executor.
      */
-    int get_num_warps_per_sm() const noexcept { return num_warps_per_sm_; }
+    int get_num_warps_per_sm() const noexcept
+    {
+        return this->cuda_exec_info_.num_work_groups_per_core;
+    }
 
     /**
      * Get the number of multiprocessor of this executor.
      */
-    int get_num_multiprocessor() const noexcept { return num_multiprocessor_; }
+    int get_num_multiprocessor() const noexcept
+    {
+        return this->cuda_exec_info_.num_cores;
+    }
 
     /**
      * Get the number of warps of this executor.
      */
     int get_num_warps() const noexcept
     {
-        return num_multiprocessor_ * num_warps_per_sm_;
+        return this->cuda_exec_info_.num_cores *
+               this->cuda_exec_info_.num_work_groups_per_core;
     }
 
     /**
      * Get the warp size of this executor.
      */
-    int get_warp_size() const noexcept { return warp_size_; }
+    int get_warp_size() const noexcept
+    {
+        return this->cuda_exec_info_.warp_size;
+    }
 
     /**
      * Get the major verion of compute capability.
      */
-    int get_major_version() const noexcept { return major_; }
+    int get_major_version() const noexcept
+    {
+        return this->cuda_exec_info_.major;
+    }
 
     /**
      * Get the minor verion of compute capability.
      */
-    int get_minor_version() const noexcept { return minor_; }
+    int get_minor_version() const noexcept
+    {
+        return this->cuda_exec_info_.minor;
+    }
 
     /**
      * Get the cublas handle for this executor
@@ -1172,13 +1192,6 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         return cusparse_handle_.get();
     }
 
-    /**
-     * Get the Executor information for this executor
-     *
-     * @return the executor info (cuda_exec_info*) for this executor
-     */
-    cuda_exec_info *get_exec_info() const { return exec_info_.get(); }
-
 protected:
     void set_gpu_property();
 
@@ -1186,20 +1199,17 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
     CudaExecutor(int device_id, std::shared_ptr<Executor> master,
                  bool device_reset = false)
-        : EnableDeviceReset{device_reset},
-          device_id_(device_id),
-          master_(master),
-          num_warps_per_sm_(0),
-          num_multiprocessor_(0),
-          major_(0),
-          minor_(0),
-          warp_size_(0)
+        : EnableDeviceReset{device_reset}, master_(master)
     {
-        assert(device_id < max_devices && device_id >= 0);
+        this->cuda_exec_info_.device_id = device_id;
+        this->cuda_exec_info_.num_cores = 0;
+        this->cuda_exec_info_.num_work_groups_per_core = 0;
+        assert(this->cuda_exec_info_.device_id < max_devices &&
+               this->cuda_exec_info_.device_id >= 0);
         this->set_gpu_property();
         this->init_handles();
-        increase_num_execs(device_id);
-        exec_info_ = cuda_exec_info::create();
+        increase_num_execs(this->cuda_exec_info_.device_id);
+        this->populate_exec_info(get_machine_topology());
     }
 
     void *raw_alloc(size_type size) const override;
@@ -1236,14 +1246,16 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         return num_execs[device_id];
     }
 
+    void populate_exec_info(const MachineTopology *mach_topo) override;
+
 private:
-    int device_id_;
     std::shared_ptr<Executor> master_;
-    int num_warps_per_sm_;
-    int num_multiprocessor_;
-    int major_;
-    int minor_;
-    int warp_size_;
+
+    struct cuda_exec_info : public exec_info {
+        int major;
+        int minor;
+        int warp_size;
+    } cuda_exec_info_;
 
     template <typename T>
     using handle_manager = std::unique_ptr<T, std::function<void(T *)>>;
@@ -1253,7 +1265,6 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     static constexpr int max_devices = 64;
     static unsigned num_execs[max_devices];
     static std::mutex mutex[max_devices];
-    std::unique_ptr<cuda_exec_info> exec_info_;
 };
 
 
@@ -1272,13 +1283,10 @@ using DefaultExecutor = CudaExecutor;
  */
 class HipExecutor : public detail::ExecutorBase<HipExecutor>,
                     public std::enable_shared_from_this<HipExecutor>,
-                    public detail::EnableDeviceReset,
-                    public machine_config::Topology<HipExecutor> {
+                    public detail::EnableDeviceReset {
     friend class detail::ExecutorBase<HipExecutor>;
 
 public:
-    using hip_exec_info = machine_config::Topology<HipExecutor>;
-
     /**
      * Creates a new HipExecutor.
      *
@@ -1290,7 +1298,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
                                                std::shared_ptr<Executor> master,
                                                bool device_reset = false);
 
-    ~HipExecutor() { decrease_num_execs(this->device_id_); }
+    ~HipExecutor() { decrease_num_execs(this->hip_exec_info_.device_id); }
 
     std::shared_ptr<Executor> get_master() noexcept override;
 
@@ -1303,7 +1311,10 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     /**
      * Get the HIP device id of the device associated to this executor.
      */
-    int get_device_id() const noexcept { return device_id_; }
+    int get_device_id() const noexcept
+    {
+        return this->hip_exec_info_.device_id;
+    }
 
     /**
      * Get the number of devices present on the system.
@@ -1313,35 +1324,51 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     /**
      * Get the number of warps per SM of this executor.
      */
-    int get_num_warps_per_sm() const noexcept { return num_warps_per_sm_; }
+    int get_num_warps_per_sm() const noexcept
+    {
+        return this->hip_exec_info_.num_work_groups_per_core;
+    }
 
     /**
      * Get the number of multiprocessor of this executor.
      */
-    int get_num_multiprocessor() const noexcept { return num_multiprocessor_; }
+    int get_num_multiprocessor() const noexcept
+    {
+        return this->hip_exec_info_.num_cores;
+    }
 
     /**
      * Get the major verion of compute capability.
      */
-    int get_major_version() const noexcept { return major_; }
+    int get_major_version() const noexcept
+    {
+        return this->hip_exec_info_.major;
+    }
 
     /**
      * Get the minor verion of compute capability.
      */
-    int get_minor_version() const noexcept { return minor_; }
+    int get_minor_version() const noexcept
+    {
+        return this->hip_exec_info_.minor;
+    }
 
     /**
      * Get the number of warps of this executor.
      */
     int get_num_warps() const noexcept
     {
-        return num_multiprocessor_ * num_warps_per_sm_;
+        return this->hip_exec_info_.num_cores *
+               this->hip_exec_info_.num_work_groups_per_core;
     }
 
     /**
      * Get the warp size of this executor.
      */
-    int get_warp_size() const noexcept { return warp_size_; }
+    int get_warp_size() const noexcept
+    {
+        return this->hip_exec_info_.warp_size;
+    }
 
     /**
      * Get the hipblas handle for this executor
@@ -1360,13 +1387,6 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         return hipsparse_handle_.get();
     }
 
-    /**
-     * Get the Executor information for this executor
-     *
-     * @return the executor info (hip_exec_info*) for this executor
-     */
-    hip_exec_info *get_exec_info() const { return exec_info_.get(); }
-
 protected:
     void set_gpu_property();
 
@@ -1374,20 +1394,16 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
     HipExecutor(int device_id, std::shared_ptr<Executor> master,
                 bool device_reset = false)
-        : EnableDeviceReset{device_reset},
-          device_id_(device_id),
-          master_(master),
-          num_multiprocessor_(0),
-          num_warps_per_sm_(0),
-          major_(0),
-          minor_(0),
-          warp_size_(0)
+        : EnableDeviceReset{device_reset}, master_(master)
     {
-        assert(device_id < max_devices);
+        this->hip_exec_info_.device_id = device_id;
+        this->hip_exec_info_.num_cores = 0;
+        this->hip_exec_info_.num_work_groups_per_core = 0;
+        assert(this->hip_exec_info_.device_id < max_devices);
         this->set_gpu_property();
         this->init_handles();
-        increase_num_execs(device_id);
-        exec_info_ = hip_exec_info::create();
+        increase_num_execs(this->hip_exec_info_.device_id);
+        this->populate_exec_info(get_machine_topology());
     }
 
     void *raw_alloc(size_type size) const override;
@@ -1424,14 +1440,16 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         return num_execs[device_id];
     }
 
+    void populate_exec_info(const MachineTopology *mach_topo) override;
+
 private:
-    int device_id_;
     std::shared_ptr<Executor> master_;
-    int num_multiprocessor_;
-    int num_warps_per_sm_;
-    int major_;
-    int minor_;
-    int warp_size_;
+
+    struct hip_exec_info : public exec_info {
+        int major = 0;
+        int minor = 0;
+        int warp_size = 0;
+    } hip_exec_info_;
 
     template <typename T>
     using handle_manager = std::unique_ptr<T, std::function<void(T *)>>;
@@ -1441,7 +1459,6 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     static constexpr int max_devices = 64;
     static int num_execs[max_devices];
     static std::mutex mutex[max_devices];
-    std::unique_ptr<hip_exec_info> exec_info_;
 };
 
 
@@ -1489,7 +1506,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      *
      * @return the DPCPP device id of the device associated to this executor
      */
-    int get_device_id() const noexcept { return device_id_; }
+    int get_device_id() const noexcept { return dpcpp_exec_info_.device_id; }
 
     ::cl::sycl::queue *get_queue() const { return queue_.get(); }
 
@@ -1509,7 +1526,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      */
     const std::vector<size_type> &get_subgroup_sizes() const noexcept
     {
-        return subgroup_sizes_;
+        return dpcpp_exec_info_.subgroup_sizes;
     }
 
     /**
@@ -1519,7 +1536,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      */
     size_type get_num_computing_units() const noexcept
     {
-        return num_computing_units_;
+        return dpcpp_exec_info_.num_computing_units;
     }
 
     /**
@@ -1529,7 +1546,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      */
     const std::vector<size_type> &get_max_workitem_sizes() const noexcept
     {
-        return max_workitem_sizes_;
+        return dpcpp_exec_info_.max_workitem_sizes;
     }
 
     /**
@@ -1539,7 +1556,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      */
     size_type get_max_workgroup_size() const noexcept
     {
-        return max_workgroup_size_;
+        return dpcpp_exec_info_.num_work_groups_per_core;
     }
 
     /**
@@ -1554,13 +1571,16 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
 
     DpcppExecutor(int device_id, std::shared_ptr<Executor> master,
                   std::string device_type = "all")
-        : device_id_(device_id), master_(master), device_type_(device_type)
+        : master_(master)
     {
-        std::for_each(device_type_.begin(), device_type_.end(),
+        std::for_each(this->dpcpp_exec_info_.device_type.begin(),
+                      this->dpcpp_exec_info_.device_type.end(),
                       [](char &c) { c = std::tolower(c); });
         this->set_device_property();
     }
 
+    void populate_exec_info(const MachineTopology *mach_topo) override;
+
     void *raw_alloc(size_type size) const override;
 
     void raw_free(void *ptr) const noexcept override;
@@ -1578,13 +1598,14 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
     bool verify_memory_to(const DpcppExecutor *dest_exec) const override;
 
 private:
-    int device_id_;
     std::shared_ptr<Executor> master_;
-    std::string device_type_;
-    int num_computing_units_{};
-    std::vector<size_type> subgroup_sizes_{};
-    std::vector<size_type> max_workitem_sizes_{};
-    size_type max_workgroup_size_{};
+
+    struct dpcpp_exec_info : public exec_info {
+        std::string device_type;
+        int num_computing_units;
+        std::vector<size_type> subgroup_sizes{};
+        std::vector<size_type> max_workitem_sizes{};
+    } dpcpp_exec_info_;
 
     template <typename T>
     using queue_manager = std::unique_ptr<T, std::function<void(T *)>>;
diff --git a/include/ginkgo/core/base/machine_config.hpp b/include/ginkgo/core/base/machine_topology.hpp
similarity index 82%
rename from include/ginkgo/core/base/machine_config.hpp
rename to include/ginkgo/core/base/machine_topology.hpp
index 360ce06643d..736cfbfb517 100644
--- a/include/ginkgo/core/base/machine_config.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_MACHINE_CONFIG_HPP_
-#define GKO_CORE_BASE_MACHINE_CONFIG_HPP_
+#ifndef GKO_CORE_BASE_MACHINE_TOPOLOGY_HPP_
+#define GKO_CORE_BASE_MACHINE_TOPOLOGY_HPP_
 
 
 #include <cassert>
@@ -65,16 +65,17 @@ struct hwloc_topology;
 
 
 namespace gko {
-/**
- * @brief The machine configuration namespace.
- *
- * @ingroup machine_config
- */
-namespace machine_config {
+
+
+class MachineTopology;
+
+extern const MachineTopology *get_machine_topology();
+
+extern std::shared_ptr<const MachineTopology> get_shared_machine_topology();
 
 
 /**
- * The topology class represents the heirarchical topology of a machine,
+ * The machine topology class represents the heirarchical topology of a machine,
  * including NUMA nodes, cores and GPUs. Various infomation of the machine are
  * gathered with the help of the Hardware Locality library (hwloc).
  *
@@ -82,13 +83,14 @@ namespace machine_config {
  * the execution objects. Binding can enhance performance by allowing data to be
  * closer to the executing object.
  */
-template <class Executor>
-class Topology {
+class MachineTopology {
+    friend const MachineTopology *get_machine_topology();
+
 private:
     /**
      * This struct holds the attributes for an object.
      */
-    struct topology_obj_info {
+    struct mach_topo_obj_info {
         /**
          * The hwloc object.
          */
@@ -109,24 +111,25 @@ class Topology {
          * For GPUs, this is their number in the numa
          */
         size_type physical_id;
+
+        /**
+         * The physical_id assigned to the object.
+         * For GPUs, this is their number in the numa
+         */
+        size_type memory_size;
     };
 
 public:
     /**
-     * Creates a new Topology object.
+     * Creates a new MachineTopology object.
      */
-    static std::unique_ptr<Topology> create()
+    static std::shared_ptr<MachineTopology> create()
     {
-        return std::unique_ptr<Topology>(new Topology());
+        return std::shared_ptr<MachineTopology>(new MachineTopology());
     }
 
-    /**
-     * Get the Topology object.
-     *
-     * @return  the topology object.
-     */
-    hwloc_topology *get_topology() const { return this->topo_.get(); }
-
+    MachineTopology(MachineTopology &) = delete;
+    MachineTopology(MachineTopology &&) = delete;
 
     /**
      * Bind the object associated with the id to a core.
@@ -147,7 +150,7 @@ class Topology {
      *
      * @param id  The id of the PU
      */
-    const topology_obj_info &get_pu(size_type id)
+    const mach_topo_obj_info &get_pu(size_type id)
     {
         GKO_ENSURE_IN_BOUNDS(id, pus_.size());
         return pus_[id];
@@ -159,7 +162,7 @@ class Topology {
      *
      * @param id  The id of the core
      */
-    const topology_obj_info &get_core(size_type id)
+    const mach_topo_obj_info &get_core(size_type id)
     {
         GKO_ENSURE_IN_BOUNDS(id, cores_.size());
         return cores_[id];
@@ -171,7 +174,7 @@ class Topology {
      *
      * @param id  The id of the gpu
      */
-    const topology_obj_info &get_gpu(size_type id)
+    const mach_topo_obj_info &get_gpu(size_type id)
     {
         GKO_ENSURE_IN_BOUNDS(id, gpus_.size());
         return gpus_[id];
@@ -209,10 +212,7 @@ class Topology {
     virtual void load_gpus() {}
 
 protected:
-    Topology(Topology &) = delete;
-    Topology(Topology &&) = delete;
-
-    Topology()
+    MachineTopology()
     {
 #if GKO_HAVE_HWLOC
 
@@ -236,7 +236,8 @@ class Topology {
     /**
      * A helper function that binds the object with an id.
      */
-    void hwloc_binding_helper(std::vector<topology_obj_info> &obj, size_type id)
+    void hwloc_binding_helper(std::vector<mach_topo_obj_info> &obj,
+                              size_type id)
     {
 #if GKO_HAVE_HWLOC
         auto bitmap = hwloc_bitmap_alloc();
@@ -279,14 +280,14 @@ class Topology {
     // The objects should be sorted by logical index since hwloc uses logical
     // index with these functions
     void load_objects(hwloc_obj_type_t type,
-                      std::vector<topology_obj_info> &vector)
+                      std::vector<mach_topo_obj_info> &vector)
     {
         unsigned nbcores = hwloc_get_nbobjs_by_type(topo_.get(), type);
         for (unsigned i = 0; i < nbcores; i++) {
             hwloc_obj_t obj = hwloc_get_obj_by_type(topo_.get(), type, i);
             vector.push_back(
-                topology_obj_info{obj, hwloc_bitmap_first(obj->nodeset),
-                                  obj->logical_index, obj->os_index});
+                mach_topo_obj_info{obj, hwloc_bitmap_first(obj->nodeset),
+                                   obj->logical_index, obj->os_index});
         }
     }
 
@@ -309,9 +310,10 @@ class Topology {
 
 
 private:
-    std::vector<topology_obj_info> gpus_;
-    std::vector<topology_obj_info> pus_;
-    std::vector<topology_obj_info> cores_;
+    std::vector<mach_topo_obj_info> gpus_;
+    std::vector<mach_topo_obj_info> pus_;
+    std::vector<mach_topo_obj_info> cores_;
+    std::vector<mach_topo_obj_info> pci_devices_;
     size_type num_numas_;
 
     template <typename T>
@@ -320,8 +322,7 @@ class Topology {
 };
 
 
-}  // namespace machine_config
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_MACHINE_CONFIG_HPP_
+#endif  // GKO_CORE_BASE_MACHINE_TOPOLOGY_HPP_
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index f2c8c27bb6d..1e8600d68a4 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -46,7 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
-#include <ginkgo/core/base/machine_config.hpp>
+#include <ginkgo/core/base/machine_topology.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/matrix_assembly_data.hpp>
 #include <ginkgo/core/base/matrix_data.hpp>

From be9e2f7962a3032b1d7c18fbbccd2c282d699e56 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Mon, 30 Nov 2020 00:12:12 +0100
Subject: [PATCH 0276/1008] Add HWLOC as a third_party dependency.

---
 CMakeLists.txt                                | 16 +++----
 cmake/DownloadNonCMakeCMakeLists.txt.in       | 14 ++++++
 ...ncies.cmake => autodetect_executors.cmake} |  8 ----
 cmake/package_helpers.cmake                   | 44 +++++++++++++++++++
 core/CMakeLists.txt                           |  3 +-
 core/device_hooks/hip_hooks.cpp               |  2 +-
 third_party/CMakeLists.txt                    |  2 +
 third_party/gflags/CMakeLists.txt             |  2 +-
 third_party/hwloc/CMakeLists.txt              | 34 ++++++++++++++
 9 files changed, 102 insertions(+), 23 deletions(-)
 create mode 100644 cmake/DownloadNonCMakeCMakeLists.txt.in
 rename cmake/{autodetect_library_dependencies.cmake => autodetect_executors.cmake} (85%)
 create mode 100644 third_party/hwloc/CMakeLists.txt

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 38e5b38b511..e99a655066a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -19,7 +19,7 @@ set(PROJECT_VERSION_TAG ${Ginkgo_VERSION_TAG})
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules/")
 # Determine which modules can be compiled
 include(cmake/hip_path.cmake)
-include(cmake/autodetect_library_dependencies.cmake)
+include(cmake/autodetect_executors.cmake)
 include(cmake/build_type_helpers.cmake)
 
 # Ginkgo configuration options
@@ -33,7 +33,6 @@ option(GINKGO_BUILD_DPCPP
     "Compile DPC++ kernels for Intel GPUs or other DPC++ enabled hardware" ${GINKGO_HAS_DPCPP})
 option(GINKGO_BUILD_CUDA "Compile kernels for NVIDIA GPUs" ${GINKGO_HAS_CUDA})
 option(GINKGO_BUILD_HIP "Compile kernels for AMD or NVIDIA GPUs" ${GINKGO_HAS_HIP})
-option(GINKGO_WITH_HWLOC "Build Ginkgo with HWLOC support" ${GINKGO_HAS_HWLOC})
 option(GINKGO_BUILD_DOC "Generate documentation" OFF)
 option(GINKGO_SKIP_DEPENDENCY_UPDATE
     "Do not update dependencies each time the project is rebuilt" ON)
@@ -145,15 +144,7 @@ find_package(PAPI OPTIONAL_COMPONENTS sde)
 if(PAPI_sde_FOUND)
     set(GINKGO_HAVE_PAPI_SDE 1)
 endif()
-
-# Find HWLOC if needed
-set(GINKGO_HAVE_HWLOC 0)
-if(GINKGO_WITH_HWLOC)
-  find_package(HWLOC 2.1 REQUIRED)
-  set(GKO_HWLOC_XMLFILE "${HWLOC_XMLFILE}")
-  message(STATUS "HWLOC XML file set to: ${GKO_HWLOC_XMLFILE}")
-  set(GINKGO_HAVE_HWLOC 1)
-endif()
+set(GINKGO_HAVE_HWLOC 1)
 
 set(GINKGO_HIP_PLATFORM_NVCC 0)
 set(GINKGO_HIP_PLATFORM_HCC 0)
@@ -206,6 +197,9 @@ include(cmake/package_helpers.cmake)
 ginkgo_find_package(GTest "GTest::GTest;GTest::Main" FALSE 1.8.1)
 ginkgo_find_package(gflags gflags FALSE 2.2.2)
 ginkgo_find_package(RapidJSON rapidjson TRUE 1.1.0)
+ginkgo_find_package(HWLOC hwloc FALSE 2.1)
+set(GKO_HWLOC_XMLFILE "${HWLOC_XMLFILE}")
+message(STATUS "HWLOC XML file set to: ${GKO_HWLOC_XMLFILE}")
 add_subdirectory(third_party)    # Third-party tools and libraries
 
 # Ginkgo core libraries
diff --git a/cmake/DownloadNonCMakeCMakeLists.txt.in b/cmake/DownloadNonCMakeCMakeLists.txt.in
new file mode 100644
index 00000000000..7f994a8fcf4
--- /dev/null
+++ b/cmake/DownloadNonCMakeCMakeLists.txt.in
@@ -0,0 +1,14 @@
+cmake_minimum_required(VERSION 3.9)
+project(${package_name})
+
+include(ExternalProject)
+ExternalProject_Add(${package_name}
+    URL "${package_url}"
+    URL_HASH "${package_hash}"
+    DOWNLOAD_NO_PROGRESS TRUE
+    SOURCE_DIR        "${CMAKE_CURRENT_BINARY_DIR}/src"
+    BINARY_DIR        "${CMAKE_CURRENT_BINARY_DIR}/build"
+    CONFIGURE_COMMAND "${config_command}"
+    INSTALL_COMMAND ""
+    UPDATE_DISCONNECTED ${GINKGO_SKIP_DEPENDENCY_UPDATE}
+    )
diff --git a/cmake/autodetect_library_dependencies.cmake b/cmake/autodetect_executors.cmake
similarity index 85%
rename from cmake/autodetect_library_dependencies.cmake
rename to cmake/autodetect_executors.cmake
index f2095599ea5..c448805fb2c 100644
--- a/cmake/autodetect_library_dependencies.cmake
+++ b/cmake/autodetect_executors.cmake
@@ -9,7 +9,6 @@ check_language(CUDA)
 try_compile(GKO_CAN_COMPILE_DPCPP ${PROJECT_BINARY_DIR}/dpcpp
     SOURCES ${PROJECT_SOURCE_DIR}/dpcpp/test_dpcpp.dp.cpp
     CXX_STANDARD 17)
-find_package(HWLOC 2.1)
 
 if(OpenMP_CXX_FOUND)
     if(NOT DEFINED GINKGO_BUILD_OMP)
@@ -18,13 +17,6 @@ if(OpenMP_CXX_FOUND)
     set(GINKGO_HAS_OMP ON)
 endif()
 
-if(HWLOC_FOUND)
-  if(NOT DEFINED GINKGO_WITH_HWLOC)
-    message(STATUS "Enabling HWLOC support")
-  endif()
-  set(GINKGO_HAS_HWLOC ON)
-endif()
-
 if(CMAKE_CUDA_COMPILER)
     if(NOT DEFINED GINKGO_BUILD_CUDA)
         message(STATUS "Enabling CUDA executor")
diff --git a/cmake/package_helpers.cmake b/cmake/package_helpers.cmake
index 74208b86e09..6bd65709a78 100644
--- a/cmake/package_helpers.cmake
+++ b/cmake/package_helpers.cmake
@@ -1,6 +1,9 @@
 set(PACKAGE_DOWNLOADER_SCRIPT
     "${CMAKE_CURRENT_LIST_DIR}/DownloadCMakeLists.txt.in")
 
+set(NON_CMAKE_PACKAGE_DOWNLOADER_SCRIPT
+    "${CMAKE_CURRENT_LIST_DIR}/DownloadNonCMakeCMakeLists.txt.in")
+
 function(ginkgo_load_git_package package_name package_url package_tag)
     set(GINKGO_THIRD_PARTY_BUILD_TYPE "Debug")
     if (CMAKE_BUILD_TYPE MATCHES "[Rr][Ee][Ll][Ee][Aa][Ss][Ee]")
@@ -43,6 +46,30 @@ function(ginkgo_load_git_package package_name package_url package_tag)
 endfunction()
 
 
+function(ginkgo_load_and_configure_package package_name package_url package_hash config_command)
+    set(GINKGO_THIRD_PARTY_BUILD_TYPE "Debug")
+    if (CMAKE_BUILD_TYPE MATCHES "[Rr][Ee][Ll][Ee][Aa][Ss][Ee]")
+        set(GINKGO_THIRD_PARTY_BUILD_TYPE "Release")
+    endif()
+    configure_file(${NON_CMAKE_PACKAGE_DOWNLOADER_SCRIPT}
+                   download/CMakeLists.txt)
+    execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
+        RESULT_VARIABLE result
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/download)
+    if(result)
+        message(FATAL_ERROR
+            "CMake step for ${package_name}/download failed: ${result}")
+    endif()
+    execute_process(COMMAND ${CMAKE_COMMAND} --build .
+      RESULT_VARIABLE result
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/download)
+    if(result)
+      message(FATAL_ERROR
+        "Build step for ${package_name}/download failed: ${result}")
+    endif()
+endfunction()
+
+
 #   Add external target to external project.
 #   Create a new target and declare it as `IMPORTED` for libraries or `INTERFACE`
 #       for header only projects.
@@ -182,3 +209,20 @@ macro(ginkgo_find_package package_name target_list header_only)
         endif()
     endif()
 endmacro(ginkgo_find_package)
+
+
+#   Download a file and verify the download
+#
+#   \param url Name of package to be found
+#   \param filename      For TPL packages, declare a new target for each library
+#   \param hash_type      For TPL packages, declare the tpl package as header only
+#   \param hash             Extra specifications for the package finder
+#
+function(ginkgo_download_file url filename hash_type hash)
+  if(NOT EXISTS ${filename})
+    file(DOWNLOAD ${url} ${filename}
+      TIMEOUT 60  # seconds
+      EXPECTED_HASH ${hash_type}=${hash}
+      TLS_VERIFY ON)
+  endif()
+endfunction(ginkgo_download_file)
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 6d4524b7c32..2ac359d2079 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -68,8 +68,7 @@ if (GINKGO_HAVE_PAPI_SDE)
     target_link_libraries(ginkgo PUBLIC PAPI::PAPI)
 endif()
 if (GINKGO_HAVE_HWLOC)
-  target_include_directories(ginkgo PUBLIC ${HWLOC_INCLUDE_DIRS})
-  target_link_libraries(ginkgo PUBLIC ${HWLOC_LIBRARIES})
+  target_link_libraries(ginkgo PUBLIC hwloc)
 endif()
 ginkgo_default_includes(ginkgo)
 ginkgo_install_library(ginkgo core)
diff --git a/core/device_hooks/hip_hooks.cpp b/core/device_hooks/hip_hooks.cpp
index 2616b9b847d..6c8659a6909 100644
--- a/core/device_hooks/hip_hooks.cpp
+++ b/core/device_hooks/hip_hooks.cpp
@@ -59,7 +59,7 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
 }
 
 
-void OmpExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+void HipExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
 
 
 void OmpExecutor::raw_copy_to(const HipExecutor *, size_type num_bytes,
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index 40618311a5f..9b4930c3015 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -13,6 +13,8 @@ if(GINKGO_BUILD_TESTS)
     ginkgo_add_subdirectory(GTest gtest)
 endif()
 
+ginkgo_add_subdirectory(hwloc hwloc)
+
 if(GINKGO_DEVEL_TOOLS)
     set(GCF_IGNORE_LIST "third_party" CACHE STRING "Ignore directories for GCF")
     add_subdirectory(git-cmake-format)
diff --git a/third_party/gflags/CMakeLists.txt b/third_party/gflags/CMakeLists.txt
index 6a2e76a2dbf..20c921241c3 100644
--- a/third_party/gflags/CMakeLists.txt
+++ b/third_party/gflags/CMakeLists.txt
@@ -26,7 +26,7 @@ else()
         )
 endif()
 if(WIN32)
-    # gflags uses gflags_nothreads_static not gflags_nothreads_static in Windows.
+    # gflags uses gflags_nothreads_static not gflags_nothreads in Windows.
     ginkgo_add_external_target(gflags gflags_nothreads_static build/include build/lib
         STATIC "_debug" gflags_external FALSE)
 else()
diff --git a/third_party/hwloc/CMakeLists.txt b/third_party/hwloc/CMakeLists.txt
new file mode 100644
index 00000000000..fe11bf6cc38
--- /dev/null
+++ b/third_party/hwloc/CMakeLists.txt
@@ -0,0 +1,34 @@
+message(STATUS "Configuring and building HWLOC")
+if(WIN32)
+  ginkgo_download_file("https://download.open-mpi.org/release/hwloc/v2.3/hwloc-win32-build-2.3.0.zip"
+    "${PROJECT_BINARY_DIR}/third_party/hwloc/hwloc.zip"
+    "SHA1"
+    "1d53dc8db809dd19d8281556f5f3785105aba97f"
+    )
+  file(ARCHIVE_EXTRACT INPUT "${PROJECT_BINARY_DIR}/third_party/hwloc/hwloc.zip")
+elseif(WIN64)
+  ginkgo_download_file("https://download.open-mpi.org/release/hwloc/v2.3/hwloc-win64-build-2.3.0.zip"
+    "${PROJECT_BINARY_DIR}/third_party/hwloc/hwloc.zip"
+    "SHA1"
+    "943e0ca50262dcf256b5956c73faded0141e06b9"
+    )
+  file(ARCHIVE_EXTRACT INPUT "${PROJECT_BINARY_DIR}/third_party/hwloc/hwloc.zip")
+else()
+ginkgo_load_and_configure_package(hwloc_external "https://download.open-mpi.org/release/hwloc/v2.3/hwloc-2.3.0.tar.gz"
+    "SHA1=69d8e2f088d365cf5e86d04f049c6b1e09e2b9c3"
+    "${CMAKE_BINARY_DIR}/third_party/hwloc/src/configure"
+    )
+
+endif()
+
+if(WIN32 OR WIN64)
+  add_library(hwloc STATIC IMPORTED GLOBAL)
+  add_dependencies(hwloc hwloc_external)
+  set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION "hwloc/lib/libhwloc.a" )
+  set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "hwloc/include")
+else()
+  add_library(hwloc SHARED IMPORTED GLOBAL)
+  add_dependencies(hwloc hwloc_external )
+  set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION "${CMAKE_BINARY_DIR}/third_party/hwloc/build/hwloc/.libs/libhwloc.so" )
+  set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}/third_party/hwloc/build/include")
+endif()

From 4cfbaf237a7e1bb07656b27dd3d10dc26f9967ef Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Mon, 30 Nov 2020 10:58:56 +0100
Subject: [PATCH 0277/1008] Allow to switch off HWLOC.

---
 CMakeLists.txt              | 10 +++++++++-
 cmake/package_helpers.cmake |  2 +-
 core/CMakeLists.txt         |  6 +++---
 third_party/CMakeLists.txt  |  4 +++-
 4 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e99a655066a..35932062147 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -72,6 +72,8 @@ set(GINKGO_HIP_AMDGPU "" CACHE STRING
     "The amdgpu_target(s) variable passed to hipcc. The default is none (auto).")
 option(GINKGO_JACOBI_FULL_OPTIMIZATIONS "Use all the optimizations for the CUDA Jacobi algorithm" OFF)
 option(BUILD_SHARED_LIBS "Build shared (.so, .dylib, .dll) libraries" ON)
+option(GINKGO_USE_EXTERNAL_HWLOC "Use external hwloc libray if available" ON)
+option(GINKGO_FORCE_NO_HWLOC "Forcibly switch off HWLOC dependency" OFF)
 
 set(GINKGO_CIRCULAR_DEPS_FLAGS "-Wl,--no-undefined")
 
@@ -144,8 +146,12 @@ find_package(PAPI OPTIONAL_COMPONENTS sde)
 if(PAPI_sde_FOUND)
     set(GINKGO_HAVE_PAPI_SDE 1)
 endif()
-set(GINKGO_HAVE_HWLOC 1)
 
+if(GINKGO_FORCE_NO_HWLOC)
+    set(GINKGO_HAVE_HWLOC 0)
+else()
+    set(GINKGO_HAVE_HWLOC 1)
+endif()
 set(GINKGO_HIP_PLATFORM_NVCC 0)
 set(GINKGO_HIP_PLATFORM_HCC 0)
 
@@ -197,7 +203,9 @@ include(cmake/package_helpers.cmake)
 ginkgo_find_package(GTest "GTest::GTest;GTest::Main" FALSE 1.8.1)
 ginkgo_find_package(gflags gflags FALSE 2.2.2)
 ginkgo_find_package(RapidJSON rapidjson TRUE 1.1.0)
+if(NOT GINKGO_FORCE_NO_HWLOC)
 ginkgo_find_package(HWLOC hwloc FALSE 2.1)
+endif()
 set(GKO_HWLOC_XMLFILE "${HWLOC_XMLFILE}")
 message(STATUS "HWLOC XML file set to: ${GKO_HWLOC_XMLFILE}")
 add_subdirectory(third_party)    # Third-party tools and libraries
diff --git a/cmake/package_helpers.cmake b/cmake/package_helpers.cmake
index 6bd65709a78..b812464ad07 100644
--- a/cmake/package_helpers.cmake
+++ b/cmake/package_helpers.cmake
@@ -200,7 +200,7 @@ macro(ginkgo_find_package package_name target_list header_only)
                     "${lib}" ${header_only})
              endforeach()
         else()
-            find_package(${package_name} QUIET ${ARGN})
+            find_package(${package_name} ${ARGN})
             if (${package_name}_FOUND)
                 message(STATUS "Using external version of package ${package_name}. In case of problems, consider setting -DGINKGO_USE_EXTERNAL_${_UPACKAGE_NAME}=OFF.")
             else()
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 2ac359d2079..be0164471e5 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -67,9 +67,9 @@ target_link_libraries(ginkgo
 if (GINKGO_HAVE_PAPI_SDE)
     target_link_libraries(ginkgo PUBLIC PAPI::PAPI)
 endif()
-if (GINKGO_HAVE_HWLOC)
-  target_link_libraries(ginkgo PUBLIC hwloc)
-endif()
+
+target_link_libraries(ginkgo PUBLIC hwloc)
+
 ginkgo_default_includes(ginkgo)
 ginkgo_install_library(ginkgo core)
 
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index 9b4930c3015..37671aaa77d 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -13,7 +13,9 @@ if(GINKGO_BUILD_TESTS)
     ginkgo_add_subdirectory(GTest gtest)
 endif()
 
-ginkgo_add_subdirectory(hwloc hwloc)
+if(NOT GINKGO_FORCE_NO_HWLOC)
+    ginkgo_add_subdirectory(HWLOC hwloc)
+endif()
 
 if(GINKGO_DEVEL_TOOLS)
     set(GCF_IGNORE_LIST "third_party" CACHE STRING "Ignore directories for GCF")

From 03adeedd4f446faf6c59ad52dd49a6bc3acd1a6d Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Mon, 30 Nov 2020 11:53:54 +0100
Subject: [PATCH 0278/1008] Fix Windows interaction

---
 core/test/base/executor.cpp        | 21 ---------------------
 cuda/test/base/cuda_executor.cu    | 24 ------------------------
 hip/test/base/hip_executor.hip.cpp | 25 -------------------------
 third_party/hwloc/CMakeLists.txt   |  4 ++--
 4 files changed, 2 insertions(+), 72 deletions(-)

diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index ceeafebb5ec..5ff74d2b68e 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -101,27 +101,6 @@ TEST(OmpExecutor, RunsCorrectLambdaOperation)
     ASSERT_EQ(1, value);
 }
 
-// #if GKO_HAVE_HWLOC
-
-
-// TEST(OmpExecutor, GetsExecInfo)
-// {
-//     int num_pus = 0;
-//     int num_cores = 0;
-//     int num_numas = 0;
-//     auto omp = gko::OmpExecutor::create();
-//     auto omp_info = omp->get_exec_info();
-//     num_pus = omp_info->get_num_pus();
-//     num_cores = omp_info->get_num_cores();
-//     num_numas = omp_info->get_num_numas();
-//     EXPECT_GT(num_numas, 0);
-//     EXPECT_GT(num_pus, 0);
-//     EXPECT_GT(num_cores, 0);
-// }
-
-
-// #endif
-
 
 TEST(OmpExecutor, AllocatesAndFreesMemory)
 {
diff --git a/cuda/test/base/cuda_executor.cu b/cuda/test/base/cuda_executor.cu
index 7c6bbe0ebfa..99db90458ee 100644
--- a/cuda/test/base/cuda_executor.cu
+++ b/cuda/test/base/cuda_executor.cu
@@ -131,30 +131,6 @@ TEST_F(CudaExecutor, MasterKnowsNumberOfDevices)
 }
 
 
-#if GKO_HAVE_HWLOC
-
-
-TEST_F(CudaExecutor, GetsExecInfo)
-{
-    int num_pus = 0;
-    int num_gpus = 0;
-    int num_cores = 0;
-    int num_numas = 0;
-    auto cuda = gko::CudaExecutor::create(0, omp);
-    auto cuda_info = cuda->get_exec_info();
-    num_pus = cuda_info->get_num_pus();
-    num_gpus = cuda_info->get_num_gpus();
-    num_cores = cuda_info->get_num_cores();
-    num_numas = cuda_info->get_num_numas();
-    EXPECT_GT(num_numas, 0);
-    EXPECT_GT(num_gpus, 0);
-    EXPECT_GT(num_pus, 0);
-    EXPECT_GT(num_cores, 0);
-}
-
-
-#endif
-
 TEST_F(CudaExecutor, AllocatesAndFreesMemory)
 {
     int *ptr = nullptr;
diff --git a/hip/test/base/hip_executor.hip.cpp b/hip/test/base/hip_executor.hip.cpp
index 1eb55530ec5..c4f4757e397 100644
--- a/hip/test/base/hip_executor.hip.cpp
+++ b/hip/test/base/hip_executor.hip.cpp
@@ -136,31 +136,6 @@ TEST_F(HipExecutor, MasterKnowsNumberOfDevices)
 }
 
 
-// #if GKO_HAVE_HWLOC
-
-
-// TEST_F(HipExecutor, GetsExecInfo)
-// {
-//     int num_pus = 0;
-//     int num_gpus = 0;
-//     int num_cores = 0;
-//     int num_numas = 0;
-//     auto hip = gko::HipExecutor::create(0, omp);
-//     auto hip_info = hip->get_exec_info();
-//     num_pus = hip_info->get_num_pus();
-//     num_gpus = hip_info->get_num_gpus();
-//     num_cores = hip_info->get_num_cores();
-//     num_numas = hip_info->get_num_numas();
-//     EXPECT_GT(num_numas, 0);
-//     EXPECT_GT(num_gpus, 0);
-//     EXPECT_GT(num_pus, 0);
-//     EXPECT_GT(num_cores, 0);
-// }
-
-
-// #endif
-
-
 TEST_F(HipExecutor, AllocatesAndFreesMemory)
 {
     int *ptr = nullptr;
diff --git a/third_party/hwloc/CMakeLists.txt b/third_party/hwloc/CMakeLists.txt
index fe11bf6cc38..a77af05a46e 100644
--- a/third_party/hwloc/CMakeLists.txt
+++ b/third_party/hwloc/CMakeLists.txt
@@ -24,8 +24,8 @@ endif()
 if(WIN32 OR WIN64)
   add_library(hwloc STATIC IMPORTED GLOBAL)
   add_dependencies(hwloc hwloc_external)
-  set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION "hwloc/lib/libhwloc.a" )
-  set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "hwloc/include")
+  set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION "${CMAKE_BINARY_DIR}/third_party/hwloc/hwloc/lib/libhwloc.a" )
+  set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}/third_party/hwloc/hwloc/include")
 else()
   add_library(hwloc SHARED IMPORTED GLOBAL)
   add_dependencies(hwloc hwloc_external )

From 79a74a46ab3402cd209202da5c948b2d9790eccd Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Mon, 30 Nov 2020 14:49:34 +0100
Subject: [PATCH 0279/1008] Create ginkgo_core_device obj to fix linking issue

---
 core/CMakeLists.txt                         | 1 -
 core/device_hooks/CMakeLists.txt            | 5 +++++
 core/devices/CMakeLists.txt                 | 2 ++
 core/devices/dpcpp/CMakeLists.txt           | 2 +-
 core/devices/hip/CMakeLists.txt             | 3 ++-
 core/{base => devices}/machine_topology.cpp | 0
 core/devices/omp/CMakeLists.txt             | 2 +-
 core/devices/reference/CMakeLists.txt       | 2 +-
 8 files changed, 12 insertions(+), 5 deletions(-)
 rename core/{base => devices}/machine_topology.cpp (100%)

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index be0164471e5..0147e678365 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -9,7 +9,6 @@ target_sources(ginkgo
     base/composition.cpp
     base/executor.cpp
     base/mtx_io.cpp
-    base/machine_topology.cpp
     base/perturbation.cpp
     base/version.cpp
     factorization/ic.cpp
diff --git a/core/device_hooks/CMakeLists.txt b/core/device_hooks/CMakeLists.txt
index 7e94cd6ab49..09629c89bbb 100644
--- a/core/device_hooks/CMakeLists.txt
+++ b/core/device_hooks/CMakeLists.txt
@@ -1,6 +1,7 @@
 if(NOT GINKGO_BUILD_CUDA)
     add_library(ginkgo_cuda
         $<TARGET_OBJECTS:ginkgo_cuda_device>
+        $<TARGET_OBJECTS:ginkgo_core_device>
         cuda_hooks.cpp)
     ginkgo_compile_features(ginkgo_cuda)
     ginkgo_default_includes(ginkgo_cuda)
@@ -10,6 +11,7 @@ endif()
 if (NOT GINKGO_BUILD_DPCPP)
     add_library(ginkgo_dpcpp
         $<TARGET_OBJECTS:ginkgo_dpcpp_device>
+        $<TARGET_OBJECTS:ginkgo_core_device>
         dpcpp_hooks.cpp)
     ginkgo_compile_features(ginkgo_dpcpp)
     ginkgo_default_includes(ginkgo_dpcpp)
@@ -19,6 +21,7 @@ endif()
 if(NOT GINKGO_BUILD_HIP)
     add_library(ginkgo_hip
         $<TARGET_OBJECTS:ginkgo_hip_device>
+        $<TARGET_OBJECTS:ginkgo_core_device>
         hip_hooks.cpp)
     ginkgo_compile_features(ginkgo_hip)
     ginkgo_default_includes(ginkgo_hip)
@@ -33,6 +36,7 @@ endif()
 if (NOT GINKGO_BUILD_OMP)
     add_library(ginkgo_omp
         $<TARGET_OBJECTS:ginkgo_omp_device>
+        $<TARGET_OBJECTS:ginkgo_core_device>
         omp_hooks.cpp)
     ginkgo_compile_features(ginkgo_omp)
     target_link_libraries(ginkgo_omp PUBLIC ginkgo_cuda)
@@ -45,6 +49,7 @@ endif()
 if (NOT GINKGO_BUILD_REFERENCE)
     add_library(ginkgo_reference
         $<TARGET_OBJECTS:ginkgo_reference_device>
+        $<TARGET_OBJECTS:ginkgo_core_device>
         reference_hooks.cpp)
     ginkgo_compile_features(ginkgo_reference)
     ginkgo_default_includes(ginkgo_reference)
diff --git a/core/devices/CMakeLists.txt b/core/devices/CMakeLists.txt
index adfd93beb03..3b544ce0500 100644
--- a/core/devices/CMakeLists.txt
+++ b/core/devices/CMakeLists.txt
@@ -6,6 +6,8 @@ function(ginkgo_add_object_library name)
     set_target_properties(${name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endfunction()
 
+ginkgo_add_object_library(ginkgo_core_device machine_topology.cpp)
+
 add_subdirectory(cuda)
 add_subdirectory(dpcpp)
 add_subdirectory(hip)
diff --git a/core/devices/dpcpp/CMakeLists.txt b/core/devices/dpcpp/CMakeLists.txt
index f6fdb354ff2..274c94fa1bc 100644
--- a/core/devices/dpcpp/CMakeLists.txt
+++ b/core/devices/dpcpp/CMakeLists.txt
@@ -1,2 +1,2 @@
 ginkgo_add_object_library(ginkgo_dpcpp_device
-    executor.cpp)
+  $<TARGET_OBJECTS:ginkgo_core_device> executor.cpp)
diff --git a/core/devices/hip/CMakeLists.txt b/core/devices/hip/CMakeLists.txt
index 7f855b3e2e9..b8a6f4973dd 100644
--- a/core/devices/hip/CMakeLists.txt
+++ b/core/devices/hip/CMakeLists.txt
@@ -1,3 +1,4 @@
 ginkgo_add_object_library(ginkgo_hip_device
-    executor.cpp)
+  $<TARGET_OBJECTS:ginkgo_core_device> executor.cpp)
+ 
 
diff --git a/core/base/machine_topology.cpp b/core/devices/machine_topology.cpp
similarity index 100%
rename from core/base/machine_topology.cpp
rename to core/devices/machine_topology.cpp
diff --git a/core/devices/omp/CMakeLists.txt b/core/devices/omp/CMakeLists.txt
index bf5ca4cfa88..fe65a123554 100644
--- a/core/devices/omp/CMakeLists.txt
+++ b/core/devices/omp/CMakeLists.txt
@@ -1,2 +1,2 @@
 ginkgo_add_object_library(ginkgo_omp_device
-    executor.cpp)
+  $<TARGET_OBJECTS:ginkgo_core_device> executor.cpp)
diff --git a/core/devices/reference/CMakeLists.txt b/core/devices/reference/CMakeLists.txt
index 2bb8cd7c79a..870b0c52aad 100644
--- a/core/devices/reference/CMakeLists.txt
+++ b/core/devices/reference/CMakeLists.txt
@@ -1,2 +1,2 @@
 ginkgo_add_object_library(ginkgo_reference_device
-    dummy.cpp)
+  $<TARGET_OBJECTS:ginkgo_core_device> dummy.cpp)

From 29ebe399667d76f4fb061c27ac47f36345fba54f Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Mon, 30 Nov 2020 22:25:27 +0100
Subject: [PATCH 0280/1008] Fix windows issues and disable for osx actions.

---
 .github/workflows/osx.yml                     |  5 +--
 CMakeLists.txt                                |  2 +-
 cmake/package_helpers.cmake                   | 15 +++++----
 core/CMakeLists.txt                           |  4 ++-
 .../CudaArchitectureSelector/CMakeLists.txt   |  1 +
 third_party/gflags/CMakeLists.txt             |  1 +
 third_party/git-cmake-format/CMakeLists.txt   |  1 +
 third_party/gtest/CMakeLists.txt              |  1 +
 third_party/hwloc/CMakeLists.txt              | 31 +++++++++----------
 third_party/rapidjson/CMakeLists.txt          |  1 +
 10 files changed, 36 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml
index 59935e07273..6516a54e6c6 100644
--- a/.github/workflows/osx.yml
+++ b/.github/workflows/osx.yml
@@ -16,7 +16,8 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: setup
-      run: brew install libomp
+      run: |
+        brew install libomp
     - name: info
       run: |
         g++ -v
@@ -25,7 +26,7 @@ jobs:
       run: |
         mkdir build
         cd build
-        cmake .. -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }}
+        cmake .. -DGINKGO_FORCE_NO_HWLOC=on -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }}
         make -j8
         ctest -j10 --output-on-failure
     - name: install
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 35932062147..a45d69fbbf0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -204,7 +204,7 @@ ginkgo_find_package(GTest "GTest::GTest;GTest::Main" FALSE 1.8.1)
 ginkgo_find_package(gflags gflags FALSE 2.2.2)
 ginkgo_find_package(RapidJSON rapidjson TRUE 1.1.0)
 if(NOT GINKGO_FORCE_NO_HWLOC)
-ginkgo_find_package(HWLOC hwloc FALSE 2.1)
+  ginkgo_find_package(HWLOC hwloc FALSE 2.1)
 endif()
 set(GKO_HWLOC_XMLFILE "${HWLOC_XMLFILE}")
 message(STATUS "HWLOC XML file set to: ${GKO_HWLOC_XMLFILE}")
diff --git a/cmake/package_helpers.cmake b/cmake/package_helpers.cmake
index b812464ad07..51441d40499 100644
--- a/cmake/package_helpers.cmake
+++ b/cmake/package_helpers.cmake
@@ -219,10 +219,13 @@ endmacro(ginkgo_find_package)
 #   \param hash             Extra specifications for the package finder
 #
 function(ginkgo_download_file url filename hash_type hash)
-  if(NOT EXISTS ${filename})
-    file(DOWNLOAD ${url} ${filename}
-      TIMEOUT 60  # seconds
-      EXPECTED_HASH ${hash_type}=${hash}
-      TLS_VERIFY ON)
-  endif()
+  file(DOWNLOAD ${url} ${filename}
+    TIMEOUT 60  # seconds
+    EXPECTED_HASH "${hash_type}=${hash}"
+    TLS_VERIFY ON)
+  if(EXISTS ${filename})
+    message(STATUS "${filename} downloaded from ${url}")
+  else()
+    message(FATAL_ERROR "Download of ${filename} failed.")
+  endif() 
 endfunction(ginkgo_download_file)
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 0147e678365..9fbb7a054dd 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -67,7 +67,9 @@ if (GINKGO_HAVE_PAPI_SDE)
     target_link_libraries(ginkgo PUBLIC PAPI::PAPI)
 endif()
 
-target_link_libraries(ginkgo PUBLIC hwloc)
+if(GINKGO_HAVE_HWLOC)
+  target_link_libraries(ginkgo PUBLIC hwloc)
+endif()
 
 ginkgo_default_includes(ginkgo)
 ginkgo_install_library(ginkgo core)
diff --git a/third_party/CudaArchitectureSelector/CMakeLists.txt b/third_party/CudaArchitectureSelector/CMakeLists.txt
index 03fdd72e485..e4d53a76a56 100644
--- a/third_party/CudaArchitectureSelector/CMakeLists.txt
+++ b/third_party/CudaArchitectureSelector/CMakeLists.txt
@@ -1,3 +1,4 @@
+message(STATUS "Configuring and building CAS")
 ginkgo_load_git_package(CudaArchitectureSelector
     "https://github.com/ginkgo-project/CudaArchitectureSelector.git"
     "cba91d58e9bb847e572fedf8cc18f701ae035410")
diff --git a/third_party/gflags/CMakeLists.txt b/third_party/gflags/CMakeLists.txt
index 20c921241c3..44618b89228 100644
--- a/third_party/gflags/CMakeLists.txt
+++ b/third_party/gflags/CMakeLists.txt
@@ -1,3 +1,4 @@
+message(STATUS "Configuring and building gflags")
 if(MSVC)
     # cmake links dynamic runtime libraries by default in Visual Studio
     # use the ginkgo's flags to use the same runtime libraries as ginkgo
diff --git a/third_party/git-cmake-format/CMakeLists.txt b/third_party/git-cmake-format/CMakeLists.txt
index b8e3d623050..ad709c14152 100644
--- a/third_party/git-cmake-format/CMakeLists.txt
+++ b/third_party/git-cmake-format/CMakeLists.txt
@@ -1,3 +1,4 @@
+message(STATUS "Configuring and building git-cmake-format")
 ginkgo_load_git_package(git-cmake-format
     "https://github.com/ginkgo-project/git-cmake-format.git"
     "29c23665d624e1cae1308bec651706fdaa8fe38b"
diff --git a/third_party/gtest/CMakeLists.txt b/third_party/gtest/CMakeLists.txt
index 9d13c970cc7..31da9c5eda3 100644
--- a/third_party/gtest/CMakeLists.txt
+++ b/third_party/gtest/CMakeLists.txt
@@ -1,3 +1,4 @@
+message(STATUS "Configuring and building gtest")
 if(MSVC)
     # Force using shared runtime library when MSVC builds shared libraries
     ginkgo_load_git_package(gtest_external
diff --git a/third_party/hwloc/CMakeLists.txt b/third_party/hwloc/CMakeLists.txt
index a77af05a46e..db9905de012 100644
--- a/third_party/hwloc/CMakeLists.txt
+++ b/third_party/hwloc/CMakeLists.txt
@@ -1,34 +1,33 @@
 message(STATUS "Configuring and building HWLOC")
-if(WIN32)
+set(TPL_HWLOC_PATH "${PROJECT_BINARY_DIR}/third_party/hwloc")
+if(MSVC OR WIN32)
+  message(STATUS "Downloading HWLOC binaries for win32")
   ginkgo_download_file("https://download.open-mpi.org/release/hwloc/v2.3/hwloc-win32-build-2.3.0.zip"
-    "${PROJECT_BINARY_DIR}/third_party/hwloc/hwloc.zip"
+    "${TPL_HWLOC_PATH}/hwloc.zip"
     "SHA1"
     "1d53dc8db809dd19d8281556f5f3785105aba97f"
     )
-  file(ARCHIVE_EXTRACT INPUT "${PROJECT_BINARY_DIR}/third_party/hwloc/hwloc.zip")
-elseif(WIN64)
-  ginkgo_download_file("https://download.open-mpi.org/release/hwloc/v2.3/hwloc-win64-build-2.3.0.zip"
-    "${PROJECT_BINARY_DIR}/third_party/hwloc/hwloc.zip"
-    "SHA1"
-    "943e0ca50262dcf256b5956c73faded0141e06b9"
+  file(ARCHIVE_EXTRACT INPUT "${TPL_HWLOC_PATH}/hwloc.zip"
+    DESTINATION "${TPL_HWLOC_PATH}/hwloc"
+    VERBOSE
     )
-  file(ARCHIVE_EXTRACT INPUT "${PROJECT_BINARY_DIR}/third_party/hwloc/hwloc.zip")
 else()
-ginkgo_load_and_configure_package(hwloc_external "https://download.open-mpi.org/release/hwloc/v2.3/hwloc-2.3.0.tar.gz"
+  message(STATUS "Downloading HWLOC tars and building")
+  ginkgo_load_and_configure_package(hwloc_external "https://download.open-mpi.org/release/hwloc/v2.3/hwloc-2.3.0.tar.gz"
     "SHA1=69d8e2f088d365cf5e86d04f049c6b1e09e2b9c3"
-    "${CMAKE_BINARY_DIR}/third_party/hwloc/src/configure"
+    "${TPL_HWLOC_PATH}/src/configure"
     )
 
 endif()
 
-if(WIN32 OR WIN64)
+if(MSVC OR WIN32)
   add_library(hwloc STATIC IMPORTED GLOBAL)
   add_dependencies(hwloc hwloc_external)
-  set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION "${CMAKE_BINARY_DIR}/third_party/hwloc/hwloc/lib/libhwloc.a" )
-  set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}/third_party/hwloc/hwloc/include")
+  set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION "${TPL_HWLOC_PATH}/hwloc/hwloc-win32-build-2.3.0/lib/libhwloc.a" )
+  set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${TPL_HWLOC_PATH}/hwloc/hwloc-win32-build-2.3.0/include")
 else()
   add_library(hwloc SHARED IMPORTED GLOBAL)
   add_dependencies(hwloc hwloc_external )
-  set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION "${CMAKE_BINARY_DIR}/third_party/hwloc/build/hwloc/.libs/libhwloc.so" )
-  set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_BINARY_DIR}/third_party/hwloc/build/include")
+  set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION "${TPL_HWLOC_PATH}/build/hwloc/.libs/libhwloc.so" )
+  set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${TPL_HWLOC_PATH}/build/include")
 endif()
diff --git a/third_party/rapidjson/CMakeLists.txt b/third_party/rapidjson/CMakeLists.txt
index bd2ae7899b1..58118263a18 100644
--- a/third_party/rapidjson/CMakeLists.txt
+++ b/third_party/rapidjson/CMakeLists.txt
@@ -1,3 +1,4 @@
+message(STATUS "Configuring and building rapidjson")
 ginkgo_load_git_package(rapidjson_external
     "https://github.com/Tencent/rapidjson.git"
     "88bd956d66d348f478bceebfdadb8e26c6844695"

From 96428b5980843768362ef493c1fb5a61526bf2be Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Tue, 1 Dec 2020 08:19:03 +0000
Subject: [PATCH 0281/1008] Format files

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 core/devices/machine_topology.cpp             | 7 +++----
 include/ginkgo/core/base/machine_topology.hpp | 6 +++---
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index b424f6f0dcb..cf2cb27d313 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -30,15 +30,14 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-
-#include <ginkgo/core/base/machine_topology.hpp>
-
-
 #include <atomic>
 #include <memory>
 #include <mutex>
 
 
+#include <ginkgo/core/base/machine_topology.hpp>
+
+
 namespace gko {
 
 
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index 736cfbfb517..0c370d71222 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_MACHINE_TOPOLOGY_HPP_
-#define GKO_CORE_BASE_MACHINE_TOPOLOGY_HPP_
+#ifndef GKO_PUBLIC_CORE_BASE_MACHINE_TOPOLOGY_HPP_
+#define GKO_PUBLIC_CORE_BASE_MACHINE_TOPOLOGY_HPP_
 
 
 #include <cassert>
@@ -325,4 +325,4 @@ class MachineTopology {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_MACHINE_TOPOLOGY_HPP_
+#endif  // GKO_PUBLIC_CORE_BASE_MACHINE_TOPOLOGY_HPP_

From ebe750ecb0dfc833a52fde1029f2a4c2bd87f992 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 1 Dec 2020 15:25:04 +0100
Subject: [PATCH 0282/1008] Revert core_device obj libs

---
 core/devices/cuda/CMakeLists.txt      | 2 +-
 core/devices/dpcpp/CMakeLists.txt     | 2 +-
 core/devices/hip/CMakeLists.txt       | 2 +-
 core/devices/omp/CMakeLists.txt       | 2 +-
 core/devices/reference/CMakeLists.txt | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/core/devices/cuda/CMakeLists.txt b/core/devices/cuda/CMakeLists.txt
index 1b164165305..17fad40ae98 100644
--- a/core/devices/cuda/CMakeLists.txt
+++ b/core/devices/cuda/CMakeLists.txt
@@ -1,3 +1,3 @@
 ginkgo_add_object_library(ginkgo_cuda_device
-    executor.cpp)
+  executor.cpp)
 
diff --git a/core/devices/dpcpp/CMakeLists.txt b/core/devices/dpcpp/CMakeLists.txt
index 274c94fa1bc..45200f64072 100644
--- a/core/devices/dpcpp/CMakeLists.txt
+++ b/core/devices/dpcpp/CMakeLists.txt
@@ -1,2 +1,2 @@
 ginkgo_add_object_library(ginkgo_dpcpp_device
-  $<TARGET_OBJECTS:ginkgo_core_device> executor.cpp)
+  executor.cpp)
diff --git a/core/devices/hip/CMakeLists.txt b/core/devices/hip/CMakeLists.txt
index b8a6f4973dd..5900ce69950 100644
--- a/core/devices/hip/CMakeLists.txt
+++ b/core/devices/hip/CMakeLists.txt
@@ -1,4 +1,4 @@
 ginkgo_add_object_library(ginkgo_hip_device
-  $<TARGET_OBJECTS:ginkgo_core_device> executor.cpp)
+  executor.cpp)
  
 
diff --git a/core/devices/omp/CMakeLists.txt b/core/devices/omp/CMakeLists.txt
index fe65a123554..5ebb37c4799 100644
--- a/core/devices/omp/CMakeLists.txt
+++ b/core/devices/omp/CMakeLists.txt
@@ -1,2 +1,2 @@
 ginkgo_add_object_library(ginkgo_omp_device
-  $<TARGET_OBJECTS:ginkgo_core_device> executor.cpp)
+  executor.cpp)
diff --git a/core/devices/reference/CMakeLists.txt b/core/devices/reference/CMakeLists.txt
index 870b0c52aad..485eb53e22d 100644
--- a/core/devices/reference/CMakeLists.txt
+++ b/core/devices/reference/CMakeLists.txt
@@ -1,2 +1,2 @@
 ginkgo_add_object_library(ginkgo_reference_device
-  $<TARGET_OBJECTS:ginkgo_core_device> dummy.cpp)
+  dummy.cpp)

From 781179e0dab5c48d005f7470c1d9934d2fbd28c6 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 3 Dec 2020 17:16:00 +0100
Subject: [PATCH 0283/1008] Update find_package and used shared_ptr for Topo

---
 cmake/Modules/FindHWLOC.cmake                 | 12 +++----
 core/device_hooks/cuda_hooks.cpp              |  4 ++-
 core/device_hooks/dpcpp_hooks.cpp             |  4 ++-
 core/device_hooks/hip_hooks.cpp               |  4 ++-
 core/devices/omp/executor.cpp                 |  4 ++-
 core/test/base/executor.cpp                   |  8 +++++
 cuda/base/executor.cpp                        |  4 ++-
 dpcpp/base/executor.dp.cpp                    |  4 ++-
 hip/CMakeLists.txt                            |  8 ++---
 hip/base/executor.hip.cpp                     |  4 ++-
 include/ginkgo/core/base/executor.hpp         | 29 ++++++++++++-----
 include/ginkgo/core/base/machine_topology.hpp | 32 +++++++------------
 12 files changed, 70 insertions(+), 47 deletions(-)

diff --git a/cmake/Modules/FindHWLOC.cmake b/cmake/Modules/FindHWLOC.cmake
index a36a47518ec..c7325ab64f5 100644
--- a/cmake/Modules/FindHWLOC.cmake
+++ b/cmake/Modules/FindHWLOC.cmake
@@ -33,7 +33,7 @@
 # To help us fund GROMACS development, we humbly ask that you cite
 # the research papers on the package. Check out http://www.gromacs.org.
 
-# FindHwloc
+# FindHWLOC
 #
 # - Locate headers and libraries for the Portable Hardware Locality (hwloc)
 #   library.
@@ -41,7 +41,7 @@
 # Usage: find_package(HWLOC VERSION)
 #        find_package(HWLOC VERSION REQUIRED)
 #
-# FindHwloc defines the following variables:
+# FindHWLOC defines the following variables:
 #
 # HWLOC_FOUND          - True if both headers and libraries were located
 # HWLOC_INCLUDE_DIRS   - Where to find hwloc headers
@@ -103,13 +103,13 @@ if(HWLOC_INCLUDE_DIRS)
         endif()
     endif()
 
-    if (NOT Hwloc_FIND_QUIETLY)
+    if (NOT HWLOC_FIND_QUIETLY)
         message(STATUS "hwloc version: ${HWLOC_VERSION}")
     endif()
 
     # Parse header if cross-compiling, or if hwloc-info was not found
     if(NOT HWLOC_VERSION)
-        # Hwloc is never installed as a framework on OS X, so this should always work.
+        # HWLOC is never installed as a framework on OS X, so this should always work.
         file(READ "${HWLOC_INCLUDE_DIRS}/hwloc.h"
              HEADER_CONTENTS LIMIT 16384)
         string(REGEX REPLACE ".*#define HWLOC_API_VERSION (0[xX][0-9a-fA-F]+).*" "\\1"
@@ -120,7 +120,7 @@ if(HWLOC_INCLUDE_DIRS)
         hex2dec(${HEX_MAJOR} DEC_MAJOR)
         hex2dec(${HEX_MINOR} DEC_MINOR)
         hex2dec(${HEX_PATCH} DEC_PATCH)
-        set(HWLOC_VERSION "${DEC_MAJOR}.${DEC_MINOR}.${DEC_PATCH}" CACHE STRING "Hwloc library version")
+        set(HWLOC_VERSION "${DEC_MAJOR}.${DEC_MINOR}.${DEC_PATCH}" CACHE STRING "HWLOC library version")
     endif()
 endif()
 
@@ -132,7 +132,7 @@ if(HWLOC_VERSION LESS_EQUAL HWLOC_FIND_VERSION)
     unset(HWLOC_INCLUDE_DIRS)
 else()
     include(FindPackageHandleStandardArgs)
-    find_package_handle_standard_args(Hwloc
+    find_package_handle_standard_args(HWLOC
         REQUIRED_VARS HWLOC_LIBRARIES HWLOC_INCLUDE_DIRS
         VERSION_VAR HWLOC_VERSION)
     mark_as_advanced(HWLOC_INCLUDE_DIRS HWLOC_LIBRARIES HWLOC_VERSION)
diff --git a/core/device_hooks/cuda_hooks.cpp b/core/device_hooks/cuda_hooks.cpp
index 3c88599f897..0221d8c935c 100644
--- a/core/device_hooks/cuda_hooks.cpp
+++ b/core/device_hooks/cuda_hooks.cpp
@@ -62,7 +62,9 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
 }
 
 
-void CudaExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+void CudaExecutor::populate_exec_info(
+    std::shared_ptr<const MachineTopology> mach_topo)
+{}
 
 
 void OmpExecutor::raw_copy_to(const CudaExecutor *, size_type num_bytes,
diff --git a/core/device_hooks/dpcpp_hooks.cpp b/core/device_hooks/dpcpp_hooks.cpp
index e336ee7340f..9085beb99d2 100644
--- a/core/device_hooks/dpcpp_hooks.cpp
+++ b/core/device_hooks/dpcpp_hooks.cpp
@@ -59,7 +59,9 @@ std::shared_ptr<DpcppExecutor> DpcppExecutor::create(
 }
 
 
-void DpcppExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+void DpcppExecutor::populate_exec_info(
+    std::shared_ptr<const MachineTopology> mach_topo)
+{}
 
 
 void OmpExecutor::raw_copy_to(const DpcppExecutor *, size_type num_bytes,
diff --git a/core/device_hooks/hip_hooks.cpp b/core/device_hooks/hip_hooks.cpp
index 6c8659a6909..dc30a4da947 100644
--- a/core/device_hooks/hip_hooks.cpp
+++ b/core/device_hooks/hip_hooks.cpp
@@ -59,7 +59,9 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
 }
 
 
-void HipExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+void HipExecutor::populate_exec_info(
+    std::shared_ptr<const MachineTopology> mach_topo)
+{}
 
 
 void OmpExecutor::raw_copy_to(const HipExecutor *, size_type num_bytes,
diff --git a/core/devices/omp/executor.cpp b/core/devices/omp/executor.cpp
index ba9a7e768cc..fd7f9f14375 100644
--- a/core/devices/omp/executor.cpp
+++ b/core/devices/omp/executor.cpp
@@ -44,7 +44,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace gko {
 
 
-void OmpExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+void OmpExecutor::populate_exec_info(
+    std::shared_ptr<const MachineTopology> mach_topo)
+{}
 
 
 void OmpExecutor::raw_free(void *ptr) const noexcept { std::free(ptr); }
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 5ff74d2b68e..03c71141a30 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -156,6 +156,14 @@ TEST(OmpExecutor, IsItsOwnMaster)
 }
 
 
+// TEST(OmpExecutor, IsItsOwnMaster)
+// {
+//     exec_ptr omp = gko::OmpExecutor::create();
+
+//     ASSERT_EQ(omp, omp->get_master());
+// }
+
+
 TEST(ReferenceExecutor, RunsCorrectOperation)
 {
     int value = 0;
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index a4cfa396d0d..f4bc9f42b9a 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -111,7 +111,9 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
 }
 
 
-void CudaExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+void CudaExecutor::populate_exec_info(
+    std::shared_ptr<const MachineTopology> mach_topo)
+{}
 
 
 void OmpExecutor::raw_copy_to(const CudaExecutor *dest, size_type num_bytes,
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 38abcd359d7..1363f193160 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -93,7 +93,9 @@ std::shared_ptr<DpcppExecutor> DpcppExecutor::create(
 }
 
 
-void DpcppExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+void DpcppExecutor::populate_exec_info(
+    std::shared_ptr<const MachineTopology> mach_topo)
+{}
 
 
 void DpcppExecutor::raw_free(void *ptr) const noexcept
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 25e51e0bc2b..5944595cfa6 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -289,7 +289,8 @@ if(GINKGO_HIP_PLATFORM MATCHES "hcc")
         # directly use HIP
         ginkgo_hip_clang_ban_hip_device_flags()
     endif()
-    target_link_libraries(ginkgo_hip PRIVATE hip::device)
+    target_link_libraries(ginkgo_hip PRIVATE hip::device
+      )
 elseif(GINKGO_HIP_PLATFORM MATCHES "nvcc")
     find_package(CUDA 9.2 REQUIRED)
     target_link_libraries(ginkgo_hip PUBLIC ${CUDA_LIBRARIES})
@@ -316,11 +317,6 @@ if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo_hip)
 endif()
 
-if (GINKGO_HAVE_HWLOC)
-    target_include_directories(ginkgo_hip PUBLIC ${HWLOC_INCLUDE_DIRS})
-    target_link_libraries(ginkgo_hip PUBLIC ${HWLOC_LIBRARIES})
-endif()
-
 if(GINKGO_BUILD_TESTS)
   # Here, we go through all of Ginkgo's dependencies to build a `-Wl,-rpath` string since for
   # some reason `hipcc` through CMake does not have rpath settings unlike the other compilers.
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 654d500a25b..1f75cf144a5 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -116,7 +116,9 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
 }
 
 
-void HipExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+void HipExecutor::populate_exec_info(
+    std::shared_ptr<const MachineTopology> mach_topo)
+{}
 
 
 void OmpExecutor::raw_copy_to(const HipExecutor *dest, size_type num_bytes,
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 701a7ad9384..12b706d9573 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -731,12 +731,14 @@ class Executor : public log::EnableLogging<Executor> {
      *
      * @param mach_topo the machine topology object.
      */
-    virtual void populate_exec_info(const MachineTopology *mach_topo) = 0;
+    virtual void populate_exec_info(
+        std::shared_ptr<const MachineTopology> mach_topo) = 0;
 
     struct exec_info {
         int device_id = -1;
         int num_cores = 0;
         int num_work_groups_per_core = 0;
+        std::string pci_bus_id = "invalid";
     };
 
 private:
@@ -988,10 +990,18 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
 
     void synchronize() const override;
 
+    const int get_num_cores() const { return this->omp_exec_info_.num_cores; }
+
+    const int get_num_threads_per_core() const
+    {
+        return this->omp_exec_info_.num_work_groups_per_core;
+    }
+
 protected:
     OmpExecutor() = default;
 
-    void populate_exec_info(const MachineTopology *mach_topo) override;
+    void populate_exec_info(
+        std::shared_ptr<const MachineTopology> mach_topo) override;
 
     void *raw_alloc(size_type size) const override;
 
@@ -1046,7 +1056,7 @@ class ReferenceExecutor : public OmpExecutor {
 protected:
     ReferenceExecutor() = default;
 
-    void populate_exec_info(const MachineTopology *) override
+    void populate_exec_info(std::shared_ptr<const MachineTopology>) override
     {
         this->ref_exec_info_.device_id = -1;
         this->ref_exec_info_.num_cores = 1;
@@ -1209,7 +1219,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         this->set_gpu_property();
         this->init_handles();
         increase_num_execs(this->cuda_exec_info_.device_id);
-        this->populate_exec_info(get_machine_topology());
+        this->populate_exec_info(get_shared_machine_topology());
     }
 
     void *raw_alloc(size_type size) const override;
@@ -1246,7 +1256,8 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         return num_execs[device_id];
     }
 
-    void populate_exec_info(const MachineTopology *mach_topo) override;
+    void populate_exec_info(
+        std::shared_ptr<const MachineTopology> mach_topo) override;
 
 private:
     std::shared_ptr<Executor> master_;
@@ -1403,7 +1414,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         this->set_gpu_property();
         this->init_handles();
         increase_num_execs(this->hip_exec_info_.device_id);
-        this->populate_exec_info(get_machine_topology());
+        // this->populate_exec_info(get_shared_machine_topology());
     }
 
     void *raw_alloc(size_type size) const override;
@@ -1440,7 +1451,8 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         return num_execs[device_id];
     }
 
-    void populate_exec_info(const MachineTopology *mach_topo) override;
+    void populate_exec_info(
+        std::shared_ptr<const MachineTopology> mach_topo) override;
 
 private:
     std::shared_ptr<Executor> master_;
@@ -1579,7 +1591,8 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
         this->set_device_property();
     }
 
-    void populate_exec_info(const MachineTopology *mach_topo) override;
+    void populate_exec_info(
+        std::shared_ptr<const MachineTopology> mach_topo) override;
 
     void *raw_alloc(size_type size) const override;
 
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index 0c370d71222..53e828bc29d 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -76,8 +76,8 @@ extern std::shared_ptr<const MachineTopology> get_shared_machine_topology();
 
 /**
  * The machine topology class represents the heirarchical topology of a machine,
- * including NUMA nodes, cores and GPUs. Various infomation of the machine are
- * gathered with the help of the Hardware Locality library (hwloc).
+ * including NUMA nodes, cores and PCI Devices. Various infomation of the
+ * machine are gathered with the help of the Hardware Locality library (hwloc).
  *
  * This class also provides functionalities to bind objects in the topology to
  * the execution objects. Binding can enhance performance by allowing data to be
@@ -108,13 +108,11 @@ class MachineTopology {
 
         /**
          * The physical_id assigned to the object.
-         * For GPUs, this is their number in the numa
          */
         size_type physical_id;
 
         /**
          * The physical_id assigned to the object.
-         * For GPUs, this is their number in the numa
          */
         size_type memory_size;
     };
@@ -170,14 +168,14 @@ class MachineTopology {
 
 
     /**
-     * Get the object of type gpu associated with the id.
+     * Get the object of type pci device associated with the id.
      *
-     * @param id  The id of the gpu
+     * @param id  The id of the pci device
      */
-    const mach_topo_obj_info &get_gpu(size_type id)
+    const mach_topo_obj_info &get_pci_device(size_type id)
     {
-        GKO_ENSURE_IN_BOUNDS(id, gpus_.size());
-        return gpus_[id];
+        GKO_ENSURE_IN_BOUNDS(id, pci_devices_.size());
+        return pci_devices_[id];
     }
 
 
@@ -194,9 +192,9 @@ class MachineTopology {
 
 
     /**
-     * Get the number of GPU objects stored in this Topology tree.
+     * Get the number of PCI device objects stored in this Topology tree.
      */
-    size_type get_num_gpus() { return gpus_.size(); }
+    size_type get_num_pci_devices() { return pci_devices_.size(); }
 
 
     /**
@@ -204,13 +202,6 @@ class MachineTopology {
      */
     size_type get_num_numas() { return num_numas_; }
 
-
-    /**
-     * Load the gpu objects. These functions are implemened by the respective
-     * GPU Executor classes.
-     */
-    virtual void load_gpus() {}
-
 protected:
     MachineTopology()
     {
@@ -222,7 +213,7 @@ class MachineTopology {
 
         load_objects(HWLOC_OBJ_CORE, this->cores_);
         load_objects(HWLOC_OBJ_PU, this->pus_);
-        this->load_gpus();
+        // load_objects(HWLOC_OBJ_PCI_DEVICE, this->pci_devices_);
 
         num_numas_ = hwloc_get_nbobjs_by_type(topo_.get(), HWLOC_OBJ_PACKAGE);
 #else
@@ -253,6 +244,8 @@ class MachineTopology {
 
 
     /**
+     * @internal
+     *
      * A helper function that prints the topology tree of an object to a given
      * depth. Provided from the hwloc library.
      */
@@ -310,7 +303,6 @@ class MachineTopology {
 
 
 private:
-    std::vector<mach_topo_obj_info> gpus_;
     std::vector<mach_topo_obj_info> pus_;
     std::vector<mach_topo_obj_info> cores_;
     std::vector<mach_topo_obj_info> pci_devices_;

From 10692de8d47df522b1b5ddf9080e428723543a18 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Fri, 4 Dec 2020 18:39:45 +0100
Subject: [PATCH 0284/1008] Fix find_package(HWLOC) and variable paths.

+ Fix link issues and cleanup paths
+ Disable HWLOC for Windows and MacOS by default
---
 .github/workflows/osx.yml                     |  2 +-
 CMakeLists.txt                                |  7 ++++++-
 cmake/Modules/FindHWLOC.cmake                 |  4 ++++
 cmake/build_helpers.cmake                     |  6 ++++++
 core/device_hooks/CMakeLists.txt              |  5 -----
 core/devices/CMakeLists.txt                   |  6 ++++++
 core/devices/machine_topology.cpp             | 12 +++++-------
 cuda/CMakeLists.txt                           |  2 +-
 dpcpp/CMakeLists.txt                          |  2 +-
 hip/CMakeLists.txt                            |  4 ++--
 include/ginkgo/core/base/executor.hpp         |  6 +++---
 include/ginkgo/core/base/machine_topology.hpp |  6 ++++++
 include/ginkgo/core/matrix/diagonal.hpp       |  1 +
 omp/CMakeLists.txt                            |  2 +-
 reference/CMakeLists.txt                      |  2 +-
 third_party/CMakeLists.txt                    |  2 +-
 third_party/hwloc/CMakeLists.txt              | 15 ++++++++++-----
 17 files changed, 55 insertions(+), 29 deletions(-)

diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml
index 6516a54e6c6..de5c9d36c7d 100644
--- a/.github/workflows/osx.yml
+++ b/.github/workflows/osx.yml
@@ -26,7 +26,7 @@ jobs:
       run: |
         mkdir build
         cd build
-        cmake .. -DGINKGO_FORCE_NO_HWLOC=on -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }}
+        cmake .. -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }}
         make -j8
         ctest -j10 --output-on-failure
     - name: install
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a45d69fbbf0..da16b2b1ec1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -152,6 +152,11 @@ if(GINKGO_FORCE_NO_HWLOC)
 else()
     set(GINKGO_HAVE_HWLOC 1)
 endif()
+# Switch off HWLOC for Windows and MacOS
+if(MSVC OR WIN32 OR CYGWIN OR APPLE)
+    set(GINKGO_HAVE_HWLOC 0)
+endif()
+
 set(GINKGO_HIP_PLATFORM_NVCC 0)
 set(GINKGO_HIP_PLATFORM_HCC 0)
 
@@ -203,7 +208,7 @@ include(cmake/package_helpers.cmake)
 ginkgo_find_package(GTest "GTest::GTest;GTest::Main" FALSE 1.8.1)
 ginkgo_find_package(gflags gflags FALSE 2.2.2)
 ginkgo_find_package(RapidJSON rapidjson TRUE 1.1.0)
-if(NOT GINKGO_FORCE_NO_HWLOC)
+if(NOT GINKGO_FORCE_NO_HWLOC AND GINKGO_HAVE_HWLOC)
   ginkgo_find_package(HWLOC hwloc FALSE 2.1)
 endif()
 set(GKO_HWLOC_XMLFILE "${HWLOC_XMLFILE}")
diff --git a/cmake/Modules/FindHWLOC.cmake b/cmake/Modules/FindHWLOC.cmake
index c7325ab64f5..519a73536ca 100644
--- a/cmake/Modules/FindHWLOC.cmake
+++ b/cmake/Modules/FindHWLOC.cmake
@@ -137,6 +137,10 @@ else()
         VERSION_VAR HWLOC_VERSION)
     mark_as_advanced(HWLOC_INCLUDE_DIRS HWLOC_LIBRARIES HWLOC_VERSION)
 endif()
+if(NOT HWLOC_FOUND)
+  unset(HWLOC_LIBRARIES)
+  unset(HWLOC_INCLUDE_DIRS)
+endif()
 
 if(HWLOC_FIND_REQUIRED AND NOT HWLOC_FOUND)
     message(SEND_ERROR "HWLOC could not be found. A version mismatch could have occured. The version found was ${HWLOC_VERSION}")
diff --git a/cmake/build_helpers.cmake b/cmake/build_helpers.cmake
index 78c3d810021..34689ecefc5 100644
--- a/cmake/build_helpers.cmake
+++ b/cmake/build_helpers.cmake
@@ -7,6 +7,12 @@ function(ginkgo_default_includes name)
             $<BUILD_INTERFACE:${Ginkgo_SOURCE_DIR}>
             $<INSTALL_INTERFACE:include>
         )
+    if(GINKGO_HAVE_HWLOC)
+      target_include_directories("${name}"
+        PUBLIC 
+        $<BUILD_INTERFACE:${HWLOC_INCLUDE_DIRS}>
+        )
+    endif()
 endfunction()
 
 function(ginkgo_compile_features name)
diff --git a/core/device_hooks/CMakeLists.txt b/core/device_hooks/CMakeLists.txt
index 09629c89bbb..a677f0fcbcf 100644
--- a/core/device_hooks/CMakeLists.txt
+++ b/core/device_hooks/CMakeLists.txt
@@ -26,11 +26,6 @@ if(NOT GINKGO_BUILD_HIP)
     ginkgo_compile_features(ginkgo_hip)
     ginkgo_default_includes(ginkgo_hip)
     ginkgo_install_library(ginkgo_hip hip)
-
-    if (GINKGO_HAVE_HWLOC)
-        target_include_directories(ginkgo_hip PUBLIC ${HWLOC_INCLUDE_DIRS})
-        target_link_libraries(ginkgo_hip PUBLIC ${HWLOC_LIBRARIES})
-    endif()
 endif()
 
 if (NOT GINKGO_BUILD_OMP)
diff --git a/core/devices/CMakeLists.txt b/core/devices/CMakeLists.txt
index 3b544ce0500..26365167654 100644
--- a/core/devices/CMakeLists.txt
+++ b/core/devices/CMakeLists.txt
@@ -3,6 +3,12 @@ function(ginkgo_add_object_library name)
     ginkgo_compile_features(${name})
     ginkgo_default_includes(${name})
     target_include_directories(${name} PUBLIC ${Ginkgo_SOURCE_DIR})
+    if(GINKGO_HAVE_HWLOC)
+      target_include_directories(${name}
+        PUBLIC 
+        $<BUILD_INTERFACE:${HWLOC_INCLUDE_DIRS}>
+        )
+    endif()
     set_target_properties(${name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endfunction()
 
diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index cf2cb27d313..4d798017076 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -54,13 +54,11 @@ std::atomic<bool> initialized_machine_topology{};
 
 const MachineTopology *get_machine_topology()
 {
-    {
-        if (!detail::initialized_machine_topology.load()) {
-            std::lock_guard<std::mutex> guard(detail::machine_topology_mutex);
-            if (!detail::machine_topology) {
-                detail::machine_topology = MachineTopology::create();
-                detail::initialized_machine_topology.store(true);
-            }
+    if (!detail::initialized_machine_topology.load()) {
+        std::lock_guard<std::mutex> guard(detail::machine_topology_mutex);
+        if (!detail::machine_topology) {
+            detail::machine_topology = MachineTopology::create();
+            detail::initialized_machine_topology.store(true);
         }
     }
     return detail::machine_topology.get();
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 708af4e931a..705639b0dfb 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -66,7 +66,7 @@ find_library(CUSPARSE cusparse
 find_library(CURAND curand
     HINT ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
 
-add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_cuda_device> "")
+add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_core_device> $<TARGET_OBJECTS:ginkgo_cuda_device> "")
 target_sources(ginkgo_cuda
     PRIVATE
     base/exception.cpp
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 9a61ade6052..24a799f6fbb 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -6,7 +6,7 @@ endif()
 ginkgo_extract_dpcpp_version(${CMAKE_CXX_COMPILER} GINKGO_DPCPP_VERSION)
 set(GINKGO_DPCPP_VERSION ${GINKGO_DPCPP_VERSION} PARENT_SCOPE)
 
-add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_dpcpp_device> "")
+add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_core_device> $<TARGET_OBJECTS:ginkgo_dpcpp_device> "")
 target_sources(ginkgo_dpcpp
     PRIVATE
     base/version.dp.cpp
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 5944595cfa6..67f4aa75a2e 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -247,14 +247,14 @@ set(GINKGO_HIP_CLANG_OPTIONS ${GINKGO_HIP_CLANG_COMPILER_FLAGS})
 
 set_source_files_properties(${GINKGO_HIP_SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
 if (GINKGO_HIP_VERSION VERSION_GREATER_EQUAL "3.5")
-    hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_hip_device> ${GINKGO_HIP_SOURCES}
+    hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_core_device> $<TARGET_OBJECTS:ginkgo_hip_device> ${GINKGO_HIP_SOURCES}
         HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS} "-std=c++14"
         HCC_OPTIONS ${GINKGO_HIP_HCC_OPTIONS}
         CLANG_OPTIONS ${GINKGO_HIP_CLANG_OPTIONS}
         NVCC_OPTIONS ${GINKGO_HIP_NVCC_OPTIONS} ${GINKGO_HIP_CUDA_HOST_COMPILER}
         ${GINKGO_STATIC_OR_SHARED})
 else()
-    hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_hip_device> ${GINKGO_HIP_SOURCES}
+    hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_core_device> $<TARGET_OBJECTS:ginkgo_hip_device> ${GINKGO_HIP_SOURCES}
         HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS} "-std=c++14"
         HCC_OPTIONS ${GINKGO_HIP_HCC_OPTIONS}
         NVCC_OPTIONS ${GINKGO_HIP_NVCC_OPTIONS} ${GINKGO_HIP_CUDA_HOST_COMPILER}
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 12b706d9573..1407cb33233 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -990,9 +990,9 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
 
     void synchronize() const override;
 
-    const int get_num_cores() const { return this->omp_exec_info_.num_cores; }
+    int get_num_cores() const { return this->omp_exec_info_.num_cores; }
 
-    const int get_num_threads_per_core() const
+    int get_num_threads_per_core() const
     {
         return this->omp_exec_info_.num_work_groups_per_core;
     }
@@ -1414,7 +1414,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         this->set_gpu_property();
         this->init_handles();
         increase_num_execs(this->hip_exec_info_.device_id);
-        // this->populate_exec_info(get_shared_machine_topology());
+        this->populate_exec_info(get_shared_machine_topology());
     }
 
     void *raw_alloc(size_type size) const override;
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index 53e828bc29d..276c62ca996 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -128,6 +128,12 @@ class MachineTopology {
 
     MachineTopology(MachineTopology &) = delete;
     MachineTopology(MachineTopology &&) = delete;
+    MachineTopology &operator=(MachineTopology &) = delete;
+    MachineTopology &operator=(MachineTopology &&) = delete;
+    ~MachineTopology()
+    {
+        std::cout << " Destructing the MachineTopology object." << std::endl;
+    }
 
     /**
      * Bind the object associated with the id to a core.
diff --git a/include/ginkgo/core/matrix/diagonal.hpp b/include/ginkgo/core/matrix/diagonal.hpp
index 70dfc0bd352..ddd4aa73bf0 100644
--- a/include/ginkgo/core/matrix/diagonal.hpp
+++ b/include/ginkgo/core/matrix/diagonal.hpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_PUBLIC_CORE_MATRIX_DIAGONAL_HPP_
 
 
+#include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
 
 
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index bedcfaa4890..608efc468e4 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -1,6 +1,6 @@
 find_package(OpenMP REQUIRED)
 
-add_library(ginkgo_omp $<TARGET_OBJECTS:ginkgo_omp_device> "")
+add_library(ginkgo_omp $<TARGET_OBJECTS:ginkgo_core_device> $<TARGET_OBJECTS:ginkgo_omp_device> "")
 target_sources(ginkgo_omp
     PRIVATE
     base/version.cpp
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index 2328e5b5763..7dc24093e35 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_library(ginkgo_reference $<TARGET_OBJECTS:ginkgo_reference_device> "")
+add_library(ginkgo_reference $<TARGET_OBJECTS:ginkgo_core_device> $<TARGET_OBJECTS:ginkgo_reference_device> "")
 target_sources(ginkgo_reference
     PRIVATE
     base/version.cpp
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index 37671aaa77d..437508a123d 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -13,7 +13,7 @@ if(GINKGO_BUILD_TESTS)
     ginkgo_add_subdirectory(GTest gtest)
 endif()
 
-if(NOT GINKGO_FORCE_NO_HWLOC)
+if(NOT GINKGO_FORCE_NO_HWLOC AND GINKGO_HAVE_HWLOC)
     ginkgo_add_subdirectory(HWLOC hwloc)
 endif()
 
diff --git a/third_party/hwloc/CMakeLists.txt b/third_party/hwloc/CMakeLists.txt
index db9905de012..5e560d5a6d2 100644
--- a/third_party/hwloc/CMakeLists.txt
+++ b/third_party/hwloc/CMakeLists.txt
@@ -1,5 +1,5 @@
 message(STATUS "Configuring and building HWLOC")
-set(TPL_HWLOC_PATH "${PROJECT_BINARY_DIR}/third_party/hwloc")
+set(TPL_HWLOC_PATH "${CMAKE_CURRENT_BINARY_DIR}")
 if(MSVC OR WIN32)
   message(STATUS "Downloading HWLOC binaries for win32")
   ginkgo_download_file("https://download.open-mpi.org/release/hwloc/v2.3/hwloc-win32-build-2.3.0.zip"
@@ -23,11 +23,16 @@ endif()
 if(MSVC OR WIN32)
   add_library(hwloc STATIC IMPORTED GLOBAL)
   add_dependencies(hwloc hwloc_external)
-  set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION "${TPL_HWLOC_PATH}/hwloc/hwloc-win32-build-2.3.0/lib/libhwloc.a" )
-  set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${TPL_HWLOC_PATH}/hwloc/hwloc-win32-build-2.3.0/include")
+  set(HWLOC_LIBRARIES "${TPL_HWLOC_PATH}/hwloc/hwloc-win32-build-2.3.0/lib/libhwloc.a" CACHE FILEPATH "The path to HWLOC library libhwloc.so" FORCE)
+  set(HWLOC_INCLUDE_DIRS "${TPL_HWLOC_PATH}/hwloc/hwloc-win32-build-2.3.0/include" CACHE PATH "The directory containing the hwloc header, hwloc.h" FORCE)
+  set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION ${HWLOC_LIBRARIES})
+  set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${HWLOC_INCLUDE_DIRS})
 else()
   add_library(hwloc SHARED IMPORTED GLOBAL)
   add_dependencies(hwloc hwloc_external )
-  set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION "${TPL_HWLOC_PATH}/build/hwloc/.libs/libhwloc.so" )
-  set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${TPL_HWLOC_PATH}/build/include")
+  set(HWLOC_LIBRARIES "${TPL_HWLOC_PATH}/build/hwloc/.libs/libhwloc.so" CACHE FILEPATH "The path to HWLOC library libhwloc.so" FORCE)
+  execute_process(COMMAND cp "${TPL_HWLOC_PATH}/build/include/hwloc/autogen/config.h" "${TPL_HWLOC_PATH}/src/include/hwloc/autogen/")
+  set(HWLOC_INCLUDE_DIRS "${TPL_HWLOC_PATH}/src/include" CACHE PATH "The directory containing the hwloc header, hwloc.h" FORCE)
+  set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION ${HWLOC_LIBRARIES})
+  set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${TPL_HWLOC_PATH}/src/include")
 endif()

From 356c8cc81e481021dc9b1f6bf33e68795aec118b Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Sat, 5 Dec 2020 10:12:51 +0100
Subject: [PATCH 0285/1008] Update to create Topo object at initialization.

---
 core/devices/machine_topology.cpp             |  1 -
 core/devices/omp/executor.cpp                 |  4 +++-
 include/ginkgo/core/base/machine_topology.hpp | 13 +++++--------
 3 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index 4d798017076..8c748ef97cd 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -67,7 +67,6 @@ const MachineTopology *get_machine_topology()
 
 std::shared_ptr<const MachineTopology> get_shared_machine_topology()
 {
-    get_machine_topology();
     return detail::machine_topology;
 }
 
diff --git a/core/devices/omp/executor.cpp b/core/devices/omp/executor.cpp
index fd7f9f14375..446a1dcaca0 100644
--- a/core/devices/omp/executor.cpp
+++ b/core/devices/omp/executor.cpp
@@ -46,7 +46,9 @@ namespace gko {
 
 void OmpExecutor::populate_exec_info(
     std::shared_ptr<const MachineTopology> mach_topo)
-{}
+{
+    this->omp_exec_info_.num_cores = mach_topo->get_num_cores();
+}
 
 
 void OmpExecutor::raw_free(void *ptr) const noexcept { std::free(ptr); }
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index 276c62ca996..3d4effcd9a0 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -130,10 +130,7 @@ class MachineTopology {
     MachineTopology(MachineTopology &&) = delete;
     MachineTopology &operator=(MachineTopology &) = delete;
     MachineTopology &operator=(MachineTopology &&) = delete;
-    ~MachineTopology()
-    {
-        std::cout << " Destructing the MachineTopology object." << std::endl;
-    }
+    ~MachineTopology() = default;
 
     /**
      * Bind the object associated with the id to a core.
@@ -188,25 +185,25 @@ class MachineTopology {
     /**
      * Get the number of PU objects stored in this Topology tree.
      */
-    size_type get_num_pus() { return pus_.size(); }
+    size_type get_num_pus() const { return pus_.size(); }
 
 
     /**
      * Get the number of core objects stored in this Topology tree.
      */
-    size_type get_num_cores() { return cores_.size(); }
+    size_type get_num_cores() const { return cores_.size(); }
 
 
     /**
      * Get the number of PCI device objects stored in this Topology tree.
      */
-    size_type get_num_pci_devices() { return pci_devices_.size(); }
+    size_type get_num_pci_devices() const { return pci_devices_.size(); }
 
 
     /**
      * Get the number of NUMA objects stored in this Topology tree.
      */
-    size_type get_num_numas() { return num_numas_; }
+    size_type get_num_numas() const { return num_numas_; }
 
 protected:
     MachineTopology()

From 014e59e7ceadf461cc855c33b1f59b255c94bfbe Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 8 Dec 2020 23:29:28 +0100
Subject: [PATCH 0286/1008] Revert to raw ptr, store global obj as unique ptr

---
 core/device_hooks/cuda_hooks.cpp              |  4 +---
 core/device_hooks/dpcpp_hooks.cpp             |  4 +---
 core/device_hooks/hip_hooks.cpp               |  4 +---
 core/devices/machine_topology.cpp             |  8 +------
 core/devices/omp/executor.cpp                 |  3 +--
 core/test/base/executor.cpp                   |  6 ++---
 cuda/base/executor.cpp                        |  7 +++---
 dpcpp/base/executor.dp.cpp                    |  4 +---
 hip/base/executor.hip.cpp                     |  7 +++---
 include/ginkgo/core/base/executor.hpp         | 23 ++++++++-----------
 include/ginkgo/core/base/machine_topology.hpp |  6 ++---
 11 files changed, 28 insertions(+), 48 deletions(-)

diff --git a/core/device_hooks/cuda_hooks.cpp b/core/device_hooks/cuda_hooks.cpp
index 0221d8c935c..3c88599f897 100644
--- a/core/device_hooks/cuda_hooks.cpp
+++ b/core/device_hooks/cuda_hooks.cpp
@@ -62,9 +62,7 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
 }
 
 
-void CudaExecutor::populate_exec_info(
-    std::shared_ptr<const MachineTopology> mach_topo)
-{}
+void CudaExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
 
 
 void OmpExecutor::raw_copy_to(const CudaExecutor *, size_type num_bytes,
diff --git a/core/device_hooks/dpcpp_hooks.cpp b/core/device_hooks/dpcpp_hooks.cpp
index 9085beb99d2..e336ee7340f 100644
--- a/core/device_hooks/dpcpp_hooks.cpp
+++ b/core/device_hooks/dpcpp_hooks.cpp
@@ -59,9 +59,7 @@ std::shared_ptr<DpcppExecutor> DpcppExecutor::create(
 }
 
 
-void DpcppExecutor::populate_exec_info(
-    std::shared_ptr<const MachineTopology> mach_topo)
-{}
+void DpcppExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
 
 
 void OmpExecutor::raw_copy_to(const DpcppExecutor *, size_type num_bytes,
diff --git a/core/device_hooks/hip_hooks.cpp b/core/device_hooks/hip_hooks.cpp
index dc30a4da947..6c8659a6909 100644
--- a/core/device_hooks/hip_hooks.cpp
+++ b/core/device_hooks/hip_hooks.cpp
@@ -59,9 +59,7 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
 }
 
 
-void HipExecutor::populate_exec_info(
-    std::shared_ptr<const MachineTopology> mach_topo)
-{}
+void HipExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
 
 
 void OmpExecutor::raw_copy_to(const HipExecutor *, size_type num_bytes,
diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index 8c748ef97cd..15b92b7219b 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -44,7 +44,7 @@ namespace gko {
 namespace detail {
 
 
-std::shared_ptr<const MachineTopology> machine_topology{};
+std::unique_ptr<const MachineTopology> machine_topology{};
 std::mutex machine_topology_mutex{};
 std::atomic<bool> initialized_machine_topology{};
 
@@ -65,10 +65,4 @@ const MachineTopology *get_machine_topology()
 }
 
 
-std::shared_ptr<const MachineTopology> get_shared_machine_topology()
-{
-    return detail::machine_topology;
-}
-
-
 }  // namespace gko
diff --git a/core/devices/omp/executor.cpp b/core/devices/omp/executor.cpp
index 446a1dcaca0..3a268be3c18 100644
--- a/core/devices/omp/executor.cpp
+++ b/core/devices/omp/executor.cpp
@@ -44,8 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace gko {
 
 
-void OmpExecutor::populate_exec_info(
-    std::shared_ptr<const MachineTopology> mach_topo)
+void OmpExecutor::populate_exec_info(const MachineTopology *mach_topo)
 {
     this->omp_exec_info_.num_cores = mach_topo->get_num_cores();
 }
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 03c71141a30..2d5f35dda1a 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -156,11 +156,11 @@ TEST(OmpExecutor, IsItsOwnMaster)
 }
 
 
-// TEST(OmpExecutor, IsItsOwnMaster)
+// TEST(OmpExecutor, CanGetNumCoresFromExecInfo)
 // {
-//     exec_ptr omp = gko::OmpExecutor::create();
+//     auto omp = gko::OmpExecutor::create();
 
-//     ASSERT_EQ(omp, omp->get_master());
+//     ASSERT_EQ(20, omp->get_num_cores());
 // }
 
 
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index f4bc9f42b9a..3434c730451 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -111,9 +111,10 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
 }
 
 
-void CudaExecutor::populate_exec_info(
-    std::shared_ptr<const MachineTopology> mach_topo)
-{}
+void CudaExecutor::populate_exec_info(const MachineTopology *mach_topo)
+{
+    this->cuda_exec_info_.num_cores = mach_topo->get_num_cores();
+}
 
 
 void OmpExecutor::raw_copy_to(const CudaExecutor *dest, size_type num_bytes,
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 1363f193160..38abcd359d7 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -93,9 +93,7 @@ std::shared_ptr<DpcppExecutor> DpcppExecutor::create(
 }
 
 
-void DpcppExecutor::populate_exec_info(
-    std::shared_ptr<const MachineTopology> mach_topo)
-{}
+void DpcppExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
 
 
 void DpcppExecutor::raw_free(void *ptr) const noexcept
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 1f75cf144a5..a7e5e94bffd 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -116,9 +116,10 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
 }
 
 
-void HipExecutor::populate_exec_info(
-    std::shared_ptr<const MachineTopology> mach_topo)
-{}
+void HipExecutor::populate_exec_info(const MachineTopology *mach_topo)
+{
+    this->hip_exec_info_.num_cores = mach_topo->get_num_cores();
+}
 
 
 void OmpExecutor::raw_copy_to(const HipExecutor *dest, size_type num_bytes,
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 1407cb33233..96e3038bf36 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -731,8 +731,7 @@ class Executor : public log::EnableLogging<Executor> {
      *
      * @param mach_topo the machine topology object.
      */
-    virtual void populate_exec_info(
-        std::shared_ptr<const MachineTopology> mach_topo) = 0;
+    virtual void populate_exec_info(const MachineTopology *mach_topo) = 0;
 
     struct exec_info {
         int device_id = -1;
@@ -998,10 +997,9 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
     }
 
 protected:
-    OmpExecutor() = default;
+    OmpExecutor() { this->populate_exec_info(get_machine_topology()); }
 
-    void populate_exec_info(
-        std::shared_ptr<const MachineTopology> mach_topo) override;
+    void populate_exec_info(const MachineTopology *mach_topo) override;
 
     void *raw_alloc(size_type size) const override;
 
@@ -1056,7 +1054,7 @@ class ReferenceExecutor : public OmpExecutor {
 protected:
     ReferenceExecutor() = default;
 
-    void populate_exec_info(std::shared_ptr<const MachineTopology>) override
+    void populate_exec_info(const MachineTopology *) override
     {
         this->ref_exec_info_.device_id = -1;
         this->ref_exec_info_.num_cores = 1;
@@ -1216,10 +1214,10 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         this->cuda_exec_info_.num_work_groups_per_core = 0;
         assert(this->cuda_exec_info_.device_id < max_devices &&
                this->cuda_exec_info_.device_id >= 0);
+        this->populate_exec_info(get_machine_topology());
         this->set_gpu_property();
         this->init_handles();
         increase_num_execs(this->cuda_exec_info_.device_id);
-        this->populate_exec_info(get_shared_machine_topology());
     }
 
     void *raw_alloc(size_type size) const override;
@@ -1256,8 +1254,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         return num_execs[device_id];
     }
 
-    void populate_exec_info(
-        std::shared_ptr<const MachineTopology> mach_topo) override;
+    void populate_exec_info(const MachineTopology *mach_topo) override;
 
 private:
     std::shared_ptr<Executor> master_;
@@ -1411,10 +1408,10 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         this->hip_exec_info_.num_cores = 0;
         this->hip_exec_info_.num_work_groups_per_core = 0;
         assert(this->hip_exec_info_.device_id < max_devices);
+        this->populate_exec_info(get_machine_topology());
         this->set_gpu_property();
         this->init_handles();
         increase_num_execs(this->hip_exec_info_.device_id);
-        this->populate_exec_info(get_shared_machine_topology());
     }
 
     void *raw_alloc(size_type size) const override;
@@ -1451,8 +1448,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         return num_execs[device_id];
     }
 
-    void populate_exec_info(
-        std::shared_ptr<const MachineTopology> mach_topo) override;
+    void populate_exec_info(const MachineTopology *mach_topo) override;
 
 private:
     std::shared_ptr<Executor> master_;
@@ -1591,8 +1587,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
         this->set_device_property();
     }
 
-    void populate_exec_info(
-        std::shared_ptr<const MachineTopology> mach_topo) override;
+    void populate_exec_info(const MachineTopology *mach_topo) override;
 
     void *raw_alloc(size_type size) const override;
 
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index 3d4effcd9a0..0efaa419a94 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -71,8 +71,6 @@ class MachineTopology;
 
 extern const MachineTopology *get_machine_topology();
 
-extern std::shared_ptr<const MachineTopology> get_shared_machine_topology();
-
 
 /**
  * The machine topology class represents the heirarchical topology of a machine,
@@ -121,9 +119,9 @@ class MachineTopology {
     /**
      * Creates a new MachineTopology object.
      */
-    static std::shared_ptr<MachineTopology> create()
+    static std::unique_ptr<MachineTopology> create()
     {
-        return std::shared_ptr<MachineTopology>(new MachineTopology());
+        return std::unique_ptr<MachineTopology>(new MachineTopology());
     }
 
     MachineTopology(MachineTopology &) = delete;

From 44932e315f0e6eb98545a21f197a404f4dcfccfe Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 9 Dec 2020 11:25:12 +0100
Subject: [PATCH 0287/1008] Change core_dev to a lib and update to shared_ptr

---
 core/CMakeLists.txt                           |  2 +-
 core/device_hooks/CMakeLists.txt              | 11 +++++------
 core/devices/CMakeLists.txt                   | 15 ++++++++++++---
 core/devices/machine_topology.cpp             |  2 +-
 cuda/CMakeLists.txt                           |  3 ++-
 dpcpp/CMakeLists.txt                          |  3 ++-
 hip/CMakeLists.txt                            |  6 ++++--
 include/ginkgo/core/base/machine_topology.hpp |  4 ++--
 omp/CMakeLists.txt                            |  3 ++-
 reference/CMakeLists.txt                      |  3 ++-
 10 files changed, 33 insertions(+), 19 deletions(-)

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 9fbb7a054dd..17990398a40 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -61,7 +61,7 @@ target_compile_options(ginkgo PRIVATE "${GINKGO_COMPILER_FLAGS}")
 # regardless of whether it is installed or added as a subdirectory
 add_library(Ginkgo::ginkgo ALIAS ginkgo)
 target_link_libraries(ginkgo
-    PUBLIC ginkgo_omp ginkgo_cuda ginkgo_reference ginkgo_hip ginkgo_dpcpp)
+    PUBLIC ginkgo_core_device ginkgo_omp ginkgo_cuda ginkgo_reference ginkgo_hip ginkgo_dpcpp)
 # The PAPI dependency needs to be exposed to the user.
 if (GINKGO_HAVE_PAPI_SDE)
     target_link_libraries(ginkgo PUBLIC PAPI::PAPI)
diff --git a/core/device_hooks/CMakeLists.txt b/core/device_hooks/CMakeLists.txt
index a677f0fcbcf..9b107364924 100644
--- a/core/device_hooks/CMakeLists.txt
+++ b/core/device_hooks/CMakeLists.txt
@@ -1,8 +1,8 @@
 if(NOT GINKGO_BUILD_CUDA)
     add_library(ginkgo_cuda
         $<TARGET_OBJECTS:ginkgo_cuda_device>
-        $<TARGET_OBJECTS:ginkgo_core_device>
         cuda_hooks.cpp)
+    target_link_libraries(ginkgo_cuda PUBLIC ginkgo_core_device)
     ginkgo_compile_features(ginkgo_cuda)
     ginkgo_default_includes(ginkgo_cuda)
     ginkgo_install_library(ginkgo_cuda cuda)
@@ -11,8 +11,8 @@ endif()
 if (NOT GINKGO_BUILD_DPCPP)
     add_library(ginkgo_dpcpp
         $<TARGET_OBJECTS:ginkgo_dpcpp_device>
-        $<TARGET_OBJECTS:ginkgo_core_device>
         dpcpp_hooks.cpp)
+    target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_core_device )
     ginkgo_compile_features(ginkgo_dpcpp)
     ginkgo_default_includes(ginkgo_dpcpp)
     ginkgo_install_library(ginkgo_dpcpp dpcpp)
@@ -21,8 +21,8 @@ endif()
 if(NOT GINKGO_BUILD_HIP)
     add_library(ginkgo_hip
         $<TARGET_OBJECTS:ginkgo_hip_device>
-        $<TARGET_OBJECTS:ginkgo_core_device>
         hip_hooks.cpp)
+    target_link_libraries(ginkgo_hip PUBLIC ginkgo_core_device)
     ginkgo_compile_features(ginkgo_hip)
     ginkgo_default_includes(ginkgo_hip)
     ginkgo_install_library(ginkgo_hip hip)
@@ -31,10 +31,9 @@ endif()
 if (NOT GINKGO_BUILD_OMP)
     add_library(ginkgo_omp
         $<TARGET_OBJECTS:ginkgo_omp_device>
-        $<TARGET_OBJECTS:ginkgo_core_device>
         omp_hooks.cpp)
     ginkgo_compile_features(ginkgo_omp)
-    target_link_libraries(ginkgo_omp PUBLIC ginkgo_cuda)
+    target_link_libraries(ginkgo_omp PUBLIC ginkgo_core_device ginkgo_cuda)
     target_link_libraries(ginkgo_omp PUBLIC ginkgo_hip)
     target_link_libraries(ginkgo_omp PUBLIC ginkgo_dpcpp)
     ginkgo_default_includes(ginkgo_omp)
@@ -44,8 +43,8 @@ endif()
 if (NOT GINKGO_BUILD_REFERENCE)
     add_library(ginkgo_reference
         $<TARGET_OBJECTS:ginkgo_reference_device>
-        $<TARGET_OBJECTS:ginkgo_core_device>
         reference_hooks.cpp)
+    target_link_libraries(ginkgo_reference PUBLIC ginkgo_core_device)
     ginkgo_compile_features(ginkgo_reference)
     ginkgo_default_includes(ginkgo_reference)
     ginkgo_install_library(ginkgo_reference reference)
diff --git a/core/devices/CMakeLists.txt b/core/devices/CMakeLists.txt
index 26365167654..e0c3dceff75 100644
--- a/core/devices/CMakeLists.txt
+++ b/core/devices/CMakeLists.txt
@@ -3,16 +3,25 @@ function(ginkgo_add_object_library name)
     ginkgo_compile_features(${name})
     ginkgo_default_includes(${name})
     target_include_directories(${name} PUBLIC ${Ginkgo_SOURCE_DIR})
+    set_target_properties(${name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+endfunction()
+
+function(ginkgo_add_normal_library name)
+    add_library(${name} ${ARGN})
+    ginkgo_compile_features(${name})
+    ginkgo_default_includes(${name})
     if(GINKGO_HAVE_HWLOC)
       target_include_directories(${name}
         PUBLIC 
         $<BUILD_INTERFACE:${HWLOC_INCLUDE_DIRS}>
         )
+    target_link_libraries(${name} ${HWLOC_LIBRARIES})
     endif()
     set_target_properties(${name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
-endfunction()
-
-ginkgo_add_object_library(ginkgo_core_device machine_topology.cpp)
+  endfunction()
+  
+ginkgo_add_normal_library(ginkgo_core_device machine_topology.cpp)
+ginkgo_install_library(ginkgo_core_device core/devices)
 
 add_subdirectory(cuda)
 add_subdirectory(dpcpp)
diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index 15b92b7219b..8e5dbf66d3f 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -44,7 +44,7 @@ namespace gko {
 namespace detail {
 
 
-std::unique_ptr<const MachineTopology> machine_topology{};
+std::shared_ptr<const MachineTopology> machine_topology{};
 std::mutex machine_topology_mutex{};
 std::atomic<bool> initialized_machine_topology{};
 
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 705639b0dfb..36badce618f 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -66,7 +66,7 @@ find_library(CUSPARSE cusparse
 find_library(CURAND curand
     HINT ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
 
-add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_core_device> $<TARGET_OBJECTS:ginkgo_cuda_device> "")
+add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_cuda_device> "")
 target_sources(ginkgo_cuda
     PRIVATE
     base/exception.cpp
@@ -155,6 +155,7 @@ ginkgo_compile_features(ginkgo_cuda)
 target_include_directories(ginkgo_cuda
     SYSTEM PRIVATE ${CUDA_INCLUDE_DIRS})
 target_link_libraries(ginkgo_cuda PRIVATE ${CUDA_RUNTIME_LIBS} ${CUBLAS} ${CUSPARSE} ${CURAND})
+target_link_libraries(ginkgo_cuda PUBLIC ginkgo_core_device)
 
 cas_target_cuda_architectures(ginkgo_cuda
     ARCHITECTURES ${GINKGO_CUDA_ARCHITECTURES}
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 24a799f6fbb..fda9eafd03b 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -6,7 +6,7 @@ endif()
 ginkgo_extract_dpcpp_version(${CMAKE_CXX_COMPILER} GINKGO_DPCPP_VERSION)
 set(GINKGO_DPCPP_VERSION ${GINKGO_DPCPP_VERSION} PARENT_SCOPE)
 
-add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_core_device> $<TARGET_OBJECTS:ginkgo_dpcpp_device> "")
+add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_dpcpp_device> "")
 target_sources(ginkgo_dpcpp
     PRIVATE
     base/version.dp.cpp
@@ -52,6 +52,7 @@ set(GINKGO_DPCPP_FLAGS ${GINKGO_COMPILER_FLAGS} -fsycl -fsycl-device-lib=libc)
 set(GINKGO_DPCPP_FLAGS ${GINKGO_DPCPP_FLAGS} PARENT_SCOPE)
 target_compile_options(ginkgo_dpcpp PRIVATE "${GINKGO_DPCPP_FLAGS}")
 target_compile_features(ginkgo_dpcpp PRIVATE cxx_std_17)
+target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_core_device)
 
 ginkgo_default_includes(ginkgo_dpcpp)
 ginkgo_install_library(ginkgo_dpcpp dpcpp)
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 67f4aa75a2e..b795a4b9494 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -247,14 +247,14 @@ set(GINKGO_HIP_CLANG_OPTIONS ${GINKGO_HIP_CLANG_COMPILER_FLAGS})
 
 set_source_files_properties(${GINKGO_HIP_SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
 if (GINKGO_HIP_VERSION VERSION_GREATER_EQUAL "3.5")
-    hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_core_device> $<TARGET_OBJECTS:ginkgo_hip_device> ${GINKGO_HIP_SOURCES}
+    hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_hip_device> ${GINKGO_HIP_SOURCES}
         HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS} "-std=c++14"
         HCC_OPTIONS ${GINKGO_HIP_HCC_OPTIONS}
         CLANG_OPTIONS ${GINKGO_HIP_CLANG_OPTIONS}
         NVCC_OPTIONS ${GINKGO_HIP_NVCC_OPTIONS} ${GINKGO_HIP_CUDA_HOST_COMPILER}
         ${GINKGO_STATIC_OR_SHARED})
 else()
-    hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_core_device> $<TARGET_OBJECTS:ginkgo_hip_device> ${GINKGO_HIP_SOURCES}
+    hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_hip_device> ${GINKGO_HIP_SOURCES}
         HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS} "-std=c++14"
         HCC_OPTIONS ${GINKGO_HIP_HCC_OPTIONS}
         NVCC_OPTIONS ${GINKGO_HIP_NVCC_OPTIONS} ${GINKGO_HIP_CUDA_HOST_COMPILER}
@@ -299,6 +299,8 @@ endif()
 
 target_link_libraries(ginkgo_hip PRIVATE roc::hipblas roc::hipsparse hip::hiprand roc::rocrand)
 
+target_link_libraries(ginkgo_hip PUBLIC ginkgo_core_device)
+
 target_include_directories(ginkgo_hip
     PUBLIC
         ${HIP_INCLUDE_DIRS}
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index 0efaa419a94..eb2aa074699 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -119,9 +119,9 @@ class MachineTopology {
     /**
      * Creates a new MachineTopology object.
      */
-    static std::unique_ptr<MachineTopology> create()
+    static std::shared_ptr<MachineTopology> create()
     {
-        return std::unique_ptr<MachineTopology>(new MachineTopology());
+        return std::shared_ptr<MachineTopology>(new MachineTopology());
     }
 
     MachineTopology(MachineTopology &) = delete;
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index 608efc468e4..1107d91a38c 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -1,6 +1,6 @@
 find_package(OpenMP REQUIRED)
 
-add_library(ginkgo_omp $<TARGET_OBJECTS:ginkgo_core_device> $<TARGET_OBJECTS:ginkgo_omp_device> "")
+add_library(ginkgo_omp $<TARGET_OBJECTS:ginkgo_omp_device> "")
 target_sources(ginkgo_omp
     PRIVATE
     base/version.cpp
@@ -41,6 +41,7 @@ target_sources(ginkgo_omp
 
 ginkgo_compile_features(ginkgo_omp)
 
+target_link_libraries(ginkgo_omp PUBLIC ginkgo_core_device)
 target_link_libraries(ginkgo_omp PRIVATE "${OpenMP_CXX_LIBRARIES}")
 target_include_directories(ginkgo_omp PRIVATE "${OpenMP_CXX_INCLUDE_DIRS}")
 # We first separate the arguments, otherwise, the target_compile_options adds it as a string
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index 7dc24093e35..6a261d56dcc 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_library(ginkgo_reference $<TARGET_OBJECTS:ginkgo_core_device> $<TARGET_OBJECTS:ginkgo_reference_device> "")
+add_library(ginkgo_reference $<TARGET_OBJECTS:ginkgo_reference_device> "")
 target_sources(ginkgo_reference
     PRIVATE
     base/version.cpp
@@ -37,6 +37,7 @@ target_sources(ginkgo_reference
     stop/criterion_kernels.cpp
     stop/residual_norm_kernels.cpp)
 
+target_link_libraries(ginkgo_reference PUBLIC ginkgo_core_device)
 ginkgo_compile_features(ginkgo_reference)
 ginkgo_default_includes(ginkgo_reference)
 ginkgo_install_library(ginkgo_reference reference)

From 26e46fe1b7ec97cfa5d7f130e4ef6f44029d4a7b Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 17 Dec 2020 15:04:34 +0100
Subject: [PATCH 0288/1008] Disable global no reset exec for Intel.

---
 core/device_hooks/dpcpp_hooks.cpp             |  7 ++---
 core/devices/cuda/executor.cpp                |  4 +--
 core/devices/hip/executor.cpp                 |  4 +--
 core/devices/machine_topology.cpp             |  1 +
 cuda/CMakeLists.txt                           |  2 +-
 cuda/test/utils.hpp                           |  3 +++
 dpcpp/base/executor.dp.cpp                    |  6 +++--
 include/ginkgo/core/base/executor.hpp         |  7 ++++-
 include/ginkgo/core/base/machine_topology.hpp | 27 ++++++++++---------
 9 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/core/device_hooks/dpcpp_hooks.cpp b/core/device_hooks/dpcpp_hooks.cpp
index e336ee7340f..444fd6c8a33 100644
--- a/core/device_hooks/dpcpp_hooks.cpp
+++ b/core/device_hooks/dpcpp_hooks.cpp
@@ -126,14 +126,15 @@ void DpcppExecutor::set_device_property() {}
 bool DpcppExecutor::verify_memory_to(const OmpExecutor *dest_exec) const
 {
     // Dummy check
-    return device_type_ == "cpu" || device_type_ == "host";
+    return this->dpcpp_exec_info_.device_type == "cpu" ||
+           this->dpcpp_exec_info_.device_type == "host";
 }
 
 bool DpcppExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
 {
     // Dummy check
-    return dest_exec->get_device_type() == device_type_ &&
-           dest_exec->get_device_id() == device_id_;
+    return dest_exec->get_device_type() == this->dpcpp_exec_info_.device_type &&
+           dest_exec->get_device_id() == this->dpcpp_exec_info_.device_id;
 }
 
 
diff --git a/core/devices/cuda/executor.cpp b/core/devices/cuda/executor.cpp
index 61ba73ca89c..31750fa2a42 100644
--- a/core/devices/cuda/executor.cpp
+++ b/core/devices/cuda/executor.cpp
@@ -50,14 +50,14 @@ std::shared_ptr<const Executor> CudaExecutor::get_master() const noexcept
 
 bool CudaExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
 {
-    return device_id_ == dest_exec->get_device_id();
+    return this->cuda_exec_info_.device_id == dest_exec->get_device_id();
 }
 
 
 bool CudaExecutor::verify_memory_to(const HipExecutor *dest_exec) const
 {
 #if GINKGO_HIP_PLATFORM_NVCC
-    return device_id_ == dest_exec->get_device_id();
+    return this->cuda_exec_info_.device_id == dest_exec->get_device_id();
 #else
     return false;
 #endif
diff --git a/core/devices/hip/executor.cpp b/core/devices/hip/executor.cpp
index 565bd50279b..3d33fd4b125 100644
--- a/core/devices/hip/executor.cpp
+++ b/core/devices/hip/executor.cpp
@@ -47,14 +47,14 @@ std::shared_ptr<const Executor> HipExecutor::get_master() const noexcept
 
 bool HipExecutor::verify_memory_to(const HipExecutor *dest_exec) const
 {
-    return device_id_ == dest_exec->get_device_id();
+    return this->hip_exec_info_.device_id == dest_exec->get_device_id();
 }
 
 
 bool HipExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
 {
 #if GINKGO_HIP_PLATFORM_NVCC
-    return device_id_ == dest_exec->get_device_id();
+    return this->hip_exec_info_.device_id == dest_exec->get_device_id();
 #else
     return false;
 #endif
diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index 8e5dbf66d3f..736d39ebd89 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -61,6 +61,7 @@ const MachineTopology *get_machine_topology()
             detail::initialized_machine_topology.store(true);
         }
     }
+    assert(detail::machine_topology.get() != nullptr);
     return detail::machine_topology.get();
 }
 
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 36badce618f..dab6efee927 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -155,7 +155,7 @@ ginkgo_compile_features(ginkgo_cuda)
 target_include_directories(ginkgo_cuda
     SYSTEM PRIVATE ${CUDA_INCLUDE_DIRS})
 target_link_libraries(ginkgo_cuda PRIVATE ${CUDA_RUNTIME_LIBS} ${CUBLAS} ${CUSPARSE} ${CURAND})
-target_link_libraries(ginkgo_cuda PUBLIC ginkgo_core_device)
+target_link_libraries(ginkgo_cuda PRIVATE ginkgo_core_device)
 
 cas_target_cuda_architectures(ginkgo_cuda
     ARCHITECTURES ${GINKGO_CUDA_ARCHITECTURES}
diff --git a/cuda/test/utils.hpp b/cuda/test/utils.hpp
index 4b6e0e0a667..085bead5885 100644
--- a/cuda/test/utils.hpp
+++ b/cuda/test/utils.hpp
@@ -42,11 +42,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace {
 
+#if defined(__ICC) || defined(__ICL)
+#else
 
 // prevent device reset after each test
 auto no_reset_exec =
     gko::CudaExecutor::create(0, gko::ReferenceExecutor::create(), true);
 
+#endif
 
 }  // namespace
 
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 38abcd359d7..8c87ec574b1 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -164,13 +164,15 @@ int DpcppExecutor::get_num_devices(std::string device_type)
 
 bool DpcppExecutor::verify_memory_to(const OmpExecutor *dest_exec) const
 {
-    auto device = detail::get_devices(device_type_)[device_id_];
+    auto device = detail::get_devices(
+        dpcpp_exec_info_.device_type)[dpcpp_exec_info_.device_id];
     return device.is_host() || device.is_cpu();
 }
 
 bool DpcppExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
 {
-    auto device = detail::get_devices(device_type_)[device_id_];
+    auto device = detail::get_devices(
+        dpcpp_exec_info_.device_type_)[dpcpp_exec_info_.device_id];
     auto other_device = detail::get_devices(
         dest_exec->get_device_type())[dest_exec->get_device_id()];
     return ((device.is_host() || device.is_cpu()) &&
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 96e3038bf36..09053431070 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -1572,7 +1572,10 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      *
      * @return a string representing the device type
      */
-    std::string get_device_type() const noexcept { return device_type_; }
+    std::string get_device_type() const noexcept
+    {
+        return dpcpp_exec_info_.device_type;
+    }
 
 protected:
     void set_device_property();
@@ -1585,6 +1588,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
                       this->dpcpp_exec_info_.device_type.end(),
                       [](char &c) { c = std::tolower(c); });
         this->set_device_property();
+        this->dpcpp_exec_info_.device_id = device_id;
     }
 
     void populate_exec_info(const MachineTopology *mach_topo) override;
@@ -1609,6 +1613,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
     std::shared_ptr<Executor> master_;
 
     struct dpcpp_exec_info : public exec_info {
+        int device_id;
         std::string device_type;
         int num_computing_units;
         std::vector<size_type> subgroup_sizes{};
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index eb2aa074699..15f24ebc0b6 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -151,8 +151,8 @@ class MachineTopology {
      */
     const mach_topo_obj_info &get_pu(size_type id)
     {
-        GKO_ENSURE_IN_BOUNDS(id, pus_.size());
-        return pus_[id];
+        GKO_ENSURE_IN_BOUNDS(id, this->pus_.size());
+        return this->pus_[id];
     }
 
 
@@ -163,8 +163,8 @@ class MachineTopology {
      */
     const mach_topo_obj_info &get_core(size_type id)
     {
-        GKO_ENSURE_IN_BOUNDS(id, cores_.size());
-        return cores_[id];
+        GKO_ENSURE_IN_BOUNDS(id, this->cores_.size());
+        return this->cores_[id];
     }
 
 
@@ -175,33 +175,33 @@ class MachineTopology {
      */
     const mach_topo_obj_info &get_pci_device(size_type id)
     {
-        GKO_ENSURE_IN_BOUNDS(id, pci_devices_.size());
-        return pci_devices_[id];
+        GKO_ENSURE_IN_BOUNDS(id, this->pci_devices_.size());
+        return this->pci_devices_[id];
     }
 
 
     /**
      * Get the number of PU objects stored in this Topology tree.
      */
-    size_type get_num_pus() const { return pus_.size(); }
+    size_type get_num_pus() const { return this->pus_.size(); }
 
 
     /**
      * Get the number of core objects stored in this Topology tree.
      */
-    size_type get_num_cores() const { return cores_.size(); }
+    size_type get_num_cores() const { return this->cores_.size(); }
 
 
     /**
      * Get the number of PCI device objects stored in this Topology tree.
      */
-    size_type get_num_pci_devices() const { return pci_devices_.size(); }
+    size_type get_num_pci_devices() const { return this->pci_devices_.size(); }
 
 
     /**
      * Get the number of NUMA objects stored in this Topology tree.
      */
-    size_type get_num_numas() const { return num_numas_; }
+    size_type get_num_numas() const { return this->num_numas_; }
 
 protected:
     MachineTopology()
@@ -216,7 +216,8 @@ class MachineTopology {
         load_objects(HWLOC_OBJ_PU, this->pus_);
         // load_objects(HWLOC_OBJ_PCI_DEVICE, this->pci_devices_);
 
-        num_numas_ = hwloc_get_nbobjs_by_type(topo_.get(), HWLOC_OBJ_PACKAGE);
+        num_numas_ =
+            hwloc_get_nbobjs_by_type(this->topo_.get(), HWLOC_OBJ_PACKAGE);
 #else
 
         this->topo_ = topo_manager<hwloc_topology>();
@@ -276,9 +277,9 @@ class MachineTopology {
     void load_objects(hwloc_obj_type_t type,
                       std::vector<mach_topo_obj_info> &vector)
     {
-        unsigned nbcores = hwloc_get_nbobjs_by_type(topo_.get(), type);
+        unsigned nbcores = hwloc_get_nbobjs_by_type(this->topo_.get(), type);
         for (unsigned i = 0; i < nbcores; i++) {
-            hwloc_obj_t obj = hwloc_get_obj_by_type(topo_.get(), type, i);
+            hwloc_obj_t obj = hwloc_get_obj_by_type(this->topo_.get(), type, i);
             vector.push_back(
                 mach_topo_obj_info{obj, hwloc_bitmap_first(obj->nodeset),
                                    obj->logical_index, obj->os_index});

From 7496edf25db88a1836f3019145d85a5768bc5d7a Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Sun, 20 Dec 2020 00:13:40 +0100
Subject: [PATCH 0289/1008] Review update, fix get info.

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 cmake/autodetect_executors.cmake      |  1 -
 cmake/get_info.cmake                  | 17 ++++++++++++-----
 cmake/package_helpers.cmake           |  2 +-
 core/devices/CMakeLists.txt           |  2 +-
 core/devices/cuda/CMakeLists.txt      |  2 +-
 core/devices/dpcpp/CMakeLists.txt     |  2 +-
 core/devices/hip/CMakeLists.txt       |  2 +-
 core/devices/omp/CMakeLists.txt       |  2 +-
 core/devices/reference/CMakeLists.txt |  2 +-
 dpcpp/base/executor.dp.cpp            | 17 +++++++++--------
 hip/CMakeLists.txt                    |  3 +--
 include/ginkgo/core/base/executor.hpp |  6 +++---
 12 files changed, 32 insertions(+), 26 deletions(-)

diff --git a/cmake/autodetect_executors.cmake b/cmake/autodetect_executors.cmake
index c448805fb2c..c432a92d593 100644
--- a/cmake/autodetect_executors.cmake
+++ b/cmake/autodetect_executors.cmake
@@ -2,7 +2,6 @@ set(GINKGO_HAS_OMP OFF)
 set(GINKGO_HAS_CUDA OFF)
 set(GINKGO_HAS_DPCPP OFF)
 set(GINKGO_HAS_HIP OFF)
-set(GINKGO_HAS_HWLOC OFF)
 find_package(OpenMP)
 include(CheckLanguage)
 check_language(CUDA)
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index bbca15acbbd..5b09bb571b8 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -127,7 +127,7 @@ foreach(log_type ${log_types})
     ginkgo_print_module_footer(${${log_type}} "User configuration:")
     ginkgo_print_module_footer(${${log_type}} "  Enabled modules:")
     ginkgo_print_foreach_variable(
-        "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP;GINKGO_WITH_HWLOC")
+        "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP")
     ginkgo_print_module_footer(${${log_type}} "  Tests, benchmarks and examples:")
     ginkgo_print_foreach_variable(
         "GINKGO_BUILD_TESTS;GINKGO_BUILD_EXAMPLES;GINKGO_EXTLIB_EXAMPLE;GINKGO_BUILD_BENCHMARKS")
@@ -168,18 +168,25 @@ IF(GINKGO_BUILD_DPCPP)
     include(dpcpp/get_info.cmake)
 ENDIF()
 
-ginkgo_print_generic_header(${detailed_log} "Optional Components:")
-ginkgo_print_variable(${detailed_log} "GKO_HAVE_PAPI_SDE")
+ginkgo_print_generic_header(${minimal_log} "  Components:")
+ginkgo_print_generic_header(${detailed_log} "  Components:")
 if(PAPI_sde_FOUND)
     ginkgo_print_variable(${detailed_log} "PAPI_VERSION")
     ginkgo_print_variable(${detailed_log} "PAPI_INCLUDE_DIR")
     ginkgo_print_flags(${detailed_log} "PAPI_LIBRARY")
 endif()
+ginkgo_print_variable(${minimal_log} "GINKGO_USE_EXTERNAL_HWLOC")
+ginkgo_print_variable(${detailed_log} "GINKGO_USE_EXTERNAL_HWLOC")
+ginkgo_print_variable(${minimal_log} "GINKGO_FORCE_NO_HWLOC")
+ginkgo_print_variable(${detailed_log} "GINKGO_FORCE_NO_HWLOC")
+ginkgo_print_variable(${detailed_log} "HWLOC_VERSION")
+ginkgo_print_variable(${detailed_log} "HWLOC_LIBRARIES")
+ginkgo_print_variable(${detailed_log} "HWLOC_INCLUDE_DIRS")
 
 _minimal(
-    "\n
+    "
 --\n--  Detailed information (More compiler flags, module configuration) can be found in detailed.log
-    ")
+--   ")
 _both(
     "\n--\n--  Now, run  cmake --build .  to compile Ginkgo!\n"
     )
diff --git a/cmake/package_helpers.cmake b/cmake/package_helpers.cmake
index 51441d40499..bb3eeb39bf0 100644
--- a/cmake/package_helpers.cmake
+++ b/cmake/package_helpers.cmake
@@ -200,7 +200,7 @@ macro(ginkgo_find_package package_name target_list header_only)
                     "${lib}" ${header_only})
              endforeach()
         else()
-            find_package(${package_name} ${ARGN})
+            find_package(${package_name} QUIET ${ARGN})
             if (${package_name}_FOUND)
                 message(STATUS "Using external version of package ${package_name}. In case of problems, consider setting -DGINKGO_USE_EXTERNAL_${_UPACKAGE_NAME}=OFF.")
             else()
diff --git a/core/devices/CMakeLists.txt b/core/devices/CMakeLists.txt
index e0c3dceff75..b1577545e9b 100644
--- a/core/devices/CMakeLists.txt
+++ b/core/devices/CMakeLists.txt
@@ -15,7 +15,7 @@ function(ginkgo_add_normal_library name)
         PUBLIC 
         $<BUILD_INTERFACE:${HWLOC_INCLUDE_DIRS}>
         )
-    target_link_libraries(${name} ${HWLOC_LIBRARIES})
+    target_link_libraries(${name} PUBLIC ${HWLOC_LIBRARIES})
     endif()
     set_target_properties(${name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
   endfunction()
diff --git a/core/devices/cuda/CMakeLists.txt b/core/devices/cuda/CMakeLists.txt
index 17fad40ae98..1b164165305 100644
--- a/core/devices/cuda/CMakeLists.txt
+++ b/core/devices/cuda/CMakeLists.txt
@@ -1,3 +1,3 @@
 ginkgo_add_object_library(ginkgo_cuda_device
-  executor.cpp)
+    executor.cpp)
 
diff --git a/core/devices/dpcpp/CMakeLists.txt b/core/devices/dpcpp/CMakeLists.txt
index 45200f64072..f6fdb354ff2 100644
--- a/core/devices/dpcpp/CMakeLists.txt
+++ b/core/devices/dpcpp/CMakeLists.txt
@@ -1,2 +1,2 @@
 ginkgo_add_object_library(ginkgo_dpcpp_device
-  executor.cpp)
+    executor.cpp)
diff --git a/core/devices/hip/CMakeLists.txt b/core/devices/hip/CMakeLists.txt
index 5900ce69950..7e5b124a157 100644
--- a/core/devices/hip/CMakeLists.txt
+++ b/core/devices/hip/CMakeLists.txt
@@ -1,4 +1,4 @@
 ginkgo_add_object_library(ginkgo_hip_device
-  executor.cpp)
+    executor.cpp)
  
 
diff --git a/core/devices/omp/CMakeLists.txt b/core/devices/omp/CMakeLists.txt
index 5ebb37c4799..bf5ca4cfa88 100644
--- a/core/devices/omp/CMakeLists.txt
+++ b/core/devices/omp/CMakeLists.txt
@@ -1,2 +1,2 @@
 ginkgo_add_object_library(ginkgo_omp_device
-  executor.cpp)
+    executor.cpp)
diff --git a/core/devices/reference/CMakeLists.txt b/core/devices/reference/CMakeLists.txt
index 485eb53e22d..2bb8cd7c79a 100644
--- a/core/devices/reference/CMakeLists.txt
+++ b/core/devices/reference/CMakeLists.txt
@@ -1,2 +1,2 @@
 ginkgo_add_object_library(ginkgo_reference_device
-  dummy.cpp)
+    dummy.cpp)
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 8c87ec574b1..6652a7ccf7e 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -172,7 +172,7 @@ bool DpcppExecutor::verify_memory_to(const OmpExecutor *dest_exec) const
 bool DpcppExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
 {
     auto device = detail::get_devices(
-        dpcpp_exec_info_.device_type_)[dpcpp_exec_info_.device_id];
+        dpcpp_exec_info_.device_type)[dpcpp_exec_info_.device_id];
     auto other_device = detail::get_devices(
         dest_exec->get_device_type())[dest_exec->get_device_id()];
     return ((device.is_host() || device.is_cpu()) &&
@@ -198,27 +198,28 @@ void delete_queue(sycl::queue *queue)
 
 void DpcppExecutor::set_device_property()
 {
-    assert(dpcpp_exec_info_.device_id <
-           DpcppExecutor::get_num_devices(dpcpp_exec_info_.device_type));
+    assert(this->dpcpp_exec_info_.device_id <
+           DpcppExecutor::get_num_devices(this->dpcpp_exec_info_.device_type));
     auto device = detail::get_devices(
-        dpcpp_exec_info_.device_type)[dpcpp_exec_info_.device_id];
+        this->dpcpp_exec_info_.device_type)[this->dpcpp_exec_info_.device_id];
     if (!device.is_host()) {
         try {
-            dpcpp_exec_info_.subgroup_sizes =
+            this->dpcpp_exec_info_.subgroup_sizes =
                 device.get_info<cl::sycl::info::device::sub_group_sizes>();
         } catch (cl::sycl::runtime_error &err) {
             GKO_NOT_SUPPORTED(device);
         }
     }
-    dpcpp_exec_info_.num_computing_units =
+    this->dpcpp_exec_info_.num_computing_units =
         device.get_info<sycl::info::device::max_compute_units>();
     auto max_workitem_sizes =
         device.get_info<sycl::info::device::max_work_item_sizes>();
     // There is no way to get the dimension of a sycl::id object
     for (std::size_t i = 0; i < 3; i++) {
-        dpcpp_exec_info_.max_workitem_sizes.push_back(max_workitem_sizes[i]);
+        this->dpcpp_exec_info_.max_workitem_sizes.push_back(
+            max_workitem_sizes[i]);
     }
-    dpcpp_exec_info_.num_work_groups_per_core =
+    this->dpcpp_exec_info_.num_work_groups_per_core =
         device.get_info<sycl::info::device::max_work_group_size>();
     // Here we declare the queue with the property `in_order` which ensures the
     // kernels are executed in the submission order. Otherwise, calls to
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index b795a4b9494..fbbb19088ad 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -289,8 +289,7 @@ if(GINKGO_HIP_PLATFORM MATCHES "hcc")
         # directly use HIP
         ginkgo_hip_clang_ban_hip_device_flags()
     endif()
-    target_link_libraries(ginkgo_hip PRIVATE hip::device
-      )
+    target_link_libraries(ginkgo_hip PRIVATE hip::device)
 elseif(GINKGO_HIP_PLATFORM MATCHES "nvcc")
     find_package(CUDA 9.2 REQUIRED)
     target_link_libraries(ginkgo_hip PUBLIC ${CUDA_LIBRARIES})
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 09053431070..0a520e70580 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -1584,11 +1584,11 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
                   std::string device_type = "all")
         : master_(master)
     {
-        std::for_each(this->dpcpp_exec_info_.device_type.begin(),
-                      this->dpcpp_exec_info_.device_type.end(),
+        std::for_each(device_type.begin(), device_type.end(),
                       [](char &c) { c = std::tolower(c); });
-        this->set_device_property();
+        this->dpcpp_exec_info_.device_type = std::string(device_type);
         this->dpcpp_exec_info_.device_id = device_id;
+        this->set_device_property();
     }
 
     void populate_exec_info(const MachineTopology *mach_topo) override;

From 43086b2c22b8e3f7924c695d3b8138375b6b96c3 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Sun, 20 Dec 2020 00:15:33 +0100
Subject: [PATCH 0290/1008] Move functions to cpp file, udpate CI jobs.

+ Forcibly switch of HWLOC for some jobs.
+ Use internal third-party HWLOC for some jobs.
---
 .gitlab-ci.yml                                |  18 ++-
 CMakeLists.txt                                |   2 +-
 cmake/get_info.cmake                          |   1 +
 core/devices/CMakeLists.txt                   |  10 +-
 core/devices/machine_topology.cpp             |  92 ++++++++++++
 core/test/base/executor.cpp                   |  12 +-
 hip/CMakeLists.txt                            |   4 +-
 include/ginkgo/core/base/machine_topology.hpp | 134 ++++++------------
 8 files changed, 166 insertions(+), 107 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8f38f81d93c..f9667b0025d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,7 +22,8 @@ stages:
   BUILD_OMP: "OFF"
   BUILD_CUDA: "OFF"
   BUILD_HIP: "OFF"
-  BUILD_HWLOC: "ON"
+  USE_EXTERNAL_HWLOC: "ON"
+  FORCE_NO_HWLOC: "OFF"
   RUN_EXAMPLES: "OFF"
   CONFIG_LOG: "ON"
   CXX_FLAGS: ""
@@ -64,7 +65,8 @@ stages:
         -DGINKGO_DEVEL_TOOLS=OFF -DGINKGO_BUILD_REFERENCE=${BUILD_REFERENCE}
         -DGINKGO_BUILD_OMP=${BUILD_OMP} -DGINKGO_BUILD_CUDA=${BUILD_CUDA}
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
-        -DGINKGO_WITH_HWLOC=${BUILD_HWLOC}
+        -DGINKGO_FORCE_NO_HWLOC=${FORCE_NO_HWLOC}
+        -DGINKGO_USE_EXTERNAL_HWLOC=${USE_EXTERNAL_HWLOC}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
@@ -96,7 +98,8 @@ stages:
         -DGINKGO_DEVEL_TOOLS=OFF -DGINKGO_BUILD_REFERENCE=${BUILD_REFERENCE}
         -DGINKGO_BUILD_OMP=${BUILD_OMP} -DGINKGO_BUILD_CUDA=${BUILD_CUDA}
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
-        -DGINKGO_WITH_HWLOC=${BUILD_HWLOC}
+        -DGINKGO_FORCE_NO_HWLOC=${FORCE_NO_HWLOC}
+        -DGINKGO_USE_EXTERNAL_HWLOC=${USE_EXTERNAL_HWLOC}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
@@ -255,6 +258,8 @@ build/cuda92/gcc/all/release/debug:
     - cuda
     - gpu
 
+# Make sure that our jobs run when HWLOC is
+# forcibly switched off
 build/cuda92/intel/cuda/release/static:
   <<: *default_build_with_test
   image: localhost:5000/gko-cuda92-gnu7-llvm50-intel2017
@@ -264,6 +269,7 @@ build/cuda92/intel/cuda/release/static:
     CXX_COMPILER: "icpc"
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
+    FORCE_NO_HWLOC: "OFF"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
@@ -276,6 +282,8 @@ build/cuda92/intel/cuda/release/static:
     - gpu
 
 # cuda 10.0 and friends
+# Make sure that our jobs run when using self-installed
+# third-party HWLOC.
 build/cuda100/gcc/all/debug/shared:
   <<: *default_build_with_test
   image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
@@ -284,6 +292,7 @@ build/cuda100/gcc/all/debug/shared:
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
+    USE_EXTERNAL_HWLOC: "OFF"
     BUILD_TYPE: "Debug"
     CUDA_ARCH: 35
   only:
@@ -324,7 +333,6 @@ build/cuda100/intel/cuda/release/shared:
     CXX_COMPILER: "icpc"
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
-    BUILD_HWLOC: "OFF"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
   only:
@@ -595,6 +603,7 @@ build/nocuda/clang/core/release/shared:
     C_COMPILER: "clang"
     CXX_COMPILER: "clang++"
     BUILD_REFERENCE: "OFF"
+    USE_EXTERNAL_HWLOC: "OFF"
     BUILD_TYPE: "Release"
   only:
     variables:
@@ -641,6 +650,7 @@ build/nocuda/clang/omp/debug/static:
     <<: *default_variables
     C_COMPILER: "clang"
     CXX_COMPILER: "clang++"
+    USE_EXTERNAL_HWLOC: "OFF"
     BUILD_OMP: "ON"
     BUILD_TYPE: "Debug"
     BUILD_SHARED_LIBS: "OFF"
diff --git a/CMakeLists.txt b/CMakeLists.txt
index da16b2b1ec1..aa1324404e9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -211,7 +211,7 @@ ginkgo_find_package(RapidJSON rapidjson TRUE 1.1.0)
 if(NOT GINKGO_FORCE_NO_HWLOC AND GINKGO_HAVE_HWLOC)
   ginkgo_find_package(HWLOC hwloc FALSE 2.1)
 endif()
-set(GKO_HWLOC_XMLFILE "${HWLOC_XMLFILE}")
+set(GKO_HWLOC_XMLFILE "$ENV{HWLOC_XMLFILE}")
 message(STATUS "HWLOC XML file set to: ${GKO_HWLOC_XMLFILE}")
 add_subdirectory(third_party)    # Third-party tools and libraries
 
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index 5b09bb571b8..20785260965 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -180,6 +180,7 @@ ginkgo_print_variable(${detailed_log} "GINKGO_USE_EXTERNAL_HWLOC")
 ginkgo_print_variable(${minimal_log} "GINKGO_FORCE_NO_HWLOC")
 ginkgo_print_variable(${detailed_log} "GINKGO_FORCE_NO_HWLOC")
 ginkgo_print_variable(${detailed_log} "HWLOC_VERSION")
+ginkgo_print_variable(${detailed_log} "GKO_HWLOC_XMLFILE")
 ginkgo_print_variable(${detailed_log} "HWLOC_LIBRARIES")
 ginkgo_print_variable(${detailed_log} "HWLOC_INCLUDE_DIRS")
 
diff --git a/core/devices/CMakeLists.txt b/core/devices/CMakeLists.txt
index b1577545e9b..c7b409450bc 100644
--- a/core/devices/CMakeLists.txt
+++ b/core/devices/CMakeLists.txt
@@ -11,14 +11,14 @@ function(ginkgo_add_normal_library name)
     ginkgo_compile_features(${name})
     ginkgo_default_includes(${name})
     if(GINKGO_HAVE_HWLOC)
-      target_include_directories(${name}
-        PUBLIC 
-        $<BUILD_INTERFACE:${HWLOC_INCLUDE_DIRS}>
+        target_include_directories(${name}
+            PUBLIC 
+            $<BUILD_INTERFACE:${HWLOC_INCLUDE_DIRS}>
         )
-    target_link_libraries(${name} PUBLIC ${HWLOC_LIBRARIES})
+        target_link_libraries(${name} PUBLIC ${HWLOC_LIBRARIES})
     endif()
     set_target_properties(${name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
-  endfunction()
+endfunction()
   
 ginkgo_add_normal_library(ginkgo_core_device machine_topology.cpp)
 ginkgo_install_library(ginkgo_core_device core/devices)
diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index 736d39ebd89..f415f76f8fe 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -66,4 +66,96 @@ const MachineTopology *get_machine_topology()
 }
 
 
+MachineTopology::MachineTopology()
+{
+#if GKO_HAVE_HWLOC
+
+    // Initialize the topology from hwloc
+    this->topo_ =
+        topo_manager<hwloc_topology>(init_topology(), hwloc_topology_destroy);
+
+    // load objects of type CORE. See HWLOC_OBJ_TYPE for more details.
+    load_objects(HWLOC_OBJ_CORE, this->cores_);
+    // load objects of type PU. See HWLOC_OBJ_TYPE for more details.
+    load_objects(HWLOC_OBJ_PU, this->pus_);
+    // load_objects(HWLOC_OBJ_PCI_DEVICE, this->pci_devices_);
+
+    num_numas_ = hwloc_get_nbobjs_by_type(this->topo_.get(), HWLOC_OBJ_PACKAGE);
+#else
+
+    this->topo_ = topo_manager<hwloc_topology>();
+
+#endif
+}
+
+
+void MachineTopology::hwloc_binding_helper(std::vector<mach_topo_obj_info> &obj,
+                                           size_type id)
+{
+#if GKO_HAVE_HWLOC
+    auto bitmap = hwloc_bitmap_alloc();
+    hwloc_bitmap_set(bitmap, obj[id].physical_id);
+    hwloc_bitmap_singlify(bitmap);
+    hwloc_set_cpubind(topo_.get(), bitmap, 0);
+    hwloc_bitmap_free(bitmap);
+#endif
+}
+
+
+void MachineTopology::hwloc_print_children(hwloc_topology *topology,
+                                           hwloc_obj_t obj, int depth)
+{
+#if GKO_HAVE_HWLOC
+    char type[32], attr[1024];
+    unsigned i;
+    hwloc_obj_type_snprintf(type, sizeof(type), obj, 0);
+    std::cout << std::string(2 * depth, ' ') << type;
+    if (obj->os_index != (unsigned)-1) {
+        std::cout << "#" << obj->os_index;
+    }
+    hwloc_obj_attr_snprintf(attr, sizeof(attr), obj, " ", 0);
+    if (*attr) {
+        std::cout << "(" << attr << ")";
+    }
+    std::cout << std::endl;
+    for (i = 0; i < obj->arity; i++) {
+        hwloc_print_children(topology, obj->children[i], depth + 1);
+    }
+#endif
+}
+
+
+void MachineTopology::load_objects(hwloc_obj_type_t type,
+                                   std::vector<mach_topo_obj_info> &vector)
+{
+#if GKO_HAVE_HWLOC
+    unsigned nbcores = hwloc_get_nbobjs_by_type(this->topo_.get(), type);
+    for (unsigned i = 0; i < nbcores; i++) {
+        hwloc_obj_t obj = hwloc_get_obj_by_type(this->topo_.get(), type, i);
+        vector.push_back(mach_topo_obj_info{obj,
+                                            hwloc_bitmap_first(obj->nodeset),
+                                            obj->logical_index, obj->os_index});
+    }
+#endif
+}
+
+
+hwloc_topology *MachineTopology::init_topology()
+{
+#if GKO_HAVE_HWLOC
+    hwloc_topology_t tmp;
+    hwloc_topology_init(&tmp);
+
+    hwloc_topology_set_io_types_filter(tmp, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
+    hwloc_topology_set_xml(tmp, GKO_HWLOC_XMLFILE);
+    hwloc_topology_load(tmp);
+
+    return tmp;
+#else
+    // MSVC complains if there is no return statement.
+    return nullptr;
+#endif
+}
+
+
 }  // namespace gko
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 2d5f35dda1a..26700a832b3 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -156,12 +156,14 @@ TEST(OmpExecutor, IsItsOwnMaster)
 }
 
 
-// TEST(OmpExecutor, CanGetNumCoresFromExecInfo)
-// {
-//     auto omp = gko::OmpExecutor::create();
+#ifdef NUM_CORES
+TEST(OmpExecutor, CanGetNumCoresFromExecInfo)
+{
+    auto omp = gko::OmpExecutor::create();
 
-//     ASSERT_EQ(20, omp->get_num_cores());
-// }
+    ASSERT_EQ(NUM_CORES, omp->get_num_cores());
+}
+#endif
 
 
 TEST(ReferenceExecutor, RunsCorrectOperation)
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index fbbb19088ad..ec36f370b43 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -328,6 +328,8 @@ if(GINKGO_BUILD_TESTS)
             set(GINKGO_RPATH_FOR_HIP "${GINKGO_RPATH_FOR_HIP}:$<TARGET_FILE_DIR:${target}>")
         endif()
     endforeach()
-
+    if(GINKGO_HAVE_HWLOC)
+        set(GINKGO_RPATH_FOR_HIP "${GINKGO_RPATH_FOR_HIP}:${HWLOC_LIBRARIES}")
+    endif()
     add_subdirectory(test)
 endif()
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index 15f24ebc0b6..2ff92199a72 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -69,6 +69,18 @@ namespace gko {
 
 class MachineTopology;
 
+
+/**
+ * This global function initializes the topology in its first call and stores
+ * the MachineTopology global object in a shared_ptr that any gko function/class
+ * can query.
+ *
+ * This makes sure that that the topology object is not re-initialized every
+ * time it is needed and as the topology is is relevant only for the machine,
+ * each re-initialization populates the exact same topology tree.
+ *
+ * @return  the global MachineTopology object.
+ */
 extern const MachineTopology *get_machine_topology();
 
 
@@ -80,6 +92,10 @@ extern const MachineTopology *get_machine_topology();
  * This class also provides functionalities to bind objects in the topology to
  * the execution objects. Binding can enhance performance by allowing data to be
  * closer to the executing object.
+ *
+ * See the hwloc documentation
+ * (https://www.open-mpi.org/projects/hwloc/doc/v2.4.0/) for a more detailed
+ * documentation on topology detection and binding interfaces.
  */
 class MachineTopology {
     friend const MachineTopology *get_machine_topology();
@@ -117,13 +133,9 @@ class MachineTopology {
 
 public:
     /**
-     * Creates a new MachineTopology object.
+     * Do not allow the MachineTopology object to be copied/moved. There should
+     * be only one global object per execution.
      */
-    static std::shared_ptr<MachineTopology> create()
-    {
-        return std::shared_ptr<MachineTopology>(new MachineTopology());
-    }
-
     MachineTopology(MachineTopology &) = delete;
     MachineTopology(MachineTopology &&) = delete;
     MachineTopology &operator=(MachineTopology &) = delete;
@@ -155,7 +167,6 @@ class MachineTopology {
         return this->pus_[id];
     }
 
-
     /**
      * Get the object of type core associated with the id.
      *
@@ -167,7 +178,6 @@ class MachineTopology {
         return this->cores_[id];
     }
 
-
     /**
      * Get the object of type pci device associated with the id.
      *
@@ -179,71 +189,44 @@ class MachineTopology {
         return this->pci_devices_[id];
     }
 
-
     /**
      * Get the number of PU objects stored in this Topology tree.
      */
     size_type get_num_pus() const { return this->pus_.size(); }
 
-
     /**
      * Get the number of core objects stored in this Topology tree.
      */
     size_type get_num_cores() const { return this->cores_.size(); }
 
-
     /**
      * Get the number of PCI device objects stored in this Topology tree.
      */
     size_type get_num_pci_devices() const { return this->pci_devices_.size(); }
 
-
     /**
      * Get the number of NUMA objects stored in this Topology tree.
      */
     size_type get_num_numas() const { return this->num_numas_; }
 
 protected:
-    MachineTopology()
+    /**
+     * Creates a new MachineTopology object.
+     */
+    static std::shared_ptr<MachineTopology> create()
     {
-#if GKO_HAVE_HWLOC
-
-
-        this->topo_ = topo_manager<hwloc_topology>(init_topology(),
-                                                   hwloc_topology_destroy);
-
-        load_objects(HWLOC_OBJ_CORE, this->cores_);
-        load_objects(HWLOC_OBJ_PU, this->pus_);
-        // load_objects(HWLOC_OBJ_PCI_DEVICE, this->pci_devices_);
-
-        num_numas_ =
-            hwloc_get_nbobjs_by_type(this->topo_.get(), HWLOC_OBJ_PACKAGE);
-#else
-
-        this->topo_ = topo_manager<hwloc_topology>();
-
-#endif
+        return std::shared_ptr<MachineTopology>(new MachineTopology());
     }
 
+    MachineTopology();
 
     /**
+     * @internal
+     *
      * A helper function that binds the object with an id.
      */
     void hwloc_binding_helper(std::vector<mach_topo_obj_info> &obj,
-                              size_type id)
-    {
-#if GKO_HAVE_HWLOC
-        auto bitmap = hwloc_bitmap_alloc();
-        hwloc_bitmap_set(bitmap, obj[id].physical_id);
-        hwloc_bitmap_singlify(bitmap);
-        hwloc_set_cpubind(topo_.get(), bitmap, 0);
-        hwloc_bitmap_free(bitmap);
-#endif
-    }
-
-
-#if GKO_HAVE_HWLOC
-
+                              size_type id);
 
     /**
      * @internal
@@ -252,56 +235,25 @@ class MachineTopology {
      * depth. Provided from the hwloc library.
      */
     static void hwloc_print_children(hwloc_topology *topology, hwloc_obj_t obj,
-                                     int depth)
-    {
-        char type[32], attr[1024];
-        unsigned i;
-        hwloc_obj_type_snprintf(type, sizeof(type), obj, 0);
-        std::cout << std::string(2 * depth, ' ') << type;
-        if (obj->os_index != (unsigned)-1) {
-            std::cout << "#" << obj->os_index;
-        }
-        hwloc_obj_attr_snprintf(attr, sizeof(attr), obj, " ", 0);
-        if (*attr) {
-            std::cout << "(" << attr << ")";
-        }
-        std::cout << std::endl;
-        for (i = 0; i < obj->arity; i++) {
-            hwloc_print_children(topology, obj->children[i], depth + 1);
-        }
-    }
+                                     int depth);
 
-
-    // The objects should be sorted by logical index since hwloc uses logical
-    // index with these functions
+    /**
+     * @internal
+     *
+     * Load the objects of a certain HWLOC type.
+     *
+     * @note The objects should be sorted by logical index since hwloc uses
+     * logical index with these functions
+     */
     void load_objects(hwloc_obj_type_t type,
-                      std::vector<mach_topo_obj_info> &vector)
-    {
-        unsigned nbcores = hwloc_get_nbobjs_by_type(this->topo_.get(), type);
-        for (unsigned i = 0; i < nbcores; i++) {
-            hwloc_obj_t obj = hwloc_get_obj_by_type(this->topo_.get(), type, i);
-            vector.push_back(
-                mach_topo_obj_info{obj, hwloc_bitmap_first(obj->nodeset),
-                                   obj->logical_index, obj->os_index});
-        }
-    }
-
-
-    hwloc_topology *init_topology()
-    {
-        hwloc_topology_t tmp;
-        hwloc_topology_init(&tmp);
-
-        hwloc_topology_set_io_types_filter(tmp,
-                                           HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
-        hwloc_topology_set_xml(tmp, GKO_HWLOC_XMLFILE);
-        hwloc_topology_load(tmp);
+                      std::vector<mach_topo_obj_info> &vector);
 
-        return tmp;
-    }
-
-
-#endif
+    /**
+     * @internal
+     *
+     * Initialize the topology object.
+     */
+    hwloc_topology *init_topology();
 
 
 private:

From e4d940f0d7e83cfcc6bcfd625afe2bb7ab2762cc Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 24 Dec 2020 17:50:19 +0100
Subject: [PATCH 0291/1008] Update PCI device detection and some doc.

+ Need to query sub-objects (OS devices) to find the type
  of the object.
+ Main PCI object only holds the PCI Bus ID of the underlying
  objects.
---
 core/devices/machine_topology.cpp             |  76 +++++++++--
 core/devices/omp/executor.cpp                 |   2 +
 core/test/base/executor.cpp                   |  12 +-
 include/ginkgo/core/base/machine_topology.hpp | 122 ++++++++++++++++--
 4 files changed, 186 insertions(+), 26 deletions(-)

diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index f415f76f8fe..2b29732359b 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -73,14 +73,16 @@ MachineTopology::MachineTopology()
     // Initialize the topology from hwloc
     this->topo_ =
         topo_manager<hwloc_topology>(init_topology(), hwloc_topology_destroy);
-
     // load objects of type CORE. See HWLOC_OBJ_TYPE for more details.
     load_objects(HWLOC_OBJ_CORE, this->cores_);
-    // load objects of type PU. See HWLOC_OBJ_TYPE for more details.
+    // load objects of type processing unit(PU). See HWLOC_OBJ_TYPE for more
+    // details.
     load_objects(HWLOC_OBJ_PU, this->pus_);
-    // load_objects(HWLOC_OBJ_PCI_DEVICE, this->pci_devices_);
-
+    // load objects of type PCI Devices See HWLOC_OBJ_TYPE for more
+    // details.
+    load_objects(HWLOC_OBJ_PCI_DEVICE, this->pci_devices_);
     num_numas_ = hwloc_get_nbobjs_by_type(this->topo_.get(), HWLOC_OBJ_PACKAGE);
+
 #else
 
     this->topo_ = topo_manager<hwloc_topology>();
@@ -89,7 +91,8 @@ MachineTopology::MachineTopology()
 }
 
 
-void MachineTopology::hwloc_binding_helper(std::vector<mach_topo_obj_info> &obj,
+template <typename ObjInfoType>
+void MachineTopology::hwloc_binding_helper(std::vector<ObjInfoType> &obj,
                                            size_type id)
 {
 #if GKO_HAVE_HWLOC
@@ -125,16 +128,65 @@ void MachineTopology::hwloc_print_children(hwloc_topology *topology,
 }
 
 
-void MachineTopology::load_objects(hwloc_obj_type_t type,
-                                   std::vector<mach_topo_obj_info> &vector)
+void MachineTopology::load_objects(
+    hwloc_obj_type_t type,
+    std::vector<MachineTopology::normal_obj_info> &vector)
+{
+#if GKO_HAVE_HWLOC
+    // Get the number of normal objects of a certain type (Core, PU, Machine
+    // etc.).
+    unsigned nbcores = hwloc_get_nbobjs_by_type(this->topo_.get(), type);
+    for (unsigned i = 0; i < nbcores; i++) {
+        // Get the actual normal object of the given type.
+        hwloc_obj_t obj = hwloc_get_obj_by_type(this->topo_.get(), type, i);
+        vector.push_back(normal_obj_info{obj, hwloc_bitmap_first(obj->nodeset),
+                                         obj->logical_index, obj->os_index,
+                                         obj->gp_index});
+    }
+#endif
+}
+
+
+void MachineTopology::load_objects(
+    hwloc_obj_type_t type, std::vector<MachineTopology::io_obj_info> &vector)
 {
 #if GKO_HAVE_HWLOC
     unsigned nbcores = hwloc_get_nbobjs_by_type(this->topo_.get(), type);
     for (unsigned i = 0; i < nbcores; i++) {
+        // Get the actual PCI object.
         hwloc_obj_t obj = hwloc_get_obj_by_type(this->topo_.get(), type, i);
-        vector.push_back(mach_topo_obj_info{obj,
-                                            hwloc_bitmap_first(obj->nodeset),
-                                            obj->logical_index, obj->os_index});
+        // Get the non-IO ancestor (which is the closest and the one that can be
+        // bound to) of the object.
+        auto ancestor = hwloc_get_non_io_ancestor_obj(this->topo_.get(), obj);
+        vector.push_back(io_obj_info{obj, obj->logical_index, obj->os_index,
+                                     obj->gp_index, ancestor,
+                                     hwloc_bitmap_first(ancestor->nodeset)});
+        // Write the PCI Bus ID from the object info.
+        char pci_busid[14];
+        snprintf(pci_busid, sizeof(pci_busid), "%04x:%02x:%02x.%01x",
+                 obj->attr->pcidev.domain, obj->attr->pcidev.bus,
+                 obj->attr->pcidev.dev, obj->attr->pcidev.func);
+        vector.back().pci_busid = std::string(pci_busid);
+        // Get the number of IO children if any. For example, the software OS
+        // devices (cuda, rsmi, ib) are listed as IO devices under a PCI device.
+        auto num_io_children = obj->io_arity;
+        hwloc_obj_t os_child{};
+        auto rem_io_children = num_io_children;
+        // Get the IO childrens and their names and store them in vectors.
+        if (num_io_children > 0) {
+            os_child = obj->io_first_child;
+            vector.back().io_children.push_back(os_child);
+            vector.back().io_children_name.push_back(os_child->name);
+            rem_io_children -= 1;
+            while (rem_io_children >= 1) {
+                auto os_child_2 = os_child->next_cousin;
+                vector.back().io_children.push_back(os_child_2);
+                vector.back().io_children_name.push_back(os_child_2->name);
+                rem_io_children -= 1;
+            }
+            GKO_ASSERT(vector.back().io_children.size() ==
+                       vector.back().io_children_name.size());
+        }
     }
 #endif
 }
@@ -147,6 +199,10 @@ hwloc_topology *MachineTopology::init_topology()
     hwloc_topology_init(&tmp);
 
     hwloc_topology_set_io_types_filter(tmp, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
+    hwloc_topology_set_type_filter(tmp, HWLOC_OBJ_BRIDGE,
+                                   HWLOC_TYPE_FILTER_KEEP_NONE);
+    hwloc_topology_set_type_filter(tmp, HWLOC_OBJ_OS_DEVICE,
+                                   HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
     hwloc_topology_set_xml(tmp, GKO_HWLOC_XMLFILE);
     hwloc_topology_load(tmp);
 
diff --git a/core/devices/omp/executor.cpp b/core/devices/omp/executor.cpp
index 3a268be3c18..9116f22ad89 100644
--- a/core/devices/omp/executor.cpp
+++ b/core/devices/omp/executor.cpp
@@ -47,6 +47,8 @@ namespace gko {
 void OmpExecutor::populate_exec_info(const MachineTopology *mach_topo)
 {
     this->omp_exec_info_.num_cores = mach_topo->get_num_cores();
+    this->omp_exec_info_.num_work_groups_per_core =
+        mach_topo->get_num_pus() / mach_topo->get_num_cores();
 }
 
 
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 26700a832b3..09cae94eb15 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
+#include <thread>
 #include <type_traits>
 
 
@@ -156,14 +157,17 @@ TEST(OmpExecutor, IsItsOwnMaster)
 }
 
 
-#ifdef NUM_CORES
-TEST(OmpExecutor, CanGetNumCoresFromExecInfo)
+TEST(OmpExecutor, CanGetNumCpusFromExecInfo)
 {
+    if (!gko::get_machine_topology()) {
+        GTEST_SKIP() << "No Machine topology information available";
+    }
     auto omp = gko::OmpExecutor::create();
 
-    ASSERT_EQ(NUM_CORES, omp->get_num_cores());
+    auto num_cpus = omp->get_num_cores() * omp->get_num_threads_per_core();
+
+    ASSERT_EQ(std::thread::hardware_concurrency(), num_cpus);
 }
-#endif
 
 
 TEST(ReferenceExecutor, RunsCorrectOperation)
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index 2ff92199a72..b69173d0b99 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -96,15 +96,20 @@ extern const MachineTopology *get_machine_topology();
  * See the hwloc documentation
  * (https://www.open-mpi.org/projects/hwloc/doc/v2.4.0/) for a more detailed
  * documentation on topology detection and binding interfaces.
+ *
+ * @note A global object of MachineTopology type is created when first needed
+ * and only destroyed at the end of the program. This means that any subsequent
+ * queries will be from the same global object and hence atomic on
+ * multi-threaded queries.
  */
 class MachineTopology {
     friend const MachineTopology *get_machine_topology();
 
 private:
     /**
-     * This struct holds the attributes for an object.
+     * This struct holds the attributes for a normal non-IO object.
      */
-    struct mach_topo_obj_info {
+    struct normal_obj_info {
         /**
          * The hwloc object.
          */
@@ -126,11 +131,80 @@ class MachineTopology {
         size_type physical_id;
 
         /**
-         * The physical_id assigned to the object.
+         * The global persistent id assigned to the object by hwloc.
+         */
+        size_type gp_id;
+
+        /**
+         * The memory size of the object.
          */
         size_type memory_size;
     };
 
+
+    /**
+     * This struct holds the attributes for an IO/Misc object.
+     *
+     * Mainly used for PCI devices. The identifier important for PCI devices is
+     * the PCI Bus ID, stored here as a string. PCI devices themselves usually
+     * contain Hard disks, network components as well as other objects that are
+     * not important for our case.
+     *
+     * In many cases, hwloc is able to identify the OS devices that belong to a
+     * certain PCI Bus ID and here they are stored in the io children vector. A
+     * list of their names are also additionally stored for easy access and
+     * comparison.
+     *
+     * @note IO children can have names such as ibX for Infiniband cards, cudaX
+     * for NVIDIA cards with CUDA and rsmiX for AMD cards.
+     */
+    struct io_obj_info {
+        /**
+         * The hwloc object.
+         */
+        hwloc_obj_t obj;
+
+        /**
+         * The logical_id assigned by the OS.
+         */
+        size_type logical_id;
+
+        /**
+         * The physical_id assigned to the object.
+         */
+        size_type physical_id;
+
+        /**
+         * The global persistent id assigned to the object by hwloc.
+         */
+        size_type gp_id;
+
+        /**
+         * The non-io parent object.
+         */
+        hwloc_obj_t non_io_ancestor;
+
+        /**
+         * The closest numa.
+         */
+        int numa;
+
+        /**
+         * The io children objects (usually software OS devices).
+         */
+        std::vector<hwloc_obj_t> io_children;
+
+        /**
+         * The names of the io children objects.
+         */
+        std::vector<std::string> io_children_name;
+
+        /**
+         * The PCI Bus ID
+         */
+        std::string pci_busid;
+    };
+
 public:
     /**
      * Do not allow the MachineTopology object to be copied/moved. There should
@@ -160,8 +234,9 @@ class MachineTopology {
      * Get the object of type PU associated with the id.
      *
      * @param id  The id of the PU
+     * @return  the PU object struct.
      */
-    const mach_topo_obj_info &get_pu(size_type id)
+    const normal_obj_info &get_pu(size_type id)
     {
         GKO_ENSURE_IN_BOUNDS(id, this->pus_.size());
         return this->pus_[id];
@@ -171,8 +246,9 @@ class MachineTopology {
      * Get the object of type core associated with the id.
      *
      * @param id  The id of the core
+     * @return  the core object struct.
      */
-    const mach_topo_obj_info &get_core(size_type id)
+    const normal_obj_info &get_core(size_type id)
     {
         GKO_ENSURE_IN_BOUNDS(id, this->cores_.size());
         return this->cores_[id];
@@ -182,8 +258,9 @@ class MachineTopology {
      * Get the object of type pci device associated with the id.
      *
      * @param id  The id of the pci device
+     * @return  the PCI object struct.
      */
-    const mach_topo_obj_info &get_pci_device(size_type id)
+    const io_obj_info &get_pci_device(size_type id)
     {
         GKO_ENSURE_IN_BOUNDS(id, this->pci_devices_.size());
         return this->pci_devices_[id];
@@ -191,21 +268,29 @@ class MachineTopology {
 
     /**
      * Get the number of PU objects stored in this Topology tree.
+     *
+     * @return  the number of PUs.
      */
     size_type get_num_pus() const { return this->pus_.size(); }
 
     /**
      * Get the number of core objects stored in this Topology tree.
+     *
+     * @return  the number of cores.
      */
     size_type get_num_cores() const { return this->cores_.size(); }
 
     /**
      * Get the number of PCI device objects stored in this Topology tree.
+     *
+     * @return  the number of PCI devices.
      */
     size_type get_num_pci_devices() const { return this->pci_devices_.size(); }
 
     /**
      * Get the number of NUMA objects stored in this Topology tree.
+     *
+     * @return  the number of NUMA objects.
      */
     size_type get_num_numas() const { return this->num_numas_; }
 
@@ -218,6 +303,9 @@ class MachineTopology {
         return std::shared_ptr<MachineTopology>(new MachineTopology());
     }
 
+    /**
+     * The default constructor that loads the different objects.
+     */
     MachineTopology();
 
     /**
@@ -225,8 +313,8 @@ class MachineTopology {
      *
      * A helper function that binds the object with an id.
      */
-    void hwloc_binding_helper(std::vector<mach_topo_obj_info> &obj,
-                              size_type id);
+    template <typename ObjInfoType>
+    void hwloc_binding_helper(std::vector<ObjInfoType> &obj, size_type id);
 
     /**
      * @internal
@@ -246,7 +334,17 @@ class MachineTopology {
      * logical index with these functions
      */
     void load_objects(hwloc_obj_type_t type,
-                      std::vector<mach_topo_obj_info> &vector);
+                      std::vector<normal_obj_info> &vector);
+
+    /**
+     * @internal
+     *
+     * Load the objects of a certain HWLOC type.
+     *
+     * @note The objects should be sorted by logical index since hwloc uses
+     * logical index with these functions
+     */
+    void load_objects(hwloc_obj_type_t type, std::vector<io_obj_info> &vector);
 
     /**
      * @internal
@@ -257,9 +355,9 @@ class MachineTopology {
 
 
 private:
-    std::vector<mach_topo_obj_info> pus_;
-    std::vector<mach_topo_obj_info> cores_;
-    std::vector<mach_topo_obj_info> pci_devices_;
+    std::vector<normal_obj_info> pus_;
+    std::vector<normal_obj_info> cores_;
+    std::vector<io_obj_info> pci_devices_;
     size_type num_numas_;
 
     template <typename T>

From c38f02f69b26ebf7b0fd270a27d0e473a037c3b7 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 29 Dec 2020 01:14:10 +0100
Subject: [PATCH 0292/1008] Get PCI Bus ID info from MachTopo.

+ Add interface for building complete topo information.
---
 core/devices/machine_topology.cpp             | 211 ++++++++++++++----
 cuda/base/executor.cpp                        |  52 +----
 hip/base/executor.hip.cpp                     |  11 +
 include/ginkgo/core/base/executor.hpp         |   6 +
 include/ginkgo/core/base/machine_topology.hpp | 154 +++++++++----
 5 files changed, 315 insertions(+), 119 deletions(-)

diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index 2b29732359b..52b5054dc76 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -66,63 +66,118 @@ const MachineTopology *get_machine_topology()
 }
 
 
+const MachineTopology::io_obj_info *MachineTopology::get_pci_device(
+    std::string &pci_bus_id) const
+{
+    for (auto id = 0; id < this->pci_devices_.size(); ++id) {
+        if (this->pci_devices_[id].pci_bus_id.compare(0, 12, pci_bus_id, 0,
+                                                      12) == 0) {
+            return &this->pci_devices_[id];
+        }
+    }
+    return nullptr;
+}
+
+
 MachineTopology::MachineTopology()
 {
 #if GKO_HAVE_HWLOC
 
     // Initialize the topology from hwloc
     this->topo_ =
-        topo_manager<hwloc_topology>(init_topology(), hwloc_topology_destroy);
-    // load objects of type CORE. See HWLOC_OBJ_TYPE for more details.
+        hwloc_manager<hwloc_topology>(init_topology(), hwloc_topology_destroy);
+    // load objects of type Package . See HWLOC_OBJ_PACKAGE for more details.
+    load_objects(HWLOC_OBJ_PACKAGE, this->packages_);
+    // load objects of type NUMA Node. See HWLOC_OBJ_NUMANODE for more details.
+    load_objects(HWLOC_OBJ_NUMANODE, this->numa_nodes_);
+    // load objects of type Core. See HWLOC_OBJ_CORE for more details.
     load_objects(HWLOC_OBJ_CORE, this->cores_);
-    // load objects of type processing unit(PU). See HWLOC_OBJ_TYPE for more
+    // load objects of type processing unit(PU). See HWLOC_OBJ_PU for more
     // details.
     load_objects(HWLOC_OBJ_PU, this->pus_);
-    // load objects of type PCI Devices See HWLOC_OBJ_TYPE for more
+    // load objects of type PCI Devices See HWLOC_OBJ_PCI_DEVICE for more
     // details.
     load_objects(HWLOC_OBJ_PCI_DEVICE, this->pci_devices_);
     num_numas_ = hwloc_get_nbobjs_by_type(this->topo_.get(), HWLOC_OBJ_PACKAGE);
 
 #else
 
-    this->topo_ = topo_manager<hwloc_topology>();
+    this->topo_ = hwloc_manager<hwloc_topology>();
 
 #endif
 }
 
 
-template <typename ObjInfoType>
-void MachineTopology::hwloc_binding_helper(std::vector<ObjInfoType> &obj,
-                                           size_type id)
+void MachineTopology::hwloc_binding_helper(
+    const std::tuple<std::vector<MachineTopology::normal_obj_info>,
+                     MachineTopology::hwloc_manager<hwloc_bitmap_s>> &obj,
+    const int *id, const size_type num_ids,
+    const MachineTopology::BitMapType &bitmap_type) const
 {
 #if GKO_HAVE_HWLOC
-    auto bitmap = hwloc_bitmap_alloc();
-    hwloc_bitmap_set(bitmap, obj[id].physical_id);
-    hwloc_bitmap_singlify(bitmap);
-    hwloc_set_cpubind(topo_.get(), bitmap, 0);
-    hwloc_bitmap_free(bitmap);
+    auto bitmap_cur = hwloc_bitmap_alloc();
+    auto bitmap_toset = hwloc_bitmap_alloc();
+    auto bitmap_res = hwloc_bitmap_alloc();
+    // Get current bitmap
+    hwloc_get_cpubind(this->topo_.get(), bitmap_cur, 0);
+    // Set the given ids to a new bitmap
+    for (auto i = 0; i < num_ids; ++i) {
+        GKO_ASSERT(id[i] < std::get<0>(obj).size());
+        GKO_ASSERT(id[i] >= 0);
+        hwloc_bitmap_set(bitmap_toset, std::get<0>(obj)[id[i]].os_id);
+        // Check if this id has been bound before, Binding multiple threads to
+        // same object may not be suitable.
+        if (hwloc_bitmap_isset(std::get<1>(obj).get(),
+                               std::get<0>(obj)[id[i]].os_id)) {
+            hwloc_bitmap_clr(bitmap_toset, std::get<0>(obj)[id[i]].os_id);
+        }
+    }
+    // Log the bound ids to the global bitmap.
+    hwloc_bitmap_and(std::get<1>(obj).get(), std::get<1>(obj).get(),
+                     bitmap_toset);
+    // Depending on the passed in BitMapType, do the binary operation
+    // Op(current_bitmap, toset_bitmap) and use the resulting bitmap to bind the
+    // cpu.
+    if (bitmap_type == BitMapType::bitmap_set) {
+        hwloc_bitmap_free(bitmap_res);
+        bitmap_res = hwloc_bitmap_dup(bitmap_toset);
+    } else if (bitmap_type == BitMapType::bitmap_or) {
+        hwloc_bitmap_or(bitmap_res, bitmap_cur, bitmap_toset);
+    } else if (bitmap_type == BitMapType::bitmap_and) {
+        hwloc_bitmap_and(bitmap_res, bitmap_cur, bitmap_toset);
+    } else if (bitmap_type == BitMapType::bitmap_andnot) {
+        hwloc_bitmap_andnot(bitmap_res, bitmap_cur, bitmap_toset);
+    }
+
+    // Singlify to reduce expensive migrations.
+    hwloc_bitmap_singlify(bitmap_res);
+    hwloc_set_cpubind(this->topo_.get(), bitmap_res, 0);
+    hwloc_bitmap_free(bitmap_cur);
+    hwloc_bitmap_free(bitmap_toset);
+    hwloc_bitmap_free(bitmap_res);
 #endif
 }
 
 
-void MachineTopology::hwloc_print_children(hwloc_topology *topology,
-                                           hwloc_obj_t obj, int depth)
+void MachineTopology::hwloc_print_children(const hwloc_obj_t obj,
+                                           const int depth)
 {
 #if GKO_HAVE_HWLOC
+    auto topology = this->topo_.get();
     char type[32], attr[1024];
     unsigned i;
     hwloc_obj_type_snprintf(type, sizeof(type), obj, 0);
-    std::cout << std::string(2 * depth, ' ') << type;
+    std::clog << std::string(2 * depth, ' ') << type;
     if (obj->os_index != (unsigned)-1) {
-        std::cout << "#" << obj->os_index;
+        std::clog << "#" << obj->os_index;
     }
     hwloc_obj_attr_snprintf(attr, sizeof(attr), obj, " ", 0);
     if (*attr) {
-        std::cout << "(" << attr << ")";
+        std::clog << "(" << attr << ")";
     }
-    std::cout << std::endl;
+    std::clog << std::endl;
     for (i = 0; i < obj->arity; i++) {
-        hwloc_print_children(topology, obj->children[i], depth + 1);
+        hwloc_print_children(obj->children[i], depth + 1);
     }
 #endif
 }
@@ -130,19 +185,50 @@ void MachineTopology::hwloc_print_children(hwloc_topology *topology,
 
 void MachineTopology::load_objects(
     hwloc_obj_type_t type,
-    std::vector<MachineTopology::normal_obj_info> &vector)
+    std::tuple<std::vector<MachineTopology::normal_obj_info>,
+               MachineTopology::hwloc_manager<hwloc_bitmap_s>> &objects)
 {
 #if GKO_HAVE_HWLOC
+    std::get<1>(objects) =
+        hwloc_manager<hwloc_bitmap_s>(init_bitmap(), hwloc_bitmap_free);
     // Get the number of normal objects of a certain type (Core, PU, Machine
     // etc.).
-    unsigned nbcores = hwloc_get_nbobjs_by_type(this->topo_.get(), type);
-    for (unsigned i = 0; i < nbcores; i++) {
+    unsigned num_objects = hwloc_get_nbobjs_by_type(this->topo_.get(), type);
+    for (unsigned i = 0; i < num_objects; i++) {
         // Get the actual normal object of the given type.
         hwloc_obj_t obj = hwloc_get_obj_by_type(this->topo_.get(), type, i);
-        vector.push_back(normal_obj_info{obj, hwloc_bitmap_first(obj->nodeset),
-                                         obj->logical_index, obj->os_index,
-                                         obj->gp_index});
+        std::get<0>(objects).push_back(
+            normal_obj_info{obj, obj->logical_index, obj->os_index,
+                            obj->gp_index, hwloc_bitmap_first(obj->nodeset)});
+    }
+#endif
+}
+
+
+inline int MachineTopology::get_obj_local_id_by_os_index(
+    std::vector<MachineTopology::normal_obj_info> &objects, int os_index)
+{
+#if GKO_HAVE_HWLOC
+    for (auto id = 0; id < objects.size(); ++id) {
+        if (objects[id].os_id == os_index) {
+            return id;
+        }
     }
+    return -1;
+#endif
+}
+
+
+inline int MachineTopology::get_obj_local_id_by_gp_index(
+    std::vector<MachineTopology::normal_obj_info> &objects, int gp_index)
+{
+#if GKO_HAVE_HWLOC
+    for (auto id = 0; id < objects.size(); ++id) {
+        if (objects[id].gp_id == gp_index) {
+            return id;
+        }
+    }
+    return -1;
 #endif
 }
 
@@ -151,37 +237,70 @@ void MachineTopology::load_objects(
     hwloc_obj_type_t type, std::vector<MachineTopology::io_obj_info> &vector)
 {
 #if GKO_HAVE_HWLOC
-    unsigned nbcores = hwloc_get_nbobjs_by_type(this->topo_.get(), type);
-    for (unsigned i = 0; i < nbcores; i++) {
+    GKO_ASSERT(std::get<0>(this->cores_).size() != 0);
+    GKO_ASSERT(std::get<0>(this->pus_).size() != 0);
+    unsigned num_objects = hwloc_get_nbobjs_by_type(this->topo_.get(), type);
+    for (unsigned i = 0; i < num_objects; i++) {
         // Get the actual PCI object.
         hwloc_obj_t obj = hwloc_get_obj_by_type(this->topo_.get(), type, i);
         // Get the non-IO ancestor (which is the closest and the one that can be
         // bound to) of the object.
         auto ancestor = hwloc_get_non_io_ancestor_obj(this->topo_.get(), obj);
-        vector.push_back(io_obj_info{obj, obj->logical_index, obj->os_index,
-                                     obj->gp_index, ancestor,
-                                     hwloc_bitmap_first(ancestor->nodeset)});
+        // Create the object.
+        vector.push_back(
+            io_obj_info{obj, obj->logical_index, obj->os_index, obj->gp_index,
+                        hwloc_bitmap_first(ancestor->nodeset), ancestor});
+        // Get the corresponding cpuset of the ancestor nodeset
+        hwloc_cpuset_t ancestor_cpuset = hwloc_bitmap_alloc();
+        hwloc_cpuset_from_nodeset(this->topo_.get(), ancestor_cpuset,
+                                  ancestor->nodeset);
+        // Find the cpu object closest to this device from the ancestor cpuset
+        // and store its id for binding purposes
+        vector.back().closest_cpu_id = get_obj_local_id_by_os_index(
+            std::get<0>(this->pus_), hwloc_bitmap_first(ancestor_cpuset));
+        // Get local id of the ancestor object.
+        if (hwloc_compare_types(ancestor->type, HWLOC_OBJ_PACKAGE) == 0) {
+            vector.back().ancestor_local_id = get_obj_local_id_by_gp_index(
+                std::get<0>(this->packages_), ancestor->gp_index);
+        } else if (hwloc_compare_types(ancestor->type, HWLOC_OBJ_CORE) == 0) {
+            vector.back().ancestor_local_id = get_obj_local_id_by_gp_index(
+                std::get<0>(this->cores_), ancestor->gp_index);
+        } else if (hwloc_compare_types(ancestor->type, HWLOC_OBJ_NUMANODE) ==
+                   0) {
+            vector.back().ancestor_local_id = get_obj_local_id_by_gp_index(
+                std::get<0>(this->numa_nodes_), ancestor->gp_index);
+        }
+        hwloc_bitmap_free(ancestor_cpuset);
+        // Get type of the ancestor object and store it as a string.
+        char ances_type[24];
+        hwloc_obj_type_snprintf(ances_type, sizeof(ances_type), ancestor, 0);
+        vector.back().ancestor_type = std::string(ances_type);
         // Write the PCI Bus ID from the object info.
-        char pci_busid[14];
-        snprintf(pci_busid, sizeof(pci_busid), "%04x:%02x:%02x.%01x",
+        char pci_bus_id[13];
+        snprintf(pci_bus_id, sizeof(pci_bus_id), "%04x:%02x:%02x.%01x",
                  obj->attr->pcidev.domain, obj->attr->pcidev.bus,
                  obj->attr->pcidev.dev, obj->attr->pcidev.func);
-        vector.back().pci_busid = std::string(pci_busid);
-        // Get the number of IO children if any. For example, the software OS
-        // devices (cuda, rsmi, ib) are listed as IO devices under a PCI device.
+        vector.back().pci_bus_id = std::string(pci_bus_id);
+        // Get the number of IO children if any. For example, the software
+        // OS devices (cuda, rsmi, ib) are listed as IO devices under a PCI
+        // device.
         auto num_io_children = obj->io_arity;
         hwloc_obj_t os_child{};
         auto rem_io_children = num_io_children;
-        // Get the IO childrens and their names and store them in vectors.
+        // Get the IO children and their names and store them in vectors.
         if (num_io_children > 0) {
+            // Get the first child
             os_child = obj->io_first_child;
             vector.back().io_children.push_back(os_child);
             vector.back().io_children_name.push_back(os_child->name);
             rem_io_children -= 1;
+            // If the io obj has more than one children, they are siblings to
+            // the first child. Store these as well.
+            hwloc_obj_t os_child_n = os_child;
             while (rem_io_children >= 1) {
-                auto os_child_2 = os_child->next_cousin;
-                vector.back().io_children.push_back(os_child_2);
-                vector.back().io_children_name.push_back(os_child_2->name);
+                os_child_n = os_child_n->next_sibling;
+                vector.back().io_children.push_back(os_child_n);
+                vector.back().io_children_name.push_back(os_child_n->name);
                 rem_io_children -= 1;
             }
             GKO_ASSERT(vector.back().io_children.size() ==
@@ -192,6 +311,18 @@ void MachineTopology::load_objects(
 }
 
 
+hwloc_bitmap_s *MachineTopology::init_bitmap()
+{
+#if GKO_HAVE_HWLOC
+    hwloc_bitmap_t tmp = hwloc_bitmap_alloc();
+    return tmp;
+#else
+    // MSVC complains if there is no return statement.
+    return nullptr;
+#endif
+}
+
+
 hwloc_topology *MachineTopology::init_topology()
 {
 #if GKO_HAVE_HWLOC
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 3434c730451..2eb0d997403 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -54,47 +54,6 @@ namespace gko {
 #include "common/base/executor.hpp.inc"
 
 
-// namespace machine_config {
-
-
-// template <>
-// void Topology<CudaExecutor>::load_gpus()
-// {
-// #if GKO_HAVE_HWLOC
-//     size_type num_in_numa = 0;
-//     int last_numa = 0;
-//     auto topology = this->topo_.get();
-//     auto n_objs = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_OS_DEVICE);
-//     for (size_type i = 1; i < n_objs; i++, num_in_numa++) {
-//         hwloc_obj_t obj = NULL;
-//         while ((obj = hwloc_get_next_osdev(topology, obj)) != NULL) {
-//             bool is_gpu = HWLOC_OBJ_OSDEV_COPROC == obj->attr->osdev.type &&
-//                           obj->name && !strncmp("cuda", obj->name, 4) &&
-//                           atoi(obj->name + 4) == (int)i;
-//             if (is_gpu) {
-//                 while (obj &&
-//                        (!obj->nodeset || hwloc_bitmap_iszero(obj->nodeset)))
-//                     obj = obj->parent;
-//                 if (obj && obj->nodeset) {
-//                     auto this_numa = hwloc_bitmap_first(obj->nodeset);
-//                     if (this_numa != last_numa) {
-//                         num_in_numa = 0;
-//                     }
-//                     this->gpus_.push_back(
-//                         topology_obj_info{obj, this_numa, i, num_in_numa});
-//                     last_numa = this_numa;
-//                 }
-//             }
-//         }
-//     }
-
-// #endif
-// }
-
-
-// }  // namespace machine_config
-
-
 std::shared_ptr<CudaExecutor> CudaExecutor::create(
     int device_id, std::shared_ptr<Executor> master, bool device_reset)
 {
@@ -114,6 +73,17 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
 void CudaExecutor::populate_exec_info(const MachineTopology *mach_topo)
 {
     this->cuda_exec_info_.num_cores = mach_topo->get_num_cores();
+    cuda::device_guard g(this->get_device_id());
+    GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetPCIBusId(
+        const_cast<char *>(this->cuda_exec_info_.pci_bus_id.data()), 13,
+        this->get_device_id()));
+
+    auto cuda_hwloc_obj =
+        mach_topo->get_pci_device(this->cuda_exec_info_.pci_bus_id);
+    if (cuda_hwloc_obj) {
+        this->cuda_exec_info_.numa = cuda_hwloc_obj->numa;
+        this->cuda_exec_info_.closest_cpu_id = cuda_hwloc_obj->closest_cpu_id;
+    }
 }
 
 
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index a7e5e94bffd..1c8a0762c43 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -119,6 +119,17 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
 void HipExecutor::populate_exec_info(const MachineTopology *mach_topo)
 {
     this->hip_exec_info_.num_cores = mach_topo->get_num_cores();
+    hip::device_guard g(this->get_device_id());
+    GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetPCIBusId(
+        const_cast<char *>(this->hip_exec_info_.pci_bus_id.data()), 13,
+        this->get_device_id()));
+
+    auto hip_hwloc_obj =
+        mach_topo->get_pci_device(this->hip_exec_info_.pci_bus_id);
+    if (hip_hwloc_obj) {
+        this->hip_exec_info_.numa = hip_hwloc_obj->numa;
+        this->hip_exec_info_.closest_cpu_id = hip_hwloc_obj->closest_cpu_id;
+    }
 }
 
 
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 0a520e70580..f2f650bfeab 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -1263,6 +1263,9 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         int major;
         int minor;
         int warp_size;
+        std::string pci_bus_id = std::string(13, 'x');
+        int numa = -1;
+        int closest_cpu_id = -1;
     } cuda_exec_info_;
 
     template <typename T>
@@ -1457,6 +1460,9 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         int major = 0;
         int minor = 0;
         int warp_size = 0;
+        std::string pci_bus_id = std::string(13, 'x');
+        int numa = -1;
+        int closest_cpu_id = -1;
     } hip_exec_info_;
 
     template <typename T>
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index b69173d0b99..6f77c485a32 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -62,6 +62,7 @@ struct hwloc_obj_t {};
 
 
 struct hwloc_topology;
+struct hwloc_bitmap_s;
 
 
 namespace gko {
@@ -105,6 +106,9 @@ extern const MachineTopology *get_machine_topology();
 class MachineTopology {
     friend const MachineTopology *get_machine_topology();
 
+    template <typename T>
+    using hwloc_manager = std::unique_ptr<T, std::function<void(T *)>>;
+
 private:
     /**
      * This struct holds the attributes for a normal non-IO object.
@@ -116,25 +120,29 @@ class MachineTopology {
         hwloc_obj_t obj;
 
         /**
-         * The numa number of the object.
-         */
-        int numa;
-
-        /**
-         * The logical_id assigned by the OS.
+         * The logical_id assigned by HWLOC (assigned according to physical
+         * proximity).
+         *
+         * @note Use this rather than os_id for all purposes other than binding.
+         * [Reference](https://www.open-mpi.org/projects/hwloc/doc/v2.4.0/a00364.php#faq_indexes)
          */
         size_type logical_id;
 
         /**
-         * The physical_id assigned to the object.
+         * The os_id assigned by the OS (assigned arbitrarily by the OS)
          */
-        size_type physical_id;
+        size_type os_id;
 
         /**
          * The global persistent id assigned to the object by hwloc.
          */
         size_type gp_id;
 
+        /**
+         * The numa number of the object.
+         */
+        int numa;
+
         /**
          * The memory size of the object.
          */
@@ -165,29 +173,47 @@ class MachineTopology {
         hwloc_obj_t obj;
 
         /**
-         * The logical_id assigned by the OS.
+         * The logical_id assigned by HWLOC (assigned according to proximity).
+         *
+         * @note Use this rather than os_id for all purposes other than binding.
+         * [Reference](https://www.open-mpi.org/projects/hwloc/doc/v2.4.0/a00364.php#faq_indexes)
          */
         size_type logical_id;
 
         /**
-         * The physical_id assigned to the object.
+         * The os_id assigned by the OS (assigned arbitrarily by the OS)
          */
-        size_type physical_id;
+        size_type os_id;
 
         /**
          * The global persistent id assigned to the object by hwloc.
          */
         size_type gp_id;
 
+        /**
+         * The closest numa.
+         */
+        int numa;
+
         /**
          * The non-io parent object.
          */
         hwloc_obj_t non_io_ancestor;
 
         /**
-         * The closest numa.
+         * The ancestor local id.
          */
-        int numa;
+        int ancestor_local_id;
+
+        /**
+         * The ancestor type.
+         */
+        std::string ancestor_type;
+
+        /**
+         * The closest CPU.
+         */
+        int closest_cpu_id;
 
         /**
          * The io children objects (usually software OS devices).
@@ -202,7 +228,7 @@ class MachineTopology {
         /**
          * The PCI Bus ID
          */
-        std::string pci_busid;
+        std::string pci_bus_id;
     };
 
 public:
@@ -216,19 +242,31 @@ class MachineTopology {
     MachineTopology &operator=(MachineTopology &&) = delete;
     ~MachineTopology() = default;
 
+    enum class BitMapType { bitmap_set, bitmap_or, bitmap_and, bitmap_andnot };
+
     /**
      * Bind the object associated with the id to a core.
      *
      * @param id  The id of the object to be bound.
      */
-    void bind_to_core(size_type id) { hwloc_binding_helper(this->cores_, id); }
+    void bind_to_cores(
+        const int *ids, const size_type num_ids,
+        const BitMapType bitmap_type = BitMapType::bitmap_set) const
+    {
+        hwloc_binding_helper(this->cores_, ids, num_ids, bitmap_type);
+    }
 
     /**
      * Bind the object associated with the id to a Processing unit(PU).
      *
      * @param id  The id of the object to be bound.
      */
-    void bind_to_pu(size_type id) { hwloc_binding_helper(this->pus_, id); }
+    void bind_to_pus(
+        const int *ids, const size_type num_ids,
+        const BitMapType bitmap_type = BitMapType::bitmap_set) const
+    {
+        hwloc_binding_helper(this->pus_, ids, num_ids, bitmap_type);
+    }
 
     /**
      * Get the object of type PU associated with the id.
@@ -236,10 +274,10 @@ class MachineTopology {
      * @param id  The id of the PU
      * @return  the PU object struct.
      */
-    const normal_obj_info &get_pu(size_type id)
+    const normal_obj_info *get_pu(size_type id) const
     {
-        GKO_ENSURE_IN_BOUNDS(id, this->pus_.size());
-        return this->pus_[id];
+        GKO_ENSURE_IN_BOUNDS(id, std::get<0>(this->pus_).size());
+        return &std::get<0>(this->pus_)[id];
     }
 
     /**
@@ -248,10 +286,10 @@ class MachineTopology {
      * @param id  The id of the core
      * @return  the core object struct.
      */
-    const normal_obj_info &get_core(size_type id)
+    const normal_obj_info *get_core(size_type id) const
     {
-        GKO_ENSURE_IN_BOUNDS(id, this->cores_.size());
-        return this->cores_[id];
+        GKO_ENSURE_IN_BOUNDS(id, std::get<0>(this->cores_).size());
+        return &std::get<0>(this->cores_)[id];
     }
 
     /**
@@ -260,25 +298,33 @@ class MachineTopology {
      * @param id  The id of the pci device
      * @return  the PCI object struct.
      */
-    const io_obj_info &get_pci_device(size_type id)
+    const io_obj_info *get_pci_device(size_type id) const
     {
         GKO_ENSURE_IN_BOUNDS(id, this->pci_devices_.size());
-        return this->pci_devices_[id];
+        return &this->pci_devices_[id];
     }
 
+    /**
+     * Get the object of type pci device associated with the PCI bus id.
+     *
+     * @param pci_bus_id  The PCI bus id of the pci device
+     * @return  the PCI object struct.
+     */
+    const io_obj_info *get_pci_device(std::string &pci_bus_id) const;
+
     /**
      * Get the number of PU objects stored in this Topology tree.
      *
      * @return  the number of PUs.
      */
-    size_type get_num_pus() const { return this->pus_.size(); }
+    size_type get_num_pus() const { return std::get<0>(this->pus_).size(); }
 
     /**
      * Get the number of core objects stored in this Topology tree.
      *
      * @return  the number of cores.
      */
-    size_type get_num_cores() const { return this->cores_.size(); }
+    size_type get_num_cores() const { return std::get<0>(this->cores_).size(); }
 
     /**
      * Get the number of PCI device objects stored in this Topology tree.
@@ -313,8 +359,11 @@ class MachineTopology {
      *
      * A helper function that binds the object with an id.
      */
-    template <typename ObjInfoType>
-    void hwloc_binding_helper(std::vector<ObjInfoType> &obj, size_type id);
+    void hwloc_binding_helper(
+        const std::tuple<std::vector<MachineTopology::normal_obj_info>,
+                         MachineTopology::hwloc_manager<hwloc_bitmap_s>> &obj,
+        const int *id, const size_type num_ids,
+        const BitMapType &bitmap_type) const;
 
     /**
      * @internal
@@ -322,24 +371,24 @@ class MachineTopology {
      * A helper function that prints the topology tree of an object to a given
      * depth. Provided from the hwloc library.
      */
-    static void hwloc_print_children(hwloc_topology *topology, hwloc_obj_t obj,
-                                     int depth);
+    void hwloc_print_children(const hwloc_obj_t obj, const int depth = 1);
 
     /**
      * @internal
      *
-     * Load the objects of a certain HWLOC type.
+     * Load the objects of a normal HWLOC type (Packages, cores, numa-nodes).
      *
      * @note The objects should be sorted by logical index since hwloc uses
      * logical index with these functions
      */
     void load_objects(hwloc_obj_type_t type,
-                      std::vector<normal_obj_info> &vector);
+                      std::tuple<std::vector<normal_obj_info>,
+                                 hwloc_manager<hwloc_bitmap_s>> &objects);
 
     /**
      * @internal
      *
-     * Load the objects of a certain HWLOC type.
+     * Load the objects of io type (PCI devices and OS devices).
      *
      * @note The objects should be sorted by logical index since hwloc uses
      * logical index with these functions
@@ -353,16 +402,45 @@ class MachineTopology {
      */
     hwloc_topology *init_topology();
 
+    /**
+     * @internal
+     *
+     * Initialize the bitmap.
+     */
+    hwloc_bitmap_s *init_bitmap();
+
+    /**
+     *
+     * @internal
+     *
+     * Get local id from the os index
+     */
+    inline int get_obj_local_id_by_os_index(
+        std::vector<normal_obj_info> &objects, int os_index);
+
+    /**
+     *
+     * @internal
+     *
+     * Get local id from the hwloc index
+     */
+    inline int get_obj_local_id_by_gp_index(
+        std::vector<normal_obj_info> &objects, int gp_index);
+
 
 private:
-    std::vector<normal_obj_info> pus_;
-    std::vector<normal_obj_info> cores_;
+    std::tuple<std::vector<normal_obj_info>, hwloc_manager<hwloc_bitmap_s>>
+        pus_;
+    std::tuple<std::vector<normal_obj_info>, hwloc_manager<hwloc_bitmap_s>>
+        cores_;
+    std::tuple<std::vector<normal_obj_info>, hwloc_manager<hwloc_bitmap_s>>
+        packages_;
+    std::tuple<std::vector<normal_obj_info>, hwloc_manager<hwloc_bitmap_s>>
+        numa_nodes_;
     std::vector<io_obj_info> pci_devices_;
     size_type num_numas_;
 
-    template <typename T>
-    using topo_manager = std::unique_ptr<T, std::function<void(T *)>>;
-    topo_manager<hwloc_topology> topo_;
+    hwloc_manager<hwloc_topology> topo_;
 };
 
 

From 4f211777f5161f54ac49e5d6778ce9efa19d549a Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 29 Dec 2020 01:15:09 +0100
Subject: [PATCH 0293/1008] Add binding interfaces to Cuda and Hip executors.

+ By default bind to the closest CPU for each (Cuda/HIP)executor.
---
 core/devices/cuda/executor.cpp        |  7 +++++++
 core/devices/hip/executor.cpp         |  7 +++++++
 include/ginkgo/core/base/executor.hpp | 24 ++++++++++++++++++++++++
 3 files changed, 38 insertions(+)

diff --git a/core/devices/cuda/executor.cpp b/core/devices/cuda/executor.cpp
index 31750fa2a42..e0bc21a2cf2 100644
--- a/core/devices/cuda/executor.cpp
+++ b/core/devices/cuda/executor.cpp
@@ -64,6 +64,13 @@ bool CudaExecutor::verify_memory_to(const HipExecutor *dest_exec) const
 }
 
 
+void CudaExecutor::bind_to_cores(const int *core_ids,
+                                 const size_type num_ids) const
+{
+    get_machine_topology()->bind_to_cores(core_ids, num_ids);
+}
+
+
 unsigned CudaExecutor::num_execs[max_devices];
 
 
diff --git a/core/devices/hip/executor.cpp b/core/devices/hip/executor.cpp
index 3d33fd4b125..0e18eb2b333 100644
--- a/core/devices/hip/executor.cpp
+++ b/core/devices/hip/executor.cpp
@@ -61,6 +61,13 @@ bool HipExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
 }
 
 
+void HipExecutor::bind_to_cores(const int *core_ids,
+                                const size_type num_ids) const
+{
+    get_machine_topology()->bind_to_cores(core_ids, num_ids);
+}
+
+
 int HipExecutor::num_execs[max_devices];
 
 
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index f2f650bfeab..4c1e86fbd9c 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -1200,6 +1200,14 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         return cusparse_handle_.get();
     }
 
+    /**
+     * Bind the current executor thread to the list of cores
+     *
+     * @param  ids  the logical ids of the cores.
+     * @param  num_ids  the number of cores to bind
+     */
+    void bind_to_cores(const int *ids, const size_type num_ids) const;
+
 protected:
     void set_gpu_property();
 
@@ -1215,6 +1223,10 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         assert(this->cuda_exec_info_.device_id < max_devices &&
                this->cuda_exec_info_.device_id >= 0);
         this->populate_exec_info(get_machine_topology());
+        if (this->cuda_exec_info_.closest_cpu_id != -1) {
+            get_machine_topology()->bind_to_pus(
+                &this->cuda_exec_info_.closest_cpu_id, 1);
+        }
         this->set_gpu_property();
         this->init_handles();
         increase_num_execs(this->cuda_exec_info_.device_id);
@@ -1398,6 +1410,14 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         return hipsparse_handle_.get();
     }
 
+    /**
+     * Bind the current executor thread to the list of cores
+     *
+     * @param  ids  the logical ids of the cores.
+     * @param  num_ids  the number of cores to bind
+     */
+    void bind_to_cores(const int *ids, const size_type num_ids) const;
+
 protected:
     void set_gpu_property();
 
@@ -1412,6 +1432,10 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         this->hip_exec_info_.num_work_groups_per_core = 0;
         assert(this->hip_exec_info_.device_id < max_devices);
         this->populate_exec_info(get_machine_topology());
+        if (this->hip_exec_info_.closest_cpu_id != -1) {
+            get_machine_topology()->bind_to_pus(
+                &this->hip_exec_info_.closest_cpu_id, 1);
+        }
         this->set_gpu_property();
         this->init_handles();
         increase_num_execs(this->hip_exec_info_.device_id);

From f7634335f057d04df82b1a692e8b0bcf3b8961f3 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 29 Dec 2020 13:45:15 +0100
Subject: [PATCH 0294/1008] Skip VerifyMemory test when only one dev exists.

+ Separate tests for VerifyMemory for different devices
+ Fix minor div by zero issue.
---
 CMakeLists.txt                |  1 +
 core/devices/omp/executor.cpp |  9 ++++--
 core/test/base/executor.cpp   | 59 +++++++++++++++++++++++++++++------
 hip/base/executor.hip.cpp     | 45 --------------------------
 4 files changed, 56 insertions(+), 58 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index aa1324404e9..a3d15625ad3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -149,6 +149,7 @@ endif()
 
 if(GINKGO_FORCE_NO_HWLOC)
     set(GINKGO_HAVE_HWLOC 0)
+    message(STATUS "HWLOC is being forcibly switched off")
 else()
     set(GINKGO_HAVE_HWLOC 1)
 endif()
diff --git a/core/devices/omp/executor.cpp b/core/devices/omp/executor.cpp
index 9116f22ad89..6b2d80f9f72 100644
--- a/core/devices/omp/executor.cpp
+++ b/core/devices/omp/executor.cpp
@@ -46,9 +46,12 @@ namespace gko {
 
 void OmpExecutor::populate_exec_info(const MachineTopology *mach_topo)
 {
-    this->omp_exec_info_.num_cores = mach_topo->get_num_cores();
-    this->omp_exec_info_.num_work_groups_per_core =
-        mach_topo->get_num_pus() / mach_topo->get_num_cores();
+    auto num_cores =
+        (mach_topo->get_num_cores() == 0 ? 1 : mach_topo->get_num_cores());
+    auto num_pus =
+        (mach_topo->get_num_pus() == 0 ? 1 : mach_topo->get_num_pus());
+    this->omp_exec_info_.num_cores = num_cores;
+    this->omp_exec_info_.num_work_groups_per_core = num_pus / num_cores;
 }
 
 
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 09cae94eb15..5f07d6b1e14 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -159,9 +159,9 @@ TEST(OmpExecutor, IsItsOwnMaster)
 
 TEST(OmpExecutor, CanGetNumCpusFromExecInfo)
 {
-    if (!gko::get_machine_topology()) {
-        GTEST_SKIP() << "No Machine topology information available";
-    }
+#if defined(_WIN32) || defined(__APPLE__) || defined(__CYGWIN__)
+    GTEST_SKIP() << " No useful machine topology information to test against";
+#endif
     auto omp = gko::OmpExecutor::create();
 
     auto num_cpus = omp->get_num_cores() * omp->get_num_threads_per_core();
@@ -497,8 +497,6 @@ TEST(Executor, CanVerifyMemory)
     auto omp2 = gko::OmpExecutor::create();
     auto hip2 = gko::HipExecutor::create(0, omp);
     auto cuda2 = gko::CudaExecutor::create(0, omp);
-    auto hip_1 = gko::HipExecutor::create(1, omp);
-    auto cuda_1 = gko::CudaExecutor::create(1, omp);
     std::shared_ptr<gko::DpcppExecutor> host_dpcpp;
     std::shared_ptr<gko::DpcppExecutor> cpu_dpcpp;
     std::shared_ptr<gko::DpcppExecutor> gpu_dpcpp;
@@ -543,20 +541,61 @@ TEST(Executor, CanVerifyMemory)
 #if GINKGO_HIP_PLATFORM_NVCC
     ASSERT_EQ(true, hip->memory_accessible(cuda));
     ASSERT_EQ(true, cuda->memory_accessible(hip));
-    ASSERT_EQ(true, hip_1->memory_accessible(cuda_1));
-    ASSERT_EQ(true, cuda_1->memory_accessible(hip_1));
 #else
     ASSERT_EQ(false, hip->memory_accessible(cuda));
     ASSERT_EQ(false, cuda->memory_accessible(hip));
-    ASSERT_EQ(false, hip_1->memory_accessible(cuda_1));
-    ASSERT_EQ(false, cuda_1->memory_accessible(hip_1));
 #endif
     ASSERT_EQ(true, omp->memory_accessible(omp2));
     ASSERT_EQ(true, hip->memory_accessible(hip2));
     ASSERT_EQ(true, cuda->memory_accessible(cuda2));
+}
+
+
+TEST(Executor, CanVerifyMemoryBetweenTwoCudaDevices)
+{
+    auto omp = gko::OmpExecutor::create();
+    auto cuda = gko::CudaExecutor::create(0, omp);
+    if (cuda->get_num_devices() < 2) {
+        GTEST_SKIP() << "Only one cuda device available";
+    }
+    auto cuda_1 = gko::CudaExecutor::create(1, omp);
+
+    ASSERT_EQ(false, cuda->memory_accessible(cuda_1));
+}
+
+
+TEST(Executor, CanVerifyMemoryBetweenTwoHipDevices)
+{
+    auto omp = gko::OmpExecutor::create();
+    auto hip = gko::HipExecutor::create(0, omp);
+    if (hip->get_num_devices() < 2) {
+        GTEST_SKIP() << "Only one hip device available";
+    }
+    auto hip_1 = gko::HipExecutor::create(1, omp);
+
     ASSERT_EQ(false, hip->memory_accessible(hip_1));
+}
+
+
+TEST(Executor, CanVerifyMemoryBetweenCudaAndHipDevices)
+{
+    auto omp = gko::OmpExecutor::create();
+    auto hip = gko::HipExecutor::create(0, omp);
+    auto cuda = gko::CudaExecutor::create(0, omp);
+    if (hip->get_num_devices() < 2 || cuda->get_num_devices() < 2) {
+        GTEST_SKIP() << "Only one hip/one cuda device available";
+    }
+    auto hip_1 = gko::HipExecutor::create(1, omp);
+    auto cuda_1 = gko::CudaExecutor::create(1, omp);
+
+#if GINKGO_HIP_PLATFORM_NVCC
+    ASSERT_EQ(true, hip_1->memory_accessible(cuda_1));
+    ASSERT_EQ(true, cuda_1->memory_accessible(hip_1));
+#else
+    ASSERT_EQ(false, hip_1->memory_accessible(cuda_1));
+    ASSERT_EQ(false, cuda_1->memory_accessible(hip_1));
+#endif
     ASSERT_EQ(false, cuda->memory_accessible(hip_1));
-    ASSERT_EQ(false, cuda->memory_accessible(cuda_1));
     ASSERT_EQ(false, hip->memory_accessible(cuda_1));
 }
 
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 1c8a0762c43..e8bb153bc1d 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -55,51 +55,6 @@ namespace gko {
 #include "common/base/executor.hpp.inc"
 
 
-// namespace machine_config {
-
-
-// template <>
-// void Topology<HipExecutor>::load_gpus()
-// {
-// #if GKO_HAVE_HWLOC
-//     size_type num_in_numa = 0;
-//     int last_numa = 0;
-//     auto topology = this->topo_.get();
-//     auto n_objs = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_OS_DEVICE);
-//     for (size_type i = 0; i < n_objs; i++, num_in_numa++) {
-//         hwloc_obj_t obj = NULL;
-//         while ((obj = hwloc_get_next_osdev(topology, obj)) != NULL) {
-//             bool is_gpu = (HWLOC_OBJ_OSDEV_GPU == obj->attr->osdev.type ||
-//                            HWLOC_OBJ_OSDEV_COPROC == obj->attr->osdev.type)
-//                            &&
-//                           obj->name &&
-//                           (!strncmp("rsmi", obj->name, 4) ||
-//                            !strncmp("cuda", obj->name, 4)) &&
-//                           atoi(obj->name + 4) == (int)i;
-//             if (is_gpu) {
-//                 while (obj &&
-//                        (!obj->nodeset || hwloc_bitmap_iszero(obj->nodeset)))
-//                     obj = obj->parent;
-//                 if (obj && obj->nodeset) {
-//                     auto this_numa = hwloc_bitmap_first(obj->nodeset);
-//                     if (this_numa != last_numa) {
-//                         num_in_numa = 0;
-//                     }
-//                     this->gpus_.push_back(
-//                         topology_obj_info{obj, this_numa, i, num_in_numa});
-//                     last_numa = this_numa;
-//                 }
-//             }
-//         }
-//     }
-
-// #endif
-// }
-
-
-// }  // namespace machine_config
-
-
 std::shared_ptr<HipExecutor> HipExecutor::create(
     int device_id, std::shared_ptr<Executor> master, bool device_reset)
 {

From 6e77ccf8336c37bd7bb9b973d0deaa8ba4fbca2d Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 29 Dec 2020 23:43:30 +0100
Subject: [PATCH 0295/1008] Add GMock support and some binding tests.

---
 cmake/create_test.cmake          |  3 ++-
 core/test/base/executor.cpp      | 39 ++++++++++++++++++++++++++++++++
 third_party/gtest/CMakeLists.txt |  4 ++++
 3 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index 51d264dd367..be4a03e9027 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -13,7 +13,8 @@ function(ginkgo_create_test test_name)
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
+    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest
+      GMock::Main GMock::GMock ${ARGN})
     add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
 endfunction(ginkgo_create_test)
 
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 5f07d6b1e14..706cfe050f5 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -33,10 +33,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
+#include <utmpx.h>
 #include <thread>
 #include <type_traits>
 
 
+#include <gmock/gmock.h>
 #include <gtest/gtest.h>
 
 
@@ -170,6 +172,43 @@ TEST(OmpExecutor, CanGetNumCpusFromExecInfo)
 }
 
 
+#if GKO_HAVE_HWLOC
+
+
+TEST(OmpExecutor, CanBindToASpecificCore)
+{
+#if defined(_WIN32) || defined(__APPLE__) || defined(__CYGWIN__)
+    GTEST_SKIP() << " No useful machine topology information to test against";
+#endif
+    auto omp = gko::OmpExecutor::create();
+    auto cpu_sys = sched_getcpu();
+
+    const int bind_core = 4;
+    gko::get_machine_topology()->bind_to_cores(&bind_core, 1);
+
+    cpu_sys = sched_getcpu();
+    ASSERT_EQ(cpu_sys, bind_core);
+}
+
+
+TEST(OmpExecutor, CanBindToARangeofCores)
+{
+#if defined(_WIN32) || defined(__APPLE__) || defined(__CYGWIN__)
+    GTEST_SKIP() << " No useful machine topology information to test against";
+#endif
+    auto omp = gko::OmpExecutor::create();
+    auto cpu_sys = sched_getcpu();
+
+    const int bind_core[2] = {6, 3};
+    gko::get_machine_topology()->bind_to_cores(bind_core, 2);
+
+    cpu_sys = sched_getcpu();
+    EXPECT_THAT(cpu_sys, testing::AnyOf(testing::Eq(3), testing::Eq(6)));
+}
+
+
+#endif
+
 TEST(ReferenceExecutor, RunsCorrectOperation)
 {
     int value = 0;
diff --git a/third_party/gtest/CMakeLists.txt b/third_party/gtest/CMakeLists.txt
index 31da9c5eda3..2ff09120582 100644
--- a/third_party/gtest/CMakeLists.txt
+++ b/third_party/gtest/CMakeLists.txt
@@ -19,3 +19,7 @@ ginkgo_add_external_target(GTest::GTest gtest src/googletest/include build/lib
     STATIC "d" gtest_external FALSE)
 ginkgo_add_external_target(GTest::Main gtest_main src/googletest/include build/lib
     STATIC "d" gtest_external FALSE)
+ginkgo_add_external_target(GMock::GMock gmock src/googlemock/include build/lib
+    STATIC "d" gmock_external FALSE)
+ginkgo_add_external_target(GMock::Main gmock_main src/googlemock/include build/lib
+    STATIC "d" gmock_external FALSE)

From a9dd17ae00a15f16e14f0e1a8c095415f1265ea5 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 30 Dec 2020 11:31:02 +0100
Subject: [PATCH 0296/1008] Fix code smells and use os id for binding tests.

---
 core/device_hooks/cuda_hooks.cpp              |  5 ++++-
 core/device_hooks/dpcpp_hooks.cpp             |  5 ++++-
 core/device_hooks/hip_hooks.cpp               |  5 ++++-
 core/devices/machine_topology.cpp             | 13 ++++++------
 core/test/base/executor.cpp                   | 20 ++++++++++++++++---
 include/ginkgo/core/base/machine_topology.hpp |  6 +++---
 6 files changed, 39 insertions(+), 15 deletions(-)

diff --git a/core/device_hooks/cuda_hooks.cpp b/core/device_hooks/cuda_hooks.cpp
index 3c88599f897..0d06e888b7f 100644
--- a/core/device_hooks/cuda_hooks.cpp
+++ b/core/device_hooks/cuda_hooks.cpp
@@ -62,7 +62,10 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
 }
 
 
-void CudaExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+void CudaExecutor::populate_exec_info(const MachineTopology *mach_topo)
+{
+    // This method is always called, so cannot throw when not compiled.
+}
 
 
 void OmpExecutor::raw_copy_to(const CudaExecutor *, size_type num_bytes,
diff --git a/core/device_hooks/dpcpp_hooks.cpp b/core/device_hooks/dpcpp_hooks.cpp
index 444fd6c8a33..30eff6fe21b 100644
--- a/core/device_hooks/dpcpp_hooks.cpp
+++ b/core/device_hooks/dpcpp_hooks.cpp
@@ -59,7 +59,10 @@ std::shared_ptr<DpcppExecutor> DpcppExecutor::create(
 }
 
 
-void DpcppExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+void DpcppExecutor::populate_exec_info(const MachineTopology *mach_topo)
+{
+    // This method is always called, so cannot throw when not compiled.
+}
 
 
 void OmpExecutor::raw_copy_to(const DpcppExecutor *, size_type num_bytes,
diff --git a/core/device_hooks/hip_hooks.cpp b/core/device_hooks/hip_hooks.cpp
index 6c8659a6909..6a5d79b6f73 100644
--- a/core/device_hooks/hip_hooks.cpp
+++ b/core/device_hooks/hip_hooks.cpp
@@ -59,7 +59,10 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
 }
 
 
-void HipExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+void HipExecutor::populate_exec_info(const MachineTopology *mach_topo)
+{
+    // This method is always called, so cannot throw when not compiled.
+}
 
 
 void OmpExecutor::raw_copy_to(const HipExecutor *, size_type num_bytes,
diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index 52b5054dc76..6ecd373cae8 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -67,7 +67,7 @@ const MachineTopology *get_machine_topology()
 
 
 const MachineTopology::io_obj_info *MachineTopology::get_pci_device(
-    std::string &pci_bus_id) const
+    const std::string &pci_bus_id) const
 {
     for (auto id = 0; id < this->pci_devices_.size(); ++id) {
         if (this->pci_devices_[id].pci_bus_id.compare(0, 12, pci_bus_id, 0,
@@ -163,7 +163,6 @@ void MachineTopology::hwloc_print_children(const hwloc_obj_t obj,
                                            const int depth)
 {
 #if GKO_HAVE_HWLOC
-    auto topology = this->topo_.get();
     char type[32], attr[1024];
     unsigned i;
     hwloc_obj_type_snprintf(type, sizeof(type), obj, 0);
@@ -206,7 +205,8 @@ void MachineTopology::load_objects(
 
 
 inline int MachineTopology::get_obj_local_id_by_os_index(
-    std::vector<MachineTopology::normal_obj_info> &objects, int os_index)
+    std::vector<MachineTopology::normal_obj_info> &objects,
+    size_type os_index) const
 {
 #if GKO_HAVE_HWLOC
     for (auto id = 0; id < objects.size(); ++id) {
@@ -220,7 +220,8 @@ inline int MachineTopology::get_obj_local_id_by_os_index(
 
 
 inline int MachineTopology::get_obj_local_id_by_gp_index(
-    std::vector<MachineTopology::normal_obj_info> &objects, int gp_index)
+    std::vector<MachineTopology::normal_obj_info> &objects,
+    size_type gp_index) const
 {
 #if GKO_HAVE_HWLOC
     for (auto id = 0; id < objects.size(); ++id) {
@@ -292,7 +293,7 @@ void MachineTopology::load_objects(
             // Get the first child
             os_child = obj->io_first_child;
             vector.back().io_children.push_back(os_child);
-            vector.back().io_children_name.push_back(os_child->name);
+            vector.back().io_children_name.emplace_back(os_child->name);
             rem_io_children -= 1;
             // If the io obj has more than one children, they are siblings to
             // the first child. Store these as well.
@@ -300,7 +301,7 @@ void MachineTopology::load_objects(
             while (rem_io_children >= 1) {
                 os_child_n = os_child_n->next_sibling;
                 vector.back().io_children.push_back(os_child_n);
-                vector.back().io_children_name.push_back(os_child_n->name);
+                vector.back().io_children_name.emplace_back(os_child_n->name);
                 rem_io_children -= 1;
             }
             GKO_ASSERT(vector.back().io_children.size() ==
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 706cfe050f5..75433ae8cee 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -33,12 +33,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
-#include <utmpx.h>
 #include <thread>
 #include <type_traits>
 
 
+#if defined(__unix__) || defined(__APPLE__)
+#include <utmpx.h>
+#endif
+
+
 #include <gmock/gmock.h>
+
+
 #include <gtest/gtest.h>
 
 
@@ -175,6 +181,12 @@ TEST(OmpExecutor, CanGetNumCpusFromExecInfo)
 #if GKO_HAVE_HWLOC
 
 
+inline int get_os_id(int log_id)
+{
+    return gko::get_machine_topology()->get_core(log_id)->os_id;
+}
+
+
 TEST(OmpExecutor, CanBindToASpecificCore)
 {
 #if defined(_WIN32) || defined(__APPLE__) || defined(__CYGWIN__)
@@ -187,7 +199,7 @@ TEST(OmpExecutor, CanBindToASpecificCore)
     gko::get_machine_topology()->bind_to_cores(&bind_core, 1);
 
     cpu_sys = sched_getcpu();
-    ASSERT_EQ(cpu_sys, bind_core);
+    ASSERT_EQ(cpu_sys, get_os_id(bind_core));
 }
 
 
@@ -203,12 +215,14 @@ TEST(OmpExecutor, CanBindToARangeofCores)
     gko::get_machine_topology()->bind_to_cores(bind_core, 2);
 
     cpu_sys = sched_getcpu();
-    EXPECT_THAT(cpu_sys, testing::AnyOf(testing::Eq(3), testing::Eq(6)));
+    EXPECT_THAT(cpu_sys, testing::AnyOf(testing::Eq(get_os_id(3)),
+                                        testing::Eq(get_os_id(6))));
 }
 
 
 #endif
 
+
 TEST(ReferenceExecutor, RunsCorrectOperation)
 {
     int value = 0;
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index 6f77c485a32..ddda8f74e95 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -310,7 +310,7 @@ class MachineTopology {
      * @param pci_bus_id  The PCI bus id of the pci device
      * @return  the PCI object struct.
      */
-    const io_obj_info *get_pci_device(std::string &pci_bus_id) const;
+    const io_obj_info *get_pci_device(const std::string &pci_bus_id) const;
 
     /**
      * Get the number of PU objects stored in this Topology tree.
@@ -416,7 +416,7 @@ class MachineTopology {
      * Get local id from the os index
      */
     inline int get_obj_local_id_by_os_index(
-        std::vector<normal_obj_info> &objects, int os_index);
+        std::vector<normal_obj_info> &objects, size_type os_index) const;
 
     /**
      *
@@ -425,7 +425,7 @@ class MachineTopology {
      * Get local id from the hwloc index
      */
     inline int get_obj_local_id_by_gp_index(
-        std::vector<normal_obj_info> &objects, int gp_index);
+        std::vector<normal_obj_info> &objects, size_type gp_index) const;
 
 
 private:

From 7c11e2b0636623ec644b7512f9153d404cf811fe Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Fri, 8 Jan 2021 23:46:55 +0100
Subject: [PATCH 0297/1008] Add binding tests for CUDA and HIP

---
 cuda/test/base/cuda_executor.cu       | 38 +++++++++++++++++++++++++++
 hip/test/base/hip_executor.hip.cpp    | 37 ++++++++++++++++++++++++++
 include/ginkgo/core/base/executor.hpp | 15 +++++++----
 3 files changed, 85 insertions(+), 5 deletions(-)

diff --git a/cuda/test/base/cuda_executor.cu b/cuda/test/base/cuda_executor.cu
index 99db90458ee..1019cef34da 100644
--- a/cuda/test/base/cuda_executor.cu
+++ b/cuda/test/base/cuda_executor.cu
@@ -34,9 +34,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <memory>
+#include <thread>
 #include <type_traits>
 
 
+#if defined(__unix__) || defined(__APPLE__)
+#include <utmpx.h>
+#endif
+
+
+#include <gmock/gmock.h>
+
+
 #include <gtest/gtest.h>
 
 
@@ -131,6 +140,35 @@ TEST_F(CudaExecutor, MasterKnowsNumberOfDevices)
 }
 
 
+#if GKO_HAVE_HWLOC
+
+
+inline int get_os_id(int log_id)
+{
+    return gko::get_machine_topology()->get_core(log_id)->os_id;
+}
+
+
+TEST_F(CudaExecutor, CanBindToCores)
+{
+#if defined(_WIN32) || defined(__APPLE__) || defined(__CYGWIN__)
+    GTEST_SKIP() << " No useful machine topology information to test against";
+#endif
+    auto cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+    auto cpu_sys = sched_getcpu();
+
+    const int bind_core[2] = {6, 3};
+    cuda->bind_to_cores(bind_core, 2);
+
+    cpu_sys = sched_getcpu();
+    EXPECT_THAT(cpu_sys, testing::AnyOf(testing::Eq(get_os_id(3)),
+                                        testing::Eq(get_os_id(6))));
+}
+
+
+#endif
+
+
 TEST_F(CudaExecutor, AllocatesAndFreesMemory)
 {
     int *ptr = nullptr;
diff --git a/hip/test/base/hip_executor.hip.cpp b/hip/test/base/hip_executor.hip.cpp
index c4f4757e397..057438148b6 100644
--- a/hip/test/base/hip_executor.hip.cpp
+++ b/hip/test/base/hip_executor.hip.cpp
@@ -40,9 +40,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <memory>
+#include <thread>
 #include <type_traits>
 
 
+#if defined(__unix__) || defined(__APPLE__)
+#include <utmpx.h>
+#endif
+
+
+#include <gmock/gmock.h>
+
+
 #include <gtest/gtest.h>
 
 
@@ -136,6 +145,34 @@ TEST_F(HipExecutor, MasterKnowsNumberOfDevices)
 }
 
 
+#if GKO_HAVE_HWLOC
+
+
+inline int get_os_id(int log_id)
+{
+    return gko::get_machine_topology()->get_core(log_id)->os_id;
+}
+
+
+TEST_F(HipExecutor, CanBindToCores)
+{
+#if defined(_WIN32) || defined(__APPLE__) || defined(__CYGWIN__)
+    GTEST_SKIP() << " No useful machine topology information to test against";
+#endif
+    auto hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+    auto cpu_sys = sched_getcpu();
+    const int bind_core[2] = {6, 3};
+    hip->bind_to_cores(bind_core, 2);
+
+    cpu_sys = sched_getcpu();
+    EXPECT_THAT(cpu_sys, testing::AnyOf(testing::Eq(get_os_id(3)),
+                                        testing::Eq(get_os_id(6))));
+}
+
+
+#endif
+
+
 TEST_F(HipExecutor, AllocatesAndFreesMemory)
 {
     int *ptr = nullptr;
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 4c1e86fbd9c..c2b65d7f51e 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -737,7 +737,6 @@ class Executor : public log::EnableLogging<Executor> {
         int device_id = -1;
         int num_cores = 0;
         int num_work_groups_per_core = 0;
-        std::string pci_bus_id = "invalid";
     };
 
 private:
@@ -997,7 +996,10 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
     }
 
 protected:
-    OmpExecutor() { this->populate_exec_info(get_machine_topology()); }
+    OmpExecutor()
+    {
+        this->OmpExecutor::populate_exec_info(get_machine_topology());
+    }
 
     void populate_exec_info(const MachineTopology *mach_topo) override;
 
@@ -1052,7 +1054,10 @@ class ReferenceExecutor : public OmpExecutor {
     }
 
 protected:
-    ReferenceExecutor() = default;
+    ReferenceExecutor()
+    {
+        this->ReferenceExecutor::populate_exec_info(get_machine_topology());
+    }
 
     void populate_exec_info(const MachineTopology *) override
     {
@@ -1222,7 +1227,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         this->cuda_exec_info_.num_work_groups_per_core = 0;
         assert(this->cuda_exec_info_.device_id < max_devices &&
                this->cuda_exec_info_.device_id >= 0);
-        this->populate_exec_info(get_machine_topology());
+        this->CudaExecutor::populate_exec_info(get_machine_topology());
         if (this->cuda_exec_info_.closest_cpu_id != -1) {
             get_machine_topology()->bind_to_pus(
                 &this->cuda_exec_info_.closest_cpu_id, 1);
@@ -1431,7 +1436,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         this->hip_exec_info_.num_cores = 0;
         this->hip_exec_info_.num_work_groups_per_core = 0;
         assert(this->hip_exec_info_.device_id < max_devices);
-        this->populate_exec_info(get_machine_topology());
+        this->HipExecutor::populate_exec_info(get_machine_topology());
         if (this->hip_exec_info_.closest_cpu_id != -1) {
             get_machine_topology()->bind_to_pus(
                 &this->hip_exec_info_.closest_cpu_id, 1);

From da675267387a374adb0aca8a5b9cdbb8a5a85234 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Fri, 8 Jan 2021 23:47:26 +0100
Subject: [PATCH 0298/1008] Review update: CMake

+ Update FindHWLOC
+ Update the CI jobs with one with only TPL hwloc.
+ Remove GMock

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 .gitlab-ci.yml                                |  12 +-
 CMakeLists.txt                                |   9 +-
 cmake/GinkgoConfig.cmake.in                   |   3 +
 cmake/Modules/FindHWLOC.cmake                 | 229 +++++++++---------
 cmake/Modules/hwloc_helpers.cmake             |  87 +++++++
 cmake/create_test.cmake                       |  12 +-
 cmake/get_info.cmake                          |   2 -
 cmake/package_helpers.cmake                   |  64 +++--
 core/CMakeLists.txt                           |   4 -
 core/devices/CMakeLists.txt                   |  12 +-
 core/devices/cuda/CMakeLists.txt              |   1 -
 core/devices/hip/CMakeLists.txt               |   2 -
 core/devices/machine_topology.cpp             |   6 +-
 core/test/base/executor.cpp                   |   6 +-
 hip/CMakeLists.txt                            |   3 -
 include/ginkgo/core/base/machine_topology.hpp |   2 +-
 test_install/CMakeLists.txt                   |  14 +-
 third_party/gtest/CMakeLists.txt              |   4 -
 third_party/hwloc/CMakeLists.txt              |  61 ++---
 19 files changed, 310 insertions(+), 223 deletions(-)
 create mode 100644 cmake/Modules/hwloc_helpers.cmake

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f9667b0025d..44caedc28db 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,7 +22,6 @@ stages:
   BUILD_OMP: "OFF"
   BUILD_CUDA: "OFF"
   BUILD_HIP: "OFF"
-  USE_EXTERNAL_HWLOC: "ON"
   FORCE_NO_HWLOC: "OFF"
   RUN_EXAMPLES: "OFF"
   CONFIG_LOG: "ON"
@@ -66,7 +65,6 @@ stages:
         -DGINKGO_BUILD_OMP=${BUILD_OMP} -DGINKGO_BUILD_CUDA=${BUILD_CUDA}
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
         -DGINKGO_FORCE_NO_HWLOC=${FORCE_NO_HWLOC}
-        -DGINKGO_USE_EXTERNAL_HWLOC=${USE_EXTERNAL_HWLOC}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
@@ -99,7 +97,6 @@ stages:
         -DGINKGO_BUILD_OMP=${BUILD_OMP} -DGINKGO_BUILD_CUDA=${BUILD_CUDA}
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
         -DGINKGO_FORCE_NO_HWLOC=${FORCE_NO_HWLOC}
-        -DGINKGO_USE_EXTERNAL_HWLOC=${USE_EXTERNAL_HWLOC}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
@@ -269,7 +266,7 @@ build/cuda92/intel/cuda/release/static:
     CXX_COMPILER: "icpc"
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
-    FORCE_NO_HWLOC: "OFF"
+    FORCE_NO_HWLOC: "ON"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
@@ -292,7 +289,6 @@ build/cuda100/gcc/all/debug/shared:
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
-    USE_EXTERNAL_HWLOC: "OFF"
     BUILD_TYPE: "Debug"
     CUDA_ARCH: 35
   only:
@@ -425,6 +421,8 @@ build/clang-cuda101/clang/cuda/debug/static:
     - gpu
 
 # cuda 10.2 and friends
+
+# Use TPL hwloc
 build/cuda102/gcc/all/debug/shared:
   <<: *default_build_with_test
   image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
@@ -434,6 +432,7 @@ build/cuda102/gcc/all/debug/shared:
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     BUILD_TYPE: "Debug"
+    FORCE_NO_HWLOC: "ON"
     CUDA_ARCH: 35
   only:
     variables:
@@ -588,6 +587,7 @@ build/nocuda/gcc/core/debug/static:
     BUILD_REFERENCE: "OFF"
     BUILD_TYPE: "Debug"
     BUILD_SHARED_LIBS: "OFF"
+    FORCE_NO_HWLOC: "ON"
   only:
     variables:
       - $RUN_CI_TAG
@@ -603,7 +603,6 @@ build/nocuda/clang/core/release/shared:
     C_COMPILER: "clang"
     CXX_COMPILER: "clang++"
     BUILD_REFERENCE: "OFF"
-    USE_EXTERNAL_HWLOC: "OFF"
     BUILD_TYPE: "Release"
   only:
     variables:
@@ -650,7 +649,6 @@ build/nocuda/clang/omp/debug/static:
     <<: *default_variables
     C_COMPILER: "clang"
     CXX_COMPILER: "clang++"
-    USE_EXTERNAL_HWLOC: "OFF"
     BUILD_OMP: "ON"
     BUILD_TYPE: "Debug"
     BUILD_SHARED_LIBS: "OFF"
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a3d15625ad3..c8896d6e5f3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -15,8 +15,6 @@ project(Ginkgo LANGUAGES C CXX VERSION 1.3.0 DESCRIPTION "A numerical linear alg
 set(Ginkgo_VERSION_TAG "master")
 set(PROJECT_VERSION_TAG ${Ginkgo_VERSION_TAG})
 
-# Add custom modules search path for the find_package functions.
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules/")
 # Determine which modules can be compiled
 include(cmake/hip_path.cmake)
 include(cmake/autodetect_executors.cmake)
@@ -72,7 +70,6 @@ set(GINKGO_HIP_AMDGPU "" CACHE STRING
     "The amdgpu_target(s) variable passed to hipcc. The default is none (auto).")
 option(GINKGO_JACOBI_FULL_OPTIMIZATIONS "Use all the optimizations for the CUDA Jacobi algorithm" OFF)
 option(BUILD_SHARED_LIBS "Build shared (.so, .dylib, .dll) libraries" ON)
-option(GINKGO_USE_EXTERNAL_HWLOC "Use external hwloc libray if available" ON)
 option(GINKGO_FORCE_NO_HWLOC "Forcibly switch off HWLOC dependency" OFF)
 
 set(GINKGO_CIRCULAR_DEPS_FLAGS "-Wl,--no-undefined")
@@ -148,9 +145,12 @@ if(PAPI_sde_FOUND)
 endif()
 
 if(GINKGO_FORCE_NO_HWLOC)
+    set(GINKGO_USE_EXTERNAL_HWLOC 0)
     set(GINKGO_HAVE_HWLOC 0)
     message(STATUS "HWLOC is being forcibly switched off")
 else()
+    # By default always use external HWLOC
+    set(GINKGO_USE_EXTERNAL_HWLOC 1)
     set(GINKGO_HAVE_HWLOC 1)
 endif()
 # Switch off HWLOC for Windows and MacOS
@@ -210,7 +210,7 @@ ginkgo_find_package(GTest "GTest::GTest;GTest::Main" FALSE 1.8.1)
 ginkgo_find_package(gflags gflags FALSE 2.2.2)
 ginkgo_find_package(RapidJSON rapidjson TRUE 1.1.0)
 if(NOT GINKGO_FORCE_NO_HWLOC AND GINKGO_HAVE_HWLOC)
-  ginkgo_find_package(HWLOC hwloc FALSE 2.1)
+    ginkgo_find_package(HWLOC hwloc FALSE 2.1)
 endif()
 set(GKO_HWLOC_XMLFILE "$ENV{HWLOC_XMLFILE}")
 message(STATUS "HWLOC XML file set to: ${GKO_HWLOC_XMLFILE}")
@@ -325,6 +325,7 @@ add_custom_target(test_install
     COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} -H${Ginkgo_SOURCE_DIR}/test_install
     -B${Ginkgo_BINARY_DIR}/test_install
     -DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX}/${GINKGO_INSTALL_CONFIG_DIR}
+    -DHWLOC_DIR=${HWLOC_DIR}
     -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
     -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
     # `--config cfg` is ignored by single-configuration generator.
diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 3f8160daeff..8fbd650d3bd 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -77,6 +77,9 @@ set(GINKGO_DPCPP_FLAGS @GINKGO_DPCPP_FLAGS@)
 
 set(GINKGO_HAVE_PAPI_SDE @GINKGO_HAVE_PAPI_SDE@)
 
+set(GINKGO_HAVE_HWLOC @GINKGO_HAVE_HWLOC@)
+set(GINKGO_FORCE_NO_HWLOC @GINKGO_FORCE_NO_HWLOC@)
+
 # Ginkgo external package variables
 set(GINKGO_USE_EXTERNAL_CAS "@GINKGO_USE_EXTERNAL_CAS@")
 set(GINKGO_USE_EXTERNAL_GTEST "@GINKGO_USE_EXTERNAL_GTEST@")
diff --git a/cmake/Modules/FindHWLOC.cmake b/cmake/Modules/FindHWLOC.cmake
index 519a73536ca..96ea17639e4 100644
--- a/cmake/Modules/FindHWLOC.cmake
+++ b/cmake/Modules/FindHWLOC.cmake
@@ -1,147 +1,136 @@
+###
 #
-# This file is a modified version of the file in the GROMACS molecular simulation
-# package.
+# @copyright (c) 2012-2020 Inria. All rights reserved.
+# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
 #
-# Copyright (c) 2015,2016,2017,2018, by the GROMACS development team, led by
-# Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
-# and including many others, as listed in the AUTHORS file in the
-# top-level source directory and at http://www.gromacs.org.
+# Copyright 2012-2013 Emmanuel Agullo
+# Copyright 2012-2013 Mathieu Faverge
+# Copyright 2012      Cedric Castagnede
+# Copyright 2013-2020 Florent Pruvost
+# Copyright 2020-2021 Ginkgo Project
 #
-# GROMACS is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public License
-# as published by the Free Software Foundation; either version 2.1
-# of the License, or (at your option) any later version.
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file MORSE-Copyright.txt for details.
 #
-# GROMACS is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of Morse, substitute the full
+#  License text for the above reference.)
 #
-# You should have received a copy of the GNU Lesser General Public
-# License along with GROMACS; if not, see
-# http://www.gnu.org/licenses, or write to the Free Software Foundation,
-# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
+# Modified for Ginkgo.
 #
-# If you want to redistribute modifications to GROMACS, please
-# consider that scientific software is very special. Version
-# control is crucial - bugs must be traceable. We will be happy to
-# consider code for inclusion in the official distribution, but
-# derived work must not be called official GROMACS. Details are found
-# in the README & COPYING files - if they are missing, get the
-# official version at http://www.gromacs.org.
+###
 #
-# To help us fund GROMACS development, we humbly ask that you cite
-# the research papers on the package. Check out http://www.gromacs.org.
-
-# FindHWLOC
-#
-# - Locate headers and libraries for the Portable Hardware Locality (hwloc)
-#   library.
-#
-# Usage: find_package(HWLOC VERSION)
-#        find_package(HWLOC VERSION REQUIRED)
+# - Find HWLOC include dirs and libraries
+# Use this module by invoking find_package with the form:
+#  find_package(HWLOC
+#               [REQUIRED] [VERSION]) # Fail with error if hwloc is not found
 #
-# FindHWLOC defines the following variables:
-#
-# HWLOC_FOUND          - True if both headers and libraries were located
-# HWLOC_INCLUDE_DIRS   - Where to find hwloc headers
-# HWLOC_LIBRARIES      - Libraries to link with to enable hwloc usage
-# HWLOC_VERSION        - Version (string) of the hwloc library
+# This module defines the following :prop_tgt:`IMPORTED` target:
+#   ``hwloc``
 #
+#=============================================================================
+include(CheckStructHasMember)
+include(CheckCSourceCompiles)
 
-# Function for converting hex version numbers from HWLOC_API_VERSION if necessary
-function(HEX2DEC str res)
-    string(LENGTH "${str}" len)
-    if("${len}" EQUAL 1)
-        if("${str}" MATCHES "[0-9]")
-            set(${res} "${str}" PARENT_SCOPE)
-        elseif( "${str}" MATCHES "[aA]")
-            set(${res} 10 PARENT_SCOPE)
-        elseif( "${str}" MATCHES "[bB]")
-            set(${res} 11 PARENT_SCOPE)
-        elseif( "${str}" MATCHES "[cC]")
-            set(${res} 12 PARENT_SCOPE)
-        elseif( "${str}" MATCHES "[dD]")
-            set(${res} 13 PARENT_SCOPE)
-        elseif( "${str}" MATCHES "[eE]")
-            set(${res} 14 PARENT_SCOPE)
-        elseif( "${str}" MATCHES "[fF]")
-            set(${res} 15 PARENT_SCOPE)
-        else()
-            return()
-        endif()
-    else()
-        string(SUBSTRING "${str}" 0 1 str1)
-        string(SUBSTRING "${str}" 1 -1 str2)
-        hex2dec(${str1} res1)
-        hex2dec(${str2} res2)
-        math(EXPR val "16 * ${res1} + ${res2}")
-        set(${res} "${val}" PARENT_SCOPE)
-    endif()
-endfunction()
+include(hwloc_helpers)
 
-find_path(HWLOC_INCLUDE_DIRS "hwloc.h")
-find_library(HWLOC_LIBRARIES "hwloc")
+find_path(HWLOC_INCLUDE_DIRS
+    NAMES "hwloc.h"
+    HINTS ${HWLOC_DIR} $ENV{HWLOC_DIR}
+    PATH_SUFFIXES include src/include
+    DOC "Find the hwloc.h main header"
+    )
 
-if(HWLOC_INCLUDE_DIRS)
-    # If we are not cross-compiling we try to use the hwloc-info program
-    if(NOT CMAKE_CROSSCOMPILING)
-        find_program(HWLOC_INFO "hwloc-info")
-        mark_as_advanced(HWLOC_INFO)
+find_library(HWLOC_LIBRARIES "hwloc"
+    HINTS ${HWLOC_DIR} $ENV{HWLOC_DIR}
+    PATH_SUFFIXES hwloc/lib lib lib64
+    DOC "Find the hwloc library"
+    )
 
-        if(HWLOC_INFO)
-            execute_process(COMMAND ${HWLOC_INFO} "--version"
-                            RESULT_VARIABLE HWLOC_INFO_RES
-                            OUTPUT_VARIABLE HWLOC_INFO_OUT
-                            ERROR_VARIABLE  HWLOC_INFO_ERR)
+if (HWLOC_INCLUDE_DIRS)
+    unset(HWLOC_FOUND CACHE)
+    set(HWLOC_FOUND 1)
 
-            if(HWLOC_INFO_ERR)
-	        message(STATUS "Error executing hwloc-info: ${HWLOC_INFO_ERR}")
-            endif()
-            string(REGEX MATCH "[0-9]+.*[0-9]+" HWLOC_INFO_OUT "${HWLOC_INFO_OUT}")
-            set(HWLOC_VERSION ${HWLOC_INFO_OUT} CACHE STRING "Hwloc library version")
-        endif()
-    endif()
-
-    if (NOT HWLOC_FIND_QUIETLY)
-        message(STATUS "hwloc version: ${HWLOC_VERSION}")
-    endif()
-
-    # Parse header if cross-compiling, or if hwloc-info was not found
+    # Find the version of hwloc found
     if(NOT HWLOC_VERSION)
-        # HWLOC is never installed as a framework on OS X, so this should always work.
         file(READ "${HWLOC_INCLUDE_DIRS}/hwloc.h"
-             HEADER_CONTENTS LIMIT 16384)
+            HEADER_CONTENTS LIMIT 16384)
         string(REGEX REPLACE ".*#define HWLOC_API_VERSION (0[xX][0-9a-fA-F]+).*" "\\1"
-               HWLOC_API_VERSION "${HEADER_CONTENTS}")
+            HWLOC_API_VERSION "${HEADER_CONTENTS}")
         string(SUBSTRING "${HWLOC_API_VERSION}" 4 2 HEX_MAJOR)
         string(SUBSTRING "${HWLOC_API_VERSION}" 6 2 HEX_MINOR)
         string(SUBSTRING "${HWLOC_API_VERSION}" 8 2 HEX_PATCH)
-        hex2dec(${HEX_MAJOR} DEC_MAJOR)
-        hex2dec(${HEX_MINOR} DEC_MINOR)
-        hex2dec(${HEX_PATCH} DEC_PATCH)
-        set(HWLOC_VERSION "${DEC_MAJOR}.${DEC_MINOR}.${DEC_PATCH}" CACHE STRING "HWLOC library version")
+        get_dec_from_hex("${HEX_MAJOR}" DEC_MAJOR)
+        get_dec_from_hex("${HEX_MINOR}" DEC_MINOR)
+        get_dec_from_hex("${HEX_PATCH}" DEC_PATCH)
+        set(HWLOC_VERSION "${DEC_MAJOR}.${DEC_MINOR}.${DEC_PATCH}" CACHE STRING "HWLOC version")
     endif()
-endif()
 
-string(SUBSTRING "${HWLOC_VERSION}" 0 3 HWLOC_VERSION)
-if(HWLOC_VERSION LESS_EQUAL HWLOC_FIND_VERSION)
-    message(STATUS "Required version ${HWLOC_FIND_VERSION}, but found version ${HWLOC_VERSION}")
-    set(HWLOC_FOUND 0)
-    unset(HWLOC_LIBRARIES)
-    unset(HWLOC_INCLUDE_DIRS)
-else()
-    include(FindPackageHandleStandardArgs)
-    find_package_handle_standard_args(HWLOC
-        REQUIRED_VARS HWLOC_LIBRARIES HWLOC_INCLUDE_DIRS
-        VERSION_VAR HWLOC_VERSION)
-    mark_as_advanced(HWLOC_INCLUDE_DIRS HWLOC_LIBRARIES HWLOC_VERSION)
+    if (NOT HWLOC_FIND_QUIETLY)
+        if (HWLOC_FOUND AND HWLOC_LIBRARIES)
+            message(STATUS "Looking for HWLOC - found version ${HWLOC_VERSION}")
+        else()
+            message(STATUS "${Magenta}Looking for HWLOC - not found"
+                "\n   Please check that your environment variable HWLOC_DIR"
+                "\n   has been set properly.${ColourReset}")
+        endif()
+    endif()
 endif()
-if(NOT HWLOC_FOUND)
-  unset(HWLOC_LIBRARIES)
-  unset(HWLOC_INCLUDE_DIRS)
+
+# check a function to validate the find
+if(HWLOC_FOUND AND HWLOC_LIBRARIES)
+
+    # set required libraries for link
+    ginkgo_set_required_test_lib_link(HWLOC)
+
+    # test link
+    unset(HWLOC_WORKS CACHE)
+    include(CheckFunctionExists)
+    check_function_exists(hwloc_topology_init HWLOC_WORKS)
+    mark_as_advanced(HWLOC_WORKS)
+
+    if(NOT HWLOC_WORKS)
+        if(NOT HWLOC_FIND_QUIETLY)
+            message(STATUS "Looking for hwloc : test of hwloc_topology_init with hwloc library fails")
+            message(STATUS "CMAKE_REQUIRED_LIBRARIES: ${CMAKE_REQUIRED_LIBRARIES}")
+            message(STATUS "CMAKE_REQUIRED_INCLUDES: ${CMAKE_REQUIRED_INCLUDES}")
+            message(STATUS "CMAKE_REQUIRED_FLAGS: ${CMAKE_REQUIRED_FLAGS}")
+            message(STATUS "Check in CMakeFiles/CMakeError.log to figure out why it fails")
+        endif()
+    endif()
+    set(CMAKE_REQUIRED_INCLUDES)
+    set(CMAKE_REQUIRED_FLAGS)
+    set(CMAKE_REQUIRED_LIBRARIES)
+
+    string(SUBSTRING "${HWLOC_VERSION}" 0 3 HWLOC_VERSION)
+    if(HWLOC_VERSION LESS_EQUAL HWLOC_FIND_VERSION)
+        message(STATUS "Required version ${HWLOC_FIND_VERSION}, but found version ${HWLOC_VERSION}")
+        set(HWLOC_FOUND 0)
+        unset(HWLOC_LIBRARIES)
+        unset(HWLOC_INCLUDE_DIRS)
+    else()
+        include(FindPackageHandleStandardArgs)
+        find_package_handle_standard_args(HWLOC
+            REQUIRED_VARS HWLOC_LIBRARIES HWLOC_INCLUDE_DIRS HWLOC_WORKS
+            VERSION_VAR HWLOC_VERSION)
+        mark_as_advanced(HWLOC_INCLUDE_DIRS HWLOC_LIBRARIES HWLOC_VERSION HWLOC_WORKS)
+    endif()
+
+endif(HWLOC_FOUND AND HWLOC_LIBRARIES)
+
+if(HWLOC_FOUND)
+    add_library(hwloc SHARED IMPORTED GLOBAL)
+    set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION ${HWLOC_LIBRARIES})
+    set_target_properties(hwloc PROPERTIES INTERFACE_LINK_LIBRARIES ${HWLOC_LIBRARIES})
+    set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${HWLOC_INCLUDE_DIRS})
 endif()
 
 if(HWLOC_FIND_REQUIRED AND NOT HWLOC_FOUND)
-    message(SEND_ERROR "HWLOC could not be found. A version mismatch could have occured. The version found was ${HWLOC_VERSION}")
+    unset(HWLOC_LIBRARIES)
+    unset(HWLOC_INCLUDE_DIRS)
+    message(SEND_ERROR "HWLOC could not be found. A version mismatch could have occured. The version found was ${HWLOC_VERSION}.\n"
+        "Hints where hwloc is installed can be given in the HWLOC_DIR variable. Current HWLOC_DIR: ${HWLOC_DIR}")
 endif()
diff --git a/cmake/Modules/hwloc_helpers.cmake b/cmake/Modules/hwloc_helpers.cmake
new file mode 100644
index 00000000000..41622611e8f
--- /dev/null
+++ b/cmake/Modules/hwloc_helpers.cmake
@@ -0,0 +1,87 @@
+# Modified function from Morse
+macro(ginkgo_set_required_test_lib_link name)
+    set(CMAKE_REQUIRED_INCLUDES "${${name}${STATIC}_INCLUDE_DIRS}")
+    if (${name}${STATIC}_CFLAGS_OTHER)
+        set(REQUIRED_FLAGS_COPY "${${name}${STATIC}_CFLAGS_OTHER}")
+        set(REQUIRED_FLAGS)
+        set(REQUIRED_DEFINITIONS)
+        foreach(_flag ${REQUIRED_FLAGS_COPY})
+            if (_flag MATCHES "^-D")
+                list(APPEND REQUIRED_DEFINITIONS "${_flag}")
+            endif()
+            string(REGEX REPLACE "^-D.*" "" _flag "${_flag}")
+            list(APPEND REQUIRED_FLAGS "${_flag}")
+        endforeach()
+    endif()
+    foreach(_var "${REQUIRED_FLAGS_COPY};${REQUIRED_FLAGS};${REQUIRED_LIBRARIES}" )
+        if(${_var})
+            list(REMOVE_DUPLICATES ${_var})
+        endif()
+    endforeach()
+    set(CMAKE_REQUIRED_DEFINITIONS "${REQUIRED_DEFINITIONS}")
+    set(CMAKE_REQUIRED_FLAGS "${REQUIRED_FLAGS}")
+    set(CMAKE_REQUIRED_LIBRARIES)
+    list(APPEND CMAKE_REQUIRED_LIBRARIES "${${name}${STATIC}_LDFLAGS_OTHER}")
+    list(APPEND CMAKE_REQUIRED_LIBRARIES "${${name}${STATIC}_LIBRARIES}")
+    string(REGEX REPLACE "^ -" "-" CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES}")
+endmacro()
+
+# Modified function from Morse
+macro(ginkgo_check_static_or_dynamic package libraries)
+    list(GET ${libraries} 0 _first_lib)
+    get_filename_component(_suffix ${_first_lib} EXT)
+    if (NOT _suffix)
+        unset (_lib_path CACHE)
+        find_library(_lib_path ${_first_lib} HINTS ${${package}_LIBDIR} ${${package}_LIBRARY_DIRS} NO_DEFAULT_PATH)
+        get_filename_component(_suffix ${_lib_path} EXT)
+    endif()
+    if (_suffix)
+        if(${_suffix} MATCHES ".so$" OR ${_suffix} MATCHES ".dylib$" OR ${_suffix} MATCHES ".dll$")
+            set(${package}_STATIC 0)
+        elseif(${_suffix} MATCHES ".a$")
+            set(${package}_STATIC 1)
+        else()
+            message(FATAL_ERROR "${package} library extension not in list .a, .so, .dylib, .dll")
+        endif()
+    else()
+        message(FATAL_ERROR "${package} could not detect library extension")
+    endif()
+endmacro()
+
+function(HEX2DEC str res)
+    string(LENGTH "${str}" len)
+    if("${len}" EQUAL 1)
+        if("${str}" MATCHES "[0-9]")
+            set(${res} "${str}" PARENT_SCOPE)
+        elseif( "${str}" MATCHES "[aA]")
+            set(${res} 10 PARENT_SCOPE)
+        elseif( "${str}" MATCHES "[bB]")
+            set(${res} 11 PARENT_SCOPE)
+        elseif( "${str}" MATCHES "[cC]")
+            set(${res} 12 PARENT_SCOPE)
+        elseif( "${str}" MATCHES "[dD]")
+            set(${res} 13 PARENT_SCOPE)
+        elseif( "${str}" MATCHES "[eE]")
+            set(${res} 14 PARENT_SCOPE)
+        elseif( "${str}" MATCHES "[fF]")
+            set(${res} 15 PARENT_SCOPE)
+        else()
+            return()
+        endif()
+    else()
+        string(SUBSTRING "${str}" 0 1 str1)
+        string(SUBSTRING "${str}" 1 -1 str2)
+        hex2dec(${str1} res1)
+        hex2dec(${str2} res2)
+        math(EXPR val "16 * ${res1} + ${res2}")
+        set(${res} "${val}" PARENT_SCOPE)
+    endif()
+endfunction()
+
+macro(get_dec_from_hex hex dec)
+    if(${CMAKE_VERSION} VERSION_GREATER 3.13)
+        math(EXPR ${dec} ${hex} OUTPUT_FORMAT DECIMAL)
+    else()
+        hex2dec(${hex} ${dec})
+    endif()
+endmacro()
diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index be4a03e9027..4ec9ddf4449 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -34,7 +34,8 @@ function(ginkgo_create_dpcpp_test test_name)
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
+    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest
+      GMock::Main GMock::GMock ${ARGN})
     add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
 endfunction(ginkgo_create_dpcpp_test)
 
@@ -55,7 +56,8 @@ function(ginkgo_create_thread_test test_name)
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest Threads::Threads ${ARGN})
+    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest
+        Threads::Threads ${ARGN})
     add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
 endfunction(ginkgo_create_thread_test)
 
@@ -122,7 +124,8 @@ function(ginkgo_create_hip_test_special_linkage test_name)
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
+    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest
+      GMock::Main GMock::GMock ${ARGN})
     add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
 endfunction(ginkgo_create_hip_test_special_linkage)
 
@@ -184,6 +187,7 @@ function(ginkgo_create_hip_test test_name)
         endif()
     endif()
 
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
+    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest
+      GMock::Main GMock::GMock ${ARGN})
     add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
 endfunction(ginkgo_create_hip_test)
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index 20785260965..b111a2e1d45 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -175,8 +175,6 @@ if(PAPI_sde_FOUND)
     ginkgo_print_variable(${detailed_log} "PAPI_INCLUDE_DIR")
     ginkgo_print_flags(${detailed_log} "PAPI_LIBRARY")
 endif()
-ginkgo_print_variable(${minimal_log} "GINKGO_USE_EXTERNAL_HWLOC")
-ginkgo_print_variable(${detailed_log} "GINKGO_USE_EXTERNAL_HWLOC")
 ginkgo_print_variable(${minimal_log} "GINKGO_FORCE_NO_HWLOC")
 ginkgo_print_variable(${detailed_log} "GINKGO_FORCE_NO_HWLOC")
 ginkgo_print_variable(${detailed_log} "HWLOC_VERSION")
diff --git a/cmake/package_helpers.cmake b/cmake/package_helpers.cmake
index bb3eeb39bf0..080fe8791af 100644
--- a/cmake/package_helpers.cmake
+++ b/cmake/package_helpers.cmake
@@ -4,13 +4,20 @@ set(PACKAGE_DOWNLOADER_SCRIPT
 set(NON_CMAKE_PACKAGE_DOWNLOADER_SCRIPT
     "${CMAKE_CURRENT_LIST_DIR}/DownloadNonCMakeCMakeLists.txt.in")
 
+
+#   Load a package from its repository
+#
+#   \param package_name  Name of the package
+#   \param package_url   Url of the package
+#   \param package_tag   Tag or version of the package to be downloaded.
+#
 function(ginkgo_load_git_package package_name package_url package_tag)
     set(GINKGO_THIRD_PARTY_BUILD_TYPE "Debug")
     if (CMAKE_BUILD_TYPE MATCHES "[Rr][Ee][Ll][Ee][Aa][Ss][Ee]")
         set(GINKGO_THIRD_PARTY_BUILD_TYPE "Release")
     endif()
     configure_file(${PACKAGE_DOWNLOADER_SCRIPT}
-                   download/CMakeLists.txt)
+        download/CMakeLists.txt)
     execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
         RESULT_VARIABLE result
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/download)
@@ -46,13 +53,20 @@ function(ginkgo_load_git_package package_name package_url package_tag)
 endfunction()
 
 
+#   Load a package from the url provided and run configure (Non-CMake projects)
+#
+#   \param package_name     Name of the package
+#   \param package_url      Url of the package
+#   \param package_tag      Tag or version of the package to be downloaded.
+#   \param config_command   The command for the configuration step.
+#
 function(ginkgo_load_and_configure_package package_name package_url package_hash config_command)
     set(GINKGO_THIRD_PARTY_BUILD_TYPE "Debug")
     if (CMAKE_BUILD_TYPE MATCHES "[Rr][Ee][Ll][Ee][Aa][Ss][Ee]")
         set(GINKGO_THIRD_PARTY_BUILD_TYPE "Release")
     endif()
     configure_file(${NON_CMAKE_PACKAGE_DOWNLOADER_SCRIPT}
-                   download/CMakeLists.txt)
+        download/CMakeLists.txt)
     execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
         RESULT_VARIABLE result
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/download)
@@ -61,11 +75,11 @@ function(ginkgo_load_and_configure_package package_name package_url package_hash
             "CMake step for ${package_name}/download failed: ${result}")
     endif()
     execute_process(COMMAND ${CMAKE_COMMAND} --build .
-      RESULT_VARIABLE result
-      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/download)
+        RESULT_VARIABLE result
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/download)
     if(result)
-      message(FATAL_ERROR
-        "Build step for ${package_name}/download failed: ${result}")
+        message(FATAL_ERROR
+            "Build step for ${package_name}/download failed: ${result}")
     endif()
 endfunction()
 
@@ -145,13 +159,13 @@ macro(ginkgo_add_external_target new_target external_name includedir libdir buil
         set_target_properties(${new_target} PROPERTIES IMPORTED_LOCATION_DEBUG ${${external_name}_LIBRARY_DEBUG})
         # Since we do not really manage other build types, let's globally use the DEBUG symbols
         if(MSVC)
-        # Only Debug build uses MDd or MTd, and others use MD or MT.
-        # MSVC would like to use same runtime library, so we use Debug third-party in Debug and Release third-party in others.
+            # Only Debug build uses MDd or MTd, and others use MD or MT.
+            # MSVC would like to use same runtime library, so we use Debug third-party in Debug and Release third-party in others.
             set_target_properties(${new_target} PROPERTIES IMPORTED_LOCATION
-                    ${${external_name}_LIBRARY_RELEASE})
+                ${${external_name}_LIBRARY_RELEASE})
         else()
             if (NOT CMAKE_BUILD_TYPE MATCHES "[Rr][Ee][Ll][Ee][Aa][Ss][Ee]"
-                AND NOT CMAKE_BUILD_TYPE MATCHES "[Dd][Ee][Bb][Uu][Gg]")
+                    AND NOT CMAKE_BUILD_TYPE MATCHES "[Dd][Ee][Bb][Uu][Gg]")
                 set_target_properties(${new_target} PROPERTIES IMPORTED_LOCATION
                     ${${external_name}_LIBRARY_DEBUG})
             endif()
@@ -198,9 +212,9 @@ macro(ginkgo_find_package package_name target_list header_only)
                 list(GET TPL_${_UPACKAGE_NAME}_LIBRARIES ${val} lib)
                 ginkgo_add_tpl_target("${target}" "${_UPACKAGE_NAME}" "${TPL_${_UPACKAGE_NAME}_INCLUDE_DIRS}"
                     "${lib}" ${header_only})
-             endforeach()
+            endforeach()
         else()
-            find_package(${package_name} QUIET ${ARGN})
+            find_package(${package_name} ${ARGN})
             if (${package_name}_FOUND)
                 message(STATUS "Using external version of package ${package_name}. In case of problems, consider setting -DGINKGO_USE_EXTERNAL_${_UPACKAGE_NAME}=OFF.")
             else()
@@ -213,19 +227,19 @@ endmacro(ginkgo_find_package)
 
 #   Download a file and verify the download
 #
-#   \param url Name of package to be found
-#   \param filename      For TPL packages, declare a new target for each library
-#   \param hash_type      For TPL packages, declare the tpl package as header only
-#   \param hash             Extra specifications for the package finder
+#   \param url          The url of file to be downloaded
+#   \param filename     The name of the file
+#   \param hash_type    The type of hash, See CMake file() documentation for more details.
+#   \param hash         The hash itself, See CMake file() documentation for more details.
 #
 function(ginkgo_download_file url filename hash_type hash)
-  file(DOWNLOAD ${url} ${filename}
-    TIMEOUT 60  # seconds
-    EXPECTED_HASH "${hash_type}=${hash}"
-    TLS_VERIFY ON)
-  if(EXISTS ${filename})
-    message(STATUS "${filename} downloaded from ${url}")
-  else()
-    message(FATAL_ERROR "Download of ${filename} failed.")
-  endif() 
+    file(DOWNLOAD ${url} ${filename}
+        TIMEOUT 60  # seconds
+        EXPECTED_HASH "${hash_type}=${hash}"
+        TLS_VERIFY ON)
+    if(EXISTS ${filename})
+        message(STATUS "${filename} downloaded from ${url}")
+    else()
+        message(FATAL_ERROR "Download of ${filename} failed.")
+    endif()
 endfunction(ginkgo_download_file)
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 17990398a40..f624c90c823 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -67,10 +67,6 @@ if (GINKGO_HAVE_PAPI_SDE)
     target_link_libraries(ginkgo PUBLIC PAPI::PAPI)
 endif()
 
-if(GINKGO_HAVE_HWLOC)
-  target_link_libraries(ginkgo PUBLIC hwloc)
-endif()
-
 ginkgo_default_includes(ginkgo)
 ginkgo_install_library(ginkgo core)
 
diff --git a/core/devices/CMakeLists.txt b/core/devices/CMakeLists.txt
index c7b409450bc..4502b7f9b93 100644
--- a/core/devices/CMakeLists.txt
+++ b/core/devices/CMakeLists.txt
@@ -6,21 +6,17 @@ function(ginkgo_add_object_library name)
     set_target_properties(${name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endfunction()
 
-function(ginkgo_add_normal_library name)
+function(ginkgo_add_library name)
     add_library(${name} ${ARGN})
     ginkgo_compile_features(${name})
     ginkgo_default_includes(${name})
     if(GINKGO_HAVE_HWLOC)
-        target_include_directories(${name}
-            PUBLIC 
-            $<BUILD_INTERFACE:${HWLOC_INCLUDE_DIRS}>
-        )
-        target_link_libraries(${name} PUBLIC ${HWLOC_LIBRARIES})
+        target_link_libraries(${name} PUBLIC hwloc)
     endif()
     set_target_properties(${name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endfunction()
-  
-ginkgo_add_normal_library(ginkgo_core_device machine_topology.cpp)
+
+ginkgo_add_library(ginkgo_core_device machine_topology.cpp)
 ginkgo_install_library(ginkgo_core_device core/devices)
 
 add_subdirectory(cuda)
diff --git a/core/devices/cuda/CMakeLists.txt b/core/devices/cuda/CMakeLists.txt
index 1b164165305..acd0455ff2a 100644
--- a/core/devices/cuda/CMakeLists.txt
+++ b/core/devices/cuda/CMakeLists.txt
@@ -1,3 +1,2 @@
 ginkgo_add_object_library(ginkgo_cuda_device
     executor.cpp)
-
diff --git a/core/devices/hip/CMakeLists.txt b/core/devices/hip/CMakeLists.txt
index 7e5b124a157..9995e51c5e7 100644
--- a/core/devices/hip/CMakeLists.txt
+++ b/core/devices/hip/CMakeLists.txt
@@ -1,4 +1,2 @@
 ginkgo_add_object_library(ginkgo_hip_device
     executor.cpp)
- 
-
diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index 6ecd373cae8..0deeda92f2c 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -214,8 +214,8 @@ inline int MachineTopology::get_obj_local_id_by_os_index(
             return id;
         }
     }
-    return -1;
 #endif
+    return -1;
 }
 
 
@@ -229,8 +229,8 @@ inline int MachineTopology::get_obj_local_id_by_gp_index(
             return id;
         }
     }
-    return -1;
 #endif
+    return -1;
 }
 
 
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 75433ae8cee..fc4a2b4f717 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -165,6 +165,9 @@ TEST(OmpExecutor, IsItsOwnMaster)
 }
 
 
+#if GKO_HAVE_HWLOC
+
+
 TEST(OmpExecutor, CanGetNumCpusFromExecInfo)
 {
 #if defined(_WIN32) || defined(__APPLE__) || defined(__CYGWIN__)
@@ -178,9 +181,6 @@ TEST(OmpExecutor, CanGetNumCpusFromExecInfo)
 }
 
 
-#if GKO_HAVE_HWLOC
-
-
 inline int get_os_id(int log_id)
 {
     return gko::get_machine_topology()->get_core(log_id)->os_id;
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index ec36f370b43..143b0eaec2e 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -328,8 +328,5 @@ if(GINKGO_BUILD_TESTS)
             set(GINKGO_RPATH_FOR_HIP "${GINKGO_RPATH_FOR_HIP}:$<TARGET_FILE_DIR:${target}>")
         endif()
     endforeach()
-    if(GINKGO_HAVE_HWLOC)
-        set(GINKGO_RPATH_FOR_HIP "${GINKGO_RPATH_FOR_HIP}:${HWLOC_LIBRARIES}")
-    endif()
     add_subdirectory(test)
 endif()
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index ddda8f74e95..2c3f3e44949 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/test_install/CMakeLists.txt b/test_install/CMakeLists.txt
index 53a588f3359..b400f7a7681 100644
--- a/test_install/CMakeLists.txt
+++ b/test_install/CMakeLists.txt
@@ -3,9 +3,9 @@ cmake_minimum_required(VERSION 3.9)
 project(TestInstall LANGUAGES CXX)
 
 find_package(Ginkgo REQUIRED
-            PATHS # The Path where ginkgo was installed
-            # Alternatively, use `cmake -DCMAKE_PREFIX_PATH=<ginkgo_install_dir>` to specify the install directory
-            )
+    PATHS # The Path where ginkgo was installed
+    # Alternatively, use `cmake -DCMAKE_PREFIX_PATH=<ginkgo_install_dir>` to specify the install directory
+    )
 
 if(MSVC)
     if(GINKGO_BUILD_SHARED_LIBS)
@@ -24,6 +24,11 @@ add_executable(test_install test_install.cpp)
 target_compile_features(test_install PUBLIC cxx_std_14)
 target_link_libraries(test_install PRIVATE Ginkgo::ginkgo)
 
+if(GINKGO_HAVE_HWLOC)
+    find_package(HWLOC REQUIRED)
+    target_link_libraries(test_install PRIVATE hwloc)
+endif()
+
 if(GINKGO_BUILD_CUDA)
     enable_language(CUDA)
     if(MSVC)
@@ -35,6 +40,9 @@ if(GINKGO_BUILD_CUDA)
     endif()
     add_executable(test_install_cuda test_install_cuda.cu)
     target_link_libraries(test_install_cuda PRIVATE Ginkgo::ginkgo)
+    if(GINKGO_HAVE_HWLOC)
+        target_link_libraries(test_install_cuda PRIVATE hwloc)
+    endif()
 endif()
 
 if(GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc"
diff --git a/third_party/gtest/CMakeLists.txt b/third_party/gtest/CMakeLists.txt
index 2ff09120582..31da9c5eda3 100644
--- a/third_party/gtest/CMakeLists.txt
+++ b/third_party/gtest/CMakeLists.txt
@@ -19,7 +19,3 @@ ginkgo_add_external_target(GTest::GTest gtest src/googletest/include build/lib
     STATIC "d" gtest_external FALSE)
 ginkgo_add_external_target(GTest::Main gtest_main src/googletest/include build/lib
     STATIC "d" gtest_external FALSE)
-ginkgo_add_external_target(GMock::GMock gmock src/googlemock/include build/lib
-    STATIC "d" gmock_external FALSE)
-ginkgo_add_external_target(GMock::Main gmock_main src/googlemock/include build/lib
-    STATIC "d" gmock_external FALSE)
diff --git a/third_party/hwloc/CMakeLists.txt b/third_party/hwloc/CMakeLists.txt
index 5e560d5a6d2..89f430ad18e 100644
--- a/third_party/hwloc/CMakeLists.txt
+++ b/third_party/hwloc/CMakeLists.txt
@@ -1,38 +1,41 @@
 message(STATUS "Configuring and building HWLOC")
-set(TPL_HWLOC_PATH "${CMAKE_CURRENT_BINARY_DIR}")
+set(TPL_HWLOC_PATH "${PROJECT_BINARY_DIR}/third_party/hwloc")
 if(MSVC OR WIN32)
-  message(STATUS "Downloading HWLOC binaries for win32")
-  ginkgo_download_file("https://download.open-mpi.org/release/hwloc/v2.3/hwloc-win32-build-2.3.0.zip"
-    "${TPL_HWLOC_PATH}/hwloc.zip"
-    "SHA1"
-    "1d53dc8db809dd19d8281556f5f3785105aba97f"
-    )
-  file(ARCHIVE_EXTRACT INPUT "${TPL_HWLOC_PATH}/hwloc.zip"
-    DESTINATION "${TPL_HWLOC_PATH}/hwloc"
-    VERBOSE
-    )
+    message(STATUS "Downloading HWLOC binaries for win32")
+    ginkgo_download_file("https://download.open-mpi.org/release/hwloc/v2.3/hwloc-win32-build-2.3.0.zip"
+        "${TPL_HWLOC_PATH}/hwloc.zip"
+        "SHA1"
+        "1d53dc8db809dd19d8281556f5f3785105aba97f"
+        )
+    file(ARCHIVE_EXTRACT INPUT "${TPL_HWLOC_PATH}/hwloc.zip"
+        DESTINATION "${TPL_HWLOC_PATH}/hwloc"
+        VERBOSE
+        )
 else()
-  message(STATUS "Downloading HWLOC tars and building")
-  ginkgo_load_and_configure_package(hwloc_external "https://download.open-mpi.org/release/hwloc/v2.3/hwloc-2.3.0.tar.gz"
-    "SHA1=69d8e2f088d365cf5e86d04f049c6b1e09e2b9c3"
-    "${TPL_HWLOC_PATH}/src/configure"
-    )
+    message(STATUS "Downloading HWLOC tars and building")
+    ginkgo_load_and_configure_package(hwloc_external "https://download.open-mpi.org/release/hwloc/v2.3/hwloc-2.3.0.tar.gz"
+        "SHA1=69d8e2f088d365cf5e86d04f049c6b1e09e2b9c3"
+        "${TPL_HWLOC_PATH}/src/configure"
+        )
 
 endif()
 
 if(MSVC OR WIN32)
-  add_library(hwloc STATIC IMPORTED GLOBAL)
-  add_dependencies(hwloc hwloc_external)
-  set(HWLOC_LIBRARIES "${TPL_HWLOC_PATH}/hwloc/hwloc-win32-build-2.3.0/lib/libhwloc.a" CACHE FILEPATH "The path to HWLOC library libhwloc.so" FORCE)
-  set(HWLOC_INCLUDE_DIRS "${TPL_HWLOC_PATH}/hwloc/hwloc-win32-build-2.3.0/include" CACHE PATH "The directory containing the hwloc header, hwloc.h" FORCE)
-  set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION ${HWLOC_LIBRARIES})
-  set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${HWLOC_INCLUDE_DIRS})
+    set(HWLOC_LIBRARIES "${TPL_HWLOC_PATH}/hwloc/hwloc-win32-build-2.3.0/lib/libhwloc.a" CACHE FILEPATH "The path to HWLOC library libhwloc.so" FORCE)
+    set(HWLOC_INCLUDE_DIRS "${TPL_HWLOC_PATH}/hwloc/hwloc-win32-build-2.3.0/include" CACHE PATH "The directory containing the hwloc header, hwloc.h" FORCE)
 else()
-  add_library(hwloc SHARED IMPORTED GLOBAL)
-  add_dependencies(hwloc hwloc_external )
-  set(HWLOC_LIBRARIES "${TPL_HWLOC_PATH}/build/hwloc/.libs/libhwloc.so" CACHE FILEPATH "The path to HWLOC library libhwloc.so" FORCE)
-  execute_process(COMMAND cp "${TPL_HWLOC_PATH}/build/include/hwloc/autogen/config.h" "${TPL_HWLOC_PATH}/src/include/hwloc/autogen/")
-  set(HWLOC_INCLUDE_DIRS "${TPL_HWLOC_PATH}/src/include" CACHE PATH "The directory containing the hwloc header, hwloc.h" FORCE)
-  set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION ${HWLOC_LIBRARIES})
-  set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${TPL_HWLOC_PATH}/src/include")
+    add_library(hwloc SHARED IMPORTED GLOBAL)
+    add_dependencies(hwloc hwloc_external )
+    set(HWLOC_DIR "${TPL_HWLOC_PATH}" CACHE FILEPATH "The path to HWLOC")
+    file(MAKE_DIRECTORY ${TPL_HWLOC_PATH}/lib/)
+    file(GLOB HWLOC_LIBS "${TPL_HWLOC_PATH}/build/hwloc/.libs/libhwloc*")
+    execute_process(COMMAND cp "${TPL_HWLOC_PATH}/build/include/hwloc/autogen/config.h" "${TPL_HWLOC_PATH}/src/include/hwloc/autogen/")
+    foreach(lib ${HWLOC_LIBS})
+        execute_process(COMMAND cp "${lib}" "${TPL_HWLOC_PATH}/lib/")
+    endforeach()
+    set(HWLOC_LIBRARIES "${TPL_HWLOC_PATH}/lib/libhwloc.so" CACHE FILEPATH "The path to HWLOC library libhwloc.so" FORCE)
+    set(HWLOC_INCLUDE_DIRS "${TPL_HWLOC_PATH}/src/include" CACHE PATH "The directory containing the hwloc header, hwloc.h" FORCE)
+    set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION ${HWLOC_LIBRARIES})
+    set_target_properties(hwloc PROPERTIES INTERFACE_LINK_LIBRARIES ${HWLOC_LIBRARIES})
+    set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${HWLOC_INCLUDE_DIRS}")
 endif()

From 94ead99c3ccb27fcbf16b6c5d39933e1027ed833 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Sun, 10 Jan 2021 23:56:40 +0100
Subject: [PATCH 0299/1008] Review update.

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 cmake/create_test.cmake                       | 12 ++---
 core/devices/machine_topology.cpp             | 31 +++++------
 core/test/base/executor.cpp                   |  6 +--
 cuda/test/base/cuda_executor.cu               |  6 +--
 hip/test/base/hip_executor.hip.cpp            |  6 +--
 include/ginkgo/core/base/machine_topology.hpp | 54 +++++++++----------
 6 files changed, 50 insertions(+), 65 deletions(-)

diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index 4ec9ddf4449..d12a479743f 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -13,8 +13,7 @@ function(ginkgo_create_test test_name)
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest
-      GMock::Main GMock::GMock ${ARGN})
+    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
     add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
 endfunction(ginkgo_create_test)
 
@@ -34,8 +33,7 @@ function(ginkgo_create_dpcpp_test test_name)
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest
-      GMock::Main GMock::GMock ${ARGN})
+    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
     add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
 endfunction(ginkgo_create_dpcpp_test)
 
@@ -124,8 +122,7 @@ function(ginkgo_create_hip_test_special_linkage test_name)
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest
-      GMock::Main GMock::GMock ${ARGN})
+    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
     add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
 endfunction(ginkgo_create_hip_test_special_linkage)
 
@@ -187,7 +184,6 @@ function(ginkgo_create_hip_test test_name)
         endif()
     endif()
 
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest
-      GMock::Main GMock::GMock ${ARGN})
+    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
     add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
 endfunction(ginkgo_create_hip_test)
diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index 0deeda92f2c..a9038952fb3 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -109,8 +109,8 @@ MachineTopology::MachineTopology()
 
 
 void MachineTopology::hwloc_binding_helper(
-    const std::tuple<std::vector<MachineTopology::normal_obj_info>,
-                     MachineTopology::hwloc_manager<hwloc_bitmap_s>> &obj,
+    const std::tuple<std::vector<MachineTopology::normal_obj_info>, topo_bitmap>
+        &obj,
     const int *id, const size_type num_ids,
     const MachineTopology::BitMapType &bitmap_type) const
 {
@@ -159,24 +159,25 @@ void MachineTopology::hwloc_binding_helper(
 }
 
 
-void MachineTopology::hwloc_print_children(const hwloc_obj_t obj,
+void MachineTopology::hwloc_print_children(std::ostream &os,
+                                           const hwloc_obj_t obj,
                                            const int depth)
 {
 #if GKO_HAVE_HWLOC
     char type[32], attr[1024];
     unsigned i;
     hwloc_obj_type_snprintf(type, sizeof(type), obj, 0);
-    std::clog << std::string(2 * depth, ' ') << type;
+    os << std::string(2 * depth, ' ') << type;
     if (obj->os_index != (unsigned)-1) {
         std::clog << "#" << obj->os_index;
     }
     hwloc_obj_attr_snprintf(attr, sizeof(attr), obj, " ", 0);
     if (*attr) {
-        std::clog << "(" << attr << ")";
+        os << "(" << attr << ")";
     }
-    std::clog << std::endl;
+    os << std::endl;
     for (i = 0; i < obj->arity; i++) {
-        hwloc_print_children(obj->children[i], depth + 1);
+        hwloc_print_children(os, obj->children[i], depth + 1);
     }
 #endif
 }
@@ -184,8 +185,8 @@ void MachineTopology::hwloc_print_children(const hwloc_obj_t obj,
 
 void MachineTopology::load_objects(
     hwloc_obj_type_t type,
-    std::tuple<std::vector<MachineTopology::normal_obj_info>,
-               MachineTopology::hwloc_manager<hwloc_bitmap_s>> &objects)
+    std::tuple<std::vector<MachineTopology::normal_obj_info>, topo_bitmap>
+        &objects)
 {
 #if GKO_HAVE_HWLOC
     std::get<1>(objects) =
@@ -204,7 +205,7 @@ void MachineTopology::load_objects(
 }
 
 
-inline int MachineTopology::get_obj_local_id_by_os_index(
+inline int MachineTopology::get_obj_id_by_os_index(
     std::vector<MachineTopology::normal_obj_info> &objects,
     size_type os_index) const
 {
@@ -219,7 +220,7 @@ inline int MachineTopology::get_obj_local_id_by_os_index(
 }
 
 
-inline int MachineTopology::get_obj_local_id_by_gp_index(
+inline int MachineTopology::get_obj_id_by_gp_index(
     std::vector<MachineTopology::normal_obj_info> &objects,
     size_type gp_index) const
 {
@@ -257,18 +258,18 @@ void MachineTopology::load_objects(
                                   ancestor->nodeset);
         // Find the cpu object closest to this device from the ancestor cpuset
         // and store its id for binding purposes
-        vector.back().closest_cpu_id = get_obj_local_id_by_os_index(
+        vector.back().closest_cpu_id = get_obj_id_by_os_index(
             std::get<0>(this->pus_), hwloc_bitmap_first(ancestor_cpuset));
         // Get local id of the ancestor object.
         if (hwloc_compare_types(ancestor->type, HWLOC_OBJ_PACKAGE) == 0) {
-            vector.back().ancestor_local_id = get_obj_local_id_by_gp_index(
+            vector.back().ancestor_local_id = get_obj_id_by_gp_index(
                 std::get<0>(this->packages_), ancestor->gp_index);
         } else if (hwloc_compare_types(ancestor->type, HWLOC_OBJ_CORE) == 0) {
-            vector.back().ancestor_local_id = get_obj_local_id_by_gp_index(
+            vector.back().ancestor_local_id = get_obj_id_by_gp_index(
                 std::get<0>(this->cores_), ancestor->gp_index);
         } else if (hwloc_compare_types(ancestor->type, HWLOC_OBJ_NUMANODE) ==
                    0) {
-            vector.back().ancestor_local_id = get_obj_local_id_by_gp_index(
+            vector.back().ancestor_local_id = get_obj_id_by_gp_index(
                 std::get<0>(this->numa_nodes_), ancestor->gp_index);
         }
         hwloc_bitmap_free(ancestor_cpuset);
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index fc4a2b4f717..4dc90bfa71e 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -42,9 +42,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 
 
-#include <gmock/gmock.h>
-
-
 #include <gtest/gtest.h>
 
 
@@ -215,8 +212,7 @@ TEST(OmpExecutor, CanBindToARangeofCores)
     gko::get_machine_topology()->bind_to_cores(bind_core, 2);
 
     cpu_sys = sched_getcpu();
-    EXPECT_THAT(cpu_sys, testing::AnyOf(testing::Eq(get_os_id(3)),
-                                        testing::Eq(get_os_id(6))));
+    ASSERT_TRUE(cpu_sys == get_os_id(3) || cpu_sys == get_os_id(6));
 }
 
 
diff --git a/cuda/test/base/cuda_executor.cu b/cuda/test/base/cuda_executor.cu
index 1019cef34da..2c1cc364723 100644
--- a/cuda/test/base/cuda_executor.cu
+++ b/cuda/test/base/cuda_executor.cu
@@ -43,9 +43,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 
 
-#include <gmock/gmock.h>
-
-
 #include <gtest/gtest.h>
 
 
@@ -161,8 +158,7 @@ TEST_F(CudaExecutor, CanBindToCores)
     cuda->bind_to_cores(bind_core, 2);
 
     cpu_sys = sched_getcpu();
-    EXPECT_THAT(cpu_sys, testing::AnyOf(testing::Eq(get_os_id(3)),
-                                        testing::Eq(get_os_id(6))));
+    ASSERT_TRUE(cpu_sys == get_os_id(3) || cpu_sys == get_os_id(6));
 }
 
 
diff --git a/hip/test/base/hip_executor.hip.cpp b/hip/test/base/hip_executor.hip.cpp
index 057438148b6..c52a96c22e5 100644
--- a/hip/test/base/hip_executor.hip.cpp
+++ b/hip/test/base/hip_executor.hip.cpp
@@ -49,9 +49,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 
 
-#include <gmock/gmock.h>
-
-
 #include <gtest/gtest.h>
 
 
@@ -165,8 +162,7 @@ TEST_F(HipExecutor, CanBindToCores)
     hip->bind_to_cores(bind_core, 2);
 
     cpu_sys = sched_getcpu();
-    EXPECT_THAT(cpu_sys, testing::AnyOf(testing::Eq(get_os_id(3)),
-                                        testing::Eq(get_os_id(6))));
+    ASSERT_TRUE(cpu_sys == get_os_id(3) || cpu_sys == get_os_id(6));
 }
 
 
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index 2c3f3e44949..86cf6bdfea7 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -76,9 +76,9 @@ class MachineTopology;
  * the MachineTopology global object in a shared_ptr that any gko function/class
  * can query.
  *
- * This makes sure that that the topology object is not re-initialized every
- * time it is needed and as the topology is is relevant only for the machine,
- * each re-initialization populates the exact same topology tree.
+ * This makes sure that the topology object is not re-initialized every
+ * time it is needed. Since the topology is tied to the machine anyway,
+ * each re-initialization would populate the exact same topology tree.
  *
  * @return  the global MachineTopology object.
  */
@@ -86,7 +86,7 @@ extern const MachineTopology *get_machine_topology();
 
 
 /**
- * The machine topology class represents the heirarchical topology of a machine,
+ * The machine topology class represents the hierarchical topology of a machine,
  * including NUMA nodes, cores and PCI Devices. Various infomation of the
  * machine are gathered with the help of the Hardware Locality library (hwloc).
  *
@@ -95,13 +95,13 @@ extern const MachineTopology *get_machine_topology();
  * closer to the executing object.
  *
  * See the hwloc documentation
- * (https://www.open-mpi.org/projects/hwloc/doc/v2.4.0/) for a more detailed
- * documentation on topology detection and binding interfaces.
+ * (https://www.open-mpi.org/projects/hwloc/doc/) for more detailed
+ * information on topology detection and binding interfaces.
  *
  * @note A global object of MachineTopology type is created when first needed
  * and only destroyed at the end of the program. This means that any subsequent
- * queries will be from the same global object and hence atomic on
- * multi-threaded queries.
+ * queries will be from the same global object and hence use an extra atomic
+ * read.
  */
 class MachineTopology {
     friend const MachineTopology *get_machine_topology();
@@ -109,6 +109,9 @@ class MachineTopology {
     template <typename T>
     using hwloc_manager = std::unique_ptr<T, std::function<void(T *)>>;
 
+    using topo_bitmap =
+        std::unique_ptr<hwloc_bitmap_s, std::function<void(hwloc_bitmap_s *)>>;
+
 private:
     /**
      * This struct holds the attributes for a normal non-IO object.
@@ -361,7 +364,7 @@ class MachineTopology {
      */
     void hwloc_binding_helper(
         const std::tuple<std::vector<MachineTopology::normal_obj_info>,
-                         MachineTopology::hwloc_manager<hwloc_bitmap_s>> &obj,
+                         topo_bitmap> &obj,
         const int *id, const size_type num_ids,
         const BitMapType &bitmap_type) const;
 
@@ -371,7 +374,8 @@ class MachineTopology {
      * A helper function that prints the topology tree of an object to a given
      * depth. Provided from the hwloc library.
      */
-    void hwloc_print_children(const hwloc_obj_t obj, const int depth = 1);
+    void hwloc_print_children(std::ostream &os, const hwloc_obj_t obj,
+                              const int depth = 1);
 
     /**
      * @internal
@@ -381,9 +385,9 @@ class MachineTopology {
      * @note The objects should be sorted by logical index since hwloc uses
      * logical index with these functions
      */
-    void load_objects(hwloc_obj_type_t type,
-                      std::tuple<std::vector<normal_obj_info>,
-                                 hwloc_manager<hwloc_bitmap_s>> &objects);
+    void load_objects(
+        hwloc_obj_type_t type,
+        std::tuple<std::vector<normal_obj_info>, topo_bitmap> &objects);
 
     /**
      * @internal
@@ -413,30 +417,26 @@ class MachineTopology {
      *
      * @internal
      *
-     * Get local id from the os index
+     * Get object id from the os index
      */
-    inline int get_obj_local_id_by_os_index(
-        std::vector<normal_obj_info> &objects, size_type os_index) const;
+    inline int get_obj_id_by_os_index(std::vector<normal_obj_info> &objects,
+                                      size_type os_index) const;
 
     /**
      *
      * @internal
      *
-     * Get local id from the hwloc index
+     * Get object id from the hwloc index
      */
-    inline int get_obj_local_id_by_gp_index(
-        std::vector<normal_obj_info> &objects, size_type gp_index) const;
+    inline int get_obj_id_by_gp_index(std::vector<normal_obj_info> &objects,
+                                      size_type gp_index) const;
 
 
 private:
-    std::tuple<std::vector<normal_obj_info>, hwloc_manager<hwloc_bitmap_s>>
-        pus_;
-    std::tuple<std::vector<normal_obj_info>, hwloc_manager<hwloc_bitmap_s>>
-        cores_;
-    std::tuple<std::vector<normal_obj_info>, hwloc_manager<hwloc_bitmap_s>>
-        packages_;
-    std::tuple<std::vector<normal_obj_info>, hwloc_manager<hwloc_bitmap_s>>
-        numa_nodes_;
+    std::tuple<std::vector<normal_obj_info>, topo_bitmap> pus_;
+    std::tuple<std::vector<normal_obj_info>, topo_bitmap> cores_;
+    std::tuple<std::vector<normal_obj_info>, topo_bitmap> packages_;
+    std::tuple<std::vector<normal_obj_info>, topo_bitmap> numa_nodes_;
     std::vector<io_obj_info> pci_devices_;
     size_type num_numas_;
 

From 8f0c84278d0974e50b1f08885253ce54c4444979 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 14 Jan 2021 01:34:51 +0100
Subject: [PATCH 0300/1008] Review update.

+ Make closest CPUs an array.
+ Allow binding to CPUs and cores, by default to CPUs.
+ Test NUMA and verify closest cpu has been set.
+ Move topology tests to separate file.
+ Remove the unnecessary bitmap logic

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 core/devices/cuda/executor.cpp                |   7 +
 core/devices/hip/executor.cpp                 |   7 +
 core/devices/machine_topology.cpp             | 154 ++++-------------
 core/test/base/executor.cpp                   |   9 -
 cuda/base/executor.cpp                        |   4 +-
 cuda/test/base/CMakeLists.txt                 |   3 +
 cuda/test/base/cuda_executor.cu               |  33 ----
 cuda/test/base/cuda_executor_topology.cu      | 155 +++++++++++++++++
 hip/base/executor.hip.cpp                     |   4 +-
 hip/test/base/CMakeLists.txt                  |   3 +
 hip/test/base/hip_executor.hip.cpp            |   3 -
 hip/test/base/hip_executor_topology.hip.cpp   | 160 ++++++++++++++++++
 include/ginkgo/core/base/executor.hpp         |  96 ++++++++---
 include/ginkgo/core/base/machine_topology.hpp |  82 +++------
 14 files changed, 469 insertions(+), 251 deletions(-)
 create mode 100644 cuda/test/base/cuda_executor_topology.cu
 create mode 100644 hip/test/base/hip_executor_topology.hip.cpp

diff --git a/core/devices/cuda/executor.cpp b/core/devices/cuda/executor.cpp
index e0bc21a2cf2..1bf34b5ffc8 100644
--- a/core/devices/cuda/executor.cpp
+++ b/core/devices/cuda/executor.cpp
@@ -64,6 +64,13 @@ bool CudaExecutor::verify_memory_to(const HipExecutor *dest_exec) const
 }
 
 
+void CudaExecutor::bind_to_cpus(const int *cpu_ids,
+                                const size_type num_ids) const
+{
+    get_machine_topology()->bind_to_pus(cpu_ids, num_ids);
+}
+
+
 void CudaExecutor::bind_to_cores(const int *core_ids,
                                  const size_type num_ids) const
 {
diff --git a/core/devices/hip/executor.cpp b/core/devices/hip/executor.cpp
index 0e18eb2b333..9aac5b54c30 100644
--- a/core/devices/hip/executor.cpp
+++ b/core/devices/hip/executor.cpp
@@ -61,6 +61,13 @@ bool HipExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
 }
 
 
+void HipExecutor::bind_to_cpus(const int *cpu_ids,
+                               const size_type num_ids) const
+{
+    get_machine_topology()->bind_to_pus(cpu_ids, num_ids);
+}
+
+
 void HipExecutor::bind_to_cores(const int *core_ids,
                                 const size_type num_ids) const
 {
diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index a9038952fb3..b42517cbb43 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -109,104 +109,47 @@ MachineTopology::MachineTopology()
 
 
 void MachineTopology::hwloc_binding_helper(
-    const std::tuple<std::vector<MachineTopology::normal_obj_info>, topo_bitmap>
-        &obj,
-    const int *id, const size_type num_ids,
-    const MachineTopology::BitMapType &bitmap_type) const
+    const std::vector<MachineTopology::normal_obj_info> &obj, const int *id,
+    const size_type num_ids) const
 {
 #if GKO_HAVE_HWLOC
-    auto bitmap_cur = hwloc_bitmap_alloc();
     auto bitmap_toset = hwloc_bitmap_alloc();
-    auto bitmap_res = hwloc_bitmap_alloc();
-    // Get current bitmap
-    hwloc_get_cpubind(this->topo_.get(), bitmap_cur, 0);
-    // Set the given ids to a new bitmap
+    // Set the given ids to a bitmap
     for (auto i = 0; i < num_ids; ++i) {
-        GKO_ASSERT(id[i] < std::get<0>(obj).size());
+        GKO_ASSERT(id[i] < obj.size());
         GKO_ASSERT(id[i] >= 0);
-        hwloc_bitmap_set(bitmap_toset, std::get<0>(obj)[id[i]].os_id);
-        // Check if this id has been bound before, Binding multiple threads to
-        // same object may not be suitable.
-        if (hwloc_bitmap_isset(std::get<1>(obj).get(),
-                               std::get<0>(obj)[id[i]].os_id)) {
-            hwloc_bitmap_clr(bitmap_toset, std::get<0>(obj)[id[i]].os_id);
-        }
-    }
-    // Log the bound ids to the global bitmap.
-    hwloc_bitmap_and(std::get<1>(obj).get(), std::get<1>(obj).get(),
-                     bitmap_toset);
-    // Depending on the passed in BitMapType, do the binary operation
-    // Op(current_bitmap, toset_bitmap) and use the resulting bitmap to bind the
-    // cpu.
-    if (bitmap_type == BitMapType::bitmap_set) {
-        hwloc_bitmap_free(bitmap_res);
-        bitmap_res = hwloc_bitmap_dup(bitmap_toset);
-    } else if (bitmap_type == BitMapType::bitmap_or) {
-        hwloc_bitmap_or(bitmap_res, bitmap_cur, bitmap_toset);
-    } else if (bitmap_type == BitMapType::bitmap_and) {
-        hwloc_bitmap_and(bitmap_res, bitmap_cur, bitmap_toset);
-    } else if (bitmap_type == BitMapType::bitmap_andnot) {
-        hwloc_bitmap_andnot(bitmap_res, bitmap_cur, bitmap_toset);
+        hwloc_bitmap_set(bitmap_toset, obj[id[i]].os_id);
     }
 
     // Singlify to reduce expensive migrations.
-    hwloc_bitmap_singlify(bitmap_res);
-    hwloc_set_cpubind(this->topo_.get(), bitmap_res, 0);
-    hwloc_bitmap_free(bitmap_cur);
+    hwloc_bitmap_singlify(bitmap_toset);
+    hwloc_set_cpubind(this->topo_.get(), bitmap_toset, 0);
     hwloc_bitmap_free(bitmap_toset);
-    hwloc_bitmap_free(bitmap_res);
-#endif
-}
-
-
-void MachineTopology::hwloc_print_children(std::ostream &os,
-                                           const hwloc_obj_t obj,
-                                           const int depth)
-{
-#if GKO_HAVE_HWLOC
-    char type[32], attr[1024];
-    unsigned i;
-    hwloc_obj_type_snprintf(type, sizeof(type), obj, 0);
-    os << std::string(2 * depth, ' ') << type;
-    if (obj->os_index != (unsigned)-1) {
-        std::clog << "#" << obj->os_index;
-    }
-    hwloc_obj_attr_snprintf(attr, sizeof(attr), obj, " ", 0);
-    if (*attr) {
-        os << "(" << attr << ")";
-    }
-    os << std::endl;
-    for (i = 0; i < obj->arity; i++) {
-        hwloc_print_children(os, obj->children[i], depth + 1);
-    }
 #endif
 }
 
 
 void MachineTopology::load_objects(
     hwloc_obj_type_t type,
-    std::tuple<std::vector<MachineTopology::normal_obj_info>, topo_bitmap>
-        &objects)
+    std::vector<MachineTopology::normal_obj_info> &objects) const
 {
 #if GKO_HAVE_HWLOC
-    std::get<1>(objects) =
-        hwloc_manager<hwloc_bitmap_s>(init_bitmap(), hwloc_bitmap_free);
     // Get the number of normal objects of a certain type (Core, PU, Machine
     // etc.).
     unsigned num_objects = hwloc_get_nbobjs_by_type(this->topo_.get(), type);
     for (unsigned i = 0; i < num_objects; i++) {
         // Get the actual normal object of the given type.
         hwloc_obj_t obj = hwloc_get_obj_by_type(this->topo_.get(), type, i);
-        std::get<0>(objects).push_back(
-            normal_obj_info{obj, obj->logical_index, obj->os_index,
-                            obj->gp_index, hwloc_bitmap_first(obj->nodeset)});
+        objects.push_back(normal_obj_info{obj, obj->logical_index,
+                                          obj->os_index, obj->gp_index,
+                                          hwloc_bitmap_first(obj->nodeset)});
     }
 #endif
 }
 
 
 inline int MachineTopology::get_obj_id_by_os_index(
-    std::vector<MachineTopology::normal_obj_info> &objects,
+    const std::vector<MachineTopology::normal_obj_info> &objects,
     size_type os_index) const
 {
 #if GKO_HAVE_HWLOC
@@ -221,7 +164,7 @@ inline int MachineTopology::get_obj_id_by_os_index(
 
 
 inline int MachineTopology::get_obj_id_by_gp_index(
-    std::vector<MachineTopology::normal_obj_info> &objects,
+    const std::vector<MachineTopology::normal_obj_info> &objects,
     size_type gp_index) const
 {
 #if GKO_HAVE_HWLOC
@@ -236,11 +179,12 @@ inline int MachineTopology::get_obj_id_by_gp_index(
 
 
 void MachineTopology::load_objects(
-    hwloc_obj_type_t type, std::vector<MachineTopology::io_obj_info> &vector)
+    hwloc_obj_type_t type,
+    std::vector<MachineTopology::io_obj_info> &vector) const
 {
 #if GKO_HAVE_HWLOC
-    GKO_ASSERT(std::get<0>(this->cores_).size() != 0);
-    GKO_ASSERT(std::get<0>(this->pus_).size() != 0);
+    GKO_ASSERT(this->cores_.size() != 0);
+    GKO_ASSERT(this->pus_.size() != 0);
     unsigned num_objects = hwloc_get_nbobjs_by_type(this->topo_.get(), type);
     for (unsigned i = 0; i < num_objects; i++) {
         // Get the actual PCI object.
@@ -256,21 +200,28 @@ void MachineTopology::load_objects(
         hwloc_cpuset_t ancestor_cpuset = hwloc_bitmap_alloc();
         hwloc_cpuset_from_nodeset(this->topo_.get(), ancestor_cpuset,
                                   ancestor->nodeset);
-        // Find the cpu object closest to this device from the ancestor cpuset
-        // and store its id for binding purposes
-        vector.back().closest_cpu_id = get_obj_id_by_os_index(
-            std::get<0>(this->pus_), hwloc_bitmap_first(ancestor_cpuset));
+        // Find the cpu objects closest to this device from the ancestor cpuset
+        // and store their ids for binding purposes
+        int closest_cpu_id = -1;
+        int closest_os_id = hwloc_bitmap_first(ancestor_cpuset);
+        // clang-format off
+        hwloc_bitmap_foreach_begin(closest_os_id, ancestor_cpuset)
+            closest_cpu_id = get_obj_id_by_os_index(this->pus_, closest_os_id);
+            vector.back().closest_cpu_ids.push_back(closest_cpu_id);
+        hwloc_bitmap_foreach_end();
+        // clang-format on
+
         // Get local id of the ancestor object.
         if (hwloc_compare_types(ancestor->type, HWLOC_OBJ_PACKAGE) == 0) {
-            vector.back().ancestor_local_id = get_obj_id_by_gp_index(
-                std::get<0>(this->packages_), ancestor->gp_index);
+            vector.back().ancestor_local_id =
+                get_obj_id_by_gp_index(this->packages_, ancestor->gp_index);
         } else if (hwloc_compare_types(ancestor->type, HWLOC_OBJ_CORE) == 0) {
-            vector.back().ancestor_local_id = get_obj_id_by_gp_index(
-                std::get<0>(this->cores_), ancestor->gp_index);
+            vector.back().ancestor_local_id =
+                get_obj_id_by_gp_index(this->cores_, ancestor->gp_index);
         } else if (hwloc_compare_types(ancestor->type, HWLOC_OBJ_NUMANODE) ==
                    0) {
-            vector.back().ancestor_local_id = get_obj_id_by_gp_index(
-                std::get<0>(this->numa_nodes_), ancestor->gp_index);
+            vector.back().ancestor_local_id =
+                get_obj_id_by_gp_index(this->numa_nodes_, ancestor->gp_index);
         }
         hwloc_bitmap_free(ancestor_cpuset);
         // Get type of the ancestor object and store it as a string.
@@ -283,48 +234,11 @@ void MachineTopology::load_objects(
                  obj->attr->pcidev.domain, obj->attr->pcidev.bus,
                  obj->attr->pcidev.dev, obj->attr->pcidev.func);
         vector.back().pci_bus_id = std::string(pci_bus_id);
-        // Get the number of IO children if any. For example, the software
-        // OS devices (cuda, rsmi, ib) are listed as IO devices under a PCI
-        // device.
-        auto num_io_children = obj->io_arity;
-        hwloc_obj_t os_child{};
-        auto rem_io_children = num_io_children;
-        // Get the IO children and their names and store them in vectors.
-        if (num_io_children > 0) {
-            // Get the first child
-            os_child = obj->io_first_child;
-            vector.back().io_children.push_back(os_child);
-            vector.back().io_children_name.emplace_back(os_child->name);
-            rem_io_children -= 1;
-            // If the io obj has more than one children, they are siblings to
-            // the first child. Store these as well.
-            hwloc_obj_t os_child_n = os_child;
-            while (rem_io_children >= 1) {
-                os_child_n = os_child_n->next_sibling;
-                vector.back().io_children.push_back(os_child_n);
-                vector.back().io_children_name.emplace_back(os_child_n->name);
-                rem_io_children -= 1;
-            }
-            GKO_ASSERT(vector.back().io_children.size() ==
-                       vector.back().io_children_name.size());
-        }
     }
 #endif
 }
 
 
-hwloc_bitmap_s *MachineTopology::init_bitmap()
-{
-#if GKO_HAVE_HWLOC
-    hwloc_bitmap_t tmp = hwloc_bitmap_alloc();
-    return tmp;
-#else
-    // MSVC complains if there is no return statement.
-    return nullptr;
-#endif
-}
-
-
 hwloc_topology *MachineTopology::init_topology()
 {
 #if GKO_HAVE_HWLOC
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 4dc90bfa71e..69a44ce7509 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -167,9 +167,6 @@ TEST(OmpExecutor, IsItsOwnMaster)
 
 TEST(OmpExecutor, CanGetNumCpusFromExecInfo)
 {
-#if defined(_WIN32) || defined(__APPLE__) || defined(__CYGWIN__)
-    GTEST_SKIP() << " No useful machine topology information to test against";
-#endif
     auto omp = gko::OmpExecutor::create();
 
     auto num_cpus = omp->get_num_cores() * omp->get_num_threads_per_core();
@@ -186,9 +183,6 @@ inline int get_os_id(int log_id)
 
 TEST(OmpExecutor, CanBindToASpecificCore)
 {
-#if defined(_WIN32) || defined(__APPLE__) || defined(__CYGWIN__)
-    GTEST_SKIP() << " No useful machine topology information to test against";
-#endif
     auto omp = gko::OmpExecutor::create();
     auto cpu_sys = sched_getcpu();
 
@@ -202,9 +196,6 @@ TEST(OmpExecutor, CanBindToASpecificCore)
 
 TEST(OmpExecutor, CanBindToARangeofCores)
 {
-#if defined(_WIN32) || defined(__APPLE__) || defined(__CYGWIN__)
-    GTEST_SKIP() << " No useful machine topology information to test against";
-#endif
     auto omp = gko::OmpExecutor::create();
     auto cpu_sys = sched_getcpu();
 
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 2eb0d997403..dbef057fc65 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -81,8 +81,8 @@ void CudaExecutor::populate_exec_info(const MachineTopology *mach_topo)
     auto cuda_hwloc_obj =
         mach_topo->get_pci_device(this->cuda_exec_info_.pci_bus_id);
     if (cuda_hwloc_obj) {
-        this->cuda_exec_info_.numa = cuda_hwloc_obj->numa;
-        this->cuda_exec_info_.closest_cpu_id = cuda_hwloc_obj->closest_cpu_id;
+        this->cuda_exec_info_.numa_node = cuda_hwloc_obj->numa;
+        this->cuda_exec_info_.closest_cpu_ids = cuda_hwloc_obj->closest_cpu_ids;
     }
 }
 
diff --git a/cuda/test/base/CMakeLists.txt b/cuda/test/base/CMakeLists.txt
index acbe3244fd4..b92138a52dc 100644
--- a/cuda/test/base/CMakeLists.txt
+++ b/cuda/test/base/CMakeLists.txt
@@ -1,5 +1,8 @@
 ginkgo_create_cuda_test(array)
 ginkgo_create_cuda_test(cuda_executor)
+if(GINKGO_HAVE_HWLOC)
+    ginkgo_create_cuda_test(cuda_executor_topology numa)
+endif()
 ginkgo_create_cuda_test(exception_helpers)
 ginkgo_create_cuda_test(lin_op)
 ginkgo_create_cuda_test(math)
diff --git a/cuda/test/base/cuda_executor.cu b/cuda/test/base/cuda_executor.cu
index 2c1cc364723..184fd032c89 100644
--- a/cuda/test/base/cuda_executor.cu
+++ b/cuda/test/base/cuda_executor.cu
@@ -38,11 +38,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <type_traits>
 
 
-#if defined(__unix__) || defined(__APPLE__)
-#include <utmpx.h>
-#endif
-
-
 #include <gtest/gtest.h>
 
 
@@ -137,34 +132,6 @@ TEST_F(CudaExecutor, MasterKnowsNumberOfDevices)
 }
 
 
-#if GKO_HAVE_HWLOC
-
-
-inline int get_os_id(int log_id)
-{
-    return gko::get_machine_topology()->get_core(log_id)->os_id;
-}
-
-
-TEST_F(CudaExecutor, CanBindToCores)
-{
-#if defined(_WIN32) || defined(__APPLE__) || defined(__CYGWIN__)
-    GTEST_SKIP() << " No useful machine topology information to test against";
-#endif
-    auto cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
-    auto cpu_sys = sched_getcpu();
-
-    const int bind_core[2] = {6, 3};
-    cuda->bind_to_cores(bind_core, 2);
-
-    cpu_sys = sched_getcpu();
-    ASSERT_TRUE(cpu_sys == get_os_id(3) || cpu_sys == get_os_id(6));
-}
-
-
-#endif
-
-
 TEST_F(CudaExecutor, AllocatesAndFreesMemory)
 {
     int *ptr = nullptr;
diff --git a/cuda/test/base/cuda_executor_topology.cu b/cuda/test/base/cuda_executor_topology.cu
new file mode 100644
index 00000000000..6ccc1a0e7a0
--- /dev/null
+++ b/cuda/test/base/cuda_executor_topology.cu
@@ -0,0 +1,155 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include <memory>
+#include <thread>
+#include <type_traits>
+
+
+#if defined(__unix__) || defined(__APPLE__)
+#include <numa.h>
+#include <utmpx.h>
+#endif
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
+#include "cuda/test/utils.hpp"
+
+
+namespace {
+
+
+class CudaExecutor : public ::testing::Test {
+protected:
+    CudaExecutor()
+        : omp(gko::OmpExecutor::create()), cuda(nullptr), cuda2(nullptr)
+    {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
+        cuda = gko::CudaExecutor::create(0, omp);
+        cuda2 = gko::CudaExecutor::create(
+            gko::CudaExecutor::get_num_devices() - 1, omp);
+    }
+
+    void TearDown()
+    {
+        if (cuda != nullptr) {
+            // ensure that previous calls finished and didn't throw an error
+            ASSERT_NO_THROW(cuda->synchronize());
+        }
+    }
+
+    std::shared_ptr<gko::Executor> omp;
+    std::shared_ptr<gko::CudaExecutor> cuda;
+    std::shared_ptr<gko::CudaExecutor> cuda2;
+};
+
+
+#if GKO_HAVE_HWLOC
+
+
+inline int get_cpu_os_id(int log_id)
+{
+    return gko::get_machine_topology()->get_pu(log_id)->os_id;
+}
+
+
+inline int get_core_os_id(int log_id)
+{
+    return gko::get_machine_topology()->get_core(log_id)->os_id;
+}
+
+
+TEST_F(CudaExecutor, CanBindToCpus)
+{
+    auto cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+    auto cpu_sys = sched_getcpu();
+
+    const int bind_cpu[2] = {6, 3};
+    cuda->bind_to_cpus(bind_cpu, 2);
+
+    cpu_sys = sched_getcpu();
+    ASSERT_TRUE(cpu_sys == get_cpu_os_id(3) || cpu_sys == get_cpu_os_id(6));
+}
+
+
+TEST_F(CudaExecutor, CanBindToCores)
+{
+    auto cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+    auto cpu_sys = sched_getcpu();
+
+    const int bind_cores[2] = {6, 3};
+    cuda->bind_to_cores(bind_cores, 2);
+
+    cpu_sys = sched_getcpu();
+    ASSERT_TRUE(cpu_sys == get_core_os_id(3) || cpu_sys == get_core_os_id(6));
+}
+
+
+TEST_F(CudaExecutor, KnowsItsClosestCpus)
+{
+    auto cuda0 = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+    auto cpu_sys = sched_getcpu();
+    auto close_cpus0 = cuda0->get_closest_cpus();
+
+    ASSERT_NE(close_cpus0[0], -1);
+}
+
+
+TEST_F(CudaExecutor, KnowsItsNuma)
+{
+    auto cuda0 = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+    auto exec_info0 = cuda0->get_exec_info();
+    auto numa0 = exec_info0.numa_node;
+    auto close_cpu0 = cuda0->get_closest_cpus()[0];
+
+    auto numa_sys0 = numa_node_of_cpu(get_cpu_os_id(close_cpu0));
+
+    ASSERT_TRUE(numa0 == numa_sys0);
+}
+
+
+#endif
+
+
+}  // namespace
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index e8bb153bc1d..1f2d0ad8f67 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -82,8 +82,8 @@ void HipExecutor::populate_exec_info(const MachineTopology *mach_topo)
     auto hip_hwloc_obj =
         mach_topo->get_pci_device(this->hip_exec_info_.pci_bus_id);
     if (hip_hwloc_obj) {
-        this->hip_exec_info_.numa = hip_hwloc_obj->numa;
-        this->hip_exec_info_.closest_cpu_id = hip_hwloc_obj->closest_cpu_id;
+        this->hip_exec_info_.numa_node = hip_hwloc_obj->numa;
+        this->hip_exec_info_.closest_cpu_ids = hip_hwloc_obj->closest_cpu_ids;
     }
 }
 
diff --git a/hip/test/base/CMakeLists.txt b/hip/test/base/CMakeLists.txt
index f7938076c9d..7938ae4870c 100644
--- a/hip/test/base/CMakeLists.txt
+++ b/hip/test/base/CMakeLists.txt
@@ -1,4 +1,7 @@
 ginkgo_create_hip_test(hip_executor)
+if(GINKGO_HAVE_HWLOC)
+    ginkgo_create_hip_test(hip_executor_topology numa)
+endif()
 ginkgo_create_hip_test(lin_op)
 ginkgo_create_hip_test(math)
 # Only hcc needs the libraries. nvcc only requires the headers.
diff --git a/hip/test/base/hip_executor.hip.cpp b/hip/test/base/hip_executor.hip.cpp
index c52a96c22e5..7e76f9778a7 100644
--- a/hip/test/base/hip_executor.hip.cpp
+++ b/hip/test/base/hip_executor.hip.cpp
@@ -153,9 +153,6 @@ inline int get_os_id(int log_id)
 
 TEST_F(HipExecutor, CanBindToCores)
 {
-#if defined(_WIN32) || defined(__APPLE__) || defined(__CYGWIN__)
-    GTEST_SKIP() << " No useful machine topology information to test against";
-#endif
     auto hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
     auto cpu_sys = sched_getcpu();
     const int bind_core[2] = {6, 3};
diff --git a/hip/test/base/hip_executor_topology.hip.cpp b/hip/test/base/hip_executor_topology.hip.cpp
new file mode 100644
index 00000000000..ca56c3494cf
--- /dev/null
+++ b/hip/test/base/hip_executor_topology.hip.cpp
@@ -0,0 +1,160 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+// force-top: on
+// prevent compilation failure related to disappearing assert(...) statements
+#include <hip/hip_runtime.h>
+// force-top: off
+
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include <memory>
+#include <thread>
+#include <type_traits>
+
+
+#if defined(__unix__) || defined(__APPLE__)
+#include <numa.h>
+#include <utmpx.h>
+#endif
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
+#include "hip/test/utils.hip.hpp"
+
+
+namespace {
+
+
+class HipExecutor : public ::testing::Test {
+protected:
+    HipExecutor() : omp(gko::OmpExecutor::create()), hip(nullptr), hip2(nullptr)
+    {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::HipExecutor::get_num_devices(), 0);
+        hip = gko::HipExecutor::create(0, omp);
+        hip2 = gko::HipExecutor::create(gko::HipExecutor::get_num_devices() - 1,
+                                        omp);
+    }
+
+    void TearDown()
+    {
+        if (hip != nullptr) {
+            // ensure that previous calls finished and didn't throw an error
+            ASSERT_NO_THROW(hip->synchronize());
+        }
+    }
+
+    std::shared_ptr<gko::Executor> omp;
+    std::shared_ptr<gko::HipExecutor> hip;
+    std::shared_ptr<gko::HipExecutor> hip2;
+};
+
+
+#if GKO_HAVE_HWLOC
+
+
+inline int get_cpu_os_id(int log_id)
+{
+    return gko::get_machine_topology()->get_pu(log_id)->os_id;
+}
+
+
+inline int get_core_os_id(int log_id)
+{
+    return gko::get_machine_topology()->get_core(log_id)->os_id;
+}
+
+
+TEST_F(HipExecutor, CanBindToCpus)
+{
+    auto hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+    auto cpu_sys = sched_getcpu();
+
+    const int bind_cpu[2] = {6, 3};
+    hip->bind_to_cpus(bind_cpu, 2);
+
+    cpu_sys = sched_getcpu();
+    ASSERT_TRUE(cpu_sys == get_cpu_os_id(3) || cpu_sys == get_cpu_os_id(6));
+}
+
+
+TEST_F(HipExecutor, CanBindToCores)
+{
+    auto hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+    auto cpu_sys = sched_getcpu();
+
+    const int bind_cores[2] = {6, 3};
+    hip->bind_to_cores(bind_cores, 2);
+
+    cpu_sys = sched_getcpu();
+    ASSERT_TRUE(cpu_sys == get_core_os_id(3) || cpu_sys == get_core_os_id(6));
+}
+
+
+TEST_F(HipExecutor, KnowsItsClosestCpus)
+{
+    auto hip0 = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+    auto cpu_sys = sched_getcpu();
+    auto close_cpus0 = hip0->get_closest_cpus();
+
+    ASSERT_NE(close_cpus0[0], -1);
+}
+
+
+TEST_F(HipExecutor, KnowsItsNuma)
+{
+    auto hip0 = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+    auto exec_info0 = hip0->get_exec_info();
+    auto numa0 = exec_info0.numa_node;
+    auto close_cpu0 = hip0->get_closest_cpus()[0];
+
+    auto numa_sys0 = numa_node_of_cpu(get_cpu_os_id(close_cpu0));
+
+    ASSERT_TRUE(numa0 == numa_sys0);
+}
+
+
+#endif
+
+
+}  // namespace
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index c2b65d7f51e..7747371486d 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -735,8 +735,14 @@ class Executor : public log::EnableLogging<Executor> {
 
     struct exec_info {
         int device_id = -1;
-        int num_cores = 0;
-        int num_work_groups_per_core = 0;
+        int numa_node = -1;
+        int num_cores = -1;
+        int num_work_groups_per_core = -1;
+        int major = -1;
+        int minor = -1;
+        int warp_size = -1;
+        std::string pci_bus_id = std::string(13, 'x');
+        std::vector<int> closest_cpu_ids{};
     };
 
 private:
@@ -1105,6 +1111,8 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     friend class detail::ExecutorBase<CudaExecutor>;
 
 public:
+    using cuda_exec_info = exec_info;
+
     /**
      * Creates a new CudaExecutor.
      *
@@ -1205,6 +1213,31 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         return cusparse_handle_.get();
     }
 
+    /**
+     * Get the closest CPUs
+     *
+     * @return  the array of CPUs closest to this device
+     */
+    std::vector<int> get_closest_cpus() const
+    {
+        return this->cuda_exec_info_.closest_cpu_ids;
+    }
+
+    /**
+     * Gets the exec info struct
+     *
+     * @return  the exec_info struct
+     */
+    cuda_exec_info get_exec_info() const { return this->cuda_exec_info_; }
+
+    /**
+     * Bind the current executor thread to the list of cpus
+     *
+     * @param  ids  the logical ids of the cpus.
+     * @param  num_ids  the number of cpus to bind
+     */
+    void bind_to_cpus(const int *ids, const size_type num_ids) const;
+
     /**
      * Bind the current executor thread to the list of cores
      *
@@ -1228,9 +1261,9 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         assert(this->cuda_exec_info_.device_id < max_devices &&
                this->cuda_exec_info_.device_id >= 0);
         this->CudaExecutor::populate_exec_info(get_machine_topology());
-        if (this->cuda_exec_info_.closest_cpu_id != -1) {
-            get_machine_topology()->bind_to_pus(
-                &this->cuda_exec_info_.closest_cpu_id, 1);
+        if (this->cuda_exec_info_.closest_cpu_ids.size()) {
+            this->bind_to_cpus(this->cuda_exec_info_.closest_cpu_ids.data(),
+                               this->cuda_exec_info_.closest_cpu_ids.size());
         }
         this->set_gpu_property();
         this->init_handles();
@@ -1275,15 +1308,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
 private:
     std::shared_ptr<Executor> master_;
-
-    struct cuda_exec_info : public exec_info {
-        int major;
-        int minor;
-        int warp_size;
-        std::string pci_bus_id = std::string(13, 'x');
-        int numa = -1;
-        int closest_cpu_id = -1;
-    } cuda_exec_info_;
+    cuda_exec_info cuda_exec_info_;
 
     template <typename T>
     using handle_manager = std::unique_ptr<T, std::function<void(T *)>>;
@@ -1315,6 +1340,8 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     friend class detail::ExecutorBase<HipExecutor>;
 
 public:
+    using hip_exec_info = exec_info;
+
     /**
      * Creates a new HipExecutor.
      *
@@ -1415,6 +1442,31 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         return hipsparse_handle_.get();
     }
 
+    /**
+     * Get the closest CPUs
+     *
+     * @return  the array of CPUs closest to this device
+     */
+    std::vector<int> get_closest_cpus() const
+    {
+        return this->hip_exec_info_.closest_cpu_ids;
+    }
+
+    /**
+     * Gets the exec info struct
+     *
+     * @return  the exec_info struct
+     */
+    hip_exec_info get_exec_info() const { return this->hip_exec_info_; }
+
+    /**
+     * Bind the current executor thread to the list of cpus
+     *
+     * @param  ids  the logical ids of the cpus.
+     * @param  num_ids  the number of cpus to bind
+     */
+    void bind_to_cpus(const int *ids, const size_type num_ids) const;
+
     /**
      * Bind the current executor thread to the list of cores
      *
@@ -1437,9 +1489,9 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         this->hip_exec_info_.num_work_groups_per_core = 0;
         assert(this->hip_exec_info_.device_id < max_devices);
         this->HipExecutor::populate_exec_info(get_machine_topology());
-        if (this->hip_exec_info_.closest_cpu_id != -1) {
-            get_machine_topology()->bind_to_pus(
-                &this->hip_exec_info_.closest_cpu_id, 1);
+        if (this->hip_exec_info_.closest_cpu_ids.size()) {
+            this->bind_to_cpus(this->hip_exec_info_.closest_cpu_ids.data(),
+                               this->hip_exec_info_.closest_cpu_ids.size());
         }
         this->set_gpu_property();
         this->init_handles();
@@ -1484,15 +1536,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
 private:
     std::shared_ptr<Executor> master_;
-
-    struct hip_exec_info : public exec_info {
-        int major = 0;
-        int minor = 0;
-        int warp_size = 0;
-        std::string pci_bus_id = std::string(13, 'x');
-        int numa = -1;
-        int closest_cpu_id = -1;
-    } hip_exec_info_;
+    hip_exec_info hip_exec_info_;
 
     template <typename T>
     using handle_manager = std::unique_ptr<T, std::function<void(T *)>>;
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index 86cf6bdfea7..b83bdca33f5 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -109,9 +109,6 @@ class MachineTopology {
     template <typename T>
     using hwloc_manager = std::unique_ptr<T, std::function<void(T *)>>;
 
-    using topo_bitmap =
-        std::unique_ptr<hwloc_bitmap_s, std::function<void(hwloc_bitmap_s *)>>;
-
 private:
     /**
      * This struct holds the attributes for a normal non-IO object.
@@ -214,19 +211,9 @@ class MachineTopology {
         std::string ancestor_type;
 
         /**
-         * The closest CPU.
-         */
-        int closest_cpu_id;
-
-        /**
-         * The io children objects (usually software OS devices).
-         */
-        std::vector<hwloc_obj_t> io_children;
-
-        /**
-         * The names of the io children objects.
+         * The array of CPU ids closest to the object.
          */
-        std::vector<std::string> io_children_name;
+        std::vector<int> closest_cpu_ids;
 
         /**
          * The PCI Bus ID
@@ -245,18 +232,14 @@ class MachineTopology {
     MachineTopology &operator=(MachineTopology &&) = delete;
     ~MachineTopology() = default;
 
-    enum class BitMapType { bitmap_set, bitmap_or, bitmap_and, bitmap_andnot };
-
     /**
      * Bind the object associated with the id to a core.
      *
      * @param id  The id of the object to be bound.
      */
-    void bind_to_cores(
-        const int *ids, const size_type num_ids,
-        const BitMapType bitmap_type = BitMapType::bitmap_set) const
+    void bind_to_cores(const int *ids, const size_type num_ids) const
     {
-        hwloc_binding_helper(this->cores_, ids, num_ids, bitmap_type);
+        hwloc_binding_helper(this->cores_, ids, num_ids);
     }
 
     /**
@@ -264,11 +247,9 @@ class MachineTopology {
      *
      * @param id  The id of the object to be bound.
      */
-    void bind_to_pus(
-        const int *ids, const size_type num_ids,
-        const BitMapType bitmap_type = BitMapType::bitmap_set) const
+    void bind_to_pus(const int *ids, const size_type num_ids) const
     {
-        hwloc_binding_helper(this->pus_, ids, num_ids, bitmap_type);
+        hwloc_binding_helper(this->pus_, ids, num_ids);
     }
 
     /**
@@ -279,8 +260,8 @@ class MachineTopology {
      */
     const normal_obj_info *get_pu(size_type id) const
     {
-        GKO_ENSURE_IN_BOUNDS(id, std::get<0>(this->pus_).size());
-        return &std::get<0>(this->pus_)[id];
+        GKO_ENSURE_IN_BOUNDS(id, this->pus_.size());
+        return &this->pus_[id];
     }
 
     /**
@@ -291,8 +272,8 @@ class MachineTopology {
      */
     const normal_obj_info *get_core(size_type id) const
     {
-        GKO_ENSURE_IN_BOUNDS(id, std::get<0>(this->cores_).size());
-        return &std::get<0>(this->cores_)[id];
+        GKO_ENSURE_IN_BOUNDS(id, this->cores_.size());
+        return &this->cores_[id];
     }
 
     /**
@@ -320,14 +301,14 @@ class MachineTopology {
      *
      * @return  the number of PUs.
      */
-    size_type get_num_pus() const { return std::get<0>(this->pus_).size(); }
+    size_type get_num_pus() const { return this->pus_.size(); }
 
     /**
      * Get the number of core objects stored in this Topology tree.
      *
      * @return  the number of cores.
      */
-    size_type get_num_cores() const { return std::get<0>(this->cores_).size(); }
+    size_type get_num_cores() const { return this->cores_.size(); }
 
     /**
      * Get the number of PCI device objects stored in this Topology tree.
@@ -363,19 +344,8 @@ class MachineTopology {
      * A helper function that binds the object with an id.
      */
     void hwloc_binding_helper(
-        const std::tuple<std::vector<MachineTopology::normal_obj_info>,
-                         topo_bitmap> &obj,
-        const int *id, const size_type num_ids,
-        const BitMapType &bitmap_type) const;
-
-    /**
-     * @internal
-     *
-     * A helper function that prints the topology tree of an object to a given
-     * depth. Provided from the hwloc library.
-     */
-    void hwloc_print_children(std::ostream &os, const hwloc_obj_t obj,
-                              const int depth = 1);
+        const std::vector<MachineTopology::normal_obj_info> &obj, const int *id,
+        const size_type num_ids) const;
 
     /**
      * @internal
@@ -385,9 +355,8 @@ class MachineTopology {
      * @note The objects should be sorted by logical index since hwloc uses
      * logical index with these functions
      */
-    void load_objects(
-        hwloc_obj_type_t type,
-        std::tuple<std::vector<normal_obj_info>, topo_bitmap> &objects);
+    void load_objects(hwloc_obj_type_t type,
+                      std::vector<normal_obj_info> &objects) const;
 
     /**
      * @internal
@@ -397,7 +366,8 @@ class MachineTopology {
      * @note The objects should be sorted by logical index since hwloc uses
      * logical index with these functions
      */
-    void load_objects(hwloc_obj_type_t type, std::vector<io_obj_info> &vector);
+    void load_objects(hwloc_obj_type_t type,
+                      std::vector<io_obj_info> &vector) const;
 
     /**
      * @internal
@@ -419,8 +389,8 @@ class MachineTopology {
      *
      * Get object id from the os index
      */
-    inline int get_obj_id_by_os_index(std::vector<normal_obj_info> &objects,
-                                      size_type os_index) const;
+    inline int get_obj_id_by_os_index(
+        const std::vector<normal_obj_info> &objects, size_type os_index) const;
 
     /**
      *
@@ -428,15 +398,15 @@ class MachineTopology {
      *
      * Get object id from the hwloc index
      */
-    inline int get_obj_id_by_gp_index(std::vector<normal_obj_info> &objects,
-                                      size_type gp_index) const;
+    inline int get_obj_id_by_gp_index(
+        const std::vector<normal_obj_info> &objects, size_type gp_index) const;
 
 
 private:
-    std::tuple<std::vector<normal_obj_info>, topo_bitmap> pus_;
-    std::tuple<std::vector<normal_obj_info>, topo_bitmap> cores_;
-    std::tuple<std::vector<normal_obj_info>, topo_bitmap> packages_;
-    std::tuple<std::vector<normal_obj_info>, topo_bitmap> numa_nodes_;
+    std::vector<normal_obj_info> pus_;
+    std::vector<normal_obj_info> cores_;
+    std::vector<normal_obj_info> packages_;
+    std::vector<normal_obj_info> numa_nodes_;
     std::vector<io_obj_info> pci_devices_;
     size_type num_numas_;
 

From d640e8527493b42916a40f9396b74d34f12b1f49 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 21 Jan 2021 05:13:39 +0100
Subject: [PATCH 0301/1008] Review update.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

+ Make MachineTopology a singleton.
+ Add Morse License
+ Improve naming.
+ Update binding interface.
+ Unify exec_info struct.
+ Remove plain pointers and use RAII classes instead.

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 .gitlab-ci.yml                                |  12 +-
 ABOUT-LICENSING.md                            |  44 +++
 CMakeLists.txt                                |   6 +-
 cmake/GinkgoConfig.cmake.in                   |   7 +-
 cmake/Modules/FindHWLOC.cmake                 |   2 +-
 cmake/Modules/hwloc_helpers.cmake             |  25 +-
 cmake/get_info.cmake                          |   4 +-
 cmake/package_helpers.cmake                   |   2 +
 core/device_hooks/dpcpp_hooks.cpp             |   8 +-
 core/devices/cuda/executor.cpp                |  18 +-
 core/devices/hip/executor.cpp                 |  18 +-
 core/devices/machine_topology.cpp             | 107 +++--
 core/devices/omp/executor.cpp                 |   4 +-
 core/test/base/executor.cpp                   |  62 +--
 cuda/base/executor.cpp                        |  54 +--
 cuda/test/base/cuda_executor.cu               |   1 -
 cuda/test/base/cuda_executor_topology.cu      |  48 ++-
 cuda/test/utils.hpp                           |   3 -
 dpcpp/base/executor.dp.cpp                    |  21 +-
 dpcpp/test/base/CMakeLists.txt                |   1 +
 dpcpp/test/base/executor_topology.dp.cpp      |  93 +++++
 hip/base/executor.hip.cpp                     |  42 +-
 hip/test/base/hip_executor.hip.cpp            |  24 --
 hip/test/base/hip_executor_topology.hip.cpp   |  50 ++-
 include/ginkgo/core/base/executor.hpp         | 368 +++++++++++-------
 include/ginkgo/core/base/machine_topology.hpp |  69 ++--
 test_install/CMakeLists.txt                   |   8 -
 27 files changed, 634 insertions(+), 467 deletions(-)
 create mode 100644 dpcpp/test/base/executor_topology.dp.cpp

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 44caedc28db..544f45e3daa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,7 +22,7 @@ stages:
   BUILD_OMP: "OFF"
   BUILD_CUDA: "OFF"
   BUILD_HIP: "OFF"
-  FORCE_NO_HWLOC: "OFF"
+  DISABLE_HWLOC: "OFF"
   RUN_EXAMPLES: "OFF"
   CONFIG_LOG: "ON"
   CXX_FLAGS: ""
@@ -64,7 +64,7 @@ stages:
         -DGINKGO_DEVEL_TOOLS=OFF -DGINKGO_BUILD_REFERENCE=${BUILD_REFERENCE}
         -DGINKGO_BUILD_OMP=${BUILD_OMP} -DGINKGO_BUILD_CUDA=${BUILD_CUDA}
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
-        -DGINKGO_FORCE_NO_HWLOC=${FORCE_NO_HWLOC}
+        -DGINKGO_DISABLE_HWLOC=${DISABLE_HWLOC}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
@@ -96,7 +96,7 @@ stages:
         -DGINKGO_DEVEL_TOOLS=OFF -DGINKGO_BUILD_REFERENCE=${BUILD_REFERENCE}
         -DGINKGO_BUILD_OMP=${BUILD_OMP} -DGINKGO_BUILD_CUDA=${BUILD_CUDA}
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
-        -DGINKGO_FORCE_NO_HWLOC=${FORCE_NO_HWLOC}
+        -DGINKGO_DISABLE_HWLOC=${DISABLE_HWLOC}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
@@ -266,7 +266,7 @@ build/cuda92/intel/cuda/release/static:
     CXX_COMPILER: "icpc"
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
-    FORCE_NO_HWLOC: "ON"
+    DISABLE_HWLOC: "ON"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
@@ -432,7 +432,7 @@ build/cuda102/gcc/all/debug/shared:
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     BUILD_TYPE: "Debug"
-    FORCE_NO_HWLOC: "ON"
+    DISABLE_HWLOC: "ON"
     CUDA_ARCH: 35
   only:
     variables:
@@ -587,7 +587,7 @@ build/nocuda/gcc/core/debug/static:
     BUILD_REFERENCE: "OFF"
     BUILD_TYPE: "Debug"
     BUILD_SHARED_LIBS: "OFF"
-    FORCE_NO_HWLOC: "ON"
+    DISABLE_HWLOC: "ON"
   only:
     variables:
       - $RUN_CI_TAG
diff --git a/ABOUT-LICENSING.md b/ABOUT-LICENSING.md
index 3adba19ba6e..e9215ee35cf 100644
--- a/ABOUT-LICENSING.md
+++ b/ABOUT-LICENSING.md
@@ -240,6 +240,50 @@ also licensed the same as the deal.II library.
 > page](https://github.com/dealii/dealii/blob/master/LICENSE.md) or on the
 > official [GNU license page](https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html).
 
+
+For detecting the HWLOC library, we used a modified version of the FindHWLOC.cmake file from the MORSE-cmake library. The library is [available on gitlab](https://gitlab.inria.fr/solverstack/morse_cmake), and its LICENSE is available below:
+
+> ###
+> #
+> # @copyright (c) 2012-2020 Inria. All rights reserved.
+> # @copyright (c) 2012-2020 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+> #
+> ###
+> #
+> # This software is a computer program whose purpose is to process
+> # Matrices Over Runtime Systems @ Exascale (MORSE). More information
+> # can be found on the following website: http://www.inria.fr/en/teams/morse.
+> #
+> # This software is governed by the CeCILL-C license under French law and
+> # abiding by the rules of distribution of free software.  You can  use,
+> # modify and/ or redistribute the software under the terms of the CeCILL-C
+> # license as circulated by CEA, CNRS and INRIA at the following URL
+> # "http://www.cecill.info".
+> #
+> # As a counterpart to the access to the source code and  rights to copy,
+> # modify and redistribute granted by the license, users are provided only
+> # with a limited warranty  and the software's author,  the holder of the
+> # economic rights,  and the successive licensors  have only  limited
+> # liability.
+> #
+> # In this respect, the user's attention is drawn to the risks associated
+> # with loading,  using,  modifying and/or developing or reproducing the
+> # software by the user in light of its specific status of free software,
+> # that may mean  that it is complicated to manipulate,  and  that  also
+> # therefore means  that it is reserved for developers  and  experienced
+> # professionals having in-depth computer knowledge. Users are therefore
+> # encouraged to load and test the software's suitability as regards their
+> # requirements in conditions enabling the security of their systems and/or
+> # data to be ensured and,  more generally, to use and operate it in the
+> # same conditions as regards security.
+> #
+> # The fact that you are presently reading this means that you have had
+> # knowledge of the CeCILL-C license and that you accept its terms.
+> #
+> ###
+
+
+
 __NOTE:__ Some of the options that pull additional software when compiling
 Ginkgo are ON by default, and have to be disabled manually to prevent
 third-party licensing. Refer to the [Installation section in
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c8896d6e5f3..7d510bd63ce 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -70,7 +70,7 @@ set(GINKGO_HIP_AMDGPU "" CACHE STRING
     "The amdgpu_target(s) variable passed to hipcc. The default is none (auto).")
 option(GINKGO_JACOBI_FULL_OPTIMIZATIONS "Use all the optimizations for the CUDA Jacobi algorithm" OFF)
 option(BUILD_SHARED_LIBS "Build shared (.so, .dylib, .dll) libraries" ON)
-option(GINKGO_FORCE_NO_HWLOC "Forcibly switch off HWLOC dependency" OFF)
+option(GINKGO_DISABLE_HWLOC "Disable HWLOC " OFF)
 
 set(GINKGO_CIRCULAR_DEPS_FLAGS "-Wl,--no-undefined")
 
@@ -144,7 +144,7 @@ if(PAPI_sde_FOUND)
     set(GINKGO_HAVE_PAPI_SDE 1)
 endif()
 
-if(GINKGO_FORCE_NO_HWLOC)
+if(GINKGO_DISABLE_HWLOC)
     set(GINKGO_USE_EXTERNAL_HWLOC 0)
     set(GINKGO_HAVE_HWLOC 0)
     message(STATUS "HWLOC is being forcibly switched off")
@@ -209,7 +209,7 @@ include(cmake/package_helpers.cmake)
 ginkgo_find_package(GTest "GTest::GTest;GTest::Main" FALSE 1.8.1)
 ginkgo_find_package(gflags gflags FALSE 2.2.2)
 ginkgo_find_package(RapidJSON rapidjson TRUE 1.1.0)
-if(NOT GINKGO_FORCE_NO_HWLOC AND GINKGO_HAVE_HWLOC)
+if(NOT GINKGO_DISABLE_HWLOC AND GINKGO_HAVE_HWLOC)
     ginkgo_find_package(HWLOC hwloc FALSE 2.1)
 endif()
 set(GKO_HWLOC_XMLFILE "$ENV{HWLOC_XMLFILE}")
diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 8fbd650d3bd..0cd56ccdccc 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -78,13 +78,14 @@ set(GINKGO_DPCPP_FLAGS @GINKGO_DPCPP_FLAGS@)
 set(GINKGO_HAVE_PAPI_SDE @GINKGO_HAVE_PAPI_SDE@)
 
 set(GINKGO_HAVE_HWLOC @GINKGO_HAVE_HWLOC@)
-set(GINKGO_FORCE_NO_HWLOC @GINKGO_FORCE_NO_HWLOC@)
+set(GINKGO_DISABLE_HWLOC @GINKGO_DISABLE_HWLOC@)
 
 # Ginkgo external package variables
 set(GINKGO_USE_EXTERNAL_CAS "@GINKGO_USE_EXTERNAL_CAS@")
 set(GINKGO_USE_EXTERNAL_GTEST "@GINKGO_USE_EXTERNAL_GTEST@")
 set(GINKGO_USE_EXTERNAL_GFLAGS "@GINKGO_USE_EXTERNAL_GFLAGS@")
 set(GINKGO_USE_EXTERNAL_RAPIDJSON "@GINKGO_USE_EXTERNAL_RAPIDJSON@")
+set(GINKGO_USE_EXTERNAL_HWLOC "@GINKGO_USE_EXTERNAL_HWLOC@")
 
 set(TPL_ENABLE_GTEST "@TPL_ENABLE_GTEST@")
 set(TPL_GTEST_LIBRARIES "@TPL_GTEST_LIBRARIES@")
@@ -153,6 +154,10 @@ if(GINKGO_HAVE_PAPI_SDE)
     find_package(PAPI REQUIRED OPTIONAL_COMPONENTS sde)
 endif()
 
+if(GINKGO_HAVE_HWLOC)
+    find_package(HWLOC REQUIRED)
+endif()
+
 # HIP depends on Threads::Threads in some circumstances, but doesn't find it
 if (GINKGO_BUILD_HIP)
     find_package(Threads REQUIRED)
diff --git a/cmake/Modules/FindHWLOC.cmake b/cmake/Modules/FindHWLOC.cmake
index 96ea17639e4..6d3686df07a 100644
--- a/cmake/Modules/FindHWLOC.cmake
+++ b/cmake/Modules/FindHWLOC.cmake
@@ -19,7 +19,7 @@
 # (To distribute this file outside of Morse, substitute the full
 #  License text for the above reference.)
 #
-# Modified for Ginkgo.
+# Modified for Ginkgo (See ABOUT-LICENSING.md for additional details).
 #
 ###
 #
diff --git a/cmake/Modules/hwloc_helpers.cmake b/cmake/Modules/hwloc_helpers.cmake
index 41622611e8f..689179142ab 100644
--- a/cmake/Modules/hwloc_helpers.cmake
+++ b/cmake/Modules/hwloc_helpers.cmake
@@ -1,4 +1,27 @@
-# Modified function from Morse
+###
+#
+# @copyright (c) 2012-2020 Inria. All rights reserved.
+# @copyright (c) 2012-2014 Bordeaux INP, CNRS (LaBRI UMR 5800), Inria, Univ. Bordeaux. All rights reserved.
+#
+# Copyright 2012-2013 Emmanuel Agullo
+# Copyright 2012-2013 Mathieu Faverge
+# Copyright 2012      Cedric Castagnede
+# Copyright 2013-2020 Florent Pruvost
+# Copyright 2020-2021 Ginkgo Project
+#
+# Distributed under the OSI-approved BSD License (the "License");
+# see accompanying file MORSE-Copyright.txt for details.
+#
+# This software is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the License for more information.
+#=============================================================================
+# (To distribute this file outside of Morse, substitute the full
+#  License text for the above reference.)
+#
+# Modified for Ginkgo (See ABOUT-LICENSING.md for additional details).
+#
+
 macro(ginkgo_set_required_test_lib_link name)
     set(CMAKE_REQUIRED_INCLUDES "${${name}${STATIC}_INCLUDE_DIRS}")
     if (${name}${STATIC}_CFLAGS_OTHER)
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index b111a2e1d45..5621c74f2b8 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -175,8 +175,8 @@ if(PAPI_sde_FOUND)
     ginkgo_print_variable(${detailed_log} "PAPI_INCLUDE_DIR")
     ginkgo_print_flags(${detailed_log} "PAPI_LIBRARY")
 endif()
-ginkgo_print_variable(${minimal_log} "GINKGO_FORCE_NO_HWLOC")
-ginkgo_print_variable(${detailed_log} "GINKGO_FORCE_NO_HWLOC")
+ginkgo_print_variable(${minimal_log} "GINKGO_DISABLE_HWLOC")
+ginkgo_print_variable(${detailed_log} "GINKGO_DISABLE_HWLOC")
 ginkgo_print_variable(${detailed_log} "HWLOC_VERSION")
 ginkgo_print_variable(${detailed_log} "GKO_HWLOC_XMLFILE")
 ginkgo_print_variable(${detailed_log} "HWLOC_LIBRARIES")
diff --git a/cmake/package_helpers.cmake b/cmake/package_helpers.cmake
index 080fe8791af..78385f1d1b7 100644
--- a/cmake/package_helpers.cmake
+++ b/cmake/package_helpers.cmake
@@ -73,6 +73,7 @@ function(ginkgo_load_and_configure_package package_name package_url package_hash
     if(result)
         message(FATAL_ERROR
             "CMake step for ${package_name}/download failed: ${result}")
+        return()
     endif()
     execute_process(COMMAND ${CMAKE_COMMAND} --build .
         RESULT_VARIABLE result
@@ -80,6 +81,7 @@ function(ginkgo_load_and_configure_package package_name package_url package_hash
     if(result)
         message(FATAL_ERROR
             "Build step for ${package_name}/download failed: ${result}")
+        return()
     endif()
 endfunction()
 
diff --git a/core/device_hooks/dpcpp_hooks.cpp b/core/device_hooks/dpcpp_hooks.cpp
index 30eff6fe21b..2fdaa5de3bc 100644
--- a/core/device_hooks/dpcpp_hooks.cpp
+++ b/core/device_hooks/dpcpp_hooks.cpp
@@ -129,15 +129,15 @@ void DpcppExecutor::set_device_property() {}
 bool DpcppExecutor::verify_memory_to(const OmpExecutor *dest_exec) const
 {
     // Dummy check
-    return this->dpcpp_exec_info_.device_type == "cpu" ||
-           this->dpcpp_exec_info_.device_type == "host";
+    return this->get_exec_info().device_type == "cpu" ||
+           this->get_exec_info().device_type == "host";
 }
 
 bool DpcppExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
 {
     // Dummy check
-    return dest_exec->get_device_type() == this->dpcpp_exec_info_.device_type &&
-           dest_exec->get_device_id() == this->dpcpp_exec_info_.device_id;
+    return dest_exec->get_device_type() == this->get_exec_info().device_type &&
+           dest_exec->get_device_id() == this->get_exec_info().device_id;
 }
 
 
diff --git a/core/devices/cuda/executor.cpp b/core/devices/cuda/executor.cpp
index 1bf34b5ffc8..7c5bb7f8dd4 100644
--- a/core/devices/cuda/executor.cpp
+++ b/core/devices/cuda/executor.cpp
@@ -50,34 +50,20 @@ std::shared_ptr<const Executor> CudaExecutor::get_master() const noexcept
 
 bool CudaExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
 {
-    return this->cuda_exec_info_.device_id == dest_exec->get_device_id();
+    return this->get_exec_info().device_id == dest_exec->get_device_id();
 }
 
 
 bool CudaExecutor::verify_memory_to(const HipExecutor *dest_exec) const
 {
 #if GINKGO_HIP_PLATFORM_NVCC
-    return this->cuda_exec_info_.device_id == dest_exec->get_device_id();
+    return this->get_exec_info().device_id == dest_exec->get_device_id();
 #else
     return false;
 #endif
 }
 
 
-void CudaExecutor::bind_to_cpus(const int *cpu_ids,
-                                const size_type num_ids) const
-{
-    get_machine_topology()->bind_to_pus(cpu_ids, num_ids);
-}
-
-
-void CudaExecutor::bind_to_cores(const int *core_ids,
-                                 const size_type num_ids) const
-{
-    get_machine_topology()->bind_to_cores(core_ids, num_ids);
-}
-
-
 unsigned CudaExecutor::num_execs[max_devices];
 
 
diff --git a/core/devices/hip/executor.cpp b/core/devices/hip/executor.cpp
index 9aac5b54c30..5020228b133 100644
--- a/core/devices/hip/executor.cpp
+++ b/core/devices/hip/executor.cpp
@@ -47,34 +47,20 @@ std::shared_ptr<const Executor> HipExecutor::get_master() const noexcept
 
 bool HipExecutor::verify_memory_to(const HipExecutor *dest_exec) const
 {
-    return this->hip_exec_info_.device_id == dest_exec->get_device_id();
+    return this->get_exec_info().device_id == dest_exec->get_device_id();
 }
 
 
 bool HipExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
 {
 #if GINKGO_HIP_PLATFORM_NVCC
-    return this->hip_exec_info_.device_id == dest_exec->get_device_id();
+    return this->get_exec_info().device_id == dest_exec->get_device_id();
 #else
     return false;
 #endif
 }
 
 
-void HipExecutor::bind_to_cpus(const int *cpu_ids,
-                               const size_type num_ids) const
-{
-    get_machine_topology()->bind_to_pus(cpu_ids, num_ids);
-}
-
-
-void HipExecutor::bind_to_cores(const int *core_ids,
-                                const size_type num_ids) const
-{
-    get_machine_topology()->bind_to_cores(core_ids, num_ids);
-}
-
-
 int HipExecutor::num_execs[max_devices];
 
 
diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index b42517cbb43..7f24593e496 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -44,28 +44,47 @@ namespace gko {
 namespace detail {
 
 
-std::shared_ptr<const MachineTopology> machine_topology{};
-std::mutex machine_topology_mutex{};
-std::atomic<bool> initialized_machine_topology{};
-
+class topo_bitmap {
+public:
+#if GKO_HAVE_HWLOC
+    using bitmap_type =
+        std::remove_pointer<decltype(hwloc_bitmap_alloc())>::type;
+    topo_bitmap() : bitmap(hwloc_bitmap_alloc()) {}
+    ~topo_bitmap() { hwloc_bitmap_free(bitmap); }
+#else
+    using bitmap_type = void;
+#endif
+    bitmap_type *get() { return bitmap; }
 
-}  // namespace detail
+private:
+    bitmap_type *bitmap;
+};
 
 
-const MachineTopology *get_machine_topology()
+hwloc_topology *init_topology()
 {
-    if (!detail::initialized_machine_topology.load()) {
-        std::lock_guard<std::mutex> guard(detail::machine_topology_mutex);
-        if (!detail::machine_topology) {
-            detail::machine_topology = MachineTopology::create();
-            detail::initialized_machine_topology.store(true);
-        }
-    }
-    assert(detail::machine_topology.get() != nullptr);
-    return detail::machine_topology.get();
+#if GKO_HAVE_HWLOC
+    hwloc_topology_t tmp;
+    hwloc_topology_init(&tmp);
+
+    hwloc_topology_set_io_types_filter(tmp, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
+    hwloc_topology_set_type_filter(tmp, HWLOC_OBJ_BRIDGE,
+                                   HWLOC_TYPE_FILTER_KEEP_NONE);
+    hwloc_topology_set_type_filter(tmp, HWLOC_OBJ_OS_DEVICE,
+                                   HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
+    hwloc_topology_set_xml(tmp, GKO_HWLOC_XMLFILE);
+    hwloc_topology_load(tmp);
+
+    return tmp;
+#else
+    return nullptr;
+#endif
 }
 
 
+}  // namespace detail
+
+
 const MachineTopology::io_obj_info *MachineTopology::get_pci_device(
     const std::string &pci_bus_id) const
 {
@@ -84,8 +103,8 @@ MachineTopology::MachineTopology()
 #if GKO_HAVE_HWLOC
 
     // Initialize the topology from hwloc
-    this->topo_ =
-        hwloc_manager<hwloc_topology>(init_topology(), hwloc_topology_destroy);
+    this->topo_ = hwloc_manager<hwloc_topology>(detail::init_topology(),
+                                                hwloc_topology_destroy);
     // load objects of type Package . See HWLOC_OBJ_PACKAGE for more details.
     load_objects(HWLOC_OBJ_PACKAGE, this->packages_);
     // load objects of type NUMA Node. See HWLOC_OBJ_NUMANODE for more details.
@@ -109,22 +128,23 @@ MachineTopology::MachineTopology()
 
 
 void MachineTopology::hwloc_binding_helper(
-    const std::vector<MachineTopology::normal_obj_info> &obj, const int *id,
-    const size_type num_ids) const
+    const std::vector<MachineTopology::normal_obj_info> &obj,
+    const std::vector<int> &bind_ids) const
 {
 #if GKO_HAVE_HWLOC
-    auto bitmap_toset = hwloc_bitmap_alloc();
+    detail::topo_bitmap bitmap_toset;
+    auto num_ids = bind_ids.size();
+    auto id = bind_ids.data();
     // Set the given ids to a bitmap
     for (auto i = 0; i < num_ids; ++i) {
         GKO_ASSERT(id[i] < obj.size());
         GKO_ASSERT(id[i] >= 0);
-        hwloc_bitmap_set(bitmap_toset, obj[id[i]].os_id);
+        hwloc_bitmap_set(bitmap_toset.get(), obj[id[i]].os_id);
     }
 
     // Singlify to reduce expensive migrations.
-    hwloc_bitmap_singlify(bitmap_toset);
-    hwloc_set_cpubind(this->topo_.get(), bitmap_toset, 0);
-    hwloc_bitmap_free(bitmap_toset);
+    hwloc_bitmap_singlify(bitmap_toset.get());
+    hwloc_set_cpubind(this->topo_.get(), bitmap_toset.get(), 0);
 #endif
 }
 
@@ -137,6 +157,7 @@ void MachineTopology::load_objects(
     // Get the number of normal objects of a certain type (Core, PU, Machine
     // etc.).
     unsigned num_objects = hwloc_get_nbobjs_by_type(this->topo_.get(), type);
+    objects.reserve(num_objects);
     for (unsigned i = 0; i < num_objects; i++) {
         // Get the actual normal object of the given type.
         hwloc_obj_t obj = hwloc_get_obj_by_type(this->topo_.get(), type, i);
@@ -186,6 +207,7 @@ void MachineTopology::load_objects(
     GKO_ASSERT(this->cores_.size() != 0);
     GKO_ASSERT(this->pus_.size() != 0);
     unsigned num_objects = hwloc_get_nbobjs_by_type(this->topo_.get(), type);
+    vector.reserve(num_objects);
     for (unsigned i = 0; i < num_objects; i++) {
         // Get the actual PCI object.
         hwloc_obj_t obj = hwloc_get_obj_by_type(this->topo_.get(), type, i);
@@ -197,17 +219,17 @@ void MachineTopology::load_objects(
             io_obj_info{obj, obj->logical_index, obj->os_index, obj->gp_index,
                         hwloc_bitmap_first(ancestor->nodeset), ancestor});
         // Get the corresponding cpuset of the ancestor nodeset
-        hwloc_cpuset_t ancestor_cpuset = hwloc_bitmap_alloc();
-        hwloc_cpuset_from_nodeset(this->topo_.get(), ancestor_cpuset,
+        detail::topo_bitmap ancestor_cpuset;
+        hwloc_cpuset_from_nodeset(this->topo_.get(), ancestor_cpuset.get(),
                                   ancestor->nodeset);
         // Find the cpu objects closest to this device from the ancestor cpuset
         // and store their ids for binding purposes
-        int closest_cpu_id = -1;
-        int closest_os_id = hwloc_bitmap_first(ancestor_cpuset);
+        int closest_pu_id = -1;
+        int closest_os_id = hwloc_bitmap_first(ancestor_cpuset.get());
         // clang-format off
-        hwloc_bitmap_foreach_begin(closest_os_id, ancestor_cpuset)
-            closest_cpu_id = get_obj_id_by_os_index(this->pus_, closest_os_id);
-            vector.back().closest_cpu_ids.push_back(closest_cpu_id);
+        hwloc_bitmap_foreach_begin(closest_os_id, ancestor_cpuset.get())
+            closest_pu_id = get_obj_id_by_os_index(this->pus_, closest_os_id);
+            vector.back().closest_pu_ids.push_back(closest_pu_id);
         hwloc_bitmap_foreach_end();
         // clang-format on
 
@@ -223,7 +245,6 @@ void MachineTopology::load_objects(
             vector.back().ancestor_local_id =
                 get_obj_id_by_gp_index(this->numa_nodes_, ancestor->gp_index);
         }
-        hwloc_bitmap_free(ancestor_cpuset);
         // Get type of the ancestor object and store it as a string.
         char ances_type[24];
         hwloc_obj_type_snprintf(ances_type, sizeof(ances_type), ancestor, 0);
@@ -239,26 +260,4 @@ void MachineTopology::load_objects(
 }
 
 
-hwloc_topology *MachineTopology::init_topology()
-{
-#if GKO_HAVE_HWLOC
-    hwloc_topology_t tmp;
-    hwloc_topology_init(&tmp);
-
-    hwloc_topology_set_io_types_filter(tmp, HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
-    hwloc_topology_set_type_filter(tmp, HWLOC_OBJ_BRIDGE,
-                                   HWLOC_TYPE_FILTER_KEEP_NONE);
-    hwloc_topology_set_type_filter(tmp, HWLOC_OBJ_OS_DEVICE,
-                                   HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
-    hwloc_topology_set_xml(tmp, GKO_HWLOC_XMLFILE);
-    hwloc_topology_load(tmp);
-
-    return tmp;
-#else
-    // MSVC complains if there is no return statement.
-    return nullptr;
-#endif
-}
-
-
 }  // namespace gko
diff --git a/core/devices/omp/executor.cpp b/core/devices/omp/executor.cpp
index 6b2d80f9f72..61b8dd02dca 100644
--- a/core/devices/omp/executor.cpp
+++ b/core/devices/omp/executor.cpp
@@ -50,8 +50,8 @@ void OmpExecutor::populate_exec_info(const MachineTopology *mach_topo)
         (mach_topo->get_num_cores() == 0 ? 1 : mach_topo->get_num_cores());
     auto num_pus =
         (mach_topo->get_num_pus() == 0 ? 1 : mach_topo->get_num_pus());
-    this->omp_exec_info_.num_cores = num_cores;
-    this->omp_exec_info_.num_work_groups_per_core = num_pus / num_cores;
+    this->get_exec_info().num_computing_units = num_cores;
+    this->get_exec_info().num_pe_per_cu = num_pus / num_cores;
 }
 
 
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 69a44ce7509..8c9a377c2f9 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -177,7 +177,7 @@ TEST(OmpExecutor, CanGetNumCpusFromExecInfo)
 
 inline int get_os_id(int log_id)
 {
-    return gko::get_machine_topology()->get_core(log_id)->os_id;
+    return gko::MachineTopology::get_instance()->get_core(log_id)->os_id;
 }
 
 
@@ -187,7 +187,8 @@ TEST(OmpExecutor, CanBindToASpecificCore)
     auto cpu_sys = sched_getcpu();
 
     const int bind_core = 4;
-    gko::get_machine_topology()->bind_to_cores(&bind_core, 1);
+    gko::MachineTopology::get_instance()->bind_to_cores(
+        std::vector<int>{bind_core});
 
     cpu_sys = sched_getcpu();
     ASSERT_EQ(cpu_sys, get_os_id(bind_core));
@@ -199,8 +200,8 @@ TEST(OmpExecutor, CanBindToARangeofCores)
     auto omp = gko::OmpExecutor::create();
     auto cpu_sys = sched_getcpu();
 
-    const int bind_core[2] = {6, 3};
-    gko::get_machine_topology()->bind_to_cores(bind_core, 2);
+    const std::vector<int> bind_core = {6, 3};
+    gko::MachineTopology::get_instance()->bind_to_cores(bind_core);
 
     cpu_sys = sched_getcpu();
     ASSERT_TRUE(cpu_sys == get_os_id(3) || cpu_sys == get_os_id(6));
@@ -537,6 +538,8 @@ TEST(Executor, CanVerifyMemory)
     auto omp2 = gko::OmpExecutor::create();
     auto hip2 = gko::HipExecutor::create(0, omp);
     auto cuda2 = gko::CudaExecutor::create(0, omp);
+    auto hip_1 = gko::HipExecutor::create(1, omp);
+    auto cuda_1 = gko::CudaExecutor::create(1, omp);
     std::shared_ptr<gko::DpcppExecutor> host_dpcpp;
     std::shared_ptr<gko::DpcppExecutor> cpu_dpcpp;
     std::shared_ptr<gko::DpcppExecutor> gpu_dpcpp;
@@ -581,61 +584,20 @@ TEST(Executor, CanVerifyMemory)
 #if GINKGO_HIP_PLATFORM_NVCC
     ASSERT_EQ(true, hip->memory_accessible(cuda));
     ASSERT_EQ(true, cuda->memory_accessible(hip));
+    ASSERT_EQ(true, hip_1->memory_accessible(cuda_1));
+    ASSERT_EQ(true, cuda_1->memory_accessible(hip_1));
 #else
     ASSERT_EQ(false, hip->memory_accessible(cuda));
     ASSERT_EQ(false, cuda->memory_accessible(hip));
+    ASSERT_EQ(false, hip_1->memory_accessible(cuda_1));
+    ASSERT_EQ(false, cuda_1->memory_accessible(hip_1));
 #endif
     ASSERT_EQ(true, omp->memory_accessible(omp2));
     ASSERT_EQ(true, hip->memory_accessible(hip2));
     ASSERT_EQ(true, cuda->memory_accessible(cuda2));
-}
-
-
-TEST(Executor, CanVerifyMemoryBetweenTwoCudaDevices)
-{
-    auto omp = gko::OmpExecutor::create();
-    auto cuda = gko::CudaExecutor::create(0, omp);
-    if (cuda->get_num_devices() < 2) {
-        GTEST_SKIP() << "Only one cuda device available";
-    }
-    auto cuda_1 = gko::CudaExecutor::create(1, omp);
-
-    ASSERT_EQ(false, cuda->memory_accessible(cuda_1));
-}
-
-
-TEST(Executor, CanVerifyMemoryBetweenTwoHipDevices)
-{
-    auto omp = gko::OmpExecutor::create();
-    auto hip = gko::HipExecutor::create(0, omp);
-    if (hip->get_num_devices() < 2) {
-        GTEST_SKIP() << "Only one hip device available";
-    }
-    auto hip_1 = gko::HipExecutor::create(1, omp);
-
     ASSERT_EQ(false, hip->memory_accessible(hip_1));
-}
-
-
-TEST(Executor, CanVerifyMemoryBetweenCudaAndHipDevices)
-{
-    auto omp = gko::OmpExecutor::create();
-    auto hip = gko::HipExecutor::create(0, omp);
-    auto cuda = gko::CudaExecutor::create(0, omp);
-    if (hip->get_num_devices() < 2 || cuda->get_num_devices() < 2) {
-        GTEST_SKIP() << "Only one hip/one cuda device available";
-    }
-    auto hip_1 = gko::HipExecutor::create(1, omp);
-    auto cuda_1 = gko::CudaExecutor::create(1, omp);
-
-#if GINKGO_HIP_PLATFORM_NVCC
-    ASSERT_EQ(true, hip_1->memory_accessible(cuda_1));
-    ASSERT_EQ(true, cuda_1->memory_accessible(hip_1));
-#else
-    ASSERT_EQ(false, hip_1->memory_accessible(cuda_1));
-    ASSERT_EQ(false, cuda_1->memory_accessible(hip_1));
-#endif
     ASSERT_EQ(false, cuda->memory_accessible(hip_1));
+    ASSERT_EQ(false, cuda->memory_accessible(cuda_1));
     ASSERT_EQ(false, hip->memory_accessible(cuda_1));
 }
 
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index dbef057fc65..c49438f055c 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -72,17 +72,20 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
 
 void CudaExecutor::populate_exec_info(const MachineTopology *mach_topo)
 {
-    this->cuda_exec_info_.num_cores = mach_topo->get_num_cores();
-    cuda::device_guard g(this->get_device_id());
-    GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetPCIBusId(
-        const_cast<char *>(this->cuda_exec_info_.pci_bus_id.data()), 13,
-        this->get_device_id()));
-
-    auto cuda_hwloc_obj =
-        mach_topo->get_pci_device(this->cuda_exec_info_.pci_bus_id);
-    if (cuda_hwloc_obj) {
-        this->cuda_exec_info_.numa_node = cuda_hwloc_obj->numa;
-        this->cuda_exec_info_.closest_cpu_ids = cuda_hwloc_obj->closest_cpu_ids;
+    if (this->get_device_id() < this->get_num_devices() &&
+        this->get_device_id() >= 0) {
+        cuda::device_guard g(this->get_device_id());
+        GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetPCIBusId(
+            const_cast<char *>(this->get_exec_info().pci_bus_id.data()), 13,
+            this->get_device_id()));
+
+        auto cuda_hwloc_obj =
+            mach_topo->get_pci_device(this->get_exec_info().pci_bus_id);
+        if (cuda_hwloc_obj) {
+            this->get_exec_info().numa_node = cuda_hwloc_obj->numa;
+            this->get_exec_info().closest_pu_ids =
+                cuda_hwloc_obj->closest_pu_ids;
+        }
     }
 }
 
@@ -210,31 +213,32 @@ int CudaExecutor::get_num_devices()
 
 void CudaExecutor::set_gpu_property()
 {
-    if (this->cuda_exec_info_.device_id < this->get_num_devices() &&
-        this->cuda_exec_info_.device_id >= 0) {
+    if (this->get_device_id() < this->get_num_devices() &&
+        this->get_device_id() >= 0) {
         cuda::device_guard g(this->get_device_id());
         GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
-            &this->cuda_exec_info_.major, cudaDevAttrComputeCapabilityMajor,
-            this->cuda_exec_info_.device_id));
+            &this->get_exec_info().major, cudaDevAttrComputeCapabilityMajor,
+            this->get_device_id()));
         GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
-            &this->cuda_exec_info_.minor, cudaDevAttrComputeCapabilityMinor,
-            this->cuda_exec_info_.device_id));
+            &this->get_exec_info().minor, cudaDevAttrComputeCapabilityMinor,
+            this->get_device_id()));
         GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
-            &this->cuda_exec_info_.num_cores, cudaDevAttrMultiProcessorCount,
-            this->cuda_exec_info_.device_id));
-        this->cuda_exec_info_.num_work_groups_per_core =
-            convert_sm_ver_to_cores(this->cuda_exec_info_.major,
-                                    this->cuda_exec_info_.minor) /
+            &this->get_exec_info().num_computing_units,
+            cudaDevAttrMultiProcessorCount, this->get_device_id()));
+        this->get_exec_info().num_pe_per_cu =
+            convert_sm_ver_to_cores(this->get_exec_info().major,
+                                    this->get_exec_info().minor) /
             kernels::cuda::config::warp_size;
-        this->cuda_exec_info_.warp_size = kernels::cuda::config::warp_size;
+        this->get_exec_info().subgroup_sizes.push_back(
+            kernels::cuda::config::warp_size);
     }
 }
 
 
 void CudaExecutor::init_handles()
 {
-    if (this->cuda_exec_info_.device_id < this->get_num_devices() &&
-        this->cuda_exec_info_.device_id >= 0) {
+    if (this->get_device_id() < this->get_num_devices() &&
+        this->get_device_id() >= 0) {
         const auto id = this->get_device_id();
         cuda::device_guard g(id);
         this->cublas_handle_ = handle_manager<cublasContext>(
diff --git a/cuda/test/base/cuda_executor.cu b/cuda/test/base/cuda_executor.cu
index 184fd032c89..99db90458ee 100644
--- a/cuda/test/base/cuda_executor.cu
+++ b/cuda/test/base/cuda_executor.cu
@@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <memory>
-#include <thread>
 #include <type_traits>
 
 
diff --git a/cuda/test/base/cuda_executor_topology.cu b/cuda/test/base/cuda_executor_topology.cu
index 6ccc1a0e7a0..f5c2bce2d8d 100644
--- a/cuda/test/base/cuda_executor_topology.cu
+++ b/cuda/test/base/cuda_executor_topology.cu
@@ -80,8 +80,8 @@ protected:
     }
 
     std::shared_ptr<gko::Executor> omp;
-    std::shared_ptr<gko::CudaExecutor> cuda;
-    std::shared_ptr<gko::CudaExecutor> cuda2;
+    std::shared_ptr<const gko::CudaExecutor> cuda;
+    std::shared_ptr<const gko::CudaExecutor> cuda2;
 };
 
 
@@ -90,47 +90,57 @@ protected:
 
 inline int get_cpu_os_id(int log_id)
 {
-    return gko::get_machine_topology()->get_pu(log_id)->os_id;
+    return gko::MachineTopology::get_instance()->get_pu(log_id)->os_id;
 }
 
 
 inline int get_core_os_id(int log_id)
 {
-    return gko::get_machine_topology()->get_core(log_id)->os_id;
+    return gko::MachineTopology::get_instance()->get_core(log_id)->os_id;
 }
 
 
-TEST_F(CudaExecutor, CanBindToCpus)
+TEST_F(CudaExecutor, CanBindToSinglePu)
 {
-    auto cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+    cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+
+    const int bind_pu = 6;
+    cuda->bind_to_pu(bind_pu);
+
     auto cpu_sys = sched_getcpu();
+    ASSERT_TRUE(cpu_sys == get_cpu_os_id(6));
+}
 
-    const int bind_cpu[2] = {6, 3};
-    cuda->bind_to_cpus(bind_cpu, 2);
 
-    cpu_sys = sched_getcpu();
+TEST_F(CudaExecutor, CanBindToPus)
+{
+    cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+
+    std::vector<int> bind_pus = {6, 3};
+    cuda->bind_to_pus(bind_pus);
+
+    auto cpu_sys = sched_getcpu();
     ASSERT_TRUE(cpu_sys == get_cpu_os_id(3) || cpu_sys == get_cpu_os_id(6));
 }
 
 
 TEST_F(CudaExecutor, CanBindToCores)
 {
-    auto cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
-    auto cpu_sys = sched_getcpu();
+    cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
 
-    const int bind_cores[2] = {6, 3};
-    cuda->bind_to_cores(bind_cores, 2);
+    std::vector<int> bind_cores = {6, 3};
+    cuda->bind_to_cores(bind_cores);
 
-    cpu_sys = sched_getcpu();
+    auto cpu_sys = sched_getcpu();
     ASSERT_TRUE(cpu_sys == get_core_os_id(3) || cpu_sys == get_core_os_id(6));
 }
 
 
 TEST_F(CudaExecutor, KnowsItsClosestCpus)
 {
-    auto cuda0 = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+    cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
     auto cpu_sys = sched_getcpu();
-    auto close_cpus0 = cuda0->get_closest_cpus();
+    auto close_cpus0 = cuda->get_closest_pus();
 
     ASSERT_NE(close_cpus0[0], -1);
 }
@@ -138,10 +148,10 @@ TEST_F(CudaExecutor, KnowsItsClosestCpus)
 
 TEST_F(CudaExecutor, KnowsItsNuma)
 {
-    auto cuda0 = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
-    auto exec_info0 = cuda0->get_exec_info();
+    cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+    auto exec_info0 = cuda->get_exec_info();
     auto numa0 = exec_info0.numa_node;
-    auto close_cpu0 = cuda0->get_closest_cpus()[0];
+    auto close_cpu0 = cuda->get_closest_pus()[0];
 
     auto numa_sys0 = numa_node_of_cpu(get_cpu_os_id(close_cpu0));
 
diff --git a/cuda/test/utils.hpp b/cuda/test/utils.hpp
index 085bead5885..4b6e0e0a667 100644
--- a/cuda/test/utils.hpp
+++ b/cuda/test/utils.hpp
@@ -42,14 +42,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace {
 
-#if defined(__ICC) || defined(__ICL)
-#else
 
 // prevent device reset after each test
 auto no_reset_exec =
     gko::CudaExecutor::create(0, gko::ReferenceExecutor::create(), true);
 
-#endif
 
 }  // namespace
 
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 6652a7ccf7e..91c418d6608 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -165,14 +165,14 @@ int DpcppExecutor::get_num_devices(std::string device_type)
 bool DpcppExecutor::verify_memory_to(const OmpExecutor *dest_exec) const
 {
     auto device = detail::get_devices(
-        dpcpp_exec_info_.device_type)[dpcpp_exec_info_.device_id];
+        get_exec_info().device_type)[get_exec_info().device_id];
     return device.is_host() || device.is_cpu();
 }
 
 bool DpcppExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
 {
     auto device = detail::get_devices(
-        dpcpp_exec_info_.device_type)[dpcpp_exec_info_.device_id];
+        get_exec_info().device_type)[get_exec_info().device_id];
     auto other_device = detail::get_devices(
         dest_exec->get_device_type())[dest_exec->get_device_id()];
     return ((device.is_host() || device.is_cpu()) &&
@@ -198,28 +198,31 @@ void delete_queue(sycl::queue *queue)
 
 void DpcppExecutor::set_device_property()
 {
-    assert(this->dpcpp_exec_info_.device_id <
-           DpcppExecutor::get_num_devices(this->dpcpp_exec_info_.device_type));
+    assert(this->get_exec_info().device_id <
+           DpcppExecutor::get_num_devices(this->get_exec_info().device_type));
     auto device = detail::get_devices(
-        this->dpcpp_exec_info_.device_type)[this->dpcpp_exec_info_.device_id];
+        this->get_exec_info().device_type)[this->get_exec_info().device_id];
     if (!device.is_host()) {
         try {
-            this->dpcpp_exec_info_.subgroup_sizes =
+            auto subgroup_sizes =
                 device.get_info<cl::sycl::info::device::sub_group_sizes>();
+            for (auto &i : subgroup_sizes) {
+                this->get_exec_info().subgroup_sizes.push_back(i);
+            }
         } catch (cl::sycl::runtime_error &err) {
             GKO_NOT_SUPPORTED(device);
         }
     }
-    this->dpcpp_exec_info_.num_computing_units =
+    this->get_exec_info().num_computing_units =
         device.get_info<sycl::info::device::max_compute_units>();
     auto max_workitem_sizes =
         device.get_info<sycl::info::device::max_work_item_sizes>();
     // There is no way to get the dimension of a sycl::id object
     for (std::size_t i = 0; i < 3; i++) {
-        this->dpcpp_exec_info_.max_workitem_sizes.push_back(
+        this->get_exec_info().max_workitem_sizes.push_back(
             max_workitem_sizes[i]);
     }
-    this->dpcpp_exec_info_.num_work_groups_per_core =
+    this->get_exec_info().num_pe_per_cu =
         device.get_info<sycl::info::device::max_work_group_size>();
     // Here we declare the queue with the property `in_order` which ensures the
     // kernels are executed in the submission order. Otherwise, calls to
diff --git a/dpcpp/test/base/CMakeLists.txt b/dpcpp/test/base/CMakeLists.txt
index adb2f2505b5..6d634f249eb 100644
--- a/dpcpp/test/base/CMakeLists.txt
+++ b/dpcpp/test/base/CMakeLists.txt
@@ -1 +1,2 @@
 ginkgo_create_dpcpp_test(executor)
+ginkgo_create_dpcpp_test(executor_topology)
diff --git a/dpcpp/test/base/executor_topology.dp.cpp b/dpcpp/test/base/executor_topology.dp.cpp
new file mode 100644
index 00000000000..98406f8ae7d
--- /dev/null
+++ b/dpcpp/test/base/executor_topology.dp.cpp
@@ -0,0 +1,93 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include <exception>
+#include <memory>
+#include <type_traits>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
+namespace {
+
+
+class DpcppExecutor : public ::testing::Test {
+protected:
+    DpcppExecutor()
+        : omp(gko::OmpExecutor::create()), dpcpp(nullptr), dpcpp2(nullptr)
+    {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices("cpu"), 0);
+        dpcpp = gko::DpcppExecutor::create(0, omp, "cpu");
+        if (gko::DpcppExecutor::get_num_devices("gpu") > 0) {
+            dpcpp2 = gko::DpcppExecutor::create(0, omp, "gpu");
+        }
+    }
+
+    void TearDown()
+    {
+        if (dpcpp != nullptr) {
+            // ensure that previous calls finished and didn't throw an error
+            ASSERT_NO_THROW(dpcpp->synchronize());
+        }
+    }
+
+    std::shared_ptr<gko::Executor> omp;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp2;
+};
+
+
+TEST_F(DpcppExecutor, CanInstantiateTwoExecutorsOnOneDevice)
+{
+    dpcpp = gko::DpcppExecutor::create(0, omp);
+    dpcpp2 = gko::DpcppExecutor::create(0, omp);
+
+    // We want automatic deinitialization to not create any error
+}
+
+
+}  // namespace
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 1f2d0ad8f67..ede895b47f3 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -73,17 +73,20 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
 
 void HipExecutor::populate_exec_info(const MachineTopology *mach_topo)
 {
-    this->hip_exec_info_.num_cores = mach_topo->get_num_cores();
-    hip::device_guard g(this->get_device_id());
-    GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetPCIBusId(
-        const_cast<char *>(this->hip_exec_info_.pci_bus_id.data()), 13,
-        this->get_device_id()));
-
-    auto hip_hwloc_obj =
-        mach_topo->get_pci_device(this->hip_exec_info_.pci_bus_id);
-    if (hip_hwloc_obj) {
-        this->hip_exec_info_.numa_node = hip_hwloc_obj->numa;
-        this->hip_exec_info_.closest_cpu_ids = hip_hwloc_obj->closest_cpu_ids;
+    if (this->get_device_id() < this->get_num_devices() &&
+        this->get_device_id() >= 0) {
+        hip::device_guard g(this->get_device_id());
+        GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetPCIBusId(
+            const_cast<char *>(this->get_exec_info().pci_bus_id.data()), 13,
+            this->get_device_id()));
+
+        auto hip_hwloc_obj =
+            mach_topo->get_pci_device(this->get_exec_info().pci_bus_id);
+        if (hip_hwloc_obj) {
+            this->get_exec_info().numa_node = hip_hwloc_obj->numa;
+            this->get_exec_info().closest_pu_ids =
+                hip_hwloc_obj->closest_pu_ids;
+        }
     }
 }
 
@@ -215,25 +218,26 @@ void HipExecutor::set_gpu_property()
         this->get_device_id() >= 0) {
         hip::device_guard g(this->get_device_id());
         GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
-            &this->hip_exec_info_.num_cores,
+            &this->get_exec_info().num_computing_units,
             hipDeviceAttributeMultiprocessorCount, this->get_device_id()));
         GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
-            &this->hip_exec_info_.major,
+            &this->get_exec_info().major,
             hipDeviceAttributeComputeCapabilityMajor, this->get_device_id()));
         GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
-            &this->hip_exec_info_.minor,
+            &this->get_exec_info().minor,
             hipDeviceAttributeComputeCapabilityMinor, this->get_device_id()));
 #if GINKGO_HIP_PLATFORM_NVCC
-        this->hip_exec_info_.num_work_groups_per_core =
-            convert_sm_ver_to_cores(this->hip_exec_info_.major,
-                                    this->hip_exec_info_.minor) /
+        this->get_exec_info().num_pe_per_cu =
+            convert_sm_ver_to_cores(this->get_exec_info().major,
+                                    this->get_exec_info().minor) /
             kernels::hip::config::warp_size;
 #else
         // In GCN (Graphics Core Next), each multiprocessor has 4 SIMD
         // Reference: https://en.wikipedia.org/wiki/Graphics_Core_Next
-        this->hip_exec_info_.num_work_groups_per_core = 4;
+        this->get_exec_info().num_pe_per_cu = 4;
 #endif  // GINKGO_HIP_PLATFORM_NVCC
-        this->hip_exec_info_.warp_size = kernels::hip::config::warp_size;
+        this->get_exec_info().subgroup_sizes.push_back(
+            kernels::hip::config::warp_size);
     }
 }
 
diff --git a/hip/test/base/hip_executor.hip.cpp b/hip/test/base/hip_executor.hip.cpp
index 7e76f9778a7..db829c4bf92 100644
--- a/hip/test/base/hip_executor.hip.cpp
+++ b/hip/test/base/hip_executor.hip.cpp
@@ -142,30 +142,6 @@ TEST_F(HipExecutor, MasterKnowsNumberOfDevices)
 }
 
 
-#if GKO_HAVE_HWLOC
-
-
-inline int get_os_id(int log_id)
-{
-    return gko::get_machine_topology()->get_core(log_id)->os_id;
-}
-
-
-TEST_F(HipExecutor, CanBindToCores)
-{
-    auto hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
-    auto cpu_sys = sched_getcpu();
-    const int bind_core[2] = {6, 3};
-    hip->bind_to_cores(bind_core, 2);
-
-    cpu_sys = sched_getcpu();
-    ASSERT_TRUE(cpu_sys == get_os_id(3) || cpu_sys == get_os_id(6));
-}
-
-
-#endif
-
-
 TEST_F(HipExecutor, AllocatesAndFreesMemory)
 {
     int *ptr = nullptr;
diff --git a/hip/test/base/hip_executor_topology.hip.cpp b/hip/test/base/hip_executor_topology.hip.cpp
index ca56c3494cf..826d464bcc3 100644
--- a/hip/test/base/hip_executor_topology.hip.cpp
+++ b/hip/test/base/hip_executor_topology.hip.cpp
@@ -85,8 +85,8 @@ class HipExecutor : public ::testing::Test {
     }
 
     std::shared_ptr<gko::Executor> omp;
-    std::shared_ptr<gko::HipExecutor> hip;
-    std::shared_ptr<gko::HipExecutor> hip2;
+    std::shared_ptr<const gko::HipExecutor> hip;
+    std::shared_ptr<const gko::HipExecutor> hip2;
 };
 
 
@@ -95,47 +95,57 @@ class HipExecutor : public ::testing::Test {
 
 inline int get_cpu_os_id(int log_id)
 {
-    return gko::get_machine_topology()->get_pu(log_id)->os_id;
+    return gko::MachineTopology::get_instance()->get_pu(log_id)->os_id;
 }
 
 
 inline int get_core_os_id(int log_id)
 {
-    return gko::get_machine_topology()->get_core(log_id)->os_id;
+    return gko::MachineTopology::get_instance()->get_core(log_id)->os_id;
 }
 
 
-TEST_F(HipExecutor, CanBindToCpus)
+TEST_F(HipExecutor, CanBindToSinglePu)
 {
-    auto hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+    hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+
+    const int bind_pu = 6;
+    hip->bind_to_pu(bind_pu);
+
     auto cpu_sys = sched_getcpu();
+    ASSERT_TRUE(cpu_sys == get_cpu_os_id(6));
+}
 
-    const int bind_cpu[2] = {6, 3};
-    hip->bind_to_cpus(bind_cpu, 2);
 
-    cpu_sys = sched_getcpu();
+TEST_F(HipExecutor, CanBindToPus)
+{
+    hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+
+    std::vector<int> bind_pus = {6, 3};
+    hip->bind_to_pus(bind_pus);
+
+    auto cpu_sys = sched_getcpu();
     ASSERT_TRUE(cpu_sys == get_cpu_os_id(3) || cpu_sys == get_cpu_os_id(6));
 }
 
 
 TEST_F(HipExecutor, CanBindToCores)
 {
-    auto hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
-    auto cpu_sys = sched_getcpu();
+    hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
 
-    const int bind_cores[2] = {6, 3};
-    hip->bind_to_cores(bind_cores, 2);
+    std::vector<int> bind_cores = {6, 3};
+    hip->bind_to_cores(bind_cores);
 
-    cpu_sys = sched_getcpu();
+    auto cpu_sys = sched_getcpu();
     ASSERT_TRUE(cpu_sys == get_core_os_id(3) || cpu_sys == get_core_os_id(6));
 }
 
 
 TEST_F(HipExecutor, KnowsItsClosestCpus)
 {
-    auto hip0 = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+    hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
     auto cpu_sys = sched_getcpu();
-    auto close_cpus0 = hip0->get_closest_cpus();
+    auto close_cpus0 = hip->get_closest_pus();
 
     ASSERT_NE(close_cpus0[0], -1);
 }
@@ -143,12 +153,12 @@ TEST_F(HipExecutor, KnowsItsClosestCpus)
 
 TEST_F(HipExecutor, KnowsItsNuma)
 {
-    auto hip0 = gko::HipExecutor::create(0, gko::OmpExecutor::create());
-    auto exec_info0 = hip0->get_exec_info();
+    hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+    auto exec_info0 = hip->get_exec_info();
     auto numa0 = exec_info0.numa_node;
-    auto close_cpu0 = hip0->get_closest_cpus()[0];
+    auto close_cpu0 = hip->get_closest_pus();
 
-    auto numa_sys0 = numa_node_of_cpu(get_cpu_os_id(close_cpu0));
+    auto numa_sys0 = numa_node_of_cpu(get_cpu_os_id(close_cpu0[0]));
 
     ASSERT_TRUE(numa0 == numa_sys0);
 }
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 7747371486d..c1ecd1f0cbf 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -34,12 +34,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_PUBLIC_CORE_BASE_EXECUTOR_HPP_
 
 
+#include <array>
 #include <memory>
 #include <mutex>
 #include <sstream>
 #include <string>
 #include <tuple>
 #include <type_traits>
+#include <vector>
 
 
 #include <ginkgo/core/base/machine_topology.hpp>
@@ -647,6 +649,97 @@ class Executor : public log::EnableLogging<Executor> {
         return this->verify_memory_from(other.get());
     }
 
+    /**
+     * A struct that abstracts the executor info for different executors
+     * classes.
+     */
+    struct exec_info {
+        /**
+         * The id of the device.
+         */
+        int device_id = -1;
+
+        /**
+         * The type of the device, relevant only for the dpcpp executor.
+         */
+        std::string device_type;
+
+        /**
+         * The numa node of the executor.
+         */
+        int numa_node = -1;
+
+        /**
+         * The number of computing units in the executor.
+         *
+         * @note In CPU executors this is equivalent to the number of cores.
+         *       In CUDA and HIP executors this is the number of Streaming
+         *       Multiprocessors. In DPCPP, this is the number of computing
+         *       units.
+         */
+        int num_computing_units = -1;
+
+        /**
+         * The number of processing elements per computing unit in the executor.
+         *
+         * @note In CPU executors this is equivalent to the number of SIMD units
+         *       per core. In CUDA and HIP executors this is the number of warps
+         *       per SM. In DPCPP, this is the max work group size.
+         */
+        int num_pe_per_cu = -1;
+
+        /**
+         * The sizes of the subgroup for the executor.
+         *
+         * @note In CPU executors this is invalid.
+         *       In CUDA and HIP executors warp size.
+         *       In DPCPP, this is the subgroup sizes for the device associated
+         *       with the dpcpp executor.
+         */
+        std::vector<int> subgroup_sizes{};
+
+        /**
+         * The sizes of the work items for the executor.
+         *
+         * @note In CPU executors this is invalid.
+         *       In CUDA and HIP executors this is invalid.
+         *       In DPCPP, this is the maximum sizes for the workitems for the
+         *       device associated with the dpcpp executor.
+         */
+        std::vector<int> max_workitem_sizes{};
+
+        /**
+         * The major version for CUDA/HIP device.
+         */
+        int major = -1;
+
+        /**
+         * The minor version for CUDA/HIP device.
+         */
+        int minor = -1;
+
+        /**
+         * The PCI bus id of the device.
+         *
+         * @note Only relevant for I/O devices (GPUs).
+         */
+        std::string pci_bus_id = std::string(13, 'x');
+
+        /**
+         * The processing units closest to the device.
+         *
+         * @note Only relevant for CUDA, HIP and DPCPP executors.
+         */
+        std::vector<int> closest_pu_ids{};
+    };
+
+    /**
+     * Gets the exec info struct
+     *
+     * @return  the exec_info struct
+     */
+    const exec_info &get_exec_info() const { return this->exec_info_; }
+
 protected:
     /**
      * Allocates raw memory in this Executor.
@@ -733,17 +826,18 @@ class Executor : public log::EnableLogging<Executor> {
      */
     virtual void populate_exec_info(const MachineTopology *mach_topo) = 0;
 
-    struct exec_info {
-        int device_id = -1;
-        int numa_node = -1;
-        int num_cores = -1;
-        int num_work_groups_per_core = -1;
-        int major = -1;
-        int minor = -1;
-        int warp_size = -1;
-        std::string pci_bus_id = std::string(13, 'x');
-        std::vector<int> closest_cpu_ids{};
-    };
+    /**
+     * Gets the modifiable exec info object
+     *
+     * @return  the pointer to the exec_info object
+     */
+    exec_info &get_exec_info()
+    {
+        return const_cast<exec_info &>(
+            static_cast<const Executor &>(*this).get_exec_info());
+    }
+
+    exec_info exec_info_;
 
 private:
     /**
@@ -913,6 +1007,57 @@ class ExecutorBase : public Executor {
 
 #undef GKO_DECLARE_EXECUTOR_FRIEND
 
+
+/**
+ * This class provides some binding functions from the MachineTopology class and
+ * the executors can derive from this class as necessary.
+ *
+ * The executor object can then be bound to the bind-able objects from the
+ * machine topology by calling the member functions of this class.
+ */
+class EnableBinding {
+public:
+    /**
+     * Bind to Processing units (PU)
+     *
+     * @param ids  The ids to be bound to the object.
+     */
+    void bind_to_pus(const std::vector<int> &ids) const
+    {
+        MachineTopology::get_instance()->bind_to_pus(ids);
+    }
+
+    /**
+     * Bind to a Processing unit (PU)
+     *
+     * @param id  The id to be bound to the object.
+     */
+    void bind_to_pu(const int &id) const
+    {
+        MachineTopology::get_instance()->bind_to_pus(std::vector<int>{id});
+    }
+
+    /**
+     * Bind to cores
+     *
+     * @param ids  The ids to be bound to the calling object.
+     */
+    void bind_to_cores(const std::vector<int> &ids) const
+    {
+        MachineTopology::get_instance()->bind_to_cores(ids);
+    }
+
+    /**
+     * Bind to core
+     *
+     * @param id  The id to be bound to the calling object.
+     */
+    void bind_to_core(const int &id) const
+    {
+        MachineTopology::get_instance()->bind_to_cores(std::vector<int>{id});
+    }
+};
+
 /**
  * Controls whether the DeviceReset function should be called thanks to a
  * boolean. Note that in any case, `DeviceReset` is called only after destroying
@@ -994,17 +1139,20 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
 
     void synchronize() const override;
 
-    int get_num_cores() const { return this->omp_exec_info_.num_cores; }
+    int get_num_cores() const
+    {
+        return this->get_exec_info().num_computing_units;
+    }
 
     int get_num_threads_per_core() const
     {
-        return this->omp_exec_info_.num_work_groups_per_core;
+        return this->get_exec_info().num_pe_per_cu;
     }
 
 protected:
     OmpExecutor()
     {
-        this->OmpExecutor::populate_exec_info(get_machine_topology());
+        this->OmpExecutor::populate_exec_info(MachineTopology::get_instance());
     }
 
     void populate_exec_info(const MachineTopology *mach_topo) override;
@@ -1024,9 +1172,6 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
     GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(CudaExecutor, false);
 
     bool verify_memory_to(const DpcppExecutor *dest_exec) const override;
-
-private:
-    exec_info omp_exec_info_;
 };
 
 
@@ -1062,14 +1207,15 @@ class ReferenceExecutor : public OmpExecutor {
 protected:
     ReferenceExecutor()
     {
-        this->ReferenceExecutor::populate_exec_info(get_machine_topology());
+        this->ReferenceExecutor::populate_exec_info(
+            MachineTopology::get_instance());
     }
 
     void populate_exec_info(const MachineTopology *) override
     {
-        this->ref_exec_info_.device_id = -1;
-        this->ref_exec_info_.num_cores = 1;
-        this->ref_exec_info_.num_work_groups_per_core = 1;
+        this->get_exec_info().device_id = -1;
+        this->get_exec_info().num_computing_units = 1;
+        this->get_exec_info().num_pe_per_cu = 1;
     }
 
     bool verify_memory_from(const Executor *src_exec) const override
@@ -1086,9 +1232,6 @@ class ReferenceExecutor : public OmpExecutor {
     GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(CudaExecutor, false);
 
     GKO_DEFAULT_OVERRIDE_VERIFY_MEMORY(HipExecutor, false);
-
-private:
-    exec_info ref_exec_info_;
 };
 
 
@@ -1107,12 +1250,11 @@ using DefaultExecutor = ReferenceExecutor;
  */
 class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
                      public std::enable_shared_from_this<CudaExecutor>,
+                     public detail::EnableBinding,
                      public detail::EnableDeviceReset {
     friend class detail::ExecutorBase<CudaExecutor>;
 
 public:
-    using cuda_exec_info = exec_info;
-
     /**
      * Creates a new CudaExecutor.
      *
@@ -1124,7 +1266,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         int device_id, std::shared_ptr<Executor> master,
         bool device_reset = false);
 
-    ~CudaExecutor() { decrease_num_execs(this->cuda_exec_info_.device_id); }
+    ~CudaExecutor() { decrease_num_execs(this->get_exec_info().device_id); }
 
     std::shared_ptr<Executor> get_master() noexcept override;
 
@@ -1139,7 +1281,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
      */
     int get_device_id() const noexcept
     {
-        return this->cuda_exec_info_.device_id;
+        return this->get_exec_info().device_id;
     }
 
     /**
@@ -1152,7 +1294,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
      */
     int get_num_warps_per_sm() const noexcept
     {
-        return this->cuda_exec_info_.num_work_groups_per_core;
+        return this->get_exec_info().num_pe_per_cu;
     }
 
     /**
@@ -1160,7 +1302,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
      */
     int get_num_multiprocessor() const noexcept
     {
-        return this->cuda_exec_info_.num_cores;
+        return this->get_exec_info().num_computing_units;
     }
 
     /**
@@ -1168,8 +1310,8 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
      */
     int get_num_warps() const noexcept
     {
-        return this->cuda_exec_info_.num_cores *
-               this->cuda_exec_info_.num_work_groups_per_core;
+        return this->get_exec_info().num_computing_units *
+               this->get_exec_info().num_pe_per_cu;
     }
 
     /**
@@ -1177,7 +1319,9 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
      */
     int get_warp_size() const noexcept
     {
-        return this->cuda_exec_info_.warp_size;
+        return (this->get_exec_info().subgroup_sizes.size()
+                    ? this->get_exec_info().subgroup_sizes[0]
+                    : -1);
     }
 
     /**
@@ -1185,7 +1329,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
      */
     int get_major_version() const noexcept
     {
-        return this->cuda_exec_info_.major;
+        return this->get_exec_info().major;
     }
 
     /**
@@ -1193,7 +1337,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
      */
     int get_minor_version() const noexcept
     {
-        return this->cuda_exec_info_.minor;
+        return this->get_exec_info().minor;
     }
 
     /**
@@ -1214,38 +1358,15 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     }
 
     /**
-     * Get the closest CPUs
+     * Get the closest PUs
      *
-     * @return  the array of CPUs closest to this device
+     * @return  the array of PUs closest to this device
      */
-    std::vector<int> get_closest_cpus() const
+    std::vector<int> get_closest_pus() const
     {
-        return this->cuda_exec_info_.closest_cpu_ids;
+        return this->get_exec_info().closest_pu_ids;
     }
 
-    /**
-     * Gets the exec info struct
-     *
-     * @return  the exec_info struct
-     */
-    cuda_exec_info get_exec_info() const { return this->cuda_exec_info_; }
-
-    /**
-     * Bind the current executor thread to the list of cpus
-     *
-     * @param  ids  the logical ids of the cpus.
-     * @param  num_ids  the number of cpus to bind
-     */
-    void bind_to_cpus(const int *ids, const size_type num_ids) const;
-
-    /**
-     * Bind the current executor thread to the list of cores
-     *
-     * @param  ids  the logical ids of the cores.
-     * @param  num_ids  the number of cores to bind
-     */
-    void bind_to_cores(const int *ids, const size_type num_ids) const;
-
 protected:
     void set_gpu_property();
 
@@ -1255,19 +1376,16 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
                  bool device_reset = false)
         : EnableDeviceReset{device_reset}, master_(master)
     {
-        this->cuda_exec_info_.device_id = device_id;
-        this->cuda_exec_info_.num_cores = 0;
-        this->cuda_exec_info_.num_work_groups_per_core = 0;
-        assert(this->cuda_exec_info_.device_id < max_devices &&
-               this->cuda_exec_info_.device_id >= 0);
-        this->CudaExecutor::populate_exec_info(get_machine_topology());
-        if (this->cuda_exec_info_.closest_cpu_ids.size()) {
-            this->bind_to_cpus(this->cuda_exec_info_.closest_cpu_ids.data(),
-                               this->cuda_exec_info_.closest_cpu_ids.size());
+        this->get_exec_info().device_id = device_id;
+        this->get_exec_info().num_computing_units = 0;
+        this->get_exec_info().num_pe_per_cu = 0;
+        this->CudaExecutor::populate_exec_info(MachineTopology::get_instance());
+        if (this->get_exec_info().closest_pu_ids.size()) {
+            this->bind_to_pus(this->get_exec_info().closest_pu_ids);
         }
         this->set_gpu_property();
         this->init_handles();
-        increase_num_execs(this->cuda_exec_info_.device_id);
+        increase_num_execs(this->get_exec_info().device_id);
     }
 
     void *raw_alloc(size_type size) const override;
@@ -1308,7 +1426,6 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
 private:
     std::shared_ptr<Executor> master_;
-    cuda_exec_info cuda_exec_info_;
 
     template <typename T>
     using handle_manager = std::unique_ptr<T, std::function<void(T *)>>;
@@ -1336,12 +1453,11 @@ using DefaultExecutor = CudaExecutor;
  */
 class HipExecutor : public detail::ExecutorBase<HipExecutor>,
                     public std::enable_shared_from_this<HipExecutor>,
+                    public detail::EnableBinding,
                     public detail::EnableDeviceReset {
     friend class detail::ExecutorBase<HipExecutor>;
 
 public:
-    using hip_exec_info = exec_info;
-
     /**
      * Creates a new HipExecutor.
      *
@@ -1353,7 +1469,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
                                                std::shared_ptr<Executor> master,
                                                bool device_reset = false);
 
-    ~HipExecutor() { decrease_num_execs(this->hip_exec_info_.device_id); }
+    ~HipExecutor() { decrease_num_execs(this->get_exec_info().device_id); }
 
     std::shared_ptr<Executor> get_master() noexcept override;
 
@@ -1368,7 +1484,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
      */
     int get_device_id() const noexcept
     {
-        return this->hip_exec_info_.device_id;
+        return this->get_exec_info().device_id;
     }
 
     /**
@@ -1381,7 +1497,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
      */
     int get_num_warps_per_sm() const noexcept
     {
-        return this->hip_exec_info_.num_work_groups_per_core;
+        return this->get_exec_info().num_pe_per_cu;
     }
 
     /**
@@ -1389,7 +1505,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
      */
     int get_num_multiprocessor() const noexcept
     {
-        return this->hip_exec_info_.num_cores;
+        return this->get_exec_info().num_computing_units;
     }
 
     /**
@@ -1397,7 +1513,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
      */
     int get_major_version() const noexcept
     {
-        return this->hip_exec_info_.major;
+        return this->get_exec_info().major;
     }
 
     /**
@@ -1405,7 +1521,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
      */
     int get_minor_version() const noexcept
     {
-        return this->hip_exec_info_.minor;
+        return this->get_exec_info().minor;
     }
 
     /**
@@ -1413,8 +1529,8 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
      */
     int get_num_warps() const noexcept
     {
-        return this->hip_exec_info_.num_cores *
-               this->hip_exec_info_.num_work_groups_per_core;
+        return this->get_exec_info().num_computing_units *
+               this->get_exec_info().num_pe_per_cu;
     }
 
     /**
@@ -1422,7 +1538,9 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
      */
     int get_warp_size() const noexcept
     {
-        return this->hip_exec_info_.warp_size;
+        return (this->get_exec_info().subgroup_sizes.size()
+                    ? this->get_exec_info().subgroup_sizes[0]
+                    : -1);
     }
 
     /**
@@ -1443,38 +1561,15 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     }
 
     /**
-     * Get the closest CPUs
+     * Get the closest PUs
      *
-     * @return  the array of CPUs closest to this device
+     * @return  the array of PUs closest to this device
      */
-    std::vector<int> get_closest_cpus() const
+    std::vector<int> get_closest_pus() const
     {
-        return this->hip_exec_info_.closest_cpu_ids;
+        return this->get_exec_info().closest_pu_ids;
     }
 
-    /**
-     * Gets the exec info struct
-     *
-     * @return  the exec_info struct
-     */
-    hip_exec_info get_exec_info() const { return this->hip_exec_info_; }
-
-    /**
-     * Bind the current executor thread to the list of cpus
-     *
-     * @param  ids  the logical ids of the cpus.
-     * @param  num_ids  the number of cpus to bind
-     */
-    void bind_to_cpus(const int *ids, const size_type num_ids) const;
-
-    /**
-     * Bind the current executor thread to the list of cores
-     *
-     * @param  ids  the logical ids of the cores.
-     * @param  num_ids  the number of cores to bind
-     */
-    void bind_to_cores(const int *ids, const size_type num_ids) const;
-
 protected:
     void set_gpu_property();
 
@@ -1484,18 +1579,16 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
                 bool device_reset = false)
         : EnableDeviceReset{device_reset}, master_(master)
     {
-        this->hip_exec_info_.device_id = device_id;
-        this->hip_exec_info_.num_cores = 0;
-        this->hip_exec_info_.num_work_groups_per_core = 0;
-        assert(this->hip_exec_info_.device_id < max_devices);
-        this->HipExecutor::populate_exec_info(get_machine_topology());
-        if (this->hip_exec_info_.closest_cpu_ids.size()) {
-            this->bind_to_cpus(this->hip_exec_info_.closest_cpu_ids.data(),
-                               this->hip_exec_info_.closest_cpu_ids.size());
+        this->get_exec_info().device_id = device_id;
+        this->get_exec_info().num_computing_units = 0;
+        this->get_exec_info().num_pe_per_cu = 0;
+        this->HipExecutor::populate_exec_info(MachineTopology::get_instance());
+        if (this->get_exec_info().closest_pu_ids.size()) {
+            this->bind_to_pus(this->get_exec_info().closest_pu_ids);
         }
         this->set_gpu_property();
         this->init_handles();
-        increase_num_execs(this->hip_exec_info_.device_id);
+        increase_num_execs(this->get_exec_info().device_id);
     }
 
     void *raw_alloc(size_type size) const override;
@@ -1536,7 +1629,6 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
 private:
     std::shared_ptr<Executor> master_;
-    hip_exec_info hip_exec_info_;
 
     template <typename T>
     using handle_manager = std::unique_ptr<T, std::function<void(T *)>>;
@@ -1593,7 +1685,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      *
      * @return the DPCPP device id of the device associated to this executor
      */
-    int get_device_id() const noexcept { return dpcpp_exec_info_.device_id; }
+    int get_device_id() const noexcept { return get_exec_info().device_id; }
 
     ::cl::sycl::queue *get_queue() const { return queue_.get(); }
 
@@ -1611,9 +1703,9 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      *
      * @return the available subgroup sizes for this device
      */
-    const std::vector<size_type> &get_subgroup_sizes() const noexcept
+    const std::vector<int> &get_subgroup_sizes() const noexcept
     {
-        return dpcpp_exec_info_.subgroup_sizes;
+        return get_exec_info().subgroup_sizes;
     }
 
     /**
@@ -1623,7 +1715,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      */
     size_type get_num_computing_units() const noexcept
     {
-        return dpcpp_exec_info_.num_computing_units;
+        return get_exec_info().num_computing_units;
     }
 
     /**
@@ -1631,9 +1723,9 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      *
      * @return the maximum work item sizes
      */
-    const std::vector<size_type> &get_max_workitem_sizes() const noexcept
+    const std::vector<int> &get_max_workitem_sizes() const noexcept
     {
-        return dpcpp_exec_info_.max_workitem_sizes;
+        return get_exec_info().max_workitem_sizes;
     }
 
     /**
@@ -1643,7 +1735,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      */
     size_type get_max_workgroup_size() const noexcept
     {
-        return dpcpp_exec_info_.num_work_groups_per_core;
+        return get_exec_info().num_pe_per_cu;
     }
 
     /**
@@ -1653,7 +1745,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      */
     std::string get_device_type() const noexcept
     {
-        return dpcpp_exec_info_.device_type;
+        return get_exec_info().device_type;
     }
 
 protected:
@@ -1665,8 +1757,8 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
     {
         std::for_each(device_type.begin(), device_type.end(),
                       [](char &c) { c = std::tolower(c); });
-        this->dpcpp_exec_info_.device_type = std::string(device_type);
-        this->dpcpp_exec_info_.device_id = device_id;
+        this->get_exec_info().device_type = std::string(device_type);
+        this->get_exec_info().device_id = device_id;
         this->set_device_property();
     }
 
@@ -1691,13 +1783,13 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
 private:
     std::shared_ptr<Executor> master_;
 
-    struct dpcpp_exec_info : public exec_info {
-        int device_id;
-        std::string device_type;
-        int num_computing_units;
-        std::vector<size_type> subgroup_sizes{};
-        std::vector<size_type> max_workitem_sizes{};
-    } dpcpp_exec_info_;
+    // struct dpcpp_exec_info : public exec_info {
+    //     int device_id;
+    //     std::string device_type;
+    //     int num_computing_units;
+    //     std::vector<size_type> subgroup_sizes{};
+    //     std::vector<size_type> max_workitem_sizes{};
+    // } get_exec_info();
 
     template <typename T>
     using queue_manager = std::unique_ptr<T, std::function<void(T *)>>;
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index b83bdca33f5..3ede524c131 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <memory>
 #include <mutex>
 #include <string>
+#include <type_traits>
 #include <vector>
 
 
@@ -68,23 +69,6 @@ struct hwloc_bitmap_s;
 namespace gko {
 
 
-class MachineTopology;
-
-
-/**
- * This global function initializes the topology in its first call and stores
- * the MachineTopology global object in a shared_ptr that any gko function/class
- * can query.
- *
- * This makes sure that the topology object is not re-initialized every
- * time it is needed. Since the topology is tied to the machine anyway,
- * each re-initialization would populate the exact same topology tree.
- *
- * @return  the global MachineTopology object.
- */
-extern const MachineTopology *get_machine_topology();
-
-
 /**
  * The machine topology class represents the hierarchical topology of a machine,
  * including NUMA nodes, cores and PCI Devices. Various infomation of the
@@ -104,8 +88,6 @@ extern const MachineTopology *get_machine_topology();
  * read.
  */
 class MachineTopology {
-    friend const MachineTopology *get_machine_topology();
-
     template <typename T>
     using hwloc_manager = std::unique_ptr<T, std::function<void(T *)>>;
 
@@ -213,7 +195,7 @@ class MachineTopology {
         /**
          * The array of CPU ids closest to the object.
          */
-        std::vector<int> closest_cpu_ids;
+        std::vector<int> closest_pu_ids;
 
         /**
          * The PCI Bus ID
@@ -222,6 +204,17 @@ class MachineTopology {
     };
 
 public:
+    /**
+     * Returns an instance of the MachineTopology object.
+     *
+     * @return  the MachineTopology instance
+     */
+    static MachineTopology *get_instance()
+    {
+        static MachineTopology instance;
+        return &instance;
+    }
+
     /**
      * Do not allow the MachineTopology object to be copied/moved. There should
      * be only one global object per execution.
@@ -233,23 +226,23 @@ class MachineTopology {
     ~MachineTopology() = default;
 
     /**
-     * Bind the object associated with the id to a core.
+     * Bind the object associated with the ids to a core.
      *
-     * @param id  The id of the object to be bound.
+     * @param ids  The ids of the object to be bound.
      */
-    void bind_to_cores(const int *ids, const size_type num_ids) const
+    void bind_to_cores(const std::vector<int> &ids) const
     {
-        hwloc_binding_helper(this->cores_, ids, num_ids);
+        hwloc_binding_helper(this->cores_, ids);
     }
 
     /**
-     * Bind the object associated with the id to a Processing unit(PU).
+     * Bind the object associated with the ids to a Processing unit(PU).
      *
-     * @param id  The id of the object to be bound.
+     * @param ids  The ids of the object to be bound.
      */
-    void bind_to_pus(const int *ids, const size_type num_ids) const
+    void bind_to_pus(const std::vector<int> &ids) const
     {
-        hwloc_binding_helper(this->pus_, ids, num_ids);
+        hwloc_binding_helper(this->pus_, ids);
     }
 
     /**
@@ -341,11 +334,11 @@ class MachineTopology {
     /**
      * @internal
      *
-     * A helper function that binds the object with an id.
+     * A helper function that binds the object with ids.
      */
     void hwloc_binding_helper(
-        const std::vector<MachineTopology::normal_obj_info> &obj, const int *id,
-        const size_type num_ids) const;
+        const std::vector<MachineTopology::normal_obj_info> &obj,
+        const std::vector<int> &ids) const;
 
     /**
      * @internal
@@ -369,20 +362,6 @@ class MachineTopology {
     void load_objects(hwloc_obj_type_t type,
                       std::vector<io_obj_info> &vector) const;
 
-    /**
-     * @internal
-     *
-     * Initialize the topology object.
-     */
-    hwloc_topology *init_topology();
-
-    /**
-     * @internal
-     *
-     * Initialize the bitmap.
-     */
-    hwloc_bitmap_s *init_bitmap();
-
     /**
      *
      * @internal
diff --git a/test_install/CMakeLists.txt b/test_install/CMakeLists.txt
index b400f7a7681..88d41bec448 100644
--- a/test_install/CMakeLists.txt
+++ b/test_install/CMakeLists.txt
@@ -24,11 +24,6 @@ add_executable(test_install test_install.cpp)
 target_compile_features(test_install PUBLIC cxx_std_14)
 target_link_libraries(test_install PRIVATE Ginkgo::ginkgo)
 
-if(GINKGO_HAVE_HWLOC)
-    find_package(HWLOC REQUIRED)
-    target_link_libraries(test_install PRIVATE hwloc)
-endif()
-
 if(GINKGO_BUILD_CUDA)
     enable_language(CUDA)
     if(MSVC)
@@ -40,9 +35,6 @@ if(GINKGO_BUILD_CUDA)
     endif()
     add_executable(test_install_cuda test_install_cuda.cu)
     target_link_libraries(test_install_cuda PRIVATE Ginkgo::ginkgo)
-    if(GINKGO_HAVE_HWLOC)
-        target_link_libraries(test_install_cuda PRIVATE hwloc)
-    endif()
 endif()
 
 if(GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc"

From 3b911421e3c681abd533056ddc73c8694965fe2e Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 21 Jan 2021 12:01:21 +0100
Subject: [PATCH 0302/1008] Dpcpp and some other review minor updates.

+ Add max threads per block to max_workitem_sizes for HIP and CUDA

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 core/devices/cuda/executor.cpp              |  4 ++--
 core/devices/hip/executor.cpp               |  4 ++--
 cuda/base/executor.cpp                      |  6 +++++
 cuda/test/base/cuda_executor_topology.cu    |  3 +--
 dpcpp/base/executor.dp.cpp                  |  6 ++++-
 dpcpp/test/base/executor_topology.dp.cpp    |  7 +++---
 hip/base/executor.hip.cpp                   |  6 +++++
 hip/test/base/hip_executor.hip.cpp          |  6 -----
 hip/test/base/hip_executor_topology.hip.cpp |  3 +--
 include/ginkgo/core/base/executor.hpp       | 26 +++++++++------------
 10 files changed, 38 insertions(+), 33 deletions(-)

diff --git a/core/devices/cuda/executor.cpp b/core/devices/cuda/executor.cpp
index 7c5bb7f8dd4..c1efe6e7d40 100644
--- a/core/devices/cuda/executor.cpp
+++ b/core/devices/cuda/executor.cpp
@@ -50,14 +50,14 @@ std::shared_ptr<const Executor> CudaExecutor::get_master() const noexcept
 
 bool CudaExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
 {
-    return this->get_exec_info().device_id == dest_exec->get_device_id();
+    return this->get_device_id() == dest_exec->get_device_id();
 }
 
 
 bool CudaExecutor::verify_memory_to(const HipExecutor *dest_exec) const
 {
 #if GINKGO_HIP_PLATFORM_NVCC
-    return this->get_exec_info().device_id == dest_exec->get_device_id();
+    return this->get_device_id() == dest_exec->get_device_id();
 #else
     return false;
 #endif
diff --git a/core/devices/hip/executor.cpp b/core/devices/hip/executor.cpp
index 5020228b133..31325fcd15f 100644
--- a/core/devices/hip/executor.cpp
+++ b/core/devices/hip/executor.cpp
@@ -47,14 +47,14 @@ std::shared_ptr<const Executor> HipExecutor::get_master() const noexcept
 
 bool HipExecutor::verify_memory_to(const HipExecutor *dest_exec) const
 {
-    return this->get_exec_info().device_id == dest_exec->get_device_id();
+    return this->get_device_id() == dest_exec->get_device_id();
 }
 
 
 bool HipExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
 {
 #if GINKGO_HIP_PLATFORM_NVCC
-    return this->get_exec_info().device_id == dest_exec->get_device_id();
+    return this->get_device_id() == dest_exec->get_device_id();
 #else
     return false;
 #endif
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index c49438f055c..067ffc05db0 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -225,6 +225,12 @@ void CudaExecutor::set_gpu_property()
         GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
             &this->get_exec_info().num_computing_units,
             cudaDevAttrMultiProcessorCount, this->get_device_id()));
+        auto max_threads_per_block = 0;
+        GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
+            &max_threads_per_block, cudaDevAttrMaxThreadsPerBlock,
+            this->get_device_id()));
+        this->get_exec_info().max_workitem_sizes.push_back(
+            max_threads_per_block);
         this->get_exec_info().num_pe_per_cu =
             convert_sm_ver_to_cores(this->get_exec_info().major,
                                     this->get_exec_info().minor) /
diff --git a/cuda/test/base/cuda_executor_topology.cu b/cuda/test/base/cuda_executor_topology.cu
index f5c2bce2d8d..5e0d0fb4965 100644
--- a/cuda/test/base/cuda_executor_topology.cu
+++ b/cuda/test/base/cuda_executor_topology.cu
@@ -136,10 +136,9 @@ TEST_F(CudaExecutor, CanBindToCores)
 }
 
 
-TEST_F(CudaExecutor, KnowsItsClosestCpus)
+TEST_F(CudaExecutor, ClosestCpusIsPopulated)
 {
     cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
-    auto cpu_sys = sched_getcpu();
     auto close_cpus0 = cuda->get_closest_pus();
 
     ASSERT_NE(close_cpus0[0], -1);
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 91c418d6608..05b62c1f306 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -93,7 +93,11 @@ std::shared_ptr<DpcppExecutor> DpcppExecutor::create(
 }
 
 
-void DpcppExecutor::populate_exec_info(const MachineTopology *mach_topo) {}
+void DpcppExecutor::populate_exec_info(const MachineTopology *mach_topo)
+{
+    // Closest CPUs, NUMA node can be updated when there is a way to identify
+    // the device itself, which is currently not available with DPC++.
+}
 
 
 void DpcppExecutor::raw_free(void *ptr) const noexcept
diff --git a/dpcpp/test/base/executor_topology.dp.cpp b/dpcpp/test/base/executor_topology.dp.cpp
index 98406f8ae7d..c30bb4adc7d 100644
--- a/dpcpp/test/base/executor_topology.dp.cpp
+++ b/dpcpp/test/base/executor_topology.dp.cpp
@@ -81,12 +81,13 @@ class DpcppExecutor : public ::testing::Test {
 };
 
 
-TEST_F(DpcppExecutor, CanInstantiateTwoExecutorsOnOneDevice)
+TEST_F(DpcppExecutor, CanGetExecInfo)
 {
     dpcpp = gko::DpcppExecutor::create(0, omp);
-    dpcpp2 = gko::DpcppExecutor::create(0, omp);
 
-    // We want automatic deinitialization to not create any error
+    auto exec_info = dpcpp->get_exec_info();
+
+    ASSERT_TRUE(exec_info.max_workitem_sizes.size() > 0);
 }
 
 
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index ede895b47f3..9c2f05ae9eb 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -226,6 +226,12 @@ void HipExecutor::set_gpu_property()
         GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
             &this->get_exec_info().minor,
             hipDeviceAttributeComputeCapabilityMinor, this->get_device_id()));
+        auto max_threads_per_block = 0;
+        GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
+            &max_threads_per_block, hipDeviceAttributeMaxThreadsPerBlock,
+            this->get_device_id()));
+        this->get_exec_info().max_workitem_sizes.push_back(
+            max_threads_per_block);
 #if GINKGO_HIP_PLATFORM_NVCC
         this->get_exec_info().num_pe_per_cu =
             convert_sm_ver_to_cores(this->get_exec_info().major,
diff --git a/hip/test/base/hip_executor.hip.cpp b/hip/test/base/hip_executor.hip.cpp
index db829c4bf92..c4f4757e397 100644
--- a/hip/test/base/hip_executor.hip.cpp
+++ b/hip/test/base/hip_executor.hip.cpp
@@ -40,15 +40,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <memory>
-#include <thread>
 #include <type_traits>
 
 
-#if defined(__unix__) || defined(__APPLE__)
-#include <utmpx.h>
-#endif
-
-
 #include <gtest/gtest.h>
 
 
diff --git a/hip/test/base/hip_executor_topology.hip.cpp b/hip/test/base/hip_executor_topology.hip.cpp
index 826d464bcc3..8afd9e5640f 100644
--- a/hip/test/base/hip_executor_topology.hip.cpp
+++ b/hip/test/base/hip_executor_topology.hip.cpp
@@ -141,10 +141,9 @@ TEST_F(HipExecutor, CanBindToCores)
 }
 
 
-TEST_F(HipExecutor, KnowsItsClosestCpus)
+TEST_F(HipExecutor, ClosestCpusIsPopulated)
 {
     hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
-    auto cpu_sys = sched_getcpu();
     auto close_cpus0 = hip->get_closest_pus();
 
     ASSERT_NE(close_cpus0[0], -1);
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index c1ecd1f0cbf..a7a4efa934a 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -728,7 +728,7 @@ class Executor : public log::EnableLogging<Executor> {
         /**
          * The processing units closest to the device.
          *
-         * @note Only relevant for CUDA, HIP and DPCPP executors.
+         * @note Currently only relevant for CUDA, HIP and DPCPP executors.
          */
         std::vector<int> closest_pu_ids{};
     };
@@ -1058,6 +1058,7 @@ class EnableBinding {
     }
 };
 
+
 /**
  * Controls whether the DeviceReset function should be called thanks to a
  * boolean. Note that in any case, `DeviceReset` is called only after destroying
@@ -1685,7 +1686,10 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      *
      * @return the DPCPP device id of the device associated to this executor
      */
-    int get_device_id() const noexcept { return get_exec_info().device_id; }
+    int get_device_id() const noexcept
+    {
+        return this->get_exec_info().device_id;
+    }
 
     ::cl::sycl::queue *get_queue() const { return queue_.get(); }
 
@@ -1705,7 +1709,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      */
     const std::vector<int> &get_subgroup_sizes() const noexcept
     {
-        return get_exec_info().subgroup_sizes;
+        return this->get_exec_info().subgroup_sizes;
     }
 
     /**
@@ -1715,7 +1719,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      */
     size_type get_num_computing_units() const noexcept
     {
-        return get_exec_info().num_computing_units;
+        return this->get_exec_info().num_computing_units;
     }
 
     /**
@@ -1725,7 +1729,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      */
     const std::vector<int> &get_max_workitem_sizes() const noexcept
     {
-        return get_exec_info().max_workitem_sizes;
+        return this->get_exec_info().max_workitem_sizes;
     }
 
     /**
@@ -1735,7 +1739,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      */
     size_type get_max_workgroup_size() const noexcept
     {
-        return get_exec_info().num_pe_per_cu;
+        return this->get_exec_info().num_pe_per_cu;
     }
 
     /**
@@ -1745,7 +1749,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      */
     std::string get_device_type() const noexcept
     {
-        return get_exec_info().device_type;
+        return this->get_exec_info().device_type;
     }
 
 protected:
@@ -1783,14 +1787,6 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
 private:
     std::shared_ptr<Executor> master_;
 
-    // struct dpcpp_exec_info : public exec_info {
-    //     int device_id;
-    //     std::string device_type;
-    //     int num_computing_units;
-    //     std::vector<size_type> subgroup_sizes{};
-    //     std::vector<size_type> max_workitem_sizes{};
-    // } get_exec_info();
-
     template <typename T>
     using queue_manager = std::unique_ptr<T, std::function<void(T *)>>;
     queue_manager<::cl::sycl::queue> queue_;

From 058759c1735ff60e5439e205ce67ceb33c013e77 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Fri, 22 Jan 2021 11:46:53 +0100
Subject: [PATCH 0303/1008] Review updates.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

+ Make some functions in MachineTopology private.
+ Update types of loop variables.

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 core/device_hooks/dpcpp_hooks.cpp             |  8 +--
 core/devices/machine_topology.cpp             | 13 ++---
 cuda/base/executor.cpp                        |  3 +-
 hip/base/executor.hip.cpp                     |  2 +-
 include/ginkgo/core/base/executor.hpp         |  4 +-
 include/ginkgo/core/base/machine_topology.hpp | 57 +++++++------------
 6 files changed, 35 insertions(+), 52 deletions(-)

diff --git a/core/device_hooks/dpcpp_hooks.cpp b/core/device_hooks/dpcpp_hooks.cpp
index 2fdaa5de3bc..65ca04dd840 100644
--- a/core/device_hooks/dpcpp_hooks.cpp
+++ b/core/device_hooks/dpcpp_hooks.cpp
@@ -129,15 +129,15 @@ void DpcppExecutor::set_device_property() {}
 bool DpcppExecutor::verify_memory_to(const OmpExecutor *dest_exec) const
 {
     // Dummy check
-    return this->get_exec_info().device_type == "cpu" ||
-           this->get_exec_info().device_type == "host";
+    return this->get_device_type() == "cpu" ||
+           this->get_device_type() == "host";
 }
 
 bool DpcppExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
 {
     // Dummy check
-    return dest_exec->get_device_type() == this->get_exec_info().device_type &&
-           dest_exec->get_device_id() == this->get_exec_info().device_id;
+    return dest_exec->get_device_type() == this->get_device_type() &&
+           dest_exec->get_device_id() == this->get_device_id();
 }
 
 
diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index 7f24593e496..a91ad5cabdd 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -46,13 +46,10 @@ namespace detail {
 
 class topo_bitmap {
 public:
+    using bitmap_type = hwloc_bitmap_s;
 #if GKO_HAVE_HWLOC
-    using bitmap_type =
-        std::remove_pointer<decltype(hwloc_bitmap_alloc())>::type;
     topo_bitmap() : bitmap(hwloc_bitmap_alloc()) {}
     ~topo_bitmap() { hwloc_bitmap_free(bitmap); }
-#else
-    using bitmap_type = void;
 #endif
     bitmap_type *get() { return bitmap; }
 
@@ -88,7 +85,7 @@ hwloc_topology *init_topology()
 const MachineTopology::io_obj_info *MachineTopology::get_pci_device(
     const std::string &pci_bus_id) const
 {
-    for (auto id = 0; id < this->pci_devices_.size(); ++id) {
+    for (size_type id = 0; id < this->pci_devices_.size(); ++id) {
         if (this->pci_devices_[id].pci_bus_id.compare(0, 12, pci_bus_id, 0,
                                                       12) == 0) {
             return &this->pci_devices_[id];
@@ -136,7 +133,7 @@ void MachineTopology::hwloc_binding_helper(
     auto num_ids = bind_ids.size();
     auto id = bind_ids.data();
     // Set the given ids to a bitmap
-    for (auto i = 0; i < num_ids; ++i) {
+    for (size_type i = 0; i < num_ids; ++i) {
         GKO_ASSERT(id[i] < obj.size());
         GKO_ASSERT(id[i] >= 0);
         hwloc_bitmap_set(bitmap_toset.get(), obj[id[i]].os_id);
@@ -174,7 +171,7 @@ inline int MachineTopology::get_obj_id_by_os_index(
     size_type os_index) const
 {
 #if GKO_HAVE_HWLOC
-    for (auto id = 0; id < objects.size(); ++id) {
+    for (size_type id = 0; id < objects.size(); ++id) {
         if (objects[id].os_id == os_index) {
             return id;
         }
@@ -189,7 +186,7 @@ inline int MachineTopology::get_obj_id_by_gp_index(
     size_type gp_index) const
 {
 #if GKO_HAVE_HWLOC
-    for (auto id = 0; id < objects.size(); ++id) {
+    for (size_type id = 0; id < objects.size(); ++id) {
         if (objects[id].gp_id == gp_index) {
             return id;
         }
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 067ffc05db0..45a18fe5832 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -48,6 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "cuda/base/cusparse_handle.hpp"
 #include "cuda/base/device_guard.hpp"
 
+
 namespace gko {
 
 
@@ -82,7 +83,7 @@ void CudaExecutor::populate_exec_info(const MachineTopology *mach_topo)
         auto cuda_hwloc_obj =
             mach_topo->get_pci_device(this->get_exec_info().pci_bus_id);
         if (cuda_hwloc_obj) {
-            this->get_exec_info().numa_node = cuda_hwloc_obj->numa;
+            this->get_exec_info().numa_node = cuda_hwloc_obj->closest_numa;
             this->get_exec_info().closest_pu_ids =
                 cuda_hwloc_obj->closest_pu_ids;
         }
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 9c2f05ae9eb..0128faf2b33 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -83,7 +83,7 @@ void HipExecutor::populate_exec_info(const MachineTopology *mach_topo)
         auto hip_hwloc_obj =
             mach_topo->get_pci_device(this->get_exec_info().pci_bus_id);
         if (hip_hwloc_obj) {
-            this->get_exec_info().numa_node = hip_hwloc_obj->numa;
+            this->get_exec_info().numa_node = hip_hwloc_obj->closest_numa;
             this->get_exec_info().closest_pu_ids =
                 hip_hwloc_obj->closest_pu_ids;
         }
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index a7a4efa934a..c55e8f5fedc 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -1267,7 +1267,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         int device_id, std::shared_ptr<Executor> master,
         bool device_reset = false);
 
-    ~CudaExecutor() { decrease_num_execs(this->get_exec_info().device_id); }
+    ~CudaExecutor() { decrease_num_execs(this->get_device_id()); }
 
     std::shared_ptr<Executor> get_master() noexcept override;
 
@@ -1470,7 +1470,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
                                                std::shared_ptr<Executor> master,
                                                bool device_reset = false);
 
-    ~HipExecutor() { decrease_num_execs(this->get_exec_info().device_id); }
+    ~HipExecutor() { decrease_num_execs(this->get_device_id()); }
 
     std::shared_ptr<Executor> get_master() noexcept override;
 
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index 3ede524c131..4a74abefb67 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -82,16 +82,15 @@ namespace gko {
  * (https://www.open-mpi.org/projects/hwloc/doc/) for more detailed
  * information on topology detection and binding interfaces.
  *
- * @note A global object of MachineTopology type is created when first needed
- * and only destroyed at the end of the program. This means that any subsequent
- * queries will be from the same global object and hence use an extra atomic
- * read.
+ * @note A global object of MachineTopology type is created in a thread safe
+ *       manner and only destroyed at the end of the program. This means that
+ *       any subsequent queries will be from the same global object and hence
+ *       use an extra atomic read.
  */
 class MachineTopology {
     template <typename T>
     using hwloc_manager = std::unique_ptr<T, std::function<void(T *)>>;
 
-private:
     /**
      * This struct holds the attributes for a normal non-IO object.
      */
@@ -106,7 +105,7 @@ class MachineTopology {
          * proximity).
          *
          * @note Use this rather than os_id for all purposes other than binding.
-         * [Reference](https://www.open-mpi.org/projects/hwloc/doc/v2.4.0/a00364.php#faq_indexes)
+         *       [Reference](https://www.open-mpi.org/projects/hwloc/doc/v2.4.0/a00364.php#faq_indexes)
          */
         size_type logical_id;
 
@@ -175,7 +174,7 @@ class MachineTopology {
         /**
          * The closest numa.
          */
-        int numa;
+        int closest_numa;
 
         /**
          * The non-io parent object.
@@ -215,16 +214,6 @@ class MachineTopology {
         return &instance;
     }
 
-    /**
-     * Do not allow the MachineTopology object to be copied/moved. There should
-     * be only one global object per execution.
-     */
-    MachineTopology(MachineTopology &) = delete;
-    MachineTopology(MachineTopology &&) = delete;
-    MachineTopology &operator=(MachineTopology &) = delete;
-    MachineTopology &operator=(MachineTopology &&) = delete;
-    ~MachineTopology() = default;
-
     /**
      * Bind the object associated with the ids to a core.
      *
@@ -317,20 +306,6 @@ class MachineTopology {
      */
     size_type get_num_numas() const { return this->num_numas_; }
 
-protected:
-    /**
-     * Creates a new MachineTopology object.
-     */
-    static std::shared_ptr<MachineTopology> create()
-    {
-        return std::shared_ptr<MachineTopology>(new MachineTopology());
-    }
-
-    /**
-     * The default constructor that loads the different objects.
-     */
-    MachineTopology();
-
     /**
      * @internal
      *
@@ -368,8 +343,8 @@ class MachineTopology {
      *
      * Get object id from the os index
      */
-    inline int get_obj_id_by_os_index(
-        const std::vector<normal_obj_info> &objects, size_type os_index) const;
+    int get_obj_id_by_os_index(const std::vector<normal_obj_info> &objects,
+                               size_type os_index) const;
 
     /**
      *
@@ -377,11 +352,21 @@ class MachineTopology {
      *
      * Get object id from the hwloc index
      */
-    inline int get_obj_id_by_gp_index(
-        const std::vector<normal_obj_info> &objects, size_type gp_index) const;
-
+    int get_obj_id_by_gp_index(const std::vector<normal_obj_info> &objects,
+                               size_type gp_index) const;
 
 private:
+    /**
+     * Do not allow the MachineTopology object to be copied/moved. There should
+     * be only one global object per execution.
+     */
+    MachineTopology();
+    MachineTopology(MachineTopology &) = delete;
+    MachineTopology(MachineTopology &&) = delete;
+    MachineTopology &operator=(MachineTopology &) = delete;
+    MachineTopology &operator=(MachineTopology &&) = delete;
+    ~MachineTopology() = default;
+
     std::vector<normal_obj_info> pus_;
     std::vector<normal_obj_info> cores_;
     std::vector<normal_obj_info> packages_;

From a8bf3ea6f7865233578ad9c73e6b966d91fbdc36 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Mon, 25 Jan 2021 00:34:32 +0100
Subject: [PATCH 0304/1008] Review update.

+ Add singlify as an optional parameter.
+ Use cores 1 and 3 for binding tests.
+ Clarify docs.

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
---
 core/devices/machine_topology.cpp             |  8 +--
 core/test/base/executor.cpp                   | 12 ++---
 cuda/test/base/cuda_executor_topology.cu      | 12 ++---
 hip/test/base/hip_executor_topology.hip.cpp   | 12 ++---
 include/ginkgo/core/base/executor.hpp         | 51 +++++++++++++------
 include/ginkgo/core/base/machine_topology.hpp | 37 ++++++++++----
 third_party/CMakeLists.txt                    |  2 +-
 7 files changed, 86 insertions(+), 48 deletions(-)

diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index a91ad5cabdd..09fcab1cd86 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -126,7 +126,7 @@ MachineTopology::MachineTopology()
 
 void MachineTopology::hwloc_binding_helper(
     const std::vector<MachineTopology::normal_obj_info> &obj,
-    const std::vector<int> &bind_ids) const
+    const std::vector<int> &bind_ids, const bool singlify) const
 {
 #if GKO_HAVE_HWLOC
     detail::topo_bitmap bitmap_toset;
@@ -139,8 +139,10 @@ void MachineTopology::hwloc_binding_helper(
         hwloc_bitmap_set(bitmap_toset.get(), obj[id[i]].os_id);
     }
 
-    // Singlify to reduce expensive migrations.
-    hwloc_bitmap_singlify(bitmap_toset.get());
+    // Singlify to reduce expensive migrations, if asked for.
+    if (singlify) {
+        hwloc_bitmap_singlify(bitmap_toset.get());
+    }
     hwloc_set_cpubind(this->topo_.get(), bitmap_toset.get(), 0);
 #endif
 }
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 8c9a377c2f9..39ea82f6b6d 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -181,12 +181,11 @@ inline int get_os_id(int log_id)
 }
 
 
-TEST(OmpExecutor, CanBindToASpecificCore)
+TEST(MachineTopology, CanBindToASpecificCore)
 {
-    auto omp = gko::OmpExecutor::create();
     auto cpu_sys = sched_getcpu();
 
-    const int bind_core = 4;
+    const int bind_core = 3;
     gko::MachineTopology::get_instance()->bind_to_cores(
         std::vector<int>{bind_core});
 
@@ -195,16 +194,15 @@ TEST(OmpExecutor, CanBindToASpecificCore)
 }
 
 
-TEST(OmpExecutor, CanBindToARangeofCores)
+TEST(MachineTopology, CanBindToARangeofCores)
 {
-    auto omp = gko::OmpExecutor::create();
     auto cpu_sys = sched_getcpu();
 
-    const std::vector<int> bind_core = {6, 3};
+    const std::vector<int> bind_core = {1, 3};
     gko::MachineTopology::get_instance()->bind_to_cores(bind_core);
 
     cpu_sys = sched_getcpu();
-    ASSERT_TRUE(cpu_sys == get_os_id(3) || cpu_sys == get_os_id(6));
+    ASSERT_TRUE(cpu_sys == get_os_id(3) || cpu_sys == get_os_id(1));
 }
 
 
diff --git a/cuda/test/base/cuda_executor_topology.cu b/cuda/test/base/cuda_executor_topology.cu
index 5e0d0fb4965..1449b233772 100644
--- a/cuda/test/base/cuda_executor_topology.cu
+++ b/cuda/test/base/cuda_executor_topology.cu
@@ -104,11 +104,11 @@ TEST_F(CudaExecutor, CanBindToSinglePu)
 {
     cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
 
-    const int bind_pu = 6;
+    const int bind_pu = 1;
     cuda->bind_to_pu(bind_pu);
 
     auto cpu_sys = sched_getcpu();
-    ASSERT_TRUE(cpu_sys == get_cpu_os_id(6));
+    ASSERT_TRUE(cpu_sys == get_cpu_os_id(1));
 }
 
 
@@ -116,11 +116,11 @@ TEST_F(CudaExecutor, CanBindToPus)
 {
     cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
 
-    std::vector<int> bind_pus = {6, 3};
+    std::vector<int> bind_pus = {1, 3};
     cuda->bind_to_pus(bind_pus);
 
     auto cpu_sys = sched_getcpu();
-    ASSERT_TRUE(cpu_sys == get_cpu_os_id(3) || cpu_sys == get_cpu_os_id(6));
+    ASSERT_TRUE(cpu_sys == get_cpu_os_id(3) || cpu_sys == get_cpu_os_id(1));
 }
 
 
@@ -128,11 +128,11 @@ TEST_F(CudaExecutor, CanBindToCores)
 {
     cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
 
-    std::vector<int> bind_cores = {6, 3};
+    std::vector<int> bind_cores = {1, 3};
     cuda->bind_to_cores(bind_cores);
 
     auto cpu_sys = sched_getcpu();
-    ASSERT_TRUE(cpu_sys == get_core_os_id(3) || cpu_sys == get_core_os_id(6));
+    ASSERT_TRUE(cpu_sys == get_core_os_id(3) || cpu_sys == get_core_os_id(1));
 }
 
 
diff --git a/hip/test/base/hip_executor_topology.hip.cpp b/hip/test/base/hip_executor_topology.hip.cpp
index 8afd9e5640f..41c2a4173e7 100644
--- a/hip/test/base/hip_executor_topology.hip.cpp
+++ b/hip/test/base/hip_executor_topology.hip.cpp
@@ -109,11 +109,11 @@ TEST_F(HipExecutor, CanBindToSinglePu)
 {
     hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
 
-    const int bind_pu = 6;
+    const int bind_pu = 1;
     hip->bind_to_pu(bind_pu);
 
     auto cpu_sys = sched_getcpu();
-    ASSERT_TRUE(cpu_sys == get_cpu_os_id(6));
+    ASSERT_TRUE(cpu_sys == get_cpu_os_id(1));
 }
 
 
@@ -121,11 +121,11 @@ TEST_F(HipExecutor, CanBindToPus)
 {
     hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
 
-    std::vector<int> bind_pus = {6, 3};
+    std::vector<int> bind_pus = {1, 3};
     hip->bind_to_pus(bind_pus);
 
     auto cpu_sys = sched_getcpu();
-    ASSERT_TRUE(cpu_sys == get_cpu_os_id(3) || cpu_sys == get_cpu_os_id(6));
+    ASSERT_TRUE(cpu_sys == get_cpu_os_id(3) || cpu_sys == get_cpu_os_id(1));
 }
 
 
@@ -133,11 +133,11 @@ TEST_F(HipExecutor, CanBindToCores)
 {
     hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
 
-    std::vector<int> bind_cores = {6, 3};
+    std::vector<int> bind_cores = {1, 3};
     hip->bind_to_cores(bind_cores);
 
     auto cpu_sys = sched_getcpu();
-    ASSERT_TRUE(cpu_sys == get_core_os_id(3) || cpu_sys == get_core_os_id(6));
+    ASSERT_TRUE(cpu_sys == get_core_os_id(3) || cpu_sys == get_core_os_id(1));
 }
 
 
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index c55e8f5fedc..23014982236 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -692,7 +692,7 @@ class Executor : public log::EnableLogging<Executor> {
          * The sizes of the subgroup for the executor.
          *
          * @note In CPU executors this is invalid.
-         *       In CUDA and HIP executors warp size.
+         *       In CUDA and HIP executors this is the warp size.
          *       In DPCPP, this is the subgroup sizes for the device associated
          *       with the dpcpp executor.
          */
@@ -702,9 +702,11 @@ class Executor : public log::EnableLogging<Executor> {
          * The sizes of the work items for the executor.
          *
          * @note In CPU executors this is invalid.
-         *       In CUDA and HIP executors this is invalid.
-         *       In DPCPP, this is the maximum sizes for the workitems for the
-         *       device associated with the dpcpp executor.
+         *       In CUDA and HIP executors this is the maximum number of threads
+         *       per block.
+         *       In DPCPP, this is the maximum number of workitems, in each
+         *       direction of the workgroup for the device associated with the
+         *       dpcpp executor.
          */
         std::vector<int> max_workitem_sizes{};
 
@@ -729,6 +731,11 @@ class Executor : public log::EnableLogging<Executor> {
          * The processing units closest to the device.
          *
          * @note Currently only relevant for CUDA, HIP and DPCPP executors.
+         *       [Definition from hwloc
+         *       documentation:](https://www.open-mpi.org/projects/hwloc/doc/v2.4.0/a00350.php)
+         *       "The smallest processing element that can be represented by a
+         *       hwloc object. It may be a single-core processor, a core of a
+         *       multicore processor, or a single thread in a SMT processor"
          */
         std::vector<int> closest_pu_ids{};
     };
@@ -834,7 +841,7 @@ class Executor : public log::EnableLogging<Executor> {
     exec_info &get_exec_info()
     {
         return const_cast<exec_info &>(
-            static_cast<const Executor &>(*this).get_exec_info());
+            const_cast<const Executor &>(*this).get_exec_info());
     }
 
     exec_info exec_info_;
@@ -1020,17 +1027,24 @@ class EnableBinding {
     /**
      * Bind to Processing units (PU)
      *
-     * @param ids  The ids to be bound to the object.
+     * @param ids  The ids of PUs to be bound to the calling process.
+     * @param singlify  The ids of PUs are singlified to prevent possibly
+     *                  expensive migrations by the OS. This means that the
+     *                  binding is performed for only one of the ids in the
+     *                  set of ids passed in.
+     *                  See hwloc doc for
+     *                  [singlify](https://www.open-mpi.org/projects/hwloc/doc/v2.4.0/a00175.php#gaa611a77c092e679246afdf9a60d5db8b)
      */
-    void bind_to_pus(const std::vector<int> &ids) const
+    void bind_to_pus(const std::vector<int> &ids,
+                     const bool singlify = true) const
     {
-        MachineTopology::get_instance()->bind_to_pus(ids);
+        MachineTopology::get_instance()->bind_to_pus(ids, singlify);
     }
 
     /**
      * Bind to a Processing unit (PU)
      *
-     * @param id  The id to be bound to the object.
+     * @param ids  The ids of PUs to be bound to the calling process.
      */
     void bind_to_pu(const int &id) const
     {
@@ -1038,19 +1052,26 @@ class EnableBinding {
     }
 
     /**
-     * Bind to cores
+     * Bind to a set of cores
      *
-     * @param ids  The ids to be bound to the calling object.
+     * @param ids  The ids of cores to be bound to the calling process.
+     * @param singlify  The ids of PUs are singlified to prevent possibly
+     *                  expensive migrations by the OS. This means that the
+     *                  binding is performed for only one of the ids in the
+     *                  set of ids passed in.
+     *                  See hwloc doc for
+     *                  [singlify](https://www.open-mpi.org/projects/hwloc/doc/v2.4.0/a00175.php#gaa611a77c092e679246afdf9a60d5db8b)
      */
-    void bind_to_cores(const std::vector<int> &ids) const
+    void bind_to_cores(const std::vector<int> &ids,
+                       const bool singlify = true) const
     {
-        MachineTopology::get_instance()->bind_to_cores(ids);
+        MachineTopology::get_instance()->bind_to_cores(ids, singlify);
     }
 
     /**
-     * Bind to core
+     * Bind to a single core
      *
-     * @param id  The id to be bound to the calling object.
+     * @param ids  The ids of the core to be bound to the calling process.
      */
     void bind_to_core(const int &id) const
     {
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index 4a74abefb67..b6a57053952 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -215,23 +215,39 @@ class MachineTopology {
     }
 
     /**
-     * Bind the object associated with the ids to a core.
+     * Bind the calling process to the CPU cores associated with
+     * the ids.
      *
-     * @param ids  The ids of the object to be bound.
+     * @param ids  The ids of cores to be bound.
+     * @param singlify  The ids of PUs are singlified to prevent possibly
+     *                  expensive migrations by the OS. This means that the
+     *                  binding is performed for only one of the ids in the
+     *                  set of ids passed in.
+     *                  See hwloc doc for
+     *                  [singlify](https://www.open-mpi.org/projects/hwloc/doc/v2.4.0/a00175.php#gaa611a77c092e679246afdf9a60d5db8b)
      */
-    void bind_to_cores(const std::vector<int> &ids) const
+    void bind_to_cores(const std::vector<int> &ids,
+                       const bool singlify = true) const
     {
-        hwloc_binding_helper(this->cores_, ids);
+        hwloc_binding_helper(this->cores_, ids, singlify);
     }
 
     /**
-     * Bind the object associated with the ids to a Processing unit(PU).
+     * Bind the calling process to PUs associated with
+     * the ids.
      *
-     * @param ids  The ids of the object to be bound.
+     * @param ids  The ids of PUs to be bound.
+     * @param singlify  The ids of PUs are singlified to prevent possibly
+     *                  expensive migrations by the OS. This means that the
+     *                  binding is performed for only one of the ids in the
+     *                  set of ids passed in.
+     *                  See hwloc doc for
+     *                  [singlify](https://www.open-mpi.org/projects/hwloc/doc/v2.4.0/a00175.php#gaa611a77c092e679246afdf9a60d5db8b)
      */
-    void bind_to_pus(const std::vector<int> &ids) const
+    void bind_to_pus(const std::vector<int> &ids,
+                     const bool singlify = true) const
     {
-        hwloc_binding_helper(this->pus_, ids);
+        hwloc_binding_helper(this->pus_, ids, singlify);
     }
 
     /**
@@ -309,11 +325,12 @@ class MachineTopology {
     /**
      * @internal
      *
-     * A helper function that binds the object with ids.
+     * A helper function that binds the calling process with the ids of `obj`
+     * object .
      */
     void hwloc_binding_helper(
         const std::vector<MachineTopology::normal_obj_info> &obj,
-        const std::vector<int> &ids) const;
+        const std::vector<int> &ids, const bool singlify = true) const;
 
     /**
      * @internal
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index 437508a123d..b4ca5067187 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -13,7 +13,7 @@ if(GINKGO_BUILD_TESTS)
     ginkgo_add_subdirectory(GTest gtest)
 endif()
 
-if(NOT GINKGO_FORCE_NO_HWLOC AND GINKGO_HAVE_HWLOC)
+if(NOT GINKGO_DISABLE_HWLOC AND GINKGO_HAVE_HWLOC)
     ginkgo_add_subdirectory(HWLOC hwloc)
 endif()
 

From a2c609ca5d59ae3fe57cce9e0c1bb97cde1c8e2f Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 26 Jan 2021 01:42:06 +0100
Subject: [PATCH 0305/1008] Review updates.

+ Remove Binding interface from Executors. Binding is now
  possible by calling member functions of the machine topology object.
+ Update exec_info objects to correct member abstractions.
+ Add tests for CUDA, HIP and DPCPP property getters.

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 core/devices/omp/executor.cpp                 |   2 +-
 cuda/base/executor.cpp                        |  20 +-
 cuda/test/base/cuda_executor.cu               |  31 +++-
 cuda/test/base/cuda_executor_topology.cu      |   9 +-
 dpcpp/base/executor.dp.cpp                    |  19 +-
 dpcpp/test/base/CMakeLists.txt                |   1 -
 dpcpp/test/base/executor.dp.cpp               |  16 +-
 dpcpp/test/base/executor_topology.dp.cpp      |  94 ----------
 hip/base/executor.hip.cpp                     |  20 +-
 hip/test/base/hip_executor.hip.cpp            |  35 +++-
 hip/test/base/hip_executor_topology.hip.cpp   |   9 +-
 include/ginkgo/core/base/executor.hpp         | 172 ++++++++----------
 include/ginkgo/core/base/machine_topology.hpp |  20 ++
 13 files changed, 224 insertions(+), 224 deletions(-)
 delete mode 100644 dpcpp/test/base/executor_topology.dp.cpp

diff --git a/core/devices/omp/executor.cpp b/core/devices/omp/executor.cpp
index 61b8dd02dca..3808d42969e 100644
--- a/core/devices/omp/executor.cpp
+++ b/core/devices/omp/executor.cpp
@@ -51,7 +51,7 @@ void OmpExecutor::populate_exec_info(const MachineTopology *mach_topo)
     auto num_pus =
         (mach_topo->get_num_pus() == 0 ? 1 : mach_topo->get_num_pus());
     this->get_exec_info().num_computing_units = num_cores;
-    this->get_exec_info().num_pe_per_cu = num_pus / num_cores;
+    this->get_exec_info().num_pu_per_cu = num_pus / num_cores;
 }
 
 
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 45a18fe5832..7f41fe87f3d 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -230,14 +230,24 @@ void CudaExecutor::set_gpu_property()
         GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
             &max_threads_per_block, cudaDevAttrMaxThreadsPerBlock,
             this->get_device_id()));
-        this->get_exec_info().max_workitem_sizes.push_back(
-            max_threads_per_block);
-        this->get_exec_info().num_pe_per_cu =
+        std::vector<int> max_threads_per_block_dim{3, 0};
+        GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
+            &max_threads_per_block_dim[0], cudaDevAttrMaxBlockDimX,
+            this->get_device_id()));
+        GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
+            &max_threads_per_block_dim[1], cudaDevAttrMaxBlockDimY,
+            this->get_device_id()));
+        GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
+            &max_threads_per_block_dim[2], cudaDevAttrMaxBlockDimZ,
+            this->get_device_id()));
+        this->get_exec_info().max_workgroup_size = max_threads_per_block;
+        this->get_exec_info().max_workitem_sizes = max_threads_per_block_dim;
+        this->get_exec_info().num_pu_per_cu =
             convert_sm_ver_to_cores(this->get_exec_info().major,
                                     this->get_exec_info().minor) /
             kernels::cuda::config::warp_size;
-        this->get_exec_info().subgroup_sizes.push_back(
-            kernels::cuda::config::warp_size);
+        this->get_exec_info().max_subgroup_size =
+            kernels::cuda::config::warp_size;
     }
 }
 
diff --git a/cuda/test/base/cuda_executor.cu b/cuda/test/base/cuda_executor.cu
index 99db90458ee..902514ee6c8 100644
--- a/cuda/test/base/cuda_executor.cu
+++ b/cuda/test/base/cuda_executor.cu
@@ -43,7 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 
-
+#include "common/base/executor.hpp.inc"
 #include "cuda/test/utils.hpp"
 
 
@@ -256,4 +256,33 @@ TEST_F(CudaExecutor, Synchronizes)
 }
 
 
+TEST_F(CudaExecutor, ExecInfoSetsCorrectProperties)
+{
+    auto dev_id = cuda->get_device_id();
+    auto num_sm = 0;
+    auto major = 0;
+    auto minor = 0;
+    auto max_threads_per_block = 0;
+    auto warp_size = 0;
+    GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
+        &num_sm, cudaDevAttrMultiProcessorCount, dev_id));
+    GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
+        &major, cudaDevAttrComputeCapabilityMajor, dev_id));
+    GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
+        &minor, cudaDevAttrComputeCapabilityMinor, dev_id));
+    GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
+        &max_threads_per_block, cudaDevAttrMaxThreadsPerBlock, dev_id));
+    GKO_ASSERT_NO_CUDA_ERRORS(
+        cudaDeviceGetAttribute(&warp_size, cudaDevAttrWarpSize, dev_id));
+    auto num_cores = convert_sm_ver_to_cores(major, minor);
+
+    ASSERT_EQ(cuda->get_major_version(), major);
+    ASSERT_EQ(cuda->get_minor_version(), minor);
+    ASSERT_EQ(cuda->get_num_multiprocessor(), num_sm);
+    ASSERT_EQ(cuda->get_warp_size(), warp_size);
+    ASSERT_EQ(cuda->get_num_warps(), num_sm * (num_cores / warp_size));
+    ASSERT_EQ(cuda->get_num_warps_per_sm(), num_cores / warp_size);
+}
+
+
 }  // namespace
diff --git a/cuda/test/base/cuda_executor_topology.cu b/cuda/test/base/cuda_executor_topology.cu
index 1449b233772..7e9eb4191da 100644
--- a/cuda/test/base/cuda_executor_topology.cu
+++ b/cuda/test/base/cuda_executor_topology.cu
@@ -105,7 +105,7 @@ TEST_F(CudaExecutor, CanBindToSinglePu)
     cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
 
     const int bind_pu = 1;
-    cuda->bind_to_pu(bind_pu);
+    gko::MachineTopology::get_instance()->bind_to_pu(bind_pu);
 
     auto cpu_sys = sched_getcpu();
     ASSERT_TRUE(cpu_sys == get_cpu_os_id(1));
@@ -117,7 +117,7 @@ TEST_F(CudaExecutor, CanBindToPus)
     cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
 
     std::vector<int> bind_pus = {1, 3};
-    cuda->bind_to_pus(bind_pus);
+    gko::MachineTopology::get_instance()->bind_to_pus(bind_pus);
 
     auto cpu_sys = sched_getcpu();
     ASSERT_TRUE(cpu_sys == get_cpu_os_id(3) || cpu_sys == get_cpu_os_id(1));
@@ -129,7 +129,7 @@ TEST_F(CudaExecutor, CanBindToCores)
     cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
 
     std::vector<int> bind_cores = {1, 3};
-    cuda->bind_to_cores(bind_cores);
+    gko::MachineTopology::get_instance()->bind_to_cores(bind_cores);
 
     auto cpu_sys = sched_getcpu();
     ASSERT_TRUE(cpu_sys == get_core_os_id(3) || cpu_sys == get_core_os_id(1));
@@ -148,8 +148,7 @@ TEST_F(CudaExecutor, ClosestCpusIsPopulated)
 TEST_F(CudaExecutor, KnowsItsNuma)
 {
     cuda = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
-    auto exec_info0 = cuda->get_exec_info();
-    auto numa0 = exec_info0.numa_node;
+    auto numa0 = cuda->get_closest_numa();
     auto close_cpu0 = cuda->get_closest_pus()[0];
 
     auto numa_sys0 = numa_node_of_cpu(get_cpu_os_id(close_cpu0));
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 05b62c1f306..9130fcf7ca5 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -217,17 +217,24 @@ void DpcppExecutor::set_device_property()
             GKO_NOT_SUPPORTED(device);
         }
     }
-    this->get_exec_info().num_computing_units =
-        device.get_info<sycl::info::device::max_compute_units>();
+    this->get_exec_info().num_computing_units = static_cast<int>(
+        device.get_info<sycl::info::device::max_compute_units>());
+    const auto subgroup_sizes = this->get_exec_info().subgroup_sizes;
+    if (subgroup_sizes.size()) {
+        this->get_exec_info().max_subgroup_size = static_cast<int>(
+            *std::max_element(subgroup_sizes.begin(), subgroup_sizes.end()));
+    }
+    this->get_exec_info().max_workgroup_size = static_cast<int>(
+        device.get_info<sycl::info::device::max_work_group_size>());
     auto max_workitem_sizes =
         device.get_info<sycl::info::device::max_work_item_sizes>();
-    // There is no way to get the dimension of a sycl::id object
-    for (std::size_t i = 0; i < 3; i++) {
+    // Get the max dimension of a sycl::id object
+    auto max_work_item_dimensions =
+        device.get_info<sycl::info::device::max_work_item_dimensions>();
+    for (uint32 i = 0; i < max_work_item_dimensions; i++) {
         this->get_exec_info().max_workitem_sizes.push_back(
             max_workitem_sizes[i]);
     }
-    this->get_exec_info().num_pe_per_cu =
-        device.get_info<sycl::info::device::max_work_group_size>();
     // Here we declare the queue with the property `in_order` which ensures the
     // kernels are executed in the submission order. Otherwise, calls to
     // `wait()` would be needed after every call to a DPC++ function or kernel.
diff --git a/dpcpp/test/base/CMakeLists.txt b/dpcpp/test/base/CMakeLists.txt
index 6d634f249eb..adb2f2505b5 100644
--- a/dpcpp/test/base/CMakeLists.txt
+++ b/dpcpp/test/base/CMakeLists.txt
@@ -1,2 +1 @@
 ginkgo_create_dpcpp_test(executor)
-ginkgo_create_dpcpp_test(executor_topology)
diff --git a/dpcpp/test/base/executor.dp.cpp b/dpcpp/test/base/executor.dp.cpp
index a83db327226..eb29ea6546c 100644
--- a/dpcpp/test/base/executor.dp.cpp
+++ b/dpcpp/test/base/executor.dp.cpp
@@ -76,8 +76,8 @@ class DpcppExecutor : public ::testing::Test {
     }
 
     std::shared_ptr<gko::Executor> omp;
-    std::shared_ptr<gko::DpcppExecutor> dpcpp;
-    std::shared_ptr<gko::DpcppExecutor> dpcpp2;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp2;
 };
 
 
@@ -90,6 +90,18 @@ TEST_F(DpcppExecutor, CanInstantiateTwoExecutorsOnOneDevice)
 }
 
 
+TEST_F(DpcppExecutor, CanGetExecInfo)
+{
+    dpcpp = gko::DpcppExecutor::create(0, omp);
+
+    ASSERT_TRUE(dpcpp->get_num_computing_units() > 0);
+    ASSERT_TRUE(dpcpp->get_subgroup_sizes().size() > 0);
+    ASSERT_TRUE(dpcpp->get_max_workitem_sizes().size() > 0);
+    ASSERT_TRUE(dpcpp->get_max_workgroup_size() > 0);
+    ASSERT_TRUE(dpcpp->get_max_subgroup_size() > 0);
+}
+
+
 TEST_F(DpcppExecutor, KnowsNumberOfDevicesOfTypeAll)
 {
     auto count = sycl::device::get_devices(sycl::info::device_type::all).size();
diff --git a/dpcpp/test/base/executor_topology.dp.cpp b/dpcpp/test/base/executor_topology.dp.cpp
deleted file mode 100644
index c30bb4adc7d..00000000000
--- a/dpcpp/test/base/executor_topology.dp.cpp
+++ /dev/null
@@ -1,94 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/base/executor.hpp>
-
-
-#include <exception>
-#include <memory>
-#include <type_traits>
-
-
-#include <CL/sycl.hpp>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/exception_helpers.hpp>
-
-
-namespace {
-
-
-class DpcppExecutor : public ::testing::Test {
-protected:
-    DpcppExecutor()
-        : omp(gko::OmpExecutor::create()), dpcpp(nullptr), dpcpp2(nullptr)
-    {}
-
-    void SetUp()
-    {
-        ASSERT_GT(gko::DpcppExecutor::get_num_devices("cpu"), 0);
-        dpcpp = gko::DpcppExecutor::create(0, omp, "cpu");
-        if (gko::DpcppExecutor::get_num_devices("gpu") > 0) {
-            dpcpp2 = gko::DpcppExecutor::create(0, omp, "gpu");
-        }
-    }
-
-    void TearDown()
-    {
-        if (dpcpp != nullptr) {
-            // ensure that previous calls finished and didn't throw an error
-            ASSERT_NO_THROW(dpcpp->synchronize());
-        }
-    }
-
-    std::shared_ptr<gko::Executor> omp;
-    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
-    std::shared_ptr<const gko::DpcppExecutor> dpcpp2;
-};
-
-
-TEST_F(DpcppExecutor, CanGetExecInfo)
-{
-    dpcpp = gko::DpcppExecutor::create(0, omp);
-
-    auto exec_info = dpcpp->get_exec_info();
-
-    ASSERT_TRUE(exec_info.max_workitem_sizes.size() > 0);
-}
-
-
-}  // namespace
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 0128faf2b33..706270aaa47 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -232,18 +232,30 @@ void HipExecutor::set_gpu_property()
             this->get_device_id()));
         this->get_exec_info().max_workitem_sizes.push_back(
             max_threads_per_block);
+        std::vector<int> max_threads_per_block_dim{3, 0};
+        GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
+            &max_threads_per_block_dim[0], hipDeviceAttributeMaxBlockDimX,
+            this->get_device_id()));
+        GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
+            &max_threads_per_block_dim[1], hipDeviceAttributeMaxBlockDimY,
+            this->get_device_id()));
+        GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
+            &max_threads_per_block_dim[2], hipDeviceAttributeMaxBlockDimZ,
+            this->get_device_id()));
+        this->get_exec_info().max_workgroup_size = max_threads_per_block;
+        this->get_exec_info().max_workitem_sizes = max_threads_per_block_dim;
 #if GINKGO_HIP_PLATFORM_NVCC
-        this->get_exec_info().num_pe_per_cu =
+        this->get_exec_info().num_pu_per_cu =
             convert_sm_ver_to_cores(this->get_exec_info().major,
                                     this->get_exec_info().minor) /
             kernels::hip::config::warp_size;
 #else
         // In GCN (Graphics Core Next), each multiprocessor has 4 SIMD
         // Reference: https://en.wikipedia.org/wiki/Graphics_Core_Next
-        this->get_exec_info().num_pe_per_cu = 4;
+        this->get_exec_info().num_pu_per_cu = 4;
 #endif  // GINKGO_HIP_PLATFORM_NVCC
-        this->get_exec_info().subgroup_sizes.push_back(
-            kernels::hip::config::warp_size);
+        this->get_exec_info().max_subgroup_size =
+            kernels::hip::config::warp_size;
     }
 }
 
diff --git a/hip/test/base/hip_executor.hip.cpp b/hip/test/base/hip_executor.hip.cpp
index c4f4757e397..dba2003450e 100644
--- a/hip/test/base/hip_executor.hip.cpp
+++ b/hip/test/base/hip_executor.hip.cpp
@@ -49,7 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 
-
+#include "common/base/executor.hpp.inc"
 #include "hip/test/utils.hip.hpp"
 
 
@@ -265,4 +265,37 @@ TEST_F(HipExecutor, Synchronizes)
 }
 
 
+TEST_F(HipExecutor, ExecInfoSetsCorrectProperties)
+{
+    auto dev_id = hip->get_device_id();
+    auto num_sm = 0;
+    auto major = 0;
+    auto minor = 0;
+    auto max_threads_per_block = 0;
+    auto warp_size = 0;
+    GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
+        &num_sm, hipDeviceAttributeMultiprocessorCount, dev_id));
+    GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
+        &major, hipDeviceAttributeComputeCapabilityMajor, dev_id));
+    GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
+        &minor, hipDeviceAttributeComputeCapabilityMinor, dev_id));
+    GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
+        &max_threads_per_block, hipDeviceAttributeMaxThreadsPerBlock, dev_id));
+    GKO_ASSERT_NO_HIP_ERRORS(
+        hipDeviceGetAttribute(&warp_size, hipDeviceAttributeWarpSize, dev_id));
+#if GINKGO_HIP_PLATFORM_NVCC
+    auto num_cores = convert_sm_ver_to_cores(major, minor);
+#else
+    auto num_cores = warp_size * 4;
+#endif
+
+    ASSERT_EQ(hip->get_major_version(), major);
+    ASSERT_EQ(hip->get_minor_version(), minor);
+    ASSERT_EQ(hip->get_num_multiprocessor(), num_sm);
+    ASSERT_EQ(hip->get_warp_size(), warp_size);
+    ASSERT_EQ(hip->get_num_warps(), num_sm * (num_cores / warp_size));
+    ASSERT_EQ(hip->get_num_warps_per_sm(), num_cores / warp_size);
+}
+
+
 }  // namespace
diff --git a/hip/test/base/hip_executor_topology.hip.cpp b/hip/test/base/hip_executor_topology.hip.cpp
index 41c2a4173e7..0f52ebb23d5 100644
--- a/hip/test/base/hip_executor_topology.hip.cpp
+++ b/hip/test/base/hip_executor_topology.hip.cpp
@@ -110,7 +110,7 @@ TEST_F(HipExecutor, CanBindToSinglePu)
     hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
 
     const int bind_pu = 1;
-    hip->bind_to_pu(bind_pu);
+    gko::MachineTopology::get_instance()->bind_to_pu(bind_pu);
 
     auto cpu_sys = sched_getcpu();
     ASSERT_TRUE(cpu_sys == get_cpu_os_id(1));
@@ -122,7 +122,7 @@ TEST_F(HipExecutor, CanBindToPus)
     hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
 
     std::vector<int> bind_pus = {1, 3};
-    hip->bind_to_pus(bind_pus);
+    gko::MachineTopology::get_instance()->bind_to_pus(bind_pus);
 
     auto cpu_sys = sched_getcpu();
     ASSERT_TRUE(cpu_sys == get_cpu_os_id(3) || cpu_sys == get_cpu_os_id(1));
@@ -134,7 +134,7 @@ TEST_F(HipExecutor, CanBindToCores)
     hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
 
     std::vector<int> bind_cores = {1, 3};
-    hip->bind_to_cores(bind_cores);
+    gko::MachineTopology::get_instance()->bind_to_cores(bind_cores);
 
     auto cpu_sys = sched_getcpu();
     ASSERT_TRUE(cpu_sys == get_core_os_id(3) || cpu_sys == get_core_os_id(1));
@@ -153,8 +153,7 @@ TEST_F(HipExecutor, ClosestCpusIsPopulated)
 TEST_F(HipExecutor, KnowsItsNuma)
 {
     hip = gko::HipExecutor::create(0, gko::OmpExecutor::create());
-    auto exec_info0 = hip->get_exec_info();
-    auto numa0 = exec_info0.numa_node;
+    auto numa0 = hip->get_closest_numa();
     auto close_cpu0 = hip->get_closest_pus();
 
     auto numa_sys0 = numa_node_of_cpu(get_cpu_os_id(close_cpu0[0]));
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 23014982236..ecd8dbc0cad 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -649,6 +649,7 @@ class Executor : public log::EnableLogging<Executor> {
         return this->verify_memory_from(other.get());
     }
 
+protected:
     /**
      * A struct that abstracts the executor info for different executors
      * classes.
@@ -680,36 +681,59 @@ class Executor : public log::EnableLogging<Executor> {
         int num_computing_units = -1;
 
         /**
-         * The number of processing elements per computing unit in the executor.
+         * The number of processing units per computing unit in the executor.
          *
          * @note In CPU executors this is equivalent to the number of SIMD units
-         *       per core. In CUDA and HIP executors this is the number of warps
-         *       per SM. In DPCPP, this is the max work group size.
+         *       per core.
+         *       In CUDA and HIP executors this is the number of warps
+         *       per SM.
+         *       In DPCPP, this is currently undefined.
          */
-        int num_pe_per_cu = -1;
+        int num_pu_per_cu = -1;
 
         /**
          * The sizes of the subgroup for the executor.
          *
          * @note In CPU executors this is invalid.
-         *       In CUDA and HIP executors this is the warp size.
+         *       In CUDA and HIP executors this is invalid.
          *       In DPCPP, this is the subgroup sizes for the device associated
          *       with the dpcpp executor.
          */
         std::vector<int> subgroup_sizes{};
 
+        /**
+         * The maximum subgroup size for the executor.
+         *
+         * @note In CPU executors this is invalid.
+         *       In CUDA and HIP executors this is the warp size.
+         *       In DPCPP, this is the maximum subgroup size for the device
+         *       associated with the dpcpp executor.
+         */
+        int max_subgroup_size = -1;
+
         /**
          * The sizes of the work items for the executor.
          *
          * @note In CPU executors this is invalid.
          *       In CUDA and HIP executors this is the maximum number of threads
-         *       per block.
+         *       in each dimension of a block (x, y, z).
          *       In DPCPP, this is the maximum number of workitems, in each
          *       direction of the workgroup for the device associated with the
          *       dpcpp executor.
          */
         std::vector<int> max_workitem_sizes{};
 
+        /**
+         * The sizes of the work items for the executor.
+         *
+         * @note In CPU executors this is invalid.
+         *       In CUDA and HIP executors this is the maximum number of threads
+         *       in block.
+         *       In DPCPP, this is the maximum number of workitems that are
+         *       permitted in a workgroup.
+         */
+        int max_workgroup_size;
+
         /**
          * The major version for CUDA/HIP device.
          */
@@ -728,7 +752,7 @@ class Executor : public log::EnableLogging<Executor> {
         std::string pci_bus_id = std::string(13, 'x');
 
         /**
-         * The processing units closest to the device.
+         * The host processing units closest to the device.
          *
          * @note Currently only relevant for CUDA, HIP and DPCPP executors.
          *       [Definition from hwloc
@@ -747,7 +771,6 @@ class Executor : public log::EnableLogging<Executor> {
      */
     const exec_info &get_exec_info() const { return this->exec_info_; }
 
-protected:
     /**
      * Allocates raw memory in this Executor.
      *
@@ -838,11 +861,7 @@ class Executor : public log::EnableLogging<Executor> {
      *
      * @return  the pointer to the exec_info object
      */
-    exec_info &get_exec_info()
-    {
-        return const_cast<exec_info &>(
-            const_cast<const Executor &>(*this).get_exec_info());
-    }
+    exec_info &get_exec_info() { return this->exec_info_; }
 
     exec_info exec_info_;
 
@@ -1015,71 +1034,6 @@ class ExecutorBase : public Executor {
 #undef GKO_DECLARE_EXECUTOR_FRIEND
 
 
-/**
- * This class provides some binding functions from the MachineTopology class and
- * the executors can derive from this class as necessary.
- *
- * The executor object can then be bound to the bind-able objects from the
- * machine topology by calling the member functions of this class.
- */
-class EnableBinding {
-public:
-    /**
-     * Bind to Processing units (PU)
-     *
-     * @param ids  The ids of PUs to be bound to the calling process.
-     * @param singlify  The ids of PUs are singlified to prevent possibly
-     *                  expensive migrations by the OS. This means that the
-     *                  binding is performed for only one of the ids in the
-     *                  set of ids passed in.
-     *                  See hwloc doc for
-     *                  [singlify](https://www.open-mpi.org/projects/hwloc/doc/v2.4.0/a00175.php#gaa611a77c092e679246afdf9a60d5db8b)
-     */
-    void bind_to_pus(const std::vector<int> &ids,
-                     const bool singlify = true) const
-    {
-        MachineTopology::get_instance()->bind_to_pus(ids, singlify);
-    }
-
-    /**
-     * Bind to a Processing unit (PU)
-     *
-     * @param ids  The ids of PUs to be bound to the calling process.
-     */
-    void bind_to_pu(const int &id) const
-    {
-        MachineTopology::get_instance()->bind_to_pus(std::vector<int>{id});
-    }
-
-    /**
-     * Bind to a set of cores
-     *
-     * @param ids  The ids of cores to be bound to the calling process.
-     * @param singlify  The ids of PUs are singlified to prevent possibly
-     *                  expensive migrations by the OS. This means that the
-     *                  binding is performed for only one of the ids in the
-     *                  set of ids passed in.
-     *                  See hwloc doc for
-     *                  [singlify](https://www.open-mpi.org/projects/hwloc/doc/v2.4.0/a00175.php#gaa611a77c092e679246afdf9a60d5db8b)
-     */
-    void bind_to_cores(const std::vector<int> &ids,
-                       const bool singlify = true) const
-    {
-        MachineTopology::get_instance()->bind_to_cores(ids, singlify);
-    }
-
-    /**
-     * Bind to a single core
-     *
-     * @param ids  The ids of the core to be bound to the calling process.
-     */
-    void bind_to_core(const int &id) const
-    {
-        MachineTopology::get_instance()->bind_to_cores(std::vector<int>{id});
-    }
-};
-
-
 /**
  * Controls whether the DeviceReset function should be called thanks to a
  * boolean. Note that in any case, `DeviceReset` is called only after destroying
@@ -1168,7 +1122,7 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
 
     int get_num_threads_per_core() const
     {
-        return this->get_exec_info().num_pe_per_cu;
+        return this->get_exec_info().num_pu_per_cu;
     }
 
 protected:
@@ -1237,7 +1191,7 @@ class ReferenceExecutor : public OmpExecutor {
     {
         this->get_exec_info().device_id = -1;
         this->get_exec_info().num_computing_units = 1;
-        this->get_exec_info().num_pe_per_cu = 1;
+        this->get_exec_info().num_pu_per_cu = 1;
     }
 
     bool verify_memory_from(const Executor *src_exec) const override
@@ -1272,7 +1226,6 @@ using DefaultExecutor = ReferenceExecutor;
  */
 class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
                      public std::enable_shared_from_this<CudaExecutor>,
-                     public detail::EnableBinding,
                      public detail::EnableDeviceReset {
     friend class detail::ExecutorBase<CudaExecutor>;
 
@@ -1316,7 +1269,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
      */
     int get_num_warps_per_sm() const noexcept
     {
-        return this->get_exec_info().num_pe_per_cu;
+        return this->get_exec_info().num_pu_per_cu;
     }
 
     /**
@@ -1333,7 +1286,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     int get_num_warps() const noexcept
     {
         return this->get_exec_info().num_computing_units *
-               this->get_exec_info().num_pe_per_cu;
+               this->get_exec_info().num_pu_per_cu;
     }
 
     /**
@@ -1341,9 +1294,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
      */
     int get_warp_size() const noexcept
     {
-        return (this->get_exec_info().subgroup_sizes.size()
-                    ? this->get_exec_info().subgroup_sizes[0]
-                    : -1);
+        return this->get_exec_info().max_subgroup_size;
     }
 
     /**
@@ -1389,6 +1340,13 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         return this->get_exec_info().closest_pu_ids;
     }
 
+    /**
+     * Get the closest NUMA node
+     *
+     * @return  the closest NUMA node closest to this device
+     */
+    int get_closest_numa() const { return this->get_exec_info().numa_node; }
+
 protected:
     void set_gpu_property();
 
@@ -1400,10 +1358,11 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     {
         this->get_exec_info().device_id = device_id;
         this->get_exec_info().num_computing_units = 0;
-        this->get_exec_info().num_pe_per_cu = 0;
+        this->get_exec_info().num_pu_per_cu = 0;
         this->CudaExecutor::populate_exec_info(MachineTopology::get_instance());
         if (this->get_exec_info().closest_pu_ids.size()) {
-            this->bind_to_pus(this->get_exec_info().closest_pu_ids);
+            MachineTopology::get_instance()->bind_to_pus(
+                this->get_closest_pus());
         }
         this->set_gpu_property();
         this->init_handles();
@@ -1475,7 +1434,6 @@ using DefaultExecutor = CudaExecutor;
  */
 class HipExecutor : public detail::ExecutorBase<HipExecutor>,
                     public std::enable_shared_from_this<HipExecutor>,
-                    public detail::EnableBinding,
                     public detail::EnableDeviceReset {
     friend class detail::ExecutorBase<HipExecutor>;
 
@@ -1519,7 +1477,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
      */
     int get_num_warps_per_sm() const noexcept
     {
-        return this->get_exec_info().num_pe_per_cu;
+        return this->get_exec_info().num_pu_per_cu;
     }
 
     /**
@@ -1552,7 +1510,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     int get_num_warps() const noexcept
     {
         return this->get_exec_info().num_computing_units *
-               this->get_exec_info().num_pe_per_cu;
+               this->get_exec_info().num_pu_per_cu;
     }
 
     /**
@@ -1560,9 +1518,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
      */
     int get_warp_size() const noexcept
     {
-        return (this->get_exec_info().subgroup_sizes.size()
-                    ? this->get_exec_info().subgroup_sizes[0]
-                    : -1);
+        return this->get_exec_info().max_subgroup_size;
     }
 
     /**
@@ -1582,6 +1538,13 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         return hipsparse_handle_.get();
     }
 
+    /**
+     * Get the closest NUMA node
+     *
+     * @return  the closest NUMA node closest to this device
+     */
+    int get_closest_numa() const { return this->get_exec_info().numa_node; }
+
     /**
      * Get the closest PUs
      *
@@ -1603,10 +1566,11 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     {
         this->get_exec_info().device_id = device_id;
         this->get_exec_info().num_computing_units = 0;
-        this->get_exec_info().num_pe_per_cu = 0;
+        this->get_exec_info().num_pu_per_cu = 0;
         this->HipExecutor::populate_exec_info(MachineTopology::get_instance());
         if (this->get_exec_info().closest_pu_ids.size()) {
-            this->bind_to_pus(this->get_exec_info().closest_pu_ids);
+            MachineTopology::get_instance()->bind_to_pus(
+                this->get_closest_pus());
         }
         this->set_gpu_property();
         this->init_handles();
@@ -1738,7 +1702,7 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      *
      * @return the number of Computing Units of this executor
      */
-    size_type get_num_computing_units() const noexcept
+    int get_num_computing_units() const noexcept
     {
         return this->get_exec_info().num_computing_units;
     }
@@ -1758,9 +1722,19 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
      *
      * @return the maximum workgroup size
      */
-    size_type get_max_workgroup_size() const noexcept
+    int get_max_workgroup_size() const noexcept
+    {
+        return this->get_exec_info().max_workgroup_size;
+    }
+
+    /**
+     * Get the maximum subgroup size.
+     *
+     * @return the maximum subgroup size
+     */
+    int get_max_subgroup_size() const noexcept
     {
-        return this->get_exec_info().num_pe_per_cu;
+        return this->get_exec_info().max_subgroup_size;
     }
 
     /**
diff --git a/include/ginkgo/core/base/machine_topology.hpp b/include/ginkgo/core/base/machine_topology.hpp
index b6a57053952..509c7ada786 100644
--- a/include/ginkgo/core/base/machine_topology.hpp
+++ b/include/ginkgo/core/base/machine_topology.hpp
@@ -232,6 +232,16 @@ class MachineTopology {
         hwloc_binding_helper(this->cores_, ids, singlify);
     }
 
+    /**
+     * Bind to a single core
+     *
+     * @param ids  The ids of the core to be bound to the calling process.
+     */
+    void bind_to_core(const int &id) const
+    {
+        MachineTopology::get_instance()->bind_to_cores(std::vector<int>{id});
+    }
+
     /**
      * Bind the calling process to PUs associated with
      * the ids.
@@ -250,6 +260,16 @@ class MachineTopology {
         hwloc_binding_helper(this->pus_, ids, singlify);
     }
 
+    /**
+     * Bind to a Processing unit (PU)
+     *
+     * @param ids  The ids of PUs to be bound to the calling process.
+     */
+    void bind_to_pu(const int &id) const
+    {
+        MachineTopology::get_instance()->bind_to_pus(std::vector<int>{id});
+    }
+
     /**
      * Get the object of type PU associated with the id.
      *

From 8df12f295c597cca0694ca574bd7910ebb8a28d6 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 28 Jan 2021 09:26:41 +0100
Subject: [PATCH 0306/1008] Review update.

+ Update to BUILD_HWLOC.

Co-authored-by: Tobias Ribizel <tobias.ribizel@kit.edu>
---
 .gitlab-ci.yml                   | 15 ++++++++-------
 CMakeLists.txt                   | 14 +++++++-------
 cmake/GinkgoConfig.cmake.in      |  2 +-
 cmake/get_info.cmake             |  4 ++--
 core/device_hooks/CMakeLists.txt |  2 +-
 third_party/CMakeLists.txt       |  2 +-
 6 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 544f45e3daa..87151c84c78 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,7 +22,7 @@ stages:
   BUILD_OMP: "OFF"
   BUILD_CUDA: "OFF"
   BUILD_HIP: "OFF"
-  DISABLE_HWLOC: "OFF"
+  BUILD_HWLOC: "ON"
   RUN_EXAMPLES: "OFF"
   CONFIG_LOG: "ON"
   CXX_FLAGS: ""
@@ -64,7 +64,7 @@ stages:
         -DGINKGO_DEVEL_TOOLS=OFF -DGINKGO_BUILD_REFERENCE=${BUILD_REFERENCE}
         -DGINKGO_BUILD_OMP=${BUILD_OMP} -DGINKGO_BUILD_CUDA=${BUILD_CUDA}
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
-        -DGINKGO_DISABLE_HWLOC=${DISABLE_HWLOC}
+        -DGINKGO_BUILD_HWLOC=${BUILD_HWLOC}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
@@ -96,7 +96,7 @@ stages:
         -DGINKGO_DEVEL_TOOLS=OFF -DGINKGO_BUILD_REFERENCE=${BUILD_REFERENCE}
         -DGINKGO_BUILD_OMP=${BUILD_OMP} -DGINKGO_BUILD_CUDA=${BUILD_CUDA}
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
-        -DGINKGO_DISABLE_HWLOC=${DISABLE_HWLOC}
+        -DGINKGO_BUILD_HWLOC=${BUILD_HWLOC}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
@@ -266,7 +266,7 @@ build/cuda92/intel/cuda/release/static:
     CXX_COMPILER: "icpc"
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
-    DISABLE_HWLOC: "ON"
+    BUILD_HWLOC: "OFF"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
@@ -422,7 +422,7 @@ build/clang-cuda101/clang/cuda/debug/static:
 
 # cuda 10.2 and friends
 
-# Use TPL hwloc
+# works when there is no hwloc and tpl hwloc is also switched off.
 build/cuda102/gcc/all/debug/shared:
   <<: *default_build_with_test
   image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
@@ -432,7 +432,7 @@ build/cuda102/gcc/all/debug/shared:
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     BUILD_TYPE: "Debug"
-    DISABLE_HWLOC: "ON"
+    BUILD_HWLOC: "OFF"
     CUDA_ARCH: 35
   only:
     variables:
@@ -442,6 +442,7 @@ build/cuda102/gcc/all/debug/shared:
     - cuda
     - gpu
 
+# Use TPL hwloc when no system hwloc is available
 build/cuda102/clang/all/release/static:
   <<: *default_build_with_test
   image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
@@ -587,7 +588,7 @@ build/nocuda/gcc/core/debug/static:
     BUILD_REFERENCE: "OFF"
     BUILD_TYPE: "Debug"
     BUILD_SHARED_LIBS: "OFF"
-    DISABLE_HWLOC: "ON"
+    BUILD_HWLOC: "OFF"
   only:
     variables:
       - $RUN_CI_TAG
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7d510bd63ce..4edf6bd972d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -70,7 +70,7 @@ set(GINKGO_HIP_AMDGPU "" CACHE STRING
     "The amdgpu_target(s) variable passed to hipcc. The default is none (auto).")
 option(GINKGO_JACOBI_FULL_OPTIMIZATIONS "Use all the optimizations for the CUDA Jacobi algorithm" OFF)
 option(BUILD_SHARED_LIBS "Build shared (.so, .dylib, .dll) libraries" ON)
-option(GINKGO_DISABLE_HWLOC "Disable HWLOC " OFF)
+option(GINKGO_BUILD_HWLOC "Build Ginkgo with HWLOC. Default is ON. If a system HWLOC is not found, then we try to build it ourselves. Switch this OFF to disable HWLOC." ON)
 
 set(GINKGO_CIRCULAR_DEPS_FLAGS "-Wl,--no-undefined")
 
@@ -144,14 +144,14 @@ if(PAPI_sde_FOUND)
     set(GINKGO_HAVE_PAPI_SDE 1)
 endif()
 
-if(GINKGO_DISABLE_HWLOC)
-    set(GINKGO_USE_EXTERNAL_HWLOC 0)
-    set(GINKGO_HAVE_HWLOC 0)
-    message(STATUS "HWLOC is being forcibly switched off")
-else()
+if(GINKGO_BUILD_HWLOC)
     # By default always use external HWLOC
     set(GINKGO_USE_EXTERNAL_HWLOC 1)
     set(GINKGO_HAVE_HWLOC 1)
+else()
+    set(GINKGO_USE_EXTERNAL_HWLOC 0)
+    set(GINKGO_HAVE_HWLOC 0)
+    message(STATUS "HWLOC is being forcibly switched off")
 endif()
 # Switch off HWLOC for Windows and MacOS
 if(MSVC OR WIN32 OR CYGWIN OR APPLE)
@@ -209,7 +209,7 @@ include(cmake/package_helpers.cmake)
 ginkgo_find_package(GTest "GTest::GTest;GTest::Main" FALSE 1.8.1)
 ginkgo_find_package(gflags gflags FALSE 2.2.2)
 ginkgo_find_package(RapidJSON rapidjson TRUE 1.1.0)
-if(NOT GINKGO_DISABLE_HWLOC AND GINKGO_HAVE_HWLOC)
+if(GINKGO_HAVE_HWLOC)
     ginkgo_find_package(HWLOC hwloc FALSE 2.1)
 endif()
 set(GKO_HWLOC_XMLFILE "$ENV{HWLOC_XMLFILE}")
diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 0cd56ccdccc..2fdcd695319 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -78,7 +78,7 @@ set(GINKGO_DPCPP_FLAGS @GINKGO_DPCPP_FLAGS@)
 set(GINKGO_HAVE_PAPI_SDE @GINKGO_HAVE_PAPI_SDE@)
 
 set(GINKGO_HAVE_HWLOC @GINKGO_HAVE_HWLOC@)
-set(GINKGO_DISABLE_HWLOC @GINKGO_DISABLE_HWLOC@)
+set(GINKGO_BUILD_HWLOC @GINKGO_BUILD_HWLOC@)
 
 # Ginkgo external package variables
 set(GINKGO_USE_EXTERNAL_CAS "@GINKGO_USE_EXTERNAL_CAS@")
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index 5621c74f2b8..78b5f5783ab 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -175,8 +175,8 @@ if(PAPI_sde_FOUND)
     ginkgo_print_variable(${detailed_log} "PAPI_INCLUDE_DIR")
     ginkgo_print_flags(${detailed_log} "PAPI_LIBRARY")
 endif()
-ginkgo_print_variable(${minimal_log} "GINKGO_DISABLE_HWLOC")
-ginkgo_print_variable(${detailed_log} "GINKGO_DISABLE_HWLOC")
+ginkgo_print_variable(${minimal_log} "GINKGO_BUILD_HWLOC")
+ginkgo_print_variable(${detailed_log} "GINKGO_BUILD_HWLOC")
 ginkgo_print_variable(${detailed_log} "HWLOC_VERSION")
 ginkgo_print_variable(${detailed_log} "GKO_HWLOC_XMLFILE")
 ginkgo_print_variable(${detailed_log} "HWLOC_LIBRARIES")
diff --git a/core/device_hooks/CMakeLists.txt b/core/device_hooks/CMakeLists.txt
index 9b107364924..ff96f6e19f9 100644
--- a/core/device_hooks/CMakeLists.txt
+++ b/core/device_hooks/CMakeLists.txt
@@ -12,7 +12,7 @@ if (NOT GINKGO_BUILD_DPCPP)
     add_library(ginkgo_dpcpp
         $<TARGET_OBJECTS:ginkgo_dpcpp_device>
         dpcpp_hooks.cpp)
-    target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_core_device )
+    target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_core_device)
     ginkgo_compile_features(ginkgo_dpcpp)
     ginkgo_default_includes(ginkgo_dpcpp)
     ginkgo_install_library(ginkgo_dpcpp dpcpp)
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index b4ca5067187..71cd76defa6 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -13,7 +13,7 @@ if(GINKGO_BUILD_TESTS)
     ginkgo_add_subdirectory(GTest gtest)
 endif()
 
-if(NOT GINKGO_DISABLE_HWLOC AND GINKGO_HAVE_HWLOC)
+if(GINKGO_HAVE_HWLOC)
     ginkgo_add_subdirectory(HWLOC hwloc)
 endif()
 

From 0e9c78688d4f4bf39854c6a1e1bb9778cabe41a7 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 19 Jan 2021 00:01:59 +0100
Subject: [PATCH 0307/1008] Tune and benchmark variables non intrusively.

This piece of code integrates and shows how to tune some variables,
namely the multiple used to decide on the number of warps used in COO.

How: This is integrated into the SPMV benchmarks for now and is entirely
controlled with the new CMake option `GINKGO_BENCHMARK_ENABLE_TUNING`.

The setup works through the use of (dynamic) global variables so that
tuning is as little intrusive as possible. This should work even with the
"implementation selection" since this also has a dynamic component which
could use the same strategy as shown here.

Caveat: With this setup, only benchmarks can be compiled, other
executable codes such as tests would require to also add
`benchmark/utils/tuning_variables.cpp` or equivalent. This option should
of course not be turned on for general Ginkgo usage, but only for
tuning.
---
 CMakeLists.txt                                | 10 ++++
 benchmark/CMakeLists.txt                      |  7 +++
 benchmark/conversions/conversions.cpp         |  5 ++
 .../matrix_generator/matrix_generator.cpp     |  5 ++
 .../matrix_statistics/matrix_statistics.cpp   |  5 ++
 benchmark/preconditioner/preconditioner.cpp   |  6 ++-
 benchmark/solver/solver.cpp                   |  5 ++
 benchmark/spmv/spmv.cpp                       | 47 ++++++++++++++++-
 benchmark/utils/tuning_variables.cpp          | 48 +++++++++++++++++
 benchmark/utils/tuning_variables.hpp          | 52 +++++++++++++++++++
 cmake/get_info.cmake                          |  2 +-
 cuda/components/format_conversion.cuh         | 11 ++++
 hip/components/format_conversion.hip.hpp      | 11 ++++
 include/ginkgo/config.hpp.in                  |  4 ++
 14 files changed, 215 insertions(+), 3 deletions(-)
 create mode 100644 benchmark/utils/tuning_variables.cpp
 create mode 100644 benchmark/utils/tuning_variables.hpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4edf6bd972d..49a6d20f9b4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -44,6 +44,9 @@ option(GINKGO_CHECK_CIRCULAR_DEPS
     OFF)
 option(GINKGO_CONFIG_LOG_DETAILED
     "Enable printing of detailed configuration log to screen in addition to the writing of files," OFF)
+option(GINKGO_BENCHMARK_ENABLE_TUNING
+    "Enable tuning variables in the benchmarks. For specific use cases, manual code changes could be required."
+    OFF)
 set(GINKGO_VERBOSE_LEVEL "1" CACHE STRING
     "Verbosity level. Put 0 to turn off. 1 activates a few important messages.")
 if(MSVC)
@@ -74,6 +77,13 @@ option(GINKGO_BUILD_HWLOC "Build Ginkgo with HWLOC. Default is ON. If a system H
 
 set(GINKGO_CIRCULAR_DEPS_FLAGS "-Wl,--no-undefined")
 
+if(GINKGO_BENCHMARK_ENABLE_TUNING)
+    # In this state, the tests and examples cannot be compiled without extra
+    # complexity/intrusiveness, so we simply disable them.
+    set(GINKGO_BUILD_TESTS OFF)
+    set(GINKGO_BUILD_EXAMPLES OFF)
+endif()
+
 if(BUILD_SHARED_LIBS AND (WIN32 OR CYGWIN) AND (GINKGO_BUILD_TESTS OR GINKGO_BUILD_EXAMPLES OR GINKGO_BUILD_BENCHMARKS))
     # Change shared libraries output only if this build has executable program
     # with shared libraries.
diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
index 9323039f6ea..0ad53594fd4 100644
--- a/benchmark/CMakeLists.txt
+++ b/benchmark/CMakeLists.txt
@@ -11,6 +11,12 @@ if (GINKGO_BUILD_CUDA AND GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc"
         "or use `export HIP_PLATFORM=nvcc` in your build environment instead.")
 endif()
 
+function(ginkgo_benchmark_add_tuning_maybe name)
+    if(GINKGO_BENCHMARK_ENABLE_TUNING)
+        target_sources(${name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../utils/tuning_variables.cpp)
+    endif()
+endfunction()
+
 function(ginkgo_benchmark_cusp_linops name)
     target_compile_definitions("${name}" PRIVATE HAS_CUDA=1)
     target_link_libraries("${name}" ginkgo ${CUDA_RUNTIME_LIBS}
@@ -69,6 +75,7 @@ function(ginkgo_add_single_benchmark_executable name use_lib_linops macro_def)
     add_executable("${name}" ${ARGN})
     target_link_libraries("${name}" ginkgo gflags rapidjson)
     target_compile_definitions("${name}" PRIVATE "${macro_def}")
+    ginkgo_benchmark_add_tuning_maybe("${name}")
     if("${use_lib_linops}")
         if (GINKGO_BUILD_CUDA)
             ginkgo_benchmark_cusp_linops("${name}")
diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index 7a3d9c61a0c..9c36ad24537 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -51,6 +51,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/types.hpp"
 
 
+#ifdef GINKGO_BENCHMARK_ENABLE_TUNING
+#include "benchmark/utils/tuning_variables.hpp"
+#endif  // GINKGO_BENCHMARK_ENABLE_TUNING
+
+
 // This function supposes that management of `FLAGS_overwrite` is done before
 // calling it
 void convert_matrix(const gko::LinOp *matrix_from, const char *format_to,
diff --git a/benchmark/matrix_generator/matrix_generator.cpp b/benchmark/matrix_generator/matrix_generator.cpp
index ab98a20d20e..308c57de421 100644
--- a/benchmark/matrix_generator/matrix_generator.cpp
+++ b/benchmark/matrix_generator/matrix_generator.cpp
@@ -43,6 +43,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/types.hpp"
 
 
+#ifdef GINKGO_BENCHMARK_ENABLE_TUNING
+#include "benchmark/utils/tuning_variables.hpp"
+#endif  // GINKGO_BENCHMARK_ENABLE_TUNING
+
+
 namespace {
 std::string input_format =
     "  [\n"
diff --git a/benchmark/matrix_statistics/matrix_statistics.cpp b/benchmark/matrix_statistics/matrix_statistics.cpp
index bc273acf70d..4501f67a489 100644
--- a/benchmark/matrix_statistics/matrix_statistics.cpp
+++ b/benchmark/matrix_statistics/matrix_statistics.cpp
@@ -45,6 +45,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/types.hpp"
 
 
+#ifdef GINKGO_BENCHMARK_ENABLE_TUNING
+#include "benchmark/utils/tuning_variables.hpp"
+#endif  // GINKGO_BENCHMARK_ENABLE_TUNING
+
+
 // See en.wikipedia.org/wiki/Five-number_summary
 // Quartile computation uses Method 3 from en.wikipedia.org/wiki/Quartile
 void compute_summary(const std::vector<gko::size_type> &dist,
diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index 00d928a9f09..9ec10b1ee4c 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -51,8 +51,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/types.hpp"
 
 
-// preconditioner generation and application
+#ifdef GINKGO_BENCHMARK_ENABLE_TUNING
+#include "benchmark/utils/tuning_variables.hpp"
+#endif  // GINKGO_BENCHMARK_ENABLE_TUNING
+
 
+// preconditioner generation and application
 std::string encode_parameters(const char *precond_name)
 {
     static std::map<std::string, std::string (*)()> encoder{
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index f75ae9216b1..f351fb74fbe 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -55,6 +55,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/types.hpp"
 
 
+#ifdef GINKGO_BENCHMARK_ENABLE_TUNING
+#include "benchmark/utils/tuning_variables.hpp"
+#endif  // GINKGO_BENCHMARK_ENABLE_TUNING
+
+
 // Command-line arguments
 DEFINE_uint32(max_iters, 1000,
               "Maximal number of iterations the solver will be run for");
diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp
index b00dbff152a..c455bbfa78f 100644
--- a/benchmark/spmv/spmv.cpp
+++ b/benchmark/spmv/spmv.cpp
@@ -51,8 +51,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "benchmark/utils/types.hpp"
 
 
-// Command-line arguments
+#ifdef GINKGO_BENCHMARK_ENABLE_TUNING
+#include "benchmark/utils/tuning_variables.hpp"
+#endif  // GINKGO_BENCHMARK_ENABLE_TUNING
+
 
+// Command-line arguments
 DEFINE_uint32(nrhs, 1, "The number of right hand sides");
 
 
@@ -94,6 +98,47 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
             system_matrix->apply(lend(b), lend(x_clone));
             exec->synchronize();
         }
+
+        // tuning run
+#ifdef GINKGO_BENCHMARK_ENABLE_TUNING
+        auto &format_case = spmv_case[format_name];
+        if (!format_case.HasMember("tuning")) {
+            format_case.AddMember(
+                "tuning", rapidjson::Value(rapidjson::kObjectType), allocator);
+        }
+        auto &tuning_case = format_case["tuning"];
+        add_or_set_member(tuning_case, "time",
+                          rapidjson::Value(rapidjson::kArrayType), allocator);
+        add_or_set_member(tuning_case, "values",
+                          rapidjson::Value(rapidjson::kArrayType), allocator);
+
+        // Enable tuning for this portion of code
+        gko::_tuning_flag = true;
+        // Select some values we want to tune.
+        std::vector<gko::size_type> tuning_values{
+            1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096};
+        for (auto val : tuning_values) {
+            // Actually set the value that will be tuned. See
+            // cuda/components/format_conversion.cuh for an example of how this
+            // variable is used.
+            gko::_tuned_value = val;
+            auto tuning_timer = get_timer(exec, FLAGS_gpu_timer);
+            for (unsigned int i = 0; i < FLAGS_repetitions; i++) {
+                auto x_clone = clone(x);
+                exec->synchronize();
+                tuning_timer->tic();
+                system_matrix->apply(lend(b), lend(x_clone));
+                tuning_timer->toc();
+            }
+            tuning_case["time"].PushBack(tuning_timer->compute_average_time(),
+                                         allocator);
+            tuning_case["values"].PushBack(val, allocator);
+        }
+        // We put back the flag to false to use the default (non-tuned) values
+        // for the following
+        gko::_tuning_flag = false;
+#endif  // GINKGO_BENCHMARK_ENABLE_TUNING
+
         // timed run
         auto timer = get_timer(exec, FLAGS_gpu_timer);
         for (unsigned int i = 0; i < FLAGS_repetitions; i++) {
diff --git a/benchmark/utils/tuning_variables.cpp b/benchmark/utils/tuning_variables.cpp
new file mode 100644
index 00000000000..d4554286532
--- /dev/null
+++ b/benchmark/utils/tuning_variables.cpp
@@ -0,0 +1,48 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "benchmark/utils/tuning_variables.hpp"
+
+
+#include <ginkgo/core/base/types.hpp>
+
+
+namespace gko {
+
+
+bool _tuning_flag = false;
+
+
+size_type _tuned_value = 0;
+
+
+}  // namespace gko
diff --git a/benchmark/utils/tuning_variables.hpp b/benchmark/utils/tuning_variables.hpp
new file mode 100644
index 00000000000..be172c02cd0
--- /dev/null
+++ b/benchmark/utils/tuning_variables.hpp
@@ -0,0 +1,52 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_BENCHMARK_UTILS_TUNING_VARIABLES_HPP_
+#define GKO_BENCHMARK_UTILS_TUNING_VARIABLES_HPP_
+
+
+#include <ginkgo/core/base/types.hpp>
+
+
+namespace gko {
+
+
+extern bool _tuning_flag;
+
+
+extern size_type _tuned_value;
+
+
+}  // namespace gko
+
+
+#endif  // GKO_BENCHMARK_UTILS_TUNING_VARIABLES_HPP_
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index 78b5f5783ab..3d84e3aca94 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -130,7 +130,7 @@ foreach(log_type ${log_types})
         "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP")
     ginkgo_print_module_footer(${${log_type}} "  Tests, benchmarks and examples:")
     ginkgo_print_foreach_variable(
-        "GINKGO_BUILD_TESTS;GINKGO_BUILD_EXAMPLES;GINKGO_EXTLIB_EXAMPLE;GINKGO_BUILD_BENCHMARKS")
+        "GINKGO_BUILD_TESTS;GINKGO_BUILD_EXAMPLES;GINKGO_EXTLIB_EXAMPLE;GINKGO_BUILD_BENCHMARKS;GINKGO_BENCHMARK_ENABLE_TUNING")
     ginkgo_print_module_footer(${${log_type}} "  Documentation:")
     ginkgo_print_foreach_variable("GINKGO_BUILD_DOC;GINKGO_VERBOSE_LEVEL")
     ginkgo_print_module_footer(${${log_type}} "  Developer helpers:")
diff --git a/cuda/components/format_conversion.cuh b/cuda/components/format_conversion.cuh
index fa411773574..3586627451d 100644
--- a/cuda/components/format_conversion.cuh
+++ b/cuda/components/format_conversion.cuh
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CUDA_COMPONENTS_FORMAT_CONVERSION_CUH_
 
 
+#include <ginkgo/config.hpp>
 #include <ginkgo/core/base/executor.hpp>
 
 
@@ -41,6 +42,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "cuda/components/thread_ids.cuh"
 
 
+#ifdef GINKGO_BENCHMARK_ENABLE_TUNING
+#include "benchmark/utils/tuning_variables.hpp"
+#endif  // GINKGO_BENCHMARK_ENABLE_TUNING
+
+
 namespace gko {
 namespace kernels {
 namespace cuda {
@@ -109,6 +115,11 @@ __host__ size_type calculate_nwarps(std::shared_ptr<const CudaExecutor> exec,
     } else if (nnz >= 2e5) {
         multiple = 32;
     }
+#ifdef GINKGO_BENCHMARK_ENABLE_TUNING
+    if (_tuning_flag) {
+        multiple = _tuned_value;
+    }
+#endif  // GINKGO_BENCHMARK_ENABLE_TUNING
     return std::min(multiple * nwarps_in_cuda,
                     size_type(ceildiv(nnz, config::warp_size)));
 }
diff --git a/hip/components/format_conversion.hip.hpp b/hip/components/format_conversion.hip.hpp
index 4a28cfae17b..c0c77869e3f 100644
--- a/hip/components/format_conversion.hip.hpp
+++ b/hip/components/format_conversion.hip.hpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <hip/hip_runtime.h>
 
 
+#include <ginkgo/config.hpp>
 #include <ginkgo/core/base/executor.hpp>
 
 
@@ -44,6 +45,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "hip/components/thread_ids.hip.hpp"
 
 
+#ifdef GINKGO_BENCHMARK_ENABLE_TUNING
+#include "benchmark/utils/tuning_variables.hpp"
+#endif  // GINKGO_BENCHMARK_ENABLE_TUNING
+
+
 namespace gko {
 namespace kernels {
 namespace hip {
@@ -121,6 +127,11 @@ __host__ size_type calculate_nwarps(std::shared_ptr<const HipExecutor> exec,
         multiple = 8;
     }
 #endif  // GINKGO_HIP_PLATFORM_NVCC
+#ifdef GINKGO_BENCHMARK_ENABLE_TUNING
+    if (_tuning_flag) {
+        multiple = _tuned_value;
+    }
+#endif  // GINKGO_BENCHMARK_ENABLE_TUNING
     return std::min(multiple * nwarps_in_hip,
                     size_type(ceildiv(nnz, config::warp_size)));
 }
diff --git a/include/ginkgo/config.hpp.in b/include/ginkgo/config.hpp.in
index 55042286815..e41862634f9 100644
--- a/include/ginkgo/config.hpp.in
+++ b/include/ginkgo/config.hpp.in
@@ -59,6 +59,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #cmakedefine GINKGO_JACOBI_FULL_OPTIMIZATIONS
 
 
+/* Should we compile Ginkgo specifically to tune values? */
+#cmakedefine GINKGO_BENCHMARK_ENABLE_TUNING
+
+
 /* What is HIP compiled for, hcc or nvcc? */
 // clang-format off
 #define GINKGO_HIP_PLATFORM_HCC @GINKGO_HIP_PLATFORM_HCC@

From 81e6eddd1f180a114f6edd267f9485249c26e800 Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Tue, 19 Jan 2021 10:00:13 +0000
Subject: [PATCH 0308/1008] Format files

Co-authored-by: tcojean <tcojean@users.noreply.github.com>
---
 benchmark/utils/tuning_variables.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/benchmark/utils/tuning_variables.cpp b/benchmark/utils/tuning_variables.cpp
index d4554286532..b01dd98d635 100644
--- a/benchmark/utils/tuning_variables.cpp
+++ b/benchmark/utils/tuning_variables.cpp
@@ -30,10 +30,10 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "benchmark/utils/tuning_variables.hpp"
+#include <ginkgo/core/base/types.hpp>
 
 
-#include <ginkgo/core/base/types.hpp>
+#include "benchmark/utils/tuning_variables.hpp"
 
 
 namespace gko {

From 17d2e73d63e59b4b4c12e34ab374e86a1669088f Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 19 Jan 2021 17:30:41 +0100
Subject: [PATCH 0309/1008] Add an R script to plot JSON tuning data using R.

---
 dev_tools/plots/tuning_heatmap.R | 49 ++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 dev_tools/plots/tuning_heatmap.R

diff --git a/dev_tools/plots/tuning_heatmap.R b/dev_tools/plots/tuning_heatmap.R
new file mode 100644
index 00000000000..cb4fc9510a8
--- /dev/null
+++ b/dev_tools/plots/tuning_heatmap.R
@@ -0,0 +1,49 @@
+# These packages are required, to install them, use the package manager or open
+# an R session and type:
+# install.packages("jsonlite", "tidyr", "ggplot2", "scales")
+library(jsonlite)
+library(ggplot2)
+library(scales)
+library(tidyr)
+
+# Manage arguments
+args <- commandArgs(trailingOnly=TRUE)
+if (length(args)!=2) {
+    stop("Usage: Rscript tuning_heatmap.R input_directory output_graphics_file\n", call=FALSE)
+}
+input <- args[1]
+output <- args[2]
+
+# Read the input json files into a dataframe
+files <- list.files(paste(input), recursive=TRUE, pattern = "*.json", full.names=TRUE)
+df_tmp <- list()
+count <- 1
+for (i in files)
+{
+    tmp <-jsonlite::fromJSON(i,flatten=TRUE)
+    df_tmp[[count]] <- as.data.frame(tmp)[,c("problem.name", "problem.nonzeros",
+                                             "spmv.coo.time", "spmv.coo.tuning.values",
+                                             "spmv.coo.tuning.time")]
+    count <- count +1
+}
+# Merge all the separate dataframes
+df_merged <- rbind_pages(df_tmp)
+# Unnest the two vectors
+df <- as.data.frame(unnest(df,spmv.coo.tuning.values,spmv.coo.tuning.time))
+# Now that all columns are vectors, compute the speedup using vector operations
+df$spmv.coo.speedup <- df$spmv.coo.time/df$spmv.coo.tuning.time
+
+# Plot the values
+ggplot(df, aes(factor(problem.nonzeros), factor(spmv.coo.tuning.values), fill=spmv.coo.speedup)) +
+    geom_tile() +
+    scale_fill_gradientn(
+        colours=c("red", "yellow", "skyblue", "darkblue"),
+        values = rescale(c(min(df$speedup),
+                           1.0,
+                           1.11,
+                           max(df$speedup)))) +
+    ggtitle("Speedup of tuned value against COO SpMV")+ xlab("nonzeros")+ ylab("tuned value (multiple)") +
+    theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1), plot.title = element_text(hjust=0.5))
+
+# Save to the output file
+ggsave(paste(output), width=9, height=7)

From 70424224b0d5ca97132e60b9e21ba298d23331e9 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 30 Jan 2021 21:17:40 +0100
Subject: [PATCH 0310/1008] fix machine_topology PCI bus ID buffer overflow

found by a compiler warning for snprintf
---
 core/devices/machine_topology.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/devices/machine_topology.cpp b/core/devices/machine_topology.cpp
index 09fcab1cd86..cc7a86f51d4 100644
--- a/core/devices/machine_topology.cpp
+++ b/core/devices/machine_topology.cpp
@@ -249,7 +249,7 @@ void MachineTopology::load_objects(
         hwloc_obj_type_snprintf(ances_type, sizeof(ances_type), ancestor, 0);
         vector.back().ancestor_type = std::string(ances_type);
         // Write the PCI Bus ID from the object info.
-        char pci_bus_id[13];
+        char pci_bus_id[14];
         snprintf(pci_bus_id, sizeof(pci_bus_id), "%04x:%02x:%02x.%01x",
                  obj->attr->pcidev.domain, obj->attr->pcidev.bus,
                  obj->attr->pcidev.dev, obj->attr->pcidev.func);

From 77358e0425a82ae79a888c9aa4c62785e27e4625 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 30 Jan 2021 22:27:50 +0100
Subject: [PATCH 0311/1008] add heat equation example with video output

---
 examples/CMakeLists.txt                  |     1 +
 examples/heat-equation/CMakeLists.txt    |    12 +
 examples/heat-equation/build.sh          |    19 +
 examples/heat-equation/data/initial.mtx  | 65538 +++++++++++++++++++++
 examples/heat-equation/data/source.mtx   | 65538 +++++++++++++++++++++
 examples/heat-equation/doc/builds-on     |     1 +
 examples/heat-equation/doc/intro.dox     |    39 +
 examples/heat-equation/doc/kind          |     1 +
 examples/heat-equation/doc/results.dox   |     5 +
 examples/heat-equation/doc/short-intro   |     1 +
 examples/heat-equation/doc/tooltip       |     1 +
 examples/heat-equation/heat-equation.cpp |   209 +
 12 files changed, 131365 insertions(+)
 create mode 100644 examples/heat-equation/CMakeLists.txt
 create mode 100755 examples/heat-equation/build.sh
 create mode 100644 examples/heat-equation/data/initial.mtx
 create mode 100644 examples/heat-equation/data/source.mtx
 create mode 100644 examples/heat-equation/doc/builds-on
 create mode 100644 examples/heat-equation/doc/intro.dox
 create mode 100644 examples/heat-equation/doc/kind
 create mode 100644 examples/heat-equation/doc/results.dox
 create mode 100644 examples/heat-equation/doc/short-intro
 create mode 100644 examples/heat-equation/doc/tooltip
 create mode 100644 examples/heat-equation/heat-equation.cpp

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index c023940383b..b35d6422098 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -21,6 +21,7 @@ set(EXAMPLES_LIST
     custom-matrix-format
     custom-stopping-criterion
     ginkgo-overhead
+    heat-equation
     minimal-cuda-solver
     papi-logging
     par-ilu-convergence
diff --git a/examples/heat-equation/CMakeLists.txt b/examples/heat-equation/CMakeLists.txt
new file mode 100644
index 00000000000..75ba544a218
--- /dev/null
+++ b/examples/heat-equation/CMakeLists.txt
@@ -0,0 +1,12 @@
+set(target_name "heat-equation")
+find_package(OpenCV)
+
+if (OpenCV_FOUND)
+    add_executable(${target_name} ${target_name}.cpp)
+    target_link_libraries(${target_name} Ginkgo::ginkgo ${OpenCV_LIBS})
+    target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+    configure_file(data/initial.mtx data/initial.mtx COPYONLY)
+    configure_file(data/source.mtx data/source.mtx COPYONLY)
+else()
+    message("OpenCV not found, skipping heat-equation example")
+endif()
diff --git a/examples/heat-equation/build.sh b/examples/heat-equation/build.sh
new file mode 100755
index 00000000000..495d054d477
--- /dev/null
+++ b/examples/heat-equation/build.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# set up script
+if [ $# -ne 1 ]; then
+    echo -e "Usage: $0 GINKGO_BUILD_DIRECTORY"
+    exit 1
+fi
+BUILD_DIR=$1
+THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" &>/dev/null && pwd )
+
+source ${THIS_DIR}/../build-setup.sh
+
+# build
+${CXX} -std=c++14 -o ${THIS_DIR}/heat-equation \
+       ${THIS_DIR}/heat-equation.cpp \
+       -I${THIS_DIR}/../../include -I${BUILD_DIR}/include \
+       `pkg-config --cflags opencv4` \
+       -L${THIS_DIR} ${LINK_FLAGS} \
+       `pkg-config --libs opencv4`
diff --git a/examples/heat-equation/data/initial.mtx b/examples/heat-equation/data/initial.mtx
new file mode 100644
index 00000000000..3d4fdcbecb5
--- /dev/null
+++ b/examples/heat-equation/data/initial.mtx
@@ -0,0 +1,65538 @@
+%%MatrixMarket matrix array real general
+65536 1
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.09700788686685691
+0.24134981294511437
+0.3866003304555665
+0.46493975774341134
+0.5023772328168228
+0.5091424310999448
+0.4879897630331653
+0.3944474308679604
+0.19000670232238637
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.1174721498628637
+0.19641728348258702
+0.2296101192536426
+0.19641728348258702
+0.1293633932048852
+0.08652970395641724
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.1174721498628637
+0.4402850125061395
+0.6222706499630242
+0.5952918676771857
+3.4134121992978272
+3.448736817474918
+4.515387913750165
+4.547905436471331
+4.531406566217803
+4.49967437303024
+3.4508560466680547
+3.4084134232106233
+0.6099759108185445
+0.593379484990824
+0.2616727643865282
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.2555541939391164
+0.5995223290890301
+0.6151073061225478
+3.3999084873857006
+3.405282092316791
+3.408257823745231
+3.4030586686795825
+3.398331392047578
+0.6048683038797913
+0.6306810902599107
+0.5728484679595169
+0.2819871622377592
+0.04986894719131918
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.08652970395641724
+0.4949361895255393
+0.6027748234445571
+3.465580337420984
+4.771893282393301
+4.914400272933426
+4.876422053888234
+4.871259604324209
+4.927710408327659
+4.914400272933426
+4.797007948311492
+4.760252191586986
+4.858812151936161
+4.954466537076697
+4.954466537076697
+4.524936356535483
+3.400810101543841
+0.6323850363479462
+0.2175816996331532
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.157275361322311
+0.6189656491157116
+3.40192156799998
+4.496400533324626
+4.706661305356881
+4.706661305356881
+4.706661305356881
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.488924457521423
+3.4028602887940522
+0.6379045049580707
+0.2935510911988004
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.2250745829383396
+0.6315576238645662
+3.4625939578942924
+4.927710408327659
+4.927710408327659
+4.914400272933426
+4.836643969491252
+4.779773978458947
+4.799397058532644
+4.914400272933426
+4.914400272933426
+4.779773978458947
+4.760252191586986
+4.856526425443426
+4.927710408327659
+4.942733045803132
+4.858812151936161
+4.74320020196667
+4.5073296469186275
+0.5958542710129855
+0.478329130669521
+0.0576866479940108
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.3397734141787456
+0.6077216126175999
+4.517639698678814
+4.706661305356881
+4.706661305356881
+4.706661305356881
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.687379768913745
+4.687379768913745
+4.545406035164644
+3.3964945993160724
+0.5973002521507231
+0.12563154621256611
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.25327535793473643
+0.6170782619076309
+4.546129552465073
+4.856526425443426
+4.914400272933426
+4.927710408327659
+4.90900814782395
+4.799397058532644
+4.779773978458947
+4.760252191586986
+4.851876601914066
+4.871259604324209
+4.760252191586986
+4.740838689292398
+4.836643969491252
+4.927710408327659
+4.927710408327659
+4.876422053888234
+4.7425356663700775
+4.74320020196667
+4.72456262033899
+3.413367042843766
+0.601668839099629
+0.08652970395641724
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.4061375121005799
+0.5899185767461592
+4.799131843864706
+4.7807449689252595
+4.706661305356881
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+3.520256162140694
+0.623527224719381
+0.36996027523321917
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.2342163616980768
+0.6239900704573369
+4.565463683036979
+4.799397058532644
+4.799397058532644
+4.836643969491252
+4.914400272933426
+4.914400272933426
+4.799397058532644
+4.760252191586986
+4.760252191586986
+4.779773978458947
+4.779773978458947
+4.740838689292398
+4.740838689292398
+4.816798834118136
+4.927710408327659
+4.927710408327659
+4.890373906609362
+4.740838689292398
+4.723513570758677
+4.72456262033899
+4.72456262033899
+3.4311617867798176
+0.6078689180196122
+0.08652970395641724
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.36632506944213183
+3.396125464650726
+4.7807449689252595
+4.917254376411946
+4.724731264199313
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.629292470505407
+3.402013881744261
+0.6221120681140002
+0.14584680845313702
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.3047863981217178
+0.6176669519964975
+4.565026059278837
+4.836643969491252
+4.779773978458947
+4.779773978458947
+4.779773978458947
+4.836643969491252
+4.914400272933426
+4.816798834118136
+4.760252191586986
+4.760252191586986
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.797007948311492
+4.927710408327659
+4.927710408327659
+4.89873092730281
+4.760252191586986
+4.721541071319574
+4.721541071319574
+4.723513570758677
+4.72456262033899
+3.4092709205692344
+0.5101841475615974
+0.0576866479940108
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.2815945019063122
+0.60697618875888
+4.706661305356881
+4.858680873820008
+4.878241460327516
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.741920921617843
+4.515513748627862
+0.6122451008034862
+0.36838302101761844
+0.02884340803556551
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.5462859882677904
+0.5899413179355788
+4.637805882449454
+4.914400272933426
+4.890373906609362
+4.799397058532644
+4.760252191586986
+4.760252191586986
+4.760252191586986
+4.856526425443426
+4.797007948311492
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.7772844700957515
+4.914400272933426
+4.927710408327659
+4.914400272933426
+4.7772844700957515
+4.721541071319574
+4.721541071319574
+4.702367606407748
+4.70461635049022
+4.70461635049022
+0.6033346157208195
+0.29090644896388973
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.15433651373052548
+0.6468371917588174
+4.517180948736314
+4.724731264199313
+4.8977894537919315
+4.8196627570631865
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.705374712396305
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.6862809068484905
+4.839146489353515
+4.979648002510261
+4.8052586551908245
+3.4055507670466887
+0.6313873661534748
+0.12563154621256611
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.3804579250412381
+0.6283636673209996
+3.4560886589405255
+4.779773978458947
+4.816798834118136
+4.89873092730281
+4.914400272933426
+4.871259604324209
+4.760252191586986
+4.760252191586986
+4.760252191586986
+4.760252191586986
+4.760252191586986
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.721541071319574
+4.760252191586986
+4.89873092730281
+4.927710408327659
+4.927710408327659
+4.797007948311492
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.492830953572344
+0.6731571051476077
+0.10651401691436746
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.08652970395641724
+0.6505752464000345
+3.482448277995883
+4.705374712396305
+4.741640717081259
+4.939112184045445
+4.779464206950053
+4.705374712396305
+4.705374712396305
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.6862809068484905
+4.6862809068484905
+4.723266788376083
+4.939112184045445
+4.979648002510261
+4.939112184045445
+4.723266788376083
+3.5213697935196873
+0.6257788837436884
+0.3288024034009762
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.1174721498628637
+0.4378629551218145
+0.6242252728394968
+3.416892015768177
+4.6396512859558126
+4.7576395655368735
+4.7576395655368735
+4.760252191586986
+4.816798834118136
+4.914400272933426
+4.914400272933426
+4.851876601914066
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.721541071319574
+4.721541071319574
+4.721541071319574
+4.740838689292398
+4.890373906609362
+4.927710408327659
+4.927710408327659
+4.812732846572471
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.702367606407748
+3.3904710512449823
+0.34789026551600166
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.3986814192409289
+3.390093982583272
+4.705374712396305
+4.705374712396305
+4.779464206950053
+4.939112184045445
+4.741920921617843
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.6862809068484905
+4.6862809068484905
+4.6862809068484905
+4.667980307053825
+4.800247028092315
+4.979648002510261
+4.979648002510261
+4.858680873820008
+4.667980307053825
+4.667980307053825
+4.550296376868264
+3.394306358192898
+0.5998540234643901
+0.12563154621256611
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.08652970395641724
+0.5023285856438795
+0.6032473782512451
+3.4555084947063524
+4.7321680251582725
+4.818365174589247
+4.7342759124717695
+4.7576395655368735
+4.7576395655368735
+4.7576395655368735
+4.738083404508356
+4.851876601914066
+4.914400272933426
+4.914400272933426
+4.797007948311492
+4.721541071319574
+4.721541071319574
+4.740838689292398
+4.740838689292398
+4.740838689292398
+4.721541071319574
+4.721541071319574
+4.721541071319574
+4.721541071319574
+4.721541071319574
+4.871259604324209
+4.927710408327659
+4.927710408327659
+4.851876601914066
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.481931321951778
+0.6819464310925891
+0.08652970395641724
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.17305757537147226
+0.6477421764976778
+4.580603563978908
+4.6862809068484905
+4.6862809068484905
+4.839146489353515
+4.918935027068591
+4.705374712396305
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.6862809068484905
+4.6862809068484905
+4.667980307053825
+4.667980307053825
+4.6862809068484905
+4.918935027068591
+4.979648002510261
+4.959350993808612
+4.760369351195106
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+3.5115025769659662
+0.6271487773617475
+0.37691273948061976
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.0576866479940108
+0.5806138431058043
+3.42369249567201
+4.773452659709149
+4.76833752096446
+4.845146974506639
+4.845146974506639
+4.748902697275286
+4.7342759124717695
+4.714774087646149
+4.738083404508356
+4.738083404508356
+4.7576395655368735
+4.870754749200952
+4.900088766172104
+4.89873092730281
+4.7772844700957515
+4.721541071319574
+4.721541071319574
+4.702367606407748
+4.721541071319574
+4.721541071319574
+4.721541071319574
+4.721541071319574
+4.721541071319574
+4.702367606407748
+4.832343613752159
+4.927710408327659
+4.927710408327659
+4.890373906609362
+4.721541071319574
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.683327299089502
+4.683327299089502
+0.6372361654016805
+0.19262923125003173
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.07799929245714843
+0.5902957830399753
+3.4346281852775626
+4.6862809068484905
+4.6862809068484905
+4.6862809068484905
+4.858812151936161
+4.878780841567253
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.6862809068484905
+4.6862809068484905
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.779602577154106
+4.979648002510261
+4.979648002510261
+4.8977894537919315
+4.667980307053825
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.573804961511842
+3.400674372454345
+0.6446648696800334
+0.25687811066009925
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.4380371529532674
+3.404076937033252
+4.858049419420977
+4.79984516624872
+4.7342759124717695
+4.79984516624872
+4.858049419420977
+4.818365174589247
+4.714774087646149
+4.714774087646149
+4.714774087646149
+4.714774087646149
+4.718625762219989
+4.793091516779753
+4.872529839105629
+4.900088766172104
+4.863636363636363
+4.740838689292398
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.721541071319574
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.797007948311492
+4.927710408327659
+4.927710408327659
+4.89873092730281
+4.740838689292398
+4.702367606407748
+4.702367606407748
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+3.3871959829706197
+0.38541578130400733
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.2406471531816594
+0.6130836407996663
+4.7047148084587604
+4.6862809068484905
+4.6862809068484905
+4.7047148084587604
+4.898823242738641
+4.838921850181822
+4.6862809068484905
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.6862809068484905
+4.6862809068484905
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.6862809068484905
+4.878241460327516
+4.979648002510261
+4.979648002510261
+4.779602577154106
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.4707363627284495
+3.3932655188344025
+0.64079515659542
+0.302207141430471
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.6323293455201187
+4.781042241478235
+4.858049419420977
+4.8584171544605805
+4.748902697275286
+4.7342759124717695
+4.842936919059271
+4.845146974506639
+4.748902697275286
+4.714774087646149
+4.695350099572867
+4.695350099572867
+4.695350099572867
+4.718625762219989
+4.836483399978973
+4.8848078319332995
+4.8848078319332995
+4.826171391227169
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.702367606407748
+4.683327299089502
+4.683327299089502
+4.702367606407748
+4.702367606407748
+4.7576395655368735
+4.914400272933426
+4.927710408327659
+4.914400272933426
+4.7576395655368735
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+3.4635252048046143
+0.6299960845817676
+0.10651401691436746
+0.02884340803556551
+0.0
+0.0
+0.02884340803556551
+0.08652970395641724
+0.654849727357143
+4.4950100072473465
+4.6862809068484905
+4.6862809068484905
+4.6862809068484905
+4.723266788376083
+4.935591242636505
+4.8196627570631865
+4.6862809068484905
+4.6862809068484905
+4.6862809068484905
+4.687379768913745
+4.687379768913745
+4.687379768913745
+4.6862809068484905
+4.6862809068484905
+4.6862809068484905
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.741226342122529
+4.959350993808612
+4.979648002510261
+4.918935027068591
+4.6862809068484905
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.614029116375104
+4.614029116375104
+4.46955485671531
+3.3971231599860086
+0.6391238293857269
+0.2060605753082827
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.08652970395641724
+0.5684107529392541
+3.4426681588491452
+4.7342759124717695
+4.79984516624872
+4.858049419420977
+4.79984516624872
+4.714774087646149
+4.76833752096446
+4.858049419420977
+4.826446087163245
+4.714774087646149
+4.695350099572867
+4.695350099572867
+4.695350099572867
+4.676017577221279
+4.729454156416239
+4.8584171544605805
+4.8848078319332995
+4.872529839105629
+4.773452659709149
+4.702367606407748
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.740838689292398
+4.89873092730281
+4.927710408327659
+4.927710408327659
+4.7772844700957515
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.50990487550728
+0.7108893910022562
+0.14421518948927226
+0.0576866479940108
+0.0
+0.0
+0.0576866479940108
+0.3099984596405294
+0.5941860158497647
+4.779773978458947
+4.6862809068484905
+4.6862809068484905
+4.6862809068484905
+4.760369351195106
+4.935591242636505
+4.779602577154106
+4.6862809068484905
+4.6862809068484905
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.6862809068484905
+4.6862809068484905
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.858680873820008
+4.979648002510261
+4.979648002510261
+4.798912023055375
+4.649828001134684
+4.649828001134684
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.462504916850652
+0.6063959649582049
+0.37691273948061976
+0.02884340803556551
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.2406471531816594
+0.6213226895949919
+4.817997864018289
+4.742906577502044
+4.729454156416239
+4.79984516624872
+4.809208233051305
+4.714774087646149
+4.714774087646149
+4.818365174589247
+4.858049419420977
+4.76833752096446
+4.695350099572867
+4.695350099572867
+4.676017577221279
+4.676017577221279
+4.676017577221279
+4.781042241478235
+4.858049419420977
+4.8848078319332995
+4.870754749200952
+4.718625762219989
+4.680045399669822
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.702367606407748
+4.871259604324209
+4.927710408327659
+4.927710408327659
+4.812732846572471
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+0.6263147453996627
+0.2595829718925291
+0.0576866479940108
+0.0
+0.02884340803556551
+0.10651401691436746
+0.6688907068459828
+4.5272956191935645
+4.720184565238619
+4.684462903548131
+4.684462903548131
+4.6862809068484905
+4.798912023055375
+4.935591242636505
+4.741226342122529
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.649828001134684
+4.723266788376083
+4.939112184045445
+4.979648002510261
+4.918935027068591
+4.6862809068484905
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.596414173087077
+4.6862809068484905
+3.401787032682269
+0.4134958711015573
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.11537255792817191
+0.6801637532859157
+4.5
+4.81729793871423
+4.781970581815818
+4.71001832849629
+4.71001832849629
+4.729454156416239
+4.695350099572867
+4.695350099572867
+4.729454156416239
+4.845146974506639
+4.842936919059271
+4.714774087646149
+4.676017577221279
+4.676017577221279
+4.676017577221279
+4.676017577221279
+4.695350099572867
+4.826446087163245
+4.858049419420977
+4.8848078319332995
+4.807023268125679
+4.680045399669822
+4.680045399669822
+4.680045399669822
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.6644299631397725
+4.812732846572471
+4.914400272933426
+4.927710408327659
+4.871259604324209
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.683327299089502
+4.681900682162952
+4.681900682162952
+3.3936590485265534
+0.5102083351063529
+0.08652970395641724
+0.02884340803556551
+0.0576866479940108
+0.26944739896229564
+0.6039406390696822
+4.90900814782395
+4.700980573576612
+4.681900682162952
+4.681900682162952
+4.684462903548131
+4.838921850181822
+4.935591242636505
+4.722186242078544
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.667980307053825
+4.667980307053825
+4.667980307053825
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.839146489353515
+4.979648002510261
+4.979648002510261
+4.800247028092315
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.596414173087077
+4.6862809068484905
+4.898823242738641
+4.979648002510261
+0.6007651230049555
+0.3016368135971197
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.08652970395641724
+0.6361009339160937
+3.424880372594676
+4.754700337540463
+4.81729793871423
+4.81729793871423
+4.754700337540463
+4.690617259906991
+4.690617259906991
+4.690617259906991
+4.695350099572867
+4.695350099572867
+4.781042241478235
+4.858049419420977
+4.781042241478235
+4.676017577221279
+4.676017577221279
+4.676017577221279
+4.676017577221279
+4.656789715712701
+4.729454156416239
+4.845146974506639
+4.858049419420977
+4.845146974506639
+4.7342759124717695
+4.680045399669822
+4.680045399669822
+4.680045399669822
+4.683327299089502
+4.6644299631397725
+4.6644299631397725
+4.7576395655368735
+4.914400272933426
+4.914400272933426
+4.890373906609362
+4.702367606407748
+4.683327299089502
+4.683327299089502
+4.681900682162952
+4.681900682162952
+4.681900682162952
+4.681900682162952
+4.4850662655980855
+0.7006178728830168
+0.11537255792817191
+0.0576866479940108
+0.10651401691436746
+0.6027146065606575
+3.451331861236609
+4.890373906609362
+4.681900682162952
+4.681900682162952
+4.681900682162952
+4.681900682162952
+4.851876601914066
+4.916065859983668
+4.7047148084587604
+4.667980307053825
+4.667980307053825
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.7047148084587604
+4.939112184045445
+4.979648002510261
+4.939112184045445
+4.6862809068484905
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.596414173087077
+4.594488612832219
+4.684268621444632
+4.918935027068591
+4.979648002510261
+4.798912023055375
+4.4578260227203375
+0.6748309658588463
+0.11537255792817191
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.27722759130246555
+0.6389578534659792
+3.4502024765421035
+4.735962024272876
+4.690617259906991
+4.781970581815818
+4.81729793871423
+4.803471126886287
+4.685126324463377
+4.671270154154301
+4.690617259906991
+4.690617259906991
+4.671270154154301
+4.690617259906991
+4.826446087163245
+4.845146974506639
+4.729454156416239
+4.676017577221279
+4.656789715712701
+4.656789715712701
+4.656789715712701
+4.676017577221279
+4.791416189934912
+4.858049419420977
+4.858049419420977
+4.809208233051305
+4.676017577221279
+4.660943261085074
+4.660943261085074
+4.660943261085074
+4.660943261085074
+4.6644299631397725
+4.718625762219989
+4.89873092730281
+4.914400272933426
+4.89873092730281
+4.718625762219989
+4.662955652364289
+4.681900682162952
+4.681900682162952
+4.681900682162952
+4.681900682162952
+4.681900682162952
+4.681900682162952
+0.6076346759138609
+0.2790850420626718
+0.11537255792817191
+0.17305757537147226
+0.6657145261825592
+4.765366729850828
+4.851876601914066
+4.681900682162952
+4.681900682162952
+4.662955652364289
+4.681900682162952
+4.890373906609362
+4.890373906609362
+4.662955652364289
+4.647295911553856
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.649828001134684
+4.8196627570631865
+4.979648002510261
+4.979648002510261
+4.8196627570631865
+4.631839101577405
+4.631839101577405
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.596414173087077
+4.594488612832219
+4.594488612832219
+4.6862809068484905
+4.918935027068591
+4.979648002510261
+4.779602577154106
+4.594488612832219
+4.577072911270099
+3.3871856774329334
+0.4164115895697904
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.07802269801585293
+0.3200076833933503
+0.6284694025551278
+3.3962455403760017
+4.522735510237164
+4.791447617972283
+4.760933045899426
+4.665846151051314
+4.685126324463377
+4.803471126886287
+4.81729793871423
+4.764423495930857
+4.671270154154301
+4.671270154154301
+4.6519944403256295
+4.671270154154301
+4.671270154154301
+4.742906577502044
+4.845146974506639
+4.809208233051305
+4.676017577221279
+4.656789715712701
+4.656789715712701
+4.656789715712701
+4.656789715712701
+4.71001832849629
+4.845146974506639
+4.858049419420977
+4.858049419420977
+4.742906577502044
+4.63767970403356
+4.63767970403356
+4.660943261085074
+4.660943261085074
+4.660943261085074
+4.680045399669822
+4.863636363636363
+4.914400272933426
+4.914400272933426
+4.7367234133223
+4.662955652364289
+4.662955652364289
+4.681900682162952
+4.681900682162952
+4.684462903548131
+4.681900682162952
+4.681900682162952
+3.4066666666666667
+0.5470912582584093
+0.17305757537147226
+0.3367206068372681
+0.598100425917039
+4.927710408327659
+4.832343613752159
+4.662955652364289
+4.662955652364289
+4.662955652364289
+4.700980573576612
+4.89873092730281
+4.851876601914066
+4.644156904882541
+4.644156904882541
+4.647295911553856
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.649828001134684
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.665803664493659
+4.918935027068591
+4.979648002510261
+4.939112184045445
+4.6862809068484905
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.596414173087077
+4.594488612832219
+4.594488612832219
+4.684462903548131
+4.918935027068591
+4.959350993808612
+4.779602577154106
+4.594488612832219
+4.577072911270099
+4.647885375522203
+4.624023941631759
+0.6441774988331241
+0.17305757537147226
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17811199232491148
+0.40250640871114296
+0.6209395623165124
+0.612528122475215
+3.4121573149713003
+4.5225662367935255
+4.672145963635914
+4.672145963635914
+4.74974235645692
+4.791447617972283
+4.709682610573851
+4.646596735835308
+4.698038989919562
+4.804805898398897
+4.804805898398897
+4.704414572748163
+4.6519944403256295
+4.6519944403256295
+4.6519944403256295
+4.6519944403256295
+4.671270154154301
+4.781970581815818
+4.843246627448329
+4.76204169956615
+4.656789715712701
+4.656789715712701
+4.656789715712701
+4.656789715712701
+4.656789715712701
+4.781042241478235
+4.858049419420977
+4.858049419420977
+4.826446087163245
+4.676017577221279
+4.63767970403356
+4.63767970403356
+4.63767970403356
+4.660943261085074
+4.660943261085074
+4.826171391227169
+4.900088766172104
+4.914400272933426
+4.797007948311492
+4.662955652364289
+4.662955652364289
+4.665803664493659
+4.662955652364289
+4.681900682162952
+4.681900682162952
+4.681900682162952
+4.47682675213889
+0.7302790609412355
+0.22140021244840558
+0.5660760801461455
+3.3996686346662637
+4.927710408327659
+4.797007948311492
+4.662955652364289
+4.662955652364289
+4.662955652364289
+4.7367234133223
+4.914400272933426
+4.832343613752159
+4.644156904882541
+4.644156904882541
+4.644156904882541
+4.644156904882541
+4.647295911553856
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.614029116375104
+4.779773978458947
+4.954466537076697
+4.954466537076697
+4.800247028092315
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.596414173087077
+4.594488612832219
+4.594488612832219
+4.594488612832219
+4.703261124315225
+4.918935027068591
+4.959350993808612
+4.760369351195106
+4.577072911270099
+4.594488612832219
+4.722186242078544
+4.918935027068591
+4.979648002510261
+4.5078487634365185
+0.6635237943349184
+0.24229536681363495
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.10654702030214391
+0.35504606621873336
+0.5918444305940929
+0.6175436328753916
+3.4041864441530785
+3.4488877508772764
+4.506066750912535
+4.653211143840427
+4.734199167934266
+4.753120216906312
+4.710759302443869
+4.640648550355574
+4.672145963635914
+4.74974235645692
+4.74473950518707
+4.646596735835308
+4.646596735835308
+4.709682610573851
+4.804805898398897
+4.746468359034328
+4.6519944403256295
+4.6519944403256295
+4.6519944403256295
+4.6519944403256295
+4.6519944403256295
+4.685126324463377
+4.817997864018289
+4.817997864018289
+4.690617259906991
+4.656789715712701
+4.656789715712701
+4.656789715712701
+4.63767970403356
+4.671270154154301
+4.842936919059271
+4.858049419420977
+4.858049419420977
+4.76204169956615
+4.63767970403356
+4.63767970403356
+4.63767970403356
+4.63767970403356
+4.640410514489808
+4.748902697275286
+4.8848078319332995
+4.8848078319332995
+4.826171391227169
+4.662955652364289
+4.662955652364289
+4.662955652364289
+4.662955652364289
+4.662955652364289
+4.681900682162952
+4.681900682162952
+4.681900682162952
+0.6956563972819012
+0.30777693354095703
+0.7152110594945071
+3.4769566928057696
+4.927710408327659
+4.775931851598417
+4.662955652364289
+4.662955652364289
+4.644156904882541
+4.756290519524092
+4.927710408327659
+4.797007948311492
+4.644156904882541
+4.644156904882541
+4.644156904882541
+4.625516671887187
+4.625516671887187
+4.625516671887187
+4.62895277070343
+4.631839101577405
+4.631839101577405
+4.631839101577405
+4.614029116375104
+4.614029116375104
+4.647295911553856
+4.896292145056855
+4.954466537076697
+4.916065859983668
+4.665803664493659
+4.596414173087077
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.596414173087077
+4.594488612832219
+4.594488612832219
+4.594488612832219
+4.594488612832219
+4.703261124315225
+4.939112184045445
+4.959350993808612
+4.760369351195106
+4.577072911270099
+4.647885375522203
+4.818281302257531
+4.959350993808612
+4.979648002510261
+4.918935027068591
+4.703261124315225
+4.442054446236271
+3.3900069940231603
+0.6493211432484394
+0.502381006373251
+0.19918352929138816
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.17811199232491148
+0.5838932072864029
+0.6060061846830029
+3.4270860452442005
+4.534349385336478
+4.728397247583595
+4.753120216906312
+4.753120216906312
+4.693155811682875
+4.640648550355574
+4.664824166136957
+4.752498445713435
+4.752498445713435
+4.653211143840427
+4.621503709837636
+4.653211143840427
+4.710759302443869
+4.659803272550208
+4.62739859759647
+4.62739859759647
+4.719749765469021
+4.760933045899426
+4.646596735835308
+4.6519944403256295
+4.6519944403256295
+4.6519944403256295
+4.6328065440738575
+4.6328065440738575
+4.735962024272876
+4.828385320427853
+4.781970581815818
+4.6519944403256295
+4.656789715712701
+4.63767970403356
+4.63767970403356
+4.63767970403356
+4.742906577502044
+4.858049419420977
+4.858049419420977
+4.842936919059271
+4.671270154154301
+4.63767970403356
+4.636068948163684
+4.636068948163684
+4.636068948163684
+4.689147125339842
+4.872529839105629
+4.8848078319332995
+4.8540296586129035
+4.678585478412721
+4.662955652364289
+4.662955652364289
+4.662955652364289
+4.662955652364289
+4.662955652364289
+4.662955652364289
+4.681900682162952
+0.6330885343407324
+0.40378354047257137
+0.779888710483915
+4.508833305020784
+4.927710408327659
+4.756290519524092
+4.644156904882541
+4.644156904882541
+4.644156904882541
+4.793091516779753
+4.927710408327659
+4.775931851598417
+4.644156904882541
+4.644156904882541
+4.625516671887187
+4.625516671887187
+4.625516671887187
+4.625516671887187
+4.625516671887187
+4.625516671887187
+4.610788162315073
+4.614029116375104
+4.614029116375104
+4.614029116375104
+4.779773978458947
+4.954466537076697
+4.954466537076697
+4.778439516729418
+4.596414173087077
+4.596414173087077
+4.596414173087077
+4.596414173087077
+4.614029116375104
+4.596414173087077
+4.596414173087077
+4.594488612832219
+4.594488612832219
+4.594488612832219
+4.594488612832219
+4.722186242078544
+4.939112184045445
+4.959350993808612
+4.741226342122529
+4.612107748250274
+4.741226342122529
+4.918935027068591
+4.979648002510261
+4.979648002510261
+4.898823242738641
+4.66600971208682
+4.5598793734488305
+4.5598793734488305
+4.5598793734488305
+4.4958606434723105
+3.4909161000893336
+3.395088895732531
+0.6229909793731245
+0.22937674882331027
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.07802269801585293
+0.5168240056601512
+0.5957551901477602
+4.499558173439981
+4.716549695711502
+4.715349243520801
+4.675060389383624
+4.6667668946905065
+4.726826567571509
+4.740587601014347
+4.740409145179578
+4.693155811682875
+4.621503709837636
+4.693155811682875
+4.753120216906312
+4.718039453057842
+4.621503709837636
+4.621503709837636
+4.621503709837636
+4.621503709837636
+4.621503709837636
+4.621503709837636
+4.646596735835308
+4.709682610573851
+4.646596735835308
+4.62739859759647
+4.6328065440738575
+4.6328065440738575
+4.6328065440738575
+4.6328065440738575
+4.6519944403256295
+4.7879891805359165
+4.81729793871423
+4.717058602539717
+4.6328065440738575
+4.6328065440738575
+4.63767970403356
+4.63767970403356
+4.656789715712701
+4.826446087163245
+4.858049419420977
+4.858049419420977
+4.76204169956615
+4.636068948163684
+4.636068948163684
+4.636068948163684
+4.636068948163684
+4.655242633831636
+4.826446087163245
+4.869137689341459
+4.872529839105629
+4.713366915704603
+4.6594318456833586
+4.6594318456833586
+4.662955652364289
+4.662955652364289
+4.662955652364289
+4.662955652364289
+4.662955652364289
+3.3739082146644273
+0.6058076208999745
+0.7084001799010258
+4.6072998258226505
+4.914400272933426
+4.7367234133223
+4.644156904882541
+4.644156904882541
+4.644156904882541
+4.851876601914066
+4.927710408327659
+4.7367234133223
+4.625516671887187
+4.625516671887187
+4.625516671887187
+4.625516671887187
+4.625516671887187
+4.625516671887187
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.610788162315073
+4.614029116375104
+4.647295911553856
+4.896292145056855
+4.954466537076697
+4.896292145056855
+4.647295911553856
+4.596414173087077
+4.596414173087077
+4.596414173087077
+4.596414173087077
+4.596414173087077
+4.594488612832219
+4.594488612832219
+4.594488612832219
+4.594488612832219
+4.594488612832219
+4.722186242078544
+4.939112184045445
+4.959350993808612
+4.760369351195106
+4.703261124315225
+4.878241460327516
+4.979648002510261
+4.979648002510261
+4.979648002510261
+4.837689816634932
+4.629912424656536
+4.5598793734488305
+4.5598793734488305
+4.5598793734488305
+4.5598793734488305
+4.5598793734488305
+4.665803664493659
+4.898823242738641
+4.599546042034984
+0.5987856305456649
+0.3200076833933503
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.12563154621256611
+0.640989063402218
+3.446276131575725
+4.7039874056290465
+4.7039874056290465
+4.701532155459308
+4.657071354732281
+4.6084885702460685
+4.6084885702460685
+4.638697904486414
+4.701532155459308
+4.728275832763921
+4.740409145179578
+4.675240430792547
+4.640648550355574
+4.710759302443869
+4.740409145179578
+4.653211143840427
+4.602391518498605
+4.602391518498605
+4.602391518498605
+4.602391518498605
+4.621503709837636
+4.621503709837636
+4.621503709837636
+4.62739859759647
+4.608270933569005
+4.608270933569005
+4.608270933569005
+4.6328065440738575
+4.6328065440738575
+4.6328065440738575
+4.698038989919562
+4.804805898398897
+4.803471126886287
+4.665846151051314
+4.6328065440738575
+4.6328065440738575
+4.6328065440738575
+4.63767970403356
+4.742906577502044
+4.858049419420977
+4.858049419420977
+4.842936919059271
+4.669748954253991
+4.636068948163684
+4.636068948163684
+4.636068948163684
+4.636068948163684
+4.781042241478235
+4.858049419420977
+4.858049419420977
+4.747524644977324
+4.6594318456833586
+4.6594318456833586
+4.6594318456833586
+4.6594318456833586
+4.644156904882541
+4.644156904882541
+4.644156904882541
+4.464972813772094
+0.8092215597727375
+0.6578501295517025
+4.832343613752159
+4.914400272933426
+4.717242610399388
+4.644156904882541
+4.644156904882541
+4.662955652364289
+4.871259604324209
+4.914400272933426
+4.717242610399388
+4.625516671887187
+4.625516671887187
+4.625516671887187
+4.625516671887187
+4.625516671887187
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.758924030677352
+4.954466537076697
+4.954466537076697
+4.760369351195106
+4.596414173087077
+4.596414173087077
+4.596414173087077
+4.596414173087077
+4.577072911270099
+4.577072911270099
+4.577072911270099
+4.577072911270099
+4.594488612832219
+4.594488612832219
+4.722186242078544
+4.939112184045445
+4.959350993808612
+4.839146489353515
+4.858680873820008
+4.959350993808612
+4.979648002510261
+4.979648002510261
+4.959350993808612
+4.798912023055375
+4.594488612832219
+4.5598793734488305
+4.5598793734488305
+4.5598793734488305
+4.5598793734488305
+4.5598793734488305
+4.703261124315225
+4.939112184045445
+4.979648002510261
+4.979648002510261
+4.7370538152010715
+0.602084246074603
+0.3249276534532015
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.1351645434877229
+0.6344548999581567
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8360663886365511
+0.8647921722156005
+0.9219773019841471
+0.6742277222771449
+4.4974714903326305
+4.69281418415078
+4.7039874056290465
+4.7039874056290465
+4.69066493212302
+4.6580308995468975
+4.6084885702460685
+4.6084885702460685
+4.589568314126108
+4.620162389412384
+4.697448484045342
+4.728275832763921
+4.728397247583595
+4.675240430792547
+4.621503709837636
+4.693155811682875
+4.664824166136957
+4.602391518498605
+4.602391518498605
+4.602391518498605
+4.602391518498605
+4.602391518498605
+4.602391518498605
+4.602391518498605
+4.602391518498605
+4.608270933569005
+4.608270933569005
+4.608270933569005
+4.608270933569005
+4.608270933569005
+4.613722468658828
+4.6328065440738575
+4.764423495930857
+4.81729793871423
+4.764423495930857
+4.6328065440738575
+4.613722468658828
+4.6328065440738575
+4.6328065440738575
+4.650413336097011
+4.826446087163245
+4.858049419420977
+4.858049419420977
+4.741519592641198
+4.636068948163684
+4.636068948163684
+4.636068948163684
+4.636068948163684
+4.7280566877824075
+4.858049419420977
+4.858049419420977
+4.781042241478235
+4.636068948163684
+4.6594318456833586
+4.6594318456833586
+4.640410514489808
+4.640410514489808
+4.644156904882541
+4.644156904882541
+4.644156904882541
+3.367293261192396
+3.3772367930894895
+4.851876601914066
+4.89873092730281
+4.700980573576612
+4.644156904882541
+4.644156904882541
+4.681900682162952
+4.89873092730281
+4.914400272933426
+4.681900682162952
+4.625516671887187
+4.625516671887187
+4.625516671887187
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.644156904882541
+4.890373906609362
+4.927710408327659
+4.876422053888234
+4.631839101577405
+4.596414173087077
+4.596414173087077
+4.577072911270099
+4.577072911270099
+4.577072911270099
+4.577072911270099
+4.577072911270099
+4.5598793734488305
+4.577072911270099
+4.722186242078544
+4.959350993808612
+4.979648002510261
+4.959350993808612
+4.959350993808612
+4.979648002510261
+4.979648002510261
+4.979648002510261
+4.959350993808612
+4.741226342122529
+4.577072911270099
+4.5598793734488305
+4.5598793734488305
+4.5598793734488305
+4.542927748567686
+4.577072911270099
+4.760369351195106
+4.959350993808612
+4.979648002510261
+4.939112184045445
+4.741226342122529
+4.557739919850564
+4.455161226213266
+0.6152045717011259
+0.27346584048501477
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.14421518948927226
+1.3457522347649986
+2.0609275239094176
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.140313153331892
+2.1491682045832015
+2.173019078934724
+2.4622677624114453
+4.475400723987205
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.7039874056290465
+4.7039874056290465
+4.665703966605539
+4.601501635773668
+4.589568314126108
+4.589568314126108
+4.620162389412384
+4.697448484045342
+4.716549695711502
+4.728397247583595
+4.657071354732281
+4.602391518498605
+4.621503709837636
+4.602391518498605
+4.602391518498605
+4.602391518498605
+4.602391518498605
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.602391518498605
+4.602391518498605
+4.608270933569005
+4.608270933569005
+4.608270933569005
+4.589232276438436
+4.594758254119056
+4.665846151051314
+4.804805898398897
+4.804805898398897
+4.698038989919562
+4.613722468658828
+4.611996828309391
+4.631157180365171
+4.631157180365171
+4.715635347795578
+4.843246627448329
+4.858049419420977
+4.826446087163245
+4.655242633831636
+4.636068948163684
+4.636068948163684
+4.636068948163684
+4.669748954253991
+4.830804040442458
+4.858049419420977
+4.826446087163245
+4.655242633831636
+4.6170189734379745
+4.6170189734379745
+4.640410514489808
+4.640410514489808
+4.640410514489808
+4.640410514489808
+4.644156904882541
+4.644156904882541
+4.644156904882541
+4.851876601914066
+4.8816267976576375
+4.662955652364289
+4.625516671887187
+4.625516671887187
+4.697859544470582
+4.914400272933426
+4.89873092730281
+4.662955652364289
+4.625516671887187
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.588765585089522
+4.752475355734676
+4.927710408327659
+4.927710408327659
+4.7367234133223
+4.592816956402557
+4.57306015489294
+4.577072911270099
+4.577072911270099
+4.577072911270099
+4.577072911270099
+4.577072911270099
+4.5598793734488305
+4.5598793734488305
+4.720184565238619
+4.935591242636505
+4.954466537076697
+4.959350993808612
+4.979648002510261
+4.979648002510261
+4.979648002510261
+4.979648002510261
+4.918935027068591
+4.703261124315225
+4.5598793734488305
+4.5598793734488305
+4.5598793734488305
+4.5598793734488305
+4.542927748567686
+4.612107748250274
+4.837689816634932
+4.979648002510261
+4.979648002510261
+4.898823242738641
+4.663789262481713
+4.524135426742258
+4.524135426742258
+4.524135426742258
+4.418012974920095
+0.6258535861392627
+0.26251218737318904
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8156864016436192
+2.1277860562756055
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.261761202020505
+2.289181849292641
+3.569141620103143
+4.716549695711502
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.7039874056290465
+4.6870297623887245
+4.620162389412384
+4.570624651733303
+4.589568314126108
+4.601501635773668
+4.681832734435403
+4.716549695711502
+4.715349243520801
+4.6309464772963524
+4.596137437755187
+4.602391518498605
+4.602391518498605
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.602391518498605
+4.602391518498605
+4.583333333333333
+4.589232276438436
+4.589232276438436
+4.589232276438436
+4.589232276438436
+4.728197166242947
+4.81729793871423
+4.7879891805359165
+4.644983451029647
+4.611996828309391
+4.611996828309391
+4.611996828309391
+4.650413336097011
+4.803471126886287
+4.843246627448329
+4.858049419420977
+4.722273263744934
+4.636068948163684
+4.636068948163684
+4.636068948163684
+4.655242633831636
+4.809208233051305
+4.858049419420977
+4.845146974506639
+4.669748954253991
+4.6170189734379745
+4.6170189734379745
+4.6170189734379745
+4.640410514489808
+4.640410514489808
+4.640410514489808
+4.621534006797198
+4.621534006797198
+4.644156904882541
+4.871259604324209
+4.8816267976576375
+4.644156904882541
+4.607048139239902
+4.607048139239902
+4.717242610399388
+4.927710408327659
+4.871259604324209
+4.625516671887187
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.588765585089522
+4.588765585089522
+4.644156904882541
+4.8816267976576375
+4.927710408327659
+4.871259604324209
+4.607048139239902
+4.5686346228467
+4.5686346228467
+4.57306015489294
+4.577072911270099
+4.577072911270099
+4.5598793734488305
+4.5598793734488305
+4.57306015489294
+4.739502382932878
+4.935591242636505
+4.954466537076697
+4.954466537076697
+4.954466537076697
+4.954466537076697
+4.979648002510261
+4.979648002510261
+4.878780841567253
+4.647295911553856
+4.5598793734488305
+4.5598793734488305
+4.5598793734488305
+4.542927748567686
+4.542927748567686
+4.647295911553856
+4.898823242738641
+4.979648002510261
+4.979648002510261
+4.837689816634932
+4.594488612832219
+4.524135426742258
+4.524135426742258
+4.524135426742258
+4.524135426742258
+4.524135426742258
+4.404238805072985
+0.619900467181778
+0.39308664550572203
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7652212687958828
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.2710578765326823
+2.627737733120367
+4.62793792443204
+4.676655864461275
+4.679949111556313
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.701532155459308
+4.6309464772963524
+4.570624651733303
+4.5516827258491075
+4.582748243886765
+4.681832734435403
+4.716549695711502
+4.715349243520801
+4.6309464772963524
+4.577112015096106
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.564349777616885
+4.583333333333333
+4.583333333333333
+4.589232276438436
+4.589232276438436
+4.589232276438436
+4.621503709837636
+4.776441943741247
+4.81729793871423
+4.728197166242947
+4.611996828309391
+4.611996828309391
+4.611996828309391
+4.611996828309391
+4.715635347795578
+4.828385320427853
+4.828385320427853
+4.798991798454706
+4.655242633831636
+4.636068948163684
+4.636068948163684
+4.636068948163684
+4.76204169956615
+4.858049419420977
+4.858049419420977
+4.722273263744934
+4.6170189734379745
+4.6170189734379745
+4.6170189734379745
+4.6170189734379745
+4.6170189734379745
+4.621534006797198
+4.621534006797198
+4.621534006797198
+4.6594318456833586
+4.863636363636363
+4.871259604324209
+4.625516671887187
+4.607048139239902
+4.607048139239902
+4.7576395655368735
+4.927710408327659
+4.832343613752159
+4.625516671887187
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.588765585089522
+4.588765585089522
+4.588765585089522
+4.772087701603047
+4.927710408327659
+4.927710408327659
+4.717242610399388
+4.5686346228467
+4.5686346228467
+4.5686346228467
+4.5686346228467
+4.550742156987144
+4.555500056192404
+4.555500056192404
+4.57306015489294
+4.758924030677352
+4.935591242636505
+4.954466537076697
+4.954466537076697
+4.954466537076697
+4.954466537076697
+4.954466537076697
+4.954466537076697
+4.837439329515021
+4.608817106569179
+4.5598793734488305
+4.5598793734488305
+4.542927748567686
+4.542927748567686
+4.5598793734488305
+4.722186242078544
+4.939112184045445
+4.979648002510261
+4.959350993808612
+4.760369351195106
+4.557739919850564
+4.524135426742258
+4.524135426742258
+4.524135426742258
+4.524135426742258
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.411424193686267
+3.3919459957853
+0.6318804001419362
+0.2175816996331532
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+1.4498872085464836
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2631715931889724
+2.3059535643590703
+4.396434317608271
+4.669446132552235
+4.63606643082577
+4.613200121218277
+4.6562728773757405
+4.679949111556313
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69066493212302
+4.640652808961297
+4.5516827258491075
+4.5516827258491075
+4.589568314126108
+4.665703966605539
+4.716549695711502
+4.701532155459308
+4.620162389412384
+4.577112015096106
+4.577112015096106
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.583333333333333
+4.564349777616885
+4.564349777616885
+4.564349777616885
+4.564349777616885
+4.564349777616885
+4.583333333333333
+4.583333333333333
+4.587375226740174
+4.587375226740174
+4.670612366078799
+4.791447617972283
+4.791122580013401
+4.664297236339458
+4.611996828309391
+4.611996828309391
+4.611996828309391
+4.631157180365171
+4.7879891805359165
+4.828385320427853
+4.81729793871423
+4.683632848810578
+4.636068948163684
+4.636068948163684
+4.6170189734379745
+4.708589746017948
+4.858049419420977
+4.858049419420977
+4.781042241478235
+4.6170189734379745
+4.6170189734379745
+4.6170189734379745
+4.598107147542447
+4.598107147542447
+4.598107147542447
+4.598107147542447
+4.621534006797198
+4.655242633831636
+4.870754749200952
+4.836483399978973
+4.621534006797198
+4.607048139239902
+4.607048139239902
+4.793091516779753
+4.927710408327659
+4.793091516779753
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.607048139239902
+4.588765585089522
+4.588765585089522
+4.588765585089522
+4.588765585089522
+4.640410514489808
+4.8816267976576375
+4.927710408327659
+4.85029320494125
+4.586736589139169
+4.5686346228467
+4.5686346228467
+4.5686346228467
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.5686346228467
+4.756290519524092
+4.935591242636505
+4.954466537076697
+4.954466537076697
+4.954466537076697
+4.954466537076697
+4.954466537076697
+4.935591242636505
+4.778439516729418
+4.57306015489294
+4.538177492226159
+4.538177492226159
+4.542927748567686
+4.542927748567686
+4.594488612832219
+4.798912023055375
+4.959350993808612
+4.979648002510261
+4.898823242738641
+4.663789262481713
+4.524135426742258
+4.524135426742258
+4.524135426742258
+4.524135426742258
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.577072911270099
+4.818281302257531
+4.5663028571346675
+0.5995445676728011
+0.37691273948061976
+0.0576866479940108
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2018996889437794
+2.339414265862871
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.2753823920420055
+2.585740560405326
+4.576767997638424
+4.6356545149438695
+4.6356545149438695
+4.620764512081358
+4.585070500657054
+4.62297914451095
+4.676655864461275
+4.679949111556313
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.649129891979821
+4.570624651733303
+4.558125621117782
+4.570624651733303
+4.675060389383624
+4.716549695711502
+4.701532155459308
+4.601501635773668
+4.558125621117782
+4.577112015096106
+4.577112015096106
+4.583333333333333
+4.564349777616885
+4.564349777616885
+4.564349777616885
+4.564349777616885
+4.564349777616885
+4.564349777616885
+4.564349777616885
+4.564349777616885
+4.562343756184406
+4.581430992457462
+4.581430992457462
+4.606503407028626
+4.754870956928874
+4.791447617972283
+4.754870956928874
+4.606503407028626
+4.611996828309391
+4.611996828309391
+4.611996828309391
+4.715635347795578
+4.81729793871423
+4.81729793871423
+4.764423495930857
+4.611996828309391
+4.611996828309391
+4.6170189734379745
+4.655242633831636
+4.842936919059271
+4.858049419420977
+4.826446087163245
+4.636068948163684
+4.598107147542447
+4.598107147542447
+4.598107147542447
+4.598107147542447
+4.598107147542447
+4.598107147542447
+4.5793486723868595
+4.636068948163684
+4.872529839105629
+4.818365174589247
+4.602815691902671
+4.584269967083597
+4.607048139239902
+4.832343613752159
+4.914400272933426
+4.7576395655368735
+4.588765585089522
+4.588765585089522
+4.607048139239902
+4.588765585089522
+4.588765585089522
+4.588765585089522
+4.588765585089522
+4.588765585089522
+4.588765585089522
+4.772087701603047
+4.927710408327659
+4.914400272933426
+4.697859544470582
+4.5686346228467
+4.5686346228467
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.5686346228467
+4.772087701603047
+4.927710408327659
+4.942733045803132
+4.954466537076697
+4.954466537076697
+4.954466537076697
+4.954466537076697
+4.935591242636505
+4.720184565238619
+4.555500056192404
+4.52111316046745
+4.52111316046745
+4.52111316046745
+4.52111316046745
+4.626979007463358
+4.878780841567253
+4.979648002510261
+4.979648002510261
+4.837689816634932
+4.594488612832219
+4.524135426742258
+4.524135426742258
+4.524135426742258
+4.524135426742258
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.626979007463358
+4.876999764984348
+4.979648002510261
+4.979648002510261
+4.798912023055375
+3.386350825680307
+0.40992356387120665
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.547964844339119
+2.0275402225265795
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.261761202020505
+2.288299328844452
+3.5828241956895424
+4.557095632157165
+4.5799545547395955
+4.631830491259161
+4.6356545149438695
+4.626881615898075
+4.566659309953631
+4.573804961511842
+4.64015341288857
+4.679949111556313
+4.679949111556313
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.6562728773757405
+4.570624651733303
+4.53920052169893
+4.558125621117782
+4.6580308995468975
+4.716549695711502
+4.701532155459308
+4.601501635773668
+4.558125621117782
+4.558125621117782
+4.558125621117782
+4.564349777616885
+4.564349777616885
+4.564349777616885
+4.564349777616885
+4.564349777616885
+4.564349777616885
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.581430992457462
+4.581430992457462
+4.651613169058767
+4.778506435350758
+4.791447617972283
+4.670612366078799
+4.587375226740174
+4.611996828309391
+4.611996828309391
+4.644983451029647
+4.7879891805359165
+4.81729793871423
+4.803471126886287
+4.644983451029647
+4.611996828309391
+4.611996828309391
+4.611996828309391
+4.791416189934912
+4.858049419420977
+4.845146974506639
+4.650413336097011
+4.598107147542447
+4.598107147542447
+4.598107147542447
+4.598107147542447
+4.5793486723868595
+4.5793486723868595
+4.5793486723868595
+4.631157180365171
+4.858049419420977
+4.781042241478235
+4.584269967083597
+4.584269967083597
+4.621534006797198
+4.863636363636363
+4.914400272933426
+4.697859544470582
+4.588765585089522
+4.570684518657339
+4.570684518657339
+4.570684518657339
+4.588765585089522
+4.588765585089522
+4.588765585089522
+4.570684518657339
+4.640410514489808
+4.89873092730281
+4.927710408327659
+4.830860339288274
+4.586736589139169
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.586736589139169
+4.791710163401963
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.942733045803132
+4.954466537076697
+4.896292145056855
+4.681900682162952
+4.538177492226159
+4.52111316046745
+4.52111316046745
+4.52111316046745
+4.538177492226159
+4.700980573576612
+4.896292145056855
+4.971271205771669
+4.939112184045445
+4.741226342122529
+4.557739919850564
+4.524135426742258
+4.524135426742258
+4.524135426742258
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.524135426742258
+4.701147711411036
+4.939112184045445
+5.0
+4.939112184045445
+4.701147711411036
+4.507737436138109
+4.491641118521462
+0.6257802964995784
+0.22147900774992912
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.983771971076508
+2.2211465944647593
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.272607966463304
+2.4627138022106574
+4.522366093046264
+4.541142071059501
+4.498352535656417
+4.541142071059501
+4.612010239449515
+4.6356545149438695
+4.626881615898075
+4.5480985287219635
+4.517290388513929
+4.585070500657054
+4.662029905531306
+4.679949111556313
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.671919701176808
+4.563931631528289
+4.53920052169893
+4.558125621117782
+4.640652808961297
+4.7039874056290465
+4.701532155459308
+4.582748243886765
+4.53920052169893
+4.53920052169893
+4.53920052169893
+4.558125621117782
+4.564349777616885
+4.564349777616885
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.581430992457462
+4.710759302443869
+4.778088299149454
+4.760933045899426
+4.606503407028626
+4.587375226740174
+4.587375226740174
+4.611996828309391
+4.735962024272876
+4.81729793871423
+4.81729793871423
+4.717058602539717
+4.5929485459444255
+4.5929485459444255
+4.611996828309391
+4.717058602539717
+4.843246627448329
+4.858049419420977
+4.722273263744934
+4.598107147542447
+4.598107147542447
+4.598107147542447
+4.5793486723868595
+4.5793486723868595
+4.5793486723868595
+4.5793486723868595
+4.650413336097011
+4.858049419420977
+4.76204169956615
+4.560759765732997
+4.5659124333978545
+4.636068948163684
+4.8584171544605805
+4.872529839105629
+4.6594318456833586
+4.570684518657339
+4.570684518657339
+4.570684518657339
+4.570684518657339
+4.570684518657339
+4.570684518657339
+4.5686346228467
+4.5686346228467
+4.791710163401963
+4.927710408327659
+4.914400272933426
+4.6594318456833586
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.586736589139169
+4.791710163401963
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.871259604324209
+4.62349952932741
+4.52111316046745
+4.52111316046745
+4.52111316046745
+4.52111316046745
+4.57306015489294
+4.778439516729418
+4.935591242636505
+4.954466537076697
+4.876422053888234
+4.64212860845057
+4.518950034056942
+4.524135426742258
+4.524135426742258
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.557739919850564
+4.779602577154106
+4.959350993808612
+4.979648002510261
+4.876999764984348
+4.608817106569179
+4.491641118521462
+4.491641118521462
+4.491641118521462
+3.5280397530367322
+0.6614334550488712
+0.08652970395641724
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7147544722886581
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2583256956715583
+2.284853433531523
+3.434543599944379
+4.624042963856418
+4.596668132191088
+4.510643118126104
+4.498352535656417
+4.510643118126104
+4.588824563121978
+4.6356545149438695
+4.6050017018392575
+4.498352535656417
+4.498352535656417
+4.5361952346332135
+4.632170355178927
+4.666403422086015
+4.679949111556313
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.676655864461275
+4.575743797536524
+4.53920052169893
+4.53920052169893
+4.62297914451095
+4.7039874056290465
+4.6870297623887245
+4.582748243886765
+4.53920052169893
+4.53920052169893
+4.53920052169893
+4.556070973540779
+4.556070973540779
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.6134253459783015
+4.752498445713435
+4.778088299149454
+4.701674543591646
+4.587375226740174
+4.587375226740174
+4.587375226740174
+4.625712366844445
+4.803471126886287
+4.81729793871423
+4.771738592817006
+4.611996828309391
+4.574028932345933
+4.574028932345933
+4.644983451029647
+4.81729793871423
+4.828385320427853
+4.773211711786747
+4.598107147542447
+4.5793486723868595
+4.5793486723868595
+4.5793486723868595
+4.5793486723868595
+4.5793486723868595
+4.5793486723868595
+4.669748954253991
+4.858049419420977
+4.742906577502044
+4.560759765732997
+4.560759765732997
+4.650413336097011
+4.872529839105629
+4.8584171544605805
+4.6170189734379745
+4.5659124333978545
+4.570684518657339
+4.570684518657339
+4.570684518657339
+4.5528218213026905
+4.550742156987144
+4.550742156987144
+4.640410514489808
+4.89873092730281
+4.927710408327659
+4.811314015577191
+4.5686346228467
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.586736589139169
+4.811314015577191
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.811314015577191
+4.582193139573988
+4.51566364335077
+4.51566364335077
+4.52111316046745
+4.52111316046745
+4.626979007463358
+4.856526425443426
+4.954466537076697
+4.954466537076697
+4.798038464273939
+4.57306015489294
+4.502167670798816
+4.502167670798816
+4.502167670798816
+4.502167670798816
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.507737436138109
+4.608817106569179
+4.857210833107264
+4.979648002510261
+4.979648002510261
+4.798912023055375
+4.557739919850564
+4.491641118521462
+4.491641118521462
+4.475872421177181
+4.475872421177181
+4.475872421177181
+0.6272847219477233
+0.2018996889437794
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+1.53524057630655
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.2694046972337256
+2.3917108314050513
+4.328370131613414
+4.548238985236116
+4.590893017970352
+4.553310052157446
+4.491801074676181
+4.4794059114735
+4.4794059114735
+4.552289840148148
+4.612010239449515
+4.541142071059501
+4.498352535656417
+4.498352535656417
+4.517290388513929
+4.566659309953631
+4.6562728773757405
+4.6562728773757405
+4.586925143439678
+4.671919701176808
+4.69281418415078
+4.69281418415078
+4.676655864461275
+4.594289814835405
+4.513899622194286
+4.513899622194286
+4.62297914451095
+4.7039874056290465
+4.6870297623887245
+4.582748243886765
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.556070973540779
+4.556070973540779
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.655506333614664
+4.753120216906312
+4.752498445713435
+4.6197826327175235
+4.5683462678453175
+4.5683462678453175
+4.5683462678453175
+4.719749765469021
+4.81729793871423
+4.804805898398897
+4.644983451029647
+4.574028932345933
+4.574028932345933
+4.5929485459444255
+4.7879891805359165
+4.828385320427853
+4.803471126886287
+4.611996828309391
+4.5793486723868595
+4.5793486723868595
+4.5793486723868595
+4.5793486723868595
+4.5793486723868595
+4.5793486723868595
+4.689147125339842
+4.858049419420977
+4.723684925163794
+4.560759765732997
+4.560759765732997
+4.683632848810578
+4.858049419420977
+4.826446087163245
+4.5793486723868595
+4.5659124333978545
+4.5659124333978545
+4.5659124333978545
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.772087701603047
+4.927710408327659
+4.89873092730281
+4.640410514489808
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.60503037786163
+4.830860339288274
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.772087701603047
+4.550742156987144
+4.51566364335077
+4.51566364335077
+4.51566364335077
+4.533078274185074
+4.697859544470582
+4.916065859983668
+4.954466537076697
+4.935591242636505
+4.720184565238619
+4.536004443501877
+4.502167670798816
+4.502167670798816
+4.502167670798816
+4.485682117209293
+4.485682117209293
+4.485682117209293
+4.507737436138109
+4.491641118521462
+4.507737436138109
+4.682407358487153
+4.918935027068591
+5.0
+4.939112184045445
+4.701147711411036
+4.507737436138109
+4.491641118521462
+4.475872421177181
+4.475872421177181
+4.475872421177181
+4.475872421177181
+4.475872421177181
+3.3796199770798063
+0.47265059722130953
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3172647300558147
+2.275992453631977
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2583256956715583
+2.281922839209146
+3.3931182886259084
+4.491801074676181
+4.482822888508683
+4.5461808736533715
+4.577601706931892
+4.51854399108187
+4.47291585209415
+4.4794059114735
+4.4794059114735
+4.504058708979198
+4.522663592490754
+4.498352535656417
+4.498352535656417
+4.4794059114735
+4.4794059114735
+4.517290388513929
+4.523942749265335
+4.513899622194286
+4.545078913708497
+4.6562728773757405
+4.69281418415078
+4.69281418415078
+4.69066493212302
+4.594289814835405
+4.513899622194286
+4.513899622194286
+4.62297914451095
+4.7039874056290465
+4.671919701176808
+4.568639765064591
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.556070973540779
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.575166900574203
+4.723400793471849
+4.753120216906312
+4.693155811682875
+4.562343756184406
+4.562343756184406
+4.5683462678453175
+4.6390051220405155
+4.778506435350758
+4.803215945464247
+4.728197166242947
+4.574028932345933
+4.574028932345933
+4.574028932345933
+4.728197166242947
+4.81729793871423
+4.804805898398897
+4.644983451029647
+4.574028932345933
+4.574028932345933
+4.5793486723868595
+4.5793486723868595
+4.560759765732997
+4.560759765732997
+4.704414572748163
+4.858049419420977
+4.702967088304484
+4.560759765732997
+4.54235790142653
+4.702967088304484
+4.869137689341459
+4.773211711786747
+4.54235790142653
+4.547760071805961
+4.545622089347077
+4.545622089347077
+4.545622089347077
+4.545622089347077
+4.550742156987144
+4.640410514489808
+4.89873092730281
+4.914400272933426
+4.772087701603047
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.550742156987144
+4.60503037786163
+4.830860339288274
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.914400272933426
+4.717242610399388
+4.533078274185074
+4.51566364335077
+4.51566364335077
+4.51566364335077
+4.5686346228467
+4.772087701603047
+4.914400272933426
+4.927710408327659
+4.871259604324209
+4.64212860845057
+4.502167670798816
+4.502167670798816
+4.502167670798816
+4.485682117209293
+4.485682117209293
+4.485682117209293
+4.485682117209293
+4.485682117209293
+4.4695199194396285
+4.518950034056942
+4.760369351195106
+4.959350993808612
+4.979648002510261
+4.876999764984348
+4.608817106569179
+4.491641118521462
+4.475872421177181
+4.475872421177181
+4.475872421177181
+4.475872421177181
+4.475872421177181
+4.460458853299791
+4.460458853299791
+3.5760771683002512
+0.7121090555268879
+0.11537255792817191
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6632793931627319
+2.100723074504167
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.2694046972337256
+2.3589310087373994
+4.304317460058428
+4.491801074676181
+4.463924095533918
+4.482822888508683
+4.555438561670694
+4.5853224209629255
+4.51854399108187
+4.444957838142898
+4.47291585209415
+4.4794059114735
+4.4794059114735
+4.4794059114735
+4.4794059114735
+4.4794059114735
+4.4794059114735
+4.4794059114735
+4.4794059114735
+4.504950980439749
+4.513899622194286
+4.526215661270152
+4.6236189421086475
+4.69281418415078
+4.69281418415078
+4.69066493212302
+4.586925143439678
+4.513899622194286
+4.511538266366625
+4.62297914451095
+4.7039874056290465
+4.681832734435403
+4.568639765064591
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.556070973540779
+4.556070973540779
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.562343756184406
+4.625661610351658
+4.753120216906312
+4.738328274948345
+4.594292603304557
+4.562343756184406
+4.562343756184406
+4.581430992457462
+4.74473950518707
+4.791447617972283
+4.760933045899426
+4.587375226740174
+4.555255275433425
+4.555255275433425
+4.65822873316879
+4.81729793871423
+4.81729793871423
+4.696574961377673
+4.555255275433425
+4.574028932345933
+4.574028932345933
+4.560759765732997
+4.560759765732997
+4.560759765732997
+4.723684925163794
+4.858049419420977
+4.664297236339458
+4.54235790142653
+4.54235790142653
+4.760666938236282
+4.858049419420977
+4.722273263744934
+4.5219170311723484
+4.5219170311723484
+4.5219170311723484
+4.545622089347077
+4.545622089347077
+4.545622089347077
+4.563809733321463
+4.805612835725778
+4.914400272933426
+4.8816267976576375
+4.600754584726002
+4.533078274185074
+4.533078274185074
+4.533078274185074
+4.60503037786163
+4.85029320494125
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.89873092730281
+4.678585478412721
+4.51566364335077
+4.51566364335077
+4.51566364335077
+4.51566364335077
+4.61947756989829
+4.851876601914066
+4.927710408327659
+4.927710408327659
+4.811314015577191
+4.582193139573988
+4.496296093052938
+4.502167670798816
+4.502167670798816
+4.485682117209293
+4.485682117209293
+4.485682117209293
+4.485682117209293
+4.485682117209293
+4.4695199194396285
+4.57306015489294
+4.837439329515021
+4.954466537076697
+4.954466537076697
+4.779602577154106
+4.540810622745219
+4.475872421177181
+4.475872421177181
+4.475872421177181
+4.475872421177181
+4.475872421177181
+4.475872421177181
+4.460458853299791
+4.460458853299791
+4.460458853299791
+4.37907711329853
+0.6436808059256123
+0.34662776824208574
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8553382116227111
+2.2211465944647593
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.261761202020505
+2.281922839209146
+3.3644985134317262
+4.479063031276068
+4.444957838142898
+4.463924095533918
+4.463924095533918
+4.457765303161006
+4.555438561670694
+4.5853224209629255
+4.500777606857457
+4.444957838142898
+4.4512597008547115
+4.454011608466691
+4.460474670771755
+4.460474670771755
+4.460474670771755
+4.4794059114735
+4.4794059114735
+4.4794059114735
+4.4794059114735
+4.504950980439749
+4.4859857040911795
+4.507366574133323
+4.632170355178927
+4.679949111556313
+4.69281418415078
+4.69066493212302
+4.603299912056848
+4.511538266366625
+4.511538266366625
+4.62297914451095
+4.7039874056290465
+4.671919701176808
+4.5495826726201285
+4.5180563038110195
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.556070973540779
+4.556070973540779
+4.562343756184406
+4.562343756184406
+4.543342744992878
+4.543342744992878
+4.562343756184406
+4.701360778922976
+4.753120216906312
+4.675240430792547
+4.543342744992878
+4.524448559224209
+4.543342744992878
+4.664824166136957
+4.778088299149454
+4.791447617972283
+4.651613169058767
+4.549435230703424
+4.555255275433425
+4.606503407028626
+4.791122580013401
+4.81729793871423
+4.764423495930857
+4.555255275433425
+4.536645877449656
+4.555255275433425
+4.555255275433425
+4.555255275433425
+4.560759765732997
+4.742906577502044
+4.858049419420977
+4.650413336097011
+4.54235790142653
+4.560759765732997
+4.791416189934912
+4.858049419420977
+4.664297236339458
+4.5219170311723484
+4.5219170311723484
+4.5219170311723484
+4.5219170311723484
+4.5219170311723484
+4.5276493769360435
+4.655242633831636
+4.872529839105629
+4.8848078319332995
+4.752475355734676
+4.533078274185074
+4.533078274185074
+4.51566364335077
+4.600754584726002
+4.851876601914066
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.927710408327659
+4.871259604324209
+4.640410514489808
+4.51566364335077
+4.51566364335077
+4.51566364335077
+4.533078274185074
+4.697859544470582
+4.89873092730281
+4.927710408327659
+4.914400272933426
+4.732895578140565
+4.530851648554603
+4.496296093052938
+4.496296093052938
+4.479437422437865
+4.479437422437865
+4.485682117209293
+4.485682117209293
+4.485682117209293
+4.4695199194396285
+4.485682117209293
+4.64212860845057
+4.896292145056855
+4.971271205771669
+4.916065859983668
+4.660902998104733
+4.485682117209293
+4.475872421177181
+4.475872421177181
+4.475872421177181
+4.475872421177181
+4.475872421177181
+4.460458853299791
+4.460458853299791
+4.460458853299791
+4.460458853299791
+4.460458853299791
+4.460458853299791
+4.404615629412585
+0.6039035554536563
+0.6252108790857731
+0.157275361322311
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+1.323460538475258
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.246067985320125
+2.2587488089142114
+2.276297393963519
+2.625116350348653
+4.451641543756818
+4.543452396465089
+4.470438427378001
+4.419975864651762
+4.444957838142898
+4.444957838142898
+4.457765303161006
+4.539257872104618
+4.5853224209629255
+4.512109974050057
+4.444957838142898
+4.425946698109906
+4.454011608466691
+4.460474670771755
+4.460474670771755
+4.460474670771755
+4.460474670771755
+4.4794059114735
+4.4794059114735
+4.460474670771755
+4.460474670771755
+4.467070908185214
+4.504950980439749
+4.614878214028467
+4.679949111556313
+4.69281418415078
+4.679949111556313
+4.603299912056848
+4.511538266366625
+4.511538266366625
+4.614878214028467
+4.69281418415078
+4.671919701176808
+4.5495826726201285
+4.5180563038110195
+4.5180563038110195
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.543342744992878
+4.543342744992878
+4.543342744992878
+4.606705029314043
+4.740409145179578
+4.738328274948345
+4.575166900574203
+4.524448559224209
+4.524448559224209
+4.575166900574203
+4.752498445713435
+4.778088299149454
+4.710759302443869
+4.549435230703424
+4.530661494648408
+4.549435230703424
+4.754870956928874
+4.81729793871423
+4.803471126886287
+4.5683462678453175
+4.536645877449656
+4.536645877449656
+4.536645877449656
+4.555255275433425
+4.555255275433425
+4.726800481807874
+4.858049419420977
+4.631157180365171
+4.54235790142653
+4.5793486723868595
+4.826446087163245
+4.858049419420977
+4.631157180365171
+4.5219170311723484
+4.5219170311723484
+4.5219170311723484
+4.5219170311723484
+4.503888260008801
+4.540163595729319
+4.79984516624872
+4.8848078319332995
+4.8584171544605805
+4.596006379601555
+4.5276493769360435
+4.51566364335077
+4.600754584726002
+4.851876601914066
+4.914400272933426
+4.914400272933426
+4.914400272933426
+4.927710408327659
+4.927710408327659
+4.832343613752159
+4.582193139573988
+4.51566364335077
+4.51566364335077
+4.51566364335077
+4.5686346228467
+4.791710163401963
+4.927710408327659
+4.927710408327659
+4.8816267976576375
+4.676464247475493
+4.513442545183218
+4.496296093052938
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.462893717258573
+4.4695199194396285
+4.518950034056942
+4.739502382932878
+4.935591242636505
+4.954466537076697
+4.837439329515021
+4.590838511188699
+4.453709526215379
+4.453709526215379
+4.453709526215379
+4.453709526215379
+4.475872421177181
+4.460458853299791
+4.460458853299791
+4.460458853299791
+4.460458853299791
+4.460458853299791
+4.460458853299791
+4.460458853299791
+4.458442905224089
+4.44343410958372
+4.44343410958372
+3.529378058130801
+0.6122074521481151
+0.5714120409432386
+0.1174721498628637
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.14421518948927226
+2.4239620560826447
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.261761202020505
+2.351362644082894
+3.4741062982310638
+4.506921741467265
+4.551801604129618
+4.550140350585692
+4.5350396637789485
+4.464612354157994
+4.419975864651762
+4.419975864651762
+4.444957838142898
+4.463924095533918
+4.529291803534183
+4.5853224209629255
+4.529291803534183
+4.438908795687557
+4.425946698109906
+4.432218388162097
+4.454011608466691
+4.460474670771755
+4.460474670771755
+4.460474670771755
+4.460474670771755
+4.460474670771755
+4.460474670771755
+4.460474670771755
+4.457759882887575
+4.4958606434723105
+4.605035406704348
+4.679949111556313
+4.679949111556313
+4.679949111556313
+4.603299912056848
+4.511538266366625
+4.530541158577594
+4.632170355178927
+4.69281418415078
+4.6562728773757405
+4.5495826726201285
+4.5180563038110195
+4.5180563038110195
+4.537026822756646
+4.537026822756646
+4.537026822756646
+4.5180563038110195
+4.5180563038110195
+4.5180563038110195
+4.537026822756646
+4.537026822756646
+4.543342744992878
+4.543342744992878
+4.684246317290842
+4.753120216906312
+4.657071354732281
+4.524448559224209
+4.524448559224209
+4.524448559224209
+4.701360778922976
+4.764851533139823
+4.752498445713435
+4.581430992457462
+4.530661494648408
+4.530661494648408
+4.683348043725572
+4.803215945464247
+4.804805898398897
+4.6390051220405155
+4.536645877449656
+4.536645877449656
+4.536645877449656
+4.518220365758712
+4.536645877449656
+4.746468359034328
+4.81729793871423
+4.606503407028626
+4.5219170311723484
+4.590812073003308
+4.830804040442458
+4.830804040442458
+4.590812073003308
+4.5219170311723484
+4.5219170311723484
+4.5219170311723484
+4.503888260008801
+4.503888260008801
+4.644983451029647
+4.858049419420977
+4.858049419420977
+4.722273263744934
+4.50991243088939
+4.50991243088939
+4.6149276187135
+4.836483399978973
+4.914400272933426
+4.914400272933426
+4.914400272933426
+4.914400272933426
+4.914400272933426
+4.791710163401963
+4.5686346228467
+4.51566364335077
+4.51566364335077
+4.51566364335077
+4.61947756989829
+4.85029320494125
+4.927710408327659
+4.927710408327659
+4.85029320494125
+4.600754584726002
+4.496296093052938
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.462893717258573
+4.462893717258573
+4.462893717258573
+4.545622089347077
+4.815381064171493
+4.954466537076697
+4.935591242636505
+4.758924030677352
+4.518950034056942
+4.453709526215379
+4.453709526215379
+4.453709526215379
+4.453709526215379
+4.453709526215379
+4.453709526215379
+4.4382813720654735
+4.460458853299791
+4.460458853299791
+4.460458853299791
+4.460458853299791
+4.458442905224089
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.44343410958372
+3.4943066622976655
+0.6279912341196221
+0.3486431132071166
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3749445413935266
+2.1701297193934703
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.261761202020505
+2.386513800755304
+4.360589293829445
+4.4642115202475745
+4.489080743021098
+4.530776542855393
+4.550140350585692
+4.550140350585692
+4.543029917126171
+4.47698040719883
+4.419975864651762
+4.419975864651762
+4.419975864651762
+4.425946698109906
+4.500777606857457
+4.5853224209629255
+4.555438561670694
+4.457765303161006
+4.425946698109906
+4.432218388162097
+4.432218388162097
+4.435112702591826
+4.441583484616433
+4.441583484616433
+4.441583484616433
+4.457759882887575
+4.457759882887575
+4.457759882887575
+4.457759882887575
+4.4768097253305825
+4.605035406704348
+4.679949111556313
+4.679949111556313
+4.666403422086015
+4.585070500657054
+4.511538266366625
+4.530541158577594
+4.632170355178927
+4.69281418415078
+4.6562728773757405
+4.5495826726201285
+4.5180563038110195
+4.5180563038110195
+4.5180563038110195
+4.5180563038110195
+4.5180563038110195
+4.511538266366625
+4.511538266366625
+4.511538266366625
+4.511538266366625
+4.5180563038110195
+4.524448559224209
+4.575166900574203
+4.752498445713435
+4.723400793471849
+4.556070973540779
+4.524448559224209
+4.5056825520772295
+4.606705029314043
+4.753120216906312
+4.753120216906312
+4.625661610351658
+4.5056825520772295
+4.5120455101596315
+4.6134253459783015
+4.791447617972283
+4.791447617972283
+4.6909830056250525
+4.536645877449656
+4.518220365758712
+4.518220365758712
+4.518220365758712
+4.536645877449656
+4.737508505270368
+4.81729793871423
+4.579235806949711
+4.491168995656453
+4.625712366844445
+4.858049419420977
+4.809208233051305
+4.540163595729319
+4.5219170311723484
+4.503888260008801
+4.503888260008801
+4.503888260008801
+4.540163595729319
+4.809208233051305
+4.858049419420977
+4.815335102610012
+4.553054939770582
+4.486100377805714
+4.6170189734379745
+4.842936919059271
+4.8848078319332995
+4.8848078319332995
+4.900088766172104
+4.914400272933426
+4.89873092730281
+4.732895578140565
+4.533078274185074
+4.498520697110432
+4.498520697110432
+4.513442545183218
+4.676464247475493
+4.89873092730281
+4.927710408327659
+4.927710408327659
+4.791710163401963
+4.566366390735457
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.462893717258573
+4.462893717258573
+4.479437422437865
+4.61947756989829
+4.863636363636363
+4.937237907553747
+4.890373906609362
+4.660902998104733
+4.4695199194396285
+4.453709526215379
+4.453709526215379
+4.453709526215379
+4.453709526215379
+4.453709526215379
+4.4382813720654735
+4.4382813720654735
+4.4382813720654735
+4.4382813720654735
+4.4382813720654735
+4.458442905224089
+4.458442905224089
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.338887181586376
+3.3820332600309895
+0.5334979361207957
+0.0576866479940108
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8156864016436192
+2.1277860562756055
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2631715931889724
+2.3145873379003574
+3.6047343523426454
+4.546907521378909
+4.436098794671977
+4.376515254795292
+4.461419383366735
+4.517386266312412
+4.5437863107195415
+4.550140350585692
+4.543029917126171
+4.471464242517082
+4.400988345734394
+4.400988345734394
+4.400988345734394
+4.40691351739986
+4.488865705516085
+4.571165324280725
+4.578780008896242
+4.470438427378001
+4.432218388162097
+4.432218388162097
+4.432218388162097
+4.432218388162097
+4.438734902106786
+4.438734902106786
+4.438734902106786
+4.438734902106786
+4.457759882887575
+4.457759882887575
+4.457759882887575
+4.4768097253305825
+4.5968888406633095
+4.668188175579596
+4.679949111556313
+4.666403422086015
+4.585070500657054
+4.511538266366625
+4.530541158577594
+4.632170355178927
+4.69281418415078
+4.6562728773757405
+4.530541158577594
+4.499181667202394
+4.499181667202394
+4.492597061738637
+4.492597061738637
+4.511538266366625
+4.511538266366625
+4.492597061738637
+4.492597061738637
+4.492597061738637
+4.511538266366625
+4.5180563038110195
+4.6667668946905065
+4.753120216906312
+4.61844277751581
+4.5056825520772295
+4.5056825520772295
+4.537026822756646
+4.738328274948345
+4.753120216906312
+4.684246317290842
+4.5056825520772295
+4.5056825520772295
+4.556070973540779
+4.76455519928248
+4.791447617972283
+4.74473950518707
+4.5120455101596315
+4.518220365758712
+4.518220365758712
+4.518220365758712
+4.530661494648408
+4.754870956928874
+4.804805898398897
+4.5601096139456985
+4.491168995656453
+4.651613169058767
+4.828385320427853
+4.771843886220461
+4.5219170311723484
+4.503888260008801
+4.503888260008801
+4.503888260008801
+4.503888260008801
+4.683632848810578
+4.858049419420977
+4.858049419420977
+4.702967088304484
+4.486100377805714
+4.611996828309391
+4.830804040442458
+4.869137689341459
+4.8848078319332995
+4.8848078319332995
+4.8848078319332995
+4.886289060517825
+4.693905508121947
+4.51566364335077
+4.498520697110432
+4.479437422437865
+4.513442545183218
+4.732895578140565
+4.914400272933426
+4.927710408327659
+4.914400272933426
+4.732895578140565
+4.513442545183218
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.462893717258573
+4.462893717258573
+4.462893717258573
+4.496296093052938
+4.713366915704603
+4.914400272933426
+4.927710408327659
+4.830860339288274
+4.563809733321463
+4.446694387067265
+4.446694387067265
+4.453709526215379
+4.453709526215379
+4.453709526215379
+4.4382813720654735
+4.4382813720654735
+4.4382813720654735
+4.4382813720654735
+4.4382813720654735
+4.4382813720654735
+4.43622316904972
+4.43622316904972
+4.43622316904972
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.428834675719275
+4.428834675719275
+4.428834675719275
+3.3833587151986046
+0.5567962144720973
+0.0576866479940108
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7853909093635885
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2551629429654723
+2.2757804963122665
+3.3579079718332823
+4.528339335359163
+4.525148982297988
+4.494288594971252
+4.390394399682197
+4.376515254795292
+4.4181306177488215
+4.508785647745265
+4.530776542855393
+4.5437863107195415
+4.536738078402639
+4.483266161362857
+4.3954670809967755
+4.400988345734394
+4.400988345734394
+4.400988345734394
+4.451863182214335
+4.56377683095028
+4.5853224209629255
+4.500777606857457
+4.432218388162097
+4.413190127900948
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.438734902106786
+4.438734902106786
+4.438734902106786
+4.438734902106786
+4.457759882887575
+4.457759882887575
+4.4702902099390895
+4.6050017018392575
+4.657957256979294
+4.668188175579596
+4.666403422086015
+4.566659309953631
+4.492597061738637
+4.523942749265335
+4.64015341288857
+4.69281418415078
+4.64015341288857
+4.504950980439749
+4.492597061738637
+4.492597061738637
+4.492597061738637
+4.492597061738637
+4.492597061738637
+4.492597061738637
+4.492597061738637
+4.492597061738637
+4.492597061738637
+4.492597061738637
+4.542937271084943
+4.726826567571509
+4.707820717224033
+4.524448559224209
+4.5056825520772295
+4.5056825520772295
+4.684246317290842
+4.753120216906312
+4.738328274948345
+4.537026822756646
+4.480426073856573
+4.499181667202394
+4.701360778922976
+4.778088299149454
+4.76455519928248
+4.562343756184406
+4.493609179742273
+4.493609179742273
+4.491168995656453
+4.528362374312382
+4.771738592817006
+4.791122580013401
+4.541058036271199
+4.491168995656453
+4.689504627805389
+4.828385320427853
+4.689504627805389
+4.497626528797163
+4.503888260008801
+4.503888260008801
+4.503888260008801
+4.553054939770582
+4.826446087163245
+4.858049419420977
+4.815335102610012
+4.553054939770582
+4.631157180365171
+4.830804040442458
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.869137689341459
+4.8584171544605805
+4.650413336097011
+4.486100377805714
+4.472934417851474
+4.479437422437865
+4.545622089347077
+4.791710163401963
+4.914400272933426
+4.914400272933426
+4.8816267976576375
+4.676464247475493
+4.496296093052938
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.462893717258573
+4.462893717258573
+4.462893717258573
+4.462893717258573
+4.545622089347077
+4.791710163401963
+4.927710408327659
+4.914400272933426
+4.732895578140565
+4.496296093052938
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.4382813720654735
+4.4382813720654735
+4.4382813720654735
+4.4382813720654735
+4.4382813720654735
+4.43622316904972
+4.43622316904972
+4.43622316904972
+4.421219991103758
+4.421219991103758
+4.421219991103758
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.44343410958372
+4.428834675719275
+4.428834675719275
+4.428834675719275
+4.428834675719275
+4.428834675719275
+4.428834675719275
+3.378969527964049
+0.429071160666906
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+1.4126304624017485
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.2713027004785546
+2.447931302334326
+4.4222248304064395
+4.50429677271331
+4.502694043727065
+4.507264102733968
+4.449037409925407
+4.376515254795292
+4.376515254795292
+4.385805856764533
+4.468584860676432
+4.525148982297988
+4.5437863107195415
+4.550140350585692
+4.494381313041727
+4.414356383420913
+4.381968556808062
+4.400988345734394
+4.400988345734394
+4.433162067515307
+4.5227409419139475
+4.577601706931892
+4.529291803534183
+4.423232002361576
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.432218388162097
+4.438734902106786
+4.438734902106786
+4.438734902106786
+4.457759882887575
+4.457759882887575
+4.489286681787702
+4.596668132191088
+4.6356545149438695
+4.657957256979294
+4.666403422086015
+4.555042161857102
+4.492597061738637
+4.504950980439749
+4.64015341288857
+4.69281418415078
+4.632170355178927
+4.504950980439749
+4.492597061738637
+4.492597061738637
+4.492597061738637
+4.4737411954365065
+4.4737411954365065
+4.492597061738637
+4.492597061738637
+4.492597061738637
+4.4737411954365065
+4.4737411954365065
+4.640652808961297
+4.728397247583595
+4.587688824985208
+4.5056825520772295
+4.487067282446096
+4.606705029314043
+4.753120216906312
+4.753120216906312
+4.599691079805129
+4.480426073856573
+4.480426073856573
+4.637060067730612
+4.764851533139823
+4.764851533139823
+4.625661610351658
+4.491168995656453
+4.491168995656453
+4.4728501212062755
+4.5096810473294475
+4.776441943741247
+4.791122580013401
+4.541058036271199
+4.491168995656453
+4.736141839889283
+4.81729793871423
+4.6197826327175235
+4.491168995656453
+4.491168995656453
+4.497626528797163
+4.503888260008801
+4.702967088304484
+4.858049419420977
+4.858049419420977
+4.702967088304484
+4.625712366844445
+4.830804040442458
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.815335102610012
+4.609875135493686
+4.486100377805714
+4.466207198930331
+4.466207198930331
+4.596006379601555
+4.84508484236849
+4.914400272933426
+4.914400272933426
+4.863636363636363
+4.600754584726002
+4.462893717258573
+4.462893717258573
+4.479437422437865
+4.479437422437865
+4.479437422437865
+4.462893717258573
+4.462893717258573
+4.462893717258573
+4.462893717258573
+4.600754584726002
+4.863636363636363
+4.937237907553747
+4.8816267976576375
+4.638346592840813
+4.462893717258573
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.4382813720654735
+4.4382813720654735
+4.43622316904972
+4.43622316904972
+4.43622316904972
+4.421219991103758
+4.421219991103758
+4.421219991103758
+4.421219991103758
+4.421219991103758
+4.421219991103758
+4.421219991103758
+4.428834675719275
+4.428834675719275
+4.428834675719275
+4.428834675719275
+4.428834675719275
+4.428834675719275
+4.428834675719275
+4.428834675719275
+4.44343410958372
+0.6273720592915861
+0.2118577334327867
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.17305757537147226
+2.3625603076388697
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2583256956715583
+2.282467208230445
+3.4027943810712595
+4.510525149050915
+4.4811159587853355
+4.475186563700894
+4.475186563700894
+4.452791429304008
+4.371568891778836
+4.3575227068135405
+4.3575227068135405
+4.371568891778836
+4.444561438329306
+4.517386266312412
+4.525148982297988
+4.5437863107195415
+4.520058934112502
+4.409131784852644
+4.381968556808062
+4.400988345734394
+4.400988345734394
+4.398150499106636
+4.488817061485224
+4.577601706931892
+4.555438561670694
+4.436280735418166
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.432218388162097
+4.438734902106786
+4.438734902106786
+4.438734902106786
+4.438734902106786
+4.489286681787702
+4.596668132191088
+4.6356545149438695
+4.648026828159334
+4.641235384144579
+4.5361952346332135
+4.4737411954365065
+4.504950980439749
+4.6562728773757405
+4.69281418415078
+4.632170355178927
+4.511538266366625
+4.4737411954365065
+4.4737411954365065
+4.4737411954365065
+4.4737411954365065
+4.4737411954365065
+4.4737411954365065
+4.4737411954365065
+4.4737411954365065
+4.4737411954365065
+4.523942749265335
+4.715349243520801
+4.649129891979821
+4.4737411954365065
+4.487067282446096
+4.5180563038110195
+4.738328274948345
+4.753120216906312
+4.684246317290842
+4.480426073856573
+4.461814107815801
+4.5495826726201285
+4.740409145179578
+4.764851533139823
+4.68279634091494
+4.466027460208323
+4.466027460208323
+4.4728501212062755
+4.5221019260413
+4.776441943741247
+4.791122580013401
+4.541058036271199
+4.5096810473294475
+4.771738592817006
+4.804805898398897
+4.562343756184406
+4.4728501212062755
+4.4728501212062755
+4.4728501212062755
+4.5601096139456985
+4.817997864018289
+4.858049419420977
+4.830804040442458
+4.715635347795578
+4.830804040442458
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.809208233051305
+4.571866014030257
+4.466207198930331
+4.466207198930331
+4.466207198930331
+4.629037720920056
+4.8584171544605805
+4.8848078319332995
+4.8848078319332995
+4.8247089911549255
+4.563809733321463
+4.462893717258573
+4.462893717258573
+4.462893717258573
+4.462893717258573
+4.462893717258573
+4.462893717258573
+4.462893717258573
+4.462893717258573
+4.479437422437865
+4.693905508121947
+4.89873092730281
+4.927710408327659
+4.811314015577191
+4.545622089347077
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.428749545454485
+4.43622316904972
+4.421219991103758
+4.421219991103758
+4.421219991103758
+4.421219991103758
+4.421219991103758
+4.421219991103758
+4.421219991103758
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.428834675719275
+4.428834675719275
+4.428834675719275
+4.428834675719275
+4.428834675719275
+4.473829599409482
+4.701147711411036
+4.502155701517554
+0.6574162717298266
+0.08652970395641724
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4997231449871
+2.070157598741992
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.2713027004785546
+2.3818535613912446
+4.399964825372442
+4.4811159587853355
+4.392755007565345
+4.3484375085333085
+4.3484375085333085
+4.363331244589073
+4.3484375085333085
+4.352675830528556
+4.3575227068135405
+4.3575227068135405
+4.3575227068135405
+4.4181306177488215
+4.494288594971252
+4.525148982297988
+4.525148982297988
+4.514260294902164
+4.427760861995566
+4.378997457380755
+4.359813865820298
+4.378997457380755
+4.378997457380755
+4.456666181535883
+4.563841360958199
+4.55656589041628
+4.467971184044464
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.438734902106786
+4.438734902106786
+4.438734902106786
+4.438734902106786
+4.489286681787702
+4.612010239449515
+4.6356545149438695
+4.6356545149438695
+4.626881615898075
+4.508225910052507
+4.4737411954365065
+4.504950980439749
+4.646852001521213
+4.69281418415078
+4.614878214028467
+4.4737411954365065
+4.4737411954365065
+4.4737411954365065
+4.4737411954365065
+4.4737411954365065
+4.4549954627911825
+4.4549954627911825
+4.4549954627911825
+4.4737411954365065
+4.4737411954365065
+4.632170355178927
+4.69066493212302
+4.523942749265335
+4.4549954627911825
+4.4737411954365065
+4.684246317290842
+4.753120216906312
+4.723400793471849
+4.511538266366625
+4.461814107815801
+4.492597061738637
+4.723400793471849
+4.753120216906312
+4.723400793471849
+4.490078739911335
+4.459144616499847
+4.466027460208323
+4.5221019260413
+4.76455519928248
+4.776441943741247
+4.50326278167366
+4.5221019260413
+4.791122580013401
+4.776441943741247
+4.5096810473294475
+4.454750524866162
+4.454750524866162
+4.4728501212062755
+4.726800481807874
+4.828385320427853
+4.843246627448329
+4.845146974506639
+4.845146974506639
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.790040730028535
+4.553054939770582
+4.466207198930331
+4.448945275450309
+4.466207198930331
+4.642838795116047
+4.845146974506639
+4.869137689341459
+4.8848078319332995
+4.779665527134621
+4.5219170311723484
+4.456023364726183
+4.456023364726183
+4.462893717258573
+4.462893717258573
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.513442545183218
+4.772087701603047
+4.927710408327659
+4.914400272933426
+4.713366915704603
+4.496296093052938
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.428749545454485
+4.428749545454485
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.421219991103758
+4.421219991103758
+4.421219991103758
+4.421219991103758
+4.421219991103758
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.428834675719275
+4.4146775790370745
+4.428834675719275
+4.570838353420942
+4.8177095132940115
+4.837439329515021
+4.553307770795782
+0.6091995609230558
+0.26944739896229564
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.9379779678995588
+2.2037404301444643
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2583256956715583
+2.279953976130526
+3.3145332729137547
+4.486334337166265
+4.397344032056895
+4.30404791709206
+4.329641292087303
+4.329641292087303
+4.329641292087303
+4.329641292087303
+4.329641292087303
+4.329641292087303
+4.333736762813394
+4.3575227068135405
+4.3575227068135405
+4.385805856764533
+4.468584860676432
+4.525148982297988
+4.525148982297988
+4.522944253053671
+4.436098794671977
+4.359813865820298
+4.359813865820298
+4.359813865820298
+4.378997457380755
+4.425150693598551
+4.541557094775911
+4.563841360958199
+4.500777606857457
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.432218388162097
+4.438734902106786
+4.438734902106786
+4.438734902106786
+4.482822888508683
+4.61789817246824
+4.6356545149438695
+4.6356545149438695
+4.612010239449515
+4.489286681787702
+4.445436290543406
+4.517290388513929
+4.662029905531306
+4.679949111556313
+4.586925143439678
+4.4737411954365065
+4.4737411954365065
+4.4737411954365065
+4.4549954627911825
+4.4549954627911825
+4.4549954627911825
+4.4549954627911825
+4.4549954627911825
+4.436386382000617
+4.517290388513929
+4.7039874056290465
+4.603299912056848
+4.4549954627911825
+4.4549954627911825
+4.592455460018538
+4.740587601014347
+4.753120216906312
+4.561910464792896
+4.459144616499847
+4.459144616499847
+4.68279634091494
+4.753120216906312
+4.740409145179578
+4.528188482906883
+4.440585238149008
+4.440585238149008
+4.530541158577594
+4.7657943350103915
+4.76455519928248
+4.50326278167366
+4.543342744992878
+4.791447617972283
+4.754870956928874
+4.4728501212062755
+4.454750524866162
+4.454750524866162
+4.562343756184406
+4.804805898398897
+4.81729793871423
+4.828385320427853
+4.828385320427853
+4.843246627448329
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.790040730028535
+4.553054939770582
+4.466207198930331
+4.448945275450309
+4.466207198930331
+4.683632848810578
+4.845146974506639
+4.858049419420977
+4.858049419420977
+4.734569035115524
+4.501557647515675
+4.448945275450309
+4.448945275450309
+4.448945275450309
+4.456023364726183
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.563809733321463
+4.830860339288274
+4.927710408327659
+4.863636363636363
+4.61947756989829
+4.462893717258573
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.428749545454485
+4.428749545454485
+4.428749545454485
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.421219991103758
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.421219991103758
+4.518950034056942
+4.741226342122529
+4.876999764984348
+4.682407358487153
+4.458442905224089
+4.4146775790370745
+3.4692915679189387
+0.6531094688252859
+0.08652970395641724
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7319868307667621
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2631715931889724
+2.2982112549716662
+3.5917018858924608
+4.479107382974854
+4.352495164972238
+4.30404791709206
+4.30404791709206
+4.310799240903608
+4.310799240903608
+4.310799240903608
+4.310799240903608
+4.329641292087303
+4.329641292087303
+4.329641292087303
+4.352675830528556
+4.3575227068135405
+4.3575227068135405
+4.444561438329306
+4.517386266312412
+4.525148982297988
+4.517386266312412
+4.466270560425329
+4.373530165834871
+4.359813865820298
+4.359813865820298
+4.359813865820298
+4.392622212842695
+4.51043569835049
+4.556828312189985
+4.5227409419139475
+4.423232002361576
+4.410395058116138
+4.410395058116138
+4.410395058116138
+4.404092515167324
+4.404092515167324
+4.404092515167324
+4.413190127900948
+4.419759557158932
+4.419759557158932
+4.482822888508683
+4.61789817246824
+4.6356545149438695
+4.6356545149438695
+4.596668132191088
+4.4512597008547115
+4.426558867884018
+4.510643118126104
+4.662029905531306
+4.666403422086015
+4.566659309953631
+4.4549954627911825
+4.448231417474431
+4.448231417474431
+4.448231417474431
+4.448231417474431
+4.436386382000617
+4.436386382000617
+4.436386382000617
+4.4549954627911825
+4.649129891979821
+4.6870297623887245
+4.467070908185214
+4.436386382000617
+4.504950980439749
+4.715349243520801
+4.728275832763921
+4.639065325410318
+4.459144616499847
+4.459144616499847
+4.599691079805129
+4.753120216906312
+4.753120216906312
+4.61844277751581
+4.440585238149008
+4.4222123806403095
+4.530541158577594
+4.753120216906312
+4.752498445713435
+4.50326278167366
+4.594292603304557
+4.791447617972283
+4.681857368058384
+4.454750524866162
+4.454750524866162
+4.4728501212062755
+4.736141839889283
+4.81729793871423
+4.81729793871423
+4.81729793871423
+4.81729793871423
+4.828385320427853
+4.828385320427853
+4.843246627448329
+4.798991798454706
+4.553054939770582
+4.448945275450309
+4.448945275450309
+4.4837541190057655
+4.702967088304484
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.715635347795578
+4.466207198930331
+4.431999507328421
+4.431999507328421
+4.431999507328421
+4.431999507328421
+4.431999507328421
+4.431999507328421
+4.439444868219007
+4.462893717258573
+4.61947756989829
+4.863636363636363
+4.914400272933426
+4.791710163401963
+4.525371911714212
+4.446694387067265
+4.446694387067265
+4.446694387067265
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.428749545454485
+4.428749545454485
+4.428749545454485
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.398374751597663
+4.398374751597663
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.406660686573971
+4.421219991103758
+4.530851648554603
+4.739502382932878
+4.83515291684364
+4.6988319217883205
+4.502167670798816
+4.4146775790370745
+4.400997163296846
+4.400997163296846
+4.400997163296846
+0.6749169490212346
+0.17305757537147226
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+1.5018024598202486
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.2784211733545465
+2.4353146260153733
+4.410795463131559
+4.469240513255388
+4.446713403848863
+4.423669530065321
+4.399300493996371
+4.352495164972238
+4.320286335236165
+4.307610503395612
+4.310799240903608
+4.310799240903608
+4.310799240903608
+4.329641292087303
+4.329641292087303
+4.326291782126872
+4.330384925242272
+4.335175833863043
+4.397327275394801
+4.494288594971252
+4.525148982297988
+4.525148982297988
+4.473136969025019
+4.373530165834871
+4.359813865820298
+4.359813865820298
+4.359813865820298
+4.378997457380755
+4.488817061485224
+4.556828312189985
+4.526170400590518
+4.436280735418166
+4.404092515167324
+4.404092515167324
+4.404092515167324
+4.404092515167324
+4.38493610611664
+4.38493610611664
+4.38493610611664
+4.413190127900948
+4.419759557158932
+4.495160837080689
+4.609159296939702
+4.6356545149438695
+4.6356545149438695
+4.572275762357333
+4.432218388162097
+4.419759557158932
+4.520325741539703
+4.666403422086015
+4.676655864461275
+4.510643118126104
+4.448231417474431
+4.448231417474431
+4.429493503939355
+4.429493503939355
+4.429493503939355
+4.429493503939355
+4.436386382000617
+4.436386382000617
+4.5361952346332135
+4.69066493212302
+4.555042161857102
+4.436386382000617
+4.452260404566443
+4.665703966605539
+4.728275832763921
+4.6870297623887245
+4.471121086488168
+4.440585238149008
+4.530541158577594
+4.740409145179578
+4.753120216906312
+4.68279634091494
+4.440585238149008
+4.4222123806403095
+4.542937271084943
+4.753120216906312
+4.738328274948345
+4.471121086488168
+4.637060067730612
+4.778088299149454
+4.6134253459783015
+4.436898945829201
+4.436898945829201
+4.575166900574203
+4.804805898398897
+4.81729793871423
+4.81729793871423
+4.81729793871423
+4.81729793871423
+4.81729793871423
+4.828385320427853
+4.803471126886287
+4.5601096139456985
+4.441702784036244
+4.448945275450309
+4.477153658161236
+4.715635347795578
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.702967088304484
+4.466207198930331
+4.431999507328421
+4.431999507328421
+4.415404133782649
+4.415404133782649
+4.415404133782649
+4.431999507328421
+4.431999507328421
+4.466207198930331
+4.689147125339842
+4.886289060517825
+4.89873092730281
+4.691713156761751
+4.462893717258573
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.43087120951505
+4.428749545454485
+4.428749545454485
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.406660686573971
+4.406660686573971
+4.43622316904972
+4.584430242518478
+4.778439516729418
+4.854921788484206
+4.6988319217883205
+4.485682117209293
+4.392582383584325
+4.392582383584325
+4.392582383584325
+4.392582383584325
+4.400997163296846
+4.400997163296846
+0.6083917582347658
+0.2982551909566391
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2884240647434684
+2.309585675329008
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2583256956715583
+2.285534687051869
+2.689922886317072
+4.450262109391606
+4.40784695706688
+4.421815150590111
+4.446713403848863
+4.459631613915226
+4.4569885833271
+4.446713403848863
+4.423669530065321
+4.383033768797953
+4.357406875112937
+4.326251880415926
+4.310799240903608
+4.307217958916177
+4.326291782126872
+4.326291782126872
+4.330384925242272
+4.335175833863043
+4.364189297235156
+4.473136969025019
+4.525148982297988
+4.525148982297988
+4.484110954809075
+4.382978344722738
+4.359813865820298
+4.359813865820298
+4.359813865820298
+4.378997457380755
+4.456666181535883
+4.543029917126171
+4.549643730922412
+4.44374109684803
+4.404092515167324
+4.38493610611664
+4.38493610611664
+4.38493610611664
+4.38493610611664
+4.38493610611664
+4.38493610611664
+4.394200048734022
+4.413190127900948
+4.507121697681015
+4.6226847512678555
+4.6226847512678555
+4.6356545149438695
+4.555394310843138
+4.413190127900948
+4.394200048734022
+4.531977395390569
+4.655021703819993
+4.662029905531306
+4.498352535656417
+4.429493503939355
+4.429493503939355
+4.429493503939355
+4.429493503939355
+4.429493503939355
+4.429493503939355
+4.410885543205496
+4.448231417474431
+4.6562728773757405
+4.649129891979821
+4.43352271756291
+4.43352271756291
+4.585070500657054
+4.716549695711502
+4.7039874056290465
+4.498352535656417
+4.414936323624801
+4.471121086488168
+4.723400793471849
+4.753120216906312
+4.738328274948345
+4.471121086488168
+4.4040579122219
+4.561910464792896
+4.753120216906312
+4.723400793471849
+4.471121086488168
+4.68279634091494
+4.764851533139823
+4.5495826726201285
+4.436898945829201
+4.4728501212062755
+4.7266202754230955
+4.791447617972283
+4.81729793871423
+4.81729793871423
+4.81729793871423
+4.81729793871423
+4.81729793871423
+4.791122580013401
+4.579235806949711
+4.434345980166132
+4.434345980166132
+4.470390948165326
+4.715635347795578
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.683632848810578
+4.448945275450309
+4.431999507328421
+4.431999507328421
+4.415404133782649
+4.415404133782649
+4.415404133782649
+4.415404133782649
+4.415404133782649
+4.477153658161236
+4.741519592641198
+4.8848078319332995
+4.825007070248277
+4.596006379601555
+4.446694387067265
+4.43087120951505
+4.415458429428599
+4.415458429428599
+4.415458429428599
+4.415458429428599
+4.43087120951505
+4.43087120951505
+4.428749545454485
+4.428749545454485
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.444561438329306
+4.61947756989829
+4.815381064171493
+4.854921788484206
+4.6798086704348965
+4.485682117209293
+4.392582383584325
+4.392582383584325
+4.392582383584325
+4.392582383584325
+4.379023901017087
+4.379023901017087
+4.379023901017087
+4.400997163296846
+3.3674002013493793
+0.5008125698600319
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6056272403357319
+2.0609275239094176
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.261761202020505
+2.290883657014198
+3.4939392236770477
+4.32539149528159
+4.264296363513784
+4.281840839476571
+4.3001441763240145
+4.334884057988927
+4.3666145665109415
+4.410905534283973
+4.449201693600046
+4.469240513255388
+4.459631613915226
+4.451997806809301
+4.401672253049345
+4.3571598314339965
+4.326291782126872
+4.326291782126872
+4.330384925242272
+4.311196423331765
+4.311196423331765
+4.3495197621021
+4.444561438329306
+4.517386266312412
+4.525148982297988
+4.508785647745265
+4.415569757481522
+4.354374661974843
+4.359813865820298
+4.359813865820298
+4.359813865820298
+4.425150693598551
+4.5350396637789485
+4.556828312189985
+4.456666181535883
+4.378997457380755
+4.38493610611664
+4.38493610611664
+4.38493610611664
+4.38493610611664
+4.38493610611664
+4.365786209121337
+4.365786209121337
+4.394200048734022
+4.536064520713749
+4.6226847512678555
+4.6226847512678555
+4.6226847512678555
+4.525181448575433
+4.394200048734022
+4.394200048734022
+4.543017950066887
+4.657957256979294
+4.629658286286764
+4.460474670771755
+4.410885543205496
+4.410885543205496
+4.429493503939355
+4.410885543205496
+4.410885543205496
+4.410885543205496
+4.410885543205496
+4.566659309953631
+4.69066493212302
+4.4958606434723105
+4.396532494921532
+4.498352535656417
+4.7039874056290465
+4.716549695711502
+4.585070500657054
+4.414936323624801
+4.414936323624801
+4.630575200519795
+4.740587601014347
+4.753120216906312
+4.542937271084943
+4.4040579122219
+4.599691079805129
+4.753120216906312
+4.723400793471849
+4.490078739911335
+4.707820717224033
+4.738328274948345
+4.471121086488168
+4.4222123806403095
+4.587688824985208
+4.791447617972283
+4.791447617972283
+4.791447617972283
+4.791447617972283
+4.81729793871423
+4.81729793871423
+4.81729793871423
+4.6134253459783015
+4.416711565401162
+4.416711565401162
+4.452249827320327
+4.700234519030086
+4.828385320427853
+4.843246627448329
+4.858049419420977
+4.683632848810578
+4.448945275450309
+4.431999507328421
+4.415404133782649
+4.415404133782649
+4.415404133782649
+4.415404133782649
+4.415404133782649
+4.415404133782649
+4.515914357186943
+4.790040730028535
+4.858049419420977
+4.771843886220461
+4.513544964723501
+4.415404133782649
+4.415404133782649
+4.407420763274691
+4.415458429428599
+4.415458429428599
+4.415458429428599
+4.415458429428599
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.472934417851474
+4.6531316008140005
+4.8435411247040765
+4.861973736884253
+4.672383068369923
+4.479437422437865
+4.392582383584325
+4.392582383584325
+4.392582383584325
+4.392582383584325
+4.379023901017087
+4.379023901017087
+4.379023901017087
+4.379023901017087
+4.379023901017087
+4.379023901017087
+3.4282319347219907
+0.6264258425886293
+0.11537255792817191
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8867520004449787
+2.212645948944707
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.2694046972337256
+2.317851220971419
+4.278262000226555
+4.270244682654806
+4.245536640869366
+4.245536640869366
+4.264296363513784
+4.264296363513784
+4.264296363513784
+4.285099320258072
+4.301694080793463
+4.351015773715063
+4.410905534283973
+4.459631613915226
+4.469240513255388
+4.462569174514068
+4.433425791049117
+4.3571598314339965
+4.307217958916177
+4.307217958916177
+4.307217958916177
+4.307217958916177
+4.330384925242272
+4.397327275394801
+4.508785647745265
+4.525148982297988
+4.517386266312412
+4.444561438329306
+4.3495197621021
+4.340622378893027
+4.359813865820298
+4.359813865820298
+4.392622212842695
+4.494381313041727
+4.550140350585692
+4.483266161362857
+4.378997457380755
+4.38493610611664
+4.38493610611664
+4.365786209121337
+4.365786209121337
+4.365786209121337
+4.365786209121337
+4.365786209121337
+4.38493610611664
+4.5461808736533715
+4.6226847512678555
+4.6226847512678555
+4.609159296939702
+4.476523478952587
+4.394200048734022
+4.394200048734022
+4.553310052157446
+4.657957256979294
+4.5968888406633095
+4.422758162648074
+4.410885543205496
+4.410885543205496
+4.410885543205496
+4.410885543205496
+4.407795588564746
+4.407795588564746
+4.457759882887575
+4.676655864461275
+4.595530339448932
+4.396532494921532
+4.426558867884018
+4.671919701176808
+4.716549695711502
+4.647609269642364
+4.414936323624801
+4.396532494921532
+4.5480985287219635
+4.716549695711502
+4.740587601014347
+4.629292470505407
+4.4222123806403095
+4.610962775441974
+4.753120216906312
+4.723400793471849
+4.521563209510312
+4.726826567571509
+4.69168036796802
+4.4222123806403095
+4.43352271756291
+4.707820717224033
+4.764851533139823
+4.778088299149454
+4.791447617972283
+4.791447617972283
+4.791447617972283
+4.803215945464247
+4.681857368058384
+4.434345980166132
+4.416711565401162
+4.434345980166132
+4.670612366078799
+4.81729793871423
+4.828385320427853
+4.828385320427853
+4.670612366078799
+4.441702784036244
+4.415404133782649
+4.415404133782649
+4.415404133782649
+4.415404133782649
+4.415404133782649
+4.415404133782649
+4.415404133782649
+4.566145733226151
+4.815335102610012
+4.858049419420977
+4.683632848810578
+4.441702784036244
+4.399196764220106
+4.399196764220106
+4.399196764220106
+4.399196764220106
+4.399196764220106
+4.407420763274691
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.413341468587734
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.413341468587734
+4.507635367903604
+4.711105210397496
+4.863636363636363
+4.8435411247040765
+4.6531316008140005
+4.456023364726183
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.392582383584325
+4.379023901017087
+4.379023901017087
+4.379023901017087
+4.379023901017087
+4.379023901017087
+4.379023901017087
+4.379023901017087
+4.379023901017087
+3.5115871017929865
+0.6938078777165453
+0.11537255792817191
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6968440009240968
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.2784211733545465
+2.3775439650252617
+4.307047904522636
+4.320220278148516
+4.2636275033140185
+4.245120702397898
+4.245536640869366
+4.245536640869366
+4.264296363513784
+4.2610653657318265
+4.2610653657318265
+4.282040951144447
+4.282040951144447
+4.332417955746382
+4.410905534283973
+4.4569885833271
+4.4812489691440245
+4.477376686372026
+4.417615488324072
+4.3229225252815375
+4.288085371739211
+4.307217958916177
+4.307217958916177
+4.311196423331765
+4.364189297235156
+4.468584860676432
+4.525148982297988
+4.525148982297988
+4.468584860676432
+4.364189297235156
+4.335175833863043
+4.359813865820298
+4.340622378893027
+4.354374661974843
+4.466270560425329
+4.556828312189985
+4.51043569835049
+4.392622212842695
+4.340622378893027
+4.365786209121337
+4.365786209121337
+4.365786209121337
+4.365786209121337
+4.365786209121337
+4.346668293077823
+4.378997457380755
+4.548365008356385
+4.613831733428877
+4.6226847512678555
+4.603447413405319
+4.438908795687557
+4.375275290067039
+4.387882008174667
+4.564743476593351
+4.6356545149438695
+4.560601716090231
+4.400860279484698
+4.392438741386968
+4.392438741386968
+4.389176702879745
+4.407795588564746
+4.407795588564746
+4.407795588564746
+4.622018630619488
+4.676655864461275
+4.438734902106786
+4.396532494921532
+4.613200121218277
+4.7039874056290465
+4.6870297623887245
+4.445436290543406
+4.396532494921532
+4.4794059114735
+4.7039874056290465
+4.728275832763921
+4.671919701176808
+4.43352271756291
+4.647405571565398
+4.753120216906312
+4.723400793471849
+4.599691079805129
+4.753120216906312
+4.629292470505407
+4.386157512505592
+4.561910464792896
+4.753120216906312
+4.753120216906312
+4.764851533139823
+4.778088299149454
+4.791447617972283
+4.791447617972283
+4.7266202754230955
+4.447681790841982
+4.399382355503642
+4.434345980166132
+4.651613169058767
+4.81729793871423
+4.81729793871423
+4.81729793871423
+4.670612366078799
+4.434345980166132
+4.399382355503642
+4.399382355503642
+4.407435659740823
+4.415404133782649
+4.415404133782649
+4.415404133782649
+4.431999507328421
+4.623573129679567
+4.845146974506639
+4.830804040442458
+4.60435607626104
+4.415404133782649
+4.399196764220106
+4.399196764220106
+4.399196764220106
+4.383418627512178
+4.399196764220106
+4.399196764220106
+4.396937585433321
+4.396937585433321
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.413341468587734
+4.413341468587734
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.413341468587734
+4.543333818464117
+4.752475355734676
+4.8816267976576375
+4.8435411247040765
+4.638346592840813
+4.446694387067265
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.3699279208395785
+4.3699279208395785
+4.379023901017087
+4.379023901017087
+4.379023901017087
+4.379023901017087
+4.379023901017087
+4.379023901017087
+4.366025403784438
+3.5211359667324267
+0.7189354001027244
+0.14421518948927226
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.11537255792817191
+2.441092972129308
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.2840022396629216
+2.4620011054082873
+4.401911550094636
+4.427387113724792
+4.405171091330308
+4.392926544974844
+4.350020346986972
+4.299316960299984
+4.242051477075593
+4.242051477075593
+4.2610653657318265
+4.2610653657318265
+4.282040951144447
+4.262824069542097
+4.282040951144447
+4.34870927122472
+4.435608687796262
+4.469240513255388
+4.484738687018837
+4.4538393189583685
+4.372425188923552
+4.307217958916177
+4.288085371739211
+4.307217958916177
+4.307217958916177
+4.326291782126872
+4.444561438329306
+4.517386266312412
+4.525148982297988
+4.508785647745265
+4.411424193686267
+4.330384925242272
+4.315955355853033
+4.340622378893027
+4.340622378893027
+4.4201516731586725
+4.543029917126171
+4.520058934112502
+4.406415144859124
+4.340622378893027
+4.340622378893027
+4.365786209121337
+4.365786209121337
+4.346668293077823
+4.346668293077823
+4.346668293077823
+4.392622212842695
+4.55656589041628
+4.612171103755108
+4.613831733428877
+4.580899023800447
+4.40691351739986
+4.356445868152194
+4.40691351739986
+4.596668132191088
+4.6356545149438695
+4.488865705516085
+4.37907711329853
+4.37907711329853
+4.389176702879745
+4.389176702879745
+4.389176702879745
+4.389176702879745
+4.510643118126104
+4.69281418415078
+4.541142071059501
+4.370736046613753
+4.529417439370056
+4.7039874056290465
+4.7039874056290465
+4.498352535656417
+4.378346264281328
+4.426558867884018
+4.6870297623887245
+4.716549695711502
+4.7039874056290465
+4.4794059114735
+4.647609269642364
+4.740587601014347
+4.7124748910849155
+4.665257422053748
+4.753120216906312
+4.542937271084943
+4.43352271756291
+4.707820717224033
+4.753120216906312
+4.753120216906312
+4.753120216906312
+4.753120216906312
+4.764851533139823
+4.752498445713435
+4.515666105800818
+4.391307496138788
+4.391307496138788
+4.594292603304557
+4.804805898398897
+4.81729793871423
+4.81729793871423
+4.681857368058384
+4.434345980166132
+4.399382355503642
+4.399382355503642
+4.399382355503642
+4.399382355503642
+4.407435659740823
+4.415404133782649
+4.431999507328421
+4.683632848810578
+4.858049419420977
+4.798991798454706
+4.553054939770582
+4.399196764220106
+4.399196764220106
+4.399196764220106
+4.383418627512178
+4.383418627512178
+4.383418627512178
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.389854478801128
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.398374751597663
+4.428749545454485
+4.5798483268413275
+4.791710163401963
+4.89873092730281
+4.830860339288274
+4.61947756989829
+4.444561438329306
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.379023901017087
+4.379023901017087
+4.366025403784438
+4.3643454850561305
+4.421219991103758
+3.4706170303476207
+0.6960283355891734
+0.17305757537147226
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3461049112705705
+2.2060580360966195
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2583256956715583
+2.2868502656880048
+2.514473207177659
+4.392918295777071
+4.405239266363791
+4.416720743103606
+4.416380983659691
+4.427387113724792
+4.417257564352126
+4.364961598717095
+4.260323041219008
+4.242051477075593
+4.2610653657318265
+4.2610653657318265
+4.2610653657318265
+4.262824069542097
+4.262824069542097
+4.280004643579906
+4.383033768797953
+4.446713403848863
+4.469240513255388
+4.4756868585019225
+4.417615488324072
+4.3229225252815375
+4.288085371739211
+4.307217958916177
+4.307217958916177
+4.307217958916177
+4.393499185832402
+4.503640969757196
+4.525148982297988
+4.517386266312412
+4.440466121467073
+4.326291782126872
+4.311196423331765
+4.315955355853033
+4.321447907498091
+4.387552875155975
+4.520058934112502
+4.528831176637506
+4.4201516731586725
+4.340622378893027
+4.321447907498091
+4.340622378893027
+4.346668293077823
+4.346668293077823
+4.346668293077823
+4.346668293077823
+4.425150693598551
+4.571165324280725
+4.590893017970352
+4.613831733428877
+4.548238985236116
+4.375275290067039
+4.356445868152194
+4.419975864651762
+4.603447413405319
+4.61789817246824
+4.438908795687557
+4.353245755530516
+4.37907711329853
+4.370736046613753
+4.370736046613753
+4.370736046613753
+4.419759557158932
+4.662029905531306
+4.646852001521213
+4.407795588564746
+4.438734902106786
+4.69066493212302
+4.69281418415078
+4.577606921643228
+4.378346264281328
+4.378346264281328
+4.630575200519795
+4.716549695711502
+4.716549695711502
+4.577606921643228
+4.671919701176808
+4.728275832763921
+4.728397247583595
+4.726826567571509
+4.740409145179578
+4.471121086488168
+4.580837290747216
+4.753120216906312
+4.753120216906312
+4.753120216906312
+4.753120216906312
+4.753120216906312
+4.753120216906312
+4.592455460018538
+4.383281307781995
+4.373948985744692
+4.534705884651435
+4.76455519928248
+4.791447617972283
+4.803215945464247
+4.700234519030086
+4.434345980166132
+4.3823982849430525
+4.3823982849430525
+4.3823982849430525
+4.3823982849430525
+4.3823982849430525
+4.3823982849430525
+4.452249827320327
+4.708250006998824
+4.858049419420977
+4.771843886220461
+4.497626528797163
+4.399196764220106
+4.399196764220106
+4.383418627512178
+4.383418627512178
+4.383418627512178
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.389854478801128
+4.398374751597663
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.439444868219007
+4.61947756989829
+4.830860339288274
+4.914400272933426
+4.811314015577191
+4.600754584726002
+4.428749545454485
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.356534660165839
+4.383889436681951
+4.460742127895382
+4.64212860845057
+4.83515291684364
+3.4115167815690217
+0.6398616579338059
+0.17305757537147226
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7683375209644598
+2.100723074504167
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2583256956715583
+2.290474143048891
+2.5242140993240794
+4.23457751825265
+4.230127781115758
+4.263553618346956
+4.282391253599503
+4.300525727563219
+4.316206246689385
+4.316206246689385
+4.260543986543284
+4.242051477075593
+4.242051477075593
+4.242051477075593
+4.2610653657318265
+4.2610653657318265
+4.2610653657318265
+4.262824069542097
+4.262824069542097
+4.315731378555368
+4.410905534283973
+4.459631613915226
+4.469240513255388
+4.451997806809301
+4.372425188923552
+4.288085371739211
+4.288085371739211
+4.307217958916177
+4.288085371739211
+4.3387810056485865
+4.464382022636259
+4.525148982297988
+4.525148982297988
+4.479478935593598
+4.360377731938609
+4.291975060537432
+4.291975060537432
+4.296737346030571
+4.3495197621021
+4.494381313041727
+4.543029917126171
+4.433633609264543
+4.315955355853033
+4.315955355853033
+4.315955355853033
+4.340622378893027
+4.346668293077823
+4.346668293077823
+4.327610812997139
+4.44374109684803
+4.577601706931892
+4.590893017970352
+4.590893017970352
+4.500777606857457
+4.353245755530516
+4.334367732518834
+4.438908795687557
+4.6226847512678555
+4.58852136234472
+4.38493610611664
+4.334367732518834
+4.334367732518834
+4.360278408571066
+4.370736046613753
+4.370736046613753
+4.570132142342086
+4.679949111556313
+4.491801074676181
+4.389176702879745
+4.646852001521213
+4.704934473220487
+4.629658286286764
+4.389176702879745
+4.360417319901349
+4.5480985287219635
+4.7039874056290465
+4.716549695711502
+4.701532155459308
+4.69066493212302
+4.716549695711502
+4.716549695711502
+4.728275832763921
+4.6870297623887245
+4.4834255156601905
+4.707820717224033
+4.753120216906312
+4.753120216906312
+4.753120216906312
+4.753120216906312
+4.753120216906312
+4.690283055605783
+4.412180874957363
+4.34817212384323
+4.452260404566443
+4.738328274948345
+4.778088299149454
+4.791447617972283
+4.709360105415576
+4.4269312816945705
+4.3823982849430525
+4.3823982849430525
+4.3823982849430525
+4.3823982849430525
+4.3823982849430525
+4.3823982849430525
+4.445120250365041
+4.736141839889283
+4.828385320427853
+4.708250006998824
+4.452249827320327
+4.399196764220106
+4.399196764220106
+4.383418627512178
+4.383418627512178
+4.383418627512178
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.365815996925577
+4.374952900097492
+4.383889436681951
+4.462893717258573
+4.633974596215562
+4.8247089911549255
+4.89873092730281
+4.791710163401963
+4.582193139573988
+4.428749545454485
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.383889436681951
+4.437253921568383
+4.5798483268413275
+4.75
+4.890373906609362
+4.955048454425975
+4.954466537076697
+3.383068941784411
+0.519387512494303
+0.14421518948927226
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8165844264500439
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2583256956715583
+2.290474143048891
+2.4912522817563203
+4.189445410195999
+4.209330964305835
+4.206280547491993
+4.206280547491993
+4.2243343199280865
+4.2243343199280865
+4.2243343199280865
+4.2243343199280865
+4.2243343199280865
+4.223061383739665
+4.242051477075593
+4.242051477075593
+4.2610653657318265
+4.2610653657318265
+4.2610653657318265
+4.2610653657318265
+4.262824069542097
+4.262824069542097
+4.332417955746382
+4.437940884635671
+4.469240513255388
+4.472314528156556
+4.415390784022273
+4.3229225252815375
+4.288085371739211
+4.288085371739211
+4.288085371739211
+4.30404791709206
+4.436755038723429
+4.512159965648277
+4.525148982297988
+4.503640969757196
+4.392755007565345
+4.311196423331765
+4.291975060537432
+4.291975060537432
+4.330384925242272
+4.461419383366735
+4.536738078402639
+4.466270560425329
+4.330384925242272
+4.315955355853033
+4.315955355853033
+4.321447907498091
+4.327610812997139
+4.327610812997139
+4.327610812997139
+4.474628088285788
+4.590893017970352
+4.590893017970352
+4.577601706931892
+4.425150693598551
+4.334367732518834
+4.334367732518834
+4.488865705516085
+4.6226847512678555
+4.5461808736533715
+4.353245755530516
+4.334367732518834
+4.334367732518834
+4.360278408571066
+4.360278408571066
+4.4702902099390895
+4.679949111556313
+4.587726262875978
+4.370736046613753
+4.570132142342086
+4.69281418415078
+4.679949111556313
+4.438734902106786
+4.334548778452816
+4.4794059114735
+4.7039874056290465
+4.7039874056290465
+4.7039874056290465
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.638630209477494
+4.585070500657054
+4.740587601014347
+4.753120216906312
+4.753120216906312
+4.753120216906312
+4.753120216906312
+4.740409145179578
+4.4834255156601905
+4.34817212384323
+4.3936790662814325
+4.673607649283862
+4.764851533139823
+4.764851533139823
+4.723400793471849
+4.459144616499847
+4.356951756012255
+4.356951756012255
+4.356951756012255
+4.3823982849430525
+4.3823982849430525
+4.365804009510418
+4.466027460208323
+4.753526533936163
+4.81729793871423
+4.651613169058767
+4.416711565401162
+4.365804009510418
+4.374649654129857
+4.374649654129857
+4.383418627512178
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.381142838860653
+4.477153658161236
+4.686946241913178
+4.863636363636363
+4.89873092730281
+4.772087701603047
+4.543333818464117
+4.398374751597663
+4.3699279208395785
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.383889436681951
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.383889436681951
+4.456023364726183
+4.582193139573988
+4.752475355734676
+4.863636363636363
+4.927710408327659
+4.937237907553747
+4.89873092730281
+4.772087701603047
+4.561499360541137
+0.6347160311526148
+0.40378354047257137
+0.14421518948927226
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+1.3791909450596869
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2583256956715583
+2.289362850895306
+2.408217453072381
+4.2924919737237754
+4.308499281988651
+4.269324112263929
+4.229957398299591
+4.226680526381007
+4.2243343199280865
+4.2243343199280865
+4.2243343199280865
+4.2243343199280865
+4.2243343199280865
+4.223061383739665
+4.242051477075593
+4.242051477075593
+4.242051477075593
+4.242051477075593
+4.242051477075593
+4.242051477075593
+4.243532542904225
+4.262824069542097
+4.280004643579906
+4.3666145665109415
+4.446713403848863
+4.469240513255388
+4.451997806809301
+4.352495164972238
+4.268914746068856
+4.268914746068856
+4.268914746068856
+4.288085371739211
+4.369978307763559
+4.498827660612522
+4.525148982297988
+4.525148982297988
+4.456924017946896
+4.326291782126872
+4.291975060537432
+4.272744302947785
+4.291975060537432
+4.429161646579058
+4.536738078402639
+4.478034066281278
+4.330384925242272
+4.296737346030571
+4.296737346030571
+4.296737346030571
+4.321447907498091
+4.327610812997139
+4.340622378893027
+4.514317882790707
+4.590893017970352
+4.590893017970352
+4.571165324280725
+4.373530165834871
+4.308646412990123
+4.346668293077823
+4.529291803534183
+4.6226847512678555
+4.488865705516085
+4.334367732518834
+4.334367732518834
+4.334367732518834
+4.315633110904785
+4.372230331526471
+4.620764512081358
+4.652239127360683
+4.419759557158932
+4.491801074676181
+4.69281418415078
+4.69281418415078
+4.510643118126104
+4.334548778452816
+4.419759557158932
+4.676655864461275
+4.7039874056290465
+4.7039874056290465
+4.7039874056290465
+4.7039874056290465
+4.716549695711502
+4.716549695711502
+4.69066493212302
+4.7039874056290465
+4.716549695711502
+4.728275832763921
+4.740587601014347
+4.753120216906312
+4.753120216906312
+4.629292470505407
+4.365563274055217
+4.365563274055217
+4.592455460018538
+4.753120216906312
+4.753120216906312
+4.726826567571509
+4.4834255156601905
+4.34817212384323
+4.34817212384323
+4.356951756012255
+4.356951756012255
+4.356951756012255
+4.340367332701628
+4.484563222116503
+4.771738592817006
+4.804805898398897
+4.6134253459783015
+4.3823982849430525
+4.349649451498392
+4.349649451498392
+4.365804009510418
+4.363362166374713
+4.372291676433294
+4.381142838860653
+4.381142838860653
+4.381142838860653
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.381142838860653
+4.495245211892045
+4.702967088304484
+4.8584171544605805
+4.872529839105629
+4.741519592641198
+4.519591849810563
+4.389854478801128
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.383889436681951
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.398374751597663
+4.472934417851474
+4.6149276187135
+4.772087701603047
+4.8816267976576375
+4.927710408327659
+4.937237907553747
+4.914400272933426
+4.8435411247040765
+4.672383068369923
+4.470708196465817
+4.38210182753176
+4.294309041013602
+0.7504206761995382
+0.3172647300558147
+0.11537255792817191
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.14421518948927226
+2.388738116386298
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2583256956715583
+2.285534687051869
+2.325359830869213
+3.617318141801828
+4.416211745185024
+4.404716270919072
+4.3815185041079445
+4.3549764147420245
+4.287843642057993
+4.206280547491993
+4.2243343199280865
+4.2243343199280865
+4.2243343199280865
+4.2243343199280865
+4.2243343199280865
+4.223061383739665
+4.222982331221807
+4.222982331221807
+4.222982331221807
+4.242051477075593
+4.242051477075593
+4.242051477075593
+4.262824069542097
+4.262824069542097
+4.297355012936745
+4.410905534283973
+4.469240513255388
+4.472314528156556
+4.397344032056895
+4.30404791709206
+4.268914746068856
+4.268914746068856
+4.268914746068856
+4.3387810056485865
+4.460527625640681
+4.519855048515984
+4.525148982297988
+4.494288594971252
+4.3571598314339965
+4.272744302947785
+4.272744302947785
+4.272744302947785
+4.393499185832402
+4.522944253053671
+4.504730153028199
+4.3495197621021
+4.277549565796887
+4.296737346030571
+4.296737346030571
+4.293673850523379
+4.299288615843937
+4.357161204883953
+4.548365008356385
+4.590893017970352
+4.590893017970352
+4.532577772405431
+4.337867549100633
+4.280057633263803
+4.359813865820298
+4.58852136234472
+4.603447413405319
+4.419975864651762
+4.315633110904785
+4.315633110904785
+4.315633110904785
+4.315633110904785
+4.51854399108187
+4.657957256979294
+4.531977395390569
+4.419759557158932
+4.652239127360683
+4.69281418415078
+4.570132142342086
+4.352512026646508
+4.370736046613753
+4.6311819590611725
+4.69281418415078
+4.7039874056290465
+4.7039874056290465
+4.7039874056290465
+4.7039874056290465
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.728275832763921
+4.7124748910849155
+4.43352271756291
+4.34817212384323
+4.5024876237639
+4.740409145179578
+4.753120216906312
+4.740409145179578
+4.540628945133276
+4.331160119833364
+4.331160119833364
+4.34817212384323
+4.34817212384323
+4.340367332701628
+4.340367332701628
+4.4778617586030505
+4.760933045899426
+4.791122580013401
+4.572949016875158
+4.365804009510418
+4.349649451498392
+4.349649451498392
+4.347162035888704
+4.347162035888704
+4.347162035888704
+4.363362166374713
+4.363362166374713
+4.356531416969382
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.396937585433321
+4.513544964723501
+4.715635347795578
+4.845146974506639
+4.830804040442458
+4.702967088304484
+4.495245211892045
+4.381142838860653
+4.351006892703883
+4.351006892703883
+4.360574686677896
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.3699279208395785
+4.356534660165839
+4.3699279208395785
+4.413341468587734
+4.507635367903604
+4.6531316008140005
+4.811314015577191
+4.89873092730281
+4.927710408327659
+4.937237907553747
+4.927710408327659
+4.861973736884253
+4.711105210397496
+4.543333818464117
+4.413341468587734
+4.354800864958
+4.342042743020706
+4.342042743020706
+3.3511576086498795
+0.8011534860348615
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4614592400945998
+2.1290389837345884
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2548582107881368
+2.2827562121821066
+2.316860727329738
+3.3708624975062507
+4.372000496216216
+4.356991511156253
+4.3549764147420245
+4.32489895623564
+4.234012918746753
+4.209330964305835
+4.209330964305835
+4.206280547491993
+4.2243343199280865
+4.2243343199280865
+4.206280547491993
+4.206280547491993
+4.206280547491993
+4.204412773112099
+4.222982331221807
+4.223061383739665
+4.223061383739665
+4.241708610495863
+4.241708610495863
+4.242051477075593
+4.243532542904225
+4.262824069542097
+4.3314640165552545
+4.435608687796262
+4.469240513255388
+4.435608687796262
+4.334884057988927
+4.2497299242386894
+4.2497299242386894
+4.2497299242386894
+4.30404791709206
+4.433425791049117
+4.507264102733968
+4.519855048515984
+4.517386266312412
+4.407725684983931
+4.288085371739211
+4.272744302947785
+4.253531640965672
+4.3571598314339965
+4.522944253053671
+4.514260294902164
+4.345288092527255
+4.274291885177432
+4.274291885177432
+4.274291885177432
+4.274291885177432
+4.280057633263803
+4.368581807425848
+4.571165324280725
+4.590893017970352
+4.590893017970352
+4.486557454816782
+4.299288615843937
+4.280057633263803
+4.406415144859124
+4.609159296939702
+4.557404016105389
+4.346668293077823
+4.2970852712950425
+4.315633110904785
+4.315633110904785
+4.419975864651762
+4.6226847512678555
+4.612010239449515
+4.394200048734022
+4.613471566824906
+4.679949111556313
+4.6311819590611725
+4.370736046613753
+4.334548778452816
+4.587726262875978
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.7039874056290465
+4.7039874056290465
+4.7039874056290465
+4.7039874056290465
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.545836543402008
+4.339552889545689
+4.412180874957363
+4.706471321315166
+4.753120216906312
+4.753120216906312
+4.610962775441974
+4.34817212384323
+4.331160119833364
+4.331160119833364
+4.331160119833364
+4.331160119833364
+4.331160119833364
+4.490078739911335
+4.74974235645692
+4.76455519928248
+4.528188482906883
+4.340367332701628
+4.349649451498392
+4.347162035888704
+4.347162035888704
+4.347162035888704
+4.347162035888704
+4.347162035888704
+4.347162035888704
+4.347162035888704
+4.347162035888704
+4.347162035888704
+4.356531416969382
+4.365815996925577
+4.365815996925577
+4.365815996925577
+4.396937585433321
+4.5471908537001
+4.7533262051837655
+4.845146974506639
+4.830804040442458
+4.677428238159029
+4.477153658161236
+4.365815996925577
+4.3367692831103835
+4.3367692831103835
+4.351006892703883
+4.351006892703883
+4.351006892703883
+4.360574686677896
+4.360574686677896
+4.3699279208395785
+4.383889436681951
+4.439444868219007
+4.543333818464117
+4.691713156761751
+4.8247089911549255
+4.914400272933426
+4.927710408327659
+4.937237907553747
+4.927710408327659
+4.8816267976576375
+4.752475355734676
+4.5798483268413275
+4.4210464010109
+4.368169508740839
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.342042743020706
+0.6559621560518574
+0.4902956806169465
+0.2018996889437794
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8825278501371363
+2.194405655016403
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.2740761375503107
+2.311089856465691
+2.417643118526024
+4.2206086405597025
+4.197071138300812
+4.17835618067723
+4.172308385784067
+4.189445410195999
+4.189445410195999
+4.209330964305835
+4.209330964305835
+4.209330964305835
+4.206280547491993
+4.206280547491993
+4.206280547491993
+4.206280547491993
+4.206280547491993
+4.206280547491993
+4.223061383739665
+4.223061383739665
+4.223061383739665
+4.223061383739665
+4.223061383739665
+4.223061383739665
+4.222982331221807
+4.243532542904225
+4.2788791060630365
+4.380505452216383
+4.459631613915226
+4.4569885833271
+4.397344032056895
+4.283023643208255
+4.2497299242386894
+4.2497299242386894
+4.2470628308633005
+4.384942201292258
+4.494343167426363
+4.519855048515984
+4.525148982297988
+4.452791429304008
+4.30404791709206
+4.253531640965672
+4.253531640965672
+4.3387810056485865
+4.508785647745265
+4.522944253053671
+4.360377731938609
+4.2549350594948505
+4.2549350594948505
+4.274291885177432
+4.274291885177432
+4.274291885177432
+4.401638244326036
+4.577601706931892
+4.590893017970352
+4.577601706931892
+4.4201516731586725
+4.280057633263803
+4.280057633263803
+4.469148351445398
+4.599002836703154
+4.494664026499109
+4.2970852712950425
+4.2970852712950425
+4.2970852712950425
+4.327610812997139
+4.58852136234472
+4.6356545149438695
+4.451863182214335
+4.536064520713749
+4.668188175579596
+4.652239127360683
+4.400860279484698
+4.313355124891885
+4.541142071059501
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.7039874056290465
+4.7039874056290465
+4.7039874056290465
+4.7039874056290465
+4.716549695711502
+4.716549695711502
+4.6705278718852306
+4.360278408571066
+4.331189168179998
+4.577606921643228
+4.740587601014347
+4.753120216906312
+4.673607649283862
+4.375380027466919
+4.331160119833364
+4.331160119833364
+4.331160119833364
+4.331160119833364
+4.3145871600249
+4.464400344563128
+4.726826567571509
+4.726826567571509
+4.509108982480729
+4.340367332701628
+4.324253970781363
+4.321599429301051
+4.321599429301051
+4.347162035888704
+4.347162035888704
+4.347162035888704
+4.331449528132573
+4.331449528132573
+4.331449528132573
+4.331449528132573
+4.347162035888704
+4.347162035888704
+4.347162035888704
+4.405121922133547
+4.566145733226151
+4.771843886220461
+4.858049419420977
+4.830804040442458
+4.677428238159029
+4.4592965723509685
+4.365815996925577
+4.3367692831103835
+4.3367692831103835
+4.3367692831103835
+4.3367692831103835
+4.3367692831103835
+4.3367692831103835
+4.351006892703883
+4.381142838860653
+4.4592965723509685
+4.590812073003308
+4.747524644977324
+4.84508484236849
+4.914400272933426
+4.914400272933426
+4.927710408327659
+4.927710408327659
+4.89873092730281
+4.791710163401963
+4.6149276187135
+4.453819126346629
+4.368169508740839
+4.354800864958
+4.354800864958
+4.354800864958
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.343755683862673
+3.3555162235739977
+0.9014987265478416
+0.3749445413935266
+0.14421518948927226
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7522202707914554
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.264828130972697
+2.2960327368554108
+2.3307592583286696
+2.6001616363942115
+4.240432247167452
+4.197071138300812
+4.17835618067723
+4.172308385784067
+4.189445410195999
+4.189445410195999
+4.189445410195999
+4.192161408306018
+4.192161408306018
+4.192161408306018
+4.206280547491993
+4.206280547491993
+4.206280547491993
+4.206280547491993
+4.206280547491993
+4.204412773112099
+4.204412773112099
+4.204412773112099
+4.204412773112099
+4.223061383739665
+4.223061383739665
+4.223061383739665
+4.222982331221807
+4.222982331221807
+4.314109655050784
+4.421815150590111
+4.469240513255388
+4.435608687796262
+4.333865602892183
+4.2280247218333145
+4.230559509060905
+4.2280247218333145
+4.334884057988927
+4.47123123823328
+4.502694043727065
+4.519855048515984
+4.489566010408934
+4.3571598314339965
+4.253531640965672
+4.25
+4.3229225252815375
+4.489566010408934
+4.517386266312412
+4.393499185832402
+4.2549350594948505
+4.2549350594948505
+4.2549350594948505
+4.2549350594948505
+4.2549350594948505
+4.446692229204218
+4.570197478760316
+4.590893017970352
+4.55656589041628
+4.3517175586913135
+4.260910257904718
+4.313053758086822
+4.541557094775911
+4.577601706931892
+4.411631509689652
+4.278776197768125
+4.278776197768125
+4.278776197768125
+4.51854399108187
+4.6356545149438695
+4.562746078431617
+4.51854399108187
+4.648026828159334
+4.657957256979294
+4.463924095533918
+4.295872802376734
+4.4702902099390895
+4.679949111556313
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.7039874056290465
+4.7039874056290465
+4.7039874056290465
+4.7039874056290465
+4.508225910052507
+4.295872802376734
+4.438734902106786
+4.69066493212302
+4.728275832763921
+4.715349243520801
+4.445436290543406
+4.331160119833364
+4.331160119833364
+4.331160119833364
+4.3145871600249
+4.3145871600249
+4.464400344563128
+4.7124748910849155
+4.726826567571509
+4.464400344563128
+4.3145871600249
+4.311815282769911
+4.321599429301051
+4.321599429301051
+4.321599429301051
+4.321599429301051
+4.321599429301051
+4.331449528132573
+4.331449528132573
+4.331449528132573
+4.331449528132573
+4.331449528132573
+4.331449528132573
+4.3799934275166095
+4.553807499358993
+4.753526533936163
+4.843246627448329
+4.830804040442458
+4.65822873316879
+4.4592965723509685
+4.365815996925577
+4.3367692831103835
+4.3367692831103835
+4.3367692831103835
+4.3367692831103835
+4.3367692831103835
+4.3367692831103835
+4.372291676433294
+4.477153658161236
+4.623573129679567
+4.771843886220461
+4.8584171544605805
+4.8848078319332995
+4.8946862347346505
+4.911393962819666
+4.914400272933426
+4.89873092730281
+4.811314015577191
+4.633974596215562
+4.470708196465817
+4.38210182753176
+4.354800864958
+4.354800864958
+4.354800864958
+4.354800864958
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.336426518579296
+0.6932492436805235
+0.547964844339119
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+1.4721776460987375
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2583256956715583
+2.2770293828519175
+2.3133893077813523
+2.339865608300966
+2.604089149770337
+4.1708710668561695
+4.197071138300812
+4.17835618067723
+4.17835618067723
+4.172308385784067
+4.172308385784067
+4.172308385784067
+4.172308385784067
+4.192161408306018
+4.192161408306018
+4.192161408306018
+4.206280547491993
+4.206280547491993
+4.206280547491993
+4.188330631831401
+4.188330631831401
+4.188330631831401
+4.204412773112099
+4.204412773112099
+4.204412773112099
+4.204412773112099
+4.204412773112099
+4.204412773112099
+4.222982331221807
+4.241708610495863
+4.365645075900559
+4.459631613915226
+4.4569885833271
+4.380505452216383
+4.260323041219008
+4.2280247218333145
+4.2280247218333145
+4.281840839476571
+4.430470736639508
+4.502694043727065
+4.502694043727065
+4.512159965648277
+4.404476728520448
+4.265623984992915
+4.2305425448815
+4.30404791709206
+4.479478935593598
+4.512159965648277
+4.390161428226321
+4.2549350594948505
+4.235636591185085
+4.235636591185085
+4.2549350594948505
+4.269458948305532
+4.492262563861891
+4.563123965563946
+4.583108031777054
+4.514317882790707
+4.293673850523379
+4.260910257904718
+4.387552875155975
+4.577601706931892
+4.541557094775911
+4.293673850523379
+4.252737371600075
+4.278776197768125
+4.419975864651762
+4.6226847512678555
+4.609159296939702
+4.58852136234472
+4.6356545149438695
+4.6356545149438695
+4.51854399108187
+4.301991565390525
+4.410395058116138
+4.666403422086015
+4.679949111556313
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.7039874056290465
+4.7039874056290465
+4.638630209477494
+4.341633051398518
+4.331189168179998
+4.638630209477494
+4.716549695711502
+4.716549695711502
+4.52708414790585
+4.313355124891885
+4.32212166094401
+4.32212166094401
+4.3145871600249
+4.3145871600249
+4.452260404566443
+4.723400793471849
+4.726826567571509
+4.464400344563128
+4.311815282769911
+4.29565309083286
+4.29565309083286
+4.311815282769911
+4.311815282769911
+4.321599429301051
+4.321599429301051
+4.305945600391937
+4.305945600391937
+4.305945600391937
+4.331449528132573
+4.331449528132573
+4.363362166374713
+4.515666105800818
+4.736141839889283
+4.81729793871423
+4.791122580013401
+4.644528994853105
+4.445120250365041
+4.34127725333279
+4.3367692831103835
+4.3367692831103835
+4.3367692831103835
+4.3367692831103835
+4.351006892703883
+4.390836464365371
+4.507271327463094
+4.656063156626178
+4.790040730028535
+4.845146974506639
+4.858049419420977
+4.858049419420977
+4.869137689341459
+4.8848078319332995
+4.872529839105629
+4.80780959574163
+4.6482824413086865
+4.4837541190057655
+4.374952900097492
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.220535793049947
+2.6231863385468026
+0.7785997875515944
+0.40378354047257137
+0.17305757537147226
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.17305757537147226
+2.406513659435264
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.2444401175322657
+2.251535166886691
+2.264828130972697
+2.2935260049111688
+2.3218005945232756
+2.345459739590308
+2.535262654748501
+4.228958610384918
+4.181121458434199
+4.1626765496543685
+4.1626765496543685
+4.1626765496543685
+4.1555211100661475
+4.172308385784067
+4.172308385784067
+4.172308385784067
+4.192161408306018
+4.192161408306018
+4.192161408306018
+4.188330631831401
+4.188330631831401
+4.188330631831401
+4.188330631831401
+4.188330631831401
+4.185803260524182
+4.185803260524182
+4.185803260524182
+4.185803260524182
+4.204412773112099
+4.204412773112099
+4.203881081227029
+4.203881081227029
+4.279174511318519
+4.423669530065321
+4.469240513255388
+4.435608687796262
+4.314109655050784
+4.208043257279148
+4.209015986244905
+4.245536640869366
+4.383033768797953
+4.490181602580803
+4.502694043727065
+4.507264102733968
+4.446948255186931
+4.282040951144447
+4.2305425448815
+4.30404791709206
+4.479478935593598
+4.525148982297988
+4.404476728520448
+4.25
+4.235636591185085
+4.235636591185085
+4.235636591185085
+4.308286843238249
+4.514260294902164
+4.563123965563946
+4.570197478760316
+4.4514998653577775
+4.260910257904718
+4.280057633263803
+4.486557454816782
+4.590893017970352
+4.4514998653577775
+4.241889569491942
+4.249171266666321
+4.321447907498091
+4.58852136234472
+4.6226847512678555
+4.6226847512678555
+4.6356545149438695
+4.6356545149438695
+4.571250454545515
+4.294017679194993
+4.346668293077823
+4.641235384144579
+4.679949111556313
+4.679949111556313
+4.679949111556313
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.508225910052507
+4.295872802376734
+4.489286681787702
+4.7039874056290465
+4.716549695711502
+4.622018630619488
+4.331189168179998
+4.295872802376734
+4.278812254077182
+4.295872802376734
+4.305089596671972
+4.426558867884018
+4.723400793471849
+4.726826567571509
+4.46182495923458
+4.311815282769911
+4.29565309083286
+4.29565309083286
+4.29565309083286
+4.29565309083286
+4.29565309083286
+4.29565309083286
+4.29565309083286
+4.305945600391937
+4.305945600391937
+4.305945600391937
+4.337781444744843
+4.49671038392666
+4.718351151603021
+4.81729793871423
+4.791122580013401
+4.644528994853105
+4.4269312816945705
+4.331449528132573
+4.316287496333247
+4.316287496333247
+4.316287496333247
+4.34127725333279
+4.405121922133547
+4.532028815955536
+4.677428238159029
+4.80780959574163
+4.845146974506639
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.797628303871802
+4.656063156626178
+4.495245211892045
+4.381142838860653
+4.321318784036765
+4.321318784036765
+4.321318784036765
+4.331811824420404
+4.342042743020706
+4.342042743020706
+4.32994289459005
+4.32994289459005
+4.32994289459005
+4.342042743020706
+4.342042743020706
+4.342042743020706
+4.343755683862673
+4.270836668788456
+2.6391154618309955
+0.9303337382556007
+0.4902956806169465
+0.2595829718925291
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.7522202707914554
+2.227587031250235
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.2060580360966195
+2.199780343719392
+2.154609032791459
+2.091620099140353
+2.0249250210520437
+2.1226824881662285
+2.410142949609684
+3.3395423085271414
+4.206236197371835
+4.1626765496543685
+4.1626765496543685
+4.1626765496543685
+4.1626765496543685
+4.1555211100661475
+4.172308385784067
+4.17835618067723
+4.181121458434199
+4.181121458434199
+4.175265798540549
+4.192161408306018
+4.188330631831401
+4.188330631831401
+4.170559406447948
+4.170559406447948
+4.170559406447948
+4.188330631831401
+4.185803260524182
+4.185803260524182
+4.185803260524182
+4.185803260524182
+4.185803260524182
+4.184777003835199
+4.223061383739665
+4.347444600831109
+4.459631613915226
+4.4569885833271
+4.380505452216383
+4.241708610495863
+4.185396469740754
+4.204796272439868
+4.333865602892183
+4.4811159587853355
+4.502694043727065
+4.502694043727065
+4.47123123823328
+4.3387810056485865
+4.2305425448815
+4.282040951144447
+4.479478935593598
+4.525148982297988
+4.421450634493992
+4.246184195954533
+4.235636591185085
+4.235636591185085
+4.235636591185085
+4.34170489193806
+4.543029917126171
+4.563123965563946
+4.556828312189985
+4.364189297235156
+4.241889569491942
+4.313053758086822
+4.55656589041628
+4.577601706931892
+4.3468683991859995
+4.239333061763718
+4.258480407358802
+4.505917988700361
+4.613831733428877
+4.6226847512678555
+4.6226847512678555
+4.6226847512678555
+4.61789817246824
+4.346668293077823
+4.28666792956462
+4.603447413405319
+4.657957256979294
+4.668188175579596
+4.679949111556313
+4.679949111556313
+4.679949111556313
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.662029905531306
+4.341633051398518
+4.341633051398518
+4.654819004174287
+4.7039874056290465
+4.676655864461275
+4.400860279484698
+4.278812254077182
+4.278812254077182
+4.278812254077182
+4.278812254077182
+4.382065911779677
+4.6856931845997325
+4.715349243520801
+4.4834255156601905
+4.29565309083286
+4.29565309083286
+4.29565309083286
+4.29565309083286
+4.29565309083286
+4.29565309083286
+4.280067129658118
+4.280067129658118
+4.280067129658118
+4.29565309083286
+4.311815282769911
+4.437966363085956
+4.673739406868355
+4.778506435350758
+4.778506435350758
+4.644528994853105
+4.4269312816945705
+4.331449528132573
+4.316287496333247
+4.316287496333247
+4.316287496333247
+4.35442503592916
+4.500816995645366
+4.66327376376416
+4.786659476922233
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.858049419420977
+4.815335102610012
+4.696574961377673
+4.507271327463094
+4.38850465887066
+4.321318784036765
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.321318784036765
+4.321318784036765
+4.321318784036765
+4.331811824420404
+4.32994289459005
+4.32994289459005
+4.32994289459005
+4.32994289459005
+4.32994289459005
+4.343755683862673
+4.342042743020706
+3.4128708719498393
+0.6611918524689804
+0.9220007075428525
+0.547964844339119
+0.2884240647434684
+0.11537255792817191
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.3461049112705705
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.38453231097115825
+0.40378354047257137
+0.4326218123061114
+0.5191309563131474
+0.6632793931627319
+0.8647921722156005
+0.9443664684730466
+0.7931129431694348
+2.6231417127608503
+3.49049925704238
+4.229762156405545
+4.206980666498831
+4.18527631598984
+4.18527631598984
+4.1626765496543685
+4.1626765496543685
+4.1626765496543685
+4.1626765496543685
+4.181121458434199
+4.181121458434199
+4.175265798540549
+4.170559406447948
+4.170559406447948
+4.175265798540549
+4.175265798540549
+4.175265798540549
+4.170559406447948
+4.167287633500955
+4.167287633500955
+4.167287633500955
+4.185803260524182
+4.185803260524182
+4.185803260524182
+4.185803260524182
+4.279174511318519
+4.421815150590111
+4.469240513255388
+4.421815150590111
+4.296670760432378
+4.181718697742469
+4.185396469740754
+4.281840839476571
+4.442556880764622
+4.502694043727065
+4.502694043727065
+4.494343167426363
+4.369978307763559
+4.226705965117246
+4.262824069542097
+4.479478935593598
+4.525148982297988
+4.434554479488243
+4.243532542904225
+4.211117229499433
+4.216438603867498
+4.216438603867498
+4.393499185832402
+4.550140350585692
+4.563123965563946
+4.514260294902164
+4.288894789602504
+4.2230503108410335
+4.4201516731586725
+4.590893017970352
+4.514317882790707
+4.252475355022676
+4.220334472865379
+4.399245415273999
+4.590893017970352
+4.613831733428877
+4.6226847512678555
+4.6226847512678555
+4.6226847512678555
+4.392622212842695
+4.275537753904143
+4.571250454545515
+4.6356545149438695
+4.6356545149438695
+4.657957256979294
+4.669446132552235
+4.679949111556313
+4.679949111556313
+4.69281418415078
+4.69281418415078
+4.534232616473268
+4.278812254077182
+4.522663592490754
+4.69281418415078
+4.7039874056290465
+4.522663592490754
+4.278812254077182
+4.262256704184564
+4.278812254077182
+4.278812254077182
+4.341633051398518
+4.638630209477494
+4.7039874056290465
+4.4702902099390895
+4.275657408429723
+4.285516381473171
+4.29565309083286
+4.29565309083286
+4.29565309083286
+4.280067129658118
+4.280067129658118
+4.280067129658118
+4.280067129658118
+4.280067129658118
+4.375380027466919
+4.610962775441974
+4.7657943350103915
+4.7657943350103915
+4.655506333614664
+4.437966363085956
+4.305945600391937
+4.29089492712542
+4.30174537583588
+4.347162035888704
+4.445120250365041
+4.6112620879661055
+4.753526533936163
+4.804805898398897
+4.828385320427853
+4.828385320427853
+4.828385320427853
+4.843246627448329
+4.858049419420977
+4.815335102610012
+4.715635347795578
+4.5471908537001
+4.407435659740823
+4.3367692831103835
+4.321318784036765
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3193194762604055
+4.331637880230829
+4.331637880230829
+4.315788338256202
+3.3448636559069373
+1.3705756052118994
+0.8141402729636837
+0.7497229403733847
+0.5191309563131474
+0.3172647300558147
+0.14421518948927226
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.08652970395641724
+0.230741499582245
+0.40378354047257137
+0.6056272403357319
+0.8557848105107277
+0.7567958781011281
+2.656497955214265
+3.5190043127715844
+4.373454032767738
+4.359130606419303
+4.330714981509336
+4.302784596518338
+4.281359252824272
+4.247576635145128
+4.206980666498831
+4.18527631598984
+4.165678958020959
+4.165678958020959
+4.158771858263004
+4.175265798540549
+4.175265798540549
+4.158771858263004
+4.158771858263004
+4.158771858263004
+4.153071544848954
+4.167287633500955
+4.167287633500955
+4.170559406447948
+4.188330631831401
+4.170559406447948
+4.163998999246982
+4.2016895928872255
+4.347444600831109
+4.459631613915226
+4.459631613915226
+4.347444600831109
+4.2016895928872255
+4.166017143754156
+4.242051477075593
+4.412959269104636
+4.490181602580803
+4.502694043727065
+4.502694043727065
+4.415390784022273
+4.243532542904225
+4.262824069542097
+4.479478935593598
+4.525148982297988
+4.434554479488243
+4.224183288970262
+4.187731060706238
+4.1917640638603615
+4.2305425448815
+4.459189377254781
+4.550140350585692
+4.563123965563946
+4.459189377254781
+4.235636591185085
+4.2549350594948505
+4.508358881478538
+4.590893017970352
+4.399245415273999
+4.220334472865379
+4.308286843238249
+4.563841360958199
+4.590893017970352
+4.590893017970352
+4.613831733428877
+4.6226847512678555
+4.464612354157994
+4.257331764657746
+4.5461808736533715
+4.6356545149438695
+4.6356545149438695
+4.6356545149438695
+4.645199135042
+4.657957256979294
+4.679949111556313
+4.679949111556313
+4.666403422086015
+4.375275290067039
+4.3634085827425615
+4.676655864461275
+4.69281418415078
+4.629658286286764
+4.323181495798052
+4.262256704184564
+4.262256704184564
+4.262256704184564
+4.313355124891885
+4.605035406704348
+4.7039874056290465
+4.489286681787702
+4.2589398463924955
+4.2589398463924955
+4.275657408429723
+4.285516381473171
+4.26937230025452
+4.280067129658118
+4.280067129658118
+4.280067129658118
+4.280067129658118
+4.33685742855189
+4.533871840011103
+4.7124748910849155
+4.753120216906312
+4.673607649283862
+4.46182495923458
+4.311815282769911
+4.2765310720585505
+4.29089492712542
+4.365563274055217
+4.528188482906883
+4.691710049093213
+4.791122580013401
+4.81729793871423
+4.81729793871423
+4.81729793871423
+4.81729793871423
+4.81729793871423
+4.791122580013401
+4.718351151603021
+4.553807499358993
+4.416711565401162
+4.3367692831103835
+4.323160554031693
+4.321318784036765
+4.321318784036765
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.310241109073139
+4.310241109073139
+4.334450971017986
+3.337756614473365
+2.633505214007222
+0.7101596369167549
+0.894993423848998
+0.8073046643997408
+0.6344548999581567
+0.4614592400945998
+0.2884240647434684
+0.14421518948927226
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.08652970395641724
+0.17305757537147226
+0.3461049112705705
+0.5191309563131474
+0.6821398934982863
+0.950145549599057
+0.740608520590464
+2.6509483927138517
+3.466713592697833
+4.394440664701463
+4.3967083155925035
+4.395466391668603
+4.394440664701463
+4.382782218537319
+4.356991511156253
+4.330714981509336
+4.281359252824272
+4.229762156405545
+4.187847747763415
+4.158771858263004
+4.158771858263004
+4.158771858263004
+4.158771858263004
+4.158771858263004
+4.158771858263004
+4.153071544848954
+4.153071544848954
+4.153071544848954
+4.149583103528658
+4.167422873107778
+4.167422873107778
+4.163998999246982
+4.279174511318519
+4.421815150590111
+4.469240513255388
+4.407393740532426
+4.24013464324727
+4.162310183365068
+4.184777003835199
+4.3666145665109415
+4.477376686372026
+4.502694043727065
+4.502694043727065
+4.442556880764622
+4.2610653657318265
+4.262824069542097
+4.464382022636259
+4.525148982297988
+4.446948255186931
+4.243532542904225
+4.187731060706238
+4.187731060706238
+4.246184195954533
+4.499344690879607
+4.550140350585692
+4.550140350585692
+4.373020992536642
+4.194387164274222
+4.3201913295651035
+4.570197478760316
+4.541557094775911
+4.2719433122175925
+4.239333061763718
+4.524127578822819
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.603682897163525
+4.539257872104618
+4.267819388224821
+4.529291803534183
+4.6356545149438695
+4.6356545149438695
+4.6356545149438695
+4.6356545149438695
+4.6356545149438695
+4.657957256979294
+4.668188175579596
+4.5968888406633095
+4.304971199174973
+4.552289840148148
+4.69281418415078
+4.69281418415078
+4.444957838142898
+4.262256704184564
+4.262256704184564
+4.262256704184564
+4.278812254077182
+4.559466509233962
+4.7039874056290465
+4.501631338735831
+4.275657408429723
+4.2589398463924955
+4.2589398463924955
+4.2589398463924955
+4.2589398463924955
+4.2589398463924955
+4.26937230025452
+4.280067129658118
+4.29565309083286
+4.46182495923458
+4.690283055605783
+4.753120216906312
+4.706471321315166
+4.4834255156601905
+4.302198258940722
+4.250998675515501
+4.29565309083286
+4.412180874957363
+4.59029185398596
+4.734199167934266
+4.791447617972283
+4.791447617972283
+4.791447617972283
+4.81729793871423
+4.81729793871423
+4.804805898398897
+4.718351151603021
+4.572949016875158
+4.408981720886764
+4.316287496333247
+4.287899053243774
+4.30174537583588
+4.312526019392585
+4.310612764776552
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.310241109073139
+4.183067585707844
+3.3884368516118326
+2.633124237316123
+0.721251949749206
+0.890687515674526
+0.8647921722156005
+0.6920998310633086
+0.6056272403357319
+0.4614592400945998
+0.3461049112705705
+0.2018996889437794
+0.11537255792817191
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.0576866479940108
+0.14421518948927226
+0.2595829718925291
+0.40378354047257137
+0.547964844339119
+0.7108936078332695
+0.9134702960435828
+0.7706936374269509
+2.6169002176238445
+3.373718318227698
+4.347339821427706
+4.348018961012276
+4.373887843350183
+4.394440664701463
+4.404716270919072
+4.404716270919072
+4.3935208595585
+4.3815185041079445
+4.342147638371092
+4.2770241344525415
+4.202832528070241
+4.175265798540549
+4.158771858263004
+4.142857142857142
+4.142857142857142
+4.139252000516036
+4.149583103528658
+4.149583103528658
+4.149583103528658
+4.167422873107778
+4.149583103528658
+4.145249727007031
+4.203651089218543
+4.363078943678145
+4.469240513255388
+4.446713403848863
+4.29737398300434
+4.162310183365068
+4.162310183365068
+4.299316960299984
+4.4676065372348495
+4.490181602580803
+4.502694043727065
+4.4676065372348495
+4.281511118199585
+4.242051477075593
+4.464382022636259
+4.525148982297988
+4.446948255186931
+4.224183288970262
+4.168302313602947
+4.168302313602947
+4.298852288588964
+4.522944253053671
+4.550140350585692
+4.514260294902164
+4.265623984992915
+4.213656267826046
+4.459189377254781
+4.5756616620083115
+4.429161646579058
+4.220334472865379
+4.4514998653577775
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.571165324280725
+4.293673850523379
+4.494664026499109
+4.6226847512678555
+4.6226847512678555
+4.6226847512678555
+4.6356545149438695
+4.6356545149438695
+4.6356545149438695
+4.6356545149438695
+4.470438427378001
+4.365786209121337
+4.652239127360683
+4.679949111556313
+4.5968888406633095
+4.287058569715656
+4.262256704184564
+4.262256704184564
+4.2589398463924955
+4.501631338735831
+4.69281418415078
+4.559466509233962
+4.275657408429723
+4.242808832535492
+4.242808832535492
+4.2589398463924955
+4.2589398463924955
+4.2589398463924955
+4.242808832535492
+4.242808832535492
+4.360278408571066
+4.603299912056848
+4.740409145179578
+4.726826567571509
+4.571600036028576
+4.339552889545689
+4.265148157089352
+4.302198258940722
+4.4427961124456905
+4.639065325410318
+4.726826567571509
+4.764851533139823
+4.764851533139823
+4.778088299149454
+4.791447617972283
+4.778506435350758
+4.709360105415576
+4.566403774558057
+4.4269312816945705
+4.331449528132573
+4.287899053243774
+4.287899053243774
+4.287899053243774
+4.287899053243774
+4.285895083247516
+4.285895083247516
+4.285895083247516
+4.297243754933228
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.3084046300143655
+4.296239884869885
+4.321318784036765
+4.351006892703883
+4.326333543660183
+3.334370823290448
+0.659306673328599
+0.8372004512939295
+0.8647921722156005
+0.6920998310633086
+0.6056272403357319
+0.4902956806169465
+0.40378354047257137
+0.2884240647434684
+0.2018996889437794
+0.11537255792817191
+0.0576866479940108
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.08652970395641724
+0.17305757537147226
+0.2884240647434684
+0.40378354047257137
+0.547964844339119
+0.6632793931627319
+0.8647921722156005
+0.8372004512939295
+0.6680677313924832
+3.320055084365214
+3.603891500879352
+4.255173391471447
+4.302784596518338
+4.3453595061404515
+4.382782218537319
+4.404716270919072
+4.4149539674021
+4.416211745185024
+4.394872548397378
+4.356613498206495
+4.291189724650094
+4.218259620984635
+4.1555211100661475
+4.139252000516036
+4.139252000516036
+4.132088428280674
+4.132088428280674
+4.132088428280674
+4.132088428280674
+4.149583103528658
+4.149583103528658
+4.167422873107778
+4.2986838154209295
+4.446713403848863
+4.4569885833271
+4.347517694966176
+4.182830894787021
+4.142890501858993
+4.260543986543284
+4.437940884635671
+4.490181602580803
+4.502694043727065
+4.4811159587853355
+4.316623615341484
+4.242051477075593
+4.460527625640681
+4.525148982297988
+4.431281148704458
+4.224183288970262
+4.16484708315636
+4.16484708315636
+4.369978307763559
+4.530776542855393
+4.550140350585692
+4.4547427227222975
+4.194387164274222
+4.28500654445743
+4.543029917126171
+4.543029917126171
+4.265623984992915
+4.361653407159187
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.577601706931892
+4.3468683991859995
+4.469148351445398
+4.613831733428877
+4.6226847512678555
+4.6226847512678555
+4.6226847512678555
+4.6226847512678555
+4.6356545149438695
+4.6226847512678555
+4.373530165834871
+4.529291803534183
+4.657957256979294
+4.655021703819993
+4.413190127900948
+4.246305058254497
+4.242808832535492
+4.2589398463924955
+4.429521246676579
+4.679949111556313
+4.605035406704348
+4.283930830651066
+4.242808832535492
+4.242808832535492
+4.242808832535492
+4.242808832535492
+4.242808832535492
+4.242808832535492
+4.275657408429723
+4.4702902099390895
+4.6870297623887245
+4.716549695711502
+4.630575200519795
+4.386252292638426
+4.265148157089352
+4.305089596671972
+4.4834255156601905
+4.673607649283862
+4.740409145179578
+4.753120216906312
+4.753120216906312
+4.753120216906312
+4.740409145179578
+4.706471321315166
+4.571600036028576
+4.430850873866957
+4.321599429301051
+4.2629461847989285
+4.2629461847989285
+4.2629461847989285
+4.287899053243774
+4.287899053243774
+4.285895083247516
+4.285895083247516
+4.285895083247516
+4.285895083247516
+4.272808871106918
+4.272808871106918
+4.285895083247516
+4.324673792101699
+4.372291676433294
+4.441702784036244
+4.5471908537001
+4.457774122525462
+3.348727737271513
+0.6995574804335964
+0.9220194164042796
+0.7683375209644598
+0.6344548999581567
+0.5191309563131474
+0.3749445413935266
+0.2884240647434684
+0.2018996889437794
+0.14421518948927226
+0.08652970395641724
+0.0576866479940108
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.08652970395641724
+0.17305757537147226
+0.2595829718925291
+0.3749445413935266
+0.5191309563131474
+0.6344548999581567
+0.7683375209644598
+0.9650888789852514
+0.7249732392691373
+2.6491430356740824
+3.437664135829692
+4.154824223110502
+4.190487592151753
+4.253456513491267
+4.330714981509336
+4.3815185041079445
+4.404716270919072
+4.416211745185024
+4.416211745185024
+4.405288240837459
+4.365873551370795
+4.291189724650094
+4.200425193415516
+4.139252000516036
+4.139252000516036
+4.115142843788829
+4.132088428280674
+4.132088428280674
+4.132088428280674
+4.149583103528658
+4.132088428280674
+4.227703995652505
+4.406095101803691
+4.469240513255388
+4.407393740532426
+4.2243248754084695
+4.145249727007031
+4.203651089218543
+4.395780918368115
+4.490181602580803
+4.490181602580803
+4.477376686372026
+4.351015773715063
+4.241708610495863
+4.442556880764622
+4.519855048515984
+4.434554479488243
+4.201961535726061
+4.145078211515794
+4.1822904867059885
+4.431281148704458
+4.525148982297988
+4.530776542855393
+4.354691823497583
+4.1752910088450745
+4.425098568663218
+4.563123965563946
+4.442260080149436
+4.28500654445743
+4.556828312189985
+4.583108031777054
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.415569757481522
+4.433633609264543
+4.590893017970352
+4.590893017970352
+4.613831733428877
+4.6226847512678555
+4.6226847512678555
+4.6226847512678555
+4.573160800504121
+4.425150693598551
+4.6226847512678555
+4.6356545149438695
+4.5789535989891
+4.257331764657746
+4.242808832535492
+4.227383303293633
+4.341633051398518
+4.646852001521213
+4.629658286286764
+4.323181495798052
+4.242808832535492
+4.242808832535492
+4.242808832535492
+4.242808832535492
+4.242808832535492
+4.227383303293633
+4.341633051398518
+4.587726262875978
+4.716549695711502
+4.6870297623887245
+4.4702902099390895
+4.275657408429723
+4.292861927065423
+4.4702902099390895
+4.6804377029718225
+4.753120216906312
+4.764851533139823
+4.753120216906312
+4.753120216906312
+4.706471321315166
+4.592455460018538
+4.423839349817026
+4.302198258940722
+4.265148157089352
+4.250998675515501
+4.250998675515501
+4.2629461847989285
+4.2629461847989285
+4.2629461847989285
+4.2629461847989285
+4.272808871106918
+4.272808871106918
+4.285895083247516
+4.3143068154002675
+4.337781444744843
+4.40657002550727
+4.494233170364849
+4.623509128848177
+4.718351151603021
+4.7879891805359165
+4.49671038392666
+3.355910963566751
+0.7318749366141102
+0.9630837087124622
+0.7209149579373282
+0.6056272403357319
+0.4614592400945998
+0.3172647300558147
+0.230741499582245
+0.14421518948927226
+0.08652970395641724
+0.0576866479940108
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.11537255792817191
+0.2018996889437794
+0.2884240647434684
+0.4326218123061114
+0.547964844339119
+0.6632793931627319
+0.8456634862694745
+0.8051255375555195
+1.370689976024737
+3.361067436462658
+4.273095217912634
+4.137131613836097
+4.179270418973757
+4.234197004870354
+4.3173943306794955
+4.3815185041079445
+4.416211745185024
+4.426368701027112
+4.426792664882731
+4.405171091330308
+4.351449655609735
+4.24386640352576
+4.1626765496543685
+4.12374457827202
+4.115142843788829
+4.132088428280674
+4.115142843788829
+4.115142843788829
+4.132088428280674
+4.172308385784067
+4.348001132847382
+4.4569885833271
+4.43358416697286
+4.280458950753834
+4.126667610015392
+4.167422873107778
+4.347444600831109
+4.477376686372026
+4.490181602580803
+4.490181602580803
+4.3666145665109415
+4.242436409101377
+4.440598382711996
+4.507264102733968
+4.415390784022273
+4.1822904867059885
+4.145078211515794
+4.242051477075593
+4.489566010408934
+4.525148982297988
+4.503640969757196
+4.243532542904225
+4.246184195954533
+4.528831176637506
+4.536738078402639
+4.35787139154943
+4.514260294902164
+4.563123965563946
+4.570197478760316
+4.590893017970352
+4.590893017970352
+4.497832329201184
+4.433633609264543
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.599002836703154
+4.613831733428877
+4.6226847512678555
+4.5461808736533715
+4.555438561670694
+4.6356545149438695
+4.6356545149438695
+4.411631509689652
+4.19219040425837
+4.202371696128198
+4.257331764657746
+4.5799545547395955
+4.652239127360683
+4.372230331526471
+4.227383303293633
+4.242808832535492
+4.242808832535492
+4.242808832535492
+4.227383303293633
+4.242808832535492
+4.4512597008547115
+4.671919701176808
+4.7039874056290465
+4.577606921643228
+4.341633051398518
+4.275657408429723
+4.4512597008547115
+4.638630209477494
+4.716549695711502
+4.728275832763921
+4.740587601014347
+4.7124748910849155
+4.610962775441974
+4.4427961124456905
+4.302198258940722
+4.250998675515501
+4.250998675515501
+4.250998675515501
+4.250998675515501
+4.250998675515501
+4.237732583564444
+4.237732583564444
+4.250998675515501
+4.280067129658118
+4.328474169847666
+4.4195207143370485
+4.487575113685434
+4.623509128848177
+4.718351151603021
+4.760933045899426
+4.791122580013401
+4.803471126886287
+4.656335710595208
+3.476393452796253
+2.633861646758893
+0.7714249580945625
+0.9029921131331431
+0.6723626437589605
+0.5382175776289602
+0.3749445413935266
+0.2595829718925291
+0.14421518948927226
+0.08652970395641724
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.14421518948927226
+0.7108936078332695
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9379779678995588
+0.9630732565314721
+0.9769912611037048
+0.9295793432470756
+0.8753367724102397
+0.8285407428931473
+0.7853909093635885
+0.7450954734242421
+0.7219183786078611
+1.3086185082177466
+1.4674073676353294
+2.7441662409954692
+3.5383457622682686
+4.137439776966195
+4.113765550959549
+4.131105436808975
+4.1753404548542585
+4.253456513491267
+4.3453595061404515
+4.394872548397378
+4.416217764170109
+4.426792664882731
+4.416380983659691
+4.380974111341195
+4.291189724650094
+4.18527631598984
+4.12374457827202
+4.0990733560973345
+4.115142843788829
+4.115142843788829
+4.115142843788829
+4.132088428280674
+4.282391253599503
+4.43358416697286
+4.4569885833271
+4.3318824654794925
+4.149583103528658
+4.126667610015392
+4.2986838154209295
+4.462569174514068
+4.490181602580803
+4.490181602580803
+4.381564064933234
+4.296644023914582
+4.4811159587853355
+4.502694043727065
+4.399300493996371
+4.162560670484979
+4.145078211515794
+4.315731378555368
+4.512159965648277
+4.525148982297988
+4.431281148704458
+4.16484708315636
+4.407725684983931
+4.550140350585692
+4.48209627902764
+4.514260294902164
+4.563123965563946
+4.563123965563946
+4.563123965563946
+4.583108031777054
+4.549643730922412
+4.463995556498123
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.600699506003629
+4.59511905321376
+4.6226847512678555
+4.6226847512678555
+4.571250454545515
+4.241889569491942
+4.19219040425837
+4.19219040425837
+4.462167072541683
+4.645199135042
+4.436280735418166
+4.227383303293633
+4.227383303293633
+4.227383303293633
+4.227383303293633
+4.227383303293633
+4.287058569715656
+4.552289840148148
+4.69281418415078
+4.662029905531306
+4.4512597008547115
+4.275657408429723
+4.391289528959374
+4.605035406704348
+4.716549695711502
+4.728275832763921
+4.7039874056290465
+4.622018630619488
+4.4702902099390895
+4.323181495798052
+4.2538624055405
+4.250998675515501
+4.250998675515501
+4.250998675515501
+4.250998675515501
+4.250998675515501
+4.265148157089352
+4.305089596671972
+4.386252292638426
+4.4834255156601905
+4.59029185398596
+4.673607649283862
+4.738328274948345
+4.778506435350758
+4.778506435350758
+4.743407471224992
+4.66327376376416
+4.575166900574203
+4.463516460681111
+4.270280676361232
+3.0
+2.5298116246948874
+1.3307395744181507
+0.7128431035659979
+0.7405035296384099
+0.7763091941917066
+0.8285407428931473
+0.8867520004449787
+0.9499400819692401
+0.9717007050706465
+0.9379779678995588
+0.9379779678995588
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.9157781068691486
+0.7010944029779997
+0.11537255792817191
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.14421518948927226
+0.7128431035659979
+2.3410810950415235
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.301758598159419
+2.294952421195424
+2.27228354593505
+2.2503257854353
+2.2110819526399554
+2.152489662966116
+2.0691651155283464
+2.033646161431456
+2.101115010430744
+2.2274730204316917
+2.6027536568337393
+3.5438788910224837
+4.15206094808109
+4.124605143116053
+4.132471371425174
+4.131105436808975
+4.200740871072689
+4.302784596518338
+4.382954350785997
+4.416217764170109
+4.426792664882731
+4.427387113724792
+4.393549265162711
+4.308499281988651
+4.18527631598984
+4.12374457827202
+4.115142843788829
+4.115142843788829
+4.115142843788829
+4.115142843788829
+4.210991133064981
+4.3914638956194585
+4.4569885833271
+4.377925230742242
+4.185475408577178
+4.10650491814166
+4.244565779881304
+4.43358416697286
+4.486334337166265
+4.490181602580803
+4.4676065372348495
+4.490181602580803
+4.502694043727065
+4.502694043727065
+4.3666145665109415
+4.142789166892736
+4.145078211515794
+4.401672253049345
+4.525148982297988
+4.525148982297988
+4.315731378555368
+4.224183288970262
+4.508785647745265
+4.550140350585692
+4.550140350585692
+4.550140350585692
+4.563123965563946
+4.563123965563946
+4.563123965563946
+4.556828312189985
+4.549643730922412
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.613831733428877
+4.6226847512678555
+4.44374109684803
+4.174992929751723
+4.174992929751723
+4.335175833863043
+4.6226847512678555
+4.512109974050057
+4.209959269971465
+4.213340523077767
+4.200690125825689
+4.212804137539644
+4.212804137539644
+4.353245755530516
+4.629658286286764
+4.69281418415078
+4.587726262875978
+4.341633051398518
+4.323181495798052
+4.552289840148148
+4.69066493212302
+4.716549695711502
+4.676655864461275
+4.559466509233962
+4.37907711329853
+4.2589398463924955
+4.212804137539644
+4.212804137539644
+4.212804137539644
+4.227383303293633
+4.26937230025452
+4.339552889545689
+4.423839349817026
+4.533871840011103
+4.639065325410318
+4.7124748910849155
+4.740409145179578
+4.740409145179578
+4.7124748910849155
+4.673607649283862
+4.571600036028576
+4.46182495923458
+4.383281307781995
+4.324253970781363
+4.2744320876537865
+4.264794256022425
+3.3502799459558017
+2.323831802286236
+2.1124228235671483
+2.0395990526220777
+2.040817592526905
+2.1158475591049335
+2.1971107715186284
+2.2478906171684057
+2.2777796781485313
+2.294952421195424
+2.301758598159419
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.312187166661021
+2.353889367864523
+0.9630732565314721
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6344548999581567
+2.353197272802601
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2743070608481237
+2.2503257854353
+2.224605730129962
+2.182689437410736
+2.1098134426472237
+2.015795365275975
+2.0860965787440415
+2.1511077696207925
+2.209217455389143
+2.532591650260483
+3.451852042322593
+4.18322211802512
+4.113765550959549
+4.109495134451669
+4.121129178702734
+4.154824223110502
+4.255173391471447
+4.356991511156253
+4.416380983659691
+4.427387113724792
+4.427387113724792
+4.405171091330308
+4.32489895623564
+4.209232919086878
+4.109355017580312
+4.0990733560973345
+4.0990733560973345
+4.0990733560973345
+4.1555211100661475
+4.3488481899731894
+4.454624403616235
+4.406095101803691
+4.206813094023217
+4.10650491814166
+4.206813094023217
+4.406095101803691
+4.469240513255388
+4.486334337166265
+4.490181602580803
+4.490181602580803
+4.502694043727065
+4.490181602580803
+4.333419371494481
+4.123000235015652
+4.181718697742469
+4.475186563700894
+4.525148982297988
+4.489566010408934
+4.222982331221807
+4.384942201292258
+4.525148982297988
+4.5437863107195415
+4.550140350585692
+4.550140350585692
+4.550140350585692
+4.563123965563946
+4.563123965563946
+4.563123965563946
+4.570197478760316
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.5853224209629255
+4.274291885177432
+4.15856099578416
+4.2230503108410335
+4.571250454545515
+4.58852136234472
+4.260910257904718
+4.174992929751723
+4.174992929751723
+4.186050824078031
+4.202371696128198
+4.425946698109906
+4.652239127360683
+4.652239127360683
+4.463924095533918
+4.304971199174973
+4.463924095533918
+4.662029905531306
+4.704934473220487
+4.676655864461275
+4.520325741539703
+4.323181495798052
+4.212804137539644
+4.199235542692652
+4.199235542692652
+4.227383303293633
+4.266212842735925
+4.372230331526471
+4.501631338735831
+4.622018630619488
+4.701532155459308
+4.753120216906312
+4.753120216906312
+4.726826567571509
+4.656514879187194
+4.571600036028576
+4.4768097253305825
+4.404981570241521
+4.386252292638426
+4.3936790662814325
+4.4549954627911825
+4.452705607583619
+4.337340933525442
+2.6769755810662073
+2.283297061686561
+2.160109727281195
+2.1003810347274205
+2.0
+2.09890148419316
+2.1678160426511073
+2.221728616568317
+2.2503257854353
+2.2625311142457543
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+0.8361751761523477
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8795831949522626
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2625311142457543
+2.224605730129962
+2.182689437410736
+2.1158475591049335
+2.0331706464402766
+2.0601496310576852
+2.135183545553083
+2.1910332398368633
+2.437345322120844
+3.3627313896351043
+4.2396401002233866
+4.122553981020209
+4.109495134451669
+4.121129178702734
+4.144376911506039
+4.223228363093585
+4.342804609758586
+4.405288240837459
+4.427387113724792
+4.427387113724792
+4.405171091330308
+4.32489895623564
+4.192810323252793
+4.109355017580312
+4.0990733560973345
+4.097112727520313
+4.113407259976061
+4.286106158638396
+4.442593054589512
+4.431848211731319
+4.244565779881304
+4.10650491814166
+4.166061373551728
+4.377925230742242
+4.469240513255388
+4.472314528156556
+4.490181602580803
+4.490181602580803
+4.490181602580803
+4.490181602580803
+4.296670760432378
+4.103232625359826
+4.278622723585689
+4.512159965648277
+4.525148982297988
+4.4182170674180945
+4.242051477075593
+4.503640969757196
+4.525148982297988
+4.530776542855393
+4.5437863107195415
+4.550140350585692
+4.550140350585692
+4.550140350585692
+4.563123965563946
+4.563123965563946
+4.563123965563946
+4.570197478760316
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.497832329201184
+4.163516600021028
+4.1475510772649535
+4.456666181535883
+4.6226847512678555
+4.368581807425848
+4.15856099578416
+4.174992929751723
+4.15856099578416
+4.201549923241762
+4.480408150189437
+4.645199135042
+4.588824563121978
+4.353245755530516
+4.353245755530516
+4.5799545547395955
+4.69281418415078
+4.679949111556313
+4.552289840148148
+4.315633110904785
+4.212804137539644
+4.199235542692652
+4.212804137539644
+4.287058569715656
+4.410395058116138
+4.559466509233962
+4.654819004174287
+4.7039874056290465
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.728275832763921
+4.728397247583595
+4.7124748910849155
+4.690283055605783
+4.690283055605783
+4.706471321315166
+4.723400793471849
+4.76455519928248
+4.681845242735892
+4.411514045438016
+2.674267853333106
+2.2392507225367644
+2.155483910477752
+2.1003810347274205
+2.0
+2.1038052061323396
+2.17374086367226
+2.224605730129962
+2.2503257854353
+2.2743070608481237
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.317246029935818
+0.7785209922500709
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.7128431035659979
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2625311142457543
+2.229462216113617
+2.1937629269159817
+2.1367106068549404
+2.0512707473844163
+2.0436562698390768
+2.1312372430148407
+2.170541996586407
+2.3420968789349415
+3.279704955838343
+3.6647777611972554
+4.1645865807695905
+4.113765550959549
+4.113175748504369
+4.122553981020209
+4.208892782922307
+4.327726831947551
+4.405288240837459
+4.427387113724792
+4.427387113724792
+4.405171091330308
+4.3117422270487165
+4.170693269388646
+4.082166585709781
+4.082166585709781
+4.082166585709781
+4.231867605852527
+4.418012974920095
+4.442593054589512
+4.284882423026124
+4.10650491814166
+4.13051993625085
+4.347517694966176
+4.469240513255388
+4.469240513255388
+4.472314528156556
+4.490181602580803
+4.490181602580803
+4.4676065372348495
+4.242436409101377
+4.1235014442649724
+4.381564064933234
+4.519855048515984
+4.512159965648277
+4.315731378555368
+4.384942201292258
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.530776542855393
+4.550140350585692
+4.550140350585692
+4.550140350585692
+4.550140350585692
+4.563123965563946
+4.563123965563946
+4.563123965563946
+4.583108031777054
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.37650047067259
+4.113710939482175
+4.286633084295397
+4.577601706931892
+4.505917988700361
+4.19219040425837
+4.15856099578416
+4.1431582649902525
+4.241889569491942
+4.529291803534183
+4.6356545149438695
+4.512109974050057
+4.299288615843937
+4.449325548723675
+4.65085686650051
+4.69281418415078
+4.5968888406633095
+4.372230331526471
+4.227383303293633
+4.212804137539644
+4.269511373603581
+4.40691351739986
+4.570132142342086
+4.662029905531306
+4.7039874056290465
+4.7039874056290465
+4.7039874056290465
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.728275832763921
+4.728275832763921
+4.75182931192929
+4.764851533139823
+4.778088299149454
+4.664294722906742
+3.563422141832019
+2.6121534849123687
+2.212819215323622
+2.1511077696207925
+2.0860965787440415
+2.015795365275975
+2.09890148419316
+2.17374086367226
+2.224605730129962
+2.255849667444121
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.4046939094161646
+0.40378354047257137
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.17305757537147226
+1.4612614002956033
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2625311142457543
+2.237685607857683
+2.202927022229015
+2.148386749763698
+2.0691651155283464
+2.02662213211285
+2.1140001744364625
+2.160109727281195
+2.2476747825223162
+2.6608724675715303
+3.5197459726697047
+4.2253855461188
+4.147219722065306
+4.11464846424741
+4.122553981020209
+4.210936623961439
+4.342804609758586
+4.416380983659691
+4.427387113724792
+4.427387113724792
+4.393549265162711
+4.2770241344525415
+4.132106904228244
+4.082166585709781
+4.082166585709781
+4.17835618067723
+4.391267854158244
+4.4525225151227055
+4.332945081527638
+4.113407259976061
+4.10650491814166
+4.316206246689385
+4.4569885833271
+4.469240513255388
+4.469240513255388
+4.486334337166265
+4.490181602580803
+4.4538393189583685
+4.188330631831401
+4.163998999246982
+4.4538393189583685
+4.514890985676352
+4.473282043330933
+4.368049152440184
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.5437863107195415
+4.550140350585692
+4.550140350585692
+4.550140350585692
+4.563123965563946
+4.563123965563946
+4.563123965563946
+4.563123965563946
+4.583108031777054
+4.563841360958199
+4.247524644265324
+4.15491515763151
+4.508358881478538
+4.563841360958199
+4.247524644265324
+4.129245250799048
+4.1431582649902525
+4.260910257904718
+4.539257872104618
+4.6226847512678555
+4.512109974050057
+4.3517175586913135
+4.539257872104618
+4.6356545149438695
+4.645199135042
+4.488865705516085
+4.248918520191973
+4.232147173910067
+4.353245755530516
+4.534232616473268
+4.646852001521213
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.7039874056290465
+4.7039874056290465
+4.7039874056290465
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.716549695711502
+4.7039874056290465
+4.6870297623887245
+4.649129891979821
+4.438823716960622
+3.474742796347404
+2.5304056759265983
+2.192769312606066
+2.1432617429014895
+2.0601496310576852
+2.0331706464402766
+2.1158475591049335
+2.182689437410736
+2.224605730129962
+2.255849667444121
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+1.4574341235805166
+0.17305757537147226
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4133322168253235
+2.3983024946691054
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2625311142457543
+2.2478906171684057
+2.2110819526399554
+2.1584557623803704
+2.0866931654730516
+2.0111326748592737
+2.0960825945482244
+2.155483910477752
+2.2082431896030905
+2.5435740731531253
+3.3689649963112624
+4.285393434112448
+4.190487592151753
+4.122553981020209
+4.113175748504369
+4.221266037517113
+4.3549764147420245
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.367190816108014
+4.227753174233639
+4.094693968165638
+4.069462169076174
+4.122164405528604
+4.350012581677216
+4.4525225151227055
+4.350012581677216
+4.137792226867703
+4.088453787285082
+4.298762055403042
+4.4569885833271
+4.469240513255388
+4.469240513255388
+4.472314528156556
+4.486334337166265
+4.410905534283973
+4.126667610015392
+4.279174511318519
+4.490181602580803
+4.502694043727065
+4.4811159587853355
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.5437863107195415
+4.550140350585692
+4.550140350585692
+4.550140350585692
+4.563123965563946
+4.563123965563946
+4.563123965563946
+4.502706439240199
+4.15491515763151
+4.38052243010171
+4.590893017970352
+4.4313653771533
+4.129245250799048
+4.129245250799048
+4.267104421859435
+4.541557094775911
+4.613831733428877
+4.586658531412266
+4.47698040719883
+4.603447413405319
+4.6356545149438695
+4.609159296939702
+4.406415144859124
+4.274291885177432
+4.430539033190921
+4.588824563121978
+4.666403422086015
+4.679949111556313
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.69281418415078
+4.676655864461275
+4.638630209477494
+4.587726262875978
+4.520325741539703
+4.441583484616433
+4.341633051398518
+4.234828115179514
+3.3943692655669393
+2.426725364121725
+2.1865790505141085
+2.135183545553083
+2.0601496310576852
+2.045597015308826
+2.1367106068549404
+2.1971107715186284
+2.237685607857683
+2.2625311142457543
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+0.7163871050283115
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8256255235064831
+2.326976269832184
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2690991488719328
+2.2503257854353
+2.224605730129962
+2.17374086367226
+2.1038052061323396
+2.02739461766745
+2.0711438279723438
+2.1432617429014895
+2.1763775028439323
+2.403163833643906
+3.2700437837185357
+4.327356979995596
+4.240845765526305
+4.144376911506039
+4.134202790326104
+4.240845765526305
+4.380386429263759
+4.427387113724792
+4.427387113724792
+4.416380983659691
+4.32489895623564
+4.14271795783301
+4.060376351998211
+4.107624531009424
+4.305505191501973
+4.44092500022344
+4.378043959316646
+4.1613904777964095
+4.082166585709781
+4.298762055403042
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.472314528156556
+4.365645075900559
+4.126667610015392
+4.395780918368115
+4.502694043727065
+4.502694043727065
+4.502694043727065
+4.507264102733968
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.530776542855393
+4.5437863107195415
+4.550140350585692
+4.550140350585692
+4.550140350585692
+4.563123965563946
+4.405511387167781
+4.204366011250862
+4.5350396637789485
+4.549643730922412
+4.213656267826046
+4.113710939482175
+4.282757389600612
+4.55656589041628
+4.590893017970352
+4.590893017970352
+4.599002836703154
+4.6226847512678555
+4.6226847512678555
+4.6226847512678555
+4.5227409419139475
+4.512109974050057
+4.609159296939702
+4.6356545149438695
+4.648026828159334
+4.657957256979294
+4.668188175579596
+4.679949111556313
+4.666403422086015
+4.645399894198835
+4.5968888406633095
+4.531977395390569
+4.444957838142898
+4.353245755530516
+4.283930830651066
+4.242808832535492
+4.228811106557455
+4.157308012802116
+4.295952195528659
+3.3338712248557067
+2.3211831636053715
+2.1673562403753768
+2.1186411559003946
+2.0436562698390768
+2.063727005490863
+2.1367106068549404
+2.1971107715186284
+2.237685607857683
+2.2625311142457543
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+0.9045078427753417
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8121773597308399
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2743070608481237
+2.2503257854353
+2.224605730129962
+2.1885638406774586
+2.1323446946695777
+2.0512707473844163
+2.0436562698390768
+2.122787738844958
+2.1673562403753768
+2.301877100132785
+2.7408362706092446
+3.5730728061118344
+4.270567858673243
+4.157675480838209
+4.1656882766508945
+4.285908874252486
+4.405288240837459
+4.427387113724792
+4.427387113724792
+4.392079691227801
+4.247576635145128
+4.0987968379282975
+4.084101723741918
+4.256447689150627
+4.429161646579058
+4.391267854158244
+4.18527631598984
+4.082166585709781
+4.317872846720725
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.282391253599503
+4.206280547491993
+4.4676065372348495
+4.490181602580803
+4.502694043727065
+4.502694043727065
+4.502694043727065
+4.519855048515984
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.530776542855393
+4.550140350585692
+4.550140350585692
+4.550140350585692
+4.298852288588964
+4.387892251749726
+4.563123965563946
+4.387892251749726
+4.1183732023423625
+4.342653094159319
+4.571165324280725
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.613831733428877
+4.6226847512678555
+4.6226847512678555
+4.6356545149438695
+4.6356545149438695
+4.6356545149438695
+4.609159296939702
+4.571250454545515
+4.500777606857457
+4.392622212842695
+4.305664264019257
+4.248918520191973
+4.2101709130931395
+4.1840024052794655
+4.172922335496928
+4.200639673579342
+4.200639673579342
+4.141719569647896
+3.5468119914791725
+2.5827129410220424
+2.2487592108855945
+2.155483910477752
+2.1003810347274205
+2.0111326748592737
+2.0866931654730516
+2.152489662966116
+2.2110819526399554
+2.237685607857683
+2.2625311142457543
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.353197272802601
+0.6723626437589605
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.06854719057135394
+1.3237599323266105
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2625311142457543
+2.237685607857683
+2.1971107715186284
+2.148386749763698
+2.0750586643658098
+2.019145534516432
+2.105457950100802
+2.160109727281195
+2.2350387770506743
+2.633162161009022
+3.4669981438100352
+4.283524110580276
+4.190487592151753
+4.208892782922307
+4.342804609758586
+4.427387113724792
+4.427387113724792
+4.416380983659691
+4.327726831947551
+4.15362856723562
+4.076735249363886
+4.221636875083904
+4.418012974920095
+4.405516093367723
+4.206980666498831
+4.094693968165638
+4.3488481899731894
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.446713403848863
+4.210991133064981
+4.331245648109325
+4.490181602580803
+4.490181602580803
+4.490181602580803
+4.502694043727065
+4.502694043727065
+4.502694043727065
+4.519855048515984
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.5437863107195415
+4.536738078402639
+4.438213139075817
+4.536738078402639
+4.528831176637506
+4.241075969322648
+4.387892251749726
+4.570197478760316
+4.583108031777054
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.603682897163525
+4.613831733428877
+4.59511905321376
+4.512109974050057
+4.387552875155975
+4.2549350594948505
+4.165009881185526
+4.1431582649902525
+4.127470160894371
+4.141950580579023
+4.156753372551671
+4.172922335496928
+4.2160995736710225
+4.21602102689975
+4.240735018037144
+3.3857054816622654
+2.4425861609117265
+2.197496488944807
+2.1432617429014895
+2.0711438279723438
+2.0194533506514256
+2.09890148419316
+2.1678160426511073
+2.2168333461740244
+2.2503257854353
+2.2625311142457543
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.432451996876125
+0.3172647300558147
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.22140021244840558
+1.5281001994867136
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2625311142457543
+2.2503257854353
+2.2110819526399554
+2.163970843013399
+2.09890148419316
+2.0112792435250872
+2.0860965787440415
+2.1511077696207925
+2.1927208376298393
+2.5255381483758743
+3.3582972295480533
+4.292846626911027
+4.213979226326154
+4.273346012383912
+4.394136322099213
+4.427387113724792
+4.427387113724792
+4.392079691227801
+4.221266037517113
+4.09250926319951
+4.192810323252793
+4.405516093367723
+4.418012974920095
+4.221636875083904
+4.119144903907343
+4.377713549555899
+4.4664508108639795
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.406095101803691
+4.227703995652505
+4.446713403848863
+4.490181602580803
+4.490181602580803
+4.490181602580803
+4.502694043727065
+4.502694043727065
+4.502694043727065
+4.507264102733968
+4.519855048515984
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.5437863107195415
+4.550140350585692
+4.466517186829626
+4.4547427227222975
+4.563123965563946
+4.563123965563946
+4.563123965563946
+4.570197478760316
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.590893017970352
+4.524127578822819
+4.37650047067259
+4.213656267826046
+4.1431582649902525
+4.1169613292012635
+4.107173483038939
+4.1169613292012635
+4.1151921680667005
+4.1415828455394195
+4.19219040425837
+4.308646412990123
+4.338128996657249
+4.353505525188759
+2.704765968797729
+2.3065407450626294
+2.1673562403753768
+2.122787738844958
+2.0436562698390768
+2.045597015308826
+2.1264928868542414
+2.182689437410736
+2.224605730129962
+2.2503257854353
+2.2743070608481237
+2.2796572735563765
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+1.4017247053402326
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5382175776289602
+2.384596050189881
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2743070608481237
+2.2503257854353
+2.224605730129962
+2.182689437410736
+2.1204808889673323
+2.0331706464402766
+2.066570901959439
+2.139334340219923
+2.1763775028439323
+2.424058950280588
+3.2807022858567167
+4.305299409549234
+4.275546918294798
+4.356613498206495
+4.416380983659691
+4.427387113724792
+4.416380983659691
+4.298727598411171
+4.113175748504369
+4.166537627720674
+4.391323950112836
+4.429161646579058
+4.241940840533436
+4.155611178860544
+4.403952440507338
+4.4525225151227055
+4.4664508108639795
+4.469240513255388
+4.469240513255388
+4.363005348045478
+4.377662548191024
+4.469240513255388
+4.472314528156556
+4.490181602580803
+4.490181602580803
+4.490181602580803
+4.502694043727065
+4.502694043727065
+4.502694043727065
+4.507264102733968
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.530776542855393
+4.536738078402639
+4.550140350585692
+4.550140350585692
+4.550140350585692
+4.563123965563946
+4.563123965563946
+4.563123965563946
+4.583108031777054
+4.590893017970352
+4.508358881478538
+4.323535752524507
+4.15491515763151
+4.099911233827896
+4.099911233827896
+4.088606037180334
+4.1053137652653495
+4.127470160894371
+4.201549923241762
+4.354374661974843
+4.5030960939417675
+4.451614189945248
+3.469949565112751
+2.5867012767877045
+2.221558662260198
+2.155483910477752
+2.1003810347274205
+2.0157495197110955
+2.0750586643658098
+2.148386749763698
+2.1971107715186284
+2.237685607857683
+2.2625311142457543
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+0.7569148146041229
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.9379779678995588
+2.2965955499850623
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.280529132403043
+2.2743070608481237
+2.2625311142457543
+2.237685607857683
+2.1937629269159817
+2.1323446946695777
+2.05698502564009
+2.0436562698390768
+2.122787738844958
+2.170541996586407
+2.3508959098624618
+3.260182981301824
+4.322349160759665
+4.346962517011327
+4.417897362922753
+4.427387113724792
+4.427387113724792
+4.353117274877057
+4.144376911506039
+4.154824223110502
+4.3786204442699175
+4.429161646579058
+4.256447689150627
+4.206980666498831
+4.430380826279388
+4.4525225151227055
+4.454624403616235
+4.4664508108639795
+4.469240513255388
+4.4569885833271
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.472314528156556
+4.490181602580803
+4.490181602580803
+4.490181602580803
+4.502694043727065
+4.502694043727065
+4.502694043727065
+4.519855048515984
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.530776542855393
+4.550140350585692
+4.550140350585692
+4.550140350585692
+4.563123965563946
+4.563123965563946
+4.512329152144901
+4.317592641512847
+4.128740395675791
+4.099911233827896
+4.099911233827896
+4.088606037180334
+4.099911233827896
+4.15491515763151
+4.321414521587279
+4.488817061485224
+4.59511905321376
+4.447806976940592
+3.337450436572998
+2.442590054821959
+2.1828331778914043
+2.1432617429014895
+2.0758977822449705
+2.015795365275975
+2.1038052061323396
+2.1678160426511073
+2.2110819526399554
+2.2503257854353
+2.2625311142457543
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2965955499850623
+0.9630732565314721
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7647373658980348
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2625311142457543
+2.237685607857683
+2.2079729546579676
+2.152489662966116
+2.0750586643658098
+2.032562994261107
+2.1186411559003946
+2.1621169737505186
+2.306811880582651
+3.2449725942218595
+3.6216779421851166
+4.416217764170109
+4.428862183491362
+4.427387113724792
+4.393549265162711
+4.208892782922307
+4.144376911506039
+4.365873551370795
+4.428170372024604
+4.291189724650094
+4.289528505696501
+4.44092500022344
+4.4525225151227055
+4.4525225151227055
+4.454624403616235
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.486334337166265
+4.490181602580803
+4.490181602580803
+4.490181602580803
+4.502694043727065
+4.502694043727065
+4.507264102733968
+4.519855048515984
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.5437863107195415
+4.550140350585692
+4.536738078402639
+4.404476728520448
+4.145078211515794
+4.057266954196868
+4.057266954196868
+4.072289591672341
+4.101269072697191
+4.227912298396953
+4.415569757481522
+4.565650682473864
+4.590893017970352
+4.408635309066533
+3.275551551235999
+2.340876918824316
+2.1673562403753768
+2.122787738844958
+2.0436562698390768
+2.045597015308826
+2.1264928868542414
+2.182689437410736
+2.224605730129962
+2.2503257854353
+2.2743070608481237
+2.2796572735563765
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.3772600626078293
+0.547964844339119
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+1.3905844909148
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2625311142457543
+2.2503257854353
+2.2110819526399554
+2.152489662966116
+2.081518910697424
+2.0157495197110955
+2.105457950100802
+2.160109727281195
+2.2777873993509488
+2.7666474563244394
+3.610811338896543
+4.416217764170109
+4.428862183491362
+4.416380983659691
+4.2664459887944055
+4.15362856723562
+4.3549764147420245
+4.427387113724792
+4.338149361251756
+4.36476001222331
+4.44092500022344
+4.4525225151227055
+4.4525225151227055
+4.4525225151227055
+4.454624403616235
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.472314528156556
+4.490181602580803
+4.490181602580803
+4.490181602580803
+4.502694043727065
+4.502694043727065
+4.502694043727065
+4.507264102733968
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.489566010408934
+4.280004643579906
+4.057266954196868
+4.044951545574025
+4.044951545574025
+4.073248530971775
+4.201961535726061
+4.421450634493992
+4.565584671587924
+4.583108031777054
+4.405809921245526
+2.7324485083436683
+2.2768529630361805
+2.160109727281195
+2.105457950100802
+2.019145534516432
+2.0750586643658098
+2.152489662966116
+2.1971107715186284
+2.237685607857683
+2.2625311142457543
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+1.5354439345570259
+0.24055939460771647
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3172647300558147
+2.4372360353739517
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2743070608481237
+2.2503257854353
+2.2110819526399554
+2.1678160426511073
+2.1038052061323396
+2.015795365275975
+2.0860965787440415
+2.1570620246091714
+2.2777873993509488
+2.7666474563244394
+3.6254519576508906
+4.428211004156933
+4.428862183491362
+4.3117422270487165
+4.1753404548542585
+4.3549764147420245
+4.427387113724792
+4.405239266363791
+4.429161646579058
+4.44092500022344
+4.44092500022344
+4.4525225151227055
+4.4525225151227055
+4.4525225151227055
+4.4664508108639795
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.472314528156556
+4.490181602580803
+4.490181602580803
+4.490181602580803
+4.502694043727065
+4.502694043727065
+4.502694043727065
+4.507264102733968
+4.525148982297988
+4.525148982297988
+4.525148982297988
+4.443596648081142
+4.162310183365068
+4.020098689803449
+4.020098689803449
+4.044951545574025
+4.145078211515794
+4.387892251749726
+4.544787027092619
+4.563123965563946
+4.397484734887035
+2.719898488397276
+2.2623212663256167
+2.163555712820396
+2.1003810347274205
+2.0
+2.092668508103671
+2.1678160426511073
+2.2110819526399554
+2.2503257854353
+2.2625311142457543
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+1.334894959804494
+0.06854719057135394
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6148520064161902
+2.3452889562727264
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2764066145123847
+2.2743070608481237
+2.2503257854353
+2.224605730129962
+2.182689437410736
+2.1098134426472237
+2.015795365275975
+2.0860965787440415
+2.1570620246091714
+2.2948898705815313
+3.2382221208941813
+4.374914026316517
+4.430458479623737
+4.342804609758586
+4.284364652204422
+4.416380983659691
+4.427387113724792
+4.427387113724792
+4.428170372024604
+4.44092500022344
+4.44092500022344
+4.44092500022344
+4.4525225151227055
+4.4525225151227055
+4.454624403616235
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.486334337166265
+4.490181602580803
+4.490181602580803
+4.490181602580803
+4.502694043727065
+4.502694043727065
+4.502694043727065
+4.519855048515984
+4.415390784022273
+4.103232625359826
+4.020098689803449
+4.020098689803449
+4.040649006191388
+4.278622723585689
+4.494288594971252
+4.564735477435103
+4.405058310646361
+3.2627492086774588
+2.2732041608987807
+2.1665875672999495
+2.1003810347274205
+2.0
+2.09890148419316
+2.17374086367226
+2.224605730129962
+2.2503257854353
+2.2743070608481237
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+0.8013943068221954
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.9237561989163847
+2.2965955499850623
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2625311142457543
+2.224605730129962
+2.182689437410736
+2.1098134426472237
+2.015795365275975
+2.0960825945482244
+2.1594910626532013
+2.424058950280588
+3.4033593703152687
+4.403607540711773
+4.417897362922753
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.428170372024604
+4.44092500022344
+4.44092500022344
+4.44092500022344
+4.4525225151227055
+4.4525225151227055
+4.454624403616235
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.472314528156556
+4.486334337166265
+4.490181602580803
+4.490181602580803
+4.490181602580803
+4.502694043727065
+4.395780918368115
+4.0712481374985465
+4.01960399960008
+4.020098689803449
+4.142890501858993
+4.415390784022273
+4.532451043151799
+4.434404132986627
+3.2997923574897783
+2.3214444725064065
+2.160109727281195
+2.1003810347274205
+2.0
+2.09890148419316
+2.1678160426511073
+2.224605730129962
+2.255849667444121
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.312187166661021
+0.8256255235064831
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7238182390859009
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2625311142457543
+2.224605730129962
+2.182689437410736
+2.1038052061323396
+2.0157495197110955
+2.1312372430148407
+2.193652390301639
+2.7386357871628335
+4.377148473974314
+4.428211004156933
+4.426792664882731
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.428170372024604
+4.439493703582002
+4.44092500022344
+4.44092500022344
+4.4525225151227055
+4.4525225151227055
+4.4525225151227055
+4.454624403616235
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.472314528156556
+4.490181602580803
+4.490181602580803
+4.395780918368115
+4.073910026193216
+4.041787974963054
+4.028346058775132
+4.279174511318519
+4.491440034664813
+4.518215987060842
+3.475078474720889
+2.464962024504153
+2.1594258087242073
+2.105457950100802
+2.0
+2.09890148419316
+2.1678160426511073
+2.224605730129962
+2.255849667444121
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.395732822205279
+0.44213125973792344
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.17305757537147226
+1.4351488673587052
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2625311142457543
+2.221728616568317
+2.152489662966116
+2.05698502564009
+2.0758977822449705
+2.1570620246091714
+2.474079002171915
+3.550701921872128
+4.428211004156933
+4.416217764170109
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.428170372024604
+4.44092500022344
+4.44092500022344
+4.44092500022344
+4.4525225151227055
+4.4525225151227055
+4.454624403616235
+4.4664508108639795
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.421815150590111
+4.0990733560973345
+4.057108257194596
+4.0712481374985465
+4.363548074929932
+4.508911515906129
+4.410566038220873
+2.7304609129191033
+2.197713610869064
+2.122787738844958
+2.02662213211285
+2.092668508103671
+2.17374086367226
+2.224605730129962
+2.255849667444121
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+1.4797210190180161
+0.17305757537147226
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.38453231097115825
+2.4107914727036284
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.280529132403043
+2.2743070608481237
+2.2503257854353
+2.1971107715186284
+2.1098134426472237
+2.0111326748592737
+2.1312372430148407
+2.3009344856470513
+3.4092698519760596
+4.416211745185024
+4.416211745185024
+4.416217764170109
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.428170372024604
+4.44092500022344
+4.44092500022344
+4.44092500022344
+4.4525225151227055
+4.4525225151227055
+4.454624403616235
+4.469240513255388
+4.469240513255388
+4.469240513255388
+4.4569885833271
+4.184104289765669
+4.07241892641154
+4.097112727520313
+4.410905534283973
+4.508911515906129
+3.602433982163511
+2.481687647876782
+2.1536364274697855
+2.066570901959439
+2.063727005490863
+2.152489662966116
+2.224605730129962
+2.2503257854353
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+0.7028916044659059
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7683375209644598
+2.3363209725409417
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2625311142457543
+2.224605730129962
+2.152489662966116
+2.045597015308826
+2.1003810347274205
+2.2242764561854735
+3.3279088601467954
+4.404716270919072
+4.416211745185024
+4.416211745185024
+4.426792664882731
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.439493703582002
+4.44092500022344
+4.44092500022344
+4.44092500022344
+4.4525225151227055
+4.4525225151227055
+4.454624403616235
+4.469240513255388
+4.316214637717017
+4.062586990861675
+4.1088251299545835
+4.405105971950175
+4.487951122439035
+3.4996046386563355
+2.326661896488758
+2.1312372430148407
+2.0111326748592737
+2.1204808889673323
+2.1971107715186284
+2.2503257854353
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+0.8753367724102397
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.841423246189211
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.280529132403043
+2.2743070608481237
+2.237685607857683
+2.182689437410736
+2.081518910697424
+2.077355219671662
+2.2023661849891183
+3.332339341975471
+4.405992259746327
+4.416211745185024
+4.416211745185024
+4.416217764170109
+4.426792664882731
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.428170372024604
+4.439493703582002
+4.44092500022344
+4.44092500022344
+4.4525225151227055
+4.4525225151227055
+4.418012974920095
+4.085785441124662
+4.096141743038126
+4.406095101803691
+4.486334337166265
+3.462924881186887
+2.2667210007255236
+2.109810149006344
+2.038449217047763
+2.163970843013399
+2.224605730129962
+2.2625311142457543
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.3410810950415235
+0.7108936078332695
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.6982561460305812
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2503257854353
+2.1971107715186284
+2.09890148419316
+2.0625720845185223
+2.223400939437198
+3.4153326189907474
+4.416217764170109
+4.404716270919072
+4.416211745185024
+4.416211745185024
+4.416217764170109
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.428170372024604
+4.44092500022344
+4.44092500022344
+4.4525225151227055
+4.275317971489297
+4.090813886481587
+4.363885490652064
+4.482790556910679
+3.526665827705376
+2.2838577347402804
+2.105457950100802
+2.05698502564009
+2.179096074810225
+2.237685607857683
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.4253402630005265
+0.3557324805227413
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.19262923125003173
+1.505818593254138
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2625311142457543
+2.2110819526399554
+2.1038052061323396
+2.077355219671662
+2.3136782418210164
+3.572769228786013
+4.405568769612216
+4.404716270919072
+4.404716270919072
+4.416211745185024
+4.416211745185024
+4.416217764170109
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.428170372024604
+4.417257564352126
+4.143145871693728
+4.275317971489297
+4.467975145367171
+4.37888240499102
+2.3841071414305532
+2.105457950100802
+2.05698502564009
+2.182689437410736
+2.2503257854353
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+1.4240069420453962
+0.11537255792817191
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4997231449871
+2.395732822205279
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2625311142457543
+2.1971107715186284
+2.0866931654730516
+2.0960825945482244
+2.5253339155275762
+4.365929836233456
+4.405992259746327
+4.404716270919072
+4.404716270919072
+4.416211745185024
+4.416211745185024
+4.416211745185024
+4.426792664882731
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.353117274877057
+4.221266037517113
+4.429161646579058
+4.410753479930216
+2.61433650252718
+2.105135466890179
+2.05698502564009
+2.182689437410736
+2.2503257854353
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+0.7425722277764768
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8825278501371363
+2.3071218036676138
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2503257854353
+2.182689437410736
+2.0512707473844163
+2.115923216982885
+3.2172127875054515
+4.395466391668603
+4.394440664701463
+4.404716270919072
+4.404716270919072
+4.404716270919072
+4.416211745185024
+4.416211745185024
+4.416217764170109
+4.427387113724792
+4.427387113724792
+4.427387113724792
+4.3549764147420245
+4.393549265162711
+4.440985698726188
+3.2651300009437985
+2.1173813288353065
+2.0331706464402766
+2.182689437410736
+2.2503257854353
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2965955499850623
+0.9650712301005928
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7818913621104633
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2478906171684057
+2.163970843013399
+2.024242880540455
+2.224950265230117
+3.532962738921775
+4.405992259746327
+4.395466391668603
+4.394440664701463
+4.404716270919072
+4.404716270919072
+4.404716270919072
+4.416211745185024
+4.416211745185024
+4.416217764170109
+4.427387113724792
+4.427387113724792
+4.440985698726188
+3.6030567214020692
+2.2879955242179557
+2.0263674134864393
+2.152489662966116
+2.237685607857683
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.366575084333111
+0.5860800426151513
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.06854719057135394
+1.368306154887538
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.224605730129962
+2.1204808889673323
+2.0711438279723438
+2.668617332320399
+4.385295485648465
+4.395466391668603
+4.395466391668603
+4.395466391668603
+4.394440664701463
+4.404716270919072
+4.404716270919072
+4.404716270919072
+4.416211745185024
+4.416211745185024
+4.426792664882731
+4.4149076540614445
+2.7461677393350667
+2.0758977822449705
+2.1204808889673323
+2.224605730129962
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.4449570708859434
+0.2693452599052222
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2884240647434684
+1.5465893039290308
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.280529132403043
+2.2625311142457543
+2.1971107715186284
+2.0581063231035372
+2.1682995337015143
+3.47930344235551
+4.395736408546402
+4.395466391668603
+4.395466391668603
+4.395466391668603
+4.395466391668603
+4.394440664701463
+4.404716270919072
+4.404716270919072
+4.416211745185024
+4.428211004156933
+3.5561228267406797
+2.2092482937679585
+2.054713117792171
+2.1971107715186284
+2.2625311142457543
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+1.3571681680736565
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5767970611512538
+2.353889367864523
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.237685607857683
+2.152489662966116
+2.0353944423380357
+2.7020283218669165
+4.386125747786783
+4.395466391668603
+4.395466391668603
+4.395466391668603
+4.395466391668603
+4.395466391668603
+4.394440664701463
+4.404716270919072
+4.404716270919072
+4.405992259746327
+2.7731637760483516
+2.0436562698390768
+2.148386749763698
+2.237685607857683
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+0.7763091941917066
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.9650712301005928
+2.2965955499850623
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2110819526399554
+2.071215874545916
+2.250843516412312
+3.6020738028972716
+4.386876885101057
+4.387147043291084
+4.395466391668603
+4.395466391668603
+4.395466391668603
+4.395466391668603
+4.395466391668603
+4.405992259746327
+4.3666498419454385
+2.298157327804374
+2.0581063231035372
+2.2110819526399554
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.3071218036676138
+0.8825278501371363
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7359825258205799
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2503257854353
+2.152489662966116
+2.046265922153117
+3.233043394124209
+4.388327680595662
+4.387147043291084
+4.387147043291084
+4.385615758860399
+4.395466391668603
+4.395466391668603
+4.395466391668603
+4.407428326716359
+3.286739248517872
+2.0558405821318013
+2.1367106068549404
+2.2503257854353
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.384596050189881
+0.48061248750569696
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.14421518948927226
+1.412865539386467
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2625311142457543
+2.1971107715186284
+2.040107581316896
+2.6193909090739607
+4.369061359433033
+4.377747634560744
+4.377747634560744
+4.387147043291084
+4.387147043291084
+4.385615758860399
+4.395466391668603
+4.385295485648465
+2.6466510922053246
+2.0355882720041394
+2.1971107715186284
+2.2625311142457543
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+1.494678486305875
+0.2018996889437794
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3461049112705705
+2.419971038201888
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.2350331725121304
+2.104241453393035
+2.2875914460808904
+4.359769301852005
+4.378973407053157
+4.377747634560744
+4.377747634560744
+4.377747634560744
+4.387147043291084
+4.386876885101057
+4.3680123619119815
+2.302239888941486
+2.104241453393035
+2.224605730129962
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+0.6951452758865653
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7108936078332695
+2.3452889562727264
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2503257854353
+2.152489662966116
+2.1063363848303007
+3.427842944333051
+4.380396058849706
+4.379738179877462
+4.377747634560744
+4.377747634560744
+4.377747634560744
+4.388327680595662
+3.414657216759596
+2.099288576514305
+2.152489662966116
+2.2503257854353
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+0.8495621167939733
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8704047159223176
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2625311142457543
+2.1971107715186284
+2.02739461766745
+3.2154644369067613
+4.380396058849706
+4.370810227989025
+4.379738179877462
+4.379738179877462
+4.377747634560744
+4.388327680595662
+3.181556897644701
+2.02739461766745
+2.1971107715186284
+2.2625311142457543
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.326976269832184
+0.7584815939613518
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.7077485255828968
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2743070608481237
+2.224605730129962
+2.075416544172951
+2.81649189746334
+4.363184059378072
+4.370810227989025
+4.370810227989025
+4.370810227989025
+4.379738179877462
+4.378757756023827
+2.7134039666737695
+2.0783683466684364
+2.224605730129962
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.41660567849633
+0.39419237910002547
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+1.4017247053402326
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.2838303016383614
+2.237685607857683
+2.104241453393035
+2.6666666666666665
+4.356449779411873
+4.364768579242243
+4.361976979184238
+4.370810227989025
+4.370810227989025
+4.367924906446838
+2.4944548376163715
+2.120374153703379
+2.237685607857683
+2.2743070608481237
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+2.285659673582315
+1.3460310727738563
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.9533025720683259
+1.4574341235805166
+1.494678486305875
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.494678486305875
+1.531709664562058
+2.4127829664478506
+2.6018431061745493
+4.347339821427706
+4.356449779411873
+4.364768579242243
+4.364768579242243
+4.370810227989025
+3.5824983806496986
+2.4769229983066006
+2.4218410027163504
+1.5242987993988644
+1.494678486305875
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.4908650045033593
+1.494678486305875
+1.4908650045033593
+0.6951452758865653
+0.4614592400945998
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.17305757537147226
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.230741499582245
+0.3172647300558147
+0.48061248750569696
+0.766750836091826
+4.395454975696837
+4.356449779411873
+4.356449779411873
+4.356449779411873
+4.364768579242243
+3.3901628360753846
+0.8935964092522806
+0.451806660765417
+0.2884240647434684
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.22140021244840558
+0.2018996889437794
+0.06854719057135394
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.2018996889437794
+0.7831407078010928
+3.5526195691168794
+4.3586769260880995
+4.3586769260880995
+4.356449779411873
+4.356449779411873
+3.3626279895276285
+0.6695114446410386
+0.17305757537147226
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.19262923125003173
+0.7805011123938099
+3.518119533002176
+4.354329469367648
+4.3617847327443595
+4.3586769260880995
+4.356449779411873
+3.358640744665906
+0.6054987150990874
+0.17305757537147226
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7778914672149311
+3.476402583504967
+4.34704458715414
+4.354329469367648
+4.354329469367648
+4.362650556348973
+3.356648080755124
+0.519387512494303
+0.17305757537147226
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7804767966616
+3.45435772355868
+4.350730623451057
+4.34704458715414
+4.354329469367648
+4.355194967601397
+3.3620486854417333
+0.4614592400945998
+0.17305757537147226
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7754850383014231
+3.4222154472822863
+4.3433457326275935
+4.3433457326275935
+4.350730623451057
+4.355194967601397
+0.6146514861726686
+0.40378354047257137
+0.14421518948927226
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7688158416003859
+3.413519709439906
+4.3433457326275935
+4.3433457326275935
+4.3433457326275935
+4.350730623451057
+0.626196231396948
+0.3749445413935266
+0.11537255792817191
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7796188342366239
+3.403334004770977
+4.347234887382945
+4.3433457326275935
+4.3433457326275935
+4.3433457326275935
+0.6381535476845652
+0.3172647300558147
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.336579979066304
+0.5670150445251751
+0.5762468110472865
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5670150445251751
+0.5762468110472865
+0.451806660765417
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7796188342366239
+3.4025832397127886
+4.340961913960934
+4.340961913960934
+4.347234887382945
+4.3433457326275935
+0.6471225330532464
+0.30777693354095703
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5285140666175412
+0.8495621167939733
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8268653780234594
+0.5382175776289602
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7796188342366239
+3.401876210713215
+4.334910196094188
+4.340961913960934
+4.340961913960934
+4.340961913960934
+0.6567269107919813
+0.30777693354095703
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.963774036650098
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.5958076617353987
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7551530895194016
+3.3881187366326473
+4.339190019517164
+4.339190019517164
+4.334910196094188
+4.340961913960934
+0.6633429124174155
+0.30777693354095703
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.230741499582245
+0.8674260761271935
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8268653780234594
+0.39419237910002547
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7551530895194016
+3.3872926401828645
+4.339190019517164
+4.339190019517164
+4.339190019517164
+4.339190019517164
+0.6701438322619957
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.40378354047257137
+0.8268653780234594
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8829731209921494
+0.21178330071085139
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7796188342366239
+3.3978458064751607
+4.337967397763282
+4.333620133481415
+4.339190019517164
+4.339190019517164
+0.6749865177929246
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6336557862849546
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.9550413197418148
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7796188342366239
+3.3978458064751607
+4.332854430148258
+4.332854430148258
+4.333620133481415
+4.333620133481415
+0.6826826421251191
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.8613593131169246
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.7683375209644598
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7674288466362427
+3.4006038179412155
+4.332854430148258
+4.332854430148258
+4.332854430148258
+4.332854430148258
+0.6826826421251191
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.1255917803265003
+0.9342501145827482
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.54752294636826
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7749854823300453
+3.415365057716329
+4.33723332223392
+4.332854430148258
+4.332854430148258
+4.332854430148258
+0.6914960532608827
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2693452599052222
+0.8529267406541052
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8394299183804375
+0.336579979066304
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.780827995344489
+3.4192368324649047
+4.332550398821635
+4.332550398821635
+4.33723332223392
+4.332854430148258
+0.6944175558639092
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.48061248750569696
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8995403851261718
+0.16387931614117957
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7794752654872443
+3.4429921340343563
+4.327833805985841
+4.327833805985841
+4.332550398821635
+4.332550398821635
+0.6944175558639092
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6909830056250525
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.9157781068691486
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7821067592393272
+3.4597708579452107
+4.332371753085499
+4.332371753085499
+4.327833805985841
+4.327833805985841
+0.7037607950418554
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.9157781068691486
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.7195984021833732
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7805011123938099
+3.484915360681428
+4.328341712918101
+4.332371753085499
+4.332371753085499
+4.332371753085499
+0.7070144357115327
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.16387931614117957
+0.8995403851261718
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.48061248750569696
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7692280381806267
+3.533052287049287
+3.6675623664868677
+4.328341712918101
+4.328341712918101
+4.332371753085499
+0.7070144357115327
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.336579979066304
+0.8394299183804375
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8495621167939733
+0.2789778070382427
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7637294987872902
+3.5905083598818424
+3.6675623664868677
+3.6675623664868677
+3.6675623664868677
+4.328341712918101
+0.7110276306660097
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5382175776289602
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.9332803587042555
+0.1351645434877229
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.17305757537147226
+0.7564047449203359
+3.63037851640973
+3.668556167091508
+3.668556167091508
+3.6675623664868677
+3.6675623664868677
+0.7210686438917335
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7766676551803027
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8706366067951148
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.22140021244840558
+0.7354188586773578
+3.6311669204486057
+3.6590433640953237
+3.6590433640953237
+3.668556167091508
+3.668556167091508
+0.7256446863795958
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.9600775485324995
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.6336557862849546
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.230741499582245
+0.722595819752951
+3.6604290860517175
+3.659819236392028
+3.6590433640953237
+3.6590433640953237
+3.63037851640973
+0.7334579454226846
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2018996889437794
+0.8829731209921494
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8268653780234594
+0.4229997026740895
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.2595829718925291
+0.7062695533929109
+3.6571830662978924
+3.6604290860517175
+3.659819236392028
+3.659819236392028
+3.6311669204486057
+0.7464068573479388
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.38453231097115825
+0.8268653780234594
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8674260761271935
+0.230741499582245
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.2884240647434684
+0.6891242947821885
+3.6571830662978924
+3.6571830662978924
+3.6604290860517175
+3.6604290860517175
+3.6311669204486057
+0.7519710559558561
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6049589697638957
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.9615285957602238
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.2884240647434684
+0.6754834543527535
+3.6480580099102515
+3.6480580099102515
+3.6571830662978924
+3.6571830662978924
+3.628330373201324
+0.7519710559558561
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.833333333333333
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.7970093430472787
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.2884240647434684
+0.6635370822318256
+3.648332150527765
+3.6480580099102515
+3.6480580099102515
+3.6476277987965524
+3.628330373201324
+0.7368127772852731
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.11537255792817191
+0.9517446316409384
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.5762468110472865
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.30777693354095703
+0.6530041104745514
+3.6390096082606376
+3.648332150527765
+3.648332150527765
+3.6480580099102515
+3.6480580099102515
+0.7285441512181112
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2501845927102355
+0.8529267406541052
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8394299183804375
+0.3557324805227413
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.30777693354095703
+0.6436597632191985
+3.6390096082606376
+3.6390096082606376
+3.6390096082606376
+3.648332150527765
+3.648332150527765
+0.7201444745328986
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.451806660765417
+0.8268653780234594
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8829731209921494
+0.17305757537147226
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.30777693354095703
+0.6362446198120244
+3.638996173543899
+3.6390096082606376
+3.6390096082606376
+3.6390096082606376
+3.6390096082606376
+0.7062695533929109
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6623325614087925
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.9406793596180201
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.3172647300558147
+0.6273903943262429
+3.638996173543899
+3.638996173543899
+3.638996173543899
+3.6390096082606376
+3.6390096082606376
+0.7122437955064962
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.8889745195479124
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.7396285443968873
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.3172647300558147
+0.6204364114904433
+3.6352246502145382
+3.6352246502145382
+3.638996173543899
+3.638996173543899
+3.6390096082606376
+0.7201444745328986
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.14421518948927226
+0.916900139649794
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.5094164061306206
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.3172647300558147
+2.62636182814419
+3.6293327884224067
+3.6256567937371553
+3.6352246502145382
+3.6352246502145382
+3.638996173543899
+0.7237507092058575
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.30777693354095703
+0.8495621167939733
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8394299183804375
+0.29813748831171427
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.11537255792817191
+0.36554510004184326
+2.635943082963963
+3.6256567937371553
+3.6256567937371553
+3.6256567937371553
+3.6256567937371553
+3.6076203497319868
+0.7396915102064616
+0.2884240647434684
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5094164061306206
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.916900139649794
+0.14421518948927226
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.11537255792817191
+0.42320293884874616
+2.643908914151852
+3.6161878900604445
+3.6122361466389306
+3.6256567937371553
+3.6256567937371553
+3.5986166028485815
+0.7553270992585563
+0.2595829718925291
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7396285443968873
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8980412833157345
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.14421518948927226
+0.48086904368685257
+3.345563419756308
+3.615777007111271
+3.611846835730545
+3.6122361466389306
+3.6122361466389306
+3.5768044292407
+0.773399350476982
+0.24055939460771647
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.9480916413381557
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.6623325614087925
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.14421518948927226
+0.519387512494303
+3.339814797789736
+3.60640522163187
+3.611352922417542
+3.611846835730545
+3.6122361466389306
+3.522512506538295
+0.7852235592687276
+0.22140021244840558
+0.08652970395641724
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.19262923125003173
+0.896381476420494
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8268653780234594
+0.44213125973792344
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.14421518948927226
+0.5386918553337177
+3.3398686007305654
+3.60640522163187
+3.602431634147373
+3.602431634147373
+3.611846835730545
+3.4725591053650846
+0.7981582714728965
+0.2018996889437794
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3557324805227413
+0.8268653780234594
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8674260761271935
+0.2501845927102355
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.17305757537147226
+0.548193339234583
+3.337722952345149
+3.6058675054218323
+3.602431634147373
+3.602431634147373
+3.602431634147373
+3.433949486604032
+0.7952518509254771
+0.17305757537147226
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5762468110472865
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.9517446316409384
+0.10651401691436746
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.17305757537147226
+0.548193339234583
+3.339222710596468
+3.605239693322749
+3.60192095283249
+3.60192095283249
+3.602431634147373
+3.411934280089485
+0.8019432589378992
+0.17305757537147226
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.8050731528963935
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.814533543275509
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.17305757537147226
+0.519387512494303
+2.6581767679401853
+3.596668967777341
+3.6013224079591977
+3.6013224079591977
+3.60192095283249
+3.397900324664534
+0.8007101677278259
+0.17305757537147226
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.10651401691436746
+0.9644359211176852
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.6049589697638957
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.17305757537147226
+0.4902956806169465
+2.647548190681089
+3.596050400837252
+3.592098998426612
+3.5926851884767963
+3.6013224079591977
+3.38524471697017
+0.7977398853553179
+0.17305757537147226
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.230741499582245
+0.8674260761271935
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8268653780234594
+0.38453231097115825
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.17305757537147226
+0.48086904368685257
+2.644890284803525
+3.5831072087996483
+3.592098998426612
+3.592098998426612
+3.5926851884767963
+3.3797180321147944
+0.7977398853553179
+0.17305757537147226
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4229997026740895
+0.8268653780234594
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8829731209921494
+0.2018996889437794
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.17305757537147226
+0.48086904368685257
+2.650346155851089
+3.591440647082618
+3.591440647082618
+3.591440647082618
+3.592098998426612
+3.366298608599595
+0.7978190722169973
+0.17305757537147226
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5094164061306206
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.8151569303072108
+0.833333333333333
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.17305757537147226
+0.48086904368685257
+2.644890284803525
+3.5824858983856354
+3.587337166476254
+3.587337166476254
+3.5831072087996483
+3.355478394593713
+0.7843371560037582
+0.17305757537147226
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.17305757537147226
+0.7481662154254112
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.8980412833157345
+0.7195984021833732
+0.19262923125003173
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.14421518948927226
+0.4326218123061114
+1.376450103746838
+3.5824858983856354
+3.5824858983856354
+3.5831072087996483
+3.591440647082618
+3.326998084168252
+0.670764537079803
+0.17305757537147226
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.11537255792817191
+0.3461049112705705
+0.7816423580346736
+3.3121934427792192
+3.305953106889765
+2.6831376611757842
+2.6193992969677624
+0.7804689843874018
+0.4326218123061114
+0.17305757537147226
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08652970395641724
+0.2595829718925291
+0.5191309563131474
+0.8648354565122771
+0.963094151909706
+0.9134702960435828
+0.7497229403733847
+0.5191309563131474
+0.2884240647434684
+0.11537255792817191
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.14421518948927226
+0.2884240647434684
+0.451806660765417
+0.5191309563131474
+0.5191309563131474
+0.4326218123061114
+0.3172647300558147
+0.17305757537147226
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0576866479940108
+0.11537255792817191
+0.2018996889437794
+0.230741499582245
+0.230741499582245
+0.17305757537147226
+0.11537255792817191
+0.0576866479940108
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.02884340803556551
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.0576866479940108
+0.02884340803556551
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
\ No newline at end of file
diff --git a/examples/heat-equation/data/source.mtx b/examples/heat-equation/data/source.mtx
new file mode 100644
index 00000000000..15b759340be
--- /dev/null
+++ b/examples/heat-equation/data/source.mtx
@@ -0,0 +1,65538 @@
+%%MatrixMarket matrix array real general
+65536 1
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.09803921568627451
+0.3058823529411765
+0.40784313725490196
+0.3058823529411765
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.33725490196078434
+0.4549019607843137
+0.611764705882353
+0.7215686274509804
+0.7686274509803922
+0.8549019607843137
+0.9215686274509803
+0.9725490196078431
+1.0
+0.9686274509803922
+0.7176470588235294
+0.023529411764705882
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0196078431372549
+0.3176470588235294
+0.4627450980392157
+0.5098039215686274
+0.5411764705882353
+0.5607843137254902
+0.5607843137254902
+0.5529411764705883
+0.5333333333333333
+0.5098039215686274
+0.48627450980392156
+0.4392156862745098
+0.36470588235294116
+0.16470588235294117
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.16862745098039217
+0.4196078431372549
+0.5215686274509804
+0.6235294117647059
+0.7529411764705882
+0.7372549019607844
+0.5137254901960784
+0.14901960784313725
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.47058823529411764
+0.6627450980392157
+0.7686274509803922
+0.8549019607843137
+0.9333333333333333
+0.9764705882352941
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.09411764705882353
+0.42745098039215684
+0.6588235294117647
+0.8274509803921568
+0.9450980392156862
+0.9725490196078431
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9921568627450981
+0.9803921568627451
+0.9568627450980393
+0.9490196078431372
+0.8745098039215686
+0.7843137254901961
+0.6470588235294118
+0.5215686274509804
+0.30196078431372547
+0.07450980392156863
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.18823529411764706
+0.4235294117647059
+0.5607843137254902
+0.6980392156862745
+0.796078431372549
+0.9176470588235294
+0.9568627450980393
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8313725490196079
+0.047058823529411764
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00392156862745098
+0.8705882352941177
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0196078431372549
+0.3568627450980392
+0.7215686274509804
+0.9411764705882353
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.9764705882352941
+0.9607843137254902
+0.6
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.03529411764705882
+0.8588235294117647
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8745098039215686
+0.06274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00392156862745098
+0.8705882352941177
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.3843137254901961
+0.803921568627451
+0.9764705882352941
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6823529411764706
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.13725490196078433
+0.9176470588235294
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8588235294117647
+0.03137254901960784
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00392156862745098
+0.8705882352941177
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.21176470588235294
+0.7137254901960784
+0.9725490196078431
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6078431372549019
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.24705882352941178
+0.9607843137254902
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8509803921568627
+0.01568627450980392
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8549019607843137
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.42745098039215684
+0.8980392156862745
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9686274509803922
+0.8274509803921568
+0.6784313725490196
+0.6039215686274509
+0.592156862745098
+0.6
+0.6352941176470588
+0.7490196078431373
+0.8666666666666667
+0.9803921568627451
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.5568627450980392
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3176470588235294
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8470588235294118
+0.00392156862745098
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.27450980392156865
+0.6745098039215687
+0.9333333333333333
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00784313725490196
+0.5529411764705883
+0.9725490196078431
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9254901960784314
+0.6470588235294118
+0.27450980392156865
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3568627450980392
+0.7058823529411765
+0.9607843137254902
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.5333333333333333
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3686274509803922
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.807843137254902
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.23137254901960785
+0.6313725490196078
+0.8862745098039215
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00784313725490196
+0.5686274509803921
+0.9803921568627451
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.8117647058823529
+0.25882352941176473
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5019607843137255
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.5215686274509804
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4627450980392157
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9725490196078431
+0.8549019607843137
+0.7098039215686275
+0.30980392156862746
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.13333333333333333
+0.9294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5411764705882353
+0.9803921568627451
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.7490196078431373
+0.09803921568627451
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.8352941176470589
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.984313725490196
+0.38823529411764707
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.20392156862745098
+0.8
+0.9803921568627451
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9607843137254902
+0.8627450980392157
+0.6392156862745098
+0.3215686274509804
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3843137254901961
+0.9568627450980393
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.792156862745098
+0.09803921568627451
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4588235294117647
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9725490196078431
+0.2196078431372549
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.43529411764705883
+0.8352941176470589
+0.996078431372549
+0.9647058823529412
+0.8156862745098039
+0.5725490196078431
+0.28627450980392155
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.1568627450980392
+0.8627450980392157
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.23529411764705882
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.01568627450980392
+0.8117647058823529
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9647058823529412
+0.09803921568627451
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.07450980392156863
+0.34901960784313724
+0.26666666666666666
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6705882352941176
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.5215686274509804
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.44313725490196076
+0.984313725490196
+1.0
+1.0
+1.0
+1.0
+0.9647058823529412
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3215686274509804
+0.9529411764705882
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8549019607843137
+0.08235294117647059
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7019607843137254
+0.8313725490196079
+0.8313725490196079
+0.8313725490196079
+0.8313725490196079
+0.7372549019607844
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00392156862745098
+0.00392156862745098
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.01568627450980392
+0.023529411764705882
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7450980392156863
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.5137254901960784
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00392156862745098
+0.00784313725490196
+0.00784313725490196
+0.00784313725490196
+0.00784313725490196
+0.00392156862745098
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.19215686274509805
+0.5372549019607843
+0.788235294117647
+0.7176470588235294
+0.42745098039215684
+0.09411764705882353
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2235294117647059
+0.5333333333333333
+0.7764705882352941
+0.7529411764705882
+0.5333333333333333
+0.07058823529411765
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.07450980392156863
+0.3176470588235294
+0.6
+0.7137254901960784
+0.8470588235294118
+0.8666666666666667
+0.8666666666666667
+0.7843137254901961
+0.6470588235294118
+0.3254901960784314
+0.03529411764705882
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.09803921568627451
+0.30980392156862746
+0.5490196078431373
+0.6784313725490196
+0.7647058823529411
+0.8627450980392157
+0.8666666666666667
+0.8666666666666667
+0.8666666666666667
+0.8313725490196079
+0.7098039215686275
+0.6392156862745098
+0.3843137254901961
+0.17254901960784313
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.26666666666666666
+0.6078431372549019
+0.7411764705882353
+0.8627450980392157
+0.8666666666666667
+0.8666666666666667
+0.8666666666666667
+0.5176470588235295
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.027450980392156862
+0.25882352941176473
+0.4980392156862745
+0.6745098039215687
+0.7803921568627451
+0.8666666666666667
+0.8705882352941177
+0.8745098039215686
+0.8705882352941177
+0.8588235294117647
+0.7372549019607844
+0.6549019607843137
+0.4
+0.1803921568627451
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.24705882352941178
+0.9450980392156862
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9333333333333333
+0.1843137254901961
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.25098039215686274
+0.5803921568627451
+0.7843137254901961
+0.9333333333333333
+1.0
+1.0
+1.0
+1.0
+0.5882352941176471
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3333333333333333
+0.6509803921568628
+0.8156862745098039
+0.9450980392156862
+1.0
+1.0
+1.0
+0.9725490196078431
+0.32941176470588235
+0.0
+0.0
+0.0
+0.0
+0.33725490196078434
+0.7019607843137254
+0.8745098039215686
+0.984313725490196
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9803921568627451
+0.8313725490196079
+0.4980392156862745
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.2627450980392157
+0.7843137254901961
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.803921568627451
+0.8
+0.3803921568627451
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.10196078431372549
+0.49411764705882355
+0.7529411764705882
+0.8980392156862745
+0.984313725490196
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9254901960784314
+0.7843137254901961
+0.5058823529411764
+0.07450980392156863
+0.0
+0.13725490196078433
+0.7058823529411765
+0.9568627450980393
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6196078431372549
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.29411764705882354
+0.6705882352941176
+0.8509803921568627
+0.9607843137254902
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9294117647058824
+0.788235294117647
+0.5294117647058824
+0.11764705882352941
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6313725490196078
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7843137254901961
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3333333333333333
+0.5647058823529412
+0.796078431372549
+0.9529411764705882
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.1411764705882353
+0.4627450980392157
+0.6745098039215687
+0.8627450980392157
+0.9803921568627451
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9764705882352941
+0.3411764705882353
+0.0
+0.0
+0.27450980392156865
+0.7176470588235294
+0.9686274509803922
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.984313725490196
+0.7137254901960784
+0.11372549019607843
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.3568627450980392
+0.9764705882352941
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.5333333333333333
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.06666666666666667
+0.5607843137254902
+0.8705882352941177
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.8392156862745098
+0.4196078431372549
+0.7529411764705882
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6196078431372549
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3215686274509804
+0.7333333333333333
+0.9607843137254902
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8666666666666667
+0.5058823529411764
+0.011764705882352941
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8784313725490196
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.5490196078431373
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00392156862745098
+0.30980392156862746
+0.596078431372549
+0.8313725490196079
+0.9725490196078431
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3254901960784314
+0.7529411764705882
+0.9372549019607843
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9764705882352941
+0.3411764705882353
+0.00392156862745098
+0.5490196078431373
+0.9333333333333333
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7607843137254902
+0.058823529411764705
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.3568627450980392
+0.9764705882352941
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.5333333333333333
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3176470588235294
+0.8196078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9764705882352941
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6196078431372549
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.050980392156862744
+0.6039215686274509
+0.9568627450980393
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9882352941176471
+0.7607843137254902
+0.17647058823529413
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2823529411764706
+0.9607843137254902
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9803921568627451
+0.30196078431372547
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3137254901960784
+0.9686274509803922
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6039215686274509
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9764705882352941
+0.3843137254901961
+0.7568627450980392
+0.9921568627450981
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.5568627450980392
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.3568627450980392
+0.9764705882352941
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.5333333333333333
+0.0
+0.0
+0.0
+0.0
+0.39215686274509803
+0.9294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9725490196078431
+0.8313725490196079
+0.7215686274509804
+0.7098039215686275
+0.7725490196078432
+0.9215686274509803
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6196078431372549
+0.0
+0.0
+0.0
+0.0
+0.0
+0.16862745098039217
+0.7803921568627451
+0.9921568627450981
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9568627450980393
+0.8627450980392157
+0.803921568627451
+0.8352941176470589
+0.9333333333333333
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8549019607843137
+0.21176470588235294
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4980392156862745
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9137254901960784
+0.13333333333333333
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.35294117647058826
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6039215686274509
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9411764705882353
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8627450980392157
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.3568627450980392
+0.9764705882352941
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.5333333333333333
+0.0
+0.0
+0.0
+0.36470588235294116
+0.9333333333333333
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8901960784313725
+0.396078431372549
+0.00784313725490196
+0.0
+0.0
+0.0
+0.1803921568627451
+0.6980392156862745
+0.9921568627450981
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6196078431372549
+0.0
+0.0
+0.0
+0.0
+0.2
+0.8509803921568627
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.984313725490196
+0.7411764705882353
+0.27450980392156865
+0.023529411764705882
+0.0
+0.0
+0.2
+0.6705882352941176
+0.9764705882352941
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8352941176470589
+0.13333333333333333
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7058823529411765
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8705882352941177
+0.011764705882352941
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3411764705882353
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6039215686274509
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9764705882352941
+0.29411764705882354
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.25882352941176473
+0.8274509803921568
+0.9019607843137255
+0.9882352941176471
+0.9921568627450981
+0.9921568627450981
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9921568627450981
+0.9921568627450981
+0.9882352941176471
+0.9686274509803922
+0.8784313725490196
+0.8431372549019608
+0.36470588235294116
+0.0
+0.0
+0.1607843137254902
+0.8745098039215686
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9450980392156862
+0.30980392156862746
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.03137254901960784
+0.7372549019607844
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6196078431372549
+0.0
+0.0
+0.0
+0.09803921568627451
+0.792156862745098
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6392156862745098
+0.03137254901960784
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00784313725490196
+0.615686274509804
+0.9921568627450981
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6980392156862745
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8392156862745098
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8196078431372549
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.12941176470588237
+0.8392156862745098
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.43529411764705883
+0.9490196078431372
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9921568627450981
+0.8862745098039215
+0.6980392156862745
+0.5647058823529412
+0.5098039215686274
+0.5764705882352941
+0.6627450980392157
+0.8627450980392157
+0.9921568627450981
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.5294117647058824
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.01568627450980392
+0.09803921568627451
+0.1803921568627451
+0.2549019607843137
+0.41568627450980394
+0.9333333333333333
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8352941176470589
+0.4235294117647059
+0.27450980392156865
+0.2
+0.1568627450980392
+0.06666666666666667
+0.011764705882352941
+0.0
+0.0
+0.0
+0.615686274509804
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.21176470588235294
+0.9411764705882353
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.792156862745098
+0.7411764705882353
+0.7411764705882353
+0.7411764705882353
+0.7411764705882353
+0.7411764705882353
+0.44313725490196076
+0.0
+0.0
+0.0
+0.6549019607843137
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8156862745098039
+0.08235294117647059
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.058823529411764705
+0.8313725490196079
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.984313725490196
+0.43137254901960786
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.054901960784313725
+0.8980392156862745
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7647058823529411
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.24705882352941178
+0.8745098039215686
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3764705882352941
+0.8431372549019608
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8549019607843137
+0.44313725490196076
+0.050980392156862744
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0392156862745098
+0.5882352941176471
+0.984313725490196
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.615686274509804
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.11764705882352941
+0.7568627450980392
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8156862745098039
+0.19607843137254902
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.06666666666666667
+0.8784313725490196
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9882352941176471
+0.3686274509803922
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00392156862745098
+0.7490196078431373
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.5450980392156862
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3843137254901961
+0.9686274509803922
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9921568627450981
+0.44313725490196076
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5450980392156862
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.803921568627451
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2196078431372549
+0.9490196078431372
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7372549019607844
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.35294117647058826
+0.9254901960784314
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.12156862745098039
+0.6941176470588235
+0.9725490196078431
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00784313725490196
+0.7490196078431373
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7254901960784313
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.058823529411764705
+0.7333333333333333
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.807843137254902
+0.1450980392156863
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.37254901960784315
+0.9921568627450981
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9529411764705882
+0.24705882352941178
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6196078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6705882352941176
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00784313725490196
+0.8
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9098039215686274
+0.10196078431372549
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.16862745098039217
+0.9490196078431372
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9803921568627451
+0.3843137254901961
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3137254901960784
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7333333333333333
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.42745098039215684
+0.9568627450980393
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6274509803921569
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5529411764705883
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.792156862745098
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.01568627450980392
+0.6862745098039216
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.796078431372549
+0.12941176470588237
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6431372549019608
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9450980392156862
+0.2196078431372549
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5411764705882353
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.788235294117647
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.32941176470588235
+0.9686274509803922
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7294117647058823
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00784313725490196
+0.8
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7176470588235294
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3215686274509804
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7450980392156863
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.611764705882353
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4470588235294118
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.26666666666666666
+0.9921568627450981
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8235294117647058
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.6549019607843137
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7607843137254902
+0.11372549019607843
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7019607843137254
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9607843137254902
+0.27450980392156865
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4745098039215686
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.5098039215686274
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6431372549019608
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8588235294117647
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3254901960784314
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7725490196078432
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.1803921568627451
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.06666666666666667
+0.0
+0.0
+0.0
+0.5882352941176471
+0.9803921568627451
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.7215686274509804
+0.11764705882352941
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.984313725490196
+0.34509803921568627
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5098039215686274
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7764705882352941
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0784313725490196
+0.8862745098039215
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9764705882352941
+0.35294117647058826
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4588235294117647
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.19607843137254902
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3215686274509804
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8196078431372549
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.023529411764705882
+0.4196078431372549
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.4627450980392157
+0.37254901960784315
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.1411764705882353
+0.9490196078431372
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9215686274509803
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8980392156862745
+0.058823529411764705
+0.0
+0.0
+0.5411764705882353
+0.9764705882352941
+1.0
+1.0
+1.0
+1.0
+0.9921568627450981
+0.7058823529411765
+0.08627450980392157
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7019607843137254
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.4627450980392157
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6549019607843137
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2549019607843137
+0.9725490196078431
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.19607843137254902
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3215686274509804
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.4117647058823529
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.30980392156862746
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8509803921568627
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.20392156862745098
+0.9333333333333333
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.9921568627450981
+0.8784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.08627450980392157
+0.8941176470588236
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9450980392156862
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8705882352941177
+0.03137254901960784
+0.0
+0.49019607843137253
+0.9725490196078431
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9411764705882353
+0.16470588235294117
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.592156862745098
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6823529411764706
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6901960784313725
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.5254901960784314
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4470588235294118
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9333333333333333
+0.06274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.24705882352941178
+0.984313725490196
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.5019607843137255
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.20784313725490197
+0.9450980392156862
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8980392156862745
+0.058823529411764705
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.20784313725490197
+0.9411764705882353
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8901960784313725
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.058823529411764705
+0.8745098039215686
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8352941176470589
+0.00784313725490196
+0.4588235294117647
+0.9607843137254902
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6745098039215687
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.32941176470588235
+0.9803921568627451
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8941176470588236
+0.13333333333333333
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0784313725490196
+0.8862745098039215
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9764705882352941
+0.23529411764705882
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6392156862745098
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9176470588235294
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.20784313725490197
+0.9647058823529412
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6313725490196078
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.03137254901960784
+0.8941176470588236
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9254901960784314
+0.16470588235294117
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.20784313725490197
+0.9411764705882353
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8901960784313725
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.8666666666666667
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8235294117647058
+0.41568627450980394
+0.9333333333333333
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9686274509803922
+0.37254901960784315
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00392156862745098
+0.8313725490196079
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.5843137254901961
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5843137254901961
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7803921568627451
+0.011764705882352941
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6862745098039216
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.1607843137254902
+0.9294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7215686274509804
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8235294117647058
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9803921568627451
+0.30196078431372547
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.20784313725490197
+0.9411764705882353
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8901960784313725
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.8666666666666667
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8549019607843137
+0.9137254901960784
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8352941176470589
+0.12549019607843137
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5254901960784314
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.5098039215686274
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4470588235294118
+0.9450980392156862
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9647058823529412
+0.3568627450980392
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7568627450980392
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9019607843137255
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.13725490196078433
+0.9137254901960784
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7215686274509804
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6901960784313725
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.4823529411764706
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.1411764705882353
+0.8470588235294118
+0.9372549019607843
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9803921568627451
+0.788235294117647
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.8666666666666667
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.615686274509804
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0196078431372549
+0.7568627450980392
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9803921568627451
+0.8117647058823529
+0.5882352941176471
+0.5019607843137255
+0.5372549019607843
+0.7568627450980392
+0.9686274509803922
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9882352941176471
+0.6196078431372549
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.807843137254902
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.13725490196078433
+0.9176470588235294
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6941176470588235
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4627450980392157
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6941176470588235
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.12156862745098039
+0.3803921568627451
+0.4627450980392157
+0.6235294117647059
+0.7254901960784313
+0.9254901960784314
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9764705882352941
+0.6235294117647059
+0.3607843137254902
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.8666666666666667
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9450980392156862
+0.33725490196078434
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2
+0.8313725490196079
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.984313725490196
+0.6588235294117647
+0.0392156862745098
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7843137254901961
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9215686274509803
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.17254901960784313
+0.9372549019607843
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.5568627450980392
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.23921568627450981
+0.9490196078431372
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8627450980392157
+0.023529411764705882
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7215686274509804
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.00392156862745098
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.8666666666666667
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9647058823529412
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.803921568627451
+0.058823529411764705
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.1607843137254902
+0.7490196078431373
+0.9803921568627451
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9333333333333333
+0.5647058823529412
+0.011764705882352941
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7176470588235294
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9372549019607843
+0.09019607843137255
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.21176470588235294
+0.9647058823529412
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.4588235294117647
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.8431372549019608
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9568627450980393
+0.23137254901960785
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7176470588235294
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.8666666666666667
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9294117647058824
+0.396078431372549
+0.9019607843137255
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9921568627450981
+0.5686274509803921
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0196078431372549
+0.4745098039215686
+0.9254901960784314
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.7176470588235294
+0.24705882352941178
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6588235294117647
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.21176470588235294
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.25098039215686274
+0.984313725490196
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9725490196078431
+0.3215686274509804
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5647058823529412
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.5254901960784314
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7176470588235294
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.8666666666666667
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9529411764705882
+0.4
+0.0
+0.5411764705882353
+0.9921568627450981
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9254901960784314
+0.23137254901960785
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.6352941176470588
+0.9882352941176471
+1.0
+1.0
+1.0
+0.996078431372549
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.9921568627450981
+0.9568627450980393
+0.807843137254902
+0.5686274509803921
+0.2235294117647059
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5568627450980392
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9803921568627451
+0.36470588235294116
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3215686274509804
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9176470588235294
+0.07058823529411765
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.16470588235294117
+0.9176470588235294
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8
+0.023529411764705882
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7176470588235294
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.8666666666666667
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9725490196078431
+0.4980392156862745
+0.0
+0.0
+0.08235294117647059
+0.8745098039215686
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7568627450980392
+0.023529411764705882
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5568627450980392
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+0.6627450980392157
+0.4549019607843137
+0.5764705882352941
+0.6352941176470588
+0.6549019607843137
+0.6627450980392157
+0.6666666666666666
+0.6509803921568628
+0.6235294117647059
+0.5333333333333333
+0.34901960784313724
+0.1568627450980392
+0.011764705882352941
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.32941176470588235
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.4980392156862745
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.44313725490196076
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.792156862745098
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6784313725490196
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.984313725490196
+0.32941176470588235
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7176470588235294
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.8666666666666667
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8274509803921568
+0.00784313725490196
+0.0
+0.0
+0.0
+0.5019607843137255
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9921568627450981
+0.49019607843137253
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3411764705882353
+0.9529411764705882
+1.0
+1.0
+1.0
+1.0
+0.984313725490196
+0.38823529411764707
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.15294117647058825
+0.9176470588235294
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7019607843137254
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6274509803921569
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.592156862745098
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7137254901960784
+0.00392156862745098
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7176470588235294
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.8666666666666667
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8313725490196079
+0.00392156862745098
+0.0
+0.0
+0.0
+0.023529411764705882
+0.8313725490196079
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9058823529411765
+0.21176470588235294
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.027450980392156862
+0.8392156862745098
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9647058823529412
+0.2980392156862745
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.803921568627451
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00392156862745098
+0.7686274509803922
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9294117647058824
+0.18823529411764706
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5882352941176471
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9647058823529412
+0.37254901960784315
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7176470588235294
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.8666666666666667
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8549019607843137
+0.0196078431372549
+0.0
+0.0
+0.0
+0.0
+0.4549019607843137
+0.984313725490196
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7098039215686275
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.47058823529411764
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.48627450980392156
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9647058823529412
+0.2980392156862745
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.11764705882352941
+0.9215686274509803
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6901960784313725
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.023529411764705882
+0.7803921568627451
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8274509803921568
+0.09411764705882353
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.8196078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.8666666666666667
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8745098039215686
+0.03529411764705882
+0.0
+0.0
+0.0
+0.0
+0.01568627450980392
+0.7764705882352941
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9725490196078431
+0.45098039215686275
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7137254901960784
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8509803921568627
+0.18823529411764706
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.10196078431372549
+0.8666666666666667
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.596078431372549
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4549019607843137
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9411764705882353
+0.26666666666666666
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.25882352941176473
+0.8941176470588236
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.7411764705882353
+0.06274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.47058823529411764
+0.9764705882352941
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6274509803921569
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.047058823529411764
+0.8666666666666667
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8784313725490196
+0.03529411764705882
+0.0
+0.0
+0.0
+0.0
+0.0
+0.37254901960784315
+0.9725490196078431
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8627450980392157
+0.12941176470588237
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7803921568627451
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8901960784313725
+0.6274509803921569
+0.4980392156862745
+0.44313725490196076
+0.4392156862745098
+0.4392156862745098
+0.4392156862745098
+0.4392156862745098
+0.4392156862745098
+0.4392156862745098
+0.4392156862745098
+0.43529411764705883
+0.41568627450980394
+0.3686274509803922
+0.23137254901960785
+0.08235294117647059
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5529411764705883
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8901960784313725
+0.1607843137254902
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.00784313725490196
+0.7490196078431373
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.5764705882352941
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3803921568627451
+0.9372549019607843
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.7725490196078432
+0.23529411764705882
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.03137254901960784
+0.49411764705882355
+0.9529411764705882
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5333333333333333
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6352941176470588
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.050980392156862744
+0.8705882352941177
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.01568627450980392
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8784313725490196
+0.03529411764705882
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.7450980392156863
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6666666666666666
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7686274509803922
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.9882352941176471
+0.9882352941176471
+0.9882352941176471
+0.9882352941176471
+0.9882352941176471
+0.9882352941176471
+0.9882352941176471
+0.9882352941176471
+0.9882352941176471
+0.9882352941176471
+0.9803921568627451
+0.9725490196078431
+0.9647058823529412
+0.9098039215686274
+0.803921568627451
+0.6313725490196078
+0.3803921568627451
+0.06666666666666667
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0392156862745098
+0.7843137254901961
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6627450980392157
+0.011764705882352941
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4549019607843137
+0.9803921568627451
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.803921568627451
+0.07058823529411765
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4235294117647059
+0.9254901960784314
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9333333333333333
+0.6392156862745098
+0.25098039215686274
+0.07058823529411765
+0.011764705882352941
+0.00392156862745098
+0.00784313725490196
+0.054901960784313725
+0.18823529411764706
+0.5254901960784314
+0.8313725490196079
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.14901960784313725
+0.34901960784313724
+0.611764705882353
+0.8627450980392157
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8549019607843137
+0.6
+0.34509803921568627
+0.17254901960784313
+0.047058823529411764
+0.0
+0.0
+0.0
+0.10196078431372549
+0.25882352941176473
+0.5450980392156862
+0.807843137254902
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8862745098039215
+0.611764705882353
+0.33725490196078434
+0.12941176470588237
+0.00392156862745098
+0.0
+0.00392156862745098
+0.11372549019607843
+0.2627450980392157
+0.5450980392156862
+0.7176470588235294
+0.9764705882352941
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.984313725490196
+0.6784313725490196
+0.49019607843137253
+0.23529411764705882
+0.09803921568627451
+0.00392156862745098
+0.0
+0.0
+0.08235294117647059
+0.2549019607843137
+0.5490196078431373
+0.7529411764705882
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9607843137254902
+0.6431372549019608
+0.3843137254901961
+0.1607843137254902
+0.01568627450980392
+0.0
+0.12156862745098039
+0.4666666666666667
+0.8235294117647058
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9686274509803922
+0.6392156862745098
+0.34509803921568627
+0.1607843137254902
+0.00784313725490196
+0.0
+0.0
+0.0
+0.7098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9882352941176471
+0.8784313725490196
+0.5137254901960784
+0.03529411764705882
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.27450980392156865
+0.8980392156862745
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.984313725490196
+0.6470588235294118
+0.08235294117647059
+0.0
+0.0
+0.0
+0.0392156862745098
+0.4666666666666667
+0.9333333333333333
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8666666666666667
+0.19607843137254902
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.33725490196078434
+0.8823529411764706
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9725490196078431
+0.8784313725490196
+0.8352941176470589
+0.8156862745098039
+0.8313725490196079
+0.8705882352941177
+0.9529411764705882
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.5450980392156862
+0.9294117647058824
+0.9921568627450981
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9921568627450981
+0.9490196078431372
+0.7098039215686275
+0.0
+0.0
+0.1607843137254902
+0.8823529411764706
+0.984313725490196
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9921568627450981
+0.9019607843137255
+0.26666666666666666
+0.0
+0.36470588235294116
+0.9019607843137255
+0.984313725490196
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.9803921568627451
+0.8901960784313725
+0.25098039215686274
+0.0
+0.07450980392156863
+0.807843137254902
+0.9803921568627451
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9921568627450981
+0.9372549019607843
+0.592156862745098
+0.0
+0.7686274509803922
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9921568627450981
+0.9372549019607843
+0.42745098039215684
+0.0
+0.0
+0.0
+0.5019607843137255
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9882352941176471
+0.7372549019607844
+0.09019607843137255
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.3215686274509804
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.8666666666666667
+0.6862745098039216
+0.6392156862745098
+0.6784313725490196
+0.8431372549019608
+0.984313725490196
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8588235294117647
+0.22745098039215686
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2
+0.7411764705882353
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9490196078431372
+0.00392156862745098
+0.0
+0.0
+0.0
+0.0
+0.6392156862745098
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8196078431372549
+0.0
+0.0
+0.21568627450980393
+0.9803921568627451
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.984313725490196
+0.3333333333333333
+0.0
+0.4470588235294118
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.3137254901960784
+0.0
+0.10980392156862745
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7215686274509804
+0.0
+0.8274509803921568
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.5333333333333333
+0.0
+0.0
+0.0
+0.12549019607843137
+0.8705882352941177
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7215686274509804
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2784313725490196
+0.8352941176470589
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9921568627450981
+0.7450980392156863
+0.17254901960784313
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4745098039215686
+0.8588235294117647
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9568627450980393
+0.8823529411764706
+0.792156862745098
+0.7568627450980392
+0.7254901960784313
+0.7137254901960784
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6392156862745098
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8196078431372549
+0.0
+0.0
+0.21568627450980393
+0.9803921568627451
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.984313725490196
+0.3333333333333333
+0.0
+0.4470588235294118
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.3137254901960784
+0.0
+0.10980392156862745
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7215686274509804
+0.0
+0.8274509803921568
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.5333333333333333
+0.0
+0.0
+0.0
+0.0
+0.2549019607843137
+0.7843137254901961
+0.984313725490196
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9803921568627451
+0.4235294117647059
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.09411764705882353
+0.6313725490196078
+0.9411764705882353
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8901960784313725
+0.5137254901960784
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.050980392156862744
+0.5490196078431373
+0.8196078431372549
+0.9686274509803922
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9921568627450981
+0.9411764705882353
+0.8509803921568627
+0.7529411764705882
+0.6078431372549019
+0.47058823529411764
+0.1803921568627451
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6392156862745098
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8196078431372549
+0.0
+0.0
+0.21176470588235294
+0.984313725490196
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9882352941176471
+0.32941176470588235
+0.0
+0.4470588235294118
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.30980392156862746
+0.0
+0.10588235294117647
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7176470588235294
+0.0
+0.8274509803921568
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.5254901960784314
+0.0
+0.0
+0.0
+0.0
+0.0
+0.25882352941176473
+0.8470588235294118
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.792156862745098
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.30196078431372547
+0.6784313725490196
+0.8941176470588236
+0.9921568627450981
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9803921568627451
+0.8509803921568627
+0.592156862745098
+0.13333333333333333
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.30980392156862746
+0.6196078431372549
+0.7764705882352941
+0.8823529411764706
+0.9294117647058824
+0.9607843137254902
+0.996078431372549
+1.0
+1.0
+0.9686274509803922
+0.9372549019607843
+0.9058823529411765
+0.8196078431372549
+0.7372549019607844
+0.615686274509804
+0.403921568627451
+0.1450980392156863
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.2784313725490196
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.3843137254901961
+0.0
+0.0
+0.054901960784313725
+0.4745098039215686
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.4823529411764706
+0.11764705882352941
+0.0
+0.1803921568627451
+0.49019607843137253
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.4823529411764706
+0.10980392156862745
+0.0
+0.01568627450980392
+0.43137254901960786
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.30196078431372547
+0.0
+0.38823529411764707
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.48627450980392156
+0.1843137254901961
+0.0
+0.0
+0.0
+0.0
+0.4117647058823529
+0.9215686274509803
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.615686274509804
+0.4470588235294118
+0.4666666666666667
+0.47843137254901963
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.49411764705882355
+0.5215686274509804
+0.6666666666666666
+0.7411764705882353
+0.8627450980392157
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9333333333333333
+0.18823529411764706
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.07058823529411765
+0.4235294117647059
+0.6980392156862745
+0.788235294117647
+0.8784313725490196
+0.9215686274509803
+0.9333333333333333
+0.9450980392156862
+0.9450980392156862
+0.9333333333333333
+0.9215686274509803
+0.8666666666666667
+0.7764705882352941
+0.6431372549019608
+0.35294117647058826
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.15294117647058825
+0.2784313725490196
+0.3686274509803922
+0.3843137254901961
+0.3764705882352941
+0.2980392156862745
+0.1843137254901961
+0.054901960784313725
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.396078431372549
+0.9490196078431372
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9411764705882353
+0.1803921568627451
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.023529411764705882
+0.4627450980392157
+0.9294117647058824
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.33725490196078434
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.10196078431372549
+0.17647058823529413
+0.21568627450980393
+0.2196078431372549
+0.1803921568627451
+0.09803921568627451
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.23529411764705882
+0.8901960784313725
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8588235294117647
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.5058823529411764
+0.9921568627450981
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.41568627450980394
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6627450980392157
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7803921568627451
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.023529411764705882
+0.8823529411764706
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.4470588235294118
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.14901960784313725
+0.9098039215686274
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7647058823529411
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7686274509803922
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.4235294117647059
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.4392156862745098
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.792156862745098
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7058823529411765
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.3411764705882353
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6784313725490196
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8941176470588236
+0.00784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7019607843137254
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9294117647058824
+0.17647058823529413
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6980392156862745
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9764705882352941
+0.37254901960784315
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.7686274509803922
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.7490196078431373
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6823529411764706
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.8117647058823529
+0.09803921568627451
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.09411764705882353
+0.8980392156862745
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9725490196078431
+0.3568627450980392
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.45098039215686275
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.7058823529411765
+0.11372549019607843
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6078431372549019
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.6941176470588235
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.16862745098039217
+0.9137254901960784
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.8627450980392157
+0.4392156862745098
+0.07058823529411765
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.011764705882352941
+0.18823529411764706
+0.6588235294117647
+0.9686274509803922
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.807843137254902
+0.12156862745098039
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.6392156862745098
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9882352941176471
+0.8901960784313725
+0.7764705882352941
+0.6196078431372549
+0.6
+0.6
+0.6039215686274509
+0.615686274509804
+0.7372549019607844
+0.8431372549019608
+0.9490196078431372
+0.996078431372549
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.792156862745098
+0.15294117647058825
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.12156862745098039
+0.8
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9803921568627451
+0.6392156862745098
+0.09019607843137255
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.17647058823529413
+0.7529411764705882
+0.9882352941176471
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.984313725490196
+0.796078431372549
+0.3764705882352941
+0.00392156862745098
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.47843137254901963
+0.8
+0.9725490196078431
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.996078431372549
+0.9176470588235294
+0.7215686274509804
+0.4196078431372549
+0.00392156862745098
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.33725490196078434
+0.6509803921568628
+0.8
+0.9098039215686274
+0.9568627450980393
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+1.0
+0.9607843137254902
+0.9254901960784314
+0.8235294117647058
+0.7254901960784313
+0.4823529411764706
+0.10196078431372549
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.050980392156862744
+0.2549019607843137
+0.43137254901960786
+0.49019607843137253
+0.5803921568627451
+0.7019607843137254
+0.7215686274509804
+0.7215686274509804
+0.7058823529411765
+0.596078431372549
+0.49411764705882355
+0.43529411764705883
+0.2784313725490196
+0.0784313725490196
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
+0.0
diff --git a/examples/heat-equation/doc/builds-on b/examples/heat-equation/doc/builds-on
new file mode 100644
index 00000000000..896db74e274
--- /dev/null
+++ b/examples/heat-equation/doc/builds-on
@@ -0,0 +1 @@
+simple-solver three-pt-stencil-solver
diff --git a/examples/heat-equation/doc/intro.dox b/examples/heat-equation/doc/intro.dox
new file mode 100644
index 00000000000..0e05f60761a
--- /dev/null
+++ b/examples/heat-equation/doc/intro.dox
@@ -0,0 +1,39 @@
+<a name="Intro"></a>
+<h1>Introduction</h1>
+This example solves a 2D heat conduction equation
+
+$
+    u : [0, d]^2 \rightarrow R\\
+    \partial_t u = \delta u + f
+$
+
+with Dirichlet boundary conditions and given initial condition and function f.
+
+The partial differential equation (PDE) is solved with a finite difference
+spatial discretization on an equidistant grid: For `n` grid points,
+and grid distance $h = 1/n$ we write
+
+$
+    u_{i,j}' =
+    \alpha\frac{u_{i-1,j}+u_{i+1,j}+u_{i,j-1}+u_{i,j+1}-4u_{i,j}}{h^2}+f_{i,j}
+$
+
+We then build an implicit Euler integrator by discretizing with time step $\tau$
+
+$
+    \frac{u_{i,j}^{k+1} - u_{i,j}^k}{\tau} =
+    \alpha\frac{u_{i-1,j}^{k+1}+u_{i+1,j}^{k+1}
+               -u_{i,j-1}^{k+1}-u_{i,j+1}^{k+1}+4u_{i,j}^{k+1}}{h^2}
+    +f_{i,j}^{k+1}
+$
+
+and solve the resulting linear system for $ u_{\cdot}^{k+1}$ using Ginkgo's CG
+solver preconditioned with an incomplete Cholesky factorization for each time
+step, occasionally writing the resulting grid values into a video file using
+OpenCV and a custom color mapping.
+
+The intention of this example is to provide a mini-app showing matrix assembly,
+vector initialization, solver setup and the use of Ginkgo in a more complex
+setting.
+
+<h3> About the example </h3>
\ No newline at end of file
diff --git a/examples/heat-equation/doc/kind b/examples/heat-equation/doc/kind
new file mode 100644
index 00000000000..c1d9154931a
--- /dev/null
+++ b/examples/heat-equation/doc/kind
@@ -0,0 +1 @@
+techniques
diff --git a/examples/heat-equation/doc/results.dox b/examples/heat-equation/doc/results.dox
new file mode 100644
index 00000000000..6bcba24ed7a
--- /dev/null
+++ b/examples/heat-equation/doc/results.dox
@@ -0,0 +1,5 @@
+<h1>Results</h1>
+The program will generate a video file named heat.mp4 and output the timestamp
+of each generated frame.
+
+<h3> Comments about programming and debugging </h3>
diff --git a/examples/heat-equation/doc/short-intro b/examples/heat-equation/doc/short-intro
new file mode 100644
index 00000000000..9ba4bd2def0
--- /dev/null
+++ b/examples/heat-equation/doc/short-intro
@@ -0,0 +1 @@
+The heat equation example.
diff --git a/examples/heat-equation/doc/tooltip b/examples/heat-equation/doc/tooltip
new file mode 100644
index 00000000000..4783754312d
--- /dev/null
+++ b/examples/heat-equation/doc/tooltip
@@ -0,0 +1 @@
+Assemble a 2D finite difference matrix. Use it to build an implicit Euler integrator.
diff --git a/examples/heat-equation/heat-equation.cpp b/examples/heat-equation/heat-equation.cpp
new file mode 100644
index 00000000000..138ad3a9575
--- /dev/null
+++ b/examples/heat-equation/heat-equation.cpp
@@ -0,0 +1,209 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+/*****************************<DESCRIPTION>***********************************
+This example solves a 2D heat conduction equation
+
+    u : [0, d]^2 \rightarrow R\\
+    \partial_t u = \delta u + f
+
+with Dirichlet boundary conditions and given initial condition and function f.
+
+The partial differential equation (PDE) is solved with a finite difference
+spatial discretization on an equidistant grid: For `n` grid points,
+and grid distance $h = 1/n$ we write
+
+    u_{i,j}' = \alpha (u_{i-1,j} + u_{i+1,j} + u_{i,j-1} + u_{i,j+1}
+                   - 4 u_{i,j}) / h^2
+               + f_{i,j}
+
+We then build an implicit Euler integrator by discretizing with time step $\tau$
+
+    (u_{i,j}^{k+1} - u_{i,j}^k) / \tau =
+    \alpha (u_{i-1,j}^{k+1} - u_{i+1,j}^{k+1}
+          + u_{i,j-1}^{k+1} - u_{i,j+1}^{k+1} - 4 u_{i,j}^{k+1}) / h^2
+    + f_{i,j}^{k+1}
+
+and solve the resulting linear system for $ u_{\cdot}^{k+1}$ using Ginkgo's CG
+solver preconditioned with an incomplete Cholesky factorization for each time
+step, occasionally writing the resulting grid values into a video file using
+OpenCV and a custom color mapping.
+
+The intention of this example is to provide a mini-app showing matrix assembly,
+vector initialization, solver setup and the use of Ginkgo in a more complex
+setting.
+*****************************<DESCRIPTION>**********************************/
+
+#include <ginkgo/ginkgo.hpp>
+
+#include <chrono>
+#include <fstream>
+#include <iostream>
+
+#include <opencv2/core.hpp>
+#include <opencv2/videoio.hpp>
+
+
+// This function implements a simple Ginkgo-themed clamped color mapping for
+// values in the range [0,5].
+void set_val(unsigned char *data, double value)
+{
+    // RGB values for the 6 colors used for values 0, 1, ..., 5
+    // We will interpolate linearly between these values.
+    double col_r[] = {255, 221, 129, 201, 249, 255};
+    double col_g[] = {255, 220, 130, 161, 158, 204};
+    double col_b[] = {255, 220, 133, 93, 24, 8};
+    value = std::max(0.0, value);
+    auto i = std::max(0, std::min(4, int(value)));
+    auto d = std::max(0.0, std::min(1.0, value - i));
+    // OpenCV uses BGR instead of RGB by default, revert indices
+    data[2] = static_cast<unsigned char>(col_r[i + 1] * d + col_r[i] * (1 - d));
+    data[1] = static_cast<unsigned char>(col_g[i + 1] * d + col_g[i] * (1 - d));
+    data[0] = static_cast<unsigned char>(col_b[i + 1] * d + col_b[i] * (1 - d));
+}
+
+
+// Initialize video output with given dimension and FPS (frames per seconds)
+std::pair<cv::VideoWriter, cv::Mat> build_output(int n, double fps)
+{
+    cv::Size videosize{n, n};
+    auto output =
+        std::make_pair(cv::VideoWriter{}, cv::Mat{videosize, CV_8UC3});
+    auto fourcc = cv::VideoWriter::fourcc('a', 'v', 'c', '1');
+    output.first.open("heat.mp4", fourcc, fps, videosize);
+    return output;
+}
+
+
+// Write the current frame to video output using the above color mapping
+void output_timestep(std::pair<cv::VideoWriter, cv::Mat> &output, int n,
+                     const double *data)
+{
+    for (int i = 0; i < n; i++) {
+        auto row = output.second.ptr(i);
+        for (int j = 0; j < n; j++) {
+            set_val(&row[3 * j], data[i * n + j]);
+        }
+    }
+    output.first.write(output.second);
+}
+
+
+int main(int argc, char *argv[])
+{
+    using mtx = gko::matrix::Csr<>;
+    using vec = gko::matrix::Dense<>;
+
+    // Problem parameters:
+    // simulation length
+    auto t0 = 5.0;
+    // diffusion factor
+    auto diffusion = 0.0005;
+    // scaling factor for heat source
+    auto source_scale = 2.5;
+    // Simulation parameters:
+    // grid size for inner points
+    auto n = 256;
+    // number of simulation steps per second
+    auto steps_per_sec = 500;
+    // number of video frames per second
+    auto fps = 25;
+    // number of grid points
+    auto n2 = n * n;
+    // grid point distance
+    auto h = 1.0 / n;
+    auto h2 = h * h;
+    // time step size for the simulation
+    auto tau = 1.0 / steps_per_sec;
+
+    // create an OpenMP executor
+    auto exec = gko::OmpExecutor::create();
+    // load heat source and initial state vectors
+    std::ifstream source_stream("data/source.mtx");
+    std::ifstream initial_stream("data/initial.mtx");
+    auto source = gko::read<vec>(source_stream, exec);
+    auto in_vector = gko::read<vec>(initial_stream, exec);
+    // create output vector with initial guess for
+    auto out_vector = in_vector->clone();
+    // create scalar for source update
+    auto tau_source_scalar = gko::initialize<vec>({source_scale * tau}, exec);
+    // create stencil matrix as shared_ptr for solver
+    auto stencil_matrix = gko::share(mtx::create(exec));
+    // assemble matrix
+    gko::matrix_data<> mtx_data{gko::dim<2>(n2, n2)};
+    for (int i = 0; i < n; i++) {
+        for (int j = 0; j < n; j++) {
+            auto c = i * n + j;
+            auto c_val = diffusion * tau * 4.0 / h2 + 1.0;
+            auto off_val = -diffusion * tau / h2;
+            // for each grid point: insert 5 stencil points
+            // with Dirichlet boundary conditions, i.e. with zero boundary value
+            // clang-format off
+            if (i > 0) mtx_data.nonzeros.emplace_back(c, c - n, off_val);
+            if (j > 0) mtx_data.nonzeros.emplace_back(c, c - 1, off_val);
+            mtx_data.nonzeros.emplace_back(c, c, c_val);
+            if (j < n - 1) mtx_data.nonzeros.emplace_back(c, c + 1, off_val);
+            if (i < n - 1) mtx_data.nonzeros.emplace_back(c, c + n, off_val);
+            // clang-format on
+        }
+    }
+    stencil_matrix->read(mtx_data);
+    // prepare video output
+    auto output = build_output(n, fps);
+    // build CG solver on stencil with incomplete Cholesky preconditioner
+    // stopping at 1e-10 relative accuracy
+    auto solver =
+        gko::solver::Cg<>::build()
+            .with_preconditioner(gko::preconditioner::Ic<>::build().on(exec))
+            .with_criteria(gko::stop::RelativeResidualNorm<>::build()
+                               .with_tolerance(1e-10)
+                               .on(exec))
+            .on(exec)
+            ->generate(stencil_matrix);
+    // time stamp of the last output frame (sentinel value)
+    double last_t = -t0;
+    // execute implicit Euler method: for each timestep, solve stencil system
+    for (double t = 0; t < t0; t += tau) {
+        // if enough time has passed, output the next frame
+        if (t - last_t > 1.0 / fps) {
+            last_t = t;
+            std::cout << t << std::endl;
+            output_timestep(output, n, in_vector->get_const_values());
+        }
+        // add heat
+        in_vector->add_scaled(lend(tau_source_scalar), lend(source));
+        // execute Euler step
+        solver->apply(lend(in_vector), lend(out_vector));
+        // swap input and output
+        std::swap(in_vector, out_vector);
+    }
+}

From 21d2386ea14229ce11873c243fa6e896bd24dd8e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 1 Feb 2021 18:13:59 +0100
Subject: [PATCH 0312/1008] Review updates

Co-authored-by: Hartwig Anzt <hanzt@icl.utk.edu>
Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
Co-authored-by: Natalie Beams <nbeams@icl.utk.edu>
---
 examples/heat-equation/doc/intro.dox     |  5 +--
 examples/heat-equation/heat-equation.cpp | 41 +++++++++++++++---------
 2 files changed, 28 insertions(+), 18 deletions(-)

diff --git a/examples/heat-equation/doc/intro.dox b/examples/heat-equation/doc/intro.dox
index 0e05f60761a..e57d2052bf3 100644
--- a/examples/heat-equation/doc/intro.dox
+++ b/examples/heat-equation/doc/intro.dox
@@ -7,7 +7,8 @@ $
     \partial_t u = \delta u + f
 $
 
-with Dirichlet boundary conditions and given initial condition and function f.
+with Dirichlet boundary conditions and given initial condition and
+constant-in-time source function f.
 
 The partial differential equation (PDE) is solved with a finite difference
 spatial discretization on an equidistant grid: For `n` grid points,
@@ -24,7 +25,7 @@ $
     \frac{u_{i,j}^{k+1} - u_{i,j}^k}{\tau} =
     \alpha\frac{u_{i-1,j}^{k+1}+u_{i+1,j}^{k+1}
                -u_{i,j-1}^{k+1}-u_{i,j+1}^{k+1}+4u_{i,j}^{k+1}}{h^2}
-    +f_{i,j}^{k+1}
+    +f_{i,j}
 $
 
 and solve the resulting linear system for $ u_{\cdot}^{k+1}$ using Ginkgo's CG
diff --git a/examples/heat-equation/heat-equation.cpp b/examples/heat-equation/heat-equation.cpp
index 138ad3a9575..5ec604d53c9 100644
--- a/examples/heat-equation/heat-equation.cpp
+++ b/examples/heat-equation/heat-equation.cpp
@@ -36,7 +36,8 @@ This example solves a 2D heat conduction equation
     u : [0, d]^2 \rightarrow R\\
     \partial_t u = \delta u + f
 
-with Dirichlet boundary conditions and given initial condition and function f.
+with Dirichlet boundary conditions and given initial condition and
+constant-in-time source function f.
 
 The partial differential equation (PDE) is solved with a finite difference
 spatial discretization on an equidistant grid: For `n` grid points,
@@ -51,7 +52,7 @@ We then build an implicit Euler integrator by discretizing with time step $\tau$
     (u_{i,j}^{k+1} - u_{i,j}^k) / \tau =
     \alpha (u_{i-1,j}^{k+1} - u_{i+1,j}^{k+1}
           + u_{i,j-1}^{k+1} - u_{i,j+1}^{k+1} - 4 u_{i,j}^{k+1}) / h^2
-    + f_{i,j}^{k+1}
+    + f_{i,j}
 
 and solve the resulting linear system for $ u_{\cdot}^{k+1}$ using Ginkgo's CG
 solver preconditioned with an incomplete Cholesky factorization for each time
@@ -65,10 +66,12 @@ setting.
 
 #include <ginkgo/ginkgo.hpp>
 
+
 #include <chrono>
 #include <fstream>
 #include <iostream>
 
+
 #include <opencv2/core.hpp>
 #include <opencv2/videoio.hpp>
 
@@ -131,7 +134,7 @@ int main(int argc, char *argv[])
     // scaling factor for heat source
     auto source_scale = 2.5;
     // Simulation parameters:
-    // grid size for inner points
+    // inner grid points per discretization direction
     auto n = 256;
     // number of simulation steps per second
     auto steps_per_sec = 500;
@@ -139,8 +142,8 @@ int main(int argc, char *argv[])
     auto fps = 25;
     // number of grid points
     auto n2 = n * n;
-    // grid point distance
-    auto h = 1.0 / n;
+    // grid point distance (ignoring boundary points)
+    auto h = 1.0 / (n + 1);
     auto h2 = h * h;
     // time step size for the simulation
     auto tau = 1.0 / steps_per_sec;
@@ -167,13 +170,19 @@ int main(int argc, char *argv[])
             auto off_val = -diffusion * tau / h2;
             // for each grid point: insert 5 stencil points
             // with Dirichlet boundary conditions, i.e. with zero boundary value
-            // clang-format off
-            if (i > 0) mtx_data.nonzeros.emplace_back(c, c - n, off_val);
-            if (j > 0) mtx_data.nonzeros.emplace_back(c, c - 1, off_val);
+            if (i > 0) {
+                mtx_data.nonzeros.emplace_back(c, c - n, off_val);
+            }
+            if (j > 0) {
+                mtx_data.nonzeros.emplace_back(c, c - 1, off_val);
+            }
             mtx_data.nonzeros.emplace_back(c, c, c_val);
-            if (j < n - 1) mtx_data.nonzeros.emplace_back(c, c + 1, off_val);
-            if (i < n - 1) mtx_data.nonzeros.emplace_back(c, c + n, off_val);
-            // clang-format on
+            if (j < n - 1) {
+                mtx_data.nonzeros.emplace_back(c, c + 1, off_val);
+            }
+            if (i < n - 1) {
+                mtx_data.nonzeros.emplace_back(c, c + n, off_val);
+            }
         }
     }
     stencil_matrix->read(mtx_data);
@@ -189,20 +198,20 @@ int main(int argc, char *argv[])
                                .on(exec))
             .on(exec)
             ->generate(stencil_matrix);
-    // time stamp of the last output frame (sentinel value)
+    // time stamp of the last output frame (initialized to a sentinel value)
     double last_t = -t0;
     // execute implicit Euler method: for each timestep, solve stencil system
     for (double t = 0; t < t0; t += tau) {
-        // if enough time has passed, output the next frame
+        // if enough time has passed, output the next video frame
         if (t - last_t > 1.0 / fps) {
             last_t = t;
             std::cout << t << std::endl;
             output_timestep(output, n, in_vector->get_const_values());
         }
-        // add heat
-        in_vector->add_scaled(lend(tau_source_scalar), lend(source));
+        // add heat source contribution
+        in_vector->add_scaled(gko::lend(tau_source_scalar), gko::lend(source));
         // execute Euler step
-        solver->apply(lend(in_vector), lend(out_vector));
+        solver->apply(gko::lend(in_vector), gko::lend(out_vector));
         // swap input and output
         std::swap(in_vector, out_vector);
     }

From cbb1ba15a4123b57ce17abc1395e8b6757c3ee84 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Sat, 30 Jan 2021 12:24:04 +0100
Subject: [PATCH 0313/1008] Improve Ginkgo device arch flags management.

1. Use a newer version of CAS which makes Auto become All when no
   architecture was detected. This ensures that compiling on a node without
   a GPU will provide good performance (at the cost of binary size and
   compile time), instead of failing to provide optimized architecture
   specific kernels.
2. Use the `variable` version of CAS instead of `target` so that we do
   not call the CAS redundantly in several tests.
3. Fix HIP AMDGPU flags which were not properly passed:
   `target_compile_options` do not seem to work with `hip_add_library`, so
   the flags need to be passed to `hip_add_libary` (or executable)
   directly.
---
 cmake/GinkgoConfig.cmake.in                   |  1 +
 cmake/create_test.cmake                       |  5 ++---
 cuda/CMakeLists.txt                           | 11 +++++++----
 cuda/get_info.cmake                           |  1 +
 hip/CMakeLists.txt                            | 19 +++++++++++--------
 .../CudaArchitectureSelector/CMakeLists.txt   |  2 +-
 6 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 2fdcd695319..85d15d16722 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -63,6 +63,7 @@ set(GINKGO_JACOBI_FULL_OPTIMIZATIONS @GINKGO_JACOBI_FULL_OPTIMIZATIONS@)
 set(GINKGO_CUDA_ARCHITECTURES @GINKGO_CUDA_ARCHITECTURES@)
 set(GINKGO_CUDA_DEFAULT_HOST_COMPILER @GINKGO_CUDA_DEFAULT_HOST_COMPILER@)
 set(GINKGO_CUDA_HOST_COMPILER @CMAKE_CUDA_HOST_COMPILER@)
+set(GINKGO_CUDA_ARCH_FLAGS @GINKGO_CUDA_ARCH_FLAGS@)
 
 set(GINKGO_HIP_COMPILER_FLAGS @GINKGO_HIP_COMPILER_FLAGS@)
 set(GINKGO_HIP_HCC_COMPILER_FLAGS @GINKGO_HIP_HCC_COMPILER_FLAGS@)
diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index d12a479743f..022de4bf61b 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -89,9 +89,8 @@ function(ginkgo_create_cuda_test test_name)
         PRIVATE
         "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
         )
-    cas_target_cuda_architectures(${TEST_TARGET_NAME}
-        ARCHITECTURES ${GINKGO_CUDA_ARCHITECTURES}
-        UNSUPPORTED "20" "21")
+    target_compile_options(${TEST_TARGET_NAME}
+        PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_ARCH_FLAGS}>")
     set_target_properties(${TEST_TARGET_NAME} PROPERTIES
         OUTPUT_NAME ${test_name})
 
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index dab6efee927..0b04f87c40c 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -35,6 +35,11 @@ set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS}" PARENT_SCOPE)
 set(CMAKE_CUDA_COMPILER_VERSION ${CMAKE_CUDA_COMPILER_VERSION} PARENT_SCOPE)
 set(CUDA_INCLUDE_DIRS ${CUDA_INCLUDE_DIRS} PARENT_SCOPE)
 
+# Detect the CUDA architecture flags and propagate to all the project
+cas_variable_cuda_architectures(GINKGO_CUDA_ARCH_FLAGS
+    ARCHITECTURES ${GINKGO_CUDA_ARCHITECTURES}
+    UNSUPPORTED "20" "21")
+set(GINKGO_CUDA_ARCH_FLAGS "${GINKGO_CUDA_ARCH_FLAGS}" PARENT_SCOPE)
 
 # MSVC nvcc uses static cudartlibrary by default, and other platforms use shared cudartlibrary.
 # add `-cudart shared` or `-cudart=shared` according system into CMAKE_CUDA_FLAGS
@@ -156,10 +161,8 @@ target_include_directories(ginkgo_cuda
     SYSTEM PRIVATE ${CUDA_INCLUDE_DIRS})
 target_link_libraries(ginkgo_cuda PRIVATE ${CUDA_RUNTIME_LIBS} ${CUBLAS} ${CUSPARSE} ${CURAND})
 target_link_libraries(ginkgo_cuda PRIVATE ginkgo_core_device)
-
-cas_target_cuda_architectures(ginkgo_cuda
-    ARCHITECTURES ${GINKGO_CUDA_ARCHITECTURES}
-    UNSUPPORTED "20" "21")
+target_compile_options(ginkgo_cuda
+        PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_ARCH_FLAGS}>")
 
 ginkgo_default_includes(ginkgo_cuda)
 ginkgo_install_library(ginkgo_cuda cuda)
diff --git a/cuda/get_info.cmake b/cuda/get_info.cmake
index 4fbf19d5dca..3d91ea9f23a 100644
--- a/cuda/get_info.cmake
+++ b/cuda/get_info.cmake
@@ -2,6 +2,7 @@ ginkgo_print_module_header(${detailed_log} "CUDA")
 ginkgo_print_variable(${detailed_log} "GINKGO_CUDA_ARCHITECTURES")
 ginkgo_print_variable(${detailed_log} "GINKGO_CUDA_COMPILER_FLAGS")
 ginkgo_print_variable(${detailed_log} "GINKGO_CUDA_DEFAULT_HOST_COMPILER")
+ginkgo_print_variable(${detailed_log} "GINKGO_CUDA_ARCH_FLAGS")
 ginkgo_print_module_footer(${detailed_log} "CUDA variables:")
 ginkgo_print_variable(${detailed_log} "CMAKE_CUDA_COMPILER")
 ginkgo_print_variable(${detailed_log} "CMAKE_CUDA_COMPILER_VERSION")
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 143b0eaec2e..7d7ce4d3ab2 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -240,10 +240,19 @@ if (GINKGO_HIP_PLATFORM MATCHES "nvcc")
         ARCHITECTURES ${GINKGO_CUDA_ARCHITECTURES}
         UNSUPPORTED "20" "21")
 endif()
+
+# `target_compile_options` do not work with hip_add_library
+# Thus, we need to pass the flags to `hip_add_library` itself
+if(GINKGO_HIP_AMDGPU)
+    foreach(target ${GINKGO_HIP_AMDGPU})
+        list(APPEND GINKGO_AMDGPU_FLAGS --amdgpu-target=${target})
+    endforeach()
+endif()
+
 set(GINKGO_HIPCC_OPTIONS ${GINKGO_HIP_COMPILER_FLAGS})
 set(GINKGO_HIP_NVCC_OPTIONS ${GINKGO_HIP_NVCC_COMPILER_FLAGS} ${GINKGO_HIP_NVCC_ARCH} ${GINKGO_HIP_NVCC_ADDITIONAL_FLAGS})
-set(GINKGO_HIP_HCC_OPTIONS ${GINKGO_HIP_HCC_COMPILER_FLAGS})
-set(GINKGO_HIP_CLANG_OPTIONS ${GINKGO_HIP_CLANG_COMPILER_FLAGS})
+set(GINKGO_HIP_HCC_OPTIONS ${GINKGO_HIP_HCC_COMPILER_FLAGS} ${GINKGO_AMDGPU_FLAGS})
+set(GINKGO_HIP_CLANG_OPTIONS ${GINKGO_HIP_CLANG_COMPILER_FLAGS} ${GINKGO_AMDGPU_FLAGS})
 
 set_source_files_properties(${GINKGO_HIP_SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
 if (GINKGO_HIP_VERSION VERSION_GREATER_EQUAL "3.5")
@@ -261,12 +270,6 @@ else()
         ${GINKGO_STATIC_OR_SHARED})
 endif()
 
-if(GINKGO_HIP_AMDGPU AND GINKGO_HIP_PLATFORM MATCHES "hcc")
-    foreach(target ${GINKGO_HIP_AMDGPU})
-        target_compile_options(ginkgo_hip PRIVATE --amdgpu-target=${target})
-        target_link_libraries(ginkgo_hip PRIVATE --amdgpu-target=${target})
-    endforeach()
-endif()
 
 target_compile_options(ginkgo_hip PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${GINKGO_COMPILER_FLAGS}>)
 if(GINKGO_WITH_CLANG_TIDY AND GINKGO_CLANG_TIDY_PATH)
diff --git a/third_party/CudaArchitectureSelector/CMakeLists.txt b/third_party/CudaArchitectureSelector/CMakeLists.txt
index e4d53a76a56..ee951455644 100644
--- a/third_party/CudaArchitectureSelector/CMakeLists.txt
+++ b/third_party/CudaArchitectureSelector/CMakeLists.txt
@@ -1,7 +1,7 @@
 message(STATUS "Configuring and building CAS")
 ginkgo_load_git_package(CudaArchitectureSelector
     "https://github.com/ginkgo-project/CudaArchitectureSelector.git"
-    "cba91d58e9bb847e572fedf8cc18f701ae035410")
+    "50277af53b8f02b49261ab62fb5bdcbab5c2e88c")
 add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/src
     ${CMAKE_CURRENT_BINARY_DIR}/build EXCLUDE_FROM_ALL)
 set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" PARENT_SCOPE)

From fbea27926194eace660a9922c9b1a7de48b5453d Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Mon, 1 Feb 2021 13:24:09 +0100
Subject: [PATCH 0314/1008] Review update.

+ Change GINKGO_AMDGPU_FLAGS to GINKGO_AMD_ARCH_FLAGS
+ Add the flag to the proper information places (get_info.cmake,
  GinkgoConfig.cmake)
+ Shorten the HIP get_info.cmake
+ Update the CAS to properly detect SM architectures with CUDA >= 11.0

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
---
 .github/workflows/windows-build.yml           |  2 +-
 cmake/GinkgoConfig.cmake.in                   |  1 +
 cmake/get_info.cmake                          | 16 ++++----
 hip/CMakeLists.txt                            |  6 +--
 hip/get_info.cmake                            | 37 ++++++++-----------
 .../CudaArchitectureSelector/CMakeLists.txt   |  2 +-
 6 files changed, 30 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml
index 25d9987b38a..ccd0d978eeb 100644
--- a/.github/workflows/windows-build.yml
+++ b/.github/workflows/windows-build.yml
@@ -30,7 +30,7 @@ jobs:
         $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
         mkdir build
         cd build
-        cmake -DCMAKE_CXX_FLAGS=/bigobj -DGINKGO_BUILD_CUDA=ON -DGINKGO_BUILD_OMP=OFF ..
+        cmake -DCMAKE_CXX_FLAGS=/bigobj -DGINKGO_BUILD_CUDA=ON -DGINKGO_BUILD_OMP=OFF -DGINKGO_CUDA_ARCHITECTURES=60 ..
         cmake --build . -j4 --config Release
 
   windows_ref:
diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 85d15d16722..961949822e8 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -72,6 +72,7 @@ set(GINKGO_HIP_CLANG_COMPILER_FLAGS @GINKGO_HIP_CLANG_COMPILER_FLAGS@)
 set(GINKGO_HIP_PLATFORM @GINKGO_HIP_PLATFORM@)
 set(GINKGO_HIP_AMDGPU @GINKGO_HIP_AMDGPU@)
 set(GINKGO_HIP_VERSION @GINKGO_HIP_VERSION@)
+set(GINKGO_AMD_ARCH_FLAGS @GINKGO_AMD_ARCH_FLAGS@)
 
 set(GINKGO_DPCPP_VERSION @GINKGO_DPCPP_VERSION@)
 set(GINKGO_DPCPP_FLAGS @GINKGO_DPCPP_FLAGS@)
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index 3d84e3aca94..c44367afe6d 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -87,9 +87,9 @@ function(ginkgo_print_env_variable log_type var_name)
 endfunction()
 
 
-macro(ginkgo_print_foreach_variable variables)
-    foreach(var ${variables})
-        ginkgo_print_variable(${${log_type}} ${var} )
+macro(ginkgo_print_foreach_variable log_type)
+    foreach(var ${ARGN})
+        ginkgo_print_variable(${log_type} ${var} )
     endforeach()
 endmacro()
 
@@ -109,7 +109,7 @@ ENDIF()
 set(log_types "detailed_log;minimal_log")
 foreach(log_type ${log_types})
     ginkgo_print_module_footer(${${log_type}} "Ginkgo configuration:")
-    ginkgo_print_foreach_variable(
+    ginkgo_print_foreach_variable(${${log_type}}
         "CMAKE_BUILD_TYPE;BUILD_SHARED_LIBS;CMAKE_INSTALL_PREFIX"
         "PROJECT_SOURCE_DIR;PROJECT_BINARY_DIR")
     string(SUBSTRING
@@ -126,15 +126,15 @@ foreach(log_type ${log_types})
     FILE(APPEND ${${log_type}} "${print_string}")
     ginkgo_print_module_footer(${${log_type}} "User configuration:")
     ginkgo_print_module_footer(${${log_type}} "  Enabled modules:")
-    ginkgo_print_foreach_variable(
+    ginkgo_print_foreach_variable(${${log_type}}
         "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP")
     ginkgo_print_module_footer(${${log_type}} "  Tests, benchmarks and examples:")
-    ginkgo_print_foreach_variable(
+    ginkgo_print_foreach_variable(${${log_type}}
         "GINKGO_BUILD_TESTS;GINKGO_BUILD_EXAMPLES;GINKGO_EXTLIB_EXAMPLE;GINKGO_BUILD_BENCHMARKS;GINKGO_BENCHMARK_ENABLE_TUNING")
     ginkgo_print_module_footer(${${log_type}} "  Documentation:")
-    ginkgo_print_foreach_variable("GINKGO_BUILD_DOC;GINKGO_VERBOSE_LEVEL")
+    ginkgo_print_foreach_variable(${${log_type}} "GINKGO_BUILD_DOC;GINKGO_VERBOSE_LEVEL")
     ginkgo_print_module_footer(${${log_type}} "  Developer helpers:")
-    ginkgo_print_foreach_variable(
+    ginkgo_print_foreach_variable(${${log_type}}
         "GINKGO_DEVEL_TOOLS;GINKGO_WITH_CLANG_TIDY;GINKGO_WITH_IWYU"
         "GINKGO_CHECK_CIRCULAR_DEPS;GINKGO_CHECK_PATH")
     ginkgo_print_module_footer(${${log_type}} "")
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 7d7ce4d3ab2..089e15250a9 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -245,14 +245,14 @@ endif()
 # Thus, we need to pass the flags to `hip_add_library` itself
 if(GINKGO_HIP_AMDGPU)
     foreach(target ${GINKGO_HIP_AMDGPU})
-        list(APPEND GINKGO_AMDGPU_FLAGS --amdgpu-target=${target})
+        list(APPEND GINKGO_AMD_ARCH_FLAGS --amdgpu-target=${target})
     endforeach()
 endif()
 
 set(GINKGO_HIPCC_OPTIONS ${GINKGO_HIP_COMPILER_FLAGS})
 set(GINKGO_HIP_NVCC_OPTIONS ${GINKGO_HIP_NVCC_COMPILER_FLAGS} ${GINKGO_HIP_NVCC_ARCH} ${GINKGO_HIP_NVCC_ADDITIONAL_FLAGS})
-set(GINKGO_HIP_HCC_OPTIONS ${GINKGO_HIP_HCC_COMPILER_FLAGS} ${GINKGO_AMDGPU_FLAGS})
-set(GINKGO_HIP_CLANG_OPTIONS ${GINKGO_HIP_CLANG_COMPILER_FLAGS} ${GINKGO_AMDGPU_FLAGS})
+set(GINKGO_HIP_HCC_OPTIONS ${GINKGO_HIP_HCC_COMPILER_FLAGS} ${GINKGO_AMD_ARCH_FLAGS})
+set(GINKGO_HIP_CLANG_OPTIONS ${GINKGO_HIP_CLANG_COMPILER_FLAGS} ${GINKGO_AMD_ARCH_FLAGS})
 
 set_source_files_properties(${GINKGO_HIP_SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
 if (GINKGO_HIP_VERSION VERSION_GREATER_EQUAL "3.5")
diff --git a/hip/get_info.cmake b/hip/get_info.cmake
index 20fe5056e90..f93a8acac79 100644
--- a/hip/get_info.cmake
+++ b/hip/get_info.cmake
@@ -1,21 +1,16 @@
-set(log_types "detailed_log")
-foreach(log_type ${log_types})
-    ginkgo_print_module_header(${${log_type}} "HIP")
-    set(print_var "GINKGO_HIPCONFIG_PATH;GINKGO_HIP_AMDGPU;GINKGO_HIP_HCC_COMPILER_FLAGS;GINKGO_HIP_NVCC_COMPILER_FLAGS;GINKGO_HIP_THRUST_PATH"
-        )
-    foreach(var ${print_var})
-        ginkgo_print_variable(${${log_type}} ${var} )
-    endforeach()
-endforeach()
-foreach(log_type ${log_types})
-    ginkgo_print_module_footer(${${log_type}} "HIP variables:")
-    set(print_var "HIP_VERSION;HIP_COMPILER;HIP_PATH;ROCM_PATH;HIP_PLATFORM;HIP_ROOT_DIR;HCC_PATH;HIP_RUNTIME;HIPBLAS_PATH;HIPSPARSE_PATH;HIPRAND_PATH;ROCRAND_PATH;HIP_CLANG_INCLUDE_PATH;HIP_CLANG_PATH;HIP_HIPCC_EXECUTABLE;HIP_HIPCONFIG_EXECUTABLE;HIP_HOST_COMPILATION_CPP"
-        )
-    foreach(var ${print_var})
-        ginkgo_print_variable(${${log_type}} ${var} )
-    endforeach()
-    ginkgo_print_flags(${detailed_log} "HIP_HCC_FLAGS")
-    ginkgo_print_flags(${detailed_log} "HIP_HIPCC_FLAGS")
-    ginkgo_print_flags(${detailed_log} "HIP_NVCC_FLAGS")
-    ginkgo_print_module_footer(${detailed_log} "")
-endforeach()
+ginkgo_print_module_header(${detailed_log} "HIP")
+ginkgo_print_foreach_variable(${detailed_log}
+    "GINKGO_HIPCONFIG_PATH;GINKGO_HIP_AMDGPU"
+    "GINKGO_HIP_HCC_COMPILER_FLAGS;GINKGO_HIP_NVCC_COMPILER_FLAGS"
+    "GINKGO_HIP_THRUST_PATH;GINKGO_AMD_ARCH_FLAGS")
+ginkgo_print_module_footer(${detailed_log} "HIP variables:")
+ginkgo_print_foreach_variable(${detailed_log}
+    "HIP_VERSION;HIP_COMPILER;HIP_PATH;ROCM_PATH"
+    "HIP_PLATFORM;HIP_ROOT_DIR;HCC_PATH;HIP_RUNTIME;HIPBLAS_PATH;HIPSPARSE_PATH"
+    "HIPRAND_PATH;ROCRAND_PATH;HIP_CLANG_INCLUDE_PATH;HIP_CLANG_PATH"
+    "HIP_HIPCC_EXECUTABLE;HIP_HIPCONFIG_EXECUTABLE;HIP_HOST_COMPILATION_CPP")
+ginkgo_print_flags(${detailed_log} "HIP_HCC_FLAGS")
+ginkgo_print_flags(${detailed_log} "HIP_HIPCC_FLAGS")
+ginkgo_print_flags(${detailed_log} "HIP_NVCC_FLAGS")
+ginkgo_print_flags(${detailed_log} "HIP_CLANG_FLAGS")
+ginkgo_print_module_footer(${detailed_log} "")
diff --git a/third_party/CudaArchitectureSelector/CMakeLists.txt b/third_party/CudaArchitectureSelector/CMakeLists.txt
index ee951455644..5530b7c691a 100644
--- a/third_party/CudaArchitectureSelector/CMakeLists.txt
+++ b/third_party/CudaArchitectureSelector/CMakeLists.txt
@@ -1,7 +1,7 @@
 message(STATUS "Configuring and building CAS")
 ginkgo_load_git_package(CudaArchitectureSelector
     "https://github.com/ginkgo-project/CudaArchitectureSelector.git"
-    "50277af53b8f02b49261ab62fb5bdcbab5c2e88c")
+    "97022db486e25ede6d7cbf683f4ac8d48694222a")
 add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/src
     ${CMAKE_CURRENT_BINARY_DIR}/build EXCLUDE_FROM_ALL)
 set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" PARENT_SCOPE)

From 2274266173e47bda295cf77d10dea25289262d0e Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 3 Feb 2021 10:58:50 +0100
Subject: [PATCH 0315/1008] Fix hwloc version detection issue.

---
 cmake/Modules/FindHWLOC.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/Modules/FindHWLOC.cmake b/cmake/Modules/FindHWLOC.cmake
index 6d3686df07a..e0d146bafe4 100644
--- a/cmake/Modules/FindHWLOC.cmake
+++ b/cmake/Modules/FindHWLOC.cmake
@@ -106,7 +106,7 @@ if(HWLOC_FOUND AND HWLOC_LIBRARIES)
     set(CMAKE_REQUIRED_LIBRARIES)
 
     string(SUBSTRING "${HWLOC_VERSION}" 0 3 HWLOC_VERSION)
-    if(HWLOC_VERSION LESS_EQUAL HWLOC_FIND_VERSION)
+    if(HWLOC_VERSION VERSION_LESS HWLOC_FIND_VERSION)
         message(STATUS "Required version ${HWLOC_FIND_VERSION}, but found version ${HWLOC_VERSION}")
         set(HWLOC_FOUND 0)
         unset(HWLOC_LIBRARIES)

From 92d67ef8f8071a24b09addd31586a01bfc2fe5bf Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 3 Feb 2021 16:57:54 +0100
Subject: [PATCH 0316/1008] fix spurious rebuilds when running CMake

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 third_party/hwloc/CMakeLists.txt | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/third_party/hwloc/CMakeLists.txt b/third_party/hwloc/CMakeLists.txt
index 89f430ad18e..9bf34b460a7 100644
--- a/third_party/hwloc/CMakeLists.txt
+++ b/third_party/hwloc/CMakeLists.txt
@@ -29,9 +29,10 @@ else()
     set(HWLOC_DIR "${TPL_HWLOC_PATH}" CACHE FILEPATH "The path to HWLOC")
     file(MAKE_DIRECTORY ${TPL_HWLOC_PATH}/lib/)
     file(GLOB HWLOC_LIBS "${TPL_HWLOC_PATH}/build/hwloc/.libs/libhwloc*")
-    execute_process(COMMAND cp "${TPL_HWLOC_PATH}/build/include/hwloc/autogen/config.h" "${TPL_HWLOC_PATH}/src/include/hwloc/autogen/")
+    configure_file("${TPL_HWLOC_PATH}/build/include/hwloc/autogen/config.h" "${TPL_HWLOC_PATH}/src/include/hwloc/autogen/config.h" COPYONLY)
     foreach(lib ${HWLOC_LIBS})
-        execute_process(COMMAND cp "${lib}" "${TPL_HWLOC_PATH}/lib/")
+        get_filename_component(lib_name ${lib} NAME)
+        configure_file("${lib}" "${TPL_HWLOC_PATH}/lib/${lib_name}" COPYONLY)
     endforeach()
     set(HWLOC_LIBRARIES "${TPL_HWLOC_PATH}/lib/libhwloc.so" CACHE FILEPATH "The path to HWLOC library libhwloc.so" FORCE)
     set(HWLOC_INCLUDE_DIRS "${TPL_HWLOC_PATH}/src/include" CACHE PATH "The directory containing the hwloc header, hwloc.h" FORCE)

From 6e462ec92a8352d18d59007156d10a418230df46 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 11 Feb 2021 13:49:42 +0100
Subject: [PATCH 0317/1008] bump CUDA generic support to 10.2 for benchmark

---
 benchmark/utils/cuda_linops.hpp |  8 ++++----
 benchmark/utils/formats.hpp     | 12 ++++++------
 cuda/base/types.hpp             |  8 ++++----
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/benchmark/utils/cuda_linops.hpp b/benchmark/utils/cuda_linops.hpp
index 30866aeb285..61dbfc23396 100644
--- a/benchmark/utils/cuda_linops.hpp
+++ b/benchmark/utils/cuda_linops.hpp
@@ -482,7 +482,7 @@ class CuspHybrid
 
 #if defined(CUDA_VERSION) &&  \
     (CUDA_VERSION >= 11000 || \
-     ((CUDA_VERSION >= 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
+     ((CUDA_VERSION >= 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
 
 
 template <typename ValueType>
@@ -678,7 +678,7 @@ class CuspGenericCoo
 
 
 #endif  // defined(CUDA_VERSION) && (CUDA_VERSION >= 11000 || ((CUDA_VERSION >=
-        // 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
+        // 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
 
 
 }  // namespace detail
@@ -695,7 +695,7 @@ using cusp_csrmm = detail::CuspCsrmm<etype>;
 
 #if defined(CUDA_VERSION) &&  \
     (CUDA_VERSION >= 11000 || \
-     ((CUDA_VERSION >= 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
+     ((CUDA_VERSION >= 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
 
 
 using cusp_gcsr = detail::CuspGenericCsr<etype>;
@@ -705,7 +705,7 @@ using cusp_gcoo = detail::CuspGenericCoo<etype>;
 
 
 #endif  // defined(CUDA_VERSION) && (CUDA_VERSION >= 11000 || ((CUDA_VERSION >=
-        // 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
+        // 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
 
 
 #if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
diff --git a/benchmark/utils/formats.hpp b/benchmark/utils/formats.hpp
index 095b44da218..d25be4657ec 100644
--- a/benchmark/utils/formats.hpp
+++ b/benchmark/utils/formats.hpp
@@ -69,10 +69,10 @@ std::string available_format =
 #endif  // defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
 #if defined(CUDA_VERSION) &&  \
     (CUDA_VERSION >= 11000 || \
-     ((CUDA_VERSION >= 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
+     ((CUDA_VERSION >= 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
     ", cusp_gcsr, cusp_gcsr2, cusp_gcoo"
 #endif  // defined(CUDA_VERSION) && (CUDA_VERSION >= 11000 || ((CUDA_VERSION >=
-        // 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
+        // 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
 #endif  // HAS_CUDA
 #ifdef HAS_HIP
     ", hipsp_csr, hipsp_csrmm, hipsp_coo, hipsp_ell, hipsp_hybrid"
@@ -115,7 +115,7 @@ std::string format_description =
     "cusp_csrex: benchmark CuSPARSE with the cusparseXcsrmvEx function."
 #if defined(CUDA_VERSION) &&  \
     (CUDA_VERSION >= 11000 || \
-     ((CUDA_VERSION >= 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
+     ((CUDA_VERSION >= 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
     "\n"
     "cusp_gcsr: benchmark CuSPARSE with the generic csr with default "
     "algorithm.\n"
@@ -124,7 +124,7 @@ std::string format_description =
     "cusp_gcoo: benchmark CuSPARSE with the generic coo with default "
     "algorithm.\n"
 #endif  // defined(CUDA_VERSION) && (CUDA_VERSION >= 11000 || ((CUDA_VERSION >=
-        // 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
+        // 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
 #endif  // HAS_CUDA
 #ifdef HAS_HIP
     "\n"
@@ -220,12 +220,12 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
         {"cusp_csrex", read_matrix_from_data<cusp_csrex>},
 #if defined(CUDA_VERSION) &&  \
     (CUDA_VERSION >= 11000 || \
-     ((CUDA_VERSION >= 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
+     ((CUDA_VERSION >= 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
         {"cusp_gcsr", read_matrix_from_data<cusp_gcsr>},
         {"cusp_gcsr2", read_matrix_from_data<cusp_gcsr2>},
         {"cusp_gcoo", read_matrix_from_data<cusp_gcoo>},
 #endif  // defined(CUDA_VERSION) && (CUDA_VERSION >= 11000 || ((CUDA_VERSION >=
-        // 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
+        // 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
 #endif  // HAS_CUDA
 #ifdef HAS_HIP
         {"hipsp_csr", read_matrix_from_data<hipsp_csr>},
diff --git a/cuda/base/types.hpp b/cuda/base/types.hpp
index ab6fa968f51..54309b3ecaa 100644
--- a/cuda/base/types.hpp
+++ b/cuda/base/types.hpp
@@ -200,7 +200,7 @@ constexpr cudaDataType_t cuda_data_type_impl<uint8>()
 
 #if defined(CUDA_VERSION) &&  \
     (CUDA_VERSION >= 11000 || \
-     ((CUDA_VERSION >= 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
+     ((CUDA_VERSION >= 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
 
 
 template <typename T>
@@ -223,7 +223,7 @@ constexpr cusparseIndexType_t cusparse_index_type_impl<int64>()
 
 
 #endif  // defined(CUDA_VERSION) && (CUDA_VERSION >= 11000 || ((CUDA_VERSION >=
-        // 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
+        // 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
 
 
 }  // namespace detail
@@ -246,7 +246,7 @@ constexpr cudaDataType_t cuda_data_type()
 
 #if defined(CUDA_VERSION) &&  \
     (CUDA_VERSION >= 11000 || \
-     ((CUDA_VERSION >= 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
+     ((CUDA_VERSION >= 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
 
 
 /**
@@ -265,7 +265,7 @@ constexpr cusparseIndexType_t cusparse_index_type()
 
 
 #endif  // defined(CUDA_VERSION) && (CUDA_VERSION >= 11000 || ((CUDA_VERSION >=
-        // 10010) && !(defined(_WIN32) || defined(__CYGWIN__))))
+        // 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
 
 
 /**

From 26b4dade0dc873177866c99455417f9f315e99df Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Sat, 6 Feb 2021 16:03:04 +0100
Subject: [PATCH 0318/1008] Add an implicit resnorm stopping criterion

+ Saves on computing a 2 norm in each iteration in the
  stopping criterion.
+ Also makes the results comparable with other libraries
  that use this type of implicit residual for convergence
  checks.
---
 core/device_hooks/common_kernels.inc.cpp   | 13 ++++
 core/stop/residual_norm.cpp                | 39 +++++++++++
 core/stop/residual_norm_kernels.hpp        | 56 +++++++++++++++
 cuda/stop/residual_norm_kernels.cu         | 79 +++++++++++++++++++++
 dpcpp/stop/residual_norm_kernels.dp.cpp    | 28 ++++++++
 hip/stop/residual_norm_kernels.hip.cpp     | 80 ++++++++++++++++++++++
 include/ginkgo/core/stop/criterion.hpp     |  3 +
 include/ginkgo/core/stop/residual_norm.hpp | 78 +++++++++++++++++++++
 omp/stop/residual_norm_kernels.cpp         | 48 +++++++++++++
 reference/stop/residual_norm_kernels.cpp   | 43 ++++++++++++
 10 files changed, 467 insertions(+)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 6fd03ce9877..053dfde5029 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -1189,6 +1189,19 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
 
 
 }  // namespace residual_norm
+
+
+namespace implicit_residual_norm {
+
+
+template <typename ValueType>
+GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
+    GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
+
+
+}  // namespace implicit_residual_norm
 }  // namespace GKO_HOOK_MODULE
 }  // namespace kernels
 }  // namespace gko
diff --git a/core/stop/residual_norm.cpp b/core/stop/residual_norm.cpp
index d6adfdf31a0..c4efb1bf57e 100644
--- a/core/stop/residual_norm.cpp
+++ b/core/stop/residual_norm.cpp
@@ -49,6 +49,16 @@ GKO_REGISTER_OPERATION(fill_array, components::fill_array);
 }  // namespace residual_norm
 
 
+namespace implicit_residual_norm {
+
+
+GKO_REGISTER_OPERATION(implicit_residual_norm,
+                       implicit_residual_norm::implicit_residual_norm);
+
+
+}  // namespace implicit_residual_norm
+
+
 template <typename ValueType>
 bool ResidualNorm<ValueType>::check_impl(uint8 stoppingId, bool setFinalized,
                                          Array<stopping_status> *stop_status,
@@ -79,6 +89,30 @@ bool ResidualNorm<ValueType>::check_impl(uint8 stoppingId, bool setFinalized,
     return all_converged;
 }
 
+
+template <typename ValueType>
+bool ImplicitResidualNormReduction<ValueType>::check_impl(
+    uint8 stoppingId, bool setFinalized, Array<stopping_status> *stop_status,
+    bool *one_changed, const Criterion::Updater &updater)
+{
+    const NormVector *dense_tau;
+    if (updater.implicit_sq_residual_norm_ != nullptr) {
+        dense_tau = as<NormVector>(updater.implicit_sq_residual_norm_);
+    } else {
+        GKO_NOT_SUPPORTED(nullptr);
+    }
+    bool all_converged = true;
+
+    this->get_executor()->run(
+        implicit_residual_norm::make_implicit_residual_norm(
+            dense_tau, this->starting_tau_.get(), this->reduction_factor_,
+            stoppingId, setFinalized, stop_status, &device_storage_,
+            &all_converged, one_changed));
+
+    return all_converged;
+}
+
+
 template <typename ValueType>
 void AbsoluteResidualNorm<ValueType>::initialize_starting_tau()
 {
@@ -92,6 +126,11 @@ void AbsoluteResidualNorm<ValueType>::initialize_starting_tau()
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_RESIDUAL_NORM);
 
 
+#define GKO_DECLARE_IMPLICIT_RESIDUAL_NORM(_type) \
+    class ImplicitResidualNormReduction<_type>
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IMPLICIT_RESIDUAL_NORM);
+
+
 #define GKO_DECLARE_ABSOLUTE_RESIDUAL_NORM(_type) \
     class AbsoluteResidualNorm<_type>
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_ABSOLUTE_RESIDUAL_NORM);
diff --git a/core/stop/residual_norm_kernels.hpp b/core/stop/residual_norm_kernels.hpp
index 9acbf9bd3fc..8b8ff732f51 100644
--- a/core/stop/residual_norm_kernels.hpp
+++ b/core/stop/residual_norm_kernels.hpp
@@ -63,12 +63,39 @@ namespace residual_norm {
 }  // namespace residual_norm
 
 
+namespace implicit_residual_norm {
+
+
+#define GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL(_type)                       \
+    void implicit_residual_norm(                                               \
+        std::shared_ptr<const DefaultExecutor> exec,                           \
+        const matrix::Dense<_type> *tau, const matrix::Dense<_type> *orig_tau, \
+        _type rel_residual_goal, uint8 stoppingId, bool setFinalized,          \
+        Array<stopping_status> *stop_status, Array<bool> *device_storage,      \
+        bool *all_converged, bool *one_changed)
+
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES2 \
+    template <typename ValueType>     \
+    GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL(ValueType)
+
+
+}  // namespace implicit_residual_norm
+
+
 namespace omp {
 namespace residual_norm {
 
 GKO_DECLARE_ALL_AS_TEMPLATES;
 
 }  // namespace residual_norm
+
+
+namespace implicit_residual_norm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES2;
+
+}  // namespace implicit_residual_norm
 }  // namespace omp
 
 
@@ -78,6 +105,13 @@ namespace residual_norm {
 GKO_DECLARE_ALL_AS_TEMPLATES;
 
 }  // namespace residual_norm
+
+
+namespace implicit_residual_norm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES2;
+
+}  // namespace implicit_residual_norm
 }  // namespace cuda
 
 
@@ -87,6 +121,13 @@ namespace residual_norm {
 GKO_DECLARE_ALL_AS_TEMPLATES;
 
 }  // namespace residual_norm
+
+
+namespace implicit_residual_norm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES2;
+
+}  // namespace implicit_residual_norm
 }  // namespace reference
 
 
@@ -96,6 +137,13 @@ namespace residual_norm {
 GKO_DECLARE_ALL_AS_TEMPLATES;
 
 }  // namespace residual_norm
+
+
+namespace implicit_residual_norm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES2;
+
+}  // namespace implicit_residual_norm
 }  // namespace hip
 
 
@@ -105,10 +153,18 @@ namespace residual_norm {
 GKO_DECLARE_ALL_AS_TEMPLATES;
 
 }  // namespace residual_norm
+
+
+namespace implicit_residual_norm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES2;
+
+}  // namespace implicit_residual_norm
 }  // namespace dpcpp
 
 
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
+#undef GKO_DECLARE_ALL_AS_TEMPLATES2
 
 }  // namespace kernels
 }  // namespace gko
diff --git a/cuda/stop/residual_norm_kernels.cu b/cuda/stop/residual_norm_kernels.cu
index dbb17c515fb..e79800c874d 100644
--- a/cuda/stop/residual_norm_kernels.cu
+++ b/cuda/stop/residual_norm_kernels.cu
@@ -121,6 +121,85 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
 
 
 }  // namespace residual_norm
+
+
+/**
+ * @brief The Residual norm stopping criterion namespace.
+ * @ref resnorm
+ * @ingroup resnorm
+ */
+namespace implicit_residual_norm {
+
+
+constexpr int default_block_size = 512;
+
+
+template <typename ValueType>
+__global__
+    __launch_bounds__(default_block_size) void implicit_residual_norm_kernel(
+        size_type num_cols, ValueType rel_residual_goal,
+        const ValueType *__restrict__ tau,
+        const ValueType *__restrict__ orig_tau, uint8 stoppingId,
+        bool setFinalized, stopping_status *__restrict__ stop_status,
+        bool *__restrict__ device_storage)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    if (tidx < num_cols) {
+        if (sqrt(tau[tidx]) < rel_residual_goal * orig_tau[tidx]) {
+            stop_status[tidx].converge(stoppingId, setFinalized);
+            device_storage[1] = true;
+        }
+        // because only false is written to all_converged, write conflicts
+        // should not cause any problem
+        else if (!stop_status[tidx].has_stopped()) {
+            device_storage[0] = false;
+        }
+    }
+}
+
+
+__global__ __launch_bounds__(1) void init_kernel(
+    bool *__restrict__ device_storage)
+{
+    device_storage[0] = true;
+    device_storage[1] = false;
+}
+
+
+template <typename ValueType>
+void implicit_residual_norm(std::shared_ptr<const CudaExecutor> exec,
+                            const matrix::Dense<ValueType> *tau,
+                            const matrix::Dense<ValueType> *orig_tau,
+                            ValueType rel_residual_goal, uint8 stoppingId,
+                            bool setFinalized,
+                            Array<stopping_status> *stop_status,
+                            Array<bool> *device_storage, bool *all_converged,
+                            bool *one_changed)
+{
+    static_assert(is_complex_s<ValueType>::value == false,
+                  "ValueType must not be complex in this function!");
+    init_kernel<<<1, 1>>>(as_cuda_type(device_storage->get_data()));
+
+    const dim3 block_size(default_block_size, 1, 1);
+    const dim3 grid_size(ceildiv(tau->get_size()[1], block_size.x), 1, 1);
+
+    implicit_residual_norm_kernel<<<grid_size, block_size>>>(
+        tau->get_size()[1], rel_residual_goal,
+        as_cuda_type(tau->get_const_values()),
+        as_cuda_type(orig_tau->get_const_values()), stoppingId, setFinalized,
+        as_cuda_type(stop_status->get_data()),
+        as_cuda_type(device_storage->get_data()));
+
+    /* Represents all_converged, one_changed */
+    *all_converged = exec->copy_val_to_host(device_storage->get_const_data());
+    *one_changed = exec->copy_val_to_host(device_storage->get_const_data() + 1);
+}
+
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
+    GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
+
+
+}  // namespace implicit_residual_norm
 }  // namespace cuda
 }  // namespace kernels
 }  // namespace gko
diff --git a/dpcpp/stop/residual_norm_kernels.dp.cpp b/dpcpp/stop/residual_norm_kernels.dp.cpp
index 0e1c6bf9642..7a498d0be47 100644
--- a/dpcpp/stop/residual_norm_kernels.dp.cpp
+++ b/dpcpp/stop/residual_norm_kernels.dp.cpp
@@ -69,6 +69,34 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
 
 
 }  // namespace residual_norm
+
+
+/**
+ * @brief The Residual norm stopping criterion namespace.
+ * @ref resnorm
+ * @ingroup resnorm
+ */
+namespace implicit_residual_norm {
+
+
+constexpr int default_group_size = 512;
+
+
+template <typename ValueType>
+void implicit_residual_norm(std::shared_ptr<const DpcppExecutor> exec,
+                            const matrix::Dense<ValueType> *tau,
+                            const matrix::Dense<ValueType> *orig_tau,
+                            ValueType rel_residual_goal, uint8 stoppingId,
+                            bool setFinalized,
+                            Array<stopping_status> *stop_status,
+                            Array<bool> *device_storage, bool *all_converged,
+                            bool *one_changed) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
+    GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
+
+
+}  // namespace implicit_residual_norm
 }  // namespace dpcpp
 }  // namespace kernels
 }  // namespace gko
diff --git a/hip/stop/residual_norm_kernels.hip.cpp b/hip/stop/residual_norm_kernels.hip.cpp
index eb0060801dd..32920f23600 100644
--- a/hip/stop/residual_norm_kernels.hip.cpp
+++ b/hip/stop/residual_norm_kernels.hip.cpp
@@ -125,6 +125,86 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
 
 
 }  // namespace residual_norm
+
+
+/**
+ * @brief The Residual norm stopping criterion namespace.
+ * @ref resnorm
+ * @ingroup resnorm
+ */
+namespace implicit_residual_norm {
+
+
+constexpr int default_block_size = 512;
+
+
+template <typename ValueType>
+__global__
+    __launch_bounds__(default_block_size) void implicit_residual_norm_kernel(
+        size_type num_cols, ValueType rel_residual_goal,
+        const ValueType *__restrict__ tau,
+        const ValueType *__restrict__ orig_tau, uint8 stoppingId,
+        bool setFinalized, stopping_status *__restrict__ stop_status,
+        bool *__restrict__ device_storage)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    if (tidx < num_cols) {
+        if (sqrt(tau[tidx]) < rel_residual_goal * orig_tau[tidx]) {
+            stop_status[tidx].converge(stoppingId, setFinalized);
+            device_storage[1] = true;
+        }
+        // because only false is written to all_converged, write conflicts
+        // should not cause any problem
+        else if (!stop_status[tidx].has_stopped()) {
+            device_storage[0] = false;
+        }
+    }
+}
+
+
+__global__ __launch_bounds__(1) void init_kernel(
+    bool *__restrict__ device_storage)
+{
+    device_storage[0] = true;
+    device_storage[1] = false;
+}
+
+
+template <typename ValueType>
+void implicit_residual_norm(std::shared_ptr<const HipExecutor> exec,
+                            const matrix::Dense<ValueType> *tau,
+                            const matrix::Dense<ValueType> *orig_tau,
+                            ValueType rel_residual_goal, uint8 stoppingId,
+                            bool setFinalized,
+                            Array<stopping_status> *stop_status,
+                            Array<bool> *device_storage, bool *all_converged,
+                            bool *one_changed)
+{
+    static_assert(is_complex_s<ValueType>::value == false,
+                  "ValueType must not be complex in this function!");
+    hipLaunchKernelGGL((init_kernel), dim3(1), dim3(1), 0, 0,
+                       as_hip_type(device_storage->get_data()));
+
+    const dim3 block_size(default_block_size, 1, 1);
+    const dim3 grid_size(ceildiv(tau->get_size()[1], block_size.x), 1, 1);
+
+    hipLaunchKernelGGL((implicit_residual_norm_kernel), dim3(grid_size),
+                       dim3(block_size), 0, 0, tau->get_size()[1],
+                       rel_residual_goal, as_hip_type(tau->get_const_values()),
+                       as_hip_type(orig_tau->get_const_values()), stoppingId,
+                       setFinalized, as_hip_type(stop_status->get_data()),
+                       as_hip_type(device_storage->get_data()));
+
+    /* Represents all_converged, one_changed */
+    *all_converged = exec->copy_val_to_host(device_storage->get_const_data());
+    *one_changed = exec->copy_val_to_host(device_storage->get_const_data() + 1);
+}
+
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
+    GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
+
+
+}  // namespace implicit_residual_norm
 }  // namespace hip
 }  // namespace kernels
 }  // namespace gko
diff --git a/include/ginkgo/core/stop/criterion.hpp b/include/ginkgo/core/stop/criterion.hpp
index 22cc74da9fd..55b3c3bebbd 100644
--- a/include/ginkgo/core/stop/criterion.hpp
+++ b/include/ginkgo/core/stop/criterion.hpp
@@ -70,6 +70,7 @@ class Criterion : public EnableAbstractPolymorphicObject<Criterion> {
      * does not build an object. This allows calling a Criterion's check in the
      * form of: stop_criterion->update() .num_iterations(num_iterations)
      *   .residual_norm(residual_norm)
+     *   .implicit_sq_residual_norm(implicit_sq_residual_norm)
      *   .residual(residual)
      *   .solution(solution)
      *   .check(converged);
@@ -117,6 +118,8 @@ class Criterion : public EnableAbstractPolymorphicObject<Criterion> {
         GKO_UPDATER_REGISTER_PARAMETER(size_type, num_iterations);
         GKO_UPDATER_REGISTER_PARAMETER(const LinOp *, residual);
         GKO_UPDATER_REGISTER_PARAMETER(const LinOp *, residual_norm);
+        GKO_UPDATER_REGISTER_PARAMETER(const LinOp *,
+                                       implicit_sq_residual_norm);
         GKO_UPDATER_REGISTER_PARAMETER(const LinOp *, solution);
 
 #undef GKO_UPDATER_REGISTER_PARAMETER
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index 6abc109d126..487d7e323c9 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -226,6 +226,84 @@ class RelativeResidualNorm : public ResidualNorm<ValueType> {
 };
 
 
+/**
+ * The ImplicitResidualNormReduction class is a stopping criterion which
+ * stops the iteration process when the residual norm is below a certain
+ * threshold recurrent to the norm of the right-hand side, i.e. when
+ * norm(residual) / norm(right_hand_side) < threshold.
+ * For better performance, the checks are run thanks to kernels on
+ * the executor where the algorithm is executed.
+ *
+ * @note To use this stopping criterion there are some dependencies. The
+ * constructor depends on `b` in order to compute the norm of the
+ * right-hand side. If this is not correctly provided, an exception
+ * ::gko::NotSupported() is thrown.
+ *
+ * @ingroup stop
+ */
+template <typename ValueType = default_precision>
+class ImplicitResidualNormReduction : public ResidualNorm<ValueType> {
+public:
+    using ComplexVector = matrix::Dense<to_complex<ValueType>>;
+    using NormVector = matrix::Dense<remove_complex<ValueType>>;
+    using Vector = matrix::Dense<ValueType>;
+
+    GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
+    {
+        /**
+         * Recurrent residual norm goal
+         */
+        remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(
+            reduction_factor, static_cast<remove_complex<ValueType>>(1e-15));
+    };
+    GKO_ENABLE_CRITERION_FACTORY(ImplicitResidualNormReduction<ValueType>,
+                                 parameters, Factory);
+    GKO_ENABLE_BUILD_METHOD(Factory);
+
+protected:
+    bool check_impl(uint8 stoppingId, bool setFinalized,
+                    Array<stopping_status> *stop_status, bool *one_changed,
+                    const Criterion::Updater &) override;
+
+    explicit ImplicitResidualNormReduction(
+        std::shared_ptr<const gko::Executor> exec)
+        : ResidualNorm<ValueType>(exec), device_storage_{exec, 2}
+    {}
+
+    explicit ImplicitResidualNormReduction(const Factory *factory,
+                                           const CriterionArgs &args)
+        : ResidualNorm<ValueType>(factory->get_executor(),
+                                  factory->get_parameters().reduction_factor),
+          parameters_{factory->get_parameters()},
+          device_storage_{factory->get_executor(), 2}
+    {
+        if (args.b == nullptr) {
+            GKO_NOT_SUPPORTED(nullptr);
+        }
+
+        auto exec = factory->get_executor();
+
+        this->reduction_factor_ = factory->get_parameters().reduction_factor;
+        this->starting_tau_ =
+            NormVector::create(exec, dim<2>{1, args.b->get_size()[1]});
+        this->u_dense_tau_ =
+            NormVector::create_with_config_of(this->starting_tau_.get());
+        if (dynamic_cast<const ComplexVector *>(args.b.get())) {
+            auto dense_rhs = as<ComplexVector>(args.b);
+            dense_rhs->compute_norm2(this->starting_tau_.get());
+        } else {
+            auto dense_rhs = as<Vector>(args.b);
+            dense_rhs->compute_norm2(this->starting_tau_.get());
+        }
+    }
+
+private:
+    remove_complex<ValueType> reduction_factor_{};
+    /* Contains device side: all_converged and one_changed booleans */
+    Array<bool> device_storage_;
+};
+
+
 /**
  * The AbsoluteResidualNorm class is a stopping criterion which stops the
  * iteration process when the residual norm is below a certain
diff --git a/omp/stop/residual_norm_kernels.cpp b/omp/stop/residual_norm_kernels.cpp
index 630fd9b05c6..5e11030a1e4 100644
--- a/omp/stop/residual_norm_kernels.cpp
+++ b/omp/stop/residual_norm_kernels.cpp
@@ -88,6 +88,54 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
 
 
 }  // namespace residual_norm
+
+
+/**
+ * @brief The Residual norm stopping criterion namespace.
+ * @ref resnorm
+ * @ingroup resnorm
+ */
+namespace implicit_residual_norm {
+
+
+template <typename ValueType>
+void implicit_residual_norm(std::shared_ptr<const OmpExecutor> exec,
+                            const matrix::Dense<ValueType> *tau,
+                            const matrix::Dense<ValueType> *orig_tau,
+                            ValueType rel_residual_goal, uint8 stoppingId,
+                            bool setFinalized,
+                            Array<stopping_status> *stop_status,
+                            Array<bool> *device_storage, bool *all_converged,
+                            bool *one_changed)
+{
+    static_assert(is_complex_s<ValueType>::value == false,
+                  "ValueType must not be complex in this function!");
+    bool local_one_changed = false;
+#pragma omp parallel for reduction(|| : local_one_changed)
+    for (size_type i = 0; i < tau->get_size()[1]; ++i) {
+        if (sqrt(tau->at(i)) < rel_residual_goal * orig_tau->at(i)) {
+            stop_status->get_data()[i].converge(stoppingId, setFinalized);
+            local_one_changed = true;
+        }
+    }
+    *one_changed = local_one_changed;
+    // No early stopping here because one cannot use break with parallel for
+    // But it's parallel so does it matter?
+    bool local_all_converged = true;
+#pragma omp parallel for reduction(&& : local_all_converged)
+    for (size_type i = 0; i < stop_status->get_num_elems(); ++i) {
+        if (!stop_status->get_const_data()[i].has_stopped()) {
+            local_all_converged = false;
+        }
+    }
+    *all_converged = local_all_converged;
+}
+
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
+    GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
+
+
+}  // namespace implicit_residual_norm
 }  // namespace omp
 }  // namespace kernels
 }  // namespace gko
diff --git a/reference/stop/residual_norm_kernels.cpp b/reference/stop/residual_norm_kernels.cpp
index c6c4393a4eb..dff14bfc8e0 100644
--- a/reference/stop/residual_norm_kernels.cpp
+++ b/reference/stop/residual_norm_kernels.cpp
@@ -85,6 +85,49 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
 
 
 }  // namespace residual_norm
+
+
+/**
+ * @brief The Recurrent Residual norm stopping criterion.
+ * @ref resnorm
+ * @ingroup resnorm
+ */
+namespace implicit_residual_norm {
+
+
+template <typename ValueType>
+void implicit_residual_norm(std::shared_ptr<const ReferenceExecutor> exec,
+                            const matrix::Dense<ValueType> *tau,
+                            const matrix::Dense<ValueType> *orig_tau,
+                            ValueType rel_residual_goal, uint8 stoppingId,
+                            bool setFinalized,
+                            Array<stopping_status> *stop_status,
+                            Array<bool> *device_storage, bool *all_converged,
+                            bool *one_changed)
+{
+    static_assert(is_complex_s<ValueType>::value == false,
+                  "ValueType must not be complex in this function!");
+    *all_converged = true;
+    *one_changed = false;
+    for (size_type i = 0; i < tau->get_size()[1]; ++i) {
+        if (sqrt(tau->at(i)) < rel_residual_goal * orig_tau->at(i)) {
+            stop_status->get_data()[i].converge(stoppingId, setFinalized);
+            *one_changed = true;
+        }
+    }
+    for (size_type i = 0; i < stop_status->get_num_elems(); ++i) {
+        if (!stop_status->get_const_data()[i].has_stopped()) {
+            *all_converged = false;
+            break;
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
+    GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
+
+
+}  // namespace implicit_residual_norm
 }  // namespace reference
 }  // namespace kernels
 }  // namespace gko

From e84e7c2c6b1a91ba4c05ee0f66f697c0150c2e74 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Sat, 6 Feb 2021 16:06:54 +0100
Subject: [PATCH 0319/1008] Add core and kernel tests.

---
 cuda/test/stop/residual_norm_kernels.cpp      | 108 +++++++++++++++
 hip/test/stop/residual_norm_kernels.cpp       | 108 +++++++++++++++
 omp/test/stop/residual_norm_kernels.cpp       | 128 ++++++++++++++++++
 reference/test/stop/residual_norm_kernels.cpp | 128 ++++++++++++++++++
 4 files changed, 472 insertions(+)

diff --git a/cuda/test/stop/residual_norm_kernels.cpp b/cuda/test/stop/residual_norm_kernels.cpp
index f962c243d3d..088def448bd 100644
--- a/cuda/test/stop/residual_norm_kernels.cpp
+++ b/cuda/test/stop/residual_norm_kernels.cpp
@@ -259,6 +259,114 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
 }
 
 
+class ImplicitResidualNormReduction : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<>;
+
+    ImplicitResidualNormReduction()
+    {
+        ref_ = gko::ReferenceExecutor::create();
+        cuda_ = gko::CudaExecutor::create(0, ref_);
+        factory_ = gko::stop::ImplicitResidualNormReduction<>::build()
+                       .with_reduction_factor(tol)
+                       .on(cuda_);
+    }
+
+    std::unique_ptr<gko::stop::ImplicitResidualNormReduction<>::Factory>
+        factory_;
+    std::shared_ptr<const gko::CudaExecutor> cuda_;
+    std::shared_ptr<gko::ReferenceExecutor> ref_;
+};
+
+
+TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoal)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto d_res = Mtx::create(cuda_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(cuda_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(cuda_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = std::pow(tol * 1.1e+1, 2);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(cuda_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = std::pow(tol * 0.9e+1, 2);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto d_res = Mtx::create(cuda_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(cuda_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(cuda_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = std::pow(tol * 0.9e+1, 2);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(cuda_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = std::pow(tol * 0.9e+1, 2);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
 class AbsoluteResidualNorm : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
diff --git a/hip/test/stop/residual_norm_kernels.cpp b/hip/test/stop/residual_norm_kernels.cpp
index 361b6545509..3360372acb9 100644
--- a/hip/test/stop/residual_norm_kernels.cpp
+++ b/hip/test/stop/residual_norm_kernels.cpp
@@ -259,6 +259,114 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
 }
 
 
+class ImplicitResidualNormReduction : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<>;
+
+    ImplicitResidualNormReduction()
+    {
+        ref_ = gko::ReferenceExecutor::create();
+        hip_ = gko::HipExecutor::create(0, ref_);
+        factory_ = gko::stop::ImplicitResidualNormReduction<>::build()
+                       .with_reduction_factor(tol)
+                       .on(hip_);
+    }
+
+    std::unique_ptr<gko::stop::ImplicitResidualNormReduction<>::Factory>
+        factory_;
+    std::shared_ptr<const gko::HipExecutor> hip_;
+    std::shared_ptr<gko::ReferenceExecutor> ref_;
+};
+
+
+TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoal)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto d_res = Mtx::create(hip_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(hip_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(hip_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = std::pow(tol * 1.1e+1, 2);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(hip_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = std::pow(tol * 0.9e+1, 2);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto d_res = Mtx::create(hip_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(hip_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(hip_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = std::pow(tol * 0.9e+1, 2);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(hip_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = std::pow(tol * 0.9e+1, 2);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
 class AbsoluteResidualNorm : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
diff --git a/omp/test/stop/residual_norm_kernels.cpp b/omp/test/stop/residual_norm_kernels.cpp
index 4d960f7b442..2923f829fe0 100644
--- a/omp/test/stop/residual_norm_kernels.cpp
+++ b/omp/test/stop/residual_norm_kernels.cpp
@@ -245,6 +245,134 @@ TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
 }
 
 
+template <typename T>
+class ImplicitResidualNormReduction : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<T>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<T>>;
+
+    ImplicitResidualNormReduction()
+    {
+        exec_ = gko::OmpExecutor::create();
+        factory_ = gko::stop::ImplicitResidualNormReduction<T>::build()
+                       .with_reduction_factor(r<T>::value)
+                       .on(exec_);
+    }
+
+    std::unique_ptr<
+        typename gko::stop::ImplicitResidualNormReduction<T>::Factory>
+        factory_;
+    std::shared_ptr<const gko::Executor> exec_;
+};
+
+TYPED_TEST_SUITE(ImplicitResidualNormReduction, gko::test::ValueTypes);
+
+
+TYPED_TEST(ImplicitResidualNormReduction, CanCreateFactory)
+{
+    ASSERT_NE(this->factory_, nullptr);
+    ASSERT_EQ(this->factory_->get_parameters().reduction_factor,
+              r<TypeParam>::value);
+    ASSERT_EQ(this->factory_->get_executor(), this->exec_);
+}
+
+
+TYPED_TEST(ImplicitResidualNormReduction, CannotCreateCriterionWithoutB)
+{
+    ASSERT_THROW(this->factory_->generate(nullptr, nullptr, nullptr, nullptr),
+                 gko::NotSupported);
+}
+
+
+TYPED_TEST(ImplicitResidualNormReduction, CanCreateCriterionWithB)
+{
+    using Mtx = typename TestFixture::Mtx;
+    std::shared_ptr<gko::LinOp> scalar =
+        gko::initialize<Mtx>({1.0}, this->exec_);
+    auto criterion =
+        this->factory_->generate(nullptr, scalar, nullptr, nullptr);
+    ASSERT_NE(criterion, nullptr);
+}
+
+
+TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoal)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using NormVector = typename TestFixture::NormVector;
+    auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->exec_);
+    auto res_norm = gko::initialize<NormVector>({100.0}, this->exec_);
+    auto criterion =
+        this->factory_->generate(nullptr, rhs, nullptr, initial_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+    stop_status.get_data()[0].reset();
+
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(res_norm.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res_norm->at(0) = std::pow(r<TypeParam>::value * 1.1e+1, 2);
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(res_norm.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
+    ASSERT_EQ(one_changed, false);
+
+    res_norm->at(0) = std::pow(r<TypeParam>::value * 0.9e+1, 2);
+    ASSERT_TRUE(
+        criterion->update()
+            .implicit_sq_residual_norm(res_norm.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+    ASSERT_EQ(one_changed, true);
+}
+
+
+TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using NormVector = typename TestFixture::NormVector;
+    using T = TypeParam;
+    using T_nc = gko::remove_complex<TypeParam>;
+    auto res = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->exec_);
+    auto res_norm =
+        gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({I<T>{10.0, 10.0}}, this->exec_);
+    auto criterion = this->factory_->generate(nullptr, rhs, nullptr, res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(this->exec_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(res_norm.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res_norm->at(0, 0) = std::pow(r<TypeParam>::value * 0.9e+1, 2);
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(res_norm.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+    ASSERT_EQ(one_changed, true);
+
+    res_norm->at(0, 1) = std::pow(r<TypeParam>::value * 0.9e+1, 2);
+    ASSERT_TRUE(
+        criterion->update()
+            .implicit_sq_residual_norm(res_norm.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    ASSERT_EQ(stop_status.get_data()[1].has_converged(), true);
+    ASSERT_EQ(one_changed, true);
+}
+
+
 template <typename T>
 class AbsoluteResidualNorm : public ::testing::Test {
 protected:
diff --git a/reference/test/stop/residual_norm_kernels.cpp b/reference/test/stop/residual_norm_kernels.cpp
index 6a25f249186..15a2033a398 100644
--- a/reference/test/stop/residual_norm_kernels.cpp
+++ b/reference/test/stop/residual_norm_kernels.cpp
@@ -302,6 +302,134 @@ TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
 }
 
 
+template <typename T>
+class ImplicitResidualNormReduction : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<T>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<T>>;
+
+    ImplicitResidualNormReduction()
+    {
+        exec_ = gko::ReferenceExecutor::create();
+        factory_ = gko::stop::ImplicitResidualNormReduction<T>::build()
+                       .with_reduction_factor(r<T>::value)
+                       .on(exec_);
+    }
+
+    std::unique_ptr<
+        typename gko::stop::ImplicitResidualNormReduction<T>::Factory>
+        factory_;
+    std::shared_ptr<const gko::Executor> exec_;
+};
+
+TYPED_TEST_SUITE(ImplicitResidualNormReduction, gko::test::ValueTypes);
+
+
+TYPED_TEST(ImplicitResidualNormReduction, CanCreateFactory)
+{
+    ASSERT_NE(this->factory_, nullptr);
+    ASSERT_EQ(this->factory_->get_parameters().reduction_factor,
+              r<TypeParam>::value);
+    ASSERT_EQ(this->factory_->get_executor(), this->exec_);
+}
+
+
+TYPED_TEST(ImplicitResidualNormReduction, CannotCreateCriterionWithoutB)
+{
+    ASSERT_THROW(this->factory_->generate(nullptr, nullptr, nullptr, nullptr),
+                 gko::NotSupported);
+}
+
+
+TYPED_TEST(ImplicitResidualNormReduction, CanCreateCriterionWithB)
+{
+    using Mtx = typename TestFixture::Mtx;
+    std::shared_ptr<gko::LinOp> scalar =
+        gko::initialize<Mtx>({1.0}, this->exec_);
+    auto criterion =
+        this->factory_->generate(nullptr, scalar, nullptr, nullptr);
+    ASSERT_NE(criterion, nullptr);
+}
+
+
+TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoal)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using NormVector = typename TestFixture::NormVector;
+    auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->exec_);
+    auto res_norm = gko::initialize<NormVector>({100.0}, this->exec_);
+    auto criterion =
+        this->factory_->generate(nullptr, rhs, nullptr, initial_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+    stop_status.get_data()[0].reset();
+
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(res_norm.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res_norm->at(0) = std::pow(r<TypeParam>::value * 1.1e+1, 2);
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(res_norm.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
+    ASSERT_EQ(one_changed, false);
+
+    res_norm->at(0) = std::pow(r<TypeParam>::value * 0.9e+1, 2);
+    ASSERT_TRUE(
+        criterion->update()
+            .implicit_sq_residual_norm(res_norm.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+    ASSERT_EQ(one_changed, true);
+}
+
+
+TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using NormVector = typename TestFixture::NormVector;
+    using T = TypeParam;
+    using T_nc = gko::remove_complex<TypeParam>;
+    auto res = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->exec_);
+    auto res_norm =
+        gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({I<T>{10.0, 10.0}}, this->exec_);
+    auto criterion = this->factory_->generate(nullptr, rhs, nullptr, res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(this->exec_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(res_norm.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res_norm->at(0, 0) = std::pow(r<TypeParam>::value * 0.9e+1, 2);
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(res_norm.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+    ASSERT_EQ(one_changed, true);
+
+    res_norm->at(0, 1) = std::pow(r<TypeParam>::value * 0.9e+1, 2);
+    ASSERT_TRUE(
+        criterion->update()
+            .implicit_sq_residual_norm(res_norm.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    ASSERT_EQ(stop_status.get_data()[1].has_converged(), true);
+    ASSERT_EQ(one_changed, true);
+}
+
+
 template <typename T>
 class AbsoluteResidualNorm : public ::testing::Test {
 protected:

From 5af1cb8f86d2d8458e1d807c48ef17637634dd88 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Sat, 6 Feb 2021 16:08:31 +0100
Subject: [PATCH 0320/1008] Update stop criterion for solvers.

+ Add criterion tests for solvers.
---
 core/solver/bicg.cpp                       |  1 +
 core/solver/bicgstab.cpp                   |  9 ++++--
 core/solver/cg.cpp                         |  1 +
 core/solver/cgs.cpp                        |  1 +
 core/solver/fcg.cpp                        |  1 +
 core/solver/idr.cpp                        |  2 ++
 reference/test/solver/bicg_kernels.cpp     | 28 +++++++++++++++++++
 reference/test/solver/bicgstab_kernels.cpp | 32 ++++++++++++++++++++++
 reference/test/solver/cg_kernels.cpp       | 31 ++++++++++++++++++++-
 reference/test/solver/cgs_kernels.cpp      | 27 ++++++++++++++++++
 reference/test/solver/fcg_kernels.cpp      | 28 +++++++++++++++++++
 reference/test/solver/gmres_kernels.cpp    | 25 +++++++++++++++++
 12 files changed, 183 insertions(+), 3 deletions(-)

diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp
index d4fa8f96921..f283d5f16d7 100644
--- a/core/solver/bicg.cpp
+++ b/core/solver/bicg.cpp
@@ -195,6 +195,7 @@ void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         if (stop_criterion->update()
                 .num_iterations(iter)
                 .residual(r.get())
+                .implicit_sq_residual_norm(rho->compute_absolute().get())
                 .solution(dense_x)
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
             break;
diff --git a/core/solver/bicgstab.cpp b/core/solver/bicgstab.cpp
index 2a452031fcc..0f359950095 100644
--- a/core/solver/bicgstab.cpp
+++ b/core/solver/bicgstab.cpp
@@ -89,6 +89,7 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 {
     using std::swap;
     using Vector = matrix::Dense<ValueType>;
+    using AbsVector = matrix::Dense<remove_complex<ValueType>>;
 
     constexpr uint8 RelativeStoppingId{1};
 
@@ -113,6 +114,7 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     auto gamma = Vector::create_with_config_of(alpha.get());
     auto prev_rho = Vector::create_with_config_of(alpha.get());
     auto rho = Vector::create_with_config_of(alpha.get());
+    auto rho_abs = AbsVector::create(exec, alpha->get_size());
     auto omega = Vector::create_with_config_of(alpha.get());
 
     bool one_changed{};
@@ -140,16 +142,18 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         ++iter;
         this->template log<log::Logger::iteration_complete>(this, iter, r.get(),
                                                             dense_x);
+        rr->compute_dot(r.get(), rho.get());
+        rho_abs = rho->compute_absolute();
+
         if (stop_criterion->update()
                 .num_iterations(iter)
                 .residual(r.get())
+                .implicit_sq_residual_norm(rho_abs.get())
                 .solution(dense_x)
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
             break;
         }
 
-        rr->compute_dot(r.get(), rho.get());
-
         exec->run(bicgstab::make_step_1(r.get(), p.get(), v.get(), rho.get(),
                                         prev_rho.get(), alpha.get(),
                                         omega.get(), &stop_status));
@@ -169,6 +173,7 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             stop_criterion->update()
                 .num_iterations(iter)
                 .residual(s.get())
+                .implicit_sq_residual_norm(rho_abs.get())
                 // .solution(dense_x) // outdated at this point
                 .check(RelativeStoppingId, false, &stop_status, &one_changed);
         if (one_changed) {
diff --git a/core/solver/cg.cpp b/core/solver/cg.cpp
index 66b8fee43df..84184d6584a 100644
--- a/core/solver/cg.cpp
+++ b/core/solver/cg.cpp
@@ -138,6 +138,7 @@ void Cg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         if (stop_criterion->update()
                 .num_iterations(iter)
                 .residual(r.get())
+                .implicit_sq_residual_norm(rho->compute_absolute().get())
                 .solution(dense_x)
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
             break;
diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index 6532835cd75..5ec0173a384 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -173,6 +173,7 @@ void Cgs<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         if (stop_criterion->update()
                 .num_iterations(iter)
                 .residual(r.get())
+                .implicit_sq_residual_norm(rho->compute_absolute().get())
                 .solution(dense_x)
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
             break;
diff --git a/core/solver/fcg.cpp b/core/solver/fcg.cpp
index a7f13060983..de3017db571 100644
--- a/core/solver/fcg.cpp
+++ b/core/solver/fcg.cpp
@@ -141,6 +141,7 @@ void Fcg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         if (stop_criterion->update()
                 .num_iterations(iter)
                 .residual(r.get())
+                .implicit_sq_residual_norm(rho->compute_absolute().get())
                 .solution(dense_x)
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
             break;
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index a76649b0fc7..88683136a00 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -155,6 +155,7 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
     residual->copy_from(dense_b);
     system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(),
                           residual.get());
+    residual->compute_norm2(residual_norm.get());
 
     // g = u = 0
     exec->run(idr::make_fill_array(
@@ -178,6 +179,7 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
         if (stop_criterion->update()
                 .num_iterations(total_iter)
                 .residual(residual.get())
+                .residual_norm(residual_norm.get())
                 .solution(dense_x)
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
             break;
diff --git a/reference/test/solver/bicg_kernels.cpp b/reference/test/solver/bicg_kernels.cpp
index d00fde64493..5b54357245a 100644
--- a/reference/test/solver/bicg_kernels.cpp
+++ b/reference/test/solver/bicg_kernels.cpp
@@ -89,6 +89,16 @@ class Bicg : public ::testing::Test {
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
+          bicg_factory_big2(
+              Solver::build()
+                  .with_criteria(
+                      gko::stop::Iteration::build().with_max_iters(100u).on(
+                          exec),
+                      gko::stop::ImplicitResidualNormReduction<
+                          value_type>::build()
+                          .with_reduction_factor(r<value_type>::value)
+                          .on(exec))
+                  .on(exec)),
           mtx_non_symmetric(gko::initialize<Mtx>(
               {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec))
 
@@ -101,6 +111,7 @@ class Bicg : public ::testing::Test {
     std::shared_ptr<Mtx> mtx_non_symmetric;
     std::unique_ptr<typename Solver::Factory> bicg_factory;
     std::unique_ptr<typename Solver::Factory> bicg_factory_big;
+    std::unique_ptr<typename Solver::Factory> bicg_factory_big2;
     std::unique_ptr<typename Solver::Factory> bicg_factory_non_symmetric;
 };
 
@@ -209,6 +220,23 @@ TYPED_TEST(Bicg, SolvesBigDenseSystem2)
 }
 
 
+TYPED_TEST(Bicg, SolvesBigDenseSystemImplicitResNormCrit)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto solver = this->bicg_factory_big2->generate(this->mtx_big);
+    auto b = gko::initialize<Mtx>(
+        {886630.5, -172578.0, 684522.0, -65310.5, 455487.5, 607436.0},
+        this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
+                        r<value_type>::value * 1e2);
+}
+
+
 TYPED_TEST(Bicg, SolvesNonSymmetricStencilSystem)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/reference/test/solver/bicgstab_kernels.cpp b/reference/test/solver/bicgstab_kernels.cpp
index 0779821d159..3189560acf2 100644
--- a/reference/test/solver/bicgstab_kernels.cpp
+++ b/reference/test/solver/bicgstab_kernels.cpp
@@ -73,6 +73,18 @@ class Bicgstab : public ::testing::Test {
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
+          bicgstab_factory2(
+              Solver::build()
+                  .with_criteria(
+                      gko::stop::Iteration::build().with_max_iters(8u).on(exec),
+                      gko::stop::Time::build()
+                          .with_time_limit(std::chrono::seconds(6))
+                          .on(exec),
+                      gko::stop::ImplicitResidualNormReduction<
+                          value_type>::build()
+                          .with_reduction_factor(r<value_type>::value)
+                          .on(exec))
+                  .on(exec)),
           bicgstab_factory_precision(
               Solver::build()
                   .with_criteria(
@@ -90,6 +102,7 @@ class Bicgstab : public ::testing::Test {
     std::shared_ptr<const gko::Executor> exec;
     std::shared_ptr<Mtx> mtx;
     std::unique_ptr<typename Solver::Factory> bicgstab_factory;
+    std::unique_ptr<typename Solver::Factory> bicgstab_factory2;
     std::unique_ptr<typename Solver::Factory> bicgstab_factory_precision;
 };
 
@@ -130,6 +143,25 @@ TYPED_TEST(Bicgstab, SolvesMultipleDenseSystems)
 }
 
 
+TYPED_TEST(Bicgstab, SolvesMultipleDenseSystemsWithImplicitResNormCrit)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using T = value_type;
+    auto half_tol = std::sqrt(r<value_type>::value);
+    auto solver = this->bicgstab_factory2->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {I<T>{-1.0, -5.0}, I<T>{3.0, 1.0}, I<T>{1.0, -2.0}}, this->exec);
+    auto x = gko::initialize<Mtx>(
+        {I<T>{0.0, 0.0}, I<T>{0.0, 0.0}, I<T>{0.0, 0.0}}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({{-4.0, 1.0}, {-1.0, 2.0}, {4.0, -1.0}}),
+                        half_tol);
+}
+
+
 TYPED_TEST(Bicgstab, SolvesDenseSystemUsingAdvancedApply)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/reference/test/solver/cg_kernels.cpp b/reference/test/solver/cg_kernels.cpp
index a2884779e03..6d49edb76de 100644
--- a/reference/test/solver/cg_kernels.cpp
+++ b/reference/test/solver/cg_kernels.cpp
@@ -64,7 +64,8 @@ class Cg : public ::testing::Test {
           cg_factory(
               Solver::build()
                   .with_criteria(
-                      gko::stop::Iteration::build().with_max_iters(4u).on(exec),
+                      gko::stop::Iteration::build().with_max_iters(400u).on(
+                          exec),
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
@@ -88,6 +89,16 @@ class Cg : public ::testing::Test {
                       gko::stop::ResidualNormReduction<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
+                  .on(exec)),
+          cg_factory_big2(
+              Solver::build()
+                  .with_criteria(
+                      gko::stop::Iteration::build().with_max_iters(100u).on(
+                          exec),
+                      gko::stop::ImplicitResidualNormReduction<
+                          value_type>::build()
+                          .with_reduction_factor(r<value_type>::value)
+                          .on(exec))
                   .on(exec))
     {}
 
@@ -96,6 +107,7 @@ class Cg : public ::testing::Test {
     std::shared_ptr<Mtx> mtx_big;
     std::unique_ptr<typename Solver::Factory> cg_factory;
     std::unique_ptr<typename Solver::Factory> cg_factory_big;
+    std::unique_ptr<typename Solver::Factory> cg_factory_big2;
 };
 
 TYPED_TEST_SUITE(Cg, gko::test::ValueTypes);
@@ -203,6 +215,23 @@ TYPED_TEST(Cg, SolvesBigDenseSystem2)
 }
 
 
+TYPED_TEST(Cg, SolvesBigDenseSystem3)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto solver = this->cg_factory_big2->generate(this->mtx_big);
+    auto b = gko::initialize<Mtx>(
+        {886630.5, -172578.0, 684522.0, -65310.5, 455487.5, 607436.0},
+        this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
+                        r<value_type>::value * 1e2);
+}
+
+
 template <typename T>
 gko::remove_complex<T> infNorm(gko::matrix::Dense<T> *mat, size_t col = 0)
 {
diff --git a/reference/test/solver/cgs_kernels.cpp b/reference/test/solver/cgs_kernels.cpp
index efaa0d28176..83612a8cb44 100644
--- a/reference/test/solver/cgs_kernels.cpp
+++ b/reference/test/solver/cgs_kernels.cpp
@@ -87,6 +87,16 @@ class Cgs : public ::testing::Test {
                       gko::stop::ResidualNormReduction<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
+                  .on(exec)),
+          cgs_factory_big2(
+              Solver::build()
+                  .with_criteria(
+                      gko::stop::Iteration::build().with_max_iters(100u).on(
+                          exec),
+                      gko::stop::ImplicitResidualNormReduction<
+                          value_type>::build()
+                          .with_reduction_factor(r<value_type>::value)
+                          .on(exec))
                   .on(exec))
     {}
 
@@ -95,6 +105,7 @@ class Cgs : public ::testing::Test {
     std::shared_ptr<Mtx> mtx_big;
     std::unique_ptr<typename Solver::Factory> cgs_factory;
     std::unique_ptr<typename Solver::Factory> cgs_factory_big;
+    std::unique_ptr<typename Solver::Factory> cgs_factory_big2;
 };
 
 TYPED_TEST_SUITE(Cgs, gko::test::ValueTypes);
@@ -188,6 +199,22 @@ TYPED_TEST(Cgs, SolvesBigDenseSystem1)
 }
 
 
+TYPED_TEST(Cgs, SolvesBigDenseSystemWithImplicitResNormCrit)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto solver = this->cgs_factory_big2->generate(this->mtx_big);
+    auto b = gko::initialize<Mtx>(
+        {17356.0, 5466.0, 748.0, -456.0, 3434.0, -7020.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({-58.0, 98.0, -16.0, -58.0, 2.0, 76.0}),
+                        r<value_type>::value * 1e2);
+}
+
+
 TYPED_TEST(Cgs, SolvesBigDenseSystem2)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/reference/test/solver/fcg_kernels.cpp b/reference/test/solver/fcg_kernels.cpp
index 947ec1d325a..aa311c37fbe 100644
--- a/reference/test/solver/fcg_kernels.cpp
+++ b/reference/test/solver/fcg_kernels.cpp
@@ -89,6 +89,16 @@ class Fcg : public ::testing::Test {
                       gko::stop::ResidualNormReduction<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
+                  .on(exec)),
+          fcg_factory_big2(
+              Solver::build()
+                  .with_criteria(
+                      gko::stop::Iteration::build().with_max_iters(100u).on(
+                          exec),
+                      gko::stop::ImplicitResidualNormReduction<
+                          value_type>::build()
+                          .with_reduction_factor(r<value_type>::value)
+                          .on(exec))
                   .on(exec))
     {}
 
@@ -97,6 +107,7 @@ class Fcg : public ::testing::Test {
     std::shared_ptr<Mtx> mtx_big;
     std::unique_ptr<typename Solver::Factory> fcg_factory;
     std::unique_ptr<typename Solver::Factory> fcg_factory_big;
+    std::unique_ptr<typename Solver::Factory> fcg_factory_big2;
 };
 
 TYPED_TEST_SUITE(Fcg, gko::test::ValueTypes);
@@ -187,6 +198,23 @@ TYPED_TEST(Fcg, SolvesBigDenseSystem1)
 }
 
 
+TYPED_TEST(Fcg, SolvesBigDenseSystemWithImplicitResNormCrit)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto solver = this->fcg_factory_big2->generate(this->mtx_big);
+    auto b = gko::initialize<Mtx>(
+        {886630.5, -172578.0, 684522.0, -65310.5, 455487.5, 607436.0},
+        this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
+                        r<value_type>::value * 1e3);
+}
+
+
 TYPED_TEST(Fcg, SolvesBigDenseSystem2)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/reference/test/solver/gmres_kernels.cpp b/reference/test/solver/gmres_kernels.cpp
index 8574af14dc3..b43cb8c38a8 100644
--- a/reference/test/solver/gmres_kernels.cpp
+++ b/reference/test/solver/gmres_kernels.cpp
@@ -91,6 +91,16 @@ class Gmres : public ::testing::Test {
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
+          gmres_factory_big2(
+              Solver::build()
+                  .with_criteria(
+                      gko::stop::Iteration::build().with_max_iters(100u).on(
+                          exec),
+                      gko::stop::ImplicitResidualNormReduction<
+                          value_type>::build()
+                          .with_reduction_factor(r<value_type>::value)
+                          .on(exec))
+                  .on(exec)),
           mtx_medium(
               gko::initialize<Mtx>({{-86.40, 153.30, -108.90, 8.60, -61.60},
                                     {7.70, -77.00, 3.30, -149.20, 74.80},
@@ -106,6 +116,7 @@ class Gmres : public ::testing::Test {
     std::shared_ptr<Mtx> mtx_big;
     std::unique_ptr<typename Solver::Factory> gmres_factory;
     std::unique_ptr<typename Solver::Factory> gmres_factory_big;
+    std::unique_ptr<typename Solver::Factory> gmres_factory_big2;
 };
 
 TYPED_TEST_SUITE(Gmres, gko::test::ValueTypes);
@@ -213,6 +224,20 @@ TYPED_TEST(Gmres, SolvesBigDenseSystem2)
 }
 
 
+TYPED_TEST(Gmres, SolveWithImplicitResNormCritIsDisabled)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto solver = this->gmres_factory_big2->generate(this->mtx_big);
+    auto b = gko::initialize<Mtx>(
+        {175352.10, 313410.50, 131114.10, -134116.30, 179529.30, -43564.90},
+        this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+
+    ASSERT_THROW(solver->apply(b.get(), x.get()), gko::NotSupported);
+}
+
+
 template <typename T>
 gko::remove_complex<T> infNorm(gko::matrix::Dense<T> *mat, size_t col = 0)
 {

From fe9df0b68b811d2ed86d30619a7fc6f565fc8290 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Mon, 8 Feb 2021 23:31:51 +0100
Subject: [PATCH 0321/1008] Review update.

+ Add a parameter relative to which the implicit residual is computed.
+ Update docs.

Co-authored-by: Natalie Beams <nbeams@icl.utk.edu>
---
 cuda/stop/residual_norm_kernels.cu            |  4 +-
 dpcpp/stop/residual_norm_kernels.dp.cpp       |  4 +-
 hip/stop/residual_norm_kernels.hip.cpp        |  4 +-
 include/ginkgo/core/stop/residual_norm.hpp    | 72 ++++++++++++++-----
 omp/stop/residual_norm_kernels.cpp            |  4 +-
 reference/stop/residual_norm_kernels.cpp      |  4 +-
 reference/test/stop/residual_norm_kernels.cpp | 31 +++++++-
 7 files changed, 95 insertions(+), 28 deletions(-)

diff --git a/cuda/stop/residual_norm_kernels.cu b/cuda/stop/residual_norm_kernels.cu
index e79800c874d..2ab22bf9a3b 100644
--- a/cuda/stop/residual_norm_kernels.cu
+++ b/cuda/stop/residual_norm_kernels.cu
@@ -124,8 +124,8 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
 
 
 /**
- * @brief The Residual norm stopping criterion namespace.
- * @ref resnorm
+ * @brief The Implicit Residual norm stopping criterion.
+ * @ref implicit_resnorm
  * @ingroup resnorm
  */
 namespace implicit_residual_norm {
diff --git a/dpcpp/stop/residual_norm_kernels.dp.cpp b/dpcpp/stop/residual_norm_kernels.dp.cpp
index 7a498d0be47..8e3fc2981b7 100644
--- a/dpcpp/stop/residual_norm_kernels.dp.cpp
+++ b/dpcpp/stop/residual_norm_kernels.dp.cpp
@@ -72,8 +72,8 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
 
 
 /**
- * @brief The Residual norm stopping criterion namespace.
- * @ref resnorm
+ * @brief The Implicit Residual norm stopping criterion.
+ * @ref implicit_resnorm
  * @ingroup resnorm
  */
 namespace implicit_residual_norm {
diff --git a/hip/stop/residual_norm_kernels.hip.cpp b/hip/stop/residual_norm_kernels.hip.cpp
index 32920f23600..a0543efde2a 100644
--- a/hip/stop/residual_norm_kernels.hip.cpp
+++ b/hip/stop/residual_norm_kernels.hip.cpp
@@ -128,8 +128,8 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
 
 
 /**
- * @brief The Residual norm stopping criterion namespace.
- * @ref resnorm
+ * @brief The Implicit Residual norm stopping criterion.
+ * @ref implicit_resnorm
  * @ingroup resnorm
  */
 namespace implicit_residual_norm {
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index 487d7e323c9..0e7e8f42c80 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -228,15 +228,16 @@ class RelativeResidualNorm : public ResidualNorm<ValueType> {
 
 /**
  * The ImplicitResidualNormReduction class is a stopping criterion which
- * stops the iteration process when the residual norm is below a certain
- * threshold recurrent to the norm of the right-hand side, i.e. when
- * norm(residual) / norm(right_hand_side) < threshold.
- * For better performance, the checks are run thanks to kernels on
- * the executor where the algorithm is executed.
+ * stops the iteration process when the implicit residual norm is below a
+ * certain threshold relative to either the norm of the right-hand side, i.e.
+ * when norm(residual) / norm(right_hand_side) < threshold or the initial
+ * residual, i.e. when norm(residual) / norm(initial_residual) < threshold. For
+ * better performance, the checks are run on the executor
+ * where the algorithm is executed.
  *
  * @note To use this stopping criterion there are some dependencies. The
- * constructor depends on `b` in order to compute the norm of the
- * right-hand side. If this is not correctly provided, an exception
+ * constructor depends on either `b` or the `initial_residual` in order to
+ * compute their norms. If this is not correctly provided, an exception
  * ::gko::NotSupported() is thrown.
  *
  * @ingroup stop
@@ -251,10 +252,16 @@ class ImplicitResidualNormReduction : public ResidualNorm<ValueType> {
     GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
     {
         /**
-         * Recurrent residual norm goal
+         * Implicit residual norm reduction factor
          */
         remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(
             reduction_factor, static_cast<remove_complex<ValueType>>(1e-15));
+
+        /**
+         * The quantity the reduction is relative to. Choices include "rhs" and
+         * "initial_residual"
+         */
+        std::string GKO_FACTORY_PARAMETER_SCALAR(relative_to, "rhs");
     };
     GKO_ENABLE_CRITERION_FACTORY(ImplicitResidualNormReduction<ValueType>,
                                  parameters, Factory);
@@ -277,28 +284,59 @@ class ImplicitResidualNormReduction : public ResidualNorm<ValueType> {
           parameters_{factory->get_parameters()},
           device_storage_{factory->get_executor(), 2}
     {
-        if (args.b == nullptr) {
+        this->reduction_factor_ = factory->get_parameters().reduction_factor;
+        this->relative_to_rhs_ =
+            factory->get_parameters().relative_to.compare("rhs") == 0;
+        this->relative_to_initial_residual_ =
+            factory->get_parameters().relative_to.compare("initial_residual") ==
+            0;
+        if (this->relative_to_rhs_ && args.b == nullptr) {
+            GKO_NOT_SUPPORTED(nullptr);
+        }
+        if (this->relative_to_initial_residual_ &&
+            args.initial_residual == nullptr) {
+            GKO_NOT_SUPPORTED(nullptr);
+        }
+
+        size_type tau_size = 0;
+        if (this->relative_to_rhs_) {
+            tau_size = args.b->get_size()[1];
+        } else if (this->relative_to_initial_residual_) {
+            tau_size = args.initial_residual->get_size()[1];
+        } else {
             GKO_NOT_SUPPORTED(nullptr);
         }
 
         auto exec = factory->get_executor();
 
-        this->reduction_factor_ = factory->get_parameters().reduction_factor;
-        this->starting_tau_ =
-            NormVector::create(exec, dim<2>{1, args.b->get_size()[1]});
+        this->starting_tau_ = NormVector::create(exec, dim<2>{1, tau_size});
         this->u_dense_tau_ =
             NormVector::create_with_config_of(this->starting_tau_.get());
-        if (dynamic_cast<const ComplexVector *>(args.b.get())) {
-            auto dense_rhs = as<ComplexVector>(args.b);
-            dense_rhs->compute_norm2(this->starting_tau_.get());
+        if (this->relative_to_rhs_) {
+            if (dynamic_cast<const ComplexVector *>(args.b.get())) {
+                auto dense_rhs = as<ComplexVector>(args.b);
+                dense_rhs->compute_norm2(this->starting_tau_.get());
+            } else {
+                auto dense_rhs = as<Vector>(args.b);
+                dense_rhs->compute_norm2(this->starting_tau_.get());
+            }
+        } else if (this->relative_to_initial_residual_) {
+            if (dynamic_cast<const ComplexVector *>(args.initial_residual)) {
+                auto dense_r = as<ComplexVector>(args.initial_residual);
+                dense_r->compute_norm2(this->starting_tau_.get());
+            } else {
+                auto dense_r = as<Vector>(args.initial_residual);
+                dense_r->compute_norm2(this->starting_tau_.get());
+            }
         } else {
-            auto dense_rhs = as<Vector>(args.b);
-            dense_rhs->compute_norm2(this->starting_tau_.get());
+            GKO_NOT_SUPPORTED(nullptr);
         }
     }
 
 private:
     remove_complex<ValueType> reduction_factor_{};
+    bool relative_to_rhs_{};
+    bool relative_to_initial_residual_{};
     /* Contains device side: all_converged and one_changed booleans */
     Array<bool> device_storage_;
 };
diff --git a/omp/stop/residual_norm_kernels.cpp b/omp/stop/residual_norm_kernels.cpp
index 5e11030a1e4..a231ff4210e 100644
--- a/omp/stop/residual_norm_kernels.cpp
+++ b/omp/stop/residual_norm_kernels.cpp
@@ -91,8 +91,8 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
 
 
 /**
- * @brief The Residual norm stopping criterion namespace.
- * @ref resnorm
+ * @brief The Implicit Residual norm stopping criterion.
+ * @ref implicit_resnorm
  * @ingroup resnorm
  */
 namespace implicit_residual_norm {
diff --git a/reference/stop/residual_norm_kernels.cpp b/reference/stop/residual_norm_kernels.cpp
index dff14bfc8e0..3e6301f8ae1 100644
--- a/reference/stop/residual_norm_kernels.cpp
+++ b/reference/stop/residual_norm_kernels.cpp
@@ -88,8 +88,8 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
 
 
 /**
- * @brief The Recurrent Residual norm stopping criterion.
- * @ref resnorm
+ * @brief The Implicit Residual norm stopping criterion.
+ * @ref implicit_resnorm
  * @ingroup resnorm
  */
 namespace implicit_residual_norm {
diff --git a/reference/test/stop/residual_norm_kernels.cpp b/reference/test/stop/residual_norm_kernels.cpp
index 15a2033a398..a443d0de858 100644
--- a/reference/test/stop/residual_norm_kernels.cpp
+++ b/reference/test/stop/residual_norm_kernels.cpp
@@ -314,11 +314,25 @@ class ImplicitResidualNormReduction : public ::testing::Test {
         factory_ = gko::stop::ImplicitResidualNormReduction<T>::build()
                        .with_reduction_factor(r<T>::value)
                        .on(exec_);
+        factory_2_ = gko::stop::ImplicitResidualNormReduction<T>::build()
+                         .with_reduction_factor(r<T>::value)
+                         .with_relative_to("initial_residual")
+                         .on(exec_);
+        factory_3_ = gko::stop::ImplicitResidualNormReduction<T>::build()
+                         .with_reduction_factor(r<T>::value)
+                         .with_relative_to("rhs")
+                         .on(exec_);
     }
 
     std::unique_ptr<
         typename gko::stop::ImplicitResidualNormReduction<T>::Factory>
         factory_;
+    std::unique_ptr<
+        typename gko::stop::ImplicitResidualNormReduction<T>::Factory>
+        factory_2_;
+    std::unique_ptr<
+        typename gko::stop::ImplicitResidualNormReduction<T>::Factory>
+        factory_3_;
     std::shared_ptr<const gko::Executor> exec_;
 };
 
@@ -330,11 +344,16 @@ TYPED_TEST(ImplicitResidualNormReduction, CanCreateFactory)
     ASSERT_NE(this->factory_, nullptr);
     ASSERT_EQ(this->factory_->get_parameters().reduction_factor,
               r<TypeParam>::value);
+    ASSERT_EQ(this->factory_->get_parameters().relative_to, "rhs");
+    ASSERT_EQ(this->factory_2_->get_parameters().relative_to,
+              "initial_residual");
+    ASSERT_EQ(this->factory_3_->get_parameters().relative_to, "rhs");
     ASSERT_EQ(this->factory_->get_executor(), this->exec_);
 }
 
 
-TYPED_TEST(ImplicitResidualNormReduction, CannotCreateCriterionWithoutB)
+TYPED_TEST(ImplicitResidualNormReduction,
+           CannotCreateCriterionWithoutBAndInitRes)
 {
     ASSERT_THROW(this->factory_->generate(nullptr, nullptr, nullptr, nullptr),
                  gko::NotSupported);
@@ -352,6 +371,16 @@ TYPED_TEST(ImplicitResidualNormReduction, CanCreateCriterionWithB)
 }
 
 
+TYPED_TEST(ImplicitResidualNormReduction, CanCreateCriterionWithInitialRes)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
+    auto criterion = this->factory_2_->generate(nullptr, nullptr, nullptr,
+                                                initial_res.get());
+    ASSERT_NE(criterion, nullptr);
+}
+
+
 TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoal)
 {
     using Mtx = typename TestFixture::Mtx;

From 1f8767f561d642d349ffae203645d12951c48b6b Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 9 Feb 2021 13:34:03 +0100
Subject: [PATCH 0322/1008] Review updates.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

+ Use enums for ImplicitResNorm options.
+ Move absolute computations to kernels.

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/device_hooks/common_kernels.inc.cpp      |   3 +-
 core/solver/bicg.cpp                          |   2 +-
 core/solver/bicgstab.cpp                      |   6 +-
 core/solver/cg.cpp                            |   2 +-
 core/solver/cgs.cpp                           |   2 +-
 core/solver/fcg.cpp                           |   2 +-
 core/stop/residual_norm.cpp                   |   4 +-
 core/stop/residual_norm_kernels.hpp           |  15 +-
 cuda/stop/residual_norm_kernels.cu            |  29 ++--
 cuda/test/stop/residual_norm_kernels.cpp      |  40 +++--
 dpcpp/stop/residual_norm_kernels.dp.cpp       |  21 ++-
 hip/stop/residual_norm_kernels.hip.cpp        |  29 ++--
 hip/test/stop/residual_norm_kernels.cpp       |  39 +++--
 include/ginkgo/core/stop/residual_norm.hpp    |  16 +-
 omp/stop/residual_norm_kernels.cpp            |  22 ++-
 omp/test/stop/residual_norm_kernels.cpp       | 138 ++++++++----------
 reference/stop/residual_norm_kernels.cpp      |  22 ++-
 reference/test/stop/residual_norm_kernels.cpp |  61 +++++---
 18 files changed, 240 insertions(+), 213 deletions(-)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 053dfde5029..964dfc305cb 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -1197,8 +1197,7 @@ namespace implicit_residual_norm {
 template <typename ValueType>
 GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
-    GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
 
 
 }  // namespace implicit_residual_norm
diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp
index f283d5f16d7..bbc4eab0e03 100644
--- a/core/solver/bicg.cpp
+++ b/core/solver/bicg.cpp
@@ -195,7 +195,7 @@ void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         if (stop_criterion->update()
                 .num_iterations(iter)
                 .residual(r.get())
-                .implicit_sq_residual_norm(rho->compute_absolute().get())
+                .implicit_sq_residual_norm(rho.get())
                 .solution(dense_x)
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
             break;
diff --git a/core/solver/bicgstab.cpp b/core/solver/bicgstab.cpp
index 0f359950095..0ee788387db 100644
--- a/core/solver/bicgstab.cpp
+++ b/core/solver/bicgstab.cpp
@@ -114,7 +114,6 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     auto gamma = Vector::create_with_config_of(alpha.get());
     auto prev_rho = Vector::create_with_config_of(alpha.get());
     auto rho = Vector::create_with_config_of(alpha.get());
-    auto rho_abs = AbsVector::create(exec, alpha->get_size());
     auto omega = Vector::create_with_config_of(alpha.get());
 
     bool one_changed{};
@@ -143,12 +142,11 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         this->template log<log::Logger::iteration_complete>(this, iter, r.get(),
                                                             dense_x);
         rr->compute_dot(r.get(), rho.get());
-        rho_abs = rho->compute_absolute();
 
         if (stop_criterion->update()
                 .num_iterations(iter)
                 .residual(r.get())
-                .implicit_sq_residual_norm(rho_abs.get())
+                .implicit_sq_residual_norm(rho.get())
                 .solution(dense_x)
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
             break;
@@ -173,7 +171,7 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             stop_criterion->update()
                 .num_iterations(iter)
                 .residual(s.get())
-                .implicit_sq_residual_norm(rho_abs.get())
+                .implicit_sq_residual_norm(rho.get())
                 // .solution(dense_x) // outdated at this point
                 .check(RelativeStoppingId, false, &stop_status, &one_changed);
         if (one_changed) {
diff --git a/core/solver/cg.cpp b/core/solver/cg.cpp
index 84184d6584a..dd0558acf75 100644
--- a/core/solver/cg.cpp
+++ b/core/solver/cg.cpp
@@ -138,7 +138,7 @@ void Cg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         if (stop_criterion->update()
                 .num_iterations(iter)
                 .residual(r.get())
-                .implicit_sq_residual_norm(rho->compute_absolute().get())
+                .implicit_sq_residual_norm(rho.get())
                 .solution(dense_x)
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
             break;
diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index 5ec0173a384..febacbb03bf 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -173,7 +173,7 @@ void Cgs<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         if (stop_criterion->update()
                 .num_iterations(iter)
                 .residual(r.get())
-                .implicit_sq_residual_norm(rho->compute_absolute().get())
+                .implicit_sq_residual_norm(rho.get())
                 .solution(dense_x)
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
             break;
diff --git a/core/solver/fcg.cpp b/core/solver/fcg.cpp
index de3017db571..dea9aeff494 100644
--- a/core/solver/fcg.cpp
+++ b/core/solver/fcg.cpp
@@ -141,7 +141,7 @@ void Fcg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         if (stop_criterion->update()
                 .num_iterations(iter)
                 .residual(r.get())
-                .implicit_sq_residual_norm(rho->compute_absolute().get())
+                .implicit_sq_residual_norm(rho.get())
                 .solution(dense_x)
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
             break;
diff --git a/core/stop/residual_norm.cpp b/core/stop/residual_norm.cpp
index c4efb1bf57e..b5db82c920f 100644
--- a/core/stop/residual_norm.cpp
+++ b/core/stop/residual_norm.cpp
@@ -95,9 +95,9 @@ bool ImplicitResidualNormReduction<ValueType>::check_impl(
     uint8 stoppingId, bool setFinalized, Array<stopping_status> *stop_status,
     bool *one_changed, const Criterion::Updater &updater)
 {
-    const NormVector *dense_tau;
+    const Vector *dense_tau;
     if (updater.implicit_sq_residual_norm_ != nullptr) {
-        dense_tau = as<NormVector>(updater.implicit_sq_residual_norm_);
+        dense_tau = as<Vector>(updater.implicit_sq_residual_norm_);
     } else {
         GKO_NOT_SUPPORTED(nullptr);
     }
diff --git a/core/stop/residual_norm_kernels.hpp b/core/stop/residual_norm_kernels.hpp
index 8b8ff732f51..cb0496b5aef 100644
--- a/core/stop/residual_norm_kernels.hpp
+++ b/core/stop/residual_norm_kernels.hpp
@@ -66,13 +66,14 @@ namespace residual_norm {
 namespace implicit_residual_norm {
 
 
-#define GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL(_type)                       \
-    void implicit_residual_norm(                                               \
-        std::shared_ptr<const DefaultExecutor> exec,                           \
-        const matrix::Dense<_type> *tau, const matrix::Dense<_type> *orig_tau, \
-        _type rel_residual_goal, uint8 stoppingId, bool setFinalized,          \
-        Array<stopping_status> *stop_status, Array<bool> *device_storage,      \
-        bool *all_converged, bool *one_changed)
+#define GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL(_type)           \
+    void implicit_residual_norm(                                   \
+        std::shared_ptr<const DefaultExecutor> exec,               \
+        const matrix::Dense<_type> *tau,                           \
+        const matrix::Dense<remove_complex<_type>> *orig_tau,      \
+        remove_complex<_type> rel_residual_goal, uint8 stoppingId, \
+        bool setFinalized, Array<stopping_status> *stop_status,    \
+        Array<bool> *device_storage, bool *all_converged, bool *one_changed)
 
 
 #define GKO_DECLARE_ALL_AS_TEMPLATES2 \
diff --git a/cuda/stop/residual_norm_kernels.cu b/cuda/stop/residual_norm_kernels.cu
index 2ab22bf9a3b..6297df89352 100644
--- a/cuda/stop/residual_norm_kernels.cu
+++ b/cuda/stop/residual_norm_kernels.cu
@@ -137,15 +137,16 @@ constexpr int default_block_size = 512;
 template <typename ValueType>
 __global__
     __launch_bounds__(default_block_size) void implicit_residual_norm_kernel(
-        size_type num_cols, ValueType rel_residual_goal,
+        size_type num_cols, remove_complex<ValueType> rel_residual_goal,
         const ValueType *__restrict__ tau,
-        const ValueType *__restrict__ orig_tau, uint8 stoppingId,
-        bool setFinalized, stopping_status *__restrict__ stop_status,
+        const remove_complex<ValueType> *__restrict__ orig_tau,
+        uint8 stoppingId, bool setFinalized,
+        stopping_status *__restrict__ stop_status,
         bool *__restrict__ device_storage)
 {
     const auto tidx = thread::get_thread_id_flat();
     if (tidx < num_cols) {
-        if (sqrt(tau[tidx]) < rel_residual_goal * orig_tau[tidx]) {
+        if (sqrt(abs(tau[tidx])) < rel_residual_goal * orig_tau[tidx]) {
             stop_status[tidx].converge(stoppingId, setFinalized);
             device_storage[1] = true;
         }
@@ -167,17 +168,14 @@ __global__ __launch_bounds__(1) void init_kernel(
 
 
 template <typename ValueType>
-void implicit_residual_norm(std::shared_ptr<const CudaExecutor> exec,
-                            const matrix::Dense<ValueType> *tau,
-                            const matrix::Dense<ValueType> *orig_tau,
-                            ValueType rel_residual_goal, uint8 stoppingId,
-                            bool setFinalized,
-                            Array<stopping_status> *stop_status,
-                            Array<bool> *device_storage, bool *all_converged,
-                            bool *one_changed)
+void implicit_residual_norm(
+    std::shared_ptr<const CudaExecutor> exec,
+    const matrix::Dense<ValueType> *tau,
+    const matrix::Dense<remove_complex<ValueType>> *orig_tau,
+    remove_complex<ValueType> rel_residual_goal, uint8 stoppingId,
+    bool setFinalized, Array<stopping_status> *stop_status,
+    Array<bool> *device_storage, bool *all_converged, bool *one_changed)
 {
-    static_assert(is_complex_s<ValueType>::value == false,
-                  "ValueType must not be complex in this function!");
     init_kernel<<<1, 1>>>(as_cuda_type(device_storage->get_data()));
 
     const dim3 block_size(default_block_size, 1, 1);
@@ -195,8 +193,7 @@ void implicit_residual_norm(std::shared_ptr<const CudaExecutor> exec,
     *one_changed = exec->copy_val_to_host(device_storage->get_const_data() + 1);
 }
 
-GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
-    GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
 
 
 }  // namespace implicit_residual_norm
diff --git a/cuda/test/stop/residual_norm_kernels.cpp b/cuda/test/stop/residual_norm_kernels.cpp
index 088def448bd..c96de7df739 100644
--- a/cuda/test/stop/residual_norm_kernels.cpp
+++ b/cuda/test/stop/residual_norm_kernels.cpp
@@ -48,6 +48,7 @@ constexpr double tol = 1.0e-14;
 class ResidualNormReduction : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
 
     ResidualNormReduction()
     {
@@ -67,6 +68,8 @@ class ResidualNormReduction : public ::testing::Test {
 TEST_F(ResidualNormReduction, WaitsTillResidualGoal)
 {
     auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto res_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    res->compute_norm2(res_norm.get());
     auto d_res = Mtx::create(cuda_);
     d_res->copy_from(res.get());
     std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
@@ -84,7 +87,7 @@ TEST_F(ResidualNormReduction, WaitsTillResidualGoal)
             .residual_norm(d_res.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res->at(0) = tol * 1.1e+2;
+    res->at(0) = tol * 1.1 * res_norm->at(0);
     d_res->copy_from(res.get());
     ASSERT_FALSE(
         criterion->update()
@@ -95,7 +98,7 @@ TEST_F(ResidualNormReduction, WaitsTillResidualGoal)
     stop_status.set_executor(cuda_);
     ASSERT_FALSE(one_changed);
 
-    res->at(0) = tol * 0.9e+2;
+    res->at(0) = tol * 0.9 * res_norm->at(0);
     d_res->copy_from(res.get());
     ASSERT_TRUE(
         criterion->update()
@@ -110,6 +113,8 @@ TEST_F(ResidualNormReduction, WaitsTillResidualGoal)
 TEST_F(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
 {
     auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto res_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    res->compute_norm2(res_norm.get());
     auto d_res = Mtx::create(cuda_);
     d_res->copy_from(res.get());
     std::shared_ptr<gko::LinOp> rhs =
@@ -129,7 +134,7 @@ TEST_F(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
             .residual_norm(d_res.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res->at(0, 0) = tol * 0.9e+2;
+    res->at(0, 0) = tol * 0.9 * res_norm->at(0, 0);
     d_res->copy_from(res.get());
     ASSERT_FALSE(
         criterion->update()
@@ -140,7 +145,7 @@ TEST_F(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
     stop_status.set_executor(cuda_);
     ASSERT_TRUE(one_changed);
 
-    res->at(0, 1) = tol * 0.9e+2;
+    res->at(0, 1) = tol * 0.9 * res_norm->at(0, 1);
     d_res->copy_from(res.get());
     ASSERT_TRUE(
         criterion->update()
@@ -155,6 +160,7 @@ TEST_F(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
 class RelativeResidualNorm : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
 
     RelativeResidualNorm()
     {
@@ -177,6 +183,8 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoal)
     auto d_res = Mtx::create(cuda_);
     d_res->copy_from(res.get());
     std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(cuda_);
     d_rhs->copy_from(rhs.get());
     auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
@@ -191,7 +199,7 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoal)
             .residual_norm(d_res.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res->at(0) = tol * 1.1e+1;
+    res->at(0) = tol * 1.1 * rhs_norm->at(0);
     d_res->copy_from(res.get());
     ASSERT_FALSE(
         criterion->update()
@@ -202,7 +210,7 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoal)
     stop_status.set_executor(cuda_);
     ASSERT_FALSE(one_changed);
 
-    res->at(0) = tol * 0.9e+1;
+    res->at(0) = tol * 0.9 * rhs_norm->at(0);
     d_res->copy_from(res.get());
     ASSERT_TRUE(
         criterion->update()
@@ -221,6 +229,8 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
     d_res->copy_from(res.get());
     std::shared_ptr<gko::LinOp> rhs =
         gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(cuda_);
     d_rhs->copy_from(rhs.get());
     auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
@@ -236,7 +246,7 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
             .residual_norm(d_res.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res->at(0, 0) = tol * 0.9e+1;
+    res->at(0, 0) = tol * 0.9 * rhs_norm->at(0, 0);
     d_res->copy_from(res.get());
     ASSERT_FALSE(
         criterion->update()
@@ -247,7 +257,7 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
     stop_status.set_executor(cuda_);
     ASSERT_TRUE(one_changed);
 
-    res->at(0, 1) = tol * 0.9e+1;
+    res->at(0, 1) = tol * 0.9 * rhs_norm->at(0, 1);
     d_res->copy_from(res.get());
     ASSERT_TRUE(
         criterion->update()
@@ -262,6 +272,7 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
 class ImplicitResidualNormReduction : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
 
     ImplicitResidualNormReduction()
     {
@@ -285,6 +296,8 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoal)
     auto d_res = Mtx::create(cuda_);
     d_res->copy_from(res.get());
     std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(cuda_);
     d_rhs->copy_from(rhs.get());
     auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
@@ -299,7 +312,7 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoal)
             .implicit_sq_residual_norm(d_res.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res->at(0) = std::pow(tol * 1.1e+1, 2);
+    res->at(0) = std::pow(tol * 1.1 * rhs_norm->at(0), 2);
     d_res->copy_from(res.get());
     ASSERT_FALSE(
         criterion->update()
@@ -310,7 +323,7 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoal)
     stop_status.set_executor(cuda_);
     ASSERT_FALSE(one_changed);
 
-    res->at(0) = std::pow(tol * 0.9e+1, 2);
+    res->at(0) = std::pow(tol * 0.9 * rhs_norm->at(0), 2);
     d_res->copy_from(res.get());
     ASSERT_TRUE(
         criterion->update()
@@ -329,6 +342,8 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
     d_res->copy_from(res.get());
     std::shared_ptr<gko::LinOp> rhs =
         gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(cuda_);
     d_rhs->copy_from(rhs.get());
     auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
@@ -344,7 +359,7 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
             .implicit_sq_residual_norm(d_res.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res->at(0, 0) = std::pow(tol * 0.9e+1, 2);
+    res->at(0, 0) = std::pow(tol * 0.9 * rhs_norm->at(0, 0), 2);
     d_res->copy_from(res.get());
     ASSERT_FALSE(
         criterion->update()
@@ -355,7 +370,7 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
     stop_status.set_executor(cuda_);
     ASSERT_TRUE(one_changed);
 
-    res->at(0, 1) = std::pow(tol * 0.9e+1, 2);
+    res->at(0, 1) = std::pow(tol * 0.9 * rhs_norm->at(0, 1), 2);
     d_res->copy_from(res.get());
     ASSERT_TRUE(
         criterion->update()
@@ -370,6 +385,7 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
 class AbsoluteResidualNorm : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
 
     AbsoluteResidualNorm()
     {
diff --git a/dpcpp/stop/residual_norm_kernels.dp.cpp b/dpcpp/stop/residual_norm_kernels.dp.cpp
index 8e3fc2981b7..71d8b07574d 100644
--- a/dpcpp/stop/residual_norm_kernels.dp.cpp
+++ b/dpcpp/stop/residual_norm_kernels.dp.cpp
@@ -83,17 +83,16 @@ constexpr int default_group_size = 512;
 
 
 template <typename ValueType>
-void implicit_residual_norm(std::shared_ptr<const DpcppExecutor> exec,
-                            const matrix::Dense<ValueType> *tau,
-                            const matrix::Dense<ValueType> *orig_tau,
-                            ValueType rel_residual_goal, uint8 stoppingId,
-                            bool setFinalized,
-                            Array<stopping_status> *stop_status,
-                            Array<bool> *device_storage, bool *all_converged,
-                            bool *one_changed) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
-    GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
+void implicit_residual_norm(
+    std::shared_ptr<const DpcppExecutor> exec,
+    const matrix::Dense<ValueType> *tau,
+    const matrix::Dense<remove_complex<ValueType>> *orig_tau,
+    remove_complex<ValueType> rel_residual_goal, uint8 stoppingId,
+    bool setFinalized, Array<stopping_status> *stop_status,
+    Array<bool> *device_storage, bool *all_converged,
+    bool *one_changed) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
 
 
 }  // namespace implicit_residual_norm
diff --git a/hip/stop/residual_norm_kernels.hip.cpp b/hip/stop/residual_norm_kernels.hip.cpp
index a0543efde2a..e4b3ddf3b95 100644
--- a/hip/stop/residual_norm_kernels.hip.cpp
+++ b/hip/stop/residual_norm_kernels.hip.cpp
@@ -141,15 +141,16 @@ constexpr int default_block_size = 512;
 template <typename ValueType>
 __global__
     __launch_bounds__(default_block_size) void implicit_residual_norm_kernel(
-        size_type num_cols, ValueType rel_residual_goal,
+        size_type num_cols, remove_complex<ValueType> rel_residual_goal,
         const ValueType *__restrict__ tau,
-        const ValueType *__restrict__ orig_tau, uint8 stoppingId,
-        bool setFinalized, stopping_status *__restrict__ stop_status,
+        const remove_complex<ValueType> *__restrict__ orig_tau,
+        uint8 stoppingId, bool setFinalized,
+        stopping_status *__restrict__ stop_status,
         bool *__restrict__ device_storage)
 {
     const auto tidx = thread::get_thread_id_flat();
     if (tidx < num_cols) {
-        if (sqrt(tau[tidx]) < rel_residual_goal * orig_tau[tidx]) {
+        if (sqrt(abs(tau[tidx])) < rel_residual_goal * orig_tau[tidx]) {
             stop_status[tidx].converge(stoppingId, setFinalized);
             device_storage[1] = true;
         }
@@ -171,17 +172,14 @@ __global__ __launch_bounds__(1) void init_kernel(
 
 
 template <typename ValueType>
-void implicit_residual_norm(std::shared_ptr<const HipExecutor> exec,
-                            const matrix::Dense<ValueType> *tau,
-                            const matrix::Dense<ValueType> *orig_tau,
-                            ValueType rel_residual_goal, uint8 stoppingId,
-                            bool setFinalized,
-                            Array<stopping_status> *stop_status,
-                            Array<bool> *device_storage, bool *all_converged,
-                            bool *one_changed)
+void implicit_residual_norm(
+    std::shared_ptr<const HipExecutor> exec,
+    const matrix::Dense<ValueType> *tau,
+    const matrix::Dense<remove_complex<ValueType>> *orig_tau,
+    remove_complex<ValueType> rel_residual_goal, uint8 stoppingId,
+    bool setFinalized, Array<stopping_status> *stop_status,
+    Array<bool> *device_storage, bool *all_converged, bool *one_changed)
 {
-    static_assert(is_complex_s<ValueType>::value == false,
-                  "ValueType must not be complex in this function!");
     hipLaunchKernelGGL((init_kernel), dim3(1), dim3(1), 0, 0,
                        as_hip_type(device_storage->get_data()));
 
@@ -200,8 +198,7 @@ void implicit_residual_norm(std::shared_ptr<const HipExecutor> exec,
     *one_changed = exec->copy_val_to_host(device_storage->get_const_data() + 1);
 }
 
-GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
-    GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
 
 
 }  // namespace implicit_residual_norm
diff --git a/hip/test/stop/residual_norm_kernels.cpp b/hip/test/stop/residual_norm_kernels.cpp
index 3360372acb9..2a6fd1a5b42 100644
--- a/hip/test/stop/residual_norm_kernels.cpp
+++ b/hip/test/stop/residual_norm_kernels.cpp
@@ -48,6 +48,7 @@ constexpr double tol = 1.0e-14;
 class ResidualNormReduction : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
 
     ResidualNormReduction()
     {
@@ -67,6 +68,8 @@ class ResidualNormReduction : public ::testing::Test {
 TEST_F(ResidualNormReduction, WaitsTillResidualGoal)
 {
     auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto res_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    res->compute_norm2(res_norm.get());
     auto d_res = Mtx::create(hip_);
     d_res->copy_from(res.get());
     std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
@@ -84,7 +87,7 @@ TEST_F(ResidualNormReduction, WaitsTillResidualGoal)
             .residual_norm(d_res.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res->at(0) = tol * 1.1e+2;
+    res->at(0) = tol * 1.1 * res_norm->at(0);
     d_res->copy_from(res.get());
     ASSERT_FALSE(
         criterion->update()
@@ -95,7 +98,7 @@ TEST_F(ResidualNormReduction, WaitsTillResidualGoal)
     stop_status.set_executor(hip_);
     ASSERT_FALSE(one_changed);
 
-    res->at(0) = tol * 0.9e+2;
+    res->at(0) = tol * 0.9 * res_norm->at(0);
     d_res->copy_from(res.get());
     ASSERT_TRUE(
         criterion->update()
@@ -110,6 +113,8 @@ TEST_F(ResidualNormReduction, WaitsTillResidualGoal)
 TEST_F(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
 {
     auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto res_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    res->compute_norm2(res_norm.get());
     auto d_res = Mtx::create(hip_);
     d_res->copy_from(res.get());
     std::shared_ptr<gko::LinOp> rhs =
@@ -129,7 +134,7 @@ TEST_F(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
             .residual_norm(d_res.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res->at(0, 0) = tol * 0.9e+2;
+    res->at(0, 0) = tol * 0.9 * res_norm->at(0, 0);
     d_res->copy_from(res.get());
     ASSERT_FALSE(
         criterion->update()
@@ -140,7 +145,7 @@ TEST_F(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
     stop_status.set_executor(hip_);
     ASSERT_TRUE(one_changed);
 
-    res->at(0, 1) = tol * 0.9e+2;
+    res->at(0, 1) = tol * 0.9 * res_norm->at(0, 1);
     d_res->copy_from(res.get());
     ASSERT_TRUE(
         criterion->update()
@@ -155,6 +160,7 @@ TEST_F(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
 class RelativeResidualNorm : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
 
     RelativeResidualNorm()
     {
@@ -177,6 +183,8 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoal)
     auto d_res = Mtx::create(hip_);
     d_res->copy_from(res.get());
     std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(hip_);
     d_rhs->copy_from(rhs.get());
     auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
@@ -191,7 +199,7 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoal)
             .residual_norm(d_res.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res->at(0) = tol * 1.1e+1;
+    res->at(0) = tol * 1.1 * rhs_norm->at(0);
     d_res->copy_from(res.get());
     ASSERT_FALSE(
         criterion->update()
@@ -202,7 +210,7 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoal)
     stop_status.set_executor(hip_);
     ASSERT_FALSE(one_changed);
 
-    res->at(0) = tol * 0.9e+1;
+    res->at(0) = tol * 0.9 * rhs_norm->at(0);
     d_res->copy_from(res.get());
     ASSERT_TRUE(
         criterion->update()
@@ -221,6 +229,8 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
     d_res->copy_from(res.get());
     std::shared_ptr<gko::LinOp> rhs =
         gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(hip_);
     d_rhs->copy_from(rhs.get());
     auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
@@ -236,7 +246,7 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
             .residual_norm(d_res.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res->at(0, 0) = tol * 0.9e+1;
+    res->at(0, 0) = tol * 0.9 * rhs_norm->at(0, 0);
     d_res->copy_from(res.get());
     ASSERT_FALSE(
         criterion->update()
@@ -247,7 +257,7 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
     stop_status.set_executor(hip_);
     ASSERT_TRUE(one_changed);
 
-    res->at(0, 1) = tol * 0.9e+1;
+    res->at(0, 1) = tol * 0.9 * rhs_norm->at(0, 1);
     d_res->copy_from(res.get());
     ASSERT_TRUE(
         criterion->update()
@@ -262,6 +272,7 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
 class ImplicitResidualNormReduction : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
 
     ImplicitResidualNormReduction()
     {
@@ -285,6 +296,8 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoal)
     auto d_res = Mtx::create(hip_);
     d_res->copy_from(res.get());
     std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(hip_);
     d_rhs->copy_from(rhs.get());
     auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
@@ -299,7 +312,7 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoal)
             .implicit_sq_residual_norm(d_res.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res->at(0) = std::pow(tol * 1.1e+1, 2);
+    res->at(0) = std::pow(tol * 1.1 * rhs_norm->at(0), 2);
     d_res->copy_from(res.get());
     ASSERT_FALSE(
         criterion->update()
@@ -310,7 +323,7 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoal)
     stop_status.set_executor(hip_);
     ASSERT_FALSE(one_changed);
 
-    res->at(0) = std::pow(tol * 0.9e+1, 2);
+    res->at(0) = std::pow(tol * 0.9 * rhs_norm->at(0), 2);
     d_res->copy_from(res.get());
     ASSERT_TRUE(
         criterion->update()
@@ -329,6 +342,8 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
     d_res->copy_from(res.get());
     std::shared_ptr<gko::LinOp> rhs =
         gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(hip_);
     d_rhs->copy_from(rhs.get());
     auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
@@ -344,7 +359,7 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
             .implicit_sq_residual_norm(d_res.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res->at(0, 0) = std::pow(tol * 0.9e+1, 2);
+    res->at(0, 0) = std::pow(tol * 0.9 * rhs_norm->at(0, 0), 2);
     d_res->copy_from(res.get());
     ASSERT_FALSE(
         criterion->update()
@@ -355,7 +370,7 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
     stop_status.set_executor(hip_);
     ASSERT_TRUE(one_changed);
 
-    res->at(0, 1) = std::pow(tol * 0.9e+1, 2);
+    res->at(0, 1) = std::pow(tol * 0.9 * rhs_norm->at(0, 1), 2);
     d_res->copy_from(res.get());
     ASSERT_TRUE(
         criterion->update()
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index 0e7e8f42c80..f05257d98e7 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -226,6 +226,11 @@ class RelativeResidualNorm : public ResidualNorm<ValueType> {
 };
 
 
+/**
+ * The enum that holds the quantity relative to the implicit residual.
+ */
+enum class norm_of { rhs, initial_residual };
+
 /**
  * The ImplicitResidualNormReduction class is a stopping criterion which
  * stops the iteration process when the implicit residual norm is below a
@@ -258,10 +263,10 @@ class ImplicitResidualNormReduction : public ResidualNorm<ValueType> {
             reduction_factor, static_cast<remove_complex<ValueType>>(1e-15));
 
         /**
-         * The quantity the reduction is relative to. Choices include "rhs" and
-         * "initial_residual"
+         * The quantity the reduction is relative to. Choices include
+         * "norm_of::rhs" and "norm_of::initial_residual"
          */
-        std::string GKO_FACTORY_PARAMETER_SCALAR(relative_to, "rhs");
+        norm_of GKO_FACTORY_PARAMETER_SCALAR(relative_to, norm_of::rhs);
     };
     GKO_ENABLE_CRITERION_FACTORY(ImplicitResidualNormReduction<ValueType>,
                                  parameters, Factory);
@@ -286,10 +291,9 @@ class ImplicitResidualNormReduction : public ResidualNorm<ValueType> {
     {
         this->reduction_factor_ = factory->get_parameters().reduction_factor;
         this->relative_to_rhs_ =
-            factory->get_parameters().relative_to.compare("rhs") == 0;
+            factory->get_parameters().relative_to == norm_of::rhs;
         this->relative_to_initial_residual_ =
-            factory->get_parameters().relative_to.compare("initial_residual") ==
-            0;
+            factory->get_parameters().relative_to == norm_of::initial_residual;
         if (this->relative_to_rhs_ && args.b == nullptr) {
             GKO_NOT_SUPPORTED(nullptr);
         }
diff --git a/omp/stop/residual_norm_kernels.cpp b/omp/stop/residual_norm_kernels.cpp
index a231ff4210e..a62dccbae48 100644
--- a/omp/stop/residual_norm_kernels.cpp
+++ b/omp/stop/residual_norm_kernels.cpp
@@ -99,21 +99,18 @@ namespace implicit_residual_norm {
 
 
 template <typename ValueType>
-void implicit_residual_norm(std::shared_ptr<const OmpExecutor> exec,
-                            const matrix::Dense<ValueType> *tau,
-                            const matrix::Dense<ValueType> *orig_tau,
-                            ValueType rel_residual_goal, uint8 stoppingId,
-                            bool setFinalized,
-                            Array<stopping_status> *stop_status,
-                            Array<bool> *device_storage, bool *all_converged,
-                            bool *one_changed)
+void implicit_residual_norm(
+    std::shared_ptr<const OmpExecutor> exec,
+    const matrix::Dense<ValueType> *tau,
+    const matrix::Dense<remove_complex<ValueType>> *orig_tau,
+    remove_complex<ValueType> rel_residual_goal, uint8 stoppingId,
+    bool setFinalized, Array<stopping_status> *stop_status,
+    Array<bool> *device_storage, bool *all_converged, bool *one_changed)
 {
-    static_assert(is_complex_s<ValueType>::value == false,
-                  "ValueType must not be complex in this function!");
     bool local_one_changed = false;
 #pragma omp parallel for reduction(|| : local_one_changed)
     for (size_type i = 0; i < tau->get_size()[1]; ++i) {
-        if (sqrt(tau->at(i)) < rel_residual_goal * orig_tau->at(i)) {
+        if (sqrt(abs(tau->at(i))) < rel_residual_goal * orig_tau->at(i)) {
             stop_status->get_data()[i].converge(stoppingId, setFinalized);
             local_one_changed = true;
         }
@@ -131,8 +128,7 @@ void implicit_residual_norm(std::shared_ptr<const OmpExecutor> exec,
     *all_converged = local_all_converged;
 }
 
-GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
-    GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
 
 
 }  // namespace implicit_residual_norm
diff --git a/omp/test/stop/residual_norm_kernels.cpp b/omp/test/stop/residual_norm_kernels.cpp
index 2923f829fe0..5eba7d93851 100644
--- a/omp/test/stop/residual_norm_kernels.cpp
+++ b/omp/test/stop/residual_norm_kernels.cpp
@@ -53,15 +53,15 @@ class ResidualNormReduction : public ::testing::Test {
 
     ResidualNormReduction()
     {
-        omp_ = gko::OmpExecutor::create();
+        exec_ = gko::OmpExecutor::create();
         factory_ = gko::stop::ResidualNormReduction<T>::build()
                        .with_reduction_factor(r<T>::value)
-                       .on(omp_);
+                       .on(exec_);
     }
 
     std::unique_ptr<typename gko::stop::ResidualNormReduction<T>::Factory>
         factory_;
-    std::shared_ptr<const gko::OmpExecutor> omp_;
+    std::shared_ptr<const gko::OmpExecutor> exec_;
 };
 
 TYPED_TEST_SUITE(ResidualNormReduction, gko::test::ValueTypes);
@@ -71,14 +71,14 @@ TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoal)
 {
     using Mtx = typename TestFixture::Mtx;
     using NormVector = typename TestFixture::NormVector;
-    auto initial_res = gko::initialize<Mtx>({100.0}, this->omp_);
-    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->omp_);
-    auto res_norm = gko::initialize<NormVector>({100.0}, this->omp_);
+    auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->exec_);
+    auto res_norm = gko::initialize<NormVector>({100.0}, this->exec_);
     auto criterion =
         this->factory_->generate(nullptr, rhs, nullptr, initial_res.get());
     bool one_changed{};
     constexpr gko::uint8 RelativeStoppingId{1};
-    gko::Array<gko::stopping_status> stop_status(this->omp_, 1);
+    gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
     stop_status.get_data()[0].reset();
 
     ASSERT_FALSE(
@@ -86,7 +86,7 @@ TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoal)
             .residual_norm(res_norm.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res_norm->at(0) = r<TypeParam>::value * 1.1e+2;
+    res_norm->at(0) = r<TypeParam>::value * 1.1 * res_norm->at(0);
     ASSERT_FALSE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -94,7 +94,7 @@ TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoal)
     ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
     ASSERT_EQ(one_changed, false);
 
-    res_norm->at(0) = r<TypeParam>::value * 0.9e+2;
+    res_norm->at(0) = r<TypeParam>::value * 0.9 * res_norm->at(0);
     ASSERT_TRUE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -110,15 +110,15 @@ TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
     using NormVector = typename TestFixture::NormVector;
     using T = TypeParam;
     using T_nc = gko::remove_complex<TypeParam>;
-    auto res = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->omp_);
+    auto res = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->exec_);
     auto res_norm =
-        gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->omp_);
+        gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
     std::shared_ptr<gko::LinOp> rhs =
-        gko::initialize<Mtx>({I<T>{10.0, 10.0}}, this->omp_);
+        gko::initialize<Mtx>({I<T>{10.0, 10.0}}, this->exec_);
     auto criterion = this->factory_->generate(nullptr, rhs, nullptr, res.get());
     bool one_changed{};
     constexpr gko::uint8 RelativeStoppingId{1};
-    gko::Array<gko::stopping_status> stop_status(this->omp_, 2);
+    gko::Array<gko::stopping_status> stop_status(this->exec_, 2);
     stop_status.get_data()[0].reset();
     stop_status.get_data()[1].reset();
 
@@ -127,7 +127,7 @@ TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
             .residual_norm(res_norm.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res_norm->at(0, 0) = r<TypeParam>::value * 0.9e+2;
+    res_norm->at(0, 0) = r<TypeParam>::value * 0.9 * res_norm->at(0, 0);
     ASSERT_FALSE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -135,7 +135,7 @@ TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
     ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
     ASSERT_EQ(one_changed, true);
 
-    res_norm->at(0, 1) = r<TypeParam>::value * 0.9e+2;
+    res_norm->at(0, 1) = r<TypeParam>::value * 0.9 * res_norm->at(0, 1);
     ASSERT_TRUE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -153,15 +153,15 @@ class RelativeResidualNorm : public ::testing::Test {
 
     RelativeResidualNorm()
     {
-        omp_ = gko::OmpExecutor::create();
+        exec_ = gko::OmpExecutor::create();
         factory_ = gko::stop::RelativeResidualNorm<T>::build()
                        .with_tolerance(r<T>::value)
-                       .on(omp_);
+                       .on(exec_);
     }
 
     std::unique_ptr<typename gko::stop::RelativeResidualNorm<T>::Factory>
         factory_;
-    std::shared_ptr<const gko::OmpExecutor> omp_;
+    std::shared_ptr<const gko::OmpExecutor> exec_;
 };
 
 TYPED_TEST_SUITE(RelativeResidualNorm, gko::test::ValueTypes);
@@ -169,16 +169,20 @@ TYPED_TEST_SUITE(RelativeResidualNorm, gko::test::ValueTypes);
 
 TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoal)
 {
+    using T = TypeParam;
+    using T_nc = gko::remove_complex<TypeParam>;
     using Mtx = typename TestFixture::Mtx;
     using NormVector = typename TestFixture::NormVector;
-    auto initial_res = gko::initialize<Mtx>({100.0}, this->omp_);
-    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->omp_);
-    auto res_norm = gko::initialize<NormVector>({100.0}, this->omp_);
+    auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->exec_);
+    auto rhs_norm = gko::initialize<NormVector>({I<T_nc>{0.0}}, this->exec_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+    auto res_norm = gko::initialize<NormVector>({100.0}, this->exec_);
     auto criterion =
         this->factory_->generate(nullptr, rhs, nullptr, initial_res.get());
     bool one_changed{};
     constexpr gko::uint8 RelativeStoppingId{1};
-    gko::Array<gko::stopping_status> stop_status(this->omp_, 1);
+    gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
     stop_status.get_data()[0].reset();
 
     ASSERT_FALSE(
@@ -186,7 +190,7 @@ TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoal)
             .residual_norm(res_norm.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res_norm->at(0) = r<TypeParam>::value * 1.1e+1;
+    res_norm->at(0) = r<TypeParam>::value * 1.1 * rhs_norm->at(0);
     ASSERT_FALSE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -194,7 +198,7 @@ TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoal)
     ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
     ASSERT_EQ(one_changed, false);
 
-    res_norm->at(0) = r<TypeParam>::value * 0.9e+1;
+    res_norm->at(0) = r<TypeParam>::value * 0.9 * rhs_norm->at(0);
     ASSERT_TRUE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -210,15 +214,18 @@ TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
     using NormVector = typename TestFixture::NormVector;
     using T = TypeParam;
     using T_nc = gko::remove_complex<TypeParam>;
-    auto res = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->omp_);
+    auto res = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->exec_);
     auto res_norm =
-        gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->omp_);
+        gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
     std::shared_ptr<gko::LinOp> rhs =
-        gko::initialize<Mtx>({I<T>{10.0, 10.0}}, this->omp_);
+        gko::initialize<Mtx>({I<T>{10.0, 10.0}}, this->exec_);
+    auto rhs_norm =
+        gko::initialize<NormVector>({I<T_nc>{0.0, 0.0}}, this->exec_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     auto criterion = this->factory_->generate(nullptr, rhs, nullptr, res.get());
     bool one_changed{};
     constexpr gko::uint8 RelativeStoppingId{1};
-    gko::Array<gko::stopping_status> stop_status(this->omp_, 2);
+    gko::Array<gko::stopping_status> stop_status(this->exec_, 2);
     stop_status.get_data()[0].reset();
     stop_status.get_data()[1].reset();
 
@@ -227,7 +234,7 @@ TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
             .residual_norm(res_norm.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res_norm->at(0, 0) = r<TypeParam>::value * 0.9e+1;
+    res_norm->at(0, 0) = r<TypeParam>::value * 0.9 * rhs_norm->at(0, 0);
     ASSERT_FALSE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -235,7 +242,7 @@ TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
     ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
     ASSERT_EQ(one_changed, true);
 
-    res_norm->at(0, 1) = r<TypeParam>::value * 0.9e+1;
+    res_norm->at(0, 1) = r<TypeParam>::value * 0.9 * rhs_norm->at(0, 1);
     ASSERT_TRUE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -268,40 +275,17 @@ class ImplicitResidualNormReduction : public ::testing::Test {
 TYPED_TEST_SUITE(ImplicitResidualNormReduction, gko::test::ValueTypes);
 
 
-TYPED_TEST(ImplicitResidualNormReduction, CanCreateFactory)
-{
-    ASSERT_NE(this->factory_, nullptr);
-    ASSERT_EQ(this->factory_->get_parameters().reduction_factor,
-              r<TypeParam>::value);
-    ASSERT_EQ(this->factory_->get_executor(), this->exec_);
-}
-
-
-TYPED_TEST(ImplicitResidualNormReduction, CannotCreateCriterionWithoutB)
-{
-    ASSERT_THROW(this->factory_->generate(nullptr, nullptr, nullptr, nullptr),
-                 gko::NotSupported);
-}
-
-
-TYPED_TEST(ImplicitResidualNormReduction, CanCreateCriterionWithB)
-{
-    using Mtx = typename TestFixture::Mtx;
-    std::shared_ptr<gko::LinOp> scalar =
-        gko::initialize<Mtx>({1.0}, this->exec_);
-    auto criterion =
-        this->factory_->generate(nullptr, scalar, nullptr, nullptr);
-    ASSERT_NE(criterion, nullptr);
-}
-
-
 TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoal)
 {
+    using T = TypeParam;
+    using T_nc = gko::remove_complex<TypeParam>;
     using Mtx = typename TestFixture::Mtx;
     using NormVector = typename TestFixture::NormVector;
     auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
     std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->exec_);
-    auto res_norm = gko::initialize<NormVector>({100.0}, this->exec_);
+    auto res_norm = gko::initialize<Mtx>({100.0}, this->exec_);
+    auto rhs_norm = gko::initialize<NormVector>({I<T_nc>{0.0}}, this->exec_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     auto criterion =
         this->factory_->generate(nullptr, rhs, nullptr, initial_res.get());
     bool one_changed{};
@@ -314,7 +298,7 @@ TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoal)
             .implicit_sq_residual_norm(res_norm.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res_norm->at(0) = std::pow(r<TypeParam>::value * 1.1e+1, 2);
+    res_norm->at(0) = std::pow(r<TypeParam>::value * 1.1 * rhs_norm->at(0), 2);
     ASSERT_FALSE(
         criterion->update()
             .implicit_sq_residual_norm(res_norm.get())
@@ -322,7 +306,7 @@ TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoal)
     ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
     ASSERT_EQ(one_changed, false);
 
-    res_norm->at(0) = std::pow(r<TypeParam>::value * 0.9e+1, 2);
+    res_norm->at(0) = std::pow(r<TypeParam>::value * 0.9 * rhs_norm->at(0), 2);
     ASSERT_TRUE(
         criterion->update()
             .implicit_sq_residual_norm(res_norm.get())
@@ -339,10 +323,12 @@ TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
     using T = TypeParam;
     using T_nc = gko::remove_complex<TypeParam>;
     auto res = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->exec_);
-    auto res_norm =
-        gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
+    auto res_norm = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->exec_);
     std::shared_ptr<gko::LinOp> rhs =
         gko::initialize<Mtx>({I<T>{10.0, 10.0}}, this->exec_);
+    auto rhs_norm =
+        gko::initialize<NormVector>({I<T_nc>{0.0, 0.0}}, this->exec_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     auto criterion = this->factory_->generate(nullptr, rhs, nullptr, res.get());
     bool one_changed{};
     constexpr gko::uint8 RelativeStoppingId{1};
@@ -355,7 +341,8 @@ TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
             .implicit_sq_residual_norm(res_norm.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res_norm->at(0, 0) = std::pow(r<TypeParam>::value * 0.9e+1, 2);
+    res_norm->at(0, 0) =
+        std::pow(r<TypeParam>::value * 0.9 * rhs_norm->at(0, 0), 2);
     ASSERT_FALSE(
         criterion->update()
             .implicit_sq_residual_norm(res_norm.get())
@@ -363,7 +350,8 @@ TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
     ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
     ASSERT_EQ(one_changed, true);
 
-    res_norm->at(0, 1) = std::pow(r<TypeParam>::value * 0.9e+1, 2);
+    res_norm->at(0, 1) =
+        std::pow(r<TypeParam>::value * 0.9 * rhs_norm->at(0, 1), 2);
     ASSERT_TRUE(
         criterion->update()
             .implicit_sq_residual_norm(res_norm.get())
@@ -381,15 +369,15 @@ class AbsoluteResidualNorm : public ::testing::Test {
 
     AbsoluteResidualNorm()
     {
-        omp_ = gko::OmpExecutor::create();
+        exec_ = gko::OmpExecutor::create();
         factory_ = gko::stop::AbsoluteResidualNorm<T>::build()
                        .with_tolerance(r<T>::value)
-                       .on(omp_);
+                       .on(exec_);
     }
 
     std::unique_ptr<typename gko::stop::AbsoluteResidualNorm<T>::Factory>
         factory_;
-    std::shared_ptr<const gko::OmpExecutor> omp_;
+    std::shared_ptr<const gko::OmpExecutor> exec_;
 };
 
 TYPED_TEST_SUITE(AbsoluteResidualNorm, gko::test::ValueTypes);
@@ -399,14 +387,14 @@ TYPED_TEST(AbsoluteResidualNorm, WaitsTillResidualGoal)
 {
     using Mtx = typename TestFixture::Mtx;
     using NormVector = typename TestFixture::NormVector;
-    auto initial_res = gko::initialize<Mtx>({100.0}, this->omp_);
-    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->omp_);
-    auto res_norm = gko::initialize<NormVector>({100.0}, this->omp_);
+    auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->exec_);
+    auto res_norm = gko::initialize<NormVector>({100.0}, this->exec_);
     auto criterion =
         this->factory_->generate(nullptr, rhs, nullptr, initial_res.get());
     bool one_changed{};
     constexpr gko::uint8 RelativeStoppingId{1};
-    gko::Array<gko::stopping_status> stop_status(this->omp_, 1);
+    gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
     stop_status.get_data()[0].reset();
 
     ASSERT_FALSE(
@@ -438,15 +426,15 @@ TYPED_TEST(AbsoluteResidualNorm, WaitsTillResidualGoalMultipleRHS)
     using NormVector = typename TestFixture::NormVector;
     using T = TypeParam;
     using T_nc = gko::remove_complex<TypeParam>;
-    auto res = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->omp_);
+    auto res = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->exec_);
     auto res_norm =
-        gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->omp_);
+        gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
     std::shared_ptr<gko::LinOp> rhs =
-        gko::initialize<Mtx>({I<T>{10.0, 10.0}}, this->omp_);
+        gko::initialize<Mtx>({I<T>{10.0, 10.0}}, this->exec_);
     auto criterion = this->factory_->generate(nullptr, rhs, nullptr, res.get());
     bool one_changed{};
     constexpr gko::uint8 RelativeStoppingId{1};
-    gko::Array<gko::stopping_status> stop_status(this->omp_, 2);
+    gko::Array<gko::stopping_status> stop_status(this->exec_, 2);
     stop_status.get_data()[0].reset();
     stop_status.get_data()[1].reset();
 
diff --git a/reference/stop/residual_norm_kernels.cpp b/reference/stop/residual_norm_kernels.cpp
index 3e6301f8ae1..b1f25b52c29 100644
--- a/reference/stop/residual_norm_kernels.cpp
+++ b/reference/stop/residual_norm_kernels.cpp
@@ -96,21 +96,18 @@ namespace implicit_residual_norm {
 
 
 template <typename ValueType>
-void implicit_residual_norm(std::shared_ptr<const ReferenceExecutor> exec,
-                            const matrix::Dense<ValueType> *tau,
-                            const matrix::Dense<ValueType> *orig_tau,
-                            ValueType rel_residual_goal, uint8 stoppingId,
-                            bool setFinalized,
-                            Array<stopping_status> *stop_status,
-                            Array<bool> *device_storage, bool *all_converged,
-                            bool *one_changed)
+void implicit_residual_norm(
+    std::shared_ptr<const ReferenceExecutor> exec,
+    const matrix::Dense<ValueType> *tau,
+    const matrix::Dense<remove_complex<ValueType>> *orig_tau,
+    remove_complex<ValueType> rel_residual_goal, uint8 stoppingId,
+    bool setFinalized, Array<stopping_status> *stop_status,
+    Array<bool> *device_storage, bool *all_converged, bool *one_changed)
 {
-    static_assert(is_complex_s<ValueType>::value == false,
-                  "ValueType must not be complex in this function!");
     *all_converged = true;
     *one_changed = false;
     for (size_type i = 0; i < tau->get_size()[1]; ++i) {
-        if (sqrt(tau->at(i)) < rel_residual_goal * orig_tau->at(i)) {
+        if (sqrt(abs(tau->at(i))) < rel_residual_goal * orig_tau->at(i)) {
             stop_status->get_data()[i].converge(stoppingId, setFinalized);
             *one_changed = true;
         }
@@ -123,8 +120,7 @@ void implicit_residual_norm(std::shared_ptr<const ReferenceExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
-    GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
 
 
 }  // namespace implicit_residual_norm
diff --git a/reference/test/stop/residual_norm_kernels.cpp b/reference/test/stop/residual_norm_kernels.cpp
index a443d0de858..f2c280c4d3a 100644
--- a/reference/test/stop/residual_norm_kernels.cpp
+++ b/reference/test/stop/residual_norm_kernels.cpp
@@ -94,6 +94,7 @@ TYPED_TEST(ResidualNormReduction, CanCreateCriterionWithB)
         gko::initialize<Mtx>({1.0}, this->exec_);
     auto criterion =
         this->factory_->generate(nullptr, nullptr, nullptr, scalar.get());
+
     ASSERT_NE(criterion, nullptr);
 }
 
@@ -117,7 +118,7 @@ TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoal)
             .residual_norm(res_norm.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res_norm->at(0) = r<TypeParam>::value * 1.1e+2;
+    res_norm->at(0) = r<TypeParam>::value * 1.1 * res_norm->at(0);
     ASSERT_FALSE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -125,7 +126,7 @@ TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoal)
     ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
     ASSERT_EQ(one_changed, false);
 
-    res_norm->at(0) = r<TypeParam>::value * 0.9e+2;
+    res_norm->at(0) = r<TypeParam>::value * 0.9 * res_norm->at(0);
     ASSERT_TRUE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -158,7 +159,7 @@ TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
             .residual_norm(res_norm.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res_norm->at(0, 0) = r<TypeParam>::value * 0.9e+2;
+    res_norm->at(0, 0) = r<TypeParam>::value * 0.9 * res_norm->at(0, 0);
     ASSERT_FALSE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -166,7 +167,7 @@ TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
     ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
     ASSERT_EQ(one_changed, true);
 
-    res_norm->at(0, 1) = r<TypeParam>::value * 0.9e+2;
+    res_norm->at(0, 1) = r<TypeParam>::value * 0.9 * res_norm->at(0, 1);
     ASSERT_TRUE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -220,16 +221,21 @@ TYPED_TEST(RelativeResidualNorm, CanCreateCriterionWithB)
         gko::initialize<Mtx>({1.0}, this->exec_);
     auto criterion =
         this->factory_->generate(nullptr, scalar, nullptr, nullptr);
+
     ASSERT_NE(criterion, nullptr);
 }
 
 
 TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoal)
 {
+    using T = TypeParam;
+    using T_nc = gko::remove_complex<TypeParam>;
     using Mtx = typename TestFixture::Mtx;
     using NormVector = typename TestFixture::NormVector;
     auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
     std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->exec_);
+    auto rhs_norm = gko::initialize<NormVector>({I<T_nc>{0.0}}, this->exec_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     auto res_norm = gko::initialize<NormVector>({100.0}, this->exec_);
     auto criterion =
         this->factory_->generate(nullptr, rhs, nullptr, initial_res.get());
@@ -243,7 +249,7 @@ TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoal)
             .residual_norm(res_norm.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res_norm->at(0) = r<TypeParam>::value * 1.1e+1;
+    res_norm->at(0) = r<TypeParam>::value * 1.1 * rhs_norm->at(0);
     ASSERT_FALSE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -251,7 +257,7 @@ TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoal)
     ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
     ASSERT_EQ(one_changed, false);
 
-    res_norm->at(0) = r<TypeParam>::value * 0.9e+1;
+    res_norm->at(0) = r<TypeParam>::value * 0.9 * rhs_norm->at(0);
     ASSERT_TRUE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -272,6 +278,9 @@ TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
         gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
     std::shared_ptr<gko::LinOp> rhs =
         gko::initialize<Mtx>({I<T>{10.0, 10.0}}, this->exec_);
+    auto rhs_norm =
+        gko::initialize<NormVector>({I<T_nc>{0.0, 0.0}}, this->exec_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     auto criterion = this->factory_->generate(nullptr, rhs, nullptr, res.get());
     bool one_changed{};
     constexpr gko::uint8 RelativeStoppingId{1};
@@ -284,7 +293,7 @@ TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
             .residual_norm(res_norm.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res_norm->at(0, 0) = r<TypeParam>::value * 0.9e+1;
+    res_norm->at(0, 0) = r<TypeParam>::value * 0.9 * rhs_norm->at(0, 0);
     ASSERT_FALSE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -292,7 +301,7 @@ TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
     ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
     ASSERT_EQ(one_changed, true);
 
-    res_norm->at(0, 1) = r<TypeParam>::value * 0.9e+1;
+    res_norm->at(0, 1) = r<TypeParam>::value * 0.9 * rhs_norm->at(0, 1);
     ASSERT_TRUE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -316,11 +325,11 @@ class ImplicitResidualNormReduction : public ::testing::Test {
                        .on(exec_);
         factory_2_ = gko::stop::ImplicitResidualNormReduction<T>::build()
                          .with_reduction_factor(r<T>::value)
-                         .with_relative_to("initial_residual")
+                         .with_relative_to(gko::stop::norm_of::initial_residual)
                          .on(exec_);
         factory_3_ = gko::stop::ImplicitResidualNormReduction<T>::build()
                          .with_reduction_factor(r<T>::value)
-                         .with_relative_to("rhs")
+                         .with_relative_to(gko::stop::norm_of::rhs)
                          .on(exec_);
     }
 
@@ -344,10 +353,12 @@ TYPED_TEST(ImplicitResidualNormReduction, CanCreateFactory)
     ASSERT_NE(this->factory_, nullptr);
     ASSERT_EQ(this->factory_->get_parameters().reduction_factor,
               r<TypeParam>::value);
-    ASSERT_EQ(this->factory_->get_parameters().relative_to, "rhs");
+    ASSERT_EQ(this->factory_->get_parameters().relative_to,
+              gko::stop::norm_of::rhs);
     ASSERT_EQ(this->factory_2_->get_parameters().relative_to,
-              "initial_residual");
-    ASSERT_EQ(this->factory_3_->get_parameters().relative_to, "rhs");
+              gko::stop::norm_of::initial_residual);
+    ASSERT_EQ(this->factory_3_->get_parameters().relative_to,
+              gko::stop::norm_of::rhs);
     ASSERT_EQ(this->factory_->get_executor(), this->exec_);
 }
 
@@ -367,6 +378,7 @@ TYPED_TEST(ImplicitResidualNormReduction, CanCreateCriterionWithB)
         gko::initialize<Mtx>({1.0}, this->exec_);
     auto criterion =
         this->factory_->generate(nullptr, scalar, nullptr, nullptr);
+
     ASSERT_NE(criterion, nullptr);
 }
 
@@ -383,11 +395,15 @@ TYPED_TEST(ImplicitResidualNormReduction, CanCreateCriterionWithInitialRes)
 
 TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoal)
 {
+    using T = TypeParam;
+    using T_nc = gko::remove_complex<TypeParam>;
     using Mtx = typename TestFixture::Mtx;
     using NormVector = typename TestFixture::NormVector;
     auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
     std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->exec_);
-    auto res_norm = gko::initialize<NormVector>({100.0}, this->exec_);
+    auto res_norm = gko::initialize<Mtx>({100.0}, this->exec_);
+    auto rhs_norm = gko::initialize<NormVector>({I<T_nc>{0.0}}, this->exec_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     auto criterion =
         this->factory_->generate(nullptr, rhs, nullptr, initial_res.get());
     bool one_changed{};
@@ -400,7 +416,7 @@ TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoal)
             .implicit_sq_residual_norm(res_norm.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res_norm->at(0) = std::pow(r<TypeParam>::value * 1.1e+1, 2);
+    res_norm->at(0) = std::pow(r<TypeParam>::value * 1.1 * rhs_norm->at(0), 2);
     ASSERT_FALSE(
         criterion->update()
             .implicit_sq_residual_norm(res_norm.get())
@@ -408,7 +424,7 @@ TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoal)
     ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
     ASSERT_EQ(one_changed, false);
 
-    res_norm->at(0) = std::pow(r<TypeParam>::value * 0.9e+1, 2);
+    res_norm->at(0) = std::pow(r<TypeParam>::value * 0.9 * rhs_norm->at(0), 2);
     ASSERT_TRUE(
         criterion->update()
             .implicit_sq_residual_norm(res_norm.get())
@@ -425,10 +441,12 @@ TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
     using T = TypeParam;
     using T_nc = gko::remove_complex<TypeParam>;
     auto res = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->exec_);
-    auto res_norm =
-        gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
+    auto res_norm = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->exec_);
     std::shared_ptr<gko::LinOp> rhs =
         gko::initialize<Mtx>({I<T>{10.0, 10.0}}, this->exec_);
+    auto rhs_norm =
+        gko::initialize<NormVector>({I<T_nc>{0.0, 0.0}}, this->exec_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
     auto criterion = this->factory_->generate(nullptr, rhs, nullptr, res.get());
     bool one_changed{};
     constexpr gko::uint8 RelativeStoppingId{1};
@@ -441,7 +459,8 @@ TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
             .implicit_sq_residual_norm(res_norm.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res_norm->at(0, 0) = std::pow(r<TypeParam>::value * 0.9e+1, 2);
+    res_norm->at(0, 0) =
+        std::pow(r<TypeParam>::value * 0.9 * rhs_norm->at(0, 0), 2);
     ASSERT_FALSE(
         criterion->update()
             .implicit_sq_residual_norm(res_norm.get())
@@ -449,7 +468,8 @@ TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
     ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
     ASSERT_EQ(one_changed, true);
 
-    res_norm->at(0, 1) = std::pow(r<TypeParam>::value * 0.9e+1, 2);
+    res_norm->at(0, 1) =
+        std::pow(r<TypeParam>::value * 0.9 * rhs_norm->at(0, 1), 2);
     ASSERT_TRUE(
         criterion->update()
             .implicit_sq_residual_norm(res_norm.get())
@@ -503,6 +523,7 @@ TYPED_TEST(AbsoluteResidualNorm, CanCreateCriterionWithB)
         gko::initialize<Mtx>({1.0}, this->exec_);
     auto criterion =
         this->factory_->generate(nullptr, scalar, nullptr, nullptr);
+
     ASSERT_NE(criterion, nullptr);
 }
 

From b26a69c318f5e6b438e6687915f099d5adcea034 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 10 Feb 2021 11:16:52 +0100
Subject: [PATCH 0323/1008] Change class name to ImplicitResidualNorm.

---
 core/stop/residual_norm.cpp                   |  4 +--
 cuda/test/stop/residual_norm_kernels.cpp      | 13 ++++---
 hip/test/stop/residual_norm_kernels.cpp       | 13 ++++---
 include/ginkgo/core/stop/residual_norm.hpp    | 17 +++++-----
 omp/test/stop/residual_norm_kernels.cpp       | 15 ++++----
 reference/test/solver/bicg_kernels.cpp        |  3 +-
 reference/test/solver/bicgstab_kernels.cpp    |  3 +-
 reference/test/solver/cg_kernels.cpp          |  3 +-
 reference/test/solver/cgs_kernels.cpp         |  3 +-
 reference/test/solver/fcg_kernels.cpp         |  3 +-
 reference/test/solver/gmres_kernels.cpp       |  3 +-
 reference/test/stop/residual_norm_kernels.cpp | 34 ++++++++-----------
 12 files changed, 50 insertions(+), 64 deletions(-)

diff --git a/core/stop/residual_norm.cpp b/core/stop/residual_norm.cpp
index b5db82c920f..6a94ffe544a 100644
--- a/core/stop/residual_norm.cpp
+++ b/core/stop/residual_norm.cpp
@@ -91,7 +91,7 @@ bool ResidualNorm<ValueType>::check_impl(uint8 stoppingId, bool setFinalized,
 
 
 template <typename ValueType>
-bool ImplicitResidualNormReduction<ValueType>::check_impl(
+bool ImplicitResidualNorm<ValueType>::check_impl(
     uint8 stoppingId, bool setFinalized, Array<stopping_status> *stop_status,
     bool *one_changed, const Criterion::Updater &updater)
 {
@@ -127,7 +127,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_RESIDUAL_NORM);
 
 
 #define GKO_DECLARE_IMPLICIT_RESIDUAL_NORM(_type) \
-    class ImplicitResidualNormReduction<_type>
+    class ImplicitResidualNorm<_type>
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IMPLICIT_RESIDUAL_NORM);
 
 
diff --git a/cuda/test/stop/residual_norm_kernels.cpp b/cuda/test/stop/residual_norm_kernels.cpp
index c96de7df739..c9a0518a328 100644
--- a/cuda/test/stop/residual_norm_kernels.cpp
+++ b/cuda/test/stop/residual_norm_kernels.cpp
@@ -269,28 +269,27 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
 }
 
 
-class ImplicitResidualNormReduction : public ::testing::Test {
+class ImplicitResidualNorm : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
     using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
 
-    ImplicitResidualNormReduction()
+    ImplicitResidualNorm()
     {
         ref_ = gko::ReferenceExecutor::create();
         cuda_ = gko::CudaExecutor::create(0, ref_);
-        factory_ = gko::stop::ImplicitResidualNormReduction<>::build()
+        factory_ = gko::stop::ImplicitResidualNorm<>::build()
                        .with_reduction_factor(tol)
                        .on(cuda_);
     }
 
-    std::unique_ptr<gko::stop::ImplicitResidualNormReduction<>::Factory>
-        factory_;
+    std::unique_ptr<gko::stop::ImplicitResidualNorm<>::Factory> factory_;
     std::shared_ptr<const gko::CudaExecutor> cuda_;
     std::shared_ptr<gko::ReferenceExecutor> ref_;
 };
 
 
-TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoal)
+TEST_F(ImplicitResidualNorm, WaitsTillResidualGoal)
 {
     auto res = gko::initialize<Mtx>({100.0}, ref_);
     auto d_res = Mtx::create(cuda_);
@@ -335,7 +334,7 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoal)
 }
 
 
-TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
+TEST_F(ImplicitResidualNorm, WaitsTillResidualGoalMultipleRHS)
 {
     auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
     auto d_res = Mtx::create(cuda_);
diff --git a/hip/test/stop/residual_norm_kernels.cpp b/hip/test/stop/residual_norm_kernels.cpp
index 2a6fd1a5b42..32c5b4b32f7 100644
--- a/hip/test/stop/residual_norm_kernels.cpp
+++ b/hip/test/stop/residual_norm_kernels.cpp
@@ -269,28 +269,27 @@ TEST_F(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
 }
 
 
-class ImplicitResidualNormReduction : public ::testing::Test {
+class ImplicitResidualNorm : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
     using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
 
-    ImplicitResidualNormReduction()
+    ImplicitResidualNorm()
     {
         ref_ = gko::ReferenceExecutor::create();
         hip_ = gko::HipExecutor::create(0, ref_);
-        factory_ = gko::stop::ImplicitResidualNormReduction<>::build()
+        factory_ = gko::stop::ImplicitResidualNorm<>::build()
                        .with_reduction_factor(tol)
                        .on(hip_);
     }
 
-    std::unique_ptr<gko::stop::ImplicitResidualNormReduction<>::Factory>
-        factory_;
+    std::unique_ptr<gko::stop::ImplicitResidualNorm<>::Factory> factory_;
     std::shared_ptr<const gko::HipExecutor> hip_;
     std::shared_ptr<gko::ReferenceExecutor> ref_;
 };
 
 
-TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoal)
+TEST_F(ImplicitResidualNorm, WaitsTillResidualGoal)
 {
     auto res = gko::initialize<Mtx>({100.0}, ref_);
     auto d_res = Mtx::create(hip_);
@@ -335,7 +334,7 @@ TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoal)
 }
 
 
-TEST_F(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
+TEST_F(ImplicitResidualNorm, WaitsTillResidualGoalMultipleRHS)
 {
     auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
     auto d_res = Mtx::create(hip_);
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index f05257d98e7..a67b00f187b 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -229,10 +229,10 @@ class RelativeResidualNorm : public ResidualNorm<ValueType> {
 /**
  * The enum that holds the quantity relative to the implicit residual.
  */
-enum class norm_of { rhs, initial_residual };
+enum class norm_of { initial_residual, rhs };
 
 /**
- * The ImplicitResidualNormReduction class is a stopping criterion which
+ * The ImplicitResidualNorm class is a stopping criterion which
  * stops the iteration process when the implicit residual norm is below a
  * certain threshold relative to either the norm of the right-hand side, i.e.
  * when norm(residual) / norm(right_hand_side) < threshold or the initial
@@ -248,7 +248,7 @@ enum class norm_of { rhs, initial_residual };
  * @ingroup stop
  */
 template <typename ValueType = default_precision>
-class ImplicitResidualNormReduction : public ResidualNorm<ValueType> {
+class ImplicitResidualNorm : public ResidualNorm<ValueType> {
 public:
     using ComplexVector = matrix::Dense<to_complex<ValueType>>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
@@ -268,8 +268,8 @@ class ImplicitResidualNormReduction : public ResidualNorm<ValueType> {
          */
         norm_of GKO_FACTORY_PARAMETER_SCALAR(relative_to, norm_of::rhs);
     };
-    GKO_ENABLE_CRITERION_FACTORY(ImplicitResidualNormReduction<ValueType>,
-                                 parameters, Factory);
+    GKO_ENABLE_CRITERION_FACTORY(ImplicitResidualNorm<ValueType>, parameters,
+                                 Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);
 
 protected:
@@ -277,13 +277,12 @@ class ImplicitResidualNormReduction : public ResidualNorm<ValueType> {
                     Array<stopping_status> *stop_status, bool *one_changed,
                     const Criterion::Updater &) override;
 
-    explicit ImplicitResidualNormReduction(
-        std::shared_ptr<const gko::Executor> exec)
+    explicit ImplicitResidualNorm(std::shared_ptr<const gko::Executor> exec)
         : ResidualNorm<ValueType>(exec), device_storage_{exec, 2}
     {}
 
-    explicit ImplicitResidualNormReduction(const Factory *factory,
-                                           const CriterionArgs &args)
+    explicit ImplicitResidualNorm(const Factory *factory,
+                                  const CriterionArgs &args)
         : ResidualNorm<ValueType>(factory->get_executor(),
                                   factory->get_parameters().reduction_factor),
           parameters_{factory->get_parameters()},
diff --git a/omp/test/stop/residual_norm_kernels.cpp b/omp/test/stop/residual_norm_kernels.cpp
index 5eba7d93851..9e025c69508 100644
--- a/omp/test/stop/residual_norm_kernels.cpp
+++ b/omp/test/stop/residual_norm_kernels.cpp
@@ -253,29 +253,28 @@ TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
 
 
 template <typename T>
-class ImplicitResidualNormReduction : public ::testing::Test {
+class ImplicitResidualNorm : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<T>;
     using NormVector = gko::matrix::Dense<gko::remove_complex<T>>;
 
-    ImplicitResidualNormReduction()
+    ImplicitResidualNorm()
     {
         exec_ = gko::OmpExecutor::create();
-        factory_ = gko::stop::ImplicitResidualNormReduction<T>::build()
+        factory_ = gko::stop::ImplicitResidualNorm<T>::build()
                        .with_reduction_factor(r<T>::value)
                        .on(exec_);
     }
 
-    std::unique_ptr<
-        typename gko::stop::ImplicitResidualNormReduction<T>::Factory>
+    std::unique_ptr<typename gko::stop::ImplicitResidualNorm<T>::Factory>
         factory_;
     std::shared_ptr<const gko::Executor> exec_;
 };
 
-TYPED_TEST_SUITE(ImplicitResidualNormReduction, gko::test::ValueTypes);
+TYPED_TEST_SUITE(ImplicitResidualNorm, gko::test::ValueTypes);
 
 
-TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoal)
+TYPED_TEST(ImplicitResidualNorm, WaitsTillResidualGoal)
 {
     using T = TypeParam;
     using T_nc = gko::remove_complex<TypeParam>;
@@ -316,7 +315,7 @@ TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoal)
 }
 
 
-TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
+TYPED_TEST(ImplicitResidualNorm, WaitsTillResidualGoalMultipleRHS)
 {
     using Mtx = typename TestFixture::Mtx;
     using NormVector = typename TestFixture::NormVector;
diff --git a/reference/test/solver/bicg_kernels.cpp b/reference/test/solver/bicg_kernels.cpp
index 5b54357245a..5af8b8d0e22 100644
--- a/reference/test/solver/bicg_kernels.cpp
+++ b/reference/test/solver/bicg_kernels.cpp
@@ -94,8 +94,7 @@ class Bicg : public ::testing::Test {
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
-                      gko::stop::ImplicitResidualNormReduction<
-                          value_type>::build()
+                      gko::stop::ImplicitResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
diff --git a/reference/test/solver/bicgstab_kernels.cpp b/reference/test/solver/bicgstab_kernels.cpp
index 3189560acf2..a95099589c1 100644
--- a/reference/test/solver/bicgstab_kernels.cpp
+++ b/reference/test/solver/bicgstab_kernels.cpp
@@ -80,8 +80,7 @@ class Bicgstab : public ::testing::Test {
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
-                      gko::stop::ImplicitResidualNormReduction<
-                          value_type>::build()
+                      gko::stop::ImplicitResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
diff --git a/reference/test/solver/cg_kernels.cpp b/reference/test/solver/cg_kernels.cpp
index 6d49edb76de..5209a96864b 100644
--- a/reference/test/solver/cg_kernels.cpp
+++ b/reference/test/solver/cg_kernels.cpp
@@ -95,8 +95,7 @@ class Cg : public ::testing::Test {
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
-                      gko::stop::ImplicitResidualNormReduction<
-                          value_type>::build()
+                      gko::stop::ImplicitResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec))
diff --git a/reference/test/solver/cgs_kernels.cpp b/reference/test/solver/cgs_kernels.cpp
index 83612a8cb44..46954492f90 100644
--- a/reference/test/solver/cgs_kernels.cpp
+++ b/reference/test/solver/cgs_kernels.cpp
@@ -93,8 +93,7 @@ class Cgs : public ::testing::Test {
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
-                      gko::stop::ImplicitResidualNormReduction<
-                          value_type>::build()
+                      gko::stop::ImplicitResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec))
diff --git a/reference/test/solver/fcg_kernels.cpp b/reference/test/solver/fcg_kernels.cpp
index aa311c37fbe..bd4d3324f2f 100644
--- a/reference/test/solver/fcg_kernels.cpp
+++ b/reference/test/solver/fcg_kernels.cpp
@@ -95,8 +95,7 @@ class Fcg : public ::testing::Test {
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
-                      gko::stop::ImplicitResidualNormReduction<
-                          value_type>::build()
+                      gko::stop::ImplicitResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec))
diff --git a/reference/test/solver/gmres_kernels.cpp b/reference/test/solver/gmres_kernels.cpp
index b43cb8c38a8..6ccabd9a7a7 100644
--- a/reference/test/solver/gmres_kernels.cpp
+++ b/reference/test/solver/gmres_kernels.cpp
@@ -96,8 +96,7 @@ class Gmres : public ::testing::Test {
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
-                      gko::stop::ImplicitResidualNormReduction<
-                          value_type>::build()
+                      gko::stop::ImplicitResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
diff --git a/reference/test/stop/residual_norm_kernels.cpp b/reference/test/stop/residual_norm_kernels.cpp
index f2c280c4d3a..ba01fbbea91 100644
--- a/reference/test/stop/residual_norm_kernels.cpp
+++ b/reference/test/stop/residual_norm_kernels.cpp
@@ -312,43 +312,40 @@ TYPED_TEST(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
 
 
 template <typename T>
-class ImplicitResidualNormReduction : public ::testing::Test {
+class ImplicitResidualNorm : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<T>;
     using NormVector = gko::matrix::Dense<gko::remove_complex<T>>;
 
-    ImplicitResidualNormReduction()
+    ImplicitResidualNorm()
     {
         exec_ = gko::ReferenceExecutor::create();
-        factory_ = gko::stop::ImplicitResidualNormReduction<T>::build()
+        factory_ = gko::stop::ImplicitResidualNorm<T>::build()
                        .with_reduction_factor(r<T>::value)
                        .on(exec_);
-        factory_2_ = gko::stop::ImplicitResidualNormReduction<T>::build()
+        factory_2_ = gko::stop::ImplicitResidualNorm<T>::build()
                          .with_reduction_factor(r<T>::value)
                          .with_relative_to(gko::stop::norm_of::initial_residual)
                          .on(exec_);
-        factory_3_ = gko::stop::ImplicitResidualNormReduction<T>::build()
+        factory_3_ = gko::stop::ImplicitResidualNorm<T>::build()
                          .with_reduction_factor(r<T>::value)
                          .with_relative_to(gko::stop::norm_of::rhs)
                          .on(exec_);
     }
 
-    std::unique_ptr<
-        typename gko::stop::ImplicitResidualNormReduction<T>::Factory>
+    std::unique_ptr<typename gko::stop::ImplicitResidualNorm<T>::Factory>
         factory_;
-    std::unique_ptr<
-        typename gko::stop::ImplicitResidualNormReduction<T>::Factory>
+    std::unique_ptr<typename gko::stop::ImplicitResidualNorm<T>::Factory>
         factory_2_;
-    std::unique_ptr<
-        typename gko::stop::ImplicitResidualNormReduction<T>::Factory>
+    std::unique_ptr<typename gko::stop::ImplicitResidualNorm<T>::Factory>
         factory_3_;
     std::shared_ptr<const gko::Executor> exec_;
 };
 
-TYPED_TEST_SUITE(ImplicitResidualNormReduction, gko::test::ValueTypes);
+TYPED_TEST_SUITE(ImplicitResidualNorm, gko::test::ValueTypes);
 
 
-TYPED_TEST(ImplicitResidualNormReduction, CanCreateFactory)
+TYPED_TEST(ImplicitResidualNorm, CanCreateFactory)
 {
     ASSERT_NE(this->factory_, nullptr);
     ASSERT_EQ(this->factory_->get_parameters().reduction_factor,
@@ -363,15 +360,14 @@ TYPED_TEST(ImplicitResidualNormReduction, CanCreateFactory)
 }
 
 
-TYPED_TEST(ImplicitResidualNormReduction,
-           CannotCreateCriterionWithoutBAndInitRes)
+TYPED_TEST(ImplicitResidualNorm, CannotCreateCriterionWithoutBAndInitRes)
 {
     ASSERT_THROW(this->factory_->generate(nullptr, nullptr, nullptr, nullptr),
                  gko::NotSupported);
 }
 
 
-TYPED_TEST(ImplicitResidualNormReduction, CanCreateCriterionWithB)
+TYPED_TEST(ImplicitResidualNorm, CanCreateCriterionWithB)
 {
     using Mtx = typename TestFixture::Mtx;
     std::shared_ptr<gko::LinOp> scalar =
@@ -383,7 +379,7 @@ TYPED_TEST(ImplicitResidualNormReduction, CanCreateCriterionWithB)
 }
 
 
-TYPED_TEST(ImplicitResidualNormReduction, CanCreateCriterionWithInitialRes)
+TYPED_TEST(ImplicitResidualNorm, CanCreateCriterionWithInitialRes)
 {
     using Mtx = typename TestFixture::Mtx;
     auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
@@ -393,7 +389,7 @@ TYPED_TEST(ImplicitResidualNormReduction, CanCreateCriterionWithInitialRes)
 }
 
 
-TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoal)
+TYPED_TEST(ImplicitResidualNorm, WaitsTillResidualGoal)
 {
     using T = TypeParam;
     using T_nc = gko::remove_complex<TypeParam>;
@@ -434,7 +430,7 @@ TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoal)
 }
 
 
-TYPED_TEST(ImplicitResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
+TYPED_TEST(ImplicitResidualNorm, WaitsTillResidualGoalMultipleRHS)
 {
     using Mtx = typename TestFixture::Mtx;
     using NormVector = typename TestFixture::NormVector;

From 9c95b2439b178b4500e66ea375c5434bc0d533f3 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 10 Feb 2021 18:06:49 +0100
Subject: [PATCH 0324/1008] Add a Array and Dense fill helper function.

---
 core/base/array.cpp                  | 25 +++++++++++++++++++++++++
 core/components/fill_array.hpp       |  4 ++--
 core/matrix/dense.cpp                |  2 --
 core/test/base/array.cpp             | 12 ++++++++++++
 core/test/matrix/dense.cpp           | 15 +++++++++++++++
 cuda/components/fill_array.cu        |  4 +---
 dpcpp/components/fill_array.dp.cpp   |  4 +---
 hip/components/fill_array.hip.cpp    |  5 +----
 include/ginkgo/core/base/array.hpp   |  7 +++++++
 include/ginkgo/core/base/types.hpp   | 18 ++++++++++++++++++
 include/ginkgo/core/matrix/dense.hpp |  9 ++++++++-
 omp/components/fill_array.cpp        |  4 +---
 reference/components/fill_array.cpp  |  4 +---
 13 files changed, 92 insertions(+), 21 deletions(-)

diff --git a/core/base/array.cpp b/core/base/array.cpp
index b74da73c803..6f24322e91c 100644
--- a/core/base/array.cpp
+++ b/core/base/array.cpp
@@ -36,6 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
+#include "core/components/fill_array.hpp"
 #include "core/components/precision_conversion.hpp"
 
 
@@ -49,6 +50,15 @@ GKO_REGISTER_OPERATION(convert, components::convert_precision);
 }  // namespace conversion
 
 
+namespace array {
+
+
+GKO_REGISTER_OPERATION(fill_array, components::fill_array);
+
+
+}  // namespace array
+
+
 namespace detail {
 
 
@@ -68,4 +78,19 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_ARRAY_CONVERSION);
 
 
 }  // namespace detail
+
+
+template <typename ValueType>
+void Array<ValueType>::fill(const ValueType value)
+{
+    GKO_ASSERT(this->get_num_elems() > 0);
+    this->get_executor()->run(
+        array::make_fill_array(this->get_data(), this->get_num_elems(), value));
+}
+
+
+#define GKO_DECLARE_ARRAY(_type) class Array<_type>
+GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_ARRAY);
+
+
 }  // namespace gko
diff --git a/core/components/fill_array.hpp b/core/components/fill_array.hpp
index e8b39ff8465..7270d168bc1 100644
--- a/core/components/fill_array.hpp
+++ b/core/components/fill_array.hpp
@@ -51,8 +51,8 @@ namespace kernels {
 
 
 #define GKO_DECLARE_ALL_AS_TEMPLATES \
-    template <typename IndexType>    \
-    GKO_DECLARE_FILL_ARRAY_KERNEL(IndexType)
+    template <typename ValueType>    \
+    GKO_DECLARE_FILL_ARRAY_KERNEL(ValueType)
 
 
 namespace omp {
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index feb67558d44..9f69997b48c 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -1014,6 +1014,4 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_MATRIX);
 
 
 }  // namespace matrix
-
-
 }  // namespace gko
diff --git a/core/test/base/array.cpp b/core/test/base/array.cpp
index 91c5e8867b9..e3f1bf5e69b 100644
--- a/core/test/base/array.cpp
+++ b/core/test/base/array.cpp
@@ -299,6 +299,18 @@ TYPED_TEST(Array, DoesNotCopyBackTemporaryCloneBetweenSameMemory)
 }
 
 
+TYPED_TEST(Array, CanBeFilledWithValue)
+{
+    this->x.fill(TypeParam{42});
+
+    ASSERT_EQ(this->x.get_num_elems(), 2);
+    ASSERT_EQ(this->x.get_data()[0], TypeParam{42});
+    ASSERT_EQ(this->x.get_data()[1], TypeParam{42});
+    ASSERT_EQ(this->x.get_const_data()[0], TypeParam{42});
+    ASSERT_EQ(this->x.get_const_data()[1], TypeParam{42});
+}
+
+
 TYPED_TEST(Array, CanBeCleared)
 {
     this->x.clear();
diff --git a/core/test/matrix/dense.cpp b/core/test/matrix/dense.cpp
index f352648fadd..4af19de0a5c 100644
--- a/core/test/matrix/dense.cpp
+++ b/core/test/matrix/dense.cpp
@@ -203,6 +203,21 @@ TYPED_TEST(Dense, CanBeDoubleListConstructedWithstride)
 }
 
 
+TYPED_TEST(Dense, CanBeFilledWithValue)
+{
+    using value_type = typename TestFixture::value_type;
+    auto m =
+        gko::initialize<gko::matrix::Dense<TypeParam>>({1.0, 2.0}, this->exec);
+    EXPECT_EQ(m->at(0), value_type{1});
+    EXPECT_EQ(m->at(1), value_type{2});
+
+    m->fill(value_type{42});
+
+    EXPECT_EQ(m->at(0), value_type{42});
+    EXPECT_EQ(m->at(1), value_type{42});
+}
+
+
 TYPED_TEST(Dense, CanBeCopied)
 {
     auto mtx_copy = gko::matrix::Dense<TypeParam>::create(this->exec);
diff --git a/cuda/components/fill_array.cu b/cuda/components/fill_array.cu
index be4876b9aa4..c12435b5773 100644
--- a/cuda/components/fill_array.cu
+++ b/cuda/components/fill_array.cu
@@ -59,9 +59,7 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
                                                         as_cuda_type(val));
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
-template GKO_DECLARE_FILL_ARRAY_KERNEL(size_type);
+GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
 }  // namespace components
diff --git a/dpcpp/components/fill_array.dp.cpp b/dpcpp/components/fill_array.dp.cpp
index 8006ddd16c9..2dfde3591b7 100644
--- a/dpcpp/components/fill_array.dp.cpp
+++ b/dpcpp/components/fill_array.dp.cpp
@@ -54,9 +54,7 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
     });
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
-template GKO_DECLARE_FILL_ARRAY_KERNEL(size_type);
+GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
 }  // namespace components
diff --git a/hip/components/fill_array.hip.cpp b/hip/components/fill_array.hip.cpp
index dc4e24720ba..80e6bbbee1a 100644
--- a/hip/components/fill_array.hip.cpp
+++ b/hip/components/fill_array.hip.cpp
@@ -64,10 +64,7 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
                        0, n, as_hip_type(array), as_hip_type(val));
 }
 
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
-template GKO_DECLARE_FILL_ARRAY_KERNEL(size_type);
+GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
 }  // namespace components
diff --git a/include/ginkgo/core/base/array.hpp b/include/ginkgo/core/base/array.hpp
index 04dea7f309b..b7f31529da9 100644
--- a/include/ginkgo/core/base/array.hpp
+++ b/include/ginkgo/core/base/array.hpp
@@ -479,6 +479,13 @@ class Array {
         }
     }
 
+    /**
+     * Fill the array with the given value.
+     *
+     * @param value the value to be filled
+     */
+    void fill(const ValueType value);
+
     /**
      * Returns the number of elements in the Array.
      *
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 2b665371285..e7fd4b46854 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -491,6 +491,24 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
     template _macro(std::complex<double>, std::complex<float>)
 
 
+/**
+ * Instantiates a template for each POD type
+ *
+ * @param _macro  A macro which expands the template instantiation
+ *                (not including the leading `template` specifier).
+ *                Should take two arguments `src` and `dst`, which
+ *                are replaced by the source and destination value type.
+ */
+#define GKO_INSTANTIATE_FOR_EACH_POD_TYPE(_macro) \
+    template _macro(float);                       \
+    template _macro(double);                      \
+    template _macro(std::complex<float>);         \
+    template _macro(std::complex<double>);        \
+    template _macro(size_type);                   \
+    template _macro(int32);                       \
+    template _macro(int64)
+
+
 }  // namespace gko
 
 
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 22bd0314392..2a5cd4a1851 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -244,6 +244,13 @@ class Dense
     std::unique_ptr<LinOp> row_permute(
         const Array<int64> *permutation_indices) const override;
 
+    /**
+     * Fill the dense matrix with a given value.
+     *
+     * @param value  the value to be filled
+     */
+    void fill(const ValueType value) { this->values_.fill(value); }
+
     /**
      * Create a Dense matrix consisting of the given rows from this matrix.
      *
@@ -693,7 +700,7 @@ class Dense
 private:
     Array<value_type> values_;
     size_type stride_;
-};  // namespace matrix
+};
 
 
 }  // namespace matrix
diff --git a/omp/components/fill_array.cpp b/omp/components/fill_array.cpp
index c528c8fe7d3..b1c00b3a76f 100644
--- a/omp/components/fill_array.cpp
+++ b/omp/components/fill_array.cpp
@@ -49,9 +49,7 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
-template GKO_DECLARE_FILL_ARRAY_KERNEL(size_type);
+GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
 }  // namespace components
diff --git a/reference/components/fill_array.cpp b/reference/components/fill_array.cpp
index 43e586c1092..949b4ee92d4 100644
--- a/reference/components/fill_array.cpp
+++ b/reference/components/fill_array.cpp
@@ -46,9 +46,7 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
     std::fill_n(array, n, val);
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
-template GKO_DECLARE_FILL_ARRAY_KERNEL(size_type);
+GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
 }  // namespace components

From 4694a89de4eaca6a01db960dd83fa4925b52da83 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 10 Feb 2021 23:03:28 +0100
Subject: [PATCH 0325/1008] Update ResidualNorm crit, make it more general.

+ Deprecate the older criteria.
+ Add tests for the new ResidualNorm criterion.
+ Update test install.
+ Add a factory parameter to select between different baseline modes.
---
 core/stop/residual_norm.cpp                   |   5 +-
 cuda/test/stop/residual_norm_kernels.cpp      | 314 ++++++++++++++++
 hip/test/stop/residual_norm_kernels.cpp       | 313 ++++++++++++++++
 include/ginkgo/core/stop/residual_norm.hpp    | 349 +++++++++++-------
 omp/test/stop/residual_norm_kernels.cpp       | 243 ++++++++++++
 reference/test/stop/residual_norm_kernels.cpp | 305 ++++++++++++++-
 test_install/test_install.cpp                 |  10 +
 test_install/test_install_cuda.cu             |  10 +
 8 files changed, 1404 insertions(+), 145 deletions(-)

diff --git a/core/stop/residual_norm.cpp b/core/stop/residual_norm.cpp
index 6a94ffe544a..a8f2599119d 100644
--- a/core/stop/residual_norm.cpp
+++ b/core/stop/residual_norm.cpp
@@ -83,8 +83,9 @@ bool ResidualNorm<ValueType>::check_impl(uint8 stoppingId, bool setFinalized,
     bool all_converged = true;
 
     this->get_executor()->run(residual_norm::make_residual_norm(
-        dense_tau, starting_tau_.get(), tolerance_, stoppingId, setFinalized,
-        stop_status, &device_storage_, &all_converged, one_changed));
+        dense_tau, starting_tau_.get(), reduction_factor_, stoppingId,
+        setFinalized, stop_status, &device_storage_, &all_converged,
+        one_changed));
 
     return all_converged;
 }
diff --git a/cuda/test/stop/residual_norm_kernels.cpp b/cuda/test/stop/residual_norm_kernels.cpp
index c9a0518a328..6af9e73c056 100644
--- a/cuda/test/stop/residual_norm_kernels.cpp
+++ b/cuda/test/stop/residual_norm_kernels.cpp
@@ -45,6 +45,320 @@ namespace {
 constexpr double tol = 1.0e-14;
 
 
+class ResidualNorm : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
+
+    ResidualNorm()
+    {
+        ref_ = gko::ReferenceExecutor::create();
+        cuda_ = gko::CudaExecutor::create(0, ref_);
+        factory_ =
+            gko::stop::ResidualNorm<>::build().with_reduction_factor(tol).on(
+                cuda_);
+        rel_factory_ = gko::stop::ResidualNorm<>::build()
+                           .with_reduction_factor(tol)
+                           .with_baseline(gko::stop::mode::initial_resnorm)
+                           .on(cuda_);
+        abs_factory_ = gko::stop::ResidualNorm<>::build()
+                           .with_reduction_factor(tol)
+                           .with_baseline(gko::stop::mode::absolute)
+                           .on(cuda_);
+    }
+
+    std::unique_ptr<gko::stop::ResidualNorm<>::Factory> factory_;
+    std::unique_ptr<gko::stop::ResidualNorm<>::Factory> rel_factory_;
+    std::unique_ptr<gko::stop::ResidualNorm<>::Factory> abs_factory_;
+    std::shared_ptr<const gko::CudaExecutor> cuda_;
+    std::shared_ptr<gko::ReferenceExecutor> ref_;
+};
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalForRhsResNorm)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto res_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(cuda_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(cuda_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(cuda_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = tol * 1.1 * rhs_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(cuda_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = tol * 0.9 * rhs_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalMultipleRHSForRhsResNorm)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto res_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(cuda_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(cuda_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(cuda_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = tol * 0.9 * rhs_norm->at(0, 0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(cuda_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = tol * 0.9 * rhs_norm->at(0, 1);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalForRelResNorm)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto res_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(cuda_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(cuda_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion =
+        rel_factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(cuda_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = tol * 1.1 * res_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(cuda_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = tol * 0.9 * res_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalMultipleRHSForRelResNorm)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto res_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(cuda_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(cuda_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion =
+        rel_factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(cuda_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = tol * 0.9 * res_norm->at(0, 0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(cuda_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = tol * 0.9 * res_norm->at(0, 1);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalForAbsResNorm)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto res_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(cuda_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(cuda_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion =
+        abs_factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(cuda_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = tol * 1.1;
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(cuda_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = tol * 0.9;
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalMultipleRHSForAbsResNorm)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto res_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(cuda_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(cuda_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion =
+        abs_factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(cuda_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = tol * 0.9;
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(cuda_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = tol * 0.9;
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
 class ResidualNormReduction : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
diff --git a/hip/test/stop/residual_norm_kernels.cpp b/hip/test/stop/residual_norm_kernels.cpp
index 32c5b4b32f7..4dfcd690c79 100644
--- a/hip/test/stop/residual_norm_kernels.cpp
+++ b/hip/test/stop/residual_norm_kernels.cpp
@@ -45,6 +45,319 @@ namespace {
 constexpr double tol = 1.0e-14;
 
 
+class ResidualNorm : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
+
+    ResidualNorm()
+    {
+        ref_ = gko::ReferenceExecutor::create();
+        hip_ = gko::HipExecutor::create(0, ref_);
+        factory_ =
+            gko::stop::ResidualNorm<>::build().with_reduction_factor(tol).on(
+                hip_);
+        rel_factory_ = gko::stop::ResidualNorm<>::build()
+                           .with_reduction_factor(tol)
+                           .with_baseline(gko::stop::mode::initial_resnorm)
+                           .on(hip_);
+        abs_factory_ = gko::stop::ResidualNorm<>::build()
+                           .with_reduction_factor(tol)
+                           .with_baseline(gko::stop::mode::absolute)
+                           .on(hip_);
+    }
+
+    std::unique_ptr<gko::stop::ResidualNorm<>::Factory> factory_;
+    std::unique_ptr<gko::stop::ResidualNorm<>::Factory> rel_factory_;
+    std::unique_ptr<gko::stop::ResidualNorm<>::Factory> abs_factory_;
+    std::shared_ptr<const gko::HipExecutor> hip_;
+    std::shared_ptr<gko::ReferenceExecutor> ref_;
+};
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalForRhsResNorm)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto res_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(hip_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(hip_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(hip_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = tol * 1.1 * rhs_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(hip_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = tol * 0.9 * rhs_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalMultipleRHSForRhsResNorm)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto res_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(hip_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(hip_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(hip_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = tol * 0.9 * rhs_norm->at(0, 0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(hip_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = tol * 0.9 * rhs_norm->at(0, 1);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalForRelResNorm)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto res_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(hip_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(hip_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion =
+        rel_factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(hip_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = tol * 1.1 * res_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(hip_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = tol * 0.9 * res_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalMultipleRHSForRelResNorm)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto res_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(hip_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(hip_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion =
+        rel_factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(hip_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = tol * 0.9 * res_norm->at(0, 0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(hip_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = tol * 0.9 * res_norm->at(0, 1);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalForAbsResNorm)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto res_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(hip_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(hip_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion =
+        abs_factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(hip_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = tol * 1.1;
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(hip_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = tol * 0.9;
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalMultipleRHSForAbsResNorm)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto res_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(hip_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(hip_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion =
+        abs_factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(hip_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = tol * 0.9;
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(hip_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = tol * 0.9;
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
 class ResidualNormReduction : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index a67b00f187b..d0ac7e06ce4 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -48,6 +48,12 @@ namespace gko {
 namespace stop {
 
 
+/**
+ * The enum that holds the quantity relative to the implicit residual.
+ */
+enum class mode { absolute, initial_resnorm, rhs_norm };
+
+
 /**
  * The ResidualNorm class provides a framework for stopping criteria
  * related to the residual norm. These criteria differ in the way they
@@ -66,6 +72,23 @@ class ResidualNorm
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
     using Vector = matrix::Dense<ValueType>;
 
+    GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
+    {
+        /**
+         * Residual norm reduction factor
+         */
+        remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(
+            reduction_factor, static_cast<remove_complex<ValueType>>(1e-15));
+
+        /**
+         * The quantity the reduction is relative to. Choices include
+         * "mode::rhs_norm", "mode::initial_resnorm" and "mode::absolute"
+         */
+        mode GKO_FACTORY_PARAMETER_SCALAR(baseline, mode::rhs_norm);
+    };
+    GKO_ENABLE_CRITERION_FACTORY(ResidualNorm<ValueType>, parameters, Factory);
+    GKO_ENABLE_BUILD_METHOD(Factory);
+
 protected:
     bool check_impl(uint8 stoppingId, bool setFinalized,
                     Array<stopping_status> *stop_status, bool *one_changed,
@@ -77,17 +100,200 @@ class ResidualNorm
     {}
 
     explicit ResidualNorm(std::shared_ptr<const gko::Executor> exec,
-                          remove_complex<ValueType> tolerance)
+                          remove_complex<ValueType> reduction_factor)
         : EnablePolymorphicObject<ResidualNorm, Criterion>(exec),
           device_storage_{exec, 2},
-          tolerance_{tolerance}
+          reduction_factor_{reduction_factor}
+    {}
+
+    explicit ResidualNorm(const Factory *factory, const CriterionArgs &args)
+        : EnablePolymorphicObject<ResidualNorm, Criterion>(
+              factory->get_executor()),
+          device_storage_{factory->get_executor(), 2},
+          parameters_{factory->get_parameters()},
+          reduction_factor_{factory->get_parameters().reduction_factor}
+    {
+        this->baseline_rhs_norm_ =
+            factory->get_parameters().baseline == mode::rhs_norm;
+        this->baseline_init_resnorm_ =
+            factory->get_parameters().baseline == mode::initial_resnorm;
+        this->baseline_absolute_ =
+            factory->get_parameters().baseline == mode::absolute;
+        auto exec = factory->get_executor();
+
+        if (this->baseline_init_resnorm_) {
+            if (args.initial_residual == nullptr) {
+                GKO_NOT_SUPPORTED(nullptr);
+            }
+            this->starting_tau_ = NormVector::create(
+                exec, dim<2>{1, args.initial_residual->get_size()[1]});
+            if (dynamic_cast<const ComplexVector *>(args.initial_residual)) {
+                auto dense_r = as<ComplexVector>(args.initial_residual);
+                dense_r->compute_norm2(this->starting_tau_.get());
+            } else {
+                auto dense_r = as<Vector>(args.initial_residual);
+                dense_r->compute_norm2(this->starting_tau_.get());
+            }
+        } else if (this->baseline_rhs_norm_) {
+            if (args.b == nullptr) {
+                GKO_NOT_SUPPORTED(nullptr);
+            }
+            this->starting_tau_ =
+                NormVector::create(exec, dim<2>{1, args.b->get_size()[1]});
+            if (dynamic_cast<const ComplexVector *>(args.b.get())) {
+                auto dense_rhs = as<ComplexVector>(args.b);
+                dense_rhs->compute_norm2(this->starting_tau_.get());
+            } else {
+                auto dense_rhs = as<Vector>(args.b);
+                dense_rhs->compute_norm2(this->starting_tau_.get());
+            }
+        } else if (this->baseline_absolute_) {
+            if (args.b == nullptr) {
+                GKO_NOT_SUPPORTED(nullptr);
+            }
+            this->starting_tau_ =
+                NormVector::create(exec, dim<2>{1, args.b->get_size()[1]});
+            this->starting_tau_->fill(gko::one<remove_complex<ValueType>>());
+        } else {
+            GKO_NOT_SUPPORTED(nullptr);
+        }
+        this->u_dense_tau_ =
+            NormVector::create_with_config_of(this->starting_tau_.get());
+    }
+
+    std::unique_ptr<NormVector> starting_tau_{};
+    std::unique_ptr<NormVector> u_dense_tau_{};
+
+private:
+    remove_complex<ValueType> reduction_factor_{};
+    bool baseline_rhs_norm_{};
+    bool baseline_init_resnorm_{};
+    bool baseline_absolute_{};
+    /* Contains device side: all_converged and one_changed booleans */
+    Array<bool> device_storage_;
+};
+
+
+/**
+ * The ImplicitResidualNorm class is a stopping criterion which
+ * stops the iteration process when the implicit residual norm is below a
+ * certain threshold relative to either the norm of the right-hand side, i.e.
+ * when norm(residual) / norm(right_hand_side) < threshold or the initial
+ * residual, i.e. when norm(residual) / norm(initial_residual) < threshold. For
+ * better performance, the checks are run on the executor
+ * where the algorithm is executed.
+ *
+ * @note To use this stopping criterion there are some dependencies. The
+ * constructor depends on either `b` or the `initial_residual` in order to
+ * compute their norms. If this is not correctly provided, an exception
+ * ::gko::NotSupported() is thrown.
+ *
+ * @ingroup stop
+ */
+template <typename ValueType = default_precision>
+class ImplicitResidualNorm
+    : public EnablePolymorphicObject<ImplicitResidualNorm<ValueType>,
+                                     Criterion> {
+    friend class EnablePolymorphicObject<ImplicitResidualNorm<ValueType>,
+                                         Criterion>;
+
+public:
+    using ComplexVector = matrix::Dense<to_complex<ValueType>>;
+    using NormVector = matrix::Dense<remove_complex<ValueType>>;
+    using Vector = matrix::Dense<ValueType>;
+
+    GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
+    {
+        /**
+         * Implicit Residual norm goal
+         */
+        remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(
+            reduction_factor, static_cast<remove_complex<ValueType>>(1e-15));
+
+        /**
+         * The quantity the reduction is relative to. Choices include
+         * "mode::rhs_norm", "mode::initial_resnorm" and "mode::absolute"
+         */
+        mode GKO_FACTORY_PARAMETER_SCALAR(baseline, mode::rhs_norm);
+    };
+    GKO_ENABLE_CRITERION_FACTORY(ImplicitResidualNorm<ValueType>, parameters,
+                                 Factory);
+    GKO_ENABLE_BUILD_METHOD(Factory);
+
+protected:
+    bool check_impl(uint8 stoppingId, bool setFinalized,
+                    Array<stopping_status> *stop_status, bool *one_changed,
+                    const Criterion::Updater &) override;
+
+    explicit ImplicitResidualNorm(std::shared_ptr<const gko::Executor> exec)
+        : EnablePolymorphicObject<ImplicitResidualNorm, Criterion>(exec),
+          device_storage_{exec, 2}
     {}
 
+    explicit ImplicitResidualNorm(const Factory *factory,
+                                  const CriterionArgs &args)
+        : EnablePolymorphicObject<ImplicitResidualNorm, Criterion>(
+              factory->get_executor()),
+          device_storage_{factory->get_executor(), 2},
+          parameters_{factory->get_parameters()},
+          reduction_factor_{factory->get_parameters().reduction_factor}
+    {
+        this->baseline_rhs_norm_ =
+            factory->get_parameters().baseline == mode::rhs_norm;
+        this->baseline_init_resnorm_ =
+            factory->get_parameters().baseline == mode::initial_resnorm;
+        this->baseline_absolute_ =
+            factory->get_parameters().baseline == mode::absolute;
+        auto exec = factory->get_executor();
+
+        if (this->baseline_init_resnorm_) {
+            if (args.initial_residual == nullptr) {
+                GKO_NOT_SUPPORTED(nullptr);
+            }
+            this->starting_tau_ = NormVector::create(
+                exec, dim<2>{1, args.initial_residual->get_size()[1]});
+            if (dynamic_cast<const ComplexVector *>(args.initial_residual)) {
+                auto dense_r = as<ComplexVector>(args.initial_residual);
+                dense_r->compute_norm2(this->starting_tau_.get());
+            } else {
+                auto dense_r = as<Vector>(args.initial_residual);
+                dense_r->compute_norm2(this->starting_tau_.get());
+            }
+        } else if (this->baseline_rhs_norm_) {
+            if (args.b == nullptr) {
+                GKO_NOT_SUPPORTED(nullptr);
+            }
+            this->starting_tau_ =
+                NormVector::create(exec, dim<2>{1, args.b->get_size()[1]});
+            if (dynamic_cast<const ComplexVector *>(args.b.get())) {
+                auto dense_rhs = as<ComplexVector>(args.b);
+                dense_rhs->compute_norm2(this->starting_tau_.get());
+            } else {
+                auto dense_rhs = as<Vector>(args.b);
+                dense_rhs->compute_norm2(this->starting_tau_.get());
+            }
+        } else if (this->baseline_absolute_) {
+            if (args.b == nullptr) {
+                GKO_NOT_SUPPORTED(nullptr);
+            }
+            this->starting_tau_ =
+                NormVector::create(exec, dim<2>{1, args.b->get_size()[1]});
+            this->starting_tau_->fill(gko::one<remove_complex<ValueType>>());
+        } else {
+            GKO_NOT_SUPPORTED(nullptr);
+        }
+        this->u_dense_tau_ =
+            NormVector::create_with_config_of(this->starting_tau_.get());
+    }
+
     std::unique_ptr<NormVector> starting_tau_{};
     std::unique_ptr<NormVector> u_dense_tau_{};
 
 private:
-    remove_complex<ValueType> tolerance_{};
+    remove_complex<ValueType> reduction_factor_{};
+    bool baseline_rhs_norm_{};
+    bool baseline_init_resnorm_{};
+    bool baseline_absolute_{};
     /* Contains device side: all_converged and one_changed booleans */
     Array<bool> device_storage_;
 };
@@ -110,7 +316,11 @@ class ResidualNorm
  * @ingroup stop
  */
 template <typename ValueType = default_precision>
-class ResidualNormReduction : public ResidualNorm<ValueType> {
+class [
+    [deprecated("Please use the ResidualNorm criterion with a init_resnorm "
+                "baseline mode instead.")]] ResidualNormReduction
+    : public ResidualNorm<ValueType>
+{
 public:
     using ComplexVector = matrix::Dense<to_complex<ValueType>>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
@@ -176,7 +386,11 @@ class ResidualNormReduction : public ResidualNorm<ValueType> {
  * @ingroup stop
  */
 template <typename ValueType = default_precision>
-class RelativeResidualNorm : public ResidualNorm<ValueType> {
+class [
+    [deprecated("Please use the ResidualNorm criterion with a rhs_norm "
+                "baseline mode instead.")]] RelativeResidualNorm
+    : public ResidualNorm<ValueType>
+{
 public:
     using ComplexVector = matrix::Dense<to_complex<ValueType>>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
@@ -226,125 +440,6 @@ class RelativeResidualNorm : public ResidualNorm<ValueType> {
 };
 
 
-/**
- * The enum that holds the quantity relative to the implicit residual.
- */
-enum class norm_of { initial_residual, rhs };
-
-/**
- * The ImplicitResidualNorm class is a stopping criterion which
- * stops the iteration process when the implicit residual norm is below a
- * certain threshold relative to either the norm of the right-hand side, i.e.
- * when norm(residual) / norm(right_hand_side) < threshold or the initial
- * residual, i.e. when norm(residual) / norm(initial_residual) < threshold. For
- * better performance, the checks are run on the executor
- * where the algorithm is executed.
- *
- * @note To use this stopping criterion there are some dependencies. The
- * constructor depends on either `b` or the `initial_residual` in order to
- * compute their norms. If this is not correctly provided, an exception
- * ::gko::NotSupported() is thrown.
- *
- * @ingroup stop
- */
-template <typename ValueType = default_precision>
-class ImplicitResidualNorm : public ResidualNorm<ValueType> {
-public:
-    using ComplexVector = matrix::Dense<to_complex<ValueType>>;
-    using NormVector = matrix::Dense<remove_complex<ValueType>>;
-    using Vector = matrix::Dense<ValueType>;
-
-    GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
-    {
-        /**
-         * Implicit residual norm reduction factor
-         */
-        remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(
-            reduction_factor, static_cast<remove_complex<ValueType>>(1e-15));
-
-        /**
-         * The quantity the reduction is relative to. Choices include
-         * "norm_of::rhs" and "norm_of::initial_residual"
-         */
-        norm_of GKO_FACTORY_PARAMETER_SCALAR(relative_to, norm_of::rhs);
-    };
-    GKO_ENABLE_CRITERION_FACTORY(ImplicitResidualNorm<ValueType>, parameters,
-                                 Factory);
-    GKO_ENABLE_BUILD_METHOD(Factory);
-
-protected:
-    bool check_impl(uint8 stoppingId, bool setFinalized,
-                    Array<stopping_status> *stop_status, bool *one_changed,
-                    const Criterion::Updater &) override;
-
-    explicit ImplicitResidualNorm(std::shared_ptr<const gko::Executor> exec)
-        : ResidualNorm<ValueType>(exec), device_storage_{exec, 2}
-    {}
-
-    explicit ImplicitResidualNorm(const Factory *factory,
-                                  const CriterionArgs &args)
-        : ResidualNorm<ValueType>(factory->get_executor(),
-                                  factory->get_parameters().reduction_factor),
-          parameters_{factory->get_parameters()},
-          device_storage_{factory->get_executor(), 2}
-    {
-        this->reduction_factor_ = factory->get_parameters().reduction_factor;
-        this->relative_to_rhs_ =
-            factory->get_parameters().relative_to == norm_of::rhs;
-        this->relative_to_initial_residual_ =
-            factory->get_parameters().relative_to == norm_of::initial_residual;
-        if (this->relative_to_rhs_ && args.b == nullptr) {
-            GKO_NOT_SUPPORTED(nullptr);
-        }
-        if (this->relative_to_initial_residual_ &&
-            args.initial_residual == nullptr) {
-            GKO_NOT_SUPPORTED(nullptr);
-        }
-
-        size_type tau_size = 0;
-        if (this->relative_to_rhs_) {
-            tau_size = args.b->get_size()[1];
-        } else if (this->relative_to_initial_residual_) {
-            tau_size = args.initial_residual->get_size()[1];
-        } else {
-            GKO_NOT_SUPPORTED(nullptr);
-        }
-
-        auto exec = factory->get_executor();
-
-        this->starting_tau_ = NormVector::create(exec, dim<2>{1, tau_size});
-        this->u_dense_tau_ =
-            NormVector::create_with_config_of(this->starting_tau_.get());
-        if (this->relative_to_rhs_) {
-            if (dynamic_cast<const ComplexVector *>(args.b.get())) {
-                auto dense_rhs = as<ComplexVector>(args.b);
-                dense_rhs->compute_norm2(this->starting_tau_.get());
-            } else {
-                auto dense_rhs = as<Vector>(args.b);
-                dense_rhs->compute_norm2(this->starting_tau_.get());
-            }
-        } else if (this->relative_to_initial_residual_) {
-            if (dynamic_cast<const ComplexVector *>(args.initial_residual)) {
-                auto dense_r = as<ComplexVector>(args.initial_residual);
-                dense_r->compute_norm2(this->starting_tau_.get());
-            } else {
-                auto dense_r = as<Vector>(args.initial_residual);
-                dense_r->compute_norm2(this->starting_tau_.get());
-            }
-        } else {
-            GKO_NOT_SUPPORTED(nullptr);
-        }
-    }
-
-private:
-    remove_complex<ValueType> reduction_factor_{};
-    bool relative_to_rhs_{};
-    bool relative_to_initial_residual_{};
-    /* Contains device side: all_converged and one_changed booleans */
-    Array<bool> device_storage_;
-};
-
-
 /**
  * The AbsoluteResidualNorm class is a stopping criterion which stops the
  * iteration process when the residual norm is below a certain
@@ -360,7 +455,11 @@ class ImplicitResidualNorm : public ResidualNorm<ValueType> {
  * @ingroup stop
  */
 template <typename ValueType = default_precision>
-class AbsoluteResidualNorm : public ResidualNorm<ValueType> {
+class [
+    [deprecated("Please use the ResidualNorm criterion with an absolute "
+                "baseline mode instead.")]] AbsoluteResidualNorm
+    : public ResidualNorm<ValueType>
+{
 public:
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
     using Vector = matrix::Dense<ValueType>;
diff --git a/omp/test/stop/residual_norm_kernels.cpp b/omp/test/stop/residual_norm_kernels.cpp
index 9e025c69508..949acf9092a 100644
--- a/omp/test/stop/residual_norm_kernels.cpp
+++ b/omp/test/stop/residual_norm_kernels.cpp
@@ -45,6 +45,249 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
+template <typename T>
+class ResidualNorm : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<T>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<T>>;
+
+    ResidualNorm()
+    {
+        exec_ = gko::OmpExecutor::create();
+        factory_ = gko::stop::ResidualNorm<T>::build()
+                       .with_reduction_factor(r<T>::value)
+                       .on(exec_);
+        rel_factory_ = gko::stop::ResidualNorm<T>::build()
+                           .with_reduction_factor(r<T>::value)
+                           .with_baseline(gko::stop::mode::initial_resnorm)
+                           .on(exec_);
+        abs_factory_ = gko::stop::ResidualNorm<T>::build()
+                           .with_reduction_factor(r<T>::value)
+                           .with_baseline(gko::stop::mode::absolute)
+                           .on(exec_);
+    }
+
+    std::unique_ptr<typename gko::stop::ResidualNorm<T>::Factory> factory_;
+    std::unique_ptr<typename gko::stop::ResidualNorm<T>::Factory> rel_factory_;
+    std::unique_ptr<typename gko::stop::ResidualNorm<T>::Factory> abs_factory_;
+    std::shared_ptr<const gko::Executor> exec_;
+};
+
+TYPED_TEST_SUITE(ResidualNorm, gko::test::ValueTypes);
+
+
+TYPED_TEST(ResidualNorm, WaitsTillResidualGoal)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using NormVector = typename TestFixture::NormVector;
+    using T_nc = gko::remove_complex<TypeParam>;
+    auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->exec_);
+    auto criterion =
+        this->factory_->generate(nullptr, rhs, nullptr, initial_res.get());
+    auto rel_criterion =
+        this->rel_factory_->generate(nullptr, rhs, nullptr, initial_res.get());
+    auto abs_criterion =
+        this->abs_factory_->generate(nullptr, rhs, nullptr, initial_res.get());
+    {
+        auto res_norm = gko::initialize<NormVector>({10.0}, this->exec_);
+        auto rhs_norm = gko::initialize<NormVector>({100.0}, this->exec_);
+        gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+        constexpr gko::uint8 RelativeStoppingId{1};
+        bool one_changed{};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+        stop_status.get_data()[0].reset();
+
+        ASSERT_FALSE(
+            criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        res_norm->at(0) = r<TypeParam>::value * 1.1 * res_norm->at(0);
+        ASSERT_FALSE(
+            criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
+        ASSERT_EQ(one_changed, false);
+
+        res_norm->at(0) = r<TypeParam>::value * 0.9 * res_norm->at(0);
+        ASSERT_TRUE(
+            criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+    {
+        auto res_norm = gko::initialize<NormVector>({100.0}, this->exec_);
+        constexpr gko::uint8 RelativeStoppingId{1};
+        bool one_changed{};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+        stop_status.get_data()[0].reset();
+
+        ASSERT_FALSE(
+            rel_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        res_norm->at(0) = r<TypeParam>::value * 1.1 * res_norm->at(0);
+        ASSERT_FALSE(
+            rel_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
+        ASSERT_EQ(one_changed, false);
+
+        res_norm->at(0) = r<TypeParam>::value * 0.9 * res_norm->at(0);
+        ASSERT_TRUE(
+            rel_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+    {
+        auto res_norm = gko::initialize<NormVector>({100.0}, this->exec_);
+        constexpr gko::uint8 RelativeStoppingId{1};
+        bool one_changed{};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+        stop_status.get_data()[0].reset();
+
+        ASSERT_FALSE(
+            abs_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        res_norm->at(0) = r<TypeParam>::value * 1.1;
+        ASSERT_FALSE(
+            abs_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
+        ASSERT_EQ(one_changed, false);
+
+        res_norm->at(0) = r<TypeParam>::value * 0.9;
+        ASSERT_TRUE(
+            abs_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+}
+
+
+TYPED_TEST(ResidualNorm, WaitsTillResidualGoalMultipleRHS)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using NormVector = typename TestFixture::NormVector;
+    using T = TypeParam;
+    using T_nc = gko::remove_complex<TypeParam>;
+    auto res = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->exec_);
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({I<T>{10.0, 10.0}}, this->exec_);
+    auto criterion = this->factory_->generate(nullptr, rhs, nullptr, res.get());
+    auto rel_criterion =
+        this->rel_factory_->generate(nullptr, rhs, nullptr, res.get());
+    auto abs_criterion =
+        this->abs_factory_->generate(nullptr, rhs, nullptr, res.get());
+    {
+        auto res_norm =
+            gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
+        auto rhs_norm =
+            gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
+        gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+        bool one_changed{};
+        constexpr gko::uint8 RelativeStoppingId{1};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 2);
+        stop_status.get_data()[0].reset();
+        stop_status.get_data()[1].reset();
+
+        ASSERT_FALSE(
+            criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        res_norm->at(0, 0) = r<TypeParam>::value * 0.9 * rhs_norm->at(0, 0);
+        ASSERT_FALSE(
+            criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+
+        res_norm->at(0, 1) = r<TypeParam>::value * 0.9 * rhs_norm->at(0, 1);
+        ASSERT_TRUE(
+            criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[1].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+    {
+        auto res_norm =
+            gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
+        bool one_changed{};
+        constexpr gko::uint8 RelativeStoppingId{1};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 2);
+        stop_status.get_data()[0].reset();
+        stop_status.get_data()[1].reset();
+
+        ASSERT_FALSE(
+            rel_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        res_norm->at(0, 0) = r<TypeParam>::value * 0.9 * res_norm->at(0, 0);
+        ASSERT_FALSE(
+            rel_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+
+        res_norm->at(0, 1) = r<TypeParam>::value * 0.9 * res_norm->at(0, 1);
+        ASSERT_TRUE(
+            rel_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[1].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+    {
+        auto res_norm =
+            gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
+        bool one_changed{};
+        constexpr gko::uint8 RelativeStoppingId{1};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 2);
+        stop_status.get_data()[0].reset();
+        stop_status.get_data()[1].reset();
+
+        ASSERT_FALSE(
+            abs_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        res_norm->at(0, 0) = r<TypeParam>::value * 0.9;
+        ASSERT_FALSE(
+            abs_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+
+        res_norm->at(0, 1) = r<TypeParam>::value * 0.9;
+        ASSERT_TRUE(
+            abs_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[1].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+}
+
+
 template <typename T>
 class ResidualNormReduction : public ::testing::Test {
 protected:
diff --git a/reference/test/stop/residual_norm_kernels.cpp b/reference/test/stop/residual_norm_kernels.cpp
index ba01fbbea91..0cd48cafb6d 100644
--- a/reference/test/stop/residual_norm_kernels.cpp
+++ b/reference/test/stop/residual_norm_kernels.cpp
@@ -50,55 +50,324 @@ namespace {
 
 
 template <typename T>
-class ResidualNormReduction : public ::testing::Test {
+class ResidualNorm : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<T>;
     using NormVector = gko::matrix::Dense<gko::remove_complex<T>>;
 
-    ResidualNormReduction()
+    ResidualNorm()
     {
         exec_ = gko::ReferenceExecutor::create();
-        factory_ = gko::stop::ResidualNormReduction<T>::build()
+        factory_ = gko::stop::ResidualNorm<T>::build()
                        .with_reduction_factor(r<T>::value)
                        .on(exec_);
+        rel_factory_ = gko::stop::ResidualNorm<T>::build()
+                           .with_reduction_factor(r<T>::value)
+                           .with_baseline(gko::stop::mode::initial_resnorm)
+                           .on(exec_);
+        abs_factory_ = gko::stop::ResidualNorm<T>::build()
+                           .with_reduction_factor(r<T>::value)
+                           .with_baseline(gko::stop::mode::absolute)
+                           .on(exec_);
     }
 
-    std::unique_ptr<typename gko::stop::ResidualNormReduction<T>::Factory>
-        factory_;
+    std::unique_ptr<typename gko::stop::ResidualNorm<T>::Factory> factory_;
+    std::unique_ptr<typename gko::stop::ResidualNorm<T>::Factory> rel_factory_;
+    std::unique_ptr<typename gko::stop::ResidualNorm<T>::Factory> abs_factory_;
     std::shared_ptr<const gko::Executor> exec_;
 };
 
-TYPED_TEST_SUITE(ResidualNormReduction, gko::test::ValueTypes);
+TYPED_TEST_SUITE(ResidualNorm, gko::test::ValueTypes);
 
 
-TYPED_TEST(ResidualNormReduction, CanCreateFactory)
+TYPED_TEST(ResidualNorm, CanCreateFactory)
 {
     ASSERT_NE(this->factory_, nullptr);
     ASSERT_EQ(this->factory_->get_parameters().reduction_factor,
               r<TypeParam>::value);
+    ASSERT_EQ(this->factory_->get_parameters().baseline,
+              gko::stop::mode::rhs_norm);
     ASSERT_EQ(this->factory_->get_executor(), this->exec_);
+    ASSERT_NE(this->rel_factory_, nullptr);
+    ASSERT_EQ(this->rel_factory_->get_parameters().reduction_factor,
+              r<TypeParam>::value);
+    ASSERT_EQ(this->rel_factory_->get_parameters().baseline,
+              gko::stop::mode::initial_resnorm);
+    ASSERT_EQ(this->rel_factory_->get_executor(), this->exec_);
+    ASSERT_NE(this->abs_factory_, nullptr);
+    ASSERT_EQ(this->abs_factory_->get_parameters().reduction_factor,
+              r<TypeParam>::value);
+    ASSERT_EQ(this->abs_factory_->get_parameters().baseline,
+              gko::stop::mode::absolute);
+    ASSERT_EQ(this->abs_factory_->get_executor(), this->exec_);
 }
 
 
-TYPED_TEST(ResidualNormReduction, CannotCreateCriterionWithoutB)
+TYPED_TEST(ResidualNorm, CannotCreateCriterionWithoutNeededInput)
 {
     ASSERT_THROW(this->factory_->generate(nullptr, nullptr, nullptr, nullptr),
                  gko::NotSupported);
+    ASSERT_THROW(
+        this->rel_factory_->generate(nullptr, nullptr, nullptr, nullptr),
+        gko::NotSupported);
+    ASSERT_THROW(
+        this->abs_factory_->generate(nullptr, nullptr, nullptr, nullptr),
+        gko::NotSupported);
 }
 
 
-TYPED_TEST(ResidualNormReduction, CanCreateCriterionWithB)
+TYPED_TEST(ResidualNorm, CanCreateCriterionWithNeededInput)
 {
     using Mtx = typename TestFixture::Mtx;
     std::shared_ptr<gko::LinOp> scalar =
         gko::initialize<Mtx>({1.0}, this->exec_);
     auto criterion =
-        this->factory_->generate(nullptr, nullptr, nullptr, scalar.get());
+        this->factory_->generate(nullptr, scalar, nullptr, nullptr);
+    auto rel_criterion =
+        this->rel_factory_->generate(nullptr, nullptr, nullptr, scalar.get());
+    auto abs_criterion =
+        this->abs_factory_->generate(nullptr, scalar, nullptr, nullptr);
 
     ASSERT_NE(criterion, nullptr);
+    ASSERT_NE(rel_criterion, nullptr);
+    ASSERT_NE(abs_criterion, nullptr);
 }
 
 
+TYPED_TEST(ResidualNorm, WaitsTillResidualGoal)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using NormVector = typename TestFixture::NormVector;
+    using T_nc = gko::remove_complex<TypeParam>;
+    auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->exec_);
+    auto criterion =
+        this->factory_->generate(nullptr, rhs, nullptr, initial_res.get());
+    auto rel_criterion =
+        this->rel_factory_->generate(nullptr, rhs, nullptr, initial_res.get());
+    auto abs_criterion =
+        this->abs_factory_->generate(nullptr, rhs, nullptr, initial_res.get());
+    {
+        auto res_norm = gko::initialize<NormVector>({10.0}, this->exec_);
+        auto rhs_norm = gko::initialize<NormVector>({100.0}, this->exec_);
+        gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+        constexpr gko::uint8 RelativeStoppingId{1};
+        bool one_changed{};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+        stop_status.get_data()[0].reset();
+
+        ASSERT_FALSE(
+            criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        res_norm->at(0) = r<TypeParam>::value * 1.1 * res_norm->at(0);
+        ASSERT_FALSE(
+            criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
+        ASSERT_EQ(one_changed, false);
+
+        res_norm->at(0) = r<TypeParam>::value * 0.9 * res_norm->at(0);
+        ASSERT_TRUE(
+            criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+    {
+        auto res_norm = gko::initialize<NormVector>({100.0}, this->exec_);
+        constexpr gko::uint8 RelativeStoppingId{1};
+        bool one_changed{};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+        stop_status.get_data()[0].reset();
+
+        ASSERT_FALSE(
+            rel_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        res_norm->at(0) = r<TypeParam>::value * 1.1 * res_norm->at(0);
+        ASSERT_FALSE(
+            rel_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
+        ASSERT_EQ(one_changed, false);
+
+        res_norm->at(0) = r<TypeParam>::value * 0.9 * res_norm->at(0);
+        ASSERT_TRUE(
+            rel_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+    {
+        auto res_norm = gko::initialize<NormVector>({100.0}, this->exec_);
+        constexpr gko::uint8 RelativeStoppingId{1};
+        bool one_changed{};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+        stop_status.get_data()[0].reset();
+
+        ASSERT_FALSE(
+            abs_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        res_norm->at(0) = r<TypeParam>::value * 1.1;
+        ASSERT_FALSE(
+            abs_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
+        ASSERT_EQ(one_changed, false);
+
+        res_norm->at(0) = r<TypeParam>::value * 0.9;
+        ASSERT_TRUE(
+            abs_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+}
+
+
+TYPED_TEST(ResidualNorm, WaitsTillResidualGoalMultipleRHS)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using NormVector = typename TestFixture::NormVector;
+    using T = TypeParam;
+    using T_nc = gko::remove_complex<TypeParam>;
+    auto res = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->exec_);
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({I<T>{10.0, 10.0}}, this->exec_);
+    auto criterion = this->factory_->generate(nullptr, rhs, nullptr, res.get());
+    auto rel_criterion =
+        this->rel_factory_->generate(nullptr, rhs, nullptr, res.get());
+    auto abs_criterion =
+        this->abs_factory_->generate(nullptr, rhs, nullptr, res.get());
+    {
+        auto res_norm =
+            gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
+        auto rhs_norm =
+            gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
+        gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+        bool one_changed{};
+        constexpr gko::uint8 RelativeStoppingId{1};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 2);
+        stop_status.get_data()[0].reset();
+        stop_status.get_data()[1].reset();
+
+        ASSERT_FALSE(
+            criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        res_norm->at(0, 0) = r<TypeParam>::value * 0.9 * rhs_norm->at(0, 0);
+        ASSERT_FALSE(
+            criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+
+        res_norm->at(0, 1) = r<TypeParam>::value * 0.9 * rhs_norm->at(0, 1);
+        ASSERT_TRUE(
+            criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[1].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+    {
+        auto res_norm =
+            gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
+        bool one_changed{};
+        constexpr gko::uint8 RelativeStoppingId{1};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 2);
+        stop_status.get_data()[0].reset();
+        stop_status.get_data()[1].reset();
+
+        ASSERT_FALSE(
+            rel_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        res_norm->at(0, 0) = r<TypeParam>::value * 0.9 * res_norm->at(0, 0);
+        ASSERT_FALSE(
+            rel_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+
+        res_norm->at(0, 1) = r<TypeParam>::value * 0.9 * res_norm->at(0, 1);
+        ASSERT_TRUE(
+            rel_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[1].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+    {
+        auto res_norm =
+            gko::initialize<NormVector>({I<T_nc>{100.0, 100.0}}, this->exec_);
+        bool one_changed{};
+        constexpr gko::uint8 RelativeStoppingId{1};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 2);
+        stop_status.get_data()[0].reset();
+        stop_status.get_data()[1].reset();
+
+        ASSERT_FALSE(
+            abs_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        res_norm->at(0, 0) = r<TypeParam>::value * 0.9;
+        ASSERT_FALSE(
+            abs_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+
+        res_norm->at(0, 1) = r<TypeParam>::value * 0.9;
+        ASSERT_TRUE(
+            abs_criterion->update()
+                .residual_norm(res_norm.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[1].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+}
+
+
+template <typename T>
+class ResidualNormReduction : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<T>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<T>>;
+
+    ResidualNormReduction()
+    {
+        exec_ = gko::ReferenceExecutor::create();
+        factory_ = gko::stop::ResidualNormReduction<T>::build()
+                       .with_reduction_factor(r<T>::value)
+                       .on(exec_);
+    }
+
+    std::unique_ptr<typename gko::stop::ResidualNormReduction<T>::Factory>
+        factory_;
+    std::shared_ptr<const gko::ReferenceExecutor> exec_;
+};
+
+TYPED_TEST_SUITE(ResidualNormReduction, gko::test::ValueTypes);
+
+
 TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoal)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -325,11 +594,11 @@ class ImplicitResidualNorm : public ::testing::Test {
                        .on(exec_);
         factory_2_ = gko::stop::ImplicitResidualNorm<T>::build()
                          .with_reduction_factor(r<T>::value)
-                         .with_relative_to(gko::stop::norm_of::initial_residual)
+                         .with_baseline(gko::stop::mode::initial_resnorm)
                          .on(exec_);
         factory_3_ = gko::stop::ImplicitResidualNorm<T>::build()
                          .with_reduction_factor(r<T>::value)
-                         .with_relative_to(gko::stop::norm_of::rhs)
+                         .with_baseline(gko::stop::mode::rhs_norm)
                          .on(exec_);
     }
 
@@ -350,12 +619,12 @@ TYPED_TEST(ImplicitResidualNorm, CanCreateFactory)
     ASSERT_NE(this->factory_, nullptr);
     ASSERT_EQ(this->factory_->get_parameters().reduction_factor,
               r<TypeParam>::value);
-    ASSERT_EQ(this->factory_->get_parameters().relative_to,
-              gko::stop::norm_of::rhs);
-    ASSERT_EQ(this->factory_2_->get_parameters().relative_to,
-              gko::stop::norm_of::initial_residual);
-    ASSERT_EQ(this->factory_3_->get_parameters().relative_to,
-              gko::stop::norm_of::rhs);
+    ASSERT_EQ(this->factory_->get_parameters().baseline,
+              gko::stop::mode::rhs_norm);
+    ASSERT_EQ(this->factory_2_->get_parameters().baseline,
+              gko::stop::mode::initial_resnorm);
+    ASSERT_EQ(this->factory_3_->get_parameters().baseline,
+              gko::stop::mode::rhs_norm);
     ASSERT_EQ(this->factory_->get_executor(), this->exec_);
 }
 
diff --git a/test_install/test_install.cpp b/test_install/test_install.cpp
index 589a52bf8c2..115e7b15b46 100644
--- a/test_install/test_install.cpp
+++ b/test_install/test_install.cpp
@@ -353,6 +353,16 @@ int main(int, char **)
                         .on(refExec);
 
         // residual_norm.hpp
+        auto main_res = gko::stop::ResidualNorm<>::build()
+                            .with_reduction_factor(1e-10)
+                            .with_baseline(gko::stop::mode_absolute)
+                            .on(refExec);
+
+        auto implicit_res = gko::stop::ImplicitResidualNorm<>::build()
+                                .with_reduction_factor(1e-10)
+                                .with_baseline(gko::stop::mode_absolute)
+                                .on(refExec);
+
         auto res_red = gko::stop::ResidualNormReduction<>::build()
                            .with_reduction_factor(1e-10)
                            .on(refExec);
diff --git a/test_install/test_install_cuda.cu b/test_install/test_install_cuda.cu
index f11b545c309..bf2dd5a895a 100644
--- a/test_install/test_install_cuda.cu
+++ b/test_install/test_install_cuda.cu
@@ -349,6 +349,16 @@ int main(int, char **)
             .with_reduction_factor(1e-10)
             .on(cudaExec);
 
+        gko::stop::ResidualNorm<>::build()
+                            .with_reduction_factor(1e-10)
+                            .with_baseline(gko::stop::mode_absolute)
+                            .on(cudaExec);
+
+        gko::stop::ImplicitResidualNorm<>::build()
+                                .with_reduction_factor(1e-10)
+                                .with_baseline(gko::stop::mode_absolute)
+                                .on(cudaExec);
+
         gko::stop::RelativeResidualNorm<>::build().with_tolerance(1e-10).on(
             cudaExec);
 

From 420f2e1e0007fbb97989b46112ceb8c79bc8ee94 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 10 Feb 2021 23:05:31 +0100
Subject: [PATCH 0326/1008] s/ResidualNormReduction/ResidualNorm as default.

+ Update all examples and benchmarks.
---
 benchmark/solver/solver.cpp                   |  2 +-
 benchmark/utils/preconditioners.hpp           |  2 +-
 core/test/solver/bicg.cpp                     |  4 +-
 core/test/solver/bicgstab.cpp                 |  2 +-
 core/test/solver/cg.cpp                       |  4 +-
 core/test/solver/cgs.cpp                      |  4 +-
 core/test/solver/fcg.cpp                      |  4 +-
 core/test/solver/gmres.cpp                    |  8 +-
 core/test/solver/idr.cpp                      |  2 +-
 core/test/solver/ir.cpp                       |  8 +-
 cuda/test/solver/bicg_kernels.cpp             |  8 +-
 cuda/test/solver/bicgstab_kernels.cpp         |  4 +-
 cuda/test/solver/cg_kernels.cpp               |  4 +-
 cuda/test/solver/cgs_kernels.cpp              |  4 +-
 cuda/test/solver/fcg_kernels.cpp              |  4 +-
 .../adaptiveprecision-blockjacobi.cpp         |  2 +-
 examples/custom-logger/custom-logger.cpp      |  2 +-
 .../custom-matrix-format.cpp                  |  2 +-
 .../external-lib-interfacing.cpp              |  2 +-
 .../ilu-preconditioned-solver.cpp             |  2 +-
 .../inverse-iteration/inverse-iteration.cpp   |  2 +-
 .../ir-ilu-preconditioned-solver.cpp          |  2 +-
 .../iterative-refinement.cpp                  |  4 +-
 .../minimal-cuda-solver.cpp                   |  2 +-
 .../mixed-precision-ir/mixed-precision-ir.cpp |  2 +-
 .../nine-pt-stencil-solver.cpp                |  2 +-
 examples/papi-logging/papi-logging.cpp        |  2 +-
 .../performance-debugging.cpp                 |  2 +-
 examples/poisson-solver/poisson-solver.cpp    |  2 +-
 .../preconditioned-solver.cpp                 |  2 +-
 .../simple-solver-logging/doc/results.dox     | 80 +++++++++----------
 .../simple-solver-logging.cpp                 |  4 +-
 examples/simple-solver/simple-solver.cpp      |  2 +-
 .../three-pt-stencil-solver.cpp               |  2 +-
 hip/test/solver/bicg_kernels.cpp              |  8 +-
 hip/test/solver/bicgstab_kernels.cpp          |  4 +-
 hip/test/solver/cg_kernels.cpp                |  4 +-
 hip/test/solver/cgs_kernels.cpp               |  4 +-
 hip/test/solver/fcg_kernels.cpp               |  4 +-
 omp/test/solver/bicg_kernels.cpp              |  8 +-
 omp/test/solver/bicgstab_kernels.cpp          |  4 +-
 omp/test/solver/cg_kernels.cpp                |  4 +-
 omp/test/solver/cgs_kernels.cpp               |  4 +-
 omp/test/solver/fcg_kernels.cpp               |  4 +-
 omp/test/solver/gmres_kernels.cpp             |  4 +-
 reference/test/preconditioner/ilu.cpp         |  4 +-
 reference/test/solver/bicg_kernels.cpp        |  4 +-
 reference/test/solver/bicgstab_kernels.cpp    |  4 +-
 reference/test/solver/cg_kernels.cpp          |  4 +-
 reference/test/solver/cgs_kernels.cpp         |  4 +-
 reference/test/solver/fcg_kernels.cpp         |  4 +-
 reference/test/solver/gmres_kernels.cpp       |  8 +-
 reference/test/solver/idr_kernels.cpp         |  8 +-
 reference/test/solver/ir_kernels.cpp          | 18 ++---
 54 files changed, 147 insertions(+), 147 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index f351fb74fbe..53fa9c1fc6c 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -141,7 +141,7 @@ std::shared_ptr<const gko::stop::CriterionFactory> create_criterion(
                 .on(exec));
     } else {
         residual_stop =
-            gko::share(gko::stop::ResidualNormReduction<rc_etype>::build()
+            gko::share(gko::stop::ResidualNorm<rc_etype>::build()
                            .with_reduction_factor(
                                static_cast<rc_etype>(FLAGS_rel_res_goal))
                            .on(exec));
diff --git a/benchmark/utils/preconditioners.hpp b/benchmark/utils/preconditioners.hpp
index 17f20c40ebe..ab58461f284 100644
--- a/benchmark/utils/preconditioners.hpp
+++ b/benchmark/utils/preconditioners.hpp
@@ -307,7 +307,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
          }},
         {"overhead", [](std::shared_ptr<const gko::Executor> exec) {
              return gko::Overhead<etype>::build()
-                 .with_criteria(gko::stop::ResidualNormReduction<etype>::build()
+                 .with_criteria(gko::stop::ResidualNorm<etype>::build()
                                     .with_reduction_factor(rc_etype{})
                                     .on(exec))
                  .on(exec);
diff --git a/core/test/solver/bicg.cpp b/core/test/solver/bicg.cpp
index 1e8de3473eb..26a6440eef4 100644
--- a/core/test/solver/bicg.cpp
+++ b/core/test/solver/bicg.cpp
@@ -67,7 +67,7 @@ class Bicg : public ::testing::Test {
               Solver::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(3u).on(exec),
-                      gko::stop::ResidualNormReduction<>::build()
+                      gko::stop::ResidualNorm<>::build()
                           .with_reduction_factor(gko::remove_complex<T>{1e-6})
                           .on(exec))
                   .on(exec)),
@@ -181,7 +181,7 @@ TYPED_TEST(Bicg, CanSetPreconditionerGenerator)
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(this->exec),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(
                         gko::remove_complex<value_type>(1e-6))
                     .on(this->exec))
diff --git a/core/test/solver/bicgstab.cpp b/core/test/solver/bicgstab.cpp
index 455d05d8d29..9ad4e883b5a 100644
--- a/core/test/solver/bicgstab.cpp
+++ b/core/test/solver/bicgstab.cpp
@@ -65,7 +65,7 @@ class Bicgstab : public ::testing::Test {
               Solver::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(3u).on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(gko::remove_complex<T>{1e-6})
                           .on(exec))
                   .on(exec)),
diff --git a/core/test/solver/cg.cpp b/core/test/solver/cg.cpp
index b4bca2fc830..d4314788d7b 100644
--- a/core/test/solver/cg.cpp
+++ b/core/test/solver/cg.cpp
@@ -67,7 +67,7 @@ class Cg : public ::testing::Test {
               Solver::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(3u).on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(gko::remove_complex<T>{1e-6})
                           .on(exec))
                   .on(exec)),
@@ -180,7 +180,7 @@ TYPED_TEST(Cg, CanSetPreconditionerGenerator)
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
+                gko::stop::ResidualNorm<value_type>::build()
                     .with_reduction_factor(
                         gko::remove_complex<value_type>(1e-6))
                     .on(this->exec))
diff --git a/core/test/solver/cgs.cpp b/core/test/solver/cgs.cpp
index 4adfa5264fd..a9bcc316346 100644
--- a/core/test/solver/cgs.cpp
+++ b/core/test/solver/cgs.cpp
@@ -67,7 +67,7 @@ class Cgs : public ::testing::Test {
               Solver::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(3u).on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(gko::remove_complex<T>{1e-6})
                           .on(exec))
                   .on(exec)),
@@ -180,7 +180,7 @@ TYPED_TEST(Cgs, CanSetPreconditionerGenerator)
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
+                gko::stop::ResidualNorm<value_type>::build()
                     .with_reduction_factor(
                         gko::remove_complex<value_type>(1e-6))
                     .on(this->exec))
diff --git a/core/test/solver/fcg.cpp b/core/test/solver/fcg.cpp
index 0f5663ca99c..43052f011e4 100644
--- a/core/test/solver/fcg.cpp
+++ b/core/test/solver/fcg.cpp
@@ -64,7 +64,7 @@ class Fcg : public ::testing::Test {
               Solver::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(3u).on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(gko::remove_complex<T>{1e-6})
                           .on(exec))
                   .on(exec)),
@@ -165,7 +165,7 @@ TYPED_TEST(Fcg, CanSetPreconditionerGenerator)
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
+                gko::stop::ResidualNorm<value_type>::build()
                     .with_reduction_factor(
                         gko::remove_complex<value_type>(1e-6))
                     .on(this->exec))
diff --git a/core/test/solver/gmres.cpp b/core/test/solver/gmres.cpp
index ceb9d1314da..3f5b9510332 100644
--- a/core/test/solver/gmres.cpp
+++ b/core/test/solver/gmres.cpp
@@ -71,7 +71,7 @@ class Gmres : public ::testing::Test {
               Solver::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(3u).on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(reduction_factor)
                           .on(exec))
                   .on(exec)),
@@ -81,7 +81,7 @@ class Gmres : public ::testing::Test {
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(128u).on(
                           exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(reduction_factor)
                           .on(exec))
                   .on(exec)),
@@ -198,7 +198,7 @@ TYPED_TEST(Gmres, CanSetPreconditionerGenerator)
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
+                gko::stop::ResidualNorm<value_type>::build()
                     .with_reduction_factor(TestFixture::reduction_factor)
                     .on(this->exec))
             .with_preconditioner(
@@ -254,7 +254,7 @@ TYPED_TEST(Gmres, CanSetKrylovDim)
             .with_krylov_dim(4u)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(4u).on(this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
+                gko::stop::ResidualNorm<value_type>::build()
                     .with_reduction_factor(TestFixture::reduction_factor)
                     .on(this->exec))
             .on(this->exec);
diff --git a/core/test/solver/idr.cpp b/core/test/solver/idr.cpp
index a1393e62675..af0080de435 100644
--- a/core/test/solver/idr.cpp
+++ b/core/test/solver/idr.cpp
@@ -65,7 +65,7 @@ class Idr : public ::testing::Test {
               Solver::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(3u).on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(gko::remove_complex<T>{1e-6})
                           .on(exec))
                   .on(exec)),
diff --git a/core/test/solver/ir.cpp b/core/test/solver/ir.cpp
index e2069153a59..93d18c99260 100644
--- a/core/test/solver/ir.cpp
+++ b/core/test/solver/ir.cpp
@@ -67,7 +67,7 @@ class Ir : public ::testing::Test {
               Solver::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(3u).on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
@@ -179,7 +179,7 @@ TYPED_TEST(Ir, CanSetInnerSolverInFactory)
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
+                gko::stop::ResidualNorm<value_type>::build()
                     .with_reduction_factor(r<value_type>::value)
                     .on(this->exec))
             .with_solver(
@@ -340,7 +340,7 @@ TYPED_TEST(Ir, DefaultRelaxationFactor)
         gko::solver::Richardson<value_type>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
+                gko::stop::ResidualNorm<value_type>::build()
                     .with_reduction_factor(r<value_type>::value)
                     .on(this->exec))
             .on(this->exec)
@@ -359,7 +359,7 @@ TYPED_TEST(Ir, UseAsRichardson)
         gko::solver::Richardson<value_type>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
+                gko::stop::ResidualNorm<value_type>::build()
                     .with_reduction_factor(r<value_type>::value)
                     .on(this->exec))
             .with_relaxation_factor(relaxation_factor)
diff --git a/cuda/test/solver/bicg_kernels.cpp b/cuda/test/solver/bicg_kernels.cpp
index c5277173ef2..a825ae5aa9b 100644
--- a/cuda/test/solver/bicg_kernels.cpp
+++ b/cuda/test/solver/bicg_kernels.cpp
@@ -298,7 +298,7 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
         gko::solver::Bicg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(ref))
             .on(ref);
@@ -306,7 +306,7 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
         gko::solver::Bicg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(cuda),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(cuda))
             .on(cuda);
@@ -332,7 +332,7 @@ TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
         gko::solver::Bicg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(ref))
             .on(ref);
@@ -340,7 +340,7 @@ TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
         gko::solver::Bicg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(cuda),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(cuda))
             .on(cuda);
diff --git a/cuda/test/solver/bicgstab_kernels.cpp b/cuda/test/solver/bicgstab_kernels.cpp
index 8a742847ab7..7808226870c 100644
--- a/cuda/test/solver/bicgstab_kernels.cpp
+++ b/cuda/test/solver/bicgstab_kernels.cpp
@@ -77,7 +77,7 @@ class Bicgstab : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(cuda),
-                    gko::stop::ResidualNormReduction<>::build()
+                    gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
                         .on(cuda))
                 .on(cuda);
@@ -85,7 +85,7 @@ class Bicgstab : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNormReduction<>::build()
+                    gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
                         .on(ref))
                 .on(ref);
diff --git a/cuda/test/solver/cg_kernels.cpp b/cuda/test/solver/cg_kernels.cpp
index 3f9f1b94e8f..d8a98a83805 100644
--- a/cuda/test/solver/cg_kernels.cpp
+++ b/cuda/test/solver/cg_kernels.cpp
@@ -247,7 +247,7 @@ TEST_F(Cg, ApplyIsEquivalentToRef)
         gko::solver::Cg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(ref))
             .on(ref);
@@ -255,7 +255,7 @@ TEST_F(Cg, ApplyIsEquivalentToRef)
         gko::solver::Cg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(cuda),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(cuda))
             .on(cuda);
diff --git a/cuda/test/solver/cgs_kernels.cpp b/cuda/test/solver/cgs_kernels.cpp
index ca530ed5a5a..543f580cf53 100644
--- a/cuda/test/solver/cgs_kernels.cpp
+++ b/cuda/test/solver/cgs_kernels.cpp
@@ -75,7 +75,7 @@ class Cgs : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(cuda),
-                    gko::stop::ResidualNormReduction<>::build()
+                    gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
                         .on(cuda))
                 .on(cuda);
@@ -83,7 +83,7 @@ class Cgs : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNormReduction<>::build()
+                    gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
                         .on(ref))
                 .on(ref);
diff --git a/cuda/test/solver/fcg_kernels.cpp b/cuda/test/solver/fcg_kernels.cpp
index 0edddbcc606..85b0b3b7017 100644
--- a/cuda/test/solver/fcg_kernels.cpp
+++ b/cuda/test/solver/fcg_kernels.cpp
@@ -260,7 +260,7 @@ TEST_F(Fcg, ApplyIsEquivalentToRef)
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(ref))
             .on(ref);
@@ -268,7 +268,7 @@ TEST_F(Fcg, ApplyIsEquivalentToRef)
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(cuda),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(cuda))
             .on(cuda);
diff --git a/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp b/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
index 261c9428824..2d1cb9dd4c5 100644
--- a/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
+++ b/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
@@ -111,7 +111,7 @@ int main(int argc, char *argv[])
     const RealValueType reduction_factor = 1e-7;
     auto iter_stop =
         gko::stop::Iteration::build().with_max_iters(10000u).on(exec);
-    auto tol_stop = gko::stop::ResidualNormReduction<ValueType>::build()
+    auto tol_stop = gko::stop::ResidualNorm<ValueType>::build()
                         .with_reduction_factor(reduction_factor)
                         .on(exec);
 
diff --git a/examples/custom-logger/custom-logger.cpp b/examples/custom-logger/custom-logger.cpp
index 94bbd065c5b..1caa5911dab 100644
--- a/examples/custom-logger/custom-logger.cpp
+++ b/examples/custom-logger/custom-logger.cpp
@@ -278,7 +278,7 @@ int main(int argc, char *argv[])
         cg::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(20u).on(exec),
-                gko::stop::ResidualNormReduction<ValueType>::build()
+                gko::stop::ResidualNorm<ValueType>::build()
                     .with_reduction_factor(reduction_factor)
                     .on(exec))
             .on(exec);
diff --git a/examples/custom-matrix-format/custom-matrix-format.cpp b/examples/custom-matrix-format/custom-matrix-format.cpp
index c586284e6f8..cfad1b3bedd 100644
--- a/examples/custom-matrix-format/custom-matrix-format.cpp
+++ b/examples/custom-matrix-format/custom-matrix-format.cpp
@@ -295,7 +295,7 @@ int main(int argc, char *argv[])
         .with_criteria(gko::stop::Iteration::build()
                            .with_max_iters(discretization_points)
                            .on(exec),
-                       gko::stop::ResidualNormReduction<ValueType>::build()
+                       gko::stop::ResidualNorm<ValueType>::build()
                            .with_reduction_factor(reduction_factor)
                            .on(exec))
         .on(exec)
diff --git a/examples/external-lib-interfacing/external-lib-interfacing.cpp b/examples/external-lib-interfacing/external-lib-interfacing.cpp
index b885eeee77f..122b73b155b 100644
--- a/examples/external-lib-interfacing/external-lib-interfacing.cpp
+++ b/examples/external-lib-interfacing/external-lib-interfacing.cpp
@@ -881,7 +881,7 @@ void AdvectionProblem<dim>::solve()
         bicgstab::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(1000).on(exec),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-12)
                     .on(exec))
             .with_preconditioner(bj::build().on(exec))
diff --git a/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp b/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
index 511689a61f9..1b9e7b5f53b 100644
--- a/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
+++ b/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp
@@ -117,7 +117,7 @@ int main(int argc, char *argv[])
         gmres::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(1000u).on(exec),
-                gko::stop::ResidualNormReduction<ValueType>::build()
+                gko::stop::ResidualNorm<ValueType>::build()
                     .with_reduction_factor(reduction_factor)
                     .on(exec))
             .with_generated_preconditioner(gko::share(ilu_preconditioner))
diff --git a/examples/inverse-iteration/inverse-iteration.cpp b/examples/inverse-iteration/inverse-iteration.cpp
index 56afa33442a..4025c743ba1 100644
--- a/examples/inverse-iteration/inverse-iteration.cpp
+++ b/examples/inverse-iteration/inverse-iteration.cpp
@@ -122,7 +122,7 @@ int main(int argc, char *argv[])
             .with_criteria(gko::stop::Iteration::build()
                                .with_max_iters(system_max_iterations)
                                .on(exec),
-                           gko::stop::ResidualNormReduction<precision>::build()
+                           gko::stop::ResidualNorm<precision>::build()
                                .with_reduction_factor(system_residual_goal)
                                .on(exec))
             .on(exec)
diff --git a/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp b/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
index 38b690090e1..aab893c6efa 100644
--- a/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
+++ b/examples/ir-ilu-preconditioned-solver/ir-ilu-preconditioned-solver.cpp
@@ -143,7 +143,7 @@ int main(int argc, char *argv[])
     const RealValueType reduction_factor{1e-12};
     auto iter_stop =
         gko::stop::Iteration::build().with_max_iters(1000u).on(exec);
-    auto tol_stop = gko::stop::ResidualNormReduction<ValueType>::build()
+    auto tol_stop = gko::stop::ResidualNorm<ValueType>::build()
                         .with_reduction_factor(reduction_factor)
                         .on(exec);
 
diff --git a/examples/iterative-refinement/iterative-refinement.cpp b/examples/iterative-refinement/iterative-refinement.cpp
index 71587eba5f7..20a3514316e 100644
--- a/examples/iterative-refinement/iterative-refinement.cpp
+++ b/examples/iterative-refinement/iterative-refinement.cpp
@@ -113,7 +113,7 @@ int main(int argc, char *argv[])
     RealValueType outer_reduction_factor{1e-12};
     auto iter_stop =
         gko::stop::Iteration::build().with_max_iters(max_iters).on(exec);
-    auto tol_stop = gko::stop::ResidualNormReduction<ValueType>::build()
+    auto tol_stop = gko::stop::ResidualNorm<ValueType>::build()
                         .with_reduction_factor(outer_reduction_factor)
                         .on(exec);
 
@@ -129,7 +129,7 @@ int main(int argc, char *argv[])
             .with_solver(
                 cg::build()
                     .with_criteria(
-                        gko::stop::ResidualNormReduction<ValueType>::build()
+                        gko::stop::ResidualNorm<ValueType>::build()
                             .with_reduction_factor(inner_reduction_factor)
                             .on(exec))
                     .on(exec))
diff --git a/examples/minimal-cuda-solver/minimal-cuda-solver.cpp b/examples/minimal-cuda-solver/minimal-cuda-solver.cpp
index e099a9cc7e8..7c8e9c77f96 100644
--- a/examples/minimal-cuda-solver/minimal-cuda-solver.cpp
+++ b/examples/minimal-cuda-solver/minimal-cuda-solver.cpp
@@ -47,7 +47,7 @@ int main()
             .with_preconditioner(gko::preconditioner::Jacobi<>::build().on(gpu))
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(20u).on(gpu),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-15)
                     .on(gpu))
             .on(gpu);
diff --git a/examples/mixed-precision-ir/mixed-precision-ir.cpp b/examples/mixed-precision-ir/mixed-precision-ir.cpp
index f7f67724e2c..499924fcfe9 100644
--- a/examples/mixed-precision-ir/mixed-precision-ir.cpp
+++ b/examples/mixed-precision-ir/mixed-precision-ir.cpp
@@ -127,7 +127,7 @@ int main(int argc, char *argv[])
     // Create inner solver
     auto inner_solver =
         cg::build()
-            .with_criteria(gko::stop::ResidualNormReduction<SolverType>::build()
+            .with_criteria(gko::stop::ResidualNorm<SolverType>::build()
                                .with_reduction_factor(inner_reduction_factor)
                                .on(exec),
                            gko::stop::Iteration::build()
diff --git a/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp b/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
index 1f469bccd4e..cf3d360c11f 100644
--- a/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
+++ b/examples/nine-pt-stencil-solver/nine-pt-stencil-solver.cpp
@@ -285,7 +285,7 @@ void solve_system(const std::string &executor_string,
         cg::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(dp_2).on(exec),
-                gko::stop::ResidualNormReduction<ValueType>::build()
+                gko::stop::ResidualNorm<ValueType>::build()
                     .with_reduction_factor(reduction_factor)
                     .on(exec))
             .with_preconditioner(bj::build().on(exec))
diff --git a/examples/papi-logging/papi-logging.cpp b/examples/papi-logging/papi-logging.cpp
index 4e2990bc3fc..6aff2772980 100644
--- a/examples/papi-logging/papi-logging.cpp
+++ b/examples/papi-logging/papi-logging.cpp
@@ -180,7 +180,7 @@ int main(int argc, char *argv[])
         cg::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(20u).on(exec),
-                gko::stop::ResidualNormReduction<ValueType>::build()
+                gko::stop::ResidualNorm<ValueType>::build()
                     .with_reduction_factor(reduction_factor)
                     .on(exec))
             .on(exec);
diff --git a/examples/performance-debugging/performance-debugging.cpp b/examples/performance-debugging/performance-debugging.cpp
index 2378bf7864d..902062c06fd 100644
--- a/examples/performance-debugging/performance-debugging.cpp
+++ b/examples/performance-debugging/performance-debugging.cpp
@@ -426,7 +426,7 @@ int main(int argc, char *argv[])
     auto solver_factory =
         solver::build()
             .with_criteria(
-                gko::stop::ResidualNormReduction<ValueType>::build()
+                gko::stop::ResidualNorm<ValueType>::build()
                     .with_reduction_factor(reduction_factor)
                     .on(exec),
                 gko::stop::Iteration::build().with_max_iters(max_iters).on(
diff --git a/examples/poisson-solver/poisson-solver.cpp b/examples/poisson-solver/poisson-solver.cpp
index b83c7825c22..2d04a84a7d3 100644
--- a/examples/poisson-solver/poisson-solver.cpp
+++ b/examples/poisson-solver/poisson-solver.cpp
@@ -185,7 +185,7 @@ int main(int argc, char *argv[])
         .with_criteria(gko::stop::Iteration::build()
                            .with_max_iters(discretization_points)
                            .on(exec),
-                       gko::stop::ResidualNormReduction<ValueType>::build()
+                       gko::stop::ResidualNorm<ValueType>::build()
                            .with_reduction_factor(reduction_factor)
                            .on(exec))
         .with_preconditioner(bj::build().on(exec))
diff --git a/examples/preconditioned-solver/preconditioned-solver.cpp b/examples/preconditioned-solver/preconditioned-solver.cpp
index 73c460e019b..ce01f0c0cc5 100644
--- a/examples/preconditioned-solver/preconditioned-solver.cpp
+++ b/examples/preconditioned-solver/preconditioned-solver.cpp
@@ -98,7 +98,7 @@ int main(int argc, char *argv[])
         cg::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(20u).on(exec),
-                gko::stop::ResidualNormReduction<ValueType>::build()
+                gko::stop::ResidualNorm<ValueType>::build()
                     .with_reduction_factor(reduction_factor)
                     .on(exec))
             // Add preconditioner, these 2 lines are the only
diff --git a/examples/simple-solver-logging/doc/results.dox b/examples/simple-solver-logging/doc/results.dox
index e70253b2079..c7cf3f0aee8 100644
--- a/examples/simple-solver-logging/doc/results.dox
+++ b/examples/simple-solver-logging/doc/results.dox
@@ -43,12 +43,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 0 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 0 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 0 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 0 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 0 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149550] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149550] with Bytes[152]
@@ -70,12 +70,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 1 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 1 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 1 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 1 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 1 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149980] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149980] with Bytes[152]
@@ -101,12 +101,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 2 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 2 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 2 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 2 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 2 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149290] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149290] with Bytes[152]
@@ -132,12 +132,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 3 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 3 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 3 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 3 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 3 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149890] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149890] with Bytes[152]
@@ -163,12 +163,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 4 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 4 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 4 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 4 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 4 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149200] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149200] with Bytes[152]
@@ -194,12 +194,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 5 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 5 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 5 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 5 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 5 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149890] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149890] with Bytes[152]
@@ -225,12 +225,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 6 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 6 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 6 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 6 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 6 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149450] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149450] with Bytes[152]
@@ -256,12 +256,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 7 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 7 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 7 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 7 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 7 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149730] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149730] with Bytes[152]
@@ -287,12 +287,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 8 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 8 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 8 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 8 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 8 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149200] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149200] with Bytes[152]
@@ -318,12 +318,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 9 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 9 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 9 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 9 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 9 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149620] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149620] with Bytes[152]
@@ -349,12 +349,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 10 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 10 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 10 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 10 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 10 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149450] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149450] with Bytes[152]
@@ -380,12 +380,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 11 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 11 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 11 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 11 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 11 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149860] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149860] with Bytes[152]
@@ -411,12 +411,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 12 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 12 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 12 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 12 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 12 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x21499a0] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x21499a0] with Bytes[152]
@@ -442,12 +442,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 13 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 13 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 13 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 13 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 13 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149490] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149490] with Bytes[152]
@@ -473,12 +473,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 14 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 14 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 14 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 14 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 14 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149b50] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149b50] with Bytes[152]
@@ -504,12 +504,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 15 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 15 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 15 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 15 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 15 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149a70] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149a70] with Bytes[152]
@@ -535,12 +535,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 16 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 16 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 16 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 16 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 16 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149970] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149970] with Bytes[152]
@@ -566,12 +566,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 17 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 17 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 17 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 17 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 17 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149780] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149780] with Bytes[152]
@@ -597,12 +597,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 18 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 18 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 18 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 18 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 18 with ID 1 and finalized set to 1. It changed one RHS 0, stopped the iteration process 0
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149620] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149620] with Bytes[152]
@@ -628,12 +628,12 @@ This is the expected output:
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_dot_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14c50] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> iteration 19 completed with solver LinOp[gko::solver::Cg<double>,0x2142d60] with residual LinOp[gko::matrix::Dense<double>,0x2147b30], solution LinOp[gko::matrix::Dense<double>,0x2143450] and residual_norm LinOp[gko::LinOp const*,0]
-[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 19 with ID 1 and finalized set to 1
+[LOG] >>> check started for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 19 with ID 1 and finalized set to 1
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::matrix::dense::compute_norm2_operation<gko::matrix::Dense<double> const*, gko::matrix::Dense<double>*>,0x7ffd93d14ad0] completed on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] started on Executor[gko::ReferenceExecutor,0x21400d0]
 [LOG] >>> Operation[gko::stop::residual_norm::residual_norm_operation<gko::matrix::Dense<double> const*&, gko::matrix::Dense<double>*, double&, unsigned char&, bool&, gko::Array<gko::stopping_status>*&, gko::Array<bool>*, bool*, bool*&>,0x7ffd93d14b90] completed on Executor[gko::ReferenceExecutor,0x21400d0]
-[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNormReduction<double>,0x2148db0] at iteration 19 with ID 1 and finalized set to 1. It changed one RHS 1, stopped the iteration process 1
+[LOG] >>> check completed for stop::Criterion[gko::stop::ResidualNorm<double>,0x2148db0] at iteration 19 with ID 1 and finalized set to 1. It changed one RHS 1, stopped the iteration process 1
 [LOG] >>> allocation started on Executor[gko::ReferenceExecutor,0x21400d0] with Bytes[152]
 [LOG] >>> allocation completed on Executor[gko::ReferenceExecutor,0x21400d0] at Location[0x2149890] with Bytes[152]
 [LOG] >>> copy started from Executor[gko::ReferenceExecutor,0x21400d0] to Executor[gko::ReferenceExecutor,0x21400d0] from Location[0x21480a0] to Location[0x2149890] with Bytes[152]
diff --git a/examples/simple-solver-logging/simple-solver-logging.cpp b/examples/simple-solver-logging/simple-solver-logging.cpp
index 01fcf93d622..a7f5bbaca84 100644
--- a/examples/simple-solver-logging/simple-solver-logging.cpp
+++ b/examples/simple-solver-logging/simple-solver-logging.cpp
@@ -123,12 +123,12 @@ int main(int argc, char *argv[])
     // Add stream_logger to the executor
     exec->add_logger(stream_logger);
 
-    // Add stream_logger only to the ResidualNormReduction criterion Factory
+    // Add stream_logger only to the ResidualNorm criterion Factory
     // Note that the logger will get automatically propagated to every criterion
     // generated from this factory.
     const RealValueType reduction_factor{1e-7};
     using ResidualCriterionFactory =
-        gko::stop::ResidualNormReduction<ValueType>::Factory;
+        gko::stop::ResidualNorm<ValueType>::Factory;
     std::shared_ptr<ResidualCriterionFactory> residual_criterion =
         ResidualCriterionFactory::create()
             .with_reduction_factor(reduction_factor)
diff --git a/examples/simple-solver/simple-solver.cpp b/examples/simple-solver/simple-solver.cpp
index 09bfb476a8f..bf8095fc373 100644
--- a/examples/simple-solver/simple-solver.cpp
+++ b/examples/simple-solver/simple-solver.cpp
@@ -132,7 +132,7 @@ int main(int argc, char *argv[])
         cg::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(20u).on(exec),
-                gko::stop::ResidualNormReduction<ValueType>::build()
+                gko::stop::ResidualNorm<ValueType>::build()
                     .with_reduction_factor(reduction_factor)
                     .on(exec))
             .on(exec);
diff --git a/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp b/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
index 8d695803a9a..a050be1a7fe 100644
--- a/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
+++ b/examples/three-pt-stencil-solver/three-pt-stencil-solver.cpp
@@ -220,7 +220,7 @@ void solve_system(const std::string &executor_string,
             .with_criteria(gko::stop::Iteration::build()
                                .with_max_iters(gko::size_type(dp))
                                .on(exec),
-                           gko::stop::ResidualNormReduction<ValueType>::build()
+                           gko::stop::ResidualNorm<ValueType>::build()
                                .with_reduction_factor(reduction_factor)
                                .on(exec))
             .with_preconditioner(bj::build().on(exec))
diff --git a/hip/test/solver/bicg_kernels.cpp b/hip/test/solver/bicg_kernels.cpp
index 3eb3359f01d..52073945050 100644
--- a/hip/test/solver/bicg_kernels.cpp
+++ b/hip/test/solver/bicg_kernels.cpp
@@ -298,7 +298,7 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
         gko::solver::Bicg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(ref))
             .on(ref);
@@ -306,7 +306,7 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
         gko::solver::Bicg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(hip),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(hip))
             .on(hip);
@@ -332,7 +332,7 @@ TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
         gko::solver::Bicg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(ref))
             .on(ref);
@@ -340,7 +340,7 @@ TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
         gko::solver::Bicg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(hip),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(hip))
             .on(hip);
diff --git a/hip/test/solver/bicgstab_kernels.cpp b/hip/test/solver/bicgstab_kernels.cpp
index ee9810e005a..bcf4191a426 100644
--- a/hip/test/solver/bicgstab_kernels.cpp
+++ b/hip/test/solver/bicgstab_kernels.cpp
@@ -77,7 +77,7 @@ class Bicgstab : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(hip),
-                    gko::stop::ResidualNormReduction<>::build()
+                    gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
                         .on(hip))
                 .on(hip);
@@ -85,7 +85,7 @@ class Bicgstab : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNormReduction<>::build()
+                    gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
                         .on(ref))
                 .on(ref);
diff --git a/hip/test/solver/cg_kernels.cpp b/hip/test/solver/cg_kernels.cpp
index 3b7e4ccb078..a46d698cc22 100644
--- a/hip/test/solver/cg_kernels.cpp
+++ b/hip/test/solver/cg_kernels.cpp
@@ -247,7 +247,7 @@ TEST_F(Cg, ApplyIsEquivalentToRef)
         gko::solver::Cg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(ref))
             .on(ref);
@@ -255,7 +255,7 @@ TEST_F(Cg, ApplyIsEquivalentToRef)
         gko::solver::Cg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(hip),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(hip))
             .on(hip);
diff --git a/hip/test/solver/cgs_kernels.cpp b/hip/test/solver/cgs_kernels.cpp
index 7069d1fe0a2..861f9498780 100644
--- a/hip/test/solver/cgs_kernels.cpp
+++ b/hip/test/solver/cgs_kernels.cpp
@@ -75,7 +75,7 @@ class Cgs : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(hip),
-                    gko::stop::ResidualNormReduction<>::build()
+                    gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
                         .on(hip))
                 .on(hip);
@@ -83,7 +83,7 @@ class Cgs : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNormReduction<>::build()
+                    gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
                         .on(ref))
                 .on(ref);
diff --git a/hip/test/solver/fcg_kernels.cpp b/hip/test/solver/fcg_kernels.cpp
index 53ff0021e19..663c811227c 100644
--- a/hip/test/solver/fcg_kernels.cpp
+++ b/hip/test/solver/fcg_kernels.cpp
@@ -260,7 +260,7 @@ TEST_F(Fcg, ApplyIsEquivalentToRef)
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(ref))
             .on(ref);
@@ -268,7 +268,7 @@ TEST_F(Fcg, ApplyIsEquivalentToRef)
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(hip),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(hip))
             .on(hip);
diff --git a/omp/test/solver/bicg_kernels.cpp b/omp/test/solver/bicg_kernels.cpp
index 6a18e176119..fc51d1847ae 100644
--- a/omp/test/solver/bicg_kernels.cpp
+++ b/omp/test/solver/bicg_kernels.cpp
@@ -278,7 +278,7 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
         gko::solver::Bicg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(ref))
             .on(ref);
@@ -286,7 +286,7 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
         gko::solver::Bicg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(omp),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(omp))
             .on(omp);
@@ -315,7 +315,7 @@ TEST_F(Bicg, ApplyWithRandomMatrixIsEquivalentToRef)
         gko::solver::Bicg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(ref))
             .on(ref);
@@ -323,7 +323,7 @@ TEST_F(Bicg, ApplyWithRandomMatrixIsEquivalentToRef)
         gko::solver::Bicg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(omp),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(omp))
             .on(omp);
diff --git a/omp/test/solver/bicgstab_kernels.cpp b/omp/test/solver/bicgstab_kernels.cpp
index 83b691b6455..74085b6e7a7 100644
--- a/omp/test/solver/bicgstab_kernels.cpp
+++ b/omp/test/solver/bicgstab_kernels.cpp
@@ -75,7 +75,7 @@ class Bicgstab : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(omp),
-                    gko::stop::ResidualNormReduction<>::build()
+                    gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
                         .on(omp))
                 .on(omp);
@@ -84,7 +84,7 @@ class Bicgstab : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNormReduction<>::build()
+                    gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
                         .on(ref))
                 .on(ref);
diff --git a/omp/test/solver/cg_kernels.cpp b/omp/test/solver/cg_kernels.cpp
index 35d09fc0f6d..e71fad019b5 100644
--- a/omp/test/solver/cg_kernels.cpp
+++ b/omp/test/solver/cg_kernels.cpp
@@ -246,7 +246,7 @@ TEST_F(Cg, ApplyIsEquivalentToRef)
         gko::solver::Cg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(ref))
             .on(ref);
@@ -254,7 +254,7 @@ TEST_F(Cg, ApplyIsEquivalentToRef)
         gko::solver::Cg<>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(omp),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(omp))
             .on(omp);
diff --git a/omp/test/solver/cgs_kernels.cpp b/omp/test/solver/cgs_kernels.cpp
index 1c64f8e22fb..f05d0604919 100644
--- a/omp/test/solver/cgs_kernels.cpp
+++ b/omp/test/solver/cgs_kernels.cpp
@@ -74,7 +74,7 @@ class Cgs : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(omp),
-                    gko::stop::ResidualNormReduction<>::build()
+                    gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
                         .on(omp))
                 .on(omp);
@@ -82,7 +82,7 @@ class Cgs : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNormReduction<>::build()
+                    gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
                         .on(ref))
                 .on(ref);
diff --git a/omp/test/solver/fcg_kernels.cpp b/omp/test/solver/fcg_kernels.cpp
index 9c55c762a78..bcf6c57d288 100644
--- a/omp/test/solver/fcg_kernels.cpp
+++ b/omp/test/solver/fcg_kernels.cpp
@@ -259,7 +259,7 @@ TEST_F(Fcg, ApplyIsEquivalentToRef)
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(ref))
             .on(ref);
@@ -267,7 +267,7 @@ TEST_F(Fcg, ApplyIsEquivalentToRef)
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(omp),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
                     .on(omp))
             .on(omp);
diff --git a/omp/test/solver/gmres_kernels.cpp b/omp/test/solver/gmres_kernels.cpp
index 8e9e8a5a762..31692fb0628 100644
--- a/omp/test/solver/gmres_kernels.cpp
+++ b/omp/test/solver/gmres_kernels.cpp
@@ -80,7 +80,7 @@ class Gmres : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(omp),
-                    gko::stop::ResidualNormReduction<>::build()
+                    gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
                         .on(omp))
                 .on(omp);
@@ -89,7 +89,7 @@ class Gmres : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNormReduction<>::build()
+                    gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
                         .on(ref))
                 .on(ref);
diff --git a/reference/test/preconditioner/ilu.cpp b/reference/test/preconditioner/ilu.cpp
index 10bc527d1b1..3a8713bd686 100644
--- a/reference/test/preconditioner/ilu.cpp
+++ b/reference/test/preconditioner/ilu.cpp
@@ -88,7 +88,7 @@ class Ilu : public ::testing::Test {
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<T>::value)
                           .on(exec))
                   .on(exec)),
@@ -100,7 +100,7 @@ class Ilu : public ::testing::Test {
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<T>::value)
                           .on(exec))
                   .on(exec)),
diff --git a/reference/test/solver/bicg_kernels.cpp b/reference/test/solver/bicg_kernels.cpp
index 5af8b8d0e22..3d74bd8a107 100644
--- a/reference/test/solver/bicg_kernels.cpp
+++ b/reference/test/solver/bicg_kernels.cpp
@@ -68,7 +68,7 @@ class Bicg : public ::testing::Test {
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
@@ -85,7 +85,7 @@ class Bicg : public ::testing::Test {
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
diff --git a/reference/test/solver/bicgstab_kernels.cpp b/reference/test/solver/bicgstab_kernels.cpp
index a95099589c1..0a53fd7c88b 100644
--- a/reference/test/solver/bicgstab_kernels.cpp
+++ b/reference/test/solver/bicgstab_kernels.cpp
@@ -69,7 +69,7 @@ class Bicgstab : public ::testing::Test {
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
@@ -92,7 +92,7 @@ class Bicgstab : public ::testing::Test {
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec))
diff --git a/reference/test/solver/cg_kernels.cpp b/reference/test/solver/cg_kernels.cpp
index 5209a96864b..2eadaa5748a 100644
--- a/reference/test/solver/cg_kernels.cpp
+++ b/reference/test/solver/cg_kernels.cpp
@@ -69,7 +69,7 @@ class Cg : public ::testing::Test {
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
@@ -86,7 +86,7 @@ class Cg : public ::testing::Test {
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
diff --git a/reference/test/solver/cgs_kernels.cpp b/reference/test/solver/cgs_kernels.cpp
index 46954492f90..cfe2a2ef10d 100644
--- a/reference/test/solver/cgs_kernels.cpp
+++ b/reference/test/solver/cgs_kernels.cpp
@@ -67,7 +67,7 @@ class Cgs : public ::testing::Test {
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(40u).on(
                           exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
@@ -84,7 +84,7 @@ class Cgs : public ::testing::Test {
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
diff --git a/reference/test/solver/fcg_kernels.cpp b/reference/test/solver/fcg_kernels.cpp
index bd4d3324f2f..e95299b9c98 100644
--- a/reference/test/solver/fcg_kernels.cpp
+++ b/reference/test/solver/fcg_kernels.cpp
@@ -69,7 +69,7 @@ class Fcg : public ::testing::Test {
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
@@ -86,7 +86,7 @@ class Fcg : public ::testing::Test {
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
diff --git a/reference/test/solver/gmres_kernels.cpp b/reference/test/solver/gmres_kernels.cpp
index 6ccabd9a7a7..cded0a059ba 100644
--- a/reference/test/solver/gmres_kernels.cpp
+++ b/reference/test/solver/gmres_kernels.cpp
@@ -70,7 +70,7 @@ class Gmres : public ::testing::Test {
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
@@ -87,7 +87,7 @@ class Gmres : public ::testing::Test {
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
@@ -331,7 +331,7 @@ TYPED_TEST(Gmres, SolvesBigDenseSystem1WithRestart)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(200u).on(
                     this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
+                gko::stop::ResidualNorm<value_type>::build()
                     .with_reduction_factor(r<value_type>::value)
                     .on(this->exec))
             .on(this->exec);
@@ -357,7 +357,7 @@ TYPED_TEST(Gmres, SolvesWithPreconditioner)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(100u).on(
                     this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
+                gko::stop::ResidualNorm<value_type>::build()
                     .with_reduction_factor(r<value_type>::value)
                     .on(this->exec))
             .with_preconditioner(
diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp
index 882c5565c5b..4240e18f214 100644
--- a/reference/test/solver/idr_kernels.cpp
+++ b/reference/test/solver/idr_kernels.cpp
@@ -73,7 +73,7 @@ class Idr : public ::testing::Test {
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
@@ -86,7 +86,7 @@ class Idr : public ::testing::Test {
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec))
@@ -131,7 +131,7 @@ TYPED_TEST(Idr, SolvesDenseSystemWithComplexSubSpace)
                 gko::stop::Time::build()
                     .with_time_limit(std::chrono::seconds(6))
                     .on(this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
+                gko::stop::ResidualNorm<value_type>::build()
                     .with_reduction_factor(r<value_type>::value)
                     .on(this->exec))
             .on(this->exec);
@@ -180,7 +180,7 @@ TYPED_TEST(Idr, SolvesMultipleDenseSystemsWithComplexSubspace)
                 gko::stop::Time::build()
                     .with_time_limit(std::chrono::seconds(6))
                     .on(this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
+                gko::stop::ResidualNorm<value_type>::build()
                     .with_reduction_factor(r<value_type>::value)
                     .on(this->exec))
             .on(this->exec);
diff --git a/reference/test/solver/ir_kernels.cpp b/reference/test/solver/ir_kernels.cpp
index 48bed351e6d..6ded3841a3d 100644
--- a/reference/test/solver/ir_kernels.cpp
+++ b/reference/test/solver/ir_kernels.cpp
@@ -69,7 +69,7 @@ class Ir : public ::testing::Test {
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(30u).on(
                           exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec))
@@ -105,7 +105,7 @@ TYPED_TEST(Ir, SolvesTriangularSystemWithIterativeInnerSolver)
     const gko::remove_complex<value_type> inner_reduction_factor = 1e-2;
     auto inner_solver_factory =
         gko::solver::Gmres<value_type>::build()
-            .with_criteria(gko::stop::ResidualNormReduction<value_type>::build()
+            .with_criteria(gko::stop::ResidualNorm<value_type>::build()
                                .with_reduction_factor(inner_reduction_factor)
                                .on(this->exec))
             .on(this->exec);
@@ -114,7 +114,7 @@ TYPED_TEST(Ir, SolvesTriangularSystemWithIterativeInnerSolver)
         gko::solver::Ir<value_type>::build()
             .with_criteria(gko::stop::Iteration::build().with_max_iters(30u).on(
                                this->exec),
-                           gko::stop::ResidualNormReduction<value_type>::build()
+                           gko::stop::ResidualNorm<value_type>::build()
                                .with_reduction_factor(r<value_type>::value)
                                .on(this->exec))
             .with_solver(gko::share(inner_solver_factory))
@@ -158,7 +158,7 @@ TYPED_TEST(Ir, SolvesTriangularSystemUsingAdvancedApply)
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), r<value_type>::value * 1e1);
 }
 
 
@@ -218,7 +218,7 @@ TYPED_TEST(Ir, RichardsonSolvesTriangularSystem)
                       .with_criteria(
                           gko::stop::Iteration::build().with_max_iters(100u).on(
                               this->exec),
-                          gko::stop::ResidualNormReduction<value_type>::build()
+                          gko::stop::ResidualNorm<value_type>::build()
                               .with_reduction_factor(r<value_type>::value)
                               .on(this->exec))
                       .with_relaxation_factor(value_type{0.9})
@@ -240,7 +240,7 @@ TYPED_TEST(Ir, RichardsonSolvesTriangularSystemWithIterativeInnerSolver)
     const gko::remove_complex<value_type> inner_reduction_factor = 1e-2;
     auto inner_solver_factory =
         gko::solver::Gmres<value_type>::build()
-            .with_criteria(gko::stop::ResidualNormReduction<value_type>::build()
+            .with_criteria(gko::stop::ResidualNorm<value_type>::build()
                                .with_reduction_factor(inner_reduction_factor)
                                .on(this->exec))
             .on(this->exec);
@@ -249,7 +249,7 @@ TYPED_TEST(Ir, RichardsonSolvesTriangularSystemWithIterativeInnerSolver)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(100u).on(
                     this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
+                gko::stop::ResidualNorm<value_type>::build()
                     .with_reduction_factor(r<value_type>::value)
                     .on(this->exec))
             .with_relaxation_factor(value_type{0.9})
@@ -272,7 +272,7 @@ TYPED_TEST(Ir, RichardsonTransposedSolvesTriangularSystem)
         gko::solver::Ir<value_type>::build()
             .with_criteria(gko::stop::Iteration::build().with_max_iters(30u).on(
                                this->exec),
-                           gko::stop::ResidualNormReduction<value_type>::build()
+                           gko::stop::ResidualNorm<value_type>::build()
                                .with_reduction_factor(r<value_type>::value)
                                .on(this->exec))
             .with_relaxation_factor(value_type{0.9})
@@ -295,7 +295,7 @@ TYPED_TEST(Ir, RichardsonConjTransposedSolvesTriangularSystem)
         gko::solver::Ir<value_type>::build()
             .with_criteria(gko::stop::Iteration::build().with_max_iters(30u).on(
                                this->exec),
-                           gko::stop::ResidualNormReduction<value_type>::build()
+                           gko::stop::ResidualNorm<value_type>::build()
                                .with_reduction_factor(r<value_type>::value)
                                .on(this->exec))
             .with_relaxation_factor(value_type{0.9})

From ffcadd7d6fc7a673fffd9e3b5f66e16ede8b6cfa Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 10 Feb 2021 23:43:08 +0100
Subject: [PATCH 0327/1008] Format and minor fixes.

---
 core/base/array.cpp                     |  5 +-
 core/test/base/array.cpp                | 12 ----
 core/test/matrix/dense.cpp              | 15 -----
 reference/test/base/CMakeLists.txt      |  1 +
 reference/test/base/array.cpp           | 79 +++++++++++++++++++++++++
 reference/test/matrix/dense_kernels.cpp | 15 +++++
 test_install/test_install.cpp           |  4 +-
 test_install/test_install_cuda.cu       | 12 ++--
 8 files changed, 106 insertions(+), 37 deletions(-)
 create mode 100644 reference/test/base/array.cpp

diff --git a/core/base/array.cpp b/core/base/array.cpp
index 6f24322e91c..a36c8655728 100644
--- a/core/base/array.cpp
+++ b/core/base/array.cpp
@@ -89,8 +89,9 @@ void Array<ValueType>::fill(const ValueType value)
 }
 
 
-#define GKO_DECLARE_ARRAY(_type) class Array<_type>
-GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_ARRAY);
+#define GKO_DECLARE_ARRAY_FILL(_type) void Array<_type>::fill(const _type value)
+
+GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_ARRAY_FILL);
 
 
 }  // namespace gko
diff --git a/core/test/base/array.cpp b/core/test/base/array.cpp
index e3f1bf5e69b..91c5e8867b9 100644
--- a/core/test/base/array.cpp
+++ b/core/test/base/array.cpp
@@ -299,18 +299,6 @@ TYPED_TEST(Array, DoesNotCopyBackTemporaryCloneBetweenSameMemory)
 }
 
 
-TYPED_TEST(Array, CanBeFilledWithValue)
-{
-    this->x.fill(TypeParam{42});
-
-    ASSERT_EQ(this->x.get_num_elems(), 2);
-    ASSERT_EQ(this->x.get_data()[0], TypeParam{42});
-    ASSERT_EQ(this->x.get_data()[1], TypeParam{42});
-    ASSERT_EQ(this->x.get_const_data()[0], TypeParam{42});
-    ASSERT_EQ(this->x.get_const_data()[1], TypeParam{42});
-}
-
-
 TYPED_TEST(Array, CanBeCleared)
 {
     this->x.clear();
diff --git a/core/test/matrix/dense.cpp b/core/test/matrix/dense.cpp
index 4af19de0a5c..f352648fadd 100644
--- a/core/test/matrix/dense.cpp
+++ b/core/test/matrix/dense.cpp
@@ -203,21 +203,6 @@ TYPED_TEST(Dense, CanBeDoubleListConstructedWithstride)
 }
 
 
-TYPED_TEST(Dense, CanBeFilledWithValue)
-{
-    using value_type = typename TestFixture::value_type;
-    auto m =
-        gko::initialize<gko::matrix::Dense<TypeParam>>({1.0, 2.0}, this->exec);
-    EXPECT_EQ(m->at(0), value_type{1});
-    EXPECT_EQ(m->at(1), value_type{2});
-
-    m->fill(value_type{42});
-
-    EXPECT_EQ(m->at(0), value_type{42});
-    EXPECT_EQ(m->at(1), value_type{42});
-}
-
-
 TYPED_TEST(Dense, CanBeCopied)
 {
     auto mtx_copy = gko::matrix::Dense<TypeParam>::create(this->exec);
diff --git a/reference/test/base/CMakeLists.txt b/reference/test/base/CMakeLists.txt
index 3386bb01e20..cf9aa96813c 100644
--- a/reference/test/base/CMakeLists.txt
+++ b/reference/test/base/CMakeLists.txt
@@ -1,3 +1,4 @@
+ginkgo_create_test(array)
 ginkgo_create_test(combination)
 ginkgo_create_test(composition)
 ginkgo_create_test(perturbation)
diff --git a/reference/test/base/array.cpp b/reference/test/base/array.cpp
new file mode 100644
index 00000000000..50ec0d3eb30
--- /dev/null
+++ b/reference/test/base/array.cpp
@@ -0,0 +1,79 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/base/array.hpp>
+
+
+#include <algorithm>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+template <typename T>
+class Array : public ::testing::Test {
+protected:
+    Array() : exec(gko::ReferenceExecutor::create()), x(exec, 2)
+    {
+        x.get_data()[0] = 5;
+        x.get_data()[1] = 2;
+    }
+
+    std::shared_ptr<const gko::Executor> exec;
+    gko::Array<T> x;
+};
+
+TYPED_TEST_SUITE(Array, gko::test::ValueAndIndexTypes);
+
+
+TYPED_TEST(Array, CanBeFilledWithValue)
+{
+    this->x.fill(TypeParam{42});
+
+    ASSERT_EQ(this->x.get_num_elems(), 2);
+    ASSERT_EQ(this->x.get_data()[0], TypeParam{42});
+    ASSERT_EQ(this->x.get_data()[1], TypeParam{42});
+    ASSERT_EQ(this->x.get_const_data()[0], TypeParam{42});
+    ASSERT_EQ(this->x.get_const_data()[1], TypeParam{42});
+}
+
+
+}  // namespace
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index d8d8758af21..9984fb7da74 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -109,6 +109,21 @@ class Dense : public ::testing::Test {
 TYPED_TEST_SUITE(Dense, gko::test::ValueTypes);
 
 
+TYPED_TEST(Dense, CanBeFilledWithValue)
+{
+    using value_type = typename TestFixture::value_type;
+    auto m =
+        gko::initialize<gko::matrix::Dense<TypeParam>>({1.0, 2.0}, this->exec);
+    EXPECT_EQ(m->at(0), value_type{1});
+    EXPECT_EQ(m->at(1), value_type{2});
+
+    m->fill(value_type{42});
+
+    EXPECT_EQ(m->at(0), value_type{42});
+    EXPECT_EQ(m->at(1), value_type{42});
+}
+
+
 TYPED_TEST(Dense, AppliesToDense)
 {
     using T = typename TestFixture::value_type;
diff --git a/test_install/test_install.cpp b/test_install/test_install.cpp
index 115e7b15b46..969a059edf3 100644
--- a/test_install/test_install.cpp
+++ b/test_install/test_install.cpp
@@ -355,12 +355,12 @@ int main(int, char **)
         // residual_norm.hpp
         auto main_res = gko::stop::ResidualNorm<>::build()
                             .with_reduction_factor(1e-10)
-                            .with_baseline(gko::stop::mode_absolute)
+                            .with_baseline(gko::stop::mode::absolute)
                             .on(refExec);
 
         auto implicit_res = gko::stop::ImplicitResidualNorm<>::build()
                                 .with_reduction_factor(1e-10)
-                                .with_baseline(gko::stop::mode_absolute)
+                                .with_baseline(gko::stop::mode::absolute)
                                 .on(refExec);
 
         auto res_red = gko::stop::ResidualNormReduction<>::build()
diff --git a/test_install/test_install_cuda.cu b/test_install/test_install_cuda.cu
index bf2dd5a895a..6def39b0c04 100644
--- a/test_install/test_install_cuda.cu
+++ b/test_install/test_install_cuda.cu
@@ -350,14 +350,14 @@ int main(int, char **)
             .on(cudaExec);
 
         gko::stop::ResidualNorm<>::build()
-                            .with_reduction_factor(1e-10)
-                            .with_baseline(gko::stop::mode_absolute)
-                            .on(cudaExec);
+            .with_reduction_factor(1e-10)
+            .with_baseline(gko::stop::mode::absolute)
+            .on(cudaExec);
 
         gko::stop::ImplicitResidualNorm<>::build()
-                                .with_reduction_factor(1e-10)
-                                .with_baseline(gko::stop::mode_absolute)
-                                .on(cudaExec);
+            .with_reduction_factor(1e-10)
+            .with_baseline(gko::stop::mode::absolute)
+            .on(cudaExec);
 
         gko::stop::RelativeResidualNorm<>::build().with_tolerance(1e-10).on(
             cudaExec);

From f1f4ce811081a55ef34e55405d84f570361034d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 16 Feb 2021 20:55:58 +0100
Subject: [PATCH 0328/1008] Remove code duplication for stopping criterions

- Create ResidualNormBase, which creates the framework for all residual
  norm stopping criterions and implements `check_impl` for ResidualNorm
- All residual stopping criterions now inherit from ResidualNormBase to
  avoid code duplication
- Remove C++ attribute deprecated because it generated too many
  warnings during compilation
---
 core/stop/residual_norm.cpp                |  18 +-
 include/ginkgo/core/stop/residual_norm.hpp | 330 ++++++++-------------
 2 files changed, 128 insertions(+), 220 deletions(-)

diff --git a/core/stop/residual_norm.cpp b/core/stop/residual_norm.cpp
index a8f2599119d..45635b1384f 100644
--- a/core/stop/residual_norm.cpp
+++ b/core/stop/residual_norm.cpp
@@ -60,10 +60,9 @@ GKO_REGISTER_OPERATION(implicit_residual_norm,
 
 
 template <typename ValueType>
-bool ResidualNorm<ValueType>::check_impl(uint8 stoppingId, bool setFinalized,
-                                         Array<stopping_status> *stop_status,
-                                         bool *one_changed,
-                                         const Criterion::Updater &updater)
+bool ResidualNormBase<ValueType>::check_impl(
+    uint8 stoppingId, bool setFinalized, Array<stopping_status> *stop_status,
+    bool *one_changed, const Criterion::Updater &updater)
 {
     const NormVector *dense_tau;
     if (updater.residual_norm_ != nullptr) {
@@ -107,22 +106,13 @@ bool ImplicitResidualNorm<ValueType>::check_impl(
     this->get_executor()->run(
         implicit_residual_norm::make_implicit_residual_norm(
             dense_tau, this->starting_tau_.get(), this->reduction_factor_,
-            stoppingId, setFinalized, stop_status, &device_storage_,
+            stoppingId, setFinalized, stop_status, &this->device_storage_,
             &all_converged, one_changed));
 
     return all_converged;
 }
 
 
-template <typename ValueType>
-void AbsoluteResidualNorm<ValueType>::initialize_starting_tau()
-{
-    this->get_executor()->run(residual_norm::make_fill_array(
-        this->starting_tau_->get_values(), this->starting_tau_->get_size()[1],
-        gko::one<remove_complex<ValueType>>()));
-}
-
-
 #define GKO_DECLARE_RESIDUAL_NORM(_type) class ResidualNorm<_type>
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_RESIDUAL_NORM);
 
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index d0ac7e06ce4..8c2faa5962c 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -55,73 +55,44 @@ enum class mode { absolute, initial_resnorm, rhs_norm };
 
 
 /**
- * The ResidualNorm class provides a framework for stopping criteria
+ * The ResidualNormBase class provides a framework for stopping criteria
  * related to the residual norm. These criteria differ in the way they
  * initialize starting_tau_, so in the value they compare the
  * residual norm against.
+ * The provided check_impl uses the actual residual to check for convergence.
  *
  * @ingroup stop
  */
-template <typename ValueType = default_precision>
-class ResidualNorm
-    : public EnablePolymorphicObject<ResidualNorm<ValueType>, Criterion> {
-    friend class EnablePolymorphicObject<ResidualNorm<ValueType>, Criterion>;
+template <typename ValueType>
+class ResidualNormBase
+    : public EnablePolymorphicObject<ResidualNormBase<ValueType>, Criterion> {
+    friend class EnablePolymorphicObject<ResidualNormBase<ValueType>,
+                                         Criterion>;
 
-public:
+protected:
     using ComplexVector = matrix::Dense<to_complex<ValueType>>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
     using Vector = matrix::Dense<ValueType>;
-
-    GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
-    {
-        /**
-         * Residual norm reduction factor
-         */
-        remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(
-            reduction_factor, static_cast<remove_complex<ValueType>>(1e-15));
-
-        /**
-         * The quantity the reduction is relative to. Choices include
-         * "mode::rhs_norm", "mode::initial_resnorm" and "mode::absolute"
-         */
-        mode GKO_FACTORY_PARAMETER_SCALAR(baseline, mode::rhs_norm);
-    };
-    GKO_ENABLE_CRITERION_FACTORY(ResidualNorm<ValueType>, parameters, Factory);
-    GKO_ENABLE_BUILD_METHOD(Factory);
-
-protected:
     bool check_impl(uint8 stoppingId, bool setFinalized,
                     Array<stopping_status> *stop_status, bool *one_changed,
                     const Criterion::Updater &) override;
 
-    explicit ResidualNorm(std::shared_ptr<const gko::Executor> exec)
-        : EnablePolymorphicObject<ResidualNorm, Criterion>(exec),
+    explicit ResidualNormBase(std::shared_ptr<const gko::Executor> exec)
+        : EnablePolymorphicObject<ResidualNormBase, Criterion>(exec),
           device_storage_{exec, 2}
     {}
 
-    explicit ResidualNorm(std::shared_ptr<const gko::Executor> exec,
-                          remove_complex<ValueType> reduction_factor)
-        : EnablePolymorphicObject<ResidualNorm, Criterion>(exec),
+    explicit ResidualNormBase(std::shared_ptr<const gko::Executor> exec,
+                              const CriterionArgs &args,
+                              remove_complex<ValueType> reduction_factor,
+                              mode baseline)
+        : EnablePolymorphicObject<ResidualNormBase, Criterion>(exec),
           device_storage_{exec, 2},
-          reduction_factor_{reduction_factor}
-    {}
-
-    explicit ResidualNorm(const Factory *factory, const CriterionArgs &args)
-        : EnablePolymorphicObject<ResidualNorm, Criterion>(
-              factory->get_executor()),
-          device_storage_{factory->get_executor(), 2},
-          parameters_{factory->get_parameters()},
-          reduction_factor_{factory->get_parameters().reduction_factor}
+          reduction_factor_{reduction_factor},
+          baseline_{baseline}
     {
-        this->baseline_rhs_norm_ =
-            factory->get_parameters().baseline == mode::rhs_norm;
-        this->baseline_init_resnorm_ =
-            factory->get_parameters().baseline == mode::initial_resnorm;
-        this->baseline_absolute_ =
-            factory->get_parameters().baseline == mode::absolute;
-        auto exec = factory->get_executor();
-
-        if (this->baseline_init_resnorm_) {
+        switch (baseline_) {
+        case mode::initial_resnorm: {
             if (args.initial_residual == nullptr) {
                 GKO_NOT_SUPPORTED(nullptr);
             }
@@ -134,7 +105,9 @@ class ResidualNorm
                 auto dense_r = as<Vector>(args.initial_residual);
                 dense_r->compute_norm2(this->starting_tau_.get());
             }
-        } else if (this->baseline_rhs_norm_) {
+            break;
+        }
+        case mode::rhs_norm: {
             if (args.b == nullptr) {
                 GKO_NOT_SUPPORTED(nullptr);
             }
@@ -147,30 +120,86 @@ class ResidualNorm
                 auto dense_rhs = as<Vector>(args.b);
                 dense_rhs->compute_norm2(this->starting_tau_.get());
             }
-        } else if (this->baseline_absolute_) {
+            break;
+        }
+        case mode::absolute: {
             if (args.b == nullptr) {
                 GKO_NOT_SUPPORTED(nullptr);
             }
             this->starting_tau_ =
                 NormVector::create(exec, dim<2>{1, args.b->get_size()[1]});
             this->starting_tau_->fill(gko::one<remove_complex<ValueType>>());
-        } else {
+            break;
+        }
+        default:
             GKO_NOT_SUPPORTED(nullptr);
         }
         this->u_dense_tau_ =
             NormVector::create_with_config_of(this->starting_tau_.get());
     }
 
+    remove_complex<ValueType> reduction_factor_{};
     std::unique_ptr<NormVector> starting_tau_{};
     std::unique_ptr<NormVector> u_dense_tau_{};
-
-private:
-    remove_complex<ValueType> reduction_factor_{};
-    bool baseline_rhs_norm_{};
-    bool baseline_init_resnorm_{};
-    bool baseline_absolute_{};
     /* Contains device side: all_converged and one_changed booleans */
     Array<bool> device_storage_;
+
+private:
+    mode baseline_{mode::rhs_norm};
+};
+
+
+/**
+ * The ResidualNorm class is a stopping criterion which
+ * stops the iteration process when the actual residual norm is below a
+ * certain threshold relative to either the norm of the right-hand side, i.e.
+ * when norm(residual) / norm(right_hand_side) < threshold or the initial
+ * residual, i.e. when norm(residual) / norm(initial_residual) < threshold. For
+ * better performance, the checks are run on the executor
+ * where the algorithm is executed.
+ *
+ * @note To use this stopping criterion there are some dependencies. The
+ * constructor depends on either `b` or the `initial_residual` in order to
+ * compute their norms. If this is not correctly provided, an exception
+ * ::gko::NotSupported() is thrown.
+ *
+ * @ingroup stop
+ */
+template <typename ValueType = default_precision>
+class ResidualNorm : public ResidualNormBase<ValueType> {
+public:
+    using ComplexVector = matrix::Dense<to_complex<ValueType>>;
+    using NormVector = matrix::Dense<remove_complex<ValueType>>;
+    using Vector = matrix::Dense<ValueType>;
+
+    GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
+    {
+        /**
+         * Residual norm reduction factor
+         */
+        remove_complex<ValueType> GKO_FACTORY_PARAMETER_SCALAR(
+            reduction_factor, static_cast<remove_complex<ValueType>>(1e-15));
+
+        /**
+         * The quantity the reduction is relative to. Choices include
+         * "mode::rhs_norm", "mode::initial_resnorm" and "mode::absolute"
+         */
+        mode GKO_FACTORY_PARAMETER_SCALAR(baseline, mode::rhs_norm);
+    };
+    GKO_ENABLE_CRITERION_FACTORY(ResidualNorm<ValueType>, parameters, Factory);
+    GKO_ENABLE_BUILD_METHOD(Factory);
+
+protected:
+    explicit ResidualNorm(std::shared_ptr<const gko::Executor> exec)
+        : ResidualNormBase<ValueType>(exec)
+    {}
+
+    explicit ResidualNorm(const Factory *factory, const CriterionArgs &args)
+        : ResidualNormBase<ValueType>(
+              factory->get_executor(), args,
+              factory->get_parameters().reduction_factor,
+              factory->get_parameters().baseline)
+    {}
 };
 
 
@@ -191,12 +220,7 @@ class ResidualNorm
  * @ingroup stop
  */
 template <typename ValueType = default_precision>
-class ImplicitResidualNorm
-    : public EnablePolymorphicObject<ImplicitResidualNorm<ValueType>,
-                                     Criterion> {
-    friend class EnablePolymorphicObject<ImplicitResidualNorm<ValueType>,
-                                         Criterion>;
-
+class ImplicitResidualNorm : public ResidualNormBase<ValueType> {
 public:
     using ComplexVector = matrix::Dense<to_complex<ValueType>>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
@@ -221,81 +245,24 @@ class ImplicitResidualNorm
     GKO_ENABLE_BUILD_METHOD(Factory);
 
 protected:
+    // check_impl needs to be overwritten again since we focus on the implicit
+    // residual here
     bool check_impl(uint8 stoppingId, bool setFinalized,
                     Array<stopping_status> *stop_status, bool *one_changed,
                     const Criterion::Updater &) override;
 
     explicit ImplicitResidualNorm(std::shared_ptr<const gko::Executor> exec)
-        : EnablePolymorphicObject<ImplicitResidualNorm, Criterion>(exec),
-          device_storage_{exec, 2}
+        : ResidualNormBase<ValueType>(exec)
     {}
 
     explicit ImplicitResidualNorm(const Factory *factory,
                                   const CriterionArgs &args)
-        : EnablePolymorphicObject<ImplicitResidualNorm, Criterion>(
-              factory->get_executor()),
-          device_storage_{factory->get_executor(), 2},
-          parameters_{factory->get_parameters()},
-          reduction_factor_{factory->get_parameters().reduction_factor}
-    {
-        this->baseline_rhs_norm_ =
-            factory->get_parameters().baseline == mode::rhs_norm;
-        this->baseline_init_resnorm_ =
-            factory->get_parameters().baseline == mode::initial_resnorm;
-        this->baseline_absolute_ =
-            factory->get_parameters().baseline == mode::absolute;
-        auto exec = factory->get_executor();
-
-        if (this->baseline_init_resnorm_) {
-            if (args.initial_residual == nullptr) {
-                GKO_NOT_SUPPORTED(nullptr);
-            }
-            this->starting_tau_ = NormVector::create(
-                exec, dim<2>{1, args.initial_residual->get_size()[1]});
-            if (dynamic_cast<const ComplexVector *>(args.initial_residual)) {
-                auto dense_r = as<ComplexVector>(args.initial_residual);
-                dense_r->compute_norm2(this->starting_tau_.get());
-            } else {
-                auto dense_r = as<Vector>(args.initial_residual);
-                dense_r->compute_norm2(this->starting_tau_.get());
-            }
-        } else if (this->baseline_rhs_norm_) {
-            if (args.b == nullptr) {
-                GKO_NOT_SUPPORTED(nullptr);
-            }
-            this->starting_tau_ =
-                NormVector::create(exec, dim<2>{1, args.b->get_size()[1]});
-            if (dynamic_cast<const ComplexVector *>(args.b.get())) {
-                auto dense_rhs = as<ComplexVector>(args.b);
-                dense_rhs->compute_norm2(this->starting_tau_.get());
-            } else {
-                auto dense_rhs = as<Vector>(args.b);
-                dense_rhs->compute_norm2(this->starting_tau_.get());
-            }
-        } else if (this->baseline_absolute_) {
-            if (args.b == nullptr) {
-                GKO_NOT_SUPPORTED(nullptr);
-            }
-            this->starting_tau_ =
-                NormVector::create(exec, dim<2>{1, args.b->get_size()[1]});
-            this->starting_tau_->fill(gko::one<remove_complex<ValueType>>());
-        } else {
-            GKO_NOT_SUPPORTED(nullptr);
-        }
-        this->u_dense_tau_ =
-            NormVector::create_with_config_of(this->starting_tau_.get());
-    }
-
-    std::unique_ptr<NormVector> starting_tau_{};
-    std::unique_ptr<NormVector> u_dense_tau_{};
-
-private:
-    remove_complex<ValueType> reduction_factor_{};
-    bool baseline_rhs_norm_{};
-    bool baseline_init_resnorm_{};
-    bool baseline_absolute_{};
-    /* Contains device side: all_converged and one_changed booleans */
-    Array<bool> device_storage_;
+        : ResidualNormBase<ValueType>(
+              factory->get_executor(), args,
+              factory->get_parameters().reduction_factor,
+              factory->get_parameters().baseline),
+          parameters_{factory->get_parameters()}
+    {}
 };
 
 
@@ -313,14 +280,13 @@ class ImplicitResidualNorm
  * `residual_norm` or the `residual` being set. When any of those is not
  * correctly provided, an exception ::gko::NotSupported() is thrown.
  *
+ * @deprecated Please use the class ResidualNorm with the factory parameter
+ *             baseline = mode::initial_resnorm
+ *
  * @ingroup stop
  */
 template <typename ValueType = default_precision>
-class [
-    [deprecated("Please use the ResidualNorm criterion with a init_resnorm "
-                "baseline mode instead.")]] ResidualNormReduction
-    : public ResidualNorm<ValueType>
-{
+class ResidualNormReduction : public ResidualNormBase<ValueType> {
 public:
     using ComplexVector = matrix::Dense<to_complex<ValueType>>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
@@ -340,33 +306,17 @@ class [
 
 protected:
     explicit ResidualNormReduction(std::shared_ptr<const gko::Executor> exec)
-        : ResidualNorm<ValueType>(exec)
+        : ResidualNormBase<ValueType>(exec)
     {}
 
     explicit ResidualNormReduction(const Factory *factory,
                                    const CriterionArgs &args)
-        : ResidualNorm<ValueType>(factory->get_executor(),
-                                  factory->get_parameters().reduction_factor),
+        : ResidualNormBase<ValueType>(
+              factory->get_executor(), args,
+              factory->get_parameters().reduction_factor,
+              mode::initial_resnorm),
           parameters_{factory->get_parameters()}
-    {
-        if (args.initial_residual == nullptr) {
-            GKO_NOT_SUPPORTED(nullptr);
-        }
-
-        auto exec = factory->get_executor();
-
-        this->starting_tau_ = NormVector::create(
-            exec, dim<2>{1, args.initial_residual->get_size()[1]});
-        this->u_dense_tau_ =
-            NormVector::create_with_config_of(this->starting_tau_.get());
-        if (dynamic_cast<const ComplexVector *>(args.initial_residual)) {
-            auto dense_r = as<ComplexVector>(args.initial_residual);
-            dense_r->compute_norm2(this->starting_tau_.get());
-        } else {
-            auto dense_r = as<Vector>(args.initial_residual);
-            dense_r->compute_norm2(this->starting_tau_.get());
-        }
-    }
+    {}
 };
 
 
@@ -383,14 +333,13 @@ class [
  * right-hand side. If this is not correctly provided, an exception
  * ::gko::NotSupported() is thrown.
  *
+ * @deprecated Please use the class ResidualNorm with the factory parameter
+ *             baseline = mode::rhs_norm
+ *
  * @ingroup stop
  */
 template <typename ValueType = default_precision>
-class [
-    [deprecated("Please use the ResidualNorm criterion with a rhs_norm "
-                "baseline mode instead.")]] RelativeResidualNorm
-    : public ResidualNorm<ValueType>
-{
+class RelativeResidualNorm : public ResidualNormBase<ValueType> {
 public:
     using ComplexVector = matrix::Dense<to_complex<ValueType>>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
@@ -410,33 +359,16 @@ class [
 
 protected:
     explicit RelativeResidualNorm(std::shared_ptr<const gko::Executor> exec)
-        : ResidualNorm<ValueType>(exec)
+        : ResidualNormBase<ValueType>(exec)
     {}
 
     explicit RelativeResidualNorm(const Factory *factory,
                                   const CriterionArgs &args)
-        : ResidualNorm<ValueType>(factory->get_executor(),
-                                  factory->get_parameters().tolerance),
+        : ResidualNormBase<ValueType>(factory->get_executor(), args,
+                                      factory->get_parameters().tolerance,
+                                      mode::rhs_norm),
           parameters_{factory->get_parameters()}
-    {
-        if (args.b == nullptr) {
-            GKO_NOT_SUPPORTED(nullptr);
-        }
-
-        auto exec = factory->get_executor();
-
-        this->starting_tau_ =
-            NormVector::create(exec, dim<2>{1, args.b->get_size()[1]});
-        this->u_dense_tau_ =
-            NormVector::create_with_config_of(this->starting_tau_.get());
-        if (dynamic_cast<const ComplexVector *>(args.b.get())) {
-            auto dense_rhs = as<ComplexVector>(args.b);
-            dense_rhs->compute_norm2(this->starting_tau_.get());
-        } else {
-            auto dense_rhs = as<Vector>(args.b);
-            dense_rhs->compute_norm2(this->starting_tau_.get());
-        }
-    }
+    {}
 };
 
 
@@ -452,14 +384,13 @@ class [
  * If this is not correctly provided, an exception ::gko::NotSupported()
  * is thrown.
  *
+ * @deprecated Please use the class ResidualNorm with the factory parameter
+ *             baseline = mode::absolute
+ *
  * @ingroup stop
  */
 template <typename ValueType = default_precision>
-class [
-    [deprecated("Please use the ResidualNorm criterion with an absolute "
-                "baseline mode instead.")]] AbsoluteResidualNorm
-    : public ResidualNorm<ValueType>
-{
+class AbsoluteResidualNorm : public ResidualNormBase<ValueType> {
 public:
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
     using Vector = matrix::Dense<ValueType>;
@@ -477,30 +408,17 @@ class [
     GKO_ENABLE_BUILD_METHOD(Factory);
 
 protected:
-    void initialize_starting_tau();
-
     explicit AbsoluteResidualNorm(std::shared_ptr<const gko::Executor> exec)
-        : ResidualNorm<ValueType>(exec)
+        : ResidualNormBase<ValueType>(exec)
     {}
 
     explicit AbsoluteResidualNorm(const Factory *factory,
                                   const CriterionArgs &args)
-        : ResidualNorm<ValueType>(factory->get_executor(),
-                                  factory->get_parameters().tolerance),
+        : ResidualNormBase<ValueType>(factory->get_executor(), args,
+                                      factory->get_parameters().tolerance,
+                                      mode::absolute),
           parameters_{factory->get_parameters()}
-    {
-        if (args.b == nullptr) {
-            GKO_NOT_SUPPORTED(nullptr);
-        }
-
-        auto exec = factory->get_executor();
-
-        this->starting_tau_ =
-            NormVector::create(exec, dim<2>{1, args.b->get_size()[1]});
-        this->u_dense_tau_ =
-            NormVector::create_with_config_of(this->starting_tau_.get());
-        initialize_starting_tau();
-    }
+    {}
 };
 
 

From 589bf5fcbbcb12c1c6cedfa1d85ce9f57f234305 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 17 Feb 2021 10:09:22 +0100
Subject: [PATCH 0329/1008] Review update.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

+ Update documentation.

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 core/test/solver/bicg.cpp                     |  4 +-
 include/ginkgo/core/preconditioner/ic.hpp     |  2 +-
 include/ginkgo/core/preconditioner/ilu.hpp    |  2 +-
 include/ginkgo/core/stop/residual_norm.hpp    | 35 ++++++++++----
 reference/test/stop/residual_norm_kernels.cpp | 46 ++++++++++---------
 5 files changed, 53 insertions(+), 36 deletions(-)

diff --git a/core/test/solver/bicg.cpp b/core/test/solver/bicg.cpp
index 26a6440eef4..f29620ad2e7 100644
--- a/core/test/solver/bicg.cpp
+++ b/core/test/solver/bicg.cpp
@@ -67,7 +67,7 @@ class Bicg : public ::testing::Test {
               Solver::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(3u).on(exec),
-                      gko::stop::ResidualNorm<>::build()
+                      gko::stop::ResidualNorm<value_type>::build()
                           .with_reduction_factor(gko::remove_complex<T>{1e-6})
                           .on(exec))
                   .on(exec)),
@@ -181,7 +181,7 @@ TYPED_TEST(Bicg, CanSetPreconditionerGenerator)
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(this->exec),
-                gko::stop::ResidualNorm<>::build()
+                gko::stop::ResidualNorm<value_type>::build()
                     .with_reduction_factor(
                         gko::remove_complex<value_type>(1e-6))
                     .on(this->exec))
diff --git a/include/ginkgo/core/preconditioner/ic.hpp b/include/ginkgo/core/preconditioner/ic.hpp
index ec1f80607b6..ead157415b0 100644
--- a/include/ginkgo/core/preconditioner/ic.hpp
+++ b/include/ginkgo/core/preconditioner/ic.hpp
@@ -302,7 +302,7 @@ class Ic : public EnableLinOp<Ic<LSolverType, IndexType>>, public Transposable {
             .with_criteria(gko::stop::Iteration::build()
                                .with_max_iters(default_max_iters)
                                .on(exec),
-                           gko::stop::ResidualNormReduction<value_type>::build()
+                           gko::stop::ResidualNorm<value_type>::build()
                                .with_reduction_factor(default_reduce_residual)
                                .on(exec))
             .on(exec)
diff --git a/include/ginkgo/core/preconditioner/ilu.hpp b/include/ginkgo/core/preconditioner/ilu.hpp
index f138212bb4a..42272288a22 100644
--- a/include/ginkgo/core/preconditioner/ilu.hpp
+++ b/include/ginkgo/core/preconditioner/ilu.hpp
@@ -329,7 +329,7 @@ class Ilu : public EnableLinOp<
             .with_criteria(gko::stop::Iteration::build()
                                .with_max_iters(default_max_iters)
                                .on(exec),
-                           gko::stop::ResidualNormReduction<value_type>::build()
+                           gko::stop::ResidualNorm<value_type>::build()
                                .with_reduction_factor(default_reduce_residual)
                                .on(exec))
             .on(exec)
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index 8c2faa5962c..a592807ac7c 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -49,7 +49,18 @@ namespace stop {
 
 
 /**
- * The enum that holds the quantity relative to the implicit residual.
+ * The mode for the residual norm criterion.
+ *
+ * - absolute:        Check for tolerance against residual norm.
+ *                    || r || < tau
+ *
+ * - initial_resnorm: Check for tolerance relative to the initial residual norm.
+ *                    || r ||/|| r_0|| < tau
+ *
+ * - rhs_resnorm:     Check for tolerance relative to the rhs norm.
+ *                    || r ||/|| b || < tau
+ *
+ * @ingroup stop
  */
 enum class mode { absolute, initial_resnorm, rhs_norm };
 
@@ -152,10 +163,13 @@ class ResidualNormBase
 /**
  * The ResidualNorm class is a stopping criterion which
  * stops the iteration process when the actual residual norm is below a
- * certain threshold relative to either the norm of the right-hand side, i.e.
- * when norm(residual) / norm(right_hand_side) < threshold or the initial
- * residual, i.e. when norm(residual) / norm(initial_residual) < threshold. For
- * better performance, the checks are run on the executor
+ * certain threshold relative to
+ * 1. the norm of the right-hand side, norm(residual) / norm(right_hand_side)
+ *                                                                  < threshold
+ * 2. the initial residual, norm(residual) / norm(initial_residual) < threshold.
+ * 3. one,  norm(residual) < threshold.
+ *
+ * For better performance, the checks are run on the executor
  * where the algorithm is executed.
  *
  * @note To use this stopping criterion there are some dependencies. The
@@ -206,11 +220,12 @@ class ResidualNorm : public ResidualNormBase<ValueType> {
 /**
  * The ImplicitResidualNorm class is a stopping criterion which
  * stops the iteration process when the implicit residual norm is below a
- * certain threshold relative to either the norm of the right-hand side, i.e.
- * when norm(residual) / norm(right_hand_side) < threshold or the initial
- * residual, i.e. when norm(residual) / norm(initial_residual) < threshold. For
- * better performance, the checks are run on the executor
- * where the algorithm is executed.
+ * certain threshold relative to
+ * 1. the norm of the right-hand side, implicit_resnorm / norm(right_hand_side)
+ *                                                          < threshold
+ * 2. the initial residual, implicit_resnorm / norm(initial_residual) <
+ *                                                          < threshold.
+ * 3. one, implicit_resnorm < threshold.
  *
  * @note To use this stopping criterion there are some dependencies. The
  * constructor depends on either `b` or the `initial_residual` in order to
diff --git a/reference/test/stop/residual_norm_kernels.cpp b/reference/test/stop/residual_norm_kernels.cpp
index 0cd48cafb6d..b7676c807ee 100644
--- a/reference/test/stop/residual_norm_kernels.cpp
+++ b/reference/test/stop/residual_norm_kernels.cpp
@@ -58,9 +58,9 @@ class ResidualNorm : public ::testing::Test {
     ResidualNorm()
     {
         exec_ = gko::ReferenceExecutor::create();
-        factory_ = gko::stop::ResidualNorm<T>::build()
-                       .with_reduction_factor(r<T>::value)
-                       .on(exec_);
+        rhs_factory_ = gko::stop::ResidualNorm<T>::build()
+                           .with_reduction_factor(r<T>::value)
+                           .on(exec_);
         rel_factory_ = gko::stop::ResidualNorm<T>::build()
                            .with_reduction_factor(r<T>::value)
                            .with_baseline(gko::stop::mode::initial_resnorm)
@@ -71,7 +71,7 @@ class ResidualNorm : public ::testing::Test {
                            .on(exec_);
     }
 
-    std::unique_ptr<typename gko::stop::ResidualNorm<T>::Factory> factory_;
+    std::unique_ptr<typename gko::stop::ResidualNorm<T>::Factory> rhs_factory_;
     std::unique_ptr<typename gko::stop::ResidualNorm<T>::Factory> rel_factory_;
     std::unique_ptr<typename gko::stop::ResidualNorm<T>::Factory> abs_factory_;
     std::shared_ptr<const gko::Executor> exec_;
@@ -82,12 +82,12 @@ TYPED_TEST_SUITE(ResidualNorm, gko::test::ValueTypes);
 
 TYPED_TEST(ResidualNorm, CanCreateFactory)
 {
-    ASSERT_NE(this->factory_, nullptr);
-    ASSERT_EQ(this->factory_->get_parameters().reduction_factor,
+    ASSERT_NE(this->rhs_factory_, nullptr);
+    ASSERT_EQ(this->rhs_factory_->get_parameters().reduction_factor,
               r<TypeParam>::value);
-    ASSERT_EQ(this->factory_->get_parameters().baseline,
+    ASSERT_EQ(this->rhs_factory_->get_parameters().baseline,
               gko::stop::mode::rhs_norm);
-    ASSERT_EQ(this->factory_->get_executor(), this->exec_);
+    ASSERT_EQ(this->rhs_factory_->get_executor(), this->exec_);
     ASSERT_NE(this->rel_factory_, nullptr);
     ASSERT_EQ(this->rel_factory_->get_parameters().reduction_factor,
               r<TypeParam>::value);
@@ -105,8 +105,9 @@ TYPED_TEST(ResidualNorm, CanCreateFactory)
 
 TYPED_TEST(ResidualNorm, CannotCreateCriterionWithoutNeededInput)
 {
-    ASSERT_THROW(this->factory_->generate(nullptr, nullptr, nullptr, nullptr),
-                 gko::NotSupported);
+    ASSERT_THROW(
+        this->rhs_factory_->generate(nullptr, nullptr, nullptr, nullptr),
+        gko::NotSupported);
     ASSERT_THROW(
         this->rel_factory_->generate(nullptr, nullptr, nullptr, nullptr),
         gko::NotSupported);
@@ -121,14 +122,14 @@ TYPED_TEST(ResidualNorm, CanCreateCriterionWithNeededInput)
     using Mtx = typename TestFixture::Mtx;
     std::shared_ptr<gko::LinOp> scalar =
         gko::initialize<Mtx>({1.0}, this->exec_);
-    auto criterion =
-        this->factory_->generate(nullptr, scalar, nullptr, nullptr);
+    auto rhs_criterion =
+        this->rhs_factory_->generate(nullptr, scalar, nullptr, nullptr);
     auto rel_criterion =
         this->rel_factory_->generate(nullptr, nullptr, nullptr, scalar.get());
     auto abs_criterion =
         this->abs_factory_->generate(nullptr, scalar, nullptr, nullptr);
 
-    ASSERT_NE(criterion, nullptr);
+    ASSERT_NE(rhs_criterion, nullptr);
     ASSERT_NE(rel_criterion, nullptr);
     ASSERT_NE(abs_criterion, nullptr);
 }
@@ -141,8 +142,8 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoal)
     using T_nc = gko::remove_complex<TypeParam>;
     auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
     std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->exec_);
-    auto criterion =
-        this->factory_->generate(nullptr, rhs, nullptr, initial_res.get());
+    auto rhs_criterion =
+        this->rhs_factory_->generate(nullptr, rhs, nullptr, initial_res.get());
     auto rel_criterion =
         this->rel_factory_->generate(nullptr, rhs, nullptr, initial_res.get());
     auto abs_criterion =
@@ -157,13 +158,13 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoal)
         stop_status.get_data()[0].reset();
 
         ASSERT_FALSE(
-            criterion->update()
+            rhs_criterion->update()
                 .residual_norm(res_norm.get())
                 .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
         res_norm->at(0) = r<TypeParam>::value * 1.1 * res_norm->at(0);
         ASSERT_FALSE(
-            criterion->update()
+            rhs_criterion->update()
                 .residual_norm(res_norm.get())
                 .check(RelativeStoppingId, true, &stop_status, &one_changed));
         ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
@@ -171,7 +172,7 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoal)
 
         res_norm->at(0) = r<TypeParam>::value * 0.9 * res_norm->at(0);
         ASSERT_TRUE(
-            criterion->update()
+            rhs_criterion->update()
                 .residual_norm(res_norm.get())
                 .check(RelativeStoppingId, true, &stop_status, &one_changed));
         ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
@@ -245,7 +246,8 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoalMultipleRHS)
     auto res = gko::initialize<Mtx>({I<T>{100.0, 100.0}}, this->exec_);
     std::shared_ptr<gko::LinOp> rhs =
         gko::initialize<Mtx>({I<T>{10.0, 10.0}}, this->exec_);
-    auto criterion = this->factory_->generate(nullptr, rhs, nullptr, res.get());
+    auto rhs_criterion =
+        this->rhs_factory_->generate(nullptr, rhs, nullptr, res.get());
     auto rel_criterion =
         this->rel_factory_->generate(nullptr, rhs, nullptr, res.get());
     auto abs_criterion =
@@ -263,13 +265,13 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoalMultipleRHS)
         stop_status.get_data()[1].reset();
 
         ASSERT_FALSE(
-            criterion->update()
+            rhs_criterion->update()
                 .residual_norm(res_norm.get())
                 .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
         res_norm->at(0, 0) = r<TypeParam>::value * 0.9 * rhs_norm->at(0, 0);
         ASSERT_FALSE(
-            criterion->update()
+            rhs_criterion->update()
                 .residual_norm(res_norm.get())
                 .check(RelativeStoppingId, true, &stop_status, &one_changed));
         ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
@@ -277,7 +279,7 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoalMultipleRHS)
 
         res_norm->at(0, 1) = r<TypeParam>::value * 0.9 * rhs_norm->at(0, 1);
         ASSERT_TRUE(
-            criterion->update()
+            rhs_criterion->update()
                 .residual_norm(res_norm.get())
                 .check(RelativeStoppingId, true, &stop_status, &one_changed));
         ASSERT_EQ(stop_status.get_data()[1].has_converged(), true);

From 167960436675891f689c1d9a7b958a85088ecc15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 17 Feb 2021 15:46:14 +0100
Subject: [PATCH 0330/1008] Small update to resolve linking issue

---
 core/stop/residual_norm.cpp                | 8 +-------
 include/ginkgo/core/stop/residual_norm.hpp | 7 ++++---
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/core/stop/residual_norm.cpp b/core/stop/residual_norm.cpp
index 45635b1384f..6d1e0de50f8 100644
--- a/core/stop/residual_norm.cpp
+++ b/core/stop/residual_norm.cpp
@@ -43,7 +43,6 @@ namespace residual_norm {
 
 
 GKO_REGISTER_OPERATION(residual_norm, residual_norm::residual_norm);
-GKO_REGISTER_OPERATION(fill_array, components::fill_array);
 
 
 }  // namespace residual_norm
@@ -113,7 +112,7 @@ bool ImplicitResidualNorm<ValueType>::check_impl(
 }
 
 
-#define GKO_DECLARE_RESIDUAL_NORM(_type) class ResidualNorm<_type>
+#define GKO_DECLARE_RESIDUAL_NORM(_type) class ResidualNormBase<_type>
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_RESIDUAL_NORM);
 
 
@@ -122,10 +121,5 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_RESIDUAL_NORM);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IMPLICIT_RESIDUAL_NORM);
 
 
-#define GKO_DECLARE_ABSOLUTE_RESIDUAL_NORM(_type) \
-    class AbsoluteResidualNorm<_type>
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_ABSOLUTE_RESIDUAL_NORM);
-
-
 }  // namespace stop
 }  // namespace gko
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index a592807ac7c..b38d8a80090 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -86,7 +86,7 @@ class ResidualNormBase
     using Vector = matrix::Dense<ValueType>;
     bool check_impl(uint8 stoppingId, bool setFinalized,
                     Array<stopping_status> *stop_status, bool *one_changed,
-                    const Criterion::Updater &) override;
+                    const Criterion::Updater &updater) override;
 
     explicit ResidualNormBase(std::shared_ptr<const gko::Executor> exec)
         : EnablePolymorphicObject<ResidualNormBase, Criterion>(exec),
@@ -212,7 +212,8 @@ class ResidualNorm : public ResidualNormBase<ValueType> {
         : ResidualNormBase<ValueType>(
               factory->get_executor(), args,
               factory->get_parameters().reduction_factor,
-              factory->get_parameters().baseline)
+              factory->get_parameters().baseline),
+          parameters_{factory->get_parameters()}
     {}
 };
 
@@ -264,7 +265,7 @@ class ImplicitResidualNorm : public ResidualNormBase<ValueType> {
     // residual here
     bool check_impl(uint8 stoppingId, bool setFinalized,
                     Array<stopping_status> *stop_status, bool *one_changed,
-                    const Criterion::Updater &) override;
+                    const Criterion::Updater &updater) override;
 
     explicit ImplicitResidualNorm(std::shared_ptr<const gko::Executor> exec)
         : ResidualNormBase<ValueType>(exec)

From eb2ac8d3805d149935ad5ab7fa0d658ac51b73f4 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 17 Feb 2021 16:32:39 +0100
Subject: [PATCH 0331/1008] Add strided fill kernels for dense.

---
 common/matrix/dense_kernels.hpp.inc      | 16 +++++++++++++
 core/device_hooks/common_kernels.inc.cpp |  5 ++++
 core/matrix/dense.cpp                    | 12 ++++++++++
 core/matrix/dense_kernels.hpp            |  6 +++++
 cuda/matrix/dense_kernels.cu             | 16 +++++++++++++
 cuda/test/matrix/dense_kernels.cpp       | 30 ++++++++++++++++++++++++
 cuda/test/utils.hpp                      |  4 ++++
 dpcpp/matrix/dense_kernels.dp.cpp        |  7 ++++++
 hip/matrix/dense_kernels.hip.cpp         | 17 ++++++++++++++
 hip/test/matrix/dense_kernels.hip.cpp    | 30 ++++++++++++++++++++++++
 include/ginkgo/core/matrix/dense.hpp     |  3 ++-
 omp/matrix/dense_kernels.cpp             | 15 ++++++++++++
 omp/test/matrix/dense_kernels.cpp        | 30 ++++++++++++++++++++++++
 reference/matrix/dense_kernels.cpp       | 14 +++++++++++
 reference/test/matrix/dense_kernels.cpp  | 30 ++++++++++++++++++++++++
 15 files changed, 234 insertions(+), 1 deletion(-)

diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index 8722726da1b..e1aaa299657 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -33,6 +33,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace kernel {
 
 
+template <size_type block_size, typename ValueType>
+__global__ __launch_bounds__(block_size) void strided_fill(
+    size_type num_rows, size_type num_cols, size_type stride,
+    ValueType *__restrict__ mat, ValueType value)
+{
+    constexpr auto warps_per_block = block_size / config::warp_size;
+    const auto global_id =
+        thread::get_thread_id<config::warp_size, warps_per_block>();
+    const auto row_id = global_id / num_cols;
+    const auto col_id = global_id % num_cols;
+    if (row_id < num_rows) {
+        mat[row_id * stride + col_id] = value;
+    }
+}
+
+
 template <size_type block_size, typename ValueType>
 __global__ __launch_bounds__(block_size) void scale(
     size_type num_rows, size_type num_cols, size_type num_alpha_cols,
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 964dfc305cb..c11562c8a33 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -126,6 +126,11 @@ GKO_DECLARE_DENSE_APPLY_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
+template <typename ValueType>
+GKO_DECLARE_DENSE_FILL_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_FILL_KERNEL);
+
 template <typename ValueType>
 GKO_DECLARE_DENSE_SCALE_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 9f69997b48c..5bdd902712e 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -62,6 +62,7 @@ namespace dense {
 
 GKO_REGISTER_OPERATION(simple_apply, dense::simple_apply);
 GKO_REGISTER_OPERATION(apply, dense::apply);
+GKO_REGISTER_OPERATION(fill, dense::fill);
 GKO_REGISTER_OPERATION(scale, dense::scale);
 GKO_REGISTER_OPERATION(add_scaled, dense::add_scaled);
 GKO_REGISTER_OPERATION(add_scaled_diag, dense::add_scaled_diag);
@@ -252,6 +253,17 @@ void Dense<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
 }
 
 
+template <typename ValueType>
+void Dense<ValueType>::fill(const ValueType value)
+{
+    if (this->get_stride() == this->get_size()[1]) {
+        this->values_.fill(value);
+    } else {
+        this->get_executor()->run(dense::make_fill(this, value));
+    }
+}
+
+
 template <typename ValueType>
 void Dense<ValueType>::scale_impl(const LinOp *alpha)
 {
diff --git a/core/matrix/dense_kernels.hpp b/core/matrix/dense_kernels.hpp
index cad52a11686..df3abaef209 100644
--- a/core/matrix/dense_kernels.hpp
+++ b/core/matrix/dense_kernels.hpp
@@ -57,6 +57,10 @@ namespace kernels {
                const matrix::Dense<_type> *a, const matrix::Dense<_type> *b, \
                const matrix::Dense<_type> *beta, matrix::Dense<_type> *c)
 
+#define GKO_DECLARE_DENSE_FILL_KERNEL(_type)               \
+    void fill(std::shared_ptr<const DefaultExecutor> exec, \
+              matrix::Dense<_type> *mat, _type value)
+
 #define GKO_DECLARE_DENSE_SCALE_KERNEL(_type)               \
     void scale(std::shared_ptr<const DefaultExecutor> exec, \
                const matrix::Dense<_type> *alpha, matrix::Dense<_type> *x)
@@ -216,6 +220,8 @@ namespace kernels {
     template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_APPLY_KERNEL(ValueType);                              \
     template <typename ValueType>                                           \
+    GKO_DECLARE_DENSE_FILL_KERNEL(ValueType);                               \
+    template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_SCALE_KERNEL(ValueType);                              \
     template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_ADD_SCALED_KERNEL(ValueType);                         \
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index 0864198fec8..6fa4ba78c17 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -117,6 +117,22 @@ void apply(std::shared_ptr<const CudaExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
+template <typename ValueType>
+void fill(std::shared_ptr<const DefaultExecutor> exec,
+          matrix::Dense<ValueType> *mat, ValueType value)
+{
+    constexpr auto block_size = default_block_size;
+    const dim3 grid_dim =
+        ceildiv(mat->get_size()[0] * mat->get_size()[1], block_size);
+    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
+    kernel::strided_fill<block_size><<<grid_dim, block_dim>>>(
+        mat->get_size()[0], mat->get_size()[1], mat->get_stride(),
+        as_cuda_type(mat->get_values()), as_cuda_type(value));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_FILL_KERNEL);
+
+
 template <typename ValueType>
 void scale(std::shared_ptr<const CudaExecutor> exec,
            const matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *x)
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index f64a4cc9c39..4b906b2e4f8 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -179,6 +179,36 @@ class Dense : public ::testing::Test {
 };
 
 
+TEST_F(Dense, CudaFillIsEquivalentToRef)
+{
+    set_up_vector_data(3);
+    auto result = Mtx::create(ref);
+
+    x->fill(42);
+    dx->fill(42);
+    result->copy_from(dx.get());
+
+    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
+}
+
+
+TEST_F(Dense, CudaStridedFillIsEquivalentToRef)
+{
+    using T = double;
+    auto x = gko::initialize<gko::matrix::Dense<T>>(
+        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, ref);
+    auto dx = gko::initialize<gko::matrix::Dense<T>>(
+        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, cuda);
+    auto result = Mtx::create(ref);
+
+    x->fill(42);
+    dx->fill(42);
+    result->copy_from(dx.get());
+
+    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
+}
+
+
 TEST_F(Dense, SingleVectorCudaScaleIsEquivalentToRef)
 {
     set_up_vector_data(1);
diff --git a/cuda/test/utils.hpp b/cuda/test/utils.hpp
index 4b6e0e0a667..54e60f53499 100644
--- a/cuda/test/utils.hpp
+++ b/cuda/test/utils.hpp
@@ -51,4 +51,8 @@ auto no_reset_exec =
 }  // namespace
 
 
+template <typename T>
+using I = std::initializer_list<T>;
+
+
 #endif  // GKO_CUDA_TEST_UTILS_HPP_
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 251ffbf18d6..88c86b6849b 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -84,6 +84,13 @@ void apply(std::shared_ptr<const DpcppExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
+template <typename ValueType>
+void fill(std::shared_ptr<const DefaultExecutor> exec,
+          matrix::Dense<ValueType> *mat, ValueType value) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_FILL_KERNEL);
+
+
 template <typename ValueType>
 void scale(std::shared_ptr<const DpcppExecutor> exec,
            const matrix::Dense<ValueType> *alpha,
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index c254e9a31a4..532e7259ffc 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -120,6 +120,23 @@ void apply(std::shared_ptr<const HipExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
+template <typename ValueType>
+void fill(std::shared_ptr<const DefaultExecutor> exec,
+          matrix::Dense<ValueType> *mat, ValueType value)
+{
+    constexpr auto block_size = default_block_size;
+    const dim3 grid_dim =
+        ceildiv(mat->get_size()[0] * mat->get_size()[1], block_size);
+    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
+    hipLaunchKernelGGL(
+        HIP_KERNEL_NAME(kernel::strided_fill<block_size>), dim3(grid_dim),
+        dim3(block_dim), 0, 0, mat->get_size()[0], mat->get_size()[1],
+        mat->get_stride(), as_hip_type(mat->get_values()), as_hip_type(value));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_FILL_KERNEL);
+
+
 template <typename ValueType>
 void scale(std::shared_ptr<const HipExecutor> exec,
            const matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *x)
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index d68c2041b20..c311571415f 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -174,6 +174,36 @@ class Dense : public ::testing::Test {
 };
 
 
+TEST_F(Dense, HipFillIsEquivalentToRef)
+{
+    set_up_vector_data(3);
+    auto result = Mtx::create(ref);
+
+    x->fill(42);
+    dx->fill(42);
+    result->copy_from(dx.get());
+
+    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
+}
+
+
+TEST_F(Dense, HipStridedFillIsEquivalentToRef)
+{
+    using T = double;
+    auto x = gko::initialize<gko::matrix::Dense<T>>(
+        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, ref);
+    auto dx = gko::initialize<gko::matrix::Dense<T>>(
+        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, hip);
+    auto result = Mtx::create(ref);
+
+    x->fill(42);
+    dx->fill(42);
+    result->copy_from(dx.get());
+
+    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
+}
+
+
 TEST_F(Dense, SingleVectorHipScaleIsEquivalentToRef)
 {
     set_up_vector_data(1);
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 2a5cd4a1851..336bb401574 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/base/mtx_io.hpp>
@@ -249,7 +250,7 @@ class Dense
      *
      * @param value  the value to be filled
      */
-    void fill(const ValueType value) { this->values_.fill(value); }
+    void fill(const ValueType value);
 
     /**
      * Create a Dense matrix consisting of the given rows from this matrix.
diff --git a/omp/matrix/dense_kernels.cpp b/omp/matrix/dense_kernels.cpp
index 6ca39cccd22..3830b96615a 100644
--- a/omp/matrix/dense_kernels.cpp
+++ b/omp/matrix/dense_kernels.cpp
@@ -127,6 +127,21 @@ void apply(std::shared_ptr<const OmpExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
+template <typename ValueType>
+void fill(std::shared_ptr<const DefaultExecutor> exec,
+          matrix::Dense<ValueType> *mat, ValueType value)
+{
+#pragma omp parallel for
+    for (size_type row = 0; row < mat->get_size()[0]; ++row) {
+        for (size_type col = 0; col < mat->get_size()[1]; ++col) {
+            mat->at(row, col) = value;
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_FILL_KERNEL);
+
+
 template <typename ValueType>
 void scale(std::shared_ptr<const OmpExecutor> exec,
            const matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *x)
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index bb860e7c820..e6a1ad38e02 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -191,6 +191,36 @@ class Dense : public ::testing::Test {
 };
 
 
+TEST_F(Dense, OmpFillIsEquivalentToRef)
+{
+    set_up_vector_data(3);
+    auto result = Mtx::create(ref);
+
+    x->fill(42);
+    dx->fill(42);
+    result->copy_from(dx.get());
+
+    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
+}
+
+
+TEST_F(Dense, OmpStridedFillIsEquivalentToRef)
+{
+    using T = double;
+    auto x = gko::initialize<gko::matrix::Dense<T>>(
+        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, ref);
+    auto dx = gko::initialize<gko::matrix::Dense<T>>(
+        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, omp);
+    auto result = Mtx::create(ref);
+
+    x->fill(42);
+    dx->fill(42);
+    result->copy_from(dx.get());
+
+    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
+}
+
+
 TEST_F(Dense, SingleVectorOmpScaleIsEquivalentToRef)
 {
     set_up_vector_data(1);
diff --git a/reference/matrix/dense_kernels.cpp b/reference/matrix/dense_kernels.cpp
index df0b8595afb..5e47b0376fa 100644
--- a/reference/matrix/dense_kernels.cpp
+++ b/reference/matrix/dense_kernels.cpp
@@ -116,6 +116,20 @@ void apply(std::shared_ptr<const ReferenceExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
+template <typename ValueType>
+void fill(std::shared_ptr<const DefaultExecutor> exec,
+          matrix::Dense<ValueType> *mat, ValueType value)
+{
+    for (size_type row = 0; row < mat->get_size()[0]; ++row) {
+        for (size_type col = 0; col < mat->get_size()[1]; ++col) {
+            mat->at(row, col) = value;
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_FILL_KERNEL);
+
+
 template <typename ValueType>
 void scale(std::shared_ptr<const ReferenceExecutor> exec,
            const matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *x)
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 9984fb7da74..ebe3e132045 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -124,6 +124,36 @@ TYPED_TEST(Dense, CanBeFilledWithValue)
 }
 
 
+TYPED_TEST(Dense, CanBeFilledWithValueForStridedMatrices)
+{
+    using value_type = typename TestFixture::value_type;
+    using T = value_type;
+    auto m = gko::initialize<gko::matrix::Dense<TypeParam>>(
+        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, this->exec);
+    auto in_stride = m->get_values()[3];
+    ASSERT_EQ(m->get_size(), gko::dim<2>(3, 2));
+    ASSERT_EQ(m->get_num_stored_elements(), 12);
+    EXPECT_EQ(m->at(0), value_type{1.0});
+    EXPECT_EQ(m->at(1), value_type{2.0});
+    EXPECT_EQ(m->at(2), value_type{3.0});
+    ASSERT_EQ(m->at(3), value_type{4.0});
+    EXPECT_EQ(m->at(4), value_type{5.0});
+    EXPECT_EQ(m->at(5), value_type{6.0});
+
+    m->fill(value_type{42});
+
+    ASSERT_EQ(m->get_size(), gko::dim<2>(3, 2));
+    EXPECT_EQ(m->get_num_stored_elements(), 12);
+    EXPECT_EQ(m->at(0), value_type{42.0});
+    EXPECT_EQ(m->at(1), value_type{42.0});
+    EXPECT_EQ(m->at(2), value_type{42.0});
+    ASSERT_EQ(m->at(3), value_type{42.0});
+    EXPECT_EQ(m->at(4), value_type{42.0});
+    EXPECT_EQ(m->at(5), value_type{42.0});
+    EXPECT_EQ(m->get_values()[3], in_stride);
+}
+
+
 TYPED_TEST(Dense, AppliesToDense)
 {
     using T = typename TestFixture::value_type;

From f8163803347bdf45a9b02ff1cd65245d5b7d0397 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 18 Feb 2021 13:22:02 +0100
Subject: [PATCH 0332/1008] Review udpate.

+ Simplify fill kernel.
+ Add implicit res to logger.
+ Some doc fixes.

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 common/matrix/dense_kernels.hpp.inc        |  8 ++----
 core/base/array.cpp                        |  3 +-
 core/matrix/dense.cpp                      |  6 +---
 core/stop/residual_norm.cpp                | 10 +++----
 cuda/components/fill_array.cu              |  2 +-
 cuda/matrix/dense_kernels.cu               |  8 ++----
 cuda/test/utils.hpp                        |  4 ---
 dpcpp/components/fill_array.dp.cpp         |  2 +-
 hip/components/fill_array.hip.cpp          |  2 +-
 hip/matrix/dense_kernels.hip.cpp           | 11 ++++----
 include/ginkgo/core/base/types.hpp         | 18 +++++-------
 include/ginkgo/core/log/logger.hpp         | 10 ++++++-
 include/ginkgo/core/stop/criterion.hpp     | 32 +++++++++++-----------
 include/ginkgo/core/stop/residual_norm.hpp |  6 ++--
 omp/components/fill_array.cpp              |  2 +-
 reference/components/fill_array.cpp        |  2 +-
 16 files changed, 58 insertions(+), 68 deletions(-)

diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index e1aaa299657..5bafe38b4d5 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -33,14 +33,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace kernel {
 
 
-template <size_type block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void strided_fill(
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void strided_fill(
     size_type num_rows, size_type num_cols, size_type stride,
     ValueType *__restrict__ mat, ValueType value)
 {
-    constexpr auto warps_per_block = block_size / config::warp_size;
-    const auto global_id =
-        thread::get_thread_id<config::warp_size, warps_per_block>();
+    const auto global_id = thread::get_thread_id_flat();
     const auto row_id = global_id / num_cols;
     const auto col_id = global_id % num_cols;
     if (row_id < num_rows) {
diff --git a/core/base/array.cpp b/core/base/array.cpp
index a36c8655728..0f31b3d9ba7 100644
--- a/core/base/array.cpp
+++ b/core/base/array.cpp
@@ -83,7 +83,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_ARRAY_CONVERSION);
 template <typename ValueType>
 void Array<ValueType>::fill(const ValueType value)
 {
-    GKO_ASSERT(this->get_num_elems() > 0);
     this->get_executor()->run(
         array::make_fill_array(this->get_data(), this->get_num_elems(), value));
 }
@@ -91,7 +90,7 @@ void Array<ValueType>::fill(const ValueType value)
 
 #define GKO_DECLARE_ARRAY_FILL(_type) void Array<_type>::fill(const _type value)
 
-GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_ARRAY_FILL);
+GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_ARRAY_FILL);
 
 
 }  // namespace gko
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 5bdd902712e..3a24dedddc5 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -256,11 +256,7 @@ void Dense<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
 template <typename ValueType>
 void Dense<ValueType>::fill(const ValueType value)
 {
-    if (this->get_stride() == this->get_size()[1]) {
-        this->values_.fill(value);
-    } else {
-        this->get_executor()->run(dense::make_fill(this, value));
-    }
+    this->get_executor()->run(dense::make_fill(this, value));
 }
 
 
diff --git a/core/stop/residual_norm.cpp b/core/stop/residual_norm.cpp
index 6d1e0de50f8..c4c37162408 100644
--- a/core/stop/residual_norm.cpp
+++ b/core/stop/residual_norm.cpp
@@ -60,7 +60,7 @@ GKO_REGISTER_OPERATION(implicit_residual_norm,
 
 template <typename ValueType>
 bool ResidualNormBase<ValueType>::check_impl(
-    uint8 stoppingId, bool setFinalized, Array<stopping_status> *stop_status,
+    uint8 stopping_id, bool set_finalized, Array<stopping_status> *stop_status,
     bool *one_changed, const Criterion::Updater &updater)
 {
     const NormVector *dense_tau;
@@ -81,8 +81,8 @@ bool ResidualNormBase<ValueType>::check_impl(
     bool all_converged = true;
 
     this->get_executor()->run(residual_norm::make_residual_norm(
-        dense_tau, starting_tau_.get(), reduction_factor_, stoppingId,
-        setFinalized, stop_status, &device_storage_, &all_converged,
+        dense_tau, starting_tau_.get(), reduction_factor_, stopping_id,
+        set_finalized, stop_status, &device_storage_, &all_converged,
         one_changed));
 
     return all_converged;
@@ -91,7 +91,7 @@ bool ResidualNormBase<ValueType>::check_impl(
 
 template <typename ValueType>
 bool ImplicitResidualNorm<ValueType>::check_impl(
-    uint8 stoppingId, bool setFinalized, Array<stopping_status> *stop_status,
+    uint8 stopping_id, bool set_finalized, Array<stopping_status> *stop_status,
     bool *one_changed, const Criterion::Updater &updater)
 {
     const Vector *dense_tau;
@@ -105,7 +105,7 @@ bool ImplicitResidualNorm<ValueType>::check_impl(
     this->get_executor()->run(
         implicit_residual_norm::make_implicit_residual_norm(
             dense_tau, this->starting_tau_.get(), this->reduction_factor_,
-            stoppingId, setFinalized, stop_status, &this->device_storage_,
+            stopping_id, set_finalized, stop_status, &this->device_storage_,
             &all_converged, one_changed));
 
     return all_converged;
diff --git a/cuda/components/fill_array.cu b/cuda/components/fill_array.cu
index c12435b5773..fba7a3104a2 100644
--- a/cuda/components/fill_array.cu
+++ b/cuda/components/fill_array.cu
@@ -59,7 +59,7 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
                                                         as_cuda_type(val));
 }
 
-GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
 }  // namespace components
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index 6fa4ba78c17..915c38b053a 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -121,11 +121,9 @@ template <typename ValueType>
 void fill(std::shared_ptr<const DefaultExecutor> exec,
           matrix::Dense<ValueType> *mat, ValueType value)
 {
-    constexpr auto block_size = default_block_size;
-    const dim3 grid_dim =
-        ceildiv(mat->get_size()[0] * mat->get_size()[1], block_size);
-    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
-    kernel::strided_fill<block_size><<<grid_dim, block_dim>>>(
+    const auto num_blocks =
+        ceildiv(mat->get_size()[0] * mat->get_size()[1], default_block_size);
+    kernel::strided_fill<<<num_blocks, default_block_size>>>(
         mat->get_size()[0], mat->get_size()[1], mat->get_stride(),
         as_cuda_type(mat->get_values()), as_cuda_type(value));
 }
diff --git a/cuda/test/utils.hpp b/cuda/test/utils.hpp
index 54e60f53499..4b6e0e0a667 100644
--- a/cuda/test/utils.hpp
+++ b/cuda/test/utils.hpp
@@ -51,8 +51,4 @@ auto no_reset_exec =
 }  // namespace
 
 
-template <typename T>
-using I = std::initializer_list<T>;
-
-
 #endif  // GKO_CUDA_TEST_UTILS_HPP_
diff --git a/dpcpp/components/fill_array.dp.cpp b/dpcpp/components/fill_array.dp.cpp
index 2dfde3591b7..f7e2307f263 100644
--- a/dpcpp/components/fill_array.dp.cpp
+++ b/dpcpp/components/fill_array.dp.cpp
@@ -54,7 +54,7 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
     });
 }
 
-GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
 }  // namespace components
diff --git a/hip/components/fill_array.hip.cpp b/hip/components/fill_array.hip.cpp
index 80e6bbbee1a..8df17d89a79 100644
--- a/hip/components/fill_array.hip.cpp
+++ b/hip/components/fill_array.hip.cpp
@@ -64,7 +64,7 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
                        0, n, as_hip_type(array), as_hip_type(val));
 }
 
-GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
 }  // namespace components
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index 532e7259ffc..628772a1c12 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -125,13 +125,12 @@ void fill(std::shared_ptr<const DefaultExecutor> exec,
           matrix::Dense<ValueType> *mat, ValueType value)
 {
     constexpr auto block_size = default_block_size;
-    const dim3 grid_dim =
+    const auto num_blocks =
         ceildiv(mat->get_size()[0] * mat->get_size()[1], block_size);
-    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
-    hipLaunchKernelGGL(
-        HIP_KERNEL_NAME(kernel::strided_fill<block_size>), dim3(grid_dim),
-        dim3(block_dim), 0, 0, mat->get_size()[0], mat->get_size()[1],
-        mat->get_stride(), as_hip_type(mat->get_values()), as_hip_type(value));
+    hipLaunchKernelGGL(kernel::strided_fill, num_blocks, block_size, 0, 0,
+                       mat->get_size()[0], mat->get_size()[1],
+                       mat->get_stride(), as_hip_type(mat->get_values()),
+                       as_hip_type(value));
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_FILL_KERNEL);
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index e7fd4b46854..e1d1d86cfe5 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -492,21 +492,17 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
 
 
 /**
- * Instantiates a template for each POD type
+ * Instantiates a template for each normal type
  *
  * @param _macro  A macro which expands the template instantiation
  *                (not including the leading `template` specifier).
- *                Should take two arguments `src` and `dst`, which
- *                are replaced by the source and destination value type.
+ *                Should take one argument, which is replaced by the
+ *                value type.
  */
-#define GKO_INSTANTIATE_FOR_EACH_POD_TYPE(_macro) \
-    template _macro(float);                       \
-    template _macro(double);                      \
-    template _macro(std::complex<float>);         \
-    template _macro(std::complex<double>);        \
-    template _macro(size_type);                   \
-    template _macro(int32);                       \
-    template _macro(int64)
+#define GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(_macro) \
+    GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(_macro);       \
+    GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(_macro);       \
+    template _macro(size_type)
 
 
 }  // namespace gko
diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp
index b276fbcfc9c..114381aaab9 100644
--- a/include/ginkgo/core/log/logger.hpp
+++ b/include/ginkgo/core/log/logger.hpp
@@ -402,8 +402,16 @@ public:                                                              \
                               const size_type &it, const LinOp *r,
                               const LinOp *x = nullptr,
                               const LinOp *tau = nullptr)
+protected:
+    virtual void on_iteration_complete(const LinOp *solver, const size_type &it,
+                                       const LinOp *r, const LinOp *x,
+                                       const LinOp *tau,
+                                       const LinOp *implicit_tau_sq) const
+    {
+        this->on_iteration_complete(solver, it, r, x, tau);
+    }
 
-
+public:
 #undef GKO_LOGGER_REGISTER_EVENT
 
     /**
diff --git a/include/ginkgo/core/stop/criterion.hpp b/include/ginkgo/core/stop/criterion.hpp
index 55b3c3bebbd..20f1ce8d1f5 100644
--- a/include/ginkgo/core/stop/criterion.hpp
+++ b/include/ginkgo/core/stop/criterion.hpp
@@ -96,10 +96,10 @@ class Criterion : public EnableAbstractPolymorphicObject<Criterion> {
          * Calls the parent Criterion object's check method
          * @copydoc Criterion::check(uint8, bool, Array<stopping_status>, bool)
          */
-        bool check(uint8 stoppingId, bool setFinalized,
+        bool check(uint8 stopping_id, bool set_finalized,
                    Array<stopping_status> *stop_status, bool *one_changed) const
         {
-            auto converged = parent_->check(stoppingId, setFinalized,
+            auto converged = parent_->check(stopping_id, set_finalized,
                                             stop_status, one_changed, *this);
             return converged;
         }
@@ -141,8 +141,8 @@ class Criterion : public EnableAbstractPolymorphicObject<Criterion> {
      * This checks whether convergence was reached for a certain criterion.
      * The actual implantation of the criterion goes here.
      *
-     * @param stoppingId  id of the stopping criterion
-     * @param setFinalized  Controls if the current version should count as
+     * @param stopping_id  id of the stopping criterion
+     * @param set_finalized  Controls if the current version should count as
      *                      finalized or not
      * @param stop_status  status of the stopping criterion
      * @param one_changed  indicates if one vector's status changed
@@ -150,20 +150,20 @@ class Criterion : public EnableAbstractPolymorphicObject<Criterion> {
      *
      * @returns whether convergence was completely reached
      */
-    bool check(uint8 stoppingId, bool setFinalized,
+    bool check(uint8 stopping_id, bool set_finalized,
                Array<stopping_status> *stop_status, bool *one_changed,
                const Updater &updater)
     {
         this->template log<log::Logger::criterion_check_started>(
             this, updater.num_iterations_, updater.residual_,
-            updater.residual_norm_, updater.solution_, stoppingId,
-            setFinalized);
+            updater.residual_norm_, updater.solution_, stopping_id,
+            set_finalized);
         auto all_converged = this->check_impl(
-            stoppingId, setFinalized, stop_status, one_changed, updater);
+            stopping_id, set_finalized, stop_status, one_changed, updater);
         this->template log<log::Logger::criterion_check_completed>(
             this, updater.num_iterations_, updater.residual_,
-            updater.residual_norm_, updater.solution_, stoppingId, setFinalized,
-            stop_status, *one_changed, all_converged);
+            updater.residual_norm_, updater.solution_, stopping_id,
+            set_finalized, stop_status, *one_changed, all_converged);
         return all_converged;
     }
 
@@ -175,8 +175,8 @@ class Criterion : public EnableAbstractPolymorphicObject<Criterion> {
      * This checks whether convergence was reached for a certain criterion.
      * The actual implantation of the criterion goes here.
      *
-     * @param stoppingId  id of the stopping criterion
-     * @param setFinalized  Controls if the current version should count as
+     * @param stopping_id  id of the stopping criterion
+     * @param set_finalized  Controls if the current version should count as
      *                      finalized or not
      * @param stop_status  status of the stopping criterion
      * @param one_changed  indicates if one vector's status changed
@@ -184,7 +184,7 @@ class Criterion : public EnableAbstractPolymorphicObject<Criterion> {
      *
      * @returns whether convergence was completely reached
      */
-    virtual bool check_impl(uint8 stoppingId, bool setFinalized,
+    virtual bool check_impl(uint8 stopping_id, bool set_finalized,
                             Array<stopping_status> *stop_status,
                             bool *one_changed, const Updater &updater) = 0;
 
@@ -193,12 +193,12 @@ class Criterion : public EnableAbstractPolymorphicObject<Criterion> {
      * stopping_status to converged. This is used in stopping criteria such as
      * Time or Iteration.
      *
-     * @param stoppingId  id of the stopping criterion
-     * @param setFinalized  Controls if the current version should count as
+     * @param stopping_id  id of the stopping criterion
+     * @param set_finalized  Controls if the current version should count as
      *                      finalized or not
      * @param stop_status  status of the stopping criterion
      */
-    void set_all_statuses(uint8 stoppingId, bool setFinalized,
+    void set_all_statuses(uint8 stopping_id, bool set_finalized,
                           Array<stopping_status> *stop_status);
 
     explicit Criterion(std::shared_ptr<const gko::Executor> exec)
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index b38d8a80090..e2456403643 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -52,13 +52,13 @@ namespace stop {
  * The mode for the residual norm criterion.
  *
  * - absolute:        Check for tolerance against residual norm.
- *                    || r || < tau
+ *                    $ || r || < \tau $
  *
  * - initial_resnorm: Check for tolerance relative to the initial residual norm.
- *                    || r ||/|| r_0|| < tau
+ *                    $ \frac{|| r ||}{|| r_0||} < \tau $
  *
  * - rhs_resnorm:     Check for tolerance relative to the rhs norm.
- *                    || r ||/|| b || < tau
+ *                    $ \frac{|| r ||}{|| b ||} < \tau $
  *
  * @ingroup stop
  */
diff --git a/omp/components/fill_array.cpp b/omp/components/fill_array.cpp
index b1c00b3a76f..7f32a4f14c8 100644
--- a/omp/components/fill_array.cpp
+++ b/omp/components/fill_array.cpp
@@ -49,7 +49,7 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
 }  // namespace components
diff --git a/reference/components/fill_array.cpp b/reference/components/fill_array.cpp
index 949b4ee92d4..099215d09f4 100644
--- a/reference/components/fill_array.cpp
+++ b/reference/components/fill_array.cpp
@@ -46,7 +46,7 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
     std::fill_n(array, n, val);
 }
 
-GKO_INSTANTIATE_FOR_EACH_POD_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
 }  // namespace components

From 0053886ace1fc6fb7832c7b2f18daa0035804e9e Mon Sep 17 00:00:00 2001
From: "aliaga@uji.es" <aliaga@tintorrum.local>
Date: Mon, 27 Apr 2020 19:14:41 +0200
Subject: [PATCH 0333/1008] First commit for the GmresMixed class:   * Only
 core and reference executors.   * test files don't compile, due to t problem
 related the macros of gtest (TEST_F -> TYPED_TEST).   * MGS, MGS with
 reorthogonalization and CGS with reorthogonalization are considered.   *
 Norms are still created in the internal routines.

---
 core/CMakeLists.txt                           |   1 +
 core/device_hooks/common_kernels.inc.cpp      |  38 ++
 core/solver/gmres_mixed.cpp                   | 298 +++++++++
 core/solver/gmres_mixed_kernels.hpp           | 165 +++++
 core/test/solver/CMakeLists.txt               |   1 +
 core/test/solver/gmres_mixed.cpp              | 307 ++++++++++
 cuda/CMakeLists.txt                           |   1 +
 cuda/test/solver/CMakeLists.txt               |   1 +
 dpcpp/CMakeLists.txt                          |   1 +
 examples/CMakeLists.txt                       |   2 +
 examples/gmres-mixed-solver/CMakeLists.txt    |   9 +
 examples/gmres-mixed-solver/build.sh          |  37 ++
 examples/gmres-mixed-solver/data/A.mtx        | 114 ++++
 examples/gmres-mixed-solver/data/b.mtx        |  21 +
 examples/gmres-mixed-solver/data/x0.mtx       |  21 +
 examples/gmres-mixed-solver/doc/builds-on     |   1 +
 examples/gmres-mixed-solver/doc/intro.dox     |  19 +
 examples/gmres-mixed-solver/doc/kind          |   1 +
 examples/gmres-mixed-solver/doc/results.dox   |  33 +
 examples/gmres-mixed-solver/doc/short-intro   |   1 +
 examples/gmres-mixed-solver/doc/tooltip       |   1 +
 .../gmres-mixed-solver-jacobi.cpp             | 158 +++++
 .../gmres-mixed-solver/gmres-mixed-solver.cpp | 153 +++++
 examples/gmres-solver/CMakeLists.txt          |   6 +
 examples/gmres-solver/build.sh                |  37 ++
 examples/gmres-solver/data/A.mtx              | 114 ++++
 examples/gmres-solver/data/b.mtx              |  21 +
 examples/gmres-solver/data/x0.mtx             |  21 +
 examples/gmres-solver/doc/builds-on           |   1 +
 examples/gmres-solver/doc/intro.dox           |  19 +
 examples/gmres-solver/doc/kind                |   1 +
 examples/gmres-solver/doc/results.dox         |  33 +
 examples/gmres-solver/doc/short-intro         |   1 +
 examples/gmres-solver/doc/tooltip             |   1 +
 examples/gmres-solver/gmres-solver.cpp        | 152 +++++
 hip/CMakeLists.txt                            |   1 +
 hip/test/solver/CMakeLists.txt                |   1 +
 include/ginkgo/core/base/types.hpp            |  36 ++
 include/ginkgo/core/solver/gmres_mixed.hpp    | 188 ++++++
 include/ginkgo/ginkgo.hpp                     |   1 +
 omp/CMakeLists.txt                            |   1 +
 omp/test/solver/CMakeLists.txt                |   1 +
 reference/CMakeLists.txt                      |   1 +
 reference/solver/gmres_mixed_kernels.cpp      | 576 ++++++++++++++++++
 reference/test/solver/CMakeLists.txt          |   1 +
 reference/test/solver/gmres_mixed_kernels.cpp | 357 +++++++++++
 46 files changed, 2955 insertions(+)
 create mode 100644 core/solver/gmres_mixed.cpp
 create mode 100644 core/solver/gmres_mixed_kernels.hpp
 create mode 100644 core/test/solver/gmres_mixed.cpp
 create mode 100644 examples/gmres-mixed-solver/CMakeLists.txt
 create mode 100755 examples/gmres-mixed-solver/build.sh
 create mode 100644 examples/gmres-mixed-solver/data/A.mtx
 create mode 100644 examples/gmres-mixed-solver/data/b.mtx
 create mode 100644 examples/gmres-mixed-solver/data/x0.mtx
 create mode 100644 examples/gmres-mixed-solver/doc/builds-on
 create mode 100644 examples/gmres-mixed-solver/doc/intro.dox
 create mode 100644 examples/gmres-mixed-solver/doc/kind
 create mode 100644 examples/gmres-mixed-solver/doc/results.dox
 create mode 100644 examples/gmres-mixed-solver/doc/short-intro
 create mode 100644 examples/gmres-mixed-solver/doc/tooltip
 create mode 100644 examples/gmres-mixed-solver/gmres-mixed-solver-jacobi.cpp
 create mode 100644 examples/gmres-mixed-solver/gmres-mixed-solver.cpp
 create mode 100644 examples/gmres-solver/CMakeLists.txt
 create mode 100755 examples/gmres-solver/build.sh
 create mode 100644 examples/gmres-solver/data/A.mtx
 create mode 100644 examples/gmres-solver/data/b.mtx
 create mode 100644 examples/gmres-solver/data/x0.mtx
 create mode 100644 examples/gmres-solver/doc/builds-on
 create mode 100644 examples/gmres-solver/doc/intro.dox
 create mode 100644 examples/gmres-solver/doc/kind
 create mode 100644 examples/gmres-solver/doc/results.dox
 create mode 100644 examples/gmres-solver/doc/short-intro
 create mode 100644 examples/gmres-solver/doc/tooltip
 create mode 100644 examples/gmres-solver/gmres-solver.cpp
 create mode 100644 include/ginkgo/core/solver/gmres_mixed.hpp
 create mode 100644 reference/solver/gmres_mixed_kernels.cpp
 create mode 100644 reference/test/solver/gmres_mixed_kernels.cpp

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index f624c90c823..e66d92bbfca 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -40,6 +40,7 @@ target_sources(ginkgo
     solver/cgs.cpp
     solver/fcg.cpp
     solver/gmres.cpp
+    solver/gmres_mixed.cpp
     solver/idr.cpp
     solver/ir.cpp
     solver/lower_trs.cpp
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index c11562c8a33..95e0e346600 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -61,6 +61,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/cgs_kernels.hpp"
 #include "core/solver/fcg_kernels.hpp"
 #include "core/solver/gmres_kernels.hpp"
+#include "core/solver/gmres_mixed_kernels.hpp"
 #include "core/solver/idr_kernels.hpp"
 #include "core/solver/ir_kernels.hpp"
 #include "core/solver/lower_trs_kernels.hpp"
@@ -570,6 +571,43 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_STEP_2_KERNEL);
 }  // namespace gmres
 
 
+namespace gmres_mixed {
+
+
+template <typename ValueType>
+GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(ValueType, ValueTypeKrylovBases)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
+
+/*
+template <typename ValueType, typename ValueTypeKrylovBases, bool
+Reorthogonalization, bool MGS_CGS>
+GKO_DECLARE_GMRES_MIXED_BOOL_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases,
+                                           Reorthogonalization, MGS_CGS)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(GKO_DECLARE_GMRES_MIXED_BOOL_STEP_1_KERNEL);
+*/
+template <typename ValueType, typename ValueTypeKrylovBases>
+GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(ValueType, ValueTypeKrylovBases)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
+
+
+}  // namespace gmres_mixed
+
+
 namespace ir {
 
 
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
new file mode 100644
index 00000000000..44d61a53098
--- /dev/null
+++ b/core/solver/gmres_mixed.cpp
@@ -0,0 +1,298 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/solver/gmres_mixed.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/name_demangling.hpp>
+#include <ginkgo/core/base/utils.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/identity.hpp>
+
+
+#include "core/solver/gmres_mixed_kernels.hpp"
+
+
+namespace gko {
+namespace solver {
+
+
+namespace gmres_mixed {
+
+
+GKO_REGISTER_OPERATION(initialize_1, gmres_mixed::initialize_1);
+GKO_REGISTER_OPERATION(initialize_2, gmres_mixed::initialize_2);
+GKO_REGISTER_OPERATION(step_1, gmres_mixed::step_1);
+GKO_REGISTER_OPERATION(step_2, gmres_mixed::step_2);
+
+
+}  // namespace gmres_mixed
+
+/*
+template <typename ValueType, typename ValueTypeKrylovBases, bool
+Reorthogonalization, bool MGS_CGS> void
+GmresMixed<ValueType,ValueTypeKrylovBases,
+                Reorthogonalization,MGS_CGS>::apply_impl(const LinOp *b, LinOp
+*x) const
+*/
+template <typename ValueType, typename ValueTypeKrylovBases>
+void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
+                                                             LinOp *x) const
+{
+    GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix_);
+
+    using Vector = matrix::Dense<ValueType>;
+    using Krylov = matrix::Dense<ValueTypeKrylovBases>;
+
+    constexpr uint8 RelativeStoppingId{1};
+
+    auto exec = this->get_executor();
+
+    auto one_op = initialize<Vector>({one<ValueType>()}, exec);
+    auto neg_one_op = initialize<Vector>({-one<ValueType>()}, exec);
+
+    auto dense_b = as<const Vector>(b);
+    auto dense_x = as<Vector>(x);
+    auto residual = Vector::create_with_config_of(dense_b);
+    auto krylov_bases = Krylov::create(
+        exec, dim<2>{system_matrix_->get_size()[1],
+                     (krylov_dim_mixed_ + 1) * dense_b->get_size()[1]});
+    auto next_krylov_basis = Vector::create_with_config_of(dense_b);
+    std::shared_ptr<matrix::Dense<ValueType>> preconditioned_vector =
+        Vector::create_with_config_of(dense_b);
+    auto hessenberg = Vector::create(
+        exec, dim<2>{krylov_dim_mixed_ + 1,
+                     krylov_dim_mixed_ * dense_b->get_size()[1]});
+    auto buffer = Vector::create(
+        exec, dim<2>{krylov_dim_mixed_ + 1, dense_b->get_size()[1]});
+    auto givens_sin =
+        Vector::create(exec, dim<2>{krylov_dim_mixed_, dense_b->get_size()[1]});
+    auto givens_cos =
+        Vector::create(exec, dim<2>{krylov_dim_mixed_, dense_b->get_size()[1]});
+    auto residual_norm_collection = Vector::create(
+        exec, dim<2>{krylov_dim_mixed_ + 1, dense_b->get_size()[1]});
+    auto residual_norm =
+        Vector::create(exec, dim<2>{1, dense_b->get_size()[1]});
+    auto b_norm = Vector::create(exec, dim<2>{1, dense_b->get_size()[1]});
+    Array<size_type> final_iter_nums(this->get_executor(),
+                                     dense_b->get_size()[1]);
+    auto y =
+        Vector::create(exec, dim<2>{krylov_dim_mixed_, dense_b->get_size()[1]});
+
+    bool one_changed{};
+    Array<stopping_status> stop_status(this->get_executor(),
+                                       dense_b->get_size()[1]);
+
+    // Initialization
+    exec->run(gmres_mixed::make_initialize_1(
+        dense_b, b_norm.get(), residual.get(), givens_sin.get(),
+        givens_cos.get(), &stop_status, krylov_dim_mixed_));
+    // b_norm = norm(b)
+    // residual = dense_b
+    // givens_sin = givens_cos = 0
+    system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(),
+                          residual.get());
+    // residual = residual - Ax
+
+    exec->run(gmres_mixed::make_initialize_2(
+        residual.get(), residual_norm.get(), residual_norm_collection.get(),
+        krylov_bases.get(), next_krylov_basis.get(), &final_iter_nums,
+        krylov_dim_mixed_));
+    // residual_norm = norm(residual)
+    // residual_norm_collection = {residual_norm, 0, ..., 0}
+    // krylov_bases(:, 1) = residual / residual_norm
+    // next_krylov_basis = residual / residual_norm
+    // final_iter_nums = {0, ..., 0}
+
+    auto stop_criterion = stop_criterion_factory_->generate(
+        system_matrix_, std::shared_ptr<const LinOp>(b, [](const LinOp *) {}),
+        x, residual.get());
+
+    int total_iter = -1;
+    size_type restart_iter = 0;
+
+    auto before_preconditioner =
+        matrix::Dense<ValueType>::create_with_config_of(dense_x);
+    auto after_preconditioner =
+        matrix::Dense<ValueType>::create_with_config_of(dense_x);
+
+    while (true) {
+        ++total_iter;
+        this->template log<log::Logger::iteration_complete>(
+            this, total_iter, residual.get(), dense_x, residual_norm.get());
+        if (stop_criterion->update()
+                .num_iterations(total_iter)
+                .residual(residual.get())
+                .residual_norm(residual_norm.get())
+                .solution(dense_x)
+                .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
+            break;
+        }
+
+        if (restart_iter == krylov_dim_mixed_) {
+            // Restart
+            exec->run(gmres_mixed::make_step_2(
+                residual_norm_collection.get(), krylov_bases.get(),
+                hessenberg.get(), y.get(), before_preconditioner.get(),
+                &final_iter_nums));
+            // Solve upper triangular.
+            // y = hessenberg \ residual_norm_collection
+
+            get_preconditioner()->apply(before_preconditioner.get(),
+                                        after_preconditioner.get());
+            dense_x->add_scaled(one_op.get(), after_preconditioner.get());
+            // Solve x
+            // x = x + get_preconditioner() * krylov_bases * y
+            residual->copy_from(dense_b);
+            // residual = dense_b
+            system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(),
+                                  residual.get());
+            // residual = residual - Ax
+            exec->run(gmres_mixed::make_initialize_2(
+                residual.get(), residual_norm.get(),
+                residual_norm_collection.get(), krylov_bases.get(),
+                next_krylov_basis.get(), &final_iter_nums, krylov_dim_mixed_));
+            // residual_norm = norm(residual)
+            // residual_norm_collection = {residual_norm, 0, ..., 0}
+            // krylov_bases(:, 1) = residual / residual_norm
+            // next_krylov_basis = residual / residual_norm
+            // final_iter_nums = {0, ..., 0}
+            restart_iter = 0;
+        }
+
+        get_preconditioner()->apply(next_krylov_basis.get(),
+                                    preconditioned_vector.get());
+        // preconditioned_vector = get_preconditioner() * next_krylov_basis
+
+        // Do Arnoldi and givens rotation
+        auto hessenberg_iter = hessenberg->create_submatrix(
+            span{0, restart_iter + 2},
+            span{dense_b->get_size()[1] * restart_iter,
+                 dense_b->get_size()[1] * (restart_iter + 1)});
+        auto buffer_iter = buffer->create_submatrix(
+            span{0, restart_iter + 2}, span{0, dense_b->get_size()[1]});
+
+        // Start of arnoldi
+        system_matrix_->apply(preconditioned_vector.get(),
+                              next_krylov_basis.get());
+        // next_krylov_basis = A * preconditioned_vector
+
+        exec->run(gmres_mixed::make_step_1(
+            next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
+            residual_norm.get(), residual_norm_collection.get(),
+            krylov_bases.get(), hessenberg_iter.get(), buffer_iter.get(),
+            b_norm.get(), restart_iter, &final_iter_nums, &stop_status));
+        // for i in 0:restart_iter
+        //     hessenberg(restart_iter, i) = next_krylov_basis' *
+        //     krylov_bases(:, i) next_krylov_basis  -= hessenberg(restart_iter,
+        //     i) * krylov_bases(:, i)
+        // end
+        // hessenberg(restart_iter, restart_iter + 1) = norm(next_krylov_basis)
+        // next_krylov_basis /= hessenberg(restart_iter, restart_iter + 1)
+        // End of arnoldi
+        // Start apply givens rotation
+        // for j in 0:restart_iter
+        //     temp             =  cos(j)*hessenberg(j) +
+        //                         sin(j)*hessenberg(j+1)
+        //     hessenberg(j+1)  = -sin(j)*hessenberg(j) +
+        //                         cos(j)*hessenberg(j+1)
+        //     hessenberg(j)    =  temp;
+        // end
+        // Calculate sin and cos
+        // hessenberg(restart_iter)   =
+        // cos(restart_iter)*hessenberg(restart_iter) +
+        //                      sin(restart_iter)*hessenberg(restart_iter)
+        // hessenberg(restart_iter+1) = 0
+        // End apply givens rotation
+        // Calculate residual norm
+
+        restart_iter++;
+    }
+
+    // Solve x
+    auto krylov_bases_small = krylov_bases->create_submatrix(
+        span{0, system_matrix_->get_size()[0]},
+        span{0, dense_b->get_size()[1] * (restart_iter + 1)});
+    auto hessenberg_small = hessenberg->create_submatrix(
+        span{0, restart_iter},
+        span{0, dense_b->get_size()[1] * (restart_iter)});
+
+    exec->run(gmres_mixed::make_step_2(
+        residual_norm_collection.get(), krylov_bases_small.get(),
+        hessenberg_small.get(), y.get(), before_preconditioner.get(),
+        &final_iter_nums));
+    // Solve upper triangular.
+    // y = hessenberg \ residual_norm_collection
+
+    get_preconditioner()->apply(before_preconditioner.get(),
+                                after_preconditioner.get());
+    dense_x->add_scaled(one_op.get(), after_preconditioner.get());
+    // Solve x
+    // x = x + get_preconditioner() * krylov_bases * y
+}
+
+
+/*
+template <typename ValueType, typename ValueTypeKrylovBases, bool
+Reorthogonalization, bool MGS_CGS> void
+GmresMixed<ValueType,ValueTypeKrylovBases,
+                Reorthogonalization,MGS_CGS>::apply_impl(const LinOp *alpha,
+const LinOp *b, const LinOp *residual_norm_collection, LinOp *x) const
+*/
+template <typename ValueType, typename ValueTypeKrylovBases>
+void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(
+    const LinOp *alpha, const LinOp *b, const LinOp *residual_norm_collection,
+    LinOp *x) const
+{
+    auto dense_x = as<matrix::Dense<ValueType>>(x);
+
+    auto x_clone = dense_x->clone();
+    this->apply(b, x_clone.get());
+    dense_x->scale(residual_norm_collection);
+    dense_x->add_scaled(alpha, x_clone.get());
+}
+
+/*
+#define GKO_DECLARE_GMRES_MIXED_BOOL(_type1, _type2, _bool1, _bool2)
+      class GmresMixed<_type1, _type2, _bool1, _bool2>
+GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(GKO_DECLARE_GMRES_MIXED_BOOL);
+*/
+#define GKO_DECLARE_GMRES_MIXED(_type1, _type2) class GmresMixed<_type1, _type2>
+GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED);
+
+}  // namespace solver
+}  // namespace gko
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
new file mode 100644
index 00000000000..561db6bc214
--- /dev/null
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -0,0 +1,165 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_SOLVER_GMRES_MIXED_KERNELS_HPP_
+#define GKO_CORE_SOLVER_GMRES_MIXED_KERNELS_HPP_
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/stopping_status.hpp>
+
+namespace gko {
+namespace kernels {
+namespace gmres_mixed {
+
+
+#define GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL(_type)                     \
+    void initialize_1(                                                         \
+        std::shared_ptr<const DefaultExecutor> exec,                           \
+        const matrix::Dense<_type> *b, matrix::Dense<_type> *b_norm,           \
+        matrix::Dense<_type> *residual, matrix::Dense<_type> *givens_sin,      \
+        matrix::Dense<_type> *givens_cos, Array<stopping_status> *stop_status, \
+        size_type krylov_dim)
+
+
+#define GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(_type1, _type2)    \
+    void initialize_2(std::shared_ptr<const DefaultExecutor> exec,     \
+                      const matrix::Dense<_type1> *residual,           \
+                      matrix::Dense<_type1> *residual_norm,            \
+                      matrix::Dense<_type1> *residual_norm_collection, \
+                      matrix::Dense<_type2> *krylov_bases,             \
+                      matrix::Dense<_type1> *next_krylov_basis,        \
+                      Array<size_type> *final_iter_nums, size_type krylov_dim)
+
+
+// #define GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(_type1, _type2, _bool1, _bool2)
+#define GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(_type1, _type2)        \
+    void step_1(std::shared_ptr<const DefaultExecutor> exec,         \
+                matrix::Dense<_type1> *next_krylov_basis,            \
+                matrix::Dense<_type1> *givens_sin,                   \
+                matrix::Dense<_type1> *givens_cos,                   \
+                matrix::Dense<_type1> *residual_norm,                \
+                matrix::Dense<_type1> *residual_norm_collection,     \
+                matrix::Dense<_type2> *krylov_bases,                 \
+                matrix::Dense<_type1> *hessenberg_iter,              \
+                matrix::Dense<_type1> *buffer_iter,                  \
+                const matrix::Dense<_type1> *b_norm, size_type iter, \
+                Array<size_type> *final_iter_nums,                   \
+                const Array<stopping_status> *stop_status)
+
+
+#define GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(_type1, _type2)          \
+    void step_2(std::shared_ptr<const DefaultExecutor> exec,           \
+                const matrix::Dense<_type1> *residual_norm_collection, \
+                const matrix::Dense<_type2> *krylov_bases,             \
+                const matrix::Dense<_type1> *hessenberg,               \
+                matrix::Dense<_type1> *y,                              \
+                matrix::Dense<_type1> *before_preconditioner,          \
+                const Array<size_type> *final_iter_nums)
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES                                        \
+    template <typename ValueType>                                           \
+    GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL(ValueType);                 \
+    template <typename ValueType, typename ValueTypeKrylovBases>            \
+    GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(ValueType,                  \
+                                                ValueTypeKrylovBases);      \
+    template <typename ValueType, typename ValueTypeKrylovBases>            \
+    GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases); \
+    template <typename ValueType, typename ValueTypeKrylovBases>            \
+    GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(ValueType, ValueTypeKrylovBases)
+/*
+    template <typename ValueType, typename ValueTypeKrylovBases, \
+              bool Reorthogonalization, bool MGS_CGS>                     \
+    GKO_DECLARE_GMRES_MIXED_BOOL_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases,
+   \
+                                          Reorthogonalization, MGS_CGS);       \
+*/
+
+}  // namespace gmres_mixed
+
+
+namespace omp {
+namespace gmres_mixed {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace gmres_mixed
+}  // namespace omp
+
+
+namespace cuda {
+namespace gmres_mixed {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace gmres_mixed
+}  // namespace cuda
+
+
+namespace reference {
+namespace gmres_mixed {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace gmres_mixed
+}  // namespace reference
+
+
+namespace hip {
+namespace gmres_mixed {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace gmres_mixed
+}  // namespace hip
+
+
+namespace dpcpp {
+namespace gmres_mixed {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace gmres_mixed
+}  // namespace dpcpp
+
+
+#undef GKO_DECLARE_ALL_AS_TEMPLATES
+
+
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_CORE_SOLVER_GMRES_MIXED_KERNELS_HPP_
diff --git a/core/test/solver/CMakeLists.txt b/core/test/solver/CMakeLists.txt
index 7454c5a925a..dc8996a5f6a 100644
--- a/core/test/solver/CMakeLists.txt
+++ b/core/test/solver/CMakeLists.txt
@@ -4,6 +4,7 @@ ginkgo_create_test(cg)
 ginkgo_create_test(cgs)
 ginkgo_create_test(fcg)
 ginkgo_create_test(gmres)
+# ginkgo_create_test(gmres_mixed)
 ginkgo_create_test(idr)
 ginkgo_create_test(ir)
 ginkgo_create_test(lower_trs)
diff --git a/core/test/solver/gmres_mixed.cpp b/core/test/solver/gmres_mixed.cpp
new file mode 100644
index 00000000000..f82553bd58a
--- /dev/null
+++ b/core/test/solver/gmres_mixed.cpp
@@ -0,0 +1,307 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/solver/gmres_mixed.hpp>
+
+
+#include <typeinfo>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm_reduction.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+template <typename T1, typename T2>
+class GmresMixed : public ::testing::Test {
+protected:
+    using value_type = T1;
+    using krylov_type = T2;
+    using Mtx = gko::matrix::Dense<value_type>;
+    using Solver = gko::solver::GmresMixed<value_type, krylov_type>;
+    using Big_solver = gko::solver::GmresMixed<double>;
+
+    static constexpr gko::remove_complex<T1> reduction_factor =
+        gko::remove_complex<T1>(1e-6);
+
+    GmresMixed()
+        : exec(gko::ReferenceExecutor::create()),
+          mtx(gko::initialize<Mtx>(
+              {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)),
+          gmres_mixed_factory(
+              Solver::build()
+                  .with_criteria(
+                      gko::stop::Iteration::build().with_max_iters(3u).on(exec),
+                      gko::stop::ResidualNormReduction<value_type>::build()
+                          .with_reduction_factor(reduction_factor)
+                          .on(exec))
+                  .on(exec)),
+          solver(gmres_mixed_factory->generate(mtx)),
+          gmres_mixed_big_factory(
+              Big_solver::build()
+                  .with_criteria(
+                      gko::stop::Iteration::build().with_max_iters(128u).on(
+                          exec),
+                      gko::stop::ResidualNormReduction<value_type>::build()
+                          .with_reduction_factor(reduction_factor)
+                          .on(exec))
+                  .on(exec)),
+          big_solver(gmres_mixed_big_factory->generate(mtx))
+    {}
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<Mtx> mtx;
+    std::unique_ptr<typename Solver::Factory> gmres_mixed_factory;
+    std::unique_ptr<gko::LinOp> solver;
+    std::unique_ptr<Big_solver::Factory> gmres_mixed_big_factory;
+    std::unique_ptr<gko::LinOp> big_solver;
+
+    static void assert_same_matrices(const Mtx *m1, const Mtx *m2)
+    {
+        ASSERT_EQ(m1->get_size()[0], m2->get_size()[0]);
+        ASSERT_EQ(m1->get_size()[1], m2->get_size()[1]);
+        for (gko::size_type i = 0; i < m1->get_size()[0]; ++i) {
+            for (gko::size_type j = 0; j < m2->get_size()[1]; ++j) {
+                EXPECT_EQ(m1->at(i, j), m2->at(i, j));
+            }
+        }
+    }
+};
+
+template <typename T1>
+constexpr gko::remove_complex<T1>
+    GmresMixed<T1, T1>::reduction_factor;  // JIAE??
+
+TYPED_TEST_CASE(GmresMixed, gko::test::ValueTypes);
+
+
+TYPED_TEST(GmresMixed, GmresMixedFactoryKnowsItsExecutor)
+{
+    ASSERT_EQ(this->gmres_mixed_factory->get_executor(), this->exec);
+}
+
+
+TYPED_TEST(GmresMixed, GmresMixedFactoryCreatesCorrectSolver)
+{
+    using Solver = typename TestFixture::Solver;
+    ASSERT_EQ(this->solver->get_size(), gko::dim<2>(3, 3));
+    auto gmres_mixed_solver = static_cast<Solver *>(this->solver.get());
+    ASSERT_NE(gmres_mixed_solver->get_system_matrix(), nullptr);
+    ASSERT_EQ(gmres_mixed_solver->get_system_matrix(), this->mtx);
+}
+
+
+TYPED_TEST(GmresMixed, CanBeCopied)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    auto copy = this->gmres_mixed_factory->generate(Mtx::create(this->exec));
+
+    copy->copy_from(this->solver.get());
+
+    ASSERT_EQ(copy->get_size(), gko::dim<2>(3, 3));
+    auto copy_mtx = static_cast<Solver *>(copy.get())->get_system_matrix();
+    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
+                               this->mtx.get());
+}
+
+
+TYPED_TEST(GmresMixed, CanBeMoved)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    auto copy = this->gmres_mixed_factory->generate(Mtx::create(this->exec));
+
+    copy->copy_from(std::move(this->solver));
+
+    ASSERT_EQ(copy->get_size(), gko::dim<2>(3, 3));
+    auto copy_mtx = static_cast<Solver *>(copy.get())->get_system_matrix();
+    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
+                               this->mtx.get());
+}
+
+
+TYPED_TEST(GmresMixed, CanBeCloned)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    auto clone = this->solver->clone();
+
+    ASSERT_EQ(clone->get_size(), gko::dim<2>(3, 3));
+    auto clone_mtx = static_cast<Solver *>(clone.get())->get_system_matrix();
+    this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
+                               this->mtx.get());
+}
+
+
+TYPED_TEST(GmresMixed, CanBeCleared)
+{
+    using Solver = typename TestFixture::Solver;
+    this->solver->clear();
+
+    ASSERT_EQ(this->solver->get_size(), gko::dim<2>(0, 0));
+    auto solver_mtx =
+        static_cast<Solver *>(this->solver.get())->get_system_matrix();
+    ASSERT_EQ(solver_mtx, nullptr);
+}
+
+
+TYPED_TEST(GmresMixed, CanSetPreconditionerGenerator)
+{
+    using Solver = typename TestFixture::Solver;
+    using value_type = typename TestFixture::value_type;
+    auto gmres_mixed_factory =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec),
+                gko::stop::ResidualNormReduction<value_type>::build()
+                    .with_reduction_factor(TestFixture::reduction_factor)
+                    .on(this->exec))
+            .with_preconditioner(Solver::build().on(this->exec))
+            .on(this->exec);
+    auto solver = gmres_mixed_factory->generate(this->mtx);
+    auto precond = dynamic_cast<const gko::solver::GmresMixed<value_type> *>(
+        static_cast<gko::solver::GmresMixed<value_type> *>(solver.get())
+            ->get_preconditioner()
+            .get());
+
+    ASSERT_NE(precond, nullptr);
+    ASSERT_EQ(precond->get_size(), gko::dim<2>(3, 3));
+    ASSERT_EQ(precond->get_system_matrix(), this->mtx);
+}
+
+
+TYPED_TEST(GmresMixed, CanSetKrylovDim)
+{
+    using Solver = typename TestFixture::Solver;
+    using value_type = typename TestFixture::value_type;
+    auto gmres_mixed_factory =
+        Solver::build()
+            .with_krylov_dim_mixed(4u)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(4u).on(this->exec),
+                gko::stop::ResidualNormReduction<value_type>::build()
+                    .with_reduction_factor(TestFixture::reduction_factor)
+                    .on(this->exec))
+            .on(this->exec);
+    auto solver = gmres_mixed_factory->generate(this->mtx);
+    auto krylov_dim_mixed = solver->get_krylov_dim_mixed();
+
+    ASSERT_EQ(krylov_dim_mixed, 4);
+}
+
+
+TYPED_TEST(GmresMixed, CanSetPreconditionerInFactory)
+{
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<Solver> gmres_mixed_precond =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .on(this->exec)
+            ->generate(this->mtx);
+
+    auto gmres_mixed_factory =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .with_generated_preconditioner(gmres_mixed_precond)
+            .on(this->exec);
+    auto solver = gmres_mixed_factory->generate(this->mtx);
+    auto precond = solver->get_preconditioner();
+
+    ASSERT_NE(precond.get(), nullptr);
+    ASSERT_EQ(precond.get(), gmres_mixed_precond.get());
+}
+
+
+TYPED_TEST(GmresMixed, ThrowsOnWrongPreconditionerInFactory)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<Mtx> wrong_sized_mtx =
+        Mtx::create(this->exec, gko::dim<2>{1, 3});
+    std::shared_ptr<Solver> gmres_mixed_precond =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .on(this->exec)
+            ->generate(wrong_sized_mtx);
+
+    auto gmres_mixed_factory =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .with_generated_preconditioner(gmres_mixed_precond)
+            .on(this->exec);
+
+    ASSERT_THROW(gmres_mixed_factory->generate(this->mtx),
+                 gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(GmresMixed, CanSetPreconditioner)
+{
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<Solver> gmres_mixed_precond =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .on(this->exec)
+            ->generate(this->mtx);
+
+    auto gmres_mixed_factory =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .on(this->exec);
+    auto solver = gmres_mixed_factory->generate(this->mtx);
+    solver->set_preconditioner(gmres_mixed_precond);
+    auto precond = solver->get_preconditioner();
+
+    ASSERT_NE(precond.get(), nullptr);
+    ASSERT_EQ(precond.get(), gmres_mixed_precond.get());
+}
+
+
+}  // namespace
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 0b04f87c40c..80d1faf556e 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -113,6 +113,7 @@ target_sources(ginkgo_cuda
     solver/cgs_kernels.cu
     solver/fcg_kernels.cu
     solver/gmres_kernels.cu
+    solver/gmres_mixed_kernels.cu
     solver/idr_kernels.cu
     solver/ir_kernels.cu
     solver/lower_trs_kernels.cu
diff --git a/cuda/test/solver/CMakeLists.txt b/cuda/test/solver/CMakeLists.txt
index f4f671add8f..0689804b253 100644
--- a/cuda/test/solver/CMakeLists.txt
+++ b/cuda/test/solver/CMakeLists.txt
@@ -4,6 +4,7 @@ ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
+ginkgo_create_test(gmres_mixed_kernels)
 ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
 ginkgo_create_test_cpp_cuda_header(lower_trs_kernels)
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index fda9eafd03b..cdd37d3556e 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -39,6 +39,7 @@ target_sources(ginkgo_dpcpp
     solver/cgs_kernels.dp.cpp
     solver/fcg_kernels.dp.cpp
     solver/gmres_kernels.dp.cpp
+    solver/gmres_mixed_kernels.dp.cpp
     solver/idr_kernels.dp.cpp
     solver/ir_kernels.dp.cpp
     solver/lower_trs_kernels.dp.cpp
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index b35d6422098..e3ce557c654 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -21,6 +21,8 @@ set(EXAMPLES_LIST
     custom-matrix-format
     custom-stopping-criterion
     ginkgo-overhead
+    gmres-solver
+    gmres-mixed-solver
     heat-equation
     minimal-cuda-solver
     papi-logging
diff --git a/examples/gmres-mixed-solver/CMakeLists.txt b/examples/gmres-mixed-solver/CMakeLists.txt
new file mode 100644
index 00000000000..8871978b11e
--- /dev/null
+++ b/examples/gmres-mixed-solver/CMakeLists.txt
@@ -0,0 +1,9 @@
+add_executable(gmres-mixed-solver gmres-mixed-solver.cpp)
+target_link_libraries(gmres-mixed-solver ginkgo)
+target_include_directories(gmres-mixed-solver PRIVATE ${PROJECT_SOURCE_DIR})
+add_executable(gmres-mixed-solver-jacobi gmres-mixed-solver-jacobi.cpp)
+target_link_libraries(gmres-mixed-solver-jacobi ginkgo)
+target_include_directories(gmres-mixed-solver-jacobi PRIVATE ${PROJECT_SOURCE_DIR})
+configure_file(data/A.mtx data/A.mtx COPYONLY)
+configure_file(data/b.mtx data/b.mtx COPYONLY)
+configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/gmres-mixed-solver/build.sh b/examples/gmres-mixed-solver/build.sh
new file mode 100755
index 00000000000..f2c94bc239c
--- /dev/null
+++ b/examples/gmres-mixed-solver/build.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# set up script
+if [ $# -ne 1 ]; then
+    echo -e "Usage: $0 GINKGO_BUILD_DIRECTORY"
+    exit 1
+fi
+BUILD_DIR=$1
+THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" &>/dev/null && pwd )
+
+# copy libraries
+LIBRARY_DIRS="core core/device_hooks reference omp cuda hip"
+LIBRARY_NAMES="ginkgo ginkgo_reference ginkgo_omp ginkgo_cuda ginkgo_hip"
+SUFFIXES=".so .dylib .dll d.so d.dylib d.dll"
+for prefix in ${LIBRARY_DIRS}; do
+    for name in ${LIBRARY_NAMES}; do
+        for suffix in ${SUFFIXES}; do
+            cp ${BUILD_DIR}/${prefix}/lib${name}${suffix} \
+                ${THIS_DIR}/lib${name}${suffix} 2>/dev/null
+        done
+    done
+done
+
+# figure out correct compiler flags
+if ls ${THIS_DIR} | grep -F "libginkgo." >/dev/null; then
+    LINK_FLAGS="-lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference -lginkgo_hip"
+else
+    LINK_FLAGS="-lginkgod -lginkgo_ompd -lginkgo_cudad -lginkgo_referenced -lginkgo_hipd"
+fi
+if [ -z "${CXX}" ]; then
+    CXX="c++"
+fi
+
+# build
+${CXX} -std=c++11 -o ${THIS_DIR}/simple-solver ${THIS_DIR}/simple-solver.cpp \
+       -I${THIS_DIR}/../../include -I${BUILD_DIR}/include \
+       -L${THIS_DIR} ${LINK_FLAGS}
diff --git a/examples/gmres-mixed-solver/data/A.mtx b/examples/gmres-mixed-solver/data/A.mtx
new file mode 100644
index 00000000000..c67437da567
--- /dev/null
+++ b/examples/gmres-mixed-solver/data/A.mtx
@@ -0,0 +1,114 @@
+%%MatrixMarket matrix coordinate integer symmetric
+%-------------------------------------------------------------------------------
+% UF Sparse Matrix Collection, Tim Davis
+% http://www.cise.ufl.edu/research/sparse/matrices/JGD_Trefethen/Trefethen_20b
+% name: JGD_Trefethen/Trefethen_20b
+% [Diagonal matrices with primes, Nick Trefethen, Oxford Univ.]
+% id: 2203
+% date: 2008
+% author: N. Trefethen
+% ed: J.-G. Dumas
+% fields: name title A id date author ed kind notes
+% kind: combinatorial problem
+%-------------------------------------------------------------------------------
+% notes:
+% Diagonal matrices with primes, Nick Trefethen, Oxford Univ.          
+% From Jean-Guillaume Dumas' Sparse Integer Matrix Collection,         
+% http://ljk.imag.fr/membres/Jean-Guillaume.Dumas/simc.html            
+%                                                                      
+% Problem 7 of the Hundred-dollar, Hundred-digit Challenge Problems,   
+% SIAM News, vol 35, no. 1.                                            
+%                                                                      
+% 7. Let A be the 20,000 x 20,000 matrix whose entries are zero        
+% everywhere except for the primes 2, 3, 5, 7, . . . , 224737 along the
+% main diagonal and the number 1 in all the positions A(i,j) with      
+% |i-j| = 1,2,4,8, . . . ,16384.  What is the (1,1) entry of inv(A)?   
+%                                                                      
+% http://www.siam.org/news/news.php?id=388                             
+%                                                                      
+% Filename in JGD collection: Trefethen/trefethen_20__19_minor.sms     
+%-------------------------------------------------------------------------------
+19 19 83
+1 1 3
+2 1 1
+3 1 1
+5 1 1
+9 1 1
+17 1 1
+2 2 5
+3 2 1
+4 2 1
+6 2 1
+10 2 1
+18 2 1
+3 3 7
+4 3 1
+5 3 1
+7 3 1
+11 3 1
+19 3 1
+4 4 11
+5 4 1
+6 4 1
+8 4 1
+12 4 1
+5 5 13
+6 5 1
+7 5 1
+9 5 1
+13 5 1
+6 6 17
+7 6 1
+8 6 1
+10 6 1
+14 6 1
+7 7 19
+8 7 1
+9 7 1
+11 7 1
+15 7 1
+8 8 23
+9 8 1
+10 8 1
+12 8 1
+16 8 1
+9 9 29
+10 9 1
+11 9 1
+13 9 1
+17 9 1
+10 10 31
+11 10 1
+12 10 1
+14 10 1
+18 10 1
+11 11 37
+12 11 1
+13 11 1
+15 11 1
+19 11 1
+12 12 41
+13 12 1
+14 12 1
+16 12 1
+13 13 43
+14 13 1
+15 13 1
+17 13 1
+14 14 47
+15 14 1
+16 14 1
+18 14 1
+15 15 53
+16 15 1
+17 15 1
+19 15 1
+16 16 59
+17 16 1
+18 16 1
+17 17 61
+18 17 1
+19 17 1
+18 18 67
+19 18 1
+19 19 71
diff --git a/examples/gmres-mixed-solver/data/b.mtx b/examples/gmres-mixed-solver/data/b.mtx
new file mode 100644
index 00000000000..05d92ecc6f7
--- /dev/null
+++ b/examples/gmres-mixed-solver/data/b.mtx
@@ -0,0 +1,21 @@
+%%MatrixMarket matrix array real general
+19 1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
diff --git a/examples/gmres-mixed-solver/data/x0.mtx b/examples/gmres-mixed-solver/data/x0.mtx
new file mode 100644
index 00000000000..91d470cdbcd
--- /dev/null
+++ b/examples/gmres-mixed-solver/data/x0.mtx
@@ -0,0 +1,21 @@
+%%MatrixMarket matrix array real general
+19 1
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
diff --git a/examples/gmres-mixed-solver/doc/builds-on b/examples/gmres-mixed-solver/doc/builds-on
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/examples/gmres-mixed-solver/doc/builds-on
@@ -0,0 +1 @@
+
diff --git a/examples/gmres-mixed-solver/doc/intro.dox b/examples/gmres-mixed-solver/doc/intro.dox
new file mode 100644
index 00000000000..2869e91a12c
--- /dev/null
+++ b/examples/gmres-mixed-solver/doc/intro.dox
@@ -0,0 +1,19 @@
+<a name="Intro"></a>
+<h1>Introduction</h1>
+This simple solver example should help you get started with Ginkgo. This example is meant for you to understand
+how Ginkgo works and how you can solve a simple linear system with Ginkgo. We encourage you to play with the code,
+change the parameters and see what is best suited for your purposes.
+
+<h3> About the example </h3>
+Each example has the following sections:
+<ol>
+  <li> <b>Introduction:</b>This gives an overview of the example and mentions
+  any interesting aspects in the example that might help the reader.
+  <li> <b>The commented program:</b> This section is intended for you to
+  understand the details of the example so that you can play with it and understand
+  Ginkgo and its features better.
+  <li> <b>Results:</b> This section shows the results of the code when run. Though the
+  results may not be completely the same, you can expect the behaviour to be similar. 
+  <li> <b>The plain program:</b> This is the complete code without any comments to have
+  an complete overview of the code.
+  </ol>
diff --git a/examples/gmres-mixed-solver/doc/kind b/examples/gmres-mixed-solver/doc/kind
new file mode 100644
index 00000000000..15a13db4511
--- /dev/null
+++ b/examples/gmres-mixed-solver/doc/kind
@@ -0,0 +1 @@
+basic
diff --git a/examples/gmres-mixed-solver/doc/results.dox b/examples/gmres-mixed-solver/doc/results.dox
new file mode 100644
index 00000000000..0615abac5de
--- /dev/null
+++ b/examples/gmres-mixed-solver/doc/results.dox
@@ -0,0 +1,33 @@
+<h1>Results</h1>
+The following is the expected result:
+
+@code{.cpp}
+Solution (x):
+%%MatrixMarket matrix array real general
+19 1
+0.252218
+0.108645
+0.0662811
+0.0630433
+0.0384088
+0.0396536
+0.0402648
+0.0338935
+0.0193098
+0.0234653
+0.0211499
+0.0196413
+0.0199151
+0.0181674
+0.0162722
+0.0150714
+0.0107016
+0.0121141
+0.0123025
+Residual norm sqrt(r^T r):
+%%MatrixMarket matrix array real general
+1 1
+2.10788e-15
+@endcode
+
+<h3> Comments about programming and debugging </h3>
diff --git a/examples/gmres-mixed-solver/doc/short-intro b/examples/gmres-mixed-solver/doc/short-intro
new file mode 100644
index 00000000000..578df81366b
--- /dev/null
+++ b/examples/gmres-mixed-solver/doc/short-intro
@@ -0,0 +1 @@
+The simple solver example.
diff --git a/examples/gmres-mixed-solver/doc/tooltip b/examples/gmres-mixed-solver/doc/tooltip
new file mode 100644
index 00000000000..52c4f6d5f44
--- /dev/null
+++ b/examples/gmres-mixed-solver/doc/tooltip
@@ -0,0 +1 @@
+Solve a simple linear system with CG. Read a matrix and right hand side from a file.
diff --git a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi.cpp b/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi.cpp
new file mode 100644
index 00000000000..200f9c05ca4
--- /dev/null
+++ b/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi.cpp
@@ -0,0 +1,158 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+// @sect3{Include files}
+
+// This is the main ginkgo header file.
+#include <ginkgo/ginkgo.hpp>
+
+// Add the fstream header to read from data from files.
+#include <fstream>
+// Add the C++ iostream header to output information to the console.
+#include <iostream>
+// Add the string manipulation header to handle strings.
+#include <string>
+
+
+int main(int argc, char *argv[])
+{
+    // Use some shortcuts. In Ginkgo, vectors are seen as a gko::matrix::Dense
+    // with one column/one row. The advantage of this concept is that using
+    // multiple vectors is a now a natural extension of adding columns/rows are
+    // necessary.
+    using vec = gko::matrix::Dense<>;
+    // The gko::matrix::Csr class is used here, but any other matrix class such
+    // as gko::matrix::Coo, gko::matrix::Hybrid, gko::matrix::Ell or
+    // gko::matrix::Sellp could also be used.
+    using mtx = gko::matrix::Csr<>;
+    // The gko::solver::Cg is used here, but any other solver class can also be
+    // used.
+    //    using gmres_mixed = gko::solver::GmresMixed<double, float>;
+    using gmres_mixed = gko::solver::GmresMixed<>;
+    using bj = gko::preconditioner::Jacobi<>;
+
+    // Print the ginkgo version information.
+    std::cout << gko::version_info::get() << std::endl;
+
+    // @sect3{Where do you want to run your solver ?}
+    // The gko::Executor class is one of the cornerstones of Ginkgo. Currently,
+    // we have support for
+    // an gko::OmpExecutor, which uses OpenMP multi-threading in most of its
+    // kernels, a gko::ReferenceExecutor, a single threaded specialization of
+    // the OpenMP executor and a gko::CudaExecutor which runs the code on a
+    // NVIDIA GPU if available.
+    // @note With the help of C++, you see that you only ever need to change the
+    // executor and all the other functions/ routines within Ginkgo should
+    // automatically work and run on the executor with any other changes.
+    std::shared_ptr<gko::Executor> exec;
+    if (argc == 1 || std::string(argv[1]) == "reference") {
+        exec = gko::ReferenceExecutor::create();
+    } else if (argc == 2 && std::string(argv[1]) == "omp") {
+        exec = gko::OmpExecutor::create();
+    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
+               gko::CudaExecutor::get_num_devices() > 0) {
+        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
+               gko::HipExecutor::get_num_devices() > 0) {
+        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+    } else {
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+        std::exit(-1);
+    }
+
+    // @sect3{Reading your data and transfer to the proper device.}
+    // Read the matrix, right hand side and the initial solution using the @ref
+    // read function.
+    // @note Ginkgo uses C++ smart pointers to automatically manage memory. To
+    // this end, we use our own object ownership transfer functions that under
+    // the hood call the required smart pointer functions to manage object
+    // ownership. The gko::share , gko::give and gko::lend are the functions
+    // that you would need to use.
+    auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
+    auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
+    auto x = gko::read<vec>(std::ifstream("data/x0.mtx"), exec);
+
+    // @sect3{Creating the solver}
+    // Generate the gko::solver factory. Ginkgo uses the concept of Factories to
+    // build solvers with certain
+    // properties. Observe the Fluent interface used here. Here a gmres_mixed
+    // solver is generated with a stopping criteria of maximum iterations of 20
+    // and a residual norm reduction of 1e-15. You also observe that the
+    // stopping criteria(gko::stop) are also generated from factories using
+    // their build methods. You need to specify the executors which each of the
+    // object needs to be built on.
+    auto solver_gen =
+        gmres_mixed::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(20u).on(exec),
+                gko::stop::ResidualNormReduction<>::build()
+                    .with_reduction_factor(1e-15)
+                    .on(exec))
+            // Add preconditioner, these 2 lines are the only
+            // difference from the simple solver example
+            .with_preconditioner(bj::build().with_max_block_size(8u).on(exec))
+            .on(exec);
+    // Generate the solver from the matrix. The solver factory built in the
+    // previous step takes a "matrix"(a gko::LinOp to be more general) as an
+    // input. In this case we provide it with a full matrix that we previously
+    // read, but as the solver only effectively uses the apply() method within
+    // the provided "matrix" object, you can effectively create a gko::LinOp
+    // class with your own apply implementation to accomplish more tasks. We
+    // will see an example of how this can be done in the custom-matrix-format
+    // example
+    auto solver = solver_gen->generate(A);
+
+    // Finally, solve the system. The solver, being a gko::LinOp, can be applied
+    // to a right hand side, b to
+    // obtain the solution, x.
+    solver->apply(lend(b), lend(x));
+
+    // Print the solution to the command line.
+    std::cout << "Solution (x): \n";
+    write(std::cout, lend(x));
+
+    // To measure if your solution has actually converged, you can measure the
+    // error of the solution.
+    // one, neg_one are objects that represent the numbers which allow for a
+    // uniform interface when computing on any device. To compute the residual,
+    // all you need to do is call the apply method, which in this case is an
+    // spmv and equivalent to the LAPACK z_spmv routine. Finally, you compute
+    // the euclidean 2-norm with the compute_norm2 function.
+    auto one = gko::initialize<vec>({1.0}, exec);
+    auto neg_one = gko::initialize<vec>({-1.0}, exec);
+    auto res = gko::initialize<vec>({0.0}, exec);
+    A->apply(lend(one), lend(x), lend(neg_one), lend(b));
+    b->compute_norm2(lend(res));
+
+    std::cout << "Residual norm sqrt(r^T r): \n";
+    write(std::cout, lend(res));
+}
diff --git a/examples/gmres-mixed-solver/gmres-mixed-solver.cpp b/examples/gmres-mixed-solver/gmres-mixed-solver.cpp
new file mode 100644
index 00000000000..12d4f669ca7
--- /dev/null
+++ b/examples/gmres-mixed-solver/gmres-mixed-solver.cpp
@@ -0,0 +1,153 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+// @sect3{Include files}
+
+// This is the main ginkgo header file.
+#include <ginkgo/ginkgo.hpp>
+
+// Add the fstream header to read from data from files.
+#include <fstream>
+// Add the C++ iostream header to output information to the console.
+#include <iostream>
+// Add the string manipulation header to handle strings.
+#include <string>
+
+int main(int argc, char *argv[])
+{
+    // Use some shortcuts. In Ginkgo, vectors are seen as a gko::matrix::Dense
+    // with one column/one row. The advantage of this concept is that using
+    // multiple vectors is a now a natural extension of adding columns/rows are
+    // necessary.
+    using vec = gko::matrix::Dense<>;
+    // The gko::matrix::Csr class is used here, but any other matrix class such
+    // as gko::matrix::Coo, gko::matrix::Hybrid, gko::matrix::Ell or
+    // gko::matrix::Sellp could also be used.
+    using mtx = gko::matrix::Csr<>;
+    // The gko::solver::Cg is used here, but any other solver class can also be
+    // used.
+    //    using gmres_mixed = gko::solver::GmresMixed<double, float>;
+    using gmres_mixed = gko::solver::GmresMixed<>;
+
+    // Print the ginkgo version information.
+    std::cout << gko::version_info::get() << std::endl;
+
+    // @sect3{Where do you want to run your solver ?}
+    // The gko::Executor class is one of the cornerstones of Ginkgo. Currently,
+    // we have support for
+    // an gko::OmpExecutor, which uses OpenMP multi-threading in most of its
+    // kernels, a gko::ReferenceExecutor, a single threaded specialization of
+    // the OpenMP executor and a gko::CudaExecutor which runs the code on a
+    // NVIDIA GPU if available.
+    // @note With the help of C++, you see that you only ever need to change the
+    // executor and all the other functions/ routines within Ginkgo should
+    // automatically work and run on the executor with any other changes.
+    std::shared_ptr<gko::Executor> exec;
+    if (argc == 1 || std::string(argv[1]) == "reference") {
+        exec = gko::ReferenceExecutor::create();
+    } else if (argc == 2 && std::string(argv[1]) == "omp") {
+        exec = gko::OmpExecutor::create();
+    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
+               gko::CudaExecutor::get_num_devices() > 0) {
+        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
+               gko::HipExecutor::get_num_devices() > 0) {
+        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+    } else {
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+        std::exit(-1);
+    }
+
+    // @sect3{Reading your data and transfer to the proper device.}
+    // Read the matrix, right hand side and the initial solution using the @ref
+    // read function.
+    // @note Ginkgo uses C++ smart pointers to automatically manage memory. To
+    // this end, we use our own object ownership transfer functions that under
+    // the hood call the required smart pointer functions to manage object
+    // ownership. The gko::share , gko::give and gko::lend are the functions
+    // that you would need to use.
+    auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
+    auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
+    auto x = gko::read<vec>(std::ifstream("data/x0.mtx"), exec);
+
+    // @sect3{Creating the solver}
+    // Generate the gko::solver factory. Ginkgo uses the concept of Factories to
+    // build solvers with certain
+    // properties. Observe the Fluent interface used here. Here a gmres_mixed
+    // solver is generated with a stopping criteria of maximum iterations of 20
+    // and a residual norm reduction of 1e-15. You also observe that the
+    // stopping criteria(gko::stop) are also generated from factories using
+    // their build methods. You need to specify the executors which each of the
+    // object needs to be built on.
+    auto solver_gen =
+        gmres_mixed::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(20u).on(exec),
+                gko::stop::ResidualNormReduction<>::build()
+                    .with_reduction_factor(1e-15)
+                    .on(exec))
+            .on(exec);
+    // Generate the solver from the matrix. The solver factory built in the
+    // previous step takes a "matrix"(a gko::LinOp to be more general) as an
+    // input. In this case we provide it with a full matrix that we previously
+    // read, but as the solver only effectively uses the apply() method within
+    // the provided "matrix" object, you can effectively create a gko::LinOp
+    // class with your own apply implementation to accomplish more tasks. We
+    // will see an example of how this can be done in the custom-matrix-format
+    // example
+    auto solver = solver_gen->generate(A);
+
+    // Finally, solve the system. The solver, being a gko::LinOp, can be applied
+    // to a right hand side, b to
+    // obtain the solution, x.
+    solver->apply(lend(b), lend(x));
+
+    // Print the solution to the command line.
+    std::cout << "Solution (x): \n";
+    write(std::cout, lend(x));
+
+    // To measure if your solution has actually converged, you can measure the
+    // error of the solution.
+    // one, neg_one are objects that represent the numbers which allow for a
+    // uniform interface when computing on any device. To compute the residual,
+    // all you need to do is call the apply method, which in this case is an
+    // spmv and equivalent to the LAPACK z_spmv routine. Finally, you compute
+    // the euclidean 2-norm with the compute_norm2 function.
+    auto one = gko::initialize<vec>({1.0}, exec);
+    auto neg_one = gko::initialize<vec>({-1.0}, exec);
+    auto res = gko::initialize<vec>({0.0}, exec);
+    A->apply(lend(one), lend(x), lend(neg_one), lend(b));
+    b->compute_norm2(lend(res));
+
+    std::cout << "Residual norm sqrt(r^T r): \n";
+    write(std::cout, lend(res));
+}
diff --git a/examples/gmres-solver/CMakeLists.txt b/examples/gmres-solver/CMakeLists.txt
new file mode 100644
index 00000000000..d00153f906c
--- /dev/null
+++ b/examples/gmres-solver/CMakeLists.txt
@@ -0,0 +1,6 @@
+add_executable(gmres-solver gmres-solver.cpp)
+target_link_libraries(gmres-solver ginkgo)
+target_include_directories(gmres-solver PRIVATE ${PROJECT_SOURCE_DIR})
+configure_file(data/A.mtx data/A.mtx COPYONLY)
+configure_file(data/b.mtx data/b.mtx COPYONLY)
+configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/gmres-solver/build.sh b/examples/gmres-solver/build.sh
new file mode 100755
index 00000000000..f2c94bc239c
--- /dev/null
+++ b/examples/gmres-solver/build.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# set up script
+if [ $# -ne 1 ]; then
+    echo -e "Usage: $0 GINKGO_BUILD_DIRECTORY"
+    exit 1
+fi
+BUILD_DIR=$1
+THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" &>/dev/null && pwd )
+
+# copy libraries
+LIBRARY_DIRS="core core/device_hooks reference omp cuda hip"
+LIBRARY_NAMES="ginkgo ginkgo_reference ginkgo_omp ginkgo_cuda ginkgo_hip"
+SUFFIXES=".so .dylib .dll d.so d.dylib d.dll"
+for prefix in ${LIBRARY_DIRS}; do
+    for name in ${LIBRARY_NAMES}; do
+        for suffix in ${SUFFIXES}; do
+            cp ${BUILD_DIR}/${prefix}/lib${name}${suffix} \
+                ${THIS_DIR}/lib${name}${suffix} 2>/dev/null
+        done
+    done
+done
+
+# figure out correct compiler flags
+if ls ${THIS_DIR} | grep -F "libginkgo." >/dev/null; then
+    LINK_FLAGS="-lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference -lginkgo_hip"
+else
+    LINK_FLAGS="-lginkgod -lginkgo_ompd -lginkgo_cudad -lginkgo_referenced -lginkgo_hipd"
+fi
+if [ -z "${CXX}" ]; then
+    CXX="c++"
+fi
+
+# build
+${CXX} -std=c++11 -o ${THIS_DIR}/simple-solver ${THIS_DIR}/simple-solver.cpp \
+       -I${THIS_DIR}/../../include -I${BUILD_DIR}/include \
+       -L${THIS_DIR} ${LINK_FLAGS}
diff --git a/examples/gmres-solver/data/A.mtx b/examples/gmres-solver/data/A.mtx
new file mode 100644
index 00000000000..c67437da567
--- /dev/null
+++ b/examples/gmres-solver/data/A.mtx
@@ -0,0 +1,114 @@
+%%MatrixMarket matrix coordinate integer symmetric
+%-------------------------------------------------------------------------------
+% UF Sparse Matrix Collection, Tim Davis
+% http://www.cise.ufl.edu/research/sparse/matrices/JGD_Trefethen/Trefethen_20b
+% name: JGD_Trefethen/Trefethen_20b
+% [Diagonal matrices with primes, Nick Trefethen, Oxford Univ.]
+% id: 2203
+% date: 2008
+% author: N. Trefethen
+% ed: J.-G. Dumas
+% fields: name title A id date author ed kind notes
+% kind: combinatorial problem
+%-------------------------------------------------------------------------------
+% notes:
+% Diagonal matrices with primes, Nick Trefethen, Oxford Univ.          
+% From Jean-Guillaume Dumas' Sparse Integer Matrix Collection,         
+% http://ljk.imag.fr/membres/Jean-Guillaume.Dumas/simc.html            
+%                                                                      
+% Problem 7 of the Hundred-dollar, Hundred-digit Challenge Problems,   
+% SIAM News, vol 35, no. 1.                                            
+%                                                                      
+% 7. Let A be the 20,000 x 20,000 matrix whose entries are zero        
+% everywhere except for the primes 2, 3, 5, 7, . . . , 224737 along the
+% main diagonal and the number 1 in all the positions A(i,j) with      
+% |i-j| = 1,2,4,8, . . . ,16384.  What is the (1,1) entry of inv(A)?   
+%                                                                      
+% http://www.siam.org/news/news.php?id=388                             
+%                                                                      
+% Filename in JGD collection: Trefethen/trefethen_20__19_minor.sms     
+%-------------------------------------------------------------------------------
+19 19 83
+1 1 3
+2 1 1
+3 1 1
+5 1 1
+9 1 1
+17 1 1
+2 2 5
+3 2 1
+4 2 1
+6 2 1
+10 2 1
+18 2 1
+3 3 7
+4 3 1
+5 3 1
+7 3 1
+11 3 1
+19 3 1
+4 4 11
+5 4 1
+6 4 1
+8 4 1
+12 4 1
+5 5 13
+6 5 1
+7 5 1
+9 5 1
+13 5 1
+6 6 17
+7 6 1
+8 6 1
+10 6 1
+14 6 1
+7 7 19
+8 7 1
+9 7 1
+11 7 1
+15 7 1
+8 8 23
+9 8 1
+10 8 1
+12 8 1
+16 8 1
+9 9 29
+10 9 1
+11 9 1
+13 9 1
+17 9 1
+10 10 31
+11 10 1
+12 10 1
+14 10 1
+18 10 1
+11 11 37
+12 11 1
+13 11 1
+15 11 1
+19 11 1
+12 12 41
+13 12 1
+14 12 1
+16 12 1
+13 13 43
+14 13 1
+15 13 1
+17 13 1
+14 14 47
+15 14 1
+16 14 1
+18 14 1
+15 15 53
+16 15 1
+17 15 1
+19 15 1
+16 16 59
+17 16 1
+18 16 1
+17 17 61
+18 17 1
+19 17 1
+18 18 67
+19 18 1
+19 19 71
diff --git a/examples/gmres-solver/data/b.mtx b/examples/gmres-solver/data/b.mtx
new file mode 100644
index 00000000000..05d92ecc6f7
--- /dev/null
+++ b/examples/gmres-solver/data/b.mtx
@@ -0,0 +1,21 @@
+%%MatrixMarket matrix array real general
+19 1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
diff --git a/examples/gmres-solver/data/x0.mtx b/examples/gmres-solver/data/x0.mtx
new file mode 100644
index 00000000000..91d470cdbcd
--- /dev/null
+++ b/examples/gmres-solver/data/x0.mtx
@@ -0,0 +1,21 @@
+%%MatrixMarket matrix array real general
+19 1
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
diff --git a/examples/gmres-solver/doc/builds-on b/examples/gmres-solver/doc/builds-on
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/examples/gmres-solver/doc/builds-on
@@ -0,0 +1 @@
+
diff --git a/examples/gmres-solver/doc/intro.dox b/examples/gmres-solver/doc/intro.dox
new file mode 100644
index 00000000000..2869e91a12c
--- /dev/null
+++ b/examples/gmres-solver/doc/intro.dox
@@ -0,0 +1,19 @@
+<a name="Intro"></a>
+<h1>Introduction</h1>
+This simple solver example should help you get started with Ginkgo. This example is meant for you to understand
+how Ginkgo works and how you can solve a simple linear system with Ginkgo. We encourage you to play with the code,
+change the parameters and see what is best suited for your purposes.
+
+<h3> About the example </h3>
+Each example has the following sections:
+<ol>
+  <li> <b>Introduction:</b>This gives an overview of the example and mentions
+  any interesting aspects in the example that might help the reader.
+  <li> <b>The commented program:</b> This section is intended for you to
+  understand the details of the example so that you can play with it and understand
+  Ginkgo and its features better.
+  <li> <b>Results:</b> This section shows the results of the code when run. Though the
+  results may not be completely the same, you can expect the behaviour to be similar. 
+  <li> <b>The plain program:</b> This is the complete code without any comments to have
+  an complete overview of the code.
+  </ol>
diff --git a/examples/gmres-solver/doc/kind b/examples/gmres-solver/doc/kind
new file mode 100644
index 00000000000..15a13db4511
--- /dev/null
+++ b/examples/gmres-solver/doc/kind
@@ -0,0 +1 @@
+basic
diff --git a/examples/gmres-solver/doc/results.dox b/examples/gmres-solver/doc/results.dox
new file mode 100644
index 00000000000..0615abac5de
--- /dev/null
+++ b/examples/gmres-solver/doc/results.dox
@@ -0,0 +1,33 @@
+<h1>Results</h1>
+The following is the expected result:
+
+@code{.cpp}
+Solution (x):
+%%MatrixMarket matrix array real general
+19 1
+0.252218
+0.108645
+0.0662811
+0.0630433
+0.0384088
+0.0396536
+0.0402648
+0.0338935
+0.0193098
+0.0234653
+0.0211499
+0.0196413
+0.0199151
+0.0181674
+0.0162722
+0.0150714
+0.0107016
+0.0121141
+0.0123025
+Residual norm sqrt(r^T r):
+%%MatrixMarket matrix array real general
+1 1
+2.10788e-15
+@endcode
+
+<h3> Comments about programming and debugging </h3>
diff --git a/examples/gmres-solver/doc/short-intro b/examples/gmres-solver/doc/short-intro
new file mode 100644
index 00000000000..578df81366b
--- /dev/null
+++ b/examples/gmres-solver/doc/short-intro
@@ -0,0 +1 @@
+The simple solver example.
diff --git a/examples/gmres-solver/doc/tooltip b/examples/gmres-solver/doc/tooltip
new file mode 100644
index 00000000000..52c4f6d5f44
--- /dev/null
+++ b/examples/gmres-solver/doc/tooltip
@@ -0,0 +1 @@
+Solve a simple linear system with CG. Read a matrix and right hand side from a file.
diff --git a/examples/gmres-solver/gmres-solver.cpp b/examples/gmres-solver/gmres-solver.cpp
new file mode 100644
index 00000000000..a8c97b2b474
--- /dev/null
+++ b/examples/gmres-solver/gmres-solver.cpp
@@ -0,0 +1,152 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+// @sect3{Include files}
+
+// This is the main ginkgo header file.
+#include <ginkgo/ginkgo.hpp>
+
+// Add the fstream header to read from data from files.
+#include <fstream>
+// Add the C++ iostream header to output information to the console.
+#include <iostream>
+// Add the string manipulation header to handle strings.
+#include <string>
+
+int main(int argc, char *argv[])
+{
+    // Use some shortcuts. In Ginkgo, vectors are seen as a gko::matrix::Dense
+    // with one column/one row. The advantage of this concept is that using
+    // multiple vectors is a now a natural extension of adding columns/rows are
+    // necessary.
+    using vec = gko::matrix::Dense<>;
+    // The gko::matrix::Csr class is used here, but any other matrix class such
+    // as gko::matrix::Coo, gko::matrix::Hybrid, gko::matrix::Ell or
+    // gko::matrix::Sellp could also be used.
+    using mtx = gko::matrix::Csr<>;
+    // The gko::solver::Cg is used here, but any other solver class can also be
+    // used.
+    using gmres = gko::solver::Gmres<>;
+
+    // Print the ginkgo version information.
+    std::cout << gko::version_info::get() << std::endl;
+
+    // @sect3{Where do you want to run your solver ?}
+    // The gko::Executor class is one of the cornerstones of Ginkgo. Currently,
+    // we have support for
+    // an gko::OmpExecutor, which uses OpenMP multi-threading in most of its
+    // kernels, a gko::ReferenceExecutor, a single threaded specialization of
+    // the OpenMP executor and a gko::CudaExecutor which runs the code on a
+    // NVIDIA GPU if available.
+    // @note With the help of C++, you see that you only ever need to change the
+    // executor and all the other functions/ routines within Ginkgo should
+    // automatically work and run on the executor with any other changes.
+    std::shared_ptr<gko::Executor> exec;
+    if (argc == 1 || std::string(argv[1]) == "reference") {
+        exec = gko::ReferenceExecutor::create();
+    } else if (argc == 2 && std::string(argv[1]) == "omp") {
+        exec = gko::OmpExecutor::create();
+    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
+               gko::CudaExecutor::get_num_devices() > 0) {
+        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
+               gko::HipExecutor::get_num_devices() > 0) {
+        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+    } else {
+        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+        std::exit(-1);
+    }
+
+    // @sect3{Reading your data and transfer to the proper device.}
+    // Read the matrix, right hand side and the initial solution using the @ref
+    // read function.
+    // @note Ginkgo uses C++ smart pointers to automatically manage memory. To
+    // this end, we use our own object ownership transfer functions that under
+    // the hood call the required smart pointer functions to manage object
+    // ownership. The gko::share , gko::give and gko::lend are the functions
+    // that you would need to use.
+    auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
+    auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
+    auto x = gko::read<vec>(std::ifstream("data/x0.mtx"), exec);
+
+    // @sect3{Creating the solver}
+    // Generate the gko::solver factory. Ginkgo uses the concept of Factories to
+    // build solvers with certain
+    // properties. Observe the Fluent interface used here. Here a gmres solver
+    // is generated with a stopping criteria of maximum iterations of 20 and a
+    // residual norm reduction of 1e-15. You also observe that the stopping
+    // criteria(gko::stop) are also generated from factories using their build
+    // methods. You need to specify the executors which each of the object needs
+    // to be built on.
+    auto solver_gen =
+        gmres::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(20u).on(exec),
+                gko::stop::ResidualNormReduction<>::build()
+                    .with_reduction_factor(1e-15)
+                    .on(exec))
+            .on(exec);
+    // Generate the solver from the matrix. The solver factory built in the
+    // previous step takes a "matrix"(a gko::LinOp to be more general) as an
+    // input. In this case we provide it with a full matrix that we previously
+    // read, but as the solver only effectively uses the apply() method within
+    // the provided "matrix" object, you can effectively create a gko::LinOp
+    // class with your own apply implementation to accomplish more tasks. We
+    // will see an example of how this can be done in the custom-matrix-format
+    // example
+    auto solver = solver_gen->generate(A);
+
+    // Finally, solve the system. The solver, being a gko::LinOp, can be applied
+    // to a right hand side, b to
+    // obtain the solution, x.
+    solver->apply(lend(b), lend(x));
+
+    // Print the solution to the command line.
+    std::cout << "Solution (x): \n";
+    write(std::cout, lend(x));
+
+    // To measure if your solution has actually converged, you can measure the
+    // error of the solution.
+    // one, neg_one are objects that represent the numbers which allow for a
+    // uniform interface when computing on any device. To compute the residual,
+    // all you need to do is call the apply method, which in this case is an
+    // spmv and equivalent to the LAPACK z_spmv routine. Finally, you compute
+    // the euclidean 2-norm with the compute_norm2 function.
+    auto one = gko::initialize<vec>({1.0}, exec);
+    auto neg_one = gko::initialize<vec>({-1.0}, exec);
+    auto res = gko::initialize<vec>({0.0}, exec);
+    A->apply(lend(one), lend(x), lend(neg_one), lend(b));
+    b->compute_norm2(lend(res));
+
+    std::cout << "Residual norm sqrt(r^T r): \n";
+    write(std::cout, lend(res));
+}
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 089e15250a9..246bb42635d 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -204,6 +204,7 @@ set(GINKGO_HIP_SOURCES
     solver/cgs_kernels.hip.cpp
     solver/fcg_kernels.hip.cpp
     solver/gmres_kernels.hip.cpp
+    solver/gmres_mixed_kernels.hip.cpp
     solver/idr_kernels.hip.cpp
     solver/ir_kernels.hip.cpp
     solver/lower_trs_kernels.hip.cpp
diff --git a/hip/test/solver/CMakeLists.txt b/hip/test/solver/CMakeLists.txt
index e8bb9cc4768..9bf14e276e3 100644
--- a/hip/test/solver/CMakeLists.txt
+++ b/hip/test/solver/CMakeLists.txt
@@ -4,6 +4,7 @@ ginkgo_create_hip_test_special_linkage(cg_kernels)
 ginkgo_create_hip_test_special_linkage(cgs_kernels)
 ginkgo_create_hip_test_special_linkage(fcg_kernels)
 ginkgo_create_hip_test_special_linkage(gmres_kernels)
+ginkgo_create_hip_test_special_linkage(gmres_mixed_kernels)
 ginkgo_create_hip_test_special_linkage(idr_kernels)
 ginkgo_create_hip_test_special_linkage(ir_kernels)
 ginkgo_create_hip_test_special_linkage(lower_trs_kernels)
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index e1d1d86cfe5..57eb2c8990f 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -443,6 +443,42 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
     template _macro(std::complex<double>)
 
 
+/**
+ * Instantiates a template for each value type compiled by Ginkgo.
+ *
+ * @param _macro  A macro which expands the template instantiation
+ *                (not including the leading `template` specifier).
+ *                Should take one argument, which is replaced by the
+ *                value type.
+ */
+#define GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(_macro) \
+    template _macro(double, float);                 \
+    template _macro(double, double)
+
+
+/**
+ * Instantiates a template for each value type compiled by Ginkgo.
+ *
+ * @param _macro  A macro which expands the template instantiation
+ *                (not including the leading `template` specifier).
+ *                Should take one argument, which is replaced by the
+ *                value type.
+ */
+/*
+#define GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(_macro) \
+    template _macro(double, float, bool, bool);                 \
+    template _macro(double, double, bool, bool)
+//    template _macro(double, float, (bool) true, (bool) true); \
+//    template _macro(double, float, (bool) true, (bool) false); \
+//    template _macro(double, float, (bool) false, (bool) true); \
+//    template _macro(double, float, (bool) false, (bool) false); \
+//    template _macro(double, double, (bool) true, (bool) true); \
+//    template _macro(double, double, (bool) true, (bool) false); \
+//    template _macro(double, double, (bool) false, (bool) true); \
+//    template _macro(double, double, (bool) false, (bool) false)
+*/
+
+
 /**
  * Instantiates a template for each index type compiled by Ginkgo.
  *
diff --git a/include/ginkgo/core/solver/gmres_mixed.hpp b/include/ginkgo/core/solver/gmres_mixed.hpp
new file mode 100644
index 00000000000..d891c86b7bd
--- /dev/null
+++ b/include/ginkgo/core/solver/gmres_mixed.hpp
@@ -0,0 +1,188 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_SOLVER_GMRES_MIXED_HPP_
+#define GKO_CORE_SOLVER_GMRES_MIXED_HPP_
+
+
+#include <vector>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/log/logger.hpp>
+#include <ginkgo/core/matrix/identity.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/criterion.hpp>
+
+
+namespace gko {
+namespace solver {
+
+
+constexpr size_type default_krylov_dim_mixed = 100u;
+
+
+/**
+ * GMRES_MIXED or the generalized minimal residual method is an iterative type
+ * Krylov subspace method which is suitable for nonsymmetric linear systems.
+ *
+ * The implementation in Ginkgo makes use of the merged kernel to make the best
+ * use of data locality. The inner operations in one iteration of GMRES_MIXED
+ * are merged into 2 separate steps.
+ *
+ * @tparam ValueType  precision of matrix elements
+ *
+ * @ingroup solvers
+ * @ingroup LinOp
+ */
+/*
+template <typename ValueType = default_precision,
+          typename ValueTypeKrylovBases = float,
+          bool Reorthogonalization = true,
+          bool MGS_CGS = true>
+*/
+template <typename ValueType = default_precision,
+          typename ValueTypeKrylovBases = float>
+//          typename ValueTypeKrylovBases = default_precision>
+class GmresMixed :
+    //		public EnableLinOp<GmresMixed<ValueType,
+    // ValueTypeKrylovBases,Reorthogonalization, MGS_CGS>>,
+    public EnableLinOp<GmresMixed<ValueType, ValueTypeKrylovBases>>,
+    public Preconditionable {
+    friend class EnableLinOp<GmresMixed>;
+    friend class EnablePolymorphicObject<GmresMixed, LinOp>;
+
+public:
+    using value_type = ValueType;
+    using krylov_type = ValueTypeKrylovBases;
+    //    using reorthogon  = Reorthogonalization;
+    //    using mgs_cgs     = MGS_CGS;
+
+    /**
+     * Gets the system operator (matrix) of the linear system.
+     *
+     * @return the system operator (matrix)
+     */
+    std::shared_ptr<const LinOp> get_system_matrix() const
+    {
+        return system_matrix_;
+    }
+
+    /**
+     * Returns the krylov dimension.
+     *
+     * @return the krylov dimension
+     */
+    size_type get_krylov_dim_mixed() const { return krylov_dim_mixed_; }
+
+    GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
+    {
+        /**
+         * Criterion factories.
+         */
+        std::vector<std::shared_ptr<const stop::CriterionFactory>>
+            GKO_FACTORY_PARAMETER(criteria, nullptr);
+
+        /**
+         * Preconditioner factory.
+         */
+        std::shared_ptr<const LinOpFactory> GKO_FACTORY_PARAMETER(
+            preconditioner, nullptr);
+
+        /**
+         * Already generated preconditioner. If one is provided, the factory
+         * `preconditioner` will be ignored.
+         */
+        std::shared_ptr<const LinOp> GKO_FACTORY_PARAMETER(
+            generated_preconditioner, nullptr);
+
+        /**
+         * krylov dimension factory.
+         */
+        size_type GKO_FACTORY_PARAMETER(krylov_dim_mixed, 0u);
+    };
+    GKO_ENABLE_LIN_OP_FACTORY(GmresMixed, parameters, Factory);
+    GKO_ENABLE_BUILD_METHOD(Factory);
+
+protected:
+    void apply_impl(const LinOp *b, LinOp *x) const override;
+
+    void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
+                    LinOp *x) const override;
+
+    explicit GmresMixed(std::shared_ptr<const Executor> exec)
+        : EnableLinOp<GmresMixed>(std::move(exec))
+    {}
+
+    explicit GmresMixed(const Factory *factory,
+                        std::shared_ptr<const LinOp> system_matrix)
+        : EnableLinOp<GmresMixed>(factory->get_executor(),
+                                  transpose(system_matrix->get_size())),
+          parameters_{factory->get_parameters()},
+          system_matrix_{std::move(system_matrix)}
+    {
+        if (parameters_.generated_preconditioner) {
+            GKO_ASSERT_EQUAL_DIMENSIONS(parameters_.generated_preconditioner,
+                                        this);
+            set_preconditioner(parameters_.generated_preconditioner);
+        } else if (parameters_.preconditioner) {
+            set_preconditioner(
+                parameters_.preconditioner->generate(system_matrix_));
+        } else {
+            set_preconditioner(matrix::Identity<ValueType>::create(
+                this->get_executor(), this->get_size()[0]));
+        }
+        if (parameters_.krylov_dim_mixed) {
+            krylov_dim_mixed_ = parameters_.krylov_dim_mixed;
+        } else {
+            krylov_dim_mixed_ = default_krylov_dim_mixed;
+        }
+        stop_criterion_factory_ =
+            stop::combine(std::move(parameters_.criteria));
+    }
+
+private:
+    std::shared_ptr<const LinOp> system_matrix_{};
+    std::shared_ptr<const stop::CriterionFactory> stop_criterion_factory_{};
+    size_type krylov_dim_mixed_;
+};
+
+
+}  // namespace solver
+}  // namespace gko
+
+
+#endif  // GKO_CORE_SOLVER_GMRES_MIXED_HPP_
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 1e8600d68a4..fc5e904fb0d 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -101,6 +101,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/cgs.hpp>
 #include <ginkgo/core/solver/fcg.hpp>
 #include <ginkgo/core/solver/gmres.hpp>
+#include <ginkgo/core/solver/gmres_mixed.hpp>
 #include <ginkgo/core/solver/idr.hpp>
 #include <ginkgo/core/solver/ir.hpp>
 #include <ginkgo/core/solver/lower_trs.hpp>
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index 1107d91a38c..d4a774f298f 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -32,6 +32,7 @@ target_sources(ginkgo_omp
     solver/cgs_kernels.cpp
     solver/fcg_kernels.cpp
     solver/gmres_kernels.cpp
+    solver/gmres_mixed_kernels.cpp
     solver/idr_kernels.cpp
     solver/ir_kernels.cpp
     solver/lower_trs_kernels.cpp
diff --git a/omp/test/solver/CMakeLists.txt b/omp/test/solver/CMakeLists.txt
index 58a1e19edd1..de3fce1fb78 100644
--- a/omp/test/solver/CMakeLists.txt
+++ b/omp/test/solver/CMakeLists.txt
@@ -4,6 +4,7 @@ ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
+ginkgo_create_test(gmres_mixed_kernels)
 ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
 ginkgo_create_test(lower_trs_kernels)
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index 6a261d56dcc..8951092dcfe 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -30,6 +30,7 @@ target_sources(ginkgo_reference
     solver/cgs_kernels.cpp
     solver/fcg_kernels.cpp
     solver/gmres_kernels.cpp
+    solver/gmres_mixed_kernels.cpp
     solver/idr_kernels.cpp
     solver/ir_kernels.cpp
     solver/lower_trs_kernels.cpp
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
new file mode 100644
index 00000000000..9b506f90550
--- /dev/null
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -0,0 +1,576 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/gmres_mixed_kernels.hpp"
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/solver/gmres_mixed.hpp>
+
+#include <iostream>
+
+
+namespace gko {
+namespace kernels {
+namespace reference {
+/**
+ * @brief The GMRES_MIXED solver namespace.
+ *
+ * @ingroup gmres_mixed
+ */
+namespace gmres_mixed {
+
+
+namespace {
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
+                    matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                    matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
+                    const stopping_status *stop_status)
+{
+    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
+        if (stop_status[i].has_stopped()) {
+            continue;
+        }
+        for (size_type k = 0; k < iter + 1; ++k) {
+            hessenberg_iter->at(k, i) = 0;
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                hessenberg_iter->at(k, i) +=
+                    next_krylov_basis->at(j, i) *
+                    krylov_bases->at(j,
+                                     next_krylov_basis->get_size()[1] * k + i);
+            }
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                next_krylov_basis->at(j, i) -=
+                    hessenberg_iter->at(k, i) *
+                    krylov_bases->at(j,
+                                     next_krylov_basis->get_size()[1] * k + i);
+            }
+        }
+        // for i in 1:iter
+        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+        // end
+
+        hessenberg_iter->at(iter + 1, i) = 0;
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            hessenberg_iter->at(iter + 1, i) +=
+                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        hessenberg_iter->at(iter + 1, i) =
+            sqrt(hessenberg_iter->at(iter + 1, i));
+        // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
+            krylov_bases->at(j, next_krylov_basis->get_size()[1] * (iter + 1) +
+                                    i) = next_krylov_basis->at(j, i);
+        }
+        // next_krylov_basis /= hessenberg(iter, iter + 1)
+        // krylov_bases(:, iter + 1) = next_krylov_basis
+        // End of arnoldi
+    }
+}
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
+                        matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                        matrix::Dense<ValueType> *hessenberg_iter,
+                        matrix::Dense<ValueType> *buffer_iter, size_type iter,
+                        const stopping_status *stop_status)
+{
+    const ValueType eta = 1.0 / sqrt(2.0);
+    ValueType nrmP = 0, nrmN = 0;
+    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
+        nrmP = 0;
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            nrmP += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        nrmP = sqrt(nrmP);
+        // nrmP = norm(next_krylov_basis)
+        if (stop_status[i].has_stopped()) {
+            continue;
+        }
+        for (size_type k = 0; k < iter + 1; ++k) {
+            hessenberg_iter->at(k, i) = 0;
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                hessenberg_iter->at(k, i) +=
+                    next_krylov_basis->at(j, i) *
+                    krylov_bases->at(j,
+                                     next_krylov_basis->get_size()[1] * k + i);
+            }
+            //            std::cout << "  " << k << " => " <<
+            //            hessenberg_iter->at(k, i) << std::endl;
+        }
+        // for i in 1:iter
+        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+        // end
+        for (size_type k = 0; k < iter + 1; ++k) {
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                next_krylov_basis->at(j, i) -=
+                    hessenberg_iter->at(k, i) *
+                    krylov_bases->at(j,
+                                     next_krylov_basis->get_size()[1] * k + i);
+            }
+        }
+        // for i in 1:iter
+        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+        // end
+        nrmN = 0;
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            nrmN += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        nrmN = sqrt(nrmN);
+        // nrmN = norm(next_krylov_basis)
+        for (size_type l = 1; nrmN < eta * nrmP && l < 3; l++) {
+            //            std::cout << l << " => " << nrmN << " - " << eta << "
+            //            - " << nrmP << std::endl;
+            nrmP = nrmN;
+            for (size_type k = 0; k < iter + 1; ++k) {
+                buffer_iter->at(k, i) = 0;
+                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
+                     ++j) {
+                    buffer_iter->at(k, i) +=
+                        next_krylov_basis->at(j, i) *
+                        krylov_bases->at(
+                            j, next_krylov_basis->get_size()[1] * k + i);
+                }
+            }
+            // for i in 1:iter
+            //     buffer(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+            // end
+            for (size_type k = 0; k < iter + 1; ++k) {
+                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
+                     ++j) {
+                    next_krylov_basis->at(j, i) -=
+                        buffer_iter->at(k, i) *
+                        krylov_bases->at(
+                            j, next_krylov_basis->get_size()[1] * k + i);
+                }
+                //                std::cout << "  " << k << " => " <<
+                //                hessenberg_iter->at(k, i); std::cout << " - "
+                //                << buffer_iter->at(k, i) << std::endl;
+                hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
+            }
+            // for i in 1:iter
+            //     next_krylov_basis   -= buffer(iter, i) * krylov_bases(:, i)
+            //     hessenberg(iter, i) += buffer(iter, i)
+            // end
+            nrmN = 0;
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                nrmN +=
+                    next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            }
+            nrmN = sqrt(nrmN);
+            // nrmN = norm(next_krylov_basis)
+        }
+        //        std::cout << nrmN << " - " << eta << " - " << nrmP <<
+        //        std::endl;
+        // reorthogonalization
+        hessenberg_iter->at(iter + 1, i) = 0;
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            hessenberg_iter->at(iter + 1, i) +=
+                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        hessenberg_iter->at(iter + 1, i) =
+            sqrt(hessenberg_iter->at(iter + 1, i));
+        //        std::cout << hessenberg_iter->at(iter + 1, i) << std::endl;
+        // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
+            krylov_bases->at(j, next_krylov_basis->get_size()[1] * (iter + 1) +
+                                    i) = next_krylov_basis->at(j, i);
+        }
+        // next_krylov_basis /= hessenberg(iter, iter + 1)
+        // krylov_bases(:, iter + 1) = next_krylov_basis
+        // End of arnoldi
+    }
+}
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
+                           matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                           matrix::Dense<ValueType> *hessenberg_iter,
+                           size_type iter, const stopping_status *stop_status)
+{
+    double nrm = 0, reorth = 0;
+
+    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
+        if (stop_status[i].has_stopped()) {
+            continue;
+        }
+        nrm = 0;
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        nrm = sqrt(nrm) * 0.99;
+        // nrm = norm(next_krylov_basis)
+        for (size_type k = 0; k < iter + 1; ++k) {
+            hessenberg_iter->at(k, i) = 0;
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                hessenberg_iter->at(k, i) +=
+                    next_krylov_basis->at(j, i) *
+                    krylov_bases->at(j,
+                                     next_krylov_basis->get_size()[1] * k + i);
+            }
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                next_krylov_basis->at(j, i) -=
+                    hessenberg_iter->at(k, i) *
+                    krylov_bases->at(j,
+                                     next_krylov_basis->get_size()[1] * k + i);
+            }
+            if (hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i) > nrm) {
+                reorth = 0;
+                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
+                     ++j) {
+                    reorth += next_krylov_basis->at(j, i) *
+                              krylov_bases->at(
+                                  j, next_krylov_basis->get_size()[1] * k + i);
+                }
+                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
+                     ++j) {
+                    next_krylov_basis->at(j, i) -=
+                        reorth *
+                        krylov_bases->at(
+                            j, next_krylov_basis->get_size()[1] * k + i);
+                }
+                hessenberg_iter->at(k, i) += reorth;
+            }
+        }
+        // for i in 1:iter
+        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+        //     if (hessenberg(iter, i)*hessenberg(iter, i)>nrm*0.99)
+        //         reorth = next_krylov_basis' * krylov_bases(:, i)
+        //         next_krylov_basis  -= reorth * krylov_bases(:, i)
+        //         hessenberg(iter, i) += reorth;
+        //     end
+        // end
+
+        hessenberg_iter->at(iter + 1, i) = 0;
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            hessenberg_iter->at(iter + 1, i) +=
+                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        hessenberg_iter->at(iter + 1, i) =
+            sqrt(hessenberg_iter->at(iter + 1, i));
+        // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
+            krylov_bases->at(j, next_krylov_basis->get_size()[1] * (iter + 1) +
+                                    i) = next_krylov_basis->at(j, i);
+        }
+        // next_krylov_basis /= hessenberg(iter, iter + 1)
+        // krylov_bases(:, iter + 1) = next_krylov_basis
+        // End of arnoldi
+    }
+}
+
+
+template <typename ValueType>
+void calculate_sin_and_cos(matrix::Dense<ValueType> *givens_sin,
+                           matrix::Dense<ValueType> *givens_cos,
+                           matrix::Dense<ValueType> *hessenberg_iter,
+                           size_type iter, const size_type rhs)
+{
+    if (hessenberg_iter->at(iter, rhs) == zero<ValueType>()) {
+        givens_cos->at(iter, rhs) = zero<ValueType>();
+        givens_sin->at(iter, rhs) = one<ValueType>();
+    } else {
+        auto hypotenuse = sqrt(hessenberg_iter->at(iter, rhs) *
+                                   hessenberg_iter->at(iter, rhs) +
+                               hessenberg_iter->at(iter + 1, rhs) *
+                                   hessenberg_iter->at(iter + 1, rhs));
+        givens_cos->at(iter, rhs) =
+            abs(hessenberg_iter->at(iter, rhs)) / hypotenuse;
+        givens_sin->at(iter, rhs) = givens_cos->at(iter, rhs) *
+                                    hessenberg_iter->at(iter + 1, rhs) /
+                                    hessenberg_iter->at(iter, rhs);
+    }
+}
+
+
+template <typename ValueType>
+void givens_rotation(matrix::Dense<ValueType> *next_krylov_basis,
+                     matrix::Dense<ValueType> *givens_sin,
+                     matrix::Dense<ValueType> *givens_cos,
+                     matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
+                     const stopping_status *stop_status)
+{
+    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
+        if (stop_status[i].has_stopped()) {
+            continue;
+        }
+        for (size_type j = 0; j < iter; ++j) {
+            auto temp = givens_cos->at(j, i) * hessenberg_iter->at(j, i) +
+                        givens_sin->at(j, i) * hessenberg_iter->at(j + 1, i);
+            hessenberg_iter->at(j + 1, i) =
+                -givens_sin->at(j, i) * hessenberg_iter->at(j, i) +
+                givens_cos->at(j, i) * hessenberg_iter->at(j + 1, i);
+            hessenberg_iter->at(j, i) = temp;
+            // temp             =  cos(j)*hessenberg(j) +
+            //                     sin(j)*hessenberg(j+1)
+            // hessenberg(j+1)  = -sin(j)*hessenberg(j) +
+            //                     cos(j)*hessenberg(j+1)
+            // hessenberg(j)    =  temp;
+        }
+
+        calculate_sin_and_cos(givens_sin, givens_cos, hessenberg_iter, iter, i);
+
+        hessenberg_iter->at(iter, i) =
+            givens_cos->at(iter, i) * hessenberg_iter->at(iter, i) +
+            givens_sin->at(iter, i) * hessenberg_iter->at(iter + 1, i);
+        hessenberg_iter->at(iter + 1, i) = zero<ValueType>();
+        // hessenberg(iter)   = cos(iter)*hessenberg(iter) +
+        //                      sin(iter)*hessenberg(iter)
+        // hessenberg(iter+1) = 0
+    }
+}
+
+
+template <typename ValueType>
+void calculate_next_residual_norm(
+    matrix::Dense<ValueType> *givens_sin, matrix::Dense<ValueType> *givens_cos,
+    matrix::Dense<ValueType> *residual_norm,
+    matrix::Dense<ValueType> *residual_norm_collection,
+    const matrix::Dense<ValueType> *b_norm, size_type iter,
+    const stopping_status *stop_status)
+{
+    for (size_type i = 0; i < residual_norm->get_size()[1]; ++i) {
+        if (stop_status[i].has_stopped()) {
+            continue;
+        }
+        residual_norm_collection->at(iter + 1, i) =
+            -givens_sin->at(iter, i) * residual_norm_collection->at(iter, i);
+        residual_norm_collection->at(iter, i) =
+            givens_cos->at(iter, i) * residual_norm_collection->at(iter, i);
+        residual_norm->at(0, i) =
+            abs(residual_norm_collection->at(iter + 1, i)) / b_norm->at(0, i);
+    }
+}
+
+
+template <typename ValueType>
+void solve_upper_triangular(
+    const matrix::Dense<ValueType> *residual_norm_collection,
+    const matrix::Dense<ValueType> *hessenberg, matrix::Dense<ValueType> *y,
+    const size_type *final_iter_nums)
+{
+    for (size_type k = 0; k < residual_norm_collection->get_size()[1]; ++k) {
+        for (int i = final_iter_nums[k] - 1; i >= 0; --i) {
+            auto temp = residual_norm_collection->at(i, k);
+            for (size_type j = i + 1; j < final_iter_nums[k]; ++j) {
+                temp -=
+                    hessenberg->at(
+                        i, j * residual_norm_collection->get_size()[1] + k) *
+                    y->at(j, k);
+            }
+            y->at(i, k) =
+                temp / hessenberg->at(
+                           i, i * residual_norm_collection->get_size()[1] + k);
+        }
+    }
+}
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void calculate_qy(const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                  const matrix::Dense<ValueType> *y,
+                  matrix::Dense<ValueType> *before_preconditioner,
+                  const size_type *final_iter_nums)
+{
+    for (size_type k = 0; k < before_preconditioner->get_size()[1]; ++k) {
+        for (size_type i = 0; i < before_preconditioner->get_size()[0]; ++i) {
+            before_preconditioner->at(i, k) = zero<ValueType>();
+            for (size_type j = 0; j < final_iter_nums[k]; ++j) {
+                before_preconditioner->at(i, k) +=
+                    krylov_bases->at(
+                        i, j * before_preconditioner->get_size()[1] + k) *
+                    y->at(j, k);
+            }
+        }
+    }
+}
+
+
+}  // namespace
+
+
+template <typename ValueType>
+void initialize_1(std::shared_ptr<const ReferenceExecutor> exec,
+                  const matrix::Dense<ValueType> *b,
+                  matrix::Dense<ValueType> *b_norm,
+                  matrix::Dense<ValueType> *residual,
+                  matrix::Dense<ValueType> *givens_sin,
+                  matrix::Dense<ValueType> *givens_cos,
+                  Array<stopping_status> *stop_status, size_type krylov_dim)
+{
+    for (size_type j = 0; j < b->get_size()[1]; ++j) {
+        // Calculate b norm
+        b_norm->at(0, j) = zero<ValueType>();
+        for (size_type i = 0; i < b->get_size()[0]; ++i) {
+            b_norm->at(0, j) += b->at(i, j) * b->at(i, j);
+        }
+        b_norm->at(0, j) = sqrt(b_norm->at(0, j));
+
+        for (size_type i = 0; i < b->get_size()[0]; ++i) {
+            residual->at(i, j) = b->at(i, j);
+        }
+        for (size_type i = 0; i < krylov_dim; ++i) {
+            givens_sin->at(i, j) = zero<ValueType>();
+            givens_cos->at(i, j) = zero<ValueType>();
+        }
+        stop_status->get_data()[j].reset();
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
+                  const matrix::Dense<ValueType> *residual,
+                  matrix::Dense<ValueType> *residual_norm,
+                  matrix::Dense<ValueType> *residual_norm_collection,
+                  matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                  matrix::Dense<ValueType> *next_krylov_basis,
+                  Array<size_type> *final_iter_nums, size_type krylov_dim)
+{
+    for (size_type j = 0; j < residual->get_size()[1]; ++j) {
+        // Calculate residual norm
+        residual_norm->at(0, j) = 0;
+        for (size_type i = 0; i < residual->get_size()[0]; ++i) {
+            residual_norm->at(0, j) += residual->at(i, j) * residual->at(i, j);
+        }
+        residual_norm->at(0, j) = sqrt(residual_norm->at(0, j));
+
+        for (size_type i = 0; i < krylov_dim + 1; ++i) {
+            if (i == 0) {
+                residual_norm_collection->at(i, j) = residual_norm->at(0, j);
+            } else {
+                residual_norm_collection->at(i, j) = zero<ValueType>();
+            }
+        }
+        for (size_type i = 0; i < residual->get_size()[0]; ++i) {
+            krylov_bases->at(i, j) =
+                residual->at(i, j) / residual_norm->at(0, j);
+            next_krylov_basis->at(i, j) =
+                residual->at(i, j) / residual_norm->at(0, j);
+        }
+        final_iter_nums->get_data()[j] = 0;
+    }
+
+    for (size_type j = residual->get_size()[1]; j < krylov_bases->get_size()[1];
+         ++j) {
+        for (size_type i = 0; i < krylov_bases->get_size()[0]; ++i) {
+            krylov_bases->at(i, j) = zero<ValueType>();
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
+
+
+// template <typename ValueType, typename ValueTypeKrylovBases, bool
+// Reorthogonalization, bool MGS_CGS>
+template <typename ValueType, typename ValueTypeKrylovBases>
+void step_1(std::shared_ptr<const ReferenceExecutor> exec,
+            matrix::Dense<ValueType> *next_krylov_basis,
+            matrix::Dense<ValueType> *givens_sin,
+            matrix::Dense<ValueType> *givens_cos,
+            matrix::Dense<ValueType> *residual_norm,
+            matrix::Dense<ValueType> *residual_norm_collection,
+            matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+            matrix::Dense<ValueType> *hessenberg_iter,
+            matrix::Dense<ValueType> *buffer_iter,
+            const matrix::Dense<ValueType> *b_norm, size_type iter,
+            Array<size_type> *final_iter_nums,
+            const Array<stopping_status> *stop_status)
+{
+    /*
+        if (Reorthogonalization) std::cout << "Reorthogonalization";
+        else  std::cout << "No Reorthogonalization";
+        if (MGS_CGS) std::cout << "Modified Gram-Schmidt";
+        else  std::cout << "Classical Gram-Schmidt";
+    */
+    for (size_type i = 0; i < final_iter_nums->get_num_elems(); ++i) {
+        final_iter_nums->get_data()[i] +=
+            (1 - stop_status->get_const_data()[i].has_stopped());
+    }
+
+    //    finish_arnoldi(next_krylov_basis, krylov_bases, hessenberg_iter, iter,
+    //                   stop_status->get_const_data());
+    //    finish_arnoldi_reorth(next_krylov_basis, krylov_bases,
+    //    hessenberg_iter, iter,
+    //                          stop_status->get_const_data());
+    finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
+                       buffer_iter, iter, stop_status->get_const_data());
+    givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
+                    iter, stop_status->get_const_data());
+    calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,
+                                 residual_norm_collection, b_norm, iter,
+                                 stop_status->get_const_data());
+}
+
+// GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(GKO_DECLARE_GMRES_MIXED_BOOL_STEP_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void step_2(std::shared_ptr<const ReferenceExecutor> exec,
+            const matrix::Dense<ValueType> *residual_norm_collection,
+            const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+            const matrix::Dense<ValueType> *hessenberg,
+            matrix::Dense<ValueType> *y,
+            matrix::Dense<ValueType> *before_preconditioner,
+            const Array<size_type> *final_iter_nums)
+{
+    solve_upper_triangular(residual_norm_collection, hessenberg, y,
+                           final_iter_nums->get_const_data());
+    calculate_qy(krylov_bases, y, before_preconditioner,
+                 final_iter_nums->get_const_data());
+}
+
+GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
+
+
+}  // namespace gmres_mixed
+}  // namespace reference
+}  // namespace kernels
+}  // namespace gko
diff --git a/reference/test/solver/CMakeLists.txt b/reference/test/solver/CMakeLists.txt
index 7c53f72d86e..6014f571dc4 100644
--- a/reference/test/solver/CMakeLists.txt
+++ b/reference/test/solver/CMakeLists.txt
@@ -4,6 +4,7 @@ ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
+#ginkgo_create_test(gmres_mixed_kernels)
 ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
 ginkgo_create_test(lower_trs)
diff --git a/reference/test/solver/gmres_mixed_kernels.cpp b/reference/test/solver/gmres_mixed_kernels.cpp
new file mode 100644
index 00000000000..b5138b4631a
--- /dev/null
+++ b/reference/test/solver/gmres_mixed_kernels.cpp
@@ -0,0 +1,357 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/solver/gmres_mixed.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/preconditioner/jacobi.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm_reduction.hpp>
+#include <ginkgo/core/stop/time.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+template <typename T1, typename T2>
+class GmresMixed : public ::testing::Test {
+protected:
+    using value_type = T1;
+    using krylov_type = T2;
+    using Mtx = gko::matrix::Dense<value_type>;
+    using Solver = gko::solver::GmresMixed<value_type, krylov_type>;
+    GmresMixed()
+        : exec(gko::ReferenceExecutor::create()),
+          mtx(gko::initialize<Mtx>(
+              {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)),
+          gmres_mixed_factory(
+              Solver::build()
+                  .with_criteria(
+                      gko::stop::Iteration::build().with_max_iters(4u).on(exec),
+                      gko::stop::Time::build()
+                          .with_time_limit(std::chrono::seconds(6))
+                          .on(exec),
+                      gko::stop::ResidualNormReduction<value_type>::build()
+                          .with_reduction_factor(r<value_type>::value)
+                          .on(exec))
+                  .on(exec)),
+          mtx_big(gko::initialize<Mtx>(
+              {{2295.7, -764.8, 1166.5, 428.9, 291.7, -774.5},
+               {2752.6, -1127.7, 1212.8, -299.1, 987.7, 786.8},
+               {138.3, 78.2, 485.5, -899.9, 392.9, 1408.9},
+               {-1907.1, 2106.6, 1026.0, 634.7, 194.6, -534.1},
+               {-365.0, -715.8, 870.7, 67.5, 279.8, 1927.8},
+               {-848.1, -280.5, -381.8, -187.1, 51.2, -176.2}},
+              exec)),
+          gmres_mixed_factory_big(
+              Solver::build()
+                  .with_criteria(
+                      gko::stop::Iteration::build().with_max_iters(100u).on(
+                          exec),
+                      gko::stop::ResidualNormReduction<value_type>::build()
+                          .with_reduction_factor(r<value_type>::value)
+                          .on(exec))
+                  .on(exec)),
+          mtx_medium(
+              gko::initialize<Mtx>({{-86.40, 153.30, -108.90, 8.60, -61.60},
+                                    {7.70, -77.00, 3.30, -149.20, 74.80},
+                                    {-121.40, 37.10, 55.30, -74.20, -19.20},
+                                    {-111.40, -22.60, 110.10, -106.20, 88.90},
+                                    {-0.70, 111.70, 154.40, 235.00, -76.50}},
+                                   exec))
+    {}
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<Mtx> mtx;
+    std::shared_ptr<Mtx> mtx_medium;
+    std::shared_ptr<Mtx> mtx_big;
+    std::unique_ptr<typename Solver::Factory> gmres_mixed_factory;
+    std::unique_ptr<typename Solver::Factory> gmres_mixed_factory_big;
+};
+
+TYPED_TEST_CASE(GmresMixed, gko::test::ValueTypes);
+
+
+TYPED_TEST(GmresMixed, SolvesStencilSystem)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto solver = this->gmres_mixed_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}), r<value_type>::value * 1e1);
+}
+
+
+TYPED_TEST(GmresMixed, SolvesMultipleStencilSystems)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using T = value_type;
+    auto solver = this->gmres_mixed_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {I<T>{13.0, 6.0}, I<T>{7.0, 4.0}, I<T>{1.0, 1.0}}, this->exec);
+    auto x = gko::initialize<Mtx>(
+        {I<T>{0.0, 0.0}, I<T>{0.0, 0.0}, I<T>{0.0, 0.0}}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({{1.0, 1.0}, {3.0, 1.0}, {2.0, 1.0}}),
+                        r<value_type>::value * 1e1);
+}
+
+
+TYPED_TEST(GmresMixed, SolvesStencilSystemUsingAdvancedApply)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto solver = this->gmres_mixed_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), r<value_type>::value * 1e1);
+}
+
+
+TYPED_TEST(GmresMixed, SolvesMultipleStencilSystemsUsingAdvancedApply)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using T = value_type;
+    auto solver = this->gmres_mixed_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {I<T>{13.0, 6.0}, I<T>{7.0, 4.0}, I<T>{1.0, 1.0}}, this->exec);
+    auto x = gko::initialize<Mtx>(
+        {I<T>{0.5, 1.0}, I<T>{1.0, 2.0}, I<T>{2.0, 3.0}}, this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({{1.5, 1.0}, {5.0, 0.0}, {2.0, -1.0}}),
+                        r<value_type>::value * 1e1);
+}
+
+
+TYPED_TEST(GmresMixed, SolvesBigDenseSystem1)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto solver = this->gmres_mixed_factory_big->generate(this->mtx_big);
+    auto b = gko::initialize<Mtx>(
+        {72748.36, 297469.88, 347229.24, 36290.66, 82958.82, -80192.15},
+        this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({52.7, 85.4, 134.2, -250.0, -16.8, 35.3}),
+                        r<value_type>::value * 1e3);
+}
+
+
+TYPED_TEST(GmresMixed, SolvesBigDenseSystem2)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto solver = this->gmres_mixed_factory_big->generate(this->mtx_big);
+    auto b = gko::initialize<Mtx>(
+        {175352.10, 313410.50, 131114.10, -134116.30, 179529.30, -43564.90},
+        this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
+                        r<value_type>::value * 1e3);
+}
+
+
+template <typename T>
+gko::remove_complex<T> infNorm(gko::matrix::Dense<T> *mat, size_t col = 0)
+{
+    using std::abs;
+    using no_cpx_t = gko::remove_complex<T>;
+    no_cpx_t norm = 0.0;
+    for (size_t i = 0; i < mat->get_size()[0]; ++i) {
+        no_cpx_t absEntry = abs(mat->at(i, col));
+        if (norm < absEntry) norm = absEntry;
+    }
+    return norm;
+}
+
+
+TYPED_TEST(GmresMixed, SolvesMultipleDenseSystemForDivergenceCheck)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto solver = this->gmres_mixed_factory_big->generate(this->mtx_big);
+    auto b1 = gko::initialize<Mtx>(
+        {1300083.0, 1018120.5, 906410.0, -42679.5, 846779.5, 1176858.5},
+        this->exec);
+    auto b2 = gko::initialize<Mtx>(
+        {886630.5, -172578.0, 684522.0, -65310.5, 455487.5, 607436.0},
+        this->exec);
+
+    auto x1 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+    auto x2 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+
+    auto bc =
+        Mtx::create(this->exec, gko::dim<2>{this->mtx_big->get_size()[0], 2});
+    auto xc =
+        Mtx::create(this->exec, gko::dim<2>{this->mtx_big->get_size()[1], 2});
+    for (size_t i = 0; i < bc->get_size()[0]; ++i) {
+        bc->at(i, 0) = b1->at(i);
+        bc->at(i, 1) = b2->at(i);
+
+        xc->at(i, 0) = x1->at(i);
+        xc->at(i, 1) = x2->at(i);
+    }
+
+    solver->apply(b1.get(), x1.get());
+    solver->apply(b2.get(), x2.get());
+    solver->apply(bc.get(), xc.get());
+    auto mergedRes = Mtx::create(this->exec, gko::dim<2>{b1->get_size()[0], 2});
+    for (size_t i = 0; i < mergedRes->get_size()[0]; ++i) {
+        mergedRes->at(i, 0) = x1->at(i);
+        mergedRes->at(i, 1) = x2->at(i);
+    }
+
+    auto alpha = gko::initialize<Mtx>({1.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+
+    auto residual1 = Mtx::create(this->exec, b1->get_size());
+    residual1->copy_from(b1.get());
+    auto residual2 = Mtx::create(this->exec, b2->get_size());
+    residual2->copy_from(b2.get());
+    auto residualC = Mtx::create(this->exec, bc->get_size());
+    residualC->copy_from(bc.get());
+
+    this->mtx_big->apply(alpha.get(), x1.get(), beta.get(), residual1.get());
+    this->mtx_big->apply(alpha.get(), x2.get(), beta.get(), residual2.get());
+    this->mtx_big->apply(alpha.get(), xc.get(), beta.get(), residualC.get());
+
+    auto normS1 = infNorm(residual1.get());
+    auto normS2 = infNorm(residual2.get());
+    auto normC1 = infNorm(residualC.get(), 0);
+    auto normC2 = infNorm(residualC.get(), 1);
+    auto normB1 = infNorm(b1.get());
+    auto normB2 = infNorm(b2.get());
+
+    // make sure that all combined solutions are as good or better than the
+    // single solutions
+    ASSERT_LE(normC1 / normB1, normS1 / normB1 + r<value_type>::value);
+    ASSERT_LE(normC2 / normB2, normS2 / normB2 + r<value_type>::value);
+
+    // Not sure if this is necessary, the assertions above should cover what is
+    // needed.
+    GKO_ASSERT_MTX_NEAR(xc, mergedRes, r<value_type>::value);
+}
+
+
+TYPED_TEST(GmresMixed, SolvesBigDenseSystem1WithRestart)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    using value_type = typename TestFixture::value_type;
+    auto half_tol = std::sqrt(r<value_type>::value);
+    auto gmres_mixed_factory_restart =
+        Solver::build()
+            .with_krylov_dim(4u)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(200u).on(
+                    this->exec),
+                gko::stop::ResidualNormReduction<value_type>::build()
+                    .with_reduction_factor(r<value_type>::value)
+                    .on(this->exec))
+            .on(this->exec);
+    auto solver = gmres_mixed_factory_restart->generate(this->mtx_medium);
+    auto b = gko::initialize<Mtx>(
+        {-13945.16, 11205.66, 16132.96, 24342.18, -10910.98}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({-140.20, -142.20, 48.80, -17.70, -19.60}),
+                        half_tol * 1e2);
+}
+
+
+TYPED_TEST(GmresMixed, SolvesWithPreconditioner)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    using value_type = typename TestFixture::value_type;
+    auto gmres_mixed_factory_preconditioner =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(100u).on(
+                    this->exec),
+                gko::stop::ResidualNormReduction<value_type>::build()
+                    .with_reduction_factor(r<value_type>::value)
+                    .on(this->exec))
+            .with_preconditioner(
+                gko::preconditioner::Jacobi<value_type>::build()
+                    .with_max_block_size(3u)
+                    .on(this->exec))
+            .on(this->exec);
+    auto solver = gmres_mixed_factory_preconditioner->generate(this->mtx_big);
+    auto b = gko::initialize<Mtx>(
+        {175352.10, 313410.50, 131114.10, -134116.30, 179529.30, -43564.90},
+        this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
+                        r<value_type>::value * 1e3);
+}
+
+
+}  // namespace

From 65a69f2da1a092c2094a6a168329cfcccd76aebd Mon Sep 17 00:00:00 2001
From: "aliaga@uji.es" <aliaga@tintorrum.local>
Date: Mon, 27 Apr 2020 23:32:41 +0200
Subject: [PATCH 0334/1008] Inclusion of the omp executor in the repository:  
 * Now, the norms are properly created in the main class.   * The test files
 are not repaired yet.

---
 core/solver/gmres_mixed.cpp              |   4 +-
 core/solver/gmres_mixed_kernels.hpp      |   3 +-
 omp/solver/gmres_mixed_kernels.cpp       | 615 +++++++++++++++++++++++
 omp/test/solver/CMakeLists.txt           |   2 +-
 reference/solver/gmres_mixed_kernels.cpp | 224 +++++----
 5 files changed, 751 insertions(+), 97 deletions(-)
 create mode 100644 omp/solver/gmres_mixed_kernels.cpp

diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 44d61a53098..d0d8d7a9e3a 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -108,6 +108,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     auto residual_norm =
         Vector::create(exec, dim<2>{1, dense_b->get_size()[1]});
     auto b_norm = Vector::create(exec, dim<2>{1, dense_b->get_size()[1]});
+    auto arnoldi_norm = Vector::create(exec, dim<2>{2, dense_b->get_size()[1]});
     Array<size_type> final_iter_nums(this->get_executor(),
                                      dense_b->get_size()[1]);
     auto y =
@@ -215,7 +216,8 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
             next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
             residual_norm.get(), residual_norm_collection.get(),
             krylov_bases.get(), hessenberg_iter.get(), buffer_iter.get(),
-            b_norm.get(), restart_iter, &final_iter_nums, &stop_status));
+            b_norm.get(), arnoldi_norm.get(), restart_iter, &final_iter_nums,
+            &stop_status));
         // for i in 0:restart_iter
         //     hessenberg(restart_iter, i) = next_krylov_basis' *
         //     krylov_bases(:, i) next_krylov_basis  -= hessenberg(restart_iter,
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index 561db6bc214..fbfc164ddbf 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -75,7 +75,8 @@ namespace gmres_mixed {
                 matrix::Dense<_type2> *krylov_bases,                 \
                 matrix::Dense<_type1> *hessenberg_iter,              \
                 matrix::Dense<_type1> *buffer_iter,                  \
-                const matrix::Dense<_type1> *b_norm, size_type iter, \
+                const matrix::Dense<_type1> *b_norm,                 \
+                matrix::Dense<_type1> *arnoldi_norm, size_type iter, \
                 Array<size_type> *final_iter_nums,                   \
                 const Array<stopping_status> *stop_status)
 
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
new file mode 100644
index 00000000000..b1379ebf0fe
--- /dev/null
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -0,0 +1,615 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/gmres_mixed_kernels.hpp"
+
+
+#include <omp.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/solver/gmres_mixed.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+/**
+ * @brief The GMRES_MIXED solver namespace.
+ *
+ * @ingroup gmres_mixed
+ */
+namespace gmres_mixed {
+
+
+namespace {
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
+                    matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                    matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
+                    const stopping_status *stop_status)
+{
+#pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
+
+    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
+        if (stop_status[i].has_stopped()) {
+            continue;
+        }
+        for (size_type k = 0; k < iter + 1; ++k) {
+            ValueType hessenberg_iter_entry = zero<ValueType>();
+
+#pragma omp parallel for reduction(add : hessenberg_iter_entry)
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                hessenberg_iter_entry +=
+                    next_krylov_basis->at(j, i) *
+                    krylov_bases->at(j,
+                                     next_krylov_basis->get_size()[1] * k + i);
+            }
+            hessenberg_iter->at(k, i) = hessenberg_iter_entry;
+
+#pragma omp parallel for
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                next_krylov_basis->at(j, i) -=
+                    hessenberg_iter->at(k, i) *
+                    krylov_bases->at(j,
+                                     next_krylov_basis->get_size()[1] * k + i);
+            }
+        }
+        // for i in 1:iter
+        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+        // end
+
+        ValueType hessenberg_iter_entry = zero<ValueType>();
+
+#pragma omp parallel for reduction(add : hessenberg_iter_entry)
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            hessenberg_iter_entry +=
+                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        hessenberg_iter->at(iter + 1, i) = sqrt(hessenberg_iter_entry);
+        // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
+#pragma omp parallel for
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
+            krylov_bases->at(j, next_krylov_basis->get_size()[1] * (iter + 1) +
+                                    i) = next_krylov_basis->at(j, i);
+        }
+        // next_krylov_basis /= hessenberg(iter, iter + 1)
+        // krylov_bases(:, iter + 1) = next_krylov_basis
+        // End of arnoldi
+    }
+}
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
+                           matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                           matrix::Dense<ValueType> *hessenberg_iter,
+                           matrix::Dense<ValueType> *arnoldi_norm,
+                           size_type iter, const stopping_status *stop_status)
+{
+#pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
+
+    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
+        if (stop_status[i].has_stopped()) {
+            continue;
+        }
+
+        ValueType nrm = zero<ValueType>();
+#pragma omp parallel for reduction(add : nrm)
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        arnoldi_norm->at(0, i) = sqrt(nrm) * 0.99;
+        // nrm = norm(next_krylov_basis)
+        for (size_type k = 0; k < iter + 1; ++k) {
+            ValueType hessenberg_iter_entry = zero<ValueType>();
+#pragma omp parallel for reduction(add : hessenberg_iter_entry)
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                hessenberg_iter_entry +=
+                    next_krylov_basis->at(j, i) *
+                    krylov_bases->at(j,
+                                     next_krylov_basis->get_size()[1] * k + i);
+            }
+            hessenberg_iter->at(k, i) = hessenberg_iter_entry;
+
+#pragma omp parallel for
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                next_krylov_basis->at(j, i) -=
+                    hessenberg_iter->at(k, i) *
+                    krylov_bases->at(j,
+                                     next_krylov_basis->get_size()[1] * k + i);
+            }
+            if (hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i) >
+                arnoldi_norm->at(0, i)) {
+                ValueType reorth = zero<ValueType>();
+#pragma omp parallel for reduction(add : reorth)
+                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
+                     ++j) {
+                    reorth += next_krylov_basis->at(j, i) *
+                              krylov_bases->at(
+                                  j, next_krylov_basis->get_size()[1] * k + i);
+                }
+                hessenberg_iter->at(k, i) += reorth;
+
+#pragma omp parallel for
+                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
+                     ++j) {
+                    next_krylov_basis->at(j, i) -=
+                        reorth *
+                        krylov_bases->at(
+                            j, next_krylov_basis->get_size()[1] * k + i);
+                }
+            }
+        }
+        // for i in 1:iter
+        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+        //     if (hessenberg(iter, i)*hessenberg(iter, i)>nrm*0.99)
+        //         reorth = next_krylov_basis' * krylov_bases(:, i)
+        //         next_krylov_basis  -= reorth * krylov_bases(:, i)
+        //         hessenberg(iter, i) += reorth;
+        //     end
+        // end
+
+        ValueType hessenberg_iter_entry = zero<ValueType>();
+#pragma omp parallel for reduction(add : hessenberg_iter_entry)
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            hessenberg_iter_entry +=
+                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        hessenberg_iter->at(iter + 1, i) = sqrt(hessenberg_iter_entry);
+        // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
+#pragma omp parallel for
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
+            krylov_bases->at(j, next_krylov_basis->get_size()[1] * (iter + 1) +
+                                    i) = next_krylov_basis->at(j, i);
+        }
+        // next_krylov_basis /= hessenberg(iter, iter + 1)
+        // krylov_bases(:, iter + 1) = next_krylov_basis
+        // End of arnoldi
+    }
+}
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
+                        matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                        matrix::Dense<ValueType> *hessenberg_iter,
+                        matrix::Dense<ValueType> *buffer_iter,
+                        matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+                        const stopping_status *stop_status)
+{
+    const ValueType eta = 1.0 / sqrt(2.0);
+#pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
+
+    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
+        if (stop_status[i].has_stopped()) {
+            continue;
+        }
+
+        ValueType nrm = zero<ValueType>();
+#pragma omp parallel for reduction(add : nrm)
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        arnoldi_norm->at(0, i) = nrm * eta;
+        // nrmP = norm(next_krylov_basis)
+        for (size_type k = 0; k < iter + 1; ++k) {
+            ValueType hessenberg_iter_entry = zero<ValueType>();
+#pragma omp parallel for reduction(add : hessenberg_iter_entry)
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                hessenberg_iter_entry +=
+                    next_krylov_basis->at(j, i) *
+                    krylov_bases->at(j,
+                                     next_krylov_basis->get_size()[1] * k + i);
+            }
+            hessenberg_iter->at(k, i) = hessenberg_iter_entry;
+        }
+        // for i in 1:iter
+        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+        // end
+        for (size_type k = 0; k < iter + 1; ++k) {
+#pragma omp parallel for
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                next_krylov_basis->at(j, i) -=
+                    hessenberg_iter->at(k, i) *
+                    krylov_bases->at(j,
+                                     next_krylov_basis->get_size()[1] * k + i);
+            }
+        }
+        // for i in 1:iter
+        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+        // end
+        // ValueType nrm = zero<ValueType>();
+        nrm = zero<ValueType>();
+#pragma omp parallel for reduction(add : nrm)
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        arnoldi_norm->at(1, i) = nrm;
+        // nrmN = norm(next_krylov_basis)
+        for (size_type l = 1;
+             arnoldi_norm->at(1, i) < arnoldi_norm->at(0, i) && l < 3; l++) {
+            arnoldi_norm->at(0, i) = arnoldi_norm->at(1, i) * eta;
+            // nrmP = nrmN
+            for (size_type k = 0; k < iter + 1; ++k) {
+                ValueType hessenberg_iter_entry = zero<ValueType>();
+#pragma omp parallel for reduction(add : hessenberg_iter_entry)
+                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
+                     ++j) {
+                    hessenberg_iter_entry +=
+                        next_krylov_basis->at(j, i) *
+                        krylov_bases->at(
+                            j, next_krylov_basis->get_size()[1] * k + i);
+                }
+                buffer_iter->at(k, i) = hessenberg_iter_entry;
+            }
+            // for i in 1:iter
+            //     buffer(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+            // end
+            for (size_type k = 0; k < iter + 1; ++k) {
+#pragma omp parallel for
+                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
+                     ++j) {
+                    next_krylov_basis->at(j, i) -=
+                        buffer_iter->at(k, i) *
+                        krylov_bases->at(
+                            j, next_krylov_basis->get_size()[1] * k + i);
+                }
+            }
+            // for i in 1:iter
+            //     next_krylov_basis  -= buffer(iter, i) * krylov_bases(:, i)
+            // end
+            ValueType nrm = zero<ValueType>();
+#pragma omp parallel for reduction(add : nrm)
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                nrm +=
+                    next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            }
+            arnoldi_norm->at(1, i) = nrm;
+            // nrmN = norm(next_krylov_basis)
+        }
+        // reorthogonalization
+
+        ValueType hessenberg_iter_entry = zero<ValueType>();
+#pragma omp parallel for reduction(add : hessenberg_iter_entry)
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            hessenberg_iter_entry +=
+                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        hessenberg_iter->at(iter + 1, i) = sqrt(hessenberg_iter_entry);
+        // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
+#pragma omp parallel for
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
+            krylov_bases->at(j, next_krylov_basis->get_size()[1] * (iter + 1) +
+                                    i) = next_krylov_basis->at(j, i);
+        }
+        // next_krylov_basis /= hessenberg(iter, iter + 1)
+        // krylov_bases(:, iter + 1) = next_krylov_basis
+        // End of arnoldi
+    }
+}
+
+
+template <typename ValueType>
+void calculate_sin_and_cos(matrix::Dense<ValueType> *givens_sin,
+                           matrix::Dense<ValueType> *givens_cos,
+                           matrix::Dense<ValueType> *hessenberg_iter,
+                           size_type iter, const size_type rhs)
+{
+    if (hessenberg_iter->at(iter, rhs) == zero<ValueType>()) {
+        givens_cos->at(iter, rhs) = zero<ValueType>();
+        givens_sin->at(iter, rhs) = one<ValueType>();
+    } else {
+        auto hypotenuse = sqrt(hessenberg_iter->at(iter, rhs) *
+                                   hessenberg_iter->at(iter, rhs) +
+                               hessenberg_iter->at(iter + 1, rhs) *
+                                   hessenberg_iter->at(iter + 1, rhs));
+        givens_cos->at(iter, rhs) =
+            abs(hessenberg_iter->at(iter, rhs)) / hypotenuse;
+        givens_sin->at(iter, rhs) = givens_cos->at(iter, rhs) *
+                                    hessenberg_iter->at(iter + 1, rhs) /
+                                    hessenberg_iter->at(iter, rhs);
+    }
+}
+
+
+template <typename ValueType>
+void givens_rotation(matrix::Dense<ValueType> *next_krylov_basis,
+                     matrix::Dense<ValueType> *givens_sin,
+                     matrix::Dense<ValueType> *givens_cos,
+                     matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
+                     const stopping_status *stop_status)
+{
+#pragma omp parallel for
+    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
+        if (stop_status[i].has_stopped()) {
+            continue;
+        }
+        for (size_type j = 0; j < iter; ++j) {
+            auto temp = givens_cos->at(j, i) * hessenberg_iter->at(j, i) +
+                        givens_sin->at(j, i) * hessenberg_iter->at(j + 1, i);
+            hessenberg_iter->at(j + 1, i) =
+                -givens_sin->at(j, i) * hessenberg_iter->at(j, i) +
+                givens_cos->at(j, i) * hessenberg_iter->at(j + 1, i);
+            hessenberg_iter->at(j, i) = temp;
+            // temp             =  cos(j)*hessenberg(j) +
+            //                     sin(j)*hessenberg(j+1)
+            // hessenberg(j+1)  = -sin(j)*hessenberg(j) +
+            //                     cos(j)*hessenberg(j+1)
+            // hessenberg(j)    =  temp;
+        }
+
+        calculate_sin_and_cos(givens_sin, givens_cos, hessenberg_iter, iter, i);
+
+        hessenberg_iter->at(iter, i) =
+            givens_cos->at(iter, i) * hessenberg_iter->at(iter, i) +
+            givens_sin->at(iter, i) * hessenberg_iter->at(iter + 1, i);
+        hessenberg_iter->at(iter + 1, i) = zero<ValueType>();
+        // hessenberg(iter)   = cos(iter)*hessenberg(iter) +
+        //                      sin(iter)*hessenberg(iter)
+        // hessenberg(iter+1) = 0
+    }
+}
+
+
+template <typename ValueType>
+void calculate_next_residual_norm(
+    matrix::Dense<ValueType> *givens_sin, matrix::Dense<ValueType> *givens_cos,
+    matrix::Dense<ValueType> *residual_norm,
+    matrix::Dense<ValueType> *residual_norm_collection,
+    const matrix::Dense<ValueType> *b_norm, size_type iter,
+    const stopping_status *stop_status)
+{
+#pragma omp parallel for
+    for (size_type i = 0; i < residual_norm->get_size()[1]; ++i) {
+        if (stop_status[i].has_stopped()) {
+            continue;
+        }
+        residual_norm_collection->at(iter + 1, i) =
+            -givens_sin->at(iter, i) * residual_norm_collection->at(iter, i);
+        residual_norm_collection->at(iter, i) =
+            givens_cos->at(iter, i) * residual_norm_collection->at(iter, i);
+        residual_norm->at(0, i) =
+            abs(residual_norm_collection->at(iter + 1, i)) / b_norm->at(0, i);
+    }
+}
+
+
+template <typename ValueType>
+void solve_upper_triangular(
+    const matrix::Dense<ValueType> *residual_norm_collection,
+    const matrix::Dense<ValueType> *hessenberg, matrix::Dense<ValueType> *y,
+    const size_type *final_iter_nums)
+{
+#pragma omp parallel for
+    for (size_type k = 0; k < residual_norm_collection->get_size()[1]; ++k) {
+        for (int i = final_iter_nums[k] - 1; i >= 0; --i) {
+            auto temp = residual_norm_collection->at(i, k);
+            for (size_type j = i + 1; j < final_iter_nums[k]; ++j) {
+                temp -=
+                    hessenberg->at(
+                        i, j * residual_norm_collection->get_size()[1] + k) *
+                    y->at(j, k);
+            }
+            y->at(i, k) =
+                temp / hessenberg->at(
+                           i, i * residual_norm_collection->get_size()[1] + k);
+        }
+    }
+}
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void calculate_qy(const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                  const matrix::Dense<ValueType> *y,
+                  matrix::Dense<ValueType> *before_preconditioner,
+                  const size_type *final_iter_nums)
+{
+#pragma omp parallel for
+    for (size_type i = 0; i < before_preconditioner->get_size()[0]; ++i) {
+        for (size_type k = 0; k < before_preconditioner->get_size()[1]; ++k) {
+            before_preconditioner->at(i, k) = zero<ValueType>();
+            for (size_type j = 0; j < final_iter_nums[k]; ++j) {
+                before_preconditioner->at(i, k) +=
+                    krylov_bases->at(
+                        i, j * before_preconditioner->get_size()[1] + k) *
+                    y->at(j, k);
+            }
+        }
+    }
+}
+
+
+}  // namespace
+
+
+template <typename ValueType>
+void initialize_1(std::shared_ptr<const OmpExecutor> exec,
+                  const matrix::Dense<ValueType> *b,
+                  matrix::Dense<ValueType> *b_norm,
+                  matrix::Dense<ValueType> *residual,
+                  matrix::Dense<ValueType> *givens_sin,
+                  matrix::Dense<ValueType> *givens_cos,
+                  Array<stopping_status> *stop_status, size_type krylov_dim)
+{
+    for (size_type j = 0; j < b->get_size()[1]; ++j) {
+        // Calculate b norm
+        ValueType norm = zero<ValueType>();
+
+#pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
+
+#pragma omp parallel for reduction(add : norm)
+        for (size_type i = 0; i < b->get_size()[0]; ++i) {
+            norm += b->at(i, j) * b->at(i, j);
+        }
+        b_norm->at(0, j) = sqrt(norm);
+
+#pragma omp parallel for
+        for (size_type i = 0; i < b->get_size()[0]; ++i) {
+            residual->at(i, j) = b->at(i, j);
+        }
+
+#pragma omp parallel for
+        for (size_type i = 0; i < krylov_dim; ++i) {
+            givens_sin->at(i, j) = zero<ValueType>();
+            givens_cos->at(i, j) = zero<ValueType>();
+        }
+        stop_status->get_data()[j].reset();
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void initialize_2(std::shared_ptr<const OmpExecutor> exec,
+                  const matrix::Dense<ValueType> *residual,
+                  matrix::Dense<ValueType> *residual_norm,
+                  matrix::Dense<ValueType> *residual_norm_collection,
+                  matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                  matrix::Dense<ValueType> *next_krylov_basis,
+                  Array<size_type> *final_iter_nums, size_type krylov_dim)
+{
+    for (size_type j = 0; j < residual->get_size()[1]; ++j) {
+        // Calculate residual norm
+        ValueType res_norm = zero<ValueType>();
+
+#pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
+
+#pragma omp parallel for reduction(add : res_norm)
+        for (size_type i = 0; i < residual->get_size()[0]; ++i) {
+            res_norm += residual->at(i, j) * residual->at(i, j);
+        }
+        residual_norm->at(0, j) = sqrt(res_norm);
+
+#pragma omp parallel for
+        for (size_type i = 0; i < krylov_dim + 1; ++i) {
+            if (i == 0) {
+                residual_norm_collection->at(i, j) = residual_norm->at(0, j);
+            } else {
+                residual_norm_collection->at(i, j) = zero<ValueType>();
+            }
+        }
+
+#pragma omp parallel for
+        for (size_type i = 0; i < residual->get_size()[0]; ++i) {
+            auto value = residual->at(i, j) / residual_norm->at(0, j);
+            krylov_bases->at(i, j) = value;
+            next_krylov_basis->at(i, j) = value;
+        }
+        final_iter_nums->get_data()[j] = 0;
+    }
+
+#pragma omp parallel for
+    for (size_type i = 0; i < krylov_bases->get_size()[0]; ++i) {
+        for (size_type j = residual->get_size()[1];
+             j < krylov_bases->get_size()[1]; ++j) {
+            krylov_bases->at(i, j) = zero<ValueType>();
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void step_1(std::shared_ptr<const OmpExecutor> exec,
+            matrix::Dense<ValueType> *next_krylov_basis,
+            matrix::Dense<ValueType> *givens_sin,
+            matrix::Dense<ValueType> *givens_cos,
+            matrix::Dense<ValueType> *residual_norm,
+            matrix::Dense<ValueType> *residual_norm_collection,
+            matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+            matrix::Dense<ValueType> *hessenberg_iter,
+            matrix::Dense<ValueType> *buffer_iter,
+            const matrix::Dense<ValueType> *b_norm,
+            matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+            Array<size_type> *final_iter_nums,
+            const Array<stopping_status> *stop_status)
+{
+#pragma omp parallel for
+    for (size_type i = 0; i < final_iter_nums->get_num_elems(); ++i) {
+        final_iter_nums->get_data()[i] +=
+            (1 - stop_status->get_const_data()[i].has_stopped());
+    }
+    //    finish_arnoldi(next_krylov_basis, krylov_bases, hessenberg_iter, iter,
+    //                   stop_status->get_const_data());
+    //    finish_arnoldi_reorth(next_krylov_basis, krylov_bases,
+    //                        hessenberg_iter, arnoldi_norm, iter,
+    //                          stop_status->get_const_data());
+    finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
+                       buffer_iter, arnoldi_norm, iter,
+                       stop_status->get_const_data());
+    givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
+                    iter, stop_status->get_const_data());
+    calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,
+                                 residual_norm_collection, b_norm, iter,
+                                 stop_status->get_const_data());
+}
+
+GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void step_2(std::shared_ptr<const OmpExecutor> exec,
+            const matrix::Dense<ValueType> *residual_norm_collection,
+            const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+            const matrix::Dense<ValueType> *hessenberg,
+            matrix::Dense<ValueType> *y,
+            matrix::Dense<ValueType> *before_preconditioner,
+            const Array<size_type> *final_iter_nums)
+{
+    solve_upper_triangular(residual_norm_collection, hessenberg, y,
+                           final_iter_nums->get_const_data());
+    calculate_qy(krylov_bases, y, before_preconditioner,
+                 final_iter_nums->get_const_data());
+}
+
+GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
+
+
+}  // namespace gmres_mixed
+}  // namespace omp
+}  // namespace kernels
+}  // namespace gko
diff --git a/omp/test/solver/CMakeLists.txt b/omp/test/solver/CMakeLists.txt
index de3fce1fb78..7c75ae347b0 100644
--- a/omp/test/solver/CMakeLists.txt
+++ b/omp/test/solver/CMakeLists.txt
@@ -4,7 +4,7 @@ ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
-ginkgo_create_test(gmres_mixed_kernels)
+# ginkgo_create_test(gmres_mixed_kernels)
 ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
 ginkgo_create_test(lower_trs_kernels)
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index 9b506f90550..e19b2c989c2 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -105,25 +105,30 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
     }
 }
 
+
 template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
-                        matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
-                        matrix::Dense<ValueType> *hessenberg_iter,
-                        matrix::Dense<ValueType> *buffer_iter, size_type iter,
-                        const stopping_status *stop_status)
+void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
+                           matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                           matrix::Dense<ValueType> *hessenberg_iter,
+                           matrix::Dense<ValueType> *arnoldi_norm,
+                           size_type iter, const stopping_status *stop_status)
 {
-    const ValueType eta = 1.0 / sqrt(2.0);
-    ValueType nrmP = 0, nrmN = 0;
+    //    ValueType nrm = 0, reorth = 0;
+
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
-        nrmP = 0;
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            nrmP += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
-        }
-        nrmP = sqrt(nrmP);
-        // nrmP = norm(next_krylov_basis)
         if (stop_status[i].has_stopped()) {
             continue;
         }
+        // nrm = 0;
+        arnoldi_norm->at(0, i) = 0;
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            arnoldi_norm->at(0, i) +=
+                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            // nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        arnoldi_norm->at(0, i) = sqrt(arnoldi_norm->at(0, i)) * 0.99;
+        // nrm = sqrt(nrm) * 0.99;
+        // nrm = norm(next_krylov_basis)
         for (size_type k = 0; k < iter + 1; ++k) {
             hessenberg_iter->at(k, i) = 0;
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
@@ -132,74 +137,48 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
                     krylov_bases->at(j,
                                      next_krylov_basis->get_size()[1] * k + i);
             }
-            //            std::cout << "  " << k << " => " <<
-            //            hessenberg_iter->at(k, i) << std::endl;
-        }
-        // for i in 1:iter
-        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-        // end
-        for (size_type k = 0; k < iter + 1; ++k) {
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
                     hessenberg_iter->at(k, i) *
                     krylov_bases->at(j,
                                      next_krylov_basis->get_size()[1] * k + i);
             }
-        }
-        // for i in 1:iter
-        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
-        // end
-        nrmN = 0;
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            nrmN += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
-        }
-        nrmN = sqrt(nrmN);
-        // nrmN = norm(next_krylov_basis)
-        for (size_type l = 1; nrmN < eta * nrmP && l < 3; l++) {
-            //            std::cout << l << " => " << nrmN << " - " << eta << "
-            //            - " << nrmP << std::endl;
-            nrmP = nrmN;
-            for (size_type k = 0; k < iter + 1; ++k) {
-                buffer_iter->at(k, i) = 0;
+            if (hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i) >
+                arnoldi_norm->at(0, i)) {
+                // if (hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i) >
+                // nrm) {
+                arnoldi_norm->at(1, i) = 0;
+                // reorth = 0;
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
-                    buffer_iter->at(k, i) +=
+                    // reorth += next_krylov_basis->at(j, i) *
+                    arnoldi_norm->at(1, i) +=
                         next_krylov_basis->at(j, i) *
                         krylov_bases->at(
                             j, next_krylov_basis->get_size()[1] * k + i);
                 }
-            }
-            // for i in 1:iter
-            //     buffer(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-            // end
-            for (size_type k = 0; k < iter + 1; ++k) {
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
-                        buffer_iter->at(k, i) *
+                        arnoldi_norm->at(1, i) *
                         krylov_bases->at(
                             j, next_krylov_basis->get_size()[1] * k + i);
+                    // reorth *
                 }
-                //                std::cout << "  " << k << " => " <<
-                //                hessenberg_iter->at(k, i); std::cout << " - "
-                //                << buffer_iter->at(k, i) << std::endl;
-                hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
-            }
-            // for i in 1:iter
-            //     next_krylov_basis   -= buffer(iter, i) * krylov_bases(:, i)
-            //     hessenberg(iter, i) += buffer(iter, i)
-            // end
-            nrmN = 0;
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                nrmN +=
-                    next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+                hessenberg_iter->at(k, i) += arnoldi_norm->at(1, i);
+                //                hessenberg_iter->at(k, i) += reorth;
             }
-            nrmN = sqrt(nrmN);
-            // nrmN = norm(next_krylov_basis)
         }
-        //        std::cout << nrmN << " - " << eta << " - " << nrmP <<
-        //        std::endl;
-        // reorthogonalization
+        // for i in 1:iter
+        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+        //     if (hessenberg(iter, i)*hessenberg(iter, i)>nrm*0.99)
+        //         reorth = next_krylov_basis' * krylov_bases(:, i)
+        //         next_krylov_basis  -= reorth * krylov_bases(:, i)
+        //         hessenberg(iter, i) += reorth;
+        //     end
+        // end
+
         hessenberg_iter->at(iter + 1, i) = 0;
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             hessenberg_iter->at(iter + 1, i) +=
@@ -207,7 +186,6 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
         }
         hessenberg_iter->at(iter + 1, i) =
             sqrt(hessenberg_iter->at(iter + 1, i));
-        //        std::cout << hessenberg_iter->at(iter + 1, i) << std::endl;
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
@@ -222,23 +200,31 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
-                           matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
-                           matrix::Dense<ValueType> *hessenberg_iter,
-                           size_type iter, const stopping_status *stop_status)
+void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
+                        matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                        matrix::Dense<ValueType> *hessenberg_iter,
+                        matrix::Dense<ValueType> *buffer_iter,
+                        matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+                        const stopping_status *stop_status)
 {
-    double nrm = 0, reorth = 0;
-
+    const ValueType eta = 1.0 / sqrt(2.0);
+    //    ValueType nrmP = 0, nrmN = 0;
+    ValueType nrmP = 0, nrmN = 0;
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
+        arnoldi_norm->at(0, i) = 0;
+        // nrmP = 0;
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            arnoldi_norm->at(0, i) +=
+                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            // nrmP += next_krylov_basis->at(j, i) * next_krylov_basis->at(j,
+            // i);
+        }
+        arnoldi_norm->at(0, i) = sqrt(arnoldi_norm->at(0, i)) * eta;
+        // nrmP = sqrt(nrmP);
+        // arnoldi_norm->at(0, i) = norm(next_krylov_basis)
         if (stop_status[i].has_stopped()) {
             continue;
         }
-        nrm = 0;
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
-        }
-        nrm = sqrt(nrm) * 0.99;
-        // nrm = norm(next_krylov_basis)
         for (size_type k = 0; k < iter + 1; ++k) {
             hessenberg_iter->at(k, i) = 0;
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
@@ -247,40 +233,87 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
                     krylov_bases->at(j,
                                      next_krylov_basis->get_size()[1] * k + i);
             }
+            //            std::cout << "  " << k << " => " <<
+            //            hessenberg_iter->at(k, i) << std::endl;
+        }
+        // for i in 1:iter
+        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+        // end
+        for (size_type k = 0; k < iter + 1; ++k) {
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
                     hessenberg_iter->at(k, i) *
                     krylov_bases->at(j,
                                      next_krylov_basis->get_size()[1] * k + i);
             }
-            if (hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i) > nrm) {
-                reorth = 0;
+        }
+        // for i in 1:iter
+        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+        // end
+        arnoldi_norm->at(1, i) = 0;
+        // nrmN = 0;
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            arnoldi_norm->at(1, i) +=
+                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            // nrmN += next_krylov_basis->at(j, i) * next_krylov_basis->at(j,
+            // i);
+        }
+        arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
+        // nrmN = sqrt(nrmN);
+        // nrmN = arnoldi_norm->at(1, i)(next_krylov_basis)
+        for (size_type l = 1;
+             arnoldi_norm->at(1, i) < arnoldi_norm->at(0, i) && l < 3; l++) {
+            // for (size_type l = 1; arnoldi_norm->at(1, i) < eta *
+            // arnoldi_norm->at(0, i) && l < 3; l++) { for (size_type l = 1;
+            // nrmN < eta * nrmP && l < 3; l++) {
+            //            std::cout << l << " => " << nrmN << " - " << eta << "
+            //            - " << nrmP << std::endl;
+            arnoldi_norm->at(0, i) = eta * arnoldi_norm->at(1, i);
+            // nrmP = nrmN;
+            for (size_type k = 0; k < iter + 1; ++k) {
+                buffer_iter->at(k, i) = 0;
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
-                    reorth += next_krylov_basis->at(j, i) *
-                              krylov_bases->at(
-                                  j, next_krylov_basis->get_size()[1] * k + i);
+                    buffer_iter->at(k, i) +=
+                        next_krylov_basis->at(j, i) *
+                        krylov_bases->at(
+                            j, next_krylov_basis->get_size()[1] * k + i);
                 }
+            }
+            // for i in 1:iter
+            //     buffer(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+            // end
+            for (size_type k = 0; k < iter + 1; ++k) {
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
-                        reorth *
+                        buffer_iter->at(k, i) *
                         krylov_bases->at(
                             j, next_krylov_basis->get_size()[1] * k + i);
                 }
-                hessenberg_iter->at(k, i) += reorth;
+                //                std::cout << "  " << k << " => " <<
+                //                hessenberg_iter->at(k, i); std::cout << " - "
+                //                << buffer_iter->at(k, i) << std::endl;
+                hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
+            }
+            // for i in 1:iter
+            //     next_krylov_basis   -= buffer(iter, i) * krylov_bases(:, i)
+            //     hessenberg(iter, i) += buffer(iter, i)
+            // end
+            arnoldi_norm->at(1, i) = 0;
+            // nrmN = 0;
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                arnoldi_norm->at(1, i) +=
+                    // nrmN +=
+                    next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
             }
+            arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
+            // nrmN = sqrt(nrmN);
+            // nrmN = norm(next_krylov_basis)
         }
-        // for i in 1:iter
-        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
-        //     if (hessenberg(iter, i)*hessenberg(iter, i)>nrm*0.99)
-        //         reorth = next_krylov_basis' * krylov_bases(:, i)
-        //         next_krylov_basis  -= reorth * krylov_bases(:, i)
-        //         hessenberg(iter, i) += reorth;
-        //     end
-        // end
-
+        //        std::cout << nrmN << " - " << eta << " - " << nrmP <<
+        //        std::endl;
+        // reorthogonalization
         hessenberg_iter->at(iter + 1, i) = 0;
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             hessenberg_iter->at(iter + 1, i) +=
@@ -288,6 +321,7 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
         }
         hessenberg_iter->at(iter + 1, i) =
             sqrt(hessenberg_iter->at(iter + 1, i));
+        //        std::cout << hessenberg_iter->at(iter + 1, i) << std::endl;
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
@@ -519,7 +553,8 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
             matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
             matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
-            const matrix::Dense<ValueType> *b_norm, size_type iter,
+            const matrix::Dense<ValueType> *b_norm,
+            matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
             Array<size_type> *final_iter_nums,
             const Array<stopping_status> *stop_status)
 {
@@ -537,10 +572,11 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
     //    finish_arnoldi(next_krylov_basis, krylov_bases, hessenberg_iter, iter,
     //                   stop_status->get_const_data());
     //    finish_arnoldi_reorth(next_krylov_basis, krylov_bases,
-    //    hessenberg_iter, iter,
+    //                          hessenberg_iter, arnoldi_norm, iter,
     //                          stop_status->get_const_data());
     finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
-                       buffer_iter, iter, stop_status->get_const_data());
+                       buffer_iter, arnoldi_norm, iter,
+                       stop_status->get_const_data());
     givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
                     iter, stop_status->get_const_data());
     calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,

From 0bfb872b5b9a04e4205a449e0020258f5aea2d3b Mon Sep 17 00:00:00 2001
From: "aliaga@uji.es" <aliaga@tintorrum.local>
Date: Wed, 29 Apr 2020 16:43:28 +0200
Subject: [PATCH 0335/1008] Inclusion of the cuda executor in the repository:  
   * For CGS, a loop of kernels is used instead of a kernel with a loop.     *
 The test files are not repaired yet.

---
 common/components/atomic.hpp.inc          |   2 +
 common/solver/gmres_mixed_kernels.hpp.inc | 597 +++++++++++++++++++
 core/solver/gmres_mixed.cpp               |   6 +-
 core/solver/gmres_mixed_kernels.hpp       |  29 +-
 cuda/solver/gmres_mixed_kernels.cu        | 688 ++++++++++++++++++++++
 cuda/test/solver/CMakeLists.txt           |   2 +-
 omp/solver/gmres_mixed_kernels.cpp        |  21 +-
 reference/solver/gmres_mixed_kernels.cpp  |  15 +-
 8 files changed, 1336 insertions(+), 24 deletions(-)
 create mode 100644 common/solver/gmres_mixed_kernels.hpp.inc
 create mode 100644 cuda/solver/gmres_mixed_kernels.cu

diff --git a/common/components/atomic.hpp.inc b/common/components/atomic.hpp.inc
index 40f87255284..f1b3f7613f2 100644
--- a/common/components/atomic.hpp.inc
+++ b/common/components/atomic.hpp.inc
@@ -81,6 +81,7 @@ __forceinline__ __device__ ResultType reinterpret(ValueType val)
 
 // Support 64-bit ATOMIC_ADD
 GKO_BIND_ATOMIC_HELPER_STRUCTURE(unsigned long long int);
+// GKO_BIND_ATOMIC_HELPER_STRUCTURE(size_type);
 // Support 32-bit ATOMIC_ADD
 GKO_BIND_ATOMIC_HELPER_STRUCTURE(unsigned int);
 
@@ -115,6 +116,7 @@ GKO_BIND_ATOMIC_ADD(unsigned int);
 GKO_BIND_ATOMIC_ADD(unsigned long long int);
 GKO_BIND_ATOMIC_ADD(float);
 
+// GKO_BIND_ATOMIC_ADD(size_type);
 
 #if !defined(__HIPCC__) || \
     (defined(__HIP_DEVICE_COMPILE__) && GINKGO_HIP_PLATFORM_NVCC)
diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
new file mode 100644
index 00000000000..4b9a8fad27a
--- /dev/null
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -0,0 +1,597 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+// Must be called with at least `max(stride_b * num_rows, krylov_dim *
+// num_cols)` threads in total.
+template <size_type block_size, typename ValueType>
+__global__ __launch_bounds__(block_size) void initialize_1_kernel(
+    size_type num_rows, size_type num_cols, size_type krylov_dim,
+    const ValueType *__restrict__ b, size_type stride_b,
+    ValueType *__restrict__ residual, size_type stride_residual,
+    ValueType *__restrict__ givens_sin, size_type stride_sin,
+    ValueType *__restrict__ givens_cos, size_type stride_cos,
+    stopping_status *__restrict__ stop_status)
+{
+    const auto global_id = thread::get_thread_id_flat();
+
+    const auto row_idx = global_id / stride_b;
+    const auto col_idx = global_id % stride_b;
+
+    if (global_id < num_cols) {
+        stop_status[global_id].reset();
+    }
+
+    if (row_idx < num_rows && col_idx < num_cols) {
+        residual[row_idx * stride_residual + col_idx] =
+            b[row_idx * stride_b + col_idx];
+    }
+
+    if (global_id < krylov_dim * num_cols) {
+        const auto row_givens = global_id / num_cols;
+        const auto col_givens = global_id % num_cols;
+
+        givens_sin[row_givens * stride_sin + col_givens] = zero<ValueType>();
+        givens_cos[row_givens * stride_cos + col_givens] = zero<ValueType>();
+    }
+}
+
+
+// Must be called with at least `num_rows * stride_krylov` threads in total.
+template <size_type block_size, typename ValueType,
+          typename ValueTypeKrylovBases>
+__global__ __launch_bounds__(block_size) void initialize_2_1_kernel(
+    size_type num_rows, size_type num_rhs, size_type krylov_dim,
+    ValueTypeKrylovBases *__restrict__ krylov_bases, size_type stride_krylov,
+    ValueType *__restrict__ residual_norm_collection,
+    size_type stride_residual_nc)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row_idx = global_id / stride_krylov;
+    const auto col_idx = global_id % stride_krylov;
+
+    if (row_idx < num_rows && col_idx < (krylov_dim + 1) * num_rhs) {
+        krylov_bases[row_idx * stride_krylov + col_idx] = zero<ValueType>();
+    }
+
+    if (row_idx < krylov_dim + 1 && col_idx < num_rhs) {
+        residual_norm_collection[row_idx * stride_residual_nc + col_idx] =
+            zero<ValueType>();
+    }
+}
+
+
+// Must be called with at least `num_rows * num_rhs` threads in total.
+template <size_type block_size, typename ValueType,
+          typename ValueTypeKrylovBases>
+__global__ __launch_bounds__(block_size) void initialize_2_2_kernel(
+    size_type num_rows, size_type num_rhs,
+    const ValueType *__restrict__ residual, size_type stride_residual,
+    const ValueType *__restrict__ residual_norm,
+    ValueType *__restrict__ residual_norm_collection,
+    ValueTypeKrylovBases *__restrict__ krylov_bases, size_type stride_krylov,
+    ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
+    size_type *__restrict__ final_iter_nums)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row_idx = global_id / num_rhs;
+    const auto col_idx = global_id % num_rhs;
+
+    if (global_id < num_rhs) {
+        residual_norm_collection[global_id] = residual_norm[global_id];
+        final_iter_nums[global_id] = 0;
+    }
+
+    if (row_idx < num_rows && col_idx < num_rhs) {
+        auto value = residual[row_idx * stride_residual + col_idx] /
+                     residual_norm[col_idx];
+        krylov_bases[row_idx * stride_krylov + col_idx] = value;
+        next_krylov_basis[row_idx * stride_next_krylov + col_idx] = value;
+    }
+}
+
+
+__global__
+    __launch_bounds__(default_block_size) void increase_final_iteration_numbers_kernel(
+        size_type *__restrict__ final_iter_nums,
+        const stopping_status *__restrict__ stop_status, size_type total_number)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    if (global_id < total_number) {
+        final_iter_nums[global_id] += !stop_status[global_id].has_stopped();
+    }
+}
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+__global__ __launch_bounds__(default_dot_size) void multidot_kernel(
+    size_type k, size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov,
+    const ValueTypeKrylovBases *__restrict__ krylov_bases,
+    size_type stride_krylov, ValueType *__restrict__ hessenberg_iter,
+    size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto tidx = threadIdx.x;
+    const auto tidy = threadIdx.y;
+    const auto col_idx = blockIdx.x * default_dot_dim + tidx;
+    const auto num = ceildiv(num_rows, gridDim.y);
+    const auto start_row = blockIdx.y * num;
+    const auto end_row =
+        ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+    __shared__
+        UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
+            reduction_helper_array;
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
+
+    ValueType local_res = zero<ValueType>();
+    const auto krylov_col = k * num_cols + col_idx;
+    if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+        for (size_type i = start_row + tidy; i < end_row;
+             i += default_dot_dim) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            const auto krylov_idx = i * stride_krylov + krylov_col;
+            local_res +=
+                next_krylov_basis[next_krylov_idx] * krylov_bases[krylov_idx];
+        }
+    }
+    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
+    __syncthreads();
+    local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
+    const auto tile_block =
+        group::tiled_partition<default_dot_dim>(group::this_thread_block());
+    const auto sum =
+        reduce(tile_block, local_res,
+               [](const ValueType &a, const ValueType &b) { return a + b; });
+    const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
+    if (tidx == 0 && new_col_idx < num_cols &&
+        !stop_status[new_col_idx].has_stopped()) {
+        const auto hessenberg_idx = k * stride_hessenberg + new_col_idx;
+        atomic_add(hessenberg_iter + hessenberg_idx, sum);
+    }
+}
+
+
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__ __launch_bounds__(block_size) void update_next_krylov_kernel(
+    size_type k, size_type num_rows, size_type num_cols,
+    ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
+    const ValueTypeKrylovBases *__restrict__ krylov_bases,
+    size_type stride_krylov, const ValueType *__restrict__ hessenberg_iter,
+    size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row_idx = global_id / stride_next_krylov;
+    const auto col_idx = global_id % stride_next_krylov;
+
+    if (row_idx < num_rows && col_idx < num_cols &&
+        !stop_status[col_idx].has_stopped()) {
+        const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+        const auto krylov_idx =
+            row_idx * stride_krylov + k * num_cols + col_idx;
+        const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+
+        next_krylov_basis[next_krylov_idx] -=
+            hessenberg_iter[hessenberg_idx] * krylov_bases[krylov_idx];
+    }
+}
+
+
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__
+    __launch_bounds__(block_size) void update_next_krylov_kernel_and_check(
+        size_type k, size_type num_rows, size_type num_cols,
+        ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
+        const ValueTypeKrylovBases *__restrict__ krylov_bases,
+        size_type stride_krylov, const ValueType *__restrict__ hessenberg_iter,
+        size_type stride_hessenberg, const ValueType *__restrict__ arnoldi_norm,
+        size_type stride_norm, const stopping_status *__restrict__ stop_status,
+        stopping_status *__restrict__ reorth_status,
+        size_type *__restrict__ num_reorth)
+
+//    stopping_status *__restrict__ reorth_status, size_type *num_reorth)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row_idx = global_id / stride_next_krylov;
+    const auto col_idx = global_id % stride_next_krylov;
+
+    if (row_idx < num_rows && col_idx < num_cols &&
+        !stop_status[col_idx].has_stopped()) {
+        const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+        const auto krylov_idx =
+            row_idx * stride_krylov + k * num_cols + col_idx;
+        const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+        //        const auto arnoldi_idx = k * stride_norm + col_idx;
+        const auto arnoldi_idx = col_idx;
+        const auto reorth_idx = col_idx;
+
+        next_krylov_basis[next_krylov_idx] -=
+            hessenberg_iter[hessenberg_idx] * krylov_bases[krylov_idx];
+        if (row_idx == 0) {
+            if ((hessenberg_iter[hessenberg_idx] *
+                 hessenberg_iter[hessenberg_idx]) >
+                (0.99 * sqrt(arnoldi_norm[arnoldi_idx]))) {
+                reorth_status[reorth_idx].reset();
+                //                reorth_status[k].reset();
+                printf("K = %d , HI = %f , AN = %f\n", k,
+                       hessenberg_iter[hessenberg_idx],
+                       0.99 * sqrt(arnoldi_norm[arnoldi_idx]));
+                //                num_reorth[0] = 1;
+                //                if (!reorth_status[arnoldi_idx].has_stopped())
+                //                    printf ("OK in %d\n", arnoldi_idx);
+                //                else
+                //                    printf ("BAD in %d\n", arnoldi_idx);
+                atomic_add(num_reorth, (size_type)1);
+            } else {
+                reorth_status[reorth_idx].stop(1);
+                //                if (reorth_status[arnoldi_idx].has_stopped())
+                //                    printf ("OK in %d\n", arnoldi_idx);
+                //                else
+                //                    printf ("BAD in %d\n", arnoldi_idx);
+                //                reorth_status[k].stop(0);
+            }
+        }
+    }
+}
+
+
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
+    size_type k, size_type num_rows, size_type num_cols,
+    ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
+    const ValueTypeKrylovBases *__restrict__ krylov_bases,
+    size_type stride_krylov, ValueType *__restrict__ hessenberg_iter,
+    size_type stride_hessenberg, const ValueType *__restrict__ buffer_iter,
+    size_type stride_buffer, const stopping_status *__restrict__ stop_status,
+    const stopping_status *__restrict__ reorth_status)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row_idx = global_id / stride_next_krylov;
+    const auto col_idx = global_id % stride_next_krylov;
+
+    if (row_idx < num_rows && col_idx < num_cols &&
+        !stop_status[col_idx].has_stopped() &&
+        reorth_status[col_idx].has_stopped()) {
+        const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+        const auto krylov_idx =
+            row_idx * stride_krylov + k * num_cols + col_idx;
+        const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+        const auto buffer_idx = k * stride_buffer + col_idx;
+        next_krylov_basis[next_krylov_idx] -=
+            buffer_iter[buffer_idx] * krylov_bases[krylov_idx];
+        if ((row_idx == 0) && reorth_status[col_idx].has_stopped()) {
+            printf(" Add(%d) = %10.5e\n", k, buffer_iter[buffer_idx]);
+            hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
+        }
+    }
+}
+
+
+// Must be called with at least `num_cols` blocks, each with `block_size`
+// threads. `block_size` must be a power of 2.
+template <int block_size, typename ValueType>
+__global__ __launch_bounds__(block_size) void check_arnoldi_norms(
+    const ValueType *__restrict__ arnoldi_norm0, size_type stride_norm0,
+    const ValueType *__restrict__ arnoldi_norm1, size_type stride_norm1,
+    const stopping_status *__restrict__ stop_status,
+    stopping_status *__restrict__ reorth_status,
+    size_type *__restrict__ num_reorth)
+//    stopping_status *__restrict__ reorth_status, size_type *num_reorth)
+//    const ValueType eta, stopping_status *__restrict__ reorth_status)
+{
+    //    const ValueType eta = 1.0 / sqrt(2.0);
+    const ValueType eta = 1.0 / 2.0;
+    const auto global_id = thread::get_thread_id_flat();
+    //    const auto tidx = threadIdx.x;
+    //    const auto col_idx = blockIdx.x;
+
+    if (global_id < stride_norm0 && !stop_status[global_id].has_stopped()) {
+        const ValueType num0 = eta * arnoldi_norm0[global_id];
+        const ValueType num1 = arnoldi_norm1[global_id];
+        //        if (arnoldi_norm1[global_id] < eta * arnoldi_norm0[global_id])
+        //        {
+        printf(" Reorth(%d) = %10.5e,%10.5e)\n", global_id, num0, num1);
+        if (num1 < num0) {
+            reorth_status[global_id].stop(1);
+            atomic_add(num_reorth, (size_type)1);
+            //            num_reorth[0] = 1;
+            //            printf (" Reorth(%10.5e,%10.5e)\n",
+            //                     eta * arnoldi_norm0[global_id],
+            //                     arnoldi_norm1[global_id]);
+        } else {
+            //            num_reorth[0] = 0;
+            reorth_status[global_id].reset();
+        }
+    }
+}
+
+
+// Must be called with at least `num_cols` blocks, each with `block_size`
+// threads. `block_size` must be a power of 2.
+template <int block_size, typename ValueType>
+__global__ __launch_bounds__(block_size) void update_hessenberg_2_kernel(
+    size_type iter, size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov, ValueType *__restrict__ hessenberg_iter,
+    size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto tidx = threadIdx.x;
+    const auto col_idx = blockIdx.x;
+
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+    __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
+
+    if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+        ValueType local_res{};
+        for (size_type i = tidx; i < num_rows; i += block_size) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            const auto next_krylov_value = next_krylov_basis[next_krylov_idx];
+
+            local_res += next_krylov_value * next_krylov_value;
+        }
+
+        reduction_helper[tidx] = local_res;
+
+        // Perform thread block reduction. Result is in reduction_helper[0]
+        reduce(group::this_thread_block(), reduction_helper,
+               [](const ValueType &a, const ValueType &b) { return a + b; });
+
+        if (tidx == 0) {
+            hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] =
+                sqrt(reduction_helper[0]);
+        }
+    }
+}
+
+
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__ __launch_bounds__(block_size) void update_krylov_next_krylov_kernel(
+    size_type iter, size_type num_rows, size_type num_cols,
+    ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
+    ValueTypeKrylovBases *__restrict__ krylov_bases, size_type stride_krylov,
+    const ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row_idx = global_id / stride_next_krylov;
+    const auto col_idx = global_id % stride_next_krylov;
+    const auto hessenberg =
+        hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx];
+
+    if (row_idx < num_rows && col_idx < num_cols &&
+        !stop_status[col_idx].has_stopped()) {
+        const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+        const auto krylov_idx =
+            row_idx * stride_krylov + num_cols * (iter + 1) + col_idx;
+
+        const auto next_krylov_value =
+            next_krylov_basis[next_krylov_idx] / hessenberg;
+
+        next_krylov_basis[next_krylov_idx] = next_krylov_value;
+        krylov_bases[krylov_idx] = next_krylov_value;
+    }
+}
+
+
+template <typename ValueType>
+__device__ void calculate_sin_and_cos_kernel(
+    size_type col_idx, size_type num_cols, size_type iter,
+    const ValueType *hessenberg_iter, size_type stride_hessenberg,
+    ValueType *givens_sin, size_type stride_sin, ValueType *givens_cos,
+    size_type stride_cos)
+{
+    if (hessenberg_iter[iter * stride_hessenberg + col_idx] ==
+        zero<ValueType>()) {
+        givens_cos[iter * stride_cos + col_idx] = zero<ValueType>();
+        givens_sin[iter * stride_sin + col_idx] = one<ValueType>();
+    } else {
+        auto hypotenuse =
+            sqrt(hessenberg_iter[iter * stride_hessenberg + col_idx] *
+                     hessenberg_iter[iter * stride_hessenberg + col_idx] +
+                 hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] *
+                     hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx]);
+        givens_cos[iter * stride_cos + col_idx] =
+            abs(hessenberg_iter[iter * stride_hessenberg + col_idx]) /
+            hypotenuse;
+        givens_sin[iter * stride_sin + col_idx] =
+            givens_cos[iter * stride_cos + col_idx] *
+            hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] /
+            hessenberg_iter[iter * stride_hessenberg + col_idx];
+    }
+}
+
+
+template <typename ValueType>
+__device__ void calculate_residual_norm_kernel(
+    size_type col_idx, size_type num_cols, size_type iter,
+    const ValueType *givens_sin, size_type stride_sin,
+    const ValueType *givens_cos, size_type stride_cos, ValueType *residual_norm,
+    ValueType *residual_norm_collection,
+    size_type stride_residual_norm_collection, const ValueType *b_norm)
+{
+    residual_norm_collection[(iter + 1) * stride_residual_norm_collection +
+                             col_idx] =
+        -givens_sin[iter * stride_sin + col_idx] *
+        residual_norm_collection[iter * stride_residual_norm_collection +
+                                 col_idx];
+    residual_norm_collection[iter * stride_residual_norm_collection + col_idx] =
+        givens_cos[iter * stride_cos + col_idx] *
+        residual_norm_collection[iter * stride_residual_norm_collection +
+                                 col_idx];
+    residual_norm[col_idx] =
+        abs(residual_norm_collection[(iter + 1) *
+                                         stride_residual_norm_collection +
+                                     col_idx]) /
+        b_norm[col_idx];
+}
+
+
+// Must be called with at least `num_cols` threads in total.
+template <size_type block_size, typename ValueType>
+__global__ __launch_bounds__(block_size) void givens_rotation_kernel(
+    size_type num_rows, size_type num_cols, size_type iter,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    ValueType *__restrict__ givens_sin, size_type stride_sin,
+    ValueType *__restrict__ givens_cos, size_type stride_cos,
+    ValueType *__restrict__ residual_norm,
+    ValueType *__restrict__ residual_norm_collection,
+    size_type stride_residual_norm_collection,
+    const ValueType *__restrict__ b_norm,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto col_idx = thread::get_thread_id_flat();
+
+    if (col_idx >= num_cols || stop_status[col_idx].has_stopped()) {
+        return;
+    }
+
+    const auto current_thread_block = group::this_thread_block();
+
+    for (size_type i = 0; i < iter; ++i) {
+        const auto tmp =
+            givens_cos[i * stride_cos + col_idx] *
+                hessenberg_iter[i * stride_hessenberg + col_idx] +
+            givens_sin[i * stride_sin + col_idx] *
+                hessenberg_iter[(i + 1) * stride_hessenberg + col_idx];
+        current_thread_block.sync();
+        hessenberg_iter[(i + 1) * stride_hessenberg + col_idx] =
+            givens_cos[i * stride_cos + col_idx] *
+                hessenberg_iter[(i + 1) * stride_hessenberg + col_idx] -
+            givens_sin[i * stride_sin + col_idx] *
+                hessenberg_iter[i * stride_hessenberg + col_idx];
+        hessenberg_iter[i * stride_hessenberg + col_idx] = tmp;
+        current_thread_block.sync();
+    }
+    // for j in 1:iter - 1
+    //     temp             =  cos(j)*hessenberg(j) +
+    //                         sin(j)*hessenberg(j+1)
+    //     hessenberg(j+1)  = -sin(j)*hessenberg(j) +
+    //                         cos(j)*hessenberg(j+1)
+    //     hessenberg(j)    =  temp;
+    // end
+
+    calculate_sin_and_cos_kernel(col_idx, num_cols, iter, hessenberg_iter,
+                                 stride_hessenberg, givens_sin, stride_sin,
+                                 givens_cos, stride_cos);
+    // Calculate sin and cos
+
+    hessenberg_iter[iter * stride_hessenberg + col_idx] =
+        givens_cos[iter * stride_cos + col_idx] *
+            hessenberg_iter[iter * stride_hessenberg + col_idx] +
+        givens_sin[iter * stride_sin + col_idx] *
+            hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx];
+    hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] =
+        zero<ValueType>();
+    // hessenberg(iter)   = cos(iter)*hessenberg(iter) +
+    //                      sin(iter)*hessenberg(iter)
+    // hessenberg(iter+1) = 0
+
+    calculate_residual_norm_kernel(col_idx, num_cols, iter, givens_sin,
+                                   stride_sin, givens_cos, stride_cos,
+                                   residual_norm, residual_norm_collection,
+                                   stride_residual_norm_collection, b_norm);
+    // Calculate residual norm
+}
+
+
+// Must be called with at least `num_rhs` threads in total.
+template <size_type block_size, typename ValueType>
+__global__ __launch_bounds__(block_size) void solve_upper_triangular_kernel(
+    size_type num_cols, size_type num_rhs,
+    const ValueType *__restrict__ residual_norm_collection,
+    size_type stride_residual_norm_collection,
+    const ValueType *__restrict__ hessenberg, size_type stride_hessenberg,
+    ValueType *__restrict__ y, size_type stride_y,
+    const size_type *__restrict__ final_iter_nums)
+{
+    const auto col_idx = thread::get_thread_id_flat();
+
+    if (col_idx >= num_rhs) {
+        return;
+    }
+
+    for (int i = final_iter_nums[col_idx] - 1; i >= 0; --i) {
+        auto temp =
+            residual_norm_collection[i * stride_residual_norm_collection +
+                                     col_idx];
+        for (size_type j = i + 1; j < final_iter_nums[col_idx]; ++j) {
+            temp -= hessenberg[i * stride_hessenberg + j * num_rhs + col_idx] *
+                    y[j * stride_y + col_idx];
+        }
+
+        y[i * stride_y + col_idx] =
+            temp / hessenberg[i * stride_hessenberg + i * num_rhs + col_idx];
+    }
+    // Solve upper triangular.
+    // y = hessenberg \ residual_norm_collection
+}
+
+
+// Must be called with at least `stride_preconditioner * num_rows` threads in
+// total.
+template <size_type block_size, typename ValueType,
+          typename ValueTypeKrylovBases>
+__global__ __launch_bounds__(block_size) void calculate_Qy_kernel(
+    size_type num_rows, size_type num_cols, size_type num_rhs,
+    const ValueTypeKrylovBases *__restrict__ krylov_bases,
+    size_type stride_krylov, const ValueType *__restrict__ y,
+    size_type stride_y, ValueType *__restrict__ before_preconditioner,
+    size_type stride_preconditioner,
+    const size_type *__restrict__ final_iter_nums)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row_id = global_id / stride_preconditioner;
+    const auto col_id = global_id % stride_preconditioner;
+
+    if (row_id < num_rows && col_id < num_cols) {
+        before_preconditioner[global_id] = zero<ValueType>();
+        for (size_type j = 0; j < final_iter_nums[col_id]; ++j) {
+            before_preconditioner[global_id] +=
+                krylov_bases[row_id * stride_krylov + j * num_rhs + col_id] *
+                y[j * stride_y + col_id];
+        }
+    }
+}
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index d0d8d7a9e3a..5741d513f4f 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -117,6 +117,10 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     bool one_changed{};
     Array<stopping_status> stop_status(this->get_executor(),
                                        dense_b->get_size()[1]);
+    Array<stopping_status> reorth_status(this->get_executor(),
+                                         dense_b->get_size()[1]);
+    //                                         krylov_dim_mixed_ + 1);
+    Array<size_type> num_reorth(this->get_executor(), 1);
 
     // Initialization
     exec->run(gmres_mixed::make_initialize_1(
@@ -217,7 +221,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
             residual_norm.get(), residual_norm_collection.get(),
             krylov_bases.get(), hessenberg_iter.get(), buffer_iter.get(),
             b_norm.get(), arnoldi_norm.get(), restart_iter, &final_iter_nums,
-            &stop_status));
+            &stop_status, &reorth_status, &num_reorth));
         // for i in 0:restart_iter
         //     hessenberg(restart_iter, i) = next_krylov_basis' *
         //     krylov_bases(:, i) next_krylov_basis  -= hessenberg(restart_iter,
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index fbfc164ddbf..a3cf99b216f 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -65,20 +65,21 @@ namespace gmres_mixed {
 
 
 // #define GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(_type1, _type2, _bool1, _bool2)
-#define GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(_type1, _type2)        \
-    void step_1(std::shared_ptr<const DefaultExecutor> exec,         \
-                matrix::Dense<_type1> *next_krylov_basis,            \
-                matrix::Dense<_type1> *givens_sin,                   \
-                matrix::Dense<_type1> *givens_cos,                   \
-                matrix::Dense<_type1> *residual_norm,                \
-                matrix::Dense<_type1> *residual_norm_collection,     \
-                matrix::Dense<_type2> *krylov_bases,                 \
-                matrix::Dense<_type1> *hessenberg_iter,              \
-                matrix::Dense<_type1> *buffer_iter,                  \
-                const matrix::Dense<_type1> *b_norm,                 \
-                matrix::Dense<_type1> *arnoldi_norm, size_type iter, \
-                Array<size_type> *final_iter_nums,                   \
-                const Array<stopping_status> *stop_status)
+#define GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(_type1, _type2)                 \
+    void step_1(                                                              \
+        std::shared_ptr<const DefaultExecutor> exec,                          \
+        matrix::Dense<_type1> *next_krylov_basis,                             \
+        matrix::Dense<_type1> *givens_sin, matrix::Dense<_type1> *givens_cos, \
+        matrix::Dense<_type1> *residual_norm,                                 \
+        matrix::Dense<_type1> *residual_norm_collection,                      \
+        matrix::Dense<_type2> *krylov_bases,                                  \
+        matrix::Dense<_type1> *hessenberg_iter,                               \
+        matrix::Dense<_type1> *buffer_iter,                                   \
+        const matrix::Dense<_type1> *b_norm,                                  \
+        matrix::Dense<_type1> *arnoldi_norm, size_type iter,                  \
+        Array<size_type> *final_iter_nums,                                    \
+        const Array<stopping_status> *stop_status,                            \
+        Array<stopping_status> *reorth_status, Array<size_type> *num_reorth)
 
 
 #define GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(_type1, _type2)          \
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
new file mode 100644
index 00000000000..bc168c766b0
--- /dev/null
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -0,0 +1,688 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/gmres_mixed_kernels.hpp"
+
+
+#include <algorithm>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/range_accessors.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/components/fill_array.hpp"
+#include "cuda/base/config.hpp"
+#include "cuda/base/cublas_bindings.hpp"
+#include "cuda/base/math.hpp"
+#include "cuda/base/types.hpp"
+#include "cuda/components/atomic.cuh"
+#include "cuda/components/cooperative_groups.cuh"
+#include "cuda/components/reduction.cuh"
+#include "cuda/components/thread_ids.cuh"
+#include "cuda/components/uninitialized_array.hpp"
+
+
+#include <iostream>
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+/**
+ * @brief The GMRES_MIXED solver namespace.
+ *
+ * @ingroup gmres_mixed
+ */
+namespace gmres_mixed {
+
+
+constexpr int default_block_size = 512;
+// default_dot_dim can not be 64 in hip because 64 * 64 exceeds their max block
+// size limit.
+constexpr int default_dot_dim = 32;
+constexpr int default_dot_size = default_dot_dim * default_dot_dim;
+
+
+#include "common/solver/gmres_mixed_kernels.hpp.inc"
+
+
+template <typename ValueType>
+void initialize_1(std::shared_ptr<const CudaExecutor> exec,
+                  const matrix::Dense<ValueType> *b,
+                  matrix::Dense<ValueType> *b_norm,
+                  matrix::Dense<ValueType> *residual,
+                  matrix::Dense<ValueType> *givens_sin,
+                  matrix::Dense<ValueType> *givens_cos,
+                  Array<stopping_status> *stop_status, size_type krylov_dim)
+{
+    const auto num_threads = std::max(b->get_size()[0] * b->get_stride(),
+                                      krylov_dim * b->get_size()[1]);
+    const dim3 grid_dim(ceildiv(num_threads, default_block_size), 1, 1);
+    const dim3 block_dim(default_block_size, 1, 1);
+    constexpr auto block_size = default_block_size;
+
+    b->compute_norm2(b_norm);
+    initialize_1_kernel<block_size><<<grid_dim, block_dim>>>(
+        b->get_size()[0], b->get_size()[1], krylov_dim,
+        as_cuda_type(b->get_const_values()), b->get_stride(),
+        as_cuda_type(residual->get_values()), residual->get_stride(),
+        as_cuda_type(givens_sin->get_values()), givens_sin->get_stride(),
+        as_cuda_type(givens_cos->get_values()), givens_cos->get_stride(),
+        as_cuda_type(stop_status->get_data()));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void initialize_2(std::shared_ptr<const CudaExecutor> exec,
+                  const matrix::Dense<ValueType> *residual,
+                  matrix::Dense<ValueType> *residual_norm,
+                  matrix::Dense<ValueType> *residual_norm_collection,
+                  matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                  matrix::Dense<ValueType> *next_krylov_basis,
+                  Array<size_type> *final_iter_nums, size_type krylov_dim)
+{
+    const auto num_rows = residual->get_size()[0];
+    const auto num_rhs = residual->get_size()[1];
+    const dim3 grid_dim_1(
+        ceildiv(num_rows * krylov_bases->get_stride(), default_block_size), 1,
+        1);
+    const dim3 block_dim(default_block_size, 1, 1);
+    constexpr auto block_size = default_block_size;
+
+    initialize_2_1_kernel<block_size><<<grid_dim_1, block_dim>>>(
+        residual->get_size()[0], residual->get_size()[1], krylov_dim,
+        as_cuda_type(krylov_bases->get_values()), krylov_bases->get_stride(),
+        as_cuda_type(residual_norm_collection->get_values()),
+        residual_norm_collection->get_stride());
+    residual->compute_norm2(residual_norm);
+
+    const dim3 grid_dim_2(ceildiv(num_rows * num_rhs, default_block_size), 1,
+                          1);
+    initialize_2_2_kernel<block_size><<<grid_dim_2, block_dim>>>(
+        residual->get_size()[0], residual->get_size()[1],
+        as_cuda_type(residual->get_const_values()), residual->get_stride(),
+        as_cuda_type(residual_norm->get_const_values()),
+        as_cuda_type(residual_norm_collection->get_values()),
+        as_cuda_type(krylov_bases->get_values()), krylov_bases->get_stride(),
+        as_cuda_type(next_krylov_basis->get_values()),
+        next_krylov_basis->get_stride(),
+        as_cuda_type(final_iter_nums->get_data()));
+}
+
+GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void finish_arnoldi(std::shared_ptr<const CudaExecutor> exec,
+                    matrix::Dense<ValueType> *next_krylov_basis,
+                    matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                    matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
+                    const stopping_status *stop_status)
+{
+    const auto stride_next_krylov = next_krylov_basis->get_stride();
+    const auto stride_krylov = krylov_bases->get_stride();
+    const auto stride_hessenberg = hessenberg_iter->get_stride();
+    const auto dim_size = next_krylov_basis->get_size();
+    auto cublas_handle = exec->get_cublas_handle();
+    const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
+                         exec->get_num_multiprocessor() * 2);
+    const dim3 block_size(default_dot_dim, default_dot_dim);
+    for (size_type k = 0; k < iter + 1; ++k) {
+        components::fill_array(
+            exec, hessenberg_iter->get_values() + k * stride_hessenberg,
+            dim_size[1], zero<ValueType>());
+        multidot_kernel<<<grid_size, block_size>>>(
+            k, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov, as_cuda_type(krylov_bases->get_const_values()),
+            stride_krylov, as_cuda_type(hessenberg_iter->get_values()),
+            stride_hessenberg, as_cuda_type(stop_status));
+        update_next_krylov_kernel<default_block_size>
+            <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+               default_block_size>>>(
+                k, dim_size[0], dim_size[1],
+                as_cuda_type(next_krylov_basis->get_values()),
+                stride_next_krylov,
+                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                as_cuda_type(hessenberg_iter->get_const_values()),
+                stride_hessenberg, as_cuda_type(stop_status));
+    }
+    // for i in 1:iter
+    //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+    //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+    // end
+
+
+    update_hessenberg_2_kernel<default_block_size>
+        <<<dim_size[1], default_block_size>>>(
+            iter, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov, as_cuda_type(hessenberg_iter->get_values()),
+            stride_hessenberg, as_cuda_type(stop_status));
+
+    update_krylov_next_krylov_kernel<default_block_size>
+        <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+           default_block_size>>>(
+            iter, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
+            as_cuda_type(krylov_bases->get_values()), stride_krylov,
+            as_cuda_type(hessenberg_iter->get_const_values()),
+            stride_hessenberg, as_cuda_type(stop_status));
+    // next_krylov_basis /= hessenberg(iter, iter + 1)
+    // krylov_bases(:, iter + 1) = next_krylov_basis
+    // End of arnoldi
+}
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
+                           matrix::Dense<ValueType> *next_krylov_basis,
+                           matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                           matrix::Dense<ValueType> *hessenberg_iter,
+                           matrix::Dense<ValueType> *buffer_iter,
+                           matrix::Dense<ValueType> *arnoldi_norm,
+                           size_type iter, const stopping_status *stop_status,
+                           stopping_status *reorth_status,
+                           Array<size_type> *num_reorth)
+{
+    const auto stride_next_krylov = next_krylov_basis->get_stride();
+    const auto stride_krylov = krylov_bases->get_stride();
+    const auto stride_hessenberg = hessenberg_iter->get_stride();
+    const auto stride_buffer = buffer_iter->get_stride();
+    const auto stride_arnoldi = arnoldi_norm->get_stride();
+    const auto dim_size = next_krylov_basis->get_size();
+    auto cublas_handle = exec->get_cublas_handle();
+    const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
+                         exec->get_num_multiprocessor() * 2);
+    const dim3 block_size(default_dot_dim, default_dot_dim);
+    size_type numReorth;
+    //    size_type num_reorth;
+    components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
+                           zero<ValueType>());
+    multidot_kernel<<<grid_size, block_size>>>(
+        0, dim_size[0], dim_size[1],
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
+    // nrm = norm(next_krylov_basis
+    for (size_type k = 0; k < iter + 1; ++k) {
+        components::fill_array(
+            exec, hessenberg_iter->get_values() + k * stride_hessenberg,
+            dim_size[1], zero<ValueType>());
+        multidot_kernel<<<grid_size, block_size>>>(
+            k, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov, as_cuda_type(krylov_bases->get_const_values()),
+            stride_krylov, as_cuda_type(hessenberg_iter->get_values()),
+            stride_hessenberg, as_cuda_type(stop_status));
+        /*
+                update_next_krylov_kernel<default_block_size>
+                    <<<ceildiv(dim_size[0] * stride_next_krylov,
+           default_block_size), default_block_size>>>( k, dim_size[0],
+           dim_size[1], as_cuda_type(next_krylov_basis->get_values()),
+                        stride_next_krylov,
+                        as_cuda_type(krylov_bases->get_const_values()),
+           stride_krylov, as_cuda_type(hessenberg_iter->get_const_values()),
+                        stride_hessenberg, as_cuda_type(stop_status));
+        */
+        //        num_reorth = 0;
+        components::fill_array(exec, num_reorth->get_data(), 1,
+                               zero<size_type>());
+        update_next_krylov_kernel_and_check<default_block_size>
+            <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+               default_block_size>>>(
+                k, dim_size[0], dim_size[1],
+                as_cuda_type(next_krylov_basis->get_values()),
+                stride_next_krylov,
+                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                as_cuda_type(hessenberg_iter->get_const_values()),
+                stride_hessenberg, as_cuda_type(arnoldi_norm->get_values()), 0,
+                //                as_cuda_type(stop_status),
+                //                as_cuda_type(reorth_status));
+                as_cuda_type(stop_status), as_cuda_type(reorth_status),
+                as_cuda_type(num_reorth->get_data()));
+        //                as_cuda_type(&num_reorth));
+        numReorth = 0;
+        exec->get_master()->copy_from(exec.get(), 1,
+                                      num_reorth->get_const_data(), &numReorth);
+        std::cout << "K = " << k << " , numReorth => " << numReorth
+                  << std::endl;
+        //        std::cout << "NREORTH = " << num_reorth << std::endl;
+        //        if (num_reorth > 0) {
+        //        if (2 > 1) {
+        if (numReorth > 0) {
+            std::cout << "REORTHOGONALIZATION" << std::endl;
+            components::fill_array(
+                exec, buffer_iter->get_values() + k * stride_buffer,
+                dim_size[1], zero<ValueType>());
+            multidot_kernel<<<grid_size, block_size>>>(
+                k, dim_size[0], dim_size[1],
+                as_cuda_type(next_krylov_basis->get_const_values()),
+                stride_next_krylov,
+                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                as_cuda_type(buffer_iter->get_values()), stride_buffer,
+                as_cuda_type(stop_status));
+            /**/
+            update_next_krylov_kernel_and_add<default_block_size><<<
+                ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+                default_block_size>>>(
+                k, dim_size[0], dim_size[1],
+                as_cuda_type(next_krylov_basis->get_values()),
+                stride_next_krylov,
+                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+                as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
+                as_cuda_type(stop_status), as_cuda_type(reorth_status));
+            /**/
+        }
+    }
+    // for i in 1:iter
+    //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+    //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+    //     if (hessenberg(iter, i)*hessenberg(iter, i)>nrm*0.99)
+    //         buffer = next_krylov_basis' * krylov_bases(:, i)
+    //         next_krylov_basis  -= buffer * krylov_bases(:, i)
+    //         hessenberg(iter, i) += buffer;
+    //     end
+    // end
+
+
+    update_hessenberg_2_kernel<default_block_size>
+        <<<dim_size[1], default_block_size>>>(
+            iter, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov, as_cuda_type(hessenberg_iter->get_values()),
+            stride_hessenberg, as_cuda_type(stop_status));
+
+    update_krylov_next_krylov_kernel<default_block_size>
+        <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+           default_block_size>>>(
+            iter, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
+            as_cuda_type(krylov_bases->get_values()), stride_krylov,
+            as_cuda_type(hessenberg_iter->get_const_values()),
+            stride_hessenberg, as_cuda_type(stop_status));
+    // next_krylov_basis /= hessenberg(iter, iter + 1)
+    // krylov_bases(:, iter + 1) = next_krylov_basis
+    // End of arnoldi
+}
+
+/**/
+template <typename ValueType, typename ValueTypeKrylovBases>
+void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
+                        matrix::Dense<ValueType> *next_krylov_basis,
+                        matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                        matrix::Dense<ValueType> *hessenberg_iter,
+                        matrix::Dense<ValueType> *buffer_iter,
+                        matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+                        const stopping_status *stop_status,
+                        stopping_status *reorth_status,
+                        Array<size_type> *num_reorth)
+{
+    const auto stride_next_krylov = next_krylov_basis->get_stride();
+    const auto stride_krylov = krylov_bases->get_stride();
+    const auto stride_hessenberg = hessenberg_iter->get_stride();
+    const auto stride_buffer = buffer_iter->get_stride();
+    const auto stride_arnoldi = arnoldi_norm->get_stride();
+    const auto dim_size = next_krylov_basis->get_size();
+    auto cublas_handle = exec->get_cublas_handle();
+    const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
+                         exec->get_num_multiprocessor() * 2);
+    const dim3 block_size(default_dot_dim, default_dot_dim);
+    //    size_type num_reorth;
+    size_type numReorth;
+    components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
+                           zero<ValueType>());
+    multidot_kernel<<<grid_size, block_size>>>(
+        0, dim_size[0], dim_size[1],
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
+    // nrmP = norm(next_krylov_basis
+    for (size_type k = 0; k < iter + 1; ++k) {
+        components::fill_array(
+            exec, hessenberg_iter->get_values() + k * stride_hessenberg,
+            dim_size[1], zero<ValueType>());
+        multidot_kernel<<<grid_size, block_size>>>(
+            k, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov, as_cuda_type(krylov_bases->get_const_values()),
+            stride_krylov, as_cuda_type(hessenberg_iter->get_values()),
+            stride_hessenberg, as_cuda_type(stop_status));
+    }
+    // for i in 1:iter
+    //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+    // end
+    for (size_type k = 0; k < iter + 1; ++k) {
+        update_next_krylov_kernel<default_block_size>
+            <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+               default_block_size>>>(
+                k, dim_size[0], dim_size[1],
+                as_cuda_type(next_krylov_basis->get_values()),
+                stride_next_krylov,
+                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                as_cuda_type(hessenberg_iter->get_const_values()),
+                stride_hessenberg, as_cuda_type(stop_status));
+    }
+    // for i in 1:iter
+    //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+    // end
+    components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
+                           dim_size[1], zero<ValueType>());
+    multidot_kernel<<<grid_size, block_size>>>(
+        0, dim_size[0], dim_size[1],
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
+        as_cuda_type(stop_status));
+    // nrmN = norm(next_krylov_basis
+    // const ValueType eta = 1.0 / sqrt(2.0);
+    //    components::fill_array(exec, num_reorth->get_values(), 1,
+    //    zero<size_type>()); components::fill_array(exec,
+    //    num_reorth->get_data(), 1, zero<size_type>()); num_reorth->get_data[0]
+    //    = 0;
+    components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
+    check_arnoldi_norms<default_block_size>
+        <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
+            as_cuda_type(arnoldi_norm->get_const_values()), stride_arnoldi,
+            as_cuda_type(arnoldi_norm->get_const_values() + stride_arnoldi),
+            stride_arnoldi,
+            //            as_cuda_type(stop_status),
+            //            as_cuda_type(reorth_status));
+            as_cuda_type(stop_status), as_cuda_type(reorth_status),
+            as_cuda_type(num_reorth->get_data()));
+    //            as_cuda_type(num_reorth->get_values()));
+    //        eta, as_cuda_type(reorth_status));
+    numReorth = 0;
+    exec->get_master()->copy_from(exec.get(), 1, num_reorth->get_const_data(),
+                                  &numReorth);
+    std::cout << " numReorth => " << numReorth << std::endl;
+    for (size_type l = 1; (l < 3); l++) {
+        //    for (size_type l = 1; (numReorth>0) && (l < 3); l++) {
+        //    for (size_type l = 1; (num_reorth->get_data[0]>0) && (l < 3); l++)
+        //    { for (size_type l = 1; (num_reorth->get_values[0]>0) && (l < 3);
+        //    l++) {
+        //             arnoldi_norm->at(1, i) < arnoldi_norm->at(0, i) && l < 3;
+        //             l++) {
+        for (size_type k = 0; k < iter + 1; ++k) {
+            components::fill_array(
+                exec, buffer_iter->get_values() + k * stride_buffer,
+                dim_size[1], zero<ValueType>());
+            multidot_kernel<<<grid_size, block_size>>>(
+                k, dim_size[0], dim_size[1],
+                as_cuda_type(next_krylov_basis->get_const_values()),
+                stride_next_krylov,
+                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                as_cuda_type(buffer_iter->get_values()), stride_buffer,
+                as_cuda_type(stop_status));
+        }
+        // for i in 1:iter
+        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+        // end
+        for (size_type k = 0; k < iter + 1; ++k) {
+            update_next_krylov_kernel_and_add<default_block_size><<<
+                ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+                default_block_size>>>(
+                k, dim_size[0], dim_size[1],
+                as_cuda_type(next_krylov_basis->get_values()),
+                stride_next_krylov,
+                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+                as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
+                as_cuda_type(stop_status), as_cuda_type(reorth_status));
+        }
+        // for i in 1:iter
+        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+        // end
+        components::fill_array(
+            exec, arnoldi_norm->get_values() + dim_size[1] * (l - 1),
+            dim_size[1], zero<ValueType>());
+        multidot_kernel<<<grid_size, block_size>>>(
+            0, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov,
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov,
+            as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
+            as_cuda_type(stop_status));
+        // nrmN = norm(next_krylov_basis
+        //        std::cout << " XX => " << (l-1) << " - " << (2-l) <<
+        //        std::endl;
+        std::cout << " XX => " << l << " - " << (2 - l) << " - " << (l - 1)
+                  << std::endl;
+        //        num_reorth = 0;
+        //        components::fill_array(exec, num_reorth->get_values(), 1,
+        //        zero<size_type>()); components::fill_array(exec,
+        //        num_reorth->get_data(), 1, zero<size_type>());
+        //        num_reorth->get_data[0] = 0;
+        components::fill_array(exec, num_reorth->get_data(), 1,
+                               zero<size_type>());
+        check_arnoldi_norms<default_block_size>
+            <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
+                as_cuda_type(arnoldi_norm->get_const_values() +
+                             stride_arnoldi * (2 - l)),
+                stride_arnoldi,
+                as_cuda_type(arnoldi_norm->get_const_values() +
+                             stride_arnoldi * (l - 1)),
+                stride_arnoldi,
+                //                as_cuda_type(stop_status),
+                //                as_cuda_type(reorth_status));
+                as_cuda_type(stop_status), as_cuda_type(reorth_status),
+                as_cuda_type(num_reorth->get_data()));
+        //                as_cuda_type(num_reorth->get_values()));
+        //        eta, as_cuda_type(reorth_status));
+        numReorth = 0;
+        exec->get_master()->copy_from(exec.get(), 1,
+                                      num_reorth->get_const_data(), &numReorth);
+        std::cout << " L = " << l << " , numReorth => " << numReorth
+                  << std::endl;
+        //        std::cout << " num_reorth => " << num_reorth << std::endl;
+    }
+
+
+    update_hessenberg_2_kernel<default_block_size>
+        <<<dim_size[1], default_block_size>>>(
+            iter, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov, as_cuda_type(hessenberg_iter->get_values()),
+            stride_hessenberg, as_cuda_type(stop_status));
+    update_krylov_next_krylov_kernel<default_block_size>
+        <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+           default_block_size>>>(
+            iter, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
+            as_cuda_type(krylov_bases->get_values()), stride_krylov,
+            as_cuda_type(hessenberg_iter->get_const_values()),
+            stride_hessenberg, as_cuda_type(stop_status));
+    // next_krylov_basis /= hessenberg(iter, iter + 1)
+    // krylov_bases(:, iter + 1) = next_krylov_basis
+    // End of arnoldi
+}
+/**/
+
+template <typename ValueType>
+void givens_rotation(std::shared_ptr<const CudaExecutor> exec,
+                     matrix::Dense<ValueType> *givens_sin,
+                     matrix::Dense<ValueType> *givens_cos,
+                     matrix::Dense<ValueType> *hessenberg_iter,
+                     matrix::Dense<ValueType> *residual_norm,
+                     matrix::Dense<ValueType> *residual_norm_collection,
+                     const matrix::Dense<ValueType> *b_norm, size_type iter,
+                     const Array<stopping_status> *stop_status)
+{
+    // TODO: tune block_size for optimal performance
+    constexpr auto block_size = default_block_size;
+    const auto num_cols = hessenberg_iter->get_size()[1];
+    const dim3 block_dim{block_size, 1, 1};
+    const dim3 grid_dim{
+        static_cast<unsigned int>(ceildiv(num_cols, block_size)), 1, 1};
+
+    givens_rotation_kernel<block_size><<<grid_dim, block_dim>>>(
+        hessenberg_iter->get_size()[0], hessenberg_iter->get_size()[1], iter,
+        as_cuda_type(hessenberg_iter->get_values()),
+        hessenberg_iter->get_stride(), as_cuda_type(givens_sin->get_values()),
+        givens_sin->get_stride(), as_cuda_type(givens_cos->get_values()),
+        givens_cos->get_stride(), as_cuda_type(residual_norm->get_values()),
+        as_cuda_type(residual_norm_collection->get_values()),
+        residual_norm_collection->get_stride(),
+        as_cuda_type(b_norm->get_const_values()),
+        as_cuda_type(stop_status->get_const_data()));
+}
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void step_1(std::shared_ptr<const CudaExecutor> exec,
+            matrix::Dense<ValueType> *next_krylov_basis,
+            matrix::Dense<ValueType> *givens_sin,
+            matrix::Dense<ValueType> *givens_cos,
+            matrix::Dense<ValueType> *residual_norm,
+            matrix::Dense<ValueType> *residual_norm_collection,
+            matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+            matrix::Dense<ValueType> *hessenberg_iter,
+            matrix::Dense<ValueType> *buffer_iter,
+            const matrix::Dense<ValueType> *b_norm,
+            matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+            Array<size_type> *final_iter_nums,
+            const Array<stopping_status> *stop_status,
+            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth)
+{
+    std::cout << "ITER = " << iter << std::endl;
+    increase_final_iteration_numbers_kernel<<<
+        static_cast<unsigned int>(
+            ceildiv(final_iter_nums->get_num_elems(), default_block_size)),
+        default_block_size>>>(as_cuda_type(final_iter_nums->get_data()),
+                              as_cuda_type(stop_status->get_const_data()),
+                              final_iter_nums->get_num_elems());
+    //    finish_arnoldi(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
+    //    iter,
+    //                   stop_status->get_const_data());
+    //    finish_arnoldi_reorth(exec, next_krylov_basis, krylov_bases,
+    //    hessenberg_iter,
+    //                          buffer_iter, arnoldi_norm, iter,
+    //                          stop_status->get_const_data(),
+    //                          reorth_status->get_data(), num_reorth);
+    //                          reorth_status->get_data());
+    finish_arnoldi_CGS(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
+                       buffer_iter, arnoldi_norm, iter,
+                       stop_status->get_const_data(), reorth_status->get_data(),
+                       num_reorth);
+    givens_rotation(exec, givens_sin, givens_cos, hessenberg_iter,
+                    residual_norm, residual_norm_collection, b_norm, iter,
+                    stop_status);
+}
+
+GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
+
+
+template <typename ValueType>
+void solve_upper_triangular(
+    const matrix::Dense<ValueType> *residual_norm_collection,
+    const matrix::Dense<ValueType> *hessenberg, matrix::Dense<ValueType> *y,
+    const Array<size_type> *final_iter_nums)
+{
+    // TODO: tune block_size for optimal performance
+    constexpr auto block_size = default_block_size;
+    const auto num_rhs = residual_norm_collection->get_size()[1];
+    const dim3 block_dim{block_size, 1, 1};
+    const dim3 grid_dim{static_cast<unsigned int>(ceildiv(num_rhs, block_size)),
+                        1, 1};
+
+    solve_upper_triangular_kernel<block_size><<<grid_dim, block_dim>>>(
+        hessenberg->get_size()[1], num_rhs,
+        as_cuda_type(residual_norm_collection->get_const_values()),
+        residual_norm_collection->get_stride(),
+        as_cuda_type(hessenberg->get_const_values()), hessenberg->get_stride(),
+        as_cuda_type(y->get_values()), y->get_stride(),
+        as_cuda_type(final_iter_nums->get_const_data()));
+}
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void calculate_qy(const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                  const matrix::Dense<ValueType> *y,
+                  matrix::Dense<ValueType> *before_preconditioner,
+                  const Array<size_type> *final_iter_nums)
+{
+    const auto num_rows = before_preconditioner->get_size()[0];
+    const auto num_cols = krylov_bases->get_size()[1];
+    const auto num_rhs = before_preconditioner->get_size()[1];
+    const auto stride_before_preconditioner =
+        before_preconditioner->get_stride();
+
+    constexpr auto block_size = default_block_size;
+    const dim3 grid_dim{
+        static_cast<unsigned int>(
+            ceildiv(num_rows * stride_before_preconditioner, block_size)),
+        1, 1};
+    const dim3 block_dim{block_size, 1, 1};
+
+
+    calculate_Qy_kernel<block_size><<<grid_dim, block_dim>>>(
+        num_rows, num_cols, num_rhs,
+        as_cuda_type(krylov_bases->get_const_values()),
+        krylov_bases->get_stride(), as_cuda_type(y->get_const_values()),
+        y->get_stride(), as_cuda_type(before_preconditioner->get_values()),
+        stride_before_preconditioner,
+        as_cuda_type(final_iter_nums->get_const_data()));
+    // Calculate qy
+    // before_preconditioner = krylov_bases * y
+}
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void step_2(std::shared_ptr<const CudaExecutor> exec,
+            const matrix::Dense<ValueType> *residual_norm_collection,
+            const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+            const matrix::Dense<ValueType> *hessenberg,
+            matrix::Dense<ValueType> *y,
+            matrix::Dense<ValueType> *before_preconditioner,
+            const Array<size_type> *final_iter_nums)
+{
+    solve_upper_triangular(residual_norm_collection, hessenberg, y,
+                           final_iter_nums);
+    calculate_qy(krylov_bases, y, before_preconditioner, final_iter_nums);
+}
+
+GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
+
+
+}  // namespace gmres_mixed
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
diff --git a/cuda/test/solver/CMakeLists.txt b/cuda/test/solver/CMakeLists.txt
index 0689804b253..4290bea2d5c 100644
--- a/cuda/test/solver/CMakeLists.txt
+++ b/cuda/test/solver/CMakeLists.txt
@@ -4,7 +4,7 @@ ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
-ginkgo_create_test(gmres_mixed_kernels)
+# ginkgo_create_test(gmres_mixed_kernels)
 ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
 ginkgo_create_test_cpp_cuda_header(lower_trs_kernels)
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index b1379ebf0fe..3ed088cffac 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -36,6 +36,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <omp.h>
 
 
+#include <iostream>
+
+
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
@@ -157,6 +160,9 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
             }
             if (hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i) >
                 arnoldi_norm->at(0, i)) {
+                std::cout << "K = " << k;
+                std::cout << " , HI = " << hessenberg_iter->at(k, i);
+                std::cout << " , AN = " << arnoldi_norm->at(0, i) << std::endl;
                 ValueType reorth = zero<ValueType>();
 #pragma omp parallel for reduction(add : reorth)
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
@@ -566,8 +572,10 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Dense<ValueType> *b_norm,
             matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
             Array<size_type> *final_iter_nums,
-            const Array<stopping_status> *stop_status)
+            const Array<stopping_status> *stop_status,
+            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth)
 {
+    std::cout << "ITER = " << iter << std::endl;
 #pragma omp parallel for
     for (size_type i = 0; i < final_iter_nums->get_num_elems(); ++i) {
         final_iter_nums->get_data()[i] +=
@@ -575,12 +583,11 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
     }
     //    finish_arnoldi(next_krylov_basis, krylov_bases, hessenberg_iter, iter,
     //                   stop_status->get_const_data());
-    //    finish_arnoldi_reorth(next_krylov_basis, krylov_bases,
-    //                        hessenberg_iter, arnoldi_norm, iter,
-    //                          stop_status->get_const_data());
-    finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
-                       buffer_iter, arnoldi_norm, iter,
-                       stop_status->get_const_data());
+    finish_arnoldi_reorth(next_krylov_basis, krylov_bases, hessenberg_iter,
+                          arnoldi_norm, iter, stop_status->get_const_data());
+    // finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
+    //                    buffer_iter, arnoldi_norm, iter,
+    //                    stop_status->get_const_data());
     givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
                     iter, stop_status->get_const_data());
     calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index e19b2c989c2..680e06382f4 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -145,6 +145,9 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
             }
             if (hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i) >
                 arnoldi_norm->at(0, i)) {
+                std::cout << "K = " << k;
+                std::cout << " , HI = " << hessenberg_iter->at(k, i);
+                std::cout << " , AN = " << arnoldi_norm->at(0, i) << std::endl;
                 // if (hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i) >
                 // nrm) {
                 arnoldi_norm->at(1, i) = 0;
@@ -166,6 +169,8 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
                     // reorth *
                 }
                 hessenberg_iter->at(k, i) += arnoldi_norm->at(1, i);
+                std::cout << " Add(" << k << ") = " << arnoldi_norm->at(1, i)
+                          << std::endl;
                 //                hessenberg_iter->at(k, i) += reorth;
             }
         }
@@ -261,6 +266,9 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
         arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
         // nrmN = sqrt(nrmN);
         // nrmN = arnoldi_norm->at(1, i)(next_krylov_basis)
+        printf("Reorth(%10.5e,%10.5e)\n",
+               arnoldi_norm->at(0, i) * arnoldi_norm->at(0, i),
+               arnoldi_norm->at(1, i) * arnoldi_norm->at(1, i));
         for (size_type l = 1;
              arnoldi_norm->at(1, i) < arnoldi_norm->at(0, i) && l < 3; l++) {
             // for (size_type l = 1; arnoldi_norm->at(1, i) < eta *
@@ -310,6 +318,9 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
             arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
             // nrmN = sqrt(nrmN);
             // nrmN = norm(next_krylov_basis)
+            printf("XX Reorth(%10.5e,%10.5e)\n",
+                   arnoldi_norm->at(0, i) * arnoldi_norm->at(0, i),
+                   arnoldi_norm->at(1, i) * arnoldi_norm->at(1, i));
         }
         //        std::cout << nrmN << " - " << eta << " - " << nrmP <<
         //        std::endl;
@@ -556,7 +567,8 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
             const matrix::Dense<ValueType> *b_norm,
             matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
             Array<size_type> *final_iter_nums,
-            const Array<stopping_status> *stop_status)
+            const Array<stopping_status> *stop_status,
+            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth)
 {
     /*
         if (Reorthogonalization) std::cout << "Reorthogonalization";
@@ -564,6 +576,7 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
         if (MGS_CGS) std::cout << "Modified Gram-Schmidt";
         else  std::cout << "Classical Gram-Schmidt";
     */
+    std::cout << "ITER = " << iter << std::endl;
     for (size_type i = 0; i < final_iter_nums->get_num_elems(); ++i) {
         final_iter_nums->get_data()[i] +=
             (1 - stop_status->get_const_data()[i].has_stopped());

From 92337ffa8b667d11727331e8f48c64c768703fc4 Mon Sep 17 00:00:00 2001
From: "aliaga@uji.es" <aliaga@tintorrum.local>
Date: Thu, 30 Apr 2020 01:23:00 +0200
Subject: [PATCH 0336/1008] The test files are finally included, but:     * The
 messages have to be removed.     * For CGS, a loop of kernels is used instead
 of a kernel with a loop.

---
 common/solver/gmres_mixed_kernels.hpp.inc     |  12 +-
 core/solver/gmres_mixed.cpp                   |   2 +-
 core/test/solver/CMakeLists.txt               |   2 +-
 core/test/solver/gmres_mixed.cpp              | 190 ++++-------
 cuda/solver/gmres_mixed_kernels.cu            |  16 +-
 cuda/test/solver/CMakeLists.txt               |   2 +-
 cuda/test/solver/gmres_mixed_kernels.cpp      | 303 ++++++++++++++++++
 omp/solver/gmres_mixed_kernels.cpp            |  12 +-
 omp/test/solver/CMakeLists.txt                |   2 +-
 omp/test/solver/gmres_mixed_kernels.cpp       | 301 +++++++++++++++++
 reference/solver/gmres_mixed_kernels.cpp      |   2 +-
 reference/test/solver/CMakeLists.txt          |   2 +-
 reference/test/solver/gmres_mixed_kernels.cpp | 263 ++++++---------
 13 files changed, 801 insertions(+), 308 deletions(-)
 create mode 100644 cuda/test/solver/gmres_mixed_kernels.cpp
 create mode 100644 omp/test/solver/gmres_mixed_kernels.cpp

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 4b9a8fad27a..f2acc9fb833 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -247,6 +247,7 @@ __global__
             if ((hessenberg_iter[hessenberg_idx] *
                  hessenberg_iter[hessenberg_idx]) >
                 (0.99 * sqrt(arnoldi_norm[arnoldi_idx]))) {
+                //                reorth_status[reorth_idx].stop(1);
                 reorth_status[reorth_idx].reset();
                 //                reorth_status[k].reset();
                 printf("K = %d , HI = %f , AN = %f\n", k,
@@ -259,6 +260,7 @@ __global__
                 //                    printf ("BAD in %d\n", arnoldi_idx);
                 atomic_add(num_reorth, (size_type)1);
             } else {
+                //                reorth_status[reorth_idx].reset();
                 reorth_status[reorth_idx].stop(1);
                 //                if (reorth_status[arnoldi_idx].has_stopped())
                 //                    printf ("OK in %d\n", arnoldi_idx);
@@ -289,7 +291,7 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
 
     if (row_idx < num_rows && col_idx < num_cols &&
         !stop_status[col_idx].has_stopped() &&
-        reorth_status[col_idx].has_stopped()) {
+        !reorth_status[col_idx].has_stopped()) {
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
         const auto krylov_idx =
             row_idx * stride_krylov + k * num_cols + col_idx;
@@ -297,7 +299,7 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
         const auto buffer_idx = k * stride_buffer + col_idx;
         next_krylov_basis[next_krylov_idx] -=
             buffer_iter[buffer_idx] * krylov_bases[krylov_idx];
-        if ((row_idx == 0) && reorth_status[col_idx].has_stopped()) {
+        if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
             printf(" Add(%d) = %10.5e\n", k, buffer_iter[buffer_idx]);
             hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
         }
@@ -330,7 +332,8 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms(
         //        {
         printf(" Reorth(%d) = %10.5e,%10.5e)\n", global_id, num0, num1);
         if (num1 < num0) {
-            reorth_status[global_id].stop(1);
+            //            reorth_status[global_id].stop(1);
+            reorth_status[global_id].reset();
             atomic_add(num_reorth, (size_type)1);
             //            num_reorth[0] = 1;
             //            printf (" Reorth(%10.5e,%10.5e)\n",
@@ -338,7 +341,8 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms(
             //                     arnoldi_norm1[global_id]);
         } else {
             //            num_reorth[0] = 0;
-            reorth_status[global_id].reset();
+            //            reorth_status[global_id].reset();
+            reorth_status[global_id].stop(1);
         }
     }
 }
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 5741d513f4f..892604af9e4 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -120,7 +120,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     Array<stopping_status> reorth_status(this->get_executor(),
                                          dense_b->get_size()[1]);
     //                                         krylov_dim_mixed_ + 1);
-    Array<size_type> num_reorth(this->get_executor(), 1);
+    Array<size_type> num_reorth(this->get_executor(), dense_b->get_size()[1]);
 
     // Initialization
     exec->run(gmres_mixed::make_initialize_1(
diff --git a/core/test/solver/CMakeLists.txt b/core/test/solver/CMakeLists.txt
index dc8996a5f6a..4fb9dbb8ba8 100644
--- a/core/test/solver/CMakeLists.txt
+++ b/core/test/solver/CMakeLists.txt
@@ -4,7 +4,7 @@ ginkgo_create_test(cg)
 ginkgo_create_test(cgs)
 ginkgo_create_test(fcg)
 ginkgo_create_test(gmres)
-# ginkgo_create_test(gmres_mixed)
+ginkgo_create_test(gmres_mixed)
 ginkgo_create_test(idr)
 ginkgo_create_test(ir)
 ginkgo_create_test(lower_trs)
diff --git a/core/test/solver/gmres_mixed.cpp b/core/test/solver/gmres_mixed.cpp
index f82553bd58a..f9c09ea39d9 100644
--- a/core/test/solver/gmres_mixed.cpp
+++ b/core/test/solver/gmres_mixed.cpp
@@ -30,40 +30,28 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include <core/solver/gmres_mixed.cpp>
 #include <ginkgo/core/solver/gmres_mixed.hpp>
 
-
 #include <typeinfo>
 
-
 #include <gtest/gtest.h>
 
-
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm_reduction.hpp>
 
-
-#include "core/test/utils.hpp"
-
-
 namespace {
 
-
-template <typename T1, typename T2>
 class GmresMixed : public ::testing::Test {
 protected:
-    using value_type = T1;
-    using krylov_type = T2;
-    using Mtx = gko::matrix::Dense<value_type>;
-    using Solver = gko::solver::GmresMixed<value_type, krylov_type>;
+    using Mtx = gko::matrix::Dense<>;
+    using Solver = gko::solver::GmresMixed<>;
+    //    using Solver = gko::solver::GmresMixed<double,double>;
     using Big_solver = gko::solver::GmresMixed<double>;
 
-    static constexpr gko::remove_complex<T1> reduction_factor =
-        gko::remove_complex<T1>(1e-6);
-
     GmresMixed()
         : exec(gko::ReferenceExecutor::create()),
           mtx(gko::initialize<Mtx>(
@@ -72,8 +60,8 @@ class GmresMixed : public ::testing::Test {
               Solver::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(3u).on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
-                          .with_reduction_factor(reduction_factor)
+                      gko::stop::ResidualNormReduction<>::build()
+                          .with_reduction_factor(1e-6)
                           .on(exec))
                   .on(exec)),
           solver(gmres_mixed_factory->generate(mtx)),
@@ -82,8 +70,8 @@ class GmresMixed : public ::testing::Test {
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(128u).on(
                           exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
-                          .with_reduction_factor(reduction_factor)
+                      gko::stop::ResidualNormReduction<>::build()
+                          .with_reduction_factor(1e-6)
                           .on(exec))
                   .on(exec)),
           big_solver(gmres_mixed_big_factory->generate(mtx))
@@ -91,7 +79,7 @@ class GmresMixed : public ::testing::Test {
 
     std::shared_ptr<const gko::Executor> exec;
     std::shared_ptr<Mtx> mtx;
-    std::unique_ptr<typename Solver::Factory> gmres_mixed_factory;
+    std::unique_ptr<Solver::Factory> gmres_mixed_factory;
     std::unique_ptr<gko::LinOp> solver;
     std::unique_ptr<Big_solver::Factory> gmres_mixed_big_factory;
     std::unique_ptr<gko::LinOp> big_solver;
@@ -108,194 +96,155 @@ class GmresMixed : public ::testing::Test {
     }
 };
 
-template <typename T1>
-constexpr gko::remove_complex<T1>
-    GmresMixed<T1, T1>::reduction_factor;  // JIAE??
-
-TYPED_TEST_CASE(GmresMixed, gko::test::ValueTypes);
-
-
-TYPED_TEST(GmresMixed, GmresMixedFactoryKnowsItsExecutor)
+TEST_F(GmresMixed, GmresMixedFactoryKnowsItsExecutor)
 {
-    ASSERT_EQ(this->gmres_mixed_factory->get_executor(), this->exec);
+    ASSERT_EQ(gmres_mixed_factory->get_executor(), exec);
 }
 
-
-TYPED_TEST(GmresMixed, GmresMixedFactoryCreatesCorrectSolver)
+TEST_F(GmresMixed, GmresMixedFactoryCreatesCorrectSolver)
 {
-    using Solver = typename TestFixture::Solver;
-    ASSERT_EQ(this->solver->get_size(), gko::dim<2>(3, 3));
-    auto gmres_mixed_solver = static_cast<Solver *>(this->solver.get());
+    ASSERT_EQ(solver->get_size(), gko::dim<2>(3, 3));
+    auto gmres_mixed_solver = static_cast<Solver *>(solver.get());
     ASSERT_NE(gmres_mixed_solver->get_system_matrix(), nullptr);
-    ASSERT_EQ(gmres_mixed_solver->get_system_matrix(), this->mtx);
+    ASSERT_EQ(gmres_mixed_solver->get_system_matrix(), mtx);
 }
 
-
-TYPED_TEST(GmresMixed, CanBeCopied)
+TEST_F(GmresMixed, CanBeCopied)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using Solver = typename TestFixture::Solver;
-    auto copy = this->gmres_mixed_factory->generate(Mtx::create(this->exec));
+    auto copy = gmres_mixed_factory->generate(Mtx::create(exec));
 
-    copy->copy_from(this->solver.get());
+    copy->copy_from(solver.get());
 
     ASSERT_EQ(copy->get_size(), gko::dim<2>(3, 3));
     auto copy_mtx = static_cast<Solver *>(copy.get())->get_system_matrix();
-    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
-                               this->mtx.get());
+    assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()), mtx.get());
 }
 
-
-TYPED_TEST(GmresMixed, CanBeMoved)
+TEST_F(GmresMixed, CanBeMoved)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using Solver = typename TestFixture::Solver;
-    auto copy = this->gmres_mixed_factory->generate(Mtx::create(this->exec));
+    auto copy = gmres_mixed_factory->generate(Mtx::create(exec));
 
-    copy->copy_from(std::move(this->solver));
+    copy->copy_from(std::move(solver));
 
     ASSERT_EQ(copy->get_size(), gko::dim<2>(3, 3));
     auto copy_mtx = static_cast<Solver *>(copy.get())->get_system_matrix();
-    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
-                               this->mtx.get());
+    assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()), mtx.get());
 }
 
-
-TYPED_TEST(GmresMixed, CanBeCloned)
+TEST_F(GmresMixed, CanBeCloned)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using Solver = typename TestFixture::Solver;
-    auto clone = this->solver->clone();
+    auto clone = solver->clone();
 
     ASSERT_EQ(clone->get_size(), gko::dim<2>(3, 3));
     auto clone_mtx = static_cast<Solver *>(clone.get())->get_system_matrix();
-    this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
-                               this->mtx.get());
+    assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()), mtx.get());
 }
 
-
-TYPED_TEST(GmresMixed, CanBeCleared)
+TEST_F(GmresMixed, CanBeCleared)
 {
-    using Solver = typename TestFixture::Solver;
-    this->solver->clear();
+    solver->clear();
 
-    ASSERT_EQ(this->solver->get_size(), gko::dim<2>(0, 0));
-    auto solver_mtx =
-        static_cast<Solver *>(this->solver.get())->get_system_matrix();
+    ASSERT_EQ(solver->get_size(), gko::dim<2>(0, 0));
+    auto solver_mtx = static_cast<Solver *>(solver.get())->get_system_matrix();
     ASSERT_EQ(solver_mtx, nullptr);
 }
 
-
-TYPED_TEST(GmresMixed, CanSetPreconditionerGenerator)
+TEST_F(GmresMixed, CanSetPreconditionerGenerator)
 {
-    using Solver = typename TestFixture::Solver;
-    using value_type = typename TestFixture::value_type;
     auto gmres_mixed_factory =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
-                    .with_reduction_factor(TestFixture::reduction_factor)
-                    .on(this->exec))
-            .with_preconditioner(Solver::build().on(this->exec))
-            .on(this->exec);
-    auto solver = gmres_mixed_factory->generate(this->mtx);
-    auto precond = dynamic_cast<const gko::solver::GmresMixed<value_type> *>(
-        static_cast<gko::solver::GmresMixed<value_type> *>(solver.get())
+                gko::stop::Iteration::build().with_max_iters(3u).on(exec),
+                gko::stop::ResidualNormReduction<>::build()
+                    .with_reduction_factor(1e-6)
+                    .on(exec))
+            .with_preconditioner(Solver::build().on(exec))
+            .on(exec);
+    auto solver = gmres_mixed_factory->generate(mtx);
+    auto precond = dynamic_cast<const gko::solver::GmresMixed<> *>(
+        static_cast<gko::solver::GmresMixed<> *>(solver.get())
             ->get_preconditioner()
             .get());
 
     ASSERT_NE(precond, nullptr);
     ASSERT_EQ(precond->get_size(), gko::dim<2>(3, 3));
-    ASSERT_EQ(precond->get_system_matrix(), this->mtx);
+    ASSERT_EQ(precond->get_system_matrix(), mtx);
 }
 
-
-TYPED_TEST(GmresMixed, CanSetKrylovDim)
+TEST_F(GmresMixed, CanSetKrylovDim)
 {
-    using Solver = typename TestFixture::Solver;
-    using value_type = typename TestFixture::value_type;
     auto gmres_mixed_factory =
         Solver::build()
             .with_krylov_dim_mixed(4u)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(4u).on(this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
-                    .with_reduction_factor(TestFixture::reduction_factor)
-                    .on(this->exec))
-            .on(this->exec);
-    auto solver = gmres_mixed_factory->generate(this->mtx);
+                gko::stop::Iteration::build().with_max_iters(4u).on(exec),
+                gko::stop::ResidualNormReduction<>::build()
+                    .with_reduction_factor(1e-6)
+                    .on(exec))
+            .on(exec);
+    auto solver = gmres_mixed_factory->generate(mtx);
     auto krylov_dim_mixed = solver->get_krylov_dim_mixed();
 
     ASSERT_EQ(krylov_dim_mixed, 4);
 }
 
-
-TYPED_TEST(GmresMixed, CanSetPreconditionerInFactory)
+TEST_F(GmresMixed, CanSetPreconditionerInFactory)
 {
-    using Solver = typename TestFixture::Solver;
     std::shared_ptr<Solver> gmres_mixed_precond =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
-            .on(this->exec)
-            ->generate(this->mtx);
+                gko::stop::Iteration::build().with_max_iters(3u).on(exec))
+            .on(exec)
+            ->generate(mtx);
 
     auto gmres_mixed_factory =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+                gko::stop::Iteration::build().with_max_iters(3u).on(exec))
             .with_generated_preconditioner(gmres_mixed_precond)
-            .on(this->exec);
-    auto solver = gmres_mixed_factory->generate(this->mtx);
+            .on(exec);
+    auto solver = gmres_mixed_factory->generate(mtx);
     auto precond = solver->get_preconditioner();
 
     ASSERT_NE(precond.get(), nullptr);
     ASSERT_EQ(precond.get(), gmres_mixed_precond.get());
 }
 
-
-TYPED_TEST(GmresMixed, ThrowsOnWrongPreconditionerInFactory)
+TEST_F(GmresMixed, ThrowsOnWrongPreconditionerInFactory)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using Solver = typename TestFixture::Solver;
-    std::shared_ptr<Mtx> wrong_sized_mtx =
-        Mtx::create(this->exec, gko::dim<2>{1, 3});
+    std::shared_ptr<Mtx> wrong_sized_mtx = Mtx::create(exec, gko::dim<2>{1, 3});
     std::shared_ptr<Solver> gmres_mixed_precond =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
-            .on(this->exec)
+                gko::stop::Iteration::build().with_max_iters(3u).on(exec))
+            .on(exec)
             ->generate(wrong_sized_mtx);
 
     auto gmres_mixed_factory =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+                gko::stop::Iteration::build().with_max_iters(3u).on(exec))
             .with_generated_preconditioner(gmres_mixed_precond)
-            .on(this->exec);
+            .on(exec);
 
-    ASSERT_THROW(gmres_mixed_factory->generate(this->mtx),
-                 gko::DimensionMismatch);
+    ASSERT_THROW(gmres_mixed_factory->generate(mtx), gko::DimensionMismatch);
 }
 
-
-TYPED_TEST(GmresMixed, CanSetPreconditioner)
+TEST_F(GmresMixed, CanSetPreconditioner)
 {
-    using Solver = typename TestFixture::Solver;
     std::shared_ptr<Solver> gmres_mixed_precond =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
-            .on(this->exec)
-            ->generate(this->mtx);
+                gko::stop::Iteration::build().with_max_iters(3u).on(exec))
+            .on(exec)
+            ->generate(mtx);
 
     auto gmres_mixed_factory =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
-            .on(this->exec);
-    auto solver = gmres_mixed_factory->generate(this->mtx);
+                gko::stop::Iteration::build().with_max_iters(3u).on(exec))
+            .on(exec);
+    auto solver = gmres_mixed_factory->generate(mtx);
     solver->set_preconditioner(gmres_mixed_precond);
     auto precond = solver->get_preconditioner();
 
@@ -303,5 +252,4 @@ TYPED_TEST(GmresMixed, CanSetPreconditioner)
     ASSERT_EQ(precond.get(), gmres_mixed_precond.get());
 }
 
-
 }  // namespace
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index bc168c766b0..712b2134323 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -287,7 +287,10 @@ void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
         //        if (num_reorth > 0) {
         //        if (2 > 1) {
         if (numReorth > 0) {
+            //        if (numReorth < 0) {
             std::cout << "REORTHOGONALIZATION" << std::endl;
+            std::cout << dim_size[0] << " - " << dim_size[1] << " - "
+                      << stride_buffer << std::endl;
             components::fill_array(
                 exec, buffer_iter->get_values() + k * stride_buffer,
                 dim_size[1], zero<ValueType>());
@@ -433,8 +436,8 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
     exec->get_master()->copy_from(exec.get(), 1, num_reorth->get_const_data(),
                                   &numReorth);
     std::cout << " numReorth => " << numReorth << std::endl;
-    for (size_type l = 1; (l < 3); l++) {
-        //    for (size_type l = 1; (numReorth>0) && (l < 3); l++) {
+    // for (size_type l = 1; (l < 3); l++) {
+    for (size_type l = 1; (numReorth > 0) && (l < 3); l++) {
         //    for (size_type l = 1; (num_reorth->get_data[0]>0) && (l < 3); l++)
         //    { for (size_type l = 1; (num_reorth->get_values[0]>0) && (l < 3);
         //    l++) {
@@ -590,14 +593,11 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
                               as_cuda_type(stop_status->get_const_data()),
                               final_iter_nums->get_num_elems());
     //    finish_arnoldi(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
-    //    iter,
-    //                   stop_status->get_const_data());
+    //                   iter, stop_status->get_const_data());
     //    finish_arnoldi_reorth(exec, next_krylov_basis, krylov_bases,
-    //    hessenberg_iter,
-    //                          buffer_iter, arnoldi_norm, iter,
-    //                          stop_status->get_const_data(),
+    //                          hessenberg_iter, buffer_iter, arnoldi_norm,
+    //                          iter, stop_status->get_const_data(),
     //                          reorth_status->get_data(), num_reorth);
-    //                          reorth_status->get_data());
     finish_arnoldi_CGS(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
                        buffer_iter, arnoldi_norm, iter,
                        stop_status->get_const_data(), reorth_status->get_data(),
diff --git a/cuda/test/solver/CMakeLists.txt b/cuda/test/solver/CMakeLists.txt
index 4290bea2d5c..0689804b253 100644
--- a/cuda/test/solver/CMakeLists.txt
+++ b/cuda/test/solver/CMakeLists.txt
@@ -4,7 +4,7 @@ ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
-# ginkgo_create_test(gmres_mixed_kernels)
+ginkgo_create_test(gmres_mixed_kernels)
 ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
 ginkgo_create_test_cpp_cuda_header(lower_trs_kernels)
diff --git a/cuda/test/solver/gmres_mixed_kernels.cpp b/cuda/test/solver/gmres_mixed_kernels.cpp
new file mode 100644
index 00000000000..d3fe27800f6
--- /dev/null
+++ b/cuda/test/solver/gmres_mixed_kernels.cpp
@@ -0,0 +1,303 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/solver/gmres_mixed.hpp>
+
+#include <random>
+
+#include <gtest/gtest.h>
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm_reduction.hpp>
+
+#include "core/solver/gmres_mixed_kernels.hpp"
+#include "core/test/utils.hpp"
+
+namespace {
+
+class GmresMixed : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<>;
+    GmresMixed() : rand_engine(30) {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        cuda = gko::CudaExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (cuda != nullptr) {
+            ASSERT_NO_THROW(cuda->synchronize());
+        }
+    }
+
+    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<Mtx>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(num_cols, num_cols),
+            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    void initialize_data()
+    {
+        int m = 597;
+        int n = 43;
+        x = gen_mtx(m, n);
+        y = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        before_preconditioner = Mtx::create_with_config_of(x.get());
+        b = gen_mtx(m, n);
+        b_norm = gen_mtx(1, n);
+        arnoldi_norm = gen_mtx(2, n);
+        krylov_bases =
+            gen_mtx(m, (gko::solver::default_krylov_dim_mixed + 1) * n);
+        next_krylov_basis = gen_mtx(m, n);
+        hessenberg = gen_mtx(gko::solver::default_krylov_dim_mixed + 1,
+                             gko::solver::default_krylov_dim_mixed * n);
+        hessenberg_iter = gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
+        buffer_iter = gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
+        residual = gen_mtx(m, n);
+        residual_norm = gen_mtx(1, n);
+        residual_norm_collection =
+            gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
+        givens_sin = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        givens_cos = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(ref, n));
+        for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
+            stop_status->get_data()[i].reset();
+        }
+        reorth_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(ref, n));
+        for (size_t i = 0; i < reorth_status->get_num_elems(); ++i) {
+            reorth_status->get_data()[i].reset();
+        }
+        final_iter_nums = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(ref, n));
+        for (size_t i = 0; i < final_iter_nums->get_num_elems(); ++i) {
+            final_iter_nums->get_data()[i] = 5;
+        }
+        num_reorth = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(ref, n));
+        for (size_t i = 0; i < num_reorth->get_num_elems(); ++i) {
+            num_reorth->get_data()[i] = 5;
+        }
+
+        d_x = Mtx::create(cuda);
+        d_x->copy_from(x.get());
+        d_before_preconditioner = Mtx::create_with_config_of(d_x.get());
+        d_y = Mtx::create(cuda);
+        d_y->copy_from(y.get());
+        d_b = Mtx::create(cuda);
+        d_b->copy_from(b.get());
+        d_b_norm = Mtx::create(cuda);
+        d_b_norm->copy_from(b_norm.get());
+        d_arnoldi_norm = Mtx::create(cuda);
+        d_arnoldi_norm->copy_from(arnoldi_norm.get());
+        d_krylov_bases = Mtx::create(cuda);
+        d_krylov_bases->copy_from(krylov_bases.get());
+        d_next_krylov_basis = Mtx::create(cuda);
+        d_next_krylov_basis->copy_from(next_krylov_basis.get());
+        d_hessenberg = Mtx::create(cuda);
+        d_hessenberg->copy_from(hessenberg.get());
+        d_hessenberg_iter = Mtx::create(cuda);
+        d_hessenberg_iter->copy_from(hessenberg_iter.get());
+        d_buffer_iter = Mtx::create(cuda);
+        d_buffer_iter->copy_from(buffer_iter.get());
+        d_residual = Mtx::create(cuda);
+        d_residual->copy_from(residual.get());
+        d_residual_norm = Mtx::create(cuda);
+        d_residual_norm->copy_from(residual_norm.get());
+        d_residual_norm_collection = Mtx::create(cuda);
+        d_residual_norm_collection->copy_from(residual_norm_collection.get());
+        d_givens_sin = Mtx::create(cuda);
+        d_givens_sin->copy_from(givens_sin.get());
+        d_givens_cos = Mtx::create(cuda);
+        d_givens_cos->copy_from(givens_cos.get());
+        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(cuda, n));
+        *d_stop_status = *stop_status;
+        d_reorth_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(cuda, n));
+        *d_reorth_status = *reorth_status;
+        d_final_iter_nums = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(cuda, n));
+        *d_final_iter_nums = *final_iter_nums;
+        d_num_reorth = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(cuda, n));
+        *d_num_reorth = *num_reorth;
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::CudaExecutor> cuda;
+
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Mtx> before_preconditioner;
+    std::unique_ptr<Mtx> x;
+    std::unique_ptr<Mtx> y;
+    std::unique_ptr<Mtx> b;
+    std::unique_ptr<Mtx> b_norm;
+    std::unique_ptr<Mtx> arnoldi_norm;
+    std::unique_ptr<Mtx> krylov_bases;
+    std::unique_ptr<Mtx> next_krylov_basis;
+    std::unique_ptr<Mtx> hessenberg;
+    std::unique_ptr<Mtx> hessenberg_iter;
+    std::unique_ptr<Mtx> buffer_iter;
+    std::unique_ptr<Mtx> residual;
+    std::unique_ptr<Mtx> residual_norm;
+    std::unique_ptr<Mtx> residual_norm_collection;
+    std::unique_ptr<Mtx> givens_sin;
+    std::unique_ptr<Mtx> givens_cos;
+    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
+    std::unique_ptr<gko::Array<gko::stopping_status>> reorth_status;
+    std::unique_ptr<gko::Array<gko::size_type>> final_iter_nums;
+    std::unique_ptr<gko::Array<gko::size_type>> num_reorth;
+
+    std::unique_ptr<Mtx> d_x;
+    std::unique_ptr<Mtx> d_before_preconditioner;
+    std::unique_ptr<Mtx> d_y;
+    std::unique_ptr<Mtx> d_b;
+    std::unique_ptr<Mtx> d_b_norm;
+    std::unique_ptr<Mtx> d_arnoldi_norm;
+    std::unique_ptr<Mtx> d_krylov_bases;
+    std::unique_ptr<Mtx> d_next_krylov_basis;
+    std::unique_ptr<Mtx> d_hessenberg;
+    std::unique_ptr<Mtx> d_hessenberg_iter;
+    std::unique_ptr<Mtx> d_buffer_iter;
+    std::unique_ptr<Mtx> d_residual;
+    std::unique_ptr<Mtx> d_residual_norm;
+    std::unique_ptr<Mtx> d_residual_norm_collection;
+    std::unique_ptr<Mtx> d_givens_sin;
+    std::unique_ptr<Mtx> d_givens_cos;
+    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
+    std::unique_ptr<gko::Array<gko::stopping_status>> d_reorth_status;
+    std::unique_ptr<gko::Array<gko::size_type>> d_final_iter_nums;
+    std::unique_ptr<gko::Array<gko::size_type>> d_num_reorth;
+};
+
+TEST_F(GmresMixed, CudaGmresMixedInitialize1IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::gmres_mixed::initialize_1(
+        ref, b.get(), b_norm.get(), residual.get(), givens_sin.get(),
+        givens_cos.get(), stop_status.get(),
+        gko::solver::default_krylov_dim_mixed);
+    gko::kernels::cuda::gmres_mixed::initialize_1(
+        cuda, d_b.get(), d_b_norm.get(), d_residual.get(), d_givens_sin.get(),
+        d_givens_cos.get(), d_stop_status.get(),
+        gko::solver::default_krylov_dim_mixed);
+
+    GKO_ASSERT_MTX_NEAR(d_b_norm, b_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual, residual, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
+    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
+}
+
+TEST_F(GmresMixed, CudaGmresMixedInitialize2IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::gmres_mixed::initialize_2(
+        ref, residual.get(), residual_norm.get(),
+        residual_norm_collection.get(), krylov_bases.get(),
+        next_krylov_basis.get(), final_iter_nums.get(),
+        gko::solver::default_krylov_dim_mixed);
+    gko::kernels::cuda::gmres_mixed::initialize_2(
+        cuda, d_residual.get(), d_residual_norm.get(),
+        d_residual_norm_collection.get(), d_krylov_bases.get(),
+        d_next_krylov_basis.get(), d_final_iter_nums.get(),
+        gko::solver::default_krylov_dim_mixed);
+
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, 1e-14);
+    GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
+}
+
+TEST_F(GmresMixed, CudaGmresMixedStep1IsEquivalentToRef)
+{
+    initialize_data();
+    int iter = 5;
+
+    gko::kernels::reference::gmres_mixed::step_1(
+        ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
+        residual_norm.get(), residual_norm_collection.get(), krylov_bases.get(),
+        hessenberg_iter.get(), buffer_iter.get(), b_norm.get(),
+        arnoldi_norm.get(), iter, final_iter_nums.get(), stop_status.get(),
+        reorth_status.get(), num_reorth.get());
+    gko::kernels::cuda::gmres_mixed::step_1(
+        cuda, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
+        d_residual_norm.get(), d_residual_norm_collection.get(),
+        d_krylov_bases.get(), d_hessenberg_iter.get(), d_buffer_iter.get(),
+        d_b_norm.get(), d_arnoldi_norm.get(), iter, d_final_iter_nums.get(),
+        d_stop_status.get(), d_reorth_status.get(), d_num_reorth.get());
+
+    GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, 1e-14);
+    GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
+}
+
+TEST_F(GmresMixed, CudaGmresMixedStep2IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::gmres_mixed::step_2(
+        ref, residual_norm_collection.get(), krylov_bases.get(),
+        hessenberg.get(), y.get(), before_preconditioner.get(),
+        final_iter_nums.get());
+    gko::kernels::cuda::gmres_mixed::step_2(
+        cuda, d_residual_norm_collection.get(), d_krylov_bases.get(),
+        d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
+        d_final_iter_nums.get());
+
+    GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+}
+
+}  // namespace
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index 3ed088cffac..c930efad7fd 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -583,11 +583,13 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
     }
     //    finish_arnoldi(next_krylov_basis, krylov_bases, hessenberg_iter, iter,
     //                   stop_status->get_const_data());
-    finish_arnoldi_reorth(next_krylov_basis, krylov_bases, hessenberg_iter,
-                          arnoldi_norm, iter, stop_status->get_const_data());
-    // finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
-    //                    buffer_iter, arnoldi_norm, iter,
-    //                    stop_status->get_const_data());
+    //    finish_arnoldi_reorth(next_krylov_basis, krylov_bases,
+    //    hessenberg_iter,
+    //                          arnoldi_norm, iter,
+    //                          stop_status->get_const_data());
+    finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
+                       buffer_iter, arnoldi_norm, iter,
+                       stop_status->get_const_data());
     givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
                     iter, stop_status->get_const_data());
     calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,
diff --git a/omp/test/solver/CMakeLists.txt b/omp/test/solver/CMakeLists.txt
index 7c75ae347b0..de3fce1fb78 100644
--- a/omp/test/solver/CMakeLists.txt
+++ b/omp/test/solver/CMakeLists.txt
@@ -4,7 +4,7 @@ ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
-# ginkgo_create_test(gmres_mixed_kernels)
+ginkgo_create_test(gmres_mixed_kernels)
 ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
 ginkgo_create_test(lower_trs_kernels)
diff --git a/omp/test/solver/gmres_mixed_kernels.cpp b/omp/test/solver/gmres_mixed_kernels.cpp
new file mode 100644
index 00000000000..917b31fa763
--- /dev/null
+++ b/omp/test/solver/gmres_mixed_kernels.cpp
@@ -0,0 +1,301 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/gmres_mixed_kernels.hpp"
+
+#include <random>
+
+#include <gtest/gtest.h>
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/solver/gmres_mixed.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm_reduction.hpp>
+
+#include "core/test/utils.hpp"
+
+namespace {
+
+class GmresMixed : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<>;
+    GmresMixed() : rand_engine(30) {}
+
+    void SetUp()
+    {
+        ref = gko::ReferenceExecutor::create();
+        omp = gko::OmpExecutor::create();
+    }
+
+    void TearDown()
+    {
+        if (omp != nullptr) {
+            ASSERT_NO_THROW(omp->synchronize());
+        }
+    }
+
+    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<Mtx>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(num_cols, num_cols),
+            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    void initialize_data()
+    {
+        int m = 597;
+        int n = 43;
+        x = gen_mtx(m, n);
+        y = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        before_preconditioner = Mtx::create_with_config_of(x.get());
+        b = gen_mtx(m, n);
+        b_norm = gen_mtx(1, n);
+        arnoldi_norm = gen_mtx(2, n);
+        krylov_bases =
+            gen_mtx(m, (gko::solver::default_krylov_dim_mixed + 1) * n);
+        next_krylov_basis = gen_mtx(m, n);
+        hessenberg = gen_mtx(gko::solver::default_krylov_dim_mixed + 1,
+                             gko::solver::default_krylov_dim_mixed * n);
+        hessenberg_iter = gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
+        buffer_iter = gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
+        residual = gen_mtx(m, n);
+        residual_norm = gen_mtx(1, n);
+        residual_norm_collection =
+            gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
+        givens_sin = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        givens_cos = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(ref, n));
+        for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
+            stop_status->get_data()[i].reset();
+        }
+        reorth_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(ref, n));
+        for (size_t i = 0; i < reorth_status->get_num_elems(); ++i) {
+            reorth_status->get_data()[i].reset();
+        }
+        final_iter_nums = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(ref, n));
+        for (size_t i = 0; i < final_iter_nums->get_num_elems(); ++i) {
+            final_iter_nums->get_data()[i] = 5;
+        }
+        num_reorth = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(ref, n));
+        for (size_t i = 0; i < num_reorth->get_num_elems(); ++i) {
+            num_reorth->get_data()[i] = 0;
+        }
+
+        d_x = Mtx::create(omp);
+        d_x->copy_from(x.get());
+        d_before_preconditioner = Mtx::create_with_config_of(d_x.get());
+        d_y = Mtx::create(omp);
+        d_y->copy_from(y.get());
+        d_b = Mtx::create(omp);
+        d_b->copy_from(b.get());
+        d_b_norm = Mtx::create(omp);
+        d_b_norm->copy_from(b_norm.get());
+        d_arnoldi_norm = Mtx::create(omp);
+        d_arnoldi_norm->copy_from(arnoldi_norm.get());
+        d_krylov_bases = Mtx::create(omp);
+        d_krylov_bases->copy_from(krylov_bases.get());
+        d_next_krylov_basis = Mtx::create(omp);
+        d_next_krylov_basis->copy_from(next_krylov_basis.get());
+        d_hessenberg = Mtx::create(omp);
+        d_hessenberg->copy_from(hessenberg.get());
+        d_hessenberg_iter = Mtx::create(omp);
+        d_hessenberg_iter->copy_from(hessenberg_iter.get());
+        d_buffer_iter = Mtx::create(omp);
+        d_residual = Mtx::create(omp);
+        d_residual->copy_from(residual.get());
+        d_residual_norm = Mtx::create(omp);
+        d_residual_norm->copy_from(residual_norm.get());
+        d_residual_norm_collection = Mtx::create(omp);
+        d_residual_norm_collection->copy_from(residual_norm_collection.get());
+        d_givens_sin = Mtx::create(omp);
+        d_givens_sin->copy_from(givens_sin.get());
+        d_givens_cos = Mtx::create(omp);
+        d_givens_cos->copy_from(givens_cos.get());
+        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(omp, n));
+        *d_stop_status = *stop_status;
+        d_reorth_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(omp, n));
+        *d_reorth_status = *reorth_status;
+        d_final_iter_nums = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(omp, n));
+        *d_final_iter_nums = *final_iter_nums;
+        d_num_reorth = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(omp, n));
+        *d_num_reorth = *num_reorth;
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::OmpExecutor> omp;
+
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Mtx> before_preconditioner;
+    std::unique_ptr<Mtx> x;
+    std::unique_ptr<Mtx> y;
+    std::unique_ptr<Mtx> b;
+    std::unique_ptr<Mtx> b_norm;
+    std::unique_ptr<Mtx> arnoldi_norm;
+    std::unique_ptr<Mtx> krylov_bases;
+    std::unique_ptr<Mtx> next_krylov_basis;
+    std::unique_ptr<Mtx> hessenberg;
+    std::unique_ptr<Mtx> hessenberg_iter;
+    std::unique_ptr<Mtx> buffer_iter;
+    std::unique_ptr<Mtx> residual;
+    std::unique_ptr<Mtx> residual_norm;
+    std::unique_ptr<Mtx> residual_norm_collection;
+    std::unique_ptr<Mtx> givens_sin;
+    std::unique_ptr<Mtx> givens_cos;
+    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
+    std::unique_ptr<gko::Array<gko::stopping_status>> reorth_status;
+    std::unique_ptr<gko::Array<gko::size_type>> final_iter_nums;
+    std::unique_ptr<gko::Array<gko::size_type>> num_reorth;
+
+    std::unique_ptr<Mtx> d_x;
+    std::unique_ptr<Mtx> d_before_preconditioner;
+    std::unique_ptr<Mtx> d_y;
+    std::unique_ptr<Mtx> d_b;
+    std::unique_ptr<Mtx> d_b_norm;
+    std::unique_ptr<Mtx> d_arnoldi_norm;
+    std::unique_ptr<Mtx> d_krylov_bases;
+    std::unique_ptr<Mtx> d_next_krylov_basis;
+    std::unique_ptr<Mtx> d_hessenberg;
+    std::unique_ptr<Mtx> d_hessenberg_iter;
+    std::unique_ptr<Mtx> d_buffer_iter;
+    std::unique_ptr<Mtx> d_residual;
+    std::unique_ptr<Mtx> d_residual_norm;
+    std::unique_ptr<Mtx> d_residual_norm_collection;
+    std::unique_ptr<Mtx> d_givens_sin;
+    std::unique_ptr<Mtx> d_givens_cos;
+    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
+    std::unique_ptr<gko::Array<gko::stopping_status>> d_reorth_status;
+    std::unique_ptr<gko::Array<gko::size_type>> d_final_iter_nums;
+    std::unique_ptr<gko::Array<gko::size_type>> d_num_reorth;
+};
+
+TEST_F(GmresMixed, OmpGmresMixedInitialize1IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::gmres_mixed::initialize_1(
+        ref, b.get(), b_norm.get(), residual.get(), givens_sin.get(),
+        givens_cos.get(), stop_status.get(),
+        gko::solver::default_krylov_dim_mixed);
+    gko::kernels::omp::gmres_mixed::initialize_1(
+        omp, d_b.get(), d_b_norm.get(), d_residual.get(), d_givens_sin.get(),
+        d_givens_cos.get(), d_stop_status.get(),
+        gko::solver::default_krylov_dim_mixed);
+
+    GKO_ASSERT_MTX_NEAR(d_b_norm, b_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual, residual, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
+    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
+}
+
+TEST_F(GmresMixed, OmpGmresMixedInitialize2IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::gmres_mixed::initialize_2(
+        ref, residual.get(), residual_norm.get(),
+        residual_norm_collection.get(), krylov_bases.get(),
+        next_krylov_basis.get(), final_iter_nums.get(),
+        gko::solver::default_krylov_dim_mixed);
+    gko::kernels::omp::gmres_mixed::initialize_2(
+        omp, d_residual.get(), d_residual_norm.get(),
+        d_residual_norm_collection.get(), d_krylov_bases.get(),
+        d_next_krylov_basis.get(), d_final_iter_nums.get(),
+        gko::solver::default_krylov_dim_mixed);
+
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, 1e-14);
+    GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
+}
+
+TEST_F(GmresMixed, OmpGmresMixedStep1IsEquivalentToRef)
+{
+    initialize_data();
+    int iter = 5;
+
+    gko::kernels::reference::gmres_mixed::step_1(
+        ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
+        residual_norm.get(), residual_norm_collection.get(), krylov_bases.get(),
+        hessenberg_iter.get(), buffer_iter.get(), b_norm.get(),
+        arnoldi_norm.get(), iter, final_iter_nums.get(), stop_status.get(),
+        reorth_status.get(), num_reorth.get());
+    gko::kernels::omp::gmres_mixed::step_1(
+        omp, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
+        d_residual_norm.get(), d_residual_norm_collection.get(),
+        d_krylov_bases.get(), d_hessenberg_iter.get(), d_buffer_iter.get(),
+        d_b_norm.get(), d_arnoldi_norm.get(), iter, d_final_iter_nums.get(),
+        d_stop_status.get(), d_reorth_status.get(), d_num_reorth.get());
+
+    GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, 1e-14);
+    GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
+}
+
+TEST_F(GmresMixed, OmpGmresMixedStep2IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::gmres_mixed::step_2(
+        ref, residual_norm_collection.get(), krylov_bases.get(),
+        hessenberg.get(), y.get(), before_preconditioner.get(),
+        final_iter_nums.get());
+    gko::kernels::omp::gmres_mixed::step_2(
+        omp, d_residual_norm_collection.get(), d_krylov_bases.get(),
+        d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
+        d_final_iter_nums.get());
+
+    GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+}
+
+}  // namespace
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index 680e06382f4..bd015bc2f2a 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -143,7 +143,7 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
                     krylov_bases->at(j,
                                      next_krylov_basis->get_size()[1] * k + i);
             }
-            if (hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i) >
+            if ((hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i)) >
                 arnoldi_norm->at(0, i)) {
                 std::cout << "K = " << k;
                 std::cout << " , HI = " << hessenberg_iter->at(k, i);
diff --git a/reference/test/solver/CMakeLists.txt b/reference/test/solver/CMakeLists.txt
index 6014f571dc4..1ad4bc4ddbf 100644
--- a/reference/test/solver/CMakeLists.txt
+++ b/reference/test/solver/CMakeLists.txt
@@ -4,7 +4,7 @@ ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
-#ginkgo_create_test(gmres_mixed_kernels)
+ginkgo_create_test(gmres_mixed_kernels)
 ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
 ginkgo_create_test(lower_trs)
diff --git a/reference/test/solver/gmres_mixed_kernels.cpp b/reference/test/solver/gmres_mixed_kernels.cpp
index b5138b4631a..4ea8b6fd0e2 100644
--- a/reference/test/solver/gmres_mixed_kernels.cpp
+++ b/reference/test/solver/gmres_mixed_kernels.cpp
@@ -32,10 +32,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/solver/gmres_mixed.hpp>
 
-
 #include <gtest/gtest.h>
 
-
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
@@ -45,33 +43,26 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/residual_norm_reduction.hpp>
 #include <ginkgo/core/stop/time.hpp>
 
-
-#include "core/test/utils.hpp"
-
+#include "core/test/utils/assertions.hpp"
 
 namespace {
 
-
-template <typename T1, typename T2>
 class GmresMixed : public ::testing::Test {
 protected:
-    using value_type = T1;
-    using krylov_type = T2;
-    using Mtx = gko::matrix::Dense<value_type>;
-    using Solver = gko::solver::GmresMixed<value_type, krylov_type>;
+    using Mtx = gko::matrix::Dense<>;
     GmresMixed()
         : exec(gko::ReferenceExecutor::create()),
           mtx(gko::initialize<Mtx>(
               {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)),
           gmres_mixed_factory(
-              Solver::build()
+              gko::solver::GmresMixed<>::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(4u).on(exec),
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
-                          .with_reduction_factor(r<value_type>::value)
+                      gko::stop::ResidualNormReduction<>::build()
+                          .with_reduction_factor(1e-15)
                           .on(exec))
                   .on(exec)),
           mtx_big(gko::initialize<Mtx>(
@@ -83,12 +74,12 @@ class GmresMixed : public ::testing::Test {
                {-848.1, -280.5, -381.8, -187.1, 51.2, -176.2}},
               exec)),
           gmres_mixed_factory_big(
-              Solver::build()
+              gko::solver::GmresMixed<>::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
-                      gko::stop::ResidualNormReduction<value_type>::build()
-                          .with_reduction_factor(r<value_type>::value)
+                      gko::stop::ResidualNormReduction<>::build()
+                          .with_reduction_factor(1e-15)
                           .on(exec))
                   .on(exec)),
           mtx_medium(
@@ -104,148 +95,107 @@ class GmresMixed : public ::testing::Test {
     std::shared_ptr<Mtx> mtx;
     std::shared_ptr<Mtx> mtx_medium;
     std::shared_ptr<Mtx> mtx_big;
-    std::unique_ptr<typename Solver::Factory> gmres_mixed_factory;
-    std::unique_ptr<typename Solver::Factory> gmres_mixed_factory_big;
+    std::unique_ptr<gko::solver::GmresMixed<>::Factory> gmres_mixed_factory;
+    std::unique_ptr<gko::solver::GmresMixed<>::Factory> gmres_mixed_factory_big;
 };
 
-TYPED_TEST_CASE(GmresMixed, gko::test::ValueTypes);
-
-
-TYPED_TEST(GmresMixed, SolvesStencilSystem)
+TEST_F(GmresMixed, SolvesStencilSystem)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using value_type = typename TestFixture::value_type;
-    auto solver = this->gmres_mixed_factory->generate(this->mtx);
-    auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, this->exec);
-    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+    auto solver = gmres_mixed_factory->generate(mtx);
+    auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}), r<value_type>::value * 1e1);
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}), 1e-14);
 }
 
-
-TYPED_TEST(GmresMixed, SolvesMultipleStencilSystems)
+TEST_F(GmresMixed, SolvesMultipleStencilSystems)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using value_type = typename TestFixture::value_type;
-    using T = value_type;
-    auto solver = this->gmres_mixed_factory->generate(this->mtx);
-    auto b = gko::initialize<Mtx>(
-        {I<T>{13.0, 6.0}, I<T>{7.0, 4.0}, I<T>{1.0, 1.0}}, this->exec);
-    auto x = gko::initialize<Mtx>(
-        {I<T>{0.0, 0.0}, I<T>{0.0, 0.0}, I<T>{0.0, 0.0}}, this->exec);
+    auto solver = gmres_mixed_factory->generate(mtx);
+    auto b = gko::initialize<Mtx>({{13.0, 6.0}, {7.0, 4.0}, {1.0, 1.0}}, exec);
+    auto x = gko::initialize<Mtx>({{0.0, 0.0}, {0.0, 0.0}, {0.0, 0.0}}, exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({{1.0, 1.0}, {3.0, 1.0}, {2.0, 1.0}}),
-                        r<value_type>::value * 1e1);
+    GKO_ASSERT_MTX_NEAR(x, l({{1.0, 1.0}, {3.0, 1.0}, {2.0, 1.0}}), 1e-14);
 }
 
-
-TYPED_TEST(GmresMixed, SolvesStencilSystemUsingAdvancedApply)
+TEST_F(GmresMixed, SolvesStencilSystemUsingAdvancedApply)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using value_type = typename TestFixture::value_type;
-    auto solver = this->gmres_mixed_factory->generate(this->mtx);
-    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
-    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
-    auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, this->exec);
-    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
+    auto solver = gmres_mixed_factory->generate(mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, exec);
+    auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, exec);
+    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, exec);
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), r<value_type>::value * 1e1);
+    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), 1e-14);
 }
 
-
-TYPED_TEST(GmresMixed, SolvesMultipleStencilSystemsUsingAdvancedApply)
+TEST_F(GmresMixed, SolvesMultipleStencilSystemsUsingAdvancedApply)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using value_type = typename TestFixture::value_type;
-    using T = value_type;
-    auto solver = this->gmres_mixed_factory->generate(this->mtx);
-    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
-    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
-    auto b = gko::initialize<Mtx>(
-        {I<T>{13.0, 6.0}, I<T>{7.0, 4.0}, I<T>{1.0, 1.0}}, this->exec);
-    auto x = gko::initialize<Mtx>(
-        {I<T>{0.5, 1.0}, I<T>{1.0, 2.0}, I<T>{2.0, 3.0}}, this->exec);
+    auto solver = gmres_mixed_factory->generate(mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, exec);
+    auto b = gko::initialize<Mtx>({{13.0, 6.0}, {7.0, 4.0}, {1.0, 1.0}}, exec);
+    auto x = gko::initialize<Mtx>({{0.5, 1.0}, {1.0, 2.0}, {2.0, 3.0}}, exec);
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({{1.5, 1.0}, {5.0, 0.0}, {2.0, -1.0}}),
-                        r<value_type>::value * 1e1);
+    GKO_ASSERT_MTX_NEAR(x, l({{1.5, 1.0}, {5.0, 0.0}, {2.0, -1.0}}), 1e-14);
 }
 
-
-TYPED_TEST(GmresMixed, SolvesBigDenseSystem1)
+TEST_F(GmresMixed, SolvesBigDenseSystem1)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using value_type = typename TestFixture::value_type;
-    auto solver = this->gmres_mixed_factory_big->generate(this->mtx_big);
+    auto solver = gmres_mixed_factory_big->generate(mtx_big);
     auto b = gko::initialize<Mtx>(
-        {72748.36, 297469.88, 347229.24, 36290.66, 82958.82, -80192.15},
-        this->exec);
-    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+        {72748.36, 297469.88, 347229.24, 36290.66, 82958.82, -80192.15}, exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({52.7, 85.4, 134.2, -250.0, -16.8, 35.3}),
-                        r<value_type>::value * 1e3);
+    GKO_ASSERT_MTX_NEAR(x, l({52.7, 85.4, 134.2, -250.0, -16.8, 35.3}), 1e-10);
 }
 
-
-TYPED_TEST(GmresMixed, SolvesBigDenseSystem2)
+TEST_F(GmresMixed, SolvesBigDenseSystem2)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using value_type = typename TestFixture::value_type;
-    auto solver = this->gmres_mixed_factory_big->generate(this->mtx_big);
+    auto solver = gmres_mixed_factory_big->generate(mtx_big);
     auto b = gko::initialize<Mtx>(
         {175352.10, 313410.50, 131114.10, -134116.30, 179529.30, -43564.90},
-        this->exec);
-    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+        exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
-                        r<value_type>::value * 1e3);
+    GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}), 1e-10);
 }
 
-
-template <typename T>
-gko::remove_complex<T> infNorm(gko::matrix::Dense<T> *mat, size_t col = 0)
+double infNorm(gko::matrix::Dense<> *mat, size_t col = 0)
 {
     using std::abs;
-    using no_cpx_t = gko::remove_complex<T>;
-    no_cpx_t norm = 0.0;
+    double norm = 0.0;
     for (size_t i = 0; i < mat->get_size()[0]; ++i) {
-        no_cpx_t absEntry = abs(mat->at(i, col));
+        double absEntry = abs(mat->at(i, col));
         if (norm < absEntry) norm = absEntry;
     }
     return norm;
 }
 
-
-TYPED_TEST(GmresMixed, SolvesMultipleDenseSystemForDivergenceCheck)
+TEST_F(GmresMixed, SolvesMultipleDenseSystemForDivergenceCheck)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using value_type = typename TestFixture::value_type;
-    auto solver = this->gmres_mixed_factory_big->generate(this->mtx_big);
+    auto solver = gmres_mixed_factory_big->generate(mtx_big);
     auto b1 = gko::initialize<Mtx>(
-        {1300083.0, 1018120.5, 906410.0, -42679.5, 846779.5, 1176858.5},
-        this->exec);
+        {1300083.0, 1018120.5, 906410.0, -42679.5, 846779.5, 1176858.5}, exec);
     auto b2 = gko::initialize<Mtx>(
-        {886630.5, -172578.0, 684522.0, -65310.5, 455487.5, 607436.0},
-        this->exec);
+        {886630.5, -172578.0, 684522.0, -65310.5, 455487.5, 607436.0}, exec);
 
-    auto x1 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
-    auto x2 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+    auto x1 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, exec);
+    auto x2 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, exec);
 
-    auto bc =
-        Mtx::create(this->exec, gko::dim<2>{this->mtx_big->get_size()[0], 2});
-    auto xc =
-        Mtx::create(this->exec, gko::dim<2>{this->mtx_big->get_size()[1], 2});
+    auto bc = Mtx::create(exec, gko::dim<2>{mtx_big->get_size()[0], 2});
+    auto xc = Mtx::create(exec, gko::dim<2>{mtx_big->get_size()[1], 2});
     for (size_t i = 0; i < bc->get_size()[0]; ++i) {
         bc->at(i, 0) = b1->at(i);
         bc->at(i, 1) = b2->at(i);
@@ -257,101 +207,86 @@ TYPED_TEST(GmresMixed, SolvesMultipleDenseSystemForDivergenceCheck)
     solver->apply(b1.get(), x1.get());
     solver->apply(b2.get(), x2.get());
     solver->apply(bc.get(), xc.get());
-    auto mergedRes = Mtx::create(this->exec, gko::dim<2>{b1->get_size()[0], 2});
+    auto mergedRes = Mtx::create(exec, gko::dim<2>{b1->get_size()[0], 2});
     for (size_t i = 0; i < mergedRes->get_size()[0]; ++i) {
         mergedRes->at(i, 0) = x1->at(i);
         mergedRes->at(i, 1) = x2->at(i);
     }
 
-    auto alpha = gko::initialize<Mtx>({1.0}, this->exec);
-    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto alpha = gko::initialize<Mtx>({1.0}, exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, exec);
 
-    auto residual1 = Mtx::create(this->exec, b1->get_size());
+    auto residual1 = Mtx::create(exec, b1->get_size());
     residual1->copy_from(b1.get());
-    auto residual2 = Mtx::create(this->exec, b2->get_size());
+    auto residual2 = Mtx::create(exec, b2->get_size());
     residual2->copy_from(b2.get());
-    auto residualC = Mtx::create(this->exec, bc->get_size());
+    auto residualC = Mtx::create(exec, bc->get_size());
     residualC->copy_from(bc.get());
 
-    this->mtx_big->apply(alpha.get(), x1.get(), beta.get(), residual1.get());
-    this->mtx_big->apply(alpha.get(), x2.get(), beta.get(), residual2.get());
-    this->mtx_big->apply(alpha.get(), xc.get(), beta.get(), residualC.get());
+    mtx_big->apply(alpha.get(), x1.get(), beta.get(), residual1.get());
+    mtx_big->apply(alpha.get(), x2.get(), beta.get(), residual2.get());
+    mtx_big->apply(alpha.get(), xc.get(), beta.get(), residualC.get());
 
-    auto normS1 = infNorm(residual1.get());
-    auto normS2 = infNorm(residual2.get());
-    auto normC1 = infNorm(residualC.get(), 0);
-    auto normC2 = infNorm(residualC.get(), 1);
-    auto normB1 = infNorm(b1.get());
-    auto normB2 = infNorm(b2.get());
+    double normS1 = infNorm(residual1.get());
+    double normS2 = infNorm(residual2.get());
+    double normC1 = infNorm(residualC.get(), 0);
+    double normC2 = infNorm(residualC.get(), 1);
+    double normB1 = infNorm(b1.get());
+    double normB2 = infNorm(b2.get());
 
     // make sure that all combined solutions are as good or better than the
     // single solutions
-    ASSERT_LE(normC1 / normB1, normS1 / normB1 + r<value_type>::value);
-    ASSERT_LE(normC2 / normB2, normS2 / normB2 + r<value_type>::value);
+    ASSERT_LE(normC1 / normB1, normS1 / normB1 + 1e-14);
+    ASSERT_LE(normC2 / normB2, normS2 / normB2 + 1e-14);
 
     // Not sure if this is necessary, the assertions above should cover what is
     // needed.
-    GKO_ASSERT_MTX_NEAR(xc, mergedRes, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(xc, mergedRes, 1e-14);
 }
 
-
-TYPED_TEST(GmresMixed, SolvesBigDenseSystem1WithRestart)
+TEST_F(GmresMixed, SolvesBigDenseSystem1WithRestart)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using Solver = typename TestFixture::Solver;
-    using value_type = typename TestFixture::value_type;
-    auto half_tol = std::sqrt(r<value_type>::value);
     auto gmres_mixed_factory_restart =
-        Solver::build()
-            .with_krylov_dim(4u)
+        gko::solver::GmresMixed<>::build()
+            .with_krylov_dim_mixed(4u)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(200u).on(
-                    this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
-                    .with_reduction_factor(r<value_type>::value)
-                    .on(this->exec))
-            .on(this->exec);
-    auto solver = gmres_mixed_factory_restart->generate(this->mtx_medium);
+                gko::stop::Iteration::build().with_max_iters(200u).on(exec),
+                gko::stop::ResidualNormReduction<>::build()
+                    .with_reduction_factor(1e-15)
+                    .on(exec))
+            .on(exec);
+    auto solver = gmres_mixed_factory_restart->generate(mtx_medium);
     auto b = gko::initialize<Mtx>(
-        {-13945.16, 11205.66, 16132.96, 24342.18, -10910.98}, this->exec);
-    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+        {-13945.16, 11205.66, 16132.96, 24342.18, -10910.98}, exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0}, exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({-140.20, -142.20, 48.80, -17.70, -19.60}),
-                        half_tol * 1e2);
+    GKO_ASSERT_MTX_NEAR(x, l({-140.20, -142.20, 48.80, -17.70, -19.60}), 1e-5);
 }
 
-
-TYPED_TEST(GmresMixed, SolvesWithPreconditioner)
+TEST_F(GmresMixed, SolvesWithPreconditioner)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using Solver = typename TestFixture::Solver;
-    using value_type = typename TestFixture::value_type;
     auto gmres_mixed_factory_preconditioner =
-        Solver::build()
+        gko::solver::GmresMixed<>::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(100u).on(
-                    this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
-                    .with_reduction_factor(r<value_type>::value)
-                    .on(this->exec))
-            .with_preconditioner(
-                gko::preconditioner::Jacobi<value_type>::build()
-                    .with_max_block_size(3u)
-                    .on(this->exec))
-            .on(this->exec);
-    auto solver = gmres_mixed_factory_preconditioner->generate(this->mtx_big);
+                gko::stop::Iteration::build().with_max_iters(100u).on(exec),
+                gko::stop::ResidualNormReduction<>::build()
+                    .with_reduction_factor(1e-15)
+                    .on(exec))
+            .with_preconditioner(gko::preconditioner::Jacobi<>::build()
+                                     .with_max_block_size(3u)
+                                     .on(exec))
+            .on(exec);
+    auto solver = gmres_mixed_factory_preconditioner->generate(mtx_big);
     auto b = gko::initialize<Mtx>(
         {175352.10, 313410.50, 131114.10, -134116.30, 179529.30, -43564.90},
-        this->exec);
-    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+        exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
-                        r<value_type>::value * 1e3);
+    GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}), 1e-10);
 }
 
-
 }  // namespace

From b19daa8fb9ceebd8c6b462db6af21466ce81e005 Mon Sep 17 00:00:00 2001
From: "aliaga@uji.es" <aliaga@tintorrum.local>
Date: Thu, 30 Apr 2020 10:39:31 +0200
Subject: [PATCH 0337/1008] The first unoptimized version is done. Next tasks: 
    * For CGS, a loop of kernels is used instead of a kernel with a loop.    
 * Consider another base_types for ValueTypeKrylovBases

---
 common/solver/gmres_mixed_kernels.hpp.inc | 35 -----------
 core/solver/gmres_mixed_kernels.hpp       |  3 +-
 cuda/solver/gmres_mixed_kernels.cu        | 76 +++--------------------
 omp/solver/gmres_mixed_kernels.cpp        |  4 --
 reference/solver/gmres_mixed_kernels.cpp  | 41 +-----------
 5 files changed, 11 insertions(+), 148 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index f2acc9fb833..fb2cf313057 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -224,8 +224,6 @@ __global__
         size_type stride_norm, const stopping_status *__restrict__ stop_status,
         stopping_status *__restrict__ reorth_status,
         size_type *__restrict__ num_reorth)
-
-//    stopping_status *__restrict__ reorth_status, size_type *num_reorth)
 {
     const auto global_id = thread::get_thread_id_flat();
     const auto row_idx = global_id / stride_next_krylov;
@@ -237,7 +235,6 @@ __global__
         const auto krylov_idx =
             row_idx * stride_krylov + k * num_cols + col_idx;
         const auto hessenberg_idx = k * stride_hessenberg + col_idx;
-        //        const auto arnoldi_idx = k * stride_norm + col_idx;
         const auto arnoldi_idx = col_idx;
         const auto reorth_idx = col_idx;
 
@@ -247,26 +244,10 @@ __global__
             if ((hessenberg_iter[hessenberg_idx] *
                  hessenberg_iter[hessenberg_idx]) >
                 (0.99 * sqrt(arnoldi_norm[arnoldi_idx]))) {
-                //                reorth_status[reorth_idx].stop(1);
                 reorth_status[reorth_idx].reset();
-                //                reorth_status[k].reset();
-                printf("K = %d , HI = %f , AN = %f\n", k,
-                       hessenberg_iter[hessenberg_idx],
-                       0.99 * sqrt(arnoldi_norm[arnoldi_idx]));
-                //                num_reorth[0] = 1;
-                //                if (!reorth_status[arnoldi_idx].has_stopped())
-                //                    printf ("OK in %d\n", arnoldi_idx);
-                //                else
-                //                    printf ("BAD in %d\n", arnoldi_idx);
                 atomic_add(num_reorth, (size_type)1);
             } else {
-                //                reorth_status[reorth_idx].reset();
                 reorth_status[reorth_idx].stop(1);
-                //                if (reorth_status[arnoldi_idx].has_stopped())
-                //                    printf ("OK in %d\n", arnoldi_idx);
-                //                else
-                //                    printf ("BAD in %d\n", arnoldi_idx);
-                //                reorth_status[k].stop(0);
             }
         }
     }
@@ -300,7 +281,6 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
         next_krylov_basis[next_krylov_idx] -=
             buffer_iter[buffer_idx] * krylov_bases[krylov_idx];
         if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
-            printf(" Add(%d) = %10.5e\n", k, buffer_iter[buffer_idx]);
             hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
         }
     }
@@ -316,32 +296,17 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms(
     const stopping_status *__restrict__ stop_status,
     stopping_status *__restrict__ reorth_status,
     size_type *__restrict__ num_reorth)
-//    stopping_status *__restrict__ reorth_status, size_type *num_reorth)
-//    const ValueType eta, stopping_status *__restrict__ reorth_status)
 {
-    //    const ValueType eta = 1.0 / sqrt(2.0);
     const ValueType eta = 1.0 / 2.0;
     const auto global_id = thread::get_thread_id_flat();
-    //    const auto tidx = threadIdx.x;
-    //    const auto col_idx = blockIdx.x;
 
     if (global_id < stride_norm0 && !stop_status[global_id].has_stopped()) {
         const ValueType num0 = eta * arnoldi_norm0[global_id];
         const ValueType num1 = arnoldi_norm1[global_id];
-        //        if (arnoldi_norm1[global_id] < eta * arnoldi_norm0[global_id])
-        //        {
-        printf(" Reorth(%d) = %10.5e,%10.5e)\n", global_id, num0, num1);
         if (num1 < num0) {
-            //            reorth_status[global_id].stop(1);
             reorth_status[global_id].reset();
             atomic_add(num_reorth, (size_type)1);
-            //            num_reorth[0] = 1;
-            //            printf (" Reorth(%10.5e,%10.5e)\n",
-            //                     eta * arnoldi_norm0[global_id],
-            //                     arnoldi_norm1[global_id]);
         } else {
-            //            num_reorth[0] = 0;
-            //            reorth_status[global_id].reset();
             reorth_status[global_id].stop(1);
         }
     }
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index a3cf99b216f..e88e7a4f06c 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -104,8 +104,7 @@ namespace gmres_mixed {
 /*
     template <typename ValueType, typename ValueTypeKrylovBases, \
               bool Reorthogonalization, bool MGS_CGS>                     \
-    GKO_DECLARE_GMRES_MIXED_BOOL_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases,
-   \
+    GKO_DECLARE_GMRES_MIXED_BOOL_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases,\
                                           Reorthogonalization, MGS_CGS);       \
 */
 
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 712b2134323..137ffb64d8b 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -251,17 +251,6 @@ void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
             stride_next_krylov, as_cuda_type(krylov_bases->get_const_values()),
             stride_krylov, as_cuda_type(hessenberg_iter->get_values()),
             stride_hessenberg, as_cuda_type(stop_status));
-        /*
-                update_next_krylov_kernel<default_block_size>
-                    <<<ceildiv(dim_size[0] * stride_next_krylov,
-           default_block_size), default_block_size>>>( k, dim_size[0],
-           dim_size[1], as_cuda_type(next_krylov_basis->get_values()),
-                        stride_next_krylov,
-                        as_cuda_type(krylov_bases->get_const_values()),
-           stride_krylov, as_cuda_type(hessenberg_iter->get_const_values()),
-                        stride_hessenberg, as_cuda_type(stop_status));
-        */
-        //        num_reorth = 0;
         components::fill_array(exec, num_reorth->get_data(), 1,
                                zero<size_type>());
         update_next_krylov_kernel_and_check<default_block_size>
@@ -273,24 +262,12 @@ void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
                 as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
                 as_cuda_type(hessenberg_iter->get_const_values()),
                 stride_hessenberg, as_cuda_type(arnoldi_norm->get_values()), 0,
-                //                as_cuda_type(stop_status),
-                //                as_cuda_type(reorth_status));
                 as_cuda_type(stop_status), as_cuda_type(reorth_status),
                 as_cuda_type(num_reorth->get_data()));
-        //                as_cuda_type(&num_reorth));
         numReorth = 0;
         exec->get_master()->copy_from(exec.get(), 1,
                                       num_reorth->get_const_data(), &numReorth);
-        std::cout << "K = " << k << " , numReorth => " << numReorth
-                  << std::endl;
-        //        std::cout << "NREORTH = " << num_reorth << std::endl;
-        //        if (num_reorth > 0) {
-        //        if (2 > 1) {
         if (numReorth > 0) {
-            //        if (numReorth < 0) {
-            std::cout << "REORTHOGONALIZATION" << std::endl;
-            std::cout << dim_size[0] << " - " << dim_size[1] << " - "
-                      << stride_buffer << std::endl;
             components::fill_array(
                 exec, buffer_iter->get_values() + k * stride_buffer,
                 dim_size[1], zero<ValueType>());
@@ -301,7 +278,6 @@ void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
                 as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
                 as_cuda_type(buffer_iter->get_values()), stride_buffer,
                 as_cuda_type(stop_status));
-            /**/
             update_next_krylov_kernel_and_add<default_block_size><<<
                 ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
                 default_block_size>>>(
@@ -312,7 +288,6 @@ void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
                 as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
                 as_cuda_type(stop_status), as_cuda_type(reorth_status));
-            /**/
         }
     }
     // for i in 1:iter
@@ -346,7 +321,7 @@ void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
     // End of arnoldi
 }
 
-/**/
+
 template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
                         matrix::Dense<ValueType> *next_krylov_basis,
@@ -368,8 +343,8 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
     const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
                          exec->get_num_multiprocessor() * 2);
     const dim3 block_size(default_dot_dim, default_dot_dim);
-    //    size_type num_reorth;
     size_type numReorth;
+
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<ValueType>());
     multidot_kernel<<<grid_size, block_size>>>(
@@ -414,35 +389,18 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
         as_cuda_type(stop_status));
-    // nrmN = norm(next_krylov_basis
-    // const ValueType eta = 1.0 / sqrt(2.0);
-    //    components::fill_array(exec, num_reorth->get_values(), 1,
-    //    zero<size_type>()); components::fill_array(exec,
-    //    num_reorth->get_data(), 1, zero<size_type>()); num_reorth->get_data[0]
-    //    = 0;
     components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
     check_arnoldi_norms<default_block_size>
         <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
             as_cuda_type(arnoldi_norm->get_const_values()), stride_arnoldi,
             as_cuda_type(arnoldi_norm->get_const_values() + stride_arnoldi),
-            stride_arnoldi,
-            //            as_cuda_type(stop_status),
-            //            as_cuda_type(reorth_status));
-            as_cuda_type(stop_status), as_cuda_type(reorth_status),
-            as_cuda_type(num_reorth->get_data()));
-    //            as_cuda_type(num_reorth->get_values()));
-    //        eta, as_cuda_type(reorth_status));
+            stride_arnoldi, as_cuda_type(stop_status),
+            as_cuda_type(reorth_status), as_cuda_type(num_reorth->get_data()));
     numReorth = 0;
     exec->get_master()->copy_from(exec.get(), 1, num_reorth->get_const_data(),
                                   &numReorth);
-    std::cout << " numReorth => " << numReorth << std::endl;
-    // for (size_type l = 1; (l < 3); l++) {
+    // numReorth <= number of next_krylov vector to be reorthogonalization
     for (size_type l = 1; (numReorth > 0) && (l < 3); l++) {
-        //    for (size_type l = 1; (num_reorth->get_data[0]>0) && (l < 3); l++)
-        //    { for (size_type l = 1; (num_reorth->get_values[0]>0) && (l < 3);
-        //    l++) {
-        //             arnoldi_norm->at(1, i) < arnoldi_norm->at(0, i) && l < 3;
-        //             l++) {
         for (size_type k = 0; k < iter + 1; ++k) {
             components::fill_array(
                 exec, buffer_iter->get_values() + k * stride_buffer,
@@ -484,16 +442,7 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
             stride_next_krylov,
             as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
             as_cuda_type(stop_status));
-        // nrmN = norm(next_krylov_basis
-        //        std::cout << " XX => " << (l-1) << " - " << (2-l) <<
-        //        std::endl;
-        std::cout << " XX => " << l << " - " << (2 - l) << " - " << (l - 1)
-                  << std::endl;
-        //        num_reorth = 0;
-        //        components::fill_array(exec, num_reorth->get_values(), 1,
-        //        zero<size_type>()); components::fill_array(exec,
-        //        num_reorth->get_data(), 1, zero<size_type>());
-        //        num_reorth->get_data[0] = 0;
+        // nrmN = norm(next_krylov_basis)
         components::fill_array(exec, num_reorth->get_data(), 1,
                                zero<size_type>());
         check_arnoldi_norms<default_block_size>
@@ -503,19 +452,13 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
                 stride_arnoldi,
                 as_cuda_type(arnoldi_norm->get_const_values() +
                              stride_arnoldi * (l - 1)),
-                stride_arnoldi,
-                //                as_cuda_type(stop_status),
-                //                as_cuda_type(reorth_status));
-                as_cuda_type(stop_status), as_cuda_type(reorth_status),
+                stride_arnoldi, as_cuda_type(stop_status),
+                as_cuda_type(reorth_status),
                 as_cuda_type(num_reorth->get_data()));
-        //                as_cuda_type(num_reorth->get_values()));
-        //        eta, as_cuda_type(reorth_status));
         numReorth = 0;
         exec->get_master()->copy_from(exec.get(), 1,
                                       num_reorth->get_const_data(), &numReorth);
-        std::cout << " L = " << l << " , numReorth => " << numReorth
-                  << std::endl;
-        //        std::cout << " num_reorth => " << num_reorth << std::endl;
+        // numReorth <= number of next_krylov vector to be reorthogonalization
     }
 
 
@@ -585,7 +528,6 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
             const Array<stopping_status> *stop_status,
             Array<stopping_status> *reorth_status, Array<size_type> *num_reorth)
 {
-    std::cout << "ITER = " << iter << std::endl;
     increase_final_iteration_numbers_kernel<<<
         static_cast<unsigned int>(
             ceildiv(final_iter_nums->get_num_elems(), default_block_size)),
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index c930efad7fd..1a2c47c9570 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -160,9 +160,6 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
             }
             if (hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i) >
                 arnoldi_norm->at(0, i)) {
-                std::cout << "K = " << k;
-                std::cout << " , HI = " << hessenberg_iter->at(k, i);
-                std::cout << " , AN = " << arnoldi_norm->at(0, i) << std::endl;
                 ValueType reorth = zero<ValueType>();
 #pragma omp parallel for reduction(add : reorth)
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
@@ -575,7 +572,6 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
             const Array<stopping_status> *stop_status,
             Array<stopping_status> *reorth_status, Array<size_type> *num_reorth)
 {
-    std::cout << "ITER = " << iter << std::endl;
 #pragma omp parallel for
     for (size_type i = 0; i < final_iter_nums->get_num_elems(); ++i) {
         final_iter_nums->get_data()[i] +=
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index bd015bc2f2a..cca64c738cb 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -145,16 +145,9 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
             }
             if ((hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i)) >
                 arnoldi_norm->at(0, i)) {
-                std::cout << "K = " << k;
-                std::cout << " , HI = " << hessenberg_iter->at(k, i);
-                std::cout << " , AN = " << arnoldi_norm->at(0, i) << std::endl;
-                // if (hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i) >
-                // nrm) {
                 arnoldi_norm->at(1, i) = 0;
-                // reorth = 0;
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
-                    // reorth += next_krylov_basis->at(j, i) *
                     arnoldi_norm->at(1, i) +=
                         next_krylov_basis->at(j, i) *
                         krylov_bases->at(
@@ -166,12 +159,8 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
                         arnoldi_norm->at(1, i) *
                         krylov_bases->at(
                             j, next_krylov_basis->get_size()[1] * k + i);
-                    // reorth *
                 }
                 hessenberg_iter->at(k, i) += arnoldi_norm->at(1, i);
-                std::cout << " Add(" << k << ") = " << arnoldi_norm->at(1, i)
-                          << std::endl;
-                //                hessenberg_iter->at(k, i) += reorth;
             }
         }
         // for i in 1:iter
@@ -238,8 +227,6 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
                     krylov_bases->at(j,
                                      next_krylov_basis->get_size()[1] * k + i);
             }
-            //            std::cout << "  " << k << " => " <<
-            //            hessenberg_iter->at(k, i) << std::endl;
         }
         // for i in 1:iter
         //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
@@ -256,28 +243,15 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
         arnoldi_norm->at(1, i) = 0;
-        // nrmN = 0;
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             arnoldi_norm->at(1, i) +=
                 next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
-            // nrmN += next_krylov_basis->at(j, i) * next_krylov_basis->at(j,
-            // i);
         }
         arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
-        // nrmN = sqrt(nrmN);
-        // nrmN = arnoldi_norm->at(1, i)(next_krylov_basis)
-        printf("Reorth(%10.5e,%10.5e)\n",
-               arnoldi_norm->at(0, i) * arnoldi_norm->at(0, i),
-               arnoldi_norm->at(1, i) * arnoldi_norm->at(1, i));
+        // nrmN = norm(next_krylov_basis)
         for (size_type l = 1;
              arnoldi_norm->at(1, i) < arnoldi_norm->at(0, i) && l < 3; l++) {
-            // for (size_type l = 1; arnoldi_norm->at(1, i) < eta *
-            // arnoldi_norm->at(0, i) && l < 3; l++) { for (size_type l = 1;
-            // nrmN < eta * nrmP && l < 3; l++) {
-            //            std::cout << l << " => " << nrmN << " - " << eta << "
-            //            - " << nrmP << std::endl;
             arnoldi_norm->at(0, i) = eta * arnoldi_norm->at(1, i);
-            // nrmP = nrmN;
             for (size_type k = 0; k < iter + 1; ++k) {
                 buffer_iter->at(k, i) = 0;
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
@@ -299,9 +273,6 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
                         krylov_bases->at(
                             j, next_krylov_basis->get_size()[1] * k + i);
                 }
-                //                std::cout << "  " << k << " => " <<
-                //                hessenberg_iter->at(k, i); std::cout << " - "
-                //                << buffer_iter->at(k, i) << std::endl;
                 hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
             }
             // for i in 1:iter
@@ -309,21 +280,13 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
             //     hessenberg(iter, i) += buffer(iter, i)
             // end
             arnoldi_norm->at(1, i) = 0;
-            // nrmN = 0;
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 arnoldi_norm->at(1, i) +=
-                    // nrmN +=
                     next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
             }
             arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
-            // nrmN = sqrt(nrmN);
             // nrmN = norm(next_krylov_basis)
-            printf("XX Reorth(%10.5e,%10.5e)\n",
-                   arnoldi_norm->at(0, i) * arnoldi_norm->at(0, i),
-                   arnoldi_norm->at(1, i) * arnoldi_norm->at(1, i));
         }
-        //        std::cout << nrmN << " - " << eta << " - " << nrmP <<
-        //        std::endl;
         // reorthogonalization
         hessenberg_iter->at(iter + 1, i) = 0;
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
@@ -332,7 +295,6 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
         }
         hessenberg_iter->at(iter + 1, i) =
             sqrt(hessenberg_iter->at(iter + 1, i));
-        //        std::cout << hessenberg_iter->at(iter + 1, i) << std::endl;
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
@@ -576,7 +538,6 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
         if (MGS_CGS) std::cout << "Modified Gram-Schmidt";
         else  std::cout << "Classical Gram-Schmidt";
     */
-    std::cout << "ITER = " << iter << std::endl;
     for (size_type i = 0; i < final_iter_nums->get_num_elems(); ++i) {
         final_iter_nums->get_data()[i] +=
             (1 - stop_status->get_const_data()[i].has_stopped());

From ab7e91fdbb3394e18ca872e2d6980dc2cbb1aa1a Mon Sep 17 00:00:00 2001
From: josealiaga <aliaga@uji.es>
Date: Thu, 30 Apr 2020 22:45:01 +0200
Subject: [PATCH 0338/1008] The default value for ValueTypeKrylovBasis has been
 changed to avoid some errors which were detected during the testing process
 in the repository. The previous value was float whereas the original results
 were executed by default_precision, and these are the reason of the errors.
 Now, the default value is also default_precision.

---
 include/ginkgo/core/solver/gmres_mixed.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/ginkgo/core/solver/gmres_mixed.hpp b/include/ginkgo/core/solver/gmres_mixed.hpp
index d891c86b7bd..bb466919c09 100644
--- a/include/ginkgo/core/solver/gmres_mixed.hpp
+++ b/include/ginkgo/core/solver/gmres_mixed.hpp
@@ -75,8 +75,8 @@ template <typename ValueType = default_precision,
           bool MGS_CGS = true>
 */
 template <typename ValueType = default_precision,
-          typename ValueTypeKrylovBases = float>
-//          typename ValueTypeKrylovBases = default_precision>
+          typename ValueTypeKrylovBases = default_precision>
+//          typename ValueTypeKrylovBases = float>
 class GmresMixed :
     //		public EnableLinOp<GmresMixed<ValueType,
     // ValueTypeKrylovBases,Reorthogonalization, MGS_CGS>>,

From 0da1b9a2e0fced24bf7d127f6d974abf4e19e10b Mon Sep 17 00:00:00 2001
From: josealiaga <aliaga@uji.es>
Date: Sun, 3 May 2020 23:42:27 +0200
Subject: [PATCH 0339/1008] Definition of the CG2 variant of the finish_arnoldi
 routine for omp and cuda executors, as a first step in the optimization
 process. Also the calls for the timing are included.

---
 common/solver/gmres_mixed_kernels.hpp.inc     |  69 +++++
 core/solver/gmres_mixed.cpp                   |  16 +
 cuda/solver/gmres_mixed_kernels.cu            | 212 ++++++++++++-
 examples/gmres-mixed-solver/CMakeLists.txt    |   4 +
 .../gmres-mixed-solver-jacobi-matrix.cpp      | 283 ++++++++++++++++++
 omp/solver/gmres_mixed_kernels.cpp            | 130 +++++++-
 6 files changed, 707 insertions(+), 7 deletions(-)
 create mode 100644 examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index fb2cf313057..604f3711ce8 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -30,6 +30,21 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void zero_matrix_kernel(
+    size_type m, size_type n, size_type stride, ValueType *__restrict__ array)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    if (tidx < n) {
+        auto pos = tidx;
+        for (size_type k = 0; k < m; ++k) {
+            array[pos] = zero<ValueType>();
+            pos += stride;
+        }
+    }
+}
+
+
 // Must be called with at least `max(stride_b * num_rows, krylov_dim *
 // num_cols)` threads in total.
 template <size_type block_size, typename ValueType>
@@ -183,6 +198,60 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
 }
 
 
+template <typename ValueType, typename ValueTypeKrylovBases>
+__global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
+    size_type num_iters, size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov,
+    const ValueTypeKrylovBases *__restrict__ krylov_bases,
+    size_type stride_krylov, ValueType *__restrict__ hessenberg_iter,
+    size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto tidx = threadIdx.x;
+    const auto tidy = threadIdx.y;
+    const auto col_idx = blockIdx.x * default_dot_dim + tidx;
+    const auto num = ceildiv(num_rows, gridDim.y);
+    const auto start_row = blockIdx.y * num;
+    const auto end_row =
+        ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+    __shared__
+        UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
+            reduction_helper_array;
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
+
+    for (size_type k = 0; k < num_iters; ++k) {
+        ValueType local_res = zero<ValueType>();
+        const auto krylov_col = k * num_cols + col_idx;
+        if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+            for (size_type i = start_row + tidy; i < end_row;
+                 i += default_dot_dim) {
+                const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+                const auto krylov_idx = i * stride_krylov + krylov_col;
+                local_res += next_krylov_basis[next_krylov_idx] *
+                             krylov_bases[krylov_idx];
+            }
+        }
+        reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
+        __syncthreads();
+        local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
+        const auto tile_block =
+            group::tiled_partition<default_dot_dim>(group::this_thread_block());
+        const auto sum = reduce(
+            tile_block, local_res,
+            [](const ValueType &a, const ValueType &b) { return a + b; });
+        const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
+        if (tidx == 0 && new_col_idx < num_cols &&
+            !stop_status[new_col_idx].has_stopped()) {
+            const auto hessenberg_idx = k * stride_hessenberg + new_col_idx;
+            atomic_add(hessenberg_iter + hessenberg_idx, sum);
+        }
+    }
+}
+
+
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
 template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 892604af9e4..1500a948f8f 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -44,9 +44,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/identity.hpp>
 
 
+#include <iostream>
+
+
 #include "core/solver/gmres_mixed_kernels.hpp"
 
 
+using double_seconds = std::chrono::duration<double>;
+
+
 namespace gko {
 namespace solver {
 
@@ -155,6 +161,8 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     auto after_preconditioner =
         matrix::Dense<ValueType>::create_with_config_of(dense_x);
 
+    auto start = std::chrono::steady_clock::now();
+
     while (true) {
         ++total_iter;
         this->template log<log::Logger::iteration_complete>(
@@ -169,6 +177,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
         }
 
         if (restart_iter == krylov_dim_mixed_) {
+            // std::cout << "Restart" << std::endl;
             // Restart
             exec->run(gmres_mixed::make_step_2(
                 residual_norm_collection.get(), krylov_bases.get(),
@@ -269,6 +278,13 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     dense_x->add_scaled(one_op.get(), after_preconditioner.get());
     // Solve x
     // x = x + get_preconditioner() * krylov_bases * y
+
+    auto time = std::chrono::steady_clock::now() - start;
+    std::cout << "total_iter = " << total_iter << std::endl;
+    std::cout << "time = "
+              << std::chrono::duration_cast<double_seconds>(time).count()
+              << std::endl;
+    write(std::cout, lend(residual_norm));
 }
 
 
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 137ffb64d8b..8af56d11064 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -78,6 +78,16 @@ constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 #include "common/solver/gmres_mixed_kernels.hpp.inc"
 
 
+template <typename ValueType>
+void zero_matrix(size_type m, size_type n, size_type stride, ValueType *array)
+{
+    const dim3 block_size(default_block_size, 1, 1);
+    const dim3 grid_size(ceildiv(n, block_size.x), 1, 1);
+    zero_matrix_kernel<<<grid_size, block_size, 0, 0>>>(m, n, stride,
+                                                        as_cuda_type(array));
+}
+
+
 template <typename ValueType>
 void initialize_1(std::shared_ptr<const CudaExecutor> exec,
                   const matrix::Dense<ValueType> *b,
@@ -482,6 +492,195 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
 }
 /**/
 
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
+                         matrix::Dense<ValueType> *next_krylov_basis,
+                         matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                         matrix::Dense<ValueType> *hessenberg_iter,
+                         matrix::Dense<ValueType> *buffer_iter,
+                         matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+                         const stopping_status *stop_status,
+                         stopping_status *reorth_status,
+                         Array<size_type> *num_reorth)
+{
+    const auto stride_next_krylov = next_krylov_basis->get_stride();
+    const auto stride_krylov = krylov_bases->get_stride();
+    const auto stride_hessenberg = hessenberg_iter->get_stride();
+    const auto stride_buffer = buffer_iter->get_stride();
+    const auto stride_arnoldi = arnoldi_norm->get_stride();
+    const auto dim_size = next_krylov_basis->get_size();
+    auto cublas_handle = exec->get_cublas_handle();
+    const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
+                         exec->get_num_multiprocessor() * 2);
+    const dim3 block_size(default_dot_dim, default_dot_dim);
+    size_type numReorth;
+
+    components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
+                           zero<ValueType>());
+    multidot_kernel<<<grid_size, block_size>>>(
+        0, dim_size[0], dim_size[1],
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
+    // nrmP = norm(next_krylov_basis
+    /*
+        for (size_type k = 0; k < iter + 1; ++k) {
+            components::fill_array(exec, hessenberg_iter->get_values() + k *
+       stride_hessenberg, dim_size[1], zero<ValueType>());
+            multidot_kernel<<<grid_size, block_size>>>(
+                k, dim_size[0], dim_size[1],
+                as_cuda_type(next_krylov_basis->get_const_values()),
+                stride_next_krylov,
+       as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+       as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+       as_cuda_type(stop_status));
+        }
+    */
+    /*
+    for (size_type k = 0; k < iter + 1; ++k) {
+        components::fill_array(exec,
+            hessenberg_iter->get_values() + k * stride_hessenberg, dim_size[1],
+            zero<ValueType>());
+    }
+    */
+    zero_matrix(iter + 1, dim_size[1], stride_hessenberg,
+                hessenberg_iter->get_values());
+    multidot_kernel_num_iters<<<grid_size, block_size>>>(
+        iter + 1, dim_size[0], dim_size[1],
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+        as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+        as_cuda_type(stop_status));
+    // for i in 1:iter
+    //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+    // end
+    for (size_type k = 0; k < iter + 1; ++k) {
+        update_next_krylov_kernel<default_block_size>
+            <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+               default_block_size>>>(
+                k, dim_size[0], dim_size[1],
+                as_cuda_type(next_krylov_basis->get_values()),
+                stride_next_krylov,
+                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                as_cuda_type(hessenberg_iter->get_const_values()),
+                stride_hessenberg, as_cuda_type(stop_status));
+    }
+    // for i in 1:iter
+    //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+    // end
+    components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
+                           dim_size[1], zero<ValueType>());
+    multidot_kernel<<<grid_size, block_size>>>(
+        0, dim_size[0], dim_size[1],
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
+        as_cuda_type(stop_status));
+    // nrmN = norm(next_krylov_basis)
+    components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
+    check_arnoldi_norms<default_block_size>
+        <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
+            as_cuda_type(arnoldi_norm->get_const_values()), stride_arnoldi,
+            as_cuda_type(arnoldi_norm->get_const_values() + stride_arnoldi),
+            stride_arnoldi, as_cuda_type(stop_status),
+            as_cuda_type(reorth_status), as_cuda_type(num_reorth->get_data()));
+    numReorth = 0;
+    exec->get_master()->copy_from(exec.get(), 1, num_reorth->get_const_data(),
+                                  &numReorth);
+    // numReorth <= number of next_krylov vector to be reorthogonalization
+    for (size_type l = 1; (numReorth > 0) && (l < 3); l++) {
+        /*
+        for (size_type k = 0; k < iter + 1; ++k) {
+            components::fill_array(exec,
+                                   buffer_iter->get_values() + k *
+        stride_buffer, dim_size[1], zero<ValueType>());
+            multidot_kernel<<<grid_size, block_size>>>(
+                k, dim_size[0], dim_size[1],
+                as_cuda_type(next_krylov_basis->get_const_values()),
+                stride_next_krylov,
+                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                as_cuda_type(buffer_iter->get_values()), stride_buffer,
+                as_cuda_type(stop_status));
+        }
+        */
+        zero_matrix(iter + 1, dim_size[1], stride_buffer,
+                    buffer_iter->get_values());
+        multidot_kernel_num_iters<<<grid_size, block_size>>>(
+            iter + 1, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov, as_cuda_type(krylov_bases->get_const_values()),
+            stride_krylov, as_cuda_type(buffer_iter->get_values()),
+            stride_buffer, as_cuda_type(stop_status));
+        // for i in 1:iter
+        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+        // end
+        for (size_type k = 0; k < iter + 1; ++k) {
+            update_next_krylov_kernel_and_add<default_block_size><<<
+                ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+                default_block_size>>>(
+                k, dim_size[0], dim_size[1],
+                as_cuda_type(next_krylov_basis->get_values()),
+                stride_next_krylov,
+                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+                as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
+                as_cuda_type(stop_status), as_cuda_type(reorth_status));
+        }
+        // for i in 1:iter
+        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+        // end
+        components::fill_array(
+            exec, arnoldi_norm->get_values() + dim_size[1] * (l - 1),
+            dim_size[1], zero<ValueType>());
+        multidot_kernel<<<grid_size, block_size>>>(
+            0, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov,
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov,
+            as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
+            as_cuda_type(stop_status));
+        // nrmN = norm(next_krylov_basis)
+        components::fill_array(exec, num_reorth->get_data(), 1,
+                               zero<size_type>());
+        check_arnoldi_norms<default_block_size>
+            <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
+                as_cuda_type(arnoldi_norm->get_const_values() +
+                             stride_arnoldi * (2 - l)),
+                stride_arnoldi,
+                as_cuda_type(arnoldi_norm->get_const_values() +
+                             stride_arnoldi * (l - 1)),
+                stride_arnoldi, as_cuda_type(stop_status),
+                as_cuda_type(reorth_status),
+                as_cuda_type(num_reorth->get_data()));
+        numReorth = 0;
+        exec->get_master()->copy_from(exec.get(), 1,
+                                      num_reorth->get_const_data(), &numReorth);
+        // numReorth <= number of next_krylov vector to be reorthogonalization
+    }
+
+
+    update_hessenberg_2_kernel<default_block_size>
+        <<<dim_size[1], default_block_size>>>(
+            iter, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov, as_cuda_type(hessenberg_iter->get_values()),
+            stride_hessenberg, as_cuda_type(stop_status));
+    update_krylov_next_krylov_kernel<default_block_size>
+        <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+           default_block_size>>>(
+            iter, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
+            as_cuda_type(krylov_bases->get_values()), stride_krylov,
+            as_cuda_type(hessenberg_iter->get_const_values()),
+            stride_hessenberg, as_cuda_type(stop_status));
+    // next_krylov_basis /= hessenberg(iter, iter + 1)
+    // krylov_bases(:, iter + 1) = next_krylov_basis
+    // End of arnoldi
+}
+/**/
+
 template <typename ValueType>
 void givens_rotation(std::shared_ptr<const CudaExecutor> exec,
                      matrix::Dense<ValueType> *givens_sin,
@@ -540,10 +739,15 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
     //                          hessenberg_iter, buffer_iter, arnoldi_norm,
     //                          iter, stop_status->get_const_data(),
     //                          reorth_status->get_data(), num_reorth);
-    finish_arnoldi_CGS(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
-                       buffer_iter, arnoldi_norm, iter,
-                       stop_status->get_const_data(), reorth_status->get_data(),
-                       num_reorth);
+    //    finish_arnoldi_CGS(exec, next_krylov_basis, krylov_bases,
+    //    hessenberg_iter,
+    //                       buffer_iter, arnoldi_norm, iter,
+    //                       stop_status->get_const_data(),
+    //                       reorth_status->get_data(), num_reorth);
+    finish_arnoldi_CGS2(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
+                        buffer_iter, arnoldi_norm, iter,
+                        stop_status->get_const_data(),
+                        reorth_status->get_data(), num_reorth);
     givens_rotation(exec, givens_sin, givens_cos, hessenberg_iter,
                     residual_norm, residual_norm_collection, b_norm, iter,
                     stop_status);
diff --git a/examples/gmres-mixed-solver/CMakeLists.txt b/examples/gmres-mixed-solver/CMakeLists.txt
index 8871978b11e..467dfff7d96 100644
--- a/examples/gmres-mixed-solver/CMakeLists.txt
+++ b/examples/gmres-mixed-solver/CMakeLists.txt
@@ -4,6 +4,10 @@ target_include_directories(gmres-mixed-solver PRIVATE ${PROJECT_SOURCE_DIR})
 add_executable(gmres-mixed-solver-jacobi gmres-mixed-solver-jacobi.cpp)
 target_link_libraries(gmres-mixed-solver-jacobi ginkgo)
 target_include_directories(gmres-mixed-solver-jacobi PRIVATE ${PROJECT_SOURCE_DIR})
+add_executable(gmres-mixed-solver-jacobi-matrix
+gmres-mixed-solver-jacobi-matrix.cpp)
+target_link_libraries(gmres-mixed-solver-jacobi-matrix ginkgo)
+target_include_directories(gmres-mixed-solver-jacobi-matrix PRIVATE ${PROJECT_SOURCE_DIR})
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp b/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
new file mode 100644
index 00000000000..bab8cf37cc8
--- /dev/null
+++ b/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
@@ -0,0 +1,283 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+// @sect3{Include files}
+
+// This is the main ginkgo header file.
+#include <ginkgo/ginkgo.hpp>
+//#include "cuda/components/zero_array.hpp"
+
+// Add the fstream header to read from data from files.
+#include <fstream>
+// Add the C++ iostream header to output information to the console.
+#include <iostream>
+// Add the string manipulation header to handle strings.
+#include <string>
+
+
+int main(int argc, char *argv[])
+{
+    // Use some shortcuts. In Ginkgo, vectors are seen as a gko::matrix::Dense
+    // with one column/one row. The advantage of this concept is that using
+    // multiple vectors is a now a natural extension of adding columns/rows are
+    // necessary.
+    using vec = gko::matrix::Dense<>;
+    // The gko::matrix::Csr class is used here, but any other matrix class such
+    // as gko::matrix::Coo, gko::matrix::Hybrid, gko::matrix::Ell or
+    // gko::matrix::Sellp could also be used.
+    using mtx = gko::matrix::Csr<>;
+    // The gko::solver::Cg is used here, but any other solver class can also be
+    // used.
+    //    using gmres_mixed = gko::solver::GmresMixed<double, float>;
+    using gmres_mixed = gko::solver::GmresMixed<>;
+    using bj = gko::preconditioner::Jacobi<>;
+
+    // Print the ginkgo version information.
+    std::cout << gko::version_info::get() << std::endl;
+
+    // @sect3{Where do you want to run your solver ?}
+    // The gko::Executor class is one of the cornerstones of Ginkgo. Currently,
+    // we have support for
+    // an gko::OmpExecutor, which uses OpenMP multi-threading in most of its
+    // kernels, a gko::ReferenceExecutor, a single threaded specialization of
+    // the OpenMP executor and a gko::CudaExecutor which runs the code on a
+    // NVIDIA GPU if available.
+    // @note With the help of C++, you see that you only ever need to change the
+    // executor and all the other functions/ routines within Ginkgo should
+    // automatically work and run on the executor with any other changes.
+    /*
+        std::shared_ptr<gko::Executor> exec;
+        if (argc == 1 || std::string(argv[1]) == "reference") {
+            exec = gko::ReferenceExecutor::create();
+        } else if (argc == 2 && std::string(argv[1]) == "omp") {
+            exec = gko::OmpExecutor::create();
+        } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
+                   gko::CudaExecutor::get_num_devices() > 0) {
+            exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+        } else if (argc == 2 && std::string(argv[1]) == "hip" &&
+                   gko::HipExecutor::get_num_devices() > 0) {
+            exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+        } else {
+            std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
+            std::exit(-1);
+        }
+    */
+    int index_matrix = -1;
+    std::shared_ptr<gko::Executor> exec;
+    if (argc == 1 || std::string(argv[1]) == "reference") {
+        exec = gko::ReferenceExecutor::create();
+        if (argc == 3) index_matrix = 2;
+    } else if (argc <= 3 && std::string(argv[1]) == "omp") {
+        exec = gko::OmpExecutor::create();
+        if (argc == 3) index_matrix = 2;
+    } else if (argc <= 3 && std::string(argv[1]) == "cuda" &&
+               gko::CudaExecutor::get_num_devices() > 0) {
+        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
+        if (argc == 3) index_matrix = 2;
+    } else if (argc <= 3 && std::string(argv[1]) == "hip" &&
+               gko::HipExecutor::get_num_devices() > 0) {
+        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
+        if (argc == 3) index_matrix = 2;
+    } else if (argc == 2) {
+        exec = gko::ReferenceExecutor::create();
+        index_matrix = 1;
+    } else {
+        std::cerr << "Usage: " << argv[0] << " [executor] [matrix]"
+                  << std::endl;
+        std::exit(-1);
+    }
+    std::cout << "index_matrix -> " << index_matrix << std::endl;
+
+    // @sect3{Reading your data and transfer to the proper device.}
+    // Read the matrix, right hand side and the initial solution using the @ref
+    // read function.
+    // @note Ginkgo uses C++ smart pointers to automatically manage memory. To
+    // this end, we use our own object ownership transfer functions that under
+    // the hood call the required smart pointer functions to manage object
+    // ownership. The gko::share , gko::give and gko::lend are the functions
+    // that you would need to use.
+    auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
+    auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
+    auto x = gko::read<vec>(std::ifstream("data/x0.mtx"), exec);
+    if (index_matrix > 0) {
+        double aux1, aux2;
+
+        std::cout << index_matrix << std::endl;
+        std::cout << index_matrix << " -> " << std::string(argv[index_matrix])
+                  << std::endl;
+        A = share(gko::read<mtx>(std::ifstream(argv[index_matrix]), exec));
+        auto sizesA = A->get_size();
+        std::cout << sizesA[0] << " - " << sizesA[1] << std::endl;
+        if (std::string(argv[1]) == "cuda") {
+            exec->get_master()->copy_from(exec.get(), 1, A->get_values(),
+                                          &aux1);
+            exec->get_master()->copy_from(
+                exec.get(), 1,
+                A->get_values() + A->get_num_stored_elements() - 1, &aux2);
+            std::cout << "A0 => " << aux1 << " - " << aux2 << std::endl;
+        } else {
+            std::cout << "A0 => " << A->get_values()[0];
+            std::cout << A->get_values()[A->get_num_stored_elements() - 1]
+                      << std::endl;
+        }
+
+        double aux[sizesA[0]];
+
+        //        x = gko::read<vec>(std::ifstream("data/one.mtx"), exec);
+        if (std::string(argv[1]) == "cuda") {
+            x = gko::read<vec>(std::ifstream("data/one.mtx"), exec);
+            //            gko::kernels::cuda::zero_array(sizesA[1],
+            //            b->get_values()); for (int i = 0; i<sizesA[0]; i++)
+            //            aux[i] = 1.0; x->copy_from(aux);
+            exec->get_master()->copy_from(exec.get(), 1, x->get_values(),
+                                          &aux1);
+            exec->get_master()->copy_from(
+                exec.get(), 1, x->get_values() + sizesA[1] - 1, &aux2);
+            std::cout << "X0 => " << aux1 << " - " << aux2 << std::endl;
+        } else {
+            x = vec::create(exec, gko::dim<2>{sizesA[0], 1});
+            for (int i = 0; i < sizesA[0]; i++) x->at(i) = 1.0;
+            std::cout << "X0 => " << x->at(0) << " - " << x->at(sizesA[0] - 1)
+                      << std::endl;
+        }
+        auto sizesx = x->get_size();
+        //        std::cout << "SIZEX -> " << sizesx[0] << " - " << sizesx[1] <<
+        //        std::endl;
+
+        //        b = gko::read<vec>(std::ifstream("data/zero.mtx"), exec);
+        if (std::string(argv[1]) == "cuda") {
+            b = gko::read<vec>(std::ifstream("data/zero.mtx"), exec);
+            exec->get_master()->copy_from(exec.get(), 1, b->get_values(),
+                                          &aux1);
+            exec->get_master()->copy_from(
+                exec.get(), 1, b->get_values() + sizesA[1] - 1, &aux2);
+            //            std::cout << "B0 => " << aux1 << " - " << aux2 <<
+            //            std::endl;
+        } else {
+            b = vec::create(exec, gko::dim<2>{sizesA[0], 1});
+            for (int i = 0; i < sizesA[0]; i++) b->at(i) = 0.0;
+            //            std::cout << "B0 => " << b->at(0) << " - " <<
+            //            b->at(sizesA[0]-1) << std::endl;
+        }
+        auto sizesb = b->get_size();
+        //        std::cout << "SIZEB -> " << sizesb[0] << " - " << sizesb[1] <<
+        //        std::endl;
+
+        A->apply(lend(x), lend(b));
+        if (std::string(argv[1]) == "cuda") {
+            exec->get_master()->copy_from(exec.get(), 1, b->get_values(),
+                                          &aux1);
+            exec->get_master()->copy_from(
+                exec.get(), 1, b->get_values() + sizesA[1] - 1, &aux2);
+            std::cout << "B1 => " << aux1 << " - " << aux2 << std::endl;
+        } else {
+            std::cout << "B1 => " << b->at(0) << " - " << b->at(sizesA[0] - 1)
+                      << std::endl;
+        }
+
+        //        x = gko::read<vec>(std::ifstream("data/zero.mtx"), exec);
+        if (std::string(argv[1]) == "cuda") {
+            x = gko::read<vec>(std::ifstream("data/zero.mtx"), exec);
+            exec->get_master()->copy_from(exec.get(), 1, x->get_values(),
+                                          &aux1);
+            exec->get_master()->copy_from(
+                exec.get(), 1, x->get_values() + sizesA[1] - 1, &aux2);
+            //            std::cout << "X0 => " << aux1 << " - " << aux2 <<
+            //            std::endl;
+        } else {
+            x = vec::create(exec, gko::dim<2>{sizesA[0], 1});
+            for (int i = 0; i < sizesA[0]; i++) x->at(i) = 0.0;
+            //            std::cout << "X0 => " << x->at(0) << " - " <<
+            //            x->at(sizesA[0]-1) << std::endl;
+        }
+    }
+    // @sect3{Creating the solver}
+    // Generate the gko::solver factory. Ginkgo uses the concept of Factories to
+    // build solvers with certain
+    // properties. Observe the Fluent interface used here. Here a gmres_mixed
+    // solver is generated with a stopping criteria of maximum iterations of 20
+    // and a residual norm reduction of 1e-15. You also observe that the
+    // stopping criteria(gko::stop) are also generated from factories using
+    // their build methods. You need to specify the executors which each of the
+    // object needs to be built on.
+    auto solver_gen =
+        gmres_mixed::build()
+            .with_criteria(
+                //                gko::stop::Iteration::build().with_max_iters(20u).on(exec),
+                gko::stop::Iteration::build()
+                    .with_max_iters(A->get_size()[0])
+                    .on(exec),
+                //                gko::stop::Iteration::build().with_max_iters(100000u).on(exec),
+                gko::stop::ResidualNormReduction<>::build()
+                    //                    .with_reduction_factor(1e-15)
+                    .with_reduction_factor(1e-12)
+                    .on(exec))
+            // Add preconditioner, these 2 lines are the only
+            // difference from the simple solver example
+            //            .with_preconditioner(bj::build().with_max_block_size(8u).on(exec))
+            .with_preconditioner(bj::build().with_max_block_size(1u).on(exec))
+            .on(exec);
+    // Generate the solver from the matrix. The solver factory built in the
+    // previous step takes a "matrix"(a gko::LinOp to be more general) as an
+    // input. In this case we provide it with a full matrix that we previously
+    // read, but as the solver only effectively uses the apply() method within
+    // the provided "matrix" object, you can effectively create a gko::LinOp
+    // class with your own apply implementation to accomplish more tasks. We
+    // will see an example of how this can be done in the custom-matrix-format
+    // example
+    auto solver = solver_gen->generate(A);
+
+    // Finally, solve the system. The solver, being a gko::LinOp, can be applied
+    // to a right hand side, b to
+    // obtain the solution, x.
+    solver->apply(lend(b), lend(x));
+
+    // Print the solution to the command line.
+    //    std::cout << "Solution (x): \n";
+    //    write(std::cout, lend(x));
+
+    // To measure if your solution has actually converged, you can measure the
+    // error of the solution.
+    // one, neg_one are objects that represent the numbers which allow for a
+    // uniform interface when computing on any device. To compute the residual,
+    // all you need to do is call the apply method, which in this case is an
+    // spmv and equivalent to the LAPACK z_spmv routine. Finally, you compute
+    // the euclidean 2-norm with the compute_norm2 function.
+    auto one = gko::initialize<vec>({1.0}, exec);
+    auto neg_one = gko::initialize<vec>({-1.0}, exec);
+    auto res = gko::initialize<vec>({0.0}, exec);
+    A->apply(lend(one), lend(x), lend(neg_one), lend(b));
+    b->compute_norm2(lend(res));
+
+    std::cout << "Residual norm sqrt(r^T r): \n";
+    write(std::cout, lend(res));
+}
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index 1a2c47c9570..29b7c52f4df 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -332,6 +332,127 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
 }
 
 
+template <typename ValueType, typename ValueTypeKrylovBases>
+void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
+                         matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                         matrix::Dense<ValueType> *hessenberg_iter,
+                         matrix::Dense<ValueType> *buffer_iter,
+                         matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+                         const stopping_status *stop_status)
+{
+    const ValueType eta = 1.0 / sqrt(2.0);
+#pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
+
+    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
+        if (stop_status[i].has_stopped()) {
+            continue;
+        }
+
+        ValueType nrm = zero<ValueType>();
+#pragma omp parallel for reduction(add : nrm)
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        arnoldi_norm->at(0, i) = nrm * eta;
+        // nrmP = norm(next_krylov_basis)
+#pragma omp parallel for
+        for (size_type k = 0; k < iter + 1; ++k) {
+            ValueType hessenberg_iter_entry = zero<ValueType>();
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                hessenberg_iter_entry +=
+                    next_krylov_basis->at(j, i) *
+                    krylov_bases->at(j,
+                                     next_krylov_basis->get_size()[1] * k + i);
+            }
+            hessenberg_iter->at(k, i) = hessenberg_iter_entry;
+        }
+        // for i in 1:iter
+        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+        // end
+        for (size_type k = 0; k < iter + 1; ++k) {
+#pragma omp parallel for
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                next_krylov_basis->at(j, i) -=
+                    hessenberg_iter->at(k, i) *
+                    krylov_bases->at(j,
+                                     next_krylov_basis->get_size()[1] * k + i);
+            }
+        }
+        // for i in 1:iter
+        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+        // end
+        // ValueType nrm = zero<ValueType>();
+        nrm = zero<ValueType>();
+#pragma omp parallel for reduction(add : nrm)
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        arnoldi_norm->at(1, i) = nrm;
+        // nrmN = norm(next_krylov_basis)
+        for (size_type l = 1;
+             arnoldi_norm->at(1, i) < arnoldi_norm->at(0, i) && l < 3; l++) {
+            arnoldi_norm->at(0, i) = arnoldi_norm->at(1, i) * eta;
+            // nrmP = nrmN
+#pragma omp parallel for
+            for (size_type k = 0; k < iter + 1; ++k) {
+                ValueType hessenberg_iter_entry = zero<ValueType>();
+                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
+                     ++j) {
+                    hessenberg_iter_entry +=
+                        next_krylov_basis->at(j, i) *
+                        krylov_bases->at(
+                            j, next_krylov_basis->get_size()[1] * k + i);
+                }
+                buffer_iter->at(k, i) = hessenberg_iter_entry;
+            }
+            // for i in 1:iter
+            //     buffer(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+            // end
+            for (size_type k = 0; k < iter + 1; ++k) {
+#pragma omp parallel for
+                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
+                     ++j) {
+                    next_krylov_basis->at(j, i) -=
+                        buffer_iter->at(k, i) *
+                        krylov_bases->at(
+                            j, next_krylov_basis->get_size()[1] * k + i);
+                }
+            }
+            // for i in 1:iter
+            //     next_krylov_basis  -= buffer(iter, i) * krylov_bases(:, i)
+            // end
+            ValueType nrm = zero<ValueType>();
+#pragma omp parallel for reduction(add : nrm)
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                nrm +=
+                    next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            }
+            arnoldi_norm->at(1, i) = nrm;
+            // nrmN = norm(next_krylov_basis)
+        }
+        // reorthogonalization
+
+        ValueType hessenberg_iter_entry = zero<ValueType>();
+#pragma omp parallel for reduction(add : hessenberg_iter_entry)
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            hessenberg_iter_entry +=
+                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        hessenberg_iter->at(iter + 1, i) = sqrt(hessenberg_iter_entry);
+        // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
+#pragma omp parallel for
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
+            krylov_bases->at(j, next_krylov_basis->get_size()[1] * (iter + 1) +
+                                    i) = next_krylov_basis->at(j, i);
+        }
+        // next_krylov_basis /= hessenberg(iter, iter + 1)
+        // krylov_bases(:, iter + 1) = next_krylov_basis
+        // End of arnoldi
+    }
+}
+
+
 template <typename ValueType>
 void calculate_sin_and_cos(matrix::Dense<ValueType> *givens_sin,
                            matrix::Dense<ValueType> *givens_cos,
@@ -583,9 +704,12 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
     //    hessenberg_iter,
     //                          arnoldi_norm, iter,
     //                          stop_status->get_const_data());
-    finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
-                       buffer_iter, arnoldi_norm, iter,
-                       stop_status->get_const_data());
+    //    finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
+    //                       buffer_iter, arnoldi_norm, iter,
+    //                       stop_status->get_const_data());
+    finish_arnoldi_CGS2(next_krylov_basis, krylov_bases, hessenberg_iter,
+                        buffer_iter, arnoldi_norm, iter,
+                        stop_status->get_const_data());
     givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
                     iter, stop_status->get_const_data());
     calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,

From 9f1543350ceebf672784e75ab4a681f7faf83bc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 7 May 2020 09:11:09 +0200
Subject: [PATCH 0340/1008] Add GMRES_mixed to benchmark

---
 benchmark/solver/solver.cpp | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 53fa9c1fc6c..db60f269ac2 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -194,6 +194,24 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                             .with_preconditioner(give(precond))
                             .on(exec);
                     }},
+                   {"gmres_mixed_double",
+                    [](std::shared_ptr<const gko::Executor> exec,
+                       std::shared_ptr<const gko::LinOpFactory> precond) {
+                        return gko::solver::GmresMixed<double, double>::build()
+                            .with_criteria(create_criterion(exec))
+                            .with_krylov_dim_mixed(FLAGS_gmres_restart)
+                            .with_preconditioner(give(precond))
+                            .on(exec);
+                    }},
+                   {"gmres_mixed_float",
+                    [](std::shared_ptr<const gko::Executor> exec,
+                       std::shared_ptr<const gko::LinOpFactory> precond) {
+                        return gko::solver::GmresMixed<double, float>::build()
+                            .with_criteria(create_criterion(exec))
+                            .with_krylov_dim_mixed(FLAGS_gmres_restart)
+                            .with_preconditioner(give(precond))
+                            .on(exec);
+                    }},
                    {"lower_trs",
                     [](std::shared_ptr<const gko::Executor> exec,
                        std::shared_ptr<const gko::LinOpFactory>) {

From 1ff5a5edbba4e3a3605e9143855915522ddfb963 Mon Sep 17 00:00:00 2001
From: josealiaga <aliaga@uji.es>
Date: Thu, 7 May 2020 15:19:25 +0200
Subject: [PATCH 0341/1008] Definition of the CGS2 version of finish_arnoldi
 method, for omp and cuda executors. For omp, the omp is trying to move to the
 outer loop For cuda, the loop of kernels is change to a kernels with a loop. 
  * The main routines (loop of dots and loop of axpy) are still too expensive.

---
 common/solver/gmres_mixed_kernels.hpp.inc     | 144 +++++++++++++++++-
 core/solver/gmres_mixed.cpp                   |  47 ++++++
 cuda/solver/gmres_mixed_kernels.cu            | 120 +++++++++++++--
 .../gmres-mixed-solver-jacobi-matrix.cpp      |  77 +---------
 include/ginkgo/core/base/types.hpp            |   7 +
 omp/solver/gmres_mixed_kernels.cpp            |  34 +++++
 reference/stop/residual_norm_kernels.cpp      |   3 +
 7 files changed, 351 insertions(+), 81 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 604f3711ce8..7005646b144 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -252,6 +252,75 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
 }
 
 
+template <typename ValueType, typename ValueTypeKrylovBases>
+__global__
+    __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_new(
+        size_type num_iters, size_type num_rows, size_type num_cols,
+        const ValueType *__restrict__ next_krylov_basis,
+        size_type stride_next_krylov,
+        const ValueTypeKrylovBases *__restrict__ krylov_bases,
+        size_type stride_krylov, ValueType *__restrict__ hessenberg_iter,
+        size_type stride_hessenberg,
+        const stopping_status *__restrict__ stop_status)
+{
+    const auto mult = 1;
+    const auto tidx = threadIdx.x;
+    const auto tidy = threadIdx.y;
+    const auto col_idx = blockIdx.x * default_dot_dim + tidx;
+    const auto num = ceildiv(num_rows, gridDim.y);
+    const auto start_row = blockIdx.y * num;
+    const auto end_row =
+        ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+    __shared__
+        UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
+            reduction_helper_array;
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
+
+    for (size_type k = 0; k < num_iters; ++k) {
+        ValueType local_res = zero<ValueType>();
+        const auto krylov_col = k * num_cols + col_idx;
+        if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+            for (size_type i = start_row + tidy; i < end_row;
+                 i += default_dot_dim * mult) {
+                const auto next_krylov_idx =
+                    i * stride_next_krylov * mult + col_idx;
+                const auto krylov_idx = i * stride_krylov * mult + krylov_col;
+                local_res += next_krylov_basis[next_krylov_idx] *
+                             krylov_bases[krylov_idx];
+                if (mult > 1) {
+                    const auto next_krylov_idx_2 =
+                        next_krylov_idx +
+                        default_dot_size * stride_next_krylov * mult;
+                    const auto krylov_idx_2 =
+                        krylov_idx + default_dot_size * stride_krylov * mult;
+                    local_res += ((i + default_dot_size) < end_row)
+                                     ? next_krylov_basis[next_krylov_idx_2] *
+                                           krylov_bases[krylov_idx_2]
+                                     : 0.0;
+                }
+            }
+
+            reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
+            __syncthreads();
+            local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
+            const auto tile_block = group::tiled_partition<default_dot_dim>(
+                group::this_thread_block());
+            const auto sum = reduce(
+                tile_block, local_res,
+                [](const ValueType &a, const ValueType &b) { return a + b; });
+            const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
+            if (tidx == 0 && new_col_idx < num_cols &&
+                !stop_status[new_col_idx].has_stopped()) {
+                const auto hessenberg_idx = k * stride_hessenberg + new_col_idx;
+                atomic_add(hessenberg_iter + hessenberg_idx, sum);
+            }
+        }
+    }
+}
+
+
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
 template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
@@ -280,6 +349,37 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel(
 }
 
 
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__
+    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters(
+        size_type num_iters, size_type num_rows, size_type num_cols,
+        ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
+        const ValueTypeKrylovBases *__restrict__ krylov_bases,
+        size_type stride_krylov, const ValueType *__restrict__ hessenberg_iter,
+        size_type stride_hessenberg,
+        const stopping_status *__restrict__ stop_status)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row_idx = global_id / stride_next_krylov;
+    const auto col_idx = global_id % stride_next_krylov;
+
+    if (row_idx < num_rows && col_idx < num_cols &&
+        !stop_status[col_idx].has_stopped()) {
+        const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+        for (size_type k = 0; k < num_iters; ++k) {
+            const auto krylov_idx =
+                row_idx * stride_krylov + k * num_cols + col_idx;
+            const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+
+            next_krylov_basis[next_krylov_idx] -=
+                hessenberg_iter[hessenberg_idx] * krylov_bases[krylov_idx];
+        }
+    }
+}
+
+
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
 template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
@@ -356,6 +456,43 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
 }
 
 
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__
+    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_and_add(
+        size_type num_iters, size_type num_rows, size_type num_cols,
+        ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
+        const ValueTypeKrylovBases *__restrict__ krylov_bases,
+        size_type stride_krylov, ValueType *__restrict__ hessenberg_iter,
+        size_type stride_hessenberg, const ValueType *__restrict__ buffer_iter,
+        size_type stride_buffer,
+        const stopping_status *__restrict__ stop_status,
+        const stopping_status *__restrict__ reorth_status)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row_idx = global_id / stride_next_krylov;
+    const auto col_idx = global_id % stride_next_krylov;
+
+    if (row_idx < num_rows && col_idx < num_cols &&
+        !stop_status[col_idx].has_stopped() &&
+        !reorth_status[col_idx].has_stopped()) {
+        const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+        for (size_type k = 0; k < num_iters; ++k) {
+            const auto krylov_idx =
+                row_idx * stride_krylov + k * num_cols + col_idx;
+            const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+            const auto buffer_idx = k * stride_buffer + col_idx;
+            next_krylov_basis[next_krylov_idx] -=
+                buffer_iter[buffer_idx] * krylov_bases[krylov_idx];
+            if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
+                hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
+            }
+        }
+    }
+}
+
+
 // Must be called with at least `num_cols` blocks, each with `block_size`
 // threads. `block_size` must be a power of 2.
 template <int block_size, typename ValueType>
@@ -396,7 +533,8 @@ __global__ __launch_bounds__(block_size) void update_hessenberg_2_kernel(
     const auto col_idx = blockIdx.x;
 
     // Used that way to get around dynamic initialization warning and
-    // template error when using `reduction_helper_array` directly in `reduce`
+    // template error when using `reduction_helper_array` directly in
+    // `reduce`
     __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
     ValueType *__restrict__ reduction_helper = reduction_helper_array;
 
@@ -608,8 +746,8 @@ __global__ __launch_bounds__(block_size) void solve_upper_triangular_kernel(
 }
 
 
-// Must be called with at least `stride_preconditioner * num_rows` threads in
-// total.
+// Must be called with at least `stride_preconditioner * num_rows` threads
+// in total.
 template <size_type block_size, typename ValueType,
           typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(block_size) void calculate_Qy_kernel(
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 1500a948f8f..c810d69c135 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -50,7 +50,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/gmres_mixed_kernels.hpp"
 
 
+// #define TIMING 1
+
+
+#ifdef TIMING
 using double_seconds = std::chrono::duration<double>;
+#endif
 
 
 namespace gko {
@@ -161,7 +166,11 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     auto after_preconditioner =
         matrix::Dense<ValueType>::create_with_config_of(dense_x);
 
+#ifdef TIMING
     auto start = std::chrono::steady_clock::now();
+    auto time_SPMV = start - start;
+    auto time_STEP1 = start - start;
+#endif
 
     while (true) {
         ++total_iter;
@@ -220,17 +229,29 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
         auto buffer_iter = buffer->create_submatrix(
             span{0, restart_iter + 2}, span{0, dense_b->get_size()[1]});
 
+#ifdef TIMING
+        auto t_aux_1 = std::chrono::steady_clock::now();
+#endif
         // Start of arnoldi
         system_matrix_->apply(preconditioned_vector.get(),
                               next_krylov_basis.get());
         // next_krylov_basis = A * preconditioned_vector
+#ifdef TIMING
+        time_SPMV += std::chrono::steady_clock::now() - t_aux_1;
+#endif
 
+#ifdef TIMING
+        auto t_aux_2 = std::chrono::steady_clock::now();
+#endif
         exec->run(gmres_mixed::make_step_1(
             next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
             residual_norm.get(), residual_norm_collection.get(),
             krylov_bases.get(), hessenberg_iter.get(), buffer_iter.get(),
             b_norm.get(), arnoldi_norm.get(), restart_iter, &final_iter_nums,
             &stop_status, &reorth_status, &num_reorth));
+#ifdef TIMING
+        time_STEP1 += std::chrono::steady_clock::now() - t_aux_2;
+#endif
         // for i in 0:restart_iter
         //     hessenberg(restart_iter, i) = next_krylov_basis' *
         //     krylov_bases(:, i) next_krylov_basis  -= hessenberg(restart_iter,
@@ -259,6 +280,9 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     }
 
     // Solve x
+#ifdef TIMING
+    auto t_aux_3 = std::chrono::steady_clock::now();
+#endif
     auto krylov_bases_small = krylov_bases->create_submatrix(
         span{0, system_matrix_->get_size()[0]},
         span{0, dense_b->get_size()[1] * (restart_iter + 1)});
@@ -270,21 +294,44 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
         residual_norm_collection.get(), krylov_bases_small.get(),
         hessenberg_small.get(), y.get(), before_preconditioner.get(),
         &final_iter_nums));
+#ifdef TIMING
+    auto time_STEP2 = std::chrono::steady_clock::now() - t_aux_3;
+#endif
     // Solve upper triangular.
     // y = hessenberg \ residual_norm_collection
 
+#ifdef TIMING
+    auto t_aux_4 = std::chrono::steady_clock::now();
+#endif
     get_preconditioner()->apply(before_preconditioner.get(),
                                 after_preconditioner.get());
     dense_x->add_scaled(one_op.get(), after_preconditioner.get());
+#ifdef TIMING
+    auto time_SOLVEX = std::chrono::steady_clock::now() - t_aux_4;
+#endif
     // Solve x
     // x = x + get_preconditioner() * krylov_bases * y
 
+#ifdef TIMING
     auto time = std::chrono::steady_clock::now() - start;
     std::cout << "total_iter = " << total_iter << std::endl;
     std::cout << "time = "
               << std::chrono::duration_cast<double_seconds>(time).count()
               << std::endl;
+    std::cout << "time_SPMV = "
+              << std::chrono::duration_cast<double_seconds>(time_SPMV).count()
+              << std::endl;
+    std::cout << "time_STEP1 = "
+              << std::chrono::duration_cast<double_seconds>(time_STEP1).count()
+              << std::endl;
+    std::cout << "time_STEP2 = "
+              << std::chrono::duration_cast<double_seconds>(time_STEP2).count()
+              << std::endl;
+    std::cout << "time_SOLVEX = "
+              << std::chrono::duration_cast<double_seconds>(time_SOLVEX).count()
+              << std::endl;
     write(std::cout, lend(residual_norm));
+#endif
 }
 
 
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 8af56d11064..b36b06eebe2 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -57,6 +57,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <iostream>
 
 
+// #define TIMING 1
+
+
+#ifdef TIMING
+using double_seconds = std::chrono::duration<double, std::milli>;
+#endif
+
+
 namespace gko {
 namespace kernels {
 namespace cuda {
@@ -524,17 +532,22 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
     // nrmP = norm(next_krylov_basis
+#ifdef TIMING
+    exec->synchronize();
+    auto start_1 = std::chrono::steady_clock::now();
+#endif
     /*
         for (size_type k = 0; k < iter + 1; ++k) {
-            components::fill_array(exec, hessenberg_iter->get_values() + k *
-       stride_hessenberg, dim_size[1], zero<ValueType>());
+            components::fill_array(
+                exec, hessenberg_iter->get_values() + k * stride_hessenberg,
+                dim_size[1], zero<ValueType>());
             multidot_kernel<<<grid_size, block_size>>>(
                 k, dim_size[0], dim_size[1],
                 as_cuda_type(next_krylov_basis->get_const_values()),
                 stride_next_krylov,
-       as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
-       as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-       as_cuda_type(stop_status));
+                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+                as_cuda_type(stop_status));
         }
     */
     /*
@@ -544,6 +557,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
             zero<ValueType>());
     }
     */
+    /* */
     zero_matrix(iter + 1, dim_size[1], stride_hessenberg,
                 hessenberg_iter->get_values());
     multidot_kernel_num_iters<<<grid_size, block_size>>>(
@@ -552,9 +566,22 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
         as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
         as_cuda_type(stop_status));
+    /* */
+#ifdef TIMING
+    exec->synchronize();
+    auto time_1 = std::chrono::steady_clock::now() - start_1;
+    std::cout << "time_1(" << iter << ") = "
+              << std::chrono::duration_cast<double_seconds>(time_1).count()
+              << std::endl;
+#endif
     // for i in 1:iter
     //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
     // end
+#ifdef TIMING
+    exec->synchronize();
+    auto start_2 = std::chrono::steady_clock::now();
+#endif
+    /*
     for (size_type k = 0; k < iter + 1; ++k) {
         update_next_krylov_kernel<default_block_size>
             <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
@@ -566,35 +593,88 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                 as_cuda_type(hessenberg_iter->get_const_values()),
                 stride_hessenberg, as_cuda_type(stop_status));
     }
+    */
+    update_next_krylov_kernel_num_iters<default_block_size>
+        <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+           default_block_size>>>(
+            iter + 1, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
+            as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+            as_cuda_type(hessenberg_iter->get_const_values()),
+            stride_hessenberg, as_cuda_type(stop_status));
+#ifdef TIMING
+    exec->synchronize();
+    auto time_2 = std::chrono::steady_clock::now() - start_2;
+    std::cout << "time_2(" << iter << ") = "
+              << std::chrono::duration_cast<double_seconds>(time_2).count()
+              << std::endl;
+    std::cout << "time_1 / time_2(" << iter << ") = "
+              << std::chrono::duration_cast<double_seconds>(time_1).count() /
+                     std::chrono::duration_cast<double_seconds>(time_2).count()
+              << std::endl;
+#endif
     // for i in 1:iter
     //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
     // end
     components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
                            dim_size[1], zero<ValueType>());
+#ifdef TIMING
+    exec->synchronize();
+    auto start_3 = std::chrono::steady_clock::now();
+#endif
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
         as_cuda_type(stop_status));
+#ifdef TIMING
+    exec->synchronize();
+    auto time_3 = std::chrono::steady_clock::now() - start_3;
+    std::cout << "time_3(" << iter << ") = "
+              << std::chrono::duration_cast<double_seconds>(time_3).count()
+              << std::endl;
+#endif
     // nrmN = norm(next_krylov_basis)
     components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
+#ifdef TIMING
+    exec->synchronize();
+    auto start_4 = std::chrono::steady_clock::now();
+#endif
     check_arnoldi_norms<default_block_size>
         <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
             as_cuda_type(arnoldi_norm->get_const_values()), stride_arnoldi,
             as_cuda_type(arnoldi_norm->get_const_values() + stride_arnoldi),
             stride_arnoldi, as_cuda_type(stop_status),
             as_cuda_type(reorth_status), as_cuda_type(num_reorth->get_data()));
+#ifdef TIMING
+    exec->synchronize();
+    auto time_4 = std::chrono::steady_clock::now() - start_4;
+    std::cout << "time_4(" << iter << ") = "
+              << std::chrono::duration_cast<double_seconds>(time_4).count()
+              << std::endl;
+#endif
+#ifdef TIMING
+    exec->synchronize();
+    auto start_5 = std::chrono::steady_clock::now();
+#endif
     numReorth = 0;
     exec->get_master()->copy_from(exec.get(), 1, num_reorth->get_const_data(),
                                   &numReorth);
+#ifdef TIMING
+    exec->synchronize();
+    auto time_5 = std::chrono::steady_clock::now() - start_5;
+    std::cout << "time_5(" << iter << ") = "
+              << std::chrono::duration_cast<double_seconds>(time_5).count()
+              << std::endl;
+#endif
     // numReorth <= number of next_krylov vector to be reorthogonalization
     for (size_type l = 1; (numReorth > 0) && (l < 3); l++) {
         /*
         for (size_type k = 0; k < iter + 1; ++k) {
-            components::fill_array(exec,
-                                   buffer_iter->get_values() + k *
-        stride_buffer, dim_size[1], zero<ValueType>());
+            components::fill_array(
+                exec, buffer_iter->get_values() + k * stride_buffer,
+                dim_size[1], zero<ValueType>());
             multidot_kernel<<<grid_size, block_size>>>(
                 k, dim_size[0], dim_size[1],
                 as_cuda_type(next_krylov_basis->get_const_values()),
@@ -615,6 +695,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         // for i in 1:iter
         //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
         // end
+        /*
         for (size_type k = 0; k < iter + 1; ++k) {
             update_next_krylov_kernel_and_add<default_block_size><<<
                 ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
@@ -627,6 +708,17 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                 as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
                 as_cuda_type(stop_status), as_cuda_type(reorth_status));
         }
+        */
+        update_next_krylov_kernel_num_iters_and_add<default_block_size>
+            <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+               default_block_size>>>(
+                iter + 1, dim_size[0], dim_size[1],
+                as_cuda_type(next_krylov_basis->get_values()),
+                stride_next_krylov,
+                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+                as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
+                as_cuda_type(stop_status), as_cuda_type(reorth_status));
         // for i in 1:iter
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
@@ -660,7 +752,10 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         // numReorth <= number of next_krylov vector to be reorthogonalization
     }
 
-
+#ifdef TIMING
+    exec->synchronize();
+    auto start_6 = std::chrono::steady_clock::now();
+#endif
     update_hessenberg_2_kernel<default_block_size>
         <<<dim_size[1], default_block_size>>>(
             iter, dim_size[0], dim_size[1],
@@ -675,6 +770,13 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
             as_cuda_type(krylov_bases->get_values()), stride_krylov,
             as_cuda_type(hessenberg_iter->get_const_values()),
             stride_hessenberg, as_cuda_type(stop_status));
+#ifdef TIMING
+    exec->synchronize();
+    auto time_6 = std::chrono::steady_clock::now() - start_6;
+    std::cout << "time_6(" << iter << ") = "
+              << std::chrono::duration_cast<double_seconds>(time_6).count()
+              << std::endl;
+#endif
     // next_krylov_basis /= hessenberg(iter, iter + 1)
     // krylov_bases(:, iter + 1) = next_krylov_basis
     // End of arnoldi
diff --git a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp b/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
index bab8cf37cc8..0100f4b041f 100644
--- a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
+++ b/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
@@ -57,8 +57,8 @@ int main(int argc, char *argv[])
     using mtx = gko::matrix::Csr<>;
     // The gko::solver::Cg is used here, but any other solver class can also be
     // used.
-    //    using gmres_mixed = gko::solver::GmresMixed<double, float>;
-    using gmres_mixed = gko::solver::GmresMixed<>;
+    using gmres_mixed = gko::solver::GmresMixed<double, float>;
+    //    using gmres_mixed = gko::solver::GmresMixed<>;
     using bj = gko::preconditioner::Jacobi<>;
 
     // Print the ginkgo version information.
@@ -115,7 +115,6 @@ int main(int argc, char *argv[])
                   << std::endl;
         std::exit(-1);
     }
-    std::cout << "index_matrix -> " << index_matrix << std::endl;
 
     // @sect3{Reading your data and transfer to the proper device.}
     // Read the matrix, right hand side and the initial solution using the @ref
@@ -129,95 +128,34 @@ int main(int argc, char *argv[])
     auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
     auto x = gko::read<vec>(std::ifstream("data/x0.mtx"), exec);
     if (index_matrix > 0) {
-        double aux1, aux2;
-
-        std::cout << index_matrix << std::endl;
-        std::cout << index_matrix << " -> " << std::string(argv[index_matrix])
-                  << std::endl;
         A = share(gko::read<mtx>(std::ifstream(argv[index_matrix]), exec));
         auto sizesA = A->get_size();
-        std::cout << sizesA[0] << " - " << sizesA[1] << std::endl;
-        if (std::string(argv[1]) == "cuda") {
-            exec->get_master()->copy_from(exec.get(), 1, A->get_values(),
-                                          &aux1);
-            exec->get_master()->copy_from(
-                exec.get(), 1,
-                A->get_values() + A->get_num_stored_elements() - 1, &aux2);
-            std::cout << "A0 => " << aux1 << " - " << aux2 << std::endl;
-        } else {
-            std::cout << "A0 => " << A->get_values()[0];
-            std::cout << A->get_values()[A->get_num_stored_elements() - 1]
-                      << std::endl;
-        }
 
         double aux[sizesA[0]];
 
-        //        x = gko::read<vec>(std::ifstream("data/one.mtx"), exec);
         if (std::string(argv[1]) == "cuda") {
             x = gko::read<vec>(std::ifstream("data/one.mtx"), exec);
-            //            gko::kernels::cuda::zero_array(sizesA[1],
-            //            b->get_values()); for (int i = 0; i<sizesA[0]; i++)
-            //            aux[i] = 1.0; x->copy_from(aux);
-            exec->get_master()->copy_from(exec.get(), 1, x->get_values(),
-                                          &aux1);
-            exec->get_master()->copy_from(
-                exec.get(), 1, x->get_values() + sizesA[1] - 1, &aux2);
-            std::cout << "X0 => " << aux1 << " - " << aux2 << std::endl;
         } else {
             x = vec::create(exec, gko::dim<2>{sizesA[0], 1});
             for (int i = 0; i < sizesA[0]; i++) x->at(i) = 1.0;
-            std::cout << "X0 => " << x->at(0) << " - " << x->at(sizesA[0] - 1)
-                      << std::endl;
         }
         auto sizesx = x->get_size();
-        //        std::cout << "SIZEX -> " << sizesx[0] << " - " << sizesx[1] <<
-        //        std::endl;
 
-        //        b = gko::read<vec>(std::ifstream("data/zero.mtx"), exec);
         if (std::string(argv[1]) == "cuda") {
             b = gko::read<vec>(std::ifstream("data/zero.mtx"), exec);
-            exec->get_master()->copy_from(exec.get(), 1, b->get_values(),
-                                          &aux1);
-            exec->get_master()->copy_from(
-                exec.get(), 1, b->get_values() + sizesA[1] - 1, &aux2);
-            //            std::cout << "B0 => " << aux1 << " - " << aux2 <<
-            //            std::endl;
         } else {
             b = vec::create(exec, gko::dim<2>{sizesA[0], 1});
             for (int i = 0; i < sizesA[0]; i++) b->at(i) = 0.0;
-            //            std::cout << "B0 => " << b->at(0) << " - " <<
-            //            b->at(sizesA[0]-1) << std::endl;
         }
         auto sizesb = b->get_size();
-        //        std::cout << "SIZEB -> " << sizesb[0] << " - " << sizesb[1] <<
-        //        std::endl;
 
         A->apply(lend(x), lend(b));
-        if (std::string(argv[1]) == "cuda") {
-            exec->get_master()->copy_from(exec.get(), 1, b->get_values(),
-                                          &aux1);
-            exec->get_master()->copy_from(
-                exec.get(), 1, b->get_values() + sizesA[1] - 1, &aux2);
-            std::cout << "B1 => " << aux1 << " - " << aux2 << std::endl;
-        } else {
-            std::cout << "B1 => " << b->at(0) << " - " << b->at(sizesA[0] - 1)
-                      << std::endl;
-        }
 
-        //        x = gko::read<vec>(std::ifstream("data/zero.mtx"), exec);
         if (std::string(argv[1]) == "cuda") {
             x = gko::read<vec>(std::ifstream("data/zero.mtx"), exec);
-            exec->get_master()->copy_from(exec.get(), 1, x->get_values(),
-                                          &aux1);
-            exec->get_master()->copy_from(
-                exec.get(), 1, x->get_values() + sizesA[1] - 1, &aux2);
-            //            std::cout << "X0 => " << aux1 << " - " << aux2 <<
-            //            std::endl;
         } else {
             x = vec::create(exec, gko::dim<2>{sizesA[0], 1});
             for (int i = 0; i < sizesA[0]; i++) x->at(i) = 0.0;
-            //            std::cout << "X0 => " << x->at(0) << " - " <<
-            //            x->at(sizesA[0]-1) << std::endl;
         }
     }
     // @sect3{Creating the solver}
@@ -232,18 +170,19 @@ int main(int argc, char *argv[])
     auto solver_gen =
         gmres_mixed::build()
             .with_criteria(
-                //                gko::stop::Iteration::build().with_max_iters(20u).on(exec),
+                //    gko::stop::Iteration::build().with_max_iters(20u).on(exec),
                 gko::stop::Iteration::build()
                     .with_max_iters(A->get_size()[0])
                     .on(exec),
-                //                gko::stop::Iteration::build().with_max_iters(100000u).on(exec),
+                //    gko::stop::Iteration::build().with_max_iters(100000u).on(exec),
                 gko::stop::ResidualNormReduction<>::build()
-                    //                    .with_reduction_factor(1e-15)
-                    .with_reduction_factor(1e-12)
+                    //    .with_reduction_factor(1e-15)
+                    //    .with_reduction_factor(1e-12)
+                    .with_reduction_factor(1e-13)
                     .on(exec))
             // Add preconditioner, these 2 lines are the only
             // difference from the simple solver example
-            //            .with_preconditioner(bj::build().with_max_block_size(8u).on(exec))
+            //    .with_preconditioner(bj::build().with_max_block_size(8u).on(exec))
             .with_preconditioner(bj::build().with_max_block_size(1u).on(exec))
             .on(exec);
     // Generate the solver from the matrix. The solver factory built in the
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 57eb2c8990f..d8a6ab723e3 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -451,9 +451,16 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
  *                Should take one argument, which is replaced by the
  *                value type.
  */
+// #if defined(__CUDACC__) || defined(__HIPCC__)
+// #define GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(_macro) \
+//     template _macro(double, half);                  \
+//     template _macro(double, float);                 \
+//     template _macro(double, double)
+// #else
 #define GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(_macro) \
     template _macro(double, float);                 \
     template _macro(double, double)
+// #endif
 
 
 /**
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index 29b7c52f4df..ea732d62249 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -46,6 +46,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/gmres_mixed.hpp>
 
 
+#include <iostream>
+
+
+// #define TIMING 1
+
+
+#ifdef TIMING 1
+using double_seconds = std::chrono::duration<double, std::milli>;
+#endif
+
+
 namespace gko {
 namespace kernels {
 namespace omp {
@@ -355,6 +366,9 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         }
         arnoldi_norm->at(0, i) = nrm * eta;
         // nrmP = norm(next_krylov_basis)
+#ifdef TIMING 1
+        auto start_1 = std::chrono::steady_clock::now();
+#endif
 #pragma omp parallel for
         for (size_type k = 0; k < iter + 1; ++k) {
             ValueType hessenberg_iter_entry = zero<ValueType>();
@@ -366,9 +380,18 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
             }
             hessenberg_iter->at(k, i) = hessenberg_iter_entry;
         }
+#ifdef TIMING 1
+        auto time_1 = std::chrono::steady_clock::now() - start_1;
+        std::cout << "time_1(" << iter << ") = "
+                  << std::chrono::duration_cast<double_seconds>(time_1).count()
+                  << std::endl;
+#endif
         // for i in 1:iter
         //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
         // end
+#ifdef TIMING 1
+        auto start_2 = std::chrono::steady_clock::now();
+#endif
         for (size_type k = 0; k < iter + 1; ++k) {
 #pragma omp parallel for
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
@@ -378,6 +401,17 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                                      next_krylov_basis->get_size()[1] * k + i);
             }
         }
+#ifdef TIMING 1
+        auto time_2 = std::chrono::steady_clock::now() - start_2;
+        std::cout << "time_2(" << iter << ") = "
+                  << std::chrono::duration_cast<double_seconds>(time_2).count()
+                  << std::endl;
+        std::cout
+            << "time_1 / time_2(" << iter << ") = "
+            << std::chrono::duration_cast<double_seconds>(time_1).count() /
+                   std::chrono::duration_cast<double_seconds>(time_2).count()
+            << std::endl;
+#endif
         // for i in 1:iter
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
diff --git a/reference/stop/residual_norm_kernels.cpp b/reference/stop/residual_norm_kernels.cpp
index b1f25b52c29..b1d06b2a732 100644
--- a/reference/stop/residual_norm_kernels.cpp
+++ b/reference/stop/residual_norm_kernels.cpp
@@ -42,6 +42,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
+#include <iostream>
+
+
 namespace gko {
 namespace kernels {
 namespace reference {

From efdbe2026eec12e77b86085903d8b01e38faa2d1 Mon Sep 17 00:00:00 2001
From: josealiaga <aliaga@uji.es>
Date: Wed, 13 May 2020 01:26:18 +0200
Subject: [PATCH 0342/1008] Finally a good implementation of the
 multidot_kernels_num_iters_1 is done. Also timing instructions are included,
 whose management is made by some define's.

The next step will be to improve the update kernels.
---
 common/solver/gmres_mixed_kernels.hpp.inc     | 112 ++++++++++--------
 core/solver/gmres_mixed.cpp                   |  56 +++++++--
 core/solver/gmres_mixed_kernels.hpp           |   3 +-
 cuda/solver/gmres_mixed_kernels.cu            | 100 ++++++----------
 cuda/test/solver/gmres_mixed_kernels.cpp      |   8 +-
 .../gmres-mixed-solver-jacobi-matrix.cpp      |  18 ++-
 include/ginkgo/core/base/types.hpp            |   1 +
 omp/solver/gmres_mixed_kernels.cpp            |   3 +-
 omp/test/solver/gmres_mixed_kernels.cpp       |   8 +-
 reference/solver/gmres_mixed_kernels.cpp      |   3 +-
 10 files changed, 174 insertions(+), 138 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 7005646b144..9a9297809b9 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -94,7 +94,8 @@ __global__ __launch_bounds__(block_size) void initialize_2_1_kernel(
     const auto col_idx = global_id % stride_krylov;
 
     if (row_idx < num_rows && col_idx < (krylov_dim + 1) * num_rhs) {
-        krylov_bases[row_idx * stride_krylov + col_idx] = zero<ValueType>();
+        krylov_bases[row_idx * stride_krylov + col_idx] =
+            zero<ValueTypeKrylovBases>();
     }
 
     if (row_idx < krylov_dim + 1 && col_idx < num_rhs) {
@@ -169,7 +170,6 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
         UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
             reduction_helper_array;
     ValueType *__restrict__ reduction_helper = reduction_helper_array;
-
     ValueType local_res = zero<ValueType>();
     const auto krylov_col = k * num_cols + col_idx;
     if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
@@ -253,24 +253,25 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-__global__
-    __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_new(
-        size_type num_iters, size_type num_rows, size_type num_cols,
-        const ValueType *__restrict__ next_krylov_basis,
-        size_type stride_next_krylov,
-        const ValueTypeKrylovBases *__restrict__ krylov_bases,
-        size_type stride_krylov, ValueType *__restrict__ hessenberg_iter,
-        size_type stride_hessenberg,
-        const stopping_status *__restrict__ stop_status)
+__global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_1(
+    size_type num_iters, size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov,
+    const ValueTypeKrylovBases *__restrict__ krylov_bases,
+    size_type stride_krylov, ValueType *__restrict__ hessenberg_iter,
+    size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status)
 {
-    const auto mult = 1;
     const auto tidx = threadIdx.x;
     const auto tidy = threadIdx.y;
-    const auto col_idx = blockIdx.x * default_dot_dim + tidx;
+    const auto num_cols_iters = num_cols * num_iters;
+    const auto col_idx_iters = blockIdx.x * default_dot_dim + tidx;
+    const auto col_idx = col_idx_iters % num_cols;
     const auto num = ceildiv(num_rows, gridDim.y);
     const auto start_row = blockIdx.y * num;
     const auto end_row =
         ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
+    const auto k = col_idx_iters / num_cols;
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
     __shared__
@@ -278,46 +279,55 @@ __global__
             reduction_helper_array;
     ValueType *__restrict__ reduction_helper = reduction_helper_array;
 
-    for (size_type k = 0; k < num_iters; ++k) {
-        ValueType local_res = zero<ValueType>();
-        const auto krylov_col = k * num_cols + col_idx;
-        if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
-            for (size_type i = start_row + tidy; i < end_row;
-                 i += default_dot_dim * mult) {
-                const auto next_krylov_idx =
-                    i * stride_next_krylov * mult + col_idx;
-                const auto krylov_idx = i * stride_krylov * mult + krylov_col;
-                local_res += next_krylov_basis[next_krylov_idx] *
-                             krylov_bases[krylov_idx];
-                if (mult > 1) {
-                    const auto next_krylov_idx_2 =
-                        next_krylov_idx +
-                        default_dot_size * stride_next_krylov * mult;
-                    const auto krylov_idx_2 =
-                        krylov_idx + default_dot_size * stride_krylov * mult;
-                    local_res += ((i + default_dot_size) < end_row)
-                                     ? next_krylov_basis[next_krylov_idx_2] *
-                                           krylov_bases[krylov_idx_2]
-                                     : 0.0;
-                }
-            }
-
-            reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
-            __syncthreads();
-            local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
-            const auto tile_block = group::tiled_partition<default_dot_dim>(
-                group::this_thread_block());
-            const auto sum = reduce(
-                tile_block, local_res,
-                [](const ValueType &a, const ValueType &b) { return a + b; });
-            const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
-            if (tidx == 0 && new_col_idx < num_cols &&
-                !stop_status[new_col_idx].has_stopped()) {
-                const auto hessenberg_idx = k * stride_hessenberg + new_col_idx;
-                atomic_add(hessenberg_iter + hessenberg_idx, sum);
-            }
+    //    for (size_type k = 0; k < num_iters; ++k) {
+    ValueType local_res = zero<ValueType>();
+    const auto krylov_col = k * num_cols + col_idx;
+    if (col_idx_iters < num_cols_iters && !stop_status[col_idx].has_stopped()) {
+        for (size_type i = start_row + tidy; i < end_row;
+             i += default_dot_dim) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            const auto krylov_idx = i * stride_krylov + krylov_col;
+            local_res +=
+                next_krylov_basis[next_krylov_idx] * krylov_bases[krylov_idx];
         }
     }
+    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
+    __syncthreads();
+    /*
+    if (((blockIdx.x + blockIdx.y) == 0) && ((tidx + tidy) == 0)) {
+        //    if ((blockIdx.x == 0) && ((tidx + tidy) == 0)) {
+        for (size_type ii = 0; ii < default_dot_dim; ii++)
+            for (size_type jj = 0; jj < default_dot_dim + 1; jj++)
+                if (jj <= num)
+                    printf("XX_%d(%d,%d) = %g\n", (int)blockIdx.y + 1,
+                           (int)ii + 1, (int)jj + 1,
+                           reduction_helper[(default_dot_dim + 1) * ii + jj]);
+    }
+    __syncthreads();
+    */
+    local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
+    const auto tile_block =
+        group::tiled_partition<default_dot_dim>(group::this_thread_block());
+    const auto sum =
+        reduce(tile_block, local_res,
+               [](const ValueType &a, const ValueType &b) { return a + b; });
+    const auto new_col_idx_iters = blockIdx.x * default_dot_dim + tidy;
+    const auto new_col_idx = new_col_idx_iters % num_cols;
+    const auto new_k = new_col_idx_iters / num_cols;
+    if (tidx == 0 && new_col_idx_iters < num_cols_iters &&
+        !stop_status[new_col_idx].has_stopped()) {
+        //        if ((blockIdx.x + blockIdx.y) == 0) {
+        /*
+        if (blockIdx.x == 0) {
+            printf("sum_%d(%d) = %g\n", (int)blockIdx.y + 1, (int)tidy + 1,
+                   sum);
+        }
+        */
+        // const auto hessenberg_idx = k * stride_hessenberg + new_col_idx;
+        const auto hessenberg_idx = new_k * stride_hessenberg + new_col_idx;
+        atomic_add(hessenberg_iter + hessenberg_idx, sum);
+    }
+    //    }
 }
 
 
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index c810d69c135..1a510bc45a4 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -50,11 +50,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/gmres_mixed_kernels.hpp"
 
 
-// #define TIMING 1
+#define TIMING 1
 
 
 #ifdef TIMING
 using double_seconds = std::chrono::duration<double>;
+#define TIMING_STEPS 1
 #endif
 
 
@@ -132,6 +133,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
                                          dense_b->get_size()[1]);
     //                                         krylov_dim_mixed_ + 1);
     Array<size_type> num_reorth(this->get_executor(), dense_b->get_size()[1]);
+    int num_restarts = 0, num_reorth_steps = 0, num_reorth_vectors = 0;
 
     // Initialization
     exec->run(gmres_mixed::make_initialize_1(
@@ -167,9 +169,13 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
         matrix::Dense<ValueType>::create_with_config_of(dense_x);
 
 #ifdef TIMING
+    exec->synchronize();
     auto start = std::chrono::steady_clock::now();
+#ifdef TIMING_STEPS
+    auto time_RSTRT = start - start;
     auto time_SPMV = start - start;
     auto time_STEP1 = start - start;
+#endif
 #endif
 
     while (true) {
@@ -186,7 +192,11 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
         }
 
         if (restart_iter == krylov_dim_mixed_) {
-            // std::cout << "Restart" << std::endl;
+#ifdef TIMING_STEPS
+            exec->synchronize();
+            auto t_aux_0 = std::chrono::steady_clock::now();
+#endif
+            num_restarts++;
             // Restart
             exec->run(gmres_mixed::make_step_2(
                 residual_norm_collection.get(), krylov_bases.get(),
@@ -215,6 +225,10 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
             // next_krylov_basis = residual / residual_norm
             // final_iter_nums = {0, ..., 0}
             restart_iter = 0;
+#ifdef TIMING_STEPS
+            exec->synchronize();
+            time_RSTRT += std::chrono::steady_clock::now() - t_aux_0;
+#endif
         }
 
         get_preconditioner()->apply(next_krylov_basis.get(),
@@ -229,18 +243,21 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
         auto buffer_iter = buffer->create_submatrix(
             span{0, restart_iter + 2}, span{0, dense_b->get_size()[1]});
 
-#ifdef TIMING
+#ifdef TIMING_STEPS
+        exec->synchronize();
         auto t_aux_1 = std::chrono::steady_clock::now();
 #endif
         // Start of arnoldi
         system_matrix_->apply(preconditioned_vector.get(),
                               next_krylov_basis.get());
         // next_krylov_basis = A * preconditioned_vector
-#ifdef TIMING
+#ifdef TIMING_STEPS
+        exec->synchronize();
         time_SPMV += std::chrono::steady_clock::now() - t_aux_1;
 #endif
 
-#ifdef TIMING
+#ifdef TIMING_STEPS
+        exec->synchronize();
         auto t_aux_2 = std::chrono::steady_clock::now();
 #endif
         exec->run(gmres_mixed::make_step_1(
@@ -248,8 +265,10 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
             residual_norm.get(), residual_norm_collection.get(),
             krylov_bases.get(), hessenberg_iter.get(), buffer_iter.get(),
             b_norm.get(), arnoldi_norm.get(), restart_iter, &final_iter_nums,
-            &stop_status, &reorth_status, &num_reorth));
-#ifdef TIMING
+            &stop_status, &reorth_status, &num_reorth, &num_reorth_steps,
+            &num_reorth_vectors));
+#ifdef TIMING_STEPS
+        exec->synchronize();
         time_STEP1 += std::chrono::steady_clock::now() - t_aux_2;
 #endif
         // for i in 0:restart_iter
@@ -280,7 +299,8 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     }
 
     // Solve x
-#ifdef TIMING
+#ifdef TIMING_STEPS
+    exec->synchronize();
     auto t_aux_3 = std::chrono::steady_clock::now();
 #endif
     auto krylov_bases_small = krylov_bases->create_submatrix(
@@ -294,30 +314,43 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
         residual_norm_collection.get(), krylov_bases_small.get(),
         hessenberg_small.get(), y.get(), before_preconditioner.get(),
         &final_iter_nums));
-#ifdef TIMING
+#ifdef TIMING_STEPS
+    exec->synchronize();
     auto time_STEP2 = std::chrono::steady_clock::now() - t_aux_3;
 #endif
     // Solve upper triangular.
     // y = hessenberg \ residual_norm_collection
 
-#ifdef TIMING
+#ifdef TIMING_STEPS
+    exec->synchronize();
     auto t_aux_4 = std::chrono::steady_clock::now();
 #endif
     get_preconditioner()->apply(before_preconditioner.get(),
                                 after_preconditioner.get());
     dense_x->add_scaled(one_op.get(), after_preconditioner.get());
-#ifdef TIMING
+#ifdef TIMING_STEPS
+    exec->synchronize();
     auto time_SOLVEX = std::chrono::steady_clock::now() - t_aux_4;
 #endif
     // Solve x
     // x = x + get_preconditioner() * krylov_bases * y
 
 #ifdef TIMING
+    exec->synchronize();
     auto time = std::chrono::steady_clock::now() - start;
+#endif
+#ifdef TIMING
     std::cout << "total_iter = " << total_iter << std::endl;
+    std::cout << "num_restarts = " << num_restarts << std::endl;
+    std::cout << "reorth_steps = " << num_reorth_steps << std::endl;
+    std::cout << "reorth_vectors = " << num_reorth_vectors << std::endl;
     std::cout << "time = "
               << std::chrono::duration_cast<double_seconds>(time).count()
               << std::endl;
+#ifdef TIMING_STEPS
+    std::cout << "time_RSTRT = "
+              << std::chrono::duration_cast<double_seconds>(time_RSTRT).count()
+              << std::endl;
     std::cout << "time_SPMV = "
               << std::chrono::duration_cast<double_seconds>(time_SPMV).count()
               << std::endl;
@@ -330,6 +363,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     std::cout << "time_SOLVEX = "
               << std::chrono::duration_cast<double_seconds>(time_SOLVEX).count()
               << std::endl;
+#endif
     write(std::cout, lend(residual_norm));
 #endif
 }
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index e88e7a4f06c..54d1fe5d353 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -79,7 +79,8 @@ namespace gmres_mixed {
         matrix::Dense<_type1> *arnoldi_norm, size_type iter,                  \
         Array<size_type> *final_iter_nums,                                    \
         const Array<stopping_status> *stop_status,                            \
-        Array<stopping_status> *reorth_status, Array<size_type> *num_reorth)
+        Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,  \
+        int *num_reorth_steps, int *num_reorth_vectors)
 
 
 #define GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(_type1, _type2)          \
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index b36b06eebe2..d554b24ac01 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -510,7 +510,8 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                          matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
                          const stopping_status *stop_status,
                          stopping_status *reorth_status,
-                         Array<size_type> *num_reorth)
+                         Array<size_type> *num_reorth, int *num_reorth_steps,
+                         int *num_reorth_vectors)
 {
     const auto stride_next_krylov = next_krylov_basis->get_stride();
     const auto stride_krylov = krylov_bases->get_stride();
@@ -521,11 +522,17 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     auto cublas_handle = exec->get_cublas_handle();
     const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
                          exec->get_num_multiprocessor() * 2);
+    const dim3 grid_size_num_iters(
+        ceildiv(dim_size[1] * (iter + 1), default_dot_dim),
+        exec->get_num_multiprocessor() * 2);
     const dim3 block_size(default_dot_dim, default_dot_dim);
     size_type numReorth;
 
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<ValueType>());
+    // std::cout << dim_size[0] << " - " << dim_size[1] << std::endl;
+    // std::cout << krylov_bases->get_size()[0] << " - "
+    //           << krylov_bases->get_size()[1] << std::endl;
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
@@ -536,36 +543,28 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     exec->synchronize();
     auto start_1 = std::chrono::steady_clock::now();
 #endif
-    /*
-        for (size_type k = 0; k < iter + 1; ++k) {
-            components::fill_array(
-                exec, hessenberg_iter->get_values() + k * stride_hessenberg,
-                dim_size[1], zero<ValueType>());
-            multidot_kernel<<<grid_size, block_size>>>(
-                k, dim_size[0], dim_size[1],
-                as_cuda_type(next_krylov_basis->get_const_values()),
-                stride_next_krylov,
-                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
-                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-                as_cuda_type(stop_status));
-        }
-    */
-    /*
-    for (size_type k = 0; k < iter + 1; ++k) {
-        components::fill_array(exec,
-            hessenberg_iter->get_values() + k * stride_hessenberg, dim_size[1],
-            zero<ValueType>());
-    }
-    */
     /* */
     zero_matrix(iter + 1, dim_size[1], stride_hessenberg,
                 hessenberg_iter->get_values());
+    /*
     multidot_kernel_num_iters<<<grid_size, block_size>>>(
         iter + 1, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
         as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
         as_cuda_type(stop_status));
+    */
+    /* */
+    //    write(std::cout, krylov_bases);
+    //    write(std::cout, next_krylov_basis);
+    multidot_kernel_num_iters_1<<<grid_size_num_iters, block_size>>>(
+        iter + 1, dim_size[0], dim_size[1],
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+        as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+        as_cuda_type(stop_status));
+    // exec->synchronize();
+    // write(std::cout, hessenberg_iter);
     /* */
 #ifdef TIMING
     exec->synchronize();
@@ -581,19 +580,6 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     exec->synchronize();
     auto start_2 = std::chrono::steady_clock::now();
 #endif
-    /*
-    for (size_type k = 0; k < iter + 1; ++k) {
-        update_next_krylov_kernel<default_block_size>
-            <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
-               default_block_size>>>(
-                k, dim_size[0], dim_size[1],
-                as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov,
-                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
-                as_cuda_type(hessenberg_iter->get_const_values()),
-                stride_hessenberg, as_cuda_type(stop_status));
-    }
-    */
     update_next_krylov_kernel_num_iters<default_block_size>
         <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
            default_block_size>>>(
@@ -670,45 +656,27 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
 #endif
     // numReorth <= number of next_krylov vector to be reorthogonalization
     for (size_type l = 1; (numReorth > 0) && (l < 3); l++) {
-        /*
-        for (size_type k = 0; k < iter + 1; ++k) {
-            components::fill_array(
-                exec, buffer_iter->get_values() + k * stride_buffer,
-                dim_size[1], zero<ValueType>());
-            multidot_kernel<<<grid_size, block_size>>>(
-                k, dim_size[0], dim_size[1],
-                as_cuda_type(next_krylov_basis->get_const_values()),
-                stride_next_krylov,
-                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
-                as_cuda_type(buffer_iter->get_values()), stride_buffer,
-                as_cuda_type(stop_status));
-        }
-        */
+        (*num_reorth_steps)++;
+        (*num_reorth_vectors) += iter;
         zero_matrix(iter + 1, dim_size[1], stride_buffer,
                     buffer_iter->get_values());
+        /*
         multidot_kernel_num_iters<<<grid_size, block_size>>>(
             iter + 1, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
             stride_next_krylov, as_cuda_type(krylov_bases->get_const_values()),
             stride_krylov, as_cuda_type(buffer_iter->get_values()),
             stride_buffer, as_cuda_type(stop_status));
+        */
+        multidot_kernel_num_iters_1<<<grid_size_num_iters, block_size>>>(
+            iter + 1, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov, as_cuda_type(krylov_bases->get_const_values()),
+            stride_krylov, as_cuda_type(buffer_iter->get_values()),
+            stride_buffer, as_cuda_type(stop_status));
         // for i in 1:iter
         //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
         // end
-        /*
-        for (size_type k = 0; k < iter + 1; ++k) {
-            update_next_krylov_kernel_and_add<default_block_size><<<
-                ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
-                default_block_size>>>(
-                k, dim_size[0], dim_size[1],
-                as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov,
-                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
-                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-                as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
-                as_cuda_type(stop_status), as_cuda_type(reorth_status));
-        }
-        */
         update_next_krylov_kernel_num_iters_and_add<default_block_size>
             <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
                default_block_size>>>(
@@ -827,7 +795,8 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
             matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
             Array<size_type> *final_iter_nums,
             const Array<stopping_status> *stop_status,
-            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth)
+            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
+            int *num_reorth_steps, int *num_reorth_vectors)
 {
     increase_final_iteration_numbers_kernel<<<
         static_cast<unsigned int>(
@@ -849,7 +818,8 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
     finish_arnoldi_CGS2(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
                         buffer_iter, arnoldi_norm, iter,
                         stop_status->get_const_data(),
-                        reorth_status->get_data(), num_reorth);
+                        reorth_status->get_data(), num_reorth, num_reorth_steps,
+                        num_reorth_vectors);
     givens_rotation(exec, givens_sin, givens_cos, hessenberg_iter,
                     residual_norm, residual_norm_collection, b_norm, iter,
                     stop_status);
diff --git a/cuda/test/solver/gmres_mixed_kernels.cpp b/cuda/test/solver/gmres_mixed_kernels.cpp
index d3fe27800f6..7637147a49e 100644
--- a/cuda/test/solver/gmres_mixed_kernels.cpp
+++ b/cuda/test/solver/gmres_mixed_kernels.cpp
@@ -258,19 +258,23 @@ TEST_F(GmresMixed, CudaGmresMixedStep1IsEquivalentToRef)
 {
     initialize_data();
     int iter = 5;
+    int num_reorth_steps = 0, num_reorth_vectors = 0;
+    int d_num_reorth_steps = 0, d_num_reorth_vectors = 0;
 
     gko::kernels::reference::gmres_mixed::step_1(
         ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
         residual_norm.get(), residual_norm_collection.get(), krylov_bases.get(),
         hessenberg_iter.get(), buffer_iter.get(), b_norm.get(),
         arnoldi_norm.get(), iter, final_iter_nums.get(), stop_status.get(),
-        reorth_status.get(), num_reorth.get());
+        reorth_status.get(), num_reorth.get(), &num_reorth_steps,
+        &num_reorth_vectors);
     gko::kernels::cuda::gmres_mixed::step_1(
         cuda, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
         d_residual_norm.get(), d_residual_norm_collection.get(),
         d_krylov_bases.get(), d_hessenberg_iter.get(), d_buffer_iter.get(),
         d_b_norm.get(), d_arnoldi_norm.get(), iter, d_final_iter_nums.get(),
-        d_stop_status.get(), d_reorth_status.get(), d_num_reorth.get());
+        d_stop_status.get(), d_reorth_status.get(), d_num_reorth.get(),
+        &num_reorth_steps, &num_reorth_vectors);
 
     GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
diff --git a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp b/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
index 0100f4b041f..857347741a2 100644
--- a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
+++ b/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
@@ -57,8 +57,8 @@ int main(int argc, char *argv[])
     using mtx = gko::matrix::Csr<>;
     // The gko::solver::Cg is used here, but any other solver class can also be
     // used.
-    using gmres_mixed = gko::solver::GmresMixed<double, float>;
-    //    using gmres_mixed = gko::solver::GmresMixed<>;
+    //    using gmres_mixed = gko::solver::GmresMixed<double, float>;
+    using gmres_mixed = gko::solver::GmresMixed<>;
     using bj = gko::preconditioner::Jacobi<>;
 
     // Print the ginkgo version information.
@@ -157,6 +157,14 @@ int main(int argc, char *argv[])
             x = vec::create(exec, gko::dim<2>{sizesA[0], 1});
             for (int i = 0; i < sizesA[0]; i++) x->at(i) = 0.0;
         }
+        /**/
+        if (std::string(argv[1]) == "cuda") {
+            b = gko::read<vec>(std::ifstream("data/one.mtx"), exec);
+        } else {
+            b = vec::create(exec, gko::dim<2>{sizesA[0], 1});
+            for (int i = 0; i < sizesA[0]; i++) b->at(i) = 1.0;
+        }
+        /**/
     }
     // @sect3{Creating the solver}
     // Generate the gko::solver factory. Ginkgo uses the concept of Factories to
@@ -172,13 +180,15 @@ int main(int argc, char *argv[])
             .with_criteria(
                 //    gko::stop::Iteration::build().with_max_iters(20u).on(exec),
                 gko::stop::Iteration::build()
-                    .with_max_iters(A->get_size()[0])
+                    //    .with_max_iters(A->get_size()[0])
+                    .with_max_iters(20000u)
                     .on(exec),
                 //    gko::stop::Iteration::build().with_max_iters(100000u).on(exec),
                 gko::stop::ResidualNormReduction<>::build()
                     //    .with_reduction_factor(1e-15)
                     //    .with_reduction_factor(1e-12)
-                    .with_reduction_factor(1e-13)
+                    //    .with_reduction_factor(1e-13)
+                    .with_reduction_factor(1e-9)
                     .on(exec))
             // Add preconditioner, these 2 lines are the only
             // difference from the simple solver example
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index d8a6ab723e3..9f51ea3d2e6 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -452,6 +452,7 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
  *                value type.
  */
 // #if defined(__CUDACC__) || defined(__HIPCC__)
+// #if defined(__CUDA_ARCH__)
 // #define GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(_macro) \
 //     template _macro(double, half);                  \
 //     template _macro(double, float);                 \
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index ea732d62249..b278d8a9085 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -725,7 +725,8 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
             matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
             Array<size_type> *final_iter_nums,
             const Array<stopping_status> *stop_status,
-            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth)
+            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
+            int *num_reorth_steps, int *num_reorth_vectors)
 {
 #pragma omp parallel for
     for (size_type i = 0; i < final_iter_nums->get_num_elems(); ++i) {
diff --git a/omp/test/solver/gmres_mixed_kernels.cpp b/omp/test/solver/gmres_mixed_kernels.cpp
index 917b31fa763..432e13c0c90 100644
--- a/omp/test/solver/gmres_mixed_kernels.cpp
+++ b/omp/test/solver/gmres_mixed_kernels.cpp
@@ -256,19 +256,23 @@ TEST_F(GmresMixed, OmpGmresMixedStep1IsEquivalentToRef)
 {
     initialize_data();
     int iter = 5;
+    int num_reorth_steps = 0, num_reorth_vectors = 0;
+    int d_num_reorth_steps = 0, d_num_reorth_vectors = 0;
 
     gko::kernels::reference::gmres_mixed::step_1(
         ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
         residual_norm.get(), residual_norm_collection.get(), krylov_bases.get(),
         hessenberg_iter.get(), buffer_iter.get(), b_norm.get(),
         arnoldi_norm.get(), iter, final_iter_nums.get(), stop_status.get(),
-        reorth_status.get(), num_reorth.get());
+        reorth_status.get(), num_reorth.get(), &num_reorth_steps,
+        &num_reorth_vectors);
     gko::kernels::omp::gmres_mixed::step_1(
         omp, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
         d_residual_norm.get(), d_residual_norm_collection.get(),
         d_krylov_bases.get(), d_hessenberg_iter.get(), d_buffer_iter.get(),
         d_b_norm.get(), d_arnoldi_norm.get(), iter, d_final_iter_nums.get(),
-        d_stop_status.get(), d_reorth_status.get(), d_num_reorth.get());
+        d_stop_status.get(), d_reorth_status.get(), d_num_reorth.get(),
+        &num_reorth_steps, &num_reorth_vectors);
 
     GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index cca64c738cb..d6e6bb12b31 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -530,7 +530,8 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
             matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
             Array<size_type> *final_iter_nums,
             const Array<stopping_status> *stop_status,
-            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth)
+            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
+            int *num_reorth_steps, int *num_reorth_vectors)
 {
     /*
         if (Reorthogonalization) std::cout << "Reorthogonalization";

From 605a388f841585999d64e043eb10a9590ccbe079 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 13 May 2020 10:37:03 +0200
Subject: [PATCH 0343/1008] Add Accessor support and extend reference test

Added an accessor header file (name might have to change in future)
and used it in all mixed precision kernels (but for now only for the
reduced precision accesses).

Also adds some minor fixes:
- removed unused code in the example in hopes that it compiles on
  windows
- added HIP stubs to allow HIP compilation
---
 common/solver/gmres_mixed_kernels.hpp.inc     | 121 ++++----
 core/components/accessor.hpp                  | 233 +++++++++++++++
 core/device_hooks/common_kernels.inc.cpp      |   8 +-
 core/solver/gmres_mixed.cpp                   |  43 ++-
 core/solver/gmres_mixed_kernels.hpp           |  46 ++-
 cuda/solver/gmres_mixed_kernels.cu            | 220 ++++++++-------
 cuda/test/solver/gmres_mixed_kernels.cpp      |  90 ++++--
 .../gmres-mixed-solver-jacobi-matrix.cpp      |   2 -
 hip/solver/gmres_mixed_kernels.hip.cpp        | 157 +++++++++++
 include/ginkgo/core/base/types.hpp            |  44 ---
 omp/solver/gmres_mixed_kernels.cpp            | 142 +++++-----
 omp/test/solver/gmres_mixed_kernels.cpp       |  98 +++++--
 reference/solver/gmres_mixed_kernels.cpp      | 106 +++----
 reference/test/solver/gmres_mixed_kernels.cpp | 265 +++++++++++-------
 14 files changed, 1089 insertions(+), 486 deletions(-)
 create mode 100644 core/components/accessor.hpp
 create mode 100644 hip/solver/gmres_mixed_kernels.hip.cpp

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 9a9297809b9..fe2a354840a 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -85,17 +85,16 @@ template <size_type block_size, typename ValueType,
           typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(block_size) void initialize_2_1_kernel(
     size_type num_rows, size_type num_rhs, size_type krylov_dim,
-    ValueTypeKrylovBases *__restrict__ krylov_bases, size_type stride_krylov,
+    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
     ValueType *__restrict__ residual_norm_collection,
     size_type stride_residual_nc)
 {
     const auto global_id = thread::get_thread_id_flat();
-    const auto row_idx = global_id / stride_krylov;
-    const auto col_idx = global_id % stride_krylov;
+    const auto row_idx = global_id / krylov_bases.get_stride();
+    const auto col_idx = global_id % krylov_bases.get_stride();
 
     if (row_idx < num_rows && col_idx < (krylov_dim + 1) * num_rhs) {
-        krylov_bases[row_idx * stride_krylov + col_idx] =
-            zero<ValueTypeKrylovBases>();
+        krylov_bases.write(row_idx, col_idx, zero<ValueType>());
     }
 
     if (row_idx < krylov_dim + 1 && col_idx < num_rhs) {
@@ -113,7 +112,7 @@ __global__ __launch_bounds__(block_size) void initialize_2_2_kernel(
     const ValueType *__restrict__ residual, size_type stride_residual,
     const ValueType *__restrict__ residual_norm,
     ValueType *__restrict__ residual_norm_collection,
-    ValueTypeKrylovBases *__restrict__ krylov_bases, size_type stride_krylov,
+    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
     size_type *__restrict__ final_iter_nums)
 {
@@ -129,7 +128,7 @@ __global__ __launch_bounds__(block_size) void initialize_2_2_kernel(
     if (row_idx < num_rows && col_idx < num_rhs) {
         auto value = residual[row_idx * stride_residual + col_idx] /
                      residual_norm[col_idx];
-        krylov_bases[row_idx * stride_krylov + col_idx] = value;
+        krylov_bases.write(row_idx, col_idx, value);
         next_krylov_basis[row_idx * stride_next_krylov + col_idx] = value;
     }
 }
@@ -152,9 +151,8 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
     size_type k, size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
     size_type stride_next_krylov,
-    const ValueTypeKrylovBases *__restrict__ krylov_bases,
-    size_type stride_krylov, ValueType *__restrict__ hessenberg_iter,
-    size_type stride_hessenberg,
+    Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
     const auto tidx = threadIdx.x;
@@ -176,9 +174,8 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
         for (size_type i = start_row + tidy; i < end_row;
              i += default_dot_dim) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
-            const auto krylov_idx = i * stride_krylov + krylov_col;
-            local_res +=
-                next_krylov_basis[next_krylov_idx] * krylov_bases[krylov_idx];
+            local_res += next_krylov_basis[next_krylov_idx] *
+                         krylov_bases.read(i, krylov_col);
         }
     }
     reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
@@ -203,9 +200,8 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
     size_type num_iters, size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
     size_type stride_next_krylov,
-    const ValueTypeKrylovBases *__restrict__ krylov_bases,
-    size_type stride_krylov, ValueType *__restrict__ hessenberg_iter,
-    size_type stride_hessenberg,
+    Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
     const auto tidx = threadIdx.x;
@@ -229,9 +225,8 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
             for (size_type i = start_row + tidy; i < end_row;
                  i += default_dot_dim) {
                 const auto next_krylov_idx = i * stride_next_krylov + col_idx;
-                const auto krylov_idx = i * stride_krylov + krylov_col;
                 local_res += next_krylov_basis[next_krylov_idx] *
-                             krylov_bases[krylov_idx];
+                             krylov_bases.read(i, krylov_col);
             }
         }
         reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
@@ -257,9 +252,8 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_1(
     size_type num_iters, size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
     size_type stride_next_krylov,
-    const ValueTypeKrylovBases *__restrict__ krylov_bases,
-    size_type stride_krylov, ValueType *__restrict__ hessenberg_iter,
-    size_type stride_hessenberg,
+    Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
     const auto tidx = threadIdx.x;
@@ -286,9 +280,8 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_1(
         for (size_type i = start_row + tidy; i < end_row;
              i += default_dot_dim) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
-            const auto krylov_idx = i * stride_krylov + krylov_col;
-            local_res +=
-                next_krylov_basis[next_krylov_idx] * krylov_bases[krylov_idx];
+            local_res += next_krylov_basis[next_krylov_idx] *
+                         krylov_bases.read(i, krylov_col);
         }
     }
     reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
@@ -337,9 +330,8 @@ template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(block_size) void update_next_krylov_kernel(
     size_type k, size_type num_rows, size_type num_cols,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-    const ValueTypeKrylovBases *__restrict__ krylov_bases,
-    size_type stride_krylov, const ValueType *__restrict__ hessenberg_iter,
-    size_type stride_hessenberg,
+    Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    const ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
     const auto global_id = thread::get_thread_id_flat();
@@ -349,12 +341,13 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel(
     if (row_idx < num_rows && col_idx < num_cols &&
         !stop_status[col_idx].has_stopped()) {
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-        const auto krylov_idx =
-            row_idx * stride_krylov + k * num_cols + col_idx;
+        // const auto krylov_idx =
+        //    row_idx * stride_krylov + k * num_cols + col_idx;
         const auto hessenberg_idx = k * stride_hessenberg + col_idx;
 
         next_krylov_basis[next_krylov_idx] -=
-            hessenberg_iter[hessenberg_idx] * krylov_bases[krylov_idx];
+            hessenberg_iter[hessenberg_idx] *
+            krylov_bases.read(row_idx, k * num_cols + col_idx);
     }
 }
 
@@ -366,8 +359,8 @@ __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        const ValueTypeKrylovBases *__restrict__ krylov_bases,
-        size_type stride_krylov, const ValueType *__restrict__ hessenberg_iter,
+        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
 {
@@ -379,12 +372,13 @@ __global__
         !stop_status[col_idx].has_stopped()) {
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
         for (size_type k = 0; k < num_iters; ++k) {
-            const auto krylov_idx =
-                row_idx * stride_krylov + k * num_cols + col_idx;
+            // const auto krylov_idx =
+            //    row_idx * stride_krylov + k * num_cols + col_idx;
             const auto hessenberg_idx = k * stride_hessenberg + col_idx;
 
             next_krylov_basis[next_krylov_idx] -=
-                hessenberg_iter[hessenberg_idx] * krylov_bases[krylov_idx];
+                hessenberg_iter[hessenberg_idx] *
+                krylov_bases.read(row_idx, k * num_cols + col_idx);
         }
     }
 }
@@ -397,8 +391,8 @@ __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_and_check(
         size_type k, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        const ValueTypeKrylovBases *__restrict__ krylov_bases,
-        size_type stride_krylov, const ValueType *__restrict__ hessenberg_iter,
+        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg, const ValueType *__restrict__ arnoldi_norm,
         size_type stride_norm, const stopping_status *__restrict__ stop_status,
         stopping_status *__restrict__ reorth_status,
@@ -411,14 +405,15 @@ __global__
     if (row_idx < num_rows && col_idx < num_cols &&
         !stop_status[col_idx].has_stopped()) {
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-        const auto krylov_idx =
-            row_idx * stride_krylov + k * num_cols + col_idx;
+        // const auto krylov_idx =
+        //    row_idx * stride_krylov + k * num_cols + col_idx;
         const auto hessenberg_idx = k * stride_hessenberg + col_idx;
         const auto arnoldi_idx = col_idx;
         const auto reorth_idx = col_idx;
 
         next_krylov_basis[next_krylov_idx] -=
-            hessenberg_iter[hessenberg_idx] * krylov_bases[krylov_idx];
+            hessenberg_iter[hessenberg_idx] *
+            krylov_bases.read(row_idx, k * num_cols + col_idx);
         if (row_idx == 0) {
             if ((hessenberg_iter[hessenberg_idx] *
                  hessenberg_iter[hessenberg_idx]) >
@@ -439,10 +434,10 @@ template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
     size_type k, size_type num_rows, size_type num_cols,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-    const ValueTypeKrylovBases *__restrict__ krylov_bases,
-    size_type stride_krylov, ValueType *__restrict__ hessenberg_iter,
-    size_type stride_hessenberg, const ValueType *__restrict__ buffer_iter,
-    size_type stride_buffer, const stopping_status *__restrict__ stop_status,
+    Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    const ValueType *__restrict__ buffer_iter, size_type stride_buffer,
+    const stopping_status *__restrict__ stop_status,
     const stopping_status *__restrict__ reorth_status)
 {
     const auto global_id = thread::get_thread_id_flat();
@@ -453,12 +448,13 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
         !stop_status[col_idx].has_stopped() &&
         !reorth_status[col_idx].has_stopped()) {
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-        const auto krylov_idx =
-            row_idx * stride_krylov + k * num_cols + col_idx;
+        // const auto krylov_idx =
+        //    row_idx * stride_krylov + k * num_cols + col_idx;
         const auto hessenberg_idx = k * stride_hessenberg + col_idx;
         const auto buffer_idx = k * stride_buffer + col_idx;
         next_krylov_basis[next_krylov_idx] -=
-            buffer_iter[buffer_idx] * krylov_bases[krylov_idx];
+            buffer_iter[buffer_idx] *
+            krylov_bases.read(row_idx, k * num_cols + col_idx);
         if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
             hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
         }
@@ -473,10 +469,9 @@ __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_and_add(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        const ValueTypeKrylovBases *__restrict__ krylov_bases,
-        size_type stride_krylov, ValueType *__restrict__ hessenberg_iter,
-        size_type stride_hessenberg, const ValueType *__restrict__ buffer_iter,
-        size_type stride_buffer,
+        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+        const ValueType *__restrict__ buffer_iter, size_type stride_buffer,
         const stopping_status *__restrict__ stop_status,
         const stopping_status *__restrict__ reorth_status)
 {
@@ -489,12 +484,13 @@ __global__
         !reorth_status[col_idx].has_stopped()) {
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
         for (size_type k = 0; k < num_iters; ++k) {
-            const auto krylov_idx =
-                row_idx * stride_krylov + k * num_cols + col_idx;
+            // const auto krylov_idx =
+            //    row_idx * stride_krylov + k * num_cols + col_idx;
             const auto hessenberg_idx = k * stride_hessenberg + col_idx;
             const auto buffer_idx = k * stride_buffer + col_idx;
             next_krylov_basis[next_krylov_idx] -=
-                buffer_iter[buffer_idx] * krylov_bases[krylov_idx];
+                buffer_iter[buffer_idx] *
+                krylov_bases.read(row_idx, k * num_cols + col_idx);
             if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
                 hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
             }
@@ -577,7 +573,7 @@ template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(block_size) void update_krylov_next_krylov_kernel(
     size_type iter, size_type num_rows, size_type num_cols,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-    ValueTypeKrylovBases *__restrict__ krylov_bases, size_type stride_krylov,
+    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
     const ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -590,14 +586,15 @@ __global__ __launch_bounds__(block_size) void update_krylov_next_krylov_kernel(
     if (row_idx < num_rows && col_idx < num_cols &&
         !stop_status[col_idx].has_stopped()) {
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-        const auto krylov_idx =
-            row_idx * stride_krylov + num_cols * (iter + 1) + col_idx;
+        // const auto krylov_idx =
+        //    row_idx * stride_krylov + num_cols * (iter + 1) + col_idx;
 
         const auto next_krylov_value =
             next_krylov_basis[next_krylov_idx] / hessenberg;
 
         next_krylov_basis[next_krylov_idx] = next_krylov_value;
-        krylov_bases[krylov_idx] = next_krylov_value;
+        krylov_bases.write(row_idx, num_cols * (iter + 1) + col_idx,
+                           next_krylov_value);
     }
 }
 
@@ -762,9 +759,9 @@ template <size_type block_size, typename ValueType,
           typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(block_size) void calculate_Qy_kernel(
     size_type num_rows, size_type num_cols, size_type num_rhs,
-    const ValueTypeKrylovBases *__restrict__ krylov_bases,
-    size_type stride_krylov, const ValueType *__restrict__ y,
-    size_type stride_y, ValueType *__restrict__ before_preconditioner,
+    Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    const ValueType *__restrict__ y, size_type stride_y,
+    ValueType *__restrict__ before_preconditioner,
     size_type stride_preconditioner,
     const size_type *__restrict__ final_iter_nums)
 {
@@ -776,7 +773,7 @@ __global__ __launch_bounds__(block_size) void calculate_Qy_kernel(
         before_preconditioner[global_id] = zero<ValueType>();
         for (size_type j = 0; j < final_iter_nums[col_id]; ++j) {
             before_preconditioner[global_id] +=
-                krylov_bases[row_id * stride_krylov + j * num_rhs + col_id] *
+                krylov_bases.read(row_id, j * num_rhs + col_id) *
                 y[j * stride_y + col_id];
         }
     }
diff --git a/core/components/accessor.hpp b/core/components/accessor.hpp
new file mode 100644
index 00000000000..9d3c5b05b3e
--- /dev/null
+++ b/core/components/accessor.hpp
@@ -0,0 +1,233 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_COMMON_ACCESSOR_HPP_
+#define GKO_CORE_COMMON_ACCESSOR_HPP_
+
+
+#include <cinttypes>
+
+
+#include <ginkgo/core/base/types.hpp>
+
+
+namespace gko {
+
+
+/**
+ * @internal
+ *
+ * The Accessor class hides the underlying storage_ format and provides a simple
+ * interface for accessing a one dimensional storage.
+ *
+ * This class only manages the accesses, however, and not the memory itself.
+ */
+template <typename StorageType, typename ArithmeticType = double>
+class Accessor {
+public:
+    using storage_type = StorageType;
+    using arithmetic_type = ArithmeticType;
+
+    /**
+     * Creates the accessor with an already allocated storage space.
+     */
+    Accessor(storage_type *storage) : storage_{storage} {}
+
+    /**
+     * Reads the value at the given index.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES arithmetic_type read(IndexType idx) const
+    {
+        // Make use of the restrict (and const) qualifier. If the restrict
+        // qualifier would have been put on the class attribute, it would be
+        // ignored by nvcc.
+        const storage_type *GKO_RESTRICT rest_storage = storage_;
+        return static_cast<arithmetic_type>(rest_storage[idx]);
+    }
+
+    /**
+     * Writes the given value at the given index.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES void write(IndexType idx, arithmetic_type value)
+    {
+        storage_type *GKO_RESTRICT rest_storage = storage_;
+        rest_storage[idx] = static_cast<storage_type>(value);
+    }
+
+private:
+    storage_type *storage_;
+};
+
+
+template <typename StorageType, typename ArithmeticType>
+class Accessor2d;
+
+
+/**
+ * @internal
+ *
+ * The Accessor2d class hides the underlying storage_ format and provides a
+ * simple interface for accessing a one dimensional storage.
+ *
+ * This class only manages the accesses, however, and not the memory itself.
+ */
+template <typename StorageType, typename ArithmeticType = double>
+class Accessor2dConst {
+public:
+    using storage_type = StorageType;
+    using arithmetic_type = ArithmeticType;
+
+    /**
+     * Creates the accessor with an already allocated storage space with a
+     * stride.
+     */
+    Accessor2dConst(const storage_type *storage, size_type stride)
+        : storage_{storage}, stride_{stride}
+    {}
+
+    /**
+     * Creates a const accessor from a non-const accessor.
+     */
+    Accessor2dConst(Accessor2d<StorageType, ArithmeticType> acc)
+        : storage_{acc.get_const_storage()}, stride_{acc.get_stride()}
+    {}
+
+    /**
+     * Reads the value at the given index.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES arithmetic_type read(IndexType row, IndexType col) const
+    {
+        // Make use of the restrict (and const) qualifier. If the restrict
+        // qualifier would have been put on the class attribute, it would be
+        // ignored by nvcc.
+        const storage_type *GKO_RESTRICT rest_storage = storage_;
+        return static_cast<arithmetic_type>(rest_storage[row * stride_ + col]);
+    }
+
+    GKO_ATTRIBUTES size_type get_stride() const { return stride_; }
+
+    GKO_ATTRIBUTES const storage_type *get_storage() const { return storage_; }
+
+    GKO_ATTRIBUTES const storage_type *get_const_storage() const
+    {
+        return storage_;
+    }
+
+private:
+    const storage_type *storage_;
+    size_type stride_;
+};
+
+
+/**
+ * @internal
+ *
+ * The Accessor2d class hides the underlying storage_ format and provides a
+ * simple interface for accessing a one dimensional storage.
+ *
+ * This class only manages the accesses, however, and not the memory itself.
+ */
+template <typename StorageType, typename ArithmeticType = double>
+class Accessor2d {
+public:
+    using storage_type = StorageType;
+    using arithmetic_type = ArithmeticType;
+
+    /**
+     * Creates an empty accessor pointing to a nullptr.
+     */
+    Accessor2d() : storage_{nullptr}, stride_{} {}
+
+    /**
+     * Creates the accessor with an already allocated storage space with a
+     * stride.
+     */
+    Accessor2d(storage_type *storage, size_type stride)
+        : storage_{storage}, stride_{stride}
+    {}
+
+    operator Accessor2dConst<storage_type, arithmetic_type>() const
+    {
+        return {storage_, stride_};
+    }
+
+    Accessor2dConst<storage_type, arithmetic_type> to_const() const
+    {
+        return {storage_, stride_};
+    }
+
+    /**
+     * Reads the value at the given index.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES arithmetic_type read(IndexType row, IndexType col) const
+    {
+        // Make use of the restrict (and const) qualifier. If the restrict
+        // qualifier would have been put on the class attribute, it would be
+        // ignored by nvcc.
+        const storage_type *GKO_RESTRICT rest_storage = storage_;
+        return static_cast<arithmetic_type>(rest_storage[row * stride_ + col]);
+    }
+
+    /**
+     * Writes the given value at the given index.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES void write(IndexType row, IndexType col,
+                              arithmetic_type value)
+    {
+        storage_type *GKO_RESTRICT rest_storage = storage_;
+        rest_storage[row * stride_ + col] = static_cast<storage_type>(value);
+    }
+
+    GKO_ATTRIBUTES size_type get_stride() const { return stride_; }
+
+    GKO_ATTRIBUTES storage_type *get_storage() { return storage_; }
+
+    GKO_ATTRIBUTES const storage_type *get_const_storage() const
+    {
+        return storage_;
+    }
+
+private:
+    storage_type *storage_;
+    size_type stride_;
+};
+
+
+}  // namespace gko
+
+
+#endif  // GKO_CORE_COMMON_ACCESSOR_HPP_
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 95e0e346600..b09326762a6 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -583,7 +583,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
 template <typename ValueType, typename ValueTypeKrylovBases>
 GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(ValueType, ValueTypeKrylovBases)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
 
 /*
@@ -597,12 +597,14 @@ GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(GKO_DECLARE_GMRES_MIXED_BOOL_STEP_1_KER
 template <typename ValueType, typename ValueTypeKrylovBases>
 GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
 
 template <typename ValueType, typename ValueTypeKrylovBases>
 GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(ValueType, ValueTypeKrylovBases)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
 
 
 }  // namespace gmres_mixed
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 1a510bc45a4..90b51968490 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -47,6 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <iostream>
 
 
+#include "core/components/accessor.hpp"
 #include "core/solver/gmres_mixed_kernels.hpp"
 
 
@@ -88,7 +89,9 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix_);
 
     using Vector = matrix::Dense<ValueType>;
-    using Krylov = matrix::Dense<ValueTypeKrylovBases>;
+    using LowArray = Array<ValueTypeKrylovBases>;
+    using KrylovAccessor = Accessor2d<ValueTypeKrylovBases, ValueType>;
+
 
     constexpr uint8 RelativeStoppingId{1};
 
@@ -100,9 +103,13 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     auto dense_b = as<const Vector>(b);
     auto dense_x = as<Vector>(x);
     auto residual = Vector::create_with_config_of(dense_b);
-    auto krylov_bases = Krylov::create(
-        exec, dim<2>{system_matrix_->get_size()[1],
-                     (krylov_dim_mixed_ + 1) * dense_b->get_size()[1]});
+    const dim<2> krylov_bases_dim{
+        system_matrix_->get_size()[1],
+        (krylov_dim_mixed_ + 1) * dense_b->get_size()[1]};
+    const size_type krylov_bases_stride = krylov_bases_dim[1];
+    LowArray krylov_bases(exec, krylov_bases_dim[0] * krylov_bases_stride);
+    KrylovAccessor krylov_bases_accessor(krylov_bases.get_data(),
+                                         krylov_bases_stride);
     auto next_krylov_basis = Vector::create_with_config_of(dense_b);
     std::shared_ptr<matrix::Dense<ValueType>> preconditioned_vector =
         Vector::create_with_config_of(dense_b);
@@ -131,7 +138,6 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
                                        dense_b->get_size()[1]);
     Array<stopping_status> reorth_status(this->get_executor(),
                                          dense_b->get_size()[1]);
-    //                                         krylov_dim_mixed_ + 1);
     Array<size_type> num_reorth(this->get_executor(), dense_b->get_size()[1]);
     int num_restarts = 0, num_reorth_steps = 0, num_reorth_vectors = 0;
 
@@ -148,7 +154,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
 
     exec->run(gmres_mixed::make_initialize_2(
         residual.get(), residual_norm.get(), residual_norm_collection.get(),
-        krylov_bases.get(), next_krylov_basis.get(), &final_iter_nums,
+        krylov_bases_accessor, next_krylov_basis.get(), &final_iter_nums,
         krylov_dim_mixed_));
     // residual_norm = norm(residual)
     // residual_norm_collection = {residual_norm, 0, ..., 0}
@@ -199,9 +205,9 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
             num_restarts++;
             // Restart
             exec->run(gmres_mixed::make_step_2(
-                residual_norm_collection.get(), krylov_bases.get(),
-                hessenberg.get(), y.get(), before_preconditioner.get(),
-                &final_iter_nums));
+                residual_norm_collection.get(),
+                krylov_bases_accessor.to_const(), hessenberg.get(), y.get(),
+                before_preconditioner.get(), &final_iter_nums));
             // Solve upper triangular.
             // y = hessenberg \ residual_norm_collection
 
@@ -217,7 +223,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
             // residual = residual - Ax
             exec->run(gmres_mixed::make_initialize_2(
                 residual.get(), residual_norm.get(),
-                residual_norm_collection.get(), krylov_bases.get(),
+                residual_norm_collection.get(), krylov_bases_accessor,
                 next_krylov_basis.get(), &final_iter_nums, krylov_dim_mixed_));
             // residual_norm = norm(residual)
             // residual_norm_collection = {residual_norm, 0, ..., 0}
@@ -263,7 +269,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
         exec->run(gmres_mixed::make_step_1(
             next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
             residual_norm.get(), residual_norm_collection.get(),
-            krylov_bases.get(), hessenberg_iter.get(), buffer_iter.get(),
+            krylov_bases_accessor, hessenberg_iter.get(), buffer_iter.get(),
             b_norm.get(), arnoldi_norm.get(), restart_iter, &final_iter_nums,
             &stop_status, &reorth_status, &num_reorth, &num_reorth_steps,
             &num_reorth_vectors));
@@ -303,15 +309,26 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     exec->synchronize();
     auto t_aux_3 = std::chrono::steady_clock::now();
 #endif
+    /*
     auto krylov_bases_small = krylov_bases->create_submatrix(
         span{0, system_matrix_->get_size()[0]},
         span{0, dense_b->get_size()[1] * (restart_iter + 1)});
+    const dim<2> krylov_bases_dim{
+        system_matrix_->get_size()[1],
+        (krylov_dim_mixed_ + 1) * dense_b->get_size()[1]};
+    */
+    const dim<2> krylov_bases_small_dim{
+        system_matrix_->get_size()[0],
+        dense_b->get_size()[1] * (restart_iter + 1)};
+    KrylovAccessor krylov_bases_small_accessor{krylov_bases.get_data(),
+                                               krylov_bases_stride};
+
     auto hessenberg_small = hessenberg->create_submatrix(
         span{0, restart_iter},
         span{0, dense_b->get_size()[1] * (restart_iter)});
 
     exec->run(gmres_mixed::make_step_2(
-        residual_norm_collection.get(), krylov_bases_small.get(),
+        residual_norm_collection.get(), krylov_bases_small_accessor.to_const(),
         hessenberg_small.get(), y.get(), before_preconditioner.get(),
         &final_iter_nums));
 #ifdef TIMING_STEPS
@@ -395,7 +412,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(
 GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(GKO_DECLARE_GMRES_MIXED_BOOL);
 */
 #define GKO_DECLARE_GMRES_MIXED(_type1, _type2) class GmresMixed<_type1, _type2>
-GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED);
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED);
 
 }  // namespace solver
 }  // namespace gko
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index 54d1fe5d353..d4ed241faa6 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -40,6 +40,37 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/stop/stopping_status.hpp>
 
+
+#include "core/components/accessor.hpp"
+
+
+/**
+ * Instantiates a template for each value type with each lower precision type
+ * supported by Ginkgo for GmresMixed.
+ *
+ * @param _macro  A macro which expands the template instantiation
+ *                (not including the leading `template` specifier).
+ *                Should take two arguments:
+ *                1. the first will be used as the regular ValueType
+ *                     (precisions supported by Ginkgo), and
+ *                2. the second the value type of the reduced precision.
+ */
+// #if defined(__CUDACC__) || defined(__HIPCC__)
+// #define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro) \
+//     template _macro(double, half);                  \
+//     template _macro(double, float);                 \
+//     template _macro(double, double)
+// #else
+#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro) \
+    template _macro(double, double);                      \
+    template _macro(double, float);                       \
+    template _macro(float, float)  //;                  \
+    template _macro(std::complex<double>, std::complex<double>);  \
+    template _macro(std::complex<double>, std::complex<float>);  \
+    template _macro(std::complex<float>, std::complex<float>)
+// #endif
+
+
 namespace gko {
 namespace kernels {
 namespace gmres_mixed {
@@ -59,12 +90,11 @@ namespace gmres_mixed {
                       const matrix::Dense<_type1> *residual,           \
                       matrix::Dense<_type1> *residual_norm,            \
                       matrix::Dense<_type1> *residual_norm_collection, \
-                      matrix::Dense<_type2> *krylov_bases,             \
+                      Accessor2d<_type2, _type1> krylov_bases,         \
                       matrix::Dense<_type1> *next_krylov_basis,        \
                       Array<size_type> *final_iter_nums, size_type krylov_dim)
 
 
-// #define GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(_type1, _type2, _bool1, _bool2)
 #define GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(_type1, _type2)                 \
     void step_1(                                                              \
         std::shared_ptr<const DefaultExecutor> exec,                          \
@@ -72,7 +102,7 @@ namespace gmres_mixed {
         matrix::Dense<_type1> *givens_sin, matrix::Dense<_type1> *givens_cos, \
         matrix::Dense<_type1> *residual_norm,                                 \
         matrix::Dense<_type1> *residual_norm_collection,                      \
-        matrix::Dense<_type2> *krylov_bases,                                  \
+        Accessor2d<_type2, _type1> krylov_bases,                              \
         matrix::Dense<_type1> *hessenberg_iter,                               \
         matrix::Dense<_type1> *buffer_iter,                                   \
         const matrix::Dense<_type1> *b_norm,                                  \
@@ -86,12 +116,13 @@ namespace gmres_mixed {
 #define GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(_type1, _type2)          \
     void step_2(std::shared_ptr<const DefaultExecutor> exec,           \
                 const matrix::Dense<_type1> *residual_norm_collection, \
-                const matrix::Dense<_type2> *krylov_bases,             \
+                Accessor2dConst<_type2, _type1> krylov_bases,          \
                 const matrix::Dense<_type1> *hessenberg,               \
                 matrix::Dense<_type1> *y,                              \
                 matrix::Dense<_type1> *before_preconditioner,          \
                 const Array<size_type> *final_iter_nums)
 
+
 #define GKO_DECLARE_ALL_AS_TEMPLATES                                        \
     template <typename ValueType>                                           \
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL(ValueType);                 \
@@ -102,12 +133,7 @@ namespace gmres_mixed {
     GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases); \
     template <typename ValueType, typename ValueTypeKrylovBases>            \
     GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(ValueType, ValueTypeKrylovBases)
-/*
-    template <typename ValueType, typename ValueTypeKrylovBases, \
-              bool Reorthogonalization, bool MGS_CGS>                     \
-    GKO_DECLARE_GMRES_MIXED_BOOL_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases,\
-                                          Reorthogonalization, MGS_CGS);       \
-*/
+
 
 }  // namespace gmres_mixed
 
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index d554b24ac01..bc9d19f87ca 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -86,6 +86,22 @@ constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 #include "common/solver/gmres_mixed_kernels.hpp.inc"
 
 
+// Specialization, so the Accessor can use the same function as regular pointers
+template <typename Type1, typename Type2>
+Accessor2d<cuda_type<Type1>, cuda_type<Type2>> as_cuda_accessor(
+    Accessor2d<Type1, Type2> acc)
+{
+    return {as_cuda_type(acc.get_storage()), acc.get_stride()};
+}
+
+template <typename Type1, typename Type2>
+Accessor2dConst<cuda_type<Type1>, cuda_type<Type2>> as_cuda_accessor(
+    const Accessor2dConst<Type1, Type2> &acc)
+{
+    return {as_cuda_type(acc.get_storage()), acc.get_stride()};
+}
+
+
 template <typename ValueType>
 void zero_matrix(size_type m, size_type n, size_type stride, ValueType *array)
 {
@@ -130,21 +146,21 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
                   const matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
-                  matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                  Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
 {
     const auto num_rows = residual->get_size()[0];
     const auto num_rhs = residual->get_size()[1];
     const dim3 grid_dim_1(
-        ceildiv(num_rows * krylov_bases->get_stride(), default_block_size), 1,
+        ceildiv(num_rows * krylov_bases.get_stride(), default_block_size), 1,
         1);
     const dim3 block_dim(default_block_size, 1, 1);
     constexpr auto block_size = default_block_size;
 
     initialize_2_1_kernel<block_size><<<grid_dim_1, block_dim>>>(
         residual->get_size()[0], residual->get_size()[1], krylov_dim,
-        as_cuda_type(krylov_bases->get_values()), krylov_bases->get_stride(),
+        as_cuda_accessor(krylov_bases),
         as_cuda_type(residual_norm_collection->get_values()),
         residual_norm_collection->get_stride());
     residual->compute_norm2(residual_norm);
@@ -156,25 +172,25 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
         as_cuda_type(residual->get_const_values()), residual->get_stride(),
         as_cuda_type(residual_norm->get_const_values()),
         as_cuda_type(residual_norm_collection->get_values()),
-        as_cuda_type(krylov_bases->get_values()), krylov_bases->get_stride(),
+        as_cuda_accessor(krylov_bases),
         as_cuda_type(next_krylov_basis->get_values()),
         next_krylov_basis->get_stride(),
         as_cuda_type(final_iter_nums->get_data()));
 }
 
-GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi(std::shared_ptr<const CudaExecutor> exec,
                     matrix::Dense<ValueType> *next_krylov_basis,
-                    matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
                     matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
                     const stopping_status *stop_status)
 {
     const auto stride_next_krylov = next_krylov_basis->get_stride();
-    const auto stride_krylov = krylov_bases->get_stride();
+    const auto stride_krylov = krylov_bases.get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
     const auto dim_size = next_krylov_basis->get_size();
     auto cublas_handle = exec->get_cublas_handle();
@@ -188,16 +204,15 @@ void finish_arnoldi(std::shared_ptr<const CudaExecutor> exec,
         multidot_kernel<<<grid_size, block_size>>>(
             k, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_type(krylov_bases->get_const_values()),
-            stride_krylov, as_cuda_type(hessenberg_iter->get_values()),
-            stride_hessenberg, as_cuda_type(stop_status));
+            stride_next_krylov, as_cuda_accessor(krylov_bases),
+            as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+            as_cuda_type(stop_status));
         update_next_krylov_kernel<default_block_size>
             <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
                default_block_size>>>(
                 k, dim_size[0], dim_size[1],
                 as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov,
-                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                stride_next_krylov, as_cuda_accessor(krylov_bases),
                 as_cuda_type(hessenberg_iter->get_const_values()),
                 stride_hessenberg, as_cuda_type(stop_status));
     }
@@ -219,7 +234,7 @@ void finish_arnoldi(std::shared_ptr<const CudaExecutor> exec,
            default_block_size>>>(
             iter, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
-            as_cuda_type(krylov_bases->get_values()), stride_krylov,
+            as_cuda_accessor(krylov_bases),
             as_cuda_type(hessenberg_iter->get_const_values()),
             stride_hessenberg, as_cuda_type(stop_status));
     // next_krylov_basis /= hessenberg(iter, iter + 1)
@@ -229,18 +244,18 @@ void finish_arnoldi(std::shared_ptr<const CudaExecutor> exec,
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
-                           matrix::Dense<ValueType> *next_krylov_basis,
-                           matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
-                           matrix::Dense<ValueType> *hessenberg_iter,
-                           matrix::Dense<ValueType> *buffer_iter,
-                           matrix::Dense<ValueType> *arnoldi_norm,
-                           size_type iter, const stopping_status *stop_status,
-                           stopping_status *reorth_status,
-                           Array<size_type> *num_reorth)
+void finish_arnoldi_reorth(
+    std::shared_ptr<const CudaExecutor> exec,
+    matrix::Dense<ValueType> *next_krylov_basis,
+    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    matrix::Dense<ValueType> *hessenberg_iter,
+    matrix::Dense<ValueType> *buffer_iter,
+    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    const stopping_status *stop_status, stopping_status *reorth_status,
+    Array<size_type> *num_reorth)
 {
     const auto stride_next_krylov = next_krylov_basis->get_stride();
-    const auto stride_krylov = krylov_bases->get_stride();
+    const auto stride_krylov = krylov_bases.get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
     const auto stride_buffer = buffer_iter->get_stride();
     const auto stride_arnoldi = arnoldi_norm->get_stride();
@@ -250,12 +265,16 @@ void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
                          exec->get_num_multiprocessor() * 2);
     const dim3 block_size(default_dot_dim, default_dot_dim);
     size_type numReorth;
+    Accessor2d<ValueType, ValueType> next_krylov_accessor{
+        next_krylov_basis->get_values(), stride_next_krylov};
+    auto next_krylov_const_accessor = next_krylov_accessor.to_const();
     //    size_type num_reorth;
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<ValueType>());
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_accessor(next_krylov_const_accessor),
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
     // nrm = norm(next_krylov_basis
@@ -266,9 +285,9 @@ void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
         multidot_kernel<<<grid_size, block_size>>>(
             k, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_type(krylov_bases->get_const_values()),
-            stride_krylov, as_cuda_type(hessenberg_iter->get_values()),
-            stride_hessenberg, as_cuda_type(stop_status));
+            stride_next_krylov, as_cuda_accessor(krylov_bases),
+            as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+            as_cuda_type(stop_status));
         components::fill_array(exec, num_reorth->get_data(), 1,
                                zero<size_type>());
         update_next_krylov_kernel_and_check<default_block_size>
@@ -276,8 +295,7 @@ void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
                default_block_size>>>(
                 k, dim_size[0], dim_size[1],
                 as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov,
-                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                stride_next_krylov, as_cuda_accessor(krylov_bases),
                 as_cuda_type(hessenberg_iter->get_const_values()),
                 stride_hessenberg, as_cuda_type(arnoldi_norm->get_values()), 0,
                 as_cuda_type(stop_status), as_cuda_type(reorth_status),
@@ -292,8 +310,7 @@ void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
             multidot_kernel<<<grid_size, block_size>>>(
                 k, dim_size[0], dim_size[1],
                 as_cuda_type(next_krylov_basis->get_const_values()),
-                stride_next_krylov,
-                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                stride_next_krylov, as_cuda_accessor(krylov_bases),
                 as_cuda_type(buffer_iter->get_values()), stride_buffer,
                 as_cuda_type(stop_status));
             update_next_krylov_kernel_and_add<default_block_size><<<
@@ -301,8 +318,7 @@ void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
                 default_block_size>>>(
                 k, dim_size[0], dim_size[1],
                 as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov,
-                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                stride_next_krylov, as_cuda_accessor(krylov_bases),
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
                 as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
                 as_cuda_type(stop_status), as_cuda_type(reorth_status));
@@ -331,7 +347,7 @@ void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
            default_block_size>>>(
             iter, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
-            as_cuda_type(krylov_bases->get_values()), stride_krylov,
+            as_cuda_accessor(krylov_bases),
             as_cuda_type(hessenberg_iter->get_const_values()),
             stride_hessenberg, as_cuda_type(stop_status));
     // next_krylov_basis /= hessenberg(iter, iter + 1)
@@ -341,18 +357,18 @@ void finish_arnoldi_reorth(std::shared_ptr<const CudaExecutor> exec,
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
-                        matrix::Dense<ValueType> *next_krylov_basis,
-                        matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
-                        matrix::Dense<ValueType> *hessenberg_iter,
-                        matrix::Dense<ValueType> *buffer_iter,
-                        matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
-                        const stopping_status *stop_status,
-                        stopping_status *reorth_status,
-                        Array<size_type> *num_reorth)
+void finish_arnoldi_CGS(
+    std::shared_ptr<const CudaExecutor> exec,
+    matrix::Dense<ValueType> *next_krylov_basis,
+    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    matrix::Dense<ValueType> *hessenberg_iter,
+    matrix::Dense<ValueType> *buffer_iter,
+    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    const stopping_status *stop_status, stopping_status *reorth_status,
+    Array<size_type> *num_reorth)
 {
     const auto stride_next_krylov = next_krylov_basis->get_stride();
-    const auto stride_krylov = krylov_bases->get_stride();
+    const auto stride_krylov = krylov_bases.get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
     const auto stride_buffer = buffer_iter->get_stride();
     const auto stride_arnoldi = arnoldi_norm->get_stride();
@@ -362,13 +378,16 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
                          exec->get_num_multiprocessor() * 2);
     const dim3 block_size(default_dot_dim, default_dot_dim);
     size_type numReorth;
+    Accessor2d<ValueType, ValueType> next_krylov_accessor{
+        next_krylov_basis->get_values(), stride_next_krylov};
+    auto next_krylov_const_accessor = next_krylov_accessor.to_const();
 
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<ValueType>());
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_accessor(next_krylov_const_accessor),
         as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
     // nrmP = norm(next_krylov_basis
     for (size_type k = 0; k < iter + 1; ++k) {
@@ -378,9 +397,9 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
         multidot_kernel<<<grid_size, block_size>>>(
             k, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_type(krylov_bases->get_const_values()),
-            stride_krylov, as_cuda_type(hessenberg_iter->get_values()),
-            stride_hessenberg, as_cuda_type(stop_status));
+            stride_next_krylov, as_cuda_accessor(krylov_bases),
+            as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+            as_cuda_type(stop_status));
     }
     // for i in 1:iter
     //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
@@ -391,8 +410,7 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
                default_block_size>>>(
                 k, dim_size[0], dim_size[1],
                 as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov,
-                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                stride_next_krylov, as_cuda_accessor(krylov_bases),
                 as_cuda_type(hessenberg_iter->get_const_values()),
                 stride_hessenberg, as_cuda_type(stop_status));
     }
@@ -404,7 +422,7 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_accessor(next_krylov_const_accessor),
         as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
         as_cuda_type(stop_status));
     components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
@@ -426,8 +444,7 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
             multidot_kernel<<<grid_size, block_size>>>(
                 k, dim_size[0], dim_size[1],
                 as_cuda_type(next_krylov_basis->get_const_values()),
-                stride_next_krylov,
-                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                stride_next_krylov, as_cuda_accessor(krylov_bases),
                 as_cuda_type(buffer_iter->get_values()), stride_buffer,
                 as_cuda_type(stop_status));
         }
@@ -440,8 +457,7 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
                 default_block_size>>>(
                 k, dim_size[0], dim_size[1],
                 as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov,
-                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                stride_next_krylov, as_cuda_accessor(krylov_bases),
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
                 as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
                 as_cuda_type(stop_status), as_cuda_type(reorth_status));
@@ -455,9 +471,7 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
         multidot_kernel<<<grid_size, block_size>>>(
             0, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov,
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov,
+            stride_next_krylov, as_cuda_accessor(next_krylov_const_accessor),
             as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
             as_cuda_type(stop_status));
         // nrmN = norm(next_krylov_basis)
@@ -491,7 +505,7 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
            default_block_size>>>(
             iter, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
-            as_cuda_type(krylov_bases->get_values()), stride_krylov,
+            as_cuda_accessor(krylov_bases),
             as_cuda_type(hessenberg_iter->get_const_values()),
             stride_hessenberg, as_cuda_type(stop_status));
     // next_krylov_basis /= hessenberg(iter, iter + 1)
@@ -502,19 +516,19 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
-                         matrix::Dense<ValueType> *next_krylov_basis,
-                         matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
-                         matrix::Dense<ValueType> *hessenberg_iter,
-                         matrix::Dense<ValueType> *buffer_iter,
-                         matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
-                         const stopping_status *stop_status,
-                         stopping_status *reorth_status,
-                         Array<size_type> *num_reorth, int *num_reorth_steps,
-                         int *num_reorth_vectors)
+void finish_arnoldi_CGS2(
+    std::shared_ptr<const CudaExecutor> exec,
+    matrix::Dense<ValueType> *next_krylov_basis,
+    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    matrix::Dense<ValueType> *hessenberg_iter,
+    matrix::Dense<ValueType> *buffer_iter,
+    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    const stopping_status *stop_status, stopping_status *reorth_status,
+    Array<size_type> *num_reorth, int *num_reorth_steps,
+    int *num_reorth_vectors)
 {
     const auto stride_next_krylov = next_krylov_basis->get_stride();
-    const auto stride_krylov = krylov_bases->get_stride();
+    const auto stride_krylov = krylov_bases.get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
     const auto stride_buffer = buffer_iter->get_stride();
     const auto stride_arnoldi = arnoldi_norm->get_stride();
@@ -528,15 +542,18 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     const dim3 block_size(default_dot_dim, default_dot_dim);
     size_type numReorth;
 
+    Accessor2d<ValueType, ValueType> next_krylov_accessor{
+        next_krylov_basis->get_values(), stride_next_krylov};
+    auto next_krylov_const_accessor = next_krylov_accessor.to_const();
+
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<ValueType>());
-    // std::cout << dim_size[0] << " - " << dim_size[1] << std::endl;
-    // std::cout << krylov_bases->get_size()[0] << " - "
-    //           << krylov_bases->get_size()[1] << std::endl;
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_accessor(next_krylov_const_accessor),
+        // as_cuda_type(next_krylov_basis->get_const_values()),
+        // stride_next_krylov,
         as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
     // nrmP = norm(next_krylov_basis
 #ifdef TIMING
@@ -550,7 +567,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     multidot_kernel_num_iters<<<grid_size, block_size>>>(
         iter + 1, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+        as_cuda_accessor(krylov_bases.to_const()),
         as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
         as_cuda_type(stop_status));
     */
@@ -560,7 +577,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     multidot_kernel_num_iters_1<<<grid_size_num_iters, block_size>>>(
         iter + 1, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+        as_cuda_accessor(krylov_bases.to_const()),
         as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
         as_cuda_type(stop_status));
     // exec->synchronize();
@@ -585,7 +602,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
            default_block_size>>>(
             iter + 1, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
-            as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+            as_cuda_accessor(krylov_bases.to_const()),
             as_cuda_type(hessenberg_iter->get_const_values()),
             stride_hessenberg, as_cuda_type(stop_status));
 #ifdef TIMING
@@ -611,7 +628,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_accessor(next_krylov_const_accessor),
         as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
         as_cuda_type(stop_status));
 #ifdef TIMING
@@ -664,16 +681,16 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         multidot_kernel_num_iters<<<grid_size, block_size>>>(
             iter + 1, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_type(krylov_bases->get_const_values()),
-            stride_krylov, as_cuda_type(buffer_iter->get_values()),
-            stride_buffer, as_cuda_type(stop_status));
+            stride_next_krylov, as_cuda_accessor(krylov_bases.to_const()),
+            as_cuda_type(buffer_iter->get_values()), stride_buffer,
+            as_cuda_type(stop_status));
         */
         multidot_kernel_num_iters_1<<<grid_size_num_iters, block_size>>>(
             iter + 1, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_type(krylov_bases->get_const_values()),
-            stride_krylov, as_cuda_type(buffer_iter->get_values()),
-            stride_buffer, as_cuda_type(stop_status));
+            stride_next_krylov, as_cuda_accessor(krylov_bases.to_const()),
+            as_cuda_type(buffer_iter->get_values()), stride_buffer,
+            as_cuda_type(stop_status));
         // for i in 1:iter
         //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
         // end
@@ -682,8 +699,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                default_block_size>>>(
                 iter + 1, dim_size[0], dim_size[1],
                 as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov,
-                as_cuda_type(krylov_bases->get_const_values()), stride_krylov,
+                stride_next_krylov, as_cuda_accessor(krylov_bases.to_const()),
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
                 as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
                 as_cuda_type(stop_status), as_cuda_type(reorth_status));
@@ -696,9 +712,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         multidot_kernel<<<grid_size, block_size>>>(
             0, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov,
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov,
+            stride_next_krylov, as_cuda_accessor(next_krylov_const_accessor),
             as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
             as_cuda_type(stop_status));
         // nrmN = norm(next_krylov_basis)
@@ -735,7 +749,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
            default_block_size>>>(
             iter, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
-            as_cuda_type(krylov_bases->get_values()), stride_krylov,
+            as_cuda_accessor(krylov_bases),
             as_cuda_type(hessenberg_iter->get_const_values()),
             stride_hessenberg, as_cuda_type(stop_status));
 #ifdef TIMING
@@ -788,7 +802,7 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
             matrix::Dense<ValueType> *givens_cos,
             matrix::Dense<ValueType> *residual_norm,
             matrix::Dense<ValueType> *residual_norm_collection,
-            matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+            Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
             matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
             const matrix::Dense<ValueType> *b_norm,
@@ -825,7 +839,8 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
                     stop_status);
 }
 
-GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
 
 
 template <typename ValueType>
@@ -852,13 +867,13 @@ void solve_upper_triangular(
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void calculate_qy(const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
-                  const matrix::Dense<ValueType> *y,
+void calculate_qy(Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+                  size_type krylov_cols, const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const Array<size_type> *final_iter_nums)
 {
     const auto num_rows = before_preconditioner->get_size()[0];
-    const auto num_cols = krylov_bases->get_size()[1];
+    const auto num_cols = krylov_cols;  // krylov_bases->get_size()[1];
     const auto num_rhs = before_preconditioner->get_size()[1];
     const auto stride_before_preconditioner =
         before_preconditioner->get_stride();
@@ -872,10 +887,9 @@ void calculate_qy(const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
 
 
     calculate_Qy_kernel<block_size><<<grid_dim, block_dim>>>(
-        num_rows, num_cols, num_rhs,
-        as_cuda_type(krylov_bases->get_const_values()),
-        krylov_bases->get_stride(), as_cuda_type(y->get_const_values()),
-        y->get_stride(), as_cuda_type(before_preconditioner->get_values()),
+        num_rows, num_cols, num_rhs, as_cuda_accessor(krylov_bases),
+        as_cuda_type(y->get_const_values()), y->get_stride(),
+        as_cuda_type(before_preconditioner->get_values()),
         stride_before_preconditioner,
         as_cuda_type(final_iter_nums->get_const_data()));
     // Calculate qy
@@ -886,18 +900,22 @@ void calculate_qy(const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
 template <typename ValueType, typename ValueTypeKrylovBases>
 void step_2(std::shared_ptr<const CudaExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+            Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
             const Array<size_type> *final_iter_nums)
 {
+    const auto res_norm_col_size = residual_norm_collection->get_size();
+    const auto krylov_cols = res_norm_col_size[0] * res_norm_col_size[1];
     solve_upper_triangular(residual_norm_collection, hessenberg, y,
                            final_iter_nums);
-    calculate_qy(krylov_bases, y, before_preconditioner, final_iter_nums);
+    calculate_qy(krylov_bases, krylov_cols, y, before_preconditioner,
+                 final_iter_nums);
 }
 
-GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
 
 
 }  // namespace gmres_mixed
diff --git a/cuda/test/solver/gmres_mixed_kernels.cpp b/cuda/test/solver/gmres_mixed_kernels.cpp
index 7637147a49e..950d0d5ca8f 100644
--- a/cuda/test/solver/gmres_mixed_kernels.cpp
+++ b/cuda/test/solver/gmres_mixed_kernels.cpp
@@ -50,7 +50,13 @@ namespace {
 
 class GmresMixed : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Dense<>;
+    using value_type = double;
+    using krylov_type = float;
+    using index_type = int;
+    using Accessor2d = gko::Accessor2d<krylov_type, value_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+    using Mtx = Dense;
+
     GmresMixed() : rand_engine(30) {}
 
     void SetUp()
@@ -58,6 +64,8 @@ class GmresMixed : public ::testing::Test {
         ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
         ref = gko::ReferenceExecutor::create();
         cuda = gko::CudaExecutor::create(0, ref);
+        krylov_bases.set_executor(ref);
+        d_krylov_bases.set_executor(cuda);
     }
 
     void TearDown()
@@ -71,8 +79,21 @@ class GmresMixed : public ::testing::Test {
     {
         return gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
-            std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+            std::uniform_int_distribution<index_type>(num_cols, num_cols),
+            std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    gko::Array<krylov_type> generate_krylov_bases(int num_rows, int num_cols)
+    {
+        gko::Array<krylov_type> result{
+            ref, static_cast<gko::size_type>(num_rows * num_cols)};
+        auto temp_krylov_bases = gko::test::generate_random_matrix<Dense>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<index_type>(num_cols, num_cols),
+            std::normal_distribution<krylov_type>(-1.0, 1.0), rand_engine, ref);
+        std::copy_n(temp_krylov_bases->get_const_values(),
+                    result.get_num_elems(), result.get_data());
+        return result;
     }
 
     void initialize_data()
@@ -85,8 +106,12 @@ class GmresMixed : public ::testing::Test {
         b = gen_mtx(m, n);
         b_norm = gen_mtx(1, n);
         arnoldi_norm = gen_mtx(2, n);
+        gko::dim<2> krylov_bases_dim(
+            m, (gko::solver::default_krylov_dim_mixed + 1) * n);
         krylov_bases =
-            gen_mtx(m, (gko::solver::default_krylov_dim_mixed + 1) * n);
+            generate_krylov_bases(krylov_bases_dim[0], krylov_bases_dim[1]);
+        krylov_bases_accessor =
+            Accessor2d{krylov_bases.get_data(), krylov_bases_dim[1]};
         next_krylov_basis = gen_mtx(m, n);
         hessenberg = gen_mtx(gko::solver::default_krylov_dim_mixed + 1,
                              gko::solver::default_krylov_dim_mixed * n);
@@ -130,8 +155,9 @@ class GmresMixed : public ::testing::Test {
         d_b_norm->copy_from(b_norm.get());
         d_arnoldi_norm = Mtx::create(cuda);
         d_arnoldi_norm->copy_from(arnoldi_norm.get());
-        d_krylov_bases = Mtx::create(cuda);
-        d_krylov_bases->copy_from(krylov_bases.get());
+        d_krylov_bases = krylov_bases;
+        d_krylov_bases_accessor =
+            Accessor2d{d_krylov_bases.get_data(), krylov_bases_dim[1]};
         d_next_krylov_basis = Mtx::create(cuda);
         d_next_krylov_basis->copy_from(next_krylov_basis.get());
         d_hessenberg = Mtx::create(cuda);
@@ -164,6 +190,19 @@ class GmresMixed : public ::testing::Test {
         *d_num_reorth = *num_reorth;
     }
 
+    void assert_krylov_bases_near()
+    {
+        gko::Array<krylov_type> d_to_host{ref};
+        d_to_host = d_krylov_bases;
+        const auto tolerance = r<krylov_type>::value;
+        using std::abs;
+        for (gko::size_type i = 0; i < krylov_bases.get_num_elems(); ++i) {
+            const auto ref_value = krylov_bases.get_const_data()[i];
+            const auto dev_value = d_to_host.get_const_data()[i];
+            ASSERT_LE(abs(dev_value - ref_value), tolerance);
+        }
+    }
+
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::CudaExecutor> cuda;
 
@@ -175,7 +214,8 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> b_norm;
     std::unique_ptr<Mtx> arnoldi_norm;
-    std::unique_ptr<Mtx> krylov_bases;
+    gko::Array<krylov_type> krylov_bases;
+    Accessor2d krylov_bases_accessor;
     std::unique_ptr<Mtx> next_krylov_basis;
     std::unique_ptr<Mtx> hessenberg;
     std::unique_ptr<Mtx> hessenberg_iter;
@@ -196,7 +236,8 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> d_b;
     std::unique_ptr<Mtx> d_b_norm;
     std::unique_ptr<Mtx> d_arnoldi_norm;
-    std::unique_ptr<Mtx> d_krylov_bases;
+    gko::Array<krylov_type> d_krylov_bases;
+    Accessor2d d_krylov_bases_accessor;
     std::unique_ptr<Mtx> d_next_krylov_basis;
     std::unique_ptr<Mtx> d_hessenberg;
     std::unique_ptr<Mtx> d_hessenberg_iter;
@@ -212,6 +253,7 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<gko::Array<gko::size_type>> d_num_reorth;
 };
 
+
 TEST_F(GmresMixed, CudaGmresMixedInitialize1IsEquivalentToRef)
 {
     initialize_data();
@@ -232,28 +274,30 @@ TEST_F(GmresMixed, CudaGmresMixedInitialize1IsEquivalentToRef)
     GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
 }
 
+
 TEST_F(GmresMixed, CudaGmresMixedInitialize2IsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::gmres_mixed::initialize_2(
         ref, residual.get(), residual_norm.get(),
-        residual_norm_collection.get(), krylov_bases.get(),
+        residual_norm_collection.get(), krylov_bases_accessor,
         next_krylov_basis.get(), final_iter_nums.get(),
         gko::solver::default_krylov_dim_mixed);
     gko::kernels::cuda::gmres_mixed::initialize_2(
         cuda, d_residual.get(), d_residual_norm.get(),
-        d_residual_norm_collection.get(), d_krylov_bases.get(),
+        d_residual_norm_collection.get(), d_krylov_bases_accessor,
         d_next_krylov_basis.get(), d_final_iter_nums.get(),
         gko::solver::default_krylov_dim_mixed);
 
     GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
                         1e-14);
-    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, 1e-14);
+    assert_krylov_bases_near();
     GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
 }
 
+
 TEST_F(GmresMixed, CudaGmresMixedStep1IsEquivalentToRef)
 {
     initialize_data();
@@ -263,15 +307,15 @@ TEST_F(GmresMixed, CudaGmresMixedStep1IsEquivalentToRef)
 
     gko::kernels::reference::gmres_mixed::step_1(
         ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
-        residual_norm.get(), residual_norm_collection.get(), krylov_bases.get(),
-        hessenberg_iter.get(), buffer_iter.get(), b_norm.get(),
-        arnoldi_norm.get(), iter, final_iter_nums.get(), stop_status.get(),
-        reorth_status.get(), num_reorth.get(), &num_reorth_steps,
-        &num_reorth_vectors);
+        residual_norm.get(), residual_norm_collection.get(),
+        krylov_bases_accessor, hessenberg_iter.get(), buffer_iter.get(),
+        b_norm.get(), arnoldi_norm.get(), iter, final_iter_nums.get(),
+        stop_status.get(), reorth_status.get(), num_reorth.get(),
+        &num_reorth_steps, &num_reorth_vectors);
     gko::kernels::cuda::gmres_mixed::step_1(
         cuda, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
         d_residual_norm.get(), d_residual_norm_collection.get(),
-        d_krylov_bases.get(), d_hessenberg_iter.get(), d_buffer_iter.get(),
+        d_krylov_bases_accessor, d_hessenberg_iter.get(), d_buffer_iter.get(),
         d_b_norm.get(), d_arnoldi_norm.get(), iter, d_final_iter_nums.get(),
         d_stop_status.get(), d_reorth_status.get(), d_num_reorth.get(),
         &num_reorth_steps, &num_reorth_vectors);
@@ -283,25 +327,27 @@ TEST_F(GmresMixed, CudaGmresMixedStep1IsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
                         1e-14);
     GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, 1e-14);
+    assert_krylov_bases_near();
     GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
 }
 
+
 TEST_F(GmresMixed, CudaGmresMixedStep2IsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::gmres_mixed::step_2(
-        ref, residual_norm_collection.get(), krylov_bases.get(),
+        ref, residual_norm_collection.get(), krylov_bases_accessor.to_const(),
         hessenberg.get(), y.get(), before_preconditioner.get(),
         final_iter_nums.get());
     gko::kernels::cuda::gmres_mixed::step_2(
-        cuda, d_residual_norm_collection.get(), d_krylov_bases.get(),
-        d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
-        d_final_iter_nums.get());
+        cuda, d_residual_norm_collection.get(),
+        d_krylov_bases_accessor.to_const(), d_hessenberg.get(), d_y.get(),
+        d_before_preconditioner.get(), d_final_iter_nums.get());
 
     GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
 }
 
+
 }  // namespace
diff --git a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp b/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
index 857347741a2..23c9d894829 100644
--- a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
+++ b/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
@@ -131,8 +131,6 @@ int main(int argc, char *argv[])
         A = share(gko::read<mtx>(std::ifstream(argv[index_matrix]), exec));
         auto sizesA = A->get_size();
 
-        double aux[sizesA[0]];
-
         if (std::string(argv[1]) == "cuda") {
             x = gko::read<vec>(std::ifstream("data/one.mtx"), exec);
         } else {
diff --git a/hip/solver/gmres_mixed_kernels.hip.cpp b/hip/solver/gmres_mixed_kernels.hip.cpp
new file mode 100644
index 00000000000..311c03fa889
--- /dev/null
+++ b/hip/solver/gmres_mixed_kernels.hip.cpp
@@ -0,0 +1,157 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/gmres_mixed_kernels.hpp"
+
+
+#include <algorithm>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/range_accessors.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "hip/base/config.hip.hpp"
+#include "hip/base/math.hip.hpp"
+#include "hip/base/types.hip.hpp"
+#include "hip/components/atomic.hip.hpp"
+#include "hip/components/cooperative_groups.hip.hpp"
+#include "hip/components/reduction.hip.hpp"
+#include "hip/components/thread_ids.hip.hpp"
+#include "hip/components/uninitialized_array.hip.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace hip {
+/**
+ * @brief The GMRES_MIXED solver namespace.
+ *
+ * @ingroup gmres_mixed
+ */
+namespace gmres_mixed {
+
+
+constexpr int default_block_size = 512;
+// default_dot_dim can not be 64 in hip because 64 * 64 exceeds their max block
+// size limit.
+constexpr int default_dot_dim = 32;
+constexpr int default_dot_size = default_dot_dim * default_dot_dim;
+
+
+#include "common/solver/gmres_mixed_kernels.hpp.inc"
+
+
+// Specialization, so the Accessor can use the same function as regular pointers
+template <typename Type1, typename Type2>
+Accessor2d<hip_type<Type1>, hip_type<Type2>> as_hip_accessor(
+    Accessor2d<Type1, Type2> acc)
+{
+    return {as_hip_type(acc.get_storage()), acc.get_stride()};
+}
+
+template <typename Type1, typename Type2>
+Accessor2dConst<hip_type<Type1>, hip_type<Type2>> as_hip_accessor(
+    const Accessor2dConst<Type1, Type2> &acc)
+{
+    return {as_hip_type(acc.get_storage()), acc.get_stride()};
+}
+
+
+template <typename ValueType>
+void initialize_1(std::shared_ptr<const HipExecutor> exec,
+                  const matrix::Dense<ValueType> *b,
+                  matrix::Dense<ValueType> *b_norm,
+                  matrix::Dense<ValueType> *residual,
+                  matrix::Dense<ValueType> *givens_sin,
+                  matrix::Dense<ValueType> *givens_cos,
+                  Array<stopping_status> *stop_status,
+                  size_type krylov_dim) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void initialize_2(std::shared_ptr<const HipExecutor> exec,
+                  const matrix::Dense<ValueType> *residual,
+                  matrix::Dense<ValueType> *residual_norm,
+                  matrix::Dense<ValueType> *residual_norm_collection,
+                  Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+                  matrix::Dense<ValueType> *next_krylov_basis,
+                  Array<size_type> *final_iter_nums,
+                  size_type krylov_dim) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void step_1(std::shared_ptr<const HipExecutor> exec,
+            matrix::Dense<ValueType> *next_krylov_basis,
+            matrix::Dense<ValueType> *givens_sin,
+            matrix::Dense<ValueType> *givens_cos,
+            matrix::Dense<ValueType> *residual_norm,
+            matrix::Dense<ValueType> *residual_norm_collection,
+            Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+            matrix::Dense<ValueType> *hessenberg_iter,
+            matrix::Dense<ValueType> *buffer_iter,
+            const matrix::Dense<ValueType> *b_norm,
+            matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+            Array<size_type> *final_iter_nums,
+            const Array<stopping_status> *stop_status,
+            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
+            int *num_reorth_steps, int *num_reorth_vectors) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+void step_2(std::shared_ptr<const HipExecutor> exec,
+            const matrix::Dense<ValueType> *residual_norm_collection,
+            Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+            const matrix::Dense<ValueType> *hessenberg,
+            matrix::Dense<ValueType> *y,
+            matrix::Dense<ValueType> *before_preconditioner,
+            const Array<size_type> *final_iter_nums) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
+
+
+}  // namespace gmres_mixed
+}  // namespace hip
+}  // namespace kernels
+}  // namespace gko
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 9f51ea3d2e6..e1d1d86cfe5 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -443,50 +443,6 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
     template _macro(std::complex<double>)
 
 
-/**
- * Instantiates a template for each value type compiled by Ginkgo.
- *
- * @param _macro  A macro which expands the template instantiation
- *                (not including the leading `template` specifier).
- *                Should take one argument, which is replaced by the
- *                value type.
- */
-// #if defined(__CUDACC__) || defined(__HIPCC__)
-// #if defined(__CUDA_ARCH__)
-// #define GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(_macro) \
-//     template _macro(double, half);                  \
-//     template _macro(double, float);                 \
-//     template _macro(double, double)
-// #else
-#define GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(_macro) \
-    template _macro(double, float);                 \
-    template _macro(double, double)
-// #endif
-
-
-/**
- * Instantiates a template for each value type compiled by Ginkgo.
- *
- * @param _macro  A macro which expands the template instantiation
- *                (not including the leading `template` specifier).
- *                Should take one argument, which is replaced by the
- *                value type.
- */
-/*
-#define GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(_macro) \
-    template _macro(double, float, bool, bool);                 \
-    template _macro(double, double, bool, bool)
-//    template _macro(double, float, (bool) true, (bool) true); \
-//    template _macro(double, float, (bool) true, (bool) false); \
-//    template _macro(double, float, (bool) false, (bool) true); \
-//    template _macro(double, float, (bool) false, (bool) false); \
-//    template _macro(double, double, (bool) true, (bool) true); \
-//    template _macro(double, double, (bool) true, (bool) false); \
-//    template _macro(double, double, (bool) false, (bool) true); \
-//    template _macro(double, double, (bool) false, (bool) false)
-*/
-
-
 /**
  * Instantiates a template for each index type compiled by Ginkgo.
  *
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index b278d8a9085..25ded584943 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -52,7 +52,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 // #define TIMING 1
 
 
-#ifdef TIMING 1
+#ifdef TIMING
 using double_seconds = std::chrono::duration<double, std::milli>;
 #endif
 
@@ -73,7 +73,7 @@ namespace {
 
 template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
-                    matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
                     matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
                     const stopping_status *stop_status)
 {
@@ -90,8 +90,8 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter_entry +=
                     next_krylov_basis->at(j, i) *
-                    krylov_bases->at(j,
-                                     next_krylov_basis->get_size()[1] * k + i);
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
             }
             hessenberg_iter->at(k, i) = hessenberg_iter_entry;
 
@@ -99,8 +99,8 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
                     hessenberg_iter->at(k, i) *
-                    krylov_bases->at(j,
-                                     next_krylov_basis->get_size()[1] * k + i);
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
             }
         }
         // for i in 1:iter
@@ -120,8 +120,9 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
 #pragma omp parallel for
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases->at(j, next_krylov_basis->get_size()[1] * (iter + 1) +
-                                    i) = next_krylov_basis->at(j, i);
+            krylov_bases.write(
+                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
+                next_krylov_basis->at(j, i));
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -131,11 +132,12 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
-                           matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
-                           matrix::Dense<ValueType> *hessenberg_iter,
-                           matrix::Dense<ValueType> *arnoldi_norm,
-                           size_type iter, const stopping_status *stop_status)
+void finish_arnoldi_reorth(
+    matrix::Dense<ValueType> *next_krylov_basis,
+    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    matrix::Dense<ValueType> *hessenberg_iter,
+    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    const stopping_status *stop_status)
 {
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
 
@@ -157,8 +159,8 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter_entry +=
                     next_krylov_basis->at(j, i) *
-                    krylov_bases->at(j,
-                                     next_krylov_basis->get_size()[1] * k + i);
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
             }
             hessenberg_iter->at(k, i) = hessenberg_iter_entry;
 
@@ -166,8 +168,8 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
                     hessenberg_iter->at(k, i) *
-                    krylov_bases->at(j,
-                                     next_krylov_basis->get_size()[1] * k + i);
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
             }
             if (hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i) >
                 arnoldi_norm->at(0, i)) {
@@ -176,7 +178,7 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     reorth += next_krylov_basis->at(j, i) *
-                              krylov_bases->at(
+                              krylov_bases.read(
                                   j, next_krylov_basis->get_size()[1] * k + i);
                 }
                 hessenberg_iter->at(k, i) += reorth;
@@ -186,7 +188,7 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
                      ++j) {
                     next_krylov_basis->at(j, i) -=
                         reorth *
-                        krylov_bases->at(
+                        krylov_bases.read(
                             j, next_krylov_basis->get_size()[1] * k + i);
                 }
             }
@@ -212,8 +214,9 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
 #pragma omp parallel for
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases->at(j, next_krylov_basis->get_size()[1] * (iter + 1) +
-                                    i) = next_krylov_basis->at(j, i);
+            krylov_bases.write(
+                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
+                next_krylov_basis->at(j, i));
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -223,12 +226,13 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
-                        matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
-                        matrix::Dense<ValueType> *hessenberg_iter,
-                        matrix::Dense<ValueType> *buffer_iter,
-                        matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
-                        const stopping_status *stop_status)
+void finish_arnoldi_CGS(
+    matrix::Dense<ValueType> *next_krylov_basis,
+    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    matrix::Dense<ValueType> *hessenberg_iter,
+    matrix::Dense<ValueType> *buffer_iter,
+    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    const stopping_status *stop_status)
 {
     const ValueType eta = 1.0 / sqrt(2.0);
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
@@ -251,8 +255,8 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter_entry +=
                     next_krylov_basis->at(j, i) *
-                    krylov_bases->at(j,
-                                     next_krylov_basis->get_size()[1] * k + i);
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
             }
             hessenberg_iter->at(k, i) = hessenberg_iter_entry;
         }
@@ -264,8 +268,8 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
                     hessenberg_iter->at(k, i) *
-                    krylov_bases->at(j,
-                                     next_krylov_basis->get_size()[1] * k + i);
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
             }
         }
         // for i in 1:iter
@@ -290,7 +294,7 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
                      ++j) {
                     hessenberg_iter_entry +=
                         next_krylov_basis->at(j, i) *
-                        krylov_bases->at(
+                        krylov_bases.read(
                             j, next_krylov_basis->get_size()[1] * k + i);
                 }
                 buffer_iter->at(k, i) = hessenberg_iter_entry;
@@ -304,7 +308,7 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
                      ++j) {
                     next_krylov_basis->at(j, i) -=
                         buffer_iter->at(k, i) *
-                        krylov_bases->at(
+                        krylov_bases.read(
                             j, next_krylov_basis->get_size()[1] * k + i);
                 }
             }
@@ -333,8 +337,9 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
 #pragma omp parallel for
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases->at(j, next_krylov_basis->get_size()[1] * (iter + 1) +
-                                    i) = next_krylov_basis->at(j, i);
+            krylov_bases.write(
+                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
+                next_krylov_basis->at(j, i));
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -344,12 +349,13 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
-                         matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
-                         matrix::Dense<ValueType> *hessenberg_iter,
-                         matrix::Dense<ValueType> *buffer_iter,
-                         matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
-                         const stopping_status *stop_status)
+void finish_arnoldi_CGS2(
+    matrix::Dense<ValueType> *next_krylov_basis,
+    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    matrix::Dense<ValueType> *hessenberg_iter,
+    matrix::Dense<ValueType> *buffer_iter,
+    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    const stopping_status *stop_status)
 {
     const ValueType eta = 1.0 / sqrt(2.0);
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
@@ -366,7 +372,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         }
         arnoldi_norm->at(0, i) = nrm * eta;
         // nrmP = norm(next_krylov_basis)
-#ifdef TIMING 1
+#ifdef TIMING
         auto start_1 = std::chrono::steady_clock::now();
 #endif
 #pragma omp parallel for
@@ -375,12 +381,12 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter_entry +=
                     next_krylov_basis->at(j, i) *
-                    krylov_bases->at(j,
-                                     next_krylov_basis->get_size()[1] * k + i);
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
             }
             hessenberg_iter->at(k, i) = hessenberg_iter_entry;
         }
-#ifdef TIMING 1
+#ifdef TIMING
         auto time_1 = std::chrono::steady_clock::now() - start_1;
         std::cout << "time_1(" << iter << ") = "
                   << std::chrono::duration_cast<double_seconds>(time_1).count()
@@ -389,7 +395,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         // for i in 1:iter
         //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
         // end
-#ifdef TIMING 1
+#ifdef TIMING
         auto start_2 = std::chrono::steady_clock::now();
 #endif
         for (size_type k = 0; k < iter + 1; ++k) {
@@ -397,11 +403,11 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
                     hessenberg_iter->at(k, i) *
-                    krylov_bases->at(j,
-                                     next_krylov_basis->get_size()[1] * k + i);
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
             }
         }
-#ifdef TIMING 1
+#ifdef TIMING
         auto time_2 = std::chrono::steady_clock::now() - start_2;
         std::cout << "time_2(" << iter << ") = "
                   << std::chrono::duration_cast<double_seconds>(time_2).count()
@@ -434,7 +440,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                      ++j) {
                     hessenberg_iter_entry +=
                         next_krylov_basis->at(j, i) *
-                        krylov_bases->at(
+                        krylov_bases.read(
                             j, next_krylov_basis->get_size()[1] * k + i);
                 }
                 buffer_iter->at(k, i) = hessenberg_iter_entry;
@@ -448,7 +454,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                      ++j) {
                     next_krylov_basis->at(j, i) -=
                         buffer_iter->at(k, i) *
-                        krylov_bases->at(
+                        krylov_bases.read(
                             j, next_krylov_basis->get_size()[1] * k + i);
                 }
             }
@@ -477,8 +483,9 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
 #pragma omp parallel for
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases->at(j, next_krylov_basis->get_size()[1] * (iter + 1) +
-                                    i) = next_krylov_basis->at(j, i);
+            krylov_bases.write(
+                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
+                next_krylov_basis->at(j, i));
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -597,7 +604,7 @@ void solve_upper_triangular(
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void calculate_qy(const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+void calculate_qy(Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
                   const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const size_type *final_iter_nums)
@@ -608,7 +615,7 @@ void calculate_qy(const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
             before_preconditioner->at(i, k) = zero<ValueType>();
             for (size_type j = 0; j < final_iter_nums[k]; ++j) {
                 before_preconditioner->at(i, k) +=
-                    krylov_bases->at(
+                    krylov_bases.read(
                         i, j * before_preconditioner->get_size()[1] + k) *
                     y->at(j, k);
             }
@@ -664,7 +671,7 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
                   const matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
-                  matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                  Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
 {
@@ -692,22 +699,25 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             auto value = residual->at(i, j) / residual_norm->at(0, j);
-            krylov_bases->at(i, j) = value;
+            krylov_bases.write(i, j, value);
             next_krylov_basis->at(i, j) = value;
         }
         final_iter_nums->get_data()[j] = 0;
     }
 
 #pragma omp parallel for
-    for (size_type i = 0; i < krylov_bases->get_size()[0]; ++i) {
+    for (size_type i = 0;
+         i < residual->get_size()[0] /*krylov_bases->get_size()[0]*/; ++i) {
         for (size_type j = residual->get_size()[1];
-             j < krylov_bases->get_size()[1]; ++j) {
-            krylov_bases->at(i, j) = zero<ValueType>();
+             j < (krylov_dim + 1) *
+                     residual->get_size()[1] /*krylov_bases->get_size()[1]*/;
+             ++j) {
+            krylov_bases.write(i, j, zero<ValueType>());
         }
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
 
 
@@ -718,7 +728,7 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
             matrix::Dense<ValueType> *givens_cos,
             matrix::Dense<ValueType> *residual_norm,
             matrix::Dense<ValueType> *residual_norm_collection,
-            matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+            Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
             matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
             const matrix::Dense<ValueType> *b_norm,
@@ -752,13 +762,14 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
                                  stop_status->get_const_data());
 }
 
-GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
 void step_2(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+            Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
@@ -770,7 +781,8 @@ void step_2(std::shared_ptr<const OmpExecutor> exec,
                  final_iter_nums->get_const_data());
 }
 
-GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
 
 
 }  // namespace gmres_mixed
diff --git a/omp/test/solver/gmres_mixed_kernels.cpp b/omp/test/solver/gmres_mixed_kernels.cpp
index 432e13c0c90..6f7913b71aa 100644
--- a/omp/test/solver/gmres_mixed_kernels.cpp
+++ b/omp/test/solver/gmres_mixed_kernels.cpp
@@ -32,10 +32,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/solver/gmres_mixed_kernels.hpp"
 
+
+#include <algorithm>
+#include <cmath>
 #include <random>
 
+
 #include <gtest/gtest.h>
 
+
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
@@ -44,19 +49,32 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm_reduction.hpp>
 
+
+#include "core/components/accessor.hpp"
 #include "core/test/utils.hpp"
 
+
 namespace {
 
+
 class GmresMixed : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Dense<>;
+    using value_type = double;
+    using krylov_type = float;
+    using index_type = int;
+    using Accessor2d = gko::Accessor2d<krylov_type, value_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+    using Mtx = Dense;
+
+
     GmresMixed() : rand_engine(30) {}
 
     void SetUp()
     {
         ref = gko::ReferenceExecutor::create();
         omp = gko::OmpExecutor::create();
+        krylov_bases.set_executor(ref);
+        d_krylov_bases.set_executor(omp);
     }
 
     void TearDown()
@@ -70,8 +88,21 @@ class GmresMixed : public ::testing::Test {
     {
         return gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
-            std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+            std::uniform_int_distribution<index_type>(num_cols, num_cols),
+            std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    gko::Array<krylov_type> generate_krylov_bases(int num_rows, int num_cols)
+    {
+        gko::Array<krylov_type> result{
+            ref, static_cast<gko::size_type>(num_rows * num_cols)};
+        auto temp_krylov_bases = gko::test::generate_random_matrix<Dense>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<index_type>(num_cols, num_cols),
+            std::normal_distribution<krylov_type>(-1.0, 1.0), rand_engine, ref);
+        std::copy_n(temp_krylov_bases->get_const_values(),
+                    result.get_num_elems(), result.get_data());
+        return result;
     }
 
     void initialize_data()
@@ -84,8 +115,13 @@ class GmresMixed : public ::testing::Test {
         b = gen_mtx(m, n);
         b_norm = gen_mtx(1, n);
         arnoldi_norm = gen_mtx(2, n);
+        gko::dim<2> krylov_bases_dim(
+            m, (gko::solver::default_krylov_dim_mixed + 1) * n);
         krylov_bases =
-            gen_mtx(m, (gko::solver::default_krylov_dim_mixed + 1) * n);
+            generate_krylov_bases(krylov_bases_dim[0], krylov_bases_dim[1]);
+        krylov_bases_accessor =
+            Accessor2d{krylov_bases.get_data(), krylov_bases_dim[1]};
+
         next_krylov_basis = gen_mtx(m, n);
         hessenberg = gen_mtx(gko::solver::default_krylov_dim_mixed + 1,
                              gko::solver::default_krylov_dim_mixed * n);
@@ -129,8 +165,9 @@ class GmresMixed : public ::testing::Test {
         d_b_norm->copy_from(b_norm.get());
         d_arnoldi_norm = Mtx::create(omp);
         d_arnoldi_norm->copy_from(arnoldi_norm.get());
-        d_krylov_bases = Mtx::create(omp);
-        d_krylov_bases->copy_from(krylov_bases.get());
+        d_krylov_bases = krylov_bases;
+        d_krylov_bases_accessor =
+            Accessor2d{d_krylov_bases.get_data(), krylov_bases_dim[1]};
         d_next_krylov_basis = Mtx::create(omp);
         d_next_krylov_basis->copy_from(next_krylov_basis.get());
         d_hessenberg = Mtx::create(omp);
@@ -162,6 +199,19 @@ class GmresMixed : public ::testing::Test {
         *d_num_reorth = *num_reorth;
     }
 
+    void assert_krylov_bases_near()
+    {
+        gko::Array<krylov_type> d_to_host{ref};
+        d_to_host = d_krylov_bases;
+        const auto tolerance = r<krylov_type>::value;
+        using std::abs;
+        for (gko::size_type i = 0; i < krylov_bases.get_num_elems(); ++i) {
+            const auto ref_value = krylov_bases.get_const_data()[i];
+            const auto dev_value = d_to_host.get_const_data()[i];
+            ASSERT_LE(abs(dev_value - ref_value), tolerance);
+        }
+    }
+
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::OmpExecutor> omp;
 
@@ -173,7 +223,8 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> b_norm;
     std::unique_ptr<Mtx> arnoldi_norm;
-    std::unique_ptr<Mtx> krylov_bases;
+    gko::Array<krylov_type> krylov_bases;
+    Accessor2d krylov_bases_accessor;
     std::unique_ptr<Mtx> next_krylov_basis;
     std::unique_ptr<Mtx> hessenberg;
     std::unique_ptr<Mtx> hessenberg_iter;
@@ -194,7 +245,8 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> d_b;
     std::unique_ptr<Mtx> d_b_norm;
     std::unique_ptr<Mtx> d_arnoldi_norm;
-    std::unique_ptr<Mtx> d_krylov_bases;
+    gko::Array<krylov_type> d_krylov_bases;
+    Accessor2d d_krylov_bases_accessor;
     std::unique_ptr<Mtx> d_next_krylov_basis;
     std::unique_ptr<Mtx> d_hessenberg;
     std::unique_ptr<Mtx> d_hessenberg_iter;
@@ -210,6 +262,7 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<gko::Array<gko::size_type>> d_num_reorth;
 };
 
+
 TEST_F(GmresMixed, OmpGmresMixedInitialize1IsEquivalentToRef)
 {
     initialize_data();
@@ -236,19 +289,19 @@ TEST_F(GmresMixed, OmpGmresMixedInitialize2IsEquivalentToRef)
 
     gko::kernels::reference::gmres_mixed::initialize_2(
         ref, residual.get(), residual_norm.get(),
-        residual_norm_collection.get(), krylov_bases.get(),
+        residual_norm_collection.get(), krylov_bases_accessor,
         next_krylov_basis.get(), final_iter_nums.get(),
         gko::solver::default_krylov_dim_mixed);
     gko::kernels::omp::gmres_mixed::initialize_2(
         omp, d_residual.get(), d_residual_norm.get(),
-        d_residual_norm_collection.get(), d_krylov_bases.get(),
+        d_residual_norm_collection.get(), d_krylov_bases_accessor,
         d_next_krylov_basis.get(), d_final_iter_nums.get(),
         gko::solver::default_krylov_dim_mixed);
 
     GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
                         1e-14);
-    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, 1e-14);
+    assert_krylov_bases_near();
     GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
 }
 
@@ -261,15 +314,15 @@ TEST_F(GmresMixed, OmpGmresMixedStep1IsEquivalentToRef)
 
     gko::kernels::reference::gmres_mixed::step_1(
         ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
-        residual_norm.get(), residual_norm_collection.get(), krylov_bases.get(),
-        hessenberg_iter.get(), buffer_iter.get(), b_norm.get(),
-        arnoldi_norm.get(), iter, final_iter_nums.get(), stop_status.get(),
-        reorth_status.get(), num_reorth.get(), &num_reorth_steps,
-        &num_reorth_vectors);
+        residual_norm.get(), residual_norm_collection.get(),
+        krylov_bases_accessor, hessenberg_iter.get(), buffer_iter.get(),
+        b_norm.get(), arnoldi_norm.get(), iter, final_iter_nums.get(),
+        stop_status.get(), reorth_status.get(), num_reorth.get(),
+        &num_reorth_steps, &num_reorth_vectors);
     gko::kernels::omp::gmres_mixed::step_1(
         omp, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
         d_residual_norm.get(), d_residual_norm_collection.get(),
-        d_krylov_bases.get(), d_hessenberg_iter.get(), d_buffer_iter.get(),
+        d_krylov_bases_accessor, d_hessenberg_iter.get(), d_buffer_iter.get(),
         d_b_norm.get(), d_arnoldi_norm.get(), iter, d_final_iter_nums.get(),
         d_stop_status.get(), d_reorth_status.get(), d_num_reorth.get(),
         &num_reorth_steps, &num_reorth_vectors);
@@ -281,7 +334,7 @@ TEST_F(GmresMixed, OmpGmresMixedStep1IsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
                         1e-14);
     GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, 1e-14);
+    assert_krylov_bases_near();
     GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
 }
 
@@ -290,16 +343,17 @@ TEST_F(GmresMixed, OmpGmresMixedStep2IsEquivalentToRef)
     initialize_data();
 
     gko::kernels::reference::gmres_mixed::step_2(
-        ref, residual_norm_collection.get(), krylov_bases.get(),
+        ref, residual_norm_collection.get(), krylov_bases_accessor.to_const(),
         hessenberg.get(), y.get(), before_preconditioner.get(),
         final_iter_nums.get());
     gko::kernels::omp::gmres_mixed::step_2(
-        omp, d_residual_norm_collection.get(), d_krylov_bases.get(),
-        d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
-        d_final_iter_nums.get());
+        omp, d_residual_norm_collection.get(),
+        d_krylov_bases_accessor.to_const(), d_hessenberg.get(), d_y.get(),
+        d_before_preconditioner.get(), d_final_iter_nums.get());
 
     GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
 }
 
+
 }  // namespace
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index d6e6bb12b31..af3dfc820b4 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -41,6 +41,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <iostream>
 
+#include "core/components/accessor.hpp"
+
 
 namespace gko {
 namespace kernels {
@@ -58,7 +60,7 @@ namespace {
 
 template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
-                    matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
                     matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
                     const stopping_status *stop_status)
 {
@@ -71,14 +73,14 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
                     next_krylov_basis->at(j, i) *
-                    krylov_bases->at(j,
-                                     next_krylov_basis->get_size()[1] * k + i);
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
             }
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
                     hessenberg_iter->at(k, i) *
-                    krylov_bases->at(j,
-                                     next_krylov_basis->get_size()[1] * k + i);
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
             }
         }
         // for i in 1:iter
@@ -96,8 +98,9 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases->at(j, next_krylov_basis->get_size()[1] * (iter + 1) +
-                                    i) = next_krylov_basis->at(j, i);
+            krylov_bases.write(
+                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
+                next_krylov_basis->at(j, i));
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -107,11 +110,12 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
-                           matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
-                           matrix::Dense<ValueType> *hessenberg_iter,
-                           matrix::Dense<ValueType> *arnoldi_norm,
-                           size_type iter, const stopping_status *stop_status)
+void finish_arnoldi_reorth(
+    matrix::Dense<ValueType> *next_krylov_basis,
+    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    matrix::Dense<ValueType> *hessenberg_iter,
+    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    const stopping_status *stop_status)
 {
     //    ValueType nrm = 0, reorth = 0;
 
@@ -134,14 +138,14 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
                     next_krylov_basis->at(j, i) *
-                    krylov_bases->at(j,
-                                     next_krylov_basis->get_size()[1] * k + i);
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
             }
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
                     hessenberg_iter->at(k, i) *
-                    krylov_bases->at(j,
-                                     next_krylov_basis->get_size()[1] * k + i);
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
             }
             if ((hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i)) >
                 arnoldi_norm->at(0, i)) {
@@ -150,14 +154,14 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
                      ++j) {
                     arnoldi_norm->at(1, i) +=
                         next_krylov_basis->at(j, i) *
-                        krylov_bases->at(
+                        krylov_bases.read(
                             j, next_krylov_basis->get_size()[1] * k + i);
                 }
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
                         arnoldi_norm->at(1, i) *
-                        krylov_bases->at(
+                        krylov_bases.read(
                             j, next_krylov_basis->get_size()[1] * k + i);
                 }
                 hessenberg_iter->at(k, i) += arnoldi_norm->at(1, i);
@@ -183,8 +187,9 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases->at(j, next_krylov_basis->get_size()[1] * (iter + 1) +
-                                    i) = next_krylov_basis->at(j, i);
+            krylov_bases.write(
+                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
+                next_krylov_basis->at(j, i));
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -194,12 +199,13 @@ void finish_arnoldi_reorth(matrix::Dense<ValueType> *next_krylov_basis,
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
-                        matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
-                        matrix::Dense<ValueType> *hessenberg_iter,
-                        matrix::Dense<ValueType> *buffer_iter,
-                        matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
-                        const stopping_status *stop_status)
+void finish_arnoldi_CGS(
+    matrix::Dense<ValueType> *next_krylov_basis,
+    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    matrix::Dense<ValueType> *hessenberg_iter,
+    matrix::Dense<ValueType> *buffer_iter,
+    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    const stopping_status *stop_status)
 {
     const ValueType eta = 1.0 / sqrt(2.0);
     //    ValueType nrmP = 0, nrmN = 0;
@@ -224,8 +230,8 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
                     next_krylov_basis->at(j, i) *
-                    krylov_bases->at(j,
-                                     next_krylov_basis->get_size()[1] * k + i);
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
             }
         }
         // for i in 1:iter
@@ -235,8 +241,8 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
                     hessenberg_iter->at(k, i) *
-                    krylov_bases->at(j,
-                                     next_krylov_basis->get_size()[1] * k + i);
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
             }
         }
         // for i in 1:iter
@@ -258,7 +264,7 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
                      ++j) {
                     buffer_iter->at(k, i) +=
                         next_krylov_basis->at(j, i) *
-                        krylov_bases->at(
+                        krylov_bases.read(
                             j, next_krylov_basis->get_size()[1] * k + i);
                 }
             }
@@ -270,7 +276,7 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
                      ++j) {
                     next_krylov_basis->at(j, i) -=
                         buffer_iter->at(k, i) *
-                        krylov_bases->at(
+                        krylov_bases.read(
                             j, next_krylov_basis->get_size()[1] * k + i);
                 }
                 hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
@@ -298,8 +304,9 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases->at(j, next_krylov_basis->get_size()[1] * (iter + 1) +
-                                    i) = next_krylov_basis->at(j, i);
+            krylov_bases.write(
+                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
+                next_krylov_basis->at(j, i));
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -415,7 +422,7 @@ void solve_upper_triangular(
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void calculate_qy(const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+void calculate_qy(Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
                   const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const size_type *final_iter_nums)
@@ -425,7 +432,7 @@ void calculate_qy(const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
             before_preconditioner->at(i, k) = zero<ValueType>();
             for (size_type j = 0; j < final_iter_nums[k]; ++j) {
                 before_preconditioner->at(i, k) +=
-                    krylov_bases->at(
+                    krylov_bases.read(
                         i, j * before_preconditioner->get_size()[1] + k) *
                     y->at(j, k);
             }
@@ -474,7 +481,7 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
                   const matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
-                  matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+                  Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
 {
@@ -494,23 +501,26 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
             }
         }
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
-            krylov_bases->at(i, j) =
-                residual->at(i, j) / residual_norm->at(0, j);
+            krylov_bases.write(i, j,
+                               residual->at(i, j) / residual_norm->at(0, j));
             next_krylov_basis->at(i, j) =
                 residual->at(i, j) / residual_norm->at(0, j);
         }
         final_iter_nums->get_data()[j] = 0;
     }
 
-    for (size_type j = residual->get_size()[1]; j < krylov_bases->get_size()[1];
+    for (size_type j = residual->get_size()[1];
+         j < (krylov_dim + 1) *
+                 residual->get_size()[1] /*krylov_bases->get_size()[1]*/;
          ++j) {
-        for (size_type i = 0; i < krylov_bases->get_size()[0]; ++i) {
-            krylov_bases->at(i, j) = zero<ValueType>();
+        for (size_type i = 0;
+             i < residual->get_size()[0] /*krylov_bases->get_size()[0]*/; ++i) {
+            krylov_bases.write(i, j, zero<ValueType>());
         }
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
 
 
@@ -523,7 +533,7 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
             matrix::Dense<ValueType> *givens_cos,
             matrix::Dense<ValueType> *residual_norm,
             matrix::Dense<ValueType> *residual_norm_collection,
-            matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+            Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
             matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
             const matrix::Dense<ValueType> *b_norm,
@@ -560,13 +570,14 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
 }
 
 // GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(GKO_DECLARE_GMRES_MIXED_BOOL_STEP_1_KERNEL);
-GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
 void step_2(std::shared_ptr<const ReferenceExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            const matrix::Dense<ValueTypeKrylovBases> *krylov_bases,
+            Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
@@ -578,7 +589,8 @@ void step_2(std::shared_ptr<const ReferenceExecutor> exec,
                  final_iter_nums->get_const_data());
 }
 
-GKO_INSTANTIATE_FOR_EACH_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+    GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
 
 
 }  // namespace gmres_mixed
diff --git a/reference/test/solver/gmres_mixed_kernels.cpp b/reference/test/solver/gmres_mixed_kernels.cpp
index 4ea8b6fd0e2..3e399dec481 100644
--- a/reference/test/solver/gmres_mixed_kernels.cpp
+++ b/reference/test/solver/gmres_mixed_kernels.cpp
@@ -32,8 +32,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/solver/gmres_mixed.hpp>
 
+
 #include <gtest/gtest.h>
 
+
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
@@ -43,25 +45,35 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/residual_norm_reduction.hpp>
 #include <ginkgo/core/stop/time.hpp>
 
-#include "core/test/utils/assertions.hpp"
+
+#include "core/test/utils.hpp"
+
 
 namespace {
 
+
+template <typename ValueKrylovType>
 class GmresMixed : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Dense<>;
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueKrylovType())>::type;
+    using krylov_type =
+        typename std::tuple_element<1, decltype(ValueKrylovType())>::type;
+    using Mtx = gko::matrix::Dense<value_type>;
+    using gmres_type = gko::solver::GmresMixed<value_type, krylov_type>;
+
     GmresMixed()
         : exec(gko::ReferenceExecutor::create()),
           mtx(gko::initialize<Mtx>(
               {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)),
           gmres_mixed_factory(
-              gko::solver::GmresMixed<>::build()
+              gmres_type::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(4u).on(exec),
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
-                      gko::stop::ResidualNormReduction<>::build()
+                      gko::stop::ResidualNormReduction<value_type>::build()
                           .with_reduction_factor(1e-15)
                           .on(exec))
                   .on(exec)),
@@ -74,11 +86,11 @@ class GmresMixed : public ::testing::Test {
                {-848.1, -280.5, -381.8, -187.1, 51.2, -176.2}},
               exec)),
           gmres_mixed_factory_big(
-              gko::solver::GmresMixed<>::build()
+              gmres_type::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
-                      gko::stop::ResidualNormReduction<>::build()
+                      gko::stop::ResidualNormReduction<value_type>::build()
                           .with_reduction_factor(1e-15)
                           .on(exec))
                   .on(exec)),
@@ -95,107 +107,155 @@ class GmresMixed : public ::testing::Test {
     std::shared_ptr<Mtx> mtx;
     std::shared_ptr<Mtx> mtx_medium;
     std::shared_ptr<Mtx> mtx_big;
-    std::unique_ptr<gko::solver::GmresMixed<>::Factory> gmres_mixed_factory;
-    std::unique_ptr<gko::solver::GmresMixed<>::Factory> gmres_mixed_factory_big;
+    std::unique_ptr<typename gmres_type::Factory> gmres_mixed_factory;
+    std::unique_ptr<typename gmres_type::Factory> gmres_mixed_factory_big;
 };
 
-TEST_F(GmresMixed, SolvesStencilSystem)
+
+using TestTypes =
+    ::testing::Types<std::tuple<double, double>, std::tuple<double, float>,
+                     std::tuple<float, float>>;  //,
+// std::tuple<std::complex<double>, std::complex<double>>,
+// std::tuple<std::complex<double>, std::complex<float>>,
+// std::tuple<std::complex<float>, std::complex<float>>>;
+
+
+TYPED_TEST_CASE(GmresMixed, TestTypes);
+
+
+TYPED_TEST(GmresMixed, SolvesStencilSystem)
 {
-    auto solver = gmres_mixed_factory->generate(mtx);
-    auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, exec);
-    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, exec);
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto solver = this->gmres_mixed_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}), 1e-14);
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}), r<T>::value * 1e1);
 }
 
-TEST_F(GmresMixed, SolvesMultipleStencilSystems)
+
+TYPED_TEST(GmresMixed, SolvesMultipleStencilSystems)
 {
-    auto solver = gmres_mixed_factory->generate(mtx);
-    auto b = gko::initialize<Mtx>({{13.0, 6.0}, {7.0, 4.0}, {1.0, 1.0}}, exec);
-    auto x = gko::initialize<Mtx>({{0.0, 0.0}, {0.0, 0.0}, {0.0, 0.0}}, exec);
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto solver = this->gmres_mixed_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {I<T>{13.0, 6.0}, I<T>{7.0, 4.0}, I<T>{1.0, 1.0}}, this->exec);
+    auto x = gko::initialize<Mtx>(
+        {I<T>{0.0, 0.0}, I<T>{0.0, 0.0}, I<T>{0.0, 0.0}}, this->exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({{1.0, 1.0}, {3.0, 1.0}, {2.0, 1.0}}), 1e-14);
+    GKO_ASSERT_MTX_NEAR(x, l({I<T>{1.0, 1.0}, I<T>{3.0, 1.0}, I<T>{2.0, 1.0}}),
+                        r<T>::value * 1e1);
 }
 
-TEST_F(GmresMixed, SolvesStencilSystemUsingAdvancedApply)
+
+TYPED_TEST(GmresMixed, SolvesStencilSystemUsingAdvancedApply)
 {
-    auto solver = gmres_mixed_factory->generate(mtx);
-    auto alpha = gko::initialize<Mtx>({2.0}, exec);
-    auto beta = gko::initialize<Mtx>({-1.0}, exec);
-    auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, exec);
-    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, exec);
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto solver = this->gmres_mixed_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), 1e-14);
+    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), r<T>::value * 1e1);
 }
 
-TEST_F(GmresMixed, SolvesMultipleStencilSystemsUsingAdvancedApply)
+
+TYPED_TEST(GmresMixed, SolvesMultipleStencilSystemsUsingAdvancedApply)
 {
-    auto solver = gmres_mixed_factory->generate(mtx);
-    auto alpha = gko::initialize<Mtx>({2.0}, exec);
-    auto beta = gko::initialize<Mtx>({-1.0}, exec);
-    auto b = gko::initialize<Mtx>({{13.0, 6.0}, {7.0, 4.0}, {1.0, 1.0}}, exec);
-    auto x = gko::initialize<Mtx>({{0.5, 1.0}, {1.0, 2.0}, {2.0, 3.0}}, exec);
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto solver = this->gmres_mixed_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {I<T>{13.0, 6.0}, I<T>{7.0, 4.0}, I<T>{1.0, 1.0}}, this->exec);
+    auto x = gko::initialize<Mtx>(
+        {I<T>{0.5, 1.0}, I<T>{1.0, 2.0}, I<T>{2.0, 3.0}}, this->exec);
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({{1.5, 1.0}, {5.0, 0.0}, {2.0, -1.0}}), 1e-14);
+    GKO_ASSERT_MTX_NEAR(x, l({I<T>{1.5, 1.0}, I<T>{5.0, 0.0}, I<T>{2.0, -1.0}}),
+                        r<T>::value * 1e1);
 }
 
-TEST_F(GmresMixed, SolvesBigDenseSystem1)
+
+TYPED_TEST(GmresMixed, SolvesBigDenseSystem1)
 {
-    auto solver = gmres_mixed_factory_big->generate(mtx_big);
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto solver = this->gmres_mixed_factory_big->generate(this->mtx_big);
     auto b = gko::initialize<Mtx>(
-        {72748.36, 297469.88, 347229.24, 36290.66, 82958.82, -80192.15}, exec);
-    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, exec);
+        {72748.36, 297469.88, 347229.24, 36290.66, 82958.82, -80192.15},
+        this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({52.7, 85.4, 134.2, -250.0, -16.8, 35.3}), 1e-10);
+    GKO_ASSERT_MTX_NEAR(x, l({52.7, 85.4, 134.2, -250.0, -16.8, 35.3}),
+                        r<T>::value * 1e3);
 }
 
-TEST_F(GmresMixed, SolvesBigDenseSystem2)
+
+TYPED_TEST(GmresMixed, SolvesBigDenseSystem2)
 {
-    auto solver = gmres_mixed_factory_big->generate(mtx_big);
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto solver = this->gmres_mixed_factory_big->generate(this->mtx_big);
     auto b = gko::initialize<Mtx>(
         {175352.10, 313410.50, 131114.10, -134116.30, 179529.30, -43564.90},
-        exec);
-    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, exec);
+        this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}), 1e-10);
+    GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
+                        r<T>::value * 1e3);
 }
 
-double infNorm(gko::matrix::Dense<> *mat, size_t col = 0)
+
+template <typename T>
+gko::remove_complex<T> infNorm(gko::matrix::Dense<T> *mat, size_t col = 0)
 {
     using std::abs;
-    double norm = 0.0;
+    using no_cpx_t = gko::remove_complex<T>;
+    no_cpx_t norm = 0.0;
     for (size_t i = 0; i < mat->get_size()[0]; ++i) {
-        double absEntry = abs(mat->at(i, col));
+        no_cpx_t absEntry = abs(mat->at(i, col));
         if (norm < absEntry) norm = absEntry;
     }
     return norm;
 }
 
-TEST_F(GmresMixed, SolvesMultipleDenseSystemForDivergenceCheck)
+
+TYPED_TEST(GmresMixed, SolvesMultipleDenseSystemForDivergenceCheck)
 {
-    auto solver = gmres_mixed_factory_big->generate(mtx_big);
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto solver = this->gmres_mixed_factory_big->generate(this->mtx_big);
     auto b1 = gko::initialize<Mtx>(
-        {1300083.0, 1018120.5, 906410.0, -42679.5, 846779.5, 1176858.5}, exec);
+        {1300083.0, 1018120.5, 906410.0, -42679.5, 846779.5, 1176858.5},
+        this->exec);
     auto b2 = gko::initialize<Mtx>(
-        {886630.5, -172578.0, 684522.0, -65310.5, 455487.5, 607436.0}, exec);
+        {886630.5, -172578.0, 684522.0, -65310.5, 455487.5, 607436.0},
+        this->exec);
 
-    auto x1 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, exec);
-    auto x2 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, exec);
+    auto x1 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
+    auto x2 = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
 
-    auto bc = Mtx::create(exec, gko::dim<2>{mtx_big->get_size()[0], 2});
-    auto xc = Mtx::create(exec, gko::dim<2>{mtx_big->get_size()[1], 2});
+    auto bc =
+        Mtx::create(this->exec, gko::dim<2>{this->mtx_big->get_size()[0], 2});
+    auto xc =
+        Mtx::create(this->exec, gko::dim<2>{this->mtx_big->get_size()[1], 2});
     for (size_t i = 0; i < bc->get_size()[0]; ++i) {
         bc->at(i, 0) = b1->at(i);
         bc->at(i, 1) = b2->at(i);
@@ -207,86 +267,101 @@ TEST_F(GmresMixed, SolvesMultipleDenseSystemForDivergenceCheck)
     solver->apply(b1.get(), x1.get());
     solver->apply(b2.get(), x2.get());
     solver->apply(bc.get(), xc.get());
-    auto mergedRes = Mtx::create(exec, gko::dim<2>{b1->get_size()[0], 2});
+    auto mergedRes = Mtx::create(this->exec, gko::dim<2>{b1->get_size()[0], 2});
     for (size_t i = 0; i < mergedRes->get_size()[0]; ++i) {
         mergedRes->at(i, 0) = x1->at(i);
         mergedRes->at(i, 1) = x2->at(i);
     }
 
-    auto alpha = gko::initialize<Mtx>({1.0}, exec);
-    auto beta = gko::initialize<Mtx>({-1.0}, exec);
+    auto alpha = gko::initialize<Mtx>({1.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
 
-    auto residual1 = Mtx::create(exec, b1->get_size());
+    auto residual1 = Mtx::create(this->exec, b1->get_size());
     residual1->copy_from(b1.get());
-    auto residual2 = Mtx::create(exec, b2->get_size());
+    auto residual2 = Mtx::create(this->exec, b2->get_size());
     residual2->copy_from(b2.get());
-    auto residualC = Mtx::create(exec, bc->get_size());
+    auto residualC = Mtx::create(this->exec, bc->get_size());
     residualC->copy_from(bc.get());
 
-    mtx_big->apply(alpha.get(), x1.get(), beta.get(), residual1.get());
-    mtx_big->apply(alpha.get(), x2.get(), beta.get(), residual2.get());
-    mtx_big->apply(alpha.get(), xc.get(), beta.get(), residualC.get());
+    this->mtx_big->apply(alpha.get(), x1.get(), beta.get(), residual1.get());
+    this->mtx_big->apply(alpha.get(), x2.get(), beta.get(), residual2.get());
+    this->mtx_big->apply(alpha.get(), xc.get(), beta.get(), residualC.get());
 
-    double normS1 = infNorm(residual1.get());
-    double normS2 = infNorm(residual2.get());
-    double normC1 = infNorm(residualC.get(), 0);
-    double normC2 = infNorm(residualC.get(), 1);
-    double normB1 = infNorm(b1.get());
-    double normB2 = infNorm(b2.get());
+    auto normS1 = infNorm(residual1.get());
+    auto normS2 = infNorm(residual2.get());
+    auto normC1 = infNorm(residualC.get(), 0);
+    auto normC2 = infNorm(residualC.get(), 1);
+    auto normB1 = infNorm(b1.get());
+    auto normB2 = infNorm(b2.get());
 
     // make sure that all combined solutions are as good or better than the
     // single solutions
-    ASSERT_LE(normC1 / normB1, normS1 / normB1 + 1e-14);
-    ASSERT_LE(normC2 / normB2, normS2 / normB2 + 1e-14);
+    ASSERT_LE(normC1 / normB1, normS1 / normB1 + r<value_type>::value);
+    ASSERT_LE(normC2 / normB2, normS2 / normB2 + r<value_type>::value);
 
     // Not sure if this is necessary, the assertions above should cover what is
     // needed.
-    GKO_ASSERT_MTX_NEAR(xc, mergedRes, 1e-14);
+    GKO_ASSERT_MTX_NEAR(xc, mergedRes, r<value_type>::value);
 }
 
-TEST_F(GmresMixed, SolvesBigDenseSystem1WithRestart)
+
+TYPED_TEST(GmresMixed, SolvesBigDenseSystem1WithRestart)
 {
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using gmres_type = typename TestFixture::gmres_type;
+    const auto half_tol = std::sqrt(r<value_type>::value);
     auto gmres_mixed_factory_restart =
-        gko::solver::GmresMixed<>::build()
+        gmres_type::build()
             .with_krylov_dim_mixed(4u)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(200u).on(exec),
-                gko::stop::ResidualNormReduction<>::build()
-                    .with_reduction_factor(1e-15)
-                    .on(exec))
-            .on(exec);
-    auto solver = gmres_mixed_factory_restart->generate(mtx_medium);
+                gko::stop::Iteration::build().with_max_iters(200u).on(
+                    this->exec),
+                gko::stop::ResidualNormReduction<value_type>::build()
+                    .with_reduction_factor(r<value_type>::value)
+                    .on(this->exec))
+            .on(this->exec);
+    auto solver = gmres_mixed_factory_restart->generate(this->mtx_medium);
     auto b = gko::initialize<Mtx>(
-        {-13945.16, 11205.66, 16132.96, 24342.18, -10910.98}, exec);
-    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0}, exec);
+        {-13945.16, 11205.66, 16132.96, 24342.18, -10910.98}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({-140.20, -142.20, 48.80, -17.70, -19.60}), 1e-5);
+    GKO_ASSERT_MTX_NEAR(x, l({-140.20, -142.20, 48.80, -17.70, -19.60}),
+                        half_tol * 1e2);
 }
 
-TEST_F(GmresMixed, SolvesWithPreconditioner)
+
+TYPED_TEST(GmresMixed, SolvesWithPreconditioner)
 {
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using gmres_type = typename TestFixture::gmres_type;
     auto gmres_mixed_factory_preconditioner =
-        gko::solver::GmresMixed<>::build()
+        gmres_type::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(100u).on(exec),
-                gko::stop::ResidualNormReduction<>::build()
+                gko::stop::Iteration::build().with_max_iters(100u).on(
+                    this->exec),
+                gko::stop::ResidualNormReduction<value_type>::build()
                     .with_reduction_factor(1e-15)
-                    .on(exec))
-            .with_preconditioner(gko::preconditioner::Jacobi<>::build()
-                                     .with_max_block_size(3u)
-                                     .on(exec))
-            .on(exec);
-    auto solver = gmres_mixed_factory_preconditioner->generate(mtx_big);
+                    .on(this->exec))
+            .with_preconditioner(
+                gko::preconditioner::Jacobi<value_type>::build()
+                    .with_max_block_size(3u)
+                    .on(this->exec))
+            .on(this->exec);
+    auto solver = gmres_mixed_factory_preconditioner->generate(this->mtx_big);
     auto b = gko::initialize<Mtx>(
         {175352.10, 313410.50, 131114.10, -134116.30, 179529.30, -43564.90},
-        exec);
-    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, exec);
+        this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}), 1e-10);
+    GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
+                        r<value_type>::value * 1e3);
 }
 
+
 }  // namespace

From b44b100a5fd6f1dc1fd43a592bf8ae55b46cfa3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 14 May 2020 00:10:26 +0200
Subject: [PATCH 0344/1008] Made GmresMixed compile with complex types

---
 common/solver/gmres_mixed_kernels.hpp.inc     | 11 +--
 core/solver/gmres_mixed_kernels.hpp           | 10 +--
 omp/solver/gmres_mixed_kernels.cpp            | 69 ++++++++++---------
 reference/solver/gmres_mixed_kernels.cpp      | 31 +++++----
 reference/test/solver/gmres_mixed_kernels.cpp | 14 ++--
 5 files changed, 71 insertions(+), 64 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index fe2a354840a..7f4df6724dc 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -415,9 +415,9 @@ __global__
             hessenberg_iter[hessenberg_idx] *
             krylov_bases.read(row_idx, k * num_cols + col_idx);
         if (row_idx == 0) {
-            if ((hessenberg_iter[hessenberg_idx] *
-                 hessenberg_iter[hessenberg_idx]) >
-                (0.99 * sqrt(arnoldi_norm[arnoldi_idx]))) {
+            // TODO remove abs() as soon as type of arnoldi_norm has changed
+            if (squared_norm(hessenberg_iter[hessenberg_idx]) >
+                abs(0.99 * sqrt(arnoldi_norm[arnoldi_idx]))) {
                 reorth_status[reorth_idx].reset();
                 atomic_add(num_reorth, (size_type)1);
             } else {
@@ -513,8 +513,9 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms(
     const auto global_id = thread::get_thread_id_flat();
 
     if (global_id < stride_norm0 && !stop_status[global_id].has_stopped()) {
-        const ValueType num0 = eta * arnoldi_norm0[global_id];
-        const ValueType num1 = arnoldi_norm1[global_id];
+        // TODO remove abs() as soon as type of arnoldi_norm has changed
+        const auto num0 = abs(eta * arnoldi_norm0[global_id]);
+        const auto num1 = abs(arnoldi_norm1[global_id]);
         if (num1 < num0) {
             reorth_status[global_id].reset();
             atomic_add(num_reorth, (size_type)1);
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index d4ed241faa6..bae812b9c70 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -61,11 +61,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 //     template _macro(double, float);                 \
 //     template _macro(double, double)
 // #else
-#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro) \
-    template _macro(double, double);                      \
-    template _macro(double, float);                       \
-    template _macro(float, float)  //;                  \
-    template _macro(std::complex<double>, std::complex<double>);  \
+#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro)        \
+    template _macro(double, double);                             \
+    template _macro(double, float);                              \
+    template _macro(float, float);                               \
+    template _macro(std::complex<double>, std::complex<double>); \
     template _macro(std::complex<double>, std::complex<float>);  \
     template _macro(std::complex<float>, std::complex<float>)
 // #endif
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index 25ded584943..e9c3e8e4071 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -140,16 +140,17 @@ void finish_arnoldi_reorth(
     const stopping_status *stop_status)
 {
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
+#pragma omp declare reduction(addnc : remove_complex<ValueType> : omp_out = omp_out + omp_in)
 
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
         if (stop_status[i].has_stopped()) {
             continue;
         }
 
-        ValueType nrm = zero<ValueType>();
-#pragma omp parallel for reduction(add : nrm)
+        remove_complex<ValueType> nrm = zero<ValueType>();
+#pragma omp parallel for reduction(addnc : nrm)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            nrm += squared_norm(next_krylov_basis->at(j, i));
         }
         arnoldi_norm->at(0, i) = sqrt(nrm) * 0.99;
         // nrm = norm(next_krylov_basis)
@@ -171,8 +172,9 @@ void finish_arnoldi_reorth(
                     krylov_bases.read(j,
                                       next_krylov_basis->get_size()[1] * k + i);
             }
-            if (hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i) >
-                arnoldi_norm->at(0, i)) {
+            // TODO remove abs() as soon as type for arnoldi_norm has changed!
+            if (squared_norm(hessenberg_iter->at(k, i)) >
+                abs(arnoldi_norm->at(0, i))) {
                 ValueType reorth = zero<ValueType>();
 #pragma omp parallel for reduction(add : reorth)
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
@@ -236,16 +238,17 @@ void finish_arnoldi_CGS(
 {
     const ValueType eta = 1.0 / sqrt(2.0);
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
+#pragma omp declare reduction(addnc : remove_complex<ValueType> : omp_out = omp_out + omp_in)
 
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
         if (stop_status[i].has_stopped()) {
             continue;
         }
 
-        ValueType nrm = zero<ValueType>();
-#pragma omp parallel for reduction(add : nrm)
+        remove_complex<ValueType> nrm = zero<remove_complex<ValueType>>();
+#pragma omp parallel for reduction(addnc : nrm)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            nrm += squared_norm(next_krylov_basis->at(j, i));
         }
         arnoldi_norm->at(0, i) = nrm * eta;
         // nrmP = norm(next_krylov_basis)
@@ -277,14 +280,17 @@ void finish_arnoldi_CGS(
         // end
         // ValueType nrm = zero<ValueType>();
         nrm = zero<ValueType>();
-#pragma omp parallel for reduction(add : nrm)
+#pragma omp parallel for reduction(addnc : nrm)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            nrm += squared_norm(next_krylov_basis->at(j, i));
         }
         arnoldi_norm->at(1, i) = nrm;
         // nrmN = norm(next_krylov_basis)
+
+        // TODO remove the abs() as soon as the type of arnoldi_norm has changed
         for (size_type l = 1;
-             arnoldi_norm->at(1, i) < arnoldi_norm->at(0, i) && l < 3; l++) {
+             abs(arnoldi_norm->at(1, i)) < abs(arnoldi_norm->at(0, i)) && l < 3;
+             l++) {
             arnoldi_norm->at(0, i) = arnoldi_norm->at(1, i) * eta;
             // nrmP = nrmN
             for (size_type k = 0; k < iter + 1; ++k) {
@@ -315,11 +321,10 @@ void finish_arnoldi_CGS(
             // for i in 1:iter
             //     next_krylov_basis  -= buffer(iter, i) * krylov_bases(:, i)
             // end
-            ValueType nrm = zero<ValueType>();
-#pragma omp parallel for reduction(add : nrm)
+            remove_complex<ValueType> nrm = zero<ValueType>();
+#pragma omp parallel for reduction(addnc : nrm)
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                nrm +=
-                    next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+                nrm += squared_norm(next_krylov_basis->at(j, i));
             }
             arnoldi_norm->at(1, i) = nrm;
             // nrmN = norm(next_krylov_basis)
@@ -357,18 +362,19 @@ void finish_arnoldi_CGS2(
     matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
     const stopping_status *stop_status)
 {
-    const ValueType eta = 1.0 / sqrt(2.0);
+    const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
+#pragma omp declare reduction(addnc : remove_complex<ValueType> : omp_out = omp_out + omp_in)
 
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
         if (stop_status[i].has_stopped()) {
             continue;
         }
 
-        ValueType nrm = zero<ValueType>();
-#pragma omp parallel for reduction(add : nrm)
+        remove_complex<ValueType> nrm = zero<remove_complex<ValueType>>();
+#pragma omp parallel for reduction(addnc : nrm)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            nrm += squared_norm(next_krylov_basis->at(j, i));
         }
         arnoldi_norm->at(0, i) = nrm * eta;
         // nrmP = norm(next_krylov_basis)
@@ -422,15 +428,18 @@ void finish_arnoldi_CGS2(
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
         // ValueType nrm = zero<ValueType>();
-        nrm = zero<ValueType>();
-#pragma omp parallel for reduction(add : nrm)
+        nrm = zero<remove_complex<ValueType>>();
+#pragma omp parallel for reduction(addnc : nrm)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            nrm += squared_norm(next_krylov_basis->at(j, i));
         }
         arnoldi_norm->at(1, i) = nrm;
         // nrmN = norm(next_krylov_basis)
+
+        // TODO remove abs() as soon as type of arnoldi_norm has changed!
         for (size_type l = 1;
-             arnoldi_norm->at(1, i) < arnoldi_norm->at(0, i) && l < 3; l++) {
+             abs(arnoldi_norm->at(1, i)) < abs(arnoldi_norm->at(0, i)) && l < 3;
+             l++) {
             arnoldi_norm->at(0, i) = arnoldi_norm->at(1, i) * eta;
             // nrmP = nrmN
 #pragma omp parallel for
@@ -461,11 +470,10 @@ void finish_arnoldi_CGS2(
             // for i in 1:iter
             //     next_krylov_basis  -= buffer(iter, i) * krylov_bases(:, i)
             // end
-            ValueType nrm = zero<ValueType>();
-#pragma omp parallel for reduction(add : nrm)
+            remove_complex<ValueType> nrm = zero<remove_complex<ValueType>>();
+#pragma omp parallel for reduction(addnc : nrm)
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                nrm +=
-                    next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+                nrm += squared_norm(next_krylov_basis->at(j, i));
             }
             arnoldi_norm->at(1, i) = nrm;
             // nrmN = norm(next_krylov_basis)
@@ -706,12 +714,9 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
     }
 
 #pragma omp parallel for
-    for (size_type i = 0;
-         i < residual->get_size()[0] /*krylov_bases->get_size()[0]*/; ++i) {
+    for (size_type i = 0; i < residual->get_size()[0]; ++i) {
         for (size_type j = residual->get_size()[1];
-             j < (krylov_dim + 1) *
-                     residual->get_size()[1] /*krylov_bases->get_size()[1]*/;
-             ++j) {
+             j < (krylov_dim + 1) * residual->get_size()[1]; ++j) {
             krylov_bases.write(i, j, zero<ValueType>());
         }
     }
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index af3dfc820b4..2f0599f0ba9 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -126,8 +126,7 @@ void finish_arnoldi_reorth(
         // nrm = 0;
         arnoldi_norm->at(0, i) = 0;
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            arnoldi_norm->at(0, i) +=
-                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            arnoldi_norm->at(0, i) += squared_norm(next_krylov_basis->at(j, i));
             // nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
         }
         arnoldi_norm->at(0, i) = sqrt(arnoldi_norm->at(0, i)) * 0.99;
@@ -147,11 +146,14 @@ void finish_arnoldi_reorth(
                     krylov_bases.read(j,
                                       next_krylov_basis->get_size()[1] * k + i);
             }
-            if ((hessenberg_iter->at(k, i) * hessenberg_iter->at(k, i)) >
+            if (squared_norm(hessenberg_iter->at(k, i) *
+                             hessenberg_iter->at(k, i)) >
                 arnoldi_norm->at(0, i)) {
                 arnoldi_norm->at(1, i) = 0;
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
+                    // TODO find good alternative here, so the type of
+                    // arnoldi_norm can change to remove_complex<ValueType>!!!
                     arnoldi_norm->at(1, i) +=
                         next_krylov_basis->at(j, i) *
                         krylov_bases.read(
@@ -207,15 +209,14 @@ void finish_arnoldi_CGS(
     matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
     const stopping_status *stop_status)
 {
-    const ValueType eta = 1.0 / sqrt(2.0);
+    const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
     //    ValueType nrmP = 0, nrmN = 0;
     ValueType nrmP = 0, nrmN = 0;
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
         arnoldi_norm->at(0, i) = 0;
         // nrmP = 0;
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            arnoldi_norm->at(0, i) +=
-                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            arnoldi_norm->at(0, i) += squared_norm(next_krylov_basis->at(j, i));
             // nrmP += next_krylov_basis->at(j, i) * next_krylov_basis->at(j,
             // i);
         }
@@ -250,13 +251,16 @@ void finish_arnoldi_CGS(
         // end
         arnoldi_norm->at(1, i) = 0;
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            arnoldi_norm->at(1, i) +=
-                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            arnoldi_norm->at(1, i) += squared_norm(next_krylov_basis->at(j, i));
         }
         arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
         // nrmN = norm(next_krylov_basis)
+
+        // TODO the abs() should be removed here as soon as the type of
+        // arnoldi_norm has changed!
         for (size_type l = 1;
-             arnoldi_norm->at(1, i) < arnoldi_norm->at(0, i) && l < 3; l++) {
+             abs(arnoldi_norm->at(1, i)) < abs(arnoldi_norm->at(0, i)) && l < 3;
+             l++) {
             arnoldi_norm->at(0, i) = eta * arnoldi_norm->at(1, i);
             for (size_type k = 0; k < iter + 1; ++k) {
                 buffer_iter->at(k, i) = 0;
@@ -288,7 +292,7 @@ void finish_arnoldi_CGS(
             arnoldi_norm->at(1, i) = 0;
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 arnoldi_norm->at(1, i) +=
-                    next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+                    squared_norm(next_krylov_basis->at(j, i));
             }
             arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
             // nrmN = norm(next_krylov_basis)
@@ -510,11 +514,8 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
     }
 
     for (size_type j = residual->get_size()[1];
-         j < (krylov_dim + 1) *
-                 residual->get_size()[1] /*krylov_bases->get_size()[1]*/;
-         ++j) {
-        for (size_type i = 0;
-             i < residual->get_size()[0] /*krylov_bases->get_size()[0]*/; ++i) {
+         j < (krylov_dim + 1) * residual->get_size()[1]; ++j) {
+        for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             krylov_bases.write(i, j, zero<ValueType>());
         }
     }
diff --git a/reference/test/solver/gmres_mixed_kernels.cpp b/reference/test/solver/gmres_mixed_kernels.cpp
index 3e399dec481..62221f10e59 100644
--- a/reference/test/solver/gmres_mixed_kernels.cpp
+++ b/reference/test/solver/gmres_mixed_kernels.cpp
@@ -74,7 +74,7 @@ class GmresMixed : public ::testing::Test {
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
                       gko::stop::ResidualNormReduction<value_type>::build()
-                          .with_reduction_factor(1e-15)
+                          .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
           mtx_big(gko::initialize<Mtx>(
@@ -91,7 +91,7 @@ class GmresMixed : public ::testing::Test {
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
                       gko::stop::ResidualNormReduction<value_type>::build()
-                          .with_reduction_factor(1e-15)
+                          .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec)),
           mtx_medium(
@@ -114,10 +114,10 @@ class GmresMixed : public ::testing::Test {
 
 using TestTypes =
     ::testing::Types<std::tuple<double, double>, std::tuple<double, float>,
-                     std::tuple<float, float>>;  //,
-// std::tuple<std::complex<double>, std::complex<double>>,
-// std::tuple<std::complex<double>, std::complex<float>>,
-// std::tuple<std::complex<float>, std::complex<float>>>;
+                     std::tuple<float, float>,
+                     std::tuple<std::complex<double>, std::complex<double>>,
+                     std::tuple<std::complex<double>, std::complex<float>>,
+                     std::tuple<std::complex<float>, std::complex<float>>>;
 
 
 TYPED_TEST_CASE(GmresMixed, TestTypes);
@@ -344,7 +344,7 @@ TYPED_TEST(GmresMixed, SolvesWithPreconditioner)
                 gko::stop::Iteration::build().with_max_iters(100u).on(
                     this->exec),
                 gko::stop::ResidualNormReduction<value_type>::build()
-                    .with_reduction_factor(1e-15)
+                    .with_reduction_factor(r<value_type>::value)
                     .on(this->exec))
             .with_preconditioner(
                 gko::preconditioner::Jacobi<value_type>::build()

From 6dac3600a11c457b19809a12a13d24a841bdbf93 Mon Sep 17 00:00:00 2001
From: josealiaga <aliaga@uji.es>
Date: Tue, 19 May 2020 13:29:18 +0200
Subject: [PATCH 0345/1008] The update routines have been improved. Now the
 computational time is close to 75s for 6221 iters.

Next steps should be:
  * Add the computation of the inf-norm for the next_krylov_basis.
  * Merge updating and norms computations.
---
 common/components/reduction.hpp.inc       |  45 +++
 common/solver/gmres_mixed_kernels.hpp.inc | 433 ++++++++++++++++++++++
 cuda/solver/gmres_mixed_kernels.cu        |  27 ++
 3 files changed, 505 insertions(+)

diff --git a/common/components/reduction.hpp.inc b/common/components/reduction.hpp.inc
index ef56a05504f..fd34841c85d 100644
--- a/common/components/reduction.hpp.inc
+++ b/common/components/reduction.hpp.inc
@@ -128,6 +128,51 @@ __device__ void reduce(const Group &__restrict__ group,
 }
 
 
+/**
+ * @internal
+ *
+ * Computes a reduction using the binary operation `reduce_op` on entire block.
+ * The data for the reduction is taken from the `data` array which has to be of
+ * size `block_size` and accessible from all threads. The `data` array is also
+ * used as work space (so its content will be destroyed in the process), as well
+ * as to store the return value - which is stored in the 0-th position of the
+ * array.
+ */
+template <
+    typename Group, typename ValueType, typename Operator,
+    typename = xstd::enable_if_t<group::is_synchronizable_group<Group>::value>>
+__device__ void multireduce(const Group &__restrict__ group,
+                            ValueType *__restrict__ data, size_type stride,
+                            size_type num, Operator reduce_op = Operator{})
+{
+    const auto local_id = group.thread_rank();
+
+    for (int k = group.size() / 2; k >= config::warp_size; k /= 2) {
+        group.sync();
+        if (local_id < k) {
+            for (int j = 0; j < num; j++) {
+                data[j * stride + local_id] =
+                    reduce_op(data[j * stride + local_id],
+                              data[j * stride + local_id + k]);
+            }
+        }
+    }
+
+    const auto warp = group::tiled_partition<config::warp_size>(group);
+    const auto warp_id = group.thread_rank() / warp.size();
+    if (warp_id > 0) {
+        return;
+    }
+    for (int j = 0; j < num; j++) {
+        auto result =
+            reduce(warp, data[j * stride + warp.thread_rank()], reduce_op);
+        if (warp.thread_rank() == 0) {
+            data[j * stride] = result;
+        }
+    }
+}
+
+
 /**
  * @internal
  *
diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 7f4df6724dc..fda76c391dd 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -384,6 +384,364 @@ __global__
 }
 
 
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__
+    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_0(
+        size_type num_iters, size_type num_rows, size_type num_cols,
+        ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
+        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        const ValueType *__restrict__ hessenberg_iter,
+        size_type stride_hessenberg,
+        const stopping_status *__restrict__ stop_status)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    //    const auto row_idx = global_id / stride_next_krylov;
+    //    const auto col_idx = global_id % stride_next_krylov;
+    const auto row_idx_krylov = global_id / (stride_next_krylov * num_iters);
+    const auto col_idx_krylov = global_id % (stride_next_krylov * num_iters);
+    //    const auto num_itr_krylov = col_idx_krylov / stride_next_krylov;
+    const auto k = col_idx_krylov / stride_next_krylov;
+    const auto col_itr_krylov = col_idx_krylov % stride_next_krylov;
+
+    if (row_idx_krylov < num_rows && col_itr_krylov < num_cols &&
+        !stop_status[col_itr_krylov].has_stopped()) {
+        const auto next_krylov_idx =
+            row_idx_krylov * stride_next_krylov + col_itr_krylov;
+        const auto hessenberg_idx = k * stride_hessenberg + col_itr_krylov;
+
+        auto value =
+            hessenberg_iter[hessenberg_idx] *
+            krylov_bases.read(row_idx_krylov, k * num_cols + col_itr_krylov);
+        atomic_add(next_krylov_basis + next_krylov_idx, -value);
+    }
+}
+
+
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__
+    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_1(
+        size_type num_iters, size_type num_rows, size_type num_cols,
+        ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
+        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        const ValueType *__restrict__ hessenberg_iter,
+        size_type stride_hessenberg,
+        const stopping_status *__restrict__ stop_status)
+{
+    if (num_cols == 1) {
+        const auto tidx = threadIdx.x;
+        const size_type row_idx = blockIdx.x;
+        const auto col_idx = 0;
+
+        // Used that way to get around dynamic initialization warning and
+        // template error when using `reduction_helper_array` directly in
+        // `reduce`
+        __shared__ UninitializedArray<ValueType, block_size>
+            reduction_helper_array;
+        ValueType *__restrict__ reduction_helper = reduction_helper_array;
+        if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+            ValueType value = zero<ValueType>();
+            for (size_type i = tidx; i < num_iters; i += blockDim.x) {
+                const auto hessenberg_idx = i * stride_hessenberg + col_idx;
+                const auto krylov_idx = i * num_cols + col_idx;
+
+                value += hessenberg_iter[hessenberg_idx] *
+                         krylov_bases.read(row_idx, krylov_idx);
+            }
+            reduction_helper[tidx] = value;
+
+            // Perform thread block reduction. Result is in
+            // reduction_helper[0]
+            reduce(
+                group::this_thread_block(), reduction_helper,
+                [](const ValueType &a, const ValueType &b) { return a + b; });
+            if (tidx == 0) {
+                const auto next_krylov_idx =
+                    row_idx * stride_next_krylov + col_idx;
+                next_krylov_basis[next_krylov_idx] -= reduction_helper[0];
+            }
+        }
+    } else {
+        const auto global_id = thread::get_thread_id_flat();
+        const auto row_idx = global_id / stride_next_krylov;
+        const auto col_idx = global_id % stride_next_krylov;
+
+        if (row_idx < num_rows && col_idx < num_cols &&
+            !stop_status[col_idx].has_stopped()) {
+            const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+            for (size_type k = 0; k < num_iters; ++k) {
+                // const auto krylov_idx =
+                //    row_idx * stride_krylov + k * num_cols + col_idx;
+                const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+
+                next_krylov_basis[next_krylov_idx] -=
+                    hessenberg_iter[hessenberg_idx] *
+                    krylov_bases.read(row_idx, k * num_cols + col_idx);
+            }
+        }
+    }
+}
+
+
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__
+    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_2(
+        size_type num_iters, size_type num_blocks, size_type num_rows,
+        size_type num_cols, ValueType *__restrict__ next_krylov_basis,
+        size_type stride_next_krylov,
+        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        const ValueType *__restrict__ hessenberg_iter,
+        size_type stride_hessenberg,
+        const stopping_status *__restrict__ stop_status)
+{
+    if (num_cols == 1) {
+        const auto tidx = threadIdx.x;
+        const size_type row_idx = blockIdx.x * num_blocks;
+        const auto col_idx = 0;
+
+        // Used that way to get around dynamic initialization warning and
+        // template error when using `reduction_helper_array` directly in
+        // `reduce`
+        __shared__ UninitializedArray<ValueType, block_size * block_size>
+            reduction_helper_array;
+        ValueType *__restrict__ reduction_helper = reduction_helper_array;
+        if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+            //            for (size_type j = 0; j < num_blocks; j++) {
+            for (size_type j = row_idx, k = 0; j < num_rows && k < num_blocks;
+                 j++, k++) {
+                ValueType value = zero<ValueType>();
+                for (size_type i = tidx; i < num_iters; i += blockDim.x) {
+                    const auto hessenberg_idx = i * stride_hessenberg + col_idx;
+                    const auto krylov_idx = i * num_cols + col_idx;
+
+                    value += hessenberg_iter[hessenberg_idx] *
+                             krylov_bases.read(j, krylov_idx);
+                    //                             krylov_bases.read(row_idx+j,
+                    //                             krylov_idx);
+                }
+                reduction_helper[k * block_size + tidx] = value;
+            }
+
+            //            for (size_type j = 0; j < num_blocks; j++) {
+            for (size_type j = row_idx, k = 0; j < num_rows && k < num_blocks;
+                 j++, k++) {
+                // Perform thread block reduction. Result is in
+                // reduction_helper[0]
+                reduce(group::this_thread_block(),
+                       reduction_helper + k * block_size,
+                       [](const ValueType &a, const ValueType &b) {
+                           return a + b;
+                       });
+            }
+            if (tidx == 0) {
+                // for (size_type j = 0; j < num_blocks; j++) {
+                for (size_type j = row_idx, k = 0;
+                     j < num_rows && k < num_blocks; j++, k++) {
+                    // (row_idx + j) * stride_next_krylov + col_idx;
+                    const auto next_krylov_idx =
+                        j * stride_next_krylov + col_idx;
+                    next_krylov_basis[next_krylov_idx] -=
+                        reduction_helper[k * block_size];
+                    // reduction_helper[j * block_size];
+                }
+            }
+        }
+    } else {
+        const auto global_id = thread::get_thread_id_flat();
+        const auto row_idx = global_id / stride_next_krylov;
+        const auto col_idx = global_id % stride_next_krylov;
+
+        if (row_idx < num_rows && col_idx < num_cols &&
+            !stop_status[col_idx].has_stopped()) {
+            const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+            for (size_type k = 0; k < num_iters; ++k) {
+                // const auto krylov_idx =
+                //    row_idx * stride_krylov + k * num_cols + col_idx;
+                const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+
+                next_krylov_basis[next_krylov_idx] -=
+                    hessenberg_iter[hessenberg_idx] *
+                    krylov_bases.read(row_idx, k * num_cols + col_idx);
+            }
+        }
+    }
+}
+
+
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__
+    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_3(
+        size_type num_iters, size_type num_blocks, size_type num_rows,
+        size_type num_cols, ValueType *__restrict__ next_krylov_basis,
+        size_type stride_next_krylov,
+        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        const ValueType *__restrict__ hessenberg_iter,
+        size_type stride_hessenberg,
+        const stopping_status *__restrict__ stop_status)
+{
+    if (num_cols == 1) {
+        const auto tidx = threadIdx.x;
+        const size_type row_idx = blockIdx.x * num_blocks;
+        const auto col_idx = 0;
+
+        // Used that way to get around dynamic initialization warning and
+        // template error when using `reduction_helper_array` directly in
+        // `reduce`
+        __shared__ UninitializedArray<ValueType, block_size * block_size>
+            reduction_helper_array;
+        ValueType *__restrict__ reduction_helper = reduction_helper_array;
+        if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+            //            for (size_type j = 0; j < num_blocks; j++) {
+
+            for (size_type j = row_idx, k = 0; j < num_rows && k < num_blocks;
+                 j++, k++) {
+                reduction_helper[k * block_size + tidx] = zero<ValueType>();
+            }
+            for (size_type i = tidx; i < num_iters; i += blockDim.x) {
+                const auto hessenberg_idx = i * stride_hessenberg + col_idx;
+                const auto krylov_idx = i * num_cols + col_idx;
+                const auto hessenberg_val = hessenberg_iter[hessenberg_idx];
+                for (size_type j = row_idx, k = 0;
+                     j < num_rows && k < num_blocks; j++, k++) {
+                    reduction_helper[k * block_size + tidx] +=
+                        hessenberg_val * krylov_bases.read(j, krylov_idx);
+                }
+            }
+
+            //            for (size_type j = 0; j < num_blocks; j++) {
+            for (size_type j = row_idx, k = 0; j < num_rows && k < num_blocks;
+                 j++, k++) {
+                // Perform thread block reduction. Result is in
+                // reduction_helper[0]
+                reduce(group::this_thread_block(),
+                       reduction_helper + k * block_size,
+                       [](const ValueType &a, const ValueType &b) {
+                           return a + b;
+                       });
+            }
+            if (tidx == 0) {
+                // for (size_type j = 0; j < num_blocks; j++) {
+                for (size_type j = row_idx, k = 0;
+                     j < num_rows && k < num_blocks; j++, k++) {
+                    // (row_idx + j) * stride_next_krylov + col_idx;
+                    const auto next_krylov_idx =
+                        j * stride_next_krylov + col_idx;
+                    next_krylov_basis[next_krylov_idx] -=
+                        reduction_helper[k * block_size];
+                    // reduction_helper[j * block_size];
+                }
+            }
+        }
+    } else {
+        const auto global_id = thread::get_thread_id_flat();
+        const auto row_idx = global_id / stride_next_krylov;
+        const auto col_idx = global_id % stride_next_krylov;
+
+        if (row_idx < num_rows && col_idx < num_cols &&
+            !stop_status[col_idx].has_stopped()) {
+            const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+            for (size_type k = 0; k < num_iters; ++k) {
+                // const auto krylov_idx =
+                //    row_idx * stride_krylov + k * num_cols + col_idx;
+                const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+
+                next_krylov_basis[next_krylov_idx] -=
+                    hessenberg_iter[hessenberg_idx] *
+                    krylov_bases.read(row_idx, k * num_cols + col_idx);
+            }
+        }
+    }
+}
+
+
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__
+    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_4(
+        size_type num_iters, size_type num_blocks, size_type num_rows,
+        size_type num_cols, ValueType *__restrict__ next_krylov_basis,
+        size_type stride_next_krylov,
+        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        const ValueType *__restrict__ hessenberg_iter,
+        size_type stride_hessenberg,
+        const stopping_status *__restrict__ stop_status)
+{
+    if (num_cols == 1) {
+        const auto tidx = threadIdx.x;
+        const size_type row_idx = blockIdx.x * num_blocks;
+        const auto col_idx = 0;
+
+        // Used that way to get around dynamic initialization warning and
+        // template error when using `reduction_helper_array` directly in
+        // `reduce`
+        __shared__ UninitializedArray<ValueType, block_size * block_size>
+            reduction_helper_array;
+        ValueType *__restrict__ reduction_helper = reduction_helper_array;
+        if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+            //            for (size_type j = 0; j < num_blocks; j++) {
+
+            for (size_type j = row_idx, k = 0; j < num_rows && k < num_blocks;
+                 j++, k++) {
+                reduction_helper[k * block_size + tidx] = zero<ValueType>();
+            }
+            for (size_type i = tidx; i < num_iters; i += blockDim.x) {
+                const auto hessenberg_idx = i * stride_hessenberg + col_idx;
+                const auto krylov_idx = i * num_cols + col_idx;
+                const auto hessenberg_val = hessenberg_iter[hessenberg_idx];
+                for (size_type j = row_idx, k = 0;
+                     j < num_rows && k < num_blocks; j++, k++) {
+                    reduction_helper[k * block_size + tidx] +=
+                        hessenberg_val * krylov_bases.read(j, krylov_idx);
+                }
+            }
+            multireduce(
+                group::this_thread_block(), reduction_helper, block_size,
+                block_size,
+                [](const ValueType &a, const ValueType &b) { return a + b; });
+
+            if (tidx == 0) {
+                // for (size_type j = 0; j < num_blocks; j++) {
+                for (size_type j = row_idx, k = 0;
+                     j < num_rows && k < num_blocks; j++, k++) {
+                    // (row_idx + j) * stride_next_krylov + col_idx;
+                    const auto next_krylov_idx =
+                        j * stride_next_krylov + col_idx;
+                    next_krylov_basis[next_krylov_idx] -=
+                        reduction_helper[k * block_size];
+                    // reduction_helper[j * block_size];
+                }
+            }
+        }
+    } else {
+        const auto global_id = thread::get_thread_id_flat();
+        const auto row_idx = global_id / stride_next_krylov;
+        const auto col_idx = global_id % stride_next_krylov;
+
+        if (row_idx < num_rows && col_idx < num_cols &&
+            !stop_status[col_idx].has_stopped()) {
+            const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+            for (size_type k = 0; k < num_iters; ++k) {
+                // const auto krylov_idx =
+                //    row_idx * stride_krylov + k * num_cols + col_idx;
+                const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+
+                next_krylov_basis[next_krylov_idx] -=
+                    hessenberg_iter[hessenberg_idx] *
+                    krylov_bases.read(row_idx, k * num_cols + col_idx);
+            }
+        }
+    }
+}
+
+
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
 template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
@@ -498,6 +856,81 @@ __global__
     }
 }
 
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__
+    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_and_add_1(
+        size_type num_iters, size_type num_rows, size_type num_cols,
+        ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
+        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+        const ValueType *__restrict__ buffer_iter, size_type stride_buffer,
+        const stopping_status *__restrict__ stop_status,
+        const stopping_status *__restrict__ reorth_status)
+{
+    if (num_cols == 1) {
+        const auto tidx = threadIdx.x;
+        const size_type row_idx = blockIdx.x;
+        const auto col_idx = 0;
+
+        // Used that way to get around dynamic initialization warning and
+        // template error when using `reduction_helper_array` directly in
+        // `reduce`
+        __shared__ UninitializedArray<ValueType, block_size>
+            reduction_helper_array;
+        ValueType *__restrict__ reduction_helper = reduction_helper_array;
+        if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+            ValueType value = zero<ValueType>();
+            for (size_type i = tidx; i < num_iters; i += blockDim.x) {
+                const auto hessenberg_idx = i * stride_hessenberg + col_idx;
+                const auto buffer_idx = i * stride_buffer + col_idx;
+                const auto krylov_idx = i * num_cols + col_idx;
+
+                value += buffer_iter[buffer_idx] *
+                         krylov_bases.read(row_idx, krylov_idx);
+                if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
+                    hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
+                }
+            }
+            reduction_helper[tidx] = value;
+
+            // Perform thread block reduction. Result is in
+            // reduction_helper[0]
+            reduce(
+                group::this_thread_block(), reduction_helper,
+                [](const ValueType &a, const ValueType &b) { return a + b; });
+            if (tidx == 0) {
+                const auto next_krylov_idx =
+                    row_idx * stride_next_krylov + col_idx;
+                next_krylov_basis[next_krylov_idx] -= reduction_helper[0];
+            }
+        }
+    } else {
+        const auto global_id = thread::get_thread_id_flat();
+        const auto row_idx = global_id / stride_next_krylov;
+        const auto col_idx = global_id % stride_next_krylov;
+
+        if (row_idx < num_rows && col_idx < num_cols &&
+            !stop_status[col_idx].has_stopped() &&
+            !reorth_status[col_idx].has_stopped()) {
+            const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+            for (size_type k = 0; k < num_iters; ++k) {
+                // const auto krylov_idx =
+                //    row_idx * stride_krylov + k * num_cols + col_idx;
+                const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+                const auto buffer_idx = k * stride_buffer + col_idx;
+                next_krylov_basis[next_krylov_idx] -=
+                    buffer_iter[buffer_idx] *
+                    krylov_bases.read(row_idx, k * num_cols + col_idx);
+                if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
+                    hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
+                }
+            }
+        }
+    }
+}
+
 
 // Must be called with at least `num_cols` blocks, each with `block_size`
 // threads. `block_size` must be a power of 2.
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index bc9d19f87ca..f3fdf02578d 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -83,6 +83,9 @@ constexpr int default_dot_dim = 32;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
+constexpr int default_update_dim = 32;
+
+
 #include "common/solver/gmres_mixed_kernels.hpp.inc"
 
 
@@ -597,6 +600,7 @@ void finish_arnoldi_CGS2(
     exec->synchronize();
     auto start_2 = std::chrono::steady_clock::now();
 #endif
+    /*
     update_next_krylov_kernel_num_iters<default_block_size>
         <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
            default_block_size>>>(
@@ -605,6 +609,18 @@ void finish_arnoldi_CGS2(
             as_cuda_accessor(krylov_bases.to_const()),
             as_cuda_type(hessenberg_iter->get_const_values()),
             stride_hessenberg, as_cuda_type(stop_status));
+    */
+    /* */
+    // update_next_krylov_kernel_num_iters_1<default_block_size>
+    //     <<<dim_size[0], default_block_size>>>(
+    update_next_krylov_kernel_num_iters_1<default_update_dim>
+        <<<dim_size[0] * stride_next_krylov, default_update_dim>>>(
+            iter + 1, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
+            as_cuda_accessor(krylov_bases.to_const()),
+            as_cuda_type(hessenberg_iter->get_const_values()),
+            stride_hessenberg, as_cuda_type(stop_status));
+    /* */
 #ifdef TIMING
     exec->synchronize();
     auto time_2 = std::chrono::steady_clock::now() - start_2;
@@ -694,6 +710,7 @@ void finish_arnoldi_CGS2(
         // for i in 1:iter
         //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
         // end
+        /*
         update_next_krylov_kernel_num_iters_and_add<default_block_size>
             <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
                default_block_size>>>(
@@ -703,6 +720,16 @@ void finish_arnoldi_CGS2(
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
                 as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
                 as_cuda_type(stop_status), as_cuda_type(reorth_status));
+        */ /* */
+        update_next_krylov_kernel_num_iters_and_add_1<default_update_dim>
+            <<<dim_size[0] * stride_next_krylov, default_update_dim>>>(
+                iter + 1, dim_size[0], dim_size[1],
+                as_cuda_type(next_krylov_basis->get_values()),
+                stride_next_krylov, as_cuda_accessor(krylov_bases.to_const()),
+                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+                as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
+                as_cuda_type(stop_status), as_cuda_type(reorth_status));
+        /* */
         // for i in 1:iter
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end

From 84a39169b18d6e8ec547133564a7e60efff69233 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 19 May 2020 11:03:10 +0200
Subject: [PATCH 0346/1008] Add specialization for integer types for Accessor

The specialization is currently set to only work with float storage
type to test the pipeline, but it can easily be modified to work with
all integer types.
The Accessor was also moved from a shared header to a gmres_mixed
exclusive header.
---
 core/components/accessor.hpp             | 233 -------------
 core/solver/gmres_mixed.cpp              |  29 +-
 core/solver/gmres_mixed_accessor.hpp     | 400 +++++++++++++++++++++++
 core/solver/gmres_mixed_kernels.hpp      |   2 +-
 cuda/solver/gmres_mixed_kernels.cu       |  29 +-
 cuda/test/solver/gmres_mixed_kernels.cpp |  32 +-
 omp/test/solver/gmres_mixed_kernels.cpp  |  27 +-
 reference/solver/gmres_mixed_kernels.cpp |   2 +-
 8 files changed, 476 insertions(+), 278 deletions(-)
 delete mode 100644 core/components/accessor.hpp
 create mode 100644 core/solver/gmres_mixed_accessor.hpp

diff --git a/core/components/accessor.hpp b/core/components/accessor.hpp
deleted file mode 100644
index 9d3c5b05b3e..00000000000
--- a/core/components/accessor.hpp
+++ /dev/null
@@ -1,233 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GKO_CORE_COMMON_ACCESSOR_HPP_
-#define GKO_CORE_COMMON_ACCESSOR_HPP_
-
-
-#include <cinttypes>
-
-
-#include <ginkgo/core/base/types.hpp>
-
-
-namespace gko {
-
-
-/**
- * @internal
- *
- * The Accessor class hides the underlying storage_ format and provides a simple
- * interface for accessing a one dimensional storage.
- *
- * This class only manages the accesses, however, and not the memory itself.
- */
-template <typename StorageType, typename ArithmeticType = double>
-class Accessor {
-public:
-    using storage_type = StorageType;
-    using arithmetic_type = ArithmeticType;
-
-    /**
-     * Creates the accessor with an already allocated storage space.
-     */
-    Accessor(storage_type *storage) : storage_{storage} {}
-
-    /**
-     * Reads the value at the given index.
-     */
-    template <typename IndexType>
-    GKO_ATTRIBUTES arithmetic_type read(IndexType idx) const
-    {
-        // Make use of the restrict (and const) qualifier. If the restrict
-        // qualifier would have been put on the class attribute, it would be
-        // ignored by nvcc.
-        const storage_type *GKO_RESTRICT rest_storage = storage_;
-        return static_cast<arithmetic_type>(rest_storage[idx]);
-    }
-
-    /**
-     * Writes the given value at the given index.
-     */
-    template <typename IndexType>
-    GKO_ATTRIBUTES void write(IndexType idx, arithmetic_type value)
-    {
-        storage_type *GKO_RESTRICT rest_storage = storage_;
-        rest_storage[idx] = static_cast<storage_type>(value);
-    }
-
-private:
-    storage_type *storage_;
-};
-
-
-template <typename StorageType, typename ArithmeticType>
-class Accessor2d;
-
-
-/**
- * @internal
- *
- * The Accessor2d class hides the underlying storage_ format and provides a
- * simple interface for accessing a one dimensional storage.
- *
- * This class only manages the accesses, however, and not the memory itself.
- */
-template <typename StorageType, typename ArithmeticType = double>
-class Accessor2dConst {
-public:
-    using storage_type = StorageType;
-    using arithmetic_type = ArithmeticType;
-
-    /**
-     * Creates the accessor with an already allocated storage space with a
-     * stride.
-     */
-    Accessor2dConst(const storage_type *storage, size_type stride)
-        : storage_{storage}, stride_{stride}
-    {}
-
-    /**
-     * Creates a const accessor from a non-const accessor.
-     */
-    Accessor2dConst(Accessor2d<StorageType, ArithmeticType> acc)
-        : storage_{acc.get_const_storage()}, stride_{acc.get_stride()}
-    {}
-
-    /**
-     * Reads the value at the given index.
-     */
-    template <typename IndexType>
-    GKO_ATTRIBUTES arithmetic_type read(IndexType row, IndexType col) const
-    {
-        // Make use of the restrict (and const) qualifier. If the restrict
-        // qualifier would have been put on the class attribute, it would be
-        // ignored by nvcc.
-        const storage_type *GKO_RESTRICT rest_storage = storage_;
-        return static_cast<arithmetic_type>(rest_storage[row * stride_ + col]);
-    }
-
-    GKO_ATTRIBUTES size_type get_stride() const { return stride_; }
-
-    GKO_ATTRIBUTES const storage_type *get_storage() const { return storage_; }
-
-    GKO_ATTRIBUTES const storage_type *get_const_storage() const
-    {
-        return storage_;
-    }
-
-private:
-    const storage_type *storage_;
-    size_type stride_;
-};
-
-
-/**
- * @internal
- *
- * The Accessor2d class hides the underlying storage_ format and provides a
- * simple interface for accessing a one dimensional storage.
- *
- * This class only manages the accesses, however, and not the memory itself.
- */
-template <typename StorageType, typename ArithmeticType = double>
-class Accessor2d {
-public:
-    using storage_type = StorageType;
-    using arithmetic_type = ArithmeticType;
-
-    /**
-     * Creates an empty accessor pointing to a nullptr.
-     */
-    Accessor2d() : storage_{nullptr}, stride_{} {}
-
-    /**
-     * Creates the accessor with an already allocated storage space with a
-     * stride.
-     */
-    Accessor2d(storage_type *storage, size_type stride)
-        : storage_{storage}, stride_{stride}
-    {}
-
-    operator Accessor2dConst<storage_type, arithmetic_type>() const
-    {
-        return {storage_, stride_};
-    }
-
-    Accessor2dConst<storage_type, arithmetic_type> to_const() const
-    {
-        return {storage_, stride_};
-    }
-
-    /**
-     * Reads the value at the given index.
-     */
-    template <typename IndexType>
-    GKO_ATTRIBUTES arithmetic_type read(IndexType row, IndexType col) const
-    {
-        // Make use of the restrict (and const) qualifier. If the restrict
-        // qualifier would have been put on the class attribute, it would be
-        // ignored by nvcc.
-        const storage_type *GKO_RESTRICT rest_storage = storage_;
-        return static_cast<arithmetic_type>(rest_storage[row * stride_ + col]);
-    }
-
-    /**
-     * Writes the given value at the given index.
-     */
-    template <typename IndexType>
-    GKO_ATTRIBUTES void write(IndexType row, IndexType col,
-                              arithmetic_type value)
-    {
-        storage_type *GKO_RESTRICT rest_storage = storage_;
-        rest_storage[row * stride_ + col] = static_cast<storage_type>(value);
-    }
-
-    GKO_ATTRIBUTES size_type get_stride() const { return stride_; }
-
-    GKO_ATTRIBUTES storage_type *get_storage() { return storage_; }
-
-    GKO_ATTRIBUTES const storage_type *get_const_storage() const
-    {
-        return storage_;
-    }
-
-private:
-    storage_type *storage_;
-    size_type stride_;
-};
-
-
-}  // namespace gko
-
-
-#endif  // GKO_CORE_COMMON_ACCESSOR_HPP_
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 90b51968490..138a4052b9f 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -47,7 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <iostream>
 
 
-#include "core/components/accessor.hpp"
+#include "core/solver/gmres_mixed_accessor.hpp"
 #include "core/solver/gmres_mixed_kernels.hpp"
 
 
@@ -75,13 +75,7 @@ GKO_REGISTER_OPERATION(step_2, gmres_mixed::step_2);
 
 }  // namespace gmres_mixed
 
-/*
-template <typename ValueType, typename ValueTypeKrylovBases, bool
-Reorthogonalization, bool MGS_CGS> void
-GmresMixed<ValueType,ValueTypeKrylovBases,
-                Reorthogonalization,MGS_CGS>::apply_impl(const LinOp *b, LinOp
-*x) const
-*/
+
 template <typename ValueType, typename ValueTypeKrylovBases>
 void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
                                                              LinOp *x) const
@@ -90,7 +84,9 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
 
     using Vector = matrix::Dense<ValueType>;
     using LowArray = Array<ValueTypeKrylovBases>;
-    using KrylovAccessor = Accessor2d<ValueTypeKrylovBases, ValueType>;
+    using KrylovAccessor = kernels::Accessor2d<ValueTypeKrylovBases, ValueType>;
+    using Accessor2dHelper =
+        kernels::Accessor2dHelper<ValueType, ValueTypeKrylovBases>;
 
 
     constexpr uint8 RelativeStoppingId{1};
@@ -106,10 +102,13 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     const dim<2> krylov_bases_dim{
         system_matrix_->get_size()[1],
         (krylov_dim_mixed_ + 1) * dense_b->get_size()[1]};
-    const size_type krylov_bases_stride = krylov_bases_dim[1];
-    LowArray krylov_bases(exec, krylov_bases_dim[0] * krylov_bases_stride);
-    KrylovAccessor krylov_bases_accessor(krylov_bases.get_data(),
-                                         krylov_bases_stride);
+    // const size_type krylov_bases_stride = krylov_bases_dim[1];
+    // LowArray krylov_bases(exec, krylov_bases_dim[0] * krylov_bases_stride);
+    // KrylovAccessor krylov_bases_accessor(krylov_bases.get_data(),
+    //                                     krylov_bases_stride);
+    Accessor2dHelper helper(krylov_bases_dim, exec);
+    auto krylov_bases_accessor = helper.get_accessor();
+
     auto next_krylov_basis = Vector::create_with_config_of(dense_b);
     std::shared_ptr<matrix::Dense<ValueType>> preconditioned_vector =
         Vector::create_with_config_of(dense_b);
@@ -317,18 +316,20 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
         system_matrix_->get_size()[1],
         (krylov_dim_mixed_ + 1) * dense_b->get_size()[1]};
     */
+    /*
     const dim<2> krylov_bases_small_dim{
         system_matrix_->get_size()[0],
         dense_b->get_size()[1] * (restart_iter + 1)};
     KrylovAccessor krylov_bases_small_accessor{krylov_bases.get_data(),
                                                krylov_bases_stride};
+    */
 
     auto hessenberg_small = hessenberg->create_submatrix(
         span{0, restart_iter},
         span{0, dense_b->get_size()[1] * (restart_iter)});
 
     exec->run(gmres_mixed::make_step_2(
-        residual_norm_collection.get(), krylov_bases_small_accessor.to_const(),
+        residual_norm_collection.get(), krylov_bases_accessor.to_const(),
         hessenberg_small.get(), y.get(), before_preconditioner.get(),
         &final_iter_nums));
 #ifdef TIMING_STEPS
diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
new file mode 100644
index 00000000000..c97bd2a2810
--- /dev/null
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -0,0 +1,400 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_SOLVER_GMRES_MIXED_ACCESSOR_HPP_
+#define GKO_CORE_SOLVER_GMRES_MIXED_ACCESSOR_HPP_
+
+
+#include <cinttypes>
+#include <type_traits>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/std_extensions.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+namespace gko {
+namespace kernels {  // TODO maybe put into another separate namespace
+
+
+using place_holder_type = float;
+
+
+template <typename StorageType, typename ArithmeticType,
+          bool = !std::is_same<StorageType, place_holder_type>::value ||
+                 std::is_same<StorageType, ArithmeticType>::value>
+class Accessor2dConst {};
+/**
+ * @internal
+ *
+ * The Accessor2d class hides the underlying storage_ format and provides a
+ * simple interface for accessing a one dimensional storage.
+ *
+ * This class only manages the accesses, however, and not the memory itself.
+ */
+template <typename StorageType, typename ArithmeticType>
+class Accessor2dConst<StorageType, ArithmeticType, true> {
+public:
+    using storage_type = StorageType;
+    using arithmetic_type = ArithmeticType;
+    // static_assert(!std::is_integral<storage_type>::value,
+    //        "storage_type must not be an integral in this class.");
+
+    static constexpr bool has_scalar{false};
+
+    /**
+     * Creates the accessor with an already allocated storage space with a
+     * stride.
+     *
+     * @internal
+     * const_cast needed, so Accessor2dConst(ac.get_storage(), ac.get_stride())
+     * works. Also, storage_ is never accessed in a non-const fashion, so it
+     * is not invalid or UB code.
+     */
+    Accessor2dConst(const storage_type *storage, size_type stride)
+        : storage_{const_cast<storage_type *>(storage)}, stride_{stride}
+    {}
+
+    /**
+     * Reads the value at the given index.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES arithmetic_type read(IndexType row, IndexType col) const
+    {
+        // Make use of the restrict (and const) qualifier. If the restrict
+        // qualifier would have been put on the class attribute, it would be
+        // ignored by nvcc.
+        const storage_type *GKO_RESTRICT rest_storage = storage_;
+        return static_cast<arithmetic_type>(rest_storage[row * stride_ + col]);
+    }
+
+    GKO_ATTRIBUTES size_type get_stride() const { return stride_; }
+
+    GKO_ATTRIBUTES const storage_type *get_storage() const { return storage_; }
+
+    GKO_ATTRIBUTES const storage_type *get_const_storage() const
+    {
+        return storage_;
+    }
+
+protected:
+    storage_type *storage_;
+    size_type stride_;
+};
+
+
+/**
+ * @internal
+ *
+ * The Accessor2d class hides the underlying storage_ format and provides a
+ * simple interface for accessing a one dimensional storage.
+ * Additionally, since it is (soon) an integer type, a scale array is also
+ * needed to do a proper conversion.
+ *
+ * This class only manages the accesses, however, and not the memory itself.
+ */
+template <typename StorageType, typename ArithmeticType>
+class Accessor2dConst<StorageType, ArithmeticType, false> {
+public:
+    using storage_type = StorageType;
+    using arithmetic_type = ArithmeticType;
+    // static_assert(std::is_integral<storage_type>::value,
+    //        "storage_type must not be an integral in this class.");
+
+    static constexpr bool has_scalar{true};
+
+    /**
+     * Creates the accessor with an already allocated storage space with a
+     * stride.
+     *
+     * @internal
+     * const_cast needed, so Accessor2dConst(ac.get_storage(), ac.get_stride())
+     * works. Also, storage_ is never accessed in a non-const fashion, so it
+     * is not invalid or UB code.
+     */
+    Accessor2dConst(const storage_type *storage, size_type stride,
+                    const arithmetic_type *scale)
+        : storage_{const_cast<storage_type *>(storage)},
+          stride_{stride},
+          scale_{const_cast<arithmetic_type *>(scale)}
+    {}
+
+    /**
+     * Reads the value at the given index.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES arithmetic_type read(IndexType row, IndexType col) const
+    {
+        // Make use of the restrict (and const) qualifier. If the restrict
+        // qualifier would have been put on the class attribute, it would be
+        // ignored by nvcc.
+        const storage_type *GKO_RESTRICT rest_storage = storage_;
+        const arithmetic_type *GKO_RESTRICT rest_scale = scale_;
+        return static_cast<arithmetic_type>(rest_storage[row * stride_ + col]) *
+               rest_scale[row];
+    }
+
+    GKO_ATTRIBUTES size_type get_stride() const { return stride_; }
+
+    GKO_ATTRIBUTES const storage_type *get_storage() const { return storage_; }
+
+    GKO_ATTRIBUTES const storage_type *get_const_storage() const
+    {
+        return storage_;
+    }
+
+    GKO_ATTRIBUTES const arithmetic_type *get_scale() const
+    {
+        return this->scale_;
+    }
+
+    GKO_ATTRIBUTES const arithmetic_type *get_const_scale() const
+    {
+        return this->scale_;
+    }
+
+protected:
+    storage_type *storage_;
+    arithmetic_type *scale_;
+    size_type stride_;
+};
+
+
+template <typename StorageType, typename ArithmeticType,
+          bool = !std::is_same<StorageType, place_holder_type>::value ||
+                 std::is_same<StorageType, ArithmeticType>::value>
+class Accessor2d : public Accessor2dConst<StorageType, ArithmeticType> {};
+
+
+/**
+ * @internal
+ *
+ * The Accessor2d class hides the underlying storage_ format and provides a
+ * simple interface for accessing a one dimensional storage.
+ *
+ * This class only manages the accesses, however, and not the memory itself.
+ */
+template <typename StorageType, typename ArithmeticType>
+class Accessor2d<StorageType, ArithmeticType, true>
+    : public Accessor2dConst<StorageType, ArithmeticType> {
+public:
+    using storage_type = StorageType;
+    using arithmetic_type = ArithmeticType;
+    // static_assert(!std::is_integral<storage_type>::value,
+    //        "storage_type must not be an integral in this class.");
+
+private:
+    using Accessor2dC = Accessor2dConst<storage_type, arithmetic_type>;
+
+public:
+    /**
+     * Creates an empty accessor pointing to a nullptr.
+     */
+    Accessor2d() : Accessor2dC(nullptr, {}) {}
+
+    /**
+     * Creates the accessor with an already allocated storage space with a
+     * stride.
+     */
+    Accessor2d(storage_type *storage, size_type stride)
+        : Accessor2dC(storage, stride)
+    {}
+    /*
+        operator Accessor2dC() const
+        {
+            return {this->storage_, this->stride_};
+        }
+    */
+
+    Accessor2dC to_const() const { return {this->storage_, this->stride_}; }
+
+    /**
+     * Writes the given value at the given index.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES void write(IndexType row, IndexType col,
+                              arithmetic_type value)
+    {
+        storage_type *GKO_RESTRICT rest_storage = this->storage_;
+        rest_storage[row * this->stride_ + col] =
+            static_cast<storage_type>(value);
+    }
+
+    GKO_ATTRIBUTES storage_type *get_storage() { return this->storage_; }
+};
+
+
+/**
+ * @internal
+ *
+ * The Accessor2d class hides the underlying storage_ format and provides a
+ * simple interface for accessing a one dimensional storage.
+ *
+ * This class only manages the accesses, however, and not the memory itself.
+ */
+template <typename StorageType, typename ArithmeticType>
+class Accessor2d<StorageType, ArithmeticType, false>
+    : public Accessor2dConst<StorageType, ArithmeticType> {
+public:
+    using storage_type = StorageType;
+    using arithmetic_type = ArithmeticType;
+    // static_assert(std::is_integral<storage_type>::value,
+    //        "storage_type must not be an integral in this class.");
+
+private:
+    using Accessor2dC = Accessor2dConst<storage_type, arithmetic_type>;
+
+public:
+    /**
+     * Creates an empty accessor pointing to a nullptr.
+     */
+    Accessor2d() : Accessor2dC(nullptr, {}, nullptr) {}
+
+    /**
+     * Creates the accessor with an already allocated storage space with a
+     * stride.
+     */
+    Accessor2d(storage_type *storage, size_type stride, arithmetic_type *scale)
+        : Accessor2dC(storage, stride, scale)
+    {}
+
+    Accessor2dC to_const() const
+    {
+        return {this->storage_, this->stride_, this->scale_};
+    }
+
+    /**
+     * Writes the given value at the given index.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES void write(IndexType row, IndexType col,
+                              arithmetic_type value)
+    {
+        storage_type *GKO_RESTRICT rest_storage = this->storage_;
+        const arithmetic_type *GKO_RESTRICT rest_scale = this->scale_;
+        rest_storage[row * this->stride_ + col] =
+            static_cast<storage_type>(value / rest_scale[row]);
+    }
+
+    /**
+     * Writes the given value at the given index for a scale.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES void set_scale(IndexType idx, arithmetic_type val)
+    {
+        arithmetic_type *GKO_RESTRICT rest_scale = this->scale_;
+        rest_scale[idx] = val;
+    }
+
+    GKO_ATTRIBUTES storage_type *get_storage() { return this->storage_; }
+
+    GKO_ATTRIBUTES arithmetic_type *get_scale() { return this->scale_; }
+};
+
+
+template <typename ValueType, typename ValueTypeKrylovBases,
+          bool = Accessor2d<ValueTypeKrylovBases, ValueType>::has_scalar>
+class Accessor2dHelper {};
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+class Accessor2dHelper<ValueType, ValueTypeKrylovBases, true> {
+    using Accessor = Accessor2d<ValueTypeKrylovBases, ValueType>;
+    static_assert(Accessor::has_scalar,
+                  "This accessor must have a scalar in this class!");
+
+public:
+    Accessor2dHelper() = default;
+
+    Accessor2dHelper(dim<2> krylov_dim, std::shared_ptr<const Executor> exec)
+        : krylov_dim_{krylov_dim},
+          bases_stride_{krylov_dim_[1]},
+          bases_{exec, krylov_dim_[0] * bases_stride_},
+          scalar_{exec, krylov_dim_[0]}
+    {
+        // For testing, initialize scalar to ones
+        Array<ValueType> h_scalar{exec->get_master(), krylov_dim[0]};
+        for (size_type i = 0; i < h_scalar.get_num_elems(); ++i) {
+            h_scalar.get_data()[i] = one<ValueType>();
+        }
+        scalar_ = h_scalar;
+    }
+
+    Accessor get_accessor()
+    {
+        return {bases_.get_data(), bases_stride_, scalar_.get_data()};
+    }
+
+    gko::Array<ValueTypeKrylovBases> &get_bases() { return bases_; }
+
+private:
+    dim<2> krylov_dim_;
+    size_type bases_stride_;
+    Array<ValueTypeKrylovBases> bases_;
+    Array<ValueType> scalar_;
+};
+
+
+template <typename ValueType, typename ValueTypeKrylovBases>
+class Accessor2dHelper<ValueType, ValueTypeKrylovBases, false> {
+    using Accessor = Accessor2d<ValueTypeKrylovBases, ValueType>;
+    static_assert(!Accessor::has_scalar,
+                  "This accessor must not have a scalar in this class!");
+
+public:
+    Accessor2dHelper() = default;
+
+    Accessor2dHelper(dim<2> krylov_dim, std::shared_ptr<const Executor> exec)
+        : krylov_dim_{krylov_dim},
+          bases_stride_{krylov_dim_[1]},
+          bases_{exec, krylov_dim_[0] * bases_stride_}
+    {}
+
+    Accessor get_accessor() { return {bases_.get_data(), bases_stride_}; }
+
+    gko::Array<ValueTypeKrylovBases> &get_bases() { return bases_; }
+
+private:
+    dim<2> krylov_dim_;
+    size_type bases_stride_;
+    Array<ValueTypeKrylovBases> bases_;
+};
+
+
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_CORE_SOLVER_GMRES_MIXED_ACCESSOR_HPP_
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index bae812b9c70..eda1bf1728a 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -41,7 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/stopping_status.hpp>
 
 
-#include "core/components/accessor.hpp"
+#include "core/solver/gmres_mixed_accessor.hpp"
 
 
 /**
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index f3fdf02578d..4ecf46eb038 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/range_accessors.hpp>
+#include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
@@ -91,19 +92,39 @@ constexpr int default_update_dim = 32;
 
 // Specialization, so the Accessor can use the same function as regular pointers
 template <typename Type1, typename Type2>
-Accessor2d<cuda_type<Type1>, cuda_type<Type2>> as_cuda_accessor(
-    Accessor2d<Type1, Type2> acc)
+xstd::enable_if_t<!Accessor2d<Type1, Type2>::has_scalar,
+                  Accessor2d<cuda_type<Type1>, cuda_type<Type2>>>
+as_cuda_accessor(Accessor2d<Type1, Type2> acc)
 {
     return {as_cuda_type(acc.get_storage()), acc.get_stride()};
 }
 
 template <typename Type1, typename Type2>
-Accessor2dConst<cuda_type<Type1>, cuda_type<Type2>> as_cuda_accessor(
-    const Accessor2dConst<Type1, Type2> &acc)
+xstd::enable_if_t<Accessor2d<Type1, Type2>::has_scalar,
+                  Accessor2d<cuda_type<Type1>, cuda_type<Type2>>>
+as_cuda_accessor(Accessor2d<Type1, Type2> acc)
+{
+    return {as_cuda_type(acc.get_storage()), acc.get_stride(),
+            as_cuda_type(acc.get_scale())};
+}
+
+template <typename Type1, typename Type2>
+xstd::enable_if_t<!Accessor2dConst<Type1, Type2>::has_scalar,
+                  Accessor2dConst<cuda_type<Type1>, cuda_type<Type2>>>
+as_cuda_accessor(const Accessor2dConst<Type1, Type2> &acc)
 {
     return {as_cuda_type(acc.get_storage()), acc.get_stride()};
 }
 
+template <typename Type1, typename Type2>
+xstd::enable_if_t<Accessor2dConst<Type1, Type2>::has_scalar,
+                  Accessor2dConst<cuda_type<Type1>, cuda_type<Type2>>>
+as_cuda_accessor(const Accessor2dConst<Type1, Type2> &acc)
+{
+    return {as_cuda_type(acc.get_storage()), acc.get_stride(),
+            as_cuda_type(acc.get_scale())};
+}
+
 
 template <typename ValueType>
 void zero_matrix(size_type m, size_type n, size_type stride, ValueType *array)
diff --git a/cuda/test/solver/gmres_mixed_kernels.cpp b/cuda/test/solver/gmres_mixed_kernels.cpp
index 950d0d5ca8f..d106f255003 100644
--- a/cuda/test/solver/gmres_mixed_kernels.cpp
+++ b/cuda/test/solver/gmres_mixed_kernels.cpp
@@ -32,10 +32,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/solver/gmres_mixed.hpp>
 
+
 #include <random>
 
+
 #include <gtest/gtest.h>
 
+
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
@@ -43,17 +46,23 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm_reduction.hpp>
 
+
+#include "core/solver/gmres_mixed_accessor.hpp"
 #include "core/solver/gmres_mixed_kernels.hpp"
 #include "core/test/utils.hpp"
 
+
 namespace {
 
+
 class GmresMixed : public ::testing::Test {
 protected:
     using value_type = double;
     using krylov_type = float;
     using index_type = int;
-    using Accessor2d = gko::Accessor2d<krylov_type, value_type>;
+    using Accessor2d = gko::kernels::Accessor2d<krylov_type, value_type>;
+    using Accessor2dHelper =
+        gko::kernels::Accessor2dHelper<value_type, krylov_type>;
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
 
@@ -64,8 +73,6 @@ class GmresMixed : public ::testing::Test {
         ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
         ref = gko::ReferenceExecutor::create();
         cuda = gko::CudaExecutor::create(0, ref);
-        krylov_bases.set_executor(ref);
-        d_krylov_bases.set_executor(cuda);
     }
 
     void TearDown()
@@ -108,10 +115,10 @@ class GmresMixed : public ::testing::Test {
         arnoldi_norm = gen_mtx(2, n);
         gko::dim<2> krylov_bases_dim(
             m, (gko::solver::default_krylov_dim_mixed + 1) * n);
-        krylov_bases =
+        acc_helper = Accessor2dHelper{krylov_bases_dim, ref};
+        acc_helper.get_bases() =
             generate_krylov_bases(krylov_bases_dim[0], krylov_bases_dim[1]);
-        krylov_bases_accessor =
-            Accessor2d{krylov_bases.get_data(), krylov_bases_dim[1]};
+        krylov_bases_accessor = acc_helper.get_accessor();
         next_krylov_basis = gen_mtx(m, n);
         hessenberg = gen_mtx(gko::solver::default_krylov_dim_mixed + 1,
                              gko::solver::default_krylov_dim_mixed * n);
@@ -155,9 +162,9 @@ class GmresMixed : public ::testing::Test {
         d_b_norm->copy_from(b_norm.get());
         d_arnoldi_norm = Mtx::create(cuda);
         d_arnoldi_norm->copy_from(arnoldi_norm.get());
-        d_krylov_bases = krylov_bases;
-        d_krylov_bases_accessor =
-            Accessor2d{d_krylov_bases.get_data(), krylov_bases_dim[1]};
+        d_acc_helper = Accessor2dHelper{{}, cuda};
+        d_acc_helper = acc_helper;
+        d_krylov_bases_accessor = d_acc_helper.get_accessor();
         d_next_krylov_basis = Mtx::create(cuda);
         d_next_krylov_basis->copy_from(next_krylov_basis.get());
         d_hessenberg = Mtx::create(cuda);
@@ -193,7 +200,8 @@ class GmresMixed : public ::testing::Test {
     void assert_krylov_bases_near()
     {
         gko::Array<krylov_type> d_to_host{ref};
-        d_to_host = d_krylov_bases;
+        auto &krylov_bases = acc_helper.get_bases();
+        d_to_host = d_acc_helper.get_bases();
         const auto tolerance = r<krylov_type>::value;
         using std::abs;
         for (gko::size_type i = 0; i < krylov_bases.get_num_elems(); ++i) {
@@ -214,7 +222,7 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> b_norm;
     std::unique_ptr<Mtx> arnoldi_norm;
-    gko::Array<krylov_type> krylov_bases;
+    Accessor2dHelper acc_helper;
     Accessor2d krylov_bases_accessor;
     std::unique_ptr<Mtx> next_krylov_basis;
     std::unique_ptr<Mtx> hessenberg;
@@ -236,7 +244,7 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> d_b;
     std::unique_ptr<Mtx> d_b_norm;
     std::unique_ptr<Mtx> d_arnoldi_norm;
-    gko::Array<krylov_type> d_krylov_bases;
+    Accessor2dHelper d_acc_helper;
     Accessor2d d_krylov_bases_accessor;
     std::unique_ptr<Mtx> d_next_krylov_basis;
     std::unique_ptr<Mtx> d_hessenberg;
diff --git a/omp/test/solver/gmres_mixed_kernels.cpp b/omp/test/solver/gmres_mixed_kernels.cpp
index 6f7913b71aa..8bd83e6c494 100644
--- a/omp/test/solver/gmres_mixed_kernels.cpp
+++ b/omp/test/solver/gmres_mixed_kernels.cpp
@@ -50,7 +50,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/residual_norm_reduction.hpp>
 
 
-#include "core/components/accessor.hpp"
+#include "core/solver/gmres_mixed_accessor.hpp"
 #include "core/test/utils.hpp"
 
 
@@ -62,7 +62,9 @@ class GmresMixed : public ::testing::Test {
     using value_type = double;
     using krylov_type = float;
     using index_type = int;
-    using Accessor2d = gko::Accessor2d<krylov_type, value_type>;
+    using Accessor2d = gko::kernels::Accessor2d<krylov_type, value_type>;
+    using Accessor2dHelper =
+        gko::kernels::Accessor2dHelper<value_type, krylov_type>;
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
 
@@ -73,8 +75,6 @@ class GmresMixed : public ::testing::Test {
     {
         ref = gko::ReferenceExecutor::create();
         omp = gko::OmpExecutor::create();
-        krylov_bases.set_executor(ref);
-        d_krylov_bases.set_executor(omp);
     }
 
     void TearDown()
@@ -117,10 +117,10 @@ class GmresMixed : public ::testing::Test {
         arnoldi_norm = gen_mtx(2, n);
         gko::dim<2> krylov_bases_dim(
             m, (gko::solver::default_krylov_dim_mixed + 1) * n);
-        krylov_bases =
+        acc_helper = Accessor2dHelper{krylov_bases_dim, ref};
+        acc_helper.get_bases() =
             generate_krylov_bases(krylov_bases_dim[0], krylov_bases_dim[1]);
-        krylov_bases_accessor =
-            Accessor2d{krylov_bases.get_data(), krylov_bases_dim[1]};
+        krylov_bases_accessor = acc_helper.get_accessor();
 
         next_krylov_basis = gen_mtx(m, n);
         hessenberg = gen_mtx(gko::solver::default_krylov_dim_mixed + 1,
@@ -165,9 +165,9 @@ class GmresMixed : public ::testing::Test {
         d_b_norm->copy_from(b_norm.get());
         d_arnoldi_norm = Mtx::create(omp);
         d_arnoldi_norm->copy_from(arnoldi_norm.get());
-        d_krylov_bases = krylov_bases;
-        d_krylov_bases_accessor =
-            Accessor2d{d_krylov_bases.get_data(), krylov_bases_dim[1]};
+        d_acc_helper = Accessor2dHelper{{}, omp};
+        d_acc_helper = acc_helper;
+        d_krylov_bases_accessor = d_acc_helper.get_accessor();
         d_next_krylov_basis = Mtx::create(omp);
         d_next_krylov_basis->copy_from(next_krylov_basis.get());
         d_hessenberg = Mtx::create(omp);
@@ -202,7 +202,8 @@ class GmresMixed : public ::testing::Test {
     void assert_krylov_bases_near()
     {
         gko::Array<krylov_type> d_to_host{ref};
-        d_to_host = d_krylov_bases;
+        auto &krylov_bases = acc_helper.get_bases();
+        d_to_host = d_acc_helper.get_bases();
         const auto tolerance = r<krylov_type>::value;
         using std::abs;
         for (gko::size_type i = 0; i < krylov_bases.get_num_elems(); ++i) {
@@ -223,7 +224,7 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> b_norm;
     std::unique_ptr<Mtx> arnoldi_norm;
-    gko::Array<krylov_type> krylov_bases;
+    Accessor2dHelper acc_helper;
     Accessor2d krylov_bases_accessor;
     std::unique_ptr<Mtx> next_krylov_basis;
     std::unique_ptr<Mtx> hessenberg;
@@ -245,7 +246,7 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> d_b;
     std::unique_ptr<Mtx> d_b_norm;
     std::unique_ptr<Mtx> d_arnoldi_norm;
-    gko::Array<krylov_type> d_krylov_bases;
+    Accessor2dHelper d_acc_helper;
     Accessor2d d_krylov_bases_accessor;
     std::unique_ptr<Mtx> d_next_krylov_basis;
     std::unique_ptr<Mtx> d_hessenberg;
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index 2f0599f0ba9..4f95b4a755f 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -41,7 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <iostream>
 
-#include "core/components/accessor.hpp"
+#include "core/solver/gmres_mixed_accessor.hpp"
 
 
 namespace gko {

From 9f44aef88b967c7c21fa20948012467f548fef05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 20 May 2020 12:17:24 +0200
Subject: [PATCH 0347/1008] Make the scale work with integer types

Also did some name changes.
---
 core/solver/gmres_mixed_accessor.hpp | 76 ++++++++++++++--------------
 cuda/solver/gmres_mixed_kernels.cu   |  8 +--
 2 files changed, 41 insertions(+), 43 deletions(-)

diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index c97bd2a2810..794f36bef4a 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -49,13 +49,10 @@ namespace gko {
 namespace kernels {  // TODO maybe put into another separate namespace
 
 
-using place_holder_type = float;
-
-
 template <typename StorageType, typename ArithmeticType,
-          bool = !std::is_same<StorageType, place_holder_type>::value ||
-                 std::is_same<StorageType, ArithmeticType>::value>
-class Accessor2dConst {};
+          bool = std::is_integral<StorageType>::value>
+class Accessor2dConst {
+};
 /**
  * @internal
  *
@@ -65,14 +62,14 @@ class Accessor2dConst {};
  * This class only manages the accesses, however, and not the memory itself.
  */
 template <typename StorageType, typename ArithmeticType>
-class Accessor2dConst<StorageType, ArithmeticType, true> {
+class Accessor2dConst<StorageType, ArithmeticType, false> {
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
-    // static_assert(!std::is_integral<storage_type>::value,
-    //        "storage_type must not be an integral in this class.");
+    static_assert(!std::is_integral<storage_type>::value,
+                  "storage_type must not be an integral in this class.");
 
-    static constexpr bool has_scalar{false};
+    static constexpr bool has_scale{false};
 
     /**
      * Creates the accessor with an already allocated storage space with a
@@ -126,14 +123,14 @@ class Accessor2dConst<StorageType, ArithmeticType, true> {
  * This class only manages the accesses, however, and not the memory itself.
  */
 template <typename StorageType, typename ArithmeticType>
-class Accessor2dConst<StorageType, ArithmeticType, false> {
+class Accessor2dConst<StorageType, ArithmeticType, true> {
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
-    // static_assert(std::is_integral<storage_type>::value,
-    //        "storage_type must not be an integral in this class.");
+    static_assert(std::is_integral<storage_type>::value,
+                  "storage_type must not be an integral in this class.");
 
-    static constexpr bool has_scalar{true};
+    static constexpr bool has_scale{true};
 
     /**
      * Creates the accessor with an already allocated storage space with a
@@ -163,7 +160,7 @@ class Accessor2dConst<StorageType, ArithmeticType, false> {
         const storage_type *GKO_RESTRICT rest_storage = storage_;
         const arithmetic_type *GKO_RESTRICT rest_scale = scale_;
         return static_cast<arithmetic_type>(rest_storage[row * stride_ + col]) *
-               rest_scale[row];
+               rest_scale[col];
     }
 
     GKO_ATTRIBUTES size_type get_stride() const { return stride_; }
@@ -193,9 +190,9 @@ class Accessor2dConst<StorageType, ArithmeticType, false> {
 
 
 template <typename StorageType, typename ArithmeticType,
-          bool = !std::is_same<StorageType, place_holder_type>::value ||
-                 std::is_same<StorageType, ArithmeticType>::value>
-class Accessor2d : public Accessor2dConst<StorageType, ArithmeticType> {};
+          bool = std::is_integral<StorageType>::value>
+class Accessor2d : public Accessor2dConst<StorageType, ArithmeticType> {
+};
 
 
 /**
@@ -207,13 +204,13 @@ class Accessor2d : public Accessor2dConst<StorageType, ArithmeticType> {};
  * This class only manages the accesses, however, and not the memory itself.
  */
 template <typename StorageType, typename ArithmeticType>
-class Accessor2d<StorageType, ArithmeticType, true>
+class Accessor2d<StorageType, ArithmeticType, false>
     : public Accessor2dConst<StorageType, ArithmeticType> {
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
-    // static_assert(!std::is_integral<storage_type>::value,
-    //        "storage_type must not be an integral in this class.");
+    static_assert(!std::is_integral<storage_type>::value,
+                  "storage_type must not be an integral in this class.");
 
 private:
     using Accessor2dC = Accessor2dConst<storage_type, arithmetic_type>;
@@ -265,13 +262,13 @@ class Accessor2d<StorageType, ArithmeticType, true>
  * This class only manages the accesses, however, and not the memory itself.
  */
 template <typename StorageType, typename ArithmeticType>
-class Accessor2d<StorageType, ArithmeticType, false>
+class Accessor2d<StorageType, ArithmeticType, true>
     : public Accessor2dConst<StorageType, ArithmeticType> {
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
-    // static_assert(std::is_integral<storage_type>::value,
-    //        "storage_type must not be an integral in this class.");
+    static_assert(std::is_integral<storage_type>::value,
+                  "storage_type must not be an integral in this class.");
 
 private:
     using Accessor2dC = Accessor2dConst<storage_type, arithmetic_type>;
@@ -305,7 +302,7 @@ class Accessor2d<StorageType, ArithmeticType, false>
         storage_type *GKO_RESTRICT rest_storage = this->storage_;
         const arithmetic_type *GKO_RESTRICT rest_scale = this->scale_;
         rest_storage[row * this->stride_ + col] =
-            static_cast<storage_type>(value / rest_scale[row]);
+            static_cast<storage_type>(value / rest_scale[col]);
     }
 
     /**
@@ -325,15 +322,16 @@ class Accessor2d<StorageType, ArithmeticType, false>
 
 
 template <typename ValueType, typename ValueTypeKrylovBases,
-          bool = Accessor2d<ValueTypeKrylovBases, ValueType>::has_scalar>
-class Accessor2dHelper {};
+          bool = Accessor2d<ValueTypeKrylovBases, ValueType>::has_scale>
+class Accessor2dHelper {
+};
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
 class Accessor2dHelper<ValueType, ValueTypeKrylovBases, true> {
     using Accessor = Accessor2d<ValueTypeKrylovBases, ValueType>;
-    static_assert(Accessor::has_scalar,
-                  "This accessor must have a scalar in this class!");
+    static_assert(Accessor::has_scale,
+                  "This accessor must have a scale in this class!");
 
 public:
     Accessor2dHelper() = default;
@@ -342,19 +340,19 @@ class Accessor2dHelper<ValueType, ValueTypeKrylovBases, true> {
         : krylov_dim_{krylov_dim},
           bases_stride_{krylov_dim_[1]},
           bases_{exec, krylov_dim_[0] * bases_stride_},
-          scalar_{exec, krylov_dim_[0]}
+          scale_{exec, krylov_dim_[1]}
     {
-        // For testing, initialize scalar to ones
-        Array<ValueType> h_scalar{exec->get_master(), krylov_dim[0]};
-        for (size_type i = 0; i < h_scalar.get_num_elems(); ++i) {
-            h_scalar.get_data()[i] = one<ValueType>();
+        // For testing, initialize scale to ones
+        Array<ValueType> h_scale{exec->get_master(), krylov_dim[0]};
+        for (size_type i = 0; i < h_scale.get_num_elems(); ++i) {
+            h_scale.get_data()[i] = one<ValueType>();
         }
-        scalar_ = h_scalar;
+        scale_ = h_scale;
     }
 
     Accessor get_accessor()
     {
-        return {bases_.get_data(), bases_stride_, scalar_.get_data()};
+        return {bases_.get_data(), bases_stride_, scale_.get_data()};
     }
 
     gko::Array<ValueTypeKrylovBases> &get_bases() { return bases_; }
@@ -363,15 +361,15 @@ class Accessor2dHelper<ValueType, ValueTypeKrylovBases, true> {
     dim<2> krylov_dim_;
     size_type bases_stride_;
     Array<ValueTypeKrylovBases> bases_;
-    Array<ValueType> scalar_;
+    Array<ValueType> scale_;
 };
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
 class Accessor2dHelper<ValueType, ValueTypeKrylovBases, false> {
     using Accessor = Accessor2d<ValueTypeKrylovBases, ValueType>;
-    static_assert(!Accessor::has_scalar,
-                  "This accessor must not have a scalar in this class!");
+    static_assert(!Accessor::has_scale,
+                  "This accessor must not have a scale in this class!");
 
 public:
     Accessor2dHelper() = default;
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 4ecf46eb038..fcfb47cfb63 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -92,7 +92,7 @@ constexpr int default_update_dim = 32;
 
 // Specialization, so the Accessor can use the same function as regular pointers
 template <typename Type1, typename Type2>
-xstd::enable_if_t<!Accessor2d<Type1, Type2>::has_scalar,
+xstd::enable_if_t<!Accessor2d<Type1, Type2>::has_scale,
                   Accessor2d<cuda_type<Type1>, cuda_type<Type2>>>
 as_cuda_accessor(Accessor2d<Type1, Type2> acc)
 {
@@ -100,7 +100,7 @@ as_cuda_accessor(Accessor2d<Type1, Type2> acc)
 }
 
 template <typename Type1, typename Type2>
-xstd::enable_if_t<Accessor2d<Type1, Type2>::has_scalar,
+xstd::enable_if_t<Accessor2d<Type1, Type2>::has_scale,
                   Accessor2d<cuda_type<Type1>, cuda_type<Type2>>>
 as_cuda_accessor(Accessor2d<Type1, Type2> acc)
 {
@@ -109,7 +109,7 @@ as_cuda_accessor(Accessor2d<Type1, Type2> acc)
 }
 
 template <typename Type1, typename Type2>
-xstd::enable_if_t<!Accessor2dConst<Type1, Type2>::has_scalar,
+xstd::enable_if_t<!Accessor2dConst<Type1, Type2>::has_scale,
                   Accessor2dConst<cuda_type<Type1>, cuda_type<Type2>>>
 as_cuda_accessor(const Accessor2dConst<Type1, Type2> &acc)
 {
@@ -117,7 +117,7 @@ as_cuda_accessor(const Accessor2dConst<Type1, Type2> &acc)
 }
 
 template <typename Type1, typename Type2>
-xstd::enable_if_t<Accessor2dConst<Type1, Type2>::has_scalar,
+xstd::enable_if_t<Accessor2dConst<Type1, Type2>::has_scale,
                   Accessor2dConst<cuda_type<Type1>, cuda_type<Type2>>>
 as_cuda_accessor(const Accessor2dConst<Type1, Type2> &acc)
 {

From 427ac8a57c9c9b00d2ea6daab00ee639070dacf4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 20 May 2020 12:48:05 +0200
Subject: [PATCH 0348/1008] Add helper to determine if we need a scale or not

---
 core/solver/gmres_mixed_accessor.hpp | 44 +++++++++++++++++++++++-----
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index 794f36bef4a..1af7a1716cc 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -49,8 +49,34 @@ namespace gko {
 namespace kernels {  // TODO maybe put into another separate namespace
 
 
+namespace detail {
+
+
+using place_holder_type = float;
+
+
+template <typename StorageType, typename ArithmeticType,
+          bool = std::is_same<StorageType, place_holder_type>::value &&
+                 !std::is_same<StorageType, ArithmeticType>::value>
+struct helper_have_scale {
+};
+
+template <typename StorageType, typename ArithmeticType>
+struct helper_have_scale<StorageType, ArithmeticType, false>
+    : public std::false_type {
+};
+
+template <typename StorageType, typename ArithmeticType>
+struct helper_have_scale<StorageType, ArithmeticType, true>
+    : public std::true_type {
+};
+
+
+}  // namespace detail
+
+
 template <typename StorageType, typename ArithmeticType,
-          bool = std::is_integral<StorageType>::value>
+          bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
 class Accessor2dConst {
 };
 /**
@@ -66,8 +92,9 @@ class Accessor2dConst<StorageType, ArithmeticType, false> {
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
-    static_assert(!std::is_integral<storage_type>::value,
-                  "storage_type must not be an integral in this class.");
+    static_assert(
+        !detail::helper_have_scale<StorageType, ArithmeticType>::value,
+        "storage_type must not be an integral in this class.");
 
     static constexpr bool has_scale{false};
 
@@ -127,7 +154,7 @@ class Accessor2dConst<StorageType, ArithmeticType, true> {
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
-    static_assert(std::is_integral<storage_type>::value,
+    static_assert(detail::helper_have_scale<StorageType, ArithmeticType>::value,
                   "storage_type must not be an integral in this class.");
 
     static constexpr bool has_scale{true};
@@ -190,7 +217,7 @@ class Accessor2dConst<StorageType, ArithmeticType, true> {
 
 
 template <typename StorageType, typename ArithmeticType,
-          bool = std::is_integral<StorageType>::value>
+          bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
 class Accessor2d : public Accessor2dConst<StorageType, ArithmeticType> {
 };
 
@@ -209,8 +236,9 @@ class Accessor2d<StorageType, ArithmeticType, false>
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
-    static_assert(!std::is_integral<storage_type>::value,
-                  "storage_type must not be an integral in this class.");
+    static_assert(
+        !detail::helper_have_scale<StorageType, ArithmeticType>::value,
+        "storage_type must not be an integral in this class.");
 
 private:
     using Accessor2dC = Accessor2dConst<storage_type, arithmetic_type>;
@@ -267,7 +295,7 @@ class Accessor2d<StorageType, ArithmeticType, true>
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
-    static_assert(std::is_integral<storage_type>::value,
+    static_assert(detail::helper_have_scale<StorageType, ArithmeticType>::value,
                   "storage_type must not be an integral in this class.");
 
 private:

From 4568bb3b89603614de5cd51c905bc22139c1c4d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 20 May 2020 13:19:12 +0200
Subject: [PATCH 0349/1008] Add a helper structure to manage the scale writing
 in common

---
 common/solver/gmres_mixed_kernels.hpp.inc | 32 +++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index fda76c391dd..8258ed38b3e 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -45,6 +45,38 @@ __global__ __launch_bounds__(default_block_size) void zero_matrix_kernel(
 }
 
 
+template <typename ValueType, typename KrylovType,
+          bool = Accessor2d<KrylovType, ValueType>::has_scale>
+struct helper_functions_accessor {};
+
+// Accessors having a scale
+template <typename ValueType, typename KrylovType>
+struct helper_functions_accessor<ValueType, KrylovType, true> {
+    template <typename IndexType>
+    static __forceinline__ __device__ void write_scale(
+        Accessor2d<KrylovType, ValueType> krylov_bases, IndexType col_idx,
+        ValueType value)
+    {
+        krylov_bases.set_scale(col_idx, value);
+    }
+};
+
+
+// Accessors not having a scale
+template <typename ValueType, typename KrylovType>
+struct helper_functions_accessor<ValueType, KrylovType, false> {
+    template <typename IndexType>
+    static __forceinline__ __device__ void write_scale(
+        Accessor2d<KrylovType, ValueType> krylov_bases, IndexType col_idx,
+        ValueType value)
+    {}
+};
+
+// calling it with:
+// helper_functions_accessor<ValueType,
+// ValueTypeKrylovBases>::write_scale(krylov_bases, col_idx, value);
+
+
 // Must be called with at least `max(stride_b * num_rows, krylov_dim *
 // num_cols)` threads in total.
 template <size_type block_size, typename ValueType>

From d26fe71034203a94c49cd755407415df7b88c159 Mon Sep 17 00:00:00 2001
From: josealiaga <aliaga@uji.es>
Date: Wed, 20 May 2020 13:54:31 +0200
Subject: [PATCH 0350/1008] Testing the push command

---
 core/solver/gmres_mixed_accessor.hpp | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index 1af7a1716cc..4727b9926ff 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -58,18 +58,15 @@ using place_holder_type = float;
 template <typename StorageType, typename ArithmeticType,
           bool = std::is_same<StorageType, place_holder_type>::value &&
                  !std::is_same<StorageType, ArithmeticType>::value>
-struct helper_have_scale {
-};
+struct helper_have_scale {};
 
 template <typename StorageType, typename ArithmeticType>
 struct helper_have_scale<StorageType, ArithmeticType, false>
-    : public std::false_type {
-};
+    : public std::false_type {};
 
 template <typename StorageType, typename ArithmeticType>
 struct helper_have_scale<StorageType, ArithmeticType, true>
-    : public std::true_type {
-};
+    : public std::true_type {};
 
 
 }  // namespace detail
@@ -77,8 +74,7 @@ struct helper_have_scale<StorageType, ArithmeticType, true>
 
 template <typename StorageType, typename ArithmeticType,
           bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
-class Accessor2dConst {
-};
+class Accessor2dConst {};
 /**
  * @internal
  *
@@ -218,8 +214,7 @@ class Accessor2dConst<StorageType, ArithmeticType, true> {
 
 template <typename StorageType, typename ArithmeticType,
           bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
-class Accessor2d : public Accessor2dConst<StorageType, ArithmeticType> {
-};
+class Accessor2d : public Accessor2dConst<StorageType, ArithmeticType> {};
 
 
 /**
@@ -351,8 +346,7 @@ class Accessor2d<StorageType, ArithmeticType, true>
 
 template <typename ValueType, typename ValueTypeKrylovBases,
           bool = Accessor2d<ValueTypeKrylovBases, ValueType>::has_scale>
-class Accessor2dHelper {
-};
+class Accessor2dHelper {};
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>

From 2cbc645b70bf9059c84efe82c42ee8480fbb8cd9 Mon Sep 17 00:00:00 2001
From: josealiaga <aliaga@uji.es>
Date: Fri, 22 May 2020 13:41:19 +0200
Subject: [PATCH 0351/1008] Definition of norm2 and norminf routines in CUDA.
 Only the first one is used because the second one needs the correct
 management of norms as noncomplex values.

The next step will be to use remove_complex for norms, and then, to use norminf.
---
 common/components/atomic.hpp.inc          | 81 +++++++++++++++++++
 common/solver/gmres_mixed_kernels.hpp.inc | 96 +++++++++++++++++++++++
 core/solver/gmres_mixed.cpp               |  2 +-
 cuda/solver/gmres_mixed_kernels.cu        | 39 ++++++++-
 4 files changed, 215 insertions(+), 3 deletions(-)

diff --git a/common/components/atomic.hpp.inc b/common/components/atomic.hpp.inc
index f1b3f7613f2..0e7418db0b3 100644
--- a/common/components/atomic.hpp.inc
+++ b/common/components/atomic.hpp.inc
@@ -44,6 +44,14 @@ struct atomic_helper {
                       "specializations are.");
         // TODO: add proper implementation of generic atomic add
     }
+    __forceinline__ __device__ static ValueType atomic_max(ValueType *,
+                                                           ValueType)
+    {
+        static_assert(sizeof(ValueType) == 0,
+                      "This default function is not implemented, only the "
+                      "specializations are.");
+        // TODO: add proper implementation of generic atomic add
+    }
 };
 
 
@@ -77,6 +85,22 @@ __forceinline__ __device__ ResultType reinterpret(ValueType val)
             } while (assumed != old);                                        \
             return reinterpret<ValueType>(old);                              \
         }                                                                    \
+        __forceinline__ __device__ static ValueType atomic_max(              \
+            ValueType *__restrict__ addr, ValueType val)                     \
+        {                                                                    \
+            CONVERTER_TYPE *address_as_converter =                           \
+                reinterpret_cast<CONVERTER_TYPE *>(addr);                    \
+            CONVERTER_TYPE old = *address_as_converter;                      \
+            CONVERTER_TYPE assumed;                                          \
+            do {                                                             \
+                assumed = old;                                               \
+                if (abs(reinterpret<ValueType>(old)) < abs(val)) {           \
+                    old = atomicCAS(address_as_converter, assumed,           \
+                                    reinterpret<CONVERTER_TYPE>(val));       \
+                }                                                            \
+            } while (assumed != old);                                        \
+            return reinterpret<ValueType>(old);                              \
+        }                                                                    \
     };
 
 // Support 64-bit ATOMIC_ADD
@@ -155,3 +179,60 @@ GKO_BIND_ATOMIC_ADD(__half2);
 
 
 #undef GKO_BIND_ATOMIC_ADD
+
+template <typename T>
+__forceinline__ __device__ T atomic_max(T *__restrict__ addr, T val)
+{
+    return detail::atomic_helper<T>::atomic_max(addr, val);
+}
+
+
+#define GKO_BIND_ATOMIC_MAX(ValueType)               \
+    __forceinline__ __device__ ValueType atomic_max( \
+        ValueType *__restrict__ addr, ValueType val) \
+    {                                                \
+        return atomicMax(addr, val);                 \
+    }
+
+GKO_BIND_ATOMIC_MAX(int);
+GKO_BIND_ATOMIC_MAX(unsigned int);
+// GKO_BIND_ATOMIC_MAX(unsigned long long int);
+// GKO_BIND_ATOMIC_MAX(float);
+
+// GKO_BIND_ATOMIC_MAX(size_type);
+
+/*
+#if !defined(__HIPCC__) || \
+    (defined(__HIP_DEVICE_COMPILE__) && GINKGO_HIP_PLATFORM_NVCC)
+
+
+#if !((defined(CUDA_VERSION) && (CUDA_VERSION < 8000)) || \
+      (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 600)))
+// CUDA 8.0 starts suppoting 64-bit double atomicAdd on devices of compute
+// capability 6.x and higher
+GKO_BIND_ATOMIC_MAX(double);
+#endif
+
+#if !((defined(CUDA_VERSION) && (CUDA_VERSION < 10000)) || \
+      (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 700)))
+// CUDA 10.0 starts supporting 16-bit __half floating-point atomicAdd on devices
+// of compute capability 7.x and higher.
+GKO_BIND_ATOMIC_MAX(__half);
+#endif
+
+#if !((defined(CUDA_VERSION) && (CUDA_VERSION < 10000)) || \
+      (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 600)))
+// CUDA 10.0 starts supporting 32-bit __half2 floating-point atomicAdd on
+// devices of compute capability 6.x and higher. note: The atomicity of the
+// __half2 add operation is guaranteed separately for each of the two __half
+// elements; the entire __half2 is not guaranteed to be atomic as a single
+// 32-bit access.
+GKO_BIND_ATOMIC_MAX(__half2);
+#endif
+
+
+#endif  // !defined(__HIPCC__) || (defined(__HIP_DEVICE_COMPILE__) &&
+        // GINKGO_HIP_PLATFORM_HCC)
+*/
+
+#undef GKO_BIND_ATOMIC_MAX
diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 8258ed38b3e..1c927a91122 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -178,6 +178,102 @@ __global__
 }
 
 
+template <typename ValueType>
+__global__ __launch_bounds__(default_dot_size) void multinorm2_kernel(
+    size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov, ValueType *__restrict__ norms,
+    size_type stride_norms, const stopping_status *__restrict__ stop_status)
+{
+    const auto tidx = threadIdx.x;
+    const auto tidy = threadIdx.y;
+    const auto col_idx = blockIdx.x * default_dot_dim + tidx;
+    const auto num = ceildiv(num_rows, gridDim.y);
+    const auto start_row = blockIdx.y * num;
+    const auto end_row =
+        ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+    __shared__
+        UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
+            reduction_helper_array;
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
+    ValueType local_res = zero<ValueType>();
+    if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+        for (size_type i = start_row + tidy; i < end_row;
+             i += default_dot_dim) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            local_res += squared_norm(next_krylov_basis[next_krylov_idx]);
+        }
+    }
+    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
+    __syncthreads();
+    local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
+    const auto tile_block =
+        group::tiled_partition<default_dot_dim>(group::this_thread_block());
+    const auto sum =
+        reduce(tile_block, local_res,
+               [](const ValueType &a, const ValueType &b) { return a + b; });
+    const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
+    if (tidx == 0 && new_col_idx < num_cols &&
+        !stop_status[new_col_idx].has_stopped()) {
+        const auto norms_idx = new_col_idx;
+        atomic_add(norms + norms_idx, sum);
+    }
+}
+
+
+template <typename ValueType>
+__global__ __launch_bounds__(default_dot_size) void multinorminf_kernel(
+    size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov, ValueType *__restrict__ norms,
+    size_type stride_norms, const stopping_status *__restrict__ stop_status)
+{
+    const auto tidx = threadIdx.x;
+    const auto tidy = threadIdx.y;
+    const auto col_idx = blockIdx.x * default_dot_dim + tidx;
+    const auto num = ceildiv(num_rows, gridDim.y);
+    const auto start_row = blockIdx.y * num;
+    const auto end_row =
+        ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+    __shared__
+        UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
+            reduction_helper_array;
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
+    remove_complex<ValueType> local_res = zero<remove_complex<ValueType>>();
+    if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+        for (size_type i = start_row + tidy; i < end_row;
+             i += default_dot_dim) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            local_res = (local_res >= abs(next_krylov_basis[next_krylov_idx]))
+                            ? local_res
+                            : abs(next_krylov_basis[next_krylov_idx]);
+            // local_res = (local_res >= next_krylov_basis[next_krylov_idx])
+            //                 ? local_res
+            //                 : next_krylov_basis[next_krylov_idx];
+        }
+    }
+    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
+    __syncthreads();
+    local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
+    const auto tile_block =
+        group::tiled_partition<default_dot_dim>(group::this_thread_block());
+    const auto value = reduce(
+        tile_block, local_res,
+        [](const remove_complex<ValueType> &a,
+           const remove_complex<ValueType> &b) { return ((a >= b) ? a : b); });
+    const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
+    if (tidx == 0 && new_col_idx < num_cols &&
+        !stop_status[new_col_idx].has_stopped()) {
+        const auto norms_idx = new_col_idx;
+        atomic_max(norms + norms_idx, value);
+    }
+}
+
+
 template <typename ValueType, typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
     size_type k, size_type num_rows, size_type num_cols,
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 138a4052b9f..4c972b1ff50 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -126,7 +126,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     auto residual_norm =
         Vector::create(exec, dim<2>{1, dense_b->get_size()[1]});
     auto b_norm = Vector::create(exec, dim<2>{1, dense_b->get_size()[1]});
-    auto arnoldi_norm = Vector::create(exec, dim<2>{2, dense_b->get_size()[1]});
+    auto arnoldi_norm = Vector::create(exec, dim<2>{3, dense_b->get_size()[1]});
     Array<size_type> final_iter_nums(this->get_executor(),
                                      dense_b->get_size()[1]);
     auto y =
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index fcfb47cfb63..5535a8871a7 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -572,6 +572,7 @@ void finish_arnoldi_CGS2(
 
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<ValueType>());
+    /*
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
@@ -579,6 +580,11 @@ void finish_arnoldi_CGS2(
         // as_cuda_type(next_krylov_basis->get_const_values()),
         // stride_next_krylov,
         as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
+    */
+    multinorm2_kernel<<<grid_size, block_size>>>(
+        dim_size[0], dim_size[1],
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
     // nrmP = norm(next_krylov_basis
 #ifdef TIMING
     exec->synchronize();
@@ -656,18 +662,32 @@ void finish_arnoldi_CGS2(
     // for i in 1:iter
     //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
     // end
-    components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
-                           dim_size[1], zero<ValueType>());
 #ifdef TIMING
     exec->synchronize();
     auto start_3 = std::chrono::steady_clock::now();
 #endif
+    components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
+                           dim_size[1], zero<ValueType>());
+    /*
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_accessor(next_krylov_const_accessor),
         as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
         as_cuda_type(stop_status));
+    */
+    multinorm2_kernel<<<grid_size, block_size>>>(
+        dim_size[0], dim_size[1],
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
+        as_cuda_type(stop_status));
+    /*
+    multinorminf_kernel<<<grid_size, block_size>>>(
+        dim_size[0], dim_size[1],
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]), 0,
+        as_cuda_type(stop_status));
+    */
 #ifdef TIMING
     exec->synchronize();
     auto time_3 = std::chrono::steady_clock::now() - start_3;
@@ -757,12 +777,27 @@ void finish_arnoldi_CGS2(
         components::fill_array(
             exec, arnoldi_norm->get_values() + dim_size[1] * (l - 1),
             dim_size[1], zero<ValueType>());
+        /*
         multidot_kernel<<<grid_size, block_size>>>(
             0, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
             stride_next_krylov, as_cuda_accessor(next_krylov_const_accessor),
             as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
             as_cuda_type(stop_status));
+        */
+        multinorm2_kernel<<<grid_size, block_size>>>(
+            dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov,
+            as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
+            as_cuda_type(stop_status));
+        /*
+        multinorminf_kernel<<<grid_size, block_size>>>(
+            dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            as_cuda_type(arnoldi_norm->get_values() + 2*dim_size[1]), 0,
+            as_cuda_type(stop_status));
+        */
         // nrmN = norm(next_krylov_basis)
         components::fill_array(exec, num_reorth->get_data(), 1,
                                zero<size_type>());

From a28272f7495e25442fba171bc1798e72f162e50a Mon Sep 17 00:00:00 2001
From: josealiaga <aliaga@uji.es>
Date: Mon, 25 May 2020 01:22:43 +0200
Subject: [PATCH 0352/1008] remove_complex has been added to the norms
 variables, and multinorm2_inf_kernel has been created.

Also, the last 2-norm computation is uses as the hessenberg component, avoiding
the use of update_hessenberg_2_kernel.

Next steps:
  * To solve the problems which have been found when the set_scale method is used.
  * To know why the reference tests don't work properly.
---
 common/components/atomic.hpp.inc          |   2 +-
 common/solver/gmres_mixed_kernels.hpp.inc | 183 ++++++++++++---
 core/solver/gmres_mixed.cpp               |  37 ++-
 core/solver/gmres_mixed_accessor.hpp      |   7 +-
 core/solver/gmres_mixed_kernels.hpp       |  26 ++-
 cuda/solver/gmres_mixed_kernels.cu        | 168 +++++++++++---
 cuda/test/solver/gmres_mixed_kernels.cpp  |   2 +-
 omp/solver/gmres_mixed_kernels.cpp        | 135 +++++++----
 omp/test/solver/gmres_mixed_kernels.cpp   |   2 +-
 reference/solver/gmres_mixed_kernels.cpp  | 264 ++++++++++++++++++----
 10 files changed, 645 insertions(+), 181 deletions(-)

diff --git a/common/components/atomic.hpp.inc b/common/components/atomic.hpp.inc
index 0e7418db0b3..1ababe8380a 100644
--- a/common/components/atomic.hpp.inc
+++ b/common/components/atomic.hpp.inc
@@ -94,7 +94,7 @@ __forceinline__ __device__ ResultType reinterpret(ValueType val)
             CONVERTER_TYPE assumed;                                          \
             do {                                                             \
                 assumed = old;                                               \
-                if (abs(reinterpret<ValueType>(old)) < abs(val)) {           \
+                if (reinterpret<ValueType>(assumed) < val) {                 \
                     old = atomicCAS(address_as_converter, assumed,           \
                                     reinterpret<CONVERTER_TYPE>(val));       \
                 }                                                            \
diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 1c927a91122..1395407ee44 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -142,7 +142,7 @@ template <size_type block_size, typename ValueType,
 __global__ __launch_bounds__(block_size) void initialize_2_2_kernel(
     size_type num_rows, size_type num_rhs,
     const ValueType *__restrict__ residual, size_type stride_residual,
-    const ValueType *__restrict__ residual_norm,
+    const remove_complex<ValueType> *__restrict__ residual_norm,
     ValueType *__restrict__ residual_norm_collection,
     Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
@@ -182,7 +182,7 @@ template <typename ValueType>
 __global__ __launch_bounds__(default_dot_size) void multinorm2_kernel(
     size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov, ValueType *__restrict__ norms,
+    size_type stride_next_krylov, remove_complex<ValueType> *__restrict__ norms,
     size_type stride_norms, const stopping_status *__restrict__ stop_status)
 {
     const auto tidx = threadIdx.x;
@@ -194,11 +194,12 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_kernel(
         ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
-    __shared__
-        UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
-            reduction_helper_array;
-    ValueType *__restrict__ reduction_helper = reduction_helper_array;
-    ValueType local_res = zero<ValueType>();
+    __shared__ UninitializedArray<remove_complex<ValueType>,
+                                  default_dot_dim *(default_dot_dim + 1)>
+        reduction_helper_array;
+    remove_complex<ValueType> *__restrict__ reduction_helper =
+        reduction_helper_array;
+    remove_complex<ValueType> local_res = zero<remove_complex<ValueType>>();
     if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
         for (size_type i = start_row + tidy; i < end_row;
              i += default_dot_dim) {
@@ -213,7 +214,8 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_kernel(
         group::tiled_partition<default_dot_dim>(group::this_thread_block());
     const auto sum =
         reduce(tile_block, local_res,
-               [](const ValueType &a, const ValueType &b) { return a + b; });
+               [](const remove_complex<ValueType> &a,
+                  const remove_complex<ValueType> &b) { return a + b; });
     const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
     if (tidx == 0 && new_col_idx < num_cols &&
         !stop_status[new_col_idx].has_stopped()) {
@@ -227,7 +229,7 @@ template <typename ValueType>
 __global__ __launch_bounds__(default_dot_size) void multinorminf_kernel(
     size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov, ValueType *__restrict__ norms,
+    size_type stride_next_krylov, remove_complex<ValueType> *__restrict__ norms,
     size_type stride_norms, const stopping_status *__restrict__ stop_status)
 {
     const auto tidx = threadIdx.x;
@@ -239,30 +241,31 @@ __global__ __launch_bounds__(default_dot_size) void multinorminf_kernel(
         ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
-    __shared__
-        UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
-            reduction_helper_array;
-    ValueType *__restrict__ reduction_helper = reduction_helper_array;
-    remove_complex<ValueType> local_res = zero<remove_complex<ValueType>>();
+    __shared__ UninitializedArray<remove_complex<ValueType>,
+                                  default_dot_dim *(default_dot_dim + 1)>
+        reduction_helper_array;
+    remove_complex<ValueType> *__restrict__ reduction_helper =
+        reduction_helper_array;
+    remove_complex<ValueType> local_max = zero<remove_complex<ValueType>>();
     if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
         for (size_type i = start_row + tidy; i < end_row;
              i += default_dot_dim) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
-            local_res = (local_res >= abs(next_krylov_basis[next_krylov_idx]))
-                            ? local_res
+            local_max = (local_max >= abs(next_krylov_basis[next_krylov_idx]))
+                            ? local_max
                             : abs(next_krylov_basis[next_krylov_idx]);
-            // local_res = (local_res >= next_krylov_basis[next_krylov_idx])
-            //                 ? local_res
+            // local_max = (local_max >= next_krylov_basis[next_krylov_idx])
+            //                 ? local_max
             //                 : next_krylov_basis[next_krylov_idx];
         }
     }
-    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
+    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_max;
     __syncthreads();
-    local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
+    local_max = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
     const auto tile_block =
         group::tiled_partition<default_dot_dim>(group::this_thread_block());
     const auto value = reduce(
-        tile_block, local_res,
+        tile_block, local_max,
         [](const remove_complex<ValueType> &a,
            const remove_complex<ValueType> &b) { return ((a >= b) ? a : b); });
     const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
@@ -274,6 +277,75 @@ __global__ __launch_bounds__(default_dot_size) void multinorminf_kernel(
 }
 
 
+template <typename ValueType>
+__global__ __launch_bounds__(default_dot_size) void multinorm2_inf_kernel(
+    size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov,
+    remove_complex<ValueType> *__restrict__ norms1,
+    remove_complex<ValueType> *__restrict__ norms2, size_type stride_norms,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto tidx = threadIdx.x;
+    const auto tidy = threadIdx.y;
+    const auto col_idx = blockIdx.x * default_dot_dim + tidx;
+    const auto num = ceildiv(num_rows, gridDim.y);
+    const auto start_row = blockIdx.y * num;
+    const auto end_row =
+        ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+    __shared__ UninitializedArray<remove_complex<ValueType>,
+                                  default_dot_dim *(default_dot_dim + 1)>
+        reduction_helper_array_1, reduction_helper_array_2;
+    remove_complex<ValueType> *__restrict__ reduction_helper_1 =
+        reduction_helper_array_1;
+    remove_complex<ValueType> *__restrict__ reduction_helper_2 =
+        reduction_helper_array_2;
+    remove_complex<ValueType> local_res = zero<remove_complex<ValueType>>();
+    remove_complex<ValueType> local_max = zero<remove_complex<ValueType>>();
+    if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+        for (size_type i = start_row + tidy; i < end_row;
+             i += default_dot_dim) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            const auto num = next_krylov_basis[next_krylov_idx];
+            local_res += squared_norm(num);
+            // local_res += squared_norm(next_krylov_basis[next_krylov_idx]);
+            local_max = ((local_max >= abs(num)) ? local_max : abs(num));
+            // local_max = (local_max >=
+            // abs(next_krylov_basis[next_krylov_idx]))
+            //                 ? local_max
+            //                 : abs(next_krylov_basis[next_krylov_idx]);
+        }
+    }
+    // Add reduction
+    reduction_helper_1[tidx * (default_dot_dim + 1) + tidy] = local_res;
+    reduction_helper_2[tidx * (default_dot_dim + 1) + tidy] = local_max;
+    __syncthreads();
+    local_res = reduction_helper_1[tidy * (default_dot_dim + 1) + tidx];
+    const auto tile_block1 =
+        group::tiled_partition<default_dot_dim>(group::this_thread_block());
+    const auto sum =
+        reduce(tile_block1, local_res,
+               [](const remove_complex<ValueType> &a,
+                  const remove_complex<ValueType> &b) { return a + b; });
+    local_max = reduction_helper_2[tidy * (default_dot_dim + 1) + tidx];
+    const auto tile_block2 =
+        group::tiled_partition<default_dot_dim>(group::this_thread_block());
+    const auto value = reduce(
+        tile_block2, local_max,
+        [](const remove_complex<ValueType> &a,
+           const remove_complex<ValueType> &b) { return ((a >= b) ? a : b); });
+    const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
+    if (tidx == 0 && new_col_idx < num_cols &&
+        !stop_status[new_col_idx].has_stopped()) {
+        const auto norms_idx = new_col_idx;
+        atomic_add(norms1 + norms_idx, sum);
+        atomic_max(norms2 + norms_idx, value);
+    }
+}
+
+
 template <typename ValueType, typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
     size_type k, size_type num_rows, size_type num_cols,
@@ -879,7 +951,8 @@ __global__
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
         Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
-        size_type stride_hessenberg, const ValueType *__restrict__ arnoldi_norm,
+        size_type stride_hessenberg,
+        const remove_complex<ValueType> *__restrict__ arnoldi_norm,
         size_type stride_norm, const stopping_status *__restrict__ stop_status,
         stopping_status *__restrict__ reorth_status,
         size_type *__restrict__ num_reorth)
@@ -901,9 +974,11 @@ __global__
             hessenberg_iter[hessenberg_idx] *
             krylov_bases.read(row_idx, k * num_cols + col_idx);
         if (row_idx == 0) {
-            // TODO remove abs() as soon as type of arnoldi_norm has changed
+            // DONE: TODO remove abs() as soon as type of arnoldi_norm has
+            // changed
             if (squared_norm(hessenberg_iter[hessenberg_idx]) >
-                abs(0.99 * sqrt(arnoldi_norm[arnoldi_idx]))) {
+                (0.99 * sqrt(arnoldi_norm[arnoldi_idx]))) {
+                // abs(0.99 * sqrt(arnoldi_norm[arnoldi_idx]))) {
                 reorth_status[reorth_idx].reset();
                 atomic_add(num_reorth, (size_type)1);
             } else {
@@ -1075,8 +1150,11 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms(
 
     if (global_id < stride_norm0 && !stop_status[global_id].has_stopped()) {
         // TODO remove abs() as soon as type of arnoldi_norm has changed
-        const auto num0 = abs(eta * arnoldi_norm0[global_id]);
-        const auto num1 = abs(arnoldi_norm1[global_id]);
+        const auto num0 = (eta * arnoldi_norm0[global_id]);
+        // const auto num0 = abs(eta * arnoldi_norm0[global_id]);
+        const auto num1 = (arnoldi_norm1[global_id]);
+        // const auto num1 = abs(arnoldi_norm1[global_id]);
+        // printf("num0 = %f , num1 = %f\n", num0, num1);
         if (num1 < num0) {
             reorth_status[global_id].reset();
             atomic_add(num_reorth, (size_type)1);
@@ -1087,6 +1165,49 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms(
 }
 
 
+// Must be called with at least `num_cols` blocks, each with `block_size`
+// threads. `block_size` must be a power of 2.
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
+    remove_complex<ValueType> *__restrict__ arnoldi_norm, size_type stride_norm,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    size_type iter,
+    Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    const stopping_status *__restrict__ stop_status,
+    stopping_status *__restrict__ reorth_status,
+    size_type *__restrict__ num_reorth)
+{
+    const remove_complex<ValueType> eta = 1.0 / 2.0;
+    const auto col_idx = thread::get_thread_id_flat();
+
+    if (col_idx < stride_norm && !stop_status[col_idx].has_stopped()) {
+        // TODO remove abs() as soon as type of arnoldi_norm has changed
+        const auto num0 = (eta * arnoldi_norm[col_idx]);
+        // const auto num0 = abs(eta * arnoldi_norm0[col_idx]);
+        const auto num11 = (arnoldi_norm[col_idx + stride_norm]);
+        const auto num12 = sqrt(num11);
+        // const auto num1 = abs(arnoldi_norm1[col_idx]);
+        const auto num2 = (arnoldi_norm[col_idx + 2 * stride_norm]);
+        // printf("num0 = %f , num11 = %f , num12 = %f , num2 = %f\n", num0,
+        // num11,
+        //        num12, num2);
+        if (num11 < num0) {
+            reorth_status[col_idx].reset();
+            atomic_add(num_reorth, (size_type)1);
+        } else {
+            reorth_status[col_idx].stop(1);
+        }
+        arnoldi_norm[col_idx] = num11;
+        arnoldi_norm[col_idx + stride_norm] = num12;
+        hessenberg_iter[iter * stride_hessenberg + col_idx] = num12;
+        // helper_functions_accessor<ValueType,
+        // ValueTypeKrylovBases>::write_scale(
+        //     krylov_bases, col_idx, num2 / num12);
+        // krylov_bases.set_scale(col_idx, num2 / num12);
+    }
+}
+
+
 // Must be called with at least `num_cols` blocks, each with `block_size`
 // threads. `block_size` must be a power of 2.
 template <int block_size, typename ValueType>
@@ -1193,9 +1314,11 @@ template <typename ValueType>
 __device__ void calculate_residual_norm_kernel(
     size_type col_idx, size_type num_cols, size_type iter,
     const ValueType *givens_sin, size_type stride_sin,
-    const ValueType *givens_cos, size_type stride_cos, ValueType *residual_norm,
+    const ValueType *givens_cos, size_type stride_cos,
+    remove_complex<ValueType> *residual_norm,
     ValueType *residual_norm_collection,
-    size_type stride_residual_norm_collection, const ValueType *b_norm)
+    size_type stride_residual_norm_collection,
+    const remove_complex<ValueType> *b_norm)
 {
     residual_norm_collection[(iter + 1) * stride_residual_norm_collection +
                              col_idx] =
@@ -1221,10 +1344,10 @@ __global__ __launch_bounds__(block_size) void givens_rotation_kernel(
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     ValueType *__restrict__ givens_sin, size_type stride_sin,
     ValueType *__restrict__ givens_cos, size_type stride_cos,
-    ValueType *__restrict__ residual_norm,
+    remove_complex<ValueType> *__restrict__ residual_norm,
     ValueType *__restrict__ residual_norm_collection,
     size_type stride_residual_norm_collection,
-    const ValueType *__restrict__ b_norm,
+    const remove_complex<ValueType> *__restrict__ b_norm,
     const stopping_status *__restrict__ stop_status)
 {
     const auto col_idx = thread::get_thread_id_flat();
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 4c972b1ff50..c6eef6cbd3e 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -51,7 +51,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/gmres_mixed_kernels.hpp"
 
 
-#define TIMING 1
+// #define TIMING 1
 
 
 #ifdef TIMING
@@ -83,6 +83,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix_);
 
     using Vector = matrix::Dense<ValueType>;
+    using VectorNorms = matrix::Dense<remove_complex<ValueType>>;
     using LowArray = Array<ValueTypeKrylovBases>;
     using KrylovAccessor = kernels::Accessor2d<ValueTypeKrylovBases, ValueType>;
     using Accessor2dHelper =
@@ -124,9 +125,10 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     auto residual_norm_collection = Vector::create(
         exec, dim<2>{krylov_dim_mixed_ + 1, dense_b->get_size()[1]});
     auto residual_norm =
-        Vector::create(exec, dim<2>{1, dense_b->get_size()[1]});
-    auto b_norm = Vector::create(exec, dim<2>{1, dense_b->get_size()[1]});
-    auto arnoldi_norm = Vector::create(exec, dim<2>{3, dense_b->get_size()[1]});
+        VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
+    auto b_norm = VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
+    auto arnoldi_norm =
+        VectorNorms::create(exec, dim<2>{3, dense_b->get_size()[1]});
     Array<size_type> final_iter_nums(this->get_executor(),
                                      dense_b->get_size()[1]);
     auto y =
@@ -201,12 +203,35 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
             exec->synchronize();
             auto t_aux_0 = std::chrono::steady_clock::now();
 #endif
+            //            std::cout << "RESTARTING" << std::endl;
             num_restarts++;
             // Restart
+            /*
+            std::cout << "[ ";
+            for (int i = 0; i <= krylov_dim_mixed_; i++) {
+                for (int j = 0; j < krylov_dim_mixed_ * dense_b->get_size()[1];
+                     j++) {
+                    if (j > 0) std::cout << ", ";
+                    std::cout << hessenberg->at(i, j);
+                }
+                std::cout << std::endl;
+            }
+            std::cout << "]" << std::endl;
+            */
             exec->run(gmres_mixed::make_step_2(
                 residual_norm_collection.get(),
                 krylov_bases_accessor.to_const(), hessenberg.get(), y.get(),
                 before_preconditioner.get(), &final_iter_nums));
+            /* */ /*
+            auto hessenberg_small = hessenberg->create_submatrix(
+                span{0, restart_iter},
+                span{0, dense_b->get_size()[1] * (restart_iter)});
+            exec->run(gmres_mixed::make_step_2(
+                residual_norm_collection.get(),
+                krylov_bases_accessor.to_const(),
+                hessenberg_small.get(), y.get(),
+                before_preconditioner.get(), &final_iter_nums));
+            */
             // Solve upper triangular.
             // y = hessenberg \ residual_norm_collection
 
@@ -332,12 +357,12 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
         residual_norm_collection.get(), krylov_bases_accessor.to_const(),
         hessenberg_small.get(), y.get(), before_preconditioner.get(),
         &final_iter_nums));
+    // Solve upper triangular.
+    // y = hessenberg \ residual_norm_collection
 #ifdef TIMING_STEPS
     exec->synchronize();
     auto time_STEP2 = std::chrono::steady_clock::now() - t_aux_3;
 #endif
-    // Solve upper triangular.
-    // y = hessenberg \ residual_norm_collection
 
 #ifdef TIMING_STEPS
     exec->synchronize();
diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index 4727b9926ff..e013dec94ae 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -45,6 +45,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
+#include <iostream>
+
+
 namespace gko {
 namespace kernels {  // TODO maybe put into another separate namespace
 
@@ -182,6 +185,7 @@ class Accessor2dConst<StorageType, ArithmeticType, true> {
         // ignored by nvcc.
         const storage_type *GKO_RESTRICT rest_storage = storage_;
         const arithmetic_type *GKO_RESTRICT rest_scale = scale_;
+        //        std::cout << col << " - A - " << rest_scale[col] << std::endl;
         return static_cast<arithmetic_type>(rest_storage[row * stride_ + col]) *
                rest_scale[col];
     }
@@ -365,7 +369,8 @@ class Accessor2dHelper<ValueType, ValueTypeKrylovBases, true> {
           scale_{exec, krylov_dim_[1]}
     {
         // For testing, initialize scale to ones
-        Array<ValueType> h_scale{exec->get_master(), krylov_dim[0]};
+        // Array<ValueType> h_scale{exec->get_master(), krylov_dim[0]};
+        Array<ValueType> h_scale{exec->get_master(), krylov_dim[1]};
         for (size_type i = 0; i < h_scale.get_num_elems(); ++i) {
             h_scale.get_data()[i] = one<ValueType>();
         }
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index eda1bf1728a..55a357689ff 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -44,6 +44,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/gmres_mixed_accessor.hpp"
 
 
+#define FINISH_ARNOLDI 3
+
+
 /**
  * Instantiates a template for each value type with each lower precision type
  * supported by Ginkgo for GmresMixed.
@@ -79,19 +82,20 @@ namespace gmres_mixed {
 #define GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL(_type)                     \
     void initialize_1(                                                         \
         std::shared_ptr<const DefaultExecutor> exec,                           \
-        const matrix::Dense<_type> *b, matrix::Dense<_type> *b_norm,           \
+        const matrix::Dense<_type> *b,                                         \
+        matrix::Dense<remove_complex<_type>> *b_norm,                          \
         matrix::Dense<_type> *residual, matrix::Dense<_type> *givens_sin,      \
         matrix::Dense<_type> *givens_cos, Array<stopping_status> *stop_status, \
         size_type krylov_dim)
 
 
-#define GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(_type1, _type2)    \
-    void initialize_2(std::shared_ptr<const DefaultExecutor> exec,     \
-                      const matrix::Dense<_type1> *residual,           \
-                      matrix::Dense<_type1> *residual_norm,            \
-                      matrix::Dense<_type1> *residual_norm_collection, \
-                      Accessor2d<_type2, _type1> krylov_bases,         \
-                      matrix::Dense<_type1> *next_krylov_basis,        \
+#define GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(_type1, _type2)         \
+    void initialize_2(std::shared_ptr<const DefaultExecutor> exec,          \
+                      const matrix::Dense<_type1> *residual,                \
+                      matrix::Dense<remove_complex<_type1>> *residual_norm, \
+                      matrix::Dense<_type1> *residual_norm_collection,      \
+                      Accessor2d<_type2, _type1> krylov_bases,              \
+                      matrix::Dense<_type1> *next_krylov_basis,             \
                       Array<size_type> *final_iter_nums, size_type krylov_dim)
 
 
@@ -100,13 +104,13 @@ namespace gmres_mixed {
         std::shared_ptr<const DefaultExecutor> exec,                          \
         matrix::Dense<_type1> *next_krylov_basis,                             \
         matrix::Dense<_type1> *givens_sin, matrix::Dense<_type1> *givens_cos, \
-        matrix::Dense<_type1> *residual_norm,                                 \
+        matrix::Dense<remove_complex<_type1>> *residual_norm,                 \
         matrix::Dense<_type1> *residual_norm_collection,                      \
         Accessor2d<_type2, _type1> krylov_bases,                              \
         matrix::Dense<_type1> *hessenberg_iter,                               \
         matrix::Dense<_type1> *buffer_iter,                                   \
-        const matrix::Dense<_type1> *b_norm,                                  \
-        matrix::Dense<_type1> *arnoldi_norm, size_type iter,                  \
+        const matrix::Dense<remove_complex<_type1>> *b_norm,                  \
+        matrix::Dense<remove_complex<_type1>> *arnoldi_norm, size_type iter,  \
         Array<size_type> *final_iter_nums,                                    \
         const Array<stopping_status> *stop_status,                            \
         Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,  \
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 5535a8871a7..0c2fcb2de95 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -139,7 +139,7 @@ void zero_matrix(size_type m, size_type n, size_type stride, ValueType *array)
 template <typename ValueType>
 void initialize_1(std::shared_ptr<const CudaExecutor> exec,
                   const matrix::Dense<ValueType> *b,
-                  matrix::Dense<ValueType> *b_norm,
+                  matrix::Dense<remove_complex<ValueType>> *b_norm,
                   matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *givens_sin,
                   matrix::Dense<ValueType> *givens_cos,
@@ -168,7 +168,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
 template <typename ValueType, typename ValueTypeKrylovBases>
 void initialize_2(std::shared_ptr<const CudaExecutor> exec,
                   const matrix::Dense<ValueType> *residual,
-                  matrix::Dense<ValueType> *residual_norm,
+                  matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
                   Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
@@ -274,7 +274,7 @@ void finish_arnoldi_reorth(
     Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
-    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
     const stopping_status *stop_status, stopping_status *reorth_status,
     Array<size_type> *num_reorth)
 {
@@ -295,13 +295,20 @@ void finish_arnoldi_reorth(
     //    size_type num_reorth;
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<ValueType>());
+    /*
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_accessor(next_krylov_const_accessor),
+        // as_cuda_type(next_krylov_basis->get_const_values()),
+    stride_next_krylov, as_cuda_type(arnoldi_norm->get_values()), 0,
+    as_cuda_type(stop_status));
+    */
+    multinorm2_kernel<<<grid_size, block_size>>>(
+        dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
-    // nrm = norm(next_krylov_basis
+    // nrm = norm(next_krylov_basis)
     for (size_type k = 0; k < iter + 1; ++k) {
         components::fill_array(
             exec, hessenberg_iter->get_values() + k * stride_hessenberg,
@@ -387,7 +394,7 @@ void finish_arnoldi_CGS(
     Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
-    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
     const stopping_status *stop_status, stopping_status *reorth_status,
     Array<size_type> *num_reorth)
 {
@@ -408,11 +415,17 @@ void finish_arnoldi_CGS(
 
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<ValueType>());
+    /*
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_accessor(next_krylov_const_accessor),
         as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
+    */
+    multinorm2_kernel<<<grid_size, block_size>>>(
+        dim_size[0], dim_size[1],
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
     // nrmP = norm(next_krylov_basis
     for (size_type k = 0; k < iter + 1; ++k) {
         components::fill_array(
@@ -443,12 +456,20 @@ void finish_arnoldi_CGS(
     // end
     components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
                            dim_size[1], zero<ValueType>());
+    /*
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_accessor(next_krylov_const_accessor),
         as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
         as_cuda_type(stop_status));
+    */
+    multinorm2_kernel<<<grid_size, block_size>>>(
+        dim_size[0], dim_size[1],
+        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
+        as_cuda_type(stop_status));
+    // nrmN = norm(next_krylov_basis)
     components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
     check_arnoldi_norms<default_block_size>
         <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
@@ -492,12 +513,20 @@ void finish_arnoldi_CGS(
         components::fill_array(
             exec, arnoldi_norm->get_values() + dim_size[1] * (l - 1),
             dim_size[1], zero<ValueType>());
+        /*
         multidot_kernel<<<grid_size, block_size>>>(
             0, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
             stride_next_krylov, as_cuda_accessor(next_krylov_const_accessor),
             as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
             as_cuda_type(stop_status));
+        */
+        multinorm2_kernel<<<grid_size, block_size>>>(
+            dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov,
+            as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
+            as_cuda_type(stop_status));
         // nrmN = norm(next_krylov_basis)
         components::fill_array(exec, num_reorth->get_data(), 1,
                                zero<size_type>());
@@ -546,11 +575,12 @@ void finish_arnoldi_CGS2(
     Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
-    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
     const stopping_status *stop_status, stopping_status *reorth_status,
     Array<size_type> *num_reorth, int *num_reorth_steps,
     int *num_reorth_vectors)
 {
+    using non_complex = remove_complex<ValueType>;
     const auto stride_next_krylov = next_krylov_basis->get_stride();
     const auto stride_krylov = krylov_bases.get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
@@ -571,7 +601,7 @@ void finish_arnoldi_CGS2(
     auto next_krylov_const_accessor = next_krylov_accessor.to_const();
 
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
-                           zero<ValueType>());
+                           zero<non_complex>());
     /*
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
@@ -667,7 +697,9 @@ void finish_arnoldi_CGS2(
     auto start_3 = std::chrono::steady_clock::now();
 #endif
     components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
-                           dim_size[1], zero<ValueType>());
+                           dim_size[1], zero<non_complex>());
+    components::fill_array(exec, arnoldi_norm->get_values() + 2 * dim_size[1],
+                           dim_size[1], zero<non_complex>());
     /*
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
@@ -676,17 +708,37 @@ void finish_arnoldi_CGS2(
         as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
         as_cuda_type(stop_status));
     */
+    /*
     multinorm2_kernel<<<grid_size, block_size>>>(
         dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
         as_cuda_type(stop_status));
+    */
     /*
-    multinorminf_kernel<<<grid_size, block_size>>>(
+ multinorminf_kernel<<<grid_size, block_size>>>(
+     dim_size[0], dim_size[1],
+     as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+     as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]), 0,
+     as_cuda_type(stop_status));
+ */ /* */
+    multinorm2_inf_kernel<<<grid_size, block_size>>>(
         dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_cuda_type(arnoldi_norm->get_values() + dim_size[1]),
         as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]), 0,
         as_cuda_type(stop_status));
+    /* */
+    /*
+    {
+        exec->synchronize();
+        remove_complex<ValueType> norm, inf;
+        exec->get_master()->copy_from(
+            exec.get(), 1, arnoldi_norm->get_values() + dim_size[1], &norm);
+        exec->get_master()->copy_from(
+            exec.get(), 1, arnoldi_norm->get_values() + 2 * dim_size[1], &inf);
+        std::cout << sqrt(norm) << " - " << inf << std::endl;
+    }
     */
 #ifdef TIMING
     exec->synchronize();
@@ -696,17 +748,27 @@ void finish_arnoldi_CGS2(
               << std::endl;
 #endif
     // nrmN = norm(next_krylov_basis)
-    components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
 #ifdef TIMING
     exec->synchronize();
     auto start_4 = std::chrono::steady_clock::now();
 #endif
+    components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
+    /*
     check_arnoldi_norms<default_block_size>
         <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
             as_cuda_type(arnoldi_norm->get_const_values()), stride_arnoldi,
             as_cuda_type(arnoldi_norm->get_const_values() + stride_arnoldi),
             stride_arnoldi, as_cuda_type(stop_status),
             as_cuda_type(reorth_status), as_cuda_type(num_reorth->get_data()));
+    */ /* */
+    check_arnoldi_norms_new<default_block_size>
+        <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
+            as_cuda_type(arnoldi_norm->get_values()), stride_arnoldi,
+            as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+            iter + 1, as_cuda_accessor(krylov_bases.to_const()),
+            as_cuda_type(stop_status), as_cuda_type(reorth_status),
+            as_cuda_type(num_reorth->get_data()));
+    /* */
 #ifdef TIMING
     exec->synchronize();
     auto time_4 = std::chrono::steady_clock::now() - start_4;
@@ -730,6 +792,7 @@ void finish_arnoldi_CGS2(
 #endif
     // numReorth <= number of next_krylov vector to be reorthogonalization
     for (size_type l = 1; (numReorth > 0) && (l < 3); l++) {
+        // std::cout << "CUDA RESTART " << l << std::endl;
         (*num_reorth_steps)++;
         (*num_reorth_vectors) += iter;
         zero_matrix(iter + 1, dim_size[1], stride_buffer,
@@ -774,9 +837,14 @@ void finish_arnoldi_CGS2(
         // for i in 1:iter
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
-        components::fill_array(
-            exec, arnoldi_norm->get_values() + dim_size[1] * (l - 1),
-            dim_size[1], zero<ValueType>());
+        components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
+                               dim_size[1], zero<non_complex>());
+        // components::fill_array(
+        //     exec, arnoldi_norm->get_values() + dim_size[1] * (l - 1),
+        //     dim_size[1], zero<ValueType>());
+        components::fill_array(exec,
+                               arnoldi_norm->get_values() + 2 * dim_size[1],
+                               dim_size[1], zero<non_complex>());
         /*
         multidot_kernel<<<grid_size, block_size>>>(
             0, dim_size[0], dim_size[1],
@@ -785,22 +853,35 @@ void finish_arnoldi_CGS2(
             as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
             as_cuda_type(stop_status));
         */
+        /*
         multinorm2_kernel<<<grid_size, block_size>>>(
             dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
             stride_next_krylov,
             as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
             as_cuda_type(stop_status));
+        */
         /*
-        multinorminf_kernel<<<grid_size, block_size>>>(
+     multinorminf_kernel<<<grid_size, block_size>>>(
+         dim_size[0], dim_size[1],
+         as_cuda_type(next_krylov_basis->get_const_values()),
+         stride_next_krylov,
+         as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]), 0,
+         as_cuda_type(stop_status));
+     */ /* */
+        multinorm2_inf_kernel<<<grid_size, block_size>>>(
             dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
-            as_cuda_type(arnoldi_norm->get_values() + 2*dim_size[1]), 0,
+            stride_next_krylov,
+            as_cuda_type(arnoldi_norm->get_values() + dim_size[1]),
+            // as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)),
+            as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]), 0,
             as_cuda_type(stop_status));
-        */
+        /* */
         // nrmN = norm(next_krylov_basis)
         components::fill_array(exec, num_reorth->get_data(), 1,
                                zero<size_type>());
+        /*
         check_arnoldi_norms<default_block_size>
             <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
                 as_cuda_type(arnoldi_norm->get_const_values() +
@@ -811,6 +892,15 @@ void finish_arnoldi_CGS2(
                 stride_arnoldi, as_cuda_type(stop_status),
                 as_cuda_type(reorth_status),
                 as_cuda_type(num_reorth->get_data()));
+        */ /* */
+        check_arnoldi_norms_new<default_block_size>
+            <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
+                as_cuda_type(arnoldi_norm->get_values()), stride_arnoldi,
+                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+                iter + 1, as_cuda_accessor(krylov_bases.to_const()),
+                as_cuda_type(stop_status), as_cuda_type(reorth_status),
+                as_cuda_type(num_reorth->get_data()));
+        /* */
         numReorth = 0;
         exec->get_master()->copy_from(exec.get(), 1,
                                       num_reorth->get_const_data(), &numReorth);
@@ -821,12 +911,14 @@ void finish_arnoldi_CGS2(
     exec->synchronize();
     auto start_6 = std::chrono::steady_clock::now();
 #endif
+    /*
     update_hessenberg_2_kernel<default_block_size>
         <<<dim_size[1], default_block_size>>>(
             iter, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
             stride_next_krylov, as_cuda_type(hessenberg_iter->get_values()),
             stride_hessenberg, as_cuda_type(stop_status));
+    */
     update_krylov_next_krylov_kernel<default_block_size>
         <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
            default_block_size>>>(
@@ -853,10 +945,10 @@ void givens_rotation(std::shared_ptr<const CudaExecutor> exec,
                      matrix::Dense<ValueType> *givens_sin,
                      matrix::Dense<ValueType> *givens_cos,
                      matrix::Dense<ValueType> *hessenberg_iter,
-                     matrix::Dense<ValueType> *residual_norm,
+                     matrix::Dense<remove_complex<ValueType>> *residual_norm,
                      matrix::Dense<ValueType> *residual_norm_collection,
-                     const matrix::Dense<ValueType> *b_norm, size_type iter,
-                     const Array<stopping_status> *stop_status)
+                     const matrix::Dense<remove_complex<ValueType>> *b_norm,
+                     size_type iter, const Array<stopping_status> *stop_status)
 {
     // TODO: tune block_size for optimal performance
     constexpr auto block_size = default_block_size;
@@ -883,14 +975,14 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
             matrix::Dense<ValueType> *next_krylov_basis,
             matrix::Dense<ValueType> *givens_sin,
             matrix::Dense<ValueType> *givens_cos,
-            matrix::Dense<ValueType> *residual_norm,
+            matrix::Dense<remove_complex<ValueType>> *residual_norm,
             matrix::Dense<ValueType> *residual_norm_collection,
             Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
             matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
-            const matrix::Dense<ValueType> *b_norm,
-            matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
-            Array<size_type> *final_iter_nums,
+            const matrix::Dense<remove_complex<ValueType>> *b_norm,
+            matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+            size_type iter, Array<size_type> *final_iter_nums,
             const Array<stopping_status> *stop_status,
             Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
             int *num_reorth_steps, int *num_reorth_vectors)
@@ -901,22 +993,30 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
         default_block_size>>>(as_cuda_type(final_iter_nums->get_data()),
                               as_cuda_type(stop_status->get_const_data()),
                               final_iter_nums->get_num_elems());
-    //    finish_arnoldi(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
-    //                   iter, stop_status->get_const_data());
-    //    finish_arnoldi_reorth(exec, next_krylov_basis, krylov_bases,
-    //                          hessenberg_iter, buffer_iter, arnoldi_norm,
-    //                          iter, stop_status->get_const_data(),
-    //                          reorth_status->get_data(), num_reorth);
-    //    finish_arnoldi_CGS(exec, next_krylov_basis, krylov_bases,
-    //    hessenberg_iter,
-    //                       buffer_iter, arnoldi_norm, iter,
-    //                       stop_status->get_const_data(),
-    //                       reorth_status->get_data(), num_reorth);
+#if FINISH_ARNOLDI == 1
+    //    std::cout << "CUDA MGS_REORTH" << std::endl;
+    finish_arnoldi_reorth(exec, next_krylov_basis, krylov_bases,
+                          hessenberg_iter, buffer_iter, arnoldi_norm, iter,
+                          stop_status->get_const_data(),
+                          reorth_status->get_data(), num_reorth);
+#elif FINISH_ARNOLDI == 2
+    //    std::cout << "CUDA CGS_REORTH" << std::endl;
+    finish_arnoldi_CGS(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
+                       buffer_iter, arnoldi_norm, iter,
+                       stop_status->get_const_data(), reorth_status->get_data(),
+                       num_reorth);
+#elif FINISH_ARNOLDI == 3
+    //    std::cout << "CUDA CGS_REORTH_2" << std::endl;
     finish_arnoldi_CGS2(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
                         buffer_iter, arnoldi_norm, iter,
                         stop_status->get_const_data(),
                         reorth_status->get_data(), num_reorth, num_reorth_steps,
                         num_reorth_vectors);
+#else
+    //    std::cout << "CUDA MGS" << std::endl;
+    finish_arnoldi(exec, next_krylov_basis, krylov_bases, hessenberg_iter, iter,
+                   stop_status->get_const_data());
+#endif
     givens_rotation(exec, givens_sin, givens_cos, hessenberg_iter,
                     residual_norm, residual_norm_collection, b_norm, iter,
                     stop_status);
diff --git a/cuda/test/solver/gmres_mixed_kernels.cpp b/cuda/test/solver/gmres_mixed_kernels.cpp
index d106f255003..85ebdaf37be 100644
--- a/cuda/test/solver/gmres_mixed_kernels.cpp
+++ b/cuda/test/solver/gmres_mixed_kernels.cpp
@@ -112,7 +112,7 @@ class GmresMixed : public ::testing::Test {
         before_preconditioner = Mtx::create_with_config_of(x.get());
         b = gen_mtx(m, n);
         b_norm = gen_mtx(1, n);
-        arnoldi_norm = gen_mtx(2, n);
+        arnoldi_norm = gen_mtx(3, n);
         gko::dim<2> krylov_bases_dim(
             m, (gko::solver::default_krylov_dim_mixed + 1) * n);
         acc_helper = Accessor2dHelper{krylov_bases_dim, ref};
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index e9c3e8e4071..b0aa3c74d2e 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -112,8 +112,8 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
 
 #pragma omp parallel for reduction(add : hessenberg_iter_entry)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            hessenberg_iter_entry +=
-                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            hessenberg_iter_entry += squared_norm(next_krylov_basis->at(j, i));
+            // next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
         }
         hessenberg_iter->at(iter + 1, i) = sqrt(hessenberg_iter_entry);
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
@@ -136,7 +136,7 @@ void finish_arnoldi_reorth(
     matrix::Dense<ValueType> *next_krylov_basis,
     Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
-    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
     const stopping_status *stop_status)
 {
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
@@ -147,7 +147,7 @@ void finish_arnoldi_reorth(
             continue;
         }
 
-        remove_complex<ValueType> nrm = zero<ValueType>();
+        remove_complex<ValueType> nrm = zero<remove_complex<ValueType>>();
 #pragma omp parallel for reduction(addnc : nrm)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             nrm += squared_norm(next_krylov_basis->at(j, i));
@@ -172,9 +172,11 @@ void finish_arnoldi_reorth(
                     krylov_bases.read(j,
                                       next_krylov_basis->get_size()[1] * k + i);
             }
-            // TODO remove abs() as soon as type for arnoldi_norm has changed!
+            // DONE: TODO remove abs() as soon as type for arnoldi_norm has
+            // changed!
             if (squared_norm(hessenberg_iter->at(k, i)) >
-                abs(arnoldi_norm->at(0, i))) {
+                arnoldi_norm->at(0, i)) {
+                // abs(arnoldi_norm->at(0, i))) {
                 ValueType reorth = zero<ValueType>();
 #pragma omp parallel for reduction(add : reorth)
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
@@ -208,8 +210,8 @@ void finish_arnoldi_reorth(
         ValueType hessenberg_iter_entry = zero<ValueType>();
 #pragma omp parallel for reduction(add : hessenberg_iter_entry)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            hessenberg_iter_entry +=
-                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            hessenberg_iter_entry += squared_norm(next_krylov_basis->at(j, i));
+            // next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
         }
         hessenberg_iter->at(iter + 1, i) = sqrt(hessenberg_iter_entry);
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
@@ -233,10 +235,10 @@ void finish_arnoldi_CGS(
     Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
-    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
     const stopping_status *stop_status)
 {
-    const ValueType eta = 1.0 / sqrt(2.0);
+    const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
 #pragma omp declare reduction(addnc : remove_complex<ValueType> : omp_out = omp_out + omp_in)
 
@@ -279,7 +281,7 @@ void finish_arnoldi_CGS(
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
         // ValueType nrm = zero<ValueType>();
-        nrm = zero<ValueType>();
+        nrm = zero<remove_complex<ValueType>>();
 #pragma omp parallel for reduction(addnc : nrm)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             nrm += squared_norm(next_krylov_basis->at(j, i));
@@ -287,9 +289,12 @@ void finish_arnoldi_CGS(
         arnoldi_norm->at(1, i) = nrm;
         // nrmN = norm(next_krylov_basis)
 
-        // TODO remove the abs() as soon as the type of arnoldi_norm has changed
+        // DONE: TODO remove the abs() as soon as the type of arnoldi_norm has
+        // changed
         for (size_type l = 1;
-             abs(arnoldi_norm->at(1, i)) < abs(arnoldi_norm->at(0, i)) && l < 3;
+             (arnoldi_norm->at(1, i)) < (arnoldi_norm->at(0, i)) && l < 3;
+             // abs(arnoldi_norm->at(1, i)) < abs(arnoldi_norm->at(0, i)) && l <
+             // 3;
              l++) {
             arnoldi_norm->at(0, i) = arnoldi_norm->at(1, i) * eta;
             // nrmP = nrmN
@@ -321,7 +326,7 @@ void finish_arnoldi_CGS(
             // for i in 1:iter
             //     next_krylov_basis  -= buffer(iter, i) * krylov_bases(:, i)
             // end
-            remove_complex<ValueType> nrm = zero<ValueType>();
+            remove_complex<ValueType> nrm = zero<remove_complex<ValueType>>();
 #pragma omp parallel for reduction(addnc : nrm)
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 nrm += squared_norm(next_krylov_basis->at(j, i));
@@ -359,12 +364,15 @@ void finish_arnoldi_CGS2(
     Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
-    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
     const stopping_status *stop_status)
 {
     const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
 #pragma omp declare reduction(addnc : remove_complex<ValueType> : omp_out = omp_out + omp_in)
+#pragma omp declare reduction(infnc : remove_complex<ValueType> : omp_out = (omp_out >= omp_in? omp_out: omp_in))
+    //#pragma omp declare reduction(infnc : remove_complex<ValueType> : omp_out
+    //= (omp_out >= abs(omp_in)? omp_out: abs(omp_in)))
 
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
         if (stop_status[i].has_stopped()) {
@@ -429,16 +437,26 @@ void finish_arnoldi_CGS2(
         // end
         // ValueType nrm = zero<ValueType>();
         nrm = zero<remove_complex<ValueType>>();
-#pragma omp parallel for reduction(addnc : nrm)
+        remove_complex<ValueType> inf = zero<remove_complex<ValueType>>();
+#pragma omp parallel for reduction(addnc : nrm) reduction(infnc : inf)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             nrm += squared_norm(next_krylov_basis->at(j, i));
-        }
-        arnoldi_norm->at(1, i) = nrm;
+            inf = (inf >= abs(next_krylov_basis->at(j, i)))
+                      ? inf
+                      : abs(next_krylov_basis->at(j, i));
+        }
+        arnoldi_norm->at(1, i) = sqrt(nrm);
+        arnoldi_norm->at(2, i) = inf;
+        // std::cout << sqrt(arnoldi_norm->at(1, i)) << " - "
+        //           << arnoldi_norm->at(2, i) << std::endl;
+        // nrmI = infnorm(next_krylov_basis)
         // nrmN = norm(next_krylov_basis)
 
-        // TODO remove abs() as soon as type of arnoldi_norm has changed!
+        // DONE: TODO remove abs() as soon as type of arnoldi_norm has changed!
         for (size_type l = 1;
-             abs(arnoldi_norm->at(1, i)) < abs(arnoldi_norm->at(0, i)) && l < 3;
+             (arnoldi_norm->at(1, i)) < (arnoldi_norm->at(0, i)) && l < 3;
+             // abs(arnoldi_norm->at(1, i)) < abs(arnoldi_norm->at(0, i)) && l <
+             // 3;
              l++) {
             arnoldi_norm->at(0, i) = arnoldi_norm->at(1, i) * eta;
             // nrmP = nrmN
@@ -471,22 +489,30 @@ void finish_arnoldi_CGS2(
             //     next_krylov_basis  -= buffer(iter, i) * krylov_bases(:, i)
             // end
             remove_complex<ValueType> nrm = zero<remove_complex<ValueType>>();
-#pragma omp parallel for reduction(addnc : nrm)
+            remove_complex<ValueType> inf = zero<remove_complex<ValueType>>();
+#pragma omp parallel for reduction(addnc : nrm) reduction(infnc : inf)
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 nrm += squared_norm(next_krylov_basis->at(j, i));
+                inf = (inf >= abs(next_krylov_basis->at(j, i)))
+                          ? inf
+                          : abs(next_krylov_basis->at(j, i));
             }
-            arnoldi_norm->at(1, i) = nrm;
+            arnoldi_norm->at(1, i) = sqrt(nrm);
+            arnoldi_norm->at(2, i) = inf;
             // nrmN = norm(next_krylov_basis)
+            // nrmI = infnorm(next_krylov_basis)
         }
         // reorthogonalization
-
+        /*
         ValueType hessenberg_iter_entry = zero<ValueType>();
 #pragma omp parallel for reduction(add : hessenberg_iter_entry)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            hessenberg_iter_entry +=
-                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+            hessenberg_iter_entry += squared_norm(next_krylov_basis->at(j, i));
+            // next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
         }
         hessenberg_iter->at(iter + 1, i) = sqrt(hessenberg_iter_entry);
+        */
+        hessenberg_iter->at(iter + 1, i) = (arnoldi_norm->at(1, i));
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
 #pragma omp parallel for
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
@@ -567,9 +593,9 @@ void givens_rotation(matrix::Dense<ValueType> *next_krylov_basis,
 template <typename ValueType>
 void calculate_next_residual_norm(
     matrix::Dense<ValueType> *givens_sin, matrix::Dense<ValueType> *givens_cos,
-    matrix::Dense<ValueType> *residual_norm,
+    matrix::Dense<remove_complex<ValueType>> *residual_norm,
     matrix::Dense<ValueType> *residual_norm_collection,
-    const matrix::Dense<ValueType> *b_norm, size_type iter,
+    const matrix::Dense<remove_complex<ValueType>> *b_norm, size_type iter,
     const stopping_status *stop_status)
 {
 #pragma omp parallel for
@@ -638,7 +664,7 @@ void calculate_qy(Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
 template <typename ValueType>
 void initialize_1(std::shared_ptr<const OmpExecutor> exec,
                   const matrix::Dense<ValueType> *b,
-                  matrix::Dense<ValueType> *b_norm,
+                  matrix::Dense<remove_complex<ValueType>> *b_norm,
                   matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *givens_sin,
                   matrix::Dense<ValueType> *givens_cos,
@@ -646,13 +672,14 @@ void initialize_1(std::shared_ptr<const OmpExecutor> exec,
 {
     for (size_type j = 0; j < b->get_size()[1]; ++j) {
         // Calculate b norm
-        ValueType norm = zero<ValueType>();
+        remove_complex<ValueType> norm = zero<remove_complex<ValueType>>();
 
-#pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
+#pragma omp declare reduction(addnc:remove_complex<ValueType> : omp_out = omp_out + omp_in)
 
-#pragma omp parallel for reduction(add : norm)
+#pragma omp parallel for reduction(addnc : norm)
         for (size_type i = 0; i < b->get_size()[0]; ++i) {
-            norm += b->at(i, j) * b->at(i, j);
+            norm += squared_norm(b->at(i, j));
+            // norm += b->at(i, j) * b->at(i, j);
         }
         b_norm->at(0, j) = sqrt(norm);
 
@@ -677,7 +704,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
 template <typename ValueType, typename ValueTypeKrylovBases>
 void initialize_2(std::shared_ptr<const OmpExecutor> exec,
                   const matrix::Dense<ValueType> *residual,
-                  matrix::Dense<ValueType> *residual_norm,
+                  matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
                   Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
@@ -685,13 +712,14 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
 {
     for (size_type j = 0; j < residual->get_size()[1]; ++j) {
         // Calculate residual norm
-        ValueType res_norm = zero<ValueType>();
+        remove_complex<ValueType> res_norm = zero<remove_complex<ValueType>>();
 
-#pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
+#pragma omp declare reduction(addnc:remove_complex<ValueType> : omp_out = omp_out + omp_in)
 
-#pragma omp parallel for reduction(add : res_norm)
+#pragma omp parallel for reduction(addnc : res_norm)
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
-            res_norm += residual->at(i, j) * residual->at(i, j);
+            res_norm += squared_norm(residual->at(i, j));
+            // res_norm += residual->at(i, j) * residual->at(i, j);
         }
         residual_norm->at(0, j) = sqrt(res_norm);
 
@@ -731,14 +759,14 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
             matrix::Dense<ValueType> *next_krylov_basis,
             matrix::Dense<ValueType> *givens_sin,
             matrix::Dense<ValueType> *givens_cos,
-            matrix::Dense<ValueType> *residual_norm,
+            matrix::Dense<remove_complex<ValueType>> *residual_norm,
             matrix::Dense<ValueType> *residual_norm_collection,
             Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
             matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
-            const matrix::Dense<ValueType> *b_norm,
-            matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
-            Array<size_type> *final_iter_nums,
+            const matrix::Dense<remove_complex<ValueType>> *b_norm,
+            matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+            size_type iter, Array<size_type> *final_iter_nums,
             const Array<stopping_status> *stop_status,
             Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
             int *num_reorth_steps, int *num_reorth_vectors)
@@ -748,18 +776,25 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
         final_iter_nums->get_data()[i] +=
             (1 - stop_status->get_const_data()[i].has_stopped());
     }
-    //    finish_arnoldi(next_krylov_basis, krylov_bases, hessenberg_iter, iter,
-    //                   stop_status->get_const_data());
-    //    finish_arnoldi_reorth(next_krylov_basis, krylov_bases,
-    //    hessenberg_iter,
-    //                          arnoldi_norm, iter,
-    //                          stop_status->get_const_data());
-    //    finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
-    //                       buffer_iter, arnoldi_norm, iter,
-    //                       stop_status->get_const_data());
+#if FINISH_ARNOLDI == 1
+    //    std::cout << "OMP MGS_REORTH" << std::endl;
+    finish_arnoldi_reorth(next_krylov_basis, krylov_bases, hessenberg_iter,
+                          arnoldi_norm, iter, stop_status->get_const_data());
+#elif FINISH_ARNOLDI == 2
+    //    std::cout << "OMP CGS_REORTH" << std::endl;
+    finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
+                       buffer_iter, arnoldi_norm, iter,
+                       stop_status->get_const_data());
+#elif FINISH_ARNOLDI == 3
+    //    std::cout << "OMP CGS_REORTH_2" << std::endl;
     finish_arnoldi_CGS2(next_krylov_basis, krylov_bases, hessenberg_iter,
                         buffer_iter, arnoldi_norm, iter,
                         stop_status->get_const_data());
+#else
+    //    std::cout << "OMP MGS" << std::endl;
+    finish_arnoldi(next_krylov_basis, krylov_bases, hessenberg_iter, iter,
+                   stop_status->get_const_data());
+#endif
     givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
                     iter, stop_status->get_const_data());
     calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,
diff --git a/omp/test/solver/gmres_mixed_kernels.cpp b/omp/test/solver/gmres_mixed_kernels.cpp
index 8bd83e6c494..69573980ed7 100644
--- a/omp/test/solver/gmres_mixed_kernels.cpp
+++ b/omp/test/solver/gmres_mixed_kernels.cpp
@@ -114,7 +114,7 @@ class GmresMixed : public ::testing::Test {
         before_preconditioner = Mtx::create_with_config_of(x.get());
         b = gen_mtx(m, n);
         b_norm = gen_mtx(1, n);
-        arnoldi_norm = gen_mtx(2, n);
+        arnoldi_norm = gen_mtx(3, n);
         gko::dim<2> krylov_bases_dim(
             m, (gko::solver::default_krylov_dim_mixed + 1) * n);
         acc_helper = Accessor2dHelper{krylov_bases_dim, ref};
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index 4f95b4a755f..a157f40004b 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -69,7 +69,7 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
             continue;
         }
         for (size_type k = 0; k < iter + 1; ++k) {
-            hessenberg_iter->at(k, i) = 0;
+            hessenberg_iter->at(k, i) = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
                     next_krylov_basis->at(j, i) *
@@ -88,10 +88,11 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
 
-        hessenberg_iter->at(iter + 1, i) = 0;
+        hessenberg_iter->at(iter + 1, i) = zero<ValueType>();
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             hessenberg_iter->at(iter + 1, i) +=
-                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+                squared_norm(next_krylov_basis->at(j, i));
+            // next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
         }
         hessenberg_iter->at(iter + 1, i) =
             sqrt(hessenberg_iter->at(iter + 1, i));
@@ -101,6 +102,12 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
             krylov_bases.write(
                 j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
                 next_krylov_basis->at(j, i));
+            // std::cout << next_krylov_basis->at(j, i) << " - "
+            //           << krylov_bases.read(
+            //                  j,
+            //                  next_krylov_basis->get_size()[1] * (iter + 1) +
+            //                  i)
+            //           << std::endl;
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -114,7 +121,8 @@ void finish_arnoldi_reorth(
     matrix::Dense<ValueType> *next_krylov_basis,
     Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
-    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    matrix::Dense<ValueType> *buffer_iter,
+    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
     const stopping_status *stop_status)
 {
     //    ValueType nrm = 0, reorth = 0;
@@ -124,7 +132,7 @@ void finish_arnoldi_reorth(
             continue;
         }
         // nrm = 0;
-        arnoldi_norm->at(0, i) = 0;
+        arnoldi_norm->at(0, i) = zero<remove_complex<ValueType>>();
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             arnoldi_norm->at(0, i) += squared_norm(next_krylov_basis->at(j, i));
             // nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
@@ -133,7 +141,7 @@ void finish_arnoldi_reorth(
         // nrm = sqrt(nrm) * 0.99;
         // nrm = norm(next_krylov_basis)
         for (size_type k = 0; k < iter + 1; ++k) {
-            hessenberg_iter->at(k, i) = 0;
+            hessenberg_iter->at(k, i) = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
                     next_krylov_basis->at(j, i) *
@@ -146,15 +154,18 @@ void finish_arnoldi_reorth(
                     krylov_bases.read(j,
                                       next_krylov_basis->get_size()[1] * k + i);
             }
-            if (squared_norm(hessenberg_iter->at(k, i) *
-                             hessenberg_iter->at(k, i)) >
+            // if (squared_norm(hessenberg_iter->at(k, i) *
+            //                  hessenberg_iter->at(k, i)) >
+            if (squared_norm(hessenberg_iter->at(k, i)) >
                 arnoldi_norm->at(0, i)) {
-                arnoldi_norm->at(1, i) = 0;
+                buffer_iter->at(k, i) = zero<ValueType>();
+                // arnoldi_norm->at(1, i) = 0;
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
-                    // TODO find good alternative here, so the type of
-                    // arnoldi_norm can change to remove_complex<ValueType>!!!
-                    arnoldi_norm->at(1, i) +=
+                    // DONE: TODO find good alternative here, so the type of
+                    // DONE: arnoldi_norm can change to
+                    // remove_complex<ValueType>!!! arnoldi_norm->at(1, i) +=
+                    buffer_iter->at(k, i) +=
                         next_krylov_basis->at(j, i) *
                         krylov_bases.read(
                             j, next_krylov_basis->get_size()[1] * k + i);
@@ -162,11 +173,13 @@ void finish_arnoldi_reorth(
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
-                        arnoldi_norm->at(1, i) *
+                        buffer_iter->at(k, i) *
+                        // arnoldi_norm->at(1, i) *
                         krylov_bases.read(
                             j, next_krylov_basis->get_size()[1] * k + i);
                 }
-                hessenberg_iter->at(k, i) += arnoldi_norm->at(1, i);
+                // hessenberg_iter->at(k, i) += arnoldi_norm->at(1, i);
+                hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
             }
         }
         // for i in 1:iter
@@ -179,10 +192,11 @@ void finish_arnoldi_reorth(
         //     end
         // end
 
-        hessenberg_iter->at(iter + 1, i) = 0;
+        hessenberg_iter->at(iter + 1, i) = zero<ValueType>();
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             hessenberg_iter->at(iter + 1, i) +=
-                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+                squared_norm(next_krylov_basis->at(j, i));
+            // next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
         }
         hessenberg_iter->at(iter + 1, i) =
             sqrt(hessenberg_iter->at(iter + 1, i));
@@ -206,14 +220,14 @@ void finish_arnoldi_CGS(
     Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
-    matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
+    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
     const stopping_status *stop_status)
 {
     const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
     //    ValueType nrmP = 0, nrmN = 0;
-    ValueType nrmP = 0, nrmN = 0;
+    //    ValueType nrmP = 0, nrmN = 0;
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
-        arnoldi_norm->at(0, i) = 0;
+        arnoldi_norm->at(0, i) = zero<remove_complex<ValueType>>();
         // nrmP = 0;
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             arnoldi_norm->at(0, i) += squared_norm(next_krylov_basis->at(j, i));
@@ -227,7 +241,7 @@ void finish_arnoldi_CGS(
             continue;
         }
         for (size_type k = 0; k < iter + 1; ++k) {
-            hessenberg_iter->at(k, i) = 0;
+            hessenberg_iter->at(k, i) = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
                     next_krylov_basis->at(j, i) *
@@ -249,21 +263,21 @@ void finish_arnoldi_CGS(
         // for i in 1:iter
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
-        arnoldi_norm->at(1, i) = 0;
+        arnoldi_norm->at(1, i) = zero<remove_complex<ValueType>>();
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             arnoldi_norm->at(1, i) += squared_norm(next_krylov_basis->at(j, i));
         }
         arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
         // nrmN = norm(next_krylov_basis)
 
-        // TODO the abs() should be removed here as soon as the type of
-        // arnoldi_norm has changed!
+        // DONE: TODO the abs() should be removed here as soon as the type of
+        // DONE: arnoldi_norm has changed!
         for (size_type l = 1;
-             abs(arnoldi_norm->at(1, i)) < abs(arnoldi_norm->at(0, i)) && l < 3;
+             (arnoldi_norm->at(1, i)) < (arnoldi_norm->at(0, i)) && l < 3;
              l++) {
             arnoldi_norm->at(0, i) = eta * arnoldi_norm->at(1, i);
             for (size_type k = 0; k < iter + 1; ++k) {
-                buffer_iter->at(k, i) = 0;
+                buffer_iter->at(k, i) = zero<ValueType>();
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     buffer_iter->at(k, i) +=
@@ -289,7 +303,7 @@ void finish_arnoldi_CGS(
             //     next_krylov_basis   -= buffer(iter, i) * krylov_bases(:, i)
             //     hessenberg(iter, i) += buffer(iter, i)
             // end
-            arnoldi_norm->at(1, i) = 0;
+            arnoldi_norm->at(1, i) = zero<remove_complex<ValueType>>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 arnoldi_norm->at(1, i) +=
                     squared_norm(next_krylov_basis->at(j, i));
@@ -298,10 +312,11 @@ void finish_arnoldi_CGS(
             // nrmN = norm(next_krylov_basis)
         }
         // reorthogonalization
-        hessenberg_iter->at(iter + 1, i) = 0;
+        hessenberg_iter->at(iter + 1, i) = zero<ValueType>();
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             hessenberg_iter->at(iter + 1, i) +=
-                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+                squared_norm(next_krylov_basis->at(j, i));
+            // next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
         }
         hessenberg_iter->at(iter + 1, i) =
             sqrt(hessenberg_iter->at(iter + 1, i));
@@ -319,6 +334,148 @@ void finish_arnoldi_CGS(
 }
 
 
+template <typename ValueType, typename ValueTypeKrylovBases>
+void finish_arnoldi_CGS2(
+    matrix::Dense<ValueType> *next_krylov_basis,
+    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    matrix::Dense<ValueType> *hessenberg_iter,
+    matrix::Dense<ValueType> *buffer_iter,
+    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
+    const stopping_status *stop_status)
+{
+    const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
+    //    ValueType nrmP = 0, nrmN = 0;
+    //    ValueType nrmP = 0, nrmN = 0;
+    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
+        arnoldi_norm->at(0, i) = zero<remove_complex<ValueType>>();
+        // nrmP = 0;
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            arnoldi_norm->at(0, i) += squared_norm(next_krylov_basis->at(j, i));
+            // nrmP += next_krylov_basis->at(j, i) * next_krylov_basis->at(j,
+            // i);
+        }
+        arnoldi_norm->at(0, i) = sqrt(arnoldi_norm->at(0, i)) * eta;
+        // nrmP = sqrt(nrmP);
+        // arnoldi_norm->at(0, i) = norm(next_krylov_basis)
+        if (stop_status[i].has_stopped()) {
+            continue;
+        }
+        for (size_type k = 0; k < iter + 1; ++k) {
+            hessenberg_iter->at(k, i) = zero<ValueType>();
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                hessenberg_iter->at(k, i) +=
+                    next_krylov_basis->at(j, i) *
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
+            }
+        }
+        // for i in 1:iter
+        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+        // end
+        for (size_type k = 0; k < iter + 1; ++k) {
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                next_krylov_basis->at(j, i) -=
+                    hessenberg_iter->at(k, i) *
+                    krylov_bases.read(j,
+                                      next_krylov_basis->get_size()[1] * k + i);
+            }
+        }
+        // for i in 1:iter
+        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+        // end
+        arnoldi_norm->at(1, i) = zero<remove_complex<ValueType>>();
+        arnoldi_norm->at(2, i) = zero<remove_complex<ValueType>>();
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            arnoldi_norm->at(1, i) += squared_norm(next_krylov_basis->at(j, i));
+            arnoldi_norm->at(2, i) =
+                (arnoldi_norm->at(2, i) >= abs(next_krylov_basis->at(j, i)))
+                    ? arnoldi_norm->at(2, i)
+                    : abs(next_krylov_basis->at(j, i));
+        }
+        arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
+        // std::cout << arnoldi_norm->at(1, i) << " - " << arnoldi_norm->at(2,
+        // i)
+        //           << " - " << 0 << std::endl;
+        // nrmN = norm(next_krylov_basis)
+
+        // DONE: TODO the abs() should be removed here as soon as the type of
+        // DONE: arnoldi_norm has changed!
+        for (size_type l = 1;
+             (arnoldi_norm->at(1, i)) < (arnoldi_norm->at(0, i)) && l < 3;
+             l++) {
+            // std::cout << "REF RESTART " << l << std::endl;
+            arnoldi_norm->at(0, i) = eta * arnoldi_norm->at(1, i);
+            for (size_type k = 0; k < iter + 1; ++k) {
+                buffer_iter->at(k, i) = zero<ValueType>();
+                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
+                     ++j) {
+                    buffer_iter->at(k, i) +=
+                        next_krylov_basis->at(j, i) *
+                        krylov_bases.read(
+                            j, next_krylov_basis->get_size()[1] * k + i);
+                }
+            }
+            // for i in 1:iter
+            //     buffer(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+            // end
+            for (size_type k = 0; k < iter + 1; ++k) {
+                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
+                     ++j) {
+                    next_krylov_basis->at(j, i) -=
+                        buffer_iter->at(k, i) *
+                        krylov_bases.read(
+                            j, next_krylov_basis->get_size()[1] * k + i);
+                }
+                hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
+            }
+            // for i in 1:iter
+            //     next_krylov_basis   -= buffer(iter, i) * krylov_bases(:, i)
+            //     hessenberg(iter, i) += buffer(iter, i)
+            // end
+            arnoldi_norm->at(1, i) = zero<remove_complex<ValueType>>();
+            arnoldi_norm->at(2, i) = zero<remove_complex<ValueType>>();
+            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+                arnoldi_norm->at(1, i) +=
+                    squared_norm(next_krylov_basis->at(j, i));
+                arnoldi_norm->at(2, i) =
+                    (arnoldi_norm->at(2, i) >= abs(next_krylov_basis->at(j, i)))
+                        ? arnoldi_norm->at(2, i)
+                        : abs(next_krylov_basis->at(j, i));
+            }
+            arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
+            // std::cout << arnoldi_norm->at(1, i) << " - "
+            //           << arnoldi_norm->at(2, i) << " - " << l << std::endl;
+            // nrmN = norm(next_krylov_basis)
+        }
+        /*
+        // reorthogonalization
+        hessenberg_iter->at(iter + 1, i) = zero<ValueType>();
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            hessenberg_iter->at(iter + 1, i) +=
+                squared_norm(next_krylov_basis->at(j, i));
+            // next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
+        }
+        hessenberg_iter->at(iter + 1, i) =
+            sqrt(hessenberg_iter->at(iter + 1, i));
+        */
+        hessenberg_iter->at(iter + 1, i) = arnoldi_norm->at(1, i);
+        // std::cout << arnoldi_norm->at(1, i) << " - " << arnoldi_norm->at(2,
+        // i)
+        //           << " - " << hessenberg_iter->at(iter + 1, i) << std::endl;
+        // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
+        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
+            next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
+            krylov_bases.write(
+                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
+                next_krylov_basis->at(j, i));
+        }
+        // next_krylov_basis /= hessenberg(iter, iter + 1)
+        // krylov_bases(:, iter + 1) = next_krylov_basis
+        // End of arnoldi
+    }
+}
+
+
 template <typename ValueType>
 void calculate_sin_and_cos(matrix::Dense<ValueType> *givens_sin,
                            matrix::Dense<ValueType> *givens_cos,
@@ -383,9 +540,9 @@ void givens_rotation(matrix::Dense<ValueType> *next_krylov_basis,
 template <typename ValueType>
 void calculate_next_residual_norm(
     matrix::Dense<ValueType> *givens_sin, matrix::Dense<ValueType> *givens_cos,
-    matrix::Dense<ValueType> *residual_norm,
+    matrix::Dense<remove_complex<ValueType>> *residual_norm,
     matrix::Dense<ValueType> *residual_norm_collection,
-    const matrix::Dense<ValueType> *b_norm, size_type iter,
+    const matrix::Dense<remove_complex<ValueType>> *b_norm, size_type iter,
     const stopping_status *stop_status)
 {
     for (size_type i = 0; i < residual_norm->get_size()[1]; ++i) {
@@ -451,7 +608,7 @@ void calculate_qy(Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
 template <typename ValueType>
 void initialize_1(std::shared_ptr<const ReferenceExecutor> exec,
                   const matrix::Dense<ValueType> *b,
-                  matrix::Dense<ValueType> *b_norm,
+                  matrix::Dense<remove_complex<ValueType>> *b_norm,
                   matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *givens_sin,
                   matrix::Dense<ValueType> *givens_cos,
@@ -459,9 +616,10 @@ void initialize_1(std::shared_ptr<const ReferenceExecutor> exec,
 {
     for (size_type j = 0; j < b->get_size()[1]; ++j) {
         // Calculate b norm
-        b_norm->at(0, j) = zero<ValueType>();
+        b_norm->at(0, j) = zero<remove_complex<ValueType>>();
         for (size_type i = 0; i < b->get_size()[0]; ++i) {
-            b_norm->at(0, j) += b->at(i, j) * b->at(i, j);
+            b_norm->at(0, j) += squared_norm(b->at(i, j));
+            // b_norm->at(0, j) += b->at(i, j) * b->at(i, j);
         }
         b_norm->at(0, j) = sqrt(b_norm->at(0, j));
 
@@ -483,7 +641,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
 template <typename ValueType, typename ValueTypeKrylovBases>
 void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
                   const matrix::Dense<ValueType> *residual,
-                  matrix::Dense<ValueType> *residual_norm,
+                  matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
                   Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
@@ -491,9 +649,11 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
 {
     for (size_type j = 0; j < residual->get_size()[1]; ++j) {
         // Calculate residual norm
-        residual_norm->at(0, j) = 0;
+        residual_norm->at(0, j) = zero<remove_complex<ValueType>>();
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
-            residual_norm->at(0, j) += residual->at(i, j) * residual->at(i, j);
+            residual_norm->at(0, j) += squared_norm(residual->at(i, j));
+            // residual_norm->at(0, j) += residual->at(i, j) * residual->at(i,
+            // j);
         }
         residual_norm->at(0, j) = sqrt(residual_norm->at(0, j));
 
@@ -532,14 +692,14 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
             matrix::Dense<ValueType> *next_krylov_basis,
             matrix::Dense<ValueType> *givens_sin,
             matrix::Dense<ValueType> *givens_cos,
-            matrix::Dense<ValueType> *residual_norm,
+            matrix::Dense<remove_complex<ValueType>> *residual_norm,
             matrix::Dense<ValueType> *residual_norm_collection,
             Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
             matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
-            const matrix::Dense<ValueType> *b_norm,
-            matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
-            Array<size_type> *final_iter_nums,
+            const matrix::Dense<remove_complex<ValueType>> *b_norm,
+            matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+            size_type iter, Array<size_type> *final_iter_nums,
             const Array<stopping_status> *stop_status,
             Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
             int *num_reorth_steps, int *num_reorth_vectors)
@@ -554,15 +714,27 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
         final_iter_nums->get_data()[i] +=
             (1 - stop_status->get_const_data()[i].has_stopped());
     }
-
-    //    finish_arnoldi(next_krylov_basis, krylov_bases, hessenberg_iter, iter,
-    //                   stop_status->get_const_data());
-    //    finish_arnoldi_reorth(next_krylov_basis, krylov_bases,
-    //                          hessenberg_iter, arnoldi_norm, iter,
-    //                          stop_status->get_const_data());
+#if FINISH_ARNOLDI == 1
+    //    std::cout << "REF MGS_REORTH" << std::endl;
+    finish_arnoldi_reorth(next_krylov_basis, krylov_bases, hessenberg_iter,
+                          buffer_iter, arnoldi_norm, iter,
+                          stop_status->get_const_data());
+
+#elif FINISH_ARNOLDI == 2
+    //    std::cout << "REF CGS_REORTH" << std::endl;
     finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
                        buffer_iter, arnoldi_norm, iter,
                        stop_status->get_const_data());
+#elif FINISH_ARNOLDI == 3
+    //    std::cout << "REF CGS_REORTH_2" << std::endl;
+    finish_arnoldi_CGS2(next_krylov_basis, krylov_bases, hessenberg_iter,
+                        buffer_iter, arnoldi_norm, iter,
+                        stop_status->get_const_data());
+#else
+    //    std::cout << "REF MGS" << std::endl;
+    finish_arnoldi(next_krylov_basis, krylov_bases, hessenberg_iter, iter,
+                   stop_status->get_const_data());
+#endif
     givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
                     iter, stop_status->get_const_data());
     calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,

From 94f9aebb247ad36bdecb1417b683468c917a3b55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 25 May 2020 08:11:57 +0200
Subject: [PATCH 0353/1008] Fixed cuda step2 to take a view into account

Previously, the full krylov_bases were always used, regardless if a
view was used or not. Now, the dimensions for krylov_bases are derived
from hessenberg to take a potential view into account.
---
 cuda/solver/gmres_mixed_kernels.cu | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 0c2fcb2de95..c39bfa2c371 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -1089,8 +1089,11 @@ void step_2(std::shared_ptr<const CudaExecutor> exec,
             matrix::Dense<ValueType> *before_preconditioner,
             const Array<size_type> *final_iter_nums)
 {
-    const auto res_norm_col_size = residual_norm_collection->get_size();
-    const auto krylov_cols = res_norm_col_size[0] * res_norm_col_size[1];
+    // since hessenberg has dims:  iters x iters * num_rhs
+    // and krylov should have dims:  sysmtx[0] x (iters + 1) * num_rhs
+    const auto num_rhs = before_preconditioner->get_size()[1];
+    const auto hessenberg_cols = hessenberg->get_size()[1];
+    const auto krylov_cols = hessenberg_cols + num_rhs;
     solve_upper_triangular(residual_norm_collection, hessenberg, y,
                            final_iter_nums);
     calculate_qy(krylov_bases, krylov_cols, y, before_preconditioner,

From c6bcffae972d39530ab17e682703743cdbb7c13a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 25 May 2020 11:02:05 +0200
Subject: [PATCH 0354/1008] Change const accessor to non-const in
 check_arnoldi_norms_new

This allows it to compile with the writing of the scale.
---
 common/solver/gmres_mixed_kernels.hpp.inc | 14 +++++---------
 cuda/solver/gmres_mixed_kernels.cu        |  7 +++----
 2 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 1395407ee44..c8797a3ee1a 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -47,7 +47,8 @@ __global__ __launch_bounds__(default_block_size) void zero_matrix_kernel(
 
 template <typename ValueType, typename KrylovType,
           bool = Accessor2d<KrylovType, ValueType>::has_scale>
-struct helper_functions_accessor {};
+struct helper_functions_accessor {
+};
 
 // Accessors having a scale
 template <typename ValueType, typename KrylovType>
@@ -1149,7 +1150,6 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms(
     const auto global_id = thread::get_thread_id_flat();
 
     if (global_id < stride_norm0 && !stop_status[global_id].has_stopped()) {
-        // TODO remove abs() as soon as type of arnoldi_norm has changed
         const auto num0 = (eta * arnoldi_norm0[global_id]);
         // const auto num0 = abs(eta * arnoldi_norm0[global_id]);
         const auto num1 = (arnoldi_norm1[global_id]);
@@ -1171,8 +1171,7 @@ template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
     remove_complex<ValueType> *__restrict__ arnoldi_norm, size_type stride_norm,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
-    size_type iter,
-    Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    size_type iter, Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
     const stopping_status *__restrict__ stop_status,
     stopping_status *__restrict__ reorth_status,
     size_type *__restrict__ num_reorth)
@@ -1181,7 +1180,6 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
     const auto col_idx = thread::get_thread_id_flat();
 
     if (col_idx < stride_norm && !stop_status[col_idx].has_stopped()) {
-        // TODO remove abs() as soon as type of arnoldi_norm has changed
         const auto num0 = (eta * arnoldi_norm[col_idx]);
         // const auto num0 = abs(eta * arnoldi_norm0[col_idx]);
         const auto num11 = (arnoldi_norm[col_idx + stride_norm]);
@@ -1200,10 +1198,8 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
         arnoldi_norm[col_idx] = num11;
         arnoldi_norm[col_idx + stride_norm] = num12;
         hessenberg_iter[iter * stride_hessenberg + col_idx] = num12;
-        // helper_functions_accessor<ValueType,
-        // ValueTypeKrylovBases>::write_scale(
-        //     krylov_bases, col_idx, num2 / num12);
-        // krylov_bases.set_scale(col_idx, num2 / num12);
+        helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
+            krylov_bases, col_idx, num2 / num12);
     }
 }
 
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index c39bfa2c371..d570f0ae682 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -765,9 +765,8 @@ void finish_arnoldi_CGS2(
         <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
             as_cuda_type(arnoldi_norm->get_values()), stride_arnoldi,
             as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-            iter + 1, as_cuda_accessor(krylov_bases.to_const()),
-            as_cuda_type(stop_status), as_cuda_type(reorth_status),
-            as_cuda_type(num_reorth->get_data()));
+            iter + 1, as_cuda_accessor(krylov_bases), as_cuda_type(stop_status),
+            as_cuda_type(reorth_status), as_cuda_type(num_reorth->get_data()));
     /* */
 #ifdef TIMING
     exec->synchronize();
@@ -897,7 +896,7 @@ void finish_arnoldi_CGS2(
             <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
                 as_cuda_type(arnoldi_norm->get_values()), stride_arnoldi,
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-                iter + 1, as_cuda_accessor(krylov_bases.to_const()),
+                iter + 1, as_cuda_accessor(krylov_bases),
                 as_cuda_type(stop_status), as_cuda_type(reorth_status),
                 as_cuda_type(num_reorth->get_data()));
         /* */

From 39539fb044117e7033b637e2e02eeb495d2e62a6 Mon Sep 17 00:00:00 2001
From: josealiaga <aliaga@uji.es>
Date: Wed, 27 May 2020 11:40:00 +0200
Subject: [PATCH 0355/1008] The set_scale method finally works!!

I have also tested the <double, long int> case and it also works.
---
 common/solver/gmres_mixed_kernels.hpp.inc     | 98 ++++++++++++++++++-
 core/solver/gmres_mixed.cpp                   | 14 ++-
 core/solver/gmres_mixed_accessor.hpp          | 56 ++++++++++-
 core/solver/gmres_mixed_kernels.hpp           |  4 +
 cuda/solver/gmres_mixed_kernels.cu            | 33 +++++++
 cuda/test/solver/gmres_mixed_kernels.cpp      | 10 +-
 .../gmres-mixed-solver-jacobi-matrix.cpp      | 47 +++++++--
 omp/solver/gmres_mixed_kernels.cpp            | 24 ++++-
 omp/test/solver/gmres_mixed_kernels.cpp       | 10 +-
 reference/solver/gmres_mixed_kernels.cpp      | 16 +++
 10 files changed, 283 insertions(+), 29 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index c8797a3ee1a..9eb495d8ae4 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -45,10 +45,10 @@ __global__ __launch_bounds__(default_block_size) void zero_matrix_kernel(
 }
 
 
+/*
 template <typename ValueType, typename KrylovType,
           bool = Accessor2d<KrylovType, ValueType>::has_scale>
-struct helper_functions_accessor {
-};
+struct helper_functions_accessor {};
 
 // Accessors having a scale
 template <typename ValueType, typename KrylovType>
@@ -76,7 +76,7 @@ struct helper_functions_accessor<ValueType, KrylovType, false> {
 // calling it with:
 // helper_functions_accessor<ValueType,
 // ValueTypeKrylovBases>::write_scale(krylov_bases, col_idx, value);
-
+*/
 
 // Must be called with at least `max(stride_b * num_rows, krylov_dim *
 // num_cols)` threads in total.
@@ -278,6 +278,58 @@ __global__ __launch_bounds__(default_dot_size) void multinorminf_kernel(
 }
 
 
+template <typename ValueType>
+__global__
+    __launch_bounds__(default_dot_size) void multinorminf_kernel_without_stop(
+        size_type num_rows, size_type num_cols,
+        const ValueType *__restrict__ next_krylov_basis,
+        size_type stride_next_krylov,
+        remove_complex<ValueType> *__restrict__ norms, size_type stride_norms)
+{
+    const auto tidx = threadIdx.x;
+    const auto tidy = threadIdx.y;
+    const auto col_idx = blockIdx.x * default_dot_dim + tidx;
+    const auto num = ceildiv(num_rows, gridDim.y);
+    const auto start_row = blockIdx.y * num;
+    const auto end_row =
+        ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+    __shared__ UninitializedArray<remove_complex<ValueType>,
+                                  default_dot_dim *(default_dot_dim + 1)>
+        reduction_helper_array;
+    remove_complex<ValueType> *__restrict__ reduction_helper =
+        reduction_helper_array;
+    remove_complex<ValueType> local_max = zero<remove_complex<ValueType>>();
+    if (col_idx < num_cols) {
+        for (size_type i = start_row + tidy; i < end_row;
+             i += default_dot_dim) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            local_max = (local_max >= abs(next_krylov_basis[next_krylov_idx]))
+                            ? local_max
+                            : abs(next_krylov_basis[next_krylov_idx]);
+            // local_max = (local_max >= next_krylov_basis[next_krylov_idx])
+            //                 ? local_max
+            //                 : next_krylov_basis[next_krylov_idx];
+        }
+    }
+    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_max;
+    __syncthreads();
+    local_max = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
+    const auto tile_block =
+        group::tiled_partition<default_dot_dim>(group::this_thread_block());
+    const auto value = reduce(
+        tile_block, local_max,
+        [](const remove_complex<ValueType> &a,
+           const remove_complex<ValueType> &b) { return ((a >= b) ? a : b); });
+    const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
+    if (tidx == 0 && new_col_idx < num_cols) {
+        const auto norms_idx = new_col_idx;
+        atomic_max(norms + norms_idx, value);
+    }
+}
+
+
 template <typename ValueType>
 __global__ __launch_bounds__(default_dot_size) void multinorm2_inf_kernel(
     size_type num_rows, size_type num_cols,
@@ -1198,8 +1250,46 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
         arnoldi_norm[col_idx] = num11;
         arnoldi_norm[col_idx + stride_norm] = num12;
         hessenberg_iter[iter * stride_hessenberg + col_idx] = num12;
+        // printf("(%d) num1 = %10.5e , num2 = %10.5e\n", cod_idx, num1, num2);
+        // printf("%d - %d - %d - %d\n", stride_norm, iter, col_idx,
+        //        stride_norm * iter + col_idx);
         helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
-            krylov_bases, col_idx, num2 / num12);
+            krylov_bases, stride_norm * iter + col_idx, num2 / num12);
+        // krylov_bases, num_cols * (iter + 1) + col_idx, num2 / num12);
+        // krylov_bases, col_idx, num2 / num12);
+    }
+}
+
+
+// Must be called with at least `num_cols` blocks, each with `block_size`
+// threads. `block_size` must be a power of 2.
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__ __launch_bounds__(block_size) void set_scale_kernel(
+    size_type num_rhs, size_type num_blocks,
+    const remove_complex<ValueType> *__restrict__ residual_norm,
+    size_type stride_residual,
+    const remove_complex<ValueType> *__restrict__ arnoldi_inf,
+    size_type stride_inf,
+    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto blk_idx = global_id / num_rhs;
+    const auto col_idx = global_id % num_rhs;
+
+    if (blk_idx < num_blocks && col_idx < num_rhs) {
+        if (blk_idx == 0) {
+            const auto num1 = residual_norm[col_idx];
+            const auto num2 = arnoldi_inf[col_idx];
+            // printf("num1 = %10.5e , num2 = %10.5e\n", num1, num2);
+            helper_functions_accessor<
+                ValueType, ValueTypeKrylovBases>::write_scale(krylov_bases,
+                                                              col_idx,
+                                                              num2 / num1);
+        } else {
+            const auto num = one<remove_complex<ValueType>>();
+            helper_functions_accessor<ValueType, ValueTypeKrylovBases>::
+                write_scale(krylov_bases, blk_idx * num_rhs + col_idx, num);
+        }
     }
 }
 
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index c6eef6cbd3e..6802447e102 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -51,7 +51,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/gmres_mixed_kernels.hpp"
 
 
-// #define TIMING 1
+#define TIMING 1
 
 
 #ifdef TIMING
@@ -142,6 +142,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     Array<size_type> num_reorth(this->get_executor(), dense_b->get_size()[1]);
     int num_restarts = 0, num_reorth_steps = 0, num_reorth_vectors = 0;
 
+    // std::cout << "Before initializate_1" << std::endl;
     // Initialization
     exec->run(gmres_mixed::make_initialize_1(
         dense_b, b_norm.get(), residual.get(), givens_sin.get(),
@@ -153,10 +154,11 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
                           residual.get());
     // residual = residual - Ax
 
+    // std::cout << "Before initializate_2" << std::endl;
     exec->run(gmres_mixed::make_initialize_2(
         residual.get(), residual_norm.get(), residual_norm_collection.get(),
-        krylov_bases_accessor, next_krylov_basis.get(), &final_iter_nums,
-        krylov_dim_mixed_));
+        arnoldi_norm.get(), krylov_bases_accessor, next_krylov_basis.get(),
+        &final_iter_nums, krylov_dim_mixed_));
     // residual_norm = norm(residual)
     // residual_norm_collection = {residual_norm, 0, ..., 0}
     // krylov_bases(:, 1) = residual / residual_norm
@@ -185,6 +187,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
 #endif
 #endif
 
+    // std::cout << "Before loop" << std::endl;
     while (true) {
         ++total_iter;
         this->template log<log::Logger::iteration_complete>(
@@ -247,8 +250,9 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
             // residual = residual - Ax
             exec->run(gmres_mixed::make_initialize_2(
                 residual.get(), residual_norm.get(),
-                residual_norm_collection.get(), krylov_bases_accessor,
-                next_krylov_basis.get(), &final_iter_nums, krylov_dim_mixed_));
+                residual_norm_collection.get(), arnoldi_norm.get(),
+                krylov_bases_accessor, next_krylov_basis.get(),
+                &final_iter_nums, krylov_dim_mixed_));
             // residual_norm = norm(residual)
             // residual_norm_collection = {residual_norm, 0, ..., 0}
             // krylov_bases(:, 1) = residual / residual_norm
diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index e013dec94ae..550f23123dc 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -59,8 +59,11 @@ using place_holder_type = float;
 
 
 template <typename StorageType, typename ArithmeticType,
-          bool = std::is_same<StorageType, place_holder_type>::value &&
-                 !std::is_same<StorageType, ArithmeticType>::value>
+          bool = (std::is_same<StorageType, place_holder_type>::value &&
+                  !std::is_same<StorageType, ArithmeticType>::value) ||
+                 std::is_integral<StorageType>::value>
+// bool = std::is_same<StorageType, place_holder_type>::value &&
+//        !std::is_same<StorageType, ArithmeticType>::value>
 struct helper_have_scale {};
 
 template <typename StorageType, typename ArithmeticType>
@@ -339,7 +342,18 @@ class Accessor2d<StorageType, ArithmeticType, true>
     GKO_ATTRIBUTES void set_scale(IndexType idx, arithmetic_type val)
     {
         arithmetic_type *GKO_RESTRICT rest_scale = this->scale_;
-        rest_scale[idx] = val;
+        storage_type max_val = one<storage_type>();
+        // printf("(A) max_val = %d\n", max_val);
+        // if (std::is_integer<storage_type>::value) {
+        if (std::is_integral<storage_type>::value) {
+            max_val = std::numeric_limits<storage_type>::max();
+            //    printf("(B) max_val = %ld\n", max_val);
+        }
+        rest_scale[idx] = val / static_cast<arithmetic_type>(max_val);
+        //    rest_scale[idx] = one<arithmetic_type>();
+        //    printf("val = %10.5e , rest_scale = %10.5e\n", val,
+        //    rest_scale[idx]); std::cout << val << " - " << rest_scale[idx] <<
+        //    std::endl;
     }
 
     GKO_ATTRIBUTES storage_type *get_storage() { return this->storage_; }
@@ -417,6 +431,42 @@ class Accessor2dHelper<ValueType, ValueTypeKrylovBases, false> {
     Array<ValueTypeKrylovBases> bases_;
 };
 
+//----------------------------------------------
+
+template <typename ValueType, typename KrylovType,
+          bool = Accessor2d<KrylovType, ValueType>::has_scale>
+struct helper_functions_accessor {};
+
+// Accessors having a scale
+template <typename ValueType, typename KrylovType>
+struct helper_functions_accessor<ValueType, KrylovType, true> {
+    template <typename IndexType>
+    static inline GKO_ATTRIBUTES void write_scale(
+        // static __forceinline__ __device__ void write_scale(
+        Accessor2d<KrylovType, ValueType> krylov_bases, IndexType col_idx,
+        ValueType value)
+    {
+        krylov_bases.set_scale(col_idx, value);
+    }
+};
+
+
+// Accessors not having a scale
+template <typename ValueType, typename KrylovType>
+struct helper_functions_accessor<ValueType, KrylovType, false> {
+    template <typename IndexType>
+    static inline GKO_ATTRIBUTES void write_scale(
+        // static __forceinline__ __device__ void write_scale(
+        Accessor2d<KrylovType, ValueType> krylov_bases, IndexType col_idx,
+        ValueType value)
+    {}
+};
+
+// calling it with:
+// helper_functions_accessor<ValueType,
+// ValueTypeKrylovBases>::write_scale(krylov_bases, col_idx, value);
+
+//----------------------------------------------
 
 }  // namespace kernels
 }  // namespace gko
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index 55a357689ff..c429512dbe9 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -67,6 +67,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro)        \
     template _macro(double, double);                             \
     template _macro(double, float);                              \
+    template _macro(double, int64);                              \
+    template _macro(double, int32);                              \
+    template _macro(double, int16);                              \
     template _macro(float, float);                               \
     template _macro(std::complex<double>, std::complex<double>); \
     template _macro(std::complex<double>, std::complex<float>);  \
@@ -94,6 +97,7 @@ namespace gmres_mixed {
                       const matrix::Dense<_type1> *residual,                \
                       matrix::Dense<remove_complex<_type1>> *residual_norm, \
                       matrix::Dense<_type1> *residual_norm_collection,      \
+                      matrix::Dense<remove_complex<_type1>> *arnoldi_norm,  \
                       Accessor2d<_type2, _type1> krylov_bases,              \
                       matrix::Dense<_type1> *next_krylov_basis,             \
                       Array<size_type> *final_iter_nums, size_type krylov_dim)
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index d570f0ae682..35526647902 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -170,6 +170,7 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
                   const matrix::Dense<ValueType> *residual,
                   matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
+                  matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
                   Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
@@ -182,13 +183,43 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
     const dim3 block_dim(default_block_size, 1, 1);
     constexpr auto block_size = default_block_size;
 
+    // exec->synchronize();
+    // std::cout << "Before initialize_2_1_kernel" << std::endl;
     initialize_2_1_kernel<block_size><<<grid_dim_1, block_dim>>>(
         residual->get_size()[0], residual->get_size()[1], krylov_dim,
         as_cuda_accessor(krylov_bases),
         as_cuda_type(residual_norm_collection->get_values()),
         residual_norm_collection->get_stride());
+    // exec->synchronize();
     residual->compute_norm2(residual_norm);
+    // exec->synchronize();
+
+    // std::cout << "Before multinorminf_kernel_without_stop" << std::endl;
+    components::fill_array(exec, arnoldi_norm->get_values() + 2 * num_rhs,
+                           num_rhs, zero<remove_complex<ValueType>>());
+    const dim3 grid_size_nrm(ceildiv(num_rhs, default_dot_dim),
+                             exec->get_num_multiprocessor() * 2);
+    const dim3 block_size_nrm(default_dot_dim, default_dot_dim);
+    multinorminf_kernel_without_stop<<<grid_size_nrm, block_size_nrm>>>(
+        num_rows, num_rhs, as_cuda_type(residual->get_const_values()), num_rhs,
+        as_cuda_type(arnoldi_norm->get_values() + 2 * num_rhs), 0);
+    // exec->synchronize();
 
+    // helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
+    //     krylov_bases, col_idx, arnoldi_norm->at(2, j) / residual_norm->at(0,
+    //     j));
+    /* */
+    // std::cout << "Before set_scale_kernel" << std::endl;
+    set_scale_kernel<default_block_size>
+        <<<ceildiv(num_rhs * (krylov_dim + 1), default_block_size),
+           default_block_size>>>(
+            num_rhs, krylov_dim + 1,
+            as_cuda_type(residual_norm->get_const_values()), num_rhs,
+            as_cuda_type(arnoldi_norm->get_const_values() + 2 * num_rhs),
+            num_rhs, as_cuda_accessor(krylov_bases));
+    // exec->synchronize();
+    /* */
+    // std::cout << "Before initialize_2_2_kernel" << std::endl;
     const dim3 grid_dim_2(ceildiv(num_rows * num_rhs, default_block_size), 1,
                           1);
     initialize_2_2_kernel<block_size><<<grid_dim_2, block_dim>>>(
@@ -200,6 +231,8 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
         as_cuda_type(next_krylov_basis->get_values()),
         next_krylov_basis->get_stride(),
         as_cuda_type(final_iter_nums->get_data()));
+    // exec->synchronize();
+    // std::cout << "After initialize_2_2_kernel" << std::endl;
 }
 
 GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
diff --git a/cuda/test/solver/gmres_mixed_kernels.cpp b/cuda/test/solver/gmres_mixed_kernels.cpp
index 85ebdaf37be..5b66f6d0e42 100644
--- a/cuda/test/solver/gmres_mixed_kernels.cpp
+++ b/cuda/test/solver/gmres_mixed_kernels.cpp
@@ -289,14 +289,14 @@ TEST_F(GmresMixed, CudaGmresMixedInitialize2IsEquivalentToRef)
 
     gko::kernels::reference::gmres_mixed::initialize_2(
         ref, residual.get(), residual_norm.get(),
-        residual_norm_collection.get(), krylov_bases_accessor,
-        next_krylov_basis.get(), final_iter_nums.get(),
+        residual_norm_collection.get(), arnoldi_norm.get(),
+        krylov_bases_accessor, next_krylov_basis.get(), final_iter_nums.get(),
         gko::solver::default_krylov_dim_mixed);
     gko::kernels::cuda::gmres_mixed::initialize_2(
         cuda, d_residual.get(), d_residual_norm.get(),
-        d_residual_norm_collection.get(), d_krylov_bases_accessor,
-        d_next_krylov_basis.get(), d_final_iter_nums.get(),
-        gko::solver::default_krylov_dim_mixed);
+        d_residual_norm_collection.get(), d_arnoldi_norm.get(),
+        d_krylov_bases_accessor, d_next_krylov_basis.get(),
+        d_final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
 
     GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
diff --git a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp b/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
index 23c9d894829..e7f2b8f5cc1 100644
--- a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
+++ b/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
@@ -57,8 +57,9 @@ int main(int argc, char *argv[])
     using mtx = gko::matrix::Csr<>;
     // The gko::solver::Cg is used here, but any other solver class can also be
     // used.
-    //    using gmres_mixed = gko::solver::GmresMixed<double, float>;
-    using gmres_mixed = gko::solver::GmresMixed<>;
+    using gmres_mixed = gko::solver::GmresMixed<double, long int>;
+    // using gmres_mixed = gko::solver::GmresMixed<double, float>;
+    // using gmres_mixed = gko::solver::GmresMixed<>;
     using bj = gko::preconditioner::Jacobi<>;
 
     // Print the ginkgo version information.
@@ -132,7 +133,15 @@ int main(int argc, char *argv[])
         auto sizesA = A->get_size();
 
         if (std::string(argv[1]) == "cuda") {
-            x = gko::read<vec>(std::ifstream("data/one.mtx"), exec);
+            /* */
+            auto host_x =
+                vec::create(exec->get_master(), gko::dim<2>(sizesA[0], 1));
+            for (auto i = 0; i < sizesA[0]; i++) {
+                host_x->at(i, 0) = 1.0;
+            }
+            x->copy_from(host_x.get());
+            /* */
+            // x = gko::read<vec>(std::ifstream("data/one.mtx"), exec);
         } else {
             x = vec::create(exec, gko::dim<2>{sizesA[0], 1});
             for (int i = 0; i < sizesA[0]; i++) x->at(i) = 1.0;
@@ -140,7 +149,15 @@ int main(int argc, char *argv[])
         auto sizesx = x->get_size();
 
         if (std::string(argv[1]) == "cuda") {
-            b = gko::read<vec>(std::ifstream("data/zero.mtx"), exec);
+            /* */
+            auto host_b =
+                vec::create(exec->get_master(), gko::dim<2>(sizesA[0], 1));
+            for (auto i = 0; i < sizesA[0]; i++) {
+                host_b->at(i, 0) = 0.0;
+            }
+            b->copy_from(host_b.get());
+            /* */
+            // b = gko::read<vec>(std::ifstream("data/zero.mtx"), exec);
         } else {
             b = vec::create(exec, gko::dim<2>{sizesA[0], 1});
             for (int i = 0; i < sizesA[0]; i++) b->at(i) = 0.0;
@@ -150,20 +167,37 @@ int main(int argc, char *argv[])
         A->apply(lend(x), lend(b));
 
         if (std::string(argv[1]) == "cuda") {
-            x = gko::read<vec>(std::ifstream("data/zero.mtx"), exec);
+            /* */
+            auto host_x =
+                vec::create(exec->get_master(), gko::dim<2>(sizesA[0], 1));
+            for (auto i = 0; i < sizesA[0]; i++) {
+                host_x->at(i, 0) = 0.0;
+            }
+            x->copy_from(host_x.get());
+            /* */
+            // x = gko::read<vec>(std::ifstream("data/zero.mtx"), exec);
         } else {
             x = vec::create(exec, gko::dim<2>{sizesA[0], 1});
             for (int i = 0; i < sizesA[0]; i++) x->at(i) = 0.0;
         }
         /**/
         if (std::string(argv[1]) == "cuda") {
-            b = gko::read<vec>(std::ifstream("data/one.mtx"), exec);
+            /* */
+            auto host_b =
+                vec::create(exec->get_master(), gko::dim<2>(sizesA[0], 1));
+            for (auto i = 0; i < sizesA[0]; i++) {
+                host_b->at(i, 0) = 1.0;
+            }
+            b->copy_from(host_b.get());
+            /* */
+            // b = gko::read<vec>(std::ifstream("data/one.mtx"), exec);
         } else {
             b = vec::create(exec, gko::dim<2>{sizesA[0], 1});
             for (int i = 0; i < sizesA[0]; i++) b->at(i) = 1.0;
         }
         /**/
     }
+    // std::cout << "Initilization is completed" << std::endl;
     // @sect3{Creating the solver}
     // Generate the gko::solver factory. Ginkgo uses the concept of Factories to
     // build solvers with certain
@@ -206,6 +240,7 @@ int main(int argc, char *argv[])
     // Finally, solve the system. The solver, being a gko::LinOp, can be applied
     // to a right hand side, b to
     // obtain the solution, x.
+    // std::cout << "Computing " << std::endl;
     solver->apply(lend(b), lend(x));
 
     // Print the solution to the command line.
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index b0aa3c74d2e..f4b49c40ba7 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -502,6 +502,9 @@ void finish_arnoldi_CGS2(
             // nrmN = norm(next_krylov_basis)
             // nrmI = infnorm(next_krylov_basis)
         }
+        helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
+            krylov_bases, next_krylov_basis->get_size()[1] * (iter + 1) + i,
+            arnoldi_norm->at(2, i) / arnoldi_norm->at(1, i));
         // reorthogonalization
         /*
         ValueType hessenberg_iter_entry = zero<ValueType>();
@@ -706,6 +709,7 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
                   const matrix::Dense<ValueType> *residual,
                   matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
+                  matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
                   Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
@@ -713,15 +717,26 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
     for (size_type j = 0; j < residual->get_size()[1]; ++j) {
         // Calculate residual norm
         remove_complex<ValueType> res_norm = zero<remove_complex<ValueType>>();
+        remove_complex<ValueType> res_inf = zero<remove_complex<ValueType>>();
 
 #pragma omp declare reduction(addnc:remove_complex<ValueType> : omp_out = omp_out + omp_in)
+#pragma omp declare reduction(infnc : remove_complex<ValueType> : omp_out = (omp_out >= omp_in? omp_out: omp_in))
 
-#pragma omp parallel for reduction(addnc : res_norm)
+#pragma omp parallel for reduction(addnc : res_norm) reduction(infnc : res_inf)
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             res_norm += squared_norm(residual->at(i, j));
             // res_norm += residual->at(i, j) * residual->at(i, j);
+            res_inf = (res_inf >= abs(residual->at(i, j)))
+                          ? res_inf
+                          : abs(residual->at(i, j));
         }
         residual_norm->at(0, j) = sqrt(res_norm);
+        arnoldi_norm->at(2, j) = res_inf;
+        // std::cout << residual_norm->at(0, j) << " - " << arnoldi_norm->at(2,
+        // j)
+        //           << std::endl;
+        helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
+            krylov_bases, j, arnoldi_norm->at(2, j) / residual_norm->at(0, j));
 
 #pragma omp parallel for
         for (size_type i = 0; i < krylov_dim + 1; ++i) {
@@ -741,6 +756,13 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
         final_iter_nums->get_data()[j] = 0;
     }
 
+#pragma omp parallel for
+    for (size_type j = residual->get_size()[1];
+         j < (krylov_dim + 1) * residual->get_size()[1]; ++j) {
+        helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
+            krylov_bases, j, one<remove_complex<ValueType>>());
+    }
+
 #pragma omp parallel for
     for (size_type i = 0; i < residual->get_size()[0]; ++i) {
         for (size_type j = residual->get_size()[1];
diff --git a/omp/test/solver/gmres_mixed_kernels.cpp b/omp/test/solver/gmres_mixed_kernels.cpp
index 69573980ed7..6114969bced 100644
--- a/omp/test/solver/gmres_mixed_kernels.cpp
+++ b/omp/test/solver/gmres_mixed_kernels.cpp
@@ -290,14 +290,14 @@ TEST_F(GmresMixed, OmpGmresMixedInitialize2IsEquivalentToRef)
 
     gko::kernels::reference::gmres_mixed::initialize_2(
         ref, residual.get(), residual_norm.get(),
-        residual_norm_collection.get(), krylov_bases_accessor,
-        next_krylov_basis.get(), final_iter_nums.get(),
+        residual_norm_collection.get(), arnoldi_norm.get(),
+        krylov_bases_accessor, next_krylov_basis.get(), final_iter_nums.get(),
         gko::solver::default_krylov_dim_mixed);
     gko::kernels::omp::gmres_mixed::initialize_2(
         omp, d_residual.get(), d_residual_norm.get(),
-        d_residual_norm_collection.get(), d_krylov_bases_accessor,
-        d_next_krylov_basis.get(), d_final_iter_nums.get(),
-        gko::solver::default_krylov_dim_mixed);
+        d_residual_norm_collection.get(), d_arnoldi_norm.get(),
+        d_krylov_bases_accessor, d_next_krylov_basis.get(),
+        d_final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
 
     GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index a157f40004b..320a2e6687d 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -447,6 +447,9 @@ void finish_arnoldi_CGS2(
             //           << arnoldi_norm->at(2, i) << " - " << l << std::endl;
             // nrmN = norm(next_krylov_basis)
         }
+        helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
+            krylov_bases, next_krylov_basis->get_size()[1] * (iter + 1) + i,
+            arnoldi_norm->at(2, i) / arnoldi_norm->at(1, i));
         /*
         // reorthogonalization
         hessenberg_iter->at(iter + 1, i) = zero<ValueType>();
@@ -643,6 +646,7 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
                   const matrix::Dense<ValueType> *residual,
                   matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
+                  matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
                   Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
@@ -650,12 +654,22 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
     for (size_type j = 0; j < residual->get_size()[1]; ++j) {
         // Calculate residual norm
         residual_norm->at(0, j) = zero<remove_complex<ValueType>>();
+        arnoldi_norm->at(2, j) = zero<remove_complex<ValueType>>();
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             residual_norm->at(0, j) += squared_norm(residual->at(i, j));
             // residual_norm->at(0, j) += residual->at(i, j) * residual->at(i,
             // j);
+            arnoldi_norm->at(2, j) =
+                (arnoldi_norm->at(2, j) >= abs(residual->at(i, j)))
+                    ? arnoldi_norm->at(2, j)
+                    : abs(residual->at(i, j));
         }
         residual_norm->at(0, j) = sqrt(residual_norm->at(0, j));
+        // std::cout << residual_norm->at(0, j) << " - " << arnoldi_norm->at(2,
+        // j)
+        //           << std::endl;
+        helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
+            krylov_bases, j, arnoldi_norm->at(2, j) / residual_norm->at(0, j));
 
         for (size_type i = 0; i < krylov_dim + 1; ++i) {
             if (i == 0) {
@@ -675,6 +689,8 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
 
     for (size_type j = residual->get_size()[1];
          j < (krylov_dim + 1) * residual->get_size()[1]; ++j) {
+        helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
+            krylov_bases, j, one<remove_complex<ValueType>>());
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             krylov_bases.write(i, j, zero<ValueType>());
         }

From 8c0d950c99856bcab7c63632448deac78ff098eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 1 Jun 2020 21:24:52 +0200
Subject: [PATCH 0356/1008] Change storage layout of krylov_bases

Previously, krylov bases b1, b2, b3 were stored
[b1[0], b2[0], b3[0],b1[1], b2[1], b3[1], ...]. Now, they are stored in
transposed fashion: [b1; b2; b3], which is equivalent to
[b1[0], b1[1], ..., b2[0], b2[1],..., b3[0], b3[1], ...]
Note that if multiple right hand sides are present, the vectors are
still stored in row major.

To make that happen, the accessor is now 3D. They store the follwing
for (x, y, z):
 x: represent which krylov vector you want to address
 y: which row-element you want to use and
 z: which column-element, aka which right hand side you want to use.

The accessor is stored row-major.
---
 common/solver/gmres_mixed_kernels.hpp.inc | 241 +++++++++-------------
 core/solver/gmres_mixed.cpp               |  17 +-
 core/solver/gmres_mixed_accessor.hpp      | 210 +++++++++++--------
 core/solver/gmres_mixed_kernels.hpp       |   6 +-
 cuda/solver/gmres_mixed_kernels.cu        | 102 ++++-----
 cuda/test/solver/gmres_mixed_kernels.cpp  |  50 +++--
 omp/solver/gmres_mixed_kernels.cpp        | 123 ++++-------
 omp/test/solver/gmres_mixed_kernels.cpp   |  49 +++--
 reference/solver/gmres_mixed_kernels.cpp  | 128 ++++--------
 9 files changed, 432 insertions(+), 494 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 9eb495d8ae4..46252ee711c 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -45,39 +45,6 @@ __global__ __launch_bounds__(default_block_size) void zero_matrix_kernel(
 }
 
 
-/*
-template <typename ValueType, typename KrylovType,
-          bool = Accessor2d<KrylovType, ValueType>::has_scale>
-struct helper_functions_accessor {};
-
-// Accessors having a scale
-template <typename ValueType, typename KrylovType>
-struct helper_functions_accessor<ValueType, KrylovType, true> {
-    template <typename IndexType>
-    static __forceinline__ __device__ void write_scale(
-        Accessor2d<KrylovType, ValueType> krylov_bases, IndexType col_idx,
-        ValueType value)
-    {
-        krylov_bases.set_scale(col_idx, value);
-    }
-};
-
-
-// Accessors not having a scale
-template <typename ValueType, typename KrylovType>
-struct helper_functions_accessor<ValueType, KrylovType, false> {
-    template <typename IndexType>
-    static __forceinline__ __device__ void write_scale(
-        Accessor2d<KrylovType, ValueType> krylov_bases, IndexType col_idx,
-        ValueType value)
-    {}
-};
-
-// calling it with:
-// helper_functions_accessor<ValueType,
-// ValueTypeKrylovBases>::write_scale(krylov_bases, col_idx, value);
-*/
-
 // Must be called with at least `max(stride_b * num_rows, krylov_dim *
 // num_cols)` threads in total.
 template <size_type block_size, typename ValueType>
@@ -118,21 +85,28 @@ template <size_type block_size, typename ValueType,
           typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(block_size) void initialize_2_1_kernel(
     size_type num_rows, size_type num_rhs, size_type krylov_dim,
-    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
     ValueType *__restrict__ residual_norm_collection,
     size_type stride_residual_nc)
 {
     const auto global_id = thread::get_thread_id_flat();
-    const auto row_idx = global_id / krylov_bases.get_stride();
-    const auto col_idx = global_id % krylov_bases.get_stride();
-
-    if (row_idx < num_rows && col_idx < (krylov_dim + 1) * num_rhs) {
-        krylov_bases.write(row_idx, col_idx, zero<ValueType>());
+    // krylov indices
+    const auto krylov_idx = global_id / krylov_bases.get_stride0();
+    const auto reminder = global_id % krylov_bases.get_stride0();
+    const auto krylov_row = reminder / krylov_bases.get_stride1();
+    const auto rhs = reminder % krylov_bases.get_stride1();
+
+    // residual_norm indices (separated for better coalesced access)
+    const auto residual_row = global_id / stride_residual_nc;
+    const auto residual_col = global_id % stride_residual_nc;
+
+    if (krylov_idx < krylov_dim + 1 && krylov_row < num_rows && rhs < num_rhs) {
+        krylov_bases.write(krylov_idx, krylov_row, rhs, zero<ValueType>());
     }
 
-    if (row_idx < krylov_dim + 1 && col_idx < num_rhs) {
-        residual_norm_collection[row_idx * stride_residual_nc + col_idx] =
-            zero<ValueType>();
+    if (residual_row < krylov_dim + 1 && residual_col < num_rhs) {
+        residual_norm_collection[residual_row * stride_residual_nc +
+                                 residual_col] = zero<ValueType>();
     }
 }
 
@@ -145,13 +119,13 @@ __global__ __launch_bounds__(block_size) void initialize_2_2_kernel(
     const ValueType *__restrict__ residual, size_type stride_residual,
     const remove_complex<ValueType> *__restrict__ residual_norm,
     ValueType *__restrict__ residual_norm_collection,
-    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
     size_type *__restrict__ final_iter_nums)
 {
     const auto global_id = thread::get_thread_id_flat();
-    const auto row_idx = global_id / num_rhs;
-    const auto col_idx = global_id % num_rhs;
+    const auto row_idx = global_id / krylov_bases.get_stride1();
+    const auto col_idx = global_id % krylov_bases.get_stride1();
 
     if (global_id < num_rhs) {
         residual_norm_collection[global_id] = residual_norm[global_id];
@@ -161,7 +135,7 @@ __global__ __launch_bounds__(block_size) void initialize_2_2_kernel(
     if (row_idx < num_rows && col_idx < num_rhs) {
         auto value = residual[row_idx * stride_residual + col_idx] /
                      residual_norm[col_idx];
-        krylov_bases.write(row_idx, col_idx, value);
+        krylov_bases.write({0}, row_idx, col_idx, value);
         next_krylov_basis[row_idx * stride_next_krylov + col_idx] = value;
     }
 }
@@ -401,16 +375,16 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_inf_kernel(
 
 template <typename ValueType, typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
-    size_type k, size_type num_rows, size_type num_cols,
+    size_type k, size_type num_rows, size_type num_rhs,
     const ValueType *__restrict__ next_krylov_basis,
     size_type stride_next_krylov,
-    Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
     const auto tidx = threadIdx.x;
     const auto tidy = threadIdx.y;
-    const auto col_idx = blockIdx.x * default_dot_dim + tidx;
+    const decltype(num_rows) col_idx = blockIdx.x * default_dot_dim + tidx;
     const auto num = ceildiv(num_rows, gridDim.y);
     const auto start_row = blockIdx.y * num;
     const auto end_row =
@@ -422,13 +396,13 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
             reduction_helper_array;
     ValueType *__restrict__ reduction_helper = reduction_helper_array;
     ValueType local_res = zero<ValueType>();
-    const auto krylov_col = k * num_cols + col_idx;
-    if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+    // const auto krylov_col = k * num_rhs + col_idx;
+    if (col_idx < num_rhs && !stop_status[col_idx].has_stopped()) {
         for (size_type i = start_row + tidy; i < end_row;
              i += default_dot_dim) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
             local_res += next_krylov_basis[next_krylov_idx] *
-                         krylov_bases.read(i, krylov_col);
+                         krylov_bases.read(k, i, col_idx);
         }
     }
     reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
@@ -440,7 +414,7 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
         reduce(tile_block, local_res,
                [](const ValueType &a, const ValueType &b) { return a + b; });
     const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
-    if (tidx == 0 && new_col_idx < num_cols &&
+    if (tidx == 0 && new_col_idx < num_rhs &&
         !stop_status[new_col_idx].has_stopped()) {
         const auto hessenberg_idx = k * stride_hessenberg + new_col_idx;
         atomic_add(hessenberg_iter + hessenberg_idx, sum);
@@ -453,7 +427,7 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
     size_type num_iters, size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
     size_type stride_next_krylov,
-    Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -473,13 +447,12 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
 
     for (size_type k = 0; k < num_iters; ++k) {
         ValueType local_res = zero<ValueType>();
-        const auto krylov_col = k * num_cols + col_idx;
         if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
             for (size_type i = start_row + tidy; i < end_row;
                  i += default_dot_dim) {
                 const auto next_krylov_idx = i * stride_next_krylov + col_idx;
                 local_res += next_krylov_basis[next_krylov_idx] *
-                             krylov_bases.read(i, krylov_col);
+                             krylov_bases.read(k, i, col_idx);
             }
         }
         reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
@@ -505,7 +478,7 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_1(
     size_type num_iters, size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
     size_type stride_next_krylov,
-    Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -528,13 +501,12 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_1(
 
     //    for (size_type k = 0; k < num_iters; ++k) {
     ValueType local_res = zero<ValueType>();
-    const auto krylov_col = k * num_cols + col_idx;
     if (col_idx_iters < num_cols_iters && !stop_status[col_idx].has_stopped()) {
         for (size_type i = start_row + tidy; i < end_row;
              i += default_dot_dim) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
             local_res += next_krylov_basis[next_krylov_idx] *
-                         krylov_bases.read(i, krylov_col);
+                         krylov_bases.read(k, i, col_idx);
         }
     }
     reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
@@ -583,7 +555,7 @@ template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(block_size) void update_next_krylov_kernel(
     size_type k, size_type num_rows, size_type num_cols,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-    Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
     const ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -594,13 +566,11 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel(
     if (row_idx < num_rows && col_idx < num_cols &&
         !stop_status[col_idx].has_stopped()) {
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-        // const auto krylov_idx =
-        //    row_idx * stride_krylov + k * num_cols + col_idx;
         const auto hessenberg_idx = k * stride_hessenberg + col_idx;
 
         next_krylov_basis[next_krylov_idx] -=
             hessenberg_iter[hessenberg_idx] *
-            krylov_bases.read(row_idx, k * num_cols + col_idx);
+            krylov_bases.read(k, row_idx, col_idx);
     }
 }
 
@@ -612,7 +582,7 @@ __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -631,42 +601,39 @@ __global__
 
             next_krylov_basis[next_krylov_idx] -=
                 hessenberg_iter[hessenberg_idx] *
-                krylov_bases.read(row_idx, k * num_cols + col_idx);
+                krylov_bases.read(k, row_idx, col_idx);
         }
     }
 }
 
 
-// Must be called with at least `num_rows * stride_next_krylov` threads in
-// total.
+// Must be called with at least `num_vectors * krylov_bases.get_stride0()`
+// threads in total.
 template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_0(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
 {
+    // CAREFUL: krylov_bases changes might not be done properly since this
+    //          kernel is currently not used!
     const auto global_id = thread::get_thread_id_flat();
-    //    const auto row_idx = global_id / stride_next_krylov;
-    //    const auto col_idx = global_id % stride_next_krylov;
-    const auto row_idx_krylov = global_id / (stride_next_krylov * num_iters);
-    const auto col_idx_krylov = global_id % (stride_next_krylov * num_iters);
-    //    const auto num_itr_krylov = col_idx_krylov / stride_next_krylov;
-    const auto k = col_idx_krylov / stride_next_krylov;
-    const auto col_itr_krylov = col_idx_krylov % stride_next_krylov;
-
-    if (row_idx_krylov < num_rows && col_itr_krylov < num_cols &&
-        !stop_status[col_itr_krylov].has_stopped()) {
-        const auto next_krylov_idx =
-            row_idx_krylov * stride_next_krylov + col_itr_krylov;
-        const auto hessenberg_idx = k * stride_hessenberg + col_itr_krylov;
-
-        auto value =
-            hessenberg_iter[hessenberg_idx] *
-            krylov_bases.read(row_idx_krylov, k * num_cols + col_itr_krylov);
+    const auto k = global_id / krylov_bases.get_stride0();
+    const auto reminder = global_id / krylov_bases.get_stride0();
+    const auto row_idx = reminder / krylov_bases.get_stride1();
+    const auto col_idx = reminder % krylov_bases.get_stride1();
+
+    if (row_idx < num_rows && col_idx < num_cols &&
+        !stop_status[col_idx].has_stopped()) {
+        const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+        const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+
+        auto value = hessenberg_iter[hessenberg_idx] *
+                     krylov_bases.read(k, row_idx, col_idx);
         atomic_add(next_krylov_basis + next_krylov_idx, -value);
     }
 }
@@ -679,7 +646,7 @@ __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_1(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -687,7 +654,7 @@ __global__
     if (num_cols == 1) {
         const auto tidx = threadIdx.x;
         const size_type row_idx = blockIdx.x;
-        const auto col_idx = 0;
+        constexpr decltype(row_idx) col_idx = 0;
 
         // Used that way to get around dynamic initialization warning and
         // template error when using `reduction_helper_array` directly in
@@ -699,10 +666,9 @@ __global__
             ValueType value = zero<ValueType>();
             for (size_type i = tidx; i < num_iters; i += blockDim.x) {
                 const auto hessenberg_idx = i * stride_hessenberg + col_idx;
-                const auto krylov_idx = i * num_cols + col_idx;
 
                 value += hessenberg_iter[hessenberg_idx] *
-                         krylov_bases.read(row_idx, krylov_idx);
+                         krylov_bases.read(i, row_idx, col_idx);
             }
             reduction_helper[tidx] = value;
 
@@ -726,13 +692,11 @@ __global__
             !stop_status[col_idx].has_stopped()) {
             const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
             for (size_type k = 0; k < num_iters; ++k) {
-                // const auto krylov_idx =
-                //    row_idx * stride_krylov + k * num_cols + col_idx;
                 const auto hessenberg_idx = k * stride_hessenberg + col_idx;
 
                 next_krylov_basis[next_krylov_idx] -=
                     hessenberg_iter[hessenberg_idx] *
-                    krylov_bases.read(row_idx, k * num_cols + col_idx);
+                    krylov_bases.read(k, row_idx, col_idx);
             }
         }
     }
@@ -747,7 +711,7 @@ __global__
         size_type num_iters, size_type num_blocks, size_type num_rows,
         size_type num_cols, ValueType *__restrict__ next_krylov_basis,
         size_type stride_next_krylov,
-        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -770,12 +734,9 @@ __global__
                 ValueType value = zero<ValueType>();
                 for (size_type i = tidx; i < num_iters; i += blockDim.x) {
                     const auto hessenberg_idx = i * stride_hessenberg + col_idx;
-                    const auto krylov_idx = i * num_cols + col_idx;
 
                     value += hessenberg_iter[hessenberg_idx] *
-                             krylov_bases.read(j, krylov_idx);
-                    //                             krylov_bases.read(row_idx+j,
-                    //                             krylov_idx);
+                             krylov_bases.read(i, j, col_idx);
                 }
                 reduction_helper[k * block_size + tidx] = value;
             }
@@ -813,13 +774,11 @@ __global__
             !stop_status[col_idx].has_stopped()) {
             const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
             for (size_type k = 0; k < num_iters; ++k) {
-                // const auto krylov_idx =
-                //    row_idx * stride_krylov + k * num_cols + col_idx;
                 const auto hessenberg_idx = k * stride_hessenberg + col_idx;
 
                 next_krylov_basis[next_krylov_idx] -=
                     hessenberg_iter[hessenberg_idx] *
-                    krylov_bases.read(row_idx, k * num_cols + col_idx);
+                    krylov_bases.read(k, row_idx, col_idx);
             }
         }
     }
@@ -834,7 +793,7 @@ __global__
         size_type num_iters, size_type num_blocks, size_type num_rows,
         size_type num_cols, ValueType *__restrict__ next_krylov_basis,
         size_type stride_next_krylov,
-        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -859,12 +818,11 @@ __global__
             }
             for (size_type i = tidx; i < num_iters; i += blockDim.x) {
                 const auto hessenberg_idx = i * stride_hessenberg + col_idx;
-                const auto krylov_idx = i * num_cols + col_idx;
                 const auto hessenberg_val = hessenberg_iter[hessenberg_idx];
                 for (size_type j = row_idx, k = 0;
                      j < num_rows && k < num_blocks; j++, k++) {
                     reduction_helper[k * block_size + tidx] +=
-                        hessenberg_val * krylov_bases.read(j, krylov_idx);
+                        hessenberg_val * krylov_bases.read(i, j, col_idx);
                 }
             }
 
@@ -907,7 +865,7 @@ __global__
 
                 next_krylov_basis[next_krylov_idx] -=
                     hessenberg_iter[hessenberg_idx] *
-                    krylov_bases.read(row_idx, k * num_cols + col_idx);
+                    krylov_bases.read(k, row_idx, col_idx);
             }
         }
     }
@@ -922,7 +880,7 @@ __global__
         size_type num_iters, size_type num_blocks, size_type num_rows,
         size_type num_cols, ValueType *__restrict__ next_krylov_basis,
         size_type stride_next_krylov,
-        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -947,12 +905,11 @@ __global__
             }
             for (size_type i = tidx; i < num_iters; i += blockDim.x) {
                 const auto hessenberg_idx = i * stride_hessenberg + col_idx;
-                const auto krylov_idx = i * num_cols + col_idx;
                 const auto hessenberg_val = hessenberg_iter[hessenberg_idx];
                 for (size_type j = row_idx, k = 0;
                      j < num_rows && k < num_blocks; j++, k++) {
                     reduction_helper[k * block_size + tidx] +=
-                        hessenberg_val * krylov_bases.read(j, krylov_idx);
+                        hessenberg_val * krylov_bases.read(i, j, col_idx);
                 }
             }
             multireduce(
@@ -982,13 +939,11 @@ __global__
             !stop_status[col_idx].has_stopped()) {
             const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
             for (size_type k = 0; k < num_iters; ++k) {
-                // const auto krylov_idx =
-                //    row_idx * stride_krylov + k * num_cols + col_idx;
                 const auto hessenberg_idx = k * stride_hessenberg + col_idx;
 
                 next_krylov_basis[next_krylov_idx] -=
                     hessenberg_iter[hessenberg_idx] *
-                    krylov_bases.read(row_idx, k * num_cols + col_idx);
+                    krylov_bases.read(k, row_idx, col_idx);
             }
         }
     }
@@ -1002,7 +957,7 @@ __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_and_check(
         size_type k, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const remove_complex<ValueType> *__restrict__ arnoldi_norm,
@@ -1017,21 +972,16 @@ __global__
     if (row_idx < num_rows && col_idx < num_cols &&
         !stop_status[col_idx].has_stopped()) {
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-        // const auto krylov_idx =
-        //    row_idx * stride_krylov + k * num_cols + col_idx;
         const auto hessenberg_idx = k * stride_hessenberg + col_idx;
         const auto arnoldi_idx = col_idx;
         const auto reorth_idx = col_idx;
 
         next_krylov_basis[next_krylov_idx] -=
             hessenberg_iter[hessenberg_idx] *
-            krylov_bases.read(row_idx, k * num_cols + col_idx);
+            krylov_bases.read(k, row_idx, col_idx);
         if (row_idx == 0) {
-            // DONE: TODO remove abs() as soon as type of arnoldi_norm has
-            // changed
             if (squared_norm(hessenberg_iter[hessenberg_idx]) >
                 (0.99 * sqrt(arnoldi_norm[arnoldi_idx]))) {
-                // abs(0.99 * sqrt(arnoldi_norm[arnoldi_idx]))) {
                 reorth_status[reorth_idx].reset();
                 atomic_add(num_reorth, (size_type)1);
             } else {
@@ -1048,7 +998,7 @@ template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
     size_type k, size_type num_rows, size_type num_cols,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-    Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const ValueType *__restrict__ buffer_iter, size_type stride_buffer,
     const stopping_status *__restrict__ stop_status,
@@ -1062,13 +1012,10 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
         !stop_status[col_idx].has_stopped() &&
         !reorth_status[col_idx].has_stopped()) {
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-        // const auto krylov_idx =
-        //    row_idx * stride_krylov + k * num_cols + col_idx;
         const auto hessenberg_idx = k * stride_hessenberg + col_idx;
         const auto buffer_idx = k * stride_buffer + col_idx;
         next_krylov_basis[next_krylov_idx] -=
-            buffer_iter[buffer_idx] *
-            krylov_bases.read(row_idx, k * num_cols + col_idx);
+            buffer_iter[buffer_idx] * krylov_bases.read(k, row_idx, col_idx);
         if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
             hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
         }
@@ -1083,7 +1030,7 @@ __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_and_add(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
         ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
         const ValueType *__restrict__ buffer_iter, size_type stride_buffer,
         const stopping_status *__restrict__ stop_status,
@@ -1104,7 +1051,7 @@ __global__
             const auto buffer_idx = k * stride_buffer + col_idx;
             next_krylov_basis[next_krylov_idx] -=
                 buffer_iter[buffer_idx] *
-                krylov_bases.read(row_idx, k * num_cols + col_idx);
+                krylov_bases.read(k, row_idx, col_idx);
             if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
                 hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
             }
@@ -1119,7 +1066,7 @@ __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_and_add_1(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
         ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
         const ValueType *__restrict__ buffer_iter, size_type stride_buffer,
         const stopping_status *__restrict__ stop_status,
@@ -1128,7 +1075,7 @@ __global__
     if (num_cols == 1) {
         const auto tidx = threadIdx.x;
         const size_type row_idx = blockIdx.x;
-        const auto col_idx = 0;
+        constexpr decltype(row_idx) col_idx = 0;
 
         // Used that way to get around dynamic initialization warning and
         // template error when using `reduction_helper_array` directly in
@@ -1141,10 +1088,9 @@ __global__
             for (size_type i = tidx; i < num_iters; i += blockDim.x) {
                 const auto hessenberg_idx = i * stride_hessenberg + col_idx;
                 const auto buffer_idx = i * stride_buffer + col_idx;
-                const auto krylov_idx = i * num_cols + col_idx;
 
                 value += buffer_iter[buffer_idx] *
-                         krylov_bases.read(row_idx, krylov_idx);
+                         krylov_bases.read(i, row_idx, col_idx);
                 if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
                     hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
                 }
@@ -1178,7 +1124,7 @@ __global__
                 const auto buffer_idx = k * stride_buffer + col_idx;
                 next_krylov_basis[next_krylov_idx] -=
                     buffer_iter[buffer_idx] *
-                    krylov_bases.read(row_idx, k * num_cols + col_idx);
+                    krylov_bases.read(k, row_idx, col_idx);
                 if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
                     hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
                 }
@@ -1223,7 +1169,7 @@ template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
     remove_complex<ValueType> *__restrict__ arnoldi_norm, size_type stride_norm,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
-    size_type iter, Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    size_type iter, Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
     const stopping_status *__restrict__ stop_status,
     stopping_status *__restrict__ reorth_status,
     size_type *__restrict__ num_reorth)
@@ -1254,7 +1200,7 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
         // printf("%d - %d - %d - %d\n", stride_norm, iter, col_idx,
         //        stride_norm * iter + col_idx);
         helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
-            krylov_bases, stride_norm * iter + col_idx, num2 / num12);
+            krylov_bases, iter, col_idx, num2 / num12);
         // krylov_bases, num_cols * (iter + 1) + col_idx, num2 / num12);
         // krylov_bases, col_idx, num2 / num12);
     }
@@ -1270,11 +1216,11 @@ __global__ __launch_bounds__(block_size) void set_scale_kernel(
     size_type stride_residual,
     const remove_complex<ValueType> *__restrict__ arnoldi_inf,
     size_type stride_inf,
-    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases)
+    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases)
 {
     const auto global_id = thread::get_thread_id_flat();
-    const auto blk_idx = global_id / num_rhs;
-    const auto col_idx = global_id % num_rhs;
+    const auto blk_idx = global_id / krylov_bases.get_stride1();
+    const auto col_idx = global_id % krylov_bases.get_stride1();
 
     if (blk_idx < num_blocks && col_idx < num_rhs) {
         if (blk_idx == 0) {
@@ -1282,13 +1228,15 @@ __global__ __launch_bounds__(block_size) void set_scale_kernel(
             const auto num2 = arnoldi_inf[col_idx];
             // printf("num1 = %10.5e , num2 = %10.5e\n", num1, num2);
             helper_functions_accessor<
-                ValueType, ValueTypeKrylovBases>::write_scale(krylov_bases,
+                ValueType, ValueTypeKrylovBases>::write_scale(krylov_bases, {0},
                                                               col_idx,
                                                               num2 / num1);
         } else {
             const auto num = one<remove_complex<ValueType>>();
-            helper_functions_accessor<ValueType, ValueTypeKrylovBases>::
-                write_scale(krylov_bases, blk_idx * num_rhs + col_idx, num);
+            helper_functions_accessor<
+                ValueType, ValueTypeKrylovBases>::write_scale(krylov_bases,
+                                                              blk_idx, col_idx,
+                                                              num);
         }
     }
 }
@@ -1342,7 +1290,7 @@ template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(block_size) void update_krylov_next_krylov_kernel(
     size_type iter, size_type num_rows, size_type num_cols,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
     const ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -1355,15 +1303,12 @@ __global__ __launch_bounds__(block_size) void update_krylov_next_krylov_kernel(
     if (row_idx < num_rows && col_idx < num_cols &&
         !stop_status[col_idx].has_stopped()) {
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-        // const auto krylov_idx =
-        //    row_idx * stride_krylov + num_cols * (iter + 1) + col_idx;
 
         const auto next_krylov_value =
             next_krylov_basis[next_krylov_idx] / hessenberg;
 
         next_krylov_basis[next_krylov_idx] = next_krylov_value;
-        krylov_bases.write(row_idx, num_cols * (iter + 1) + col_idx,
-                           next_krylov_value);
+        krylov_bases.write(iter + 1, row_idx, col_idx, next_krylov_value);
     }
 }
 
@@ -1529,8 +1474,8 @@ __global__ __launch_bounds__(block_size) void solve_upper_triangular_kernel(
 template <size_type block_size, typename ValueType,
           typename ValueTypeKrylovBases>
 __global__ __launch_bounds__(block_size) void calculate_Qy_kernel(
-    size_type num_rows, size_type num_cols, size_type num_rhs,
-    Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    size_type num_rows, size_type num_cols,
+    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
     const ValueType *__restrict__ y, size_type stride_y,
     ValueType *__restrict__ before_preconditioner,
     size_type stride_preconditioner,
@@ -1541,11 +1486,11 @@ __global__ __launch_bounds__(block_size) void calculate_Qy_kernel(
     const auto col_id = global_id % stride_preconditioner;
 
     if (row_id < num_rows && col_id < num_cols) {
-        before_preconditioner[global_id] = zero<ValueType>();
+        ValueType temp = zero<ValueType>();
         for (size_type j = 0; j < final_iter_nums[col_id]; ++j) {
-            before_preconditioner[global_id] +=
-                krylov_bases.read(row_id, j * num_rhs + col_id) *
-                y[j * stride_y + col_id];
+            temp +=
+                krylov_bases.read(j, row_id, col_id) * y[j * stride_y + col_id];
         }
+        before_preconditioner[global_id] = temp;
     }
 }
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 6802447e102..622d9f5d25e 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -85,9 +85,9 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     using Vector = matrix::Dense<ValueType>;
     using VectorNorms = matrix::Dense<remove_complex<ValueType>>;
     using LowArray = Array<ValueTypeKrylovBases>;
-    using KrylovAccessor = kernels::Accessor2d<ValueTypeKrylovBases, ValueType>;
-    using Accessor2dHelper =
-        kernels::Accessor2dHelper<ValueType, ValueTypeKrylovBases>;
+    using KrylovAccessor = kernels::Accessor3d<ValueTypeKrylovBases, ValueType>;
+    using Accessor3dHelper =
+        kernels::Accessor3dHelper<ValueType, ValueTypeKrylovBases>;
 
 
     constexpr uint8 RelativeStoppingId{1};
@@ -100,14 +100,17 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     auto dense_b = as<const Vector>(b);
     auto dense_x = as<Vector>(x);
     auto residual = Vector::create_with_config_of(dense_b);
-    const dim<2> krylov_bases_dim{
-        system_matrix_->get_size()[1],
-        (krylov_dim_mixed_ + 1) * dense_b->get_size()[1]};
+    const dim<3> krylov_bases_dim{krylov_dim_mixed_ + 1,
+                                  system_matrix_->get_size()[1],
+                                  dense_b->get_size()[1]};
+    // const dim<2> krylov_bases_dim{
+    //     system_matrix_->get_size()[1],
+    //     (krylov_dim_mixed_ + 1) * dense_b->get_size()[1]};
     // const size_type krylov_bases_stride = krylov_bases_dim[1];
     // LowArray krylov_bases(exec, krylov_bases_dim[0] * krylov_bases_stride);
     // KrylovAccessor krylov_bases_accessor(krylov_bases.get_data(),
     //                                     krylov_bases_stride);
-    Accessor2dHelper helper(krylov_bases_dim, exec);
+    Accessor3dHelper helper(exec, krylov_bases_dim);
     auto krylov_bases_accessor = helper.get_accessor();
 
     auto next_krylov_basis = Vector::create_with_config_of(dense_b);
diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index 550f23123dc..153df5b982e 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -35,11 +35,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <cinttypes>
+#include <memory>
 #include <type_traits>
 
 
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/types.hpp>
@@ -80,17 +82,24 @@ struct helper_have_scale<StorageType, ArithmeticType, true>
 
 template <typename StorageType, typename ArithmeticType,
           bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
-class Accessor2dConst {};
+class Accessor3dConst {};
 /**
  * @internal
  *
- * The Accessor2d class hides the underlying storage_ format and provides a
+ * The Accessor3d class hides the underlying storage_ format and provides a
  * simple interface for accessing a one dimensional storage.
  *
  * This class only manages the accesses, however, and not the memory itself.
+ *
+ * The dimensions {x, y, z} explained for the krylov_bases:
+ * - x: selects the krylov vector (usually has krylov_dim + 1 vectors)
+ * - y: selects the (row-)element of said krylov vector
+ * - z: selects which column-element of said krylov vector should be used
+ *
+ * The accessor uses row-major access.
  */
 template <typename StorageType, typename ArithmeticType>
-class Accessor2dConst<StorageType, ArithmeticType, false> {
+class Accessor3dConst<StorageType, ArithmeticType, false> {
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
@@ -105,28 +114,33 @@ class Accessor2dConst<StorageType, ArithmeticType, false> {
      * stride.
      *
      * @internal
-     * const_cast needed, so Accessor2dConst(ac.get_storage(), ac.get_stride())
+     * const_cast needed, so Accessor3dConst(ac.get_storage(), ...)
      * works. Also, storage_ is never accessed in a non-const fashion, so it
      * is not invalid or UB code.
      */
-    Accessor2dConst(const storage_type *storage, size_type stride)
-        : storage_{const_cast<storage_type *>(storage)}, stride_{stride}
+    Accessor3dConst(const storage_type *storage, size_type stride0,
+                    size_type stride1)
+        : storage_{const_cast<storage_type *>(storage)},
+          stride_{stride0, stride1}
     {}
 
     /**
      * Reads the value at the given index.
      */
     template <typename IndexType>
-    GKO_ATTRIBUTES arithmetic_type read(IndexType row, IndexType col) const
+    GKO_ATTRIBUTES arithmetic_type read(IndexType x, IndexType y,
+                                        IndexType z) const
     {
         // Make use of the restrict (and const) qualifier. If the restrict
         // qualifier would have been put on the class attribute, it would be
         // ignored by nvcc.
         const storage_type *GKO_RESTRICT rest_storage = storage_;
-        return static_cast<arithmetic_type>(rest_storage[row * stride_ + col]);
+        return static_cast<arithmetic_type>(
+            rest_storage[x * stride_[0] + y * stride_[1] + z]);
     }
 
-    GKO_ATTRIBUTES size_type get_stride() const { return stride_; }
+    GKO_ATTRIBUTES size_type get_stride0() const { return stride_[0]; }
+    GKO_ATTRIBUTES size_type get_stride1() const { return stride_[1]; }
 
     GKO_ATTRIBUTES const storage_type *get_storage() const { return storage_; }
 
@@ -137,22 +151,29 @@ class Accessor2dConst<StorageType, ArithmeticType, false> {
 
 protected:
     storage_type *storage_;
-    size_type stride_;
+    size_type stride_[2];
 };
 
 
 /**
  * @internal
  *
- * The Accessor2d class hides the underlying storage_ format and provides a
+ * The Accessor3d class hides the underlying storage_ format and provides a
  * simple interface for accessing a one dimensional storage.
- * Additionally, since it is (soon) an integer type, a scale array is also
- * needed to do a proper conversion.
+ * Additionally, this accessor posesses a scale array, which is used for each
+ * read and write operation to do a proper conversion.
  *
  * This class only manages the accesses, however, and not the memory itself.
+ *
+ * The dimensions {x, y, z} explained for the krylov_bases:
+ * - x: selects the krylov vector (usually has krylov_dim + 1 vectors)
+ * - y: selects the (row-)element of said krylov vector
+ * - z: selects which column-element of said krylov vector should be used
+ *
+ * The accessor uses row-major access.
  */
 template <typename StorageType, typename ArithmeticType>
-class Accessor2dConst<StorageType, ArithmeticType, true> {
+class Accessor3dConst<StorageType, ArithmeticType, true> {
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
@@ -166,34 +187,45 @@ class Accessor2dConst<StorageType, ArithmeticType, true> {
      * stride.
      *
      * @internal
-     * const_cast needed, so Accessor2dConst(ac.get_storage(), ac.get_stride())
+     * const_cast needed, so Accessor3dConst(ac.get_storage(), ...)
      * works. Also, storage_ is never accessed in a non-const fashion, so it
      * is not invalid or UB code.
      */
-    Accessor2dConst(const storage_type *storage, size_type stride,
-                    const arithmetic_type *scale)
+    Accessor3dConst(const storage_type *storage, size_type stride0,
+                    size_type stride1, const arithmetic_type *scale)
         : storage_{const_cast<storage_type *>(storage)},
-          stride_{stride},
+          stride_{stride0, stride1},
           scale_{const_cast<arithmetic_type *>(scale)}
     {}
 
+    /**
+     * Reads the scale value at the given indices.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES arithmetic_type read_scale(IndexType x, IndexType z) const
+    {
+        const arithmetic_type *GKO_RESTRICT rest_scale = scale_;
+        return rest_scale[x * stride_[1] + z];
+    }
+
     /**
      * Reads the value at the given index.
      */
     template <typename IndexType>
-    GKO_ATTRIBUTES arithmetic_type read(IndexType row, IndexType col) const
+    GKO_ATTRIBUTES arithmetic_type read(IndexType x, IndexType y,
+                                        IndexType z) const
     {
         // Make use of the restrict (and const) qualifier. If the restrict
         // qualifier would have been put on the class attribute, it would be
         // ignored by nvcc.
         const storage_type *GKO_RESTRICT rest_storage = storage_;
-        const arithmetic_type *GKO_RESTRICT rest_scale = scale_;
-        //        std::cout << col << " - A - " << rest_scale[col] << std::endl;
-        return static_cast<arithmetic_type>(rest_storage[row * stride_ + col]) *
-               rest_scale[col];
+        return static_cast<arithmetic_type>(
+                   rest_storage[x * stride_[0] + y * stride_[1] + z]) *
+               read_scale(x, z);
     }
 
-    GKO_ATTRIBUTES size_type get_stride() const { return stride_; }
+    GKO_ATTRIBUTES size_type get_stride0() const { return stride_[0]; }
+    GKO_ATTRIBUTES size_type get_stride1() const { return stride_[1]; }
 
     GKO_ATTRIBUTES const storage_type *get_storage() const { return storage_; }
 
@@ -214,27 +246,27 @@ class Accessor2dConst<StorageType, ArithmeticType, true> {
 
 protected:
     storage_type *storage_;
+    size_type stride_[2];
     arithmetic_type *scale_;
-    size_type stride_;
 };
 
 
 template <typename StorageType, typename ArithmeticType,
           bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
-class Accessor2d : public Accessor2dConst<StorageType, ArithmeticType> {};
+class Accessor3d : public Accessor3dConst<StorageType, ArithmeticType> {};
 
 
 /**
  * @internal
  *
- * The Accessor2d class hides the underlying storage_ format and provides a
+ * The Accessor3d class hides the underlying storage_ format and provides a
  * simple interface for accessing a one dimensional storage.
  *
  * This class only manages the accesses, however, and not the memory itself.
  */
 template <typename StorageType, typename ArithmeticType>
-class Accessor2d<StorageType, ArithmeticType, false>
-    : public Accessor2dConst<StorageType, ArithmeticType> {
+class Accessor3d<StorageType, ArithmeticType, false>
+    : public Accessor3dConst<StorageType, ArithmeticType> {
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
@@ -243,39 +275,36 @@ class Accessor2d<StorageType, ArithmeticType, false>
         "storage_type must not be an integral in this class.");
 
 private:
-    using Accessor2dC = Accessor2dConst<storage_type, arithmetic_type>;
+    using Accessor3dC = Accessor3dConst<storage_type, arithmetic_type>;
 
 public:
     /**
      * Creates an empty accessor pointing to a nullptr.
      */
-    Accessor2d() : Accessor2dC(nullptr, {}) {}
+    Accessor3d() : Accessor3dC(nullptr, {}, {}) {}
 
     /**
      * Creates the accessor with an already allocated storage space with a
      * stride.
      */
-    Accessor2d(storage_type *storage, size_type stride)
-        : Accessor2dC(storage, stride)
+    Accessor3d(storage_type *storage, size_type stride0, size_type stride1)
+        : Accessor3dC(storage, stride0, stride1)
     {}
-    /*
-        operator Accessor2dC() const
-        {
-            return {this->storage_, this->stride_};
-        }
-    */
 
-    Accessor2dC to_const() const { return {this->storage_, this->stride_}; }
+    Accessor3dC to_const() const
+    {
+        return {this->storage_, this->stride_[0], this->stride_[1]};
+    }
 
     /**
      * Writes the given value at the given index.
      */
     template <typename IndexType>
-    GKO_ATTRIBUTES void write(IndexType row, IndexType col,
+    GKO_ATTRIBUTES void write(IndexType x, IndexType y, IndexType z,
                               arithmetic_type value)
     {
         storage_type *GKO_RESTRICT rest_storage = this->storage_;
-        rest_storage[row * this->stride_ + col] =
+        rest_storage[x * this->stride_[0] + y * this->stride_[1] + z] =
             static_cast<storage_type>(value);
     }
 
@@ -286,14 +315,14 @@ class Accessor2d<StorageType, ArithmeticType, false>
 /**
  * @internal
  *
- * The Accessor2d class hides the underlying storage_ format and provides a
+ * The Accessor3d class hides the underlying storage_ format and provides a
  * simple interface for accessing a one dimensional storage.
  *
  * This class only manages the accesses, however, and not the memory itself.
  */
 template <typename StorageType, typename ArithmeticType>
-class Accessor2d<StorageType, ArithmeticType, true>
-    : public Accessor2dConst<StorageType, ArithmeticType> {
+class Accessor3d<StorageType, ArithmeticType, true>
+    : public Accessor3dConst<StorageType, ArithmeticType> {
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
@@ -301,45 +330,48 @@ class Accessor2d<StorageType, ArithmeticType, true>
                   "storage_type must not be an integral in this class.");
 
 private:
-    using Accessor2dC = Accessor2dConst<storage_type, arithmetic_type>;
+    using Accessor3dC = Accessor3dConst<storage_type, arithmetic_type>;
 
 public:
     /**
      * Creates an empty accessor pointing to a nullptr.
      */
-    Accessor2d() : Accessor2dC(nullptr, {}, nullptr) {}
+    Accessor3d() : Accessor3dC(nullptr, {}, {}, nullptr) {}
 
     /**
      * Creates the accessor with an already allocated storage space with a
      * stride.
      */
-    Accessor2d(storage_type *storage, size_type stride, arithmetic_type *scale)
-        : Accessor2dC(storage, stride, scale)
+    Accessor3d(storage_type *storage, size_type stride0, size_type stride1,
+               arithmetic_type *scale)
+        : Accessor3dC(storage, stride0, stride1, scale)
     {}
 
-    Accessor2dC to_const() const
+    Accessor3dC to_const() const
     {
-        return {this->storage_, this->stride_, this->scale_};
+        return {this->storage_, this->stride_[0], this->stride_[1],
+                this->scale_};
     }
 
     /**
      * Writes the given value at the given index.
      */
     template <typename IndexType>
-    GKO_ATTRIBUTES void write(IndexType row, IndexType col,
+    GKO_ATTRIBUTES void write(IndexType x, IndexType y, IndexType z,
                               arithmetic_type value)
     {
         storage_type *GKO_RESTRICT rest_storage = this->storage_;
-        const arithmetic_type *GKO_RESTRICT rest_scale = this->scale_;
-        rest_storage[row * this->stride_ + col] =
-            static_cast<storage_type>(value / rest_scale[col]);
+        const auto stride0 = this->stride_[0];
+        const auto stride1 = this->stride_[1];
+        rest_storage[x * stride0 + y * stride1 + z] =
+            static_cast<storage_type>(value / this->read_scale(x, z));
     }
 
     /**
      * Writes the given value at the given index for a scale.
      */
     template <typename IndexType>
-    GKO_ATTRIBUTES void set_scale(IndexType idx, arithmetic_type val)
+    GKO_ATTRIBUTES void set_scale(IndexType x, IndexType z, arithmetic_type val)
     {
         arithmetic_type *GKO_RESTRICT rest_scale = this->scale_;
         storage_type max_val = one<storage_type>();
@@ -349,7 +381,8 @@ class Accessor2d<StorageType, ArithmeticType, true>
             max_val = std::numeric_limits<storage_type>::max();
             //    printf("(B) max_val = %ld\n", max_val);
         }
-        rest_scale[idx] = val / static_cast<arithmetic_type>(max_val);
+        rest_scale[x * this->stride_[1] + z] =
+            val / static_cast<arithmetic_type>(max_val);
         //    rest_scale[idx] = one<arithmetic_type>();
         //    printf("val = %10.5e , rest_scale = %10.5e\n", val,
         //    rest_scale[idx]); std::cout << val << " - " << rest_scale[idx] <<
@@ -363,28 +396,28 @@ class Accessor2d<StorageType, ArithmeticType, true>
 
 
 template <typename ValueType, typename ValueTypeKrylovBases,
-          bool = Accessor2d<ValueTypeKrylovBases, ValueType>::has_scale>
-class Accessor2dHelper {};
+          bool = Accessor3d<ValueTypeKrylovBases, ValueType>::has_scale>
+class Accessor3dHelper {};
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-class Accessor2dHelper<ValueType, ValueTypeKrylovBases, true> {
-    using Accessor = Accessor2d<ValueTypeKrylovBases, ValueType>;
+class Accessor3dHelper<ValueType, ValueTypeKrylovBases, true> {
+    using Accessor = Accessor3d<ValueTypeKrylovBases, ValueType>;
     static_assert(Accessor::has_scale,
                   "This accessor must have a scale in this class!");
 
 public:
-    Accessor2dHelper() = default;
+    Accessor3dHelper() = default;
 
-    Accessor2dHelper(dim<2> krylov_dim, std::shared_ptr<const Executor> exec)
+    Accessor3dHelper(std::shared_ptr<const Executor> exec, dim<3> krylov_dim)
         : krylov_dim_{krylov_dim},
-          bases_stride_{krylov_dim_[1]},
-          bases_{exec, krylov_dim_[0] * bases_stride_},
-          scale_{exec, krylov_dim_[1]}
+          bases_{exec, krylov_dim_[0] * krylov_dim_[1] * krylov_dim_[2]},
+          scale_{exec, krylov_dim_[0] * krylov_dim_[2]}
     {
         // For testing, initialize scale to ones
         // Array<ValueType> h_scale{exec->get_master(), krylov_dim[0]};
-        Array<ValueType> h_scale{exec->get_master(), krylov_dim[1]};
+        Array<ValueType> h_scale{exec->get_master(),
+                                 krylov_dim[0] * krylov_dim[2]};
         for (size_type i = 0; i < h_scale.get_num_elems(); ++i) {
             h_scale.get_data()[i] = one<ValueType>();
         }
@@ -393,48 +426,53 @@ class Accessor2dHelper<ValueType, ValueTypeKrylovBases, true> {
 
     Accessor get_accessor()
     {
-        return {bases_.get_data(), bases_stride_, scale_.get_data()};
+        const auto stride0 = krylov_dim_[1] * krylov_dim_[2];
+        const auto stride1 = krylov_dim_[2];
+        return {bases_.get_data(), stride0, stride1, scale_.get_data()};
     }
 
     gko::Array<ValueTypeKrylovBases> &get_bases() { return bases_; }
 
 private:
-    dim<2> krylov_dim_;
-    size_type bases_stride_;
+    dim<3> krylov_dim_;
     Array<ValueTypeKrylovBases> bases_;
     Array<ValueType> scale_;
 };
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-class Accessor2dHelper<ValueType, ValueTypeKrylovBases, false> {
-    using Accessor = Accessor2d<ValueTypeKrylovBases, ValueType>;
+class Accessor3dHelper<ValueType, ValueTypeKrylovBases, false> {
+    using Accessor = Accessor3d<ValueTypeKrylovBases, ValueType>;
     static_assert(!Accessor::has_scale,
                   "This accessor must not have a scale in this class!");
 
 public:
-    Accessor2dHelper() = default;
+    Accessor3dHelper() = default;
 
-    Accessor2dHelper(dim<2> krylov_dim, std::shared_ptr<const Executor> exec)
+    Accessor3dHelper(std::shared_ptr<const Executor> exec, dim<3> krylov_dim)
         : krylov_dim_{krylov_dim},
-          bases_stride_{krylov_dim_[1]},
-          bases_{exec, krylov_dim_[0] * bases_stride_}
+          bases_{std::move(exec),
+                 krylov_dim_[0] * krylov_dim_[1] * krylov_dim_[2]}
     {}
 
-    Accessor get_accessor() { return {bases_.get_data(), bases_stride_}; }
+    Accessor get_accessor()
+    {
+        const auto stride0 = krylov_dim_[1] * krylov_dim_[2];
+        const auto stride1 = krylov_dim_[2];
+        return {bases_.get_data(), stride0, stride1};
+    }
 
     gko::Array<ValueTypeKrylovBases> &get_bases() { return bases_; }
 
 private:
-    dim<2> krylov_dim_;
-    size_type bases_stride_;
+    dim<3> krylov_dim_;
     Array<ValueTypeKrylovBases> bases_;
 };
 
 //----------------------------------------------
 
 template <typename ValueType, typename KrylovType,
-          bool = Accessor2d<KrylovType, ValueType>::has_scale>
+          bool = Accessor3d<KrylovType, ValueType>::has_scale>
 struct helper_functions_accessor {};
 
 // Accessors having a scale
@@ -442,11 +480,10 @@ template <typename ValueType, typename KrylovType>
 struct helper_functions_accessor<ValueType, KrylovType, true> {
     template <typename IndexType>
     static inline GKO_ATTRIBUTES void write_scale(
-        // static __forceinline__ __device__ void write_scale(
-        Accessor2d<KrylovType, ValueType> krylov_bases, IndexType col_idx,
-        ValueType value)
+        Accessor3d<KrylovType, ValueType> krylov_bases, IndexType vector_idx,
+        IndexType col_idx, ValueType value)
     {
-        krylov_bases.set_scale(col_idx, value);
+        krylov_bases.set_scale(vector_idx, col_idx, value);
     }
 };
 
@@ -456,9 +493,8 @@ template <typename ValueType, typename KrylovType>
 struct helper_functions_accessor<ValueType, KrylovType, false> {
     template <typename IndexType>
     static inline GKO_ATTRIBUTES void write_scale(
-        // static __forceinline__ __device__ void write_scale(
-        Accessor2d<KrylovType, ValueType> krylov_bases, IndexType col_idx,
-        ValueType value)
+        Accessor3d<KrylovType, ValueType> krylov_bases, IndexType vector_idx,
+        IndexType col_idx, ValueType value)
     {}
 };
 
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index c429512dbe9..4ef2a12c80c 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -98,7 +98,7 @@ namespace gmres_mixed {
                       matrix::Dense<remove_complex<_type1>> *residual_norm, \
                       matrix::Dense<_type1> *residual_norm_collection,      \
                       matrix::Dense<remove_complex<_type1>> *arnoldi_norm,  \
-                      Accessor2d<_type2, _type1> krylov_bases,              \
+                      Accessor3d<_type2, _type1> krylov_bases,              \
                       matrix::Dense<_type1> *next_krylov_basis,             \
                       Array<size_type> *final_iter_nums, size_type krylov_dim)
 
@@ -110,7 +110,7 @@ namespace gmres_mixed {
         matrix::Dense<_type1> *givens_sin, matrix::Dense<_type1> *givens_cos, \
         matrix::Dense<remove_complex<_type1>> *residual_norm,                 \
         matrix::Dense<_type1> *residual_norm_collection,                      \
-        Accessor2d<_type2, _type1> krylov_bases,                              \
+        Accessor3d<_type2, _type1> krylov_bases,                              \
         matrix::Dense<_type1> *hessenberg_iter,                               \
         matrix::Dense<_type1> *buffer_iter,                                   \
         const matrix::Dense<remove_complex<_type1>> *b_norm,                  \
@@ -124,7 +124,7 @@ namespace gmres_mixed {
 #define GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(_type1, _type2)          \
     void step_2(std::shared_ptr<const DefaultExecutor> exec,           \
                 const matrix::Dense<_type1> *residual_norm_collection, \
-                Accessor2dConst<_type2, _type1> krylov_bases,          \
+                Accessor3dConst<_type2, _type1> krylov_bases,          \
                 const matrix::Dense<_type1> *hessenberg,               \
                 matrix::Dense<_type1> *y,                              \
                 matrix::Dense<_type1> *before_preconditioner,          \
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 35526647902..accaaf1f135 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -92,37 +92,39 @@ constexpr int default_update_dim = 32;
 
 // Specialization, so the Accessor can use the same function as regular pointers
 template <typename Type1, typename Type2>
-xstd::enable_if_t<!Accessor2d<Type1, Type2>::has_scale,
-                  Accessor2d<cuda_type<Type1>, cuda_type<Type2>>>
-as_cuda_accessor(Accessor2d<Type1, Type2> acc)
+xstd::enable_if_t<!Accessor3d<Type1, Type2>::has_scale,
+                  Accessor3d<cuda_type<Type1>, cuda_type<Type2>>>
+as_cuda_accessor(Accessor3d<Type1, Type2> acc)
 {
-    return {as_cuda_type(acc.get_storage()), acc.get_stride()};
+    return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
+            acc.get_stride1()};
 }
 
 template <typename Type1, typename Type2>
-xstd::enable_if_t<Accessor2d<Type1, Type2>::has_scale,
-                  Accessor2d<cuda_type<Type1>, cuda_type<Type2>>>
-as_cuda_accessor(Accessor2d<Type1, Type2> acc)
+xstd::enable_if_t<Accessor3d<Type1, Type2>::has_scale,
+                  Accessor3d<cuda_type<Type1>, cuda_type<Type2>>>
+as_cuda_accessor(Accessor3d<Type1, Type2> acc)
 {
-    return {as_cuda_type(acc.get_storage()), acc.get_stride(),
-            as_cuda_type(acc.get_scale())};
+    return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
+            acc.get_stride1(), as_cuda_type(acc.get_scale())};
 }
 
 template <typename Type1, typename Type2>
-xstd::enable_if_t<!Accessor2dConst<Type1, Type2>::has_scale,
-                  Accessor2dConst<cuda_type<Type1>, cuda_type<Type2>>>
-as_cuda_accessor(const Accessor2dConst<Type1, Type2> &acc)
+xstd::enable_if_t<!Accessor3dConst<Type1, Type2>::has_scale,
+                  Accessor3dConst<cuda_type<Type1>, cuda_type<Type2>>>
+as_cuda_accessor(const Accessor3dConst<Type1, Type2> &acc)
 {
-    return {as_cuda_type(acc.get_storage()), acc.get_stride()};
+    return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
+            acc.get_stride1()};
 }
 
 template <typename Type1, typename Type2>
-xstd::enable_if_t<Accessor2dConst<Type1, Type2>::has_scale,
-                  Accessor2dConst<cuda_type<Type1>, cuda_type<Type2>>>
-as_cuda_accessor(const Accessor2dConst<Type1, Type2> &acc)
+xstd::enable_if_t<Accessor3dConst<Type1, Type2>::has_scale,
+                  Accessor3dConst<cuda_type<Type1>, cuda_type<Type2>>>
+as_cuda_accessor(const Accessor3dConst<Type1, Type2> &acc)
 {
-    return {as_cuda_type(acc.get_storage()), acc.get_stride(),
-            as_cuda_type(acc.get_scale())};
+    return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
+            acc.get_stride1(), as_cuda_type(acc.get_scale())};
 }
 
 
@@ -171,15 +173,15 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
                   matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
                   matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
-                  Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+                  Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
 {
     const auto num_rows = residual->get_size()[0];
     const auto num_rhs = residual->get_size()[1];
-    const dim3 grid_dim_1(
-        ceildiv(num_rows * krylov_bases.get_stride(), default_block_size), 1,
-        1);
+    const dim3 grid_dim_1(ceildiv((krylov_dim + 1) * krylov_bases.get_stride0(),
+                                  default_block_size),
+                          1, 1);
     const dim3 block_dim(default_block_size, 1, 1);
     constexpr auto block_size = default_block_size;
 
@@ -220,8 +222,9 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
     // exec->synchronize();
     /* */
     // std::cout << "Before initialize_2_2_kernel" << std::endl;
-    const dim3 grid_dim_2(ceildiv(num_rows * num_rhs, default_block_size), 1,
-                          1);
+    const dim3 grid_dim_2(
+        ceildiv(num_rows * krylov_bases.get_stride1(), default_block_size), 1,
+        1);
     initialize_2_2_kernel<block_size><<<grid_dim_2, block_dim>>>(
         residual->get_size()[0], residual->get_size()[1],
         as_cuda_type(residual->get_const_values()), residual->get_stride(),
@@ -242,12 +245,11 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
 template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi(std::shared_ptr<const CudaExecutor> exec,
                     matrix::Dense<ValueType> *next_krylov_basis,
-                    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+                    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
                     matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
                     const stopping_status *stop_status)
 {
     const auto stride_next_krylov = next_krylov_basis->get_stride();
-    const auto stride_krylov = krylov_bases.get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
     const auto dim_size = next_krylov_basis->get_size();
     auto cublas_handle = exec->get_cublas_handle();
@@ -304,7 +306,7 @@ template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi_reorth(
     std::shared_ptr<const CudaExecutor> exec,
     matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
     matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
@@ -312,7 +314,6 @@ void finish_arnoldi_reorth(
     Array<size_type> *num_reorth)
 {
     const auto stride_next_krylov = next_krylov_basis->get_stride();
-    const auto stride_krylov = krylov_bases.get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
     const auto stride_buffer = buffer_iter->get_stride();
     const auto stride_arnoldi = arnoldi_norm->get_stride();
@@ -322,8 +323,10 @@ void finish_arnoldi_reorth(
                          exec->get_num_multiprocessor() * 2);
     const dim3 block_size(default_dot_dim, default_dot_dim);
     size_type numReorth;
-    Accessor2d<ValueType, ValueType> next_krylov_accessor{
-        next_krylov_basis->get_values(), stride_next_krylov};
+    // Only 2D accessor, modeled as a 3D accessor:
+    Accessor3d<ValueType, ValueType> next_krylov_accessor{
+        next_krylov_basis->get_values(), stride_next_krylov,
+        stride_next_krylov};
     auto next_krylov_const_accessor = next_krylov_accessor.to_const();
     //    size_type num_reorth;
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
@@ -424,7 +427,7 @@ template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi_CGS(
     std::shared_ptr<const CudaExecutor> exec,
     matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
     matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
@@ -432,7 +435,6 @@ void finish_arnoldi_CGS(
     Array<size_type> *num_reorth)
 {
     const auto stride_next_krylov = next_krylov_basis->get_stride();
-    const auto stride_krylov = krylov_bases.get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
     const auto stride_buffer = buffer_iter->get_stride();
     const auto stride_arnoldi = arnoldi_norm->get_stride();
@@ -442,8 +444,9 @@ void finish_arnoldi_CGS(
                          exec->get_num_multiprocessor() * 2);
     const dim3 block_size(default_dot_dim, default_dot_dim);
     size_type numReorth;
-    Accessor2d<ValueType, ValueType> next_krylov_accessor{
-        next_krylov_basis->get_values(), stride_next_krylov};
+    Accessor3d<ValueType, ValueType> next_krylov_accessor{
+        next_krylov_basis->get_values(), stride_next_krylov,
+        stride_next_krylov};
     auto next_krylov_const_accessor = next_krylov_accessor.to_const();
 
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
@@ -605,7 +608,7 @@ template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi_CGS2(
     std::shared_ptr<const CudaExecutor> exec,
     matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
     matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
@@ -615,7 +618,6 @@ void finish_arnoldi_CGS2(
 {
     using non_complex = remove_complex<ValueType>;
     const auto stride_next_krylov = next_krylov_basis->get_stride();
-    const auto stride_krylov = krylov_bases.get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
     const auto stride_buffer = buffer_iter->get_stride();
     const auto stride_arnoldi = arnoldi_norm->get_stride();
@@ -629,8 +631,9 @@ void finish_arnoldi_CGS2(
     const dim3 block_size(default_dot_dim, default_dot_dim);
     size_type numReorth;
 
-    Accessor2d<ValueType, ValueType> next_krylov_accessor{
-        next_krylov_basis->get_values(), stride_next_krylov};
+    Accessor3d<ValueType, ValueType> next_krylov_accessor{
+        next_krylov_basis->get_values(), stride_next_krylov,
+        stride_next_krylov};
     auto next_krylov_const_accessor = next_krylov_accessor.to_const();
 
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
@@ -1009,7 +1012,7 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
             matrix::Dense<ValueType> *givens_cos,
             matrix::Dense<remove_complex<ValueType>> *residual_norm,
             matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+            Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
             matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
             const matrix::Dense<remove_complex<ValueType>> *b_norm,
@@ -1082,14 +1085,13 @@ void solve_upper_triangular(
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void calculate_qy(Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
-                  size_type krylov_cols, const matrix::Dense<ValueType> *y,
+void calculate_qy(Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+                  size_type num_krylov_bases, const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const Array<size_type> *final_iter_nums)
 {
     const auto num_rows = before_preconditioner->get_size()[0];
-    const auto num_cols = krylov_cols;  // krylov_bases->get_size()[1];
-    const auto num_rhs = before_preconditioner->get_size()[1];
+    const auto num_cols = before_preconditioner->get_size()[1];
     const auto stride_before_preconditioner =
         before_preconditioner->get_stride();
 
@@ -1102,7 +1104,7 @@ void calculate_qy(Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
 
 
     calculate_Qy_kernel<block_size><<<grid_dim, block_dim>>>(
-        num_rows, num_cols, num_rhs, as_cuda_accessor(krylov_bases),
+        num_rows, num_cols, as_cuda_accessor(krylov_bases),
         as_cuda_type(y->get_const_values()), y->get_stride(),
         as_cuda_type(before_preconditioner->get_values()),
         stride_before_preconditioner,
@@ -1115,20 +1117,20 @@ void calculate_qy(Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
 template <typename ValueType, typename ValueTypeKrylovBases>
 void step_2(std::shared_ptr<const CudaExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+            Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
             const Array<size_type> *final_iter_nums)
 {
     // since hessenberg has dims:  iters x iters * num_rhs
-    // and krylov should have dims:  sysmtx[0] x (iters + 1) * num_rhs
-    const auto num_rhs = before_preconditioner->get_size()[1];
-    const auto hessenberg_cols = hessenberg->get_size()[1];
-    const auto krylov_cols = hessenberg_cols + num_rhs;
+    // krylov_bases has dims:  (iters + 1) x sysmtx[0] x num_rhs
+    const auto iters =
+        hessenberg->get_size()[1] / before_preconditioner->get_size()[1];
+    const auto num_krylov_bases = iters + 1;
     solve_upper_triangular(residual_norm_collection, hessenberg, y,
                            final_iter_nums);
-    calculate_qy(krylov_bases, krylov_cols, y, before_preconditioner,
+    calculate_qy(krylov_bases, num_krylov_bases, y, before_preconditioner,
                  final_iter_nums);
 }
 
diff --git a/cuda/test/solver/gmres_mixed_kernels.cpp b/cuda/test/solver/gmres_mixed_kernels.cpp
index 5b66f6d0e42..28d3701c1a9 100644
--- a/cuda/test/solver/gmres_mixed_kernels.cpp
+++ b/cuda/test/solver/gmres_mixed_kernels.cpp
@@ -60,9 +60,10 @@ class GmresMixed : public ::testing::Test {
     using value_type = double;
     using krylov_type = float;
     using index_type = int;
-    using Accessor2d = gko::kernels::Accessor2d<krylov_type, value_type>;
-    using Accessor2dHelper =
-        gko::kernels::Accessor2dHelper<value_type, krylov_type>;
+    using size_type = gko::size_type;
+    using Accessor3d = gko::kernels::Accessor3d<krylov_type, value_type>;
+    using Accessor3dHelper =
+        gko::kernels::Accessor3dHelper<value_type, krylov_type>;
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
 
@@ -90,17 +91,31 @@ class GmresMixed : public ::testing::Test {
             std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
     }
 
-    gko::Array<krylov_type> generate_krylov_bases(int num_rows, int num_cols)
+    Accessor3dHelper generate_krylov_helper(gko::dim<3> size)
     {
-        gko::Array<krylov_type> result{
-            ref, static_cast<gko::size_type>(num_rows * num_cols)};
+        auto helper = Accessor3dHelper{ref, size};
+        auto &bases = helper.get_bases();
+        const auto num_rows = size[0] * size[1];
+        const auto num_cols = size[2];
         auto temp_krylov_bases = gko::test::generate_random_matrix<Dense>(
             num_rows, num_cols,
             std::uniform_int_distribution<index_type>(num_cols, num_cols),
             std::normal_distribution<krylov_type>(-1.0, 1.0), rand_engine, ref);
         std::copy_n(temp_krylov_bases->get_const_values(),
-                    result.get_num_elems(), result.get_data());
-        return result;
+                    bases.get_num_elems(), bases.get_data());
+        if (Accessor3d::has_scale) {
+            auto accessor = helper.get_accessor();
+            auto dist = std::normal_distribution<value_type>(-1, 1);
+            for (size_type k = 0; k < size[0]; ++k) {
+                for (size_type i = 0; i < size[2]; ++i) {
+                    gko::kernels::helper_functions_accessor<
+                        value_type, krylov_type>::write_scale(accessor, k, i,
+                                                              dist(
+                                                                  rand_engine));
+                }
+            }
+        }
+        return helper;
     }
 
     void initialize_data()
@@ -113,12 +128,11 @@ class GmresMixed : public ::testing::Test {
         b = gen_mtx(m, n);
         b_norm = gen_mtx(1, n);
         arnoldi_norm = gen_mtx(3, n);
-        gko::dim<2> krylov_bases_dim(
-            m, (gko::solver::default_krylov_dim_mixed + 1) * n);
-        acc_helper = Accessor2dHelper{krylov_bases_dim, ref};
-        acc_helper.get_bases() =
-            generate_krylov_bases(krylov_bases_dim[0], krylov_bases_dim[1]);
+        gko::dim<3> krylov_bases_dim(gko::solver::default_krylov_dim_mixed + 1,
+                                     m, n);
+        acc_helper = generate_krylov_helper(krylov_bases_dim);
         krylov_bases_accessor = acc_helper.get_accessor();
+
         next_krylov_basis = gen_mtx(m, n);
         hessenberg = gen_mtx(gko::solver::default_krylov_dim_mixed + 1,
                              gko::solver::default_krylov_dim_mixed * n);
@@ -162,7 +176,7 @@ class GmresMixed : public ::testing::Test {
         d_b_norm->copy_from(b_norm.get());
         d_arnoldi_norm = Mtx::create(cuda);
         d_arnoldi_norm->copy_from(arnoldi_norm.get());
-        d_acc_helper = Accessor2dHelper{{}, cuda};
+        d_acc_helper = Accessor3dHelper{cuda, {}};
         d_acc_helper = acc_helper;
         d_krylov_bases_accessor = d_acc_helper.get_accessor();
         d_next_krylov_basis = Mtx::create(cuda);
@@ -222,8 +236,8 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> b_norm;
     std::unique_ptr<Mtx> arnoldi_norm;
-    Accessor2dHelper acc_helper;
-    Accessor2d krylov_bases_accessor;
+    Accessor3dHelper acc_helper;
+    Accessor3d krylov_bases_accessor;
     std::unique_ptr<Mtx> next_krylov_basis;
     std::unique_ptr<Mtx> hessenberg;
     std::unique_ptr<Mtx> hessenberg_iter;
@@ -244,8 +258,8 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> d_b;
     std::unique_ptr<Mtx> d_b_norm;
     std::unique_ptr<Mtx> d_arnoldi_norm;
-    Accessor2dHelper d_acc_helper;
-    Accessor2d d_krylov_bases_accessor;
+    Accessor3dHelper d_acc_helper;
+    Accessor3d d_krylov_bases_accessor;
     std::unique_ptr<Mtx> d_next_krylov_basis;
     std::unique_ptr<Mtx> d_hessenberg;
     std::unique_ptr<Mtx> d_hessenberg_iter;
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index f4b49c40ba7..ec0fd01bd44 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -73,7 +73,7 @@ namespace {
 
 template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
-                    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+                    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
                     matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
                     const stopping_status *stop_status)
 {
@@ -89,18 +89,14 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
 #pragma omp parallel for reduction(add : hessenberg_iter_entry)
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter_entry +=
-                    next_krylov_basis->at(j, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
             }
             hessenberg_iter->at(k, i) = hessenberg_iter_entry;
 
 #pragma omp parallel for
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
             }
         }
         // for i in 1:iter
@@ -120,9 +116,7 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
 #pragma omp parallel for
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(
-                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
-                next_krylov_basis->at(j, i));
+            krylov_bases.write(iter + 1, j, i, next_krylov_basis->at(j, i));
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -134,7 +128,7 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
 template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi_reorth(
     matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
     const stopping_status *stop_status)
@@ -159,18 +153,14 @@ void finish_arnoldi_reorth(
 #pragma omp parallel for reduction(add : hessenberg_iter_entry)
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter_entry +=
-                    next_krylov_basis->at(j, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
             }
             hessenberg_iter->at(k, i) = hessenberg_iter_entry;
 
 #pragma omp parallel for
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
             }
             // DONE: TODO remove abs() as soon as type for arnoldi_norm has
             // changed!
@@ -182,8 +172,7 @@ void finish_arnoldi_reorth(
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     reorth += next_krylov_basis->at(j, i) *
-                              krylov_bases.read(
-                                  j, next_krylov_basis->get_size()[1] * k + i);
+                              krylov_bases.read(k, j, i);
                 }
                 hessenberg_iter->at(k, i) += reorth;
 
@@ -191,9 +180,7 @@ void finish_arnoldi_reorth(
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
-                        reorth *
-                        krylov_bases.read(
-                            j, next_krylov_basis->get_size()[1] * k + i);
+                        reorth * krylov_bases.read(k, j, i);
                 }
             }
         }
@@ -218,9 +205,7 @@ void finish_arnoldi_reorth(
 #pragma omp parallel for
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(
-                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
-                next_krylov_basis->at(j, i));
+            krylov_bases.write(iter + 1, j, i, next_krylov_basis->at(j, i));
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -232,7 +217,7 @@ void finish_arnoldi_reorth(
 template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi_CGS(
     matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
     matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
@@ -259,9 +244,7 @@ void finish_arnoldi_CGS(
 #pragma omp parallel for reduction(add : hessenberg_iter_entry)
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter_entry +=
-                    next_krylov_basis->at(j, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
             }
             hessenberg_iter->at(k, i) = hessenberg_iter_entry;
         }
@@ -272,9 +255,7 @@ void finish_arnoldi_CGS(
 #pragma omp parallel for
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
             }
         }
         // for i in 1:iter
@@ -303,10 +284,8 @@ void finish_arnoldi_CGS(
 #pragma omp parallel for reduction(add : hessenberg_iter_entry)
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
-                    hessenberg_iter_entry +=
-                        next_krylov_basis->at(j, i) *
-                        krylov_bases.read(
-                            j, next_krylov_basis->get_size()[1] * k + i);
+                    hessenberg_iter_entry += next_krylov_basis->at(j, i) *
+                                             krylov_bases.read(k, j, i);
                 }
                 buffer_iter->at(k, i) = hessenberg_iter_entry;
             }
@@ -318,9 +297,7 @@ void finish_arnoldi_CGS(
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
-                        buffer_iter->at(k, i) *
-                        krylov_bases.read(
-                            j, next_krylov_basis->get_size()[1] * k + i);
+                        buffer_iter->at(k, i) * krylov_bases.read(k, j, i);
                 }
             }
             // for i in 1:iter
@@ -347,9 +324,7 @@ void finish_arnoldi_CGS(
 #pragma omp parallel for
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(
-                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
-                next_krylov_basis->at(j, i));
+            krylov_bases.write(iter + 1, j, i, next_krylov_basis->at(j, i));
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -361,7 +336,7 @@ void finish_arnoldi_CGS(
 template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi_CGS2(
     matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
     matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
@@ -394,9 +369,7 @@ void finish_arnoldi_CGS2(
             ValueType hessenberg_iter_entry = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter_entry +=
-                    next_krylov_basis->at(j, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
             }
             hessenberg_iter->at(k, i) = hessenberg_iter_entry;
         }
@@ -416,9 +389,7 @@ void finish_arnoldi_CGS2(
 #pragma omp parallel for
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
             }
         }
 #ifdef TIMING
@@ -465,10 +436,8 @@ void finish_arnoldi_CGS2(
                 ValueType hessenberg_iter_entry = zero<ValueType>();
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
-                    hessenberg_iter_entry +=
-                        next_krylov_basis->at(j, i) *
-                        krylov_bases.read(
-                            j, next_krylov_basis->get_size()[1] * k + i);
+                    hessenberg_iter_entry += next_krylov_basis->at(j, i) *
+                                             krylov_bases.read(k, j, i);
                 }
                 buffer_iter->at(k, i) = hessenberg_iter_entry;
             }
@@ -480,9 +449,7 @@ void finish_arnoldi_CGS2(
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
-                        buffer_iter->at(k, i) *
-                        krylov_bases.read(
-                            j, next_krylov_basis->get_size()[1] * k + i);
+                        buffer_iter->at(k, i) * krylov_bases.read(k, j, i);
                 }
             }
             // for i in 1:iter
@@ -503,7 +470,7 @@ void finish_arnoldi_CGS2(
             // nrmI = infnorm(next_krylov_basis)
         }
         helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
-            krylov_bases, next_krylov_basis->get_size()[1] * (iter + 1) + i,
+            krylov_bases, iter + 1, i,
             arnoldi_norm->at(2, i) / arnoldi_norm->at(1, i));
         // reorthogonalization
         /*
@@ -520,9 +487,7 @@ void finish_arnoldi_CGS2(
 #pragma omp parallel for
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(
-                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
-                next_krylov_basis->at(j, i));
+            krylov_bases.write(iter + 1, j, i, next_krylov_basis->at(j, i));
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -641,7 +606,7 @@ void solve_upper_triangular(
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void calculate_qy(Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+void calculate_qy(Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
                   const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const size_type *final_iter_nums)
@@ -652,9 +617,7 @@ void calculate_qy(Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
             before_preconditioner->at(i, k) = zero<ValueType>();
             for (size_type j = 0; j < final_iter_nums[k]; ++j) {
                 before_preconditioner->at(i, k) +=
-                    krylov_bases.read(
-                        i, j * before_preconditioner->get_size()[1] + k) *
-                    y->at(j, k);
+                    krylov_bases.read(j, i, k) * y->at(j, k);
             }
         }
     }
@@ -710,7 +673,7 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
                   matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
                   matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
-                  Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+                  Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
 {
@@ -736,7 +699,8 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
         // j)
         //           << std::endl;
         helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
-            krylov_bases, j, arnoldi_norm->at(2, j) / residual_norm->at(0, j));
+            krylov_bases, {0}, j,
+            arnoldi_norm->at(2, j) / residual_norm->at(0, j));
 
 #pragma omp parallel for
         for (size_type i = 0; i < krylov_dim + 1; ++i) {
@@ -750,24 +714,23 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             auto value = residual->at(i, j) / residual_norm->at(0, j);
-            krylov_bases.write(i, j, value);
+            krylov_bases.write({0}, i, j, value);
             next_krylov_basis->at(i, j) = value;
         }
         final_iter_nums->get_data()[j] = 0;
     }
 
 #pragma omp parallel for
-    for (size_type j = residual->get_size()[1];
-         j < (krylov_dim + 1) * residual->get_size()[1]; ++j) {
-        helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
-            krylov_bases, j, one<remove_complex<ValueType>>());
-    }
-
-#pragma omp parallel for
-    for (size_type i = 0; i < residual->get_size()[0]; ++i) {
-        for (size_type j = residual->get_size()[1];
-             j < (krylov_dim + 1) * residual->get_size()[1]; ++j) {
-            krylov_bases.write(i, j, zero<ValueType>());
+    for (size_type k = 1; k < krylov_dim + 1; ++k) {
+        for (size_type j = 0; j < residual->get_size()[1]; ++j) {
+            helper_functions_accessor<ValueType, ValueTypeKrylovBases>::
+                write_scale(krylov_bases, k, j,
+                            one<remove_complex<ValueType>>());
+        }
+        for (size_type i = 0; i < residual->get_size()[0]; ++i) {
+            for (size_type j = 0; j < residual->get_size()[1]; ++j) {
+                krylov_bases.write(k, i, j, zero<ValueType>());
+            }
         }
     }
 }
@@ -783,7 +746,7 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
             matrix::Dense<ValueType> *givens_cos,
             matrix::Dense<remove_complex<ValueType>> *residual_norm,
             matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+            Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
             matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
             const matrix::Dense<remove_complex<ValueType>> *b_norm,
@@ -831,7 +794,7 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
 template <typename ValueType, typename ValueTypeKrylovBases>
 void step_2(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+            Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
diff --git a/omp/test/solver/gmres_mixed_kernels.cpp b/omp/test/solver/gmres_mixed_kernels.cpp
index 6114969bced..47b8c23484b 100644
--- a/omp/test/solver/gmres_mixed_kernels.cpp
+++ b/omp/test/solver/gmres_mixed_kernels.cpp
@@ -62,9 +62,10 @@ class GmresMixed : public ::testing::Test {
     using value_type = double;
     using krylov_type = float;
     using index_type = int;
-    using Accessor2d = gko::kernels::Accessor2d<krylov_type, value_type>;
-    using Accessor2dHelper =
-        gko::kernels::Accessor2dHelper<value_type, krylov_type>;
+    using size_type = gko::size_type;
+    using Accessor3d = gko::kernels::Accessor3d<krylov_type, value_type>;
+    using Accessor3dHelper =
+        gko::kernels::Accessor3dHelper<value_type, krylov_type>;
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
 
@@ -92,17 +93,31 @@ class GmresMixed : public ::testing::Test {
             std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
     }
 
-    gko::Array<krylov_type> generate_krylov_bases(int num_rows, int num_cols)
+    Accessor3dHelper generate_krylov_helper(gko::dim<3> size)
     {
-        gko::Array<krylov_type> result{
-            ref, static_cast<gko::size_type>(num_rows * num_cols)};
+        auto helper = Accessor3dHelper{ref, size};
+        auto &bases = helper.get_bases();
+        const auto num_rows = size[0] * size[1];
+        const auto num_cols = size[2];
         auto temp_krylov_bases = gko::test::generate_random_matrix<Dense>(
             num_rows, num_cols,
             std::uniform_int_distribution<index_type>(num_cols, num_cols),
             std::normal_distribution<krylov_type>(-1.0, 1.0), rand_engine, ref);
         std::copy_n(temp_krylov_bases->get_const_values(),
-                    result.get_num_elems(), result.get_data());
-        return result;
+                    bases.get_num_elems(), bases.get_data());
+        if (Accessor3d::has_scale) {
+            auto accessor = helper.get_accessor();
+            auto dist = std::normal_distribution<value_type>(-1, 1);
+            for (size_type k = 0; k < size[0]; ++k) {
+                for (size_type i = 0; i < size[2]; ++i) {
+                    gko::kernels::helper_functions_accessor<
+                        value_type, krylov_type>::write_scale(accessor, k, i,
+                                                              dist(
+                                                                  rand_engine));
+                }
+            }
+        }
+        return helper;
     }
 
     void initialize_data()
@@ -115,11 +130,9 @@ class GmresMixed : public ::testing::Test {
         b = gen_mtx(m, n);
         b_norm = gen_mtx(1, n);
         arnoldi_norm = gen_mtx(3, n);
-        gko::dim<2> krylov_bases_dim(
-            m, (gko::solver::default_krylov_dim_mixed + 1) * n);
-        acc_helper = Accessor2dHelper{krylov_bases_dim, ref};
-        acc_helper.get_bases() =
-            generate_krylov_bases(krylov_bases_dim[0], krylov_bases_dim[1]);
+        gko::dim<3> krylov_bases_dim(gko::solver::default_krylov_dim_mixed + 1,
+                                     m, n);
+        acc_helper = generate_krylov_helper(krylov_bases_dim);
         krylov_bases_accessor = acc_helper.get_accessor();
 
         next_krylov_basis = gen_mtx(m, n);
@@ -165,7 +178,7 @@ class GmresMixed : public ::testing::Test {
         d_b_norm->copy_from(b_norm.get());
         d_arnoldi_norm = Mtx::create(omp);
         d_arnoldi_norm->copy_from(arnoldi_norm.get());
-        d_acc_helper = Accessor2dHelper{{}, omp};
+        d_acc_helper = Accessor3dHelper{omp, {}};
         d_acc_helper = acc_helper;
         d_krylov_bases_accessor = d_acc_helper.get_accessor();
         d_next_krylov_basis = Mtx::create(omp);
@@ -224,8 +237,8 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> b_norm;
     std::unique_ptr<Mtx> arnoldi_norm;
-    Accessor2dHelper acc_helper;
-    Accessor2d krylov_bases_accessor;
+    Accessor3dHelper acc_helper;
+    Accessor3d krylov_bases_accessor;
     std::unique_ptr<Mtx> next_krylov_basis;
     std::unique_ptr<Mtx> hessenberg;
     std::unique_ptr<Mtx> hessenberg_iter;
@@ -246,8 +259,8 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> d_b;
     std::unique_ptr<Mtx> d_b_norm;
     std::unique_ptr<Mtx> d_arnoldi_norm;
-    Accessor2dHelper d_acc_helper;
-    Accessor2d d_krylov_bases_accessor;
+    Accessor3dHelper d_acc_helper;
+    Accessor3d d_krylov_bases_accessor;
     std::unique_ptr<Mtx> d_next_krylov_basis;
     std::unique_ptr<Mtx> d_hessenberg;
     std::unique_ptr<Mtx> d_hessenberg_iter;
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index 320a2e6687d..b1124fe0d4d 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <iostream>
 
+
 #include "core/solver/gmres_mixed_accessor.hpp"
 
 
@@ -60,7 +61,7 @@ namespace {
 
 template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
-                    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+                    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
                     matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
                     const stopping_status *stop_status)
 {
@@ -72,15 +73,11 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
             hessenberg_iter->at(k, i) = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
-                    next_krylov_basis->at(j, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
             }
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
             }
         }
         // for i in 1:iter
@@ -99,9 +96,7 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(
-                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
-                next_krylov_basis->at(j, i));
+            krylov_bases.write((iter + 1), j, i, next_krylov_basis->at(j, i));
             // std::cout << next_krylov_basis->at(j, i) << " - "
             //           << krylov_bases.read(
             //                  j,
@@ -119,7 +114,7 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
 template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi_reorth(
     matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
     matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
@@ -144,15 +139,11 @@ void finish_arnoldi_reorth(
             hessenberg_iter->at(k, i) = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
-                    next_krylov_basis->at(j, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
             }
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
             }
             // if (squared_norm(hessenberg_iter->at(k, i) *
             //                  hessenberg_iter->at(k, i)) >
@@ -165,18 +156,14 @@ void finish_arnoldi_reorth(
                     // DONE: TODO find good alternative here, so the type of
                     // DONE: arnoldi_norm can change to
                     // remove_complex<ValueType>!!! arnoldi_norm->at(1, i) +=
-                    buffer_iter->at(k, i) +=
-                        next_krylov_basis->at(j, i) *
-                        krylov_bases.read(
-                            j, next_krylov_basis->get_size()[1] * k + i);
+                    buffer_iter->at(k, i) += next_krylov_basis->at(j, i) *
+                                             krylov_bases.read(k, j, i);
                 }
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
-                    next_krylov_basis->at(j, i) -=
-                        buffer_iter->at(k, i) *
-                        // arnoldi_norm->at(1, i) *
-                        krylov_bases.read(
-                            j, next_krylov_basis->get_size()[1] * k + i);
+                    next_krylov_basis->at(j, i) -= buffer_iter->at(k, i) *
+                                                   // arnoldi_norm->at(1, i) *
+                                                   krylov_bases.read(k, j, i);
                 }
                 // hessenberg_iter->at(k, i) += arnoldi_norm->at(1, i);
                 hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
@@ -203,9 +190,7 @@ void finish_arnoldi_reorth(
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(
-                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
-                next_krylov_basis->at(j, i));
+            krylov_bases.write(iter + 1, j, i, next_krylov_basis->at(j, i));
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -217,7 +202,7 @@ void finish_arnoldi_reorth(
 template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi_CGS(
     matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
     matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
@@ -244,9 +229,7 @@ void finish_arnoldi_CGS(
             hessenberg_iter->at(k, i) = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
-                    next_krylov_basis->at(j, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
             }
         }
         // for i in 1:iter
@@ -255,9 +238,7 @@ void finish_arnoldi_CGS(
         for (size_type k = 0; k < iter + 1; ++k) {
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
             }
         }
         // for i in 1:iter
@@ -270,8 +251,6 @@ void finish_arnoldi_CGS(
         arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
         // nrmN = norm(next_krylov_basis)
 
-        // DONE: TODO the abs() should be removed here as soon as the type of
-        // DONE: arnoldi_norm has changed!
         for (size_type l = 1;
              (arnoldi_norm->at(1, i)) < (arnoldi_norm->at(0, i)) && l < 3;
              l++) {
@@ -280,10 +259,8 @@ void finish_arnoldi_CGS(
                 buffer_iter->at(k, i) = zero<ValueType>();
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
-                    buffer_iter->at(k, i) +=
-                        next_krylov_basis->at(j, i) *
-                        krylov_bases.read(
-                            j, next_krylov_basis->get_size()[1] * k + i);
+                    buffer_iter->at(k, i) += next_krylov_basis->at(j, i) *
+                                             krylov_bases.read(k, j, i);
                 }
             }
             // for i in 1:iter
@@ -293,9 +270,7 @@ void finish_arnoldi_CGS(
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
-                        buffer_iter->at(k, i) *
-                        krylov_bases.read(
-                            j, next_krylov_basis->get_size()[1] * k + i);
+                        buffer_iter->at(k, i) * krylov_bases.read(k, j, i);
                 }
                 hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
             }
@@ -323,9 +298,7 @@ void finish_arnoldi_CGS(
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(
-                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
-                next_krylov_basis->at(j, i));
+            krylov_bases.write(iter + 1, j, i, next_krylov_basis->at(j, i));
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -337,7 +310,7 @@ void finish_arnoldi_CGS(
 template <typename ValueType, typename ValueTypeKrylovBases>
 void finish_arnoldi_CGS2(
     matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
     matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
@@ -364,9 +337,7 @@ void finish_arnoldi_CGS2(
             hessenberg_iter->at(k, i) = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
-                    next_krylov_basis->at(j, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
             }
         }
         // for i in 1:iter
@@ -375,9 +346,7 @@ void finish_arnoldi_CGS2(
         for (size_type k = 0; k < iter + 1; ++k) {
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) *
-                    krylov_bases.read(j,
-                                      next_krylov_basis->get_size()[1] * k + i);
+                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
             }
         }
         // for i in 1:iter
@@ -398,8 +367,6 @@ void finish_arnoldi_CGS2(
         //           << " - " << 0 << std::endl;
         // nrmN = norm(next_krylov_basis)
 
-        // DONE: TODO the abs() should be removed here as soon as the type of
-        // DONE: arnoldi_norm has changed!
         for (size_type l = 1;
              (arnoldi_norm->at(1, i)) < (arnoldi_norm->at(0, i)) && l < 3;
              l++) {
@@ -409,10 +376,8 @@ void finish_arnoldi_CGS2(
                 buffer_iter->at(k, i) = zero<ValueType>();
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
-                    buffer_iter->at(k, i) +=
-                        next_krylov_basis->at(j, i) *
-                        krylov_bases.read(
-                            j, next_krylov_basis->get_size()[1] * k + i);
+                    buffer_iter->at(k, i) += next_krylov_basis->at(j, i) *
+                                             krylov_bases.read(k, j, i);
                 }
             }
             // for i in 1:iter
@@ -422,9 +387,7 @@ void finish_arnoldi_CGS2(
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
-                        buffer_iter->at(k, i) *
-                        krylov_bases.read(
-                            j, next_krylov_basis->get_size()[1] * k + i);
+                        buffer_iter->at(k, i) * krylov_bases.read(k, j, i);
                 }
                 hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
             }
@@ -448,7 +411,7 @@ void finish_arnoldi_CGS2(
             // nrmN = norm(next_krylov_basis)
         }
         helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
-            krylov_bases, next_krylov_basis->get_size()[1] * (iter + 1) + i,
+            krylov_bases, iter + 1, i,
             arnoldi_norm->at(2, i) / arnoldi_norm->at(1, i));
         /*
         // reorthogonalization
@@ -468,9 +431,7 @@ void finish_arnoldi_CGS2(
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(
-                j, next_krylov_basis->get_size()[1] * (iter + 1) + i,
-                next_krylov_basis->at(j, i));
+            krylov_bases.write(iter + 1, j, i, next_krylov_basis->at(j, i));
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -586,7 +547,7 @@ void solve_upper_triangular(
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-void calculate_qy(Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+void calculate_qy(Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
                   const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const size_type *final_iter_nums)
@@ -596,9 +557,7 @@ void calculate_qy(Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
             before_preconditioner->at(i, k) = zero<ValueType>();
             for (size_type j = 0; j < final_iter_nums[k]; ++j) {
                 before_preconditioner->at(i, k) +=
-                    krylov_bases.read(
-                        i, j * before_preconditioner->get_size()[1] + k) *
-                    y->at(j, k);
+                    krylov_bases.read(j, i, k) * y->at(j, k);
             }
         }
     }
@@ -647,7 +606,7 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
                   matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
                   matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
-                  Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+                  Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
 {
@@ -669,7 +628,8 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
         // j)
         //           << std::endl;
         helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
-            krylov_bases, j, arnoldi_norm->at(2, j) / residual_norm->at(0, j));
+            krylov_bases, {0}, j,
+            arnoldi_norm->at(2, j) / residual_norm->at(0, j));
 
         for (size_type i = 0; i < krylov_dim + 1; ++i) {
             if (i == 0) {
@@ -679,7 +639,7 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
             }
         }
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
-            krylov_bases.write(i, j,
+            krylov_bases.write({0}, i, j,
                                residual->at(i, j) / residual_norm->at(0, j));
             next_krylov_basis->at(i, j) =
                 residual->at(i, j) / residual_norm->at(0, j);
@@ -687,12 +647,14 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
         final_iter_nums->get_data()[j] = 0;
     }
 
-    for (size_type j = residual->get_size()[1];
-         j < (krylov_dim + 1) * residual->get_size()[1]; ++j) {
-        helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
-            krylov_bases, j, one<remove_complex<ValueType>>());
-        for (size_type i = 0; i < residual->get_size()[0]; ++i) {
-            krylov_bases.write(i, j, zero<ValueType>());
+    for (size_type k = 1; k < krylov_dim + 1; ++k) {
+        for (size_type j = 0; j < residual->get_size()[1]; ++j) {
+            helper_functions_accessor<ValueType, ValueTypeKrylovBases>::
+                write_scale(krylov_bases, k, j,
+                            one<remove_complex<ValueType>>());
+            for (size_type i = 0; i < residual->get_size()[0]; ++i) {
+                krylov_bases.write(k, i, j, zero<ValueType>());
+            }
         }
     }
 }
@@ -710,7 +672,7 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
             matrix::Dense<ValueType> *givens_cos,
             matrix::Dense<remove_complex<ValueType>> *residual_norm,
             matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+            Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
             matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
             const matrix::Dense<remove_complex<ValueType>> *b_norm,
@@ -766,7 +728,7 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
 template <typename ValueType, typename ValueTypeKrylovBases>
 void step_2(std::shared_ptr<const ReferenceExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+            Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,

From 54084b7b8f4fb407f834660ff931b7c437d0c951 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 2 Jun 2020 09:13:36 +0200
Subject: [PATCH 0357/1008] Make memory access to krylov_bases coalesced again

This is only done for finish_arnoldi_CGS2 by changing the kernels to
adapt to the new storage format of krylov_bases.
---
 common/solver/gmres_mixed_kernels.hpp.inc | 114 +++++++++++++++++++++-
 cuda/solver/gmres_mixed_kernels.cu        |  84 +++++++++-------
 2 files changed, 163 insertions(+), 35 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 46252ee711c..28878cd20bc 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -433,7 +433,7 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
 {
     const auto tidx = threadIdx.x;
     const auto tidy = threadIdx.y;
-    const auto col_idx = blockIdx.x * default_dot_dim + tidx;
+    const size_type col_idx = blockIdx.x * default_dot_dim + tidx;
     const auto num = ceildiv(num_rows, gridDim.y);
     const auto start_row = blockIdx.y * num;
     const auto end_row =
@@ -485,7 +485,7 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_1(
     const auto tidx = threadIdx.x;
     const auto tidy = threadIdx.y;
     const auto num_cols_iters = num_cols * num_iters;
-    const auto col_idx_iters = blockIdx.x * default_dot_dim + tidx;
+    const auto col_idx_iters = blockIdx.x * blockDim.x + tidx;
     const auto col_idx = col_idx_iters % num_cols;
     const auto num = ceildiv(num_rows, gridDim.y);
     const auto start_row = blockIdx.y * num;
@@ -548,6 +548,116 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_1(
     //    }
 }
 
+template <int dot_dim, typename ValueType, typename ValueTypeKrylovBases>
+__global__ __launch_bounds__(dot_dim *dot_dim) void multidot_kernel_num_iters_2(
+    size_type num_iters, size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov,
+    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status)
+{
+    /*
+     * In general in this kernel (thread indices appear to be col-major):
+     * grid_dim
+     *   x: for col_idx (^= which right hand side)
+     *   y: for row_idx
+     *   z: for num_iters (number of krylov vectors)
+     * block_dim
+     *   x: for col_idx
+     *   y: for row_idx
+     *   (z not used, must be set to 1 in dim)
+     */
+    const size_type tidx = threadIdx.x;
+    const size_type tidy = threadIdx.y;
+    const size_type col_idx = blockIdx.x * blockDim.x + threadIdx.x;
+    const size_type num_rows_per_thread = ceildiv(num_rows, gridDim.y);
+    const size_type start_row = blockIdx.y * num_rows_per_thread + threadIdx.y;
+    const auto end_row = min((blockIdx.y + 1) * num_rows_per_thread, num_rows);
+    const size_type k = blockIdx.z;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+    __shared__ UninitializedArray<ValueType, dot_dim * dot_dim>
+        reduction_helper_array;
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
+
+    ValueType local_res = zero<ValueType>();
+    if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+        for (size_type i = start_row; i < end_row; i += blockDim.y) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            local_res += next_krylov_basis[next_krylov_idx] *
+                         krylov_bases.read(k, i, col_idx);
+        }
+    }
+    // Transpose local_res, so each warp contains a local_res from the same
+    // right hand side
+    reduction_helper[tidx * dot_dim + tidy] = local_res;
+    auto thread_block = group::this_thread_block();
+    thread_block.sync();
+    local_res = reduction_helper[tidy * dot_dim + tidx];
+    const auto new_col_idx = blockIdx.x * blockDim.x + tidy;
+    const auto tile_block = group::tiled_partition<dot_dim>(thread_block);
+    const auto sum =
+        reduce(tile_block, local_res,
+               [](const ValueType &a, const ValueType &b) { return a + b; });
+    if (tidx == 0 && new_col_idx < num_cols &&
+        !stop_status[new_col_idx].has_stopped()) {
+        const auto hessenberg_idx = k * stride_hessenberg + new_col_idx;
+        atomic_add(hessenberg_iter + hessenberg_idx, sum);
+    }
+}
+
+
+template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+__global__ __launch_bounds__(block_size) void singledot_kernel_num_iters_2(
+    size_type num_iters, size_type num_rows,
+    const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov,
+    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status)
+{
+    /*
+     * In general in this kernel (thread indices appear to be col-major):
+     * grid_dim
+     *   x: for row_idx
+     *   y: for num_iters (number of krylov vectors)
+     * block_dim
+     *   x: for row_idx (must be block_size)
+     *   (y not used, must be set to 1 in dim)
+     */
+    const size_type tidx = threadIdx.x;
+    constexpr size_type col_idx{0};
+    const size_type num_rows_per_thread = ceildiv(num_rows, gridDim.x);
+    const size_type start_row = blockIdx.x * num_rows_per_thread + threadIdx.x;
+    const auto end_row = min((blockIdx.x + 1) * num_rows_per_thread, num_rows);
+    const size_type k = blockIdx.y;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+    __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
+
+    ValueType local_res = zero<ValueType>();
+    if (!stop_status[col_idx].has_stopped()) {
+        for (size_type i = start_row; i < end_row; i += block_size) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            local_res += next_krylov_basis[next_krylov_idx] *
+                         krylov_bases.read(k, i, col_idx);
+        }
+    }
+    // Transpose local_res, so each warp contains a local_res from the same
+    // right hand side
+    reduction_helper[tidx] = local_res;
+    auto thread_block = group::this_thread_block();
+    thread_block.sync();
+    reduce(thread_block, reduction_helper,
+           [](const ValueType &a, const ValueType &b) { return a + b; });
+    if (tidx == 0 && !stop_status[col_idx].has_stopped()) {
+        const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+        atomic_add(hessenberg_iter + hessenberg_idx, reduction_helper[0]);
+    }
+}
+
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index accaaf1f135..34803d2c9b6 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -628,7 +628,13 @@ void finish_arnoldi_CGS2(
     const dim3 grid_size_num_iters(
         ceildiv(dim_size[1] * (iter + 1), default_dot_dim),
         exec->get_num_multiprocessor() * 2);
+    const dim3 grid_size_num_iters_2(ceildiv(dim_size[1], default_dot_dim),
+                                     exec->get_num_multiprocessor() * 2,
+                                     iter + 1);
     const dim3 block_size(default_dot_dim, default_dot_dim);
+    const dim3 grid_size_iters_single(exec->get_num_multiprocessor() * 2,
+                                      iter + 1);
+    const dim3 block_size_iters_single(default_block_size);
     size_type numReorth;
 
     Accessor3d<ValueType, ValueType> next_krylov_accessor{
@@ -666,19 +672,30 @@ void finish_arnoldi_CGS2(
         as_cuda_accessor(krylov_bases.to_const()),
         as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
         as_cuda_type(stop_status));
-    */
-    /* */
+    /*/
     //    write(std::cout, krylov_bases);
     //    write(std::cout, next_krylov_basis);
-    multidot_kernel_num_iters_1<<<grid_size_num_iters, block_size>>>(
-        iter + 1, dim_size[0], dim_size[1],
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_accessor(krylov_bases.to_const()),
-        as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-        as_cuda_type(stop_status));
-    // exec->synchronize();
-    // write(std::cout, hessenberg_iter);
-    /* */
+    if (dim_size[1] > 1) {
+        multidot_kernel_num_iters_2<default_dot_dim>
+            <<<grid_size_num_iters_2, block_size>>>(
+                iter + 1, dim_size[0], dim_size[1],
+                as_cuda_type(next_krylov_basis->get_const_values()),
+                stride_next_krylov, as_cuda_accessor(krylov_bases.to_const()),
+                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+                as_cuda_type(stop_status));
+    } else {
+        singledot_kernel_num_iters_2<default_block_size>
+            <<<grid_size_iters_single, block_size_iters_single>>>(
+                iter + 1, dim_size[0],
+                as_cuda_type(next_krylov_basis->get_const_values()),
+                stride_next_krylov, as_cuda_accessor(krylov_bases.to_const()),
+                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+                as_cuda_type(stop_status));
+    }
+        // */
+        // exec->synchronize();
+        // write(std::cout, hessenberg_iter);
+        /* */
 #ifdef TIMING
     exec->synchronize();
     auto time_1 = std::chrono::steady_clock::now() - start_1;
@@ -693,7 +710,6 @@ void finish_arnoldi_CGS2(
     exec->synchronize();
     auto start_2 = std::chrono::steady_clock::now();
 #endif
-    /*
     update_next_krylov_kernel_num_iters<default_block_size>
         <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
            default_block_size>>>(
@@ -702,18 +718,7 @@ void finish_arnoldi_CGS2(
             as_cuda_accessor(krylov_bases.to_const()),
             as_cuda_type(hessenberg_iter->get_const_values()),
             stride_hessenberg, as_cuda_type(stop_status));
-    */
-    /* */
-    // update_next_krylov_kernel_num_iters_1<default_block_size>
-    //     <<<dim_size[0], default_block_size>>>(
-    update_next_krylov_kernel_num_iters_1<default_update_dim>
-        <<<dim_size[0] * stride_next_krylov, default_update_dim>>>(
-            iter + 1, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
-            as_cuda_accessor(krylov_bases.to_const()),
-            as_cuda_type(hessenberg_iter->get_const_values()),
-            stride_hessenberg, as_cuda_type(stop_status));
-    /* */
+
 #ifdef TIMING
     exec->synchronize();
     auto time_2 = std::chrono::steady_clock::now() - start_2;
@@ -840,16 +845,29 @@ void finish_arnoldi_CGS2(
             as_cuda_type(buffer_iter->get_values()), stride_buffer,
             as_cuda_type(stop_status));
         */
-        multidot_kernel_num_iters_1<<<grid_size_num_iters, block_size>>>(
-            iter + 1, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_accessor(krylov_bases.to_const()),
-            as_cuda_type(buffer_iter->get_values()), stride_buffer,
-            as_cuda_type(stop_status));
+        if (dim_size[1] > 1) {
+            multidot_kernel_num_iters_2<default_dot_dim>
+                <<<grid_size_num_iters_2, block_size>>>(
+                    iter + 1, dim_size[0], dim_size[1],
+                    as_cuda_type(next_krylov_basis->get_const_values()),
+                    stride_next_krylov,
+                    as_cuda_accessor(krylov_bases.to_const()),
+                    as_cuda_type(buffer_iter->get_values()), stride_buffer,
+                    as_cuda_type(stop_status));
+        } else {
+            singledot_kernel_num_iters_2<default_block_size>
+                <<<grid_size_iters_single, block_size_iters_single>>>(
+                    iter + 1, dim_size[0],
+                    as_cuda_type(next_krylov_basis->get_const_values()),
+                    stride_next_krylov,
+                    as_cuda_accessor(krylov_bases.to_const()),
+                    as_cuda_type(buffer_iter->get_values()), stride_buffer,
+                    as_cuda_type(stop_status));
+        }
         // for i in 1:iter
         //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
         // end
-        /*
+        //*
         update_next_krylov_kernel_num_iters_and_add<default_block_size>
             <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
                default_block_size>>>(
@@ -859,7 +877,7 @@ void finish_arnoldi_CGS2(
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
                 as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
                 as_cuda_type(stop_status), as_cuda_type(reorth_status));
-        */ /* */
+        /*/
         update_next_krylov_kernel_num_iters_and_add_1<default_update_dim>
             <<<dim_size[0] * stride_next_krylov, default_update_dim>>>(
                 iter + 1, dim_size[0], dim_size[1],
@@ -868,7 +886,7 @@ void finish_arnoldi_CGS2(
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
                 as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
                 as_cuda_type(stop_status), as_cuda_type(reorth_status));
-        /* */
+        // */
         // for i in 1:iter
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end

From 775763f88342523bf9bd9f6ed0494878bed34dba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 4 Jun 2020 17:38:18 +0200
Subject: [PATCH 0358/1008] Transpose grid when launching singledot kernel

Transposing is supposed to avoid write-conflicts during the atomic_add
at the end. Obviously, the kernel was changed accordingly, so the
behavior has not changed.
Also contains:
- Scaling is now only added for integer types (not for float anymore)
- Add benchmarks for GmresMixed integer
- Some cleanup for debug output in CUDA
---
 benchmark/solver/solver.cpp               | 133 ++++++++++++----------
 common/solver/gmres_mixed_kernels.hpp.inc |  14 ++-
 core/solver/gmres_mixed_accessor.hpp      |   8 +-
 cuda/solver/gmres_mixed_kernels.cu        |  58 +++++-----
 4 files changed, 119 insertions(+), 94 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index db60f269ac2..7692930eeb0 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -170,63 +170,82 @@ std::unique_ptr<gko::LinOpFactory> create_solver(
 const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                                 std::shared_ptr<const gko::Executor>,
                                 std::shared_ptr<const gko::LinOpFactory>)>>
-    solver_factory{{"bicgstab", create_solver<gko::solver::Bicgstab<etype>>},
-                   {"bicg", create_solver<gko::solver::Bicg<etype>>},
-                   {"cg", create_solver<gko::solver::Cg<etype>>},
-                   {"cgs", create_solver<gko::solver::Cgs<etype>>},
-                   {"fcg", create_solver<gko::solver::Fcg<etype>>},
-                   {"idr",
-                    [](std::shared_ptr<const gko::Executor> exec,
-                       std::shared_ptr<const gko::LinOpFactory> precond) {
-                        return gko::solver::Idr<etype>::build()
-                            .with_criteria(create_criterion(exec))
-                            .with_subspace_dim(FLAGS_idr_subspace_dim)
-                            .with_kappa(static_cast<rc_etype>(FLAGS_idr_kappa))
-                            .with_preconditioner(give(precond))
-                            .on(exec);
-                    }},
-                   {"gmres",
-                    [](std::shared_ptr<const gko::Executor> exec,
-                       std::shared_ptr<const gko::LinOpFactory> precond) {
-                        return gko::solver::Gmres<etype>::build()
-                            .with_criteria(create_criterion(exec))
-                            .with_krylov_dim(FLAGS_gmres_restart)
-                            .with_preconditioner(give(precond))
-                            .on(exec);
-                    }},
-                   {"gmres_mixed_double",
-                    [](std::shared_ptr<const gko::Executor> exec,
-                       std::shared_ptr<const gko::LinOpFactory> precond) {
-                        return gko::solver::GmresMixed<double, double>::build()
-                            .with_criteria(create_criterion(exec))
-                            .with_krylov_dim_mixed(FLAGS_gmres_restart)
-                            .with_preconditioner(give(precond))
-                            .on(exec);
-                    }},
-                   {"gmres_mixed_float",
-                    [](std::shared_ptr<const gko::Executor> exec,
-                       std::shared_ptr<const gko::LinOpFactory> precond) {
-                        return gko::solver::GmresMixed<double, float>::build()
-                            .with_criteria(create_criterion(exec))
-                            .with_krylov_dim_mixed(FLAGS_gmres_restart)
-                            .with_preconditioner(give(precond))
-                            .on(exec);
-                    }},
-                   {"lower_trs",
-                    [](std::shared_ptr<const gko::Executor> exec,
-                       std::shared_ptr<const gko::LinOpFactory>) {
-                        return gko::solver::LowerTrs<etype>::build()
-                            .with_num_rhs(FLAGS_nrhs)
-                            .on(exec);
-                    }},
-                   {"upper_trs",
-                    [](std::shared_ptr<const gko::Executor> exec,
-                       std::shared_ptr<const gko::LinOpFactory>) {
-                        return gko::solver::UpperTrs<etype>::build()
-                            .with_num_rhs(FLAGS_nrhs)
-                            .on(exec);
-                    }},
-                   {"overhead", create_solver<gko::Overhead<etype>>}};
+    solver_factory{
+        {"bicgstab", create_solver<gko::solver::Bicgstab<etype>>},
+        {"bicg", create_solver<gko::solver::Bicg<etype>>},
+        {"cg", create_solver<gko::solver::Cg<etype>>},
+        {"cgs", create_solver<gko::solver::Cgs<etype>>},
+        {"fcg", create_solver<gko::solver::Fcg<etype>>},
+        {"idr",
+         [](std::shared_ptr<const gko::Executor> exec,
+            std::shared_ptr<const gko::LinOpFactory> precond) {
+             return gko::solver::Idr<etype>::build()
+                 .with_criteria(create_criterion(exec))
+                 .with_subspace_dim(FLAGS_idr_subspace_dim)
+                 .with_kappa(static_cast<rc_etype>(FLAGS_idr_kappa))
+                 .with_preconditioner(give(precond))
+                 .on(exec);
+         }},
+        {"gmres",
+         [](std::shared_ptr<const gko::Executor> exec,
+            std::shared_ptr<const gko::LinOpFactory> precond) {
+             return gko::solver::Gmres<etype>::build()
+                 .with_criteria(create_criterion(exec))
+                 .with_krylov_dim(FLAGS_gmres_restart)
+                 .with_preconditioner(give(precond))
+                 .on(exec);
+         }},
+        {"gmres_mixed_double",
+         [](std::shared_ptr<const gko::Executor> exec,
+            std::shared_ptr<const gko::LinOpFactory> precond) {
+             return gko::solver::GmresMixed<double, double>::build()
+                 .with_criteria(create_criterion(exec))
+                 .with_krylov_dim_mixed(FLAGS_gmres_restart)
+                 .with_preconditioner(give(precond))
+                 .on(exec);
+         }},
+        {"gmres_mixed_float",
+         [](std::shared_ptr<const gko::Executor> exec,
+            std::shared_ptr<const gko::LinOpFactory> precond) {
+             return gko::solver::GmresMixed<double, float>::build()
+                 .with_criteria(create_criterion(exec))
+                 .with_krylov_dim_mixed(FLAGS_gmres_restart)
+                 .with_preconditioner(give(precond))
+                 .on(exec);
+         }},
+        {"gmres_mixed_int32",
+         [](std::shared_ptr<const gko::Executor> exec,
+            std::shared_ptr<const gko::LinOpFactory> precond) {
+             return gko::solver::GmresMixed<double, gko::int32>::build()
+                 .with_criteria(create_criterion(exec))
+                 .with_krylov_dim_mixed(FLAGS_gmres_restart)
+                 .with_preconditioner(give(precond))
+                 .on(exec);
+         }},
+        {"gmres_mixed_int16",
+         [](std::shared_ptr<const gko::Executor> exec,
+            std::shared_ptr<const gko::LinOpFactory> precond) {
+             return gko::solver::GmresMixed<double, gko::int16>::build()
+                 .with_criteria(create_criterion(exec))
+                 .with_krylov_dim_mixed(FLAGS_gmres_restart)
+                 .with_preconditioner(give(precond))
+                 .on(exec);
+         }},
+        {"lower_trs",
+         [](std::shared_ptr<const gko::Executor> exec,
+            std::shared_ptr<const gko::LinOpFactory>) {
+             return gko::solver::LowerTrs<etype>::build()
+                 .with_num_rhs(FLAGS_nrhs)
+                 .on(exec);
+         }},
+        {"upper_trs",
+         [](std::shared_ptr<const gko::Executor> exec,
+            std::shared_ptr<const gko::LinOpFactory>) {
+             return gko::solver::UpperTrs<etype>::build()
+                 .with_num_rhs(FLAGS_nrhs)
+                 .on(exec);
+         }},
+        {"overhead", create_solver<gko::Overhead<etype>>}};
 
 
 void write_precond_info(const gko::LinOp *precond,
diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 28878cd20bc..a7c542d084e 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -620,18 +620,20 @@ __global__ __launch_bounds__(block_size) void singledot_kernel_num_iters_2(
     /*
      * In general in this kernel (thread indices appear to be col-major):
      * grid_dim
-     *   x: for row_idx
-     *   y: for num_iters (number of krylov vectors)
+     *   x: for num_iters (number of krylov vectors)
+     *   y: for row_idx
      * block_dim
      *   x: for row_idx (must be block_size)
      *   (y not used, must be set to 1 in dim)
+     * Note: the grid_dim is transposed to the storage layout intentioally
+     *       so less conflicts with `atomic_add` occur at the same time.
      */
     const size_type tidx = threadIdx.x;
     constexpr size_type col_idx{0};
-    const size_type num_rows_per_thread = ceildiv(num_rows, gridDim.x);
-    const size_type start_row = blockIdx.x * num_rows_per_thread + threadIdx.x;
-    const auto end_row = min((blockIdx.x + 1) * num_rows_per_thread, num_rows);
-    const size_type k = blockIdx.y;
+    const size_type k = blockIdx.x;
+    const size_type num_rows_per_thread = ceildiv(num_rows, gridDim.y);
+    const size_type start_row = blockIdx.y * num_rows_per_thread + threadIdx.x;
+    const auto end_row = min((blockIdx.y + 1) * num_rows_per_thread, num_rows);
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
     __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index 153df5b982e..af3c9d46c6f 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -57,15 +57,15 @@ namespace kernels {  // TODO maybe put into another separate namespace
 namespace detail {
 
 
-using place_holder_type = float;
+// using place_holder_type = float;
 
 
 template <typename StorageType, typename ArithmeticType,
-          bool = (std::is_same<StorageType, place_holder_type>::value &&
+          bool = std::is_integral<StorageType>::value>
+/*          bool = (std::is_same<StorageType, place_holder_type>::value &&
                   !std::is_same<StorageType, ArithmeticType>::value) ||
                  std::is_integral<StorageType>::value>
-// bool = std::is_same<StorageType, place_holder_type>::value &&
-//        !std::is_same<StorageType, ArithmeticType>::value>
+*/
 struct helper_have_scale {};
 
 template <typename StorageType, typename ArithmeticType>
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 34803d2c9b6..179c7d2faf6 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -55,13 +55,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "cuda/components/uninitialized_array.hpp"
 
 
-#include <iostream>
-
-
 // #define TIMING 1
 
 
 #ifdef TIMING
+#include <chrono>
+#include <iostream>
+
+
 using double_seconds = std::chrono::duration<double, std::milli>;
 #endif
 
@@ -186,7 +187,7 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
     constexpr auto block_size = default_block_size;
 
     // exec->synchronize();
-    // std::cout << "Before initialize_2_1_kernel" << std::endl;
+    // std::cout << "Before initialize_2_1_kernel" << '\n';
     initialize_2_1_kernel<block_size><<<grid_dim_1, block_dim>>>(
         residual->get_size()[0], residual->get_size()[1], krylov_dim,
         as_cuda_accessor(krylov_bases),
@@ -196,7 +197,7 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
     residual->compute_norm2(residual_norm);
     // exec->synchronize();
 
-    // std::cout << "Before multinorminf_kernel_without_stop" << std::endl;
+    // std::cout << "Before multinorminf_kernel_without_stop" << '\n';
     components::fill_array(exec, arnoldi_norm->get_values() + 2 * num_rhs,
                            num_rhs, zero<remove_complex<ValueType>>());
     const dim3 grid_size_nrm(ceildiv(num_rhs, default_dot_dim),
@@ -211,7 +212,7 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
     //     krylov_bases, col_idx, arnoldi_norm->at(2, j) / residual_norm->at(0,
     //     j));
     /* */
-    // std::cout << "Before set_scale_kernel" << std::endl;
+    // std::cout << "Before set_scale_kernel" << '\n';
     set_scale_kernel<default_block_size>
         <<<ceildiv(num_rhs * (krylov_dim + 1), default_block_size),
            default_block_size>>>(
@@ -221,7 +222,7 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
             num_rhs, as_cuda_accessor(krylov_bases));
     // exec->synchronize();
     /* */
-    // std::cout << "Before initialize_2_2_kernel" << std::endl;
+    // std::cout << "Before initialize_2_2_kernel" << '\n';
     const dim3 grid_dim_2(
         ceildiv(num_rows * krylov_bases.get_stride1(), default_block_size), 1,
         1);
@@ -235,7 +236,7 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
         next_krylov_basis->get_stride(),
         as_cuda_type(final_iter_nums->get_data()));
     // exec->synchronize();
-    // std::cout << "After initialize_2_2_kernel" << std::endl;
+    // std::cout << "After initialize_2_2_kernel" << '\n';
 }
 
 GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
@@ -632,8 +633,11 @@ void finish_arnoldi_CGS2(
                                      exec->get_num_multiprocessor() * 2,
                                      iter + 1);
     const dim3 block_size(default_dot_dim, default_dot_dim);
-    const dim3 grid_size_iters_single(exec->get_num_multiprocessor() * 2,
-                                      iter + 1);
+    // Note: having iter first (instead of row_idx information) is likely
+    //       beneficial for avoiding atomic_add conflicts. Maybe, we need to
+    //       modify `grid_size_num_iters_2` as well
+    const dim3 grid_size_iters_single(iter + 1,
+                                      exec->get_num_multiprocessor() * 2);
     const dim3 block_size_iters_single(default_block_size);
     size_type numReorth;
 
@@ -692,16 +696,16 @@ void finish_arnoldi_CGS2(
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
                 as_cuda_type(stop_status));
     }
-        // */
-        // exec->synchronize();
-        // write(std::cout, hessenberg_iter);
-        /* */
+    // */
+    // exec->synchronize();
+    // write(std::cout, hessenberg_iter);
+    /* */
 #ifdef TIMING
     exec->synchronize();
     auto time_1 = std::chrono::steady_clock::now() - start_1;
     std::cout << "time_1(" << iter << ") = "
               << std::chrono::duration_cast<double_seconds>(time_1).count()
-              << std::endl;
+              << '\n';
 #endif
     // for i in 1:iter
     //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
@@ -724,11 +728,11 @@ void finish_arnoldi_CGS2(
     auto time_2 = std::chrono::steady_clock::now() - start_2;
     std::cout << "time_2(" << iter << ") = "
               << std::chrono::duration_cast<double_seconds>(time_2).count()
-              << std::endl;
+              << '\n';
     std::cout << "time_1 / time_2(" << iter << ") = "
               << std::chrono::duration_cast<double_seconds>(time_1).count() /
                      std::chrono::duration_cast<double_seconds>(time_2).count()
-              << std::endl;
+              << '\n';
 #endif
     // for i in 1:iter
     //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
@@ -778,7 +782,7 @@ void finish_arnoldi_CGS2(
             exec.get(), 1, arnoldi_norm->get_values() + dim_size[1], &norm);
         exec->get_master()->copy_from(
             exec.get(), 1, arnoldi_norm->get_values() + 2 * dim_size[1], &inf);
-        std::cout << sqrt(norm) << " - " << inf << std::endl;
+        std::cout << sqrt(norm) << " - " << inf << '\n';
     }
     */
 #ifdef TIMING
@@ -786,7 +790,7 @@ void finish_arnoldi_CGS2(
     auto time_3 = std::chrono::steady_clock::now() - start_3;
     std::cout << "time_3(" << iter << ") = "
               << std::chrono::duration_cast<double_seconds>(time_3).count()
-              << std::endl;
+              << '\n';
 #endif
     // nrmN = norm(next_krylov_basis)
 #ifdef TIMING
@@ -814,7 +818,7 @@ void finish_arnoldi_CGS2(
     auto time_4 = std::chrono::steady_clock::now() - start_4;
     std::cout << "time_4(" << iter << ") = "
               << std::chrono::duration_cast<double_seconds>(time_4).count()
-              << std::endl;
+              << '\n';
 #endif
 #ifdef TIMING
     exec->synchronize();
@@ -828,11 +832,11 @@ void finish_arnoldi_CGS2(
     auto time_5 = std::chrono::steady_clock::now() - start_5;
     std::cout << "time_5(" << iter << ") = "
               << std::chrono::duration_cast<double_seconds>(time_5).count()
-              << std::endl;
+              << '\n';
 #endif
     // numReorth <= number of next_krylov vector to be reorthogonalization
     for (size_type l = 1; (numReorth > 0) && (l < 3); l++) {
-        // std::cout << "CUDA RESTART " << l << std::endl;
+        // std::cout << "CUDA RESTART " << l << '\n';
         (*num_reorth_steps)++;
         (*num_reorth_vectors) += iter;
         zero_matrix(iter + 1, dim_size[1], stride_buffer,
@@ -985,7 +989,7 @@ void finish_arnoldi_CGS2(
     auto time_6 = std::chrono::steady_clock::now() - start_6;
     std::cout << "time_6(" << iter << ") = "
               << std::chrono::duration_cast<double_seconds>(time_6).count()
-              << std::endl;
+              << '\n';
 #endif
     // next_krylov_basis /= hessenberg(iter, iter + 1)
     // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -1047,26 +1051,26 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
                               as_cuda_type(stop_status->get_const_data()),
                               final_iter_nums->get_num_elems());
 #if FINISH_ARNOLDI == 1
-    //    std::cout << "CUDA MGS_REORTH" << std::endl;
+    //    std::cout << "CUDA MGS_REORTH" << '\n';
     finish_arnoldi_reorth(exec, next_krylov_basis, krylov_bases,
                           hessenberg_iter, buffer_iter, arnoldi_norm, iter,
                           stop_status->get_const_data(),
                           reorth_status->get_data(), num_reorth);
 #elif FINISH_ARNOLDI == 2
-    //    std::cout << "CUDA CGS_REORTH" << std::endl;
+    //    std::cout << "CUDA CGS_REORTH" << '\n';
     finish_arnoldi_CGS(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
                        buffer_iter, arnoldi_norm, iter,
                        stop_status->get_const_data(), reorth_status->get_data(),
                        num_reorth);
 #elif FINISH_ARNOLDI == 3
-    //    std::cout << "CUDA CGS_REORTH_2" << std::endl;
+    //    std::cout << "CUDA CGS_REORTH_2" << '\n';
     finish_arnoldi_CGS2(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
                         buffer_iter, arnoldi_norm, iter,
                         stop_status->get_const_data(),
                         reorth_status->get_data(), num_reorth, num_reorth_steps,
                         num_reorth_vectors);
 #else
-    //    std::cout << "CUDA MGS" << std::endl;
+    //    std::cout << "CUDA MGS" << '\n';
     finish_arnoldi(exec, next_krylov_basis, krylov_bases, hessenberg_iter, iter,
                    stop_status->get_const_data());
 #endif

From 4f40075b5d6e61796b269e92ba4678ba7dbc1375 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 4 Jun 2020 19:17:48 +0200
Subject: [PATCH 0359/1008] Reversed the transpose of the grid dim

The transposing needs further investigation, but a single test on a
V100 lead to worse results than before, so I am undoing this change
for now.
---
 common/solver/gmres_mixed_kernels.hpp.inc | 14 ++++++--------
 cuda/solver/gmres_mixed_kernels.cu        |  8 ++++----
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index a7c542d084e..3fc07fc7eb7 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -620,20 +620,18 @@ __global__ __launch_bounds__(block_size) void singledot_kernel_num_iters_2(
     /*
      * In general in this kernel (thread indices appear to be col-major):
      * grid_dim
-     *   x: for num_iters (number of krylov vectors)
-     *   y: for row_idx
+     *   x: for row_idx
+     *   y: for num_iters (number of krylov vectors)
      * block_dim
      *   x: for row_idx (must be block_size)
      *   (y not used, must be set to 1 in dim)
-     * Note: the grid_dim is transposed to the storage layout intentioally
-     *       so less conflicts with `atomic_add` occur at the same time.
      */
     const size_type tidx = threadIdx.x;
     constexpr size_type col_idx{0};
-    const size_type k = blockIdx.x;
-    const size_type num_rows_per_thread = ceildiv(num_rows, gridDim.y);
-    const size_type start_row = blockIdx.y * num_rows_per_thread + threadIdx.x;
-    const auto end_row = min((blockIdx.y + 1) * num_rows_per_thread, num_rows);
+    const size_type k = blockIdx.y;
+    const size_type num_rows_per_thread = ceildiv(num_rows, gridDim.x);
+    const size_type start_row = blockIdx.x * num_rows_per_thread + threadIdx.x;
+    const auto end_row = min((blockIdx.x + 1) * num_rows_per_thread, num_rows);
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
     __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 179c7d2faf6..895100ec8ab 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -634,10 +634,10 @@ void finish_arnoldi_CGS2(
                                      iter + 1);
     const dim3 block_size(default_dot_dim, default_dot_dim);
     // Note: having iter first (instead of row_idx information) is likely
-    //       beneficial for avoiding atomic_add conflicts. Maybe, we need to
-    //       modify `grid_size_num_iters_2` as well
-    const dim3 grid_size_iters_single(iter + 1,
-                                      exec->get_num_multiprocessor() * 2);
+    //       beneficial for avoiding atomic_add conflicts, but that needs
+    //       further investigation.
+    const dim3 grid_size_iters_single(exec->get_num_multiprocessor() * 2,
+                                      iter + 1);
     const dim3 block_size_iters_single(default_block_size);
     size_type numReorth;
 

From 347d25902b164fbbd82d16bb01ced61f5f565abe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 15 Jun 2020 18:46:41 +0200
Subject: [PATCH 0360/1008] Add half precision support to GmresMixed

---
 benchmark/solver/solver.cpp                   | 18 ++++++++++++++++++
 core/solver/gmres_mixed.cpp                   |  1 +
 core/solver/gmres_mixed_kernels.hpp           |  3 +++
 reference/test/solver/gmres_mixed_kernels.cpp | 17 ++++++++++-------
 4 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 7692930eeb0..fa029be7d2d 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -213,6 +213,24 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
+        {"gmres_mixed_half",
+         [](std::shared_ptr<const gko::Executor> exec,
+            std::shared_ptr<const gko::LinOpFactory> precond) {
+             return gko::solver::GmresMixed<double, gko::half>::build()
+                 .with_criteria(create_criterion(exec))
+                 .with_krylov_dim_mixed(FLAGS_gmres_restart)
+                 .with_preconditioner(give(precond))
+                 .on(exec);
+         }},
+        {"gmres_mixed_int64",
+         [](std::shared_ptr<const gko::Executor> exec,
+            std::shared_ptr<const gko::LinOpFactory> precond) {
+             return gko::solver::GmresMixed<double, gko::int64>::build()
+                 .with_criteria(create_criterion(exec))
+                 .with_krylov_dim_mixed(FLAGS_gmres_restart)
+                 .with_preconditioner(give(precond))
+                 .on(exec);
+         }},
         {"gmres_mixed_int32",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 622d9f5d25e..9c4beb6117c 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -447,5 +447,6 @@ GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(GKO_DECLARE_GMRES_MIXED_BOOL);
 #define GKO_DECLARE_GMRES_MIXED(_type1, _type2) class GmresMixed<_type1, _type2>
 GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED);
 
+
 }  // namespace solver
 }  // namespace gko
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index 4ef2a12c80c..cfc83636a8f 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/stopping_status.hpp>
 
 
+#include "core/base/extended_float.hpp"
 #include "core/solver/gmres_mixed_accessor.hpp"
 
 
@@ -70,7 +71,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     template _macro(double, int64);                              \
     template _macro(double, int32);                              \
     template _macro(double, int16);                              \
+    template _macro(double, half);                               \
     template _macro(float, float);                               \
+    template _macro(float, half);                                \
     template _macro(std::complex<double>, std::complex<double>); \
     template _macro(std::complex<double>, std::complex<float>);  \
     template _macro(std::complex<float>, std::complex<float>)
diff --git a/reference/test/solver/gmres_mixed_kernels.cpp b/reference/test/solver/gmres_mixed_kernels.cpp
index 62221f10e59..345a85778c4 100644
--- a/reference/test/solver/gmres_mixed_kernels.cpp
+++ b/reference/test/solver/gmres_mixed_kernels.cpp
@@ -69,7 +69,8 @@ class GmresMixed : public ::testing::Test {
           gmres_mixed_factory(
               gmres_type::build()
                   .with_criteria(
-                      gko::stop::Iteration::build().with_max_iters(4u).on(exec),
+                      gko::stop::Iteration::build().with_max_iters(100u).on(
+                          exec),
                       gko::stop::Time::build()
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
@@ -112,12 +113,14 @@ class GmresMixed : public ::testing::Test {
 };
 
 
-using TestTypes =
-    ::testing::Types<std::tuple<double, double>, std::tuple<double, float>,
-                     std::tuple<float, float>,
-                     std::tuple<std::complex<double>, std::complex<double>>,
-                     std::tuple<std::complex<double>, std::complex<float>>,
-                     std::tuple<std::complex<float>, std::complex<float>>>;
+using TestTypes = ::testing::Types<
+    std::tuple<double, double>, std::tuple<double, float>,
+    std::tuple<double, gko::int64>, std::tuple<double, gko::int32>,
+    std::tuple<double, gko::int16>, std::tuple<double, gko::half>,
+    std::tuple<float, float>, std::tuple<float, gko::half>,
+    std::tuple<std::complex<double>, std::complex<double>>,
+    std::tuple<std::complex<double>, std::complex<float>>,
+    std::tuple<std::complex<float>, std::complex<float>>>;
 
 
 TYPED_TEST_CASE(GmresMixed, TestTypes);

From 03c062458757e556176fba3aea3b7b52e890d686 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 15 Jun 2020 18:51:35 +0200
Subject: [PATCH 0361/1008] Hopefully improve singledot performance

---
 cuda/solver/gmres_mixed_kernels.cu | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 895100ec8ab..b72e3a17c5a 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -618,6 +618,8 @@ void finish_arnoldi_CGS2(
     int *num_reorth_vectors)
 {
     using non_complex = remove_complex<ValueType>;
+    // optimization parameter
+    constexpr int singledot_block_size = default_dot_dim;
     const auto stride_next_krylov = next_krylov_basis->get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
     const auto stride_buffer = buffer_iter->get_stride();
@@ -638,7 +640,7 @@ void finish_arnoldi_CGS2(
     //       further investigation.
     const dim3 grid_size_iters_single(exec->get_num_multiprocessor() * 2,
                                       iter + 1);
-    const dim3 block_size_iters_single(default_block_size);
+    const dim3 block_size_iters_single(singledot_block_size);
     size_type numReorth;
 
     Accessor3d<ValueType, ValueType> next_krylov_accessor{
@@ -688,7 +690,7 @@ void finish_arnoldi_CGS2(
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
                 as_cuda_type(stop_status));
     } else {
-        singledot_kernel_num_iters_2<default_block_size>
+        singledot_kernel_num_iters_2<singledot_block_size>
             <<<grid_size_iters_single, block_size_iters_single>>>(
                 iter + 1, dim_size[0],
                 as_cuda_type(next_krylov_basis->get_const_values()),
@@ -859,7 +861,7 @@ void finish_arnoldi_CGS2(
                     as_cuda_type(buffer_iter->get_values()), stride_buffer,
                     as_cuda_type(stop_status));
         } else {
-            singledot_kernel_num_iters_2<default_block_size>
+            singledot_kernel_num_iters_2<singledot_block_size>
                 <<<grid_size_iters_single, block_size_iters_single>>>(
                     iter + 1, dim_size[0],
                     as_cuda_type(next_krylov_basis->get_const_values()),

From fd342c06ce61ed1ef18cb22ff06ed74886aef9bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 17 Jun 2020 16:22:27 +0200
Subject: [PATCH 0362/1008] Infinity norm only computed when scale is present

Additionally, removed the debug output and removed unused kernel
parameter.
---
 common/solver/gmres_mixed_kernels.hpp.inc | 75 ++++++++++++-----------
 core/solver/gmres_mixed.cpp               | 10 ++-
 core/solver/gmres_mixed_accessor.hpp      |  4 +-
 cuda/solver/gmres_mixed_kernels.cu        | 35 ++++++-----
 4 files changed, 68 insertions(+), 56 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 3fc07fc7eb7..d6c895bc47a 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -158,7 +158,7 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_kernel(
     size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
     size_type stride_next_krylov, remove_complex<ValueType> *__restrict__ norms,
-    size_type stride_norms, const stopping_status *__restrict__ stop_status)
+    const stopping_status *__restrict__ stop_status)
 {
     const auto tidx = threadIdx.x;
     const auto tidy = threadIdx.y;
@@ -304,15 +304,16 @@ __global__
 }
 
 
-template <typename ValueType>
+template <bool compute_inf, typename ValueType>
 __global__ __launch_bounds__(default_dot_size) void multinorm2_inf_kernel(
     size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
     size_type stride_next_krylov,
     remove_complex<ValueType> *__restrict__ norms1,
-    remove_complex<ValueType> *__restrict__ norms2, size_type stride_norms,
+    remove_complex<ValueType> *__restrict__ norms2,
     const stopping_status *__restrict__ stop_status)
 {
+    using non_complex = remove_complex<ValueType>;
     const auto tidx = threadIdx.x;
     const auto tidy = threadIdx.y;
     const auto col_idx = blockIdx.x * default_dot_dim + tidx;
@@ -322,53 +323,54 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_inf_kernel(
         ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
-    __shared__ UninitializedArray<remove_complex<ValueType>,
-                                  default_dot_dim *(default_dot_dim + 1)>
-        reduction_helper_array_1, reduction_helper_array_2;
-    remove_complex<ValueType> *__restrict__ reduction_helper_1 =
-        reduction_helper_array_1;
-    remove_complex<ValueType> *__restrict__ reduction_helper_2 =
-        reduction_helper_array_2;
-    remove_complex<ValueType> local_res = zero<remove_complex<ValueType>>();
-    remove_complex<ValueType> local_max = zero<remove_complex<ValueType>>();
+    __shared__ UninitializedArray<
+        non_complex, (1 + compute_inf) * default_dot_dim *(default_dot_dim + 1)>
+        reduction_helper_array;
+    non_complex *__restrict__ reduction_helper_add = reduction_helper_array;
+    non_complex *__restrict__ reduction_helper_max =
+        static_cast<non_complex *>(reduction_helper_array) +
+        default_dot_dim * (default_dot_dim + 1);
+    non_complex local_res = zero<non_complex>();
+    non_complex local_max = zero<non_complex>();
     if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
         for (size_type i = start_row + tidy; i < end_row;
              i += default_dot_dim) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
             const auto num = next_krylov_basis[next_krylov_idx];
             local_res += squared_norm(num);
-            // local_res += squared_norm(next_krylov_basis[next_krylov_idx]);
-            local_max = ((local_max >= abs(num)) ? local_max : abs(num));
-            // local_max = (local_max >=
-            // abs(next_krylov_basis[next_krylov_idx]))
-            //                 ? local_max
-            //                 : abs(next_krylov_basis[next_krylov_idx]);
+            if (compute_inf) {
+                local_max = ((local_max >= abs(num)) ? local_max : abs(num));
+            }
         }
     }
     // Add reduction
-    reduction_helper_1[tidx * (default_dot_dim + 1) + tidy] = local_res;
-    reduction_helper_2[tidx * (default_dot_dim + 1) + tidy] = local_max;
+    reduction_helper_add[tidx * (default_dot_dim + 1) + tidy] = local_res;
+    if (compute_inf) {
+        reduction_helper_max[tidx * (default_dot_dim + 1) + tidy] = local_max;
+    }
     __syncthreads();
-    local_res = reduction_helper_1[tidy * (default_dot_dim + 1) + tidx];
-    const auto tile_block1 =
-        group::tiled_partition<default_dot_dim>(group::this_thread_block());
-    const auto sum =
-        reduce(tile_block1, local_res,
-               [](const remove_complex<ValueType> &a,
-                  const remove_complex<ValueType> &b) { return a + b; });
-    local_max = reduction_helper_2[tidy * (default_dot_dim + 1) + tidx];
-    const auto tile_block2 =
+    local_res = reduction_helper_add[tidy * (default_dot_dim + 1) + tidx];
+    const auto tile_block =
         group::tiled_partition<default_dot_dim>(group::this_thread_block());
-    const auto value = reduce(
-        tile_block2, local_max,
-        [](const remove_complex<ValueType> &a,
-           const remove_complex<ValueType> &b) { return ((a >= b) ? a : b); });
+    const auto sum = reduce(
+        tile_block, local_res,
+        [](const non_complex &a, const non_complex &b) { return a + b; });
+    non_complex reduced_max{};
+    if (compute_inf) {
+        local_max = reduction_helper_max[tidy * (default_dot_dim + 1) + tidx];
+        reduced_max = reduce(tile_block, local_max,
+                             [](const non_complex &a, const non_complex &b) {
+                                 return ((a >= b) ? a : b);
+                             });
+    }
     const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
     if (tidx == 0 && new_col_idx < num_cols &&
         !stop_status[new_col_idx].has_stopped()) {
         const auto norms_idx = new_col_idx;
         atomic_add(norms1 + norms_idx, sum);
-        atomic_max(norms2 + norms_idx, value);
+        if (compute_inf) {
+            atomic_max(norms2 + norms_idx, reduced_max);
+        }
     }
 }
 
@@ -1284,6 +1286,8 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
     stopping_status *__restrict__ reorth_status,
     size_type *__restrict__ num_reorth)
 {
+    constexpr bool has_scale =
+        Accessor3d<ValueTypeKrylovBases, ValueType>::has_scale;
     const remove_complex<ValueType> eta = 1.0 / 2.0;
     const auto col_idx = thread::get_thread_id_flat();
 
@@ -1293,7 +1297,8 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
         const auto num11 = (arnoldi_norm[col_idx + stride_norm]);
         const auto num12 = sqrt(num11);
         // const auto num1 = abs(arnoldi_norm1[col_idx]);
-        const auto num2 = (arnoldi_norm[col_idx + 2 * stride_norm]);
+        const auto num2 = has_scale ? (arnoldi_norm[col_idx + 2 * stride_norm])
+                                    : remove_complex<ValueType>{};
         // printf("num0 = %f , num11 = %f , num12 = %f , num2 = %f\n", num0,
         // num11,
         //        num12, num2);
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 9c4beb6117c..4dee1655817 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -51,7 +51,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/gmres_mixed_kernels.hpp"
 
 
-#define TIMING 1
+//#define TIMING 1
 
 
 #ifdef TIMING
@@ -130,8 +130,12 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     auto residual_norm =
         VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
     auto b_norm = VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
-    auto arnoldi_norm =
-        VectorNorms::create(exec, dim<2>{3, dense_b->get_size()[1]});
+    // TODO: write description what the different rows represent
+    // The optional entry stores the infinity_norm of each next_krylov_vector,
+    // which is only used to compute the scale
+    auto arnoldi_norm = VectorNorms::create(
+        exec, dim<2>{2 + Accessor3dHelper::Accessor::has_scale,
+                     dense_b->get_size()[1]});
     Array<size_type> final_iter_nums(this->get_executor(),
                                      dense_b->get_size()[1]);
     auto y =
diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index af3c9d46c6f..2641f177876 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -402,11 +402,11 @@ class Accessor3dHelper {};
 
 template <typename ValueType, typename ValueTypeKrylovBases>
 class Accessor3dHelper<ValueType, ValueTypeKrylovBases, true> {
+public:
     using Accessor = Accessor3d<ValueTypeKrylovBases, ValueType>;
     static_assert(Accessor::has_scale,
                   "This accessor must have a scale in this class!");
 
-public:
     Accessor3dHelper() = default;
 
     Accessor3dHelper(std::shared_ptr<const Executor> exec, dim<3> krylov_dim)
@@ -442,11 +442,11 @@ class Accessor3dHelper<ValueType, ValueTypeKrylovBases, true> {
 
 template <typename ValueType, typename ValueTypeKrylovBases>
 class Accessor3dHelper<ValueType, ValueTypeKrylovBases, false> {
+public:
     using Accessor = Accessor3d<ValueTypeKrylovBases, ValueType>;
     static_assert(!Accessor::has_scale,
                   "This accessor must not have a scale in this class!");
 
-public:
     Accessor3dHelper() = default;
 
     Accessor3dHelper(std::shared_ptr<const Executor> exec, dim<3> krylov_dim)
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index b72e3a17c5a..68c124cd2f4 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -213,13 +213,14 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
     //     j));
     /* */
     // std::cout << "Before set_scale_kernel" << '\n';
+    const auto stride_arnoldi = arnoldi_norm->get_stride();
     set_scale_kernel<default_block_size>
         <<<ceildiv(num_rhs * (krylov_dim + 1), default_block_size),
            default_block_size>>>(
             num_rhs, krylov_dim + 1,
             as_cuda_type(residual_norm->get_const_values()), num_rhs,
-            as_cuda_type(arnoldi_norm->get_const_values() + 2 * num_rhs),
-            num_rhs, as_cuda_accessor(krylov_bases));
+            as_cuda_type(arnoldi_norm->get_const_values() + 2 * stride_arnoldi),
+            stride_arnoldi, as_cuda_accessor(krylov_bases));
     // exec->synchronize();
     /* */
     // std::cout << "Before initialize_2_2_kernel" << '\n';
@@ -344,7 +345,7 @@ void finish_arnoldi_reorth(
     multinorm2_kernel<<<grid_size, block_size>>>(
         dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
+        as_cuda_type(arnoldi_norm->get_values()), as_cuda_type(stop_status));
     // nrm = norm(next_krylov_basis)
     for (size_type k = 0; k < iter + 1; ++k) {
         components::fill_array(
@@ -462,7 +463,7 @@ void finish_arnoldi_CGS(
     multinorm2_kernel<<<grid_size, block_size>>>(
         dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
+        as_cuda_type(arnoldi_norm->get_values()), as_cuda_type(stop_status));
     // nrmP = norm(next_krylov_basis
     for (size_type k = 0; k < iter + 1; ++k) {
         components::fill_array(
@@ -491,14 +492,14 @@ void finish_arnoldi_CGS(
     // for i in 1:iter
     //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
     // end
-    components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
+    components::fill_array(exec, arnoldi_norm->get_values() + stride_arnoldi,
                            dim_size[1], zero<ValueType>());
     /*
     multidot_kernel<<<grid_size, block_size>>>(
         0, dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_accessor(next_krylov_const_accessor),
-        as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
+        as_cuda_type(arnoldi_norm->get_values() + stride_arnoldi), 0,
         as_cuda_type(stop_status));
     */
     multinorm2_kernel<<<grid_size, block_size>>>(
@@ -548,7 +549,7 @@ void finish_arnoldi_CGS(
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
         components::fill_array(
-            exec, arnoldi_norm->get_values() + dim_size[1] * (l - 1),
+            exec, arnoldi_norm->get_values() + stride_arnoldi * (l - 1),
             dim_size[1], zero<ValueType>());
         /*
         multidot_kernel<<<grid_size, block_size>>>(
@@ -562,8 +563,8 @@ void finish_arnoldi_CGS(
             dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
             stride_next_krylov,
-            as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
-            as_cuda_type(stop_status));
+            as_cuda_type(arnoldi_norm->get_values() + stride_arnoldi * (l - 1)),
+            0, as_cuda_type(stop_status));
         // nrmN = norm(next_krylov_basis)
         components::fill_array(exec, num_reorth->get_data(), 1,
                                zero<size_type>());
@@ -620,6 +621,8 @@ void finish_arnoldi_CGS2(
     using non_complex = remove_complex<ValueType>;
     // optimization parameter
     constexpr int singledot_block_size = default_dot_dim;
+    constexpr bool use_scale =
+        Accessor3d<ValueTypeKrylovBases, ValueType>::has_scale;
     const auto stride_next_krylov = next_krylov_basis->get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
     const auto stride_buffer = buffer_iter->get_stride();
@@ -662,7 +665,7 @@ void finish_arnoldi_CGS2(
     multinorm2_kernel<<<grid_size, block_size>>>(
         dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
+        as_cuda_type(arnoldi_norm->get_values()), as_cuda_type(stop_status));
     // nrmP = norm(next_krylov_basis
 #ifdef TIMING
     exec->synchronize();
@@ -759,7 +762,7 @@ void finish_arnoldi_CGS2(
     multinorm2_kernel<<<grid_size, block_size>>>(
         dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
+        as_cuda_type(arnoldi_norm->get_values() + dim_size[1]),
         as_cuda_type(stop_status));
     */
     /*
@@ -769,11 +772,11 @@ void finish_arnoldi_CGS2(
      as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]), 0,
      as_cuda_type(stop_status));
  */ /* */
-    multinorm2_inf_kernel<<<grid_size, block_size>>>(
+    multinorm2_inf_kernel<use_scale><<<grid_size, block_size>>>(
         dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_type(arnoldi_norm->get_values() + dim_size[1]),
-        as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]), 0,
+        as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]),
         as_cuda_type(stop_status));
     /* */
     /*
@@ -917,7 +920,7 @@ void finish_arnoldi_CGS2(
             dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
             stride_next_krylov,
-            as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
+            as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)),
             as_cuda_type(stop_status));
         */
         /*
@@ -928,13 +931,13 @@ void finish_arnoldi_CGS2(
          as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]), 0,
          as_cuda_type(stop_status));
      */ /* */
-        multinorm2_inf_kernel<<<grid_size, block_size>>>(
+        multinorm2_inf_kernel<use_scale><<<grid_size, block_size>>>(
             dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
             stride_next_krylov,
             as_cuda_type(arnoldi_norm->get_values() + dim_size[1]),
             // as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)),
-            as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]), 0,
+            as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]),
             as_cuda_type(stop_status));
         /* */
         // nrmN = norm(next_krylov_basis)

From f7e8b3d0f60f44951115bbffe64d718fe136310a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 23 Jun 2020 07:20:47 +0200
Subject: [PATCH 0363/1008] Add another RHS generation in the benchmark

- The benchmark script now has the option to change the initial guess
- Add option to generate the RHS with:
  b = A * (s / |s|) with s(i) = sin(i)
---
 BENCHMARKING.md                 | 13 +++--
 benchmark/run_all_benchmarks.sh | 41 +++++++++++++--
 benchmark/solver/solver.cpp     | 89 ++++++++++++++++++++++++++-------
 benchmark/utils/general.hpp     | 68 +++++++++++++++++++------
 4 files changed, 170 insertions(+), 41 deletions(-)

diff --git a/BENCHMARKING.md b/BENCHMARKING.md
index 33e4505c0d6..9d691f309d3 100644
--- a/BENCHMARKING.md
+++ b/BENCHMARKING.md
@@ -143,7 +143,7 @@ The benchmark suite can take a number of configuration parameters. Benchmarks
 can be run only for `sparse matrix vector products (spmv)`, for full solvers
 (with or without preconditioners), or for preconditioners only when supported.
 The benchmark suite also allows to target a sub-part of the SuiteSparse matrix
-collection. For details, see the [available benchmark options](### 5: Available
+collection. For details, see the [available benchmark options](### 6: Available
 benchmark options). Here are the most important options:
 * `BENCHMARK={spmv, solver, preconditioner}` - allows to select the type of
     benchmark to be ran.
@@ -303,9 +303,16 @@ The supported environment variables are described in the following list:
     the solver should stop. The default is `1e-6`.
 * `SOLVERS_MAX_ITERATION=<number>` - the maximum number of iterations with which
     a solver should be ran. The default is `10000`.
-* `SOLVERS_RHS={unit, random}` - whether to use a vector of all ones or random
-    values as the right-hand side in solver benchmarks. Default is `unit`.
+* `SOLVERS_RHS={1, random, sinus}` - whether to use a vector of all ones,
+    random values or $b = A * (s / |s|)$ with $s(i) = sin(i)$
+    as the right-hand side in solver benchmarks. Default is `1`.
+* `SOLVERS_INITIAL_GUESS`={rhs,0,random} - the initial guess generation of the
+    solvers. `rhs` uses the right-hand side, `0` uses a zero vector and `random`
+    generates a random vector as the initial guess.
 * `DETAILED={0,1}` - selects whether detailed benchmarks should be ran for the
     solver benchmarks, can be either `0` (off) or `1` (on). The default is `0`.
 * `GPU_TIMER={true, false}` - If set to `true`, use the gpu timer, which is
     valid for cuda/hip executor, to measure the timing. Default is `false`.
+* `SOLVERS_JACOBI_MAX_BS` - sets the maximum block size for the Jacobi
+    preconditioner (if used, otherwise, it does nothing) in the solvers
+    benchmark
diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index 9105bd85e10..e6996f0503c 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -60,9 +60,15 @@ if [ ! "${DEVICE_ID}" ]; then
     DEVICE_ID="0"
 fi
 
+if [ ! "${SOLVERS_JACOBI_MAX_BS}" ]; then
+    SOLVERS_JACOBI_MAX_BS="32"
+    "SOLVERS_JACOBI_MAX_BS environment variable not set - assuming \"${SOLVERS_JACOBI_MAX_BS}\"" 1>&2
+fi
+
+
 if [ ! "${SOLVERS_RHS}" ]; then
-    echo "SOLVERS_RHS environment variable not set - assuming \"unit\"" 1>&2
-    SOLVERS_RHS="unit"
+    SOLVERS_RHS="1"
+    echo "SOLVERS_RHS environment variable not set - assuming \"${SOLVERS_RHS}\"" 1>&2
 fi
 
 if [ ! "${BENCHMARK_PRECISION}" ]; then
@@ -85,9 +91,20 @@ else
 fi
 
 if [ "${SOLVERS_RHS}" == "random" ]; then
-    SOLVERS_RHS_FLAG="--random_rhs=true"
+    SOLVERS_RHS_FLAG="--rhs_generation=random"
+elif [ "${SOLVERS_RHS}" == "1" ]; then
+    SOLVERS_RHS_FLAG="--rhs_generation=1"
+elif [ "${SOLVERS_RHS}" == "sinus" ]; then
+    SOLVERS_RHS_FLAG="--rhs_generation=sinus"
 else
-    SOLVERS_RHS_FLAG="--random_rhs=false"
+    echo "SOLVERS_RHS does not support the value \"${SOLVERS_RHS}\"." 1>&2
+    echo "The following values are supported: \"1\", \"random\" and \"sinus\"" 1>&2
+    exit 1
+fi
+
+if [ ! "${SOLVERS_INITIAL_GUESS}" ]; then
+    SOLVERS_INITIAL_GUESS="rhs"
+    echo "SOLVERS_RHS environment variable not set - assuming \"${SOLVERS_INITIAL_GUESS}\"" 1>&2
 fi
 
 if [ ! "${GPU_TIMER}" ]; then
@@ -95,6 +112,18 @@ if [ ! "${GPU_TIMER}" ]; then
     GPU_TIMER="false"
 fi
 
+if [ "${SOLVERS_INITIAL_GUESS}" == "random" ]; then
+    SOLVERS_INITIAL_GUESS_FLAG="--initial_guess_generation=random"
+elif [ "${SOLVERS_INITIAL_GUESS}" == "0" ]; then
+    SOLVERS_INITIAL_GUESS_FLAG="--initial_guess_generation=0"
+elif [ "${SOLVERS_INITIAL_GUESS}" == "rhs" ]; then
+    SOLVERS_INITIAL_GUESS_FLAG="--initial_guess_generation=rhs"
+else
+    echo "SOLVERS_RHS does not support the value \"${SOLVERS_RHS}\"." 1>&2
+    echo "The following values are supported: \"1\", \"random\" and \"sinus\"" 1>&2
+    exit 1
+fi
+
 # Control whether to run detailed benchmarks or not.
 # Default setting is detailed=false. To activate, set DETAILED=1.
 if  [ ! "${DETAILED}" ] || [ "${DETAILED}" -eq 0 ]; then
@@ -202,7 +231,9 @@ run_solver_benchmarks() {
                     --executor="${EXECUTOR}" --solvers="${SOLVERS}" \
                     --preconditioners="${PRECONDS}" \
                     --max_iters=${SOLVERS_MAX_ITERATIONS} --rel_res_goal=${SOLVERS_PRECISION} \
-                    ${SOLVERS_RHS_FLAG} ${DETAILED_STR} --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
+                    ${SOLVERS_RHS_FLAG} ${DETAILED_STR} ${SOLVERS_INITIAL_GUESS_FLAG} \
+                    --gpu_timer=${GPU_TIMER} \
+                    --jacobi_max_block_size=${SOLVERS_JACOBI_MAX_BS} --device_id="${DEVICE_ID}" \
                     <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index fa029be7d2d..81dec07c45f 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -89,11 +89,19 @@ DEFINE_double(
     idr_kappa, 0.7,
     "the number to check whether Av_n and v_n are too close or not in IDR");
 
-DEFINE_bool(random_rhs, false,
-            "Use a random vector for the rhs (otherwise use all ones)");
-
-DEFINE_bool(random_initial_guess, false,
-            "Use a random vector for the initial guess (otherwise use rhs)");
+DEFINE_string(
+    rhs_generation, "1",
+    "Method used to generate the right hand side. Supported values are:"
+    "1, random, sinus. 1 sets all values of the right hand side to 1, "
+    "random assigns the values to a uniformly distributed random number "
+    "in [-1, 1), and sinus assigns b = A * (s / |s|) with A := system matrix, "
+    "s := vector with s(i) = sin(i).");
+
+DEFINE_string(
+    initial_guess_generation, "rhs",
+    "Method used to generate the initial guess. Supported values are: "
+    "random, rhs, 0. random uses a random vector, rhs uses the right "
+    "hand side, and 0 uses a zero vector as the initial guess.");
 
 // This allows to benchmark the overhead of a solver by using the following
 // data: A=[1.0], x=[0.0], b=[nan]. This data can be used to benchmark normal
@@ -118,6 +126,62 @@ DEFINE_bool(overhead, false,
 }
 
 
+template <typename Engine>
+std::unique_ptr<vec<etype>> generate_rhs(
+    std::shared_ptr<const gko::Executor> exec,
+    std::shared_ptr<const gko::LinOp> system_matrix, Engine engine)
+{
+    using rc_etype = gko::remove_complex<etype>;
+    gko::dim<2> vec_size{system_matrix->get_size()[0], FLAGS_nrhs};
+    if (FLAGS_rhs_generation == "1") {
+        return create_matrix<etype>(exec, vec_size, gko::one<etype>());
+    } else if (FLAGS_rhs_generation == "random") {
+        return create_matrix<etype>(exec, vec_size, engine);
+    } else if (FLAGS_rhs_generation == "sinus") {
+        auto rhs = vec<etype>::create(exec, vec_size);
+
+        auto tmp = create_matrix_sin<etype>(exec, vec_size);
+        auto scalar = gko::matrix::Dense<rc_etype>::create(
+            exec->get_master(), gko::dim<2>{1, vec_size[1]});
+        tmp->compute_norm2(scalar.get());
+        for (gko::size_type i = 0; i < vec_size[1]; ++i) {
+            scalar->at(0, i) = gko::one<rc_etype>() / scalar->at(0, i);
+        }
+        // normalize sin-vector
+        if (gko::is_complex_s<etype>::value) {
+            tmp->scale(scalar->make_complex().get());
+        } else {
+            tmp->scale(scalar.get());
+        }
+        system_matrix->apply(tmp.get(), rhs.get());
+        return rhs;
+    }
+    throw std::invalid_argument(std::string("\"rhs_generation\" = ") +
+                                FLAGS_rhs_generation + " is not supported!");
+}
+
+
+template <typename Engine>
+std::unique_ptr<vec<etype>> generate_initial_guess(
+    std::shared_ptr<const gko::Executor> exec,
+    std::shared_ptr<const gko::LinOp> system_matrix, const vec<etype> *rhs,
+    Engine engine)
+{
+    using rc_etype = gko::remove_complex<etype>;
+    gko::dim<2> vec_size{system_matrix->get_size()[1], FLAGS_nrhs};
+    if (FLAGS_initial_guess_generation == "0") {
+        return create_matrix<etype>(exec, vec_size, gko::zero<etype>());
+    } else if (FLAGS_initial_guess_generation == "random") {
+        return create_matrix<etype>(exec, vec_size, engine);
+    } else if (FLAGS_initial_guess_generation == "rhs") {
+        return rhs->clone();
+    }
+    throw std::invalid_argument(std::string("\"initial_guess_generation\" = ") +
+                                FLAGS_initial_guess_generation +
+                                " is not supported!");
+}
+
+
 void validate_option_object(const rapidjson::Value &value)
 {
     if (!value.IsObject() || !value.HasMember("optimal") ||
@@ -552,19 +616,10 @@ int main(int argc, char *argv[])
                     std::ifstream rhs_fd{test_case["rhs"].GetString()};
                     b = gko::read<Vec>(rhs_fd, exec);
                 } else {
-                    b = create_matrix<etype>(
-                        exec,
-                        gko::dim<2>{system_matrix->get_size()[0], FLAGS_nrhs},
-                        engine, FLAGS_random_rhs);
-                }
-                if (FLAGS_random_initial_guess) {
-                    x = create_matrix<etype>(
-                        exec,
-                        gko::dim<2>{system_matrix->get_size()[0], FLAGS_nrhs},
-                        engine);
-                } else {
-                    x = b->clone();
+                    b = generate_rhs(exec, system_matrix, engine);
                 }
+                x = generate_initial_guess(exec, system_matrix, b.get(),
+                                           engine);
             }
 
             std::clog << "Matrix is of size (" << system_matrix->get_size()[0]
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index 3094083c26d..ee8cdddf4a4 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -287,22 +287,51 @@ template <typename ValueType>
 using vec = gko::matrix::Dense<ValueType>;
 
 
-// creates a zero vector
+// Create a matrix with value indices s[i, j] = sin(i)
 template <typename ValueType>
-std::unique_ptr<vec<ValueType>> create_vector(
-    std::shared_ptr<const gko::Executor> exec, gko::size_type size)
+std::enable_if_t<!gko::is_complex_s<ValueType>::value,
+                 std::unique_ptr<vec<ValueType>>>
+create_matrix_sin(std::shared_ptr<const gko::Executor> exec, gko::dim<2> size)
 {
+    auto h_res = vec<ValueType>::create(exec->get_master(), size);
+    for (gko::size_type i = 0; i < size[0]; ++i) {
+        for (gko::size_type j = 0; j < size[1]; ++j) {
+            h_res->at(i, j) = std::sin(static_cast<ValueType>(i));
+        }
+    }
     auto res = vec<ValueType>::create(exec);
-    res->read(gko::matrix_data<ValueType>(gko::dim<2>{size, 1}));
+    h_res->move_to(res.get());
     return res;
 }
 
+// Note: complex values are assigned s[i, j] = {sin(2 * i, sin(2 * i + 1))}
+template <typename ValueType>
+std::enable_if_t<gko::is_complex_s<ValueType>::value,
+                 std::unique_ptr<vec<ValueType>>>
+create_matrix_sin(std::shared_ptr<const gko::Executor> exec, gko::dim<2> size)
+{
+    auto h_res = vec<ValueType>::create(exec->get_master(), size);
+    for (gko::size_type i = 0; i < size[0]; ++i) {
+        for (gko::size_type j = 0; j < size[1]; ++j) {
+            h_res->at(i, j) = ValueType{
+                std::sin(static_cast<gko::remove_complex<ValueType>>(2 * i)),
+                std::sin(
+                    static_cast<gko::remove_complex<ValueType>>(2 * i + 1))};
+        }
+    }
+    auto res = vec<ValueType>::create(exec);
+    h_res->move_to(res.get());
+    return res;
+}
+
+
 template <typename ValueType>
 std::unique_ptr<vec<ValueType>> create_matrix(
-    std::shared_ptr<const gko::Executor> exec, gko::dim<2> size)
+    std::shared_ptr<const gko::Executor> exec, gko::dim<2> size,
+    ValueType value)
 {
     auto res = vec<ValueType>::create(exec);
-    res->read(gko::matrix_data<ValueType>(size));
+    res->read(gko::matrix_data<ValueType>(size, value));
     return res;
 }
 
@@ -311,18 +340,25 @@ std::unique_ptr<vec<ValueType>> create_matrix(
 template <typename ValueType, typename RandomEngine>
 std::unique_ptr<vec<ValueType>> create_matrix(
     std::shared_ptr<const gko::Executor> exec, gko::dim<2> size,
-    RandomEngine &engine, bool random = true)
+    RandomEngine &engine)
 {
     auto res = vec<ValueType>::create(exec);
-    if (random) {
-        res->read(gko::matrix_data<ValueType>(
-            size,
-            std::uniform_real_distribution<gko::remove_complex<ValueType>>(-1.0,
-                                                                           1.0),
-            engine));
-    } else {
-        res->read(gko::matrix_data<ValueType>(size, gko::one<ValueType>()));
-    }
+    res->read(gko::matrix_data<ValueType>(
+        size,
+        std::uniform_real_distribution<gko::remove_complex<ValueType>>(-1.0,
+                                                                       1.0),
+        engine));
+    return res;
+}
+
+
+// creates a zero vector
+template <typename ValueType>
+std::unique_ptr<vec<ValueType>> create_vector(
+    std::shared_ptr<const gko::Executor> exec, gko::size_type size)
+{
+    auto res = vec<ValueType>::create(exec);
+    res->read(gko::matrix_data<ValueType>(gko::dim<2>{size, 1}));
     return res;
 }
 

From c2f77903d097d85e00a8bd9cd363219f1f0ab973 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 23 Jun 2020 20:25:30 +0200
Subject: [PATCH 0364/1008] Fix residual_norm calculation in GmresMixed

---
 common/solver/gmres_mixed_kernels.hpp.inc | 10 ++++------
 omp/solver/gmres_mixed_kernels.cpp        |  5 +++--
 reference/solver/gmres_mixed_kernels.cpp  |  5 +++--
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index d6c895bc47a..17391c668ed 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -1468,18 +1468,16 @@ __device__ void calculate_residual_norm_kernel(
 {
     residual_norm_collection[(iter + 1) * stride_residual_norm_collection +
                              col_idx] =
-        -givens_sin[iter * stride_sin + col_idx] *
+        -conj(givens_sin[iter * stride_sin + col_idx]) *
         residual_norm_collection[iter * stride_residual_norm_collection +
                                  col_idx];
     residual_norm_collection[iter * stride_residual_norm_collection + col_idx] =
         givens_cos[iter * stride_cos + col_idx] *
         residual_norm_collection[iter * stride_residual_norm_collection +
                                  col_idx];
-    residual_norm[col_idx] =
-        abs(residual_norm_collection[(iter + 1) *
-                                         stride_residual_norm_collection +
-                                     col_idx]) /
-        b_norm[col_idx];
+    residual_norm[col_idx] = abs(
+        residual_norm_collection[(iter + 1) * stride_residual_norm_collection +
+                                 col_idx]);
 }
 
 
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index ec0fd01bd44..e7311bbf5e4 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -572,11 +572,12 @@ void calculate_next_residual_norm(
             continue;
         }
         residual_norm_collection->at(iter + 1, i) =
-            -givens_sin->at(iter, i) * residual_norm_collection->at(iter, i);
+            -conj(givens_sin->at(iter, i)) *
+            residual_norm_collection->at(iter, i);
         residual_norm_collection->at(iter, i) =
             givens_cos->at(iter, i) * residual_norm_collection->at(iter, i);
         residual_norm->at(0, i) =
-            abs(residual_norm_collection->at(iter + 1, i)) / b_norm->at(0, i);
+            abs(residual_norm_collection->at(iter + 1, i));
     }
 }
 
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index b1124fe0d4d..eb53fe910ef 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -514,11 +514,12 @@ void calculate_next_residual_norm(
             continue;
         }
         residual_norm_collection->at(iter + 1, i) =
-            -givens_sin->at(iter, i) * residual_norm_collection->at(iter, i);
+            -conj(givens_sin->at(iter, i)) *
+            residual_norm_collection->at(iter, i);
         residual_norm_collection->at(iter, i) =
             givens_cos->at(iter, i) * residual_norm_collection->at(iter, i);
         residual_norm->at(0, i) =
-            abs(residual_norm_collection->at(iter + 1, i)) / b_norm->at(0, i);
+            abs(residual_norm_collection->at(iter + 1, i));
     }
 }
 

From 515e7ae3231e364cfc3b33efcf06551885c2d404 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 25 Jun 2020 21:06:12 +0200
Subject: [PATCH 0365/1008] Make sure GmresMixed does not exit early

If the stopping criterion is met, perform a reset of GMRES and check
the residual again. If it is still correct, exit, otherwise, keep
calculating.

Other changes:
- Renamed krylov_dim_mixed to krylov_dim in GmresMixed (to be
  consistent with Gmres)
---
 core/solver/gmres_mixed.cpp                   | 72 ++++++++++++-------
 core/test/solver/gmres_mixed.cpp              |  6 +-
 include/ginkgo/core/solver/gmres_mixed.hpp    | 12 ++--
 reference/test/solver/gmres_mixed_kernels.cpp |  2 +-
 4 files changed, 57 insertions(+), 35 deletions(-)

diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 4dee1655817..c49ea9ba75f 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -100,12 +100,11 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     auto dense_b = as<const Vector>(b);
     auto dense_x = as<Vector>(x);
     auto residual = Vector::create_with_config_of(dense_b);
-    const dim<3> krylov_bases_dim{krylov_dim_mixed_ + 1,
-                                  system_matrix_->get_size()[1],
-                                  dense_b->get_size()[1]};
+    const dim<3> krylov_bases_dim{
+        krylov_dim_ + 1, system_matrix_->get_size()[1], dense_b->get_size()[1]};
     // const dim<2> krylov_bases_dim{
     //     system_matrix_->get_size()[1],
-    //     (krylov_dim_mixed_ + 1) * dense_b->get_size()[1]};
+    //     (krylov_dim_ + 1) * dense_b->get_size()[1]};
     // const size_type krylov_bases_stride = krylov_bases_dim[1];
     // LowArray krylov_bases(exec, krylov_bases_dim[0] * krylov_bases_stride);
     // KrylovAccessor krylov_bases_accessor(krylov_bases.get_data(),
@@ -117,16 +116,15 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     std::shared_ptr<matrix::Dense<ValueType>> preconditioned_vector =
         Vector::create_with_config_of(dense_b);
     auto hessenberg = Vector::create(
-        exec, dim<2>{krylov_dim_mixed_ + 1,
-                     krylov_dim_mixed_ * dense_b->get_size()[1]});
-    auto buffer = Vector::create(
-        exec, dim<2>{krylov_dim_mixed_ + 1, dense_b->get_size()[1]});
+        exec, dim<2>{krylov_dim_ + 1, krylov_dim_ * dense_b->get_size()[1]});
+    auto buffer =
+        Vector::create(exec, dim<2>{krylov_dim_ + 1, dense_b->get_size()[1]});
     auto givens_sin =
-        Vector::create(exec, dim<2>{krylov_dim_mixed_, dense_b->get_size()[1]});
+        Vector::create(exec, dim<2>{krylov_dim_, dense_b->get_size()[1]});
     auto givens_cos =
-        Vector::create(exec, dim<2>{krylov_dim_mixed_, dense_b->get_size()[1]});
-    auto residual_norm_collection = Vector::create(
-        exec, dim<2>{krylov_dim_mixed_ + 1, dense_b->get_size()[1]});
+        Vector::create(exec, dim<2>{krylov_dim_, dense_b->get_size()[1]});
+    auto residual_norm_collection =
+        Vector::create(exec, dim<2>{krylov_dim_ + 1, dense_b->get_size()[1]});
     auto residual_norm =
         VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
     auto b_norm = VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
@@ -138,8 +136,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
                      dense_b->get_size()[1]});
     Array<size_type> final_iter_nums(this->get_executor(),
                                      dense_b->get_size()[1]);
-    auto y =
-        Vector::create(exec, dim<2>{krylov_dim_mixed_, dense_b->get_size()[1]});
+    auto y = Vector::create(exec, dim<2>{krylov_dim_, dense_b->get_size()[1]});
 
     bool one_changed{};
     Array<stopping_status> stop_status(this->get_executor(),
@@ -153,7 +150,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     // Initialization
     exec->run(gmres_mixed::make_initialize_1(
         dense_b, b_norm.get(), residual.get(), givens_sin.get(),
-        givens_cos.get(), &stop_status, krylov_dim_mixed_));
+        givens_cos.get(), &stop_status, krylov_dim_));
     // b_norm = norm(b)
     // residual = dense_b
     // givens_sin = givens_cos = 0
@@ -165,7 +162,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     exec->run(gmres_mixed::make_initialize_2(
         residual.get(), residual_norm.get(), residual_norm_collection.get(),
         arnoldi_norm.get(), krylov_bases_accessor, next_krylov_basis.get(),
-        &final_iter_nums, krylov_dim_mixed_));
+        &final_iter_nums, krylov_dim_));
     // residual_norm = norm(residual)
     // residual_norm_collection = {residual_norm, 0, ..., 0}
     // krylov_bases(:, 1) = residual / residual_norm
@@ -193,7 +190,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     auto time_STEP1 = start - start;
 #endif
 #endif
-
+    bool stop_already_encountered{false};
     // std::cout << "Before loop" << std::endl;
     while (true) {
         ++total_iter;
@@ -205,10 +202,30 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
                 .residual_norm(residual_norm.get())
                 .solution(dense_x)
                 .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
-            break;
+            if (stop_already_encountered) {
+                break;
+            }
+            stop_already_encountered = true;
+            Array<stopping_status> host_stop_status(
+                this->get_executor()->get_master(), stop_status);
+            bool host_array_changed{false};
+            for (size_type i = 0; i < host_stop_status.get_num_elems(); ++i) {
+                auto local_status = host_stop_status.get_data() + i;
+                if (local_status->has_converged()) {
+                    local_status->reset();
+                    host_array_changed = true;
+                }
+            }
+            if (host_array_changed) {
+                stop_status = host_stop_status;
+            } else {
+                break;
+            }
+        } else {
+            stop_already_encountered = false;
         }
 
-        if (restart_iter == krylov_dim_mixed_) {
+        if (stop_already_encountered || restart_iter == krylov_dim_) {
 #ifdef TIMING_STEPS
             exec->synchronize();
             auto t_aux_0 = std::chrono::steady_clock::now();
@@ -218,8 +235,8 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
             // Restart
             /*
             std::cout << "[ ";
-            for (int i = 0; i <= krylov_dim_mixed_; i++) {
-                for (int j = 0; j < krylov_dim_mixed_ * dense_b->get_size()[1];
+            for (int i = 0; i <= krylov_dim_; i++) {
+                for (int j = 0; j < krylov_dim_ * dense_b->get_size()[1];
                      j++) {
                     if (j > 0) std::cout << ", ";
                     std::cout << hessenberg->at(i, j);
@@ -228,10 +245,15 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
             }
             std::cout << "]" << std::endl;
             */
+            // use a view in case this is called earlier
+            auto hessenberg_view = hessenberg->create_submatrix(
+                span{0, restart_iter},
+                span{0, dense_b->get_size()[1] * (restart_iter)});
+
             exec->run(gmres_mixed::make_step_2(
                 residual_norm_collection.get(),
-                krylov_bases_accessor.to_const(), hessenberg.get(), y.get(),
-                before_preconditioner.get(), &final_iter_nums));
+                krylov_bases_accessor.to_const(), hessenberg_view.get(),
+                y.get(), before_preconditioner.get(), &final_iter_nums));
             /* */ /*
             auto hessenberg_small = hessenberg->create_submatrix(
                 span{0, restart_iter},
@@ -259,7 +281,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
                 residual.get(), residual_norm.get(),
                 residual_norm_collection.get(), arnoldi_norm.get(),
                 krylov_bases_accessor, next_krylov_basis.get(),
-                &final_iter_nums, krylov_dim_mixed_));
+                &final_iter_nums, krylov_dim_));
             // residual_norm = norm(residual)
             // residual_norm_collection = {residual_norm, 0, ..., 0}
             // krylov_bases(:, 1) = residual / residual_norm
@@ -350,7 +372,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
         span{0, dense_b->get_size()[1] * (restart_iter + 1)});
     const dim<2> krylov_bases_dim{
         system_matrix_->get_size()[1],
-        (krylov_dim_mixed_ + 1) * dense_b->get_size()[1]};
+        (krylov_dim_ + 1) * dense_b->get_size()[1]};
     */
     /*
     const dim<2> krylov_bases_small_dim{
diff --git a/core/test/solver/gmres_mixed.cpp b/core/test/solver/gmres_mixed.cpp
index f9c09ea39d9..42db208ba4d 100644
--- a/core/test/solver/gmres_mixed.cpp
+++ b/core/test/solver/gmres_mixed.cpp
@@ -175,7 +175,7 @@ TEST_F(GmresMixed, CanSetKrylovDim)
 {
     auto gmres_mixed_factory =
         Solver::build()
-            .with_krylov_dim_mixed(4u)
+            .with_krylov_dim(4u)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(4u).on(exec),
                 gko::stop::ResidualNormReduction<>::build()
@@ -183,9 +183,9 @@ TEST_F(GmresMixed, CanSetKrylovDim)
                     .on(exec))
             .on(exec);
     auto solver = gmres_mixed_factory->generate(mtx);
-    auto krylov_dim_mixed = solver->get_krylov_dim_mixed();
+    auto krylov_dim = solver->get_krylov_dim();
 
-    ASSERT_EQ(krylov_dim_mixed, 4);
+    ASSERT_EQ(krylov_dim, 4);
 }
 
 TEST_F(GmresMixed, CanSetPreconditionerInFactory)
diff --git a/include/ginkgo/core/solver/gmres_mixed.hpp b/include/ginkgo/core/solver/gmres_mixed.hpp
index bb466919c09..ed189f31915 100644
--- a/include/ginkgo/core/solver/gmres_mixed.hpp
+++ b/include/ginkgo/core/solver/gmres_mixed.hpp
@@ -106,7 +106,7 @@ class GmresMixed :
      *
      * @return the krylov dimension
      */
-    size_type get_krylov_dim_mixed() const { return krylov_dim_mixed_; }
+    size_type get_krylov_dim() const { return krylov_dim_; }
 
     GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
     {
@@ -132,7 +132,7 @@ class GmresMixed :
         /**
          * krylov dimension factory.
          */
-        size_type GKO_FACTORY_PARAMETER(krylov_dim_mixed, 0u);
+        size_type GKO_FACTORY_PARAMETER(krylov_dim, 0u);
     };
     GKO_ENABLE_LIN_OP_FACTORY(GmresMixed, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);
@@ -165,10 +165,10 @@ class GmresMixed :
             set_preconditioner(matrix::Identity<ValueType>::create(
                 this->get_executor(), this->get_size()[0]));
         }
-        if (parameters_.krylov_dim_mixed) {
-            krylov_dim_mixed_ = parameters_.krylov_dim_mixed;
+        if (parameters_.krylov_dim) {
+            krylov_dim_ = parameters_.krylov_dim;
         } else {
-            krylov_dim_mixed_ = default_krylov_dim_mixed;
+            krylov_dim_ = default_krylov_dim_mixed;
         }
         stop_criterion_factory_ =
             stop::combine(std::move(parameters_.criteria));
@@ -177,7 +177,7 @@ class GmresMixed :
 private:
     std::shared_ptr<const LinOp> system_matrix_{};
     std::shared_ptr<const stop::CriterionFactory> stop_criterion_factory_{};
-    size_type krylov_dim_mixed_;
+    size_type krylov_dim_;
 };
 
 
diff --git a/reference/test/solver/gmres_mixed_kernels.cpp b/reference/test/solver/gmres_mixed_kernels.cpp
index 345a85778c4..abd1c0e767d 100644
--- a/reference/test/solver/gmres_mixed_kernels.cpp
+++ b/reference/test/solver/gmres_mixed_kernels.cpp
@@ -316,7 +316,7 @@ TYPED_TEST(GmresMixed, SolvesBigDenseSystem1WithRestart)
     const auto half_tol = std::sqrt(r<value_type>::value);
     auto gmres_mixed_factory_restart =
         gmres_type::build()
-            .with_krylov_dim_mixed(4u)
+            .with_krylov_dim(4u)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(200u).on(
                     this->exec),

From c0377fa1e17fb12ebe93ebf75dfcf2d3e3c17a11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 26 Jun 2020 06:39:34 +0200
Subject: [PATCH 0366/1008] Add benchmark parameter for GMRES krylov_dim

Adds the parameter in the benchmark script (and documentation).
---
 BENCHMARKING.md                 | 2 ++
 benchmark/run_all_benchmarks.sh | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/BENCHMARKING.md b/BENCHMARKING.md
index 9d691f309d3..ea2802510c7 100644
--- a/BENCHMARKING.md
+++ b/BENCHMARKING.md
@@ -316,3 +316,5 @@ The supported environment variables are described in the following list:
 * `SOLVERS_JACOBI_MAX_BS` - sets the maximum block size for the Jacobi
     preconditioner (if used, otherwise, it does nothing) in the solvers
     benchmark
+* `SOLVERS_GMRES_RESTART` - the maximum dimension of the Krylov space to use in
+    GMRES. The default is `100`.
diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index e6996f0503c..eb859fbea7a 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -50,6 +50,11 @@ if [ ! "${SOLVERS_MAX_ITERATIONS}" ]; then
     SOLVERS_MAX_ITERATIONS=10000
 fi
 
+if [ ! "${SOLVERS_GMRES_RESTART}" ]; then
+    SOLVERS_GMRES_RESTART=100
+    echo "SOLVERS_GMRES_RESTART    environment variable not set - assuming \"${SOLVERS_GMRES_RESTART}\"" 1>&2
+fi
+
 if [ ! "${SYSTEM_NAME}" ]; then
     echo "SYSTEM_MANE environment variable not set - assuming \"unknown\"" 1>&2
     SYSTEM_NAME="unknown"
@@ -234,6 +239,7 @@ run_solver_benchmarks() {
                     ${SOLVERS_RHS_FLAG} ${DETAILED_STR} ${SOLVERS_INITIAL_GUESS_FLAG} \
                     --gpu_timer=${GPU_TIMER} \
                     --jacobi_max_block_size=${SOLVERS_JACOBI_MAX_BS} --device_id="${DEVICE_ID}" \
+                    --gmres_restart="${SOLVERS_GMRES_RESTART}" \
                     <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }

From 4e307a8f51ca013a29a71025c66bddfebf7d612a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 26 Jun 2020 11:44:03 +0200
Subject: [PATCH 0367/1008] Add forced iterations when convergence is detected

---
 core/solver/gmres_mixed.cpp | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index c49ea9ba75f..9915a8301ce 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -191,18 +191,27 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
 #endif
 #endif
     bool stop_already_encountered{false};
-    // std::cout << "Before loop" << std::endl;
+    bool perform_reset{false};
+    decltype(krylov_dim_) forced_iterations{0};
+    const decltype(forced_iterations) forced_limit{krylov_dim_ / 10};
+    // TODO: take care of multiple RHS. Currently, we restart everything,
+    //       even though a lot of parts might have already converged!
+    //       Use `one_changed` to take care of that!
     while (true) {
         ++total_iter;
         this->template log<log::Logger::iteration_complete>(
             this, total_iter, residual.get(), dense_x, residual_norm.get());
-        if (stop_criterion->update()
-                .num_iterations(total_iter)
-                .residual(residual.get())
-                .residual_norm(residual_norm.get())
-                .solution(dense_x)
-                .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
+        if (stop_already_encountered && forced_iterations < forced_limit) {
+            ++forced_iterations;
+        } else if (stop_criterion->update()
+                       .num_iterations(total_iter)
+                       .residual(residual.get())
+                       .residual_norm(residual_norm.get())
+                       .solution(dense_x)
+                       .check(RelativeStoppingId, true, &stop_status,
+                              &one_changed)) {
             if (stop_already_encountered) {
+                // forced_iterations = 0;
                 break;
             }
             stop_already_encountered = true;
@@ -211,21 +220,25 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
             bool host_array_changed{false};
             for (size_type i = 0; i < host_stop_status.get_num_elems(); ++i) {
                 auto local_status = host_stop_status.get_data() + i;
+                // TODO: ignore all actually converged ones!
                 if (local_status->has_converged()) {
                     local_status->reset();
                     host_array_changed = true;
                 }
             }
             if (host_array_changed) {
+                perform_reset = true;
                 stop_status = host_stop_status;
             } else {
                 break;
             }
         } else {
+            forced_iterations = 0;
             stop_already_encountered = false;
         }
 
-        if (stop_already_encountered || restart_iter == krylov_dim_) {
+        if (perform_reset || restart_iter == krylov_dim_) {
+            perform_reset = false;
 #ifdef TIMING_STEPS
             exec->synchronize();
             auto t_aux_0 = std::chrono::steady_clock::now();

From ed2d55e0cc8ac3545805acfcc0c85967bb45389d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 26 Jun 2020 13:38:02 +0200
Subject: [PATCH 0368/1008] Add debug output to forced iterations

---
 core/solver/gmres_mixed.cpp | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 9915a8301ce..99ef281fd6b 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <iostream>
+#include <typeinfo>
 
 
 #include "core/solver/gmres_mixed_accessor.hpp"
@@ -75,6 +76,24 @@ GKO_REGISTER_OPERATION(step_2, gmres_mixed::step_2);
 
 }  // namespace gmres_mixed
 
+// TODO: Remove output
+template <typename T>
+struct type_string {
+    static const char *get() { return typeid(T).name(); }
+};
+
+#define GKO_SPECIALIZE_TYPE_STRING(type)           \
+    template <>                                    \
+    struct type_string<type> {                     \
+        static const char *get() { return #type; } \
+    }
+
+GKO_SPECIALIZE_TYPE_STRING(int32);
+GKO_SPECIALIZE_TYPE_STRING(int16);
+GKO_SPECIALIZE_TYPE_STRING(double);
+GKO_SPECIALIZE_TYPE_STRING(float);
+GKO_SPECIALIZE_TYPE_STRING(half);
+
 
 template <typename ValueType, typename ValueTypeKrylovBases>
 void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
@@ -194,6 +213,8 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     bool perform_reset{false};
     decltype(krylov_dim_) forced_iterations{0};
     const decltype(forced_iterations) forced_limit{krylov_dim_ / 10};
+    decltype(krylov_dim_) total_checks{0};  // TODO: Remove debug output
+    const char *type_str = type_string<ValueTypeKrylovBases>::get();
     // TODO: take care of multiple RHS. Currently, we restart everything,
     //       even though a lot of parts might have already converged!
     //       Use `one_changed` to take care of that!
@@ -215,6 +236,9 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
                 break;
             }
             stop_already_encountered = true;
+            std::cout << type_str << ": " << ++total_checks
+                      << ". check in iteration " << total_iter << "; "
+                      << forced_iterations << " / " << forced_limit << '\n';
             Array<stopping_status> host_stop_status(
                 this->get_executor()->get_master(), stop_status);
             bool host_array_changed{false};
@@ -373,6 +397,9 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
 
         restart_iter++;
     }
+    std::cout << type_str << ": " << total_checks << ": exiting in iteration "
+              << total_iter << "; " << forced_iterations << " / "
+              << forced_limit << '\n';
 
     // Solve x
 #ifdef TIMING_STEPS

From 5a09d0788ab40ef0a85030ba112cfd5f1524fd1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 8 Jul 2020 17:03:43 +0200
Subject: [PATCH 0369/1008] Fix reference bug in GmresMixed

---
 core/solver/gmres_mixed.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 99ef281fd6b..5c97b6b3187 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -150,9 +150,8 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
     // TODO: write description what the different rows represent
     // The optional entry stores the infinity_norm of each next_krylov_vector,
     // which is only used to compute the scale
-    auto arnoldi_norm = VectorNorms::create(
-        exec, dim<2>{2 + Accessor3dHelper::Accessor::has_scale,
-                     dense_b->get_size()[1]});
+    auto arnoldi_norm =
+        VectorNorms::create(exec, dim<2>{3, dense_b->get_size()[1]});
     Array<size_type> final_iter_nums(this->get_executor(),
                                      dense_b->get_size()[1]);
     auto y = Vector::create(exec, dim<2>{krylov_dim_, dense_b->get_size()[1]});

From 83bbee2d23bcbc0018c65f195d70b828e9a36a83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 30 Jul 2020 09:38:16 +0200
Subject: [PATCH 0370/1008] DEBUG: Add write output for integral accessor

---
 core/solver/gmres_mixed_accessor.hpp          | 35 ++++++++++++++-----
 core/test/solver/gmres_mixed.cpp              |  7 +++-
 reference/test/solver/gmres_mixed_kernels.cpp | 30 ++++++++++++++++
 3 files changed, 63 insertions(+), 9 deletions(-)

diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index 2641f177876..0abe1a6f8b1 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -66,15 +66,18 @@ template <typename StorageType, typename ArithmeticType,
                   !std::is_same<StorageType, ArithmeticType>::value) ||
                  std::is_integral<StorageType>::value>
 */
-struct helper_have_scale {};
+struct helper_have_scale {
+};
 
 template <typename StorageType, typename ArithmeticType>
 struct helper_have_scale<StorageType, ArithmeticType, false>
-    : public std::false_type {};
+    : public std::false_type {
+};
 
 template <typename StorageType, typename ArithmeticType>
 struct helper_have_scale<StorageType, ArithmeticType, true>
-    : public std::true_type {};
+    : public std::true_type {
+};
 
 
 }  // namespace detail
@@ -82,7 +85,8 @@ struct helper_have_scale<StorageType, ArithmeticType, true>
 
 template <typename StorageType, typename ArithmeticType,
           bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
-class Accessor3dConst {};
+class Accessor3dConst {
+};
 /**
  * @internal
  *
@@ -253,7 +257,8 @@ class Accessor3dConst<StorageType, ArithmeticType, true> {
 
 template <typename StorageType, typename ArithmeticType,
           bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
-class Accessor3d : public Accessor3dConst<StorageType, ArithmeticType> {};
+class Accessor3d : public Accessor3dConst<StorageType, ArithmeticType> {
+};
 
 
 /**
@@ -365,6 +370,12 @@ class Accessor3d<StorageType, ArithmeticType, true>
         const auto stride1 = this->stride_[1];
         rest_storage[x * stride0 + y * stride1 + z] =
             static_cast<storage_type>(value / this->read_scale(x, z));
+#if not defined(__CUDA_ARCH__)
+        std::cout << "storage[" << x << ", " << y << ", " << z
+                  << "] = " << rest_storage[x * stride0 + y * stride1 + z]
+                  << "; value = " << value
+                  << "; scale = " << this->read_scale(x, z) << '\n';
+#endif
     }
 
     /**
@@ -376,13 +387,19 @@ class Accessor3d<StorageType, ArithmeticType, true>
         arithmetic_type *GKO_RESTRICT rest_scale = this->scale_;
         storage_type max_val = one<storage_type>();
         // printf("(A) max_val = %d\n", max_val);
-        // if (std::is_integer<storage_type>::value) {
         if (std::is_integral<storage_type>::value) {
             max_val = std::numeric_limits<storage_type>::max();
             //    printf("(B) max_val = %ld\n", max_val);
         }
         rest_scale[x * this->stride_[1] + z] =
             val / static_cast<arithmetic_type>(max_val);
+#if not defined(__CUDA_ARCH__)
+        std::cout << "scale[" << x << ", " << z
+                  << "] = " << rest_scale[x * this->stride_[1] + z]
+                  << "; val = " << val
+                  << "; max_val = " << static_cast<arithmetic_type>(max_val)
+                  << '\n';
+#endif
         //    rest_scale[idx] = one<arithmetic_type>();
         //    printf("val = %10.5e , rest_scale = %10.5e\n", val,
         //    rest_scale[idx]); std::cout << val << " - " << rest_scale[idx] <<
@@ -397,7 +414,8 @@ class Accessor3d<StorageType, ArithmeticType, true>
 
 template <typename ValueType, typename ValueTypeKrylovBases,
           bool = Accessor3d<ValueTypeKrylovBases, ValueType>::has_scale>
-class Accessor3dHelper {};
+class Accessor3dHelper {
+};
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
@@ -473,7 +491,8 @@ class Accessor3dHelper<ValueType, ValueTypeKrylovBases, false> {
 
 template <typename ValueType, typename KrylovType,
           bool = Accessor3d<KrylovType, ValueType>::has_scale>
-struct helper_functions_accessor {};
+struct helper_functions_accessor {
+};
 
 // Accessors having a scale
 template <typename ValueType, typename KrylovType>
diff --git a/core/test/solver/gmres_mixed.cpp b/core/test/solver/gmres_mixed.cpp
index 42db208ba4d..ff22f71956f 100644
--- a/core/test/solver/gmres_mixed.cpp
+++ b/core/test/solver/gmres_mixed.cpp
@@ -158,7 +158,12 @@ TEST_F(GmresMixed, CanSetPreconditionerGenerator)
                 gko::stop::ResidualNormReduction<>::build()
                     .with_reduction_factor(1e-6)
                     .on(exec))
-            .with_preconditioner(Solver::build().on(exec))
+            .with_preconditioner(
+                Solver::build()
+                    .with_criteria(
+                        gko::stop::Iteration::build().with_max_iters(3u).on(
+                            exec))
+                    .on(exec))
             .on(exec);
     auto solver = gmres_mixed_factory->generate(mtx);
     auto precond = dynamic_cast<const gko::solver::GmresMixed<> *>(
diff --git a/reference/test/solver/gmres_mixed_kernels.cpp b/reference/test/solver/gmres_mixed_kernels.cpp
index abd1c0e767d..54206435f1a 100644
--- a/reference/test/solver/gmres_mixed_kernels.cpp
+++ b/reference/test/solver/gmres_mixed_kernels.cpp
@@ -66,6 +66,8 @@ class GmresMixed : public ::testing::Test {
         : exec(gko::ReferenceExecutor::create()),
           mtx(gko::initialize<Mtx>(
               {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)),
+          mtx2(gko::initialize<Mtx>(
+              {{1.0, 2.0, 3.0}, {4.0, 2.0, 1.0}, {0.0, 1.0, 2.0}}, exec)),
           gmres_mixed_factory(
               gmres_type::build()
                   .with_criteria(
@@ -106,6 +108,7 @@ class GmresMixed : public ::testing::Test {
 
     std::shared_ptr<const gko::Executor> exec;
     std::shared_ptr<Mtx> mtx;
+    std::shared_ptr<Mtx> mtx2;
     std::shared_ptr<Mtx> mtx_medium;
     std::shared_ptr<Mtx> mtx_big;
     std::unique_ptr<typename gmres_type::Factory> gmres_mixed_factory;
@@ -140,6 +143,33 @@ TYPED_TEST(GmresMixed, SolvesStencilSystem)
 }
 
 
+TYPED_TEST(GmresMixed, SolvesStencilSystem2)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    using gmres_type = typename TestFixture::gmres_type;
+    auto factory =
+        gmres_type::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(100u).on(
+                    this->exec),
+                gko::stop::Time::build()
+                    .with_time_limit(std::chrono::seconds(6))
+                    .on(this->exec),
+                gko::stop::ResidualNormReduction<T>::build()
+                    .with_reduction_factor(r<T>::value)
+                    .on(this->exec))
+            .on(this->exec);
+    auto solver = factory->generate(this->mtx2);
+    auto b = gko::initialize<Mtx>({33.0, 20.0, 20.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 4.0, 8.0}), r<T>::value * 1e1);
+}
+
+
 TYPED_TEST(GmresMixed, SolvesMultipleStencilSystems)
 {
     using Mtx = typename TestFixture::Mtx;

From 5af7f65fb21de666f6f2d4750434d01531ff3bb0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 14 Aug 2020 18:57:36 +0200
Subject: [PATCH 0371/1008] DEBUG: Move towards `at` with accessor

---
 common/solver/gmres_mixed_kernels.hpp.inc     | 136 +++++-------
 core/solver/gmres_mixed.cpp                   |   4 +-
 core/solver/gmres_mixed_accessor.hpp          | 196 +++++++++++++++---
 core/solver/gmres_mixed_kernels.hpp           |  59 ++++--
 cuda/solver/gmres_mixed_kernels.cu            |  70 +++----
 reference/solver/gmres_mixed_kernels.cpp      | 137 ++++++------
 reference/test/solver/gmres_mixed_kernels.cpp |  18 +-
 7 files changed, 385 insertions(+), 235 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 17391c668ed..822efca448f 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -81,12 +81,10 @@ __global__ __launch_bounds__(block_size) void initialize_1_kernel(
 
 
 // Must be called with at least `num_rows * stride_krylov` threads in total.
-template <size_type block_size, typename ValueType,
-          typename ValueTypeKrylovBases>
+template <size_type block_size, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(block_size) void initialize_2_1_kernel(
     size_type num_rows, size_type num_rhs, size_type krylov_dim,
-    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
-    ValueType *__restrict__ residual_norm_collection,
+    Accessor3d krylov_bases, ValueType *__restrict__ residual_norm_collection,
     size_type stride_residual_nc)
 {
     const auto global_id = thread::get_thread_id_flat();
@@ -112,14 +110,12 @@ __global__ __launch_bounds__(block_size) void initialize_2_1_kernel(
 
 
 // Must be called with at least `num_rows * num_rhs` threads in total.
-template <size_type block_size, typename ValueType,
-          typename ValueTypeKrylovBases>
+template <size_type block_size, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(block_size) void initialize_2_2_kernel(
     size_type num_rows, size_type num_rhs,
     const ValueType *__restrict__ residual, size_type stride_residual,
     const remove_complex<ValueType> *__restrict__ residual_norm,
-    ValueType *__restrict__ residual_norm_collection,
-    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    ValueType *__restrict__ residual_norm_collection, Accessor3d krylov_bases,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
     size_type *__restrict__ final_iter_nums)
 {
@@ -375,12 +371,11 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_inf_kernel(
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3dConst>
 __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
     size_type k, size_type num_rows, size_type num_rhs,
     const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov,
-    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    size_type stride_next_krylov, Accessor3dConst krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -424,12 +419,11 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
     size_type num_iters, size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov,
-    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    size_type stride_next_krylov, Accessor3dConst krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -475,12 +469,11 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3dConst>
 __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_1(
     size_type num_iters, size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov,
-    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    size_type stride_next_krylov, Accessor3dConst krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -550,12 +543,11 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_1(
     //    }
 }
 
-template <int dot_dim, typename ValueType, typename ValueTypeKrylovBases>
+template <int dot_dim, typename ValueType, typename Accessor3dConst>
 __global__ __launch_bounds__(dot_dim *dot_dim) void multidot_kernel_num_iters_2(
     size_type num_iters, size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov,
-    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    size_type stride_next_krylov, Accessor3dConst krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -610,12 +602,11 @@ __global__ __launch_bounds__(dot_dim *dot_dim) void multidot_kernel_num_iters_2(
 }
 
 
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3dConst>
 __global__ __launch_bounds__(block_size) void singledot_kernel_num_iters_2(
     size_type num_iters, size_type num_rows,
     const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov,
-    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    size_type stride_next_krylov, Accessor3dConst krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -663,12 +654,12 @@ __global__ __launch_bounds__(block_size) void singledot_kernel_num_iters_2(
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3dConst>
 __global__ __launch_bounds__(block_size) void update_next_krylov_kernel(
     size_type k, size_type num_rows, size_type num_cols,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
-    const ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    Accessor3dConst krylov_bases, const ValueType *__restrict__ hessenberg_iter,
+    size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
     const auto global_id = thread::get_thread_id_flat();
@@ -689,12 +680,12 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel(
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3dConst>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        Accessor3dConst krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -721,12 +712,12 @@ __global__
 
 // Must be called with at least `num_vectors * krylov_bases.get_stride0()`
 // threads in total.
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3dConst>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_0(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        Accessor3dConst krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -753,12 +744,12 @@ __global__
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3dConst>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_1(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        Accessor3dConst krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -817,13 +808,12 @@ __global__
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3dConst>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_2(
         size_type num_iters, size_type num_blocks, size_type num_rows,
         size_type num_cols, ValueType *__restrict__ next_krylov_basis,
-        size_type stride_next_krylov,
-        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        size_type stride_next_krylov, Accessor3dConst krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -899,13 +889,12 @@ __global__
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3dConst>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_3(
         size_type num_iters, size_type num_blocks, size_type num_rows,
         size_type num_cols, ValueType *__restrict__ next_krylov_basis,
-        size_type stride_next_krylov,
-        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        size_type stride_next_krylov, Accessor3dConst krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -986,13 +975,12 @@ __global__
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3dConst>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_4(
         size_type num_iters, size_type num_blocks, size_type num_rows,
         size_type num_cols, ValueType *__restrict__ next_krylov_basis,
-        size_type stride_next_krylov,
-        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        size_type stride_next_krylov, Accessor3dConst krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -1064,12 +1052,12 @@ __global__
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3dConst>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_and_check(
         size_type k, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+        Accessor3dConst krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const remove_complex<ValueType> *__restrict__ arnoldi_norm,
@@ -1106,14 +1094,13 @@ __global__
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3dConst>
 __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
     size_type k, size_type num_rows, size_type num_cols,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
-    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
-    const ValueType *__restrict__ buffer_iter, size_type stride_buffer,
-    const stopping_status *__restrict__ stop_status,
+    Accessor3dConst krylov_bases, ValueType *__restrict__ hessenberg_iter,
+    size_type stride_hessenberg, const ValueType *__restrict__ buffer_iter,
+    size_type stride_buffer, const stopping_status *__restrict__ stop_status,
     const stopping_status *__restrict__ reorth_status)
 {
     const auto global_id = thread::get_thread_id_flat();
@@ -1137,14 +1124,14 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3dConst>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_and_add(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
-        ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
-        const ValueType *__restrict__ buffer_iter, size_type stride_buffer,
+        Accessor3dConst krylov_bases, ValueType *__restrict__ hessenberg_iter,
+        size_type stride_hessenberg, const ValueType *__restrict__ buffer_iter,
+        size_type stride_buffer,
         const stopping_status *__restrict__ stop_status,
         const stopping_status *__restrict__ reorth_status)
 {
@@ -1173,14 +1160,14 @@ __global__
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3dConst>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_and_add_1(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
-        ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
-        const ValueType *__restrict__ buffer_iter, size_type stride_buffer,
+        Accessor3dConst krylov_bases, ValueType *__restrict__ hessenberg_iter,
+        size_type stride_hessenberg, const ValueType *__restrict__ buffer_iter,
+        size_type stride_buffer,
         const stopping_status *__restrict__ stop_status,
         const stopping_status *__restrict__ reorth_status)
 {
@@ -1277,17 +1264,15 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms(
 
 // Must be called with at least `num_cols` blocks, each with `block_size`
 // threads. `block_size` must be a power of 2.
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
     remove_complex<ValueType> *__restrict__ arnoldi_norm, size_type stride_norm,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
-    size_type iter, Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    size_type iter, Accessor3d krylov_bases,
     const stopping_status *__restrict__ stop_status,
     stopping_status *__restrict__ reorth_status,
     size_type *__restrict__ num_reorth)
 {
-    constexpr bool has_scale =
-        Accessor3d<ValueTypeKrylovBases, ValueType>::has_scale;
     const remove_complex<ValueType> eta = 1.0 / 2.0;
     const auto col_idx = thread::get_thread_id_flat();
 
@@ -1314,7 +1299,7 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
         // printf("(%d) num1 = %10.5e , num2 = %10.5e\n", cod_idx, num1, num2);
         // printf("%d - %d - %d - %d\n", stride_norm, iter, col_idx,
         //        stride_norm * iter + col_idx);
-        helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
+        helper_functions_accessor<Accessor3d>::write_scale(
             krylov_bases, iter, col_idx, num2 / num12);
         // krylov_bases, num_cols * (iter + 1) + col_idx, num2 / num12);
         // krylov_bases, col_idx, num2 / num12);
@@ -1324,14 +1309,13 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
 
 // Must be called with at least `num_cols` blocks, each with `block_size`
 // threads. `block_size` must be a power of 2.
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(block_size) void set_scale_kernel(
     size_type num_rhs, size_type num_blocks,
     const remove_complex<ValueType> *__restrict__ residual_norm,
     size_type stride_residual,
     const remove_complex<ValueType> *__restrict__ arnoldi_inf,
-    size_type stride_inf,
-    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases)
+    size_type stride_inf, Accessor3d krylov_bases)
 {
     const auto global_id = thread::get_thread_id_flat();
     const auto blk_idx = global_id / krylov_bases.get_stride1();
@@ -1342,16 +1326,12 @@ __global__ __launch_bounds__(block_size) void set_scale_kernel(
             const auto num1 = residual_norm[col_idx];
             const auto num2 = arnoldi_inf[col_idx];
             // printf("num1 = %10.5e , num2 = %10.5e\n", num1, num2);
-            helper_functions_accessor<
-                ValueType, ValueTypeKrylovBases>::write_scale(krylov_bases, {0},
-                                                              col_idx,
-                                                              num2 / num1);
+            helper_functions_accessor<Accessor3d>::write_scale(
+                krylov_bases, {0}, col_idx, num2 / num1);
         } else {
             const auto num = one<remove_complex<ValueType>>();
-            helper_functions_accessor<
-                ValueType, ValueTypeKrylovBases>::write_scale(krylov_bases,
-                                                              blk_idx, col_idx,
-                                                              num);
+            helper_functions_accessor<Accessor3d>::write_scale(
+                krylov_bases, blk_idx, col_idx, num);
         }
     }
 }
@@ -1401,12 +1381,12 @@ __global__ __launch_bounds__(block_size) void update_hessenberg_2_kernel(
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename ValueTypeKrylovBases>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(block_size) void update_krylov_next_krylov_kernel(
     size_type iter, size_type num_rows, size_type num_cols,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
-    const ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    Accessor3d krylov_bases, const ValueType *__restrict__ hessenberg_iter,
+    size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
     const auto global_id = thread::get_thread_id_flat();
@@ -1584,11 +1564,9 @@ __global__ __launch_bounds__(block_size) void solve_upper_triangular_kernel(
 
 // Must be called with at least `stride_preconditioner * num_rows` threads
 // in total.
-template <size_type block_size, typename ValueType,
-          typename ValueTypeKrylovBases>
+template <size_type block_size, typename ValueType, typename Accessor3dConst>
 __global__ __launch_bounds__(block_size) void calculate_Qy_kernel(
-    size_type num_rows, size_type num_cols,
-    Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+    size_type num_rows, size_type num_cols, Accessor3dConst krylov_bases,
     const ValueType *__restrict__ y, size_type stride_y,
     ValueType *__restrict__ before_preconditioner,
     size_type stride_preconditioner,
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 5c97b6b3187..c7920afe628 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -235,9 +235,11 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
                 break;
             }
             stop_already_encountered = true;
+            /*
             std::cout << type_str << ": " << ++total_checks
                       << ". check in iteration " << total_iter << "; "
                       << forced_iterations << " / " << forced_limit << '\n';
+            */
             Array<stopping_status> host_stop_status(
                 this->get_executor()->get_master(), stop_status);
             bool host_array_changed{false};
@@ -510,7 +512,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(
 GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(GKO_DECLARE_GMRES_MIXED_BOOL);
 */
 #define GKO_DECLARE_GMRES_MIXED(_type1, _type2) class GmresMixed<_type1, _type2>
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(GKO_DECLARE_GMRES_MIXED);
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE2(GKO_DECLARE_GMRES_MIXED);
 
 
 }  // namespace solver
diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index 0abe1a6f8b1..2f71973c245 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -66,18 +66,15 @@ template <typename StorageType, typename ArithmeticType,
                   !std::is_same<StorageType, ArithmeticType>::value) ||
                  std::is_integral<StorageType>::value>
 */
-struct helper_have_scale {
-};
+struct helper_have_scale {};
 
 template <typename StorageType, typename ArithmeticType>
 struct helper_have_scale<StorageType, ArithmeticType, false>
-    : public std::false_type {
-};
+    : public std::false_type {};
 
 template <typename StorageType, typename ArithmeticType>
 struct helper_have_scale<StorageType, ArithmeticType, true>
-    : public std::true_type {
-};
+    : public std::true_type {};
 
 
 }  // namespace detail
@@ -85,8 +82,7 @@ struct helper_have_scale<StorageType, ArithmeticType, true>
 
 template <typename StorageType, typename ArithmeticType,
           bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
-class Accessor3dConst {
-};
+class Accessor3dConst {};
 /**
  * @internal
  *
@@ -112,7 +108,33 @@ class Accessor3dConst<StorageType, ArithmeticType, false> {
         "storage_type must not be an integral in this class.");
 
     static constexpr bool has_scale{false};
+    using const_type = Accessor3dConst;
+
+protected:
+    class reference {
+    public:
+        reference() = delete;
+        reference(reference &&) = delete;
+        reference(const reference &) = delete;
+        reference &operator=(const reference &) = delete;
+        reference &operator=(reference &&) = delete;
+
+        reference(storage_type *const GKO_RESTRICT ptr) : ptr_{ptr} {}
+        operator arithmetic_type() const
+        {
+            return static_cast<arithmetic_type>(*ptr_);
+        }
+        arithmetic_type operator=(arithmetic_type val) &&
+        {
+            *ptr_ = static_cast<storage_type>(val);
+            return val;
+        }
+
+    private:
+        storage_type *const GKO_RESTRICT ptr_;
+    };
 
+public:
     /**
      * Creates the accessor with an already allocated storage space with a
      * stride.
@@ -128,6 +150,18 @@ class Accessor3dConst<StorageType, ArithmeticType, false> {
           stride_{stride0, stride1}
     {}
 
+    /**
+     * Reads the value at the given index.
+     *
+     * @internal The const is important since it prevents writes
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES const reference at(IndexType x, IndexType y,
+                                      IndexType z) const
+    {
+        return {storage_ + x * stride_[0] + y * stride_[1] + z};
+    }
+
     /**
      * Reads the value at the given index.
      */
@@ -185,7 +219,63 @@ class Accessor3dConst<StorageType, ArithmeticType, true> {
                   "storage_type must not be an integral in this class.");
 
     static constexpr bool has_scale{true};
+    using const_type = Accessor3dConst;
+
+protected:
+    class scale_reference {
+    public:
+        scale_reference() = delete;
+        scale_reference(scale_reference &&) = delete;
+        scale_reference(const scale_reference &) = delete;
+        scale_reference &operator=(const scale_reference &) = delete;
+        scale_reference &operator=(scale_reference &&) = delete;
+
+        scale_reference(arithmetic_type *const GKO_RESTRICT scale)
+            : scale_{scale}
+        {}
+        operator arithmetic_type() const { return *scale_; }
+        arithmetic_type operator=(arithmetic_type val) &&
+        {
+            storage_type max_val = one<storage_type>();
+            if (std::is_integral<storage_type>::value) {
+                max_val = std::numeric_limits<storage_type>::max();
+            }
+            *scale_ = val / static_cast<arithmetic_type>(max_val);
+            return val;
+        }
+
+    private:
+        arithmetic_type *const GKO_RESTRICT scale_;
+    };
+
+    class value_reference {
+    public:
+        value_reference() = delete;
+        value_reference(value_reference &&) = delete;
+        value_reference(const value_reference &) = delete;
+        value_reference &operator=(const value_reference &) = delete;
+        value_reference &operator=(value_reference &&) = delete;
+
+        value_reference(storage_type *const GKO_RESTRICT value,
+                        arithmetic_type scale)
+            : value_{value}, scale_{scale}
+        {}
+        operator arithmetic_type() const
+        {
+            return static_cast<arithmetic_type>(*value_) * scale_;
+        }
+        arithmetic_type operator=(arithmetic_type val) &&
+        {
+            *value_ = static_cast<storage_type>(val / scale_);
+            return val;
+        }
 
+    private:
+        storage_type *const GKO_RESTRICT value_;
+        const arithmetic_type scale_;
+    };
+
+public:
     /**
      * Creates the accessor with an already allocated storage space with a
      * stride.
@@ -202,6 +292,20 @@ class Accessor3dConst<StorageType, ArithmeticType, true> {
           scale_{const_cast<arithmetic_type *>(scale)}
     {}
 
+    template <typename IndexType>
+    GKO_ATTRIBUTES const scale_reference scale_at(IndexType x,
+                                                  IndexType z) const
+    {
+        return {scale_ + x * stride_[1] + z};
+    }
+
+    template <typename IndexType>
+    GKO_ATTRIBUTES const value_reference at(IndexType x, IndexType y,
+                                            IndexType z) const
+    {
+        return {storage_ + x * stride_[0] + y * stride_[1] + z,
+                read_scale(x, z)};
+    }
     /**
      * Reads the scale value at the given indices.
      */
@@ -257,8 +361,7 @@ class Accessor3dConst<StorageType, ArithmeticType, true> {
 
 template <typename StorageType, typename ArithmeticType,
           bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
-class Accessor3d : public Accessor3dConst<StorageType, ArithmeticType> {
-};
+class Accessor3d : public Accessor3dConst<StorageType, ArithmeticType> {};
 
 
 /**
@@ -281,8 +384,10 @@ class Accessor3d<StorageType, ArithmeticType, false>
 
 private:
     using Accessor3dC = Accessor3dConst<storage_type, arithmetic_type>;
+    using reference = typename Accessor3dC::reference;
 
 public:
+    using const_type = Accessor3dC;
     /**
      * Creates an empty accessor pointing to a nullptr.
      */
@@ -301,6 +406,13 @@ class Accessor3d<StorageType, ArithmeticType, false>
         return {this->storage_, this->stride_[0], this->stride_[1]};
     }
 
+
+    template <typename IndexType>
+    GKO_ATTRIBUTES reference at(IndexType x, IndexType y, IndexType z)
+    {
+        return {this->storage_ + x * this->stride_[0] + y * this->stride_[1] +
+                z};
+    }
     /**
      * Writes the given value at the given index.
      */
@@ -336,8 +448,11 @@ class Accessor3d<StorageType, ArithmeticType, true>
 
 private:
     using Accessor3dC = Accessor3dConst<storage_type, arithmetic_type>;
+    using scale_reference = typename Accessor3dC::scale_reference;
+    using value_reference = typename Accessor3dC::value_reference;
 
 public:
+    using const_type = Accessor3dC;
     /**
      * Creates an empty accessor pointing to a nullptr.
      */
@@ -358,6 +473,20 @@ class Accessor3d<StorageType, ArithmeticType, true>
                 this->scale_};
     }
 
+    template <typename IndexType>
+    GKO_ATTRIBUTES scale_reference scale_at(IndexType x, IndexType z)
+    {
+        return {this->scale_ + x * this->stride_[1] + z};
+    }
+
+    template <typename IndexType>
+    GKO_ATTRIBUTES value_reference at(IndexType x, IndexType y, IndexType z)
+    {
+        return {
+            this->storage_ + x * this->stride_[0] + y * this->stride_[1] + z,
+            this->read_scale(x, z)};
+    }
+
     /**
      * Writes the given value at the given index.
      */
@@ -370,7 +499,7 @@ class Accessor3d<StorageType, ArithmeticType, true>
         const auto stride1 = this->stride_[1];
         rest_storage[x * stride0 + y * stride1 + z] =
             static_cast<storage_type>(value / this->read_scale(x, z));
-#if not defined(__CUDA_ARCH__)
+#if false && not defined(__CUDA_ARCH__)
         std::cout << "storage[" << x << ", " << y << ", " << z
                   << "] = " << rest_storage[x * stride0 + y * stride1 + z]
                   << "; value = " << value
@@ -393,7 +522,7 @@ class Accessor3d<StorageType, ArithmeticType, true>
         }
         rest_scale[x * this->stride_[1] + z] =
             val / static_cast<arithmetic_type>(max_val);
-#if not defined(__CUDA_ARCH__)
+#if false && not defined(__CUDA_ARCH__)
         std::cout << "scale[" << x << ", " << z
                   << "] = " << rest_scale[x * this->stride_[1] + z]
                   << "; val = " << val
@@ -414,8 +543,7 @@ class Accessor3d<StorageType, ArithmeticType, true>
 
 template <typename ValueType, typename ValueTypeKrylovBases,
           bool = Accessor3d<ValueTypeKrylovBases, ValueType>::has_scale>
-class Accessor3dHelper {
-};
+class Accessor3dHelper {};
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
@@ -489,37 +617,43 @@ class Accessor3dHelper<ValueType, ValueTypeKrylovBases, false> {
 
 //----------------------------------------------
 
-template <typename ValueType, typename KrylovType,
-          bool = Accessor3d<KrylovType, ValueType>::has_scale>
-struct helper_functions_accessor {
-};
+template <typename Accessor3d, bool = Accessor3d::has_scale>
+struct helper_functions_accessor {};
 
 // Accessors having a scale
-template <typename ValueType, typename KrylovType>
-struct helper_functions_accessor<ValueType, KrylovType, true> {
+template <typename Accessor3d>
+struct helper_functions_accessor<Accessor3d, true> {
+    using value_type = typename Accessor3d::arithmetic_type;
+    static_assert(Accessor3d::has_scale, "Accessor must have a scale here!");
     template <typename IndexType>
-    static inline GKO_ATTRIBUTES void write_scale(
-        Accessor3d<KrylovType, ValueType> krylov_bases, IndexType vector_idx,
-        IndexType col_idx, ValueType value)
+    static inline GKO_ATTRIBUTES void write_scale(Accessor3d krylov_bases,
+                                                  IndexType vector_idx,
+                                                  IndexType col_idx,
+                                                  value_type value)
     {
-        krylov_bases.set_scale(vector_idx, col_idx, value);
+        // krylov_bases.set_scale(vector_idx, col_idx, value);
+        krylov_bases.scale_at(vector_idx, col_idx) = value;
     }
 };
 
 
 // Accessors not having a scale
-template <typename ValueType, typename KrylovType>
-struct helper_functions_accessor<ValueType, KrylovType, false> {
+template <typename Accessor3d>
+struct helper_functions_accessor<Accessor3d, false> {
+    using value_type = typename Accessor3d::arithmetic_type;
+    static_assert(!Accessor3d::has_scale,
+                  "Accessor must not have a scale here!");
     template <typename IndexType>
-    static inline GKO_ATTRIBUTES void write_scale(
-        Accessor3d<KrylovType, ValueType> krylov_bases, IndexType vector_idx,
-        IndexType col_idx, ValueType value)
+    static inline GKO_ATTRIBUTES void write_scale(Accessor3d krylov_bases,
+                                                  IndexType vector_idx,
+                                                  IndexType col_idx,
+                                                  value_type value)
     {}
 };
 
 // calling it with:
-// helper_functions_accessor<ValueType,
-// ValueTypeKrylovBases>::write_scale(krylov_bases, col_idx, value);
+// helper_functions_accessor<Accessor3d>::write_scale(krylov_bases, col_idx,
+// value);
 
 //----------------------------------------------
 
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index cfc83636a8f..b113dcfd967 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -65,19 +65,40 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 //     template _macro(double, float);                 \
 //     template _macro(double, double)
 // #else
-#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro)        \
+#define GKO_UNPACK(...) __VA_ARGS__
+#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE2(_macro)       \
     template _macro(double, double);                             \
     template _macro(double, float);                              \
     template _macro(double, int64);                              \
     template _macro(double, int32);                              \
     template _macro(double, int16);                              \
     template _macro(double, half);                               \
-    template _macro(float, float);                               \
-    template _macro(float, half);                                \
+    template _macro(float, float, float >);                      \
+    template _macro(float, half, float >);                       \
     template _macro(std::complex<double>, std::complex<double>); \
     template _macro(std::complex<double>, std::complex<float>);  \
     template _macro(std::complex<float>, std::complex<float>)
+
+#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro)                    \
+    template _macro(double, GKO_UNPACK(Accessor3d<double, double>));         \
+    template _macro(double, GKO_UNPACK(Accessor3d<float, double>));          \
+    template _macro(double, GKO_UNPACK(Accessor3d<int64, double>));          \
+    template _macro(double, GKO_UNPACK(Accessor3d<int32, double>));          \
+    template _macro(double, GKO_UNPACK(Accessor3d<int16, double>));          \
+    template _macro(double, GKO_UNPACK(Accessor3d<half, double>));           \
+    template _macro(float, GKO_UNPACK(Accessor3d<float, float>));            \
+    template _macro(float, GKO_UNPACK(Accessor3d<half, float>));             \
+    template _macro(                                                         \
+        std::complex<double>,                                                \
+        GKO_UNPACK(Accessor3d<std::complex<double>, std::complex<double>>)); \
+    template _macro(                                                         \
+        std::complex<double>,                                                \
+        GKO_UNPACK(Accessor3d<std::complex<float>, std::complex<double>>));  \
+    template _macro(                                                         \
+        std::complex<float>,                                                 \
+        GKO_UNPACK(Accessor3d<std::complex<float>, std::complex<float>>))
 // #endif
+// #undef GKO_UNPACK
 
 
 namespace gko {
@@ -95,26 +116,25 @@ namespace gmres_mixed {
         size_type krylov_dim)
 
 
-#define GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(_type1, _type2)         \
+#define GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(_type1, _accessor)      \
     void initialize_2(std::shared_ptr<const DefaultExecutor> exec,          \
                       const matrix::Dense<_type1> *residual,                \
                       matrix::Dense<remove_complex<_type1>> *residual_norm, \
                       matrix::Dense<_type1> *residual_norm_collection,      \
                       matrix::Dense<remove_complex<_type1>> *arnoldi_norm,  \
-                      Accessor3d<_type2, _type1> krylov_bases,              \
+                      _accessor krylov_bases,                               \
                       matrix::Dense<_type1> *next_krylov_basis,             \
                       Array<size_type> *final_iter_nums, size_type krylov_dim)
 
 
-#define GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(_type1, _type2)                 \
+#define GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(_type1, _accessor)              \
     void step_1(                                                              \
         std::shared_ptr<const DefaultExecutor> exec,                          \
         matrix::Dense<_type1> *next_krylov_basis,                             \
         matrix::Dense<_type1> *givens_sin, matrix::Dense<_type1> *givens_cos, \
         matrix::Dense<remove_complex<_type1>> *residual_norm,                 \
         matrix::Dense<_type1> *residual_norm_collection,                      \
-        Accessor3d<_type2, _type1> krylov_bases,                              \
-        matrix::Dense<_type1> *hessenberg_iter,                               \
+        _accessor krylov_bases, matrix::Dense<_type1> *hessenberg_iter,       \
         matrix::Dense<_type1> *buffer_iter,                                   \
         const matrix::Dense<remove_complex<_type1>> *b_norm,                  \
         matrix::Dense<remove_complex<_type1>> *arnoldi_norm, size_type iter,  \
@@ -124,26 +144,25 @@ namespace gmres_mixed {
         int *num_reorth_steps, int *num_reorth_vectors)
 
 
-#define GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(_type1, _type2)          \
+#define GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(_type1, _accessor)       \
     void step_2(std::shared_ptr<const DefaultExecutor> exec,           \
                 const matrix::Dense<_type1> *residual_norm_collection, \
-                Accessor3dConst<_type2, _type1> krylov_bases,          \
+                _accessor /*::const_type*/ krylov_bases,               \
                 const matrix::Dense<_type1> *hessenberg,               \
                 matrix::Dense<_type1> *y,                              \
                 matrix::Dense<_type1> *before_preconditioner,          \
                 const Array<size_type> *final_iter_nums)
 
 
-#define GKO_DECLARE_ALL_AS_TEMPLATES                                        \
-    template <typename ValueType>                                           \
-    GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL(ValueType);                 \
-    template <typename ValueType, typename ValueTypeKrylovBases>            \
-    GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(ValueType,                  \
-                                                ValueTypeKrylovBases);      \
-    template <typename ValueType, typename ValueTypeKrylovBases>            \
-    GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases); \
-    template <typename ValueType, typename ValueTypeKrylovBases>            \
-    GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(ValueType, ValueTypeKrylovBases)
+#define GKO_DECLARE_ALL_AS_TEMPLATES                                    \
+    template <typename ValueType>                                       \
+    GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL(ValueType);             \
+    template <typename ValueType, typename Accessor3d>                  \
+    GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(ValueType, Accessor3d); \
+    template <typename ValueType, typename Accessor3d>                  \
+    GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(ValueType, Accessor3d);       \
+    template <typename ValueType, typename Accessor3d>                  \
+    GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(ValueType, Accessor3d)
 
 
 }  // namespace gmres_mixed
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 68c124cd2f4..c6ef53eccd9 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -168,13 +168,13 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void initialize_2(std::shared_ptr<const CudaExecutor> exec,
                   const matrix::Dense<ValueType> *residual,
                   matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
                   matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
-                  Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
+                  Accessor3d krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
 {
@@ -244,10 +244,10 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void finish_arnoldi(std::shared_ptr<const CudaExecutor> exec,
                     matrix::Dense<ValueType> *next_krylov_basis,
-                    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
+                    Accessor3d krylov_bases,
                     matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
                     const stopping_status *stop_status)
 {
@@ -304,11 +304,10 @@ void finish_arnoldi(std::shared_ptr<const CudaExecutor> exec,
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void finish_arnoldi_reorth(
     std::shared_ptr<const CudaExecutor> exec,
-    matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    matrix::Dense<ValueType> *next_krylov_basis, Accessor3d krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
     matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
@@ -425,16 +424,16 @@ void finish_arnoldi_reorth(
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_CGS(
-    std::shared_ptr<const CudaExecutor> exec,
-    matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
-    matrix::Dense<ValueType> *hessenberg_iter,
-    matrix::Dense<ValueType> *buffer_iter,
-    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
-    const stopping_status *stop_status, stopping_status *reorth_status,
-    Array<size_type> *num_reorth)
+template <typename ValueType, typename Accessor3d>
+void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
+                        matrix::Dense<ValueType> *next_krylov_basis,
+                        Accessor3d krylov_bases,
+                        matrix::Dense<ValueType> *hessenberg_iter,
+                        matrix::Dense<ValueType> *buffer_iter,
+                        matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+                        size_type iter, const stopping_status *stop_status,
+                        stopping_status *reorth_status,
+                        Array<size_type> *num_reorth)
 {
     const auto stride_next_krylov = next_krylov_basis->get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
@@ -606,17 +605,17 @@ void finish_arnoldi_CGS(
 /**/
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_CGS2(
-    std::shared_ptr<const CudaExecutor> exec,
-    matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
-    matrix::Dense<ValueType> *hessenberg_iter,
-    matrix::Dense<ValueType> *buffer_iter,
-    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
-    const stopping_status *stop_status, stopping_status *reorth_status,
-    Array<size_type> *num_reorth, int *num_reorth_steps,
-    int *num_reorth_vectors)
+template <typename ValueType, typename Accessor3d>
+void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
+                         matrix::Dense<ValueType> *next_krylov_basis,
+                         Accessor3d krylov_bases,
+                         matrix::Dense<ValueType> *hessenberg_iter,
+                         matrix::Dense<ValueType> *buffer_iter,
+                         matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+                         size_type iter, const stopping_status *stop_status,
+                         stopping_status *reorth_status,
+                         Array<size_type> *num_reorth, int *num_reorth_steps,
+                         int *num_reorth_vectors)
 {
     using non_complex = remove_complex<ValueType>;
     // optimization parameter
@@ -1032,15 +1031,14 @@ void givens_rotation(std::shared_ptr<const CudaExecutor> exec,
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void step_1(std::shared_ptr<const CudaExecutor> exec,
             matrix::Dense<ValueType> *next_krylov_basis,
             matrix::Dense<ValueType> *givens_sin,
             matrix::Dense<ValueType> *givens_cos,
             matrix::Dense<remove_complex<ValueType>> *residual_norm,
             matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
-            matrix::Dense<ValueType> *hessenberg_iter,
+            Accessor3d krylov_bases, matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
             const matrix::Dense<remove_complex<ValueType>> *b_norm,
             matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
@@ -1111,9 +1109,9 @@ void solve_upper_triangular(
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
-void calculate_qy(Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
-                  size_type num_krylov_bases, const matrix::Dense<ValueType> *y,
+template <typename ValueType, typename Accessor3dConst>
+void calculate_qy(Accessor3dConst krylov_bases, size_type num_krylov_bases,
+                  const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const Array<size_type> *final_iter_nums)
 {
@@ -1141,10 +1139,10 @@ void calculate_qy(Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3dConst>
 void step_2(std::shared_ptr<const CudaExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+            Accessor3dConst krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index eb53fe910ef..43f391f0a8b 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -59,12 +59,15 @@ namespace gmres_mixed {
 namespace {
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
-                    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
+                    Accessor3d krylov_bases,
                     matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
                     const stopping_status *stop_status)
 {
+    static_assert(
+        std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
+        "ValueType must match arithmetic_type of accessor!");
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
         if (stop_status[i].has_stopped()) {
             continue;
@@ -73,11 +76,11 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
             hessenberg_iter->at(k, i) = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
-                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
+                    next_krylov_basis->at(j, i) * krylov_bases.at(k, j, i);
             }
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
+                    hessenberg_iter->at(k, i) * krylov_bases.at(k, j, i);
             }
         }
         // for i in 1:iter
@@ -96,7 +99,7 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write((iter + 1), j, i, next_krylov_basis->at(j, i));
+            krylov_bases.at((iter + 1), j, i) = next_krylov_basis->at(j, i);
             // std::cout << next_krylov_basis->at(j, i) << " - "
             //           << krylov_bases.read(
             //                  j,
@@ -111,15 +114,17 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void finish_arnoldi_reorth(
-    matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    matrix::Dense<ValueType> *next_krylov_basis, Accessor3d krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
     matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
     const stopping_status *stop_status)
 {
+    static_assert(
+        std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
+        "ValueType must match arithmetic_type of accessor!");
     //    ValueType nrm = 0, reorth = 0;
 
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
@@ -139,11 +144,11 @@ void finish_arnoldi_reorth(
             hessenberg_iter->at(k, i) = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
-                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
+                    next_krylov_basis->at(j, i) * krylov_bases.at(k, j, i);
             }
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
+                    hessenberg_iter->at(k, i) * krylov_bases.at(k, j, i);
             }
             // if (squared_norm(hessenberg_iter->at(k, i) *
             //                  hessenberg_iter->at(k, i)) >
@@ -156,14 +161,14 @@ void finish_arnoldi_reorth(
                     // DONE: TODO find good alternative here, so the type of
                     // DONE: arnoldi_norm can change to
                     // remove_complex<ValueType>!!! arnoldi_norm->at(1, i) +=
-                    buffer_iter->at(k, i) += next_krylov_basis->at(j, i) *
-                                             krylov_bases.read(k, j, i);
+                    buffer_iter->at(k, i) +=
+                        next_krylov_basis->at(j, i) * krylov_bases.at(k, j, i);
                 }
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -= buffer_iter->at(k, i) *
                                                    // arnoldi_norm->at(1, i) *
-                                                   krylov_bases.read(k, j, i);
+                                                   krylov_bases.at(k, j, i);
                 }
                 // hessenberg_iter->at(k, i) += arnoldi_norm->at(1, i);
                 hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
@@ -190,7 +195,7 @@ void finish_arnoldi_reorth(
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(iter + 1, j, i, next_krylov_basis->at(j, i));
+            krylov_bases.at(iter + 1, j, i) = next_krylov_basis->at(j, i);
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -199,15 +204,17 @@ void finish_arnoldi_reorth(
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_CGS(
-    matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
-    matrix::Dense<ValueType> *hessenberg_iter,
-    matrix::Dense<ValueType> *buffer_iter,
-    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
-    const stopping_status *stop_status)
+template <typename ValueType, typename Accessor3d>
+void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
+                        Accessor3d krylov_bases,
+                        matrix::Dense<ValueType> *hessenberg_iter,
+                        matrix::Dense<ValueType> *buffer_iter,
+                        matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+                        size_type iter, const stopping_status *stop_status)
 {
+    static_assert(
+        std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
+        "ValueType must match arithmetic_type of accessor!");
     const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
     //    ValueType nrmP = 0, nrmN = 0;
     //    ValueType nrmP = 0, nrmN = 0;
@@ -229,7 +236,7 @@ void finish_arnoldi_CGS(
             hessenberg_iter->at(k, i) = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
-                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
+                    next_krylov_basis->at(j, i) * krylov_bases.at(k, j, i);
             }
         }
         // for i in 1:iter
@@ -238,7 +245,7 @@ void finish_arnoldi_CGS(
         for (size_type k = 0; k < iter + 1; ++k) {
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
+                    hessenberg_iter->at(k, i) * krylov_bases.at(k, j, i);
             }
         }
         // for i in 1:iter
@@ -259,8 +266,8 @@ void finish_arnoldi_CGS(
                 buffer_iter->at(k, i) = zero<ValueType>();
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
-                    buffer_iter->at(k, i) += next_krylov_basis->at(j, i) *
-                                             krylov_bases.read(k, j, i);
+                    buffer_iter->at(k, i) +=
+                        next_krylov_basis->at(j, i) * krylov_bases.at(k, j, i);
                 }
             }
             // for i in 1:iter
@@ -270,7 +277,7 @@ void finish_arnoldi_CGS(
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
-                        buffer_iter->at(k, i) * krylov_bases.read(k, j, i);
+                        buffer_iter->at(k, i) * krylov_bases.at(k, j, i);
                 }
                 hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
             }
@@ -298,7 +305,7 @@ void finish_arnoldi_CGS(
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(iter + 1, j, i, next_krylov_basis->at(j, i));
+            krylov_bases.at(iter + 1, j, i) = next_krylov_basis->at(j, i);
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -307,15 +314,17 @@ void finish_arnoldi_CGS(
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_CGS2(
-    matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
-    matrix::Dense<ValueType> *hessenberg_iter,
-    matrix::Dense<ValueType> *buffer_iter,
-    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
-    const stopping_status *stop_status)
+template <typename ValueType, typename Accessor3d>
+void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
+                         Accessor3d krylov_bases,
+                         matrix::Dense<ValueType> *hessenberg_iter,
+                         matrix::Dense<ValueType> *buffer_iter,
+                         matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+                         size_type iter, const stopping_status *stop_status)
 {
+    static_assert(
+        std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
+        "ValueType must match arithmetic_type of accessor!");
     const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
     //    ValueType nrmP = 0, nrmN = 0;
     //    ValueType nrmP = 0, nrmN = 0;
@@ -337,7 +346,7 @@ void finish_arnoldi_CGS2(
             hessenberg_iter->at(k, i) = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
-                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
+                    next_krylov_basis->at(j, i) * krylov_bases.at(k, j, i);
             }
         }
         // for i in 1:iter
@@ -346,7 +355,7 @@ void finish_arnoldi_CGS2(
         for (size_type k = 0; k < iter + 1; ++k) {
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
+                    hessenberg_iter->at(k, i) * krylov_bases.at(k, j, i);
             }
         }
         // for i in 1:iter
@@ -376,8 +385,8 @@ void finish_arnoldi_CGS2(
                 buffer_iter->at(k, i) = zero<ValueType>();
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
-                    buffer_iter->at(k, i) += next_krylov_basis->at(j, i) *
-                                             krylov_bases.read(k, j, i);
+                    buffer_iter->at(k, i) +=
+                        next_krylov_basis->at(j, i) * krylov_bases.at(k, j, i);
                 }
             }
             // for i in 1:iter
@@ -387,7 +396,7 @@ void finish_arnoldi_CGS2(
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
-                        buffer_iter->at(k, i) * krylov_bases.read(k, j, i);
+                        buffer_iter->at(k, i) * krylov_bases.at(k, j, i);
                 }
                 hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
             }
@@ -410,7 +419,7 @@ void finish_arnoldi_CGS2(
             //           << arnoldi_norm->at(2, i) << " - " << l << std::endl;
             // nrmN = norm(next_krylov_basis)
         }
-        helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
+        helper_functions_accessor<Accessor3d>::write_scale(
             krylov_bases, iter + 1, i,
             arnoldi_norm->at(2, i) / arnoldi_norm->at(1, i));
         /*
@@ -431,7 +440,7 @@ void finish_arnoldi_CGS2(
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(iter + 1, j, i, next_krylov_basis->at(j, i));
+            krylov_bases.at(iter + 1, j, i) = next_krylov_basis->at(j, i);
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -547,18 +556,22 @@ void solve_upper_triangular(
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
-void calculate_qy(Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+template <typename ValueType, typename Accessor3dConst>
+void calculate_qy(Accessor3dConst krylov_bases,
                   const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const size_type *final_iter_nums)
 {
+    static_assert(
+        std::is_same<ValueType,
+                     typename Accessor3dConst::arithmetic_type>::value,
+        "ValueType must match arithmetic_type of accessor!");
     for (size_type k = 0; k < before_preconditioner->get_size()[1]; ++k) {
         for (size_type i = 0; i < before_preconditioner->get_size()[0]; ++i) {
             before_preconditioner->at(i, k) = zero<ValueType>();
             for (size_type j = 0; j < final_iter_nums[k]; ++j) {
                 before_preconditioner->at(i, k) +=
-                    krylov_bases.read(j, i, k) * y->at(j, k);
+                    krylov_bases.at(j, i, k) * y->at(j, k);
             }
         }
     }
@@ -601,16 +614,19 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
                   const matrix::Dense<ValueType> *residual,
                   matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
                   matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
-                  Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
+                  Accessor3d krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
 {
+    static_assert(
+        std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
+        "ValueType must match arithmetic_type of accessor!");
     for (size_type j = 0; j < residual->get_size()[1]; ++j) {
         // Calculate residual norm
         residual_norm->at(0, j) = zero<remove_complex<ValueType>>();
@@ -628,7 +644,7 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
         // std::cout << residual_norm->at(0, j) << " - " << arnoldi_norm->at(2,
         // j)
         //           << std::endl;
-        helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
+        helper_functions_accessor<Accessor3d>::write_scale(
             krylov_bases, {0}, j,
             arnoldi_norm->at(2, j) / residual_norm->at(0, j));
 
@@ -640,8 +656,8 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
             }
         }
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
-            krylov_bases.write({0}, i, j,
-                               residual->at(i, j) / residual_norm->at(0, j));
+            krylov_bases.at({0}, i, j) =
+                residual->at(i, j) / residual_norm->at(0, j);
             next_krylov_basis->at(i, j) =
                 residual->at(i, j) / residual_norm->at(0, j);
         }
@@ -650,11 +666,10 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
 
     for (size_type k = 1; k < krylov_dim + 1; ++k) {
         for (size_type j = 0; j < residual->get_size()[1]; ++j) {
-            helper_functions_accessor<ValueType, ValueTypeKrylovBases>::
-                write_scale(krylov_bases, k, j,
-                            one<remove_complex<ValueType>>());
+            helper_functions_accessor<Accessor3d>::write_scale(
+                krylov_bases, k, j, one<remove_complex<ValueType>>());
             for (size_type i = 0; i < residual->get_size()[0]; ++i) {
-                krylov_bases.write(k, i, j, zero<ValueType>());
+                krylov_bases.at(k, i, j) = zero<ValueType>();
             }
         }
     }
@@ -666,15 +681,14 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
 
 // template <typename ValueType, typename ValueTypeKrylovBases, bool
 // Reorthogonalization, bool MGS_CGS>
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void step_1(std::shared_ptr<const ReferenceExecutor> exec,
             matrix::Dense<ValueType> *next_krylov_basis,
             matrix::Dense<ValueType> *givens_sin,
             matrix::Dense<ValueType> *givens_cos,
             matrix::Dense<remove_complex<ValueType>> *residual_norm,
             matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
-            matrix::Dense<ValueType> *hessenberg_iter,
+            Accessor3d krylov_bases, matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
             const matrix::Dense<remove_complex<ValueType>> *b_norm,
             matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
@@ -683,6 +697,9 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
             Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
             int *num_reorth_steps, int *num_reorth_vectors)
 {
+    static_assert(
+        std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
+        "ValueType must match arithmetic_type of accessor!");
     /*
         if (Reorthogonalization) std::cout << "Reorthogonalization";
         else  std::cout << "No Reorthogonalization";
@@ -726,10 +743,10 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3dConst>
 void step_2(std::shared_ptr<const ReferenceExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+            Accessor3dConst krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
diff --git a/reference/test/solver/gmres_mixed_kernels.cpp b/reference/test/solver/gmres_mixed_kernels.cpp
index 54206435f1a..92a8a0c2030 100644
--- a/reference/test/solver/gmres_mixed_kernels.cpp
+++ b/reference/test/solver/gmres_mixed_kernels.cpp
@@ -116,14 +116,16 @@ class GmresMixed : public ::testing::Test {
 };
 
 
-using TestTypes = ::testing::Types<
-    std::tuple<double, double>, std::tuple<double, float>,
-    std::tuple<double, gko::int64>, std::tuple<double, gko::int32>,
-    std::tuple<double, gko::int16>, std::tuple<double, gko::half>,
-    std::tuple<float, float>, std::tuple<float, gko::half>,
-    std::tuple<std::complex<double>, std::complex<double>>,
-    std::tuple<std::complex<double>, std::complex<float>>,
-    std::tuple<std::complex<float>, std::complex<float>>>;
+using TestTypes =
+    ::testing::Types<std::tuple<double, double>, std::tuple<double, float>,
+                     // std::tuple<double, gko::int64>,
+                     std::tuple<double, gko::int32>,
+                     std::tuple<double, gko::int16>,
+                     std::tuple<double, gko::half>, std::tuple<float, float>,
+                     std::tuple<float, gko::half>,
+                     std::tuple<std::complex<double>, std::complex<double>>,
+                     std::tuple<std::complex<double>, std::complex<float>>,
+                     std::tuple<std::complex<float>, std::complex<float>>>;
 
 
 TYPED_TEST_CASE(GmresMixed, TestTypes);

From 5c547d9e334a9ce457813757a5c9cd7a8fe45204 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 17 Aug 2020 06:59:19 +0200
Subject: [PATCH 0372/1008] Remove Accessor3dConst

Reference executor now compiles, with the donwside that an Accessor
can no longer be created from const pointers.
---
 common/solver/gmres_mixed_kernels.hpp.inc |  68 ++--
 core/solver/gmres_mixed.cpp               |  10 +-
 core/solver/gmres_mixed_accessor.hpp      | 390 +++++++++-------------
 core/solver/gmres_mixed_kernels.hpp       |   6 +-
 reference/solver/gmres_mixed_kernels.cpp  |  11 +-
 5 files changed, 206 insertions(+), 279 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 822efca448f..7e6b872e3cd 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -371,11 +371,11 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_inf_kernel(
 }
 
 
-template <typename ValueType, typename Accessor3dConst>
+template <typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
     size_type k, size_type num_rows, size_type num_rhs,
     const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov, Accessor3dConst krylov_bases,
+    size_type stride_next_krylov, const Accessor3d krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -423,7 +423,7 @@ template <typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
     size_type num_iters, size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov, Accessor3dConst krylov_bases,
+    size_type stride_next_krylov, const Accessor3d krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -469,11 +469,11 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
 }
 
 
-template <typename ValueType, typename Accessor3dConst>
+template <typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_1(
     size_type num_iters, size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov, Accessor3dConst krylov_bases,
+    size_type stride_next_krylov, const Accessor3d krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -543,11 +543,11 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_1(
     //    }
 }
 
-template <int dot_dim, typename ValueType, typename Accessor3dConst>
+template <int dot_dim, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(dot_dim *dot_dim) void multidot_kernel_num_iters_2(
     size_type num_iters, size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov, Accessor3dConst krylov_bases,
+    size_type stride_next_krylov, const Accessor3d krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -602,11 +602,11 @@ __global__ __launch_bounds__(dot_dim *dot_dim) void multidot_kernel_num_iters_2(
 }
 
 
-template <int block_size, typename ValueType, typename Accessor3dConst>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(block_size) void singledot_kernel_num_iters_2(
     size_type num_iters, size_type num_rows,
     const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov, Accessor3dConst krylov_bases,
+    size_type stride_next_krylov, const Accessor3d krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
@@ -654,12 +654,12 @@ __global__ __launch_bounds__(block_size) void singledot_kernel_num_iters_2(
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename Accessor3dConst>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(block_size) void update_next_krylov_kernel(
     size_type k, size_type num_rows, size_type num_cols,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-    Accessor3dConst krylov_bases, const ValueType *__restrict__ hessenberg_iter,
-    size_type stride_hessenberg,
+    const Accessor3d krylov_bases,
+    const ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
 {
     const auto global_id = thread::get_thread_id_flat();
@@ -680,12 +680,12 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel(
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename Accessor3dConst>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor3dConst krylov_bases,
+        const Accessor3d krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -712,12 +712,12 @@ __global__
 
 // Must be called with at least `num_vectors * krylov_bases.get_stride0()`
 // threads in total.
-template <int block_size, typename ValueType, typename Accessor3dConst>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_0(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor3dConst krylov_bases,
+        const Accessor3d krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -744,12 +744,12 @@ __global__
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename Accessor3dConst>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_1(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor3dConst krylov_bases,
+        const Accessor3d krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -808,12 +808,12 @@ __global__
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename Accessor3dConst>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_2(
         size_type num_iters, size_type num_blocks, size_type num_rows,
         size_type num_cols, ValueType *__restrict__ next_krylov_basis,
-        size_type stride_next_krylov, Accessor3dConst krylov_bases,
+        size_type stride_next_krylov, const Accessor3d krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -889,12 +889,12 @@ __global__
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename Accessor3dConst>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_3(
         size_type num_iters, size_type num_blocks, size_type num_rows,
         size_type num_cols, ValueType *__restrict__ next_krylov_basis,
-        size_type stride_next_krylov, Accessor3dConst krylov_bases,
+        size_type stride_next_krylov, const Accessor3d krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -975,12 +975,12 @@ __global__
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename Accessor3dConst>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_4(
         size_type num_iters, size_type num_blocks, size_type num_rows,
         size_type num_cols, ValueType *__restrict__ next_krylov_basis,
-        size_type stride_next_krylov, Accessor3dConst krylov_bases,
+        size_type stride_next_krylov, const Accessor3d krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const stopping_status *__restrict__ stop_status)
@@ -1052,12 +1052,12 @@ __global__
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename Accessor3dConst>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_and_check(
         size_type k, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor3dConst krylov_bases,
+        const Accessor3d krylov_bases,
         const ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg,
         const remove_complex<ValueType> *__restrict__ arnoldi_norm,
@@ -1094,11 +1094,11 @@ __global__
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename Accessor3dConst>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
     size_type k, size_type num_rows, size_type num_cols,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-    Accessor3dConst krylov_bases, ValueType *__restrict__ hessenberg_iter,
+    const Accessor3d krylov_bases, ValueType *__restrict__ hessenberg_iter,
     size_type stride_hessenberg, const ValueType *__restrict__ buffer_iter,
     size_type stride_buffer, const stopping_status *__restrict__ stop_status,
     const stopping_status *__restrict__ reorth_status)
@@ -1124,12 +1124,12 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename Accessor3dConst>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_and_add(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor3dConst krylov_bases, ValueType *__restrict__ hessenberg_iter,
+        const Accessor3d krylov_bases, ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg, const ValueType *__restrict__ buffer_iter,
         size_type stride_buffer,
         const stopping_status *__restrict__ stop_status,
@@ -1160,12 +1160,12 @@ __global__
 
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
-template <int block_size, typename ValueType, typename Accessor3dConst>
+template <int block_size, typename ValueType, typename Accessor3d>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_and_add_1(
         size_type num_iters, size_type num_rows, size_type num_cols,
         ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        Accessor3dConst krylov_bases, ValueType *__restrict__ hessenberg_iter,
+        const Accessor3d krylov_bases, ValueType *__restrict__ hessenberg_iter,
         size_type stride_hessenberg, const ValueType *__restrict__ buffer_iter,
         size_type stride_buffer,
         const stopping_status *__restrict__ stop_status,
@@ -1564,9 +1564,9 @@ __global__ __launch_bounds__(block_size) void solve_upper_triangular_kernel(
 
 // Must be called with at least `stride_preconditioner * num_rows` threads
 // in total.
-template <size_type block_size, typename ValueType, typename Accessor3dConst>
+template <size_type block_size, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(block_size) void calculate_Qy_kernel(
-    size_type num_rows, size_type num_cols, Accessor3dConst krylov_bases,
+    size_type num_rows, size_type num_cols, const Accessor3d krylov_bases,
     const ValueType *__restrict__ y, size_type stride_y,
     ValueType *__restrict__ before_preconditioner,
     size_type stride_preconditioner,
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index c7920afe628..59f007a491b 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -289,16 +289,16 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
                 span{0, dense_b->get_size()[1] * (restart_iter)});
 
             exec->run(gmres_mixed::make_step_2(
-                residual_norm_collection.get(),
-                krylov_bases_accessor.to_const(), hessenberg_view.get(),
-                y.get(), before_preconditioner.get(), &final_iter_nums));
+                residual_norm_collection.get(), krylov_bases_accessor,
+                hessenberg_view.get(), y.get(), before_preconditioner.get(),
+                &final_iter_nums));
             /* */ /*
             auto hessenberg_small = hessenberg->create_submatrix(
                 span{0, restart_iter},
                 span{0, dense_b->get_size()[1] * (restart_iter)});
             exec->run(gmres_mixed::make_step_2(
                 residual_norm_collection.get(),
-                krylov_bases_accessor.to_const(),
+                krylov_bases_accessor,
                 hessenberg_small.get(), y.get(),
                 before_preconditioner.get(), &final_iter_nums));
             */
@@ -428,7 +428,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
         span{0, dense_b->get_size()[1] * (restart_iter)});
 
     exec->run(gmres_mixed::make_step_2(
-        residual_norm_collection.get(), krylov_bases_accessor.to_const(),
+        residual_norm_collection.get(), krylov_bases_accessor,
         hessenberg_small.get(), y.get(), before_preconditioner.get(),
         &final_iter_nums));
     // Solve upper triangular.
diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index 2f71973c245..8a59f2543e8 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <cinttypes>
+#include <limits>
 #include <memory>
 #include <type_traits>
 
@@ -48,6 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <iostream>
+#define GKO_DEBUG_OUTPUT false
 
 
 namespace gko {
@@ -66,15 +68,84 @@ template <typename StorageType, typename ArithmeticType,
                   !std::is_same<StorageType, ArithmeticType>::value) ||
                  std::is_integral<StorageType>::value>
 */
-struct helper_have_scale {};
+struct helper_have_scale {
+};
 
 template <typename StorageType, typename ArithmeticType>
 struct helper_have_scale<StorageType, ArithmeticType, false>
-    : public std::false_type {};
+    : public std::false_type {
+};
 
 template <typename StorageType, typename ArithmeticType>
 struct helper_have_scale<StorageType, ArithmeticType, true>
-    : public std::true_type {};
+    : public std::true_type {
+};
+
+/**
+ * This is a mixin which defines the binary operators for *, /, +, - for the
+ * Reference class, the unary operator -, and the assignment operators
+ * *=, /=, +=, -=
+ * Additionally, it prevents the default generation of copy and move constructor
+ * and copy and move assignment.
+ */
+template <typename Reference, typename ArithmeticType>
+struct enable_reference {
+    using arithmetic_type = ArithmeticType;
+
+    enable_reference() =
+        default;  // Needs to exist, so creation is not probibited
+    enable_reference(enable_reference &&) = delete;
+    enable_reference(const enable_reference &) = delete;
+    enable_reference &operator=(const enable_reference &) = delete;
+    enable_reference &operator=(enable_reference &&) = delete;
+
+#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_oper, _op)            \
+    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper( \
+        const Reference &ref1, const Reference &ref2)                 \
+    {                                                                 \
+        return static_cast<arithmetic_type>(ref1)                     \
+            _op static_cast<arithmetic_type>(ref2);                   \
+    }                                                                 \
+    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper( \
+        const Reference &ref, const arithmetic_type &a)               \
+    {                                                                 \
+        return static_cast<arithmetic_type>(ref) _op a;               \
+    }                                                                 \
+    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper( \
+        const arithmetic_type &a, const Reference &ref)               \
+    {                                                                 \
+        return a _op static_cast<arithmetic_type>(ref);               \
+    }
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator*, *)
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator/, /)
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator+, +)
+    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator-, -)
+#undef GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD
+
+#define GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(_oper, _op)           \
+    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper(    \
+        Reference &&ref1, const Reference &ref2)                         \
+    {                                                                    \
+        return std::move(ref1) = static_cast<arithmetic_type>(ref1)      \
+                   _op static_cast<arithmetic_type>(ref2);               \
+    }                                                                    \
+    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper(    \
+        Reference &&ref, const arithmetic_type &a)                       \
+    {                                                                    \
+        return std::move(ref) = static_cast<arithmetic_type>(ref) _op a; \
+    }
+
+    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator*=, *)
+    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator/=, /)
+    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator+=, +)
+    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator-=, -)
+#undef GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD
+    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type
+    operator-(const Reference &ref)
+    {
+        return -static_cast<arithmetic_type>(ref);
+    }
+};
 
 
 }  // namespace detail
@@ -82,7 +153,8 @@ struct helper_have_scale<StorageType, ArithmeticType, true>
 
 template <typename StorageType, typename ArithmeticType,
           bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
-class Accessor3dConst {};
+class Accessor3d {
+};
 /**
  * @internal
  *
@@ -99,35 +171,34 @@ class Accessor3dConst {};
  * The accessor uses row-major access.
  */
 template <typename StorageType, typename ArithmeticType>
-class Accessor3dConst<StorageType, ArithmeticType, false> {
+class Accessor3d<StorageType, ArithmeticType, false> {
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
     static_assert(
         !detail::helper_have_scale<StorageType, ArithmeticType>::value,
         "storage_type must not be an integral in this class.");
-
     static constexpr bool has_scale{false};
-    using const_type = Accessor3dConst;
 
 protected:
-    class reference {
+    class reference
+        : public detail::enable_reference<reference, arithmetic_type> {
     public:
         reference() = delete;
-        reference(reference &&) = delete;
-        reference(const reference &) = delete;
-        reference &operator=(const reference &) = delete;
-        reference &operator=(reference &&) = delete;
-
         reference(storage_type *const GKO_RESTRICT ptr) : ptr_{ptr} {}
-        operator arithmetic_type() const
+        GKO_ATTRIBUTES constexpr GKO_INLINE operator arithmetic_type() const
         {
             return static_cast<arithmetic_type>(*ptr_);
         }
-        arithmetic_type operator=(arithmetic_type val) &&
+        GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type
+        operator=(arithmetic_type val) &&
+        {
+            return *ptr_ = static_cast<storage_type>(val);
+        }
+        GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type
+        operator=(const reference &ref) &&
         {
-            *ptr_ = static_cast<storage_type>(val);
-            return val;
+            return std::move(*this) = static_cast<arithmetic_type>(ref);
         }
 
     private:
@@ -136,32 +207,19 @@ class Accessor3dConst<StorageType, ArithmeticType, false> {
 
 public:
     /**
+     * TODO update
      * Creates the accessor with an already allocated storage space with a
      * stride.
      *
      * @internal
-     * const_cast needed, so Accessor3dConst(ac.get_storage(), ...)
+     * const_cast needed, so Accessor3d(ac.get_storage(), ...)
      * works. Also, storage_ is never accessed in a non-const fashion, so it
      * is not invalid or UB code.
      */
-    Accessor3dConst(const storage_type *storage, size_type stride0,
-                    size_type stride1)
-        : storage_{const_cast<storage_type *>(storage)},
-          stride_{stride0, stride1}
+    Accessor3d(storage_type *storage, size_type stride0, size_type stride1)
+        : storage_{storage}, stride_{stride0, stride1}
     {}
 
-    /**
-     * Reads the value at the given index.
-     *
-     * @internal The const is important since it prevents writes
-     */
-    template <typename IndexType>
-    GKO_ATTRIBUTES const reference at(IndexType x, IndexType y,
-                                      IndexType z) const
-    {
-        return {storage_ + x * stride_[0] + y * stride_[1] + z};
-    }
-
     /**
      * Reads the value at the given index.
      */
@@ -177,6 +235,13 @@ class Accessor3dConst<StorageType, ArithmeticType, false> {
             rest_storage[x * stride_[0] + y * stride_[1] + z]);
     }
 
+    template <typename IndexType>
+    GKO_ATTRIBUTES arithmetic_type at(IndexType x, IndexType y,
+                                      IndexType z) const
+    {
+        return reference{storage_ + x * stride_[0] + y * stride_[1] + z};
+    }
+
     GKO_ATTRIBUTES size_type get_stride0() const { return stride_[0]; }
     GKO_ATTRIBUTES size_type get_stride1() const { return stride_[1]; }
 
@@ -186,13 +251,32 @@ class Accessor3dConst<StorageType, ArithmeticType, false> {
     {
         return storage_;
     }
+    /**
+     * Writes the given value at the given index.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES void write(IndexType x, IndexType y, IndexType z,
+                              arithmetic_type value)
+    {
+        storage_type *GKO_RESTRICT rest_storage = this->storage_;
+        rest_storage[x * this->stride_[0] + y * this->stride_[1] + z] =
+            static_cast<storage_type>(value);
+    }
+
+    template <typename IndexType>
+    GKO_ATTRIBUTES reference at(IndexType x, IndexType y, IndexType z)
+    {
+        return {this->storage_ + x * this->stride_[0] + y * this->stride_[1] +
+                z};
+    }
+
+    GKO_ATTRIBUTES storage_type *get_storage() { return this->storage_; }
 
 protected:
     storage_type *storage_;
     size_type stride_[2];
 };
 
-
 /**
  * @internal
  *
@@ -211,7 +295,7 @@ class Accessor3dConst<StorageType, ArithmeticType, false> {
  * The accessor uses row-major access.
  */
 template <typename StorageType, typename ArithmeticType>
-class Accessor3dConst<StorageType, ArithmeticType, true> {
+class Accessor3d<StorageType, ArithmeticType, true> {
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
@@ -219,93 +303,51 @@ class Accessor3dConst<StorageType, ArithmeticType, true> {
                   "storage_type must not be an integral in this class.");
 
     static constexpr bool has_scale{true};
-    using const_type = Accessor3dConst;
 
 protected:
-    class scale_reference {
+    class reference
+        : public detail::enable_reference<reference, arithmetic_type> {
     public:
-        scale_reference() = delete;
-        scale_reference(scale_reference &&) = delete;
-        scale_reference(const scale_reference &) = delete;
-        scale_reference &operator=(const scale_reference &) = delete;
-        scale_reference &operator=(scale_reference &&) = delete;
-
-        scale_reference(arithmetic_type *const GKO_RESTRICT scale)
-            : scale_{scale}
+        reference() = delete;
+        reference(storage_type *const GKO_RESTRICT ptr, arithmetic_type scale)
+            : ptr_{ptr}, scale_{scale}
         {}
-        operator arithmetic_type() const { return *scale_; }
-        arithmetic_type operator=(arithmetic_type val) &&
+        GKO_ATTRIBUTES constexpr GKO_INLINE operator arithmetic_type() const
         {
-            storage_type max_val = one<storage_type>();
-            if (std::is_integral<storage_type>::value) {
-                max_val = std::numeric_limits<storage_type>::max();
-            }
-            *scale_ = val / static_cast<arithmetic_type>(max_val);
-            return val;
+            return static_cast<arithmetic_type>(*ptr_) * scale_;
         }
-
-    private:
-        arithmetic_type *const GKO_RESTRICT scale_;
-    };
-
-    class value_reference {
-    public:
-        value_reference() = delete;
-        value_reference(value_reference &&) = delete;
-        value_reference(const value_reference &) = delete;
-        value_reference &operator=(const value_reference &) = delete;
-        value_reference &operator=(value_reference &&) = delete;
-
-        value_reference(storage_type *const GKO_RESTRICT value,
-                        arithmetic_type scale)
-            : value_{value}, scale_{scale}
-        {}
-        operator arithmetic_type() const
+        GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type
+        operator=(arithmetic_type val) &&
         {
-            return static_cast<arithmetic_type>(*value_) * scale_;
+            return *ptr_ = static_cast<storage_type>(val / scale_), val;
         }
-        arithmetic_type operator=(arithmetic_type val) &&
+        GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type
+        operator=(const reference &ref) &&
         {
-            *value_ = static_cast<storage_type>(val / scale_);
-            return val;
+            return std::move(*this) = static_cast<arithmetic_type>(ref);
         }
 
     private:
-        storage_type *const GKO_RESTRICT value_;
+        storage_type *const GKO_RESTRICT ptr_;
         const arithmetic_type scale_;
     };
 
 public:
     /**
+     * TODO update
      * Creates the accessor with an already allocated storage space with a
      * stride.
      *
      * @internal
-     * const_cast needed, so Accessor3dConst(ac.get_storage(), ...)
+     * const_cast needed, so Accessor3d(ac.get_storage(), ...)
      * works. Also, storage_ is never accessed in a non-const fashion, so it
      * is not invalid or UB code.
      */
-    Accessor3dConst(const storage_type *storage, size_type stride0,
-                    size_type stride1, const arithmetic_type *scale)
-        : storage_{const_cast<storage_type *>(storage)},
-          stride_{stride0, stride1},
-          scale_{const_cast<arithmetic_type *>(scale)}
+    Accessor3d(storage_type *storage, size_type stride0, size_type stride1,
+               arithmetic_type *scale)
+        : storage_{storage}, stride_{stride0, stride1}, scale_{scale}
     {}
 
-    template <typename IndexType>
-    GKO_ATTRIBUTES const scale_reference scale_at(IndexType x,
-                                                  IndexType z) const
-    {
-        return {scale_ + x * stride_[1] + z};
-    }
-
-    template <typename IndexType>
-    GKO_ATTRIBUTES const value_reference at(IndexType x, IndexType y,
-                                            IndexType z) const
-    {
-        return {storage_ + x * stride_[0] + y * stride_[1] + z,
-                read_scale(x, z)};
-    }
     /**
      * Reads the scale value at the given indices.
      */
@@ -316,6 +358,14 @@ class Accessor3dConst<StorageType, ArithmeticType, true> {
         return rest_scale[x * stride_[1] + z];
     }
 
+    template <typename IndexType>
+    GKO_ATTRIBUTES const reference at(IndexType x, IndexType y,
+                                      IndexType z) const
+    {
+        return {storage_ + x * stride_[0] + y * stride_[1] + z,
+                read_scale(x, z)};
+    }
+
     /**
      * Reads the value at the given index.
      */
@@ -351,136 +401,8 @@ class Accessor3dConst<StorageType, ArithmeticType, true> {
     {
         return this->scale_;
     }
-
-protected:
-    storage_type *storage_;
-    size_type stride_[2];
-    arithmetic_type *scale_;
-};
-
-
-template <typename StorageType, typename ArithmeticType,
-          bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
-class Accessor3d : public Accessor3dConst<StorageType, ArithmeticType> {};
-
-
-/**
- * @internal
- *
- * The Accessor3d class hides the underlying storage_ format and provides a
- * simple interface for accessing a one dimensional storage.
- *
- * This class only manages the accesses, however, and not the memory itself.
- */
-template <typename StorageType, typename ArithmeticType>
-class Accessor3d<StorageType, ArithmeticType, false>
-    : public Accessor3dConst<StorageType, ArithmeticType> {
-public:
-    using storage_type = StorageType;
-    using arithmetic_type = ArithmeticType;
-    static_assert(
-        !detail::helper_have_scale<StorageType, ArithmeticType>::value,
-        "storage_type must not be an integral in this class.");
-
-private:
-    using Accessor3dC = Accessor3dConst<storage_type, arithmetic_type>;
-    using reference = typename Accessor3dC::reference;
-
-public:
-    using const_type = Accessor3dC;
-    /**
-     * Creates an empty accessor pointing to a nullptr.
-     */
-    Accessor3d() : Accessor3dC(nullptr, {}, {}) {}
-
-    /**
-     * Creates the accessor with an already allocated storage space with a
-     * stride.
-     */
-    Accessor3d(storage_type *storage, size_type stride0, size_type stride1)
-        : Accessor3dC(storage, stride0, stride1)
-    {}
-
-    Accessor3dC to_const() const
-    {
-        return {this->storage_, this->stride_[0], this->stride_[1]};
-    }
-
-
     template <typename IndexType>
     GKO_ATTRIBUTES reference at(IndexType x, IndexType y, IndexType z)
-    {
-        return {this->storage_ + x * this->stride_[0] + y * this->stride_[1] +
-                z};
-    }
-    /**
-     * Writes the given value at the given index.
-     */
-    template <typename IndexType>
-    GKO_ATTRIBUTES void write(IndexType x, IndexType y, IndexType z,
-                              arithmetic_type value)
-    {
-        storage_type *GKO_RESTRICT rest_storage = this->storage_;
-        rest_storage[x * this->stride_[0] + y * this->stride_[1] + z] =
-            static_cast<storage_type>(value);
-    }
-
-    GKO_ATTRIBUTES storage_type *get_storage() { return this->storage_; }
-};
-
-
-/**
- * @internal
- *
- * The Accessor3d class hides the underlying storage_ format and provides a
- * simple interface for accessing a one dimensional storage.
- *
- * This class only manages the accesses, however, and not the memory itself.
- */
-template <typename StorageType, typename ArithmeticType>
-class Accessor3d<StorageType, ArithmeticType, true>
-    : public Accessor3dConst<StorageType, ArithmeticType> {
-public:
-    using storage_type = StorageType;
-    using arithmetic_type = ArithmeticType;
-    static_assert(detail::helper_have_scale<StorageType, ArithmeticType>::value,
-                  "storage_type must not be an integral in this class.");
-
-private:
-    using Accessor3dC = Accessor3dConst<storage_type, arithmetic_type>;
-    using scale_reference = typename Accessor3dC::scale_reference;
-    using value_reference = typename Accessor3dC::value_reference;
-
-public:
-    using const_type = Accessor3dC;
-    /**
-     * Creates an empty accessor pointing to a nullptr.
-     */
-    Accessor3d() : Accessor3dC(nullptr, {}, {}, nullptr) {}
-
-    /**
-     * Creates the accessor with an already allocated storage space with a
-     * stride.
-     */
-    Accessor3d(storage_type *storage, size_type stride0, size_type stride1,
-               arithmetic_type *scale)
-        : Accessor3dC(storage, stride0, stride1, scale)
-    {}
-
-    Accessor3dC to_const() const
-    {
-        return {this->storage_, this->stride_[0], this->stride_[1],
-                this->scale_};
-    }
-
-    template <typename IndexType>
-    GKO_ATTRIBUTES scale_reference scale_at(IndexType x, IndexType z)
-    {
-        return {this->scale_ + x * this->stride_[1] + z};
-    }
-
-    template <typename IndexType>
-    GKO_ATTRIBUTES value_reference at(IndexType x, IndexType y, IndexType z)
     {
         return {
             this->storage_ + x * this->stride_[0] + y * this->stride_[1] + z,
@@ -499,7 +421,7 @@ class Accessor3d<StorageType, ArithmeticType, true>
         const auto stride1 = this->stride_[1];
         rest_storage[x * stride0 + y * stride1 + z] =
             static_cast<storage_type>(value / this->read_scale(x, z));
-#if false && not defined(__CUDA_ARCH__)
+#if GKO_DEBUG_OUTPUT && not defined(__CUDA_ARCH__)
         std::cout << "storage[" << x << ", " << y << ", " << z
                   << "] = " << rest_storage[x * stride0 + y * stride1 + z]
                   << "; value = " << value
@@ -522,7 +444,7 @@ class Accessor3d<StorageType, ArithmeticType, true>
         }
         rest_scale[x * this->stride_[1] + z] =
             val / static_cast<arithmetic_type>(max_val);
-#if false && not defined(__CUDA_ARCH__)
+#if GKO_DEBUG_OUTPUT && not defined(__CUDA_ARCH__)
         std::cout << "scale[" << x << ", " << z
                   << "] = " << rest_scale[x * this->stride_[1] + z]
                   << "; val = " << val
@@ -538,12 +460,18 @@ class Accessor3d<StorageType, ArithmeticType, true>
     GKO_ATTRIBUTES storage_type *get_storage() { return this->storage_; }
 
     GKO_ATTRIBUTES arithmetic_type *get_scale() { return this->scale_; }
+
+protected:
+    storage_type *storage_;
+    size_type stride_[2];
+    arithmetic_type *scale_;
 };
 
 
 template <typename ValueType, typename ValueTypeKrylovBases,
           bool = Accessor3d<ValueTypeKrylovBases, ValueType>::has_scale>
-class Accessor3dHelper {};
+class Accessor3dHelper {
+};
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
@@ -618,7 +546,8 @@ class Accessor3dHelper<ValueType, ValueTypeKrylovBases, false> {
 //----------------------------------------------
 
 template <typename Accessor3d, bool = Accessor3d::has_scale>
-struct helper_functions_accessor {};
+struct helper_functions_accessor {
+};
 
 // Accessors having a scale
 template <typename Accessor3d>
@@ -631,8 +560,7 @@ struct helper_functions_accessor<Accessor3d, true> {
                                                   IndexType col_idx,
                                                   value_type value)
     {
-        // krylov_bases.set_scale(vector_idx, col_idx, value);
-        krylov_bases.scale_at(vector_idx, col_idx) = value;
+        krylov_bases.set_scale(vector_idx, col_idx, value);
     }
 };
 
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index b113dcfd967..72de109ba36 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -73,8 +73,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     template _macro(double, int32);                              \
     template _macro(double, int16);                              \
     template _macro(double, half);                               \
-    template _macro(float, float, float >);                      \
-    template _macro(float, half, float >);                       \
+    template _macro(float, float);                               \
+    template _macro(float, half);                                \
     template _macro(std::complex<double>, std::complex<double>); \
     template _macro(std::complex<double>, std::complex<float>);  \
     template _macro(std::complex<float>, std::complex<float>)
@@ -147,7 +147,7 @@ namespace gmres_mixed {
 #define GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(_type1, _accessor)       \
     void step_2(std::shared_ptr<const DefaultExecutor> exec,           \
                 const matrix::Dense<_type1> *residual_norm_collection, \
-                _accessor /*::const_type*/ krylov_bases,               \
+                const _accessor &krylov_bases,                         \
                 const matrix::Dense<_type1> *hessenberg,               \
                 matrix::Dense<_type1> *y,                              \
                 matrix::Dense<_type1> *before_preconditioner,          \
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index 43f391f0a8b..5f495f4a33e 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -556,15 +556,14 @@ void solve_upper_triangular(
 }
 
 
-template <typename ValueType, typename Accessor3dConst>
-void calculate_qy(Accessor3dConst krylov_bases,
+template <typename ValueType, typename Accessor3d>
+void calculate_qy(const Accessor3d &krylov_bases,
                   const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const size_type *final_iter_nums)
 {
     static_assert(
-        std::is_same<ValueType,
-                     typename Accessor3dConst::arithmetic_type>::value,
+        std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
         "ValueType must match arithmetic_type of accessor!");
     for (size_type k = 0; k < before_preconditioner->get_size()[1]; ++k) {
         for (size_type i = 0; i < before_preconditioner->get_size()[0]; ++i) {
@@ -743,10 +742,10 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
 
 
-template <typename ValueType, typename Accessor3dConst>
+template <typename ValueType, typename Accessor3d>
 void step_2(std::shared_ptr<const ReferenceExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor3dConst krylov_bases,
+            const Accessor3d &krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,

From 5ff50aa8c3df5f781092f6031196872a659a6b14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 17 Aug 2020 12:04:08 +0200
Subject: [PATCH 0373/1008] Adopt OpenMP support to new Accessor

So it compiles and works, it does not yet use `at` or similar.
---
 core/solver/gmres_mixed_accessor.hpp    | 20 +++-----
 omp/solver/gmres_mixed_kernels.cpp      | 65 ++++++++++++-------------
 omp/test/solver/gmres_mixed_kernels.cpp | 13 +++--
 3 files changed, 44 insertions(+), 54 deletions(-)

diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index 8a59f2543e8..f8437be9295 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -68,18 +68,15 @@ template <typename StorageType, typename ArithmeticType,
                   !std::is_same<StorageType, ArithmeticType>::value) ||
                  std::is_integral<StorageType>::value>
 */
-struct helper_have_scale {
-};
+struct helper_have_scale {};
 
 template <typename StorageType, typename ArithmeticType>
 struct helper_have_scale<StorageType, ArithmeticType, false>
-    : public std::false_type {
-};
+    : public std::false_type {};
 
 template <typename StorageType, typename ArithmeticType>
 struct helper_have_scale<StorageType, ArithmeticType, true>
-    : public std::true_type {
-};
+    : public std::true_type {};
 
 /**
  * This is a mixin which defines the binary operators for *, /, +, - for the
@@ -153,8 +150,7 @@ struct enable_reference {
 
 template <typename StorageType, typename ArithmeticType,
           bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
-class Accessor3d {
-};
+class Accessor3d {};
 /**
  * @internal
  *
@@ -206,6 +202,7 @@ class Accessor3d<StorageType, ArithmeticType, false> {
     };
 
 public:
+    Accessor3d() : storage_{nullptr}, stride_{0, 0} {}
     /**
      * TODO update
      * Creates the accessor with an already allocated storage space with a
@@ -333,6 +330,7 @@ class Accessor3d<StorageType, ArithmeticType, true> {
     };
 
 public:
+    Accessor3d() : storage_{nullptr}, stride_{0, 0}, scale_{nullptr} {}
     /**
      * TODO update
      * Creates the accessor with an already allocated storage space with a
@@ -470,8 +468,7 @@ class Accessor3d<StorageType, ArithmeticType, true> {
 
 template <typename ValueType, typename ValueTypeKrylovBases,
           bool = Accessor3d<ValueTypeKrylovBases, ValueType>::has_scale>
-class Accessor3dHelper {
-};
+class Accessor3dHelper {};
 
 
 template <typename ValueType, typename ValueTypeKrylovBases>
@@ -546,8 +543,7 @@ class Accessor3dHelper<ValueType, ValueTypeKrylovBases, false> {
 //----------------------------------------------
 
 template <typename Accessor3d, bool = Accessor3d::has_scale>
-struct helper_functions_accessor {
-};
+struct helper_functions_accessor {};
 
 // Accessors having a scale
 template <typename Accessor3d>
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index e7311bbf5e4..2ed3bd375d3 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -71,9 +71,9 @@ namespace gmres_mixed {
 namespace {
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
-                    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
+                    Accessor3d krylov_bases,
                     matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
                     const stopping_status *stop_status)
 {
@@ -125,10 +125,9 @@ void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void finish_arnoldi_reorth(
-    matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
+    matrix::Dense<ValueType> *next_krylov_basis, Accessor3d krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
     const stopping_status *stop_status)
@@ -214,14 +213,13 @@ void finish_arnoldi_reorth(
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_CGS(
-    matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
-    matrix::Dense<ValueType> *hessenberg_iter,
-    matrix::Dense<ValueType> *buffer_iter,
-    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
-    const stopping_status *stop_status)
+template <typename ValueType, typename Accessor3d>
+void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
+                        Accessor3d krylov_bases,
+                        matrix::Dense<ValueType> *hessenberg_iter,
+                        matrix::Dense<ValueType> *buffer_iter,
+                        matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+                        size_type iter, const stopping_status *stop_status)
 {
     const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
@@ -333,14 +331,13 @@ void finish_arnoldi_CGS(
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
-void finish_arnoldi_CGS2(
-    matrix::Dense<ValueType> *next_krylov_basis,
-    Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
-    matrix::Dense<ValueType> *hessenberg_iter,
-    matrix::Dense<ValueType> *buffer_iter,
-    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
-    const stopping_status *stop_status)
+template <typename ValueType, typename Accessor3d>
+void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
+                         Accessor3d krylov_bases,
+                         matrix::Dense<ValueType> *hessenberg_iter,
+                         matrix::Dense<ValueType> *buffer_iter,
+                         matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+                         size_type iter, const stopping_status *stop_status)
 {
     const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
@@ -469,7 +466,7 @@ void finish_arnoldi_CGS2(
             // nrmN = norm(next_krylov_basis)
             // nrmI = infnorm(next_krylov_basis)
         }
-        helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
+        helper_functions_accessor<Accessor3d>::write_scale(
             krylov_bases, iter + 1, i,
             arnoldi_norm->at(2, i) / arnoldi_norm->at(1, i));
         // reorthogonalization
@@ -606,8 +603,8 @@ void solve_upper_triangular(
 }
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
-void calculate_qy(Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+template <typename ValueType, typename Accessor3d>
+void calculate_qy(const Accessor3d &krylov_bases,
                   const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const size_type *final_iter_nums)
@@ -668,13 +665,13 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void initialize_2(std::shared_ptr<const OmpExecutor> exec,
                   const matrix::Dense<ValueType> *residual,
                   matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
                   matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
-                  Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
+                  Accessor3d krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
 {
@@ -699,7 +696,7 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
         // std::cout << residual_norm->at(0, j) << " - " << arnoldi_norm->at(2,
         // j)
         //           << std::endl;
-        helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
+        helper_functions_accessor<Accessor3d>::write_scale(
             krylov_bases, {0}, j,
             arnoldi_norm->at(2, j) / residual_norm->at(0, j));
 
@@ -724,9 +721,8 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for
     for (size_type k = 1; k < krylov_dim + 1; ++k) {
         for (size_type j = 0; j < residual->get_size()[1]; ++j) {
-            helper_functions_accessor<ValueType, ValueTypeKrylovBases>::
-                write_scale(krylov_bases, k, j,
-                            one<remove_complex<ValueType>>());
+            helper_functions_accessor<Accessor3d>::write_scale(
+                krylov_bases, k, j, one<remove_complex<ValueType>>());
         }
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             for (size_type j = 0; j < residual->get_size()[1]; ++j) {
@@ -740,15 +736,14 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void step_1(std::shared_ptr<const OmpExecutor> exec,
             matrix::Dense<ValueType> *next_krylov_basis,
             matrix::Dense<ValueType> *givens_sin,
             matrix::Dense<ValueType> *givens_cos,
             matrix::Dense<remove_complex<ValueType>> *residual_norm,
             matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor3d<ValueTypeKrylovBases, ValueType> krylov_bases,
-            matrix::Dense<ValueType> *hessenberg_iter,
+            Accessor3d krylov_bases, matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
             const matrix::Dense<remove_complex<ValueType>> *b_norm,
             matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
@@ -792,10 +787,10 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void step_2(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor3dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+            const Accessor3d &krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
diff --git a/omp/test/solver/gmres_mixed_kernels.cpp b/omp/test/solver/gmres_mixed_kernels.cpp
index 47b8c23484b..21e5245c88f 100644
--- a/omp/test/solver/gmres_mixed_kernels.cpp
+++ b/omp/test/solver/gmres_mixed_kernels.cpp
@@ -111,9 +111,8 @@ class GmresMixed : public ::testing::Test {
             for (size_type k = 0; k < size[0]; ++k) {
                 for (size_type i = 0; i < size[2]; ++i) {
                     gko::kernels::helper_functions_accessor<
-                        value_type, krylov_type>::write_scale(accessor, k, i,
-                                                              dist(
-                                                                  rand_engine));
+                        Accessor3d>::write_scale(accessor, k, i,
+                                                 dist(rand_engine));
                 }
             }
         }
@@ -357,13 +356,13 @@ TEST_F(GmresMixed, OmpGmresMixedStep2IsEquivalentToRef)
     initialize_data();
 
     gko::kernels::reference::gmres_mixed::step_2(
-        ref, residual_norm_collection.get(), krylov_bases_accessor.to_const(),
+        ref, residual_norm_collection.get(), krylov_bases_accessor,
         hessenberg.get(), y.get(), before_preconditioner.get(),
         final_iter_nums.get());
     gko::kernels::omp::gmres_mixed::step_2(
-        omp, d_residual_norm_collection.get(),
-        d_krylov_bases_accessor.to_const(), d_hessenberg.get(), d_y.get(),
-        d_before_preconditioner.get(), d_final_iter_nums.get());
+        omp, d_residual_norm_collection.get(), d_krylov_bases_accessor,
+        d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
+        d_final_iter_nums.get());
 
     GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);

From 0fd1af01f893376870094e062bf6b02806ae5601 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 17 Aug 2020 12:09:25 +0200
Subject: [PATCH 0374/1008] Remove unused GMRES_mixed code from Ref & OMP

---
 omp/solver/gmres_mixed_kernels.cpp       | 296 ---------------------
 reference/solver/gmres_mixed_kernels.cpp | 313 -----------------------
 2 files changed, 609 deletions(-)

diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index 2ed3bd375d3..4602f0d4367 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -71,266 +71,6 @@ namespace gmres_mixed {
 namespace {
 
 
-template <typename ValueType, typename Accessor3d>
-void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
-                    Accessor3d krylov_bases,
-                    matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
-                    const stopping_status *stop_status)
-{
-#pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
-
-    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
-        if (stop_status[i].has_stopped()) {
-            continue;
-        }
-        for (size_type k = 0; k < iter + 1; ++k) {
-            ValueType hessenberg_iter_entry = zero<ValueType>();
-
-#pragma omp parallel for reduction(add : hessenberg_iter_entry)
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                hessenberg_iter_entry +=
-                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
-            }
-            hessenberg_iter->at(k, i) = hessenberg_iter_entry;
-
-#pragma omp parallel for
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
-            }
-        }
-        // for i in 1:iter
-        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
-        // end
-
-        ValueType hessenberg_iter_entry = zero<ValueType>();
-
-#pragma omp parallel for reduction(add : hessenberg_iter_entry)
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            hessenberg_iter_entry += squared_norm(next_krylov_basis->at(j, i));
-            // next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
-        }
-        hessenberg_iter->at(iter + 1, i) = sqrt(hessenberg_iter_entry);
-        // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
-#pragma omp parallel for
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(iter + 1, j, i, next_krylov_basis->at(j, i));
-        }
-        // next_krylov_basis /= hessenberg(iter, iter + 1)
-        // krylov_bases(:, iter + 1) = next_krylov_basis
-        // End of arnoldi
-    }
-}
-
-
-template <typename ValueType, typename Accessor3d>
-void finish_arnoldi_reorth(
-    matrix::Dense<ValueType> *next_krylov_basis, Accessor3d krylov_bases,
-    matrix::Dense<ValueType> *hessenberg_iter,
-    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
-    const stopping_status *stop_status)
-{
-#pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
-#pragma omp declare reduction(addnc : remove_complex<ValueType> : omp_out = omp_out + omp_in)
-
-    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
-        if (stop_status[i].has_stopped()) {
-            continue;
-        }
-
-        remove_complex<ValueType> nrm = zero<remove_complex<ValueType>>();
-#pragma omp parallel for reduction(addnc : nrm)
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            nrm += squared_norm(next_krylov_basis->at(j, i));
-        }
-        arnoldi_norm->at(0, i) = sqrt(nrm) * 0.99;
-        // nrm = norm(next_krylov_basis)
-        for (size_type k = 0; k < iter + 1; ++k) {
-            ValueType hessenberg_iter_entry = zero<ValueType>();
-#pragma omp parallel for reduction(add : hessenberg_iter_entry)
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                hessenberg_iter_entry +=
-                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
-            }
-            hessenberg_iter->at(k, i) = hessenberg_iter_entry;
-
-#pragma omp parallel for
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
-            }
-            // DONE: TODO remove abs() as soon as type for arnoldi_norm has
-            // changed!
-            if (squared_norm(hessenberg_iter->at(k, i)) >
-                arnoldi_norm->at(0, i)) {
-                // abs(arnoldi_norm->at(0, i))) {
-                ValueType reorth = zero<ValueType>();
-#pragma omp parallel for reduction(add : reorth)
-                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
-                     ++j) {
-                    reorth += next_krylov_basis->at(j, i) *
-                              krylov_bases.read(k, j, i);
-                }
-                hessenberg_iter->at(k, i) += reorth;
-
-#pragma omp parallel for
-                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
-                     ++j) {
-                    next_krylov_basis->at(j, i) -=
-                        reorth * krylov_bases.read(k, j, i);
-                }
-            }
-        }
-        // for i in 1:iter
-        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
-        //     if (hessenberg(iter, i)*hessenberg(iter, i)>nrm*0.99)
-        //         reorth = next_krylov_basis' * krylov_bases(:, i)
-        //         next_krylov_basis  -= reorth * krylov_bases(:, i)
-        //         hessenberg(iter, i) += reorth;
-        //     end
-        // end
-
-        ValueType hessenberg_iter_entry = zero<ValueType>();
-#pragma omp parallel for reduction(add : hessenberg_iter_entry)
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            hessenberg_iter_entry += squared_norm(next_krylov_basis->at(j, i));
-            // next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
-        }
-        hessenberg_iter->at(iter + 1, i) = sqrt(hessenberg_iter_entry);
-        // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
-#pragma omp parallel for
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(iter + 1, j, i, next_krylov_basis->at(j, i));
-        }
-        // next_krylov_basis /= hessenberg(iter, iter + 1)
-        // krylov_bases(:, iter + 1) = next_krylov_basis
-        // End of arnoldi
-    }
-}
-
-
-template <typename ValueType, typename Accessor3d>
-void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
-                        Accessor3d krylov_bases,
-                        matrix::Dense<ValueType> *hessenberg_iter,
-                        matrix::Dense<ValueType> *buffer_iter,
-                        matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
-                        size_type iter, const stopping_status *stop_status)
-{
-    const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
-#pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
-#pragma omp declare reduction(addnc : remove_complex<ValueType> : omp_out = omp_out + omp_in)
-
-    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
-        if (stop_status[i].has_stopped()) {
-            continue;
-        }
-
-        remove_complex<ValueType> nrm = zero<remove_complex<ValueType>>();
-#pragma omp parallel for reduction(addnc : nrm)
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            nrm += squared_norm(next_krylov_basis->at(j, i));
-        }
-        arnoldi_norm->at(0, i) = nrm * eta;
-        // nrmP = norm(next_krylov_basis)
-        for (size_type k = 0; k < iter + 1; ++k) {
-            ValueType hessenberg_iter_entry = zero<ValueType>();
-#pragma omp parallel for reduction(add : hessenberg_iter_entry)
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                hessenberg_iter_entry +=
-                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
-            }
-            hessenberg_iter->at(k, i) = hessenberg_iter_entry;
-        }
-        // for i in 1:iter
-        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-        // end
-        for (size_type k = 0; k < iter + 1; ++k) {
-#pragma omp parallel for
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
-            }
-        }
-        // for i in 1:iter
-        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
-        // end
-        // ValueType nrm = zero<ValueType>();
-        nrm = zero<remove_complex<ValueType>>();
-#pragma omp parallel for reduction(addnc : nrm)
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            nrm += squared_norm(next_krylov_basis->at(j, i));
-        }
-        arnoldi_norm->at(1, i) = nrm;
-        // nrmN = norm(next_krylov_basis)
-
-        // DONE: TODO remove the abs() as soon as the type of arnoldi_norm has
-        // changed
-        for (size_type l = 1;
-             (arnoldi_norm->at(1, i)) < (arnoldi_norm->at(0, i)) && l < 3;
-             // abs(arnoldi_norm->at(1, i)) < abs(arnoldi_norm->at(0, i)) && l <
-             // 3;
-             l++) {
-            arnoldi_norm->at(0, i) = arnoldi_norm->at(1, i) * eta;
-            // nrmP = nrmN
-            for (size_type k = 0; k < iter + 1; ++k) {
-                ValueType hessenberg_iter_entry = zero<ValueType>();
-#pragma omp parallel for reduction(add : hessenberg_iter_entry)
-                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
-                     ++j) {
-                    hessenberg_iter_entry += next_krylov_basis->at(j, i) *
-                                             krylov_bases.read(k, j, i);
-                }
-                buffer_iter->at(k, i) = hessenberg_iter_entry;
-            }
-            // for i in 1:iter
-            //     buffer(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-            // end
-            for (size_type k = 0; k < iter + 1; ++k) {
-#pragma omp parallel for
-                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
-                     ++j) {
-                    next_krylov_basis->at(j, i) -=
-                        buffer_iter->at(k, i) * krylov_bases.read(k, j, i);
-                }
-            }
-            // for i in 1:iter
-            //     next_krylov_basis  -= buffer(iter, i) * krylov_bases(:, i)
-            // end
-            remove_complex<ValueType> nrm = zero<remove_complex<ValueType>>();
-#pragma omp parallel for reduction(addnc : nrm)
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                nrm += squared_norm(next_krylov_basis->at(j, i));
-            }
-            arnoldi_norm->at(1, i) = nrm;
-            // nrmN = norm(next_krylov_basis)
-        }
-        // reorthogonalization
-
-        ValueType hessenberg_iter_entry = zero<ValueType>();
-#pragma omp parallel for reduction(add : hessenberg_iter_entry)
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            hessenberg_iter_entry +=
-                next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
-        }
-        hessenberg_iter->at(iter + 1, i) = sqrt(hessenberg_iter_entry);
-        // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
-#pragma omp parallel for
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(iter + 1, j, i, next_krylov_basis->at(j, i));
-        }
-        // next_krylov_basis /= hessenberg(iter, iter + 1)
-        // krylov_bases(:, iter + 1) = next_krylov_basis
-        // End of arnoldi
-    }
-}
-
-
 template <typename ValueType, typename Accessor3d>
 void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                          Accessor3d krylov_bases,
@@ -343,8 +83,6 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
 #pragma omp declare reduction(addnc : remove_complex<ValueType> : omp_out = omp_out + omp_in)
 #pragma omp declare reduction(infnc : remove_complex<ValueType> : omp_out = (omp_out >= omp_in? omp_out: omp_in))
-    //#pragma omp declare reduction(infnc : remove_complex<ValueType> : omp_out
-    //= (omp_out >= abs(omp_in)? omp_out: abs(omp_in)))
 
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
         if (stop_status[i].has_stopped()) {
@@ -415,16 +153,11 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         }
         arnoldi_norm->at(1, i) = sqrt(nrm);
         arnoldi_norm->at(2, i) = inf;
-        // std::cout << sqrt(arnoldi_norm->at(1, i)) << " - "
-        //           << arnoldi_norm->at(2, i) << std::endl;
         // nrmI = infnorm(next_krylov_basis)
         // nrmN = norm(next_krylov_basis)
 
-        // DONE: TODO remove abs() as soon as type of arnoldi_norm has changed!
         for (size_type l = 1;
              (arnoldi_norm->at(1, i)) < (arnoldi_norm->at(0, i)) && l < 3;
-             // abs(arnoldi_norm->at(1, i)) < abs(arnoldi_norm->at(0, i)) && l <
-             // 3;
              l++) {
             arnoldi_norm->at(0, i) = arnoldi_norm->at(1, i) * eta;
             // nrmP = nrmN
@@ -470,15 +203,6 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
             krylov_bases, iter + 1, i,
             arnoldi_norm->at(2, i) / arnoldi_norm->at(1, i));
         // reorthogonalization
-        /*
-        ValueType hessenberg_iter_entry = zero<ValueType>();
-#pragma omp parallel for reduction(add : hessenberg_iter_entry)
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            hessenberg_iter_entry += squared_norm(next_krylov_basis->at(j, i));
-            // next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
-        }
-        hessenberg_iter->at(iter + 1, i) = sqrt(hessenberg_iter_entry);
-        */
         hessenberg_iter->at(iter + 1, i) = (arnoldi_norm->at(1, i));
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
 #pragma omp parallel for
@@ -686,16 +410,12 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for reduction(addnc : res_norm) reduction(infnc : res_inf)
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             res_norm += squared_norm(residual->at(i, j));
-            // res_norm += residual->at(i, j) * residual->at(i, j);
             res_inf = (res_inf >= abs(residual->at(i, j)))
                           ? res_inf
                           : abs(residual->at(i, j));
         }
         residual_norm->at(0, j) = sqrt(res_norm);
         arnoldi_norm->at(2, j) = res_inf;
-        // std::cout << residual_norm->at(0, j) << " - " << arnoldi_norm->at(2,
-        // j)
-        //           << std::endl;
         helper_functions_accessor<Accessor3d>::write_scale(
             krylov_bases, {0}, j,
             arnoldi_norm->at(2, j) / residual_norm->at(0, j));
@@ -757,25 +477,9 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
         final_iter_nums->get_data()[i] +=
             (1 - stop_status->get_const_data()[i].has_stopped());
     }
-#if FINISH_ARNOLDI == 1
-    //    std::cout << "OMP MGS_REORTH" << std::endl;
-    finish_arnoldi_reorth(next_krylov_basis, krylov_bases, hessenberg_iter,
-                          arnoldi_norm, iter, stop_status->get_const_data());
-#elif FINISH_ARNOLDI == 2
-    //    std::cout << "OMP CGS_REORTH" << std::endl;
-    finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
-                       buffer_iter, arnoldi_norm, iter,
-                       stop_status->get_const_data());
-#elif FINISH_ARNOLDI == 3
-    //    std::cout << "OMP CGS_REORTH_2" << std::endl;
     finish_arnoldi_CGS2(next_krylov_basis, krylov_bases, hessenberg_iter,
                         buffer_iter, arnoldi_norm, iter,
                         stop_status->get_const_data());
-#else
-    //    std::cout << "OMP MGS" << std::endl;
-    finish_arnoldi(next_krylov_basis, krylov_bases, hessenberg_iter, iter,
-                   stop_status->get_const_data());
-#endif
     givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
                     iter, stop_status->get_const_data());
     calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index 5f495f4a33e..f25c830494b 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -59,261 +59,6 @@ namespace gmres_mixed {
 namespace {
 
 
-template <typename ValueType, typename Accessor3d>
-void finish_arnoldi(matrix::Dense<ValueType> *next_krylov_basis,
-                    Accessor3d krylov_bases,
-                    matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
-                    const stopping_status *stop_status)
-{
-    static_assert(
-        std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
-        "ValueType must match arithmetic_type of accessor!");
-    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
-        if (stop_status[i].has_stopped()) {
-            continue;
-        }
-        for (size_type k = 0; k < iter + 1; ++k) {
-            hessenberg_iter->at(k, i) = zero<ValueType>();
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                hessenberg_iter->at(k, i) +=
-                    next_krylov_basis->at(j, i) * krylov_bases.at(k, j, i);
-            }
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) * krylov_bases.at(k, j, i);
-            }
-        }
-        // for i in 1:iter
-        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
-        // end
-
-        hessenberg_iter->at(iter + 1, i) = zero<ValueType>();
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            hessenberg_iter->at(iter + 1, i) +=
-                squared_norm(next_krylov_basis->at(j, i));
-            // next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
-        }
-        hessenberg_iter->at(iter + 1, i) =
-            sqrt(hessenberg_iter->at(iter + 1, i));
-        // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.at((iter + 1), j, i) = next_krylov_basis->at(j, i);
-            // std::cout << next_krylov_basis->at(j, i) << " - "
-            //           << krylov_bases.read(
-            //                  j,
-            //                  next_krylov_basis->get_size()[1] * (iter + 1) +
-            //                  i)
-            //           << std::endl;
-        }
-        // next_krylov_basis /= hessenberg(iter, iter + 1)
-        // krylov_bases(:, iter + 1) = next_krylov_basis
-        // End of arnoldi
-    }
-}
-
-
-template <typename ValueType, typename Accessor3d>
-void finish_arnoldi_reorth(
-    matrix::Dense<ValueType> *next_krylov_basis, Accessor3d krylov_bases,
-    matrix::Dense<ValueType> *hessenberg_iter,
-    matrix::Dense<ValueType> *buffer_iter,
-    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
-    const stopping_status *stop_status)
-{
-    static_assert(
-        std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
-        "ValueType must match arithmetic_type of accessor!");
-    //    ValueType nrm = 0, reorth = 0;
-
-    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
-        if (stop_status[i].has_stopped()) {
-            continue;
-        }
-        // nrm = 0;
-        arnoldi_norm->at(0, i) = zero<remove_complex<ValueType>>();
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            arnoldi_norm->at(0, i) += squared_norm(next_krylov_basis->at(j, i));
-            // nrm += next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
-        }
-        arnoldi_norm->at(0, i) = sqrt(arnoldi_norm->at(0, i)) * 0.99;
-        // nrm = sqrt(nrm) * 0.99;
-        // nrm = norm(next_krylov_basis)
-        for (size_type k = 0; k < iter + 1; ++k) {
-            hessenberg_iter->at(k, i) = zero<ValueType>();
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                hessenberg_iter->at(k, i) +=
-                    next_krylov_basis->at(j, i) * krylov_bases.at(k, j, i);
-            }
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) * krylov_bases.at(k, j, i);
-            }
-            // if (squared_norm(hessenberg_iter->at(k, i) *
-            //                  hessenberg_iter->at(k, i)) >
-            if (squared_norm(hessenberg_iter->at(k, i)) >
-                arnoldi_norm->at(0, i)) {
-                buffer_iter->at(k, i) = zero<ValueType>();
-                // arnoldi_norm->at(1, i) = 0;
-                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
-                     ++j) {
-                    // DONE: TODO find good alternative here, so the type of
-                    // DONE: arnoldi_norm can change to
-                    // remove_complex<ValueType>!!! arnoldi_norm->at(1, i) +=
-                    buffer_iter->at(k, i) +=
-                        next_krylov_basis->at(j, i) * krylov_bases.at(k, j, i);
-                }
-                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
-                     ++j) {
-                    next_krylov_basis->at(j, i) -= buffer_iter->at(k, i) *
-                                                   // arnoldi_norm->at(1, i) *
-                                                   krylov_bases.at(k, j, i);
-                }
-                // hessenberg_iter->at(k, i) += arnoldi_norm->at(1, i);
-                hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
-            }
-        }
-        // for i in 1:iter
-        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
-        //     if (hessenberg(iter, i)*hessenberg(iter, i)>nrm*0.99)
-        //         reorth = next_krylov_basis' * krylov_bases(:, i)
-        //         next_krylov_basis  -= reorth * krylov_bases(:, i)
-        //         hessenberg(iter, i) += reorth;
-        //     end
-        // end
-
-        hessenberg_iter->at(iter + 1, i) = zero<ValueType>();
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            hessenberg_iter->at(iter + 1, i) +=
-                squared_norm(next_krylov_basis->at(j, i));
-            // next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
-        }
-        hessenberg_iter->at(iter + 1, i) =
-            sqrt(hessenberg_iter->at(iter + 1, i));
-        // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.at(iter + 1, j, i) = next_krylov_basis->at(j, i);
-        }
-        // next_krylov_basis /= hessenberg(iter, iter + 1)
-        // krylov_bases(:, iter + 1) = next_krylov_basis
-        // End of arnoldi
-    }
-}
-
-
-template <typename ValueType, typename Accessor3d>
-void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
-                        Accessor3d krylov_bases,
-                        matrix::Dense<ValueType> *hessenberg_iter,
-                        matrix::Dense<ValueType> *buffer_iter,
-                        matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
-                        size_type iter, const stopping_status *stop_status)
-{
-    static_assert(
-        std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
-        "ValueType must match arithmetic_type of accessor!");
-    const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
-    //    ValueType nrmP = 0, nrmN = 0;
-    //    ValueType nrmP = 0, nrmN = 0;
-    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
-        arnoldi_norm->at(0, i) = zero<remove_complex<ValueType>>();
-        // nrmP = 0;
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            arnoldi_norm->at(0, i) += squared_norm(next_krylov_basis->at(j, i));
-            // nrmP += next_krylov_basis->at(j, i) * next_krylov_basis->at(j,
-            // i);
-        }
-        arnoldi_norm->at(0, i) = sqrt(arnoldi_norm->at(0, i)) * eta;
-        // nrmP = sqrt(nrmP);
-        // arnoldi_norm->at(0, i) = norm(next_krylov_basis)
-        if (stop_status[i].has_stopped()) {
-            continue;
-        }
-        for (size_type k = 0; k < iter + 1; ++k) {
-            hessenberg_iter->at(k, i) = zero<ValueType>();
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                hessenberg_iter->at(k, i) +=
-                    next_krylov_basis->at(j, i) * krylov_bases.at(k, j, i);
-            }
-        }
-        // for i in 1:iter
-        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-        // end
-        for (size_type k = 0; k < iter + 1; ++k) {
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) * krylov_bases.at(k, j, i);
-            }
-        }
-        // for i in 1:iter
-        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
-        // end
-        arnoldi_norm->at(1, i) = zero<remove_complex<ValueType>>();
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            arnoldi_norm->at(1, i) += squared_norm(next_krylov_basis->at(j, i));
-        }
-        arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
-        // nrmN = norm(next_krylov_basis)
-
-        for (size_type l = 1;
-             (arnoldi_norm->at(1, i)) < (arnoldi_norm->at(0, i)) && l < 3;
-             l++) {
-            arnoldi_norm->at(0, i) = eta * arnoldi_norm->at(1, i);
-            for (size_type k = 0; k < iter + 1; ++k) {
-                buffer_iter->at(k, i) = zero<ValueType>();
-                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
-                     ++j) {
-                    buffer_iter->at(k, i) +=
-                        next_krylov_basis->at(j, i) * krylov_bases.at(k, j, i);
-                }
-            }
-            // for i in 1:iter
-            //     buffer(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-            // end
-            for (size_type k = 0; k < iter + 1; ++k) {
-                for (size_type j = 0; j < next_krylov_basis->get_size()[0];
-                     ++j) {
-                    next_krylov_basis->at(j, i) -=
-                        buffer_iter->at(k, i) * krylov_bases.at(k, j, i);
-                }
-                hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
-            }
-            // for i in 1:iter
-            //     next_krylov_basis   -= buffer(iter, i) * krylov_bases(:, i)
-            //     hessenberg(iter, i) += buffer(iter, i)
-            // end
-            arnoldi_norm->at(1, i) = zero<remove_complex<ValueType>>();
-            for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                arnoldi_norm->at(1, i) +=
-                    squared_norm(next_krylov_basis->at(j, i));
-            }
-            arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
-            // nrmN = norm(next_krylov_basis)
-        }
-        // reorthogonalization
-        hessenberg_iter->at(iter + 1, i) = zero<ValueType>();
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            hessenberg_iter->at(iter + 1, i) +=
-                squared_norm(next_krylov_basis->at(j, i));
-            // next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
-        }
-        hessenberg_iter->at(iter + 1, i) =
-            sqrt(hessenberg_iter->at(iter + 1, i));
-        // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.at(iter + 1, j, i) = next_krylov_basis->at(j, i);
-        }
-        // next_krylov_basis /= hessenberg(iter, iter + 1)
-        // krylov_bases(:, iter + 1) = next_krylov_basis
-        // End of arnoldi
-    }
-}
-
-
 template <typename ValueType, typename Accessor3d>
 void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                          Accessor3d krylov_bases,
@@ -326,18 +71,12 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
         "ValueType must match arithmetic_type of accessor!");
     const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
-    //    ValueType nrmP = 0, nrmN = 0;
-    //    ValueType nrmP = 0, nrmN = 0;
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
         arnoldi_norm->at(0, i) = zero<remove_complex<ValueType>>();
-        // nrmP = 0;
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             arnoldi_norm->at(0, i) += squared_norm(next_krylov_basis->at(j, i));
-            // nrmP += next_krylov_basis->at(j, i) * next_krylov_basis->at(j,
-            // i);
         }
         arnoldi_norm->at(0, i) = sqrt(arnoldi_norm->at(0, i)) * eta;
-        // nrmP = sqrt(nrmP);
         // arnoldi_norm->at(0, i) = norm(next_krylov_basis)
         if (stop_status[i].has_stopped()) {
             continue;
@@ -371,15 +110,10 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                     : abs(next_krylov_basis->at(j, i));
         }
         arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
-        // std::cout << arnoldi_norm->at(1, i) << " - " << arnoldi_norm->at(2,
-        // i)
-        //           << " - " << 0 << std::endl;
-        // nrmN = norm(next_krylov_basis)
 
         for (size_type l = 1;
              (arnoldi_norm->at(1, i)) < (arnoldi_norm->at(0, i)) && l < 3;
              l++) {
-            // std::cout << "REF RESTART " << l << std::endl;
             arnoldi_norm->at(0, i) = eta * arnoldi_norm->at(1, i);
             for (size_type k = 0; k < iter + 1; ++k) {
                 buffer_iter->at(k, i) = zero<ValueType>();
@@ -415,28 +149,12 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                         : abs(next_krylov_basis->at(j, i));
             }
             arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
-            // std::cout << arnoldi_norm->at(1, i) << " - "
-            //           << arnoldi_norm->at(2, i) << " - " << l << std::endl;
             // nrmN = norm(next_krylov_basis)
         }
         helper_functions_accessor<Accessor3d>::write_scale(
             krylov_bases, iter + 1, i,
             arnoldi_norm->at(2, i) / arnoldi_norm->at(1, i));
-        /*
-        // reorthogonalization
-        hessenberg_iter->at(iter + 1, i) = zero<ValueType>();
-        for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-            hessenberg_iter->at(iter + 1, i) +=
-                squared_norm(next_krylov_basis->at(j, i));
-            // next_krylov_basis->at(j, i) * next_krylov_basis->at(j, i);
-        }
-        hessenberg_iter->at(iter + 1, i) =
-            sqrt(hessenberg_iter->at(iter + 1, i));
-        */
         hessenberg_iter->at(iter + 1, i) = arnoldi_norm->at(1, i);
-        // std::cout << arnoldi_norm->at(1, i) << " - " << arnoldi_norm->at(2,
-        // i)
-        //           << " - " << hessenberg_iter->at(iter + 1, i) << std::endl;
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
@@ -594,7 +312,6 @@ void initialize_1(std::shared_ptr<const ReferenceExecutor> exec,
         b_norm->at(0, j) = zero<remove_complex<ValueType>>();
         for (size_type i = 0; i < b->get_size()[0]; ++i) {
             b_norm->at(0, j) += squared_norm(b->at(i, j));
-            // b_norm->at(0, j) += b->at(i, j) * b->at(i, j);
         }
         b_norm->at(0, j) = sqrt(b_norm->at(0, j));
 
@@ -632,17 +349,12 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
         arnoldi_norm->at(2, j) = zero<remove_complex<ValueType>>();
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             residual_norm->at(0, j) += squared_norm(residual->at(i, j));
-            // residual_norm->at(0, j) += residual->at(i, j) * residual->at(i,
-            // j);
             arnoldi_norm->at(2, j) =
                 (arnoldi_norm->at(2, j) >= abs(residual->at(i, j)))
                     ? arnoldi_norm->at(2, j)
                     : abs(residual->at(i, j));
         }
         residual_norm->at(0, j) = sqrt(residual_norm->at(0, j));
-        // std::cout << residual_norm->at(0, j) << " - " << arnoldi_norm->at(2,
-        // j)
-        //           << std::endl;
         helper_functions_accessor<Accessor3d>::write_scale(
             krylov_bases, {0}, j,
             arnoldi_norm->at(2, j) / residual_norm->at(0, j));
@@ -699,37 +411,13 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
     static_assert(
         std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
         "ValueType must match arithmetic_type of accessor!");
-    /*
-        if (Reorthogonalization) std::cout << "Reorthogonalization";
-        else  std::cout << "No Reorthogonalization";
-        if (MGS_CGS) std::cout << "Modified Gram-Schmidt";
-        else  std::cout << "Classical Gram-Schmidt";
-    */
     for (size_type i = 0; i < final_iter_nums->get_num_elems(); ++i) {
         final_iter_nums->get_data()[i] +=
             (1 - stop_status->get_const_data()[i].has_stopped());
     }
-#if FINISH_ARNOLDI == 1
-    //    std::cout << "REF MGS_REORTH" << std::endl;
-    finish_arnoldi_reorth(next_krylov_basis, krylov_bases, hessenberg_iter,
-                          buffer_iter, arnoldi_norm, iter,
-                          stop_status->get_const_data());
-
-#elif FINISH_ARNOLDI == 2
-    //    std::cout << "REF CGS_REORTH" << std::endl;
-    finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
-                       buffer_iter, arnoldi_norm, iter,
-                       stop_status->get_const_data());
-#elif FINISH_ARNOLDI == 3
-    //    std::cout << "REF CGS_REORTH_2" << std::endl;
     finish_arnoldi_CGS2(next_krylov_basis, krylov_bases, hessenberg_iter,
                         buffer_iter, arnoldi_norm, iter,
                         stop_status->get_const_data());
-#else
-    //    std::cout << "REF MGS" << std::endl;
-    finish_arnoldi(next_krylov_basis, krylov_bases, hessenberg_iter, iter,
-                   stop_status->get_const_data());
-#endif
     givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
                     iter, stop_status->get_const_data());
     calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,
@@ -737,7 +425,6 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
                                  stop_status->get_const_data());
 }
 
-// GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(GKO_DECLARE_GMRES_MIXED_BOOL_STEP_1_KERNEL);
 GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
 

From 1305c006a518f4a25cccbe43ce944aace1c8c45b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 17 Aug 2020 12:25:13 +0200
Subject: [PATCH 0375/1008] Adopt CUDA to the new accessor format (NOT `at`)

---
 cuda/solver/gmres_mixed_kernels.cu | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index c6ef53eccd9..02e2b29b198 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -110,6 +110,7 @@ as_cuda_accessor(Accessor3d<Type1, Type2> acc)
             acc.get_stride1(), as_cuda_type(acc.get_scale())};
 }
 
+/*
 template <typename Type1, typename Type2>
 xstd::enable_if_t<!Accessor3dConst<Type1, Type2>::has_scale,
                   Accessor3dConst<cuda_type<Type1>, cuda_type<Type2>>>
@@ -127,6 +128,7 @@ as_cuda_accessor(const Accessor3dConst<Type1, Type2> &acc)
     return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
             acc.get_stride1(), as_cuda_type(acc.get_scale())};
 }
+*/
 
 
 template <typename ValueType>
@@ -328,7 +330,7 @@ void finish_arnoldi_reorth(
     Accessor3d<ValueType, ValueType> next_krylov_accessor{
         next_krylov_basis->get_values(), stride_next_krylov,
         stride_next_krylov};
-    auto next_krylov_const_accessor = next_krylov_accessor.to_const();
+    const auto next_krylov_const_accessor = next_krylov_accessor;
     //    size_type num_reorth;
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<ValueType>());
@@ -448,7 +450,7 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
     Accessor3d<ValueType, ValueType> next_krylov_accessor{
         next_krylov_basis->get_values(), stride_next_krylov,
         stride_next_krylov};
-    auto next_krylov_const_accessor = next_krylov_accessor.to_const();
+    const auto next_krylov_const_accessor = next_krylov_accessor;
 
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<ValueType>());
@@ -648,7 +650,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     Accessor3d<ValueType, ValueType> next_krylov_accessor{
         next_krylov_basis->get_values(), stride_next_krylov,
         stride_next_krylov};
-    auto next_krylov_const_accessor = next_krylov_accessor.to_const();
+    const auto next_krylov_const_accessor = next_krylov_accessor;
 
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<non_complex>());
@@ -688,7 +690,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
             <<<grid_size_num_iters_2, block_size>>>(
                 iter + 1, dim_size[0], dim_size[1],
                 as_cuda_type(next_krylov_basis->get_const_values()),
-                stride_next_krylov, as_cuda_accessor(krylov_bases.to_const()),
+                stride_next_krylov, as_cuda_accessor(krylov_bases),
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
                 as_cuda_type(stop_status));
     } else {
@@ -696,7 +698,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
             <<<grid_size_iters_single, block_size_iters_single>>>(
                 iter + 1, dim_size[0],
                 as_cuda_type(next_krylov_basis->get_const_values()),
-                stride_next_krylov, as_cuda_accessor(krylov_bases.to_const()),
+                stride_next_krylov, as_cuda_accessor(krylov_bases),
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
                 as_cuda_type(stop_status));
     }
@@ -723,7 +725,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
            default_block_size>>>(
             iter + 1, dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
-            as_cuda_accessor(krylov_bases.to_const()),
+            as_cuda_accessor(krylov_bases),
             as_cuda_type(hessenberg_iter->get_const_values()),
             stride_hessenberg, as_cuda_type(stop_status));
 
@@ -858,8 +860,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                 <<<grid_size_num_iters_2, block_size>>>(
                     iter + 1, dim_size[0], dim_size[1],
                     as_cuda_type(next_krylov_basis->get_const_values()),
-                    stride_next_krylov,
-                    as_cuda_accessor(krylov_bases.to_const()),
+                    stride_next_krylov, as_cuda_accessor(krylov_bases),
                     as_cuda_type(buffer_iter->get_values()), stride_buffer,
                     as_cuda_type(stop_status));
         } else {
@@ -867,8 +868,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                 <<<grid_size_iters_single, block_size_iters_single>>>(
                     iter + 1, dim_size[0],
                     as_cuda_type(next_krylov_basis->get_const_values()),
-                    stride_next_krylov,
-                    as_cuda_accessor(krylov_bases.to_const()),
+                    stride_next_krylov, as_cuda_accessor(krylov_bases),
                     as_cuda_type(buffer_iter->get_values()), stride_buffer,
                     as_cuda_type(stop_status));
         }
@@ -881,7 +881,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                default_block_size>>>(
                 iter + 1, dim_size[0], dim_size[1],
                 as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov, as_cuda_accessor(krylov_bases.to_const()),
+                stride_next_krylov, as_cuda_accessor(krylov_bases),
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
                 as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
                 as_cuda_type(stop_status), as_cuda_type(reorth_status));
@@ -1109,8 +1109,8 @@ void solve_upper_triangular(
 }
 
 
-template <typename ValueType, typename Accessor3dConst>
-void calculate_qy(Accessor3dConst krylov_bases, size_type num_krylov_bases,
+template <typename ValueType, typename Accessor3d>
+void calculate_qy(const Accessor3d &krylov_bases, size_type num_krylov_bases,
                   const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const Array<size_type> *final_iter_nums)
@@ -1139,10 +1139,10 @@ void calculate_qy(Accessor3dConst krylov_bases, size_type num_krylov_bases,
 }
 
 
-template <typename ValueType, typename Accessor3dConst>
+template <typename ValueType, typename Accessor3d>
 void step_2(std::shared_ptr<const CudaExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor3dConst krylov_bases,
+            const Accessor3d &krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,

From 04961ca6eb34b98e9480957a777766e3a0a34ae9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 17 Aug 2020 14:22:39 +0200
Subject: [PATCH 0376/1008] Make HIP and CUDA work with new accessor (NOT at)

---
 common/solver/gmres_mixed_kernels.hpp.inc | 14 ++++++++------
 cuda/solver/gmres_mixed_kernels.cu        | 16 +++++++---------
 cuda/test/solver/gmres_mixed_kernels.cpp  | 13 ++++++-------
 hip/solver/gmres_mixed_kernels.hip.cpp    | 13 ++++++-------
 4 files changed, 27 insertions(+), 29 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 7e6b872e3cd..5f308dd9b16 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -1275,6 +1275,7 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
 {
     const remove_complex<ValueType> eta = 1.0 / 2.0;
     const auto col_idx = thread::get_thread_id_flat();
+    constexpr bool has_scale = Accessor3d::has_scale;
 
     if (col_idx < stride_norm && !stop_status[col_idx].has_stopped()) {
         const auto num0 = (eta * arnoldi_norm[col_idx]);
@@ -1309,14 +1310,15 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
 
 // Must be called with at least `num_cols` blocks, each with `block_size`
 // threads. `block_size` must be a power of 2.
-template <int block_size, typename ValueType, typename Accessor3d>
+template <int block_size, typename RealValueType, typename Accessor3d>
 __global__ __launch_bounds__(block_size) void set_scale_kernel(
     size_type num_rhs, size_type num_blocks,
-    const remove_complex<ValueType> *__restrict__ residual_norm,
-    size_type stride_residual,
-    const remove_complex<ValueType> *__restrict__ arnoldi_inf,
-    size_type stride_inf, Accessor3d krylov_bases)
+    const RealValueType *__restrict__ residual_norm, size_type stride_residual,
+    const RealValueType *__restrict__ arnoldi_inf, size_type stride_inf,
+    Accessor3d krylov_bases)
 {
+    static_assert(!is_complex_s<RealValueType>::value,
+                  "ValueType must not be complex!");
     const auto global_id = thread::get_thread_id_flat();
     const auto blk_idx = global_id / krylov_bases.get_stride1();
     const auto col_idx = global_id % krylov_bases.get_stride1();
@@ -1329,7 +1331,7 @@ __global__ __launch_bounds__(block_size) void set_scale_kernel(
             helper_functions_accessor<Accessor3d>::write_scale(
                 krylov_bases, {0}, col_idx, num2 / num1);
         } else {
-            const auto num = one<remove_complex<ValueType>>();
+            const auto num = one<RealValueType>();
             helper_functions_accessor<Accessor3d>::write_scale(
                 krylov_bases, blk_idx, col_idx, num);
         }
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 02e2b29b198..b474722e18a 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -306,10 +306,10 @@ void finish_arnoldi(std::shared_ptr<const CudaExecutor> exec,
 }
 
 
-template <typename ValueType, typename Accessor3d>
+template <typename ValueType, typename Accessor3dim>
 void finish_arnoldi_reorth(
     std::shared_ptr<const CudaExecutor> exec,
-    matrix::Dense<ValueType> *next_krylov_basis, Accessor3d krylov_bases,
+    matrix::Dense<ValueType> *next_krylov_basis, Accessor3dim krylov_bases,
     matrix::Dense<ValueType> *hessenberg_iter,
     matrix::Dense<ValueType> *buffer_iter,
     matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
@@ -426,10 +426,10 @@ void finish_arnoldi_reorth(
 }
 
 
-template <typename ValueType, typename Accessor3d>
+template <typename ValueType, typename Accessor3dim>
 void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
                         matrix::Dense<ValueType> *next_krylov_basis,
-                        Accessor3d krylov_bases,
+                        Accessor3dim krylov_bases,
                         matrix::Dense<ValueType> *hessenberg_iter,
                         matrix::Dense<ValueType> *buffer_iter,
                         matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
@@ -607,10 +607,10 @@ void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
 /**/
 
 
-template <typename ValueType, typename Accessor3d>
+template <typename ValueType, typename Accessor3dim>
 void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                          matrix::Dense<ValueType> *next_krylov_basis,
-                         Accessor3d krylov_bases,
+                         Accessor3dim krylov_bases,
                          matrix::Dense<ValueType> *hessenberg_iter,
                          matrix::Dense<ValueType> *buffer_iter,
                          matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
@@ -622,8 +622,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     using non_complex = remove_complex<ValueType>;
     // optimization parameter
     constexpr int singledot_block_size = default_dot_dim;
-    constexpr bool use_scale =
-        Accessor3d<ValueTypeKrylovBases, ValueType>::has_scale;
+    constexpr bool use_scale = Accessor3dim::has_scale;
     const auto stride_next_krylov = next_krylov_basis->get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
     const auto stride_buffer = buffer_iter->get_stride();
@@ -650,7 +649,6 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     Accessor3d<ValueType, ValueType> next_krylov_accessor{
         next_krylov_basis->get_values(), stride_next_krylov,
         stride_next_krylov};
-    const auto next_krylov_const_accessor = next_krylov_accessor;
 
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<non_complex>());
diff --git a/cuda/test/solver/gmres_mixed_kernels.cpp b/cuda/test/solver/gmres_mixed_kernels.cpp
index 28d3701c1a9..6520fdec23a 100644
--- a/cuda/test/solver/gmres_mixed_kernels.cpp
+++ b/cuda/test/solver/gmres_mixed_kernels.cpp
@@ -109,9 +109,8 @@ class GmresMixed : public ::testing::Test {
             for (size_type k = 0; k < size[0]; ++k) {
                 for (size_type i = 0; i < size[2]; ++i) {
                     gko::kernels::helper_functions_accessor<
-                        value_type, krylov_type>::write_scale(accessor, k, i,
-                                                              dist(
-                                                                  rand_engine));
+                        Accessor3d>::write_scale(accessor, k, i,
+                                                 dist(rand_engine));
                 }
             }
         }
@@ -359,13 +358,13 @@ TEST_F(GmresMixed, CudaGmresMixedStep2IsEquivalentToRef)
     initialize_data();
 
     gko::kernels::reference::gmres_mixed::step_2(
-        ref, residual_norm_collection.get(), krylov_bases_accessor.to_const(),
+        ref, residual_norm_collection.get(), krylov_bases_accessor,
         hessenberg.get(), y.get(), before_preconditioner.get(),
         final_iter_nums.get());
     gko::kernels::cuda::gmres_mixed::step_2(
-        cuda, d_residual_norm_collection.get(),
-        d_krylov_bases_accessor.to_const(), d_hessenberg.get(), d_y.get(),
-        d_before_preconditioner.get(), d_final_iter_nums.get());
+        cuda, d_residual_norm_collection.get(), d_krylov_bases_accessor,
+        d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
+        d_final_iter_nums.get());
 
     GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
diff --git a/hip/solver/gmres_mixed_kernels.hip.cpp b/hip/solver/gmres_mixed_kernels.hip.cpp
index 311c03fa889..33282abcdef 100644
--- a/hip/solver/gmres_mixed_kernels.hip.cpp
+++ b/hip/solver/gmres_mixed_kernels.hip.cpp
@@ -103,12 +103,12 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void initialize_2(std::shared_ptr<const HipExecutor> exec,
                   const matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
-                  Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
+                  Accessor3d krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums,
                   size_type krylov_dim) GKO_NOT_IMPLEMENTED;
@@ -117,15 +117,14 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void step_1(std::shared_ptr<const HipExecutor> exec,
             matrix::Dense<ValueType> *next_krylov_basis,
             matrix::Dense<ValueType> *givens_sin,
             matrix::Dense<ValueType> *givens_cos,
             matrix::Dense<ValueType> *residual_norm,
             matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor2d<ValueTypeKrylovBases, ValueType> krylov_bases,
-            matrix::Dense<ValueType> *hessenberg_iter,
+            Accessor3d krylov_bases, matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
             const matrix::Dense<ValueType> *b_norm,
             matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
@@ -138,10 +137,10 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
+template <typename ValueType, typename Accessor3d>
 void step_2(std::shared_ptr<const HipExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            Accessor2dConst<ValueTypeKrylovBases, ValueType> krylov_bases,
+            const Accessor3d &krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,

From 8e0effcc303cfa634aaaf0fde86bc44a1f9796f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 17 Aug 2020 14:49:01 +0200
Subject: [PATCH 0377/1008] Remove unused code from CUDA

---
 core/solver/gmres_mixed_kernels.hpp |   3 -
 cuda/solver/gmres_mixed_kernels.cu  | 380 ----------------------------
 2 files changed, 383 deletions(-)

diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index 72de109ba36..1855ad1ae6f 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -45,9 +45,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/gmres_mixed_accessor.hpp"
 
 
-#define FINISH_ARNOLDI 3
-
-
 /**
  * Instantiates a template for each value type with each lower precision type
  * supported by Ginkgo for GmresMixed.
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index b474722e18a..b19f2027619 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -246,367 +246,6 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
 
 
-template <typename ValueType, typename Accessor3d>
-void finish_arnoldi(std::shared_ptr<const CudaExecutor> exec,
-                    matrix::Dense<ValueType> *next_krylov_basis,
-                    Accessor3d krylov_bases,
-                    matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
-                    const stopping_status *stop_status)
-{
-    const auto stride_next_krylov = next_krylov_basis->get_stride();
-    const auto stride_hessenberg = hessenberg_iter->get_stride();
-    const auto dim_size = next_krylov_basis->get_size();
-    auto cublas_handle = exec->get_cublas_handle();
-    const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
-                         exec->get_num_multiprocessor() * 2);
-    const dim3 block_size(default_dot_dim, default_dot_dim);
-    for (size_type k = 0; k < iter + 1; ++k) {
-        components::fill_array(
-            exec, hessenberg_iter->get_values() + k * stride_hessenberg,
-            dim_size[1], zero<ValueType>());
-        multidot_kernel<<<grid_size, block_size>>>(
-            k, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_accessor(krylov_bases),
-            as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-            as_cuda_type(stop_status));
-        update_next_krylov_kernel<default_block_size>
-            <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
-               default_block_size>>>(
-                k, dim_size[0], dim_size[1],
-                as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov, as_cuda_accessor(krylov_bases),
-                as_cuda_type(hessenberg_iter->get_const_values()),
-                stride_hessenberg, as_cuda_type(stop_status));
-    }
-    // for i in 1:iter
-    //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-    //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
-    // end
-
-
-    update_hessenberg_2_kernel<default_block_size>
-        <<<dim_size[1], default_block_size>>>(
-            iter, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_type(hessenberg_iter->get_values()),
-            stride_hessenberg, as_cuda_type(stop_status));
-
-    update_krylov_next_krylov_kernel<default_block_size>
-        <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
-           default_block_size>>>(
-            iter, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
-            as_cuda_accessor(krylov_bases),
-            as_cuda_type(hessenberg_iter->get_const_values()),
-            stride_hessenberg, as_cuda_type(stop_status));
-    // next_krylov_basis /= hessenberg(iter, iter + 1)
-    // krylov_bases(:, iter + 1) = next_krylov_basis
-    // End of arnoldi
-}
-
-
-template <typename ValueType, typename Accessor3dim>
-void finish_arnoldi_reorth(
-    std::shared_ptr<const CudaExecutor> exec,
-    matrix::Dense<ValueType> *next_krylov_basis, Accessor3dim krylov_bases,
-    matrix::Dense<ValueType> *hessenberg_iter,
-    matrix::Dense<ValueType> *buffer_iter,
-    matrix::Dense<remove_complex<ValueType>> *arnoldi_norm, size_type iter,
-    const stopping_status *stop_status, stopping_status *reorth_status,
-    Array<size_type> *num_reorth)
-{
-    const auto stride_next_krylov = next_krylov_basis->get_stride();
-    const auto stride_hessenberg = hessenberg_iter->get_stride();
-    const auto stride_buffer = buffer_iter->get_stride();
-    const auto stride_arnoldi = arnoldi_norm->get_stride();
-    const auto dim_size = next_krylov_basis->get_size();
-    auto cublas_handle = exec->get_cublas_handle();
-    const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
-                         exec->get_num_multiprocessor() * 2);
-    const dim3 block_size(default_dot_dim, default_dot_dim);
-    size_type numReorth;
-    // Only 2D accessor, modeled as a 3D accessor:
-    Accessor3d<ValueType, ValueType> next_krylov_accessor{
-        next_krylov_basis->get_values(), stride_next_krylov,
-        stride_next_krylov};
-    const auto next_krylov_const_accessor = next_krylov_accessor;
-    //    size_type num_reorth;
-    components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
-                           zero<ValueType>());
-    /*
-    multidot_kernel<<<grid_size, block_size>>>(
-        0, dim_size[0], dim_size[1],
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_accessor(next_krylov_const_accessor),
-        // as_cuda_type(next_krylov_basis->get_const_values()),
-    stride_next_krylov, as_cuda_type(arnoldi_norm->get_values()), 0,
-    as_cuda_type(stop_status));
-    */
-    multinorm2_kernel<<<grid_size, block_size>>>(
-        dim_size[0], dim_size[1],
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(arnoldi_norm->get_values()), as_cuda_type(stop_status));
-    // nrm = norm(next_krylov_basis)
-    for (size_type k = 0; k < iter + 1; ++k) {
-        components::fill_array(
-            exec, hessenberg_iter->get_values() + k * stride_hessenberg,
-            dim_size[1], zero<ValueType>());
-        multidot_kernel<<<grid_size, block_size>>>(
-            k, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_accessor(krylov_bases),
-            as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-            as_cuda_type(stop_status));
-        components::fill_array(exec, num_reorth->get_data(), 1,
-                               zero<size_type>());
-        update_next_krylov_kernel_and_check<default_block_size>
-            <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
-               default_block_size>>>(
-                k, dim_size[0], dim_size[1],
-                as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov, as_cuda_accessor(krylov_bases),
-                as_cuda_type(hessenberg_iter->get_const_values()),
-                stride_hessenberg, as_cuda_type(arnoldi_norm->get_values()), 0,
-                as_cuda_type(stop_status), as_cuda_type(reorth_status),
-                as_cuda_type(num_reorth->get_data()));
-        numReorth = 0;
-        exec->get_master()->copy_from(exec.get(), 1,
-                                      num_reorth->get_const_data(), &numReorth);
-        if (numReorth > 0) {
-            components::fill_array(
-                exec, buffer_iter->get_values() + k * stride_buffer,
-                dim_size[1], zero<ValueType>());
-            multidot_kernel<<<grid_size, block_size>>>(
-                k, dim_size[0], dim_size[1],
-                as_cuda_type(next_krylov_basis->get_const_values()),
-                stride_next_krylov, as_cuda_accessor(krylov_bases),
-                as_cuda_type(buffer_iter->get_values()), stride_buffer,
-                as_cuda_type(stop_status));
-            update_next_krylov_kernel_and_add<default_block_size><<<
-                ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
-                default_block_size>>>(
-                k, dim_size[0], dim_size[1],
-                as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov, as_cuda_accessor(krylov_bases),
-                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-                as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
-                as_cuda_type(stop_status), as_cuda_type(reorth_status));
-        }
-    }
-    // for i in 1:iter
-    //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-    //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
-    //     if (hessenberg(iter, i)*hessenberg(iter, i)>nrm*0.99)
-    //         buffer = next_krylov_basis' * krylov_bases(:, i)
-    //         next_krylov_basis  -= buffer * krylov_bases(:, i)
-    //         hessenberg(iter, i) += buffer;
-    //     end
-    // end
-
-
-    update_hessenberg_2_kernel<default_block_size>
-        <<<dim_size[1], default_block_size>>>(
-            iter, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_type(hessenberg_iter->get_values()),
-            stride_hessenberg, as_cuda_type(stop_status));
-
-    update_krylov_next_krylov_kernel<default_block_size>
-        <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
-           default_block_size>>>(
-            iter, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
-            as_cuda_accessor(krylov_bases),
-            as_cuda_type(hessenberg_iter->get_const_values()),
-            stride_hessenberg, as_cuda_type(stop_status));
-    // next_krylov_basis /= hessenberg(iter, iter + 1)
-    // krylov_bases(:, iter + 1) = next_krylov_basis
-    // End of arnoldi
-}
-
-
-template <typename ValueType, typename Accessor3dim>
-void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
-                        matrix::Dense<ValueType> *next_krylov_basis,
-                        Accessor3dim krylov_bases,
-                        matrix::Dense<ValueType> *hessenberg_iter,
-                        matrix::Dense<ValueType> *buffer_iter,
-                        matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
-                        size_type iter, const stopping_status *stop_status,
-                        stopping_status *reorth_status,
-                        Array<size_type> *num_reorth)
-{
-    const auto stride_next_krylov = next_krylov_basis->get_stride();
-    const auto stride_hessenberg = hessenberg_iter->get_stride();
-    const auto stride_buffer = buffer_iter->get_stride();
-    const auto stride_arnoldi = arnoldi_norm->get_stride();
-    const auto dim_size = next_krylov_basis->get_size();
-    auto cublas_handle = exec->get_cublas_handle();
-    const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
-                         exec->get_num_multiprocessor() * 2);
-    const dim3 block_size(default_dot_dim, default_dot_dim);
-    size_type numReorth;
-    Accessor3d<ValueType, ValueType> next_krylov_accessor{
-        next_krylov_basis->get_values(), stride_next_krylov,
-        stride_next_krylov};
-    const auto next_krylov_const_accessor = next_krylov_accessor;
-
-    components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
-                           zero<ValueType>());
-    /*
-    multidot_kernel<<<grid_size, block_size>>>(
-        0, dim_size[0], dim_size[1],
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_accessor(next_krylov_const_accessor),
-        as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
-    */
-    multinorm2_kernel<<<grid_size, block_size>>>(
-        dim_size[0], dim_size[1],
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(arnoldi_norm->get_values()), as_cuda_type(stop_status));
-    // nrmP = norm(next_krylov_basis
-    for (size_type k = 0; k < iter + 1; ++k) {
-        components::fill_array(
-            exec, hessenberg_iter->get_values() + k * stride_hessenberg,
-            dim_size[1], zero<ValueType>());
-        multidot_kernel<<<grid_size, block_size>>>(
-            k, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_accessor(krylov_bases),
-            as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-            as_cuda_type(stop_status));
-    }
-    // for i in 1:iter
-    //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-    // end
-    for (size_type k = 0; k < iter + 1; ++k) {
-        update_next_krylov_kernel<default_block_size>
-            <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
-               default_block_size>>>(
-                k, dim_size[0], dim_size[1],
-                as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov, as_cuda_accessor(krylov_bases),
-                as_cuda_type(hessenberg_iter->get_const_values()),
-                stride_hessenberg, as_cuda_type(stop_status));
-    }
-    // for i in 1:iter
-    //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
-    // end
-    components::fill_array(exec, arnoldi_norm->get_values() + stride_arnoldi,
-                           dim_size[1], zero<ValueType>());
-    /*
-    multidot_kernel<<<grid_size, block_size>>>(
-        0, dim_size[0], dim_size[1],
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_accessor(next_krylov_const_accessor),
-        as_cuda_type(arnoldi_norm->get_values() + stride_arnoldi), 0,
-        as_cuda_type(stop_status));
-    */
-    multinorm2_kernel<<<grid_size, block_size>>>(
-        dim_size[0], dim_size[1],
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
-        as_cuda_type(stop_status));
-    // nrmN = norm(next_krylov_basis)
-    components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
-    check_arnoldi_norms<default_block_size>
-        <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
-            as_cuda_type(arnoldi_norm->get_const_values()), stride_arnoldi,
-            as_cuda_type(arnoldi_norm->get_const_values() + stride_arnoldi),
-            stride_arnoldi, as_cuda_type(stop_status),
-            as_cuda_type(reorth_status), as_cuda_type(num_reorth->get_data()));
-    numReorth = 0;
-    exec->get_master()->copy_from(exec.get(), 1, num_reorth->get_const_data(),
-                                  &numReorth);
-    // numReorth <= number of next_krylov vector to be reorthogonalization
-    for (size_type l = 1; (numReorth > 0) && (l < 3); l++) {
-        for (size_type k = 0; k < iter + 1; ++k) {
-            components::fill_array(
-                exec, buffer_iter->get_values() + k * stride_buffer,
-                dim_size[1], zero<ValueType>());
-            multidot_kernel<<<grid_size, block_size>>>(
-                k, dim_size[0], dim_size[1],
-                as_cuda_type(next_krylov_basis->get_const_values()),
-                stride_next_krylov, as_cuda_accessor(krylov_bases),
-                as_cuda_type(buffer_iter->get_values()), stride_buffer,
-                as_cuda_type(stop_status));
-        }
-        // for i in 1:iter
-        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
-        // end
-        for (size_type k = 0; k < iter + 1; ++k) {
-            update_next_krylov_kernel_and_add<default_block_size><<<
-                ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
-                default_block_size>>>(
-                k, dim_size[0], dim_size[1],
-                as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov, as_cuda_accessor(krylov_bases),
-                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-                as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
-                as_cuda_type(stop_status), as_cuda_type(reorth_status));
-        }
-        // for i in 1:iter
-        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
-        // end
-        components::fill_array(
-            exec, arnoldi_norm->get_values() + stride_arnoldi * (l - 1),
-            dim_size[1], zero<ValueType>());
-        /*
-        multidot_kernel<<<grid_size, block_size>>>(
-            0, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_accessor(next_krylov_const_accessor),
-            as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
-            as_cuda_type(stop_status));
-        */
-        multinorm2_kernel<<<grid_size, block_size>>>(
-            dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov,
-            as_cuda_type(arnoldi_norm->get_values() + stride_arnoldi * (l - 1)),
-            0, as_cuda_type(stop_status));
-        // nrmN = norm(next_krylov_basis)
-        components::fill_array(exec, num_reorth->get_data(), 1,
-                               zero<size_type>());
-        check_arnoldi_norms<default_block_size>
-            <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
-                as_cuda_type(arnoldi_norm->get_const_values() +
-                             stride_arnoldi * (2 - l)),
-                stride_arnoldi,
-                as_cuda_type(arnoldi_norm->get_const_values() +
-                             stride_arnoldi * (l - 1)),
-                stride_arnoldi, as_cuda_type(stop_status),
-                as_cuda_type(reorth_status),
-                as_cuda_type(num_reorth->get_data()));
-        numReorth = 0;
-        exec->get_master()->copy_from(exec.get(), 1,
-                                      num_reorth->get_const_data(), &numReorth);
-        // numReorth <= number of next_krylov vector to be reorthogonalization
-    }
-
-
-    update_hessenberg_2_kernel<default_block_size>
-        <<<dim_size[1], default_block_size>>>(
-            iter, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_type(hessenberg_iter->get_values()),
-            stride_hessenberg, as_cuda_type(stop_status));
-    update_krylov_next_krylov_kernel<default_block_size>
-        <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
-           default_block_size>>>(
-            iter, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_values()), stride_next_krylov,
-            as_cuda_accessor(krylov_bases),
-            as_cuda_type(hessenberg_iter->get_const_values()),
-            stride_hessenberg, as_cuda_type(stop_status));
-    // next_krylov_basis /= hessenberg(iter, iter + 1)
-    // krylov_bases(:, iter + 1) = next_krylov_basis
-    // End of arnoldi
-}
-/**/
-
-
 template <typename ValueType, typename Accessor3dim>
 void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                          matrix::Dense<ValueType> *next_krylov_basis,
@@ -1051,30 +690,11 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
         default_block_size>>>(as_cuda_type(final_iter_nums->get_data()),
                               as_cuda_type(stop_status->get_const_data()),
                               final_iter_nums->get_num_elems());
-#if FINISH_ARNOLDI == 1
-    //    std::cout << "CUDA MGS_REORTH" << '\n';
-    finish_arnoldi_reorth(exec, next_krylov_basis, krylov_bases,
-                          hessenberg_iter, buffer_iter, arnoldi_norm, iter,
-                          stop_status->get_const_data(),
-                          reorth_status->get_data(), num_reorth);
-#elif FINISH_ARNOLDI == 2
-    //    std::cout << "CUDA CGS_REORTH" << '\n';
-    finish_arnoldi_CGS(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
-                       buffer_iter, arnoldi_norm, iter,
-                       stop_status->get_const_data(), reorth_status->get_data(),
-                       num_reorth);
-#elif FINISH_ARNOLDI == 3
-    //    std::cout << "CUDA CGS_REORTH_2" << '\n';
     finish_arnoldi_CGS2(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
                         buffer_iter, arnoldi_norm, iter,
                         stop_status->get_const_data(),
                         reorth_status->get_data(), num_reorth, num_reorth_steps,
                         num_reorth_vectors);
-#else
-    //    std::cout << "CUDA MGS" << '\n';
-    finish_arnoldi(exec, next_krylov_basis, krylov_bases, hessenberg_iter, iter,
-                   stop_status->get_const_data());
-#endif
     givens_rotation(exec, givens_sin, givens_cos, hessenberg_iter,
                     residual_norm, residual_norm_collection, b_norm, iter,
                     stop_status);

From d3cabd3c1c5c3ddf476c5cbf33ee7d16b29bddd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 18 Aug 2020 02:38:26 +0200
Subject: [PATCH 0378/1008] CUDA implementation is now using `at`

---
 common/solver/gmres_mixed_kernels.hpp.inc | 51 +++++++++++------------
 1 file changed, 25 insertions(+), 26 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 5f308dd9b16..1ecb4e8f6a0 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -99,7 +99,7 @@ __global__ __launch_bounds__(block_size) void initialize_2_1_kernel(
     const auto residual_col = global_id % stride_residual_nc;
 
     if (krylov_idx < krylov_dim + 1 && krylov_row < num_rows && rhs < num_rhs) {
-        krylov_bases.write(krylov_idx, krylov_row, rhs, zero<ValueType>());
+        krylov_bases.at(krylov_idx, krylov_row, rhs) = zero<ValueType>();
     }
 
     if (residual_row < krylov_dim + 1 && residual_col < num_rhs) {
@@ -131,7 +131,7 @@ __global__ __launch_bounds__(block_size) void initialize_2_2_kernel(
     if (row_idx < num_rows && col_idx < num_rhs) {
         auto value = residual[row_idx * stride_residual + col_idx] /
                      residual_norm[col_idx];
-        krylov_bases.write({0}, row_idx, col_idx, value);
+        krylov_bases.at({0}, row_idx, col_idx) = value;
         next_krylov_basis[row_idx * stride_next_krylov + col_idx] = value;
     }
 }
@@ -399,7 +399,7 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
              i += default_dot_dim) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
             local_res += next_krylov_basis[next_krylov_idx] *
-                         krylov_bases.read(k, i, col_idx);
+                         krylov_bases.at(k, i, col_idx);
         }
     }
     reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
@@ -448,7 +448,7 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
                  i += default_dot_dim) {
                 const auto next_krylov_idx = i * stride_next_krylov + col_idx;
                 local_res += next_krylov_basis[next_krylov_idx] *
-                             krylov_bases.read(k, i, col_idx);
+                             krylov_bases.at(k, i, col_idx);
             }
         }
         reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
@@ -501,7 +501,7 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_1(
              i += default_dot_dim) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
             local_res += next_krylov_basis[next_krylov_idx] *
-                         krylov_bases.read(k, i, col_idx);
+                         krylov_bases.at(k, i, col_idx);
         }
     }
     reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
@@ -580,7 +580,7 @@ __global__ __launch_bounds__(dot_dim *dot_dim) void multidot_kernel_num_iters_2(
         for (size_type i = start_row; i < end_row; i += blockDim.y) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
             local_res += next_krylov_basis[next_krylov_idx] *
-                         krylov_bases.read(k, i, col_idx);
+                         krylov_bases.at(k, i, col_idx);
         }
     }
     // Transpose local_res, so each warp contains a local_res from the same
@@ -635,7 +635,7 @@ __global__ __launch_bounds__(block_size) void singledot_kernel_num_iters_2(
         for (size_type i = start_row; i < end_row; i += block_size) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
             local_res += next_krylov_basis[next_krylov_idx] *
-                         krylov_bases.read(k, i, col_idx);
+                         krylov_bases.at(k, i, col_idx);
         }
     }
     // Transpose local_res, so each warp contains a local_res from the same
@@ -673,7 +673,7 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel(
 
         next_krylov_basis[next_krylov_idx] -=
             hessenberg_iter[hessenberg_idx] *
-            krylov_bases.read(k, row_idx, col_idx);
+            krylov_bases.at(k, row_idx, col_idx);
     }
 }
 
@@ -704,7 +704,7 @@ __global__
 
             next_krylov_basis[next_krylov_idx] -=
                 hessenberg_iter[hessenberg_idx] *
-                krylov_bases.read(k, row_idx, col_idx);
+                krylov_bases.at(k, row_idx, col_idx);
         }
     }
 }
@@ -736,7 +736,7 @@ __global__
         const auto hessenberg_idx = k * stride_hessenberg + col_idx;
 
         auto value = hessenberg_iter[hessenberg_idx] *
-                     krylov_bases.read(k, row_idx, col_idx);
+                     krylov_bases.at(k, row_idx, col_idx);
         atomic_add(next_krylov_basis + next_krylov_idx, -value);
     }
 }
@@ -771,7 +771,7 @@ __global__
                 const auto hessenberg_idx = i * stride_hessenberg + col_idx;
 
                 value += hessenberg_iter[hessenberg_idx] *
-                         krylov_bases.read(i, row_idx, col_idx);
+                         krylov_bases.at(i, row_idx, col_idx);
             }
             reduction_helper[tidx] = value;
 
@@ -799,7 +799,7 @@ __global__
 
                 next_krylov_basis[next_krylov_idx] -=
                     hessenberg_iter[hessenberg_idx] *
-                    krylov_bases.read(k, row_idx, col_idx);
+                    krylov_bases.at(k, row_idx, col_idx);
             }
         }
     }
@@ -838,7 +838,7 @@ __global__
                     const auto hessenberg_idx = i * stride_hessenberg + col_idx;
 
                     value += hessenberg_iter[hessenberg_idx] *
-                             krylov_bases.read(i, j, col_idx);
+                             krylov_bases.at(i, j, col_idx);
                 }
                 reduction_helper[k * block_size + tidx] = value;
             }
@@ -880,7 +880,7 @@ __global__
 
                 next_krylov_basis[next_krylov_idx] -=
                     hessenberg_iter[hessenberg_idx] *
-                    krylov_bases.read(k, row_idx, col_idx);
+                    krylov_bases.at(k, row_idx, col_idx);
             }
         }
     }
@@ -923,7 +923,7 @@ __global__
                 for (size_type j = row_idx, k = 0;
                      j < num_rows && k < num_blocks; j++, k++) {
                     reduction_helper[k * block_size + tidx] +=
-                        hessenberg_val * krylov_bases.read(i, j, col_idx);
+                        hessenberg_val * krylov_bases.at(i, j, col_idx);
                 }
             }
 
@@ -966,7 +966,7 @@ __global__
 
                 next_krylov_basis[next_krylov_idx] -=
                     hessenberg_iter[hessenberg_idx] *
-                    krylov_bases.read(k, row_idx, col_idx);
+                    krylov_bases.at(k, row_idx, col_idx);
             }
         }
     }
@@ -1009,7 +1009,7 @@ __global__
                 for (size_type j = row_idx, k = 0;
                      j < num_rows && k < num_blocks; j++, k++) {
                     reduction_helper[k * block_size + tidx] +=
-                        hessenberg_val * krylov_bases.read(i, j, col_idx);
+                        hessenberg_val * krylov_bases.at(i, j, col_idx);
                 }
             }
             multireduce(
@@ -1043,7 +1043,7 @@ __global__
 
                 next_krylov_basis[next_krylov_idx] -=
                     hessenberg_iter[hessenberg_idx] *
-                    krylov_bases.read(k, row_idx, col_idx);
+                    krylov_bases.at(k, row_idx, col_idx);
             }
         }
     }
@@ -1078,7 +1078,7 @@ __global__
 
         next_krylov_basis[next_krylov_idx] -=
             hessenberg_iter[hessenberg_idx] *
-            krylov_bases.read(k, row_idx, col_idx);
+            krylov_bases.at(k, row_idx, col_idx);
         if (row_idx == 0) {
             if (squared_norm(hessenberg_iter[hessenberg_idx]) >
                 (0.99 * sqrt(arnoldi_norm[arnoldi_idx]))) {
@@ -1114,7 +1114,7 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
         const auto hessenberg_idx = k * stride_hessenberg + col_idx;
         const auto buffer_idx = k * stride_buffer + col_idx;
         next_krylov_basis[next_krylov_idx] -=
-            buffer_iter[buffer_idx] * krylov_bases.read(k, row_idx, col_idx);
+            buffer_iter[buffer_idx] * krylov_bases.at(k, row_idx, col_idx);
         if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
             hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
         }
@@ -1149,8 +1149,7 @@ __global__
             const auto hessenberg_idx = k * stride_hessenberg + col_idx;
             const auto buffer_idx = k * stride_buffer + col_idx;
             next_krylov_basis[next_krylov_idx] -=
-                buffer_iter[buffer_idx] *
-                krylov_bases.read(k, row_idx, col_idx);
+                buffer_iter[buffer_idx] * krylov_bases.at(k, row_idx, col_idx);
             if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
                 hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
             }
@@ -1189,7 +1188,7 @@ __global__
                 const auto buffer_idx = i * stride_buffer + col_idx;
 
                 value += buffer_iter[buffer_idx] *
-                         krylov_bases.read(i, row_idx, col_idx);
+                         krylov_bases.at(i, row_idx, col_idx);
                 if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
                     hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
                 }
@@ -1223,7 +1222,7 @@ __global__
                 const auto buffer_idx = k * stride_buffer + col_idx;
                 next_krylov_basis[next_krylov_idx] -=
                     buffer_iter[buffer_idx] *
-                    krylov_bases.read(k, row_idx, col_idx);
+                    krylov_bases.at(k, row_idx, col_idx);
                 if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
                     hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
                 }
@@ -1405,7 +1404,7 @@ __global__ __launch_bounds__(block_size) void update_krylov_next_krylov_kernel(
             next_krylov_basis[next_krylov_idx] / hessenberg;
 
         next_krylov_basis[next_krylov_idx] = next_krylov_value;
-        krylov_bases.write(iter + 1, row_idx, col_idx, next_krylov_value);
+        krylov_bases.at(iter + 1, row_idx, col_idx) = next_krylov_value;
     }
 }
 
@@ -1582,7 +1581,7 @@ __global__ __launch_bounds__(block_size) void calculate_Qy_kernel(
         ValueType temp = zero<ValueType>();
         for (size_type j = 0; j < final_iter_nums[col_id]; ++j) {
             temp +=
-                krylov_bases.read(j, row_id, col_id) * y[j * stride_y + col_id];
+                krylov_bases.at(j, row_id, col_id) * y[j * stride_y + col_id];
         }
         before_preconditioner[global_id] = temp;
     }

From aa79b595545c723103e17c9f84e1acfb1acc0ecc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 18 Aug 2020 18:39:20 +0200
Subject: [PATCH 0379/1008] Re-add ConstAccessor

Also add instantiation macro for ConstAccessors
---
 core/device_hooks/common_kernels.inc.cpp |   2 +-
 core/solver/gmres_mixed.cpp              |   8 +-
 core/solver/gmres_mixed_accessor.hpp     | 176 +++++++++++++++++------
 core/solver/gmres_mixed_kernels.hpp      |  23 ++-
 cuda/solver/gmres_mixed_kernels.cu       |  24 ++--
 cuda/test/solver/gmres_mixed_kernels.cpp |   8 +-
 hip/solver/gmres_mixed_kernels.hip.cpp   |   6 +-
 omp/solver/gmres_mixed_kernels.cpp       |  10 +-
 omp/test/solver/gmres_mixed_kernels.cpp  |   8 +-
 reference/solver/gmres_mixed_kernels.cpp |  13 +-
 10 files changed, 194 insertions(+), 84 deletions(-)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index b09326762a6..8eeecc23c96 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -603,7 +603,7 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
 template <typename ValueType, typename ValueTypeKrylovBases>
 GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(ValueType, ValueTypeKrylovBases)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
 
 
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 59f007a491b..3167f4ccaab 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -289,9 +289,9 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
                 span{0, dense_b->get_size()[1] * (restart_iter)});
 
             exec->run(gmres_mixed::make_step_2(
-                residual_norm_collection.get(), krylov_bases_accessor,
-                hessenberg_view.get(), y.get(), before_preconditioner.get(),
-                &final_iter_nums));
+                residual_norm_collection.get(),
+                krylov_bases_accessor.to_const(), hessenberg_view.get(),
+                y.get(), before_preconditioner.get(), &final_iter_nums));
             /* */ /*
             auto hessenberg_small = hessenberg->create_submatrix(
                 span{0, restart_iter},
@@ -428,7 +428,7 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
         span{0, dense_b->get_size()[1] * (restart_iter)});
 
     exec->run(gmres_mixed::make_step_2(
-        residual_norm_collection.get(), krylov_bases_accessor,
+        residual_norm_collection.get(), krylov_bases_accessor.to_const(),
         hessenberg_small.get(), y.get(), before_preconditioner.get(),
         &final_iter_nums));
     // Solve upper triangular.
diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index f8437be9295..a905ab8d3c5 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -89,7 +89,7 @@ template <typename Reference, typename ArithmeticType>
 struct enable_reference {
     using arithmetic_type = ArithmeticType;
 
-    enable_reference() =
+    GKO_ATTRIBUTES enable_reference() =
         default;  // Needs to exist, so creation is not probibited
     enable_reference(enable_reference &&) = delete;
     enable_reference(const enable_reference &) = delete;
@@ -137,6 +137,9 @@ struct enable_reference {
     GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator+=, +)
     GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator-=, -)
 #undef GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD
+
+    // TODO test if comparison operators need to be overloaded as well
+
     friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type
     operator-(const Reference &ref)
     {
@@ -150,7 +153,8 @@ struct enable_reference {
 
 template <typename StorageType, typename ArithmeticType,
           bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
-class Accessor3d {};
+class ConstAccessor3d {};
+
 /**
  * @internal
  *
@@ -167,10 +171,11 @@ class Accessor3d {};
  * The accessor uses row-major access.
  */
 template <typename StorageType, typename ArithmeticType>
-class Accessor3d<StorageType, ArithmeticType, false> {
+class ConstAccessor3d<StorageType, ArithmeticType, false> {
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
+    using const_accessor = ConstAccessor3d;
     static_assert(
         !detail::helper_have_scale<StorageType, ArithmeticType>::value,
         "storage_type must not be an integral in this class.");
@@ -181,7 +186,9 @@ class Accessor3d<StorageType, ArithmeticType, false> {
         : public detail::enable_reference<reference, arithmetic_type> {
     public:
         reference() = delete;
-        reference(storage_type *const GKO_RESTRICT ptr) : ptr_{ptr} {}
+        GKO_ATTRIBUTES reference(storage_type *const GKO_RESTRICT ptr)
+            : ptr_{ptr}
+        {}
         GKO_ATTRIBUTES constexpr GKO_INLINE operator arithmetic_type() const
         {
             return static_cast<arithmetic_type>(*ptr_);
@@ -202,9 +209,7 @@ class Accessor3d<StorageType, ArithmeticType, false> {
     };
 
 public:
-    Accessor3d() : storage_{nullptr}, stride_{0, 0} {}
     /**
-     * TODO update
      * Creates the accessor with an already allocated storage space with a
      * stride.
      *
@@ -213,10 +218,14 @@ class Accessor3d<StorageType, ArithmeticType, false> {
      * works. Also, storage_ is never accessed in a non-const fashion, so it
      * is not invalid or UB code.
      */
-    Accessor3d(storage_type *storage, size_type stride0, size_type stride1)
-        : storage_{storage}, stride_{stride0, stride1}
+    GKO_ATTRIBUTES ConstAccessor3d(const storage_type *storage,
+                                   size_type stride0, size_type stride1)
+        : storage_{const_cast<storage_type *>(storage)},
+          stride_{stride0, stride1}
     {}
 
+    GKO_ATTRIBUTES const_accessor to_const() const { return *this; }
+
     /**
      * Reads the value at the given index.
      */
@@ -248,26 +257,6 @@ class Accessor3d<StorageType, ArithmeticType, false> {
     {
         return storage_;
     }
-    /**
-     * Writes the given value at the given index.
-     */
-    template <typename IndexType>
-    GKO_ATTRIBUTES void write(IndexType x, IndexType y, IndexType z,
-                              arithmetic_type value)
-    {
-        storage_type *GKO_RESTRICT rest_storage = this->storage_;
-        rest_storage[x * this->stride_[0] + y * this->stride_[1] + z] =
-            static_cast<storage_type>(value);
-    }
-
-    template <typename IndexType>
-    GKO_ATTRIBUTES reference at(IndexType x, IndexType y, IndexType z)
-    {
-        return {this->storage_ + x * this->stride_[0] + y * this->stride_[1] +
-                z};
-    }
-
-    GKO_ATTRIBUTES storage_type *get_storage() { return this->storage_; }
 
 protected:
     storage_type *storage_;
@@ -292,10 +281,11 @@ class Accessor3d<StorageType, ArithmeticType, false> {
  * The accessor uses row-major access.
  */
 template <typename StorageType, typename ArithmeticType>
-class Accessor3d<StorageType, ArithmeticType, true> {
+class ConstAccessor3d<StorageType, ArithmeticType, true> {
 public:
     using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
+    using const_accessor = ConstAccessor3d;
     static_assert(detail::helper_have_scale<StorageType, ArithmeticType>::value,
                   "storage_type must not be an integral in this class.");
 
@@ -306,7 +296,8 @@ class Accessor3d<StorageType, ArithmeticType, true> {
         : public detail::enable_reference<reference, arithmetic_type> {
     public:
         reference() = delete;
-        reference(storage_type *const GKO_RESTRICT ptr, arithmetic_type scale)
+        GKO_ATTRIBUTES reference(storage_type *const GKO_RESTRICT ptr,
+                                 arithmetic_type scale)
             : ptr_{ptr}, scale_{scale}
         {}
         GKO_ATTRIBUTES constexpr GKO_INLINE operator arithmetic_type() const
@@ -330,9 +321,7 @@ class Accessor3d<StorageType, ArithmeticType, true> {
     };
 
 public:
-    Accessor3d() : storage_{nullptr}, stride_{0, 0}, scale_{nullptr} {}
     /**
-     * TODO update
      * Creates the accessor with an already allocated storage space with a
      * stride.
      *
@@ -341,11 +330,16 @@ class Accessor3d<StorageType, ArithmeticType, true> {
      * works. Also, storage_ is never accessed in a non-const fashion, so it
      * is not invalid or UB code.
      */
-    Accessor3d(storage_type *storage, size_type stride0, size_type stride1,
-               arithmetic_type *scale)
-        : storage_{storage}, stride_{stride0, stride1}, scale_{scale}
+    GKO_ATTRIBUTES ConstAccessor3d(const storage_type *storage,
+                                   size_type stride0, size_type stride1,
+                                   const arithmetic_type *scale)
+        : storage_{const_cast<storage_type *>(storage)},
+          stride_{stride0, stride1},
+          scale_{const_cast<arithmetic_type *>(scale)}
     {}
 
+    GKO_ATTRIBUTES const_accessor to_const() const { return *this; }
+
     /**
      * Reads the scale value at the given indices.
      */
@@ -357,11 +351,11 @@ class Accessor3d<StorageType, ArithmeticType, true> {
     }
 
     template <typename IndexType>
-    GKO_ATTRIBUTES const reference at(IndexType x, IndexType y,
+    GKO_ATTRIBUTES arithmetic_type at(IndexType x, IndexType y,
                                       IndexType z) const
     {
-        return {storage_ + x * stride_[0] + y * stride_[1] + z,
-                read_scale(x, z)};
+        return reference{storage_ + x * stride_[0] + y * stride_[1] + z,
+                         read_scale(x, z)};
     }
 
     /**
@@ -399,6 +393,105 @@ class Accessor3d<StorageType, ArithmeticType, true> {
     {
         return this->scale_;
     }
+
+protected:
+    storage_type *storage_;
+    size_type stride_[2];
+    arithmetic_type *scale_;
+};
+
+template <typename StorageType, typename ArithmeticType,
+          bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
+class Accessor3d : public ConstAccessor3d<StorageType, ArithmeticType> {};
+
+template <typename StorageType, typename ArithmeticType>
+class Accessor3d<StorageType, ArithmeticType, false>
+    : public ConstAccessor3d<StorageType, ArithmeticType, false> {
+public:
+    using storage_type = StorageType;
+    using arithmetic_type = ArithmeticType;
+    using const_accessor = ConstAccessor3d<storage_type, arithmetic_type>;
+    static_assert(
+        !detail::helper_have_scale<StorageType, ArithmeticType>::value,
+        "storage_type must not be an integral in this class.");
+    static constexpr bool has_scale{false};
+
+protected:
+    using reference = typename const_accessor::reference;
+
+public:
+    GKO_ATTRIBUTES Accessor3d() : const_accessor(nullptr, 0, 0) {}
+
+    GKO_ATTRIBUTES Accessor3d(storage_type *storage, size_type stride0,
+                              size_type stride1)
+        : const_accessor(storage, stride0, stride1)
+    {}
+
+    GKO_ATTRIBUTES const_accessor to_const() const
+    {
+        return {this->storage_, this->stride_[0], this->stride_[1]};
+    }
+
+    GKO_ATTRIBUTES operator const_accessor() const { return to_const(); }
+
+    using const_accessor::get_storage;
+    GKO_ATTRIBUTES storage_type *get_storage() { return this->storage_; }
+
+    /**
+     * Writes the given value at the given index.
+     */
+    template <typename IndexType>
+    GKO_ATTRIBUTES void write(IndexType x, IndexType y, IndexType z,
+                              arithmetic_type value)
+    {
+        storage_type *GKO_RESTRICT rest_storage = this->storage_;
+        rest_storage[x * this->stride_[0] + y * this->stride_[1] + z] =
+            static_cast<storage_type>(value);
+    }
+
+    using const_accessor::at;  // Makes sure the at() const function is visible
+    template <typename IndexType>
+    GKO_ATTRIBUTES reference at(IndexType x, IndexType y, IndexType z)
+    {
+        return {this->storage_ + x * this->stride_[0] + y * this->stride_[1] +
+                z};
+    }
+};
+
+
+template <typename StorageType, typename ArithmeticType>
+class Accessor3d<StorageType, ArithmeticType, true>
+    : public ConstAccessor3d<StorageType, ArithmeticType, true> {
+public:
+    using storage_type = StorageType;
+    using arithmetic_type = ArithmeticType;
+    using const_accessor = ConstAccessor3d<storage_type, arithmetic_type>;
+    static_assert(detail::helper_have_scale<StorageType, ArithmeticType>::value,
+                  "storage_type must not be an integral in this class.");
+
+    static constexpr bool has_scale{true};
+
+protected:
+    using reference = typename const_accessor::reference;
+
+public:
+    GKO_ATTRIBUTES Accessor3d() : const_accessor(nullptr, {}, {}, nullptr) {}
+
+    GKO_ATTRIBUTES Accessor3d(storage_type *storage, size_type stride0,
+                              size_type stride1, arithmetic_type *scale)
+        : const_accessor(storage, stride0, stride1, scale)
+    {}
+
+    GKO_ATTRIBUTES const_accessor to_const() const
+    {
+        return {this->storage_, this->stride_[0], this->stride_[1],
+                this->scale_};
+    }
+
+    GKO_ATTRIBUTES operator const_accessor() const { return to_const(); }
+
+
+    using const_accessor::at;
     template <typename IndexType>
     GKO_ATTRIBUTES reference at(IndexType x, IndexType y, IndexType z)
     {
@@ -455,14 +548,11 @@ class Accessor3d<StorageType, ArithmeticType, true> {
         //    std::endl;
     }
 
+    using const_accessor::get_storage;
     GKO_ATTRIBUTES storage_type *get_storage() { return this->storage_; }
 
+    using const_accessor::get_scale;
     GKO_ATTRIBUTES arithmetic_type *get_scale() { return this->scale_; }
-
-protected:
-    storage_type *storage_;
-    size_type stride_[2];
-    arithmetic_type *scale_;
 };
 
 
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index 1855ad1ae6f..998356b3c30 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -94,6 +94,27 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     template _macro(                                                         \
         std::complex<float>,                                                 \
         GKO_UNPACK(Accessor3d<std::complex<float>, std::complex<float>>))
+
+#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(_macro)            \
+    template _macro(double, GKO_UNPACK(ConstAccessor3d<double, double>));  \
+    template _macro(double, GKO_UNPACK(ConstAccessor3d<float, double>));   \
+    template _macro(double, GKO_UNPACK(ConstAccessor3d<int64, double>));   \
+    template _macro(double, GKO_UNPACK(ConstAccessor3d<int32, double>));   \
+    template _macro(double, GKO_UNPACK(ConstAccessor3d<int16, double>));   \
+    template _macro(double, GKO_UNPACK(ConstAccessor3d<half, double>));    \
+    template _macro(float, GKO_UNPACK(ConstAccessor3d<float, float>));     \
+    template _macro(float, GKO_UNPACK(ConstAccessor3d<half, float>));      \
+    template _macro(                                                       \
+        std::complex<double>,                                              \
+        GKO_UNPACK(                                                        \
+            ConstAccessor3d<std::complex<double>, std::complex<double>>)); \
+    template _macro(                                                       \
+        std::complex<double>,                                              \
+        GKO_UNPACK(                                                        \
+            ConstAccessor3d<std::complex<float>, std::complex<double>>));  \
+    template _macro(                                                       \
+        std::complex<float>,                                               \
+        GKO_UNPACK(ConstAccessor3d<std::complex<float>, std::complex<float>>))
 // #endif
 // #undef GKO_UNPACK
 
@@ -144,7 +165,7 @@ namespace gmres_mixed {
 #define GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(_type1, _accessor)       \
     void step_2(std::shared_ptr<const DefaultExecutor> exec,           \
                 const matrix::Dense<_type1> *residual_norm_collection, \
-                const _accessor &krylov_bases,                         \
+                _accessor krylov_bases,                                \
                 const matrix::Dense<_type1> *hessenberg,               \
                 matrix::Dense<_type1> *y,                              \
                 matrix::Dense<_type1> *before_preconditioner,          \
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index b19f2027619..7631c83d393 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -110,25 +110,23 @@ as_cuda_accessor(Accessor3d<Type1, Type2> acc)
             acc.get_stride1(), as_cuda_type(acc.get_scale())};
 }
 
-/*
 template <typename Type1, typename Type2>
-xstd::enable_if_t<!Accessor3dConst<Type1, Type2>::has_scale,
-                  Accessor3dConst<cuda_type<Type1>, cuda_type<Type2>>>
-as_cuda_accessor(const Accessor3dConst<Type1, Type2> &acc)
+xstd::enable_if_t<!ConstAccessor3d<Type1, Type2>::has_scale,
+                  ConstAccessor3d<cuda_type<Type1>, cuda_type<Type2>>>
+as_cuda_accessor(const ConstAccessor3d<Type1, Type2> &acc)
 {
     return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
             acc.get_stride1()};
 }
 
 template <typename Type1, typename Type2>
-xstd::enable_if_t<Accessor3dConst<Type1, Type2>::has_scale,
-                  Accessor3dConst<cuda_type<Type1>, cuda_type<Type2>>>
-as_cuda_accessor(const Accessor3dConst<Type1, Type2> &acc)
+xstd::enable_if_t<ConstAccessor3d<Type1, Type2>::has_scale,
+                  ConstAccessor3d<cuda_type<Type1>, cuda_type<Type2>>>
+as_cuda_accessor(const ConstAccessor3d<Type1, Type2> &acc)
 {
     return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
             acc.get_stride1(), as_cuda_type(acc.get_scale())};
 }
-*/
 
 
 template <typename ValueType>
@@ -727,8 +725,8 @@ void solve_upper_triangular(
 }
 
 
-template <typename ValueType, typename Accessor3d>
-void calculate_qy(const Accessor3d &krylov_bases, size_type num_krylov_bases,
+template <typename ValueType, typename ConstAccessor3d>
+void calculate_qy(ConstAccessor3d krylov_bases, size_type num_krylov_bases,
                   const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const Array<size_type> *final_iter_nums)
@@ -757,10 +755,10 @@ void calculate_qy(const Accessor3d &krylov_bases, size_type num_krylov_bases,
 }
 
 
-template <typename ValueType, typename Accessor3d>
+template <typename ValueType, typename ConstAccessor3d>
 void step_2(std::shared_ptr<const CudaExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            const Accessor3d &krylov_bases,
+            ConstAccessor3d krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
@@ -777,7 +775,7 @@ void step_2(std::shared_ptr<const CudaExecutor> exec,
                  final_iter_nums);
 }
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
 
 
diff --git a/cuda/test/solver/gmres_mixed_kernels.cpp b/cuda/test/solver/gmres_mixed_kernels.cpp
index 6520fdec23a..2a0e391f6cc 100644
--- a/cuda/test/solver/gmres_mixed_kernels.cpp
+++ b/cuda/test/solver/gmres_mixed_kernels.cpp
@@ -358,13 +358,13 @@ TEST_F(GmresMixed, CudaGmresMixedStep2IsEquivalentToRef)
     initialize_data();
 
     gko::kernels::reference::gmres_mixed::step_2(
-        ref, residual_norm_collection.get(), krylov_bases_accessor,
+        ref, residual_norm_collection.get(), krylov_bases_accessor.to_const(),
         hessenberg.get(), y.get(), before_preconditioner.get(),
         final_iter_nums.get());
     gko::kernels::cuda::gmres_mixed::step_2(
-        cuda, d_residual_norm_collection.get(), d_krylov_bases_accessor,
-        d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
-        d_final_iter_nums.get());
+        cuda, d_residual_norm_collection.get(),
+        d_krylov_bases_accessor.to_const(), d_hessenberg.get(), d_y.get(),
+        d_before_preconditioner.get(), d_final_iter_nums.get());
 
     GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
diff --git a/hip/solver/gmres_mixed_kernels.hip.cpp b/hip/solver/gmres_mixed_kernels.hip.cpp
index 33282abcdef..4632665128e 100644
--- a/hip/solver/gmres_mixed_kernels.hip.cpp
+++ b/hip/solver/gmres_mixed_kernels.hip.cpp
@@ -137,16 +137,16 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
 
 
-template <typename ValueType, typename Accessor3d>
+template <typename ValueType, typename ConstAccessor3d>
 void step_2(std::shared_ptr<const HipExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            const Accessor3d &krylov_bases,
+            ConstAccessor3d krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
             const Array<size_type> *final_iter_nums) GKO_NOT_IMPLEMENTED;
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
 
 
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index 4602f0d4367..72af0253047 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -327,8 +327,8 @@ void solve_upper_triangular(
 }
 
 
-template <typename ValueType, typename Accessor3d>
-void calculate_qy(const Accessor3d &krylov_bases,
+template <typename ValueType, typename ConstAccessor3d>
+void calculate_qy(ConstAccessor3d krylov_bases,
                   const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const size_type *final_iter_nums)
@@ -491,10 +491,10 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
 
 
-template <typename ValueType, typename Accessor3d>
+template <typename ValueType, typename ConstAccessor3d>
 void step_2(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            const Accessor3d &krylov_bases,
+            ConstAccessor3d krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
@@ -506,7 +506,7 @@ void step_2(std::shared_ptr<const OmpExecutor> exec,
                  final_iter_nums->get_const_data());
 }
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
 
 
diff --git a/omp/test/solver/gmres_mixed_kernels.cpp b/omp/test/solver/gmres_mixed_kernels.cpp
index 21e5245c88f..09959a2ddea 100644
--- a/omp/test/solver/gmres_mixed_kernels.cpp
+++ b/omp/test/solver/gmres_mixed_kernels.cpp
@@ -356,13 +356,13 @@ TEST_F(GmresMixed, OmpGmresMixedStep2IsEquivalentToRef)
     initialize_data();
 
     gko::kernels::reference::gmres_mixed::step_2(
-        ref, residual_norm_collection.get(), krylov_bases_accessor,
+        ref, residual_norm_collection.get(), krylov_bases_accessor.to_const(),
         hessenberg.get(), y.get(), before_preconditioner.get(),
         final_iter_nums.get());
     gko::kernels::omp::gmres_mixed::step_2(
-        omp, d_residual_norm_collection.get(), d_krylov_bases_accessor,
-        d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
-        d_final_iter_nums.get());
+        omp, d_residual_norm_collection.get(),
+        d_krylov_bases_accessor.to_const(), d_hessenberg.get(), d_y.get(),
+        d_before_preconditioner.get(), d_final_iter_nums.get());
 
     GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index f25c830494b..a1b3c5336d7 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -274,14 +274,15 @@ void solve_upper_triangular(
 }
 
 
-template <typename ValueType, typename Accessor3d>
-void calculate_qy(const Accessor3d &krylov_bases,
+template <typename ValueType, typename ConstAccessor3d>
+void calculate_qy(ConstAccessor3d krylov_bases,
                   const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const size_type *final_iter_nums)
 {
     static_assert(
-        std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
+        std::is_same<ValueType,
+                     typename ConstAccessor3d::arithmetic_type>::value,
         "ValueType must match arithmetic_type of accessor!");
     for (size_type k = 0; k < before_preconditioner->get_size()[1]; ++k) {
         for (size_type i = 0; i < before_preconditioner->get_size()[0]; ++i) {
@@ -429,10 +430,10 @@ GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
 
 
-template <typename ValueType, typename Accessor3d>
+template <typename ValueType, typename ConstAccessor3d>
 void step_2(std::shared_ptr<const ReferenceExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
-            const Accessor3d &krylov_bases,
+            ConstAccessor3d krylov_bases,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
@@ -444,7 +445,7 @@ void step_2(std::shared_ptr<const ReferenceExecutor> exec,
                  final_iter_nums->get_const_data());
 }
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
+GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
 
 

From 012c184951c1b6ac0a9115421e2ea3b6e24f15c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 19 Aug 2020 19:12:55 +0200
Subject: [PATCH 0380/1008] Fix accessor by adding additional __restrict__

---
 core/solver/gmres_mixed_accessor.hpp | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index a905ab8d3c5..945f2814cb8 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -189,16 +189,18 @@ class ConstAccessor3d<StorageType, ArithmeticType, false> {
         GKO_ATTRIBUTES reference(storage_type *const GKO_RESTRICT ptr)
             : ptr_{ptr}
         {}
-        GKO_ATTRIBUTES constexpr GKO_INLINE operator arithmetic_type() const
+        GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
         {
-            return static_cast<arithmetic_type>(*ptr_);
+            const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+            return static_cast<arithmetic_type>(*r_ptr);
         }
-        GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type
+        GKO_ATTRIBUTES GKO_INLINE arithmetic_type
         operator=(arithmetic_type val) &&
         {
-            return *ptr_ = static_cast<storage_type>(val);
+            storage_type *const GKO_RESTRICT r_ptr = ptr_;
+            return *r_ptr = static_cast<storage_type>(val);
         }
-        GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type
+        GKO_ATTRIBUTES GKO_INLINE arithmetic_type
         operator=(const reference &ref) &&
         {
             return std::move(*this) = static_cast<arithmetic_type>(ref);
@@ -300,16 +302,18 @@ class ConstAccessor3d<StorageType, ArithmeticType, true> {
                                  arithmetic_type scale)
             : ptr_{ptr}, scale_{scale}
         {}
-        GKO_ATTRIBUTES constexpr GKO_INLINE operator arithmetic_type() const
+        GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
         {
-            return static_cast<arithmetic_type>(*ptr_) * scale_;
+            const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+            return static_cast<arithmetic_type>(*r_ptr) * scale_;
         }
-        GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type
+        GKO_ATTRIBUTES GKO_INLINE arithmetic_type
         operator=(arithmetic_type val) &&
         {
-            return *ptr_ = static_cast<storage_type>(val / scale_), val;
+            storage_type *const GKO_RESTRICT r_ptr = ptr_;
+            return *r_ptr = static_cast<storage_type>(val / scale_), val;
         }
-        GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type
+        GKO_ATTRIBUTES GKO_INLINE arithmetic_type
         operator=(const reference &ref) &&
         {
             return std::move(*this) = static_cast<arithmetic_type>(ref);

From 6526186f3ba5d7f79a7047aaca3a67b8a2eaeac6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 25 Aug 2020 09:21:51 +0200
Subject: [PATCH 0381/1008] GmresMixed storage prec is now a factory parameter

Currently, only core is adapted with the reference test started
(not all precision combinations are tested properly).
---
 core/solver/gmres_mixed.cpp                   | 793 ++++++++++--------
 .../gmres-mixed-solver-jacobi-matrix.cpp      |   5 +-
 include/ginkgo/core/solver/gmres_mixed.hpp    |  38 +-
 reference/test/solver/gmres_mixed_kernels.cpp |  19 +-
 4 files changed, 472 insertions(+), 383 deletions(-)

diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 3167f4ccaab..f8ef75b7cd7 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -95,408 +95,486 @@ GKO_SPECIALIZE_TYPE_STRING(float);
 GKO_SPECIALIZE_TYPE_STRING(half);
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
-void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(const LinOp *b,
-                                                             LinOp *x) const
+template <typename T>
+struct to_integer_impl {
+    using type = T;
+};
+
+template <>
+struct to_integer_impl<double> {
+    using type = int64;
+};
+
+template <>
+struct to_integer_impl<float> {
+    using type = int32;
+};
+
+template <>
+struct to_integer_impl<half> {
+    using type = int16;
+};
+
+template <typename T>
+using to_integer = typename to_integer_impl<T>::type;
+
+
+template <typename ValueType>
+struct helper {
+    template <typename Callable>
+    static void call(Callable callable,
+                     gko::solver::gmres_mixed_storage_precision st)
+    {
+        callable(ValueType{});
+    }
+};
+
+template <>
+struct helper<double> {
+    using ValueType = double;
+
+    template <typename Callable>
+    static void call(Callable callable,
+                     gko::solver::gmres_mixed_storage_precision st)
+    {
+        switch (st) {
+        case gmres_mixed_storage_precision::reduce1:
+            callable(reduce_precision<ValueType>{});
+            break;
+        case gmres_mixed_storage_precision::reduce2:
+            callable(reduce_precision<reduce_precision<ValueType>>{});
+            break;
+        case gmres_mixed_storage_precision::integer:
+            // check_for_complex();
+            callable(to_integer<ValueType>{});
+            break;
+        case gmres_mixed_storage_precision::ireduce1:
+            // check_for_complex();
+            callable(to_integer<reduce_precision<ValueType>>{});
+            break;
+        case gmres_mixed_storage_precision::ireduce2:
+            // check_for_complex();
+            callable(
+                to_integer<reduce_precision<reduce_precision<ValueType>>>{});
+            break;
+        default:
+            callable(ValueType{});
+        }
+    }
+};
+
+
+template <typename ValueType>
+void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix_);
-
-    using Vector = matrix::Dense<ValueType>;
-    using VectorNorms = matrix::Dense<remove_complex<ValueType>>;
-    using LowArray = Array<ValueTypeKrylovBases>;
-    using KrylovAccessor = kernels::Accessor3d<ValueTypeKrylovBases, ValueType>;
-    using Accessor3dHelper =
-        kernels::Accessor3dHelper<ValueType, ValueTypeKrylovBases>;
-
-
-    constexpr uint8 RelativeStoppingId{1};
-
-    auto exec = this->get_executor();
-
-    auto one_op = initialize<Vector>({one<ValueType>()}, exec);
-    auto neg_one_op = initialize<Vector>({-one<ValueType>()}, exec);
-
-    auto dense_b = as<const Vector>(b);
-    auto dense_x = as<Vector>(x);
-    auto residual = Vector::create_with_config_of(dense_b);
-    const dim<3> krylov_bases_dim{
-        krylov_dim_ + 1, system_matrix_->get_size()[1], dense_b->get_size()[1]};
-    // const dim<2> krylov_bases_dim{
-    //     system_matrix_->get_size()[1],
-    //     (krylov_dim_ + 1) * dense_b->get_size()[1]};
-    // const size_type krylov_bases_stride = krylov_bases_dim[1];
-    // LowArray krylov_bases(exec, krylov_bases_dim[0] * krylov_bases_stride);
-    // KrylovAccessor krylov_bases_accessor(krylov_bases.get_data(),
-    //                                     krylov_bases_stride);
-    Accessor3dHelper helper(exec, krylov_bases_dim);
-    auto krylov_bases_accessor = helper.get_accessor();
-
-    auto next_krylov_basis = Vector::create_with_config_of(dense_b);
-    std::shared_ptr<matrix::Dense<ValueType>> preconditioned_vector =
-        Vector::create_with_config_of(dense_b);
-    auto hessenberg = Vector::create(
-        exec, dim<2>{krylov_dim_ + 1, krylov_dim_ * dense_b->get_size()[1]});
-    auto buffer =
-        Vector::create(exec, dim<2>{krylov_dim_ + 1, dense_b->get_size()[1]});
-    auto givens_sin =
-        Vector::create(exec, dim<2>{krylov_dim_, dense_b->get_size()[1]});
-    auto givens_cos =
-        Vector::create(exec, dim<2>{krylov_dim_, dense_b->get_size()[1]});
-    auto residual_norm_collection =
-        Vector::create(exec, dim<2>{krylov_dim_ + 1, dense_b->get_size()[1]});
-    auto residual_norm =
-        VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
-    auto b_norm = VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
-    // TODO: write description what the different rows represent
-    // The optional entry stores the infinity_norm of each next_krylov_vector,
-    // which is only used to compute the scale
-    auto arnoldi_norm =
-        VectorNorms::create(exec, dim<2>{3, dense_b->get_size()[1]});
-    Array<size_type> final_iter_nums(this->get_executor(),
-                                     dense_b->get_size()[1]);
-    auto y = Vector::create(exec, dim<2>{krylov_dim_, dense_b->get_size()[1]});
-
-    bool one_changed{};
-    Array<stopping_status> stop_status(this->get_executor(),
-                                       dense_b->get_size()[1]);
-    Array<stopping_status> reorth_status(this->get_executor(),
+    // Current workaround to get a lambda with a template argument (only the
+    // type of `value` matters, the content does not)
+    auto apply_templated = [&](auto value) {
+        using storage_type = decltype(value);
+        GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix_);
+
+        using Vector = matrix::Dense<ValueType>;
+        using VectorNorms = matrix::Dense<remove_complex<ValueType>>;
+        using LowArray = Array<storage_type>;
+        using KrylovAccessor = kernels::Accessor3d<storage_type, ValueType>;
+        using Accessor3dHelper =
+            kernels::Accessor3dHelper<ValueType, storage_type>;
+
+
+        constexpr uint8 RelativeStoppingId{1};
+
+        auto exec = this->get_executor();
+
+        auto one_op = initialize<Vector>({one<ValueType>()}, exec);
+        auto neg_one_op = initialize<Vector>({-one<ValueType>()}, exec);
+
+        auto dense_b = as<const Vector>(b);
+        auto dense_x = as<Vector>(x);
+        auto residual = Vector::create_with_config_of(dense_b);
+        const dim<3> krylov_bases_dim{krylov_dim_ + 1,
+                                      system_matrix_->get_size()[1],
+                                      dense_b->get_size()[1]};
+        // const dim<2> krylov_bases_dim{
+        //     system_matrix_->get_size()[1],
+        //     (krylov_dim_ + 1) * dense_b->get_size()[1]};
+        // const size_type krylov_bases_stride = krylov_bases_dim[1];
+        // LowArray krylov_bases(exec, krylov_bases_dim[0] *
+        // krylov_bases_stride); KrylovAccessor
+        // krylov_bases_accessor(krylov_bases.get_data(),
+        //                                     krylov_bases_stride);
+        Accessor3dHelper helper(exec, krylov_bases_dim);
+        auto krylov_bases_accessor = helper.get_accessor();
+
+        auto next_krylov_basis = Vector::create_with_config_of(dense_b);
+        std::shared_ptr<matrix::Dense<ValueType>> preconditioned_vector =
+            Vector::create_with_config_of(dense_b);
+        auto hessenberg = Vector::create(
+            exec,
+            dim<2>{krylov_dim_ + 1, krylov_dim_ * dense_b->get_size()[1]});
+        auto buffer = Vector::create(
+            exec, dim<2>{krylov_dim_ + 1, dense_b->get_size()[1]});
+        auto givens_sin =
+            Vector::create(exec, dim<2>{krylov_dim_, dense_b->get_size()[1]});
+        auto givens_cos =
+            Vector::create(exec, dim<2>{krylov_dim_, dense_b->get_size()[1]});
+        auto residual_norm_collection = Vector::create(
+            exec, dim<2>{krylov_dim_ + 1, dense_b->get_size()[1]});
+        auto residual_norm =
+            VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
+        auto b_norm =
+            VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
+        // TODO: write description what the different rows represent
+        // The optional entry stores the infinity_norm of each
+        // next_krylov_vector, which is only used to compute the scale
+        auto arnoldi_norm =
+            VectorNorms::create(exec, dim<2>{3, dense_b->get_size()[1]});
+        Array<size_type> final_iter_nums(this->get_executor(),
                                          dense_b->get_size()[1]);
-    Array<size_type> num_reorth(this->get_executor(), dense_b->get_size()[1]);
-    int num_restarts = 0, num_reorth_steps = 0, num_reorth_vectors = 0;
-
-    // std::cout << "Before initializate_1" << std::endl;
-    // Initialization
-    exec->run(gmres_mixed::make_initialize_1(
-        dense_b, b_norm.get(), residual.get(), givens_sin.get(),
-        givens_cos.get(), &stop_status, krylov_dim_));
-    // b_norm = norm(b)
-    // residual = dense_b
-    // givens_sin = givens_cos = 0
-    system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(),
-                          residual.get());
-    // residual = residual - Ax
-
-    // std::cout << "Before initializate_2" << std::endl;
-    exec->run(gmres_mixed::make_initialize_2(
-        residual.get(), residual_norm.get(), residual_norm_collection.get(),
-        arnoldi_norm.get(), krylov_bases_accessor, next_krylov_basis.get(),
-        &final_iter_nums, krylov_dim_));
-    // residual_norm = norm(residual)
-    // residual_norm_collection = {residual_norm, 0, ..., 0}
-    // krylov_bases(:, 1) = residual / residual_norm
-    // next_krylov_basis = residual / residual_norm
-    // final_iter_nums = {0, ..., 0}
-
-    auto stop_criterion = stop_criterion_factory_->generate(
-        system_matrix_, std::shared_ptr<const LinOp>(b, [](const LinOp *) {}),
-        x, residual.get());
-
-    int total_iter = -1;
-    size_type restart_iter = 0;
-
-    auto before_preconditioner =
-        matrix::Dense<ValueType>::create_with_config_of(dense_x);
-    auto after_preconditioner =
-        matrix::Dense<ValueType>::create_with_config_of(dense_x);
+        auto y =
+            Vector::create(exec, dim<2>{krylov_dim_, dense_b->get_size()[1]});
+
+        bool one_changed{};
+        Array<stopping_status> stop_status(this->get_executor(),
+                                           dense_b->get_size()[1]);
+        Array<stopping_status> reorth_status(this->get_executor(),
+                                             dense_b->get_size()[1]);
+        Array<size_type> num_reorth(this->get_executor(),
+                                    dense_b->get_size()[1]);
+        int num_restarts = 0, num_reorth_steps = 0, num_reorth_vectors = 0;
+
+        // std::cout << "Before initializate_1" << std::endl;
+        // Initialization
+        exec->run(gmres_mixed::make_initialize_1(
+            dense_b, b_norm.get(), residual.get(), givens_sin.get(),
+            givens_cos.get(), &stop_status, krylov_dim_));
+        // b_norm = norm(b)
+        // residual = dense_b
+        // givens_sin = givens_cos = 0
+        system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(),
+                              residual.get());
+        // residual = residual - Ax
+
+        // std::cout << "Before initializate_2" << std::endl;
+        exec->run(gmres_mixed::make_initialize_2(
+            residual.get(), residual_norm.get(), residual_norm_collection.get(),
+            arnoldi_norm.get(), krylov_bases_accessor, next_krylov_basis.get(),
+            &final_iter_nums, krylov_dim_));
+        // residual_norm = norm(residual)
+        // residual_norm_collection = {residual_norm, 0, ..., 0}
+        // krylov_bases(:, 1) = residual / residual_norm
+        // next_krylov_basis = residual / residual_norm
+        // final_iter_nums = {0, ..., 0}
+
+        auto stop_criterion = stop_criterion_factory_->generate(
+            system_matrix_,
+            std::shared_ptr<const LinOp>(b, [](const LinOp *) {}), x,
+            residual.get());
+
+        int total_iter = -1;
+        size_type restart_iter = 0;
+
+        auto before_preconditioner =
+            matrix::Dense<ValueType>::create_with_config_of(dense_x);
+        auto after_preconditioner =
+            matrix::Dense<ValueType>::create_with_config_of(dense_x);
 
 #ifdef TIMING
-    exec->synchronize();
-    auto start = std::chrono::steady_clock::now();
+        exec->synchronize();
+        auto start = std::chrono::steady_clock::now();
 #ifdef TIMING_STEPS
-    auto time_RSTRT = start - start;
-    auto time_SPMV = start - start;
-    auto time_STEP1 = start - start;
+        auto time_RSTRT = start - start;
+        auto time_SPMV = start - start;
+        auto time_STEP1 = start - start;
 #endif
 #endif
-    bool stop_already_encountered{false};
-    bool perform_reset{false};
-    decltype(krylov_dim_) forced_iterations{0};
-    const decltype(forced_iterations) forced_limit{krylov_dim_ / 10};
-    decltype(krylov_dim_) total_checks{0};  // TODO: Remove debug output
-    const char *type_str = type_string<ValueTypeKrylovBases>::get();
-    // TODO: take care of multiple RHS. Currently, we restart everything,
-    //       even though a lot of parts might have already converged!
-    //       Use `one_changed` to take care of that!
-    while (true) {
-        ++total_iter;
-        this->template log<log::Logger::iteration_complete>(
-            this, total_iter, residual.get(), dense_x, residual_norm.get());
-        if (stop_already_encountered && forced_iterations < forced_limit) {
-            ++forced_iterations;
-        } else if (stop_criterion->update()
-                       .num_iterations(total_iter)
-                       .residual(residual.get())
-                       .residual_norm(residual_norm.get())
-                       .solution(dense_x)
-                       .check(RelativeStoppingId, true, &stop_status,
-                              &one_changed)) {
-            if (stop_already_encountered) {
-                // forced_iterations = 0;
-                break;
-            }
-            stop_already_encountered = true;
-            /*
-            std::cout << type_str << ": " << ++total_checks
-                      << ". check in iteration " << total_iter << "; "
-                      << forced_iterations << " / " << forced_limit << '\n';
-            */
-            Array<stopping_status> host_stop_status(
-                this->get_executor()->get_master(), stop_status);
-            bool host_array_changed{false};
-            for (size_type i = 0; i < host_stop_status.get_num_elems(); ++i) {
-                auto local_status = host_stop_status.get_data() + i;
-                // TODO: ignore all actually converged ones!
-                if (local_status->has_converged()) {
-                    local_status->reset();
-                    host_array_changed = true;
+        bool stop_already_encountered{false};
+        bool perform_reset{false};
+        decltype(krylov_dim_) forced_iterations{0};
+        const decltype(forced_iterations) forced_limit{krylov_dim_ / 10};
+        decltype(krylov_dim_) total_checks{0};  // TODO: Remove debug output
+        const char *type_str = type_string<storage_type>::get();
+        // TODO: take care of multiple RHS. Currently, we restart
+        // everything,
+        //       even though a lot of parts might have already converged!
+        //       Use `one_changed` to take care of that!
+        while (true) {
+            ++total_iter;
+            this->template log<log::Logger::iteration_complete>(
+                this, total_iter, residual.get(), dense_x, residual_norm.get());
+            if (stop_already_encountered && forced_iterations < forced_limit) {
+                ++forced_iterations;
+            } else if (stop_criterion->update()
+                           .num_iterations(total_iter)
+                           .residual(residual.get())
+                           .residual_norm(residual_norm.get())
+                           .solution(dense_x)
+                           .check(RelativeStoppingId, true, &stop_status,
+                                  &one_changed)) {
+                if (stop_already_encountered) {
+                    // forced_iterations = 0;
+                    break;
+                }
+                stop_already_encountered = true;
+                /*
+                std::cout << type_str << ": " << ++total_checks
+                          << ". check in iteration " << total_iter << "; "
+                          << forced_iterations << " / " << forced_limit <<
+                '\n';
+                */
+                Array<stopping_status> host_stop_status(
+                    this->get_executor()->get_master(), stop_status);
+                bool host_array_changed{false};
+                for (size_type i = 0; i < host_stop_status.get_num_elems();
+                     ++i) {
+                    auto local_status = host_stop_status.get_data() + i;
+                    // TODO: ignore all actually converged ones!
+                    if (local_status->has_converged()) {
+                        local_status->reset();
+                        host_array_changed = true;
+                    }
+                }
+                if (host_array_changed) {
+                    perform_reset = true;
+                    stop_status = host_stop_status;
+                } else {
+                    break;
                 }
-            }
-            if (host_array_changed) {
-                perform_reset = true;
-                stop_status = host_stop_status;
             } else {
-                break;
+                forced_iterations = 0;
+                stop_already_encountered = false;
             }
-        } else {
-            forced_iterations = 0;
-            stop_already_encountered = false;
-        }
 
-        if (perform_reset || restart_iter == krylov_dim_) {
-            perform_reset = false;
+            if (perform_reset || restart_iter == krylov_dim_) {
+                perform_reset = false;
 #ifdef TIMING_STEPS
-            exec->synchronize();
-            auto t_aux_0 = std::chrono::steady_clock::now();
+                exec->synchronize();
+                auto t_aux_0 = std::chrono::steady_clock::now();
 #endif
-            //            std::cout << "RESTARTING" << std::endl;
-            num_restarts++;
-            // Restart
-            /*
-            std::cout << "[ ";
-            for (int i = 0; i <= krylov_dim_; i++) {
-                for (int j = 0; j < krylov_dim_ * dense_b->get_size()[1];
-                     j++) {
-                    if (j > 0) std::cout << ", ";
-                    std::cout << hessenberg->at(i, j);
-                }
-                std::cout << std::endl;
-            }
-            std::cout << "]" << std::endl;
-            */
-            // use a view in case this is called earlier
-            auto hessenberg_view = hessenberg->create_submatrix(
-                span{0, restart_iter},
-                span{0, dense_b->get_size()[1] * (restart_iter)});
-
-            exec->run(gmres_mixed::make_step_2(
-                residual_norm_collection.get(),
-                krylov_bases_accessor.to_const(), hessenberg_view.get(),
-                y.get(), before_preconditioner.get(), &final_iter_nums));
-            /* */ /*
-            auto hessenberg_small = hessenberg->create_submatrix(
-                span{0, restart_iter},
-                span{0, dense_b->get_size()[1] * (restart_iter)});
-            exec->run(gmres_mixed::make_step_2(
-                residual_norm_collection.get(),
-                krylov_bases_accessor,
-                hessenberg_small.get(), y.get(),
-                before_preconditioner.get(), &final_iter_nums));
-            */
-            // Solve upper triangular.
-            // y = hessenberg \ residual_norm_collection
-
-            get_preconditioner()->apply(before_preconditioner.get(),
-                                        after_preconditioner.get());
-            dense_x->add_scaled(one_op.get(), after_preconditioner.get());
-            // Solve x
-            // x = x + get_preconditioner() * krylov_bases * y
-            residual->copy_from(dense_b);
-            // residual = dense_b
-            system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(),
-                                  residual.get());
-            // residual = residual - Ax
-            exec->run(gmres_mixed::make_initialize_2(
-                residual.get(), residual_norm.get(),
-                residual_norm_collection.get(), arnoldi_norm.get(),
-                krylov_bases_accessor, next_krylov_basis.get(),
-                &final_iter_nums, krylov_dim_));
-            // residual_norm = norm(residual)
-            // residual_norm_collection = {residual_norm, 0, ..., 0}
-            // krylov_bases(:, 1) = residual / residual_norm
-            // next_krylov_basis = residual / residual_norm
-            // final_iter_nums = {0, ..., 0}
-            restart_iter = 0;
+                //            std::cout << "RESTARTING" << std::endl;
+                num_restarts++;
+                // Restart
+                // use a view in case this is called earlier
+                auto hessenberg_view = hessenberg->create_submatrix(
+                    span{0, restart_iter},
+                    span{0, dense_b->get_size()[1] * (restart_iter)});
+
+                exec->run(gmres_mixed::make_step_2(
+                    residual_norm_collection.get(),
+                    krylov_bases_accessor.to_const(), hessenberg_view.get(),
+                    y.get(), before_preconditioner.get(), &final_iter_nums));
+                // Solve upper triangular.
+                // y = hessenberg \ residual_norm_collection
+
+                get_preconditioner()->apply(before_preconditioner.get(),
+                                            after_preconditioner.get());
+                dense_x->add_scaled(one_op.get(), after_preconditioner.get());
+                // Solve x
+                // x = x + get_preconditioner() * krylov_bases * y
+                residual->copy_from(dense_b);
+                // residual = dense_b
+                system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(),
+                                      residual.get());
+                // residual = residual - Ax
+                exec->run(gmres_mixed::make_initialize_2(
+                    residual.get(), residual_norm.get(),
+                    residual_norm_collection.get(), arnoldi_norm.get(),
+                    krylov_bases_accessor, next_krylov_basis.get(),
+                    &final_iter_nums, krylov_dim_));
+                // residual_norm = norm(residual)
+                // residual_norm_collection = {residual_norm, 0, ..., 0}
+                // krylov_bases(:, 1) = residual / residual_norm
+                // next_krylov_basis = residual / residual_norm
+                // final_iter_nums = {0, ..., 0}
+                restart_iter = 0;
 #ifdef TIMING_STEPS
-            exec->synchronize();
-            time_RSTRT += std::chrono::steady_clock::now() - t_aux_0;
+                exec->synchronize();
+                time_RSTRT += std::chrono::steady_clock::now() - t_aux_0;
 #endif
-        }
+            }
 
-        get_preconditioner()->apply(next_krylov_basis.get(),
-                                    preconditioned_vector.get());
-        // preconditioned_vector = get_preconditioner() * next_krylov_basis
+            get_preconditioner()->apply(next_krylov_basis.get(),
+                                        preconditioned_vector.get());
+            // preconditioned_vector = get_preconditioner() *
+            // next_krylov_basis
 
-        // Do Arnoldi and givens rotation
-        auto hessenberg_iter = hessenberg->create_submatrix(
-            span{0, restart_iter + 2},
-            span{dense_b->get_size()[1] * restart_iter,
-                 dense_b->get_size()[1] * (restart_iter + 1)});
-        auto buffer_iter = buffer->create_submatrix(
-            span{0, restart_iter + 2}, span{0, dense_b->get_size()[1]});
+            // Do Arnoldi and givens rotation
+            auto hessenberg_iter = hessenberg->create_submatrix(
+                span{0, restart_iter + 2},
+                span{dense_b->get_size()[1] * restart_iter,
+                     dense_b->get_size()[1] * (restart_iter + 1)});
+            auto buffer_iter = buffer->create_submatrix(
+                span{0, restart_iter + 2}, span{0, dense_b->get_size()[1]});
 
 #ifdef TIMING_STEPS
-        exec->synchronize();
-        auto t_aux_1 = std::chrono::steady_clock::now();
+            exec->synchronize();
+            auto t_aux_1 = std::chrono::steady_clock::now();
 #endif
-        // Start of arnoldi
-        system_matrix_->apply(preconditioned_vector.get(),
-                              next_krylov_basis.get());
-        // next_krylov_basis = A * preconditioned_vector
+            // Start of arnoldi
+            system_matrix_->apply(preconditioned_vector.get(),
+                                  next_krylov_basis.get());
+            // next_krylov_basis = A * preconditioned_vector
 #ifdef TIMING_STEPS
-        exec->synchronize();
-        time_SPMV += std::chrono::steady_clock::now() - t_aux_1;
+            exec->synchronize();
+            time_SPMV += std::chrono::steady_clock::now() - t_aux_1;
 #endif
 
 #ifdef TIMING_STEPS
-        exec->synchronize();
-        auto t_aux_2 = std::chrono::steady_clock::now();
+            exec->synchronize();
+            auto t_aux_2 = std::chrono::steady_clock::now();
 #endif
-        exec->run(gmres_mixed::make_step_1(
-            next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
-            residual_norm.get(), residual_norm_collection.get(),
-            krylov_bases_accessor, hessenberg_iter.get(), buffer_iter.get(),
-            b_norm.get(), arnoldi_norm.get(), restart_iter, &final_iter_nums,
-            &stop_status, &reorth_status, &num_reorth, &num_reorth_steps,
-            &num_reorth_vectors));
+            exec->run(gmres_mixed::make_step_1(
+                next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
+                residual_norm.get(), residual_norm_collection.get(),
+                krylov_bases_accessor, hessenberg_iter.get(), buffer_iter.get(),
+                b_norm.get(), arnoldi_norm.get(), restart_iter,
+                &final_iter_nums, &stop_status, &reorth_status, &num_reorth,
+                &num_reorth_steps, &num_reorth_vectors));
 #ifdef TIMING_STEPS
-        exec->synchronize();
-        time_STEP1 += std::chrono::steady_clock::now() - t_aux_2;
+            exec->synchronize();
+            time_STEP1 += std::chrono::steady_clock::now() - t_aux_2;
 #endif
-        // for i in 0:restart_iter
-        //     hessenberg(restart_iter, i) = next_krylov_basis' *
-        //     krylov_bases(:, i) next_krylov_basis  -= hessenberg(restart_iter,
-        //     i) * krylov_bases(:, i)
-        // end
-        // hessenberg(restart_iter, restart_iter + 1) = norm(next_krylov_basis)
-        // next_krylov_basis /= hessenberg(restart_iter, restart_iter + 1)
-        // End of arnoldi
-        // Start apply givens rotation
-        // for j in 0:restart_iter
-        //     temp             =  cos(j)*hessenberg(j) +
-        //                         sin(j)*hessenberg(j+1)
-        //     hessenberg(j+1)  = -sin(j)*hessenberg(j) +
-        //                         cos(j)*hessenberg(j+1)
-        //     hessenberg(j)    =  temp;
-        // end
-        // Calculate sin and cos
-        // hessenberg(restart_iter)   =
-        // cos(restart_iter)*hessenberg(restart_iter) +
-        //                      sin(restart_iter)*hessenberg(restart_iter)
-        // hessenberg(restart_iter+1) = 0
-        // End apply givens rotation
-        // Calculate residual norm
-
-        restart_iter++;
-    }
-    std::cout << type_str << ": " << total_checks << ": exiting in iteration "
-              << total_iter << "; " << forced_iterations << " / "
-              << forced_limit << '\n';
+            // for i in 0:restart_iter
+            //     hessenberg(restart_iter, i) = next_krylov_basis' *
+            //     krylov_bases(:, i) next_krylov_basis  -=
+            //     hessenberg(restart_iter, i) * krylov_bases(:, i)
+            // end
+            // hessenberg(restart_iter, restart_iter + 1) =
+            // norm(next_krylov_basis) next_krylov_basis /=
+            // hessenberg(restart_iter, restart_iter + 1) End of arnoldi
+            // Start apply givens rotation for j in 0:restart_iter
+            //     temp             =  cos(j)*hessenberg(j) +
+            //                         sin(j)*hessenberg(j+1)
+            //     hessenberg(j+1)  = -sin(j)*hessenberg(j) +
+            //                         cos(j)*hessenberg(j+1)
+            //     hessenberg(j)    =  temp;
+            // end
+            // Calculate sin and cos
+            // hessenberg(restart_iter)   =
+            // cos(restart_iter)*hessenberg(restart_iter) +
+            //                      sin(restart_iter)*hessenberg(restart_iter)
+            // hessenberg(restart_iter+1) = 0
+            // End apply givens rotation
+            // Calculate residual norm
+
+            restart_iter++;
+        }
+        std::cout << type_str << ": " << total_checks
+                  << ": exiting in iteration " << total_iter << "; "
+                  << forced_iterations << " / " << forced_limit << '\n';
 
-    // Solve x
+        // Solve x
 #ifdef TIMING_STEPS
-    exec->synchronize();
-    auto t_aux_3 = std::chrono::steady_clock::now();
+        exec->synchronize();
+        auto t_aux_3 = std::chrono::steady_clock::now();
 #endif
-    /*
-    auto krylov_bases_small = krylov_bases->create_submatrix(
-        span{0, system_matrix_->get_size()[0]},
-        span{0, dense_b->get_size()[1] * (restart_iter + 1)});
-    const dim<2> krylov_bases_dim{
-        system_matrix_->get_size()[1],
-        (krylov_dim_ + 1) * dense_b->get_size()[1]};
-    */
-    /*
-    const dim<2> krylov_bases_small_dim{
-        system_matrix_->get_size()[0],
-        dense_b->get_size()[1] * (restart_iter + 1)};
-    KrylovAccessor krylov_bases_small_accessor{krylov_bases.get_data(),
-                                               krylov_bases_stride};
-    */
 
-    auto hessenberg_small = hessenberg->create_submatrix(
-        span{0, restart_iter},
-        span{0, dense_b->get_size()[1] * (restart_iter)});
+        auto hessenberg_small = hessenberg->create_submatrix(
+            span{0, restart_iter},
+            span{0, dense_b->get_size()[1] * (restart_iter)});
 
-    exec->run(gmres_mixed::make_step_2(
-        residual_norm_collection.get(), krylov_bases_accessor.to_const(),
-        hessenberg_small.get(), y.get(), before_preconditioner.get(),
-        &final_iter_nums));
-    // Solve upper triangular.
-    // y = hessenberg \ residual_norm_collection
+        exec->run(gmres_mixed::make_step_2(
+            residual_norm_collection.get(), krylov_bases_accessor.to_const(),
+            hessenberg_small.get(), y.get(), before_preconditioner.get(),
+            &final_iter_nums));
+        // Solve upper triangular.
+        // y = hessenberg \ residual_norm_collection
 #ifdef TIMING_STEPS
-    exec->synchronize();
-    auto time_STEP2 = std::chrono::steady_clock::now() - t_aux_3;
+        exec->synchronize();
+        auto time_STEP2 = std::chrono::steady_clock::now() - t_aux_3;
 #endif
 
 #ifdef TIMING_STEPS
-    exec->synchronize();
-    auto t_aux_4 = std::chrono::steady_clock::now();
+        exec->synchronize();
+        auto t_aux_4 = std::chrono::steady_clock::now();
 #endif
-    get_preconditioner()->apply(before_preconditioner.get(),
-                                after_preconditioner.get());
-    dense_x->add_scaled(one_op.get(), after_preconditioner.get());
+        get_preconditioner()->apply(before_preconditioner.get(),
+                                    after_preconditioner.get());
+        dense_x->add_scaled(one_op.get(), after_preconditioner.get());
 #ifdef TIMING_STEPS
-    exec->synchronize();
-    auto time_SOLVEX = std::chrono::steady_clock::now() - t_aux_4;
+        exec->synchronize();
+        auto time_SOLVEX = std::chrono::steady_clock::now() - t_aux_4;
 #endif
-    // Solve x
-    // x = x + get_preconditioner() * krylov_bases * y
+        // Solve x
+        // x = x + get_preconditioner() * krylov_bases * y
 
 #ifdef TIMING
-    exec->synchronize();
-    auto time = std::chrono::steady_clock::now() - start;
+        exec->synchronize();
+        auto time = std::chrono::steady_clock::now() - start;
 #endif
 #ifdef TIMING
-    std::cout << "total_iter = " << total_iter << std::endl;
-    std::cout << "num_restarts = " << num_restarts << std::endl;
-    std::cout << "reorth_steps = " << num_reorth_steps << std::endl;
-    std::cout << "reorth_vectors = " << num_reorth_vectors << std::endl;
-    std::cout << "time = "
-              << std::chrono::duration_cast<double_seconds>(time).count()
-              << std::endl;
+        std::cout << "total_iter = " << total_iter << std::endl;
+        std::cout << "num_restarts = " << num_restarts << std::endl;
+        std::cout << "reorth_steps = " << num_reorth_steps << std::endl;
+        std::cout << "reorth_vectors = " << num_reorth_vectors << std::endl;
+        std::cout << "time = "
+                  << std::chrono::duration_cast<double_seconds>(time).count()
+                  << std::endl;
 #ifdef TIMING_STEPS
-    std::cout << "time_RSTRT = "
-              << std::chrono::duration_cast<double_seconds>(time_RSTRT).count()
-              << std::endl;
-    std::cout << "time_SPMV = "
-              << std::chrono::duration_cast<double_seconds>(time_SPMV).count()
-              << std::endl;
-    std::cout << "time_STEP1 = "
-              << std::chrono::duration_cast<double_seconds>(time_STEP1).count()
-              << std::endl;
-    std::cout << "time_STEP2 = "
-              << std::chrono::duration_cast<double_seconds>(time_STEP2).count()
-              << std::endl;
-    std::cout << "time_SOLVEX = "
-              << std::chrono::duration_cast<double_seconds>(time_SOLVEX).count()
-              << std::endl;
+        std::cout
+            << "time_RSTRT = "
+            << std::chrono::duration_cast<double_seconds>(time_RSTRT).count()
+            << std::endl;
+        std::cout
+            << "time_SPMV = "
+            << std::chrono::duration_cast<double_seconds>(time_SPMV).count()
+            << std::endl;
+        std::cout
+            << "time_STEP1 = "
+            << std::chrono::duration_cast<double_seconds>(time_STEP1).count()
+            << std::endl;
+        std::cout
+            << "time_STEP2 = "
+            << std::chrono::duration_cast<double_seconds>(time_STEP2).count()
+            << std::endl;
+        std::cout
+            << "time_SOLVEX = "
+            << std::chrono::duration_cast<double_seconds>(time_SOLVEX).count()
+            << std::endl;
 #endif
-    write(std::cout, lend(residual_norm));
+        write(std::cout, lend(residual_norm));
 #endif
+    };  // End of apply_lambda
+
+    auto check_for_complex = [storage_precision = storage_precision_]()
+    {
+        if (is_complex_s<ValueType>::value) {
+            GKO_NOT_SUPPORTED(storage_precision);
+        }
+    };
+    // Look which precision to use as the storage type
+    helper<ValueType>::call(apply_templated, storage_precision_);
+    /*
+    switch (storage_precision_) {
+    case gmres_mixed_storage_precision::reduce1:
+        apply_templated(reduce_precision<ValueType>{});
+        break;
+    case gmres_mixed_storage_precision::reduce2:
+        apply_templated(reduce_precision<reduce_precision<ValueType>>{});
+        break;
+    case gmres_mixed_storage_precision::integer:
+        check_for_complex();
+        apply_templated(to_integer<ValueType>{});
+        break;
+    case gmres_mixed_storage_precision::ireduce1:
+        check_for_complex();
+        apply_templated(to_integer<reduce_precision<ValueType>>{});
+        break;
+    case gmres_mixed_storage_precision::ireduce2:
+        check_for_complex();
+        apply_templated(
+            to_integer<reduce_precision<reduce_precision<ValueType>>>{});
+        break;
+    default:
+        apply_templated(ValueType{});
+    }
+    */
 }
 
 
-/*
-template <typename ValueType, typename ValueTypeKrylovBases, bool
-Reorthogonalization, bool MGS_CGS> void
-GmresMixed<ValueType,ValueTypeKrylovBases,
-                Reorthogonalization,MGS_CGS>::apply_impl(const LinOp *alpha,
-const LinOp *b, const LinOp *residual_norm_collection, LinOp *x) const
-*/
-template <typename ValueType, typename ValueTypeKrylovBases>
-void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(
-    const LinOp *alpha, const LinOp *b, const LinOp *residual_norm_collection,
-    LinOp *x) const
+template <typename ValueType>
+void GmresMixed<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
+                                       const LinOp *residual_norm_collection,
+                                       LinOp *x) const
 {
     auto dense_x = as<matrix::Dense<ValueType>>(x);
 
@@ -506,13 +584,8 @@ void GmresMixed<ValueType, ValueTypeKrylovBases>::apply_impl(
     dense_x->add_scaled(alpha, x_clone.get());
 }
 
-/*
-#define GKO_DECLARE_GMRES_MIXED_BOOL(_type1, _type2, _bool1, _bool2)
-      class GmresMixed<_type1, _type2, _bool1, _bool2>
-GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(GKO_DECLARE_GMRES_MIXED_BOOL);
-*/
-#define GKO_DECLARE_GMRES_MIXED(_type1, _type2) class GmresMixed<_type1, _type2>
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE2(GKO_DECLARE_GMRES_MIXED);
+#define GKO_DECLARE_GMRES_MIXED(_type1) class GmresMixed<_type1>
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_MIXED);
 
 
 }  // namespace solver
diff --git a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp b/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
index e7f2b8f5cc1..a767bda82a3 100644
--- a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
+++ b/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
@@ -57,8 +57,7 @@ int main(int argc, char *argv[])
     using mtx = gko::matrix::Csr<>;
     // The gko::solver::Cg is used here, but any other solver class can also be
     // used.
-    using gmres_mixed = gko::solver::GmresMixed<double, long int>;
-    // using gmres_mixed = gko::solver::GmresMixed<double, float>;
+    using gmres_mixed = gko::solver::GmresMixed<double>;
     // using gmres_mixed = gko::solver::GmresMixed<>;
     using bj = gko::preconditioner::Jacobi<>;
 
@@ -225,6 +224,8 @@ int main(int argc, char *argv[])
             // Add preconditioner, these 2 lines are the only
             // difference from the simple solver example
             //    .with_preconditioner(bj::build().with_max_block_size(8u).on(exec))
+            .with_storage_precision(
+                gko::solver::gmres_mixed_storage_precision::keep)
             .with_preconditioner(bj::build().with_max_block_size(1u).on(exec))
             .on(exec);
     // Generate the solver from the matrix. The solver factory built in the
diff --git a/include/ginkgo/core/solver/gmres_mixed.hpp b/include/ginkgo/core/solver/gmres_mixed.hpp
index ed189f31915..4bd017e6486 100644
--- a/include/ginkgo/core/solver/gmres_mixed.hpp
+++ b/include/ginkgo/core/solver/gmres_mixed.hpp
@@ -55,6 +55,16 @@ namespace solver {
 constexpr size_type default_krylov_dim_mixed = 100u;
 
 
+enum class gmres_mixed_storage_precision {
+    keep,
+    reduce1,
+    reduce2,
+    integer,
+    ireduce1,
+    ireduce2
+};
+
+
 /**
  * GMRES_MIXED or the generalized minimal residual method is an iterative type
  * Krylov subspace method which is suitable for nonsymmetric linear systems.
@@ -68,28 +78,14 @@ constexpr size_type default_krylov_dim_mixed = 100u;
  * @ingroup solvers
  * @ingroup LinOp
  */
-/*
-template <typename ValueType = default_precision,
-          typename ValueTypeKrylovBases = float,
-          bool Reorthogonalization = true,
-          bool MGS_CGS = true>
-*/
-template <typename ValueType = default_precision,
-          typename ValueTypeKrylovBases = default_precision>
-//          typename ValueTypeKrylovBases = float>
-class GmresMixed :
-    //		public EnableLinOp<GmresMixed<ValueType,
-    // ValueTypeKrylovBases,Reorthogonalization, MGS_CGS>>,
-    public EnableLinOp<GmresMixed<ValueType, ValueTypeKrylovBases>>,
-    public Preconditionable {
+template <typename ValueType = default_precision>
+class GmresMixed : public EnableLinOp<GmresMixed<ValueType>>,
+                   public Preconditionable {
     friend class EnableLinOp<GmresMixed>;
     friend class EnablePolymorphicObject<GmresMixed, LinOp>;
 
 public:
     using value_type = ValueType;
-    using krylov_type = ValueTypeKrylovBases;
-    //    using reorthogon  = Reorthogonalization;
-    //    using mgs_cgs     = MGS_CGS;
 
     /**
      * Gets the system operator (matrix) of the linear system.
@@ -110,6 +106,12 @@ class GmresMixed :
 
     GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
     {
+        /**
+         * Determines which storage type is used.
+         */
+        gmres_mixed_storage_precision GKO_FACTORY_PARAMETER(
+            storage_precision, gmres_mixed_storage_precision::keep);
+
         /**
          * Criterion factories.
          */
@@ -172,12 +174,14 @@ class GmresMixed :
         }
         stop_criterion_factory_ =
             stop::combine(std::move(parameters_.criteria));
+        storage_precision_ = parameters_.storage_precision;
     }
 
 private:
     std::shared_ptr<const LinOp> system_matrix_{};
     std::shared_ptr<const stop::CriterionFactory> stop_criterion_factory_{};
     size_type krylov_dim_;
+    gmres_mixed_storage_precision storage_precision_;
 };
 
 
diff --git a/reference/test/solver/gmres_mixed_kernels.cpp b/reference/test/solver/gmres_mixed_kernels.cpp
index 92a8a0c2030..872fd0f755c 100644
--- a/reference/test/solver/gmres_mixed_kernels.cpp
+++ b/reference/test/solver/gmres_mixed_kernels.cpp
@@ -52,15 +52,19 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
-template <typename ValueKrylovType>
+template <typename ValueType>
 class GmresMixed : public ::testing::Test {
 protected:
+    using value_type = ValueType;
+    /*
     using value_type =
         typename std::tuple_element<0, decltype(ValueKrylovType())>::type;
     using krylov_type =
         typename std::tuple_element<1, decltype(ValueKrylovType())>::type;
+    */
     using Mtx = gko::matrix::Dense<value_type>;
-    using gmres_type = gko::solver::GmresMixed<value_type, krylov_type>;
+    using gmres_type = gko::solver::GmresMixed<value_type>;
+    // using gmres_type = gko::solver::GmresMixed<value_type, krylov_type>;
 
     GmresMixed()
         : exec(gko::ReferenceExecutor::create()),
@@ -68,8 +72,11 @@ class GmresMixed : public ::testing::Test {
               {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)),
           mtx2(gko::initialize<Mtx>(
               {{1.0, 2.0, 3.0}, {4.0, 2.0, 1.0}, {0.0, 1.0, 2.0}}, exec)),
+          storage_precision{
+              gko::solver::gmres_mixed_storage_precision::ireduce2},
           gmres_mixed_factory(
               gmres_type::build()
+                  .with_storage_precision(storage_precision)
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
@@ -90,6 +97,7 @@ class GmresMixed : public ::testing::Test {
               exec)),
           gmres_mixed_factory_big(
               gmres_type::build()
+                  .with_storage_precision(storage_precision)
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
@@ -111,11 +119,13 @@ class GmresMixed : public ::testing::Test {
     std::shared_ptr<Mtx> mtx2;
     std::shared_ptr<Mtx> mtx_medium;
     std::shared_ptr<Mtx> mtx_big;
+    gko::solver::gmres_mixed_storage_precision storage_precision;
     std::unique_ptr<typename gmres_type::Factory> gmres_mixed_factory;
     std::unique_ptr<typename gmres_type::Factory> gmres_mixed_factory_big;
 };
 
 
+/*
 using TestTypes =
     ::testing::Types<std::tuple<double, double>, std::tuple<double, float>,
                      // std::tuple<double, gko::int64>,
@@ -126,9 +136,10 @@ using TestTypes =
                      std::tuple<std::complex<double>, std::complex<double>>,
                      std::tuple<std::complex<double>, std::complex<float>>,
                      std::tuple<std::complex<float>, std::complex<float>>>;
+*/
 
-
-TYPED_TEST_CASE(GmresMixed, TestTypes);
+TYPED_TEST_CASE(GmresMixed, gko::test::ValueTypes /*TestTypes */);
+// TODO TYPED_TEST_CASE_P
 
 
 TYPED_TEST(GmresMixed, SolvesStencilSystem)

From 670b04c9818da354f674576ab074b50d80ced11d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 25 Aug 2020 19:00:05 +0200
Subject: [PATCH 0382/1008] Improve reference test and include the enum there

---
 core/solver/gmres_mixed_kernels.hpp           |  12 --
 reference/test/solver/gmres_mixed_kernels.cpp | 137 +++++++++++++-----
 2 files changed, 97 insertions(+), 52 deletions(-)

diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index 998356b3c30..398b4c25aec 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -63,18 +63,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 //     template _macro(double, double)
 // #else
 #define GKO_UNPACK(...) __VA_ARGS__
-#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE2(_macro)       \
-    template _macro(double, double);                             \
-    template _macro(double, float);                              \
-    template _macro(double, int64);                              \
-    template _macro(double, int32);                              \
-    template _macro(double, int16);                              \
-    template _macro(double, half);                               \
-    template _macro(float, float);                               \
-    template _macro(float, half);                                \
-    template _macro(std::complex<double>, std::complex<double>); \
-    template _macro(std::complex<double>, std::complex<float>);  \
-    template _macro(std::complex<float>, std::complex<float>)
 
 #define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro)                    \
     template _macro(double, GKO_UNPACK(Accessor3d<double, double>));         \
diff --git a/reference/test/solver/gmres_mixed_kernels.cpp b/reference/test/solver/gmres_mixed_kernels.cpp
index 872fd0f755c..e7e93028461 100644
--- a/reference/test/solver/gmres_mixed_kernels.cpp
+++ b/reference/test/solver/gmres_mixed_kernels.cpp
@@ -33,6 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/gmres_mixed.hpp>
 
 
+#include <type_traits>
+
+
 #include <gtest/gtest.h>
 
 
@@ -52,19 +55,33 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
-template <typename ValueType>
+template <typename ValueEnumType>
 class GmresMixed : public ::testing::Test {
 protected:
-    using value_type = ValueType;
-    /*
     using value_type =
-        typename std::tuple_element<0, decltype(ValueKrylovType())>::type;
-    using krylov_type =
-        typename std::tuple_element<1, decltype(ValueKrylovType())>::type;
-    */
+        typename std::tuple_element<0, decltype(ValueEnumType())>::type;
+    using nc_value_type = gko::remove_complex<value_type>;
+    using storage_helper_type =
+        typename std::tuple_element<1, decltype(ValueEnumType())>::type;
     using Mtx = gko::matrix::Dense<value_type>;
     using gmres_type = gko::solver::GmresMixed<value_type>;
-    // using gmres_type = gko::solver::GmresMixed<value_type, krylov_type>;
+
+    /**
+     * Used to extract the precision of the given value_type.
+     *
+     * @internal
+     * This struct is required because the `r<T>` stores a value of type
+     * remove_complex<T>, which is not possible for half since it can't be
+     * constructed as a constexpr.
+     */
+    template <typename T>
+    struct precision_target {
+        using rc_type = gko::remove_complex<T>;
+        static constexpr nc_value_type value =
+            std::is_same<rc_type, double>::value
+                ? 1e-14
+                : std::is_same<rc_type, float>::value ? 1e-7 : 1e-3;
+    };
 
     GmresMixed()
         : exec(gko::ReferenceExecutor::create()),
@@ -72,8 +89,7 @@ class GmresMixed : public ::testing::Test {
               {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)),
           mtx2(gko::initialize<Mtx>(
               {{1.0, 2.0, 3.0}, {4.0, 2.0, 1.0}, {0.0, 1.0, 2.0}}, exec)),
-          storage_precision{
-              gko::solver::gmres_mixed_storage_precision::ireduce2},
+          storage_precision{storage_helper_type::value},
           gmres_mixed_factory(
               gmres_type::build()
                   .with_storage_precision(storage_precision)
@@ -84,7 +100,7 @@ class GmresMixed : public ::testing::Test {
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
                       gko::stop::ResidualNormReduction<value_type>::build()
-                          .with_reduction_factor(r<value_type>::value)
+                          .with_reduction_factor(this->precision())
                           .on(exec))
                   .on(exec)),
           mtx_big(gko::initialize<Mtx>(
@@ -102,7 +118,7 @@ class GmresMixed : public ::testing::Test {
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
                       gko::stop::ResidualNormReduction<value_type>::build()
-                          .with_reduction_factor(r<value_type>::value)
+                          .with_reduction_factor(this->precision())
                           .on(exec))
                   .on(exec)),
           mtx_medium(
@@ -114,6 +130,28 @@ class GmresMixed : public ::testing::Test {
                                    exec))
     {}
 
+    nc_value_type precision() const noexcept
+    {
+        using gko::reduce_precision;
+        using gko::solver::gmres_mixed_storage_precision;
+
+        // Note: integer and floating point are assumed to have similar
+        //       target precision.
+        switch (storage_precision) {
+        case gmres_mixed_storage_precision::reduce1:
+        case gmres_mixed_storage_precision::ireduce1:
+            return precision_target<reduce_precision<value_type>>::value * 1e-2;
+        case gmres_mixed_storage_precision::reduce2:
+        case gmres_mixed_storage_precision::ireduce2:
+            return precision_target<
+                       reduce_precision<reduce_precision<value_type>>>::value *
+                   1e-2;
+        case gmres_mixed_storage_precision::integer:
+        default:
+            return precision_target<value_type>::value;
+        }
+    }
+
     std::shared_ptr<const gko::Executor> exec;
     std::shared_ptr<Mtx> mtx;
     std::shared_ptr<Mtx> mtx2;
@@ -125,21 +163,37 @@ class GmresMixed : public ::testing::Test {
 };
 
 
-/*
+/**
+ * This creates a helper structure which translates a type into an enum
+ * parameter.
+ */
+using st_enum = gko::solver::gmres_mixed_storage_precision;
+
+template <st_enum P>
+struct st_helper_type {
+    static constexpr st_enum value{P};
+};
+
+using st_keep = st_helper_type<st_enum::keep>;
+using st_r1 = st_helper_type<st_enum::reduce1>;
+using st_r2 = st_helper_type<st_enum::reduce2>;
+using st_i = st_helper_type<st_enum::integer>;
+using st_ir1 = st_helper_type<st_enum::ireduce1>;
+using st_ir2 = st_helper_type<st_enum::ireduce2>;
+
 using TestTypes =
-    ::testing::Types<std::tuple<double, double>, std::tuple<double, float>,
-                     // std::tuple<double, gko::int64>,
-                     std::tuple<double, gko::int32>,
-                     std::tuple<double, gko::int16>,
-                     std::tuple<double, gko::half>, std::tuple<float, float>,
-                     std::tuple<float, gko::half>,
-                     std::tuple<std::complex<double>, std::complex<double>>,
-                     std::tuple<std::complex<double>, std::complex<float>>,
-                     std::tuple<std::complex<float>, std::complex<float>>>;
-*/
+    ::testing::Types<std::tuple<double, st_keep>, std::tuple<double, st_r1>,
+                     std::tuple<double, st_r2>, std::tuple<double, st_i>,
+                     std::tuple<double, st_ir1>, std::tuple<double, st_ir2>,
+                     std::tuple<float, st_keep>, std::tuple<float, st_r1>,
+                     std::tuple<float, st_r2>, std::tuple<float, st_i>,
+                     std::tuple<float, st_ir1>, std::tuple<float, st_ir2>,
+                     std::tuple<std::complex<double>, st_keep>,
+                     std::tuple<std::complex<double>, st_r1>,
+                     std::tuple<std::complex<double>, st_r2>,
+                     std::tuple<std::complex<float>, st_keep>>;
 
-TYPED_TEST_CASE(GmresMixed, gko::test::ValueTypes /*TestTypes */);
-// TODO TYPED_TEST_CASE_P
+TYPED_TEST_CASE(GmresMixed, TestTypes);
 
 
 TYPED_TEST(GmresMixed, SolvesStencilSystem)
@@ -152,7 +206,7 @@ TYPED_TEST(GmresMixed, SolvesStencilSystem)
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}), r<T>::value * 1e1);
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}), this->precision() * 1e1);
 }
 
 
@@ -163,6 +217,7 @@ TYPED_TEST(GmresMixed, SolvesStencilSystem2)
     using gmres_type = typename TestFixture::gmres_type;
     auto factory =
         gmres_type::build()
+            .with_storage_precision(this->storage_precision)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(100u).on(
                     this->exec),
@@ -170,7 +225,7 @@ TYPED_TEST(GmresMixed, SolvesStencilSystem2)
                     .with_time_limit(std::chrono::seconds(6))
                     .on(this->exec),
                 gko::stop::ResidualNormReduction<T>::build()
-                    .with_reduction_factor(r<T>::value)
+                    .with_reduction_factor(this->precision())
                     .on(this->exec))
             .on(this->exec);
     auto solver = factory->generate(this->mtx2);
@@ -179,7 +234,7 @@ TYPED_TEST(GmresMixed, SolvesStencilSystem2)
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.0, 4.0, 8.0}), r<T>::value * 1e1);
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 4.0, 8.0}), this->precision() * 1e1);
 }
 
 
@@ -196,7 +251,7 @@ TYPED_TEST(GmresMixed, SolvesMultipleStencilSystems)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({I<T>{1.0, 1.0}, I<T>{3.0, 1.0}, I<T>{2.0, 1.0}}),
-                        r<T>::value * 1e1);
+                        this->precision() * 1e1);
 }
 
 
@@ -212,7 +267,7 @@ TYPED_TEST(GmresMixed, SolvesStencilSystemUsingAdvancedApply)
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), r<T>::value * 1e1);
+    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), this->precision() * 1e1);
 }
 
 
@@ -231,7 +286,7 @@ TYPED_TEST(GmresMixed, SolvesMultipleStencilSystemsUsingAdvancedApply)
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({I<T>{1.5, 1.0}, I<T>{5.0, 0.0}, I<T>{2.0, -1.0}}),
-                        r<T>::value * 1e1);
+                        this->precision() * 1e1);
 }
 
 
@@ -248,7 +303,7 @@ TYPED_TEST(GmresMixed, SolvesBigDenseSystem1)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({52.7, 85.4, 134.2, -250.0, -16.8, 35.3}),
-                        r<T>::value * 1e3);
+                        this->precision() * 1e3);
 }
 
 
@@ -265,7 +320,7 @@ TYPED_TEST(GmresMixed, SolvesBigDenseSystem2)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
-                        r<T>::value * 1e3);
+                        this->precision() * 1e3);
 }
 
 
@@ -342,12 +397,12 @@ TYPED_TEST(GmresMixed, SolvesMultipleDenseSystemForDivergenceCheck)
 
     // make sure that all combined solutions are as good or better than the
     // single solutions
-    ASSERT_LE(normC1 / normB1, normS1 / normB1 + r<value_type>::value);
-    ASSERT_LE(normC2 / normB2, normS2 / normB2 + r<value_type>::value);
+    ASSERT_LE(normC1 / normB1, normS1 / normB1 + this->precision());
+    ASSERT_LE(normC2 / normB2, normS2 / normB2 + this->precision());
 
     // Not sure if this is necessary, the assertions above should cover what is
     // needed.
-    GKO_ASSERT_MTX_NEAR(xc, mergedRes, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(xc, mergedRes, this->precision());
 }
 
 
@@ -356,15 +411,16 @@ TYPED_TEST(GmresMixed, SolvesBigDenseSystem1WithRestart)
     using Mtx = typename TestFixture::Mtx;
     using value_type = typename TestFixture::value_type;
     using gmres_type = typename TestFixture::gmres_type;
-    const auto half_tol = std::sqrt(r<value_type>::value);
+    const auto half_tol = std::sqrt(this->precision());
     auto gmres_mixed_factory_restart =
         gmres_type::build()
             .with_krylov_dim(4u)
+            .with_storage_precision(this->storage_precision)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(200u).on(
                     this->exec),
                 gko::stop::ResidualNormReduction<value_type>::build()
-                    .with_reduction_factor(r<value_type>::value)
+                    .with_reduction_factor(this->precision())
                     .on(this->exec))
             .on(this->exec);
     auto solver = gmres_mixed_factory_restart->generate(this->mtx_medium);
@@ -386,11 +442,12 @@ TYPED_TEST(GmresMixed, SolvesWithPreconditioner)
     using gmres_type = typename TestFixture::gmres_type;
     auto gmres_mixed_factory_preconditioner =
         gmres_type::build()
+            .with_storage_precision(this->storage_precision)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(100u).on(
                     this->exec),
                 gko::stop::ResidualNormReduction<value_type>::build()
-                    .with_reduction_factor(r<value_type>::value)
+                    .with_reduction_factor(this->precision())
                     .on(this->exec))
             .with_preconditioner(
                 gko::preconditioner::Jacobi<value_type>::build()
@@ -406,7 +463,7 @@ TYPED_TEST(GmresMixed, SolvesWithPreconditioner)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
-                        r<value_type>::value * 1e3);
+                        this->precision() * 1e3);
 }
 
 

From 08c6da2cef08098ffd6d7896277d9bf80325eb77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 26 Aug 2020 17:19:45 +0200
Subject: [PATCH 0383/1008] Fix the reference test to pass

Also CUDA and OpenMP compiles now for the new accessor layout.
Benchmarks is still TODO.
---
 common/solver/gmres_mixed_kernels.hpp.inc |   3 +-
 core/solver/gmres_mixed.cpp               |   5 +-
 core/solver/gmres_mixed_accessor.hpp      | 587 ++++++++++++----------
 core/solver/gmres_mixed_kernels.hpp       |  87 ++--
 cuda/solver/gmres_mixed_kernels.cu        |  25 +-
 cuda/test/solver/gmres_mixed_kernels.cpp  |  29 +-
 omp/solver/gmres_mixed_kernels.cpp        |  18 +-
 omp/test/solver/gmres_mixed_kernels.cpp   |  29 +-
 reference/solver/gmres_mixed_kernels.cpp  |  16 +-
 9 files changed, 431 insertions(+), 368 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 1ecb4e8f6a0..5f2259ece11 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -1274,7 +1274,8 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
 {
     const remove_complex<ValueType> eta = 1.0 / 2.0;
     const auto col_idx = thread::get_thread_id_flat();
-    constexpr bool has_scale = Accessor3d::has_scale;
+    constexpr bool has_scale =
+        kernels::detail::is_3d_scaled_accessor<Accessor3d>::value;
 
     if (col_idx < stride_norm && !stop_status[col_idx].has_stopped()) {
         const auto num0 = (eta * arnoldi_norm[col_idx]);
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index f8ef75b7cd7..8795e9186c4 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -176,7 +176,7 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         using Vector = matrix::Dense<ValueType>;
         using VectorNorms = matrix::Dense<remove_complex<ValueType>>;
         using LowArray = Array<storage_type>;
-        using KrylovAccessor = kernels::Accessor3d<storage_type, ValueType>;
+        // using KrylovAccessor = kernels::Accessor3d<storage_type, ValueType>;
         using Accessor3dHelper =
             kernels::Accessor3dHelper<ValueType, storage_type>;
 
@@ -535,8 +535,7 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 #endif
     };  // End of apply_lambda
 
-    auto check_for_complex = [storage_precision = storage_precision_]()
-    {
+    auto check_for_complex = [storage_precision = storage_precision_]() {
         if (is_complex_s<ValueType>::value) {
             GKO_NOT_SUPPORTED(storage_precision);
         }
diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index 945f2814cb8..ab380851b12 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -148,20 +148,100 @@ struct enable_reference {
 };
 
 
-}  // namespace detail
+namespace reference {
 
 
-template <typename StorageType, typename ArithmeticType,
-          bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
-class ConstAccessor3d {};
+/**
+ * Reference class for a different storage than arithmetic type. The conversion
+ * between both formats is done with a simple static_cast.
+ */
+template <typename ArithmeticType, typename StorageType>
+class ReducedStorageReference
+    : public detail::enable_reference<
+          ReducedStorageReference<ArithmeticType, StorageType>,
+          ArithmeticType> {
+public:
+    using arithmetic_type = ArithmeticType;
+    using storage_type = StorageType;
+    ReducedStorageReference() = delete;
+    GKO_ATTRIBUTES ReducedStorageReference(storage_type *const GKO_RESTRICT ptr)
+        : ptr_{ptr}
+    {}
+    GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
+    {
+        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return static_cast<arithmetic_type>(*r_ptr);
+    }
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator=(arithmetic_type val) &&
+    {
+        storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return *r_ptr = static_cast<storage_type>(val);
+    }
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type
+    operator=(const ReducedStorageReference &ref) &&
+    {
+        return std::move(*this) = static_cast<arithmetic_type>(ref);
+    }
+
+private:
+    storage_type *const GKO_RESTRICT ptr_;
+};
+
+/**
+ * Reference class for a different storage than arithmetic type. The conversion
+ * between both formats is done with a simple static_cast followed by a
+ * multiplication with a scale.
+ */
+template <typename ArithmeticType, typename StorageType>
+class ScaledReducedStorageReference
+    : public detail::enable_reference<
+          ScaledReducedStorageReference<ArithmeticType, StorageType>,
+          ArithmeticType> {
+public:
+    using arithmetic_type = ArithmeticType;
+    using storage_type = StorageType;
+
+    ScaledReducedStorageReference() = delete;
+    GKO_ATTRIBUTES ScaledReducedStorageReference(
+        storage_type *const GKO_RESTRICT ptr, arithmetic_type scale)
+        : ptr_{ptr}, scale_{scale}
+    {}
+    GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
+    {
+        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return static_cast<arithmetic_type>(*r_ptr) * scale_;
+    }
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator=(arithmetic_type val) &&
+    {
+        storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        return *r_ptr = static_cast<storage_type>(val / scale_), val;
+    }
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type
+    operator=(const ScaledReducedStorageReference &ref) &&
+    {
+        return std::move(*this) = static_cast<arithmetic_type>(ref);
+    }
+
+private:
+    storage_type *const GKO_RESTRICT ptr_;
+    const arithmetic_type scale_;
+};
+
+
+}  // namespace reference
+}  // namespace detail
+
 
 /**
  * @internal
  *
- * The Accessor3d class hides the underlying storage_ format and provides a
- * simple interface for accessing a one dimensional storage.
+ * The ReducedStorage3d class hides the underlying storage_ format and provides
+ * a simple interface for accessing a one dimensional storage.
  *
- * This class only manages the accesses, however, and not the memory itself.
+ * This class only manages the accesses and not the memory itself.
+ *
+ * TODO move this documentation to the core since this is a general purpose
+ * accessor
  *
  * The dimensions {x, y, z} explained for the krylov_bases:
  * - x: selects the krylov vector (usually has krylov_dim + 1 vectors)
@@ -170,45 +250,18 @@ class ConstAccessor3d {};
  *
  * The accessor uses row-major access.
  */
-template <typename StorageType, typename ArithmeticType>
-class ConstAccessor3d<StorageType, ArithmeticType, false> {
+template <typename ArithmeticType, typename StorageType>
+class ConstReducedStorage3d {
 public:
-    using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
-    using const_accessor = ConstAccessor3d;
-    static_assert(
-        !detail::helper_have_scale<StorageType, ArithmeticType>::value,
-        "storage_type must not be an integral in this class.");
-    static constexpr bool has_scale{false};
+    using storage_type = StorageType;
+    using const_accessor = ConstReducedStorage3d;
+    static_assert(!std::is_const<storage_type>::value,
+                  "StorageType must not be const!");
 
 protected:
-    class reference
-        : public detail::enable_reference<reference, arithmetic_type> {
-    public:
-        reference() = delete;
-        GKO_ATTRIBUTES reference(storage_type *const GKO_RESTRICT ptr)
-            : ptr_{ptr}
-        {}
-        GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
-        {
-            const storage_type *const GKO_RESTRICT r_ptr = ptr_;
-            return static_cast<arithmetic_type>(*r_ptr);
-        }
-        GKO_ATTRIBUTES GKO_INLINE arithmetic_type
-        operator=(arithmetic_type val) &&
-        {
-            storage_type *const GKO_RESTRICT r_ptr = ptr_;
-            return *r_ptr = static_cast<storage_type>(val);
-        }
-        GKO_ATTRIBUTES GKO_INLINE arithmetic_type
-        operator=(const reference &ref) &&
-        {
-            return std::move(*this) = static_cast<arithmetic_type>(ref);
-        }
-
-    private:
-        storage_type *const GKO_RESTRICT ptr_;
-    };
+    using reference =
+        detail::reference::ReducedStorageReference<ArithmeticType, StorageType>;
 
 public:
     /**
@@ -216,65 +269,126 @@ class ConstAccessor3d<StorageType, ArithmeticType, false> {
      * stride.
      *
      * @internal
-     * const_cast needed, so Accessor3d(ac.get_storage(), ...)
-     * works. Also, storage_ is never accessed in a non-const fashion, so it
-     * is not invalid or UB code.
+     * const_cast needed, so ReducedStorage3d(ac.get_storage(), ...)
+     * works. Also, storage_ is never accessed in a non-const fashion in this
+     * class, so it is not invalid or UB code.
      */
-    GKO_ATTRIBUTES ConstAccessor3d(const storage_type *storage,
-                                   size_type stride0, size_type stride1)
+    GKO_ATTRIBUTES ConstReducedStorage3d(const storage_type *storage,
+                                         size_type stride0, size_type stride1)
         : storage_{const_cast<storage_type *>(storage)},
           stride_{stride0, stride1}
     {}
 
-    GKO_ATTRIBUTES const_accessor to_const() const { return *this; }
+    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const { return *this; }
 
-    /**
-     * Reads the value at the given index.
-     */
     template <typename IndexType>
-    GKO_ATTRIBUTES arithmetic_type read(IndexType x, IndexType y,
-                                        IndexType z) const
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator()(IndexType x,
+                                                         IndexType y,
+                                                         IndexType z) const
     {
-        // Make use of the restrict (and const) qualifier. If the restrict
-        // qualifier would have been put on the class attribute, it would be
-        // ignored by nvcc.
-        const storage_type *GKO_RESTRICT rest_storage = storage_;
-        return static_cast<arithmetic_type>(
-            rest_storage[x * stride_[0] + y * stride_[1] + z]);
+        return reference{storage_ + compute_index(x, y, z)};
     }
 
     template <typename IndexType>
-    GKO_ATTRIBUTES arithmetic_type at(IndexType x, IndexType y,
-                                      IndexType z) const
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type at(IndexType x, IndexType y,
+                                                 IndexType z) const
     {
-        return reference{storage_ + x * stride_[0] + y * stride_[1] + z};
+        return reference{storage_ + compute_index(x, y, z)};
     }
 
-    GKO_ATTRIBUTES size_type get_stride0() const { return stride_[0]; }
-    GKO_ATTRIBUTES size_type get_stride1() const { return stride_[1]; }
+    GKO_ATTRIBUTES GKO_INLINE size_type get_stride0() const
+    {
+        return stride_[0];
+    }
+    GKO_ATTRIBUTES GKO_INLINE size_type get_stride1() const
+    {
+        return stride_[1];
+    }
 
-    GKO_ATTRIBUTES const storage_type *get_storage() const { return storage_; }
+    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_storage() const
+    {
+        return storage_;
+    }
 
-    GKO_ATTRIBUTES const storage_type *get_const_storage() const
+    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_const_storage() const
     {
         return storage_;
     }
 
 protected:
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE constexpr size_type compute_index(
+        IndexType x, IndexType y, IndexType z) const
+    {
+        return x * stride_[0] + y * stride_[1] + z;
+    }
+
     storage_type *storage_;
     size_type stride_[2];
 };
 
+template <typename ArithmeticType, typename StorageType>
+class ReducedStorage3d
+    : public ConstReducedStorage3d<ArithmeticType, StorageType> {
+public:
+    using arithmetic_type = ArithmeticType;
+    using storage_type = StorageType;
+    using const_accessor = ConstReducedStorage3d<arithmetic_type, storage_type>;
+
+protected:
+    using reference = typename const_accessor::reference;
+
+public:
+    GKO_ATTRIBUTES ReducedStorage3d() : const_accessor(nullptr, 0, 0) {}
+
+    GKO_ATTRIBUTES ReducedStorage3d(storage_type *storage, size_type stride0,
+                                    size_type stride1)
+        : const_accessor(storage, stride0, stride1)
+    {}
+
+    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const
+    {
+        return {this->storage_, this->stride_[0], this->stride_[1]};
+    }
+
+    using const_accessor::get_storage;
+    GKO_ATTRIBUTES GKO_INLINE storage_type *get_storage()
+    {
+        return this->storage_;
+    }
+
+    // Makes sure the operator() const function is visible
+    using const_accessor::operator();
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE reference operator()(IndexType x, IndexType y,
+                                                   IndexType z)
+    {
+        return {this->storage_ + const_accessor::compute_index(x, y, z)};
+    }
+
+    using const_accessor::at;  // Makes sure the at() const function is visible
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE reference at(IndexType x, IndexType y,
+                                           IndexType z)
+    {
+        return {this->storage_ + const_accessor::compute_index(x, y, z)};
+    }
+};
+
+
 /**
  * @internal
  *
- * The Accessor3d class hides the underlying storage_ format and provides a
- * simple interface for accessing a one dimensional storage.
+ * The ScaledReducedStorage3d class hides the underlying storage_ format and
+ * provides a simple interface for accessing a one dimensional storage.
  * Additionally, this accessor posesses a scale array, which is used for each
  * read and write operation to do a proper conversion.
  *
  * This class only manages the accesses, however, and not the memory itself.
  *
+ * TODO move this documentation to the core since this is a general purpose
+ * accessor
+ *
  * The dimensions {x, y, z} explained for the krylov_bases:
  * - x: selects the krylov vector (usually has krylov_dim + 1 vectors)
  * - y: selects the (row-)element of said krylov vector
@@ -282,47 +396,19 @@ class ConstAccessor3d<StorageType, ArithmeticType, false> {
  *
  * The accessor uses row-major access.
  */
-template <typename StorageType, typename ArithmeticType>
-class ConstAccessor3d<StorageType, ArithmeticType, true> {
+template <typename ArithmeticType, typename StorageType>
+class ConstScaledReducedStorage3d {
 public:
-    using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
-    using const_accessor = ConstAccessor3d;
-    static_assert(detail::helper_have_scale<StorageType, ArithmeticType>::value,
-                  "storage_type must not be an integral in this class.");
-
-    static constexpr bool has_scale{true};
+    using storage_type = StorageType;
+    using const_accessor = ConstScaledReducedStorage3d;
+    static_assert(!std::is_const<storage_type>::value,
+                  "StorageType must not be const!");
 
 protected:
-    class reference
-        : public detail::enable_reference<reference, arithmetic_type> {
-    public:
-        reference() = delete;
-        GKO_ATTRIBUTES reference(storage_type *const GKO_RESTRICT ptr,
-                                 arithmetic_type scale)
-            : ptr_{ptr}, scale_{scale}
-        {}
-        GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
-        {
-            const storage_type *const GKO_RESTRICT r_ptr = ptr_;
-            return static_cast<arithmetic_type>(*r_ptr) * scale_;
-        }
-        GKO_ATTRIBUTES GKO_INLINE arithmetic_type
-        operator=(arithmetic_type val) &&
-        {
-            storage_type *const GKO_RESTRICT r_ptr = ptr_;
-            return *r_ptr = static_cast<storage_type>(val / scale_), val;
-        }
-        GKO_ATTRIBUTES GKO_INLINE arithmetic_type
-        operator=(const reference &ref) &&
-        {
-            return std::move(*this) = static_cast<arithmetic_type>(ref);
-        }
-
-    private:
-        storage_type *const GKO_RESTRICT ptr_;
-        const arithmetic_type scale_;
-    };
+    using reference =
+        detail::reference::ScaledReducedStorageReference<ArithmeticType,
+                                                         StorageType>;
 
 public:
     /**
@@ -330,195 +416,152 @@ class ConstAccessor3d<StorageType, ArithmeticType, true> {
      * stride.
      *
      * @internal
-     * const_cast needed, so Accessor3d(ac.get_storage(), ...)
-     * works. Also, storage_ is never accessed in a non-const fashion, so it
-     * is not invalid or UB code.
+     * const_cast needed, so ScaledReducedStorage3d(ac.get_storage(), ...)
+     * works. Also, storage_ is never accessed in a non-const fashion (from
+     * ConstScaledReducedStorage3d), so it is not invalid or UB code.
      */
-    GKO_ATTRIBUTES ConstAccessor3d(const storage_type *storage,
-                                   size_type stride0, size_type stride1,
-                                   const arithmetic_type *scale)
+    GKO_ATTRIBUTES ConstScaledReducedStorage3d(const storage_type *storage,
+                                               size_type stride0,
+                                               size_type stride1,
+                                               const arithmetic_type *scale)
         : storage_{const_cast<storage_type *>(storage)},
           stride_{stride0, stride1},
           scale_{const_cast<arithmetic_type *>(scale)}
     {}
 
-    GKO_ATTRIBUTES const_accessor to_const() const { return *this; }
+    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const { return *this; }
 
     /**
      * Reads the scale value at the given indices.
      */
     template <typename IndexType>
-    GKO_ATTRIBUTES arithmetic_type read_scale(IndexType x, IndexType z) const
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type read_scale(IndexType x,
+                                                         IndexType z) const
     {
         const arithmetic_type *GKO_RESTRICT rest_scale = scale_;
         return rest_scale[x * stride_[1] + z];
     }
 
     template <typename IndexType>
-    GKO_ATTRIBUTES arithmetic_type at(IndexType x, IndexType y,
-                                      IndexType z) const
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator()(IndexType x,
+                                                         IndexType y,
+                                                         IndexType z) const
     {
-        return reference{storage_ + x * stride_[0] + y * stride_[1] + z,
-                         read_scale(x, z)};
+        return reference{storage_ + compute_index(x, y, z), read_scale(x, z)};
     }
 
-    /**
-     * Reads the value at the given index.
-     */
     template <typename IndexType>
-    GKO_ATTRIBUTES arithmetic_type read(IndexType x, IndexType y,
-                                        IndexType z) const
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type at(IndexType x, IndexType y,
+                                                 IndexType z) const
     {
-        // Make use of the restrict (and const) qualifier. If the restrict
-        // qualifier would have been put on the class attribute, it would be
-        // ignored by nvcc.
-        const storage_type *GKO_RESTRICT rest_storage = storage_;
-        return static_cast<arithmetic_type>(
-                   rest_storage[x * stride_[0] + y * stride_[1] + z]) *
-               read_scale(x, z);
+        return reference{storage_ + compute_index(x, y, z), read_scale(x, z)};
     }
 
-    GKO_ATTRIBUTES size_type get_stride0() const { return stride_[0]; }
-    GKO_ATTRIBUTES size_type get_stride1() const { return stride_[1]; }
-
-    GKO_ATTRIBUTES const storage_type *get_storage() const { return storage_; }
+    GKO_ATTRIBUTES GKO_INLINE size_type get_stride0() const
+    {
+        return stride_[0];
+    }
+    GKO_ATTRIBUTES GKO_INLINE size_type get_stride1() const
+    {
+        return stride_[1];
+    }
 
-    GKO_ATTRIBUTES const storage_type *get_const_storage() const
+    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_storage() const
     {
         return storage_;
     }
 
-    GKO_ATTRIBUTES const arithmetic_type *get_scale() const
+    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_const_storage() const
     {
-        return this->scale_;
+        return storage_;
     }
 
-    GKO_ATTRIBUTES const arithmetic_type *get_const_scale() const
+    GKO_ATTRIBUTES GKO_INLINE const arithmetic_type *get_scale() const
     {
         return this->scale_;
     }
 
-protected:
-    storage_type *storage_;
-    size_type stride_[2];
-    arithmetic_type *scale_;
-};
-
-template <typename StorageType, typename ArithmeticType,
-          bool = detail::helper_have_scale<StorageType, ArithmeticType>::value>
-class Accessor3d : public ConstAccessor3d<StorageType, ArithmeticType> {};
-
-template <typename StorageType, typename ArithmeticType>
-class Accessor3d<StorageType, ArithmeticType, false>
-    : public ConstAccessor3d<StorageType, ArithmeticType, false> {
-public:
-    using storage_type = StorageType;
-    using arithmetic_type = ArithmeticType;
-    using const_accessor = ConstAccessor3d<storage_type, arithmetic_type>;
-    static_assert(
-        !detail::helper_have_scale<StorageType, ArithmeticType>::value,
-        "storage_type must not be an integral in this class.");
-    static constexpr bool has_scale{false};
-
-protected:
-    using reference = typename const_accessor::reference;
-
-public:
-    GKO_ATTRIBUTES Accessor3d() : const_accessor(nullptr, 0, 0) {}
-
-    GKO_ATTRIBUTES Accessor3d(storage_type *storage, size_type stride0,
-                              size_type stride1)
-        : const_accessor(storage, stride0, stride1)
-    {}
-
-    GKO_ATTRIBUTES const_accessor to_const() const
+    GKO_ATTRIBUTES GKO_INLINE const arithmetic_type *get_const_scale() const
     {
-        return {this->storage_, this->stride_[0], this->stride_[1]};
+        return this->scale_;
     }
 
-    GKO_ATTRIBUTES operator const_accessor() const { return to_const(); }
-
-    using const_accessor::get_storage;
-    GKO_ATTRIBUTES storage_type *get_storage() { return this->storage_; }
-
-    /**
-     * Writes the given value at the given index.
-     */
+protected:
     template <typename IndexType>
-    GKO_ATTRIBUTES void write(IndexType x, IndexType y, IndexType z,
-                              arithmetic_type value)
+    GKO_ATTRIBUTES constexpr GKO_INLINE size_type
+    compute_index(IndexType x, IndexType y, IndexType z) const
     {
-        storage_type *GKO_RESTRICT rest_storage = this->storage_;
-        rest_storage[x * this->stride_[0] + y * this->stride_[1] + z] =
-            static_cast<storage_type>(value);
+        return x * stride_[0] + y * stride_[1] + z;
     }
 
-    using const_accessor::at;  // Makes sure the at() const function is visible
-    template <typename IndexType>
-    GKO_ATTRIBUTES reference at(IndexType x, IndexType y, IndexType z)
-    {
-        return {this->storage_ + x * this->stride_[0] + y * this->stride_[1] +
-                z};
-    }
+    storage_type *storage_;
+    size_type stride_[2];
+    arithmetic_type *scale_;
 };
 
 
-template <typename StorageType, typename ArithmeticType>
-class Accessor3d<StorageType, ArithmeticType, true>
-    : public ConstAccessor3d<StorageType, ArithmeticType, true> {
+template <typename ArithmeticType, typename StorageType>
+class ScaledReducedStorage3d
+    : public ConstScaledReducedStorage3d<ArithmeticType, StorageType> {
 public:
-    using storage_type = StorageType;
     using arithmetic_type = ArithmeticType;
-    using const_accessor = ConstAccessor3d<storage_type, arithmetic_type>;
-    static_assert(detail::helper_have_scale<StorageType, ArithmeticType>::value,
-                  "storage_type must not be an integral in this class.");
-
-    static constexpr bool has_scale{true};
+    using storage_type = StorageType;
+    using const_accessor =
+        ConstScaledReducedStorage3d<arithmetic_type, storage_type>;
 
 protected:
     using reference = typename const_accessor::reference;
 
 public:
-    GKO_ATTRIBUTES Accessor3d() : const_accessor(nullptr, {}, {}, nullptr) {}
+    GKO_ATTRIBUTES ScaledReducedStorage3d()
+        : const_accessor(nullptr, {}, {}, nullptr)
+    {}
 
-    GKO_ATTRIBUTES Accessor3d(storage_type *storage, size_type stride0,
-                              size_type stride1, arithmetic_type *scale)
+    GKO_ATTRIBUTES ScaledReducedStorage3d(storage_type *storage,
+                                          size_type stride0, size_type stride1,
+                                          arithmetic_type *scale)
         : const_accessor(storage, stride0, stride1, scale)
     {}
 
-    GKO_ATTRIBUTES const_accessor to_const() const
+    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const
     {
         return {this->storage_, this->stride_[0], this->stride_[1],
                 this->scale_};
     }
 
-    GKO_ATTRIBUTES operator const_accessor() const { return to_const(); }
+    // Makes sure the operator() const function is visible
+    using const_accessor::operator();
+    template <typename IndexType>
+    GKO_ATTRIBUTES GKO_INLINE reference operator()(IndexType x, IndexType y,
+                                                   IndexType z)
+    {
+        return {this->storage_ + const_accessor::compute_index(x, y, z),
+                this->read_scale(x, z)};
+    }
 
 
     using const_accessor::at;
     template <typename IndexType>
-    GKO_ATTRIBUTES reference at(IndexType x, IndexType y, IndexType z)
+    GKO_ATTRIBUTES GKO_INLINE reference at(IndexType x, IndexType y,
+                                           IndexType z)
     {
-        return {
-            this->storage_ + x * this->stride_[0] + y * this->stride_[1] + z,
-            this->read_scale(x, z)};
+        return {this->storage_ + const_accessor::compute_index(x, y, z),
+                this->read_scale(x, z)};
     }
 
     /**
      * Writes the given value at the given index.
      */
     template <typename IndexType>
-    GKO_ATTRIBUTES void write(IndexType x, IndexType y, IndexType z,
-                              arithmetic_type value)
+    GKO_ATTRIBUTES GKO_INLINE void write(IndexType x, IndexType y, IndexType z,
+                                         arithmetic_type value)
     {
         storage_type *GKO_RESTRICT rest_storage = this->storage_;
-        const auto stride0 = this->stride_[0];
-        const auto stride1 = this->stride_[1];
-        rest_storage[x * stride0 + y * stride1 + z] =
+        rest_storage[const_accessor::compute_index(x, y, z)] =
             static_cast<storage_type>(value / this->read_scale(x, z));
 #if GKO_DEBUG_OUTPUT && not defined(__CUDA_ARCH__)
-        std::cout << "storage[" << x << ", " << y << ", " << z
-                  << "] = " << rest_storage[x * stride0 + y * stride1 + z]
+        std::cout << "storage[" << x << ", " << y << ", " << z << "] = "
+                  << rest_storage[const_accessor::compute_index(x, y, z)]
                   << "; value = " << value
                   << "; scale = " << this->read_scale(x, z) << '\n';
 #endif
@@ -528,49 +571,59 @@ class Accessor3d<StorageType, ArithmeticType, true>
      * Writes the given value at the given index for a scale.
      */
     template <typename IndexType>
-    GKO_ATTRIBUTES void set_scale(IndexType x, IndexType z, arithmetic_type val)
+    GKO_ATTRIBUTES GKO_INLINE void set_scale(IndexType x, IndexType z,
+                                             arithmetic_type val)
     {
         arithmetic_type *GKO_RESTRICT rest_scale = this->scale_;
-        storage_type max_val = one<storage_type>();
-        // printf("(A) max_val = %d\n", max_val);
-        if (std::is_integral<storage_type>::value) {
-            max_val = std::numeric_limits<storage_type>::max();
-            //    printf("(B) max_val = %ld\n", max_val);
-        }
-        rest_scale[x * this->stride_[1] + z] =
-            val / static_cast<arithmetic_type>(max_val);
-#if GKO_DEBUG_OUTPUT && not defined(__CUDA_ARCH__)
-        std::cout << "scale[" << x << ", " << z
-                  << "] = " << rest_scale[x * this->stride_[1] + z]
-                  << "; val = " << val
-                  << "; max_val = " << static_cast<arithmetic_type>(max_val)
-                  << '\n';
-#endif
-        //    rest_scale[idx] = one<arithmetic_type>();
-        //    printf("val = %10.5e , rest_scale = %10.5e\n", val,
-        //    rest_scale[idx]); std::cout << val << " - " << rest_scale[idx] <<
-        //    std::endl;
+        rest_scale[x * this->stride_[1] + z] = val;
     }
 
     using const_accessor::get_storage;
-    GKO_ATTRIBUTES storage_type *get_storage() { return this->storage_; }
+    GKO_ATTRIBUTES GKO_INLINE storage_type *get_storage()
+    {
+        return this->storage_;
+    }
 
     using const_accessor::get_scale;
-    GKO_ATTRIBUTES arithmetic_type *get_scale() { return this->scale_; }
+    GKO_ATTRIBUTES GKO_INLINE arithmetic_type *get_scale()
+    {
+        return this->scale_;
+    }
 };
 
 
-template <typename ValueType, typename ValueTypeKrylovBases,
-          bool = Accessor3d<ValueTypeKrylovBases, ValueType>::has_scale>
+namespace detail {
+
+
+template <typename Accessor>
+struct is_3d_scaled_accessor : public std::false_type {};
+
+template <typename... Args>
+struct is_3d_scaled_accessor<ScaledReducedStorage3d<Args...>>
+    : public std::true_type {};
+
+template <typename StorageType, bool = std::is_integral<StorageType>::value>
+struct helper_require_scale {};
+
+template <typename StorageType>
+struct helper_require_scale<StorageType, false> : public std::false_type {};
+
+template <typename StorageType>
+struct helper_require_scale<StorageType, true> : public std::true_type {};
+
+
+}  // namespace detail
+
+
+template <typename ValueType, typename StorageType,
+          bool = detail::helper_require_scale<StorageType>::value>
 class Accessor3dHelper {};
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
-class Accessor3dHelper<ValueType, ValueTypeKrylovBases, true> {
+template <typename ValueType, typename StorageType>
+class Accessor3dHelper<ValueType, StorageType, true> {
 public:
-    using Accessor = Accessor3d<ValueTypeKrylovBases, ValueType>;
-    static_assert(Accessor::has_scale,
-                  "This accessor must have a scale in this class!");
+    using Accessor = ScaledReducedStorage3d<ValueType, StorageType>;
 
     Accessor3dHelper() = default;
 
@@ -596,21 +649,19 @@ class Accessor3dHelper<ValueType, ValueTypeKrylovBases, true> {
         return {bases_.get_data(), stride0, stride1, scale_.get_data()};
     }
 
-    gko::Array<ValueTypeKrylovBases> &get_bases() { return bases_; }
+    gko::Array<StorageType> &get_bases() { return bases_; }
 
 private:
     dim<3> krylov_dim_;
-    Array<ValueTypeKrylovBases> bases_;
+    Array<StorageType> bases_;
     Array<ValueType> scale_;
 };
 
 
-template <typename ValueType, typename ValueTypeKrylovBases>
-class Accessor3dHelper<ValueType, ValueTypeKrylovBases, false> {
+template <typename ValueType, typename StorageType>
+class Accessor3dHelper<ValueType, StorageType, false> {
 public:
-    using Accessor = Accessor3d<ValueTypeKrylovBases, ValueType>;
-    static_assert(!Accessor::has_scale,
-                  "This accessor must not have a scale in this class!");
+    using Accessor = ReducedStorage3d<ValueType, StorageType>;
 
     Accessor3dHelper() = default;
 
@@ -627,45 +678,53 @@ class Accessor3dHelper<ValueType, ValueTypeKrylovBases, false> {
         return {bases_.get_data(), stride0, stride1};
     }
 
-    gko::Array<ValueTypeKrylovBases> &get_bases() { return bases_; }
+    gko::Array<StorageType> &get_bases() { return bases_; }
 
 private:
     dim<3> krylov_dim_;
-    Array<ValueTypeKrylovBases> bases_;
+    Array<StorageType> bases_;
 };
 
 //----------------------------------------------
 
-template <typename Accessor3d, bool = Accessor3d::has_scale>
+template <typename Accessor3d,
+          bool = detail::is_3d_scaled_accessor<Accessor3d>::value>
 struct helper_functions_accessor {};
 
 // Accessors having a scale
 template <typename Accessor3d>
 struct helper_functions_accessor<Accessor3d, true> {
-    using value_type = typename Accessor3d::arithmetic_type;
-    static_assert(Accessor3d::has_scale, "Accessor must have a scale here!");
+    using arithmetic_type = typename Accessor3d::arithmetic_type;
+    static_assert(detail::is_3d_scaled_accessor<Accessor3d>::value,
+                  "Accessor must have a scale here!");
     template <typename IndexType>
     static inline GKO_ATTRIBUTES void write_scale(Accessor3d krylov_bases,
                                                   IndexType vector_idx,
                                                   IndexType col_idx,
-                                                  value_type value)
+                                                  arithmetic_type value)
     {
-        krylov_bases.set_scale(vector_idx, col_idx, value);
+        using storage_type = typename Accessor3d::storage_type;
+        constexpr arithmetic_type correction =
+            std::is_integral<storage_type>::value
+                // Use 2 instead of 1 here to allow for a bit more room
+                ? 2 / static_cast<arithmetic_type>(
+                          std::numeric_limits<storage_type>::max())
+                : 1;
+        krylov_bases.set_scale(vector_idx, col_idx, value * correction);
     }
 };
 
-
 // Accessors not having a scale
 template <typename Accessor3d>
 struct helper_functions_accessor<Accessor3d, false> {
-    using value_type = typename Accessor3d::arithmetic_type;
-    static_assert(!Accessor3d::has_scale,
+    using arithmetic_type = typename Accessor3d::arithmetic_type;
+    static_assert(!detail::is_3d_scaled_accessor<Accessor3d>::value,
                   "Accessor must not have a scale here!");
     template <typename IndexType>
     static inline GKO_ATTRIBUTES void write_scale(Accessor3d krylov_bases,
                                                   IndexType vector_idx,
                                                   IndexType col_idx,
-                                                  value_type value)
+                                                  arithmetic_type value)
     {}
 };
 
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index 398b4c25aec..10d32e13535 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -64,45 +64,54 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 // #else
 #define GKO_UNPACK(...) __VA_ARGS__
 
-#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro)                    \
-    template _macro(double, GKO_UNPACK(Accessor3d<double, double>));         \
-    template _macro(double, GKO_UNPACK(Accessor3d<float, double>));          \
-    template _macro(double, GKO_UNPACK(Accessor3d<int64, double>));          \
-    template _macro(double, GKO_UNPACK(Accessor3d<int32, double>));          \
-    template _macro(double, GKO_UNPACK(Accessor3d<int16, double>));          \
-    template _macro(double, GKO_UNPACK(Accessor3d<half, double>));           \
-    template _macro(float, GKO_UNPACK(Accessor3d<float, float>));            \
-    template _macro(float, GKO_UNPACK(Accessor3d<half, float>));             \
-    template _macro(                                                         \
-        std::complex<double>,                                                \
-        GKO_UNPACK(Accessor3d<std::complex<double>, std::complex<double>>)); \
-    template _macro(                                                         \
-        std::complex<double>,                                                \
-        GKO_UNPACK(Accessor3d<std::complex<float>, std::complex<double>>));  \
-    template _macro(                                                         \
-        std::complex<float>,                                                 \
-        GKO_UNPACK(Accessor3d<std::complex<float>, std::complex<float>>))
-
-#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(_macro)            \
-    template _macro(double, GKO_UNPACK(ConstAccessor3d<double, double>));  \
-    template _macro(double, GKO_UNPACK(ConstAccessor3d<float, double>));   \
-    template _macro(double, GKO_UNPACK(ConstAccessor3d<int64, double>));   \
-    template _macro(double, GKO_UNPACK(ConstAccessor3d<int32, double>));   \
-    template _macro(double, GKO_UNPACK(ConstAccessor3d<int16, double>));   \
-    template _macro(double, GKO_UNPACK(ConstAccessor3d<half, double>));    \
-    template _macro(float, GKO_UNPACK(ConstAccessor3d<float, float>));     \
-    template _macro(float, GKO_UNPACK(ConstAccessor3d<half, float>));      \
-    template _macro(                                                       \
-        std::complex<double>,                                              \
-        GKO_UNPACK(                                                        \
-            ConstAccessor3d<std::complex<double>, std::complex<double>>)); \
-    template _macro(                                                       \
-        std::complex<double>,                                              \
-        GKO_UNPACK(                                                        \
-            ConstAccessor3d<std::complex<float>, std::complex<double>>));  \
-    template _macro(                                                       \
-        std::complex<float>,                                               \
-        GKO_UNPACK(ConstAccessor3d<std::complex<float>, std::complex<float>>))
+#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro)                   \
+    template _macro(double, GKO_UNPACK(ReducedStorage3d<double, double>));  \
+    template _macro(double, GKO_UNPACK(ReducedStorage3d<double, float>));   \
+    template _macro(double,                                                 \
+                    GKO_UNPACK(ScaledReducedStorage3d<double, int64>));     \
+    template _macro(double,                                                 \
+                    GKO_UNPACK(ScaledReducedStorage3d<double, int32>));     \
+    template _macro(double,                                                 \
+                    GKO_UNPACK(ScaledReducedStorage3d<double, int16>));     \
+    template _macro(double, GKO_UNPACK(ReducedStorage3d<double, half>));    \
+    template _macro(float, GKO_UNPACK(ReducedStorage3d<float, float>));     \
+    template _macro(float, GKO_UNPACK(ReducedStorage3d<float, half>));      \
+    template _macro(                                                        \
+        std::complex<double>,                                               \
+        GKO_UNPACK(                                                         \
+            ReducedStorage3d<std::complex<double>, std::complex<double>>)); \
+    template _macro(                                                        \
+        std::complex<double>,                                               \
+        GKO_UNPACK(                                                         \
+            ReducedStorage3d<std::complex<double>, std::complex<float>>));  \
+    template _macro(                                                        \
+        std::complex<float>,                                                \
+        GKO_UNPACK(                                                         \
+            ReducedStorage3d<std::complex<float>, std::complex<float>>))
+
+#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(_macro)                \
+    template _macro(double,                                                    \
+                    GKO_UNPACK(ConstReducedStorage3d<double, double>));        \
+    template _macro(double, GKO_UNPACK(ConstReducedStorage3d<double, float>)); \
+    template _macro(double,                                                    \
+                    GKO_UNPACK(ConstScaledReducedStorage3d<double, int64>));   \
+    template _macro(double,                                                    \
+                    GKO_UNPACK(ConstScaledReducedStorage3d<double, int32>));   \
+    template _macro(double,                                                    \
+                    GKO_UNPACK(ConstScaledReducedStorage3d<double, int16>));   \
+    template _macro(double, GKO_UNPACK(ConstReducedStorage3d<double, half>));  \
+    template _macro(float, GKO_UNPACK(ConstReducedStorage3d<float, float>));   \
+    template _macro(float, GKO_UNPACK(ConstReducedStorage3d<float, half>));    \
+    template _macro(std::complex<double>,                                      \
+                    GKO_UNPACK(ConstReducedStorage3d<std::complex<double>,     \
+                                                     std::complex<double>>));  \
+    template _macro(std::complex<double>,                                      \
+                    GKO_UNPACK(ConstReducedStorage3d<std::complex<double>,     \
+                                                     std::complex<float>>));   \
+    template _macro(                                                           \
+        std::complex<float>,                                                   \
+        GKO_UNPACK(                                                            \
+            ConstReducedStorage3d<std::complex<float>, std::complex<float>>))
 // #endif
 // #undef GKO_UNPACK
 
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 7631c83d393..19fbdb79dd1 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -93,36 +93,32 @@ constexpr int default_update_dim = 32;
 
 // Specialization, so the Accessor can use the same function as regular pointers
 template <typename Type1, typename Type2>
-xstd::enable_if_t<!Accessor3d<Type1, Type2>::has_scale,
-                  Accessor3d<cuda_type<Type1>, cuda_type<Type2>>>
-as_cuda_accessor(Accessor3d<Type1, Type2> acc)
+ReducedStorage3d<cuda_type<Type1>, cuda_type<Type2>> as_cuda_accessor(
+    ReducedStorage3d<Type1, Type2> acc)
 {
     return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
             acc.get_stride1()};
 }
 
 template <typename Type1, typename Type2>
-xstd::enable_if_t<Accessor3d<Type1, Type2>::has_scale,
-                  Accessor3d<cuda_type<Type1>, cuda_type<Type2>>>
-as_cuda_accessor(Accessor3d<Type1, Type2> acc)
+ScaledReducedStorage3d<cuda_type<Type1>, cuda_type<Type2>> as_cuda_accessor(
+    ScaledReducedStorage3d<Type1, Type2> acc)
 {
     return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
             acc.get_stride1(), as_cuda_type(acc.get_scale())};
 }
 
 template <typename Type1, typename Type2>
-xstd::enable_if_t<!ConstAccessor3d<Type1, Type2>::has_scale,
-                  ConstAccessor3d<cuda_type<Type1>, cuda_type<Type2>>>
-as_cuda_accessor(const ConstAccessor3d<Type1, Type2> &acc)
+ConstReducedStorage3d<cuda_type<Type1>, cuda_type<Type2>> as_cuda_accessor(
+    const ConstReducedStorage3d<Type1, Type2> &acc)
 {
     return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
             acc.get_stride1()};
 }
 
 template <typename Type1, typename Type2>
-xstd::enable_if_t<ConstAccessor3d<Type1, Type2>::has_scale,
-                  ConstAccessor3d<cuda_type<Type1>, cuda_type<Type2>>>
-as_cuda_accessor(const ConstAccessor3d<Type1, Type2> &acc)
+ConstScaledReducedStorage3d<cuda_type<Type1>, cuda_type<Type2>>
+as_cuda_accessor(const ConstScaledReducedStorage3d<Type1, Type2> &acc)
 {
     return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
             acc.get_stride1(), as_cuda_type(acc.get_scale())};
@@ -259,7 +255,8 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     using non_complex = remove_complex<ValueType>;
     // optimization parameter
     constexpr int singledot_block_size = default_dot_dim;
-    constexpr bool use_scale = Accessor3dim::has_scale;
+    constexpr bool use_scale =
+        kernels::detail::is_3d_scaled_accessor<Accessor3dim>::value;
     const auto stride_next_krylov = next_krylov_basis->get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
     const auto stride_buffer = buffer_iter->get_stride();
@@ -283,7 +280,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     const dim3 block_size_iters_single(singledot_block_size);
     size_type numReorth;
 
-    Accessor3d<ValueType, ValueType> next_krylov_accessor{
+    ReducedStorage3d<ValueType, ValueType> next_krylov_accessor{
         next_krylov_basis->get_values(), stride_next_krylov,
         stride_next_krylov};
 
diff --git a/cuda/test/solver/gmres_mixed_kernels.cpp b/cuda/test/solver/gmres_mixed_kernels.cpp
index 2a0e391f6cc..cfbbfb9a9fc 100644
--- a/cuda/test/solver/gmres_mixed_kernels.cpp
+++ b/cuda/test/solver/gmres_mixed_kernels.cpp
@@ -58,12 +58,12 @@ namespace {
 class GmresMixed : public ::testing::Test {
 protected:
     using value_type = double;
-    using krylov_type = float;
+    using storage_type = float;
     using index_type = int;
     using size_type = gko::size_type;
-    using Accessor3d = gko::kernels::Accessor3d<krylov_type, value_type>;
+    using Accessor3d = gko::kernels::ReducedStorage3d<value_type, storage_type>;
     using Accessor3dHelper =
-        gko::kernels::Accessor3dHelper<value_type, krylov_type>;
+        gko::kernels::Accessor3dHelper<value_type, storage_type>;
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
 
@@ -100,18 +100,17 @@ class GmresMixed : public ::testing::Test {
         auto temp_krylov_bases = gko::test::generate_random_matrix<Dense>(
             num_rows, num_cols,
             std::uniform_int_distribution<index_type>(num_cols, num_cols),
-            std::normal_distribution<krylov_type>(-1.0, 1.0), rand_engine, ref);
+            std::normal_distribution<storage_type>(-1.0, 1.0), rand_engine,
+            ref);
         std::copy_n(temp_krylov_bases->get_const_values(),
                     bases.get_num_elems(), bases.get_data());
-        if (Accessor3d::has_scale) {
-            auto accessor = helper.get_accessor();
-            auto dist = std::normal_distribution<value_type>(-1, 1);
-            for (size_type k = 0; k < size[0]; ++k) {
-                for (size_type i = 0; i < size[2]; ++i) {
-                    gko::kernels::helper_functions_accessor<
-                        Accessor3d>::write_scale(accessor, k, i,
-                                                 dist(rand_engine));
-                }
+        // Only useful when the Accessor actually has a scale
+        auto accessor = helper.get_accessor();
+        auto dist = std::normal_distribution<value_type>(-1, 1);
+        for (size_type k = 0; k < size[0]; ++k) {
+            for (size_type i = 0; i < size[2]; ++i) {
+                gko::kernels::helper_functions_accessor<
+                    Accessor3d>::write_scale(accessor, k, i, dist(rand_engine));
             }
         }
         return helper;
@@ -212,10 +211,10 @@ class GmresMixed : public ::testing::Test {
 
     void assert_krylov_bases_near()
     {
-        gko::Array<krylov_type> d_to_host{ref};
+        gko::Array<storage_type> d_to_host{ref};
         auto &krylov_bases = acc_helper.get_bases();
         d_to_host = d_acc_helper.get_bases();
-        const auto tolerance = r<krylov_type>::value;
+        const auto tolerance = r<storage_type>::value;
         using std::abs;
         for (gko::size_type i = 0; i < krylov_bases.get_num_elems(); ++i) {
             const auto ref_value = krylov_bases.get_const_data()[i];
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index 72af0253047..9c69542906a 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -104,7 +104,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
             ValueType hessenberg_iter_entry = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter_entry +=
-                    next_krylov_basis->at(j, i) * krylov_bases.read(k, j, i);
+                    next_krylov_basis->at(j, i) * krylov_bases(k, j, i);
             }
             hessenberg_iter->at(k, i) = hessenberg_iter_entry;
         }
@@ -124,7 +124,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
 #pragma omp parallel for
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) * krylov_bases.read(k, j, i);
+                    hessenberg_iter->at(k, i) * krylov_bases(k, j, i);
             }
         }
 #ifdef TIMING
@@ -166,8 +166,8 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                 ValueType hessenberg_iter_entry = zero<ValueType>();
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
-                    hessenberg_iter_entry += next_krylov_basis->at(j, i) *
-                                             krylov_bases.read(k, j, i);
+                    hessenberg_iter_entry +=
+                        next_krylov_basis->at(j, i) * krylov_bases(k, j, i);
                 }
                 buffer_iter->at(k, i) = hessenberg_iter_entry;
             }
@@ -179,7 +179,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
-                        buffer_iter->at(k, i) * krylov_bases.read(k, j, i);
+                        buffer_iter->at(k, i) * krylov_bases(k, j, i);
                 }
             }
             // for i in 1:iter
@@ -208,7 +208,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
 #pragma omp parallel for
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.write(iter + 1, j, i, next_krylov_basis->at(j, i));
+            krylov_bases(iter + 1, j, i) = next_krylov_basis->at(j, i);
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -339,7 +339,7 @@ void calculate_qy(ConstAccessor3d krylov_bases,
             before_preconditioner->at(i, k) = zero<ValueType>();
             for (size_type j = 0; j < final_iter_nums[k]; ++j) {
                 before_preconditioner->at(i, k) +=
-                    krylov_bases.read(j, i, k) * y->at(j, k);
+                    krylov_bases(j, i, k) * y->at(j, k);
             }
         }
     }
@@ -432,7 +432,7 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             auto value = residual->at(i, j) / residual_norm->at(0, j);
-            krylov_bases.write({0}, i, j, value);
+            krylov_bases({0}, i, j) = value;
             next_krylov_basis->at(i, j) = value;
         }
         final_iter_nums->get_data()[j] = 0;
@@ -446,7 +446,7 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
         }
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             for (size_type j = 0; j < residual->get_size()[1]; ++j) {
-                krylov_bases.write(k, i, j, zero<ValueType>());
+                krylov_bases(k, i, j) = zero<ValueType>();
             }
         }
     }
diff --git a/omp/test/solver/gmres_mixed_kernels.cpp b/omp/test/solver/gmres_mixed_kernels.cpp
index 09959a2ddea..566ccd5fcfc 100644
--- a/omp/test/solver/gmres_mixed_kernels.cpp
+++ b/omp/test/solver/gmres_mixed_kernels.cpp
@@ -60,12 +60,12 @@ namespace {
 class GmresMixed : public ::testing::Test {
 protected:
     using value_type = double;
-    using krylov_type = float;
+    using storage_type = float;
     using index_type = int;
     using size_type = gko::size_type;
-    using Accessor3d = gko::kernels::Accessor3d<krylov_type, value_type>;
+    using Accessor3d = gko::kernels::ReducedStorage3d<value_type, storage_type>;
     using Accessor3dHelper =
-        gko::kernels::Accessor3dHelper<value_type, krylov_type>;
+        gko::kernels::Accessor3dHelper<value_type, storage_type>;
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
 
@@ -102,18 +102,17 @@ class GmresMixed : public ::testing::Test {
         auto temp_krylov_bases = gko::test::generate_random_matrix<Dense>(
             num_rows, num_cols,
             std::uniform_int_distribution<index_type>(num_cols, num_cols),
-            std::normal_distribution<krylov_type>(-1.0, 1.0), rand_engine, ref);
+            std::normal_distribution<storage_type>(-1.0, 1.0), rand_engine,
+            ref);
         std::copy_n(temp_krylov_bases->get_const_values(),
                     bases.get_num_elems(), bases.get_data());
-        if (Accessor3d::has_scale) {
-            auto accessor = helper.get_accessor();
-            auto dist = std::normal_distribution<value_type>(-1, 1);
-            for (size_type k = 0; k < size[0]; ++k) {
-                for (size_type i = 0; i < size[2]; ++i) {
-                    gko::kernels::helper_functions_accessor<
-                        Accessor3d>::write_scale(accessor, k, i,
-                                                 dist(rand_engine));
-                }
+        // Only useful when the Accessor actually has a scale
+        auto accessor = helper.get_accessor();
+        auto dist = std::normal_distribution<value_type>(-1, 1);
+        for (size_type k = 0; k < size[0]; ++k) {
+            for (size_type i = 0; i < size[2]; ++i) {
+                gko::kernels::helper_functions_accessor<
+                    Accessor3d>::write_scale(accessor, k, i, dist(rand_engine));
             }
         }
         return helper;
@@ -213,10 +212,10 @@ class GmresMixed : public ::testing::Test {
 
     void assert_krylov_bases_near()
     {
-        gko::Array<krylov_type> d_to_host{ref};
+        gko::Array<storage_type> d_to_host{ref};
         auto &krylov_bases = acc_helper.get_bases();
         d_to_host = d_acc_helper.get_bases();
-        const auto tolerance = r<krylov_type>::value;
+        const auto tolerance = r<storage_type>::value;
         using std::abs;
         for (gko::size_type i = 0; i < krylov_bases.get_num_elems(); ++i) {
             const auto ref_value = krylov_bases.get_const_data()[i];
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index a1b3c5336d7..50ca1914eaa 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -85,7 +85,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
             hessenberg_iter->at(k, i) = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
-                    next_krylov_basis->at(j, i) * krylov_bases.at(k, j, i);
+                    next_krylov_basis->at(j, i) * krylov_bases(k, j, i);
             }
         }
         // for i in 1:iter
@@ -94,7 +94,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         for (size_type k = 0; k < iter + 1; ++k) {
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 next_krylov_basis->at(j, i) -=
-                    hessenberg_iter->at(k, i) * krylov_bases.at(k, j, i);
+                    hessenberg_iter->at(k, i) * krylov_bases(k, j, i);
             }
         }
         // for i in 1:iter
@@ -120,7 +120,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     buffer_iter->at(k, i) +=
-                        next_krylov_basis->at(j, i) * krylov_bases.at(k, j, i);
+                        next_krylov_basis->at(j, i) * krylov_bases(k, j, i);
                 }
             }
             // for i in 1:iter
@@ -130,7 +130,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
-                        buffer_iter->at(k, i) * krylov_bases.at(k, j, i);
+                        buffer_iter->at(k, i) * krylov_bases(k, j, i);
                 }
                 hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
             }
@@ -158,7 +158,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         // hessenberg(iter, iter + 1) = norm(next_krylov_basis)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             next_krylov_basis->at(j, i) /= hessenberg_iter->at(iter + 1, i);
-            krylov_bases.at(iter + 1, j, i) = next_krylov_basis->at(j, i);
+            krylov_bases(iter + 1, j, i) = next_krylov_basis->at(j, i);
         }
         // next_krylov_basis /= hessenberg(iter, iter + 1)
         // krylov_bases(:, iter + 1) = next_krylov_basis
@@ -289,7 +289,7 @@ void calculate_qy(ConstAccessor3d krylov_bases,
             before_preconditioner->at(i, k) = zero<ValueType>();
             for (size_type j = 0; j < final_iter_nums[k]; ++j) {
                 before_preconditioner->at(i, k) +=
-                    krylov_bases.at(j, i, k) * y->at(j, k);
+                    krylov_bases(j, i, k) * y->at(j, k);
             }
         }
     }
@@ -368,7 +368,7 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
             }
         }
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
-            krylov_bases.at({0}, i, j) =
+            krylov_bases({0}, i, j) =
                 residual->at(i, j) / residual_norm->at(0, j);
             next_krylov_basis->at(i, j) =
                 residual->at(i, j) / residual_norm->at(0, j);
@@ -381,7 +381,7 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
             helper_functions_accessor<Accessor3d>::write_scale(
                 krylov_bases, k, j, one<remove_complex<ValueType>>());
             for (size_type i = 0; i < residual->get_size()[0]; ++i) {
-                krylov_bases.at(k, i, j) = zero<ValueType>();
+                krylov_bases(k, i, j) = zero<ValueType>();
             }
         }
     }

From 3bf5da0a22b0cc64aa8fd1a9d7dded8b7e861e43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 1 Sep 2020 16:46:30 +0200
Subject: [PATCH 0384/1008] Adopt to new parameter macros

---
 include/ginkgo/core/solver/gmres_mixed.hpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/include/ginkgo/core/solver/gmres_mixed.hpp b/include/ginkgo/core/solver/gmres_mixed.hpp
index 4bd017e6486..13145be56c6 100644
--- a/include/ginkgo/core/solver/gmres_mixed.hpp
+++ b/include/ginkgo/core/solver/gmres_mixed.hpp
@@ -109,32 +109,32 @@ class GmresMixed : public EnableLinOp<GmresMixed<ValueType>>,
         /**
          * Determines which storage type is used.
          */
-        gmres_mixed_storage_precision GKO_FACTORY_PARAMETER(
+        gmres_mixed_storage_precision GKO_FACTORY_PARAMETER_SCALAR(
             storage_precision, gmres_mixed_storage_precision::keep);
 
         /**
          * Criterion factories.
          */
         std::vector<std::shared_ptr<const stop::CriterionFactory>>
-            GKO_FACTORY_PARAMETER(criteria, nullptr);
+            GKO_FACTORY_PARAMETER_VECTOR(criteria, nullptr);
 
         /**
          * Preconditioner factory.
          */
-        std::shared_ptr<const LinOpFactory> GKO_FACTORY_PARAMETER(
+        std::shared_ptr<const LinOpFactory> GKO_FACTORY_PARAMETER_SCALAR(
             preconditioner, nullptr);
 
         /**
          * Already generated preconditioner. If one is provided, the factory
          * `preconditioner` will be ignored.
          */
-        std::shared_ptr<const LinOp> GKO_FACTORY_PARAMETER(
+        std::shared_ptr<const LinOp> GKO_FACTORY_PARAMETER_SCALAR(
             generated_preconditioner, nullptr);
 
         /**
          * krylov dimension factory.
          */
-        size_type GKO_FACTORY_PARAMETER(krylov_dim, 0u);
+        size_type GKO_FACTORY_PARAMETER_SCALAR(krylov_dim, 0u);
     };
     GKO_ENABLE_LIN_OP_FACTORY(GmresMixed, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);

From 256f13c15cb947f1081e5dc4e71306989407782a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 7 Sep 2020 19:52:10 +0200
Subject: [PATCH 0385/1008] Update the helper to throw when complex

Also add instantiation for single precision floating point
---
 core/solver/gmres_mixed.cpp         | 79 +++++++++++++++++++++++------
 core/solver/gmres_mixed_kernels.hpp | 54 +++++++++++---------
 2 files changed, 94 insertions(+), 39 deletions(-)

diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 8795e9186c4..22c9942fb3e 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -119,19 +119,76 @@ template <typename T>
 using to_integer = typename to_integer_impl<T>::type;
 
 
+template <typename T, typename Skip>
+using reduce_precision_skip =
+    typename std::conditional_t<std::is_same<reduce_precision<T>, Skip>::value,
+                                T, reduce_precision<T>>;
+
+
+namespace detail {
+
+
+template <typename T, typename Skip, int count>
+struct reduce_precision_skip_count_impl {
+    static_assert(count > 0,
+                  "The count variable must be larger or equal zero.");
+    using type = typename reduce_precision_skip_count_impl<
+        reduce_precision_skip<T, Skip>, Skip, count - 1>::type;
+};
+
+template <typename T, typename Skip>
+struct reduce_precision_skip_count_impl<T, Skip, 0> {
+    using type = T;
+};
+
+
+}  // namespace detail
+
+
+template <typename T, typename Skip, int count>
+using reduce_precision_skip_count =
+    typename detail::reduce_precision_skip_count_impl<T, Skip, count>::type;
+
+
+template <typename T, int count>
+using reduce_precision_count =
+    typename detail::reduce_precision_skip_count_impl<T, void, count>::type;
+
+
 template <typename ValueType>
 struct helper {
     template <typename Callable>
     static void call(Callable callable,
                      gko::solver::gmres_mixed_storage_precision st)
     {
-        callable(ValueType{});
+        switch (st) {
+        case gmres_mixed_storage_precision::reduce1:
+            callable(reduce_precision_count<ValueType, 1>{});
+            break;
+        case gmres_mixed_storage_precision::reduce2:
+            callable(reduce_precision_count<ValueType, 2>{});
+            break;
+        case gmres_mixed_storage_precision::integer:
+            callable(to_integer<ValueType>{});
+            break;
+        case gmres_mixed_storage_precision::ireduce1:
+            callable(to_integer<reduce_precision_count<ValueType, 1>>{});
+            break;
+        case gmres_mixed_storage_precision::ireduce2:
+            callable(to_integer<reduce_precision_count<ValueType, 2>>{});
+            break;
+        default:
+            callable(ValueType{});
+        }
     }
 };
 
-template <>
-struct helper<double> {
-    using ValueType = double;
+
+// helper for complex numbers
+template <typename T>
+struct helper<std::complex<T>> {
+    using ValueType = std::complex<T>;
+    using skip_type = std::complex<half>;
 
     template <typename Callable>
     static void call(Callable callable,
@@ -139,23 +196,15 @@ struct helper<double> {
     {
         switch (st) {
         case gmres_mixed_storage_precision::reduce1:
-            callable(reduce_precision<ValueType>{});
+            callable(reduce_precision_skip_count<ValueType, skip_type, 1>{});
             break;
         case gmres_mixed_storage_precision::reduce2:
-            callable(reduce_precision<reduce_precision<ValueType>>{});
+            callable(reduce_precision_skip_count<ValueType, skip_type, 2>{});
             break;
         case gmres_mixed_storage_precision::integer:
-            // check_for_complex();
-            callable(to_integer<ValueType>{});
-            break;
         case gmres_mixed_storage_precision::ireduce1:
-            // check_for_complex();
-            callable(to_integer<reduce_precision<ValueType>>{});
-            break;
         case gmres_mixed_storage_precision::ireduce2:
-            // check_for_complex();
-            callable(
-                to_integer<reduce_precision<reduce_precision<ValueType>>>{});
+            GKO_NOT_SUPPORTED(st);
             break;
         default:
             callable(ValueType{});
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index 10d32e13535..b4db2b61a7b 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -64,44 +64,50 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 // #else
 #define GKO_UNPACK(...) __VA_ARGS__
 
-#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro)                   \
-    template _macro(double, GKO_UNPACK(ReducedStorage3d<double, double>));  \
-    template _macro(double, GKO_UNPACK(ReducedStorage3d<double, float>));   \
-    template _macro(double,                                                 \
-                    GKO_UNPACK(ScaledReducedStorage3d<double, int64>));     \
-    template _macro(double,                                                 \
-                    GKO_UNPACK(ScaledReducedStorage3d<double, int32>));     \
-    template _macro(double,                                                 \
-                    GKO_UNPACK(ScaledReducedStorage3d<double, int16>));     \
-    template _macro(double, GKO_UNPACK(ReducedStorage3d<double, half>));    \
-    template _macro(float, GKO_UNPACK(ReducedStorage3d<float, float>));     \
-    template _macro(float, GKO_UNPACK(ReducedStorage3d<float, half>));      \
-    template _macro(                                                        \
-        std::complex<double>,                                               \
-        GKO_UNPACK(                                                         \
-            ReducedStorage3d<std::complex<double>, std::complex<double>>)); \
-    template _macro(                                                        \
-        std::complex<double>,                                               \
-        GKO_UNPACK(                                                         \
-            ReducedStorage3d<std::complex<double>, std::complex<float>>));  \
-    template _macro(                                                        \
-        std::complex<float>,                                                \
-        GKO_UNPACK(                                                         \
+#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro)                     \
+    template _macro(double, GKO_UNPACK(ReducedStorage3d<double, double>));    \
+    template _macro(double, GKO_UNPACK(ReducedStorage3d<double, float>));     \
+    template _macro(double, GKO_UNPACK(ReducedStorage3d<double, half>));      \
+    template _macro(double,                                                   \
+                    GKO_UNPACK(ScaledReducedStorage3d<double, int64>));       \
+    template _macro(double,                                                   \
+                    GKO_UNPACK(ScaledReducedStorage3d<double, int32>));       \
+    template _macro(double,                                                   \
+                    GKO_UNPACK(ScaledReducedStorage3d<double, int16>));       \
+    template _macro(float, GKO_UNPACK(ReducedStorage3d<float, float>));       \
+    template _macro(float, GKO_UNPACK(ReducedStorage3d<float, half>));        \
+    template _macro(float, GKO_UNPACK(ScaledReducedStorage3d<float, int32>)); \
+    template _macro(float, GKO_UNPACK(ScaledReducedStorage3d<float, int16>)); \
+    template _macro(                                                          \
+        std::complex<double>,                                                 \
+        GKO_UNPACK(                                                           \
+            ReducedStorage3d<std::complex<double>, std::complex<double>>));   \
+    template _macro(                                                          \
+        std::complex<double>,                                                 \
+        GKO_UNPACK(                                                           \
+            ReducedStorage3d<std::complex<double>, std::complex<float>>));    \
+    template _macro(                                                          \
+        std::complex<float>,                                                  \
+        GKO_UNPACK(                                                           \
             ReducedStorage3d<std::complex<float>, std::complex<float>>))
 
 #define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(_macro)                \
     template _macro(double,                                                    \
                     GKO_UNPACK(ConstReducedStorage3d<double, double>));        \
     template _macro(double, GKO_UNPACK(ConstReducedStorage3d<double, float>)); \
+    template _macro(double, GKO_UNPACK(ConstReducedStorage3d<double, half>));  \
     template _macro(double,                                                    \
                     GKO_UNPACK(ConstScaledReducedStorage3d<double, int64>));   \
     template _macro(double,                                                    \
                     GKO_UNPACK(ConstScaledReducedStorage3d<double, int32>));   \
     template _macro(double,                                                    \
                     GKO_UNPACK(ConstScaledReducedStorage3d<double, int16>));   \
-    template _macro(double, GKO_UNPACK(ConstReducedStorage3d<double, half>));  \
     template _macro(float, GKO_UNPACK(ConstReducedStorage3d<float, float>));   \
     template _macro(float, GKO_UNPACK(ConstReducedStorage3d<float, half>));    \
+    template _macro(float,                                                     \
+                    GKO_UNPACK(ConstScaledReducedStorage3d<float, int32>));    \
+    template _macro(float,                                                     \
+                    GKO_UNPACK(ConstScaledReducedStorage3d<float, int16>));    \
     template _macro(std::complex<double>,                                      \
                     GKO_UNPACK(ConstReducedStorage3d<std::complex<double>,     \
                                                      std::complex<double>>));  \

From dd644266be1b2bef9b40154fce853722bceaed18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 9 Sep 2020 17:38:11 +0200
Subject: [PATCH 0386/1008] Make GmresMixed work properly with multiple RHS

Also adjust test precision to be more accurate.
---
 core/solver/gmres_mixed.cpp                   | 130 ++++++++----------
 reference/test/solver/gmres_mixed_kernels.cpp |  16 +--
 2 files changed, 69 insertions(+), 77 deletions(-)

diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 22c9942fb3e..5ad3fbbc341 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -225,7 +225,8 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         using Vector = matrix::Dense<ValueType>;
         using VectorNorms = matrix::Dense<remove_complex<ValueType>>;
         using LowArray = Array<storage_type>;
-        // using KrylovAccessor = kernels::Accessor3d<storage_type, ValueType>;
+        // using KrylovAccessor = kernels::Accessor3d<storage_type,
+        // ValueType>;
         using Accessor3dHelper =
             kernels::Accessor3dHelper<ValueType, storage_type>;
 
@@ -336,7 +337,14 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         auto time_STEP1 = start - start;
 #endif
 #endif
-        bool stop_already_encountered{false};
+        Array<bool> stop_encountered_rhs(exec->get_master(),
+                                         dense_b->get_size()[1]);
+        Array<bool> fully_converged_rhs(exec->get_master(),
+                                        dense_b->get_size()[1]);
+        for (size_type i = 0; i < stop_encountered_rhs.get_num_elems(); ++i) {
+            stop_encountered_rhs.get_data()[i] = false;
+            fully_converged_rhs.get_data()[i] = false;
+        }
         bool perform_reset{false};
         decltype(krylov_dim_) forced_iterations{0};
         const decltype(forced_iterations) forced_limit{krylov_dim_ / 10};
@@ -350,47 +358,62 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             ++total_iter;
             this->template log<log::Logger::iteration_complete>(
                 this, total_iter, residual.get(), dense_x, residual_norm.get());
-            if (stop_already_encountered && forced_iterations < forced_limit) {
+            if (forced_iterations < forced_limit) {
                 ++forced_iterations;
-            } else if (stop_criterion->update()
-                           .num_iterations(total_iter)
-                           .residual(residual.get())
-                           .residual_norm(residual_norm.get())
-                           .solution(dense_x)
-                           .check(RelativeStoppingId, true, &stop_status,
-                                  &one_changed)) {
-                if (stop_already_encountered) {
-                    // forced_iterations = 0;
-                    break;
-                }
-                stop_already_encountered = true;
-                /*
-                std::cout << type_str << ": " << ++total_checks
-                          << ". check in iteration " << total_iter << "; "
-                          << forced_iterations << " / " << forced_limit <<
-                '\n';
-                */
-                Array<stopping_status> host_stop_status(
-                    this->get_executor()->get_master(), stop_status);
-                bool host_array_changed{false};
-                for (size_type i = 0; i < host_stop_status.get_num_elems();
-                     ++i) {
-                    auto local_status = host_stop_status.get_data() + i;
-                    // TODO: ignore all actually converged ones!
-                    if (local_status->has_converged()) {
-                        local_status->reset();
-                        host_array_changed = true;
+            } else {
+                bool all_changed = stop_criterion->update()
+                                       .num_iterations(total_iter)
+                                       .residual(residual.get())
+                                       .residual_norm(residual_norm.get())
+                                       .solution(dense_x)
+                                       .check(RelativeStoppingId, true,
+                                              &stop_status, &one_changed);
+                if (one_changed || all_changed) {
+                    /*
+                    std::cout << type_str << ": " << ++total_checks
+                              << ". check in iteration " << total_iter << ";
+                    "
+                              << forced_iterations << " / " << forced_limit
+                    <<
+                    '\n';
+                    */
+                    Array<stopping_status> host_stop_status(
+                        this->get_executor()->get_master(), stop_status);
+                    bool host_array_changed{false};
+                    for (size_type i = 0; i < host_stop_status.get_num_elems();
+                         ++i) {
+                        auto local_status = host_stop_status.get_data() + i;
+                        // TODO: ignore all actually converged ones!
+                        if (fully_converged_rhs.get_data()[i]) {
+                            continue;
+                        }
+                        if (local_status->has_converged()) {
+                            if (stop_encountered_rhs.get_data()[i]) {
+                                fully_converged_rhs.get_data()[i] = true;
+                            } else {
+                                stop_encountered_rhs.get_data()[i] = true;
+                                local_status->reset();
+                                host_array_changed = true;
+                            }
+                        }
                     }
-                }
-                if (host_array_changed) {
-                    perform_reset = true;
-                    stop_status = host_stop_status;
+                    if (host_array_changed) {
+                        perform_reset = true;
+                        stop_status = host_stop_status;
+                    } else {
+                        // Stop here can happen if all RHS are "fully_converged"
+                        // or if it was stopped for non-convergence reason
+                        // (like time or iteration)
+                        break;
+                    }
+
                 } else {
-                    break;
+                    for (size_type i = 0;
+                         i < stop_encountered_rhs.get_num_elems(); ++i) {
+                        stop_encountered_rhs.get_data()[i] = false;
+                    }
                 }
-            } else {
                 forced_iterations = 0;
-                stop_already_encountered = false;
             }
 
             if (perform_reset || restart_iter == krylov_dim_) {
@@ -399,7 +422,6 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                 exec->synchronize();
                 auto t_aux_0 = std::chrono::steady_clock::now();
 #endif
-                //            std::cout << "RESTARTING" << std::endl;
                 num_restarts++;
                 // Restart
                 // use a view in case this is called earlier
@@ -584,38 +606,8 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 #endif
     };  // End of apply_lambda
 
-    auto check_for_complex = [storage_precision = storage_precision_]() {
-        if (is_complex_s<ValueType>::value) {
-            GKO_NOT_SUPPORTED(storage_precision);
-        }
-    };
     // Look which precision to use as the storage type
     helper<ValueType>::call(apply_templated, storage_precision_);
-    /*
-    switch (storage_precision_) {
-    case gmres_mixed_storage_precision::reduce1:
-        apply_templated(reduce_precision<ValueType>{});
-        break;
-    case gmres_mixed_storage_precision::reduce2:
-        apply_templated(reduce_precision<reduce_precision<ValueType>>{});
-        break;
-    case gmres_mixed_storage_precision::integer:
-        check_for_complex();
-        apply_templated(to_integer<ValueType>{});
-        break;
-    case gmres_mixed_storage_precision::ireduce1:
-        check_for_complex();
-        apply_templated(to_integer<reduce_precision<ValueType>>{});
-        break;
-    case gmres_mixed_storage_precision::ireduce2:
-        check_for_complex();
-        apply_templated(
-            to_integer<reduce_precision<reduce_precision<ValueType>>>{});
-        break;
-    default:
-        apply_templated(ValueType{});
-    }
-    */
 }
 
 
diff --git a/reference/test/solver/gmres_mixed_kernels.cpp b/reference/test/solver/gmres_mixed_kernels.cpp
index e7e93028461..aa6b2ec19cd 100644
--- a/reference/test/solver/gmres_mixed_kernels.cpp
+++ b/reference/test/solver/gmres_mixed_kernels.cpp
@@ -206,7 +206,7 @@ TYPED_TEST(GmresMixed, SolvesStencilSystem)
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}), this->precision() * 1e1);
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}), this->precision());
 }
 
 
@@ -251,7 +251,7 @@ TYPED_TEST(GmresMixed, SolvesMultipleStencilSystems)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({I<T>{1.0, 1.0}, I<T>{3.0, 1.0}, I<T>{2.0, 1.0}}),
-                        this->precision() * 1e1);
+                        this->precision());
 }
 
 
@@ -267,7 +267,7 @@ TYPED_TEST(GmresMixed, SolvesStencilSystemUsingAdvancedApply)
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), this->precision() * 1e1);
+    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), this->precision());
 }
 
 
@@ -286,7 +286,7 @@ TYPED_TEST(GmresMixed, SolvesMultipleStencilSystemsUsingAdvancedApply)
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({I<T>{1.5, 1.0}, I<T>{5.0, 0.0}, I<T>{2.0, -1.0}}),
-                        this->precision() * 1e1);
+                        this->precision());
 }
 
 
@@ -303,7 +303,7 @@ TYPED_TEST(GmresMixed, SolvesBigDenseSystem1)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({52.7, 85.4, 134.2, -250.0, -16.8, 35.3}),
-                        this->precision() * 1e3);
+                        this->precision() * 1e1);
 }
 
 
@@ -320,7 +320,7 @@ TYPED_TEST(GmresMixed, SolvesBigDenseSystem2)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
-                        this->precision() * 1e3);
+                        this->precision() * 1e1);
 }
 
 
@@ -431,7 +431,7 @@ TYPED_TEST(GmresMixed, SolvesBigDenseSystem1WithRestart)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({-140.20, -142.20, 48.80, -17.70, -19.60}),
-                        half_tol * 1e2);
+                        half_tol);
 }
 
 
@@ -463,7 +463,7 @@ TYPED_TEST(GmresMixed, SolvesWithPreconditioner)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
-                        this->precision() * 1e3);
+                        this->precision() * 1e1);
 }
 
 

From 2f08803264cabab73a7e9377c3ffae310742437b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 10 Sep 2020 19:50:59 +0200
Subject: [PATCH 0387/1008] Fix benchmark to work with new GmresMixed layout

Make GmresMixed reference test work on CI.
---
 benchmark/solver/solver.cpp                   | 36 ++++++++++++-------
 reference/test/solver/gmres_mixed_kernels.cpp |  8 ++---
 2 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 81dec07c45f..60dea916357 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -262,54 +262,66 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"gmres_mixed_double",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::GmresMixed<double, double>::build()
+             return gko::solver::GmresMixed<double>::build()
+                 .with_storage_precision(
+                     gko::solver::gmres_mixed_storage_precision::keep)
                  .with_criteria(create_criterion(exec))
-                 .with_krylov_dim_mixed(FLAGS_gmres_restart)
+                 .with_krylov_dim(FLAGS_gmres_restart)
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
         {"gmres_mixed_float",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::GmresMixed<double, float>::build()
+             return gko::solver::GmresMixed<double>::build()
+                 .with_storage_precision(
+                     gko::solver::gmres_mixed_storage_precision::reduce1)
                  .with_criteria(create_criterion(exec))
-                 .with_krylov_dim_mixed(FLAGS_gmres_restart)
+                 .with_krylov_dim(FLAGS_gmres_restart)
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
         {"gmres_mixed_half",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::GmresMixed<double, gko::half>::build()
+             return gko::solver::GmresMixed<double>::build()
+                 .with_storage_precision(
+                     gko::solver::gmres_mixed_storage_precision::reduce2)
                  .with_criteria(create_criterion(exec))
-                 .with_krylov_dim_mixed(FLAGS_gmres_restart)
+                 .with_krylov_dim(FLAGS_gmres_restart)
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
         {"gmres_mixed_int64",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::GmresMixed<double, gko::int64>::build()
+             return gko::solver::GmresMixed<double>::build()
+                 .with_storage_precision(
+                     gko::solver::gmres_mixed_storage_precision::integer)
                  .with_criteria(create_criterion(exec))
-                 .with_krylov_dim_mixed(FLAGS_gmres_restart)
+                 .with_krylov_dim(FLAGS_gmres_restart)
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
         {"gmres_mixed_int32",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::GmresMixed<double, gko::int32>::build()
+             return gko::solver::GmresMixed<double>::build()
+                 .with_storage_precision(
+                     gko::solver::gmres_mixed_storage_precision::ireduce1)
                  .with_criteria(create_criterion(exec))
-                 .with_krylov_dim_mixed(FLAGS_gmres_restart)
+                 .with_krylov_dim(FLAGS_gmres_restart)
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
         {"gmres_mixed_int16",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::GmresMixed<double, gko::int16>::build()
+             return gko::solver::GmresMixed<double>::build()
+                 .with_storage_precision(
+                     gko::solver::gmres_mixed_storage_precision::ireduce2)
                  .with_criteria(create_criterion(exec))
-                 .with_krylov_dim_mixed(FLAGS_gmres_restart)
+                 .with_krylov_dim(FLAGS_gmres_restart)
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
diff --git a/reference/test/solver/gmres_mixed_kernels.cpp b/reference/test/solver/gmres_mixed_kernels.cpp
index aa6b2ec19cd..d190df910b0 100644
--- a/reference/test/solver/gmres_mixed_kernels.cpp
+++ b/reference/test/solver/gmres_mixed_kernels.cpp
@@ -80,7 +80,7 @@ class GmresMixed : public ::testing::Test {
         static constexpr nc_value_type value =
             std::is_same<rc_type, double>::value
                 ? 1e-14
-                : std::is_same<rc_type, float>::value ? 1e-7 : 1e-3;
+                : std::is_same<rc_type, float>::value ? 1e-6 : 1e-3;
     };
 
     GmresMixed()
@@ -303,7 +303,7 @@ TYPED_TEST(GmresMixed, SolvesBigDenseSystem1)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({52.7, 85.4, 134.2, -250.0, -16.8, 35.3}),
-                        this->precision() * 1e1);
+                        this->precision());
 }
 
 
@@ -320,7 +320,7 @@ TYPED_TEST(GmresMixed, SolvesBigDenseSystem2)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
-                        this->precision() * 1e1);
+                        this->precision());
 }
 
 
@@ -463,7 +463,7 @@ TYPED_TEST(GmresMixed, SolvesWithPreconditioner)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
-                        this->precision() * 1e1);
+                        this->precision());
 }
 
 

From f39e59969732e891397ce842f28f09a131909f0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 11 Sep 2020 18:00:36 +0200
Subject: [PATCH 0388/1008] Use new reduced_row_major Accessor in GmresMixed

Of course also use scaled_reduced_row_major.
Both are from the newly-added Accessors from the private headers.
---
 common/solver/gmres_mixed_kernels.hpp.inc | 100 ++--
 core/solver/gmres_mixed.cpp               |  24 +-
 core/solver/gmres_mixed_accessor.hpp      | 613 ++--------------------
 core/solver/gmres_mixed_kernels.hpp       | 172 +++---
 cuda/solver/gmres_mixed_kernels.cu        |  61 +--
 cuda/test/solver/gmres_mixed_kernels.cpp  |  59 +--
 omp/solver/gmres_mixed_kernels.cpp        |   8 +-
 omp/test/solver/gmres_mixed_kernels.cpp   |  59 +--
 reference/solver/gmres_mixed_kernels.cpp  |  22 +-
 9 files changed, 319 insertions(+), 799 deletions(-)

diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/gmres_mixed_kernels.hpp.inc
index 5f2259ece11..e4da9eb65b7 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/gmres_mixed_kernels.hpp.inc
@@ -88,18 +88,20 @@ __global__ __launch_bounds__(block_size) void initialize_2_1_kernel(
     size_type stride_residual_nc)
 {
     const auto global_id = thread::get_thread_id_flat();
+    const auto krylov_stride =
+        helper_functions_accessor<Accessor3d>::get_stride(krylov_bases);
     // krylov indices
-    const auto krylov_idx = global_id / krylov_bases.get_stride0();
-    const auto reminder = global_id % krylov_bases.get_stride0();
-    const auto krylov_row = reminder / krylov_bases.get_stride1();
-    const auto rhs = reminder % krylov_bases.get_stride1();
+    const auto krylov_idx = global_id / krylov_stride[0];
+    const auto reminder = global_id % krylov_stride[0];
+    const auto krylov_row = reminder / krylov_stride[1];
+    const auto rhs = reminder % krylov_stride[1];
 
     // residual_norm indices (separated for better coalesced access)
     const auto residual_row = global_id / stride_residual_nc;
     const auto residual_col = global_id % stride_residual_nc;
 
     if (krylov_idx < krylov_dim + 1 && krylov_row < num_rows && rhs < num_rhs) {
-        krylov_bases.at(krylov_idx, krylov_row, rhs) = zero<ValueType>();
+        krylov_bases(krylov_idx, krylov_row, rhs) = zero<ValueType>();
     }
 
     if (residual_row < krylov_dim + 1 && residual_col < num_rhs) {
@@ -120,8 +122,10 @@ __global__ __launch_bounds__(block_size) void initialize_2_2_kernel(
     size_type *__restrict__ final_iter_nums)
 {
     const auto global_id = thread::get_thread_id_flat();
-    const auto row_idx = global_id / krylov_bases.get_stride1();
-    const auto col_idx = global_id % krylov_bases.get_stride1();
+    const auto krylov_stride =
+        helper_functions_accessor<Accessor3d>::get_stride(krylov_bases);
+    const auto row_idx = global_id / krylov_stride[1];
+    const auto col_idx = global_id % krylov_stride[1];
 
     if (global_id < num_rhs) {
         residual_norm_collection[global_id] = residual_norm[global_id];
@@ -131,7 +135,7 @@ __global__ __launch_bounds__(block_size) void initialize_2_2_kernel(
     if (row_idx < num_rows && col_idx < num_rhs) {
         auto value = residual[row_idx * stride_residual + col_idx] /
                      residual_norm[col_idx];
-        krylov_bases.at({0}, row_idx, col_idx) = value;
+        krylov_bases(0, row_idx, col_idx) = value;
         next_krylov_basis[row_idx * stride_next_krylov + col_idx] = value;
     }
 }
@@ -399,7 +403,7 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel(
              i += default_dot_dim) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
             local_res += next_krylov_basis[next_krylov_idx] *
-                         krylov_bases.at(k, i, col_idx);
+                         krylov_bases(k, i, col_idx);
         }
     }
     reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
@@ -448,7 +452,7 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
                  i += default_dot_dim) {
                 const auto next_krylov_idx = i * stride_next_krylov + col_idx;
                 local_res += next_krylov_basis[next_krylov_idx] *
-                             krylov_bases.at(k, i, col_idx);
+                             krylov_bases(k, i, col_idx);
             }
         }
         reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
@@ -501,7 +505,7 @@ __global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_1(
              i += default_dot_dim) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
             local_res += next_krylov_basis[next_krylov_idx] *
-                         krylov_bases.at(k, i, col_idx);
+                         krylov_bases(k, i, col_idx);
         }
     }
     reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
@@ -580,7 +584,7 @@ __global__ __launch_bounds__(dot_dim *dot_dim) void multidot_kernel_num_iters_2(
         for (size_type i = start_row; i < end_row; i += blockDim.y) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
             local_res += next_krylov_basis[next_krylov_idx] *
-                         krylov_bases.at(k, i, col_idx);
+                         krylov_bases(k, i, col_idx);
         }
     }
     // Transpose local_res, so each warp contains a local_res from the same
@@ -635,7 +639,7 @@ __global__ __launch_bounds__(block_size) void singledot_kernel_num_iters_2(
         for (size_type i = start_row; i < end_row; i += block_size) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
             local_res += next_krylov_basis[next_krylov_idx] *
-                         krylov_bases.at(k, i, col_idx);
+                         krylov_bases(k, i, col_idx);
         }
     }
     // Transpose local_res, so each warp contains a local_res from the same
@@ -672,8 +676,7 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel(
         const auto hessenberg_idx = k * stride_hessenberg + col_idx;
 
         next_krylov_basis[next_krylov_idx] -=
-            hessenberg_iter[hessenberg_idx] *
-            krylov_bases.at(k, row_idx, col_idx);
+            hessenberg_iter[hessenberg_idx] * krylov_bases(k, row_idx, col_idx);
     }
 }
 
@@ -704,14 +707,14 @@ __global__
 
             next_krylov_basis[next_krylov_idx] -=
                 hessenberg_iter[hessenberg_idx] *
-                krylov_bases.at(k, row_idx, col_idx);
+                krylov_bases(k, row_idx, col_idx);
         }
     }
 }
 
 
-// Must be called with at least `num_vectors * krylov_bases.get_stride0()`
-// threads in total.
+// Must be called with at least `num_vectors *
+// krylov_bases.get_accessor().get_stride()[0]` threads in total.
 template <int block_size, typename ValueType, typename Accessor3d>
 __global__
     __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_0(
@@ -725,18 +728,20 @@ __global__
     // CAREFUL: krylov_bases changes might not be done properly since this
     //          kernel is currently not used!
     const auto global_id = thread::get_thread_id_flat();
-    const auto k = global_id / krylov_bases.get_stride0();
-    const auto reminder = global_id / krylov_bases.get_stride0();
-    const auto row_idx = reminder / krylov_bases.get_stride1();
-    const auto col_idx = reminder % krylov_bases.get_stride1();
+    const auto krylov_stride =
+        helper_functions_accessor<Accessor3d>::get_stride(krylov_bases);
+    const auto k = global_id / krylov_stride[0];
+    const auto reminder = global_id / krylov_stride[0];
+    const auto row_idx = reminder / krylov_stride[1];
+    const auto col_idx = reminder % krylov_stride[1];
 
     if (row_idx < num_rows && col_idx < num_cols &&
         !stop_status[col_idx].has_stopped()) {
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
         const auto hessenberg_idx = k * stride_hessenberg + col_idx;
 
-        auto value = hessenberg_iter[hessenberg_idx] *
-                     krylov_bases.at(k, row_idx, col_idx);
+        auto value =
+            hessenberg_iter[hessenberg_idx] * krylov_bases(k, row_idx, col_idx);
         atomic_add(next_krylov_basis + next_krylov_idx, -value);
     }
 }
@@ -771,7 +776,7 @@ __global__
                 const auto hessenberg_idx = i * stride_hessenberg + col_idx;
 
                 value += hessenberg_iter[hessenberg_idx] *
-                         krylov_bases.at(i, row_idx, col_idx);
+                         krylov_bases(i, row_idx, col_idx);
             }
             reduction_helper[tidx] = value;
 
@@ -799,7 +804,7 @@ __global__
 
                 next_krylov_basis[next_krylov_idx] -=
                     hessenberg_iter[hessenberg_idx] *
-                    krylov_bases.at(k, row_idx, col_idx);
+                    krylov_bases(k, row_idx, col_idx);
             }
         }
     }
@@ -838,7 +843,7 @@ __global__
                     const auto hessenberg_idx = i * stride_hessenberg + col_idx;
 
                     value += hessenberg_iter[hessenberg_idx] *
-                             krylov_bases.at(i, j, col_idx);
+                             krylov_bases(i, j, col_idx);
                 }
                 reduction_helper[k * block_size + tidx] = value;
             }
@@ -880,7 +885,7 @@ __global__
 
                 next_krylov_basis[next_krylov_idx] -=
                     hessenberg_iter[hessenberg_idx] *
-                    krylov_bases.at(k, row_idx, col_idx);
+                    krylov_bases(k, row_idx, col_idx);
             }
         }
     }
@@ -923,7 +928,7 @@ __global__
                 for (size_type j = row_idx, k = 0;
                      j < num_rows && k < num_blocks; j++, k++) {
                     reduction_helper[k * block_size + tidx] +=
-                        hessenberg_val * krylov_bases.at(i, j, col_idx);
+                        hessenberg_val * krylov_bases(i, j, col_idx);
                 }
             }
 
@@ -966,7 +971,7 @@ __global__
 
                 next_krylov_basis[next_krylov_idx] -=
                     hessenberg_iter[hessenberg_idx] *
-                    krylov_bases.at(k, row_idx, col_idx);
+                    krylov_bases(k, row_idx, col_idx);
             }
         }
     }
@@ -1009,7 +1014,7 @@ __global__
                 for (size_type j = row_idx, k = 0;
                      j < num_rows && k < num_blocks; j++, k++) {
                     reduction_helper[k * block_size + tidx] +=
-                        hessenberg_val * krylov_bases.at(i, j, col_idx);
+                        hessenberg_val * krylov_bases(i, j, col_idx);
                 }
             }
             multireduce(
@@ -1043,7 +1048,7 @@ __global__
 
                 next_krylov_basis[next_krylov_idx] -=
                     hessenberg_iter[hessenberg_idx] *
-                    krylov_bases.at(k, row_idx, col_idx);
+                    krylov_bases(k, row_idx, col_idx);
             }
         }
     }
@@ -1077,8 +1082,7 @@ __global__
         const auto reorth_idx = col_idx;
 
         next_krylov_basis[next_krylov_idx] -=
-            hessenberg_iter[hessenberg_idx] *
-            krylov_bases.at(k, row_idx, col_idx);
+            hessenberg_iter[hessenberg_idx] * krylov_bases(k, row_idx, col_idx);
         if (row_idx == 0) {
             if (squared_norm(hessenberg_iter[hessenberg_idx]) >
                 (0.99 * sqrt(arnoldi_norm[arnoldi_idx]))) {
@@ -1114,7 +1118,7 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
         const auto hessenberg_idx = k * stride_hessenberg + col_idx;
         const auto buffer_idx = k * stride_buffer + col_idx;
         next_krylov_basis[next_krylov_idx] -=
-            buffer_iter[buffer_idx] * krylov_bases.at(k, row_idx, col_idx);
+            buffer_iter[buffer_idx] * krylov_bases(k, row_idx, col_idx);
         if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
             hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
         }
@@ -1149,7 +1153,7 @@ __global__
             const auto hessenberg_idx = k * stride_hessenberg + col_idx;
             const auto buffer_idx = k * stride_buffer + col_idx;
             next_krylov_basis[next_krylov_idx] -=
-                buffer_iter[buffer_idx] * krylov_bases.at(k, row_idx, col_idx);
+                buffer_iter[buffer_idx] * krylov_bases(k, row_idx, col_idx);
             if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
                 hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
             }
@@ -1187,8 +1191,8 @@ __global__
                 const auto hessenberg_idx = i * stride_hessenberg + col_idx;
                 const auto buffer_idx = i * stride_buffer + col_idx;
 
-                value += buffer_iter[buffer_idx] *
-                         krylov_bases.at(i, row_idx, col_idx);
+                value +=
+                    buffer_iter[buffer_idx] * krylov_bases(i, row_idx, col_idx);
                 if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
                     hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
                 }
@@ -1221,8 +1225,7 @@ __global__
                 const auto hessenberg_idx = k * stride_hessenberg + col_idx;
                 const auto buffer_idx = k * stride_buffer + col_idx;
                 next_krylov_basis[next_krylov_idx] -=
-                    buffer_iter[buffer_idx] *
-                    krylov_bases.at(k, row_idx, col_idx);
+                    buffer_iter[buffer_idx] * krylov_bases(k, row_idx, col_idx);
                 if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
                     hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
                 }
@@ -1300,7 +1303,7 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
         // printf("(%d) num1 = %10.5e , num2 = %10.5e\n", cod_idx, num1, num2);
         // printf("%d - %d - %d - %d\n", stride_norm, iter, col_idx,
         //        stride_norm * iter + col_idx);
-        helper_functions_accessor<Accessor3d>::write_scale(
+        helper_functions_accessor<Accessor3d>::write_scalar(
             krylov_bases, iter, col_idx, num2 / num12);
         // krylov_bases, num_cols * (iter + 1) + col_idx, num2 / num12);
         // krylov_bases, col_idx, num2 / num12);
@@ -1320,19 +1323,21 @@ __global__ __launch_bounds__(block_size) void set_scale_kernel(
     static_assert(!is_complex_s<RealValueType>::value,
                   "ValueType must not be complex!");
     const auto global_id = thread::get_thread_id_flat();
-    const auto blk_idx = global_id / krylov_bases.get_stride1();
-    const auto col_idx = global_id % krylov_bases.get_stride1();
+    const auto krylov_stride =
+        helper_functions_accessor<Accessor3d>::get_stride(krylov_bases);
+    const auto blk_idx = global_id / krylov_stride[1];
+    const auto col_idx = global_id % krylov_stride[1];
 
     if (blk_idx < num_blocks && col_idx < num_rhs) {
         if (blk_idx == 0) {
             const auto num1 = residual_norm[col_idx];
             const auto num2 = arnoldi_inf[col_idx];
             // printf("num1 = %10.5e , num2 = %10.5e\n", num1, num2);
-            helper_functions_accessor<Accessor3d>::write_scale(
+            helper_functions_accessor<Accessor3d>::write_scalar(
                 krylov_bases, {0}, col_idx, num2 / num1);
         } else {
             const auto num = one<RealValueType>();
-            helper_functions_accessor<Accessor3d>::write_scale(
+            helper_functions_accessor<Accessor3d>::write_scalar(
                 krylov_bases, blk_idx, col_idx, num);
         }
     }
@@ -1405,7 +1410,7 @@ __global__ __launch_bounds__(block_size) void update_krylov_next_krylov_kernel(
             next_krylov_basis[next_krylov_idx] / hessenberg;
 
         next_krylov_basis[next_krylov_idx] = next_krylov_value;
-        krylov_bases.at(iter + 1, row_idx, col_idx) = next_krylov_value;
+        krylov_bases(iter + 1, row_idx, col_idx) = next_krylov_value;
     }
 }
 
@@ -1581,8 +1586,7 @@ __global__ __launch_bounds__(block_size) void calculate_Qy_kernel(
     if (row_id < num_rows && col_id < num_cols) {
         ValueType temp = zero<ValueType>();
         for (size_type j = 0; j < final_iter_nums[col_id]; ++j) {
-            temp +=
-                krylov_bases.at(j, row_id, col_id) * y[j * stride_y + col_id];
+            temp += krylov_bases(j, row_id, col_id) * y[j * stride_y + col_id];
         }
         before_preconditioner[global_id] = temp;
     }
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 5ad3fbbc341..e95a4add464 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -241,6 +241,12 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         auto dense_b = as<const Vector>(b);
         auto dense_x = as<Vector>(x);
         auto residual = Vector::create_with_config_of(dense_b);
+        /* The dimensions {x, y, z} explained for the krylov_bases:
+         * - x: selects the krylov vector (which has krylov_dim + 1 vectors)
+         * - y: selects the (row-)element of said krylov vector
+         * - z: selects which column-element of said krylov vector should be
+         *      used
+         */
         const dim<3> krylov_bases_dim{krylov_dim_ + 1,
                                       system_matrix_->get_size()[1],
                                       dense_b->get_size()[1]};
@@ -250,10 +256,10 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         // const size_type krylov_bases_stride = krylov_bases_dim[1];
         // LowArray krylov_bases(exec, krylov_bases_dim[0] *
         // krylov_bases_stride); KrylovAccessor
-        // krylov_bases_accessor(krylov_bases.get_data(),
+        // krylov_bases_range(krylov_bases.get_data(),
         //                                     krylov_bases_stride);
         Accessor3dHelper helper(exec, krylov_bases_dim);
-        auto krylov_bases_accessor = helper.get_accessor();
+        auto krylov_bases_range = helper.get_range();
 
         auto next_krylov_basis = Vector::create_with_config_of(dense_b);
         std::shared_ptr<matrix::Dense<ValueType>> preconditioned_vector =
@@ -307,7 +313,7 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         // std::cout << "Before initializate_2" << std::endl;
         exec->run(gmres_mixed::make_initialize_2(
             residual.get(), residual_norm.get(), residual_norm_collection.get(),
-            arnoldi_norm.get(), krylov_bases_accessor, next_krylov_basis.get(),
+            arnoldi_norm.get(), krylov_bases_range, next_krylov_basis.get(),
             &final_iter_nums, krylov_dim_));
         // residual_norm = norm(residual)
         // residual_norm_collection = {residual_norm, 0, ..., 0}
@@ -431,8 +437,9 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
                 exec->run(gmres_mixed::make_step_2(
                     residual_norm_collection.get(),
-                    krylov_bases_accessor.to_const(), hessenberg_view.get(),
-                    y.get(), before_preconditioner.get(), &final_iter_nums));
+                    krylov_bases_range.get_accessor().to_const(),
+                    hessenberg_view.get(), y.get(), before_preconditioner.get(),
+                    &final_iter_nums));
                 // Solve upper triangular.
                 // y = hessenberg \ residual_norm_collection
 
@@ -449,7 +456,7 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                 exec->run(gmres_mixed::make_initialize_2(
                     residual.get(), residual_norm.get(),
                     residual_norm_collection.get(), arnoldi_norm.get(),
-                    krylov_bases_accessor, next_krylov_basis.get(),
+                    krylov_bases_range, next_krylov_basis.get(),
                     &final_iter_nums, krylov_dim_));
                 // residual_norm = norm(residual)
                 // residual_norm_collection = {residual_norm, 0, ..., 0}
@@ -496,7 +503,7 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             exec->run(gmres_mixed::make_step_1(
                 next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
                 residual_norm.get(), residual_norm_collection.get(),
-                krylov_bases_accessor, hessenberg_iter.get(), buffer_iter.get(),
+                krylov_bases_range, hessenberg_iter.get(), buffer_iter.get(),
                 b_norm.get(), arnoldi_norm.get(), restart_iter,
                 &final_iter_nums, &stop_status, &reorth_status, &num_reorth,
                 &num_reorth_steps, &num_reorth_vectors));
@@ -544,7 +551,8 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             span{0, dense_b->get_size()[1] * (restart_iter)});
 
         exec->run(gmres_mixed::make_step_2(
-            residual_norm_collection.get(), krylov_bases_accessor.to_const(),
+            residual_norm_collection.get(),
+            krylov_bases_range.get_accessor().to_const(),
             hessenberg_small.get(), y.get(), before_preconditioner.get(),
             &final_iter_nums));
         // Solve upper triangular.
diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/gmres_mixed_accessor.hpp
index ab380851b12..e34bd299c1d 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/gmres_mixed_accessor.hpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_SOLVER_GMRES_MIXED_ACCESSOR_HPP_
 
 
+#include <array>
 #include <cinttypes>
 #include <limits>
 #include <memory>
@@ -44,10 +45,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/dim.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/types.hpp>
 
 
+#include "core/base/accessors.hpp"
+
+
 #include <iostream>
 #define GKO_DEBUG_OUTPUT false
 
@@ -59,547 +64,12 @@ namespace kernels {  // TODO maybe put into another separate namespace
 namespace detail {
 
 
-// using place_holder_type = float;
-
-
-template <typename StorageType, typename ArithmeticType,
-          bool = std::is_integral<StorageType>::value>
-/*          bool = (std::is_same<StorageType, place_holder_type>::value &&
-                  !std::is_same<StorageType, ArithmeticType>::value) ||
-                 std::is_integral<StorageType>::value>
-*/
-struct helper_have_scale {};
-
-template <typename StorageType, typename ArithmeticType>
-struct helper_have_scale<StorageType, ArithmeticType, false>
-    : public std::false_type {};
-
-template <typename StorageType, typename ArithmeticType>
-struct helper_have_scale<StorageType, ArithmeticType, true>
-    : public std::true_type {};
-
-/**
- * This is a mixin which defines the binary operators for *, /, +, - for the
- * Reference class, the unary operator -, and the assignment operators
- * *=, /=, +=, -=
- * Additionally, it prevents the default generation of copy and move constructor
- * and copy and move assignment.
- */
-template <typename Reference, typename ArithmeticType>
-struct enable_reference {
-    using arithmetic_type = ArithmeticType;
-
-    GKO_ATTRIBUTES enable_reference() =
-        default;  // Needs to exist, so creation is not probibited
-    enable_reference(enable_reference &&) = delete;
-    enable_reference(const enable_reference &) = delete;
-    enable_reference &operator=(const enable_reference &) = delete;
-    enable_reference &operator=(enable_reference &&) = delete;
-
-#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_oper, _op)            \
-    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper( \
-        const Reference &ref1, const Reference &ref2)                 \
-    {                                                                 \
-        return static_cast<arithmetic_type>(ref1)                     \
-            _op static_cast<arithmetic_type>(ref2);                   \
-    }                                                                 \
-    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper( \
-        const Reference &ref, const arithmetic_type &a)               \
-    {                                                                 \
-        return static_cast<arithmetic_type>(ref) _op a;               \
-    }                                                                 \
-    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper( \
-        const arithmetic_type &a, const Reference &ref)               \
-    {                                                                 \
-        return a _op static_cast<arithmetic_type>(ref);               \
-    }
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator*, *)
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator/, /)
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator+, +)
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(operator-, -)
-#undef GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD
-
-#define GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(_oper, _op)           \
-    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper(    \
-        Reference &&ref1, const Reference &ref2)                         \
-    {                                                                    \
-        return std::move(ref1) = static_cast<arithmetic_type>(ref1)      \
-                   _op static_cast<arithmetic_type>(ref2);               \
-    }                                                                    \
-    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type _oper(    \
-        Reference &&ref, const arithmetic_type &a)                       \
-    {                                                                    \
-        return std::move(ref) = static_cast<arithmetic_type>(ref) _op a; \
-    }
-
-    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator*=, *)
-    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator/=, /)
-    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator+=, +)
-    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator-=, -)
-#undef GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD
-
-    // TODO test if comparison operators need to be overloaded as well
-
-    friend GKO_ATTRIBUTES constexpr GKO_INLINE arithmetic_type
-    operator-(const Reference &ref)
-    {
-        return -static_cast<arithmetic_type>(ref);
-    }
-};
-
-
-namespace reference {
-
-
-/**
- * Reference class for a different storage than arithmetic type. The conversion
- * between both formats is done with a simple static_cast.
- */
-template <typename ArithmeticType, typename StorageType>
-class ReducedStorageReference
-    : public detail::enable_reference<
-          ReducedStorageReference<ArithmeticType, StorageType>,
-          ArithmeticType> {
-public:
-    using arithmetic_type = ArithmeticType;
-    using storage_type = StorageType;
-    ReducedStorageReference() = delete;
-    GKO_ATTRIBUTES ReducedStorageReference(storage_type *const GKO_RESTRICT ptr)
-        : ptr_{ptr}
-    {}
-    GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
-    {
-        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return static_cast<arithmetic_type>(*r_ptr);
-    }
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator=(arithmetic_type val) &&
-    {
-        storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return *r_ptr = static_cast<storage_type>(val);
-    }
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type
-    operator=(const ReducedStorageReference &ref) &&
-    {
-        return std::move(*this) = static_cast<arithmetic_type>(ref);
-    }
-
-private:
-    storage_type *const GKO_RESTRICT ptr_;
-};
-
-/**
- * Reference class for a different storage than arithmetic type. The conversion
- * between both formats is done with a simple static_cast followed by a
- * multiplication with a scale.
- */
-template <typename ArithmeticType, typename StorageType>
-class ScaledReducedStorageReference
-    : public detail::enable_reference<
-          ScaledReducedStorageReference<ArithmeticType, StorageType>,
-          ArithmeticType> {
-public:
-    using arithmetic_type = ArithmeticType;
-    using storage_type = StorageType;
-
-    ScaledReducedStorageReference() = delete;
-    GKO_ATTRIBUTES ScaledReducedStorageReference(
-        storage_type *const GKO_RESTRICT ptr, arithmetic_type scale)
-        : ptr_{ptr}, scale_{scale}
-    {}
-    GKO_ATTRIBUTES GKO_INLINE operator arithmetic_type() const
-    {
-        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return static_cast<arithmetic_type>(*r_ptr) * scale_;
-    }
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator=(arithmetic_type val) &&
-    {
-        storage_type *const GKO_RESTRICT r_ptr = ptr_;
-        return *r_ptr = static_cast<storage_type>(val / scale_), val;
-    }
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type
-    operator=(const ScaledReducedStorageReference &ref) &&
-    {
-        return std::move(*this) = static_cast<arithmetic_type>(ref);
-    }
-
-private:
-    storage_type *const GKO_RESTRICT ptr_;
-    const arithmetic_type scale_;
-};
-
-
-}  // namespace reference
-}  // namespace detail
-
-
-/**
- * @internal
- *
- * The ReducedStorage3d class hides the underlying storage_ format and provides
- * a simple interface for accessing a one dimensional storage.
- *
- * This class only manages the accesses and not the memory itself.
- *
- * TODO move this documentation to the core since this is a general purpose
- * accessor
- *
- * The dimensions {x, y, z} explained for the krylov_bases:
- * - x: selects the krylov vector (usually has krylov_dim + 1 vectors)
- * - y: selects the (row-)element of said krylov vector
- * - z: selects which column-element of said krylov vector should be used
- *
- * The accessor uses row-major access.
- */
-template <typename ArithmeticType, typename StorageType>
-class ConstReducedStorage3d {
-public:
-    using arithmetic_type = ArithmeticType;
-    using storage_type = StorageType;
-    using const_accessor = ConstReducedStorage3d;
-    static_assert(!std::is_const<storage_type>::value,
-                  "StorageType must not be const!");
-
-protected:
-    using reference =
-        detail::reference::ReducedStorageReference<ArithmeticType, StorageType>;
-
-public:
-    /**
-     * Creates the accessor with an already allocated storage space with a
-     * stride.
-     *
-     * @internal
-     * const_cast needed, so ReducedStorage3d(ac.get_storage(), ...)
-     * works. Also, storage_ is never accessed in a non-const fashion in this
-     * class, so it is not invalid or UB code.
-     */
-    GKO_ATTRIBUTES ConstReducedStorage3d(const storage_type *storage,
-                                         size_type stride0, size_type stride1)
-        : storage_{const_cast<storage_type *>(storage)},
-          stride_{stride0, stride1}
-    {}
-
-    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const { return *this; }
-
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator()(IndexType x,
-                                                         IndexType y,
-                                                         IndexType z) const
-    {
-        return reference{storage_ + compute_index(x, y, z)};
-    }
-
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type at(IndexType x, IndexType y,
-                                                 IndexType z) const
-    {
-        return reference{storage_ + compute_index(x, y, z)};
-    }
-
-    GKO_ATTRIBUTES GKO_INLINE size_type get_stride0() const
-    {
-        return stride_[0];
-    }
-    GKO_ATTRIBUTES GKO_INLINE size_type get_stride1() const
-    {
-        return stride_[1];
-    }
-
-    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_storage() const
-    {
-        return storage_;
-    }
-
-    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_const_storage() const
-    {
-        return storage_;
-    }
-
-protected:
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE constexpr size_type compute_index(
-        IndexType x, IndexType y, IndexType z) const
-    {
-        return x * stride_[0] + y * stride_[1] + z;
-    }
-
-    storage_type *storage_;
-    size_type stride_[2];
-};
-
-template <typename ArithmeticType, typename StorageType>
-class ReducedStorage3d
-    : public ConstReducedStorage3d<ArithmeticType, StorageType> {
-public:
-    using arithmetic_type = ArithmeticType;
-    using storage_type = StorageType;
-    using const_accessor = ConstReducedStorage3d<arithmetic_type, storage_type>;
-
-protected:
-    using reference = typename const_accessor::reference;
-
-public:
-    GKO_ATTRIBUTES ReducedStorage3d() : const_accessor(nullptr, 0, 0) {}
-
-    GKO_ATTRIBUTES ReducedStorage3d(storage_type *storage, size_type stride0,
-                                    size_type stride1)
-        : const_accessor(storage, stride0, stride1)
-    {}
-
-    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const
-    {
-        return {this->storage_, this->stride_[0], this->stride_[1]};
-    }
-
-    using const_accessor::get_storage;
-    GKO_ATTRIBUTES GKO_INLINE storage_type *get_storage()
-    {
-        return this->storage_;
-    }
-
-    // Makes sure the operator() const function is visible
-    using const_accessor::operator();
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE reference operator()(IndexType x, IndexType y,
-                                                   IndexType z)
-    {
-        return {this->storage_ + const_accessor::compute_index(x, y, z)};
-    }
-
-    using const_accessor::at;  // Makes sure the at() const function is visible
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE reference at(IndexType x, IndexType y,
-                                           IndexType z)
-    {
-        return {this->storage_ + const_accessor::compute_index(x, y, z)};
-    }
-};
-
-
-/**
- * @internal
- *
- * The ScaledReducedStorage3d class hides the underlying storage_ format and
- * provides a simple interface for accessing a one dimensional storage.
- * Additionally, this accessor posesses a scale array, which is used for each
- * read and write operation to do a proper conversion.
- *
- * This class only manages the accesses, however, and not the memory itself.
- *
- * TODO move this documentation to the core since this is a general purpose
- * accessor
- *
- * The dimensions {x, y, z} explained for the krylov_bases:
- * - x: selects the krylov vector (usually has krylov_dim + 1 vectors)
- * - y: selects the (row-)element of said krylov vector
- * - z: selects which column-element of said krylov vector should be used
- *
- * The accessor uses row-major access.
- */
-template <typename ArithmeticType, typename StorageType>
-class ConstScaledReducedStorage3d {
-public:
-    using arithmetic_type = ArithmeticType;
-    using storage_type = StorageType;
-    using const_accessor = ConstScaledReducedStorage3d;
-    static_assert(!std::is_const<storage_type>::value,
-                  "StorageType must not be const!");
-
-protected:
-    using reference =
-        detail::reference::ScaledReducedStorageReference<ArithmeticType,
-                                                         StorageType>;
-
-public:
-    /**
-     * Creates the accessor with an already allocated storage space with a
-     * stride.
-     *
-     * @internal
-     * const_cast needed, so ScaledReducedStorage3d(ac.get_storage(), ...)
-     * works. Also, storage_ is never accessed in a non-const fashion (from
-     * ConstScaledReducedStorage3d), so it is not invalid or UB code.
-     */
-    GKO_ATTRIBUTES ConstScaledReducedStorage3d(const storage_type *storage,
-                                               size_type stride0,
-                                               size_type stride1,
-                                               const arithmetic_type *scale)
-        : storage_{const_cast<storage_type *>(storage)},
-          stride_{stride0, stride1},
-          scale_{const_cast<arithmetic_type *>(scale)}
-    {}
-
-    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const { return *this; }
-
-    /**
-     * Reads the scale value at the given indices.
-     */
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type read_scale(IndexType x,
-                                                         IndexType z) const
-    {
-        const arithmetic_type *GKO_RESTRICT rest_scale = scale_;
-        return rest_scale[x * stride_[1] + z];
-    }
-
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type operator()(IndexType x,
-                                                         IndexType y,
-                                                         IndexType z) const
-    {
-        return reference{storage_ + compute_index(x, y, z), read_scale(x, z)};
-    }
-
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type at(IndexType x, IndexType y,
-                                                 IndexType z) const
-    {
-        return reference{storage_ + compute_index(x, y, z), read_scale(x, z)};
-    }
-
-    GKO_ATTRIBUTES GKO_INLINE size_type get_stride0() const
-    {
-        return stride_[0];
-    }
-    GKO_ATTRIBUTES GKO_INLINE size_type get_stride1() const
-    {
-        return stride_[1];
-    }
-
-    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_storage() const
-    {
-        return storage_;
-    }
-
-    GKO_ATTRIBUTES GKO_INLINE const storage_type *get_const_storage() const
-    {
-        return storage_;
-    }
-
-    GKO_ATTRIBUTES GKO_INLINE const arithmetic_type *get_scale() const
-    {
-        return this->scale_;
-    }
-
-    GKO_ATTRIBUTES GKO_INLINE const arithmetic_type *get_const_scale() const
-    {
-        return this->scale_;
-    }
-
-protected:
-    template <typename IndexType>
-    GKO_ATTRIBUTES constexpr GKO_INLINE size_type
-    compute_index(IndexType x, IndexType y, IndexType z) const
-    {
-        return x * stride_[0] + y * stride_[1] + z;
-    }
-
-    storage_type *storage_;
-    size_type stride_[2];
-    arithmetic_type *scale_;
-};
-
-
-template <typename ArithmeticType, typename StorageType>
-class ScaledReducedStorage3d
-    : public ConstScaledReducedStorage3d<ArithmeticType, StorageType> {
-public:
-    using arithmetic_type = ArithmeticType;
-    using storage_type = StorageType;
-    using const_accessor =
-        ConstScaledReducedStorage3d<arithmetic_type, storage_type>;
-
-protected:
-    using reference = typename const_accessor::reference;
-
-public:
-    GKO_ATTRIBUTES ScaledReducedStorage3d()
-        : const_accessor(nullptr, {}, {}, nullptr)
-    {}
-
-    GKO_ATTRIBUTES ScaledReducedStorage3d(storage_type *storage,
-                                          size_type stride0, size_type stride1,
-                                          arithmetic_type *scale)
-        : const_accessor(storage, stride0, stride1, scale)
-    {}
-
-    GKO_ATTRIBUTES GKO_INLINE const_accessor to_const() const
-    {
-        return {this->storage_, this->stride_[0], this->stride_[1],
-                this->scale_};
-    }
-
-    // Makes sure the operator() const function is visible
-    using const_accessor::operator();
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE reference operator()(IndexType x, IndexType y,
-                                                   IndexType z)
-    {
-        return {this->storage_ + const_accessor::compute_index(x, y, z),
-                this->read_scale(x, z)};
-    }
-
-
-    using const_accessor::at;
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE reference at(IndexType x, IndexType y,
-                                           IndexType z)
-    {
-        return {this->storage_ + const_accessor::compute_index(x, y, z),
-                this->read_scale(x, z)};
-    }
-
-    /**
-     * Writes the given value at the given index.
-     */
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE void write(IndexType x, IndexType y, IndexType z,
-                                         arithmetic_type value)
-    {
-        storage_type *GKO_RESTRICT rest_storage = this->storage_;
-        rest_storage[const_accessor::compute_index(x, y, z)] =
-            static_cast<storage_type>(value / this->read_scale(x, z));
-#if GKO_DEBUG_OUTPUT && not defined(__CUDA_ARCH__)
-        std::cout << "storage[" << x << ", " << y << ", " << z << "] = "
-                  << rest_storage[const_accessor::compute_index(x, y, z)]
-                  << "; value = " << value
-                  << "; scale = " << this->read_scale(x, z) << '\n';
-#endif
-    }
-
-    /**
-     * Writes the given value at the given index for a scale.
-     */
-    template <typename IndexType>
-    GKO_ATTRIBUTES GKO_INLINE void set_scale(IndexType x, IndexType z,
-                                             arithmetic_type val)
-    {
-        arithmetic_type *GKO_RESTRICT rest_scale = this->scale_;
-        rest_scale[x * this->stride_[1] + z] = val;
-    }
-
-    using const_accessor::get_storage;
-    GKO_ATTRIBUTES GKO_INLINE storage_type *get_storage()
-    {
-        return this->storage_;
-    }
-
-    using const_accessor::get_scale;
-    GKO_ATTRIBUTES GKO_INLINE arithmetic_type *get_scale()
-    {
-        return this->scale_;
-    }
-};
-
-
-namespace detail {
-
-
 template <typename Accessor>
 struct is_3d_scaled_accessor : public std::false_type {};
 
-template <typename... Args>
-struct is_3d_scaled_accessor<ScaledReducedStorage3d<Args...>>
+template <typename T1, typename T2, size_type mask>
+struct is_3d_scaled_accessor<
+    range<accessor::scaled_reduced_row_major<3, T1, T2, mask>>>
     : public std::true_type {};
 
 template <typename StorageType, bool = std::is_integral<StorageType>::value>
@@ -623,7 +93,9 @@ class Accessor3dHelper {};
 template <typename ValueType, typename StorageType>
 class Accessor3dHelper<ValueType, StorageType, true> {
 public:
-    using Accessor = ScaledReducedStorage3d<ValueType, StorageType>;
+    using Accessor =
+        accessor::scaled_reduced_row_major<3, ValueType, StorageType, 0b101>;
+    using Range = range<Accessor>;
 
     Accessor3dHelper() = default;
 
@@ -635,18 +107,16 @@ class Accessor3dHelper<ValueType, StorageType, true> {
         // For testing, initialize scale to ones
         // Array<ValueType> h_scale{exec->get_master(), krylov_dim[0]};
         Array<ValueType> h_scale{exec->get_master(),
-                                 krylov_dim[0] * krylov_dim[2]};
+                                 krylov_dim_[0] * krylov_dim_[2]};
         for (size_type i = 0; i < h_scale.get_num_elems(); ++i) {
             h_scale.get_data()[i] = one<ValueType>();
         }
         scale_ = h_scale;
     }
 
-    Accessor get_accessor()
+    Range get_range()
     {
-        const auto stride0 = krylov_dim_[1] * krylov_dim_[2];
-        const auto stride1 = krylov_dim_[2];
-        return {bases_.get_data(), stride0, stride1, scale_.get_data()};
+        return Range(krylov_dim_, bases_.get_data(), scale_.get_data());
     }
 
     gko::Array<StorageType> &get_bases() { return bases_; }
@@ -661,7 +131,8 @@ class Accessor3dHelper<ValueType, StorageType, true> {
 template <typename ValueType, typename StorageType>
 class Accessor3dHelper<ValueType, StorageType, false> {
 public:
-    using Accessor = ReducedStorage3d<ValueType, StorageType>;
+    using Accessor = accessor::reduced_row_major<3, ValueType, StorageType>;
+    using Range = range<Accessor>;
 
     Accessor3dHelper() = default;
 
@@ -671,12 +142,7 @@ class Accessor3dHelper<ValueType, StorageType, false> {
                  krylov_dim_[0] * krylov_dim_[1] * krylov_dim_[2]}
     {}
 
-    Accessor get_accessor()
-    {
-        const auto stride0 = krylov_dim_[1] * krylov_dim_[2];
-        const auto stride1 = krylov_dim_[2];
-        return {bases_.get_data(), stride0, stride1};
-    }
+    Range get_range() { return Range(krylov_dim_, bases_.get_data()); }
 
     gko::Array<StorageType> &get_bases() { return bases_; }
 
@@ -694,46 +160,63 @@ struct helper_functions_accessor {};
 // Accessors having a scale
 template <typename Accessor3d>
 struct helper_functions_accessor<Accessor3d, true> {
-    using arithmetic_type = typename Accessor3d::arithmetic_type;
+    using arithmetic_type = typename Accessor3d::accessor::arithmetic_type;
+    static constexpr size_type dimensionality = Accessor3d::dimensionality;
     static_assert(detail::is_3d_scaled_accessor<Accessor3d>::value,
-                  "Accessor must have a scale here!");
+                  "Accessor must have a scalar here!");
     template <typename IndexType>
-    static inline GKO_ATTRIBUTES void write_scale(Accessor3d krylov_bases,
-                                                  IndexType vector_idx,
-                                                  IndexType col_idx,
-                                                  arithmetic_type value)
+    static inline GKO_ATTRIBUTES void write_scalar(Accessor3d krylov_bases,
+                                                   IndexType vector_idx,
+                                                   IndexType col_idx,
+                                                   arithmetic_type value)
     {
-        using storage_type = typename Accessor3d::storage_type;
+        using storage_type = typename Accessor3d::accessor::storage_type;
         constexpr arithmetic_type correction =
             std::is_integral<storage_type>::value
                 // Use 2 instead of 1 here to allow for a bit more room
                 ? 2 / static_cast<arithmetic_type>(
                           std::numeric_limits<storage_type>::max())
                 : 1;
-        krylov_bases.set_scale(vector_idx, col_idx, value * correction);
+        krylov_bases.get_accessor().write_scalar_direct(value * correction,
+                                                        vector_idx, col_idx);
+    }
+
+    static constexpr GKO_ATTRIBUTES std::array<size_type, dimensionality - 1>
+    get_stride(Accessor3d krylov_bases)
+    {
+        return krylov_bases.get_accessor().get_storage_stride();
     }
 };
 
 // Accessors not having a scale
 template <typename Accessor3d>
 struct helper_functions_accessor<Accessor3d, false> {
-    using arithmetic_type = typename Accessor3d::arithmetic_type;
+    using arithmetic_type = typename Accessor3d::accessor::arithmetic_type;
+    static constexpr size_type dimensionality = Accessor3d::dimensionality;
     static_assert(!detail::is_3d_scaled_accessor<Accessor3d>::value,
                   "Accessor must not have a scale here!");
+
     template <typename IndexType>
-    static inline GKO_ATTRIBUTES void write_scale(Accessor3d krylov_bases,
-                                                  IndexType vector_idx,
-                                                  IndexType col_idx,
-                                                  arithmetic_type value)
+    static inline GKO_ATTRIBUTES void write_scalar(Accessor3d krylov_bases,
+                                                   IndexType vector_idx,
+                                                   IndexType col_idx,
+                                                   arithmetic_type value)
     {}
+
+    static constexpr GKO_ATTRIBUTES std::array<size_type, dimensionality - 1>
+    get_stride(Accessor3d krylov_bases)
+    {
+        return krylov_bases.get_accessor().get_stride();
+    }
 };
 
 // calling it with:
-// helper_functions_accessor<Accessor3d>::write_scale(krylov_bases, col_idx,
+// helper_functions_accessor<Accessor3d>::write_scalar(krylov_bases, col_idx,
 // value);
 
 //----------------------------------------------
 
+
 }  // namespace kernels
 }  // namespace gko
 
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/gmres_mixed_kernels.hpp
index b4db2b61a7b..1a03e102bf7 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/gmres_mixed_kernels.hpp
@@ -36,15 +36,19 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/stop/stopping_status.hpp>
 
 
+#include "core/base/accessors.hpp"
 #include "core/base/extended_float.hpp"
 #include "core/solver/gmres_mixed_accessor.hpp"
 
 
+// TODO Find way around using it!
+#define GKO_UNPACK(...) __VA_ARGS__
 /**
  * Instantiates a template for each value type with each lower precision type
  * supported by Ginkgo for GmresMixed.
@@ -56,70 +60,107 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *                     (precisions supported by Ginkgo), and
  *                2. the second the value type of the reduced precision.
  */
-// #if defined(__CUDACC__) || defined(__HIPCC__)
-// #define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro) \
-//     template _macro(double, half);                  \
-//     template _macro(double, float);                 \
-//     template _macro(double, double)
-// #else
-#define GKO_UNPACK(...) __VA_ARGS__
-
-#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro)                     \
-    template _macro(double, GKO_UNPACK(ReducedStorage3d<double, double>));    \
-    template _macro(double, GKO_UNPACK(ReducedStorage3d<double, float>));     \
-    template _macro(double, GKO_UNPACK(ReducedStorage3d<double, half>));      \
-    template _macro(double,                                                   \
-                    GKO_UNPACK(ScaledReducedStorage3d<double, int64>));       \
-    template _macro(double,                                                   \
-                    GKO_UNPACK(ScaledReducedStorage3d<double, int32>));       \
-    template _macro(double,                                                   \
-                    GKO_UNPACK(ScaledReducedStorage3d<double, int16>));       \
-    template _macro(float, GKO_UNPACK(ReducedStorage3d<float, float>));       \
-    template _macro(float, GKO_UNPACK(ReducedStorage3d<float, half>));        \
-    template _macro(float, GKO_UNPACK(ScaledReducedStorage3d<float, int32>)); \
-    template _macro(float, GKO_UNPACK(ScaledReducedStorage3d<float, int16>)); \
-    template _macro(                                                          \
-        std::complex<double>,                                                 \
-        GKO_UNPACK(                                                           \
-            ReducedStorage3d<std::complex<double>, std::complex<double>>));   \
-    template _macro(                                                          \
-        std::complex<double>,                                                 \
-        GKO_UNPACK(                                                           \
-            ReducedStorage3d<std::complex<double>, std::complex<float>>));    \
-    template _macro(                                                          \
-        std::complex<float>,                                                  \
-        GKO_UNPACK(                                                           \
-            ReducedStorage3d<std::complex<float>, std::complex<float>>))
+#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro)                      \
+    template _macro(                                                           \
+        double,                                                                \
+        GKO_UNPACK(range<accessor::reduced_row_major<3, double, double>>));    \
+    template _macro(                                                           \
+        double,                                                                \
+        GKO_UNPACK(range<accessor::reduced_row_major<3, double, float>>));     \
+    template _macro(                                                           \
+        double,                                                                \
+        GKO_UNPACK(range<accessor::reduced_row_major<3, double, half>>));      \
+    template _macro(                                                           \
+        double,                                                                \
+        GKO_UNPACK(range<accessor::scaled_reduced_row_major<3, double, int64,  \
+                                                            0b101>>));         \
+    template _macro(                                                           \
+        double,                                                                \
+        GKO_UNPACK(range<accessor::scaled_reduced_row_major<3, double, int32,  \
+                                                            0b101>>));         \
+    template _macro(                                                           \
+        double,                                                                \
+        GKO_UNPACK(range<accessor::scaled_reduced_row_major<3, double, int16,  \
+                                                            0b101>>));         \
+    template _macro(                                                           \
+        float,                                                                 \
+        GKO_UNPACK(range<accessor::reduced_row_major<3, float, float>>));      \
+    template _macro(                                                           \
+        float,                                                                 \
+        GKO_UNPACK(range<accessor::reduced_row_major<3, float, half>>));       \
+    template _macro(                                                           \
+        float,                                                                 \
+        GKO_UNPACK(                                                            \
+            range<                                                             \
+                accessor::scaled_reduced_row_major<3, float, int32, 0b101>>)); \
+    template _macro(                                                           \
+        float,                                                                 \
+        GKO_UNPACK(                                                            \
+            range<                                                             \
+                accessor::scaled_reduced_row_major<3, float, int16, 0b101>>)); \
+    template _macro(                                                           \
+        std::complex<double>,                                                  \
+        GKO_UNPACK(range<accessor::reduced_row_major<3, std::complex<double>,  \
+                                                     std::complex<double>>>)); \
+    template _macro(                                                           \
+        std::complex<double>,                                                  \
+        GKO_UNPACK(range<accessor::reduced_row_major<3, std::complex<double>,  \
+                                                     std::complex<float>>>));  \
+    template _macro(                                                           \
+        std::complex<float>,                                                   \
+        GKO_UNPACK(range<accessor::reduced_row_major<3, std::complex<float>,   \
+                                                     std::complex<float>>>))
 
 #define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(_macro)                \
+    template _macro(                                                           \
+        double,                                                                \
+        GKO_UNPACK(                                                            \
+            range<accessor::reduced_row_major<3, double, const double>>));     \
+    template _macro(                                                           \
+        double,                                                                \
+        GKO_UNPACK(                                                            \
+            range<accessor::reduced_row_major<3, double, const float>>));      \
+    template _macro(                                                           \
+        double,                                                                \
+        GKO_UNPACK(                                                            \
+            range<accessor::reduced_row_major<3, double, const half>>));       \
     template _macro(double,                                                    \
-                    GKO_UNPACK(ConstReducedStorage3d<double, double>));        \
-    template _macro(double, GKO_UNPACK(ConstReducedStorage3d<double, float>)); \
-    template _macro(double, GKO_UNPACK(ConstReducedStorage3d<double, half>));  \
-    template _macro(double,                                                    \
-                    GKO_UNPACK(ConstScaledReducedStorage3d<double, int64>));   \
+                    GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
+                                   3, double, const int64, 0b101>>));          \
     template _macro(double,                                                    \
-                    GKO_UNPACK(ConstScaledReducedStorage3d<double, int32>));   \
+                    GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
+                                   3, double, const int32, 0b101>>));          \
     template _macro(double,                                                    \
-                    GKO_UNPACK(ConstScaledReducedStorage3d<double, int16>));   \
-    template _macro(float, GKO_UNPACK(ConstReducedStorage3d<float, float>));   \
-    template _macro(float, GKO_UNPACK(ConstReducedStorage3d<float, half>));    \
+                    GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
+                                   3, double, const int16, 0b101>>));          \
+    template _macro(                                                           \
+        float,                                                                 \
+        GKO_UNPACK(                                                            \
+            range<accessor::reduced_row_major<3, float, const float>>));       \
+    template _macro(                                                           \
+        float,                                                                 \
+        GKO_UNPACK(range<accessor::reduced_row_major<3, float, const half>>)); \
     template _macro(float,                                                     \
-                    GKO_UNPACK(ConstScaledReducedStorage3d<float, int32>));    \
+                    GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
+                                   3, float, const int32, 0b101>>));           \
     template _macro(float,                                                     \
-                    GKO_UNPACK(ConstScaledReducedStorage3d<float, int16>));    \
-    template _macro(std::complex<double>,                                      \
-                    GKO_UNPACK(ConstReducedStorage3d<std::complex<double>,     \
-                                                     std::complex<double>>));  \
-    template _macro(std::complex<double>,                                      \
-                    GKO_UNPACK(ConstReducedStorage3d<std::complex<double>,     \
-                                                     std::complex<float>>));   \
+                    GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
+                                   3, float, const int16, 0b101>>));           \
+    template _macro(                                                           \
+        std::complex<double>,                                                  \
+        GKO_UNPACK(                                                            \
+            range<accessor::reduced_row_major<3, std::complex<double>,         \
+                                              const std::complex<double>>>));  \
+    template _macro(                                                           \
+        std::complex<double>,                                                  \
+        GKO_UNPACK(                                                            \
+            range<accessor::reduced_row_major<3, std::complex<double>,         \
+                                              const std::complex<float>>>));   \
     template _macro(                                                           \
         std::complex<float>,                                                   \
         GKO_UNPACK(                                                            \
-            ConstReducedStorage3d<std::complex<float>, std::complex<float>>))
-// #endif
-// #undef GKO_UNPACK
+            range<accessor::reduced_row_major<3, std::complex<float>,          \
+                                              const std::complex<float>>>))
 
 
 namespace gko {
@@ -137,25 +178,25 @@ namespace gmres_mixed {
         size_type krylov_dim)
 
 
-#define GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(_type1, _accessor)      \
+#define GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(_type1, _range)         \
     void initialize_2(std::shared_ptr<const DefaultExecutor> exec,          \
                       const matrix::Dense<_type1> *residual,                \
                       matrix::Dense<remove_complex<_type1>> *residual_norm, \
                       matrix::Dense<_type1> *residual_norm_collection,      \
                       matrix::Dense<remove_complex<_type1>> *arnoldi_norm,  \
-                      _accessor krylov_bases,                               \
+                      _range krylov_bases,                                  \
                       matrix::Dense<_type1> *next_krylov_basis,             \
                       Array<size_type> *final_iter_nums, size_type krylov_dim)
 
 
-#define GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(_type1, _accessor)              \
+#define GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(_type1, _range)                 \
     void step_1(                                                              \
         std::shared_ptr<const DefaultExecutor> exec,                          \
         matrix::Dense<_type1> *next_krylov_basis,                             \
         matrix::Dense<_type1> *givens_sin, matrix::Dense<_type1> *givens_cos, \
         matrix::Dense<remove_complex<_type1>> *residual_norm,                 \
-        matrix::Dense<_type1> *residual_norm_collection,                      \
-        _accessor krylov_bases, matrix::Dense<_type1> *hessenberg_iter,       \
+        matrix::Dense<_type1> *residual_norm_collection, _range krylov_bases, \
+        matrix::Dense<_type1> *hessenberg_iter,                               \
         matrix::Dense<_type1> *buffer_iter,                                   \
         const matrix::Dense<remove_complex<_type1>> *b_norm,                  \
         matrix::Dense<remove_complex<_type1>> *arnoldi_norm, size_type iter,  \
@@ -165,13 +206,12 @@ namespace gmres_mixed {
         int *num_reorth_steps, int *num_reorth_vectors)
 
 
-#define GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(_type1, _accessor)       \
-    void step_2(std::shared_ptr<const DefaultExecutor> exec,           \
-                const matrix::Dense<_type1> *residual_norm_collection, \
-                _accessor krylov_bases,                                \
-                const matrix::Dense<_type1> *hessenberg,               \
-                matrix::Dense<_type1> *y,                              \
-                matrix::Dense<_type1> *before_preconditioner,          \
+#define GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(_type1, _range)                 \
+    void step_2(std::shared_ptr<const DefaultExecutor> exec,                  \
+                const matrix::Dense<_type1> *residual_norm_collection,        \
+                _range krylov_bases, const matrix::Dense<_type1> *hessenberg, \
+                matrix::Dense<_type1> *y,                                     \
+                matrix::Dense<_type1> *before_preconditioner,                 \
                 const Array<size_type> *final_iter_nums)
 
 
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 19fbdb79dd1..ec0aefeffbb 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -38,11 +38,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/range_accessors.hpp>
+#include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "core/base/accessors.hpp"
 #include "core/components/fill_array.hpp"
 #include "cuda/base/config.hpp"
 #include "cuda/base/cublas_bindings.hpp"
@@ -92,36 +93,26 @@ constexpr int default_update_dim = 32;
 
 
 // Specialization, so the Accessor can use the same function as regular pointers
-template <typename Type1, typename Type2>
-ReducedStorage3d<cuda_type<Type1>, cuda_type<Type2>> as_cuda_accessor(
-    ReducedStorage3d<Type1, Type2> acc)
+template <int dim, typename Type1, typename Type2>
+GKO_INLINE auto as_cuda_accessor(
+    const range<accessor::reduced_row_major<dim, Type1, Type2>> &acc)
 {
-    return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
-            acc.get_stride1()};
+    return range<
+        accessor::reduced_row_major<dim, cuda_type<Type1>, cuda_type<Type2>>>(
+        acc.get_accessor().get_size(), as_cuda_type(acc.get_accessor().get_stored_data()),
+        acc.get_accessor().get_stride());
 }
 
-template <typename Type1, typename Type2>
-ScaledReducedStorage3d<cuda_type<Type1>, cuda_type<Type2>> as_cuda_accessor(
-    ScaledReducedStorage3d<Type1, Type2> acc)
+template <int dim, typename Type1, typename Type2, size_type mask>
+GKO_INLINE auto as_cuda_accessor(
+    const range<accessor::scaled_reduced_row_major<dim, Type1, Type2, mask>> &acc)
 {
-    return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
-            acc.get_stride1(), as_cuda_type(acc.get_scale())};
-}
-
-template <typename Type1, typename Type2>
-ConstReducedStorage3d<cuda_type<Type1>, cuda_type<Type2>> as_cuda_accessor(
-    const ConstReducedStorage3d<Type1, Type2> &acc)
-{
-    return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
-            acc.get_stride1()};
-}
-
-template <typename Type1, typename Type2>
-ConstScaledReducedStorage3d<cuda_type<Type1>, cuda_type<Type2>>
-as_cuda_accessor(const ConstScaledReducedStorage3d<Type1, Type2> &acc)
-{
-    return {as_cuda_type(acc.get_storage()), acc.get_stride0(),
-            acc.get_stride1(), as_cuda_type(acc.get_scale())};
+    return range<accessor::scaled_reduced_row_major<dim, cuda_type<Type1>,
+                                                    cuda_type<Type2>, mask>>(
+        acc.get_accessor().get_size(), as_cuda_type(acc.get_accessor().get_stored_data()),
+        acc.get_accessor().get_storage_stride(),
+        as_cuda_type(acc.get_accessor().get_scalar()),
+        acc.get_accessor().get_scalar_stride());
 }
 
 
@@ -176,9 +167,10 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
 {
     const auto num_rows = residual->get_size()[0];
     const auto num_rhs = residual->get_size()[1];
-    const dim3 grid_dim_1(ceildiv((krylov_dim + 1) * krylov_bases.get_stride0(),
-                                  default_block_size),
-                          1, 1);
+    const auto krylov_stride =
+        helper_functions_accessor<Accessor3d>::get_stride(krylov_bases);
+    const dim3 grid_dim_1(
+        ceildiv((krylov_dim + 1) * krylov_stride[0], default_block_size), 1, 1);
     const dim3 block_dim(default_block_size, 1, 1);
     constexpr auto block_size = default_block_size;
 
@@ -204,7 +196,7 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
         as_cuda_type(arnoldi_norm->get_values() + 2 * num_rhs), 0);
     // exec->synchronize();
 
-    // helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scale(
+    // helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scalar(
     //     krylov_bases, col_idx, arnoldi_norm->at(2, j) / residual_norm->at(0,
     //     j));
     /* */
@@ -221,8 +213,7 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
     /* */
     // std::cout << "Before initialize_2_2_kernel" << '\n';
     const dim3 grid_dim_2(
-        ceildiv(num_rows * krylov_bases.get_stride1(), default_block_size), 1,
-        1);
+        ceildiv(num_rows * krylov_stride[1], default_block_size), 1, 1);
     initialize_2_2_kernel<block_size><<<grid_dim_2, block_dim>>>(
         residual->get_size()[0], residual->get_size()[1],
         as_cuda_type(residual->get_const_values()), residual->get_stride(),
@@ -280,10 +271,6 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     const dim3 block_size_iters_single(singledot_block_size);
     size_type numReorth;
 
-    ReducedStorage3d<ValueType, ValueType> next_krylov_accessor{
-        next_krylov_basis->get_values(), stride_next_krylov,
-        stride_next_krylov};
-
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<non_complex>());
     /*
diff --git a/cuda/test/solver/gmres_mixed_kernels.cpp b/cuda/test/solver/gmres_mixed_kernels.cpp
index cfbbfb9a9fc..9fd61b0f2f1 100644
--- a/cuda/test/solver/gmres_mixed_kernels.cpp
+++ b/cuda/test/solver/gmres_mixed_kernels.cpp
@@ -61,9 +61,9 @@ class GmresMixed : public ::testing::Test {
     using storage_type = float;
     using index_type = int;
     using size_type = gko::size_type;
-    using Accessor3d = gko::kernels::ReducedStorage3d<value_type, storage_type>;
-    using Accessor3dHelper =
+    using Range3dHelper =
         gko::kernels::Accessor3dHelper<value_type, storage_type>;
+    using Range3d = typename Range3dHelper::Range;
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
 
@@ -91,9 +91,9 @@ class GmresMixed : public ::testing::Test {
             std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
     }
 
-    Accessor3dHelper generate_krylov_helper(gko::dim<3> size)
+    Range3dHelper generate_krylov_helper(gko::dim<3> size)
     {
-        auto helper = Accessor3dHelper{ref, size};
+        auto helper = Range3dHelper{ref, size};
         auto &bases = helper.get_bases();
         const auto num_rows = size[0] * size[1];
         const auto num_cols = size[2];
@@ -105,12 +105,12 @@ class GmresMixed : public ::testing::Test {
         std::copy_n(temp_krylov_bases->get_const_values(),
                     bases.get_num_elems(), bases.get_data());
         // Only useful when the Accessor actually has a scale
-        auto accessor = helper.get_accessor();
+        auto range = helper.get_range();
         auto dist = std::normal_distribution<value_type>(-1, 1);
         for (size_type k = 0; k < size[0]; ++k) {
             for (size_type i = 0; i < size[2]; ++i) {
-                gko::kernels::helper_functions_accessor<
-                    Accessor3d>::write_scale(accessor, k, i, dist(rand_engine));
+                gko::kernels::helper_functions_accessor<Range3d>::write_scalar(
+                    range, k, i, dist(rand_engine));
             }
         }
         return helper;
@@ -128,8 +128,7 @@ class GmresMixed : public ::testing::Test {
         arnoldi_norm = gen_mtx(3, n);
         gko::dim<3> krylov_bases_dim(gko::solver::default_krylov_dim_mixed + 1,
                                      m, n);
-        acc_helper = generate_krylov_helper(krylov_bases_dim);
-        krylov_bases_accessor = acc_helper.get_accessor();
+        range_helper = generate_krylov_helper(krylov_bases_dim);
 
         next_krylov_basis = gen_mtx(m, n);
         hessenberg = gen_mtx(gko::solver::default_krylov_dim_mixed + 1,
@@ -174,9 +173,8 @@ class GmresMixed : public ::testing::Test {
         d_b_norm->copy_from(b_norm.get());
         d_arnoldi_norm = Mtx::create(cuda);
         d_arnoldi_norm->copy_from(arnoldi_norm.get());
-        d_acc_helper = Accessor3dHelper{cuda, {}};
-        d_acc_helper = acc_helper;
-        d_krylov_bases_accessor = d_acc_helper.get_accessor();
+        d_range_helper = Range3dHelper{cuda, {}};
+        d_range_helper = range_helper;
         d_next_krylov_basis = Mtx::create(cuda);
         d_next_krylov_basis->copy_from(next_krylov_basis.get());
         d_hessenberg = Mtx::create(cuda);
@@ -212,8 +210,8 @@ class GmresMixed : public ::testing::Test {
     void assert_krylov_bases_near()
     {
         gko::Array<storage_type> d_to_host{ref};
-        auto &krylov_bases = acc_helper.get_bases();
-        d_to_host = d_acc_helper.get_bases();
+        auto &krylov_bases = range_helper.get_bases();
+        d_to_host = d_range_helper.get_bases();
         const auto tolerance = r<storage_type>::value;
         using std::abs;
         for (gko::size_type i = 0; i < krylov_bases.get_num_elems(); ++i) {
@@ -234,8 +232,7 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> b_norm;
     std::unique_ptr<Mtx> arnoldi_norm;
-    Accessor3dHelper acc_helper;
-    Accessor3d krylov_bases_accessor;
+    Range3dHelper range_helper;
     std::unique_ptr<Mtx> next_krylov_basis;
     std::unique_ptr<Mtx> hessenberg;
     std::unique_ptr<Mtx> hessenberg_iter;
@@ -256,8 +253,7 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> d_b;
     std::unique_ptr<Mtx> d_b_norm;
     std::unique_ptr<Mtx> d_arnoldi_norm;
-    Accessor3dHelper d_acc_helper;
-    Accessor3d d_krylov_bases_accessor;
+    Range3dHelper d_range_helper;
     std::unique_ptr<Mtx> d_next_krylov_basis;
     std::unique_ptr<Mtx> d_hessenberg;
     std::unique_ptr<Mtx> d_hessenberg_iter;
@@ -302,12 +298,12 @@ TEST_F(GmresMixed, CudaGmresMixedInitialize2IsEquivalentToRef)
     gko::kernels::reference::gmres_mixed::initialize_2(
         ref, residual.get(), residual_norm.get(),
         residual_norm_collection.get(), arnoldi_norm.get(),
-        krylov_bases_accessor, next_krylov_basis.get(), final_iter_nums.get(),
-        gko::solver::default_krylov_dim_mixed);
+        range_helper.get_range(), next_krylov_basis.get(),
+        final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
     gko::kernels::cuda::gmres_mixed::initialize_2(
         cuda, d_residual.get(), d_residual_norm.get(),
         d_residual_norm_collection.get(), d_arnoldi_norm.get(),
-        d_krylov_bases_accessor, d_next_krylov_basis.get(),
+        d_range_helper.get_range(), d_next_krylov_basis.get(),
         d_final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
 
     GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
@@ -328,17 +324,17 @@ TEST_F(GmresMixed, CudaGmresMixedStep1IsEquivalentToRef)
     gko::kernels::reference::gmres_mixed::step_1(
         ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
         residual_norm.get(), residual_norm_collection.get(),
-        krylov_bases_accessor, hessenberg_iter.get(), buffer_iter.get(),
+        range_helper.get_range(), hessenberg_iter.get(), buffer_iter.get(),
         b_norm.get(), arnoldi_norm.get(), iter, final_iter_nums.get(),
         stop_status.get(), reorth_status.get(), num_reorth.get(),
         &num_reorth_steps, &num_reorth_vectors);
     gko::kernels::cuda::gmres_mixed::step_1(
         cuda, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
         d_residual_norm.get(), d_residual_norm_collection.get(),
-        d_krylov_bases_accessor, d_hessenberg_iter.get(), d_buffer_iter.get(),
-        d_b_norm.get(), d_arnoldi_norm.get(), iter, d_final_iter_nums.get(),
-        d_stop_status.get(), d_reorth_status.get(), d_num_reorth.get(),
-        &num_reorth_steps, &num_reorth_vectors);
+        d_range_helper.get_range(), d_hessenberg_iter.get(),
+        d_buffer_iter.get(), d_b_norm.get(), d_arnoldi_norm.get(), iter,
+        d_final_iter_nums.get(), d_stop_status.get(), d_reorth_status.get(),
+        d_num_reorth.get(), &num_reorth_steps, &num_reorth_vectors);
 
     GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
@@ -357,13 +353,14 @@ TEST_F(GmresMixed, CudaGmresMixedStep2IsEquivalentToRef)
     initialize_data();
 
     gko::kernels::reference::gmres_mixed::step_2(
-        ref, residual_norm_collection.get(), krylov_bases_accessor.to_const(),
-        hessenberg.get(), y.get(), before_preconditioner.get(),
-        final_iter_nums.get());
+        ref, residual_norm_collection.get(),
+        range_helper.get_range().get_accessor().to_const(), hessenberg.get(),
+        y.get(), before_preconditioner.get(), final_iter_nums.get());
     gko::kernels::cuda::gmres_mixed::step_2(
         cuda, d_residual_norm_collection.get(),
-        d_krylov_bases_accessor.to_const(), d_hessenberg.get(), d_y.get(),
-        d_before_preconditioner.get(), d_final_iter_nums.get());
+        d_range_helper.get_range().get_accessor().to_const(),
+        d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
+        d_final_iter_nums.get());
 
     GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/gmres_mixed_kernels.cpp
index 9c69542906a..fa1b66afa79 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/gmres_mixed_kernels.cpp
@@ -199,7 +199,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
             // nrmN = norm(next_krylov_basis)
             // nrmI = infnorm(next_krylov_basis)
         }
-        helper_functions_accessor<Accessor3d>::write_scale(
+        helper_functions_accessor<Accessor3d>::write_scalar(
             krylov_bases, iter + 1, i,
             arnoldi_norm->at(2, i) / arnoldi_norm->at(1, i));
         // reorthogonalization
@@ -416,7 +416,7 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
         }
         residual_norm->at(0, j) = sqrt(res_norm);
         arnoldi_norm->at(2, j) = res_inf;
-        helper_functions_accessor<Accessor3d>::write_scale(
+        helper_functions_accessor<Accessor3d>::write_scalar(
             krylov_bases, {0}, j,
             arnoldi_norm->at(2, j) / residual_norm->at(0, j));
 
@@ -432,7 +432,7 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             auto value = residual->at(i, j) / residual_norm->at(0, j);
-            krylov_bases({0}, i, j) = value;
+            krylov_bases(0, i, j) = value;
             next_krylov_basis->at(i, j) = value;
         }
         final_iter_nums->get_data()[j] = 0;
@@ -441,7 +441,7 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for
     for (size_type k = 1; k < krylov_dim + 1; ++k) {
         for (size_type j = 0; j < residual->get_size()[1]; ++j) {
-            helper_functions_accessor<Accessor3d>::write_scale(
+            helper_functions_accessor<Accessor3d>::write_scalar(
                 krylov_bases, k, j, one<remove_complex<ValueType>>());
         }
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
diff --git a/omp/test/solver/gmres_mixed_kernels.cpp b/omp/test/solver/gmres_mixed_kernels.cpp
index 566ccd5fcfc..dec133547be 100644
--- a/omp/test/solver/gmres_mixed_kernels.cpp
+++ b/omp/test/solver/gmres_mixed_kernels.cpp
@@ -63,9 +63,9 @@ class GmresMixed : public ::testing::Test {
     using storage_type = float;
     using index_type = int;
     using size_type = gko::size_type;
-    using Accessor3d = gko::kernels::ReducedStorage3d<value_type, storage_type>;
-    using Accessor3dHelper =
+    using Range3dHelper =
         gko::kernels::Accessor3dHelper<value_type, storage_type>;
+    using Range3d = typename Range3dHelper::Range;
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
 
@@ -93,9 +93,9 @@ class GmresMixed : public ::testing::Test {
             std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
     }
 
-    Accessor3dHelper generate_krylov_helper(gko::dim<3> size)
+    Range3dHelper generate_krylov_helper(gko::dim<3> size)
     {
-        auto helper = Accessor3dHelper{ref, size};
+        auto helper = Range3dHelper{ref, size};
         auto &bases = helper.get_bases();
         const auto num_rows = size[0] * size[1];
         const auto num_cols = size[2];
@@ -107,12 +107,12 @@ class GmresMixed : public ::testing::Test {
         std::copy_n(temp_krylov_bases->get_const_values(),
                     bases.get_num_elems(), bases.get_data());
         // Only useful when the Accessor actually has a scale
-        auto accessor = helper.get_accessor();
+        auto range = helper.get_range();
         auto dist = std::normal_distribution<value_type>(-1, 1);
         for (size_type k = 0; k < size[0]; ++k) {
             for (size_type i = 0; i < size[2]; ++i) {
-                gko::kernels::helper_functions_accessor<
-                    Accessor3d>::write_scale(accessor, k, i, dist(rand_engine));
+                gko::kernels::helper_functions_accessor<Range3d>::write_scalar(
+                    range, k, i, dist(rand_engine));
             }
         }
         return helper;
@@ -130,8 +130,7 @@ class GmresMixed : public ::testing::Test {
         arnoldi_norm = gen_mtx(3, n);
         gko::dim<3> krylov_bases_dim(gko::solver::default_krylov_dim_mixed + 1,
                                      m, n);
-        acc_helper = generate_krylov_helper(krylov_bases_dim);
-        krylov_bases_accessor = acc_helper.get_accessor();
+        range_helper = generate_krylov_helper(krylov_bases_dim);
 
         next_krylov_basis = gen_mtx(m, n);
         hessenberg = gen_mtx(gko::solver::default_krylov_dim_mixed + 1,
@@ -176,9 +175,8 @@ class GmresMixed : public ::testing::Test {
         d_b_norm->copy_from(b_norm.get());
         d_arnoldi_norm = Mtx::create(omp);
         d_arnoldi_norm->copy_from(arnoldi_norm.get());
-        d_acc_helper = Accessor3dHelper{omp, {}};
-        d_acc_helper = acc_helper;
-        d_krylov_bases_accessor = d_acc_helper.get_accessor();
+        d_range_helper = Range3dHelper{omp, {}};
+        d_range_helper = range_helper;
         d_next_krylov_basis = Mtx::create(omp);
         d_next_krylov_basis->copy_from(next_krylov_basis.get());
         d_hessenberg = Mtx::create(omp);
@@ -213,8 +211,8 @@ class GmresMixed : public ::testing::Test {
     void assert_krylov_bases_near()
     {
         gko::Array<storage_type> d_to_host{ref};
-        auto &krylov_bases = acc_helper.get_bases();
-        d_to_host = d_acc_helper.get_bases();
+        auto &krylov_bases = range_helper.get_bases();
+        d_to_host = d_range_helper.get_bases();
         const auto tolerance = r<storage_type>::value;
         using std::abs;
         for (gko::size_type i = 0; i < krylov_bases.get_num_elems(); ++i) {
@@ -235,8 +233,7 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> b_norm;
     std::unique_ptr<Mtx> arnoldi_norm;
-    Accessor3dHelper acc_helper;
-    Accessor3d krylov_bases_accessor;
+    Range3dHelper range_helper;
     std::unique_ptr<Mtx> next_krylov_basis;
     std::unique_ptr<Mtx> hessenberg;
     std::unique_ptr<Mtx> hessenberg_iter;
@@ -257,8 +254,7 @@ class GmresMixed : public ::testing::Test {
     std::unique_ptr<Mtx> d_b;
     std::unique_ptr<Mtx> d_b_norm;
     std::unique_ptr<Mtx> d_arnoldi_norm;
-    Accessor3dHelper d_acc_helper;
-    Accessor3d d_krylov_bases_accessor;
+    Range3dHelper d_range_helper;
     std::unique_ptr<Mtx> d_next_krylov_basis;
     std::unique_ptr<Mtx> d_hessenberg;
     std::unique_ptr<Mtx> d_hessenberg_iter;
@@ -302,12 +298,12 @@ TEST_F(GmresMixed, OmpGmresMixedInitialize2IsEquivalentToRef)
     gko::kernels::reference::gmres_mixed::initialize_2(
         ref, residual.get(), residual_norm.get(),
         residual_norm_collection.get(), arnoldi_norm.get(),
-        krylov_bases_accessor, next_krylov_basis.get(), final_iter_nums.get(),
-        gko::solver::default_krylov_dim_mixed);
+        range_helper.get_range(), next_krylov_basis.get(),
+        final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
     gko::kernels::omp::gmres_mixed::initialize_2(
         omp, d_residual.get(), d_residual_norm.get(),
         d_residual_norm_collection.get(), d_arnoldi_norm.get(),
-        d_krylov_bases_accessor, d_next_krylov_basis.get(),
+        d_range_helper.get_range(), d_next_krylov_basis.get(),
         d_final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
 
     GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
@@ -327,17 +323,17 @@ TEST_F(GmresMixed, OmpGmresMixedStep1IsEquivalentToRef)
     gko::kernels::reference::gmres_mixed::step_1(
         ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
         residual_norm.get(), residual_norm_collection.get(),
-        krylov_bases_accessor, hessenberg_iter.get(), buffer_iter.get(),
+        range_helper.get_range(), hessenberg_iter.get(), buffer_iter.get(),
         b_norm.get(), arnoldi_norm.get(), iter, final_iter_nums.get(),
         stop_status.get(), reorth_status.get(), num_reorth.get(),
         &num_reorth_steps, &num_reorth_vectors);
     gko::kernels::omp::gmres_mixed::step_1(
         omp, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
         d_residual_norm.get(), d_residual_norm_collection.get(),
-        d_krylov_bases_accessor, d_hessenberg_iter.get(), d_buffer_iter.get(),
-        d_b_norm.get(), d_arnoldi_norm.get(), iter, d_final_iter_nums.get(),
-        d_stop_status.get(), d_reorth_status.get(), d_num_reorth.get(),
-        &num_reorth_steps, &num_reorth_vectors);
+        d_range_helper.get_range(), d_hessenberg_iter.get(),
+        d_buffer_iter.get(), d_b_norm.get(), d_arnoldi_norm.get(), iter,
+        d_final_iter_nums.get(), d_stop_status.get(), d_reorth_status.get(),
+        d_num_reorth.get(), &num_reorth_steps, &num_reorth_vectors);
 
     GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
@@ -355,13 +351,14 @@ TEST_F(GmresMixed, OmpGmresMixedStep2IsEquivalentToRef)
     initialize_data();
 
     gko::kernels::reference::gmres_mixed::step_2(
-        ref, residual_norm_collection.get(), krylov_bases_accessor.to_const(),
-        hessenberg.get(), y.get(), before_preconditioner.get(),
-        final_iter_nums.get());
+        ref, residual_norm_collection.get(),
+        range_helper.get_range().get_accessor().to_const(), hessenberg.get(),
+        y.get(), before_preconditioner.get(), final_iter_nums.get());
     gko::kernels::omp::gmres_mixed::step_2(
         omp, d_residual_norm_collection.get(),
-        d_krylov_bases_accessor.to_const(), d_hessenberg.get(), d_y.get(),
-        d_before_preconditioner.get(), d_final_iter_nums.get());
+        d_range_helper.get_range().get_accessor().to_const(),
+        d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
+        d_final_iter_nums.get());
 
     GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/gmres_mixed_kernels.cpp
index 50ca1914eaa..a6476b030c1 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/gmres_mixed_kernels.cpp
@@ -68,7 +68,8 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                          size_type iter, const stopping_status *stop_status)
 {
     static_assert(
-        std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
+        std::is_same<ValueType,
+                     typename Accessor3d::accessor::arithmetic_type>::value,
         "ValueType must match arithmetic_type of accessor!");
     const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
@@ -151,7 +152,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
             arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
             // nrmN = norm(next_krylov_basis)
         }
-        helper_functions_accessor<Accessor3d>::write_scale(
+        helper_functions_accessor<Accessor3d>::write_scalar(
             krylov_bases, iter + 1, i,
             arnoldi_norm->at(2, i) / arnoldi_norm->at(1, i));
         hessenberg_iter->at(iter + 1, i) = arnoldi_norm->at(1, i);
@@ -281,8 +282,9 @@ void calculate_qy(ConstAccessor3d krylov_bases,
                   const size_type *final_iter_nums)
 {
     static_assert(
-        std::is_same<ValueType,
-                     typename ConstAccessor3d::arithmetic_type>::value,
+        std::is_same<
+            ValueType,
+            typename ConstAccessor3d::accessor::arithmetic_type>::value,
         "ValueType must match arithmetic_type of accessor!");
     for (size_type k = 0; k < before_preconditioner->get_size()[1]; ++k) {
         for (size_type i = 0; i < before_preconditioner->get_size()[0]; ++i) {
@@ -342,7 +344,8 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
 {
     static_assert(
-        std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
+        std::is_same<ValueType,
+                     typename Accessor3d::accessor::arithmetic_type>::value,
         "ValueType must match arithmetic_type of accessor!");
     for (size_type j = 0; j < residual->get_size()[1]; ++j) {
         // Calculate residual norm
@@ -356,7 +359,7 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
                     : abs(residual->at(i, j));
         }
         residual_norm->at(0, j) = sqrt(residual_norm->at(0, j));
-        helper_functions_accessor<Accessor3d>::write_scale(
+        helper_functions_accessor<Accessor3d>::write_scalar(
             krylov_bases, {0}, j,
             arnoldi_norm->at(2, j) / residual_norm->at(0, j));
 
@@ -368,7 +371,7 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
             }
         }
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
-            krylov_bases({0}, i, j) =
+            krylov_bases(0, i, j) =
                 residual->at(i, j) / residual_norm->at(0, j);
             next_krylov_basis->at(i, j) =
                 residual->at(i, j) / residual_norm->at(0, j);
@@ -378,7 +381,7 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
 
     for (size_type k = 1; k < krylov_dim + 1; ++k) {
         for (size_type j = 0; j < residual->get_size()[1]; ++j) {
-            helper_functions_accessor<Accessor3d>::write_scale(
+            helper_functions_accessor<Accessor3d>::write_scalar(
                 krylov_bases, k, j, one<remove_complex<ValueType>>());
             for (size_type i = 0; i < residual->get_size()[0]; ++i) {
                 krylov_bases(k, i, j) = zero<ValueType>();
@@ -410,7 +413,8 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
             int *num_reorth_steps, int *num_reorth_vectors)
 {
     static_assert(
-        std::is_same<ValueType, typename Accessor3d::arithmetic_type>::value,
+        std::is_same<ValueType,
+                     typename Accessor3d::accessor::arithmetic_type>::value,
         "ValueType must match arithmetic_type of accessor!");
     for (size_type i = 0; i < final_iter_nums->get_num_elems(); ++i) {
         final_iter_nums->get_data()[i] +=

From 29ed79a19cab2018a73c3e0f5434604642923644 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 14 Sep 2020 07:59:36 +0200
Subject: [PATCH 0389/1008] Remove unnecessary code from CUDA GmresMixed

---
 cuda/solver/gmres_mixed_kernels.cu | 232 +----------------------------
 1 file changed, 1 insertion(+), 231 deletions(-)

diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index ec0aefeffbb..063876ce2d1 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -39,7 +39,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/range.hpp>
-#include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
@@ -56,18 +55,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "cuda/components/uninitialized_array.hpp"
 
 
-// #define TIMING 1
-
-
-#ifdef TIMING
-#include <chrono>
-#include <iostream>
-
-
-using double_seconds = std::chrono::duration<double, std::milli>;
-#endif
-
-
 namespace gko {
 namespace kernels {
 namespace cuda {
@@ -174,18 +161,13 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
     const dim3 block_dim(default_block_size, 1, 1);
     constexpr auto block_size = default_block_size;
 
-    // exec->synchronize();
-    // std::cout << "Before initialize_2_1_kernel" << '\n';
     initialize_2_1_kernel<block_size><<<grid_dim_1, block_dim>>>(
         residual->get_size()[0], residual->get_size()[1], krylov_dim,
         as_cuda_accessor(krylov_bases),
         as_cuda_type(residual_norm_collection->get_values()),
         residual_norm_collection->get_stride());
-    // exec->synchronize();
     residual->compute_norm2(residual_norm);
-    // exec->synchronize();
 
-    // std::cout << "Before multinorminf_kernel_without_stop" << '\n';
     components::fill_array(exec, arnoldi_norm->get_values() + 2 * num_rhs,
                            num_rhs, zero<remove_complex<ValueType>>());
     const dim3 grid_size_nrm(ceildiv(num_rhs, default_dot_dim),
@@ -194,7 +176,6 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
     multinorminf_kernel_without_stop<<<grid_size_nrm, block_size_nrm>>>(
         num_rows, num_rhs, as_cuda_type(residual->get_const_values()), num_rhs,
         as_cuda_type(arnoldi_norm->get_values() + 2 * num_rhs), 0);
-    // exec->synchronize();
 
     // helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scalar(
     //     krylov_bases, col_idx, arnoldi_norm->at(2, j) / residual_norm->at(0,
@@ -209,9 +190,7 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
             as_cuda_type(residual_norm->get_const_values()), num_rhs,
             as_cuda_type(arnoldi_norm->get_const_values() + 2 * stride_arnoldi),
             stride_arnoldi, as_cuda_accessor(krylov_bases));
-    // exec->synchronize();
-    /* */
-    // std::cout << "Before initialize_2_2_kernel" << '\n';
+
     const dim3 grid_dim_2(
         ceildiv(num_rows * krylov_stride[1], default_block_size), 1, 1);
     initialize_2_2_kernel<block_size><<<grid_dim_2, block_dim>>>(
@@ -223,8 +202,6 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
         as_cuda_type(next_krylov_basis->get_values()),
         next_krylov_basis->get_stride(),
         as_cuda_type(final_iter_nums->get_data()));
-    // exec->synchronize();
-    // std::cout << "After initialize_2_2_kernel" << '\n';
 }
 
 GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
@@ -273,37 +250,13 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
 
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<non_complex>());
-    /*
-    multidot_kernel<<<grid_size, block_size>>>(
-        0, dim_size[0], dim_size[1],
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_accessor(next_krylov_const_accessor),
-        // as_cuda_type(next_krylov_basis->get_const_values()),
-        // stride_next_krylov,
-        as_cuda_type(arnoldi_norm->get_values()), 0, as_cuda_type(stop_status));
-    */
     multinorm2_kernel<<<grid_size, block_size>>>(
         dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_type(arnoldi_norm->get_values()), as_cuda_type(stop_status));
     // nrmP = norm(next_krylov_basis
-#ifdef TIMING
-    exec->synchronize();
-    auto start_1 = std::chrono::steady_clock::now();
-#endif
-    /* */
     zero_matrix(iter + 1, dim_size[1], stride_hessenberg,
                 hessenberg_iter->get_values());
-    /*
-    multidot_kernel_num_iters<<<grid_size, block_size>>>(
-        iter + 1, dim_size[0], dim_size[1],
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_accessor(krylov_bases.to_const()),
-        as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-        as_cuda_type(stop_status));
-    /*/
-    //    write(std::cout, krylov_bases);
-    //    write(std::cout, next_krylov_basis);
     if (dim_size[1] > 1) {
         multidot_kernel_num_iters_2<default_dot_dim>
             <<<grid_size_num_iters_2, block_size>>>(
@@ -321,24 +274,9 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
                 as_cuda_type(stop_status));
     }
-    // */
-    // exec->synchronize();
-    // write(std::cout, hessenberg_iter);
-    /* */
-#ifdef TIMING
-    exec->synchronize();
-    auto time_1 = std::chrono::steady_clock::now() - start_1;
-    std::cout << "time_1(" << iter << ") = "
-              << std::chrono::duration_cast<double_seconds>(time_1).count()
-              << '\n';
-#endif
     // for i in 1:iter
     //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
     // end
-#ifdef TIMING
-    exec->synchronize();
-    auto start_2 = std::chrono::steady_clock::now();
-#endif
     update_next_krylov_kernel_num_iters<default_block_size>
         <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
            default_block_size>>>(
@@ -348,132 +286,36 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
             as_cuda_type(hessenberg_iter->get_const_values()),
             stride_hessenberg, as_cuda_type(stop_status));
 
-#ifdef TIMING
-    exec->synchronize();
-    auto time_2 = std::chrono::steady_clock::now() - start_2;
-    std::cout << "time_2(" << iter << ") = "
-              << std::chrono::duration_cast<double_seconds>(time_2).count()
-              << '\n';
-    std::cout << "time_1 / time_2(" << iter << ") = "
-              << std::chrono::duration_cast<double_seconds>(time_1).count() /
-                     std::chrono::duration_cast<double_seconds>(time_2).count()
-              << '\n';
-#endif
     // for i in 1:iter
     //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
     // end
-#ifdef TIMING
-    exec->synchronize();
-    auto start_3 = std::chrono::steady_clock::now();
-#endif
     components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
                            dim_size[1], zero<non_complex>());
     components::fill_array(exec, arnoldi_norm->get_values() + 2 * dim_size[1],
                            dim_size[1], zero<non_complex>());
-    /*
-    multidot_kernel<<<grid_size, block_size>>>(
-        0, dim_size[0], dim_size[1],
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_accessor(next_krylov_const_accessor),
-        as_cuda_type(arnoldi_norm->get_values() + dim_size[1]), 0,
-        as_cuda_type(stop_status));
-    */
-    /*
-    multinorm2_kernel<<<grid_size, block_size>>>(
-        dim_size[0], dim_size[1],
-        as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(arnoldi_norm->get_values() + dim_size[1]),
-        as_cuda_type(stop_status));
-    */
-    /*
- multinorminf_kernel<<<grid_size, block_size>>>(
-     dim_size[0], dim_size[1],
-     as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-     as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]), 0,
-     as_cuda_type(stop_status));
- */ /* */
     multinorm2_inf_kernel<use_scale><<<grid_size, block_size>>>(
         dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_type(arnoldi_norm->get_values() + dim_size[1]),
         as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]),
         as_cuda_type(stop_status));
-    /* */
-    /*
-    {
-        exec->synchronize();
-        remove_complex<ValueType> norm, inf;
-        exec->get_master()->copy_from(
-            exec.get(), 1, arnoldi_norm->get_values() + dim_size[1], &norm);
-        exec->get_master()->copy_from(
-            exec.get(), 1, arnoldi_norm->get_values() + 2 * dim_size[1], &inf);
-        std::cout << sqrt(norm) << " - " << inf << '\n';
-    }
-    */
-#ifdef TIMING
-    exec->synchronize();
-    auto time_3 = std::chrono::steady_clock::now() - start_3;
-    std::cout << "time_3(" << iter << ") = "
-              << std::chrono::duration_cast<double_seconds>(time_3).count()
-              << '\n';
-#endif
     // nrmN = norm(next_krylov_basis)
-#ifdef TIMING
-    exec->synchronize();
-    auto start_4 = std::chrono::steady_clock::now();
-#endif
     components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
-    /*
-    check_arnoldi_norms<default_block_size>
-        <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
-            as_cuda_type(arnoldi_norm->get_const_values()), stride_arnoldi,
-            as_cuda_type(arnoldi_norm->get_const_values() + stride_arnoldi),
-            stride_arnoldi, as_cuda_type(stop_status),
-            as_cuda_type(reorth_status), as_cuda_type(num_reorth->get_data()));
-    */ /* */
     check_arnoldi_norms_new<default_block_size>
         <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
             as_cuda_type(arnoldi_norm->get_values()), stride_arnoldi,
             as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
             iter + 1, as_cuda_accessor(krylov_bases), as_cuda_type(stop_status),
             as_cuda_type(reorth_status), as_cuda_type(num_reorth->get_data()));
-    /* */
-#ifdef TIMING
-    exec->synchronize();
-    auto time_4 = std::chrono::steady_clock::now() - start_4;
-    std::cout << "time_4(" << iter << ") = "
-              << std::chrono::duration_cast<double_seconds>(time_4).count()
-              << '\n';
-#endif
-#ifdef TIMING
-    exec->synchronize();
-    auto start_5 = std::chrono::steady_clock::now();
-#endif
     numReorth = 0;
     exec->get_master()->copy_from(exec.get(), 1, num_reorth->get_const_data(),
                                   &numReorth);
-#ifdef TIMING
-    exec->synchronize();
-    auto time_5 = std::chrono::steady_clock::now() - start_5;
-    std::cout << "time_5(" << iter << ") = "
-              << std::chrono::duration_cast<double_seconds>(time_5).count()
-              << '\n';
-#endif
     // numReorth <= number of next_krylov vector to be reorthogonalization
     for (size_type l = 1; (numReorth > 0) && (l < 3); l++) {
-        // std::cout << "CUDA RESTART " << l << '\n';
         (*num_reorth_steps)++;
         (*num_reorth_vectors) += iter;
         zero_matrix(iter + 1, dim_size[1], stride_buffer,
                     buffer_iter->get_values());
-        /*
-        multidot_kernel_num_iters<<<grid_size, block_size>>>(
-            iter + 1, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_accessor(krylov_bases.to_const()),
-            as_cuda_type(buffer_iter->get_values()), stride_buffer,
-            as_cuda_type(stop_status));
-        */
         if (dim_size[1] > 1) {
             multidot_kernel_num_iters_2<default_dot_dim>
                 <<<grid_size_num_iters_2, block_size>>>(
@@ -504,75 +346,24 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
                 as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
                 as_cuda_type(stop_status), as_cuda_type(reorth_status));
-        /*/
-        update_next_krylov_kernel_num_iters_and_add_1<default_update_dim>
-            <<<dim_size[0] * stride_next_krylov, default_update_dim>>>(
-                iter + 1, dim_size[0], dim_size[1],
-                as_cuda_type(next_krylov_basis->get_values()),
-                stride_next_krylov, as_cuda_accessor(krylov_bases.to_const()),
-                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-                as_cuda_type(buffer_iter->get_const_values()), stride_buffer,
-                as_cuda_type(stop_status), as_cuda_type(reorth_status));
-        // */
         // for i in 1:iter
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
         components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
                                dim_size[1], zero<non_complex>());
-        // components::fill_array(
-        //     exec, arnoldi_norm->get_values() + dim_size[1] * (l - 1),
-        //     dim_size[1], zero<ValueType>());
         components::fill_array(exec,
                                arnoldi_norm->get_values() + 2 * dim_size[1],
                                dim_size[1], zero<non_complex>());
-        /*
-        multidot_kernel<<<grid_size, block_size>>>(
-            0, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_accessor(next_krylov_const_accessor),
-            as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)), 0,
-            as_cuda_type(stop_status));
-        */
-        /*
-        multinorm2_kernel<<<grid_size, block_size>>>(
-            dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov,
-            as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)),
-            as_cuda_type(stop_status));
-        */
-        /*
-     multinorminf_kernel<<<grid_size, block_size>>>(
-         dim_size[0], dim_size[1],
-         as_cuda_type(next_krylov_basis->get_const_values()),
-         stride_next_krylov,
-         as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]), 0,
-         as_cuda_type(stop_status));
-     */ /* */
         multinorm2_inf_kernel<use_scale><<<grid_size, block_size>>>(
             dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
             stride_next_krylov,
             as_cuda_type(arnoldi_norm->get_values() + dim_size[1]),
-            // as_cuda_type(arnoldi_norm->get_values() + dim_size[1] * (l - 1)),
             as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]),
             as_cuda_type(stop_status));
-        /* */
         // nrmN = norm(next_krylov_basis)
         components::fill_array(exec, num_reorth->get_data(), 1,
                                zero<size_type>());
-        /*
-        check_arnoldi_norms<default_block_size>
-            <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
-                as_cuda_type(arnoldi_norm->get_const_values() +
-                             stride_arnoldi * (2 - l)),
-                stride_arnoldi,
-                as_cuda_type(arnoldi_norm->get_const_values() +
-                             stride_arnoldi * (l - 1)),
-                stride_arnoldi, as_cuda_type(stop_status),
-                as_cuda_type(reorth_status),
-                as_cuda_type(num_reorth->get_data()));
-        */ /* */
         check_arnoldi_norms_new<default_block_size>
             <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
                 as_cuda_type(arnoldi_norm->get_values()), stride_arnoldi,
@@ -580,25 +371,12 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                 iter + 1, as_cuda_accessor(krylov_bases),
                 as_cuda_type(stop_status), as_cuda_type(reorth_status),
                 as_cuda_type(num_reorth->get_data()));
-        /* */
         numReorth = 0;
         exec->get_master()->copy_from(exec.get(), 1,
                                       num_reorth->get_const_data(), &numReorth);
         // numReorth <= number of next_krylov vector to be reorthogonalization
     }
 
-#ifdef TIMING
-    exec->synchronize();
-    auto start_6 = std::chrono::steady_clock::now();
-#endif
-    /*
-    update_hessenberg_2_kernel<default_block_size>
-        <<<dim_size[1], default_block_size>>>(
-            iter, dim_size[0], dim_size[1],
-            as_cuda_type(next_krylov_basis->get_const_values()),
-            stride_next_krylov, as_cuda_type(hessenberg_iter->get_values()),
-            stride_hessenberg, as_cuda_type(stop_status));
-    */
     update_krylov_next_krylov_kernel<default_block_size>
         <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
            default_block_size>>>(
@@ -607,18 +385,10 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
             as_cuda_accessor(krylov_bases),
             as_cuda_type(hessenberg_iter->get_const_values()),
             stride_hessenberg, as_cuda_type(stop_status));
-#ifdef TIMING
-    exec->synchronize();
-    auto time_6 = std::chrono::steady_clock::now() - start_6;
-    std::cout << "time_6(" << iter << ") = "
-              << std::chrono::duration_cast<double_seconds>(time_6).count()
-              << '\n';
-#endif
     // next_krylov_basis /= hessenberg(iter, iter + 1)
     // krylov_bases(:, iter + 1) = next_krylov_basis
     // End of arnoldi
 }
-/**/
 
 template <typename ValueType>
 void givens_rotation(std::shared_ptr<const CudaExecutor> exec,

From 7df0c2093269ef914a570ae32888030e2fe9bb91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 14 Sep 2020 10:00:17 +0200
Subject: [PATCH 0390/1008] Add HIP kernels

---
 cuda/solver/gmres_mixed_kernels.cu      |   2 -
 hip/solver/gmres_mixed_kernels.hip.cpp  | 429 ++++++++++++++++++++++--
 hip/test/solver/gmres_mixed_kernels.cpp | 370 ++++++++++++++++++++
 3 files changed, 779 insertions(+), 22 deletions(-)
 create mode 100644 hip/test/solver/gmres_mixed_kernels.cpp

diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/gmres_mixed_kernels.cu
index 063876ce2d1..347d3486b84 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/gmres_mixed_kernels.cu
@@ -45,7 +45,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/base/accessors.hpp"
 #include "core/components/fill_array.hpp"
 #include "cuda/base/config.hpp"
-#include "cuda/base/cublas_bindings.hpp"
 #include "cuda/base/math.hpp"
 #include "cuda/base/types.hpp"
 #include "cuda/components/atomic.cuh"
@@ -230,7 +229,6 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     const auto stride_buffer = buffer_iter->get_stride();
     const auto stride_arnoldi = arnoldi_norm->get_stride();
     const auto dim_size = next_krylov_basis->get_size();
-    auto cublas_handle = exec->get_cublas_handle();
     const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
                          exec->get_num_multiprocessor() * 2);
     const dim3 grid_size_num_iters(
diff --git a/hip/solver/gmres_mixed_kernels.hip.cpp b/hip/solver/gmres_mixed_kernels.hip.cpp
index 4632665128e..8858f4f992a 100644
--- a/hip/solver/gmres_mixed_kernels.hip.cpp
+++ b/hip/solver/gmres_mixed_kernels.hip.cpp
@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "core/components/fill_array.hpp"
 #include "hip/base/config.hip.hpp"
 #include "hip/base/math.hip.hpp"
 #include "hip/base/types.hip.hpp"
@@ -74,30 +75,67 @@ constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
 // Specialization, so the Accessor can use the same function as regular pointers
-template <typename Type1, typename Type2>
-Accessor2d<hip_type<Type1>, hip_type<Type2>> as_hip_accessor(
-    Accessor2d<Type1, Type2> acc)
+template <int dim, typename Type1, typename Type2>
+GKO_INLINE auto as_hip_accessor(
+    const range<accessor::reduced_row_major<dim, Type1, Type2>> &acc)
 {
-    return {as_hip_type(acc.get_storage()), acc.get_stride()};
+    return range<
+        accessor::reduced_row_major<dim, hip_type<Type1>, hip_type<Type2>>>(
+        acc.get_accessor().get_size(),
+        as_hip_type(acc.get_accessor().get_stored_data()),
+        acc.get_accessor().get_stride());
 }
 
-template <typename Type1, typename Type2>
-Accessor2dConst<hip_type<Type1>, hip_type<Type2>> as_hip_accessor(
-    const Accessor2dConst<Type1, Type2> &acc)
+template <int dim, typename Type1, typename Type2, size_type mask>
+GKO_INLINE auto as_hip_accessor(
+    const range<accessor::scaled_reduced_row_major<dim, Type1, Type2, mask>>
+        &acc)
 {
-    return {as_hip_type(acc.get_storage()), acc.get_stride()};
+    return range<accessor::scaled_reduced_row_major<dim, hip_type<Type1>,
+                                                    hip_type<Type2>, mask>>(
+        acc.get_accessor().get_size(),
+        as_hip_type(acc.get_accessor().get_stored_data()),
+        acc.get_accessor().get_storage_stride(),
+        as_hip_type(acc.get_accessor().get_scalar()),
+        acc.get_accessor().get_scalar_stride());
+}
+
+
+template <typename ValueType>
+void zero_matrix(size_type m, size_type n, size_type stride, ValueType *array)
+{
+    const dim3 block_size(default_block_size, 1, 1);
+    const dim3 grid_size(ceildiv(n, block_size.x), 1, 1);
+    hipLaunchKernelGGL(zero_matrix_kernel, grid_size, block_size, 0, 0, m, n,
+                       stride, as_hip_type(array));
 }
 
 
 template <typename ValueType>
 void initialize_1(std::shared_ptr<const HipExecutor> exec,
                   const matrix::Dense<ValueType> *b,
-                  matrix::Dense<ValueType> *b_norm,
+                  matrix::Dense<remove_complex<ValueType>> *b_norm,
                   matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *givens_sin,
                   matrix::Dense<ValueType> *givens_cos,
-                  Array<stopping_status> *stop_status,
-                  size_type krylov_dim) GKO_NOT_IMPLEMENTED;
+                  Array<stopping_status> *stop_status, size_type krylov_dim)
+{
+    const auto num_threads = std::max(b->get_size()[0] * b->get_stride(),
+                                      krylov_dim * b->get_size()[1]);
+    const dim3 grid_dim(ceildiv(num_threads, default_block_size), 1, 1);
+    const dim3 block_dim(default_block_size, 1, 1);
+    constexpr auto block_size = default_block_size;
+
+    b->compute_norm2(b_norm);
+    hipLaunchKernelGGL(
+        initialize_1_kernel<block_size>, grid_dim, block_dim, 0, 0,
+        b->get_size()[0], b->get_size()[1], krylov_dim,
+        as_hip_type(b->get_const_values()), b->get_stride(),
+        as_hip_type(residual->get_values()), residual->get_stride(),
+        as_hip_type(givens_sin->get_values()), givens_sin->get_stride(),
+        as_hip_type(givens_cos->get_values()), givens_cos->get_stride(),
+        as_hip_type(stop_status->get_data()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
@@ -106,37 +144,376 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
 template <typename ValueType, typename Accessor3d>
 void initialize_2(std::shared_ptr<const HipExecutor> exec,
                   const matrix::Dense<ValueType> *residual,
-                  matrix::Dense<ValueType> *residual_norm,
+                  matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
+                  matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
                   Accessor3d krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
-                  Array<size_type> *final_iter_nums,
-                  size_type krylov_dim) GKO_NOT_IMPLEMENTED;
+                  Array<size_type> *final_iter_nums, size_type krylov_dim)
+{
+    const auto num_rows = residual->get_size()[0];
+    const auto num_rhs = residual->get_size()[1];
+    const auto krylov_stride =
+        helper_functions_accessor<Accessor3d>::get_stride(krylov_bases);
+    const dim3 grid_dim_1(
+        ceildiv((krylov_dim + 1) * krylov_stride[0], default_block_size), 1, 1);
+    const dim3 block_dim(default_block_size, 1, 1);
+    constexpr auto block_size = default_block_size;
+
+    hipLaunchKernelGGL(initialize_2_1_kernel<block_size>, grid_dim_1, block_dim,
+                       0, 0, residual->get_size()[0], residual->get_size()[1],
+                       krylov_dim, as_hip_accessor(krylov_bases),
+                       as_hip_type(residual_norm_collection->get_values()),
+                       residual_norm_collection->get_stride());
+    residual->compute_norm2(residual_norm);
+
+    components::fill_array(exec, arnoldi_norm->get_values() + 2 * num_rhs,
+                           num_rhs, zero<remove_complex<ValueType>>());
+    const dim3 grid_size_nrm(ceildiv(num_rhs, default_dot_dim),
+                             exec->get_num_multiprocessor() * 2);
+    const dim3 block_size_nrm(default_dot_dim, default_dot_dim);
+    hipLaunchKernelGGL(
+        multinorminf_kernel_without_stop, grid_size_nrm, block_size_nrm, 0, 0,
+        num_rows, num_rhs, as_hip_type(residual->get_const_values()), num_rhs,
+        as_hip_type(arnoldi_norm->get_values() + 2 * num_rhs), 0);
+
+    const auto stride_arnoldi = arnoldi_norm->get_stride();
+    hipLaunchKernelGGL(
+        set_scale_kernel<default_block_size>,
+        dim3(ceildiv(num_rhs * (krylov_dim + 1), default_block_size)),
+        dim3(default_block_size), 0, 0, num_rhs, krylov_dim + 1,
+        as_hip_type(residual_norm->get_const_values()), num_rhs,
+        as_hip_type(arnoldi_norm->get_const_values() + 2 * stride_arnoldi),
+        stride_arnoldi, as_hip_accessor(krylov_bases));
+
+    const dim3 grid_dim_2(
+        ceildiv(num_rows * krylov_stride[1], default_block_size), 1, 1);
+    hipLaunchKernelGGL(initialize_2_2_kernel<block_size>, grid_dim_2, block_dim,
+                       0, 0, residual->get_size()[0], residual->get_size()[1],
+                       as_hip_type(residual->get_const_values()),
+                       residual->get_stride(),
+                       as_hip_type(residual_norm->get_const_values()),
+                       as_hip_type(residual_norm_collection->get_values()),
+                       as_hip_accessor(krylov_bases),
+                       as_hip_type(next_krylov_basis->get_values()),
+                       next_krylov_basis->get_stride(),
+                       as_hip_type(final_iter_nums->get_data()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
 
 
+template <typename ValueType, typename Accessor3dim>
+void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
+                         matrix::Dense<ValueType> *next_krylov_basis,
+                         Accessor3dim krylov_bases,
+                         matrix::Dense<ValueType> *hessenberg_iter,
+                         matrix::Dense<ValueType> *buffer_iter,
+                         matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+                         size_type iter, const stopping_status *stop_status,
+                         stopping_status *reorth_status,
+                         Array<size_type> *num_reorth, int *num_reorth_steps,
+                         int *num_reorth_vectors)
+{
+    using non_complex = remove_complex<ValueType>;
+    // optimization parameter
+    constexpr int singledot_block_size = default_dot_dim;
+    constexpr bool use_scale =
+        kernels::detail::is_3d_scaled_accessor<Accessor3dim>::value;
+    const auto stride_next_krylov = next_krylov_basis->get_stride();
+    const auto stride_hessenberg = hessenberg_iter->get_stride();
+    const auto stride_buffer = buffer_iter->get_stride();
+    const auto stride_arnoldi = arnoldi_norm->get_stride();
+    const auto dim_size = next_krylov_basis->get_size();
+    const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
+                         exec->get_num_multiprocessor() * 2);
+    const dim3 grid_size_num_iters(
+        ceildiv(dim_size[1] * (iter + 1), default_dot_dim),
+        exec->get_num_multiprocessor() * 2);
+    const dim3 grid_size_num_iters_2(ceildiv(dim_size[1], default_dot_dim),
+                                     exec->get_num_multiprocessor() * 2,
+                                     iter + 1);
+    const dim3 block_size(default_dot_dim, default_dot_dim);
+    // Note: having iter first (instead of row_idx information) is likely
+    //       beneficial for avoiding atomic_add conflicts, but that needs
+    //       further investigation.
+    const dim3 grid_size_iters_single(exec->get_num_multiprocessor() * 2,
+                                      iter + 1);
+    const dim3 block_size_iters_single(singledot_block_size);
+    size_type numReorth;
+
+    components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
+                           zero<non_complex>());
+    hipLaunchKernelGGL(
+        multinorm2_kernel, grid_size, block_size, 0, 0, dim_size[0],
+        dim_size[1], as_hip_type(next_krylov_basis->get_const_values()),
+        stride_next_krylov, as_hip_type(arnoldi_norm->get_values()),
+        as_hip_type(stop_status));
+    // nrmP = norm(next_krylov_basis
+    zero_matrix(iter + 1, dim_size[1], stride_hessenberg,
+                hessenberg_iter->get_values());
+    if (dim_size[1] > 1) {
+        hipLaunchKernelGGL(multidot_kernel_num_iters_2<default_dot_dim>,
+                           grid_size_num_iters_2, block_size, 0, 0, iter + 1,
+                           dim_size[0], dim_size[1],
+                           as_hip_type(next_krylov_basis->get_const_values()),
+                           stride_next_krylov, as_hip_accessor(krylov_bases),
+                           as_hip_type(hessenberg_iter->get_values()),
+                           stride_hessenberg, as_hip_type(stop_status));
+    } else {
+        hipLaunchKernelGGL(singledot_kernel_num_iters_2<singledot_block_size>,
+                           grid_size_iters_single, block_size_iters_single, 0,
+                           0, iter + 1, dim_size[0],
+                           as_hip_type(next_krylov_basis->get_const_values()),
+                           stride_next_krylov, as_hip_accessor(krylov_bases),
+                           as_hip_type(hessenberg_iter->get_values()),
+                           stride_hessenberg, as_hip_type(stop_status));
+    }
+    // for i in 1:iter
+    //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+    // end
+    hipLaunchKernelGGL(
+        update_next_krylov_kernel_num_iters<default_block_size>,
+        dim3(ceildiv(dim_size[0] * stride_next_krylov, default_block_size)),
+        dim3(default_block_size), 0, 0, iter + 1, dim_size[0], dim_size[1],
+        as_hip_type(next_krylov_basis->get_values()), stride_next_krylov,
+        as_hip_accessor(krylov_bases),
+        as_hip_type(hessenberg_iter->get_const_values()), stride_hessenberg,
+        as_hip_type(stop_status));
+
+    // for i in 1:iter
+    //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+    // end
+    components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
+                           dim_size[1], zero<non_complex>());
+    components::fill_array(exec, arnoldi_norm->get_values() + 2 * dim_size[1],
+                           dim_size[1], zero<non_complex>());
+    hipLaunchKernelGGL(
+        multinorm2_inf_kernel<use_scale>, grid_size, block_size, 0, 0,
+        dim_size[0], dim_size[1],
+        as_hip_type(next_krylov_basis->get_const_values()), stride_next_krylov,
+        as_hip_type(arnoldi_norm->get_values() + dim_size[1]),
+        as_hip_type(arnoldi_norm->get_values() + 2 * dim_size[1]),
+        as_hip_type(stop_status));
+    // nrmN = norm(next_krylov_basis)
+    components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
+    hipLaunchKernelGGL(
+        check_arnoldi_norms_new<default_block_size>,
+        dim3(ceildiv(dim_size[1], default_block_size)),
+        dim3(default_block_size), 0, 0, as_hip_type(arnoldi_norm->get_values()),
+        stride_arnoldi, as_hip_type(hessenberg_iter->get_values()),
+        stride_hessenberg, iter + 1, as_hip_accessor(krylov_bases),
+        as_hip_type(stop_status), as_hip_type(reorth_status),
+        as_hip_type(num_reorth->get_data()));
+    numReorth = 0;
+    exec->get_master()->copy_from(exec.get(), 1, num_reorth->get_const_data(),
+                                  &numReorth);
+    // numReorth <= number of next_krylov vector to be reorthogonalization
+    for (size_type l = 1; (numReorth > 0) && (l < 3); l++) {
+        (*num_reorth_steps)++;
+        (*num_reorth_vectors) += iter;
+        zero_matrix(iter + 1, dim_size[1], stride_buffer,
+                    buffer_iter->get_values());
+        if (dim_size[1] > 1) {
+            hipLaunchKernelGGL(
+                multidot_kernel_num_iters_2<default_dot_dim>,
+                grid_size_num_iters_2, block_size, 0, 0, iter + 1, dim_size[0],
+                dim_size[1], as_hip_type(next_krylov_basis->get_const_values()),
+                stride_next_krylov, as_hip_accessor(krylov_bases),
+                as_hip_type(buffer_iter->get_values()), stride_buffer,
+                as_hip_type(stop_status));
+        } else {
+            hipLaunchKernelGGL(
+                singledot_kernel_num_iters_2<singledot_block_size>,
+                grid_size_iters_single, block_size_iters_single, 0, 0, iter + 1,
+                dim_size[0], as_hip_type(next_krylov_basis->get_const_values()),
+                stride_next_krylov, as_hip_accessor(krylov_bases),
+                as_hip_type(buffer_iter->get_values()), stride_buffer,
+                as_hip_type(stop_status));
+        }
+        // for i in 1:iter
+        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+        // end
+        //*
+        hipLaunchKernelGGL(
+            update_next_krylov_kernel_num_iters_and_add<default_block_size>,
+            dim3(ceildiv(dim_size[0] * stride_next_krylov, default_block_size)),
+            dim3(default_block_size), 0, 0, iter + 1, dim_size[0], dim_size[1],
+            as_hip_type(next_krylov_basis->get_values()), stride_next_krylov,
+            as_hip_accessor(krylov_bases),
+            as_hip_type(hessenberg_iter->get_values()), stride_hessenberg,
+            as_hip_type(buffer_iter->get_const_values()), stride_buffer,
+            as_hip_type(stop_status), as_hip_type(reorth_status));
+        // for i in 1:iter
+        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+        // end
+        components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
+                               dim_size[1], zero<non_complex>());
+        components::fill_array(exec,
+                               arnoldi_norm->get_values() + 2 * dim_size[1],
+                               dim_size[1], zero<non_complex>());
+        hipLaunchKernelGGL(
+            multinorm2_inf_kernel<use_scale>, grid_size, block_size, 0, 0,
+            dim_size[0], dim_size[1],
+            as_hip_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov,
+            as_hip_type(arnoldi_norm->get_values() + dim_size[1]),
+            as_hip_type(arnoldi_norm->get_values() + 2 * dim_size[1]),
+            as_hip_type(stop_status));
+        // nrmN = norm(next_krylov_basis)
+        components::fill_array(exec, num_reorth->get_data(), 1,
+                               zero<size_type>());
+        hipLaunchKernelGGL(
+            check_arnoldi_norms_new<default_block_size>,
+            dim3(ceildiv(dim_size[1], default_block_size)),
+            dim3(default_block_size), 0, 0,
+            as_hip_type(arnoldi_norm->get_values()), stride_arnoldi,
+            as_hip_type(hessenberg_iter->get_values()), stride_hessenberg,
+            iter + 1, as_hip_accessor(krylov_bases), as_hip_type(stop_status),
+            as_hip_type(reorth_status), as_hip_type(num_reorth->get_data()));
+        numReorth = 0;
+        exec->get_master()->copy_from(exec.get(), 1,
+                                      num_reorth->get_const_data(), &numReorth);
+        // numReorth <= number of next_krylov vector to be reorthogonalization
+    }
+
+    hipLaunchKernelGGL(
+        update_krylov_next_krylov_kernel<default_block_size>,
+        dim3(ceildiv(dim_size[0] * stride_next_krylov, default_block_size)),
+        dim3(default_block_size), 0, 0, iter, dim_size[0], dim_size[1],
+        as_hip_type(next_krylov_basis->get_values()), stride_next_krylov,
+        as_hip_accessor(krylov_bases),
+        as_hip_type(hessenberg_iter->get_const_values()), stride_hessenberg,
+        as_hip_type(stop_status));
+    // next_krylov_basis /= hessenberg(iter, iter + 1)
+    // krylov_bases(:, iter + 1) = next_krylov_basis
+    // End of arnoldi
+}
+
+template <typename ValueType>
+void givens_rotation(std::shared_ptr<const HipExecutor> exec,
+                     matrix::Dense<ValueType> *givens_sin,
+                     matrix::Dense<ValueType> *givens_cos,
+                     matrix::Dense<ValueType> *hessenberg_iter,
+                     matrix::Dense<remove_complex<ValueType>> *residual_norm,
+                     matrix::Dense<ValueType> *residual_norm_collection,
+                     const matrix::Dense<remove_complex<ValueType>> *b_norm,
+                     size_type iter, const Array<stopping_status> *stop_status)
+{
+    // TODO: tune block_size for optimal performance
+    constexpr auto block_size = default_block_size;
+    const auto num_cols = hessenberg_iter->get_size()[1];
+    const dim3 block_dim{block_size, 1, 1};
+    const dim3 grid_dim{
+        static_cast<unsigned int>(ceildiv(num_cols, block_size)), 1, 1};
+
+    hipLaunchKernelGGL(
+        givens_rotation_kernel<block_size>, grid_dim, block_dim, 0, 0,
+        hessenberg_iter->get_size()[0], hessenberg_iter->get_size()[1], iter,
+        as_hip_type(hessenberg_iter->get_values()),
+        hessenberg_iter->get_stride(), as_hip_type(givens_sin->get_values()),
+        givens_sin->get_stride(), as_hip_type(givens_cos->get_values()),
+        givens_cos->get_stride(), as_hip_type(residual_norm->get_values()),
+        as_hip_type(residual_norm_collection->get_values()),
+        residual_norm_collection->get_stride(),
+        as_hip_type(b_norm->get_const_values()),
+        as_hip_type(stop_status->get_const_data()));
+}
+
+
 template <typename ValueType, typename Accessor3d>
 void step_1(std::shared_ptr<const HipExecutor> exec,
             matrix::Dense<ValueType> *next_krylov_basis,
             matrix::Dense<ValueType> *givens_sin,
             matrix::Dense<ValueType> *givens_cos,
-            matrix::Dense<ValueType> *residual_norm,
+            matrix::Dense<remove_complex<ValueType>> *residual_norm,
             matrix::Dense<ValueType> *residual_norm_collection,
             Accessor3d krylov_bases, matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
-            const matrix::Dense<ValueType> *b_norm,
-            matrix::Dense<ValueType> *arnoldi_norm, size_type iter,
-            Array<size_type> *final_iter_nums,
+            const matrix::Dense<remove_complex<ValueType>> *b_norm,
+            matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+            size_type iter, Array<size_type> *final_iter_nums,
             const Array<stopping_status> *stop_status,
             Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
-            int *num_reorth_steps, int *num_reorth_vectors) GKO_NOT_IMPLEMENTED;
+            int *num_reorth_steps, int *num_reorth_vectors)
+{
+    hipLaunchKernelGGL(
+        increase_final_iteration_numbers_kernel,
+        dim3(static_cast<unsigned int>(
+            ceildiv(final_iter_nums->get_num_elems(), default_block_size))),
+        dim3(default_block_size), 0, 0,
+        as_hip_type(final_iter_nums->get_data()),
+        as_hip_type(stop_status->get_const_data()),
+        final_iter_nums->get_num_elems());
+    finish_arnoldi_CGS2(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
+                        buffer_iter, arnoldi_norm, iter,
+                        stop_status->get_const_data(),
+                        reorth_status->get_data(), num_reorth, num_reorth_steps,
+                        num_reorth_vectors);
+    givens_rotation(exec, givens_sin, givens_cos, hessenberg_iter,
+                    residual_norm, residual_norm_collection, b_norm, iter,
+                    stop_status);
+}
 
 GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
 
 
+template <typename ValueType>
+void solve_upper_triangular(
+    const matrix::Dense<ValueType> *residual_norm_collection,
+    const matrix::Dense<ValueType> *hessenberg, matrix::Dense<ValueType> *y,
+    const Array<size_type> *final_iter_nums)
+{
+    // TODO: tune block_size for optimal performance
+    constexpr auto block_size = default_block_size;
+    const auto num_rhs = residual_norm_collection->get_size()[1];
+    const dim3 block_dim{block_size, 1, 1};
+    const dim3 grid_dim{static_cast<unsigned int>(ceildiv(num_rhs, block_size)),
+                        1, 1};
+
+    hipLaunchKernelGGL(
+        solve_upper_triangular_kernel<block_size>, grid_dim, block_dim, 0, 0,
+        hessenberg->get_size()[1], num_rhs,
+        as_hip_type(residual_norm_collection->get_const_values()),
+        residual_norm_collection->get_stride(),
+        as_hip_type(hessenberg->get_const_values()), hessenberg->get_stride(),
+        as_hip_type(y->get_values()), y->get_stride(),
+        as_hip_type(final_iter_nums->get_const_data()));
+}
+
+
+template <typename ValueType, typename ConstAccessor3d>
+void calculate_qy(ConstAccessor3d krylov_bases, size_type num_krylov_bases,
+                  const matrix::Dense<ValueType> *y,
+                  matrix::Dense<ValueType> *before_preconditioner,
+                  const Array<size_type> *final_iter_nums)
+{
+    const auto num_rows = before_preconditioner->get_size()[0];
+    const auto num_cols = before_preconditioner->get_size()[1];
+    const auto stride_before_preconditioner =
+        before_preconditioner->get_stride();
+
+    constexpr auto block_size = default_block_size;
+    const dim3 grid_dim{
+        static_cast<unsigned int>(
+            ceildiv(num_rows * stride_before_preconditioner, block_size)),
+        1, 1};
+    const dim3 block_dim{block_size, 1, 1};
+
+
+    hipLaunchKernelGGL(calculate_Qy_kernel<block_size>, grid_dim, block_dim, 0,
+                       0, num_rows, num_cols, as_hip_accessor(krylov_bases),
+                       as_hip_type(y->get_const_values()), y->get_stride(),
+                       as_hip_type(before_preconditioner->get_values()),
+                       stride_before_preconditioner,
+                       as_hip_type(final_iter_nums->get_const_data()));
+    // Calculate qy
+    // before_preconditioner = krylov_bases * y
+}
+
+
 template <typename ValueType, typename ConstAccessor3d>
 void step_2(std::shared_ptr<const HipExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
@@ -144,7 +521,19 @@ void step_2(std::shared_ptr<const HipExecutor> exec,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
-            const Array<size_type> *final_iter_nums) GKO_NOT_IMPLEMENTED;
+            const Array<size_type> *final_iter_nums)
+{
+    // since hessenberg has dims:  iters x iters * num_rhs
+    // krylov_bases has dims:  (iters + 1) x sysmtx[0] x num_rhs
+    const auto iters =
+        hessenberg->get_size()[1] / before_preconditioner->get_size()[1];
+    const auto num_krylov_bases = iters + 1;
+    solve_upper_triangular(residual_norm_collection, hessenberg, y,
+                           final_iter_nums);
+    calculate_qy(krylov_bases, num_krylov_bases, y, before_preconditioner,
+                 final_iter_nums);
+}
+
 
 GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(
     GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
diff --git a/hip/test/solver/gmres_mixed_kernels.cpp b/hip/test/solver/gmres_mixed_kernels.cpp
new file mode 100644
index 00000000000..4247ebe26ee
--- /dev/null
+++ b/hip/test/solver/gmres_mixed_kernels.cpp
@@ -0,0 +1,370 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/solver/gmres_mixed.hpp>
+
+
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm_reduction.hpp>
+
+
+#include "core/solver/gmres_mixed_accessor.hpp"
+#include "core/solver/gmres_mixed_kernels.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class GmresMixed : public ::testing::Test {
+protected:
+    using value_type = double;
+    using storage_type = float;
+    using index_type = int;
+    using size_type = gko::size_type;
+    using Range3dHelper =
+        gko::kernels::Accessor3dHelper<value_type, storage_type>;
+    using Range3d = typename Range3dHelper::Range;
+    using Dense = gko::matrix::Dense<value_type>;
+    using Mtx = Dense;
+
+    GmresMixed() : rand_engine(30) {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::HipExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        hip = gko::HipExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (hip != nullptr) {
+            ASSERT_NO_THROW(hip->synchronize());
+        }
+    }
+
+    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<Mtx>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<index_type>(num_cols, num_cols),
+            std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    Range3dHelper generate_krylov_helper(gko::dim<3> size)
+    {
+        auto helper = Range3dHelper{ref, size};
+        auto &bases = helper.get_bases();
+        const auto num_rows = size[0] * size[1];
+        const auto num_cols = size[2];
+        auto temp_krylov_bases = gko::test::generate_random_matrix<Dense>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<index_type>(num_cols, num_cols),
+            std::normal_distribution<storage_type>(-1.0, 1.0), rand_engine,
+            ref);
+        std::copy_n(temp_krylov_bases->get_const_values(),
+                    bases.get_num_elems(), bases.get_data());
+        // Only useful when the Accessor actually has a scale
+        auto range = helper.get_range();
+        auto dist = std::normal_distribution<value_type>(-1, 1);
+        for (size_type k = 0; k < size[0]; ++k) {
+            for (size_type i = 0; i < size[2]; ++i) {
+                gko::kernels::helper_functions_accessor<Range3d>::write_scalar(
+                    range, k, i, dist(rand_engine));
+            }
+        }
+        return helper;
+    }
+
+    void initialize_data()
+    {
+        int m = 597;
+        int n = 43;
+        x = gen_mtx(m, n);
+        y = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        before_preconditioner = Mtx::create_with_config_of(x.get());
+        b = gen_mtx(m, n);
+        b_norm = gen_mtx(1, n);
+        arnoldi_norm = gen_mtx(3, n);
+        gko::dim<3> krylov_bases_dim(gko::solver::default_krylov_dim_mixed + 1,
+                                     m, n);
+        range_helper = generate_krylov_helper(krylov_bases_dim);
+
+        next_krylov_basis = gen_mtx(m, n);
+        hessenberg = gen_mtx(gko::solver::default_krylov_dim_mixed + 1,
+                             gko::solver::default_krylov_dim_mixed * n);
+        hessenberg_iter = gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
+        buffer_iter = gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
+        residual = gen_mtx(m, n);
+        residual_norm = gen_mtx(1, n);
+        residual_norm_collection =
+            gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
+        givens_sin = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        givens_cos = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(ref, n));
+        for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
+            stop_status->get_data()[i].reset();
+        }
+        reorth_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(ref, n));
+        for (size_t i = 0; i < reorth_status->get_num_elems(); ++i) {
+            reorth_status->get_data()[i].reset();
+        }
+        final_iter_nums = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(ref, n));
+        for (size_t i = 0; i < final_iter_nums->get_num_elems(); ++i) {
+            final_iter_nums->get_data()[i] = 5;
+        }
+        num_reorth = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(ref, n));
+        for (size_t i = 0; i < num_reorth->get_num_elems(); ++i) {
+            num_reorth->get_data()[i] = 5;
+        }
+
+        d_x = Mtx::create(hip);
+        d_x->copy_from(x.get());
+        d_before_preconditioner = Mtx::create_with_config_of(d_x.get());
+        d_y = Mtx::create(hip);
+        d_y->copy_from(y.get());
+        d_b = Mtx::create(hip);
+        d_b->copy_from(b.get());
+        d_b_norm = Mtx::create(hip);
+        d_b_norm->copy_from(b_norm.get());
+        d_arnoldi_norm = Mtx::create(hip);
+        d_arnoldi_norm->copy_from(arnoldi_norm.get());
+        d_range_helper = Range3dHelper{hip, {}};
+        d_range_helper = range_helper;
+        d_next_krylov_basis = Mtx::create(hip);
+        d_next_krylov_basis->copy_from(next_krylov_basis.get());
+        d_hessenberg = Mtx::create(hip);
+        d_hessenberg->copy_from(hessenberg.get());
+        d_hessenberg_iter = Mtx::create(hip);
+        d_hessenberg_iter->copy_from(hessenberg_iter.get());
+        d_buffer_iter = Mtx::create(hip);
+        d_buffer_iter->copy_from(buffer_iter.get());
+        d_residual = Mtx::create(hip);
+        d_residual->copy_from(residual.get());
+        d_residual_norm = Mtx::create(hip);
+        d_residual_norm->copy_from(residual_norm.get());
+        d_residual_norm_collection = Mtx::create(hip);
+        d_residual_norm_collection->copy_from(residual_norm_collection.get());
+        d_givens_sin = Mtx::create(hip);
+        d_givens_sin->copy_from(givens_sin.get());
+        d_givens_cos = Mtx::create(hip);
+        d_givens_cos->copy_from(givens_cos.get());
+        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(hip, n));
+        *d_stop_status = *stop_status;
+        d_reorth_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(hip, n));
+        *d_reorth_status = *reorth_status;
+        d_final_iter_nums = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(hip, n));
+        *d_final_iter_nums = *final_iter_nums;
+        d_num_reorth = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(hip, n));
+        *d_num_reorth = *num_reorth;
+    }
+
+    void assert_krylov_bases_near()
+    {
+        gko::Array<storage_type> d_to_host{ref};
+        auto &krylov_bases = range_helper.get_bases();
+        d_to_host = d_range_helper.get_bases();
+        const auto tolerance = r<storage_type>::value;
+        using std::abs;
+        for (gko::size_type i = 0; i < krylov_bases.get_num_elems(); ++i) {
+            const auto ref_value = krylov_bases.get_const_data()[i];
+            const auto dev_value = d_to_host.get_const_data()[i];
+            ASSERT_LE(abs(dev_value - ref_value), tolerance);
+        }
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::HipExecutor> hip;
+
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Mtx> before_preconditioner;
+    std::unique_ptr<Mtx> x;
+    std::unique_ptr<Mtx> y;
+    std::unique_ptr<Mtx> b;
+    std::unique_ptr<Mtx> b_norm;
+    std::unique_ptr<Mtx> arnoldi_norm;
+    Range3dHelper range_helper;
+    std::unique_ptr<Mtx> next_krylov_basis;
+    std::unique_ptr<Mtx> hessenberg;
+    std::unique_ptr<Mtx> hessenberg_iter;
+    std::unique_ptr<Mtx> buffer_iter;
+    std::unique_ptr<Mtx> residual;
+    std::unique_ptr<Mtx> residual_norm;
+    std::unique_ptr<Mtx> residual_norm_collection;
+    std::unique_ptr<Mtx> givens_sin;
+    std::unique_ptr<Mtx> givens_cos;
+    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
+    std::unique_ptr<gko::Array<gko::stopping_status>> reorth_status;
+    std::unique_ptr<gko::Array<gko::size_type>> final_iter_nums;
+    std::unique_ptr<gko::Array<gko::size_type>> num_reorth;
+
+    std::unique_ptr<Mtx> d_x;
+    std::unique_ptr<Mtx> d_before_preconditioner;
+    std::unique_ptr<Mtx> d_y;
+    std::unique_ptr<Mtx> d_b;
+    std::unique_ptr<Mtx> d_b_norm;
+    std::unique_ptr<Mtx> d_arnoldi_norm;
+    Range3dHelper d_range_helper;
+    std::unique_ptr<Mtx> d_next_krylov_basis;
+    std::unique_ptr<Mtx> d_hessenberg;
+    std::unique_ptr<Mtx> d_hessenberg_iter;
+    std::unique_ptr<Mtx> d_buffer_iter;
+    std::unique_ptr<Mtx> d_residual;
+    std::unique_ptr<Mtx> d_residual_norm;
+    std::unique_ptr<Mtx> d_residual_norm_collection;
+    std::unique_ptr<Mtx> d_givens_sin;
+    std::unique_ptr<Mtx> d_givens_cos;
+    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
+    std::unique_ptr<gko::Array<gko::stopping_status>> d_reorth_status;
+    std::unique_ptr<gko::Array<gko::size_type>> d_final_iter_nums;
+    std::unique_ptr<gko::Array<gko::size_type>> d_num_reorth;
+};
+
+
+TEST_F(GmresMixed, HipGmresMixedInitialize1IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::gmres_mixed::initialize_1(
+        ref, b.get(), b_norm.get(), residual.get(), givens_sin.get(),
+        givens_cos.get(), stop_status.get(),
+        gko::solver::default_krylov_dim_mixed);
+    gko::kernels::hip::gmres_mixed::initialize_1(
+        hip, d_b.get(), d_b_norm.get(), d_residual.get(), d_givens_sin.get(),
+        d_givens_cos.get(), d_stop_status.get(),
+        gko::solver::default_krylov_dim_mixed);
+
+    GKO_ASSERT_MTX_NEAR(d_b_norm, b_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual, residual, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
+    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
+}
+
+
+TEST_F(GmresMixed, HipGmresMixedInitialize2IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::gmres_mixed::initialize_2(
+        ref, residual.get(), residual_norm.get(),
+        residual_norm_collection.get(), arnoldi_norm.get(),
+        range_helper.get_range(), next_krylov_basis.get(),
+        final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
+    gko::kernels::hip::gmres_mixed::initialize_2(
+        hip, d_residual.get(), d_residual_norm.get(),
+        d_residual_norm_collection.get(), d_arnoldi_norm.get(),
+        d_range_helper.get_range(), d_next_krylov_basis.get(),
+        d_final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
+
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
+                        1e-14);
+    assert_krylov_bases_near();
+    GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
+}
+
+
+TEST_F(GmresMixed, HipGmresMixedStep1IsEquivalentToRef)
+{
+    initialize_data();
+    int iter = 5;
+    int num_reorth_steps = 0, num_reorth_vectors = 0;
+    int d_num_reorth_steps = 0, d_num_reorth_vectors = 0;
+
+    gko::kernels::reference::gmres_mixed::step_1(
+        ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
+        residual_norm.get(), residual_norm_collection.get(),
+        range_helper.get_range(), hessenberg_iter.get(), buffer_iter.get(),
+        b_norm.get(), arnoldi_norm.get(), iter, final_iter_nums.get(),
+        stop_status.get(), reorth_status.get(), num_reorth.get(),
+        &num_reorth_steps, &num_reorth_vectors);
+    gko::kernels::hip::gmres_mixed::step_1(
+        hip, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
+        d_residual_norm.get(), d_residual_norm_collection.get(),
+        d_range_helper.get_range(), d_hessenberg_iter.get(),
+        d_buffer_iter.get(), d_b_norm.get(), d_arnoldi_norm.get(), iter,
+        d_final_iter_nums.get(), d_stop_status.get(), d_reorth_status.get(),
+        d_num_reorth.get(), &num_reorth_steps, &num_reorth_vectors);
+
+    GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter, 1e-14);
+    assert_krylov_bases_near();
+    GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
+}
+
+
+TEST_F(GmresMixed, HipGmresMixedStep2IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::gmres_mixed::step_2(
+        ref, residual_norm_collection.get(),
+        range_helper.get_range().get_accessor().to_const(), hessenberg.get(),
+        y.get(), before_preconditioner.get(), final_iter_nums.get());
+    gko::kernels::hip::gmres_mixed::step_2(
+        hip, d_residual_norm_collection.get(),
+        d_range_helper.get_range().get_accessor().to_const(),
+        d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
+        d_final_iter_nums.get());
+
+    GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+}
+
+
+}  // namespace

From dec3d654557aa085f6842c35a5114d75b2922a4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 17 Sep 2020 11:18:41 +0200
Subject: [PATCH 0391/1008] Fix GmresMixed core problem

---
 core/solver/gmres_mixed.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index e95a4add464..702ace8451f 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -443,8 +443,8 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                 // Solve upper triangular.
                 // y = hessenberg \ residual_norm_collection
 
-                get_preconditioner()->apply(before_preconditioner.get(),
-                                            after_preconditioner.get());
+                this->get_preconditioner()->apply(before_preconditioner.get(),
+                                                  after_preconditioner.get());
                 dense_x->add_scaled(one_op.get(), after_preconditioner.get());
                 // Solve x
                 // x = x + get_preconditioner() * krylov_bases * y
@@ -470,8 +470,8 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 #endif
             }
 
-            get_preconditioner()->apply(next_krylov_basis.get(),
-                                        preconditioned_vector.get());
+            this->get_preconditioner()->apply(next_krylov_basis.get(),
+                                              preconditioned_vector.get());
             // preconditioned_vector = get_preconditioner() *
             // next_krylov_basis
 
@@ -566,8 +566,8 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         exec->synchronize();
         auto t_aux_4 = std::chrono::steady_clock::now();
 #endif
-        get_preconditioner()->apply(before_preconditioner.get(),
-                                    after_preconditioner.get());
+        this->get_preconditioner()->apply(before_preconditioner.get(),
+                                          after_preconditioner.get());
         dense_x->add_scaled(one_op.get(), after_preconditioner.get());
 #ifdef TIMING_STEPS
         exec->synchronize();

From b611908ef666ec038da896f147df94ca45ef3d5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Sat, 12 Dec 2020 09:34:54 +0100
Subject: [PATCH 0392/1008] Improve force-reset behavior

---
 core/solver/gmres_mixed.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/core/solver/gmres_mixed.cpp b/core/solver/gmres_mixed.cpp
index 702ace8451f..18e88700836 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/gmres_mixed.cpp
@@ -347,6 +347,8 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                                          dense_b->get_size()[1]);
         Array<bool> fully_converged_rhs(exec->get_master(),
                                         dense_b->get_size()[1]);
+        Array<stopping_status> host_stop_status(
+            this->get_executor()->get_master(), stop_status);
         for (size_type i = 0; i < stop_encountered_rhs.get_num_elems(); ++i) {
             stop_encountered_rhs.get_data()[i] = false;
             fully_converged_rhs.get_data()[i] = false;
@@ -383,8 +385,7 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                     <<
                     '\n';
                     */
-                    Array<stopping_status> host_stop_status(
-                        this->get_executor()->get_master(), stop_status);
+                    host_stop_status = stop_status;
                     bool host_array_changed{false};
                     for (size_type i = 0; i < host_stop_status.get_num_elems();
                          ++i) {
@@ -412,6 +413,7 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                         // (like time or iteration)
                         break;
                     }
+                    forced_iterations = 0;
 
                 } else {
                     for (size_type i = 0;
@@ -419,7 +421,6 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                         stop_encountered_rhs.get_data()[i] = false;
                     }
                 }
-                forced_iterations = 0;
             }
 
             if (perform_reset || restart_iter == krylov_dim_) {

From 64f107cdb3bdaf9609a368d3eca861a441446e37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Sat, 23 Jan 2021 00:59:31 +0100
Subject: [PATCH 0393/1008] Rename GmresMixed to CbGmres

---
 benchmark/solver/solver.cpp                   |  36 +++----
 ...rnels.hpp.inc => cb_gmres_kernels.hpp.inc} |   2 +-
 core/CMakeLists.txt                           |   2 +-
 core/device_hooks/common_kernels.inc.cpp      |  32 +++---
 core/solver/{gmres_mixed.cpp => cb_gmres.cpp} |  68 ++++++------
 ...xed_accessor.hpp => cb_gmres_accessor.hpp} |   8 +-
 ...mixed_kernels.hpp => cb_gmres_kernels.hpp} |  66 ++++++------
 core/test/solver/CMakeLists.txt               |   2 +-
 .../solver/{gmres_mixed.cpp => cb_gmres.cpp}  | 102 +++++++++---------
 cuda/CMakeLists.txt                           |   2 +-
 ...s_mixed_kernels.cu => cb_gmres_kernels.cu} |  37 +++----
 cuda/test/solver/CMakeLists.txt               |   2 +-
 ...mixed_kernels.cpp => cb_gmres_kernels.cpp} |  36 +++----
 dpcpp/CMakeLists.txt                          |   2 +-
 examples/CMakeLists.txt                       |   2 +-
 examples/cb-gmres-solver/CMakeLists.txt       |  13 +++
 .../build.sh                                  |   0
 .../cb-gmres-solver-jacobi-matrix.cpp}        |  12 +--
 .../cb-gmres-solver-jacobi.cpp}               |  10 +-
 .../cb-gmres-solver.cpp}                      |  10 +-
 .../data/A.mtx                                |   0
 .../data/b.mtx                                |   0
 .../data/x0.mtx                               |   0
 .../doc/builds-on                             |   0
 .../doc/intro.dox                             |   0
 .../doc/kind                                  |   0
 .../doc/results.dox                           |   0
 .../doc/short-intro                           |   0
 .../doc/tooltip                               |   0
 examples/gmres-mixed-solver/CMakeLists.txt    |  13 ---
 examples/gmres-solver/gmres-solver.cpp        |   2 +-
 hip/CMakeLists.txt                            |   2 +-
 ...rnels.hip.cpp => cb_gmres_kernels.hip.cpp} |  28 +++--
 hip/test/solver/CMakeLists.txt                |   2 +-
 ...mixed_kernels.cpp => cb_gmres_kernels.cpp} |  36 +++----
 .../solver/{gmres_mixed.hpp => cb_gmres.hpp}  |  42 ++++----
 include/ginkgo/ginkgo.hpp                     |   2 +-
 omp/CMakeLists.txt                            |   2 +-
 ...mixed_kernels.cpp => cb_gmres_kernels.cpp} |  28 +++--
 omp/test/solver/CMakeLists.txt                |   2 +-
 ...mixed_kernels.cpp => cb_gmres_kernels.cpp} |  36 +++----
 reference/CMakeLists.txt                      |   2 +-
 ...mixed_kernels.cpp => cb_gmres_kernels.cpp} |  30 +++---
 reference/test/solver/CMakeLists.txt          |   2 +-
 ...mixed_kernels.cpp => cb_gmres_kernels.cpp} |  78 +++++++-------
 45 files changed, 372 insertions(+), 379 deletions(-)
 rename common/solver/{gmres_mixed_kernels.hpp.inc => cb_gmres_kernels.hpp.inc} (99%)
 rename core/solver/{gmres_mixed.cpp => cb_gmres.cpp} (92%)
 rename core/solver/{gmres_mixed_accessor.hpp => cb_gmres_accessor.hpp} (97%)
 rename core/solver/{gmres_mixed_kernels.hpp => cb_gmres_kernels.hpp} (91%)
 rename core/test/solver/{gmres_mixed.cpp => cb_gmres.cpp} (72%)
 rename cuda/solver/{gmres_mixed_kernels.cu => cb_gmres_kernels.cu} (96%)
 rename cuda/test/solver/{gmres_mixed_kernels.cpp => cb_gmres_kernels.cpp} (94%)
 create mode 100644 examples/cb-gmres-solver/CMakeLists.txt
 rename examples/{gmres-mixed-solver => cb-gmres-solver}/build.sh (100%)
 rename examples/{gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp => cb-gmres-solver/cb-gmres-solver-jacobi-matrix.cpp} (97%)
 rename examples/{gmres-mixed-solver/gmres-mixed-solver-jacobi.cpp => cb-gmres-solver/cb-gmres-solver-jacobi.cpp} (97%)
 rename examples/{gmres-mixed-solver/gmres-mixed-solver.cpp => cb-gmres-solver/cb-gmres-solver.cpp} (97%)
 rename examples/{gmres-mixed-solver => cb-gmres-solver}/data/A.mtx (100%)
 rename examples/{gmres-mixed-solver => cb-gmres-solver}/data/b.mtx (100%)
 rename examples/{gmres-mixed-solver => cb-gmres-solver}/data/x0.mtx (100%)
 rename examples/{gmres-mixed-solver => cb-gmres-solver}/doc/builds-on (100%)
 rename examples/{gmres-mixed-solver => cb-gmres-solver}/doc/intro.dox (100%)
 rename examples/{gmres-mixed-solver => cb-gmres-solver}/doc/kind (100%)
 rename examples/{gmres-mixed-solver => cb-gmres-solver}/doc/results.dox (100%)
 rename examples/{gmres-mixed-solver => cb-gmres-solver}/doc/short-intro (100%)
 rename examples/{gmres-mixed-solver => cb-gmres-solver}/doc/tooltip (100%)
 delete mode 100644 examples/gmres-mixed-solver/CMakeLists.txt
 rename hip/solver/{gmres_mixed_kernels.hip.cpp => cb_gmres_kernels.hip.cpp} (97%)
 rename hip/test/solver/{gmres_mixed_kernels.cpp => cb_gmres_kernels.cpp} (94%)
 rename include/ginkgo/core/solver/{gmres_mixed.hpp => cb_gmres.hpp} (82%)
 rename omp/solver/{gmres_mixed_kernels.cpp => cb_gmres_kernels.cpp} (97%)
 rename omp/test/solver/{gmres_mixed_kernels.cpp => cb_gmres_kernels.cpp} (94%)
 rename reference/solver/{gmres_mixed_kernels.cpp => cb_gmres_kernels.cpp} (96%)
 rename reference/test/solver/{gmres_mixed_kernels.cpp => cb_gmres_kernels.cpp} (88%)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 60dea916357..2091013c77a 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -259,67 +259,67 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
-        {"gmres_mixed_double",
+        {"cb_gmres_double",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::GmresMixed<double>::build()
+             return gko::solver::CbGmres<double>::build()
                  .with_storage_precision(
-                     gko::solver::gmres_mixed_storage_precision::keep)
+                     gko::solver::cb_gmres_storage_precision::keep)
                  .with_criteria(create_criterion(exec))
                  .with_krylov_dim(FLAGS_gmres_restart)
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
-        {"gmres_mixed_float",
+        {"cb_gmres_float",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::GmresMixed<double>::build()
+             return gko::solver::CbGmres<double>::build()
                  .with_storage_precision(
-                     gko::solver::gmres_mixed_storage_precision::reduce1)
+                     gko::solver::cb_gmres_storage_precision::reduce1)
                  .with_criteria(create_criterion(exec))
                  .with_krylov_dim(FLAGS_gmres_restart)
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
-        {"gmres_mixed_half",
+        {"cb_gmres_half",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::GmresMixed<double>::build()
+             return gko::solver::CbGmres<double>::build()
                  .with_storage_precision(
-                     gko::solver::gmres_mixed_storage_precision::reduce2)
+                     gko::solver::cb_gmres_storage_precision::reduce2)
                  .with_criteria(create_criterion(exec))
                  .with_krylov_dim(FLAGS_gmres_restart)
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
-        {"gmres_mixed_int64",
+        {"cb_gmres_int64",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::GmresMixed<double>::build()
+             return gko::solver::CbGmres<double>::build()
                  .with_storage_precision(
-                     gko::solver::gmres_mixed_storage_precision::integer)
+                     gko::solver::cb_gmres_storage_precision::integer)
                  .with_criteria(create_criterion(exec))
                  .with_krylov_dim(FLAGS_gmres_restart)
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
-        {"gmres_mixed_int32",
+        {"cb_gmres_int32",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::GmresMixed<double>::build()
+             return gko::solver::CbGmres<double>::build()
                  .with_storage_precision(
-                     gko::solver::gmres_mixed_storage_precision::ireduce1)
+                     gko::solver::cb_gmres_storage_precision::ireduce1)
                  .with_criteria(create_criterion(exec))
                  .with_krylov_dim(FLAGS_gmres_restart)
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
-        {"gmres_mixed_int16",
+        {"cb_gmres_int16",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::GmresMixed<double>::build()
+             return gko::solver::CbGmres<double>::build()
                  .with_storage_precision(
-                     gko::solver::gmres_mixed_storage_precision::ireduce2)
+                     gko::solver::cb_gmres_storage_precision::ireduce2)
                  .with_criteria(create_criterion(exec))
                  .with_krylov_dim(FLAGS_gmres_restart)
                  .with_preconditioner(give(precond))
diff --git a/common/solver/gmres_mixed_kernels.hpp.inc b/common/solver/cb_gmres_kernels.hpp.inc
similarity index 99%
rename from common/solver/gmres_mixed_kernels.hpp.inc
rename to common/solver/cb_gmres_kernels.hpp.inc
index e4da9eb65b7..54a54209cbd 100644
--- a/common/solver/gmres_mixed_kernels.hpp.inc
+++ b/common/solver/cb_gmres_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index e66d92bbfca..e8a206089e5 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -40,7 +40,7 @@ target_sources(ginkgo
     solver/cgs.cpp
     solver/fcg.cpp
     solver/gmres.cpp
-    solver/gmres_mixed.cpp
+    solver/cb_gmres.cpp
     solver/idr.cpp
     solver/ir.cpp
     solver/lower_trs.cpp
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 8eeecc23c96..cade2c75062 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -57,11 +57,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/reorder/rcm_kernels.hpp"
 #include "core/solver/bicg_kernels.hpp"
 #include "core/solver/bicgstab_kernels.hpp"
+#include "core/solver/cb_gmres_kernels.hpp"
 #include "core/solver/cg_kernels.hpp"
 #include "core/solver/cgs_kernels.hpp"
 #include "core/solver/fcg_kernels.hpp"
 #include "core/solver/gmres_kernels.hpp"
-#include "core/solver/gmres_mixed_kernels.hpp"
 #include "core/solver/idr_kernels.hpp"
 #include "core/solver/ir_kernels.hpp"
 #include "core/solver/lower_trs_kernels.hpp"
@@ -571,43 +571,41 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_STEP_2_KERNEL);
 }  // namespace gmres
 
 
-namespace gmres_mixed {
+namespace cb_gmres {
 
 
 template <typename ValueType>
-GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL(ValueType)
+GKO_DECLARE_CB_GMRES_INITIALIZE_1_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CB_GMRES_INITIALIZE_1_KERNEL);
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(ValueType, ValueTypeKrylovBases)
+GKO_DECLARE_CB_GMRES_INITIALIZE_2_KERNEL(ValueType, ValueTypeKrylovBases)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
-    GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(
+    GKO_DECLARE_CB_GMRES_INITIALIZE_2_KERNEL);
 
 /*
 template <typename ValueType, typename ValueTypeKrylovBases, bool
 Reorthogonalization, bool MGS_CGS>
-GKO_DECLARE_GMRES_MIXED_BOOL_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases,
+GKO_DECLARE_CB_GMRES_BOOL_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases,
                                            Reorthogonalization, MGS_CGS)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(GKO_DECLARE_GMRES_MIXED_BOOL_STEP_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(GKO_DECLARE_CB_GMRES_BOOL_STEP_1_KERNEL);
 */
 template <typename ValueType, typename ValueTypeKrylovBases>
-GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases)
+GKO_DECLARE_CB_GMRES_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
-    GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(GKO_DECLARE_CB_GMRES_STEP_1_KERNEL);
 
 template <typename ValueType, typename ValueTypeKrylovBases>
-GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(ValueType, ValueTypeKrylovBases)
+GKO_DECLARE_CB_GMRES_STEP_2_KERNEL(ValueType, ValueTypeKrylovBases)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(
-    GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_CONST_TYPE(
+    GKO_DECLARE_CB_GMRES_STEP_2_KERNEL);
 
 
-}  // namespace gmres_mixed
+}  // namespace cb_gmres
 
 
 namespace ir {
diff --git a/core/solver/gmres_mixed.cpp b/core/solver/cb_gmres.cpp
similarity index 92%
rename from core/solver/gmres_mixed.cpp
rename to core/solver/cb_gmres.cpp
index 18e88700836..8c5ebf378e5 100644
--- a/core/solver/gmres_mixed.cpp
+++ b/core/solver/cb_gmres.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <ginkgo/core/solver/gmres_mixed.hpp>
+#include <ginkgo/core/solver/cb_gmres.hpp>
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -48,8 +48,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <typeinfo>
 
 
-#include "core/solver/gmres_mixed_accessor.hpp"
-#include "core/solver/gmres_mixed_kernels.hpp"
+#include "core/solver/cb_gmres_accessor.hpp"
+#include "core/solver/cb_gmres_kernels.hpp"
 
 
 //#define TIMING 1
@@ -65,16 +65,16 @@ namespace gko {
 namespace solver {
 
 
-namespace gmres_mixed {
+namespace cb_gmres {
 
 
-GKO_REGISTER_OPERATION(initialize_1, gmres_mixed::initialize_1);
-GKO_REGISTER_OPERATION(initialize_2, gmres_mixed::initialize_2);
-GKO_REGISTER_OPERATION(step_1, gmres_mixed::step_1);
-GKO_REGISTER_OPERATION(step_2, gmres_mixed::step_2);
+GKO_REGISTER_OPERATION(initialize_1, cb_gmres::initialize_1);
+GKO_REGISTER_OPERATION(initialize_2, cb_gmres::initialize_2);
+GKO_REGISTER_OPERATION(step_1, cb_gmres::step_1);
+GKO_REGISTER_OPERATION(step_2, cb_gmres::step_2);
 
 
-}  // namespace gmres_mixed
+}  // namespace cb_gmres
 
 // TODO: Remove output
 template <typename T>
@@ -159,22 +159,22 @@ template <typename ValueType>
 struct helper {
     template <typename Callable>
     static void call(Callable callable,
-                     gko::solver::gmres_mixed_storage_precision st)
+                     gko::solver::cb_gmres_storage_precision st)
     {
         switch (st) {
-        case gmres_mixed_storage_precision::reduce1:
+        case cb_gmres_storage_precision::reduce1:
             callable(reduce_precision_count<ValueType, 1>{});
             break;
-        case gmres_mixed_storage_precision::reduce2:
+        case cb_gmres_storage_precision::reduce2:
             callable(reduce_precision_count<ValueType, 2>{});
             break;
-        case gmres_mixed_storage_precision::integer:
+        case cb_gmres_storage_precision::integer:
             callable(to_integer<ValueType>{});
             break;
-        case gmres_mixed_storage_precision::ireduce1:
+        case cb_gmres_storage_precision::ireduce1:
             callable(to_integer<reduce_precision_count<ValueType, 1>>{});
             break;
-        case gmres_mixed_storage_precision::ireduce2:
+        case cb_gmres_storage_precision::ireduce2:
             callable(to_integer<reduce_precision_count<ValueType, 2>>{});
             break;
         default:
@@ -192,18 +192,18 @@ struct helper<std::complex<T>> {
 
     template <typename Callable>
     static void call(Callable callable,
-                     gko::solver::gmres_mixed_storage_precision st)
+                     gko::solver::cb_gmres_storage_precision st)
     {
         switch (st) {
-        case gmres_mixed_storage_precision::reduce1:
+        case cb_gmres_storage_precision::reduce1:
             callable(reduce_precision_skip_count<ValueType, skip_type, 1>{});
             break;
-        case gmres_mixed_storage_precision::reduce2:
+        case cb_gmres_storage_precision::reduce2:
             callable(reduce_precision_skip_count<ValueType, skip_type, 2>{});
             break;
-        case gmres_mixed_storage_precision::integer:
-        case gmres_mixed_storage_precision::ireduce1:
-        case gmres_mixed_storage_precision::ireduce2:
+        case cb_gmres_storage_precision::integer:
+        case cb_gmres_storage_precision::ireduce1:
+        case cb_gmres_storage_precision::ireduce2:
             GKO_NOT_SUPPORTED(st);
             break;
         default:
@@ -214,7 +214,7 @@ struct helper<std::complex<T>> {
 
 
 template <typename ValueType>
-void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
+void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 {
     // Current workaround to get a lambda with a template argument (only the
     // type of `value` matters, the content does not)
@@ -300,7 +300,7 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
         // std::cout << "Before initializate_1" << std::endl;
         // Initialization
-        exec->run(gmres_mixed::make_initialize_1(
+        exec->run(cb_gmres::make_initialize_1(
             dense_b, b_norm.get(), residual.get(), givens_sin.get(),
             givens_cos.get(), &stop_status, krylov_dim_));
         // b_norm = norm(b)
@@ -311,7 +311,7 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         // residual = residual - Ax
 
         // std::cout << "Before initializate_2" << std::endl;
-        exec->run(gmres_mixed::make_initialize_2(
+        exec->run(cb_gmres::make_initialize_2(
             residual.get(), residual_norm.get(), residual_norm_collection.get(),
             arnoldi_norm.get(), krylov_bases_range, next_krylov_basis.get(),
             &final_iter_nums, krylov_dim_));
@@ -436,7 +436,7 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                     span{0, restart_iter},
                     span{0, dense_b->get_size()[1] * (restart_iter)});
 
-                exec->run(gmres_mixed::make_step_2(
+                exec->run(cb_gmres::make_step_2(
                     residual_norm_collection.get(),
                     krylov_bases_range.get_accessor().to_const(),
                     hessenberg_view.get(), y.get(), before_preconditioner.get(),
@@ -454,7 +454,7 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                 system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(),
                                       residual.get());
                 // residual = residual - Ax
-                exec->run(gmres_mixed::make_initialize_2(
+                exec->run(cb_gmres::make_initialize_2(
                     residual.get(), residual_norm.get(),
                     residual_norm_collection.get(), arnoldi_norm.get(),
                     krylov_bases_range, next_krylov_basis.get(),
@@ -501,7 +501,7 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             exec->synchronize();
             auto t_aux_2 = std::chrono::steady_clock::now();
 #endif
-            exec->run(gmres_mixed::make_step_1(
+            exec->run(cb_gmres::make_step_1(
                 next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
                 residual_norm.get(), residual_norm_collection.get(),
                 krylov_bases_range, hessenberg_iter.get(), buffer_iter.get(),
@@ -551,7 +551,7 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             span{0, restart_iter},
             span{0, dense_b->get_size()[1] * (restart_iter)});
 
-        exec->run(gmres_mixed::make_step_2(
+        exec->run(cb_gmres::make_step_2(
             residual_norm_collection.get(),
             krylov_bases_range.get_accessor().to_const(),
             hessenberg_small.get(), y.get(), before_preconditioner.get(),
@@ -621,9 +621,9 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
 
 template <typename ValueType>
-void GmresMixed<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
-                                       const LinOp *residual_norm_collection,
-                                       LinOp *x) const
+void CbGmres<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
+                                    const LinOp *residual_norm_collection,
+                                    LinOp *x) const
 {
     auto dense_x = as<matrix::Dense<ValueType>>(x);
 
@@ -633,8 +633,8 @@ void GmresMixed<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
     dense_x->add_scaled(alpha, x_clone.get());
 }
 
-#define GKO_DECLARE_GMRES_MIXED(_type1) class GmresMixed<_type1>
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_MIXED);
+#define GKO_DECLARE_CB_GMRES(_type1) class CbGmres<_type1>
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CB_GMRES);
 
 
 }  // namespace solver
diff --git a/core/solver/gmres_mixed_accessor.hpp b/core/solver/cb_gmres_accessor.hpp
similarity index 97%
rename from core/solver/gmres_mixed_accessor.hpp
rename to core/solver/cb_gmres_accessor.hpp
index e34bd299c1d..d1f185dc33b 100644
--- a/core/solver/gmres_mixed_accessor.hpp
+++ b/core/solver/cb_gmres_accessor.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_GMRES_MIXED_ACCESSOR_HPP_
-#define GKO_CORE_SOLVER_GMRES_MIXED_ACCESSOR_HPP_
+#ifndef GKO_CORE_SOLVER_CB_GMRES_ACCESSOR_HPP_
+#define GKO_CORE_SOLVER_CB_GMRES_ACCESSOR_HPP_
 
 
 #include <array>
@@ -221,4 +221,4 @@ struct helper_functions_accessor<Accessor3d, false> {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_GMRES_MIXED_ACCESSOR_HPP_
+#endif  // GKO_CORE_SOLVER_CB_GMRES_ACCESSOR_HPP_
diff --git a/core/solver/gmres_mixed_kernels.hpp b/core/solver/cb_gmres_kernels.hpp
similarity index 91%
rename from core/solver/gmres_mixed_kernels.hpp
rename to core/solver/cb_gmres_kernels.hpp
index 1a03e102bf7..2a2122a4c90 100644
--- a/core/solver/gmres_mixed_kernels.hpp
+++ b/core/solver/cb_gmres_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_GMRES_MIXED_KERNELS_HPP_
-#define GKO_CORE_SOLVER_GMRES_MIXED_KERNELS_HPP_
+#ifndef GKO_CORE_SOLVER_CB_GMRES_KERNELS_HPP_
+#define GKO_CORE_SOLVER_CB_GMRES_KERNELS_HPP_
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -44,14 +44,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/base/accessors.hpp"
 #include "core/base/extended_float.hpp"
-#include "core/solver/gmres_mixed_accessor.hpp"
+#include "core/solver/cb_gmres_accessor.hpp"
 
 
 // TODO Find way around using it!
 #define GKO_UNPACK(...) __VA_ARGS__
 /**
  * Instantiates a template for each value type with each lower precision type
- * supported by Ginkgo for GmresMixed.
+ * supported by Ginkgo for CbGmres.
  *
  * @param _macro  A macro which expands the template instantiation
  *                (not including the leading `template` specifier).
@@ -60,7 +60,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *                     (precisions supported by Ginkgo), and
  *                2. the second the value type of the reduced precision.
  */
-#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(_macro)                      \
+#define GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(_macro)                         \
     template _macro(                                                           \
         double,                                                                \
         GKO_UNPACK(range<accessor::reduced_row_major<3, double, double>>));    \
@@ -111,7 +111,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         GKO_UNPACK(range<accessor::reduced_row_major<3, std::complex<float>,   \
                                                      std::complex<float>>>))
 
-#define GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(_macro)                \
+#define GKO_INSTANTIATE_FOR_EACH_CB_GMRES_CONST_TYPE(_macro)                   \
     template _macro(                                                           \
         double,                                                                \
         GKO_UNPACK(                                                            \
@@ -165,10 +165,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace gko {
 namespace kernels {
-namespace gmres_mixed {
+namespace cb_gmres {
 
 
-#define GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL(_type)                     \
+#define GKO_DECLARE_CB_GMRES_INITIALIZE_1_KERNEL(_type)                        \
     void initialize_1(                                                         \
         std::shared_ptr<const DefaultExecutor> exec,                           \
         const matrix::Dense<_type> *b,                                         \
@@ -178,7 +178,7 @@ namespace gmres_mixed {
         size_type krylov_dim)
 
 
-#define GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(_type1, _range)         \
+#define GKO_DECLARE_CB_GMRES_INITIALIZE_2_KERNEL(_type1, _range)            \
     void initialize_2(std::shared_ptr<const DefaultExecutor> exec,          \
                       const matrix::Dense<_type1> *residual,                \
                       matrix::Dense<remove_complex<_type1>> *residual_norm, \
@@ -189,7 +189,7 @@ namespace gmres_mixed {
                       Array<size_type> *final_iter_nums, size_type krylov_dim)
 
 
-#define GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(_type1, _range)                 \
+#define GKO_DECLARE_CB_GMRES_STEP_1_KERNEL(_type1, _range)                    \
     void step_1(                                                              \
         std::shared_ptr<const DefaultExecutor> exec,                          \
         matrix::Dense<_type1> *next_krylov_basis,                             \
@@ -206,7 +206,7 @@ namespace gmres_mixed {
         int *num_reorth_steps, int *num_reorth_vectors)
 
 
-#define GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(_type1, _range)                 \
+#define GKO_DECLARE_CB_GMRES_STEP_2_KERNEL(_type1, _range)                    \
     void step_2(std::shared_ptr<const DefaultExecutor> exec,                  \
                 const matrix::Dense<_type1> *residual_norm_collection,        \
                 _range krylov_bases, const matrix::Dense<_type1> *hessenberg, \
@@ -215,62 +215,62 @@ namespace gmres_mixed {
                 const Array<size_type> *final_iter_nums)
 
 
-#define GKO_DECLARE_ALL_AS_TEMPLATES                                    \
-    template <typename ValueType>                                       \
-    GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL(ValueType);             \
-    template <typename ValueType, typename Accessor3d>                  \
-    GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL(ValueType, Accessor3d); \
-    template <typename ValueType, typename Accessor3d>                  \
-    GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL(ValueType, Accessor3d);       \
-    template <typename ValueType, typename Accessor3d>                  \
-    GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL(ValueType, Accessor3d)
+#define GKO_DECLARE_ALL_AS_TEMPLATES                                 \
+    template <typename ValueType>                                    \
+    GKO_DECLARE_CB_GMRES_INITIALIZE_1_KERNEL(ValueType);             \
+    template <typename ValueType, typename Accessor3d>               \
+    GKO_DECLARE_CB_GMRES_INITIALIZE_2_KERNEL(ValueType, Accessor3d); \
+    template <typename ValueType, typename Accessor3d>               \
+    GKO_DECLARE_CB_GMRES_STEP_1_KERNEL(ValueType, Accessor3d);       \
+    template <typename ValueType, typename Accessor3d>               \
+    GKO_DECLARE_CB_GMRES_STEP_2_KERNEL(ValueType, Accessor3d)
 
 
-}  // namespace gmres_mixed
+}  // namespace cb_gmres
 
 
 namespace omp {
-namespace gmres_mixed {
+namespace cb_gmres {
 
 GKO_DECLARE_ALL_AS_TEMPLATES;
 
-}  // namespace gmres_mixed
+}  // namespace cb_gmres
 }  // namespace omp
 
 
 namespace cuda {
-namespace gmres_mixed {
+namespace cb_gmres {
 
 GKO_DECLARE_ALL_AS_TEMPLATES;
 
-}  // namespace gmres_mixed
+}  // namespace cb_gmres
 }  // namespace cuda
 
 
 namespace reference {
-namespace gmres_mixed {
+namespace cb_gmres {
 
 GKO_DECLARE_ALL_AS_TEMPLATES;
 
-}  // namespace gmres_mixed
+}  // namespace cb_gmres
 }  // namespace reference
 
 
 namespace hip {
-namespace gmres_mixed {
+namespace cb_gmres {
 
 GKO_DECLARE_ALL_AS_TEMPLATES;
 
-}  // namespace gmres_mixed
+}  // namespace cb_gmres
 }  // namespace hip
 
 
 namespace dpcpp {
-namespace gmres_mixed {
+namespace cb_gmres {
 
 GKO_DECLARE_ALL_AS_TEMPLATES;
 
-}  // namespace gmres_mixed
+}  // namespace cb_gmres
 }  // namespace dpcpp
 
 
@@ -281,4 +281,4 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_GMRES_MIXED_KERNELS_HPP_
+#endif  // GKO_CORE_SOLVER_CB_GMRES_KERNELS_HPP_
diff --git a/core/test/solver/CMakeLists.txt b/core/test/solver/CMakeLists.txt
index 4fb9dbb8ba8..384563c9ef3 100644
--- a/core/test/solver/CMakeLists.txt
+++ b/core/test/solver/CMakeLists.txt
@@ -4,7 +4,7 @@ ginkgo_create_test(cg)
 ginkgo_create_test(cgs)
 ginkgo_create_test(fcg)
 ginkgo_create_test(gmres)
-ginkgo_create_test(gmres_mixed)
+ginkgo_create_test(cb_gmres)
 ginkgo_create_test(idr)
 ginkgo_create_test(ir)
 ginkgo_create_test(lower_trs)
diff --git a/core/test/solver/gmres_mixed.cpp b/core/test/solver/cb_gmres.cpp
similarity index 72%
rename from core/test/solver/gmres_mixed.cpp
rename to core/test/solver/cb_gmres.cpp
index ff22f71956f..2867b1b2244 100644
--- a/core/test/solver/gmres_mixed.cpp
+++ b/core/test/solver/cb_gmres.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <core/solver/gmres_mixed.cpp>
-#include <ginkgo/core/solver/gmres_mixed.hpp>
+#include <core/solver/cb_gmres.cpp>
+#include <ginkgo/core/solver/cb_gmres.hpp>
 
 #include <typeinfo>
 
@@ -45,18 +45,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace {
 
-class GmresMixed : public ::testing::Test {
+class CbGmres : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Dense<>;
-    using Solver = gko::solver::GmresMixed<>;
-    //    using Solver = gko::solver::GmresMixed<double,double>;
-    using Big_solver = gko::solver::GmresMixed<double>;
+    using Solver = gko::solver::CbGmres<>;
+    //    using Solver = gko::solver::CbGmres<double,double>;
+    using Big_solver = gko::solver::CbGmres<double>;
 
-    GmresMixed()
+    CbGmres()
         : exec(gko::ReferenceExecutor::create()),
           mtx(gko::initialize<Mtx>(
               {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)),
-          gmres_mixed_factory(
+          cb_gmres_factory(
               Solver::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(3u).on(exec),
@@ -64,8 +64,8 @@ class GmresMixed : public ::testing::Test {
                           .with_reduction_factor(1e-6)
                           .on(exec))
                   .on(exec)),
-          solver(gmres_mixed_factory->generate(mtx)),
-          gmres_mixed_big_factory(
+          solver(cb_gmres_factory->generate(mtx)),
+          cb_gmres_big_factory(
               Big_solver::build()
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(128u).on(
@@ -74,14 +74,14 @@ class GmresMixed : public ::testing::Test {
                           .with_reduction_factor(1e-6)
                           .on(exec))
                   .on(exec)),
-          big_solver(gmres_mixed_big_factory->generate(mtx))
+          big_solver(cb_gmres_big_factory->generate(mtx))
     {}
 
     std::shared_ptr<const gko::Executor> exec;
     std::shared_ptr<Mtx> mtx;
-    std::unique_ptr<Solver::Factory> gmres_mixed_factory;
+    std::unique_ptr<Solver::Factory> cb_gmres_factory;
     std::unique_ptr<gko::LinOp> solver;
-    std::unique_ptr<Big_solver::Factory> gmres_mixed_big_factory;
+    std::unique_ptr<Big_solver::Factory> cb_gmres_big_factory;
     std::unique_ptr<gko::LinOp> big_solver;
 
     static void assert_same_matrices(const Mtx *m1, const Mtx *m2)
@@ -96,22 +96,22 @@ class GmresMixed : public ::testing::Test {
     }
 };
 
-TEST_F(GmresMixed, GmresMixedFactoryKnowsItsExecutor)
+TEST_F(CbGmres, CbGmresFactoryKnowsItsExecutor)
 {
-    ASSERT_EQ(gmres_mixed_factory->get_executor(), exec);
+    ASSERT_EQ(cb_gmres_factory->get_executor(), exec);
 }
 
-TEST_F(GmresMixed, GmresMixedFactoryCreatesCorrectSolver)
+TEST_F(CbGmres, CbGmresFactoryCreatesCorrectSolver)
 {
     ASSERT_EQ(solver->get_size(), gko::dim<2>(3, 3));
-    auto gmres_mixed_solver = static_cast<Solver *>(solver.get());
-    ASSERT_NE(gmres_mixed_solver->get_system_matrix(), nullptr);
-    ASSERT_EQ(gmres_mixed_solver->get_system_matrix(), mtx);
+    auto cb_gmres_solver = static_cast<Solver *>(solver.get());
+    ASSERT_NE(cb_gmres_solver->get_system_matrix(), nullptr);
+    ASSERT_EQ(cb_gmres_solver->get_system_matrix(), mtx);
 }
 
-TEST_F(GmresMixed, CanBeCopied)
+TEST_F(CbGmres, CanBeCopied)
 {
-    auto copy = gmres_mixed_factory->generate(Mtx::create(exec));
+    auto copy = cb_gmres_factory->generate(Mtx::create(exec));
 
     copy->copy_from(solver.get());
 
@@ -120,9 +120,9 @@ TEST_F(GmresMixed, CanBeCopied)
     assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()), mtx.get());
 }
 
-TEST_F(GmresMixed, CanBeMoved)
+TEST_F(CbGmres, CanBeMoved)
 {
-    auto copy = gmres_mixed_factory->generate(Mtx::create(exec));
+    auto copy = cb_gmres_factory->generate(Mtx::create(exec));
 
     copy->copy_from(std::move(solver));
 
@@ -131,7 +131,7 @@ TEST_F(GmresMixed, CanBeMoved)
     assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()), mtx.get());
 }
 
-TEST_F(GmresMixed, CanBeCloned)
+TEST_F(CbGmres, CanBeCloned)
 {
     auto clone = solver->clone();
 
@@ -140,7 +140,7 @@ TEST_F(GmresMixed, CanBeCloned)
     assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()), mtx.get());
 }
 
-TEST_F(GmresMixed, CanBeCleared)
+TEST_F(CbGmres, CanBeCleared)
 {
     solver->clear();
 
@@ -149,9 +149,9 @@ TEST_F(GmresMixed, CanBeCleared)
     ASSERT_EQ(solver_mtx, nullptr);
 }
 
-TEST_F(GmresMixed, CanSetPreconditionerGenerator)
+TEST_F(CbGmres, CanSetPreconditionerGenerator)
 {
-    auto gmres_mixed_factory =
+    auto cb_gmres_factory =
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(exec),
@@ -165,9 +165,9 @@ TEST_F(GmresMixed, CanSetPreconditionerGenerator)
                             exec))
                     .on(exec))
             .on(exec);
-    auto solver = gmres_mixed_factory->generate(mtx);
-    auto precond = dynamic_cast<const gko::solver::GmresMixed<> *>(
-        static_cast<gko::solver::GmresMixed<> *>(solver.get())
+    auto solver = cb_gmres_factory->generate(mtx);
+    auto precond = dynamic_cast<const gko::solver::CbGmres<> *>(
+        static_cast<gko::solver::CbGmres<> *>(solver.get())
             ->get_preconditioner()
             .get());
 
@@ -176,9 +176,9 @@ TEST_F(GmresMixed, CanSetPreconditionerGenerator)
     ASSERT_EQ(precond->get_system_matrix(), mtx);
 }
 
-TEST_F(GmresMixed, CanSetKrylovDim)
+TEST_F(CbGmres, CanSetKrylovDim)
 {
-    auto gmres_mixed_factory =
+    auto cb_gmres_factory =
         Solver::build()
             .with_krylov_dim(4u)
             .with_criteria(
@@ -187,74 +187,74 @@ TEST_F(GmresMixed, CanSetKrylovDim)
                     .with_reduction_factor(1e-6)
                     .on(exec))
             .on(exec);
-    auto solver = gmres_mixed_factory->generate(mtx);
+    auto solver = cb_gmres_factory->generate(mtx);
     auto krylov_dim = solver->get_krylov_dim();
 
     ASSERT_EQ(krylov_dim, 4);
 }
 
-TEST_F(GmresMixed, CanSetPreconditionerInFactory)
+TEST_F(CbGmres, CanSetPreconditionerInFactory)
 {
-    std::shared_ptr<Solver> gmres_mixed_precond =
+    std::shared_ptr<Solver> cb_gmres_precond =
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(exec))
             .on(exec)
             ->generate(mtx);
 
-    auto gmres_mixed_factory =
+    auto cb_gmres_factory =
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(exec))
-            .with_generated_preconditioner(gmres_mixed_precond)
+            .with_generated_preconditioner(cb_gmres_precond)
             .on(exec);
-    auto solver = gmres_mixed_factory->generate(mtx);
+    auto solver = cb_gmres_factory->generate(mtx);
     auto precond = solver->get_preconditioner();
 
     ASSERT_NE(precond.get(), nullptr);
-    ASSERT_EQ(precond.get(), gmres_mixed_precond.get());
+    ASSERT_EQ(precond.get(), cb_gmres_precond.get());
 }
 
-TEST_F(GmresMixed, ThrowsOnWrongPreconditionerInFactory)
+TEST_F(CbGmres, ThrowsOnWrongPreconditionerInFactory)
 {
     std::shared_ptr<Mtx> wrong_sized_mtx = Mtx::create(exec, gko::dim<2>{1, 3});
-    std::shared_ptr<Solver> gmres_mixed_precond =
+    std::shared_ptr<Solver> cb_gmres_precond =
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(exec))
             .on(exec)
             ->generate(wrong_sized_mtx);
 
-    auto gmres_mixed_factory =
+    auto cb_gmres_factory =
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(exec))
-            .with_generated_preconditioner(gmres_mixed_precond)
+            .with_generated_preconditioner(cb_gmres_precond)
             .on(exec);
 
-    ASSERT_THROW(gmres_mixed_factory->generate(mtx), gko::DimensionMismatch);
+    ASSERT_THROW(cb_gmres_factory->generate(mtx), gko::DimensionMismatch);
 }
 
-TEST_F(GmresMixed, CanSetPreconditioner)
+TEST_F(CbGmres, CanSetPreconditioner)
 {
-    std::shared_ptr<Solver> gmres_mixed_precond =
+    std::shared_ptr<Solver> cb_gmres_precond =
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(exec))
             .on(exec)
             ->generate(mtx);
 
-    auto gmres_mixed_factory =
+    auto cb_gmres_factory =
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(3u).on(exec))
             .on(exec);
-    auto solver = gmres_mixed_factory->generate(mtx);
-    solver->set_preconditioner(gmres_mixed_precond);
+    auto solver = cb_gmres_factory->generate(mtx);
+    solver->set_preconditioner(cb_gmres_precond);
     auto precond = solver->get_preconditioner();
 
     ASSERT_NE(precond.get(), nullptr);
-    ASSERT_EQ(precond.get(), gmres_mixed_precond.get());
+    ASSERT_EQ(precond.get(), cb_gmres_precond.get());
 }
 
 }  // namespace
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 80d1faf556e..883525061f7 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -113,7 +113,7 @@ target_sources(ginkgo_cuda
     solver/cgs_kernels.cu
     solver/fcg_kernels.cu
     solver/gmres_kernels.cu
-    solver/gmres_mixed_kernels.cu
+    solver/cb_gmres_kernels.cu
     solver/idr_kernels.cu
     solver/ir_kernels.cu
     solver/lower_trs_kernels.cu
diff --git a/cuda/solver/gmres_mixed_kernels.cu b/cuda/solver/cb_gmres_kernels.cu
similarity index 96%
rename from cuda/solver/gmres_mixed_kernels.cu
rename to cuda/solver/cb_gmres_kernels.cu
index 347d3486b84..9f9157a5a04 100644
--- a/cuda/solver/gmres_mixed_kernels.cu
+++ b/cuda/solver/cb_gmres_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/solver/gmres_mixed_kernels.hpp"
+#include "core/solver/cb_gmres_kernels.hpp"
 
 
 #include <algorithm>
@@ -58,11 +58,11 @@ namespace gko {
 namespace kernels {
 namespace cuda {
 /**
- * @brief The GMRES_MIXED solver namespace.
+ * @brief The CB_GMRES solver namespace.
  *
- * @ingroup gmres_mixed
+ * @ingroup cb_gmres
  */
-namespace gmres_mixed {
+namespace cb_gmres {
 
 
 constexpr int default_block_size = 512;
@@ -75,7 +75,7 @@ constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 constexpr int default_update_dim = 32;
 
 
-#include "common/solver/gmres_mixed_kernels.hpp.inc"
+#include "common/solver/cb_gmres_kernels.hpp.inc"
 
 
 // Specialization, so the Accessor can use the same function as regular pointers
@@ -85,17 +85,20 @@ GKO_INLINE auto as_cuda_accessor(
 {
     return range<
         accessor::reduced_row_major<dim, cuda_type<Type1>, cuda_type<Type2>>>(
-        acc.get_accessor().get_size(), as_cuda_type(acc.get_accessor().get_stored_data()),
+        acc.get_accessor().get_size(),
+        as_cuda_type(acc.get_accessor().get_stored_data()),
         acc.get_accessor().get_stride());
 }
 
 template <int dim, typename Type1, typename Type2, size_type mask>
 GKO_INLINE auto as_cuda_accessor(
-    const range<accessor::scaled_reduced_row_major<dim, Type1, Type2, mask>> &acc)
+    const range<accessor::scaled_reduced_row_major<dim, Type1, Type2, mask>>
+        &acc)
 {
     return range<accessor::scaled_reduced_row_major<dim, cuda_type<Type1>,
                                                     cuda_type<Type2>, mask>>(
-        acc.get_accessor().get_size(), as_cuda_type(acc.get_accessor().get_stored_data()),
+        acc.get_accessor().get_size(),
+        as_cuda_type(acc.get_accessor().get_stored_data()),
         acc.get_accessor().get_storage_stride(),
         as_cuda_type(acc.get_accessor().get_scalar()),
         acc.get_accessor().get_scalar_stride());
@@ -137,8 +140,7 @@ void initialize_1(std::shared_ptr<const CudaExecutor> exec,
         as_cuda_type(stop_status->get_data()));
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CB_GMRES_INITIALIZE_1_KERNEL);
 
 
 template <typename ValueType, typename Accessor3d>
@@ -203,8 +205,8 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
         as_cuda_type(final_iter_nums->get_data()));
 }
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
-    GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(
+    GKO_DECLARE_CB_GMRES_INITIALIZE_2_KERNEL);
 
 
 template <typename ValueType, typename Accessor3dim>
@@ -450,8 +452,7 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
                     stop_status);
 }
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
-    GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(GKO_DECLARE_CB_GMRES_STEP_1_KERNEL);
 
 
 template <typename ValueType>
@@ -527,11 +528,11 @@ void step_2(std::shared_ptr<const CudaExecutor> exec,
                  final_iter_nums);
 }
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(
-    GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_CONST_TYPE(
+    GKO_DECLARE_CB_GMRES_STEP_2_KERNEL);
 
 
-}  // namespace gmres_mixed
+}  // namespace cb_gmres
 }  // namespace cuda
 }  // namespace kernels
 }  // namespace gko
diff --git a/cuda/test/solver/CMakeLists.txt b/cuda/test/solver/CMakeLists.txt
index 0689804b253..623c3076ed8 100644
--- a/cuda/test/solver/CMakeLists.txt
+++ b/cuda/test/solver/CMakeLists.txt
@@ -4,7 +4,7 @@ ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
-ginkgo_create_test(gmres_mixed_kernels)
+ginkgo_create_test(cb_gmres_kernels)
 ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
 ginkgo_create_test_cpp_cuda_header(lower_trs_kernels)
diff --git a/cuda/test/solver/gmres_mixed_kernels.cpp b/cuda/test/solver/cb_gmres_kernels.cpp
similarity index 94%
rename from cuda/test/solver/gmres_mixed_kernels.cpp
rename to cuda/test/solver/cb_gmres_kernels.cpp
index 9fd61b0f2f1..ed81c9dab6f 100644
--- a/cuda/test/solver/gmres_mixed_kernels.cpp
+++ b/cuda/test/solver/cb_gmres_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <ginkgo/core/solver/gmres_mixed.hpp>
+#include <ginkgo/core/solver/cb_gmres.hpp>
 
 
 #include <random>
@@ -47,15 +47,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/residual_norm_reduction.hpp>
 
 
-#include "core/solver/gmres_mixed_accessor.hpp"
-#include "core/solver/gmres_mixed_kernels.hpp"
+#include "core/solver/cb_gmres_accessor.hpp"
+#include "core/solver/cb_gmres_kernels.hpp"
 #include "core/test/utils.hpp"
 
 
 namespace {
 
 
-class GmresMixed : public ::testing::Test {
+class CbGmres : public ::testing::Test {
 protected:
     using value_type = double;
     using storage_type = float;
@@ -67,7 +67,7 @@ class GmresMixed : public ::testing::Test {
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
 
-    GmresMixed() : rand_engine(30) {}
+    CbGmres() : rand_engine(30) {}
 
     void SetUp()
     {
@@ -270,15 +270,15 @@ class GmresMixed : public ::testing::Test {
 };
 
 
-TEST_F(GmresMixed, CudaGmresMixedInitialize1IsEquivalentToRef)
+TEST_F(CbGmres, CudaCbGmresInitialize1IsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::gmres_mixed::initialize_1(
+    gko::kernels::reference::cb_gmres::initialize_1(
         ref, b.get(), b_norm.get(), residual.get(), givens_sin.get(),
         givens_cos.get(), stop_status.get(),
         gko::solver::default_krylov_dim_mixed);
-    gko::kernels::cuda::gmres_mixed::initialize_1(
+    gko::kernels::cuda::cb_gmres::initialize_1(
         cuda, d_b.get(), d_b_norm.get(), d_residual.get(), d_givens_sin.get(),
         d_givens_cos.get(), d_stop_status.get(),
         gko::solver::default_krylov_dim_mixed);
@@ -291,16 +291,16 @@ TEST_F(GmresMixed, CudaGmresMixedInitialize1IsEquivalentToRef)
 }
 
 
-TEST_F(GmresMixed, CudaGmresMixedInitialize2IsEquivalentToRef)
+TEST_F(CbGmres, CudaCbGmresInitialize2IsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::gmres_mixed::initialize_2(
+    gko::kernels::reference::cb_gmres::initialize_2(
         ref, residual.get(), residual_norm.get(),
         residual_norm_collection.get(), arnoldi_norm.get(),
         range_helper.get_range(), next_krylov_basis.get(),
         final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
-    gko::kernels::cuda::gmres_mixed::initialize_2(
+    gko::kernels::cuda::cb_gmres::initialize_2(
         cuda, d_residual.get(), d_residual_norm.get(),
         d_residual_norm_collection.get(), d_arnoldi_norm.get(),
         d_range_helper.get_range(), d_next_krylov_basis.get(),
@@ -314,21 +314,21 @@ TEST_F(GmresMixed, CudaGmresMixedInitialize2IsEquivalentToRef)
 }
 
 
-TEST_F(GmresMixed, CudaGmresMixedStep1IsEquivalentToRef)
+TEST_F(CbGmres, CudaCbGmresStep1IsEquivalentToRef)
 {
     initialize_data();
     int iter = 5;
     int num_reorth_steps = 0, num_reorth_vectors = 0;
     int d_num_reorth_steps = 0, d_num_reorth_vectors = 0;
 
-    gko::kernels::reference::gmres_mixed::step_1(
+    gko::kernels::reference::cb_gmres::step_1(
         ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
         residual_norm.get(), residual_norm_collection.get(),
         range_helper.get_range(), hessenberg_iter.get(), buffer_iter.get(),
         b_norm.get(), arnoldi_norm.get(), iter, final_iter_nums.get(),
         stop_status.get(), reorth_status.get(), num_reorth.get(),
         &num_reorth_steps, &num_reorth_vectors);
-    gko::kernels::cuda::gmres_mixed::step_1(
+    gko::kernels::cuda::cb_gmres::step_1(
         cuda, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
         d_residual_norm.get(), d_residual_norm_collection.get(),
         d_range_helper.get_range(), d_hessenberg_iter.get(),
@@ -348,15 +348,15 @@ TEST_F(GmresMixed, CudaGmresMixedStep1IsEquivalentToRef)
 }
 
 
-TEST_F(GmresMixed, CudaGmresMixedStep2IsEquivalentToRef)
+TEST_F(CbGmres, CudaCbGmresStep2IsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::gmres_mixed::step_2(
+    gko::kernels::reference::cb_gmres::step_2(
         ref, residual_norm_collection.get(),
         range_helper.get_range().get_accessor().to_const(), hessenberg.get(),
         y.get(), before_preconditioner.get(), final_iter_nums.get());
-    gko::kernels::cuda::gmres_mixed::step_2(
+    gko::kernels::cuda::cb_gmres::step_2(
         cuda, d_residual_norm_collection.get(),
         d_range_helper.get_range().get_accessor().to_const(),
         d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index cdd37d3556e..81fe4492f8a 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -39,7 +39,7 @@ target_sources(ginkgo_dpcpp
     solver/cgs_kernels.dp.cpp
     solver/fcg_kernels.dp.cpp
     solver/gmres_kernels.dp.cpp
-    solver/gmres_mixed_kernels.dp.cpp
+    solver/cb_gmres_kernels.dp.cpp
     solver/idr_kernels.dp.cpp
     solver/ir_kernels.dp.cpp
     solver/lower_trs_kernels.dp.cpp
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index e3ce557c654..c4948d43b56 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -21,8 +21,8 @@ set(EXAMPLES_LIST
     custom-matrix-format
     custom-stopping-criterion
     ginkgo-overhead
+    cb-gmres-solver
     gmres-solver
-    gmres-mixed-solver
     heat-equation
     minimal-cuda-solver
     papi-logging
diff --git a/examples/cb-gmres-solver/CMakeLists.txt b/examples/cb-gmres-solver/CMakeLists.txt
new file mode 100644
index 00000000000..960df2ab077
--- /dev/null
+++ b/examples/cb-gmres-solver/CMakeLists.txt
@@ -0,0 +1,13 @@
+add_executable(cb-gmres-solver cb-gmres-solver.cpp)
+target_link_libraries(cb-gmres-solver ginkgo)
+target_include_directories(cb-gmres-solver PRIVATE ${PROJECT_SOURCE_DIR})
+add_executable(cb-gmres-solver-jacobi cb-gmres-solver-jacobi.cpp)
+target_link_libraries(cb-gmres-solver-jacobi ginkgo)
+target_include_directories(cb-gmres-solver-jacobi PRIVATE ${PROJECT_SOURCE_DIR})
+add_executable(cb-gmres-solver-jacobi-matrix
+cb-gmres-solver-jacobi-matrix.cpp)
+target_link_libraries(cb-gmres-solver-jacobi-matrix ginkgo)
+target_include_directories(cb-gmres-solver-jacobi-matrix PRIVATE ${PROJECT_SOURCE_DIR})
+configure_file(data/A.mtx data/A.mtx COPYONLY)
+configure_file(data/b.mtx data/b.mtx COPYONLY)
+configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/gmres-mixed-solver/build.sh b/examples/cb-gmres-solver/build.sh
similarity index 100%
rename from examples/gmres-mixed-solver/build.sh
rename to examples/cb-gmres-solver/build.sh
diff --git a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp b/examples/cb-gmres-solver/cb-gmres-solver-jacobi-matrix.cpp
similarity index 97%
rename from examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
rename to examples/cb-gmres-solver/cb-gmres-solver-jacobi-matrix.cpp
index a767bda82a3..d6ffaf887d1 100644
--- a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi-matrix.cpp
+++ b/examples/cb-gmres-solver/cb-gmres-solver-jacobi-matrix.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -57,8 +57,8 @@ int main(int argc, char *argv[])
     using mtx = gko::matrix::Csr<>;
     // The gko::solver::Cg is used here, but any other solver class can also be
     // used.
-    using gmres_mixed = gko::solver::GmresMixed<double>;
-    // using gmres_mixed = gko::solver::GmresMixed<>;
+    using cb_gmres = gko::solver::CbGmres<double>;
+    // using cb_gmres = gko::solver::CbGmres<>;
     using bj = gko::preconditioner::Jacobi<>;
 
     // Print the ginkgo version information.
@@ -200,14 +200,14 @@ int main(int argc, char *argv[])
     // @sect3{Creating the solver}
     // Generate the gko::solver factory. Ginkgo uses the concept of Factories to
     // build solvers with certain
-    // properties. Observe the Fluent interface used here. Here a gmres_mixed
+    // properties. Observe the Fluent interface used here. Here a cb_gmres
     // solver is generated with a stopping criteria of maximum iterations of 20
     // and a residual norm reduction of 1e-15. You also observe that the
     // stopping criteria(gko::stop) are also generated from factories using
     // their build methods. You need to specify the executors which each of the
     // object needs to be built on.
     auto solver_gen =
-        gmres_mixed::build()
+        cb_gmres::build()
             .with_criteria(
                 //    gko::stop::Iteration::build().with_max_iters(20u).on(exec),
                 gko::stop::Iteration::build()
@@ -225,7 +225,7 @@ int main(int argc, char *argv[])
             // difference from the simple solver example
             //    .with_preconditioner(bj::build().with_max_block_size(8u).on(exec))
             .with_storage_precision(
-                gko::solver::gmres_mixed_storage_precision::keep)
+                gko::solver::cb_gmres_storage_precision::keep)
             .with_preconditioner(bj::build().with_max_block_size(1u).on(exec))
             .on(exec);
     // Generate the solver from the matrix. The solver factory built in the
diff --git a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi.cpp b/examples/cb-gmres-solver/cb-gmres-solver-jacobi.cpp
similarity index 97%
rename from examples/gmres-mixed-solver/gmres-mixed-solver-jacobi.cpp
rename to examples/cb-gmres-solver/cb-gmres-solver-jacobi.cpp
index 200f9c05ca4..585920029cc 100644
--- a/examples/gmres-mixed-solver/gmres-mixed-solver-jacobi.cpp
+++ b/examples/cb-gmres-solver/cb-gmres-solver-jacobi.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -56,8 +56,8 @@ int main(int argc, char *argv[])
     using mtx = gko::matrix::Csr<>;
     // The gko::solver::Cg is used here, but any other solver class can also be
     // used.
-    //    using gmres_mixed = gko::solver::GmresMixed<double, float>;
-    using gmres_mixed = gko::solver::GmresMixed<>;
+    //    using cb_gmres = gko::solver::CbGmres<double, float>;
+    using cb_gmres = gko::solver::CbGmres<>;
     using bj = gko::preconditioner::Jacobi<>;
 
     // Print the ginkgo version information.
@@ -104,14 +104,14 @@ int main(int argc, char *argv[])
     // @sect3{Creating the solver}
     // Generate the gko::solver factory. Ginkgo uses the concept of Factories to
     // build solvers with certain
-    // properties. Observe the Fluent interface used here. Here a gmres_mixed
+    // properties. Observe the Fluent interface used here. Here a cb_gmres
     // solver is generated with a stopping criteria of maximum iterations of 20
     // and a residual norm reduction of 1e-15. You also observe that the
     // stopping criteria(gko::stop) are also generated from factories using
     // their build methods. You need to specify the executors which each of the
     // object needs to be built on.
     auto solver_gen =
-        gmres_mixed::build()
+        cb_gmres::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(20u).on(exec),
                 gko::stop::ResidualNormReduction<>::build()
diff --git a/examples/gmres-mixed-solver/gmres-mixed-solver.cpp b/examples/cb-gmres-solver/cb-gmres-solver.cpp
similarity index 97%
rename from examples/gmres-mixed-solver/gmres-mixed-solver.cpp
rename to examples/cb-gmres-solver/cb-gmres-solver.cpp
index 12d4f669ca7..bcd2e9dfed2 100644
--- a/examples/gmres-mixed-solver/gmres-mixed-solver.cpp
+++ b/examples/cb-gmres-solver/cb-gmres-solver.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -55,8 +55,8 @@ int main(int argc, char *argv[])
     using mtx = gko::matrix::Csr<>;
     // The gko::solver::Cg is used here, but any other solver class can also be
     // used.
-    //    using gmres_mixed = gko::solver::GmresMixed<double, float>;
-    using gmres_mixed = gko::solver::GmresMixed<>;
+    //    using cb_gmres = gko::solver::CbGmres<double, float>;
+    using cb_gmres = gko::solver::CbGmres<>;
 
     // Print the ginkgo version information.
     std::cout << gko::version_info::get() << std::endl;
@@ -102,14 +102,14 @@ int main(int argc, char *argv[])
     // @sect3{Creating the solver}
     // Generate the gko::solver factory. Ginkgo uses the concept of Factories to
     // build solvers with certain
-    // properties. Observe the Fluent interface used here. Here a gmres_mixed
+    // properties. Observe the Fluent interface used here. Here a cb_gmres
     // solver is generated with a stopping criteria of maximum iterations of 20
     // and a residual norm reduction of 1e-15. You also observe that the
     // stopping criteria(gko::stop) are also generated from factories using
     // their build methods. You need to specify the executors which each of the
     // object needs to be built on.
     auto solver_gen =
-        gmres_mixed::build()
+        cb_gmres::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(20u).on(exec),
                 gko::stop::ResidualNormReduction<>::build()
diff --git a/examples/gmres-mixed-solver/data/A.mtx b/examples/cb-gmres-solver/data/A.mtx
similarity index 100%
rename from examples/gmres-mixed-solver/data/A.mtx
rename to examples/cb-gmres-solver/data/A.mtx
diff --git a/examples/gmres-mixed-solver/data/b.mtx b/examples/cb-gmres-solver/data/b.mtx
similarity index 100%
rename from examples/gmres-mixed-solver/data/b.mtx
rename to examples/cb-gmres-solver/data/b.mtx
diff --git a/examples/gmres-mixed-solver/data/x0.mtx b/examples/cb-gmres-solver/data/x0.mtx
similarity index 100%
rename from examples/gmres-mixed-solver/data/x0.mtx
rename to examples/cb-gmres-solver/data/x0.mtx
diff --git a/examples/gmres-mixed-solver/doc/builds-on b/examples/cb-gmres-solver/doc/builds-on
similarity index 100%
rename from examples/gmres-mixed-solver/doc/builds-on
rename to examples/cb-gmres-solver/doc/builds-on
diff --git a/examples/gmres-mixed-solver/doc/intro.dox b/examples/cb-gmres-solver/doc/intro.dox
similarity index 100%
rename from examples/gmres-mixed-solver/doc/intro.dox
rename to examples/cb-gmres-solver/doc/intro.dox
diff --git a/examples/gmres-mixed-solver/doc/kind b/examples/cb-gmres-solver/doc/kind
similarity index 100%
rename from examples/gmres-mixed-solver/doc/kind
rename to examples/cb-gmres-solver/doc/kind
diff --git a/examples/gmres-mixed-solver/doc/results.dox b/examples/cb-gmres-solver/doc/results.dox
similarity index 100%
rename from examples/gmres-mixed-solver/doc/results.dox
rename to examples/cb-gmres-solver/doc/results.dox
diff --git a/examples/gmres-mixed-solver/doc/short-intro b/examples/cb-gmres-solver/doc/short-intro
similarity index 100%
rename from examples/gmres-mixed-solver/doc/short-intro
rename to examples/cb-gmres-solver/doc/short-intro
diff --git a/examples/gmres-mixed-solver/doc/tooltip b/examples/cb-gmres-solver/doc/tooltip
similarity index 100%
rename from examples/gmres-mixed-solver/doc/tooltip
rename to examples/cb-gmres-solver/doc/tooltip
diff --git a/examples/gmres-mixed-solver/CMakeLists.txt b/examples/gmres-mixed-solver/CMakeLists.txt
deleted file mode 100644
index 467dfff7d96..00000000000
--- a/examples/gmres-mixed-solver/CMakeLists.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-add_executable(gmres-mixed-solver gmres-mixed-solver.cpp)
-target_link_libraries(gmres-mixed-solver ginkgo)
-target_include_directories(gmres-mixed-solver PRIVATE ${PROJECT_SOURCE_DIR})
-add_executable(gmres-mixed-solver-jacobi gmres-mixed-solver-jacobi.cpp)
-target_link_libraries(gmres-mixed-solver-jacobi ginkgo)
-target_include_directories(gmres-mixed-solver-jacobi PRIVATE ${PROJECT_SOURCE_DIR})
-add_executable(gmres-mixed-solver-jacobi-matrix
-gmres-mixed-solver-jacobi-matrix.cpp)
-target_link_libraries(gmres-mixed-solver-jacobi-matrix ginkgo)
-target_include_directories(gmres-mixed-solver-jacobi-matrix PRIVATE ${PROJECT_SOURCE_DIR})
-configure_file(data/A.mtx data/A.mtx COPYONLY)
-configure_file(data/b.mtx data/b.mtx COPYONLY)
-configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/gmres-solver/gmres-solver.cpp b/examples/gmres-solver/gmres-solver.cpp
index a8c97b2b474..e782aca1a6f 100644
--- a/examples/gmres-solver/gmres-solver.cpp
+++ b/examples/gmres-solver/gmres-solver.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 246bb42635d..4999cbd8e0b 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -204,7 +204,7 @@ set(GINKGO_HIP_SOURCES
     solver/cgs_kernels.hip.cpp
     solver/fcg_kernels.hip.cpp
     solver/gmres_kernels.hip.cpp
-    solver/gmres_mixed_kernels.hip.cpp
+    solver/cb_gmres_kernels.hip.cpp
     solver/idr_kernels.hip.cpp
     solver/ir_kernels.hip.cpp
     solver/lower_trs_kernels.hip.cpp
diff --git a/hip/solver/gmres_mixed_kernels.hip.cpp b/hip/solver/cb_gmres_kernels.hip.cpp
similarity index 97%
rename from hip/solver/gmres_mixed_kernels.hip.cpp
rename to hip/solver/cb_gmres_kernels.hip.cpp
index 8858f4f992a..941e7e6e69f 100644
--- a/hip/solver/gmres_mixed_kernels.hip.cpp
+++ b/hip/solver/cb_gmres_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/solver/gmres_mixed_kernels.hpp"
+#include "core/solver/cb_gmres_kernels.hpp"
 
 
 #include <algorithm>
@@ -57,11 +57,11 @@ namespace gko {
 namespace kernels {
 namespace hip {
 /**
- * @brief The GMRES_MIXED solver namespace.
+ * @brief The CB_GMRES solver namespace.
  *
- * @ingroup gmres_mixed
+ * @ingroup cb_gmres
  */
-namespace gmres_mixed {
+namespace cb_gmres {
 
 
 constexpr int default_block_size = 512;
@@ -71,7 +71,7 @@ constexpr int default_dot_dim = 32;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
-#include "common/solver/gmres_mixed_kernels.hpp.inc"
+#include "common/solver/cb_gmres_kernels.hpp.inc"
 
 
 // Specialization, so the Accessor can use the same function as regular pointers
@@ -137,8 +137,7 @@ void initialize_1(std::shared_ptr<const HipExecutor> exec,
         as_hip_type(stop_status->get_data()));
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CB_GMRES_INITIALIZE_1_KERNEL);
 
 
 template <typename ValueType, typename Accessor3d>
@@ -200,8 +199,8 @@ void initialize_2(std::shared_ptr<const HipExecutor> exec,
                        as_hip_type(final_iter_nums->get_data()));
 }
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
-    GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(
+    GKO_DECLARE_CB_GMRES_INITIALIZE_2_KERNEL);
 
 
 template <typename ValueType, typename Accessor3dim>
@@ -456,8 +455,7 @@ void step_1(std::shared_ptr<const HipExecutor> exec,
                     stop_status);
 }
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
-    GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(GKO_DECLARE_CB_GMRES_STEP_1_KERNEL);
 
 
 template <typename ValueType>
@@ -535,11 +533,11 @@ void step_2(std::shared_ptr<const HipExecutor> exec,
 }
 
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(
-    GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_CONST_TYPE(
+    GKO_DECLARE_CB_GMRES_STEP_2_KERNEL);
 
 
-}  // namespace gmres_mixed
+}  // namespace cb_gmres
 }  // namespace hip
 }  // namespace kernels
 }  // namespace gko
diff --git a/hip/test/solver/CMakeLists.txt b/hip/test/solver/CMakeLists.txt
index 9bf14e276e3..1e36230d874 100644
--- a/hip/test/solver/CMakeLists.txt
+++ b/hip/test/solver/CMakeLists.txt
@@ -4,7 +4,7 @@ ginkgo_create_hip_test_special_linkage(cg_kernels)
 ginkgo_create_hip_test_special_linkage(cgs_kernels)
 ginkgo_create_hip_test_special_linkage(fcg_kernels)
 ginkgo_create_hip_test_special_linkage(gmres_kernels)
-ginkgo_create_hip_test_special_linkage(gmres_mixed_kernels)
+ginkgo_create_hip_test_special_linkage(cb_gmres_kernels)
 ginkgo_create_hip_test_special_linkage(idr_kernels)
 ginkgo_create_hip_test_special_linkage(ir_kernels)
 ginkgo_create_hip_test_special_linkage(lower_trs_kernels)
diff --git a/hip/test/solver/gmres_mixed_kernels.cpp b/hip/test/solver/cb_gmres_kernels.cpp
similarity index 94%
rename from hip/test/solver/gmres_mixed_kernels.cpp
rename to hip/test/solver/cb_gmres_kernels.cpp
index 4247ebe26ee..de58c3f140b 100644
--- a/hip/test/solver/gmres_mixed_kernels.cpp
+++ b/hip/test/solver/cb_gmres_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <ginkgo/core/solver/gmres_mixed.hpp>
+#include <ginkgo/core/solver/cb_gmres.hpp>
 
 
 #include <random>
@@ -47,15 +47,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/residual_norm_reduction.hpp>
 
 
-#include "core/solver/gmres_mixed_accessor.hpp"
-#include "core/solver/gmres_mixed_kernels.hpp"
+#include "core/solver/cb_gmres_accessor.hpp"
+#include "core/solver/cb_gmres_kernels.hpp"
 #include "core/test/utils.hpp"
 
 
 namespace {
 
 
-class GmresMixed : public ::testing::Test {
+class CbGmres : public ::testing::Test {
 protected:
     using value_type = double;
     using storage_type = float;
@@ -67,7 +67,7 @@ class GmresMixed : public ::testing::Test {
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
 
-    GmresMixed() : rand_engine(30) {}
+    CbGmres() : rand_engine(30) {}
 
     void SetUp()
     {
@@ -270,15 +270,15 @@ class GmresMixed : public ::testing::Test {
 };
 
 
-TEST_F(GmresMixed, HipGmresMixedInitialize1IsEquivalentToRef)
+TEST_F(CbGmres, HipCbGmresInitialize1IsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::gmres_mixed::initialize_1(
+    gko::kernels::reference::cb_gmres::initialize_1(
         ref, b.get(), b_norm.get(), residual.get(), givens_sin.get(),
         givens_cos.get(), stop_status.get(),
         gko::solver::default_krylov_dim_mixed);
-    gko::kernels::hip::gmres_mixed::initialize_1(
+    gko::kernels::hip::cb_gmres::initialize_1(
         hip, d_b.get(), d_b_norm.get(), d_residual.get(), d_givens_sin.get(),
         d_givens_cos.get(), d_stop_status.get(),
         gko::solver::default_krylov_dim_mixed);
@@ -291,16 +291,16 @@ TEST_F(GmresMixed, HipGmresMixedInitialize1IsEquivalentToRef)
 }
 
 
-TEST_F(GmresMixed, HipGmresMixedInitialize2IsEquivalentToRef)
+TEST_F(CbGmres, HipCbGmresInitialize2IsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::gmres_mixed::initialize_2(
+    gko::kernels::reference::cb_gmres::initialize_2(
         ref, residual.get(), residual_norm.get(),
         residual_norm_collection.get(), arnoldi_norm.get(),
         range_helper.get_range(), next_krylov_basis.get(),
         final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
-    gko::kernels::hip::gmres_mixed::initialize_2(
+    gko::kernels::hip::cb_gmres::initialize_2(
         hip, d_residual.get(), d_residual_norm.get(),
         d_residual_norm_collection.get(), d_arnoldi_norm.get(),
         d_range_helper.get_range(), d_next_krylov_basis.get(),
@@ -314,21 +314,21 @@ TEST_F(GmresMixed, HipGmresMixedInitialize2IsEquivalentToRef)
 }
 
 
-TEST_F(GmresMixed, HipGmresMixedStep1IsEquivalentToRef)
+TEST_F(CbGmres, HipCbGmresStep1IsEquivalentToRef)
 {
     initialize_data();
     int iter = 5;
     int num_reorth_steps = 0, num_reorth_vectors = 0;
     int d_num_reorth_steps = 0, d_num_reorth_vectors = 0;
 
-    gko::kernels::reference::gmres_mixed::step_1(
+    gko::kernels::reference::cb_gmres::step_1(
         ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
         residual_norm.get(), residual_norm_collection.get(),
         range_helper.get_range(), hessenberg_iter.get(), buffer_iter.get(),
         b_norm.get(), arnoldi_norm.get(), iter, final_iter_nums.get(),
         stop_status.get(), reorth_status.get(), num_reorth.get(),
         &num_reorth_steps, &num_reorth_vectors);
-    gko::kernels::hip::gmres_mixed::step_1(
+    gko::kernels::hip::cb_gmres::step_1(
         hip, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
         d_residual_norm.get(), d_residual_norm_collection.get(),
         d_range_helper.get_range(), d_hessenberg_iter.get(),
@@ -348,15 +348,15 @@ TEST_F(GmresMixed, HipGmresMixedStep1IsEquivalentToRef)
 }
 
 
-TEST_F(GmresMixed, HipGmresMixedStep2IsEquivalentToRef)
+TEST_F(CbGmres, HipCbGmresStep2IsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::gmres_mixed::step_2(
+    gko::kernels::reference::cb_gmres::step_2(
         ref, residual_norm_collection.get(),
         range_helper.get_range().get_accessor().to_const(), hessenberg.get(),
         y.get(), before_preconditioner.get(), final_iter_nums.get());
-    gko::kernels::hip::gmres_mixed::step_2(
+    gko::kernels::hip::cb_gmres::step_2(
         hip, d_residual_norm_collection.get(),
         d_range_helper.get_range().get_accessor().to_const(),
         d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
diff --git a/include/ginkgo/core/solver/gmres_mixed.hpp b/include/ginkgo/core/solver/cb_gmres.hpp
similarity index 82%
rename from include/ginkgo/core/solver/gmres_mixed.hpp
rename to include/ginkgo/core/solver/cb_gmres.hpp
index 13145be56c6..aab1821edf5 100644
--- a/include/ginkgo/core/solver/gmres_mixed.hpp
+++ b/include/ginkgo/core/solver/cb_gmres.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_GMRES_MIXED_HPP_
-#define GKO_CORE_SOLVER_GMRES_MIXED_HPP_
+#ifndef GKO_CORE_SOLVER_CB_GMRES_HPP_
+#define GKO_CORE_SOLVER_CB_GMRES_HPP_
 
 
 #include <vector>
@@ -55,7 +55,7 @@ namespace solver {
 constexpr size_type default_krylov_dim_mixed = 100u;
 
 
-enum class gmres_mixed_storage_precision {
+enum class cb_gmres_storage_precision {
     keep,
     reduce1,
     reduce2,
@@ -66,11 +66,11 @@ enum class gmres_mixed_storage_precision {
 
 
 /**
- * GMRES_MIXED or the generalized minimal residual method is an iterative type
+ * CB-GMRES or the generalized minimal residual method is an iterative type
  * Krylov subspace method which is suitable for nonsymmetric linear systems.
  *
  * The implementation in Ginkgo makes use of the merged kernel to make the best
- * use of data locality. The inner operations in one iteration of GMRES_MIXED
+ * use of data locality. The inner operations in one iteration of CB-GMRES
  * are merged into 2 separate steps.
  *
  * @tparam ValueType  precision of matrix elements
@@ -79,10 +79,10 @@ enum class gmres_mixed_storage_precision {
  * @ingroup LinOp
  */
 template <typename ValueType = default_precision>
-class GmresMixed : public EnableLinOp<GmresMixed<ValueType>>,
-                   public Preconditionable {
-    friend class EnableLinOp<GmresMixed>;
-    friend class EnablePolymorphicObject<GmresMixed, LinOp>;
+class CbGmres : public EnableLinOp<CbGmres<ValueType>>,
+                public Preconditionable {
+    friend class EnableLinOp<CbGmres>;
+    friend class EnablePolymorphicObject<CbGmres, LinOp>;
 
 public:
     using value_type = ValueType;
@@ -109,8 +109,8 @@ class GmresMixed : public EnableLinOp<GmresMixed<ValueType>>,
         /**
          * Determines which storage type is used.
          */
-        gmres_mixed_storage_precision GKO_FACTORY_PARAMETER_SCALAR(
-            storage_precision, gmres_mixed_storage_precision::keep);
+        cb_gmres_storage_precision GKO_FACTORY_PARAMETER_SCALAR(
+            storage_precision, cb_gmres_storage_precision::keep);
 
         /**
          * Criterion factories.
@@ -136,7 +136,7 @@ class GmresMixed : public EnableLinOp<GmresMixed<ValueType>>,
          */
         size_type GKO_FACTORY_PARAMETER_SCALAR(krylov_dim, 0u);
     };
-    GKO_ENABLE_LIN_OP_FACTORY(GmresMixed, parameters, Factory);
+    GKO_ENABLE_LIN_OP_FACTORY(CbGmres, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);
 
 protected:
@@ -145,14 +145,14 @@ class GmresMixed : public EnableLinOp<GmresMixed<ValueType>>,
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override;
 
-    explicit GmresMixed(std::shared_ptr<const Executor> exec)
-        : EnableLinOp<GmresMixed>(std::move(exec))
+    explicit CbGmres(std::shared_ptr<const Executor> exec)
+        : EnableLinOp<CbGmres>(std::move(exec))
     {}
 
-    explicit GmresMixed(const Factory *factory,
-                        std::shared_ptr<const LinOp> system_matrix)
-        : EnableLinOp<GmresMixed>(factory->get_executor(),
-                                  transpose(system_matrix->get_size())),
+    explicit CbGmres(const Factory *factory,
+                     std::shared_ptr<const LinOp> system_matrix)
+        : EnableLinOp<CbGmres>(factory->get_executor(),
+                               transpose(system_matrix->get_size())),
           parameters_{factory->get_parameters()},
           system_matrix_{std::move(system_matrix)}
     {
@@ -181,7 +181,7 @@ class GmresMixed : public EnableLinOp<GmresMixed<ValueType>>,
     std::shared_ptr<const LinOp> system_matrix_{};
     std::shared_ptr<const stop::CriterionFactory> stop_criterion_factory_{};
     size_type krylov_dim_;
-    gmres_mixed_storage_precision storage_precision_;
+    cb_gmres_storage_precision storage_precision_;
 };
 
 
@@ -189,4 +189,4 @@ class GmresMixed : public EnableLinOp<GmresMixed<ValueType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_GMRES_MIXED_HPP_
+#endif  // GKO_CORE_SOLVER_CB_GMRES_HPP_
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index fc5e904fb0d..b883785fbeb 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -97,11 +97,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/solver/bicg.hpp>
 #include <ginkgo/core/solver/bicgstab.hpp>
+#include <ginkgo/core/solver/cb_gmres.hpp>
 #include <ginkgo/core/solver/cg.hpp>
 #include <ginkgo/core/solver/cgs.hpp>
 #include <ginkgo/core/solver/fcg.hpp>
 #include <ginkgo/core/solver/gmres.hpp>
-#include <ginkgo/core/solver/gmres_mixed.hpp>
 #include <ginkgo/core/solver/idr.hpp>
 #include <ginkgo/core/solver/ir.hpp>
 #include <ginkgo/core/solver/lower_trs.hpp>
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index d4a774f298f..fcb8798a045 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -32,7 +32,7 @@ target_sources(ginkgo_omp
     solver/cgs_kernels.cpp
     solver/fcg_kernels.cpp
     solver/gmres_kernels.cpp
-    solver/gmres_mixed_kernels.cpp
+    solver/cb_gmres_kernels.cpp
     solver/idr_kernels.cpp
     solver/ir_kernels.cpp
     solver/lower_trs_kernels.cpp
diff --git a/omp/solver/gmres_mixed_kernels.cpp b/omp/solver/cb_gmres_kernels.cpp
similarity index 97%
rename from omp/solver/gmres_mixed_kernels.cpp
rename to omp/solver/cb_gmres_kernels.cpp
index fa1b66afa79..acdccd59b06 100644
--- a/omp/solver/gmres_mixed_kernels.cpp
+++ b/omp/solver/cb_gmres_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/solver/gmres_mixed_kernels.hpp"
+#include "core/solver/cb_gmres_kernels.hpp"
 
 
 #include <omp.h>
@@ -43,7 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/solver/gmres_mixed.hpp>
+#include <ginkgo/core/solver/cb_gmres.hpp>
 
 
 #include <iostream>
@@ -61,11 +61,11 @@ namespace gko {
 namespace kernels {
 namespace omp {
 /**
- * @brief The GMRES_MIXED solver namespace.
+ * @brief The CB_GMRES solver namespace.
  *
- * @ingroup gmres_mixed
+ * @ingroup cb_gmres
  */
-namespace gmres_mixed {
+namespace cb_gmres {
 
 
 namespace {
@@ -385,8 +385,7 @@ void initialize_1(std::shared_ptr<const OmpExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CB_GMRES_INITIALIZE_1_KERNEL);
 
 
 template <typename ValueType, typename Accessor3d>
@@ -452,8 +451,8 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
-    GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(
+    GKO_DECLARE_CB_GMRES_INITIALIZE_2_KERNEL);
 
 
 template <typename ValueType, typename Accessor3d>
@@ -487,8 +486,7 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
                                  stop_status->get_const_data());
 }
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
-    GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(GKO_DECLARE_CB_GMRES_STEP_1_KERNEL);
 
 
 template <typename ValueType, typename ConstAccessor3d>
@@ -506,11 +504,11 @@ void step_2(std::shared_ptr<const OmpExecutor> exec,
                  final_iter_nums->get_const_data());
 }
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(
-    GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_CONST_TYPE(
+    GKO_DECLARE_CB_GMRES_STEP_2_KERNEL);
 
 
-}  // namespace gmres_mixed
+}  // namespace cb_gmres
 }  // namespace omp
 }  // namespace kernels
 }  // namespace gko
diff --git a/omp/test/solver/CMakeLists.txt b/omp/test/solver/CMakeLists.txt
index de3fce1fb78..a69af95bd22 100644
--- a/omp/test/solver/CMakeLists.txt
+++ b/omp/test/solver/CMakeLists.txt
@@ -4,7 +4,7 @@ ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
-ginkgo_create_test(gmres_mixed_kernels)
+ginkgo_create_test(cb_gmres_kernels)
 ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
 ginkgo_create_test(lower_trs_kernels)
diff --git a/omp/test/solver/gmres_mixed_kernels.cpp b/omp/test/solver/cb_gmres_kernels.cpp
similarity index 94%
rename from omp/test/solver/gmres_mixed_kernels.cpp
rename to omp/test/solver/cb_gmres_kernels.cpp
index dec133547be..d0037bb82ac 100644
--- a/omp/test/solver/gmres_mixed_kernels.cpp
+++ b/omp/test/solver/cb_gmres_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/solver/gmres_mixed_kernels.hpp"
+#include "core/solver/cb_gmres_kernels.hpp"
 
 
 #include <algorithm>
@@ -44,20 +44,20 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/solver/gmres_mixed.hpp>
+#include <ginkgo/core/solver/cb_gmres.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm_reduction.hpp>
 
 
-#include "core/solver/gmres_mixed_accessor.hpp"
+#include "core/solver/cb_gmres_accessor.hpp"
 #include "core/test/utils.hpp"
 
 
 namespace {
 
 
-class GmresMixed : public ::testing::Test {
+class CbGmres : public ::testing::Test {
 protected:
     using value_type = double;
     using storage_type = float;
@@ -70,7 +70,7 @@ class GmresMixed : public ::testing::Test {
     using Mtx = Dense;
 
 
-    GmresMixed() : rand_engine(30) {}
+    CbGmres() : rand_engine(30) {}
 
     void SetUp()
     {
@@ -271,15 +271,15 @@ class GmresMixed : public ::testing::Test {
 };
 
 
-TEST_F(GmresMixed, OmpGmresMixedInitialize1IsEquivalentToRef)
+TEST_F(CbGmres, OmpCbGmresInitialize1IsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::gmres_mixed::initialize_1(
+    gko::kernels::reference::cb_gmres::initialize_1(
         ref, b.get(), b_norm.get(), residual.get(), givens_sin.get(),
         givens_cos.get(), stop_status.get(),
         gko::solver::default_krylov_dim_mixed);
-    gko::kernels::omp::gmres_mixed::initialize_1(
+    gko::kernels::omp::cb_gmres::initialize_1(
         omp, d_b.get(), d_b_norm.get(), d_residual.get(), d_givens_sin.get(),
         d_givens_cos.get(), d_stop_status.get(),
         gko::solver::default_krylov_dim_mixed);
@@ -291,16 +291,16 @@ TEST_F(GmresMixed, OmpGmresMixedInitialize1IsEquivalentToRef)
     GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
 }
 
-TEST_F(GmresMixed, OmpGmresMixedInitialize2IsEquivalentToRef)
+TEST_F(CbGmres, OmpCbGmresInitialize2IsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::gmres_mixed::initialize_2(
+    gko::kernels::reference::cb_gmres::initialize_2(
         ref, residual.get(), residual_norm.get(),
         residual_norm_collection.get(), arnoldi_norm.get(),
         range_helper.get_range(), next_krylov_basis.get(),
         final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
-    gko::kernels::omp::gmres_mixed::initialize_2(
+    gko::kernels::omp::cb_gmres::initialize_2(
         omp, d_residual.get(), d_residual_norm.get(),
         d_residual_norm_collection.get(), d_arnoldi_norm.get(),
         d_range_helper.get_range(), d_next_krylov_basis.get(),
@@ -313,21 +313,21 @@ TEST_F(GmresMixed, OmpGmresMixedInitialize2IsEquivalentToRef)
     GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
 }
 
-TEST_F(GmresMixed, OmpGmresMixedStep1IsEquivalentToRef)
+TEST_F(CbGmres, OmpCbGmresStep1IsEquivalentToRef)
 {
     initialize_data();
     int iter = 5;
     int num_reorth_steps = 0, num_reorth_vectors = 0;
     int d_num_reorth_steps = 0, d_num_reorth_vectors = 0;
 
-    gko::kernels::reference::gmres_mixed::step_1(
+    gko::kernels::reference::cb_gmres::step_1(
         ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
         residual_norm.get(), residual_norm_collection.get(),
         range_helper.get_range(), hessenberg_iter.get(), buffer_iter.get(),
         b_norm.get(), arnoldi_norm.get(), iter, final_iter_nums.get(),
         stop_status.get(), reorth_status.get(), num_reorth.get(),
         &num_reorth_steps, &num_reorth_vectors);
-    gko::kernels::omp::gmres_mixed::step_1(
+    gko::kernels::omp::cb_gmres::step_1(
         omp, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
         d_residual_norm.get(), d_residual_norm_collection.get(),
         d_range_helper.get_range(), d_hessenberg_iter.get(),
@@ -346,15 +346,15 @@ TEST_F(GmresMixed, OmpGmresMixedStep1IsEquivalentToRef)
     GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
 }
 
-TEST_F(GmresMixed, OmpGmresMixedStep2IsEquivalentToRef)
+TEST_F(CbGmres, OmpCbGmresStep2IsEquivalentToRef)
 {
     initialize_data();
 
-    gko::kernels::reference::gmres_mixed::step_2(
+    gko::kernels::reference::cb_gmres::step_2(
         ref, residual_norm_collection.get(),
         range_helper.get_range().get_accessor().to_const(), hessenberg.get(),
         y.get(), before_preconditioner.get(), final_iter_nums.get());
-    gko::kernels::omp::gmres_mixed::step_2(
+    gko::kernels::omp::cb_gmres::step_2(
         omp, d_residual_norm_collection.get(),
         d_range_helper.get_range().get_accessor().to_const(),
         d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index 8951092dcfe..7333eac1d12 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -30,7 +30,7 @@ target_sources(ginkgo_reference
     solver/cgs_kernels.cpp
     solver/fcg_kernels.cpp
     solver/gmres_kernels.cpp
-    solver/gmres_mixed_kernels.cpp
+    solver/cb_gmres_kernels.cpp
     solver/idr_kernels.cpp
     solver/ir_kernels.cpp
     solver/lower_trs_kernels.cpp
diff --git a/reference/solver/gmres_mixed_kernels.cpp b/reference/solver/cb_gmres_kernels.cpp
similarity index 96%
rename from reference/solver/gmres_mixed_kernels.cpp
rename to reference/solver/cb_gmres_kernels.cpp
index a6476b030c1..4105eceffc4 100644
--- a/reference/solver/gmres_mixed_kernels.cpp
+++ b/reference/solver/cb_gmres_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -30,30 +30,30 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/solver/gmres_mixed_kernels.hpp"
+#include "core/solver/cb_gmres_kernels.hpp"
 
 
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/solver/gmres_mixed.hpp>
+#include <ginkgo/core/solver/cb_gmres.hpp>
 
 #include <iostream>
 
 
-#include "core/solver/gmres_mixed_accessor.hpp"
+#include "core/solver/cb_gmres_accessor.hpp"
 
 
 namespace gko {
 namespace kernels {
 namespace reference {
 /**
- * @brief The GMRES_MIXED solver namespace.
+ * @brief The CB_GMRES solver namespace.
  *
- * @ingroup gmres_mixed
+ * @ingroup cb_gmres
  */
-namespace gmres_mixed {
+namespace cb_gmres {
 
 
 namespace {
@@ -329,8 +329,7 @@ void initialize_1(std::shared_ptr<const ReferenceExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_GMRES_MIXED_INITIALIZE_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CB_GMRES_INITIALIZE_1_KERNEL);
 
 
 template <typename ValueType, typename Accessor3d>
@@ -390,8 +389,8 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
-    GKO_DECLARE_GMRES_MIXED_INITIALIZE_2_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(
+    GKO_DECLARE_CB_GMRES_INITIALIZE_2_KERNEL);
 
 
 // template <typename ValueType, typename ValueTypeKrylovBases, bool
@@ -430,8 +429,7 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
                                  stop_status->get_const_data());
 }
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_TYPE(
-    GKO_DECLARE_GMRES_MIXED_STEP_1_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(GKO_DECLARE_CB_GMRES_STEP_1_KERNEL);
 
 
 template <typename ValueType, typename ConstAccessor3d>
@@ -449,11 +447,11 @@ void step_2(std::shared_ptr<const ReferenceExecutor> exec,
                  final_iter_nums->get_const_data());
 }
 
-GKO_INSTANTIATE_FOR_EACH_GMRES_MIXED_CONST_TYPE(
-    GKO_DECLARE_GMRES_MIXED_STEP_2_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_CONST_TYPE(
+    GKO_DECLARE_CB_GMRES_STEP_2_KERNEL);
 
 
-}  // namespace gmres_mixed
+}  // namespace cb_gmres
 }  // namespace reference
 }  // namespace kernels
 }  // namespace gko
diff --git a/reference/test/solver/CMakeLists.txt b/reference/test/solver/CMakeLists.txt
index 1ad4bc4ddbf..e14582fabba 100644
--- a/reference/test/solver/CMakeLists.txt
+++ b/reference/test/solver/CMakeLists.txt
@@ -4,7 +4,7 @@ ginkgo_create_test(cg_kernels)
 ginkgo_create_test(cgs_kernels)
 ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
-ginkgo_create_test(gmres_mixed_kernels)
+ginkgo_create_test(cb_gmres_kernels)
 ginkgo_create_test(idr_kernels)
 ginkgo_create_test(ir_kernels)
 ginkgo_create_test(lower_trs)
diff --git a/reference/test/solver/gmres_mixed_kernels.cpp b/reference/test/solver/cb_gmres_kernels.cpp
similarity index 88%
rename from reference/test/solver/gmres_mixed_kernels.cpp
rename to reference/test/solver/cb_gmres_kernels.cpp
index d190df910b0..2a85dcf7920 100644
--- a/reference/test/solver/gmres_mixed_kernels.cpp
+++ b/reference/test/solver/cb_gmres_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <ginkgo/core/solver/gmres_mixed.hpp>
+#include <ginkgo/core/solver/cb_gmres.hpp>
 
 
 #include <type_traits>
@@ -56,7 +56,7 @@ namespace {
 
 
 template <typename ValueEnumType>
-class GmresMixed : public ::testing::Test {
+class CbGmres : public ::testing::Test {
 protected:
     using value_type =
         typename std::tuple_element<0, decltype(ValueEnumType())>::type;
@@ -64,7 +64,7 @@ class GmresMixed : public ::testing::Test {
     using storage_helper_type =
         typename std::tuple_element<1, decltype(ValueEnumType())>::type;
     using Mtx = gko::matrix::Dense<value_type>;
-    using gmres_type = gko::solver::GmresMixed<value_type>;
+    using gmres_type = gko::solver::CbGmres<value_type>;
 
     /**
      * Used to extract the precision of the given value_type.
@@ -83,14 +83,14 @@ class GmresMixed : public ::testing::Test {
                 : std::is_same<rc_type, float>::value ? 1e-6 : 1e-3;
     };
 
-    GmresMixed()
+    CbGmres()
         : exec(gko::ReferenceExecutor::create()),
           mtx(gko::initialize<Mtx>(
               {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)),
           mtx2(gko::initialize<Mtx>(
               {{1.0, 2.0, 3.0}, {4.0, 2.0, 1.0}, {0.0, 1.0, 2.0}}, exec)),
           storage_precision{storage_helper_type::value},
-          gmres_mixed_factory(
+          cb_gmres_factory(
               gmres_type::build()
                   .with_storage_precision(storage_precision)
                   .with_criteria(
@@ -111,7 +111,7 @@ class GmresMixed : public ::testing::Test {
                {-365.0, -715.8, 870.7, 67.5, 279.8, 1927.8},
                {-848.1, -280.5, -381.8, -187.1, 51.2, -176.2}},
               exec)),
-          gmres_mixed_factory_big(
+          cb_gmres_factory_big(
               gmres_type::build()
                   .with_storage_precision(storage_precision)
                   .with_criteria(
@@ -133,20 +133,20 @@ class GmresMixed : public ::testing::Test {
     nc_value_type precision() const noexcept
     {
         using gko::reduce_precision;
-        using gko::solver::gmres_mixed_storage_precision;
+        using gko::solver::cb_gmres_storage_precision;
 
         // Note: integer and floating point are assumed to have similar
         //       target precision.
         switch (storage_precision) {
-        case gmres_mixed_storage_precision::reduce1:
-        case gmres_mixed_storage_precision::ireduce1:
+        case cb_gmres_storage_precision::reduce1:
+        case cb_gmres_storage_precision::ireduce1:
             return precision_target<reduce_precision<value_type>>::value * 1e-2;
-        case gmres_mixed_storage_precision::reduce2:
-        case gmres_mixed_storage_precision::ireduce2:
+        case cb_gmres_storage_precision::reduce2:
+        case cb_gmres_storage_precision::ireduce2:
             return precision_target<
                        reduce_precision<reduce_precision<value_type>>>::value *
                    1e-2;
-        case gmres_mixed_storage_precision::integer:
+        case cb_gmres_storage_precision::integer:
         default:
             return precision_target<value_type>::value;
         }
@@ -157,9 +157,9 @@ class GmresMixed : public ::testing::Test {
     std::shared_ptr<Mtx> mtx2;
     std::shared_ptr<Mtx> mtx_medium;
     std::shared_ptr<Mtx> mtx_big;
-    gko::solver::gmres_mixed_storage_precision storage_precision;
-    std::unique_ptr<typename gmres_type::Factory> gmres_mixed_factory;
-    std::unique_ptr<typename gmres_type::Factory> gmres_mixed_factory_big;
+    gko::solver::cb_gmres_storage_precision storage_precision;
+    std::unique_ptr<typename gmres_type::Factory> cb_gmres_factory;
+    std::unique_ptr<typename gmres_type::Factory> cb_gmres_factory_big;
 };
 
 
@@ -167,7 +167,7 @@ class GmresMixed : public ::testing::Test {
  * This creates a helper structure which translates a type into an enum
  * parameter.
  */
-using st_enum = gko::solver::gmres_mixed_storage_precision;
+using st_enum = gko::solver::cb_gmres_storage_precision;
 
 template <st_enum P>
 struct st_helper_type {
@@ -193,14 +193,14 @@ using TestTypes =
                      std::tuple<std::complex<double>, st_r2>,
                      std::tuple<std::complex<float>, st_keep>>;
 
-TYPED_TEST_CASE(GmresMixed, TestTypes);
+TYPED_TEST_CASE(CbGmres, TestTypes);
 
 
-TYPED_TEST(GmresMixed, SolvesStencilSystem)
+TYPED_TEST(CbGmres, SolvesStencilSystem)
 {
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
-    auto solver = this->gmres_mixed_factory->generate(this->mtx);
+    auto solver = this->cb_gmres_factory->generate(this->mtx);
     auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, this->exec);
     auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
 
@@ -210,7 +210,7 @@ TYPED_TEST(GmresMixed, SolvesStencilSystem)
 }
 
 
-TYPED_TEST(GmresMixed, SolvesStencilSystem2)
+TYPED_TEST(CbGmres, SolvesStencilSystem2)
 {
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
@@ -238,11 +238,11 @@ TYPED_TEST(GmresMixed, SolvesStencilSystem2)
 }
 
 
-TYPED_TEST(GmresMixed, SolvesMultipleStencilSystems)
+TYPED_TEST(CbGmres, SolvesMultipleStencilSystems)
 {
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
-    auto solver = this->gmres_mixed_factory->generate(this->mtx);
+    auto solver = this->cb_gmres_factory->generate(this->mtx);
     auto b = gko::initialize<Mtx>(
         {I<T>{13.0, 6.0}, I<T>{7.0, 4.0}, I<T>{1.0, 1.0}}, this->exec);
     auto x = gko::initialize<Mtx>(
@@ -255,11 +255,11 @@ TYPED_TEST(GmresMixed, SolvesMultipleStencilSystems)
 }
 
 
-TYPED_TEST(GmresMixed, SolvesStencilSystemUsingAdvancedApply)
+TYPED_TEST(CbGmres, SolvesStencilSystemUsingAdvancedApply)
 {
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
-    auto solver = this->gmres_mixed_factory->generate(this->mtx);
+    auto solver = this->cb_gmres_factory->generate(this->mtx);
     auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
     auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
     auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, this->exec);
@@ -271,11 +271,11 @@ TYPED_TEST(GmresMixed, SolvesStencilSystemUsingAdvancedApply)
 }
 
 
-TYPED_TEST(GmresMixed, SolvesMultipleStencilSystemsUsingAdvancedApply)
+TYPED_TEST(CbGmres, SolvesMultipleStencilSystemsUsingAdvancedApply)
 {
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
-    auto solver = this->gmres_mixed_factory->generate(this->mtx);
+    auto solver = this->cb_gmres_factory->generate(this->mtx);
     auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
     auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
     auto b = gko::initialize<Mtx>(
@@ -290,11 +290,11 @@ TYPED_TEST(GmresMixed, SolvesMultipleStencilSystemsUsingAdvancedApply)
 }
 
 
-TYPED_TEST(GmresMixed, SolvesBigDenseSystem1)
+TYPED_TEST(CbGmres, SolvesBigDenseSystem1)
 {
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
-    auto solver = this->gmres_mixed_factory_big->generate(this->mtx_big);
+    auto solver = this->cb_gmres_factory_big->generate(this->mtx_big);
     auto b = gko::initialize<Mtx>(
         {72748.36, 297469.88, 347229.24, 36290.66, 82958.82, -80192.15},
         this->exec);
@@ -307,11 +307,11 @@ TYPED_TEST(GmresMixed, SolvesBigDenseSystem1)
 }
 
 
-TYPED_TEST(GmresMixed, SolvesBigDenseSystem2)
+TYPED_TEST(CbGmres, SolvesBigDenseSystem2)
 {
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
-    auto solver = this->gmres_mixed_factory_big->generate(this->mtx_big);
+    auto solver = this->cb_gmres_factory_big->generate(this->mtx_big);
     auto b = gko::initialize<Mtx>(
         {175352.10, 313410.50, 131114.10, -134116.30, 179529.30, -43564.90},
         this->exec);
@@ -338,11 +338,11 @@ gko::remove_complex<T> infNorm(gko::matrix::Dense<T> *mat, size_t col = 0)
 }
 
 
-TYPED_TEST(GmresMixed, SolvesMultipleDenseSystemForDivergenceCheck)
+TYPED_TEST(CbGmres, SolvesMultipleDenseSystemForDivergenceCheck)
 {
     using Mtx = typename TestFixture::Mtx;
     using value_type = typename TestFixture::value_type;
-    auto solver = this->gmres_mixed_factory_big->generate(this->mtx_big);
+    auto solver = this->cb_gmres_factory_big->generate(this->mtx_big);
     auto b1 = gko::initialize<Mtx>(
         {1300083.0, 1018120.5, 906410.0, -42679.5, 846779.5, 1176858.5},
         this->exec);
@@ -406,13 +406,13 @@ TYPED_TEST(GmresMixed, SolvesMultipleDenseSystemForDivergenceCheck)
 }
 
 
-TYPED_TEST(GmresMixed, SolvesBigDenseSystem1WithRestart)
+TYPED_TEST(CbGmres, SolvesBigDenseSystem1WithRestart)
 {
     using Mtx = typename TestFixture::Mtx;
     using value_type = typename TestFixture::value_type;
     using gmres_type = typename TestFixture::gmres_type;
     const auto half_tol = std::sqrt(this->precision());
-    auto gmres_mixed_factory_restart =
+    auto cb_gmres_factory_restart =
         gmres_type::build()
             .with_krylov_dim(4u)
             .with_storage_precision(this->storage_precision)
@@ -423,7 +423,7 @@ TYPED_TEST(GmresMixed, SolvesBigDenseSystem1WithRestart)
                     .with_reduction_factor(this->precision())
                     .on(this->exec))
             .on(this->exec);
-    auto solver = gmres_mixed_factory_restart->generate(this->mtx_medium);
+    auto solver = cb_gmres_factory_restart->generate(this->mtx_medium);
     auto b = gko::initialize<Mtx>(
         {-13945.16, 11205.66, 16132.96, 24342.18, -10910.98}, this->exec);
     auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0, 0.0, 0.0}, this->exec);
@@ -435,12 +435,12 @@ TYPED_TEST(GmresMixed, SolvesBigDenseSystem1WithRestart)
 }
 
 
-TYPED_TEST(GmresMixed, SolvesWithPreconditioner)
+TYPED_TEST(CbGmres, SolvesWithPreconditioner)
 {
     using Mtx = typename TestFixture::Mtx;
     using value_type = typename TestFixture::value_type;
     using gmres_type = typename TestFixture::gmres_type;
-    auto gmres_mixed_factory_preconditioner =
+    auto cb_gmres_factory_preconditioner =
         gmres_type::build()
             .with_storage_precision(this->storage_precision)
             .with_criteria(
@@ -454,7 +454,7 @@ TYPED_TEST(GmresMixed, SolvesWithPreconditioner)
                     .with_max_block_size(3u)
                     .on(this->exec))
             .on(this->exec);
-    auto solver = gmres_mixed_factory_preconditioner->generate(this->mtx_big);
+    auto solver = cb_gmres_factory_preconditioner->generate(this->mtx_big);
     auto b = gko::initialize<Mtx>(
         {175352.10, 313410.50, 131114.10, -134116.30, 179529.30, -43564.90},
         this->exec);

From 3ea56588b86d779be3231642c963c1d58d4fbf07 Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Sat, 23 Jan 2021 01:16:08 +0000
Subject: [PATCH 0394/1008] Format files
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thoasm@users.noreply.github.com>
---
 core/solver/cb_gmres.cpp                 | 8 ++++----
 core/solver/cb_gmres_accessor.hpp        | 4 +---
 core/test/solver/cb_gmres.cpp            | 7 ++++++-
 include/ginkgo/core/solver/cb_gmres.hpp  | 6 +++---
 omp/solver/cb_gmres_kernels.cpp          | 7 ++-----
 omp/test/solver/cb_gmres_kernels.cpp     | 4 ++--
 reference/solver/cb_gmres_kernels.cpp    | 5 +++--
 reference/stop/residual_norm_kernels.cpp | 4 +---
 8 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/core/solver/cb_gmres.cpp b/core/solver/cb_gmres.cpp
index 8c5ebf378e5..5bf8ad72ed8 100644
--- a/core/solver/cb_gmres.cpp
+++ b/core/solver/cb_gmres.cpp
@@ -33,6 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/cb_gmres.hpp>
 
 
+#include <iostream>
+#include <typeinfo>
+
+
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
@@ -44,10 +48,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/identity.hpp>
 
 
-#include <iostream>
-#include <typeinfo>
-
-
 #include "core/solver/cb_gmres_accessor.hpp"
 #include "core/solver/cb_gmres_kernels.hpp"
 
diff --git a/core/solver/cb_gmres_accessor.hpp b/core/solver/cb_gmres_accessor.hpp
index d1f185dc33b..9cdbb6ba7ce 100644
--- a/core/solver/cb_gmres_accessor.hpp
+++ b/core/solver/cb_gmres_accessor.hpp
@@ -36,6 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <array>
 #include <cinttypes>
+#include <iostream>
 #include <limits>
 #include <memory>
 #include <type_traits>
@@ -51,9 +52,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/base/accessors.hpp"
-
-
-#include <iostream>
 #define GKO_DEBUG_OUTPUT false
 
 
diff --git a/core/test/solver/cb_gmres.cpp b/core/test/solver/cb_gmres.cpp
index 2867b1b2244..462db27712f 100644
--- a/core/test/solver/cb_gmres.cpp
+++ b/core/test/solver/cb_gmres.cpp
@@ -30,19 +30,24 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <core/solver/cb_gmres.cpp>
 #include <ginkgo/core/solver/cb_gmres.hpp>
 
+
 #include <typeinfo>
 
+
 #include <gtest/gtest.h>
 
+
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm_reduction.hpp>
 
+
+#include "core/solver/cb_gmres.cpp"
+
 namespace {
 
 class CbGmres : public ::testing::Test {
diff --git a/include/ginkgo/core/solver/cb_gmres.hpp b/include/ginkgo/core/solver/cb_gmres.hpp
index aab1821edf5..75318ea0631 100644
--- a/include/ginkgo/core/solver/cb_gmres.hpp
+++ b/include/ginkgo/core/solver/cb_gmres.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_CB_GMRES_HPP_
-#define GKO_CORE_SOLVER_CB_GMRES_HPP_
+#ifndef GKO_PUBLIC_CORE_SOLVER_CB_GMRES_HPP_
+#define GKO_PUBLIC_CORE_SOLVER_CB_GMRES_HPP_
 
 
 #include <vector>
@@ -189,4 +189,4 @@ class CbGmres : public EnableLinOp<CbGmres<ValueType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_CB_GMRES_HPP_
+#endif  // GKO_PUBLIC_CORE_SOLVER_CB_GMRES_HPP_
diff --git a/omp/solver/cb_gmres_kernels.cpp b/omp/solver/cb_gmres_kernels.cpp
index acdccd59b06..4bc635444e5 100644
--- a/omp/solver/cb_gmres_kernels.cpp
+++ b/omp/solver/cb_gmres_kernels.cpp
@@ -33,10 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/cb_gmres_kernels.hpp"
 
 
-#include <omp.h>
+#include <iostream>
 
 
-#include <iostream>
+#include <omp.h>
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -46,9 +46,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/cb_gmres.hpp>
 
 
-#include <iostream>
-
-
 // #define TIMING 1
 
 
diff --git a/omp/test/solver/cb_gmres_kernels.cpp b/omp/test/solver/cb_gmres_kernels.cpp
index d0037bb82ac..8b4662cf2ed 100644
--- a/omp/test/solver/cb_gmres_kernels.cpp
+++ b/omp/test/solver/cb_gmres_kernels.cpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/solver/cb_gmres_kernels.hpp"
+#include <ginkgo/core/solver/cb_gmres.hpp>
 
 
 #include <algorithm>
@@ -44,13 +44,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/solver/cb_gmres.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm_reduction.hpp>
 
 
 #include "core/solver/cb_gmres_accessor.hpp"
+#include "core/solver/cb_gmres_kernels.hpp"
 #include "core/test/utils.hpp"
 
 
diff --git a/reference/solver/cb_gmres_kernels.cpp b/reference/solver/cb_gmres_kernels.cpp
index 4105eceffc4..42211e346e6 100644
--- a/reference/solver/cb_gmres_kernels.cpp
+++ b/reference/solver/cb_gmres_kernels.cpp
@@ -33,14 +33,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/cb_gmres_kernels.hpp"
 
 
+#include <iostream>
+
+
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/solver/cb_gmres.hpp>
 
-#include <iostream>
-
 
 #include "core/solver/cb_gmres_accessor.hpp"
 
diff --git a/reference/stop/residual_norm_kernels.cpp b/reference/stop/residual_norm_kernels.cpp
index b1d06b2a732..0189762061f 100644
--- a/reference/stop/residual_norm_kernels.cpp
+++ b/reference/stop/residual_norm_kernels.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
+#include <iostream>
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -42,9 +43,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
-#include <iostream>
-
-
 namespace gko {
 namespace kernels {
 namespace reference {

From d95bb7fd327425012c38cea1fa28fce8ed4cc8e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Sun, 24 Jan 2021 15:01:15 +0100
Subject: [PATCH 0395/1008] Add DPCPP stubs to allow compilation

---
 dpcpp/solver/cb_gmres_kernels.dp.cpp | 120 +++++++++++++++++++++++++++
 1 file changed, 120 insertions(+)
 create mode 100644 dpcpp/solver/cb_gmres_kernels.dp.cpp

diff --git a/dpcpp/solver/cb_gmres_kernels.dp.cpp b/dpcpp/solver/cb_gmres_kernels.dp.cpp
new file mode 100644
index 00000000000..f051fc13d9e
--- /dev/null
+++ b/dpcpp/solver/cb_gmres_kernels.dp.cpp
@@ -0,0 +1,120 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/solver/cb_gmres_kernels.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/range.hpp>
+
+
+#include "core/base/accessors.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The CB_GMRES solver namespace.
+ *
+ * @ingroup cb_gmres
+ */
+namespace cb_gmres {
+
+
+template <typename ValueType>
+void initialize_1(std::shared_ptr<const DpcppExecutor> exec,
+                  const matrix::Dense<ValueType> *b,
+                  matrix::Dense<remove_complex<ValueType>> *b_norm,
+                  matrix::Dense<ValueType> *residual,
+                  matrix::Dense<ValueType> *givens_sin,
+                  matrix::Dense<ValueType> *givens_cos,
+                  Array<stopping_status> *stop_status,
+                  size_type krylov_dim) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CB_GMRES_INITIALIZE_1_KERNEL);
+
+
+template <typename ValueType, typename Accessor3d>
+void initialize_2(std::shared_ptr<const DpcppExecutor> exec,
+                  const matrix::Dense<ValueType> *residual,
+                  matrix::Dense<remove_complex<ValueType>> *residual_norm,
+                  matrix::Dense<ValueType> *residual_norm_collection,
+                  matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+                  Accessor3d krylov_bases,
+                  matrix::Dense<ValueType> *next_krylov_basis,
+                  Array<size_type> *final_iter_nums,
+                  size_type krylov_dim) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(
+    GKO_DECLARE_CB_GMRES_INITIALIZE_2_KERNEL);
+
+
+template <typename ValueType, typename Accessor3d>
+void step_1(std::shared_ptr<const DpcppExecutor> exec,
+            matrix::Dense<ValueType> *next_krylov_basis,
+            matrix::Dense<ValueType> *givens_sin,
+            matrix::Dense<ValueType> *givens_cos,
+            matrix::Dense<remove_complex<ValueType>> *residual_norm,
+            matrix::Dense<ValueType> *residual_norm_collection,
+            Accessor3d krylov_bases, matrix::Dense<ValueType> *hessenberg_iter,
+            matrix::Dense<ValueType> *buffer_iter,
+            const matrix::Dense<remove_complex<ValueType>> *b_norm,
+            matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+            size_type iter, Array<size_type> *final_iter_nums,
+            const Array<stopping_status> *stop_status,
+            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
+            int *num_reorth_steps, int *num_reorth_vectors) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(GKO_DECLARE_CB_GMRES_STEP_1_KERNEL);
+
+
+template <typename ValueType, typename ConstAccessor3d>
+void step_2(std::shared_ptr<const DpcppExecutor> exec,
+            const matrix::Dense<ValueType> *residual_norm_collection,
+            ConstAccessor3d krylov_bases,
+            const matrix::Dense<ValueType> *hessenberg,
+            matrix::Dense<ValueType> *y,
+            matrix::Dense<ValueType> *before_preconditioner,
+            const Array<size_type> *final_iter_nums) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_CB_GMRES_CONST_TYPE(
+    GKO_DECLARE_CB_GMRES_STEP_2_KERNEL);
+
+
+}  // namespace cb_gmres
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko

From fe1231916f8ca2b4187f395660fe2c450a6190c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 28 Jan 2021 11:08:25 +0100
Subject: [PATCH 0396/1008] Make cb-gmres benchmarks dependent on etype

TODO: Rename cb-gmres benchmark strings
---
 benchmark/solver/solver.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 2091013c77a..6f5a5409b18 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -262,7 +262,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"cb_gmres_double",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::CbGmres<double>::build()
+             return gko::solver::CbGmres<etype>::build()
                  .with_storage_precision(
                      gko::solver::cb_gmres_storage_precision::keep)
                  .with_criteria(create_criterion(exec))
@@ -273,7 +273,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"cb_gmres_float",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::CbGmres<double>::build()
+             return gko::solver::CbGmres<etype>::build()
                  .with_storage_precision(
                      gko::solver::cb_gmres_storage_precision::reduce1)
                  .with_criteria(create_criterion(exec))
@@ -284,7 +284,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"cb_gmres_half",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::CbGmres<double>::build()
+             return gko::solver::CbGmres<etype>::build()
                  .with_storage_precision(
                      gko::solver::cb_gmres_storage_precision::reduce2)
                  .with_criteria(create_criterion(exec))
@@ -295,7 +295,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"cb_gmres_int64",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::CbGmres<double>::build()
+             return gko::solver::CbGmres<etype>::build()
                  .with_storage_precision(
                      gko::solver::cb_gmres_storage_precision::integer)
                  .with_criteria(create_criterion(exec))
@@ -306,7 +306,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"cb_gmres_int32",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::CbGmres<double>::build()
+             return gko::solver::CbGmres<etype>::build()
                  .with_storage_precision(
                      gko::solver::cb_gmres_storage_precision::ireduce1)
                  .with_criteria(create_criterion(exec))
@@ -317,7 +317,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"cb_gmres_int16",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::CbGmres<double>::build()
+             return gko::solver::CbGmres<etype>::build()
                  .with_storage_precision(
                      gko::solver::cb_gmres_storage_precision::ireduce2)
                  .with_criteria(create_criterion(exec))

From b65fcfe17089e633213dbf4b424b096042bd87c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 1 Feb 2021 05:43:06 +0100
Subject: [PATCH 0397/1008] Fix implementation and reference test for CB-GMRES

Having forced iteration for small matrices can lead to NaN values during
CB-GMRES. To prevent that, forced iterations now only happen after the
10th total iteration. Before that, a vector is immediately declared as
converged as soon as the stopping criterion said so.

Also done:
- Removed useless debug-output from CB-GMRES
- Changed the tolerances for the reference test to always pass
---
 core/solver/cb_gmres.cpp                   | 163 +++------------------
 reference/test/solver/cb_gmres_kernels.cpp |  74 ++++------
 2 files changed, 53 insertions(+), 184 deletions(-)

diff --git a/core/solver/cb_gmres.cpp b/core/solver/cb_gmres.cpp
index 5bf8ad72ed8..3e610f9e62e 100644
--- a/core/solver/cb_gmres.cpp
+++ b/core/solver/cb_gmres.cpp
@@ -33,8 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/cb_gmres.hpp>
 
 
-#include <iostream>
-#include <typeinfo>
+#include <type_traits>
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -52,15 +51,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/cb_gmres_kernels.hpp"
 
 
-//#define TIMING 1
-
-
-#ifdef TIMING
-using double_seconds = std::chrono::duration<double>;
-#define TIMING_STEPS 1
-#endif
-
-
 namespace gko {
 namespace solver {
 
@@ -76,24 +66,6 @@ GKO_REGISTER_OPERATION(step_2, cb_gmres::step_2);
 
 }  // namespace cb_gmres
 
-// TODO: Remove output
-template <typename T>
-struct type_string {
-    static const char *get() { return typeid(T).name(); }
-};
-
-#define GKO_SPECIALIZE_TYPE_STRING(type)           \
-    template <>                                    \
-    struct type_string<type> {                     \
-        static const char *get() { return #type; } \
-    }
-
-GKO_SPECIALIZE_TYPE_STRING(int32);
-GKO_SPECIALIZE_TYPE_STRING(int16);
-GKO_SPECIALIZE_TYPE_STRING(double);
-GKO_SPECIALIZE_TYPE_STRING(float);
-GKO_SPECIALIZE_TYPE_STRING(half);
-
 
 template <typename T>
 struct to_integer_impl {
@@ -298,7 +270,6 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                                     dense_b->get_size()[1]);
         int num_restarts = 0, num_reorth_steps = 0, num_reorth_vectors = 0;
 
-        // std::cout << "Before initializate_1" << std::endl;
         // Initialization
         exec->run(cb_gmres::make_initialize_1(
             dense_b, b_norm.get(), residual.get(), givens_sin.get(),
@@ -310,7 +281,6 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                               residual.get());
         // residual = residual - Ax
 
-        // std::cout << "Before initializate_2" << std::endl;
         exec->run(cb_gmres::make_initialize_2(
             residual.get(), residual_norm.get(), residual_norm_collection.get(),
             arnoldi_norm.get(), krylov_bases_range, next_krylov_basis.get(),
@@ -334,15 +304,6 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         auto after_preconditioner =
             matrix::Dense<ValueType>::create_with_config_of(dense_x);
 
-#ifdef TIMING
-        exec->synchronize();
-        auto start = std::chrono::steady_clock::now();
-#ifdef TIMING_STEPS
-        auto time_RSTRT = start - start;
-        auto time_SPMV = start - start;
-        auto time_STEP1 = start - start;
-#endif
-#endif
         Array<bool> stop_encountered_rhs(exec->get_master(),
                                          dense_b->get_size()[1]);
         Array<bool> fully_converged_rhs(exec->get_master(),
@@ -353,20 +314,26 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             stop_encountered_rhs.get_data()[i] = false;
             fully_converged_rhs.get_data()[i] = false;
         }
+        // Start only after this value with performing forced iterations after
+        // convergence detection
+        constexpr decltype(total_iter) start_force_reset{10};
         bool perform_reset{false};
-        decltype(krylov_dim_) forced_iterations{0};
-        const decltype(forced_iterations) forced_limit{krylov_dim_ / 10};
-        decltype(krylov_dim_) total_checks{0};  // TODO: Remove debug output
-        const char *type_str = type_string<storage_type>::get();
-        // TODO: take care of multiple RHS. Currently, we restart
-        // everything,
-        //       even though a lot of parts might have already converged!
-        //       Use `one_changed` to take care of that!
+        // Fraction of the krylov_dim_ (or total_iter if it is lower),
+        // determining the number of forced iteration to perform
+        constexpr decltype(krylov_dim_) forced_iteration_fraction{10};
+        const decltype(krylov_dim_) forced_limit{krylov_dim_ /
+                                                 forced_iteration_fraction};
+        // Counter for the forced iterations. Start at max in order to properly
+        // test convergence at the beginning
+        decltype(krylov_dim_) forced_iterations{forced_limit};
+
         while (true) {
             ++total_iter;
             this->template log<log::Logger::iteration_complete>(
                 this, total_iter, residual.get(), dense_x, residual_norm.get());
-            if (forced_iterations < forced_limit) {
+            // In the beginning, only force a fraction of the total iterations
+            if (forced_iterations < forced_limit &&
+                forced_iterations < total_iter / forced_iteration_fraction) {
                 ++forced_iterations;
             } else {
                 bool all_changed = stop_criterion->update()
@@ -377,25 +344,20 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                                        .check(RelativeStoppingId, true,
                                               &stop_status, &one_changed);
                 if (one_changed || all_changed) {
-                    /*
-                    std::cout << type_str << ": " << ++total_checks
-                              << ". check in iteration " << total_iter << ";
-                    "
-                              << forced_iterations << " / " << forced_limit
-                    <<
-                    '\n';
-                    */
                     host_stop_status = stop_status;
                     bool host_array_changed{false};
                     for (size_type i = 0; i < host_stop_status.get_num_elems();
                          ++i) {
                         auto local_status = host_stop_status.get_data() + i;
-                        // TODO: ignore all actually converged ones!
+                        // Ignore all actually converged ones!
                         if (fully_converged_rhs.get_data()[i]) {
                             continue;
                         }
                         if (local_status->has_converged()) {
-                            if (stop_encountered_rhs.get_data()[i]) {
+                            // If convergence was detected earlier, or
+                            // at the very beginning:
+                            if (stop_encountered_rhs.get_data()[i] ||
+                                total_iter < start_force_reset) {
                                 fully_converged_rhs.get_data()[i] = true;
                             } else {
                                 stop_encountered_rhs.get_data()[i] = true;
@@ -425,10 +387,6 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
             if (perform_reset || restart_iter == krylov_dim_) {
                 perform_reset = false;
-#ifdef TIMING_STEPS
-                exec->synchronize();
-                auto t_aux_0 = std::chrono::steady_clock::now();
-#endif
                 num_restarts++;
                 // Restart
                 // use a view in case this is called earlier
@@ -465,10 +423,6 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                 // next_krylov_basis = residual / residual_norm
                 // final_iter_nums = {0, ..., 0}
                 restart_iter = 0;
-#ifdef TIMING_STEPS
-                exec->synchronize();
-                time_RSTRT += std::chrono::steady_clock::now() - t_aux_0;
-#endif
             }
 
             this->get_preconditioner()->apply(next_krylov_basis.get(),
@@ -484,23 +438,10 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             auto buffer_iter = buffer->create_submatrix(
                 span{0, restart_iter + 2}, span{0, dense_b->get_size()[1]});
 
-#ifdef TIMING_STEPS
-            exec->synchronize();
-            auto t_aux_1 = std::chrono::steady_clock::now();
-#endif
             // Start of arnoldi
             system_matrix_->apply(preconditioned_vector.get(),
                                   next_krylov_basis.get());
             // next_krylov_basis = A * preconditioned_vector
-#ifdef TIMING_STEPS
-            exec->synchronize();
-            time_SPMV += std::chrono::steady_clock::now() - t_aux_1;
-#endif
-
-#ifdef TIMING_STEPS
-            exec->synchronize();
-            auto t_aux_2 = std::chrono::steady_clock::now();
-#endif
             exec->run(cb_gmres::make_step_1(
                 next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
                 residual_norm.get(), residual_norm_collection.get(),
@@ -508,10 +449,6 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                 b_norm.get(), arnoldi_norm.get(), restart_iter,
                 &final_iter_nums, &stop_status, &reorth_status, &num_reorth,
                 &num_reorth_steps, &num_reorth_vectors));
-#ifdef TIMING_STEPS
-            exec->synchronize();
-            time_STEP1 += std::chrono::steady_clock::now() - t_aux_2;
-#endif
             // for i in 0:restart_iter
             //     hessenberg(restart_iter, i) = next_krylov_basis' *
             //     krylov_bases(:, i) next_krylov_basis  -=
@@ -537,15 +474,7 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
             restart_iter++;
         }
-        std::cout << type_str << ": " << total_checks
-                  << ": exiting in iteration " << total_iter << "; "
-                  << forced_iterations << " / " << forced_limit << '\n';
-
         // Solve x
-#ifdef TIMING_STEPS
-        exec->synchronize();
-        auto t_aux_3 = std::chrono::steady_clock::now();
-#endif
 
         auto hessenberg_small = hessenberg->create_submatrix(
             span{0, restart_iter},
@@ -558,61 +487,11 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             &final_iter_nums));
         // Solve upper triangular.
         // y = hessenberg \ residual_norm_collection
-#ifdef TIMING_STEPS
-        exec->synchronize();
-        auto time_STEP2 = std::chrono::steady_clock::now() - t_aux_3;
-#endif
-
-#ifdef TIMING_STEPS
-        exec->synchronize();
-        auto t_aux_4 = std::chrono::steady_clock::now();
-#endif
         this->get_preconditioner()->apply(before_preconditioner.get(),
                                           after_preconditioner.get());
         dense_x->add_scaled(one_op.get(), after_preconditioner.get());
-#ifdef TIMING_STEPS
-        exec->synchronize();
-        auto time_SOLVEX = std::chrono::steady_clock::now() - t_aux_4;
-#endif
         // Solve x
         // x = x + get_preconditioner() * krylov_bases * y
-
-#ifdef TIMING
-        exec->synchronize();
-        auto time = std::chrono::steady_clock::now() - start;
-#endif
-#ifdef TIMING
-        std::cout << "total_iter = " << total_iter << std::endl;
-        std::cout << "num_restarts = " << num_restarts << std::endl;
-        std::cout << "reorth_steps = " << num_reorth_steps << std::endl;
-        std::cout << "reorth_vectors = " << num_reorth_vectors << std::endl;
-        std::cout << "time = "
-                  << std::chrono::duration_cast<double_seconds>(time).count()
-                  << std::endl;
-#ifdef TIMING_STEPS
-        std::cout
-            << "time_RSTRT = "
-            << std::chrono::duration_cast<double_seconds>(time_RSTRT).count()
-            << std::endl;
-        std::cout
-            << "time_SPMV = "
-            << std::chrono::duration_cast<double_seconds>(time_SPMV).count()
-            << std::endl;
-        std::cout
-            << "time_STEP1 = "
-            << std::chrono::duration_cast<double_seconds>(time_STEP1).count()
-            << std::endl;
-        std::cout
-            << "time_STEP2 = "
-            << std::chrono::duration_cast<double_seconds>(time_STEP2).count()
-            << std::endl;
-        std::cout
-            << "time_SOLVEX = "
-            << std::chrono::duration_cast<double_seconds>(time_SOLVEX).count()
-            << std::endl;
-#endif
-        write(std::cout, lend(residual_norm));
-#endif
     };  // End of apply_lambda
 
     // Look which precision to use as the storage type
diff --git a/reference/test/solver/cb_gmres_kernels.cpp b/reference/test/solver/cb_gmres_kernels.cpp
index 2a85dcf7920..9d529b83e53 100644
--- a/reference/test/solver/cb_gmres_kernels.cpp
+++ b/reference/test/solver/cb_gmres_kernels.cpp
@@ -66,23 +66,6 @@ class CbGmres : public ::testing::Test {
     using Mtx = gko::matrix::Dense<value_type>;
     using gmres_type = gko::solver::CbGmres<value_type>;
 
-    /**
-     * Used to extract the precision of the given value_type.
-     *
-     * @internal
-     * This struct is required because the `r<T>` stores a value of type
-     * remove_complex<T>, which is not possible for half since it can't be
-     * constructed as a constexpr.
-     */
-    template <typename T>
-    struct precision_target {
-        using rc_type = gko::remove_complex<T>;
-        static constexpr nc_value_type value =
-            std::is_same<rc_type, double>::value
-                ? 1e-14
-                : std::is_same<rc_type, float>::value ? 1e-6 : 1e-3;
-    };
-
     CbGmres()
         : exec(gko::ReferenceExecutor::create()),
           mtx(gko::initialize<Mtx>(
@@ -100,7 +83,7 @@ class CbGmres : public ::testing::Test {
                           .with_time_limit(std::chrono::seconds(6))
                           .on(exec),
                       gko::stop::ResidualNormReduction<value_type>::build()
-                          .with_reduction_factor(this->precision())
+                          .with_reduction_factor(this->reduction_factor())
                           .on(exec))
                   .on(exec)),
           mtx_big(gko::initialize<Mtx>(
@@ -118,7 +101,7 @@ class CbGmres : public ::testing::Test {
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
                       gko::stop::ResidualNormReduction<value_type>::build()
-                          .with_reduction_factor(this->precision())
+                          .with_reduction_factor(this->reduction_factor())
                           .on(exec))
                   .on(exec)),
           mtx_medium(
@@ -130,7 +113,14 @@ class CbGmres : public ::testing::Test {
                                    exec))
     {}
 
-    nc_value_type precision() const noexcept
+    constexpr nc_value_type reduction_factor() const noexcept
+    {
+        return r<nc_value_type>::value;
+    }
+
+    // For such small matrices, it seems to be impossible to reach the actual
+    // `reduction_factor()` when operating in lower precision.
+    nc_value_type assert_precision() const noexcept
     {
         using gko::reduce_precision;
         using gko::solver::cb_gmres_storage_precision;
@@ -140,15 +130,14 @@ class CbGmres : public ::testing::Test {
         switch (storage_precision) {
         case cb_gmres_storage_precision::reduce1:
         case cb_gmres_storage_precision::ireduce1:
-            return precision_target<reduce_precision<value_type>>::value * 1e-2;
+            return r<reduce_precision<value_type>, nc_value_type>::value;
         case cb_gmres_storage_precision::reduce2:
         case cb_gmres_storage_precision::ireduce2:
-            return precision_target<
-                       reduce_precision<reduce_precision<value_type>>>::value *
-                   1e-2;
+            return r<reduce_precision<reduce_precision<value_type>>,
+                     nc_value_type>::value;
         case cb_gmres_storage_precision::integer:
         default:
-            return precision_target<value_type>::value;
+            return r<nc_value_type>::value;
         }
     }
 
@@ -206,7 +195,7 @@ TYPED_TEST(CbGmres, SolvesStencilSystem)
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}), this->precision());
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}), this->assert_precision());
 }
 
 
@@ -225,7 +214,7 @@ TYPED_TEST(CbGmres, SolvesStencilSystem2)
                     .with_time_limit(std::chrono::seconds(6))
                     .on(this->exec),
                 gko::stop::ResidualNormReduction<T>::build()
-                    .with_reduction_factor(this->precision())
+                    .with_reduction_factor(this->reduction_factor())
                     .on(this->exec))
             .on(this->exec);
     auto solver = factory->generate(this->mtx2);
@@ -234,7 +223,7 @@ TYPED_TEST(CbGmres, SolvesStencilSystem2)
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.0, 4.0, 8.0}), this->precision() * 1e1);
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 4.0, 8.0}), 4 * this->assert_precision());
 }
 
 
@@ -251,7 +240,7 @@ TYPED_TEST(CbGmres, SolvesMultipleStencilSystems)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({I<T>{1.0, 1.0}, I<T>{3.0, 1.0}, I<T>{2.0, 1.0}}),
-                        this->precision());
+                        this->assert_precision());
 }
 
 
@@ -267,7 +256,7 @@ TYPED_TEST(CbGmres, SolvesStencilSystemUsingAdvancedApply)
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), this->precision());
+    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), this->assert_precision());
 }
 
 
@@ -286,7 +275,7 @@ TYPED_TEST(CbGmres, SolvesMultipleStencilSystemsUsingAdvancedApply)
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({I<T>{1.5, 1.0}, I<T>{5.0, 0.0}, I<T>{2.0, -1.0}}),
-                        this->precision());
+                        this->assert_precision());
 }
 
 
@@ -303,7 +292,7 @@ TYPED_TEST(CbGmres, SolvesBigDenseSystem1)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({52.7, 85.4, 134.2, -250.0, -16.8, 35.3}),
-                        this->precision());
+                        this->assert_precision());
 }
 
 
@@ -320,7 +309,7 @@ TYPED_TEST(CbGmres, SolvesBigDenseSystem2)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
-                        this->precision());
+                        this->assert_precision());
 }
 
 
@@ -328,10 +317,11 @@ template <typename T>
 gko::remove_complex<T> infNorm(gko::matrix::Dense<T> *mat, size_t col = 0)
 {
     using std::abs;
+    auto host_data = clone(mat->get_executor()->get_master(), mat);
     using no_cpx_t = gko::remove_complex<T>;
     no_cpx_t norm = 0.0;
-    for (size_t i = 0; i < mat->get_size()[0]; ++i) {
-        no_cpx_t absEntry = abs(mat->at(i, col));
+    for (size_t i = 0; i < host_data->get_size()[0]; ++i) {
+        no_cpx_t absEntry = abs(host_data->at(i, col));
         if (norm < absEntry) norm = absEntry;
     }
     return norm;
@@ -397,12 +387,12 @@ TYPED_TEST(CbGmres, SolvesMultipleDenseSystemForDivergenceCheck)
 
     // make sure that all combined solutions are as good or better than the
     // single solutions
-    ASSERT_LE(normC1 / normB1, normS1 / normB1 + this->precision());
-    ASSERT_LE(normC2 / normB2, normS2 / normB2 + this->precision());
+    ASSERT_LE(normC1 / normB1, normS1 / normB1 + this->assert_precision());
+    ASSERT_LE(normC2 / normB2, normS2 / normB2 + this->assert_precision());
 
     // Not sure if this is necessary, the assertions above should cover what is
     // needed.
-    GKO_ASSERT_MTX_NEAR(xc, mergedRes, this->precision());
+    GKO_ASSERT_MTX_NEAR(xc, mergedRes, this->assert_precision());
 }
 
 
@@ -411,7 +401,7 @@ TYPED_TEST(CbGmres, SolvesBigDenseSystem1WithRestart)
     using Mtx = typename TestFixture::Mtx;
     using value_type = typename TestFixture::value_type;
     using gmres_type = typename TestFixture::gmres_type;
-    const auto half_tol = std::sqrt(this->precision());
+    const auto half_tol = std::sqrt(this->assert_precision());
     auto cb_gmres_factory_restart =
         gmres_type::build()
             .with_krylov_dim(4u)
@@ -420,7 +410,7 @@ TYPED_TEST(CbGmres, SolvesBigDenseSystem1WithRestart)
                 gko::stop::Iteration::build().with_max_iters(200u).on(
                     this->exec),
                 gko::stop::ResidualNormReduction<value_type>::build()
-                    .with_reduction_factor(this->precision())
+                    .with_reduction_factor(this->reduction_factor())
                     .on(this->exec))
             .on(this->exec);
     auto solver = cb_gmres_factory_restart->generate(this->mtx_medium);
@@ -447,7 +437,7 @@ TYPED_TEST(CbGmres, SolvesWithPreconditioner)
                 gko::stop::Iteration::build().with_max_iters(100u).on(
                     this->exec),
                 gko::stop::ResidualNormReduction<value_type>::build()
-                    .with_reduction_factor(this->precision())
+                    .with_reduction_factor(this->reduction_factor())
                     .on(this->exec))
             .with_preconditioner(
                 gko::preconditioner::Jacobi<value_type>::build()
@@ -463,7 +453,7 @@ TYPED_TEST(CbGmres, SolvesWithPreconditioner)
     solver->apply(b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, l({33.0, -56.0, 81.0, -30.0, 21.0, 40.0}),
-                        this->precision());
+                        4 * this->assert_precision());
 }
 
 

From f220a6b4a17526ccf04109a595fab9bd46ca0bb4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 1 Feb 2021 11:18:16 +0100
Subject: [PATCH 0398/1008] Update tolerance for one reference CB-GMRES test

This is necessary for the Intel compiler to pass the test
`SolvesStencilSystem2`.
---
 reference/test/solver/cb_gmres_kernels.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reference/test/solver/cb_gmres_kernels.cpp b/reference/test/solver/cb_gmres_kernels.cpp
index 9d529b83e53..115b3a10391 100644
--- a/reference/test/solver/cb_gmres_kernels.cpp
+++ b/reference/test/solver/cb_gmres_kernels.cpp
@@ -223,7 +223,7 @@ TYPED_TEST(CbGmres, SolvesStencilSystem2)
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.0, 4.0, 8.0}), 4 * this->assert_precision());
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 4.0, 8.0}), 8 * this->assert_precision());
 }
 
 

From e3ae51b79489bca3cd15c17f52704dd99b355325 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 2 Feb 2021 03:28:32 +0100
Subject: [PATCH 0399/1008] Update atomic_max

- Remove unnecessary code
- Add wrapper function to `atomic_helper` in order make it easier to
  implement other atomic operations (atomic_add and atomic_max use this
  wrapper)
---
 common/components/atomic.hpp.inc | 82 +++++++++++++-------------------
 1 file changed, 32 insertions(+), 50 deletions(-)

diff --git a/common/components/atomic.hpp.inc b/common/components/atomic.hpp.inc
index 1ababe8380a..a99c07dc654 100644
--- a/common/components/atomic.hpp.inc
+++ b/common/components/atomic.hpp.inc
@@ -50,7 +50,7 @@ struct atomic_helper {
         static_assert(sizeof(ValueType) == 0,
                       "This default function is not implemented, only the "
                       "specializations are.");
-        // TODO: add proper implementation of generic atomic add
+        // TODO: add proper implementation of generic atomic max
     }
 };
 
@@ -73,20 +73,31 @@ __forceinline__ __device__ ResultType reinterpret(ValueType val)
         __forceinline__ __device__ static ValueType atomic_add(              \
             ValueType *__restrict__ addr, ValueType val)                     \
         {                                                                    \
-            CONVERTER_TYPE *address_as_converter =                           \
-                reinterpret_cast<CONVERTER_TYPE *>(addr);                    \
-            CONVERTER_TYPE old = *address_as_converter;                      \
-            CONVERTER_TYPE assumed;                                          \
-            do {                                                             \
-                assumed = old;                                               \
-                old = atomicCAS(address_as_converter, assumed,               \
-                                reinterpret<CONVERTER_TYPE>(                 \
+            using c_type = CONVERTER_TYPE;                                   \
+            return atomic_wrapper(addr, [&val](c_type &old, c_type assumed,  \
+                                               c_type *c_addr) {             \
+                old = atomicCAS(c_addr, assumed,                             \
+                                reinterpret<c_type>(                         \
                                     val + reinterpret<ValueType>(assumed))); \
-            } while (assumed != old);                                        \
-            return reinterpret<ValueType>(old);                              \
+            });                                                              \
         }                                                                    \
         __forceinline__ __device__ static ValueType atomic_max(              \
             ValueType *__restrict__ addr, ValueType val)                     \
+        {                                                                    \
+            using c_type = CONVERTER_TYPE;                                   \
+            return atomic_wrapper(                                           \
+                addr, [&val](c_type &old, c_type assumed, c_type *c_addr) {  \
+                    if (reinterpret<ValueType>(assumed) < val) {             \
+                        old = atomicCAS(c_addr, assumed,                     \
+                                        reinterpret<c_type>(val));           \
+                    }                                                        \
+                });                                                          \
+        }                                                                    \
+                                                                             \
+    private:                                                                 \
+        template <typename Callable>                                         \
+        __forceinline__ __device__ static ValueType atomic_wrapper(          \
+            ValueType *__restrict__ addr, Callable set_old)                  \
         {                                                                    \
             CONVERTER_TYPE *address_as_converter =                           \
                 reinterpret_cast<CONVERTER_TYPE *>(addr);                    \
@@ -94,19 +105,15 @@ __forceinline__ __device__ ResultType reinterpret(ValueType val)
             CONVERTER_TYPE assumed;                                          \
             do {                                                             \
                 assumed = old;                                               \
-                if (reinterpret<ValueType>(assumed) < val) {                 \
-                    old = atomicCAS(address_as_converter, assumed,           \
-                                    reinterpret<CONVERTER_TYPE>(val));       \
-                }                                                            \
+                set_old(old, assumed, address_as_converter);                 \
             } while (assumed != old);                                        \
             return reinterpret<ValueType>(old);                              \
         }                                                                    \
-    };
+    };  // namespace detail
 
-// Support 64-bit ATOMIC_ADD
+// Support 64-bit ATOMIC_ADD and ATOMIC_MAX
 GKO_BIND_ATOMIC_HELPER_STRUCTURE(unsigned long long int);
-// GKO_BIND_ATOMIC_HELPER_STRUCTURE(size_type);
-// Support 32-bit ATOMIC_ADD
+// Support 32-bit ATOMIC_ADD and ATOMIC_MAX
 GKO_BIND_ATOMIC_HELPER_STRUCTURE(unsigned int);
 
 
@@ -140,8 +147,6 @@ GKO_BIND_ATOMIC_ADD(unsigned int);
 GKO_BIND_ATOMIC_ADD(unsigned long long int);
 GKO_BIND_ATOMIC_ADD(float);
 
-// GKO_BIND_ATOMIC_ADD(size_type);
-
 #if !defined(__HIPCC__) || \
     (defined(__HIP_DEVICE_COMPILE__) && GINKGO_HIP_PLATFORM_NVCC)
 
@@ -196,43 +201,20 @@ __forceinline__ __device__ T atomic_max(T *__restrict__ addr, T val)
 
 GKO_BIND_ATOMIC_MAX(int);
 GKO_BIND_ATOMIC_MAX(unsigned int);
-// GKO_BIND_ATOMIC_MAX(unsigned long long int);
-// GKO_BIND_ATOMIC_MAX(float);
-
-// GKO_BIND_ATOMIC_MAX(size_type);
 
-/*
 #if !defined(__HIPCC__) || \
     (defined(__HIP_DEVICE_COMPILE__) && GINKGO_HIP_PLATFORM_NVCC)
 
 
-#if !((defined(CUDA_VERSION) && (CUDA_VERSION < 8000)) || \
-      (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 600)))
-// CUDA 8.0 starts suppoting 64-bit double atomicAdd on devices of compute
-// capability 6.x and higher
-GKO_BIND_ATOMIC_MAX(double);
-#endif
-
-#if !((defined(CUDA_VERSION) && (CUDA_VERSION < 10000)) || \
-      (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 700)))
-// CUDA 10.0 starts supporting 16-bit __half floating-point atomicAdd on devices
-// of compute capability 7.x and higher.
-GKO_BIND_ATOMIC_MAX(__half);
-#endif
-
-#if !((defined(CUDA_VERSION) && (CUDA_VERSION < 10000)) || \
-      (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 600)))
-// CUDA 10.0 starts supporting 32-bit __half2 floating-point atomicAdd on
-// devices of compute capability 6.x and higher. note: The atomicity of the
-// __half2 add operation is guaranteed separately for each of the two __half
-// elements; the entire __half2 is not guaranteed to be atomic as a single
-// 32-bit access.
-GKO_BIND_ATOMIC_MAX(__half2);
+#if defined(__CUDA_ARCH__) && (350 <= __CUDA_ARCH__)
+// Only Compute Capability 3.5 and higher supports 64-bit atomicMax
+GKO_BIND_ATOMIC_MAX(unsigned long long int);
 #endif
 
-
+#else   // Is HIP platform & on AMD hardware
+GKO_BIND_ATOMIC_MAX(unsigned long long int);
 #endif  // !defined(__HIPCC__) || (defined(__HIP_DEVICE_COMPILE__) &&
         // GINKGO_HIP_PLATFORM_HCC)
-*/
+
 
 #undef GKO_BIND_ATOMIC_MAX

From 969cb428c2b08ba63312de236d4aef1429abcb11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 2 Feb 2021 19:37:34 +0100
Subject: [PATCH 0400/1008] Remove unnecessary kernels and properly name them

in CB-GMRES

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Yuhsiang M. Tsai <yhmtsai@gmail.com>
---
 common/solver/cb_gmres_kernels.hpp.inc | 926 ++-----------------------
 cuda/solver/cb_gmres_kernels.cu        |  29 +-
 hip/solver/cb_gmres_kernels.hip.cpp    |  26 +-
 3 files changed, 78 insertions(+), 903 deletions(-)

diff --git a/common/solver/cb_gmres_kernels.hpp.inc b/common/solver/cb_gmres_kernels.hpp.inc
index 54a54209cbd..8ddb6ea0149 100644
--- a/common/solver/cb_gmres_kernels.hpp.inc
+++ b/common/solver/cb_gmres_kernels.hpp.inc
@@ -160,6 +160,7 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_kernel(
     size_type stride_next_krylov, remove_complex<ValueType> *__restrict__ norms,
     const stopping_status *__restrict__ stop_status)
 {
+    using rc_vtype = remove_complex<ValueType>;
     const auto tidx = threadIdx.x;
     const auto tidy = threadIdx.y;
     const auto col_idx = blockIdx.x * default_dot_dim + tidx;
@@ -169,12 +170,11 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_kernel(
         ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
-    __shared__ UninitializedArray<remove_complex<ValueType>,
-                                  default_dot_dim *(default_dot_dim + 1)>
-        reduction_helper_array;
-    remove_complex<ValueType> *__restrict__ reduction_helper =
-        reduction_helper_array;
-    remove_complex<ValueType> local_res = zero<remove_complex<ValueType>>();
+    __shared__
+        UninitializedArray<rc_vtype, default_dot_dim *(default_dot_dim + 1)>
+            reduction_helper_array;
+    rc_vtype *__restrict__ reduction_helper = reduction_helper_array;
+    rc_vtype local_res = zero<rc_vtype>();
     if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
         for (size_type i = start_row + tidy; i < end_row;
              i += default_dot_dim) {
@@ -183,14 +183,13 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_kernel(
         }
     }
     reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
-    __syncthreads();
+    group::this_thread_block().sync();
     local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
     const auto tile_block =
         group::tiled_partition<default_dot_dim>(group::this_thread_block());
     const auto sum =
         reduce(tile_block, local_res,
-               [](const remove_complex<ValueType> &a,
-                  const remove_complex<ValueType> &b) { return a + b; });
+               [](const rc_vtype &a, const rc_vtype &b) { return a + b; });
     const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
     if (tidx == 0 && new_col_idx < num_cols &&
         !stop_status[new_col_idx].has_stopped()) {
@@ -200,66 +199,15 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_kernel(
 }
 
 
-template <typename ValueType>
-__global__ __launch_bounds__(default_dot_size) void multinorminf_kernel(
-    size_type num_rows, size_type num_cols,
-    const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov, remove_complex<ValueType> *__restrict__ norms,
-    size_type stride_norms, const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = threadIdx.x;
-    const auto tidy = threadIdx.y;
-    const auto col_idx = blockIdx.x * default_dot_dim + tidx;
-    const auto num = ceildiv(num_rows, gridDim.y);
-    const auto start_row = blockIdx.y * num;
-    const auto end_row =
-        ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
-    // Used that way to get around dynamic initialization warning and
-    // template error when using `reduction_helper_array` directly in `reduce`
-    __shared__ UninitializedArray<remove_complex<ValueType>,
-                                  default_dot_dim *(default_dot_dim + 1)>
-        reduction_helper_array;
-    remove_complex<ValueType> *__restrict__ reduction_helper =
-        reduction_helper_array;
-    remove_complex<ValueType> local_max = zero<remove_complex<ValueType>>();
-    if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
-        for (size_type i = start_row + tidy; i < end_row;
-             i += default_dot_dim) {
-            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
-            local_max = (local_max >= abs(next_krylov_basis[next_krylov_idx]))
-                            ? local_max
-                            : abs(next_krylov_basis[next_krylov_idx]);
-            // local_max = (local_max >= next_krylov_basis[next_krylov_idx])
-            //                 ? local_max
-            //                 : next_krylov_basis[next_krylov_idx];
-        }
-    }
-    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_max;
-    __syncthreads();
-    local_max = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
-    const auto tile_block =
-        group::tiled_partition<default_dot_dim>(group::this_thread_block());
-    const auto value = reduce(
-        tile_block, local_max,
-        [](const remove_complex<ValueType> &a,
-           const remove_complex<ValueType> &b) { return ((a >= b) ? a : b); });
-    const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
-    if (tidx == 0 && new_col_idx < num_cols &&
-        !stop_status[new_col_idx].has_stopped()) {
-        const auto norms_idx = new_col_idx;
-        atomic_max(norms + norms_idx, value);
-    }
-}
-
-
 template <typename ValueType>
 __global__
-    __launch_bounds__(default_dot_size) void multinorminf_kernel_without_stop(
+    __launch_bounds__(default_dot_size) void multinorminf_without_stop_kernel(
         size_type num_rows, size_type num_cols,
         const ValueType *__restrict__ next_krylov_basis,
         size_type stride_next_krylov,
         remove_complex<ValueType> *__restrict__ norms, size_type stride_norms)
 {
+    using rc_vtype = remove_complex<ValueType>;
     const auto tidx = threadIdx.x;
     const auto tidy = threadIdx.y;
     const auto col_idx = blockIdx.x * default_dot_dim + tidx;
@@ -269,12 +217,11 @@ __global__
         ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
-    __shared__ UninitializedArray<remove_complex<ValueType>,
-                                  default_dot_dim *(default_dot_dim + 1)>
-        reduction_helper_array;
-    remove_complex<ValueType> *__restrict__ reduction_helper =
-        reduction_helper_array;
-    remove_complex<ValueType> local_max = zero<remove_complex<ValueType>>();
+    __shared__
+        UninitializedArray<rc_vtype, default_dot_dim *(default_dot_dim + 1)>
+            reduction_helper_array;
+    rc_vtype *__restrict__ reduction_helper = reduction_helper_array;
+    rc_vtype local_max = zero<rc_vtype>();
     if (col_idx < num_cols) {
         for (size_type i = start_row + tidy; i < end_row;
              i += default_dot_dim) {
@@ -282,20 +229,17 @@ __global__
             local_max = (local_max >= abs(next_krylov_basis[next_krylov_idx]))
                             ? local_max
                             : abs(next_krylov_basis[next_krylov_idx]);
-            // local_max = (local_max >= next_krylov_basis[next_krylov_idx])
-            //                 ? local_max
-            //                 : next_krylov_basis[next_krylov_idx];
         }
     }
     reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_max;
-    __syncthreads();
+    group::this_thread_block().sync();
     local_max = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
     const auto tile_block =
         group::tiled_partition<default_dot_dim>(group::this_thread_block());
-    const auto value = reduce(
-        tile_block, local_max,
-        [](const remove_complex<ValueType> &a,
-           const remove_complex<ValueType> &b) { return ((a >= b) ? a : b); });
+    const auto value =
+        reduce(tile_block, local_max, [](const rc_vtype &a, const rc_vtype &b) {
+            return ((a >= b) ? a : b);
+        });
     const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
     if (tidx == 0 && new_col_idx < num_cols) {
         const auto norms_idx = new_col_idx;
@@ -313,7 +257,7 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_inf_kernel(
     remove_complex<ValueType> *__restrict__ norms2,
     const stopping_status *__restrict__ stop_status)
 {
-    using non_complex = remove_complex<ValueType>;
+    using rc_vtype = remove_complex<ValueType>;
     const auto tidx = threadIdx.x;
     const auto tidy = threadIdx.y;
     const auto col_idx = blockIdx.x * default_dot_dim + tidx;
@@ -324,14 +268,14 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_inf_kernel(
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
     __shared__ UninitializedArray<
-        non_complex, (1 + compute_inf) * default_dot_dim *(default_dot_dim + 1)>
+        rc_vtype, (1 + compute_inf) * default_dot_dim *(default_dot_dim + 1)>
         reduction_helper_array;
-    non_complex *__restrict__ reduction_helper_add = reduction_helper_array;
-    non_complex *__restrict__ reduction_helper_max =
-        static_cast<non_complex *>(reduction_helper_array) +
+    rc_vtype *__restrict__ reduction_helper_add = reduction_helper_array;
+    rc_vtype *__restrict__ reduction_helper_max =
+        static_cast<rc_vtype *>(reduction_helper_array) +
         default_dot_dim * (default_dot_dim + 1);
-    non_complex local_res = zero<non_complex>();
-    non_complex local_max = zero<non_complex>();
+    rc_vtype local_res = zero<rc_vtype>();
+    rc_vtype local_max = zero<rc_vtype>();
     if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
         for (size_type i = start_row + tidy; i < end_row;
              i += default_dot_dim) {
@@ -348,18 +292,18 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_inf_kernel(
     if (compute_inf) {
         reduction_helper_max[tidx * (default_dot_dim + 1) + tidy] = local_max;
     }
-    __syncthreads();
+    group::this_thread_block().sync();
     local_res = reduction_helper_add[tidy * (default_dot_dim + 1) + tidx];
     const auto tile_block =
         group::tiled_partition<default_dot_dim>(group::this_thread_block());
-    const auto sum = reduce(
-        tile_block, local_res,
-        [](const non_complex &a, const non_complex &b) { return a + b; });
-    non_complex reduced_max{};
+    const auto sum =
+        reduce(tile_block, local_res,
+               [](const rc_vtype &a, const rc_vtype &b) { return a + b; });
+    rc_vtype reduced_max{};
     if (compute_inf) {
         local_max = reduction_helper_max[tidy * (default_dot_dim + 1) + tidx];
         reduced_max = reduce(tile_block, local_max,
-                             [](const non_complex &a, const non_complex &b) {
+                             [](const rc_vtype &a, const rc_vtype &b) {
                                  return ((a >= b) ? a : b);
                              });
     }
@@ -375,180 +319,8 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_inf_kernel(
 }
 
 
-template <typename ValueType, typename Accessor3d>
-__global__ __launch_bounds__(default_dot_size) void multidot_kernel(
-    size_type k, size_type num_rows, size_type num_rhs,
-    const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov, const Accessor3d krylov_bases,
-    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = threadIdx.x;
-    const auto tidy = threadIdx.y;
-    const decltype(num_rows) col_idx = blockIdx.x * default_dot_dim + tidx;
-    const auto num = ceildiv(num_rows, gridDim.y);
-    const auto start_row = blockIdx.y * num;
-    const auto end_row =
-        ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
-    // Used that way to get around dynamic initialization warning and
-    // template error when using `reduction_helper_array` directly in `reduce`
-    __shared__
-        UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
-            reduction_helper_array;
-    ValueType *__restrict__ reduction_helper = reduction_helper_array;
-    ValueType local_res = zero<ValueType>();
-    // const auto krylov_col = k * num_rhs + col_idx;
-    if (col_idx < num_rhs && !stop_status[col_idx].has_stopped()) {
-        for (size_type i = start_row + tidy; i < end_row;
-             i += default_dot_dim) {
-            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
-            local_res += next_krylov_basis[next_krylov_idx] *
-                         krylov_bases(k, i, col_idx);
-        }
-    }
-    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
-    __syncthreads();
-    local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
-    const auto tile_block =
-        group::tiled_partition<default_dot_dim>(group::this_thread_block());
-    const auto sum =
-        reduce(tile_block, local_res,
-               [](const ValueType &a, const ValueType &b) { return a + b; });
-    const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
-    if (tidx == 0 && new_col_idx < num_rhs &&
-        !stop_status[new_col_idx].has_stopped()) {
-        const auto hessenberg_idx = k * stride_hessenberg + new_col_idx;
-        atomic_add(hessenberg_iter + hessenberg_idx, sum);
-    }
-}
-
-
-template <typename ValueType, typename Accessor3d>
-__global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters(
-    size_type num_iters, size_type num_rows, size_type num_cols,
-    const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov, const Accessor3d krylov_bases,
-    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = threadIdx.x;
-    const auto tidy = threadIdx.y;
-    const size_type col_idx = blockIdx.x * default_dot_dim + tidx;
-    const auto num = ceildiv(num_rows, gridDim.y);
-    const auto start_row = blockIdx.y * num;
-    const auto end_row =
-        ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
-    // Used that way to get around dynamic initialization warning and
-    // template error when using `reduction_helper_array` directly in `reduce`
-    __shared__
-        UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
-            reduction_helper_array;
-    ValueType *__restrict__ reduction_helper = reduction_helper_array;
-
-    for (size_type k = 0; k < num_iters; ++k) {
-        ValueType local_res = zero<ValueType>();
-        if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
-            for (size_type i = start_row + tidy; i < end_row;
-                 i += default_dot_dim) {
-                const auto next_krylov_idx = i * stride_next_krylov + col_idx;
-                local_res += next_krylov_basis[next_krylov_idx] *
-                             krylov_bases(k, i, col_idx);
-            }
-        }
-        reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
-        __syncthreads();
-        local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
-        const auto tile_block =
-            group::tiled_partition<default_dot_dim>(group::this_thread_block());
-        const auto sum = reduce(
-            tile_block, local_res,
-            [](const ValueType &a, const ValueType &b) { return a + b; });
-        const auto new_col_idx = blockIdx.x * default_dot_dim + tidy;
-        if (tidx == 0 && new_col_idx < num_cols &&
-            !stop_status[new_col_idx].has_stopped()) {
-            const auto hessenberg_idx = k * stride_hessenberg + new_col_idx;
-            atomic_add(hessenberg_iter + hessenberg_idx, sum);
-        }
-    }
-}
-
-
-template <typename ValueType, typename Accessor3d>
-__global__ __launch_bounds__(default_dot_size) void multidot_kernel_num_iters_1(
-    size_type num_iters, size_type num_rows, size_type num_cols,
-    const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov, const Accessor3d krylov_bases,
-    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = threadIdx.x;
-    const auto tidy = threadIdx.y;
-    const auto num_cols_iters = num_cols * num_iters;
-    const auto col_idx_iters = blockIdx.x * blockDim.x + tidx;
-    const auto col_idx = col_idx_iters % num_cols;
-    const auto num = ceildiv(num_rows, gridDim.y);
-    const auto start_row = blockIdx.y * num;
-    const auto end_row =
-        ((blockIdx.y + 1) * num > num_rows) ? num_rows : (blockIdx.y + 1) * num;
-    const auto k = col_idx_iters / num_cols;
-    // Used that way to get around dynamic initialization warning and
-    // template error when using `reduction_helper_array` directly in `reduce`
-    __shared__
-        UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
-            reduction_helper_array;
-    ValueType *__restrict__ reduction_helper = reduction_helper_array;
-
-    //    for (size_type k = 0; k < num_iters; ++k) {
-    ValueType local_res = zero<ValueType>();
-    if (col_idx_iters < num_cols_iters && !stop_status[col_idx].has_stopped()) {
-        for (size_type i = start_row + tidy; i < end_row;
-             i += default_dot_dim) {
-            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
-            local_res += next_krylov_basis[next_krylov_idx] *
-                         krylov_bases(k, i, col_idx);
-        }
-    }
-    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
-    __syncthreads();
-    /*
-    if (((blockIdx.x + blockIdx.y) == 0) && ((tidx + tidy) == 0)) {
-        //    if ((blockIdx.x == 0) && ((tidx + tidy) == 0)) {
-        for (size_type ii = 0; ii < default_dot_dim; ii++)
-            for (size_type jj = 0; jj < default_dot_dim + 1; jj++)
-                if (jj <= num)
-                    printf("XX_%d(%d,%d) = %g\n", (int)blockIdx.y + 1,
-                           (int)ii + 1, (int)jj + 1,
-                           reduction_helper[(default_dot_dim + 1) * ii + jj]);
-    }
-    __syncthreads();
-    */
-    local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
-    const auto tile_block =
-        group::tiled_partition<default_dot_dim>(group::this_thread_block());
-    const auto sum =
-        reduce(tile_block, local_res,
-               [](const ValueType &a, const ValueType &b) { return a + b; });
-    const auto new_col_idx_iters = blockIdx.x * default_dot_dim + tidy;
-    const auto new_col_idx = new_col_idx_iters % num_cols;
-    const auto new_k = new_col_idx_iters / num_cols;
-    if (tidx == 0 && new_col_idx_iters < num_cols_iters &&
-        !stop_status[new_col_idx].has_stopped()) {
-        //        if ((blockIdx.x + blockIdx.y) == 0) {
-        /*
-        if (blockIdx.x == 0) {
-            printf("sum_%d(%d) = %g\n", (int)blockIdx.y + 1, (int)tidy + 1,
-                   sum);
-        }
-        */
-        // const auto hessenberg_idx = k * stride_hessenberg + new_col_idx;
-        const auto hessenberg_idx = new_k * stride_hessenberg + new_col_idx;
-        atomic_add(hessenberg_iter + hessenberg_idx, sum);
-    }
-    //    }
-}
-
 template <int dot_dim, typename ValueType, typename Accessor3d>
-__global__ __launch_bounds__(dot_dim *dot_dim) void multidot_kernel_num_iters_2(
+__global__ __launch_bounds__(dot_dim *dot_dim) void multidot_kernel(
     size_type num_iters, size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
     size_type stride_next_krylov, const Accessor3d krylov_bases,
@@ -556,14 +328,14 @@ __global__ __launch_bounds__(dot_dim *dot_dim) void multidot_kernel_num_iters_2(
     const stopping_status *__restrict__ stop_status)
 {
     /*
-     * In general in this kernel (thread indices appear to be col-major):
+     * In general in this kernel:
      * grid_dim
      *   x: for col_idx (^= which right hand side)
      *   y: for row_idx
      *   z: for num_iters (number of krylov vectors)
      * block_dim
-     *   x: for col_idx
-     *   y: for row_idx
+     *   x: for col_idx (must be < dot_dim)
+     *   y: for row_idx (must be < dot_dim)
      *   (z not used, must be set to 1 in dim)
      */
     const size_type tidx = threadIdx.x;
@@ -607,7 +379,7 @@ __global__ __launch_bounds__(dot_dim *dot_dim) void multidot_kernel_num_iters_2(
 
 
 template <int block_size, typename ValueType, typename Accessor3d>
-__global__ __launch_bounds__(block_size) void singledot_kernel_num_iters_2(
+__global__ __launch_bounds__(block_size) void singledot_kernel(
     size_type num_iters, size_type num_rows,
     const ValueType *__restrict__ next_krylov_basis,
     size_type stride_next_krylov, const Accessor3d krylov_bases,
@@ -615,13 +387,13 @@ __global__ __launch_bounds__(block_size) void singledot_kernel_num_iters_2(
     const stopping_status *__restrict__ stop_status)
 {
     /*
-     * In general in this kernel (thread indices appear to be col-major):
+     * In general in this kernel:
      * grid_dim
      *   x: for row_idx
      *   y: for num_iters (number of krylov vectors)
      * block_dim
      *   x: for row_idx (must be block_size)
-     *   (y not used, must be set to 1 in dim)
+     *   (y and z not used, must be set to 1 in dim)
      */
     const size_type tidx = threadIdx.x;
     constexpr size_type col_idx{0};
@@ -660,7 +432,7 @@ __global__ __launch_bounds__(block_size) void singledot_kernel_num_iters_2(
 // total.
 template <int block_size, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(block_size) void update_next_krylov_kernel(
-    size_type k, size_type num_rows, size_type num_cols,
+    size_type num_iters, size_type num_rows, size_type num_cols,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
     const Accessor3d krylov_bases,
     const ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
@@ -673,425 +445,14 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel(
     if (row_idx < num_rows && col_idx < num_cols &&
         !stop_status[col_idx].has_stopped()) {
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-        const auto hessenberg_idx = k * stride_hessenberg + col_idx;
-
-        next_krylov_basis[next_krylov_idx] -=
-            hessenberg_iter[hessenberg_idx] * krylov_bases(k, row_idx, col_idx);
-    }
-}
-
-
-// Must be called with at least `num_rows * stride_next_krylov` threads in
-// total.
-template <int block_size, typename ValueType, typename Accessor3d>
-__global__
-    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters(
-        size_type num_iters, size_type num_rows, size_type num_cols,
-        ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        const Accessor3d krylov_bases,
-        const ValueType *__restrict__ hessenberg_iter,
-        size_type stride_hessenberg,
-        const stopping_status *__restrict__ stop_status)
-{
-    const auto global_id = thread::get_thread_id_flat();
-    const auto row_idx = global_id / stride_next_krylov;
-    const auto col_idx = global_id % stride_next_krylov;
-
-    if (row_idx < num_rows && col_idx < num_cols &&
-        !stop_status[col_idx].has_stopped()) {
-        const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+        auto local_res = next_krylov_basis[next_krylov_idx];
         for (size_type k = 0; k < num_iters; ++k) {
-            // const auto krylov_idx =
-            //    row_idx * stride_krylov + k * num_cols + col_idx;
             const auto hessenberg_idx = k * stride_hessenberg + col_idx;
 
-            next_krylov_basis[next_krylov_idx] -=
-                hessenberg_iter[hessenberg_idx] *
-                krylov_bases(k, row_idx, col_idx);
-        }
-    }
-}
-
-
-// Must be called with at least `num_vectors *
-// krylov_bases.get_accessor().get_stride()[0]` threads in total.
-template <int block_size, typename ValueType, typename Accessor3d>
-__global__
-    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_0(
-        size_type num_iters, size_type num_rows, size_type num_cols,
-        ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        const Accessor3d krylov_bases,
-        const ValueType *__restrict__ hessenberg_iter,
-        size_type stride_hessenberg,
-        const stopping_status *__restrict__ stop_status)
-{
-    // CAREFUL: krylov_bases changes might not be done properly since this
-    //          kernel is currently not used!
-    const auto global_id = thread::get_thread_id_flat();
-    const auto krylov_stride =
-        helper_functions_accessor<Accessor3d>::get_stride(krylov_bases);
-    const auto k = global_id / krylov_stride[0];
-    const auto reminder = global_id / krylov_stride[0];
-    const auto row_idx = reminder / krylov_stride[1];
-    const auto col_idx = reminder % krylov_stride[1];
-
-    if (row_idx < num_rows && col_idx < num_cols &&
-        !stop_status[col_idx].has_stopped()) {
-        const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-        const auto hessenberg_idx = k * stride_hessenberg + col_idx;
-
-        auto value =
-            hessenberg_iter[hessenberg_idx] * krylov_bases(k, row_idx, col_idx);
-        atomic_add(next_krylov_basis + next_krylov_idx, -value);
-    }
-}
-
-
-// Must be called with at least `num_rows * stride_next_krylov` threads in
-// total.
-template <int block_size, typename ValueType, typename Accessor3d>
-__global__
-    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_1(
-        size_type num_iters, size_type num_rows, size_type num_cols,
-        ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        const Accessor3d krylov_bases,
-        const ValueType *__restrict__ hessenberg_iter,
-        size_type stride_hessenberg,
-        const stopping_status *__restrict__ stop_status)
-{
-    if (num_cols == 1) {
-        const auto tidx = threadIdx.x;
-        const size_type row_idx = blockIdx.x;
-        constexpr decltype(row_idx) col_idx = 0;
-
-        // Used that way to get around dynamic initialization warning and
-        // template error when using `reduction_helper_array` directly in
-        // `reduce`
-        __shared__ UninitializedArray<ValueType, block_size>
-            reduction_helper_array;
-        ValueType *__restrict__ reduction_helper = reduction_helper_array;
-        if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
-            ValueType value = zero<ValueType>();
-            for (size_type i = tidx; i < num_iters; i += blockDim.x) {
-                const auto hessenberg_idx = i * stride_hessenberg + col_idx;
-
-                value += hessenberg_iter[hessenberg_idx] *
-                         krylov_bases(i, row_idx, col_idx);
-            }
-            reduction_helper[tidx] = value;
-
-            // Perform thread block reduction. Result is in
-            // reduction_helper[0]
-            reduce(
-                group::this_thread_block(), reduction_helper,
-                [](const ValueType &a, const ValueType &b) { return a + b; });
-            if (tidx == 0) {
-                const auto next_krylov_idx =
-                    row_idx * stride_next_krylov + col_idx;
-                next_krylov_basis[next_krylov_idx] -= reduction_helper[0];
-            }
-        }
-    } else {
-        const auto global_id = thread::get_thread_id_flat();
-        const auto row_idx = global_id / stride_next_krylov;
-        const auto col_idx = global_id % stride_next_krylov;
-
-        if (row_idx < num_rows && col_idx < num_cols &&
-            !stop_status[col_idx].has_stopped()) {
-            const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-            for (size_type k = 0; k < num_iters; ++k) {
-                const auto hessenberg_idx = k * stride_hessenberg + col_idx;
-
-                next_krylov_basis[next_krylov_idx] -=
-                    hessenberg_iter[hessenberg_idx] *
-                    krylov_bases(k, row_idx, col_idx);
-            }
-        }
-    }
-}
-
-
-// Must be called with at least `num_rows * stride_next_krylov` threads in
-// total.
-template <int block_size, typename ValueType, typename Accessor3d>
-__global__
-    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_2(
-        size_type num_iters, size_type num_blocks, size_type num_rows,
-        size_type num_cols, ValueType *__restrict__ next_krylov_basis,
-        size_type stride_next_krylov, const Accessor3d krylov_bases,
-        const ValueType *__restrict__ hessenberg_iter,
-        size_type stride_hessenberg,
-        const stopping_status *__restrict__ stop_status)
-{
-    if (num_cols == 1) {
-        const auto tidx = threadIdx.x;
-        const size_type row_idx = blockIdx.x * num_blocks;
-        const auto col_idx = 0;
-
-        // Used that way to get around dynamic initialization warning and
-        // template error when using `reduction_helper_array` directly in
-        // `reduce`
-        __shared__ UninitializedArray<ValueType, block_size * block_size>
-            reduction_helper_array;
-        ValueType *__restrict__ reduction_helper = reduction_helper_array;
-        if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
-            //            for (size_type j = 0; j < num_blocks; j++) {
-            for (size_type j = row_idx, k = 0; j < num_rows && k < num_blocks;
-                 j++, k++) {
-                ValueType value = zero<ValueType>();
-                for (size_type i = tidx; i < num_iters; i += blockDim.x) {
-                    const auto hessenberg_idx = i * stride_hessenberg + col_idx;
-
-                    value += hessenberg_iter[hessenberg_idx] *
-                             krylov_bases(i, j, col_idx);
-                }
-                reduction_helper[k * block_size + tidx] = value;
-            }
-
-            //            for (size_type j = 0; j < num_blocks; j++) {
-            for (size_type j = row_idx, k = 0; j < num_rows && k < num_blocks;
-                 j++, k++) {
-                // Perform thread block reduction. Result is in
-                // reduction_helper[0]
-                reduce(group::this_thread_block(),
-                       reduction_helper + k * block_size,
-                       [](const ValueType &a, const ValueType &b) {
-                           return a + b;
-                       });
-            }
-            if (tidx == 0) {
-                // for (size_type j = 0; j < num_blocks; j++) {
-                for (size_type j = row_idx, k = 0;
-                     j < num_rows && k < num_blocks; j++, k++) {
-                    // (row_idx + j) * stride_next_krylov + col_idx;
-                    const auto next_krylov_idx =
-                        j * stride_next_krylov + col_idx;
-                    next_krylov_basis[next_krylov_idx] -=
-                        reduction_helper[k * block_size];
-                    // reduction_helper[j * block_size];
-                }
-            }
-        }
-    } else {
-        const auto global_id = thread::get_thread_id_flat();
-        const auto row_idx = global_id / stride_next_krylov;
-        const auto col_idx = global_id % stride_next_krylov;
-
-        if (row_idx < num_rows && col_idx < num_cols &&
-            !stop_status[col_idx].has_stopped()) {
-            const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-            for (size_type k = 0; k < num_iters; ++k) {
-                const auto hessenberg_idx = k * stride_hessenberg + col_idx;
-
-                next_krylov_basis[next_krylov_idx] -=
-                    hessenberg_iter[hessenberg_idx] *
-                    krylov_bases(k, row_idx, col_idx);
-            }
-        }
-    }
-}
-
-
-// Must be called with at least `num_rows * stride_next_krylov` threads in
-// total.
-template <int block_size, typename ValueType, typename Accessor3d>
-__global__
-    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_3(
-        size_type num_iters, size_type num_blocks, size_type num_rows,
-        size_type num_cols, ValueType *__restrict__ next_krylov_basis,
-        size_type stride_next_krylov, const Accessor3d krylov_bases,
-        const ValueType *__restrict__ hessenberg_iter,
-        size_type stride_hessenberg,
-        const stopping_status *__restrict__ stop_status)
-{
-    if (num_cols == 1) {
-        const auto tidx = threadIdx.x;
-        const size_type row_idx = blockIdx.x * num_blocks;
-        const auto col_idx = 0;
-
-        // Used that way to get around dynamic initialization warning and
-        // template error when using `reduction_helper_array` directly in
-        // `reduce`
-        __shared__ UninitializedArray<ValueType, block_size * block_size>
-            reduction_helper_array;
-        ValueType *__restrict__ reduction_helper = reduction_helper_array;
-        if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
-            //            for (size_type j = 0; j < num_blocks; j++) {
-
-            for (size_type j = row_idx, k = 0; j < num_rows && k < num_blocks;
-                 j++, k++) {
-                reduction_helper[k * block_size + tidx] = zero<ValueType>();
-            }
-            for (size_type i = tidx; i < num_iters; i += blockDim.x) {
-                const auto hessenberg_idx = i * stride_hessenberg + col_idx;
-                const auto hessenberg_val = hessenberg_iter[hessenberg_idx];
-                for (size_type j = row_idx, k = 0;
-                     j < num_rows && k < num_blocks; j++, k++) {
-                    reduction_helper[k * block_size + tidx] +=
-                        hessenberg_val * krylov_bases(i, j, col_idx);
-                }
-            }
-
-            //            for (size_type j = 0; j < num_blocks; j++) {
-            for (size_type j = row_idx, k = 0; j < num_rows && k < num_blocks;
-                 j++, k++) {
-                // Perform thread block reduction. Result is in
-                // reduction_helper[0]
-                reduce(group::this_thread_block(),
-                       reduction_helper + k * block_size,
-                       [](const ValueType &a, const ValueType &b) {
-                           return a + b;
-                       });
-            }
-            if (tidx == 0) {
-                // for (size_type j = 0; j < num_blocks; j++) {
-                for (size_type j = row_idx, k = 0;
-                     j < num_rows && k < num_blocks; j++, k++) {
-                    // (row_idx + j) * stride_next_krylov + col_idx;
-                    const auto next_krylov_idx =
-                        j * stride_next_krylov + col_idx;
-                    next_krylov_basis[next_krylov_idx] -=
-                        reduction_helper[k * block_size];
-                    // reduction_helper[j * block_size];
-                }
-            }
-        }
-    } else {
-        const auto global_id = thread::get_thread_id_flat();
-        const auto row_idx = global_id / stride_next_krylov;
-        const auto col_idx = global_id % stride_next_krylov;
-
-        if (row_idx < num_rows && col_idx < num_cols &&
-            !stop_status[col_idx].has_stopped()) {
-            const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-            for (size_type k = 0; k < num_iters; ++k) {
-                // const auto krylov_idx =
-                //    row_idx * stride_krylov + k * num_cols + col_idx;
-                const auto hessenberg_idx = k * stride_hessenberg + col_idx;
-
-                next_krylov_basis[next_krylov_idx] -=
-                    hessenberg_iter[hessenberg_idx] *
-                    krylov_bases(k, row_idx, col_idx);
-            }
-        }
-    }
-}
-
-
-// Must be called with at least `num_rows * stride_next_krylov` threads in
-// total.
-template <int block_size, typename ValueType, typename Accessor3d>
-__global__
-    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_4(
-        size_type num_iters, size_type num_blocks, size_type num_rows,
-        size_type num_cols, ValueType *__restrict__ next_krylov_basis,
-        size_type stride_next_krylov, const Accessor3d krylov_bases,
-        const ValueType *__restrict__ hessenberg_iter,
-        size_type stride_hessenberg,
-        const stopping_status *__restrict__ stop_status)
-{
-    if (num_cols == 1) {
-        const auto tidx = threadIdx.x;
-        const size_type row_idx = blockIdx.x * num_blocks;
-        const auto col_idx = 0;
-
-        // Used that way to get around dynamic initialization warning and
-        // template error when using `reduction_helper_array` directly in
-        // `reduce`
-        __shared__ UninitializedArray<ValueType, block_size * block_size>
-            reduction_helper_array;
-        ValueType *__restrict__ reduction_helper = reduction_helper_array;
-        if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
-            //            for (size_type j = 0; j < num_blocks; j++) {
-
-            for (size_type j = row_idx, k = 0; j < num_rows && k < num_blocks;
-                 j++, k++) {
-                reduction_helper[k * block_size + tidx] = zero<ValueType>();
-            }
-            for (size_type i = tidx; i < num_iters; i += blockDim.x) {
-                const auto hessenberg_idx = i * stride_hessenberg + col_idx;
-                const auto hessenberg_val = hessenberg_iter[hessenberg_idx];
-                for (size_type j = row_idx, k = 0;
-                     j < num_rows && k < num_blocks; j++, k++) {
-                    reduction_helper[k * block_size + tidx] +=
-                        hessenberg_val * krylov_bases(i, j, col_idx);
-                }
-            }
-            multireduce(
-                group::this_thread_block(), reduction_helper, block_size,
-                block_size,
-                [](const ValueType &a, const ValueType &b) { return a + b; });
-
-            if (tidx == 0) {
-                // for (size_type j = 0; j < num_blocks; j++) {
-                for (size_type j = row_idx, k = 0;
-                     j < num_rows && k < num_blocks; j++, k++) {
-                    // (row_idx + j) * stride_next_krylov + col_idx;
-                    const auto next_krylov_idx =
-                        j * stride_next_krylov + col_idx;
-                    next_krylov_basis[next_krylov_idx] -=
-                        reduction_helper[k * block_size];
-                    // reduction_helper[j * block_size];
-                }
-            }
-        }
-    } else {
-        const auto global_id = thread::get_thread_id_flat();
-        const auto row_idx = global_id / stride_next_krylov;
-        const auto col_idx = global_id % stride_next_krylov;
-
-        if (row_idx < num_rows && col_idx < num_cols &&
-            !stop_status[col_idx].has_stopped()) {
-            const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-            for (size_type k = 0; k < num_iters; ++k) {
-                const auto hessenberg_idx = k * stride_hessenberg + col_idx;
-
-                next_krylov_basis[next_krylov_idx] -=
-                    hessenberg_iter[hessenberg_idx] *
-                    krylov_bases(k, row_idx, col_idx);
-            }
-        }
-    }
-}
-
-
-// Must be called with at least `num_rows * stride_next_krylov` threads in
-// total.
-template <int block_size, typename ValueType, typename Accessor3d>
-__global__
-    __launch_bounds__(block_size) void update_next_krylov_kernel_and_check(
-        size_type k, size_type num_rows, size_type num_cols,
-        ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        const Accessor3d krylov_bases,
-        const ValueType *__restrict__ hessenberg_iter,
-        size_type stride_hessenberg,
-        const remove_complex<ValueType> *__restrict__ arnoldi_norm,
-        size_type stride_norm, const stopping_status *__restrict__ stop_status,
-        stopping_status *__restrict__ reorth_status,
-        size_type *__restrict__ num_reorth)
-{
-    const auto global_id = thread::get_thread_id_flat();
-    const auto row_idx = global_id / stride_next_krylov;
-    const auto col_idx = global_id % stride_next_krylov;
-
-    if (row_idx < num_rows && col_idx < num_cols &&
-        !stop_status[col_idx].has_stopped()) {
-        const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-        const auto hessenberg_idx = k * stride_hessenberg + col_idx;
-        const auto arnoldi_idx = col_idx;
-        const auto reorth_idx = col_idx;
-
-        next_krylov_basis[next_krylov_idx] -=
-            hessenberg_iter[hessenberg_idx] * krylov_bases(k, row_idx, col_idx);
-        if (row_idx == 0) {
-            if (squared_norm(hessenberg_iter[hessenberg_idx]) >
-                (0.99 * sqrt(arnoldi_norm[arnoldi_idx]))) {
-                reorth_status[reorth_idx].reset();
-                atomic_add(num_reorth, (size_type)1);
-            } else {
-                reorth_status[reorth_idx].stop(1);
-            }
+            local_res -= hessenberg_iter[hessenberg_idx] *
+                         krylov_bases(k, row_idx, col_idx);
         }
+        next_krylov_basis[next_krylov_idx] = local_res;
     }
 }
 
@@ -1099,8 +460,8 @@ __global__
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
 template <int block_size, typename ValueType, typename Accessor3d>
-__global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
-    size_type k, size_type num_rows, size_type num_cols,
+__global__ __launch_bounds__(block_size) void update_next_krylov_and_add_kernel(
+    size_type num_iters, size_type num_rows, size_type num_cols,
     ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
     const Accessor3d krylov_bases, ValueType *__restrict__ hessenberg_iter,
     size_type stride_hessenberg, const ValueType *__restrict__ buffer_iter,
@@ -1115,151 +476,19 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_kernel_and_add(
         !stop_status[col_idx].has_stopped() &&
         !reorth_status[col_idx].has_stopped()) {
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-        const auto hessenberg_idx = k * stride_hessenberg + col_idx;
-        const auto buffer_idx = k * stride_buffer + col_idx;
-        next_krylov_basis[next_krylov_idx] -=
-            buffer_iter[buffer_idx] * krylov_bases(k, row_idx, col_idx);
-        if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
-            hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
-        }
-    }
-}
-
-
-// Must be called with at least `num_rows * stride_next_krylov` threads in
-// total.
-template <int block_size, typename ValueType, typename Accessor3d>
-__global__
-    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_and_add(
-        size_type num_iters, size_type num_rows, size_type num_cols,
-        ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        const Accessor3d krylov_bases, ValueType *__restrict__ hessenberg_iter,
-        size_type stride_hessenberg, const ValueType *__restrict__ buffer_iter,
-        size_type stride_buffer,
-        const stopping_status *__restrict__ stop_status,
-        const stopping_status *__restrict__ reorth_status)
-{
-    const auto global_id = thread::get_thread_id_flat();
-    const auto row_idx = global_id / stride_next_krylov;
-    const auto col_idx = global_id % stride_next_krylov;
-
-    if (row_idx < num_rows && col_idx < num_cols &&
-        !stop_status[col_idx].has_stopped() &&
-        !reorth_status[col_idx].has_stopped()) {
-        const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+        auto local_res = next_krylov_basis[next_krylov_idx];
         for (size_type k = 0; k < num_iters; ++k) {
             // const auto krylov_idx =
             //    row_idx * stride_krylov + k * num_cols + col_idx;
             const auto hessenberg_idx = k * stride_hessenberg + col_idx;
             const auto buffer_idx = k * stride_buffer + col_idx;
-            next_krylov_basis[next_krylov_idx] -=
+            local_res -=
                 buffer_iter[buffer_idx] * krylov_bases(k, row_idx, col_idx);
             if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
                 hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
             }
         }
-    }
-}
-
-// Must be called with at least `num_rows * stride_next_krylov` threads in
-// total.
-template <int block_size, typename ValueType, typename Accessor3d>
-__global__
-    __launch_bounds__(block_size) void update_next_krylov_kernel_num_iters_and_add_1(
-        size_type num_iters, size_type num_rows, size_type num_cols,
-        ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
-        const Accessor3d krylov_bases, ValueType *__restrict__ hessenberg_iter,
-        size_type stride_hessenberg, const ValueType *__restrict__ buffer_iter,
-        size_type stride_buffer,
-        const stopping_status *__restrict__ stop_status,
-        const stopping_status *__restrict__ reorth_status)
-{
-    if (num_cols == 1) {
-        const auto tidx = threadIdx.x;
-        const size_type row_idx = blockIdx.x;
-        constexpr decltype(row_idx) col_idx = 0;
-
-        // Used that way to get around dynamic initialization warning and
-        // template error when using `reduction_helper_array` directly in
-        // `reduce`
-        __shared__ UninitializedArray<ValueType, block_size>
-            reduction_helper_array;
-        ValueType *__restrict__ reduction_helper = reduction_helper_array;
-        if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
-            ValueType value = zero<ValueType>();
-            for (size_type i = tidx; i < num_iters; i += blockDim.x) {
-                const auto hessenberg_idx = i * stride_hessenberg + col_idx;
-                const auto buffer_idx = i * stride_buffer + col_idx;
-
-                value +=
-                    buffer_iter[buffer_idx] * krylov_bases(i, row_idx, col_idx);
-                if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
-                    hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
-                }
-            }
-            reduction_helper[tidx] = value;
-
-            // Perform thread block reduction. Result is in
-            // reduction_helper[0]
-            reduce(
-                group::this_thread_block(), reduction_helper,
-                [](const ValueType &a, const ValueType &b) { return a + b; });
-            if (tidx == 0) {
-                const auto next_krylov_idx =
-                    row_idx * stride_next_krylov + col_idx;
-                next_krylov_basis[next_krylov_idx] -= reduction_helper[0];
-            }
-        }
-    } else {
-        const auto global_id = thread::get_thread_id_flat();
-        const auto row_idx = global_id / stride_next_krylov;
-        const auto col_idx = global_id % stride_next_krylov;
-
-        if (row_idx < num_rows && col_idx < num_cols &&
-            !stop_status[col_idx].has_stopped() &&
-            !reorth_status[col_idx].has_stopped()) {
-            const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
-            for (size_type k = 0; k < num_iters; ++k) {
-                // const auto krylov_idx =
-                //    row_idx * stride_krylov + k * num_cols + col_idx;
-                const auto hessenberg_idx = k * stride_hessenberg + col_idx;
-                const auto buffer_idx = k * stride_buffer + col_idx;
-                next_krylov_basis[next_krylov_idx] -=
-                    buffer_iter[buffer_idx] * krylov_bases(k, row_idx, col_idx);
-                if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
-                    hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
-                }
-            }
-        }
-    }
-}
-
-
-// Must be called with at least `num_cols` blocks, each with `block_size`
-// threads. `block_size` must be a power of 2.
-template <int block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void check_arnoldi_norms(
-    const ValueType *__restrict__ arnoldi_norm0, size_type stride_norm0,
-    const ValueType *__restrict__ arnoldi_norm1, size_type stride_norm1,
-    const stopping_status *__restrict__ stop_status,
-    stopping_status *__restrict__ reorth_status,
-    size_type *__restrict__ num_reorth)
-{
-    const ValueType eta = 1.0 / 2.0;
-    const auto global_id = thread::get_thread_id_flat();
-
-    if (global_id < stride_norm0 && !stop_status[global_id].has_stopped()) {
-        const auto num0 = (eta * arnoldi_norm0[global_id]);
-        // const auto num0 = abs(eta * arnoldi_norm0[global_id]);
-        const auto num1 = (arnoldi_norm1[global_id]);
-        // const auto num1 = abs(arnoldi_norm1[global_id]);
-        // printf("num0 = %f , num1 = %f\n", num0, num1);
-        if (num1 < num0) {
-            reorth_status[global_id].reset();
-            atomic_add(num_reorth, (size_type)1);
-        } else {
-            reorth_status[global_id].stop(1);
-        }
+        next_krylov_basis[next_krylov_idx] = local_res;
     }
 }
 
@@ -1267,7 +496,7 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms(
 // Must be called with at least `num_cols` blocks, each with `block_size`
 // threads. `block_size` must be a power of 2.
 template <int block_size, typename ValueType, typename Accessor3d>
-__global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
+__global__ __launch_bounds__(block_size) void check_arnoldi_norms(
     remove_complex<ValueType> *__restrict__ arnoldi_norm, size_type stride_norm,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     size_type iter, Accessor3d krylov_bases,
@@ -1282,15 +511,10 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
 
     if (col_idx < stride_norm && !stop_status[col_idx].has_stopped()) {
         const auto num0 = (eta * arnoldi_norm[col_idx]);
-        // const auto num0 = abs(eta * arnoldi_norm0[col_idx]);
         const auto num11 = (arnoldi_norm[col_idx + stride_norm]);
         const auto num12 = sqrt(num11);
-        // const auto num1 = abs(arnoldi_norm1[col_idx]);
         const auto num2 = has_scale ? (arnoldi_norm[col_idx + 2 * stride_norm])
                                     : remove_complex<ValueType>{};
-        // printf("num0 = %f , num11 = %f , num12 = %f , num2 = %f\n", num0,
-        // num11,
-        //        num12, num2);
         if (num11 < num0) {
             reorth_status[col_idx].reset();
             atomic_add(num_reorth, (size_type)1);
@@ -1300,13 +524,8 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms_new(
         arnoldi_norm[col_idx] = num11;
         arnoldi_norm[col_idx + stride_norm] = num12;
         hessenberg_iter[iter * stride_hessenberg + col_idx] = num12;
-        // printf("(%d) num1 = %10.5e , num2 = %10.5e\n", cod_idx, num1, num2);
-        // printf("%d - %d - %d - %d\n", stride_norm, iter, col_idx,
-        //        stride_norm * iter + col_idx);
         helper_functions_accessor<Accessor3d>::write_scalar(
             krylov_bases, iter, col_idx, num2 / num12);
-        // krylov_bases, num_cols * (iter + 1) + col_idx, num2 / num12);
-        // krylov_bases, col_idx, num2 / num12);
     }
 }
 
@@ -1332,7 +551,6 @@ __global__ __launch_bounds__(block_size) void set_scale_kernel(
         if (blk_idx == 0) {
             const auto num1 = residual_norm[col_idx];
             const auto num2 = arnoldi_inf[col_idx];
-            // printf("num1 = %10.5e , num2 = %10.5e\n", num1, num2);
             helper_functions_accessor<Accessor3d>::write_scalar(
                 krylov_bases, {0}, col_idx, num2 / num1);
         } else {
@@ -1344,48 +562,6 @@ __global__ __launch_bounds__(block_size) void set_scale_kernel(
 }
 
 
-// Must be called with at least `num_cols` blocks, each with `block_size`
-// threads. `block_size` must be a power of 2.
-template <int block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void update_hessenberg_2_kernel(
-    size_type iter, size_type num_rows, size_type num_cols,
-    const ValueType *__restrict__ next_krylov_basis,
-    size_type stride_next_krylov, ValueType *__restrict__ hessenberg_iter,
-    size_type stride_hessenberg,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = threadIdx.x;
-    const auto col_idx = blockIdx.x;
-
-    // Used that way to get around dynamic initialization warning and
-    // template error when using `reduction_helper_array` directly in
-    // `reduce`
-    __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
-    ValueType *__restrict__ reduction_helper = reduction_helper_array;
-
-    if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
-        ValueType local_res{};
-        for (size_type i = tidx; i < num_rows; i += block_size) {
-            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
-            const auto next_krylov_value = next_krylov_basis[next_krylov_idx];
-
-            local_res += next_krylov_value * next_krylov_value;
-        }
-
-        reduction_helper[tidx] = local_res;
-
-        // Perform thread block reduction. Result is in reduction_helper[0]
-        reduce(group::this_thread_block(), reduction_helper,
-               [](const ValueType &a, const ValueType &b) { return a + b; });
-
-        if (tidx == 0) {
-            hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] =
-                sqrt(reduction_helper[0]);
-        }
-    }
-}
-
-
 // Must be called with at least `num_rows * stride_next_krylov` threads in
 // total.
 template <int block_size, typename ValueType, typename Accessor3d>
diff --git a/cuda/solver/cb_gmres_kernels.cu b/cuda/solver/cb_gmres_kernels.cu
index 9f9157a5a04..ca55042052a 100644
--- a/cuda/solver/cb_gmres_kernels.cu
+++ b/cuda/solver/cb_gmres_kernels.cu
@@ -174,7 +174,7 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
     const dim3 grid_size_nrm(ceildiv(num_rhs, default_dot_dim),
                              exec->get_num_multiprocessor() * 2);
     const dim3 block_size_nrm(default_dot_dim, default_dot_dim);
-    multinorminf_kernel_without_stop<<<grid_size_nrm, block_size_nrm>>>(
+    multinorminf_without_stop_kernel<<<grid_size_nrm, block_size_nrm>>>(
         num_rows, num_rhs, as_cuda_type(residual->get_const_values()), num_rhs,
         as_cuda_type(arnoldi_norm->get_values() + 2 * num_rhs), 0);
 
@@ -258,15 +258,14 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     zero_matrix(iter + 1, dim_size[1], stride_hessenberg,
                 hessenberg_iter->get_values());
     if (dim_size[1] > 1) {
-        multidot_kernel_num_iters_2<default_dot_dim>
-            <<<grid_size_num_iters_2, block_size>>>(
-                iter + 1, dim_size[0], dim_size[1],
-                as_cuda_type(next_krylov_basis->get_const_values()),
-                stride_next_krylov, as_cuda_accessor(krylov_bases),
-                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-                as_cuda_type(stop_status));
+        multidot_kernel<default_dot_dim><<<grid_size_num_iters_2, block_size>>>(
+            iter + 1, dim_size[0], dim_size[1],
+            as_cuda_type(next_krylov_basis->get_const_values()),
+            stride_next_krylov, as_cuda_accessor(krylov_bases),
+            as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
+            as_cuda_type(stop_status));
     } else {
-        singledot_kernel_num_iters_2<singledot_block_size>
+        singledot_kernel<singledot_block_size>
             <<<grid_size_iters_single, block_size_iters_single>>>(
                 iter + 1, dim_size[0],
                 as_cuda_type(next_krylov_basis->get_const_values()),
@@ -277,7 +276,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     // for i in 1:iter
     //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
     // end
-    update_next_krylov_kernel_num_iters<default_block_size>
+    update_next_krylov_kernel<default_block_size>
         <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
            default_block_size>>>(
             iter + 1, dim_size[0], dim_size[1],
@@ -301,7 +300,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         as_cuda_type(stop_status));
     // nrmN = norm(next_krylov_basis)
     components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
-    check_arnoldi_norms_new<default_block_size>
+    check_arnoldi_norms<default_block_size>
         <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
             as_cuda_type(arnoldi_norm->get_values()), stride_arnoldi,
             as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
@@ -317,7 +316,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         zero_matrix(iter + 1, dim_size[1], stride_buffer,
                     buffer_iter->get_values());
         if (dim_size[1] > 1) {
-            multidot_kernel_num_iters_2<default_dot_dim>
+            multidot_kernel<default_dot_dim>
                 <<<grid_size_num_iters_2, block_size>>>(
                     iter + 1, dim_size[0], dim_size[1],
                     as_cuda_type(next_krylov_basis->get_const_values()),
@@ -325,7 +324,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                     as_cuda_type(buffer_iter->get_values()), stride_buffer,
                     as_cuda_type(stop_status));
         } else {
-            singledot_kernel_num_iters_2<singledot_block_size>
+            singledot_kernel<singledot_block_size>
                 <<<grid_size_iters_single, block_size_iters_single>>>(
                     iter + 1, dim_size[0],
                     as_cuda_type(next_krylov_basis->get_const_values()),
@@ -337,7 +336,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
         // end
         //*
-        update_next_krylov_kernel_num_iters_and_add<default_block_size>
+        update_next_krylov_and_add_kernel<default_block_size>
             <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
                default_block_size>>>(
                 iter + 1, dim_size[0], dim_size[1],
@@ -364,7 +363,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         // nrmN = norm(next_krylov_basis)
         components::fill_array(exec, num_reorth->get_data(), 1,
                                zero<size_type>());
-        check_arnoldi_norms_new<default_block_size>
+        check_arnoldi_norms<default_block_size>
             <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
                 as_cuda_type(arnoldi_norm->get_values()), stride_arnoldi,
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
diff --git a/hip/solver/cb_gmres_kernels.hip.cpp b/hip/solver/cb_gmres_kernels.hip.cpp
index 941e7e6e69f..02c8b519c85 100644
--- a/hip/solver/cb_gmres_kernels.hip.cpp
+++ b/hip/solver/cb_gmres_kernels.hip.cpp
@@ -172,7 +172,7 @@ void initialize_2(std::shared_ptr<const HipExecutor> exec,
                              exec->get_num_multiprocessor() * 2);
     const dim3 block_size_nrm(default_dot_dim, default_dot_dim);
     hipLaunchKernelGGL(
-        multinorminf_kernel_without_stop, grid_size_nrm, block_size_nrm, 0, 0,
+        multinorminf_without_stop_kernel, grid_size_nrm, block_size_nrm, 0, 0,
         num_rows, num_rhs, as_hip_type(residual->get_const_values()), num_rhs,
         as_hip_type(arnoldi_norm->get_values() + 2 * num_rhs), 0);
 
@@ -253,7 +253,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
     zero_matrix(iter + 1, dim_size[1], stride_hessenberg,
                 hessenberg_iter->get_values());
     if (dim_size[1] > 1) {
-        hipLaunchKernelGGL(multidot_kernel_num_iters_2<default_dot_dim>,
+        hipLaunchKernelGGL(multidot_kernel<default_dot_dim>,
                            grid_size_num_iters_2, block_size, 0, 0, iter + 1,
                            dim_size[0], dim_size[1],
                            as_hip_type(next_krylov_basis->get_const_values()),
@@ -261,7 +261,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
                            as_hip_type(hessenberg_iter->get_values()),
                            stride_hessenberg, as_hip_type(stop_status));
     } else {
-        hipLaunchKernelGGL(singledot_kernel_num_iters_2<singledot_block_size>,
+        hipLaunchKernelGGL(singledot_kernel<singledot_block_size>,
                            grid_size_iters_single, block_size_iters_single, 0,
                            0, iter + 1, dim_size[0],
                            as_hip_type(next_krylov_basis->get_const_values()),
@@ -273,7 +273,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
     //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
     // end
     hipLaunchKernelGGL(
-        update_next_krylov_kernel_num_iters<default_block_size>,
+        update_next_krylov_kernel<default_block_size>,
         dim3(ceildiv(dim_size[0] * stride_next_krylov, default_block_size)),
         dim3(default_block_size), 0, 0, iter + 1, dim_size[0], dim_size[1],
         as_hip_type(next_krylov_basis->get_values()), stride_next_krylov,
@@ -298,7 +298,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
     // nrmN = norm(next_krylov_basis)
     components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
     hipLaunchKernelGGL(
-        check_arnoldi_norms_new<default_block_size>,
+        check_arnoldi_norms<default_block_size>,
         dim3(ceildiv(dim_size[1], default_block_size)),
         dim3(default_block_size), 0, 0, as_hip_type(arnoldi_norm->get_values()),
         stride_arnoldi, as_hip_type(hessenberg_iter->get_values()),
@@ -316,17 +316,17 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
                     buffer_iter->get_values());
         if (dim_size[1] > 1) {
             hipLaunchKernelGGL(
-                multidot_kernel_num_iters_2<default_dot_dim>,
-                grid_size_num_iters_2, block_size, 0, 0, iter + 1, dim_size[0],
-                dim_size[1], as_hip_type(next_krylov_basis->get_const_values()),
+                multidot_kernel<default_dot_dim>, grid_size_num_iters_2,
+                block_size, 0, 0, iter + 1, dim_size[0], dim_size[1],
+                as_hip_type(next_krylov_basis->get_const_values()),
                 stride_next_krylov, as_hip_accessor(krylov_bases),
                 as_hip_type(buffer_iter->get_values()), stride_buffer,
                 as_hip_type(stop_status));
         } else {
             hipLaunchKernelGGL(
-                singledot_kernel_num_iters_2<singledot_block_size>,
-                grid_size_iters_single, block_size_iters_single, 0, 0, iter + 1,
-                dim_size[0], as_hip_type(next_krylov_basis->get_const_values()),
+                singledot_kernel<singledot_block_size>, grid_size_iters_single,
+                block_size_iters_single, 0, 0, iter + 1, dim_size[0],
+                as_hip_type(next_krylov_basis->get_const_values()),
                 stride_next_krylov, as_hip_accessor(krylov_bases),
                 as_hip_type(buffer_iter->get_values()), stride_buffer,
                 as_hip_type(stop_status));
@@ -336,7 +336,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
         // end
         //*
         hipLaunchKernelGGL(
-            update_next_krylov_kernel_num_iters_and_add<default_block_size>,
+            update_next_krylov_and_add_kernel<default_block_size>,
             dim3(ceildiv(dim_size[0] * stride_next_krylov, default_block_size)),
             dim3(default_block_size), 0, 0, iter + 1, dim_size[0], dim_size[1],
             as_hip_type(next_krylov_basis->get_values()), stride_next_krylov,
@@ -364,7 +364,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
         components::fill_array(exec, num_reorth->get_data(), 1,
                                zero<size_type>());
         hipLaunchKernelGGL(
-            check_arnoldi_norms_new<default_block_size>,
+            check_arnoldi_norms<default_block_size>,
             dim3(ceildiv(dim_size[1], default_block_size)),
             dim3(default_block_size), 0, 0,
             as_hip_type(arnoldi_norm->get_values()), stride_arnoldi,

From fbdefcff90b05bc744990a8d02f191ccc4b582bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 3 Feb 2021 00:26:54 +0100
Subject: [PATCH 0401/1008] Review update

- Add arnoldi_norm documentation and add it in the tests
  (was not part of it previously, and needs a fix)
- Add documentation to benchmarking
- Rename namespace and helper for range helper

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Yuhsiang M. Tsai <yhmtsai@gmail.com>
---
 BENCHMARKING.md                          |  7 +--
 benchmark/run_all_benchmarks.sh          | 29 ++++++-----
 benchmark/solver/solver.cpp              | 27 ++++++-----
 benchmark/utils/general.hpp              | 10 ++--
 common/components/reduction.hpp.inc      | 13 +++--
 common/solver/cb_gmres_kernels.hpp.inc   | 24 ++++-----
 core/device_hooks/common_kernels.inc.cpp |  8 ---
 core/solver/cb_gmres.cpp                 | 30 +++++-------
 core/solver/cb_gmres_accessor.hpp        | 39 ++++++---------
 cuda/solver/cb_gmres_kernels.cu          | 62 ++++++++++++------------
 cuda/test/solver/cb_gmres_kernels.cpp    |  6 ++-
 hip/solver/cb_gmres_kernels.hip.cpp      | 50 +++++++++++--------
 hip/test/solver/cb_gmres_kernels.cpp     |  6 ++-
 omp/solver/cb_gmres_kernels.cpp          | 42 ++--------------
 omp/test/solver/cb_gmres_kernels.cpp     |  6 ++-
 reference/solver/cb_gmres_kernels.cpp    |  6 +--
 16 files changed, 163 insertions(+), 202 deletions(-)

diff --git a/BENCHMARKING.md b/BENCHMARKING.md
index ea2802510c7..441412d8acd 100644
--- a/BENCHMARKING.md
+++ b/BENCHMARKING.md
@@ -304,9 +304,10 @@ The supported environment variables are described in the following list:
 * `SOLVERS_MAX_ITERATION=<number>` - the maximum number of iterations with which
     a solver should be ran. The default is `10000`.
 * `SOLVERS_RHS={1, random, sinus}` - whether to use a vector of all ones,
-    random values or $b = A * (s / |s|)$ with $s(i) = sin(i)$
+    random values or $b = A * (s / |s|)$ with $s(idx) = sin(idx)$ (for complex
+    numbers, $s(idx) = sin(2*idx) + i * sin(2*idx+1)$)
     as the right-hand side in solver benchmarks. Default is `1`.
-* `SOLVERS_INITIAL_GUESS`={rhs,0,random} - the initial guess generation of the
+* `SOLVERS_INITIAL_GUESS={rhs,0,random}` - the initial guess generation of the
     solvers. `rhs` uses the right-hand side, `0` uses a zero vector and `random`
     generates a random vector as the initial guess.
 * `DETAILED={0,1}` - selects whether detailed benchmarks should be ran for the
@@ -315,6 +316,6 @@ The supported environment variables are described in the following list:
     valid for cuda/hip executor, to measure the timing. Default is `false`.
 * `SOLVERS_JACOBI_MAX_BS` - sets the maximum block size for the Jacobi
     preconditioner (if used, otherwise, it does nothing) in the solvers
-    benchmark
+    benchmark. The default is '32'.
 * `SOLVERS_GMRES_RESTART` - the maximum dimension of the Krylov space to use in
     GMRES. The default is `100`.
diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index eb859fbea7a..422b28c68f1 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -41,18 +41,18 @@ if [ ! "${SOLVERS}" ]; then
 fi
 
 if [ ! "${SOLVERS_PRECISION}" ]; then
-    echo "SOLVERS_PRECISION    environment variable not set - assuming \"1e-6\"" 1>&2
+    echo "SOLVERS_PRECISION environment variable not set - assuming \"1e-6\"" 1>&2
     SOLVERS_PRECISION=1e-6
 fi
 
 if [ ! "${SOLVERS_MAX_ITERATIONS}" ]; then
-    echo "SOLVERS_MAX_ITERATIONS    environment variable not set - assuming \"10000\"" 1>&2
+    echo "SOLVERS_MAX_ITERATIONS environment variable not set - assuming \"10000\"" 1>&2
     SOLVERS_MAX_ITERATIONS=10000
 fi
 
 if [ ! "${SOLVERS_GMRES_RESTART}" ]; then
     SOLVERS_GMRES_RESTART=100
-    echo "SOLVERS_GMRES_RESTART    environment variable not set - assuming \"${SOLVERS_GMRES_RESTART}\"" 1>&2
+    echo "SOLVERS_GMRES_RESTART environment variable not set - assuming \"${SOLVERS_GMRES_RESTART}\"" 1>&2
 fi
 
 if [ ! "${SYSTEM_NAME}" ]; then
@@ -70,12 +70,6 @@ if [ ! "${SOLVERS_JACOBI_MAX_BS}" ]; then
     "SOLVERS_JACOBI_MAX_BS environment variable not set - assuming \"${SOLVERS_JACOBI_MAX_BS}\"" 1>&2
 fi
 
-
-if [ ! "${SOLVERS_RHS}" ]; then
-    SOLVERS_RHS="1"
-    echo "SOLVERS_RHS environment variable not set - assuming \"${SOLVERS_RHS}\"" 1>&2
-fi
-
 if [ ! "${BENCHMARK_PRECISION}" ]; then
     BENCHMARK_PRECISION="double"
     echo "BENCHMARK_PRECISION not set - assuming \"${BENCHMARK_PRECISION}\"" 1>&2
@@ -95,6 +89,11 @@ else
     exit 1
 fi
 
+if [ ! "${SOLVERS_RHS}" ]; then
+    SOLVERS_RHS="1"
+    echo "SOLVERS_RHS environment variable not set - assuming \"${SOLVERS_RHS}\"" 1>&2
+fi
+
 if [ "${SOLVERS_RHS}" == "random" ]; then
     SOLVERS_RHS_FLAG="--rhs_generation=random"
 elif [ "${SOLVERS_RHS}" == "1" ]; then
@@ -112,11 +111,6 @@ if [ ! "${SOLVERS_INITIAL_GUESS}" ]; then
     echo "SOLVERS_RHS environment variable not set - assuming \"${SOLVERS_INITIAL_GUESS}\"" 1>&2
 fi
 
-if [ ! "${GPU_TIMER}" ]; then
-    echo "GPU_TIMER    environment variable not set - assuming \"false\"" 1>&2
-    GPU_TIMER="false"
-fi
-
 if [ "${SOLVERS_INITIAL_GUESS}" == "random" ]; then
     SOLVERS_INITIAL_GUESS_FLAG="--initial_guess_generation=random"
 elif [ "${SOLVERS_INITIAL_GUESS}" == "0" ]; then
@@ -125,10 +119,15 @@ elif [ "${SOLVERS_INITIAL_GUESS}" == "rhs" ]; then
     SOLVERS_INITIAL_GUESS_FLAG="--initial_guess_generation=rhs"
 else
     echo "SOLVERS_RHS does not support the value \"${SOLVERS_RHS}\"." 1>&2
-    echo "The following values are supported: \"1\", \"random\" and \"sinus\"" 1>&2
+    echo "The following values are supported: \"0\", \"random\" and \"rhs\"" 1>&2
     exit 1
 fi
 
+if [ ! "${GPU_TIMER}" ]; then
+    echo "GPU_TIMER    environment variable not set - assuming \"false\"" 1>&2
+    GPU_TIMER="false"
+fi
+
 # Control whether to run detailed benchmarks or not.
 # Default setting is detailed=false. To activate, set DETAILED=1.
 if  [ ! "${DETAILED}" ] || [ "${DETAILED}" -eq 0 ]; then
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 6f5a5409b18..257f10c4f69 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -92,16 +92,17 @@ DEFINE_double(
 DEFINE_string(
     rhs_generation, "1",
     "Method used to generate the right hand side. Supported values are:"
-    "1, random, sinus. 1 sets all values of the right hand side to 1, "
-    "random assigns the values to a uniformly distributed random number "
-    "in [-1, 1), and sinus assigns b = A * (s / |s|) with A := system matrix, "
-    "s := vector with s(i) = sin(i).");
+    "`1`, `random`, `sinus`. `1` sets all values of the right hand side to 1, "
+    "`random` assigns the values to a uniformly distributed random number "
+    "in [-1, 1), and `sinus` assigns b = A * (s / |s|) with A := system matrix,"
+    " s := vector with s(idx) = sin(idx) for non-complex types, and "
+    "s(idx) = sin(2*idx) + i * sin(2*idx+1).");
 
 DEFINE_string(
     initial_guess_generation, "rhs",
     "Method used to generate the initial guess. Supported values are: "
-    "random, rhs, 0. random uses a random vector, rhs uses the right "
-    "hand side, and 0 uses a zero vector as the initial guess.");
+    "`random`, `rhs`, `0`. `random` uses a random vector, `rhs` uses the right "
+    "hand side, and `0 uses a zero vector as the initial guess.");
 
 // This allows to benchmark the overhead of a solver by using the following
 // data: A=[1.0], x=[0.0], b=[nan]. This data can be used to benchmark normal
@@ -111,6 +112,8 @@ DEFINE_bool(overhead, false,
             "If set, uses dummy data to benchmark Ginkgo overhead");
 
 
+// TODO: create string list parameter for CB-GMRES solver
+
 // input validation
 [[noreturn]] void print_config_error_and_exit()
 {
@@ -259,7 +262,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
-        {"cb_gmres_double",
+        {"cb_gmres_keep",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
              return gko::solver::CbGmres<etype>::build()
@@ -270,7 +273,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
-        {"cb_gmres_float",
+        {"cb_gmres_reduced",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
              return gko::solver::CbGmres<etype>::build()
@@ -281,7 +284,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
-        {"cb_gmres_half",
+        {"cb_gmres_reduced2",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
              return gko::solver::CbGmres<etype>::build()
@@ -292,7 +295,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
-        {"cb_gmres_int64",
+        {"cb_gmres_int",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
              return gko::solver::CbGmres<etype>::build()
@@ -303,7 +306,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
-        {"cb_gmres_int32",
+        {"cb_gmres_int_reduced",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
              return gko::solver::CbGmres<etype>::build()
@@ -314,7 +317,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_preconditioner(give(precond))
                  .on(exec);
          }},
-        {"cb_gmres_int16",
+        {"cb_gmres_int_reduced2",
          [](std::shared_ptr<const gko::Executor> exec,
             std::shared_ptr<const gko::LinOpFactory> precond) {
              return gko::solver::CbGmres<etype>::build()
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index ee8cdddf4a4..d03875d87b1 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -304,19 +304,19 @@ create_matrix_sin(std::shared_ptr<const gko::Executor> exec, gko::dim<2> size)
     return res;
 }
 
-// Note: complex values are assigned s[i, j] = {sin(2 * i, sin(2 * i + 1))}
+// Note: complex values are assigned s[i, j] = {sin(2 * i), sin(2 * i + 1)}
 template <typename ValueType>
 std::enable_if_t<gko::is_complex_s<ValueType>::value,
                  std::unique_ptr<vec<ValueType>>>
 create_matrix_sin(std::shared_ptr<const gko::Executor> exec, gko::dim<2> size)
 {
+    using rc_vtype = gko::remove_complex<ValueType>;
     auto h_res = vec<ValueType>::create(exec->get_master(), size);
     for (gko::size_type i = 0; i < size[0]; ++i) {
         for (gko::size_type j = 0; j < size[1]; ++j) {
-            h_res->at(i, j) = ValueType{
-                std::sin(static_cast<gko::remove_complex<ValueType>>(2 * i)),
-                std::sin(
-                    static_cast<gko::remove_complex<ValueType>>(2 * i + 1))};
+            h_res->at(i, j) =
+                ValueType{std::sin(static_cast<rc_vtype>(2 * i)),
+                          std::sin(static_cast<rc_vtype>(2 * i + 1))};
         }
     }
     auto res = vec<ValueType>::create(exec);
diff --git a/common/components/reduction.hpp.inc b/common/components/reduction.hpp.inc
index fd34841c85d..bbdaf1217ae 100644
--- a/common/components/reduction.hpp.inc
+++ b/common/components/reduction.hpp.inc
@@ -131,12 +131,15 @@ __device__ void reduce(const Group &__restrict__ group,
 /**
  * @internal
  *
- * Computes a reduction using the binary operation `reduce_op` on entire block.
- * The data for the reduction is taken from the `data` array which has to be of
- * size `block_size` and accessible from all threads. The `data` array is also
+ * Computes `num` reductions using the binary operation `reduce_op` on an
+ * entire block.
+ * The data range for the ith (i < num) reduction is:
+ * [data + i * stride, data + block_size) (block_size == group.size())
+ * The `data` array for each reduction must be of size `block_size` and
+ * accessible from all threads. The `data` array is also
  * used as work space (so its content will be destroyed in the process), as well
- * as to store the return value - which is stored in the 0-th position of the
- * array.
+ * as to store the return value - which is stored in the (i * stride)-th
+ * position of the array.
  */
 template <
     typename Group, typename ValueType, typename Operator,
diff --git a/common/solver/cb_gmres_kernels.hpp.inc b/common/solver/cb_gmres_kernels.hpp.inc
index 8ddb6ea0149..a50ac0f00a5 100644
--- a/common/solver/cb_gmres_kernels.hpp.inc
+++ b/common/solver/cb_gmres_kernels.hpp.inc
@@ -89,7 +89,8 @@ __global__ __launch_bounds__(block_size) void initialize_2_1_kernel(
 {
     const auto global_id = thread::get_thread_id_flat();
     const auto krylov_stride =
-        helper_functions_accessor<Accessor3d>::get_stride(krylov_bases);
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::get_stride(
+            krylov_bases);
     // krylov indices
     const auto krylov_idx = global_id / krylov_stride[0];
     const auto reminder = global_id % krylov_stride[0];
@@ -123,7 +124,8 @@ __global__ __launch_bounds__(block_size) void initialize_2_2_kernel(
 {
     const auto global_id = thread::get_thread_id_flat();
     const auto krylov_stride =
-        helper_functions_accessor<Accessor3d>::get_stride(krylov_bases);
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::get_stride(
+            krylov_bases);
     const auto row_idx = global_id / krylov_stride[1];
     const auto col_idx = global_id % krylov_stride[1];
 
@@ -248,6 +250,7 @@ __global__
 }
 
 
+// ONLY computes the inf-norm (into norms2) when compute_inf is true
 template <bool compute_inf, typename ValueType>
 __global__ __launch_bounds__(default_dot_size) void multinorm2_inf_kernel(
     size_type num_rows, size_type num_cols,
@@ -507,11 +510,11 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms(
     const remove_complex<ValueType> eta = 1.0 / 2.0;
     const auto col_idx = thread::get_thread_id_flat();
     constexpr bool has_scale =
-        kernels::detail::is_3d_scaled_accessor<Accessor3d>::value;
+        gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value;
 
     if (col_idx < stride_norm && !stop_status[col_idx].has_stopped()) {
         const auto num0 = (eta * arnoldi_norm[col_idx]);
-        const auto num11 = (arnoldi_norm[col_idx + stride_norm]);
+        const auto num11 = arnoldi_norm[col_idx + stride_norm];
         const auto num12 = sqrt(num11);
         const auto num2 = has_scale ? (arnoldi_norm[col_idx + 2 * stride_norm])
                                     : remove_complex<ValueType>{};
@@ -524,16 +527,14 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms(
         arnoldi_norm[col_idx] = num11;
         arnoldi_norm[col_idx + stride_norm] = num12;
         hessenberg_iter[iter * stride_hessenberg + col_idx] = num12;
-        helper_functions_accessor<Accessor3d>::write_scalar(
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
             krylov_bases, iter, col_idx, num2 / num12);
     }
 }
 
 
-// Must be called with at least `num_cols` blocks, each with `block_size`
-// threads. `block_size` must be a power of 2.
 template <int block_size, typename RealValueType, typename Accessor3d>
-__global__ __launch_bounds__(block_size) void set_scale_kernel(
+__global__ __launch_bounds__(block_size) void set_scalar_kernel(
     size_type num_rhs, size_type num_blocks,
     const RealValueType *__restrict__ residual_norm, size_type stride_residual,
     const RealValueType *__restrict__ arnoldi_inf, size_type stride_inf,
@@ -543,7 +544,8 @@ __global__ __launch_bounds__(block_size) void set_scale_kernel(
                   "ValueType must not be complex!");
     const auto global_id = thread::get_thread_id_flat();
     const auto krylov_stride =
-        helper_functions_accessor<Accessor3d>::get_stride(krylov_bases);
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::get_stride(
+            krylov_bases);
     const auto blk_idx = global_id / krylov_stride[1];
     const auto col_idx = global_id % krylov_stride[1];
 
@@ -551,11 +553,11 @@ __global__ __launch_bounds__(block_size) void set_scale_kernel(
         if (blk_idx == 0) {
             const auto num1 = residual_norm[col_idx];
             const auto num2 = arnoldi_inf[col_idx];
-            helper_functions_accessor<Accessor3d>::write_scalar(
+            gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
                 krylov_bases, {0}, col_idx, num2 / num1);
         } else {
             const auto num = one<RealValueType>();
-            helper_functions_accessor<Accessor3d>::write_scalar(
+            gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
                 krylov_bases, blk_idx, col_idx, num);
         }
     }
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index cade2c75062..4afd590647c 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -585,14 +585,6 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(
     GKO_DECLARE_CB_GMRES_INITIALIZE_2_KERNEL);
 
-/*
-template <typename ValueType, typename ValueTypeKrylovBases, bool
-Reorthogonalization, bool MGS_CGS>
-GKO_DECLARE_CB_GMRES_BOOL_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases,
-                                           Reorthogonalization, MGS_CGS)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_MIXED_BOOL_TYPE(GKO_DECLARE_CB_GMRES_BOOL_STEP_1_KERNEL);
-*/
 template <typename ValueType, typename ValueTypeKrylovBases>
 GKO_DECLARE_CB_GMRES_STEP_1_KERNEL(ValueType, ValueTypeKrylovBases)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/solver/cb_gmres.cpp b/core/solver/cb_gmres.cpp
index 3e610f9e62e..f562ca18af8 100644
--- a/core/solver/cb_gmres.cpp
+++ b/core/solver/cb_gmres.cpp
@@ -44,7 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/name_demangling.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/identity.hpp>
+//#include <ginkgo/core/matrix/identity.hpp>
 
 
 #include "core/solver/cb_gmres_accessor.hpp"
@@ -103,7 +103,7 @@ namespace detail {
 template <typename T, typename Skip, int count>
 struct reduce_precision_skip_count_impl {
     static_assert(count > 0,
-                  "The count variable must be larger or equal zero.");
+                  "The count variable must be larger or equal to zero.");
     using type = typename reduce_precision_skip_count_impl<
         reduce_precision_skip<T, Skip>, Skip, count - 1>::type;
 };
@@ -197,10 +197,8 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         using Vector = matrix::Dense<ValueType>;
         using VectorNorms = matrix::Dense<remove_complex<ValueType>>;
         using LowArray = Array<storage_type>;
-        // using KrylovAccessor = kernels::Accessor3d<storage_type,
-        // ValueType>;
-        using Accessor3dHelper =
-            kernels::Accessor3dHelper<ValueType, storage_type>;
+        using Range3dHelper =
+            gko::cb_gmres::Range3dHelper<ValueType, storage_type>;
 
 
         constexpr uint8 RelativeStoppingId{1};
@@ -222,15 +220,7 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         const dim<3> krylov_bases_dim{krylov_dim_ + 1,
                                       system_matrix_->get_size()[1],
                                       dense_b->get_size()[1]};
-        // const dim<2> krylov_bases_dim{
-        //     system_matrix_->get_size()[1],
-        //     (krylov_dim_ + 1) * dense_b->get_size()[1]};
-        // const size_type krylov_bases_stride = krylov_bases_dim[1];
-        // LowArray krylov_bases(exec, krylov_bases_dim[0] *
-        // krylov_bases_stride); KrylovAccessor
-        // krylov_bases_range(krylov_bases.get_data(),
-        //                                     krylov_bases_stride);
-        Accessor3dHelper helper(exec, krylov_bases_dim);
+        Range3dHelper helper(exec, krylov_bases_dim);
         auto krylov_bases_range = helper.get_range();
 
         auto next_krylov_basis = Vector::create_with_config_of(dense_b);
@@ -251,9 +241,13 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
         auto b_norm =
             VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
-        // TODO: write description what the different rows represent
+        // 1st row of arnoldi_norm: TODO
+        // 2nd row of arnoldi_norm: The actual arnoldi norm
+        //                          == norm2(next_krylov_basis)
+        // 3rd row of arnoldi_norm: the infinity norm of next_krylov_basis
+        //                          (ONLY when using a scalar accessor)
         // The optional entry stores the infinity_norm of each
-        // next_krylov_vector, which is only used to compute the scale
+        // next_krylov_vector, which is only used to compute the scalar
         auto arnoldi_norm =
             VectorNorms::create(exec, dim<2>{3, dense_b->get_size()[1]});
         Array<size_type> final_iter_nums(this->get_executor(),
@@ -473,7 +467,7 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             // Calculate residual norm
 
             restart_iter++;
-        }
+        }  // closes while(true)
         // Solve x
 
         auto hessenberg_small = hessenberg->create_submatrix(
diff --git a/core/solver/cb_gmres_accessor.hpp b/core/solver/cb_gmres_accessor.hpp
index 9cdbb6ba7ce..e5727b1036e 100644
--- a/core/solver/cb_gmres_accessor.hpp
+++ b/core/solver/cb_gmres_accessor.hpp
@@ -36,7 +36,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <array>
 #include <cinttypes>
-#include <iostream>
 #include <limits>
 #include <memory>
 #include <type_traits>
@@ -52,21 +51,20 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/base/accessors.hpp"
-#define GKO_DEBUG_OUTPUT false
 
 
 namespace gko {
-namespace kernels {  // TODO maybe put into another separate namespace
+namespace cb_gmres {
 
 
 namespace detail {
 
 
 template <typename Accessor>
-struct is_3d_scaled_accessor : public std::false_type {};
+struct has_3d_scaled_accessor : public std::false_type {};
 
 template <typename T1, typename T2, size_type mask>
-struct is_3d_scaled_accessor<
+struct has_3d_scaled_accessor<
     range<accessor::scaled_reduced_row_major<3, T1, T2, mask>>>
     : public std::true_type {};
 
@@ -85,25 +83,23 @@ struct helper_require_scale<StorageType, true> : public std::true_type {};
 
 template <typename ValueType, typename StorageType,
           bool = detail::helper_require_scale<StorageType>::value>
-class Accessor3dHelper {};
+class Range3dHelper {};
 
 
 template <typename ValueType, typename StorageType>
-class Accessor3dHelper<ValueType, StorageType, true> {
+class Range3dHelper<ValueType, StorageType, true> {
 public:
     using Accessor =
         accessor::scaled_reduced_row_major<3, ValueType, StorageType, 0b101>;
     using Range = range<Accessor>;
 
-    Accessor3dHelper() = default;
+    Range3dHelper() = default;
 
-    Accessor3dHelper(std::shared_ptr<const Executor> exec, dim<3> krylov_dim)
+    Range3dHelper(std::shared_ptr<const Executor> exec, dim<3> krylov_dim)
         : krylov_dim_{krylov_dim},
           bases_{exec, krylov_dim_[0] * krylov_dim_[1] * krylov_dim_[2]},
           scale_{exec, krylov_dim_[0] * krylov_dim_[2]}
     {
-        // For testing, initialize scale to ones
-        // Array<ValueType> h_scale{exec->get_master(), krylov_dim[0]};
         Array<ValueType> h_scale{exec->get_master(),
                                  krylov_dim_[0] * krylov_dim_[2]};
         for (size_type i = 0; i < h_scale.get_num_elems(); ++i) {
@@ -127,14 +123,14 @@ class Accessor3dHelper<ValueType, StorageType, true> {
 
 
 template <typename ValueType, typename StorageType>
-class Accessor3dHelper<ValueType, StorageType, false> {
+class Range3dHelper<ValueType, StorageType, false> {
 public:
     using Accessor = accessor::reduced_row_major<3, ValueType, StorageType>;
     using Range = range<Accessor>;
 
-    Accessor3dHelper() = default;
+    Range3dHelper() = default;
 
-    Accessor3dHelper(std::shared_ptr<const Executor> exec, dim<3> krylov_dim)
+    Range3dHelper(std::shared_ptr<const Executor> exec, dim<3> krylov_dim)
         : krylov_dim_{krylov_dim},
           bases_{std::move(exec),
                  krylov_dim_[0] * krylov_dim_[1] * krylov_dim_[2]}
@@ -149,10 +145,9 @@ class Accessor3dHelper<ValueType, StorageType, false> {
     Array<StorageType> bases_;
 };
 
-//----------------------------------------------
 
 template <typename Accessor3d,
-          bool = detail::is_3d_scaled_accessor<Accessor3d>::value>
+          bool = detail::has_3d_scaled_accessor<Accessor3d>::value>
 struct helper_functions_accessor {};
 
 // Accessors having a scale
@@ -160,7 +155,7 @@ template <typename Accessor3d>
 struct helper_functions_accessor<Accessor3d, true> {
     using arithmetic_type = typename Accessor3d::accessor::arithmetic_type;
     static constexpr size_type dimensionality = Accessor3d::dimensionality;
-    static_assert(detail::is_3d_scaled_accessor<Accessor3d>::value,
+    static_assert(detail::has_3d_scaled_accessor<Accessor3d>::value,
                   "Accessor must have a scalar here!");
     template <typename IndexType>
     static inline GKO_ATTRIBUTES void write_scalar(Accessor3d krylov_bases,
@@ -191,7 +186,7 @@ template <typename Accessor3d>
 struct helper_functions_accessor<Accessor3d, false> {
     using arithmetic_type = typename Accessor3d::accessor::arithmetic_type;
     static constexpr size_type dimensionality = Accessor3d::dimensionality;
-    static_assert(!detail::is_3d_scaled_accessor<Accessor3d>::value,
+    static_assert(!detail::has_3d_scaled_accessor<Accessor3d>::value,
                   "Accessor must not have a scale here!");
 
     template <typename IndexType>
@@ -208,14 +203,8 @@ struct helper_functions_accessor<Accessor3d, false> {
     }
 };
 
-// calling it with:
-// helper_functions_accessor<Accessor3d>::write_scalar(krylov_bases, col_idx,
-// value);
 
-//----------------------------------------------
-
-
-}  // namespace kernels
+}  // namespace cb_gmres
 }  // namespace gko
 
 
diff --git a/cuda/solver/cb_gmres_kernels.cu b/cuda/solver/cb_gmres_kernels.cu
index ca55042052a..5885868ad78 100644
--- a/cuda/solver/cb_gmres_kernels.cu
+++ b/cuda/solver/cb_gmres_kernels.cu
@@ -72,9 +72,6 @@ constexpr int default_dot_dim = 32;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
-constexpr int default_update_dim = 32;
-
-
 #include "common/solver/cb_gmres_kernels.hpp.inc"
 
 
@@ -156,11 +153,13 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
     const auto num_rows = residual->get_size()[0];
     const auto num_rhs = residual->get_size()[1];
     const auto krylov_stride =
-        helper_functions_accessor<Accessor3d>::get_stride(krylov_bases);
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::get_stride(
+            krylov_bases);
     const dim3 grid_dim_1(
         ceildiv((krylov_dim + 1) * krylov_stride[0], default_block_size), 1, 1);
     const dim3 block_dim(default_block_size, 1, 1);
     constexpr auto block_size = default_block_size;
+    const auto stride_arnoldi = arnoldi_norm->get_stride();
 
     initialize_2_1_kernel<block_size><<<grid_dim_1, block_dim>>>(
         residual->get_size()[0], residual->get_size()[1], krylov_dim,
@@ -169,28 +168,28 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
         residual_norm_collection->get_stride());
     residual->compute_norm2(residual_norm);
 
-    components::fill_array(exec, arnoldi_norm->get_values() + 2 * num_rhs,
+    components::fill_array(exec,
+                           arnoldi_norm->get_values() + 2 * stride_arnoldi,
                            num_rhs, zero<remove_complex<ValueType>>());
     const dim3 grid_size_nrm(ceildiv(num_rhs, default_dot_dim),
                              exec->get_num_multiprocessor() * 2);
     const dim3 block_size_nrm(default_dot_dim, default_dot_dim);
     multinorminf_without_stop_kernel<<<grid_size_nrm, block_size_nrm>>>(
-        num_rows, num_rhs, as_cuda_type(residual->get_const_values()), num_rhs,
-        as_cuda_type(arnoldi_norm->get_values() + 2 * num_rhs), 0);
-
-    // helper_functions_accessor<ValueType, ValueTypeKrylovBases>::write_scalar(
-    //     krylov_bases, col_idx, arnoldi_norm->at(2, j) / residual_norm->at(0,
-    //     j));
-    /* */
-    // std::cout << "Before set_scale_kernel" << '\n';
-    const auto stride_arnoldi = arnoldi_norm->get_stride();
-    set_scale_kernel<default_block_size>
-        <<<ceildiv(num_rhs * (krylov_dim + 1), default_block_size),
-           default_block_size>>>(
-            num_rhs, krylov_dim + 1,
-            as_cuda_type(residual_norm->get_const_values()), num_rhs,
-            as_cuda_type(arnoldi_norm->get_const_values() + 2 * stride_arnoldi),
-            stride_arnoldi, as_cuda_accessor(krylov_bases));
+        num_rows, num_rhs, as_cuda_type(residual->get_const_values()),
+        residual->get_stride(),
+        as_cuda_type(arnoldi_norm->get_values() + 2 * stride_arnoldi), 0);
+
+    if (gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value) {
+        set_scalar_kernel<default_block_size>
+            <<<ceildiv(num_rhs * (krylov_dim + 1), default_block_size),
+               default_block_size>>>(
+                num_rhs, krylov_dim + 1,
+                as_cuda_type(residual_norm->get_const_values()),
+                residual_norm->get_stride(),
+                as_cuda_type(arnoldi_norm->get_const_values() +
+                             2 * stride_arnoldi),
+                stride_arnoldi, as_cuda_accessor(krylov_bases));
+    }
 
     const dim3 grid_dim_2(
         ceildiv(num_rows * krylov_stride[1], default_block_size), 1, 1);
@@ -225,7 +224,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     // optimization parameter
     constexpr int singledot_block_size = default_dot_dim;
     constexpr bool use_scale =
-        kernels::detail::is_3d_scaled_accessor<Accessor3dim>::value;
+        gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3dim>::value;
     const auto stride_next_krylov = next_krylov_basis->get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
     const auto stride_buffer = buffer_iter->get_stride();
@@ -254,7 +253,6 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_type(arnoldi_norm->get_values()), as_cuda_type(stop_status));
-    // nrmP = norm(next_krylov_basis
     zero_matrix(iter + 1, dim_size[1], stride_hessenberg,
                 hessenberg_iter->get_values());
     if (dim_size[1] > 1) {
@@ -288,15 +286,16 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     // for i in 1:iter
     //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
     // end
-    components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
+    components::fill_array(exec, arnoldi_norm->get_values() + stride_arnoldi,
                            dim_size[1], zero<non_complex>());
-    components::fill_array(exec, arnoldi_norm->get_values() + 2 * dim_size[1],
+    components::fill_array(exec,
+                           arnoldi_norm->get_values() + 2 * stride_arnoldi,
                            dim_size[1], zero<non_complex>());
     multinorm2_inf_kernel<use_scale><<<grid_size, block_size>>>(
         dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_cuda_type(arnoldi_norm->get_values() + dim_size[1]),
-        as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]),
+        as_cuda_type(arnoldi_norm->get_values() + stride_arnoldi),
+        as_cuda_type(arnoldi_norm->get_values() + 2 * stride_arnoldi),
         as_cuda_type(stop_status));
     // nrmN = norm(next_krylov_basis)
     components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
@@ -348,17 +347,18 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         // for i in 1:iter
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
-        components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
+        components::fill_array(exec,
+                               arnoldi_norm->get_values() + stride_arnoldi,
                                dim_size[1], zero<non_complex>());
         components::fill_array(exec,
-                               arnoldi_norm->get_values() + 2 * dim_size[1],
+                               arnoldi_norm->get_values() + 2 * stride_arnoldi,
                                dim_size[1], zero<non_complex>());
         multinorm2_inf_kernel<use_scale><<<grid_size, block_size>>>(
             dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
             stride_next_krylov,
-            as_cuda_type(arnoldi_norm->get_values() + dim_size[1]),
-            as_cuda_type(arnoldi_norm->get_values() + 2 * dim_size[1]),
+            as_cuda_type(arnoldi_norm->get_values() + stride_arnoldi),
+            as_cuda_type(arnoldi_norm->get_values() + 2 * stride_arnoldi),
             as_cuda_type(stop_status));
         // nrmN = norm(next_krylov_basis)
         components::fill_array(exec, num_reorth->get_data(), 1,
diff --git a/cuda/test/solver/cb_gmres_kernels.cpp b/cuda/test/solver/cb_gmres_kernels.cpp
index ed81c9dab6f..474d178fbf2 100644
--- a/cuda/test/solver/cb_gmres_kernels.cpp
+++ b/cuda/test/solver/cb_gmres_kernels.cpp
@@ -62,7 +62,7 @@ class CbGmres : public ::testing::Test {
     using index_type = int;
     using size_type = gko::size_type;
     using Range3dHelper =
-        gko::kernels::Accessor3dHelper<value_type, storage_type>;
+        gko::cb_gmres::Range3dHelper<value_type, storage_type>;
     using Range3d = typename Range3dHelper::Range;
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
@@ -109,7 +109,7 @@ class CbGmres : public ::testing::Test {
         auto dist = std::normal_distribution<value_type>(-1, 1);
         for (size_type k = 0; k < size[0]; ++k) {
             for (size_type i = 0; i < size[2]; ++i) {
-                gko::kernels::helper_functions_accessor<Range3d>::write_scalar(
+                gko::cb_gmres::helper_functions_accessor<Range3d>::write_scalar(
                     range, k, i, dist(rand_engine));
             }
         }
@@ -306,6 +306,7 @@ TEST_F(CbGmres, CudaCbGmresInitialize2IsEquivalentToRef)
         d_range_helper.get_range(), d_next_krylov_basis.get(),
         d_final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
 
+    GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
                         1e-14);
@@ -336,6 +337,7 @@ TEST_F(CbGmres, CudaCbGmresStep1IsEquivalentToRef)
         d_final_iter_nums.get(), d_stop_status.get(), d_reorth_status.get(),
         d_num_reorth.get(), &num_reorth_steps, &num_reorth_vectors);
 
+    GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
diff --git a/hip/solver/cb_gmres_kernels.hip.cpp b/hip/solver/cb_gmres_kernels.hip.cpp
index 02c8b519c85..12ee6717202 100644
--- a/hip/solver/cb_gmres_kernels.hip.cpp
+++ b/hip/solver/cb_gmres_kernels.hip.cpp
@@ -153,11 +153,13 @@ void initialize_2(std::shared_ptr<const HipExecutor> exec,
     const auto num_rows = residual->get_size()[0];
     const auto num_rhs = residual->get_size()[1];
     const auto krylov_stride =
-        helper_functions_accessor<Accessor3d>::get_stride(krylov_bases);
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::get_stride(
+            krylov_bases);
     const dim3 grid_dim_1(
         ceildiv((krylov_dim + 1) * krylov_stride[0], default_block_size), 1, 1);
     const dim3 block_dim(default_block_size, 1, 1);
     constexpr auto block_size = default_block_size;
+    const auto stride_arnoldi = arnoldi_norm->get_stride();
 
     hipLaunchKernelGGL(initialize_2_1_kernel<block_size>, grid_dim_1, block_dim,
                        0, 0, residual->get_size()[0], residual->get_size()[1],
@@ -166,24 +168,28 @@ void initialize_2(std::shared_ptr<const HipExecutor> exec,
                        residual_norm_collection->get_stride());
     residual->compute_norm2(residual_norm);
 
-    components::fill_array(exec, arnoldi_norm->get_values() + 2 * num_rhs,
+    components::fill_array(exec,
+                           arnoldi_norm->get_values() + 2 * stride_arnoldi,
                            num_rhs, zero<remove_complex<ValueType>>());
     const dim3 grid_size_nrm(ceildiv(num_rhs, default_dot_dim),
                              exec->get_num_multiprocessor() * 2);
     const dim3 block_size_nrm(default_dot_dim, default_dot_dim);
     hipLaunchKernelGGL(
         multinorminf_without_stop_kernel, grid_size_nrm, block_size_nrm, 0, 0,
-        num_rows, num_rhs, as_hip_type(residual->get_const_values()), num_rhs,
-        as_hip_type(arnoldi_norm->get_values() + 2 * num_rhs), 0);
+        num_rows, num_rhs, as_hip_type(residual->get_const_values()),
+        residual->get_stride(),
+        as_hip_type(arnoldi_norm->get_values() + 2 * stride_arnoldi), 0);
 
-    const auto stride_arnoldi = arnoldi_norm->get_stride();
-    hipLaunchKernelGGL(
-        set_scale_kernel<default_block_size>,
-        dim3(ceildiv(num_rhs * (krylov_dim + 1), default_block_size)),
-        dim3(default_block_size), 0, 0, num_rhs, krylov_dim + 1,
-        as_hip_type(residual_norm->get_const_values()), num_rhs,
-        as_hip_type(arnoldi_norm->get_const_values() + 2 * stride_arnoldi),
-        stride_arnoldi, as_hip_accessor(krylov_bases));
+    if (gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value) {
+        hipLaunchKernelGGL(
+            set_scalar_kernel<default_block_size>,
+            dim3(ceildiv(num_rhs * (krylov_dim + 1), default_block_size)),
+            dim3(default_block_size), 0, 0, num_rhs, krylov_dim + 1,
+            as_hip_type(residual_norm->get_const_values()),
+            residual_norm->get_stride(),
+            as_hip_type(arnoldi_norm->get_const_values() + 2 * stride_arnoldi),
+            stride_arnoldi, as_hip_accessor(krylov_bases));
+    }
 
     const dim3 grid_dim_2(
         ceildiv(num_rows * krylov_stride[1], default_block_size), 1, 1);
@@ -219,7 +225,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
     // optimization parameter
     constexpr int singledot_block_size = default_dot_dim;
     constexpr bool use_scale =
-        kernels::detail::is_3d_scaled_accessor<Accessor3dim>::value;
+        gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3dim>::value;
     const auto stride_next_krylov = next_krylov_basis->get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
     const auto stride_buffer = buffer_iter->get_stride();
@@ -284,16 +290,17 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
     // for i in 1:iter
     //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
     // end
-    components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
+    components::fill_array(exec, arnoldi_norm->get_values() + stride_arnoldi,
                            dim_size[1], zero<non_complex>());
-    components::fill_array(exec, arnoldi_norm->get_values() + 2 * dim_size[1],
+    components::fill_array(exec,
+                           arnoldi_norm->get_values() + 2 * stride_arnoldi,
                            dim_size[1], zero<non_complex>());
     hipLaunchKernelGGL(
         multinorm2_inf_kernel<use_scale>, grid_size, block_size, 0, 0,
         dim_size[0], dim_size[1],
         as_hip_type(next_krylov_basis->get_const_values()), stride_next_krylov,
-        as_hip_type(arnoldi_norm->get_values() + dim_size[1]),
-        as_hip_type(arnoldi_norm->get_values() + 2 * dim_size[1]),
+        as_hip_type(arnoldi_norm->get_values() + stride_arnoldi),
+        as_hip_type(arnoldi_norm->get_values() + 2 * stride_arnoldi),
         as_hip_type(stop_status));
     // nrmN = norm(next_krylov_basis)
     components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
@@ -347,18 +354,19 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
         // for i in 1:iter
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
-        components::fill_array(exec, arnoldi_norm->get_values() + dim_size[1],
+        components::fill_array(exec,
+                               arnoldi_norm->get_values() + stride_arnoldi,
                                dim_size[1], zero<non_complex>());
         components::fill_array(exec,
-                               arnoldi_norm->get_values() + 2 * dim_size[1],
+                               arnoldi_norm->get_values() + 2 * stride_arnoldi,
                                dim_size[1], zero<non_complex>());
         hipLaunchKernelGGL(
             multinorm2_inf_kernel<use_scale>, grid_size, block_size, 0, 0,
             dim_size[0], dim_size[1],
             as_hip_type(next_krylov_basis->get_const_values()),
             stride_next_krylov,
-            as_hip_type(arnoldi_norm->get_values() + dim_size[1]),
-            as_hip_type(arnoldi_norm->get_values() + 2 * dim_size[1]),
+            as_hip_type(arnoldi_norm->get_values() + stride_arnoldi),
+            as_hip_type(arnoldi_norm->get_values() + 2 * stride_arnoldi),
             as_hip_type(stop_status));
         // nrmN = norm(next_krylov_basis)
         components::fill_array(exec, num_reorth->get_data(), 1,
diff --git a/hip/test/solver/cb_gmres_kernels.cpp b/hip/test/solver/cb_gmres_kernels.cpp
index de58c3f140b..f228675391c 100644
--- a/hip/test/solver/cb_gmres_kernels.cpp
+++ b/hip/test/solver/cb_gmres_kernels.cpp
@@ -62,7 +62,7 @@ class CbGmres : public ::testing::Test {
     using index_type = int;
     using size_type = gko::size_type;
     using Range3dHelper =
-        gko::kernels::Accessor3dHelper<value_type, storage_type>;
+        gko::cb_gmres::Range3dHelper<value_type, storage_type>;
     using Range3d = typename Range3dHelper::Range;
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
@@ -109,7 +109,7 @@ class CbGmres : public ::testing::Test {
         auto dist = std::normal_distribution<value_type>(-1, 1);
         for (size_type k = 0; k < size[0]; ++k) {
             for (size_type i = 0; i < size[2]; ++i) {
-                gko::kernels::helper_functions_accessor<Range3d>::write_scalar(
+                gko::cb_gmres::helper_functions_accessor<Range3d>::write_scalar(
                     range, k, i, dist(rand_engine));
             }
         }
@@ -306,6 +306,7 @@ TEST_F(CbGmres, HipCbGmresInitialize2IsEquivalentToRef)
         d_range_helper.get_range(), d_next_krylov_basis.get(),
         d_final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
 
+    GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
                         1e-14);
@@ -336,6 +337,7 @@ TEST_F(CbGmres, HipCbGmresStep1IsEquivalentToRef)
         d_final_iter_nums.get(), d_stop_status.get(), d_reorth_status.get(),
         d_num_reorth.get(), &num_reorth_steps, &num_reorth_vectors);
 
+    GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
diff --git a/omp/solver/cb_gmres_kernels.cpp b/omp/solver/cb_gmres_kernels.cpp
index 4bc635444e5..54fd439d98b 100644
--- a/omp/solver/cb_gmres_kernels.cpp
+++ b/omp/solver/cb_gmres_kernels.cpp
@@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/cb_gmres_kernels.hpp"
 
 
-#include <iostream>
-
-
 #include <omp.h>
 
 
@@ -46,14 +43,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/cb_gmres.hpp>
 
 
-// #define TIMING 1
-
-
-#ifdef TIMING
-using double_seconds = std::chrono::duration<double, std::milli>;
-#endif
-
-
 namespace gko {
 namespace kernels {
 namespace omp {
@@ -93,9 +82,6 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         }
         arnoldi_norm->at(0, i) = nrm * eta;
         // nrmP = norm(next_krylov_basis)
-#ifdef TIMING
-        auto start_1 = std::chrono::steady_clock::now();
-#endif
 #pragma omp parallel for
         for (size_type k = 0; k < iter + 1; ++k) {
             ValueType hessenberg_iter_entry = zero<ValueType>();
@@ -105,18 +91,9 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
             }
             hessenberg_iter->at(k, i) = hessenberg_iter_entry;
         }
-#ifdef TIMING
-        auto time_1 = std::chrono::steady_clock::now() - start_1;
-        std::cout << "time_1(" << iter << ") = "
-                  << std::chrono::duration_cast<double_seconds>(time_1).count()
-                  << std::endl;
-#endif
         // for i in 1:iter
         //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
         // end
-#ifdef TIMING
-        auto start_2 = std::chrono::steady_clock::now();
-#endif
         for (size_type k = 0; k < iter + 1; ++k) {
 #pragma omp parallel for
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
@@ -124,17 +101,6 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                     hessenberg_iter->at(k, i) * krylov_bases(k, j, i);
             }
         }
-#ifdef TIMING
-        auto time_2 = std::chrono::steady_clock::now() - start_2;
-        std::cout << "time_2(" << iter << ") = "
-                  << std::chrono::duration_cast<double_seconds>(time_2).count()
-                  << std::endl;
-        std::cout
-            << "time_1 / time_2(" << iter << ") = "
-            << std::chrono::duration_cast<double_seconds>(time_1).count() /
-                   std::chrono::duration_cast<double_seconds>(time_2).count()
-            << std::endl;
-#endif
         // for i in 1:iter
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
@@ -150,8 +116,6 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         }
         arnoldi_norm->at(1, i) = sqrt(nrm);
         arnoldi_norm->at(2, i) = inf;
-        // nrmI = infnorm(next_krylov_basis)
-        // nrmN = norm(next_krylov_basis)
 
         for (size_type l = 1;
              (arnoldi_norm->at(1, i)) < (arnoldi_norm->at(0, i)) && l < 3;
@@ -196,7 +160,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
             // nrmN = norm(next_krylov_basis)
             // nrmI = infnorm(next_krylov_basis)
         }
-        helper_functions_accessor<Accessor3d>::write_scalar(
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
             krylov_bases, iter + 1, i,
             arnoldi_norm->at(2, i) / arnoldi_norm->at(1, i));
         // reorthogonalization
@@ -412,7 +376,7 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
         }
         residual_norm->at(0, j) = sqrt(res_norm);
         arnoldi_norm->at(2, j) = res_inf;
-        helper_functions_accessor<Accessor3d>::write_scalar(
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
             krylov_bases, {0}, j,
             arnoldi_norm->at(2, j) / residual_norm->at(0, j));
 
@@ -437,7 +401,7 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for
     for (size_type k = 1; k < krylov_dim + 1; ++k) {
         for (size_type j = 0; j < residual->get_size()[1]; ++j) {
-            helper_functions_accessor<Accessor3d>::write_scalar(
+            gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
                 krylov_bases, k, j, one<remove_complex<ValueType>>());
         }
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
diff --git a/omp/test/solver/cb_gmres_kernels.cpp b/omp/test/solver/cb_gmres_kernels.cpp
index 8b4662cf2ed..c9c8c5d45cf 100644
--- a/omp/test/solver/cb_gmres_kernels.cpp
+++ b/omp/test/solver/cb_gmres_kernels.cpp
@@ -64,7 +64,7 @@ class CbGmres : public ::testing::Test {
     using index_type = int;
     using size_type = gko::size_type;
     using Range3dHelper =
-        gko::kernels::Accessor3dHelper<value_type, storage_type>;
+        gko::cb_gmres::Range3dHelper<value_type, storage_type>;
     using Range3d = typename Range3dHelper::Range;
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
@@ -111,7 +111,7 @@ class CbGmres : public ::testing::Test {
         auto dist = std::normal_distribution<value_type>(-1, 1);
         for (size_type k = 0; k < size[0]; ++k) {
             for (size_type i = 0; i < size[2]; ++i) {
-                gko::kernels::helper_functions_accessor<Range3d>::write_scalar(
+                gko::cb_gmres::helper_functions_accessor<Range3d>::write_scalar(
                     range, k, i, dist(rand_engine));
             }
         }
@@ -306,6 +306,7 @@ TEST_F(CbGmres, OmpCbGmresInitialize2IsEquivalentToRef)
         d_range_helper.get_range(), d_next_krylov_basis.get(),
         d_final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
 
+    GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
                         1e-14);
@@ -335,6 +336,7 @@ TEST_F(CbGmres, OmpCbGmresStep1IsEquivalentToRef)
         d_final_iter_nums.get(), d_stop_status.get(), d_reorth_status.get(),
         d_num_reorth.get(), &num_reorth_steps, &num_reorth_vectors);
 
+    GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
diff --git a/reference/solver/cb_gmres_kernels.cpp b/reference/solver/cb_gmres_kernels.cpp
index 42211e346e6..5eafa9c84f0 100644
--- a/reference/solver/cb_gmres_kernels.cpp
+++ b/reference/solver/cb_gmres_kernels.cpp
@@ -153,7 +153,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
             arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
             // nrmN = norm(next_krylov_basis)
         }
-        helper_functions_accessor<Accessor3d>::write_scalar(
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
             krylov_bases, iter + 1, i,
             arnoldi_norm->at(2, i) / arnoldi_norm->at(1, i));
         hessenberg_iter->at(iter + 1, i) = arnoldi_norm->at(1, i);
@@ -359,7 +359,7 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
                     : abs(residual->at(i, j));
         }
         residual_norm->at(0, j) = sqrt(residual_norm->at(0, j));
-        helper_functions_accessor<Accessor3d>::write_scalar(
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
             krylov_bases, {0}, j,
             arnoldi_norm->at(2, j) / residual_norm->at(0, j));
 
@@ -381,7 +381,7 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
 
     for (size_type k = 1; k < krylov_dim + 1; ++k) {
         for (size_type j = 0; j < residual->get_size()[1]; ++j) {
-            helper_functions_accessor<Accessor3d>::write_scalar(
+            gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
                 krylov_bases, k, j, one<remove_complex<ValueType>>());
             for (size_type i = 0; i < residual->get_size()[0]; ++i) {
                 krylov_bases(k, i, j) = zero<ValueType>();

From 13be6b5d2e86a9e2e3d50fe7bfe43e956254e055 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 3 Feb 2021 01:22:32 +0100
Subject: [PATCH 0402/1008] Add Helper INSTANTIATE macro for CB-GMRES

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 core/solver/cb_gmres_kernels.hpp | 88 +++++++++-----------------------
 1 file changed, 23 insertions(+), 65 deletions(-)

diff --git a/core/solver/cb_gmres_kernels.hpp b/core/solver/cb_gmres_kernels.hpp
index 2a2122a4c90..04756ef196d 100644
--- a/core/solver/cb_gmres_kernels.hpp
+++ b/core/solver/cb_gmres_kernels.hpp
@@ -59,108 +59,66 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *                1. the first will be used as the regular ValueType
  *                     (precisions supported by Ginkgo), and
  *                2. the second the value type of the reduced precision.
+ * @param _const  qualifier used for the storage type, indicating if it is a
+ *                const accessor, or not.
  */
-#define GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(_macro)                         \
+#define GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE_HELPER(_macro, _const)          \
     template _macro(                                                           \
         double,                                                                \
-        GKO_UNPACK(range<accessor::reduced_row_major<3, double, double>>));    \
-    template _macro(                                                           \
-        double,                                                                \
-        GKO_UNPACK(range<accessor::reduced_row_major<3, double, float>>));     \
-    template _macro(                                                           \
-        double,                                                                \
-        GKO_UNPACK(range<accessor::reduced_row_major<3, double, half>>));      \
-    template _macro(                                                           \
-        double,                                                                \
-        GKO_UNPACK(range<accessor::scaled_reduced_row_major<3, double, int64,  \
-                                                            0b101>>));         \
-    template _macro(                                                           \
-        double,                                                                \
-        GKO_UNPACK(range<accessor::scaled_reduced_row_major<3, double, int32,  \
-                                                            0b101>>));         \
-    template _macro(                                                           \
-        double,                                                                \
-        GKO_UNPACK(range<accessor::scaled_reduced_row_major<3, double, int16,  \
-                                                            0b101>>));         \
-    template _macro(                                                           \
-        float,                                                                 \
-        GKO_UNPACK(range<accessor::reduced_row_major<3, float, float>>));      \
-    template _macro(                                                           \
-        float,                                                                 \
-        GKO_UNPACK(range<accessor::reduced_row_major<3, float, half>>));       \
-    template _macro(                                                           \
-        float,                                                                 \
         GKO_UNPACK(                                                            \
-            range<                                                             \
-                accessor::scaled_reduced_row_major<3, float, int32, 0b101>>)); \
-    template _macro(                                                           \
-        float,                                                                 \
-        GKO_UNPACK(                                                            \
-            range<                                                             \
-                accessor::scaled_reduced_row_major<3, float, int16, 0b101>>)); \
-    template _macro(                                                           \
-        std::complex<double>,                                                  \
-        GKO_UNPACK(range<accessor::reduced_row_major<3, std::complex<double>,  \
-                                                     std::complex<double>>>)); \
-    template _macro(                                                           \
-        std::complex<double>,                                                  \
-        GKO_UNPACK(range<accessor::reduced_row_major<3, std::complex<double>,  \
-                                                     std::complex<float>>>));  \
-    template _macro(                                                           \
-        std::complex<float>,                                                   \
-        GKO_UNPACK(range<accessor::reduced_row_major<3, std::complex<float>,   \
-                                                     std::complex<float>>>))
-
-#define GKO_INSTANTIATE_FOR_EACH_CB_GMRES_CONST_TYPE(_macro)                   \
-    template _macro(                                                           \
-        double,                                                                \
-        GKO_UNPACK(                                                            \
-            range<accessor::reduced_row_major<3, double, const double>>));     \
+            range<accessor::reduced_row_major<3, double, _const double>>));    \
     template _macro(                                                           \
         double,                                                                \
         GKO_UNPACK(                                                            \
-            range<accessor::reduced_row_major<3, double, const float>>));      \
+            range<accessor::reduced_row_major<3, double, _const float>>));     \
     template _macro(                                                           \
         double,                                                                \
         GKO_UNPACK(                                                            \
-            range<accessor::reduced_row_major<3, double, const half>>));       \
+            range<accessor::reduced_row_major<3, double, _const half>>));      \
     template _macro(double,                                                    \
                     GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
-                                   3, double, const int64, 0b101>>));          \
+                                   3, double, _const int64, 0b101>>));         \
     template _macro(double,                                                    \
                     GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
-                                   3, double, const int32, 0b101>>));          \
+                                   3, double, _const int32, 0b101>>));         \
     template _macro(double,                                                    \
                     GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
-                                   3, double, const int16, 0b101>>));          \
+                                   3, double, _const int16, 0b101>>));         \
     template _macro(                                                           \
         float,                                                                 \
         GKO_UNPACK(                                                            \
-            range<accessor::reduced_row_major<3, float, const float>>));       \
+            range<accessor::reduced_row_major<3, float, _const float>>));      \
     template _macro(                                                           \
         float,                                                                 \
-        GKO_UNPACK(range<accessor::reduced_row_major<3, float, const half>>)); \
+        GKO_UNPACK(                                                            \
+            range<accessor::reduced_row_major<3, float, _const half>>));       \
     template _macro(float,                                                     \
                     GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
-                                   3, float, const int32, 0b101>>));           \
+                                   3, float, _const int32, 0b101>>));          \
     template _macro(float,                                                     \
                     GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
-                                   3, float, const int16, 0b101>>));           \
+                                   3, float, _const int16, 0b101>>));          \
     template _macro(                                                           \
         std::complex<double>,                                                  \
         GKO_UNPACK(                                                            \
             range<accessor::reduced_row_major<3, std::complex<double>,         \
-                                              const std::complex<double>>>));  \
+                                              _const std::complex<double>>>)); \
     template _macro(                                                           \
         std::complex<double>,                                                  \
         GKO_UNPACK(                                                            \
             range<accessor::reduced_row_major<3, std::complex<double>,         \
-                                              const std::complex<float>>>));   \
+                                              _const std::complex<float>>>));  \
     template _macro(                                                           \
         std::complex<float>,                                                   \
         GKO_UNPACK(                                                            \
             range<accessor::reduced_row_major<3, std::complex<float>,          \
-                                              const std::complex<float>>>))
+                                              _const std::complex<float>>>))
+
+#define GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(_macro) \
+    GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE_HELPER(_macro, )
+
+#define GKO_INSTANTIATE_FOR_EACH_CB_GMRES_CONST_TYPE(_macro) \
+    GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE_HELPER(_macro, const)
 
 
 namespace gko {

From a7623722d9d4aedb164b9b55ba413d52a210eb30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 3 Feb 2021 08:24:30 +0100
Subject: [PATCH 0403/1008] Remove CB-GMRES and GMRES example

Both examples are removed because the functionality is so similar to
simple-solver, so it does not add a lot of value.
---
 core/test/solver/cb_gmres.cpp                 | 252 +++++++++++------
 cuda/solver/cb_gmres_kernels.cu               |   2 -
 examples/CMakeLists.txt                       |   2 -
 examples/cb-gmres-solver/CMakeLists.txt       |  13 -
 examples/cb-gmres-solver/build.sh             |  37 ---
 .../cb-gmres-solver-jacobi-matrix.cpp         | 266 ------------------
 .../cb-gmres-solver-jacobi.cpp                | 158 -----------
 examples/cb-gmres-solver/cb-gmres-solver.cpp  | 153 ----------
 examples/cb-gmres-solver/data/A.mtx           | 114 --------
 examples/cb-gmres-solver/data/b.mtx           |  21 --
 examples/cb-gmres-solver/data/x0.mtx          |  21 --
 examples/cb-gmres-solver/doc/builds-on        |   1 -
 examples/cb-gmres-solver/doc/intro.dox        |  19 --
 examples/cb-gmres-solver/doc/kind             |   1 -
 examples/cb-gmres-solver/doc/results.dox      |  33 ---
 examples/cb-gmres-solver/doc/short-intro      |   1 -
 examples/cb-gmres-solver/doc/tooltip          |   1 -
 examples/gmres-solver/CMakeLists.txt          |   6 -
 examples/gmres-solver/build.sh                |  37 ---
 examples/gmres-solver/data/A.mtx              | 114 --------
 examples/gmres-solver/data/b.mtx              |  21 --
 examples/gmres-solver/data/x0.mtx             |  21 --
 examples/gmres-solver/doc/builds-on           |   1 -
 examples/gmres-solver/doc/intro.dox           |  19 --
 examples/gmres-solver/doc/kind                |   1 -
 examples/gmres-solver/doc/results.dox         |  33 ---
 examples/gmres-solver/doc/short-intro         |   1 -
 examples/gmres-solver/doc/tooltip             |   1 -
 examples/gmres-solver/gmres-solver.cpp        | 152 ----------
 include/ginkgo/core/solver/cb_gmres.hpp       |  10 +
 30 files changed, 182 insertions(+), 1330 deletions(-)
 delete mode 100644 examples/cb-gmres-solver/CMakeLists.txt
 delete mode 100755 examples/cb-gmres-solver/build.sh
 delete mode 100644 examples/cb-gmres-solver/cb-gmres-solver-jacobi-matrix.cpp
 delete mode 100644 examples/cb-gmres-solver/cb-gmres-solver-jacobi.cpp
 delete mode 100644 examples/cb-gmres-solver/cb-gmres-solver.cpp
 delete mode 100644 examples/cb-gmres-solver/data/A.mtx
 delete mode 100644 examples/cb-gmres-solver/data/b.mtx
 delete mode 100644 examples/cb-gmres-solver/data/x0.mtx
 delete mode 100644 examples/cb-gmres-solver/doc/builds-on
 delete mode 100644 examples/cb-gmres-solver/doc/intro.dox
 delete mode 100644 examples/cb-gmres-solver/doc/kind
 delete mode 100644 examples/cb-gmres-solver/doc/results.dox
 delete mode 100644 examples/cb-gmres-solver/doc/short-intro
 delete mode 100644 examples/cb-gmres-solver/doc/tooltip
 delete mode 100644 examples/gmres-solver/CMakeLists.txt
 delete mode 100755 examples/gmres-solver/build.sh
 delete mode 100644 examples/gmres-solver/data/A.mtx
 delete mode 100644 examples/gmres-solver/data/b.mtx
 delete mode 100644 examples/gmres-solver/data/x0.mtx
 delete mode 100644 examples/gmres-solver/doc/builds-on
 delete mode 100644 examples/gmres-solver/doc/intro.dox
 delete mode 100644 examples/gmres-solver/doc/kind
 delete mode 100644 examples/gmres-solver/doc/results.dox
 delete mode 100644 examples/gmres-solver/doc/short-intro
 delete mode 100644 examples/gmres-solver/doc/tooltip
 delete mode 100644 examples/gmres-solver/gmres-solver.cpp

diff --git a/core/test/solver/cb_gmres.cpp b/core/test/solver/cb_gmres.cpp
index 462db27712f..6c273c1189f 100644
--- a/core/test/solver/cb_gmres.cpp
+++ b/core/test/solver/cb_gmres.cpp
@@ -47,47 +47,58 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/solver/cb_gmres.cpp"
+#include "core/test/utils.hpp"
+
 
 namespace {
 
+
+template <typename ValueEnumType>
 class CbGmres : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Dense<>;
-    using Solver = gko::solver::CbGmres<>;
-    //    using Solver = gko::solver::CbGmres<double,double>;
-    using Big_solver = gko::solver::CbGmres<double>;
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueEnumType())>::type;
+    using nc_value_type = gko::remove_complex<value_type>;
+    using storage_helper_type =
+        typename std::tuple_element<1, decltype(ValueEnumType())>::type;
+    using Mtx = gko::matrix::Dense<value_type>;
+    using Solver = gko::solver::CbGmres<value_type>;
 
     CbGmres()
         : exec(gko::ReferenceExecutor::create()),
           mtx(gko::initialize<Mtx>(
               {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)),
+          storage_precision{storage_helper_type::value},
           cb_gmres_factory(
               Solver::build()
+                  .with_storage_precision(storage_precision)
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(3u).on(exec),
-                      gko::stop::ResidualNormReduction<>::build()
-                          .with_reduction_factor(1e-6)
+                      gko::stop::ResidualNormReduction<value_type>::build()
+                          .with_reduction_factor(nc_value_type{1e-6})
                           .on(exec))
                   .on(exec)),
           solver(cb_gmres_factory->generate(mtx)),
           cb_gmres_big_factory(
-              Big_solver::build()
+              Solver::build()
+                  .with_storage_precision(storage_precision)
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(128u).on(
                           exec),
-                      gko::stop::ResidualNormReduction<>::build()
-                          .with_reduction_factor(1e-6)
+                      gko::stop::ResidualNormReduction<value_type>::build()
+                          .with_reduction_factor(nc_value_type{1e-6})
                           .on(exec))
                   .on(exec)),
           big_solver(cb_gmres_big_factory->generate(mtx))
     {}
 
+    gko::solver::cb_gmres_storage_precision storage_precision;
     std::shared_ptr<const gko::Executor> exec;
     std::shared_ptr<Mtx> mtx;
-    std::unique_ptr<Solver::Factory> cb_gmres_factory;
-    std::unique_ptr<gko::LinOp> solver;
-    std::unique_ptr<Big_solver::Factory> cb_gmres_big_factory;
-    std::unique_ptr<gko::LinOp> big_solver;
+    std::unique_ptr<typename Solver::Factory> cb_gmres_factory;
+    std::unique_ptr<Solver> solver;
+    std::unique_ptr<typename Solver::Factory> cb_gmres_big_factory;
+    std::unique_ptr<Solver> big_solver;
 
     static void assert_same_matrices(const Mtx *m1, const Mtx *m2)
     {
@@ -101,160 +112,240 @@ class CbGmres : public ::testing::Test {
     }
 };
 
-TEST_F(CbGmres, CbGmresFactoryKnowsItsExecutor)
+
+/**
+ * This creates a helper structure which translates a type into an enum
+ * parameter.
+ */
+using st_enum = gko::solver::cb_gmres_storage_precision;
+
+template <st_enum P>
+struct st_helper_type {
+    static constexpr st_enum value{P};
+};
+
+using st_keep = st_helper_type<st_enum::keep>;
+using st_r1 = st_helper_type<st_enum::reduce1>;
+using st_r2 = st_helper_type<st_enum::reduce2>;
+using st_i = st_helper_type<st_enum::integer>;
+using st_ir1 = st_helper_type<st_enum::ireduce1>;
+using st_ir2 = st_helper_type<st_enum::ireduce2>;
+
+using TestTypes =
+    ::testing::Types<std::tuple<double, st_keep>, std::tuple<double, st_r1>,
+                     std::tuple<double, st_r2>, std::tuple<double, st_i>,
+                     std::tuple<double, st_ir1>, std::tuple<double, st_ir2>,
+                     std::tuple<float, st_keep>, std::tuple<float, st_r1>,
+                     std::tuple<float, st_r2>, std::tuple<float, st_i>,
+                     std::tuple<float, st_ir1>, std::tuple<float, st_ir2>,
+                     std::tuple<std::complex<double>, st_keep>,
+                     std::tuple<std::complex<double>, st_r1>,
+                     std::tuple<std::complex<double>, st_r2>,
+                     std::tuple<std::complex<float>, st_keep>>;
+
+TYPED_TEST_CASE(CbGmres, TestTypes);
+
+
+TYPED_TEST(CbGmres, CbGmresFactoryKnowsItsExecutor)
 {
-    ASSERT_EQ(cb_gmres_factory->get_executor(), exec);
+    ASSERT_EQ(this->cb_gmres_factory->get_executor(), this->exec);
 }
 
-TEST_F(CbGmres, CbGmresFactoryCreatesCorrectSolver)
+
+TYPED_TEST(CbGmres, CbGmresFactoryCreatesCorrectSolver)
 {
-    ASSERT_EQ(solver->get_size(), gko::dim<2>(3, 3));
-    auto cb_gmres_solver = static_cast<Solver *>(solver.get());
+    using Solver = typename TestFixture::Solver;
+    ASSERT_EQ(this->solver->get_size(), gko::dim<2>(3, 3));
+    auto cb_gmres_solver = static_cast<Solver *>(this->solver.get());
     ASSERT_NE(cb_gmres_solver->get_system_matrix(), nullptr);
-    ASSERT_EQ(cb_gmres_solver->get_system_matrix(), mtx);
+    ASSERT_EQ(cb_gmres_solver->get_system_matrix(), this->mtx);
+    ASSERT_EQ(cb_gmres_solver->get_krylov_dim(), 100u);
+    ASSERT_EQ(cb_gmres_solver->get_storage_precision(),
+              this->storage_precision);
 }
 
-TEST_F(CbGmres, CanBeCopied)
+
+TYPED_TEST(CbGmres, CanBeCopied)
 {
-    auto copy = cb_gmres_factory->generate(Mtx::create(exec));
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    auto copy = this->cb_gmres_factory->generate(Mtx::create(this->exec));
+    auto r_copy = static_cast<Solver *>(copy.get());
 
-    copy->copy_from(solver.get());
+    copy->copy_from(this->solver.get());
 
     ASSERT_EQ(copy->get_size(), gko::dim<2>(3, 3));
-    auto copy_mtx = static_cast<Solver *>(copy.get())->get_system_matrix();
-    assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()), mtx.get());
+    auto copy_mtx = r_copy->get_system_matrix();
+    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
+                               this->mtx.get());
+    ASSERT_EQ(r_copy->get_storage_precision(),
+              this->solver->get_storage_precision());
+    ASSERT_EQ(r_copy->get_krylov_dim(), this->solver->get_krylov_dim());
 }
 
-TEST_F(CbGmres, CanBeMoved)
+
+TYPED_TEST(CbGmres, CanBeMoved)
 {
-    auto copy = cb_gmres_factory->generate(Mtx::create(exec));
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    auto copy = this->cb_gmres_factory->generate(Mtx::create(this->exec));
+    auto r_copy = static_cast<Solver *>(copy.get());
 
-    copy->copy_from(std::move(solver));
+    copy->copy_from(std::move(this->solver));
 
     ASSERT_EQ(copy->get_size(), gko::dim<2>(3, 3));
-    auto copy_mtx = static_cast<Solver *>(copy.get())->get_system_matrix();
-    assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()), mtx.get());
+    auto copy_mtx = r_copy->get_system_matrix();
+    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
+                               this->mtx.get());
+    ASSERT_EQ(r_copy->get_storage_precision(), this->storage_precision);
+    ASSERT_EQ(r_copy->get_krylov_dim(), 100u);
 }
 
-TEST_F(CbGmres, CanBeCloned)
+
+TYPED_TEST(CbGmres, CanBeCloned)
 {
-    auto clone = solver->clone();
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    auto clone = this->solver->clone();
+    auto r_clone = static_cast<Solver *>(clone.get());
 
     ASSERT_EQ(clone->get_size(), gko::dim<2>(3, 3));
-    auto clone_mtx = static_cast<Solver *>(clone.get())->get_system_matrix();
-    assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()), mtx.get());
+    auto clone_mtx = r_clone->get_system_matrix();
+    this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
+                               this->mtx.get());
+    ASSERT_EQ(r_clone->get_storage_precision(),
+              this->solver->get_storage_precision());
+    ASSERT_EQ(r_clone->get_krylov_dim(), this->solver->get_krylov_dim());
 }
 
-TEST_F(CbGmres, CanBeCleared)
+
+TYPED_TEST(CbGmres, CanBeCleared)
 {
-    solver->clear();
+    using Solver = typename TestFixture::Solver;
+    this->solver->clear();
 
-    ASSERT_EQ(solver->get_size(), gko::dim<2>(0, 0));
-    auto solver_mtx = static_cast<Solver *>(solver.get())->get_system_matrix();
+    ASSERT_EQ(this->solver->get_size(), gko::dim<2>(0, 0));
+    auto solver_mtx =
+        static_cast<Solver *>(this->solver.get())->get_system_matrix();
     ASSERT_EQ(solver_mtx, nullptr);
 }
 
-TEST_F(CbGmres, CanSetPreconditionerGenerator)
+
+TYPED_TEST(CbGmres, CanSetPreconditionerGenerator)
 {
+    using value_type = typename TestFixture::value_type;
+    using nc_value_type = typename TestFixture::nc_value_type;
+    using Solver = typename TestFixture::Solver;
     auto cb_gmres_factory =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(3u).on(exec),
-                gko::stop::ResidualNormReduction<>::build()
-                    .with_reduction_factor(1e-6)
-                    .on(exec))
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec),
+                gko::stop::ResidualNormReduction<value_type>::build()
+                    .with_reduction_factor(nc_value_type{1e-6})
+                    .on(this->exec))
             .with_preconditioner(
                 Solver::build()
                     .with_criteria(
                         gko::stop::Iteration::build().with_max_iters(3u).on(
-                            exec))
-                    .on(exec))
-            .on(exec);
-    auto solver = cb_gmres_factory->generate(mtx);
-    auto precond = dynamic_cast<const gko::solver::CbGmres<> *>(
-        static_cast<gko::solver::CbGmres<> *>(solver.get())
-            ->get_preconditioner()
-            .get());
+                            this->exec))
+                    .on(this->exec))
+            .on(this->exec);
+    auto solver = cb_gmres_factory->generate(this->mtx);
+    auto precond =
+        static_cast<const Solver *>(solver.get()->get_preconditioner().get());
 
     ASSERT_NE(precond, nullptr);
     ASSERT_EQ(precond->get_size(), gko::dim<2>(3, 3));
-    ASSERT_EQ(precond->get_system_matrix(), mtx);
+    ASSERT_EQ(precond->get_system_matrix(), this->mtx);
 }
 
-TEST_F(CbGmres, CanSetKrylovDim)
+
+TYPED_TEST(CbGmres, CanSetKrylovDim)
 {
+    using value_type = typename TestFixture::value_type;
+    using nc_value_type = typename TestFixture::nc_value_type;
+    using Solver = typename TestFixture::Solver;
     auto cb_gmres_factory =
         Solver::build()
             .with_krylov_dim(4u)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(4u).on(exec),
-                gko::stop::ResidualNormReduction<>::build()
-                    .with_reduction_factor(1e-6)
-                    .on(exec))
-            .on(exec);
-    auto solver = cb_gmres_factory->generate(mtx);
+                gko::stop::Iteration::build().with_max_iters(4u).on(this->exec),
+                gko::stop::ResidualNormReduction<value_type>::build()
+                    .with_reduction_factor(nc_value_type{1e-6})
+                    .on(this->exec))
+            .on(this->exec);
+    auto solver = cb_gmres_factory->generate(this->mtx);
     auto krylov_dim = solver->get_krylov_dim();
 
-    ASSERT_EQ(krylov_dim, 4);
+    ASSERT_EQ(solver->get_storage_precision(),
+              gko::solver::cb_gmres_storage_precision::keep);
 }
 
-TEST_F(CbGmres, CanSetPreconditionerInFactory)
+
+TYPED_TEST(CbGmres, CanSetPreconditionerInFactory)
 {
+    using Solver = typename TestFixture::Solver;
     std::shared_ptr<Solver> cb_gmres_precond =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(3u).on(exec))
-            .on(exec)
-            ->generate(mtx);
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .on(this->exec)
+            ->generate(this->mtx);
 
     auto cb_gmres_factory =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(3u).on(exec))
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
             .with_generated_preconditioner(cb_gmres_precond)
-            .on(exec);
-    auto solver = cb_gmres_factory->generate(mtx);
+            .on(this->exec);
+    auto solver = cb_gmres_factory->generate(this->mtx);
     auto precond = solver->get_preconditioner();
 
     ASSERT_NE(precond.get(), nullptr);
     ASSERT_EQ(precond.get(), cb_gmres_precond.get());
 }
 
-TEST_F(CbGmres, ThrowsOnWrongPreconditionerInFactory)
+
+TYPED_TEST(CbGmres, ThrowsOnWrongPreconditionerInFactory)
 {
-    std::shared_ptr<Mtx> wrong_sized_mtx = Mtx::create(exec, gko::dim<2>{1, 3});
+    using Mtx = typename TestFixture::Mtx;
+    using Solver = typename TestFixture::Solver;
+    std::shared_ptr<Mtx> wrong_sized_mtx =
+        Mtx::create(this->exec, gko::dim<2>{1, 3});
     std::shared_ptr<Solver> cb_gmres_precond =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(3u).on(exec))
-            .on(exec)
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .on(this->exec)
             ->generate(wrong_sized_mtx);
 
     auto cb_gmres_factory =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(3u).on(exec))
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
             .with_generated_preconditioner(cb_gmres_precond)
-            .on(exec);
+            .on(this->exec);
 
-    ASSERT_THROW(cb_gmres_factory->generate(mtx), gko::DimensionMismatch);
+    ASSERT_THROW(cb_gmres_factory->generate(this->mtx), gko::DimensionMismatch);
 }
 
-TEST_F(CbGmres, CanSetPreconditioner)
+
+TYPED_TEST(CbGmres, CanSetPreconditioner)
 {
+    using Solver = typename TestFixture::Solver;
     std::shared_ptr<Solver> cb_gmres_precond =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(3u).on(exec))
-            .on(exec)
-            ->generate(mtx);
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .on(this->exec)
+            ->generate(this->mtx);
 
     auto cb_gmres_factory =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(3u).on(exec))
-            .on(exec);
-    auto solver = cb_gmres_factory->generate(mtx);
+                gko::stop::Iteration::build().with_max_iters(3u).on(this->exec))
+            .on(this->exec);
+    auto solver = cb_gmres_factory->generate(this->mtx);
     solver->set_preconditioner(cb_gmres_precond);
     auto precond = solver->get_preconditioner();
 
@@ -262,4 +353,5 @@ TEST_F(CbGmres, CanSetPreconditioner)
     ASSERT_EQ(precond.get(), cb_gmres_precond.get());
 }
 
+
 }  // namespace
diff --git a/cuda/solver/cb_gmres_kernels.cu b/cuda/solver/cb_gmres_kernels.cu
index 5885868ad78..7a11a018abd 100644
--- a/cuda/solver/cb_gmres_kernels.cu
+++ b/cuda/solver/cb_gmres_kernels.cu
@@ -66,8 +66,6 @@ namespace cb_gmres {
 
 
 constexpr int default_block_size = 512;
-// default_dot_dim can not be 64 in hip because 64 * 64 exceeds their max block
-// size limit.
 constexpr int default_dot_dim = 32;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index c4948d43b56..b35d6422098 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -21,8 +21,6 @@ set(EXAMPLES_LIST
     custom-matrix-format
     custom-stopping-criterion
     ginkgo-overhead
-    cb-gmres-solver
-    gmres-solver
     heat-equation
     minimal-cuda-solver
     papi-logging
diff --git a/examples/cb-gmres-solver/CMakeLists.txt b/examples/cb-gmres-solver/CMakeLists.txt
deleted file mode 100644
index 960df2ab077..00000000000
--- a/examples/cb-gmres-solver/CMakeLists.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-add_executable(cb-gmres-solver cb-gmres-solver.cpp)
-target_link_libraries(cb-gmres-solver ginkgo)
-target_include_directories(cb-gmres-solver PRIVATE ${PROJECT_SOURCE_DIR})
-add_executable(cb-gmres-solver-jacobi cb-gmres-solver-jacobi.cpp)
-target_link_libraries(cb-gmres-solver-jacobi ginkgo)
-target_include_directories(cb-gmres-solver-jacobi PRIVATE ${PROJECT_SOURCE_DIR})
-add_executable(cb-gmres-solver-jacobi-matrix
-cb-gmres-solver-jacobi-matrix.cpp)
-target_link_libraries(cb-gmres-solver-jacobi-matrix ginkgo)
-target_include_directories(cb-gmres-solver-jacobi-matrix PRIVATE ${PROJECT_SOURCE_DIR})
-configure_file(data/A.mtx data/A.mtx COPYONLY)
-configure_file(data/b.mtx data/b.mtx COPYONLY)
-configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/cb-gmres-solver/build.sh b/examples/cb-gmres-solver/build.sh
deleted file mode 100755
index f2c94bc239c..00000000000
--- a/examples/cb-gmres-solver/build.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-# set up script
-if [ $# -ne 1 ]; then
-    echo -e "Usage: $0 GINKGO_BUILD_DIRECTORY"
-    exit 1
-fi
-BUILD_DIR=$1
-THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" &>/dev/null && pwd )
-
-# copy libraries
-LIBRARY_DIRS="core core/device_hooks reference omp cuda hip"
-LIBRARY_NAMES="ginkgo ginkgo_reference ginkgo_omp ginkgo_cuda ginkgo_hip"
-SUFFIXES=".so .dylib .dll d.so d.dylib d.dll"
-for prefix in ${LIBRARY_DIRS}; do
-    for name in ${LIBRARY_NAMES}; do
-        for suffix in ${SUFFIXES}; do
-            cp ${BUILD_DIR}/${prefix}/lib${name}${suffix} \
-                ${THIS_DIR}/lib${name}${suffix} 2>/dev/null
-        done
-    done
-done
-
-# figure out correct compiler flags
-if ls ${THIS_DIR} | grep -F "libginkgo." >/dev/null; then
-    LINK_FLAGS="-lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference -lginkgo_hip"
-else
-    LINK_FLAGS="-lginkgod -lginkgo_ompd -lginkgo_cudad -lginkgo_referenced -lginkgo_hipd"
-fi
-if [ -z "${CXX}" ]; then
-    CXX="c++"
-fi
-
-# build
-${CXX} -std=c++11 -o ${THIS_DIR}/simple-solver ${THIS_DIR}/simple-solver.cpp \
-       -I${THIS_DIR}/../../include -I${BUILD_DIR}/include \
-       -L${THIS_DIR} ${LINK_FLAGS}
diff --git a/examples/cb-gmres-solver/cb-gmres-solver-jacobi-matrix.cpp b/examples/cb-gmres-solver/cb-gmres-solver-jacobi-matrix.cpp
deleted file mode 100644
index d6ffaf887d1..00000000000
--- a/examples/cb-gmres-solver/cb-gmres-solver-jacobi-matrix.cpp
+++ /dev/null
@@ -1,266 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-// @sect3{Include files}
-
-// This is the main ginkgo header file.
-#include <ginkgo/ginkgo.hpp>
-//#include "cuda/components/zero_array.hpp"
-
-// Add the fstream header to read from data from files.
-#include <fstream>
-// Add the C++ iostream header to output information to the console.
-#include <iostream>
-// Add the string manipulation header to handle strings.
-#include <string>
-
-
-int main(int argc, char *argv[])
-{
-    // Use some shortcuts. In Ginkgo, vectors are seen as a gko::matrix::Dense
-    // with one column/one row. The advantage of this concept is that using
-    // multiple vectors is a now a natural extension of adding columns/rows are
-    // necessary.
-    using vec = gko::matrix::Dense<>;
-    // The gko::matrix::Csr class is used here, but any other matrix class such
-    // as gko::matrix::Coo, gko::matrix::Hybrid, gko::matrix::Ell or
-    // gko::matrix::Sellp could also be used.
-    using mtx = gko::matrix::Csr<>;
-    // The gko::solver::Cg is used here, but any other solver class can also be
-    // used.
-    using cb_gmres = gko::solver::CbGmres<double>;
-    // using cb_gmres = gko::solver::CbGmres<>;
-    using bj = gko::preconditioner::Jacobi<>;
-
-    // Print the ginkgo version information.
-    std::cout << gko::version_info::get() << std::endl;
-
-    // @sect3{Where do you want to run your solver ?}
-    // The gko::Executor class is one of the cornerstones of Ginkgo. Currently,
-    // we have support for
-    // an gko::OmpExecutor, which uses OpenMP multi-threading in most of its
-    // kernels, a gko::ReferenceExecutor, a single threaded specialization of
-    // the OpenMP executor and a gko::CudaExecutor which runs the code on a
-    // NVIDIA GPU if available.
-    // @note With the help of C++, you see that you only ever need to change the
-    // executor and all the other functions/ routines within Ginkgo should
-    // automatically work and run on the executor with any other changes.
-    /*
-        std::shared_ptr<gko::Executor> exec;
-        if (argc == 1 || std::string(argv[1]) == "reference") {
-            exec = gko::ReferenceExecutor::create();
-        } else if (argc == 2 && std::string(argv[1]) == "omp") {
-            exec = gko::OmpExecutor::create();
-        } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-                   gko::CudaExecutor::get_num_devices() > 0) {
-            exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
-        } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-                   gko::HipExecutor::get_num_devices() > 0) {
-            exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
-        } else {
-            std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
-            std::exit(-1);
-        }
-    */
-    int index_matrix = -1;
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-        if (argc == 3) index_matrix = 2;
-    } else if (argc <= 3 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-        if (argc == 3) index_matrix = 2;
-    } else if (argc <= 3 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
-        if (argc == 3) index_matrix = 2;
-    } else if (argc <= 3 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
-        if (argc == 3) index_matrix = 2;
-    } else if (argc == 2) {
-        exec = gko::ReferenceExecutor::create();
-        index_matrix = 1;
-    } else {
-        std::cerr << "Usage: " << argv[0] << " [executor] [matrix]"
-                  << std::endl;
-        std::exit(-1);
-    }
-
-    // @sect3{Reading your data and transfer to the proper device.}
-    // Read the matrix, right hand side and the initial solution using the @ref
-    // read function.
-    // @note Ginkgo uses C++ smart pointers to automatically manage memory. To
-    // this end, we use our own object ownership transfer functions that under
-    // the hood call the required smart pointer functions to manage object
-    // ownership. The gko::share , gko::give and gko::lend are the functions
-    // that you would need to use.
-    auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
-    auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
-    auto x = gko::read<vec>(std::ifstream("data/x0.mtx"), exec);
-    if (index_matrix > 0) {
-        A = share(gko::read<mtx>(std::ifstream(argv[index_matrix]), exec));
-        auto sizesA = A->get_size();
-
-        if (std::string(argv[1]) == "cuda") {
-            /* */
-            auto host_x =
-                vec::create(exec->get_master(), gko::dim<2>(sizesA[0], 1));
-            for (auto i = 0; i < sizesA[0]; i++) {
-                host_x->at(i, 0) = 1.0;
-            }
-            x->copy_from(host_x.get());
-            /* */
-            // x = gko::read<vec>(std::ifstream("data/one.mtx"), exec);
-        } else {
-            x = vec::create(exec, gko::dim<2>{sizesA[0], 1});
-            for (int i = 0; i < sizesA[0]; i++) x->at(i) = 1.0;
-        }
-        auto sizesx = x->get_size();
-
-        if (std::string(argv[1]) == "cuda") {
-            /* */
-            auto host_b =
-                vec::create(exec->get_master(), gko::dim<2>(sizesA[0], 1));
-            for (auto i = 0; i < sizesA[0]; i++) {
-                host_b->at(i, 0) = 0.0;
-            }
-            b->copy_from(host_b.get());
-            /* */
-            // b = gko::read<vec>(std::ifstream("data/zero.mtx"), exec);
-        } else {
-            b = vec::create(exec, gko::dim<2>{sizesA[0], 1});
-            for (int i = 0; i < sizesA[0]; i++) b->at(i) = 0.0;
-        }
-        auto sizesb = b->get_size();
-
-        A->apply(lend(x), lend(b));
-
-        if (std::string(argv[1]) == "cuda") {
-            /* */
-            auto host_x =
-                vec::create(exec->get_master(), gko::dim<2>(sizesA[0], 1));
-            for (auto i = 0; i < sizesA[0]; i++) {
-                host_x->at(i, 0) = 0.0;
-            }
-            x->copy_from(host_x.get());
-            /* */
-            // x = gko::read<vec>(std::ifstream("data/zero.mtx"), exec);
-        } else {
-            x = vec::create(exec, gko::dim<2>{sizesA[0], 1});
-            for (int i = 0; i < sizesA[0]; i++) x->at(i) = 0.0;
-        }
-        /**/
-        if (std::string(argv[1]) == "cuda") {
-            /* */
-            auto host_b =
-                vec::create(exec->get_master(), gko::dim<2>(sizesA[0], 1));
-            for (auto i = 0; i < sizesA[0]; i++) {
-                host_b->at(i, 0) = 1.0;
-            }
-            b->copy_from(host_b.get());
-            /* */
-            // b = gko::read<vec>(std::ifstream("data/one.mtx"), exec);
-        } else {
-            b = vec::create(exec, gko::dim<2>{sizesA[0], 1});
-            for (int i = 0; i < sizesA[0]; i++) b->at(i) = 1.0;
-        }
-        /**/
-    }
-    // std::cout << "Initilization is completed" << std::endl;
-    // @sect3{Creating the solver}
-    // Generate the gko::solver factory. Ginkgo uses the concept of Factories to
-    // build solvers with certain
-    // properties. Observe the Fluent interface used here. Here a cb_gmres
-    // solver is generated with a stopping criteria of maximum iterations of 20
-    // and a residual norm reduction of 1e-15. You also observe that the
-    // stopping criteria(gko::stop) are also generated from factories using
-    // their build methods. You need to specify the executors which each of the
-    // object needs to be built on.
-    auto solver_gen =
-        cb_gmres::build()
-            .with_criteria(
-                //    gko::stop::Iteration::build().with_max_iters(20u).on(exec),
-                gko::stop::Iteration::build()
-                    //    .with_max_iters(A->get_size()[0])
-                    .with_max_iters(20000u)
-                    .on(exec),
-                //    gko::stop::Iteration::build().with_max_iters(100000u).on(exec),
-                gko::stop::ResidualNormReduction<>::build()
-                    //    .with_reduction_factor(1e-15)
-                    //    .with_reduction_factor(1e-12)
-                    //    .with_reduction_factor(1e-13)
-                    .with_reduction_factor(1e-9)
-                    .on(exec))
-            // Add preconditioner, these 2 lines are the only
-            // difference from the simple solver example
-            //    .with_preconditioner(bj::build().with_max_block_size(8u).on(exec))
-            .with_storage_precision(
-                gko::solver::cb_gmres_storage_precision::keep)
-            .with_preconditioner(bj::build().with_max_block_size(1u).on(exec))
-            .on(exec);
-    // Generate the solver from the matrix. The solver factory built in the
-    // previous step takes a "matrix"(a gko::LinOp to be more general) as an
-    // input. In this case we provide it with a full matrix that we previously
-    // read, but as the solver only effectively uses the apply() method within
-    // the provided "matrix" object, you can effectively create a gko::LinOp
-    // class with your own apply implementation to accomplish more tasks. We
-    // will see an example of how this can be done in the custom-matrix-format
-    // example
-    auto solver = solver_gen->generate(A);
-
-    // Finally, solve the system. The solver, being a gko::LinOp, can be applied
-    // to a right hand side, b to
-    // obtain the solution, x.
-    // std::cout << "Computing " << std::endl;
-    solver->apply(lend(b), lend(x));
-
-    // Print the solution to the command line.
-    //    std::cout << "Solution (x): \n";
-    //    write(std::cout, lend(x));
-
-    // To measure if your solution has actually converged, you can measure the
-    // error of the solution.
-    // one, neg_one are objects that represent the numbers which allow for a
-    // uniform interface when computing on any device. To compute the residual,
-    // all you need to do is call the apply method, which in this case is an
-    // spmv and equivalent to the LAPACK z_spmv routine. Finally, you compute
-    // the euclidean 2-norm with the compute_norm2 function.
-    auto one = gko::initialize<vec>({1.0}, exec);
-    auto neg_one = gko::initialize<vec>({-1.0}, exec);
-    auto res = gko::initialize<vec>({0.0}, exec);
-    A->apply(lend(one), lend(x), lend(neg_one), lend(b));
-    b->compute_norm2(lend(res));
-
-    std::cout << "Residual norm sqrt(r^T r): \n";
-    write(std::cout, lend(res));
-}
diff --git a/examples/cb-gmres-solver/cb-gmres-solver-jacobi.cpp b/examples/cb-gmres-solver/cb-gmres-solver-jacobi.cpp
deleted file mode 100644
index 585920029cc..00000000000
--- a/examples/cb-gmres-solver/cb-gmres-solver-jacobi.cpp
+++ /dev/null
@@ -1,158 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-// @sect3{Include files}
-
-// This is the main ginkgo header file.
-#include <ginkgo/ginkgo.hpp>
-
-// Add the fstream header to read from data from files.
-#include <fstream>
-// Add the C++ iostream header to output information to the console.
-#include <iostream>
-// Add the string manipulation header to handle strings.
-#include <string>
-
-
-int main(int argc, char *argv[])
-{
-    // Use some shortcuts. In Ginkgo, vectors are seen as a gko::matrix::Dense
-    // with one column/one row. The advantage of this concept is that using
-    // multiple vectors is a now a natural extension of adding columns/rows are
-    // necessary.
-    using vec = gko::matrix::Dense<>;
-    // The gko::matrix::Csr class is used here, but any other matrix class such
-    // as gko::matrix::Coo, gko::matrix::Hybrid, gko::matrix::Ell or
-    // gko::matrix::Sellp could also be used.
-    using mtx = gko::matrix::Csr<>;
-    // The gko::solver::Cg is used here, but any other solver class can also be
-    // used.
-    //    using cb_gmres = gko::solver::CbGmres<double, float>;
-    using cb_gmres = gko::solver::CbGmres<>;
-    using bj = gko::preconditioner::Jacobi<>;
-
-    // Print the ginkgo version information.
-    std::cout << gko::version_info::get() << std::endl;
-
-    // @sect3{Where do you want to run your solver ?}
-    // The gko::Executor class is one of the cornerstones of Ginkgo. Currently,
-    // we have support for
-    // an gko::OmpExecutor, which uses OpenMP multi-threading in most of its
-    // kernels, a gko::ReferenceExecutor, a single threaded specialization of
-    // the OpenMP executor and a gko::CudaExecutor which runs the code on a
-    // NVIDIA GPU if available.
-    // @note With the help of C++, you see that you only ever need to change the
-    // executor and all the other functions/ routines within Ginkgo should
-    // automatically work and run on the executor with any other changes.
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
-    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
-    } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
-        std::exit(-1);
-    }
-
-    // @sect3{Reading your data and transfer to the proper device.}
-    // Read the matrix, right hand side and the initial solution using the @ref
-    // read function.
-    // @note Ginkgo uses C++ smart pointers to automatically manage memory. To
-    // this end, we use our own object ownership transfer functions that under
-    // the hood call the required smart pointer functions to manage object
-    // ownership. The gko::share , gko::give and gko::lend are the functions
-    // that you would need to use.
-    auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
-    auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
-    auto x = gko::read<vec>(std::ifstream("data/x0.mtx"), exec);
-
-    // @sect3{Creating the solver}
-    // Generate the gko::solver factory. Ginkgo uses the concept of Factories to
-    // build solvers with certain
-    // properties. Observe the Fluent interface used here. Here a cb_gmres
-    // solver is generated with a stopping criteria of maximum iterations of 20
-    // and a residual norm reduction of 1e-15. You also observe that the
-    // stopping criteria(gko::stop) are also generated from factories using
-    // their build methods. You need to specify the executors which each of the
-    // object needs to be built on.
-    auto solver_gen =
-        cb_gmres::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(20u).on(exec),
-                gko::stop::ResidualNormReduction<>::build()
-                    .with_reduction_factor(1e-15)
-                    .on(exec))
-            // Add preconditioner, these 2 lines are the only
-            // difference from the simple solver example
-            .with_preconditioner(bj::build().with_max_block_size(8u).on(exec))
-            .on(exec);
-    // Generate the solver from the matrix. The solver factory built in the
-    // previous step takes a "matrix"(a gko::LinOp to be more general) as an
-    // input. In this case we provide it with a full matrix that we previously
-    // read, but as the solver only effectively uses the apply() method within
-    // the provided "matrix" object, you can effectively create a gko::LinOp
-    // class with your own apply implementation to accomplish more tasks. We
-    // will see an example of how this can be done in the custom-matrix-format
-    // example
-    auto solver = solver_gen->generate(A);
-
-    // Finally, solve the system. The solver, being a gko::LinOp, can be applied
-    // to a right hand side, b to
-    // obtain the solution, x.
-    solver->apply(lend(b), lend(x));
-
-    // Print the solution to the command line.
-    std::cout << "Solution (x): \n";
-    write(std::cout, lend(x));
-
-    // To measure if your solution has actually converged, you can measure the
-    // error of the solution.
-    // one, neg_one are objects that represent the numbers which allow for a
-    // uniform interface when computing on any device. To compute the residual,
-    // all you need to do is call the apply method, which in this case is an
-    // spmv and equivalent to the LAPACK z_spmv routine. Finally, you compute
-    // the euclidean 2-norm with the compute_norm2 function.
-    auto one = gko::initialize<vec>({1.0}, exec);
-    auto neg_one = gko::initialize<vec>({-1.0}, exec);
-    auto res = gko::initialize<vec>({0.0}, exec);
-    A->apply(lend(one), lend(x), lend(neg_one), lend(b));
-    b->compute_norm2(lend(res));
-
-    std::cout << "Residual norm sqrt(r^T r): \n";
-    write(std::cout, lend(res));
-}
diff --git a/examples/cb-gmres-solver/cb-gmres-solver.cpp b/examples/cb-gmres-solver/cb-gmres-solver.cpp
deleted file mode 100644
index bcd2e9dfed2..00000000000
--- a/examples/cb-gmres-solver/cb-gmres-solver.cpp
+++ /dev/null
@@ -1,153 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-// @sect3{Include files}
-
-// This is the main ginkgo header file.
-#include <ginkgo/ginkgo.hpp>
-
-// Add the fstream header to read from data from files.
-#include <fstream>
-// Add the C++ iostream header to output information to the console.
-#include <iostream>
-// Add the string manipulation header to handle strings.
-#include <string>
-
-int main(int argc, char *argv[])
-{
-    // Use some shortcuts. In Ginkgo, vectors are seen as a gko::matrix::Dense
-    // with one column/one row. The advantage of this concept is that using
-    // multiple vectors is a now a natural extension of adding columns/rows are
-    // necessary.
-    using vec = gko::matrix::Dense<>;
-    // The gko::matrix::Csr class is used here, but any other matrix class such
-    // as gko::matrix::Coo, gko::matrix::Hybrid, gko::matrix::Ell or
-    // gko::matrix::Sellp could also be used.
-    using mtx = gko::matrix::Csr<>;
-    // The gko::solver::Cg is used here, but any other solver class can also be
-    // used.
-    //    using cb_gmres = gko::solver::CbGmres<double, float>;
-    using cb_gmres = gko::solver::CbGmres<>;
-
-    // Print the ginkgo version information.
-    std::cout << gko::version_info::get() << std::endl;
-
-    // @sect3{Where do you want to run your solver ?}
-    // The gko::Executor class is one of the cornerstones of Ginkgo. Currently,
-    // we have support for
-    // an gko::OmpExecutor, which uses OpenMP multi-threading in most of its
-    // kernels, a gko::ReferenceExecutor, a single threaded specialization of
-    // the OpenMP executor and a gko::CudaExecutor which runs the code on a
-    // NVIDIA GPU if available.
-    // @note With the help of C++, you see that you only ever need to change the
-    // executor and all the other functions/ routines within Ginkgo should
-    // automatically work and run on the executor with any other changes.
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
-    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
-    } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
-        std::exit(-1);
-    }
-
-    // @sect3{Reading your data and transfer to the proper device.}
-    // Read the matrix, right hand side and the initial solution using the @ref
-    // read function.
-    // @note Ginkgo uses C++ smart pointers to automatically manage memory. To
-    // this end, we use our own object ownership transfer functions that under
-    // the hood call the required smart pointer functions to manage object
-    // ownership. The gko::share , gko::give and gko::lend are the functions
-    // that you would need to use.
-    auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
-    auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
-    auto x = gko::read<vec>(std::ifstream("data/x0.mtx"), exec);
-
-    // @sect3{Creating the solver}
-    // Generate the gko::solver factory. Ginkgo uses the concept of Factories to
-    // build solvers with certain
-    // properties. Observe the Fluent interface used here. Here a cb_gmres
-    // solver is generated with a stopping criteria of maximum iterations of 20
-    // and a residual norm reduction of 1e-15. You also observe that the
-    // stopping criteria(gko::stop) are also generated from factories using
-    // their build methods. You need to specify the executors which each of the
-    // object needs to be built on.
-    auto solver_gen =
-        cb_gmres::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(20u).on(exec),
-                gko::stop::ResidualNormReduction<>::build()
-                    .with_reduction_factor(1e-15)
-                    .on(exec))
-            .on(exec);
-    // Generate the solver from the matrix. The solver factory built in the
-    // previous step takes a "matrix"(a gko::LinOp to be more general) as an
-    // input. In this case we provide it with a full matrix that we previously
-    // read, but as the solver only effectively uses the apply() method within
-    // the provided "matrix" object, you can effectively create a gko::LinOp
-    // class with your own apply implementation to accomplish more tasks. We
-    // will see an example of how this can be done in the custom-matrix-format
-    // example
-    auto solver = solver_gen->generate(A);
-
-    // Finally, solve the system. The solver, being a gko::LinOp, can be applied
-    // to a right hand side, b to
-    // obtain the solution, x.
-    solver->apply(lend(b), lend(x));
-
-    // Print the solution to the command line.
-    std::cout << "Solution (x): \n";
-    write(std::cout, lend(x));
-
-    // To measure if your solution has actually converged, you can measure the
-    // error of the solution.
-    // one, neg_one are objects that represent the numbers which allow for a
-    // uniform interface when computing on any device. To compute the residual,
-    // all you need to do is call the apply method, which in this case is an
-    // spmv and equivalent to the LAPACK z_spmv routine. Finally, you compute
-    // the euclidean 2-norm with the compute_norm2 function.
-    auto one = gko::initialize<vec>({1.0}, exec);
-    auto neg_one = gko::initialize<vec>({-1.0}, exec);
-    auto res = gko::initialize<vec>({0.0}, exec);
-    A->apply(lend(one), lend(x), lend(neg_one), lend(b));
-    b->compute_norm2(lend(res));
-
-    std::cout << "Residual norm sqrt(r^T r): \n";
-    write(std::cout, lend(res));
-}
diff --git a/examples/cb-gmres-solver/data/A.mtx b/examples/cb-gmres-solver/data/A.mtx
deleted file mode 100644
index c67437da567..00000000000
--- a/examples/cb-gmres-solver/data/A.mtx
+++ /dev/null
@@ -1,114 +0,0 @@
-%%MatrixMarket matrix coordinate integer symmetric
-%-------------------------------------------------------------------------------
-% UF Sparse Matrix Collection, Tim Davis
-% http://www.cise.ufl.edu/research/sparse/matrices/JGD_Trefethen/Trefethen_20b
-% name: JGD_Trefethen/Trefethen_20b
-% [Diagonal matrices with primes, Nick Trefethen, Oxford Univ.]
-% id: 2203
-% date: 2008
-% author: N. Trefethen
-% ed: J.-G. Dumas
-% fields: name title A id date author ed kind notes
-% kind: combinatorial problem
-%-------------------------------------------------------------------------------
-% notes:
-% Diagonal matrices with primes, Nick Trefethen, Oxford Univ.          
-% From Jean-Guillaume Dumas' Sparse Integer Matrix Collection,         
-% http://ljk.imag.fr/membres/Jean-Guillaume.Dumas/simc.html            
-%                                                                      
-% Problem 7 of the Hundred-dollar, Hundred-digit Challenge Problems,   
-% SIAM News, vol 35, no. 1.                                            
-%                                                                      
-% 7. Let A be the 20,000 x 20,000 matrix whose entries are zero        
-% everywhere except for the primes 2, 3, 5, 7, . . . , 224737 along the
-% main diagonal and the number 1 in all the positions A(i,j) with      
-% |i-j| = 1,2,4,8, . . . ,16384.  What is the (1,1) entry of inv(A)?   
-%                                                                      
-% http://www.siam.org/news/news.php?id=388                             
-%                                                                      
-% Filename in JGD collection: Trefethen/trefethen_20__19_minor.sms     
-%-------------------------------------------------------------------------------
-19 19 83
-1 1 3
-2 1 1
-3 1 1
-5 1 1
-9 1 1
-17 1 1
-2 2 5
-3 2 1
-4 2 1
-6 2 1
-10 2 1
-18 2 1
-3 3 7
-4 3 1
-5 3 1
-7 3 1
-11 3 1
-19 3 1
-4 4 11
-5 4 1
-6 4 1
-8 4 1
-12 4 1
-5 5 13
-6 5 1
-7 5 1
-9 5 1
-13 5 1
-6 6 17
-7 6 1
-8 6 1
-10 6 1
-14 6 1
-7 7 19
-8 7 1
-9 7 1
-11 7 1
-15 7 1
-8 8 23
-9 8 1
-10 8 1
-12 8 1
-16 8 1
-9 9 29
-10 9 1
-11 9 1
-13 9 1
-17 9 1
-10 10 31
-11 10 1
-12 10 1
-14 10 1
-18 10 1
-11 11 37
-12 11 1
-13 11 1
-15 11 1
-19 11 1
-12 12 41
-13 12 1
-14 12 1
-16 12 1
-13 13 43
-14 13 1
-15 13 1
-17 13 1
-14 14 47
-15 14 1
-16 14 1
-18 14 1
-15 15 53
-16 15 1
-17 15 1
-19 15 1
-16 16 59
-17 16 1
-18 16 1
-17 17 61
-18 17 1
-19 17 1
-18 18 67
-19 18 1
-19 19 71
diff --git a/examples/cb-gmres-solver/data/b.mtx b/examples/cb-gmres-solver/data/b.mtx
deleted file mode 100644
index 05d92ecc6f7..00000000000
--- a/examples/cb-gmres-solver/data/b.mtx
+++ /dev/null
@@ -1,21 +0,0 @@
-%%MatrixMarket matrix array real general
-19 1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
diff --git a/examples/cb-gmres-solver/data/x0.mtx b/examples/cb-gmres-solver/data/x0.mtx
deleted file mode 100644
index 91d470cdbcd..00000000000
--- a/examples/cb-gmres-solver/data/x0.mtx
+++ /dev/null
@@ -1,21 +0,0 @@
-%%MatrixMarket matrix array real general
-19 1
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
diff --git a/examples/cb-gmres-solver/doc/builds-on b/examples/cb-gmres-solver/doc/builds-on
deleted file mode 100644
index 8b137891791..00000000000
--- a/examples/cb-gmres-solver/doc/builds-on
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/examples/cb-gmres-solver/doc/intro.dox b/examples/cb-gmres-solver/doc/intro.dox
deleted file mode 100644
index 2869e91a12c..00000000000
--- a/examples/cb-gmres-solver/doc/intro.dox
+++ /dev/null
@@ -1,19 +0,0 @@
-<a name="Intro"></a>
-<h1>Introduction</h1>
-This simple solver example should help you get started with Ginkgo. This example is meant for you to understand
-how Ginkgo works and how you can solve a simple linear system with Ginkgo. We encourage you to play with the code,
-change the parameters and see what is best suited for your purposes.
-
-<h3> About the example </h3>
-Each example has the following sections:
-<ol>
-  <li> <b>Introduction:</b>This gives an overview of the example and mentions
-  any interesting aspects in the example that might help the reader.
-  <li> <b>The commented program:</b> This section is intended for you to
-  understand the details of the example so that you can play with it and understand
-  Ginkgo and its features better.
-  <li> <b>Results:</b> This section shows the results of the code when run. Though the
-  results may not be completely the same, you can expect the behaviour to be similar. 
-  <li> <b>The plain program:</b> This is the complete code without any comments to have
-  an complete overview of the code.
-  </ol>
diff --git a/examples/cb-gmres-solver/doc/kind b/examples/cb-gmres-solver/doc/kind
deleted file mode 100644
index 15a13db4511..00000000000
--- a/examples/cb-gmres-solver/doc/kind
+++ /dev/null
@@ -1 +0,0 @@
-basic
diff --git a/examples/cb-gmres-solver/doc/results.dox b/examples/cb-gmres-solver/doc/results.dox
deleted file mode 100644
index 0615abac5de..00000000000
--- a/examples/cb-gmres-solver/doc/results.dox
+++ /dev/null
@@ -1,33 +0,0 @@
-<h1>Results</h1>
-The following is the expected result:
-
-@code{.cpp}
-Solution (x):
-%%MatrixMarket matrix array real general
-19 1
-0.252218
-0.108645
-0.0662811
-0.0630433
-0.0384088
-0.0396536
-0.0402648
-0.0338935
-0.0193098
-0.0234653
-0.0211499
-0.0196413
-0.0199151
-0.0181674
-0.0162722
-0.0150714
-0.0107016
-0.0121141
-0.0123025
-Residual norm sqrt(r^T r):
-%%MatrixMarket matrix array real general
-1 1
-2.10788e-15
-@endcode
-
-<h3> Comments about programming and debugging </h3>
diff --git a/examples/cb-gmres-solver/doc/short-intro b/examples/cb-gmres-solver/doc/short-intro
deleted file mode 100644
index 578df81366b..00000000000
--- a/examples/cb-gmres-solver/doc/short-intro
+++ /dev/null
@@ -1 +0,0 @@
-The simple solver example.
diff --git a/examples/cb-gmres-solver/doc/tooltip b/examples/cb-gmres-solver/doc/tooltip
deleted file mode 100644
index 52c4f6d5f44..00000000000
--- a/examples/cb-gmres-solver/doc/tooltip
+++ /dev/null
@@ -1 +0,0 @@
-Solve a simple linear system with CG. Read a matrix and right hand side from a file.
diff --git a/examples/gmres-solver/CMakeLists.txt b/examples/gmres-solver/CMakeLists.txt
deleted file mode 100644
index d00153f906c..00000000000
--- a/examples/gmres-solver/CMakeLists.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-add_executable(gmres-solver gmres-solver.cpp)
-target_link_libraries(gmres-solver ginkgo)
-target_include_directories(gmres-solver PRIVATE ${PROJECT_SOURCE_DIR})
-configure_file(data/A.mtx data/A.mtx COPYONLY)
-configure_file(data/b.mtx data/b.mtx COPYONLY)
-configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/gmres-solver/build.sh b/examples/gmres-solver/build.sh
deleted file mode 100755
index f2c94bc239c..00000000000
--- a/examples/gmres-solver/build.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-# set up script
-if [ $# -ne 1 ]; then
-    echo -e "Usage: $0 GINKGO_BUILD_DIRECTORY"
-    exit 1
-fi
-BUILD_DIR=$1
-THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" &>/dev/null && pwd )
-
-# copy libraries
-LIBRARY_DIRS="core core/device_hooks reference omp cuda hip"
-LIBRARY_NAMES="ginkgo ginkgo_reference ginkgo_omp ginkgo_cuda ginkgo_hip"
-SUFFIXES=".so .dylib .dll d.so d.dylib d.dll"
-for prefix in ${LIBRARY_DIRS}; do
-    for name in ${LIBRARY_NAMES}; do
-        for suffix in ${SUFFIXES}; do
-            cp ${BUILD_DIR}/${prefix}/lib${name}${suffix} \
-                ${THIS_DIR}/lib${name}${suffix} 2>/dev/null
-        done
-    done
-done
-
-# figure out correct compiler flags
-if ls ${THIS_DIR} | grep -F "libginkgo." >/dev/null; then
-    LINK_FLAGS="-lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference -lginkgo_hip"
-else
-    LINK_FLAGS="-lginkgod -lginkgo_ompd -lginkgo_cudad -lginkgo_referenced -lginkgo_hipd"
-fi
-if [ -z "${CXX}" ]; then
-    CXX="c++"
-fi
-
-# build
-${CXX} -std=c++11 -o ${THIS_DIR}/simple-solver ${THIS_DIR}/simple-solver.cpp \
-       -I${THIS_DIR}/../../include -I${BUILD_DIR}/include \
-       -L${THIS_DIR} ${LINK_FLAGS}
diff --git a/examples/gmres-solver/data/A.mtx b/examples/gmres-solver/data/A.mtx
deleted file mode 100644
index c67437da567..00000000000
--- a/examples/gmres-solver/data/A.mtx
+++ /dev/null
@@ -1,114 +0,0 @@
-%%MatrixMarket matrix coordinate integer symmetric
-%-------------------------------------------------------------------------------
-% UF Sparse Matrix Collection, Tim Davis
-% http://www.cise.ufl.edu/research/sparse/matrices/JGD_Trefethen/Trefethen_20b
-% name: JGD_Trefethen/Trefethen_20b
-% [Diagonal matrices with primes, Nick Trefethen, Oxford Univ.]
-% id: 2203
-% date: 2008
-% author: N. Trefethen
-% ed: J.-G. Dumas
-% fields: name title A id date author ed kind notes
-% kind: combinatorial problem
-%-------------------------------------------------------------------------------
-% notes:
-% Diagonal matrices with primes, Nick Trefethen, Oxford Univ.          
-% From Jean-Guillaume Dumas' Sparse Integer Matrix Collection,         
-% http://ljk.imag.fr/membres/Jean-Guillaume.Dumas/simc.html            
-%                                                                      
-% Problem 7 of the Hundred-dollar, Hundred-digit Challenge Problems,   
-% SIAM News, vol 35, no. 1.                                            
-%                                                                      
-% 7. Let A be the 20,000 x 20,000 matrix whose entries are zero        
-% everywhere except for the primes 2, 3, 5, 7, . . . , 224737 along the
-% main diagonal and the number 1 in all the positions A(i,j) with      
-% |i-j| = 1,2,4,8, . . . ,16384.  What is the (1,1) entry of inv(A)?   
-%                                                                      
-% http://www.siam.org/news/news.php?id=388                             
-%                                                                      
-% Filename in JGD collection: Trefethen/trefethen_20__19_minor.sms     
-%-------------------------------------------------------------------------------
-19 19 83
-1 1 3
-2 1 1
-3 1 1
-5 1 1
-9 1 1
-17 1 1
-2 2 5
-3 2 1
-4 2 1
-6 2 1
-10 2 1
-18 2 1
-3 3 7
-4 3 1
-5 3 1
-7 3 1
-11 3 1
-19 3 1
-4 4 11
-5 4 1
-6 4 1
-8 4 1
-12 4 1
-5 5 13
-6 5 1
-7 5 1
-9 5 1
-13 5 1
-6 6 17
-7 6 1
-8 6 1
-10 6 1
-14 6 1
-7 7 19
-8 7 1
-9 7 1
-11 7 1
-15 7 1
-8 8 23
-9 8 1
-10 8 1
-12 8 1
-16 8 1
-9 9 29
-10 9 1
-11 9 1
-13 9 1
-17 9 1
-10 10 31
-11 10 1
-12 10 1
-14 10 1
-18 10 1
-11 11 37
-12 11 1
-13 11 1
-15 11 1
-19 11 1
-12 12 41
-13 12 1
-14 12 1
-16 12 1
-13 13 43
-14 13 1
-15 13 1
-17 13 1
-14 14 47
-15 14 1
-16 14 1
-18 14 1
-15 15 53
-16 15 1
-17 15 1
-19 15 1
-16 16 59
-17 16 1
-18 16 1
-17 17 61
-18 17 1
-19 17 1
-18 18 67
-19 18 1
-19 19 71
diff --git a/examples/gmres-solver/data/b.mtx b/examples/gmres-solver/data/b.mtx
deleted file mode 100644
index 05d92ecc6f7..00000000000
--- a/examples/gmres-solver/data/b.mtx
+++ /dev/null
@@ -1,21 +0,0 @@
-%%MatrixMarket matrix array real general
-19 1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
diff --git a/examples/gmres-solver/data/x0.mtx b/examples/gmres-solver/data/x0.mtx
deleted file mode 100644
index 91d470cdbcd..00000000000
--- a/examples/gmres-solver/data/x0.mtx
+++ /dev/null
@@ -1,21 +0,0 @@
-%%MatrixMarket matrix array real general
-19 1
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
-0
diff --git a/examples/gmres-solver/doc/builds-on b/examples/gmres-solver/doc/builds-on
deleted file mode 100644
index 8b137891791..00000000000
--- a/examples/gmres-solver/doc/builds-on
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/examples/gmres-solver/doc/intro.dox b/examples/gmres-solver/doc/intro.dox
deleted file mode 100644
index 2869e91a12c..00000000000
--- a/examples/gmres-solver/doc/intro.dox
+++ /dev/null
@@ -1,19 +0,0 @@
-<a name="Intro"></a>
-<h1>Introduction</h1>
-This simple solver example should help you get started with Ginkgo. This example is meant for you to understand
-how Ginkgo works and how you can solve a simple linear system with Ginkgo. We encourage you to play with the code,
-change the parameters and see what is best suited for your purposes.
-
-<h3> About the example </h3>
-Each example has the following sections:
-<ol>
-  <li> <b>Introduction:</b>This gives an overview of the example and mentions
-  any interesting aspects in the example that might help the reader.
-  <li> <b>The commented program:</b> This section is intended for you to
-  understand the details of the example so that you can play with it and understand
-  Ginkgo and its features better.
-  <li> <b>Results:</b> This section shows the results of the code when run. Though the
-  results may not be completely the same, you can expect the behaviour to be similar. 
-  <li> <b>The plain program:</b> This is the complete code without any comments to have
-  an complete overview of the code.
-  </ol>
diff --git a/examples/gmres-solver/doc/kind b/examples/gmres-solver/doc/kind
deleted file mode 100644
index 15a13db4511..00000000000
--- a/examples/gmres-solver/doc/kind
+++ /dev/null
@@ -1 +0,0 @@
-basic
diff --git a/examples/gmres-solver/doc/results.dox b/examples/gmres-solver/doc/results.dox
deleted file mode 100644
index 0615abac5de..00000000000
--- a/examples/gmres-solver/doc/results.dox
+++ /dev/null
@@ -1,33 +0,0 @@
-<h1>Results</h1>
-The following is the expected result:
-
-@code{.cpp}
-Solution (x):
-%%MatrixMarket matrix array real general
-19 1
-0.252218
-0.108645
-0.0662811
-0.0630433
-0.0384088
-0.0396536
-0.0402648
-0.0338935
-0.0193098
-0.0234653
-0.0211499
-0.0196413
-0.0199151
-0.0181674
-0.0162722
-0.0150714
-0.0107016
-0.0121141
-0.0123025
-Residual norm sqrt(r^T r):
-%%MatrixMarket matrix array real general
-1 1
-2.10788e-15
-@endcode
-
-<h3> Comments about programming and debugging </h3>
diff --git a/examples/gmres-solver/doc/short-intro b/examples/gmres-solver/doc/short-intro
deleted file mode 100644
index 578df81366b..00000000000
--- a/examples/gmres-solver/doc/short-intro
+++ /dev/null
@@ -1 +0,0 @@
-The simple solver example.
diff --git a/examples/gmres-solver/doc/tooltip b/examples/gmres-solver/doc/tooltip
deleted file mode 100644
index 52c4f6d5f44..00000000000
--- a/examples/gmres-solver/doc/tooltip
+++ /dev/null
@@ -1 +0,0 @@
-Solve a simple linear system with CG. Read a matrix and right hand side from a file.
diff --git a/examples/gmres-solver/gmres-solver.cpp b/examples/gmres-solver/gmres-solver.cpp
deleted file mode 100644
index e782aca1a6f..00000000000
--- a/examples/gmres-solver/gmres-solver.cpp
+++ /dev/null
@@ -1,152 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-// @sect3{Include files}
-
-// This is the main ginkgo header file.
-#include <ginkgo/ginkgo.hpp>
-
-// Add the fstream header to read from data from files.
-#include <fstream>
-// Add the C++ iostream header to output information to the console.
-#include <iostream>
-// Add the string manipulation header to handle strings.
-#include <string>
-
-int main(int argc, char *argv[])
-{
-    // Use some shortcuts. In Ginkgo, vectors are seen as a gko::matrix::Dense
-    // with one column/one row. The advantage of this concept is that using
-    // multiple vectors is a now a natural extension of adding columns/rows are
-    // necessary.
-    using vec = gko::matrix::Dense<>;
-    // The gko::matrix::Csr class is used here, but any other matrix class such
-    // as gko::matrix::Coo, gko::matrix::Hybrid, gko::matrix::Ell or
-    // gko::matrix::Sellp could also be used.
-    using mtx = gko::matrix::Csr<>;
-    // The gko::solver::Cg is used here, but any other solver class can also be
-    // used.
-    using gmres = gko::solver::Gmres<>;
-
-    // Print the ginkgo version information.
-    std::cout << gko::version_info::get() << std::endl;
-
-    // @sect3{Where do you want to run your solver ?}
-    // The gko::Executor class is one of the cornerstones of Ginkgo. Currently,
-    // we have support for
-    // an gko::OmpExecutor, which uses OpenMP multi-threading in most of its
-    // kernels, a gko::ReferenceExecutor, a single threaded specialization of
-    // the OpenMP executor and a gko::CudaExecutor which runs the code on a
-    // NVIDIA GPU if available.
-    // @note With the help of C++, you see that you only ever need to change the
-    // executor and all the other functions/ routines within Ginkgo should
-    // automatically work and run on the executor with any other changes.
-    std::shared_ptr<gko::Executor> exec;
-    if (argc == 1 || std::string(argv[1]) == "reference") {
-        exec = gko::ReferenceExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "omp") {
-        exec = gko::OmpExecutor::create();
-    } else if (argc == 2 && std::string(argv[1]) == "cuda" &&
-               gko::CudaExecutor::get_num_devices() > 0) {
-        exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
-    } else if (argc == 2 && std::string(argv[1]) == "hip" &&
-               gko::HipExecutor::get_num_devices() > 0) {
-        exec = gko::HipExecutor::create(0, gko::OmpExecutor::create());
-    } else {
-        std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
-        std::exit(-1);
-    }
-
-    // @sect3{Reading your data and transfer to the proper device.}
-    // Read the matrix, right hand side and the initial solution using the @ref
-    // read function.
-    // @note Ginkgo uses C++ smart pointers to automatically manage memory. To
-    // this end, we use our own object ownership transfer functions that under
-    // the hood call the required smart pointer functions to manage object
-    // ownership. The gko::share , gko::give and gko::lend are the functions
-    // that you would need to use.
-    auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
-    auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
-    auto x = gko::read<vec>(std::ifstream("data/x0.mtx"), exec);
-
-    // @sect3{Creating the solver}
-    // Generate the gko::solver factory. Ginkgo uses the concept of Factories to
-    // build solvers with certain
-    // properties. Observe the Fluent interface used here. Here a gmres solver
-    // is generated with a stopping criteria of maximum iterations of 20 and a
-    // residual norm reduction of 1e-15. You also observe that the stopping
-    // criteria(gko::stop) are also generated from factories using their build
-    // methods. You need to specify the executors which each of the object needs
-    // to be built on.
-    auto solver_gen =
-        gmres::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(20u).on(exec),
-                gko::stop::ResidualNormReduction<>::build()
-                    .with_reduction_factor(1e-15)
-                    .on(exec))
-            .on(exec);
-    // Generate the solver from the matrix. The solver factory built in the
-    // previous step takes a "matrix"(a gko::LinOp to be more general) as an
-    // input. In this case we provide it with a full matrix that we previously
-    // read, but as the solver only effectively uses the apply() method within
-    // the provided "matrix" object, you can effectively create a gko::LinOp
-    // class with your own apply implementation to accomplish more tasks. We
-    // will see an example of how this can be done in the custom-matrix-format
-    // example
-    auto solver = solver_gen->generate(A);
-
-    // Finally, solve the system. The solver, being a gko::LinOp, can be applied
-    // to a right hand side, b to
-    // obtain the solution, x.
-    solver->apply(lend(b), lend(x));
-
-    // Print the solution to the command line.
-    std::cout << "Solution (x): \n";
-    write(std::cout, lend(x));
-
-    // To measure if your solution has actually converged, you can measure the
-    // error of the solution.
-    // one, neg_one are objects that represent the numbers which allow for a
-    // uniform interface when computing on any device. To compute the residual,
-    // all you need to do is call the apply method, which in this case is an
-    // spmv and equivalent to the LAPACK z_spmv routine. Finally, you compute
-    // the euclidean 2-norm with the compute_norm2 function.
-    auto one = gko::initialize<vec>({1.0}, exec);
-    auto neg_one = gko::initialize<vec>({-1.0}, exec);
-    auto res = gko::initialize<vec>({0.0}, exec);
-    A->apply(lend(one), lend(x), lend(neg_one), lend(b));
-    b->compute_norm2(lend(res));
-
-    std::cout << "Residual norm sqrt(r^T r): \n";
-    write(std::cout, lend(res));
-}
diff --git a/include/ginkgo/core/solver/cb_gmres.hpp b/include/ginkgo/core/solver/cb_gmres.hpp
index 75318ea0631..f3fb512c577 100644
--- a/include/ginkgo/core/solver/cb_gmres.hpp
+++ b/include/ginkgo/core/solver/cb_gmres.hpp
@@ -104,6 +104,16 @@ class CbGmres : public EnableLinOp<CbGmres<ValueType>>,
      */
     size_type get_krylov_dim() const { return krylov_dim_; }
 
+    /**
+     * Returns the storage precision used internally.
+     *
+     * @return the storage precision used internally
+     */
+    cb_gmres_storage_precision get_storage_precision() const
+    {
+        return storage_precision_;
+    }
+
     GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
     {
         /**

From 37d9dd8c82b07b8b11313372d7223ba6ef73ed7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 3 Feb 2021 09:31:15 +0100
Subject: [PATCH 0404/1008] Review update

- Add more documentation to CB-GMRES
- Add CB-GMRES to test-install

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Yuhsiang M. Tsai <yhmtsai@gmail.com>
---
 core/solver/cb_gmres.cpp                 |  2 -
 cuda/test/solver/cb_gmres_kernels.cpp    | 31 +++++++-------
 hip/test/solver/cb_gmres_kernels.cpp     | 31 +++++++-------
 include/ginkgo/core/solver/cb_gmres.hpp  | 52 ++++++++++++++++++------
 omp/solver/cb_gmres_kernels.cpp          | 40 ++++++++++--------
 omp/test/solver/cb_gmres_kernels.cpp     | 31 +++++++-------
 reference/solver/cb_gmres_kernels.cpp    | 24 ++++++-----
 reference/stop/residual_norm_kernels.cpp |  1 -
 test_install/test_install.cpp            | 10 +++++
 9 files changed, 128 insertions(+), 94 deletions(-)

diff --git a/core/solver/cb_gmres.cpp b/core/solver/cb_gmres.cpp
index f562ca18af8..98444e27331 100644
--- a/core/solver/cb_gmres.cpp
+++ b/core/solver/cb_gmres.cpp
@@ -41,10 +41,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/name_demangling.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
-//#include <ginkgo/core/matrix/identity.hpp>
 
 
 #include "core/solver/cb_gmres_accessor.hpp"
diff --git a/cuda/test/solver/cb_gmres_kernels.cpp b/cuda/test/solver/cb_gmres_kernels.cpp
index 474d178fbf2..d0e0862f048 100644
--- a/cuda/test/solver/cb_gmres_kernels.cpp
+++ b/cuda/test/solver/cb_gmres_kernels.cpp
@@ -66,6 +66,7 @@ class CbGmres : public ::testing::Test {
     using Range3d = typename Range3dHelper::Range;
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
+    static constexpr unsigned int default_krylov_dim_mixed{100};
 
     CbGmres() : rand_engine(30) {}
 
@@ -121,26 +122,24 @@ class CbGmres : public ::testing::Test {
         int m = 597;
         int n = 43;
         x = gen_mtx(m, n);
-        y = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        y = gen_mtx(default_krylov_dim_mixed, n);
         before_preconditioner = Mtx::create_with_config_of(x.get());
         b = gen_mtx(m, n);
         b_norm = gen_mtx(1, n);
         arnoldi_norm = gen_mtx(3, n);
-        gko::dim<3> krylov_bases_dim(gko::solver::default_krylov_dim_mixed + 1,
-                                     m, n);
+        gko::dim<3> krylov_bases_dim(default_krylov_dim_mixed + 1, m, n);
         range_helper = generate_krylov_helper(krylov_bases_dim);
 
         next_krylov_basis = gen_mtx(m, n);
-        hessenberg = gen_mtx(gko::solver::default_krylov_dim_mixed + 1,
-                             gko::solver::default_krylov_dim_mixed * n);
-        hessenberg_iter = gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
-        buffer_iter = gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
+        hessenberg =
+            gen_mtx(default_krylov_dim_mixed + 1, default_krylov_dim_mixed * n);
+        hessenberg_iter = gen_mtx(default_krylov_dim_mixed + 1, n);
+        buffer_iter = gen_mtx(default_krylov_dim_mixed + 1, n);
         residual = gen_mtx(m, n);
         residual_norm = gen_mtx(1, n);
-        residual_norm_collection =
-            gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
-        givens_sin = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
-        givens_cos = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        residual_norm_collection = gen_mtx(default_krylov_dim_mixed + 1, n);
+        givens_sin = gen_mtx(default_krylov_dim_mixed, n);
+        givens_cos = gen_mtx(default_krylov_dim_mixed, n);
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(ref, n));
         for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
@@ -276,12 +275,10 @@ TEST_F(CbGmres, CudaCbGmresInitialize1IsEquivalentToRef)
 
     gko::kernels::reference::cb_gmres::initialize_1(
         ref, b.get(), b_norm.get(), residual.get(), givens_sin.get(),
-        givens_cos.get(), stop_status.get(),
-        gko::solver::default_krylov_dim_mixed);
+        givens_cos.get(), stop_status.get(), default_krylov_dim_mixed);
     gko::kernels::cuda::cb_gmres::initialize_1(
         cuda, d_b.get(), d_b_norm.get(), d_residual.get(), d_givens_sin.get(),
-        d_givens_cos.get(), d_stop_status.get(),
-        gko::solver::default_krylov_dim_mixed);
+        d_givens_cos.get(), d_stop_status.get(), default_krylov_dim_mixed);
 
     GKO_ASSERT_MTX_NEAR(d_b_norm, b_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual, residual, 1e-14);
@@ -299,12 +296,12 @@ TEST_F(CbGmres, CudaCbGmresInitialize2IsEquivalentToRef)
         ref, residual.get(), residual_norm.get(),
         residual_norm_collection.get(), arnoldi_norm.get(),
         range_helper.get_range(), next_krylov_basis.get(),
-        final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
+        final_iter_nums.get(), default_krylov_dim_mixed);
     gko::kernels::cuda::cb_gmres::initialize_2(
         cuda, d_residual.get(), d_residual_norm.get(),
         d_residual_norm_collection.get(), d_arnoldi_norm.get(),
         d_range_helper.get_range(), d_next_krylov_basis.get(),
-        d_final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
+        d_final_iter_nums.get(), default_krylov_dim_mixed);
 
     GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
diff --git a/hip/test/solver/cb_gmres_kernels.cpp b/hip/test/solver/cb_gmres_kernels.cpp
index f228675391c..15b220b5bfe 100644
--- a/hip/test/solver/cb_gmres_kernels.cpp
+++ b/hip/test/solver/cb_gmres_kernels.cpp
@@ -66,6 +66,7 @@ class CbGmres : public ::testing::Test {
     using Range3d = typename Range3dHelper::Range;
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
+    static constexpr unsigned int default_krylov_dim_mixed{100};
 
     CbGmres() : rand_engine(30) {}
 
@@ -121,26 +122,24 @@ class CbGmres : public ::testing::Test {
         int m = 597;
         int n = 43;
         x = gen_mtx(m, n);
-        y = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        y = gen_mtx(default_krylov_dim_mixed, n);
         before_preconditioner = Mtx::create_with_config_of(x.get());
         b = gen_mtx(m, n);
         b_norm = gen_mtx(1, n);
         arnoldi_norm = gen_mtx(3, n);
-        gko::dim<3> krylov_bases_dim(gko::solver::default_krylov_dim_mixed + 1,
-                                     m, n);
+        gko::dim<3> krylov_bases_dim(default_krylov_dim_mixed + 1, m, n);
         range_helper = generate_krylov_helper(krylov_bases_dim);
 
         next_krylov_basis = gen_mtx(m, n);
-        hessenberg = gen_mtx(gko::solver::default_krylov_dim_mixed + 1,
-                             gko::solver::default_krylov_dim_mixed * n);
-        hessenberg_iter = gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
-        buffer_iter = gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
+        hessenberg =
+            gen_mtx(default_krylov_dim_mixed + 1, default_krylov_dim_mixed * n);
+        hessenberg_iter = gen_mtx(default_krylov_dim_mixed + 1, n);
+        buffer_iter = gen_mtx(default_krylov_dim_mixed + 1, n);
         residual = gen_mtx(m, n);
         residual_norm = gen_mtx(1, n);
-        residual_norm_collection =
-            gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
-        givens_sin = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
-        givens_cos = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        residual_norm_collection = gen_mtx(default_krylov_dim_mixed + 1, n);
+        givens_sin = gen_mtx(default_krylov_dim_mixed, n);
+        givens_cos = gen_mtx(default_krylov_dim_mixed, n);
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(ref, n));
         for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
@@ -276,12 +275,10 @@ TEST_F(CbGmres, HipCbGmresInitialize1IsEquivalentToRef)
 
     gko::kernels::reference::cb_gmres::initialize_1(
         ref, b.get(), b_norm.get(), residual.get(), givens_sin.get(),
-        givens_cos.get(), stop_status.get(),
-        gko::solver::default_krylov_dim_mixed);
+        givens_cos.get(), stop_status.get(), default_krylov_dim_mixed);
     gko::kernels::hip::cb_gmres::initialize_1(
         hip, d_b.get(), d_b_norm.get(), d_residual.get(), d_givens_sin.get(),
-        d_givens_cos.get(), d_stop_status.get(),
-        gko::solver::default_krylov_dim_mixed);
+        d_givens_cos.get(), d_stop_status.get(), default_krylov_dim_mixed);
 
     GKO_ASSERT_MTX_NEAR(d_b_norm, b_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual, residual, 1e-14);
@@ -299,12 +296,12 @@ TEST_F(CbGmres, HipCbGmresInitialize2IsEquivalentToRef)
         ref, residual.get(), residual_norm.get(),
         residual_norm_collection.get(), arnoldi_norm.get(),
         range_helper.get_range(), next_krylov_basis.get(),
-        final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
+        final_iter_nums.get(), default_krylov_dim_mixed);
     gko::kernels::hip::cb_gmres::initialize_2(
         hip, d_residual.get(), d_residual_norm.get(),
         d_residual_norm_collection.get(), d_arnoldi_norm.get(),
         d_range_helper.get_range(), d_next_krylov_basis.get(),
-        d_final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
+        d_final_iter_nums.get(), default_krylov_dim_mixed);
 
     GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
diff --git a/include/ginkgo/core/solver/cb_gmres.hpp b/include/ginkgo/core/solver/cb_gmres.hpp
index f3fb512c577..e215c9ccff3 100644
--- a/include/ginkgo/core/solver/cb_gmres.hpp
+++ b/include/ginkgo/core/solver/cb_gmres.hpp
@@ -52,9 +52,33 @@ namespace gko {
 namespace solver {
 
 
-constexpr size_type default_krylov_dim_mixed = 100u;
-
-
+/**
+ * Describes the storage precision that is used in CB-GMRES.
+ *
+ * The storage precision is described relative to the ValueType:
+ * - keep: The storage precision is the same as the ValueType.
+ * - reduce1: The storage type is the ValueType reduced in precision once,
+ *            for example, ValueType == double -> storage precision == float
+ * - reduce2: ValueType precision is reduced twice
+ * - integer: The storage precision is an integer of the same size of
+ *            ValueType. Note that complex values are not supported.
+ * - ireduce1: The storage precision is an integer of the same size as
+ *             a reduced ValueType.
+ * - ireduce2: The storage precision is an integer of the same size as
+ *             a twice reduced ValueType.
+ *
+ * Precision reduction works as follows:
+ * - double -> float -> half -> half -> ... (half is the lowest supported
+ *   precision)
+ * - std::complex<double> -> std::complex<float> -> std::complex<half>
+ *   -> std::complex<half> ... (std::complex<half> is the lowest supported
+ *   precision)
+ *
+ * To integer conversions:
+ * - double -> int64
+ * - float -> int32
+ * - half -> int16
+ */
 enum class cb_gmres_storage_precision {
     keep,
     reduce1,
@@ -66,14 +90,20 @@ enum class cb_gmres_storage_precision {
 
 
 /**
- * CB-GMRES or the generalized minimal residual method is an iterative type
- * Krylov subspace method which is suitable for nonsymmetric linear systems.
+ * CB-GMRES or the compressed basis generalized minimal residual method is an
+ * iterative type Krylov subspace method which is suitable for nonsymmetric
+ * linear systems.
  *
  * The implementation in Ginkgo makes use of the merged kernel to make the best
  * use of data locality. The inner operations in one iteration of CB-GMRES
- * are merged into 2 separate steps.
+ * are merged into 2 separate steps. Classical Gram-Schmidt is used.
+ *
+ * The krylov basis can be stored in reduced precision (compressed) to reduce
+ * memory accesses, while all computations (including krylov basis operations)
+ * are performed in the same arithmetic precision ValueType.
  *
- * @tparam ValueType  precision of matrix elements
+ * @tparam ValueType  the arithmetic precision and the precision of matrix
+ *                    elements
  *
  * @ingroup solvers
  * @ingroup LinOp
@@ -144,7 +174,7 @@ class CbGmres : public EnableLinOp<CbGmres<ValueType>>,
         /**
          * krylov dimension factory.
          */
-        size_type GKO_FACTORY_PARAMETER_SCALAR(krylov_dim, 0u);
+        size_type GKO_FACTORY_PARAMETER_SCALAR(krylov_dim, 100u);
     };
     GKO_ENABLE_LIN_OP_FACTORY(CbGmres, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);
@@ -177,11 +207,7 @@ class CbGmres : public EnableLinOp<CbGmres<ValueType>>,
             set_preconditioner(matrix::Identity<ValueType>::create(
                 this->get_executor(), this->get_size()[0]));
         }
-        if (parameters_.krylov_dim) {
-            krylov_dim_ = parameters_.krylov_dim;
-        } else {
-            krylov_dim_ = default_krylov_dim_mixed;
-        }
+        krylov_dim_ = parameters_.krylov_dim;
         stop_criterion_factory_ =
             stop::combine(std::move(parameters_.criteria));
         storage_precision_ = parameters_.storage_precision;
diff --git a/omp/solver/cb_gmres_kernels.cpp b/omp/solver/cb_gmres_kernels.cpp
index 54fd439d98b..3e3357fee78 100644
--- a/omp/solver/cb_gmres_kernels.cpp
+++ b/omp/solver/cb_gmres_kernels.cpp
@@ -65,17 +65,20 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                          matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
                          size_type iter, const stopping_status *stop_status)
 {
-    const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
+    using rc_vtype = remove_complex<ValueType>;
+    const rc_vtype eta = 1.0 / sqrt(2.0);
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
-#pragma omp declare reduction(addnc : remove_complex<ValueType> : omp_out = omp_out + omp_in)
-#pragma omp declare reduction(infnc : remove_complex<ValueType> : omp_out = (omp_out >= omp_in? omp_out: omp_in))
+#pragma omp declare reduction(addnc:rc_vtype : omp_out = omp_out + omp_in)
+#pragma omp declare reduction(infnc:rc_vtype \
+                              : omp_out =    \
+                                    (omp_out >= omp_in ? omp_out : omp_in))
 
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
         if (stop_status[i].has_stopped()) {
             continue;
         }
 
-        remove_complex<ValueType> nrm = zero<remove_complex<ValueType>>();
+        auto nrm = zero<rc_vtype>();
 #pragma omp parallel for reduction(addnc : nrm)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             nrm += squared_norm(next_krylov_basis->at(j, i));
@@ -104,9 +107,8 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         // for i in 1:iter
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
-        // ValueType nrm = zero<ValueType>();
-        nrm = zero<remove_complex<ValueType>>();
-        remove_complex<ValueType> inf = zero<remove_complex<ValueType>>();
+        nrm = zero<rc_vtype>();
+        auto inf = zero<rc_vtype>();
 #pragma omp parallel for reduction(addnc : nrm) reduction(infnc : inf)
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             nrm += squared_norm(next_krylov_basis->at(j, i));
@@ -146,8 +148,8 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
             // for i in 1:iter
             //     next_krylov_basis  -= buffer(iter, i) * krylov_bases(:, i)
             // end
-            remove_complex<ValueType> nrm = zero<remove_complex<ValueType>>();
-            remove_complex<ValueType> inf = zero<remove_complex<ValueType>>();
+            auto nrm = zero<rc_vtype>();
+            auto inf = zero<rc_vtype>();
 #pragma omp parallel for reduction(addnc : nrm) reduction(infnc : inf)
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 nrm += squared_norm(next_krylov_basis->at(j, i));
@@ -319,11 +321,13 @@ void initialize_1(std::shared_ptr<const OmpExecutor> exec,
                   matrix::Dense<ValueType> *givens_cos,
                   Array<stopping_status> *stop_status, size_type krylov_dim)
 {
+    using rc_vtype = remove_complex<ValueType>;
+
     for (size_type j = 0; j < b->get_size()[1]; ++j) {
         // Calculate b norm
-        remove_complex<ValueType> norm = zero<remove_complex<ValueType>>();
+        auto norm = zero<rc_vtype>();
 
-#pragma omp declare reduction(addnc:remove_complex<ValueType> : omp_out = omp_out + omp_in)
+#pragma omp declare reduction(addnc:rc_vtype : omp_out = omp_out + omp_in)
 
 #pragma omp parallel for reduction(addnc : norm)
         for (size_type i = 0; i < b->get_size()[0]; ++i) {
@@ -359,13 +363,17 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
 {
+    using rc_vtype = remove_complex<ValueType>;
+
     for (size_type j = 0; j < residual->get_size()[1]; ++j) {
         // Calculate residual norm
-        remove_complex<ValueType> res_norm = zero<remove_complex<ValueType>>();
-        remove_complex<ValueType> res_inf = zero<remove_complex<ValueType>>();
+        auto res_norm = zero<rc_vtype>();
+        auto res_inf = zero<rc_vtype>();
 
-#pragma omp declare reduction(addnc:remove_complex<ValueType> : omp_out = omp_out + omp_in)
-#pragma omp declare reduction(infnc : remove_complex<ValueType> : omp_out = (omp_out >= omp_in? omp_out: omp_in))
+#pragma omp declare reduction(addnc:rc_vtype : omp_out = omp_out + omp_in)
+#pragma omp declare reduction(infnc:rc_vtype \
+                              : omp_out =    \
+                                    (omp_out >= omp_in ? omp_out : omp_in))
 
 #pragma omp parallel for reduction(addnc : res_norm) reduction(infnc : res_inf)
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
@@ -402,7 +410,7 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
     for (size_type k = 1; k < krylov_dim + 1; ++k) {
         for (size_type j = 0; j < residual->get_size()[1]; ++j) {
             gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
-                krylov_bases, k, j, one<remove_complex<ValueType>>());
+                krylov_bases, k, j, one<rc_vtype>());
         }
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             for (size_type j = 0; j < residual->get_size()[1]; ++j) {
diff --git a/omp/test/solver/cb_gmres_kernels.cpp b/omp/test/solver/cb_gmres_kernels.cpp
index c9c8c5d45cf..3fbb704a2e6 100644
--- a/omp/test/solver/cb_gmres_kernels.cpp
+++ b/omp/test/solver/cb_gmres_kernels.cpp
@@ -68,6 +68,7 @@ class CbGmres : public ::testing::Test {
     using Range3d = typename Range3dHelper::Range;
     using Dense = gko::matrix::Dense<value_type>;
     using Mtx = Dense;
+    static constexpr unsigned int default_krylov_dim_mixed{100};
 
 
     CbGmres() : rand_engine(30) {}
@@ -123,26 +124,24 @@ class CbGmres : public ::testing::Test {
         int m = 597;
         int n = 43;
         x = gen_mtx(m, n);
-        y = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        y = gen_mtx(default_krylov_dim_mixed, n);
         before_preconditioner = Mtx::create_with_config_of(x.get());
         b = gen_mtx(m, n);
         b_norm = gen_mtx(1, n);
         arnoldi_norm = gen_mtx(3, n);
-        gko::dim<3> krylov_bases_dim(gko::solver::default_krylov_dim_mixed + 1,
-                                     m, n);
+        gko::dim<3> krylov_bases_dim(default_krylov_dim_mixed + 1, m, n);
         range_helper = generate_krylov_helper(krylov_bases_dim);
 
         next_krylov_basis = gen_mtx(m, n);
-        hessenberg = gen_mtx(gko::solver::default_krylov_dim_mixed + 1,
-                             gko::solver::default_krylov_dim_mixed * n);
-        hessenberg_iter = gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
-        buffer_iter = gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
+        hessenberg =
+            gen_mtx(default_krylov_dim_mixed + 1, default_krylov_dim_mixed * n);
+        hessenberg_iter = gen_mtx(default_krylov_dim_mixed + 1, n);
+        buffer_iter = gen_mtx(default_krylov_dim_mixed + 1, n);
         residual = gen_mtx(m, n);
         residual_norm = gen_mtx(1, n);
-        residual_norm_collection =
-            gen_mtx(gko::solver::default_krylov_dim_mixed + 1, n);
-        givens_sin = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
-        givens_cos = gen_mtx(gko::solver::default_krylov_dim_mixed, n);
+        residual_norm_collection = gen_mtx(default_krylov_dim_mixed + 1, n);
+        givens_sin = gen_mtx(default_krylov_dim_mixed, n);
+        givens_cos = gen_mtx(default_krylov_dim_mixed, n);
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(ref, n));
         for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
@@ -277,12 +276,10 @@ TEST_F(CbGmres, OmpCbGmresInitialize1IsEquivalentToRef)
 
     gko::kernels::reference::cb_gmres::initialize_1(
         ref, b.get(), b_norm.get(), residual.get(), givens_sin.get(),
-        givens_cos.get(), stop_status.get(),
-        gko::solver::default_krylov_dim_mixed);
+        givens_cos.get(), stop_status.get(), default_krylov_dim_mixed);
     gko::kernels::omp::cb_gmres::initialize_1(
         omp, d_b.get(), d_b_norm.get(), d_residual.get(), d_givens_sin.get(),
-        d_givens_cos.get(), d_stop_status.get(),
-        gko::solver::default_krylov_dim_mixed);
+        d_givens_cos.get(), d_stop_status.get(), default_krylov_dim_mixed);
 
     GKO_ASSERT_MTX_NEAR(d_b_norm, b_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual, residual, 1e-14);
@@ -299,12 +296,12 @@ TEST_F(CbGmres, OmpCbGmresInitialize2IsEquivalentToRef)
         ref, residual.get(), residual_norm.get(),
         residual_norm_collection.get(), arnoldi_norm.get(),
         range_helper.get_range(), next_krylov_basis.get(),
-        final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
+        final_iter_nums.get(), default_krylov_dim_mixed);
     gko::kernels::omp::cb_gmres::initialize_2(
         omp, d_residual.get(), d_residual_norm.get(),
         d_residual_norm_collection.get(), d_arnoldi_norm.get(),
         d_range_helper.get_range(), d_next_krylov_basis.get(),
-        d_final_iter_nums.get(), gko::solver::default_krylov_dim_mixed);
+        d_final_iter_nums.get(), default_krylov_dim_mixed);
 
     GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
diff --git a/reference/solver/cb_gmres_kernels.cpp b/reference/solver/cb_gmres_kernels.cpp
index 5eafa9c84f0..596d0463654 100644
--- a/reference/solver/cb_gmres_kernels.cpp
+++ b/reference/solver/cb_gmres_kernels.cpp
@@ -72,9 +72,11 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         std::is_same<ValueType,
                      typename Accessor3d::accessor::arithmetic_type>::value,
         "ValueType must match arithmetic_type of accessor!");
-    const remove_complex<ValueType> eta = 1.0 / sqrt(2.0);
+    using rc_vtype = remove_complex<ValueType>;
+    const rc_vtype eta = 1.0 / sqrt(2.0);
+
     for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
-        arnoldi_norm->at(0, i) = zero<remove_complex<ValueType>>();
+        arnoldi_norm->at(0, i) = zero<rc_vtype>();
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             arnoldi_norm->at(0, i) += squared_norm(next_krylov_basis->at(j, i));
         }
@@ -102,8 +104,8 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         // for i in 1:iter
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
-        arnoldi_norm->at(1, i) = zero<remove_complex<ValueType>>();
-        arnoldi_norm->at(2, i) = zero<remove_complex<ValueType>>();
+        arnoldi_norm->at(1, i) = zero<rc_vtype>();
+        arnoldi_norm->at(2, i) = zero<rc_vtype>();
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             arnoldi_norm->at(1, i) += squared_norm(next_krylov_basis->at(j, i));
             arnoldi_norm->at(2, i) =
@@ -140,8 +142,8 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
             //     next_krylov_basis   -= buffer(iter, i) * krylov_bases(:, i)
             //     hessenberg(iter, i) += buffer(iter, i)
             // end
-            arnoldi_norm->at(1, i) = zero<remove_complex<ValueType>>();
-            arnoldi_norm->at(2, i) = zero<remove_complex<ValueType>>();
+            arnoldi_norm->at(1, i) = zero<rc_vtype>();
+            arnoldi_norm->at(2, i) = zero<rc_vtype>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 arnoldi_norm->at(1, i) +=
                     squared_norm(next_krylov_basis->at(j, i));
@@ -347,10 +349,12 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
         std::is_same<ValueType,
                      typename Accessor3d::accessor::arithmetic_type>::value,
         "ValueType must match arithmetic_type of accessor!");
+    using rc_vtype = remove_complex<ValueType>;
+
     for (size_type j = 0; j < residual->get_size()[1]; ++j) {
         // Calculate residual norm
-        residual_norm->at(0, j) = zero<remove_complex<ValueType>>();
-        arnoldi_norm->at(2, j) = zero<remove_complex<ValueType>>();
+        residual_norm->at(0, j) = zero<rc_vtype>();
+        arnoldi_norm->at(2, j) = zero<rc_vtype>();
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             residual_norm->at(0, j) += squared_norm(residual->at(i, j));
             arnoldi_norm->at(2, j) =
@@ -382,7 +386,7 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
     for (size_type k = 1; k < krylov_dim + 1; ++k) {
         for (size_type j = 0; j < residual->get_size()[1]; ++j) {
             gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
-                krylov_bases, k, j, one<remove_complex<ValueType>>());
+                krylov_bases, k, j, one<rc_vtype>());
             for (size_type i = 0; i < residual->get_size()[0]; ++i) {
                 krylov_bases(k, i, j) = zero<ValueType>();
             }
@@ -394,8 +398,6 @@ GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(
     GKO_DECLARE_CB_GMRES_INITIALIZE_2_KERNEL);
 
 
-// template <typename ValueType, typename ValueTypeKrylovBases, bool
-// Reorthogonalization, bool MGS_CGS>
 template <typename ValueType, typename Accessor3d>
 void step_1(std::shared_ptr<const ReferenceExecutor> exec,
             matrix::Dense<ValueType> *next_krylov_basis,
diff --git a/reference/stop/residual_norm_kernels.cpp b/reference/stop/residual_norm_kernels.cpp
index 0189762061f..b1f25b52c29 100644
--- a/reference/stop/residual_norm_kernels.cpp
+++ b/reference/stop/residual_norm_kernels.cpp
@@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
-#include <iostream>
 
 
 #include <ginkgo/core/base/array.hpp>
diff --git a/test_install/test_install.cpp b/test_install/test_install.cpp
index 969a059edf3..927045bfd3f 100644
--- a/test_install/test_install.cpp
+++ b/test_install/test_install.cpp
@@ -285,6 +285,16 @@ int main(int, char **)
                         .on(refExec);
     }
 
+    // core/solver/cb_gmres.hpp
+    {
+        using Solver = gko::solver::CbGmres<>;
+        auto test = Solver::build()
+                        .with_criteria(
+                            gko::stop::Iteration::build().with_max_iters(1u).on(
+                                refExec))
+                        .on(refExec);
+    }
+
     // core/solver/cg.hpp
     {
         using Solver = gko::solver::Cg<>;

From 7fb2fdee8feccb73f87d0af33253869ac7c4c720 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 3 Feb 2021 10:41:52 +0100
Subject: [PATCH 0405/1008] Remove unnecessary includes of iostream and time.h

---
 cuda/test/solver/idr_kernels.cpp      | 1 -
 hip/test/solver/idr_kernels.cpp       | 1 -
 include/ginkgo/core/solver/idr.hpp    | 1 -
 omp/solver/idr_kernels.cpp            | 2 --
 omp/test/matrix/diagonal_kernels.cpp  | 1 -
 reference/solver/cb_gmres_kernels.cpp | 3 ---
 reference/solver/idr_kernels.cpp      | 2 --
 reference/test/solver/idr_kernels.cpp | 3 ---
 8 files changed, 14 deletions(-)

diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index ef80acf5965..04ea559b6cc 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -33,7 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/idr.hpp>
 
 
-#include <iostream>
 #include <random>
 
 
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index 18ed225d095..c59ed90c9a7 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -33,7 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/idr.hpp>
 
 
-#include <iostream>
 #include <random>
 
 
diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index 110ebf8c686..939ecfcbb2c 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_PUBLIC_CORE_SOLVER_IDR_HPP_
 
 
-#include <iostream>
 #include <random>
 #include <typeinfo>
 #include <vector>
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index 59edde11e76..debfaf86eb8 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -34,9 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
-#include <iostream>
 #include <random>
-#include <time.h>
 
 
 #include <omp.h>
diff --git a/omp/test/matrix/diagonal_kernels.cpp b/omp/test/matrix/diagonal_kernels.cpp
index 9fcd618ad38..16de6c4d7f5 100644
--- a/omp/test/matrix/diagonal_kernels.cpp
+++ b/omp/test/matrix/diagonal_kernels.cpp
@@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
-#include <iostream>
 #include <numeric>
 #include <random>
 #include <vector>
diff --git a/reference/solver/cb_gmres_kernels.cpp b/reference/solver/cb_gmres_kernels.cpp
index 596d0463654..71f1078ef4e 100644
--- a/reference/solver/cb_gmres_kernels.cpp
+++ b/reference/solver/cb_gmres_kernels.cpp
@@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/cb_gmres_kernels.hpp"
 
 
-#include <iostream>
-
-
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index e419b2b1556..c1e40c1786c 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -34,9 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
-#include <iostream>
 #include <random>
-#include <time.h>
 
 
 #include <ginkgo/core/base/array.hpp>
diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp
index 4240e18f214..d473a94518e 100644
--- a/reference/test/solver/idr_kernels.cpp
+++ b/reference/test/solver/idr_kernels.cpp
@@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/idr.hpp>
 
 
-#include <iostream>
-
-
 #include <gtest/gtest.h>
 
 

From 906170cae23d01b355287ac98a204d273d59b405 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 3 Feb 2021 11:34:54 +0100
Subject: [PATCH 0406/1008] Remove circular dependency of compute_norm2 in
 (CB)-GMRES

---
 cuda/solver/cb_gmres_kernels.cu     | 5 +++--
 cuda/solver/gmres_kernels.cu        | 3 ++-
 hip/solver/cb_gmres_kernels.hip.cpp | 5 +++--
 hip/solver/gmres_kernels.hip.cpp    | 3 ++-
 4 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/cuda/solver/cb_gmres_kernels.cu b/cuda/solver/cb_gmres_kernels.cu
index 7a11a018abd..9ca7bfba3a1 100644
--- a/cuda/solver/cb_gmres_kernels.cu
+++ b/cuda/solver/cb_gmres_kernels.cu
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/base/accessors.hpp"
 #include "core/components/fill_array.hpp"
+#include "core/matrix/dense_kernels.hpp"
 #include "cuda/base/config.hpp"
 #include "cuda/base/math.hpp"
 #include "cuda/base/types.hpp"
@@ -125,7 +126,7 @@ void initialize_1(std::shared_ptr<const CudaExecutor> exec,
     const dim3 block_dim(default_block_size, 1, 1);
     constexpr auto block_size = default_block_size;
 
-    b->compute_norm2(b_norm);
+    kernels::cuda::dense::compute_norm2(exec, b, b_norm);
     initialize_1_kernel<block_size><<<grid_dim, block_dim>>>(
         b->get_size()[0], b->get_size()[1], krylov_dim,
         as_cuda_type(b->get_const_values()), b->get_stride(),
@@ -164,7 +165,7 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
         as_cuda_accessor(krylov_bases),
         as_cuda_type(residual_norm_collection->get_values()),
         residual_norm_collection->get_stride());
-    residual->compute_norm2(residual_norm);
+    kernels::cuda::dense::compute_norm2(exec, residual, residual_norm);
 
     components::fill_array(exec,
                            arnoldi_norm->get_values() + 2 * stride_arnoldi,
diff --git a/cuda/solver/gmres_kernels.cu b/cuda/solver/gmres_kernels.cu
index a5bb25f06d5..bacabd0e6c5 100644
--- a/cuda/solver/gmres_kernels.cu
+++ b/cuda/solver/gmres_kernels.cu
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/components/fill_array.hpp"
+#include "core/matrix/dense_kernels.hpp"
 #include "cuda/base/config.hpp"
 #include "cuda/base/cublas_bindings.hpp"
 #include "cuda/base/math.hpp"
@@ -118,7 +119,7 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
     const dim3 block_dim(default_block_size, 1, 1);
     constexpr auto block_size = default_block_size;
 
-    residual->compute_norm2(residual_norm);
+    kernels::cuda::dense::compute_norm2(exec, residual, residual_norm);
 
     const dim3 grid_dim_2(ceildiv(num_rows * num_rhs, default_block_size), 1,
                           1);
diff --git a/hip/solver/cb_gmres_kernels.hip.cpp b/hip/solver/cb_gmres_kernels.hip.cpp
index 12ee6717202..eb3432ceb04 100644
--- a/hip/solver/cb_gmres_kernels.hip.cpp
+++ b/hip/solver/cb_gmres_kernels.hip.cpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/components/fill_array.hpp"
+#include "core/matrix/dense_kernels.hpp"
 #include "hip/base/config.hip.hpp"
 #include "hip/base/math.hip.hpp"
 #include "hip/base/types.hip.hpp"
@@ -126,7 +127,7 @@ void initialize_1(std::shared_ptr<const HipExecutor> exec,
     const dim3 block_dim(default_block_size, 1, 1);
     constexpr auto block_size = default_block_size;
 
-    b->compute_norm2(b_norm);
+    kernels::hip::dense::compute_norm2(exec, b, b_norm);
     hipLaunchKernelGGL(
         initialize_1_kernel<block_size>, grid_dim, block_dim, 0, 0,
         b->get_size()[0], b->get_size()[1], krylov_dim,
@@ -166,7 +167,7 @@ void initialize_2(std::shared_ptr<const HipExecutor> exec,
                        krylov_dim, as_hip_accessor(krylov_bases),
                        as_hip_type(residual_norm_collection->get_values()),
                        residual_norm_collection->get_stride());
-    residual->compute_norm2(residual_norm);
+    kernels::hip::dense::compute_norm2(exec, residual, residual_norm);
 
     components::fill_array(exec,
                            arnoldi_norm->get_values() + 2 * stride_arnoldi,
diff --git a/hip/solver/gmres_kernels.hip.cpp b/hip/solver/gmres_kernels.hip.cpp
index cd430ffa916..d9d0034b915 100644
--- a/hip/solver/gmres_kernels.hip.cpp
+++ b/hip/solver/gmres_kernels.hip.cpp
@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/components/fill_array.hpp"
+#include "core/matrix/dense_kernels.hpp"
 #include "hip/base/config.hip.hpp"
 #include "hip/base/hipblas_bindings.hip.hpp"
 #include "hip/base/math.hip.hpp"
@@ -122,7 +123,7 @@ void initialize_2(std::shared_ptr<const HipExecutor> exec,
     const dim3 block_dim(default_block_size, 1, 1);
     constexpr auto block_size = default_block_size;
 
-    residual->compute_norm2(residual_norm);
+    kernels::hip::dense::compute_norm2(exec, residual, residual_norm);
 
     const dim3 grid_dim_2(ceildiv(num_rows * num_rhs, default_block_size), 1,
                           1);

From 0677a61982bb802df33113ac1b2520a35df90cbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 3 Feb 2021 21:41:04 +0100
Subject: [PATCH 0407/1008] Update solver generation in benchmark

---
 BENCHMARKING.md             |   9 +-
 benchmark/solver/solver.cpp | 213 ++++++++++++++++--------------------
 2 files changed, 97 insertions(+), 125 deletions(-)

diff --git a/BENCHMARKING.md b/BENCHMARKING.md
index 441412d8acd..8dd73bbf5ee 100644
--- a/BENCHMARKING.md
+++ b/BENCHMARKING.md
@@ -295,10 +295,11 @@ The supported environment variables are described in the following list:
     library formats (cuSPARSE with `cusp_` prefix or hipSPARSE with `hipsp_`
     prefix) can be used as well. Multiple options can be passed. The default is
     `csr,coo,ell,hybrid,sellp`.
-* `SOLVERS={bicgstab,bicg,cg,cgs,fcg,gmres,lower_trs,upper_trs}` - the solvers
-    which should be benchmarked. Multiple options can be passed. The default
-    is `cg`. Note that `lower/upper_trs` by default don't use a preconditioner,
-    as they are by default exact direct solvers.
+* `SOLVERS={bicgstab,bicg,cg,cgs,fcg,gmres,cb_gmres_{keep,reduce1,reduce2,integer,ireduce1,ireduce2},lower_trs,upper_trs}`
+    - the solvers which should be benchmarked. Multiple options can be passed.
+    The default is `bicgstab,cg,cgs.fcg.gmres,idr`. Note that `lower/upper_trs`
+    by default don't use a preconditioner, as they are by default exact direct
+    solvers.
 * `SOLVERS_PRECISION=<precision>` - the minimal residual reduction before which
     the solver should stop. The default is `1e-6`.
 * `SOLVERS_MAX_ITERATION=<number>` - the maximum number of iterations with which
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 257f10c4f69..35fa014dd85 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -72,7 +72,9 @@ DEFINE_bool(
 
 DEFINE_string(solvers, "cg",
               "A comma-separated list of solvers to run. "
-              "Supported values are: bicgstab, bicg, cg, cgs, fcg, gmres, idr, "
+              "Supported values are: bicgstab, bicg, cb_gmres_keep, "
+              "cb_gmres_reduce1, cb_gmres_reduce2, cb_gmres_integer, "
+              "cb_gmres_ireduce1, cb_gmres_ireduce2, cg, cgs, fcg, gmres, idr, "
               "lower_trs, upper_trs, overhead");
 
 DEFINE_uint32(
@@ -112,8 +114,6 @@ DEFINE_bool(overhead, false,
             "If set, uses dummy data to benchmark Ginkgo overhead");
 
 
-// TODO: create string list parameter for CB-GMRES solver
-
 // input validation
 [[noreturn]] void print_config_error_and_exit()
 {
@@ -221,128 +221,99 @@ std::shared_ptr<const gko::stop::CriterionFactory> create_criterion(
 }
 
 
-// solver mapping
-template <typename SolverType>
-std::unique_ptr<gko::LinOpFactory> create_solver(
-    std::shared_ptr<const gko::Executor> exec,
+template <typename SolverIntermediate>
+std::unique_ptr<gko::LinOpFactory> add_criteria_precond_finalize(
+    SolverIntermediate &&inter,
+    const std::shared_ptr<const gko::Executor> &exec,
     std::shared_ptr<const gko::LinOpFactory> precond)
 {
-    return SolverType::build()
-        .with_criteria(create_criterion(exec))
+    return inter.with_criteria(create_criterion(exec))
         .with_preconditioner(give(precond))
         .on(exec);
 }
 
+template <typename Solver>
+std::unique_ptr<gko::LinOpFactory> add_criteria_precond_finalize(
+    const std::shared_ptr<const gko::Executor> &exec,
+    std::shared_ptr<const gko::LinOpFactory> precond)
+{
+    return add_criteria_precond_finalize(Solver::build(), exec, precond);
+}
+
 
-const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
-                                std::shared_ptr<const gko::Executor>,
-                                std::shared_ptr<const gko::LinOpFactory>)>>
-    solver_factory{
-        {"bicgstab", create_solver<gko::solver::Bicgstab<etype>>},
-        {"bicg", create_solver<gko::solver::Bicg<etype>>},
-        {"cg", create_solver<gko::solver::Cg<etype>>},
-        {"cgs", create_solver<gko::solver::Cgs<etype>>},
-        {"fcg", create_solver<gko::solver::Fcg<etype>>},
-        {"idr",
-         [](std::shared_ptr<const gko::Executor> exec,
-            std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::Idr<etype>::build()
-                 .with_criteria(create_criterion(exec))
-                 .with_subspace_dim(FLAGS_idr_subspace_dim)
-                 .with_kappa(static_cast<rc_etype>(FLAGS_idr_kappa))
-                 .with_preconditioner(give(precond))
-                 .on(exec);
-         }},
-        {"gmres",
-         [](std::shared_ptr<const gko::Executor> exec,
-            std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::Gmres<etype>::build()
-                 .with_criteria(create_criterion(exec))
-                 .with_krylov_dim(FLAGS_gmres_restart)
-                 .with_preconditioner(give(precond))
-                 .on(exec);
-         }},
-        {"cb_gmres_keep",
-         [](std::shared_ptr<const gko::Executor> exec,
-            std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::CbGmres<etype>::build()
-                 .with_storage_precision(
-                     gko::solver::cb_gmres_storage_precision::keep)
-                 .with_criteria(create_criterion(exec))
-                 .with_krylov_dim(FLAGS_gmres_restart)
-                 .with_preconditioner(give(precond))
-                 .on(exec);
-         }},
-        {"cb_gmres_reduced",
-         [](std::shared_ptr<const gko::Executor> exec,
-            std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::CbGmres<etype>::build()
-                 .with_storage_precision(
-                     gko::solver::cb_gmres_storage_precision::reduce1)
-                 .with_criteria(create_criterion(exec))
-                 .with_krylov_dim(FLAGS_gmres_restart)
-                 .with_preconditioner(give(precond))
-                 .on(exec);
-         }},
-        {"cb_gmres_reduced2",
-         [](std::shared_ptr<const gko::Executor> exec,
-            std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::CbGmres<etype>::build()
-                 .with_storage_precision(
-                     gko::solver::cb_gmres_storage_precision::reduce2)
-                 .with_criteria(create_criterion(exec))
-                 .with_krylov_dim(FLAGS_gmres_restart)
-                 .with_preconditioner(give(precond))
-                 .on(exec);
-         }},
-        {"cb_gmres_int",
-         [](std::shared_ptr<const gko::Executor> exec,
-            std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::CbGmres<etype>::build()
-                 .with_storage_precision(
-                     gko::solver::cb_gmres_storage_precision::integer)
-                 .with_criteria(create_criterion(exec))
-                 .with_krylov_dim(FLAGS_gmres_restart)
-                 .with_preconditioner(give(precond))
-                 .on(exec);
-         }},
-        {"cb_gmres_int_reduced",
-         [](std::shared_ptr<const gko::Executor> exec,
-            std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::CbGmres<etype>::build()
-                 .with_storage_precision(
-                     gko::solver::cb_gmres_storage_precision::ireduce1)
-                 .with_criteria(create_criterion(exec))
-                 .with_krylov_dim(FLAGS_gmres_restart)
-                 .with_preconditioner(give(precond))
-                 .on(exec);
-         }},
-        {"cb_gmres_int_reduced2",
-         [](std::shared_ptr<const gko::Executor> exec,
-            std::shared_ptr<const gko::LinOpFactory> precond) {
-             return gko::solver::CbGmres<etype>::build()
-                 .with_storage_precision(
-                     gko::solver::cb_gmres_storage_precision::ireduce2)
-                 .with_criteria(create_criterion(exec))
-                 .with_krylov_dim(FLAGS_gmres_restart)
-                 .with_preconditioner(give(precond))
-                 .on(exec);
-         }},
-        {"lower_trs",
-         [](std::shared_ptr<const gko::Executor> exec,
-            std::shared_ptr<const gko::LinOpFactory>) {
-             return gko::solver::LowerTrs<etype>::build()
-                 .with_num_rhs(FLAGS_nrhs)
-                 .on(exec);
-         }},
-        {"upper_trs",
-         [](std::shared_ptr<const gko::Executor> exec,
-            std::shared_ptr<const gko::LinOpFactory>) {
-             return gko::solver::UpperTrs<etype>::build()
-                 .with_num_rhs(FLAGS_nrhs)
-                 .on(exec);
-         }},
-        {"overhead", create_solver<gko::Overhead<etype>>}};
+std::unique_ptr<gko::LinOpFactory> generate_solver(
+    const std::shared_ptr<const gko::Executor> &exec,
+    std::shared_ptr<const gko::LinOpFactory> precond,
+    const std::string &description)
+{
+    std::string cb_gmres_prefix("cb_gmres_");
+    if (description.find(cb_gmres_prefix) == 0) {
+        auto s_prec = gko::solver::cb_gmres_storage_precision::keep;
+        const auto spec = description.substr(cb_gmres_prefix.length());
+        if (spec == "keep") {
+            s_prec = gko::solver::cb_gmres_storage_precision::keep;
+        } else if (spec == "reduce1") {
+            s_prec = gko::solver::cb_gmres_storage_precision::reduce1;
+        } else if (spec == "reduce2") {
+            s_prec = gko::solver::cb_gmres_storage_precision::reduce2;
+        } else if (spec == "integer") {
+            s_prec = gko::solver::cb_gmres_storage_precision::integer;
+        } else if (spec == "ireduce1") {
+            s_prec = gko::solver::cb_gmres_storage_precision::ireduce1;
+        } else if (spec == "ireduce2") {
+            s_prec = gko::solver::cb_gmres_storage_precision::ireduce2;
+        } else {
+            throw std::range_error(
+                std::string(
+                    "CB-GMRES does not have a corresponding solver to <") +
+                description + ">!");
+        }
+        return add_criteria_precond_finalize(
+            gko::solver::CbGmres<etype>::build()
+                .with_krylov_dim(FLAGS_gmres_restart)
+                .with_storage_precision(s_prec),
+            exec, precond);
+    } else if (description == "bicgstab") {
+        return add_criteria_precond_finalize<gko::solver::Bicgstab<etype>>(
+            exec, precond);
+    } else if (description == "bicg") {
+        return add_criteria_precond_finalize<gko::solver::Bicg<etype>>(exec,
+                                                                       precond);
+    } else if (description == "cg") {
+        return add_criteria_precond_finalize<gko::solver::Cg<etype>>(exec,
+                                                                     precond);
+    } else if (description == "cgs") {
+        return add_criteria_precond_finalize<gko::solver::Cgs<etype>>(exec,
+                                                                      precond);
+    } else if (description == "fcg") {
+        return add_criteria_precond_finalize<gko::solver::Fcg<etype>>(exec,
+                                                                      precond);
+    } else if (description == "idr") {
+        return add_criteria_precond_finalize(
+            gko::solver::Idr<etype>::build()
+                .with_subspace_dim(FLAGS_idr_subspace_dim)
+                .with_kappa(static_cast<rc_etype>(FLAGS_idr_kappa)),
+            exec, precond);
+    } else if (description == "gmres") {
+        return add_criteria_precond_finalize(
+            gko::solver::Gmres<etype>::build().with_krylov_dim(
+                FLAGS_gmres_restart),
+            exec, precond);
+    } else if (description == "lower_trs") {
+        return gko::solver::LowerTrs<etype>::build()
+            .with_num_rhs(FLAGS_nrhs)
+            .on(exec);
+    } else if (description == "upper_trs") {
+        return gko::solver::UpperTrs<etype>::build()
+            .with_num_rhs(FLAGS_nrhs)
+            .on(exec);
+    } else if (description == "overhead") {
+        return add_criteria_precond_finalize<gko::Overhead<etype>>(exec,
+                                                                   precond);
+    }
+    throw std::range_error(std::string("The provided string <") + description +
+                           "> does not match any solver!");
+}
 
 
 void write_precond_info(const gko::LinOp *precond,
@@ -433,7 +404,7 @@ void solve_system(const std::string &solver_name,
         for (unsigned int i = 0; i < FLAGS_warmup; i++) {
             auto x_clone = clone(x);
             auto precond = precond_factory.at(precond_name)(exec);
-            auto solver = solver_factory.at(solver_name)(exec, give(precond))
+            auto solver = generate_solver(exec, give(precond), solver_name)
                               ->generate(system_matrix);
             solver->add_logger(it_logger);
             solver->apply(lend(b), lend(x_clone));
@@ -454,7 +425,7 @@ void solve_system(const std::string &solver_name,
             exec->add_logger(gen_logger);
 
             auto precond = precond_factory.at(precond_name)(exec);
-            auto solver = solver_factory.at(solver_name)(exec, give(precond))
+            auto solver = generate_solver(exec, give(precond), solver_name)
                               ->generate(system_matrix);
 
             exec->remove_logger(gko::lend(gen_logger));
@@ -504,7 +475,7 @@ void solve_system(const std::string &solver_name,
             exec->synchronize();
             generate_timer->tic();
             auto precond = precond_factory.at(precond_name)(exec);
-            auto solver = solver_factory.at(solver_name)(exec, give(precond))
+            auto solver = generate_solver(exec, give(precond), solver_name)
                               ->generate(system_matrix);
             generate_timer->toc();
 

From 84578994586520b8e8148f9e27e17376b8a53587 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 4 Feb 2021 09:09:45 +0100
Subject: [PATCH 0408/1008] Update eta and arnoldi_norms in CB-GMRES

---
 common/solver/cb_gmres_kernels.hpp.inc | 21 ++++++-----
 core/solver/cb_gmres.cpp               |  7 ++--
 cuda/solver/cb_gmres_kernels.cu        | 46 ++++++++++++++----------
 hip/solver/cb_gmres_kernels.hip.cpp    | 48 +++++++++++++++-----------
 omp/solver/cb_gmres_kernels.cpp        | 20 ++++++++---
 reference/solver/cb_gmres_kernels.cpp  | 34 ++++++++++++------
 6 files changed, 107 insertions(+), 69 deletions(-)

diff --git a/common/solver/cb_gmres_kernels.hpp.inc b/common/solver/cb_gmres_kernels.hpp.inc
index a50ac0f00a5..59d82cb53b9 100644
--- a/common/solver/cb_gmres_kernels.hpp.inc
+++ b/common/solver/cb_gmres_kernels.hpp.inc
@@ -507,28 +507,27 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms(
     stopping_status *__restrict__ reorth_status,
     size_type *__restrict__ num_reorth)
 {
-    const remove_complex<ValueType> eta = 1.0 / 2.0;
+    const remove_complex<ValueType> eta_squared = 1.0 / 2.0;
     const auto col_idx = thread::get_thread_id_flat();
-    constexpr bool has_scale =
+    constexpr bool has_scalar =
         gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value;
 
     if (col_idx < stride_norm && !stop_status[col_idx].has_stopped()) {
-        const auto num0 = (eta * arnoldi_norm[col_idx]);
-        const auto num11 = arnoldi_norm[col_idx + stride_norm];
-        const auto num12 = sqrt(num11);
-        const auto num2 = has_scale ? (arnoldi_norm[col_idx + 2 * stride_norm])
-                                    : remove_complex<ValueType>{};
+        const auto num0 = (sqrt(eta_squared * arnoldi_norm[col_idx]));
+        const auto num11 = sqrt(arnoldi_norm[col_idx + stride_norm]);
+        const auto num2 = has_scalar ? (arnoldi_norm[col_idx + 2 * stride_norm])
+                                     : remove_complex<ValueType>{};
         if (num11 < num0) {
             reorth_status[col_idx].reset();
             atomic_add(num_reorth, (size_type)1);
         } else {
             reorth_status[col_idx].stop(1);
         }
-        arnoldi_norm[col_idx] = num11;
-        arnoldi_norm[col_idx + stride_norm] = num12;
-        hessenberg_iter[iter * stride_hessenberg + col_idx] = num12;
+        arnoldi_norm[col_idx] = num0;
+        arnoldi_norm[col_idx + stride_norm] = num11;
+        hessenberg_iter[iter * stride_hessenberg + col_idx] = num11;
         gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
-            krylov_bases, iter, col_idx, num2 / num12);
+            krylov_bases, iter, col_idx, num2 / num11);
     }
 }
 
diff --git a/core/solver/cb_gmres.cpp b/core/solver/cb_gmres.cpp
index 98444e27331..d08d63d496e 100644
--- a/core/solver/cb_gmres.cpp
+++ b/core/solver/cb_gmres.cpp
@@ -239,13 +239,14 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
         auto b_norm =
             VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
-        // 1st row of arnoldi_norm: TODO
+        // 1st row of arnoldi_norm: == eta * norm2(old_next_krylov_basis)
+        //                          with eta == 1 / sqrt(2)
+        //                          (computed right before updating
+        //                          next_krylov_basis)
         // 2nd row of arnoldi_norm: The actual arnoldi norm
         //                          == norm2(next_krylov_basis)
         // 3rd row of arnoldi_norm: the infinity norm of next_krylov_basis
         //                          (ONLY when using a scalar accessor)
-        // The optional entry stores the infinity_norm of each
-        // next_krylov_vector, which is only used to compute the scalar
         auto arnoldi_norm =
             VectorNorms::create(exec, dim<2>{3, dense_b->get_size()[1]});
         Array<size_type> final_iter_nums(this->get_executor(),
diff --git a/cuda/solver/cb_gmres_kernels.cu b/cuda/solver/cb_gmres_kernels.cu
index 9ca7bfba3a1..25a652d6211 100644
--- a/cuda/solver/cb_gmres_kernels.cu
+++ b/cuda/solver/cb_gmres_kernels.cu
@@ -149,6 +149,8 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
 {
+    constexpr bool use_scalar =
+        gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value;
     const auto num_rows = residual->get_size()[0];
     const auto num_rhs = residual->get_size()[1];
     const auto krylov_stride =
@@ -167,16 +169,18 @@ void initialize_2(std::shared_ptr<const CudaExecutor> exec,
         residual_norm_collection->get_stride());
     kernels::cuda::dense::compute_norm2(exec, residual, residual_norm);
 
-    components::fill_array(exec,
-                           arnoldi_norm->get_values() + 2 * stride_arnoldi,
-                           num_rhs, zero<remove_complex<ValueType>>());
-    const dim3 grid_size_nrm(ceildiv(num_rhs, default_dot_dim),
-                             exec->get_num_multiprocessor() * 2);
-    const dim3 block_size_nrm(default_dot_dim, default_dot_dim);
-    multinorminf_without_stop_kernel<<<grid_size_nrm, block_size_nrm>>>(
-        num_rows, num_rhs, as_cuda_type(residual->get_const_values()),
-        residual->get_stride(),
-        as_cuda_type(arnoldi_norm->get_values() + 2 * stride_arnoldi), 0);
+    if (use_scalar) {
+        components::fill_array(exec,
+                               arnoldi_norm->get_values() + 2 * stride_arnoldi,
+                               num_rhs, zero<remove_complex<ValueType>>());
+        const dim3 grid_size_nrm(ceildiv(num_rhs, default_dot_dim),
+                                 exec->get_num_multiprocessor() * 2);
+        const dim3 block_size_nrm(default_dot_dim, default_dot_dim);
+        multinorminf_without_stop_kernel<<<grid_size_nrm, block_size_nrm>>>(
+            num_rows, num_rhs, as_cuda_type(residual->get_const_values()),
+            residual->get_stride(),
+            as_cuda_type(arnoldi_norm->get_values() + 2 * stride_arnoldi), 0);
+    }
 
     if (gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value) {
         set_scalar_kernel<default_block_size>
@@ -222,7 +226,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     using non_complex = remove_complex<ValueType>;
     // optimization parameter
     constexpr int singledot_block_size = default_dot_dim;
-    constexpr bool use_scale =
+    constexpr bool use_scalar =
         gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3dim>::value;
     const auto stride_next_krylov = next_krylov_basis->get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
@@ -287,10 +291,12 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     // end
     components::fill_array(exec, arnoldi_norm->get_values() + stride_arnoldi,
                            dim_size[1], zero<non_complex>());
-    components::fill_array(exec,
-                           arnoldi_norm->get_values() + 2 * stride_arnoldi,
-                           dim_size[1], zero<non_complex>());
-    multinorm2_inf_kernel<use_scale><<<grid_size, block_size>>>(
+    if (use_scalar) {
+        components::fill_array(exec,
+                               arnoldi_norm->get_values() + 2 * stride_arnoldi,
+                               dim_size[1], zero<non_complex>());
+    }
+    multinorm2_inf_kernel<use_scalar><<<grid_size, block_size>>>(
         dim_size[0], dim_size[1],
         as_cuda_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_cuda_type(arnoldi_norm->get_values() + stride_arnoldi),
@@ -349,10 +355,12 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         components::fill_array(exec,
                                arnoldi_norm->get_values() + stride_arnoldi,
                                dim_size[1], zero<non_complex>());
-        components::fill_array(exec,
-                               arnoldi_norm->get_values() + 2 * stride_arnoldi,
-                               dim_size[1], zero<non_complex>());
-        multinorm2_inf_kernel<use_scale><<<grid_size, block_size>>>(
+        if (use_scalar) {
+            components::fill_array(
+                exec, arnoldi_norm->get_values() + 2 * stride_arnoldi,
+                dim_size[1], zero<non_complex>());
+        }
+        multinorm2_inf_kernel<use_scalar><<<grid_size, block_size>>>(
             dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
             stride_next_krylov,
diff --git a/hip/solver/cb_gmres_kernels.hip.cpp b/hip/solver/cb_gmres_kernels.hip.cpp
index eb3432ceb04..cce0851b853 100644
--- a/hip/solver/cb_gmres_kernels.hip.cpp
+++ b/hip/solver/cb_gmres_kernels.hip.cpp
@@ -151,6 +151,8 @@ void initialize_2(std::shared_ptr<const HipExecutor> exec,
                   matrix::Dense<ValueType> *next_krylov_basis,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
 {
+    constexpr bool use_scalar =
+        gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value;
     const auto num_rows = residual->get_size()[0];
     const auto num_rhs = residual->get_size()[1];
     const auto krylov_stride =
@@ -169,17 +171,19 @@ void initialize_2(std::shared_ptr<const HipExecutor> exec,
                        residual_norm_collection->get_stride());
     kernels::hip::dense::compute_norm2(exec, residual, residual_norm);
 
-    components::fill_array(exec,
-                           arnoldi_norm->get_values() + 2 * stride_arnoldi,
-                           num_rhs, zero<remove_complex<ValueType>>());
-    const dim3 grid_size_nrm(ceildiv(num_rhs, default_dot_dim),
-                             exec->get_num_multiprocessor() * 2);
-    const dim3 block_size_nrm(default_dot_dim, default_dot_dim);
-    hipLaunchKernelGGL(
-        multinorminf_without_stop_kernel, grid_size_nrm, block_size_nrm, 0, 0,
-        num_rows, num_rhs, as_hip_type(residual->get_const_values()),
-        residual->get_stride(),
-        as_hip_type(arnoldi_norm->get_values() + 2 * stride_arnoldi), 0);
+    if (use_scalar) {
+        components::fill_array(exec,
+                               arnoldi_norm->get_values() + 2 * stride_arnoldi,
+                               num_rhs, zero<remove_complex<ValueType>>());
+        const dim3 grid_size_nrm(ceildiv(num_rhs, default_dot_dim),
+                                 exec->get_num_multiprocessor() * 2);
+        const dim3 block_size_nrm(default_dot_dim, default_dot_dim);
+        hipLaunchKernelGGL(
+            multinorminf_without_stop_kernel, grid_size_nrm, block_size_nrm, 0,
+            0, num_rows, num_rhs, as_hip_type(residual->get_const_values()),
+            residual->get_stride(),
+            as_hip_type(arnoldi_norm->get_values() + 2 * stride_arnoldi), 0);
+    }
 
     if (gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value) {
         hipLaunchKernelGGL(
@@ -225,7 +229,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
     using non_complex = remove_complex<ValueType>;
     // optimization parameter
     constexpr int singledot_block_size = default_dot_dim;
-    constexpr bool use_scale =
+    constexpr bool use_scalar =
         gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3dim>::value;
     const auto stride_next_krylov = next_krylov_basis->get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
@@ -293,11 +297,13 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
     // end
     components::fill_array(exec, arnoldi_norm->get_values() + stride_arnoldi,
                            dim_size[1], zero<non_complex>());
-    components::fill_array(exec,
-                           arnoldi_norm->get_values() + 2 * stride_arnoldi,
-                           dim_size[1], zero<non_complex>());
+    if (use_scalar) {
+        components::fill_array(exec,
+                               arnoldi_norm->get_values() + 2 * stride_arnoldi,
+                               dim_size[1], zero<non_complex>());
+    }
     hipLaunchKernelGGL(
-        multinorm2_inf_kernel<use_scale>, grid_size, block_size, 0, 0,
+        multinorm2_inf_kernel<use_scalar>, grid_size, block_size, 0, 0,
         dim_size[0], dim_size[1],
         as_hip_type(next_krylov_basis->get_const_values()), stride_next_krylov,
         as_hip_type(arnoldi_norm->get_values() + stride_arnoldi),
@@ -358,11 +364,13 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
         components::fill_array(exec,
                                arnoldi_norm->get_values() + stride_arnoldi,
                                dim_size[1], zero<non_complex>());
-        components::fill_array(exec,
-                               arnoldi_norm->get_values() + 2 * stride_arnoldi,
-                               dim_size[1], zero<non_complex>());
+        if (use_scalar) {
+            components::fill_array(
+                exec, arnoldi_norm->get_values() + 2 * stride_arnoldi,
+                dim_size[1], zero<non_complex>());
+        }
         hipLaunchKernelGGL(
-            multinorm2_inf_kernel<use_scale>, grid_size, block_size, 0, 0,
+            multinorm2_inf_kernel<use_scalar>, grid_size, block_size, 0, 0,
             dim_size[0], dim_size[1],
             as_hip_type(next_krylov_basis->get_const_values()),
             stride_next_krylov,
diff --git a/omp/solver/cb_gmres_kernels.cpp b/omp/solver/cb_gmres_kernels.cpp
index 3e3357fee78..6f7d22917d0 100644
--- a/omp/solver/cb_gmres_kernels.cpp
+++ b/omp/solver/cb_gmres_kernels.cpp
@@ -66,6 +66,8 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                          size_type iter, const stopping_status *stop_status)
 {
     using rc_vtype = remove_complex<ValueType>;
+    constexpr bool has_scalar =
+        gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value;
     const rc_vtype eta = 1.0 / sqrt(2.0);
 #pragma omp declare reduction(add:ValueType : omp_out = omp_out + omp_in)
 #pragma omp declare reduction(addnc:rc_vtype : omp_out = omp_out + omp_in)
@@ -83,7 +85,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             nrm += squared_norm(next_krylov_basis->at(j, i));
         }
-        arnoldi_norm->at(0, i) = nrm * eta;
+        arnoldi_norm->at(0, i) = eta * sqrt(nrm);
         // nrmP = norm(next_krylov_basis)
 #pragma omp parallel for
         for (size_type k = 0; k < iter + 1; ++k) {
@@ -117,12 +119,14 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                       : abs(next_krylov_basis->at(j, i));
         }
         arnoldi_norm->at(1, i) = sqrt(nrm);
-        arnoldi_norm->at(2, i) = inf;
+        if (has_scalar) {
+            arnoldi_norm->at(2, i) = inf;
+        }
 
         for (size_type l = 1;
              (arnoldi_norm->at(1, i)) < (arnoldi_norm->at(0, i)) && l < 3;
              l++) {
-            arnoldi_norm->at(0, i) = arnoldi_norm->at(1, i) * eta;
+            arnoldi_norm->at(0, i) = eta * arnoldi_norm->at(1, i);
             // nrmP = nrmN
 #pragma omp parallel for
             for (size_type k = 0; k < iter + 1; ++k) {
@@ -158,7 +162,9 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
                           : abs(next_krylov_basis->at(j, i));
             }
             arnoldi_norm->at(1, i) = sqrt(nrm);
-            arnoldi_norm->at(2, i) = inf;
+            if (has_scalar) {
+                arnoldi_norm->at(2, i) = inf;
+            }
             // nrmN = norm(next_krylov_basis)
             // nrmI = infnorm(next_krylov_basis)
         }
@@ -364,6 +370,8 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
                   Array<size_type> *final_iter_nums, size_type krylov_dim)
 {
     using rc_vtype = remove_complex<ValueType>;
+    constexpr bool has_scalar =
+        gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value;
 
     for (size_type j = 0; j < residual->get_size()[1]; ++j) {
         // Calculate residual norm
@@ -383,7 +391,9 @@ void initialize_2(std::shared_ptr<const OmpExecutor> exec,
                           : abs(residual->at(i, j));
         }
         residual_norm->at(0, j) = sqrt(res_norm);
-        arnoldi_norm->at(2, j) = res_inf;
+        if (has_scalar) {
+            arnoldi_norm->at(2, j) = res_inf;
+        }
         gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
             krylov_bases, {0}, j,
             arnoldi_norm->at(2, j) / residual_norm->at(0, j));
diff --git a/reference/solver/cb_gmres_kernels.cpp b/reference/solver/cb_gmres_kernels.cpp
index 71f1078ef4e..023666590f6 100644
--- a/reference/solver/cb_gmres_kernels.cpp
+++ b/reference/solver/cb_gmres_kernels.cpp
@@ -69,6 +69,8 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         std::is_same<ValueType,
                      typename Accessor3d::accessor::arithmetic_type>::value,
         "ValueType must match arithmetic_type of accessor!");
+    constexpr bool has_scalar =
+        gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value;
     using rc_vtype = remove_complex<ValueType>;
     const rc_vtype eta = 1.0 / sqrt(2.0);
 
@@ -77,7 +79,7 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             arnoldi_norm->at(0, i) += squared_norm(next_krylov_basis->at(j, i));
         }
-        arnoldi_norm->at(0, i) = sqrt(arnoldi_norm->at(0, i)) * eta;
+        arnoldi_norm->at(0, i) = eta * sqrt(arnoldi_norm->at(0, i));
         // arnoldi_norm->at(0, i) = norm(next_krylov_basis)
         if (stop_status[i].has_stopped()) {
             continue;
@@ -102,13 +104,17 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
         //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
         // end
         arnoldi_norm->at(1, i) = zero<rc_vtype>();
-        arnoldi_norm->at(2, i) = zero<rc_vtype>();
+        if (has_scalar) {
+            arnoldi_norm->at(2, i) = zero<rc_vtype>();
+        }
         for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
             arnoldi_norm->at(1, i) += squared_norm(next_krylov_basis->at(j, i));
-            arnoldi_norm->at(2, i) =
-                (arnoldi_norm->at(2, i) >= abs(next_krylov_basis->at(j, i)))
-                    ? arnoldi_norm->at(2, i)
-                    : abs(next_krylov_basis->at(j, i));
+            if (has_scalar) {
+                arnoldi_norm->at(2, i) =
+                    (arnoldi_norm->at(2, i) >= abs(next_krylov_basis->at(j, i)))
+                        ? arnoldi_norm->at(2, i)
+                        : abs(next_krylov_basis->at(j, i));
+            }
         }
         arnoldi_norm->at(1, i) = sqrt(arnoldi_norm->at(1, i));
 
@@ -347,17 +353,23 @@ void initialize_2(std::shared_ptr<const ReferenceExecutor> exec,
                      typename Accessor3d::accessor::arithmetic_type>::value,
         "ValueType must match arithmetic_type of accessor!");
     using rc_vtype = remove_complex<ValueType>;
+    constexpr bool has_scalar =
+        gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value;
 
     for (size_type j = 0; j < residual->get_size()[1]; ++j) {
         // Calculate residual norm
         residual_norm->at(0, j) = zero<rc_vtype>();
-        arnoldi_norm->at(2, j) = zero<rc_vtype>();
+        if (has_scalar) {
+            arnoldi_norm->at(2, j) = zero<rc_vtype>();
+        }
         for (size_type i = 0; i < residual->get_size()[0]; ++i) {
             residual_norm->at(0, j) += squared_norm(residual->at(i, j));
-            arnoldi_norm->at(2, j) =
-                (arnoldi_norm->at(2, j) >= abs(residual->at(i, j)))
-                    ? arnoldi_norm->at(2, j)
-                    : abs(residual->at(i, j));
+            if (has_scalar) {
+                arnoldi_norm->at(2, j) =
+                    (arnoldi_norm->at(2, j) >= abs(residual->at(i, j)))
+                        ? arnoldi_norm->at(2, j)
+                        : abs(residual->at(i, j));
+            }
         }
         residual_norm->at(0, j) = sqrt(residual_norm->at(0, j));
         gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(

From 24bf3965676c987e7b3b2543e273626cd41873ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 4 Feb 2021 09:14:03 +0100
Subject: [PATCH 0409/1008] Remove CUDA 9.0 exception for constexpr parameter

`--expt-relaxed-constexpr` is now used for every CUDA version
---
 cuda/CMakeLists.txt | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 883525061f7..b773f4bf039 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -121,11 +121,9 @@ target_sources(ginkgo_cuda
     stop/criterion_kernels.cu
     stop/residual_norm_kernels.cu)
 
-# This creates a compilation bug on nvcc 9.0.102 *with* the new array_deleter
-# merged at commit ed12b3df5d26, and the parameter is not recognized by clang-cuda
-if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA" AND
-   NOT CMAKE_CUDA_COMPILER_VERSION MATCHES "9.0")
+if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
     # remove false positive CUDA warnings when calling one<T>() and zero<T>()
+    # and allows the usage of std::array for nvidia GPUs
     target_compile_options(ginkgo_cuda
         PRIVATE
             $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>)

From 6902affe760e2971aeaab7d25ac130f91914a056 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 5 Feb 2021 17:38:10 +0100
Subject: [PATCH 0410/1008] Review Update

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 BENCHMARKING.md                        | 6 +++---
 benchmark/run_all_benchmarks.sh        | 2 +-
 common/components/atomic.hpp.inc       | 2 +-
 common/solver/cb_gmres_kernels.hpp.inc | 2 --
 4 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/BENCHMARKING.md b/BENCHMARKING.md
index 8dd73bbf5ee..bc419f57386 100644
--- a/BENCHMARKING.md
+++ b/BENCHMARKING.md
@@ -297,7 +297,7 @@ The supported environment variables are described in the following list:
     `csr,coo,ell,hybrid,sellp`.
 * `SOLVERS={bicgstab,bicg,cg,cgs,fcg,gmres,cb_gmres_{keep,reduce1,reduce2,integer,ireduce1,ireduce2},lower_trs,upper_trs}`
     - the solvers which should be benchmarked. Multiple options can be passed.
-    The default is `bicgstab,cg,cgs.fcg.gmres,idr`. Note that `lower/upper_trs`
+    The default is `bicgstab,cg,cgs,fcg,gmres,idr`. Note that `lower/upper_trs`
     by default don't use a preconditioner, as they are by default exact direct
     solvers.
 * `SOLVERS_PRECISION=<precision>` - the minimal residual reduction before which
@@ -305,8 +305,8 @@ The supported environment variables are described in the following list:
 * `SOLVERS_MAX_ITERATION=<number>` - the maximum number of iterations with which
     a solver should be ran. The default is `10000`.
 * `SOLVERS_RHS={1, random, sinus}` - whether to use a vector of all ones,
-    random values or $b = A * (s / |s|)$ with $s(idx) = sin(idx)$ (for complex
-    numbers, $s(idx) = sin(2*idx) + i * sin(2*idx+1)$)
+    random values or b = A * (s / |s|)$ with s(idx) = sin(idx) (for complex
+    numbers, s(idx) = sin(2*idx) + i * sin(2*idx+1))
     as the right-hand side in solver benchmarks. Default is `1`.
 * `SOLVERS_INITIAL_GUESS={rhs,0,random}` - the initial guess generation of the
     solvers. `rhs` uses the right-hand side, `0` uses a zero vector and `random`
diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index 422b28c68f1..be2d09229dd 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -37,7 +37,7 @@ fi
 
 if [ ! "${SOLVERS}" ]; then
     echo "SOLVERS    environment variable not set - assuming \"bicgstab,cg,cgs,fcg,gmres,idr\"" 1>&2
-    SOLVERS="bicgstab,cg,cgs,fcg,gmres,idr"
+    SOLVERS="bicgstab,cg,cgs,fcg,gmres,cb_gmres_reduce1,idr"
 fi
 
 if [ ! "${SOLVERS_PRECISION}" ]; then
diff --git a/common/components/atomic.hpp.inc b/common/components/atomic.hpp.inc
index a99c07dc654..a1c514f8a55 100644
--- a/common/components/atomic.hpp.inc
+++ b/common/components/atomic.hpp.inc
@@ -109,7 +109,7 @@ __forceinline__ __device__ ResultType reinterpret(ValueType val)
             } while (assumed != old);                                        \
             return reinterpret<ValueType>(old);                              \
         }                                                                    \
-    };  // namespace detail
+    };
 
 // Support 64-bit ATOMIC_ADD and ATOMIC_MAX
 GKO_BIND_ATOMIC_HELPER_STRUCTURE(unsigned long long int);
diff --git a/common/solver/cb_gmres_kernels.hpp.inc b/common/solver/cb_gmres_kernels.hpp.inc
index 59d82cb53b9..0f4af782e2b 100644
--- a/common/solver/cb_gmres_kernels.hpp.inc
+++ b/common/solver/cb_gmres_kernels.hpp.inc
@@ -481,8 +481,6 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_and_add_kernel(
         const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
         auto local_res = next_krylov_basis[next_krylov_idx];
         for (size_type k = 0; k < num_iters; ++k) {
-            // const auto krylov_idx =
-            //    row_idx * stride_krylov + k * num_cols + col_idx;
             const auto hessenberg_idx = k * stride_hessenberg + col_idx;
             const auto buffer_idx = k * stride_buffer + col_idx;
             local_res -=

From 2d056545e39bd0158123610acc18c9f2e8d7c34d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Sat, 6 Feb 2021 18:31:34 +0100
Subject: [PATCH 0411/1008] Sonarcloud update

The following modifications were done for CB-GMRES:
- Remove unused kernel parameters `num_reorth_steps` and
  `num_reorth_vectors`
- Remove unused `b_norm`
- Make unused kernel parameters unnamed
- Add some explicit casts to prevent warning
---
 benchmark/solver/solver.cpp            |  5 +---
 common/solver/cb_gmres_kernels.hpp.inc |  8 +++----
 core/solver/cb_gmres.cpp               | 15 ++++--------
 core/solver/cb_gmres_accessor.hpp      | 10 ++++----
 core/solver/cb_gmres_kernels.hpp       | 23 ++++++------------
 cuda/solver/cb_gmres_kernels.cu        | 19 ++++-----------
 cuda/test/solver/cb_gmres_kernels.cpp  | 23 ++++++------------
 dpcpp/solver/cb_gmres_kernels.dp.cpp   |  6 ++---
 hip/solver/cb_gmres_kernels.hip.cpp    | 19 ++++-----------
 hip/test/solver/cb_gmres_kernels.cpp   | 23 ++++++------------
 omp/solver/cb_gmres_kernels.cpp        | 33 +++++++-------------------
 omp/test/solver/cb_gmres_kernels.cpp   | 23 ++++++------------
 reference/solver/cb_gmres_kernels.cpp  | 24 ++++++-------------
 13 files changed, 67 insertions(+), 164 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 35fa014dd85..bc4dc2504ad 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -134,7 +134,6 @@ std::unique_ptr<vec<etype>> generate_rhs(
     std::shared_ptr<const gko::Executor> exec,
     std::shared_ptr<const gko::LinOp> system_matrix, Engine engine)
 {
-    using rc_etype = gko::remove_complex<etype>;
     gko::dim<2> vec_size{system_matrix->get_size()[0], FLAGS_nrhs};
     if (FLAGS_rhs_generation == "1") {
         return create_matrix<etype>(exec, vec_size, gko::one<etype>());
@@ -170,7 +169,6 @@ std::unique_ptr<vec<etype>> generate_initial_guess(
     std::shared_ptr<const gko::LinOp> system_matrix, const vec<etype> *rhs,
     Engine engine)
 {
-    using rc_etype = gko::remove_complex<etype>;
     gko::dim<2> vec_size{system_matrix->get_size()[1], FLAGS_nrhs};
     if (FLAGS_initial_guess_generation == "0") {
         return create_matrix<etype>(exec, vec_size, gko::zero<etype>());
@@ -223,8 +221,7 @@ std::shared_ptr<const gko::stop::CriterionFactory> create_criterion(
 
 template <typename SolverIntermediate>
 std::unique_ptr<gko::LinOpFactory> add_criteria_precond_finalize(
-    SolverIntermediate &&inter,
-    const std::shared_ptr<const gko::Executor> &exec,
+    SolverIntermediate inter, const std::shared_ptr<const gko::Executor> &exec,
     std::shared_ptr<const gko::LinOpFactory> precond)
 {
     return inter.with_criteria(create_criterion(exec))
diff --git a/common/solver/cb_gmres_kernels.hpp.inc b/common/solver/cb_gmres_kernels.hpp.inc
index 0f4af782e2b..946d8aeac80 100644
--- a/common/solver/cb_gmres_kernels.hpp.inc
+++ b/common/solver/cb_gmres_kernels.hpp.inc
@@ -625,8 +625,7 @@ __device__ void calculate_residual_norm_kernel(
     const ValueType *givens_cos, size_type stride_cos,
     remove_complex<ValueType> *residual_norm,
     ValueType *residual_norm_collection,
-    size_type stride_residual_norm_collection,
-    const remove_complex<ValueType> *b_norm)
+    size_type stride_residual_norm_collection)
 {
     residual_norm_collection[(iter + 1) * stride_residual_norm_collection +
                              col_idx] =
@@ -653,7 +652,6 @@ __global__ __launch_bounds__(block_size) void givens_rotation_kernel(
     remove_complex<ValueType> *__restrict__ residual_norm,
     ValueType *__restrict__ residual_norm_collection,
     size_type stride_residual_norm_collection,
-    const remove_complex<ValueType> *__restrict__ b_norm,
     const stopping_status *__restrict__ stop_status)
 {
     const auto col_idx = thread::get_thread_id_flat();
@@ -706,7 +704,7 @@ __global__ __launch_bounds__(block_size) void givens_rotation_kernel(
     calculate_residual_norm_kernel(col_idx, num_cols, iter, givens_sin,
                                    stride_sin, givens_cos, stride_cos,
                                    residual_norm, residual_norm_collection,
-                                   stride_residual_norm_collection, b_norm);
+                                   stride_residual_norm_collection);
     // Calculate residual norm
 }
 
@@ -727,7 +725,7 @@ __global__ __launch_bounds__(block_size) void solve_upper_triangular_kernel(
         return;
     }
 
-    for (int i = final_iter_nums[col_idx] - 1; i >= 0; --i) {
+    for (int64 i = final_iter_nums[col_idx] - 1; i >= 0; --i) {
         auto temp =
             residual_norm_collection[i * stride_residual_norm_collection +
                                      col_idx];
diff --git a/core/solver/cb_gmres.cpp b/core/solver/cb_gmres.cpp
index d08d63d496e..f3a3db1c622 100644
--- a/core/solver/cb_gmres.cpp
+++ b/core/solver/cb_gmres.cpp
@@ -194,7 +194,6 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
         using Vector = matrix::Dense<ValueType>;
         using VectorNorms = matrix::Dense<remove_complex<ValueType>>;
-        using LowArray = Array<storage_type>;
         using Range3dHelper =
             gko::cb_gmres::Range3dHelper<ValueType, storage_type>;
 
@@ -237,8 +236,6 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             exec, dim<2>{krylov_dim_ + 1, dense_b->get_size()[1]});
         auto residual_norm =
             VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
-        auto b_norm =
-            VectorNorms::create(exec, dim<2>{1, dense_b->get_size()[1]});
         // 1st row of arnoldi_norm: == eta * norm2(old_next_krylov_basis)
         //                          with eta == 1 / sqrt(2)
         //                          (computed right before updating
@@ -261,13 +258,11 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                                              dense_b->get_size()[1]);
         Array<size_type> num_reorth(this->get_executor(),
                                     dense_b->get_size()[1]);
-        int num_restarts = 0, num_reorth_steps = 0, num_reorth_vectors = 0;
 
         // Initialization
         exec->run(cb_gmres::make_initialize_1(
-            dense_b, b_norm.get(), residual.get(), givens_sin.get(),
-            givens_cos.get(), &stop_status, krylov_dim_));
-        // b_norm = norm(b)
+            dense_b, residual.get(), givens_sin.get(), givens_cos.get(),
+            &stop_status, krylov_dim_));
         // residual = dense_b
         // givens_sin = givens_cos = 0
         system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(),
@@ -380,7 +375,6 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
             if (perform_reset || restart_iter == krylov_dim_) {
                 perform_reset = false;
-                num_restarts++;
                 // Restart
                 // use a view in case this is called earlier
                 auto hessenberg_view = hessenberg->create_submatrix(
@@ -439,9 +433,8 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                 next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
                 residual_norm.get(), residual_norm_collection.get(),
                 krylov_bases_range, hessenberg_iter.get(), buffer_iter.get(),
-                b_norm.get(), arnoldi_norm.get(), restart_iter,
-                &final_iter_nums, &stop_status, &reorth_status, &num_reorth,
-                &num_reorth_steps, &num_reorth_vectors));
+                arnoldi_norm.get(), restart_iter, &final_iter_nums,
+                &stop_status, &reorth_status, &num_reorth));
             // for i in 0:restart_iter
             //     hessenberg(restart_iter, i) = next_krylov_basis' *
             //     krylov_bases(:, i) next_krylov_basis  -=
diff --git a/core/solver/cb_gmres_accessor.hpp b/core/solver/cb_gmres_accessor.hpp
index e5727b1036e..87c77cea276 100644
--- a/core/solver/cb_gmres_accessor.hpp
+++ b/core/solver/cb_gmres_accessor.hpp
@@ -190,11 +190,11 @@ struct helper_functions_accessor<Accessor3d, false> {
                   "Accessor must not have a scale here!");
 
     template <typename IndexType>
-    static inline GKO_ATTRIBUTES void write_scalar(Accessor3d krylov_bases,
-                                                   IndexType vector_idx,
-                                                   IndexType col_idx,
-                                                   arithmetic_type value)
-    {}
+    static inline GKO_ATTRIBUTES void write_scalar(Accessor3d, IndexType,
+                                                   IndexType, arithmetic_type)
+    {
+        // Since there is no scalar, there is nothing to write.
+    }
 
     static constexpr GKO_ATTRIBUTES std::array<size_type, dimensionality - 1>
     get_stride(Accessor3d krylov_bases)
diff --git a/core/solver/cb_gmres_kernels.hpp b/core/solver/cb_gmres_kernels.hpp
index 04756ef196d..305c2f84c61 100644
--- a/core/solver/cb_gmres_kernels.hpp
+++ b/core/solver/cb_gmres_kernels.hpp
@@ -123,17 +123,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 namespace gko {
 namespace kernels {
-namespace cb_gmres {
 
 
-#define GKO_DECLARE_CB_GMRES_INITIALIZE_1_KERNEL(_type)                        \
-    void initialize_1(                                                         \
-        std::shared_ptr<const DefaultExecutor> exec,                           \
-        const matrix::Dense<_type> *b,                                         \
-        matrix::Dense<remove_complex<_type>> *b_norm,                          \
-        matrix::Dense<_type> *residual, matrix::Dense<_type> *givens_sin,      \
-        matrix::Dense<_type> *givens_cos, Array<stopping_status> *stop_status, \
-        size_type krylov_dim)
+#define GKO_DECLARE_CB_GMRES_INITIALIZE_1_KERNEL(_type)                     \
+    void initialize_1(                                                      \
+        std::shared_ptr<const DefaultExecutor> exec,                        \
+        const matrix::Dense<_type> *b, matrix::Dense<_type> *residual,      \
+        matrix::Dense<_type> *givens_sin, matrix::Dense<_type> *givens_cos, \
+        Array<stopping_status> *stop_status, size_type krylov_dim)
 
 
 #define GKO_DECLARE_CB_GMRES_INITIALIZE_2_KERNEL(_type1, _range)            \
@@ -156,13 +153,10 @@ namespace cb_gmres {
         matrix::Dense<_type1> *residual_norm_collection, _range krylov_bases, \
         matrix::Dense<_type1> *hessenberg_iter,                               \
         matrix::Dense<_type1> *buffer_iter,                                   \
-        const matrix::Dense<remove_complex<_type1>> *b_norm,                  \
         matrix::Dense<remove_complex<_type1>> *arnoldi_norm, size_type iter,  \
         Array<size_type> *final_iter_nums,                                    \
         const Array<stopping_status> *stop_status,                            \
-        Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,  \
-        int *num_reorth_steps, int *num_reorth_vectors)
-
+        Array<stopping_status> *reorth_status, Array<size_type> *num_reorth)
 
 #define GKO_DECLARE_CB_GMRES_STEP_2_KERNEL(_type1, _range)                    \
     void step_2(std::shared_ptr<const DefaultExecutor> exec,                  \
@@ -184,9 +178,6 @@ namespace cb_gmres {
     GKO_DECLARE_CB_GMRES_STEP_2_KERNEL(ValueType, Accessor3d)
 
 
-}  // namespace cb_gmres
-
-
 namespace omp {
 namespace cb_gmres {
 
diff --git a/cuda/solver/cb_gmres_kernels.cu b/cuda/solver/cb_gmres_kernels.cu
index 25a652d6211..bc8a20e54ae 100644
--- a/cuda/solver/cb_gmres_kernels.cu
+++ b/cuda/solver/cb_gmres_kernels.cu
@@ -114,7 +114,6 @@ void zero_matrix(size_type m, size_type n, size_type stride, ValueType *array)
 template <typename ValueType>
 void initialize_1(std::shared_ptr<const CudaExecutor> exec,
                   const matrix::Dense<ValueType> *b,
-                  matrix::Dense<remove_complex<ValueType>> *b_norm,
                   matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *givens_sin,
                   matrix::Dense<ValueType> *givens_cos,
@@ -126,7 +125,6 @@ void initialize_1(std::shared_ptr<const CudaExecutor> exec,
     const dim3 block_dim(default_block_size, 1, 1);
     constexpr auto block_size = default_block_size;
 
-    kernels::cuda::dense::compute_norm2(exec, b, b_norm);
     initialize_1_kernel<block_size><<<grid_dim, block_dim>>>(
         b->get_size()[0], b->get_size()[1], krylov_dim,
         as_cuda_type(b->get_const_values()), b->get_stride(),
@@ -220,8 +218,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                          matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
                          size_type iter, const stopping_status *stop_status,
                          stopping_status *reorth_status,
-                         Array<size_type> *num_reorth, int *num_reorth_steps,
-                         int *num_reorth_vectors)
+                         Array<size_type> *num_reorth)
 {
     using non_complex = remove_complex<ValueType>;
     // optimization parameter
@@ -315,8 +312,6 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                                   &numReorth);
     // numReorth <= number of next_krylov vector to be reorthogonalization
     for (size_type l = 1; (numReorth > 0) && (l < 3); l++) {
-        (*num_reorth_steps)++;
-        (*num_reorth_vectors) += iter;
         zero_matrix(iter + 1, dim_size[1], stride_buffer,
                     buffer_iter->get_values());
         if (dim_size[1] > 1) {
@@ -403,7 +398,6 @@ void givens_rotation(std::shared_ptr<const CudaExecutor> exec,
                      matrix::Dense<ValueType> *hessenberg_iter,
                      matrix::Dense<remove_complex<ValueType>> *residual_norm,
                      matrix::Dense<ValueType> *residual_norm_collection,
-                     const matrix::Dense<remove_complex<ValueType>> *b_norm,
                      size_type iter, const Array<stopping_status> *stop_status)
 {
     // TODO: tune block_size for optimal performance
@@ -421,7 +415,6 @@ void givens_rotation(std::shared_ptr<const CudaExecutor> exec,
         givens_cos->get_stride(), as_cuda_type(residual_norm->get_values()),
         as_cuda_type(residual_norm_collection->get_values()),
         residual_norm_collection->get_stride(),
-        as_cuda_type(b_norm->get_const_values()),
         as_cuda_type(stop_status->get_const_data()));
 }
 
@@ -435,12 +428,10 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
             matrix::Dense<ValueType> *residual_norm_collection,
             Accessor3d krylov_bases, matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
-            const matrix::Dense<remove_complex<ValueType>> *b_norm,
             matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
             size_type iter, Array<size_type> *final_iter_nums,
             const Array<stopping_status> *stop_status,
-            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
-            int *num_reorth_steps, int *num_reorth_vectors)
+            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth)
 {
     increase_final_iteration_numbers_kernel<<<
         static_cast<unsigned int>(
@@ -451,11 +442,9 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
     finish_arnoldi_CGS2(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
                         buffer_iter, arnoldi_norm, iter,
                         stop_status->get_const_data(),
-                        reorth_status->get_data(), num_reorth, num_reorth_steps,
-                        num_reorth_vectors);
+                        reorth_status->get_data(), num_reorth);
     givens_rotation(exec, givens_sin, givens_cos, hessenberg_iter,
-                    residual_norm, residual_norm_collection, b_norm, iter,
-                    stop_status);
+                    residual_norm, residual_norm_collection, iter, stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(GKO_DECLARE_CB_GMRES_STEP_1_KERNEL);
diff --git a/cuda/test/solver/cb_gmres_kernels.cpp b/cuda/test/solver/cb_gmres_kernels.cpp
index d0e0862f048..0a3869b9666 100644
--- a/cuda/test/solver/cb_gmres_kernels.cpp
+++ b/cuda/test/solver/cb_gmres_kernels.cpp
@@ -125,7 +125,6 @@ class CbGmres : public ::testing::Test {
         y = gen_mtx(default_krylov_dim_mixed, n);
         before_preconditioner = Mtx::create_with_config_of(x.get());
         b = gen_mtx(m, n);
-        b_norm = gen_mtx(1, n);
         arnoldi_norm = gen_mtx(3, n);
         gko::dim<3> krylov_bases_dim(default_krylov_dim_mixed + 1, m, n);
         range_helper = generate_krylov_helper(krylov_bases_dim);
@@ -168,8 +167,6 @@ class CbGmres : public ::testing::Test {
         d_y->copy_from(y.get());
         d_b = Mtx::create(cuda);
         d_b->copy_from(b.get());
-        d_b_norm = Mtx::create(cuda);
-        d_b_norm->copy_from(b_norm.get());
         d_arnoldi_norm = Mtx::create(cuda);
         d_arnoldi_norm->copy_from(arnoldi_norm.get());
         d_range_helper = Range3dHelper{cuda, {}};
@@ -229,7 +226,6 @@ class CbGmres : public ::testing::Test {
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> y;
     std::unique_ptr<Mtx> b;
-    std::unique_ptr<Mtx> b_norm;
     std::unique_ptr<Mtx> arnoldi_norm;
     Range3dHelper range_helper;
     std::unique_ptr<Mtx> next_krylov_basis;
@@ -250,7 +246,6 @@ class CbGmres : public ::testing::Test {
     std::unique_ptr<Mtx> d_before_preconditioner;
     std::unique_ptr<Mtx> d_y;
     std::unique_ptr<Mtx> d_b;
-    std::unique_ptr<Mtx> d_b_norm;
     std::unique_ptr<Mtx> d_arnoldi_norm;
     Range3dHelper d_range_helper;
     std::unique_ptr<Mtx> d_next_krylov_basis;
@@ -274,13 +269,12 @@ TEST_F(CbGmres, CudaCbGmresInitialize1IsEquivalentToRef)
     initialize_data();
 
     gko::kernels::reference::cb_gmres::initialize_1(
-        ref, b.get(), b_norm.get(), residual.get(), givens_sin.get(),
-        givens_cos.get(), stop_status.get(), default_krylov_dim_mixed);
+        ref, b.get(), residual.get(), givens_sin.get(), givens_cos.get(),
+        stop_status.get(), default_krylov_dim_mixed);
     gko::kernels::cuda::cb_gmres::initialize_1(
-        cuda, d_b.get(), d_b_norm.get(), d_residual.get(), d_givens_sin.get(),
+        cuda, d_b.get(), d_residual.get(), d_givens_sin.get(),
         d_givens_cos.get(), d_stop_status.get(), default_krylov_dim_mixed);
 
-    GKO_ASSERT_MTX_NEAR(d_b_norm, b_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual, residual, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
@@ -316,23 +310,20 @@ TEST_F(CbGmres, CudaCbGmresStep1IsEquivalentToRef)
 {
     initialize_data();
     int iter = 5;
-    int num_reorth_steps = 0, num_reorth_vectors = 0;
-    int d_num_reorth_steps = 0, d_num_reorth_vectors = 0;
 
     gko::kernels::reference::cb_gmres::step_1(
         ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
         residual_norm.get(), residual_norm_collection.get(),
         range_helper.get_range(), hessenberg_iter.get(), buffer_iter.get(),
-        b_norm.get(), arnoldi_norm.get(), iter, final_iter_nums.get(),
-        stop_status.get(), reorth_status.get(), num_reorth.get(),
-        &num_reorth_steps, &num_reorth_vectors);
+        arnoldi_norm.get(), iter, final_iter_nums.get(), stop_status.get(),
+        reorth_status.get(), num_reorth.get());
     gko::kernels::cuda::cb_gmres::step_1(
         cuda, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
         d_residual_norm.get(), d_residual_norm_collection.get(),
         d_range_helper.get_range(), d_hessenberg_iter.get(),
-        d_buffer_iter.get(), d_b_norm.get(), d_arnoldi_norm.get(), iter,
+        d_buffer_iter.get(), d_arnoldi_norm.get(), iter,
         d_final_iter_nums.get(), d_stop_status.get(), d_reorth_status.get(),
-        d_num_reorth.get(), &num_reorth_steps, &num_reorth_vectors);
+        d_num_reorth.get());
 
     GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
diff --git a/dpcpp/solver/cb_gmres_kernels.dp.cpp b/dpcpp/solver/cb_gmres_kernels.dp.cpp
index f051fc13d9e..f6d5345d3a4 100644
--- a/dpcpp/solver/cb_gmres_kernels.dp.cpp
+++ b/dpcpp/solver/cb_gmres_kernels.dp.cpp
@@ -57,7 +57,6 @@ namespace cb_gmres {
 template <typename ValueType>
 void initialize_1(std::shared_ptr<const DpcppExecutor> exec,
                   const matrix::Dense<ValueType> *b,
-                  matrix::Dense<remove_complex<ValueType>> *b_norm,
                   matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *givens_sin,
                   matrix::Dense<ValueType> *givens_cos,
@@ -91,12 +90,11 @@ void step_1(std::shared_ptr<const DpcppExecutor> exec,
             matrix::Dense<ValueType> *residual_norm_collection,
             Accessor3d krylov_bases, matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
-            const matrix::Dense<remove_complex<ValueType>> *b_norm,
             matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
             size_type iter, Array<size_type> *final_iter_nums,
             const Array<stopping_status> *stop_status,
-            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
-            int *num_reorth_steps, int *num_reorth_vectors) GKO_NOT_IMPLEMENTED;
+            Array<stopping_status> *reorth_status,
+            Array<size_type> *num_reorth) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(GKO_DECLARE_CB_GMRES_STEP_1_KERNEL);
 
diff --git a/hip/solver/cb_gmres_kernels.hip.cpp b/hip/solver/cb_gmres_kernels.hip.cpp
index cce0851b853..e54652a2d82 100644
--- a/hip/solver/cb_gmres_kernels.hip.cpp
+++ b/hip/solver/cb_gmres_kernels.hip.cpp
@@ -115,7 +115,6 @@ void zero_matrix(size_type m, size_type n, size_type stride, ValueType *array)
 template <typename ValueType>
 void initialize_1(std::shared_ptr<const HipExecutor> exec,
                   const matrix::Dense<ValueType> *b,
-                  matrix::Dense<remove_complex<ValueType>> *b_norm,
                   matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *givens_sin,
                   matrix::Dense<ValueType> *givens_cos,
@@ -127,7 +126,6 @@ void initialize_1(std::shared_ptr<const HipExecutor> exec,
     const dim3 block_dim(default_block_size, 1, 1);
     constexpr auto block_size = default_block_size;
 
-    kernels::hip::dense::compute_norm2(exec, b, b_norm);
     hipLaunchKernelGGL(
         initialize_1_kernel<block_size>, grid_dim, block_dim, 0, 0,
         b->get_size()[0], b->get_size()[1], krylov_dim,
@@ -223,8 +221,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
                          matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
                          size_type iter, const stopping_status *stop_status,
                          stopping_status *reorth_status,
-                         Array<size_type> *num_reorth, int *num_reorth_steps,
-                         int *num_reorth_vectors)
+                         Array<size_type> *num_reorth)
 {
     using non_complex = remove_complex<ValueType>;
     // optimization parameter
@@ -324,8 +321,6 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
                                   &numReorth);
     // numReorth <= number of next_krylov vector to be reorthogonalization
     for (size_type l = 1; (numReorth > 0) && (l < 3); l++) {
-        (*num_reorth_steps)++;
-        (*num_reorth_vectors) += iter;
         zero_matrix(iter + 1, dim_size[1], stride_buffer,
                     buffer_iter->get_values());
         if (dim_size[1] > 1) {
@@ -414,7 +409,6 @@ void givens_rotation(std::shared_ptr<const HipExecutor> exec,
                      matrix::Dense<ValueType> *hessenberg_iter,
                      matrix::Dense<remove_complex<ValueType>> *residual_norm,
                      matrix::Dense<ValueType> *residual_norm_collection,
-                     const matrix::Dense<remove_complex<ValueType>> *b_norm,
                      size_type iter, const Array<stopping_status> *stop_status)
 {
     // TODO: tune block_size for optimal performance
@@ -433,7 +427,6 @@ void givens_rotation(std::shared_ptr<const HipExecutor> exec,
         givens_cos->get_stride(), as_hip_type(residual_norm->get_values()),
         as_hip_type(residual_norm_collection->get_values()),
         residual_norm_collection->get_stride(),
-        as_hip_type(b_norm->get_const_values()),
         as_hip_type(stop_status->get_const_data()));
 }
 
@@ -447,12 +440,10 @@ void step_1(std::shared_ptr<const HipExecutor> exec,
             matrix::Dense<ValueType> *residual_norm_collection,
             Accessor3d krylov_bases, matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
-            const matrix::Dense<remove_complex<ValueType>> *b_norm,
             matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
             size_type iter, Array<size_type> *final_iter_nums,
             const Array<stopping_status> *stop_status,
-            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
-            int *num_reorth_steps, int *num_reorth_vectors)
+            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth)
 {
     hipLaunchKernelGGL(
         increase_final_iteration_numbers_kernel,
@@ -465,11 +456,9 @@ void step_1(std::shared_ptr<const HipExecutor> exec,
     finish_arnoldi_CGS2(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
                         buffer_iter, arnoldi_norm, iter,
                         stop_status->get_const_data(),
-                        reorth_status->get_data(), num_reorth, num_reorth_steps,
-                        num_reorth_vectors);
+                        reorth_status->get_data(), num_reorth);
     givens_rotation(exec, givens_sin, givens_cos, hessenberg_iter,
-                    residual_norm, residual_norm_collection, b_norm, iter,
-                    stop_status);
+                    residual_norm, residual_norm_collection, iter, stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(GKO_DECLARE_CB_GMRES_STEP_1_KERNEL);
diff --git a/hip/test/solver/cb_gmres_kernels.cpp b/hip/test/solver/cb_gmres_kernels.cpp
index 15b220b5bfe..5ba2743a289 100644
--- a/hip/test/solver/cb_gmres_kernels.cpp
+++ b/hip/test/solver/cb_gmres_kernels.cpp
@@ -125,7 +125,6 @@ class CbGmres : public ::testing::Test {
         y = gen_mtx(default_krylov_dim_mixed, n);
         before_preconditioner = Mtx::create_with_config_of(x.get());
         b = gen_mtx(m, n);
-        b_norm = gen_mtx(1, n);
         arnoldi_norm = gen_mtx(3, n);
         gko::dim<3> krylov_bases_dim(default_krylov_dim_mixed + 1, m, n);
         range_helper = generate_krylov_helper(krylov_bases_dim);
@@ -168,8 +167,6 @@ class CbGmres : public ::testing::Test {
         d_y->copy_from(y.get());
         d_b = Mtx::create(hip);
         d_b->copy_from(b.get());
-        d_b_norm = Mtx::create(hip);
-        d_b_norm->copy_from(b_norm.get());
         d_arnoldi_norm = Mtx::create(hip);
         d_arnoldi_norm->copy_from(arnoldi_norm.get());
         d_range_helper = Range3dHelper{hip, {}};
@@ -229,7 +226,6 @@ class CbGmres : public ::testing::Test {
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> y;
     std::unique_ptr<Mtx> b;
-    std::unique_ptr<Mtx> b_norm;
     std::unique_ptr<Mtx> arnoldi_norm;
     Range3dHelper range_helper;
     std::unique_ptr<Mtx> next_krylov_basis;
@@ -250,7 +246,6 @@ class CbGmres : public ::testing::Test {
     std::unique_ptr<Mtx> d_before_preconditioner;
     std::unique_ptr<Mtx> d_y;
     std::unique_ptr<Mtx> d_b;
-    std::unique_ptr<Mtx> d_b_norm;
     std::unique_ptr<Mtx> d_arnoldi_norm;
     Range3dHelper d_range_helper;
     std::unique_ptr<Mtx> d_next_krylov_basis;
@@ -274,13 +269,12 @@ TEST_F(CbGmres, HipCbGmresInitialize1IsEquivalentToRef)
     initialize_data();
 
     gko::kernels::reference::cb_gmres::initialize_1(
-        ref, b.get(), b_norm.get(), residual.get(), givens_sin.get(),
-        givens_cos.get(), stop_status.get(), default_krylov_dim_mixed);
+        ref, b.get(), residual.get(), givens_sin.get(), givens_cos.get(),
+        stop_status.get(), default_krylov_dim_mixed);
     gko::kernels::hip::cb_gmres::initialize_1(
-        hip, d_b.get(), d_b_norm.get(), d_residual.get(), d_givens_sin.get(),
+        hip, d_b.get(), d_residual.get(), d_givens_sin.get(),
         d_givens_cos.get(), d_stop_status.get(), default_krylov_dim_mixed);
 
-    GKO_ASSERT_MTX_NEAR(d_b_norm, b_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual, residual, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
@@ -316,23 +310,20 @@ TEST_F(CbGmres, HipCbGmresStep1IsEquivalentToRef)
 {
     initialize_data();
     int iter = 5;
-    int num_reorth_steps = 0, num_reorth_vectors = 0;
-    int d_num_reorth_steps = 0, d_num_reorth_vectors = 0;
 
     gko::kernels::reference::cb_gmres::step_1(
         ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
         residual_norm.get(), residual_norm_collection.get(),
         range_helper.get_range(), hessenberg_iter.get(), buffer_iter.get(),
-        b_norm.get(), arnoldi_norm.get(), iter, final_iter_nums.get(),
-        stop_status.get(), reorth_status.get(), num_reorth.get(),
-        &num_reorth_steps, &num_reorth_vectors);
+        arnoldi_norm.get(), iter, final_iter_nums.get(), stop_status.get(),
+        reorth_status.get(), num_reorth.get());
     gko::kernels::hip::cb_gmres::step_1(
         hip, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
         d_residual_norm.get(), d_residual_norm_collection.get(),
         d_range_helper.get_range(), d_hessenberg_iter.get(),
-        d_buffer_iter.get(), d_b_norm.get(), d_arnoldi_norm.get(), iter,
+        d_buffer_iter.get(), d_arnoldi_norm.get(), iter,
         d_final_iter_nums.get(), d_stop_status.get(), d_reorth_status.get(),
-        d_num_reorth.get(), &num_reorth_steps, &num_reorth_vectors);
+        d_num_reorth.get());
 
     GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
diff --git a/omp/solver/cb_gmres_kernels.cpp b/omp/solver/cb_gmres_kernels.cpp
index 6f7d22917d0..2bea7acc2c0 100644
--- a/omp/solver/cb_gmres_kernels.cpp
+++ b/omp/solver/cb_gmres_kernels.cpp
@@ -152,8 +152,8 @@ void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
             // for i in 1:iter
             //     next_krylov_basis  -= buffer(iter, i) * krylov_bases(:, i)
             // end
-            auto nrm = zero<rc_vtype>();
-            auto inf = zero<rc_vtype>();
+            nrm = zero<rc_vtype>();
+            inf = zero<rc_vtype>();
 #pragma omp parallel for reduction(addnc : nrm) reduction(infnc : inf)
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 nrm += squared_norm(next_krylov_basis->at(j, i));
@@ -252,8 +252,7 @@ template <typename ValueType>
 void calculate_next_residual_norm(
     matrix::Dense<ValueType> *givens_sin, matrix::Dense<ValueType> *givens_cos,
     matrix::Dense<remove_complex<ValueType>> *residual_norm,
-    matrix::Dense<ValueType> *residual_norm_collection,
-    const matrix::Dense<remove_complex<ValueType>> *b_norm, size_type iter,
+    matrix::Dense<ValueType> *residual_norm_collection, size_type iter,
     const stopping_status *stop_status)
 {
 #pragma omp parallel for
@@ -280,7 +279,7 @@ void solve_upper_triangular(
 {
 #pragma omp parallel for
     for (size_type k = 0; k < residual_norm_collection->get_size()[1]; ++k) {
-        for (int i = final_iter_nums[k] - 1; i >= 0; --i) {
+        for (int64 i = final_iter_nums[k] - 1; i >= 0; --i) {
             auto temp = residual_norm_collection->at(i, k);
             for (size_type j = i + 1; j < final_iter_nums[k]; ++j) {
                 temp -=
@@ -321,7 +320,6 @@ void calculate_qy(ConstAccessor3d krylov_bases,
 template <typename ValueType>
 void initialize_1(std::shared_ptr<const OmpExecutor> exec,
                   const matrix::Dense<ValueType> *b,
-                  matrix::Dense<remove_complex<ValueType>> *b_norm,
                   matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *givens_sin,
                   matrix::Dense<ValueType> *givens_cos,
@@ -330,18 +328,6 @@ void initialize_1(std::shared_ptr<const OmpExecutor> exec,
     using rc_vtype = remove_complex<ValueType>;
 
     for (size_type j = 0; j < b->get_size()[1]; ++j) {
-        // Calculate b norm
-        auto norm = zero<rc_vtype>();
-
-#pragma omp declare reduction(addnc:rc_vtype : omp_out = omp_out + omp_in)
-
-#pragma omp parallel for reduction(addnc : norm)
-        for (size_type i = 0; i < b->get_size()[0]; ++i) {
-            norm += squared_norm(b->at(i, j));
-            // norm += b->at(i, j) * b->at(i, j);
-        }
-        b_norm->at(0, j) = sqrt(norm);
-
 #pragma omp parallel for
         for (size_type i = 0; i < b->get_size()[0]; ++i) {
             residual->at(i, j) = b->at(i, j);
@@ -443,17 +429,16 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
             matrix::Dense<ValueType> *residual_norm_collection,
             Accessor3d krylov_bases, matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
-            const matrix::Dense<remove_complex<ValueType>> *b_norm,
             matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
             size_type iter, Array<size_type> *final_iter_nums,
-            const Array<stopping_status> *stop_status,
-            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
-            int *num_reorth_steps, int *num_reorth_vectors)
+            const Array<stopping_status> *stop_status, Array<stopping_status> *,
+            Array<size_type> *)
 {
 #pragma omp parallel for
     for (size_type i = 0; i < final_iter_nums->get_num_elems(); ++i) {
         final_iter_nums->get_data()[i] +=
-            (1 - stop_status->get_const_data()[i].has_stopped());
+            (1 - static_cast<size_type>(
+                     stop_status->get_const_data()[i].has_stopped()));
     }
     finish_arnoldi_CGS2(next_krylov_basis, krylov_bases, hessenberg_iter,
                         buffer_iter, arnoldi_norm, iter,
@@ -461,7 +446,7 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
     givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
                     iter, stop_status->get_const_data());
     calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,
-                                 residual_norm_collection, b_norm, iter,
+                                 residual_norm_collection, iter,
                                  stop_status->get_const_data());
 }
 
diff --git a/omp/test/solver/cb_gmres_kernels.cpp b/omp/test/solver/cb_gmres_kernels.cpp
index 3fbb704a2e6..cbeb919be7c 100644
--- a/omp/test/solver/cb_gmres_kernels.cpp
+++ b/omp/test/solver/cb_gmres_kernels.cpp
@@ -127,7 +127,6 @@ class CbGmres : public ::testing::Test {
         y = gen_mtx(default_krylov_dim_mixed, n);
         before_preconditioner = Mtx::create_with_config_of(x.get());
         b = gen_mtx(m, n);
-        b_norm = gen_mtx(1, n);
         arnoldi_norm = gen_mtx(3, n);
         gko::dim<3> krylov_bases_dim(default_krylov_dim_mixed + 1, m, n);
         range_helper = generate_krylov_helper(krylov_bases_dim);
@@ -170,8 +169,6 @@ class CbGmres : public ::testing::Test {
         d_y->copy_from(y.get());
         d_b = Mtx::create(omp);
         d_b->copy_from(b.get());
-        d_b_norm = Mtx::create(omp);
-        d_b_norm->copy_from(b_norm.get());
         d_arnoldi_norm = Mtx::create(omp);
         d_arnoldi_norm->copy_from(arnoldi_norm.get());
         d_range_helper = Range3dHelper{omp, {}};
@@ -230,7 +227,6 @@ class CbGmres : public ::testing::Test {
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> y;
     std::unique_ptr<Mtx> b;
-    std::unique_ptr<Mtx> b_norm;
     std::unique_ptr<Mtx> arnoldi_norm;
     Range3dHelper range_helper;
     std::unique_ptr<Mtx> next_krylov_basis;
@@ -251,7 +247,6 @@ class CbGmres : public ::testing::Test {
     std::unique_ptr<Mtx> d_before_preconditioner;
     std::unique_ptr<Mtx> d_y;
     std::unique_ptr<Mtx> d_b;
-    std::unique_ptr<Mtx> d_b_norm;
     std::unique_ptr<Mtx> d_arnoldi_norm;
     Range3dHelper d_range_helper;
     std::unique_ptr<Mtx> d_next_krylov_basis;
@@ -275,13 +270,12 @@ TEST_F(CbGmres, OmpCbGmresInitialize1IsEquivalentToRef)
     initialize_data();
 
     gko::kernels::reference::cb_gmres::initialize_1(
-        ref, b.get(), b_norm.get(), residual.get(), givens_sin.get(),
-        givens_cos.get(), stop_status.get(), default_krylov_dim_mixed);
+        ref, b.get(), residual.get(), givens_sin.get(), givens_cos.get(),
+        stop_status.get(), default_krylov_dim_mixed);
     gko::kernels::omp::cb_gmres::initialize_1(
-        omp, d_b.get(), d_b_norm.get(), d_residual.get(), d_givens_sin.get(),
+        omp, d_b.get(), d_residual.get(), d_givens_sin.get(),
         d_givens_cos.get(), d_stop_status.get(), default_krylov_dim_mixed);
 
-    GKO_ASSERT_MTX_NEAR(d_b_norm, b_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_residual, residual, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
@@ -315,23 +309,20 @@ TEST_F(CbGmres, OmpCbGmresStep1IsEquivalentToRef)
 {
     initialize_data();
     int iter = 5;
-    int num_reorth_steps = 0, num_reorth_vectors = 0;
-    int d_num_reorth_steps = 0, d_num_reorth_vectors = 0;
 
     gko::kernels::reference::cb_gmres::step_1(
         ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
         residual_norm.get(), residual_norm_collection.get(),
         range_helper.get_range(), hessenberg_iter.get(), buffer_iter.get(),
-        b_norm.get(), arnoldi_norm.get(), iter, final_iter_nums.get(),
-        stop_status.get(), reorth_status.get(), num_reorth.get(),
-        &num_reorth_steps, &num_reorth_vectors);
+        arnoldi_norm.get(), iter, final_iter_nums.get(), stop_status.get(),
+        reorth_status.get(), num_reorth.get());
     gko::kernels::omp::cb_gmres::step_1(
         omp, d_next_krylov_basis.get(), d_givens_sin.get(), d_givens_cos.get(),
         d_residual_norm.get(), d_residual_norm_collection.get(),
         d_range_helper.get_range(), d_hessenberg_iter.get(),
-        d_buffer_iter.get(), d_b_norm.get(), d_arnoldi_norm.get(), iter,
+        d_buffer_iter.get(), d_arnoldi_norm.get(), iter,
         d_final_iter_nums.get(), d_stop_status.get(), d_reorth_status.get(),
-        d_num_reorth.get(), &num_reorth_steps, &num_reorth_vectors);
+        d_num_reorth.get());
 
     GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
diff --git a/reference/solver/cb_gmres_kernels.cpp b/reference/solver/cb_gmres_kernels.cpp
index 023666590f6..a3e940aaa76 100644
--- a/reference/solver/cb_gmres_kernels.cpp
+++ b/reference/solver/cb_gmres_kernels.cpp
@@ -239,8 +239,7 @@ template <typename ValueType>
 void calculate_next_residual_norm(
     matrix::Dense<ValueType> *givens_sin, matrix::Dense<ValueType> *givens_cos,
     matrix::Dense<remove_complex<ValueType>> *residual_norm,
-    matrix::Dense<ValueType> *residual_norm_collection,
-    const matrix::Dense<remove_complex<ValueType>> *b_norm, size_type iter,
+    matrix::Dense<ValueType> *residual_norm_collection, size_type iter,
     const stopping_status *stop_status)
 {
     for (size_type i = 0; i < residual_norm->get_size()[1]; ++i) {
@@ -265,7 +264,7 @@ void solve_upper_triangular(
     const size_type *final_iter_nums)
 {
     for (size_type k = 0; k < residual_norm_collection->get_size()[1]; ++k) {
-        for (int i = final_iter_nums[k] - 1; i >= 0; --i) {
+        for (int64 i = final_iter_nums[k] - 1; i >= 0; --i) {
             auto temp = residual_norm_collection->at(i, k);
             for (size_type j = i + 1; j < final_iter_nums[k]; ++j) {
                 temp -=
@@ -310,20 +309,12 @@ void calculate_qy(ConstAccessor3d krylov_bases,
 template <typename ValueType>
 void initialize_1(std::shared_ptr<const ReferenceExecutor> exec,
                   const matrix::Dense<ValueType> *b,
-                  matrix::Dense<remove_complex<ValueType>> *b_norm,
                   matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *givens_sin,
                   matrix::Dense<ValueType> *givens_cos,
                   Array<stopping_status> *stop_status, size_type krylov_dim)
 {
     for (size_type j = 0; j < b->get_size()[1]; ++j) {
-        // Calculate b norm
-        b_norm->at(0, j) = zero<remove_complex<ValueType>>();
-        for (size_type i = 0; i < b->get_size()[0]; ++i) {
-            b_norm->at(0, j) += squared_norm(b->at(i, j));
-        }
-        b_norm->at(0, j) = sqrt(b_norm->at(0, j));
-
         for (size_type i = 0; i < b->get_size()[0]; ++i) {
             residual->at(i, j) = b->at(i, j);
         }
@@ -416,12 +407,10 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
             matrix::Dense<ValueType> *residual_norm_collection,
             Accessor3d krylov_bases, matrix::Dense<ValueType> *hessenberg_iter,
             matrix::Dense<ValueType> *buffer_iter,
-            const matrix::Dense<remove_complex<ValueType>> *b_norm,
             matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
             size_type iter, Array<size_type> *final_iter_nums,
-            const Array<stopping_status> *stop_status,
-            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth,
-            int *num_reorth_steps, int *num_reorth_vectors)
+            const Array<stopping_status> *stop_status, Array<stopping_status> *,
+            Array<size_type> *)
 {
     static_assert(
         std::is_same<ValueType,
@@ -429,7 +418,8 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
         "ValueType must match arithmetic_type of accessor!");
     for (size_type i = 0; i < final_iter_nums->get_num_elems(); ++i) {
         final_iter_nums->get_data()[i] +=
-            (1 - stop_status->get_const_data()[i].has_stopped());
+            (1 - static_cast<size_type>(
+                     stop_status->get_const_data()[i].has_stopped()));
     }
     finish_arnoldi_CGS2(next_krylov_basis, krylov_bases, hessenberg_iter,
                         buffer_iter, arnoldi_norm, iter,
@@ -437,7 +427,7 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
     givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
                     iter, stop_status->get_const_data());
     calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,
-                                 residual_norm_collection, b_norm, iter,
+                                 residual_norm_collection, iter,
                                  stop_status->get_const_data());
 }
 

From 9f156c5c1a0b63774d61bb6e511fab01a799205a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 8 Feb 2021 11:46:24 +0100
Subject: [PATCH 0412/1008] Review update; Improve run_all_benchmarks.sh

- Update documentation of GMRES to mention the usage of MGS
- Use reduced precision in CB-GMRES by default
---
 benchmark/run_all_benchmarks.sh         | 20 ++++++++++----------
 core/test/solver/cb_gmres.cpp           |  2 +-
 include/ginkgo/core/solver/cb_gmres.hpp |  5 +++--
 include/ginkgo/core/solver/gmres.hpp    |  2 +-
 4 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index be2d09229dd..bd481169057 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -2,18 +2,18 @@
 # Environment variable detection
 
 if [ ! "${BENCHMARK}" ]; then
-    echo "BENCHMARK   environment variable not set - assuming \"spmv\"" 1>&2
     BENCHMARK="spmv"
+    echo "BENCHMARK   environment variable not set - assuming \"${BENCHMARK}\"" 1>&2
 fi
 
 if [ ! "${DRY_RUN}" ]; then
-    echo "DRY_RUN     environment variable not set - assuming \"false\"" 1>&2
     DRY_RUN="false"
+    echo "DRY_RUN     environment variable not set - assuming \"${DRY_RUN}\"" 1>&2
 fi
 
 if [ ! "${EXECUTOR}" ]; then
-    echo "EXECUTOR    environment variable not set - assuming \"cuda\"" 1>&2
     EXECUTOR="cuda"
+    echo "EXECUTOR    environment variable not set - assuming \"${EXECUTOR}\"" 1>&2
 fi
 
 if [ ! "${SEGMENTS}" ]; then
@@ -26,8 +26,8 @@ elif [ ! "${SEGMENT_ID}" ]; then
 fi
 
 if [ ! "${PRECONDS}" ]; then
-    echo "PRECONDS    environment variable not set - assuming \"none\"" 1>&2
     PRECONDS="none"
+    echo "PRECONDS    environment variable not set - assuming \"${PRECONDS}\"" 1>&2
 fi
 
 if [ ! "${FORMATS}" ]; then
@@ -36,18 +36,18 @@ if [ ! "${FORMATS}" ]; then
 fi
 
 if [ ! "${SOLVERS}" ]; then
-    echo "SOLVERS    environment variable not set - assuming \"bicgstab,cg,cgs,fcg,gmres,idr\"" 1>&2
     SOLVERS="bicgstab,cg,cgs,fcg,gmres,cb_gmres_reduce1,idr"
+    echo "SOLVERS    environment variable not set - assuming \"${SOLVERS}\"" 1>&2
 fi
 
 if [ ! "${SOLVERS_PRECISION}" ]; then
-    echo "SOLVERS_PRECISION environment variable not set - assuming \"1e-6\"" 1>&2
     SOLVERS_PRECISION=1e-6
+    echo "SOLVERS_PRECISION environment variable not set - assuming \"${SOLVERS_PRECISION}\"" 1>&2
 fi
 
 if [ ! "${SOLVERS_MAX_ITERATIONS}" ]; then
-    echo "SOLVERS_MAX_ITERATIONS environment variable not set - assuming \"10000\"" 1>&2
     SOLVERS_MAX_ITERATIONS=10000
+    echo "SOLVERS_MAX_ITERATIONS environment variable not set - assuming \"${SOLVERS_MAX_ITERATIONS}\"" 1>&2
 fi
 
 if [ ! "${SOLVERS_GMRES_RESTART}" ]; then
@@ -56,13 +56,13 @@ if [ ! "${SOLVERS_GMRES_RESTART}" ]; then
 fi
 
 if [ ! "${SYSTEM_NAME}" ]; then
-    echo "SYSTEM_MANE environment variable not set - assuming \"unknown\"" 1>&2
     SYSTEM_NAME="unknown"
+    echo "SYSTEM_MANE environment variable not set - assuming \"${SYSTEM_NAME}\"" 1>&2
 fi
 
 if [ ! "${DEVICE_ID}" ]; then
-    echo "DEVICE_ID environment variable not set - assuming \"0\"" 1>&2
     DEVICE_ID="0"
+    echo "DEVICE_ID environment variable not set - assuming \"${DEVICE_ID}\"" 1>&2
 fi
 
 if [ ! "${SOLVERS_JACOBI_MAX_BS}" ]; then
@@ -124,8 +124,8 @@ else
 fi
 
 if [ ! "${GPU_TIMER}" ]; then
-    echo "GPU_TIMER    environment variable not set - assuming \"false\"" 1>&2
     GPU_TIMER="false"
+    echo "GPU_TIMER    environment variable not set - assuming \"${GPU_TIMER}\"" 1>&2
 fi
 
 # Control whether to run detailed benchmarks or not.
diff --git a/core/test/solver/cb_gmres.cpp b/core/test/solver/cb_gmres.cpp
index 6c273c1189f..229cec1752c 100644
--- a/core/test/solver/cb_gmres.cpp
+++ b/core/test/solver/cb_gmres.cpp
@@ -278,7 +278,7 @@ TYPED_TEST(CbGmres, CanSetKrylovDim)
     auto krylov_dim = solver->get_krylov_dim();
 
     ASSERT_EQ(solver->get_storage_precision(),
-              gko::solver::cb_gmres_storage_precision::keep);
+              gko::solver::cb_gmres_storage_precision::reduce1);
 }
 
 
diff --git a/include/ginkgo/core/solver/cb_gmres.hpp b/include/ginkgo/core/solver/cb_gmres.hpp
index e215c9ccff3..920b2097218 100644
--- a/include/ginkgo/core/solver/cb_gmres.hpp
+++ b/include/ginkgo/core/solver/cb_gmres.hpp
@@ -100,7 +100,8 @@ enum class cb_gmres_storage_precision {
  *
  * The krylov basis can be stored in reduced precision (compressed) to reduce
  * memory accesses, while all computations (including krylov basis operations)
- * are performed in the same arithmetic precision ValueType.
+ * are performed in the same arithmetic precision ValueType. By default, the
+ * krylov basis are stored in one precision lower than ValueType.
  *
  * @tparam ValueType  the arithmetic precision and the precision of matrix
  *                    elements
@@ -150,7 +151,7 @@ class CbGmres : public EnableLinOp<CbGmres<ValueType>>,
          * Determines which storage type is used.
          */
         cb_gmres_storage_precision GKO_FACTORY_PARAMETER_SCALAR(
-            storage_precision, cb_gmres_storage_precision::keep);
+            storage_precision, cb_gmres_storage_precision::reduce1);
 
         /**
          * Criterion factories.
diff --git a/include/ginkgo/core/solver/gmres.hpp b/include/ginkgo/core/solver/gmres.hpp
index 2e426be55f8..555234a3da1 100644
--- a/include/ginkgo/core/solver/gmres.hpp
+++ b/include/ginkgo/core/solver/gmres.hpp
@@ -61,7 +61,7 @@ constexpr size_type default_krylov_dim = 100u;
  *
  * The implementation in Ginkgo makes use of the merged kernel to make the best
  * use of data locality. The inner operations in one iteration of GMRES are
- * merged into 2 separate steps.
+ * merged into 2 separate steps. Modified Gram-Schmidt is used.
  *
  * @tparam ValueType  precision of matrix elements
  *

From 7e01e2dc1b2049d30ad73e738ae7dc66f86cace3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 9 Feb 2021 11:14:03 +0100
Subject: [PATCH 0413/1008] Put storage_precision enum into cb_gmres namespace

---
 benchmark/solver/solver.cpp                | 14 +++++-----
 core/solver/cb_gmres.cpp                   | 24 ++++++++---------
 core/test/solver/cb_gmres.cpp              |  6 ++---
 include/ginkgo/core/solver/cb_gmres.hpp    | 16 ++++++++----
 reference/test/solver/cb_gmres_kernels.cpp | 30 +++++++++++-----------
 5 files changed, 48 insertions(+), 42 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index bc4dc2504ad..2da3679cade 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -245,20 +245,20 @@ std::unique_ptr<gko::LinOpFactory> generate_solver(
 {
     std::string cb_gmres_prefix("cb_gmres_");
     if (description.find(cb_gmres_prefix) == 0) {
-        auto s_prec = gko::solver::cb_gmres_storage_precision::keep;
+        auto s_prec = gko::solver::cb_gmres::storage_precision::keep;
         const auto spec = description.substr(cb_gmres_prefix.length());
         if (spec == "keep") {
-            s_prec = gko::solver::cb_gmres_storage_precision::keep;
+            s_prec = gko::solver::cb_gmres::storage_precision::keep;
         } else if (spec == "reduce1") {
-            s_prec = gko::solver::cb_gmres_storage_precision::reduce1;
+            s_prec = gko::solver::cb_gmres::storage_precision::reduce1;
         } else if (spec == "reduce2") {
-            s_prec = gko::solver::cb_gmres_storage_precision::reduce2;
+            s_prec = gko::solver::cb_gmres::storage_precision::reduce2;
         } else if (spec == "integer") {
-            s_prec = gko::solver::cb_gmres_storage_precision::integer;
+            s_prec = gko::solver::cb_gmres::storage_precision::integer;
         } else if (spec == "ireduce1") {
-            s_prec = gko::solver::cb_gmres_storage_precision::ireduce1;
+            s_prec = gko::solver::cb_gmres::storage_precision::ireduce1;
         } else if (spec == "ireduce2") {
-            s_prec = gko::solver::cb_gmres_storage_precision::ireduce2;
+            s_prec = gko::solver::cb_gmres::storage_precision::ireduce2;
         } else {
             throw std::range_error(
                 std::string(
diff --git a/core/solver/cb_gmres.cpp b/core/solver/cb_gmres.cpp
index f3a3db1c622..b66cf317b8b 100644
--- a/core/solver/cb_gmres.cpp
+++ b/core/solver/cb_gmres.cpp
@@ -129,22 +129,22 @@ template <typename ValueType>
 struct helper {
     template <typename Callable>
     static void call(Callable callable,
-                     gko::solver::cb_gmres_storage_precision st)
+                     gko::solver::cb_gmres::storage_precision st)
     {
         switch (st) {
-        case cb_gmres_storage_precision::reduce1:
+        case cb_gmres::storage_precision::reduce1:
             callable(reduce_precision_count<ValueType, 1>{});
             break;
-        case cb_gmres_storage_precision::reduce2:
+        case cb_gmres::storage_precision::reduce2:
             callable(reduce_precision_count<ValueType, 2>{});
             break;
-        case cb_gmres_storage_precision::integer:
+        case cb_gmres::storage_precision::integer:
             callable(to_integer<ValueType>{});
             break;
-        case cb_gmres_storage_precision::ireduce1:
+        case cb_gmres::storage_precision::ireduce1:
             callable(to_integer<reduce_precision_count<ValueType, 1>>{});
             break;
-        case cb_gmres_storage_precision::ireduce2:
+        case cb_gmres::storage_precision::ireduce2:
             callable(to_integer<reduce_precision_count<ValueType, 2>>{});
             break;
         default:
@@ -162,18 +162,18 @@ struct helper<std::complex<T>> {
 
     template <typename Callable>
     static void call(Callable callable,
-                     gko::solver::cb_gmres_storage_precision st)
+                     gko::solver::cb_gmres::storage_precision st)
     {
         switch (st) {
-        case cb_gmres_storage_precision::reduce1:
+        case cb_gmres::storage_precision::reduce1:
             callable(reduce_precision_skip_count<ValueType, skip_type, 1>{});
             break;
-        case cb_gmres_storage_precision::reduce2:
+        case cb_gmres::storage_precision::reduce2:
             callable(reduce_precision_skip_count<ValueType, skip_type, 2>{});
             break;
-        case cb_gmres_storage_precision::integer:
-        case cb_gmres_storage_precision::ireduce1:
-        case cb_gmres_storage_precision::ireduce2:
+        case cb_gmres::storage_precision::integer:
+        case cb_gmres::storage_precision::ireduce1:
+        case cb_gmres::storage_precision::ireduce2:
             GKO_NOT_SUPPORTED(st);
             break;
         default:
diff --git a/core/test/solver/cb_gmres.cpp b/core/test/solver/cb_gmres.cpp
index 229cec1752c..2ec3e61ad69 100644
--- a/core/test/solver/cb_gmres.cpp
+++ b/core/test/solver/cb_gmres.cpp
@@ -92,7 +92,7 @@ class CbGmres : public ::testing::Test {
           big_solver(cb_gmres_big_factory->generate(mtx))
     {}
 
-    gko::solver::cb_gmres_storage_precision storage_precision;
+    gko::solver::cb_gmres::storage_precision storage_precision;
     std::shared_ptr<const gko::Executor> exec;
     std::shared_ptr<Mtx> mtx;
     std::unique_ptr<typename Solver::Factory> cb_gmres_factory;
@@ -117,7 +117,7 @@ class CbGmres : public ::testing::Test {
  * This creates a helper structure which translates a type into an enum
  * parameter.
  */
-using st_enum = gko::solver::cb_gmres_storage_precision;
+using st_enum = gko::solver::cb_gmres::storage_precision;
 
 template <st_enum P>
 struct st_helper_type {
@@ -278,7 +278,7 @@ TYPED_TEST(CbGmres, CanSetKrylovDim)
     auto krylov_dim = solver->get_krylov_dim();
 
     ASSERT_EQ(solver->get_storage_precision(),
-              gko::solver::cb_gmres_storage_precision::reduce1);
+              gko::solver::cb_gmres::storage_precision::reduce1);
 }
 
 
diff --git a/include/ginkgo/core/solver/cb_gmres.hpp b/include/ginkgo/core/solver/cb_gmres.hpp
index 920b2097218..473773f2c7e 100644
--- a/include/ginkgo/core/solver/cb_gmres.hpp
+++ b/include/ginkgo/core/solver/cb_gmres.hpp
@@ -52,6 +52,9 @@ namespace gko {
 namespace solver {
 
 
+namespace cb_gmres {
+
+
 /**
  * Describes the storage precision that is used in CB-GMRES.
  *
@@ -79,7 +82,7 @@ namespace solver {
  * - float -> int32
  * - half -> int16
  */
-enum class cb_gmres_storage_precision {
+enum class storage_precision {
     keep,
     reduce1,
     reduce2,
@@ -89,6 +92,9 @@ enum class cb_gmres_storage_precision {
 };
 
 
+}  // namespace cb_gmres
+
+
 /**
  * CB-GMRES or the compressed basis generalized minimal residual method is an
  * iterative type Krylov subspace method which is suitable for nonsymmetric
@@ -140,7 +146,7 @@ class CbGmres : public EnableLinOp<CbGmres<ValueType>>,
      *
      * @return the storage precision used internally
      */
-    cb_gmres_storage_precision get_storage_precision() const
+    cb_gmres::storage_precision get_storage_precision() const
     {
         return storage_precision_;
     }
@@ -150,8 +156,8 @@ class CbGmres : public EnableLinOp<CbGmres<ValueType>>,
         /**
          * Determines which storage type is used.
          */
-        cb_gmres_storage_precision GKO_FACTORY_PARAMETER_SCALAR(
-            storage_precision, cb_gmres_storage_precision::reduce1);
+        cb_gmres::storage_precision GKO_FACTORY_PARAMETER_SCALAR(
+            storage_precision, cb_gmres::storage_precision::reduce1);
 
         /**
          * Criterion factories.
@@ -218,7 +224,7 @@ class CbGmres : public EnableLinOp<CbGmres<ValueType>>,
     std::shared_ptr<const LinOp> system_matrix_{};
     std::shared_ptr<const stop::CriterionFactory> stop_criterion_factory_{};
     size_type krylov_dim_;
-    cb_gmres_storage_precision storage_precision_;
+    cb_gmres::storage_precision storage_precision_;
 };
 
 
diff --git a/reference/test/solver/cb_gmres_kernels.cpp b/reference/test/solver/cb_gmres_kernels.cpp
index 115b3a10391..70d0a5fe94d 100644
--- a/reference/test/solver/cb_gmres_kernels.cpp
+++ b/reference/test/solver/cb_gmres_kernels.cpp
@@ -72,10 +72,10 @@ class CbGmres : public ::testing::Test {
               {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)),
           mtx2(gko::initialize<Mtx>(
               {{1.0, 2.0, 3.0}, {4.0, 2.0, 1.0}, {0.0, 1.0, 2.0}}, exec)),
-          storage_precision{storage_helper_type::value},
+          storage_prec{storage_helper_type::value},
           cb_gmres_factory(
               gmres_type::build()
-                  .with_storage_precision(storage_precision)
+                  .with_storage_precision(storage_prec)
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
@@ -96,7 +96,7 @@ class CbGmres : public ::testing::Test {
               exec)),
           cb_gmres_factory_big(
               gmres_type::build()
-                  .with_storage_precision(storage_precision)
+                  .with_storage_precision(storage_prec)
                   .with_criteria(
                       gko::stop::Iteration::build().with_max_iters(100u).on(
                           exec),
@@ -123,19 +123,19 @@ class CbGmres : public ::testing::Test {
     nc_value_type assert_precision() const noexcept
     {
         using gko::reduce_precision;
-        using gko::solver::cb_gmres_storage_precision;
+        using gko::solver::cb_gmres::storage_precision;
 
         // Note: integer and floating point are assumed to have similar
         //       target precision.
-        switch (storage_precision) {
-        case cb_gmres_storage_precision::reduce1:
-        case cb_gmres_storage_precision::ireduce1:
+        switch (storage_prec) {
+        case storage_precision::reduce1:
+        case storage_precision::ireduce1:
             return r<reduce_precision<value_type>, nc_value_type>::value;
-        case cb_gmres_storage_precision::reduce2:
-        case cb_gmres_storage_precision::ireduce2:
+        case storage_precision::reduce2:
+        case storage_precision::ireduce2:
             return r<reduce_precision<reduce_precision<value_type>>,
                      nc_value_type>::value;
-        case cb_gmres_storage_precision::integer:
+        case storage_precision::integer:
         default:
             return r<nc_value_type>::value;
         }
@@ -146,7 +146,7 @@ class CbGmres : public ::testing::Test {
     std::shared_ptr<Mtx> mtx2;
     std::shared_ptr<Mtx> mtx_medium;
     std::shared_ptr<Mtx> mtx_big;
-    gko::solver::cb_gmres_storage_precision storage_precision;
+    gko::solver::cb_gmres::storage_precision storage_prec;
     std::unique_ptr<typename gmres_type::Factory> cb_gmres_factory;
     std::unique_ptr<typename gmres_type::Factory> cb_gmres_factory_big;
 };
@@ -156,7 +156,7 @@ class CbGmres : public ::testing::Test {
  * This creates a helper structure which translates a type into an enum
  * parameter.
  */
-using st_enum = gko::solver::cb_gmres_storage_precision;
+using st_enum = gko::solver::cb_gmres::storage_precision;
 
 template <st_enum P>
 struct st_helper_type {
@@ -206,7 +206,7 @@ TYPED_TEST(CbGmres, SolvesStencilSystem2)
     using gmres_type = typename TestFixture::gmres_type;
     auto factory =
         gmres_type::build()
-            .with_storage_precision(this->storage_precision)
+            .with_storage_precision(this->storage_prec)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(100u).on(
                     this->exec),
@@ -405,7 +405,7 @@ TYPED_TEST(CbGmres, SolvesBigDenseSystem1WithRestart)
     auto cb_gmres_factory_restart =
         gmres_type::build()
             .with_krylov_dim(4u)
-            .with_storage_precision(this->storage_precision)
+            .with_storage_precision(this->storage_prec)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(200u).on(
                     this->exec),
@@ -432,7 +432,7 @@ TYPED_TEST(CbGmres, SolvesWithPreconditioner)
     using gmres_type = typename TestFixture::gmres_type;
     auto cb_gmres_factory_preconditioner =
         gmres_type::build()
-            .with_storage_precision(this->storage_precision)
+            .with_storage_precision(this->storage_prec)
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(100u).on(
                     this->exec),

From 15f8d61426eed31d2038c5079131a28a6de8d55f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 9 Feb 2021 23:53:25 +0100
Subject: [PATCH 0414/1008] Add CB-GMRES example

---
 examples/CMakeLists.txt           |   1 +
 examples/cb-gmres/CMakeLists.txt  |   5 +
 examples/cb-gmres/build.sh        |  16 +++
 examples/cb-gmres/cb-gmres.cpp    | 220 ++++++++++++++++++++++++++++++
 examples/cb-gmres/doc/builds-on   |   1 +
 examples/cb-gmres/doc/intro.dox   |  15 ++
 examples/cb-gmres/doc/kind        |   1 +
 examples/cb-gmres/doc/results.dox |  21 +++
 examples/cb-gmres/doc/short-intro |   1 +
 examples/cb-gmres/doc/tooltip     |   1 +
 10 files changed, 282 insertions(+)
 create mode 100644 examples/cb-gmres/CMakeLists.txt
 create mode 100755 examples/cb-gmres/build.sh
 create mode 100644 examples/cb-gmres/cb-gmres.cpp
 create mode 100644 examples/cb-gmres/doc/builds-on
 create mode 100644 examples/cb-gmres/doc/intro.dox
 create mode 100644 examples/cb-gmres/doc/kind
 create mode 100644 examples/cb-gmres/doc/results.dox
 create mode 100644 examples/cb-gmres/doc/short-intro
 create mode 100644 examples/cb-gmres/doc/tooltip

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index b35d6422098..43b6f027990 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -3,6 +3,7 @@ option(GINKGO_RUN_EXAMPLES " Compile run and validation targets for the examples
 
 set(EXAMPLES_EXEC_LIST
     adaptiveprecision-blockjacobi
+    cb-gmres
     custom-logger
     ginkgo-ranges
     ilu-preconditioned-solver
diff --git a/examples/cb-gmres/CMakeLists.txt b/examples/cb-gmres/CMakeLists.txt
new file mode 100644
index 00000000000..e3265883919
--- /dev/null
+++ b/examples/cb-gmres/CMakeLists.txt
@@ -0,0 +1,5 @@
+set(target_name "cb-gmres")
+add_executable(${target_name} ${target_name}.cpp)
+target_link_libraries(${target_name} Ginkgo::ginkgo)
+target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+configure_file("${Ginkgo_SOURCE_DIR}/matrices/test/ani1.mtx" data/A.mtx COPYONLY)
diff --git a/examples/cb-gmres/build.sh b/examples/cb-gmres/build.sh
new file mode 100755
index 00000000000..9cbd24ca500
--- /dev/null
+++ b/examples/cb-gmres/build.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# set up script
+if [ $# -ne 1 ]; then
+    echo -e "Usage: $0 GINKGO_BUILD_DIRECTORY"
+    exit 1
+fi
+BUILD_DIR=$1
+THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" &>/dev/null && pwd )
+
+source ${THIS_DIR}/../build-setup.sh
+
+# build
+${CXX} -std=c++14 -o ${THIS_DIR}/cb-gmres ${THIS_DIR}/cb-gmres.cpp \
+       -I${THIS_DIR}/../../include -I${BUILD_DIR}/include \
+       -L${THIS_DIR} ${LINK_FLAGS}
diff --git a/examples/cb-gmres/cb-gmres.cpp b/examples/cb-gmres/cb-gmres.cpp
new file mode 100644
index 00000000000..ffb52579132
--- /dev/null
+++ b/examples/cb-gmres/cb-gmres.cpp
@@ -0,0 +1,220 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+// This is the main ginkgo header file.
+#include <ginkgo/ginkgo.hpp>
+
+#include <chrono>
+#include <cmath>
+#include <fstream>
+#include <iostream>
+#include <map>
+#include <string>
+
+
+// Helper function which measures the time of `solver->apply(b, x)` in seconds
+// To get an accurate result, the solve is repeated multiple times (while
+// ensuring the initial guess is always the same). The result of the solve will
+// be written to x.
+double measure_solve_time_in_s(const gko::Executor *exec, gko::LinOp *solver,
+                               const gko::LinOp *b, gko::LinOp *x)
+{
+    constexpr int repeats{5};
+    double duration{0};
+    // Make a copy of x, so we can re-use the same initial guess multiple times
+    auto x_copy = clone(x);
+    for (int i = 0; i < repeats; ++i) {
+        // No need to copy it in the first iteration
+        if (i != 0) {
+            x_copy->copy_from(x);
+        }
+        // Make sure all previous executor operations have finished before
+        // starting the time
+        exec->synchronize();
+        auto tic = std::chrono::steady_clock::now();
+        solver->apply(b, lend(x_copy));
+        // Make sure all computations are done before stopping the time
+        exec->synchronize();
+        auto tac = std::chrono::steady_clock::now();
+        duration += std::chrono::duration<double>(tac - tic).count();
+    }
+    // Copy the solution back to x, so the caller has the result
+    x->copy_from(lend(x_copy));
+    return duration / static_cast<double>(repeats);
+}
+
+
+int main(int argc, char *argv[])
+{
+    // Use some shortcuts. In Ginkgo, vectors are seen as a gko::matrix::Dense
+    // with one column/one row. The advantage of this concept is that using
+    // multiple vectors is a now a natural extension of adding columns/rows are
+    // necessary.
+    using ValueType = double;
+    using RealValueType = gko::remove_complex<ValueType>;
+    using IndexType = int;
+    using vec = gko::matrix::Dense<ValueType>;
+    using real_vec = gko::matrix::Dense<RealValueType>;
+    // The gko::matrix::Csr class is used here, but any other matrix class such
+    // as gko::matrix::Coo, gko::matrix::Hybrid, gko::matrix::Ell or
+    // gko::matrix::Sellp could also be used.
+    using mtx = gko::matrix::Csr<ValueType, IndexType>;
+    // The gko::solver::CbGmres is used here, but any other solver class can
+    // also be used.
+    using cb_gmres = gko::solver::CbGmres<ValueType>;
+
+    // Print the ginkgo version information.
+    std::cout << gko::version_info::get() << std::endl;
+
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
+        std::cerr << "Usage: " << argv[0] << " [executor] " << std::endl;
+        std::exit(-1);
+    }
+
+    // Map which generates the appropriate executor
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
+
+    // Note: this matrix is copied from "SOURCE_DIR/matrices" instead of from
+    //       the local directory. For details, see
+    //       "examples/cb-gmres/CMakeLists.txt"
+    auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
+    // Create a uniform right-hand side with a norm2 of 1 on the host
+    // (norm2(b) == 1), followed by copying it to the actual executor
+    // (to make sure it also works for GPUs)
+    const auto A_size = A->get_size();
+    auto b_host = vec::create(exec->get_master(), gko::dim<2>{A_size[0], 1});
+    for (gko::size_type i = 0; i < A_size[0]; ++i) {
+        b_host->at(i, 0) =
+            ValueType{1} / std::sqrt(static_cast<ValueType>(A_size[0]));
+    }
+    auto b_norm = gko::initialize<real_vec>({0.0}, exec);
+    b_host->compute_norm2(lend(b_norm));
+    auto b = clone(exec, lend(b_host));
+
+    // As an initial guess, use the right-hand side
+    auto x_keep = clone(lend(b));
+    auto x_reduce = clone(x_keep);
+
+    const RealValueType reduction_factor{1e-6};
+
+    // Generate two solver factories: `_keep` uses the same precision for the
+    // krylov basis as the matrix, and `_reduce` uses one precision below it.
+    // If `ValueType` is double, then `_reduce` uses float as the krylov basis
+    // storage type
+    auto solver_gen_keep =
+        cb_gmres::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1000u).on(exec),
+                gko::stop::RelativeResidualNorm<ValueType>::build()
+                    .with_tolerance(reduction_factor)
+                    .on(exec))
+            .with_krylov_dim(100u)
+            .with_storage_precision(
+                gko::solver::cb_gmres::storage_precision::keep)
+            .on(exec);
+
+    auto solver_gen_reduce =
+        cb_gmres::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1000u).on(exec),
+                gko::stop::RelativeResidualNorm<ValueType>::build()
+                    .with_tolerance(reduction_factor)
+                    .on(exec))
+            .with_krylov_dim(100u)
+            .with_storage_precision(
+                gko::solver::cb_gmres::storage_precision::reduce1)
+            .on(exec);
+    // Generate the actual solver from the factory and the matrix.
+    auto solver_keep = solver_gen_keep->generate(A);
+    auto solver_reduce = solver_gen_reduce->generate(A);
+
+    // Solve both system and measure the time for each.
+    auto time_keep = measure_solve_time_in_s(lend(exec), lend(solver_keep),
+                                             lend(b), lend(x_keep));
+    auto time_reduce = measure_solve_time_in_s(lend(exec), lend(solver_reduce),
+                                               lend(b), lend(x_reduce));
+
+    // Make sure the output is in scientific notation for easier comparison
+    std::cout << std::scientific;
+    // Note: The time might not be significantly different since the matrix is
+    //       quite small
+    std::cout << "Solve time without compression: " << time_keep << " s\n"
+              << "Solve time with compression:    " << time_reduce << " s\n";
+
+    // To measure if your solution has actually converged, the error of the
+    // solution is measured.
+    // one, neg_one are objects that represent the numbers which allow for a
+    // uniform interface when computing on any device. To compute the residual,
+    // the (advanced) apply method is used.
+    auto one = gko::initialize<vec>({1.0}, exec);
+    auto neg_one = gko::initialize<vec>({-1.0}, exec);
+
+    auto res_norm_keep = gko::initialize<real_vec>({0.0}, exec);
+    auto res_norm_reduce = gko::initialize<real_vec>({0.0}, exec);
+    auto tmp = gko::clone(gko::lend(b));
+
+    // tmp = Ax - tmp
+    A->apply(lend(one), lend(x_keep), lend(neg_one), lend(tmp));
+    tmp->compute_norm2(lend(res_norm_keep));
+
+    std::cout << "\nResidual norm without compression:\n";
+    write(std::cout, lend(res_norm_keep));
+
+    tmp->copy_from(lend(b));
+    A->apply(lend(one), lend(x_reduce), lend(neg_one), lend(tmp));
+    tmp->compute_norm2(lend(res_norm_reduce));
+
+    std::cout << "\nResidual norm with compression:\n";
+    write(std::cout, lend(res_norm_reduce));
+}
diff --git a/examples/cb-gmres/doc/builds-on b/examples/cb-gmres/doc/builds-on
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/examples/cb-gmres/doc/builds-on
@@ -0,0 +1 @@
+
diff --git a/examples/cb-gmres/doc/intro.dox b/examples/cb-gmres/doc/intro.dox
new file mode 100644
index 00000000000..06145d2d68c
--- /dev/null
+++ b/examples/cb-gmres/doc/intro.dox
@@ -0,0 +1,15 @@
+<a name="Intro"></a>
+<h1>Introduction</h1>
+
+<h3> About the example </h3>
+This example showcases the usage of the Ginkgo solver CB-GMRES (Compressed
+Basis GMRES). A small system is solved with two un-preconditioned CB-GMRES
+solvers:
+  1. without compressing the krylov basis; it uses double precision for
+     both the matrix and the krylov basis, and
+  2. with a compression of the krylov basis; it uses double precision for the
+     matrix and all arithmetic operations, while using single precision for the
+     krylov basis
+
+Both solves are timed and the residual norm of each solution is computed to
+show that both solutions are correct.
diff --git a/examples/cb-gmres/doc/kind b/examples/cb-gmres/doc/kind
new file mode 100644
index 00000000000..c1d9154931a
--- /dev/null
+++ b/examples/cb-gmres/doc/kind
@@ -0,0 +1 @@
+techniques
diff --git a/examples/cb-gmres/doc/results.dox b/examples/cb-gmres/doc/results.dox
new file mode 100644
index 00000000000..268d893c979
--- /dev/null
+++ b/examples/cb-gmres/doc/results.dox
@@ -0,0 +1,21 @@
+<h1>Results</h1>
+The following is the expected result:
+
+@code{.cpp}
+
+Solve time without compression: 1.842690e-04 s
+Solve time with compression:    1.589936e-04 s
+
+Residual norm without compression:
+%%MatrixMarket matrix array real general
+1 1
+2.430544e-07
+
+Residual norm with compression:
+%%MatrixMarket matrix array real general
+1 1
+3.437257e-07
+
+@endcode
+
+<h3> Comments about programming and debugging </h3>
diff --git a/examples/cb-gmres/doc/short-intro b/examples/cb-gmres/doc/short-intro
new file mode 100644
index 00000000000..1970978c02c
--- /dev/null
+++ b/examples/cb-gmres/doc/short-intro
@@ -0,0 +1 @@
+The CB-GMRES solver example.
diff --git a/examples/cb-gmres/doc/tooltip b/examples/cb-gmres/doc/tooltip
new file mode 100644
index 00000000000..75201cbf08d
--- /dev/null
+++ b/examples/cb-gmres/doc/tooltip
@@ -0,0 +1 @@
+Solve a linear system with CB-GMRES, both with and without compression. Benchmark the solve time and validate the result.

From 1cc02865c7f4343cc4e574a3254675466d62f544 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 10 Feb 2021 00:32:39 +0100
Subject: [PATCH 0415/1008] Remove unnecessary included files for CB-GMRES

---
 core/solver/cb_gmres.cpp                   | 4 +++-
 core/solver/cb_gmres_kernels.hpp           | 4 +---
 core/test/solver/cb_gmres.cpp              | 4 ++--
 cuda/solver/cb_gmres_kernels.cu            | 3 ++-
 cuda/test/solver/cb_gmres_kernels.cpp      | 2 +-
 hip/solver/cb_gmres_kernels.hip.cpp        | 3 ++-
 hip/test/solver/cb_gmres_kernels.cpp       | 2 +-
 include/ginkgo/core/solver/cb_gmres.hpp    | 1 +
 omp/solver/cb_gmres_kernels.cpp            | 6 ++++--
 omp/test/solver/cb_gmres_kernels.cpp       | 2 +-
 reference/solver/cb_gmres_kernels.cpp      | 7 ++++---
 reference/test/solver/cb_gmres_kernels.cpp | 4 ++--
 12 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/core/solver/cb_gmres.cpp b/core/solver/cb_gmres.cpp
index b66cf317b8b..a77989c8228 100644
--- a/core/solver/cb_gmres.cpp
+++ b/core/solver/cb_gmres.cpp
@@ -41,10 +41,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/utils.hpp>
+#include <ginkgo/core/base/utils_helper.hpp>
+#include <ginkgo/core/log/logger.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "core/base/extended_float.hpp"
 #include "core/solver/cb_gmres_accessor.hpp"
 #include "core/solver/cb_gmres_kernels.hpp"
 
diff --git a/core/solver/cb_gmres_kernels.hpp b/core/solver/cb_gmres_kernels.hpp
index 305c2f84c61..7163f4527cc 100644
--- a/core/solver/cb_gmres_kernels.hpp
+++ b/core/solver/cb_gmres_kernels.hpp
@@ -35,16 +35,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/stop/stopping_status.hpp>
 
 
 #include "core/base/accessors.hpp"
 #include "core/base/extended_float.hpp"
-#include "core/solver/cb_gmres_accessor.hpp"
 
 
 // TODO Find way around using it!
diff --git a/core/test/solver/cb_gmres.cpp b/core/test/solver/cb_gmres.cpp
index 2ec3e61ad69..7b41667fc28 100644
--- a/core/test/solver/cb_gmres.cpp
+++ b/core/test/solver/cb_gmres.cpp
@@ -33,7 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/cb_gmres.hpp>
 
 
-#include <typeinfo>
+#include <tuple>
 
 
 #include <gtest/gtest.h>
@@ -43,7 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm_reduction.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
 
 
 #include "core/solver/cb_gmres.cpp"
diff --git a/cuda/solver/cb_gmres_kernels.cu b/cuda/solver/cb_gmres_kernels.cu
index bc8a20e54ae..b2dc33a3634 100644
--- a/cuda/solver/cb_gmres_kernels.cu
+++ b/cuda/solver/cb_gmres_kernels.cu
@@ -36,15 +36,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <algorithm>
 
 
-#include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/stopping_status.hpp>
 
 
 #include "core/base/accessors.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/matrix/dense_kernels.hpp"
+#include "core/solver/cb_gmres_accessor.hpp"
 #include "cuda/base/config.hpp"
 #include "cuda/base/math.hpp"
 #include "cuda/base/types.hpp"
diff --git a/cuda/test/solver/cb_gmres_kernels.cpp b/cuda/test/solver/cb_gmres_kernels.cpp
index 0a3869b9666..e4c6164e258 100644
--- a/cuda/test/solver/cb_gmres_kernels.cpp
+++ b/cuda/test/solver/cb_gmres_kernels.cpp
@@ -44,7 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm_reduction.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
 
 
 #include "core/solver/cb_gmres_accessor.hpp"
diff --git a/hip/solver/cb_gmres_kernels.hip.cpp b/hip/solver/cb_gmres_kernels.hip.cpp
index e54652a2d82..345f84fce31 100644
--- a/hip/solver/cb_gmres_kernels.hip.cpp
+++ b/hip/solver/cb_gmres_kernels.hip.cpp
@@ -38,12 +38,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/range_accessors.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/stopping_status.hpp>
 
 
 #include "core/components/fill_array.hpp"
 #include "core/matrix/dense_kernels.hpp"
+#include "core/solver/cb_gmres_accessor.hpp"
 #include "hip/base/config.hip.hpp"
 #include "hip/base/math.hip.hpp"
 #include "hip/base/types.hip.hpp"
diff --git a/hip/test/solver/cb_gmres_kernels.cpp b/hip/test/solver/cb_gmres_kernels.cpp
index 5ba2743a289..872b23a1556 100644
--- a/hip/test/solver/cb_gmres_kernels.cpp
+++ b/hip/test/solver/cb_gmres_kernels.cpp
@@ -44,7 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm_reduction.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
 
 
 #include "core/solver/cb_gmres_accessor.hpp"
diff --git a/include/ginkgo/core/solver/cb_gmres.hpp b/include/ginkgo/core/solver/cb_gmres.hpp
index 473773f2c7e..d6334c11abc 100644
--- a/include/ginkgo/core/solver/cb_gmres.hpp
+++ b/include/ginkgo/core/solver/cb_gmres.hpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_PUBLIC_CORE_SOLVER_CB_GMRES_HPP_
 
 
+#include <memory>
 #include <vector>
 
 
diff --git a/omp/solver/cb_gmres_kernels.cpp b/omp/solver/cb_gmres_kernels.cpp
index 2bea7acc2c0..05ffe886632 100644
--- a/omp/solver/cb_gmres_kernels.cpp
+++ b/omp/solver/cb_gmres_kernels.cpp
@@ -36,11 +36,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <omp.h>
 
 
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/solver/cb_gmres.hpp>
+#include <ginkgo/core/stop/stopping_status.hpp>
+
+
+#include "core/solver/cb_gmres_accessor.hpp"
 
 
 namespace gko {
diff --git a/omp/test/solver/cb_gmres_kernels.cpp b/omp/test/solver/cb_gmres_kernels.cpp
index cbeb919be7c..27f891eb9fe 100644
--- a/omp/test/solver/cb_gmres_kernels.cpp
+++ b/omp/test/solver/cb_gmres_kernels.cpp
@@ -46,7 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm_reduction.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
 
 
 #include "core/solver/cb_gmres_accessor.hpp"
diff --git a/reference/solver/cb_gmres_kernels.cpp b/reference/solver/cb_gmres_kernels.cpp
index a3e940aaa76..65f4e248916 100644
--- a/reference/solver/cb_gmres_kernels.cpp
+++ b/reference/solver/cb_gmres_kernels.cpp
@@ -33,11 +33,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/cb_gmres_kernels.hpp"
 
 
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception_helpers.hpp>
+#include <type_traits>
+
+
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/solver/cb_gmres.hpp>
+#include <ginkgo/core/stop/stopping_status.hpp>
 
 
 #include "core/solver/cb_gmres_accessor.hpp"
diff --git a/reference/test/solver/cb_gmres_kernels.cpp b/reference/test/solver/cb_gmres_kernels.cpp
index 70d0a5fe94d..e78d9a91338 100644
--- a/reference/test/solver/cb_gmres_kernels.cpp
+++ b/reference/test/solver/cb_gmres_kernels.cpp
@@ -33,7 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/cb_gmres.hpp>
 
 
-#include <type_traits>
+#include <tuple>
 
 
 #include <gtest/gtest.h>
@@ -45,7 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/preconditioner/jacobi.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm_reduction.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
 #include <ginkgo/core/stop/time.hpp>
 
 

From e3817e027e8cf76c95be910c525a53c3c1a3a542 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 15 Feb 2021 03:21:24 +0100
Subject: [PATCH 0416/1008] Review update

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 benchmark/solver/solver.cpp                |  1 +
 common/solver/cb_gmres_kernels.hpp.inc     | 18 +++--
 core/solver/cb_gmres.cpp                   |  8 ++-
 cuda/solver/cb_gmres_kernels.cu            | 78 ++++++++++------------
 examples/cb-gmres/doc/intro.dox            |  2 +-
 hip/solver/cb_gmres_kernels.hip.cpp        | 77 ++++++++++-----------
 include/ginkgo/core/solver/cb_gmres.hpp    |  3 +-
 omp/solver/cb_gmres_kernels.cpp            | 18 ++---
 reference/solver/cb_gmres_kernels.cpp      | 18 ++---
 reference/test/solver/cb_gmres_kernels.cpp | 18 ++---
 10 files changed, 115 insertions(+), 126 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 2da3679cade..d73ae9151b6 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -229,6 +229,7 @@ std::unique_ptr<gko::LinOpFactory> add_criteria_precond_finalize(
         .on(exec);
 }
 
+
 template <typename Solver>
 std::unique_ptr<gko::LinOpFactory> add_criteria_precond_finalize(
     const std::shared_ptr<const gko::Executor> &exec,
diff --git a/common/solver/cb_gmres_kernels.hpp.inc b/common/solver/cb_gmres_kernels.hpp.inc
index 946d8aeac80..d2f3dfc5ce0 100644
--- a/common/solver/cb_gmres_kernels.hpp.inc
+++ b/common/solver/cb_gmres_kernels.hpp.inc
@@ -324,7 +324,7 @@ __global__ __launch_bounds__(default_dot_size) void multinorm2_inf_kernel(
 
 template <int dot_dim, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(dot_dim *dot_dim) void multidot_kernel(
-    size_type num_iters, size_type num_rows, size_type num_cols,
+    size_type num_rows, size_type num_cols,
     const ValueType *__restrict__ next_krylov_basis,
     size_type stride_next_krylov, const Accessor3d krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
@@ -383,8 +383,7 @@ __global__ __launch_bounds__(dot_dim *dot_dim) void multidot_kernel(
 
 template <int block_size, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(block_size) void singledot_kernel(
-    size_type num_iters, size_type num_rows,
-    const ValueType *__restrict__ next_krylov_basis,
+    size_type num_rows, const ValueType *__restrict__ next_krylov_basis,
     size_type stride_next_krylov, const Accessor3d krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status)
@@ -494,13 +493,12 @@ __global__ __launch_bounds__(block_size) void update_next_krylov_and_add_kernel(
 }
 
 
-// Must be called with at least `num_cols` blocks, each with `block_size`
-// threads. `block_size` must be a power of 2.
+// Must be called with at least `num_rhs` threads
 template <int block_size, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(block_size) void check_arnoldi_norms(
-    remove_complex<ValueType> *__restrict__ arnoldi_norm, size_type stride_norm,
-    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
-    size_type iter, Accessor3d krylov_bases,
+    size_type num_rhs, remove_complex<ValueType> *__restrict__ arnoldi_norm,
+    size_type stride_norm, ValueType *__restrict__ hessenberg_iter,
+    size_type stride_hessenberg, size_type iter, Accessor3d krylov_bases,
     const stopping_status *__restrict__ stop_status,
     stopping_status *__restrict__ reorth_status,
     size_type *__restrict__ num_reorth)
@@ -510,14 +508,14 @@ __global__ __launch_bounds__(block_size) void check_arnoldi_norms(
     constexpr bool has_scalar =
         gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value;
 
-    if (col_idx < stride_norm && !stop_status[col_idx].has_stopped()) {
+    if (col_idx < num_rhs && !stop_status[col_idx].has_stopped()) {
         const auto num0 = (sqrt(eta_squared * arnoldi_norm[col_idx]));
         const auto num11 = sqrt(arnoldi_norm[col_idx + stride_norm]);
         const auto num2 = has_scalar ? (arnoldi_norm[col_idx + 2 * stride_norm])
                                      : remove_complex<ValueType>{};
         if (num11 < num0) {
             reorth_status[col_idx].reset();
-            atomic_add(num_reorth, (size_type)1);
+            atomic_add(num_reorth, one<size_type>());
         } else {
             reorth_status[col_idx].stop(1);
         }
diff --git a/core/solver/cb_gmres.cpp b/core/solver/cb_gmres.cpp
index a77989c8228..c7b7384df1a 100644
--- a/core/solver/cb_gmres.cpp
+++ b/core/solver/cb_gmres.cpp
@@ -256,10 +256,14 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         bool one_changed{};
         Array<stopping_status> stop_status(this->get_executor(),
                                            dense_b->get_size()[1]);
+        // reorth_status and num_reorth are both helper variables for GPU
+        // implementations at the moment.
+        // num_reorth := Number of vectors which require a re-orthogonalization
+        // reorth_status := stopping status for the re-orthogonalization,
+        //                  marking which RHS requires one, and which does not
         Array<stopping_status> reorth_status(this->get_executor(),
                                              dense_b->get_size()[1]);
-        Array<size_type> num_reorth(this->get_executor(),
-                                    dense_b->get_size()[1]);
+        Array<size_type> num_reorth(this->get_executor(), 1);
 
         // Initialization
         exec->run(cb_gmres::make_initialize_1(
diff --git a/cuda/solver/cb_gmres_kernels.cu b/cuda/solver/cb_gmres_kernels.cu
index b2dc33a3634..817eb7d4364 100644
--- a/cuda/solver/cb_gmres_kernels.cu
+++ b/cuda/solver/cb_gmres_kernels.cu
@@ -211,15 +211,15 @@ GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(
 
 
 template <typename ValueType, typename Accessor3dim>
-void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
-                         matrix::Dense<ValueType> *next_krylov_basis,
-                         Accessor3dim krylov_bases,
-                         matrix::Dense<ValueType> *hessenberg_iter,
-                         matrix::Dense<ValueType> *buffer_iter,
-                         matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
-                         size_type iter, const stopping_status *stop_status,
-                         stopping_status *reorth_status,
-                         Array<size_type> *num_reorth)
+void finish_arnoldi_CGS(std::shared_ptr<const CudaExecutor> exec,
+                        matrix::Dense<ValueType> *next_krylov_basis,
+                        Accessor3dim krylov_bases,
+                        matrix::Dense<ValueType> *hessenberg_iter,
+                        matrix::Dense<ValueType> *buffer_iter,
+                        matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+                        size_type iter, const stopping_status *stop_status,
+                        stopping_status *reorth_status,
+                        Array<size_type> *num_reorth)
 {
     using non_complex = remove_complex<ValueType>;
     // optimization parameter
@@ -233,12 +233,9 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     const auto dim_size = next_krylov_basis->get_size();
     const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
                          exec->get_num_multiprocessor() * 2);
-    const dim3 grid_size_num_iters(
-        ceildiv(dim_size[1] * (iter + 1), default_dot_dim),
-        exec->get_num_multiprocessor() * 2);
-    const dim3 grid_size_num_iters_2(ceildiv(dim_size[1], default_dot_dim),
-                                     exec->get_num_multiprocessor() * 2,
-                                     iter + 1);
+    const dim3 grid_size_num_iters(ceildiv(dim_size[1], default_dot_dim),
+                                   exec->get_num_multiprocessor() * 2,
+                                   iter + 1);
     const dim3 block_size(default_dot_dim, default_dot_dim);
     // Note: having iter first (instead of row_idx information) is likely
     //       beneficial for avoiding atomic_add conflicts, but that needs
@@ -246,7 +243,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     const dim3 grid_size_iters_single(exec->get_num_multiprocessor() * 2,
                                       iter + 1);
     const dim3 block_size_iters_single(singledot_block_size);
-    size_type numReorth;
+    size_type num_reorth_host;
 
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<non_complex>());
@@ -257,8 +254,8 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     zero_matrix(iter + 1, dim_size[1], stride_hessenberg,
                 hessenberg_iter->get_values());
     if (dim_size[1] > 1) {
-        multidot_kernel<default_dot_dim><<<grid_size_num_iters_2, block_size>>>(
-            iter + 1, dim_size[0], dim_size[1],
+        multidot_kernel<default_dot_dim><<<grid_size_num_iters, block_size>>>(
+            dim_size[0], dim_size[1],
             as_cuda_type(next_krylov_basis->get_const_values()),
             stride_next_krylov, as_cuda_accessor(krylov_bases),
             as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
@@ -266,7 +263,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     } else {
         singledot_kernel<singledot_block_size>
             <<<grid_size_iters_single, block_size_iters_single>>>(
-                iter + 1, dim_size[0],
+                dim_size[0],
                 as_cuda_type(next_krylov_basis->get_const_values()),
                 stride_next_krylov, as_cuda_accessor(krylov_bases),
                 as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
@@ -304,21 +301,20 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
     components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
     check_arnoldi_norms<default_block_size>
         <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
-            as_cuda_type(arnoldi_norm->get_values()), stride_arnoldi,
-            as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-            iter + 1, as_cuda_accessor(krylov_bases), as_cuda_type(stop_status),
-            as_cuda_type(reorth_status), as_cuda_type(num_reorth->get_data()));
-    numReorth = 0;
-    exec->get_master()->copy_from(exec.get(), 1, num_reorth->get_const_data(),
-                                  &numReorth);
-    // numReorth <= number of next_krylov vector to be reorthogonalization
-    for (size_type l = 1; (numReorth > 0) && (l < 3); l++) {
+            dim_size[1], as_cuda_type(arnoldi_norm->get_values()),
+            stride_arnoldi, as_cuda_type(hessenberg_iter->get_values()),
+            stride_hessenberg, iter + 1, as_cuda_accessor(krylov_bases),
+            as_cuda_type(stop_status), as_cuda_type(reorth_status),
+            as_cuda_type(num_reorth->get_data()));
+    num_reorth_host = exec->copy_val_to_host(num_reorth->get_const_data());
+    // num_reorth_host := number of next_krylov vector to be reorthogonalization
+    for (size_type l = 1; (num_reorth_host > 0) && (l < 3); l++) {
         zero_matrix(iter + 1, dim_size[1], stride_buffer,
                     buffer_iter->get_values());
         if (dim_size[1] > 1) {
             multidot_kernel<default_dot_dim>
-                <<<grid_size_num_iters_2, block_size>>>(
-                    iter + 1, dim_size[0], dim_size[1],
+                <<<grid_size_num_iters, block_size>>>(
+                    dim_size[0], dim_size[1],
                     as_cuda_type(next_krylov_basis->get_const_values()),
                     stride_next_krylov, as_cuda_accessor(krylov_bases),
                     as_cuda_type(buffer_iter->get_values()), stride_buffer,
@@ -326,7 +322,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         } else {
             singledot_kernel<singledot_block_size>
                 <<<grid_size_iters_single, block_size_iters_single>>>(
-                    iter + 1, dim_size[0],
+                    dim_size[0],
                     as_cuda_type(next_krylov_basis->get_const_values()),
                     stride_next_krylov, as_cuda_accessor(krylov_bases),
                     as_cuda_type(buffer_iter->get_values()), stride_buffer,
@@ -335,7 +331,6 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
         // for i in 1:iter
         //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
         // end
-        //*
         update_next_krylov_and_add_kernel<default_block_size>
             <<<ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
                default_block_size>>>(
@@ -368,15 +363,12 @@ void finish_arnoldi_CGS2(std::shared_ptr<const CudaExecutor> exec,
                                zero<size_type>());
         check_arnoldi_norms<default_block_size>
             <<<ceildiv(dim_size[1], default_block_size), default_block_size>>>(
-                as_cuda_type(arnoldi_norm->get_values()), stride_arnoldi,
-                as_cuda_type(hessenberg_iter->get_values()), stride_hessenberg,
-                iter + 1, as_cuda_accessor(krylov_bases),
+                dim_size[1], as_cuda_type(arnoldi_norm->get_values()),
+                stride_arnoldi, as_cuda_type(hessenberg_iter->get_values()),
+                stride_hessenberg, iter + 1, as_cuda_accessor(krylov_bases),
                 as_cuda_type(stop_status), as_cuda_type(reorth_status),
                 as_cuda_type(num_reorth->get_data()));
-        numReorth = 0;
-        exec->get_master()->copy_from(exec.get(), 1,
-                                      num_reorth->get_const_data(), &numReorth);
-        // numReorth <= number of next_krylov vector to be reorthogonalization
+        num_reorth_host = exec->copy_val_to_host(num_reorth->get_const_data());
     }
 
     update_krylov_next_krylov_kernel<default_block_size>
@@ -440,10 +432,10 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
         default_block_size>>>(as_cuda_type(final_iter_nums->get_data()),
                               as_cuda_type(stop_status->get_const_data()),
                               final_iter_nums->get_num_elems());
-    finish_arnoldi_CGS2(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
-                        buffer_iter, arnoldi_norm, iter,
-                        stop_status->get_const_data(),
-                        reorth_status->get_data(), num_reorth);
+    finish_arnoldi_CGS(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
+                       buffer_iter, arnoldi_norm, iter,
+                       stop_status->get_const_data(), reorth_status->get_data(),
+                       num_reorth);
     givens_rotation(exec, givens_sin, givens_cos, hessenberg_iter,
                     residual_norm, residual_norm_collection, iter, stop_status);
 }
diff --git a/examples/cb-gmres/doc/intro.dox b/examples/cb-gmres/doc/intro.dox
index 06145d2d68c..f6fe753fb5f 100644
--- a/examples/cb-gmres/doc/intro.dox
+++ b/examples/cb-gmres/doc/intro.dox
@@ -9,7 +9,7 @@ solvers:
      both the matrix and the krylov basis, and
   2. with a compression of the krylov basis; it uses double precision for the
      matrix and all arithmetic operations, while using single precision for the
-     krylov basis
+     storage of the krylov basis
 
 Both solves are timed and the residual norm of each solution is computed to
 show that both solutions are correct.
diff --git a/hip/solver/cb_gmres_kernels.hip.cpp b/hip/solver/cb_gmres_kernels.hip.cpp
index 345f84fce31..7b899e1c118 100644
--- a/hip/solver/cb_gmres_kernels.hip.cpp
+++ b/hip/solver/cb_gmres_kernels.hip.cpp
@@ -214,15 +214,15 @@ GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(
 
 
 template <typename ValueType, typename Accessor3dim>
-void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
-                         matrix::Dense<ValueType> *next_krylov_basis,
-                         Accessor3dim krylov_bases,
-                         matrix::Dense<ValueType> *hessenberg_iter,
-                         matrix::Dense<ValueType> *buffer_iter,
-                         matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
-                         size_type iter, const stopping_status *stop_status,
-                         stopping_status *reorth_status,
-                         Array<size_type> *num_reorth)
+void finish_arnoldi_CGS(std::shared_ptr<const HipExecutor> exec,
+                        matrix::Dense<ValueType> *next_krylov_basis,
+                        Accessor3dim krylov_bases,
+                        matrix::Dense<ValueType> *hessenberg_iter,
+                        matrix::Dense<ValueType> *buffer_iter,
+                        matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+                        size_type iter, const stopping_status *stop_status,
+                        stopping_status *reorth_status,
+                        Array<size_type> *num_reorth)
 {
     using non_complex = remove_complex<ValueType>;
     // optimization parameter
@@ -236,12 +236,9 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
     const auto dim_size = next_krylov_basis->get_size();
     const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
                          exec->get_num_multiprocessor() * 2);
-    const dim3 grid_size_num_iters(
-        ceildiv(dim_size[1] * (iter + 1), default_dot_dim),
-        exec->get_num_multiprocessor() * 2);
-    const dim3 grid_size_num_iters_2(ceildiv(dim_size[1], default_dot_dim),
-                                     exec->get_num_multiprocessor() * 2,
-                                     iter + 1);
+    const dim3 grid_size_num_iters(ceildiv(dim_size[1], default_dot_dim),
+                                   exec->get_num_multiprocessor() * 2,
+                                   iter + 1);
     const dim3 block_size(default_dot_dim, default_dot_dim);
     // Note: having iter first (instead of row_idx information) is likely
     //       beneficial for avoiding atomic_add conflicts, but that needs
@@ -249,7 +246,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
     const dim3 grid_size_iters_single(exec->get_num_multiprocessor() * 2,
                                       iter + 1);
     const dim3 block_size_iters_single(singledot_block_size);
-    size_type numReorth;
+    size_type num_reorth_host;
 
     components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
                            zero<non_complex>());
@@ -263,8 +260,8 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
                 hessenberg_iter->get_values());
     if (dim_size[1] > 1) {
         hipLaunchKernelGGL(multidot_kernel<default_dot_dim>,
-                           grid_size_num_iters_2, block_size, 0, 0, iter + 1,
-                           dim_size[0], dim_size[1],
+                           grid_size_num_iters, block_size, 0, 0, dim_size[0],
+                           dim_size[1],
                            as_hip_type(next_krylov_basis->get_const_values()),
                            stride_next_krylov, as_hip_accessor(krylov_bases),
                            as_hip_type(hessenberg_iter->get_values()),
@@ -272,7 +269,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
     } else {
         hipLaunchKernelGGL(singledot_kernel<singledot_block_size>,
                            grid_size_iters_single, block_size_iters_single, 0,
-                           0, iter + 1, dim_size[0],
+                           0, dim_size[0],
                            as_hip_type(next_krylov_basis->get_const_values()),
                            stride_next_krylov, as_hip_accessor(krylov_bases),
                            as_hip_type(hessenberg_iter->get_values()),
@@ -312,22 +309,20 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
     hipLaunchKernelGGL(
         check_arnoldi_norms<default_block_size>,
         dim3(ceildiv(dim_size[1], default_block_size)),
-        dim3(default_block_size), 0, 0, as_hip_type(arnoldi_norm->get_values()),
-        stride_arnoldi, as_hip_type(hessenberg_iter->get_values()),
-        stride_hessenberg, iter + 1, as_hip_accessor(krylov_bases),
-        as_hip_type(stop_status), as_hip_type(reorth_status),
-        as_hip_type(num_reorth->get_data()));
-    numReorth = 0;
-    exec->get_master()->copy_from(exec.get(), 1, num_reorth->get_const_data(),
-                                  &numReorth);
-    // numReorth <= number of next_krylov vector to be reorthogonalization
-    for (size_type l = 1; (numReorth > 0) && (l < 3); l++) {
+        dim3(default_block_size), 0, 0, dim_size[1],
+        as_hip_type(arnoldi_norm->get_values()), stride_arnoldi,
+        as_hip_type(hessenberg_iter->get_values()), stride_hessenberg, iter + 1,
+        as_hip_accessor(krylov_bases), as_hip_type(stop_status),
+        as_hip_type(reorth_status), as_hip_type(num_reorth->get_data()));
+    num_reorth_host = exec->copy_val_to_host(num_reorth->get_const_data());
+    // num_reorth_host := number of next_krylov vector to be reorthogonalization
+    for (size_type l = 1; (num_reorth_host > 0) && (l < 3); l++) {
         zero_matrix(iter + 1, dim_size[1], stride_buffer,
                     buffer_iter->get_values());
         if (dim_size[1] > 1) {
             hipLaunchKernelGGL(
-                multidot_kernel<default_dot_dim>, grid_size_num_iters_2,
-                block_size, 0, 0, iter + 1, dim_size[0], dim_size[1],
+                multidot_kernel<default_dot_dim>, grid_size_num_iters,
+                block_size, 0, 0, dim_size[0], dim_size[1],
                 as_hip_type(next_krylov_basis->get_const_values()),
                 stride_next_krylov, as_hip_accessor(krylov_bases),
                 as_hip_type(buffer_iter->get_values()), stride_buffer,
@@ -335,7 +330,7 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
         } else {
             hipLaunchKernelGGL(
                 singledot_kernel<singledot_block_size>, grid_size_iters_single,
-                block_size_iters_single, 0, 0, iter + 1, dim_size[0],
+                block_size_iters_single, 0, 0, dim_size[0],
                 as_hip_type(next_krylov_basis->get_const_values()),
                 stride_next_krylov, as_hip_accessor(krylov_bases),
                 as_hip_type(buffer_iter->get_values()), stride_buffer,
@@ -344,7 +339,6 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
         // for i in 1:iter
         //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
         // end
-        //*
         hipLaunchKernelGGL(
             update_next_krylov_and_add_kernel<default_block_size>,
             dim3(ceildiv(dim_size[0] * stride_next_krylov, default_block_size)),
@@ -379,15 +373,14 @@ void finish_arnoldi_CGS2(std::shared_ptr<const HipExecutor> exec,
         hipLaunchKernelGGL(
             check_arnoldi_norms<default_block_size>,
             dim3(ceildiv(dim_size[1], default_block_size)),
-            dim3(default_block_size), 0, 0,
+            dim3(default_block_size), 0, 0, dim_size[1],
             as_hip_type(arnoldi_norm->get_values()), stride_arnoldi,
             as_hip_type(hessenberg_iter->get_values()), stride_hessenberg,
             iter + 1, as_hip_accessor(krylov_bases), as_hip_type(stop_status),
             as_hip_type(reorth_status), as_hip_type(num_reorth->get_data()));
-        numReorth = 0;
-        exec->get_master()->copy_from(exec.get(), 1,
-                                      num_reorth->get_const_data(), &numReorth);
-        // numReorth <= number of next_krylov vector to be reorthogonalization
+        num_reorth_host = exec->copy_val_to_host(num_reorth->get_const_data());
+        // num_reorth_host := number of next_krylov vector to be
+        // reorthogonalization
     }
 
     hipLaunchKernelGGL(
@@ -454,10 +447,10 @@ void step_1(std::shared_ptr<const HipExecutor> exec,
         as_hip_type(final_iter_nums->get_data()),
         as_hip_type(stop_status->get_const_data()),
         final_iter_nums->get_num_elems());
-    finish_arnoldi_CGS2(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
-                        buffer_iter, arnoldi_norm, iter,
-                        stop_status->get_const_data(),
-                        reorth_status->get_data(), num_reorth);
+    finish_arnoldi_CGS(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
+                       buffer_iter, arnoldi_norm, iter,
+                       stop_status->get_const_data(), reorth_status->get_data(),
+                       num_reorth);
     givens_rotation(exec, givens_sin, givens_cos, hessenberg_iter,
                     residual_norm, residual_norm_collection, iter, stop_status);
 }
diff --git a/include/ginkgo/core/solver/cb_gmres.hpp b/include/ginkgo/core/solver/cb_gmres.hpp
index d6334c11abc..08af9de1766 100644
--- a/include/ginkgo/core/solver/cb_gmres.hpp
+++ b/include/ginkgo/core/solver/cb_gmres.hpp
@@ -103,7 +103,8 @@ enum class storage_precision {
  *
  * The implementation in Ginkgo makes use of the merged kernel to make the best
  * use of data locality. The inner operations in one iteration of CB-GMRES
- * are merged into 2 separate steps. Classical Gram-Schmidt is used.
+ * are merged into 2 separate steps. Classical Gram-Schmidt with
+ * reorthogonalization is used.
  *
  * The krylov basis can be stored in reduced precision (compressed) to reduce
  * memory accesses, while all computations (including krylov basis operations)
diff --git a/omp/solver/cb_gmres_kernels.cpp b/omp/solver/cb_gmres_kernels.cpp
index 05ffe886632..b1a87d0e871 100644
--- a/omp/solver/cb_gmres_kernels.cpp
+++ b/omp/solver/cb_gmres_kernels.cpp
@@ -60,12 +60,12 @@ namespace {
 
 
 template <typename ValueType, typename Accessor3d>
-void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
-                         Accessor3d krylov_bases,
-                         matrix::Dense<ValueType> *hessenberg_iter,
-                         matrix::Dense<ValueType> *buffer_iter,
-                         matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
-                         size_type iter, const stopping_status *stop_status)
+void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
+                        Accessor3d krylov_bases,
+                        matrix::Dense<ValueType> *hessenberg_iter,
+                        matrix::Dense<ValueType> *buffer_iter,
+                        matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+                        size_type iter, const stopping_status *stop_status)
 {
     using rc_vtype = remove_complex<ValueType>;
     constexpr bool has_scalar =
@@ -442,9 +442,9 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
             (1 - static_cast<size_type>(
                      stop_status->get_const_data()[i].has_stopped()));
     }
-    finish_arnoldi_CGS2(next_krylov_basis, krylov_bases, hessenberg_iter,
-                        buffer_iter, arnoldi_norm, iter,
-                        stop_status->get_const_data());
+    finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
+                       buffer_iter, arnoldi_norm, iter,
+                       stop_status->get_const_data());
     givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
                     iter, stop_status->get_const_data());
     calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,
diff --git a/reference/solver/cb_gmres_kernels.cpp b/reference/solver/cb_gmres_kernels.cpp
index 65f4e248916..a4bcd6289c9 100644
--- a/reference/solver/cb_gmres_kernels.cpp
+++ b/reference/solver/cb_gmres_kernels.cpp
@@ -59,12 +59,12 @@ namespace {
 
 
 template <typename ValueType, typename Accessor3d>
-void finish_arnoldi_CGS2(matrix::Dense<ValueType> *next_krylov_basis,
-                         Accessor3d krylov_bases,
-                         matrix::Dense<ValueType> *hessenberg_iter,
-                         matrix::Dense<ValueType> *buffer_iter,
-                         matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
-                         size_type iter, const stopping_status *stop_status)
+void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
+                        Accessor3d krylov_bases,
+                        matrix::Dense<ValueType> *hessenberg_iter,
+                        matrix::Dense<ValueType> *buffer_iter,
+                        matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+                        size_type iter, const stopping_status *stop_status)
 {
     static_assert(
         std::is_same<ValueType,
@@ -422,9 +422,9 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
             (1 - static_cast<size_type>(
                      stop_status->get_const_data()[i].has_stopped()));
     }
-    finish_arnoldi_CGS2(next_krylov_basis, krylov_bases, hessenberg_iter,
-                        buffer_iter, arnoldi_norm, iter,
-                        stop_status->get_const_data());
+    finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
+                       buffer_iter, arnoldi_norm, iter,
+                       stop_status->get_const_data());
     givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
                     iter, stop_status->get_const_data());
     calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,
diff --git a/reference/test/solver/cb_gmres_kernels.cpp b/reference/test/solver/cb_gmres_kernels.cpp
index e78d9a91338..ae5e3fa4ec9 100644
--- a/reference/test/solver/cb_gmres_kernels.cpp
+++ b/reference/test/solver/cb_gmres_kernels.cpp
@@ -314,15 +314,15 @@ TYPED_TEST(CbGmres, SolvesBigDenseSystem2)
 
 
 template <typename T>
-gko::remove_complex<T> infNorm(gko::matrix::Dense<T> *mat, size_t col = 0)
+gko::remove_complex<T> inf_norm(gko::matrix::Dense<T> *mat, size_t col = 0)
 {
     using std::abs;
     auto host_data = clone(mat->get_executor()->get_master(), mat);
     using no_cpx_t = gko::remove_complex<T>;
     no_cpx_t norm = 0.0;
     for (size_t i = 0; i < host_data->get_size()[0]; ++i) {
-        no_cpx_t absEntry = abs(host_data->at(i, col));
-        if (norm < absEntry) norm = absEntry;
+        no_cpx_t abs_entry = abs(host_data->at(i, col));
+        if (norm < abs_entry) norm = abs_entry;
     }
     return norm;
 }
@@ -378,12 +378,12 @@ TYPED_TEST(CbGmres, SolvesMultipleDenseSystemForDivergenceCheck)
     this->mtx_big->apply(alpha.get(), x2.get(), beta.get(), residual2.get());
     this->mtx_big->apply(alpha.get(), xc.get(), beta.get(), residualC.get());
 
-    auto normS1 = infNorm(residual1.get());
-    auto normS2 = infNorm(residual2.get());
-    auto normC1 = infNorm(residualC.get(), 0);
-    auto normC2 = infNorm(residualC.get(), 1);
-    auto normB1 = infNorm(b1.get());
-    auto normB2 = infNorm(b2.get());
+    auto normS1 = inf_norm(residual1.get());
+    auto normS2 = inf_norm(residual2.get());
+    auto normC1 = inf_norm(residualC.get(), 0);
+    auto normC2 = inf_norm(residualC.get(), 1);
+    auto normB1 = inf_norm(b1.get());
+    auto normB2 = inf_norm(b2.get());
 
     // make sure that all combined solutions are as good or better than the
     // single solutions

From 5ecd29f28e7697a80af49ec512037544609a4833 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 16 Feb 2021 05:46:54 +0100
Subject: [PATCH 0417/1008] Review update

- Extract GPU kernels for CB-GMRES and GMRES into a new file to avoid
  duplication.
- Adopt the updated GMRES functionality for these kernels for CPU and
  GPU

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 common/solver/cb_gmres_kernels.hpp.inc     | 190 +-------------------
 common/solver/common_gmres_kernels.hpp.inc | 199 +++++++++++++++++++++
 common/solver/gmres_kernels.hpp.inc        | 168 +----------------
 omp/solver/cb_gmres_kernels.cpp            |  36 ++--
 reference/solver/cb_gmres_kernels.cpp      |  36 ++--
 5 files changed, 237 insertions(+), 392 deletions(-)
 create mode 100644 common/solver/common_gmres_kernels.hpp.inc

diff --git a/common/solver/cb_gmres_kernels.hpp.inc b/common/solver/cb_gmres_kernels.hpp.inc
index d2f3dfc5ce0..524be799374 100644
--- a/common/solver/cb_gmres_kernels.hpp.inc
+++ b/common/solver/cb_gmres_kernels.hpp.inc
@@ -30,6 +30,9 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include "common_gmres_kernels.hpp.inc"
+
+
 template <typename ValueType>
 __global__ __launch_bounds__(default_block_size) void zero_matrix_kernel(
     size_type m, size_type n, size_type stride, ValueType *__restrict__ array)
@@ -45,41 +48,6 @@ __global__ __launch_bounds__(default_block_size) void zero_matrix_kernel(
 }
 
 
-// Must be called with at least `max(stride_b * num_rows, krylov_dim *
-// num_cols)` threads in total.
-template <size_type block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void initialize_1_kernel(
-    size_type num_rows, size_type num_cols, size_type krylov_dim,
-    const ValueType *__restrict__ b, size_type stride_b,
-    ValueType *__restrict__ residual, size_type stride_residual,
-    ValueType *__restrict__ givens_sin, size_type stride_sin,
-    ValueType *__restrict__ givens_cos, size_type stride_cos,
-    stopping_status *__restrict__ stop_status)
-{
-    const auto global_id = thread::get_thread_id_flat();
-
-    const auto row_idx = global_id / stride_b;
-    const auto col_idx = global_id % stride_b;
-
-    if (global_id < num_cols) {
-        stop_status[global_id].reset();
-    }
-
-    if (row_idx < num_rows && col_idx < num_cols) {
-        residual[row_idx * stride_residual + col_idx] =
-            b[row_idx * stride_b + col_idx];
-    }
-
-    if (global_id < krylov_dim * num_cols) {
-        const auto row_givens = global_id / num_cols;
-        const auto col_givens = global_id % num_cols;
-
-        givens_sin[row_givens * stride_sin + col_givens] = zero<ValueType>();
-        givens_cos[row_givens * stride_cos + col_givens] = zero<ValueType>();
-    }
-}
-
-
 // Must be called with at least `num_rows * stride_krylov` threads in total.
 template <size_type block_size, typename ValueType, typename Accessor3d>
 __global__ __launch_bounds__(block_size) void initialize_2_1_kernel(
@@ -588,158 +556,6 @@ __global__ __launch_bounds__(block_size) void update_krylov_next_krylov_kernel(
 }
 
 
-template <typename ValueType>
-__device__ void calculate_sin_and_cos_kernel(
-    size_type col_idx, size_type num_cols, size_type iter,
-    const ValueType *hessenberg_iter, size_type stride_hessenberg,
-    ValueType *givens_sin, size_type stride_sin, ValueType *givens_cos,
-    size_type stride_cos)
-{
-    if (hessenberg_iter[iter * stride_hessenberg + col_idx] ==
-        zero<ValueType>()) {
-        givens_cos[iter * stride_cos + col_idx] = zero<ValueType>();
-        givens_sin[iter * stride_sin + col_idx] = one<ValueType>();
-    } else {
-        auto hypotenuse =
-            sqrt(hessenberg_iter[iter * stride_hessenberg + col_idx] *
-                     hessenberg_iter[iter * stride_hessenberg + col_idx] +
-                 hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] *
-                     hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx]);
-        givens_cos[iter * stride_cos + col_idx] =
-            abs(hessenberg_iter[iter * stride_hessenberg + col_idx]) /
-            hypotenuse;
-        givens_sin[iter * stride_sin + col_idx] =
-            givens_cos[iter * stride_cos + col_idx] *
-            hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] /
-            hessenberg_iter[iter * stride_hessenberg + col_idx];
-    }
-}
-
-
-template <typename ValueType>
-__device__ void calculate_residual_norm_kernel(
-    size_type col_idx, size_type num_cols, size_type iter,
-    const ValueType *givens_sin, size_type stride_sin,
-    const ValueType *givens_cos, size_type stride_cos,
-    remove_complex<ValueType> *residual_norm,
-    ValueType *residual_norm_collection,
-    size_type stride_residual_norm_collection)
-{
-    residual_norm_collection[(iter + 1) * stride_residual_norm_collection +
-                             col_idx] =
-        -conj(givens_sin[iter * stride_sin + col_idx]) *
-        residual_norm_collection[iter * stride_residual_norm_collection +
-                                 col_idx];
-    residual_norm_collection[iter * stride_residual_norm_collection + col_idx] =
-        givens_cos[iter * stride_cos + col_idx] *
-        residual_norm_collection[iter * stride_residual_norm_collection +
-                                 col_idx];
-    residual_norm[col_idx] = abs(
-        residual_norm_collection[(iter + 1) * stride_residual_norm_collection +
-                                 col_idx]);
-}
-
-
-// Must be called with at least `num_cols` threads in total.
-template <size_type block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void givens_rotation_kernel(
-    size_type num_rows, size_type num_cols, size_type iter,
-    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
-    ValueType *__restrict__ givens_sin, size_type stride_sin,
-    ValueType *__restrict__ givens_cos, size_type stride_cos,
-    remove_complex<ValueType> *__restrict__ residual_norm,
-    ValueType *__restrict__ residual_norm_collection,
-    size_type stride_residual_norm_collection,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto col_idx = thread::get_thread_id_flat();
-
-    if (col_idx >= num_cols || stop_status[col_idx].has_stopped()) {
-        return;
-    }
-
-    const auto current_thread_block = group::this_thread_block();
-
-    for (size_type i = 0; i < iter; ++i) {
-        const auto tmp =
-            givens_cos[i * stride_cos + col_idx] *
-                hessenberg_iter[i * stride_hessenberg + col_idx] +
-            givens_sin[i * stride_sin + col_idx] *
-                hessenberg_iter[(i + 1) * stride_hessenberg + col_idx];
-        current_thread_block.sync();
-        hessenberg_iter[(i + 1) * stride_hessenberg + col_idx] =
-            givens_cos[i * stride_cos + col_idx] *
-                hessenberg_iter[(i + 1) * stride_hessenberg + col_idx] -
-            givens_sin[i * stride_sin + col_idx] *
-                hessenberg_iter[i * stride_hessenberg + col_idx];
-        hessenberg_iter[i * stride_hessenberg + col_idx] = tmp;
-        current_thread_block.sync();
-    }
-    // for j in 1:iter - 1
-    //     temp             =  cos(j)*hessenberg(j) +
-    //                         sin(j)*hessenberg(j+1)
-    //     hessenberg(j+1)  = -sin(j)*hessenberg(j) +
-    //                         cos(j)*hessenberg(j+1)
-    //     hessenberg(j)    =  temp;
-    // end
-
-    calculate_sin_and_cos_kernel(col_idx, num_cols, iter, hessenberg_iter,
-                                 stride_hessenberg, givens_sin, stride_sin,
-                                 givens_cos, stride_cos);
-    // Calculate sin and cos
-
-    hessenberg_iter[iter * stride_hessenberg + col_idx] =
-        givens_cos[iter * stride_cos + col_idx] *
-            hessenberg_iter[iter * stride_hessenberg + col_idx] +
-        givens_sin[iter * stride_sin + col_idx] *
-            hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx];
-    hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] =
-        zero<ValueType>();
-    // hessenberg(iter)   = cos(iter)*hessenberg(iter) +
-    //                      sin(iter)*hessenberg(iter)
-    // hessenberg(iter+1) = 0
-
-    calculate_residual_norm_kernel(col_idx, num_cols, iter, givens_sin,
-                                   stride_sin, givens_cos, stride_cos,
-                                   residual_norm, residual_norm_collection,
-                                   stride_residual_norm_collection);
-    // Calculate residual norm
-}
-
-
-// Must be called with at least `num_rhs` threads in total.
-template <size_type block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void solve_upper_triangular_kernel(
-    size_type num_cols, size_type num_rhs,
-    const ValueType *__restrict__ residual_norm_collection,
-    size_type stride_residual_norm_collection,
-    const ValueType *__restrict__ hessenberg, size_type stride_hessenberg,
-    ValueType *__restrict__ y, size_type stride_y,
-    const size_type *__restrict__ final_iter_nums)
-{
-    const auto col_idx = thread::get_thread_id_flat();
-
-    if (col_idx >= num_rhs) {
-        return;
-    }
-
-    for (int64 i = final_iter_nums[col_idx] - 1; i >= 0; --i) {
-        auto temp =
-            residual_norm_collection[i * stride_residual_norm_collection +
-                                     col_idx];
-        for (size_type j = i + 1; j < final_iter_nums[col_idx]; ++j) {
-            temp -= hessenberg[i * stride_hessenberg + j * num_rhs + col_idx] *
-                    y[j * stride_y + col_idx];
-        }
-
-        y[i * stride_y + col_idx] =
-            temp / hessenberg[i * stride_hessenberg + i * num_rhs + col_idx];
-    }
-    // Solve upper triangular.
-    // y = hessenberg \ residual_norm_collection
-}
-
-
 // Must be called with at least `stride_preconditioner * num_rows` threads
 // in total.
 template <size_type block_size, typename ValueType, typename Accessor3d>
diff --git a/common/solver/common_gmres_kernels.hpp.inc b/common/solver/common_gmres_kernels.hpp.inc
new file mode 100644
index 00000000000..84ee6f52f03
--- /dev/null
+++ b/common/solver/common_gmres_kernels.hpp.inc
@@ -0,0 +1,199 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+// Must be called with at least `max(stride_b * num_rows, krylov_dim *
+// num_cols)` threads in total.
+template <size_type block_size, typename ValueType>
+__global__ __launch_bounds__(block_size) void initialize_1_kernel(
+    size_type num_rows, size_type num_cols, size_type krylov_dim,
+    const ValueType *__restrict__ b, size_type stride_b,
+    ValueType *__restrict__ residual, size_type stride_residual,
+    ValueType *__restrict__ givens_sin, size_type stride_sin,
+    ValueType *__restrict__ givens_cos, size_type stride_cos,
+    stopping_status *__restrict__ stop_status)
+{
+    const auto global_id = thread::get_thread_id_flat();
+
+    const auto row_idx = global_id / stride_b;
+    const auto col_idx = global_id % stride_b;
+
+    if (global_id < num_cols) {
+        stop_status[global_id].reset();
+    }
+
+    if (row_idx < num_rows && col_idx < num_cols) {
+        residual[row_idx * stride_residual + col_idx] =
+            b[row_idx * stride_b + col_idx];
+    }
+
+    if (global_id < krylov_dim * num_cols) {
+        const auto row_givens = global_id / num_cols;
+        const auto col_givens = global_id % num_cols;
+
+        givens_sin[row_givens * stride_sin + col_givens] = zero<ValueType>();
+        givens_cos[row_givens * stride_cos + col_givens] = zero<ValueType>();
+    }
+}
+
+
+template <typename ValueType>
+__device__ void calculate_sin_and_cos_kernel(
+    size_type col_idx, size_type num_cols, size_type iter,
+    const ValueType &this_hess, const ValueType &next_hess,
+    ValueType *givens_sin, size_type stride_sin, ValueType *givens_cos,
+    size_type stride_cos, ValueType &register_sin, ValueType &register_cos)
+{
+    if (this_hess == zero<ValueType>()) {
+        register_cos = zero<ValueType>();
+        register_sin = one<ValueType>();
+    } else {
+        const auto scale = abs(this_hess) + abs(next_hess);
+        const auto hypotenuse =
+            scale * sqrt(abs(this_hess / scale) * abs(this_hess / scale) +
+                         abs(next_hess / scale) * abs(next_hess / scale));
+        register_cos = conj(this_hess) / hypotenuse;
+        register_sin = conj(next_hess) / hypotenuse;
+    }
+    givens_cos[iter * stride_cos + col_idx] = register_cos;
+    givens_sin[iter * stride_sin + col_idx] = register_sin;
+}
+
+
+template <typename ValueType>
+__device__ void calculate_residual_norm_kernel(
+    size_type col_idx, size_type num_cols, size_type iter,
+    const ValueType &register_sin, const ValueType &register_cos,
+    remove_complex<ValueType> *residual_norm,
+    ValueType *residual_norm_collection,
+    size_type stride_residual_norm_collection)
+{
+    const auto this_rnc =
+        residual_norm_collection[iter * stride_residual_norm_collection +
+                                 col_idx];
+    const auto next_rnc = -conj(register_sin) * this_rnc;
+    residual_norm_collection[iter * stride_residual_norm_collection + col_idx] =
+        register_cos * this_rnc;
+    residual_norm[col_idx] = abs(next_rnc);
+    residual_norm_collection[(iter + 1) * stride_residual_norm_collection +
+                             col_idx] = next_rnc;
+}
+
+
+// Must be called with at least `num_cols` threads in total.
+template <size_type block_size, typename ValueType>
+__global__ __launch_bounds__(block_size) void givens_rotation_kernel(
+    size_type num_rows, size_type num_cols, size_type iter,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    ValueType *__restrict__ givens_sin, size_type stride_sin,
+    ValueType *__restrict__ givens_cos, size_type stride_cos,
+    remove_complex<ValueType> *__restrict__ residual_norm,
+    ValueType *__restrict__ residual_norm_collection,
+    size_type stride_residual_norm_collection,
+    const stopping_status *__restrict__ stop_status)
+{
+    const auto col_idx = thread::get_thread_id_flat();
+
+    if (col_idx >= num_cols || stop_status[col_idx].has_stopped()) {
+        return;
+    }
+
+    auto this_hess = hessenberg_iter[col_idx];
+    auto next_hess = hessenberg_iter[stride_hessenberg + col_idx];
+    for (size_type i = 0; i < iter; ++i) {
+        const auto cos = givens_cos[i * stride_cos + col_idx];
+        const auto sin = givens_sin[i * stride_sin + col_idx];
+        hessenberg_iter[i * stride_hessenberg + col_idx] =
+            cos * this_hess + sin * next_hess;
+        this_hess = conj(cos) * next_hess - conj(sin) * this_hess;
+        next_hess = hessenberg_iter[(i + 2) * stride_hessenberg + col_idx];
+    }
+    // for j in 0:iter - 1
+    //     temp             =  cos(j)*hessenberg(j) +
+    //                         sin(j)*hessenberg(j+1)
+    //     hessenberg(j+1)  = -sin(j)*hessenberg(j) +
+    //                         cos(j)*hessenberg(j+1)
+    //     hessenberg(j)    =  temp;
+    // end
+
+    ValueType register_sin;
+    ValueType register_cos;
+    calculate_sin_and_cos_kernel(col_idx, num_cols, iter, this_hess, next_hess,
+                                 givens_sin, stride_sin, givens_cos, stride_cos,
+                                 register_sin, register_cos);
+    // Calculate sin and cos on hessenberg(iter) and hessenberg(iter+1)
+
+    hessenberg_iter[iter * stride_hessenberg + col_idx] =
+        register_cos * this_hess + register_sin * next_hess;
+    hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] =
+        zero<ValueType>();
+    // hessenberg(iter)   = cos(iter)*hessenberg(iter) +
+    //                      sin(iter)*hessenberg(iter+1)
+    // hessenberg(iter+1) = 0
+
+    calculate_residual_norm_kernel(
+        col_idx, num_cols, iter, register_sin, register_cos, residual_norm,
+        residual_norm_collection, stride_residual_norm_collection);
+    // Calculate residual norm
+}
+
+
+// Must be called with at least `num_rhs` threads in total.
+template <size_type block_size, typename ValueType>
+__global__ __launch_bounds__(block_size) void solve_upper_triangular_kernel(
+    size_type num_cols, size_type num_rhs,
+    const ValueType *__restrict__ residual_norm_collection,
+    size_type stride_residual_norm_collection,
+    const ValueType *__restrict__ hessenberg, size_type stride_hessenberg,
+    ValueType *__restrict__ y, size_type stride_y,
+    const size_type *__restrict__ final_iter_nums)
+{
+    const auto col_idx = thread::get_thread_id_flat();
+
+    if (col_idx >= num_rhs) {
+        return;
+    }
+
+    for (int i = final_iter_nums[col_idx] - 1; i >= 0; --i) {
+        auto temp =
+            residual_norm_collection[i * stride_residual_norm_collection +
+                                     col_idx];
+        for (size_type j = i + 1; j < final_iter_nums[col_idx]; ++j) {
+            temp -= hessenberg[i * stride_hessenberg + j * num_rhs + col_idx] *
+                    y[j * stride_y + col_idx];
+        }
+
+        y[i * stride_y + col_idx] =
+            temp / hessenberg[i * stride_hessenberg + i * num_rhs + col_idx];
+    }
+    // Solve upper triangular.
+    // y = hessenberg \ residual_norm_collection
+}
diff --git a/common/solver/gmres_kernels.hpp.inc b/common/solver/gmres_kernels.hpp.inc
index 9956c38a622..1f44ea93cb8 100644
--- a/common/solver/gmres_kernels.hpp.inc
+++ b/common/solver/gmres_kernels.hpp.inc
@@ -30,39 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-// Must be called with at least `max(stride_b * num_rows, krylov_dim *
-// num_cols)` threads in total.
-template <size_type block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void initialize_1_kernel(
-    size_type num_rows, size_type num_cols, size_type krylov_dim,
-    const ValueType *__restrict__ b, size_type stride_b,
-    ValueType *__restrict__ residual, size_type stride_residual,
-    ValueType *__restrict__ givens_sin, size_type stride_sin,
-    ValueType *__restrict__ givens_cos, size_type stride_cos,
-    stopping_status *__restrict__ stop_status)
-{
-    const auto global_id = thread::get_thread_id_flat();
-
-    const auto row_idx = global_id / stride_b;
-    const auto col_idx = global_id % stride_b;
-
-    if (global_id < num_cols) {
-        stop_status[global_id].reset();
-    }
-
-    if (row_idx < num_rows && col_idx < num_cols) {
-        residual[row_idx * stride_residual + col_idx] =
-            b[row_idx * stride_b + col_idx];
-    }
-
-    if (global_id < krylov_dim * num_cols) {
-        const auto row_givens = global_id / num_cols;
-        const auto col_givens = global_id % num_cols;
-
-        givens_sin[row_givens * stride_sin + col_givens] = zero<ValueType>();
-        givens_cos[row_givens * stride_cos + col_givens] = zero<ValueType>();
-    }
-}
+#include "common_gmres_kernels.hpp.inc"
 
 
 // Must be called with at least `num_rows * num_rhs` threads in total.
@@ -243,140 +211,6 @@ __global__ __launch_bounds__(block_size) void update_krylov_kernel(
 }
 
 
-template <typename ValueType>
-__device__ void calculate_sin_and_cos_kernel(
-    size_type col_idx, size_type num_cols, size_type iter,
-    const ValueType &this_hess, const ValueType &next_hess,
-    ValueType *givens_sin, size_type stride_sin, ValueType *givens_cos,
-    size_type stride_cos, ValueType &register_sin, ValueType &register_cos)
-{
-    if (this_hess == zero<ValueType>()) {
-        register_cos = zero<ValueType>();
-        register_sin = one<ValueType>();
-    } else {
-        const auto scale = abs(this_hess) + abs(next_hess);
-        const auto hypotenuse =
-            scale * sqrt(abs(this_hess / scale) * abs(this_hess / scale) +
-                         abs(next_hess / scale) * abs(next_hess / scale));
-        register_cos = conj(this_hess) / hypotenuse;
-        register_sin = conj(next_hess) / hypotenuse;
-    }
-    givens_cos[iter * stride_cos + col_idx] = register_cos;
-    givens_sin[iter * stride_sin + col_idx] = register_sin;
-}
-
-
-template <typename ValueType>
-__device__ void calculate_residual_norm_kernel(
-    size_type col_idx, size_type num_cols, size_type iter,
-    const ValueType &register_sin, const ValueType &register_cos,
-    remove_complex<ValueType> *residual_norm,
-    ValueType *residual_norm_collection,
-    size_type stride_residual_norm_collection)
-{
-    const auto this_rnc =
-        residual_norm_collection[iter * stride_residual_norm_collection +
-                                 col_idx];
-    const auto next_rnc = -conj(register_sin) * this_rnc;
-    residual_norm_collection[iter * stride_residual_norm_collection + col_idx] =
-        register_cos * this_rnc;
-    residual_norm[col_idx] = abs(next_rnc);
-    residual_norm_collection[(iter + 1) * stride_residual_norm_collection +
-                             col_idx] = next_rnc;
-}
-
-
-// Must be called with at least `num_cols` threads in total.
-template <size_type block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void givens_rotation_kernel(
-    size_type num_rows, size_type num_cols, size_type iter,
-    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
-    ValueType *__restrict__ givens_sin, size_type stride_sin,
-    ValueType *__restrict__ givens_cos, size_type stride_cos,
-    remove_complex<ValueType> *__restrict__ residual_norm,
-    ValueType *__restrict__ residual_norm_collection,
-    size_type stride_residual_norm_collection,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto col_idx = thread::get_thread_id_flat();
-
-    if (col_idx >= num_cols || stop_status[col_idx].has_stopped()) {
-        return;
-    }
-
-    auto this_hess = hessenberg_iter[col_idx];
-    auto next_hess = hessenberg_iter[stride_hessenberg + col_idx];
-    for (size_type i = 0; i < iter; ++i) {
-        const auto cos = givens_cos[i * stride_cos + col_idx];
-        const auto sin = givens_sin[i * stride_sin + col_idx];
-        hessenberg_iter[i * stride_hessenberg + col_idx] =
-            cos * this_hess + sin * next_hess;
-        this_hess = conj(cos) * next_hess - conj(sin) * this_hess;
-        next_hess = hessenberg_iter[(i + 2) * stride_hessenberg + col_idx];
-    }
-    // for j in 0:iter - 1
-    //     temp             =  cos(j)*hessenberg(j) +
-    //                         sin(j)*hessenberg(j+1)
-    //     hessenberg(j+1)  = -sin(j)*hessenberg(j) +
-    //                         cos(j)*hessenberg(j+1)
-    //     hessenberg(j)    =  temp;
-    // end
-
-    ValueType register_sin;
-    ValueType register_cos;
-    calculate_sin_and_cos_kernel(col_idx, num_cols, iter, this_hess, next_hess,
-                                 givens_sin, stride_sin, givens_cos, stride_cos,
-                                 register_sin, register_cos);
-    // Calculate sin and cos on hessenberg(iter) and hessenberg(iter+1)
-
-    hessenberg_iter[iter * stride_hessenberg + col_idx] =
-        register_cos * this_hess + register_sin * next_hess;
-    hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] =
-        zero<ValueType>();
-    // hessenberg(iter)   = cos(iter)*hessenberg(iter) +
-    //                      sin(iter)*hessenberg(iter+1)
-    // hessenberg(iter+1) = 0
-
-    calculate_residual_norm_kernel(
-        col_idx, num_cols, iter, register_sin, register_cos, residual_norm,
-        residual_norm_collection, stride_residual_norm_collection);
-    // Calculate residual norm
-}
-
-
-// Must be called with at least `num_rhs` threads in total.
-template <size_type block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void solve_upper_triangular_kernel(
-    size_type num_cols, size_type num_rhs,
-    const ValueType *__restrict__ residual_norm_collection,
-    size_type stride_residual_norm_collection,
-    const ValueType *__restrict__ hessenberg, size_type stride_hessenberg,
-    ValueType *__restrict__ y, size_type stride_y,
-    const size_type *__restrict__ final_iter_nums)
-{
-    const auto col_idx = thread::get_thread_id_flat();
-
-    if (col_idx >= num_rhs) {
-        return;
-    }
-
-    for (int i = final_iter_nums[col_idx] - 1; i >= 0; --i) {
-        auto temp =
-            residual_norm_collection[i * stride_residual_norm_collection +
-                                     col_idx];
-        for (size_type j = i + 1; j < final_iter_nums[col_idx]; ++j) {
-            temp -= hessenberg[i * stride_hessenberg + j * num_rhs + col_idx] *
-                    y[j * stride_y + col_idx];
-        }
-
-        y[i * stride_y + col_idx] =
-            temp / hessenberg[i * stride_hessenberg + i * num_rhs + col_idx];
-    }
-    // Solve upper triangular.
-    // y = hessenberg \ residual_norm_collection
-}
-
-
 // Must be called with at least `stride_preconditioner * num_rows` threads in
 // total.
 template <size_type block_size, typename ValueType>
diff --git a/omp/solver/cb_gmres_kernels.cpp b/omp/solver/cb_gmres_kernels.cpp
index b1a87d0e871..46a982e44bb 100644
--- a/omp/solver/cb_gmres_kernels.cpp
+++ b/omp/solver/cb_gmres_kernels.cpp
@@ -198,28 +198,26 @@ void calculate_sin_and_cos(matrix::Dense<ValueType> *givens_sin,
         givens_cos->at(iter, rhs) = zero<ValueType>();
         givens_sin->at(iter, rhs) = one<ValueType>();
     } else {
-        auto hypotenuse = sqrt(hessenberg_iter->at(iter, rhs) *
-                                   hessenberg_iter->at(iter, rhs) +
-                               hessenberg_iter->at(iter + 1, rhs) *
-                                   hessenberg_iter->at(iter + 1, rhs));
-        givens_cos->at(iter, rhs) =
-            abs(hessenberg_iter->at(iter, rhs)) / hypotenuse;
-        givens_sin->at(iter, rhs) = givens_cos->at(iter, rhs) *
-                                    hessenberg_iter->at(iter + 1, rhs) /
-                                    hessenberg_iter->at(iter, rhs);
+        auto this_hess = hessenberg_iter->at(iter, rhs);
+        auto next_hess = hessenberg_iter->at(iter + 1, rhs);
+        const auto scale = abs(this_hess) + abs(next_hess);
+        const auto hypotenuse =
+            scale * sqrt(abs(this_hess / scale) * abs(this_hess / scale) +
+                         abs(next_hess / scale) * abs(next_hess / scale));
+        givens_cos->at(iter, rhs) = conj(this_hess) / hypotenuse;
+        givens_sin->at(iter, rhs) = conj(next_hess) / hypotenuse;
     }
 }
 
 
 template <typename ValueType>
-void givens_rotation(matrix::Dense<ValueType> *next_krylov_basis,
-                     matrix::Dense<ValueType> *givens_sin,
+void givens_rotation(matrix::Dense<ValueType> *givens_sin,
                      matrix::Dense<ValueType> *givens_cos,
                      matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
                      const stopping_status *stop_status)
 {
 #pragma omp parallel for
-    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
+    for (size_type i = 0; i < hessenberg_iter->get_size()[1]; ++i) {
         if (stop_status[i].has_stopped()) {
             continue;
         }
@@ -227,13 +225,13 @@ void givens_rotation(matrix::Dense<ValueType> *next_krylov_basis,
             auto temp = givens_cos->at(j, i) * hessenberg_iter->at(j, i) +
                         givens_sin->at(j, i) * hessenberg_iter->at(j + 1, i);
             hessenberg_iter->at(j + 1, i) =
-                -givens_sin->at(j, i) * hessenberg_iter->at(j, i) +
-                givens_cos->at(j, i) * hessenberg_iter->at(j + 1, i);
+                -conj(givens_sin->at(j, i)) * hessenberg_iter->at(j, i) +
+                conj(givens_cos->at(j, i)) * hessenberg_iter->at(j + 1, i);
             hessenberg_iter->at(j, i) = temp;
             // temp             =  cos(j)*hessenberg(j) +
             //                     sin(j)*hessenberg(j+1)
-            // hessenberg(j+1)  = -sin(j)*hessenberg(j) +
-            //                     cos(j)*hessenberg(j+1)
+            // hessenberg(j+1)  = -conj(sin(j))*hessenberg(j) +
+            //                     conj(cos(j))*hessenberg(j+1)
             // hessenberg(j)    =  temp;
         }
 
@@ -244,7 +242,7 @@ void givens_rotation(matrix::Dense<ValueType> *next_krylov_basis,
             givens_sin->at(iter, i) * hessenberg_iter->at(iter + 1, i);
         hessenberg_iter->at(iter + 1, i) = zero<ValueType>();
         // hessenberg(iter)   = cos(iter)*hessenberg(iter) +
-        //                      sin(iter)*hessenberg(iter)
+        //                      sin(iter)*hessenberg(iter + 1)
         // hessenberg(iter+1) = 0
     }
 }
@@ -445,8 +443,8 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
     finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
                        buffer_iter, arnoldi_norm, iter,
                        stop_status->get_const_data());
-    givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
-                    iter, stop_status->get_const_data());
+    givens_rotation(givens_sin, givens_cos, hessenberg_iter, iter,
+                    stop_status->get_const_data());
     calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,
                                  residual_norm_collection, iter,
                                  stop_status->get_const_data());
diff --git a/reference/solver/cb_gmres_kernels.cpp b/reference/solver/cb_gmres_kernels.cpp
index a4bcd6289c9..811926a67ab 100644
--- a/reference/solver/cb_gmres_kernels.cpp
+++ b/reference/solver/cb_gmres_kernels.cpp
@@ -185,27 +185,25 @@ void calculate_sin_and_cos(matrix::Dense<ValueType> *givens_sin,
         givens_cos->at(iter, rhs) = zero<ValueType>();
         givens_sin->at(iter, rhs) = one<ValueType>();
     } else {
-        auto hypotenuse = sqrt(hessenberg_iter->at(iter, rhs) *
-                                   hessenberg_iter->at(iter, rhs) +
-                               hessenberg_iter->at(iter + 1, rhs) *
-                                   hessenberg_iter->at(iter + 1, rhs));
-        givens_cos->at(iter, rhs) =
-            abs(hessenberg_iter->at(iter, rhs)) / hypotenuse;
-        givens_sin->at(iter, rhs) = givens_cos->at(iter, rhs) *
-                                    hessenberg_iter->at(iter + 1, rhs) /
-                                    hessenberg_iter->at(iter, rhs);
+        auto this_hess = hessenberg_iter->at(iter, rhs);
+        auto next_hess = hessenberg_iter->at(iter + 1, rhs);
+        const auto scale = abs(this_hess) + abs(next_hess);
+        const auto hypotenuse =
+            scale * sqrt(abs(this_hess / scale) * abs(this_hess / scale) +
+                         abs(next_hess / scale) * abs(next_hess / scale));
+        givens_cos->at(iter, rhs) = conj(this_hess) / hypotenuse;
+        givens_sin->at(iter, rhs) = conj(next_hess) / hypotenuse;
     }
 }
 
 
 template <typename ValueType>
-void givens_rotation(matrix::Dense<ValueType> *next_krylov_basis,
-                     matrix::Dense<ValueType> *givens_sin,
+void givens_rotation(matrix::Dense<ValueType> *givens_sin,
                      matrix::Dense<ValueType> *givens_cos,
                      matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
                      const stopping_status *stop_status)
 {
-    for (size_type i = 0; i < next_krylov_basis->get_size()[1]; ++i) {
+    for (size_type i = 0; i < hessenberg_iter->get_size()[1]; ++i) {
         if (stop_status[i].has_stopped()) {
             continue;
         }
@@ -213,13 +211,13 @@ void givens_rotation(matrix::Dense<ValueType> *next_krylov_basis,
             auto temp = givens_cos->at(j, i) * hessenberg_iter->at(j, i) +
                         givens_sin->at(j, i) * hessenberg_iter->at(j + 1, i);
             hessenberg_iter->at(j + 1, i) =
-                -givens_sin->at(j, i) * hessenberg_iter->at(j, i) +
-                givens_cos->at(j, i) * hessenberg_iter->at(j + 1, i);
+                -conj(givens_sin->at(j, i)) * hessenberg_iter->at(j, i) +
+                conj(givens_cos->at(j, i)) * hessenberg_iter->at(j + 1, i);
             hessenberg_iter->at(j, i) = temp;
             // temp             =  cos(j)*hessenberg(j) +
             //                     sin(j)*hessenberg(j+1)
-            // hessenberg(j+1)  = -sin(j)*hessenberg(j) +
-            //                     cos(j)*hessenberg(j+1)
+            // hessenberg(j+1)  = -conj(sin(j))*hessenberg(j) +
+            //                     conj(cos(j))*hessenberg(j+1)
             // hessenberg(j)    =  temp;
         }
 
@@ -230,7 +228,7 @@ void givens_rotation(matrix::Dense<ValueType> *next_krylov_basis,
             givens_sin->at(iter, i) * hessenberg_iter->at(iter + 1, i);
         hessenberg_iter->at(iter + 1, i) = zero<ValueType>();
         // hessenberg(iter)   = cos(iter)*hessenberg(iter) +
-        //                      sin(iter)*hessenberg(iter)
+        //                      sin(iter)*hessenberg(iter + 1)
         // hessenberg(iter+1) = 0
     }
 }
@@ -425,8 +423,8 @@ void step_1(std::shared_ptr<const ReferenceExecutor> exec,
     finish_arnoldi_CGS(next_krylov_basis, krylov_bases, hessenberg_iter,
                        buffer_iter, arnoldi_norm, iter,
                        stop_status->get_const_data());
-    givens_rotation(next_krylov_basis, givens_sin, givens_cos, hessenberg_iter,
-                    iter, stop_status->get_const_data());
+    givens_rotation(givens_sin, givens_cos, hessenberg_iter, iter,
+                    stop_status->get_const_data());
     calculate_next_residual_norm(givens_sin, givens_cos, residual_norm,
                                  residual_norm_collection, iter,
                                  stop_status->get_const_data());

From 8faba62a5619a11331d85dba05f761715898f5d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20I=2E=20Aliaga?= <aliaga@uji.es>
Date: Fri, 19 Feb 2021 13:34:16 +0100
Subject: [PATCH 0418/1008] Update contributors.txt

---
 contributors.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contributors.txt b/contributors.txt
index ce7f8045f4f..746a1e03e5e 100644
--- a/contributors.txt
+++ b/contributors.txt
@@ -19,3 +19,4 @@ Olenik Gregor <go@hpsim.de> HPSim
 Ribizel Tobias <mail@upsj.de> Karlsruhe Institute of Technology
 Riemer Lukas <lksriemer@gmail.com> Karlsruhe Institute of Technology
 Tsai Yuhsiang <yhmtsai@gmail.com> National Taiwan University
+José I. Aliaga <aliaga@uji.s> Universitat Jaume I

From 6ce7181f2d0dac59fcadf8092d6c1b1eaf6e0846 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20I=2E=20Aliaga?= <aliaga@uji.es>
Date: Fri, 19 Feb 2021 13:35:51 +0100
Subject: [PATCH 0419/1008] Update contributors.txt

---
 contributors.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contributors.txt b/contributors.txt
index 746a1e03e5e..0700d45fa78 100644
--- a/contributors.txt
+++ b/contributors.txt
@@ -19,4 +19,4 @@ Olenik Gregor <go@hpsim.de> HPSim
 Ribizel Tobias <mail@upsj.de> Karlsruhe Institute of Technology
 Riemer Lukas <lksriemer@gmail.com> Karlsruhe Institute of Technology
 Tsai Yuhsiang <yhmtsai@gmail.com> National Taiwan University
-José I. Aliaga <aliaga@uji.s> Universitat Jaume I
+José I. Aliaga <aliaga@uji.es> Universitat Jaume I

From 5335d0a4c84f23cb2e3affde4e97b27eacb132ad Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 18 Feb 2021 14:22:30 +0100
Subject: [PATCH 0420/1008] make heat-equation example run on GPUs

---
 examples/heat-equation/heat-equation.cpp | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/examples/heat-equation/heat-equation.cpp b/examples/heat-equation/heat-equation.cpp
index 5ec604d53c9..60ca7f53eb3 100644
--- a/examples/heat-equation/heat-equation.cpp
+++ b/examples/heat-equation/heat-equation.cpp
@@ -148,8 +148,8 @@ int main(int argc, char *argv[])
     // time step size for the simulation
     auto tau = 1.0 / steps_per_sec;
 
-    // create an OpenMP executor
-    auto exec = gko::OmpExecutor::create();
+    // create a CUDA executor with an associated OpenMP host executor
+    auto exec = gko::CudaExecutor::create(0, gko::OmpExecutor::create());
     // load heat source and initial state vectors
     std::ifstream source_stream("data/source.mtx");
     std::ifstream initial_stream("data/initial.mtx");
@@ -206,7 +206,10 @@ int main(int argc, char *argv[])
         if (t - last_t > 1.0 / fps) {
             last_t = t;
             std::cout << t << std::endl;
-            output_timestep(output, n, in_vector->get_const_values());
+            output_timestep(
+                output, n,
+                gko::make_temporary_clone(exec->get_master(), in_vector.get())
+                    ->get_const_values());
         }
         // add heat source contribution
         in_vector->add_scaled(gko::lend(tau_source_scalar), gko::lend(source));

From 385fe12de26d537545eac7c5a58b602702cd9de6 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Mon, 8 Feb 2021 10:05:29 +0100
Subject: [PATCH 0421/1008] parallel prefix sum implementation for openmp

---
 omp/components/prefix_sum.cpp | 75 +++++++++++++++++++++++++++++++----
 1 file changed, 68 insertions(+), 7 deletions(-)

diff --git a/omp/components/prefix_sum.cpp b/omp/components/prefix_sum.cpp
index 8e2e964934b..1158a7de726 100644
--- a/omp/components/prefix_sum.cpp
+++ b/omp/components/prefix_sum.cpp
@@ -33,21 +33,82 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/prefix_sum.hpp"
 
 
+#include <omp.h>
+#include <algorithm>
+
+
 namespace gko {
 namespace kernels {
 namespace omp {
 namespace components {
 
 
+template <typename T>
+static inline T power(const T x, const int exponent)
+{
+    T ans = static_cast<T>(1);
+    for (int i = 0; i < exponent; i++) {
+        ans *= x;
+    }
+    return ans;
+}
+
+
+/*
+ * The last entry of the input array is never used, but is replaced.
+ */
 template <typename IndexType>
-void prefix_sum(std::shared_ptr<const OmpExecutor> exec, IndexType *counts,
-                size_type num_entries)
+void prefix_sum(std::shared_ptr<const OmpExecutor> exec,
+                IndexType *const counts, const size_type num_entries)
 {
-    IndexType partial_sum{};
-    for (size_type i = 0; i < num_entries; ++i) {
-        auto nnz = counts[i];
-        counts[i] = partial_sum;
-        partial_sum += nnz;
+    const auto nentries = static_cast<IndexType>(num_entries);
+    if (num_entries <= 1) {
+        return;
+    }
+
+    const int nthreads = omp_get_max_threads();
+    const IndexType def_num_witems = (num_entries - 1) / nthreads + 1;
+
+#pragma omp parallel
+    {
+        const int thread_id = omp_get_thread_num();
+        const IndexType startidx = thread_id * def_num_witems;
+        const IndexType endidx =
+            std::min(nentries, (thread_id + 1) * def_num_witems);
+        const IndexType startval = counts[startidx];
+
+#pragma omp barrier
+
+        IndexType partial_sum = startval;
+        for (IndexType i = startidx + 1; i < endidx; ++i) {
+            auto nnz = counts[i];
+            counts[i] = partial_sum;
+            partial_sum += nnz;
+        }
+        if (thread_id != nthreads - 1) {
+            counts[endidx] = partial_sum;
+        }
+    }
+
+    counts[0] = 0;
+
+    const int levels = static_cast<IndexType>(std::ceil(std::log(nthreads)));
+    for (IndexType ilvl = 0; ilvl < levels; ilvl++) {
+        const IndexType factor = power(2, (ilvl + 1));
+        const IndexType lvl_num_witems = factor * def_num_witems;
+        const int ntasks = (nthreads - 1) / factor + 1;
+
+#pragma omp parallel for
+        for (int itask = 0; itask < ntasks; itask++) {
+            const IndexType startidx = std::min(
+                nentries, lvl_num_witems / 2 + itask * lvl_num_witems + 1);
+            const IndexType endidx =
+                std::min(nentries, (itask + 1) * lvl_num_witems + 1);
+            const IndexType baseval = counts[startidx - 1];
+            for (int i = startidx; i < endidx; i++) {
+                counts[i] += baseval;
+            }
+        }
     }
 }
 

From 46c6b615c30e532adc8b3db1825260b722102c17 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Mon, 8 Feb 2021 10:54:15 +0100
Subject: [PATCH 0422/1008] minor index type fix

---
 omp/components/prefix_sum.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/omp/components/prefix_sum.cpp b/omp/components/prefix_sum.cpp
index 1158a7de726..c98b5d293b7 100644
--- a/omp/components/prefix_sum.cpp
+++ b/omp/components/prefix_sum.cpp
@@ -92,8 +92,8 @@ void prefix_sum(std::shared_ptr<const OmpExecutor> exec,
 
     counts[0] = 0;
 
-    const int levels = static_cast<IndexType>(std::ceil(std::log(nthreads)));
-    for (IndexType ilvl = 0; ilvl < levels; ilvl++) {
+    const auto levels = static_cast<int>(std::ceil(std::log(nthreads)));
+    for (int ilvl = 0; ilvl < levels; ilvl++) {
         const IndexType factor = power(2, (ilvl + 1));
         const IndexType lvl_num_witems = factor * def_num_witems;
         const int ntasks = (nthreads - 1) / factor + 1;

From 042a9ee9142d310eb6fffefab7e3c0c0dd8a37ef Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Tue, 9 Feb 2021 00:52:15 +0100
Subject: [PATCH 0423/1008] Replaced the algorithm with a simpler but less
 parallel one

This implementation has less memory acceses, so should typically be
faster.

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 omp/components/prefix_sum.cpp | 60 ++++++++++++-----------------------
 1 file changed, 21 insertions(+), 39 deletions(-)

diff --git a/omp/components/prefix_sum.cpp b/omp/components/prefix_sum.cpp
index c98b5d293b7..c57f3b6e759 100644
--- a/omp/components/prefix_sum.cpp
+++ b/omp/components/prefix_sum.cpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <omp.h>
 #include <algorithm>
+#include <vector>
 
 
 namespace gko {
@@ -43,17 +44,6 @@ namespace omp {
 namespace components {
 
 
-template <typename T>
-static inline T power(const T x, const int exponent)
-{
-    T ans = static_cast<T>(1);
-    for (int i = 0; i < exponent; i++) {
-        ans *= x;
-    }
-    return ans;
-}
-
-
 /*
  * The last entry of the input array is never used, but is replaced.
  */
@@ -62,11 +52,11 @@ void prefix_sum(std::shared_ptr<const OmpExecutor> exec,
                 IndexType *const counts, const size_type num_entries)
 {
     const auto nentries = static_cast<IndexType>(num_entries);
-    if (num_entries <= 1) {
-        return;
-    }
+    // the operation only makes sense for arrays of size at least 2
+    GKO_ENSURE_COMPATIBLE_BOUNDS(2, nentries);
 
     const int nthreads = omp_get_max_threads();
+    std::vector<IndexType> proc_sums(nthreads, 0);
     const IndexType def_num_witems = (num_entries - 1) / nthreads + 1;
 
 #pragma omp parallel
@@ -75,41 +65,33 @@ void prefix_sum(std::shared_ptr<const OmpExecutor> exec,
         const IndexType startidx = thread_id * def_num_witems;
         const IndexType endidx =
             std::min(nentries, (thread_id + 1) * def_num_witems);
-        const IndexType startval = counts[startidx];
-
-#pragma omp barrier
 
-        IndexType partial_sum = startval;
-        for (IndexType i = startidx + 1; i < endidx; ++i) {
+        IndexType partial_sum{0};
+        for (IndexType i = startidx; i < endidx; ++i) {
             auto nnz = counts[i];
             counts[i] = partial_sum;
             partial_sum += nnz;
         }
-        if (thread_id != nthreads - 1) {
-            counts[endidx] = partial_sum;
-        }
-    }
 
-    counts[0] = 0;
+        proc_sums[thread_id] = partial_sum;
 
-    const auto levels = static_cast<int>(std::ceil(std::log(nthreads)));
-    for (int ilvl = 0; ilvl < levels; ilvl++) {
-        const IndexType factor = power(2, (ilvl + 1));
-        const IndexType lvl_num_witems = factor * def_num_witems;
-        const int ntasks = (nthreads - 1) / factor + 1;
-
-#pragma omp parallel for
-        for (int itask = 0; itask < ntasks; itask++) {
-            const IndexType startidx = std::min(
-                nentries, lvl_num_witems / 2 + itask * lvl_num_witems + 1);
-            const IndexType endidx =
-                std::min(nentries, (itask + 1) * lvl_num_witems + 1);
-            const IndexType baseval = counts[startidx - 1];
-            for (int i = startidx; i < endidx; i++) {
-                counts[i] += baseval;
+#pragma omp barrier
+
+#pragma omp single
+        {
+            for (int i = 0; i < nthreads - 1; i++) {
+                proc_sums[i + 1] += proc_sums[i];
+            }
+        }
+
+        if (thread_id > 0) {
+            for (IndexType i = startidx; i < endidx; i++) {
+                counts[i] += proc_sums[thread_id - 1];
             }
         }
     }
+
+    counts[0] = 0;
 }
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_PREFIX_SUM_KERNEL);

From aabb9649327c6bc315b205fc99174d013d6fe831 Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Mon, 8 Feb 2021 23:59:19 +0000
Subject: [PATCH 0424/1008] Format files

Co-authored-by: Aditya <aditya.kashi@kit.edu>
---
 omp/components/prefix_sum.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/omp/components/prefix_sum.cpp b/omp/components/prefix_sum.cpp
index c57f3b6e759..c9a1ca8fa87 100644
--- a/omp/components/prefix_sum.cpp
+++ b/omp/components/prefix_sum.cpp
@@ -33,11 +33,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/prefix_sum.hpp"
 
 
-#include <omp.h>
 #include <algorithm>
 #include <vector>
 
 
+#include <omp.h>
+
+
 namespace gko {
 namespace kernels {
 namespace omp {

From 674cf83af4c2167469a14c603e46d13a259c07cd Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Thu, 18 Feb 2021 23:15:18 +0100
Subject: [PATCH 0425/1008] Incorporated review suggestions and added test for
 trivial cases

---
 omp/components/prefix_sum.cpp      | 30 ++++++++++++++++++------------
 omp/test/components/prefix_sum.cpp |  7 ++++++-
 2 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/omp/components/prefix_sum.cpp b/omp/components/prefix_sum.cpp
index c9a1ca8fa87..6ac9d4e9291 100644
--- a/omp/components/prefix_sum.cpp
+++ b/omp/components/prefix_sum.cpp
@@ -34,12 +34,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
-#include <vector>
 
 
 #include <omp.h>
 
 
+#include "core/base/allocator.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace omp {
@@ -53,23 +55,29 @@ template <typename IndexType>
 void prefix_sum(std::shared_ptr<const OmpExecutor> exec,
                 IndexType *const counts, const size_type num_entries)
 {
-    const auto nentries = static_cast<IndexType>(num_entries);
     // the operation only makes sense for arrays of size at least 2
-    GKO_ENSURE_COMPATIBLE_BOUNDS(2, nentries);
+    if (num_entries < 2) {
+        if (num_entries == 0) {
+            return;
+        } else {
+            counts[0] = 0;
+            return;
+        }
+    }
 
     const int nthreads = omp_get_max_threads();
-    std::vector<IndexType> proc_sums(nthreads, 0);
-    const IndexType def_num_witems = (num_entries - 1) / nthreads + 1;
+    vector<IndexType> proc_sums(nthreads, 0, {exec});
+    const size_type def_num_witems = (num_entries - 1) / nthreads + 1;
 
 #pragma omp parallel
     {
         const int thread_id = omp_get_thread_num();
-        const IndexType startidx = thread_id * def_num_witems;
-        const IndexType endidx =
-            std::min(nentries, (thread_id + 1) * def_num_witems);
+        const size_type startidx = thread_id * def_num_witems;
+        const size_type endidx =
+            std::min(num_entries, (thread_id + 1) * def_num_witems);
 
         IndexType partial_sum{0};
-        for (IndexType i = startidx; i < endidx; ++i) {
+        for (size_type i = startidx; i < endidx; ++i) {
             auto nnz = counts[i];
             counts[i] = partial_sum;
             partial_sum += nnz;
@@ -87,13 +95,11 @@ void prefix_sum(std::shared_ptr<const OmpExecutor> exec,
         }
 
         if (thread_id > 0) {
-            for (IndexType i = startidx; i < endidx; i++) {
+            for (size_type i = startidx; i < endidx; i++) {
                 counts[i] += proc_sums[thread_id - 1];
             }
         }
     }
-
-    counts[0] = 0;
 }
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_PREFIX_SUM_KERNEL);
diff --git a/omp/test/components/prefix_sum.cpp b/omp/test/components/prefix_sum.cpp
index 1322f8a7188..e336b4f3732 100644
--- a/omp/test/components/prefix_sum.cpp
+++ b/omp/test/components/prefix_sum.cpp
@@ -89,8 +89,13 @@ class PrefixSum : public ::testing::Test {
 TYPED_TEST_SUITE(PrefixSum, gko::test::IndexTypes);
 
 
-TYPED_TEST(PrefixSum, SmallEqualsReference) { this->test(100); }
+TYPED_TEST(PrefixSum, TrivialCasesEqualReference)
+{
+    this->test(0);
+    this->test(1);
+}
 
+TYPED_TEST(PrefixSum, SmallEqualsReference) { this->test(100); }
 
 TYPED_TEST(PrefixSum, BigEqualsReference) { this->test(this->total_size); }
 

From 135f569a31315941fadc2d8ed656411504de84e1 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Fri, 19 Feb 2021 17:58:10 +0100
Subject: [PATCH 0426/1008] added documentation for prefix_sum

---
 core/components/prefix_sum.hpp | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/core/components/prefix_sum.hpp b/core/components/prefix_sum.hpp
index 90848050590..b0cdf34018d 100644
--- a/core/components/prefix_sum.hpp
+++ b/core/components/prefix_sum.hpp
@@ -45,6 +45,22 @@ namespace gko {
 namespace kernels {
 
 
+/**
+ * \fn prefix_sum
+ * Computes an exclusive prefix sum or exclusive scan of the input array.
+ *
+ * As with the standard definition of exclusive scan, the last entry of the
+ * input array is not read at all, but is written to.
+ * If the input is [3,4,1,9,100], it will be replaced by
+ *  [0,3,7,8,17].
+ *
+ * \tparam IndexType  Type of entries to be scanned (summed).
+ *
+ * \param exec  Executor on which to run the scan operation
+ * \param counts  The input/output array to be scanned with the sum operation
+ * \param num_entries  Size of the array, equal to one more than the number
+ *                     of entries to be summed.
+ */
 #define GKO_DECLARE_PREFIX_SUM_KERNEL(IndexType)                 \
     void prefix_sum(std::shared_ptr<const DefaultExecutor> exec, \
                     IndexType *counts, size_type num_entries)

From bb7346f5570b4d356eb5acc7a910d9b5cab5b176 Mon Sep 17 00:00:00 2001
From: Aditya <aditya.kashi@yahoo.com>
Date: Tue, 23 Feb 2021 19:19:54 +0100
Subject: [PATCH 0427/1008] Apply suggestions from code review

whitespace changes

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 omp/test/components/prefix_sum.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/omp/test/components/prefix_sum.cpp b/omp/test/components/prefix_sum.cpp
index e336b4f3732..4dbd82e3e90 100644
--- a/omp/test/components/prefix_sum.cpp
+++ b/omp/test/components/prefix_sum.cpp
@@ -95,8 +95,10 @@ TYPED_TEST(PrefixSum, TrivialCasesEqualReference)
     this->test(1);
 }
 
+
 TYPED_TEST(PrefixSum, SmallEqualsReference) { this->test(100); }
 
+
 TYPED_TEST(PrefixSum, BigEqualsReference) { this->test(this->total_size); }
 
 

From 292c128d1b94434b9a6551cf4ce64bb300fcbace Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Mon, 22 Feb 2021 17:48:19 +0100
Subject: [PATCH 0428/1008] Update the `HIP` dependencies in INSTALL.md

---
 INSTALL.md | 61 ++++++++++++------------------------------------------
 1 file changed, 13 insertions(+), 48 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index 7ef20b65aef..56118d452fd 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -177,57 +177,13 @@ of HIP either at `/opt/rocm/hip` or at the path specified by `HIP_PATH` as a
 CMake parameter (`-DHIP_PATH=`) or environment variable (`export HIP_PATH=`),
 unless `-DGINKGO_BUILD_HIP=ON/OFF` is set explicitly.
 
-#### Correctly installing HIP toolkits and dependencies for Ginkgo
-In general, Ginkgo's HIP backend requires the following packages:
+#### HIP dependencies for Ginkgo
+Ginkgo's HIP backend adds a dependency to the following packages:
 + HIP,
 + hipBLAS,
 + hipSPARSE,
-+ Thrust.
-
-It is necessary to provide some details about the different ways to
-procure and install these packages, in particular for NVIDIA systems since
-getting a correct, non bloated setup is not straightforward.
-
-For AMD systems, the simplest way is to follow the [instructions provided
-here](https://github.com/ROCm-Developer-Tools/HIP/blob/master/INSTALL.md) which
-provide package installers for most Linux distributions. Ginkgo also needs the
-installation of the [hipBLAS](https://github.com/ROCmSoftwarePlatform/hipBLAS)
-and [hipSPARSE](https://github.com/ROCmSoftwarePlatform/hipSPARSE) interfaces.
-Optionally if you do not already have a thrust installation, [the ROCm provided
-rocThrust package can be
-used](https://github.com/ROCmSoftwarePlatform/rocThrust).
-
-For NVIDIA systems, the traditional installation (package `hip_nvcc`), albeit
-working properly is currently odd: it depends on all the `hcc` related packages,
-although the `nvcc` backend seems to entirely rely on the CUDA suite. [See this
-issue for more
-details](https://github.com/ROCmSoftwarePlatform/hipBLAS/issues/53). It is
-advised in this case to compile everything manually, including using forks of
-`hipBLAS` and `hipSPARSE` specifically made to not depend on the `hcc` specific
-packages. `Thrust` is often provided by CUDA and this Thrust version should work
-with `HIP`. Here is a sample procedure for installing `HIP`, `hipBLAS` and
-`hipSPARSE`.
-
-
-```bash
-# HIP
-git clone https://github.com/ROCm-Developer-Tools/HIP.git
-pushd HIP && mkdir build && pushd build
-cmake .. && make install
-popd && popd
-
-# hipBLAS
-git clone https://github.com/tcojean/hipBLAS.git
-pushd hipBLAS && mkdir build && pushd build
-cmake .. && make install
-popd && popd
-
-# hipSPARSE
-git clone https://github.com/tcojean/hipSPARSE.git
-pushd hipSPARSE && mkdir build && pushd build
-cmake -DBUILD_CUDA=ON .. && make install
-popd && popd
-```
++ rocRAND,
++ rocThrust.
 
 
 #### Changing the paths to search for HIP and other packages
@@ -257,12 +213,21 @@ like so:
 export HIP_PLATFORM=nvcc
 ```
 
+When using `HIP_PLATFORM=hcc`, note that two `HIP` compilers can be set, the old
+`hcc` or since ROCm 3.5, `clang`. On newer installations, `clang` should be set
+by default, but if encountering any problem try to manually set the `HIP`
+compiler to `clang`:
+```
+export HIP_COMPILER=clang
+```
+
 #### Setting platform specific compilation flags
 Platform specific compilation flags can be given through the following
 CMake variables:
 + `-DGINKGO_HIP_COMPILER_FLAGS=`: compilation flags given to all platforms.
 + `-DGINKGO_HIP_HCC_COMPILER_FLAGS=`: compilation flags given to AMD platforms.
 + `-DGINKGO_HIP_NVCC_COMPILER_FLAGS=`: compilation flags given to NVIDIA platforms.
++ `-DGINKGO_HIP_CLANG_COMPILER_FLAGS=`: compilation flags given to AMD clang compiler.
 
 
 ### Third party libraries and packages

From ff8bca63d675afb939543fd47861392600339377 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 19 Feb 2021 20:11:44 +0800
Subject: [PATCH 0429/1008] add GINKGO_BUILD_HWLOC in INTSALL.md

---
 INSTALL.md | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index 56118d452fd..c827e5a1cd6 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -41,6 +41,8 @@ Ginkgo adds the following additional switches to control what is being built:
     `OFF` otherwise.
 *   `-DGINKGO_HIP_AMDGPU="gpuarch1;gpuarch2"` the amdgpu_target(s) variable
     passed to hipcc for the `hcc` HIP backend. The default is none (auto).
+*   `-DGINKGO_BUILD_HWLOC={ON, OFF}` builds Ginkgo with HWLOC. If system HWLOC
+    is not found, Ginkgo will try to build it. Default is `ON`.
 *   `-DGINKGO_BUILD_DOC={ON, OFF}` creates an HTML version of Ginkgo's documentation
     from inline comments in the code. The default is `OFF`.
 *   `-DGINKGO_DOC_GENERATE_EXAMPLES={ON, OFF}` generates the documentation of examples
@@ -246,13 +248,17 @@ packages can be turned off by disabling the relevant options.
 + GINKGO_DEVEL_TOOLS=ON:
   [git-cmake-format](https://github.com/gflegar/git-cmake-format) is our CMake
   helper for code formatting.
++ GINKGO_BUILD_HWLOC=ON:
+  [hwloc](https://www.open-mpi.org/projects/hwloc) to help us to control cores
+  and devices.
 
-By default, Ginkgo uses the internal version of each package. For each of the
+Usually, Ginkgo uses the internal version of each package. For each of the
 packages `GTEST`, `GFLAGS`, `RAPIDJSON` and `CAS`, it is possible to force
 Ginkgo to try to use an external version of a package. For this, Ginkgo provides
 two ways to find packages. To rely on the CMake `find_package` command, use the
-CMake option `-DGINKGO_USE_EXTERNAL_<package>=ON`. Note that, if the external
-packages were not installed to the default location, the CMake option
+CMake option `-DGINKGO_USE_EXTERNAL_<package>=ON`. `HWLOC` always looks for
+system `HWLOC` first currently. Note that, if the external packages were not
+installed to the default location, the CMake option
 `-DCMAKE_PREFIX_PATH=<path-list>` needs to be set to the semicolon (`;`)
 separated list of install paths of these external packages. For more
 Information, see the [CMake documentation for

From e3be04a9f0ffb5e59df947703a10b3f8dab03d3a Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Mon, 22 Feb 2021 18:07:42 +0100
Subject: [PATCH 0430/1008] Update HWLOC instructions as well.

---
 INSTALL.md | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index c827e5a1cd6..4d07826cb8b 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -249,19 +249,21 @@ packages can be turned off by disabling the relevant options.
   [git-cmake-format](https://github.com/gflegar/git-cmake-format) is our CMake
   helper for code formatting.
 + GINKGO_BUILD_HWLOC=ON:
-  [hwloc](https://www.open-mpi.org/projects/hwloc) to help us to control cores
+  [hwloc](https://www.open-mpi.org/projects/hwloc) to detect and control cores
   and devices.
 
 Usually, Ginkgo uses the internal version of each package. For each of the
 packages `GTEST`, `GFLAGS`, `RAPIDJSON` and `CAS`, it is possible to force
 Ginkgo to try to use an external version of a package. For this, Ginkgo provides
 two ways to find packages. To rely on the CMake `find_package` command, use the
-CMake option `-DGINKGO_USE_EXTERNAL_<package>=ON`. `HWLOC` always looks for
-system `HWLOC` first currently. Note that, if the external packages were not
-installed to the default location, the CMake option
-`-DCMAKE_PREFIX_PATH=<path-list>` needs to be set to the semicolon (`;`)
-separated list of install paths of these external packages. For more
-Information, see the [CMake documentation for
+CMake option `-DGINKGO_USE_EXTERNAL_<package>=ON`. `HWLOC` works the opposite
+way, Ginkgo always looks for the system's `hwloc` first. Set
+`-DGINKGO_USE_EXTERNAL_HWLOC=OFF` to not use the system provided one.
+
+Note that, if the external packages were not installed to the default location,
+the CMake option `-DCMAKE_PREFIX_PATH=<path-list>` needs to be set to the
+semicolon (`;`) separated list of install paths of these external packages. For
+more Information, see the [CMake documentation for
 CMAKE_PREFIX_PATH](https://cmake.org/cmake/help/v3.9/variable/CMAKE_PREFIX_PATH.html)
 for details.
 

From 08f658bb7d1e1b76b07863f1181e978aea7ee4be Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Mon, 22 Feb 2021 18:20:40 +0100
Subject: [PATCH 0431/1008] Also count `.md` files as documentation.

---
 .github/label.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/label.sh b/.github/label.sh
index 8849f53d01c..aa6b4f13f75 100755
--- a/.github/label.sh
+++ b/.github/label.sh
@@ -28,7 +28,7 @@ LABELS=$LABELS$(label_match reg:benchmarking '^benchmark/')
 LABELS=$LABELS$(label_match reg:example '^examples/')
 LABELS=$LABELS$(label_match reg:build '(cm|CM)ake')
 LABELS=$LABELS$(label_match reg:ci-cd '(^\.github/|\.yml$)')
-LABELS=$LABELS$(label_match reg:documentation '^doc/')
+LABELS=$LABELS$(label_match reg:documentation '^doc/|\.md$')
 LABELS=$LABELS$(label_match reg:testing /test/)
 LABELS=$LABELS$(label_match reg:helper-scripts '^dev_tools/')
 LABELS=$LABELS$(label_match type:factorization /factorization/)

From db9c4943e0f6fd4d2f839acf3d9f5be5d0da122f Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Sat, 27 Feb 2021 12:15:30 +0100
Subject: [PATCH 0432/1008] Review update.

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Yuhsiang M. Tsai <yhmtsai@gmail.com>
Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
---
 INSTALL.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index 4d07826cb8b..7849e1c40d9 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -257,8 +257,7 @@ packages `GTEST`, `GFLAGS`, `RAPIDJSON` and `CAS`, it is possible to force
 Ginkgo to try to use an external version of a package. For this, Ginkgo provides
 two ways to find packages. To rely on the CMake `find_package` command, use the
 CMake option `-DGINKGO_USE_EXTERNAL_<package>=ON`. `HWLOC` works the opposite
-way, Ginkgo always looks for the system's `hwloc` first. Set
-`-DGINKGO_USE_EXTERNAL_HWLOC=OFF` to not use the system provided one.
+way, Ginkgo always looks for the system's `hwloc` first.
 
 Note that, if the external packages were not installed to the default location,
 the CMake option `-DCMAKE_PREFIX_PATH=<path-list>` needs to be set to the

From 20895f081786b1e7ac3a46f9328d02a60692cad3 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 7 Jan 2021 18:37:06 +0100
Subject: [PATCH 0433/1008] move solver kernel comments before calls

Co-authored-by: Hartwig Anzt <hanzt@icl.utk.edu>
---
 core/solver/bicg.cpp     | 10 ++++-----
 core/solver/bicgstab.cpp | 14 ++++++-------
 core/solver/cg.cpp       |  8 ++++----
 core/solver/cgs.cpp      | 17 ++++++++--------
 core/solver/fcg.cpp      |  8 ++++----
 core/solver/gmres.cpp    | 44 +++++++++++++++++++---------------------
 core/solver/idr.cpp      | 33 +++++++++++++++---------------
 7 files changed, 65 insertions(+), 69 deletions(-)

diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp
index bbc4eab0e03..300b5eb4774 100644
--- a/core/solver/bicg.cpp
+++ b/core/solver/bicg.cpp
@@ -201,21 +201,21 @@ void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             break;
         }
 
-        exec->run(bicg::make_step_1(p.get(), z.get(), p2.get(), z2.get(),
-                                    rho.get(), prev_rho.get(), &stop_status));
         // tmp = rho / prev_rho
         // p = z + tmp * p
         // p2 = z2 + tmp * p2
+        exec->run(bicg::make_step_1(p.get(), z.get(), p2.get(), z2.get(),
+                                    rho.get(), prev_rho.get(), &stop_status));
         system_matrix_->apply(p.get(), q.get());
         conj_trans_A->apply(p2.get(), q2.get());
         p2->compute_dot(q.get(), beta.get());
-        exec->run(bicg::make_step_2(dense_x, r.get(), r2.get(), p.get(),
-                                    q.get(), q2.get(), beta.get(), rho.get(),
-                                    &stop_status));
         // tmp = rho / beta
         // x = x + tmp * p
         // r = r - tmp * q
         // r2 = r2 - tmp * q2
+        exec->run(bicg::make_step_2(dense_x, r.get(), r2.get(), p.get(),
+                                    q.get(), q2.get(), beta.get(), rho.get(),
+                                    &stop_status));
         swap(prev_rho, rho);
     }
 }
diff --git a/core/solver/bicgstab.cpp b/core/solver/bicgstab.cpp
index 0ee788387db..c545da6b313 100644
--- a/core/solver/bicgstab.cpp
+++ b/core/solver/bicgstab.cpp
@@ -152,19 +152,19 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             break;
         }
 
+        // tmp = rho / prev_rho * alpha / omega
+        // p = r + tmp * (p - omega * v)
         exec->run(bicgstab::make_step_1(r.get(), p.get(), v.get(), rho.get(),
                                         prev_rho.get(), alpha.get(),
                                         omega.get(), &stop_status));
-        // tmp = rho / prev_rho * alpha / omega
-        // p = r + tmp * (p - omega * v)
 
         get_preconditioner()->apply(p.get(), y.get());
         system_matrix_->apply(y.get(), v.get());
         rr->compute_dot(v.get(), beta.get());
-        exec->run(bicgstab::make_step_2(r.get(), s.get(), v.get(), rho.get(),
-                                        alpha.get(), beta.get(), &stop_status));
         // alpha = rho / beta
         // s = r - alpha * v
+        exec->run(bicgstab::make_step_2(r.get(), s.get(), v.get(), rho.get(),
+                                        alpha.get(), beta.get(), &stop_status));
 
         ++iter;
         auto all_converged =
@@ -188,12 +188,12 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         system_matrix_->apply(z.get(), t.get());
         s->compute_dot(t.get(), gamma.get());
         t->compute_dot(t.get(), beta.get());
-        exec->run(bicgstab::make_step_3(
-            dense_x, r.get(), s.get(), t.get(), y.get(), z.get(), alpha.get(),
-            beta.get(), gamma.get(), omega.get(), &stop_status));
         // omega = gamma / beta
         // x = x + alpha * y + omega * z
         // r = s - omega * t
+        exec->run(bicgstab::make_step_3(
+            dense_x, r.get(), s.get(), t.get(), y.get(), z.get(), alpha.get(),
+            beta.get(), gamma.get(), omega.get(), &stop_status));
         swap(prev_rho, rho);
     }
 }  // namespace solver
diff --git a/core/solver/cg.cpp b/core/solver/cg.cpp
index dd0558acf75..dd0d56c6f22 100644
--- a/core/solver/cg.cpp
+++ b/core/solver/cg.cpp
@@ -144,17 +144,17 @@ void Cg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             break;
         }
 
-        exec->run(cg::make_step_1(p.get(), z.get(), rho.get(), prev_rho.get(),
-                                  &stop_status));
         // tmp = rho / prev_rho
         // p = z + tmp * p
+        exec->run(cg::make_step_1(p.get(), z.get(), rho.get(), prev_rho.get(),
+                                  &stop_status));
         system_matrix_->apply(p.get(), q.get());
         p->compute_dot(q.get(), beta.get());
-        exec->run(cg::make_step_2(dense_x, r.get(), p.get(), q.get(),
-                                  beta.get(), rho.get(), &stop_status));
         // tmp = rho / beta
         // x = x + tmp * p
         // r = r - tmp * q
+        exec->run(cg::make_step_2(dense_x, r.get(), p.get(), q.get(),
+                                  beta.get(), rho.get(), &stop_status));
         swap(prev_rho, rho);
     }
 }
diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index febacbb03bf..229e5de622d 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -140,15 +140,18 @@ void Cgs<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     int iter = 0;
     while (true) {
         r->compute_dot(r_tld.get(), rho.get());
+        // beta = rho / rho_prev
+        // u = r + beta * q
+        // p = u + beta * ( q + beta * p )
         exec->run(cgs::make_step_1(r.get(), u.get(), p.get(), q.get(),
                                    beta.get(), rho.get(), rho_prev.get(),
                                    &stop_status));
-        // beta = rho / rho_prev
-        // u = r + beta * q;
-        // p = u + beta * ( q + beta * p );
         get_preconditioner()->apply(p.get(), t.get());
         system_matrix_->apply(t.get(), v_hat.get());
         r_tld->compute_dot(v_hat.get(), gamma.get());
+        // alpha = rho / gamma
+        // q = u - alpha * v_hat
+        // t = u + q
         exec->run(cgs::make_step_2(u.get(), v_hat.get(), q.get(), t.get(),
                                    alpha.get(), rho.get(), gamma.get(),
                                    &stop_status));
@@ -156,16 +159,12 @@ void Cgs<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         ++iter;
         this->template log<log::Logger::iteration_complete>(this, iter, r.get(),
                                                             dense_x);
-
-        // alpha = rho / gamma
-        // q = u - alpha * v_hat
-        // t = u + q
         get_preconditioner()->apply(t.get(), u_hat.get());
         system_matrix_->apply(u_hat.get(), t.get());
+        // r = r - alpha * t
+        // x = x + alpha * u_hat
         exec->run(cgs::make_step_3(t.get(), u_hat.get(), r.get(), dense_x,
                                    alpha.get(), &stop_status));
-        // r = r -alpha * t
-        // x = x + alpha * u_hat
 
         ++iter;
         this->template log<log::Logger::iteration_complete>(this, iter, r.get(),
diff --git a/core/solver/fcg.cpp b/core/solver/fcg.cpp
index dea9aeff494..f46e37efe13 100644
--- a/core/solver/fcg.cpp
+++ b/core/solver/fcg.cpp
@@ -147,19 +147,19 @@ void Fcg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             break;
         }
 
-        exec->run(fcg::make_step_1(p.get(), z.get(), rho_t.get(),
-                                   prev_rho.get(), &stop_status));
         // tmp = rho_t / prev_rho
         // p = z + tmp * p
+        exec->run(fcg::make_step_1(p.get(), z.get(), rho_t.get(),
+                                   prev_rho.get(), &stop_status));
         system_matrix_->apply(p.get(), q.get());
         p->compute_dot(q.get(), beta.get());
-        exec->run(fcg::make_step_2(dense_x, r.get(), t.get(), p.get(), q.get(),
-                                   beta.get(), rho.get(), &stop_status));
         // tmp = rho / beta
         // [prev_r = r] in registers
         // x = x + tmp * p
         // r = r - tmp * q
         // t = r - [prev_r]
+        exec->run(fcg::make_step_2(dense_x, r.get(), t.get(), p.get(), q.get(),
+                                   beta.get(), rho.get(), &stop_status));
         swap(prev_rho, rho);
     }
 }
diff --git a/core/solver/gmres.cpp b/core/solver/gmres.cpp
index 3ee99cd3ed3..fb6d4fe012c 100644
--- a/core/solver/gmres.cpp
+++ b/core/solver/gmres.cpp
@@ -175,32 +175,31 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
         if (restart_iter == krylov_dim_) {
             // Restart
+            // Solve upper triangular.
+            // y = hessenberg \ residual_norm_collection
+            // before_preconditioner = krylov_bases * y
             exec->run(gmres::make_step_2(residual_norm_collection.get(),
                                          krylov_bases.get(), hessenberg.get(),
                                          y.get(), before_preconditioner.get(),
                                          &final_iter_nums));
-            // Solve upper triangular.
-            // y = hessenberg \ residual_norm_collection
-            // before_preconditioner = krylov_bases * y
 
+            // x = x + get_preconditioner() * before_preconditioner
             get_preconditioner()->apply(before_preconditioner.get(),
                                         after_preconditioner.get());
             dense_x->add_scaled(one_op.get(), after_preconditioner.get());
-            // Solve x
-            // x = x + get_preconditioner() * before_preconditioner
-            residual->copy_from(dense_b);
             // residual = dense_b
+            residual->copy_from(dense_b);
+            // residual = residual - Ax
             system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(),
                                   residual.get());
-            // residual = residual - Ax
-            exec->run(gmres::make_initialize_2(
-                residual.get(), residual_norm.get(),
-                residual_norm_collection.get(), krylov_bases.get(),
-                &final_iter_nums, krylov_dim_));
             // residual_norm = norm(residual)
             // residual_norm_collection = {residual_norm, unchanged}
             // krylov_bases(:, 1) = residual / residual_norm
             // final_iter_nums = {0, ..., 0}
+            exec->run(gmres::make_initialize_2(
+                residual.get(), residual_norm.get(),
+                residual_norm_collection.get(), krylov_bases.get(),
+                &final_iter_nums, krylov_dim_));
             restart_iter = 0;
         }
         auto this_krylov = krylov_bases->create_submatrix(
@@ -212,9 +211,9 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             span{system_matrix_->get_size()[0] * (restart_iter + 1),
                  system_matrix_->get_size()[0] * (restart_iter + 2)},
             span{0, dense_b->get_size()[1]});
+        // preconditioned_vector = get_preconditioner() * this_krylov
         get_preconditioner()->apply(this_krylov.get(),
                                     preconditioned_vector.get());
-        // preconditioned_vector = get_preconditioner() * this_krylov
 
         // Do Arnoldi and givens rotation
         auto hessenberg_iter = hessenberg->create_submatrix(
@@ -223,14 +222,9 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                  dense_b->get_size()[1] * (restart_iter + 1)});
 
         // Start of arnoldi
-        system_matrix_->apply(preconditioned_vector.get(), next_krylov.get());
         // next_krylov = A * preconditioned_vector
+        system_matrix_->apply(preconditioned_vector.get(), next_krylov.get());
 
-        exec->run(gmres::make_step_1(
-            dense_b->get_size()[0], givens_sin.get(), givens_cos.get(),
-            residual_norm.get(), residual_norm_collection.get(),
-            krylov_bases.get(), hessenberg_iter.get(), restart_iter,
-            &final_iter_nums, &stop_status));
         // final_iter_nums += 1 (unconverged)
         // next_krylov_basis is alias for (restart_iter + 1)-th krylov_bases
         // for i in 0:restart_iter(include)
@@ -267,6 +261,11 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         // residual_norm_collection(restart_iter) = cos(restart_iter) * this_rnc
         // residual_norm = abs(next_rnc)
         // residual_norm_collection(restart_iter + 1) = next_rnc
+        exec->run(gmres::make_step_1(
+            dense_b->get_size()[0], givens_sin.get(), givens_cos.get(),
+            residual_norm.get(), residual_norm_collection.get(),
+            krylov_bases.get(), hessenberg_iter.get(), restart_iter,
+            &final_iter_nums, &stop_status));
 
         restart_iter++;
     }
@@ -279,18 +278,17 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         span{0, restart_iter},
         span{0, dense_b->get_size()[1] * (restart_iter)});
 
+    // Solve upper triangular.
+    // y = hessenberg \ residual_norm_collection
+    // before_preconditioner = krylov_bases * y
     exec->run(gmres::make_step_2(
         residual_norm_collection.get(), krylov_bases_small.get(),
         hessenberg_small.get(), y.get(), before_preconditioner.get(),
         &final_iter_nums));
-    // Solve upper triangular.
-    // y = hessenberg \ residual_norm_collection
-    // before_preconditioner = krylov_bases * y
+    // x = x + get_preconditioner() * before_preconditioner
     get_preconditioner()->apply(before_preconditioner.get(),
                                 after_preconditioner.get());
     dense_x->add_scaled(one_op.get(), after_preconditioner.get());
-    // Solve x
-    // x = x + get_preconditioner() * before_preconditioner
 }
 
 
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index 88683136a00..d4428abbe08 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -185,32 +185,28 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
             break;
         }
 
-        subspace_vectors->apply(residual.get(), f.get());
         // f = P^H * residual
+        subspace_vectors->apply(residual.get(), f.get());
 
         for (size_type k = 0; k < subspace_dim_; k++) {
+            // c = M \ f = (c_1, ..., c_s)^T
+            // v = residual - c_k * g_k - ... - c_s * g_s
             exec->run(idr::make_step_1(nrhs, k, m.get(), f.get(),
                                        residual.get(), g.get(), c.get(),
                                        v.get(), &stop_status));
-            // c = M \ f = (c_1, ..., c_s)^T
-            // v = residual - c_k * g_k - ... - c_s * g_s
 
             get_preconditioner()->apply(v.get(), helper.get());
 
+            // u_k = omega * preconditioned_vector + c_k * u_k + ... + c_s * u_s
             exec->run(idr::make_step_2(nrhs, k, omega.get(), helper.get(),
                                        c.get(), u.get(), &stop_status));
-            // u_k = omega * preconditioned_vector + c_k * u_k + ... + c_s * u_s
 
             auto u_k = u->create_submatrix(span{0, problem_size},
                                            span{k * nrhs, (k + 1) * nrhs});
 
-            system_matrix_->apply(u_k.get(), helper.get());
             // g_k = Au_k
+            system_matrix_->apply(u_k.get(), helper.get());
 
-            exec->run(idr::make_step_3(nrhs, k, subspace_vectors.get(), g.get(),
-                                       helper.get(), u.get(), m.get(), f.get(),
-                                       alpha.get(), residual.get(), dense_x,
-                                       &stop_status));
             // for i = 1 to k - 1 do
             //     alpha = p^H_i * g_k / m_i,i
             //     g_k -= alpha * g_i
@@ -223,6 +219,10 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
             // residual -= beta * g_k
             // dense_x += beta * u_k
             // f = (0,...,0,f_k+1 - beta * m_k+1,k,...,f_s - beta * m_s,k)
+            exec->run(idr::make_step_3(nrhs, k, subspace_vectors.get(), g.get(),
+                                       helper.get(), u.get(), m.get(), f.get(),
+                                       alpha.get(), residual.get(), dense_x,
+                                       &stop_status));
         }
 
         get_preconditioner()->apply(residual.get(), helper.get());
@@ -232,14 +232,6 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
         t->compute_dot(t.get(), tht.get());
         residual->compute_norm2(residual_norm.get());
 
-        exec->run(idr::make_compute_omega(nrhs, kappa_, tht.get(),
-                                          residual_norm.get(), omega.get(),
-                                          &stop_status));
-
-        t->scale(subspace_neg_one_op.get());
-        residual->add_scaled(omega.get(), t.get());
-        dense_x->add_scaled(omega.get(), helper.get());
-
         // omega = (t^H * residual) / (t^H * t)
         // rho = (t^H * residual) / (norm(t) * norm(residual))
         // if abs(rho) < kappa then
@@ -247,6 +239,13 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
         // end if
         // residual -= omega * t
         // dense_x += omega * v
+        exec->run(idr::make_compute_omega(nrhs, kappa_, tht.get(),
+                                          residual_norm.get(), omega.get(),
+                                          &stop_status));
+
+        t->scale(subspace_neg_one_op.get());
+        residual->add_scaled(omega.get(), t.get());
+        dense_x->add_scaled(omega.get(), helper.get());
     }
 }
 

From 7dfa98a69f7bded95101cd85ca50d0428853d30c Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 7 Jan 2021 18:37:53 +0100
Subject: [PATCH 0434/1008] use C++ headers for time.h

---
 cuda/solver/idr_kernels.cu       | 2 +-
 hip/solver/idr_kernels.hip.cpp   | 2 +-
 omp/solver/idr_kernels.cpp       | 1 +
 reference/solver/idr_kernels.cpp | 1 +
 4 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index a493b2c8105..13dd40cd762 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -33,8 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
+#include <ctime>
 #include <random>
-#include <time.h>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index 5c6b3bd6e92..a467504e95c 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -33,8 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
+#include <ctime>
 #include <random>
-#include <time.h>
 
 
 #include <hip/hip_runtime.h>
diff --git a/omp/solver/idr_kernels.cpp b/omp/solver/idr_kernels.cpp
index debfaf86eb8..44dc5d870be 100644
--- a/omp/solver/idr_kernels.cpp
+++ b/omp/solver/idr_kernels.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
+#include <ctime>
 #include <random>
 
 
diff --git a/reference/solver/idr_kernels.cpp b/reference/solver/idr_kernels.cpp
index c1e40c1786c..18a152cbf91 100644
--- a/reference/solver/idr_kernels.cpp
+++ b/reference/solver/idr_kernels.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
+#include <ctime>
 #include <random>
 
 

From af49e5e5bca34af239d6758b31c9d367b7c7ab16 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 7 Jan 2021 18:38:18 +0100
Subject: [PATCH 0435/1008] add memory movement estimates to solvers

Co-authored-by: Hartwig Anzt <hanzt@icl.utk.edu>
---
 core/solver/bicg.cpp     |  8 ++++++++
 core/solver/bicgstab.cpp | 11 +++++++++++
 core/solver/cg.cpp       |  8 ++++++++
 core/solver/cgs.cpp      |  9 +++++++++
 core/solver/fcg.cpp      |  8 ++++++++
 core/solver/gmres.cpp    | 11 +++++++++++
 core/solver/idr.cpp      | 16 ++++++++++++++++
 7 files changed, 71 insertions(+)

diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp
index 300b5eb4774..ea23dca52fd 100644
--- a/core/solver/bicg.cpp
+++ b/core/solver/bicg.cpp
@@ -184,6 +184,14 @@ void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     int iter = -1;
 
+    /* Memory movement summary:
+     * 27n * values + 2 * matrix/preconditioner storage
+     * 2x SpMV:                4n * values + 2 * storage
+     * 2x Preconditioner:      4n * values + 2 * storage
+     * 2x dot                  4n
+     * 1x step 1 (axpys)       6n
+     * 1x step 2 (axpys)       9n
+     */
     while (true) {
         get_preconditioner()->apply(r.get(), z.get());
         conj_trans_preconditioner->apply(r2.get(), z2.get());
diff --git a/core/solver/bicgstab.cpp b/core/solver/bicgstab.cpp
index c545da6b313..4d2216ca57b 100644
--- a/core/solver/bicgstab.cpp
+++ b/core/solver/bicgstab.cpp
@@ -137,6 +137,17 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     rr->copy_from(r.get());
 
     int iter = -1;
+
+    /* Memory movement summary:
+     * 29n * values + 2 * matrix/preconditioner storage
+     * 2x SpMV:                4n * values + 2 * storage
+     * 2x Preconditioner:      4n * values + 2 * storage
+     * 3x dot                  6n
+     * 1x norm2                 n
+     * 1x step 1 (fused axpys) 4n
+     * 1x step 2 (axpy)        3n
+     * 1x step 3 (fused axpys) 7n
+     */
     while (true) {
         ++iter;
         this->template log<log::Logger::iteration_complete>(this, iter, r.get(),
diff --git a/core/solver/cg.cpp b/core/solver/cg.cpp
index dd0d56c6f22..13ecfc89e82 100644
--- a/core/solver/cg.cpp
+++ b/core/solver/cg.cpp
@@ -128,6 +128,14 @@ void Cg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         x, r.get());
 
     int iter = -1;
+    /* Memory movement summary:
+     * 17n * values + matrix/preconditioner storage
+     * 1x SpMV:           2n * values + storage
+     * 1x Preconditioner: 2n * values + storage
+     * 2x dot             4n
+     * 1x step 1 (axpy)   3n
+     * 1x step 2 (axpys)  6n
+     */
     while (true) {
         get_preconditioner()->apply(r.get(), z.get());
         r->compute_dot(z.get(), rho.get());
diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index 229e5de622d..4689af5bb4f 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -138,6 +138,15 @@ void Cgs<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     r_tld->copy_from(r.get());
 
     int iter = 0;
+    /* Memory movement summary:
+     * 27n * values + 2 * matrix/preconditioner storage
+     * 2x SpMV:                4n * values + 2 * storage
+     * 2x Preconditioner:      4n * values + 2 * storage
+     * 2x dot                  4n
+     * 1x step 1 (fused axpys) 5n
+     * 1x step 2 (fused axpys) 4n
+     * 1x step 3 (axpys)       6n
+     */
     while (true) {
         r->compute_dot(r_tld.get(), rho.get());
         // beta = rho / rho_prev
diff --git a/core/solver/fcg.cpp b/core/solver/fcg.cpp
index f46e37efe13..74cb939ef31 100644
--- a/core/solver/fcg.cpp
+++ b/core/solver/fcg.cpp
@@ -130,6 +130,14 @@ void Fcg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         x, r.get());
 
     int iter = -1;
+    /* Memory movement summary:
+     * 20n * values + matrix/preconditioner storage
+     * 1x SpMV:                2n * values + storage
+     * 1x Preconditioner:      2n * values + storage
+     * 3x dot                  6n
+     * 1x step 1 (axpy)        3n
+     * 1x step 2 (fused axpys) 7n
+     */
     while (true) {
         get_preconditioner()->apply(r.get(), z.get());
         r->compute_dot(z.get(), rho.get());
diff --git a/core/solver/gmres.cpp b/core/solver/gmres.cpp
index fb6d4fe012c..1a231dd005f 100644
--- a/core/solver/gmres.cpp
+++ b/core/solver/gmres.cpp
@@ -159,6 +159,17 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     auto after_preconditioner =
         matrix::Dense<ValueType>::create_with_config_of(dense_x);
 
+    /* Memory movement summary for iteration in Krylov subspace of dimension d
+     * (== krylov_dim / 2 on average), ignoring restarts:
+     * (5d+7)n * values + matrix/preconditioner storage
+     * 1x SpMV:                2n * values + storage
+     * 1x Preconditioner:      2n * values + storage
+     * MGS:               (5d+3)n
+     *   dx dot               2dn
+     *   dx axpys             3dn
+     *   1x norm2               n
+     *   1x scal               2n
+     */
     while (true) {
         ++total_iter;
         this->template log<log::Logger::iteration_complete>(
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index d4428abbe08..d862f6c1f78 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -171,6 +171,22 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
 
     int total_iter = -1;
 
+    /* Memory movement summary for iteration with subspace dimension d
+     * (3d²+12(d+1))n * values + d * matrix/preconditioner storage
+     * dx SpMV:                      2dn * values + d * storage
+     * dx Preconditioner:            2dn * values + d * storage
+     * 1x multidot (gemm)         (d+1)n
+     * dx step 1 (fused axpys) d(d/2+2)n on average
+     * dx step 2 (fused axpys) d(d/2+2)n on average
+     * dx step 3:               d(2d+4)n on average
+     *   1x orthogonalize g+u    (3k+2)n in kth iteration
+     *   1x multidot (gemm)           kn in (d-k)th iteration
+     *   2x axpy                      6n
+     * 2x dot                         4n
+     * 1x norm2                        n
+     * 1x scale                       2n
+     * 2x axpy                        4n
+     */
     while (true) {
         ++total_iter;
         this->template log<log::Logger::iteration_complete>(

From dd54e929d0cbf9201aee84d03f0fdb96081b9b05 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sun, 10 Jan 2021 19:59:54 +0100
Subject: [PATCH 0436/1008] update GMRES estimates to include precise counts

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
---
 core/solver/gmres.cpp | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/core/solver/gmres.cpp b/core/solver/gmres.cpp
index 1a231dd005f..26525b4318e 100644
--- a/core/solver/gmres.cpp
+++ b/core/solver/gmres.cpp
@@ -159,16 +159,18 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     auto after_preconditioner =
         matrix::Dense<ValueType>::create_with_config_of(dense_x);
 
-    /* Memory movement summary for iteration in Krylov subspace of dimension d
-     * (== krylov_dim / 2 on average), ignoring restarts:
-     * (5d+7)n * values + matrix/preconditioner storage
+    /* Memory movement summary for average iteration with krylov_dim d:
+     * (5/2d+15/2)n * values + matrix/preconditioner storage
      * 1x SpMV:                2n * values + storage
      * 1x Preconditioner:      2n * values + storage
-     * MGS:               (5d+3)n
-     *   dx dot               2dn
-     *   dx axpys             3dn
-     *   1x norm2               n
-     *   1x scal               2n
+     * MGS:           (5/2d+5/2)n = sum k=0 to d-1 of (5k+5)n/d
+     *       1x dots           2(k+1)n in iteration k (0-based)
+     *       1x axpys          3(k+1)n in iteration k (0-based)
+     *       1x norm2               n
+     *       1x scal               2n
+     * Restart (average):       n = approx (d+1)n/d
+     *       1x gemm           (d+1)n
+     *       plus some O(n) terms we can ignore for larger d
      */
     while (true) {
         ++total_iter;

From b5f04757983e1c209ccd1071b9bf45452a6129b3 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sun, 10 Jan 2021 20:43:00 +0100
Subject: [PATCH 0437/1008] count SpMV operations as iterations in IDR

---
 core/solver/idr.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index d862f6c1f78..e679f69d02c 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -205,6 +205,10 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
         subspace_vectors->apply(residual.get(), f.get());
 
         for (size_type k = 0; k < subspace_dim_; k++) {
+            ++total_iter;
+            this->template log<log::Logger::iteration_complete>(
+                this, total_iter, residual.get(), dense_x);
+
             // c = M \ f = (c_1, ..., c_s)^T
             // v = residual - c_k * g_k - ... - c_s * g_s
             exec->run(idr::make_step_1(nrhs, k, m.get(), f.get(),

From 2ad22380f5bc60491aa4493f30063283f11cfaed Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sun, 10 Jan 2021 20:44:03 +0100
Subject: [PATCH 0438/1008] update IDR counts

---
 core/solver/idr.cpp | 40 ++++++++++++++++++++--------------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index e679f69d02c..0939ab5984a 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -171,21 +171,21 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
 
     int total_iter = -1;
 
-    /* Memory movement summary for iteration with subspace dimension d
-     * (3d²+12(d+1))n * values + d * matrix/preconditioner storage
-     * dx SpMV:                      2dn * values + d * storage
-     * dx Preconditioner:            2dn * values + d * storage
-     * 1x multidot (gemm)         (d+1)n
-     * dx step 1 (fused axpys) d(d/2+2)n on average
-     * dx step 2 (fused axpys) d(d/2+2)n on average
-     * dx step 3:               d(2d+4)n on average
-     *   1x orthogonalize g+u    (3k+2)n in kth iteration
-     *   1x multidot (gemm)           kn in (d-k)th iteration
-     *   2x axpy                      6n
-     * 2x dot                         4n
-     * 1x norm2                        n
-     * 1x scale                       2n
-     * 2x axpy                        4n
+    /* Memory movement summary for iteration with subspace dimension s
+     * (11/2s^2+27/2d+...)n * values + (s+1) * matrix/preconditioner storage
+     * dx SpMV:                    2(s+1)n * values + (s+1) * storage
+     * dx Preconditioner:          2(s+1)n * values + (s+1) * storage
+     * 1x multidot (gemm)           (s+1)n
+     * dx step 1 (fused axpys) s(s/2+5/2)n = approx 1 + sum k=[0,s) of (s-k+1)n
+     * dx step 2 (fused axpys) s(s/2+5/2)n = approx 1 + sum k=[0,s) of (s-k+1)n
+     * dx step 3:             s(9/2s+7/2)n = sum k=[0,s) of (8k+s-k+1+6)n
+     *       1x orthogonalize g+u          8kn in iteration k (0-based)
+     *       1x multidot (gemm)       (s-k+1)n in iteration k (0-based)
+     *       2x axpy                        6n
+     * 1x dot                           2n
+     * 2x norm2                         2n
+     * 1x scale                         2n
+     * 2x axpy                          4n
      */
     while (true) {
         ++total_iter;
@@ -210,14 +210,14 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
                 this, total_iter, residual.get(), dense_x);
 
             // c = M \ f = (c_1, ..., c_s)^T
-            // v = residual - c_k * g_k - ... - c_s * g_s
+            // v = residual - sum i=[k,s) of (c_i * g_i)
             exec->run(idr::make_step_1(nrhs, k, m.get(), f.get(),
                                        residual.get(), g.get(), c.get(),
                                        v.get(), &stop_status));
 
             get_preconditioner()->apply(v.get(), helper.get());
 
-            // u_k = omega * preconditioned_vector + c_k * u_k + ... + c_s * u_s
+            // u_k = omega * precond_vector + sum i=[k,s) of (c_i * u_i)
             exec->run(idr::make_step_2(nrhs, k, omega.get(), helper.get(),
                                        c.get(), u.get(), &stop_status));
 
@@ -227,18 +227,18 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
             // g_k = Au_k
             system_matrix_->apply(u_k.get(), helper.get());
 
-            // for i = 1 to k - 1 do
+            // for i = [0,k)
             //     alpha = p^H_i * g_k / m_i,i
             //     g_k -= alpha * g_i
             //     u_k -= alpha * u_i
             // end for
-            // for i = k to s do
+            // for i = [k,s)
             //     m_i,k = p^H_i * g_k
             // end for
             // beta = f_k / m_k,k
             // residual -= beta * g_k
             // dense_x += beta * u_k
-            // f = (0,...,0,f_k+1 - beta * m_k+1,k,...,f_s - beta * m_s,k)
+            // f = (0,...,0,f_k+1 - beta * m_k+1,k,...,f_s-1 - beta * m_s-1,k)
             exec->run(idr::make_step_3(nrhs, k, subspace_vectors.get(), g.get(),
                                        helper.get(), u.get(), m.get(), f.get(),
                                        alpha.get(), residual.get(), dense_x,

From 51a92f3a604fe4981ceac995753e8dcf2aba466b Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 11 Jan 2021 16:12:40 +0100
Subject: [PATCH 0439/1008] Review updates

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
---
 core/solver/bicg.cpp     |  9 ++++++---
 core/solver/bicgstab.cpp |  3 +++
 core/solver/cgs.cpp      |  3 +++
 core/solver/idr.cpp      | 10 +++++++---
 4 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp
index ea23dca52fd..0c80443169b 100644
--- a/core/solver/bicg.cpp
+++ b/core/solver/bicg.cpp
@@ -185,9 +185,12 @@ void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     int iter = -1;
 
     /* Memory movement summary:
-     * 27n * values + 2 * matrix/preconditioner storage
-     * 2x SpMV:                4n * values + 2 * storage
-     * 2x Preconditioner:      4n * values + 2 * storage
+     * Per iteration:
+     * 13.5n * values + matrix/preconditioner storage
+     * Two iterations:
+     * 27n * values + matrix/preconditioner storage + conj storage
+     * 2x SpMV:                4n * values + storage + conj storage
+     * 2x Preconditioner:      4n * values + storage + conj storage
      * 2x dot                  4n
      * 1x step 1 (axpys)       6n
      * 1x step 2 (axpys)       9n
diff --git a/core/solver/bicgstab.cpp b/core/solver/bicgstab.cpp
index 4d2216ca57b..238acc48e45 100644
--- a/core/solver/bicgstab.cpp
+++ b/core/solver/bicgstab.cpp
@@ -139,6 +139,9 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     int iter = -1;
 
     /* Memory movement summary:
+     * Per iteration:
+     * 14.5n * values + matrix/preconditioner storage
+     * Two iterations:
      * 29n * values + 2 * matrix/preconditioner storage
      * 2x SpMV:                4n * values + 2 * storage
      * 2x Preconditioner:      4n * values + 2 * storage
diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index 4689af5bb4f..af8cc1fee7b 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -139,6 +139,9 @@ void Cgs<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     int iter = 0;
     /* Memory movement summary:
+     * Per iteration:
+     * 13.5n * values + matrix/preconditioner storage
+     * Two iterations:
      * 27n * values + 2 * matrix/preconditioner storage
      * 2x SpMV:                4n * values + 2 * storage
      * 2x Preconditioner:      4n * values + 2 * storage
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index 0939ab5984a..7f61f290a35 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -172,14 +172,17 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
     int total_iter = -1;
 
     /* Memory movement summary for iteration with subspace dimension s
-     * (11/2s^2+27/2d+...)n * values + (s+1) * matrix/preconditioner storage
+     * Per iteration:
+     * (11/2s+10+5/(s+1))n * values + matrix/preconditioner storage
+     * For (s+1) iterations:
+     * (11/2s^2+31/2s+15)n * values + (s+1) * matrix/preconditioner storage
      * dx SpMV:                    2(s+1)n * values + (s+1) * storage
      * dx Preconditioner:          2(s+1)n * values + (s+1) * storage
      * 1x multidot (gemm)           (s+1)n
      * dx step 1 (fused axpys) s(s/2+5/2)n = approx 1 + sum k=[0,s) of (s-k+1)n
      * dx step 2 (fused axpys) s(s/2+5/2)n = approx 1 + sum k=[0,s) of (s-k+1)n
-     * dx step 3:             s(9/2s+7/2)n = sum k=[0,s) of (8k+s-k+1+6)n
-     *       1x orthogonalize g+u          8kn in iteration k (0-based)
+     * dx step 3:            s(9/2s+11/2)n = sum k=[0,s) of (8k+2+s-k+1+6)n
+     *       1x orthogonalize g+u      (8k+2)n in iteration k (0-based)
      *       1x multidot (gemm)       (s-k+1)n in iteration k (0-based)
      *       2x axpy                        6n
      * 1x dot                           2n
@@ -232,6 +235,7 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
             //     g_k -= alpha * g_i
             //     u_k -= alpha * u_i
             // end for
+            // store g_k to g
             // for i = [k,s)
             //     m_i,k = p^H_i * g_k
             // end for

From b6e62773ef3c20d14667c6b7b57537854c8349d0 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 11 Jan 2021 16:22:22 +0100
Subject: [PATCH 0440/1008] exact gmres restart estimate

---
 core/solver/gmres.cpp | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/core/solver/gmres.cpp b/core/solver/gmres.cpp
index 26525b4318e..91224a33c05 100644
--- a/core/solver/gmres.cpp
+++ b/core/solver/gmres.cpp
@@ -160,7 +160,7 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         matrix::Dense<ValueType>::create_with_config_of(dense_x);
 
     /* Memory movement summary for average iteration with krylov_dim d:
-     * (5/2d+15/2)n * values + matrix/preconditioner storage
+     * (5/2d+15/2+12/d)n * values + (1+1/d) * matrix/preconditioner storage
      * 1x SpMV:                2n * values + storage
      * 1x Preconditioner:      2n * values + storage
      * MGS:           (5/2d+5/2)n = sum k=0 to d-1 of (5k+5)n/d
@@ -168,9 +168,13 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
      *       1x axpys          3(k+1)n in iteration k (0-based)
      *       1x norm2               n
      *       1x scal               2n
-     * Restart (average):       n = approx (d+1)n/d
+     * Restart:         (1+12/d)n  (every dth iteration)
      *       1x gemm           (d+1)n
-     *       plus some O(n) terms we can ignore for larger d
+     *       1x Preconditioner     2n * values + storage
+     *       1x axpy               3n
+     *       1x Advanced SpMV      3n * values + storage
+     *       1x norm2               n
+     *       2x scal               2n
      */
     while (true) {
         ++total_iter;

From 2428d19b3e7c97eebbc55918f5f9d58386f29292 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 20 Jan 2021 17:42:17 +0100
Subject: [PATCH 0441/1008] update GMRES estimates

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
---
 core/solver/gmres.cpp | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/core/solver/gmres.cpp b/core/solver/gmres.cpp
index 91224a33c05..730d539df46 100644
--- a/core/solver/gmres.cpp
+++ b/core/solver/gmres.cpp
@@ -160,21 +160,22 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         matrix::Dense<ValueType>::create_with_config_of(dense_x);
 
     /* Memory movement summary for average iteration with krylov_dim d:
-     * (5/2d+15/2+12/d)n * values + (1+1/d) * matrix/preconditioner storage
+     * (5/2d+21/2+14/d)n * values + (1+1/d) * matrix/preconditioner storage
      * 1x SpMV:                2n * values + storage
      * 1x Preconditioner:      2n * values + storage
-     * MGS:           (5/2d+5/2)n = sum k=0 to d-1 of (5k+5)n/d
+     * MGS:          (5/2d+11/2)n = sum k=0 to d-1 of (5k+8)n/d
      *       1x dots           2(k+1)n in iteration k (0-based)
      *       1x axpys          3(k+1)n in iteration k (0-based)
      *       1x norm2               n
      *       1x scal               2n
-     * Restart:         (1+12/d)n  (every dth iteration)
+     * Restart:         (1+14/d)n  (every dth iteration)
      *       1x gemm           (d+1)n
      *       1x Preconditioner     2n * values + storage
      *       1x axpy               3n
+     *       1x copy               2n
      *       1x Advanced SpMV      3n * values + storage
      *       1x norm2               n
-     *       2x scal               2n
+     *       1x scal               2n
      */
     while (true) {
         ++total_iter;

From a5bca555e9e7e333b819dec4ca172e01d86db427 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 22 Jan 2021 11:42:58 +0100
Subject: [PATCH 0442/1008] update IDR estimates

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
---
 core/solver/gmres.cpp |  2 +-
 core/solver/idr.cpp   | 20 ++++++++++----------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/core/solver/gmres.cpp b/core/solver/gmres.cpp
index 730d539df46..a5727cdb151 100644
--- a/core/solver/gmres.cpp
+++ b/core/solver/gmres.cpp
@@ -169,7 +169,7 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
      *       1x norm2               n
      *       1x scal               2n
      * Restart:         (1+14/d)n  (every dth iteration)
-     *       1x gemm           (d+1)n
+     *       1x gemv           (d+1)n
      *       1x Preconditioner     2n * values + storage
      *       1x axpy               3n
      *       1x copy               2n
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index 7f61f290a35..9cdc35e1295 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -173,22 +173,22 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
 
     /* Memory movement summary for iteration with subspace dimension s
      * Per iteration:
-     * (11/2s+10+5/(s+1))n * values + matrix/preconditioner storage
+     * (11/2s+10+7/(s+1))n * values + matrix/preconditioner storage
      * For (s+1) iterations:
-     * (11/2s^2+31/2s+15)n * values + (s+1) * matrix/preconditioner storage
-     * dx SpMV:                    2(s+1)n * values + (s+1) * storage
-     * dx Preconditioner:          2(s+1)n * values + (s+1) * storage
-     * 1x multidot (gemm)           (s+1)n
-     * dx step 1 (fused axpys) s(s/2+5/2)n = approx 1 + sum k=[0,s) of (s-k+1)n
-     * dx step 2 (fused axpys) s(s/2+5/2)n = approx 1 + sum k=[0,s) of (s-k+1)n
-     * dx step 3:            s(9/2s+11/2)n = sum k=[0,s) of (8k+2+s-k+1+6)n
+     * (11/2s^2+31/2s+17)n * values + (s+1) * matrix/preconditioner storage
+     * (s+1)x SpMV:                2(s+1)n * values + (s+1) * storage
+     * (s+1)x Preconditioner:      2(s+1)n * values + (s+1) * storage
+     * 1x multidot (gemv)           (s+1)n
+     * sx step 1 (fused axpys) s(s/2+5/2)n = sum k=[0,s) of (s-k+2)n
+     * sx step 2 (fused axpys) s(s/2+5/2)n = sum k=[0,s) of (s-k+2)n
+     * sx step 3:            s(9/2s+11/2)n = sum k=[0,s) of (8k+2+s-k+1+6)n
      *       1x orthogonalize g+u      (8k+2)n in iteration k (0-based)
-     *       1x multidot (gemm)       (s-k+1)n in iteration k (0-based)
+     *       1x multidot (gemv)       (s-k+1)n in iteration k (0-based)
      *       2x axpy                        6n
      * 1x dot                           2n
      * 2x norm2                         2n
      * 1x scale                         2n
-     * 2x axpy                          4n
+     * 2x axpy                          6n
      */
     while (true) {
         ++total_iter;

From dc198858fccdc3c0e4725eb30b19b5d5fc2522bb Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 5 Feb 2021 13:01:54 +0100
Subject: [PATCH 0443/1008] Revert "count SpMV operations as iterations in IDR"

This reverts commit b59befece0db8b3abc34b1e28709c50bf83a6f53.
---
 core/solver/idr.cpp | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index 9cdc35e1295..8dcde2e00d9 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -173,9 +173,7 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
 
     /* Memory movement summary for iteration with subspace dimension s
      * Per iteration:
-     * (11/2s+10+7/(s+1))n * values + matrix/preconditioner storage
-     * For (s+1) iterations:
-     * (11/2s^2+31/2s+17)n * values + (s+1) * matrix/preconditioner storage
+     * (11/2s^2+31/2s+18)n * values + (s+1) * matrix/preconditioner storage
      * (s+1)x SpMV:                2(s+1)n * values + (s+1) * storage
      * (s+1)x Preconditioner:      2(s+1)n * values + (s+1) * storage
      * 1x multidot (gemv)           (s+1)n
@@ -189,6 +187,7 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
      * 2x norm2                         2n
      * 1x scale                         2n
      * 2x axpy                          6n
+     * 1x norm2 residual                 n
      */
     while (true) {
         ++total_iter;
@@ -208,10 +207,6 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
         subspace_vectors->apply(residual.get(), f.get());
 
         for (size_type k = 0; k < subspace_dim_; k++) {
-            ++total_iter;
-            this->template log<log::Logger::iteration_complete>(
-                this, total_iter, residual.get(), dense_x);
-
             // c = M \ f = (c_1, ..., c_s)^T
             // v = residual - sum i=[k,s) of (c_i * g_i)
             exec->run(idr::make_step_1(nrhs, k, m.get(), f.get(),

From 8e1f3cbaacbfe9a7154dd55fe20c65bfd6d67aec Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 5 Feb 2021 13:26:12 +0100
Subject: [PATCH 0444/1008] add stopping criterion norm2 call

---
 core/solver/bicg.cpp     | 5 +++--
 core/solver/bicgstab.cpp | 5 +++--
 core/solver/cg.cpp       | 3 ++-
 core/solver/cgs.cpp      | 5 +++--
 core/solver/fcg.cpp      | 3 ++-
 5 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp
index 0c80443169b..9e58930e63b 100644
--- a/core/solver/bicg.cpp
+++ b/core/solver/bicg.cpp
@@ -186,14 +186,15 @@ void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     /* Memory movement summary:
      * Per iteration:
-     * 13.5n * values + matrix/preconditioner storage
+     * 14n * values + matrix/preconditioner storage
      * Two iterations:
-     * 27n * values + matrix/preconditioner storage + conj storage
+     * 28n * values + matrix/preconditioner storage + conj storage
      * 2x SpMV:                4n * values + storage + conj storage
      * 2x Preconditioner:      4n * values + storage + conj storage
      * 2x dot                  4n
      * 1x step 1 (axpys)       6n
      * 1x step 2 (axpys)       9n
+     * 1x norm2 residual        n
      */
     while (true) {
         get_preconditioner()->apply(r.get(), z.get());
diff --git a/core/solver/bicgstab.cpp b/core/solver/bicgstab.cpp
index 238acc48e45..e05a3150dc5 100644
--- a/core/solver/bicgstab.cpp
+++ b/core/solver/bicgstab.cpp
@@ -140,9 +140,9 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     /* Memory movement summary:
      * Per iteration:
-     * 14.5n * values + matrix/preconditioner storage
+     * 15.5n * values + matrix/preconditioner storage
      * Two iterations:
-     * 29n * values + 2 * matrix/preconditioner storage
+     * 31n * values + 2 * matrix/preconditioner storage
      * 2x SpMV:                4n * values + 2 * storage
      * 2x Preconditioner:      4n * values + 2 * storage
      * 3x dot                  6n
@@ -150,6 +150,7 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
      * 1x step 1 (fused axpys) 4n
      * 1x step 2 (axpy)        3n
      * 1x step 3 (fused axpys) 7n
+     * 2x norm2 residual       2n
      */
     while (true) {
         ++iter;
diff --git a/core/solver/cg.cpp b/core/solver/cg.cpp
index 13ecfc89e82..5a76392abff 100644
--- a/core/solver/cg.cpp
+++ b/core/solver/cg.cpp
@@ -129,12 +129,13 @@ void Cg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     int iter = -1;
     /* Memory movement summary:
-     * 17n * values + matrix/preconditioner storage
+     * 18n * values + matrix/preconditioner storage
      * 1x SpMV:           2n * values + storage
      * 1x Preconditioner: 2n * values + storage
      * 2x dot             4n
      * 1x step 1 (axpy)   3n
      * 1x step 2 (axpys)  6n
+     * 1x norm2 residual   n
      */
     while (true) {
         get_preconditioner()->apply(r.get(), z.get());
diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index af8cc1fee7b..da7db6ee70f 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -140,15 +140,16 @@ void Cgs<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     int iter = 0;
     /* Memory movement summary:
      * Per iteration:
-     * 13.5n * values + matrix/preconditioner storage
+     * 14n * values + matrix/preconditioner storage
      * Two iterations:
-     * 27n * values + 2 * matrix/preconditioner storage
+     * 28n * values + 2 * matrix/preconditioner storage
      * 2x SpMV:                4n * values + 2 * storage
      * 2x Preconditioner:      4n * values + 2 * storage
      * 2x dot                  4n
      * 1x step 1 (fused axpys) 5n
      * 1x step 2 (fused axpys) 4n
      * 1x step 3 (axpys)       6n
+     * 1x norm2 residual        n
      */
     while (true) {
         r->compute_dot(r_tld.get(), rho.get());
diff --git a/core/solver/fcg.cpp b/core/solver/fcg.cpp
index 74cb939ef31..1e38cdae02d 100644
--- a/core/solver/fcg.cpp
+++ b/core/solver/fcg.cpp
@@ -131,12 +131,13 @@ void Fcg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     int iter = -1;
     /* Memory movement summary:
-     * 20n * values + matrix/preconditioner storage
+     * 21n * values + matrix/preconditioner storage
      * 1x SpMV:                2n * values + storage
      * 1x Preconditioner:      2n * values + storage
      * 3x dot                  6n
      * 1x step 1 (axpy)        3n
      * 1x step 2 (fused axpys) 7n
+     * 1x norm2 residual        n
      */
     while (true) {
         get_preconditioner()->apply(r.get(), z.get());

From 348ead28c7a1692184c3700b98cebd8ba03d6cc8 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 5 Feb 2021 14:08:08 +0100
Subject: [PATCH 0445/1008] fix bicg iteration count in estimate

---
 core/solver/bicg.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp
index 9e58930e63b..7cb51f2f404 100644
--- a/core/solver/bicg.cpp
+++ b/core/solver/bicg.cpp
@@ -185,9 +185,6 @@ void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     int iter = -1;
 
     /* Memory movement summary:
-     * Per iteration:
-     * 14n * values + matrix/preconditioner storage
-     * Two iterations:
      * 28n * values + matrix/preconditioner storage + conj storage
      * 2x SpMV:                4n * values + storage + conj storage
      * 2x Preconditioner:      4n * values + storage + conj storage

From 433b5e619e04e0e773f673fa1a2402fa0c69fd76 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 5 Feb 2021 16:00:26 +0100
Subject: [PATCH 0446/1008] make cgs iterations match stopping checks

---
 core/solver/cgs.cpp | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index da7db6ee70f..99438ba4a1f 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -139,9 +139,6 @@ void Cgs<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     int iter = 0;
     /* Memory movement summary:
-     * Per iteration:
-     * 14n * values + matrix/preconditioner storage
-     * Two iterations:
      * 28n * values + 2 * matrix/preconditioner storage
      * 2x SpMV:                4n * values + 2 * storage
      * 2x Preconditioner:      4n * values + 2 * storage
@@ -169,9 +166,6 @@ void Cgs<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                                    alpha.get(), rho.get(), gamma.get(),
                                    &stop_status));
 
-        ++iter;
-        this->template log<log::Logger::iteration_complete>(this, iter, r.get(),
-                                                            dense_x);
         get_preconditioner()->apply(t.get(), u_hat.get());
         system_matrix_->apply(u_hat.get(), t.get());
         // r = r - alpha * t

From 5c558b3439b8b50c066d056068625e3a34b60a9c Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 5 Feb 2021 16:00:44 +0100
Subject: [PATCH 0447/1008] fix BiCGSTAB half iteration log

---
 core/solver/bicgstab.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/solver/bicgstab.cpp b/core/solver/bicgstab.cpp
index e05a3150dc5..1a3e38b9fba 100644
--- a/core/solver/bicgstab.cpp
+++ b/core/solver/bicgstab.cpp
@@ -194,7 +194,7 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                                               &stop_status));
         }
         this->template log<log::Logger::iteration_complete>(this, iter,
-                                                            r.get());
+                                                            s.get());
         if (all_converged) {
             break;
         }

From c8f0d77b88bec258cc4592b706c280461dc63efd Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 1 Mar 2021 19:33:35 +0100
Subject: [PATCH 0448/1008] replace BiCGSTAB half-iteration by full iteration

but still keep the half-iteration stop check in place
---
 core/solver/bicgstab.cpp | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/core/solver/bicgstab.cpp b/core/solver/bicgstab.cpp
index 1a3e38b9fba..f3828797fdb 100644
--- a/core/solver/bicgstab.cpp
+++ b/core/solver/bicgstab.cpp
@@ -139,9 +139,6 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     int iter = -1;
 
     /* Memory movement summary:
-     * Per iteration:
-     * 15.5n * values + matrix/preconditioner storage
-     * Two iterations:
      * 31n * values + 2 * matrix/preconditioner storage
      * 2x SpMV:                4n * values + 2 * storage
      * 2x Preconditioner:      4n * values + 2 * storage
@@ -181,7 +178,6 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         exec->run(bicgstab::make_step_2(r.get(), s.get(), v.get(), rho.get(),
                                         alpha.get(), beta.get(), &stop_status));
 
-        ++iter;
         auto all_converged =
             stop_criterion->update()
                 .num_iterations(iter)
@@ -193,8 +189,6 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             exec->run(bicgstab::make_finalize(dense_x, y.get(), alpha.get(),
                                               &stop_status));
         }
-        this->template log<log::Logger::iteration_complete>(this, iter,
-                                                            s.get());
         if (all_converged) {
             break;
         }

From 0c5f31c3df74407d5d9278207eb7f4d4c162da09 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 1 Mar 2021 22:38:35 +0100
Subject: [PATCH 0449/1008] use correct provided paths for installation

install windows and HIP helpers only on demand
---
 cmake/GinkgoConfig.cmake.in |  8 +++++--
 cmake/install_helpers.cmake | 45 ++++++++++++++++++++++---------------
 2 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 961949822e8..51a2e8a7249 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -140,8 +140,12 @@ set(GINKGO_OPENMP_LIBRARIES @OpenMP_CXX_LIBRARIES@)
 set(GINKGO_OPENMP_FLAGS "@OpenMP_CXX_FLAGS@")
 
 # Provide useful HIP helper functions
-include(${CMAKE_CURRENT_LIST_DIR}/hip_helpers.cmake)
-include(${CMAKE_CURRENT_LIST_DIR}/windows_helpers.cmake)
+if(GINKGO_BUILD_HIP)
+    include(${CMAKE_CURRENT_LIST_DIR}/hip_helpers.cmake)
+endif()
+if (WIN32 OR CYGWIN)
+    include(${CMAKE_CURRENT_LIST_DIR}/windows_helpers.cmake)
+endif()
 
 # NOTE: we do not export benchmarks, examples, tests or devel tools
 #     so `third_party` libraries are currently unneeded.
diff --git a/cmake/install_helpers.cmake b/cmake/install_helpers.cmake
index 68a75a5f9a5..74ae6f8b6b2 100644
--- a/cmake/install_helpers.cmake
+++ b/cmake/install_helpers.cmake
@@ -2,28 +2,28 @@ include(CMakePackageConfigHelpers)
 include(GNUInstallDirs)
 
 
-set(GINKGO_INSTALL_INCLUDE_DIR "include")
-set(GINKGO_INSTALL_LIBRARY_DIR "lib")
-set(GINKGO_INSTALL_PKGCONFIG_DIR "lib/pkgconfig")
-set(GINKGO_INSTALL_CONFIG_DIR "lib/cmake/Ginkgo")
-set(GINKGO_INSTALL_MODULE_DIR "lib/cmake/Ginkgo/Modules")
+set(GINKGO_INSTALL_INCLUDE_DIR "${CMAKE_INSTALL_INCLUDEDIR}")
+set(GINKGO_INSTALL_LIBRARY_DIR "${CMAKE_INSTALL_LIBDIR}")
+set(GINKGO_INSTALL_PKGCONFIG_DIR "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
+set(GINKGO_INSTALL_CONFIG_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Ginkgo")
+set(GINKGO_INSTALL_MODULE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Ginkgo/Modules")
 
 function(ginkgo_install_library name subdir)
-    
+
     if (WIN32 OR CYGWIN)
         # dll is considered as runtime
         install(TARGETS "${name}"
             EXPORT Ginkgo
-            LIBRARY DESTINATION ${GINKGO_INSTALL_LIBRARY_DIR}
-            ARCHIVE DESTINATION ${GINKGO_INSTALL_LIBRARY_DIR}
-            RUNTIME DESTINATION ${GINKGO_INSTALL_LIBRARY_DIR}
+            LIBRARY DESTINATION "${GINKGO_INSTALL_LIBRARY_DIR}"
+            ARCHIVE DESTINATION "${GINKGO_INSTALL_LIBRARY_DIR}"
+            RUNTIME DESTINATION "${GINKGO_INSTALL_LIBRARY_DIR}"
             )
     else ()
         # install .so and .a files
         install(TARGETS "${name}"
             EXPORT Ginkgo
-            LIBRARY DESTINATION ${GINKGO_INSTALL_LIBRARY_DIR}
-            ARCHIVE DESTINATION ${GINKGO_INSTALL_LIBRARY_DIR}
+            LIBRARY DESTINATION "${GINKGO_INSTALL_LIBRARY_DIR}"
+            ARCHIVE DESTINATION "${GINKGO_INSTALL_LIBRARY_DIR}"
         )
     endif ()
 endfunction()
@@ -37,9 +37,8 @@ function(ginkgo_install)
         DESTINATION "${GINKGO_INSTALL_INCLUDE_DIR}"
         FILES_MATCHING PATTERN "*.hpp"
         )
-    install(DIRECTORY "${Ginkgo_BINARY_DIR}/include/"
-        DESTINATION "${GINKGO_INSTALL_INCLUDE_DIR}"
-        FILES_MATCHING PATTERN "*.hpp"
+    install(FILES "${Ginkgo_BINARY_DIR}/include/ginkgo/config.hpp"
+        DESTINATION "${GINKGO_INSTALL_INCLUDE_DIR}/ginkgo"
         )
     if (GINKGO_HAVE_PAPI_SDE)
         install(FILES "${Ginkgo_SOURCE_DIR}/third_party/papi_sde/papi_sde_interface.h"
@@ -60,7 +59,7 @@ function(ginkgo_install)
     write_basic_package_version_file(
         "${Ginkgo_BINARY_DIR}/GinkgoConfigVersion.cmake"
         VERSION "${PROJECT_VERSION}"
-        COMPATIBILITY AnyNewerVersion
+        COMPATIBILITY SameMajorVersion
         )
     configure_package_config_file(
         "${Ginkgo_SOURCE_DIR}/cmake/GinkgoConfig.cmake.in"
@@ -75,11 +74,21 @@ function(ginkgo_install)
     install(FILES
         "${Ginkgo_BINARY_DIR}/GinkgoConfig.cmake"
         "${Ginkgo_BINARY_DIR}/GinkgoConfigVersion.cmake"
-        "${Ginkgo_SOURCE_DIR}/cmake/hip_helpers.cmake"
-        "${Ginkgo_SOURCE_DIR}/cmake/windows_helpers.cmake"
         DESTINATION "${GINKGO_INSTALL_CONFIG_DIR}"
         )
-      install(EXPORT Ginkgo
+    if (WIN32 OR CYGWIN)
+        install(FILES
+            "${Ginkgo_SOURCE_DIR}/cmake/windows_helpers.cmake"
+            DESTINATION "${GINKGO_INSTALL_CONFIG_DIR}"
+            )
+    endif()
+    if (GINKGO_BUILD_HIP)
+        install(FILES
+            "${Ginkgo_SOURCE_DIR}/cmake/hip_helpers.cmake"
+            DESTINATION "${GINKGO_INSTALL_CONFIG_DIR}"
+            )
+    endif()
+    install(EXPORT Ginkgo
         NAMESPACE Ginkgo::
         FILE GinkgoTargets.cmake
         DESTINATION "${GINKGO_INSTALL_CONFIG_DIR}")

From 3846ff7de55ebba1c8e304dff8b585f280da8e50 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 3 Mar 2021 11:52:55 +0100
Subject: [PATCH 0450/1008] add implicit residual log to solvers

---
 benchmark/solver/solver.cpp |  6 ++++++
 benchmark/utils/loggers.hpp | 30 +++++++++++++++++++++++++++++-
 core/solver/bicg.cpp        |  4 ++--
 core/solver/bicgstab.cpp    |  4 ++--
 core/solver/cg.cpp          |  4 ++--
 core/solver/cgs.cpp         |  4 ++--
 core/solver/fcg.cpp         |  4 ++--
 7 files changed, 45 insertions(+), 11 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index d73ae9151b6..e90bc738f4c 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -382,6 +382,8 @@ void solve_system(const std::string &solver_name,
                           rapidjson::Value(rapidjson::kArrayType), allocator);
         add_or_set_member(solver_json, "true_residuals",
                           rapidjson::Value(rapidjson::kArrayType), allocator);
+        add_or_set_member(solver_json, "implicit_residuals",
+                          rapidjson::Value(rapidjson::kArrayType), allocator);
         add_or_set_member(solver_json, "iteration_timestamps",
                           rapidjson::Value(rapidjson::kArrayType), allocator);
         if (b->get_size()[1] == 1 && !FLAGS_overhead) {
@@ -457,9 +459,13 @@ void solve_system(const std::string &solver_name,
                     exec, lend(system_matrix), b,
                     solver_json["recurrent_residuals"],
                     solver_json["true_residuals"],
+                    solver_json["implicit_residuals"],
                     solver_json["iteration_timestamps"], allocator);
                 solver->add_logger(res_logger);
                 solver->apply(lend(b), lend(x_clone));
+                if (!res_logger->has_implicit_res_norms()) {
+                    solver_json.RemoveMember("implicit_residuals");
+                }
             }
             exec->synchronize();
         }
diff --git a/benchmark/utils/loggers.hpp b/benchmark/utils/loggers.hpp
index 5edf8f96b69..983cd2951fb 100644
--- a/benchmark/utils/loggers.hpp
+++ b/benchmark/utils/loggers.hpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <chrono>
+#include <cmath>
 #include <mutex>
 #include <regex>
 #include <unordered_map>
@@ -202,10 +203,21 @@ template <typename ValueType>
 struct ResidualLogger : gko::log::Logger {
     using rc_vtype = gko::remove_complex<ValueType>;
 
-    void on_iteration_complete(const gko::LinOp *, const gko::size_type &,
+    // TODO2.0: Remove when deprecating simple overload
+    void on_iteration_complete(const gko::LinOp *solver,
+                               const gko::size_type &it,
                                const gko::LinOp *residual,
                                const gko::LinOp *solution,
                                const gko::LinOp *residual_norm) const override
+    {
+        on_iteration_complete(solver, it, residual, solution, residual_norm,
+                              nullptr);
+    }
+
+    void on_iteration_complete(
+        const gko::LinOp *, const gko::size_type &, const gko::LinOp *residual,
+        const gko::LinOp *solution, const gko::LinOp *residual_norm,
+        const gko::LinOp *implicit_sq_residual_norm) const override
     {
         timestamps.PushBack(std::chrono::duration<double>(
                                 std::chrono::steady_clock::now() - start)
@@ -226,12 +238,22 @@ struct ResidualLogger : gko::log::Logger {
         } else {
             true_res_norms.PushBack(-1.0, alloc);
         }
+        if (implicit_sq_residual_norm) {
+            implicit_res_norms.PushBack(
+                std::sqrt(get_norm(
+                    gko::as<vec<rc_vtype>>(implicit_sq_residual_norm))),
+                alloc);
+            has_implicit_res_norm = true;
+        } else {
+            implicit_res_norms.PushBack(-1.0, alloc);
+        }
     }
 
     ResidualLogger(std::shared_ptr<const gko::Executor> exec,
                    const gko::LinOp *matrix, const vec<ValueType> *b,
                    rapidjson::Value &rec_res_norms,
                    rapidjson::Value &true_res_norms,
+                   rapidjson::Value &implicit_res_norms,
                    rapidjson::Value &timestamps,
                    rapidjson::MemoryPoolAllocator<> &alloc)
         : gko::log::Logger(exec, gko::log::Logger::iteration_complete_mask),
@@ -240,16 +262,22 @@ struct ResidualLogger : gko::log::Logger {
           start{std::chrono::steady_clock::now()},
           rec_res_norms{rec_res_norms},
           true_res_norms{true_res_norms},
+          has_implicit_res_norm{},
+          implicit_res_norms{implicit_res_norms},
           timestamps{timestamps},
           alloc{alloc}
     {}
 
+    bool has_implicit_res_norms() const { return has_implicit_res_norm; }
+
 private:
     const gko::LinOp *matrix;
     const vec<ValueType> *b;
     std::chrono::steady_clock::time_point start;
     rapidjson::Value &rec_res_norms;
     rapidjson::Value &true_res_norms;
+    mutable bool has_implicit_res_norm;
+    rapidjson::Value &implicit_res_norms;
     rapidjson::Value &timestamps;
     rapidjson::MemoryPoolAllocator<> &alloc;
 };
diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp
index 7cb51f2f404..300270fd106 100644
--- a/core/solver/bicg.cpp
+++ b/core/solver/bicg.cpp
@@ -199,8 +199,8 @@ void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         z->compute_dot(r2.get(), rho.get());
 
         ++iter;
-        this->template log<log::Logger::iteration_complete>(this, iter, r.get(),
-                                                            dense_x);
+        this->template log<log::Logger::iteration_complete>(
+            this, iter, r.get(), dense_x, nullptr, rho.get());
         if (stop_criterion->update()
                 .num_iterations(iter)
                 .residual(r.get())
diff --git a/core/solver/bicgstab.cpp b/core/solver/bicgstab.cpp
index f3828797fdb..640af69ad45 100644
--- a/core/solver/bicgstab.cpp
+++ b/core/solver/bicgstab.cpp
@@ -151,8 +151,8 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
      */
     while (true) {
         ++iter;
-        this->template log<log::Logger::iteration_complete>(this, iter, r.get(),
-                                                            dense_x);
+        this->template log<log::Logger::iteration_complete>(
+            this, iter, r.get(), dense_x, nullptr, rho.get());
         rr->compute_dot(r.get(), rho.get());
 
         if (stop_criterion->update()
diff --git a/core/solver/cg.cpp b/core/solver/cg.cpp
index 5a76392abff..ba19aa1b1d0 100644
--- a/core/solver/cg.cpp
+++ b/core/solver/cg.cpp
@@ -142,8 +142,8 @@ void Cg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         r->compute_dot(z.get(), rho.get());
 
         ++iter;
-        this->template log<log::Logger::iteration_complete>(this, iter, r.get(),
-                                                            dense_x);
+        this->template log<log::Logger::iteration_complete>(
+            this, iter, r.get(), dense_x, nullptr, rho.get());
         if (stop_criterion->update()
                 .num_iterations(iter)
                 .residual(r.get())
diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index 99438ba4a1f..ffcf800b9d2 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -174,8 +174,8 @@ void Cgs<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                                    alpha.get(), &stop_status));
 
         ++iter;
-        this->template log<log::Logger::iteration_complete>(this, iter, r.get(),
-                                                            dense_x);
+        this->template log<log::Logger::iteration_complete>(
+            this, iter, r.get(), dense_x, nullptr, rho.get());
         if (stop_criterion->update()
                 .num_iterations(iter)
                 .residual(r.get())
diff --git a/core/solver/fcg.cpp b/core/solver/fcg.cpp
index 1e38cdae02d..bc41dfeca58 100644
--- a/core/solver/fcg.cpp
+++ b/core/solver/fcg.cpp
@@ -145,8 +145,8 @@ void Fcg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         t->compute_dot(z.get(), rho_t.get());
 
         ++iter;
-        this->template log<log::Logger::iteration_complete>(this, iter, r.get(),
-                                                            dense_x);
+        this->template log<log::Logger::iteration_complete>(
+            this, iter, r.get(), dense_x, nullptr, rho.get());
         if (stop_criterion->update()
                 .num_iterations(iter)
                 .residual(r.get())

From 75ffe5403497220a2ed614f2596121e3cb98d337 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 3 Mar 2021 23:52:18 +0100
Subject: [PATCH 0451/1008] add implicit residual to custom-logger example

---
 examples/custom-logger/custom-logger.cpp | 129 ++++++++++++++---------
 1 file changed, 78 insertions(+), 51 deletions(-)

diff --git a/examples/custom-logger/custom-logger.cpp b/examples/custom-logger/custom-logger.cpp
index 1caa5911dab..6afd4e8c1c8 100644
--- a/examples/custom-logger/custom-logger.cpp
+++ b/examples/custom-logger/custom-logger.cpp
@@ -93,18 +93,20 @@ struct ResidualLogger : gko::log::Logger {
         std::cout << "Recurrent vs true residual norm:" << std::endl;
         std::cout << '|' << std::setw(10) << "Iteration" << '|' << std::setw(25)
                   << "Recurrent Residual Norm" << '|' << std::setw(25)
-                  << "True Residual Norm" << '|' << std::endl;
+                  << "True Residual Norm" << '|' << std::setw(25)
+                  << "Implicit Residual Norm" << '|' << std::endl;
         // Print a separation line. Note that for creating `10` characters
         // `std::setw()` should be set to `11`.
         std::cout << '|' << std::setfill('-') << std::setw(11) << '|'
                   << std::setw(26) << '|' << std::setw(26) << '|'
-                  << std::setfill(' ') << std::endl;
+                  << std::setw(26) << '|' << std::setfill(' ') << std::endl;
         // Print the data one by one in the form
         std::cout << std::scientific;
         for (std::size_t i = 0; i < iterations.size(); i++) {
             std::cout << '|' << std::setw(10) << iterations[i] << '|'
                       << std::setw(25) << recurrent_norms[i] << '|'
-                      << std::setw(25) << real_norms[i] << '|' << std::endl;
+                      << std::setw(25) << real_norms[i] << '|' << std::setw(25)
+                      << implicit_norms[i] << '|' << std::endl;
         }
         // std::defaultfloat could be used here but some compilers
         // do not support it properly, e.g. the Intel compiler
@@ -112,19 +114,30 @@ struct ResidualLogger : gko::log::Logger {
         // Print a separation line
         std::cout << '|' << std::setfill('-') << std::setw(11) << '|'
                   << std::setw(26) << '|' << std::setw(26) << '|'
-                  << std::setfill(' ') << std::endl;
+                  << std::setw(26) << '|' << std::setfill(' ') << std::endl;
     }
 
     using gko_dense = gko::matrix::Dense<ValueType>;
     using gko_real_dense = gko::matrix::Dense<RealValueType>;
 
-    // Customize the logging hook which is called everytime an iteration is
-    // completed
-    void on_iteration_complete(const gko::LinOp *,
+    // This overload is necessary to avoid interface breaks for Ginkgo 2.0
+    void on_iteration_complete(const gko::LinOp *solver,
                                const gko::size_type &iteration,
                                const gko::LinOp *residual,
                                const gko::LinOp *solution,
                                const gko::LinOp *residual_norm) const override
+    {
+        this->on_iteration_complete(solver, iteration, residual, solution,
+                                    residual_norm, nullptr);
+    }
+
+    // Customize the logging hook which is called everytime an iteration is
+    // completed
+    void on_iteration_complete(
+        const gko::LinOp *, const gko::size_type &iteration,
+        const gko::LinOp *residual, const gko::LinOp *solution,
+        const gko::LinOp *residual_norm,
+        const gko::LinOp *implicit_sq_residual_norm) const override
     {
         // If the solver shares a residual norm, log its value
         if (residual_norm) {
@@ -164,6 +177,18 @@ struct ResidualLogger : gko::log::Logger {
             real_norms.push_back(-1.0);
         }
 
+        if (implicit_sq_residual_norm) {
+            auto dense_norm =
+                gko::as<gko_real_dense>(implicit_sq_residual_norm);
+            // Add the norm to the `implicit_norms` vector
+            implicit_norms.push_back(
+                std::sqrt(get_first_element(gko::lend(dense_norm))));
+        } else {
+            // Add to the `implicit_norms` vector the value -1.0 if it could not
+            // be computed
+            implicit_norms.push_back(-1.0);
+        }
+
         // Add the current iteration number to the `iterations` vector
         iterations.push_back(iteration);
     }
@@ -185,6 +210,8 @@ struct ResidualLogger : gko::log::Logger {
     mutable std::vector<RealValueType> recurrent_norms{};
     // Vector which stores all the real residual norms
     mutable std::vector<RealValueType> real_norms{};
+    // Vector which stores all the implicit residual norms
+    mutable std::vector<RealValueType> implicit_norms{};
     // Vector which stores all the iteration numbers
     mutable std::vector<std::size_t> iterations{};
 };
@@ -192,36 +219,36 @@ struct ResidualLogger : gko::log::Logger {
 
 int main(int argc, char *argv[])
 {
-    // Use some shortcuts. In Ginkgo, vectors are seen as a gko::matrix::Dense
-    // with one column/one row. The advantage of this concept is that using
-    // multiple vectors is a now a natural extension of adding columns/rows are
-    // necessary.
+    // Use some shortcuts. In Ginkgo, vectors are seen as a
+    // gko::matrix::Dense with one column/one row. The advantage of this
+    // concept is that using multiple vectors is a now a natural extension
+    // of adding columns/rows are necessary.
     using ValueType = double;
     using RealValueType = gko::remove_complex<ValueType>;
     using IndexType = int;
     using vec = gko::matrix::Dense<ValueType>;
     using real_vec = gko::matrix::Dense<RealValueType>;
-    // The gko::matrix::Csr class is used here, but any other matrix class such
-    // as gko::matrix::Coo, gko::matrix::Hybrid, gko::matrix::Ell or
+    // The gko::matrix::Csr class is used here, but any other matrix class
+    // such as gko::matrix::Coo, gko::matrix::Hybrid, gko::matrix::Ell or
     // gko::matrix::Sellp could also be used.
     using mtx = gko::matrix::Csr<ValueType, IndexType>;
-    // The gko::solver::Cg is used here, but any other solver class can also be
-    // used.
+    // The gko::solver::Cg is used here, but any other solver class can also
+    // be used.
     using cg = gko::solver::Cg<ValueType>;
 
     // Print the ginkgo version information.
     std::cout << gko::version_info::get() << std::endl;
 
     // @sect3{Where do you want to run your solver ?}
-    // The gko::Executor class is one of the cornerstones of Ginkgo. Currently,
-    // we have support for
-    // an gko::OmpExecutor, which uses OpenMP multi-threading in most of its
-    // kernels, a gko::ReferenceExecutor, a single threaded specialization of
-    // the OpenMP executor and a gko::CudaExecutor which runs the code on a
-    // NVIDIA GPU if available.
-    // @note With the help of C++, you see that you only ever need to change the
-    // executor and all the other functions/ routines within Ginkgo should
-    // automatically work and run on the executor with any other changes.
+    // The gko::Executor class is one of the cornerstones of Ginkgo.
+    // Currently, we have support for an gko::OmpExecutor, which uses OpenMP
+    // multi-threading in most of its kernels, a gko::ReferenceExecutor, a
+    // single threaded specialization of the OpenMP executor and a
+    // gko::CudaExecutor which runs the code on a NVIDIA GPU if available.
+    // @note With the help of C++, you see that you only ever need to change
+    // the executor and all the other functions/ routines within Ginkgo
+    // should automatically work and run on the executor with any other
+    // changes.
     if (argc == 2 && (std::string(argv[1]) == "--help")) {
         std::cerr << "Usage: " << argv[0] << " [executor]" << std::endl;
         std::exit(-1);
@@ -253,27 +280,27 @@ int main(int argc, char *argv[])
     const auto exec = exec_map.at(executor_string)();  // throws if not valid
 
     // @sect3{Reading your data and transfer to the proper device.}
-    // Read the matrix, right hand side and the initial solution using the @ref
-    // read function.
-    // @note Ginkgo uses C++ smart pointers to automatically manage memory. To
-    // this end, we use our own object ownership transfer functions that under
-    // the hood call the required smart pointer functions to manage object
-    // ownership. The gko::share , gko::give and gko::lend are the functions
-    // that you would need to use.
+    // Read the matrix, right hand side and the initial solution using the
+    // @ref read function.
+    // @note Ginkgo uses C++ smart pointers to automatically manage memory.
+    // To this end, we use our own object ownership transfer functions that
+    // under the hood call the required smart pointer functions to manage
+    // object ownership. The gko::share , gko::give and gko::lend are the
+    // functions that you would need to use.
     auto A = share(gko::read<mtx>(std::ifstream("data/A.mtx"), exec));
     auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
     auto x = gko::read<vec>(std::ifstream("data/x0.mtx"), exec);
     const RealValueType reduction_factor = 1e-7;
 
     // @sect3{Creating the solver}
-    // Generate the gko::solver factory. Ginkgo uses the concept of Factories to
-    // build solvers with certain
-    // properties. Observe the Fluent interface used here. Here a cg solver is
-    // generated with a stopping criteria of maximum iterations of 20 and a
-    // residual norm reduction of 1e-15. You also observe that the stopping
-    // criteria(gko::stop) are also generated from factories using their build
-    // methods. You need to specify the executors which each of the object needs
-    // to be built on.
+    // Generate the gko::solver factory. Ginkgo uses the concept of
+    // Factories to build solvers with certain properties. Observe the
+    // Fluent interface used here. Here a cg solver is generated with a
+    // stopping criteria of maximum iterations of 20 and a residual norm
+    // reduction of 1e-15. You also observe that the stopping
+    // criteria(gko::stop) are also generated from factories using their
+    // build methods. You need to specify the executors which each of the
+    // object needs to be built on.
     auto solver_gen =
         cg::build()
             .with_criteria(
@@ -287,8 +314,9 @@ int main(int argc, char *argv[])
     auto logger = std::make_shared<ResidualLogger<ValueType>>(
         exec, gko::lend(A), gko::lend(b));
 
-    // Add the previously created logger to the solver factory. The logger will
-    // be automatically propagated to all solvers created from this factory.
+    // Add the previously created logger to the solver factory. The logger
+    // will be automatically propagated to all solvers created from this
+    // factory.
     solver_gen->add_logger(logger);
 
     // Generate the solver from the matrix. The solver factory built in the
@@ -302,9 +330,8 @@ int main(int argc, char *argv[])
     auto solver = solver_gen->generate(A);
 
 
-    // Finally, solve the system. The solver, being a gko::LinOp, can be applied
-    // to a right hand side, b to
-    // obtain the solution, x.
+    // Finally, solve the system. The solver, being a gko::LinOp, can be
+    // applied to a right hand side, b to obtain the solution, x.
     solver->apply(gko::lend(b), gko::lend(x));
 
     // Print the solution to the command line.
@@ -314,13 +341,13 @@ int main(int argc, char *argv[])
     // Print the table of the residuals obtained from the logger
     logger->write();
 
-    // To measure if your solution has actually converged, you can measure the
-    // error of the solution.
-    // one, neg_one are objects that represent the numbers which allow for a
-    // uniform interface when computing on any device. To compute the residual,
-    // all you need to do is call the apply method, which in this case is an
-    // spmv and equivalent to the LAPACK z_spmv routine. Finally, you compute
-    // the euclidean 2-norm with the compute_norm2 function.
+    // To measure if your solution has actually converged, you can measure
+    // the error of the solution. one, neg_one are objects that represent
+    // the numbers which allow for a uniform interface when computing on any
+    // device. To compute the residual, all you need to do is call the apply
+    // method, which in this case is an spmv and equivalent to the LAPACK
+    // z_spmv routine. Finally, you compute the euclidean 2-norm with the
+    // compute_norm2 function.
     auto one = gko::initialize<vec>({1.0}, exec);
     auto neg_one = gko::initialize<vec>({-1.0}, exec);
     auto res = gko::initialize<real_vec>({0.0}, exec);

From c4a269f036dc17126b97a4ec00a1df24b20c681f Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 5 Mar 2021 11:37:30 +0100
Subject: [PATCH 0452/1008] update custom-logger output

---
 examples/custom-logger/custom-logger.cpp |  3 +-
 examples/custom-logger/doc/results.dox   | 48 ++++++++++++------------
 2 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/examples/custom-logger/custom-logger.cpp b/examples/custom-logger/custom-logger.cpp
index 6afd4e8c1c8..39fcb440eb1 100644
--- a/examples/custom-logger/custom-logger.cpp
+++ b/examples/custom-logger/custom-logger.cpp
@@ -90,7 +90,8 @@ struct ResidualLogger : gko::log::Logger {
     void write() const
     {
         // Print a header for the table
-        std::cout << "Recurrent vs true residual norm:" << std::endl;
+        std::cout << "Recurrent vs true vs implicit residual norm:"
+                  << std::endl;
         std::cout << '|' << std::setw(10) << "Iteration" << '|' << std::setw(25)
                   << "Recurrent Residual Norm" << '|' << std::setw(25)
                   << "True Residual Norm" << '|' << std::setw(25)
diff --git a/examples/custom-logger/doc/results.dox b/examples/custom-logger/doc/results.dox
index 13ef8f5b746..2d8185091a7 100644
--- a/examples/custom-logger/doc/results.dox
+++ b/examples/custom-logger/doc/results.dox
@@ -25,30 +25,30 @@ Solution (x):
 0.0107016
 0.0121141
 0.0123025
-Recurrent vs true residual norm:
-| Iteration|  Recurrent Residual Norm|       True Residual Norm|
-|----------|-------------------------|-------------------------|
-|         0|             4.358899e+00|             4.358899e+00|
-|         1|             2.304548e+00|             2.304548e+00|
-|         2|             1.467706e+00|             1.467706e+00|
-|         3|             9.848751e-01|             9.848751e-01|
-|         4|             7.418330e-01|             7.418330e-01|
-|         5|             5.136231e-01|             5.136231e-01|
-|         6|             3.841650e-01|             3.841650e-01|
-|         7|             3.164394e-01|             3.164394e-01|
-|         8|             2.277088e-01|             2.277088e-01|
-|         9|             1.703121e-01|             1.703121e-01|
-|        10|             9.737220e-02|             9.737220e-02|
-|        11|             6.168306e-02|             6.168306e-02|
-|        12|             4.541231e-02|             4.541231e-02|
-|        13|             3.195304e-02|             3.195304e-02|
-|        14|             1.616058e-02|             1.616058e-02|
-|        15|             6.570152e-03|             6.570152e-03|
-|        16|             2.643669e-03|             2.643669e-03|
-|        17|             8.588089e-04|             8.588089e-04|
-|        18|             2.864613e-04|             2.864613e-04|
-|        19|             1.641952e-15|             2.107881e-15|
-|----------|-------------------------|-------------------------|
+Recurrent vs true vs implicit residual norm:
+| Iteration|  Recurrent Residual Norm|       True Residual Norm|   Implicit Residual Norm|
+|----------|-------------------------|-------------------------|-------------------------|
+|         0|             4.358899e+00|             4.358899e+00|             4.358899e+00|
+|         1|             2.304548e+00|             2.304548e+00|             2.304548e+00|
+|         2|             1.467706e+00|             1.467706e+00|             1.467706e+00|
+|         3|             9.848751e-01|             9.848751e-01|             9.848751e-01|
+|         4|             7.418330e-01|             7.418330e-01|             7.418330e-01|
+|         5|             5.136231e-01|             5.136231e-01|             5.136231e-01|
+|         6|             3.841650e-01|             3.841650e-01|             3.841650e-01|
+|         7|             3.164394e-01|             3.164394e-01|             3.164394e-01|
+|         8|             2.277088e-01|             2.277088e-01|             2.277088e-01|
+|         9|             1.703121e-01|             1.703121e-01|             1.703121e-01|
+|        10|             9.737220e-02|             9.737220e-02|             9.737220e-02|
+|        11|             6.168306e-02|             6.168306e-02|             6.168306e-02|
+|        12|             4.541231e-02|             4.541231e-02|             4.541231e-02|
+|        13|             3.195304e-02|             3.195304e-02|             3.195304e-02|
+|        14|             1.616058e-02|             1.616058e-02|             1.616058e-02|
+|        15|             6.570152e-03|             6.570152e-03|             6.570152e-03|
+|        16|             2.643669e-03|             2.643669e-03|             2.643669e-03|
+|        17|             8.588089e-04|             8.588089e-04|             8.588089e-04|
+|        18|             2.864613e-04|             2.864613e-04|             2.864613e-04|
+|        19|             1.641952e-15|             2.107881e-15|             1.641952e-15|
+|----------|-------------------------|-------------------------|-------------------------|
 Residual norm sqrt(r^T r):
 %%MatrixMarket matrix array real general
 1 1

From 39f4c31b6faedad60a2520a7605c358e53993119 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 6 Mar 2021 08:26:23 +0100
Subject: [PATCH 0453/1008] add implicit residual norm to other loggers

---
 core/log/papi.cpp                  | 11 +++++++++++
 core/log/record.cpp                | 14 +++++++++++++-
 core/log/stream.cpp                | 21 +++++++++++++++++++--
 core/test/log/papi.cpp             |  2 +-
 core/test/log/record.cpp           |  5 ++++-
 core/test/log/stream.cpp           |  4 +++-
 include/ginkgo/core/log/papi.hpp   |  6 ++++++
 include/ginkgo/core/log/record.hpp | 16 ++++++++++++++--
 include/ginkgo/core/log/stream.hpp |  6 ++++++
 9 files changed, 77 insertions(+), 8 deletions(-)

diff --git a/core/log/papi.cpp b/core/log/papi.cpp
index 0ae7d334333..bfb5a16d00b 100644
--- a/core/log/papi.cpp
+++ b/core/log/papi.cpp
@@ -256,6 +256,17 @@ void Papi<ValueType>::on_iteration_complete(const LinOp *solver,
                                             const LinOp *residual,
                                             const LinOp *solution,
                                             const LinOp *residual_norm) const
+{
+    this->on_iteration_complete(solver, num_iterations, residual, solution,
+                                residual_norm, nullptr);
+}
+
+
+template <typename ValueType>
+void Papi<ValueType>::on_iteration_complete(
+    const LinOp *solver, const size_type &num_iterations, const LinOp *residual,
+    const LinOp *solution, const LinOp *residual_norm,
+    const LinOp *implicit_sq_residual_norm) const
 {
     iteration_complete.get_counter(solver) = num_iterations;
 }
diff --git a/core/log/record.cpp b/core/log/record.cpp
index c4543e3e4fc..32be42931c9 100644
--- a/core/log/record.cpp
+++ b/core/log/record.cpp
@@ -261,11 +261,23 @@ void Record::on_iteration_complete(const LinOp *solver,
                                    const size_type &num_iterations,
                                    const LinOp *residual, const LinOp *solution,
                                    const LinOp *residual_norm) const
+{
+    this->on_iteration_complete(solver, num_iterations, residual, solution,
+                                residual_norm, nullptr);
+}
+
+
+void Record::on_iteration_complete(const LinOp *solver,
+                                   const size_type &num_iterations,
+                                   const LinOp *residual, const LinOp *solution,
+                                   const LinOp *residual_norm,
+                                   const LinOp *implicit_sq_residual_norm) const
 {
     append_deque(
         data_.iteration_completed,
         (std::unique_ptr<iteration_complete_data>(new iteration_complete_data{
-            solver, num_iterations, residual, solution, residual_norm})));
+            solver, num_iterations, residual, solution, residual_norm,
+            implicit_sq_residual_norm})));
 }
 
 
diff --git a/core/log/stream.cpp b/core/log/stream.cpp
index 92e3f51da64..bc0c32a7c48 100644
--- a/core/log/stream.cpp
+++ b/core/log/stream.cpp
@@ -428,12 +428,24 @@ void Stream<ValueType>::on_iteration_complete(const LinOp *solver,
                                               const LinOp *residual,
                                               const LinOp *solution,
                                               const LinOp *residual_norm) const
+{
+    this->on_iteration_complete(solver, num_iterations, residual, solution,
+                                residual_norm, nullptr);
+}
+
+
+template <typename ValueType>
+void Stream<ValueType>::on_iteration_complete(
+    const LinOp *solver, const size_type &num_iterations, const LinOp *residual,
+    const LinOp *solution, const LinOp *residual_norm,
+    const LinOp *implicit_sq_residual_norm) const
 {
     os_ << prefix_ << "iteration " << num_iterations
         << " completed with solver " << demangle_name(solver)
         << " with residual " << demangle_name(residual) << ", solution "
-        << demangle_name(solution) << " and residual_norm "
-        << demangle_name(residual_norm) << std::endl;
+        << demangle_name(solution) << ", residual_norm "
+        << demangle_name(residual_norm) << " and implicit_sq_residual_norm "
+        << demangle_name(implicit_sq_residual_norm) << std::endl;
     if (verbose_) {
         os_ << demangle_name(residual)
             << as<gko::matrix::Dense<ValueType>>(residual) << std::endl;
@@ -446,6 +458,11 @@ void Stream<ValueType>::on_iteration_complete(const LinOp *solver,
                 << as<gko::matrix::Dense<ValueType>>(residual_norm)
                 << std::endl;
         }
+        if (implicit_sq_residual_norm != nullptr) {
+            os_ << demangle_name(implicit_sq_residual_norm)
+                << as<gko::matrix::Dense<ValueType>>(implicit_sq_residual_norm)
+                << std::endl;
+        }
     }
 }
 
diff --git a/core/test/log/papi.cpp b/core/test/log/papi.cpp
index 9c72b8233e7..c5fb0d98f95 100644
--- a/core/test/log/papi.cpp
+++ b/core/test/log/papi.cpp
@@ -482,7 +482,7 @@ TYPED_TEST(Papi, CatchesIterationComplete)
 
     this->start();
     this->logger->template on<gko::log::Logger::iteration_complete>(
-        A.get(), 42, nullptr, nullptr, nullptr);
+        A.get(), 42, nullptr, nullptr, nullptr, nullptr);
     long long int value = 0;
     this->stop(&value);
 
diff --git a/core/test/log/record.cpp b/core/test/log/record.cpp
index 647691d517d..ffde5bb0e23 100644
--- a/core/test/log/record.cpp
+++ b/core/test/log/record.cpp
@@ -509,11 +509,12 @@ TEST(Record, CatchesIterations)
     auto residual = gko::initialize<Dense>({-4.4}, exec);
     auto solution = gko::initialize<Dense>({-2.2}, exec);
     auto residual_norm = gko::initialize<Dense>({-3.3}, exec);
+    auto implicit_sq_residual_norm = gko::initialize<Dense>({-3.5}, exec);
 
 
     logger->on<gko::log::Logger::iteration_complete>(
         solver.get(), num_iters, residual.get(), solution.get(),
-        residual_norm.get());
+        residual_norm.get(), implicit_sq_residual_norm.get());
 
     auto &data = logger->get().iteration_completed.back();
     ASSERT_NE(data->solver.get(), nullptr);
@@ -522,6 +523,8 @@ TEST(Record, CatchesIterations)
     GKO_ASSERT_MTX_NEAR(gko::as<Dense>(data->solution.get()), solution, 0);
     GKO_ASSERT_MTX_NEAR(gko::as<Dense>(data->residual_norm.get()),
                         residual_norm, 0);
+    GKO_ASSERT_MTX_NEAR(gko::as<Dense>(data->implicit_sq_residual_norm.get()),
+                        implicit_sq_residual_norm, 0);
 }
 
 
diff --git a/core/test/log/stream.cpp b/core/test/log/stream.cpp
index 3ea1fc8b5b9..42e5a665417 100644
--- a/core/test/log/stream.cpp
+++ b/core/test/log/stream.cpp
@@ -708,13 +708,15 @@ TYPED_TEST(Stream, CatchesIterations)
     auto residual = Dense::create(exec);
     auto solution = Dense::create(exec);
     auto residual_norm = Dense::create(exec);
+    auto implicit_sq_residual_norm = Dense::create(exec);
     std::stringstream ptrstream_solver;
     ptrstream_solver << solver.get();
     std::stringstream ptrstream_residual;
     ptrstream_residual << residual.get();
 
     logger->template on<gko::log::Logger::iteration_complete>(
-        solver.get(), num_iters, residual.get());
+        solver.get(), num_iters, residual.get(), solution.get(),
+        residual_norm.get(), implicit_sq_residual_norm.get());
 
     GKO_ASSERT_STR_CONTAINS(out.str(),
                             "iteration " + std::to_string(num_iters));
diff --git a/include/ginkgo/core/log/papi.hpp b/include/ginkgo/core/log/papi.hpp
index 1f470dd1bb4..436dc59d1ef 100644
--- a/include/ginkgo/core/log/papi.hpp
+++ b/include/ginkgo/core/log/papi.hpp
@@ -177,6 +177,12 @@ class Papi : public Logger {
         const LinOp *residual, const LinOp *solution = nullptr,
         const LinOp *residual_norm = nullptr) const override;
 
+    void on_iteration_complete(
+        const LinOp *solver, const size_type &num_iterations,
+        const LinOp *residual, const LinOp *solution,
+        const LinOp *residual_norm,
+        const LinOp *implicit_sq_residual_norm) const override;
+
     /**
      * Creates a Papi Logger.
      *
diff --git a/include/ginkgo/core/log/record.hpp b/include/ginkgo/core/log/record.hpp
index 2fe1d4295c8..eb20ae0a40a 100644
--- a/include/ginkgo/core/log/record.hpp
+++ b/include/ginkgo/core/log/record.hpp
@@ -62,16 +62,19 @@ struct iteration_complete_data {
     std::unique_ptr<const LinOp> residual;
     std::unique_ptr<const LinOp> solution;
     std::unique_ptr<const LinOp> residual_norm;
+    std::unique_ptr<const LinOp> implicit_sq_residual_norm;
 
     iteration_complete_data(const LinOp *solver, const size_type num_iterations,
                             const LinOp *residual = nullptr,
                             const LinOp *solution = nullptr,
-                            const LinOp *residual_norm = nullptr)
+                            const LinOp *residual_norm = nullptr,
+                            const LinOp *implicit_sq_residual_norm = nullptr)
         : solver{nullptr},
           num_iterations{num_iterations},
           residual{nullptr},
           solution{nullptr},
-          residual_norm{nullptr}
+          residual_norm{nullptr},
+          implicit_sq_residual_norm{nullptr}
     {
         this->solver = solver->clone();
         if (residual != nullptr) {
@@ -83,6 +86,10 @@ struct iteration_complete_data {
         if (residual_norm != nullptr) {
             this->residual_norm = residual_norm->clone();
         }
+        if (implicit_sq_residual_norm != nullptr) {
+            this->implicit_sq_residual_norm =
+                implicit_sq_residual_norm->clone();
+        }
     }
 };
 
@@ -369,6 +376,11 @@ class Record : public Logger {
         const LinOp *residual, const LinOp *solution = nullptr,
         const LinOp *residual_norm = nullptr) const override;
 
+    void on_iteration_complete(
+        const LinOp *solver, const size_type &num_iterations,
+        const LinOp *residual, const LinOp *solution,
+        const LinOp *residual_norm,
+        const LinOp *implicit_sq_residual_norm) const override;
 
     /**
      * Creates a Record logger. This dynamically allocates the memory,
diff --git a/include/ginkgo/core/log/stream.hpp b/include/ginkgo/core/log/stream.hpp
index 634ba7e2a9b..664ee5a47e4 100644
--- a/include/ginkgo/core/log/stream.hpp
+++ b/include/ginkgo/core/log/stream.hpp
@@ -153,6 +153,12 @@ class Stream : public Logger {
         const LinOp *residual, const LinOp *solution = nullptr,
         const LinOp *residual_norm = nullptr) const override;
 
+    void on_iteration_complete(
+        const LinOp *solver, const size_type &num_iterations,
+        const LinOp *residual, const LinOp *solution,
+        const LinOp *residual_norm,
+        const LinOp *implicit_sq_residual_norm) const override;
+
     /**
      * Creates a Stream logger. This dynamically allocates the memory,
      * constructs the object and returns an std::unique_ptr to this object.

From c60b2be4359e873c7092ecb8302d96b9ccd46cdb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 5 Feb 2021 06:25:35 +0100
Subject: [PATCH 0454/1008] Extend row_major accessor for arbitrary dimensions

---
 core/test/base/range_accessors.cpp           | 140 ++++++
 include/ginkgo/core/base/range_accessors.hpp | 437 ++++++++++++++++++-
 2 files changed, 571 insertions(+), 6 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index eecac4ae60f..7c0a104822e 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -62,6 +62,35 @@ class RowMajorAccessor : public ::testing::Test {
 };
 
 
+TEST_F(RowMajorAccessor, CanCreateWithDim)
+{
+    row_major_int_range r2{data, gko::dim<2>{3, 2}, 3u};
+
+    EXPECT_EQ(r2(0, 0), 1);
+    EXPECT_EQ(r2(0, 1), 2);
+    EXPECT_EQ(r2(1, 0), 3);
+    EXPECT_EQ(r2(1, 1), 4);
+    EXPECT_EQ(r2(2, 0), 5);
+    EXPECT_EQ(r2(2, 1), 6);
+}
+
+
+TEST_F(RowMajorAccessor, CanCreateDefaultStride)
+{
+    row_major_int_range r2{data, gko::dim<2>{3, 3}};
+
+    EXPECT_EQ(r2(0, 0), 1);
+    EXPECT_EQ(r2(0, 1), 2);
+    EXPECT_EQ(r2(0, 2), -1);
+    EXPECT_EQ(r2(1, 0), 3);
+    EXPECT_EQ(r2(1, 1), 4);
+    EXPECT_EQ(r2(1, 2), -2);
+    EXPECT_EQ(r2(2, 0), 5);
+    EXPECT_EQ(r2(2, 1), 6);
+    EXPECT_EQ(r2(2, 2), -3);
+}
+
+
 TEST_F(RowMajorAccessor, CanAccessData)
 {
     EXPECT_EQ(r(0, 0), 1);
@@ -135,4 +164,115 @@ TEST_F(RowMajorAccessor, CanAssignSubranges)
 }
 
 
+class RowMajorAccessor3d : public ::testing::Test {
+protected:
+    using span = gko::span;
+    static constexpr gko::size_type dimensionality{3};
+
+    using row_major_int_range =
+        gko::range<gko::accessor::row_major<int, dimensionality>>;
+
+    // clang-format off
+    int data[2 * 3 * 4]{
+        1, 2, -1, 11,
+        3, 4, -2, 12,
+        5, 6, -3, 13,
+
+        21, 22, 23, 24,
+        25, 26, 27, 28,
+        29, 30, 31, 32
+    };
+    // clang-format on
+    const gko::dim<dimensionality> dim1{2, 3, 4};
+    const gko::dim<dimensionality> dim2{2, 2, 3};
+    row_major_int_range default_r{data, dim1};
+    row_major_int_range custom_r{
+        data, dim2,
+        std::array<const gko::size_type, dimensionality - 1>{12, 4}};
+};
+
+
+TEST_F(RowMajorAccessor3d, CanAccessData)
+{
+    EXPECT_EQ(default_r(0, 0, 0), 1);
+    EXPECT_EQ(custom_r(0, 0, 0), 1);
+    EXPECT_EQ(default_r(0, 1, 0), 3);
+    EXPECT_EQ(custom_r(0, 1, 0), 3);
+    EXPECT_EQ(default_r(0, 1, 3), 12);
+    EXPECT_EQ(default_r(0, 2, 2), -3);
+    EXPECT_EQ(default_r(1, 2, 1), 30);
+    EXPECT_EQ(default_r(1, 2, 2), 31);
+    EXPECT_EQ(default_r(1, 2, 3), 32);
+}
+
+
+TEST_F(RowMajorAccessor3d, CanWriteData)
+{
+    default_r(0, 0, 0) = 4;
+    custom_r(1, 1, 1) = 100;
+
+    EXPECT_EQ(default_r(0, 0, 0), 4);
+    EXPECT_EQ(custom_r(0, 0, 0), 4);
+    EXPECT_EQ(default_r(1, 1, 1), 100);
+    EXPECT_EQ(custom_r(1, 1, 1), 100);
+}
+
+
+TEST_F(RowMajorAccessor3d, CanCreateSubrange)
+{
+    auto subr = custom_r(span{0, 2}, span{1, 2}, span{1, 3});
+
+    EXPECT_EQ(subr(0, 0, 0), 4);
+    EXPECT_EQ(subr(0, 0, 1), -2);
+    EXPECT_EQ(subr(1, 0, 0), 26);
+    EXPECT_EQ(subr(1, 0, 1), 27);
+}
+
+
+TEST_F(RowMajorAccessor3d, CanCreateRowVector)
+{
+    auto subr = default_r(1u, 2u, span{0, 2});
+
+    EXPECT_EQ(subr(0, 0, 0), 29);
+    EXPECT_EQ(subr(0, 0, 1), 30);
+}
+
+
+TEST_F(RowMajorAccessor3d, CanCreateColumnVector)
+{
+    auto subr = default_r(span{0u, 2u}, 1u, 3u);
+
+    EXPECT_EQ(subr(0, 0, 0), 12);
+    EXPECT_EQ(subr(1, 0, 0), 28);
+}
+
+
+TEST_F(RowMajorAccessor3d, CanAssignValues)
+{
+    default_r(1, 1, 1) = default_r(0, 0, 0);
+
+    EXPECT_EQ(data[17], 1);
+}
+
+
+TEST_F(RowMajorAccessor3d, CanAssignSubranges)
+{
+    default_r(1u, span{0, 2}, span{0, 3}) =
+        custom_r(0u, span{0, 2}, span{0, 3});
+
+    EXPECT_EQ(data[12], 1);
+    EXPECT_EQ(data[13], 2);
+    EXPECT_EQ(data[14], -1);
+    EXPECT_EQ(data[15], 24);
+    EXPECT_EQ(data[16], 3);
+    EXPECT_EQ(data[17], 4);
+    EXPECT_EQ(data[18], -2);
+    EXPECT_EQ(data[19], 28);
+    EXPECT_EQ(data[20], 29);
+    EXPECT_EQ(data[21], 30);
+    EXPECT_EQ(data[22], 31);
+    EXPECT_EQ(data[23], 32);
+}
+
+
 }  // namespace
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index f4fd01247d4..b0528956b38 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -35,8 +35,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <array>
+#include <type_traits>
 
 
+#include <ginkgo/core/base/dim.hpp>
 #include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/base/types.hpp>
 
@@ -50,6 +52,228 @@ namespace gko {
 namespace accessor {
 
 
+namespace detail {
+
+
+/**
+ * This helper runs from first to last dimension in order to compute the index.
+ * The index is computed like this:
+ * indices: x1, x2, x3, ...
+ * compute(stride, x1, x2, x3) -> x1 * stride[0] + x2 * stride[1] + x3
+ */
+template <typename ValueType, size_type total_dim, size_type current_iter = 1>
+struct row_major_helper_s {
+    static_assert(total_dim >= 1, "Dimensionality must be >= 1");
+    static_assert(current_iter < total_dim, "Iteration must be < total_dim!");
+
+    static constexpr size_type dim_idx{current_iter - 1};
+
+    template <typename FirstType, typename... Indices>
+    static constexpr GKO_ATTRIBUTES ValueType
+    compute(const std::array<ValueType, total_dim> &size,
+            const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
+                &stride,
+            FirstType first, Indices &&...idxs)
+    {
+        return GKO_ASSERT(first < size[dim_idx]),
+               first * stride[dim_idx] +
+                   row_major_helper_s<ValueType, total_dim, current_iter + 1>::
+                       compute(size, stride, std::forward<Indices>(idxs)...);
+    }
+};
+
+template <typename ValueType, size_type total_dim>
+struct row_major_helper_s<ValueType, total_dim, total_dim> {
+    template <typename FirstType>
+    static constexpr GKO_ATTRIBUTES ValueType
+    compute(const std::array<ValueType, total_dim> &size,
+            const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>,
+            FirstType first)
+    {
+        return GKO_ASSERT(first < size[total_dim - 1]), first;
+    }
+};
+
+
+/**
+ * Computes the storage index for the given indices with respect to the given
+ * stride array
+ */
+template <typename ValueType, size_type total_dim, typename... Indices>
+constexpr GKO_ATTRIBUTES ValueType compute_storage_index(
+    const std::array<ValueType, total_dim> &size,
+    const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)> &stride,
+    Indices &&...idxs)
+{
+    return row_major_helper_s<ValueType, total_dim>::compute(
+        size, stride, std::forward<Indices>(idxs)...);
+}
+
+
+template <size_type iter, typename ValueType, size_type N>
+constexpr GKO_ATTRIBUTES std::enable_if_t<iter == N, int> spans_in_size(
+    const std::array<ValueType, N> &)
+{
+    return 0;
+}
+
+template <size_type iter, typename ValueType, size_type N, typename First,
+          typename... Remaining>
+constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), int> spans_in_size(
+    const std::array<ValueType, N> &size, First first, Remaining &&...remaining)
+{
+    static_assert(sizeof...(Remaining) + 1 == N - iter,
+                  "Number of remaining spans must be equal to N - iter");
+    return GKO_ASSERT(span{first}.is_valid()),
+           GKO_ASSERT(span{first} <= span{size[iter]}),
+           spans_in_size<iter + 1>(size, std::forward<Remaining>(remaining)...);
+}
+
+
+template <typename ValueType, size_type N, typename... Spans>
+constexpr GKO_ATTRIBUTES int validate_spans(
+    const std::array<ValueType, N> &size, Spans &&...spans)
+{
+    return detail::spans_in_size<0>(size, std::forward<Spans>(spans)...);
+}
+
+
+template <bool has_span, typename... Args>
+struct are_span_compatible_impl
+    : public std::integral_constant<bool, has_span> {};
+
+template <bool has_span, typename First, typename... Args>
+struct are_span_compatible_impl<has_span, First, Args...>
+    : public std::conditional<
+          std::is_integral<std::decay_t<First>>::value ||
+              std::is_same<std::decay_t<First>, span>::value,
+          are_span_compatible_impl<
+              has_span || std::is_same<std::decay_t<First>, span>::value,
+              Args...>,
+          std::false_type>::type {};
+
+
+/**
+ * Evaluates if at least one type of Args is a gko::span and the others either
+ * also gko::span or fulfill std::is_integral
+ */
+template <typename... Args>
+using are_span_compatible = are_span_compatible_impl<false, Args...>;
+
+
+template <typename ValueType, size_type N, size_type current, typename... Dims>
+constexpr GKO_ATTRIBUTES
+    std::enable_if_t<(current == N), std::array<ValueType, N>>
+    to_array_impl(const dim<N> &size, Dims &&...dims)
+{
+    static_assert(sizeof...(Dims) == N,
+                  "Number of arguments must match dimensionality!");
+    return {{std::forward<Dims>(dims)...}};
+}
+
+
+template <typename ValueType, size_type N, size_type current, typename... Dims>
+constexpr GKO_ATTRIBUTES
+    std::enable_if_t<(current < N), std::array<ValueType, N>>
+    to_array_impl(const dim<N> &size, Dims &&...dims)
+{
+    return to_array_impl<ValueType, N, current + 1>(
+        size, std::forward<Dims>(dims)..., size[current]);
+}
+
+
+template <typename ValueType, size_type N>
+constexpr GKO_ATTRIBUTES std::array<ValueType, N> to_array(const dim<N> &size)
+{
+    return to_array_impl<ValueType, N, 0>(size);
+}
+
+
+template <size_type iter, typename ValueType, size_type N>
+constexpr GKO_ATTRIBUTES std::enable_if_t<iter == N, ValueType>
+mult_dim_upwards_impl(const std::array<ValueType, N> &)
+{
+    return 1;
+}
+
+template <size_type iter, typename ValueType, size_type N>
+constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), ValueType>
+mult_dim_upwards_impl(const std::array<ValueType, N> &size)
+{
+    return size[iter] * mult_dim_upwards_impl<iter + 1>(size);
+}
+
+
+template <size_type iter = 1, typename ValueType, size_type N, typename... Args>
+constexpr GKO_ATTRIBUTES
+    std::enable_if_t<N == 0 || (iter == N && iter == sizeof...(Args) + 1),
+                     std::array<ValueType, N == 0 ? 0 : N - 1>>
+    compute_default_stride_array_impl(const std::array<ValueType, N> &,
+                                      Args &&...args)
+{
+    return {{std::forward<Args>(args)...}};
+}
+
+template <size_type iter = 1, typename ValueType, size_type N, typename... Args>
+constexpr GKO_ATTRIBUTES std::enable_if_t<
+    (iter < N) && (iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
+compute_default_stride_array_impl(const std::array<ValueType, N> &size,
+                                  Args &&...args)
+{
+    return compute_default_stride_array_impl<iter + 1>(
+        size, std::forward<Args>(args)..., mult_dim_upwards_impl<iter>(size));
+}
+
+
+template <typename ValueType, size_type dimensions>
+constexpr GKO_ATTRIBUTES
+    std::array<ValueType, (dimensions > 0 ? dimensions - 1 : 0)>
+    compute_default_stride_array(const std::array<ValueType, dimensions> &size)
+{
+    return compute_default_stride_array_impl(size);
+}
+
+
+template <size_type iter, typename ValueType, size_type N, typename Callable,
+          typename... Indices>
+GKO_ATTRIBUTES std::enable_if_t<iter == N> multidim_for_each_impl(
+    const std::array<ValueType, N> &, Callable callable, Indices &&...indices)
+{
+    static_assert(iter == sizeof...(Indices),
+                  "Number arguments must match current iteration!");
+    callable(std::forward<Indices>(indices)...);
+}
+
+template <size_type iter, typename ValueType, size_type N, typename Callable,
+          typename... Indices>
+GKO_ATTRIBUTES std::enable_if_t<(iter < N)> multidim_for_each_impl(
+    const std::array<ValueType, N> &size, Callable &&callable,
+    Indices &&...indices)
+{
+    static_assert(iter == sizeof...(Indices),
+                  "Number arguments must match current iteration!");
+    for (size_type i = 0; i < size[iter]; ++i) {
+        multidim_for_each_impl<iter + 1>(size, std::forward<Callable>(callable),
+                                         std::forward<Indices>(indices)..., i);
+    }
+}
+
+
+/**
+ * Creates a recursive for-loop for each dimension and calls dest(indices...) =
+ * source(indices...)
+ */
+template <typename ValueType, size_type N, typename Callable>
+GKO_ATTRIBUTES void multidim_for_each(const std::array<ValueType, N> &size,
+                                      Callable &&callable)
+{
+    multidim_for_each_impl<0>(size, std::forward<Callable>(callable));
+}
+
+
+}  // namespace detail
+
+
 /**
  * A row_major accessor is a bridge between a range and the row-major memory
  * layout.
@@ -59,19 +283,20 @@ namespace accessor {
  * constructor parameters for this class to the range (it will forward it to
  * this class).
  *
- * @warning The current implementation is incomplete, and only allows for
- *          2-dimensional ranges.
+ * @warning For backward compatability reasons, a specialization is provided
+ *          for dimensionality == 2.
  *
  * @tparam ValueType  type of values this accessor returns
- * @tparam Dimensionality  number of dimensions of this accessor (has to be 2)
+ * @tparam Dimensionality  number of dimensions of this accessor
  */
 template <typename ValueType, size_type Dimensionality>
 class row_major {
 public:
     friend class range<row_major>;
 
-    static_assert(Dimensionality == 2,
-                  "This accessor is only implemented for matrices");
+    static_assert(Dimensionality != 0,
+                  "This accessor does not support a dimensionality of 0!");
+    static constexpr size_type dimensionality = Dimensionality;
 
     /**
      * Type of values returned by the accessor.
@@ -86,9 +311,179 @@ class row_major {
     /**
      * Number of dimensions of the accessor.
      */
-    static constexpr size_type dimensionality = 2;
 
     using const_accessor = row_major<const ValueType, Dimensionality>;
+    using stride_type = std::array<const size_type, dimensionality - 1>;
+    using length_type = std::array<const size_type, dimensionality>;
+
+protected:
+    /**
+     * Creates a row_major accessor.
+     *
+     * @param data  pointer to the block of memory containing the data
+     * @param lengths size / length of the accesses of each dimension
+     * @param stride  distance (in elements) between starting positions of
+     *                the dimensions (i.e.
+     *                `x_1 * stride_1 + x_2 * stride_2 * ... + x_n`
+     *                points to the element at (x_1, x_2, ..., x_n))
+     */
+    constexpr GKO_ATTRIBUTES explicit row_major(data_type data,
+                                                dim<dimensionality> size,
+                                                stride_type stride)
+        : data{data},
+          lengths{detail::to_array<const size_type>(size)},
+          stride{stride}
+    {}
+
+    /**
+     * Creates a row_major accessor with a default stride (assumes no padding)
+     *
+     * @param data  pointer to the block of memory containing the data
+     * @param lengths size / length of the accesses of each dimension
+     */
+    constexpr GKO_ATTRIBUTES explicit row_major(data_type data,
+                                                dim<dimensionality> size)
+        : data{data},
+          lengths{detail::to_array<const size_type>(size)},
+          stride{detail::compute_default_stride_array(lengths)}
+    {}
+
+public:
+    /**
+     * Creates a row_major range which contains a read-only version of the
+     * current accessor.
+     *
+     * @returns  a row major range which is read-only.
+     */
+    constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
+    {
+        // TODO Remove this functionality all together (if requested)
+        return range<const_accessor>(data, lengths, stride);
+    }
+
+    /**
+     * Returns the data element at the specified indices
+     *
+     * @param row  row index
+     * @param col  column index
+     *
+     * @return data element at (indices...)
+     */
+    template <typename... Indices>
+    constexpr GKO_ATTRIBUTES
+        std::enable_if_t<are_all_integral<Indices...>::value, value_type &>
+        operator()(Indices &&...indices) const
+    {
+        return data[detail::compute_storage_index(
+            lengths, stride, std::forward<Indices>(indices)...)];
+    }
+
+    /**
+     * Returns the sub-range spanning the range (x1_span, x2_span, ...)
+     *
+     * @param rows  row span
+     * @param cols  column span
+     *
+     * @return sub-range spanning the given spans
+     */
+    template <typename... SpanTypes>
+    constexpr GKO_ATTRIBUTES std::enable_if_t<
+        detail::are_span_compatible<SpanTypes...>::value, range<row_major>>
+    operator()(SpanTypes... spans) const
+    {
+        return detail::validate_spans(lengths, spans...),
+               range<row_major>{
+                   data + detail::compute_storage_index(lengths, stride,
+                                                        (span{spans}.begin)...),
+                   dim<dimensionality>{
+                       (span{spans}.end - span{spans}.begin)...},
+                   stride};
+    }
+
+    /**
+     * Returns the length in dimension `dimension`.
+     *
+     * @param dimension  a dimension index
+     *
+     * @return length in dimension `dimension`
+     */
+    constexpr GKO_ATTRIBUTES size_type length(size_type dimension) const
+    {
+        return lengths[dimension];
+    }
+
+    /**
+     * Copies data from another accessor
+     *
+     * @warning Do not use this function since it is not optimized for a
+     *          specific executor. It will always be performed sequentially.
+     *          Please write an optimized version (adjusted to the architecture)
+     *          by iterating through the values yourself.
+     *
+     * @tparam OtherAccessor  type of the other accessor
+     *
+     * @param other  other accessor
+     */
+    template <typename OtherAccessor>
+    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
+    {
+        detail::multidim_for_each(lengths, [this, &other](auto... indices) {
+            (*this)(indices...) = other(indices...);
+        });
+    }
+
+    /**
+     * Reference to the underlying data.
+     */
+    const data_type data;
+
+    /**
+     * An array of dimension sizes.
+     */
+    const length_type lengths;
+
+    /**
+     * Distance between consecutive rows for each dimension (except the first).
+     */
+    const stride_type stride;
+};
+
+
+/**
+ * A row_major accessor is a bridge between a range and the row-major memory
+ * layout.
+ *
+ * You should never try to explicitly create an instance of this accessor.
+ * Instead, supply it as a template parameter to a range, and pass the
+ * constructor parameters for this class to the range (it will forward it to
+ * this class).
+ *
+ * @note  This is the original implementation, which is now the specialization
+ *        for Dimensionality = 2.
+ *
+ * @tparam ValueType  type of values this accessor returns
+ */
+template <typename ValueType>
+class row_major<ValueType, 2> {
+public:
+    friend class range<row_major>;
+    static constexpr size_type dimensionality = 2;
+
+    /**
+     * Type of values returned by the accessor.
+     */
+    using value_type = ValueType;
+
+    /**
+     * Type of underlying data storage.
+     */
+    using data_type = value_type *;
+
+    /**
+     * Number of dimensions of the accessor.
+     */
+
+    using const_accessor = row_major<const ValueType, dimensionality>;
 
 protected:
     /**
@@ -108,6 +503,36 @@ class row_major {
         : data{data}, lengths{num_rows, num_cols}, stride{stride}
     {}
 
+    /**
+     * Creates a row_major accessor.
+     *
+     * @param data  pointer to the block of memory containing the data
+     * @param lengths size / length of the accesses of each dimension
+     * @param stride  distance (in elements) between starting positions of
+     *                consecutive rows (i.e. `data + i * stride` points to
+     *                the `i`-th row)
+     */
+    constexpr GKO_ATTRIBUTES explicit row_major(data_type data,
+                                                dim<dimensionality> size,
+                                                size_type stride)
+        : data{data},
+          lengths{detail::to_array<const size_type>(size)},
+          stride{stride}
+    {}
+
+    /**
+     * Creates a row_major accessor with a default stride (assumes no padding)
+     *
+     * @param data  pointer to the block of memory containing the data
+     * @param lengths size / length of the accesses of each dimension
+     */
+    constexpr GKO_ATTRIBUTES explicit row_major(data_type data,
+                                                dim<dimensionality> size)
+        : data{data},
+          lengths{detail::to_array<const size_type>(size)},
+          stride{size[1]}
+    {}
+
 public:
     /**
      * Creates a row_major range which contains a read-only version of the

From 10ff42789166f4a86f4fbde16460f93c3018afa3 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Thu, 18 Feb 2021 15:56:39 +0100
Subject: [PATCH 0455/1008] added multidimensional column-major accessor and
 tests

---
 core/test/base/range_accessors.cpp           | 133 ++++++++
 include/ginkgo/core/base/range_accessors.hpp | 320 ++++++++++++++++++-
 2 files changed, 442 insertions(+), 11 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 7c0a104822e..0694541278b 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -275,4 +275,137 @@ TEST_F(RowMajorAccessor3d, CanAssignSubranges)
 }
 
 
+class ColMajorAccessor3d : public ::testing::Test {
+protected:
+    using span = gko::span;
+    static constexpr gko::size_type dimensionality{3};
+
+    using col_major_range =
+        gko::range<gko::accessor::col_major<int, dimensionality>>;
+
+    // clang-format off
+    int data[2 * 3 * 4]{
+         1, 3, 5,
+         2, 4, 6,
+		-1,-2,-3,
+		11,12,13,
+
+		21,25,29,
+		22,26,30,
+		23,27,31,
+		24,28,32
+
+		/* This matrix actually looks like
+        1, 2, -1, 11,
+        3, 4, -2, 12,
+        5, 6, -3, 13,
+
+        21, 22, 23, 24,
+        25, 26, 27, 28,
+        29, 30, 31, 32
+	    */
+    };
+    // clang-format on
+    const gko::dim<dimensionality> dim1{2, 3, 4};
+    const gko::dim<dimensionality> dim2{2, 2, 3};
+    col_major_range default_r{data, dim1};
+    col_major_range custom_r{
+        data, dim2,
+        std::array<const gko::size_type, dimensionality - 1>{12, 3}};
+};
+
+
+TEST_F(ColMajorAccessor3d, ComputesCorrectStride)
+{
+    auto range_stride = default_r.get_accessor().stride;
+    auto check_stride = std::array<const gko::size_type, 2>{12, 3};
+
+    ASSERT_EQ(range_stride, check_stride);
+}
+
+
+TEST_F(ColMajorAccessor3d, CanAccessData)
+{
+    EXPECT_EQ(default_r(0, 0, 0), 1);
+    EXPECT_EQ(custom_r(0, 0, 0), 1);
+    EXPECT_EQ(default_r(0, 1, 0), 3);
+    EXPECT_EQ(custom_r(0, 1, 0), 3);
+    EXPECT_EQ(default_r(0, 1, 1), 4);
+    EXPECT_EQ(default_r(0, 1, 3), 12);
+    EXPECT_EQ(default_r(0, 2, 2), -3);
+    EXPECT_EQ(default_r(1, 2, 1), 30);
+    EXPECT_EQ(default_r(1, 2, 2), 31);
+    EXPECT_EQ(default_r(1, 2, 3), 32);
+}
+
+
+TEST_F(ColMajorAccessor3d, CanWriteData)
+{
+    default_r(0, 0, 0) = 4;
+    custom_r(1, 1, 1) = 100;
+
+    EXPECT_EQ(default_r(0, 0, 0), 4);
+    EXPECT_EQ(custom_r(0, 0, 0), 4);
+    EXPECT_EQ(default_r(1, 1, 1), 100);
+    EXPECT_EQ(custom_r(1, 1, 1), 100);
+}
+
+
+TEST_F(ColMajorAccessor3d, CanCreateSubrange)
+{
+    auto subr = custom_r(span{0, 2}, span{1, 2}, span{1, 3});
+
+    EXPECT_EQ(subr(0, 0, 0), 4);
+    EXPECT_EQ(subr(0, 0, 1), -2);
+    EXPECT_EQ(subr(1, 0, 0), 26);
+    EXPECT_EQ(subr(1, 0, 1), 27);
+}
+
+
+TEST_F(ColMajorAccessor3d, CanCreateRowVector)
+{
+    auto subr = default_r(1u, 2u, span{0, 2});
+
+    EXPECT_EQ(subr(0, 0, 0), 29);
+    EXPECT_EQ(subr(0, 0, 1), 30);
+}
+
+
+TEST_F(ColMajorAccessor3d, CanCreateColumnVector)
+{
+    auto subr = default_r(span{0u, 2u}, 1u, 3u);
+
+    EXPECT_EQ(subr(0, 0, 0), 12);
+    EXPECT_EQ(subr(1, 0, 0), 28);
+}
+
+
+TEST_F(ColMajorAccessor3d, CanAssignValues)
+{
+    default_r(1, 1, 1) = default_r(0, 0, 0);
+
+    EXPECT_EQ(data[16], 1);
+}
+
+
+TEST_F(ColMajorAccessor3d, CanAssignSubranges)
+{
+    default_r(1u, span{0, 2}, span{0, 3}) =
+        custom_r(0u, span{0, 2}, span{0, 3});
+
+    EXPECT_EQ(data[12], 1);
+    EXPECT_EQ(data[15], 2);
+    EXPECT_EQ(data[18], -1);
+    EXPECT_EQ(data[21], 24);
+    EXPECT_EQ(data[13], 3);
+    EXPECT_EQ(data[16], 4);
+    EXPECT_EQ(data[19], -2);
+    EXPECT_EQ(data[22], 28);
+    EXPECT_EQ(data[14], 29);
+    EXPECT_EQ(data[17], 30);
+    EXPECT_EQ(data[20], 31);
+    EXPECT_EQ(data[23], 32);
+}
+
+
 }  // namespace
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index b0528956b38..902f5239769 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -73,7 +73,7 @@ struct row_major_helper_s {
     compute(const std::array<ValueType, total_dim> &size,
             const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
                 &stride,
-            FirstType first, Indices &&...idxs)
+            FirstType first, Indices &&... idxs)
     {
         return GKO_ASSERT(first < size[dim_idx]),
                first * stride[dim_idx] +
@@ -103,7 +103,7 @@ template <typename ValueType, size_type total_dim, typename... Indices>
 constexpr GKO_ATTRIBUTES ValueType compute_storage_index(
     const std::array<ValueType, total_dim> &size,
     const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)> &stride,
-    Indices &&...idxs)
+    Indices &&... idxs)
 {
     return row_major_helper_s<ValueType, total_dim>::compute(
         size, stride, std::forward<Indices>(idxs)...);
@@ -120,7 +120,8 @@ constexpr GKO_ATTRIBUTES std::enable_if_t<iter == N, int> spans_in_size(
 template <size_type iter, typename ValueType, size_type N, typename First,
           typename... Remaining>
 constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), int> spans_in_size(
-    const std::array<ValueType, N> &size, First first, Remaining &&...remaining)
+    const std::array<ValueType, N> &size, First first,
+    Remaining &&... remaining)
 {
     static_assert(sizeof...(Remaining) + 1 == N - iter,
                   "Number of remaining spans must be equal to N - iter");
@@ -132,7 +133,7 @@ constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), int> spans_in_size(
 
 template <typename ValueType, size_type N, typename... Spans>
 constexpr GKO_ATTRIBUTES int validate_spans(
-    const std::array<ValueType, N> &size, Spans &&...spans)
+    const std::array<ValueType, N> &size, Spans &&... spans)
 {
     return detail::spans_in_size<0>(size, std::forward<Spans>(spans)...);
 }
@@ -164,7 +165,7 @@ using are_span_compatible = are_span_compatible_impl<false, Args...>;
 template <typename ValueType, size_type N, size_type current, typename... Dims>
 constexpr GKO_ATTRIBUTES
     std::enable_if_t<(current == N), std::array<ValueType, N>>
-    to_array_impl(const dim<N> &size, Dims &&...dims)
+    to_array_impl(const dim<N> &size, Dims &&... dims)
 {
     static_assert(sizeof...(Dims) == N,
                   "Number of arguments must match dimensionality!");
@@ -175,7 +176,7 @@ constexpr GKO_ATTRIBUTES
 template <typename ValueType, size_type N, size_type current, typename... Dims>
 constexpr GKO_ATTRIBUTES
     std::enable_if_t<(current < N), std::array<ValueType, N>>
-    to_array_impl(const dim<N> &size, Dims &&...dims)
+    to_array_impl(const dim<N> &size, Dims &&... dims)
 {
     return to_array_impl<ValueType, N, current + 1>(
         size, std::forward<Dims>(dims)..., size[current]);
@@ -209,7 +210,7 @@ constexpr GKO_ATTRIBUTES
     std::enable_if_t<N == 0 || (iter == N && iter == sizeof...(Args) + 1),
                      std::array<ValueType, N == 0 ? 0 : N - 1>>
     compute_default_stride_array_impl(const std::array<ValueType, N> &,
-                                      Args &&...args)
+                                      Args &&... args)
 {
     return {{std::forward<Args>(args)...}};
 }
@@ -218,7 +219,7 @@ template <size_type iter = 1, typename ValueType, size_type N, typename... Args>
 constexpr GKO_ATTRIBUTES std::enable_if_t<
     (iter < N) && (iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
 compute_default_stride_array_impl(const std::array<ValueType, N> &size,
-                                  Args &&...args)
+                                  Args &&... args)
 {
     return compute_default_stride_array_impl<iter + 1>(
         size, std::forward<Args>(args)..., mult_dim_upwards_impl<iter>(size));
@@ -237,7 +238,7 @@ constexpr GKO_ATTRIBUTES
 template <size_type iter, typename ValueType, size_type N, typename Callable,
           typename... Indices>
 GKO_ATTRIBUTES std::enable_if_t<iter == N> multidim_for_each_impl(
-    const std::array<ValueType, N> &, Callable callable, Indices &&...indices)
+    const std::array<ValueType, N> &, Callable callable, Indices &&... indices)
 {
     static_assert(iter == sizeof...(Indices),
                   "Number arguments must match current iteration!");
@@ -248,7 +249,7 @@ template <size_type iter, typename ValueType, size_type N, typename Callable,
           typename... Indices>
 GKO_ATTRIBUTES std::enable_if_t<(iter < N)> multidim_for_each_impl(
     const std::array<ValueType, N> &size, Callable &&callable,
-    Indices &&...indices)
+    Indices &&... indices)
 {
     static_assert(iter == sizeof...(Indices),
                   "Number arguments must match current iteration!");
@@ -372,7 +373,7 @@ class row_major {
     template <typename... Indices>
     constexpr GKO_ATTRIBUTES
         std::enable_if_t<are_all_integral<Indices...>::value, value_type &>
-        operator()(Indices &&...indices) const
+        operator()(Indices &&... indices) const
     {
         return data[detail::compute_storage_index(
             lengths, stride, std::forward<Indices>(indices)...)];
@@ -630,6 +631,303 @@ class row_major<ValueType, 2> {
 };
 
 
+namespace detail_colmajor {
+
+
+/**
+ * This helper runs from first to last dimension in order to compute the index.
+ * The index is computed like this:
+ * indices: x1, x2, x3, ..., xn
+ * compute(stride, x1, x2, x3, ..., x(n-1), xn) ->
+ *  x1 * stride[0] + x2 * stride[1] + ...
+ *    + x(n-2) * stride[n-3] + x(n-1) + xn * stride[n-2]
+ * Note that swap of the last two strides, making this 'column major'.
+ */
+template <typename ValueType, size_type total_dim, size_type current_iter = 1>
+struct index_helper_s {
+    static_assert(total_dim >= 1, "Dimensionality must be >= 1");
+    static_assert(current_iter <= total_dim, "Iteration must be <= total_dim!");
+
+    static constexpr size_type dim_idx{current_iter - 1};
+
+    template <typename FirstType, typename... Indices>
+    static constexpr GKO_ATTRIBUTES ValueType
+    compute(const std::array<ValueType, total_dim> &size,
+            const std::array<ValueType, (total_dim > 0 ? total_dim - 1 : 0)>
+                &stride,
+            FirstType first, Indices &&... idxs)
+    {
+        if (current_iter == total_dim - 1) {
+            return first +
+                   index_helper_s<ValueType, total_dim, current_iter + 1>::
+                       compute(size, stride, std::forward<Indices>(idxs)...);
+        }
+
+        return GKO_ASSERT(first < size[dim_idx]),
+               first * stride[dim_idx] +
+                   index_helper_s<ValueType, total_dim, current_iter + 1>::
+                       compute(size, stride, std::forward<Indices>(idxs)...);
+    }
+};
+
+template <typename ValueType, size_type total_dim>
+struct index_helper_s<ValueType, total_dim, total_dim> {
+    static_assert(total_dim >= 2, "Dimensionality must be >= 2");
+
+    // static constexpr size_type current_iter = total_dim;
+    static constexpr size_type dim_idx{total_dim - 1};
+
+    template <typename FirstType, typename... Indices>
+    static constexpr GKO_ATTRIBUTES ValueType
+    compute(const std::array<ValueType, total_dim> &size,
+            const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
+                &stride,
+            FirstType first, Indices &&... idxs)
+    {
+        return GKO_ASSERT(first < size[total_dim - 1]),
+               first * stride[dim_idx - 1];
+    }
+};
+
+/**
+ * Computes the storage index for the given indices with respect to the given
+ * stride array
+ */
+template <typename ValueType, size_type total_dim, typename... Indices>
+constexpr GKO_ATTRIBUTES ValueType compute_index(
+    const std::array<ValueType, total_dim> &size,
+    const std::array<ValueType, (total_dim > 0 ? total_dim - 1 : 0)> &stride,
+    Indices &&... idxs)
+{
+    return index_helper_s<ValueType, total_dim>::compute(
+        size, stride, std::forward<Indices>(idxs)...);
+}
+
+
+template <size_type iter, typename ValueType, size_type N>
+constexpr GKO_ATTRIBUTES std::enable_if_t<iter == N, ValueType>
+mult_dim_upwards_impl(const std::array<ValueType, N> &size)
+{
+    return 1;
+}
+
+template <size_type iter, typename ValueType, size_type N>
+constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), ValueType>
+mult_dim_upwards_impl(const std::array<ValueType, N> &size)
+{
+    return size[iter] * mult_dim_upwards_impl<iter + 1>(size);
+}
+
+template <size_type iter = 1, typename ValueType, size_type N, typename... Args>
+constexpr GKO_ATTRIBUTES
+    std::enable_if_t<(iter == N - 1) && (iter == sizeof...(Args) + 1),
+                     std::array<ValueType, N - 1>>
+    default_stride_array_impl(const std::array<ValueType, N> &size,
+                              Args &&... args)
+{
+    return {{std::forward<Args>(args)..., size[N - 2]}};
+}
+
+template <size_type iter = 1, typename ValueType, size_type N, typename... Args>
+constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N - 1 || iter == N) &&
+                                              (iter == sizeof...(Args) + 1),
+                                          std::array<ValueType, N - 1>>
+default_stride_array_impl(const std::array<ValueType, N> &size, Args &&... args)
+{
+    return default_stride_array_impl<iter + 1>(
+        size, std::forward<Args>(args)..., mult_dim_upwards_impl<iter>(size));
+}
+
+template <typename ValueType, size_type dimensions>
+constexpr GKO_ATTRIBUTES
+    std::array<ValueType, (dimensions > 0 ? dimensions - 1 : 0)>
+    default_stride_array(const std::array<ValueType, dimensions> &size)
+{
+    return default_stride_array_impl(size);
+}
+
+
+}  // namespace detail_colmajor
+
+
+/**
+ * A bridge between a range and a column-major memory layout.
+ *
+ * Note that this is not completely a 'layout right'. Only the innermost
+ * two dimensions are regarded as defining a column-major matrix, and
+ * the rest of the dimensions are treated identically to \ref row_major.
+ *
+ * You should not try to explicitly create an instance of this accessor.
+ * Instead, supply it as a template parameter to a range, and pass the
+ * constructor parameters for this class to the range (it will forward it to
+ * this class).
+ *
+ * @tparam ValueType  type of values this accessor returns
+ * @tparam Dimensionality  number of dimensions of this accessor
+ */
+template <typename ValueType, size_type Dimensionality>
+class col_major {
+public:
+    friend class range<col_major>;
+
+    static_assert(Dimensionality != 0,
+                  "This accessor does not support a dimensionality of 0!");
+    static_assert(Dimensionality != 1,
+                  "Please use row_major accessor for 1D ranges.");
+    static constexpr size_type dimensionality = Dimensionality;
+
+    /**
+     * Type of values returned by the accessor.
+     */
+    using value_type = ValueType;
+
+    /**
+     * Type of underlying data storage.
+     */
+    using data_type = value_type *;
+
+    /**
+     * Number of dimensions of the accessor.
+     */
+
+    using const_accessor = col_major<const ValueType, Dimensionality>;
+    using stride_type = std::array<const size_type, dimensionality - 1>;
+    using length_type = std::array<const size_type, dimensionality>;
+
+protected:
+    /**
+     * Creates a col_major accessor.
+     *
+     * @param data  pointer to the block of memory containing the data
+     * @param lengths size / length of the accesses of each dimension
+     * @param stride  distance (in elements) between starting positions of
+     *                the dimensions (i.e.
+     *   `x_1 * stride_1 + x_2 * stride_2 * ... + x_(n-1) + x_n * stride_(n-1)`
+     *                points to the element at (x_1, x_2, ..., x_n))
+     */
+    constexpr GKO_ATTRIBUTES explicit col_major(data_type data,
+                                                dim<dimensionality> size,
+                                                stride_type stride)
+        : data{data},
+          lengths{detail::to_array<const size_type>(size)},
+          stride{stride}
+    {}
+
+    /**
+     * Creates a col_major accessor with a default stride (assumes no padding)
+     *
+     * @param data  pointer to the block of memory containing the data
+     * @param lengths size / length of the accesses of each dimension
+     */
+    constexpr GKO_ATTRIBUTES explicit col_major(data_type data,
+                                                dim<dimensionality> size)
+        : data{data},
+          lengths{detail::to_array<const size_type>(size)},
+          stride{detail_colmajor::default_stride_array(lengths)}
+    {}
+
+public:
+    /**
+     * Creates a col_major range which contains a read-only version of the
+     * current accessor.
+     *
+     * @returns  a col major range which is read-only.
+     */
+    constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
+    {
+        // TODO Remove this functionality all together (if requested)
+        return range<const_accessor>(data, lengths, stride);
+    }
+
+    /**
+     * Returns the data element at the specified indices
+     *
+     * @param row  row index
+     * @param col  column index
+     *
+     * @return data element at (indices...)
+     */
+    template <typename... Indices>
+    constexpr GKO_ATTRIBUTES
+        std::enable_if_t<are_all_integral<Indices...>::value, value_type &>
+        operator()(Indices &&... indices) const
+    {
+        return data[detail_colmajor::compute_index(
+            lengths, stride, std::forward<Indices>(indices)...)];
+    }
+
+    /**
+     * Returns the sub-range spanning the range (x1_span, x2_span, ...)
+     *
+     * @param rows  row span
+     * @param cols  column span
+     *
+     * @return sub-range spanning the given spans
+     */
+    template <typename... SpanTypes>
+    constexpr GKO_ATTRIBUTES std::enable_if_t<
+        detail::are_span_compatible<SpanTypes...>::value, range<col_major>>
+    operator()(SpanTypes... spans) const
+    {
+        return detail::validate_spans(lengths, spans...),
+               range<col_major>{
+                   data + detail_colmajor::compute_index(
+                              lengths, stride, (span{spans}.begin)...),
+                   dim<dimensionality>{
+                       (span{spans}.end - span{spans}.begin)...},
+                   stride};
+    }
+
+    /**
+     * Returns the length in dimension `dimension`.
+     *
+     * @param dimension  a dimension index
+     *
+     * @return length in dimension `dimension`
+     */
+    constexpr GKO_ATTRIBUTES size_type length(size_type dimension) const
+    {
+        return lengths[dimension];
+    }
+
+    /**
+     * Copies data from another accessor
+     *
+     * @warning Do not use this function since it is not optimized for a
+     *          specific executor. It will always be performed sequentially.
+     *          Please write an optimized version (adjusted to the architecture)
+     *          by iterating through the values yourself.
+     *
+     * @tparam OtherAccessor  type of the other accessor
+     *
+     * @param other  other accessor
+     */
+    template <typename OtherAccessor>
+    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
+    {
+        detail::multidim_for_each(lengths, [this, &other](auto... indices) {
+            (*this)(indices...) = other(indices...);
+        });
+    }
+
+    /**
+     * Reference to the underlying data.
+     */
+    const data_type data;
+
+    /**
+     * An array of dimension sizes.
+     */
+    const length_type lengths;
+
+    /**
+     * Distance between consecutive 'layers' for each dimension
+     * (except the second, for which it is 1).
+     */
+    const stride_type stride;
+};
+
+
 }  // namespace accessor
 }  // namespace gko
 

From df4e8d8c4cded7c9c429e246fea6e54c93fb1d72 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Fri, 19 Feb 2021 21:07:17 +0100
Subject: [PATCH 0456/1008] attempt to fix compile issues with Intel 17

---
 include/ginkgo/core/base/range_accessors.hpp | 22 ++++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 902f5239769..39c2b0b5f07 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -332,8 +332,8 @@ class row_major {
                                                 dim<dimensionality> size,
                                                 stride_type stride)
         : data{data},
-          lengths{detail::to_array<const size_type>(size)},
-          stride{stride}
+          lengths(detail::to_array<const size_type>(size)),
+          stride(stride)
     {}
 
     /**
@@ -345,8 +345,8 @@ class row_major {
     constexpr GKO_ATTRIBUTES explicit row_major(data_type data,
                                                 dim<dimensionality> size)
         : data{data},
-          lengths{detail::to_array<const size_type>(size)},
-          stride{detail::compute_default_stride_array(lengths)}
+          lengths(detail::to_array<const size_type>(size)),
+          stride(detail::compute_default_stride_array(lengths))
     {}
 
 public:
@@ -517,8 +517,8 @@ class row_major<ValueType, 2> {
                                                 dim<dimensionality> size,
                                                 size_type stride)
         : data{data},
-          lengths{detail::to_array<const size_type>(size)},
-          stride{stride}
+          lengths(detail::to_array<const size_type>(size)),
+          stride(stride)
     {}
 
     /**
@@ -530,7 +530,7 @@ class row_major<ValueType, 2> {
     constexpr GKO_ATTRIBUTES explicit row_major(data_type data,
                                                 dim<dimensionality> size)
         : data{data},
-          lengths{detail::to_array<const size_type>(size)},
+          lengths(detail::to_array<const size_type>(size)),
           stride{size[1]}
     {}
 
@@ -809,8 +809,8 @@ class col_major {
                                                 dim<dimensionality> size,
                                                 stride_type stride)
         : data{data},
-          lengths{detail::to_array<const size_type>(size)},
-          stride{stride}
+          lengths(detail::to_array<const size_type>(size)),
+          stride(stride)
     {}
 
     /**
@@ -822,8 +822,8 @@ class col_major {
     constexpr GKO_ATTRIBUTES explicit col_major(data_type data,
                                                 dim<dimensionality> size)
         : data{data},
-          lengths{detail::to_array<const size_type>(size)},
-          stride{detail_colmajor::default_stride_array(lengths)}
+          lengths(detail::to_array<const size_type>(size)),
+          stride(detail_colmajor::default_stride_array(lengths))
     {}
 
 public:

From 2e3b2177d5bfa6dea7a91e5d6300ace53ac1c77f Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Mon, 22 Feb 2021 16:12:46 +0100
Subject: [PATCH 0457/1008] removed some redundant code and unused variables

---
 include/ginkgo/core/base/range_accessors.hpp | 22 ++++----------------
 1 file changed, 4 insertions(+), 18 deletions(-)

diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 39c2b0b5f07..aa91b025e17 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -165,7 +165,7 @@ using are_span_compatible = are_span_compatible_impl<false, Args...>;
 template <typename ValueType, size_type N, size_type current, typename... Dims>
 constexpr GKO_ATTRIBUTES
     std::enable_if_t<(current == N), std::array<ValueType, N>>
-    to_array_impl(const dim<N> &size, Dims &&... dims)
+    to_array_impl(const dim<N> &, Dims &&... dims)
 {
     static_assert(sizeof...(Dims) == N,
                   "Number of arguments must match dimensionality!");
@@ -674,7 +674,6 @@ template <typename ValueType, size_type total_dim>
 struct index_helper_s<ValueType, total_dim, total_dim> {
     static_assert(total_dim >= 2, "Dimensionality must be >= 2");
 
-    // static constexpr size_type current_iter = total_dim;
     static constexpr size_type dim_idx{total_dim - 1};
 
     template <typename FirstType, typename... Indices>
@@ -682,7 +681,7 @@ struct index_helper_s<ValueType, total_dim, total_dim> {
     compute(const std::array<ValueType, total_dim> &size,
             const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
                 &stride,
-            FirstType first, Indices &&... idxs)
+            FirstType first, Indices &&...)
     {
         return GKO_ASSERT(first < size[total_dim - 1]),
                first * stride[dim_idx - 1];
@@ -704,20 +703,6 @@ constexpr GKO_ATTRIBUTES ValueType compute_index(
 }
 
 
-template <size_type iter, typename ValueType, size_type N>
-constexpr GKO_ATTRIBUTES std::enable_if_t<iter == N, ValueType>
-mult_dim_upwards_impl(const std::array<ValueType, N> &size)
-{
-    return 1;
-}
-
-template <size_type iter, typename ValueType, size_type N>
-constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), ValueType>
-mult_dim_upwards_impl(const std::array<ValueType, N> &size)
-{
-    return size[iter] * mult_dim_upwards_impl<iter + 1>(size);
-}
-
 template <size_type iter = 1, typename ValueType, size_type N, typename... Args>
 constexpr GKO_ATTRIBUTES
     std::enable_if_t<(iter == N - 1) && (iter == sizeof...(Args) + 1),
@@ -735,7 +720,8 @@ constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N - 1 || iter == N) &&
 default_stride_array_impl(const std::array<ValueType, N> &size, Args &&... args)
 {
     return default_stride_array_impl<iter + 1>(
-        size, std::forward<Args>(args)..., mult_dim_upwards_impl<iter>(size));
+        size, std::forward<Args>(args)...,
+        detail::mult_dim_upwards_impl<iter>(size));
 }
 
 template <typename ValueType, size_type dimensions>

From ed0587bdb2689819438bad98288baedbefada758 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Tue, 23 Feb 2021 22:25:19 +0100
Subject: [PATCH 0458/1008] Addressed review comments including renaming the
 new accessor to block_col_major
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 core/test/base/range_accessors.cpp           | 26 +++---
 include/ginkgo/core/base/range_accessors.hpp | 86 +++++++++++---------
 2 files changed, 60 insertions(+), 52 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 0694541278b..7027bed7420 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -275,13 +275,13 @@ TEST_F(RowMajorAccessor3d, CanAssignSubranges)
 }
 
 
-class ColMajorAccessor3d : public ::testing::Test {
+class BlockColMajorAccessor3d : public ::testing::Test {
 protected:
     using span = gko::span;
     static constexpr gko::size_type dimensionality{3};
 
-    using col_major_range =
-        gko::range<gko::accessor::col_major<int, dimensionality>>;
+    using blk_col_major_range =
+        gko::range<gko::accessor::block_col_major<int, dimensionality>>;
 
     // clang-format off
     int data[2 * 3 * 4]{
@@ -308,14 +308,14 @@ class ColMajorAccessor3d : public ::testing::Test {
     // clang-format on
     const gko::dim<dimensionality> dim1{2, 3, 4};
     const gko::dim<dimensionality> dim2{2, 2, 3};
-    col_major_range default_r{data, dim1};
-    col_major_range custom_r{
+    blk_col_major_range default_r{data, dim1};
+    blk_col_major_range custom_r{
         data, dim2,
         std::array<const gko::size_type, dimensionality - 1>{12, 3}};
 };
 
 
-TEST_F(ColMajorAccessor3d, ComputesCorrectStride)
+TEST_F(BlockColMajorAccessor3d, ComputesCorrectStride)
 {
     auto range_stride = default_r.get_accessor().stride;
     auto check_stride = std::array<const gko::size_type, 2>{12, 3};
@@ -324,7 +324,7 @@ TEST_F(ColMajorAccessor3d, ComputesCorrectStride)
 }
 
 
-TEST_F(ColMajorAccessor3d, CanAccessData)
+TEST_F(BlockColMajorAccessor3d, CanAccessData)
 {
     EXPECT_EQ(default_r(0, 0, 0), 1);
     EXPECT_EQ(custom_r(0, 0, 0), 1);
@@ -339,7 +339,7 @@ TEST_F(ColMajorAccessor3d, CanAccessData)
 }
 
 
-TEST_F(ColMajorAccessor3d, CanWriteData)
+TEST_F(BlockColMajorAccessor3d, CanWriteData)
 {
     default_r(0, 0, 0) = 4;
     custom_r(1, 1, 1) = 100;
@@ -351,7 +351,7 @@ TEST_F(ColMajorAccessor3d, CanWriteData)
 }
 
 
-TEST_F(ColMajorAccessor3d, CanCreateSubrange)
+TEST_F(BlockColMajorAccessor3d, CanCreateSubrange)
 {
     auto subr = custom_r(span{0, 2}, span{1, 2}, span{1, 3});
 
@@ -362,7 +362,7 @@ TEST_F(ColMajorAccessor3d, CanCreateSubrange)
 }
 
 
-TEST_F(ColMajorAccessor3d, CanCreateRowVector)
+TEST_F(BlockColMajorAccessor3d, CanCreateRowVector)
 {
     auto subr = default_r(1u, 2u, span{0, 2});
 
@@ -371,7 +371,7 @@ TEST_F(ColMajorAccessor3d, CanCreateRowVector)
 }
 
 
-TEST_F(ColMajorAccessor3d, CanCreateColumnVector)
+TEST_F(BlockColMajorAccessor3d, CanCreateColumnVector)
 {
     auto subr = default_r(span{0u, 2u}, 1u, 3u);
 
@@ -380,7 +380,7 @@ TEST_F(ColMajorAccessor3d, CanCreateColumnVector)
 }
 
 
-TEST_F(ColMajorAccessor3d, CanAssignValues)
+TEST_F(BlockColMajorAccessor3d, CanAssignValues)
 {
     default_r(1, 1, 1) = default_r(0, 0, 0);
 
@@ -388,7 +388,7 @@ TEST_F(ColMajorAccessor3d, CanAssignValues)
 }
 
 
-TEST_F(ColMajorAccessor3d, CanAssignSubranges)
+TEST_F(BlockColMajorAccessor3d, CanAssignSubranges)
 {
     default_r(1u, span{0, 2}, span{0, 3}) =
         custom_r(0u, span{0, 2}, span{0, 3});
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index aa91b025e17..8e380fa6ee4 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -468,6 +468,10 @@ template <typename ValueType>
 class row_major<ValueType, 2> {
 public:
     friend class range<row_major>;
+
+    /**
+     * Number of dimensions of the accessor.
+     */
     static constexpr size_type dimensionality = 2;
 
     /**
@@ -480,10 +484,6 @@ class row_major<ValueType, 2> {
      */
     using data_type = value_type *;
 
-    /**
-     * Number of dimensions of the accessor.
-     */
-
     using const_accessor = row_major<const ValueType, dimensionality>;
 
 protected:
@@ -631,7 +631,11 @@ class row_major<ValueType, 2> {
 };
 
 
-namespace detail_colmajor {
+/**
+ * Namespace for helper functions and structs for
+ * the block column major accessor.
+ */
+namespace detail_block_col_major {
 
 
 /**
@@ -658,9 +662,11 @@ struct index_helper_s {
             FirstType first, Indices &&... idxs)
     {
         if (current_iter == total_dim - 1) {
-            return first +
-                   index_helper_s<ValueType, total_dim, current_iter + 1>::
-                       compute(size, stride, std::forward<Indices>(idxs)...);
+            return GKO_ASSERT(first < size[dim_idx]),
+                   first +
+                       index_helper_s<ValueType, total_dim, current_iter + 1>::
+                           compute(size, stride,
+                                   std::forward<Indices>(idxs)...);
         }
 
         return GKO_ASSERT(first < size[dim_idx]),
@@ -676,12 +682,12 @@ struct index_helper_s<ValueType, total_dim, total_dim> {
 
     static constexpr size_type dim_idx{total_dim - 1};
 
-    template <typename FirstType, typename... Indices>
+    template <typename FirstType>
     static constexpr GKO_ATTRIBUTES ValueType
     compute(const std::array<ValueType, total_dim> &size,
             const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
                 &stride,
-            FirstType first, Indices &&...)
+            FirstType first)
     {
         return GKO_ASSERT(first < size[total_dim - 1]),
                first * stride[dim_idx - 1];
@@ -733,15 +739,15 @@ constexpr GKO_ATTRIBUTES
 }
 
 
-}  // namespace detail_colmajor
+}  // namespace detail_block_col_major
 
 
 /**
- * A bridge between a range and a column-major memory layout.
+ * A bridge between a range and a block-column-major memory layout.
  *
- * Note that this is not completely a 'layout right'. Only the innermost
- * two dimensions are regarded as defining a column-major matrix, and
- * the rest of the dimensions are treated identically to \ref row_major.
+ * Only the innermost two dimensions are regarded as defining
+ * a column-major matrix, and the rest of the dimensions are treated
+ * identically to \ref row_major.
  *
  * You should not try to explicitly create an instance of this accessor.
  * Instead, supply it as a template parameter to a range, and pass the
@@ -752,14 +758,18 @@ constexpr GKO_ATTRIBUTES
  * @tparam Dimensionality  number of dimensions of this accessor
  */
 template <typename ValueType, size_type Dimensionality>
-class col_major {
+class block_col_major {
 public:
-    friend class range<col_major>;
+    friend class range<block_col_major>;
 
     static_assert(Dimensionality != 0,
                   "This accessor does not support a dimensionality of 0!");
     static_assert(Dimensionality != 1,
                   "Please use row_major accessor for 1D ranges.");
+
+    /**
+     * Number of dimensions of the accessor.
+     */
     static constexpr size_type dimensionality = Dimensionality;
 
     /**
@@ -772,17 +782,13 @@ class col_major {
      */
     using data_type = value_type *;
 
-    /**
-     * Number of dimensions of the accessor.
-     */
-
-    using const_accessor = col_major<const ValueType, Dimensionality>;
+    using const_accessor = block_col_major<const ValueType, Dimensionality>;
     using stride_type = std::array<const size_type, dimensionality - 1>;
     using length_type = std::array<const size_type, dimensionality>;
 
 protected:
     /**
-     * Creates a col_major accessor.
+     * Creates a block_col_major accessor.
      *
      * @param data  pointer to the block of memory containing the data
      * @param lengths size / length of the accesses of each dimension
@@ -791,33 +797,34 @@ class col_major {
      *   `x_1 * stride_1 + x_2 * stride_2 * ... + x_(n-1) + x_n * stride_(n-1)`
      *                points to the element at (x_1, x_2, ..., x_n))
      */
-    constexpr GKO_ATTRIBUTES explicit col_major(data_type data,
-                                                dim<dimensionality> size,
-                                                stride_type stride)
+    constexpr GKO_ATTRIBUTES explicit block_col_major(data_type data,
+                                                      dim<dimensionality> size,
+                                                      stride_type stride)
         : data{data},
           lengths(detail::to_array<const size_type>(size)),
           stride(stride)
     {}
 
     /**
-     * Creates a col_major accessor with a default stride (assumes no padding)
+     * Creates a block_col_major accessor with a default stride
+     * (assumes no padding)
      *
      * @param data  pointer to the block of memory containing the data
      * @param lengths size / length of the accesses of each dimension
      */
-    constexpr GKO_ATTRIBUTES explicit col_major(data_type data,
-                                                dim<dimensionality> size)
+    constexpr GKO_ATTRIBUTES explicit block_col_major(data_type data,
+                                                      dim<dimensionality> size)
         : data{data},
           lengths(detail::to_array<const size_type>(size)),
-          stride(detail_colmajor::default_stride_array(lengths))
+          stride(detail_block_col_major::default_stride_array(lengths))
     {}
 
 public:
     /**
-     * Creates a col_major range which contains a read-only version of the
-     * current accessor.
+     * Creates a block_col_major range which contains a read-only version of
+     * the current accessor.
      *
-     * @returns  a col major range which is read-only.
+     * @returns  a block column major range which is read-only.
      */
     constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
     {
@@ -838,7 +845,7 @@ class col_major {
         std::enable_if_t<are_all_integral<Indices...>::value, value_type &>
         operator()(Indices &&... indices) const
     {
-        return data[detail_colmajor::compute_index(
+        return data[detail_block_col_major::compute_index(
             lengths, stride, std::forward<Indices>(indices)...)];
     }
 
@@ -851,13 +858,14 @@ class col_major {
      * @return sub-range spanning the given spans
      */
     template <typename... SpanTypes>
-    constexpr GKO_ATTRIBUTES std::enable_if_t<
-        detail::are_span_compatible<SpanTypes...>::value, range<col_major>>
-    operator()(SpanTypes... spans) const
+    constexpr GKO_ATTRIBUTES
+        std::enable_if_t<detail::are_span_compatible<SpanTypes...>::value,
+                         range<block_col_major>>
+        operator()(SpanTypes... spans) const
     {
         return detail::validate_spans(lengths, spans...),
-               range<col_major>{
-                   data + detail_colmajor::compute_index(
+               range<block_col_major>{
+                   data + detail_block_col_major::compute_index(
                               lengths, stride, (span{spans}.begin)...),
                    dim<dimensionality>{
                        (span{spans}.end - span{spans}.begin)...},

From 439831ca743dd0e0e938f91a5b07059a23dcb594 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 24 Feb 2021 00:36:56 +0100
Subject: [PATCH 0459/1008] some documentation improvements

---
 include/ginkgo/core/base/range_accessors.hpp | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 8e380fa6ee4..6d2a3f328db 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -97,7 +97,11 @@ struct row_major_helper_s<ValueType, total_dim, total_dim> {
 
 /**
  * Computes the storage index for the given indices with respect to the given
- * stride array
+ * stride array for row-major access
+ *
+ * @param size  the multi-dimensional sizes of the range of values
+ * @param stride  the stride array
+ * @param idxs  the multi-dimensional indices of the desired entry
  */
 template <typename ValueType, size_type total_dim, typename... Indices>
 constexpr GKO_ATTRIBUTES ValueType compute_storage_index(
@@ -639,13 +643,14 @@ namespace detail_block_col_major {
 
 
 /**
- * This helper runs from first to last dimension in order to compute the index.
+ * Runs from first to last dimension in order to compute the index.
+ *
  * The index is computed like this:
  * indices: x1, x2, x3, ..., xn
  * compute(stride, x1, x2, x3, ..., x(n-1), xn) ->
  *  x1 * stride[0] + x2 * stride[1] + ...
  *    + x(n-2) * stride[n-3] + x(n-1) + xn * stride[n-2]
- * Note that swap of the last two strides, making this 'column major'.
+ * Note that swap of the last two strides, making this 'block column major'.
  */
 template <typename ValueType, size_type total_dim, size_type current_iter = 1>
 struct index_helper_s {
@@ -695,8 +700,11 @@ struct index_helper_s<ValueType, total_dim, total_dim> {
 };
 
 /**
- * Computes the storage index for the given indices with respect to the given
- * stride array
+ * Computes the flat storage index for block-column-major access.
+ *
+ * @param size  the multi-dimensional sizes of the range of values
+ * @param stride  the stride array
+ * @param idxs  the multi-dimensional indices of the desired entry
  */
 template <typename ValueType, size_type total_dim, typename... Indices>
 constexpr GKO_ATTRIBUTES ValueType compute_index(

From 0294913072e369552fc8d8d7f6d7950b659900d6 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Tue, 9 Mar 2021 00:47:16 +0100
Subject: [PATCH 0460/1008] minor cosmetic improvements

---
 core/test/base/range_accessors.cpp           | 16 ++++++++--------
 include/ginkgo/core/base/range_accessors.hpp | 12 +++++++-----
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 7027bed7420..7a5ffe7c18a 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -287,15 +287,15 @@ class BlockColMajorAccessor3d : public ::testing::Test {
     int data[2 * 3 * 4]{
          1, 3, 5,
          2, 4, 6,
-		-1,-2,-3,
-		11,12,13,
+        -1,-2,-3,
+        11,12,13,
 
-		21,25,29,
-		22,26,30,
-		23,27,31,
-		24,28,32
+        21,25,29,
+        22,26,30,
+        23,27,31,
+        24,28,32
 
-		/* This matrix actually looks like
+        /* This matrix actually looks like
         1, 2, -1, 11,
         3, 4, -2, 12,
         5, 6, -3, 13,
@@ -303,7 +303,7 @@ class BlockColMajorAccessor3d : public ::testing::Test {
         21, 22, 23, 24,
         25, 26, 27, 28,
         29, 30, 31, 32
-	    */
+        */
     };
     // clang-format on
     const gko::dim<dimensionality> dim1{2, 3, 4};
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index 6d2a3f328db..c4abdde4e45 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -635,11 +635,12 @@ class row_major<ValueType, 2> {
 };
 
 
+namespace detail {
 /**
  * Namespace for helper functions and structs for
  * the block column major accessor.
  */
-namespace detail_block_col_major {
+namespace blk_col_major {
 
 
 /**
@@ -747,7 +748,8 @@ constexpr GKO_ATTRIBUTES
 }
 
 
-}  // namespace detail_block_col_major
+}  // namespace blk_col_major
+}  // namespace detail
 
 
 /**
@@ -824,7 +826,7 @@ class block_col_major {
                                                       dim<dimensionality> size)
         : data{data},
           lengths(detail::to_array<const size_type>(size)),
-          stride(detail_block_col_major::default_stride_array(lengths))
+          stride(detail::blk_col_major::default_stride_array(lengths))
     {}
 
 public:
@@ -853,7 +855,7 @@ class block_col_major {
         std::enable_if_t<are_all_integral<Indices...>::value, value_type &>
         operator()(Indices &&... indices) const
     {
-        return data[detail_block_col_major::compute_index(
+        return data[detail::blk_col_major::compute_index(
             lengths, stride, std::forward<Indices>(indices)...)];
     }
 
@@ -873,7 +875,7 @@ class block_col_major {
     {
         return detail::validate_spans(lengths, spans...),
                range<block_col_major>{
-                   data + detail_block_col_major::compute_index(
+                   data + detail::blk_col_major::compute_index(
                               lengths, stride, (span{spans}.begin)...),
                    dim<dimensionality>{
                        (span{spans}.end - span{spans}.begin)...},

From 1157c11c44369208fbd05b1af686b01ccb79c7e2 Mon Sep 17 00:00:00 2001
From: yhmtsai <yhmtsai@gmail.com>
Date: Wed, 22 Apr 2020 18:37:42 +0200
Subject: [PATCH 0461/1008] create EnableCFOp, CFOpFactory and implement
 matrix-dependent function inside AmgxPgm

---
 core/CMakeLists.txt                           |   1 +
 core/device_hooks/common_kernels.inc.cpp      |  63 +++
 core/multigrid/amgx_pgm.cpp                   | 172 ++++++++
 core/multigrid/amgx_pgm_kernels.hpp           | 180 ++++++++
 core/test/CMakeLists.txt                      |   1 +
 core/test/multigrid/CMakeLists.txt            |   1 +
 core/test/multigrid/amgx_pgm.cpp              |  48 ++
 cuda/CMakeLists.txt                           |   1 +
 cuda/multigrid/amgx_pgm_kernels.cu            | 169 +++++++
 cuda/test/CMakeLists.txt                      |   1 +
 cuda/test/multigrid/CMakeLists.txt            |   1 +
 cuda/test/multigrid/amgx_pgm_kernels.cpp      |  56 +++
 hip/CMakeLists.txt                            |   1 +
 hip/multigrid/amgx_pgm_kernels.hip.cpp        | 169 +++++++
 hip/test/CMakeLists.txt                       |   1 +
 hip/test/multigrid/CMakeLists.txt             |   1 +
 hip/test/multigrid/amgx_pgm_kernels.cpp       |  53 +++
 include/ginkgo/core/base/coarse_fine.hpp      | 307 +++++++++++++
 include/ginkgo/core/matrix/csr.hpp            |   1 +
 include/ginkgo/core/multigrid/amgx_pgm.hpp    | 152 +++++++
 omp/CMakeLists.txt                            |   1 +
 omp/multigrid/amgx_pgm_kernels.cpp            | 166 +++++++
 omp/test/CMakeLists.txt                       |   1 +
 omp/test/multigrid/CMakeLists.txt             |   1 +
 omp/test/multigrid/amgx_pgm_kernels.cpp       |  53 +++
 reference/CMakeLists.txt                      |   1 +
 reference/multigrid/amgx_pgm_kernels.cpp      | 376 ++++++++++++++++
 reference/test/CMakeLists.txt                 |   1 +
 reference/test/matrix/csr_kernels.cpp         |   1 +
 reference/test/multigrid/CMakeLists.txt       |   1 +
 reference/test/multigrid/amgx_pgm_kernels.cpp | 413 ++++++++++++++++++
 31 files changed, 2394 insertions(+)
 create mode 100644 core/multigrid/amgx_pgm.cpp
 create mode 100644 core/multigrid/amgx_pgm_kernels.hpp
 create mode 100644 core/test/multigrid/CMakeLists.txt
 create mode 100644 core/test/multigrid/amgx_pgm.cpp
 create mode 100644 cuda/multigrid/amgx_pgm_kernels.cu
 create mode 100644 cuda/test/multigrid/CMakeLists.txt
 create mode 100644 cuda/test/multigrid/amgx_pgm_kernels.cpp
 create mode 100644 hip/multigrid/amgx_pgm_kernels.hip.cpp
 create mode 100644 hip/test/multigrid/CMakeLists.txt
 create mode 100644 hip/test/multigrid/amgx_pgm_kernels.cpp
 create mode 100644 include/ginkgo/core/base/coarse_fine.hpp
 create mode 100644 include/ginkgo/core/multigrid/amgx_pgm.hpp
 create mode 100644 omp/multigrid/amgx_pgm_kernels.cpp
 create mode 100644 omp/test/multigrid/CMakeLists.txt
 create mode 100644 omp/test/multigrid/amgx_pgm_kernels.cpp
 create mode 100644 reference/multigrid/amgx_pgm_kernels.cpp
 create mode 100644 reference/test/multigrid/CMakeLists.txt
 create mode 100644 reference/test/multigrid/amgx_pgm_kernels.cpp

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index e8a206089e5..a4f8594249f 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -31,6 +31,7 @@ target_sources(ginkgo
     matrix/permutation.cpp
     matrix/sellp.cpp
     matrix/sparsity_csr.cpp
+    multigrid/amgx_pgm.cpp
     preconditioner/isai.cpp
     preconditioner/jacobi.cpp
     reorder/rcm.cpp
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 4afd590647c..52b08f98d98 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -52,6 +52,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/hybrid_kernels.hpp"
 #include "core/matrix/sellp_kernels.hpp"
 #include "core/matrix/sparsity_csr_kernels.hpp"
+#include "core/multigrid/amgx_pgm_kernels.hpp"
 #include "core/preconditioner/isai_kernels.hpp"
 #include "core/preconditioner/jacobi_kernels.hpp"
 #include "core/reorder/rcm_kernels.hpp"
@@ -1203,6 +1204,68 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 }  // namespace rcm
 
 
+namespace amgx_pgm {
+
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
+
+template <typename IndexType>
+GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL(IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL);
+
+template <typename IndexType>
+GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL(IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL);
+
+template <typename IndexType>
+GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL(IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL);
+
+template <typename IndexType>
+GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL(IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_AMGX_PGM_GENERATE(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
+
+
+}  // namespace amgx_pgm
+
+
 namespace set_all_statuses {
 
 
diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
new file mode 100644
index 00000000000..d422503ca19
--- /dev/null
+++ b/core/multigrid/amgx_pgm.cpp
@@ -0,0 +1,172 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/polymorphic_object.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/base/utils.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/matrix/csr_builder.hpp"
+#include "core/multigrid/amgx_pgm_kernels.hpp"
+
+
+namespace gko {
+namespace multigrid {
+namespace amgx_pgm {
+
+
+GKO_REGISTER_OPERATION(restrict_apply, amgx_pgm::restrict_apply);
+GKO_REGISTER_OPERATION(prolongate_applyadd, amgx_pgm::prolongate_applyadd);
+GKO_REGISTER_OPERATION(initial, amgx_pgm::initial);
+GKO_REGISTER_OPERATION(match_edge, amgx_pgm::match_edge);
+GKO_REGISTER_OPERATION(count_unagg, amgx_pgm::count_unagg);
+GKO_REGISTER_OPERATION(renumber, amgx_pgm::renumber);
+GKO_REGISTER_OPERATION(extract_diag, amgx_pgm::extract_diag);
+GKO_REGISTER_OPERATION(find_strongest_neighbor,
+                       amgx_pgm::find_strongest_neighbor);
+GKO_REGISTER_OPERATION(assign_to_exist_agg, amgx_pgm::assign_to_exist_agg);
+GKO_REGISTER_OPERATION(amgx_pgm_generate, amgx_pgm::amgx_pgm_generate);
+
+
+}  // namespace amgx_pgm
+
+namespace {
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<LinOp> amgx_pgm_generate(
+    std::shared_ptr<const Executor> exec,
+    const matrix::Csr<ValueType, IndexType> *source, const size_type num_agg,
+    const Array<IndexType> &agg)
+{
+    auto coarse = matrix::Csr<ValueType, IndexType>::create(
+        exec, dim<2>{num_agg, num_agg}, 0, source->get_strategy());
+    exec->run(amgx_pgm::make_amgx_pgm_generate(source, agg, coarse.get()));
+    matrix::CsrBuilder<ValueType, IndexType> builder(coarse.get());
+    return std::move(coarse);
+}
+
+
+}  // namespace
+
+
+template <typename ValueType, typename IndexType>
+void AmgxPgm<ValueType, IndexType>::generate()
+{
+    using matrix_type = matrix::Csr<ValueType, IndexType>;
+    auto exec = this->get_executor();
+    const auto num = this->system_matrix_->get_size()[0];
+    Array<ValueType> diag(this->get_executor(), num);
+    Array<IndexType> strongest_neighbor(this->get_executor(), num);
+    Array<IndexType> intermediate_agg(this->get_executor(),
+                                      parameters_.deterministic * num);
+    const auto amgxpgm_op = gko::as<matrix_type>(this->system_matrix_.get());
+    // Initial agg = -1
+    exec->run(amgx_pgm::make_initial(agg_));
+    // Extract the diagonal value of matrix
+    exec->run(amgx_pgm::make_extract_diag(amgxpgm_op, diag));
+    size_type num_unagg{0};
+    size_type num_unagg_prev{0};
+    for (int i = 0; i < parameters_.max_iterations; i++) {
+        // Find the strongest neighbor of each row
+        exec->run(amgx_pgm::make_find_strongest_neighbor(amgxpgm_op, diag, agg_,
+                                                         strongest_neighbor));
+        // Match edges
+        exec->run(amgx_pgm::make_match_edge(strongest_neighbor, agg_));
+        // Get the num_unagg
+        exec->run(amgx_pgm::make_count_unagg(agg_, &num_unagg));
+        // no new match, all match, or the ratio of num_unagg/num is lower
+        // than parameter.max_unassigned_percentage
+        if (num_unagg == 0 || num_unagg == num_unagg_prev ||
+            static_cast<double>(num_unagg) / num <
+                parameters_.max_unassigned_percentage) {
+            break;
+        }
+        num_unagg_prev = num_unagg;
+    }
+    // Handle the left unassign points
+    if (num_unagg != 0 && intermediate_agg.get_num_elems() > 0) {
+        // copy the agg to intermediate_agg
+        intermediate_agg = agg_;
+    }
+    while (num_unagg != 0) {
+        exec->run(amgx_pgm::make_assign_to_exist_agg(amgxpgm_op, diag, agg_,
+                                                     intermediate_agg));
+        exec->run(amgx_pgm::make_count_unagg(agg_, &num_unagg));
+    }
+    size_type num_agg;
+    // Renumber the index
+    exec->run(amgx_pgm::make_renumber(agg_, &num_agg));
+
+
+    // Construct the coarse matrix
+    auto coarse = amgx_pgm_generate(exec, amgxpgm_op, num_agg, agg_);
+    this->set_coarse_fine(std::move(coarse), num);
+}
+
+
+template <typename ValueType, typename IndexType>
+void AmgxPgm<ValueType, IndexType>::restrict_apply_impl(const LinOp *b,
+                                                        LinOp *x) const
+{
+    auto exec = this->get_executor();
+    exec->run(amgx_pgm::make_restrict_apply(agg_,
+                                            as<matrix::Dense<ValueType>>(b),
+                                            as<matrix::Dense<ValueType>>(x)));
+}
+
+
+template <typename ValueType, typename IndexType>
+void AmgxPgm<ValueType, IndexType>::prolongate_applyadd_impl(const LinOp *b,
+                                                             LinOp *x) const
+{
+    auto exec = this->get_executor();
+    exec->run(amgx_pgm::make_prolongate_applyadd(
+        agg_, as<matrix::Dense<ValueType>>(b),
+        as<matrix::Dense<ValueType>>(x)));
+}
+
+
+#define GKO_DECLARE_AMGX_PGM(_vtype, _itype) class AmgxPgm<_vtype, _itype>
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM);
+
+
+}  // namespace multigrid
+}  // namespace gko
diff --git a/core/multigrid/amgx_pgm_kernels.hpp b/core/multigrid/amgx_pgm_kernels.hpp
new file mode 100644
index 00000000000..2edefd61ea2
--- /dev/null
+++ b/core/multigrid/amgx_pgm_kernels.hpp
@@ -0,0 +1,180 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_SOLVER_AMGX_PGM_KERNELS_HPP_
+#define GKO_CORE_SOLVER_AMGX_PGM_KERNELS_HPP_
+
+
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace amgx_pgm {
+
+
+#define GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL(_vtype, _itype)             \
+    void restrict_apply(                                                       \
+        std::shared_ptr<const DefaultExecutor> exec, const Array<_itype> &agg, \
+        const matrix::Dense<_vtype> *b, matrix::Dense<_vtype> *x)
+
+
+#define GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL(_vtype, _itype)           \
+    void prolongate_applyadd(                                                  \
+        std::shared_ptr<const DefaultExecutor> exec, const Array<_itype> &agg, \
+        const matrix::Dense<_vtype> *b, matrix::Dense<_vtype> *x)
+
+
+#define GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL(_itype)           \
+    void initial(std::shared_ptr<const DefaultExecutor> exec, \
+                 Array<_itype> &agg)
+
+
+#define GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL(_itype)           \
+    void match_edge(std::shared_ptr<const DefaultExecutor> exec, \
+                    const Array<_itype> &strongest_neighbor,     \
+                    Array<_itype> &agg)
+
+
+#define GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL(_itype)           \
+    void count_unagg(std::shared_ptr<const DefaultExecutor> exec, \
+                     const Array<_itype> &agg, size_type *num_unagg)
+
+
+#define GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL(_itype)           \
+    void renumber(std::shared_ptr<const DefaultExecutor> exec, \
+                  Array<_itype> &agg, size_type *num_agg)
+
+#define GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG(ValueType, IndexType)        \
+    void extract_diag(std::shared_ptr<const DefaultExecutor> exec,     \
+                      const matrix::Csr<ValueType, IndexType> *source, \
+                      Array<ValueType> &diag)
+
+#define GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR(ValueType, IndexType) \
+    void find_strongest_neighbor(                                          \
+        std::shared_ptr<const DefaultExecutor> exec,                       \
+        const matrix::Csr<ValueType, IndexType> *source,                   \
+        const Array<ValueType> &diag, Array<IndexType> &agg,               \
+        Array<IndexType> &strongest_neighbor)
+
+#define GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG(ValueType, IndexType)        \
+    void assign_to_exist_agg(std::shared_ptr<const DefaultExecutor> exec,     \
+                             const matrix::Csr<ValueType, IndexType> *source, \
+                             const Array<ValueType> &diag,                    \
+                             Array<IndexType> &agg,                           \
+                             Array<IndexType> &intermediate_agg)
+
+#define GKO_DECLARE_AMGX_PGM_GENERATE(ValueType, IndexType)                 \
+    void amgx_pgm_generate(std::shared_ptr<const DefaultExecutor> exec,     \
+                           const matrix::Csr<ValueType, IndexType> *source, \
+                           const Array<IndexType> &agg,                     \
+                           matrix::Csr<ValueType, IndexType> *coarse)
+
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES                                    \
+    template <typename ValueType, typename IndexType>                   \
+    GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL(ValueType, IndexType);   \
+    template <typename ValueType, typename IndexType>                   \
+    GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL(ValueType, IndexType); \
+    template <typename IndexType>                                       \
+    GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL(IndexType);                     \
+    template <typename IndexType>                                       \
+    GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL(IndexType);                  \
+    template <typename IndexType>                                       \
+    GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL(IndexType);                 \
+    template <typename IndexType>                                       \
+    GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL(IndexType);                    \
+    template <typename ValueType, typename IndexType>                   \
+    GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG(ValueType, IndexType);            \
+    template <typename ValueType, typename IndexType>                   \
+    GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR(ValueType, IndexType); \
+    template <typename ValueType, typename IndexType>                   \
+    GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG(ValueType, IndexType);     \
+    template <typename ValueType, typename IndexType>                   \
+    GKO_DECLARE_AMGX_PGM_GENERATE(ValueType, IndexType)
+
+
+}  // namespace amgx_pgm
+
+
+namespace omp {
+namespace amgx_pgm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace amgx_pgm
+}  // namespace omp
+
+
+namespace cuda {
+namespace amgx_pgm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace amgx_pgm
+}  // namespace cuda
+
+
+namespace reference {
+namespace amgx_pgm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace amgx_pgm
+}  // namespace reference
+
+
+namespace hip {
+namespace amgx_pgm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace amgx_pgm
+}  // namespace hip
+
+
+#undef GKO_DECLARE_ALL_AS_TEMPLATES
+
+
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_CORE_SOLVER_AMGX_PGM_KERNELS_HPP_
diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt
index aeed9c81392..0639b38a63f 100644
--- a/core/test/CMakeLists.txt
+++ b/core/test/CMakeLists.txt
@@ -4,6 +4,7 @@ add_subdirectory(base)
 add_subdirectory(factorization)
 add_subdirectory(log)
 add_subdirectory(matrix)
+add_subdirectory(multigrid)
 add_subdirectory(preconditioner)
 add_subdirectory(reorder)
 add_subdirectory(solver)
diff --git a/core/test/multigrid/CMakeLists.txt b/core/test/multigrid/CMakeLists.txt
new file mode 100644
index 00000000000..12a292ba7d6
--- /dev/null
+++ b/core/test/multigrid/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_test(amgx_pgm)
\ No newline at end of file
diff --git a/core/test/multigrid/amgx_pgm.cpp b/core/test/multigrid/amgx_pgm.cpp
new file mode 100644
index 00000000000..5982402402b
--- /dev/null
+++ b/core/test/multigrid/amgx_pgm.cpp
@@ -0,0 +1,48 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+
+
+#include <memory>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {}  // namespace
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index b773f4bf039..5e4505d8187 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -101,6 +101,7 @@ target_sources(ginkgo_cuda
     matrix/hybrid_kernels.cu
     matrix/sellp_kernels.cu
     matrix/sparsity_csr_kernels.cu
+    multigrid/amgx_pgm_kernels.cu
     preconditioner/isai_kernels.cu
     preconditioner/jacobi_advanced_apply_kernel.cu
     preconditioner/jacobi_generate_kernel.cu
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
new file mode 100644
index 00000000000..c00d227331e
--- /dev/null
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -0,0 +1,169 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/multigrid/amgx_pgm_kernels.hpp"
+
+
+#include <memory>
+
+
+#include <cuda.h>
+#include <cusparse.h>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+
+
+#include "cuda/base/cusparse_bindings.hpp"
+#include "cuda/base/math.hpp"
+#include "cuda/base/types.hpp"
+#include "cuda/solver/common_trs_kernels.cuh"
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+/**
+ * @brief The AMGX_PGM solver namespace.
+ *
+ * @ingroup amgx_pgm
+ */
+namespace amgx_pgm {
+
+
+template <typename ValueType, typename IndexType>
+void restrict_apply(std::shared_ptr<const CudaExecutor> exec,
+                    const Array<IndexType> &agg,
+                    const matrix::Dense<ValueType> *b,
+                    matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void prolongate_applyadd(std::shared_ptr<const CudaExecutor> exec,
+                         const Array<IndexType> &agg,
+                         const matrix::Dense<ValueType> *b,
+                         matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
+
+
+template <typename IndexType>
+void initial(std::shared_ptr<const CudaExecutor> exec,
+             Array<IndexType> &agg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL);
+
+
+template <typename IndexType>
+void match_edge(std::shared_ptr<const CudaExecutor> exec,
+                const Array<IndexType> &strongest_neighbor,
+                Array<IndexType> &agg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL);
+
+
+template <typename IndexType>
+void count_unagg(std::shared_ptr<const CudaExecutor> exec,
+                 const Array<IndexType> &agg,
+                 size_type *num_unagg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL);
+
+
+template <typename IndexType>
+void renumber(std::shared_ptr<const CudaExecutor> exec, Array<IndexType> &agg,
+              size_type *num_agg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void extract_diag(std::shared_ptr<const CudaExecutor> exec,
+                  const matrix::Csr<ValueType, IndexType> *source,
+                  Array<ValueType> &diag)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
+
+
+template <typename ValueType, typename IndexType>
+void find_strongest_neighbor(std::shared_ptr<const CudaExecutor> exec,
+                             const matrix::Csr<ValueType, IndexType> *source,
+                             const Array<ValueType> &diag,
+                             Array<IndexType> &agg,
+                             Array<IndexType> &strongest_neighbor)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
+
+
+template <typename ValueType, typename IndexType>
+void assign_to_exist_agg(std::shared_ptr<const CudaExecutor> exec,
+                         const matrix::Csr<ValueType, IndexType> *source,
+                         const Array<ValueType> &diag, Array<IndexType> &agg,
+                         Array<IndexType> &intermediate_agg)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
+
+
+template <typename ValueType, typename IndexType>
+void amgx_pgm_generate(std::shared_ptr<const CudaExecutor> exec,
+                       const matrix::Csr<ValueType, IndexType> *source,
+                       const Array<IndexType> &agg,
+                       matrix::Csr<ValueType, IndexType> *coarse)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
+
+
+}  // namespace amgx_pgm
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
diff --git a/cuda/test/CMakeLists.txt b/cuda/test/CMakeLists.txt
index 83d7b39f35e..fb66aaf270f 100644
--- a/cuda/test/CMakeLists.txt
+++ b/cuda/test/CMakeLists.txt
@@ -4,6 +4,7 @@ add_subdirectory(base)
 add_subdirectory(components)
 add_subdirectory(factorization)
 add_subdirectory(matrix)
+add_subdirectory(multigrid)
 add_subdirectory(preconditioner)
 add_subdirectory(reorder)
 add_subdirectory(solver)
diff --git a/cuda/test/multigrid/CMakeLists.txt b/cuda/test/multigrid/CMakeLists.txt
new file mode 100644
index 00000000000..55bea27db18
--- /dev/null
+++ b/cuda/test/multigrid/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_test_cpp_cuda_header(amgx_pgm_kernels)
\ No newline at end of file
diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
new file mode 100644
index 00000000000..b72889b923f
--- /dev/null
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -0,0 +1,56 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+
+
+#include <memory>
+#include <random>
+
+
+#include <cuda.h>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/multigrid/amgx_pgm_kernels.hpp"
+#include "cuda/test/utils.hpp"
+
+
+namespace {}  // namespace
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 4999cbd8e0b..4e34da9466f 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -192,6 +192,7 @@ set(GINKGO_HIP_SOURCES
     matrix/hybrid_kernels.hip.cpp
     matrix/sellp_kernels.hip.cpp
     matrix/sparsity_csr_kernels.hip.cpp
+    multigrid/amgx_pgm_kernels.hip.cpp
     preconditioner/isai_kernels.hip.cpp
     preconditioner/jacobi_advanced_apply_kernel.hip.cpp
     preconditioner/jacobi_generate_kernel.hip.cpp
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
new file mode 100644
index 00000000000..d4583be8dce
--- /dev/null
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -0,0 +1,169 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/multigrid/amgx_pgm_kernels.hpp"
+
+
+#include <memory>
+
+
+#include <hip/hip_runtime.h>
+#include <hipsparse.h>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+
+
+#include "hip/base/hipsparse_bindings.hip.hpp"
+#include "hip/base/math.hip.hpp"
+#include "hip/base/types.hip.hpp"
+#include "hip/solver/common_trs_kernels.hip.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace hip {
+/**
+ * @brief The AMGX_PGM solver namespace.
+ *
+ * @ingroup amgx_pgm
+ */
+namespace amgx_pgm {
+
+
+template <typename ValueType, typename IndexType>
+void restrict_apply(std::shared_ptr<const HipExecutor> exec,
+                    const Array<IndexType> &agg,
+                    const matrix::Dense<ValueType> *b,
+                    matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void prolongate_applyadd(std::shared_ptr<const HipExecutor> exec,
+                         const Array<IndexType> &agg,
+                         const matrix::Dense<ValueType> *b,
+                         matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
+
+
+template <typename IndexType>
+void initial(std::shared_ptr<const HipExecutor> exec,
+             Array<IndexType> &agg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL);
+
+
+template <typename IndexType>
+void match_edge(std::shared_ptr<const HipExecutor> exec,
+                const Array<IndexType> &strongest_neighbor,
+                Array<IndexType> &agg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL);
+
+
+template <typename IndexType>
+void count_unagg(std::shared_ptr<const HipExecutor> exec,
+                 const Array<IndexType> &agg,
+                 size_type *num_unagg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL);
+
+
+template <typename IndexType>
+void renumber(std::shared_ptr<const HipExecutor> exec, Array<IndexType> &agg,
+              size_type *num_agg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void extract_diag(std::shared_ptr<const HipExecutor> exec,
+                  const matrix::Csr<ValueType, IndexType> *source,
+                  Array<ValueType> &diag)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
+
+
+template <typename ValueType, typename IndexType>
+void find_strongest_neighbor(std::shared_ptr<const HipExecutor> exec,
+                             const matrix::Csr<ValueType, IndexType> *source,
+                             const Array<ValueType> &diag,
+                             Array<IndexType> &agg,
+                             Array<IndexType> &strongest_neighbor)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
+
+
+template <typename ValueType, typename IndexType>
+void assign_to_exist_agg(std::shared_ptr<const HipExecutor> exec,
+                         const matrix::Csr<ValueType, IndexType> *source,
+                         const Array<ValueType> &diag, Array<IndexType> &agg,
+                         Array<IndexType> &intermediate_agg)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
+
+
+template <typename ValueType, typename IndexType>
+void amgx_pgm_generate(std::shared_ptr<const HipExecutor> exec,
+                       const matrix::Csr<ValueType, IndexType> *source,
+                       const Array<IndexType> &agg,
+                       matrix::Csr<ValueType, IndexType> *coarse)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
+
+
+}  // namespace amgx_pgm
+}  // namespace hip
+}  // namespace kernels
+}  // namespace gko
diff --git a/hip/test/CMakeLists.txt b/hip/test/CMakeLists.txt
index 9c8e39ca4ab..a5f126893bf 100644
--- a/hip/test/CMakeLists.txt
+++ b/hip/test/CMakeLists.txt
@@ -4,6 +4,7 @@ add_subdirectory(base)
 add_subdirectory(components)
 add_subdirectory(factorization)
 add_subdirectory(matrix)
+add_subdirectory(multigrid)
 add_subdirectory(solver)
 add_subdirectory(preconditioner)
 add_subdirectory(stop)
diff --git a/hip/test/multigrid/CMakeLists.txt b/hip/test/multigrid/CMakeLists.txt
new file mode 100644
index 00000000000..481c2cc1bf2
--- /dev/null
+++ b/hip/test/multigrid/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_hip_test_special_linkage(amgx_pgm_kernels)
diff --git a/hip/test/multigrid/amgx_pgm_kernels.cpp b/hip/test/multigrid/amgx_pgm_kernels.cpp
new file mode 100644
index 00000000000..13c99bc6a2b
--- /dev/null
+++ b/hip/test/multigrid/amgx_pgm_kernels.cpp
@@ -0,0 +1,53 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+
+
+#include <memory>
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/multigrid/amgx_pgm_kernels.hpp"
+#include "hip/test/utils.hip.hpp"
+
+
+namespace {}  // namespace
diff --git a/include/ginkgo/core/base/coarse_fine.hpp b/include/ginkgo/core/base/coarse_fine.hpp
new file mode 100644
index 00000000000..1388cbe676d
--- /dev/null
+++ b/include/ginkgo/core/base/coarse_fine.hpp
@@ -0,0 +1,307 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_BASE_COARSE_FINE_HPP_
+#define GKO_CORE_BASE_COARSE_FINE_HPP_
+
+
+#include <functional>
+#include <memory>
+
+#include <ginkgo/core/base/abstract_factory.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+namespace gko {
+
+
+/**
+ * The CoarseFine class can be used to construct restrict_apply and
+ * prolongate_applyadd. R is the restrict operator (fine level -> coarse
+ * level) and P is prolongate operator (coarse level -> fine level).
+ * restrict_apply(b, x) -> x = R(b)
+ * prolongate_applyadd(b, x) -> x = P(b) + x
+ *
+ * @ingroup CoarseFine
+ */
+
+class CoarseFine : public EnableAbstractPolymorphicObject<CoarseFine> {
+public:
+    /**
+     * Returns the coarse operator (matrix) which is R * matrix * P
+     *
+     * @return the coarse operator (matrix)
+     */
+    std::shared_ptr<const LinOp> get_coarse_operator() const { return coarse_; }
+
+    /**
+     * Applies a restrict operator to a vector (or a sequence of vectors).
+     *
+     * Performs the operation x = restrict(b).
+     *
+     * @param b  the input vector(s) on which the operator is applied
+     * @param x  the output vector(s) where the result is stored
+     *
+     * @return this
+     */
+    CoarseFine *restrict_apply(const LinOp *b, LinOp *x)
+    {
+        this->validate_restrict_parameters(b, x);
+        auto exec = this->get_executor();
+        this->restrict_apply_impl(make_temporary_clone(exec, b).get(),
+                                  make_temporary_clone(exec, x).get());
+        return this;
+    }
+
+    /**
+     * @copydoc restrict_apply(const LinOp *, LinOp *)
+     */
+    const CoarseFine *restrict_apply(const LinOp *b, LinOp *x) const
+    {
+        this->validate_restrict_parameters(b, x);
+        auto exec = this->get_executor();
+        this->restrict_apply_impl(make_temporary_clone(exec, b).get(),
+                                  make_temporary_clone(exec, x).get());
+        return this;
+    }
+
+    /**
+     * Applies a prolongate operator to a vector (or a sequence of vectors).
+     *
+     * Performs the operation x = prolongate(b) + x.
+     *
+     * @param b  the input vector(s) on which the operator is applied
+     * @param x  the output vector(s) where the result is stored
+     *
+     * @return this
+     */
+    CoarseFine *prolongate_applyadd(const LinOp *b, LinOp *x)
+    {
+        this->validate_prolongate_parameters(b, x);
+        auto exec = this->get_executor();
+        this->prolongate_applyadd_impl(make_temporary_clone(exec, b).get(),
+                                       make_temporary_clone(exec, x).get());
+        return this;
+    }
+
+    /**
+     * @copydoc prolongate_applyadd(const LinOp *, LinOp *)
+     */
+    const CoarseFine *prolongate_applyadd(const LinOp *b, LinOp *x) const
+    {
+        this->validate_prolongate_parameters(b, x);
+        auto exec = this->get_executor();
+        this->prolongate_applyadd_impl(make_temporary_clone(exec, b).get(),
+                                       make_temporary_clone(exec, x).get());
+        return this;
+    }
+
+protected:
+    void validate_restrict_parameters(const LinOp *b, LinOp *x) const
+    {
+        GKO_ASSERT_EQ(fine_dim_, b->get_size()[0]);
+        GKO_ASSERT_EQ(coarse_dim_, x->get_size()[0]);
+        GKO_ASSERT_EQUAL_COLS(b, x);
+    }
+    void validate_prolongate_parameters(const LinOp *b, LinOp *x) const
+    {
+        GKO_ASSERT_EQ(coarse_dim_, b->get_size()[0]);
+        GKO_ASSERT_EQ(fine_dim_, x->get_size()[0]);
+        GKO_ASSERT_EQUAL_COLS(b, x);
+    }
+
+    /**
+     * Implementers of CFOp should override this function instead
+     * of restrict_apply(const LinOp *, LinOp *).
+     *
+     * Performs the operation x = restrict(b), where op is the restrict
+     * operator.
+     *
+     * @param b  the input vector(s) on which the operator is applied
+     * @param x  the output vector(s) where the result is stored
+     */
+    virtual void restrict_apply_impl(const LinOp *b, LinOp *x) const = 0;
+
+    /**
+     * Implementers of CFOp should override this function instead
+     * of prolongate_apply(const LinOp *, LinOp *).
+     *
+     * Performs the operation x = prolongate(b), where op is the prolongate
+     * operator.
+     *
+     * @param b  the input vector(s) on which the operator is applied
+     * @param x  the output vector(s) where the result is stored
+     */
+    virtual void prolongate_applyadd_impl(const LinOp *b, LinOp *x) const = 0;
+
+    /**
+     * Sets the components of CoarseFine
+     *
+     * @param coarse  the coarse matrix
+     * @param fine_dim  the fine_level size
+     */
+    void set_coarse_fine(std::shared_ptr<const LinOp> coarse,
+                         size_type fine_dim)
+    {
+        coarse_ = coarse;
+        fine_dim_ = fine_dim;
+        coarse_dim_ = coarse->get_size()[0];
+    }
+
+    /**
+     * Creates a coarse fine.
+     *
+     * @param exec  the executor where all the operations are performed
+     */
+    explicit CoarseFine(std::shared_ptr<const Executor> exec)
+        : EnableAbstractPolymorphicObject<CoarseFine>(exec)
+    {}
+
+private:
+    std::shared_ptr<const LinOp> coarse_{};
+    size_type fine_dim_;
+    size_type coarse_dim_;
+};
+
+class CFFactory
+    : public AbstractFactory<CoarseFine, std::shared_ptr<const LinOp>> {
+public:
+    using AbstractFactory<CoarseFine,
+                          std::shared_ptr<const LinOp>>::AbstractFactory;
+
+    std::unique_ptr<CoarseFine> generate(
+        std::shared_ptr<const LinOp> input) const
+    {
+        auto generated = AbstractFactory::generate(input);
+        return generated;
+    }
+};
+
+template <typename ConcreteFactory, typename ConcreteCriterion,
+          typename ParametersType, typename PolymorphicBase = CFFactory>
+using EnableDefaultCFFactory =
+    EnableDefaultFactory<ConcreteFactory, ConcreteCriterion, ParametersType,
+                         PolymorphicBase>;
+
+
+template <typename ConcreteLinOp, typename PolymorphicBase = CoarseFine>
+class EnableCFOp
+    : public EnablePolymorphicObject<ConcreteLinOp, PolymorphicBase>,
+      public EnablePolymorphicAssignment<ConcreteLinOp> {
+public:
+    using EnablePolymorphicObject<ConcreteLinOp,
+                                  PolymorphicBase>::EnablePolymorphicObject;
+
+    ConcreteLinOp *restrict_apply(const LinOp *b, LinOp *x)
+    {
+        this->validate_restrict_parameters(b, x);
+        auto exec = this->get_executor();
+        this->restrict_apply_impl(make_temporary_clone(exec, b).get(),
+                                  make_temporary_clone(exec, x).get());
+        return self();
+    }
+
+    const ConcreteLinOp *restrict_apply(const LinOp *b, LinOp *x) const
+    {
+        this->validate_restrict_parameters(b, x);
+        auto exec = this->get_executor();
+        this->restrict_apply_impl(make_temporary_clone(exec, b).get(),
+                                  make_temporary_clone(exec, x).get());
+        return self();
+    }
+
+
+    ConcreteLinOp *prolongate_applyadd(const LinOp *b, LinOp *x)
+    {
+        this->validate_prolongate_parameters(b, x);
+        auto exec = this->get_executor();
+        this->prolongate_applyadd_impl(make_temporary_clone(exec, b).get(),
+                                       make_temporary_clone(exec, x).get());
+        return self();
+    }
+
+    const ConcreteLinOp *prolongate_applyadd(const LinOp *b, LinOp *x) const
+    {
+        this->validate_prolongate_parameters(b, x);
+        auto exec = this->get_executor();
+        this->prolongate_applyadd_impl(make_temporary_clone(exec, b).get(),
+                                       make_temporary_clone(exec, x).get());
+        return self();
+    }
+
+protected:
+    GKO_ENABLE_SELF(ConcreteLinOp);
+};
+
+
+#define GKO_ENABLE_CFOP_FACTORY(_cfop, _parameters_name, _factory_name)      \
+public:                                                                      \
+    const _parameters_name##_type &get_##_parameters_name() const            \
+    {                                                                        \
+        return _parameters_name##_;                                          \
+    }                                                                        \
+                                                                             \
+    class _factory_name                                                      \
+        : public ::gko::EnableDefaultCFFactory<_factory_name, _cfop,         \
+                                               _parameters_name##_type> {    \
+        friend class ::gko::EnablePolymorphicObject<_factory_name,           \
+                                                    ::gko::CFFactory>;       \
+        friend class ::gko::enable_parameters_type<_parameters_name##_type,  \
+                                                   _factory_name>;           \
+        explicit _factory_name(std::shared_ptr<const ::gko::Executor> exec)  \
+            : ::gko::EnableDefaultCFFactory<_factory_name, _cfop,            \
+                                            _parameters_name##_type>(        \
+                  std::move(exec))                                           \
+        {}                                                                   \
+        explicit _factory_name(std::shared_ptr<const ::gko::Executor> exec,  \
+                               const _parameters_name##_type &parameters)    \
+            : ::gko::EnableDefaultCFFactory<_factory_name, _cfop,            \
+                                            _parameters_name##_type>(        \
+                  std::move(exec), parameters)                               \
+        {}                                                                   \
+    };                                                                       \
+    friend ::gko::EnableDefaultCFFactory<_factory_name, _cfop,               \
+                                         _parameters_name##_type>;           \
+                                                                             \
+private:                                                                     \
+    _parameters_name##_type _parameters_name##_;                             \
+                                                                             \
+public:                                                                      \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
+                  "semi-colon warnings")
+
+
+}  // namespace gko
+
+
+#endif  // GKO_CORE_BASE_COARSE_FINE_HPP_
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index 7e01c1293f6..72223a37db1 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/coarse_fine.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/base/math.hpp>
 
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
new file mode 100644
index 00000000000..5e90b1a6db1
--- /dev/null
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -0,0 +1,152 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_MULTIGRID_AMGX_PGM_HPP_
+#define GKO_CORE_MULTIGRID_AMGX_PGM_HPP_
+
+
+#include <vector>
+
+
+#include <ginkgo/core/base/coarse_fine.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+namespace gko {
+namespace multigrid {
+
+
+/**
+ * Amgx parallel graph match (AmgxPgm) is the aggregate method introduced in the
+ * paper M. Naumov et al., "AmgX: A Library for GPU Accelerated Algebraic
+ * Multigrid and Preconditioned Iterative Methods"
+ *
+ * AmgxPgm creates the aggreagate group according to the matrix value not the
+ * structure.
+ *
+ * @tparam ValueType  precision of matrix elements
+ * @tparam IndexType  precision of matrix indexes
+ *
+ * @ingroup CoarseFine
+ * @ingroup LinOp
+ */
+template <typename ValueType = default_precision, typename IndexType = int32>
+class AmgxPgm : public EnableCFOp<AmgxPgm<ValueType, IndexType>> {
+    friend class EnableCFOp<AmgxPgm>;
+    friend class EnablePolymorphicObject<AmgxPgm, CoarseFine>;
+
+public:
+    using value_type = ValueType;
+    using index_type = IndexType;
+
+    /**
+     * Returns the system operator (matrix) of the linear system.
+     *
+     * @return the system operator (matrix)
+     */
+    std::shared_ptr<const LinOp> get_system_matrix() const
+    {
+        return system_matrix_;
+    }
+
+    /**
+     * Returns the aggregate group.
+     *
+     * @return the aggregate group.
+     */
+    IndexType *get_agg() noexcept { return agg_.get_data(); }
+
+    /**
+     * @copydoc AmgxPgm::get_agg()
+     *
+     * @note This is the constant version of the function, which can be
+     *       significantly more memory efficient than the non-constant version,
+     *       so always prefer this version.
+     */
+    const IndexType *get_const_agg() const noexcept
+    {
+        return agg_.get_const_data();
+    }
+
+    GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
+    {
+        /**
+         * The maximum number of iteration.
+         */
+        unsigned GKO_FACTORY_PARAMETER(max_iterations, 15);
+
+        /**
+         * The maximum ratio of unassigned number.
+         */
+        double GKO_FACTORY_PARAMETER(max_unassigned_percentage, 0.05);
+
+        /**
+         * Use the deterministic assign_to_exist_agg method or not.
+         */
+        bool GKO_FACTORY_PARAMETER(deterministic, false);
+    };
+    GKO_ENABLE_CFOP_FACTORY(AmgxPgm, parameters, Factory);
+    GKO_ENABLE_BUILD_METHOD(Factory);
+
+protected:
+    void restrict_apply_impl(const LinOp *b, LinOp *x) const override;
+
+    void prolongate_applyadd_impl(const LinOp *b, LinOp *x) const override;
+    explicit AmgxPgm(std::shared_ptr<const Executor> exec)
+        : EnableCFOp<AmgxPgm>(std::move(exec))
+    {}
+    explicit AmgxPgm(const Factory *factory,
+                     std::shared_ptr<const LinOp> system_matrix)
+        : EnableCFOp<AmgxPgm>(factory->get_executor()),
+          parameters_{factory->get_parameters()},
+          system_matrix_{std::move(system_matrix)},
+          agg_(factory->get_executor(), system_matrix_->get_size()[0])
+    {
+        this->generate();
+    }
+
+    void generate();
+
+private:
+    std::shared_ptr<const LinOp> system_matrix_{};
+    Array<IndexType> agg_;
+};
+
+
+}  // namespace multigrid
+}  // namespace gko
+
+
+#endif  // GKO_CORE_MULTIGRID_AMGX_PGM_HPP_
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index fcb8798a045..4fd677d5b5c 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -23,6 +23,7 @@ target_sources(ginkgo_omp
     matrix/hybrid_kernels.cpp
     matrix/sellp_kernels.cpp
     matrix/sparsity_csr_kernels.cpp
+    multigrid/amgx_pgm_kernels.cpp
     preconditioner/isai_kernels.cpp
     preconditioner/jacobi_kernels.cpp
     reorder/rcm_kernels.cpp
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
new file mode 100644
index 00000000000..445411dea24
--- /dev/null
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -0,0 +1,166 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/multigrid/amgx_pgm_kernels.hpp"
+
+
+#include <memory>
+
+
+#include <omp.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+/**
+ * @brief The AMGX_PGM solver namespace.
+ *
+ * @ingroup amgx_pgm
+ */
+namespace amgx_pgm {
+
+
+template <typename ValueType, typename IndexType>
+void restrict_apply(std::shared_ptr<const OmpExecutor> exec,
+                    const Array<IndexType> &agg,
+                    const matrix::Dense<ValueType> *b,
+                    matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void prolongate_applyadd(std::shared_ptr<const OmpExecutor> exec,
+                         const Array<IndexType> &agg,
+                         const matrix::Dense<ValueType> *b,
+                         matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
+
+
+template <typename IndexType>
+void initial(std::shared_ptr<const OmpExecutor> exec,
+             Array<IndexType> &agg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL);
+
+
+template <typename IndexType>
+void match_edge(std::shared_ptr<const OmpExecutor> exec,
+                const Array<IndexType> &strongest_neighbor,
+                Array<IndexType> &agg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL);
+
+
+template <typename IndexType>
+void count_unagg(std::shared_ptr<const OmpExecutor> exec,
+                 const Array<IndexType> &agg,
+                 size_type *num_unagg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL);
+
+
+template <typename IndexType>
+void renumber(std::shared_ptr<const OmpExecutor> exec, Array<IndexType> &agg,
+              size_type *num_agg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void extract_diag(std::shared_ptr<const OmpExecutor> exec,
+                  const matrix::Csr<ValueType, IndexType> *source,
+                  Array<ValueType> &diag)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
+
+
+template <typename ValueType, typename IndexType>
+void find_strongest_neighbor(std::shared_ptr<const OmpExecutor> exec,
+                             const matrix::Csr<ValueType, IndexType> *source,
+                             const Array<ValueType> &diag,
+                             Array<IndexType> &agg,
+                             Array<IndexType> &strongest_neighbor)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
+
+
+template <typename ValueType, typename IndexType>
+void assign_to_exist_agg(std::shared_ptr<const OmpExecutor> exec,
+                         const matrix::Csr<ValueType, IndexType> *source,
+                         const Array<ValueType> &diag, Array<IndexType> &agg,
+                         Array<IndexType> &intermediate_agg)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
+
+
+template <typename ValueType, typename IndexType>
+void amgx_pgm_generate(std::shared_ptr<const OmpExecutor> exec,
+                       const matrix::Csr<ValueType, IndexType> *source,
+                       const Array<IndexType> &agg,
+                       matrix::Csr<ValueType, IndexType> *coarse)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
+
+
+}  // namespace amgx_pgm
+}  // namespace omp
+}  // namespace kernels
+}  // namespace gko
diff --git a/omp/test/CMakeLists.txt b/omp/test/CMakeLists.txt
index 2f506e51109..3759a39bc77 100644
--- a/omp/test/CMakeLists.txt
+++ b/omp/test/CMakeLists.txt
@@ -3,6 +3,7 @@ include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
 add_subdirectory(components)
 add_subdirectory(factorization)
 add_subdirectory(matrix)
+add_subdirectory(multigrid)
 add_subdirectory(preconditioner)
 add_subdirectory(reorder)
 add_subdirectory(solver)
diff --git a/omp/test/multigrid/CMakeLists.txt b/omp/test/multigrid/CMakeLists.txt
new file mode 100644
index 00000000000..8fe8bbeba48
--- /dev/null
+++ b/omp/test/multigrid/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_test(amgx_pgm_kernels)
diff --git a/omp/test/multigrid/amgx_pgm_kernels.cpp b/omp/test/multigrid/amgx_pgm_kernels.cpp
new file mode 100644
index 00000000000..4d2f5ed1fa6
--- /dev/null
+++ b/omp/test/multigrid/amgx_pgm_kernels.cpp
@@ -0,0 +1,53 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+
+
+#include <memory>
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/multigrid/amgx_pgm_kernels.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {}  // namespace
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index 7333eac1d12..b17b75ca8ba 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -21,6 +21,7 @@ target_sources(ginkgo_reference
     matrix/hybrid_kernels.cpp
     matrix/sellp_kernels.cpp
     matrix/sparsity_csr_kernels.cpp
+    multigrid/amgx_pgm_kernels.cpp
     preconditioner/isai_kernels.cpp
     preconditioner/jacobi_kernels.cpp
     reorder/rcm_kernels.cpp
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
new file mode 100644
index 00000000000..6ed7e11b2ef
--- /dev/null
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -0,0 +1,376 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/multigrid/amgx_pgm_kernels.hpp"
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+#include "core/components/prefix_sum.hpp"
+
+
+#include "core/base/allocator.hpp"
+#include "core/matrix/csr_builder.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace reference {
+/**
+ * @brief The AMGX_PGM solver namespace.
+ *
+ * @ingroup amgx_pgm
+ */
+namespace amgx_pgm {
+
+
+template <typename ValueType, typename IndexType>
+void restrict_apply(std::shared_ptr<const ReferenceExecutor> exec,
+                    const Array<IndexType> &agg,
+                    const matrix::Dense<ValueType> *b,
+                    matrix::Dense<ValueType> *x)
+{
+    auto x_vals = x->get_values();
+    const auto x_stride = x->get_stride();
+    auto x_dim = x->get_size();
+    for (size_type i = 0; i < x_dim[0]; i++) {
+        for (size_type j = 0; j < x_dim[1]; j++) {
+            x->at(i, j) = zero<ValueType>();
+        }
+    }
+    for (size_type i = 0; i < agg.get_num_elems(); i++) {
+        const auto x_row = agg.get_const_data()[i];
+        for (size_type j = 0; j < x_dim[1]; j++) {
+            x->at(x_row, j) += b->at(i, j);
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void prolongate_applyadd(std::shared_ptr<const ReferenceExecutor> exec,
+                         const Array<IndexType> &agg,
+                         const matrix::Dense<ValueType> *b,
+                         matrix::Dense<ValueType> *x)
+{
+    auto x_vals = x->get_values();
+    const auto x_stride = x->get_stride();
+    auto x_dim = x->get_size();
+    for (size_type i = 0; i < agg.get_num_elems(); i++) {
+        const auto b_row = agg.get_const_data()[i];
+        for (size_type j = 0; j < x_dim[1]; j++) {
+            x->at(i, j) += b->at(b_row, j);
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
+
+
+template <typename IndexType>
+void initial(std::shared_ptr<const ReferenceExecutor> exec,
+             Array<IndexType> &agg)
+{
+    for (size_type i = 0; i < agg.get_num_elems(); i++) {
+        agg.get_data()[i] = -1;
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL);
+
+
+template <typename IndexType>
+void match_edge(std::shared_ptr<const ReferenceExecutor> exec,
+                const Array<IndexType> &strongest_neighbor,
+                Array<IndexType> &agg)
+{
+    auto agg_vals = agg.get_data();
+    auto strongest_neighbor_vals = strongest_neighbor.get_const_data();
+    for (size_type i = 0; i < agg.get_num_elems(); i++) {
+        if (agg_vals[i] == -1) {
+            auto neighbor = strongest_neighbor_vals[i];
+            if (neighbor != -1 && strongest_neighbor_vals[neighbor] == i) {
+                agg_vals[i] = i;
+                agg_vals[neighbor] = i;
+                // Use the smaller index as agg point
+            }
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL);
+
+
+template <typename IndexType>
+void count_unagg(std::shared_ptr<const ReferenceExecutor> exec,
+                 const Array<IndexType> &agg, size_type *num_unagg)
+{
+    IndexType unagg = 0;
+    for (size_type i = 0; i < agg.get_num_elems(); i++) {
+        unagg += (agg.get_const_data()[i] == -1);
+    }
+    *num_unagg = unagg;
+}
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL);
+
+
+template <typename IndexType>
+void renumber(std::shared_ptr<const ReferenceExecutor> exec,
+              Array<IndexType> &agg, size_type *num_agg)
+{
+    const auto num = agg.get_num_elems();
+    Array<IndexType> agg_map(exec, num + 1);
+    auto agg_vals = agg.get_data();
+    auto agg_map_vals = agg_map.get_data();
+    for (size_type i = 0; i < num + 1; i++) {
+        agg_map_vals[i] = 0;
+    }
+    for (size_type i = 0; i < num; i++) {
+        agg_map_vals[agg_vals[i]] = 1;
+    }
+    components::prefix_sum(exec, agg_map_vals, num + 1);
+    for (size_type i = 0; i < num; i++) {
+        agg_vals[i] = agg_map_vals[agg_vals[i]];
+    }
+    *num_agg = agg_map_vals[num];
+}
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void extract_diag(std::shared_ptr<const ReferenceExecutor> exec,
+                  const matrix::Csr<ValueType, IndexType> *source,
+                  Array<ValueType> &diag)
+{
+    const auto dim_size = source->get_size();
+    const auto row_ptrs = source->get_const_row_ptrs();
+    const auto col_idxs = source->get_const_col_idxs();
+    const auto vals = source->get_const_values();
+    auto diag_val = diag.get_data();
+    for (IndexType i = 0; i < dim_size[0]; i++) {
+        for (auto j = row_ptrs[i]; j < row_ptrs[i + 1]; j++) {
+            if (col_idxs[j] == i) {
+                diag_val[i] = vals[j];
+                break;
+            }
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
+
+
+template <typename ValueType, typename IndexType>
+void find_strongest_neighbor(std::shared_ptr<const ReferenceExecutor> exec,
+                             const matrix::Csr<ValueType, IndexType> *source,
+                             const Array<ValueType> &diag,
+                             Array<IndexType> &agg,
+                             Array<IndexType> &strongest_neighbor)
+{
+    const auto row_ptrs = source->get_const_row_ptrs();
+    const auto col_idxs = source->get_const_col_idxs();
+    const auto vals = source->get_const_values();
+    for (size_type row = 0; row < agg.get_num_elems(); row++) {
+        auto max_weight_unagg = zero<remove_complex<ValueType>>();
+        auto max_weight_agg = zero<remove_complex<ValueType>>();
+        IndexType strongest_unagg = -1;
+        IndexType strongest_agg = -1;
+        if (agg.get_const_data()[row] == -1) {
+            for (auto ind = row_ptrs[row]; ind < row_ptrs[row + 1]; ind++) {
+                auto weight = zero<remove_complex<ValueType>>();
+                auto col = col_idxs[ind];
+                if (col == row) {
+                    continue;
+                }
+                // Handle the unsymmetric values cases
+                for (auto trans_ind = row_ptrs[col];
+                     trans_ind < row_ptrs[col + 1]; trans_ind++) {
+                    if (col_idxs[trans_ind] == row) {
+                        weight = 0.5 * (abs(vals[ind]) + abs(vals[trans_ind])) /
+                                 max(abs(diag.get_const_data()[row]),
+                                     abs(diag.get_const_data()[col]));
+                        break;
+                    }
+                }
+
+                if (agg.get_const_data()[col] == -1 &&
+                    (weight > max_weight_unagg ||
+                     (weight == max_weight_unagg && col > strongest_unagg))) {
+                    max_weight_unagg = weight;
+                    strongest_unagg = col;
+                } else if (agg.get_const_data()[col] != -1 &&
+                           (weight > max_weight_agg ||
+                            (weight == max_weight_agg &&
+                             col > strongest_agg))) {
+                    max_weight_agg = weight;
+                    strongest_agg = col;
+                }
+            }
+
+            if (strongest_unagg == -1 && strongest_agg != -1) {
+                // all neighbor is agg, connect to the strongest agg
+                agg.get_data()[row] = agg.get_data()[strongest_agg];
+            } else if (strongest_unagg != -1) {
+                // set the strongest neighbor in the unagg group
+                strongest_neighbor.get_data()[row] = strongest_unagg;
+            } else {
+                // no neighbor
+                strongest_neighbor.get_data()[row] = row;
+            }
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
+
+
+template <typename ValueType, typename IndexType>
+void assign_to_exist_agg(std::shared_ptr<const ReferenceExecutor> exec,
+                         const matrix::Csr<ValueType, IndexType> *source,
+                         const Array<ValueType> &diag, Array<IndexType> &agg,
+                         Array<IndexType> &intermediate_agg)
+{
+    const auto row_ptrs = source->get_const_row_ptrs();
+    const auto col_idxs = source->get_const_col_idxs();
+    const auto vals = source->get_const_values();
+    const auto diag_vals = diag.get_const_data();
+    auto max_weight_agg = zero<remove_complex<ValueType>>();
+    const auto agg_const_val = agg.get_const_data();
+    auto agg_val = (intermediate_agg.get_num_elems() > 0)
+                       ? intermediate_agg.get_data()
+                       : agg.get_data();
+
+    for (IndexType row = 0; row < agg.get_num_elems(); row++) {
+        if (agg_const_val[row] != -1) {
+            continue;
+        }
+        IndexType strongest_agg = -1;
+        for (auto idx = row_ptrs[row]; idx < row_ptrs[row + 1]; idx++) {
+            auto col = col_idxs[idx];
+            if (col == row) {
+                break;
+            }
+            auto weight =
+                abs(vals[idx]) / max(abs(diag_vals[row]), abs(diag_vals[col]));
+            if (agg_const_val[col] != -1 &&
+                (weight > max_weight_agg ||
+                 (weight == max_weight_agg && col > strongest_agg))) {
+                max_weight_agg = weight;
+                strongest_agg = col;
+            }
+        }
+        if (strongest_agg != -1) {
+            agg_val[row] = agg_const_val[strongest_agg];
+        } else {
+            agg_val[row] = row;
+        }
+    }
+
+    if (intermediate_agg.get_num_elems() > 0) {
+        // Copy the intermediate_agg to agg
+        agg = intermediate_agg;
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
+
+
+template <typename ValueType, typename IndexType>
+void amgx_pgm_generate(std::shared_ptr<const ReferenceExecutor> exec,
+                       const matrix::Csr<ValueType, IndexType> *source,
+                       const Array<IndexType> &agg,
+                       matrix::Csr<ValueType, IndexType> *coarse)
+{
+    // agg[i] -> I, agg[j] -> J
+    const auto coarse_nrows = coarse->get_size()[0];
+    const auto source_nrows = source->get_size()[0];
+    const auto source_row_ptrs = source->get_const_row_ptrs();
+    const auto source_col_idxs = source->get_const_col_idxs();
+    const auto source_vals = source->get_const_values();
+    vector<map<IndexType, ValueType>> row_list(
+        source_nrows, map<IndexType, ValueType>{exec}, exec);
+    for (size_type i = 0; i < source_nrows; i++) {
+        IndexType row_idx = agg.get_const_data()[i];
+        for (auto j = source_row_ptrs[i]; j < source_row_ptrs[i + 1]; j++) {
+            const auto col = agg.get_const_data()[source_col_idxs[j]];
+            const auto val = source_vals[j];
+            row_list[row_idx][col] += val;
+        }
+    }
+    auto coarse_row_ptrs = coarse->get_row_ptrs();
+    for (size_type i = 0; i < coarse_nrows; i++) {
+        coarse_row_ptrs[i] = row_list[i].size();
+    }
+    components::prefix_sum(exec, coarse_row_ptrs, coarse_nrows + 1);
+
+    auto nnz = coarse_row_ptrs[coarse_nrows];
+    matrix::CsrBuilder<ValueType, IndexType> coarse_builder{coarse};
+    auto &coarse_col_idxs_array = coarse_builder.get_col_idx_array();
+    auto &coarse_vals_array = coarse_builder.get_value_array();
+    coarse_col_idxs_array.resize_and_reset(nnz);
+    coarse_vals_array.resize_and_reset(nnz);
+    auto coarse_col_idxs = coarse_col_idxs_array.get_data();
+    auto coarse_vals = coarse_vals_array.get_data();
+
+    for (size_type i = 0; i < coarse_nrows; i++) {
+        auto ind = coarse_row_ptrs[i];
+        for (auto pair : row_list[i]) {
+            coarse_col_idxs[ind] = pair.first;
+            coarse_vals[ind] = pair.second;
+            ind++;
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
+
+
+}  // namespace amgx_pgm
+}  // namespace reference
+}  // namespace kernels
+}  // namespace gko
diff --git a/reference/test/CMakeLists.txt b/reference/test/CMakeLists.txt
index 384ce384e08..e145507be4a 100644
--- a/reference/test/CMakeLists.txt
+++ b/reference/test/CMakeLists.txt
@@ -5,6 +5,7 @@ add_subdirectory(components)
 add_subdirectory(factorization)
 add_subdirectory(log)
 add_subdirectory(matrix)
+add_subdirectory(multigrid)
 add_subdirectory(preconditioner)
 add_subdirectory(reorder)
 add_subdirectory(solver)
diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp
index 186aa700598..654a96acd61 100644
--- a/reference/test/matrix/csr_kernels.cpp
+++ b/reference/test/matrix/csr_kernels.cpp
@@ -198,6 +198,7 @@ class Csr : public ::testing::Test {
         cols_u[6] = 0;
     }
 
+
     void assert_equal_to_mtx(const Coo *m)
     {
         auto v = m->get_const_values();
diff --git a/reference/test/multigrid/CMakeLists.txt b/reference/test/multigrid/CMakeLists.txt
new file mode 100644
index 00000000000..8fe8bbeba48
--- /dev/null
+++ b/reference/test/multigrid/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_test(amgx_pgm_kernels)
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
new file mode 100644
index 00000000000..3398a954f4f
--- /dev/null
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -0,0 +1,413 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+
+
+#include <memory>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm_reduction.hpp>
+#include <ginkgo/core/stop/time.hpp>
+
+
+#include "core/multigrid/amgx_pgm_kernels.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+template <typename ValueIndexType>
+class AmgxPgm : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using Mtx = gko::matrix::Csr<value_type, index_type>;
+    using Vec = gko::matrix::Dense<value_type>;
+    using Multigrid = gko::multigrid::AmgxPgm<value_type, index_type>;
+    using T = value_type;
+    AmgxPgm()
+        : exec(gko::ReferenceExecutor::create()),
+          AmgxPgm_factory(Multigrid::build()
+                              .with_max_iterations(2u)
+                              .with_max_unassigned_percentage(0.1)
+                              .on(exec)),
+          fine_b(gko::initialize<Vec>(
+              {I<T>({2.0, -1.0}), I<T>({-1.0, 2.0}), I<T>({0.0, -1.0}),
+               I<T>({3.0, -2.0}), I<T>({-2.0, 1.0})},
+              exec)),
+          coarse_b(gko::initialize<Vec>({I<T>({2.0, -1.0}), I<T>({0.0, -1.0})},
+                                        exec)),
+          restrict_ans((gko::initialize<Vec>(
+              {I<T>({0.0, -1.0}), I<T>({2.0, 0.0})}, exec))),
+          prolongate_ans(gko::initialize<Vec>(
+              {I<T>({0.0, -2.0}), I<T>({1.0, -2.0}), I<T>({1.0, -2.0}),
+               I<T>({0.0, -1.0}), I<T>({2.0, 1.0})},
+              exec)),
+          fine_x(gko::initialize<Vec>(
+              {I<T>({-2.0, -1.0}), I<T>({1.0, -1.0}), I<T>({-1.0, -1.0}),
+               I<T>({0.0, 0.0}), I<T>({0.0, 2.0})},
+              exec)),
+          mtx(Mtx::create(exec, gko::dim<2>(5, 5), 15,
+                          std::make_shared<typename Mtx::classical>())),
+          mtx_diag(exec, 5),
+          agg(exec, 5)
+    {
+        this->create_mtx(mtx.get(), &mtx_diag, &agg);
+    }
+
+    void create_mtx(Mtx *fine, gko::Array<value_type> *diag,
+                    gko::Array<index_type> *agg)
+    {
+        auto vals = fine->get_values();
+        auto cols = fine->get_col_idxs();
+        auto rows = fine->get_row_ptrs();
+        auto diag_val = diag->get_data();
+        auto agg_val = agg->get_data();
+        /* this matrix is stored:
+         *  5 -3 -3  0  0
+         * -3  5  0 -2 -1
+         * -3  0  5  0 -1
+         *  0 -3  0  5  0
+         *  0 -2 -2  0  5
+         */
+        vals[0] = 5;
+        vals[1] = -3;
+        vals[2] = -3;
+        vals[3] = -3;
+        vals[4] = 5;
+        vals[5] = -2;
+        vals[6] = -1;
+        vals[7] = -3;
+        vals[8] = 5;
+        vals[9] = -1;
+        vals[10] = -3;
+        vals[11] = 5;
+        vals[12] = -2;
+        vals[13] = -2;
+        vals[14] = 5;
+
+        rows[0] = 0;
+        rows[1] = 3;
+        rows[2] = 7;
+        rows[3] = 10;
+        rows[4] = 12;
+        rows[5] = 15;
+
+        cols[0] = 0;
+        cols[1] = 1;
+        cols[2] = 2;
+        cols[3] = 0;
+        cols[4] = 1;
+        cols[5] = 3;
+        cols[6] = 4;
+        cols[7] = 0;
+        cols[8] = 2;
+        cols[9] = 4;
+        cols[10] = 1;
+        cols[11] = 3;
+        cols[12] = 1;
+        cols[13] = 2;
+        cols[14] = 4;
+
+        diag_val[0] = 5;
+        diag_val[1] = 5;
+        diag_val[2] = 5;
+        diag_val[3] = 5;
+        diag_val[4] = 5;
+
+        agg_val[0] = 0;
+        agg_val[1] = 1;
+        agg_val[2] = 0;
+        agg_val[3] = 1;
+        agg_val[4] = 0;
+    }
+
+    std::shared_ptr<const gko::ReferenceExecutor> exec;
+    std::shared_ptr<Mtx> mtx;
+    gko::Array<value_type> mtx_diag;
+    gko::Array<index_type> agg;
+    std::shared_ptr<Vec> coarse_b;
+    std::shared_ptr<Vec> fine_b;
+    std::shared_ptr<Vec> restrict_ans;
+    std::shared_ptr<Vec> prolongate_ans;
+    std::shared_ptr<Vec> fine_x;
+    std::unique_ptr<typename Multigrid::Factory> AmgxPgm_factory;
+};
+
+TYPED_TEST_CASE(AmgxPgm, gko::test::ValueIndexTypes);
+
+TYPED_TEST(AmgxPgm, RestrictApply)
+{
+    // fine->coarse
+    using Vec = typename TestFixture::Vec;
+    using value_type = typename TestFixture::value_type;
+    auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
+    gko::kernels::reference::amgx_pgm::restrict_apply(
+        this->exec, this->agg, this->fine_b.get(), x.get());
+    GKO_ASSERT_MTX_NEAR(x, this->restrict_ans,
+                        gko::remove_complex<value_type>{0});
+}
+
+TYPED_TEST(AmgxPgm, ProlongateApplyadd)
+{
+    using value_type = typename TestFixture::value_type;
+    auto x = gko::clone(this->fine_x);
+    gko::kernels::reference::amgx_pgm::prolongate_applyadd(
+        this->exec, this->agg, this->coarse_b.get(), x.get());
+    GKO_ASSERT_MTX_NEAR(x, this->prolongate_ans,
+                        gko::remove_complex<value_type>{0});
+}
+
+TYPED_TEST(AmgxPgm, MatchEdge)
+{
+    using index_type = typename TestFixture::index_type;
+    gko::Array<index_type> agg(this->exec, 5);
+    gko::Array<index_type> snb(this->exec, 5);
+    auto agg_val = agg.get_data();
+    auto snb_val = snb.get_data();
+    for (int i = 0; i < 5; i++) {
+        agg_val[i] = -1;
+    }
+    snb_val[0] = 2;
+    snb_val[1] = 0;
+    snb_val[2] = 0;
+    snb_val[3] = 1;
+    snb_val[4] = 2;
+
+    gko::kernels::reference::amgx_pgm::match_edge(this->exec, snb, agg);
+
+    ASSERT_EQ(agg_val[0], 0);
+    ASSERT_EQ(agg_val[1], -1);
+    ASSERT_EQ(agg_val[2], 0);
+    ASSERT_EQ(agg_val[3], -1);
+    ASSERT_EQ(agg_val[4], -1);
+}
+
+TYPED_TEST(AmgxPgm, CountUnagg)
+{
+    using index_type = typename TestFixture::index_type;
+    gko::Array<index_type> agg(this->exec, 5);
+    auto agg_val = agg.get_data();
+    gko::size_type num_unagg = 0;
+    agg_val[0] = 0;
+    agg_val[1] = -1;
+    agg_val[2] = 0;
+    agg_val[3] = -1;
+    agg_val[4] = -1;
+
+    gko::kernels::reference::amgx_pgm::count_unagg(this->exec, agg, &num_unagg);
+
+    ASSERT_EQ(num_unagg, 3);
+}
+
+
+TYPED_TEST(AmgxPgm, Renumber)
+{
+    using index_type = typename TestFixture::index_type;
+    gko::Array<index_type> agg(this->exec, 5);
+    auto agg_val = agg.get_data();
+    gko::size_type num_agg = 0;
+    agg_val[0] = 0;
+    agg_val[1] = 1;
+    agg_val[2] = 0;
+    agg_val[3] = 1;
+    agg_val[4] = 4;
+
+    gko::kernels::reference::amgx_pgm::renumber(this->exec, agg, &num_agg);
+
+    ASSERT_EQ(num_agg, 3);
+    ASSERT_EQ(agg_val[0], 0);
+    ASSERT_EQ(agg_val[1], 1);
+    ASSERT_EQ(agg_val[2], 0);
+    ASSERT_EQ(agg_val[3], 1);
+    ASSERT_EQ(agg_val[4], 2);
+}
+
+
+TYPED_TEST(AmgxPgm, Generate)
+{
+    auto coarse_fine = this->AmgxPgm_factory->generate(this->mtx);
+
+    auto agg_result = coarse_fine->get_const_agg();
+
+    ASSERT_EQ(agg_result[0], 0);
+    ASSERT_EQ(agg_result[1], 1);
+    ASSERT_EQ(agg_result[2], 0);
+    ASSERT_EQ(agg_result[3], 1);
+    ASSERT_EQ(agg_result[4], 0);
+}
+
+
+TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
+{
+    std::unique_ptr<gko::CoarseFine> amgx_pgm{
+        this->AmgxPgm_factory->generate(this->mtx)};
+
+    // fine->coarse
+    using Vec = typename TestFixture::Vec;
+    using value_type = typename TestFixture::value_type;
+    auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
+    amgx_pgm->restrict_apply(this->fine_b.get(), x.get());
+    GKO_ASSERT_MTX_NEAR(x, this->restrict_ans,
+                        gko::remove_complex<value_type>{0});
+}
+
+
+TYPED_TEST(AmgxPgm, CoarseFineProlongateApplyadd)
+{
+    std::unique_ptr<gko::CoarseFine> amgx_pgm{
+        this->AmgxPgm_factory->generate(this->mtx)};
+
+    using value_type = typename TestFixture::value_type;
+    auto x = gko::clone(this->fine_x);
+    amgx_pgm->prolongate_applyadd(this->coarse_b.get(), x.get());
+    GKO_ASSERT_MTX_NEAR(x, this->prolongate_ans,
+                        gko::remove_complex<value_type>{0});
+}
+
+
+TYPED_TEST(AmgxPgm, ExtractDiag)
+{
+    using value_type = typename TestFixture::value_type;
+
+    gko::Array<value_type> diag(this->exec, 5);
+    gko::kernels::reference::amgx_pgm::extract_diag(this->exec, this->mtx.get(),
+                                                    diag);
+    GKO_ASSERT_ARRAY_EQ(diag, this->mtx_diag);
+}
+
+
+TYPED_TEST(AmgxPgm, FindStrongestNeighbor)
+{
+    using index_type = typename TestFixture::index_type;
+    gko::Array<index_type> strongest_neighbor(this->exec, 5);
+    gko::Array<index_type> agg(this->exec, 5);
+    auto snb_vals = strongest_neighbor.get_data();
+    auto agg_vals = agg.get_data();
+    for (int i = 0; i < 5; i++) {
+        snb_vals[i] = -1;
+        agg_vals[i] = -1;
+    }
+    gko::kernels::reference::amgx_pgm::find_strongest_neighbor(
+        this->exec, this->mtx.get(), this->mtx_diag, agg, strongest_neighbor);
+    ASSERT_EQ(snb_vals[0], 2);
+    ASSERT_EQ(snb_vals[1], 0);
+    ASSERT_EQ(snb_vals[2], 0);
+    ASSERT_EQ(snb_vals[3], 1);
+    ASSERT_EQ(snb_vals[4], 2);
+}
+
+
+TYPED_TEST(AmgxPgm, AssignToExistAgg)
+{
+    using index_type = typename TestFixture::index_type;
+    gko::Array<index_type> agg(this->exec, 5);
+    gko::Array<index_type> intermediate_agg(this->exec, 0);
+    auto agg_vals = agg.get_data();
+    // 0 - 2, 1 - 3
+    agg_vals[0] = 0;
+    agg_vals[1] = 1;
+    agg_vals[2] = 0;
+    agg_vals[3] = 1;
+    agg_vals[4] = -1;
+    gko::kernels::reference::amgx_pgm::assign_to_exist_agg(
+        this->exec, this->mtx.get(), this->mtx_diag, agg, intermediate_agg);
+    ASSERT_EQ(agg_vals[0], 0);
+    ASSERT_EQ(agg_vals[1], 1);
+    ASSERT_EQ(agg_vals[2], 0);
+    ASSERT_EQ(agg_vals[3], 1);
+    ASSERT_EQ(agg_vals[4], 0);
+}
+
+
+TYPED_TEST(AmgxPgm, GenerateMtx)
+{
+    using index_type = typename TestFixture::index_type;
+    using value_type = typename TestFixture::value_type;
+    using mtx_type = typename TestFixture::Mtx;
+    gko::Array<index_type> agg(this->exec, 5);
+    auto agg_vals = agg.get_data();
+    // 0 - 2, 1 - 3, 4
+    agg_vals[0] = 0;
+    agg_vals[1] = 1;
+    agg_vals[2] = 0;
+    agg_vals[3] = 1;
+    agg_vals[4] = 2;
+    auto csr_coarse = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
+    gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
+        this->exec, this->mtx.get(), agg, csr_coarse.get());
+
+    ASSERT_EQ(csr_coarse->get_size(), gko::dim<2>(3, 3));
+    ASSERT_EQ(csr_coarse->get_num_stored_elements(), 9);
+    auto r = csr_coarse->get_const_row_ptrs();
+    auto c = csr_coarse->get_const_col_idxs();
+    auto v = csr_coarse->get_const_values();
+    ASSERT_EQ(r[0], 0);
+    ASSERT_EQ(r[1], 3);
+    ASSERT_EQ(r[2], 6);
+    ASSERT_EQ(r[3], 9);
+    ASSERT_EQ(c[0], 0);
+    ASSERT_EQ(c[1], 1);
+    ASSERT_EQ(c[2], 2);
+    ASSERT_EQ(c[3], 0);
+    ASSERT_EQ(c[4], 1);
+    ASSERT_EQ(c[5], 2);
+    ASSERT_EQ(c[6], 0);
+    ASSERT_EQ(c[7], 1);
+    ASSERT_EQ(c[8], 2);
+    ASSERT_EQ(v[0], value_type{4});
+    ASSERT_EQ(v[1], value_type{-3});
+    ASSERT_EQ(v[2], value_type{-1});
+    ASSERT_EQ(v[3], value_type{-3});
+    ASSERT_EQ(v[4], value_type{5});
+    ASSERT_EQ(v[5], value_type{-1});
+    ASSERT_EQ(v[6], value_type{-2});
+    ASSERT_EQ(v[7], value_type{-2});
+    ASSERT_EQ(v[8], value_type{5});
+}
+
+
+}  // namespace

From ada5d883c295400d5a67550ff921eb99dabc1a57 Mon Sep 17 00:00:00 2001
From: yhmtsai <yhmtsai@gmail.com>
Date: Wed, 13 May 2020 15:21:39 +0200
Subject: [PATCH 0462/1008] rename CoarseFine to RestrictProlong

---
 core/multigrid/amgx_pgm.cpp                   |  12 +-
 core/multigrid/amgx_pgm_kernels.hpp           |   8 +-
 cuda/multigrid/amgx_pgm_kernels.cu            |   8 +-
 hip/multigrid/amgx_pgm_kernels.hip.cpp        |   8 +-
 include/ginkgo/core/matrix/csr.hpp            |   1 -
 include/ginkgo/core/multigrid/amgx_pgm.hpp    |  18 +-
 .../restrict_prolong.hpp}                     | 196 ++++++++++--------
 omp/multigrid/amgx_pgm_kernels.cpp            |   8 +-
 reference/multigrid/amgx_pgm_kernels.cpp      |  10 +-
 reference/test/multigrid/amgx_pgm_kernels.cpp |  20 +-
 10 files changed, 155 insertions(+), 134 deletions(-)
 rename include/ginkgo/core/{base/coarse_fine.hpp => multigrid/restrict_prolong.hpp} (60%)

diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index d422503ca19..db02285e90c 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -53,7 +53,7 @@ namespace amgx_pgm {
 
 
 GKO_REGISTER_OPERATION(restrict_apply, amgx_pgm::restrict_apply);
-GKO_REGISTER_OPERATION(prolongate_applyadd, amgx_pgm::prolongate_applyadd);
+GKO_REGISTER_OPERATION(prolong_applyadd, amgx_pgm::prolong_applyadd);
 GKO_REGISTER_OPERATION(initial, amgx_pgm::initial);
 GKO_REGISTER_OPERATION(match_edge, amgx_pgm::match_edge);
 GKO_REGISTER_OPERATION(count_unagg, amgx_pgm::count_unagg);
@@ -154,13 +154,13 @@ void AmgxPgm<ValueType, IndexType>::restrict_apply_impl(const LinOp *b,
 
 
 template <typename ValueType, typename IndexType>
-void AmgxPgm<ValueType, IndexType>::prolongate_applyadd_impl(const LinOp *b,
-                                                             LinOp *x) const
+void AmgxPgm<ValueType, IndexType>::prolong_applyadd_impl(const LinOp *b,
+                                                          LinOp *x) const
 {
     auto exec = this->get_executor();
-    exec->run(amgx_pgm::make_prolongate_applyadd(
-        agg_, as<matrix::Dense<ValueType>>(b),
-        as<matrix::Dense<ValueType>>(x)));
+    exec->run(amgx_pgm::make_prolong_applyadd(agg_,
+                                              as<matrix::Dense<ValueType>>(b),
+                                              as<matrix::Dense<ValueType>>(x)));
 }
 
 
diff --git a/core/multigrid/amgx_pgm_kernels.hpp b/core/multigrid/amgx_pgm_kernels.hpp
index 2edefd61ea2..68e6f57f857 100644
--- a/core/multigrid/amgx_pgm_kernels.hpp
+++ b/core/multigrid/amgx_pgm_kernels.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_SOLVER_AMGX_PGM_KERNELS_HPP_
-#define GKO_CORE_SOLVER_AMGX_PGM_KERNELS_HPP_
+#ifndef GKO_CORE_MULTIGRID_AMGX_PGM_KERNELS_HPP_
+#define GKO_CORE_MULTIGRID_AMGX_PGM_KERNELS_HPP_
 
 
 #include <ginkgo/core/multigrid/amgx_pgm.hpp>
@@ -57,7 +57,7 @@ namespace amgx_pgm {
 
 
 #define GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL(_vtype, _itype)           \
-    void prolongate_applyadd(                                                  \
+    void prolong_applyadd(                                                     \
         std::shared_ptr<const DefaultExecutor> exec, const Array<_itype> &agg, \
         const matrix::Dense<_vtype> *b, matrix::Dense<_vtype> *x)
 
@@ -177,4 +177,4 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace gko
 
 
-#endif  // GKO_CORE_SOLVER_AMGX_PGM_KERNELS_HPP_
+#endif  // GKO_CORE_MULTIGRID_AMGX_PGM_KERNELS_HPP_
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index c00d227331e..310883ecae3 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -73,10 +73,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void prolongate_applyadd(std::shared_ptr<const CudaExecutor> exec,
-                         const Array<IndexType> &agg,
-                         const matrix::Dense<ValueType> *b,
-                         matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+void prolong_applyadd(std::shared_ptr<const CudaExecutor> exec,
+                      const Array<IndexType> &agg,
+                      const matrix::Dense<ValueType> *b,
+                      matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index d4583be8dce..0ff2784258b 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -73,10 +73,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void prolongate_applyadd(std::shared_ptr<const HipExecutor> exec,
-                         const Array<IndexType> &agg,
-                         const matrix::Dense<ValueType> *b,
-                         matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+void prolong_applyadd(std::shared_ptr<const HipExecutor> exec,
+                      const Array<IndexType> &agg,
+                      const matrix::Dense<ValueType> *b,
+                      matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index 72223a37db1..7e01c1293f6 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -35,7 +35,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/coarse_fine.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/base/math.hpp>
 
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index 5e90b1a6db1..d3da801110f 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -37,11 +37,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <vector>
 
 
-#include <ginkgo/core/base/coarse_fine.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/multigrid/restrict_prolong.hpp>
 
 
 namespace gko {
@@ -59,13 +59,13 @@ namespace multigrid {
  * @tparam ValueType  precision of matrix elements
  * @tparam IndexType  precision of matrix indexes
  *
- * @ingroup CoarseFine
+ * @ingroup RestrictProlong
  * @ingroup LinOp
  */
 template <typename ValueType = default_precision, typename IndexType = int32>
-class AmgxPgm : public EnableCFOp<AmgxPgm<ValueType, IndexType>> {
-    friend class EnableCFOp<AmgxPgm>;
-    friend class EnablePolymorphicObject<AmgxPgm, CoarseFine>;
+class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
+    friend class EnableRestrictProlong<AmgxPgm>;
+    friend class EnablePolymorphicObject<AmgxPgm, RestrictProlong>;
 
 public:
     using value_type = ValueType;
@@ -117,19 +117,19 @@ class AmgxPgm : public EnableCFOp<AmgxPgm<ValueType, IndexType>> {
          */
         bool GKO_FACTORY_PARAMETER(deterministic, false);
     };
-    GKO_ENABLE_CFOP_FACTORY(AmgxPgm, parameters, Factory);
+    GKO_ENABLE_RESTRICT_PROLONG_FACTORY(AmgxPgm, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);
 
 protected:
     void restrict_apply_impl(const LinOp *b, LinOp *x) const override;
 
-    void prolongate_applyadd_impl(const LinOp *b, LinOp *x) const override;
+    void prolong_applyadd_impl(const LinOp *b, LinOp *x) const override;
     explicit AmgxPgm(std::shared_ptr<const Executor> exec)
-        : EnableCFOp<AmgxPgm>(std::move(exec))
+        : EnableRestrictProlong<AmgxPgm>(std::move(exec))
     {}
     explicit AmgxPgm(const Factory *factory,
                      std::shared_ptr<const LinOp> system_matrix)
-        : EnableCFOp<AmgxPgm>(factory->get_executor()),
+        : EnableRestrictProlong<AmgxPgm>(factory->get_executor()),
           parameters_{factory->get_parameters()},
           system_matrix_{std::move(system_matrix)},
           agg_(factory->get_executor(), system_matrix_->get_size()[0])
diff --git a/include/ginkgo/core/base/coarse_fine.hpp b/include/ginkgo/core/multigrid/restrict_prolong.hpp
similarity index 60%
rename from include/ginkgo/core/base/coarse_fine.hpp
rename to include/ginkgo/core/multigrid/restrict_prolong.hpp
index 1388cbe676d..5a7537bddb1 100644
--- a/include/ginkgo/core/base/coarse_fine.hpp
+++ b/include/ginkgo/core/multigrid/restrict_prolong.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_COARSE_FINE_HPP_
-#define GKO_CORE_BASE_COARSE_FINE_HPP_
+#ifndef GKO_CORE_MULTIGRID_RESTRICT_PROLONG_HPP_
+#define GKO_CORE_MULTIGRID_RESTRICT_PROLONG_HPP_
 
 
 #include <functional>
@@ -41,20 +41,23 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
+
 namespace gko {
+namespace multigrid {
 
 
 /**
- * The CoarseFine class can be used to construct restrict_apply and
- * prolongate_applyadd. R is the restrict operator (fine level -> coarse
- * level) and P is prolongate operator (coarse level -> fine level).
+ * The RestrictProlong class can be used to construct restrict_apply and
+ * prolong_applyadd. R is the restrict operator (fine level -> coarse
+ * level) and P is prolong operator (coarse level -> fine level).
  * restrict_apply(b, x) -> x = R(b)
- * prolongate_applyadd(b, x) -> x = P(b) + x
+ * prolong_applyadd(b, x) -> x = P(b) + x
  *
- * @ingroup CoarseFine
+ * @ingroup RestrictProlong
  */
 
-class CoarseFine : public EnableAbstractPolymorphicObject<CoarseFine> {
+class RestrictProlong
+    : public EnableAbstractPolymorphicObject<RestrictProlong> {
 public:
     /**
      * Returns the coarse operator (matrix) which is R * matrix * P
@@ -73,7 +76,7 @@ class CoarseFine : public EnableAbstractPolymorphicObject<CoarseFine> {
      *
      * @return this
      */
-    CoarseFine *restrict_apply(const LinOp *b, LinOp *x)
+    RestrictProlong *restrict_apply(const LinOp *b, LinOp *x)
     {
         this->validate_restrict_parameters(b, x);
         auto exec = this->get_executor();
@@ -85,7 +88,7 @@ class CoarseFine : public EnableAbstractPolymorphicObject<CoarseFine> {
     /**
      * @copydoc restrict_apply(const LinOp *, LinOp *)
      */
-    const CoarseFine *restrict_apply(const LinOp *b, LinOp *x) const
+    const RestrictProlong *restrict_apply(const LinOp *b, LinOp *x) const
     {
         this->validate_restrict_parameters(b, x);
         auto exec = this->get_executor();
@@ -95,52 +98,69 @@ class CoarseFine : public EnableAbstractPolymorphicObject<CoarseFine> {
     }
 
     /**
-     * Applies a prolongate operator to a vector (or a sequence of vectors).
+     * Applies a prolong operator to a vector (or a sequence of vectors).
      *
-     * Performs the operation x = prolongate(b) + x.
+     * Performs the operation x = prolong(b) + x.
      *
      * @param b  the input vector(s) on which the operator is applied
      * @param x  the output vector(s) where the result is stored
      *
      * @return this
      */
-    CoarseFine *prolongate_applyadd(const LinOp *b, LinOp *x)
+    RestrictProlong *prolong_applyadd(const LinOp *b, LinOp *x)
     {
-        this->validate_prolongate_parameters(b, x);
+        this->validate_prolong_parameters(b, x);
         auto exec = this->get_executor();
-        this->prolongate_applyadd_impl(make_temporary_clone(exec, b).get(),
-                                       make_temporary_clone(exec, x).get());
+        this->prolong_applyadd_impl(make_temporary_clone(exec, b).get(),
+                                    make_temporary_clone(exec, x).get());
         return this;
     }
 
     /**
-     * @copydoc prolongate_applyadd(const LinOp *, LinOp *)
+     * @copydoc prolong_applyadd(const LinOp *, LinOp *)
      */
-    const CoarseFine *prolongate_applyadd(const LinOp *b, LinOp *x) const
+    const RestrictProlong *prolong_applyadd(const LinOp *b, LinOp *x) const
     {
-        this->validate_prolongate_parameters(b, x);
+        this->validate_prolong_parameters(b, x);
         auto exec = this->get_executor();
-        this->prolongate_applyadd_impl(make_temporary_clone(exec, b).get(),
-                                       make_temporary_clone(exec, x).get());
+        this->prolong_applyadd_impl(make_temporary_clone(exec, b).get(),
+                                    make_temporary_clone(exec, x).get());
         return this;
     }
 
 protected:
+    /**
+     * Throws a DimensionMismatch exception if the parameters to
+     * `restrict_apply` are of the wrong size.
+     *
+     * @param b  vector(s) on which the operator is applied
+     * @param x  output vector(s)
+     */
     void validate_restrict_parameters(const LinOp *b, LinOp *x) const
     {
-        GKO_ASSERT_EQ(fine_dim_, b->get_size()[0]);
-        GKO_ASSERT_EQ(coarse_dim_, x->get_size()[0]);
+        auto restirct_dim = gko::dim<2>(coarse_dim_, fine_dim_);
+        GKO_ASSERT_CONFORMANT(restirct_dim, b);
+        GKO_ASSERT_EQUAL_ROWS(restirct_dim, x);
         GKO_ASSERT_EQUAL_COLS(b, x);
     }
-    void validate_prolongate_parameters(const LinOp *b, LinOp *x) const
+
+    /**
+     * Throws a DimensionMismatch exception if the parameters to
+     * `prolong_applyadd` are of the wrong size.
+     *
+     * @param b  vector(s) on which the operator is applied
+     * @param x  output vector(s)
+     */
+    void validate_prolong_parameters(const LinOp *b, LinOp *x) const
     {
-        GKO_ASSERT_EQ(coarse_dim_, b->get_size()[0]);
-        GKO_ASSERT_EQ(fine_dim_, x->get_size()[0]);
+        auto prolong_dim = gko::dim<2>(fine_dim_, coarse_dim_);
+        GKO_ASSERT_CONFORMANT(prolong_dim, b);
+        GKO_ASSERT_EQUAL_ROWS(prolong_dim, x);
         GKO_ASSERT_EQUAL_COLS(b, x);
     }
 
     /**
-     * Implementers of CFOp should override this function instead
+     * Implementers of RestrictProlong should override this function instead
      * of restrict_apply(const LinOp *, LinOp *).
      *
      * Performs the operation x = restrict(b), where op is the restrict
@@ -152,19 +172,19 @@ class CoarseFine : public EnableAbstractPolymorphicObject<CoarseFine> {
     virtual void restrict_apply_impl(const LinOp *b, LinOp *x) const = 0;
 
     /**
-     * Implementers of CFOp should override this function instead
-     * of prolongate_apply(const LinOp *, LinOp *).
+     * Implementers of RestrictProlong should override this function instead
+     * of prolong_apply(const LinOp *, LinOp *).
      *
-     * Performs the operation x = prolongate(b), where op is the prolongate
+     * Performs the operation x = prolong(b), where op is the prolong
      * operator.
      *
      * @param b  the input vector(s) on which the operator is applied
      * @param x  the output vector(s) where the result is stored
      */
-    virtual void prolongate_applyadd_impl(const LinOp *b, LinOp *x) const = 0;
+    virtual void prolong_applyadd_impl(const LinOp *b, LinOp *x) const = 0;
 
     /**
-     * Sets the components of CoarseFine
+     * Sets the components of RestrictProlong
      *
      * @param coarse  the coarse matrix
      * @param fine_dim  the fine_level size
@@ -182,8 +202,8 @@ class CoarseFine : public EnableAbstractPolymorphicObject<CoarseFine> {
      *
      * @param exec  the executor where all the operations are performed
      */
-    explicit CoarseFine(std::shared_ptr<const Executor> exec)
-        : EnableAbstractPolymorphicObject<CoarseFine>(exec)
+    explicit RestrictProlong(std::shared_ptr<const Executor> exec)
+        : EnableAbstractPolymorphicObject<RestrictProlong>(exec)
     {}
 
 private:
@@ -192,13 +212,13 @@ class CoarseFine : public EnableAbstractPolymorphicObject<CoarseFine> {
     size_type coarse_dim_;
 };
 
-class CFFactory
-    : public AbstractFactory<CoarseFine, std::shared_ptr<const LinOp>> {
+class RestrictProlongFactory
+    : public AbstractFactory<RestrictProlong, std::shared_ptr<const LinOp>> {
 public:
-    using AbstractFactory<CoarseFine,
+    using AbstractFactory<RestrictProlong,
                           std::shared_ptr<const LinOp>>::AbstractFactory;
 
-    std::unique_ptr<CoarseFine> generate(
+    std::unique_ptr<RestrictProlong> generate(
         std::shared_ptr<const LinOp> input) const
     {
         auto generated = AbstractFactory::generate(input);
@@ -207,14 +227,15 @@ class CFFactory
 };
 
 template <typename ConcreteFactory, typename ConcreteCriterion,
-          typename ParametersType, typename PolymorphicBase = CFFactory>
-using EnableDefaultCFFactory =
+          typename ParametersType,
+          typename PolymorphicBase = RestrictProlongFactory>
+using EnableDefaultRestrictProlongFactory =
     EnableDefaultFactory<ConcreteFactory, ConcreteCriterion, ParametersType,
                          PolymorphicBase>;
 
 
-template <typename ConcreteLinOp, typename PolymorphicBase = CoarseFine>
-class EnableCFOp
+template <typename ConcreteLinOp, typename PolymorphicBase = RestrictProlong>
+class EnableRestrictProlong
     : public EnablePolymorphicObject<ConcreteLinOp, PolymorphicBase>,
       public EnablePolymorphicAssignment<ConcreteLinOp> {
 public:
@@ -240,21 +261,21 @@ class EnableCFOp
     }
 
 
-    ConcreteLinOp *prolongate_applyadd(const LinOp *b, LinOp *x)
+    ConcreteLinOp *prolong_applyadd(const LinOp *b, LinOp *x)
     {
-        this->validate_prolongate_parameters(b, x);
+        this->validate_prolong_parameters(b, x);
         auto exec = this->get_executor();
-        this->prolongate_applyadd_impl(make_temporary_clone(exec, b).get(),
-                                       make_temporary_clone(exec, x).get());
+        this->prolong_applyadd_impl(make_temporary_clone(exec, b).get(),
+                                    make_temporary_clone(exec, x).get());
         return self();
     }
 
-    const ConcreteLinOp *prolongate_applyadd(const LinOp *b, LinOp *x) const
+    const ConcreteLinOp *prolong_applyadd(const LinOp *b, LinOp *x) const
     {
-        this->validate_prolongate_parameters(b, x);
+        this->validate_prolong_parameters(b, x);
         auto exec = this->get_executor();
-        this->prolongate_applyadd_impl(make_temporary_clone(exec, b).get(),
-                                       make_temporary_clone(exec, x).get());
+        this->prolong_applyadd_impl(make_temporary_clone(exec, b).get(),
+                                    make_temporary_clone(exec, x).get());
         return self();
     }
 
@@ -263,45 +284,46 @@ class EnableCFOp
 };
 
 
-#define GKO_ENABLE_CFOP_FACTORY(_cfop, _parameters_name, _factory_name)      \
-public:                                                                      \
-    const _parameters_name##_type &get_##_parameters_name() const            \
-    {                                                                        \
-        return _parameters_name##_;                                          \
-    }                                                                        \
-                                                                             \
-    class _factory_name                                                      \
-        : public ::gko::EnableDefaultCFFactory<_factory_name, _cfop,         \
-                                               _parameters_name##_type> {    \
-        friend class ::gko::EnablePolymorphicObject<_factory_name,           \
-                                                    ::gko::CFFactory>;       \
-        friend class ::gko::enable_parameters_type<_parameters_name##_type,  \
-                                                   _factory_name>;           \
-        explicit _factory_name(std::shared_ptr<const ::gko::Executor> exec)  \
-            : ::gko::EnableDefaultCFFactory<_factory_name, _cfop,            \
-                                            _parameters_name##_type>(        \
-                  std::move(exec))                                           \
-        {}                                                                   \
-        explicit _factory_name(std::shared_ptr<const ::gko::Executor> exec,  \
-                               const _parameters_name##_type &parameters)    \
-            : ::gko::EnableDefaultCFFactory<_factory_name, _cfop,            \
-                                            _parameters_name##_type>(        \
-                  std::move(exec), parameters)                               \
-        {}                                                                   \
-    };                                                                       \
-    friend ::gko::EnableDefaultCFFactory<_factory_name, _cfop,               \
-                                         _parameters_name##_type>;           \
-                                                                             \
-private:                                                                     \
-    _parameters_name##_type _parameters_name##_;                             \
-                                                                             \
-public:                                                                      \
-    static_assert(true,                                                      \
-                  "This assert is used to counter the false positive extra " \
+#define GKO_ENABLE_RESTRICT_PROLONG_FACTORY(_restrict_prolong,                \
+                                            _parameters_name, _factory_name)  \
+public:                                                                       \
+    const _parameters_name##_type &get_##_parameters_name() const             \
+    {                                                                         \
+        return _parameters_name##_;                                           \
+    }                                                                         \
+                                                                              \
+    class _factory_name                                                       \
+        : public ::gko::multigrid::EnableDefaultRestrictProlongFactory<       \
+              _factory_name, _restrict_prolong, _parameters_name##_type> {    \
+        friend class ::gko::EnablePolymorphicObject<                          \
+            _factory_name, ::gko::multigrid::RestrictProlongFactory>;         \
+        friend class ::gko::enable_parameters_type<_parameters_name##_type,   \
+                                                   _factory_name>;            \
+        explicit _factory_name(std::shared_ptr<const ::gko::Executor> exec)   \
+            : ::gko::multigrid::EnableDefaultRestrictProlongFactory<          \
+                  _factory_name, _restrict_prolong, _parameters_name##_type>( \
+                  std::move(exec))                                            \
+        {}                                                                    \
+        explicit _factory_name(std::shared_ptr<const ::gko::Executor> exec,   \
+                               const _parameters_name##_type &parameters)     \
+            : ::gko::multigrid::EnableDefaultRestrictProlongFactory<          \
+                  _factory_name, _restrict_prolong, _parameters_name##_type>( \
+                  std::move(exec), parameters)                                \
+        {}                                                                    \
+    };                                                                        \
+    friend ::gko::multigrid::EnableDefaultRestrictProlongFactory<             \
+        _factory_name, _restrict_prolong, _parameters_name##_type>;           \
+                                                                              \
+private:                                                                      \
+    _parameters_name##_type _parameters_name##_;                              \
+                                                                              \
+public:                                                                       \
+    static_assert(true,                                                       \
+                  "This assert is used to counter the false positive extra "  \
                   "semi-colon warnings")
 
-
+}  // namespace multigrid
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_COARSE_FINE_HPP_
+#endif  // GKO_CORE_MULTIGRID_RESTRICT_PROLONG_HPP_
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index 445411dea24..982260d193b 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -70,10 +70,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void prolongate_applyadd(std::shared_ptr<const OmpExecutor> exec,
-                         const Array<IndexType> &agg,
-                         const matrix::Dense<ValueType> *b,
-                         matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+void prolong_applyadd(std::shared_ptr<const OmpExecutor> exec,
+                      const Array<IndexType> &agg,
+                      const matrix::Dense<ValueType> *b,
+                      matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index 6ed7e11b2ef..053cb46ec45 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -42,10 +42,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/multigrid/amgx_pgm.hpp>
-#include "core/components/prefix_sum.hpp"
 
 
 #include "core/base/allocator.hpp"
+#include "core/components/prefix_sum.hpp"
 #include "core/matrix/csr_builder.hpp"
 
 
@@ -87,10 +87,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void prolongate_applyadd(std::shared_ptr<const ReferenceExecutor> exec,
-                         const Array<IndexType> &agg,
-                         const matrix::Dense<ValueType> *b,
-                         matrix::Dense<ValueType> *x)
+void prolong_applyadd(std::shared_ptr<const ReferenceExecutor> exec,
+                      const Array<IndexType> &agg,
+                      const matrix::Dense<ValueType> *b,
+                      matrix::Dense<ValueType> *x)
 {
     auto x_vals = x->get_values();
     const auto x_stride = x->get_stride();
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index 3398a954f4f..ff9507e2639 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -82,7 +82,7 @@ class AmgxPgm : public ::testing::Test {
                                         exec)),
           restrict_ans((gko::initialize<Vec>(
               {I<T>({0.0, -1.0}), I<T>({2.0, 0.0})}, exec))),
-          prolongate_ans(gko::initialize<Vec>(
+          prolong_ans(gko::initialize<Vec>(
               {I<T>({0.0, -2.0}), I<T>({1.0, -2.0}), I<T>({1.0, -2.0}),
                I<T>({0.0, -1.0}), I<T>({2.0, 1.0})},
               exec)),
@@ -172,7 +172,7 @@ class AmgxPgm : public ::testing::Test {
     std::shared_ptr<Vec> coarse_b;
     std::shared_ptr<Vec> fine_b;
     std::shared_ptr<Vec> restrict_ans;
-    std::shared_ptr<Vec> prolongate_ans;
+    std::shared_ptr<Vec> prolong_ans;
     std::shared_ptr<Vec> fine_x;
     std::unique_ptr<typename Multigrid::Factory> AmgxPgm_factory;
 };
@@ -191,13 +191,13 @@ TYPED_TEST(AmgxPgm, RestrictApply)
                         gko::remove_complex<value_type>{0});
 }
 
-TYPED_TEST(AmgxPgm, ProlongateApplyadd)
+TYPED_TEST(AmgxPgm, ProlongApplyadd)
 {
     using value_type = typename TestFixture::value_type;
     auto x = gko::clone(this->fine_x);
-    gko::kernels::reference::amgx_pgm::prolongate_applyadd(
+    gko::kernels::reference::amgx_pgm::prolong_applyadd(
         this->exec, this->agg, this->coarse_b.get(), x.get());
-    GKO_ASSERT_MTX_NEAR(x, this->prolongate_ans,
+    GKO_ASSERT_MTX_NEAR(x, this->prolong_ans,
                         gko::remove_complex<value_type>{0});
 }
 
@@ -283,7 +283,7 @@ TYPED_TEST(AmgxPgm, Generate)
 
 TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
 {
-    std::unique_ptr<gko::CoarseFine> amgx_pgm{
+    std::unique_ptr<gko::multigrid::RestrictProlong> amgx_pgm{
         this->AmgxPgm_factory->generate(this->mtx)};
 
     // fine->coarse
@@ -296,15 +296,15 @@ TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
 }
 
 
-TYPED_TEST(AmgxPgm, CoarseFineProlongateApplyadd)
+TYPED_TEST(AmgxPgm, CoarseFineProlongApplyadd)
 {
-    std::unique_ptr<gko::CoarseFine> amgx_pgm{
+    std::unique_ptr<gko::multigrid::RestrictProlong> amgx_pgm{
         this->AmgxPgm_factory->generate(this->mtx)};
 
     using value_type = typename TestFixture::value_type;
     auto x = gko::clone(this->fine_x);
-    amgx_pgm->prolongate_applyadd(this->coarse_b.get(), x.get());
-    GKO_ASSERT_MTX_NEAR(x, this->prolongate_ans,
+    amgx_pgm->prolong_applyadd(this->coarse_b.get(), x.get());
+    GKO_ASSERT_MTX_NEAR(x, this->prolong_ans,
                         gko::remove_complex<value_type>{0});
 }
 

From b9990b1dfd7f281bcebd99eb4394aa1f1fa2061a Mon Sep 17 00:00:00 2001
From: yhmtsai <yhmtsai@gmail.com>
Date: Fri, 29 May 2020 14:42:01 +0200
Subject: [PATCH 0463/1008] add core test

---
 core/test/multigrid/amgx_pgm.cpp              | 233 +++++++++++++++++-
 include/ginkgo/core/multigrid/amgx_pgm.hpp    |   8 +-
 reference/test/multigrid/amgx_pgm_kernels.cpp |  12 +-
 3 files changed, 244 insertions(+), 9 deletions(-)

diff --git a/core/test/multigrid/amgx_pgm.cpp b/core/test/multigrid/amgx_pgm.cpp
index 5982402402b..33e196c11aa 100644
--- a/core/test/multigrid/amgx_pgm.cpp
+++ b/core/test/multigrid/amgx_pgm.cpp
@@ -45,4 +45,235 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/test/utils.hpp"
 
 
-namespace {}  // namespace
+namespace {
+
+
+template <typename ValueIndexType>
+class AmgxPgm : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using Mtx = gko::matrix::Csr<value_type, index_type>;
+    using Vec = gko::matrix::Dense<value_type>;
+    using RestrictProlong = gko::multigrid::AmgxPgm<value_type, index_type>;
+    using T = value_type;
+    AmgxPgm()
+        : exec(gko::ReferenceExecutor::create()),
+          amgxpgm_factory(RestrictProlong::build()
+                              .with_max_iterations(2u)
+                              .with_max_unassigned_percentage(0.1)
+                              .on(exec)),
+          mtx(Mtx::create(exec, gko::dim<2>(5, 5), 15,
+                          std::make_shared<typename Mtx::classical>())),
+          agg(exec, 5),
+          coarse(Mtx::create(exec, gko::dim<2>(2, 2), 4,
+                             std::make_shared<typename Mtx::classical>()))
+
+    {
+        this->create_mtx(mtx.get(), &agg, coarse.get());
+        rstr_prlg = amgxpgm_factory->generate(mtx);
+    }
+
+    void create_mtx(Mtx *fine, gko::Array<index_type> *agg, Mtx *coarse)
+    {
+        auto f_vals = fine->get_values();
+        auto f_cols = fine->get_col_idxs();
+        auto f_rows = fine->get_row_ptrs();
+        auto c_vals = coarse->get_values();
+        auto c_cols = coarse->get_col_idxs();
+        auto c_rows = coarse->get_row_ptrs();
+        auto agg_val = agg->get_data();
+        /* this matrix is stored:
+         *  5 -3 -3  0  0
+         * -3  5  0 -2 -1
+         * -3  0  5  0 -1
+         *  0 -3  0  5  0
+         *  0 -2 -2  0  5
+         */
+        f_vals[0] = 5;
+        f_vals[1] = -3;
+        f_vals[2] = -3;
+        f_vals[3] = -3;
+        f_vals[4] = 5;
+        f_vals[5] = -2;
+        f_vals[6] = -1;
+        f_vals[7] = -3;
+        f_vals[8] = 5;
+        f_vals[9] = -1;
+        f_vals[10] = -3;
+        f_vals[11] = 5;
+        f_vals[12] = -2;
+        f_vals[13] = -2;
+        f_vals[14] = 5;
+
+        f_rows[0] = 0;
+        f_rows[1] = 3;
+        f_rows[2] = 7;
+        f_rows[3] = 10;
+        f_rows[4] = 12;
+        f_rows[5] = 15;
+
+        f_cols[0] = 0;
+        f_cols[1] = 1;
+        f_cols[2] = 2;
+        f_cols[3] = 0;
+        f_cols[4] = 1;
+        f_cols[5] = 3;
+        f_cols[6] = 4;
+        f_cols[7] = 0;
+        f_cols[8] = 2;
+        f_cols[9] = 4;
+        f_cols[10] = 1;
+        f_cols[11] = 3;
+        f_cols[12] = 1;
+        f_cols[13] = 2;
+        f_cols[14] = 4;
+
+        agg_val[0] = 0;
+        agg_val[1] = 1;
+        agg_val[2] = 0;
+        agg_val[3] = 1;
+        agg_val[4] = 0;
+        /* this coarse is stored:
+         *  6 -5
+         * -4  5
+         */
+        c_vals[0] = 6;
+        c_vals[1] = -5;
+        c_vals[2] = -4;
+        c_vals[3] = 5;
+
+        c_rows[0] = 0;
+        c_rows[1] = 2;
+        c_rows[2] = 4;
+
+        c_cols[0] = 0;
+        c_cols[1] = 1;
+        c_cols[2] = 0;
+        c_cols[3] = 1;
+    }
+
+    static void assert_same_matrices(const Mtx *m1, const Mtx *m2)
+    {
+        ASSERT_EQ(m1->get_size()[0], m2->get_size()[0]);
+        ASSERT_EQ(m1->get_size()[1], m2->get_size()[1]);
+        ASSERT_EQ(m1->get_num_stored_elements(), m2->get_num_stored_elements());
+        for (gko::size_type i = 0; i < m1->get_size() + 1; i++) {
+            ASSERT_EQ(m1->get_const_row_ptrs()[i], m2->get_const_row_ptrs()[i]);
+        }
+        for (gko::size_type i = 0; i < m1->get_num_stored_elements(); ++i) {
+            EXPECT_EQ(m1->get_const_values()[i], m2->get_const_values()[i]);
+            EXPECT_EQ(m1->get_const_col_idxs()[i], m2->get_const_col_idxs()[i]);
+        }
+    }
+
+    static void assert_same_agg(const index_type *m1, const index_type *m2,
+                                gko::size_type len)
+    {
+        for (gko::size_type i = 0; i < len; ++i) {
+            EXPECT_EQ(m1[i], m2[i]);
+        }
+    }
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<Mtx> mtx;
+    std::shared_ptr<Mtx> coarse;
+    gko::Array<index_type> agg;
+    std::unique_ptr<typename RestrictProlong::Factory> amgxpgm_factory;
+    std::unique_ptr<RestrictProlong> rstr_prlg;
+};
+
+TYPED_TEST_CASE(AmgxPgm, gko::test::ValueIndexTypes);
+
+
+TYPED_TEST(AmgxPgm, FactoryKnowsItsExecutor)
+{
+    ASSERT_EQ(this->amgxpgm_factory->get_executor(), this->exec);
+}
+
+
+TYPED_TEST(AmgxPgm, CanBeCopied)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using RestrictProlong = typename TestFixture::RestrictProlong;
+    auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
+
+    // copy->copy_from(this->rstr_prlg.get());
+
+    // auto copy_mtx =
+    //     static_cast<RestrictProlong *>(copy.get())->get_system_matrix();
+    // auto copy_agg = static_cast<RestrictProlong
+    // *>(copy.get())->get_const_agg(); auto copy_coarse =
+    // copy->get_coarse_operator();
+
+    // this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
+    //                            this->mtx.get());
+    // this->assert_same_agg(copy_agg, this->agg.get_data(),
+    // this->agg.get_num_elems()); this->assert_same_matrices(static_cast<const
+    // Mtx *>(copy_coarse.get()), this->coarse.get());
+}
+
+
+TYPED_TEST(AmgxPgm, CanBeMoved)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using RestrictProlong = typename TestFixture::RestrictProlong;
+    auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
+
+    copy->copy_from(std::move(this->rstr_prlg));
+
+    auto copy_mtx =
+        static_cast<RestrictProlong *>(copy.get())->get_system_matrix();
+    auto copy_agg = static_cast<RestrictProlong *>(copy.get())->get_const_agg();
+    auto copy_coarse = copy->get_coarse_operator();
+
+    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
+                               this->mtx.get());
+    this->assert_same_agg(copy_agg, this->agg.get_data(),
+                          this->agg.get_num_elems());
+    this->assert_same_matrices(static_cast<const Mtx *>(copy_coarse.get()),
+                               this->coarse.get());
+}
+
+
+TYPED_TEST(AmgxPgm, CanBeCloned)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using RestrictProlong = typename TestFixture::RestrictProlong;
+    auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
+
+    auto clone = this->rstr_prlg->clone();
+
+    auto clone_mtx =
+        static_cast<RestrictProlong *>(clone.get())->get_system_matrix();
+    auto clone_agg =
+        static_cast<RestrictProlong *>(clone.get())->get_const_agg();
+    auto clone_coarse = clone->get_coarse_operator();
+
+    this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
+                               this->mtx.get());
+    this->assert_same_agg(clone_agg, this->agg.get_data(),
+                          this->agg.get_num_elems());
+    this->assert_same_matrices(static_cast<const Mtx *>(clone_coarse.get()),
+                               this->coarse.get());
+}
+
+
+TYPED_TEST(AmgxPgm, CanBeCleared)
+{
+    using RestrictProlong = typename TestFixture::RestrictProlong;
+    this->rstr_prlg->clear();
+
+    auto mtx = static_cast<RestrictProlong *>(this->rstr_prlg.get())
+                   ->get_system_matrix();
+    auto coarse = this->rstr_prlg->get_coarse_operator();
+    auto agg = static_cast<RestrictProlong *>(this->rstr_prlg.get())->get_agg();
+    ASSERT_EQ(mtx, nullptr);
+    ASSERT_EQ(coarse, nullptr);
+    ASSERT_EQ(agg, nullptr);
+}
+
+
+}  // namespace
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index d3da801110f..ecac15c6f0b 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -51,7 +51,8 @@ namespace multigrid {
 /**
  * Amgx parallel graph match (AmgxPgm) is the aggregate method introduced in the
  * paper M. Naumov et al., "AmgX: A Library for GPU Accelerated Algebraic
- * Multigrid and Preconditioned Iterative Methods"
+ * Multigrid and Preconditioned Iterative Methods". Current implemenation only
+ * contains size = 2 version.
  *
  * AmgxPgm creates the aggreagate group according to the matrix value not the
  * structure.
@@ -134,7 +135,10 @@ class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
           system_matrix_{std::move(system_matrix)},
           agg_(factory->get_executor(), system_matrix_->get_size()[0])
     {
-        this->generate();
+        if (system_matrix_->get_size()[0] != 0) {
+            // generate on the existed matrix
+            this->generate();
+        }
     }
 
     void generate();
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index ff9507e2639..ea16d233de3 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -66,11 +66,11 @@ class AmgxPgm : public ::testing::Test {
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using Mtx = gko::matrix::Csr<value_type, index_type>;
     using Vec = gko::matrix::Dense<value_type>;
-    using Multigrid = gko::multigrid::AmgxPgm<value_type, index_type>;
+    using RestrictProlong = gko::multigrid::AmgxPgm<value_type, index_type>;
     using T = value_type;
     AmgxPgm()
         : exec(gko::ReferenceExecutor::create()),
-          AmgxPgm_factory(Multigrid::build()
+          Amgxpgm_factory(RestrictProlong::build()
                               .with_max_iterations(2u)
                               .with_max_unassigned_percentage(0.1)
                               .on(exec)),
@@ -174,7 +174,7 @@ class AmgxPgm : public ::testing::Test {
     std::shared_ptr<Vec> restrict_ans;
     std::shared_ptr<Vec> prolong_ans;
     std::shared_ptr<Vec> fine_x;
-    std::unique_ptr<typename Multigrid::Factory> AmgxPgm_factory;
+    std::unique_ptr<typename RestrictProlong::Factory> Amgxpgm_factory;
 };
 
 TYPED_TEST_CASE(AmgxPgm, gko::test::ValueIndexTypes);
@@ -269,7 +269,7 @@ TYPED_TEST(AmgxPgm, Renumber)
 
 TYPED_TEST(AmgxPgm, Generate)
 {
-    auto coarse_fine = this->AmgxPgm_factory->generate(this->mtx);
+    auto coarse_fine = this->Amgxpgm_factory->generate(this->mtx);
 
     auto agg_result = coarse_fine->get_const_agg();
 
@@ -284,7 +284,7 @@ TYPED_TEST(AmgxPgm, Generate)
 TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
 {
     std::unique_ptr<gko::multigrid::RestrictProlong> amgx_pgm{
-        this->AmgxPgm_factory->generate(this->mtx)};
+        this->Amgxpgm_factory->generate(this->mtx)};
 
     // fine->coarse
     using Vec = typename TestFixture::Vec;
@@ -299,7 +299,7 @@ TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
 TYPED_TEST(AmgxPgm, CoarseFineProlongApplyadd)
 {
     std::unique_ptr<gko::multigrid::RestrictProlong> amgx_pgm{
-        this->AmgxPgm_factory->generate(this->mtx)};
+        this->Amgxpgm_factory->generate(this->mtx)};
 
     using value_type = typename TestFixture::value_type;
     auto x = gko::clone(this->fine_x);

From 6d28e3a76b8cf38e34b88d656074bdaf0f274ea0 Mon Sep 17 00:00:00 2001
From: yhmtsai <yhmtsai@gmail.com>
Date: Wed, 3 Jun 2020 14:21:24 +0200
Subject: [PATCH 0464/1008] move the test to correct place and format

---
 core/multigrid/amgx_pgm.cpp                   |   1 +
 core/multigrid/amgx_pgm_kernels.hpp           |   6 -
 core/test/multigrid/amgx_pgm.cpp              | 204 ++----------------
 cuda/test/CMakeLists.txt                      |   1 -
 cuda/test/multigrid/CMakeLists.txt            |   1 -
 cuda/test/multigrid/amgx_pgm_kernels.cpp      |  56 -----
 hip/test/CMakeLists.txt                       |   1 -
 hip/test/multigrid/CMakeLists.txt             |   1 -
 hip/test/multigrid/amgx_pgm_kernels.cpp       |  53 -----
 include/ginkgo/core/multigrid/amgx_pgm.hpp    |   4 +-
 .../core/multigrid/restrict_prolong.hpp       |   4 +-
 include/ginkgo/ginkgo.hpp                     |   3 +
 omp/test/CMakeLists.txt                       |   1 -
 omp/test/multigrid/CMakeLists.txt             |   1 -
 omp/test/multigrid/amgx_pgm_kernels.cpp       |  53 -----
 reference/test/matrix/csr_kernels.cpp         |   1 -
 reference/test/multigrid/amgx_pgm_kernels.cpp | 166 ++++++++++++--
 test_install/test_install.cpp                 |   5 +
 test_install/test_install_cuda.cu             |   5 +
 19 files changed, 193 insertions(+), 374 deletions(-)
 delete mode 100644 cuda/test/multigrid/CMakeLists.txt
 delete mode 100644 cuda/test/multigrid/amgx_pgm_kernels.cpp
 delete mode 100644 hip/test/multigrid/CMakeLists.txt
 delete mode 100644 hip/test/multigrid/amgx_pgm_kernels.cpp
 delete mode 100644 omp/test/multigrid/CMakeLists.txt
 delete mode 100644 omp/test/multigrid/amgx_pgm_kernels.cpp

diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index db02285e90c..4a2f9fdf56a 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -67,6 +67,7 @@ GKO_REGISTER_OPERATION(amgx_pgm_generate, amgx_pgm::amgx_pgm_generate);
 
 }  // namespace amgx_pgm
 
+
 namespace {
 
 
diff --git a/core/multigrid/amgx_pgm_kernels.hpp b/core/multigrid/amgx_pgm_kernels.hpp
index 68e6f57f857..26031d570fd 100644
--- a/core/multigrid/amgx_pgm_kernels.hpp
+++ b/core/multigrid/amgx_pgm_kernels.hpp
@@ -55,29 +55,24 @@ namespace amgx_pgm {
         std::shared_ptr<const DefaultExecutor> exec, const Array<_itype> &agg, \
         const matrix::Dense<_vtype> *b, matrix::Dense<_vtype> *x)
 
-
 #define GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL(_vtype, _itype)           \
     void prolong_applyadd(                                                     \
         std::shared_ptr<const DefaultExecutor> exec, const Array<_itype> &agg, \
         const matrix::Dense<_vtype> *b, matrix::Dense<_vtype> *x)
 
-
 #define GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL(_itype)           \
     void initial(std::shared_ptr<const DefaultExecutor> exec, \
                  Array<_itype> &agg)
 
-
 #define GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL(_itype)           \
     void match_edge(std::shared_ptr<const DefaultExecutor> exec, \
                     const Array<_itype> &strongest_neighbor,     \
                     Array<_itype> &agg)
 
-
 #define GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL(_itype)           \
     void count_unagg(std::shared_ptr<const DefaultExecutor> exec, \
                      const Array<_itype> &agg, size_type *num_unagg)
 
-
 #define GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL(_itype)           \
     void renumber(std::shared_ptr<const DefaultExecutor> exec, \
                   Array<_itype> &agg, size_type *num_agg)
@@ -107,7 +102,6 @@ namespace amgx_pgm {
                            const Array<IndexType> &agg,                     \
                            matrix::Csr<ValueType, IndexType> *coarse)
 
-
 #define GKO_DECLARE_ALL_AS_TEMPLATES                                    \
     template <typename ValueType, typename IndexType>                   \
     GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL(ValueType, IndexType);   \
diff --git a/core/test/multigrid/amgx_pgm.cpp b/core/test/multigrid/amgx_pgm.cpp
index 33e196c11aa..3ffb3d115ab 100644
--- a/core/test/multigrid/amgx_pgm.cpp
+++ b/core/test/multigrid/amgx_pgm.cpp
@@ -49,7 +49,7 @@ namespace {
 
 
 template <typename ValueIndexType>
-class AmgxPgm : public ::testing::Test {
+class AmgxPgmFactory : public ::testing::Test {
 protected:
     using value_type =
         typename std::tuple_element<0, decltype(ValueIndexType())>::type;
@@ -59,220 +59,56 @@ class AmgxPgm : public ::testing::Test {
     using Vec = gko::matrix::Dense<value_type>;
     using RestrictProlong = gko::multigrid::AmgxPgm<value_type, index_type>;
     using T = value_type;
-    AmgxPgm()
+    AmgxPgmFactory()
         : exec(gko::ReferenceExecutor::create()),
           amgxpgm_factory(RestrictProlong::build()
                               .with_max_iterations(2u)
                               .with_max_unassigned_percentage(0.1)
-                              .on(exec)),
-          mtx(Mtx::create(exec, gko::dim<2>(5, 5), 15,
-                          std::make_shared<typename Mtx::classical>())),
-          agg(exec, 5),
-          coarse(Mtx::create(exec, gko::dim<2>(2, 2), 4,
-                             std::make_shared<typename Mtx::classical>()))
-
-    {
-        this->create_mtx(mtx.get(), &agg, coarse.get());
-        rstr_prlg = amgxpgm_factory->generate(mtx);
-    }
-
-    void create_mtx(Mtx *fine, gko::Array<index_type> *agg, Mtx *coarse)
-    {
-        auto f_vals = fine->get_values();
-        auto f_cols = fine->get_col_idxs();
-        auto f_rows = fine->get_row_ptrs();
-        auto c_vals = coarse->get_values();
-        auto c_cols = coarse->get_col_idxs();
-        auto c_rows = coarse->get_row_ptrs();
-        auto agg_val = agg->get_data();
-        /* this matrix is stored:
-         *  5 -3 -3  0  0
-         * -3  5  0 -2 -1
-         * -3  0  5  0 -1
-         *  0 -3  0  5  0
-         *  0 -2 -2  0  5
-         */
-        f_vals[0] = 5;
-        f_vals[1] = -3;
-        f_vals[2] = -3;
-        f_vals[3] = -3;
-        f_vals[4] = 5;
-        f_vals[5] = -2;
-        f_vals[6] = -1;
-        f_vals[7] = -3;
-        f_vals[8] = 5;
-        f_vals[9] = -1;
-        f_vals[10] = -3;
-        f_vals[11] = 5;
-        f_vals[12] = -2;
-        f_vals[13] = -2;
-        f_vals[14] = 5;
-
-        f_rows[0] = 0;
-        f_rows[1] = 3;
-        f_rows[2] = 7;
-        f_rows[3] = 10;
-        f_rows[4] = 12;
-        f_rows[5] = 15;
-
-        f_cols[0] = 0;
-        f_cols[1] = 1;
-        f_cols[2] = 2;
-        f_cols[3] = 0;
-        f_cols[4] = 1;
-        f_cols[5] = 3;
-        f_cols[6] = 4;
-        f_cols[7] = 0;
-        f_cols[8] = 2;
-        f_cols[9] = 4;
-        f_cols[10] = 1;
-        f_cols[11] = 3;
-        f_cols[12] = 1;
-        f_cols[13] = 2;
-        f_cols[14] = 4;
-
-        agg_val[0] = 0;
-        agg_val[1] = 1;
-        agg_val[2] = 0;
-        agg_val[3] = 1;
-        agg_val[4] = 0;
-        /* this coarse is stored:
-         *  6 -5
-         * -4  5
-         */
-        c_vals[0] = 6;
-        c_vals[1] = -5;
-        c_vals[2] = -4;
-        c_vals[3] = 5;
-
-        c_rows[0] = 0;
-        c_rows[1] = 2;
-        c_rows[2] = 4;
-
-        c_cols[0] = 0;
-        c_cols[1] = 1;
-        c_cols[2] = 0;
-        c_cols[3] = 1;
-    }
-
-    static void assert_same_matrices(const Mtx *m1, const Mtx *m2)
-    {
-        ASSERT_EQ(m1->get_size()[0], m2->get_size()[0]);
-        ASSERT_EQ(m1->get_size()[1], m2->get_size()[1]);
-        ASSERT_EQ(m1->get_num_stored_elements(), m2->get_num_stored_elements());
-        for (gko::size_type i = 0; i < m1->get_size() + 1; i++) {
-            ASSERT_EQ(m1->get_const_row_ptrs()[i], m2->get_const_row_ptrs()[i]);
-        }
-        for (gko::size_type i = 0; i < m1->get_num_stored_elements(); ++i) {
-            EXPECT_EQ(m1->get_const_values()[i], m2->get_const_values()[i]);
-            EXPECT_EQ(m1->get_const_col_idxs()[i], m2->get_const_col_idxs()[i]);
-        }
-    }
-
-    static void assert_same_agg(const index_type *m1, const index_type *m2,
-                                gko::size_type len)
-    {
-        for (gko::size_type i = 0; i < len; ++i) {
-            EXPECT_EQ(m1[i], m2[i]);
-        }
-    }
+                              .with_deterministic(true)
+                              .on(exec))
+
+    {}
 
     std::shared_ptr<const gko::Executor> exec;
-    std::shared_ptr<Mtx> mtx;
-    std::shared_ptr<Mtx> coarse;
-    gko::Array<index_type> agg;
     std::unique_ptr<typename RestrictProlong::Factory> amgxpgm_factory;
-    std::unique_ptr<RestrictProlong> rstr_prlg;
 };
 
-TYPED_TEST_CASE(AmgxPgm, gko::test::ValueIndexTypes);
+TYPED_TEST_CASE(AmgxPgmFactory, gko::test::ValueIndexTypes);
 
 
-TYPED_TEST(AmgxPgm, FactoryKnowsItsExecutor)
+TYPED_TEST(AmgxPgmFactory, FactoryKnowsItsExecutor)
 {
     ASSERT_EQ(this->amgxpgm_factory->get_executor(), this->exec);
 }
 
 
-TYPED_TEST(AmgxPgm, CanBeCopied)
+TYPED_TEST(AmgxPgmFactory, DefaultSetting)
 {
-    using Mtx = typename TestFixture::Mtx;
     using RestrictProlong = typename TestFixture::RestrictProlong;
-    auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
-
-    // copy->copy_from(this->rstr_prlg.get());
-
-    // auto copy_mtx =
-    //     static_cast<RestrictProlong *>(copy.get())->get_system_matrix();
-    // auto copy_agg = static_cast<RestrictProlong
-    // *>(copy.get())->get_const_agg(); auto copy_coarse =
-    // copy->get_coarse_operator();
+    auto factory = RestrictProlong::build().on(this->exec);
 
-    // this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
-    //                            this->mtx.get());
-    // this->assert_same_agg(copy_agg, this->agg.get_data(),
-    // this->agg.get_num_elems()); this->assert_same_matrices(static_cast<const
-    // Mtx *>(copy_coarse.get()), this->coarse.get());
+    ASSERT_EQ(factory->get_parameters().max_iterations, 15u);
+    ASSERT_EQ(factory->get_parameters().max_unassigned_percentage, 0.05);
+    ASSERT_EQ(factory->get_parameters().deterministic, false);
 }
 
 
-TYPED_TEST(AmgxPgm, CanBeMoved)
+TYPED_TEST(AmgxPgmFactory, SetMaxIterations)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using RestrictProlong = typename TestFixture::RestrictProlong;
-    auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
-
-    copy->copy_from(std::move(this->rstr_prlg));
-
-    auto copy_mtx =
-        static_cast<RestrictProlong *>(copy.get())->get_system_matrix();
-    auto copy_agg = static_cast<RestrictProlong *>(copy.get())->get_const_agg();
-    auto copy_coarse = copy->get_coarse_operator();
-
-    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
-                               this->mtx.get());
-    this->assert_same_agg(copy_agg, this->agg.get_data(),
-                          this->agg.get_num_elems());
-    this->assert_same_matrices(static_cast<const Mtx *>(copy_coarse.get()),
-                               this->coarse.get());
+    ASSERT_EQ(this->amgxpgm_factory->get_parameters().max_iterations, 2u);
 }
 
 
-TYPED_TEST(AmgxPgm, CanBeCloned)
+TYPED_TEST(AmgxPgmFactory, SetMaxUnassignedPercentage)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using RestrictProlong = typename TestFixture::RestrictProlong;
-    auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
-
-    auto clone = this->rstr_prlg->clone();
-
-    auto clone_mtx =
-        static_cast<RestrictProlong *>(clone.get())->get_system_matrix();
-    auto clone_agg =
-        static_cast<RestrictProlong *>(clone.get())->get_const_agg();
-    auto clone_coarse = clone->get_coarse_operator();
-
-    this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
-                               this->mtx.get());
-    this->assert_same_agg(clone_agg, this->agg.get_data(),
-                          this->agg.get_num_elems());
-    this->assert_same_matrices(static_cast<const Mtx *>(clone_coarse.get()),
-                               this->coarse.get());
+    ASSERT_EQ(this->amgxpgm_factory->get_parameters().max_unassigned_percentage,
+              0.1);
 }
 
 
-TYPED_TEST(AmgxPgm, CanBeCleared)
+TYPED_TEST(AmgxPgmFactory, SetDeterministic)
 {
-    using RestrictProlong = typename TestFixture::RestrictProlong;
-    this->rstr_prlg->clear();
-
-    auto mtx = static_cast<RestrictProlong *>(this->rstr_prlg.get())
-                   ->get_system_matrix();
-    auto coarse = this->rstr_prlg->get_coarse_operator();
-    auto agg = static_cast<RestrictProlong *>(this->rstr_prlg.get())->get_agg();
-    ASSERT_EQ(mtx, nullptr);
-    ASSERT_EQ(coarse, nullptr);
-    ASSERT_EQ(agg, nullptr);
+    ASSERT_EQ(this->amgxpgm_factory->get_parameters().deterministic, true);
 }
 
 
diff --git a/cuda/test/CMakeLists.txt b/cuda/test/CMakeLists.txt
index fb66aaf270f..83d7b39f35e 100644
--- a/cuda/test/CMakeLists.txt
+++ b/cuda/test/CMakeLists.txt
@@ -4,7 +4,6 @@ add_subdirectory(base)
 add_subdirectory(components)
 add_subdirectory(factorization)
 add_subdirectory(matrix)
-add_subdirectory(multigrid)
 add_subdirectory(preconditioner)
 add_subdirectory(reorder)
 add_subdirectory(solver)
diff --git a/cuda/test/multigrid/CMakeLists.txt b/cuda/test/multigrid/CMakeLists.txt
deleted file mode 100644
index 55bea27db18..00000000000
--- a/cuda/test/multigrid/CMakeLists.txt
+++ /dev/null
@@ -1 +0,0 @@
-ginkgo_create_test_cpp_cuda_header(amgx_pgm_kernels)
\ No newline at end of file
diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
deleted file mode 100644
index b72889b923f..00000000000
--- a/cuda/test/multigrid/amgx_pgm_kernels.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/multigrid/amgx_pgm.hpp>
-
-
-#include <memory>
-#include <random>
-
-
-#include <cuda.h>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-
-
-#include "core/multigrid/amgx_pgm_kernels.hpp"
-#include "cuda/test/utils.hpp"
-
-
-namespace {}  // namespace
diff --git a/hip/test/CMakeLists.txt b/hip/test/CMakeLists.txt
index a5f126893bf..9c8e39ca4ab 100644
--- a/hip/test/CMakeLists.txt
+++ b/hip/test/CMakeLists.txt
@@ -4,7 +4,6 @@ add_subdirectory(base)
 add_subdirectory(components)
 add_subdirectory(factorization)
 add_subdirectory(matrix)
-add_subdirectory(multigrid)
 add_subdirectory(solver)
 add_subdirectory(preconditioner)
 add_subdirectory(stop)
diff --git a/hip/test/multigrid/CMakeLists.txt b/hip/test/multigrid/CMakeLists.txt
deleted file mode 100644
index 481c2cc1bf2..00000000000
--- a/hip/test/multigrid/CMakeLists.txt
+++ /dev/null
@@ -1 +0,0 @@
-ginkgo_create_hip_test_special_linkage(amgx_pgm_kernels)
diff --git a/hip/test/multigrid/amgx_pgm_kernels.cpp b/hip/test/multigrid/amgx_pgm_kernels.cpp
deleted file mode 100644
index 13c99bc6a2b..00000000000
--- a/hip/test/multigrid/amgx_pgm_kernels.cpp
+++ /dev/null
@@ -1,53 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/multigrid/amgx_pgm.hpp>
-
-
-#include <memory>
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-
-
-#include "core/multigrid/amgx_pgm_kernels.hpp"
-#include "hip/test/utils.hip.hpp"
-
-
-namespace {}  // namespace
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index ecac15c6f0b..1925fdbf6a5 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -106,7 +106,7 @@ class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
         /**
          * The maximum number of iteration.
          */
-        unsigned GKO_FACTORY_PARAMETER(max_iterations, 15);
+        unsigned GKO_FACTORY_PARAMETER(max_iterations, 15u);
 
         /**
          * The maximum ratio of unassigned number.
@@ -125,9 +125,11 @@ class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
     void restrict_apply_impl(const LinOp *b, LinOp *x) const override;
 
     void prolong_applyadd_impl(const LinOp *b, LinOp *x) const override;
+
     explicit AmgxPgm(std::shared_ptr<const Executor> exec)
         : EnableRestrictProlong<AmgxPgm>(std::move(exec))
     {}
+
     explicit AmgxPgm(const Factory *factory,
                      std::shared_ptr<const LinOp> system_matrix)
         : EnableRestrictProlong<AmgxPgm>(factory->get_executor()),
diff --git a/include/ginkgo/core/multigrid/restrict_prolong.hpp b/include/ginkgo/core/multigrid/restrict_prolong.hpp
index 5a7537bddb1..13baa11cdb5 100644
--- a/include/ginkgo/core/multigrid/restrict_prolong.hpp
+++ b/include/ginkgo/core/multigrid/restrict_prolong.hpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <functional>
 #include <memory>
 
+
 #include <ginkgo/core/base/abstract_factory.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
@@ -55,7 +56,6 @@ namespace multigrid {
  *
  * @ingroup RestrictProlong
  */
-
 class RestrictProlong
     : public EnableAbstractPolymorphicObject<RestrictProlong> {
 public:
@@ -226,6 +226,7 @@ class RestrictProlongFactory
     }
 };
 
+
 template <typename ConcreteFactory, typename ConcreteCriterion,
           typename ParametersType,
           typename PolymorphicBase = RestrictProlongFactory>
@@ -260,7 +261,6 @@ class EnableRestrictProlong
         return self();
     }
 
-
     ConcreteLinOp *prolong_applyadd(const LinOp *b, LinOp *x)
     {
         this->validate_prolong_parameters(b, x);
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index b883785fbeb..1942000117b 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -87,6 +87,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sellp.hpp>
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+#include <ginkgo/core/multigrid/restrict_prolong.hpp>
+
 #include <ginkgo/core/preconditioner/ic.hpp>
 #include <ginkgo/core/preconditioner/ilu.hpp>
 #include <ginkgo/core/preconditioner/isai.hpp>
diff --git a/omp/test/CMakeLists.txt b/omp/test/CMakeLists.txt
index 3759a39bc77..2f506e51109 100644
--- a/omp/test/CMakeLists.txt
+++ b/omp/test/CMakeLists.txt
@@ -3,7 +3,6 @@ include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
 add_subdirectory(components)
 add_subdirectory(factorization)
 add_subdirectory(matrix)
-add_subdirectory(multigrid)
 add_subdirectory(preconditioner)
 add_subdirectory(reorder)
 add_subdirectory(solver)
diff --git a/omp/test/multigrid/CMakeLists.txt b/omp/test/multigrid/CMakeLists.txt
deleted file mode 100644
index 8fe8bbeba48..00000000000
--- a/omp/test/multigrid/CMakeLists.txt
+++ /dev/null
@@ -1 +0,0 @@
-ginkgo_create_test(amgx_pgm_kernels)
diff --git a/omp/test/multigrid/amgx_pgm_kernels.cpp b/omp/test/multigrid/amgx_pgm_kernels.cpp
deleted file mode 100644
index 4d2f5ed1fa6..00000000000
--- a/omp/test/multigrid/amgx_pgm_kernels.cpp
+++ /dev/null
@@ -1,53 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/multigrid/amgx_pgm.hpp>
-
-
-#include <memory>
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-
-
-#include "core/multigrid/amgx_pgm_kernels.hpp"
-#include "core/test/utils.hpp"
-
-
-namespace {}  // namespace
diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp
index 654a96acd61..186aa700598 100644
--- a/reference/test/matrix/csr_kernels.cpp
+++ b/reference/test/matrix/csr_kernels.cpp
@@ -198,7 +198,6 @@ class Csr : public ::testing::Test {
         cols_u[6] = 0;
     }
 
-
     void assert_equal_to_mtx(const Coo *m)
     {
         auto v = m->get_const_values();
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index ea16d233de3..bbbdd5b1968 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -70,7 +70,7 @@ class AmgxPgm : public ::testing::Test {
     using T = value_type;
     AmgxPgm()
         : exec(gko::ReferenceExecutor::create()),
-          Amgxpgm_factory(RestrictProlong::build()
+          amgxpgm_factory(RestrictProlong::build()
                               .with_max_iterations(2u)
                               .with_max_unassigned_percentage(0.1)
                               .on(exec)),
@@ -92,20 +92,26 @@ class AmgxPgm : public ::testing::Test {
               exec)),
           mtx(Mtx::create(exec, gko::dim<2>(5, 5), 15,
                           std::make_shared<typename Mtx::classical>())),
+          coarse(Mtx::create(exec, gko::dim<2>(2, 2), 4,
+                             std::make_shared<typename Mtx::classical>())),
           mtx_diag(exec, 5),
           agg(exec, 5)
     {
-        this->create_mtx(mtx.get(), &mtx_diag, &agg);
+        this->create_mtx(mtx.get(), &mtx_diag, &agg, coarse.get());
+        rstr_prlg = amgxpgm_factory->generate(mtx);
     }
 
     void create_mtx(Mtx *fine, gko::Array<value_type> *diag,
-                    gko::Array<index_type> *agg)
+                    gko::Array<index_type> *agg, Mtx *coarse)
     {
         auto vals = fine->get_values();
         auto cols = fine->get_col_idxs();
         auto rows = fine->get_row_ptrs();
         auto diag_val = diag->get_data();
         auto agg_val = agg->get_data();
+        auto c_vals = coarse->get_values();
+        auto c_cols = coarse->get_col_idxs();
+        auto c_rows = coarse->get_row_ptrs();
         /* this matrix is stored:
          *  5 -3 -3  0  0
          * -3  5  0 -2 -1
@@ -163,10 +169,51 @@ class AmgxPgm : public ::testing::Test {
         agg_val[2] = 0;
         agg_val[3] = 1;
         agg_val[4] = 0;
+
+        /* this coarse is stored:
+         *  6 -5
+         * -4  5
+         */
+        c_vals[0] = 6;
+        c_vals[1] = -5;
+        c_vals[2] = -4;
+        c_vals[3] = 5;
+
+        c_rows[0] = 0;
+        c_rows[1] = 2;
+        c_rows[2] = 4;
+
+        c_cols[0] = 0;
+        c_cols[1] = 1;
+        c_cols[2] = 0;
+        c_cols[3] = 1;
+    }
+
+    static void assert_same_matrices(const Mtx *m1, const Mtx *m2)
+    {
+        ASSERT_EQ(m1->get_size()[0], m2->get_size()[0]);
+        ASSERT_EQ(m1->get_size()[1], m2->get_size()[1]);
+        ASSERT_EQ(m1->get_num_stored_elements(), m2->get_num_stored_elements());
+        for (gko::size_type i = 0; i < m1->get_size() + 1; i++) {
+            ASSERT_EQ(m1->get_const_row_ptrs()[i], m2->get_const_row_ptrs()[i]);
+        }
+        for (gko::size_type i = 0; i < m1->get_num_stored_elements(); ++i) {
+            EXPECT_EQ(m1->get_const_values()[i], m2->get_const_values()[i]);
+            EXPECT_EQ(m1->get_const_col_idxs()[i], m2->get_const_col_idxs()[i]);
+        }
+    }
+
+    static void assert_same_agg(const index_type *m1, const index_type *m2,
+                                gko::size_type len)
+    {
+        for (gko::size_type i = 0; i < len; ++i) {
+            EXPECT_EQ(m1[i], m2[i]);
+        }
     }
 
     std::shared_ptr<const gko::ReferenceExecutor> exec;
     std::shared_ptr<Mtx> mtx;
+    std::shared_ptr<Mtx> coarse;
     gko::Array<value_type> mtx_diag;
     gko::Array<index_type> agg;
     std::shared_ptr<Vec> coarse_b;
@@ -174,19 +221,107 @@ class AmgxPgm : public ::testing::Test {
     std::shared_ptr<Vec> restrict_ans;
     std::shared_ptr<Vec> prolong_ans;
     std::shared_ptr<Vec> fine_x;
-    std::unique_ptr<typename RestrictProlong::Factory> Amgxpgm_factory;
+    std::unique_ptr<typename RestrictProlong::Factory> amgxpgm_factory;
+    std::unique_ptr<RestrictProlong> rstr_prlg;
 };
 
 TYPED_TEST_CASE(AmgxPgm, gko::test::ValueIndexTypes);
 
+
+TYPED_TEST(AmgxPgm, CanBeCopied)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using RestrictProlong = typename TestFixture::RestrictProlong;
+    auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
+
+    copy->copy_from(this->rstr_prlg.get());
+
+    auto copy_mtx =
+        static_cast<RestrictProlong *>(copy.get())->get_system_matrix();
+    auto copy_agg = static_cast<RestrictProlong *>(copy.get())->get_const_agg();
+    auto copy_coarse = copy->get_coarse_operator();
+
+    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
+                               this->mtx.get());
+    this->assert_same_agg(copy_agg, this->agg.get_data(),
+                          this->agg.get_num_elems());
+    this->assert_same_matrices(static_cast<const Mtx *>(copy_coarse.get()),
+                               this->coarse.get());
+}
+
+
+TYPED_TEST(AmgxPgm, CanBeMoved)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using RestrictProlong = typename TestFixture::RestrictProlong;
+    auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
+
+    copy->copy_from(std::move(this->rstr_prlg));
+
+    auto copy_mtx =
+        static_cast<RestrictProlong *>(copy.get())->get_system_matrix();
+    auto copy_agg = static_cast<RestrictProlong *>(copy.get())->get_const_agg();
+    auto copy_coarse = copy->get_coarse_operator();
+
+    this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
+                               this->mtx.get());
+    this->assert_same_agg(copy_agg, this->agg.get_data(),
+                          this->agg.get_num_elems());
+    this->assert_same_matrices(static_cast<const Mtx *>(copy_coarse.get()),
+                               this->coarse.get());
+}
+
+
+TYPED_TEST(AmgxPgm, CanBeCloned)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using RestrictProlong = typename TestFixture::RestrictProlong;
+    auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
+
+    auto clone = this->rstr_prlg->clone();
+
+    auto clone_mtx =
+        static_cast<RestrictProlong *>(clone.get())->get_system_matrix();
+    auto clone_agg =
+        static_cast<RestrictProlong *>(clone.get())->get_const_agg();
+    auto clone_coarse = clone->get_coarse_operator();
+
+    this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
+                               this->mtx.get());
+    this->assert_same_agg(clone_agg, this->agg.get_data(),
+                          this->agg.get_num_elems());
+    this->assert_same_matrices(static_cast<const Mtx *>(clone_coarse.get()),
+                               this->coarse.get());
+}
+
+
+TYPED_TEST(AmgxPgm, CanBeCleared)
+{
+    using RestrictProlong = typename TestFixture::RestrictProlong;
+
+    this->rstr_prlg->clear();
+
+    auto mtx = static_cast<RestrictProlong *>(this->rstr_prlg.get())
+                   ->get_system_matrix();
+    auto coarse = this->rstr_prlg->get_coarse_operator();
+    auto agg = static_cast<RestrictProlong *>(this->rstr_prlg.get())->get_agg();
+
+    ASSERT_EQ(mtx, nullptr);
+    ASSERT_EQ(coarse, nullptr);
+    ASSERT_EQ(agg, nullptr);
+}
+
+
 TYPED_TEST(AmgxPgm, RestrictApply)
 {
     // fine->coarse
     using Vec = typename TestFixture::Vec;
     using value_type = typename TestFixture::value_type;
     auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
+
     gko::kernels::reference::amgx_pgm::restrict_apply(
         this->exec, this->agg, this->fine_b.get(), x.get());
+
     GKO_ASSERT_MTX_NEAR(x, this->restrict_ans,
                         gko::remove_complex<value_type>{0});
 }
@@ -195,8 +330,10 @@ TYPED_TEST(AmgxPgm, ProlongApplyadd)
 {
     using value_type = typename TestFixture::value_type;
     auto x = gko::clone(this->fine_x);
+
     gko::kernels::reference::amgx_pgm::prolong_applyadd(
         this->exec, this->agg, this->coarse_b.get(), x.get());
+
     GKO_ASSERT_MTX_NEAR(x, this->prolong_ans,
                         gko::remove_complex<value_type>{0});
 }
@@ -269,7 +406,7 @@ TYPED_TEST(AmgxPgm, Renumber)
 
 TYPED_TEST(AmgxPgm, Generate)
 {
-    auto coarse_fine = this->Amgxpgm_factory->generate(this->mtx);
+    auto coarse_fine = this->amgxpgm_factory->generate(this->mtx);
 
     auto agg_result = coarse_fine->get_const_agg();
 
@@ -284,7 +421,7 @@ TYPED_TEST(AmgxPgm, Generate)
 TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
 {
     std::unique_ptr<gko::multigrid::RestrictProlong> amgx_pgm{
-        this->Amgxpgm_factory->generate(this->mtx)};
+        this->amgxpgm_factory->generate(this->mtx)};
 
     // fine->coarse
     using Vec = typename TestFixture::Vec;
@@ -298,11 +435,11 @@ TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
 
 TYPED_TEST(AmgxPgm, CoarseFineProlongApplyadd)
 {
-    std::unique_ptr<gko::multigrid::RestrictProlong> amgx_pgm{
-        this->Amgxpgm_factory->generate(this->mtx)};
-
     using value_type = typename TestFixture::value_type;
+    std::unique_ptr<gko::multigrid::RestrictProlong> amgx_pgm{
+        this->amgxpgm_factory->generate(this->mtx)};
     auto x = gko::clone(this->fine_x);
+
     amgx_pgm->prolong_applyadd(this->coarse_b.get(), x.get());
     GKO_ASSERT_MTX_NEAR(x, this->prolong_ans,
                         gko::remove_complex<value_type>{0});
@@ -312,8 +449,8 @@ TYPED_TEST(AmgxPgm, CoarseFineProlongApplyadd)
 TYPED_TEST(AmgxPgm, ExtractDiag)
 {
     using value_type = typename TestFixture::value_type;
-
     gko::Array<value_type> diag(this->exec, 5);
+
     gko::kernels::reference::amgx_pgm::extract_diag(this->exec, this->mtx.get(),
                                                     diag);
     GKO_ASSERT_ARRAY_EQ(diag, this->mtx_diag);
@@ -331,8 +468,10 @@ TYPED_TEST(AmgxPgm, FindStrongestNeighbor)
         snb_vals[i] = -1;
         agg_vals[i] = -1;
     }
+
     gko::kernels::reference::amgx_pgm::find_strongest_neighbor(
         this->exec, this->mtx.get(), this->mtx_diag, agg, strongest_neighbor);
+
     ASSERT_EQ(snb_vals[0], 2);
     ASSERT_EQ(snb_vals[1], 0);
     ASSERT_EQ(snb_vals[2], 0);
@@ -353,8 +492,10 @@ TYPED_TEST(AmgxPgm, AssignToExistAgg)
     agg_vals[2] = 0;
     agg_vals[3] = 1;
     agg_vals[4] = -1;
+
     gko::kernels::reference::amgx_pgm::assign_to_exist_agg(
         this->exec, this->mtx.get(), this->mtx_diag, agg, intermediate_agg);
+
     ASSERT_EQ(agg_vals[0], 0);
     ASSERT_EQ(agg_vals[1], 1);
     ASSERT_EQ(agg_vals[2], 0);
@@ -377,14 +518,15 @@ TYPED_TEST(AmgxPgm, GenerateMtx)
     agg_vals[3] = 1;
     agg_vals[4] = 2;
     auto csr_coarse = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
+
     gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
         this->exec, this->mtx.get(), agg, csr_coarse.get());
 
-    ASSERT_EQ(csr_coarse->get_size(), gko::dim<2>(3, 3));
-    ASSERT_EQ(csr_coarse->get_num_stored_elements(), 9);
     auto r = csr_coarse->get_const_row_ptrs();
     auto c = csr_coarse->get_const_col_idxs();
     auto v = csr_coarse->get_const_values();
+    ASSERT_EQ(csr_coarse->get_size(), gko::dim<2>(3, 3));
+    ASSERT_EQ(csr_coarse->get_num_stored_elements(), 9);
     ASSERT_EQ(r[0], 0);
     ASSERT_EQ(r[1], 3);
     ASSERT_EQ(r[2], 6);
diff --git a/test_install/test_install.cpp b/test_install/test_install.cpp
index 927045bfd3f..e15518d5acc 100644
--- a/test_install/test_install.cpp
+++ b/test_install/test_install.cpp
@@ -258,6 +258,11 @@ int main(int, char **)
         auto test = Mtx::create(refExec, gko::dim<2>{2, 2});
     }
 
+    // core/multigrid/amgx_pgm.hpp
+    {
+        auto test = gko::multigrid::AmgxPgm<>::build().on(refExec);
+    }
+
     // core/preconditioner/ilu.hpp
     {
         auto test = gko::preconditioner::Ilu<>::build().on(refExec);
diff --git a/test_install/test_install_cuda.cu b/test_install/test_install_cuda.cu
index 6def39b0c04..5158e3f00fa 100644
--- a/test_install/test_install_cuda.cu
+++ b/test_install/test_install_cuda.cu
@@ -262,6 +262,11 @@ int main(int, char **)
         Mtx::create(cudaExec, gko::dim<2>{2, 2});
     }
 
+    // core/multigrid/amgx_pgm.hpp
+    {
+        gko::multigrid::AmgxPgm<>::build().on(cudaExec);
+    }
+
     // core/preconditioner/ilu.hpp
     {
         gko::preconditioner::Ilu<>::build().on(cudaExec);

From 5a493b9ca97372e664fd343fd997ec101866b1ec Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 6 Jul 2020 17:51:11 +0200
Subject: [PATCH 0465/1008] add csr absolute and store weight explicitly

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/device_hooks/common_kernels.inc.cpp      |  6 +-
 core/multigrid/amgx_pgm.cpp                   | 30 +++++---
 core/multigrid/amgx_pgm_kernels.hpp           | 14 ++--
 cuda/multigrid/amgx_pgm_kernels.cu            | 18 ++---
 hip/multigrid/amgx_pgm_kernels.hip.cpp        | 18 ++---
 include/ginkgo/core/base/types.hpp            | 29 +++++--
 include/ginkgo/core/matrix/csr.hpp            |  1 +
 omp/multigrid/amgx_pgm_kernels.cpp            | 18 ++---
 omp/test/matrix/csr_kernels.cpp               |  1 +
 reference/multigrid/amgx_pgm_kernels.cpp      | 58 ++++++--------
 reference/test/matrix/csr_kernels.cpp         |  2 +
 reference/test/multigrid/amgx_pgm_kernels.cpp | 76 ++++++++++++++++---
 12 files changed, 174 insertions(+), 97 deletions(-)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 52b08f98d98..a78d4879813 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -1242,19 +1242,19 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
 
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
 
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
 
 template <typename ValueType, typename IndexType>
diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index 4a2f9fdf56a..2cc81fe5b37 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/identity.hpp>
 
 
 #include "core/matrix/csr_builder.hpp"
@@ -92,23 +93,35 @@ template <typename ValueType, typename IndexType>
 void AmgxPgm<ValueType, IndexType>::generate()
 {
     using matrix_type = matrix::Csr<ValueType, IndexType>;
+    using rmc_value_type = remove_complex<ValueType>;
+    using weight_matrix_type = remove_complex<matrix_type>;
     auto exec = this->get_executor();
     const auto num = this->system_matrix_->get_size()[0];
-    Array<ValueType> diag(this->get_executor(), num);
+    Array<rmc_value_type> diag(this->get_executor(), num);
     Array<IndexType> strongest_neighbor(this->get_executor(), num);
     Array<IndexType> intermediate_agg(this->get_executor(),
                                       parameters_.deterministic * num);
     const auto amgxpgm_op = gko::as<matrix_type>(this->system_matrix_.get());
     // Initial agg = -1
     exec->run(amgx_pgm::make_initial(agg_));
-    // Extract the diagonal value of matrix
-    exec->run(amgx_pgm::make_extract_diag(amgxpgm_op, diag));
     size_type num_unagg{0};
     size_type num_unagg_prev{0};
+    // TODO: if mtx is a hermitian matrix, weight_mtx = abs(mtx)
+    // compute weight_mtx = (abs(mtx) + abs(mtx'))/2;
+    auto abs_mtx = amgxpgm_op->compute_absolute();
+    // abs_mtx is already real valuetype, so transpose is enough
+    auto weight_mtx = gko::as<weight_matrix_type>(abs_mtx->transpose());
+    auto half_scaler = initialize<matrix::Dense<rmc_value_type>>({0.5}, exec);
+    auto identity = matrix::Identity<rmc_value_type>::create(exec, num);
+    // W = (abs_mtx + transpose(abs_mtx))/2
+    abs_mtx->apply(lend(half_scaler), lend(identity), lend(half_scaler),
+                   lend(weight_mtx));
+    // Extract the diagonal value of matrix
+    exec->run(amgx_pgm::make_extract_diag(weight_mtx.get(), diag));
     for (int i = 0; i < parameters_.max_iterations; i++) {
         // Find the strongest neighbor of each row
-        exec->run(amgx_pgm::make_find_strongest_neighbor(amgxpgm_op, diag, agg_,
-                                                         strongest_neighbor));
+        exec->run(amgx_pgm::make_find_strongest_neighbor(
+            weight_mtx.get(), diag, agg_, strongest_neighbor));
         // Match edges
         exec->run(amgx_pgm::make_match_edge(strongest_neighbor, agg_));
         // Get the num_unagg
@@ -116,8 +129,7 @@ void AmgxPgm<ValueType, IndexType>::generate()
         // no new match, all match, or the ratio of num_unagg/num is lower
         // than parameter.max_unassigned_percentage
         if (num_unagg == 0 || num_unagg == num_unagg_prev ||
-            static_cast<double>(num_unagg) / num <
-                parameters_.max_unassigned_percentage) {
+            num_unagg < parameters_.max_unassigned_percentage * num) {
             break;
         }
         num_unagg_prev = num_unagg;
@@ -128,8 +140,8 @@ void AmgxPgm<ValueType, IndexType>::generate()
         intermediate_agg = agg_;
     }
     while (num_unagg != 0) {
-        exec->run(amgx_pgm::make_assign_to_exist_agg(amgxpgm_op, diag, agg_,
-                                                     intermediate_agg));
+        exec->run(amgx_pgm::make_assign_to_exist_agg(weight_mtx.get(), diag,
+                                                     agg_, intermediate_agg));
         exec->run(amgx_pgm::make_count_unagg(agg_, &num_unagg));
     }
     size_type num_agg;
diff --git a/core/multigrid/amgx_pgm_kernels.hpp b/core/multigrid/amgx_pgm_kernels.hpp
index 26031d570fd..b3ea2f3d4bc 100644
--- a/core/multigrid/amgx_pgm_kernels.hpp
+++ b/core/multigrid/amgx_pgm_kernels.hpp
@@ -85,16 +85,16 @@ namespace amgx_pgm {
 #define GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR(ValueType, IndexType) \
     void find_strongest_neighbor(                                          \
         std::shared_ptr<const DefaultExecutor> exec,                       \
-        const matrix::Csr<ValueType, IndexType> *source,                   \
+        const matrix::Csr<ValueType, IndexType> *weight_mtx,               \
         const Array<ValueType> &diag, Array<IndexType> &agg,               \
         Array<IndexType> &strongest_neighbor)
 
-#define GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG(ValueType, IndexType)        \
-    void assign_to_exist_agg(std::shared_ptr<const DefaultExecutor> exec,     \
-                             const matrix::Csr<ValueType, IndexType> *source, \
-                             const Array<ValueType> &diag,                    \
-                             Array<IndexType> &agg,                           \
-                             Array<IndexType> &intermediate_agg)
+#define GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG(ValueType, IndexType) \
+    void assign_to_exist_agg(                                          \
+        std::shared_ptr<const DefaultExecutor> exec,                   \
+        const matrix::Csr<ValueType, IndexType> *weight_mtx,           \
+        const Array<ValueType> &diag, Array<IndexType> &agg,           \
+        Array<IndexType> &intermediate_agg)
 
 #define GKO_DECLARE_AMGX_PGM_GENERATE(ValueType, IndexType)                 \
     void amgx_pgm_generate(std::shared_ptr<const DefaultExecutor> exec,     \
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index 310883ecae3..bec3b7aa840 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -120,34 +120,34 @@ void extract_diag(std::shared_ptr<const CudaExecutor> exec,
     GKO_NOT_IMPLEMENTED;
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
 
 
 template <typename ValueType, typename IndexType>
-void find_strongest_neighbor(std::shared_ptr<const CudaExecutor> exec,
-                             const matrix::Csr<ValueType, IndexType> *source,
-                             const Array<ValueType> &diag,
-                             Array<IndexType> &agg,
-                             Array<IndexType> &strongest_neighbor)
+void find_strongest_neighbor(
+    std::shared_ptr<const CudaExecutor> exec,
+    const matrix::Csr<ValueType, IndexType> *weight_mtx,
+    const Array<ValueType> &diag, Array<IndexType> &agg,
+    Array<IndexType> &strongest_neighbor)
 {
     GKO_NOT_IMPLEMENTED;
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
 
 
 template <typename ValueType, typename IndexType>
 void assign_to_exist_agg(std::shared_ptr<const CudaExecutor> exec,
-                         const matrix::Csr<ValueType, IndexType> *source,
+                         const matrix::Csr<ValueType, IndexType> *weight_mtx,
                          const Array<ValueType> &diag, Array<IndexType> &agg,
                          Array<IndexType> &intermediate_agg)
 {
     GKO_NOT_IMPLEMENTED;
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
 
 
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index 0ff2784258b..24636180253 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -120,34 +120,34 @@ void extract_diag(std::shared_ptr<const HipExecutor> exec,
     GKO_NOT_IMPLEMENTED;
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
 
 
 template <typename ValueType, typename IndexType>
-void find_strongest_neighbor(std::shared_ptr<const HipExecutor> exec,
-                             const matrix::Csr<ValueType, IndexType> *source,
-                             const Array<ValueType> &diag,
-                             Array<IndexType> &agg,
-                             Array<IndexType> &strongest_neighbor)
+void find_strongest_neighbor(
+    std::shared_ptr<const HipExecutor> exec,
+    const matrix::Csr<ValueType, IndexType> *weight_mtx,
+    const Array<ValueType> &diag, Array<IndexType> &agg,
+    Array<IndexType> &strongest_neighbor)
 {
     GKO_NOT_IMPLEMENTED;
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
 
 
 template <typename ValueType, typename IndexType>
 void assign_to_exist_agg(std::shared_ptr<const HipExecutor> exec,
-                         const matrix::Csr<ValueType, IndexType> *source,
+                         const matrix::Csr<ValueType, IndexType> *weight_mtx,
                          const Array<ValueType> &diag, Array<IndexType> &agg,
                          Array<IndexType> &intermediate_agg)
 {
     GKO_NOT_IMPLEMENTED;
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
 
 
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index e1d1d86cfe5..d42325f2118 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -456,6 +456,22 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
     template _macro(int64)
 
 
+/**
+ * Instantiates a template for each non-complex value and index type compiled by
+ * Ginkgo.
+ *
+ * @param _macro  A macro which expands the template instantiation
+ *                (not including the leading `template` specifier).
+ *                Should take two arguments, which are replaced by the
+ *                value and index types.
+ */
+#define GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(_macro) \
+    template _macro(float, int32);                                        \
+    template _macro(double, int32);                                       \
+    template _macro(float, int64);                                        \
+    template _macro(double, int64)
+
+
 /**
  * Instantiates a template for each value and index type compiled by Ginkgo.
  *
@@ -464,14 +480,11 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
  *                Should take two arguments, which are replaced by the
  *                value and index types.
  */
-#define GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(_macro) \
-    template _macro(float, int32);                            \
-    template _macro(double, int32);                           \
-    template _macro(std::complex<float>, int32);              \
-    template _macro(std::complex<double>, int32);             \
-    template _macro(float, int64);                            \
-    template _macro(double, int64);                           \
-    template _macro(std::complex<float>, int64);              \
+#define GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(_macro)          \
+    GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(_macro); \
+    template _macro(std::complex<float>, int32);                       \
+    template _macro(std::complex<double>, int32);                      \
+    template _macro(std::complex<float>, int64);                       \
     template _macro(std::complex<double>, int64)
 
 
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index 7e01c1293f6..eded7da0434 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -655,6 +655,7 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
             detail::strategy_rebuild_helper(result);
         }
     }
+
     friend class Csr<next_precision<ValueType>, IndexType>;
 
     void convert_to(
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index 982260d193b..502531354ad 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -117,34 +117,34 @@ void extract_diag(std::shared_ptr<const OmpExecutor> exec,
     GKO_NOT_IMPLEMENTED;
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
 
 
 template <typename ValueType, typename IndexType>
-void find_strongest_neighbor(std::shared_ptr<const OmpExecutor> exec,
-                             const matrix::Csr<ValueType, IndexType> *source,
-                             const Array<ValueType> &diag,
-                             Array<IndexType> &agg,
-                             Array<IndexType> &strongest_neighbor)
+void find_strongest_neighbor(
+    std::shared_ptr<const OmpExecutor> exec,
+    const matrix::Csr<ValueType, IndexType> *weight_mtx,
+    const Array<ValueType> &diag, Array<IndexType> &agg,
+    Array<IndexType> &strongest_neighbor)
 {
     GKO_NOT_IMPLEMENTED;
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
 
 
 template <typename ValueType, typename IndexType>
 void assign_to_exist_agg(std::shared_ptr<const OmpExecutor> exec,
-                         const matrix::Csr<ValueType, IndexType> *source,
+                         const matrix::Csr<ValueType, IndexType> *weight_mtx,
                          const Array<ValueType> &diag, Array<IndexType> &agg,
                          Array<IndexType> &intermediate_agg)
 {
     GKO_NOT_IMPLEMENTED;
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
 
 
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index a27f6638a04..6e505e70f0c 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -63,6 +63,7 @@ class Csr : public ::testing::Test {
 protected:
     using Arr = gko::Array<int>;
     using Mtx = gko::matrix::Csr<>;
+    using AbsMtx = gko::matrix::Csr<>;
     using Vec = gko::matrix::Dense<>;
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
     using ComplexMtx = gko::matrix::Csr<std::complex<double>>;
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index 053cb46ec45..6cecc36a076 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -199,43 +199,33 @@ void extract_diag(std::shared_ptr<const ReferenceExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
 
 
 template <typename ValueType, typename IndexType>
-void find_strongest_neighbor(std::shared_ptr<const ReferenceExecutor> exec,
-                             const matrix::Csr<ValueType, IndexType> *source,
-                             const Array<ValueType> &diag,
-                             Array<IndexType> &agg,
-                             Array<IndexType> &strongest_neighbor)
+void find_strongest_neighbor(
+    std::shared_ptr<const ReferenceExecutor> exec,
+    const matrix::Csr<ValueType, IndexType> *weight_mtx,
+    const Array<ValueType> &diag, Array<IndexType> &agg,
+    Array<IndexType> &strongest_neighbor)
 {
-    const auto row_ptrs = source->get_const_row_ptrs();
-    const auto col_idxs = source->get_const_col_idxs();
-    const auto vals = source->get_const_values();
+    const auto row_ptrs = weight_mtx->get_const_row_ptrs();
+    const auto col_idxs = weight_mtx->get_const_col_idxs();
+    const auto vals = weight_mtx->get_const_values();
     for (size_type row = 0; row < agg.get_num_elems(); row++) {
-        auto max_weight_unagg = zero<remove_complex<ValueType>>();
-        auto max_weight_agg = zero<remove_complex<ValueType>>();
+        auto max_weight_unagg = zero<ValueType>();
+        auto max_weight_agg = zero<ValueType>();
         IndexType strongest_unagg = -1;
         IndexType strongest_agg = -1;
         if (agg.get_const_data()[row] == -1) {
-            for (auto ind = row_ptrs[row]; ind < row_ptrs[row + 1]; ind++) {
-                auto weight = zero<remove_complex<ValueType>>();
-                auto col = col_idxs[ind];
+            for (auto idx = row_ptrs[row]; idx < row_ptrs[row + 1]; idx++) {
+                auto col = col_idxs[idx];
                 if (col == row) {
                     continue;
                 }
-                // Handle the unsymmetric values cases
-                for (auto trans_ind = row_ptrs[col];
-                     trans_ind < row_ptrs[col + 1]; trans_ind++) {
-                    if (col_idxs[trans_ind] == row) {
-                        weight = 0.5 * (abs(vals[ind]) + abs(vals[trans_ind])) /
-                                 max(abs(diag.get_const_data()[row]),
-                                     abs(diag.get_const_data()[col]));
-                        break;
-                    }
-                }
-
+                auto weight = vals[idx] / max(abs(diag.get_const_data()[row]),
+                                              abs(diag.get_const_data()[col]));
                 if (agg.get_const_data()[col] == -1 &&
                     (weight > max_weight_unagg ||
                      (weight == max_weight_unagg && col > strongest_unagg))) {
@@ -264,21 +254,21 @@ void find_strongest_neighbor(std::shared_ptr<const ReferenceExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
 
 
 template <typename ValueType, typename IndexType>
 void assign_to_exist_agg(std::shared_ptr<const ReferenceExecutor> exec,
-                         const matrix::Csr<ValueType, IndexType> *source,
+                         const matrix::Csr<ValueType, IndexType> *weight_mtx,
                          const Array<ValueType> &diag, Array<IndexType> &agg,
                          Array<IndexType> &intermediate_agg)
 {
-    const auto row_ptrs = source->get_const_row_ptrs();
-    const auto col_idxs = source->get_const_col_idxs();
-    const auto vals = source->get_const_values();
+    const auto row_ptrs = weight_mtx->get_const_row_ptrs();
+    const auto col_idxs = weight_mtx->get_const_col_idxs();
+    const auto vals = weight_mtx->get_const_values();
     const auto diag_vals = diag.get_const_data();
-    auto max_weight_agg = zero<remove_complex<ValueType>>();
+    auto max_weight_agg = zero<ValueType>();
     const auto agg_const_val = agg.get_const_data();
     auto agg_val = (intermediate_agg.get_num_elems() > 0)
                        ? intermediate_agg.get_data()
@@ -294,8 +284,8 @@ void assign_to_exist_agg(std::shared_ptr<const ReferenceExecutor> exec,
             if (col == row) {
                 break;
             }
-            auto weight =
-                abs(vals[idx]) / max(abs(diag_vals[row]), abs(diag_vals[col]));
+            auto weight = vals[idx] / max(abs(diag.get_const_data()[row]),
+                                          abs(diag.get_const_data()[col]));
             if (agg_const_val[col] != -1 &&
                 (weight > max_weight_agg ||
                  (weight == max_weight_agg && col > strongest_agg))) {
@@ -316,7 +306,7 @@ void assign_to_exist_agg(std::shared_ptr<const ReferenceExecutor> exec,
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
 
 
diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp
index 186aa700598..3c53f5509e8 100644
--- a/reference/test/matrix/csr_kernels.cpp
+++ b/reference/test/matrix/csr_kernels.cpp
@@ -69,6 +69,8 @@ class Csr : public ::testing::Test {
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using Coo = gko::matrix::Coo<value_type, index_type>;
     using Mtx = gko::matrix::Csr<value_type, index_type>;
+    using AbsMtx =
+        gko::matrix::Csr<gko::remove_complex<value_type>, index_type>;
     using Sellp = gko::matrix::Sellp<value_type, index_type>;
     using SparsityCsr = gko::matrix::SparsityCsr<value_type, index_type>;
     using Ell = gko::matrix::Ell<value_type, index_type>;
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index bbbdd5b1968..76911118c67 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -68,6 +68,8 @@ class AmgxPgm : public ::testing::Test {
     using Vec = gko::matrix::Dense<value_type>;
     using RestrictProlong = gko::multigrid::AmgxPgm<value_type, index_type>;
     using T = value_type;
+    using rmc_value_type = gko::remove_complex<value_type>;
+    using WeightMtx = gko::matrix::Csr<rmc_value_type, index_type>;
     AmgxPgm()
         : exec(gko::ReferenceExecutor::create()),
           amgxpgm_factory(RestrictProlong::build()
@@ -92,21 +94,29 @@ class AmgxPgm : public ::testing::Test {
               exec)),
           mtx(Mtx::create(exec, gko::dim<2>(5, 5), 15,
                           std::make_shared<typename Mtx::classical>())),
+          weight(WeightMtx::create(
+              exec, gko::dim<2>(5, 5), 15,
+              std::make_shared<typename WeightMtx::classical>())),
           coarse(Mtx::create(exec, gko::dim<2>(2, 2), 4,
                              std::make_shared<typename Mtx::classical>())),
           mtx_diag(exec, 5),
           agg(exec, 5)
     {
-        this->create_mtx(mtx.get(), &mtx_diag, &agg, coarse.get());
+        this->create_mtx(mtx.get(), weight.get(), &mtx_diag, &agg,
+                         coarse.get());
         rstr_prlg = amgxpgm_factory->generate(mtx);
     }
 
-    void create_mtx(Mtx *fine, gko::Array<value_type> *diag,
+    void create_mtx(Mtx *fine, WeightMtx *weight,
+                    gko::Array<rmc_value_type> *diag,
                     gko::Array<index_type> *agg, Mtx *coarse)
     {
         auto vals = fine->get_values();
         auto cols = fine->get_col_idxs();
         auto rows = fine->get_row_ptrs();
+        auto w_vals = weight->get_values();
+        auto w_cols = weight->get_col_idxs();
+        auto w_rows = weight->get_row_ptrs();
         auto diag_val = diag->get_data();
         auto agg_val = agg->get_data();
         auto c_vals = coarse->get_values();
@@ -158,6 +168,52 @@ class AmgxPgm : public ::testing::Test {
         cols[13] = 2;
         cols[14] = 4;
 
+        /* weight matrix is stored:
+         * 5   3   3   0   0
+         * 3   5   0   2.5 1.5
+         * 3   0   5   0   1.5
+         * 0   2.5 0   5   0
+         * 0   1.5 1.5 0   5
+         */
+        w_vals[0] = 5;
+        w_vals[1] = 3;
+        w_vals[2] = 3;
+        w_vals[3] = 3;
+        w_vals[4] = 5;
+        w_vals[5] = 2.5;
+        w_vals[6] = 1.5;
+        w_vals[7] = 3;
+        w_vals[8] = 5;
+        w_vals[9] = 1.5;
+        w_vals[10] = 2.5;
+        w_vals[11] = 5;
+        w_vals[12] = 1.5;
+        w_vals[13] = 1.5;
+        w_vals[14] = 5;
+
+        w_rows[0] = 0;
+        w_rows[1] = 3;
+        w_rows[2] = 7;
+        w_rows[3] = 10;
+        w_rows[4] = 12;
+        w_rows[5] = 15;
+
+        w_cols[0] = 0;
+        w_cols[1] = 1;
+        w_cols[2] = 2;
+        w_cols[3] = 0;
+        w_cols[4] = 1;
+        w_cols[5] = 3;
+        w_cols[6] = 4;
+        w_cols[7] = 0;
+        w_cols[8] = 2;
+        w_cols[9] = 4;
+        w_cols[10] = 1;
+        w_cols[11] = 3;
+        w_cols[12] = 1;
+        w_cols[13] = 2;
+        w_cols[14] = 4;
+
         diag_val[0] = 5;
         diag_val[1] = 5;
         diag_val[2] = 5;
@@ -214,7 +270,8 @@ class AmgxPgm : public ::testing::Test {
     std::shared_ptr<const gko::ReferenceExecutor> exec;
     std::shared_ptr<Mtx> mtx;
     std::shared_ptr<Mtx> coarse;
-    gko::Array<value_type> mtx_diag;
+    std::shared_ptr<WeightMtx> weight;
+    gko::Array<rmc_value_type> mtx_diag;
     gko::Array<index_type> agg;
     std::shared_ptr<Vec> coarse_b;
     std::shared_ptr<Vec> fine_b;
@@ -448,11 +505,11 @@ TYPED_TEST(AmgxPgm, CoarseFineProlongApplyadd)
 
 TYPED_TEST(AmgxPgm, ExtractDiag)
 {
-    using value_type = typename TestFixture::value_type;
-    gko::Array<value_type> diag(this->exec, 5);
+    using rmc_value_type = typename TestFixture::rmc_value_type;
+    gko::Array<rmc_value_type> diag(this->exec, 5);
 
-    gko::kernels::reference::amgx_pgm::extract_diag(this->exec, this->mtx.get(),
-                                                    diag);
+    gko::kernels::reference::amgx_pgm::extract_diag(this->exec,
+                                                    this->weight.get(), diag);
     GKO_ASSERT_ARRAY_EQ(diag, this->mtx_diag);
 }
 
@@ -470,7 +527,8 @@ TYPED_TEST(AmgxPgm, FindStrongestNeighbor)
     }
 
     gko::kernels::reference::amgx_pgm::find_strongest_neighbor(
-        this->exec, this->mtx.get(), this->mtx_diag, agg, strongest_neighbor);
+        this->exec, this->weight.get(), this->mtx_diag, agg,
+        strongest_neighbor);
 
     ASSERT_EQ(snb_vals[0], 2);
     ASSERT_EQ(snb_vals[1], 0);
@@ -494,7 +552,7 @@ TYPED_TEST(AmgxPgm, AssignToExistAgg)
     agg_vals[4] = -1;
 
     gko::kernels::reference::amgx_pgm::assign_to_exist_agg(
-        this->exec, this->mtx.get(), this->mtx_diag, agg, intermediate_agg);
+        this->exec, this->weight.get(), this->mtx_diag, agg, intermediate_agg);
 
     ASSERT_EQ(agg_vals[0], 0);
     ASSERT_EQ(agg_vals[1], 1);

From 65ef0d17d4ae2443f0eaeb38806a3a9a1c48a651 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 6 Jul 2020 21:41:32 +0200
Subject: [PATCH 0466/1008] replace extract_diagonal

---
 core/device_hooks/common_kernels.inc.cpp      |  6 ---
 core/multigrid/amgx_pgm.cpp                   | 10 ++---
 core/multigrid/amgx_pgm_kernels.hpp           | 20 ++++-----
 cuda/multigrid/amgx_pgm_kernels.cu            | 17 ++------
 hip/multigrid/amgx_pgm_kernels.hip.cpp        | 17 ++------
 omp/multigrid/amgx_pgm_kernels.cpp            | 17 ++------
 reference/multigrid/amgx_pgm_kernels.cpp      | 42 +++++--------------
 reference/test/multigrid/amgx_pgm_kernels.cpp | 36 ++++------------
 8 files changed, 39 insertions(+), 126 deletions(-)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index a78d4879813..79738480867 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -1239,12 +1239,6 @@ GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL(IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
 
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
-
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index 2cc81fe5b37..788b1a25fb4 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -59,7 +59,6 @@ GKO_REGISTER_OPERATION(initial, amgx_pgm::initial);
 GKO_REGISTER_OPERATION(match_edge, amgx_pgm::match_edge);
 GKO_REGISTER_OPERATION(count_unagg, amgx_pgm::count_unagg);
 GKO_REGISTER_OPERATION(renumber, amgx_pgm::renumber);
-GKO_REGISTER_OPERATION(extract_diag, amgx_pgm::extract_diag);
 GKO_REGISTER_OPERATION(find_strongest_neighbor,
                        amgx_pgm::find_strongest_neighbor);
 GKO_REGISTER_OPERATION(assign_to_exist_agg, amgx_pgm::assign_to_exist_agg);
@@ -97,7 +96,6 @@ void AmgxPgm<ValueType, IndexType>::generate()
     using weight_matrix_type = remove_complex<matrix_type>;
     auto exec = this->get_executor();
     const auto num = this->system_matrix_->get_size()[0];
-    Array<rmc_value_type> diag(this->get_executor(), num);
     Array<IndexType> strongest_neighbor(this->get_executor(), num);
     Array<IndexType> intermediate_agg(this->get_executor(),
                                       parameters_.deterministic * num);
@@ -117,11 +115,11 @@ void AmgxPgm<ValueType, IndexType>::generate()
     abs_mtx->apply(lend(half_scaler), lend(identity), lend(half_scaler),
                    lend(weight_mtx));
     // Extract the diagonal value of matrix
-    exec->run(amgx_pgm::make_extract_diag(weight_mtx.get(), diag));
+    auto diag = weight_mtx->extract_diagonal();
     for (int i = 0; i < parameters_.max_iterations; i++) {
         // Find the strongest neighbor of each row
         exec->run(amgx_pgm::make_find_strongest_neighbor(
-            weight_mtx.get(), diag, agg_, strongest_neighbor));
+            weight_mtx.get(), diag.get(), agg_, strongest_neighbor));
         // Match edges
         exec->run(amgx_pgm::make_match_edge(strongest_neighbor, agg_));
         // Get the num_unagg
@@ -140,8 +138,8 @@ void AmgxPgm<ValueType, IndexType>::generate()
         intermediate_agg = agg_;
     }
     while (num_unagg != 0) {
-        exec->run(amgx_pgm::make_assign_to_exist_agg(weight_mtx.get(), diag,
-                                                     agg_, intermediate_agg));
+        exec->run(amgx_pgm::make_assign_to_exist_agg(
+            weight_mtx.get(), diag.get(), agg_, intermediate_agg));
         exec->run(amgx_pgm::make_count_unagg(agg_, &num_unagg));
     }
     size_type num_agg;
diff --git a/core/multigrid/amgx_pgm_kernels.hpp b/core/multigrid/amgx_pgm_kernels.hpp
index b3ea2f3d4bc..c3e2b46f8a7 100644
--- a/core/multigrid/amgx_pgm_kernels.hpp
+++ b/core/multigrid/amgx_pgm_kernels.hpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
 
 
 namespace gko {
@@ -77,23 +78,18 @@ namespace amgx_pgm {
     void renumber(std::shared_ptr<const DefaultExecutor> exec, \
                   Array<_itype> &agg, size_type *num_agg)
 
-#define GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG(ValueType, IndexType)        \
-    void extract_diag(std::shared_ptr<const DefaultExecutor> exec,     \
-                      const matrix::Csr<ValueType, IndexType> *source, \
-                      Array<ValueType> &diag)
-
 #define GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR(ValueType, IndexType) \
     void find_strongest_neighbor(                                          \
         std::shared_ptr<const DefaultExecutor> exec,                       \
         const matrix::Csr<ValueType, IndexType> *weight_mtx,               \
-        const Array<ValueType> &diag, Array<IndexType> &agg,               \
+        const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,    \
         Array<IndexType> &strongest_neighbor)
 
-#define GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG(ValueType, IndexType) \
-    void assign_to_exist_agg(                                          \
-        std::shared_ptr<const DefaultExecutor> exec,                   \
-        const matrix::Csr<ValueType, IndexType> *weight_mtx,           \
-        const Array<ValueType> &diag, Array<IndexType> &agg,           \
+#define GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG(ValueType, IndexType)  \
+    void assign_to_exist_agg(                                           \
+        std::shared_ptr<const DefaultExecutor> exec,                    \
+        const matrix::Csr<ValueType, IndexType> *weight_mtx,            \
+        const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg, \
         Array<IndexType> &intermediate_agg)
 
 #define GKO_DECLARE_AMGX_PGM_GENERATE(ValueType, IndexType)                 \
@@ -116,8 +112,6 @@ namespace amgx_pgm {
     template <typename IndexType>                                       \
     GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL(IndexType);                    \
     template <typename ValueType, typename IndexType>                   \
-    GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG(ValueType, IndexType);            \
-    template <typename ValueType, typename IndexType>                   \
     GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR(ValueType, IndexType); \
     template <typename ValueType, typename IndexType>                   \
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG(ValueType, IndexType);     \
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index bec3b7aa840..cebee78d0a3 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -112,23 +112,11 @@ void renumber(std::shared_ptr<const CudaExecutor> exec, Array<IndexType> &agg,
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void extract_diag(std::shared_ptr<const CudaExecutor> exec,
-                  const matrix::Csr<ValueType, IndexType> *source,
-                  Array<ValueType> &diag)
-{
-    GKO_NOT_IMPLEMENTED;
-}
-
-GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
-
-
 template <typename ValueType, typename IndexType>
 void find_strongest_neighbor(
     std::shared_ptr<const CudaExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *weight_mtx,
-    const Array<ValueType> &diag, Array<IndexType> &agg,
+    const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
     Array<IndexType> &strongest_neighbor)
 {
     GKO_NOT_IMPLEMENTED;
@@ -141,7 +129,8 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void assign_to_exist_agg(std::shared_ptr<const CudaExecutor> exec,
                          const matrix::Csr<ValueType, IndexType> *weight_mtx,
-                         const Array<ValueType> &diag, Array<IndexType> &agg,
+                         const matrix::Diagonal<ValueType> *diag,
+                         Array<IndexType> &agg,
                          Array<IndexType> &intermediate_agg)
 {
     GKO_NOT_IMPLEMENTED;
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index 24636180253..7104cc57984 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -112,23 +112,11 @@ void renumber(std::shared_ptr<const HipExecutor> exec, Array<IndexType> &agg,
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void extract_diag(std::shared_ptr<const HipExecutor> exec,
-                  const matrix::Csr<ValueType, IndexType> *source,
-                  Array<ValueType> &diag)
-{
-    GKO_NOT_IMPLEMENTED;
-}
-
-GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
-
-
 template <typename ValueType, typename IndexType>
 void find_strongest_neighbor(
     std::shared_ptr<const HipExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *weight_mtx,
-    const Array<ValueType> &diag, Array<IndexType> &agg,
+    const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
     Array<IndexType> &strongest_neighbor)
 {
     GKO_NOT_IMPLEMENTED;
@@ -141,7 +129,8 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void assign_to_exist_agg(std::shared_ptr<const HipExecutor> exec,
                          const matrix::Csr<ValueType, IndexType> *weight_mtx,
-                         const Array<ValueType> &diag, Array<IndexType> &agg,
+                         const matrix::Diagonal<ValueType> *diag,
+                         Array<IndexType> &agg,
                          Array<IndexType> &intermediate_agg)
 {
     GKO_NOT_IMPLEMENTED;
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index 502531354ad..5afa2a7f150 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -109,23 +109,11 @@ void renumber(std::shared_ptr<const OmpExecutor> exec, Array<IndexType> &agg,
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void extract_diag(std::shared_ptr<const OmpExecutor> exec,
-                  const matrix::Csr<ValueType, IndexType> *source,
-                  Array<ValueType> &diag)
-{
-    GKO_NOT_IMPLEMENTED;
-}
-
-GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
-
-
 template <typename ValueType, typename IndexType>
 void find_strongest_neighbor(
     std::shared_ptr<const OmpExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *weight_mtx,
-    const Array<ValueType> &diag, Array<IndexType> &agg,
+    const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
     Array<IndexType> &strongest_neighbor)
 {
     GKO_NOT_IMPLEMENTED;
@@ -138,7 +126,8 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void assign_to_exist_agg(std::shared_ptr<const OmpExecutor> exec,
                          const matrix::Csr<ValueType, IndexType> *weight_mtx,
-                         const Array<ValueType> &diag, Array<IndexType> &agg,
+                         const matrix::Diagonal<ValueType> *diag,
+                         Array<IndexType> &agg,
                          Array<IndexType> &intermediate_agg)
 {
     GKO_NOT_IMPLEMENTED;
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index 6cecc36a076..c6b3b2b2531 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
 #include <ginkgo/core/multigrid/amgx_pgm.hpp>
 
 
@@ -179,40 +180,17 @@ void renumber(std::shared_ptr<const ReferenceExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void extract_diag(std::shared_ptr<const ReferenceExecutor> exec,
-                  const matrix::Csr<ValueType, IndexType> *source,
-                  Array<ValueType> &diag)
-{
-    const auto dim_size = source->get_size();
-    const auto row_ptrs = source->get_const_row_ptrs();
-    const auto col_idxs = source->get_const_col_idxs();
-    const auto vals = source->get_const_values();
-    auto diag_val = diag.get_data();
-    for (IndexType i = 0; i < dim_size[0]; i++) {
-        for (auto j = row_ptrs[i]; j < row_ptrs[i + 1]; j++) {
-            if (col_idxs[j] == i) {
-                diag_val[i] = vals[j];
-                break;
-            }
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_EXTRACT_DIAG);
-
-
 template <typename ValueType, typename IndexType>
 void find_strongest_neighbor(
     std::shared_ptr<const ReferenceExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *weight_mtx,
-    const Array<ValueType> &diag, Array<IndexType> &agg,
+    const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
     Array<IndexType> &strongest_neighbor)
 {
     const auto row_ptrs = weight_mtx->get_const_row_ptrs();
     const auto col_idxs = weight_mtx->get_const_col_idxs();
     const auto vals = weight_mtx->get_const_values();
+    const auto diag_vals = diag->get_const_values();
     for (size_type row = 0; row < agg.get_num_elems(); row++) {
         auto max_weight_unagg = zero<ValueType>();
         auto max_weight_agg = zero<ValueType>();
@@ -224,8 +202,8 @@ void find_strongest_neighbor(
                 if (col == row) {
                     continue;
                 }
-                auto weight = vals[idx] / max(abs(diag.get_const_data()[row]),
-                                              abs(diag.get_const_data()[col]));
+                auto weight =
+                    vals[idx] / max(abs(diag_vals[row]), abs(diag_vals[col]));
                 if (agg.get_const_data()[col] == -1 &&
                     (weight > max_weight_unagg ||
                      (weight == max_weight_unagg && col > strongest_unagg))) {
@@ -261,19 +239,19 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void assign_to_exist_agg(std::shared_ptr<const ReferenceExecutor> exec,
                          const matrix::Csr<ValueType, IndexType> *weight_mtx,
-                         const Array<ValueType> &diag, Array<IndexType> &agg,
+                         const matrix::Diagonal<ValueType> *diag,
+                         Array<IndexType> &agg,
                          Array<IndexType> &intermediate_agg)
 {
     const auto row_ptrs = weight_mtx->get_const_row_ptrs();
     const auto col_idxs = weight_mtx->get_const_col_idxs();
     const auto vals = weight_mtx->get_const_values();
-    const auto diag_vals = diag.get_const_data();
     auto max_weight_agg = zero<ValueType>();
     const auto agg_const_val = agg.get_const_data();
     auto agg_val = (intermediate_agg.get_num_elems() > 0)
                        ? intermediate_agg.get_data()
                        : agg.get_data();
-
+    const auto diag_vals = diag->get_const_values();
     for (IndexType row = 0; row < agg.get_num_elems(); row++) {
         if (agg_const_val[row] != -1) {
             continue;
@@ -284,8 +262,8 @@ void assign_to_exist_agg(std::shared_ptr<const ReferenceExecutor> exec,
             if (col == row) {
                 break;
             }
-            auto weight = vals[idx] / max(abs(diag.get_const_data()[row]),
-                                          abs(diag.get_const_data()[col]));
+            auto weight =
+                vals[idx] / max(abs(diag_vals[row]), abs(diag_vals[col]));
             if (agg_const_val[col] != -1 &&
                 (weight > max_weight_agg ||
                  (weight == max_weight_agg && col > strongest_agg))) {
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index 76911118c67..797c9bd33f8 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm_reduction.hpp>
@@ -99,17 +100,15 @@ class AmgxPgm : public ::testing::Test {
               std::make_shared<typename WeightMtx::classical>())),
           coarse(Mtx::create(exec, gko::dim<2>(2, 2), 4,
                              std::make_shared<typename Mtx::classical>())),
-          mtx_diag(exec, 5),
           agg(exec, 5)
     {
-        this->create_mtx(mtx.get(), weight.get(), &mtx_diag, &agg,
-                         coarse.get());
+        this->create_mtx(mtx.get(), weight.get(), &agg, coarse.get());
         rstr_prlg = amgxpgm_factory->generate(mtx);
+        mtx_diag = weight->extract_diagonal();
     }
 
-    void create_mtx(Mtx *fine, WeightMtx *weight,
-                    gko::Array<rmc_value_type> *diag,
-                    gko::Array<index_type> *agg, Mtx *coarse)
+    void create_mtx(Mtx *fine, WeightMtx *weight, gko::Array<index_type> *agg,
+                    Mtx *coarse)
     {
         auto vals = fine->get_values();
         auto cols = fine->get_col_idxs();
@@ -117,7 +116,6 @@ class AmgxPgm : public ::testing::Test {
         auto w_vals = weight->get_values();
         auto w_cols = weight->get_col_idxs();
         auto w_rows = weight->get_row_ptrs();
-        auto diag_val = diag->get_data();
         auto agg_val = agg->get_data();
         auto c_vals = coarse->get_values();
         auto c_cols = coarse->get_col_idxs();
@@ -214,12 +212,6 @@ class AmgxPgm : public ::testing::Test {
         w_cols[13] = 2;
         w_cols[14] = 4;
 
-        diag_val[0] = 5;
-        diag_val[1] = 5;
-        diag_val[2] = 5;
-        diag_val[3] = 5;
-        diag_val[4] = 5;
-
         agg_val[0] = 0;
         agg_val[1] = 1;
         agg_val[2] = 0;
@@ -271,7 +263,7 @@ class AmgxPgm : public ::testing::Test {
     std::shared_ptr<Mtx> mtx;
     std::shared_ptr<Mtx> coarse;
     std::shared_ptr<WeightMtx> weight;
-    gko::Array<rmc_value_type> mtx_diag;
+    std::shared_ptr<gko::matrix::Diagonal<rmc_value_type>> mtx_diag;
     gko::Array<index_type> agg;
     std::shared_ptr<Vec> coarse_b;
     std::shared_ptr<Vec> fine_b;
@@ -503,17 +495,6 @@ TYPED_TEST(AmgxPgm, CoarseFineProlongApplyadd)
 }
 
 
-TYPED_TEST(AmgxPgm, ExtractDiag)
-{
-    using rmc_value_type = typename TestFixture::rmc_value_type;
-    gko::Array<rmc_value_type> diag(this->exec, 5);
-
-    gko::kernels::reference::amgx_pgm::extract_diag(this->exec,
-                                                    this->weight.get(), diag);
-    GKO_ASSERT_ARRAY_EQ(diag, this->mtx_diag);
-}
-
-
 TYPED_TEST(AmgxPgm, FindStrongestNeighbor)
 {
     using index_type = typename TestFixture::index_type;
@@ -527,7 +508,7 @@ TYPED_TEST(AmgxPgm, FindStrongestNeighbor)
     }
 
     gko::kernels::reference::amgx_pgm::find_strongest_neighbor(
-        this->exec, this->weight.get(), this->mtx_diag, agg,
+        this->exec, this->weight.get(), this->mtx_diag.get(), agg,
         strongest_neighbor);
 
     ASSERT_EQ(snb_vals[0], 2);
@@ -552,7 +533,8 @@ TYPED_TEST(AmgxPgm, AssignToExistAgg)
     agg_vals[4] = -1;
 
     gko::kernels::reference::amgx_pgm::assign_to_exist_agg(
-        this->exec, this->weight.get(), this->mtx_diag, agg, intermediate_agg);
+        this->exec, this->weight.get(), this->mtx_diag.get(), agg,
+        intermediate_agg);
 
     ASSERT_EQ(agg_vals[0], 0);
     ASSERT_EQ(agg_vals[1], 1);

From bb58b5a132d48475f1013428d253b941fe48e0e1 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 8 Jul 2020 12:04:59 +0200
Subject: [PATCH 0467/1008] fix control flow and use fill_array

---
 core/device_hooks/common_kernels.inc.cpp |  5 -----
 core/multigrid/amgx_pgm.cpp              |  6 ++++--
 core/multigrid/amgx_pgm_kernels.hpp      |  6 ------
 cuda/multigrid/amgx_pgm_kernels.cu       |  7 -------
 hip/multigrid/amgx_pgm_kernels.hip.cpp   |  7 -------
 omp/multigrid/amgx_pgm_kernels.cpp       |  7 -------
 reference/multigrid/amgx_pgm_kernels.cpp | 14 +-------------
 7 files changed, 5 insertions(+), 47 deletions(-)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 79738480867..bafc8e07ad8 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -1219,11 +1219,6 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
 
-template <typename IndexType>
-GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL(IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL);
-
 template <typename IndexType>
 GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL(IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index 788b1a25fb4..b171fa174ef 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/identity.hpp>
 
 
+#include "core/components/fill_array.hpp"
 #include "core/matrix/csr_builder.hpp"
 #include "core/multigrid/amgx_pgm_kernels.hpp"
 
@@ -55,7 +56,6 @@ namespace amgx_pgm {
 
 GKO_REGISTER_OPERATION(restrict_apply, amgx_pgm::restrict_apply);
 GKO_REGISTER_OPERATION(prolong_applyadd, amgx_pgm::prolong_applyadd);
-GKO_REGISTER_OPERATION(initial, amgx_pgm::initial);
 GKO_REGISTER_OPERATION(match_edge, amgx_pgm::match_edge);
 GKO_REGISTER_OPERATION(count_unagg, amgx_pgm::count_unagg);
 GKO_REGISTER_OPERATION(renumber, amgx_pgm::renumber);
@@ -63,6 +63,7 @@ GKO_REGISTER_OPERATION(find_strongest_neighbor,
                        amgx_pgm::find_strongest_neighbor);
 GKO_REGISTER_OPERATION(assign_to_exist_agg, amgx_pgm::assign_to_exist_agg);
 GKO_REGISTER_OPERATION(amgx_pgm_generate, amgx_pgm::amgx_pgm_generate);
+GKO_REGISTER_OPERATION(fill_array, components::fill_array);
 
 
 }  // namespace amgx_pgm
@@ -101,7 +102,8 @@ void AmgxPgm<ValueType, IndexType>::generate()
                                       parameters_.deterministic * num);
     const auto amgxpgm_op = gko::as<matrix_type>(this->system_matrix_.get());
     // Initial agg = -1
-    exec->run(amgx_pgm::make_initial(agg_));
+    exec->run(amgx_pgm::make_fill_array(agg_.get_data(), agg_.get_num_elems(),
+                                        -one<IndexType>()));
     size_type num_unagg{0};
     size_type num_unagg_prev{0};
     // TODO: if mtx is a hermitian matrix, weight_mtx = abs(mtx)
diff --git a/core/multigrid/amgx_pgm_kernels.hpp b/core/multigrid/amgx_pgm_kernels.hpp
index c3e2b46f8a7..a48e5f7a3ae 100644
--- a/core/multigrid/amgx_pgm_kernels.hpp
+++ b/core/multigrid/amgx_pgm_kernels.hpp
@@ -61,10 +61,6 @@ namespace amgx_pgm {
         std::shared_ptr<const DefaultExecutor> exec, const Array<_itype> &agg, \
         const matrix::Dense<_vtype> *b, matrix::Dense<_vtype> *x)
 
-#define GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL(_itype)           \
-    void initial(std::shared_ptr<const DefaultExecutor> exec, \
-                 Array<_itype> &agg)
-
 #define GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL(_itype)           \
     void match_edge(std::shared_ptr<const DefaultExecutor> exec, \
                     const Array<_itype> &strongest_neighbor,     \
@@ -104,8 +100,6 @@ namespace amgx_pgm {
     template <typename ValueType, typename IndexType>                   \
     GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL(ValueType, IndexType); \
     template <typename IndexType>                                       \
-    GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL(IndexType);                     \
-    template <typename IndexType>                                       \
     GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL(IndexType);                  \
     template <typename IndexType>                                       \
     GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL(IndexType);                 \
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index cebee78d0a3..5fe7bf37ba4 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -82,13 +82,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
 
 
-template <typename IndexType>
-void initial(std::shared_ptr<const CudaExecutor> exec,
-             Array<IndexType> &agg) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL);
-
-
 template <typename IndexType>
 void match_edge(std::shared_ptr<const CudaExecutor> exec,
                 const Array<IndexType> &strongest_neighbor,
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index 7104cc57984..53b9a4972bb 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -82,13 +82,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
 
 
-template <typename IndexType>
-void initial(std::shared_ptr<const HipExecutor> exec,
-             Array<IndexType> &agg) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL);
-
-
 template <typename IndexType>
 void match_edge(std::shared_ptr<const HipExecutor> exec,
                 const Array<IndexType> &strongest_neighbor,
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index 5afa2a7f150..87d81fb7ff8 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -79,13 +79,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
 
 
-template <typename IndexType>
-void initial(std::shared_ptr<const OmpExecutor> exec,
-             Array<IndexType> &agg) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL);
-
-
 template <typename IndexType>
 void match_edge(std::shared_ptr<const OmpExecutor> exec,
                 const Array<IndexType> &strongest_neighbor,
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index c6b3b2b2531..403dcef0db7 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -108,18 +108,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
 
 
-template <typename IndexType>
-void initial(std::shared_ptr<const ReferenceExecutor> exec,
-             Array<IndexType> &agg)
-{
-    for (size_type i = 0; i < agg.get_num_elems(); i++) {
-        agg.get_data()[i] = -1;
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_INITIAL_KERNEL);
-
-
 template <typename IndexType>
 void match_edge(std::shared_ptr<const ReferenceExecutor> exec,
                 const Array<IndexType> &strongest_neighbor,
@@ -260,7 +248,7 @@ void assign_to_exist_agg(std::shared_ptr<const ReferenceExecutor> exec,
         for (auto idx = row_ptrs[row]; idx < row_ptrs[row + 1]; idx++) {
             auto col = col_idxs[idx];
             if (col == row) {
-                break;
+                continue;
             }
             auto weight =
                 vals[idx] / max(abs(diag_vals[row]), abs(diag_vals[col]));

From bd82e543b1ee67ac1541262dc29b3b82580f83f7 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 12 Aug 2020 11:31:41 +0200
Subject: [PATCH 0468/1008] fix amg_pgm_kernel determinstic problem

---
 reference/multigrid/amgx_pgm_kernels.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index 403dcef0db7..170972e9fc2 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -234,7 +234,6 @@ void assign_to_exist_agg(std::shared_ptr<const ReferenceExecutor> exec,
     const auto row_ptrs = weight_mtx->get_const_row_ptrs();
     const auto col_idxs = weight_mtx->get_const_col_idxs();
     const auto vals = weight_mtx->get_const_values();
-    auto max_weight_agg = zero<ValueType>();
     const auto agg_const_val = agg.get_const_data();
     auto agg_val = (intermediate_agg.get_num_elems() > 0)
                        ? intermediate_agg.get_data()
@@ -244,6 +243,7 @@ void assign_to_exist_agg(std::shared_ptr<const ReferenceExecutor> exec,
         if (agg_const_val[row] != -1) {
             continue;
         }
+        auto max_weight_agg = zero<ValueType>();
         IndexType strongest_agg = -1;
         for (auto idx = row_ptrs[row]; idx < row_ptrs[row + 1]; idx++) {
             auto col = col_idxs[idx];

From 8a396dd11acf4dd692577d812b1b9ea4dfdbe936 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 25 Aug 2020 16:54:09 +0200
Subject: [PATCH 0469/1008] converting to csr if the system matrix is not csr

---
 core/multigrid/amgx_pgm.cpp | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index b171fa174ef..29a230cbcd0 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -100,7 +100,18 @@ void AmgxPgm<ValueType, IndexType>::generate()
     Array<IndexType> strongest_neighbor(this->get_executor(), num);
     Array<IndexType> intermediate_agg(this->get_executor(),
                                       parameters_.deterministic * num);
-    const auto amgxpgm_op = gko::as<matrix_type>(this->system_matrix_.get());
+    // Only support csr matrix currently.
+    const matrix_type *amgxpgm_op = nullptr;
+    // Store the csr matrix if needed
+    auto amgxpgm_op_unique_ptr = matrix_type::create(exec);
+    if (!(amgxpgm_op =
+              dynamic_cast<const matrix_type *>(system_matrix_.get()))) {
+        // if original matrix is not csr, converting it to csr.
+        as<ConvertibleTo<matrix_type>>(this->system_matrix_.get())
+            ->convert_to(amgxpgm_op_unique_ptr.get());
+        amgxpgm_op = amgxpgm_op_unique_ptr.get();
+    }
+
     // Initial agg = -1
     exec->run(amgx_pgm::make_fill_array(agg_.get_data(), agg_.get_num_elems(),
                                         -one<IndexType>()));

From fbe79ecaf4a30017715bfedf0d226701d4311c07 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 26 Aug 2020 11:09:02 +0200
Subject: [PATCH 0470/1008] remove unneeded code, add description of agg_group

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/multigrid/amgx_pgm.cpp                | 9 +++------
 core/test/multigrid/CMakeLists.txt         | 2 +-
 include/ginkgo/core/multigrid/amgx_pgm.hpp | 4 ++++
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index 29a230cbcd0..475cb644dcd 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -81,7 +81,6 @@ std::unique_ptr<LinOp> amgx_pgm_generate(
     auto coarse = matrix::Csr<ValueType, IndexType>::create(
         exec, dim<2>{num_agg, num_agg}, 0, source->get_strategy());
     exec->run(amgx_pgm::make_amgx_pgm_generate(source, agg, coarse.get()));
-    matrix::CsrBuilder<ValueType, IndexType> builder(coarse.get());
     return std::move(coarse);
 }
 
@@ -150,11 +149,9 @@ void AmgxPgm<ValueType, IndexType>::generate()
         // copy the agg to intermediate_agg
         intermediate_agg = agg_;
     }
-    while (num_unagg != 0) {
-        exec->run(amgx_pgm::make_assign_to_exist_agg(
-            weight_mtx.get(), diag.get(), agg_, intermediate_agg));
-        exec->run(amgx_pgm::make_count_unagg(agg_, &num_unagg));
-    }
+    // Assign all left points
+    exec->run(amgx_pgm::make_assign_to_exist_agg(weight_mtx.get(), diag.get(),
+                                                 agg_, intermediate_agg));
     size_type num_agg;
     // Renumber the index
     exec->run(amgx_pgm::make_renumber(agg_, &num_agg));
diff --git a/core/test/multigrid/CMakeLists.txt b/core/test/multigrid/CMakeLists.txt
index 12a292ba7d6..2ce9d5007d8 100644
--- a/core/test/multigrid/CMakeLists.txt
+++ b/core/test/multigrid/CMakeLists.txt
@@ -1 +1 @@
-ginkgo_create_test(amgx_pgm)
\ No newline at end of file
+ginkgo_create_test(amgx_pgm)
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index 1925fdbf6a5..8017e8360c5 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -85,6 +85,10 @@ class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
     /**
      * Returns the aggregate group.
      *
+     * Aggregate group whose size is same as the number of rows stores the
+     * mapping information from row index to coarse row index.
+     * i.e., agg[row_idx] = coarse_row_idx.
+     *
      * @return the aggregate group.
      */
     IndexType *get_agg() noexcept { return agg_.get_data(); }

From 7e75f1d6c8129b2760234a201fc105945806ccba Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 27 Aug 2020 14:33:08 +0200
Subject: [PATCH 0471/1008] use AbsoluteComputable and update documentation

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 include/ginkgo/core/multigrid/amgx_pgm.hpp         | 13 +++++++++++--
 include/ginkgo/core/multigrid/restrict_prolong.hpp | 11 +++++------
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index 8017e8360c5..ec9ff72717e 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -55,7 +55,12 @@ namespace multigrid {
  * contains size = 2 version.
  *
  * AmgxPgm creates the aggreagate group according to the matrix value not the
- * structure.
+ * structure. AmgxPgm gives two steps (one-phase handshaking) to group the
+ * elements.
+ * 1: get the strongest neighbor of each unaggregated element.
+ * 2: group the elements whose strongest neighbor is each other.
+ * repeating until reaching the given conditions. After that, assigns the
+ * unaggregated elements to aggregated group or stay alone.
  *
  * @tparam ValueType  precision of matrix elements
  * @tparam IndexType  precision of matrix indexes
@@ -108,7 +113,7 @@ class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
     GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
     {
         /**
-         * The maximum number of iteration.
+         * The maximum number of iterations.
          */
         unsigned GKO_FACTORY_PARAMETER(max_iterations, 15u);
 
@@ -119,6 +124,10 @@ class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
 
         /**
          * Use the deterministic assign_to_exist_agg method or not.
+         *
+         * If the deterministic is true, always get the same aggregated group
+         * from the same matrix. Otherwise, the aggregated group might be
+         * differenet depending on the execution ordering.
          */
         bool GKO_FACTORY_PARAMETER(deterministic, false);
     };
diff --git a/include/ginkgo/core/multigrid/restrict_prolong.hpp b/include/ginkgo/core/multigrid/restrict_prolong.hpp
index 13baa11cdb5..14ef5c5d0b2 100644
--- a/include/ginkgo/core/multigrid/restrict_prolong.hpp
+++ b/include/ginkgo/core/multigrid/restrict_prolong.hpp
@@ -138,9 +138,9 @@ class RestrictProlong
      */
     void validate_restrict_parameters(const LinOp *b, LinOp *x) const
     {
-        auto restirct_dim = gko::dim<2>(coarse_dim_, fine_dim_);
-        GKO_ASSERT_CONFORMANT(restirct_dim, b);
-        GKO_ASSERT_EQUAL_ROWS(restirct_dim, x);
+        auto restrict_dim = gko::dim<2>(coarse_dim_, fine_dim_);
+        GKO_ASSERT_CONFORMANT(restrict_dim, b);
+        GKO_ASSERT_EQUAL_ROWS(restrict_dim, x);
         GKO_ASSERT_EQUAL_COLS(b, x);
     }
 
@@ -198,7 +198,7 @@ class RestrictProlong
     }
 
     /**
-     * Creates a coarse fine.
+     * Creates a RestrictProlong.
      *
      * @param exec  the executor where all the operations are performed
      */
@@ -221,8 +221,7 @@ class RestrictProlongFactory
     std::unique_ptr<RestrictProlong> generate(
         std::shared_ptr<const LinOp> input) const
     {
-        auto generated = AbstractFactory::generate(input);
-        return generated;
+        return AbstractFactory::generate(input);
     }
 };
 

From 2533fff9b9f0f1b22373e928f70ac6a3577f405e Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 28 Aug 2020 15:58:34 +0200
Subject: [PATCH 0472/1008] update focumentation and fix format

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 include/ginkgo/core/multigrid/amgx_pgm.hpp    |  8 +-
 .../core/multigrid/restrict_prolong.hpp       | 97 +++++++++++++++----
 2 files changed, 86 insertions(+), 19 deletions(-)

diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index ec9ff72717e..3d9b7f8cce3 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -66,6 +66,7 @@ namespace multigrid {
  * @tparam IndexType  precision of matrix indexes
  *
  * @ingroup RestrictProlong
+ * @ingroup Multigrid
  * @ingroup LinOp
  */
 template <typename ValueType = default_precision, typename IndexType = int32>
@@ -118,7 +119,8 @@ class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
         unsigned GKO_FACTORY_PARAMETER(max_iterations, 15u);
 
         /**
-         * The maximum ratio of unassigned number.
+         * The maximum ratio of unassigned number, which is valid in the
+         * interval 0.0 ~ 1.0.
          */
         double GKO_FACTORY_PARAMETER(max_unassigned_percentage, 0.05);
 
@@ -127,7 +129,7 @@ class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
          *
          * If the deterministic is true, always get the same aggregated group
          * from the same matrix. Otherwise, the aggregated group might be
-         * differenet depending on the execution ordering.
+         * different depending on the execution ordering.
          */
         bool GKO_FACTORY_PARAMETER(deterministic, false);
     };
@@ -150,6 +152,8 @@ class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
           system_matrix_{std::move(system_matrix)},
           agg_(factory->get_executor(), system_matrix_->get_size()[0])
     {
+        GKO_ASSERT(parameters_.max_unassigned_percentage <= 1.0);
+        GKO_ASSERT(parameters_.max_unassigned_percentage >= 0.0);
         if (system_matrix_->get_size()[0] != 0) {
             // generate on the existed matrix
             this->generate();
diff --git a/include/ginkgo/core/multigrid/restrict_prolong.hpp b/include/ginkgo/core/multigrid/restrict_prolong.hpp
index 14ef5c5d0b2..ee2b8fb3bd5 100644
--- a/include/ginkgo/core/multigrid/restrict_prolong.hpp
+++ b/include/ginkgo/core/multigrid/restrict_prolong.hpp
@@ -50,11 +50,12 @@ namespace multigrid {
 /**
  * The RestrictProlong class can be used to construct restrict_apply and
  * prolong_applyadd. R is the restrict operator (fine level -> coarse
- * level) and P is prolong operator (coarse level -> fine level).
+ * level) and P is the prolong operator (coarse level -> fine level).
  * restrict_apply(b, x) -> x = R(b)
  * prolong_applyadd(b, x) -> x = P(b) + x
  *
  * @ingroup RestrictProlong
+ * @ingroup Multigrid
  */
 class RestrictProlong
     : public EnableAbstractPolymorphicObject<RestrictProlong> {
@@ -163,8 +164,7 @@ class RestrictProlong
      * Implementers of RestrictProlong should override this function instead
      * of restrict_apply(const LinOp *, LinOp *).
      *
-     * Performs the operation x = restrict(b), where op is the restrict
-     * operator.
+     * Performs the operation x = restrict(b).
      *
      * @param b  the input vector(s) on which the operator is applied
      * @param x  the output vector(s) where the result is stored
@@ -173,10 +173,9 @@ class RestrictProlong
 
     /**
      * Implementers of RestrictProlong should override this function instead
-     * of prolong_apply(const LinOp *, LinOp *).
+     * of prolong_applyadd(const LinOp *, LinOp *).
      *
-     * Performs the operation x = prolong(b), where op is the prolong
-     * operator.
+     * Performs the operation x += prolong(b).
      *
      * @param b  the input vector(s) on which the operator is applied
      * @param x  the output vector(s) where the result is stored
@@ -212,6 +211,18 @@ class RestrictProlong
     size_type coarse_dim_;
 };
 
+
+/**
+ * A RestrictProlongFactory represents a higher order mapping which transforms
+ * one linear operator into the behavior between multigrid level.
+ *
+ * A RestrictProlongFactory transforms a linear operator $F$ (fine level) to
+ * $R$ (restrict), $P$ (prolong), and $C=RFP$ (coarse level), which defines the
+ * mapping between fine level and coarse level. R: F -> C and P: C -> F.
+ *
+ * @ingroup RestrictProlong
+ * @ingroup Multigrid
+ */
 class RestrictProlongFactory
     : public AbstractFactory<RestrictProlong, std::shared_ptr<const LinOp>> {
 public:
@@ -226,23 +237,74 @@ class RestrictProlongFactory
 };
 
 
-template <typename ConcreteFactory, typename ConcreteCriterion,
+/**
+ * This is an alias for the EnableDefaultFactory mixin, which correctly sets the
+ * template parameters to enable a subclass of RestrictProlongFactory.
+ *
+ * @tparam ConcreteFactory  the concrete factory which is being implemented
+ *                          [CRTP parmeter]
+ * @tparam ConcreteRstrPrlg  the concrete RestrictProlong type which this
+ * factory produces, needs to have a constructor which takes a const
+ * ConcreteFactory *, and an std::shared_ptr<const LinOp> as parameters.
+ * @tparam ParametersType  a subclass of enable_parameters_type template which
+ *                         defines all of the parameters of the factory
+ * @tparam PolymorphicBase  parent of ConcreteFactory in the polymorphic
+ *                          hierarchy, has to be a subclass of
+ * RestrictProlongFactory
+ *
+ * @ingroup LinOp
+ */
+template <typename ConcreteFactory, typename ConcreteRstrPrlg,
           typename ParametersType,
           typename PolymorphicBase = RestrictProlongFactory>
 using EnableDefaultRestrictProlongFactory =
-    EnableDefaultFactory<ConcreteFactory, ConcreteCriterion, ParametersType,
+    EnableDefaultFactory<ConcreteFactory, ConcreteRstrPrlg, ParametersType,
                          PolymorphicBase>;
 
 
-template <typename ConcreteLinOp, typename PolymorphicBase = RestrictProlong>
+/**
+ * The EnableRestrictProlong mixin can be used to provide sensible default
+ * implementations of the majority of the RestrictProlong and PolymorphicObject
+ * interface.
+ *
+ * *
+ * The goal of the mixin is to facilitate the development of new
+ * RestrictProlong, by enabling the implementers to focus on the important parts
+ * of their operator, while the library takes care of generating the trivial
+ * utility functions. The mixin will provide default implementations for the
+ * entire PolymorphicObject interface, including a default implementation of
+ * `copy_from` between objects of the new LinOp type. It will also hide the
+ * default RestrictPrlong::restrict_apply() and
+ * RestrictPrlong::prolong_applyadd() methods with versions that preserve the
+ * static type of the object.
+ *
+ * Implementers of new RestrictProlong are required to specify only the
+ * following aspects:
+ *
+ * 1.  Creation of the RestrictProlong: This can be facilitated via
+ *     GKO_ENABLE_RESTRICT_PROLONG_FACTORY macro.
+ * 2.  Application of the RestrictProlong: Implementers have to override
+ *     RestrictPrlong::restrict_apply_impl() and
+ *     RestrictPrlong::prolong_applyadd_impl() virtual methods and use
+ *     set_coarse_fine() to set the generated results.
+ *
+ * @tparam ConcreteRstrPrlg  the concrete RestrictProlong which is being
+ *                           implemented [CRTP parameter]
+ * @tparam PolymorphicBase  parent of ConcreteRstrPrlg in the polymorphic
+ *                          hierarchy, has to be a subclass of RestrictProlong
+ *
+ * @ingroup RestrictProlong
+ * @ingroup Multigrid
+ */
+template <typename ConcreteRstrPrlg, typename PolymorphicBase = RestrictProlong>
 class EnableRestrictProlong
-    : public EnablePolymorphicObject<ConcreteLinOp, PolymorphicBase>,
-      public EnablePolymorphicAssignment<ConcreteLinOp> {
+    : public EnablePolymorphicObject<ConcreteRstrPrlg, PolymorphicBase>,
+      public EnablePolymorphicAssignment<ConcreteRstrPrlg> {
 public:
-    using EnablePolymorphicObject<ConcreteLinOp,
+    using EnablePolymorphicObject<ConcreteRstrPrlg,
                                   PolymorphicBase>::EnablePolymorphicObject;
 
-    ConcreteLinOp *restrict_apply(const LinOp *b, LinOp *x)
+    ConcreteRstrPrlg *restrict_apply(const LinOp *b, LinOp *x)
     {
         this->validate_restrict_parameters(b, x);
         auto exec = this->get_executor();
@@ -251,7 +313,7 @@ class EnableRestrictProlong
         return self();
     }
 
-    const ConcreteLinOp *restrict_apply(const LinOp *b, LinOp *x) const
+    const ConcreteRstrPrlg *restrict_apply(const LinOp *b, LinOp *x) const
     {
         this->validate_restrict_parameters(b, x);
         auto exec = this->get_executor();
@@ -260,7 +322,7 @@ class EnableRestrictProlong
         return self();
     }
 
-    ConcreteLinOp *prolong_applyadd(const LinOp *b, LinOp *x)
+    ConcreteRstrPrlg *prolong_applyadd(const LinOp *b, LinOp *x)
     {
         this->validate_prolong_parameters(b, x);
         auto exec = this->get_executor();
@@ -269,7 +331,7 @@ class EnableRestrictProlong
         return self();
     }
 
-    const ConcreteLinOp *prolong_applyadd(const LinOp *b, LinOp *x) const
+    const ConcreteRstrPrlg *prolong_applyadd(const LinOp *b, LinOp *x) const
     {
         this->validate_prolong_parameters(b, x);
         auto exec = this->get_executor();
@@ -279,7 +341,7 @@ class EnableRestrictProlong
     }
 
 protected:
-    GKO_ENABLE_SELF(ConcreteLinOp);
+    GKO_ENABLE_SELF(ConcreteRstrPrlg);
 };
 
 
@@ -321,6 +383,7 @@ public:                                                                       \
                   "This assert is used to counter the false positive extra "  \
                   "semi-colon warnings")
 
+
 }  // namespace multigrid
 }  // namespace gko
 

From c549d4eb0dbf72af2c53d71d04c1c0881ea89570 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 20 Oct 2020 11:30:38 +0200
Subject: [PATCH 0473/1008] review update

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/multigrid/amgx_pgm.cpp                   | 19 +++++++++----------
 include/ginkgo/core/multigrid/amgx_pgm.hpp    |  8 +++++---
 .../core/multigrid/restrict_prolong.hpp       |  3 +--
 reference/multigrid/amgx_pgm_kernels.cpp      |  4 ++--
 reference/test/multigrid/amgx_pgm_kernels.cpp |  8 ++++----
 5 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index 475cb644dcd..af008f50b1f 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -92,13 +92,13 @@ template <typename ValueType, typename IndexType>
 void AmgxPgm<ValueType, IndexType>::generate()
 {
     using matrix_type = matrix::Csr<ValueType, IndexType>;
-    using rmc_value_type = remove_complex<ValueType>;
+    using real_type = remove_complex<ValueType>;
     using weight_matrix_type = remove_complex<matrix_type>;
     auto exec = this->get_executor();
-    const auto num = this->system_matrix_->get_size()[0];
-    Array<IndexType> strongest_neighbor(this->get_executor(), num);
+    const auto num_rows = this->system_matrix_->get_size()[0];
+    Array<IndexType> strongest_neighbor(this->get_executor(), num_rows);
     Array<IndexType> intermediate_agg(this->get_executor(),
-                                      parameters_.deterministic * num);
+                                      parameters_.deterministic * num_rows);
     // Only support csr matrix currently.
     const matrix_type *amgxpgm_op = nullptr;
     // Store the csr matrix if needed
@@ -121,10 +121,10 @@ void AmgxPgm<ValueType, IndexType>::generate()
     auto abs_mtx = amgxpgm_op->compute_absolute();
     // abs_mtx is already real valuetype, so transpose is enough
     auto weight_mtx = gko::as<weight_matrix_type>(abs_mtx->transpose());
-    auto half_scaler = initialize<matrix::Dense<rmc_value_type>>({0.5}, exec);
-    auto identity = matrix::Identity<rmc_value_type>::create(exec, num);
+    auto half_scalar = initialize<matrix::Dense<real_type>>({0.5}, exec);
+    auto identity = matrix::Identity<real_type>::create(exec, num_rows);
     // W = (abs_mtx + transpose(abs_mtx))/2
-    abs_mtx->apply(lend(half_scaler), lend(identity), lend(half_scaler),
+    abs_mtx->apply(lend(half_scalar), lend(identity), lend(half_scalar),
                    lend(weight_mtx));
     // Extract the diagonal value of matrix
     auto diag = weight_mtx->extract_diagonal();
@@ -139,7 +139,7 @@ void AmgxPgm<ValueType, IndexType>::generate()
         // no new match, all match, or the ratio of num_unagg/num is lower
         // than parameter.max_unassigned_percentage
         if (num_unagg == 0 || num_unagg == num_unagg_prev ||
-            num_unagg < parameters_.max_unassigned_percentage * num) {
+            num_unagg < parameters_.max_unassigned_percentage * num_rows) {
             break;
         }
         num_unagg_prev = num_unagg;
@@ -156,10 +156,9 @@ void AmgxPgm<ValueType, IndexType>::generate()
     // Renumber the index
     exec->run(amgx_pgm::make_renumber(agg_, &num_agg));
 
-
     // Construct the coarse matrix
     auto coarse = amgx_pgm_generate(exec, amgxpgm_op, num_agg, agg_);
-    this->set_coarse_fine(std::move(coarse), num);
+    this->set_coarse_fine(std::move(coarse), num_rows);
 }
 
 
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index 3d9b7f8cce3..3a8089273f3 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -59,8 +59,9 @@ namespace multigrid {
  * elements.
  * 1: get the strongest neighbor of each unaggregated element.
  * 2: group the elements whose strongest neighbor is each other.
- * repeating until reaching the given conditions. After that, assigns the
- * unaggregated elements to aggregated group or stay alone.
+ * repeating until reaching the given conditions. After that, the
+ * un-aggregated elements are assigned to an aggregated group
+ * or are left alone.
  *
  * @tparam ValueType  precision of matrix elements
  * @tparam IndexType  precision of matrix indexes
@@ -120,7 +121,8 @@ class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
 
         /**
          * The maximum ratio of unassigned number, which is valid in the
-         * interval 0.0 ~ 1.0.
+         * interval 0.0 ~ 1.0. We use the same default value as NVIDIA AMGX
+         * Rerference Manual (October 2017, API Version 2).
          */
         double GKO_FACTORY_PARAMETER(max_unassigned_percentage, 0.05);
 
diff --git a/include/ginkgo/core/multigrid/restrict_prolong.hpp b/include/ginkgo/core/multigrid/restrict_prolong.hpp
index ee2b8fb3bd5..63073cc21c3 100644
--- a/include/ginkgo/core/multigrid/restrict_prolong.hpp
+++ b/include/ginkgo/core/multigrid/restrict_prolong.hpp
@@ -250,7 +250,7 @@ class RestrictProlongFactory
  *                         defines all of the parameters of the factory
  * @tparam PolymorphicBase  parent of ConcreteFactory in the polymorphic
  *                          hierarchy, has to be a subclass of
- * RestrictProlongFactory
+ *                          RestrictProlongFactory
  *
  * @ingroup LinOp
  */
@@ -267,7 +267,6 @@ using EnableDefaultRestrictProlongFactory =
  * implementations of the majority of the RestrictProlong and PolymorphicObject
  * interface.
  *
- * *
  * The goal of the mixin is to facilitate the development of new
  * RestrictProlong, by enabling the implementers to focus on the important parts
  * of their operator, while the library takes care of generating the trivial
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index 170972e9fc2..4d90f3037e8 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -288,8 +288,8 @@ void amgx_pgm_generate(std::shared_ptr<const ReferenceExecutor> exec,
     const auto source_row_ptrs = source->get_const_row_ptrs();
     const auto source_col_idxs = source->get_const_col_idxs();
     const auto source_vals = source->get_const_values();
-    vector<map<IndexType, ValueType>> row_list(
-        source_nrows, map<IndexType, ValueType>{exec}, exec);
+    gko::vector<gko::map<IndexType, ValueType>> row_list(
+        source_nrows, gko::map<IndexType, ValueType>{exec}, exec);
     for (size_type i = 0; i < source_nrows; i++) {
         IndexType row_idx = agg.get_const_data()[i];
         for (auto j = source_row_ptrs[i]; j < source_row_ptrs[i + 1]; j++) {
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index 797c9bd33f8..edfb0493d1b 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -69,8 +69,8 @@ class AmgxPgm : public ::testing::Test {
     using Vec = gko::matrix::Dense<value_type>;
     using RestrictProlong = gko::multigrid::AmgxPgm<value_type, index_type>;
     using T = value_type;
-    using rmc_value_type = gko::remove_complex<value_type>;
-    using WeightMtx = gko::matrix::Csr<rmc_value_type, index_type>;
+    using real_type = gko::remove_complex<value_type>;
+    using WeightMtx = gko::matrix::Csr<real_type, index_type>;
     AmgxPgm()
         : exec(gko::ReferenceExecutor::create()),
           amgxpgm_factory(RestrictProlong::build()
@@ -242,7 +242,7 @@ class AmgxPgm : public ::testing::Test {
         ASSERT_EQ(m1->get_size()[0], m2->get_size()[0]);
         ASSERT_EQ(m1->get_size()[1], m2->get_size()[1]);
         ASSERT_EQ(m1->get_num_stored_elements(), m2->get_num_stored_elements());
-        for (gko::size_type i = 0; i < m1->get_size() + 1; i++) {
+        for (gko::size_type i = 0; i < m1->get_size()[0] + 1; i++) {
             ASSERT_EQ(m1->get_const_row_ptrs()[i], m2->get_const_row_ptrs()[i]);
         }
         for (gko::size_type i = 0; i < m1->get_num_stored_elements(); ++i) {
@@ -263,7 +263,7 @@ class AmgxPgm : public ::testing::Test {
     std::shared_ptr<Mtx> mtx;
     std::shared_ptr<Mtx> coarse;
     std::shared_ptr<WeightMtx> weight;
-    std::shared_ptr<gko::matrix::Diagonal<rmc_value_type>> mtx_diag;
+    std::shared_ptr<gko::matrix::Diagonal<real_type>> mtx_diag;
     gko::Array<index_type> agg;
     std::shared_ptr<Vec> coarse_b;
     std::shared_ptr<Vec> fine_b;

From 185ac8610dbb400395cf51eb9c739524dfc96893 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 20 Jan 2021 20:24:27 +0800
Subject: [PATCH 0474/1008] add multigrid related interface and move to linop

---
 core/multigrid/amgx_pgm.cpp                   |  21 +-
 core/multigrid/amgx_pgm_kernels.hpp           |  11 +-
 core/test/multigrid/amgx_pgm.cpp              |   3 +-
 cuda/multigrid/amgx_pgm_kernels.cu            |   2 +-
 dpcpp/CMakeLists.txt                          |   1 +
 dpcpp/multigrid/amgx_pgm_kernels.dp.cpp       | 144 +++++++
 hip/multigrid/amgx_pgm_kernels.hip.cpp        |   2 +-
 include/ginkgo/core/multigrid/amgx_pgm.hpp    |  55 ++-
 include/ginkgo/core/multigrid/interface.hpp   | 293 +++++++++++++
 .../core/multigrid/restrict_prolong.hpp       | 390 ------------------
 include/ginkgo/ginkgo.hpp                     |   2 +-
 omp/multigrid/amgx_pgm_kernels.cpp            |   2 +-
 reference/multigrid/amgx_pgm_kernels.cpp      |   2 +-
 reference/test/multigrid/amgx_pgm_kernels.cpp |  58 ++-
 14 files changed, 553 insertions(+), 433 deletions(-)
 create mode 100644 dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
 create mode 100644 include/ginkgo/core/multigrid/interface.hpp
 delete mode 100644 include/ginkgo/core/multigrid/restrict_prolong.hpp

diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index af008f50b1f..a0fcc72bb2b 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -157,8 +157,9 @@ void AmgxPgm<ValueType, IndexType>::generate()
     exec->run(amgx_pgm::make_renumber(agg_, &num_agg));
 
     // Construct the coarse matrix
-    auto coarse = amgx_pgm_generate(exec, amgxpgm_op, num_agg, agg_);
-    this->set_coarse_fine(std::move(coarse), num_rows);
+    auto coarse_matrix =
+        share(amgx_pgm_generate(exec, amgxpgm_op, num_agg, agg_));
+    this->set_multigrid_level(system_matrix_, coarse_matrix);
 }
 
 
@@ -184,6 +185,20 @@ void AmgxPgm<ValueType, IndexType>::prolong_applyadd_impl(const LinOp *b,
 }
 
 
+template <typename ValueType, typename IndexType>
+void AmgxPgm<ValueType, IndexType>::prolong_apply_impl(const LinOp *b,
+                                                       LinOp *x) const
+{
+    auto exec = this->get_executor();
+    auto vec_x = as<matrix::Dense<ValueType>>(x);
+    exec->run(amgx_pgm::make_fill_array(
+        vec_x->get_values(), vec_x->get_size()[0] * vec_x->get_stride(),
+        zero<ValueType>()));
+    exec->run(amgx_pgm::make_prolong_applyadd(
+        agg_, as<matrix::Dense<ValueType>>(b), vec_x));
+}
+
+
 #define GKO_DECLARE_AMGX_PGM(_vtype, _itype) class AmgxPgm<_vtype, _itype>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM);
 
diff --git a/core/multigrid/amgx_pgm_kernels.hpp b/core/multigrid/amgx_pgm_kernels.hpp
index a48e5f7a3ae..b86099b23c6 100644
--- a/core/multigrid/amgx_pgm_kernels.hpp
+++ b/core/multigrid/amgx_pgm_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -152,6 +152,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace amgx_pgm {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace amgx_pgm
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/core/test/multigrid/amgx_pgm.cpp b/core/test/multigrid/amgx_pgm.cpp
index 3ffb3d115ab..1f04da999a8 100644
--- a/core/test/multigrid/amgx_pgm.cpp
+++ b/core/test/multigrid/amgx_pgm.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -58,7 +58,6 @@ class AmgxPgmFactory : public ::testing::Test {
     using Mtx = gko::matrix::Csr<value_type, index_type>;
     using Vec = gko::matrix::Dense<value_type>;
     using RestrictProlong = gko::multigrid::AmgxPgm<value_type, index_type>;
-    using T = value_type;
     AmgxPgmFactory()
         : exec(gko::ReferenceExecutor::create()),
           amgxpgm_factory(RestrictProlong::build()
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index 5fe7bf37ba4..937faa54616 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 81fe4492f8a..9e340ba33fe 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -30,6 +30,7 @@ target_sources(ginkgo_dpcpp
     matrix/hybrid_kernels.dp.cpp
     matrix/sellp_kernels.dp.cpp
     matrix/sparsity_csr_kernels.dp.cpp
+    multigrid/amgx_pgm_kernels.dp.cpp
     preconditioner/isai_kernels.dp.cpp
     preconditioner/jacobi_kernels.dp.cpp
     reorder/rcm_kernels.dp.cpp
diff --git a/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp b/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
new file mode 100644
index 00000000000..3e7ba1a3329
--- /dev/null
+++ b/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
@@ -0,0 +1,144 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/multigrid/amgx_pgm_kernels.hpp"
+
+
+#include <memory>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The AMGX_PGM solver namespace.
+ *
+ * @ingroup amgx_pgm
+ */
+namespace amgx_pgm {
+
+
+template <typename ValueType, typename IndexType>
+void restrict_apply(std::shared_ptr<const DpcppExecutor> exec,
+                    const Array<IndexType> &agg,
+                    const matrix::Dense<ValueType> *b,
+                    matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void prolong_applyadd(std::shared_ptr<const DpcppExecutor> exec,
+                      const Array<IndexType> &agg,
+                      const matrix::Dense<ValueType> *b,
+                      matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
+
+
+template <typename IndexType>
+void match_edge(std::shared_ptr<const DpcppExecutor> exec,
+                const Array<IndexType> &strongest_neighbor,
+                Array<IndexType> &agg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL);
+
+
+template <typename IndexType>
+void count_unagg(std::shared_ptr<const DpcppExecutor> exec,
+                 const Array<IndexType> &agg,
+                 size_type *num_unagg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL);
+
+
+template <typename IndexType>
+void renumber(std::shared_ptr<const DpcppExecutor> exec, Array<IndexType> &agg,
+              size_type *num_agg) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void find_strongest_neighbor(
+    std::shared_ptr<const DpcppExecutor> exec,
+    const matrix::Csr<ValueType, IndexType> *weight_mtx,
+    const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
+    Array<IndexType> &strongest_neighbor)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
+
+
+template <typename ValueType, typename IndexType>
+void assign_to_exist_agg(std::shared_ptr<const DpcppExecutor> exec,
+                         const matrix::Csr<ValueType, IndexType> *weight_mtx,
+                         const matrix::Diagonal<ValueType> *diag,
+                         Array<IndexType> &agg,
+                         Array<IndexType> &intermediate_agg)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
+
+
+template <typename ValueType, typename IndexType>
+void amgx_pgm_generate(std::shared_ptr<const DpcppExecutor> exec,
+                       const matrix::Csr<ValueType, IndexType> *source,
+                       const Array<IndexType> &agg,
+                       matrix::Csr<ValueType, IndexType> *coarse)
+{
+    GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
+
+
+}  // namespace amgx_pgm
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index 53b9a4972bb..5d76fea55c4 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index 3a8089273f3..f0777b602d7 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_MULTIGRID_AMGX_PGM_HPP_
-#define GKO_CORE_MULTIGRID_AMGX_PGM_HPP_
+#ifndef GKO_PUBLIC_CORE_MULTIGRID_AMGX_PGM_HPP_
+#define GKO_PUBLIC_CORE_MULTIGRID_AMGX_PGM_HPP_
 
 
 #include <vector>
@@ -41,7 +41,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/multigrid/restrict_prolong.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/multigrid/interface.hpp>
 
 
 namespace gko {
@@ -71,9 +72,10 @@ namespace multigrid {
  * @ingroup LinOp
  */
 template <typename ValueType = default_precision, typename IndexType = int32>
-class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
-    friend class EnableRestrictProlong<AmgxPgm>;
-    friend class EnablePolymorphicObject<AmgxPgm, RestrictProlong>;
+class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
+                public MultigridLevel {
+    friend class EnableLinOp<AmgxPgm>;
+    friend class EnablePolymorphicObject<AmgxPgm, LinOp>;
 
 public:
     using value_type = ValueType;
@@ -135,7 +137,7 @@ class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
          */
         bool GKO_FACTORY_PARAMETER(deterministic, false);
     };
-    GKO_ENABLE_RESTRICT_PROLONG_FACTORY(AmgxPgm, parameters, Factory);
+    GKO_ENABLE_LIN_OP_FACTORY(AmgxPgm, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);
 
 protected:
@@ -143,13 +145,43 @@ class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
 
     void prolong_applyadd_impl(const LinOp *b, LinOp *x) const override;
 
+    void prolong_apply_impl(const LinOp *b, LinOp *x) const override;
+
+    void apply_impl(const LinOp *b, LinOp *x) const override
+    {
+        // x = op(b) = prolong(coarse(restrict(b)))
+        auto exec = this->get_executor();
+        auto num_cols = b->get_size()[1];
+        auto coarse = this->get_coarse_matrix();
+        auto coarse_b = matrix::Dense<ValueType>::create(
+            exec, dim<2>{coarse->get_size()[1], num_cols});
+        auto coarse_x = matrix::Dense<ValueType>::create(
+            exec, dim<2>{coarse->get_size()[0], num_cols});
+        this->restrict_apply(b, lend(coarse_b));
+        coarse->apply(lend(coarse_b), lend(coarse_x));
+        this->prolong_apply(lend(coarse_x), x);
+    }
+
+    void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
+                    LinOp *x) const override
+    {
+        auto dense_x = as<matrix::Dense<ValueType>>(x);
+
+        auto x_clone = dense_x->clone();
+        this->apply(b, x_clone.get());
+        dense_x->scale(beta);
+        dense_x->add_scaled(alpha, x_clone.get());
+    }
+
     explicit AmgxPgm(std::shared_ptr<const Executor> exec)
-        : EnableRestrictProlong<AmgxPgm>(std::move(exec))
+        : EnableLinOp<AmgxPgm>(exec), MultigridLevel(exec, true)
     {}
 
     explicit AmgxPgm(const Factory *factory,
                      std::shared_ptr<const LinOp> system_matrix)
-        : EnableRestrictProlong<AmgxPgm>(factory->get_executor()),
+        : EnableLinOp<AmgxPgm>(factory->get_executor(),
+                               system_matrix->get_size()),
+          MultigridLevel(factory->get_executor(), true),
           parameters_{factory->get_parameters()},
           system_matrix_{std::move(system_matrix)},
           agg_(factory->get_executor(), system_matrix_->get_size()[0])
@@ -161,7 +193,6 @@ class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
             this->generate();
         }
     }
-
     void generate();
 
 private:
@@ -174,4 +205,4 @@ class AmgxPgm : public EnableRestrictProlong<AmgxPgm<ValueType, IndexType>> {
 }  // namespace gko
 
 
-#endif  // GKO_CORE_MULTIGRID_AMGX_PGM_HPP_
+#endif  // GKO_PUBLIC_CORE_MULTIGRID_AMGX_PGM_HPP_
diff --git a/include/ginkgo/core/multigrid/interface.hpp b/include/ginkgo/core/multigrid/interface.hpp
new file mode 100644
index 00000000000..25fbbe8e638
--- /dev/null
+++ b/include/ginkgo/core/multigrid/interface.hpp
@@ -0,0 +1,293 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_PUBLIC_CORE_MULTIGRID_INTERFACE_HPP_
+#define GKO_PUBLIC_CORE_MULTIGRID_INTERFACE_HPP_
+
+
+#include <functional>
+#include <memory>
+
+
+#include <ginkgo/core/base/abstract_factory.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+namespace gko {
+namespace multigrid {
+
+
+/**
+ * A class implementing this interface can be restrict operation.
+ *
+ * @ingroup multigrid
+ */
+class RestrictOp {
+public:
+    /**
+     * Applies a prolong operator to a vector (or a sequence of vectors).
+     *
+     * Performs the operation x = op(b), where op is this prolong operator.
+     *
+     * @param b  the input vector(s) on which the operator is applied
+     * @param x  the output vector(s) where the result is stored
+     */
+    void restrict_apply(const LinOp *b, LinOp *x) const
+    {
+        this->validate_parameters(b, x);
+        this->restrict_apply_impl(make_temporary_clone(exec_, b).get(),
+                                  make_temporary_clone(exec_, x).get());
+    }
+
+protected:
+    /**
+     * Sets the size of the restrict operator.
+     *
+     * @param value  the new size of the restrict operator
+     */
+    void set_restrict_size(const dim<2> &value) noexcept { size_ = value; }
+
+    virtual void restrict_apply_impl(const LinOp *b, LinOp *x) const = 0;
+
+    /**
+     * Creates a RestrictOp with settings
+     *
+     * @param exec  Executor associated to the RestrictOp
+     * @param size  the size of RestrictOp.
+     *
+     * @note the multigrid is generated in the generation not the construction,
+     *       so needs to call `set_restrict_size` to set restrict size after
+     *       generation.
+     */
+    explicit RestrictOp(std::shared_ptr<const Executor> exec,
+                        const gko::dim<2> &size = gko::dim<2>{})
+        : exec_(exec), size_(size)
+    {}
+
+private:
+    /**
+     * Throws a DimensionMismatch exception if the parameters to
+     * `restrict_apply` are of the wrong size.
+     *
+     * @param b  vector(s) on which the operator is applied
+     * @param x  output vector(s)
+     */
+    void validate_parameters(const LinOp *b, LinOp *x) const
+    {
+        GKO_ASSERT_CONFORMANT(size_, b);
+        GKO_ASSERT_EQUAL_ROWS(size_, x);
+        GKO_ASSERT_EQUAL_COLS(b, x);
+    }
+
+    std::shared_ptr<const Executor> exec_;
+    gko::dim<2> size_;
+};
+
+
+/**
+ * A class implementing this interface can be prolong operation.
+ *
+ * @ingroup multigrid
+ */
+class ProlongOp {
+public:
+    /**
+     * Applies a prolong operator to a vector (or a sequence of vectors).
+     *
+     * Performs the operation x = op(b), where op is this prolong operator.
+     *
+     * @param b  the input vector(s) on which the operator is applied
+     * @param x  the output vector(s) where the result is stored
+     */
+    void prolong_apply(const LinOp *b, LinOp *x) const
+    {
+        this->validate_parameters(b, x);
+        this->prolong_apply_impl(make_temporary_clone(exec_, b).get(),
+                                 make_temporary_clone(exec_, x).get());
+    }
+
+    /**
+     * Applies a prolong operator onto a vector (or a sequence of vectors).
+     *
+     * Performs the operation x += op(b), where op is this prolong operator.
+     *
+     * @param b  the input vector(s) on which the operator is applied
+     * @param x  the output vector(s) where the result is stored
+     */
+    void prolong_applyadd(const LinOp *b, LinOp *x) const
+    {
+        this->validate_parameters(b, x);
+        this->prolong_applyadd_impl(make_temporary_clone(exec_, b).get(),
+                                    make_temporary_clone(exec_, x).get());
+    }
+
+protected:
+    /**
+     * Sets the size of the prolong operator.
+     *
+     * @param value  the new size of the prolong operator
+     */
+    void set_prolong_size(const dim<2> &value) noexcept { size_ = value; }
+
+    virtual void prolong_apply_impl(const LinOp *b, LinOp *x) const = 0;
+
+    virtual void prolong_applyadd_impl(const LinOp *b, LinOp *x) const = 0;
+
+    /**
+     * Creates a ProlongOp with settings
+     *
+     * @param exec  Executor associated to the ProlongOp
+     * @param native_applyadd  the prolong applyadd is natively implemented in
+     *                         one kernel or not. This argument does not has
+     *                         effect yet.
+     * @param size  the size of ProlongOp.
+     *
+     * @note the multigrid is generated in the generation not the construction,
+     *       so needs to call `set_prolong_size` to set prolong size after
+     *       generation.
+     */
+    explicit ProlongOp(std::shared_ptr<const Executor> exec,
+                       bool native_applyadd = false,
+                       const gko::dim<2> &size = gko::dim<2>{})
+        : exec_(exec), native_applyadd_(native_applyadd), size_(size)
+    {}
+
+private:
+    /**
+     * Throws a DimensionMismatch exception if the parameters to
+     * `prolong_applyadd` are of the wrong size.
+     *
+     * @param b  vector(s) on which the operator is applied
+     * @param x  output vector(s)
+     */
+    void validate_parameters(const LinOp *b, LinOp *x) const
+    {
+        GKO_ASSERT_CONFORMANT(size_, b);
+        GKO_ASSERT_EQUAL_ROWS(size_, x);
+        GKO_ASSERT_EQUAL_COLS(b, x);
+    }
+
+    std::shared_ptr<const Executor> exec_;
+    gko::dim<2> size_;
+    bool native_applyadd_;
+};
+
+
+/**
+ * A class implementing this interface can be coarse generation operation.
+ *
+ * @ingroup multigrid
+ */
+class CoarseOp {
+public:
+    /**
+     * Gets the fine operator (matrix)
+     *
+     * @return the fine operator (matrix)
+     */
+    std::shared_ptr<const LinOp> get_fine_matrix() const { return fine_; }
+
+    /**
+     * Gets the coarse operator (matrix)
+     *
+     * @return the coarse operator (matrix)
+     */
+    std::shared_ptr<const LinOp> get_coarse_matrix() const { return coarse_; }
+
+protected:
+    /**
+     * Sets the fine and coarse information.
+     *
+     * @param fine  the matrix on fine level
+     * @param coarse  the matrix on coarse level
+     */
+    void set_fine_coarse(std::shared_ptr<const LinOp> fine,
+                         std::shared_ptr<const LinOp> coarse)
+    {
+        fine_ = fine;
+        coarse_ = coarse;
+    }
+
+private:
+    std::shared_ptr<const LinOp> coarse_;
+    std::shared_ptr<const LinOp> fine_;
+};
+
+
+/**
+ * A class implementing this interface can contain restrict, prolong, coarse
+ * generation operation.
+ *
+ * @ingroup multigrid
+ */
+class MultigridLevel : public RestrictOp, public ProlongOp, public CoarseOp {
+protected:
+    /**
+     * Sets the fine and coarse information.
+     *
+     * @param fine  the matrix on fine level
+     * @param coarse  the matrix on coarse level
+     */
+    void set_multigrid_level(std::shared_ptr<const LinOp> fine,
+                             std::shared_ptr<const LinOp> coarse)
+    {
+        this->set_fine_coarse(fine, coarse);
+        auto fine_dim = fine->get_size()[0];
+        auto coarse_dim = coarse->get_size()[0];
+        this->set_restrict_size(dim<2>{coarse_dim, fine_dim});
+        this->set_prolong_size(dim<2>{fine_dim, coarse_dim});
+    }
+
+    /**
+     * Creates a MultigridLevel with settings
+     *
+     * @param exec  Executor associated to the multigrid_level
+     * @param native_applyadd  the prolong applyadd is natively implemented in
+     *                         one kernel or not. This argument does not has
+     *                         effect yet.
+     *
+     * @note the multigrid is generated in the generation not the construction,
+     *       so needs to call `set_multigrid_level` to set corresponding
+     *       information after generation.
+     */
+    explicit MultigridLevel(std::shared_ptr<const Executor> exec,
+                            bool native_applyadd = false)
+        : RestrictOp(exec), ProlongOp(exec, native_applyadd)
+    {}
+};
+
+
+}  // namespace multigrid
+}  // namespace gko
+
+
+#endif  // GKO_PUBLIC_CORE_MULTIGRID_INTERFACE_HPP_
diff --git a/include/ginkgo/core/multigrid/restrict_prolong.hpp b/include/ginkgo/core/multigrid/restrict_prolong.hpp
deleted file mode 100644
index 63073cc21c3..00000000000
--- a/include/ginkgo/core/multigrid/restrict_prolong.hpp
+++ /dev/null
@@ -1,390 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GKO_CORE_MULTIGRID_RESTRICT_PROLONG_HPP_
-#define GKO_CORE_MULTIGRID_RESTRICT_PROLONG_HPP_
-
-
-#include <functional>
-#include <memory>
-
-
-#include <ginkgo/core/base/abstract_factory.hpp>
-#include <ginkgo/core/base/lin_op.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-
-
-namespace gko {
-namespace multigrid {
-
-
-/**
- * The RestrictProlong class can be used to construct restrict_apply and
- * prolong_applyadd. R is the restrict operator (fine level -> coarse
- * level) and P is the prolong operator (coarse level -> fine level).
- * restrict_apply(b, x) -> x = R(b)
- * prolong_applyadd(b, x) -> x = P(b) + x
- *
- * @ingroup RestrictProlong
- * @ingroup Multigrid
- */
-class RestrictProlong
-    : public EnableAbstractPolymorphicObject<RestrictProlong> {
-public:
-    /**
-     * Returns the coarse operator (matrix) which is R * matrix * P
-     *
-     * @return the coarse operator (matrix)
-     */
-    std::shared_ptr<const LinOp> get_coarse_operator() const { return coarse_; }
-
-    /**
-     * Applies a restrict operator to a vector (or a sequence of vectors).
-     *
-     * Performs the operation x = restrict(b).
-     *
-     * @param b  the input vector(s) on which the operator is applied
-     * @param x  the output vector(s) where the result is stored
-     *
-     * @return this
-     */
-    RestrictProlong *restrict_apply(const LinOp *b, LinOp *x)
-    {
-        this->validate_restrict_parameters(b, x);
-        auto exec = this->get_executor();
-        this->restrict_apply_impl(make_temporary_clone(exec, b).get(),
-                                  make_temporary_clone(exec, x).get());
-        return this;
-    }
-
-    /**
-     * @copydoc restrict_apply(const LinOp *, LinOp *)
-     */
-    const RestrictProlong *restrict_apply(const LinOp *b, LinOp *x) const
-    {
-        this->validate_restrict_parameters(b, x);
-        auto exec = this->get_executor();
-        this->restrict_apply_impl(make_temporary_clone(exec, b).get(),
-                                  make_temporary_clone(exec, x).get());
-        return this;
-    }
-
-    /**
-     * Applies a prolong operator to a vector (or a sequence of vectors).
-     *
-     * Performs the operation x = prolong(b) + x.
-     *
-     * @param b  the input vector(s) on which the operator is applied
-     * @param x  the output vector(s) where the result is stored
-     *
-     * @return this
-     */
-    RestrictProlong *prolong_applyadd(const LinOp *b, LinOp *x)
-    {
-        this->validate_prolong_parameters(b, x);
-        auto exec = this->get_executor();
-        this->prolong_applyadd_impl(make_temporary_clone(exec, b).get(),
-                                    make_temporary_clone(exec, x).get());
-        return this;
-    }
-
-    /**
-     * @copydoc prolong_applyadd(const LinOp *, LinOp *)
-     */
-    const RestrictProlong *prolong_applyadd(const LinOp *b, LinOp *x) const
-    {
-        this->validate_prolong_parameters(b, x);
-        auto exec = this->get_executor();
-        this->prolong_applyadd_impl(make_temporary_clone(exec, b).get(),
-                                    make_temporary_clone(exec, x).get());
-        return this;
-    }
-
-protected:
-    /**
-     * Throws a DimensionMismatch exception if the parameters to
-     * `restrict_apply` are of the wrong size.
-     *
-     * @param b  vector(s) on which the operator is applied
-     * @param x  output vector(s)
-     */
-    void validate_restrict_parameters(const LinOp *b, LinOp *x) const
-    {
-        auto restrict_dim = gko::dim<2>(coarse_dim_, fine_dim_);
-        GKO_ASSERT_CONFORMANT(restrict_dim, b);
-        GKO_ASSERT_EQUAL_ROWS(restrict_dim, x);
-        GKO_ASSERT_EQUAL_COLS(b, x);
-    }
-
-    /**
-     * Throws a DimensionMismatch exception if the parameters to
-     * `prolong_applyadd` are of the wrong size.
-     *
-     * @param b  vector(s) on which the operator is applied
-     * @param x  output vector(s)
-     */
-    void validate_prolong_parameters(const LinOp *b, LinOp *x) const
-    {
-        auto prolong_dim = gko::dim<2>(fine_dim_, coarse_dim_);
-        GKO_ASSERT_CONFORMANT(prolong_dim, b);
-        GKO_ASSERT_EQUAL_ROWS(prolong_dim, x);
-        GKO_ASSERT_EQUAL_COLS(b, x);
-    }
-
-    /**
-     * Implementers of RestrictProlong should override this function instead
-     * of restrict_apply(const LinOp *, LinOp *).
-     *
-     * Performs the operation x = restrict(b).
-     *
-     * @param b  the input vector(s) on which the operator is applied
-     * @param x  the output vector(s) where the result is stored
-     */
-    virtual void restrict_apply_impl(const LinOp *b, LinOp *x) const = 0;
-
-    /**
-     * Implementers of RestrictProlong should override this function instead
-     * of prolong_applyadd(const LinOp *, LinOp *).
-     *
-     * Performs the operation x += prolong(b).
-     *
-     * @param b  the input vector(s) on which the operator is applied
-     * @param x  the output vector(s) where the result is stored
-     */
-    virtual void prolong_applyadd_impl(const LinOp *b, LinOp *x) const = 0;
-
-    /**
-     * Sets the components of RestrictProlong
-     *
-     * @param coarse  the coarse matrix
-     * @param fine_dim  the fine_level size
-     */
-    void set_coarse_fine(std::shared_ptr<const LinOp> coarse,
-                         size_type fine_dim)
-    {
-        coarse_ = coarse;
-        fine_dim_ = fine_dim;
-        coarse_dim_ = coarse->get_size()[0];
-    }
-
-    /**
-     * Creates a RestrictProlong.
-     *
-     * @param exec  the executor where all the operations are performed
-     */
-    explicit RestrictProlong(std::shared_ptr<const Executor> exec)
-        : EnableAbstractPolymorphicObject<RestrictProlong>(exec)
-    {}
-
-private:
-    std::shared_ptr<const LinOp> coarse_{};
-    size_type fine_dim_;
-    size_type coarse_dim_;
-};
-
-
-/**
- * A RestrictProlongFactory represents a higher order mapping which transforms
- * one linear operator into the behavior between multigrid level.
- *
- * A RestrictProlongFactory transforms a linear operator $F$ (fine level) to
- * $R$ (restrict), $P$ (prolong), and $C=RFP$ (coarse level), which defines the
- * mapping between fine level and coarse level. R: F -> C and P: C -> F.
- *
- * @ingroup RestrictProlong
- * @ingroup Multigrid
- */
-class RestrictProlongFactory
-    : public AbstractFactory<RestrictProlong, std::shared_ptr<const LinOp>> {
-public:
-    using AbstractFactory<RestrictProlong,
-                          std::shared_ptr<const LinOp>>::AbstractFactory;
-
-    std::unique_ptr<RestrictProlong> generate(
-        std::shared_ptr<const LinOp> input) const
-    {
-        return AbstractFactory::generate(input);
-    }
-};
-
-
-/**
- * This is an alias for the EnableDefaultFactory mixin, which correctly sets the
- * template parameters to enable a subclass of RestrictProlongFactory.
- *
- * @tparam ConcreteFactory  the concrete factory which is being implemented
- *                          [CRTP parmeter]
- * @tparam ConcreteRstrPrlg  the concrete RestrictProlong type which this
- * factory produces, needs to have a constructor which takes a const
- * ConcreteFactory *, and an std::shared_ptr<const LinOp> as parameters.
- * @tparam ParametersType  a subclass of enable_parameters_type template which
- *                         defines all of the parameters of the factory
- * @tparam PolymorphicBase  parent of ConcreteFactory in the polymorphic
- *                          hierarchy, has to be a subclass of
- *                          RestrictProlongFactory
- *
- * @ingroup LinOp
- */
-template <typename ConcreteFactory, typename ConcreteRstrPrlg,
-          typename ParametersType,
-          typename PolymorphicBase = RestrictProlongFactory>
-using EnableDefaultRestrictProlongFactory =
-    EnableDefaultFactory<ConcreteFactory, ConcreteRstrPrlg, ParametersType,
-                         PolymorphicBase>;
-
-
-/**
- * The EnableRestrictProlong mixin can be used to provide sensible default
- * implementations of the majority of the RestrictProlong and PolymorphicObject
- * interface.
- *
- * The goal of the mixin is to facilitate the development of new
- * RestrictProlong, by enabling the implementers to focus on the important parts
- * of their operator, while the library takes care of generating the trivial
- * utility functions. The mixin will provide default implementations for the
- * entire PolymorphicObject interface, including a default implementation of
- * `copy_from` between objects of the new LinOp type. It will also hide the
- * default RestrictPrlong::restrict_apply() and
- * RestrictPrlong::prolong_applyadd() methods with versions that preserve the
- * static type of the object.
- *
- * Implementers of new RestrictProlong are required to specify only the
- * following aspects:
- *
- * 1.  Creation of the RestrictProlong: This can be facilitated via
- *     GKO_ENABLE_RESTRICT_PROLONG_FACTORY macro.
- * 2.  Application of the RestrictProlong: Implementers have to override
- *     RestrictPrlong::restrict_apply_impl() and
- *     RestrictPrlong::prolong_applyadd_impl() virtual methods and use
- *     set_coarse_fine() to set the generated results.
- *
- * @tparam ConcreteRstrPrlg  the concrete RestrictProlong which is being
- *                           implemented [CRTP parameter]
- * @tparam PolymorphicBase  parent of ConcreteRstrPrlg in the polymorphic
- *                          hierarchy, has to be a subclass of RestrictProlong
- *
- * @ingroup RestrictProlong
- * @ingroup Multigrid
- */
-template <typename ConcreteRstrPrlg, typename PolymorphicBase = RestrictProlong>
-class EnableRestrictProlong
-    : public EnablePolymorphicObject<ConcreteRstrPrlg, PolymorphicBase>,
-      public EnablePolymorphicAssignment<ConcreteRstrPrlg> {
-public:
-    using EnablePolymorphicObject<ConcreteRstrPrlg,
-                                  PolymorphicBase>::EnablePolymorphicObject;
-
-    ConcreteRstrPrlg *restrict_apply(const LinOp *b, LinOp *x)
-    {
-        this->validate_restrict_parameters(b, x);
-        auto exec = this->get_executor();
-        this->restrict_apply_impl(make_temporary_clone(exec, b).get(),
-                                  make_temporary_clone(exec, x).get());
-        return self();
-    }
-
-    const ConcreteRstrPrlg *restrict_apply(const LinOp *b, LinOp *x) const
-    {
-        this->validate_restrict_parameters(b, x);
-        auto exec = this->get_executor();
-        this->restrict_apply_impl(make_temporary_clone(exec, b).get(),
-                                  make_temporary_clone(exec, x).get());
-        return self();
-    }
-
-    ConcreteRstrPrlg *prolong_applyadd(const LinOp *b, LinOp *x)
-    {
-        this->validate_prolong_parameters(b, x);
-        auto exec = this->get_executor();
-        this->prolong_applyadd_impl(make_temporary_clone(exec, b).get(),
-                                    make_temporary_clone(exec, x).get());
-        return self();
-    }
-
-    const ConcreteRstrPrlg *prolong_applyadd(const LinOp *b, LinOp *x) const
-    {
-        this->validate_prolong_parameters(b, x);
-        auto exec = this->get_executor();
-        this->prolong_applyadd_impl(make_temporary_clone(exec, b).get(),
-                                    make_temporary_clone(exec, x).get());
-        return self();
-    }
-
-protected:
-    GKO_ENABLE_SELF(ConcreteRstrPrlg);
-};
-
-
-#define GKO_ENABLE_RESTRICT_PROLONG_FACTORY(_restrict_prolong,                \
-                                            _parameters_name, _factory_name)  \
-public:                                                                       \
-    const _parameters_name##_type &get_##_parameters_name() const             \
-    {                                                                         \
-        return _parameters_name##_;                                           \
-    }                                                                         \
-                                                                              \
-    class _factory_name                                                       \
-        : public ::gko::multigrid::EnableDefaultRestrictProlongFactory<       \
-              _factory_name, _restrict_prolong, _parameters_name##_type> {    \
-        friend class ::gko::EnablePolymorphicObject<                          \
-            _factory_name, ::gko::multigrid::RestrictProlongFactory>;         \
-        friend class ::gko::enable_parameters_type<_parameters_name##_type,   \
-                                                   _factory_name>;            \
-        explicit _factory_name(std::shared_ptr<const ::gko::Executor> exec)   \
-            : ::gko::multigrid::EnableDefaultRestrictProlongFactory<          \
-                  _factory_name, _restrict_prolong, _parameters_name##_type>( \
-                  std::move(exec))                                            \
-        {}                                                                    \
-        explicit _factory_name(std::shared_ptr<const ::gko::Executor> exec,   \
-                               const _parameters_name##_type &parameters)     \
-            : ::gko::multigrid::EnableDefaultRestrictProlongFactory<          \
-                  _factory_name, _restrict_prolong, _parameters_name##_type>( \
-                  std::move(exec), parameters)                                \
-        {}                                                                    \
-    };                                                                        \
-    friend ::gko::multigrid::EnableDefaultRestrictProlongFactory<             \
-        _factory_name, _restrict_prolong, _parameters_name##_type>;           \
-                                                                              \
-private:                                                                      \
-    _parameters_name##_type _parameters_name##_;                              \
-                                                                              \
-public:                                                                       \
-    static_assert(true,                                                       \
-                  "This assert is used to counter the false positive extra "  \
-                  "semi-colon warnings")
-
-
-}  // namespace multigrid
-}  // namespace gko
-
-
-#endif  // GKO_CORE_MULTIGRID_RESTRICT_PROLONG_HPP_
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 1942000117b..f83e6dbd82e 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -88,7 +88,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 #include <ginkgo/core/multigrid/amgx_pgm.hpp>
-#include <ginkgo/core/multigrid/restrict_prolong.hpp>
+#include <ginkgo/core/multigrid/interface.hpp>
 
 #include <ginkgo/core/preconditioner/ic.hpp>
 #include <ginkgo/core/preconditioner/ilu.hpp>
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index 87d81fb7ff8..a756f137708 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index 4d90f3037e8..f7a06d5dd60 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index edfb0493d1b..e7e356de485 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -68,7 +68,7 @@ class AmgxPgm : public ::testing::Test {
     using Mtx = gko::matrix::Csr<value_type, index_type>;
     using Vec = gko::matrix::Dense<value_type>;
     using RestrictProlong = gko::multigrid::AmgxPgm<value_type, index_type>;
-    using T = value_type;
+    using VT = value_type;
     using real_type = gko::remove_complex<value_type>;
     using WeightMtx = gko::matrix::Csr<real_type, index_type>;
     AmgxPgm()
@@ -78,20 +78,24 @@ class AmgxPgm : public ::testing::Test {
                               .with_max_unassigned_percentage(0.1)
                               .on(exec)),
           fine_b(gko::initialize<Vec>(
-              {I<T>({2.0, -1.0}), I<T>({-1.0, 2.0}), I<T>({0.0, -1.0}),
-               I<T>({3.0, -2.0}), I<T>({-2.0, 1.0})},
+              {I<VT>({2.0, -1.0}), I<VT>({-1.0, 2.0}), I<VT>({0.0, -1.0}),
+               I<VT>({3.0, -2.0}), I<VT>({-2.0, 1.0})},
               exec)),
-          coarse_b(gko::initialize<Vec>({I<T>({2.0, -1.0}), I<T>({0.0, -1.0})},
-                                        exec)),
+          coarse_b(gko::initialize<Vec>(
+              {I<VT>({2.0, -1.0}), I<VT>({0.0, -1.0})}, exec)),
           restrict_ans((gko::initialize<Vec>(
-              {I<T>({0.0, -1.0}), I<T>({2.0, 0.0})}, exec))),
+              {I<VT>({0.0, -1.0}), I<VT>({2.0, 0.0})}, exec))),
           prolong_ans(gko::initialize<Vec>(
-              {I<T>({0.0, -2.0}), I<T>({1.0, -2.0}), I<T>({1.0, -2.0}),
-               I<T>({0.0, -1.0}), I<T>({2.0, 1.0})},
+              {I<VT>({0.0, -2.0}), I<VT>({1.0, -2.0}), I<VT>({1.0, -2.0}),
+               I<VT>({0.0, -1.0}), I<VT>({2.0, 1.0})},
+              exec)),
+          prolong_applyans(gko::initialize<Vec>(
+              {I<VT>({2.0, -1.0}), I<VT>({0.0, -1.0}), I<VT>({2.0, -1.0}),
+               I<VT>({0.0, -1.0}), I<VT>({2.0, -1.0})},
               exec)),
           fine_x(gko::initialize<Vec>(
-              {I<T>({-2.0, -1.0}), I<T>({1.0, -1.0}), I<T>({-1.0, -1.0}),
-               I<T>({0.0, 0.0}), I<T>({0.0, 2.0})},
+              {I<VT>({-2.0, -1.0}), I<VT>({1.0, -1.0}), I<VT>({-1.0, -1.0}),
+               I<VT>({0.0, 0.0}), I<VT>({0.0, 2.0})},
               exec)),
           mtx(Mtx::create(exec, gko::dim<2>(5, 5), 15,
                           std::make_shared<typename Mtx::classical>())),
@@ -269,12 +273,13 @@ class AmgxPgm : public ::testing::Test {
     std::shared_ptr<Vec> fine_b;
     std::shared_ptr<Vec> restrict_ans;
     std::shared_ptr<Vec> prolong_ans;
+    std::shared_ptr<Vec> prolong_applyans;
     std::shared_ptr<Vec> fine_x;
     std::unique_ptr<typename RestrictProlong::Factory> amgxpgm_factory;
     std::unique_ptr<RestrictProlong> rstr_prlg;
 };
 
-TYPED_TEST_CASE(AmgxPgm, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(AmgxPgm, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(AmgxPgm, CanBeCopied)
@@ -288,7 +293,7 @@ TYPED_TEST(AmgxPgm, CanBeCopied)
     auto copy_mtx =
         static_cast<RestrictProlong *>(copy.get())->get_system_matrix();
     auto copy_agg = static_cast<RestrictProlong *>(copy.get())->get_const_agg();
-    auto copy_coarse = copy->get_coarse_operator();
+    auto copy_coarse = copy->get_coarse_matrix();
 
     this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
                                this->mtx.get());
@@ -310,7 +315,7 @@ TYPED_TEST(AmgxPgm, CanBeMoved)
     auto copy_mtx =
         static_cast<RestrictProlong *>(copy.get())->get_system_matrix();
     auto copy_agg = static_cast<RestrictProlong *>(copy.get())->get_const_agg();
-    auto copy_coarse = copy->get_coarse_operator();
+    auto copy_coarse = copy->get_coarse_matrix();
 
     this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
                                this->mtx.get());
@@ -333,7 +338,7 @@ TYPED_TEST(AmgxPgm, CanBeCloned)
         static_cast<RestrictProlong *>(clone.get())->get_system_matrix();
     auto clone_agg =
         static_cast<RestrictProlong *>(clone.get())->get_const_agg();
-    auto clone_coarse = clone->get_coarse_operator();
+    auto clone_coarse = clone->get_coarse_matrix();
 
     this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
                                this->mtx.get());
@@ -352,7 +357,7 @@ TYPED_TEST(AmgxPgm, CanBeCleared)
 
     auto mtx = static_cast<RestrictProlong *>(this->rstr_prlg.get())
                    ->get_system_matrix();
-    auto coarse = this->rstr_prlg->get_coarse_operator();
+    auto coarse = this->rstr_prlg->get_coarse_matrix();
     auto agg = static_cast<RestrictProlong *>(this->rstr_prlg.get())->get_agg();
 
     ASSERT_EQ(mtx, nullptr);
@@ -469,14 +474,14 @@ TYPED_TEST(AmgxPgm, Generate)
 
 TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
 {
-    std::unique_ptr<gko::multigrid::RestrictProlong> amgx_pgm{
-        this->amgxpgm_factory->generate(this->mtx)};
+    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
 
     // fine->coarse
     using Vec = typename TestFixture::Vec;
     using value_type = typename TestFixture::value_type;
     auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
     amgx_pgm->restrict_apply(this->fine_b.get(), x.get());
+
     GKO_ASSERT_MTX_NEAR(x, this->restrict_ans,
                         gko::remove_complex<value_type>{0});
 }
@@ -485,16 +490,29 @@ TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
 TYPED_TEST(AmgxPgm, CoarseFineProlongApplyadd)
 {
     using value_type = typename TestFixture::value_type;
-    std::unique_ptr<gko::multigrid::RestrictProlong> amgx_pgm{
-        this->amgxpgm_factory->generate(this->mtx)};
+    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
     auto x = gko::clone(this->fine_x);
 
     amgx_pgm->prolong_applyadd(this->coarse_b.get(), x.get());
+
     GKO_ASSERT_MTX_NEAR(x, this->prolong_ans,
                         gko::remove_complex<value_type>{0});
 }
 
 
+TYPED_TEST(AmgxPgm, CoarseFineProlongApply)
+{
+    using value_type = typename TestFixture::value_type;
+    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
+    auto x = gko::clone(this->fine_x);
+
+    amgx_pgm->prolong_apply(this->coarse_b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, this->prolong_applyans,
+                        gko::remove_complex<value_type>{0});
+}
+
+
 TYPED_TEST(AmgxPgm, FindStrongestNeighbor)
 {
     using index_type = typename TestFixture::index_type;

From 8ef3209277f7886e0388cb6b8dfa7b5416907af6 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 21 Jan 2021 13:05:42 +0800
Subject: [PATCH 0475/1008] add multigrid_level default apply and test

---
 include/ginkgo/core/multigrid/amgx_pgm.hpp    | 20 +------
 include/ginkgo/core/multigrid/interface.hpp   | 54 ++++++++++++++++-
 reference/test/multigrid/amgx_pgm_kernels.cpp | 59 ++++++++++++++-----
 3 files changed, 101 insertions(+), 32 deletions(-)

diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index f0777b602d7..b902e64e4da 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -149,28 +149,14 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
 
     void apply_impl(const LinOp *b, LinOp *x) const override
     {
-        // x = op(b) = prolong(coarse(restrict(b)))
-        auto exec = this->get_executor();
-        auto num_cols = b->get_size()[1];
-        auto coarse = this->get_coarse_matrix();
-        auto coarse_b = matrix::Dense<ValueType>::create(
-            exec, dim<2>{coarse->get_size()[1], num_cols});
-        auto coarse_x = matrix::Dense<ValueType>::create(
-            exec, dim<2>{coarse->get_size()[0], num_cols});
-        this->restrict_apply(b, lend(coarse_b));
-        coarse->apply(lend(coarse_b), lend(coarse_x));
-        this->prolong_apply(lend(coarse_x), x);
+        this->template multigrid_level_default_apply<ValueType>(b, x);
     }
 
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override
     {
-        auto dense_x = as<matrix::Dense<ValueType>>(x);
-
-        auto x_clone = dense_x->clone();
-        this->apply(b, x_clone.get());
-        dense_x->scale(beta);
-        dense_x->add_scaled(alpha, x_clone.get());
+        this->template multigrid_level_default_apply<ValueType>(alpha, b, beta,
+                                                                x);
     }
 
     explicit AmgxPgm(std::shared_ptr<const Executor> exec)
diff --git a/include/ginkgo/core/multigrid/interface.hpp b/include/ginkgo/core/multigrid/interface.hpp
index 25fbbe8e638..d77c27aaa45 100644
--- a/include/ginkgo/core/multigrid/interface.hpp
+++ b/include/ginkgo/core/multigrid/interface.hpp
@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
+
 namespace gko {
 namespace multigrid {
 
@@ -267,6 +268,54 @@ class MultigridLevel : public RestrictOp, public ProlongOp, public CoarseOp {
         this->set_prolong_size(dim<2>{fine_dim, coarse_dim});
     }
 
+    /**
+     * A default apply of multigrid_level
+     *
+     * Performs the operation x = op(b) = prolong(coarse(restrict(b)))
+     *
+     * @tparam ValueType  precision of input and result vectors
+     *
+     * @param b  the input vector(s) on which the operator is applied
+     * @param x  the output vector(s) where the result is stored
+     */
+    template <typename ValueType>
+    void multigrid_level_default_apply(const LinOp *b, LinOp *x) const
+    {
+        auto num_cols = b->get_size()[1];
+        auto coarse = this->get_coarse_matrix();
+        auto coarse_b = matrix::Dense<ValueType>::create(
+            exec_, dim<2>{coarse->get_size()[1], num_cols});
+        auto coarse_x = matrix::Dense<ValueType>::create(
+            exec_, dim<2>{coarse->get_size()[0], num_cols});
+        this->restrict_apply(b, lend(coarse_b));
+        coarse->apply(lend(coarse_b), lend(coarse_x));
+        this->prolong_apply(lend(coarse_x), x);
+    }
+
+    /**
+     * A default advanced_apply of multigrid_level
+     *
+     * Performs the operation x = alpha * op(b) + beta * x
+     * = alpha * prolong(coarse(restrict(b))) + beta * x
+     *
+     * @tparam ValueType  precision of input and result vectors
+     *
+     * @param b  the input vector(s) on which the operator is applied
+     * @param x  the output vector(s) where the result is stored
+     */
+    template <typename ValueType>
+    void multigrid_level_default_apply(const LinOp *alpha, const LinOp *b,
+                                       const LinOp *beta, LinOp *x) const
+    {
+        auto dense_x = as<matrix::Dense<ValueType>>(x);
+
+        auto x_clone = dense_x->clone();
+        this->template multigrid_level_default_apply<ValueType>(b,
+                                                                x_clone.get());
+        dense_x->scale(beta);
+        dense_x->add_scaled(alpha, x_clone.get());
+    }
+
     /**
      * Creates a MultigridLevel with settings
      *
@@ -281,8 +330,11 @@ class MultigridLevel : public RestrictOp, public ProlongOp, public CoarseOp {
      */
     explicit MultigridLevel(std::shared_ptr<const Executor> exec,
                             bool native_applyadd = false)
-        : RestrictOp(exec), ProlongOp(exec, native_applyadd)
+        : exec_(exec), RestrictOp(exec), ProlongOp(exec, native_applyadd)
     {}
+
+private:
+    std::shared_ptr<const Executor> exec_;
 };
 
 
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index e7e356de485..8a92e750ba1 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -289,7 +289,6 @@ TYPED_TEST(AmgxPgm, CanBeCopied)
     auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
 
     copy->copy_from(this->rstr_prlg.get());
-
     auto copy_mtx =
         static_cast<RestrictProlong *>(copy.get())->get_system_matrix();
     auto copy_agg = static_cast<RestrictProlong *>(copy.get())->get_const_agg();
@@ -311,7 +310,6 @@ TYPED_TEST(AmgxPgm, CanBeMoved)
     auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
 
     copy->copy_from(std::move(this->rstr_prlg));
-
     auto copy_mtx =
         static_cast<RestrictProlong *>(copy.get())->get_system_matrix();
     auto copy_agg = static_cast<RestrictProlong *>(copy.get())->get_const_agg();
@@ -333,7 +331,6 @@ TYPED_TEST(AmgxPgm, CanBeCloned)
     auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
 
     auto clone = this->rstr_prlg->clone();
-
     auto clone_mtx =
         static_cast<RestrictProlong *>(clone.get())->get_system_matrix();
     auto clone_agg =
@@ -354,7 +351,6 @@ TYPED_TEST(AmgxPgm, CanBeCleared)
     using RestrictProlong = typename TestFixture::RestrictProlong;
 
     this->rstr_prlg->clear();
-
     auto mtx = static_cast<RestrictProlong *>(this->rstr_prlg.get())
                    ->get_system_matrix();
     auto coarse = this->rstr_prlg->get_coarse_matrix();
@@ -376,8 +372,7 @@ TYPED_TEST(AmgxPgm, RestrictApply)
     gko::kernels::reference::amgx_pgm::restrict_apply(
         this->exec, this->agg, this->fine_b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, this->restrict_ans,
-                        gko::remove_complex<value_type>{0});
+    GKO_ASSERT_MTX_NEAR(x, this->restrict_ans, r<value_type>::value);
 }
 
 TYPED_TEST(AmgxPgm, ProlongApplyadd)
@@ -388,8 +383,7 @@ TYPED_TEST(AmgxPgm, ProlongApplyadd)
     gko::kernels::reference::amgx_pgm::prolong_applyadd(
         this->exec, this->agg, this->coarse_b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, this->prolong_ans,
-                        gko::remove_complex<value_type>{0});
+    GKO_ASSERT_MTX_NEAR(x, this->prolong_ans, r<value_type>::value);
 }
 
 TYPED_TEST(AmgxPgm, MatchEdge)
@@ -482,8 +476,7 @@ TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
     auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
     amgx_pgm->restrict_apply(this->fine_b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, this->restrict_ans,
-                        gko::remove_complex<value_type>{0});
+    GKO_ASSERT_MTX_NEAR(x, this->restrict_ans, r<value_type>::value);
 }
 
 
@@ -495,8 +488,7 @@ TYPED_TEST(AmgxPgm, CoarseFineProlongApplyadd)
 
     amgx_pgm->prolong_applyadd(this->coarse_b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, this->prolong_ans,
-                        gko::remove_complex<value_type>{0});
+    GKO_ASSERT_MTX_NEAR(x, this->prolong_ans, r<value_type>::value);
 }
 
 
@@ -508,8 +500,47 @@ TYPED_TEST(AmgxPgm, CoarseFineProlongApply)
 
     amgx_pgm->prolong_apply(this->coarse_b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, this->prolong_applyans,
-                        gko::remove_complex<value_type>{0});
+    GKO_ASSERT_MTX_NEAR(x, this->prolong_applyans, r<value_type>::value);
+}
+
+
+TYPED_TEST(AmgxPgm, Apply)
+{
+    using VT = typename TestFixture::value_type;
+    using Vec = typename TestFixture::Vec;
+    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
+    auto b = gko::clone(this->fine_x);
+    auto x = gko::clone(this->fine_x);
+    auto exec = amgx_pgm->get_executor();
+    auto answer = gko::initialize<Vec>(
+        {I<VT>({-23.0, 5.0}), I<VT>({17.0, -5.0}), I<VT>({-23.0, 5.0}),
+         I<VT>({17.0, -5.0}), I<VT>({-23.0, 5.0})},
+        exec);
+
+    amgx_pgm->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, answer, r<VT>::value);
+}
+
+
+TYPED_TEST(AmgxPgm, AdvancedApply)
+{
+    using VT = typename TestFixture::value_type;
+    using Vec = typename TestFixture::Vec;
+    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
+    auto b = gko::clone(this->fine_x);
+    auto x = gko::clone(this->fine_x);
+    auto exec = amgx_pgm->get_executor();
+    auto alpha = gko::initialize<Vec>({1.0}, exec);
+    auto beta = gko::initialize<Vec>({2.0}, exec);
+    auto answer = gko::initialize<Vec>(
+        {I<VT>({-27.0, 3.0}), I<VT>({19.0, -7.0}), I<VT>({-25.0, 3.0}),
+         I<VT>({17.0, -5.0}), I<VT>({-23.0, 9.0})},
+        exec);
+
+    amgx_pgm->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, answer, r<VT>::value);
 }
 
 

From 26fc2657917989ac4c8c7d48f5c9be09c2f072d3 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 25 Jan 2021 13:10:08 +0800
Subject: [PATCH 0476/1008] Apply suggestions from code review - use consistent
 indextype of num_agg/num_unagg - format - use read to build the csr - improve
 document

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/multigrid/amgx_pgm.cpp                   |  12 +-
 core/multigrid/amgx_pgm_kernels.hpp           |   4 +-
 cuda/multigrid/amgx_pgm_kernels.cu            |  24 +--
 dpcpp/multigrid/amgx_pgm_kernels.dp.cpp       |  24 +--
 hip/multigrid/amgx_pgm_kernels.hip.cpp        |  24 +--
 include/ginkgo/core/multigrid/amgx_pgm.hpp    |   2 +-
 include/ginkgo/core/multigrid/interface.hpp   |  43 ++--
 omp/multigrid/amgx_pgm_kernels.cpp            |  24 +--
 reference/multigrid/amgx_pgm_kernels.cpp      |   4 +-
 reference/test/multigrid/amgx_pgm_kernels.cpp | 192 ++++++------------
 10 files changed, 131 insertions(+), 222 deletions(-)

diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index a0fcc72bb2b..b7a697fb5a5 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -103,8 +103,8 @@ void AmgxPgm<ValueType, IndexType>::generate()
     const matrix_type *amgxpgm_op = nullptr;
     // Store the csr matrix if needed
     auto amgxpgm_op_unique_ptr = matrix_type::create(exec);
-    if (!(amgxpgm_op =
-              dynamic_cast<const matrix_type *>(system_matrix_.get()))) {
+    amgxpgm_op = dynamic_cast<const matrix_type *>(system_matrix_.get());
+    if (!amgxpgm_op) {
         // if original matrix is not csr, converting it to csr.
         as<ConvertibleTo<matrix_type>>(this->system_matrix_.get())
             ->convert_to(amgxpgm_op_unique_ptr.get());
@@ -114,8 +114,8 @@ void AmgxPgm<ValueType, IndexType>::generate()
     // Initial agg = -1
     exec->run(amgx_pgm::make_fill_array(agg_.get_data(), agg_.get_num_elems(),
                                         -one<IndexType>()));
-    size_type num_unagg{0};
-    size_type num_unagg_prev{0};
+    IndexType num_unagg{0};
+    IndexType num_unagg_prev{0};
     // TODO: if mtx is a hermitian matrix, weight_mtx = abs(mtx)
     // compute weight_mtx = (abs(mtx) + abs(mtx'))/2;
     auto abs_mtx = amgxpgm_op->compute_absolute();
@@ -145,14 +145,14 @@ void AmgxPgm<ValueType, IndexType>::generate()
         num_unagg_prev = num_unagg;
     }
     // Handle the left unassign points
-    if (num_unagg != 0 && intermediate_agg.get_num_elems() > 0) {
+    if (num_unagg != 0 && parameters_.deterministic) {
         // copy the agg to intermediate_agg
         intermediate_agg = agg_;
     }
     // Assign all left points
     exec->run(amgx_pgm::make_assign_to_exist_agg(weight_mtx.get(), diag.get(),
                                                  agg_, intermediate_agg));
-    size_type num_agg;
+    IndexType num_agg;
     // Renumber the index
     exec->run(amgx_pgm::make_renumber(agg_, &num_agg));
 
diff --git a/core/multigrid/amgx_pgm_kernels.hpp b/core/multigrid/amgx_pgm_kernels.hpp
index b86099b23c6..cbe44124472 100644
--- a/core/multigrid/amgx_pgm_kernels.hpp
+++ b/core/multigrid/amgx_pgm_kernels.hpp
@@ -68,11 +68,11 @@ namespace amgx_pgm {
 
 #define GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL(_itype)           \
     void count_unagg(std::shared_ptr<const DefaultExecutor> exec, \
-                     const Array<_itype> &agg, size_type *num_unagg)
+                     const Array<_itype> &agg, _itype *num_unagg)
 
 #define GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL(_itype)           \
     void renumber(std::shared_ptr<const DefaultExecutor> exec, \
-                  Array<_itype> &agg, size_type *num_agg)
+                  Array<_itype> &agg, _itype *num_agg)
 
 #define GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR(ValueType, IndexType) \
     void find_strongest_neighbor(                                          \
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index 937faa54616..2adc3fb19fe 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -93,14 +93,14 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL);
 template <typename IndexType>
 void count_unagg(std::shared_ptr<const CudaExecutor> exec,
                  const Array<IndexType> &agg,
-                 size_type *num_unagg) GKO_NOT_IMPLEMENTED;
+                 IndexType *num_unagg) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL);
 
 
 template <typename IndexType>
 void renumber(std::shared_ptr<const CudaExecutor> exec, Array<IndexType> &agg,
-              size_type *num_agg) GKO_NOT_IMPLEMENTED;
+              IndexType *num_agg) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
 
@@ -110,24 +110,18 @@ void find_strongest_neighbor(
     std::shared_ptr<const CudaExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *weight_mtx,
     const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
-    Array<IndexType> &strongest_neighbor)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    Array<IndexType> &strongest_neighbor) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
 
 
 template <typename ValueType, typename IndexType>
-void assign_to_exist_agg(std::shared_ptr<const CudaExecutor> exec,
-                         const matrix::Csr<ValueType, IndexType> *weight_mtx,
-                         const matrix::Diagonal<ValueType> *diag,
-                         Array<IndexType> &agg,
-                         Array<IndexType> &intermediate_agg)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+void assign_to_exist_agg(
+    std::shared_ptr<const CudaExecutor> exec,
+    const matrix::Csr<ValueType, IndexType> *weight_mtx,
+    const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
+    Array<IndexType> &intermediate_agg) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
@@ -138,9 +132,7 @@ void amgx_pgm_generate(std::shared_ptr<const CudaExecutor> exec,
                        const matrix::Csr<ValueType, IndexType> *source,
                        const Array<IndexType> &agg,
                        matrix::Csr<ValueType, IndexType> *coarse)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
 
diff --git a/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp b/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
index 3e7ba1a3329..98d42dcd7f3 100644
--- a/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
+++ b/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
@@ -86,14 +86,14 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL);
 template <typename IndexType>
 void count_unagg(std::shared_ptr<const DpcppExecutor> exec,
                  const Array<IndexType> &agg,
-                 size_type *num_unagg) GKO_NOT_IMPLEMENTED;
+                 IndexType *num_unagg) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL);
 
 
 template <typename IndexType>
 void renumber(std::shared_ptr<const DpcppExecutor> exec, Array<IndexType> &agg,
-              size_type *num_agg) GKO_NOT_IMPLEMENTED;
+              IndexType *num_agg) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
 
@@ -103,24 +103,18 @@ void find_strongest_neighbor(
     std::shared_ptr<const DpcppExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *weight_mtx,
     const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
-    Array<IndexType> &strongest_neighbor)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    Array<IndexType> &strongest_neighbor) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
 
 
 template <typename ValueType, typename IndexType>
-void assign_to_exist_agg(std::shared_ptr<const DpcppExecutor> exec,
-                         const matrix::Csr<ValueType, IndexType> *weight_mtx,
-                         const matrix::Diagonal<ValueType> *diag,
-                         Array<IndexType> &agg,
-                         Array<IndexType> &intermediate_agg)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+void assign_to_exist_agg(
+    std::shared_ptr<const DpcppExecutor> exec,
+    const matrix::Csr<ValueType, IndexType> *weight_mtx,
+    const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
+    Array<IndexType> &intermediate_agg) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
@@ -131,9 +125,7 @@ void amgx_pgm_generate(std::shared_ptr<const DpcppExecutor> exec,
                        const matrix::Csr<ValueType, IndexType> *source,
                        const Array<IndexType> &agg,
                        matrix::Csr<ValueType, IndexType> *coarse)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
 
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index 5d76fea55c4..80eb5d768ec 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -93,14 +93,14 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL);
 template <typename IndexType>
 void count_unagg(std::shared_ptr<const HipExecutor> exec,
                  const Array<IndexType> &agg,
-                 size_type *num_unagg) GKO_NOT_IMPLEMENTED;
+                 IndexType *num_unagg) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL);
 
 
 template <typename IndexType>
 void renumber(std::shared_ptr<const HipExecutor> exec, Array<IndexType> &agg,
-              size_type *num_agg) GKO_NOT_IMPLEMENTED;
+              IndexType *num_agg) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
 
@@ -110,24 +110,18 @@ void find_strongest_neighbor(
     std::shared_ptr<const HipExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *weight_mtx,
     const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
-    Array<IndexType> &strongest_neighbor)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    Array<IndexType> &strongest_neighbor) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
 
 
 template <typename ValueType, typename IndexType>
-void assign_to_exist_agg(std::shared_ptr<const HipExecutor> exec,
-                         const matrix::Csr<ValueType, IndexType> *weight_mtx,
-                         const matrix::Diagonal<ValueType> *diag,
-                         Array<IndexType> &agg,
-                         Array<IndexType> &intermediate_agg)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+void assign_to_exist_agg(
+    std::shared_ptr<const HipExecutor> exec,
+    const matrix::Csr<ValueType, IndexType> *weight_mtx,
+    const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
+    Array<IndexType> &intermediate_agg) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
@@ -138,9 +132,7 @@ void amgx_pgm_generate(std::shared_ptr<const HipExecutor> exec,
                        const matrix::Csr<ValueType, IndexType> *source,
                        const Array<IndexType> &agg,
                        matrix::Csr<ValueType, IndexType> *coarse)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
 
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index b902e64e4da..6b905a949c5 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -124,7 +124,7 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
         /**
          * The maximum ratio of unassigned number, which is valid in the
          * interval 0.0 ~ 1.0. We use the same default value as NVIDIA AMGX
-         * Rerference Manual (October 2017, API Version 2).
+         * Reference Manual (October 2017, API Version 2).
          */
         double GKO_FACTORY_PARAMETER(max_unassigned_percentage, 0.05);
 
diff --git a/include/ginkgo/core/multigrid/interface.hpp b/include/ginkgo/core/multigrid/interface.hpp
index d77c27aaa45..26795ac1e8c 100644
--- a/include/ginkgo/core/multigrid/interface.hpp
+++ b/include/ginkgo/core/multigrid/interface.hpp
@@ -39,7 +39,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/abstract_factory.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
@@ -48,16 +50,20 @@ namespace multigrid {
 
 
 /**
- * A class implementing this interface can be restrict operation.
+ * This interface allows to implement the restrict operation of
+ * a multigrid method.
  *
- * @ingroup multigrid
+ * Implementers of this interface only need to overload the
+ * RestrictOp::restrict_apply_impl() method.
+ *
+ * @ingroup Multigrid
  */
 class RestrictOp {
 public:
     /**
-     * Applies a prolong operator to a vector (or a sequence of vectors).
+     * Applies a restrict operator to a vector (or a sequence of vectors).
      *
-     * Performs the operation x = op(b), where op is this prolong operator.
+     * Performs the operation x = op(b), where op is this restrict operator.
      *
      * @param b  the input vector(s) on which the operator is applied
      * @param x  the output vector(s) where the result is stored
@@ -86,8 +92,8 @@ class RestrictOp {
      * @param size  the size of RestrictOp.
      *
      * @note the multigrid is generated in the generation not the construction,
-     *       so needs to call `set_restrict_size` to set restrict size after
-     *       generation.
+     *       so calling `set_restrict_size` is needed to set the restrict
+     *       size after generation.
      */
     explicit RestrictOp(std::shared_ptr<const Executor> exec,
                         const gko::dim<2> &size = gko::dim<2>{})
@@ -115,9 +121,14 @@ class RestrictOp {
 
 
 /**
- * A class implementing this interface can be prolong operation.
+ * This interface allows to implement the prolongation operation
+ * of a multigrid method.
+ *
+ * Implementers of this interface need to overload both the
+ * ProlongOp::prolong_apply_impl() and
+ * ProlongOp::prolong_applyadd_impl() methods.
  *
- * @ingroup multigrid
+ * @ingroup Multigrid
  */
 class ProlongOp {
 public:
@@ -204,9 +215,10 @@ class ProlongOp {
 
 
 /**
- * A class implementing this interface can be coarse generation operation.
+ * This class provides functionality for managing the coarse and fine operator
+ * for a multigrid method.
  *
- * @ingroup multigrid
+ * @ingroup Multigrid
  */
 class CoarseOp {
 public:
@@ -246,9 +258,16 @@ class CoarseOp {
 
 /**
  * A class implementing this interface can contain restrict, prolong, coarse
- * generation operation.
+ * generation operation. MultigridLevel provides multigrid_level_default_apply
+ * to represent op(b) = prolong(coarse(restrict(b))). Those classes implementing
+ * all MultigridLevel operation must inherit this class.
+ *
+ * Implementers of this interface need to overloadthe
+ * MultigridLevel::prolong_apply_impl(),
+ * MultigridLevel::prolong_applyadd_impl(), and
+ * MultigridLevel::restrict_apply_impl() methods.
  *
- * @ingroup multigrid
+ * @ingroup Multigrid
  */
 class MultigridLevel : public RestrictOp, public ProlongOp, public CoarseOp {
 protected:
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index a756f137708..82a40e33b22 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -90,14 +90,14 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL);
 template <typename IndexType>
 void count_unagg(std::shared_ptr<const OmpExecutor> exec,
                  const Array<IndexType> &agg,
-                 size_type *num_unagg) GKO_NOT_IMPLEMENTED;
+                 IndexType *num_unagg) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL);
 
 
 template <typename IndexType>
 void renumber(std::shared_ptr<const OmpExecutor> exec, Array<IndexType> &agg,
-              size_type *num_agg) GKO_NOT_IMPLEMENTED;
+              IndexType *num_agg) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
 
@@ -107,24 +107,18 @@ void find_strongest_neighbor(
     std::shared_ptr<const OmpExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *weight_mtx,
     const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
-    Array<IndexType> &strongest_neighbor)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+    Array<IndexType> &strongest_neighbor) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
 
 
 template <typename ValueType, typename IndexType>
-void assign_to_exist_agg(std::shared_ptr<const OmpExecutor> exec,
-                         const matrix::Csr<ValueType, IndexType> *weight_mtx,
-                         const matrix::Diagonal<ValueType> *diag,
-                         Array<IndexType> &agg,
-                         Array<IndexType> &intermediate_agg)
-{
-    GKO_NOT_IMPLEMENTED;
-}
+void assign_to_exist_agg(
+    std::shared_ptr<const OmpExecutor> exec,
+    const matrix::Csr<ValueType, IndexType> *weight_mtx,
+    const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
+    Array<IndexType> &intermediate_agg) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
@@ -135,9 +129,7 @@ void amgx_pgm_generate(std::shared_ptr<const OmpExecutor> exec,
                        const matrix::Csr<ValueType, IndexType> *source,
                        const Array<IndexType> &agg,
                        matrix::Csr<ValueType, IndexType> *coarse)
-{
     GKO_NOT_IMPLEMENTED;
-}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
 
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index f7a06d5dd60..159eeba9402 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -132,7 +132,7 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL);
 
 template <typename IndexType>
 void count_unagg(std::shared_ptr<const ReferenceExecutor> exec,
-                 const Array<IndexType> &agg, size_type *num_unagg)
+                 const Array<IndexType> &agg, IndexType *num_unagg)
 {
     IndexType unagg = 0;
     for (size_type i = 0; i < agg.get_num_elems(); i++) {
@@ -146,7 +146,7 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL);
 
 template <typename IndexType>
 void renumber(std::shared_ptr<const ReferenceExecutor> exec,
-              Array<IndexType> &agg, size_type *num_agg)
+              Array<IndexType> &agg, IndexType *num_agg)
 {
     const auto num = agg.get_num_elems();
     Array<IndexType> agg_map(exec, num + 1);
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index 8a92e750ba1..ba367ffca55 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -114,16 +114,13 @@ class AmgxPgm : public ::testing::Test {
     void create_mtx(Mtx *fine, WeightMtx *weight, gko::Array<index_type> *agg,
                     Mtx *coarse)
     {
-        auto vals = fine->get_values();
-        auto cols = fine->get_col_idxs();
-        auto rows = fine->get_row_ptrs();
-        auto w_vals = weight->get_values();
-        auto w_cols = weight->get_col_idxs();
-        auto w_rows = weight->get_row_ptrs();
         auto agg_val = agg->get_data();
-        auto c_vals = coarse->get_values();
-        auto c_cols = coarse->get_col_idxs();
-        auto c_rows = coarse->get_row_ptrs();
+        agg_val[0] = 0;
+        agg_val[1] = 1;
+        agg_val[2] = 0;
+        agg_val[3] = 1;
+        agg_val[4] = 0;
+
         /* this matrix is stored:
          *  5 -3 -3  0  0
          * -3  5  0 -2 -1
@@ -131,44 +128,22 @@ class AmgxPgm : public ::testing::Test {
          *  0 -3  0  5  0
          *  0 -2 -2  0  5
          */
-        vals[0] = 5;
-        vals[1] = -3;
-        vals[2] = -3;
-        vals[3] = -3;
-        vals[4] = 5;
-        vals[5] = -2;
-        vals[6] = -1;
-        vals[7] = -3;
-        vals[8] = 5;
-        vals[9] = -1;
-        vals[10] = -3;
-        vals[11] = 5;
-        vals[12] = -2;
-        vals[13] = -2;
-        vals[14] = 5;
-
-        rows[0] = 0;
-        rows[1] = 3;
-        rows[2] = 7;
-        rows[3] = 10;
-        rows[4] = 12;
-        rows[5] = 15;
-
-        cols[0] = 0;
-        cols[1] = 1;
-        cols[2] = 2;
-        cols[3] = 0;
-        cols[4] = 1;
-        cols[5] = 3;
-        cols[6] = 4;
-        cols[7] = 0;
-        cols[8] = 2;
-        cols[9] = 4;
-        cols[10] = 1;
-        cols[11] = 3;
-        cols[12] = 1;
-        cols[13] = 2;
-        cols[14] = 4;
+        fine->read({{5, 5},
+                    {{0, 0, 5},
+                     {0, 1, -3},
+                     {0, 2, -3},
+                     {1, 0, -3},
+                     {1, 1, 5},
+                     {1, 3, -2},
+                     {1, 4, -1},
+                     {2, 0, -3},
+                     {2, 2, 5},
+                     {2, 4, -1},
+                     {3, 1, -3},
+                     {3, 3, 5},
+                     {4, 1, -2},
+                     {4, 2, -2},
+                     {4, 4, 5}}});
 
         /* weight matrix is stored:
          * 5   3   3   0   0
@@ -177,68 +152,28 @@ class AmgxPgm : public ::testing::Test {
          * 0   2.5 0   5   0
          * 0   1.5 1.5 0   5
          */
-        w_vals[0] = 5;
-        w_vals[1] = 3;
-        w_vals[2] = 3;
-        w_vals[3] = 3;
-        w_vals[4] = 5;
-        w_vals[5] = 2.5;
-        w_vals[6] = 1.5;
-        w_vals[7] = 3;
-        w_vals[8] = 5;
-        w_vals[9] = 1.5;
-        w_vals[10] = 2.5;
-        w_vals[11] = 5;
-        w_vals[12] = 1.5;
-        w_vals[13] = 1.5;
-        w_vals[14] = 5;
-
-        w_rows[0] = 0;
-        w_rows[1] = 3;
-        w_rows[2] = 7;
-        w_rows[3] = 10;
-        w_rows[4] = 12;
-        w_rows[5] = 15;
-
-        w_cols[0] = 0;
-        w_cols[1] = 1;
-        w_cols[2] = 2;
-        w_cols[3] = 0;
-        w_cols[4] = 1;
-        w_cols[5] = 3;
-        w_cols[6] = 4;
-        w_cols[7] = 0;
-        w_cols[8] = 2;
-        w_cols[9] = 4;
-        w_cols[10] = 1;
-        w_cols[11] = 3;
-        w_cols[12] = 1;
-        w_cols[13] = 2;
-        w_cols[14] = 4;
-
-        agg_val[0] = 0;
-        agg_val[1] = 1;
-        agg_val[2] = 0;
-        agg_val[3] = 1;
-        agg_val[4] = 0;
+        weight->read({{5, 5},
+                      {{0, 0, 5.0},
+                       {0, 1, 3.0},
+                       {0, 2, 3.0},
+                       {1, 0, 3.0},
+                       {1, 1, 5.0},
+                       {1, 3, 2.5},
+                       {1, 4, 1.5},
+                       {2, 0, 3.0},
+                       {2, 2, 5.0},
+                       {2, 4, 1.5},
+                       {3, 1, 2.5},
+                       {3, 3, 5.0},
+                       {4, 1, 1.5},
+                       {4, 2, 1.5},
+                       {4, 4, 5.0}}});
 
         /* this coarse is stored:
          *  6 -5
          * -4  5
          */
-        c_vals[0] = 6;
-        c_vals[1] = -5;
-        c_vals[2] = -4;
-        c_vals[3] = 5;
-
-        c_rows[0] = 0;
-        c_rows[1] = 2;
-        c_rows[2] = 4;
-
-        c_cols[0] = 0;
-        c_cols[1] = 1;
-        c_cols[2] = 0;
-        c_cols[3] = 1;
+        coarse->read({{2, 2}, {{0, 0, 6}, {0, 1, -5}, {1, 0, -4}, {1, 1, 5}}});
     }
 
     static void assert_same_matrices(const Mtx *m1, const Mtx *m2)
@@ -375,6 +310,7 @@ TYPED_TEST(AmgxPgm, RestrictApply)
     GKO_ASSERT_MTX_NEAR(x, this->restrict_ans, r<value_type>::value);
 }
 
+
 TYPED_TEST(AmgxPgm, ProlongApplyadd)
 {
     using value_type = typename TestFixture::value_type;
@@ -386,6 +322,7 @@ TYPED_TEST(AmgxPgm, ProlongApplyadd)
     GKO_ASSERT_MTX_NEAR(x, this->prolong_ans, r<value_type>::value);
 }
 
+
 TYPED_TEST(AmgxPgm, MatchEdge)
 {
     using index_type = typename TestFixture::index_type;
@@ -411,12 +348,13 @@ TYPED_TEST(AmgxPgm, MatchEdge)
     ASSERT_EQ(agg_val[4], -1);
 }
 
+
 TYPED_TEST(AmgxPgm, CountUnagg)
 {
     using index_type = typename TestFixture::index_type;
     gko::Array<index_type> agg(this->exec, 5);
     auto agg_val = agg.get_data();
-    gko::size_type num_unagg = 0;
+    index_type num_unagg = 0;
     agg_val[0] = 0;
     agg_val[1] = -1;
     agg_val[2] = 0;
@@ -434,7 +372,7 @@ TYPED_TEST(AmgxPgm, Renumber)
     using index_type = typename TestFixture::index_type;
     gko::Array<index_type> agg(this->exec, 5);
     auto agg_val = agg.get_data();
-    gko::size_type num_agg = 0;
+    index_type num_agg = 0;
     agg_val[0] = 0;
     agg_val[1] = 1;
     agg_val[2] = 0;
@@ -469,11 +407,10 @@ TYPED_TEST(AmgxPgm, Generate)
 TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
 {
     auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
-
-    // fine->coarse
     using Vec = typename TestFixture::Vec;
     using value_type = typename TestFixture::value_type;
     auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
+
     amgx_pgm->restrict_apply(this->fine_b.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, this->restrict_ans, r<value_type>::value);
@@ -606,38 +543,23 @@ TYPED_TEST(AmgxPgm, GenerateMtx)
     agg_vals[2] = 0;
     agg_vals[3] = 1;
     agg_vals[4] = 2;
+    auto coarse_ans = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
+    coarse_ans->read({{3, 3},
+                      {{0, 0, 4},
+                       {0, 1, -3},
+                       {0, 2, -1},
+                       {1, 0, -3},
+                       {1, 1, 5},
+                       {1, 2, -1},
+                       {2, 0, -2},
+                       {2, 1, -2},
+                       {2, 2, 5}}});
     auto csr_coarse = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
 
     gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
         this->exec, this->mtx.get(), agg, csr_coarse.get());
 
-    auto r = csr_coarse->get_const_row_ptrs();
-    auto c = csr_coarse->get_const_col_idxs();
-    auto v = csr_coarse->get_const_values();
-    ASSERT_EQ(csr_coarse->get_size(), gko::dim<2>(3, 3));
-    ASSERT_EQ(csr_coarse->get_num_stored_elements(), 9);
-    ASSERT_EQ(r[0], 0);
-    ASSERT_EQ(r[1], 3);
-    ASSERT_EQ(r[2], 6);
-    ASSERT_EQ(r[3], 9);
-    ASSERT_EQ(c[0], 0);
-    ASSERT_EQ(c[1], 1);
-    ASSERT_EQ(c[2], 2);
-    ASSERT_EQ(c[3], 0);
-    ASSERT_EQ(c[4], 1);
-    ASSERT_EQ(c[5], 2);
-    ASSERT_EQ(c[6], 0);
-    ASSERT_EQ(c[7], 1);
-    ASSERT_EQ(c[8], 2);
-    ASSERT_EQ(v[0], value_type{4});
-    ASSERT_EQ(v[1], value_type{-3});
-    ASSERT_EQ(v[2], value_type{-1});
-    ASSERT_EQ(v[3], value_type{-3});
-    ASSERT_EQ(v[4], value_type{5});
-    ASSERT_EQ(v[5], value_type{-1});
-    ASSERT_EQ(v[6], value_type{-2});
-    ASSERT_EQ(v[7], value_type{-2});
-    ASSERT_EQ(v[8], value_type{5});
+    GKO_ASSERT_MTX_NEAR(csr_coarse, coarse_ans, r<value_type>::value);
 }
 
 

From 1cb4e25415ca0e11c1b48ba7cbe0b6d1d74f5a14 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 27 Jan 2021 15:40:35 +0800
Subject: [PATCH 0477/1008] Rename the multigrid interface, improve doc

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 include/ginkgo/core/multigrid/amgx_pgm.hpp    | 10 ++--
 .../{interface.hpp => multigrid_base.hpp}     | 48 +++++++++++++++++--
 include/ginkgo/ginkgo.hpp                     |  2 +-
 omp/test/matrix/csr_kernels.cpp               |  1 -
 reference/test/matrix/csr_kernels.cpp         |  2 -
 5 files changed, 53 insertions(+), 10 deletions(-)
 rename include/ginkgo/core/multigrid/{interface.hpp => multigrid_base.hpp} (85%)

diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index 6b905a949c5..f7d9d193718 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -42,7 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/multigrid/interface.hpp>
+#include <ginkgo/core/multigrid/multigrid_base.hpp>
 
 
 namespace gko {
@@ -117,14 +117,17 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
     GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
     {
         /**
-         * The maximum number of iterations.
+         * The maximum number of iterations. We use the same default value as
+         * NVIDIA AMGX Reference Manual (October 2017, API Version 2,
+         * https://github.com/NVIDIA/AMGX/blob/main/doc/AMGX_Reference.pdf).
          */
         unsigned GKO_FACTORY_PARAMETER(max_iterations, 15u);
 
         /**
          * The maximum ratio of unassigned number, which is valid in the
          * interval 0.0 ~ 1.0. We use the same default value as NVIDIA AMGX
-         * Reference Manual (October 2017, API Version 2).
+         * Reference Manual (October 2017, API Version 2,
+         * https://github.com/NVIDIA/AMGX/blob/main/doc/AMGX_Reference.pdf).
          */
         double GKO_FACTORY_PARAMETER(max_unassigned_percentage, 0.05);
 
@@ -179,6 +182,7 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
             this->generate();
         }
     }
+
     void generate();
 
 private:
diff --git a/include/ginkgo/core/multigrid/interface.hpp b/include/ginkgo/core/multigrid/multigrid_base.hpp
similarity index 85%
rename from include/ginkgo/core/multigrid/interface.hpp
rename to include/ginkgo/core/multigrid/multigrid_base.hpp
index 26795ac1e8c..108e13d4735 100644
--- a/include/ginkgo/core/multigrid/interface.hpp
+++ b/include/ginkgo/core/multigrid/multigrid_base.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_PUBLIC_CORE_MULTIGRID_INTERFACE_HPP_
-#define GKO_PUBLIC_CORE_MULTIGRID_INTERFACE_HPP_
+#ifndef GKO_PUBLIC_CORE_MULTIGRID_MULTIGRID_BASE_HPP_
+#define GKO_PUBLIC_CORE_MULTIGRID_MULTIGRID_BASE_HPP_
 
 
 #include <functional>
@@ -267,6 +267,48 @@ class CoarseOp {
  * MultigridLevel::prolong_applyadd_impl(), and
  * MultigridLevel::restrict_apply_impl() methods.
  *
+ * ```c++
+ * class MyMultigridLevel : public EnableLinOp<MyMultigridLevel>,
+ *                          public MultigridLevel {
+ *     GKO_CREATE_FACTORY_PARAMETERS(MyMultigridLevel, Factory) {
+ *         // some parameter settings as LinOpFactory
+ *     };
+ *     GKO_ENABLE_LIN_OP_FACTORY(MyMultigridLevel, parameters, Factory);
+ *     GKO_ENABLE_BUILD_METHOD(Factory);
+ *
+ *     // Need to implement the following override function:
+ *     // prolong_apply_impl, prolong_applyadd_impl, restict_apply_impl
+ *
+ *     // could use the default apply implementation of MultigridLevel
+ *     void apply_impl(const LinOp* b, LinOp* x) const override {
+ *         this->template multigrid_level_default_apply<ValueType>(b, x);
+ *     }
+ *
+ *     // could use the default apply implementation of MultigridLevel
+ *     void apply_impl(const LinOp *alpha, const LinOp *b,
+ *                     const LinOp *beta, LinOp *x) const override {
+ *         this->template multigrid_level_default_apply<ValueType>(
+ *             alpha, b, beta, x);
+ *     }
+ *
+ *     // constructor needed by EnableLinOp
+ *     explicit MyMultigridLevel(std::shared_ptr<const Executor> exec) {
+ *         : EnableLinOp<MyMultigridLevel>(exec) {}
+ *
+ *     // constructor needed by the factory
+ *     explicit MyMultigridLevel(const Factory *factory,
+ *                      std::shared_ptr<const LinOp> matrix)
+ *         : EnableLinOp<MyMultigridLevel>(factory->get_executor()),
+ *                                         matrix->get_size()),
+ *           // store factory's parameters locally
+ *           my_parameters_{factory->get_parameters()},
+ *     {
+ *          // do something with parameter to generate fine/coarse matrices
+ *          // set the information such that the checks work properly
+ *          this->set_fine_coarse(fine, corase);
+ *     }
+ * ```
+ *
  * @ingroup Multigrid
  */
 class MultigridLevel : public RestrictOp, public ProlongOp, public CoarseOp {
@@ -361,4 +403,4 @@ class MultigridLevel : public RestrictOp, public ProlongOp, public CoarseOp {
 }  // namespace gko
 
 
-#endif  // GKO_PUBLIC_CORE_MULTIGRID_INTERFACE_HPP_
+#endif  // GKO_PUBLIC_CORE_MULTIGRID_MULTIGRID_BASE_HPP_
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index f83e6dbd82e..ad09f72d716 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -88,7 +88,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 #include <ginkgo/core/multigrid/amgx_pgm.hpp>
-#include <ginkgo/core/multigrid/interface.hpp>
+#include <ginkgo/core/multigrid/multigrid_base.hpp>
 
 #include <ginkgo/core/preconditioner/ic.hpp>
 #include <ginkgo/core/preconditioner/ilu.hpp>
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index 6e505e70f0c..a27f6638a04 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -63,7 +63,6 @@ class Csr : public ::testing::Test {
 protected:
     using Arr = gko::Array<int>;
     using Mtx = gko::matrix::Csr<>;
-    using AbsMtx = gko::matrix::Csr<>;
     using Vec = gko::matrix::Dense<>;
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
     using ComplexMtx = gko::matrix::Csr<std::complex<double>>;
diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp
index 3c53f5509e8..186aa700598 100644
--- a/reference/test/matrix/csr_kernels.cpp
+++ b/reference/test/matrix/csr_kernels.cpp
@@ -69,8 +69,6 @@ class Csr : public ::testing::Test {
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using Coo = gko::matrix::Coo<value_type, index_type>;
     using Mtx = gko::matrix::Csr<value_type, index_type>;
-    using AbsMtx =
-        gko::matrix::Csr<gko::remove_complex<value_type>, index_type>;
     using Sellp = gko::matrix::Sellp<value_type, index_type>;
     using SparsityCsr = gko::matrix::SparsityCsr<value_type, index_type>;
     using Ell = gko::matrix::Ell<value_type, index_type>;

From 9fc57b2cc0bf2c3191fc7647820b1ba380db985b Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 5 Feb 2021 00:28:21 -0800
Subject: [PATCH 0478/1008] WIP: add UseComposition and ApplyAddable interface

---
 include/ginkgo/core/base/composition.hpp      | 17 +++++
 include/ginkgo/core/base/lin_op.hpp           | 12 +++
 include/ginkgo/core/multigrid/amgx_pgm.hpp    |  2 +-
 .../ginkgo/core/multigrid/multigrid_base.hpp  | 76 +++++--------------
 4 files changed, 51 insertions(+), 56 deletions(-)

diff --git a/include/ginkgo/core/base/composition.hpp b/include/ginkgo/core/base/composition.hpp
index d7ba65bb436..a73b8e58dc1 100644
--- a/include/ginkgo/core/base/composition.hpp
+++ b/include/ginkgo/core/base/composition.hpp
@@ -167,6 +167,23 @@ class Composition : public EnableLinOp<Composition<ValueType>>,
 };
 
 
+template <typename ValueType = default_precision>
+class UseComposition {
+    public:
+	using value_type = ValueType;
+	std::shared_ptr<Composition<ValueType>> get_composition() const {
+            return composition_;
+	}
+    protected:
+	template <typename... LinOp>
+	void set_composition(LinOp &&... linop) {
+	    composition_ = Composition<ValueType>::create(std::forward<LinOp>(linop)...);
+	}
+    private:
+	std::shared_ptr<Composition<ValueType>> composition_;
+}
+
+
 }  // namespace gko
 
 
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index cde3840c30f..53953d3f163 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -654,6 +654,18 @@ class Preconditionable {
 };
 
 
+class ApplyAddable() {
+    public:
+        void apply2(const LinOp* b, LinOp* x) const {
+	    // check the comformat size
+	    this->apply2_impl(b, x);
+	}
+
+    protected:
+	virtual void apply2_impl(const LinOp* b, LinOp* x) const = 0;
+}
+
+
 /**
  * The diagonal of a LinOp implementing this interface can be extracted.
  * extract_diagonal extracts the elements whose col and row index are the
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index f7d9d193718..397cba1da1a 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -170,7 +170,7 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
                      std::shared_ptr<const LinOp> system_matrix)
         : EnableLinOp<AmgxPgm>(factory->get_executor(),
                                system_matrix->get_size()),
-          MultigridLevel(factory->get_executor(), true),
+          MultigridLevel(system_matrix),
           parameters_{factory->get_parameters()},
           system_matrix_{std::move(system_matrix)},
           agg_(factory->get_executor(), system_matrix_->get_size()[0])
diff --git a/include/ginkgo/core/multigrid/multigrid_base.hpp b/include/ginkgo/core/multigrid/multigrid_base.hpp
index 108e13d4735..2df401d1aa6 100644
--- a/include/ginkgo/core/multigrid/multigrid_base.hpp
+++ b/include/ginkgo/core/multigrid/multigrid_base.hpp
@@ -311,7 +311,7 @@ class CoarseOp {
  *
  * @ingroup Multigrid
  */
-class MultigridLevel : public RestrictOp, public ProlongOp, public CoarseOp {
+class MultigridLevel : public UseComposition {
 protected:
     /**
      * Sets the fine and coarse information.
@@ -319,64 +319,31 @@ class MultigridLevel : public RestrictOp, public ProlongOp, public CoarseOp {
      * @param fine  the matrix on fine level
      * @param coarse  the matrix on coarse level
      */
-    void set_multigrid_level(std::shared_ptr<const LinOp> fine,
-                             std::shared_ptr<const LinOp> coarse)
+    void set_multigrid_level(std::shared_ptr<const LinOp> prolong_op,
+                             std::shared_ptr<const LinOp> coarse_op,
+			     std::shared_ptr<const LinOp> restrict_op)
     {
-        this->set_fine_coarse(fine, coarse);
-        auto fine_dim = fine->get_size()[0];
-        auto coarse_dim = coarse->get_size()[0];
-        this->set_restrict_size(dim<2>{coarse_dim, fine_dim});
-        this->set_prolong_size(dim<2>{fine_dim, coarse_dim});
+	gko::dim<2> mg_size{prolong_op->get_size()[0], restrict_op->get_size()[1]};
+	// check mg_size is the same as fine_size
+	this->set_composition(prolong_op, coarse_op, restrict_op);
     }
 
-    /**
-     * A default apply of multigrid_level
-     *
-     * Performs the operation x = op(b) = prolong(coarse(restrict(b)))
-     *
-     * @tparam ValueType  precision of input and result vectors
-     *
-     * @param b  the input vector(s) on which the operator is applied
-     * @param x  the output vector(s) where the result is stored
-     */
-    template <typename ValueType>
-    void multigrid_level_default_apply(const LinOp *b, LinOp *x) const
-    {
-        auto num_cols = b->get_size()[1];
-        auto coarse = this->get_coarse_matrix();
-        auto coarse_b = matrix::Dense<ValueType>::create(
-            exec_, dim<2>{coarse->get_size()[1], num_cols});
-        auto coarse_x = matrix::Dense<ValueType>::create(
-            exec_, dim<2>{coarse->get_size()[0], num_cols});
-        this->restrict_apply(b, lend(coarse_b));
-        coarse->apply(lend(coarse_b), lend(coarse_x));
-        this->prolong_apply(lend(coarse_x), x);
+    std::shared_ptr<const LinOp> get_fine_op() const {
+        return fine_op_;
     }
 
-    /**
-     * A default advanced_apply of multigrid_level
-     *
-     * Performs the operation x = alpha * op(b) + beta * x
-     * = alpha * prolong(coarse(restrict(b))) + beta * x
-     *
-     * @tparam ValueType  precision of input and result vectors
-     *
-     * @param b  the input vector(s) on which the operator is applied
-     * @param x  the output vector(s) where the result is stored
-     */
-    template <typename ValueType>
-    void multigrid_level_default_apply(const LinOp *alpha, const LinOp *b,
-                                       const LinOp *beta, LinOp *x) const
-    {
-        auto dense_x = as<matrix::Dense<ValueType>>(x);
+    std::shared_ptr<const LinOp> get_restrict_op() const {
+        return this->get_composition()[2];
+    }
 
-        auto x_clone = dense_x->clone();
-        this->template multigrid_level_default_apply<ValueType>(b,
-                                                                x_clone.get());
-        dense_x->scale(beta);
-        dense_x->add_scaled(alpha, x_clone.get());
+    std::shared_ptr<const LinOp> get_coarse_op() const {
+        return this->get_composition()[1];
     }
 
+    std::shared_ptr<const LinOp> get_prolong_op() const {
+        return this->get_composition()[0];
+    }
+   
     /**
      * Creates a MultigridLevel with settings
      *
@@ -389,13 +356,12 @@ class MultigridLevel : public RestrictOp, public ProlongOp, public CoarseOp {
      *       so needs to call `set_multigrid_level` to set corresponding
      *       information after generation.
      */
-    explicit MultigridLevel(std::shared_ptr<const Executor> exec,
-                            bool native_applyadd = false)
-        : exec_(exec), RestrictOp(exec), ProlongOp(exec, native_applyadd)
+    explicit MultigridLevel(std::shared_ptr<const LinOp> fine_op)
+        : fine_op_(fine_op)
     {}
 
 private:
-    std::shared_ptr<const Executor> exec_;
+    std::shared_ptr<const LinOp> fine_op_;
 };
 
 

From 1352a66cda618d79e9d9115e0d07b6a597b31c99 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 2 Feb 2021 16:40:07 +0800
Subject: [PATCH 0479/1008] WIP

---
 core/multigrid/amgx_pgm.cpp                   |  11 +-
 include/ginkgo/core/multigrid/amgx_pgm.hpp    |  29 +-
 include/ginkgo/core/multigrid/mapping.hpp     | 106 ++++
 .../ginkgo/core/multigrid/multigrid_base.hpp  |  75 +++
 .../test/factorization/par_ilu_kernels.cpp    |   3 +
 reference/test/multigrid/amgx_pgm_kernels.cpp | 575 +++++++++---------
 6 files changed, 497 insertions(+), 302 deletions(-)
 create mode 100644 include/ginkgo/core/multigrid/mapping.hpp

diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index b7a697fb5a5..5344e05e517 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -89,7 +89,8 @@ std::unique_ptr<LinOp> amgx_pgm_generate(
 
 
 template <typename ValueType, typename IndexType>
-void AmgxPgm<ValueType, IndexType>::generate()
+std::unique_ptr<MultigridLevelOp<ValueType>>
+AmgxPgm<ValueType, IndexType>::generate()
 {
     using matrix_type = matrix::Csr<ValueType, IndexType>;
     using real_type = remove_complex<ValueType>;
@@ -160,6 +161,14 @@ void AmgxPgm<ValueType, IndexType>::generate()
     auto coarse_matrix =
         share(amgx_pgm_generate(exec, amgxpgm_op, num_agg, agg_));
     this->set_multigrid_level(system_matrix_, coarse_matrix);
+    auto coarse_dim = coarse_matrix->get_size()[0];
+    auto fine_dim = system_matrix_->get_size()[0];
+    auto restrict_op = Mapping<ValueType, IndexType>::create(
+        exec, gko::dim<2>{coarse_dim, fine_dim}, agg_);
+    auto prolong_op = Mapping<ValueType, IndexType>::create(
+        exec, gko::dim<2>{fine_dim, coarse_dim}, agg_, false);
+    return MultigridLevelOp<ValueType>::create(share(prolong_op), coarse_matrix,
+                                               share(restrict_op));
 }
 
 
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index 397cba1da1a..50e97f7c1fb 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -42,9 +42,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/multigrid/mapping.hpp>
 #include <ginkgo/core/multigrid/multigrid_base.hpp>
 
-
 namespace gko {
 namespace multigrid {
 
@@ -72,11 +72,7 @@ namespace multigrid {
  * @ingroup LinOp
  */
 template <typename ValueType = default_precision, typename IndexType = int32>
-class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
-                public MultigridLevel {
-    friend class EnableLinOp<AmgxPgm>;
-    friend class EnablePolymorphicObject<AmgxPgm, LinOp>;
-
+class AmgxPgm : public MultigridLevelOp<ValueType>, public MultigridLevel {
 public:
     using value_type = ValueType;
     using index_type = IndexType;
@@ -114,6 +110,10 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
         return agg_.get_const_data();
     }
 
+    template <typename... Args>
+    static std::unique_ptr<MultigridLevelOp<ValueType>> create(
+        Args &&... args) = delete;
+
     GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
     {
         /**
@@ -163,27 +163,26 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
     }
 
     explicit AmgxPgm(std::shared_ptr<const Executor> exec)
-        : EnableLinOp<AmgxPgm>(exec), MultigridLevel(exec, true)
+        : MultigridLevelOp<ValueType>(exec), MultigridLevel(exec, true)
     {}
 
     explicit AmgxPgm(const Factory *factory,
                      std::shared_ptr<const LinOp> system_matrix)
-        : EnableLinOp<AmgxPgm>(factory->get_executor(),
-                               system_matrix->get_size()),
-          MultigridLevel(system_matrix),
+        : MultigridLevelOp<ValueType>(factory->get_executor()),
+          MultigridLevel(factory->get_executor(), true),
           parameters_{factory->get_parameters()},
           system_matrix_{std::move(system_matrix)},
           agg_(factory->get_executor(), system_matrix_->get_size()[0])
     {
         GKO_ASSERT(parameters_.max_unassigned_percentage <= 1.0);
         GKO_ASSERT(parameters_.max_unassigned_percentage >= 0.0);
-        if (system_matrix_->get_size()[0] != 0) {
-            // generate on the existed matrix
-            this->generate();
-        }
+        // if (system_matrix_->get_size()[0] != 0) {
+        // generate on the existed matrix
+        this->generate()->move_to(this);
+        // }
     }
 
-    void generate();
+    std::unique_ptr<MultigridLevelOp<ValueType>> generate();
 
 private:
     std::shared_ptr<const LinOp> system_matrix_{};
diff --git a/include/ginkgo/core/multigrid/mapping.hpp b/include/ginkgo/core/multigrid/mapping.hpp
new file mode 100644
index 00000000000..a327d51fb92
--- /dev/null
+++ b/include/ginkgo/core/multigrid/mapping.hpp
@@ -0,0 +1,106 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_PUBLIC_CORE_MULTIGRID_MAPPING_HPP_
+#define GKO_PUBLIC_CORE_MULTIGRID_MAPPING_HPP_
+
+
+#include <vector>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/multigrid/multigrid_base.hpp>
+
+
+namespace gko {
+namespace multigrid {
+
+
+template <typename ValueType, typename IndexType>
+class Mapping : public EnableLinOp<Mapping<ValueType, IndexType>>,
+                public EnableCreateMethod<Mapping<ValueType, IndexType>> {
+    friend class EnableCreateMethod<Mapping>;
+    friend class EnablePolymorphicObject<Mapping, LinOp>;
+
+public:
+    using index_type = IndexType;
+
+protected:
+    /**
+     * Creates an uninitialized Mapping arrays on the specified executor..
+     *
+     * @param exec  Executor associated to the LinOp
+     */
+    Mapping(std::shared_ptr<const Executor> exec)
+        : EnableLinOp<Mapping>(exec), mapping_{exec}, is_restrict_(true)
+    {}
+
+    template <typename IndicesArray>
+    Mapping(std::shared_ptr<const Executor> exec, const dim<2> &size,
+            IndicesArray &&mapping_indices, const bool &is_restrict = true)
+        : EnableLinOp<Mapping>(exec, size),
+          mapping_{exec, std::forward<IndicesArray>(mapping_indices)},
+          is_restrict_(is_restrict)
+    {
+        if (is_restrict_) {
+            GKO_ASSERT_EQ(size[1], mapping_.get_num_elems());
+        }
+        if (!is_restrict_) {
+            GKO_ASSERT_EQ(size[0], mapping_.get_num_elems());
+        }
+    }
+
+    void apply_impl(const LinOp *b, LinOp *x) const override
+    {
+        GKO_NOT_IMPLEMENTED;
+    }
+
+    void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
+                    LinOp *x) const override
+    {
+        GKO_NOT_IMPLEMENTED;
+    }
+
+private:
+    Array<index_type> mapping_;
+    bool is_restrict_;
+};
+
+
+}  // namespace multigrid
+}  // namespace gko
+
+#endif  // GKO_PUBLIC_CORE_MULTIGRID_MAPPING_HPP_
\ No newline at end of file
diff --git a/include/ginkgo/core/multigrid/multigrid_base.hpp b/include/ginkgo/core/multigrid/multigrid_base.hpp
index 2df401d1aa6..afd11d2628a 100644
--- a/include/ginkgo/core/multigrid/multigrid_base.hpp
+++ b/include/ginkgo/core/multigrid/multigrid_base.hpp
@@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/abstract_factory.hpp>
+#include <ginkgo/core/base/composition.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/base/utils.hpp>
@@ -365,6 +366,80 @@ class MultigridLevel : public UseComposition {
 };
 
 
+// template <typename ConcreteObject>
+// class EnableNewDefault {
+// protected:
+
+//     std::unique_ptr<PolymorphicObject> create_default_impl(
+//         std::shared_ptr<const Executor> exec) const override
+//     {
+//         return std::unique_ptr<ConcreteObject>{new ConcreteObject(exec)};
+//     }
+
+//     PolymorphicObject *copy_from_impl(const PolymorphicObject *other)
+//     override
+//     {
+//         as<ConvertibleTo<ConcreteObject>>(other)->convert_to(self());
+//         return this;
+//     }
+
+//     PolymorphicObject *copy_from_impl(
+//         std::unique_ptr<PolymorphicObject> other) override
+//     {
+//         as<ConvertibleTo<ConcreteObject>>(other.get())->move_to(self());
+//         return this;
+//     }
+
+//     PolymorphicObject *clear_impl() override
+//     {
+//         *self() = ConcreteObject{this->get_executor()};
+//         return this;
+//     }
+
+// private:
+//     GKO_ENABLE_SELF(ConcreteObject);
+// };
+
+
+template <typename ValueType>
+class MultigridLevelOp
+    : public Composition<ValueType>,
+      public EnableCreateMethod<MultigridLevelOp<ValueType>> {
+    // friend class EnablePolymorphicObject<MultigridLevelOp, LinOp>;
+    friend class EnableCreateMethod<MultigridLevelOp>;
+    // friend class EnableNewDefault<MultigridLevelOp>;
+
+public:
+    using value_type = ValueType;
+    using EnableCreateMethod<MultigridLevelOp>::create;
+    // using EnableAbstractPolymorphicObject<MultigridLevelOp,
+    //                                          LinOp>::EnableAbstractPolymorphicObject;
+    // EnablePolymorphicObject
+    // auto clone_ilu = par_ilu->clone();
+    // auto clone_l = clone_ilu->get_l_factor();
+    // using EnableLinOp<MultigridLevelOp>::PolymorphicObject;
+    // using EnableLinOp<MultigridLevelOp>::get_executor;
+    // using EnableLinOp<MultigridLevelOp>::EnablePolymorphicAssignment;
+
+    // using Composition<ValueType>::apply;
+protected:
+    /**
+     * Creates an empty operator composition (0x0 operator).
+     *
+     * @param exec  Executor associated to the composition
+     */
+    explicit MultigridLevelOp(std::shared_ptr<const Executor> exec)
+        : Composition<ValueType>(exec)
+    {}
+
+    explicit MultigridLevelOp(std::shared_ptr<const LinOp> prolong,
+                              std::shared_ptr<const LinOp> coarse,
+                              std::shared_ptr<const LinOp> restrict)
+        : Composition<ValueType>(prolong, coarse, restrict)
+    {}
+};
+
+
 }  // namespace multigrid
 }  // namespace gko
 
diff --git a/reference/test/factorization/par_ilu_kernels.cpp b/reference/test/factorization/par_ilu_kernels.cpp
index 388d54232ad..c6ea588ad51 100644
--- a/reference/test/factorization/par_ilu_kernels.cpp
+++ b/reference/test/factorization/par_ilu_kernels.cpp
@@ -483,6 +483,9 @@ TYPED_TEST(ParIlu, SetLStrategy)
         par_ilu_type::build().with_l_strategy(l_strategy).on(this->ref);
     auto par_ilu = factory->generate(this->mtx_small);
 
+    auto clone_ilu = par_ilu->clone();
+    auto clone_l = dynamic_cast<par_ilu_type *>(clone_ilu.get());
+    ASSERT_NE(clone_l, nullptr);
     ASSERT_EQ(factory->get_parameters().l_strategy, l_strategy);
     ASSERT_EQ(par_ilu->get_l_factor()->get_strategy()->get_name(),
               l_strategy->get_name());
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index ba367ffca55..59259746c44 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -264,303 +264,306 @@ TYPED_TEST(AmgxPgm, CanBeCloned)
     using Mtx = typename TestFixture::Mtx;
     using RestrictProlong = typename TestFixture::RestrictProlong;
     auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
-
-    auto clone = this->rstr_prlg->clone();
-    auto clone_mtx =
-        static_cast<RestrictProlong *>(clone.get())->get_system_matrix();
-    auto clone_agg =
-        static_cast<RestrictProlong *>(clone.get())->get_const_agg();
-    auto clone_coarse = clone->get_coarse_matrix();
-
-    this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
-                               this->mtx.get());
-    this->assert_same_agg(clone_agg, this->agg.get_data(),
-                          this->agg.get_num_elems());
+    auto copy_coarse = copy->get_coarse_matrix();
+    auto clone = copy->clone();
+    // auto clone_mtx =
+    //     clone->get_system_matrix();
+    // auto clone_agg =
+    //     static_cast<RestrictProlong *>(clone.get())->get_const_agg();
+    auto clone_coarse =
+        dynamic_cast<RestrictProlong *>(clone.get())->get_coarse_matrix();
+
+    // this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
+    //                            this->mtx.get());
+    // this->assert_same_agg(clone_agg, this->agg.get_data(),
+    //                       this->agg.get_num_elems());
     this->assert_same_matrices(static_cast<const Mtx *>(clone_coarse.get()),
                                this->coarse.get());
 }
 
 
-TYPED_TEST(AmgxPgm, CanBeCleared)
-{
-    using RestrictProlong = typename TestFixture::RestrictProlong;
-
-    this->rstr_prlg->clear();
-    auto mtx = static_cast<RestrictProlong *>(this->rstr_prlg.get())
-                   ->get_system_matrix();
-    auto coarse = this->rstr_prlg->get_coarse_matrix();
-    auto agg = static_cast<RestrictProlong *>(this->rstr_prlg.get())->get_agg();
-
-    ASSERT_EQ(mtx, nullptr);
-    ASSERT_EQ(coarse, nullptr);
-    ASSERT_EQ(agg, nullptr);
-}
-
-
-TYPED_TEST(AmgxPgm, RestrictApply)
-{
-    // fine->coarse
-    using Vec = typename TestFixture::Vec;
-    using value_type = typename TestFixture::value_type;
-    auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
-
-    gko::kernels::reference::amgx_pgm::restrict_apply(
-        this->exec, this->agg, this->fine_b.get(), x.get());
-
-    GKO_ASSERT_MTX_NEAR(x, this->restrict_ans, r<value_type>::value);
-}
-
-
-TYPED_TEST(AmgxPgm, ProlongApplyadd)
-{
-    using value_type = typename TestFixture::value_type;
-    auto x = gko::clone(this->fine_x);
-
-    gko::kernels::reference::amgx_pgm::prolong_applyadd(
-        this->exec, this->agg, this->coarse_b.get(), x.get());
-
-    GKO_ASSERT_MTX_NEAR(x, this->prolong_ans, r<value_type>::value);
-}
-
-
-TYPED_TEST(AmgxPgm, MatchEdge)
-{
-    using index_type = typename TestFixture::index_type;
-    gko::Array<index_type> agg(this->exec, 5);
-    gko::Array<index_type> snb(this->exec, 5);
-    auto agg_val = agg.get_data();
-    auto snb_val = snb.get_data();
-    for (int i = 0; i < 5; i++) {
-        agg_val[i] = -1;
-    }
-    snb_val[0] = 2;
-    snb_val[1] = 0;
-    snb_val[2] = 0;
-    snb_val[3] = 1;
-    snb_val[4] = 2;
-
-    gko::kernels::reference::amgx_pgm::match_edge(this->exec, snb, agg);
-
-    ASSERT_EQ(agg_val[0], 0);
-    ASSERT_EQ(agg_val[1], -1);
-    ASSERT_EQ(agg_val[2], 0);
-    ASSERT_EQ(agg_val[3], -1);
-    ASSERT_EQ(agg_val[4], -1);
-}
-
-
-TYPED_TEST(AmgxPgm, CountUnagg)
-{
-    using index_type = typename TestFixture::index_type;
-    gko::Array<index_type> agg(this->exec, 5);
-    auto agg_val = agg.get_data();
-    index_type num_unagg = 0;
-    agg_val[0] = 0;
-    agg_val[1] = -1;
-    agg_val[2] = 0;
-    agg_val[3] = -1;
-    agg_val[4] = -1;
-
-    gko::kernels::reference::amgx_pgm::count_unagg(this->exec, agg, &num_unagg);
-
-    ASSERT_EQ(num_unagg, 3);
-}
-
-
-TYPED_TEST(AmgxPgm, Renumber)
-{
-    using index_type = typename TestFixture::index_type;
-    gko::Array<index_type> agg(this->exec, 5);
-    auto agg_val = agg.get_data();
-    index_type num_agg = 0;
-    agg_val[0] = 0;
-    agg_val[1] = 1;
-    agg_val[2] = 0;
-    agg_val[3] = 1;
-    agg_val[4] = 4;
-
-    gko::kernels::reference::amgx_pgm::renumber(this->exec, agg, &num_agg);
-
-    ASSERT_EQ(num_agg, 3);
-    ASSERT_EQ(agg_val[0], 0);
-    ASSERT_EQ(agg_val[1], 1);
-    ASSERT_EQ(agg_val[2], 0);
-    ASSERT_EQ(agg_val[3], 1);
-    ASSERT_EQ(agg_val[4], 2);
-}
-
-
-TYPED_TEST(AmgxPgm, Generate)
-{
-    auto coarse_fine = this->amgxpgm_factory->generate(this->mtx);
-
-    auto agg_result = coarse_fine->get_const_agg();
-
-    ASSERT_EQ(agg_result[0], 0);
-    ASSERT_EQ(agg_result[1], 1);
-    ASSERT_EQ(agg_result[2], 0);
-    ASSERT_EQ(agg_result[3], 1);
-    ASSERT_EQ(agg_result[4], 0);
-}
-
-
-TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
-{
-    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
-    using Vec = typename TestFixture::Vec;
-    using value_type = typename TestFixture::value_type;
-    auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
-
-    amgx_pgm->restrict_apply(this->fine_b.get(), x.get());
-
-    GKO_ASSERT_MTX_NEAR(x, this->restrict_ans, r<value_type>::value);
-}
-
-
-TYPED_TEST(AmgxPgm, CoarseFineProlongApplyadd)
-{
-    using value_type = typename TestFixture::value_type;
-    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
-    auto x = gko::clone(this->fine_x);
-
-    amgx_pgm->prolong_applyadd(this->coarse_b.get(), x.get());
-
-    GKO_ASSERT_MTX_NEAR(x, this->prolong_ans, r<value_type>::value);
-}
-
-
-TYPED_TEST(AmgxPgm, CoarseFineProlongApply)
-{
-    using value_type = typename TestFixture::value_type;
-    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
-    auto x = gko::clone(this->fine_x);
-
-    amgx_pgm->prolong_apply(this->coarse_b.get(), x.get());
-
-    GKO_ASSERT_MTX_NEAR(x, this->prolong_applyans, r<value_type>::value);
-}
-
-
-TYPED_TEST(AmgxPgm, Apply)
-{
-    using VT = typename TestFixture::value_type;
-    using Vec = typename TestFixture::Vec;
-    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
-    auto b = gko::clone(this->fine_x);
-    auto x = gko::clone(this->fine_x);
-    auto exec = amgx_pgm->get_executor();
-    auto answer = gko::initialize<Vec>(
-        {I<VT>({-23.0, 5.0}), I<VT>({17.0, -5.0}), I<VT>({-23.0, 5.0}),
-         I<VT>({17.0, -5.0}), I<VT>({-23.0, 5.0})},
-        exec);
-
-    amgx_pgm->apply(b.get(), x.get());
-
-    GKO_ASSERT_MTX_NEAR(x, answer, r<VT>::value);
-}
+// TYPED_TEST(AmgxPgm, CanBeCleared)
+// {
+//     using RestrictProlong = typename TestFixture::RestrictProlong;
+
+//     this->rstr_prlg->clear();
+//     auto mtx = static_cast<RestrictProlong *>(this->rstr_prlg.get())
+//                    ->get_system_matrix();
+//     auto coarse = this->rstr_prlg->get_coarse_matrix();
+//     auto agg = static_cast<RestrictProlong
+//     *>(this->rstr_prlg.get())->get_agg();
+
+//     ASSERT_EQ(mtx, nullptr);
+//     ASSERT_EQ(coarse, nullptr);
+//     ASSERT_EQ(agg, nullptr);
+// }
+
+
+// TYPED_TEST(AmgxPgm, RestrictApply)
+// {
+//     // fine->coarse
+//     using Vec = typename TestFixture::Vec;
+//     using value_type = typename TestFixture::value_type;
+//     auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
+
+//     gko::kernels::reference::amgx_pgm::restrict_apply(
+//         this->exec, this->agg, this->fine_b.get(), x.get());
+
+//     GKO_ASSERT_MTX_NEAR(x, this->restrict_ans, r<value_type>::value);
+// }
+
+
+// TYPED_TEST(AmgxPgm, ProlongApplyadd)
+// {
+//     using value_type = typename TestFixture::value_type;
+//     auto x = gko::clone(this->fine_x);
+
+//     gko::kernels::reference::amgx_pgm::prolong_applyadd(
+//         this->exec, this->agg, this->coarse_b.get(), x.get());
+
+//     GKO_ASSERT_MTX_NEAR(x, this->prolong_ans, r<value_type>::value);
+// }
+
+
+// TYPED_TEST(AmgxPgm, MatchEdge)
+// {
+//     using index_type = typename TestFixture::index_type;
+//     gko::Array<index_type> agg(this->exec, 5);
+//     gko::Array<index_type> snb(this->exec, 5);
+//     auto agg_val = agg.get_data();
+//     auto snb_val = snb.get_data();
+//     for (int i = 0; i < 5; i++) {
+//         agg_val[i] = -1;
+//     }
+//     snb_val[0] = 2;
+//     snb_val[1] = 0;
+//     snb_val[2] = 0;
+//     snb_val[3] = 1;
+//     snb_val[4] = 2;
 
+//     gko::kernels::reference::amgx_pgm::match_edge(this->exec, snb, agg);
 
-TYPED_TEST(AmgxPgm, AdvancedApply)
-{
-    using VT = typename TestFixture::value_type;
-    using Vec = typename TestFixture::Vec;
-    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
-    auto b = gko::clone(this->fine_x);
-    auto x = gko::clone(this->fine_x);
-    auto exec = amgx_pgm->get_executor();
-    auto alpha = gko::initialize<Vec>({1.0}, exec);
-    auto beta = gko::initialize<Vec>({2.0}, exec);
-    auto answer = gko::initialize<Vec>(
-        {I<VT>({-27.0, 3.0}), I<VT>({19.0, -7.0}), I<VT>({-25.0, 3.0}),
-         I<VT>({17.0, -5.0}), I<VT>({-23.0, 9.0})},
-        exec);
-
-    amgx_pgm->apply(alpha.get(), b.get(), beta.get(), x.get());
-
-    GKO_ASSERT_MTX_NEAR(x, answer, r<VT>::value);
-}
+//     ASSERT_EQ(agg_val[0], 0);
+//     ASSERT_EQ(agg_val[1], -1);
+//     ASSERT_EQ(agg_val[2], 0);
+//     ASSERT_EQ(agg_val[3], -1);
+//     ASSERT_EQ(agg_val[4], -1);
+// }
 
 
-TYPED_TEST(AmgxPgm, FindStrongestNeighbor)
-{
-    using index_type = typename TestFixture::index_type;
-    gko::Array<index_type> strongest_neighbor(this->exec, 5);
-    gko::Array<index_type> agg(this->exec, 5);
-    auto snb_vals = strongest_neighbor.get_data();
-    auto agg_vals = agg.get_data();
-    for (int i = 0; i < 5; i++) {
-        snb_vals[i] = -1;
-        agg_vals[i] = -1;
-    }
+// TYPED_TEST(AmgxPgm, CountUnagg)
+// {
+//     using index_type = typename TestFixture::index_type;
+//     gko::Array<index_type> agg(this->exec, 5);
+//     auto agg_val = agg.get_data();
+//     index_type num_unagg = 0;
+//     agg_val[0] = 0;
+//     agg_val[1] = -1;
+//     agg_val[2] = 0;
+//     agg_val[3] = -1;
+//     agg_val[4] = -1;
 
-    gko::kernels::reference::amgx_pgm::find_strongest_neighbor(
-        this->exec, this->weight.get(), this->mtx_diag.get(), agg,
-        strongest_neighbor);
+//     gko::kernels::reference::amgx_pgm::count_unagg(this->exec, agg,
+//     &num_unagg);
 
-    ASSERT_EQ(snb_vals[0], 2);
-    ASSERT_EQ(snb_vals[1], 0);
-    ASSERT_EQ(snb_vals[2], 0);
-    ASSERT_EQ(snb_vals[3], 1);
-    ASSERT_EQ(snb_vals[4], 2);
-}
-
-
-TYPED_TEST(AmgxPgm, AssignToExistAgg)
-{
-    using index_type = typename TestFixture::index_type;
-    gko::Array<index_type> agg(this->exec, 5);
-    gko::Array<index_type> intermediate_agg(this->exec, 0);
-    auto agg_vals = agg.get_data();
-    // 0 - 2, 1 - 3
-    agg_vals[0] = 0;
-    agg_vals[1] = 1;
-    agg_vals[2] = 0;
-    agg_vals[3] = 1;
-    agg_vals[4] = -1;
-
-    gko::kernels::reference::amgx_pgm::assign_to_exist_agg(
-        this->exec, this->weight.get(), this->mtx_diag.get(), agg,
-        intermediate_agg);
-
-    ASSERT_EQ(agg_vals[0], 0);
-    ASSERT_EQ(agg_vals[1], 1);
-    ASSERT_EQ(agg_vals[2], 0);
-    ASSERT_EQ(agg_vals[3], 1);
-    ASSERT_EQ(agg_vals[4], 0);
-}
-
-
-TYPED_TEST(AmgxPgm, GenerateMtx)
-{
-    using index_type = typename TestFixture::index_type;
-    using value_type = typename TestFixture::value_type;
-    using mtx_type = typename TestFixture::Mtx;
-    gko::Array<index_type> agg(this->exec, 5);
-    auto agg_vals = agg.get_data();
-    // 0 - 2, 1 - 3, 4
-    agg_vals[0] = 0;
-    agg_vals[1] = 1;
-    agg_vals[2] = 0;
-    agg_vals[3] = 1;
-    agg_vals[4] = 2;
-    auto coarse_ans = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
-    coarse_ans->read({{3, 3},
-                      {{0, 0, 4},
-                       {0, 1, -3},
-                       {0, 2, -1},
-                       {1, 0, -3},
-                       {1, 1, 5},
-                       {1, 2, -1},
-                       {2, 0, -2},
-                       {2, 1, -2},
-                       {2, 2, 5}}});
-    auto csr_coarse = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
-
-    gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
-        this->exec, this->mtx.get(), agg, csr_coarse.get());
-
-    GKO_ASSERT_MTX_NEAR(csr_coarse, coarse_ans, r<value_type>::value);
-}
+//     ASSERT_EQ(num_unagg, 3);
+// }
+
+
+// TYPED_TEST(AmgxPgm, Renumber)
+// {
+//     using index_type = typename TestFixture::index_type;
+//     gko::Array<index_type> agg(this->exec, 5);
+//     auto agg_val = agg.get_data();
+//     index_type num_agg = 0;
+//     agg_val[0] = 0;
+//     agg_val[1] = 1;
+//     agg_val[2] = 0;
+//     agg_val[3] = 1;
+//     agg_val[4] = 4;
+
+//     gko::kernels::reference::amgx_pgm::renumber(this->exec, agg, &num_agg);
+
+//     ASSERT_EQ(num_agg, 3);
+//     ASSERT_EQ(agg_val[0], 0);
+//     ASSERT_EQ(agg_val[1], 1);
+//     ASSERT_EQ(agg_val[2], 0);
+//     ASSERT_EQ(agg_val[3], 1);
+//     ASSERT_EQ(agg_val[4], 2);
+// }
+
+
+// TYPED_TEST(AmgxPgm, Generate)
+// {
+//     auto coarse_fine = this->amgxpgm_factory->generate(this->mtx);
+
+//     auto agg_result = coarse_fine->get_const_agg();
+
+//     ASSERT_EQ(agg_result[0], 0);
+//     ASSERT_EQ(agg_result[1], 1);
+//     ASSERT_EQ(agg_result[2], 0);
+//     ASSERT_EQ(agg_result[3], 1);
+//     ASSERT_EQ(agg_result[4], 0);
+// }
+
+
+// TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
+// {
+//     auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
+//     using Vec = typename TestFixture::Vec;
+//     using value_type = typename TestFixture::value_type;
+//     auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
+
+//     amgx_pgm->restrict_apply(this->fine_b.get(), x.get());
+
+//     GKO_ASSERT_MTX_NEAR(x, this->restrict_ans, r<value_type>::value);
+// }
+
+
+// TYPED_TEST(AmgxPgm, CoarseFineProlongApplyadd)
+// {
+//     using value_type = typename TestFixture::value_type;
+//     auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
+//     auto x = gko::clone(this->fine_x);
+
+//     amgx_pgm->prolong_applyadd(this->coarse_b.get(), x.get());
+
+//     GKO_ASSERT_MTX_NEAR(x, this->prolong_ans, r<value_type>::value);
+// }
+
+
+// TYPED_TEST(AmgxPgm, CoarseFineProlongApply)
+// {
+//     using value_type = typename TestFixture::value_type;
+//     auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
+//     auto x = gko::clone(this->fine_x);
+
+//     amgx_pgm->prolong_apply(this->coarse_b.get(), x.get());
+
+//     GKO_ASSERT_MTX_NEAR(x, this->prolong_applyans, r<value_type>::value);
+// }
+
+
+// TYPED_TEST(AmgxPgm, Apply)
+// {
+//     using VT = typename TestFixture::value_type;
+//     using Vec = typename TestFixture::Vec;
+//     auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
+//     auto b = gko::clone(this->fine_x);
+//     auto x = gko::clone(this->fine_x);
+//     auto exec = amgx_pgm->get_executor();
+//     auto answer = gko::initialize<Vec>(
+//         {I<VT>({-23.0, 5.0}), I<VT>({17.0, -5.0}), I<VT>({-23.0, 5.0}),
+//          I<VT>({17.0, -5.0}), I<VT>({-23.0, 5.0})},
+//         exec);
+
+//     amgx_pgm->apply(b.get(), x.get());
+
+//     GKO_ASSERT_MTX_NEAR(x, answer, r<VT>::value);
+// }
+
+
+// TYPED_TEST(AmgxPgm, AdvancedApply)
+// {
+//     using VT = typename TestFixture::value_type;
+//     using Vec = typename TestFixture::Vec;
+//     auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
+//     auto b = gko::clone(this->fine_x);
+//     auto x = gko::clone(this->fine_x);
+//     auto exec = amgx_pgm->get_executor();
+//     auto alpha = gko::initialize<Vec>({1.0}, exec);
+//     auto beta = gko::initialize<Vec>({2.0}, exec);
+//     auto answer = gko::initialize<Vec>(
+//         {I<VT>({-27.0, 3.0}), I<VT>({19.0, -7.0}), I<VT>({-25.0, 3.0}),
+//          I<VT>({17.0, -5.0}), I<VT>({-23.0, 9.0})},
+//         exec);
+
+//     amgx_pgm->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+//     GKO_ASSERT_MTX_NEAR(x, answer, r<VT>::value);
+// }
+
+
+// TYPED_TEST(AmgxPgm, FindStrongestNeighbor)
+// {
+//     using index_type = typename TestFixture::index_type;
+//     gko::Array<index_type> strongest_neighbor(this->exec, 5);
+//     gko::Array<index_type> agg(this->exec, 5);
+//     auto snb_vals = strongest_neighbor.get_data();
+//     auto agg_vals = agg.get_data();
+//     for (int i = 0; i < 5; i++) {
+//         snb_vals[i] = -1;
+//         agg_vals[i] = -1;
+//     }
+
+//     gko::kernels::reference::amgx_pgm::find_strongest_neighbor(
+//         this->exec, this->weight.get(), this->mtx_diag.get(), agg,
+//         strongest_neighbor);
+
+//     ASSERT_EQ(snb_vals[0], 2);
+//     ASSERT_EQ(snb_vals[1], 0);
+//     ASSERT_EQ(snb_vals[2], 0);
+//     ASSERT_EQ(snb_vals[3], 1);
+//     ASSERT_EQ(snb_vals[4], 2);
+// }
+
+
+// TYPED_TEST(AmgxPgm, AssignToExistAgg)
+// {
+//     using index_type = typename TestFixture::index_type;
+//     gko::Array<index_type> agg(this->exec, 5);
+//     gko::Array<index_type> intermediate_agg(this->exec, 0);
+//     auto agg_vals = agg.get_data();
+//     // 0 - 2, 1 - 3
+//     agg_vals[0] = 0;
+//     agg_vals[1] = 1;
+//     agg_vals[2] = 0;
+//     agg_vals[3] = 1;
+//     agg_vals[4] = -1;
+
+//     gko::kernels::reference::amgx_pgm::assign_to_exist_agg(
+//         this->exec, this->weight.get(), this->mtx_diag.get(), agg,
+//         intermediate_agg);
+
+//     ASSERT_EQ(agg_vals[0], 0);
+//     ASSERT_EQ(agg_vals[1], 1);
+//     ASSERT_EQ(agg_vals[2], 0);
+//     ASSERT_EQ(agg_vals[3], 1);
+//     ASSERT_EQ(agg_vals[4], 0);
+// }
+
+
+// TYPED_TEST(AmgxPgm, GenerateMtx)
+// {
+//     using index_type = typename TestFixture::index_type;
+//     using value_type = typename TestFixture::value_type;
+//     using mtx_type = typename TestFixture::Mtx;
+//     gko::Array<index_type> agg(this->exec, 5);
+//     auto agg_vals = agg.get_data();
+//     // 0 - 2, 1 - 3, 4
+//     agg_vals[0] = 0;
+//     agg_vals[1] = 1;
+//     agg_vals[2] = 0;
+//     agg_vals[3] = 1;
+//     agg_vals[4] = 2;
+//     auto coarse_ans = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
+//     coarse_ans->read({{3, 3},
+//                       {{0, 0, 4},
+//                        {0, 1, -3},
+//                        {0, 2, -1},
+//                        {1, 0, -3},
+//                        {1, 1, 5},
+//                        {1, 2, -1},
+//                        {2, 0, -2},
+//                        {2, 1, -2},
+//                        {2, 2, 5}}});
+//     auto csr_coarse = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
+
+//     gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
+//         this->exec, this->mtx.get(), agg, csr_coarse.get());
+
+//     GKO_ASSERT_MTX_NEAR(csr_coarse, coarse_ans, r<value_type>::value);
+// }
 
 
 }  // namespace

From 8a290bb44605136dfb51ad41c167b56797928e9e Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <19565938+yhmtsai@users.noreply.github.com>
Date: Thu, 18 Feb 2021 01:00:55 +0800
Subject: [PATCH 0480/1008] mapping, multigridlevel, usecomp, applyaddable

---
 core/CMakeLists.txt                           |   1 +
 core/device_hooks/common_kernels.inc.cpp      |  20 +
 core/multigrid/amgx_pgm.cpp                   |   8 +-
 core/multigrid/amgx_pgm_kernels.hpp           |  36 +-
 core/multigrid/mapping.cpp                    |  89 +++
 core/multigrid/mapping_kernels.hpp            | 126 ++++
 cuda/CMakeLists.txt                           |   1 +
 cuda/multigrid/mapping_kernels.cu             |  88 +++
 dpcpp/CMakeLists.txt                          |   1 +
 dpcpp/multigrid/mapping_kernels.dp.cpp        |  88 +++
 hip/CMakeLists.txt                            |   1 +
 hip/multigrid/mapping_kernels.hip.cpp         |  88 +++
 include/ginkgo/core/base/composition.hpp      |  40 +-
 include/ginkgo/core/base/lin_op.hpp           |  30 +-
 include/ginkgo/core/multigrid/amgx_pgm.hpp    |  41 +-
 include/ginkgo/core/multigrid/mapping.hpp     |  60 +-
 .../ginkgo/core/multigrid/multigrid_base.hpp  | 124 +---
 include/ginkgo/ginkgo.hpp                     |   1 +
 omp/CMakeLists.txt                            |   1 +
 omp/multigrid/mapping_kernels.cpp             |  88 +++
 reference/CMakeLists.txt                      |   1 +
 reference/multigrid/mapping_kernels.cpp       | 118 ++++
 .../test/factorization/par_ilu_kernels.cpp    |   3 -
 reference/test/multigrid/amgx_pgm_kernels.cpp | 576 +++++++++---------
 24 files changed, 1173 insertions(+), 457 deletions(-)
 create mode 100644 core/multigrid/mapping.cpp
 create mode 100644 core/multigrid/mapping_kernels.hpp
 create mode 100644 cuda/multigrid/mapping_kernels.cu
 create mode 100644 dpcpp/multigrid/mapping_kernels.dp.cpp
 create mode 100644 hip/multigrid/mapping_kernels.hip.cpp
 create mode 100644 omp/multigrid/mapping_kernels.cpp
 create mode 100644 reference/multigrid/mapping_kernels.cpp

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index a4f8594249f..2fadd5e53b7 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -32,6 +32,7 @@ target_sources(ginkgo
     matrix/sellp.cpp
     matrix/sparsity_csr.cpp
     multigrid/amgx_pgm.cpp
+    multigrid/mapping.cpp
     preconditioner/isai.cpp
     preconditioner/jacobi.cpp
     reorder/rcm.cpp
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index bafc8e07ad8..90205943074 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -53,6 +53,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/sellp_kernels.hpp"
 #include "core/matrix/sparsity_csr_kernels.hpp"
 #include "core/multigrid/amgx_pgm_kernels.hpp"
+#include "core/multigrid/mapping_kernels.hpp"
 #include "core/preconditioner/isai_kernels.hpp"
 #include "core/preconditioner/jacobi_kernels.hpp"
 #include "core/reorder/rcm_kernels.hpp"
@@ -1288,6 +1289,25 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
 
 
 }  // namespace implicit_residual_norm
+
+
+namespace mapping {
+
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_MAPPING_APPLYADD_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_MAPPING_APPLYADD_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL);
+
+
+}  // namespace mapping
 }  // namespace GKO_HOOK_MODULE
 }  // namespace kernels
 }  // namespace gko
diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index 5344e05e517..6860482229a 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -89,8 +89,7 @@ std::unique_ptr<LinOp> amgx_pgm_generate(
 
 
 template <typename ValueType, typename IndexType>
-std::unique_ptr<MultigridLevelOp<ValueType>>
-AmgxPgm<ValueType, IndexType>::generate()
+void AmgxPgm<ValueType, IndexType>::generate()
 {
     using matrix_type = matrix::Csr<ValueType, IndexType>;
     using real_type = remove_complex<ValueType>;
@@ -160,15 +159,14 @@ AmgxPgm<ValueType, IndexType>::generate()
     // Construct the coarse matrix
     auto coarse_matrix =
         share(amgx_pgm_generate(exec, amgxpgm_op, num_agg, agg_));
-    this->set_multigrid_level(system_matrix_, coarse_matrix);
+    // this->set_multigrid_level(system_matrix_, coarse_matrix);
     auto coarse_dim = coarse_matrix->get_size()[0];
     auto fine_dim = system_matrix_->get_size()[0];
     auto restrict_op = Mapping<ValueType, IndexType>::create(
         exec, gko::dim<2>{coarse_dim, fine_dim}, agg_);
     auto prolong_op = Mapping<ValueType, IndexType>::create(
         exec, gko::dim<2>{fine_dim, coarse_dim}, agg_, false);
-    return MultigridLevelOp<ValueType>::create(share(prolong_op), coarse_matrix,
-                                               share(restrict_op));
+    this->set_composition(share(prolong_op), coarse_matrix, share(restrict_op));
 }
 
 
diff --git a/core/multigrid/amgx_pgm_kernels.hpp b/core/multigrid/amgx_pgm_kernels.hpp
index cbe44124472..5dcaddddaad 100644
--- a/core/multigrid/amgx_pgm_kernels.hpp
+++ b/core/multigrid/amgx_pgm_kernels.hpp
@@ -51,28 +51,30 @@ namespace kernels {
 namespace amgx_pgm {
 
 
-#define GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL(_vtype, _itype)             \
-    void restrict_apply(                                                       \
-        std::shared_ptr<const DefaultExecutor> exec, const Array<_itype> &agg, \
-        const matrix::Dense<_vtype> *b, matrix::Dense<_vtype> *x)
-
-#define GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL(_vtype, _itype)           \
-    void prolong_applyadd(                                                     \
-        std::shared_ptr<const DefaultExecutor> exec, const Array<_itype> &agg, \
-        const matrix::Dense<_vtype> *b, matrix::Dense<_vtype> *x)
-
-#define GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL(_itype)           \
+#define GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL(ValueType, IndexType) \
+    void restrict_apply(std::shared_ptr<const DefaultExecutor> exec,     \
+                        const Array<IndexType> &agg,                     \
+                        const matrix::Dense<ValueType> *b,               \
+                        matrix::Dense<ValueType> *x)
+
+#define GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL(ValueType, IndexType) \
+    void prolong_applyadd(std::shared_ptr<const DefaultExecutor> exec,     \
+                          const Array<IndexType> &agg,                     \
+                          const matrix::Dense<ValueType> *b,               \
+                          matrix::Dense<ValueType> *x)
+
+#define GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL(IndexType)        \
     void match_edge(std::shared_ptr<const DefaultExecutor> exec, \
-                    const Array<_itype> &strongest_neighbor,     \
-                    Array<_itype> &agg)
+                    const Array<IndexType> &strongest_neighbor,  \
+                    Array<IndexType> &agg)
 
-#define GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL(_itype)           \
+#define GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL(IndexType)        \
     void count_unagg(std::shared_ptr<const DefaultExecutor> exec, \
-                     const Array<_itype> &agg, _itype *num_unagg)
+                     const Array<IndexType> &agg, IndexType *num_unagg)
 
-#define GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL(_itype)           \
+#define GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL(IndexType)        \
     void renumber(std::shared_ptr<const DefaultExecutor> exec, \
-                  Array<_itype> &agg, _itype *num_agg)
+                  Array<IndexType> &agg, IndexType *num_agg)
 
 #define GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR(ValueType, IndexType) \
     void find_strongest_neighbor(                                          \
diff --git a/core/multigrid/mapping.cpp b/core/multigrid/mapping.cpp
new file mode 100644
index 00000000000..bb4ed40a061
--- /dev/null
+++ b/core/multigrid/mapping.cpp
@@ -0,0 +1,89 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/multigrid/mapping.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/polymorphic_object.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/base/utils.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/identity.hpp>
+
+
+#include "core/components/fill_array.hpp"
+#include "core/matrix/csr_builder.hpp"
+#include "core/multigrid/mapping_kernels.hpp"
+
+
+namespace gko {
+namespace multigrid {
+namespace mapping {
+
+
+GKO_REGISTER_OPERATION(applyadd, mapping::applyadd);
+GKO_REGISTER_OPERATION(advanced_applyadd, mapping::advanced_applyadd);
+
+
+}  // namespace mapping
+
+
+template <typename ValueType, typename IndexType>
+void Mapping<ValueType, IndexType>::apply2_impl(const LinOp *b, LinOp *x) const
+{
+    auto exec = this->get_executor();
+    exec->run(mapping::make_applyadd(this, as<matrix::Dense<ValueType>>(b),
+                                     as<matrix::Dense<ValueType>>(x)));
+}
+
+
+template <typename ValueType, typename IndexType>
+void Mapping<ValueType, IndexType>::apply2_impl(const LinOp *alpha,
+                                                const LinOp *b, LinOp *x) const
+{
+    auto exec = this->get_executor();
+    exec->run(mapping::make_advanced_applyadd(
+        this, as<matrix::Dense<ValueType>>(alpha),
+        as<matrix::Dense<ValueType>>(b), as<matrix::Dense<ValueType>>(x)));
+}
+
+
+#define GKO_DECLARE_MAPPING(_vtype, _itype) class Mapping<_vtype, _itype>
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_MAPPING);
+
+
+}  // namespace multigrid
+}  // namespace gko
diff --git a/core/multigrid/mapping_kernels.hpp b/core/multigrid/mapping_kernels.hpp
new file mode 100644
index 00000000000..eb5506ad418
--- /dev/null
+++ b/core/multigrid/mapping_kernels.hpp
@@ -0,0 +1,126 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_MULTIGRID_MAPPING_KERNELS_HPP_
+#define GKO_CORE_MULTIGRID_MAPPING_KERNELS_HPP_
+
+
+#include <ginkgo/core/multigrid/mapping.hpp>
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+
+
+namespace gko {
+namespace kernels {
+
+
+#define GKO_DECLARE_MAPPING_APPLYADD_KERNEL(ValueType, IndexType)         \
+    void applyadd(std::shared_ptr<const DefaultExecutor> exec,            \
+                  const multigrid::Mapping<ValueType, IndexType> *source, \
+                  const matrix::Dense<ValueType> *b,                      \
+                  matrix::Dense<ValueType> *x)
+
+#define GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL(ValueType, IndexType) \
+    void advanced_applyadd(                                                \
+        std::shared_ptr<const DefaultExecutor> exec,                       \
+        const multigrid::Mapping<ValueType, IndexType> *source,            \
+        const matrix::Dense<ValueType> *alpha,                             \
+        const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *x)
+
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES                           \
+    template <typename ValueType, typename IndexType>          \
+    GKO_DECLARE_MAPPING_APPLYADD_KERNEL(ValueType, IndexType); \
+    template <typename ValueType, typename IndexType>          \
+    GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL(ValueType, IndexType)
+
+
+namespace omp {
+namespace mapping {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace mapping
+}  // namespace omp
+
+
+namespace cuda {
+namespace mapping {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace mapping
+}  // namespace cuda
+
+
+namespace reference {
+namespace mapping {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace mapping
+}  // namespace reference
+
+
+namespace hip {
+namespace mapping {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace mapping
+}  // namespace hip
+
+
+namespace dpcpp {
+namespace mapping {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace mapping
+}  // namespace dpcpp
+
+
+#undef GKO_DECLARE_ALL_AS_TEMPLATES
+
+
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_CORE_MULTIGRID_MAPPING_KERNELS_HPP_
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 5e4505d8187..f8b9d31951e 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -102,6 +102,7 @@ target_sources(ginkgo_cuda
     matrix/sellp_kernels.cu
     matrix/sparsity_csr_kernels.cu
     multigrid/amgx_pgm_kernels.cu
+    multigrid/mapping_kernels.cu
     preconditioner/isai_kernels.cu
     preconditioner/jacobi_advanced_apply_kernel.cu
     preconditioner/jacobi_generate_kernel.cu
diff --git a/cuda/multigrid/mapping_kernels.cu b/cuda/multigrid/mapping_kernels.cu
new file mode 100644
index 00000000000..bd757fae834
--- /dev/null
+++ b/cuda/multigrid/mapping_kernels.cu
@@ -0,0 +1,88 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/multigrid/mapping_kernels.hpp"
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+#include <ginkgo/core/multigrid/mapping.hpp>
+
+
+#include "core/base/allocator.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/csr_builder.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+/**
+ * @brief The MAPPING solver namespace.
+ *
+ * @ingroup mapping
+ */
+namespace mapping {
+
+
+template <typename ValueType, typename IndexType>
+void applyadd(std::shared_ptr<const DefaultExecutor> exec,
+              const multigrid::Mapping<ValueType, IndexType> *source,
+              const matrix::Dense<ValueType> *b,
+              matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_MAPPING_APPLYADD_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_applyadd(std::shared_ptr<const DefaultExecutor> exec,
+                       const multigrid::Mapping<ValueType, IndexType> *source,
+                       const matrix::Dense<ValueType> *alpha,
+                       const matrix::Dense<ValueType> *b,
+                       matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL);
+
+
+}  // namespace mapping
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 9e340ba33fe..01c0c621f6c 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -31,6 +31,7 @@ target_sources(ginkgo_dpcpp
     matrix/sellp_kernels.dp.cpp
     matrix/sparsity_csr_kernels.dp.cpp
     multigrid/amgx_pgm_kernels.dp.cpp
+    multigrid/mapping_kernels.dp.cpp
     preconditioner/isai_kernels.dp.cpp
     preconditioner/jacobi_kernels.dp.cpp
     reorder/rcm_kernels.dp.cpp
diff --git a/dpcpp/multigrid/mapping_kernels.dp.cpp b/dpcpp/multigrid/mapping_kernels.dp.cpp
new file mode 100644
index 00000000000..0e172460f86
--- /dev/null
+++ b/dpcpp/multigrid/mapping_kernels.dp.cpp
@@ -0,0 +1,88 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/multigrid/mapping_kernels.hpp"
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+#include <ginkgo/core/multigrid/mapping.hpp>
+
+
+#include "core/base/allocator.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/csr_builder.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The MAPPING solver namespace.
+ *
+ * @ingroup mapping
+ */
+namespace mapping {
+
+
+template <typename ValueType, typename IndexType>
+void applyadd(std::shared_ptr<const DefaultExecutor> exec,
+              const multigrid::Mapping<ValueType, IndexType> *source,
+              const matrix::Dense<ValueType> *b,
+              matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_MAPPING_APPLYADD_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_applyadd(std::shared_ptr<const DefaultExecutor> exec,
+                       const multigrid::Mapping<ValueType, IndexType> *source,
+                       const matrix::Dense<ValueType> *alpha,
+                       const matrix::Dense<ValueType> *b,
+                       matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL);
+
+
+}  // namespace mapping
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 4e34da9466f..2c388326c54 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -193,6 +193,7 @@ set(GINKGO_HIP_SOURCES
     matrix/sellp_kernels.hip.cpp
     matrix/sparsity_csr_kernels.hip.cpp
     multigrid/amgx_pgm_kernels.hip.cpp
+    multigrid/mapping_kernels.hip.cpp
     preconditioner/isai_kernels.hip.cpp
     preconditioner/jacobi_advanced_apply_kernel.hip.cpp
     preconditioner/jacobi_generate_kernel.hip.cpp
diff --git a/hip/multigrid/mapping_kernels.hip.cpp b/hip/multigrid/mapping_kernels.hip.cpp
new file mode 100644
index 00000000000..849d0bbafd6
--- /dev/null
+++ b/hip/multigrid/mapping_kernels.hip.cpp
@@ -0,0 +1,88 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/multigrid/mapping_kernels.hpp"
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+#include <ginkgo/core/multigrid/mapping.hpp>
+
+
+#include "core/base/allocator.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/csr_builder.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace hip {
+/**
+ * @brief The MAPPING solver namespace.
+ *
+ * @ingroup mapping
+ */
+namespace mapping {
+
+
+template <typename ValueType, typename IndexType>
+void applyadd(std::shared_ptr<const DefaultExecutor> exec,
+              const multigrid::Mapping<ValueType, IndexType> *source,
+              const matrix::Dense<ValueType> *b,
+              matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_MAPPING_APPLYADD_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_applyadd(std::shared_ptr<const DefaultExecutor> exec,
+                       const multigrid::Mapping<ValueType, IndexType> *source,
+                       const matrix::Dense<ValueType> *alpha,
+                       const matrix::Dense<ValueType> *b,
+                       matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL);
+
+
+}  // namespace mapping
+}  // namespace hip
+}  // namespace kernels
+}  // namespace gko
diff --git a/include/ginkgo/core/base/composition.hpp b/include/ginkgo/core/base/composition.hpp
index a73b8e58dc1..6ba86355b04 100644
--- a/include/ginkgo/core/base/composition.hpp
+++ b/include/ginkgo/core/base/composition.hpp
@@ -169,19 +169,33 @@ class Composition : public EnableLinOp<Composition<ValueType>>,
 
 template <typename ValueType = default_precision>
 class UseComposition {
-    public:
-	using value_type = ValueType;
-	std::shared_ptr<Composition<ValueType>> get_composition() const {
-            return composition_;
-	}
-    protected:
-	template <typename... LinOp>
-	void set_composition(LinOp &&... linop) {
-	    composition_ = Composition<ValueType>::create(std::forward<LinOp>(linop)...);
-	}
-    private:
-	std::shared_ptr<Composition<ValueType>> composition_;
-}
+public:
+    using value_type = ValueType;
+    std::shared_ptr<Composition<ValueType>> get_composition() const
+    {
+        return composition_;
+    }
+
+    std::shared_ptr<const LinOp> get_composition_ithop(int index) const
+    {
+        if (composition_ == nullptr) {
+            return nullptr;
+        } else {
+            return composition_->get_operators().at(index);
+        }
+    }
+
+protected:
+    template <typename... LinOp>
+    void set_composition(LinOp &&... linop)
+    {
+        composition_ =
+            Composition<ValueType>::create(std::forward<LinOp>(linop)...);
+    }
+
+private:
+    std::shared_ptr<Composition<ValueType>> composition_;
+};
 
 
 }  // namespace gko
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index 53953d3f163..be2760d5902 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -654,16 +654,26 @@ class Preconditionable {
 };
 
 
-class ApplyAddable() {
-    public:
-        void apply2(const LinOp* b, LinOp* x) const {
-	    // check the comformat size
-	    this->apply2_impl(b, x);
-	}
-
-    protected:
-	virtual void apply2_impl(const LinOp* b, LinOp* x) const = 0;
-}
+class ApplyAddable {
+public:
+    void apply2(const LinOp *b, LinOp *x) const
+    {
+        // check the comformat size
+        this->apply2_impl(b, x);
+    }
+
+    void apply2(const LinOp *alpha, const LinOp *b, LinOp *x) const
+    {
+        // check the comformat size
+        this->apply2_impl(alpha, b, x);
+    }
+
+protected:
+    virtual void apply2_impl(const LinOp *b, LinOp *x) const = 0;
+
+    virtual void apply2_impl(const LinOp *alpha, const LinOp *b,
+                             LinOp *x) const = 0;
+};
 
 
 /**
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index 50e97f7c1fb..3face49cc6d 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <vector>
 
 
+#include <ginkgo/core/base/composition.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/base/types.hpp>
@@ -72,7 +73,11 @@ namespace multigrid {
  * @ingroup LinOp
  */
 template <typename ValueType = default_precision, typename IndexType = int32>
-class AmgxPgm : public MultigridLevelOp<ValueType>, public MultigridLevel {
+class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
+                public MultigridLevel<ValueType> {
+    friend class EnableLinOp<AmgxPgm>;
+    friend class EnablePolymorphicObject<AmgxPgm, LinOp>;
+
 public:
     using value_type = ValueType;
     using index_type = IndexType;
@@ -110,10 +115,6 @@ class AmgxPgm : public MultigridLevelOp<ValueType>, public MultigridLevel {
         return agg_.get_const_data();
     }
 
-    template <typename... Args>
-    static std::unique_ptr<MultigridLevelOp<ValueType>> create(
-        Args &&... args) = delete;
-
     GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
     {
         /**
@@ -144,45 +145,45 @@ class AmgxPgm : public MultigridLevelOp<ValueType>, public MultigridLevel {
     GKO_ENABLE_BUILD_METHOD(Factory);
 
 protected:
-    void restrict_apply_impl(const LinOp *b, LinOp *x) const override;
+    void restrict_apply_impl(const LinOp *b, LinOp *x) const;
 
-    void prolong_applyadd_impl(const LinOp *b, LinOp *x) const override;
+    void prolong_applyadd_impl(const LinOp *b, LinOp *x) const;
 
-    void prolong_apply_impl(const LinOp *b, LinOp *x) const override;
+    void prolong_apply_impl(const LinOp *b, LinOp *x) const;
 
     void apply_impl(const LinOp *b, LinOp *x) const override
     {
-        this->template multigrid_level_default_apply<ValueType>(b, x);
+        this->get_composition()->apply(b, x);
     }
 
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override
     {
-        this->template multigrid_level_default_apply<ValueType>(alpha, b, beta,
-                                                                x);
+        this->get_composition()->apply(alpha, b, beta, x);
     }
 
     explicit AmgxPgm(std::shared_ptr<const Executor> exec)
-        : MultigridLevelOp<ValueType>(exec), MultigridLevel(exec, true)
+        : EnableLinOp<AmgxPgm>(std::move(exec)), MultigridLevel<ValueType>()
     {}
 
     explicit AmgxPgm(const Factory *factory,
                      std::shared_ptr<const LinOp> system_matrix)
-        : MultigridLevelOp<ValueType>(factory->get_executor()),
-          MultigridLevel(factory->get_executor(), true),
+        : EnableLinOp<AmgxPgm>(factory->get_executor(),
+                               system_matrix->get_size()),
+          MultigridLevel<ValueType>(system_matrix),
           parameters_{factory->get_parameters()},
-          system_matrix_{std::move(system_matrix)},
+          system_matrix_{system_matrix},
           agg_(factory->get_executor(), system_matrix_->get_size()[0])
     {
         GKO_ASSERT(parameters_.max_unassigned_percentage <= 1.0);
         GKO_ASSERT(parameters_.max_unassigned_percentage >= 0.0);
-        // if (system_matrix_->get_size()[0] != 0) {
-        // generate on the existed matrix
-        this->generate()->move_to(this);
-        // }
+        if (system_matrix_->get_size()[0] != 0) {
+            // generate on the existed matrix
+            this->generate();
+        }
     }
 
-    std::unique_ptr<MultigridLevelOp<ValueType>> generate();
+    void generate();
 
 private:
     std::shared_ptr<const LinOp> system_matrix_{};
diff --git a/include/ginkgo/core/multigrid/mapping.hpp b/include/ginkgo/core/multigrid/mapping.hpp
index a327d51fb92..de75f6ac6cf 100644
--- a/include/ginkgo/core/multigrid/mapping.hpp
+++ b/include/ginkgo/core/multigrid/mapping.hpp
@@ -51,13 +51,51 @@ namespace multigrid {
 
 template <typename ValueType, typename IndexType>
 class Mapping : public EnableLinOp<Mapping<ValueType, IndexType>>,
-                public EnableCreateMethod<Mapping<ValueType, IndexType>> {
+                public EnableCreateMethod<Mapping<ValueType, IndexType>>,
+                public ApplyAddable {
     friend class EnableCreateMethod<Mapping>;
     friend class EnablePolymorphicObject<Mapping, LinOp>;
 
 public:
+    using value_type = ValueType;
     using index_type = IndexType;
 
+    /**
+     * Returns the mapping indexes.
+     *
+     * @return the mapping indexes.
+     */
+    index_type *get_mapping_index() noexcept { return mapping_.get_data(); }
+
+    /**
+     * @copydoc Mapping::get_mapping_index()
+     *
+     * @note This is the constant version of the function, which can be
+     *       significantly more memory efficient than the non-constant version,
+     *       so always prefer this version.
+     */
+    const index_type *get_const_mapping_index() const noexcept
+    {
+        return mapping_.get_const_data();
+    }
+
+    /**
+     * Returns true as it is restrict operation.
+     *
+     * @return true as it is restrict operation.
+     */
+    bool is_restrict() const noexcept { return is_restrict_; }
+
+    /**
+     * Returns the number of elements explicitly stored in the mapping index.
+     *
+     * @return the number of elements explicitly stored in the mapping index
+     */
+    size_type get_num_stored_elements() const noexcept
+    {
+        return mapping_.get_num_elems();
+    }
+
 protected:
     /**
      * Creates an uninitialized Mapping arrays on the specified executor..
@@ -85,15 +123,29 @@ class Mapping : public EnableLinOp<Mapping<ValueType, IndexType>>,
 
     void apply_impl(const LinOp *b, LinOp *x) const override
     {
-        GKO_NOT_IMPLEMENTED;
+        auto dense_x = as<matrix::Dense<ValueType>>(x);
+
+        // TODO: write dense fill kernel
+        auto zero = gko::initialize<matrix::Dense<ValueType>>(
+            {gko::zero<ValueType>()}, this->get_executor());
+        dense_x->scale(lend(zero));
+        this->apply2(b, dense_x);
     }
 
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override
     {
-        GKO_NOT_IMPLEMENTED;
+        auto dense_x = as<matrix::Dense<ValueType>>(x);
+        dense_x->scale(beta);
+        this->apply2(alpha, b, dense_x);
     }
 
+    void apply2_impl(const LinOp *b, LinOp *x) const override;
+
+    void apply2_impl(const LinOp *alpha, const LinOp *b,
+                     LinOp *x) const override;
+
+
 private:
     Array<index_type> mapping_;
     bool is_restrict_;
@@ -103,4 +155,4 @@ class Mapping : public EnableLinOp<Mapping<ValueType, IndexType>>,
 }  // namespace multigrid
 }  // namespace gko
 
-#endif  // GKO_PUBLIC_CORE_MULTIGRID_MAPPING_HPP_
\ No newline at end of file
+#endif  // GKO_PUBLIC_CORE_MULTIGRID_MAPPING_HPP_
diff --git a/include/ginkgo/core/multigrid/multigrid_base.hpp b/include/ginkgo/core/multigrid/multigrid_base.hpp
index afd11d2628a..70d292a1e47 100644
--- a/include/ginkgo/core/multigrid/multigrid_base.hpp
+++ b/include/ginkgo/core/multigrid/multigrid_base.hpp
@@ -312,7 +312,28 @@ class CoarseOp {
  *
  * @ingroup Multigrid
  */
-class MultigridLevel : public UseComposition {
+template <typename ValueType>
+class MultigridLevel : public UseComposition<ValueType> {
+public:
+    using value_type = ValueType;
+
+    std::shared_ptr<const LinOp> get_fine_op() const { return fine_op_; }
+
+    std::shared_ptr<const LinOp> get_restrict_op() const
+    {
+        return this->get_composition_ithop(2);
+    }
+
+    std::shared_ptr<const LinOp> get_coarse_op() const
+    {
+        return this->get_composition_ithop(1);
+    }
+
+    std::shared_ptr<const LinOp> get_prolong_op() const
+    {
+        return this->get_composition_ithop(0);
+    }
+
 protected:
     /**
      * Sets the fine and coarse information.
@@ -322,29 +343,16 @@ class MultigridLevel : public UseComposition {
      */
     void set_multigrid_level(std::shared_ptr<const LinOp> prolong_op,
                              std::shared_ptr<const LinOp> coarse_op,
-			     std::shared_ptr<const LinOp> restrict_op)
+                             std::shared_ptr<const LinOp> restrict_op)
     {
-	gko::dim<2> mg_size{prolong_op->get_size()[0], restrict_op->get_size()[1]};
-	// check mg_size is the same as fine_size
-	this->set_composition(prolong_op, coarse_op, restrict_op);
-    }
-
-    std::shared_ptr<const LinOp> get_fine_op() const {
-        return fine_op_;
+        gko::dim<2> mg_size{prolong_op->get_size()[0],
+                            restrict_op->get_size()[1]};
+        GKO_ASSERT_EQUAL_DIMENSIONS(fine_op_->get_size(), mg_size);
+        // check mg_size is the same as fine_size
+        this->set_composition(prolong_op, coarse_op, restrict_op);
     }
 
-    std::shared_ptr<const LinOp> get_restrict_op() const {
-        return this->get_composition()[2];
-    }
-
-    std::shared_ptr<const LinOp> get_coarse_op() const {
-        return this->get_composition()[1];
-    }
-
-    std::shared_ptr<const LinOp> get_prolong_op() const {
-        return this->get_composition()[0];
-    }
-   
+    explicit MultigridLevel() {}
     /**
      * Creates a MultigridLevel with settings
      *
@@ -366,80 +374,6 @@ class MultigridLevel : public UseComposition {
 };
 
 
-// template <typename ConcreteObject>
-// class EnableNewDefault {
-// protected:
-
-//     std::unique_ptr<PolymorphicObject> create_default_impl(
-//         std::shared_ptr<const Executor> exec) const override
-//     {
-//         return std::unique_ptr<ConcreteObject>{new ConcreteObject(exec)};
-//     }
-
-//     PolymorphicObject *copy_from_impl(const PolymorphicObject *other)
-//     override
-//     {
-//         as<ConvertibleTo<ConcreteObject>>(other)->convert_to(self());
-//         return this;
-//     }
-
-//     PolymorphicObject *copy_from_impl(
-//         std::unique_ptr<PolymorphicObject> other) override
-//     {
-//         as<ConvertibleTo<ConcreteObject>>(other.get())->move_to(self());
-//         return this;
-//     }
-
-//     PolymorphicObject *clear_impl() override
-//     {
-//         *self() = ConcreteObject{this->get_executor()};
-//         return this;
-//     }
-
-// private:
-//     GKO_ENABLE_SELF(ConcreteObject);
-// };
-
-
-template <typename ValueType>
-class MultigridLevelOp
-    : public Composition<ValueType>,
-      public EnableCreateMethod<MultigridLevelOp<ValueType>> {
-    // friend class EnablePolymorphicObject<MultigridLevelOp, LinOp>;
-    friend class EnableCreateMethod<MultigridLevelOp>;
-    // friend class EnableNewDefault<MultigridLevelOp>;
-
-public:
-    using value_type = ValueType;
-    using EnableCreateMethod<MultigridLevelOp>::create;
-    // using EnableAbstractPolymorphicObject<MultigridLevelOp,
-    //                                          LinOp>::EnableAbstractPolymorphicObject;
-    // EnablePolymorphicObject
-    // auto clone_ilu = par_ilu->clone();
-    // auto clone_l = clone_ilu->get_l_factor();
-    // using EnableLinOp<MultigridLevelOp>::PolymorphicObject;
-    // using EnableLinOp<MultigridLevelOp>::get_executor;
-    // using EnableLinOp<MultigridLevelOp>::EnablePolymorphicAssignment;
-
-    // using Composition<ValueType>::apply;
-protected:
-    /**
-     * Creates an empty operator composition (0x0 operator).
-     *
-     * @param exec  Executor associated to the composition
-     */
-    explicit MultigridLevelOp(std::shared_ptr<const Executor> exec)
-        : Composition<ValueType>(exec)
-    {}
-
-    explicit MultigridLevelOp(std::shared_ptr<const LinOp> prolong,
-                              std::shared_ptr<const LinOp> coarse,
-                              std::shared_ptr<const LinOp> restrict)
-        : Composition<ValueType>(prolong, coarse, restrict)
-    {}
-};
-
-
 }  // namespace multigrid
 }  // namespace gko
 
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index ad09f72d716..aa6cae5047d 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -88,6 +88,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 #include <ginkgo/core/multigrid/amgx_pgm.hpp>
+#include <ginkgo/core/multigrid/mapping.hpp>
 #include <ginkgo/core/multigrid/multigrid_base.hpp>
 
 #include <ginkgo/core/preconditioner/ic.hpp>
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index 4fd677d5b5c..03b0649e04e 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -24,6 +24,7 @@ target_sources(ginkgo_omp
     matrix/sellp_kernels.cpp
     matrix/sparsity_csr_kernels.cpp
     multigrid/amgx_pgm_kernels.cpp
+    multigrid/mapping_kernels.cpp
     preconditioner/isai_kernels.cpp
     preconditioner/jacobi_kernels.cpp
     reorder/rcm_kernels.cpp
diff --git a/omp/multigrid/mapping_kernels.cpp b/omp/multigrid/mapping_kernels.cpp
new file mode 100644
index 00000000000..e614c13c29d
--- /dev/null
+++ b/omp/multigrid/mapping_kernels.cpp
@@ -0,0 +1,88 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/multigrid/mapping_kernels.hpp"
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+#include <ginkgo/core/multigrid/mapping.hpp>
+
+
+#include "core/base/allocator.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/csr_builder.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+/**
+ * @brief The MAPPING solver namespace.
+ *
+ * @ingroup mapping
+ */
+namespace mapping {
+
+
+template <typename ValueType, typename IndexType>
+void applyadd(std::shared_ptr<const DefaultExecutor> exec,
+              const multigrid::Mapping<ValueType, IndexType> *source,
+              const matrix::Dense<ValueType> *b,
+              matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_MAPPING_APPLYADD_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_applyadd(std::shared_ptr<const DefaultExecutor> exec,
+                       const multigrid::Mapping<ValueType, IndexType> *source,
+                       const matrix::Dense<ValueType> *alpha,
+                       const matrix::Dense<ValueType> *b,
+                       matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL);
+
+
+}  // namespace mapping
+}  // namespace omp
+}  // namespace kernels
+}  // namespace gko
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index b17b75ca8ba..a9e598a5cbc 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -22,6 +22,7 @@ target_sources(ginkgo_reference
     matrix/sellp_kernels.cpp
     matrix/sparsity_csr_kernels.cpp
     multigrid/amgx_pgm_kernels.cpp
+    multigrid/mapping_kernels.cpp
     preconditioner/isai_kernels.cpp
     preconditioner/jacobi_kernels.cpp
     reorder/rcm_kernels.cpp
diff --git a/reference/multigrid/mapping_kernels.cpp b/reference/multigrid/mapping_kernels.cpp
new file mode 100644
index 00000000000..ed43add542d
--- /dev/null
+++ b/reference/multigrid/mapping_kernels.cpp
@@ -0,0 +1,118 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/multigrid/mapping_kernels.hpp"
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+#include <ginkgo/core/multigrid/mapping.hpp>
+
+
+#include "core/base/allocator.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/csr_builder.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace reference {
+/**
+ * @brief The MAPPING solver namespace.
+ *
+ * @ingroup mapping
+ */
+namespace mapping {
+
+
+template <typename ValueType, typename IndexType>
+void applyadd(std::shared_ptr<const DefaultExecutor> exec,
+              const multigrid::Mapping<ValueType, IndexType> *source,
+              const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *x)
+{
+    auto x_vals = x->get_values();
+    const auto x_stride = x->get_stride();
+    auto x_dim = x->get_size();
+    auto mapindex = source->get_const_mapping_index();
+    auto num = source->get_num_stored_elements();
+    auto is_restrict = source->is_restrict();
+    for (size_type i = 0; i < num; i++) {
+        const auto x_row = is_restrict ? mapindex[i] : i;
+        const auto b_row = is_restrict ? i : mapindex[i];
+        for (size_type j = 0; j < x_dim[1]; j++) {
+            x->at(x_row, j) += b->at(b_row, j);
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_MAPPING_APPLYADD_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_applyadd(std::shared_ptr<const DefaultExecutor> exec,
+                       const multigrid::Mapping<ValueType, IndexType> *source,
+                       const matrix::Dense<ValueType> *alpha,
+                       const matrix::Dense<ValueType> *b,
+                       matrix::Dense<ValueType> *x)
+{
+    auto x_vals = x->get_values();
+    const auto x_stride = x->get_stride();
+    auto x_dim = x->get_size();
+    auto mapindex = source->get_const_mapping_index();
+    auto num = source->get_num_stored_elements();
+    auto is_restrict = source->is_restrict();
+    auto alpha_val = alpha->at(0, 0);
+    for (size_type i = 0; i < num; i++) {
+        const auto x_row = is_restrict ? mapindex[i] : i;
+        const auto b_row = is_restrict ? i : mapindex[i];
+        for (size_type j = 0; j < x_dim[1]; j++) {
+            x->at(x_row, j) += alpha_val * b->at(b_row, j);
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL);
+
+
+}  // namespace mapping
+}  // namespace reference
+}  // namespace kernels
+}  // namespace gko
diff --git a/reference/test/factorization/par_ilu_kernels.cpp b/reference/test/factorization/par_ilu_kernels.cpp
index c6ea588ad51..388d54232ad 100644
--- a/reference/test/factorization/par_ilu_kernels.cpp
+++ b/reference/test/factorization/par_ilu_kernels.cpp
@@ -483,9 +483,6 @@ TYPED_TEST(ParIlu, SetLStrategy)
         par_ilu_type::build().with_l_strategy(l_strategy).on(this->ref);
     auto par_ilu = factory->generate(this->mtx_small);
 
-    auto clone_ilu = par_ilu->clone();
-    auto clone_l = dynamic_cast<par_ilu_type *>(clone_ilu.get());
-    ASSERT_NE(clone_l, nullptr);
     ASSERT_EQ(factory->get_parameters().l_strategy, l_strategy);
     ASSERT_EQ(par_ilu->get_l_factor()->get_strategy()->get_name(),
               l_strategy->get_name());
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index 59259746c44..3af1cc8eab7 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -227,7 +227,7 @@ TYPED_TEST(AmgxPgm, CanBeCopied)
     auto copy_mtx =
         static_cast<RestrictProlong *>(copy.get())->get_system_matrix();
     auto copy_agg = static_cast<RestrictProlong *>(copy.get())->get_const_agg();
-    auto copy_coarse = copy->get_coarse_matrix();
+    auto copy_coarse = copy->get_coarse_op();
 
     this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
                                this->mtx.get());
@@ -248,7 +248,7 @@ TYPED_TEST(AmgxPgm, CanBeMoved)
     auto copy_mtx =
         static_cast<RestrictProlong *>(copy.get())->get_system_matrix();
     auto copy_agg = static_cast<RestrictProlong *>(copy.get())->get_const_agg();
-    auto copy_coarse = copy->get_coarse_matrix();
+    auto copy_coarse = copy->get_coarse_op();
 
     this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
                                this->mtx.get());
@@ -263,307 +263,303 @@ TYPED_TEST(AmgxPgm, CanBeCloned)
 {
     using Mtx = typename TestFixture::Mtx;
     using RestrictProlong = typename TestFixture::RestrictProlong;
-    auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
-    auto copy_coarse = copy->get_coarse_matrix();
-    auto clone = copy->clone();
-    // auto clone_mtx =
-    //     clone->get_system_matrix();
-    // auto clone_agg =
-    //     static_cast<RestrictProlong *>(clone.get())->get_const_agg();
+    auto clone = this->rstr_prlg->clone();
+    auto clone_mtx = clone->get_system_matrix();
+    auto clone_agg =
+        static_cast<RestrictProlong *>(clone.get())->get_const_agg();
     auto clone_coarse =
-        dynamic_cast<RestrictProlong *>(clone.get())->get_coarse_matrix();
+        dynamic_cast<RestrictProlong *>(clone.get())->get_coarse_op();
 
-    // this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
-    //                            this->mtx.get());
-    // this->assert_same_agg(clone_agg, this->agg.get_data(),
-    //                       this->agg.get_num_elems());
+    this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
+                               this->mtx.get());
+    this->assert_same_agg(clone_agg, this->agg.get_data(),
+                          this->agg.get_num_elems());
     this->assert_same_matrices(static_cast<const Mtx *>(clone_coarse.get()),
                                this->coarse.get());
 }
 
 
-// TYPED_TEST(AmgxPgm, CanBeCleared)
-// {
-//     using RestrictProlong = typename TestFixture::RestrictProlong;
-
-//     this->rstr_prlg->clear();
-//     auto mtx = static_cast<RestrictProlong *>(this->rstr_prlg.get())
-//                    ->get_system_matrix();
-//     auto coarse = this->rstr_prlg->get_coarse_matrix();
-//     auto agg = static_cast<RestrictProlong
-//     *>(this->rstr_prlg.get())->get_agg();
-
-//     ASSERT_EQ(mtx, nullptr);
-//     ASSERT_EQ(coarse, nullptr);
-//     ASSERT_EQ(agg, nullptr);
-// }
-
-
-// TYPED_TEST(AmgxPgm, RestrictApply)
-// {
-//     // fine->coarse
-//     using Vec = typename TestFixture::Vec;
-//     using value_type = typename TestFixture::value_type;
-//     auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
-
-//     gko::kernels::reference::amgx_pgm::restrict_apply(
-//         this->exec, this->agg, this->fine_b.get(), x.get());
-
-//     GKO_ASSERT_MTX_NEAR(x, this->restrict_ans, r<value_type>::value);
-// }
-
-
-// TYPED_TEST(AmgxPgm, ProlongApplyadd)
-// {
-//     using value_type = typename TestFixture::value_type;
-//     auto x = gko::clone(this->fine_x);
-
-//     gko::kernels::reference::amgx_pgm::prolong_applyadd(
-//         this->exec, this->agg, this->coarse_b.get(), x.get());
-
-//     GKO_ASSERT_MTX_NEAR(x, this->prolong_ans, r<value_type>::value);
-// }
-
-
-// TYPED_TEST(AmgxPgm, MatchEdge)
-// {
-//     using index_type = typename TestFixture::index_type;
-//     gko::Array<index_type> agg(this->exec, 5);
-//     gko::Array<index_type> snb(this->exec, 5);
-//     auto agg_val = agg.get_data();
-//     auto snb_val = snb.get_data();
-//     for (int i = 0; i < 5; i++) {
-//         agg_val[i] = -1;
-//     }
-//     snb_val[0] = 2;
-//     snb_val[1] = 0;
-//     snb_val[2] = 0;
-//     snb_val[3] = 1;
-//     snb_val[4] = 2;
+TYPED_TEST(AmgxPgm, CanBeCleared)
+{
+    using RestrictProlong = typename TestFixture::RestrictProlong;
+
+    this->rstr_prlg->clear();
+    auto mtx = static_cast<RestrictProlong *>(this->rstr_prlg.get())
+                   ->get_system_matrix();
+    auto coarse = this->rstr_prlg->get_coarse_op();
+    auto agg = static_cast<RestrictProlong *>(this->rstr_prlg.get())->get_agg();
+
+    ASSERT_EQ(mtx, nullptr);
+    ASSERT_EQ(coarse, nullptr);
+    ASSERT_EQ(agg, nullptr);
+}
+
+
+TYPED_TEST(AmgxPgm, RestrictApply)
+{
+    // fine->coarse
+    using Vec = typename TestFixture::Vec;
+    using value_type = typename TestFixture::value_type;
+    auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
+
+    gko::kernels::reference::amgx_pgm::restrict_apply(
+        this->exec, this->agg, this->fine_b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, this->restrict_ans, r<value_type>::value);
+}
+
+
+TYPED_TEST(AmgxPgm, ProlongApplyadd)
+{
+    using value_type = typename TestFixture::value_type;
+    auto x = gko::clone(this->fine_x);
+
+    gko::kernels::reference::amgx_pgm::prolong_applyadd(
+        this->exec, this->agg, this->coarse_b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, this->prolong_ans, r<value_type>::value);
+}
+
+
+TYPED_TEST(AmgxPgm, MatchEdge)
+{
+    using index_type = typename TestFixture::index_type;
+    gko::Array<index_type> agg(this->exec, 5);
+    gko::Array<index_type> snb(this->exec, 5);
+    auto agg_val = agg.get_data();
+    auto snb_val = snb.get_data();
+    for (int i = 0; i < 5; i++) {
+        agg_val[i] = -1;
+    }
+    snb_val[0] = 2;
+    snb_val[1] = 0;
+    snb_val[2] = 0;
+    snb_val[3] = 1;
+    snb_val[4] = 2;
+
+    gko::kernels::reference::amgx_pgm::match_edge(this->exec, snb, agg);
+
+    ASSERT_EQ(agg_val[0], 0);
+    ASSERT_EQ(agg_val[1], -1);
+    ASSERT_EQ(agg_val[2], 0);
+    ASSERT_EQ(agg_val[3], -1);
+    ASSERT_EQ(agg_val[4], -1);
+}
+
+
+TYPED_TEST(AmgxPgm, CountUnagg)
+{
+    using index_type = typename TestFixture::index_type;
+    gko::Array<index_type> agg(this->exec, 5);
+    auto agg_val = agg.get_data();
+    index_type num_unagg = 0;
+    agg_val[0] = 0;
+    agg_val[1] = -1;
+    agg_val[2] = 0;
+    agg_val[3] = -1;
+    agg_val[4] = -1;
+
+    gko::kernels::reference::amgx_pgm::count_unagg(this->exec, agg, &num_unagg);
+
+    ASSERT_EQ(num_unagg, 3);
+}
+
+
+TYPED_TEST(AmgxPgm, Renumber)
+{
+    using index_type = typename TestFixture::index_type;
+    gko::Array<index_type> agg(this->exec, 5);
+    auto agg_val = agg.get_data();
+    index_type num_agg = 0;
+    agg_val[0] = 0;
+    agg_val[1] = 1;
+    agg_val[2] = 0;
+    agg_val[3] = 1;
+    agg_val[4] = 4;
+
+    gko::kernels::reference::amgx_pgm::renumber(this->exec, agg, &num_agg);
+
+    ASSERT_EQ(num_agg, 3);
+    ASSERT_EQ(agg_val[0], 0);
+    ASSERT_EQ(agg_val[1], 1);
+    ASSERT_EQ(agg_val[2], 0);
+    ASSERT_EQ(agg_val[3], 1);
+    ASSERT_EQ(agg_val[4], 2);
+}
+
+
+TYPED_TEST(AmgxPgm, Generate)
+{
+    auto coarse_fine = this->amgxpgm_factory->generate(this->mtx);
+
+    auto agg_result = coarse_fine->get_const_agg();
+
+    ASSERT_EQ(agg_result[0], 0);
+    ASSERT_EQ(agg_result[1], 1);
+    ASSERT_EQ(agg_result[2], 0);
+    ASSERT_EQ(agg_result[3], 1);
+    ASSERT_EQ(agg_result[4], 0);
+}
+
+
+TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
+{
+    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
+    using Vec = typename TestFixture::Vec;
+    using value_type = typename TestFixture::value_type;
+    auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
+
+    amgx_pgm->get_restrict_op()->apply(this->fine_b.get(), x.get());
 
-//     gko::kernels::reference::amgx_pgm::match_edge(this->exec, snb, agg);
+    GKO_ASSERT_MTX_NEAR(x, this->restrict_ans, r<value_type>::value);
+}
 
-//     ASSERT_EQ(agg_val[0], 0);
-//     ASSERT_EQ(agg_val[1], -1);
-//     ASSERT_EQ(agg_val[2], 0);
-//     ASSERT_EQ(agg_val[3], -1);
-//     ASSERT_EQ(agg_val[4], -1);
-// }
 
+TYPED_TEST(AmgxPgm, CoarseFineProlongApplyadd)
+{
+    using value_type = typename TestFixture::value_type;
+    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
+    auto x = gko::clone(this->fine_x);
 
-// TYPED_TEST(AmgxPgm, CountUnagg)
-// {
-//     using index_type = typename TestFixture::index_type;
-//     gko::Array<index_type> agg(this->exec, 5);
-//     auto agg_val = agg.get_data();
-//     index_type num_unagg = 0;
-//     agg_val[0] = 0;
-//     agg_val[1] = -1;
-//     agg_val[2] = 0;
-//     agg_val[3] = -1;
-//     agg_val[4] = -1;
+    gko::as<gko::ApplyAddable>(amgx_pgm->get_prolong_op())
+        ->apply2(this->coarse_b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, this->prolong_ans, r<value_type>::value);
+}
 
-//     gko::kernels::reference::amgx_pgm::count_unagg(this->exec, agg,
-//     &num_unagg);
 
-//     ASSERT_EQ(num_unagg, 3);
-// }
-
-
-// TYPED_TEST(AmgxPgm, Renumber)
-// {
-//     using index_type = typename TestFixture::index_type;
-//     gko::Array<index_type> agg(this->exec, 5);
-//     auto agg_val = agg.get_data();
-//     index_type num_agg = 0;
-//     agg_val[0] = 0;
-//     agg_val[1] = 1;
-//     agg_val[2] = 0;
-//     agg_val[3] = 1;
-//     agg_val[4] = 4;
-
-//     gko::kernels::reference::amgx_pgm::renumber(this->exec, agg, &num_agg);
-
-//     ASSERT_EQ(num_agg, 3);
-//     ASSERT_EQ(agg_val[0], 0);
-//     ASSERT_EQ(agg_val[1], 1);
-//     ASSERT_EQ(agg_val[2], 0);
-//     ASSERT_EQ(agg_val[3], 1);
-//     ASSERT_EQ(agg_val[4], 2);
-// }
-
-
-// TYPED_TEST(AmgxPgm, Generate)
-// {
-//     auto coarse_fine = this->amgxpgm_factory->generate(this->mtx);
-
-//     auto agg_result = coarse_fine->get_const_agg();
-
-//     ASSERT_EQ(agg_result[0], 0);
-//     ASSERT_EQ(agg_result[1], 1);
-//     ASSERT_EQ(agg_result[2], 0);
-//     ASSERT_EQ(agg_result[3], 1);
-//     ASSERT_EQ(agg_result[4], 0);
-// }
-
-
-// TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
-// {
-//     auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
-//     using Vec = typename TestFixture::Vec;
-//     using value_type = typename TestFixture::value_type;
-//     auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
-
-//     amgx_pgm->restrict_apply(this->fine_b.get(), x.get());
-
-//     GKO_ASSERT_MTX_NEAR(x, this->restrict_ans, r<value_type>::value);
-// }
-
-
-// TYPED_TEST(AmgxPgm, CoarseFineProlongApplyadd)
-// {
-//     using value_type = typename TestFixture::value_type;
-//     auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
-//     auto x = gko::clone(this->fine_x);
-
-//     amgx_pgm->prolong_applyadd(this->coarse_b.get(), x.get());
-
-//     GKO_ASSERT_MTX_NEAR(x, this->prolong_ans, r<value_type>::value);
-// }
-
-
-// TYPED_TEST(AmgxPgm, CoarseFineProlongApply)
-// {
-//     using value_type = typename TestFixture::value_type;
-//     auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
-//     auto x = gko::clone(this->fine_x);
-
-//     amgx_pgm->prolong_apply(this->coarse_b.get(), x.get());
-
-//     GKO_ASSERT_MTX_NEAR(x, this->prolong_applyans, r<value_type>::value);
-// }
-
-
-// TYPED_TEST(AmgxPgm, Apply)
-// {
-//     using VT = typename TestFixture::value_type;
-//     using Vec = typename TestFixture::Vec;
-//     auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
-//     auto b = gko::clone(this->fine_x);
-//     auto x = gko::clone(this->fine_x);
-//     auto exec = amgx_pgm->get_executor();
-//     auto answer = gko::initialize<Vec>(
-//         {I<VT>({-23.0, 5.0}), I<VT>({17.0, -5.0}), I<VT>({-23.0, 5.0}),
-//          I<VT>({17.0, -5.0}), I<VT>({-23.0, 5.0})},
-//         exec);
-
-//     amgx_pgm->apply(b.get(), x.get());
-
-//     GKO_ASSERT_MTX_NEAR(x, answer, r<VT>::value);
-// }
-
-
-// TYPED_TEST(AmgxPgm, AdvancedApply)
-// {
-//     using VT = typename TestFixture::value_type;
-//     using Vec = typename TestFixture::Vec;
-//     auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
-//     auto b = gko::clone(this->fine_x);
-//     auto x = gko::clone(this->fine_x);
-//     auto exec = amgx_pgm->get_executor();
-//     auto alpha = gko::initialize<Vec>({1.0}, exec);
-//     auto beta = gko::initialize<Vec>({2.0}, exec);
-//     auto answer = gko::initialize<Vec>(
-//         {I<VT>({-27.0, 3.0}), I<VT>({19.0, -7.0}), I<VT>({-25.0, 3.0}),
-//          I<VT>({17.0, -5.0}), I<VT>({-23.0, 9.0})},
-//         exec);
-
-//     amgx_pgm->apply(alpha.get(), b.get(), beta.get(), x.get());
-
-//     GKO_ASSERT_MTX_NEAR(x, answer, r<VT>::value);
-// }
-
-
-// TYPED_TEST(AmgxPgm, FindStrongestNeighbor)
-// {
-//     using index_type = typename TestFixture::index_type;
-//     gko::Array<index_type> strongest_neighbor(this->exec, 5);
-//     gko::Array<index_type> agg(this->exec, 5);
-//     auto snb_vals = strongest_neighbor.get_data();
-//     auto agg_vals = agg.get_data();
-//     for (int i = 0; i < 5; i++) {
-//         snb_vals[i] = -1;
-//         agg_vals[i] = -1;
-//     }
-
-//     gko::kernels::reference::amgx_pgm::find_strongest_neighbor(
-//         this->exec, this->weight.get(), this->mtx_diag.get(), agg,
-//         strongest_neighbor);
-
-//     ASSERT_EQ(snb_vals[0], 2);
-//     ASSERT_EQ(snb_vals[1], 0);
-//     ASSERT_EQ(snb_vals[2], 0);
-//     ASSERT_EQ(snb_vals[3], 1);
-//     ASSERT_EQ(snb_vals[4], 2);
-// }
-
-
-// TYPED_TEST(AmgxPgm, AssignToExistAgg)
-// {
-//     using index_type = typename TestFixture::index_type;
-//     gko::Array<index_type> agg(this->exec, 5);
-//     gko::Array<index_type> intermediate_agg(this->exec, 0);
-//     auto agg_vals = agg.get_data();
-//     // 0 - 2, 1 - 3
-//     agg_vals[0] = 0;
-//     agg_vals[1] = 1;
-//     agg_vals[2] = 0;
-//     agg_vals[3] = 1;
-//     agg_vals[4] = -1;
-
-//     gko::kernels::reference::amgx_pgm::assign_to_exist_agg(
-//         this->exec, this->weight.get(), this->mtx_diag.get(), agg,
-//         intermediate_agg);
-
-//     ASSERT_EQ(agg_vals[0], 0);
-//     ASSERT_EQ(agg_vals[1], 1);
-//     ASSERT_EQ(agg_vals[2], 0);
-//     ASSERT_EQ(agg_vals[3], 1);
-//     ASSERT_EQ(agg_vals[4], 0);
-// }
-
-
-// TYPED_TEST(AmgxPgm, GenerateMtx)
-// {
-//     using index_type = typename TestFixture::index_type;
-//     using value_type = typename TestFixture::value_type;
-//     using mtx_type = typename TestFixture::Mtx;
-//     gko::Array<index_type> agg(this->exec, 5);
-//     auto agg_vals = agg.get_data();
-//     // 0 - 2, 1 - 3, 4
-//     agg_vals[0] = 0;
-//     agg_vals[1] = 1;
-//     agg_vals[2] = 0;
-//     agg_vals[3] = 1;
-//     agg_vals[4] = 2;
-//     auto coarse_ans = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
-//     coarse_ans->read({{3, 3},
-//                       {{0, 0, 4},
-//                        {0, 1, -3},
-//                        {0, 2, -1},
-//                        {1, 0, -3},
-//                        {1, 1, 5},
-//                        {1, 2, -1},
-//                        {2, 0, -2},
-//                        {2, 1, -2},
-//                        {2, 2, 5}}});
-//     auto csr_coarse = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
-
-//     gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
-//         this->exec, this->mtx.get(), agg, csr_coarse.get());
-
-//     GKO_ASSERT_MTX_NEAR(csr_coarse, coarse_ans, r<value_type>::value);
-// }
+TYPED_TEST(AmgxPgm, CoarseFineProlongApply)
+{
+    using value_type = typename TestFixture::value_type;
+    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
+    auto x = gko::clone(this->fine_x);
+
+    amgx_pgm->get_prolong_op()->apply(this->coarse_b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, this->prolong_applyans, r<value_type>::value);
+}
+
+
+TYPED_TEST(AmgxPgm, Apply)
+{
+    using VT = typename TestFixture::value_type;
+    using Vec = typename TestFixture::Vec;
+    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
+    auto b = gko::clone(this->fine_x);
+    auto x = gko::clone(this->fine_x);
+    auto exec = amgx_pgm->get_executor();
+    auto answer = gko::initialize<Vec>(
+        {I<VT>({-23.0, 5.0}), I<VT>({17.0, -5.0}), I<VT>({-23.0, 5.0}),
+         I<VT>({17.0, -5.0}), I<VT>({-23.0, 5.0})},
+        exec);
+
+    amgx_pgm->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, answer, r<VT>::value);
+}
+
+
+TYPED_TEST(AmgxPgm, AdvancedApply)
+{
+    using VT = typename TestFixture::value_type;
+    using Vec = typename TestFixture::Vec;
+    auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
+    auto b = gko::clone(this->fine_x);
+    auto x = gko::clone(this->fine_x);
+    auto exec = amgx_pgm->get_executor();
+    auto alpha = gko::initialize<Vec>({1.0}, exec);
+    auto beta = gko::initialize<Vec>({2.0}, exec);
+    auto answer = gko::initialize<Vec>(
+        {I<VT>({-27.0, 3.0}), I<VT>({19.0, -7.0}), I<VT>({-25.0, 3.0}),
+         I<VT>({17.0, -5.0}), I<VT>({-23.0, 9.0})},
+        exec);
+
+    amgx_pgm->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, answer, r<VT>::value);
+}
+
+
+TYPED_TEST(AmgxPgm, FindStrongestNeighbor)
+{
+    using index_type = typename TestFixture::index_type;
+    gko::Array<index_type> strongest_neighbor(this->exec, 5);
+    gko::Array<index_type> agg(this->exec, 5);
+    auto snb_vals = strongest_neighbor.get_data();
+    auto agg_vals = agg.get_data();
+    for (int i = 0; i < 5; i++) {
+        snb_vals[i] = -1;
+        agg_vals[i] = -1;
+    }
+
+    gko::kernels::reference::amgx_pgm::find_strongest_neighbor(
+        this->exec, this->weight.get(), this->mtx_diag.get(), agg,
+        strongest_neighbor);
+
+    ASSERT_EQ(snb_vals[0], 2);
+    ASSERT_EQ(snb_vals[1], 0);
+    ASSERT_EQ(snb_vals[2], 0);
+    ASSERT_EQ(snb_vals[3], 1);
+    ASSERT_EQ(snb_vals[4], 2);
+}
+
+
+TYPED_TEST(AmgxPgm, AssignToExistAgg)
+{
+    using index_type = typename TestFixture::index_type;
+    gko::Array<index_type> agg(this->exec, 5);
+    gko::Array<index_type> intermediate_agg(this->exec, 0);
+    auto agg_vals = agg.get_data();
+    // 0 - 2, 1 - 3
+    agg_vals[0] = 0;
+    agg_vals[1] = 1;
+    agg_vals[2] = 0;
+    agg_vals[3] = 1;
+    agg_vals[4] = -1;
+
+    gko::kernels::reference::amgx_pgm::assign_to_exist_agg(
+        this->exec, this->weight.get(), this->mtx_diag.get(), agg,
+        intermediate_agg);
+
+    ASSERT_EQ(agg_vals[0], 0);
+    ASSERT_EQ(agg_vals[1], 1);
+    ASSERT_EQ(agg_vals[2], 0);
+    ASSERT_EQ(agg_vals[3], 1);
+    ASSERT_EQ(agg_vals[4], 0);
+}
+
+
+TYPED_TEST(AmgxPgm, GenerateMtx)
+{
+    using index_type = typename TestFixture::index_type;
+    using value_type = typename TestFixture::value_type;
+    using mtx_type = typename TestFixture::Mtx;
+    gko::Array<index_type> agg(this->exec, 5);
+    auto agg_vals = agg.get_data();
+    // 0 - 2, 1 - 3, 4
+    agg_vals[0] = 0;
+    agg_vals[1] = 1;
+    agg_vals[2] = 0;
+    agg_vals[3] = 1;
+    agg_vals[4] = 2;
+    auto coarse_ans = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
+    coarse_ans->read({{3, 3},
+                      {{0, 0, 4},
+                       {0, 1, -3},
+                       {0, 2, -1},
+                       {1, 0, -3},
+                       {1, 1, 5},
+                       {1, 2, -1},
+                       {2, 0, -2},
+                       {2, 1, -2},
+                       {2, 2, 5}}});
+    auto csr_coarse = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
+
+    gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
+        this->exec, this->mtx.get(), agg, csr_coarse.get());
+
+    GKO_ASSERT_MTX_NEAR(csr_coarse, coarse_ans, r<value_type>::value);
+}
 
 
 }  // namespace

From a00e1d820743038694376fd8b615c2789203b6ae Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 4 Mar 2021 17:02:25 +0800
Subject: [PATCH 0481/1008] implement fill_seq_array and add test

---
 common/components/fill_array.hpp.inc     | 11 +++++++++++
 core/components/fill_array.hpp           | 12 +++++++++---
 core/device_hooks/common_kernels.inc.cpp |  9 ++++++---
 cuda/components/fill_array.cu            | 13 +++++++++++++
 cuda/test/components/fill_array.cpp      | 15 ++++++++++++++-
 dpcpp/components/fill_array.dp.cpp       | 15 +++++++++++++++
 dpcpp/test/components/fill_array.cpp     | 16 +++++++++++++++-
 hip/components/fill_array.hip.cpp        | 13 +++++++++++++
 hip/test/components/fill_array.hip.cpp   | 15 ++++++++++++++-
 omp/components/fill_array.cpp            | 13 +++++++++++++
 omp/test/components/fill_array.cpp       | 16 +++++++++++++++-
 reference/components/fill_array.cpp      | 13 +++++++++++++
 reference/test/components/fill_array.cpp | 16 +++++++++++++++-
 13 files changed, 166 insertions(+), 11 deletions(-)

diff --git a/common/components/fill_array.hpp.inc b/common/components/fill_array.hpp.inc
index 1d219c7c634..d33350636c3 100644
--- a/common/components/fill_array.hpp.inc
+++ b/common/components/fill_array.hpp.inc
@@ -45,4 +45,15 @@ __global__ __launch_bounds__(default_block_size) void fill_array(
 }
 
 
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void fill_seq_array(
+    size_type n, ValueType *__restrict__ array)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    if (tidx < n) {
+        array[tidx] = tidx;
+    }
+}
+
+
 }  // namespace kernel
diff --git a/core/components/fill_array.hpp b/core/components/fill_array.hpp
index 7270d168bc1..0845f03701f 100644
--- a/core/components/fill_array.hpp
+++ b/core/components/fill_array.hpp
@@ -49,10 +49,16 @@ namespace kernels {
     void fill_array(std::shared_ptr<const DefaultExecutor> exec, \
                     ValueType *data, size_type num_entries, ValueType val)
 
+#define GKO_DECLARE_FILL_SEQ_ARRAY_KERNEL(ValueType)                 \
+    void fill_seq_array(std::shared_ptr<const DefaultExecutor> exec, \
+                        ValueType *data, size_type num_entries)
 
-#define GKO_DECLARE_ALL_AS_TEMPLATES \
-    template <typename ValueType>    \
-    GKO_DECLARE_FILL_ARRAY_KERNEL(ValueType)
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES          \
+    template <typename ValueType>             \
+    GKO_DECLARE_FILL_ARRAY_KERNEL(ValueType); \
+    template <typename ValueType>             \
+    GKO_DECLARE_FILL_SEQ_ARRAY_KERNEL(ValueType)
 
 
 namespace omp {
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 90205943074..c965154def0 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -98,9 +98,12 @@ template GKO_DECLARE_PREFIX_SUM_KERNEL(size_type);
 template <typename IndexType>
 GKO_DECLARE_FILL_ARRAY_KERNEL(IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
-template GKO_DECLARE_FILL_ARRAY_KERNEL(size_type);
+GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
+
+template <typename IndexType>
+GKO_DECLARE_FILL_SEQ_ARRAY_KERNEL(IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_SEQ_ARRAY_KERNEL);
 
 template <typename ValueType>
 GKO_DECLARE_INPLACE_ABSOLUTE_ARRAY_KERNEL(ValueType)
diff --git a/cuda/components/fill_array.cu b/cuda/components/fill_array.cu
index fba7a3104a2..6098399064f 100644
--- a/cuda/components/fill_array.cu
+++ b/cuda/components/fill_array.cu
@@ -62,6 +62,19 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
 GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
+template <typename ValueType>
+void fill_seq_array(std::shared_ptr<const DefaultExecutor> exec,
+                    ValueType *array, size_type n)
+{
+    const dim3 block_size(default_block_size, 1, 1);
+    const dim3 grid_size(ceildiv(n, block_size.x), 1, 1);
+    kernel::fill_seq_array<<<grid_size, block_size, 0, 0>>>(
+        n, as_cuda_type(array));
+}
+
+GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_SEQ_ARRAY_KERNEL);
+
+
 }  // namespace components
 }  // namespace cuda
 }  // namespace kernels
diff --git a/cuda/test/components/fill_array.cpp b/cuda/test/components/fill_array.cpp
index 1a774a0ddac..cb4c06c571f 100644
--- a/cuda/test/components/fill_array.cpp
+++ b/cuda/test/components/fill_array.cpp
@@ -58,9 +58,11 @@ class FillArray : public ::testing::Test {
           exec(gko::CudaExecutor::create(0, ref)),
           total_size(6344),
           vals(ref, total_size),
-          dvals(exec, total_size)
+          dvals(exec, total_size),
+          seqs(ref, total_size)
     {
         std::fill_n(vals.get_data(), total_size, 1234.0);
+        std::iota(seqs.get_data(), seqs.get_data() + total_size, 0);
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -68,6 +70,7 @@ class FillArray : public ::testing::Test {
     gko::size_type total_size;
     gko::Array<value_type> vals;
     gko::Array<value_type> dvals;
+    gko::Array<value_type> seqs;
 };
 
 
@@ -75,8 +78,18 @@ TEST_F(FillArray, EqualsReference)
 {
     gko::kernels::cuda::components::fill_array(exec, dvals.get_data(),
                                                total_size, 1234.0);
+
     GKO_ASSERT_ARRAY_EQ(vals, dvals);
 }
 
 
+TEST_F(FillArray, FillSeqEqualsReference)
+{
+    gko::kernels::cuda::components::fill_seq_array(exec, dvals.get_data(),
+                                                   total_size);
+
+    GKO_ASSERT_ARRAY_EQ(seqs, dvals);
+}
+
+
 }  // namespace
diff --git a/dpcpp/components/fill_array.dp.cpp b/dpcpp/components/fill_array.dp.cpp
index f7e2307f263..9edffc8eaf6 100644
--- a/dpcpp/components/fill_array.dp.cpp
+++ b/dpcpp/components/fill_array.dp.cpp
@@ -57,6 +57,21 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
 GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
+template <typename ValueType>
+void fill_seq_array(std::shared_ptr<const DefaultExecutor> exec,
+                    ValueType *array, size_type n)
+{
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{n}, [=](sycl::id<1> idx_id) {
+            const auto idx = idx_id[0];
+            array[idx] = idx;
+        });
+    });
+}
+
+GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_SEQ_ARRAY_KERNEL);
+
+
 }  // namespace components
 }  // namespace dpcpp
 }  // namespace kernels
diff --git a/dpcpp/test/components/fill_array.cpp b/dpcpp/test/components/fill_array.cpp
index 1f8f6e04858..75474ea4ca8 100644
--- a/dpcpp/test/components/fill_array.cpp
+++ b/dpcpp/test/components/fill_array.cpp
@@ -59,9 +59,11 @@ class FillArray : public ::testing::Test {
           exec(gko::DpcppExecutor::create(0, ref)),
           total_size(63531),
           vals(ref, total_size),
-          dvals(exec, total_size)
+          dvals(exec, total_size),
+          seqs(ref, total_size)
     {
         std::fill_n(vals.get_data(), total_size, T(1523));
+        std::iota(seqs.get_data(), seqs.get_data() + total_size, 0);
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -69,6 +71,7 @@ class FillArray : public ::testing::Test {
     gko::size_type total_size;
     gko::Array<value_type> vals;
     gko::Array<value_type> dvals;
+    gko::Array<value_type> seqs;
 };
 
 TYPED_TEST_SUITE(FillArray, gko::test::ValueAndIndexTypes);
@@ -79,8 +82,19 @@ TYPED_TEST(FillArray, EqualsReference)
     using T = typename TestFixture::value_type;
     gko::kernels::dpcpp::components::fill_array(
         this->exec, this->dvals.get_data(), this->total_size, T(1523));
+
     GKO_ASSERT_ARRAY_EQ(this->vals, this->dvals);
 }
 
 
+TYPED_TEST(FillArray, EqualsReference)
+{
+    using T = typename TestFixture::value_type;
+    gko::kernels::dpcpp::components::fill_seq_array(
+        this->exec, this->dvals.get_data(), this->total_size);
+
+    GKO_ASSERT_ARRAY_EQ(this->seqs, this->dvals);
+}
+
+
 }  // namespace
diff --git a/hip/components/fill_array.hip.cpp b/hip/components/fill_array.hip.cpp
index 8df17d89a79..936c4324c86 100644
--- a/hip/components/fill_array.hip.cpp
+++ b/hip/components/fill_array.hip.cpp
@@ -67,6 +67,19 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
 GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
+template <typename ValueType>
+void fill_seq_array(std::shared_ptr<const DefaultExecutor> exec,
+                    ValueType *array, size_type n)
+{
+    const dim3 block_size(default_block_size, 1, 1);
+    const dim3 grid_size(ceildiv(n, block_size.x), 1, 1);
+    hipLaunchKernelGGL(kernel::fill_seq_array, dim3(grid_size),
+                       dim3(block_size), 0, 0, n, as_hip_type(array));
+}
+
+GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_SEQ_ARRAY_KERNEL);
+
+
 }  // namespace components
 }  // namespace hip
 }  // namespace kernels
diff --git a/hip/test/components/fill_array.hip.cpp b/hip/test/components/fill_array.hip.cpp
index c139829596a..d9f5e20f447 100644
--- a/hip/test/components/fill_array.hip.cpp
+++ b/hip/test/components/fill_array.hip.cpp
@@ -65,9 +65,11 @@ class FillArray : public ::testing::Test {
           exec(gko::HipExecutor::create(0, ref)),
           total_size(6344),
           vals(ref, total_size),
-          dvals(exec, total_size)
+          dvals(exec, total_size),
+          seqs(ref, total_size)
     {
         std::fill_n(vals.get_data(), total_size, 1234.0);
+        std::iota(seqs.get_data(), seqs.get_data() + total_size, 0);
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -75,6 +77,7 @@ class FillArray : public ::testing::Test {
     gko::size_type total_size;
     gko::Array<value_type> vals;
     gko::Array<value_type> dvals;
+    gko::Array<value_type> seqs;
 };
 
 
@@ -82,8 +85,18 @@ TEST_F(FillArray, EqualsReference)
 {
     gko::kernels::hip::components::fill_array(exec, dvals.get_data(),
                                               total_size, 1234.0);
+
     GKO_ASSERT_ARRAY_EQ(vals, dvals);
 }
 
 
+TEST_F(FillArray, FillSeqEqualsReference)
+{
+    gko::kernels::hip::components::fill_seq_array(exec, dvals.get_data(),
+                                                  total_size);
+
+    GKO_ASSERT_ARRAY_EQ(seqs, dvals);
+}
+
+
 }  // namespace
diff --git a/omp/components/fill_array.cpp b/omp/components/fill_array.cpp
index 7f32a4f14c8..5b489af4b8d 100644
--- a/omp/components/fill_array.cpp
+++ b/omp/components/fill_array.cpp
@@ -52,6 +52,19 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
 GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
+template <typename ValueType>
+void fill_seq_array(std::shared_ptr<const DefaultExecutor> exec,
+                    ValueType *array, size_type n)
+{
+#pragma omp parallel for
+    for (size_type i = 0; i < n; ++i) {
+        array[i] = i;
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_SEQ_ARRAY_KERNEL);
+
+
 }  // namespace components
 }  // namespace omp
 }  // namespace kernels
diff --git a/omp/test/components/fill_array.cpp b/omp/test/components/fill_array.cpp
index eddf3e9eb51..f1810fe57cc 100644
--- a/omp/test/components/fill_array.cpp
+++ b/omp/test/components/fill_array.cpp
@@ -59,9 +59,11 @@ class FillArray : public ::testing::Test {
           exec(gko::OmpExecutor::create()),
           total_size(63531),
           vals(ref, total_size),
-          dvals(exec, total_size)
+          dvals(exec, total_size),
+          seqs(ref, total_size)
     {
         std::fill_n(vals.get_data(), total_size, T(1523));
+        std::iota(seqs.get_data(), seqs.get_data() + total_size, 0);
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -69,6 +71,7 @@ class FillArray : public ::testing::Test {
     gko::size_type total_size;
     gko::Array<value_type> vals;
     gko::Array<value_type> dvals;
+    gko::Array<value_type> seqs;
 };
 
 TYPED_TEST_SUITE(FillArray, gko::test::ValueAndIndexTypes);
@@ -79,8 +82,19 @@ TYPED_TEST(FillArray, EqualsReference)
     using T = typename TestFixture::value_type;
     gko::kernels::omp::components::fill_array(
         this->exec, this->dvals.get_data(), this->total_size, T(1523));
+
     GKO_ASSERT_ARRAY_EQ(this->vals, this->dvals);
 }
 
 
+TYPED_TEST(FillArray, FillSeqEqualsReference)
+{
+    using T = typename TestFixture::value_type;
+    gko::kernels::omp::components::fill_seq_array(
+        this->exec, this->dvals.get_data(), this->total_size);
+
+    GKO_ASSERT_ARRAY_EQ(this->seqs, this->dvals);
+}
+
+
 }  // namespace
diff --git a/reference/components/fill_array.cpp b/reference/components/fill_array.cpp
index 099215d09f4..f2fdba585ec 100644
--- a/reference/components/fill_array.cpp
+++ b/reference/components/fill_array.cpp
@@ -30,6 +30,9 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include <numeric>
+
+
 #include "core/components/fill_array.hpp"
 
 
@@ -49,6 +52,16 @@ void fill_array(std::shared_ptr<const DefaultExecutor> exec, ValueType *array,
 GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_ARRAY_KERNEL);
 
 
+template <typename ValueType>
+void fill_seq_array(std::shared_ptr<const DefaultExecutor> exec,
+                    ValueType *array, size_type n)
+{
+    std::iota(array, array + n, 0);
+}
+
+GKO_INSTANTIATE_FOR_EACH_TEMPLATE_TYPE(GKO_DECLARE_FILL_SEQ_ARRAY_KERNEL);
+
+
 }  // namespace components
 }  // namespace reference
 }  // namespace kernels
diff --git a/reference/test/components/fill_array.cpp b/reference/test/components/fill_array.cpp
index c58b2d27995..0daa37aaee1 100644
--- a/reference/test/components/fill_array.cpp
+++ b/reference/test/components/fill_array.cpp
@@ -58,15 +58,18 @@ class FillArray : public ::testing::Test {
         : ref(gko::ReferenceExecutor::create()),
           total_size(6344),
           expected(ref, total_size),
-          vals(ref, total_size)
+          vals(ref, total_size),
+          seqs(ref, total_size)
     {
         std::fill_n(expected.get_data(), total_size, T(6453));
+        std::iota(seqs.get_data(), seqs.get_data() + total_size, 0);
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
     gko::size_type total_size;
     gko::Array<value_type> expected;
     gko::Array<value_type> vals;
+    gko::Array<value_type> seqs;
 };
 
 TYPED_TEST_SUITE(FillArray, gko::test::ValueAndIndexTypes);
@@ -77,8 +80,19 @@ TYPED_TEST(FillArray, EqualsReference)
     using T = typename TestFixture::value_type;
     gko::kernels::reference::components::fill_array(
         this->ref, this->vals.get_data(), this->total_size, T(6453));
+
     GKO_ASSERT_ARRAY_EQ(this->vals, this->expected);
 }
 
 
+TYPED_TEST(FillArray, FillSeqEqualsReference)
+{
+    using T = typename TestFixture::value_type;
+    gko::kernels::reference::components::fill_seq_array(
+        this->ref, this->vals.get_data(), this->total_size);
+
+    GKO_ASSERT_ARRAY_EQ(this->vals, this->seqs);
+}
+
+
 }  // namespace

From 29c1aaee9795a7c6eee0516bc2279e4863561f00 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 4 Mar 2021 19:56:21 +0800
Subject: [PATCH 0482/1008] use csr to represent restrict/prolong

---
 core/multigrid/amgx_pgm.cpp                   | 21 +++++++++++++++----
 reference/test/multigrid/amgx_pgm_kernels.cpp |  6 ++++--
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index 6860482229a..8fe8cb19233 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -64,6 +64,7 @@ GKO_REGISTER_OPERATION(find_strongest_neighbor,
 GKO_REGISTER_OPERATION(assign_to_exist_agg, amgx_pgm::assign_to_exist_agg);
 GKO_REGISTER_OPERATION(amgx_pgm_generate, amgx_pgm::amgx_pgm_generate);
 GKO_REGISTER_OPERATION(fill_array, components::fill_array);
+GKO_REGISTER_OPERATION(fill_seq_array, components::fill_seq_array);
 
 
 }  // namespace amgx_pgm
@@ -162,10 +163,22 @@ void AmgxPgm<ValueType, IndexType>::generate()
     // this->set_multigrid_level(system_matrix_, coarse_matrix);
     auto coarse_dim = coarse_matrix->get_size()[0];
     auto fine_dim = system_matrix_->get_size()[0];
-    auto restrict_op = Mapping<ValueType, IndexType>::create(
-        exec, gko::dim<2>{coarse_dim, fine_dim}, agg_);
-    auto prolong_op = Mapping<ValueType, IndexType>::create(
-        exec, gko::dim<2>{fine_dim, coarse_dim}, agg_, false);
+
+    // TODO: prolong_op can be done with lightway format
+    auto prolong_op = share(
+        matrix_type::create(exec, gko::dim<2>{fine_dim, coarse_dim}, fine_dim));
+    exec->copy_from(exec.get(), agg_.get_num_elems(), agg_.get_const_data(),
+                    prolong_op->get_col_idxs());
+    exec->run(amgx_pgm::make_fill_seq_array(prolong_op->get_row_ptrs(),
+                                            fine_dim + 1));
+    exec->run(amgx_pgm::make_fill_array(prolong_op->get_values(), fine_dim,
+                                        one<ValueType>()));
+    // TODO: implement the restrict_op from aggregation.
+    auto restrict_op = share(prolong_op->transpose());
+    // auto restrict_op = Mapping<ValueType, IndexType>::create(
+    //     exec, gko::dim<2>{coarse_dim, fine_dim}, agg_);
+    // auto prolong_op = Mapping<ValueType, IndexType>::create(
+    //     exec, gko::dim<2>{fine_dim, coarse_dim}, agg_, false);
     this->set_composition(share(prolong_op), coarse_matrix, share(restrict_op));
 }
 
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index 3af1cc8eab7..03bf86099da 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -418,11 +418,13 @@ TYPED_TEST(AmgxPgm, CoarseFineRestrictApply)
 TYPED_TEST(AmgxPgm, CoarseFineProlongApplyadd)
 {
     using value_type = typename TestFixture::value_type;
+    using Vec = typename TestFixture::Vec;
     auto amgx_pgm = this->amgxpgm_factory->generate(this->mtx);
+    auto one = gko::initialize<Vec>({value_type{1.0}}, this->exec);
     auto x = gko::clone(this->fine_x);
 
-    gko::as<gko::ApplyAddable>(amgx_pgm->get_prolong_op())
-        ->apply2(this->coarse_b.get(), x.get());
+    amgx_pgm->get_prolong_op()->apply(one.get(), this->coarse_b.get(),
+                                      one.get(), x.get());
 
     GKO_ASSERT_MTX_NEAR(x, this->prolong_ans, r<value_type>::value);
 }

From 07e4978bcd3c18374f2e57b36ab2c952b8dc02ac Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 5 Mar 2021 16:41:11 +0800
Subject: [PATCH 0483/1008] remove unused func, add documentation

---
 core/CMakeLists.txt                           |   1 -
 core/device_hooks/common_kernels.inc.cpp      |  32 --
 core/multigrid/amgx_pgm.cpp                   |  50 +--
 core/multigrid/amgx_pgm_kernels.hpp           |  16 -
 core/multigrid/mapping.cpp                    |  89 ----
 core/multigrid/mapping_kernels.hpp            | 126 ------
 core/test/multigrid/amgx_pgm.cpp              |   6 +-
 cuda/CMakeLists.txt                           |   1 -
 cuda/multigrid/amgx_pgm_kernels.cu            |  20 -
 cuda/multigrid/mapping_kernels.cu             |  88 ----
 dpcpp/CMakeLists.txt                          |   1 -
 dpcpp/multigrid/amgx_pgm_kernels.dp.cpp       |  20 -
 dpcpp/multigrid/mapping_kernels.dp.cpp        |  88 ----
 hip/CMakeLists.txt                            |   1 -
 hip/multigrid/amgx_pgm_kernels.hip.cpp        |  20 -
 hip/multigrid/mapping_kernels.hip.cpp         |  88 ----
 include/ginkgo/core/base/composition.hpp      |  25 +-
 include/ginkgo/core/multigrid/amgx_pgm.hpp    |  17 +-
 include/ginkgo/core/multigrid/mapping.hpp     | 158 --------
 .../ginkgo/core/multigrid/multigrid_base.hpp  | 381 ------------------
 .../ginkgo/core/multigrid/multigrid_level.hpp | 148 +++++++
 include/ginkgo/ginkgo.hpp                     |   3 +-
 omp/CMakeLists.txt                            |   1 -
 omp/multigrid/amgx_pgm_kernels.cpp            |  20 -
 omp/multigrid/mapping_kernels.cpp             |  88 ----
 reference/CMakeLists.txt                      |   1 -
 reference/components/fill_array.cpp           |   4 +-
 reference/multigrid/amgx_pgm_kernels.cpp      |  47 ---
 reference/multigrid/mapping_kernels.cpp       | 118 ------
 reference/test/multigrid/amgx_pgm_kernels.cpp |  28 +-
 30 files changed, 188 insertions(+), 1498 deletions(-)
 delete mode 100644 core/multigrid/mapping.cpp
 delete mode 100644 core/multigrid/mapping_kernels.hpp
 delete mode 100644 cuda/multigrid/mapping_kernels.cu
 delete mode 100644 dpcpp/multigrid/mapping_kernels.dp.cpp
 delete mode 100644 hip/multigrid/mapping_kernels.hip.cpp
 delete mode 100644 include/ginkgo/core/multigrid/mapping.hpp
 delete mode 100644 include/ginkgo/core/multigrid/multigrid_base.hpp
 create mode 100644 include/ginkgo/core/multigrid/multigrid_level.hpp
 delete mode 100644 omp/multigrid/mapping_kernels.cpp
 delete mode 100644 reference/multigrid/mapping_kernels.cpp

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 2fadd5e53b7..a4f8594249f 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -32,7 +32,6 @@ target_sources(ginkgo
     matrix/sellp.cpp
     matrix/sparsity_csr.cpp
     multigrid/amgx_pgm.cpp
-    multigrid/mapping.cpp
     preconditioner/isai.cpp
     preconditioner/jacobi.cpp
     reorder/rcm.cpp
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index c965154def0..408d6e4dbf0 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -53,7 +53,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/sellp_kernels.hpp"
 #include "core/matrix/sparsity_csr_kernels.hpp"
 #include "core/multigrid/amgx_pgm_kernels.hpp"
-#include "core/multigrid/mapping_kernels.hpp"
 #include "core/preconditioner/isai_kernels.hpp"
 #include "core/preconditioner/jacobi_kernels.hpp"
 #include "core/reorder/rcm_kernels.hpp"
@@ -1211,18 +1210,6 @@ GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_RCM_GET_DEGREE_OF_NODES_KERNEL);
 namespace amgx_pgm {
 
 
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL);
-
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
-
 template <typename IndexType>
 GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL(IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
@@ -1292,25 +1279,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
 
 
 }  // namespace implicit_residual_norm
-
-
-namespace mapping {
-
-
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_MAPPING_APPLYADD_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_MAPPING_APPLYADD_KERNEL);
-
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL);
-
-
-}  // namespace mapping
 }  // namespace GKO_HOOK_MODULE
 }  // namespace kernels
 }  // namespace gko
diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index 8fe8cb19233..09edd76d101 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -54,8 +54,6 @@ namespace multigrid {
 namespace amgx_pgm {
 
 
-GKO_REGISTER_OPERATION(restrict_apply, amgx_pgm::restrict_apply);
-GKO_REGISTER_OPERATION(prolong_applyadd, amgx_pgm::prolong_applyadd);
 GKO_REGISTER_OPERATION(match_edge, amgx_pgm::match_edge);
 GKO_REGISTER_OPERATION(count_unagg, amgx_pgm::count_unagg);
 GKO_REGISTER_OPERATION(renumber, amgx_pgm::renumber);
@@ -138,9 +136,9 @@ void AmgxPgm<ValueType, IndexType>::generate()
         // Get the num_unagg
         exec->run(amgx_pgm::make_count_unagg(agg_, &num_unagg));
         // no new match, all match, or the ratio of num_unagg/num is lower
-        // than parameter.max_unassigned_percentage
+        // than parameter.max_unassigned_ratio
         if (num_unagg == 0 || num_unagg == num_unagg_prev ||
-            num_unagg < parameters_.max_unassigned_percentage * num_rows) {
+            num_unagg < parameters_.max_unassigned_ratio * num_rows) {
             break;
         }
         num_unagg_prev = num_unagg;
@@ -153,7 +151,7 @@ void AmgxPgm<ValueType, IndexType>::generate()
     // Assign all left points
     exec->run(amgx_pgm::make_assign_to_exist_agg(weight_mtx.get(), diag.get(),
                                                  agg_, intermediate_agg));
-    IndexType num_agg;
+    IndexType num_agg = 0;
     // Renumber the index
     exec->run(amgx_pgm::make_renumber(agg_, &num_agg));
 
@@ -175,47 +173,7 @@ void AmgxPgm<ValueType, IndexType>::generate()
                                         one<ValueType>()));
     // TODO: implement the restrict_op from aggregation.
     auto restrict_op = share(prolong_op->transpose());
-    // auto restrict_op = Mapping<ValueType, IndexType>::create(
-    //     exec, gko::dim<2>{coarse_dim, fine_dim}, agg_);
-    // auto prolong_op = Mapping<ValueType, IndexType>::create(
-    //     exec, gko::dim<2>{fine_dim, coarse_dim}, agg_, false);
-    this->set_composition(share(prolong_op), coarse_matrix, share(restrict_op));
-}
-
-
-template <typename ValueType, typename IndexType>
-void AmgxPgm<ValueType, IndexType>::restrict_apply_impl(const LinOp *b,
-                                                        LinOp *x) const
-{
-    auto exec = this->get_executor();
-    exec->run(amgx_pgm::make_restrict_apply(agg_,
-                                            as<matrix::Dense<ValueType>>(b),
-                                            as<matrix::Dense<ValueType>>(x)));
-}
-
-
-template <typename ValueType, typename IndexType>
-void AmgxPgm<ValueType, IndexType>::prolong_applyadd_impl(const LinOp *b,
-                                                          LinOp *x) const
-{
-    auto exec = this->get_executor();
-    exec->run(amgx_pgm::make_prolong_applyadd(agg_,
-                                              as<matrix::Dense<ValueType>>(b),
-                                              as<matrix::Dense<ValueType>>(x)));
-}
-
-
-template <typename ValueType, typename IndexType>
-void AmgxPgm<ValueType, IndexType>::prolong_apply_impl(const LinOp *b,
-                                                       LinOp *x) const
-{
-    auto exec = this->get_executor();
-    auto vec_x = as<matrix::Dense<ValueType>>(x);
-    exec->run(amgx_pgm::make_fill_array(
-        vec_x->get_values(), vec_x->get_size()[0] * vec_x->get_stride(),
-        zero<ValueType>()));
-    exec->run(amgx_pgm::make_prolong_applyadd(
-        agg_, as<matrix::Dense<ValueType>>(b), vec_x));
+    this->set_multigrid_level(prolong_op, coarse_matrix, restrict_op);
 }
 
 
diff --git a/core/multigrid/amgx_pgm_kernels.hpp b/core/multigrid/amgx_pgm_kernels.hpp
index 5dcaddddaad..9109c59e610 100644
--- a/core/multigrid/amgx_pgm_kernels.hpp
+++ b/core/multigrid/amgx_pgm_kernels.hpp
@@ -51,18 +51,6 @@ namespace kernels {
 namespace amgx_pgm {
 
 
-#define GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL(ValueType, IndexType) \
-    void restrict_apply(std::shared_ptr<const DefaultExecutor> exec,     \
-                        const Array<IndexType> &agg,                     \
-                        const matrix::Dense<ValueType> *b,               \
-                        matrix::Dense<ValueType> *x)
-
-#define GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL(ValueType, IndexType) \
-    void prolong_applyadd(std::shared_ptr<const DefaultExecutor> exec,     \
-                          const Array<IndexType> &agg,                     \
-                          const matrix::Dense<ValueType> *b,               \
-                          matrix::Dense<ValueType> *x)
-
 #define GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL(IndexType)        \
     void match_edge(std::shared_ptr<const DefaultExecutor> exec, \
                     const Array<IndexType> &strongest_neighbor,  \
@@ -97,10 +85,6 @@ namespace amgx_pgm {
                            matrix::Csr<ValueType, IndexType> *coarse)
 
 #define GKO_DECLARE_ALL_AS_TEMPLATES                                    \
-    template <typename ValueType, typename IndexType>                   \
-    GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL(ValueType, IndexType);   \
-    template <typename ValueType, typename IndexType>                   \
-    GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL(ValueType, IndexType); \
     template <typename IndexType>                                       \
     GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL(IndexType);                  \
     template <typename IndexType>                                       \
diff --git a/core/multigrid/mapping.cpp b/core/multigrid/mapping.cpp
deleted file mode 100644
index bb4ed40a061..00000000000
--- a/core/multigrid/mapping.cpp
+++ /dev/null
@@ -1,89 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/multigrid/mapping.hpp>
-
-
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/base/polymorphic_object.hpp>
-#include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/base/utils.hpp>
-#include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/identity.hpp>
-
-
-#include "core/components/fill_array.hpp"
-#include "core/matrix/csr_builder.hpp"
-#include "core/multigrid/mapping_kernels.hpp"
-
-
-namespace gko {
-namespace multigrid {
-namespace mapping {
-
-
-GKO_REGISTER_OPERATION(applyadd, mapping::applyadd);
-GKO_REGISTER_OPERATION(advanced_applyadd, mapping::advanced_applyadd);
-
-
-}  // namespace mapping
-
-
-template <typename ValueType, typename IndexType>
-void Mapping<ValueType, IndexType>::apply2_impl(const LinOp *b, LinOp *x) const
-{
-    auto exec = this->get_executor();
-    exec->run(mapping::make_applyadd(this, as<matrix::Dense<ValueType>>(b),
-                                     as<matrix::Dense<ValueType>>(x)));
-}
-
-
-template <typename ValueType, typename IndexType>
-void Mapping<ValueType, IndexType>::apply2_impl(const LinOp *alpha,
-                                                const LinOp *b, LinOp *x) const
-{
-    auto exec = this->get_executor();
-    exec->run(mapping::make_advanced_applyadd(
-        this, as<matrix::Dense<ValueType>>(alpha),
-        as<matrix::Dense<ValueType>>(b), as<matrix::Dense<ValueType>>(x)));
-}
-
-
-#define GKO_DECLARE_MAPPING(_vtype, _itype) class Mapping<_vtype, _itype>
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_MAPPING);
-
-
-}  // namespace multigrid
-}  // namespace gko
diff --git a/core/multigrid/mapping_kernels.hpp b/core/multigrid/mapping_kernels.hpp
deleted file mode 100644
index eb5506ad418..00000000000
--- a/core/multigrid/mapping_kernels.hpp
+++ /dev/null
@@ -1,126 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GKO_CORE_MULTIGRID_MAPPING_KERNELS_HPP_
-#define GKO_CORE_MULTIGRID_MAPPING_KERNELS_HPP_
-
-
-#include <ginkgo/core/multigrid/mapping.hpp>
-
-
-#include <memory>
-
-
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/diagonal.hpp>
-
-
-namespace gko {
-namespace kernels {
-
-
-#define GKO_DECLARE_MAPPING_APPLYADD_KERNEL(ValueType, IndexType)         \
-    void applyadd(std::shared_ptr<const DefaultExecutor> exec,            \
-                  const multigrid::Mapping<ValueType, IndexType> *source, \
-                  const matrix::Dense<ValueType> *b,                      \
-                  matrix::Dense<ValueType> *x)
-
-#define GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL(ValueType, IndexType) \
-    void advanced_applyadd(                                                \
-        std::shared_ptr<const DefaultExecutor> exec,                       \
-        const multigrid::Mapping<ValueType, IndexType> *source,            \
-        const matrix::Dense<ValueType> *alpha,                             \
-        const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *x)
-
-
-#define GKO_DECLARE_ALL_AS_TEMPLATES                           \
-    template <typename ValueType, typename IndexType>          \
-    GKO_DECLARE_MAPPING_APPLYADD_KERNEL(ValueType, IndexType); \
-    template <typename ValueType, typename IndexType>          \
-    GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL(ValueType, IndexType)
-
-
-namespace omp {
-namespace mapping {
-
-GKO_DECLARE_ALL_AS_TEMPLATES;
-
-}  // namespace mapping
-}  // namespace omp
-
-
-namespace cuda {
-namespace mapping {
-
-GKO_DECLARE_ALL_AS_TEMPLATES;
-
-}  // namespace mapping
-}  // namespace cuda
-
-
-namespace reference {
-namespace mapping {
-
-GKO_DECLARE_ALL_AS_TEMPLATES;
-
-}  // namespace mapping
-}  // namespace reference
-
-
-namespace hip {
-namespace mapping {
-
-GKO_DECLARE_ALL_AS_TEMPLATES;
-
-}  // namespace mapping
-}  // namespace hip
-
-
-namespace dpcpp {
-namespace mapping {
-
-GKO_DECLARE_ALL_AS_TEMPLATES;
-
-}  // namespace mapping
-}  // namespace dpcpp
-
-
-#undef GKO_DECLARE_ALL_AS_TEMPLATES
-
-
-}  // namespace kernels
-}  // namespace gko
-
-
-#endif  // GKO_CORE_MULTIGRID_MAPPING_KERNELS_HPP_
diff --git a/core/test/multigrid/amgx_pgm.cpp b/core/test/multigrid/amgx_pgm.cpp
index 1f04da999a8..6a897cee8b8 100644
--- a/core/test/multigrid/amgx_pgm.cpp
+++ b/core/test/multigrid/amgx_pgm.cpp
@@ -62,7 +62,7 @@ class AmgxPgmFactory : public ::testing::Test {
         : exec(gko::ReferenceExecutor::create()),
           amgxpgm_factory(RestrictProlong::build()
                               .with_max_iterations(2u)
-                              .with_max_unassigned_percentage(0.1)
+                              .with_max_unassigned_ratio(0.1)
                               .with_deterministic(true)
                               .on(exec))
 
@@ -87,7 +87,7 @@ TYPED_TEST(AmgxPgmFactory, DefaultSetting)
     auto factory = RestrictProlong::build().on(this->exec);
 
     ASSERT_EQ(factory->get_parameters().max_iterations, 15u);
-    ASSERT_EQ(factory->get_parameters().max_unassigned_percentage, 0.05);
+    ASSERT_EQ(factory->get_parameters().max_unassigned_ratio, 0.05);
     ASSERT_EQ(factory->get_parameters().deterministic, false);
 }
 
@@ -100,7 +100,7 @@ TYPED_TEST(AmgxPgmFactory, SetMaxIterations)
 
 TYPED_TEST(AmgxPgmFactory, SetMaxUnassignedPercentage)
 {
-    ASSERT_EQ(this->amgxpgm_factory->get_parameters().max_unassigned_percentage,
+    ASSERT_EQ(this->amgxpgm_factory->get_parameters().max_unassigned_ratio,
               0.1);
 }
 
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index f8b9d31951e..5e4505d8187 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -102,7 +102,6 @@ target_sources(ginkgo_cuda
     matrix/sellp_kernels.cu
     matrix/sparsity_csr_kernels.cu
     multigrid/amgx_pgm_kernels.cu
-    multigrid/mapping_kernels.cu
     preconditioner/isai_kernels.cu
     preconditioner/jacobi_advanced_apply_kernel.cu
     preconditioner/jacobi_generate_kernel.cu
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index 2adc3fb19fe..9153f120d93 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -62,26 +62,6 @@ namespace cuda {
 namespace amgx_pgm {
 
 
-template <typename ValueType, typename IndexType>
-void restrict_apply(std::shared_ptr<const CudaExecutor> exec,
-                    const Array<IndexType> &agg,
-                    const matrix::Dense<ValueType> *b,
-                    matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void prolong_applyadd(std::shared_ptr<const CudaExecutor> exec,
-                      const Array<IndexType> &agg,
-                      const matrix::Dense<ValueType> *b,
-                      matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
-
-
 template <typename IndexType>
 void match_edge(std::shared_ptr<const CudaExecutor> exec,
                 const Array<IndexType> &strongest_neighbor,
diff --git a/cuda/multigrid/mapping_kernels.cu b/cuda/multigrid/mapping_kernels.cu
deleted file mode 100644
index bd757fae834..00000000000
--- a/cuda/multigrid/mapping_kernels.cu
+++ /dev/null
@@ -1,88 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/multigrid/mapping_kernels.hpp"
-
-
-#include <memory>
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/diagonal.hpp>
-#include <ginkgo/core/multigrid/mapping.hpp>
-
-
-#include "core/base/allocator.hpp"
-#include "core/components/prefix_sum.hpp"
-#include "core/matrix/csr_builder.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace cuda {
-/**
- * @brief The MAPPING solver namespace.
- *
- * @ingroup mapping
- */
-namespace mapping {
-
-
-template <typename ValueType, typename IndexType>
-void applyadd(std::shared_ptr<const DefaultExecutor> exec,
-              const multigrid::Mapping<ValueType, IndexType> *source,
-              const matrix::Dense<ValueType> *b,
-              matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_MAPPING_APPLYADD_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void advanced_applyadd(std::shared_ptr<const DefaultExecutor> exec,
-                       const multigrid::Mapping<ValueType, IndexType> *source,
-                       const matrix::Dense<ValueType> *alpha,
-                       const matrix::Dense<ValueType> *b,
-                       matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL);
-
-
-}  // namespace mapping
-}  // namespace cuda
-}  // namespace kernels
-}  // namespace gko
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 01c0c621f6c..9e340ba33fe 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -31,7 +31,6 @@ target_sources(ginkgo_dpcpp
     matrix/sellp_kernels.dp.cpp
     matrix/sparsity_csr_kernels.dp.cpp
     multigrid/amgx_pgm_kernels.dp.cpp
-    multigrid/mapping_kernels.dp.cpp
     preconditioner/isai_kernels.dp.cpp
     preconditioner/jacobi_kernels.dp.cpp
     reorder/rcm_kernels.dp.cpp
diff --git a/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp b/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
index 98d42dcd7f3..933fc6fda7d 100644
--- a/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
+++ b/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
@@ -55,26 +55,6 @@ namespace dpcpp {
 namespace amgx_pgm {
 
 
-template <typename ValueType, typename IndexType>
-void restrict_apply(std::shared_ptr<const DpcppExecutor> exec,
-                    const Array<IndexType> &agg,
-                    const matrix::Dense<ValueType> *b,
-                    matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void prolong_applyadd(std::shared_ptr<const DpcppExecutor> exec,
-                      const Array<IndexType> &agg,
-                      const matrix::Dense<ValueType> *b,
-                      matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
-
-
 template <typename IndexType>
 void match_edge(std::shared_ptr<const DpcppExecutor> exec,
                 const Array<IndexType> &strongest_neighbor,
diff --git a/dpcpp/multigrid/mapping_kernels.dp.cpp b/dpcpp/multigrid/mapping_kernels.dp.cpp
deleted file mode 100644
index 0e172460f86..00000000000
--- a/dpcpp/multigrid/mapping_kernels.dp.cpp
+++ /dev/null
@@ -1,88 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/multigrid/mapping_kernels.hpp"
-
-
-#include <memory>
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/diagonal.hpp>
-#include <ginkgo/core/multigrid/mapping.hpp>
-
-
-#include "core/base/allocator.hpp"
-#include "core/components/prefix_sum.hpp"
-#include "core/matrix/csr_builder.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace dpcpp {
-/**
- * @brief The MAPPING solver namespace.
- *
- * @ingroup mapping
- */
-namespace mapping {
-
-
-template <typename ValueType, typename IndexType>
-void applyadd(std::shared_ptr<const DefaultExecutor> exec,
-              const multigrid::Mapping<ValueType, IndexType> *source,
-              const matrix::Dense<ValueType> *b,
-              matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_MAPPING_APPLYADD_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void advanced_applyadd(std::shared_ptr<const DefaultExecutor> exec,
-                       const multigrid::Mapping<ValueType, IndexType> *source,
-                       const matrix::Dense<ValueType> *alpha,
-                       const matrix::Dense<ValueType> *b,
-                       matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL);
-
-
-}  // namespace mapping
-}  // namespace dpcpp
-}  // namespace kernels
-}  // namespace gko
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 2c388326c54..4e34da9466f 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -193,7 +193,6 @@ set(GINKGO_HIP_SOURCES
     matrix/sellp_kernels.hip.cpp
     matrix/sparsity_csr_kernels.hip.cpp
     multigrid/amgx_pgm_kernels.hip.cpp
-    multigrid/mapping_kernels.hip.cpp
     preconditioner/isai_kernels.hip.cpp
     preconditioner/jacobi_advanced_apply_kernel.hip.cpp
     preconditioner/jacobi_generate_kernel.hip.cpp
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index 80eb5d768ec..d401bb72b72 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -62,26 +62,6 @@ namespace hip {
 namespace amgx_pgm {
 
 
-template <typename ValueType, typename IndexType>
-void restrict_apply(std::shared_ptr<const HipExecutor> exec,
-                    const Array<IndexType> &agg,
-                    const matrix::Dense<ValueType> *b,
-                    matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void prolong_applyadd(std::shared_ptr<const HipExecutor> exec,
-                      const Array<IndexType> &agg,
-                      const matrix::Dense<ValueType> *b,
-                      matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
-
-
 template <typename IndexType>
 void match_edge(std::shared_ptr<const HipExecutor> exec,
                 const Array<IndexType> &strongest_neighbor,
diff --git a/hip/multigrid/mapping_kernels.hip.cpp b/hip/multigrid/mapping_kernels.hip.cpp
deleted file mode 100644
index 849d0bbafd6..00000000000
--- a/hip/multigrid/mapping_kernels.hip.cpp
+++ /dev/null
@@ -1,88 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/multigrid/mapping_kernels.hpp"
-
-
-#include <memory>
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/diagonal.hpp>
-#include <ginkgo/core/multigrid/mapping.hpp>
-
-
-#include "core/base/allocator.hpp"
-#include "core/components/prefix_sum.hpp"
-#include "core/matrix/csr_builder.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace hip {
-/**
- * @brief The MAPPING solver namespace.
- *
- * @ingroup mapping
- */
-namespace mapping {
-
-
-template <typename ValueType, typename IndexType>
-void applyadd(std::shared_ptr<const DefaultExecutor> exec,
-              const multigrid::Mapping<ValueType, IndexType> *source,
-              const matrix::Dense<ValueType> *b,
-              matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_MAPPING_APPLYADD_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void advanced_applyadd(std::shared_ptr<const DefaultExecutor> exec,
-                       const multigrid::Mapping<ValueType, IndexType> *source,
-                       const matrix::Dense<ValueType> *alpha,
-                       const matrix::Dense<ValueType> *b,
-                       matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL);
-
-
-}  // namespace mapping
-}  // namespace hip
-}  // namespace kernels
-}  // namespace gko
diff --git a/include/ginkgo/core/base/composition.hpp b/include/ginkgo/core/base/composition.hpp
index 6ba86355b04..f5ffc05a4a5 100644
--- a/include/ginkgo/core/base/composition.hpp
+++ b/include/ginkgo/core/base/composition.hpp
@@ -167,16 +167,36 @@ class Composition : public EnableLinOp<Composition<ValueType>>,
 };
 
 
+/**
+ * The UseComposition class can be used to store the composition information in
+ * LinOp.
+ *
+ * @tparam ValueType  precision of input and result vectors
+ */
 template <typename ValueType = default_precision>
 class UseComposition {
 public:
     using value_type = ValueType;
+    /**
+     * Returns the composition opertor.
+     *
+     * @return composition
+     */
     std::shared_ptr<Composition<ValueType>> get_composition() const
     {
         return composition_;
     }
 
-    std::shared_ptr<const LinOp> get_composition_ithop(int index) const
+    /**
+     * Returns the operator at index-th poistion of composition
+     *
+     * @return index-th operator
+     *
+     * @note when this composition is not set, this function always returns
+     *       nullptr. However, when this composition is set, it will throw
+     *       exception when exceeding index.
+     */
+    std::shared_ptr<const LinOp> get_operator_at(size_type index) const
     {
         if (composition_ == nullptr) {
             return nullptr;
@@ -186,6 +206,9 @@ class UseComposition {
     }
 
 protected:
+    /**
+     * Sets the composition with a list of operators
+     */
     template <typename... LinOp>
     void set_composition(LinOp &&... linop)
     {
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index 3face49cc6d..99d2dc6ddbd 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -43,8 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/multigrid/mapping.hpp>
-#include <ginkgo/core/multigrid/multigrid_base.hpp>
+#include <ginkgo/core/multigrid/multigrid_level.hpp>
 
 namespace gko {
 namespace multigrid {
@@ -130,7 +129,7 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
          * Reference Manual (October 2017, API Version 2,
          * https://github.com/NVIDIA/AMGX/blob/main/doc/AMGX_Reference.pdf).
          */
-        double GKO_FACTORY_PARAMETER(max_unassigned_percentage, 0.05);
+        double GKO_FACTORY_PARAMETER(max_unassigned_ratio, 0.05);
 
         /**
          * Use the deterministic assign_to_exist_agg method or not.
@@ -145,12 +144,6 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
     GKO_ENABLE_BUILD_METHOD(Factory);
 
 protected:
-    void restrict_apply_impl(const LinOp *b, LinOp *x) const;
-
-    void prolong_applyadd_impl(const LinOp *b, LinOp *x) const;
-
-    void prolong_apply_impl(const LinOp *b, LinOp *x) const;
-
     void apply_impl(const LinOp *b, LinOp *x) const override
     {
         this->get_composition()->apply(b, x);
@@ -163,7 +156,7 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
     }
 
     explicit AmgxPgm(std::shared_ptr<const Executor> exec)
-        : EnableLinOp<AmgxPgm>(std::move(exec)), MultigridLevel<ValueType>()
+        : EnableLinOp<AmgxPgm>(std::move(exec))
     {}
 
     explicit AmgxPgm(const Factory *factory,
@@ -175,8 +168,8 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
           system_matrix_{system_matrix},
           agg_(factory->get_executor(), system_matrix_->get_size()[0])
     {
-        GKO_ASSERT(parameters_.max_unassigned_percentage <= 1.0);
-        GKO_ASSERT(parameters_.max_unassigned_percentage >= 0.0);
+        GKO_ASSERT(parameters_.max_unassigned_ratio <= 1.0);
+        GKO_ASSERT(parameters_.max_unassigned_ratio >= 0.0);
         if (system_matrix_->get_size()[0] != 0) {
             // generate on the existed matrix
             this->generate();
diff --git a/include/ginkgo/core/multigrid/mapping.hpp b/include/ginkgo/core/multigrid/mapping.hpp
deleted file mode 100644
index de75f6ac6cf..00000000000
--- a/include/ginkgo/core/multigrid/mapping.hpp
+++ /dev/null
@@ -1,158 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GKO_PUBLIC_CORE_MULTIGRID_MAPPING_HPP_
-#define GKO_PUBLIC_CORE_MULTIGRID_MAPPING_HPP_
-
-
-#include <vector>
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/lin_op.hpp>
-#include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/multigrid/multigrid_base.hpp>
-
-
-namespace gko {
-namespace multigrid {
-
-
-template <typename ValueType, typename IndexType>
-class Mapping : public EnableLinOp<Mapping<ValueType, IndexType>>,
-                public EnableCreateMethod<Mapping<ValueType, IndexType>>,
-                public ApplyAddable {
-    friend class EnableCreateMethod<Mapping>;
-    friend class EnablePolymorphicObject<Mapping, LinOp>;
-
-public:
-    using value_type = ValueType;
-    using index_type = IndexType;
-
-    /**
-     * Returns the mapping indexes.
-     *
-     * @return the mapping indexes.
-     */
-    index_type *get_mapping_index() noexcept { return mapping_.get_data(); }
-
-    /**
-     * @copydoc Mapping::get_mapping_index()
-     *
-     * @note This is the constant version of the function, which can be
-     *       significantly more memory efficient than the non-constant version,
-     *       so always prefer this version.
-     */
-    const index_type *get_const_mapping_index() const noexcept
-    {
-        return mapping_.get_const_data();
-    }
-
-    /**
-     * Returns true as it is restrict operation.
-     *
-     * @return true as it is restrict operation.
-     */
-    bool is_restrict() const noexcept { return is_restrict_; }
-
-    /**
-     * Returns the number of elements explicitly stored in the mapping index.
-     *
-     * @return the number of elements explicitly stored in the mapping index
-     */
-    size_type get_num_stored_elements() const noexcept
-    {
-        return mapping_.get_num_elems();
-    }
-
-protected:
-    /**
-     * Creates an uninitialized Mapping arrays on the specified executor..
-     *
-     * @param exec  Executor associated to the LinOp
-     */
-    Mapping(std::shared_ptr<const Executor> exec)
-        : EnableLinOp<Mapping>(exec), mapping_{exec}, is_restrict_(true)
-    {}
-
-    template <typename IndicesArray>
-    Mapping(std::shared_ptr<const Executor> exec, const dim<2> &size,
-            IndicesArray &&mapping_indices, const bool &is_restrict = true)
-        : EnableLinOp<Mapping>(exec, size),
-          mapping_{exec, std::forward<IndicesArray>(mapping_indices)},
-          is_restrict_(is_restrict)
-    {
-        if (is_restrict_) {
-            GKO_ASSERT_EQ(size[1], mapping_.get_num_elems());
-        }
-        if (!is_restrict_) {
-            GKO_ASSERT_EQ(size[0], mapping_.get_num_elems());
-        }
-    }
-
-    void apply_impl(const LinOp *b, LinOp *x) const override
-    {
-        auto dense_x = as<matrix::Dense<ValueType>>(x);
-
-        // TODO: write dense fill kernel
-        auto zero = gko::initialize<matrix::Dense<ValueType>>(
-            {gko::zero<ValueType>()}, this->get_executor());
-        dense_x->scale(lend(zero));
-        this->apply2(b, dense_x);
-    }
-
-    void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
-                    LinOp *x) const override
-    {
-        auto dense_x = as<matrix::Dense<ValueType>>(x);
-        dense_x->scale(beta);
-        this->apply2(alpha, b, dense_x);
-    }
-
-    void apply2_impl(const LinOp *b, LinOp *x) const override;
-
-    void apply2_impl(const LinOp *alpha, const LinOp *b,
-                     LinOp *x) const override;
-
-
-private:
-    Array<index_type> mapping_;
-    bool is_restrict_;
-};
-
-
-}  // namespace multigrid
-}  // namespace gko
-
-#endif  // GKO_PUBLIC_CORE_MULTIGRID_MAPPING_HPP_
diff --git a/include/ginkgo/core/multigrid/multigrid_base.hpp b/include/ginkgo/core/multigrid/multigrid_base.hpp
deleted file mode 100644
index 70d292a1e47..00000000000
--- a/include/ginkgo/core/multigrid/multigrid_base.hpp
+++ /dev/null
@@ -1,381 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GKO_PUBLIC_CORE_MULTIGRID_MULTIGRID_BASE_HPP_
-#define GKO_PUBLIC_CORE_MULTIGRID_MULTIGRID_BASE_HPP_
-
-
-#include <functional>
-#include <memory>
-
-
-#include <ginkgo/core/base/abstract_factory.hpp>
-#include <ginkgo/core/base/composition.hpp>
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/lin_op.hpp>
-#include <ginkgo/core/base/utils.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-
-
-namespace gko {
-namespace multigrid {
-
-
-/**
- * This interface allows to implement the restrict operation of
- * a multigrid method.
- *
- * Implementers of this interface only need to overload the
- * RestrictOp::restrict_apply_impl() method.
- *
- * @ingroup Multigrid
- */
-class RestrictOp {
-public:
-    /**
-     * Applies a restrict operator to a vector (or a sequence of vectors).
-     *
-     * Performs the operation x = op(b), where op is this restrict operator.
-     *
-     * @param b  the input vector(s) on which the operator is applied
-     * @param x  the output vector(s) where the result is stored
-     */
-    void restrict_apply(const LinOp *b, LinOp *x) const
-    {
-        this->validate_parameters(b, x);
-        this->restrict_apply_impl(make_temporary_clone(exec_, b).get(),
-                                  make_temporary_clone(exec_, x).get());
-    }
-
-protected:
-    /**
-     * Sets the size of the restrict operator.
-     *
-     * @param value  the new size of the restrict operator
-     */
-    void set_restrict_size(const dim<2> &value) noexcept { size_ = value; }
-
-    virtual void restrict_apply_impl(const LinOp *b, LinOp *x) const = 0;
-
-    /**
-     * Creates a RestrictOp with settings
-     *
-     * @param exec  Executor associated to the RestrictOp
-     * @param size  the size of RestrictOp.
-     *
-     * @note the multigrid is generated in the generation not the construction,
-     *       so calling `set_restrict_size` is needed to set the restrict
-     *       size after generation.
-     */
-    explicit RestrictOp(std::shared_ptr<const Executor> exec,
-                        const gko::dim<2> &size = gko::dim<2>{})
-        : exec_(exec), size_(size)
-    {}
-
-private:
-    /**
-     * Throws a DimensionMismatch exception if the parameters to
-     * `restrict_apply` are of the wrong size.
-     *
-     * @param b  vector(s) on which the operator is applied
-     * @param x  output vector(s)
-     */
-    void validate_parameters(const LinOp *b, LinOp *x) const
-    {
-        GKO_ASSERT_CONFORMANT(size_, b);
-        GKO_ASSERT_EQUAL_ROWS(size_, x);
-        GKO_ASSERT_EQUAL_COLS(b, x);
-    }
-
-    std::shared_ptr<const Executor> exec_;
-    gko::dim<2> size_;
-};
-
-
-/**
- * This interface allows to implement the prolongation operation
- * of a multigrid method.
- *
- * Implementers of this interface need to overload both the
- * ProlongOp::prolong_apply_impl() and
- * ProlongOp::prolong_applyadd_impl() methods.
- *
- * @ingroup Multigrid
- */
-class ProlongOp {
-public:
-    /**
-     * Applies a prolong operator to a vector (or a sequence of vectors).
-     *
-     * Performs the operation x = op(b), where op is this prolong operator.
-     *
-     * @param b  the input vector(s) on which the operator is applied
-     * @param x  the output vector(s) where the result is stored
-     */
-    void prolong_apply(const LinOp *b, LinOp *x) const
-    {
-        this->validate_parameters(b, x);
-        this->prolong_apply_impl(make_temporary_clone(exec_, b).get(),
-                                 make_temporary_clone(exec_, x).get());
-    }
-
-    /**
-     * Applies a prolong operator onto a vector (or a sequence of vectors).
-     *
-     * Performs the operation x += op(b), where op is this prolong operator.
-     *
-     * @param b  the input vector(s) on which the operator is applied
-     * @param x  the output vector(s) where the result is stored
-     */
-    void prolong_applyadd(const LinOp *b, LinOp *x) const
-    {
-        this->validate_parameters(b, x);
-        this->prolong_applyadd_impl(make_temporary_clone(exec_, b).get(),
-                                    make_temporary_clone(exec_, x).get());
-    }
-
-protected:
-    /**
-     * Sets the size of the prolong operator.
-     *
-     * @param value  the new size of the prolong operator
-     */
-    void set_prolong_size(const dim<2> &value) noexcept { size_ = value; }
-
-    virtual void prolong_apply_impl(const LinOp *b, LinOp *x) const = 0;
-
-    virtual void prolong_applyadd_impl(const LinOp *b, LinOp *x) const = 0;
-
-    /**
-     * Creates a ProlongOp with settings
-     *
-     * @param exec  Executor associated to the ProlongOp
-     * @param native_applyadd  the prolong applyadd is natively implemented in
-     *                         one kernel or not. This argument does not has
-     *                         effect yet.
-     * @param size  the size of ProlongOp.
-     *
-     * @note the multigrid is generated in the generation not the construction,
-     *       so needs to call `set_prolong_size` to set prolong size after
-     *       generation.
-     */
-    explicit ProlongOp(std::shared_ptr<const Executor> exec,
-                       bool native_applyadd = false,
-                       const gko::dim<2> &size = gko::dim<2>{})
-        : exec_(exec), native_applyadd_(native_applyadd), size_(size)
-    {}
-
-private:
-    /**
-     * Throws a DimensionMismatch exception if the parameters to
-     * `prolong_applyadd` are of the wrong size.
-     *
-     * @param b  vector(s) on which the operator is applied
-     * @param x  output vector(s)
-     */
-    void validate_parameters(const LinOp *b, LinOp *x) const
-    {
-        GKO_ASSERT_CONFORMANT(size_, b);
-        GKO_ASSERT_EQUAL_ROWS(size_, x);
-        GKO_ASSERT_EQUAL_COLS(b, x);
-    }
-
-    std::shared_ptr<const Executor> exec_;
-    gko::dim<2> size_;
-    bool native_applyadd_;
-};
-
-
-/**
- * This class provides functionality for managing the coarse and fine operator
- * for a multigrid method.
- *
- * @ingroup Multigrid
- */
-class CoarseOp {
-public:
-    /**
-     * Gets the fine operator (matrix)
-     *
-     * @return the fine operator (matrix)
-     */
-    std::shared_ptr<const LinOp> get_fine_matrix() const { return fine_; }
-
-    /**
-     * Gets the coarse operator (matrix)
-     *
-     * @return the coarse operator (matrix)
-     */
-    std::shared_ptr<const LinOp> get_coarse_matrix() const { return coarse_; }
-
-protected:
-    /**
-     * Sets the fine and coarse information.
-     *
-     * @param fine  the matrix on fine level
-     * @param coarse  the matrix on coarse level
-     */
-    void set_fine_coarse(std::shared_ptr<const LinOp> fine,
-                         std::shared_ptr<const LinOp> coarse)
-    {
-        fine_ = fine;
-        coarse_ = coarse;
-    }
-
-private:
-    std::shared_ptr<const LinOp> coarse_;
-    std::shared_ptr<const LinOp> fine_;
-};
-
-
-/**
- * A class implementing this interface can contain restrict, prolong, coarse
- * generation operation. MultigridLevel provides multigrid_level_default_apply
- * to represent op(b) = prolong(coarse(restrict(b))). Those classes implementing
- * all MultigridLevel operation must inherit this class.
- *
- * Implementers of this interface need to overloadthe
- * MultigridLevel::prolong_apply_impl(),
- * MultigridLevel::prolong_applyadd_impl(), and
- * MultigridLevel::restrict_apply_impl() methods.
- *
- * ```c++
- * class MyMultigridLevel : public EnableLinOp<MyMultigridLevel>,
- *                          public MultigridLevel {
- *     GKO_CREATE_FACTORY_PARAMETERS(MyMultigridLevel, Factory) {
- *         // some parameter settings as LinOpFactory
- *     };
- *     GKO_ENABLE_LIN_OP_FACTORY(MyMultigridLevel, parameters, Factory);
- *     GKO_ENABLE_BUILD_METHOD(Factory);
- *
- *     // Need to implement the following override function:
- *     // prolong_apply_impl, prolong_applyadd_impl, restict_apply_impl
- *
- *     // could use the default apply implementation of MultigridLevel
- *     void apply_impl(const LinOp* b, LinOp* x) const override {
- *         this->template multigrid_level_default_apply<ValueType>(b, x);
- *     }
- *
- *     // could use the default apply implementation of MultigridLevel
- *     void apply_impl(const LinOp *alpha, const LinOp *b,
- *                     const LinOp *beta, LinOp *x) const override {
- *         this->template multigrid_level_default_apply<ValueType>(
- *             alpha, b, beta, x);
- *     }
- *
- *     // constructor needed by EnableLinOp
- *     explicit MyMultigridLevel(std::shared_ptr<const Executor> exec) {
- *         : EnableLinOp<MyMultigridLevel>(exec) {}
- *
- *     // constructor needed by the factory
- *     explicit MyMultigridLevel(const Factory *factory,
- *                      std::shared_ptr<const LinOp> matrix)
- *         : EnableLinOp<MyMultigridLevel>(factory->get_executor()),
- *                                         matrix->get_size()),
- *           // store factory's parameters locally
- *           my_parameters_{factory->get_parameters()},
- *     {
- *          // do something with parameter to generate fine/coarse matrices
- *          // set the information such that the checks work properly
- *          this->set_fine_coarse(fine, corase);
- *     }
- * ```
- *
- * @ingroup Multigrid
- */
-template <typename ValueType>
-class MultigridLevel : public UseComposition<ValueType> {
-public:
-    using value_type = ValueType;
-
-    std::shared_ptr<const LinOp> get_fine_op() const { return fine_op_; }
-
-    std::shared_ptr<const LinOp> get_restrict_op() const
-    {
-        return this->get_composition_ithop(2);
-    }
-
-    std::shared_ptr<const LinOp> get_coarse_op() const
-    {
-        return this->get_composition_ithop(1);
-    }
-
-    std::shared_ptr<const LinOp> get_prolong_op() const
-    {
-        return this->get_composition_ithop(0);
-    }
-
-protected:
-    /**
-     * Sets the fine and coarse information.
-     *
-     * @param fine  the matrix on fine level
-     * @param coarse  the matrix on coarse level
-     */
-    void set_multigrid_level(std::shared_ptr<const LinOp> prolong_op,
-                             std::shared_ptr<const LinOp> coarse_op,
-                             std::shared_ptr<const LinOp> restrict_op)
-    {
-        gko::dim<2> mg_size{prolong_op->get_size()[0],
-                            restrict_op->get_size()[1]};
-        GKO_ASSERT_EQUAL_DIMENSIONS(fine_op_->get_size(), mg_size);
-        // check mg_size is the same as fine_size
-        this->set_composition(prolong_op, coarse_op, restrict_op);
-    }
-
-    explicit MultigridLevel() {}
-    /**
-     * Creates a MultigridLevel with settings
-     *
-     * @param exec  Executor associated to the multigrid_level
-     * @param native_applyadd  the prolong applyadd is natively implemented in
-     *                         one kernel or not. This argument does not has
-     *                         effect yet.
-     *
-     * @note the multigrid is generated in the generation not the construction,
-     *       so needs to call `set_multigrid_level` to set corresponding
-     *       information after generation.
-     */
-    explicit MultigridLevel(std::shared_ptr<const LinOp> fine_op)
-        : fine_op_(fine_op)
-    {}
-
-private:
-    std::shared_ptr<const LinOp> fine_op_;
-};
-
-
-}  // namespace multigrid
-}  // namespace gko
-
-
-#endif  // GKO_PUBLIC_CORE_MULTIGRID_MULTIGRID_BASE_HPP_
diff --git a/include/ginkgo/core/multigrid/multigrid_level.hpp b/include/ginkgo/core/multigrid/multigrid_level.hpp
new file mode 100644
index 00000000000..c1249c2dad5
--- /dev/null
+++ b/include/ginkgo/core/multigrid/multigrid_level.hpp
@@ -0,0 +1,148 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_PUBLIC_CORE_MULTIGRID_MULTIGRID_LEVEL_HPP_
+#define GKO_PUBLIC_CORE_MULTIGRID_MULTIGRID_LEVEL_HPP_
+
+
+#include <functional>
+#include <memory>
+
+
+#include <ginkgo/core/base/abstract_factory.hpp>
+#include <ginkgo/core/base/composition.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/utils.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+namespace gko {
+namespace multigrid {
+
+
+/**
+ * This class represents two levels in a multigrid hierarchy.
+ *
+ * A class implementing this interface should use the
+ * this->get_compositions()->apply(...) as its own apply, which represents
+ * op(b) = prolong(coarse(restrict(b))). All MultigridLevel generator should
+ * inherit from this class.
+ *
+ * @ingroup Multigrid
+ */
+template <typename ValueType>
+class MultigridLevel : public UseComposition<ValueType> {
+public:
+    using value_type = ValueType;
+
+    /**
+     * Returns the operator on fine level.
+     *
+     * @return  the operator on fine level.
+     */
+    std::shared_ptr<const LinOp> get_fine_op() const { return fine_op_; }
+
+    /**
+     * Returns the restrict operator.
+     *
+     * @return  the restrict operator.
+     */
+    std::shared_ptr<const LinOp> get_restrict_op() const
+    {
+        return this->get_operator_at(2);
+    }
+
+    /**
+     * Returns the operator on coarse level.
+     *
+     * @return  the operator on coarse level.
+     */
+    std::shared_ptr<const LinOp> get_coarse_op() const
+    {
+        return this->get_operator_at(1);
+    }
+
+    /**
+     * Returns the prolong operator.
+     *
+     * @return  the prolong operator.
+     */
+    std::shared_ptr<const LinOp> get_prolong_op() const
+    {
+        return this->get_operator_at(0);
+    }
+
+protected:
+    /**
+     * Sets the multigrid level information.
+     *
+     * @param prolong_op  the prolong operator
+     * @param coarse_op  the coarse operator
+     * @param restrict_op  the restrict operator
+     */
+    void set_multigrid_level(std::shared_ptr<const LinOp> prolong_op,
+                             std::shared_ptr<const LinOp> coarse_op,
+                             std::shared_ptr<const LinOp> restrict_op)
+    {
+        gko::dim<2> mg_size{prolong_op->get_size()[0],
+                            restrict_op->get_size()[1]};
+        GKO_ASSERT_EQUAL_DIMENSIONS(fine_op_->get_size(), mg_size);
+        // check mg_size is the same as fine_size
+        this->set_composition(prolong_op, coarse_op, restrict_op);
+    }
+
+    explicit MultigridLevel() {}
+
+    /**
+     * Creates a MultigridLevel with the given fine operator
+     *
+     * @param fine_op  The fine operotor associated to the multigrid_level
+     *
+     * @note the multigrid is generated in the generation not the construction,
+     *       so needs to call `set_multigrid_level` to set corresponding
+     *       information after generation.
+     */
+    explicit MultigridLevel(std::shared_ptr<const LinOp> fine_op)
+        : fine_op_(fine_op)
+    {}
+
+private:
+    std::shared_ptr<const LinOp> fine_op_;
+};
+
+
+}  // namespace multigrid
+}  // namespace gko
+
+
+#endif  // GKO_PUBLIC_CORE_MULTIGRID_MULTIGRID_LEVEL_HPP_
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index aa6cae5047d..3ae5bcfdd96 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -88,8 +88,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 #include <ginkgo/core/multigrid/amgx_pgm.hpp>
-#include <ginkgo/core/multigrid/mapping.hpp>
-#include <ginkgo/core/multigrid/multigrid_base.hpp>
+#include <ginkgo/core/multigrid/multigrid_level.hpp>
 
 #include <ginkgo/core/preconditioner/ic.hpp>
 #include <ginkgo/core/preconditioner/ilu.hpp>
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index 03b0649e04e..4fd677d5b5c 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -24,7 +24,6 @@ target_sources(ginkgo_omp
     matrix/sellp_kernels.cpp
     matrix/sparsity_csr_kernels.cpp
     multigrid/amgx_pgm_kernels.cpp
-    multigrid/mapping_kernels.cpp
     preconditioner/isai_kernels.cpp
     preconditioner/jacobi_kernels.cpp
     reorder/rcm_kernels.cpp
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index 82a40e33b22..d56b3ff9dab 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -59,26 +59,6 @@ namespace omp {
 namespace amgx_pgm {
 
 
-template <typename ValueType, typename IndexType>
-void restrict_apply(std::shared_ptr<const OmpExecutor> exec,
-                    const Array<IndexType> &agg,
-                    const matrix::Dense<ValueType> *b,
-                    matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void prolong_applyadd(std::shared_ptr<const OmpExecutor> exec,
-                      const Array<IndexType> &agg,
-                      const matrix::Dense<ValueType> *b,
-                      matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
-
-
 template <typename IndexType>
 void match_edge(std::shared_ptr<const OmpExecutor> exec,
                 const Array<IndexType> &strongest_neighbor,
diff --git a/omp/multigrid/mapping_kernels.cpp b/omp/multigrid/mapping_kernels.cpp
deleted file mode 100644
index e614c13c29d..00000000000
--- a/omp/multigrid/mapping_kernels.cpp
+++ /dev/null
@@ -1,88 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/multigrid/mapping_kernels.hpp"
-
-
-#include <memory>
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/diagonal.hpp>
-#include <ginkgo/core/multigrid/mapping.hpp>
-
-
-#include "core/base/allocator.hpp"
-#include "core/components/prefix_sum.hpp"
-#include "core/matrix/csr_builder.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace omp {
-/**
- * @brief The MAPPING solver namespace.
- *
- * @ingroup mapping
- */
-namespace mapping {
-
-
-template <typename ValueType, typename IndexType>
-void applyadd(std::shared_ptr<const DefaultExecutor> exec,
-              const multigrid::Mapping<ValueType, IndexType> *source,
-              const matrix::Dense<ValueType> *b,
-              matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_MAPPING_APPLYADD_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void advanced_applyadd(std::shared_ptr<const DefaultExecutor> exec,
-                       const multigrid::Mapping<ValueType, IndexType> *source,
-                       const matrix::Dense<ValueType> *alpha,
-                       const matrix::Dense<ValueType> *b,
-                       matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL);
-
-
-}  // namespace mapping
-}  // namespace omp
-}  // namespace kernels
-}  // namespace gko
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index a9e598a5cbc..b17b75ca8ba 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -22,7 +22,6 @@ target_sources(ginkgo_reference
     matrix/sellp_kernels.cpp
     matrix/sparsity_csr_kernels.cpp
     multigrid/amgx_pgm_kernels.cpp
-    multigrid/mapping_kernels.cpp
     preconditioner/isai_kernels.cpp
     preconditioner/jacobi_kernels.cpp
     reorder/rcm_kernels.cpp
diff --git a/reference/components/fill_array.cpp b/reference/components/fill_array.cpp
index f2fdba585ec..b4c905f9e5d 100644
--- a/reference/components/fill_array.cpp
+++ b/reference/components/fill_array.cpp
@@ -30,10 +30,10 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <numeric>
+#include "core/components/fill_array.hpp"
 
 
-#include "core/components/fill_array.hpp"
+#include <numeric>
 
 
 namespace gko {
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index 159eeba9402..1ae7ee860b0 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -61,53 +61,6 @@ namespace reference {
 namespace amgx_pgm {
 
 
-template <typename ValueType, typename IndexType>
-void restrict_apply(std::shared_ptr<const ReferenceExecutor> exec,
-                    const Array<IndexType> &agg,
-                    const matrix::Dense<ValueType> *b,
-                    matrix::Dense<ValueType> *x)
-{
-    auto x_vals = x->get_values();
-    const auto x_stride = x->get_stride();
-    auto x_dim = x->get_size();
-    for (size_type i = 0; i < x_dim[0]; i++) {
-        for (size_type j = 0; j < x_dim[1]; j++) {
-            x->at(i, j) = zero<ValueType>();
-        }
-    }
-    for (size_type i = 0; i < agg.get_num_elems(); i++) {
-        const auto x_row = agg.get_const_data()[i];
-        for (size_type j = 0; j < x_dim[1]; j++) {
-            x->at(x_row, j) += b->at(i, j);
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_RESTRICT_APPLY_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void prolong_applyadd(std::shared_ptr<const ReferenceExecutor> exec,
-                      const Array<IndexType> &agg,
-                      const matrix::Dense<ValueType> *b,
-                      matrix::Dense<ValueType> *x)
-{
-    auto x_vals = x->get_values();
-    const auto x_stride = x->get_stride();
-    auto x_dim = x->get_size();
-    for (size_type i = 0; i < agg.get_num_elems(); i++) {
-        const auto b_row = agg.get_const_data()[i];
-        for (size_type j = 0; j < x_dim[1]; j++) {
-            x->at(i, j) += b->at(b_row, j);
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_AMGX_PGM_PROLONGATE_APPLY_KERNEL);
-
-
 template <typename IndexType>
 void match_edge(std::shared_ptr<const ReferenceExecutor> exec,
                 const Array<IndexType> &strongest_neighbor,
diff --git a/reference/multigrid/mapping_kernels.cpp b/reference/multigrid/mapping_kernels.cpp
deleted file mode 100644
index ed43add542d..00000000000
--- a/reference/multigrid/mapping_kernels.cpp
+++ /dev/null
@@ -1,118 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/multigrid/mapping_kernels.hpp"
-
-
-#include <memory>
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/diagonal.hpp>
-#include <ginkgo/core/multigrid/mapping.hpp>
-
-
-#include "core/base/allocator.hpp"
-#include "core/components/prefix_sum.hpp"
-#include "core/matrix/csr_builder.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace reference {
-/**
- * @brief The MAPPING solver namespace.
- *
- * @ingroup mapping
- */
-namespace mapping {
-
-
-template <typename ValueType, typename IndexType>
-void applyadd(std::shared_ptr<const DefaultExecutor> exec,
-              const multigrid::Mapping<ValueType, IndexType> *source,
-              const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *x)
-{
-    auto x_vals = x->get_values();
-    const auto x_stride = x->get_stride();
-    auto x_dim = x->get_size();
-    auto mapindex = source->get_const_mapping_index();
-    auto num = source->get_num_stored_elements();
-    auto is_restrict = source->is_restrict();
-    for (size_type i = 0; i < num; i++) {
-        const auto x_row = is_restrict ? mapindex[i] : i;
-        const auto b_row = is_restrict ? i : mapindex[i];
-        for (size_type j = 0; j < x_dim[1]; j++) {
-            x->at(x_row, j) += b->at(b_row, j);
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_MAPPING_APPLYADD_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void advanced_applyadd(std::shared_ptr<const DefaultExecutor> exec,
-                       const multigrid::Mapping<ValueType, IndexType> *source,
-                       const matrix::Dense<ValueType> *alpha,
-                       const matrix::Dense<ValueType> *b,
-                       matrix::Dense<ValueType> *x)
-{
-    auto x_vals = x->get_values();
-    const auto x_stride = x->get_stride();
-    auto x_dim = x->get_size();
-    auto mapindex = source->get_const_mapping_index();
-    auto num = source->get_num_stored_elements();
-    auto is_restrict = source->is_restrict();
-    auto alpha_val = alpha->at(0, 0);
-    for (size_type i = 0; i < num; i++) {
-        const auto x_row = is_restrict ? mapindex[i] : i;
-        const auto b_row = is_restrict ? i : mapindex[i];
-        for (size_type j = 0; j < x_dim[1]; j++) {
-            x->at(x_row, j) += alpha_val * b->at(b_row, j);
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_MAPPING_ADVANCED_APPLYADD_KERNEL);
-
-
-}  // namespace mapping
-}  // namespace reference
-}  // namespace kernels
-}  // namespace gko
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index 03bf86099da..d3cd8e19994 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -75,7 +75,7 @@ class AmgxPgm : public ::testing::Test {
         : exec(gko::ReferenceExecutor::create()),
           amgxpgm_factory(RestrictProlong::build()
                               .with_max_iterations(2u)
-                              .with_max_unassigned_percentage(0.1)
+                              .with_max_unassigned_ratio(0.1)
                               .on(exec)),
           fine_b(gko::initialize<Vec>(
               {I<VT>({2.0, -1.0}), I<VT>({-1.0, 2.0}), I<VT>({0.0, -1.0}),
@@ -295,32 +295,6 @@ TYPED_TEST(AmgxPgm, CanBeCleared)
 }
 
 
-TYPED_TEST(AmgxPgm, RestrictApply)
-{
-    // fine->coarse
-    using Vec = typename TestFixture::Vec;
-    using value_type = typename TestFixture::value_type;
-    auto x = Vec::create_with_config_of(gko::lend(this->coarse_b));
-
-    gko::kernels::reference::amgx_pgm::restrict_apply(
-        this->exec, this->agg, this->fine_b.get(), x.get());
-
-    GKO_ASSERT_MTX_NEAR(x, this->restrict_ans, r<value_type>::value);
-}
-
-
-TYPED_TEST(AmgxPgm, ProlongApplyadd)
-{
-    using value_type = typename TestFixture::value_type;
-    auto x = gko::clone(this->fine_x);
-
-    gko::kernels::reference::amgx_pgm::prolong_applyadd(
-        this->exec, this->agg, this->coarse_b.get(), x.get());
-
-    GKO_ASSERT_MTX_NEAR(x, this->prolong_ans, r<value_type>::value);
-}
-
-
 TYPED_TEST(AmgxPgm, MatchEdge)
 {
     using index_type = typename TestFixture::index_type;

From 47d04f744715d6548ac2ca97f78cb3ee6895bbbb Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 9 Mar 2021 15:28:00 +0800
Subject: [PATCH 0484/1008] Add fill_array TypedTest and update documentation

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 cuda/test/components/fill_array.cpp           | 26 ++++++++++++-------
 dpcpp/test/components/fill_array.cpp          |  6 ++---
 hip/test/components/fill_array.hip.cpp        | 25 +++++++++++-------
 include/ginkgo/core/base/composition.hpp      |  3 +++
 include/ginkgo/core/base/lin_op.hpp           | 22 ----------------
 include/ginkgo/core/multigrid/amgx_pgm.hpp    |  8 +++---
 .../ginkgo/core/multigrid/multigrid_level.hpp | 10 +++----
 7 files changed, 46 insertions(+), 54 deletions(-)

diff --git a/cuda/test/components/fill_array.cpp b/cuda/test/components/fill_array.cpp
index cb4c06c571f..06706b174e8 100644
--- a/cuda/test/components/fill_array.cpp
+++ b/cuda/test/components/fill_array.cpp
@@ -45,14 +45,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/test/utils/assertions.hpp"
+#include "cuda/test/utils.hpp"
 
 
 namespace {
 
 
+template <typename T>
 class FillArray : public ::testing::Test {
 protected:
-    using value_type = double;
+    using value_type = T;
     FillArray()
         : ref(gko::ReferenceExecutor::create()),
           exec(gko::CudaExecutor::create(0, ref)),
@@ -61,7 +63,7 @@ class FillArray : public ::testing::Test {
           dvals(exec, total_size),
           seqs(ref, total_size)
     {
-        std::fill_n(vals.get_data(), total_size, 1234.0);
+        std::fill_n(vals.get_data(), total_size, T(1234));
         std::iota(seqs.get_data(), seqs.get_data() + total_size, 0);
     }
 
@@ -73,22 +75,26 @@ class FillArray : public ::testing::Test {
     gko::Array<value_type> seqs;
 };
 
+TYPED_TEST_SUITE(FillArray, gko::test::ValueAndIndexTypes);
 
-TEST_F(FillArray, EqualsReference)
+
+TYPED_TEST(FillArray, EqualsReference)
 {
-    gko::kernels::cuda::components::fill_array(exec, dvals.get_data(),
-                                               total_size, 1234.0);
+    using T = typename TestFixture::value_type;
+    gko::kernels::cuda::components::fill_array(
+        this->exec, this->dvals.get_data(), this->total_size, T(1234));
 
-    GKO_ASSERT_ARRAY_EQ(vals, dvals);
+    GKO_ASSERT_ARRAY_EQ(this->vals, this->dvals);
 }
 
 
-TEST_F(FillArray, FillSeqEqualsReference)
+TYPED_TEST(FillArray, FillSeqEqualsReference)
 {
-    gko::kernels::cuda::components::fill_seq_array(exec, dvals.get_data(),
-                                                   total_size);
+    using T = typename TestFixture::value_type;
+    gko::kernels::cuda::components::fill_seq_array(
+        this->exec, this->dvals.get_data(), this->total_size);
 
-    GKO_ASSERT_ARRAY_EQ(seqs, dvals);
+    GKO_ASSERT_ARRAY_EQ(this->seqs, this->dvals);
 }
 
 
diff --git a/dpcpp/test/components/fill_array.cpp b/dpcpp/test/components/fill_array.cpp
index 75474ea4ca8..1871793b83b 100644
--- a/dpcpp/test/components/fill_array.cpp
+++ b/dpcpp/test/components/fill_array.cpp
@@ -62,7 +62,7 @@ class FillArray : public ::testing::Test {
           dvals(exec, total_size),
           seqs(ref, total_size)
     {
-        std::fill_n(vals.get_data(), total_size, T(1523));
+        std::fill_n(vals.get_data(), total_size, T(1234));
         std::iota(seqs.get_data(), seqs.get_data() + total_size, 0);
     }
 
@@ -81,13 +81,13 @@ TYPED_TEST(FillArray, EqualsReference)
 {
     using T = typename TestFixture::value_type;
     gko::kernels::dpcpp::components::fill_array(
-        this->exec, this->dvals.get_data(), this->total_size, T(1523));
+        this->exec, this->dvals.get_data(), this->total_size, T(1234));
 
     GKO_ASSERT_ARRAY_EQ(this->vals, this->dvals);
 }
 
 
-TYPED_TEST(FillArray, EqualsReference)
+TYPED_TEST(FillArray, FillSeqEqualsReference)
 {
     using T = typename TestFixture::value_type;
     gko::kernels::dpcpp::components::fill_seq_array(
diff --git a/hip/test/components/fill_array.hip.cpp b/hip/test/components/fill_array.hip.cpp
index d9f5e20f447..348bee3f329 100644
--- a/hip/test/components/fill_array.hip.cpp
+++ b/hip/test/components/fill_array.hip.cpp
@@ -57,9 +57,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
+template <typename T>
 class FillArray : public ::testing::Test {
 protected:
-    using value_type = double;
+    using value_type = T;
     FillArray()
         : ref(gko::ReferenceExecutor::create()),
           exec(gko::HipExecutor::create(0, ref)),
@@ -68,7 +69,7 @@ class FillArray : public ::testing::Test {
           dvals(exec, total_size),
           seqs(ref, total_size)
     {
-        std::fill_n(vals.get_data(), total_size, 1234.0);
+        std::fill_n(vals.get_data(), total_size, T(1234));
         std::iota(seqs.get_data(), seqs.get_data() + total_size, 0);
     }
 
@@ -80,22 +81,26 @@ class FillArray : public ::testing::Test {
     gko::Array<value_type> seqs;
 };
 
+TYPED_TEST_SUITE(FillArray, gko::test::ValueAndIndexTypes);
 
-TEST_F(FillArray, EqualsReference)
+
+TYPED_TEST(FillArray, EqualsReference)
 {
-    gko::kernels::hip::components::fill_array(exec, dvals.get_data(),
-                                              total_size, 1234.0);
+    using T = typename TestFixture::value_type;
+    gko::kernels::hip::components::fill_array(
+        this->exec, this->dvals.get_data(), this->total_size, T(1234));
 
-    GKO_ASSERT_ARRAY_EQ(vals, dvals);
+    GKO_ASSERT_ARRAY_EQ(this->vals, this->dvals);
 }
 
 
-TEST_F(FillArray, FillSeqEqualsReference)
+TYPED_TEST(FillArray, FillSeqEqualsReference)
 {
-    gko::kernels::hip::components::fill_seq_array(exec, dvals.get_data(),
-                                                  total_size);
+    using T = typename TestFixture::value_type;
+    gko::kernels::hip::components::fill_seq_array(
+        this->exec, this->dvals.get_data(), this->total_size);
 
-    GKO_ASSERT_ARRAY_EQ(seqs, dvals);
+    GKO_ASSERT_ARRAY_EQ(this->seqs, this->dvals);
 }
 
 
diff --git a/include/ginkgo/core/base/composition.hpp b/include/ginkgo/core/base/composition.hpp
index f5ffc05a4a5..3be5ca30db3 100644
--- a/include/ginkgo/core/base/composition.hpp
+++ b/include/ginkgo/core/base/composition.hpp
@@ -195,6 +195,9 @@ class UseComposition {
      * @note when this composition is not set, this function always returns
      *       nullptr. However, when this composition is set, it will throw
      *       exception when exceeding index.
+     *
+     * @throw std::out_of_range if index is out of bound when composition is
+     *        existed.
      */
     std::shared_ptr<const LinOp> get_operator_at(size_type index) const
     {
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index be2760d5902..cde3840c30f 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -654,28 +654,6 @@ class Preconditionable {
 };
 
 
-class ApplyAddable {
-public:
-    void apply2(const LinOp *b, LinOp *x) const
-    {
-        // check the comformat size
-        this->apply2_impl(b, x);
-    }
-
-    void apply2(const LinOp *alpha, const LinOp *b, LinOp *x) const
-    {
-        // check the comformat size
-        this->apply2_impl(alpha, b, x);
-    }
-
-protected:
-    virtual void apply2_impl(const LinOp *b, LinOp *x) const = 0;
-
-    virtual void apply2_impl(const LinOp *alpha, const LinOp *b,
-                             LinOp *x) const = 0;
-};
-
-
 /**
  * The diagonal of a LinOp implementing this interface can be extracted.
  * extract_diagonal extracts the elements whose col and row index are the
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index 99d2dc6ddbd..c021df150d9 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -52,10 +52,10 @@ namespace multigrid {
 /**
  * Amgx parallel graph match (AmgxPgm) is the aggregate method introduced in the
  * paper M. Naumov et al., "AmgX: A Library for GPU Accelerated Algebraic
- * Multigrid and Preconditioned Iterative Methods". Current implemenation only
+ * Multigrid and Preconditioned Iterative Methods". Current implementation only
  * contains size = 2 version.
  *
- * AmgxPgm creates the aggreagate group according to the matrix value not the
+ * AmgxPgm creates the aggregate group according to the matrix value not the
  * structure. AmgxPgm gives two steps (one-phase handshaking) to group the
  * elements.
  * 1: get the strongest neighbor of each unaggregated element.
@@ -94,7 +94,7 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
     /**
      * Returns the aggregate group.
      *
-     * Aggregate group whose size is same as the number of rows stores the
+     * Aggregate group whose size is same as the number of rows. Stores the
      * mapping information from row index to coarse row index.
      * i.e., agg[row_idx] = coarse_row_idx.
      *
@@ -134,7 +134,7 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
         /**
          * Use the deterministic assign_to_exist_agg method or not.
          *
-         * If the deterministic is true, always get the same aggregated group
+         * If deterministic is set to true, always get the same aggregated group
          * from the same matrix. Otherwise, the aggregated group might be
          * different depending on the execution ordering.
          */
diff --git a/include/ginkgo/core/multigrid/multigrid_level.hpp b/include/ginkgo/core/multigrid/multigrid_level.hpp
index c1249c2dad5..5847a039410 100644
--- a/include/ginkgo/core/multigrid/multigrid_level.hpp
+++ b/include/ginkgo/core/multigrid/multigrid_level.hpp
@@ -43,7 +43,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/base/utils.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
 
 
 namespace gko {
@@ -104,7 +103,8 @@ class MultigridLevel : public UseComposition<ValueType> {
 
 protected:
     /**
-     * Sets the multigrid level information.
+     * Sets the multigrid level information. The stored composition will be
+     * prolong_op x coarse_op x restrict_op.
      *
      * @param prolong_op  the prolong operator
      * @param coarse_op  the coarse operator
@@ -126,11 +126,11 @@ class MultigridLevel : public UseComposition<ValueType> {
     /**
      * Creates a MultigridLevel with the given fine operator
      *
-     * @param fine_op  The fine operotor associated to the multigrid_level
+     * @param fine_op  The fine operator associated to the multigrid_level
      *
      * @note the multigrid is generated in the generation not the construction,
-     *       so needs to call `set_multigrid_level` to set corresponding
-     *       information after generation.
+     *       so the user needs to call `set_multigrid_level` to set the
+     *       corresponding information after generation.
      */
     explicit MultigridLevel(std::shared_ptr<const LinOp> fine_op)
         : fine_op_(fine_op)

From 610084e155c4995f60ad2feeb3e7c850d8d53540 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 10 Mar 2021 01:51:38 +0800
Subject: [PATCH 0485/1008] RestrictProlong -> MgLevel, rstr_prlg -> mg_level

---
 core/test/multigrid/amgx_pgm.cpp              | 10 ++--
 include/ginkgo/core/multigrid/amgx_pgm.hpp    |  2 +-
 reference/test/multigrid/amgx_pgm_kernels.cpp | 49 +++++++++----------
 3 files changed, 28 insertions(+), 33 deletions(-)

diff --git a/core/test/multigrid/amgx_pgm.cpp b/core/test/multigrid/amgx_pgm.cpp
index 6a897cee8b8..8648f38e3ba 100644
--- a/core/test/multigrid/amgx_pgm.cpp
+++ b/core/test/multigrid/amgx_pgm.cpp
@@ -57,10 +57,10 @@ class AmgxPgmFactory : public ::testing::Test {
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using Mtx = gko::matrix::Csr<value_type, index_type>;
     using Vec = gko::matrix::Dense<value_type>;
-    using RestrictProlong = gko::multigrid::AmgxPgm<value_type, index_type>;
+    using MgLevel = gko::multigrid::AmgxPgm<value_type, index_type>;
     AmgxPgmFactory()
         : exec(gko::ReferenceExecutor::create()),
-          amgxpgm_factory(RestrictProlong::build()
+          amgxpgm_factory(MgLevel::build()
                               .with_max_iterations(2u)
                               .with_max_unassigned_ratio(0.1)
                               .with_deterministic(true)
@@ -69,7 +69,7 @@ class AmgxPgmFactory : public ::testing::Test {
     {}
 
     std::shared_ptr<const gko::Executor> exec;
-    std::unique_ptr<typename RestrictProlong::Factory> amgxpgm_factory;
+    std::unique_ptr<typename MgLevel::Factory> amgxpgm_factory;
 };
 
 TYPED_TEST_CASE(AmgxPgmFactory, gko::test::ValueIndexTypes);
@@ -83,8 +83,8 @@ TYPED_TEST(AmgxPgmFactory, FactoryKnowsItsExecutor)
 
 TYPED_TEST(AmgxPgmFactory, DefaultSetting)
 {
-    using RestrictProlong = typename TestFixture::RestrictProlong;
-    auto factory = RestrictProlong::build().on(this->exec);
+    using MgLevel = typename TestFixture::MgLevel;
+    auto factory = MgLevel::build().on(this->exec);
 
     ASSERT_EQ(factory->get_parameters().max_iterations, 15u);
     ASSERT_EQ(factory->get_parameters().max_unassigned_ratio, 0.05);
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index c021df150d9..e2a1a579713 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -67,7 +67,7 @@ namespace multigrid {
  * @tparam ValueType  precision of matrix elements
  * @tparam IndexType  precision of matrix indexes
  *
- * @ingroup RestrictProlong
+ * @ingroup MultigridLevel
  * @ingroup Multigrid
  * @ingroup LinOp
  */
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index d3cd8e19994..63f8baf2e62 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -67,13 +67,13 @@ class AmgxPgm : public ::testing::Test {
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using Mtx = gko::matrix::Csr<value_type, index_type>;
     using Vec = gko::matrix::Dense<value_type>;
-    using RestrictProlong = gko::multigrid::AmgxPgm<value_type, index_type>;
+    using MgLevel = gko::multigrid::AmgxPgm<value_type, index_type>;
     using VT = value_type;
     using real_type = gko::remove_complex<value_type>;
     using WeightMtx = gko::matrix::Csr<real_type, index_type>;
     AmgxPgm()
         : exec(gko::ReferenceExecutor::create()),
-          amgxpgm_factory(RestrictProlong::build()
+          amgxpgm_factory(MgLevel::build()
                               .with_max_iterations(2u)
                               .with_max_unassigned_ratio(0.1)
                               .on(exec)),
@@ -107,7 +107,7 @@ class AmgxPgm : public ::testing::Test {
           agg(exec, 5)
     {
         this->create_mtx(mtx.get(), weight.get(), &agg, coarse.get());
-        rstr_prlg = amgxpgm_factory->generate(mtx);
+        mg_level = amgxpgm_factory->generate(mtx);
         mtx_diag = weight->extract_diagonal();
     }
 
@@ -210,8 +210,8 @@ class AmgxPgm : public ::testing::Test {
     std::shared_ptr<Vec> prolong_ans;
     std::shared_ptr<Vec> prolong_applyans;
     std::shared_ptr<Vec> fine_x;
-    std::unique_ptr<typename RestrictProlong::Factory> amgxpgm_factory;
-    std::unique_ptr<RestrictProlong> rstr_prlg;
+    std::unique_ptr<typename MgLevel::Factory> amgxpgm_factory;
+    std::unique_ptr<MgLevel> mg_level;
 };
 
 TYPED_TEST_SUITE(AmgxPgm, gko::test::ValueIndexTypes);
@@ -220,13 +220,12 @@ TYPED_TEST_SUITE(AmgxPgm, gko::test::ValueIndexTypes);
 TYPED_TEST(AmgxPgm, CanBeCopied)
 {
     using Mtx = typename TestFixture::Mtx;
-    using RestrictProlong = typename TestFixture::RestrictProlong;
+    using MgLevel = typename TestFixture::MgLevel;
     auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
 
-    copy->copy_from(this->rstr_prlg.get());
-    auto copy_mtx =
-        static_cast<RestrictProlong *>(copy.get())->get_system_matrix();
-    auto copy_agg = static_cast<RestrictProlong *>(copy.get())->get_const_agg();
+    copy->copy_from(this->mg_level.get());
+    auto copy_mtx = copy->get_system_matrix();
+    auto copy_agg = copy->get_const_agg();
     auto copy_coarse = copy->get_coarse_op();
 
     this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
@@ -241,13 +240,12 @@ TYPED_TEST(AmgxPgm, CanBeCopied)
 TYPED_TEST(AmgxPgm, CanBeMoved)
 {
     using Mtx = typename TestFixture::Mtx;
-    using RestrictProlong = typename TestFixture::RestrictProlong;
+    using MgLevel = typename TestFixture::MgLevel;
     auto copy = this->amgxpgm_factory->generate(Mtx::create(this->exec));
 
-    copy->copy_from(std::move(this->rstr_prlg));
-    auto copy_mtx =
-        static_cast<RestrictProlong *>(copy.get())->get_system_matrix();
-    auto copy_agg = static_cast<RestrictProlong *>(copy.get())->get_const_agg();
+    copy->copy_from(std::move(this->mg_level));
+    auto copy_mtx = copy->get_system_matrix();
+    auto copy_agg = copy->get_const_agg();
     auto copy_coarse = copy->get_coarse_op();
 
     this->assert_same_matrices(static_cast<const Mtx *>(copy_mtx.get()),
@@ -262,13 +260,11 @@ TYPED_TEST(AmgxPgm, CanBeMoved)
 TYPED_TEST(AmgxPgm, CanBeCloned)
 {
     using Mtx = typename TestFixture::Mtx;
-    using RestrictProlong = typename TestFixture::RestrictProlong;
-    auto clone = this->rstr_prlg->clone();
+    using MgLevel = typename TestFixture::MgLevel;
+    auto clone = this->mg_level->clone();
     auto clone_mtx = clone->get_system_matrix();
-    auto clone_agg =
-        static_cast<RestrictProlong *>(clone.get())->get_const_agg();
-    auto clone_coarse =
-        dynamic_cast<RestrictProlong *>(clone.get())->get_coarse_op();
+    auto clone_agg = clone->get_const_agg();
+    auto clone_coarse = clone->get_coarse_op();
 
     this->assert_same_matrices(static_cast<const Mtx *>(clone_mtx.get()),
                                this->mtx.get());
@@ -281,13 +277,12 @@ TYPED_TEST(AmgxPgm, CanBeCloned)
 
 TYPED_TEST(AmgxPgm, CanBeCleared)
 {
-    using RestrictProlong = typename TestFixture::RestrictProlong;
+    using MgLevel = typename TestFixture::MgLevel;
 
-    this->rstr_prlg->clear();
-    auto mtx = static_cast<RestrictProlong *>(this->rstr_prlg.get())
-                   ->get_system_matrix();
-    auto coarse = this->rstr_prlg->get_coarse_op();
-    auto agg = static_cast<RestrictProlong *>(this->rstr_prlg.get())->get_agg();
+    this->mg_level->clear();
+    auto mtx = this->mg_level->get_system_matrix();
+    auto coarse = this->mg_level->get_coarse_op();
+    auto agg = this->mg_level->get_agg();
 
     ASSERT_EQ(mtx, nullptr);
     ASSERT_EQ(coarse, nullptr);

From 6e52611073d110480db429a7831c84c18307b140 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 11 Mar 2021 17:09:07 +0800
Subject: [PATCH 0486/1008] split MultigridLevel EnableMultigridLevel -
 MultigridLevel as interface without ValueType information -
 EnableMultigridLevel gives the default impl with composition

---
 include/ginkgo/core/multigrid/amgx_pgm.hpp    |  4 +-
 .../ginkgo/core/multigrid/multigrid_level.hpp | 68 +++++++++++++------
 2 files changed, 49 insertions(+), 23 deletions(-)

diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index e2a1a579713..5c5b36ebf7c 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -73,7 +73,7 @@ namespace multigrid {
  */
 template <typename ValueType = default_precision, typename IndexType = int32>
 class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
-                public MultigridLevel<ValueType> {
+                public EnableMultigridLevel<ValueType> {
     friend class EnableLinOp<AmgxPgm>;
     friend class EnablePolymorphicObject<AmgxPgm, LinOp>;
 
@@ -163,7 +163,7 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
                      std::shared_ptr<const LinOp> system_matrix)
         : EnableLinOp<AmgxPgm>(factory->get_executor(),
                                system_matrix->get_size()),
-          MultigridLevel<ValueType>(system_matrix),
+          EnableMultigridLevel<ValueType>(system_matrix),
           parameters_{factory->get_parameters()},
           system_matrix_{system_matrix},
           agg_(factory->get_executor(), system_matrix_->get_size()[0])
diff --git a/include/ginkgo/core/multigrid/multigrid_level.hpp b/include/ginkgo/core/multigrid/multigrid_level.hpp
index 5847a039410..e86e51bc53b 100644
--- a/include/ginkgo/core/multigrid/multigrid_level.hpp
+++ b/include/ginkgo/core/multigrid/multigrid_level.hpp
@@ -52,51 +52,77 @@ namespace multigrid {
 /**
  * This class represents two levels in a multigrid hierarchy.
  *
- * A class implementing this interface should use the
- * this->get_compositions()->apply(...) as its own apply, which represents
- * op(b) = prolong(coarse(restrict(b))). All MultigridLevel generator should
- * inherit from this class.
+ * The MultigridLevel is an interface that allows to get the individual
+ * components of multigrid level. Each implementation of a multigrid level
+ * should inherit from this interface. Use EnableMultigridLevel<ValueType> to
+ * implement this interface with composition by default.
  *
  * @ingroup Multigrid
  */
-template <typename ValueType>
-class MultigridLevel : public UseComposition<ValueType> {
+class MultigridLevel {
 public:
-    using value_type = ValueType;
-
     /**
      * Returns the operator on fine level.
      *
      * @return  the operator on fine level.
      */
-    std::shared_ptr<const LinOp> get_fine_op() const { return fine_op_; }
+    virtual std::shared_ptr<const LinOp> get_fine_op() const = 0;
 
     /**
      * Returns the restrict operator.
      *
      * @return  the restrict operator.
      */
-    std::shared_ptr<const LinOp> get_restrict_op() const
-    {
-        return this->get_operator_at(2);
-    }
+    virtual std::shared_ptr<const LinOp> get_restrict_op() const = 0;
 
     /**
      * Returns the operator on coarse level.
      *
      * @return  the operator on coarse level.
      */
-    std::shared_ptr<const LinOp> get_coarse_op() const
-    {
-        return this->get_operator_at(1);
-    }
+    virtual std::shared_ptr<const LinOp> get_coarse_op() const = 0;
 
     /**
      * Returns the prolong operator.
      *
      * @return  the prolong operator.
      */
-    std::shared_ptr<const LinOp> get_prolong_op() const
+    virtual std::shared_ptr<const LinOp> get_prolong_op() const = 0;
+};
+
+
+/**
+ * The EnableMultigridLevel gives the default implementation of MultigridLevel
+ * with composition and provides `set_multigrid_level` function.
+ *
+ * A class inherit from EnableMultigridLevel should use the
+ * this->get_compositions()->apply(...) as its own apply, which represents
+ * op(b) = prolong(coarse(restrict(b))).
+ *
+ * @ingroup Multigrid
+ */
+template <typename ValueType>
+class EnableMultigridLevel : public MultigridLevel,
+                             public UseComposition<ValueType> {
+public:
+    using value_type = ValueType;
+
+    std::shared_ptr<const LinOp> get_fine_op() const override
+    {
+        return fine_op_;
+    }
+
+    std::shared_ptr<const LinOp> get_restrict_op() const override
+    {
+        return this->get_operator_at(2);
+    }
+
+    std::shared_ptr<const LinOp> get_coarse_op() const override
+    {
+        return this->get_operator_at(1);
+    }
+
+    std::shared_ptr<const LinOp> get_prolong_op() const override
     {
         return this->get_operator_at(0);
     }
@@ -121,10 +147,10 @@ class MultigridLevel : public UseComposition<ValueType> {
         this->set_composition(prolong_op, coarse_op, restrict_op);
     }
 
-    explicit MultigridLevel() {}
+    explicit EnableMultigridLevel() {}
 
     /**
-     * Creates a MultigridLevel with the given fine operator
+     * Creates a EnableMultigridLevel with the given fine operator
      *
      * @param fine_op  The fine operator associated to the multigrid_level
      *
@@ -132,7 +158,7 @@ class MultigridLevel : public UseComposition<ValueType> {
      *       so the user needs to call `set_multigrid_level` to set the
      *       corresponding information after generation.
      */
-    explicit MultigridLevel(std::shared_ptr<const LinOp> fine_op)
+    explicit EnableMultigridLevel(std::shared_ptr<const LinOp> fine_op)
         : fine_op_(fine_op)
     {}
 

From f2d1f5b71294092cb3c85125a97d6af962cb1530 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 19 Feb 2021 16:23:14 +0100
Subject: [PATCH 0487/1008] Make reduced_row_major independent of Ginkgo

Move the accessor {scaled,}reduced_row_major to `accessor/` in order
for others to use it without compiling Ginkgo. RowMajor was not copied
yet (it must not be moved since it is part of the public interface).
---
 {core/base => accessor}/accessor_helper.hpp   | 242 +++++++-----
 .../base => accessor}/accessor_references.hpp | 184 ++++-----
 accessor/index_span.hpp                       | 100 +++++
 accessor/range.hpp                            | 144 ++++++++
 accessor/reduced_row_major.hpp                | 299 +++++++++++++++
 .../scaled_reduced_row_major.hpp              | 348 +++---------------
 accessor/utils.hpp                            | 150 ++++++++
 core/test/base/accessors_reduced_storage.cpp  |  89 ++---
 8 files changed, 1049 insertions(+), 507 deletions(-)
 rename {core/base => accessor}/accessor_helper.hpp (69%)
 rename {core/base => accessor}/accessor_references.hpp (73%)
 create mode 100644 accessor/index_span.hpp
 create mode 100644 accessor/range.hpp
 create mode 100644 accessor/reduced_row_major.hpp
 rename core/base/accessors.hpp => accessor/scaled_reduced_row_major.hpp (58%)
 create mode 100644 accessor/utils.hpp

diff --git a/core/base/accessor_helper.hpp b/accessor/accessor_helper.hpp
similarity index 69%
rename from core/base/accessor_helper.hpp
rename to accessor/accessor_helper.hpp
index ac5246f5625..72099aee856 100644
--- a/core/base/accessor_helper.hpp
+++ b/accessor/accessor_helper.hpp
@@ -30,23 +30,23 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_ACCESSOR_HELPER_HPP_
-#define GKO_CORE_BASE_ACCESSOR_HELPER_HPP_
+#ifndef GKO_ACCESSOR_ACCESSOR_HELPER_HPP_
+#define GKO_ACCESSOR_ACCESSOR_HELPER_HPP_
 
 
 #include <array>
+#include <memory>
 #include <tuple>
 #include <type_traits>
 #include <utility>
 
 
-#include <ginkgo/core/base/dim.hpp>
-#include <ginkgo/core/base/range.hpp>
-#include <ginkgo/core/base/types.hpp>
+#include "accessor/index_span.hpp"
+#include "accessor/utils.hpp"
 
 
 namespace gko {
-namespace accessor {
+namespace acc {
 /**
  * This namespace contains helper functionality for the accessors.
  *
@@ -71,18 +71,18 @@ struct row_major_helper_s {
     static constexpr size_type dim_idx{current_iter - 1};
 
     template <typename FirstType, typename... Indices>
-    static constexpr GKO_ATTRIBUTES ValueType
-    compute(const dim<total_dim> &size,
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    compute(const std::array<size_type, total_dim> &size,
             const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
                 &stride,
             FirstType first, Indices &&... idxs)
     {
         // The ASSERT size check must NOT be indexed with `dim_idx` directy,
         // otherwise, it leads to a linker error. The reason is likely that
-        // `dim<N>::operator[](const size_type &)` uses a reference. Since
-        // `dim_idx` is constexpr (and not defined in a namespace scope), it
-        // can't be odr-used.
-        return GKO_ASSERT(first < size[static_cast<size_type>(dim_idx)]),
+        // `std::array<size_type, N>::operator[](const size_type &)` uses a
+        // reference. Since `dim_idx` is constexpr (and not defined in a
+        // namespace scope), it can't be odr-used.
+        return GKO_ACC_ASSERT(first < size[static_cast<size_type>(dim_idx)]),
                first * stride[dim_idx] +
                    row_major_helper_s<ValueType, total_dim, current_iter + 1>::
                        compute(size, stride, std::forward<Indices>(idxs)...);
@@ -92,16 +92,57 @@ struct row_major_helper_s {
 template <typename ValueType, size_type total_dim>
 struct row_major_helper_s<ValueType, total_dim, total_dim> {
     template <typename FirstType>
-    static constexpr GKO_ATTRIBUTES ValueType
-    compute(const dim<total_dim> &size,
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    compute(const std::array<size_type, total_dim> &size,
             const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>,
             FirstType first)
     {
-        return GKO_ASSERT(first < size[total_dim - 1]), first;
+        return GKO_ACC_ASSERT(first < size[total_dim - 1]), first;
     }
 };
 
 
+template <typename ValueType, size_type iter, size_type N,
+          typename DimensionType>
+constexpr GKO_ACC_ATTRIBUTES std::enable_if_t<iter == N, ValueType>
+mult_dim_upwards(const std::array<DimensionType, N> &)
+{
+    return 1;
+}
+
+template <typename ValueType, size_type iter, size_type N,
+          typename DimensionType>
+constexpr GKO_ACC_ATTRIBUTES std::enable_if_t<(iter < N), ValueType>
+mult_dim_upwards(const std::array<DimensionType, N> &size)
+{
+    return size[iter] * mult_dim_upwards<ValueType, iter + 1>(size);
+}
+
+
+template <typename ValueType, size_type iter = 1, size_type N,
+          typename DimensionType, typename... Args>
+constexpr GKO_ACC_ATTRIBUTES
+    std::enable_if_t<N == 0 || (iter == N && iter == sizeof...(Args) + 1),
+                     std::array<ValueType, N == 0 ? 0 : N - 1>>
+    compute_default_stride_array(const std::array<DimensionType, N> &,
+                                 Args &&... args)
+{
+    return {{std::forward<Args>(args)...}};
+}
+
+template <typename ValueType, size_type iter = 1, size_type N,
+          typename DimensionType, typename... Args>
+constexpr GKO_ACC_ATTRIBUTES std::enable_if_t<
+    (iter < N) && (iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
+compute_default_stride_array(const std::array<DimensionType, N> &size,
+                             Args &&... args)
+{
+    return compute_default_stride_array<ValueType, iter + 1>(
+        size, std::forward<Args>(args)...,
+        mult_dim_upwards<ValueType, iter>(size));
+}
+
+
 }  // namespace detail
 
 
@@ -110,8 +151,8 @@ struct row_major_helper_s<ValueType, total_dim, total_dim> {
  * stride array
  */
 template <typename ValueType, size_type total_dim, typename... Indices>
-constexpr GKO_ATTRIBUTES ValueType compute_storage_index(
-    const dim<total_dim> &size,
+constexpr GKO_ACC_ATTRIBUTES ValueType compute_storage_index(
+    const std::array<size_type, total_dim> &size,
     const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)> &stride,
     Indices &&... idxs)
 {
@@ -120,6 +161,31 @@ constexpr GKO_ATTRIBUTES ValueType compute_storage_index(
 }
 
 
+/**
+ * Computes the default stride array from a given size, assuming there is no
+ * padding.
+ *
+ * Example: std::array<ValueType, 4> size={2, 3, 5, 7} results in a return value
+ * of: std::array<ValueType, 3> = {3*5*7, 5*7, 7}
+ *
+ * @tparam ValueType  value type of the values in the returned array
+ *
+ * @tparam dimensions  number of dimensions in the given size
+ *
+ * @tparam DimensionType  value type of the stored size
+ *
+ * @returns an std::array<ValueType, dimensions - 1> with the stride
+ *          information.
+ */
+template <typename ValueType, size_type dimensions, typename DimensionType>
+constexpr GKO_ACC_ATTRIBUTES std::array<ValueType,
+                                        (dimensions > 0 ? dimensions - 1 : 0)>
+compute_default_stride_array(const std::array<DimensionType, dimensions> &size)
+{
+    return detail::compute_default_stride_array<ValueType>(size);
+}
+
+
 namespace detail {
 
 
@@ -164,8 +230,8 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
                   "The processed bits must be < total number of set bits!");
 
     template <typename... Args>
-    static constexpr GKO_ATTRIBUTES std::array<ValueType, stride_size>
-    build_stride(const dim<total_dim> &size, Args &&... args)
+    static constexpr GKO_ACC_ATTRIBUTES std::array<ValueType, stride_size>
+    build_stride(const std::array<size_type, total_dim> &size, Args &&... args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
@@ -173,8 +239,8 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
                                      mult_size_upwards(size));
     }
 
-    static constexpr GKO_ATTRIBUTES ValueType
-    mult_size_upwards(const dim<total_dim> &size)
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    mult_size_upwards(const std::array<size_type, total_dim> &size)
     {
         return size[dim_idx] *
                row_major_masked_helper_s<
@@ -183,15 +249,15 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     }
 
     template <typename First, typename... Indices>
-    static constexpr GKO_ATTRIBUTES ValueType
-    compute_mask_idx(const dim<total_dim> &size,
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    compute_mask_idx(const std::array<size_type, total_dim> &size,
                      const std::array<ValueType, stride_size> &stride,
                      First first, Indices &&... idxs)
     {
         static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
                       "Mismatching number of Idxs!");
         // If it is the last set dimension, there is no need for a stride
-        return GKO_ASSERT(first < size[dim_idx]),
+        return GKO_ACC_ASSERT(first < size[dim_idx]),
                first * (set_bits_processed == stride_size
                             ? 1
                             : stride[set_bits_processed]) +
@@ -204,15 +270,15 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     }
 
     template <typename First, typename... Indices>
-    static constexpr GKO_ATTRIBUTES ValueType
-    compute_direct_idx(const dim<total_dim> &size,
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    compute_direct_idx(const std::array<size_type, total_dim> &size,
                        const std::array<ValueType, stride_size> &stride,
                        First first, Indices &&... idxs)
     {
         static_assert(sizeof...(Indices) == stride_size - set_bits_processed,
                       "Mismatching number of Idxs!");
         // If it is the last set dimension, there is no need for a stride
-        return GKO_ASSERT(first < size[dim_idx]),
+        return GKO_ACC_ASSERT(first < size[dim_idx]),
                first * (set_bits_processed == stride_size
                             ? 1
                             : stride[set_bits_processed]) +
@@ -240,16 +306,16 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
                   "The iteration count must be smaller than total_dim here!!!");
 
     template <typename... Args>
-    static constexpr GKO_ATTRIBUTES std::array<ValueType, stride_size>
-    build_stride(const dim<total_dim> &size, Args &&... args)
+    static constexpr GKO_ACC_ATTRIBUTES std::array<ValueType, stride_size>
+    build_stride(const std::array<size_type, total_dim> &size, Args &&... args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
             total_dim>::build_stride(size, std::forward<Args>(args)...);
     }
 
-    static constexpr GKO_ATTRIBUTES ValueType
-    mult_size_upwards(const dim<total_dim> &size)
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    mult_size_upwards(const std::array<size_type, total_dim> &size)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
@@ -257,15 +323,15 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
     }
 
     template <typename First, typename... Indices>
-    static constexpr GKO_ATTRIBUTES ValueType
-    compute_mask_idx(const dim<total_dim> &size,
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    compute_mask_idx(const std::array<size_type, total_dim> &size,
                      const std::array<ValueType, stride_size> &stride,
                      First first, Indices &&... idxs)
     {
         static_assert(sizeof...(Indices) + 1 == total_dim - dim_idx,
                       "Mismatching number of Idxs!");
         // If it is the last set dimension, there is no need for a stride
-        return GKO_ASSERT(first < size[dim_idx]),
+        return GKO_ACC_ASSERT(first < size[dim_idx]),
                first * (set_bits_processed == stride_size
                             ? 1
                             : stride[set_bits_processed]) +
@@ -278,15 +344,15 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
     }
 
     template <typename First, typename... Indices>
-    static constexpr GKO_ATTRIBUTES ValueType
-    compute_direct_idx(const dim<total_dim> &size,
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    compute_direct_idx(const std::array<size_type, total_dim> &size,
                        const std::array<ValueType, stride_size> &stride,
                        First first, Indices &&... idxs)
     {
         static_assert(sizeof...(Indices) == stride_size - set_bits_processed,
                       "Mismatching number of Idxs!");
         // If it is the last set dimension, there is no need for a stride
-        return GKO_ASSERT(first < size[dim_idx]),
+        return GKO_ACC_ASSERT(first < size[dim_idx]),
                first * (set_bits_processed == stride_size
                             ? 1
                             : stride[set_bits_processed]) +
@@ -313,16 +379,16 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     static_assert(set_bits_processed <= stride_size + 1,
                   "The processed bits must be < total number of set bits!");
     template <typename... Args>
-    static constexpr GKO_ATTRIBUTES std::array<ValueType, stride_size>
-    build_stride(const dim<total_dim> &size, Args &&... args)
+    static constexpr GKO_ACC_ATTRIBUTES std::array<ValueType, stride_size>
+    build_stride(const std::array<size_type, total_dim> &size, Args &&... args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed, stride_size, dim_idx + 1,
             total_dim>::build_stride(size, std::forward<Args>(args)...);
     }
 
-    static constexpr GKO_ATTRIBUTES ValueType
-    mult_size_upwards(const dim<total_dim> &size)
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    mult_size_upwards(const std::array<size_type, total_dim> &size)
     {
         return row_major_masked_helper_s<ValueType, mask, set_bits_processed,
                                          stride_size, dim_idx + 1,
@@ -330,8 +396,8 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     }
 
     template <typename First, typename... Indices>
-    static constexpr GKO_ATTRIBUTES ValueType
-    compute_mask_idx(const dim<total_dim> &size,
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    compute_mask_idx(const std::array<size_type, total_dim> &size,
                      const std::array<ValueType, stride_size> &stride, First,
                      Indices &&... idxs)
     {
@@ -344,8 +410,8 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
     }
 
     template <typename... Indices>
-    static constexpr GKO_ATTRIBUTES ValueType compute_direct_idx(
-        const dim<total_dim> &size,
+    static constexpr GKO_ACC_ATTRIBUTES ValueType compute_direct_idx(
+        const std::array<size_type, total_dim> &size,
         const std::array<ValueType, stride_size> &stride, Indices &&... idxs)
     {
         return row_major_masked_helper_s<
@@ -365,24 +431,26 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
                   "The processed bits must be smaller than the total number of "
                   "set bits!");
     template <typename... Args>
-    static constexpr GKO_ATTRIBUTES std::array<ValueType, stride_size>
-    build_stride(const dim<total_dim> &, Args &&... args)
+    static constexpr GKO_ACC_ATTRIBUTES std::array<ValueType, stride_size>
+    build_stride(const std::array<size_type, total_dim> &, Args &&... args)
     {
         return {{std::forward<Args>(args)...}};
     }
-    static constexpr GKO_ATTRIBUTES ValueType
-    mult_size_upwards(const dim<total_dim> &)
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    mult_size_upwards(const std::array<size_type, total_dim> &)
     {
         return 1;
     }
 
-    static constexpr GKO_ATTRIBUTES ValueType compute_mask_idx(
-        const dim<total_dim> &, const std::array<ValueType, stride_size> &)
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    compute_mask_idx(const std::array<size_type, total_dim> &,
+                     const std::array<ValueType, stride_size> &)
     {
         return 0;
     }
-    static constexpr GKO_ATTRIBUTES ValueType compute_direct_idx(
-        const dim<total_dim> &, const std::array<ValueType, stride_size> &)
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    compute_direct_idx(const std::array<size_type, total_dim> &,
+                       const std::array<ValueType, stride_size> &)
     {
         return 0;
     }
@@ -398,8 +466,8 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
  */
 template <typename ValueType, size_type mask, size_type stride_size,
           size_type total_dim, typename... Indices>
-constexpr GKO_ATTRIBUTES auto compute_masked_index(
-    const dim<total_dim> &size,
+constexpr GKO_ACC_ATTRIBUTES auto compute_masked_index(
+    const std::array<size_type, total_dim> &size,
     const std::array<ValueType, stride_size> &stride, Indices &&... idxs)
 {
     return detail::row_major_masked_helper_s<
@@ -414,8 +482,8 @@ constexpr GKO_ATTRIBUTES auto compute_masked_index(
  */
 template <typename ValueType, size_type mask, size_type stride_size,
           size_type total_dim, typename... Indices>
-constexpr GKO_ATTRIBUTES auto compute_masked_index_direct(
-    const dim<total_dim> &size,
+constexpr GKO_ACC_ATTRIBUTES auto compute_masked_index_direct(
+    const std::array<size_type, total_dim> &size,
     const std::array<ValueType, stride_size> &stride, Indices &&... idxs)
 {
     return detail::row_major_masked_helper_s<
@@ -432,8 +500,8 @@ constexpr GKO_ATTRIBUTES auto compute_masked_index_direct(
  */
 template <typename ValueType, size_type mask, size_type stride_size,
           size_type total_dim>
-constexpr GKO_ATTRIBUTES auto compute_default_masked_stride_array(
-    const dim<total_dim> &size)
+constexpr GKO_ACC_ATTRIBUTES auto compute_default_masked_stride_array(
+    const std::array<size_type, total_dim> &size)
 {
     return detail::row_major_masked_helper_s<ValueType, mask, 0, stride_size, 0,
                                              total_dim>::build_stride(size);
@@ -444,16 +512,16 @@ namespace detail {
 
 
 template <bool has_span, typename... Args>
-struct are_span_compatible_impl
+struct are_index_span_compatible_impl
     : public std::integral_constant<bool, has_span> {};
 
 template <bool has_span, typename First, typename... Args>
-struct are_span_compatible_impl<has_span, First, Args...>
+struct are_index_span_compatible_impl<has_span, First, Args...>
     : public std::conditional<
           std::is_integral<std::decay_t<First>>::value ||
-              std::is_same<std::decay_t<First>, span>::value,
-          are_span_compatible_impl<
-              has_span || std::is_same<std::decay_t<First>, span>::value,
+              std::is_same<std::decay_t<First>, index_span>::value,
+          are_index_span_compatible_impl<
+              has_span || std::is_same<std::decay_t<First>, index_span>::value,
               Args...>,
           std::false_type>::type {};
 
@@ -462,19 +530,20 @@ struct are_span_compatible_impl<has_span, First, Args...>
 
 
 /**
- * Evaluates if at least one type of Args is a gko::span and the others either
- * also gko::span or fulfill std::is_integral
+ * Evaluates if at least one type of Args is a gko::acc::index_span and the
+ * others either also gko::acc::index_span or fulfill std::is_integral
  */
 template <typename... Args>
-using are_span_compatible = detail::are_span_compatible_impl<false, Args...>;
+using are_index_span_compatible =
+    detail::are_index_span_compatible_impl<false, Args...>;
 
 
 namespace detail {
 
 
 template <size_type iter, size_type N, typename Callable, typename... Indices>
-GKO_ATTRIBUTES std::enable_if_t<iter == N> multidim_for_each_impl(
-    const dim<N> &, Callable callable, Indices &&... indices)
+GKO_ACC_ATTRIBUTES std::enable_if_t<iter == N> multidim_for_each_impl(
+    const std::array<size_type, N> &, Callable callable, Indices &&... indices)
 {
     static_assert(iter == sizeof...(Indices),
                   "Number arguments must match current iteration!");
@@ -482,8 +551,9 @@ GKO_ATTRIBUTES std::enable_if_t<iter == N> multidim_for_each_impl(
 }
 
 template <size_type iter, size_type N, typename Callable, typename... Indices>
-GKO_ATTRIBUTES std::enable_if_t<(iter < N)> multidim_for_each_impl(
-    const dim<N> &size, Callable &&callable, Indices &&... indices)
+GKO_ACC_ATTRIBUTES std::enable_if_t<(iter < N)> multidim_for_each_impl(
+    const std::array<size_type, N> &size, Callable &&callable,
+    Indices &&... indices)
 {
     static_assert(iter == sizeof...(Indices),
                   "Number arguments must match current iteration!");
@@ -502,7 +572,8 @@ GKO_ATTRIBUTES std::enable_if_t<(iter < N)> multidim_for_each_impl(
  * source(indices...)
  */
 template <size_type N, typename Callable>
-GKO_ATTRIBUTES void multidim_for_each(const dim<N> &size, Callable &&callable)
+GKO_ACC_ATTRIBUTES void multidim_for_each(const std::array<size_type, N> &size,
+                                          Callable &&callable)
 {
     detail::multidim_for_each_impl<0>(size, std::forward<Callable>(callable));
 }
@@ -512,21 +583,24 @@ namespace detail {
 
 
 template <size_type iter, size_type N>
-constexpr GKO_ATTRIBUTES std::enable_if_t<iter == N, int> spans_in_size(
-    const dim<N> &)
+constexpr GKO_ACC_ATTRIBUTES std::enable_if_t<iter == N, int>
+index_spans_in_size(const std::array<size_type, N> &)
 {
     return 0;
 }
 
 template <size_type iter, size_type N, typename First, typename... Remaining>
-constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), int> spans_in_size(
-    const dim<N> &size, First first, Remaining &&... remaining)
+constexpr GKO_ACC_ATTRIBUTES std::enable_if_t<(iter < N), int>
+index_spans_in_size(const std::array<size_type, N> &size, First first,
+                    Remaining &&... remaining)
 {
     static_assert(sizeof...(Remaining) + 1 == N - iter,
                   "Number of remaining spans must be equal to N - iter");
-    return GKO_ASSERT(span{first}.is_valid()),
-           GKO_ASSERT(span{first} <= span{size[iter]}),
-           spans_in_size<iter + 1>(size, std::forward<Remaining>(remaining)...);
+    // TODO: Figure out why this assert causes errors outside of Ginkgo
+    return GKO_ACC_ASSERT(index_span{first}.is_valid()),
+           GKO_ACC_ASSERT(index_span{first} <= index_span{size[iter]}),
+           index_spans_in_size<iter + 1>(size,
+                                         std::forward<Remaining>(remaining)...);
 }
 
 
@@ -534,10 +608,10 @@ constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), int> spans_in_size(
 
 
 template <size_type N, typename... Spans>
-constexpr GKO_ATTRIBUTES int validate_spans(const dim<N> &size,
-                                            Spans &&... spans)
+constexpr GKO_ACC_ATTRIBUTES int validate_index_spans(
+    const std::array<size_type, N> &size, Spans &&... spans)
 {
-    return detail::spans_in_size<0>(size, std::forward<Spans>(spans)...);
+    return detail::index_spans_in_size<0>(size, std::forward<Spans>(spans)...);
 }
 
 
@@ -571,8 +645,8 @@ constexpr size_type count_mask_dimensionality()
 
 
 }  // namespace helper
-}  // namespace accessor
+}  // namespace acc
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_ACCESSOR_HELPER_HPP_
+#endif  // GKO_ACCESSOR_ACCESSOR_HELPER_HPP_
diff --git a/core/base/accessor_references.hpp b/accessor/accessor_references.hpp
similarity index 73%
rename from core/base/accessor_references.hpp
rename to accessor/accessor_references.hpp
index e392275e1eb..3d9bd1bd136 100644
--- a/core/base/accessor_references.hpp
+++ b/accessor/accessor_references.hpp
@@ -30,17 +30,15 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_ACCESSOR_REFERENCES_HPP_
-#define GKO_CORE_BASE_ACCESSOR_REFERENCES_HPP_
+#ifndef GKO_ACCESSOR_ACCESSOR_REFERENCES_HPP_
+#define GKO_ACCESSOR_ACCESSOR_REFERENCES_HPP_
 
 
 #include <type_traits>
 #include <utility>
 
 
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/std_extensions.hpp>
-#include <ginkgo/core/base/types.hpp>
+#include "accessor/utils.hpp"
 
 
 // CUDA TOOLKIT < 11 does not support constexpr in combination with
@@ -48,17 +46,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #if defined(__CUDA_ARCH__) && defined(__CUDACC_VER_MAJOR__) && \
     (__CUDACC_VER_MAJOR__ < 11)
 
-#define GKO_ENABLE_REFERENCE_CONSTEXPR
+#define GKO_ACC_ENABLE_REFERENCE_CONSTEXPR
 
 #else
 
-#define GKO_ENABLE_REFERENCE_CONSTEXPR constexpr
+#define GKO_ACC_ENABLE_REFERENCE_CONSTEXPR constexpr
 
 #endif  // __CUDA_ARCH__ && __CUDACC_VER_MAJOR__ && __CUDACC_VER_MAJOR__ < 11
 
 
 namespace gko {
-namespace accessor {
+namespace acc {
 /**
  * This namespace is not part of the public interface and can change without
  * notice.
@@ -89,7 +87,7 @@ struct has_cast_operator<
  * overloaded `operator value_type()`.
  */
 template <typename ValueType, typename Ref>
-constexpr GKO_ATTRIBUTES
+constexpr GKO_ACC_ATTRIBUTES
     std::enable_if_t<has_cast_operator<Ref, ValueType>::value, ValueType>
     to_value_type(const Ref &ref)
 {
@@ -97,7 +95,7 @@ constexpr GKO_ATTRIBUTES
 }
 
 template <typename ValueType, typename Ref>
-constexpr GKO_ATTRIBUTES
+constexpr GKO_ACC_ATTRIBUTES
     std::enable_if_t<!has_cast_operator<Ref, ValueType>::value, ValueType>
     to_value_type(const Ref &ref)
 {
@@ -125,76 +123,79 @@ template <typename Reference, typename ArithmeticType>
 struct enable_reference_operators {
     using arithmetic_type = std::remove_cv_t<ArithmeticType>;
 
-#define GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(_op)                  \
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES  \
-        arithmetic_type                                              \
+#define GKO_ACC_REFERENCE_BINARY_OPERATOR_OVERLOAD(_op)              \
+    friend GKO_ACC_ENABLE_REFERENCE_CONSTEXPR GKO_ACC_INLINE         \
+        GKO_ACC_ATTRIBUTES arithmetic_type                           \
         operator _op(const Reference &ref1, const Reference &ref2)   \
     {                                                                \
         return to_value_type<arithmetic_type>(ref1)                  \
             _op to_value_type<arithmetic_type>(ref2);                \
     }                                                                \
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES  \
-        arithmetic_type                                              \
+    friend GKO_ACC_ENABLE_REFERENCE_CONSTEXPR GKO_ACC_INLINE         \
+        GKO_ACC_ATTRIBUTES arithmetic_type                           \
         operator _op(const Reference &ref, const arithmetic_type &a) \
     {                                                                \
         return to_value_type<arithmetic_type>(ref) _op a;            \
     }                                                                \
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES  \
-        arithmetic_type                                              \
+    friend GKO_ACC_ENABLE_REFERENCE_CONSTEXPR GKO_ACC_INLINE         \
+        GKO_ACC_ATTRIBUTES arithmetic_type                           \
         operator _op(const arithmetic_type &a, const Reference &ref) \
     {                                                                \
         return a _op to_value_type<arithmetic_type>(ref);            \
     }
 
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(*)
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(/)
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(+)
-    GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD(-)
-#undef GKO_REFERENCE_BINARY_OPERATOR_OVERLOAD
+    GKO_ACC_REFERENCE_BINARY_OPERATOR_OVERLOAD(*)
+    GKO_ACC_REFERENCE_BINARY_OPERATOR_OVERLOAD(/)
+    GKO_ACC_REFERENCE_BINARY_OPERATOR_OVERLOAD(+)
+    GKO_ACC_REFERENCE_BINARY_OPERATOR_OVERLOAD(-)
+#undef GKO_ACC_REFERENCE_BINARY_OPERATOR_OVERLOAD
 
-#define GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(_oper, _op)             \
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES        \
-        arithmetic_type                                                    \
+#define GKO_ACC_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(_oper, _op)         \
+    friend GKO_ACC_ENABLE_REFERENCE_CONSTEXPR GKO_ACC_INLINE               \
+        GKO_ACC_ATTRIBUTES arithmetic_type                                 \
         _oper(Reference &&ref1, const Reference &ref2)                     \
     {                                                                      \
         return std::move(ref1) = to_value_type<arithmetic_type>(ref1)      \
                    _op to_value_type<arithmetic_type>(ref2);               \
     }                                                                      \
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES        \
-        arithmetic_type                                                    \
+    friend GKO_ACC_ENABLE_REFERENCE_CONSTEXPR GKO_ACC_INLINE               \
+        GKO_ACC_ATTRIBUTES arithmetic_type                                 \
         _oper(Reference &&ref, const arithmetic_type &a)                   \
     {                                                                      \
         return std::move(ref) = to_value_type<arithmetic_type>(ref) _op a; \
     }
 
-    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator*=, *)
-    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator/=, /)
-    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator+=, +)
-    GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator-=, -)
-#undef GKO_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD
-
-#define GKO_REFERENCE_COMPARISON_OPERATOR_OVERLOAD(_op)                  \
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES bool \
-    operator _op(const Reference &ref1, const Reference &ref2)           \
-    {                                                                    \
-        return to_value_type<arithmetic_type>(ref1)                      \
-            _op to_value_type<arithmetic_type>(ref2);                    \
-    }                                                                    \
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES bool \
-    operator _op(const Reference &ref, const arithmetic_type &a)         \
-    {                                                                    \
-        return to_value_type<arithmetic_type>(ref) _op a;                \
-    }                                                                    \
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES bool \
-    operator _op(const arithmetic_type &a, const Reference &ref)         \
-    {                                                                    \
-        return a _op to_value_type<arithmetic_type>(ref);                \
+    GKO_ACC_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator*=, *)
+    GKO_ACC_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator/=, /)
+    GKO_ACC_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator+=, +)
+    GKO_ACC_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD(operator-=, -)
+#undef GKO_ACC_REFERENCE_ASSIGNMENT_OPERATOR_OVERLOAD
+
+#define GKO_ACC_REFERENCE_COMPARISON_OPERATOR_OVERLOAD(_op)          \
+    friend GKO_ACC_ENABLE_REFERENCE_CONSTEXPR GKO_ACC_INLINE         \
+        GKO_ACC_ATTRIBUTES bool                                      \
+        operator _op(const Reference &ref1, const Reference &ref2)   \
+    {                                                                \
+        return to_value_type<arithmetic_type>(ref1)                  \
+            _op to_value_type<arithmetic_type>(ref2);                \
+    }                                                                \
+    friend GKO_ACC_ENABLE_REFERENCE_CONSTEXPR GKO_ACC_INLINE         \
+        GKO_ACC_ATTRIBUTES bool                                      \
+        operator _op(const Reference &ref, const arithmetic_type &a) \
+    {                                                                \
+        return to_value_type<arithmetic_type>(ref) _op a;            \
+    }                                                                \
+    friend GKO_ACC_ENABLE_REFERENCE_CONSTEXPR GKO_ACC_INLINE         \
+        GKO_ACC_ATTRIBUTES bool                                      \
+        operator _op(const arithmetic_type &a, const Reference &ref) \
+    {                                                                \
+        return a _op to_value_type<arithmetic_type>(ref);            \
     }
 
-    GKO_REFERENCE_COMPARISON_OPERATOR_OVERLOAD(==)
-#undef GKO_REFERENCE_COMPARISON_OPERATOR_OVERLOAD
+    GKO_ACC_REFERENCE_COMPARISON_OPERATOR_OVERLOAD(==)
+#undef GKO_ACC_REFERENCE_COMPARISON_OPERATOR_OVERLOAD
 
-    friend GKO_ENABLE_REFERENCE_CONSTEXPR GKO_INLINE GKO_ATTRIBUTES
+    friend GKO_ACC_ENABLE_REFERENCE_CONSTEXPR GKO_ACC_INLINE GKO_ACC_ATTRIBUTES
         arithmetic_type
         operator-(const Reference &ref)
     {
@@ -203,7 +204,7 @@ struct enable_reference_operators {
 };
 
 // There is no more need for this macro in this file
-#undef GKO_ENABLE_REFERENCE_CONSTEXPR
+#undef GKO_ACC_ENABLE_REFERENCE_CONSTEXPR
 
 
 }  // namespace detail
@@ -252,41 +253,41 @@ class reduced_storage
     // Forbid copy construction
     reduced_storage(const reduced_storage &) = delete;
 
-    constexpr explicit GKO_ATTRIBUTES reduced_storage(
-        storage_type *const GKO_RESTRICT ptr)
+    constexpr explicit GKO_ACC_ATTRIBUTES reduced_storage(
+        storage_type *const GKO_ACC_RESTRICT ptr)
         : ptr_{ptr}
     {}
 
-    constexpr GKO_ATTRIBUTES operator arithmetic_type() const
+    constexpr GKO_ACC_ATTRIBUTES operator arithmetic_type() const
     {
-        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        const storage_type *const GKO_ACC_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr);
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type operator=(arithmetic_type val) &&
-        noexcept
+    constexpr GKO_ACC_ATTRIBUTES arithmetic_type
+    operator=(arithmetic_type val) &&noexcept
     {
-        storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        storage_type *const GKO_ACC_RESTRICT r_ptr = ptr_;
         *r_ptr = static_cast<storage_type>(val);
         return val;
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type
+    constexpr GKO_ACC_ATTRIBUTES arithmetic_type
     operator=(const reduced_storage &ref) &&
     {
         std::move(*this) = static_cast<arithmetic_type>(ref);
         return static_cast<arithmetic_type>(*this);
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type operator=(reduced_storage &&ref) &&
-        noexcept
+    constexpr GKO_ACC_ATTRIBUTES arithmetic_type
+    operator=(reduced_storage &&ref) &&noexcept
     {
         std::move(*this) = static_cast<arithmetic_type>(ref);
         return static_cast<arithmetic_type>(*this);
     }
 
 private:
-    storage_type *const GKO_RESTRICT ptr_;
+    storage_type *const GKO_ACC_RESTRICT ptr_;
 };
 
 // Specialization for const storage_type to prevent `operator=`
@@ -310,28 +311,28 @@ class reduced_storage<ArithmeticType, const StorageType>
 
     reduced_storage &operator=(reduced_storage &&) = delete;
 
-    constexpr explicit GKO_ATTRIBUTES reduced_storage(
-        storage_type *const GKO_RESTRICT ptr)
+    constexpr explicit GKO_ACC_ATTRIBUTES reduced_storage(
+        storage_type *const GKO_ACC_RESTRICT ptr)
         : ptr_{ptr}
     {}
 
-    constexpr GKO_ATTRIBUTES operator arithmetic_type() const
+    constexpr GKO_ACC_ATTRIBUTES operator arithmetic_type() const
     {
-        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        const storage_type *const GKO_ACC_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr);
     }
 
 private:
-    storage_type *const GKO_RESTRICT ptr_;
+    storage_type *const GKO_ACC_RESTRICT ptr_;
 };
 
 
 template <typename ArithmeticType, typename StorageType>
-constexpr remove_complex<ArithmeticType> abs(
+constexpr remove_complex_t<ArithmeticType> abs(
     const reduced_storage<ArithmeticType, StorageType> &ref)
 {
     // For some reason necessary to get the correct abs function
-    using gko::abs;
+    using std::abs;
     return abs(static_cast<ArithmeticType>(ref));
 }
 
@@ -373,42 +374,41 @@ class scaled_reduced_storage
     // Forbid copy construction
     scaled_reduced_storage(const scaled_reduced_storage &) = delete;
 
-    constexpr explicit GKO_ATTRIBUTES scaled_reduced_storage(
-        storage_type *const GKO_RESTRICT ptr, arithmetic_type scalar)
+    constexpr explicit GKO_ACC_ATTRIBUTES scaled_reduced_storage(
+        storage_type *const GKO_ACC_RESTRICT ptr, arithmetic_type scalar)
         : ptr_{ptr}, scalar_{scalar}
     {}
 
-    constexpr GKO_ATTRIBUTES operator arithmetic_type() const
+    constexpr GKO_ACC_ATTRIBUTES operator arithmetic_type() const
     {
-        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        const storage_type *const GKO_ACC_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr) * scalar_;
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type operator=(arithmetic_type val) &&
-        noexcept
+    constexpr GKO_ACC_ATTRIBUTES arithmetic_type
+    operator=(arithmetic_type val) &&noexcept
     {
-        storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        storage_type *const GKO_ACC_RESTRICT r_ptr = ptr_;
         *r_ptr = static_cast<storage_type>(val / scalar_);
         return val;
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type
+    constexpr GKO_ACC_ATTRIBUTES arithmetic_type
     operator=(const scaled_reduced_storage &ref) &&
     {
         std::move(*this) = static_cast<arithmetic_type>(ref);
         return static_cast<arithmetic_type>(*this);
     }
 
-    constexpr GKO_ATTRIBUTES arithmetic_type
-        operator=(scaled_reduced_storage &&ref) &&
-        noexcept
+    constexpr GKO_ACC_ATTRIBUTES arithmetic_type
+    operator=(scaled_reduced_storage &&ref) &&noexcept
     {
         std::move(*this) = static_cast<arithmetic_type>(ref);
         return static_cast<arithmetic_type>(*this);
     }
 
 private:
-    storage_type *const GKO_RESTRICT ptr_;
+    storage_type *const GKO_ACC_RESTRICT ptr_;
     const arithmetic_type scalar_;
 };
 
@@ -434,36 +434,36 @@ class scaled_reduced_storage<ArithmeticType, const StorageType>
 
     scaled_reduced_storage &operator=(scaled_reduced_storage &&) = delete;
 
-    constexpr explicit GKO_ATTRIBUTES scaled_reduced_storage(
-        storage_type *const GKO_RESTRICT ptr, arithmetic_type scalar)
+    constexpr explicit GKO_ACC_ATTRIBUTES scaled_reduced_storage(
+        storage_type *const GKO_ACC_RESTRICT ptr, arithmetic_type scalar)
         : ptr_{ptr}, scalar_{scalar}
     {}
 
-    constexpr GKO_ATTRIBUTES operator arithmetic_type() const
+    constexpr GKO_ACC_ATTRIBUTES operator arithmetic_type() const
     {
-        const storage_type *const GKO_RESTRICT r_ptr = ptr_;
+        const storage_type *const GKO_ACC_RESTRICT r_ptr = ptr_;
         return static_cast<arithmetic_type>(*r_ptr) * scalar_;
     }
 
 private:
-    storage_type *const GKO_RESTRICT ptr_;
+    storage_type *const GKO_ACC_RESTRICT ptr_;
     const arithmetic_type scalar_;
 };
 
 
 template <typename ArithmeticType, typename StorageType>
-constexpr remove_complex<ArithmeticType> abs(
+constexpr remove_complex_t<ArithmeticType> abs(
     const scaled_reduced_storage<ArithmeticType, StorageType> &ref)
 {
     // For some reason necessary to get the correct abs function
-    using gko::abs;
+    using std::abs;
     return abs(static_cast<ArithmeticType>(ref));
 }
 
 
 }  // namespace reference_class
-}  // namespace accessor
+}  // namespace acc
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_ACCESSOR_REFERENCES_HPP_
+#endif  // GKO_ACCESSOR_ACCESSOR_REFERENCES_HPP_
diff --git a/accessor/index_span.hpp b/accessor/index_span.hpp
new file mode 100644
index 00000000000..5b71f3cdc4e
--- /dev/null
+++ b/accessor/index_span.hpp
@@ -0,0 +1,100 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_ACCESSOR_INDEX_SPAN_HPP_
+#define GKO_ACCESSOR_INDEX_SPAN_HPP_
+
+#include "accessor/utils.hpp"
+
+
+namespace gko {
+namespace acc {
+
+
+struct index_span {
+    GKO_ACC_ATTRIBUTES constexpr index_span(size_type begin,
+                                            size_type end) noexcept
+        : begin{begin}, end{end}
+    {}
+
+    GKO_ACC_ATTRIBUTES constexpr index_span(size_type point) noexcept
+        : index_span{point, point + 1}
+    {}
+
+    GKO_ACC_ATTRIBUTES constexpr bool is_valid() const { return begin < end; }
+
+    friend GKO_ACC_ATTRIBUTES constexpr bool operator<(const index_span &first,
+                                                       const index_span &second)
+    {
+        return first.end < second.begin;
+    }
+
+    friend GKO_ACC_ATTRIBUTES constexpr bool operator<=(
+        const index_span &first, const index_span &second)
+    {
+        return first.end <= second.begin;
+    }
+
+    friend GKO_ACC_ATTRIBUTES constexpr bool operator>(const index_span &first,
+                                                       const index_span &second)
+    {
+        return second < first;
+    }
+
+    friend GKO_ACC_ATTRIBUTES constexpr bool operator>=(
+        const index_span &first, const index_span &second)
+    {
+        return second <= first;
+    }
+
+    friend GKO_ACC_ATTRIBUTES constexpr bool operator==(
+        const index_span &first, const index_span &second)
+    {
+        return first.begin == second.begin && first.end == second.end;
+    }
+
+    friend GKO_ACC_ATTRIBUTES constexpr bool operator!=(
+        const index_span &first, const index_span &second)
+    {
+        return !(first == second);
+    }
+
+    const size_type begin;
+    const size_type end;
+};
+
+
+}  // namespace acc
+}  // namespace gko
+
+
+#endif  // GKO_ACCESSOR_INDEX_SPAN_HPP_
diff --git a/accessor/range.hpp b/accessor/range.hpp
new file mode 100644
index 00000000000..2b9447acb6a
--- /dev/null
+++ b/accessor/range.hpp
@@ -0,0 +1,144 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_ACCESSOR_RANGE_HPP_
+#define GKO_ACCESSOR_RANGE_HPP_
+
+#include <utility>
+
+
+#include "accessor/utils.hpp"
+
+
+namespace gko {
+namespace acc {
+
+
+template <typename Accessor>
+class range {
+public:
+    /**
+     * The type of the underlying accessor.
+     */
+    using accessor = Accessor;
+
+    /**
+     * The number of dimensions of the range.
+     */
+    static constexpr size_type dimensionality = accessor::dimensionality;
+
+    /**
+     * Use the default destructor.
+     */
+    ~range() = default;
+
+    /**
+     * Creates a new range.
+     *
+     * @tparam AccessorParam  types of parameters forwarded to the accessor
+     *                        constructor
+     *
+     * @param params  parameters forwarded to Accessor constructor.
+     */
+    template <typename... AccessorParams>
+    GKO_ACC_ATTRIBUTES constexpr explicit range(AccessorParams &&... params)
+        : accessor_{std::forward<AccessorParams>(params)...}
+    {}
+
+    /**
+     * Returns a value (or a sub-range) with the specified indexes.
+     *
+     * @tparam DimensionTypes  The types of indexes. Supported types depend on
+     *                         the underlying accessor, but are usually either
+     *                         integer types or spans. If at least one index is
+     *                         a span, the returned value will be a sub-range.
+     *
+     * @param dimensions  the indexes of the values.
+     *
+     * @return a value on position `(dimensions...)`.
+     */
+    template <typename... DimensionTypes>
+    GKO_ACC_ATTRIBUTES constexpr auto operator()(
+        DimensionTypes &&... dimensions) const
+        -> decltype(std::declval<accessor>()(
+            std::forward<DimensionTypes>(dimensions)...))
+    {
+        static_assert(sizeof...(dimensions) <= dimensionality,
+                      "Too many dimensions in range call");
+        return accessor_(std::forward<DimensionTypes>(dimensions)...);
+    }
+
+    range(const range &other) = default;
+
+    /**
+     * Returns the length of the specified dimension of the range.
+     *
+     * @param dimension  the dimensions whose length is returned
+     *
+     * @return  the length of the `dimension`-th dimension of the range
+     */
+    GKO_ACC_ATTRIBUTES constexpr size_type length(size_type dimension) const
+    {
+        return accessor_.length(dimension);
+    }
+
+    /**
+     * Returns a pointer to the accessor.
+     *
+     * Can be used to access data and functions of a specific accessor.
+     *
+     * @return pointer to the accessor
+     */
+    GKO_ACC_ATTRIBUTES constexpr const accessor *operator->() const noexcept
+    {
+        return &accessor_;
+    }
+
+    /**
+     * `Returns a reference to the accessor.
+     *
+     * @return reference to the accessor
+     */
+    GKO_ACC_ATTRIBUTES constexpr const accessor &get_accessor() const noexcept
+    {
+        return accessor_;
+    }
+
+private:
+    accessor accessor_;
+};
+
+
+}  // namespace acc
+}  // namespace gko
+
+#endif  // GKO_ACCESSOR_RANGE_HPP_
diff --git a/accessor/reduced_row_major.hpp b/accessor/reduced_row_major.hpp
new file mode 100644
index 00000000000..db31e1777fe
--- /dev/null
+++ b/accessor/reduced_row_major.hpp
@@ -0,0 +1,299 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_ACCESSOR_REDUCED_ROW_MAJOR_HPP_
+#define GKO_ACCESSOR_REDUCED_ROW_MAJOR_HPP_
+
+
+#include <array>
+#include <memory>
+#include <type_traits>
+#include <utility>
+
+
+#include "accessor/accessor_helper.hpp"
+#include "accessor/accessor_references.hpp"
+#include "accessor/index_span.hpp"
+#include "accessor/range.hpp"
+#include "accessor/utils.hpp"
+
+
+namespace gko {
+/**
+ * @brief The accessor namespace.
+ *
+ * @ingroup accessor
+ */
+namespace acc {
+
+
+/**
+ * The reduced_row_major class allows a storage format that is different from
+ * the arithmetic format (which is returned from the brace operator).
+ * As storage, the StorageType is used.
+ *
+ * This accessor uses row-major access. For example for three dimensions,
+ * neighboring z coordinates are next to each other in memory, followed by y
+ * coordinates and then x coordinates.
+ *
+ * @tparam Dimensionality  The number of dimensions managed by this accessor
+ *
+ * @tparam ArithmeticType  Value type used for arithmetic operations and
+ *                         for in- and output
+ *
+ * @tparam StorageType  Value type used for storing the actual value to memory
+ *
+ * @note  This class only manages the accesses and not the memory itself.
+ */
+template <int Dimensionality, typename ArithmeticType, typename StorageType>
+class reduced_row_major {
+public:
+    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
+    using storage_type = StorageType;
+    static constexpr size_type dimensionality{Dimensionality};
+    static constexpr bool is_const{std::is_const<storage_type>::value};
+    using const_accessor =
+        reduced_row_major<dimensionality, arithmetic_type, const storage_type>;
+
+    static_assert(Dimensionality >= 1,
+                  "Dimensionality must be a positive number!");
+
+    friend class range<reduced_row_major>;
+
+protected:
+    using storage_stride_type = std::array<size_type, dimensionality - 1>;
+    using reference_type =
+        reference_class::reduced_storage<arithmetic_type, storage_type>;
+
+    /**
+     * Creates the accessor for an already allocated storage space with a
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
+     *
+     * @param size  multidimensional size of the memory
+     * @param storage  pointer to the block of memory containing the storage
+     * @param stride  stride array used for memory accesses
+     */
+    constexpr GKO_ACC_ATTRIBUTES reduced_row_major(
+        std::array<size_type, dimensionality> size, storage_type *storage,
+        storage_stride_type stride)
+        : size_(size), storage_{storage}, stride_(stride)
+    {}
+
+    /**
+     * Creates the accessor for an already allocated storage space with a
+     * stride. The first stride is used for computing the index for the first
+     * index, the second stride for the second index, and so on.
+     *
+     * @param storage  pointer to the block of memory containing the storage
+     * @param size  multidimensional size of the memory
+     * @param strides  strides used for memory accesses
+     */
+    template <typename... Strides>
+    constexpr GKO_ACC_ATTRIBUTES reduced_row_major(
+        std::array<size_type, dimensionality> size, storage_type *storage,
+        Strides &&... strides)
+        : reduced_row_major{
+              size, storage,
+              storage_stride_type{{std::forward<Strides>(strides)...}}}
+    {
+        static_assert(sizeof...(Strides) + 1 == dimensionality,
+                      "Number of provided Strides must be dimensionality - 1!");
+    }
+
+    /**
+     * Creates the accessor for an already allocated storage space.
+     * It is assumed that all accesses are without padding.
+     *
+     * @param storage  pointer to the block of memory containing the storage
+     * @param size  multidimensional size of the memory
+     */
+    constexpr GKO_ACC_ATTRIBUTES reduced_row_major(
+        std::array<size_type, dimensionality> size, storage_type *storage)
+        : reduced_row_major{
+              size, storage,
+              helper::compute_default_stride_array<size_type>(size)}
+    {}
+
+    /**
+     * Creates an empty accessor (pointing nowhere with an empty size)
+     */
+    constexpr GKO_ACC_ATTRIBUTES reduced_row_major()
+        : reduced_row_major{{0, 0, 0}, nullptr}
+    {}
+
+public:
+    /**
+     * Creates a reduced_row_major range which contains a read-only version of
+     * the current accessor.
+     *
+     * @returns  a reduced_row_major major range which is read-only.
+     */
+    constexpr GKO_ACC_ATTRIBUTES range<const_accessor> to_const() const
+    {
+        return range<const_accessor>{size_, storage_, stride_};
+    }
+
+    /**
+     * Returns the length in dimension `dimension`.
+     *
+     * @param dimension  a dimension index
+     *
+     * @returns  length in dimension `dimension`
+     */
+    constexpr GKO_ACC_ATTRIBUTES size_type length(size_type dimension) const
+    {
+        return dimension < dimensionality ? size_[dimension] : 1;
+    }
+
+    /**
+     * Copies data from another accessor
+     *
+     * @warning Do not use this function since it is not optimized for a
+     *          specific executor. It will always be performed sequentially.
+     *          Please write an optimized version (adjusted to the architecture)
+     *          by iterating through the values yourself.
+     *
+     * @tparam OtherAccessor  type of the other accessor
+     *
+     * @param other  other accessor
+     */
+    template <typename OtherAccessor>
+    GKO_ACC_ATTRIBUTES void copy_from(const OtherAccessor &other) const
+    {
+        helper::multidim_for_each(size_, [this, &other](auto... indices) {
+            (*this)(indices...) = other(indices...);
+        });
+    }
+
+    /**
+     * Returns the stored value for the given indices. If the storage is const,
+     * a value is returned, otherwise, a reference is returned.
+     *
+     * @param indices  indices which value is supposed to access
+     *
+     * @returns  the stored value if the accessor is const (if the storage type
+     *           is const), or a reference if the accessor is non-const
+     */
+    template <typename... Indices>
+    constexpr GKO_ACC_ATTRIBUTES std::enable_if_t<
+        are_all_integral<Indices...>::value,
+        std::conditional_t<is_const, arithmetic_type, reference_type>>
+    operator()(Indices &&... indices) const
+    {
+        return reference_type{storage_ +
+                              compute_index(std::forward<Indices>(indices)...)};
+    }
+
+    /**
+     * Returns a sub-range spanning the current range (x1_span, x2_span, ...)
+     *
+     * @param spans  span for the indices
+     *
+     * @returns a sub-range for the given spans.
+     */
+    template <typename... SpanTypes>
+    constexpr GKO_ACC_ATTRIBUTES
+        std::enable_if_t<helper::are_index_span_compatible<SpanTypes...>::value,
+                         range<reduced_row_major>>
+        operator()(SpanTypes... spans) const
+    {
+        return helper::validate_index_spans(size_, spans...),
+               range<reduced_row_major>{
+                   std::array<size_type, dimensionality>{
+                       (index_span{spans}.end - index_span{spans}.begin)...},
+                   storage_ + compute_index((index_span{spans}.begin)...),
+                   stride_};
+    }
+
+    /**
+     * Returns the size of the accessor
+     *
+     * @returns the size of the accessor
+     */
+    constexpr GKO_ACC_ATTRIBUTES std::array<size_type, dimensionality>
+    get_size() const
+    {
+        return size_;
+    }
+
+    /**
+     * Returns a pointer to a stride array of size dimensionality - 1
+     *
+     * @returns returns a pointer to a stride array of size dimensionality - 1
+     */
+    GKO_ACC_ATTRIBUTES
+    constexpr const storage_stride_type &get_stride() const { return stride_; }
+
+    /**
+     * Returns the pointer to the storage data
+     *
+     * @returns the pointer to the storage data
+     */
+    constexpr GKO_ACC_ATTRIBUTES storage_type *get_stored_data() const
+    {
+        return storage_;
+    }
+
+    /**
+     * Returns a const pointer to the storage data
+     *
+     * @returns a const pointer to the storage data
+     */
+    constexpr GKO_ACC_ATTRIBUTES const storage_type *get_const_storage() const
+    {
+        return storage_;
+    }
+
+protected:
+    template <typename... Indices>
+    constexpr GKO_ACC_ATTRIBUTES size_type
+    compute_index(Indices &&... indices) const
+    {
+        static_assert(sizeof...(Indices) == dimensionality,
+                      "Number of indices must match dimensionality!");
+        return helper::compute_storage_index<size_type, dimensionality>(
+            size_, stride_, std::forward<Indices>(indices)...);
+    }
+
+private:
+    const std::array<size_type, dimensionality> size_;
+    storage_type *storage_;
+    const storage_stride_type stride_;
+};
+
+
+}  // namespace acc
+}  // namespace gko
+
+
+#endif  // GKO_ACCESSOR_REDUCED_ROW_MAJOR_HPP_
diff --git a/core/base/accessors.hpp b/accessor/scaled_reduced_row_major.hpp
similarity index 58%
rename from core/base/accessors.hpp
rename to accessor/scaled_reduced_row_major.hpp
index fb1b1700c0b..93546a01909 100644
--- a/core/base/accessors.hpp
+++ b/accessor/scaled_reduced_row_major.hpp
@@ -30,25 +30,20 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_ACCESSORS_HPP_
-#define GKO_CORE_BASE_ACCESSORS_HPP_
+#ifndef GKO_ACCESSOR_SCALED_REDUCED_ROW_MAJOR_HPP_
+#define GKO_ACCESSOR_SCALED_REDUCED_ROW_MAJOR_HPP_
 
 
 #include <array>
-#include <memory>
 #include <type_traits>
 #include <utility>
 
 
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/dim.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/range.hpp>
-#include <ginkgo/core/base/types.hpp>
-
-
-#include "core/base/accessor_helper.hpp"
-#include "core/base/accessor_references.hpp"
+#include "accessor/accessor_helper.hpp"
+#include "accessor/accessor_references.hpp"
+#include "accessor/index_span.hpp"
+#include "accessor/range.hpp"
+#include "accessor/utils.hpp"
 
 
 namespace gko {
@@ -57,240 +52,7 @@ namespace gko {
  *
  * @ingroup accessor
  */
-namespace accessor {
-
-
-/**
- * The reduced_row_major class allows a storage format that is different from
- * the arithmetic format (which is returned from the brace operator).
- * As storage, the StorageType is used.
- *
- * This accessor uses row-major access. For example for three dimensions,
- * neighboring z coordinates are next to each other in memory, followed by y
- * coordinates and then x coordinates.
- *
- * @tparam Dimensionality  The number of dimensions managed by this accessor
- *
- * @tparam ArithmeticType  Value type used for arithmetic operations and
- *                         for in- and output
- *
- * @tparam StorageType  Value type used for storing the actual value to memory
- *
- * @note  This class only manages the accesses and not the memory itself.
- */
-template <int Dimensionality, typename ArithmeticType, typename StorageType>
-class reduced_row_major {
-public:
-    using arithmetic_type = std::remove_cv_t<ArithmeticType>;
-    using storage_type = StorageType;
-    static constexpr size_type dimensionality{Dimensionality};
-    static constexpr bool is_const{std::is_const<storage_type>::value};
-    using const_accessor =
-        reduced_row_major<dimensionality, arithmetic_type, const storage_type>;
-
-    static_assert(Dimensionality >= 1,
-                  "Dimensionality must be a positive number!");
-
-    friend class range<reduced_row_major>;
-
-protected:
-    using storage_stride_type = std::array<size_type, dimensionality - 1>;
-    using reference_type =
-        reference_class::reduced_storage<arithmetic_type, storage_type>;
-
-    /**
-     * Creates the accessor for an already allocated storage space with a
-     * stride. The first stride is used for computing the index for the first
-     * index, the second stride for the second index, and so on.
-     *
-     * @param size  multidimensional size of the memory
-     * @param storage  pointer to the block of memory containing the storage
-     * @param stride  stride array used for memory accesses
-     */
-    constexpr GKO_ATTRIBUTES reduced_row_major(gko::dim<dimensionality> size,
-                                               storage_type *storage,
-                                               storage_stride_type stride)
-        : size_{size}, storage_{storage}, stride_(stride)
-    {}
-
-    /**
-     * Creates the accessor for an already allocated storage space with a
-     * stride. The first stride is used for computing the index for the first
-     * index, the second stride for the second index, and so on.
-     *
-     * @param storage  pointer to the block of memory containing the storage
-     * @param size  multidimensional size of the memory
-     * @param strides  strides used for memory accesses
-     */
-    template <typename... Strides>
-    constexpr GKO_ATTRIBUTES reduced_row_major(gko::dim<dimensionality> size,
-                                               storage_type *storage,
-                                               Strides &&... strides)
-        : reduced_row_major{
-              size, storage,
-              storage_stride_type{{std::forward<Strides>(strides)...}}}
-    {
-        static_assert(sizeof...(Strides) + 1 == dimensionality,
-                      "Number of provided Strides must be dimensionality - 1!");
-    }
-
-    /**
-     * Creates the accessor for an already allocated storage space.
-     * It is assumed that all accesses are without padding.
-     *
-     * @param storage  pointer to the block of memory containing the storage
-     * @param size  multidimensional size of the memory
-     */
-    constexpr GKO_ATTRIBUTES reduced_row_major(dim<dimensionality> size,
-                                               storage_type *storage)
-        : reduced_row_major{size, storage,
-                            compute_default_stride_array<size_type>(size)}
-    {}
-
-    /**
-     * Creates an empty accessor (pointing nowhere with an empty size)
-     */
-    constexpr GKO_ATTRIBUTES reduced_row_major()
-        : reduced_row_major{{0, 0, 0}, nullptr}
-    {}
-
-public:
-    /**
-     * Creates a reduced_row_major range which contains a read-only version of
-     * the current accessor.
-     *
-     * @returns  a reduced_row_major major range which is read-only.
-     */
-    constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
-    {
-        return range<const_accessor>{size_, storage_, stride_};
-    }
-
-    /**
-     * Returns the length in dimension `dimension`.
-     *
-     * @param dimension  a dimension index
-     *
-     * @returns  length in dimension `dimension`
-     */
-    constexpr GKO_ATTRIBUTES size_type length(size_type dimension) const
-    {
-        return dimension < dimensionality ? size_[dimension] : 1;
-    }
-
-    /**
-     * Copies data from another accessor
-     *
-     * @warning Do not use this function since it is not optimized for a
-     *          specific executor. It will always be performed sequentially.
-     *          Please write an optimized version (adjusted to the architecture)
-     *          by iterating through the values yourself.
-     *
-     * @tparam OtherAccessor  type of the other accessor
-     *
-     * @param other  other accessor
-     */
-    template <typename OtherAccessor>
-    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
-    {
-        helper::multidim_for_each(size_, [this, &other](auto... indices) {
-            (*this)(indices...) = other(indices...);
-        });
-    }
-
-    /**
-     * Returns the stored value for the given indices. If the storage is const,
-     * a value is returned, otherwise, a reference is returned.
-     *
-     * @param indices  indices which value is supposed to access
-     *
-     * @returns  the stored value if the accessor is const (if the storage type
-     *           is const), or a reference if the accessor is non-const
-     */
-    template <typename... Indices>
-    constexpr GKO_ATTRIBUTES std::enable_if_t<
-        are_all_integral<Indices...>::value,
-        std::conditional_t<is_const, arithmetic_type, reference_type>>
-    operator()(Indices &&... indices) const
-    {
-        return reference_type{storage_ +
-                              compute_index(std::forward<Indices>(indices)...)};
-    }
-
-    /**
-     * Returns a sub-range spanning the current range (x1_span, x2_span, ...)
-     *
-     * @param spans  span for the indices
-     *
-     * @returns a sub-range for the given spans.
-     */
-    template <typename... SpanTypes>
-    constexpr GKO_ATTRIBUTES
-        std::enable_if_t<helper::are_span_compatible<SpanTypes...>::value,
-                         range<reduced_row_major>>
-        operator()(SpanTypes... spans) const
-    {
-        return helper::validate_spans(size_, spans...),
-               range<reduced_row_major>{
-                   dim<dimensionality>{
-                       (span{spans}.end - span{spans}.begin)...},
-                   storage_ + compute_index((span{spans}.begin)...), stride_};
-    }
-
-    /**
-     * Returns the size of the accessor
-     *
-     * @returns the size of the accessor
-     */
-    constexpr GKO_ATTRIBUTES dim<dimensionality> get_size() const
-    {
-        return size_;
-    }
-
-    /**
-     * Returns a pointer to a stride array of size dimensionality - 1
-     *
-     * @returns returns a pointer to a stride array of size dimensionality - 1
-     */
-    GKO_ATTRIBUTES
-    constexpr const storage_stride_type &get_stride() const { return stride_; }
-
-    /**
-     * Returns the pointer to the storage data
-     *
-     * @returns the pointer to the storage data
-     */
-    constexpr GKO_ATTRIBUTES storage_type *get_stored_data() const
-    {
-        return storage_;
-    }
-
-    /**
-     * Returns a const pointer to the storage data
-     *
-     * @returns a const pointer to the storage data
-     */
-    constexpr GKO_ATTRIBUTES const storage_type *get_const_storage() const
-    {
-        return storage_;
-    }
-
-protected:
-    template <typename... Indices>
-    constexpr GKO_ATTRIBUTES size_type
-    compute_index(Indices &&... indices) const
-    {
-        static_assert(sizeof...(Indices) == dimensionality,
-                      "Number of indices must match dimensionality!");
-        return helper::compute_storage_index<size_type, dimensionality>(
-            size_, stride_, std::forward<Indices>(indices)...);
-    }
-
-private:
-    const dim<dimensionality> size_;
-    storage_type *storage_;
-    const storage_stride_type stride_;
-};
+namespace acc {
 
 
 namespace detail {
@@ -322,12 +84,12 @@ struct enable_write_scalar<Dimensionality, Accessor, ScalarType, false> {
      * @returns the written value.
      */
     template <typename... Indices>
-    constexpr GKO_ATTRIBUTES scalar_type
+    constexpr GKO_ACC_ATTRIBUTES scalar_type
     write_scalar_masked(scalar_type value, Indices &&... indices) const
     {
         static_assert(sizeof...(Indices) == Dimensionality,
                       "Number of indices must match dimensionality!");
-        scalar_type *GKO_RESTRICT rest_scalar = self()->scalar_;
+        scalar_type *GKO_ACC_RESTRICT rest_scalar = self()->scalar_;
         return rest_scalar[self()->compute_mask_scalar_index(
                    std::forward<Indices>(indices)...)] = value;
     }
@@ -344,17 +106,17 @@ struct enable_write_scalar<Dimensionality, Accessor, ScalarType, false> {
      * @returns the written value.
      */
     template <typename... Indices>
-    constexpr GKO_ATTRIBUTES scalar_type
+    constexpr GKO_ACC_ATTRIBUTES scalar_type
     write_scalar_direct(scalar_type value, Indices &&... indices) const
     {
-        scalar_type *GKO_RESTRICT rest_scalar = self()->scalar_;
+        scalar_type *GKO_ACC_RESTRICT rest_scalar = self()->scalar_;
         return rest_scalar[self()->compute_direct_scalar_index(
                    std::forward<Indices>(indices)...)] = value;
     }
 
 
 private:
-    constexpr GKO_ATTRIBUTES const Accessor *self() const
+    constexpr GKO_ACC_ATTRIBUTES const Accessor *self() const
     {
         return static_cast<const Accessor *>(this);
     }
@@ -448,11 +210,11 @@ class scaled_reduced_row_major
      *                values.
      * @param scalar_stride  stride array used for memory accesses to scalar
      */
-    constexpr GKO_ATTRIBUTES scaled_reduced_row_major(
-        dim<dimensionality> size, storage_type *storage,
+    constexpr GKO_ACC_ATTRIBUTES scaled_reduced_row_major(
+        std::array<size_type, dimensionality> size, storage_type *storage,
         storage_stride_type storage_stride, scalar_type *scalar,
         scalar_stride_type scalar_stride)
-        : size_{size},
+        : size_(size),
           storage_{storage},
           storage_stride_(storage_stride),
           scalar_{scalar},
@@ -470,8 +232,8 @@ class scaled_reduced_row_major
      * @param scalar  pointer to the block of memory containing the scalar
      *                values.
      */
-    constexpr GKO_ATTRIBUTES scaled_reduced_row_major(
-        dim<dimensionality> size, storage_type *storage,
+    constexpr GKO_ACC_ATTRIBUTES scaled_reduced_row_major(
+        std::array<size_type, dimensionality> size, storage_type *storage,
         storage_stride_type stride, scalar_type *scalar)
         : scaled_reduced_row_major{
               size, storage, stride, scalar,
@@ -489,18 +251,18 @@ class scaled_reduced_row_major
      * @param scalar  pointer to the block of memory containing the scalar
      *                values.
      */
-    constexpr GKO_ATTRIBUTES scaled_reduced_row_major(dim<dimensionality> size,
-                                                      storage_type *storage,
-                                                      scalar_type *scalar)
+    constexpr GKO_ACC_ATTRIBUTES scaled_reduced_row_major(
+        std::array<size_type, dimensionality> size, storage_type *storage,
+        scalar_type *scalar)
         : scaled_reduced_row_major{
-              size, storage, compute_default_stride_array<size_type>(size),
-              scalar}
+              size, storage,
+              helper::compute_default_stride_array<size_type>(size), scalar}
     {}
 
     /**
      * Creates an empty accessor (pointing nowhere with an empty size)
      */
-    constexpr GKO_ATTRIBUTES scaled_reduced_row_major()
+    constexpr GKO_ACC_ATTRIBUTES scaled_reduced_row_major()
         : scaled_reduced_row_major{{0, 0, 0}, nullptr, nullptr}
     {}
 
@@ -511,7 +273,7 @@ class scaled_reduced_row_major
      *
      * @returns  a scaled_reduced_row_major major range which is read-only.
      */
-    constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
+    constexpr GKO_ACC_ATTRIBUTES range<const_accessor> to_const() const
     {
         return range<const_accessor>{size_, storage_, storage_stride_, scalar_,
                                      scalar_stride_};
@@ -526,10 +288,10 @@ class scaled_reduced_row_major
      * @returns the scalar value at the given indices.
      */
     template <typename... Indices>
-    constexpr GKO_ATTRIBUTES scalar_type
+    constexpr GKO_ACC_ATTRIBUTES scalar_type
     read_scalar_masked(Indices &&... indices) const
     {
-        const arithmetic_type *GKO_RESTRICT rest_scalar = scalar_;
+        const arithmetic_type *GKO_ACC_RESTRICT rest_scalar = scalar_;
         return rest_scalar[compute_mask_scalar_index(
             std::forward<Indices>(indices)...)];
     }
@@ -544,10 +306,10 @@ class scaled_reduced_row_major
      * @returns the scalar value at the given indices.
      */
     template <typename... Indices>
-    constexpr GKO_ATTRIBUTES scalar_type
+    constexpr GKO_ACC_ATTRIBUTES scalar_type
     read_scalar_direct(Indices &&... indices) const
     {
-        const arithmetic_type *GKO_RESTRICT rest_scalar = scalar_;
+        const arithmetic_type *GKO_ACC_RESTRICT rest_scalar = scalar_;
         return rest_scalar[compute_direct_scalar_index(
             std::forward<Indices>(indices)...)];
     }
@@ -559,7 +321,7 @@ class scaled_reduced_row_major
      *
      * @returns length in dimension `dimension`
      */
-    constexpr GKO_ATTRIBUTES size_type length(size_type dimension) const
+    constexpr GKO_ACC_ATTRIBUTES size_type length(size_type dimension) const
     {
         return dimension < dimensionality ? size_[dimension] : 1;
     }
@@ -577,7 +339,7 @@ class scaled_reduced_row_major
      * @param other  other accessor
      */
     template <typename OtherAccessor>
-    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
+    GKO_ACC_ATTRIBUTES void copy_from(const OtherAccessor &other) const
     {
         helper::multidim_for_each(size_, [this, &other](auto... indices) {
             // especially inefficient if mask has not all bits set
@@ -597,7 +359,7 @@ class scaled_reduced_row_major
      *           is const), or a reference if the accessor is non-const
      */
     template <typename... Indices>
-    constexpr GKO_ATTRIBUTES std::enable_if_t<
+    constexpr GKO_ACC_ATTRIBUTES std::enable_if_t<
         are_all_integral<Indices...>::value,
         std::conditional_t<is_const, arithmetic_type, reference_type>>
     operator()(Indices... indices) const
@@ -614,18 +376,19 @@ class scaled_reduced_row_major
      * @returns a sub-range for the given spans.
      */
     template <typename... SpanTypes>
-    constexpr GKO_ATTRIBUTES
-        std::enable_if_t<helper::are_span_compatible<SpanTypes...>::value,
+    constexpr GKO_ACC_ATTRIBUTES
+        std::enable_if_t<helper::are_index_span_compatible<SpanTypes...>::value,
                          range<scaled_reduced_row_major>>
         operator()(SpanTypes... spans) const
     {
-        return helper::validate_spans(size_, spans...),
+        return helper::validate_index_spans(size_, spans...),
                range<scaled_reduced_row_major>{
-                   dim<dimensionality>{
-                       (span{spans}.end - span{spans}.begin)...},
-                   storage_ + compute_index((span{spans}.begin)...),
+                   std::array<size_type, dimensionality>{
+                       (index_span{spans}.end - index_span{spans}.begin)...},
+                   storage_ + compute_index((index_span{spans}.begin)...),
                    storage_stride_,
-                   scalar_ + compute_mask_scalar_index(span{spans}.begin...),
+                   scalar_ +
+                       compute_mask_scalar_index(index_span{spans}.begin...),
                    scalar_stride_};
     }
 
@@ -634,7 +397,8 @@ class scaled_reduced_row_major
      *
      * @returns the size of the accessor
      */
-    constexpr GKO_ATTRIBUTES dim<dimensionality> get_size() const
+    constexpr GKO_ACC_ATTRIBUTES std::array<size_type, dimensionality>
+    get_size() const
     {
         return size_;
     }
@@ -646,7 +410,7 @@ class scaled_reduced_row_major
      * @returns a const reference to the storage stride array of size
      *          dimensionality - 1
      */
-    constexpr GKO_ATTRIBUTES const storage_stride_type &get_storage_stride()
+    constexpr GKO_ACC_ATTRIBUTES const storage_stride_type &get_storage_stride()
         const
     {
         return storage_stride_;
@@ -657,7 +421,8 @@ class scaled_reduced_row_major
      *
      * @returns a const reference to the scalar stride array
      */
-    constexpr GKO_ATTRIBUTES const scalar_stride_type &get_scalar_stride() const
+    constexpr GKO_ACC_ATTRIBUTES const scalar_stride_type &get_scalar_stride()
+        const
     {
         return scalar_stride_;
     }
@@ -667,7 +432,7 @@ class scaled_reduced_row_major
      *
      * @returns the pointer to the storage data
      */
-    constexpr GKO_ATTRIBUTES storage_type *get_stored_data() const
+    constexpr GKO_ACC_ATTRIBUTES storage_type *get_stored_data() const
     {
         return storage_;
     }
@@ -677,7 +442,7 @@ class scaled_reduced_row_major
      *
      * @returns a const pointer to the storage data
      */
-    constexpr GKO_ATTRIBUTES const storage_type *get_const_storage() const
+    constexpr GKO_ACC_ATTRIBUTES const storage_type *get_const_storage() const
     {
         return storage_;
     }
@@ -687,21 +452,24 @@ class scaled_reduced_row_major
      *
      * @returns the pointer to the scalar data
      */
-    constexpr GKO_ATTRIBUTES scalar_type *get_scalar() const { return scalar_; }
+    constexpr GKO_ACC_ATTRIBUTES scalar_type *get_scalar() const
+    {
+        return scalar_;
+    }
 
     /**
      * Returns a const pointer to the scalar data
      *
      * @returns a const pointer to the scalar data
      */
-    constexpr GKO_ATTRIBUTES const scalar_type *get_const_scalar() const
+    constexpr GKO_ACC_ATTRIBUTES const scalar_type *get_const_scalar() const
     {
         return scalar_;
     }
 
 protected:
     template <typename... Indices>
-    constexpr GKO_ATTRIBUTES size_type
+    constexpr GKO_ACC_ATTRIBUTES size_type
     compute_index(Indices &&... indices) const
     {
         static_assert(sizeof...(Indices) == dimensionality,
@@ -711,7 +479,7 @@ class scaled_reduced_row_major
     }
 
     template <typename... Indices>
-    constexpr GKO_ATTRIBUTES size_type
+    constexpr GKO_ACC_ATTRIBUTES size_type
     compute_mask_scalar_index(Indices &&... indices) const
     {
         static_assert(sizeof...(Indices) == dimensionality,
@@ -722,7 +490,7 @@ class scaled_reduced_row_major
     }
 
     template <typename... Indices>
-    constexpr GKO_ATTRIBUTES size_type
+    constexpr GKO_ACC_ATTRIBUTES size_type
     compute_direct_scalar_index(Indices &&... indices) const
     {
         static_assert(
@@ -735,7 +503,7 @@ class scaled_reduced_row_major
 
 
 private:
-    const dim<dimensionality> size_;
+    const std::array<size_type, dimensionality> size_;
     storage_type *storage_;
     const storage_stride_type storage_stride_;
     scalar_type *scalar_;
@@ -743,8 +511,8 @@ class scaled_reduced_row_major
 };
 
 
-}  // namespace accessor
+}  // namespace acc
 }  // namespace gko
 
 
-#endif  // GKO_CORE_BASE_ACCESSORS_HPP_
+#endif  // GKO_ACCESSOR_SCALED_REDUCED_ROW_MAJOR_HPP_
diff --git a/accessor/utils.hpp b/accessor/utils.hpp
new file mode 100644
index 00000000000..4ce433bf259
--- /dev/null
+++ b/accessor/utils.hpp
@@ -0,0 +1,150 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_ACCESSOR_UTILS_HPP_
+#define GKO_ACCESSOR_UTILS_HPP_
+
+#include <cassert>
+#include <complex>
+#include <cstddef>  // for std::size_t
+
+
+#if defined(__CUDACC__) || defined(__HIPCC__)
+#include <thrust/complex.h>
+#endif
+
+
+#if defined(__CUDACC__) || defined(__HIPCC__)
+#define GKO_ACC_ATTRIBUTES __host__ __device__
+#define GKO_ACC_INLINE __forceinline__
+#define GKO_ACC_RESTRICT __restrict__
+#else
+#define GKO_ACC_ATTRIBUTES
+#define GKO_ACC_INLINE inline
+#define GKO_ACC_RESTRICT
+#endif  // defined(__CUDACC__) || defined(__HIPCC__)
+
+
+#if (defined(__CUDA_ARCH__) && defined(__APPLE__)) || \
+    defined(__HIP_DEVICE_COMPILE__)
+
+#ifdef NDEBUG
+#define GKO_ACC_ASSERT(condition) ((void)0)
+#else  // NDEBUG
+// Poor man's assertions on GPUs for MACs. They won't terminate the program
+// but will at least print something on the screen
+#define GKO_ACC_ASSERT(condition)                                           \
+    ((condition)                                                            \
+         ? ((void)0)                                                        \
+         : ((void)printf("%s: %d: %s: Assertion `" #condition "' failed\n", \
+                         __FILE__, __LINE__, __func__)))
+#endif  // NDEBUG
+
+#else  // (defined(__CUDA_ARCH__) && defined(__APPLE__)) ||
+       // defined(__HIP_DEVICE_COMPILE__)
+
+// Handle assertions normally on other systems
+#define GKO_ACC_ASSERT(condition) assert(condition)
+
+#endif  // (defined(__CUDA_ARCH__) && defined(__APPLE__)) ||
+        // defined(__HIP_DEVICE_COMPILE__)
+
+
+namespace gko {
+namespace acc {
+
+namespace xstd {
+
+
+template <typename...>
+using void_t = void;
+
+
+}
+
+
+using size_type = std::size_t;
+
+
+namespace detail {
+
+
+template <typename T>
+struct remove_complex_impl {
+    using type = T;
+};
+
+
+template <typename T>
+struct remove_complex_impl<std::complex<T>> {
+    using type = T;
+};
+
+
+#if defined(__CUDACC__) || defined(__HIPCC__)
+template <typename T>
+struct remove_complex_impl<thrust::complex<T>> {
+    using type = T;
+};
+#endif
+
+
+}  // namespace detail
+
+
+template <typename T>
+using remove_complex_t = typename detail::remove_complex_impl<T>::type;
+
+
+/**
+ * Evaluates if all template arguments Args fulfill std::is_integral. If that is
+ * the case, this class inherits from `std::true_type`, otherwise, it inherits
+ * from `std::false_type`.
+ * If no values are passed in, `std::true_type` is inherited from.
+ *
+ * @tparam Args...  Arguments to test for std::is_integral
+ */
+template <typename... Args>
+struct are_all_integral : public std::true_type {};
+
+template <typename First, typename... Args>
+struct are_all_integral<First, Args...>
+    : public std::conditional<std::is_integral<std::decay_t<First>>::value,
+                              are_all_integral<Args...>,
+                              std::false_type>::type {};
+
+
+}  // namespace acc
+}  // namespace gko
+
+
+#endif  // GKO_ACCESSOR_UTILS_HPP_
diff --git a/core/test/base/accessors_reduced_storage.cpp b/core/test/base/accessors_reduced_storage.cpp
index a59f85dd1f6..7d759fc1c2e 100644
--- a/core/test/base/accessors_reduced_storage.cpp
+++ b/core/test/base/accessors_reduced_storage.cpp
@@ -30,6 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include <array>
 #include <tuple>
 #include <type_traits>
 
@@ -37,12 +38,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
-#include <ginkgo/core/base/dim.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
 
 
-#include "core/base/accessors.hpp"
+#include "accessor/index_span.hpp"
+#include "accessor/reduced_row_major.hpp"
+#include "accessor/scaled_reduced_row_major.hpp"
 #include "core/base/extended_float.hpp"
 #include "core/test/utils.hpp"
 
@@ -64,15 +66,16 @@ class ReducedStorage3d : public ::testing::Test {
 
     // Type for `check_accessor_correctness` to forward the indices
     using t = std::tuple<int, int, int>;
+    using i_span = gko::acc::index_span;
 
-    using accessor = gko::accessor::reduced_row_major<3, ar_type, st_type>;
+    using accessor = gko::acc::reduced_row_major<3, ar_type, st_type>;
     using const_accessor =
-        gko::accessor::reduced_row_major<3, ar_type, const st_type>;
+        gko::acc::reduced_row_major<3, ar_type, const st_type>;
 
-    using reduced_storage = gko::range<accessor>;
-    using const_reduced_storage = gko::range<const_accessor>;
+    using reduced_storage = gko::acc::range<accessor>;
+    using const_reduced_storage = gko::acc::range<const_accessor>;
 
-    const gko::dim<3> size{4u, 3u, 2u};
+    const std::array<gko::size_type, 3> size{{4u, 3u, 2u}};
     static constexpr gko::size_type data_elements{4 * 3 * 2};
     // clang-format off
     st_type data[data_elements] {
@@ -184,6 +187,7 @@ TYPED_TEST(ReducedStorage3d, CanReadData)
 }
 
 
+/*
 TYPED_TEST(ReducedStorage3d, CopyFrom)
 {
     using st_type = typename TestFixture::st_type;
@@ -196,6 +200,7 @@ TYPED_TEST(ReducedStorage3d, CopyFrom)
 
     this->check_accessor_correctness(cpy);
 }
+*/
 
 
 TYPED_TEST(ReducedStorage3d, CanImplicitlyConvertToConst)
@@ -226,7 +231,7 @@ TYPED_TEST(ReducedStorage3d, CanCreateWithStride)
 {
     using reduced_storage = typename TestFixture::reduced_storage;
     using ar_type = typename TestFixture::ar_type;
-    auto size = gko::dim<3>{2, 2, 2};
+    auto size = std::array<gko::size_type, 3>{{2, 2, 2}};
     auto stride = std::array<gko::size_type, 2>{{12, 2}};
 
     auto range = reduced_storage{size, this->data, stride};
@@ -426,7 +431,8 @@ TYPED_TEST(ReducedStorage3d, UnaryMinus)
 
 TYPED_TEST(ReducedStorage3d, CanCreateSubrange)
 {
-    auto subr = this->r(gko::span{1u, 3u}, gko::span{0u, 2u}, 0u);
+    using i_span = typename TestFixture::i_span;
+    auto subr = this->r(i_span{1u, 3u}, i_span{0u, 2u}, 0u);
 
     EXPECT_EQ(subr(0, 0, 0), this->c_st_ar(5.06));
     EXPECT_EQ(subr(0, 1, 0), this->c_st_ar(2.08));
@@ -437,8 +443,8 @@ TYPED_TEST(ReducedStorage3d, CanCreateSubrange)
 
 TYPED_TEST(ReducedStorage3d, CanCreateSubrange2)
 {
-    auto subr =
-        this->cr(gko::span{1u, 3u}, gko::span{0u, 2u}, gko::span{0u, 1u});
+    using i_span = typename TestFixture::i_span;
+    auto subr = this->cr(i_span{1u, 3u}, i_span{0u, 2u}, i_span{0u, 1u});
 
     EXPECT_EQ(subr(0, 0, 0), this->c_st_ar(5.06));
     EXPECT_EQ(subr(0, 1, 0), this->c_st_ar(2.08));
@@ -449,18 +455,17 @@ TYPED_TEST(ReducedStorage3d, CanCreateSubrange2)
 
 class ReducedStorageXd : public ::testing::Test {
 protected:
-    using span = gko::span;
     using ar_type = double;
     using st_type = float;
     using size_type = gko::size_type;
     static constexpr ar_type delta{::r<st_type>::value};
 
-    using accessor1d = gko::accessor::reduced_row_major<1, ar_type, st_type>;
-    using accessor2d = gko::accessor::reduced_row_major<2, ar_type, st_type>;
+    using accessor1d = gko::acc::reduced_row_major<1, ar_type, st_type>;
+    using accessor2d = gko::acc::reduced_row_major<2, ar_type, st_type>;
     using const_accessor1d =
-        gko::accessor::reduced_row_major<1, ar_type, const st_type>;
+        gko::acc::reduced_row_major<1, ar_type, const st_type>;
     using const_accessor2d =
-        gko::accessor::reduced_row_major<2, ar_type, const st_type>;
+        gko::acc::reduced_row_major<2, ar_type, const st_type>;
     static_assert(std::is_same<const_accessor1d,
                                typename accessor1d::const_accessor>::value,
                   "Const accessors must be the same!");
@@ -468,15 +473,15 @@ class ReducedStorageXd : public ::testing::Test {
                                typename accessor2d::const_accessor>::value,
                   "Const accessors must be the same!");
 
-    using reduced_storage1d = gko::range<accessor1d>;
-    using reduced_storage2d = gko::range<accessor2d>;
-    using const_reduced_storage2d = gko::range<const_accessor2d>;
-    using const_reduced_storage1d = gko::range<const_accessor1d>;
+    using reduced_storage1d = gko::acc::range<accessor1d>;
+    using reduced_storage2d = gko::acc::range<accessor2d>;
+    using const_reduced_storage2d = gko::acc::range<const_accessor2d>;
+    using const_reduced_storage1d = gko::acc::range<const_accessor1d>;
 
     const std::array<size_type, 0> stride0{{}};
     const std::array<size_type, 1> stride1{{4}};
-    const gko::dim<1> size_1d{8u};
-    const gko::dim<2> size_2d{2u, 4u};
+    const std::array<gko::size_type, 1> size_1d{{8u}};
+    const std::array<gko::size_type, 2> size_2d{{2u, 4u}};
     static constexpr gko::size_type data_elements{8};
     st_type data[data_elements]{1.1f, 2.2f, 3.3f, 4.4f,
                                 5.5f, 6.6f, 7.7f, -8.8f};
@@ -543,19 +548,19 @@ class ScaledReducedStorage3d : public ::testing::Test {
         typename std::tuple_element<1, decltype(ArithmeticStorageType{})>::type;
     // Type for `check_accessor_correctness` to forward the indices
     using t = std::tuple<int, int, int>;
+    using i_span = gko::acc::index_span;
 
     static constexpr ar_type delta{::r<ar_type>::value};
 
     using accessor =
-        gko::accessor::scaled_reduced_row_major<3, ar_type, st_type, 0b0101>;
+        gko::acc::scaled_reduced_row_major<3, ar_type, st_type, 0b0101>;
     using const_accessor =
-        gko::accessor::scaled_reduced_row_major<3, ar_type, const st_type,
-                                                0b0101>;
+        gko::acc::scaled_reduced_row_major<3, ar_type, const st_type, 0b0101>;
 
-    using reduced_storage = gko::range<accessor>;
-    using const_reduced_storage = gko::range<const_accessor>;
+    using reduced_storage = gko::acc::range<accessor>;
+    using const_reduced_storage = gko::acc::range<const_accessor>;
 
-    const gko::dim<3> size{1u, 4u, 2u};
+    const std::array<gko::size_type, 3> size{{1u, 4u, 2u}};
     static constexpr gko::size_type data_elements{8};
     static constexpr gko::size_type scalar_elements{8};
     // clang-format off
@@ -640,6 +645,7 @@ TYPED_TEST(ScaledReducedStorage3d, CanReadData)
 }
 
 
+/*
 TYPED_TEST(ScaledReducedStorage3d, CopyFrom)
 {
     using ar_type = typename TestFixture::ar_type;
@@ -654,6 +660,7 @@ TYPED_TEST(ScaledReducedStorage3d, CopyFrom)
 
     this->check_accessor_correctness(cpy);
 }
+*/
 
 
 TYPED_TEST(ScaledReducedStorage3d, CanImplicitlyConvertToConst)
@@ -683,7 +690,7 @@ TYPED_TEST(ScaledReducedStorage3d, CanCreateWithStride)
 {
     using reduced_storage = typename TestFixture::reduced_storage;
     using ar_type = typename TestFixture::ar_type;
-    gko::dim<3> size{2, 1, 2};
+    std::array<gko::size_type, 3> size{{2, 1, 2}};
     std::array<gko::size_type, 2> stride_storage{{5, 2}};
     std::array<gko::size_type, 1> stride_scalar{{4}};
 
@@ -700,7 +707,8 @@ TYPED_TEST(ScaledReducedStorage3d, CanCreateWithStride)
 
 TYPED_TEST(ScaledReducedStorage3d, Subrange)
 {
-    auto subr = this->cr(0u, gko::span{0u, 2u}, 1u);
+    using i_span = typename TestFixture::i_span;
+    auto subr = this->cr(0u, i_span{0u, 2u}, 1u);
 
     EXPECT_EQ(subr(0, 0, 0), 22.);
     EXPECT_EQ(subr(0, 1, 0), 26.);
@@ -886,20 +894,19 @@ TYPED_TEST(ScaledReducedStorage3d, UnaryMinus)
 
 class ScaledReducedStorageXd : public ::testing::Test {
 protected:
-    using span = gko::span;
     using ar_type = double;
     using st_type = int;
     using size_type = gko::size_type;
     static constexpr ar_type delta{0.1};
 
     using accessor1d =
-        gko::accessor::scaled_reduced_row_major<1, ar_type, st_type, 1>;
+        gko::acc::scaled_reduced_row_major<1, ar_type, st_type, 1>;
     using accessor2d =
-        gko::accessor::scaled_reduced_row_major<2, ar_type, st_type, 3>;
+        gko::acc::scaled_reduced_row_major<2, ar_type, st_type, 3>;
     using const_accessor1d =
-        gko::accessor::scaled_reduced_row_major<1, ar_type, const st_type, 1>;
+        gko::acc::scaled_reduced_row_major<1, ar_type, const st_type, 1>;
     using const_accessor2d =
-        gko::accessor::scaled_reduced_row_major<2, ar_type, const st_type, 3>;
+        gko::acc::scaled_reduced_row_major<2, ar_type, const st_type, 3>;
     static_assert(std::is_same<const_accessor1d,
                                typename accessor1d::const_accessor>::value,
                   "Const accessors must be the same!");
@@ -907,16 +914,16 @@ class ScaledReducedStorageXd : public ::testing::Test {
                                typename accessor2d::const_accessor>::value,
                   "Const accessors must be the same!");
 
-    using reduced_storage1d = gko::range<accessor1d>;
-    using reduced_storage2d = gko::range<accessor2d>;
-    using const_reduced_storage2d = gko::range<const_accessor2d>;
-    using const_reduced_storage1d = gko::range<const_accessor1d>;
+    using reduced_storage1d = gko::acc::range<accessor1d>;
+    using reduced_storage2d = gko::acc::range<accessor2d>;
+    using const_reduced_storage2d = gko::acc::range<const_accessor2d>;
+    using const_reduced_storage1d = gko::acc::range<const_accessor1d>;
 
     const std::array<size_type, 0> stride0{{}};
     const std::array<size_type, 1> stride1{{4}};
     const std::array<size_type, 1> stride_sc{{5}};
-    const gko::dim<1> size_1d{8u};
-    const gko::dim<2> size_2d{2u, 2u};
+    const std::array<gko::size_type, 1> size_1d{{8u}};
+    const std::array<gko::size_type, 2> size_2d{{2u, 2u}};
 
     static constexpr gko::size_type data_elements{8};
     st_type data[data_elements]{10, 22, 32, 44, 54, 66, 76, -88};

From 4c497207b10882942572da9063458718c3f97299 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 22 Feb 2021 12:29:44 +0100
Subject: [PATCH 0488/1008] Adopt the CB-GMRES to the accessor changes

---
 core/solver/cb_gmres_accessor.hpp    | 23 +++++++++++-----------
 core/solver/cb_gmres_kernels.hpp     | 29 ++++++++++++++--------------
 cuda/solver/cb_gmres_kernels.cu      | 14 ++++++++------
 dpcpp/solver/cb_gmres_kernels.dp.cpp |  4 ----
 hip/solver/cb_gmres_kernels.hip.cpp  | 13 ++++++++-----
 5 files changed, 43 insertions(+), 40 deletions(-)

diff --git a/core/solver/cb_gmres_accessor.hpp b/core/solver/cb_gmres_accessor.hpp
index 87c77cea276..5c2e9329b99 100644
--- a/core/solver/cb_gmres_accessor.hpp
+++ b/core/solver/cb_gmres_accessor.hpp
@@ -45,12 +45,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/dim.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/types.hpp>
 
 
-#include "core/base/accessors.hpp"
+#include "accessor/range.hpp"
+#include "accessor/reduced_row_major.hpp"
+#include "accessor/scaled_reduced_row_major.hpp"
 
 
 namespace gko {
@@ -65,7 +66,7 @@ struct has_3d_scaled_accessor : public std::false_type {};
 
 template <typename T1, typename T2, size_type mask>
 struct has_3d_scaled_accessor<
-    range<accessor::scaled_reduced_row_major<3, T1, T2, mask>>>
+    acc::range<acc::scaled_reduced_row_major<3, T1, T2, mask>>>
     : public std::true_type {};
 
 template <typename StorageType, bool = std::is_integral<StorageType>::value>
@@ -90,13 +91,13 @@ template <typename ValueType, typename StorageType>
 class Range3dHelper<ValueType, StorageType, true> {
 public:
     using Accessor =
-        accessor::scaled_reduced_row_major<3, ValueType, StorageType, 0b101>;
-    using Range = range<Accessor>;
+        acc::scaled_reduced_row_major<3, ValueType, StorageType, 0b101>;
+    using Range = acc::range<Accessor>;
 
     Range3dHelper() = default;
 
     Range3dHelper(std::shared_ptr<const Executor> exec, dim<3> krylov_dim)
-        : krylov_dim_{krylov_dim},
+        : krylov_dim_{{krylov_dim[0], krylov_dim[1], krylov_dim[2]}},
           bases_{exec, krylov_dim_[0] * krylov_dim_[1] * krylov_dim_[2]},
           scale_{exec, krylov_dim_[0] * krylov_dim_[2]}
     {
@@ -116,7 +117,7 @@ class Range3dHelper<ValueType, StorageType, true> {
     gko::Array<StorageType> &get_bases() { return bases_; }
 
 private:
-    dim<3> krylov_dim_;
+    std::array<size_type, 3> krylov_dim_;
     Array<StorageType> bases_;
     Array<ValueType> scale_;
 };
@@ -125,13 +126,13 @@ class Range3dHelper<ValueType, StorageType, true> {
 template <typename ValueType, typename StorageType>
 class Range3dHelper<ValueType, StorageType, false> {
 public:
-    using Accessor = accessor::reduced_row_major<3, ValueType, StorageType>;
-    using Range = range<Accessor>;
+    using Accessor = acc::reduced_row_major<3, ValueType, StorageType>;
+    using Range = acc::range<Accessor>;
 
     Range3dHelper() = default;
 
     Range3dHelper(std::shared_ptr<const Executor> exec, dim<3> krylov_dim)
-        : krylov_dim_{krylov_dim},
+        : krylov_dim_{{krylov_dim[0], krylov_dim[1], krylov_dim[2]}},
           bases_{std::move(exec),
                  krylov_dim_[0] * krylov_dim_[1] * krylov_dim_[2]}
     {}
@@ -141,7 +142,7 @@ class Range3dHelper<ValueType, StorageType, false> {
     gko::Array<StorageType> &get_bases() { return bases_; }
 
 private:
-    dim<3> krylov_dim_;
+    std::array<size_type, 3> krylov_dim_;
     Array<StorageType> bases_;
 };
 
diff --git a/core/solver/cb_gmres_kernels.hpp b/core/solver/cb_gmres_kernels.hpp
index 7163f4527cc..1a7a8765d8d 100644
--- a/core/solver/cb_gmres_kernels.hpp
+++ b/core/solver/cb_gmres_kernels.hpp
@@ -41,7 +41,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
-#include "core/base/accessors.hpp"
+#include "accessor/reduced_row_major.hpp"
+#include "accessor/scaled_reduced_row_major.hpp"
 #include "core/base/extended_float.hpp"
 
 
@@ -64,52 +65,52 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     template _macro(                                                           \
         double,                                                                \
         GKO_UNPACK(                                                            \
-            range<accessor::reduced_row_major<3, double, _const double>>));    \
+            acc::range<acc::reduced_row_major<3, double, _const double>>));    \
     template _macro(                                                           \
         double,                                                                \
         GKO_UNPACK(                                                            \
-            range<accessor::reduced_row_major<3, double, _const float>>));     \
+            acc::range<acc::reduced_row_major<3, double, _const float>>));     \
     template _macro(                                                           \
         double,                                                                \
         GKO_UNPACK(                                                            \
-            range<accessor::reduced_row_major<3, double, _const half>>));      \
+            acc::range<acc::reduced_row_major<3, double, _const half>>));      \
     template _macro(double,                                                    \
-                    GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
+                    GKO_UNPACK(acc::range<acc::scaled_reduced_row_major<       \
                                    3, double, _const int64, 0b101>>));         \
     template _macro(double,                                                    \
-                    GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
+                    GKO_UNPACK(acc::range<acc::scaled_reduced_row_major<       \
                                    3, double, _const int32, 0b101>>));         \
     template _macro(double,                                                    \
-                    GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
+                    GKO_UNPACK(acc::range<acc::scaled_reduced_row_major<       \
                                    3, double, _const int16, 0b101>>));         \
     template _macro(                                                           \
         float,                                                                 \
         GKO_UNPACK(                                                            \
-            range<accessor::reduced_row_major<3, float, _const float>>));      \
+            acc::range<acc::reduced_row_major<3, float, _const float>>));      \
     template _macro(                                                           \
         float,                                                                 \
         GKO_UNPACK(                                                            \
-            range<accessor::reduced_row_major<3, float, _const half>>));       \
+            acc::range<acc::reduced_row_major<3, float, _const half>>));       \
     template _macro(float,                                                     \
-                    GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
+                    GKO_UNPACK(acc::range<acc::scaled_reduced_row_major<       \
                                    3, float, _const int32, 0b101>>));          \
     template _macro(float,                                                     \
-                    GKO_UNPACK(range<accessor::scaled_reduced_row_major<       \
+                    GKO_UNPACK(acc::range<acc::scaled_reduced_row_major<       \
                                    3, float, _const int16, 0b101>>));          \
     template _macro(                                                           \
         std::complex<double>,                                                  \
         GKO_UNPACK(                                                            \
-            range<accessor::reduced_row_major<3, std::complex<double>,         \
+            acc::range<acc::reduced_row_major<3, std::complex<double>,         \
                                               _const std::complex<double>>>)); \
     template _macro(                                                           \
         std::complex<double>,                                                  \
         GKO_UNPACK(                                                            \
-            range<accessor::reduced_row_major<3, std::complex<double>,         \
+            acc::range<acc::reduced_row_major<3, std::complex<double>,         \
                                               _const std::complex<float>>>));  \
     template _macro(                                                           \
         std::complex<float>,                                                   \
         GKO_UNPACK(                                                            \
-            range<accessor::reduced_row_major<3, std::complex<float>,          \
+            acc::range<acc::reduced_row_major<3, std::complex<float>,          \
                                               _const std::complex<float>>>))
 
 #define GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(_macro) \
diff --git a/cuda/solver/cb_gmres_kernels.cu b/cuda/solver/cb_gmres_kernels.cu
index 817eb7d4364..7c8b138a1a6 100644
--- a/cuda/solver/cb_gmres_kernels.cu
+++ b/cuda/solver/cb_gmres_kernels.cu
@@ -42,7 +42,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/stopping_status.hpp>
 
 
-#include "core/base/accessors.hpp"
+#include "accessor/range.hpp"
+#include "accessor/reduced_row_major.hpp"
+#include "accessor/scaled_reduced_row_major.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/matrix/dense_kernels.hpp"
 #include "core/solver/cb_gmres_accessor.hpp"
@@ -78,10 +80,10 @@ constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 // Specialization, so the Accessor can use the same function as regular pointers
 template <int dim, typename Type1, typename Type2>
 GKO_INLINE auto as_cuda_accessor(
-    const range<accessor::reduced_row_major<dim, Type1, Type2>> &acc)
+    const acc::range<acc::reduced_row_major<dim, Type1, Type2>> &acc)
 {
-    return range<
-        accessor::reduced_row_major<dim, cuda_type<Type1>, cuda_type<Type2>>>(
+    return acc::range<
+        acc::reduced_row_major<dim, cuda_type<Type1>, cuda_type<Type2>>>(
         acc.get_accessor().get_size(),
         as_cuda_type(acc.get_accessor().get_stored_data()),
         acc.get_accessor().get_stride());
@@ -89,10 +91,10 @@ GKO_INLINE auto as_cuda_accessor(
 
 template <int dim, typename Type1, typename Type2, size_type mask>
 GKO_INLINE auto as_cuda_accessor(
-    const range<accessor::scaled_reduced_row_major<dim, Type1, Type2, mask>>
+    const acc::range<acc::scaled_reduced_row_major<dim, Type1, Type2, mask>>
         &acc)
 {
-    return range<accessor::scaled_reduced_row_major<dim, cuda_type<Type1>,
+    return acc::range<acc::scaled_reduced_row_major<dim, cuda_type<Type1>,
                                                     cuda_type<Type2>, mask>>(
         acc.get_accessor().get_size(),
         as_cuda_type(acc.get_accessor().get_stored_data()),
diff --git a/dpcpp/solver/cb_gmres_kernels.dp.cpp b/dpcpp/solver/cb_gmres_kernels.dp.cpp
index f6d5345d3a4..b30fe70c76b 100644
--- a/dpcpp/solver/cb_gmres_kernels.dp.cpp
+++ b/dpcpp/solver/cb_gmres_kernels.dp.cpp
@@ -37,10 +37,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/range.hpp>
-
-
-#include "core/base/accessors.hpp"
 
 
 namespace gko {
diff --git a/hip/solver/cb_gmres_kernels.hip.cpp b/hip/solver/cb_gmres_kernels.hip.cpp
index 7b899e1c118..510009665c0 100644
--- a/hip/solver/cb_gmres_kernels.hip.cpp
+++ b/hip/solver/cb_gmres_kernels.hip.cpp
@@ -42,6 +42,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/stopping_status.hpp>
 
 
+#include "accessor/range.hpp"
+#include "accessor/reduced_row_major.hpp"
+#include "accessor/scaled_reduced_row_major.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/matrix/dense_kernels.hpp"
 #include "core/solver/cb_gmres_accessor.hpp"
@@ -79,10 +82,10 @@ constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 // Specialization, so the Accessor can use the same function as regular pointers
 template <int dim, typename Type1, typename Type2>
 GKO_INLINE auto as_hip_accessor(
-    const range<accessor::reduced_row_major<dim, Type1, Type2>> &acc)
+    const acc::range<acc::reduced_row_major<dim, Type1, Type2>> &acc)
 {
-    return range<
-        accessor::reduced_row_major<dim, hip_type<Type1>, hip_type<Type2>>>(
+    return acc::range<
+        acc::reduced_row_major<dim, hip_type<Type1>, hip_type<Type2>>>(
         acc.get_accessor().get_size(),
         as_hip_type(acc.get_accessor().get_stored_data()),
         acc.get_accessor().get_stride());
@@ -90,10 +93,10 @@ GKO_INLINE auto as_hip_accessor(
 
 template <int dim, typename Type1, typename Type2, size_type mask>
 GKO_INLINE auto as_hip_accessor(
-    const range<accessor::scaled_reduced_row_major<dim, Type1, Type2, mask>>
+    const acc::range<acc::scaled_reduced_row_major<dim, Type1, Type2, mask>>
         &acc)
 {
-    return range<accessor::scaled_reduced_row_major<dim, hip_type<Type1>,
+    return acc::range<acc::scaled_reduced_row_major<dim, hip_type<Type1>,
                                                     hip_type<Type2>, mask>>(
         acc.get_accessor().get_size(),
         as_hip_type(acc.get_accessor().get_stored_data()),

From 8fbe2ba64e828277cc93d43da271a8faf43c5f8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 24 Feb 2021 05:34:54 +0100
Subject: [PATCH 0489/1008] Small improvements to the accessor directory

Includes:
- Add documentation to index_span
- Update documentation of range
- Give accessor helper functions more descriptive names
---
 accessor/accessor_helper.hpp          | 34 ++++++++++++------------
 accessor/index_span.hpp               | 37 +++++++++++++++++++++++++++
 accessor/range.hpp                    | 11 +++++---
 accessor/reduced_row_major.hpp        |  4 +--
 accessor/scaled_reduced_row_major.hpp |  7 ++---
 5 files changed, 66 insertions(+), 27 deletions(-)

diff --git a/accessor/accessor_helper.hpp b/accessor/accessor_helper.hpp
index 72099aee856..ee2178a41d6 100644
--- a/accessor/accessor_helper.hpp
+++ b/accessor/accessor_helper.hpp
@@ -124,8 +124,8 @@ template <typename ValueType, size_type iter = 1, size_type N,
 constexpr GKO_ACC_ATTRIBUTES
     std::enable_if_t<N == 0 || (iter == N && iter == sizeof...(Args) + 1),
                      std::array<ValueType, N == 0 ? 0 : N - 1>>
-    compute_default_stride_array(const std::array<DimensionType, N> &,
-                                 Args &&... args)
+    compute_default_row_major_stride_array(const std::array<DimensionType, N> &,
+                                           Args &&... args)
 {
     return {{std::forward<Args>(args)...}};
 }
@@ -134,10 +134,10 @@ template <typename ValueType, size_type iter = 1, size_type N,
           typename DimensionType, typename... Args>
 constexpr GKO_ACC_ATTRIBUTES std::enable_if_t<
     (iter < N) && (iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
-compute_default_stride_array(const std::array<DimensionType, N> &size,
-                             Args &&... args)
+compute_default_row_major_stride_array(const std::array<DimensionType, N> &size,
+                                       Args &&... args)
 {
-    return compute_default_stride_array<ValueType, iter + 1>(
+    return compute_default_row_major_stride_array<ValueType, iter + 1>(
         size, std::forward<Args>(args)...,
         mult_dim_upwards<ValueType, iter>(size));
 }
@@ -151,7 +151,7 @@ compute_default_stride_array(const std::array<DimensionType, N> &size,
  * stride array
  */
 template <typename ValueType, size_type total_dim, typename... Indices>
-constexpr GKO_ACC_ATTRIBUTES ValueType compute_storage_index(
+constexpr GKO_ACC_ATTRIBUTES ValueType compute_row_major_index(
     const std::array<size_type, total_dim> &size,
     const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)> &stride,
     Indices &&... idxs)
@@ -178,11 +178,12 @@ constexpr GKO_ACC_ATTRIBUTES ValueType compute_storage_index(
  *          information.
  */
 template <typename ValueType, size_type dimensions, typename DimensionType>
-constexpr GKO_ACC_ATTRIBUTES std::array<ValueType,
-                                        (dimensions > 0 ? dimensions - 1 : 0)>
-compute_default_stride_array(const std::array<DimensionType, dimensions> &size)
+constexpr GKO_ACC_ATTRIBUTES
+    std::array<ValueType, (dimensions > 0 ? dimensions - 1 : 0)>
+    compute_default_row_major_stride_array(
+        const std::array<DimensionType, dimensions> &size)
 {
-    return detail::compute_default_stride_array<ValueType>(size);
+    return detail::compute_default_row_major_stride_array<ValueType>(size);
 }
 
 
@@ -500,7 +501,7 @@ constexpr GKO_ACC_ATTRIBUTES auto compute_masked_index_direct(
  */
 template <typename ValueType, size_type mask, size_type stride_size,
           size_type total_dim>
-constexpr GKO_ACC_ATTRIBUTES auto compute_default_masked_stride_array(
+constexpr GKO_ACC_ATTRIBUTES auto compute_default_masked_row_major_stride_array(
     const std::array<size_type, total_dim> &size)
 {
     return detail::row_major_masked_helper_s<ValueType, mask, 0, stride_size, 0,
@@ -517,13 +518,13 @@ struct are_index_span_compatible_impl
 
 template <bool has_span, typename First, typename... Args>
 struct are_index_span_compatible_impl<has_span, First, Args...>
-    : public std::conditional<
+    : public std::conditional_t<
           std::is_integral<std::decay_t<First>>::value ||
               std::is_same<std::decay_t<First>, index_span>::value,
           are_index_span_compatible_impl<
               has_span || std::is_same<std::decay_t<First>, index_span>::value,
               Args...>,
-          std::false_type>::type {};
+          std::false_type> {};
 
 
 }  // namespace detail
@@ -552,14 +553,12 @@ GKO_ACC_ATTRIBUTES std::enable_if_t<iter == N> multidim_for_each_impl(
 
 template <size_type iter, size_type N, typename Callable, typename... Indices>
 GKO_ACC_ATTRIBUTES std::enable_if_t<(iter < N)> multidim_for_each_impl(
-    const std::array<size_type, N> &size, Callable &&callable,
-    Indices &&... indices)
+    const std::array<size_type, N> &size, Callable callable, Indices... indices)
 {
     static_assert(iter == sizeof...(Indices),
                   "Number arguments must match current iteration!");
     for (size_type i = 0; i < size[iter]; ++i) {
-        multidim_for_each_impl<iter + 1>(size, std::forward<Callable>(callable),
-                                         std::forward<Indices>(indices)..., i);
+        multidim_for_each_impl<iter + 1>(size, callable, indices..., i);
     }
 }
 
@@ -596,7 +595,6 @@ index_spans_in_size(const std::array<size_type, N> &size, First first,
 {
     static_assert(sizeof...(Remaining) + 1 == N - iter,
                   "Number of remaining spans must be equal to N - iter");
-    // TODO: Figure out why this assert causes errors outside of Ginkgo
     return GKO_ACC_ASSERT(index_span{first}.is_valid()),
            GKO_ACC_ASSERT(index_span{first} <= index_span{size[iter]}),
            index_spans_in_size<iter + 1>(size,
diff --git a/accessor/index_span.hpp b/accessor/index_span.hpp
index 5b71f3cdc4e..91522ed667b 100644
--- a/accessor/index_span.hpp
+++ b/accessor/index_span.hpp
@@ -40,16 +40,53 @@ namespace gko {
 namespace acc {
 
 
+/**
+ * An index_span is a lightweight structure used to describe a contiguous span
+ * of indices of one dimension.
+ *
+ * The main purpose of the index_span is to create sub-ranges from other ranges.
+ * An index_span `is` represents a contiguous set of indexes in one dimension of
+ * the range, starting from `is.begin` (inclusive) and ending at index `is.end`
+ * (exclusive). An index_span is only valid if its end is larger than its
+ * beginning.
+ *
+ * index_spans can be compared using `==` and `!=` operators. Two spans are
+ * equal iff both their `begin` and `end` values are identical.
+ *
+ * index_sets also have two distinct partial orders defined:
+ * 1. `x < y` (`y > x`) iff `x.end < y.begin`
+ * 2. `x <= y` (`y >= x`) iff `x.end <= y.begin`
+ * Note: `x < y || x == y` is not equivalent to `x <= y`.
+ */
 struct index_span {
+    /**
+     * Creates an index_span.
+     *
+     * @param begin  the beginning (inclusive) of the index_span
+     * @param end  the end (exclusive) of the index_span
+     *
+     */
     GKO_ACC_ATTRIBUTES constexpr index_span(size_type begin,
                                             size_type end) noexcept
         : begin{begin}, end{end}
     {}
 
+    /**
+     * Creates an index_span representing the point `point`.
+     *
+     * The begin is set to `point`, and the end to `point + 1`
+     *
+     * @param point  the point which the index_span represents
+     */
     GKO_ACC_ATTRIBUTES constexpr index_span(size_type point) noexcept
         : index_span{point, point + 1}
     {}
 
+    /**
+     * Checks if an index_span is valid.
+     *
+     * @returns true iff `this->begin < this->end`
+     */
     GKO_ACC_ATTRIBUTES constexpr bool is_valid() const { return begin < end; }
 
     friend GKO_ACC_ATTRIBUTES constexpr bool operator<(const index_span &first,
diff --git a/accessor/range.hpp b/accessor/range.hpp
index 2b9447acb6a..258923464b1 100644
--- a/accessor/range.hpp
+++ b/accessor/range.hpp
@@ -79,12 +79,15 @@ class range {
      *
      * @tparam DimensionTypes  The types of indexes. Supported types depend on
      *                         the underlying accessor, but are usually either
-     *                         integer types or spans. If at least one index is
-     *                         a span, the returned value will be a sub-range.
+     *                         integer types or index_spans. If at least one
+     *                         index is a span, the returned value will be a
+     *                         sub-range (if that is supported by the accessor).
      *
-     * @param dimensions  the indexes of the values.
+     * @param dimensions  the indexes of the values or index_spans for the new
+     *                    range.
      *
-     * @return a value on position `(dimensions...)`.
+     * @return a value on position `(dimensions...)` or a sub-range with the
+     *         given index_spans.
      */
     template <typename... DimensionTypes>
     GKO_ACC_ATTRIBUTES constexpr auto operator()(
diff --git a/accessor/reduced_row_major.hpp b/accessor/reduced_row_major.hpp
index db31e1777fe..772607a1be7 100644
--- a/accessor/reduced_row_major.hpp
+++ b/accessor/reduced_row_major.hpp
@@ -141,7 +141,7 @@ class reduced_row_major {
         std::array<size_type, dimensionality> size, storage_type *storage)
         : reduced_row_major{
               size, storage,
-              helper::compute_default_stride_array<size_type>(size)}
+              helper::compute_default_row_major_stride_array<size_type>(size)}
     {}
 
     /**
@@ -281,7 +281,7 @@ class reduced_row_major {
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
-        return helper::compute_storage_index<size_type, dimensionality>(
+        return helper::compute_row_major_index<size_type, dimensionality>(
             size_, stride_, std::forward<Indices>(indices)...);
     }
 
diff --git a/accessor/scaled_reduced_row_major.hpp b/accessor/scaled_reduced_row_major.hpp
index 93546a01909..9b605577865 100644
--- a/accessor/scaled_reduced_row_major.hpp
+++ b/accessor/scaled_reduced_row_major.hpp
@@ -237,7 +237,7 @@ class scaled_reduced_row_major
         storage_stride_type stride, scalar_type *scalar)
         : scaled_reduced_row_major{
               size, storage, stride, scalar,
-              helper::compute_default_masked_stride_array<
+              helper::compute_default_masked_row_major_stride_array<
                   size_type, scalar_mask, scalar_stride_dim, dimensionality>(
                   size)}
     {}
@@ -256,7 +256,8 @@ class scaled_reduced_row_major
         scalar_type *scalar)
         : scaled_reduced_row_major{
               size, storage,
-              helper::compute_default_stride_array<size_type>(size), scalar}
+              helper::compute_default_row_major_stride_array<size_type>(size),
+              scalar}
     {}
 
     /**
@@ -474,7 +475,7 @@ class scaled_reduced_row_major
     {
         static_assert(sizeof...(Indices) == dimensionality,
                       "Number of indices must match dimensionality!");
-        return helper::compute_storage_index<size_type, dimensionality>(
+        return helper::compute_row_major_index<size_type, dimensionality>(
             size_, storage_stride_, std::forward<Indices>(indices)...);
     }
 

From 5c2bbfdbbfc2c295f6f900dfc67e4e03758bf35d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 24 Feb 2021 06:40:55 +0100
Subject: [PATCH 0490/1008] Move the accessor tests to their own directory

---
 core/test/CMakeLists.txt                      |   1 +
 core/test/accessor/CMakeLists.txt             |   4 +
 core/test/accessor/index_span.cpp             | 132 +++++
 core/test/accessor/range.cpp                  | 111 ++++
 .../reduced_row_major.cpp}                    | 477 +----------------
 .../accessor/scaled_reduced_row_major.cpp     | 494 ++++++++++++++++++
 core/test/base/CMakeLists.txt                 |   1 -
 7 files changed, 745 insertions(+), 475 deletions(-)
 create mode 100644 core/test/accessor/CMakeLists.txt
 create mode 100644 core/test/accessor/index_span.cpp
 create mode 100644 core/test/accessor/range.cpp
 rename core/test/{base/accessors_reduced_storage.cpp => accessor/reduced_row_major.cpp} (54%)
 create mode 100644 core/test/accessor/scaled_reduced_row_major.cpp

diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt
index 0639b38a63f..fcf1cf64777 100644
--- a/core/test/CMakeLists.txt
+++ b/core/test/CMakeLists.txt
@@ -1,5 +1,6 @@
 include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
 
+add_subdirectory(accessor)
 add_subdirectory(base)
 add_subdirectory(factorization)
 add_subdirectory(log)
diff --git a/core/test/accessor/CMakeLists.txt b/core/test/accessor/CMakeLists.txt
new file mode 100644
index 00000000000..7ee90215614
--- /dev/null
+++ b/core/test/accessor/CMakeLists.txt
@@ -0,0 +1,4 @@
+ginkgo_create_test(index_span)
+ginkgo_create_test(range)
+ginkgo_create_test(reduced_row_major)
+ginkgo_create_test(scaled_reduced_row_major)
diff --git a/core/test/accessor/index_span.cpp b/core/test/accessor/index_span.cpp
new file mode 100644
index 00000000000..690c7d7bb27
--- /dev/null
+++ b/core/test/accessor/index_span.cpp
@@ -0,0 +1,132 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "accessor/index_span.hpp"
+
+
+#include <gtest/gtest.h>
+
+
+namespace {
+
+
+TEST(IndexSpan, CreatesSpan)
+{
+    gko::acc::index_span s{3, 5};
+
+    ASSERT_EQ(s.begin, 3);
+    ASSERT_EQ(s.end, 5);
+}
+
+
+TEST(IndexSpan, CreatesPoint)
+{
+    gko::acc::index_span s{3};
+
+    ASSERT_EQ(s.begin, 3);
+    ASSERT_EQ(s.end, 4);
+}
+
+
+TEST(IndexSpan, LessThanEvaluatesToTrue)
+{
+    ASSERT_TRUE(gko::acc::index_span(2, 3) < gko::acc::index_span(4, 7));
+}
+
+
+TEST(IndexSpan, LessThanEvaluatesToFalse)
+{
+    ASSERT_FALSE(gko::acc::index_span(2, 4) < gko::acc::index_span(4, 7));
+}
+
+
+TEST(IndexSpan, LessOrEqualEvaluatesToTrue)
+{
+    ASSERT_TRUE(gko::acc::index_span(2, 4) <= gko::acc::index_span(4, 7));
+}
+
+
+TEST(IndexSpan, LessOrEqualEvaluatesToFalse)
+{
+    ASSERT_FALSE(gko::acc::index_span(2, 5) <= gko::acc::index_span(4, 7));
+}
+
+
+TEST(IndexSpan, GreaterThanEvaluatesToTrue)
+{
+    ASSERT_TRUE(gko::acc::index_span(4, 7) > gko::acc::index_span(2, 3));
+}
+
+
+TEST(IndexSpan, GreaterThanEvaluatesToFalse)
+{
+    ASSERT_FALSE(gko::acc::index_span(4, 7) > gko::acc::index_span(2, 4));
+}
+
+
+TEST(IndexSpan, GreaterOrEqualEvaluatesToTrue)
+{
+    ASSERT_TRUE(gko::acc::index_span(4, 7) >= gko::acc::index_span(2, 4));
+}
+
+
+TEST(IndexSpan, GreaterOrEqualEvaluatesToFalse)
+{
+    ASSERT_FALSE(gko::acc::index_span(4, 7) >= gko::acc::index_span(2, 5));
+}
+
+
+TEST(IndexSpan, EqualityEvaluatesToTrue)
+{
+    ASSERT_TRUE(gko::acc::index_span(2, 4) == gko::acc::index_span(2, 4));
+}
+
+
+TEST(IndexSpan, EqualityEvaluatesToFalse)
+{
+    ASSERT_FALSE(gko::acc::index_span(3, 4) == gko::acc::index_span(2, 5));
+}
+
+
+TEST(IndexSpan, NotEqualEvaluatesToTrue)
+{
+    ASSERT_TRUE(gko::acc::index_span(3, 4) != gko::acc::index_span(2, 5));
+}
+
+
+TEST(IndexSpan, NotEqualEvaluatesToFalse)
+{
+    ASSERT_FALSE(gko::acc::index_span(2, 4) != gko::acc::index_span(2, 4));
+}
+
+
+}  // namespace
diff --git a/core/test/accessor/range.cpp b/core/test/accessor/range.cpp
new file mode 100644
index 00000000000..63b84e16f98
--- /dev/null
+++ b/core/test/accessor/range.cpp
@@ -0,0 +1,111 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "accessor/range.hpp"
+
+
+#include <array>
+#include <cstddef>
+
+
+#include <gtest/gtest.h>
+
+
+namespace {
+
+
+// 0-memory constant accessor, which "stores" x*i + y*j + k at location
+// (i, j, k)
+struct dummy_accessor {
+    static constexpr std::size_t dimensionality = 3;
+
+    dummy_accessor(std::size_t size, int x, int y)
+        : sizes{size, size, size}, x{x}, y{y}
+    {}
+
+    dummy_accessor(std::size_t size_x, std::size_t size_y, std::size_t size_z,
+                   int x, int y)
+        : sizes{size_x, size_y, size_z}, x{x}, y{y}
+    {}
+
+    int operator()(int a, int b, int c) const { return x * a + y * b + c; }
+
+    gko::acc::size_type length(std::size_t dim) const { return sizes[dim]; }
+
+    std::array<std::size_t, 3> sizes;
+    mutable int x;
+    mutable int y;
+};
+
+
+using dummy_range = gko::acc::range<dummy_accessor>;
+
+
+TEST(Range, CreatesRange)
+{
+    dummy_range r{5u, 2, 3};
+
+    EXPECT_EQ(r->x, 2);
+    ASSERT_EQ(r->y, 3);
+}
+
+
+TEST(Range, ForwardsCallsToAccessor)
+{
+    dummy_range r{5u, 2, 3};
+
+    EXPECT_EQ(r(1, 2, 3), 2 * 1 + 3 * 2 + 3);
+    ASSERT_EQ(r(4, 2, 5), 2 * 4 + 3 * 2 + 5);
+}
+
+
+TEST(Range, ForwardsCopyToAccessor)
+{
+    dummy_range r{5u, 2, 3};
+    r = dummy_range{5u, 2, 5};
+
+    EXPECT_EQ(r->x, 2);
+    ASSERT_EQ(r->y, 5);
+}
+
+
+TEST(Range, ForwardsLength)
+{
+    dummy_range r{5u, 2, 3};
+
+    EXPECT_EQ(r->length(0), 5);
+    EXPECT_EQ(r->length(1), 5);
+    ASSERT_EQ(r->length(2), 5);
+}
+
+
+}  // namespace
diff --git a/core/test/base/accessors_reduced_storage.cpp b/core/test/accessor/reduced_row_major.cpp
similarity index 54%
rename from core/test/base/accessors_reduced_storage.cpp
rename to core/test/accessor/reduced_row_major.cpp
index 7d759fc1c2e..027b68da55a 100644
--- a/core/test/base/accessors_reduced_storage.cpp
+++ b/core/test/accessor/reduced_row_major.cpp
@@ -30,6 +30,9 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include "accessor/reduced_row_major.hpp"
+
+
 #include <array>
 #include <tuple>
 #include <type_traits>
@@ -43,8 +46,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "accessor/index_span.hpp"
-#include "accessor/reduced_row_major.hpp"
-#include "accessor/scaled_reduced_row_major.hpp"
 #include "core/base/extended_float.hpp"
 #include "core/test/utils.hpp"
 
@@ -187,22 +188,6 @@ TYPED_TEST(ReducedStorage3d, CanReadData)
 }
 
 
-/*
-TYPED_TEST(ReducedStorage3d, CopyFrom)
-{
-    using st_type = typename TestFixture::st_type;
-    using reduced_storage = typename TestFixture::reduced_storage;
-    st_type data2[TestFixture::data_elements];
-    reduced_storage cpy(this->size, data2);
-
-    // Do not use this in regular code since the implementation is slow
-    cpy = this->r;
-
-    this->check_accessor_correctness(cpy);
-}
-*/
-
-
 TYPED_TEST(ReducedStorage3d, CanImplicitlyConvertToConst)
 {
     using const_reduced_storage = typename TestFixture::const_reduced_storage;
@@ -539,460 +524,4 @@ TEST_F(ReducedStorageXd, CanWrite2)
 }
 
 
-template <typename ArithmeticStorageType>
-class ScaledReducedStorage3d : public ::testing::Test {
-protected:
-    using ar_type =
-        typename std::tuple_element<0, decltype(ArithmeticStorageType{})>::type;
-    using st_type =
-        typename std::tuple_element<1, decltype(ArithmeticStorageType{})>::type;
-    // Type for `check_accessor_correctness` to forward the indices
-    using t = std::tuple<int, int, int>;
-    using i_span = gko::acc::index_span;
-
-    static constexpr ar_type delta{::r<ar_type>::value};
-
-    using accessor =
-        gko::acc::scaled_reduced_row_major<3, ar_type, st_type, 0b0101>;
-    using const_accessor =
-        gko::acc::scaled_reduced_row_major<3, ar_type, const st_type, 0b0101>;
-
-    using reduced_storage = gko::acc::range<accessor>;
-    using const_reduced_storage = gko::acc::range<const_accessor>;
-
-    const std::array<gko::size_type, 3> size{{1u, 4u, 2u}};
-    static constexpr gko::size_type data_elements{8};
-    static constexpr gko::size_type scalar_elements{8};
-    // clang-format off
-    st_type data[8]{
-        10, 11,
-        -12, 13,
-        14, -115,
-        6, 77
-    };
-    ar_type scalar[scalar_elements]{
-        1., 2., 3., 4., 5., 6., 7., 8.
-    };
-    // clang-format on
-    const std::array<gko::size_type, 2> storage_stride{{8, 2}};
-    const std::array<gko::size_type, 1> scalar_stride{{2}};
-    reduced_storage r{size, data, storage_stride, scalar, scalar_stride};
-    const_reduced_storage cr{size, data, scalar};
-
-    template <typename Accessor>
-    static void check_accessor_correctness(
-        const Accessor &a,
-        std::tuple<int, int, int> ignore = std::tuple<int, int, int>(99, 99,
-                                                                     99))
-    {
-        // Test for equality is fine here since they should not be modified
-        // clang-format off
-        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), ar_type{10.});   }
-        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), ar_type{22.});   }
-        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), ar_type{-12.});  }
-        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), ar_type{26.});   }
-        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), ar_type{14.});   }
-        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), ar_type{-230.}); }
-        if (ignore != t(0, 3, 0)) { EXPECT_EQ(a(0, 3, 0), ar_type{6.});    }
-        if (ignore != t(0, 3, 1)) { EXPECT_EQ(a(0, 3, 1), ar_type{154.});  }
-        // clang-format on
-    }
-};
-
-using ScaledReducedStorage3dTypes = ::testing::Types<
-    std::tuple<double, gko::int64>, std::tuple<double, gko::int32>,
-    std::tuple<double, gko::int16>, std::tuple<float, gko::int32>,
-    std::tuple<float, gko::int16>>;
-
-TYPED_TEST_SUITE(ScaledReducedStorage3d, ScaledReducedStorage3dTypes);
-
-
-TYPED_TEST(ScaledReducedStorage3d, CorrectLengths)
-{
-    EXPECT_EQ(this->r.length(0), this->size[0]);
-    EXPECT_EQ(this->r.length(1), this->size[1]);
-    EXPECT_EQ(this->r.length(2), this->size[2]);
-    EXPECT_EQ(this->r.length(3), 1);
-    EXPECT_EQ(this->r->get_size(), this->size);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CorrectStride)
-{
-    EXPECT_EQ(this->r->get_scalar_stride(), this->scalar_stride);
-    EXPECT_EQ(this->r->get_storage_stride(), this->storage_stride);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CorrectStorage)
-{
-    EXPECT_EQ(this->r->get_stored_data(), this->data);
-    EXPECT_EQ(this->r->get_const_storage(), this->data);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CorrectScale)
-{
-    EXPECT_EQ(this->r->get_scalar(), this->scalar);
-    EXPECT_EQ(this->r->get_const_scalar(), this->scalar);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CanReadData)
-{
-    this->check_accessor_correctness(this->r);
-    this->check_accessor_correctness(this->cr);
-}
-
-
-/*
-TYPED_TEST(ScaledReducedStorage3d, CopyFrom)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using st_type = typename TestFixture::st_type;
-    using reduced_storage = typename TestFixture::reduced_storage;
-    st_type data2[TestFixture::data_elements];
-    ar_type scale2[TestFixture::scalar_elements];
-    reduced_storage cpy(this->size, data2, scale2);
-
-    // Do not use this in regular code since the implementation is slow
-    cpy = this->r;
-
-    this->check_accessor_correctness(cpy);
-}
-*/
-
-
-TYPED_TEST(ScaledReducedStorage3d, CanImplicitlyConvertToConst)
-{
-    using const_reduced_storage = typename TestFixture::const_reduced_storage;
-
-    const_reduced_storage const_rs = this->r->to_const();
-    const_reduced_storage const_rs2 = this->cr;
-
-    this->check_accessor_correctness(const_rs);
-    this->check_accessor_correctness(const_rs2);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, ToConstWorks)
-{
-    using const_reduced_storage = typename TestFixture::const_reduced_storage;
-    auto cr2 = this->r->to_const();
-
-    static_assert(std::is_same<decltype(cr2), const_reduced_storage>::value,
-                  "Types must be equal!");
-    this->check_accessor_correctness(cr2);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CanCreateWithStride)
-{
-    using reduced_storage = typename TestFixture::reduced_storage;
-    using ar_type = typename TestFixture::ar_type;
-    std::array<gko::size_type, 3> size{{2, 1, 2}};
-    std::array<gko::size_type, 2> stride_storage{{5, 2}};
-    std::array<gko::size_type, 1> stride_scalar{{4}};
-
-    reduced_storage range{size, this->data, stride_storage, this->scalar,
-                          stride_scalar};
-    range(1, 0, 0) = ar_type{15};
-
-    EXPECT_EQ(range(0, 0, 0), ar_type{10});
-    EXPECT_EQ(range(0, 0, 1), ar_type{22});
-    EXPECT_EQ(range(1, 0, 0), ar_type{15});
-    EXPECT_EQ(range(1, 0, 1), ar_type{36});
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Subrange)
-{
-    using i_span = typename TestFixture::i_span;
-    auto subr = this->cr(0u, i_span{0u, 2u}, 1u);
-
-    EXPECT_EQ(subr(0, 0, 0), 22.);
-    EXPECT_EQ(subr(0, 1, 0), 26.);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CanWriteScale)
-{
-    using ar_type = typename TestFixture::ar_type;
-
-    this->r->write_scalar_masked(10., 0, 0, 0);
-
-    EXPECT_EQ(this->r(0, 0, 0), ar_type{100.});
-    EXPECT_EQ(this->r(0, 0, 1), ar_type{22.});
-    EXPECT_EQ(this->r(0, 1, 0), ar_type{-120.});
-    EXPECT_EQ(this->r(0, 1, 1), ar_type{26.});
-    EXPECT_EQ(this->r(0, 2, 0), ar_type{140.});
-    EXPECT_EQ(this->r(0, 2, 1), ar_type{-230.});
-    EXPECT_EQ(this->r(0, 3, 0), ar_type{60.});
-    EXPECT_EQ(this->r(0, 3, 1), ar_type{154.});
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CanWriteMaskedScale)
-{
-    using ar_type = typename TestFixture::ar_type;
-
-    this->r->write_scalar_direct(10., 0, 0);
-
-    EXPECT_EQ(this->r(0, 0, 0), ar_type{100.});
-    EXPECT_EQ(this->r(0, 0, 1), ar_type{22.});
-    EXPECT_EQ(this->r(0, 1, 0), ar_type{-120.});
-    EXPECT_EQ(this->r(0, 1, 1), ar_type{26.});
-    EXPECT_EQ(this->r(0, 2, 0), ar_type{140.});
-    EXPECT_EQ(this->r(0, 2, 1), ar_type{-230.});
-    EXPECT_EQ(this->r(0, 3, 0), ar_type{60.});
-    EXPECT_EQ(this->r(0, 3, 1), ar_type{154.});
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, CanReadScale)
-{
-    EXPECT_EQ(this->r->read_scalar_masked(0, 0, 0), 1.);
-    EXPECT_EQ(this->r->read_scalar_masked(0, 0, 1), 2.);
-    EXPECT_EQ(this->r->read_scalar_direct(0, 0), 1.);
-    EXPECT_EQ(this->r->read_scalar_direct(0, 1), 2.);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Addition)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = 10. + 3.;
-
-    const auto result = this->cr(0, 0, 0) + 3.;
-    this->r(0, 0, 0) += 3.;
-
-    this->check_accessor_correctness(this->r, t(0, 0, 0));
-    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Addition2)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = 10. + 22.;
-
-    const auto result = this->cr(0, 0, 0) + this->r(0, 0, 1);
-    this->r(0, 0, 0) += this->cr(0, 0, 1);
-
-    this->check_accessor_correctness(this->r, t(0, 0, 0));
-    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Subtraction)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = 22. - 2.;
-
-    const auto result = this->cr(0, 0, 1) - 2.;
-    this->r(0, 0, 1) -= 2.;
-
-    this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Subtraction2)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = -12. - 26.;
-
-    const auto result = this->cr(0, 1, 0) - this->r(0, 1, 1);
-    this->r(0, 1, 0) -= this->r(0, 1, 1);
-
-    this->check_accessor_correctness(this->r, t(0, 1, 0));
-    EXPECT_NEAR(this->r(0, 1, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Multiplication)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = 26. * 3.;
-
-    const auto result = this->cr(0, 1, 1) * 3.;
-    this->r(0, 1, 1) *= 3.;
-
-    this->check_accessor_correctness(this->r, t(0, 1, 1));
-    EXPECT_NEAR(this->r(0, 1, 1), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Multiplication2)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = 14. * 10.;
-
-    const auto result = this->r(0, 2, 0) * this->r(0, 0, 0);
-    this->r(0, 2, 0) *= this->r(0, 0, 0);
-
-    this->check_accessor_correctness(this->r, t(0, 2, 0));
-    EXPECT_NEAR(this->r(0, 2, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Division)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = 10. / 2.;
-
-    const auto result = this->cr(0, 0, 0) / 2.;
-    this->r(0, 0, 0) /= 2.;
-
-    this->check_accessor_correctness(this->r, t(0, 0, 0));
-    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, Division2)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type expected = -12. / 6.;
-
-    const auto result = this->r(0, 1, 0) / this->r(0, 3, 0);
-    this->r(0, 1, 0) /= this->r(0, 3, 0);
-
-    this->check_accessor_correctness(this->r, t(0, 1, 0));
-    EXPECT_NEAR(this->r(0, 1, 0), expected, TestFixture::delta);
-    EXPECT_NEAR(result, expected, TestFixture::delta);
-}
-
-
-TYPED_TEST(ScaledReducedStorage3d, UnaryMinus)
-{
-    using ar_type = typename TestFixture::ar_type;
-    using t = typename TestFixture::t;
-    const ar_type neg_expected = this->r(0, 1, 1);
-    const ar_type expected = -neg_expected;
-
-    auto result = -this->r(0, 1, 1);
-
-    this->check_accessor_correctness(this->r);
-    EXPECT_EQ(result, expected);
-}
-
-
-class ScaledReducedStorageXd : public ::testing::Test {
-protected:
-    using ar_type = double;
-    using st_type = int;
-    using size_type = gko::size_type;
-    static constexpr ar_type delta{0.1};
-
-    using accessor1d =
-        gko::acc::scaled_reduced_row_major<1, ar_type, st_type, 1>;
-    using accessor2d =
-        gko::acc::scaled_reduced_row_major<2, ar_type, st_type, 3>;
-    using const_accessor1d =
-        gko::acc::scaled_reduced_row_major<1, ar_type, const st_type, 1>;
-    using const_accessor2d =
-        gko::acc::scaled_reduced_row_major<2, ar_type, const st_type, 3>;
-    static_assert(std::is_same<const_accessor1d,
-                               typename accessor1d::const_accessor>::value,
-                  "Const accessors must be the same!");
-    static_assert(std::is_same<const_accessor2d,
-                               typename accessor2d::const_accessor>::value,
-                  "Const accessors must be the same!");
-
-    using reduced_storage1d = gko::acc::range<accessor1d>;
-    using reduced_storage2d = gko::acc::range<accessor2d>;
-    using const_reduced_storage2d = gko::acc::range<const_accessor2d>;
-    using const_reduced_storage1d = gko::acc::range<const_accessor1d>;
-
-    const std::array<size_type, 0> stride0{{}};
-    const std::array<size_type, 1> stride1{{4}};
-    const std::array<size_type, 1> stride_sc{{5}};
-    const std::array<gko::size_type, 1> size_1d{{8u}};
-    const std::array<gko::size_type, 2> size_2d{{2u, 2u}};
-
-    static constexpr gko::size_type data_elements{8};
-    st_type data[data_elements]{10, 22, 32, 44, 54, 66, 76, -88};
-    ar_type scalar[data_elements]{1e0,  5e-1, 1e-1, 5e-2,
-                                  1e-2, 5e-3, 1e-3, 5e-4};
-
-    reduced_storage1d r1{size_1d, data, scalar};
-    reduced_storage2d r2{size_2d, data, stride1, scalar, stride_sc};
-    const_reduced_storage1d cr1{size_1d, data, stride0, scalar};
-    const_reduced_storage2d cr2{size_2d, data, stride1, scalar, stride_sc};
-
-    void data_equal_except_for(int idx)
-    {
-        // clang-format off
-        if (idx != 0) { EXPECT_EQ(data[0], 10); }
-        if (idx != 1) { EXPECT_EQ(data[1], 22); }
-        if (idx != 2) { EXPECT_EQ(data[2], 32); }
-        if (idx != 3) { EXPECT_EQ(data[3], 44); }
-        if (idx != 4) { EXPECT_EQ(data[4], 54); }
-        if (idx != 5) { EXPECT_EQ(data[5], 66); }
-        if (idx != 6) { EXPECT_EQ(data[6], 76); }
-        if (idx != 7) { EXPECT_EQ(data[7], -88); }
-        // clang-format on
-    }
-    void scalar_equal_except_for(int idx)
-    {
-        // clang-format off
-        if (idx != 0) { EXPECT_EQ(scalar[0], ar_type{1e0}); }
-        if (idx != 1) { EXPECT_EQ(scalar[1], ar_type{5e-1}); }
-        if (idx != 2) { EXPECT_EQ(scalar[2], ar_type{1e-1}); }
-        if (idx != 3) { EXPECT_EQ(scalar[3], ar_type{5e-2}); }
-        if (idx != 4) { EXPECT_EQ(scalar[4], ar_type{1e-2}); }
-        if (idx != 5) { EXPECT_EQ(scalar[5], ar_type{5e-3}); }
-        if (idx != 6) { EXPECT_EQ(scalar[6], ar_type{1e-3}); }
-        if (idx != 7) { EXPECT_EQ(scalar[7], ar_type{5e-4}); }
-        // clang-format on
-    }
-};
-
-
-TEST_F(ScaledReducedStorageXd, CanRead)
-{
-    EXPECT_NEAR(cr1(1), 11., delta);
-    EXPECT_NEAR(cr2(0, 1), 11., delta);
-    EXPECT_NEAR(cr2(1, 1), 66e-3, delta);
-    EXPECT_NEAR(r1(1), 11., delta);
-    EXPECT_NEAR(r2(0, 1), 11., delta);
-    EXPECT_NEAR(r2(1, 1), 66e-3, delta);
-}
-
-
-TEST_F(ScaledReducedStorageXd, CanWrite1)
-{
-    r1(2) = 0.2;
-
-    data_equal_except_for(2);
-    scalar_equal_except_for(99);
-    EXPECT_NEAR(r1(2), 0.2, delta);
-}
-
-
-TEST_F(ScaledReducedStorageXd, CanWrite2)
-{
-    r2(1, 1) = 0.5;
-
-    data_equal_except_for(5);
-    scalar_equal_except_for(99);
-    EXPECT_NEAR(r2(1, 1), 0.5, delta);
-}
-
-
 }  // namespace
diff --git a/core/test/accessor/scaled_reduced_row_major.cpp b/core/test/accessor/scaled_reduced_row_major.cpp
new file mode 100644
index 00000000000..d7418716fe2
--- /dev/null
+++ b/core/test/accessor/scaled_reduced_row_major.cpp
@@ -0,0 +1,494 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "accessor/scaled_reduced_row_major.hpp"
+
+
+#include <array>
+#include <tuple>
+#include <type_traits>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+#include "accessor/index_span.hpp"
+#include "core/base/extended_float.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+template <typename ArithmeticStorageType>
+class ScaledReducedStorage3d : public ::testing::Test {
+protected:
+    using ar_type =
+        typename std::tuple_element<0, decltype(ArithmeticStorageType{})>::type;
+    using st_type =
+        typename std::tuple_element<1, decltype(ArithmeticStorageType{})>::type;
+    // Type for `check_accessor_correctness` to forward the indices
+    using t = std::tuple<int, int, int>;
+    using i_span = gko::acc::index_span;
+
+    static constexpr ar_type delta{::r<ar_type>::value};
+
+    using accessor =
+        gko::acc::scaled_reduced_row_major<3, ar_type, st_type, 0b0101>;
+    using const_accessor =
+        gko::acc::scaled_reduced_row_major<3, ar_type, const st_type, 0b0101>;
+
+    using reduced_storage = gko::acc::range<accessor>;
+    using const_reduced_storage = gko::acc::range<const_accessor>;
+
+    const std::array<gko::size_type, 3> size{{1u, 4u, 2u}};
+    static constexpr gko::size_type data_elements{8};
+    static constexpr gko::size_type scalar_elements{8};
+    // clang-format off
+    st_type data[8]{
+        10, 11,
+        -12, 13,
+        14, -115,
+        6, 77
+    };
+    ar_type scalar[scalar_elements]{
+        1., 2., 3., 4., 5., 6., 7., 8.
+    };
+    // clang-format on
+    const std::array<gko::size_type, 2> storage_stride{{8, 2}};
+    const std::array<gko::size_type, 1> scalar_stride{{2}};
+    reduced_storage r{size, data, storage_stride, scalar, scalar_stride};
+    const_reduced_storage cr{size, data, scalar};
+
+    template <typename Accessor>
+    static void check_accessor_correctness(
+        const Accessor &a,
+        std::tuple<int, int, int> ignore = std::tuple<int, int, int>(99, 99,
+                                                                     99))
+    {
+        // Test for equality is fine here since they should not be modified
+        // clang-format off
+        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), ar_type{10.});   }
+        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), ar_type{22.});   }
+        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), ar_type{-12.});  }
+        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), ar_type{26.});   }
+        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), ar_type{14.});   }
+        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), ar_type{-230.}); }
+        if (ignore != t(0, 3, 0)) { EXPECT_EQ(a(0, 3, 0), ar_type{6.});    }
+        if (ignore != t(0, 3, 1)) { EXPECT_EQ(a(0, 3, 1), ar_type{154.});  }
+        // clang-format on
+    }
+};
+
+using ScaledReducedStorage3dTypes = ::testing::Types<
+    std::tuple<double, gko::int64>, std::tuple<double, gko::int32>,
+    std::tuple<double, gko::int16>, std::tuple<float, gko::int32>,
+    std::tuple<float, gko::int16>>;
+
+TYPED_TEST_SUITE(ScaledReducedStorage3d, ScaledReducedStorage3dTypes);
+
+
+TYPED_TEST(ScaledReducedStorage3d, CorrectLengths)
+{
+    EXPECT_EQ(this->r.length(0), this->size[0]);
+    EXPECT_EQ(this->r.length(1), this->size[1]);
+    EXPECT_EQ(this->r.length(2), this->size[2]);
+    EXPECT_EQ(this->r.length(3), 1);
+    EXPECT_EQ(this->r->get_size(), this->size);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CorrectStride)
+{
+    EXPECT_EQ(this->r->get_scalar_stride(), this->scalar_stride);
+    EXPECT_EQ(this->r->get_storage_stride(), this->storage_stride);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CorrectStorage)
+{
+    EXPECT_EQ(this->r->get_stored_data(), this->data);
+    EXPECT_EQ(this->r->get_const_storage(), this->data);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CorrectScale)
+{
+    EXPECT_EQ(this->r->get_scalar(), this->scalar);
+    EXPECT_EQ(this->r->get_const_scalar(), this->scalar);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CanReadData)
+{
+    this->check_accessor_correctness(this->r);
+    this->check_accessor_correctness(this->cr);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CanImplicitlyConvertToConst)
+{
+    using const_reduced_storage = typename TestFixture::const_reduced_storage;
+
+    const_reduced_storage const_rs = this->r->to_const();
+    const_reduced_storage const_rs2 = this->cr;
+
+    this->check_accessor_correctness(const_rs);
+    this->check_accessor_correctness(const_rs2);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, ToConstWorks)
+{
+    using const_reduced_storage = typename TestFixture::const_reduced_storage;
+    auto cr2 = this->r->to_const();
+
+    static_assert(std::is_same<decltype(cr2), const_reduced_storage>::value,
+                  "Types must be equal!");
+    this->check_accessor_correctness(cr2);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CanCreateWithStride)
+{
+    using reduced_storage = typename TestFixture::reduced_storage;
+    using ar_type = typename TestFixture::ar_type;
+    std::array<gko::size_type, 3> size{{2, 1, 2}};
+    std::array<gko::size_type, 2> stride_storage{{5, 2}};
+    std::array<gko::size_type, 1> stride_scalar{{4}};
+
+    reduced_storage range{size, this->data, stride_storage, this->scalar,
+                          stride_scalar};
+    range(1, 0, 0) = ar_type{15};
+
+    EXPECT_EQ(range(0, 0, 0), ar_type{10});
+    EXPECT_EQ(range(0, 0, 1), ar_type{22});
+    EXPECT_EQ(range(1, 0, 0), ar_type{15});
+    EXPECT_EQ(range(1, 0, 1), ar_type{36});
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Subrange)
+{
+    using i_span = typename TestFixture::i_span;
+    auto subr = this->cr(0u, i_span{0u, 2u}, 1u);
+
+    EXPECT_EQ(subr(0, 0, 0), 22.);
+    EXPECT_EQ(subr(0, 1, 0), 26.);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CanWriteScale)
+{
+    using ar_type = typename TestFixture::ar_type;
+
+    this->r->write_scalar_masked(10., 0, 0, 0);
+
+    EXPECT_EQ(this->r(0, 0, 0), ar_type{100.});
+    EXPECT_EQ(this->r(0, 0, 1), ar_type{22.});
+    EXPECT_EQ(this->r(0, 1, 0), ar_type{-120.});
+    EXPECT_EQ(this->r(0, 1, 1), ar_type{26.});
+    EXPECT_EQ(this->r(0, 2, 0), ar_type{140.});
+    EXPECT_EQ(this->r(0, 2, 1), ar_type{-230.});
+    EXPECT_EQ(this->r(0, 3, 0), ar_type{60.});
+    EXPECT_EQ(this->r(0, 3, 1), ar_type{154.});
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CanWriteMaskedScale)
+{
+    using ar_type = typename TestFixture::ar_type;
+
+    this->r->write_scalar_direct(10., 0, 0);
+
+    EXPECT_EQ(this->r(0, 0, 0), ar_type{100.});
+    EXPECT_EQ(this->r(0, 0, 1), ar_type{22.});
+    EXPECT_EQ(this->r(0, 1, 0), ar_type{-120.});
+    EXPECT_EQ(this->r(0, 1, 1), ar_type{26.});
+    EXPECT_EQ(this->r(0, 2, 0), ar_type{140.});
+    EXPECT_EQ(this->r(0, 2, 1), ar_type{-230.});
+    EXPECT_EQ(this->r(0, 3, 0), ar_type{60.});
+    EXPECT_EQ(this->r(0, 3, 1), ar_type{154.});
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, CanReadScale)
+{
+    EXPECT_EQ(this->r->read_scalar_masked(0, 0, 0), 1.);
+    EXPECT_EQ(this->r->read_scalar_masked(0, 0, 1), 2.);
+    EXPECT_EQ(this->r->read_scalar_direct(0, 0), 1.);
+    EXPECT_EQ(this->r->read_scalar_direct(0, 1), 2.);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Addition)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = 10. + 3.;
+
+    const auto result = this->cr(0, 0, 0) + 3.;
+    this->r(0, 0, 0) += 3.;
+
+    this->check_accessor_correctness(this->r, t(0, 0, 0));
+    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Addition2)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = 10. + 22.;
+
+    const auto result = this->cr(0, 0, 0) + this->r(0, 0, 1);
+    this->r(0, 0, 0) += this->cr(0, 0, 1);
+
+    this->check_accessor_correctness(this->r, t(0, 0, 0));
+    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Subtraction)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = 22. - 2.;
+
+    const auto result = this->cr(0, 0, 1) - 2.;
+    this->r(0, 0, 1) -= 2.;
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(this->r(0, 0, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Subtraction2)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = -12. - 26.;
+
+    const auto result = this->cr(0, 1, 0) - this->r(0, 1, 1);
+    this->r(0, 1, 0) -= this->r(0, 1, 1);
+
+    this->check_accessor_correctness(this->r, t(0, 1, 0));
+    EXPECT_NEAR(this->r(0, 1, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Multiplication)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = 26. * 3.;
+
+    const auto result = this->cr(0, 1, 1) * 3.;
+    this->r(0, 1, 1) *= 3.;
+
+    this->check_accessor_correctness(this->r, t(0, 1, 1));
+    EXPECT_NEAR(this->r(0, 1, 1), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Multiplication2)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = 14. * 10.;
+
+    const auto result = this->r(0, 2, 0) * this->r(0, 0, 0);
+    this->r(0, 2, 0) *= this->r(0, 0, 0);
+
+    this->check_accessor_correctness(this->r, t(0, 2, 0));
+    EXPECT_NEAR(this->r(0, 2, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Division)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = 10. / 2.;
+
+    const auto result = this->cr(0, 0, 0) / 2.;
+    this->r(0, 0, 0) /= 2.;
+
+    this->check_accessor_correctness(this->r, t(0, 0, 0));
+    EXPECT_NEAR(this->r(0, 0, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, Division2)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type expected = -12. / 6.;
+
+    const auto result = this->r(0, 1, 0) / this->r(0, 3, 0);
+    this->r(0, 1, 0) /= this->r(0, 3, 0);
+
+    this->check_accessor_correctness(this->r, t(0, 1, 0));
+    EXPECT_NEAR(this->r(0, 1, 0), expected, TestFixture::delta);
+    EXPECT_NEAR(result, expected, TestFixture::delta);
+}
+
+
+TYPED_TEST(ScaledReducedStorage3d, UnaryMinus)
+{
+    using ar_type = typename TestFixture::ar_type;
+    using t = typename TestFixture::t;
+    const ar_type neg_expected = this->r(0, 1, 1);
+    const ar_type expected = -neg_expected;
+
+    auto result = -this->r(0, 1, 1);
+
+    this->check_accessor_correctness(this->r);
+    EXPECT_EQ(result, expected);
+}
+
+
+class ScaledReducedStorageXd : public ::testing::Test {
+protected:
+    using ar_type = double;
+    using st_type = int;
+    using size_type = gko::size_type;
+    static constexpr ar_type delta{0.1};
+
+    using accessor1d =
+        gko::acc::scaled_reduced_row_major<1, ar_type, st_type, 1>;
+    using accessor2d =
+        gko::acc::scaled_reduced_row_major<2, ar_type, st_type, 3>;
+    using const_accessor1d =
+        gko::acc::scaled_reduced_row_major<1, ar_type, const st_type, 1>;
+    using const_accessor2d =
+        gko::acc::scaled_reduced_row_major<2, ar_type, const st_type, 3>;
+    static_assert(std::is_same<const_accessor1d,
+                               typename accessor1d::const_accessor>::value,
+                  "Const accessors must be the same!");
+    static_assert(std::is_same<const_accessor2d,
+                               typename accessor2d::const_accessor>::value,
+                  "Const accessors must be the same!");
+
+    using reduced_storage1d = gko::acc::range<accessor1d>;
+    using reduced_storage2d = gko::acc::range<accessor2d>;
+    using const_reduced_storage2d = gko::acc::range<const_accessor2d>;
+    using const_reduced_storage1d = gko::acc::range<const_accessor1d>;
+
+    const std::array<size_type, 0> stride0{{}};
+    const std::array<size_type, 1> stride1{{4}};
+    const std::array<size_type, 1> stride_sc{{5}};
+    const std::array<gko::size_type, 1> size_1d{{8u}};
+    const std::array<gko::size_type, 2> size_2d{{2u, 2u}};
+
+    static constexpr gko::size_type data_elements{8};
+    st_type data[data_elements]{10, 22, 32, 44, 54, 66, 76, -88};
+    ar_type scalar[data_elements]{1e0,  5e-1, 1e-1, 5e-2,
+                                  1e-2, 5e-3, 1e-3, 5e-4};
+
+    reduced_storage1d r1{size_1d, data, scalar};
+    reduced_storage2d r2{size_2d, data, stride1, scalar, stride_sc};
+    const_reduced_storage1d cr1{size_1d, data, stride0, scalar};
+    const_reduced_storage2d cr2{size_2d, data, stride1, scalar, stride_sc};
+
+    void data_equal_except_for(int idx)
+    {
+        // clang-format off
+        if (idx != 0) { EXPECT_EQ(data[0], 10); }
+        if (idx != 1) { EXPECT_EQ(data[1], 22); }
+        if (idx != 2) { EXPECT_EQ(data[2], 32); }
+        if (idx != 3) { EXPECT_EQ(data[3], 44); }
+        if (idx != 4) { EXPECT_EQ(data[4], 54); }
+        if (idx != 5) { EXPECT_EQ(data[5], 66); }
+        if (idx != 6) { EXPECT_EQ(data[6], 76); }
+        if (idx != 7) { EXPECT_EQ(data[7], -88); }
+        // clang-format on
+    }
+    void scalar_equal_except_for(int idx)
+    {
+        // clang-format off
+        if (idx != 0) { EXPECT_EQ(scalar[0], ar_type{1e0}); }
+        if (idx != 1) { EXPECT_EQ(scalar[1], ar_type{5e-1}); }
+        if (idx != 2) { EXPECT_EQ(scalar[2], ar_type{1e-1}); }
+        if (idx != 3) { EXPECT_EQ(scalar[3], ar_type{5e-2}); }
+        if (idx != 4) { EXPECT_EQ(scalar[4], ar_type{1e-2}); }
+        if (idx != 5) { EXPECT_EQ(scalar[5], ar_type{5e-3}); }
+        if (idx != 6) { EXPECT_EQ(scalar[6], ar_type{1e-3}); }
+        if (idx != 7) { EXPECT_EQ(scalar[7], ar_type{5e-4}); }
+        // clang-format on
+    }
+};
+
+
+TEST_F(ScaledReducedStorageXd, CanRead)
+{
+    EXPECT_NEAR(cr1(1), 11., delta);
+    EXPECT_NEAR(cr2(0, 1), 11., delta);
+    EXPECT_NEAR(cr2(1, 1), 66e-3, delta);
+    EXPECT_NEAR(r1(1), 11., delta);
+    EXPECT_NEAR(r2(0, 1), 11., delta);
+    EXPECT_NEAR(r2(1, 1), 66e-3, delta);
+}
+
+
+TEST_F(ScaledReducedStorageXd, CanWrite1)
+{
+    r1(2) = 0.2;
+
+    data_equal_except_for(2);
+    scalar_equal_except_for(99);
+    EXPECT_NEAR(r1(2), 0.2, delta);
+}
+
+
+TEST_F(ScaledReducedStorageXd, CanWrite2)
+{
+    r2(1, 1) = 0.5;
+
+    data_equal_except_for(5);
+    scalar_equal_except_for(99);
+    EXPECT_NEAR(r2(1, 1), 0.5, delta);
+}
+
+
+}  // namespace
diff --git a/core/test/base/CMakeLists.txt b/core/test/base/CMakeLists.txt
index 7b1b2cefddf..6f8c7291165 100644
--- a/core/test/base/CMakeLists.txt
+++ b/core/test/base/CMakeLists.txt
@@ -1,5 +1,4 @@
 ginkgo_create_test(abstract_factory)
-ginkgo_create_test(accessors_reduced_storage)
 ginkgo_create_test(allocator)
 ginkgo_create_test(array)
 ginkgo_create_test(combination)

From 8792dbeb51a9e4b963df3529c4166ce0a54cfe42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 24 Feb 2021 06:49:45 +0100
Subject: [PATCH 0491/1008] Sort and re-format contributors.txt

---
 contributors.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contributors.txt b/contributors.txt
index 0700d45fa78..fde5b54d2c3 100644
--- a/contributors.txt
+++ b/contributors.txt
@@ -1,6 +1,7 @@
 I hereby place all my contributions in this codebase under a BSD-3-Clause
 license, as specified in the repository's LICENSE file.
 
+Aliaga José I. <aliaga@uji.es> Universitat Jaume I
 Anzt Hartwig <hartwig.anzt@kit.edu> Karlsruhe Institute of Technology, The University of Tennessee Knoxville
 Boman Erik <egboman@sandia.gov> Sandia National Laboratories
 Chen Yenchen <yanjen224@gmail.com> National Taiwan University
@@ -19,4 +20,3 @@ Olenik Gregor <go@hpsim.de> HPSim
 Ribizel Tobias <mail@upsj.de> Karlsruhe Institute of Technology
 Riemer Lukas <lksriemer@gmail.com> Karlsruhe Institute of Technology
 Tsai Yuhsiang <yhmtsai@gmail.com> National Taiwan University
-José I. Aliaga <aliaga@uji.es> Universitat Jaume I

From 6778a6fd3da0934749fe529b92eb2b24776cc144 Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Wed, 24 Feb 2021 06:09:11 +0000
Subject: [PATCH 0492/1008] Format files
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thoasm@users.noreply.github.com>
---
 core/test/accessor/index_span.cpp               | 4 ++--
 core/test/accessor/range.cpp                    | 6 +++---
 core/test/accessor/reduced_row_major.cpp        | 4 +---
 core/test/accessor/scaled_reduced_row_major.cpp | 4 +---
 4 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/core/test/accessor/index_span.cpp b/core/test/accessor/index_span.cpp
index 690c7d7bb27..3c69141d764 100644
--- a/core/test/accessor/index_span.cpp
+++ b/core/test/accessor/index_span.cpp
@@ -30,10 +30,10 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "accessor/index_span.hpp"
+#include <gtest/gtest.h>
 
 
-#include <gtest/gtest.h>
+#include "accessor/index_span.hpp"
 
 
 namespace {
diff --git a/core/test/accessor/range.cpp b/core/test/accessor/range.cpp
index 63b84e16f98..3708486b0f2 100644
--- a/core/test/accessor/range.cpp
+++ b/core/test/accessor/range.cpp
@@ -30,9 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "accessor/range.hpp"
-
-
 #include <array>
 #include <cstddef>
 
@@ -40,6 +37,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
+#include "accessor/range.hpp"
+
+
 namespace {
 
 
diff --git a/core/test/accessor/reduced_row_major.cpp b/core/test/accessor/reduced_row_major.cpp
index 027b68da55a..745c7e09b1d 100644
--- a/core/test/accessor/reduced_row_major.cpp
+++ b/core/test/accessor/reduced_row_major.cpp
@@ -30,9 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "accessor/reduced_row_major.hpp"
-
-
 #include <array>
 #include <tuple>
 #include <type_traits>
@@ -46,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "accessor/index_span.hpp"
+#include "accessor/reduced_row_major.hpp"
 #include "core/base/extended_float.hpp"
 #include "core/test/utils.hpp"
 
diff --git a/core/test/accessor/scaled_reduced_row_major.cpp b/core/test/accessor/scaled_reduced_row_major.cpp
index d7418716fe2..7d8f95c981a 100644
--- a/core/test/accessor/scaled_reduced_row_major.cpp
+++ b/core/test/accessor/scaled_reduced_row_major.cpp
@@ -30,9 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "accessor/scaled_reduced_row_major.hpp"
-
-
 #include <array>
 #include <tuple>
 #include <type_traits>
@@ -46,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "accessor/index_span.hpp"
+#include "accessor/scaled_reduced_row_major.hpp"
 #include "core/base/extended_float.hpp"
 #include "core/test/utils.hpp"
 

From 8670aa06894bd7edd30ef8c62adeb4773949d7b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 24 Feb 2021 07:58:06 +0100
Subject: [PATCH 0493/1008] Move row_major and block_col_major to accessor/

Only the new multidimensional row_major is moved, the 2D row_major that
is part of the public interface remains.
Additionally, unnecessary code from public accessors was removed.
The code that was removed was never in the release and originally
intended for usage inside the accessors. Since they moved to an
independent directory, that code is no longer necessary.
This removal prevents it from ever becomming part of the public
interface.
---
 accessor/accessor_helper.hpp                 | 330 +++++---
 accessor/block_col_major.hpp                 | 228 ++++++
 accessor/row_major.hpp                       | 227 ++++++
 core/test/accessor/CMakeLists.txt            |   3 +
 core/test/accessor/block_col_major.cpp       | 187 +++++
 core/test/accessor/row_major.cpp             | 290 +++++++
 core/test/base/dim.cpp                       |  61 --
 core/test/base/range_accessors.cpp           | 273 -------
 include/ginkgo/core/base/dim.hpp             |  73 --
 include/ginkgo/core/base/range_accessors.hpp | 750 +------------------
 10 files changed, 1177 insertions(+), 1245 deletions(-)
 create mode 100644 accessor/block_col_major.hpp
 create mode 100644 accessor/row_major.hpp
 create mode 100644 core/test/accessor/block_col_major.cpp
 create mode 100644 core/test/accessor/row_major.cpp

diff --git a/accessor/accessor_helper.hpp b/accessor/accessor_helper.hpp
index ee2178a41d6..ae29995f3ce 100644
--- a/accessor/accessor_helper.hpp
+++ b/accessor/accessor_helper.hpp
@@ -63,7 +63,8 @@ namespace detail {
  * indices: x1, x2, x3, ...
  * compute(stride, x1, x2, x3) -> x1 * stride[0] + x2 * stride[1] + x3
  */
-template <typename ValueType, size_type total_dim, size_type current_iter = 1>
+template <typename ValueType, size_type total_dim, typename DimensionType,
+          size_type current_iter = 1>
 struct row_major_helper_s {
     static_assert(total_dim >= 1, "Dimensionality must be >= 1");
     static_assert(current_iter < total_dim, "Iteration must be < total_dim!");
@@ -72,7 +73,7 @@ struct row_major_helper_s {
 
     template <typename FirstType, typename... Indices>
     static constexpr GKO_ACC_ATTRIBUTES ValueType
-    compute(const std::array<size_type, total_dim> &size,
+    compute(const std::array<DimensionType, total_dim> &size,
             const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
                 &stride,
             FirstType first, Indices &&... idxs)
@@ -84,16 +85,19 @@ struct row_major_helper_s {
         // namespace scope), it can't be odr-used.
         return GKO_ACC_ASSERT(first < size[static_cast<size_type>(dim_idx)]),
                first * stride[dim_idx] +
-                   row_major_helper_s<ValueType, total_dim, current_iter + 1>::
-                       compute(size, stride, std::forward<Indices>(idxs)...);
+                   row_major_helper_s<ValueType, total_dim, DimensionType,
+                                      current_iter +
+                                          1>::compute(size, stride,
+                                                      std::forward<Indices>(
+                                                          idxs)...);
     }
 };
 
-template <typename ValueType, size_type total_dim>
-struct row_major_helper_s<ValueType, total_dim, total_dim> {
+template <typename ValueType, size_type total_dim, typename DimensionType>
+struct row_major_helper_s<ValueType, total_dim, DimensionType, total_dim> {
     template <typename FirstType>
     static constexpr GKO_ACC_ATTRIBUTES ValueType
-    compute(const std::array<size_type, total_dim> &size,
+    compute(const std::array<DimensionType, total_dim> &size,
             const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>,
             FirstType first)
     {
@@ -150,14 +154,17 @@ compute_default_row_major_stride_array(const std::array<DimensionType, N> &size,
  * Computes the storage index for the given indices with respect to the given
  * stride array
  */
-template <typename ValueType, size_type total_dim, typename... Indices>
+template <typename ValueType, size_type total_dim, typename DimensionType,
+          typename... Indices>
 constexpr GKO_ACC_ATTRIBUTES ValueType compute_row_major_index(
-    const std::array<size_type, total_dim> &size,
+    const std::array<DimensionType, total_dim> &size,
     const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)> &stride,
     Indices &&... idxs)
 {
-    return detail::row_major_helper_s<ValueType, total_dim>::compute(
-        size, stride, std::forward<Indices>(idxs)...);
+    return detail::row_major_helper_s<
+        ValueType, total_dim, DimensionType>::compute(size, stride,
+                                                      std::forward<Indices>(
+                                                          idxs)...);
 }
 
 
@@ -206,6 +213,7 @@ namespace detail {
  */
 template <typename ValueType, size_type mask, size_type set_bits_processed,
           size_type stride_size, size_type dim_idx, size_type total_dim,
+          typename DimensionType,
           // Determine if the bit in the mask is set for dim_idx
           // If the end is reached (dim_idx == total_dim), set it to false in
           // order to only need one specialization
@@ -218,9 +226,11 @@ struct row_major_masked_helper_s {};
 
 // bit for current dimensionality is set
 template <typename ValueType, size_type mask, size_type set_bits_processed,
-          size_type stride_size, size_type dim_idx, size_type total_dim>
+          size_type stride_size, size_type dim_idx, size_type total_dim,
+          typename DimensionType>
 struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
-                                 stride_size, dim_idx, total_dim, true> {
+                                 stride_size, dim_idx, total_dim, DimensionType,
+                                 true> {
     static_assert(mask &
                       (size_type{1}
                        << (dim_idx < total_dim ? total_dim - 1 - dim_idx : 0)),
@@ -232,26 +242,28 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
 
     template <typename... Args>
     static constexpr GKO_ACC_ATTRIBUTES std::array<ValueType, stride_size>
-    build_stride(const std::array<size_type, total_dim> &size, Args &&... args)
+    build_stride(const std::array<DimensionType, total_dim> &size,
+                 Args &&... args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
-            total_dim>::build_stride(size, std::forward<Args>(args)...,
-                                     mult_size_upwards(size));
+            total_dim, DimensionType>::build_stride(size,
+                                                    std::forward<Args>(args)...,
+                                                    mult_size_upwards(size));
     }
 
     static constexpr GKO_ACC_ATTRIBUTES ValueType
-    mult_size_upwards(const std::array<size_type, total_dim> &size)
+    mult_size_upwards(const std::array<DimensionType, total_dim> &size)
     {
-        return size[dim_idx] *
-               row_major_masked_helper_s<
-                   ValueType, mask, set_bits_processed + 1, stride_size,
-                   dim_idx + 1, total_dim>::mult_size_upwards(size);
+        return size[dim_idx] * row_major_masked_helper_s<
+                                   ValueType, mask, set_bits_processed + 1,
+                                   stride_size, dim_idx + 1, total_dim,
+                                   DimensionType>::mult_size_upwards(size);
     }
 
     template <typename First, typename... Indices>
     static constexpr GKO_ACC_ATTRIBUTES ValueType
-    compute_mask_idx(const std::array<size_type, total_dim> &size,
+    compute_mask_idx(const std::array<DimensionType, total_dim> &size,
                      const std::array<ValueType, stride_size> &stride,
                      First first, Indices &&... idxs)
     {
@@ -264,15 +276,15 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
                             : stride[set_bits_processed]) +
                    row_major_masked_helper_s<
                        ValueType, mask, set_bits_processed + 1, stride_size,
-                       dim_idx + 1,
-                       total_dim>::compute_mask_idx(size, stride,
-                                                    std::forward<Indices>(
-                                                        idxs)...);
+                       dim_idx + 1, total_dim,
+                       DimensionType>::compute_mask_idx(size, stride,
+                                                        std::forward<Indices>(
+                                                            idxs)...);
     }
 
     template <typename First, typename... Indices>
     static constexpr GKO_ACC_ATTRIBUTES ValueType
-    compute_direct_idx(const std::array<size_type, total_dim> &size,
+    compute_direct_idx(const std::array<DimensionType, total_dim> &size,
                        const std::array<ValueType, stride_size> &stride,
                        First first, Indices &&... idxs)
     {
@@ -285,19 +297,19 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
                             : stride[set_bits_processed]) +
                    row_major_masked_helper_s<
                        ValueType, mask, set_bits_processed + 1, stride_size,
-                       dim_idx + 1,
-                       total_dim>::compute_direct_idx(size, stride,
-                                                      std::forward<Indices>(
-                                                          idxs)...);
+                       dim_idx + 1, total_dim,
+                       DimensionType>::compute_direct_idx(size, stride,
+                                                          std::forward<Indices>(
+                                                              idxs)...);
     }
 };
 
 // first set bit (from the left) for current dimensionality encountered:
 //     Do not calculate stride value for it (since no lower dimension needs it)!
 template <typename ValueType, size_type mask, size_type stride_size,
-          size_type dim_idx, size_type total_dim>
+          size_type dim_idx, size_type total_dim, typename DimensionType>
 struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
-                                 total_dim, true> {
+                                 total_dim, DimensionType, true> {
     static constexpr size_type set_bits_processed{0};
     static_assert(mask &
                       (size_type{1}
@@ -308,24 +320,27 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
 
     template <typename... Args>
     static constexpr GKO_ACC_ATTRIBUTES std::array<ValueType, stride_size>
-    build_stride(const std::array<size_type, total_dim> &size, Args &&... args)
+    build_stride(const std::array<DimensionType, total_dim> &size,
+                 Args &&... args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
-            total_dim>::build_stride(size, std::forward<Args>(args)...);
+            total_dim, DimensionType>::build_stride(size,
+                                                    std::forward<Args>(
+                                                        args)...);
     }
 
     static constexpr GKO_ACC_ATTRIBUTES ValueType
-    mult_size_upwards(const std::array<size_type, total_dim> &size)
+    mult_size_upwards(const std::array<DimensionType, total_dim> &size)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed + 1, stride_size, dim_idx + 1,
-            total_dim>::mult_size_upwards(size);
+            total_dim, DimensionType>::mult_size_upwards(size);
     }
 
     template <typename First, typename... Indices>
     static constexpr GKO_ACC_ATTRIBUTES ValueType
-    compute_mask_idx(const std::array<size_type, total_dim> &size,
+    compute_mask_idx(const std::array<DimensionType, total_dim> &size,
                      const std::array<ValueType, stride_size> &stride,
                      First first, Indices &&... idxs)
     {
@@ -338,15 +353,15 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
                             : stride[set_bits_processed]) +
                    row_major_masked_helper_s<
                        ValueType, mask, set_bits_processed + 1, stride_size,
-                       dim_idx + 1,
-                       total_dim>::compute_mask_idx(size, stride,
-                                                    std::forward<Indices>(
-                                                        idxs)...);
+                       dim_idx + 1, total_dim,
+                       DimensionType>::compute_mask_idx(size, stride,
+                                                        std::forward<Indices>(
+                                                            idxs)...);
     }
 
     template <typename First, typename... Indices>
     static constexpr GKO_ACC_ATTRIBUTES ValueType
-    compute_direct_idx(const std::array<size_type, total_dim> &size,
+    compute_direct_idx(const std::array<DimensionType, total_dim> &size,
                        const std::array<ValueType, stride_size> &stride,
                        First first, Indices &&... idxs)
     {
@@ -359,18 +374,20 @@ struct row_major_masked_helper_s<ValueType, mask, 0, stride_size, dim_idx,
                             : stride[set_bits_processed]) +
                    row_major_masked_helper_s<
                        ValueType, mask, set_bits_processed + 1, stride_size,
-                       dim_idx + 1,
-                       total_dim>::compute_direct_idx(size, stride,
-                                                      std::forward<Indices>(
-                                                          idxs)...);
+                       dim_idx + 1, total_dim,
+                       DimensionType>::compute_direct_idx(size, stride,
+                                                          std::forward<Indices>(
+                                                              idxs)...);
     }
 };
 
 // bit for current dimensionality is not set
 template <typename ValueType, size_type mask, size_type set_bits_processed,
-          size_type stride_size, size_type dim_idx, size_type total_dim>
+          size_type stride_size, size_type dim_idx, size_type total_dim,
+          typename DimensionType>
 struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
-                                 stride_size, dim_idx, total_dim, false> {
+                                 stride_size, dim_idx, total_dim, DimensionType,
+                                 false> {
     static_assert((mask & (size_type{1}
                            << (dim_idx < total_dim ? total_dim - 1 - dim_idx
                                                    : 0))) == 0,
@@ -381,24 +398,27 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
                   "The processed bits must be < total number of set bits!");
     template <typename... Args>
     static constexpr GKO_ACC_ATTRIBUTES std::array<ValueType, stride_size>
-    build_stride(const std::array<size_type, total_dim> &size, Args &&... args)
+    build_stride(const std::array<DimensionType, total_dim> &size,
+                 Args &&... args)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed, stride_size, dim_idx + 1,
-            total_dim>::build_stride(size, std::forward<Args>(args)...);
+            total_dim, DimensionType>::build_stride(size,
+                                                    std::forward<Args>(
+                                                        args)...);
     }
 
     static constexpr GKO_ACC_ATTRIBUTES ValueType
-    mult_size_upwards(const std::array<size_type, total_dim> &size)
+    mult_size_upwards(const std::array<DimensionType, total_dim> &size)
     {
-        return row_major_masked_helper_s<ValueType, mask, set_bits_processed,
-                                         stride_size, dim_idx + 1,
-                                         total_dim>::mult_size_upwards(size);
+        return row_major_masked_helper_s<
+            ValueType, mask, set_bits_processed, stride_size, dim_idx + 1,
+            total_dim, DimensionType>::mult_size_upwards(size);
     }
 
     template <typename First, typename... Indices>
     static constexpr GKO_ACC_ATTRIBUTES ValueType
-    compute_mask_idx(const std::array<size_type, total_dim> &size,
+    compute_mask_idx(const std::array<DimensionType, total_dim> &size,
                      const std::array<ValueType, stride_size> &stride, First,
                      Indices &&... idxs)
     {
@@ -406,51 +426,54 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
                       "Mismatching number of Idxs!");
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed, stride_size, dim_idx + 1,
-            total_dim>::compute_mask_idx(size, stride,
-                                         std::forward<Indices>(idxs)...);
+            total_dim, DimensionType>::compute_mask_idx(size, stride,
+                                                        std::forward<Indices>(
+                                                            idxs)...);
     }
 
     template <typename... Indices>
     static constexpr GKO_ACC_ATTRIBUTES ValueType compute_direct_idx(
-        const std::array<size_type, total_dim> &size,
+        const std::array<DimensionType, total_dim> &size,
         const std::array<ValueType, stride_size> &stride, Indices &&... idxs)
     {
         return row_major_masked_helper_s<
             ValueType, mask, set_bits_processed, stride_size, dim_idx + 1,
-            total_dim>::compute_direct_idx(size, stride,
-                                           std::forward<Indices>(idxs)...);
+            total_dim, DimensionType>::compute_direct_idx(size, stride,
+                                                          std::forward<Indices>(
+                                                              idxs)...);
     }
 };
 
 // Specialization for the end of recursion: build_stride array from created
 // arguments
 template <typename ValueType, size_type mask, size_type set_bits_processed,
-          size_type stride_size, size_type total_dim>
+          size_type stride_size, size_type total_dim, typename DimensionType>
 struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
-                                 stride_size, total_dim, total_dim, false> {
+                                 stride_size, total_dim, total_dim,
+                                 DimensionType, false> {
     static_assert(set_bits_processed <= stride_size + 1,
                   "The processed bits must be smaller than the total number of "
                   "set bits!");
     template <typename... Args>
     static constexpr GKO_ACC_ATTRIBUTES std::array<ValueType, stride_size>
-    build_stride(const std::array<size_type, total_dim> &, Args &&... args)
+    build_stride(const std::array<DimensionType, total_dim> &, Args &&... args)
     {
         return {{std::forward<Args>(args)...}};
     }
     static constexpr GKO_ACC_ATTRIBUTES ValueType
-    mult_size_upwards(const std::array<size_type, total_dim> &)
+    mult_size_upwards(const std::array<DimensionType, total_dim> &)
     {
         return 1;
     }
 
     static constexpr GKO_ACC_ATTRIBUTES ValueType
-    compute_mask_idx(const std::array<size_type, total_dim> &,
+    compute_mask_idx(const std::array<DimensionType, total_dim> &,
                      const std::array<ValueType, stride_size> &)
     {
         return 0;
     }
     static constexpr GKO_ACC_ATTRIBUTES ValueType
-    compute_direct_idx(const std::array<size_type, total_dim> &,
+    compute_direct_idx(const std::array<DimensionType, total_dim> &,
                        const std::array<ValueType, stride_size> &)
     {
         return 0;
@@ -466,15 +489,15 @@ struct row_major_masked_helper_s<ValueType, mask, set_bits_processed,
  * Only indices are considered where the corresponding mask bit is set.
  */
 template <typename ValueType, size_type mask, size_type stride_size,
-          size_type total_dim, typename... Indices>
+          size_type total_dim, typename DimensionType, typename... Indices>
 constexpr GKO_ACC_ATTRIBUTES auto compute_masked_index(
-    const std::array<size_type, total_dim> &size,
+    const std::array<DimensionType, total_dim> &size,
     const std::array<ValueType, stride_size> &stride, Indices &&... idxs)
 {
     return detail::row_major_masked_helper_s<
-        ValueType, mask, 0, stride_size, 0,
-        total_dim>::compute_mask_idx(size, stride,
-                                     std::forward<Indices>(idxs)...);
+        ValueType, mask, 0, stride_size, 0, total_dim,
+        DimensionType>::compute_mask_idx(size, stride,
+                                         std::forward<Indices>(idxs)...);
 }
 
 
@@ -482,15 +505,15 @@ constexpr GKO_ACC_ATTRIBUTES auto compute_masked_index(
  * Computes the memory index for the given indices considering the stride.
  */
 template <typename ValueType, size_type mask, size_type stride_size,
-          size_type total_dim, typename... Indices>
+          size_type total_dim, typename DimensionType, typename... Indices>
 constexpr GKO_ACC_ATTRIBUTES auto compute_masked_index_direct(
-    const std::array<size_type, total_dim> &size,
+    const std::array<DimensionType, total_dim> &size,
     const std::array<ValueType, stride_size> &stride, Indices &&... idxs)
 {
     return detail::row_major_masked_helper_s<
-        ValueType, mask, 0, stride_size, 0,
-        total_dim>::compute_direct_idx(size, stride,
-                                       std::forward<Indices>(idxs)...);
+        ValueType, mask, 0, stride_size, 0, total_dim,
+        DimensionType>::compute_direct_idx(size, stride,
+                                           std::forward<Indices>(idxs)...);
 }
 
 
@@ -500,12 +523,13 @@ constexpr GKO_ACC_ATTRIBUTES auto compute_masked_index_direct(
  * padding
  */
 template <typename ValueType, size_type mask, size_type stride_size,
-          size_type total_dim>
+          size_type total_dim, typename DimensionType>
 constexpr GKO_ACC_ATTRIBUTES auto compute_default_masked_row_major_stride_array(
-    const std::array<size_type, total_dim> &size)
+    const std::array<DimensionType, total_dim> &size)
 {
     return detail::row_major_masked_helper_s<ValueType, mask, 0, stride_size, 0,
-                                             total_dim>::build_stride(size);
+                                             total_dim,
+                                             DimensionType>::build_stride(size);
 }
 
 
@@ -542,18 +566,22 @@ using are_index_span_compatible =
 namespace detail {
 
 
-template <size_type iter, size_type N, typename Callable, typename... Indices>
+template <size_type iter, size_type N, typename DimensionType,
+          typename Callable, typename... Indices>
 GKO_ACC_ATTRIBUTES std::enable_if_t<iter == N> multidim_for_each_impl(
-    const std::array<size_type, N> &, Callable callable, Indices &&... indices)
+    const std::array<DimensionType, N> &, Callable callable,
+    Indices &&... indices)
 {
     static_assert(iter == sizeof...(Indices),
                   "Number arguments must match current iteration!");
     callable(std::forward<Indices>(indices)...);
 }
 
-template <size_type iter, size_type N, typename Callable, typename... Indices>
+template <size_type iter, size_type N, typename DimensionType,
+          typename Callable, typename... Indices>
 GKO_ACC_ATTRIBUTES std::enable_if_t<(iter < N)> multidim_for_each_impl(
-    const std::array<size_type, N> &size, Callable callable, Indices... indices)
+    const std::array<DimensionType, N> &size, Callable callable,
+    Indices... indices)
 {
     static_assert(iter == sizeof...(Indices),
                   "Number arguments must match current iteration!");
@@ -570,9 +598,9 @@ GKO_ACC_ATTRIBUTES std::enable_if_t<(iter < N)> multidim_for_each_impl(
  * Creates a recursive for-loop for each dimension and calls dest(indices...) =
  * source(indices...)
  */
-template <size_type N, typename Callable>
-GKO_ACC_ATTRIBUTES void multidim_for_each(const std::array<size_type, N> &size,
-                                          Callable &&callable)
+template <size_type N, typename DimensionType, typename Callable>
+GKO_ACC_ATTRIBUTES void multidim_for_each(
+    const std::array<DimensionType, N> &size, Callable &&callable)
 {
     detail::multidim_for_each_impl<0>(size, std::forward<Callable>(callable));
 }
@@ -581,16 +609,17 @@ GKO_ACC_ATTRIBUTES void multidim_for_each(const std::array<size_type, N> &size,
 namespace detail {
 
 
-template <size_type iter, size_type N>
+template <size_type iter, typename DimensionType, size_type N>
 constexpr GKO_ACC_ATTRIBUTES std::enable_if_t<iter == N, int>
-index_spans_in_size(const std::array<size_type, N> &)
+index_spans_in_size(const std::array<DimensionType, N> &)
 {
     return 0;
 }
 
-template <size_type iter, size_type N, typename First, typename... Remaining>
+template <size_type iter, typename DimensionType, size_type N, typename First,
+          typename... Remaining>
 constexpr GKO_ACC_ATTRIBUTES std::enable_if_t<(iter < N), int>
-index_spans_in_size(const std::array<size_type, N> &size, First first,
+index_spans_in_size(const std::array<DimensionType, N> &size, First first,
                     Remaining &&... remaining)
 {
     static_assert(sizeof...(Remaining) + 1 == N - iter,
@@ -605,9 +634,9 @@ index_spans_in_size(const std::array<size_type, N> &size, First first,
 }  // namespace detail
 
 
-template <size_type N, typename... Spans>
+template <typename DimensionType, size_type N, typename... Spans>
 constexpr GKO_ACC_ATTRIBUTES int validate_index_spans(
-    const std::array<size_type, N> &size, Spans &&... spans)
+    const std::array<DimensionType, N> &size, Spans &&... spans)
 {
     return detail::index_spans_in_size<0>(size, std::forward<Spans>(spans)...);
 }
@@ -642,6 +671,121 @@ constexpr size_type count_mask_dimensionality()
 }
 
 
+/**
+ * Namespace for helper functions and structs for
+ * the block column major accessor.
+ */
+namespace blk_col_major {
+
+
+/**
+ * Runs from first to last dimension in order to compute the index.
+ *
+ * The index is computed like this:
+ * indices: x1, x2, x3, ..., xn
+ * compute(stride, x1, x2, x3, ..., x(n-1), xn) ->
+ *  x1 * stride[0] + x2 * stride[1] + ...
+ *    + x(n-2) * stride[n-3] + x(n-1) + xn * stride[n-2]
+ * Note that swap of the last two strides, making this 'block column major'.
+ */
+template <typename ValueType, size_type total_dim, size_type current_iter = 1>
+struct index_helper_s {
+    static_assert(total_dim >= 1, "Dimensionality must be >= 1");
+    static_assert(current_iter <= total_dim, "Iteration must be <= total_dim!");
+
+    static constexpr size_type dim_idx{current_iter - 1};
+
+    template <typename FirstType, typename... Indices>
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    compute(const std::array<ValueType, total_dim> &size,
+            const std::array<ValueType, (total_dim > 0 ? total_dim - 1 : 0)>
+                &stride,
+            FirstType first, Indices &&... idxs)
+    {
+        if (current_iter == total_dim - 1) {
+            return GKO_ACC_ASSERT(first < size[dim_idx]),
+                   first +
+                       index_helper_s<ValueType, total_dim, current_iter + 1>::
+                           compute(size, stride,
+                                   std::forward<Indices>(idxs)...);
+        }
+
+        return GKO_ACC_ASSERT(first < size[dim_idx]),
+               first * stride[dim_idx] +
+                   index_helper_s<ValueType, total_dim, current_iter + 1>::
+                       compute(size, stride, std::forward<Indices>(idxs)...);
+    }
+};
+
+template <typename ValueType, size_type total_dim>
+struct index_helper_s<ValueType, total_dim, total_dim> {
+    static_assert(total_dim >= 2, "Dimensionality must be >= 2");
+
+    static constexpr size_type dim_idx{total_dim - 1};
+
+    template <typename FirstType>
+    static constexpr GKO_ACC_ATTRIBUTES ValueType
+    compute(const std::array<ValueType, total_dim> &size,
+            const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
+                &stride,
+            FirstType first)
+    {
+        return GKO_ACC_ASSERT(first < size[total_dim - 1]),
+               first * stride[dim_idx - 1];
+    }
+};
+
+/**
+ * Computes the flat storage index for block-column-major access.
+ *
+ * @param size  the multi-dimensional sizes of the range of values
+ * @param stride  the stride array
+ * @param idxs  the multi-dimensional indices of the desired entry
+ */
+template <typename ValueType, size_type total_dim, typename... Indices>
+constexpr GKO_ACC_ATTRIBUTES ValueType compute_index(
+    const std::array<ValueType, total_dim> &size,
+    const std::array<ValueType, (total_dim > 0 ? total_dim - 1 : 0)> &stride,
+    Indices &&... idxs)
+{
+    return index_helper_s<ValueType, total_dim>::compute(
+        size, stride, std::forward<Indices>(idxs)...);
+}
+
+
+template <size_type iter = 1, typename ValueType, size_type N, typename... Args>
+constexpr GKO_ACC_ATTRIBUTES
+    std::enable_if_t<(iter == N - 1) && (iter == sizeof...(Args) + 1),
+                     std::array<ValueType, N - 1>>
+    default_stride_array_impl(const std::array<ValueType, N> &size,
+                              Args &&... args)
+{
+    return {{std::forward<Args>(args)..., size[N - 2]}};
+}
+
+template <size_type iter = 1, typename ValueType, size_type N, typename... Args>
+constexpr GKO_ACC_ATTRIBUTES std::enable_if_t<(iter < N - 1 || iter == N) &&
+                                                  (iter == sizeof...(Args) + 1),
+                                              std::array<ValueType, N - 1>>
+default_stride_array_impl(const std::array<ValueType, N> &size, Args &&... args)
+{
+    return default_stride_array_impl<iter + 1>(
+        size, std::forward<Args>(args)...,
+        detail::mult_dim_upwards<size_type, iter>(size));
+}
+
+template <typename ValueType, size_type dimensions>
+constexpr GKO_ACC_ATTRIBUTES
+    std::array<ValueType, (dimensions > 0 ? dimensions - 1 : 0)>
+    default_stride_array(const std::array<ValueType, dimensions> &size)
+{
+    return default_stride_array_impl(size);
+}
+
+
+}  // namespace blk_col_major
+
+
 }  // namespace helper
 }  // namespace acc
 }  // namespace gko
diff --git a/accessor/block_col_major.hpp b/accessor/block_col_major.hpp
new file mode 100644
index 00000000000..275dc387f43
--- /dev/null
+++ b/accessor/block_col_major.hpp
@@ -0,0 +1,228 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_ACCESSOR_BLOCK_COL_MAJOR_HPP_
+#define GKO_ACCESSOR_BLOCK_COL_MAJOR_HPP_
+
+#include <array>
+
+#include "accessor_helper.hpp"
+#include "range.hpp"
+#include "utils.hpp"
+
+
+namespace gko {
+namespace acc {
+
+
+/**
+ * A bridge between a range and a block-column-major memory layout.
+ *
+ * Only the innermost two dimensions are regarded as defining
+ * a column-major matrix, and the rest of the dimensions are treated
+ * identically to \ref row_major.
+ *
+ * You should not try to explicitly create an instance of this accessor.
+ * Instead, supply it as a template parameter to a range, and pass the
+ * constructor parameters for this class to the range (it will forward it to
+ * this class).
+ *
+ * @tparam ValueType  type of values this accessor returns
+ * @tparam Dimensionality  number of dimensions of this accessor
+ */
+template <typename ValueType, size_type Dimensionality>
+class block_col_major {
+public:
+    friend class range<block_col_major>;
+
+    static_assert(Dimensionality != 0,
+                  "This accessor does not support a dimensionality of 0!");
+    static_assert(Dimensionality != 1,
+                  "Please use row_major accessor for 1D ranges.");
+
+    /**
+     * Number of dimensions of the accessor.
+     */
+    static constexpr size_type dimensionality = Dimensionality;
+
+    /**
+     * Type of values returned by the accessor.
+     */
+    using value_type = ValueType;
+
+    /**
+     * Type of underlying data storage.
+     */
+    using data_type = value_type *;
+
+    using const_accessor = block_col_major<const ValueType, Dimensionality>;
+    using stride_type = std::array<const size_type, dimensionality - 1>;
+    using length_type = std::array<const size_type, dimensionality>;
+
+protected:
+    /**
+     * Creates a block_col_major accessor.
+     *
+     * @param data  pointer to the block of memory containing the data
+     * @param lengths size / length of the accesses of each dimension
+     * @param stride  distance (in elements) between starting positions of
+     *                the dimensions (i.e.
+     *   `x_1 * stride_1 + x_2 * stride_2 * ... + x_(n-1) + x_n * stride_(n-1)`
+     *                points to the element at (x_1, x_2, ..., x_n))
+     */
+    constexpr GKO_ACC_ATTRIBUTES explicit block_col_major(data_type data,
+                                                          length_type size,
+                                                          stride_type stride)
+        : data{data}, lengths(size), stride(stride)
+    {}
+
+    /**
+     * Creates a block_col_major accessor with a default stride
+     * (assumes no padding)
+     *
+     * @param data  pointer to the block of memory containing the data
+     * @param lengths size / length of the accesses of each dimension
+     */
+    constexpr GKO_ACC_ATTRIBUTES explicit block_col_major(data_type data,
+                                                          length_type size)
+        : data{data},
+          lengths(size),
+          stride(helper::blk_col_major::default_stride_array(lengths))
+    {}
+
+public:
+    /**
+     * Creates a block_col_major range which contains a read-only version of
+     * the current accessor.
+     *
+     * @returns  a block column major range which is read-only.
+     */
+    constexpr GKO_ACC_ATTRIBUTES range<const_accessor> to_const() const
+    {
+        // TODO Remove this functionality all together (if requested)
+        return range<const_accessor>(data, lengths, stride);
+    }
+
+    /**
+     * Returns the data element at the specified indices
+     *
+     * @param row  row index
+     * @param col  column index
+     *
+     * @return data element at (indices...)
+     */
+    template <typename... Indices>
+    constexpr GKO_ACC_ATTRIBUTES
+        std::enable_if_t<are_all_integral<Indices...>::value, value_type &>
+        operator()(Indices &&... indices) const
+    {
+        return data[helper::blk_col_major::compute_index(
+            lengths, stride, std::forward<Indices>(indices)...)];
+    }
+
+    /**
+     * Returns the sub-range spanning the range (x1_span, x2_span, ...)
+     *
+     * @param rows  row span
+     * @param cols  column span
+     *
+     * @return sub-range spanning the given spans
+     */
+    template <typename... SpanTypes>
+    constexpr GKO_ACC_ATTRIBUTES
+        std::enable_if_t<helper::are_index_span_compatible<SpanTypes...>::value,
+                         range<block_col_major>>
+        operator()(SpanTypes... spans) const
+    {
+        return helper::validate_index_spans(lengths, spans...),
+               range<block_col_major>{
+                   data + helper::blk_col_major::compute_index(
+                              lengths, stride, (index_span{spans}.begin)...),
+                   length_type{
+                       (index_span{spans}.end - index_span{spans}.begin)...},
+                   stride};
+    }
+
+    /**
+     * Returns the length in dimension `dimension`.
+     *
+     * @param dimension  a dimension index
+     *
+     * @return length in dimension `dimension`
+     */
+    constexpr GKO_ACC_ATTRIBUTES size_type length(size_type dimension) const
+    {
+        return lengths[dimension];
+    }
+
+    /**
+     * Copies data from another accessor
+     *
+     * @warning Do not use this function since it is not optimized for a
+     *          specific executor. It will always be performed sequentially.
+     *          Please write an optimized version (adjusted to the architecture)
+     *          by iterating through the values yourself.
+     *
+     * @tparam OtherAccessor  type of the other accessor
+     *
+     * @param other  other accessor
+     */
+    template <typename OtherAccessor>
+    GKO_ACC_ATTRIBUTES void copy_from(const OtherAccessor &other) const
+    {
+        helper::multidim_for_each(lengths, [this, &other](auto... indices) {
+            (*this)(indices...) = other(indices...);
+        });
+    }
+
+    /**
+     * Reference to the underlying data.
+     */
+    const data_type data;
+
+    /**
+     * An array of dimension sizes.
+     */
+    const length_type lengths;
+
+    /**
+     * Distance between consecutive 'layers' for each dimension
+     * (except the second, for which it is 1).
+     */
+    const stride_type stride;
+};
+
+
+}  // namespace acc
+}  // namespace gko
+
+#endif  // GKO_ACCESSOR_BLOCK_COL_MAJOR_HPP_
diff --git a/accessor/row_major.hpp b/accessor/row_major.hpp
new file mode 100644
index 00000000000..1facb6d4b33
--- /dev/null
+++ b/accessor/row_major.hpp
@@ -0,0 +1,227 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_ACCESSOR_ROW_MAJOR_HPP_
+#define GKO_ACCESSOR_ROW_MAJOR_HPP_
+
+#include <array>
+
+#include "accessor_helper.hpp"
+#include "range.hpp"
+#include "utils.hpp"
+
+
+namespace gko {
+namespace acc {
+
+
+/**
+ * A row_major accessor is a bridge between a range and the row-major memory
+ * layout.
+ *
+ * You should never try to explicitly create an instance of this accessor.
+ * Instead, supply it as a template parameter to a range, and pass the
+ * constructor parameters for this class to the range (it will forward it to
+ * this class).
+ *
+ * @warning For backward compatability reasons, a specialization is provided
+ *          for dimensionality == 2.
+ *
+ * @tparam ValueType  type of values this accessor returns
+ * @tparam Dimensionality  number of dimensions of this accessor
+ */
+template <typename ValueType, size_type Dimensionality>
+class row_major {
+public:
+    friend class range<row_major>;
+
+    static_assert(Dimensionality != 0,
+                  "This accessor does not support a dimensionality of 0!");
+    static constexpr size_type dimensionality = Dimensionality;
+
+    /**
+     * Type of values returned by the accessor.
+     */
+    using value_type = ValueType;
+
+    /**
+     * Type of underlying data storage.
+     */
+    using data_type = value_type *;
+
+    /**
+     * Number of dimensions of the accessor.
+     */
+
+    using const_accessor = row_major<const ValueType, Dimensionality>;
+    using stride_type = std::array<const size_type, dimensionality - 1>;
+    using length_type = std::array<const size_type, dimensionality>;
+
+protected:
+    /**
+     * Creates a row_major accessor.
+     *
+     * @param data  pointer to the block of memory containing the data
+     * @param lengths size / length of the accesses of each dimension
+     * @param stride  distance (in elements) between starting positions of
+     *                the dimensions (i.e.
+     *                `x_1 * stride_1 + x_2 * stride_2 * ... + x_n`
+     *                points to the element at (x_1, x_2, ..., x_n))
+     */
+    constexpr GKO_ACC_ATTRIBUTES explicit row_major(data_type data,
+                                                    length_type size,
+                                                    stride_type stride)
+        : data{data}, lengths(size), stride(stride)
+    {}
+
+    /**
+     * Creates a row_major accessor with a default stride (assumes no
+     * padding)
+     *
+     * @param data  pointer to the block of memory containing the data
+     * @param lengths size / length of the accesses of each dimension
+     */
+    constexpr GKO_ACC_ATTRIBUTES explicit row_major(data_type data,
+                                                    length_type size)
+        : row_major{
+              data, size,
+              helper::compute_default_row_major_stride_array<const size_type>(
+                  size)}
+    {}
+
+public:
+    /**
+     * Creates a row_major range which contains a read-only version of the
+     * current accessor.
+     *
+     * @returns  a row major range which is read-only.
+     */
+    constexpr GKO_ACC_ATTRIBUTES range<const_accessor> to_const() const
+    {
+        // TODO Remove this functionality all together (if requested)
+        return range<const_accessor>(data, lengths, stride);
+    }
+
+    /**
+     * Returns the data element at the specified indices
+     *
+     * @param row  row index
+     * @param col  column index
+     *
+     * @return data element at (indices...)
+     */
+    template <typename... Indices>
+    constexpr GKO_ACC_ATTRIBUTES
+        std::enable_if_t<are_all_integral<Indices...>::value, value_type &>
+        operator()(Indices &&... indices) const
+    {
+        return data[helper::compute_row_major_index(
+            lengths, stride, std::forward<Indices>(indices)...)];
+    }
+
+    /**
+     * Returns the sub-range spanning the range (x1_span, x2_span, ...)
+     *
+     * @param rows  row span
+     * @param cols  column span
+     *
+     * @return sub-range spanning the given spans
+     */
+    template <typename... SpanTypes>
+    constexpr GKO_ACC_ATTRIBUTES
+        std::enable_if_t<helper::are_index_span_compatible<SpanTypes...>::value,
+                         range<row_major>>
+        operator()(SpanTypes... spans) const
+    {
+        return helper::validate_index_spans(lengths, spans...),
+               range<row_major>{
+                   data + helper::compute_row_major_index(
+                              lengths, stride, (index_span{spans}.begin)...),
+                   length_type{
+                       (index_span{spans}.end - index_span{spans}.begin)...},
+                   stride};
+    }
+
+    /**
+     * Returns the length in dimension `dimension`.
+     *
+     * @param dimension  a dimension index
+     *
+     * @return length in dimension `dimension`
+     */
+    constexpr GKO_ACC_ATTRIBUTES size_type length(size_type dimension) const
+    {
+        return lengths[dimension];
+    }
+
+    /**
+     * Copies data from another accessor
+     *
+     * @warning Do not use this function since it is not optimized for a
+     *          specific executor. It will always be performed sequentially.
+     *          Please write an optimized version (adjusted to the
+     * architecture) by iterating through the values yourself.
+     *
+     * @tparam OtherAccessor  type of the other accessor
+     *
+     * @param other  other accessor
+     */
+    template <typename OtherAccessor>
+    GKO_ACC_ATTRIBUTES void copy_from(const OtherAccessor &other) const
+    {
+        helper::multidim_for_each(lengths, [this, &other](auto... indices) {
+            (*this)(indices...) = other(indices...);
+        });
+    }
+
+    /**
+     * Reference to the underlying data.
+     */
+    const data_type data;
+
+    /**
+     * An array of dimension sizes.
+     */
+    const length_type lengths;
+
+    /**
+     * Distance between consecutive rows for each dimension (except the
+     * first).
+     */
+    const stride_type stride;
+};
+
+
+}  // namespace acc
+}  // namespace gko
+
+#endif  // GKO_ACCESSOR_ROW_MAJOR_HPP_
diff --git a/core/test/accessor/CMakeLists.txt b/core/test/accessor/CMakeLists.txt
index 7ee90215614..b83e171b19a 100644
--- a/core/test/accessor/CMakeLists.txt
+++ b/core/test/accessor/CMakeLists.txt
@@ -1,4 +1,7 @@
 ginkgo_create_test(index_span)
 ginkgo_create_test(range)
+# Accessor tests:
+ginkgo_create_test(block_col_major)
 ginkgo_create_test(reduced_row_major)
+ginkgo_create_test(row_major)
 ginkgo_create_test(scaled_reduced_row_major)
diff --git a/core/test/accessor/block_col_major.cpp b/core/test/accessor/block_col_major.cpp
new file mode 100644
index 00000000000..b64ddf5ff99
--- /dev/null
+++ b/core/test/accessor/block_col_major.cpp
@@ -0,0 +1,187 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <array>
+#include <tuple>
+#include <type_traits>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+#include "accessor/block_col_major.hpp"
+#include "accessor/index_span.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class BlockColMajorAccessor3d : public ::testing::Test {
+protected:
+    using span = gko::acc::index_span;
+    static constexpr gko::acc::size_type dimensionality{3};
+
+    using blk_col_major_range =
+        gko::acc::range<gko::acc::block_col_major<int, dimensionality>>;
+
+    // clang-format off
+    int data[2 * 3 * 4]{
+         1, 3, 5,
+         2, 4, 6,
+        -1,-2,-3,
+        11,12,13,
+
+        21,25,29,
+        22,26,30,
+        23,27,31,
+        24,28,32
+
+        /* This matrix actually looks like
+        1, 2, -1, 11,
+        3, 4, -2, 12,
+        5, 6, -3, 13,
+
+        21, 22, 23, 24,
+        25, 26, 27, 28,
+        29, 30, 31, 32
+        */
+    };
+    // clang-format on
+    const std::array<const gko::acc::size_type, dimensionality> dim1{2, 3, 4};
+    const std::array<const gko::acc::size_type, dimensionality> dim2{2, 2, 3};
+    blk_col_major_range default_r{data, dim1};
+    blk_col_major_range custom_r{
+        data, dim2,
+        std::array<const gko::acc::size_type, dimensionality - 1>{12, 3}};
+};
+
+
+TEST_F(BlockColMajorAccessor3d, ComputesCorrectStride)
+{
+    auto range_stride = default_r.get_accessor().stride;
+    auto check_stride = std::array<const gko::acc::size_type, 2>{12, 3};
+
+    ASSERT_EQ(range_stride, check_stride);
+}
+
+
+TEST_F(BlockColMajorAccessor3d, CanAccessData)
+{
+    EXPECT_EQ(default_r(0, 0, 0), 1);
+    EXPECT_EQ(custom_r(0, 0, 0), 1);
+    EXPECT_EQ(default_r(0, 1, 0), 3);
+    EXPECT_EQ(custom_r(0, 1, 0), 3);
+    EXPECT_EQ(default_r(0, 1, 1), 4);
+    EXPECT_EQ(default_r(0, 1, 3), 12);
+    EXPECT_EQ(default_r(0, 2, 2), -3);
+    EXPECT_EQ(default_r(1, 2, 1), 30);
+    EXPECT_EQ(default_r(1, 2, 2), 31);
+    EXPECT_EQ(default_r(1, 2, 3), 32);
+}
+
+
+TEST_F(BlockColMajorAccessor3d, CanWriteData)
+{
+    default_r(0, 0, 0) = 4;
+    custom_r(1, 1, 1) = 100;
+
+    EXPECT_EQ(default_r(0, 0, 0), 4);
+    EXPECT_EQ(custom_r(0, 0, 0), 4);
+    EXPECT_EQ(default_r(1, 1, 1), 100);
+    EXPECT_EQ(custom_r(1, 1, 1), 100);
+}
+
+
+TEST_F(BlockColMajorAccessor3d, CanCreateSubrange)
+{
+    auto subr = custom_r(span{0, 2}, span{1, 2}, span{1, 3});
+
+    EXPECT_EQ(subr(0, 0, 0), 4);
+    EXPECT_EQ(subr(0, 0, 1), -2);
+    EXPECT_EQ(subr(1, 0, 0), 26);
+    EXPECT_EQ(subr(1, 0, 1), 27);
+}
+
+
+TEST_F(BlockColMajorAccessor3d, CanCreateRowVector)
+{
+    auto subr = default_r(1u, 2u, span{0, 2});
+
+    EXPECT_EQ(subr(0, 0, 0), 29);
+    EXPECT_EQ(subr(0, 0, 1), 30);
+}
+
+
+TEST_F(BlockColMajorAccessor3d, CanCreateColumnVector)
+{
+    auto subr = default_r(span{0u, 2u}, 1u, 3u);
+
+    EXPECT_EQ(subr(0, 0, 0), 12);
+    EXPECT_EQ(subr(1, 0, 0), 28);
+}
+
+/*
+TEST_F(BlockColMajorAccessor3d, CanAssignValues)
+{
+    default_r(1, 1, 1) = default_r(0, 0, 0);
+
+    EXPECT_EQ(data[16], 1);
+}
+
+
+TEST_F(BlockColMajorAccessor3d, CanAssignSubranges)
+{
+    default_r(1u, span{0, 2}, span{0, 3}) =
+        custom_r(0u, span{0, 2}, span{0, 3});
+
+    EXPECT_EQ(data[12], 1);
+    EXPECT_EQ(data[15], 2);
+    EXPECT_EQ(data[18], -1);
+    EXPECT_EQ(data[21], 24);
+    EXPECT_EQ(data[13], 3);
+    EXPECT_EQ(data[16], 4);
+    EXPECT_EQ(data[19], -2);
+    EXPECT_EQ(data[22], 28);
+    EXPECT_EQ(data[14], 29);
+    EXPECT_EQ(data[17], 30);
+    EXPECT_EQ(data[20], 31);
+    EXPECT_EQ(data[23], 32);
+}
+*/
+
+
+}  // namespace
diff --git a/core/test/accessor/row_major.cpp b/core/test/accessor/row_major.cpp
new file mode 100644
index 00000000000..fcc3f118972
--- /dev/null
+++ b/core/test/accessor/row_major.cpp
@@ -0,0 +1,290 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <array>
+#include <tuple>
+#include <type_traits>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+#include "accessor/index_span.hpp"
+#include "accessor/row_major.hpp"
+#include "core/base/extended_float.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class RowMajorAccessor : public ::testing::Test {
+protected:
+    using span = gko::acc::index_span;
+    using dim_type = std::array<const gko::acc::size_type, 2>;
+    using stride_type = std::array<const gko::acc::size_type, 1>;
+
+    using row_major_int_range = gko::acc::range<gko::acc::row_major<int, 2>>;
+
+    // clang-format off
+    int data[9]{
+        1, 2, -1,
+        3, 4, -2,
+        5, 6, -3
+    };
+    // clang-format on
+    row_major_int_range r{data, dim_type{3u, 2u}, stride_type{3u}};
+};
+
+
+TEST_F(RowMajorAccessor, CanCreateWithDim)
+{
+    row_major_int_range r2{data, dim_type{3, 2}, stride_type{3u}};
+
+    EXPECT_EQ(r2(0, 0), 1);
+    EXPECT_EQ(r2(0, 1), 2);
+    EXPECT_EQ(r2(1, 0), 3);
+    EXPECT_EQ(r2(1, 1), 4);
+    EXPECT_EQ(r2(2, 0), 5);
+    EXPECT_EQ(r2(2, 1), 6);
+}
+
+
+TEST_F(RowMajorAccessor, CanCreateDefaultStride)
+{
+    row_major_int_range r2{data, dim_type{3, 3}};
+
+    EXPECT_EQ(r2(0, 0), 1);
+    EXPECT_EQ(r2(0, 1), 2);
+    EXPECT_EQ(r2(0, 2), -1);
+    EXPECT_EQ(r2(1, 0), 3);
+    EXPECT_EQ(r2(1, 1), 4);
+    EXPECT_EQ(r2(1, 2), -2);
+    EXPECT_EQ(r2(2, 0), 5);
+    EXPECT_EQ(r2(2, 1), 6);
+    EXPECT_EQ(r2(2, 2), -3);
+}
+
+
+TEST_F(RowMajorAccessor, CanAccessData)
+{
+    EXPECT_EQ(r(0, 0), 1);
+    EXPECT_EQ(r(0, 1), 2);
+    EXPECT_EQ(r(1, 0), 3);
+    EXPECT_EQ(r(1, 1), 4);
+    EXPECT_EQ(r(2, 0), 5);
+    EXPECT_EQ(r(2, 1), 6);
+}
+
+
+TEST_F(RowMajorAccessor, CanWriteData)
+{
+    r(0, 0) = 4;
+
+    EXPECT_EQ(r(0, 0), 4);
+}
+
+
+TEST_F(RowMajorAccessor, CanCreateSubrange)
+{
+    auto subr = r(span{1, 3}, span{0, 2});
+
+    EXPECT_EQ(subr(0, 0), 3);
+    EXPECT_EQ(subr(0, 1), 4);
+    EXPECT_EQ(subr(1, 0), 5);
+    EXPECT_EQ(subr(1, 1), 6);
+}
+
+
+TEST_F(RowMajorAccessor, CanCreateRowVector)
+{
+    auto subr = r(2, span{0, 2});
+
+    EXPECT_EQ(subr(0, 0), 5);
+    EXPECT_EQ(subr(0, 1), 6);
+}
+
+
+TEST_F(RowMajorAccessor, CanCreateColumnVector)
+{
+    auto subr = r(span{0, 3}, 0);
+
+    EXPECT_EQ(subr(0, 0), 1);
+    EXPECT_EQ(subr(1, 0), 3);
+    EXPECT_EQ(subr(2, 0), 5);
+}
+
+
+TEST_F(RowMajorAccessor, CanAssignValues)
+{
+    r(1, 1) = r(0, 0);
+
+    EXPECT_EQ(data[4], 1);
+}
+
+
+/*
+TEST_F(RowMajorAccessor, CanAssignSubranges)
+{
+    r(0, span{0, 2}) = r(1, span{0, 2});
+
+    EXPECT_EQ(data[0], 3);
+    EXPECT_EQ(data[1], 4);
+    EXPECT_EQ(data[2], -1);
+    EXPECT_EQ(data[3], 3);
+    EXPECT_EQ(data[4], 4);
+    EXPECT_EQ(data[5], -2);
+    EXPECT_EQ(data[6], 5);
+    EXPECT_EQ(data[7], 6);
+    EXPECT_EQ(data[8], -3);
+}
+*/
+
+
+class RowMajorAccessor3d : public ::testing::Test {
+protected:
+    using span = gko::acc::index_span;
+    static constexpr gko::size_type dimensionality{3};
+
+    using row_major_int_range =
+        gko::acc::range<gko::acc::row_major<int, dimensionality>>;
+
+    // clang-format off
+    int data[2 * 3 * 4]{
+        1, 2, -1, 11,
+        3, 4, -2, 12,
+        5, 6, -3, 13,
+
+        21, 22, 23, 24,
+        25, 26, 27, 28,
+        29, 30, 31, 32
+    };
+    // clang-format on
+    const std::array<const gko::acc::size_type, dimensionality> dim1{2, 3, 4};
+    const std::array<const gko::acc::size_type, dimensionality> dim2{2, 2, 3};
+    row_major_int_range default_r{data, dim1};
+    row_major_int_range custom_r{
+        data, dim2,
+        std::array<const gko::size_type, dimensionality - 1>{12, 4}};
+};
+
+
+TEST_F(RowMajorAccessor3d, CanAccessData)
+{
+    EXPECT_EQ(default_r(0, 0, 0), 1);
+    EXPECT_EQ(custom_r(0, 0, 0), 1);
+    EXPECT_EQ(default_r(0, 1, 0), 3);
+    EXPECT_EQ(custom_r(0, 1, 0), 3);
+    EXPECT_EQ(default_r(0, 1, 3), 12);
+    EXPECT_EQ(default_r(0, 2, 2), -3);
+    EXPECT_EQ(default_r(1, 2, 1), 30);
+    EXPECT_EQ(default_r(1, 2, 2), 31);
+    EXPECT_EQ(default_r(1, 2, 3), 32);
+}
+
+
+TEST_F(RowMajorAccessor3d, CanWriteData)
+{
+    default_r(0, 0, 0) = 4;
+    custom_r(1, 1, 1) = 100;
+
+    EXPECT_EQ(default_r(0, 0, 0), 4);
+    EXPECT_EQ(custom_r(0, 0, 0), 4);
+    EXPECT_EQ(default_r(1, 1, 1), 100);
+    EXPECT_EQ(custom_r(1, 1, 1), 100);
+}
+
+
+TEST_F(RowMajorAccessor3d, CanCreateSubrange)
+{
+    auto subr = custom_r(span{0, 2}, span{1, 2}, span{1, 3});
+
+    EXPECT_EQ(subr(0, 0, 0), 4);
+    EXPECT_EQ(subr(0, 0, 1), -2);
+    EXPECT_EQ(subr(1, 0, 0), 26);
+    EXPECT_EQ(subr(1, 0, 1), 27);
+}
+
+
+TEST_F(RowMajorAccessor3d, CanCreateRowVector)
+{
+    auto subr = default_r(1u, 2u, span{0, 2});
+
+    EXPECT_EQ(subr(0, 0, 0), 29);
+    EXPECT_EQ(subr(0, 0, 1), 30);
+}
+
+
+TEST_F(RowMajorAccessor3d, CanCreateColumnVector)
+{
+    auto subr = default_r(span{0u, 2u}, 1u, 3u);
+
+    EXPECT_EQ(subr(0, 0, 0), 12);
+    EXPECT_EQ(subr(1, 0, 0), 28);
+}
+
+
+TEST_F(RowMajorAccessor3d, CanAssignValues)
+{
+    default_r(1, 1, 1) = default_r(0, 0, 0);
+
+    EXPECT_EQ(data[17], 1);
+}
+
+
+/*
+TEST_F(RowMajorAccessor3d, CanAssignSubranges)
+{
+    default_r(1u, span{0, 2}, span{0, 3}) =
+        custom_r(0u, span{0, 2}, span{0, 3});
+
+    EXPECT_EQ(data[12], 1);
+    EXPECT_EQ(data[13], 2);
+    EXPECT_EQ(data[14], -1);
+    EXPECT_EQ(data[15], 24);
+    EXPECT_EQ(data[16], 3);
+    EXPECT_EQ(data[17], 4);
+    EXPECT_EQ(data[18], -2);
+    EXPECT_EQ(data[19], 28);
+    EXPECT_EQ(data[20], 29);
+    EXPECT_EQ(data[21], 30);
+    EXPECT_EQ(data[22], 31);
+    EXPECT_EQ(data[23], 32);
+}
+*/
+
+
+}  // namespace
diff --git a/core/test/base/dim.cpp b/core/test/base/dim.cpp
index 6e60a975b7e..6d48bcab794 100644
--- a/core/test/base/dim.cpp
+++ b/core/test/base/dim.cpp
@@ -33,7 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/dim.hpp>
 
 
-#include <array>
 #include <memory>
 
 
@@ -80,66 +79,6 @@ TEST(Dim, ConstructsNullObject)
 }
 
 
-TEST(Dim, ComputeDefaultStrideArrayConstexpr)
-{
-    using value_type = unsigned int;
-    constexpr int dimens{4};
-    constexpr gko::dim<dimens, int> d{2, 3, 5, 7};
-    constexpr std::array<value_type, dimens - 1> expected{3 * 5 * 7, 5 * 7, 7};
-
-    constexpr auto res = gko::compute_default_stride_array<value_type>(d);
-
-    ::testing::StaticAssertTypeEq<decltype(res),
-                                  const std::array<value_type, dimens - 1>>();
-    ASSERT_EQ(res, expected);
-}
-
-
-TEST(Dim, ComputeDefaultStrideArray)
-{
-    using value_type = gko::size_type;
-    constexpr int dimens{4};
-    gko::dim<dimens, int> d{2, 3, 5, 7};
-    std::array<value_type, dimens - 1> expected{3 * 5 * 7, 5 * 7, 7};
-
-    auto res = gko::compute_default_stride_array<value_type>(d);
-
-    ::testing::StaticAssertTypeEq<decltype(res),
-                                  std::array<value_type, dimens - 1>>();
-    ASSERT_EQ(res, expected);
-}
-
-
-TEST(Dim, ComputeDefaultStrideArrayEmpty)
-{
-    using value_type = gko::size_type;
-    constexpr int dimens{1};
-    gko::dim<dimens> d{2};
-    std::array<value_type, 0> expected{};
-
-    auto res = gko::compute_default_stride_array<value_type>(d);
-
-    ::testing::StaticAssertTypeEq<decltype(res),
-                                  std::array<value_type, dimens - 1>>();
-    ASSERT_EQ(res, expected);
-}
-
-
-TEST(Dim, ComputeDefaultStrideArrayEmptyConstexpr)
-{
-    using value_type = gko::size_type;
-    constexpr int dimens{1};
-    constexpr gko::dim<dimens> d{2};
-    constexpr std::array<value_type, 0> expected{};
-
-    constexpr auto res = gko::compute_default_stride_array<value_type>(d);
-
-    ::testing::StaticAssertTypeEq<decltype(res),
-                                  const std::array<value_type, dimens - 1>>();
-    ASSERT_EQ(res, expected);
-}
-
-
 class dim_manager {
 public:
     using dim = gko::dim<3>;
diff --git a/core/test/base/range_accessors.cpp b/core/test/base/range_accessors.cpp
index 7a5ffe7c18a..eecac4ae60f 100644
--- a/core/test/base/range_accessors.cpp
+++ b/core/test/base/range_accessors.cpp
@@ -62,35 +62,6 @@ class RowMajorAccessor : public ::testing::Test {
 };
 
 
-TEST_F(RowMajorAccessor, CanCreateWithDim)
-{
-    row_major_int_range r2{data, gko::dim<2>{3, 2}, 3u};
-
-    EXPECT_EQ(r2(0, 0), 1);
-    EXPECT_EQ(r2(0, 1), 2);
-    EXPECT_EQ(r2(1, 0), 3);
-    EXPECT_EQ(r2(1, 1), 4);
-    EXPECT_EQ(r2(2, 0), 5);
-    EXPECT_EQ(r2(2, 1), 6);
-}
-
-
-TEST_F(RowMajorAccessor, CanCreateDefaultStride)
-{
-    row_major_int_range r2{data, gko::dim<2>{3, 3}};
-
-    EXPECT_EQ(r2(0, 0), 1);
-    EXPECT_EQ(r2(0, 1), 2);
-    EXPECT_EQ(r2(0, 2), -1);
-    EXPECT_EQ(r2(1, 0), 3);
-    EXPECT_EQ(r2(1, 1), 4);
-    EXPECT_EQ(r2(1, 2), -2);
-    EXPECT_EQ(r2(2, 0), 5);
-    EXPECT_EQ(r2(2, 1), 6);
-    EXPECT_EQ(r2(2, 2), -3);
-}
-
-
 TEST_F(RowMajorAccessor, CanAccessData)
 {
     EXPECT_EQ(r(0, 0), 1);
@@ -164,248 +135,4 @@ TEST_F(RowMajorAccessor, CanAssignSubranges)
 }
 
 
-class RowMajorAccessor3d : public ::testing::Test {
-protected:
-    using span = gko::span;
-    static constexpr gko::size_type dimensionality{3};
-
-    using row_major_int_range =
-        gko::range<gko::accessor::row_major<int, dimensionality>>;
-
-    // clang-format off
-    int data[2 * 3 * 4]{
-        1, 2, -1, 11,
-        3, 4, -2, 12,
-        5, 6, -3, 13,
-
-        21, 22, 23, 24,
-        25, 26, 27, 28,
-        29, 30, 31, 32
-    };
-    // clang-format on
-    const gko::dim<dimensionality> dim1{2, 3, 4};
-    const gko::dim<dimensionality> dim2{2, 2, 3};
-    row_major_int_range default_r{data, dim1};
-    row_major_int_range custom_r{
-        data, dim2,
-        std::array<const gko::size_type, dimensionality - 1>{12, 4}};
-};
-
-
-TEST_F(RowMajorAccessor3d, CanAccessData)
-{
-    EXPECT_EQ(default_r(0, 0, 0), 1);
-    EXPECT_EQ(custom_r(0, 0, 0), 1);
-    EXPECT_EQ(default_r(0, 1, 0), 3);
-    EXPECT_EQ(custom_r(0, 1, 0), 3);
-    EXPECT_EQ(default_r(0, 1, 3), 12);
-    EXPECT_EQ(default_r(0, 2, 2), -3);
-    EXPECT_EQ(default_r(1, 2, 1), 30);
-    EXPECT_EQ(default_r(1, 2, 2), 31);
-    EXPECT_EQ(default_r(1, 2, 3), 32);
-}
-
-
-TEST_F(RowMajorAccessor3d, CanWriteData)
-{
-    default_r(0, 0, 0) = 4;
-    custom_r(1, 1, 1) = 100;
-
-    EXPECT_EQ(default_r(0, 0, 0), 4);
-    EXPECT_EQ(custom_r(0, 0, 0), 4);
-    EXPECT_EQ(default_r(1, 1, 1), 100);
-    EXPECT_EQ(custom_r(1, 1, 1), 100);
-}
-
-
-TEST_F(RowMajorAccessor3d, CanCreateSubrange)
-{
-    auto subr = custom_r(span{0, 2}, span{1, 2}, span{1, 3});
-
-    EXPECT_EQ(subr(0, 0, 0), 4);
-    EXPECT_EQ(subr(0, 0, 1), -2);
-    EXPECT_EQ(subr(1, 0, 0), 26);
-    EXPECT_EQ(subr(1, 0, 1), 27);
-}
-
-
-TEST_F(RowMajorAccessor3d, CanCreateRowVector)
-{
-    auto subr = default_r(1u, 2u, span{0, 2});
-
-    EXPECT_EQ(subr(0, 0, 0), 29);
-    EXPECT_EQ(subr(0, 0, 1), 30);
-}
-
-
-TEST_F(RowMajorAccessor3d, CanCreateColumnVector)
-{
-    auto subr = default_r(span{0u, 2u}, 1u, 3u);
-
-    EXPECT_EQ(subr(0, 0, 0), 12);
-    EXPECT_EQ(subr(1, 0, 0), 28);
-}
-
-
-TEST_F(RowMajorAccessor3d, CanAssignValues)
-{
-    default_r(1, 1, 1) = default_r(0, 0, 0);
-
-    EXPECT_EQ(data[17], 1);
-}
-
-
-TEST_F(RowMajorAccessor3d, CanAssignSubranges)
-{
-    default_r(1u, span{0, 2}, span{0, 3}) =
-        custom_r(0u, span{0, 2}, span{0, 3});
-
-    EXPECT_EQ(data[12], 1);
-    EXPECT_EQ(data[13], 2);
-    EXPECT_EQ(data[14], -1);
-    EXPECT_EQ(data[15], 24);
-    EXPECT_EQ(data[16], 3);
-    EXPECT_EQ(data[17], 4);
-    EXPECT_EQ(data[18], -2);
-    EXPECT_EQ(data[19], 28);
-    EXPECT_EQ(data[20], 29);
-    EXPECT_EQ(data[21], 30);
-    EXPECT_EQ(data[22], 31);
-    EXPECT_EQ(data[23], 32);
-}
-
-
-class BlockColMajorAccessor3d : public ::testing::Test {
-protected:
-    using span = gko::span;
-    static constexpr gko::size_type dimensionality{3};
-
-    using blk_col_major_range =
-        gko::range<gko::accessor::block_col_major<int, dimensionality>>;
-
-    // clang-format off
-    int data[2 * 3 * 4]{
-         1, 3, 5,
-         2, 4, 6,
-        -1,-2,-3,
-        11,12,13,
-
-        21,25,29,
-        22,26,30,
-        23,27,31,
-        24,28,32
-
-        /* This matrix actually looks like
-        1, 2, -1, 11,
-        3, 4, -2, 12,
-        5, 6, -3, 13,
-
-        21, 22, 23, 24,
-        25, 26, 27, 28,
-        29, 30, 31, 32
-        */
-    };
-    // clang-format on
-    const gko::dim<dimensionality> dim1{2, 3, 4};
-    const gko::dim<dimensionality> dim2{2, 2, 3};
-    blk_col_major_range default_r{data, dim1};
-    blk_col_major_range custom_r{
-        data, dim2,
-        std::array<const gko::size_type, dimensionality - 1>{12, 3}};
-};
-
-
-TEST_F(BlockColMajorAccessor3d, ComputesCorrectStride)
-{
-    auto range_stride = default_r.get_accessor().stride;
-    auto check_stride = std::array<const gko::size_type, 2>{12, 3};
-
-    ASSERT_EQ(range_stride, check_stride);
-}
-
-
-TEST_F(BlockColMajorAccessor3d, CanAccessData)
-{
-    EXPECT_EQ(default_r(0, 0, 0), 1);
-    EXPECT_EQ(custom_r(0, 0, 0), 1);
-    EXPECT_EQ(default_r(0, 1, 0), 3);
-    EXPECT_EQ(custom_r(0, 1, 0), 3);
-    EXPECT_EQ(default_r(0, 1, 1), 4);
-    EXPECT_EQ(default_r(0, 1, 3), 12);
-    EXPECT_EQ(default_r(0, 2, 2), -3);
-    EXPECT_EQ(default_r(1, 2, 1), 30);
-    EXPECT_EQ(default_r(1, 2, 2), 31);
-    EXPECT_EQ(default_r(1, 2, 3), 32);
-}
-
-
-TEST_F(BlockColMajorAccessor3d, CanWriteData)
-{
-    default_r(0, 0, 0) = 4;
-    custom_r(1, 1, 1) = 100;
-
-    EXPECT_EQ(default_r(0, 0, 0), 4);
-    EXPECT_EQ(custom_r(0, 0, 0), 4);
-    EXPECT_EQ(default_r(1, 1, 1), 100);
-    EXPECT_EQ(custom_r(1, 1, 1), 100);
-}
-
-
-TEST_F(BlockColMajorAccessor3d, CanCreateSubrange)
-{
-    auto subr = custom_r(span{0, 2}, span{1, 2}, span{1, 3});
-
-    EXPECT_EQ(subr(0, 0, 0), 4);
-    EXPECT_EQ(subr(0, 0, 1), -2);
-    EXPECT_EQ(subr(1, 0, 0), 26);
-    EXPECT_EQ(subr(1, 0, 1), 27);
-}
-
-
-TEST_F(BlockColMajorAccessor3d, CanCreateRowVector)
-{
-    auto subr = default_r(1u, 2u, span{0, 2});
-
-    EXPECT_EQ(subr(0, 0, 0), 29);
-    EXPECT_EQ(subr(0, 0, 1), 30);
-}
-
-
-TEST_F(BlockColMajorAccessor3d, CanCreateColumnVector)
-{
-    auto subr = default_r(span{0u, 2u}, 1u, 3u);
-
-    EXPECT_EQ(subr(0, 0, 0), 12);
-    EXPECT_EQ(subr(1, 0, 0), 28);
-}
-
-
-TEST_F(BlockColMajorAccessor3d, CanAssignValues)
-{
-    default_r(1, 1, 1) = default_r(0, 0, 0);
-
-    EXPECT_EQ(data[16], 1);
-}
-
-
-TEST_F(BlockColMajorAccessor3d, CanAssignSubranges)
-{
-    default_r(1u, span{0, 2}, span{0, 3}) =
-        custom_r(0u, span{0, 2}, span{0, 3});
-
-    EXPECT_EQ(data[12], 1);
-    EXPECT_EQ(data[15], 2);
-    EXPECT_EQ(data[18], -1);
-    EXPECT_EQ(data[21], 24);
-    EXPECT_EQ(data[13], 3);
-    EXPECT_EQ(data[16], 4);
-    EXPECT_EQ(data[19], -2);
-    EXPECT_EQ(data[22], 28);
-    EXPECT_EQ(data[14], 29);
-    EXPECT_EQ(data[17], 30);
-    EXPECT_EQ(data[20], 31);
-    EXPECT_EQ(data[23], 32);
-}
-
-
 }  // namespace
diff --git a/include/ginkgo/core/base/dim.hpp b/include/ginkgo/core/base/dim.hpp
index 64645fe3f8d..0881d954aa0 100644
--- a/include/ginkgo/core/base/dim.hpp
+++ b/include/ginkgo/core/base/dim.hpp
@@ -34,9 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_PUBLIC_CORE_BASE_DIM_HPP_
 
 
-#include <array>
-
-
 #include <ginkgo/core/base/types.hpp>
 
 
@@ -245,76 +242,6 @@ constexpr GKO_ATTRIBUTES GKO_INLINE dim<2, DimensionType> transpose(
 }
 
 
-namespace detail {
-
-
-template <typename ValueType, size_type iter, size_type N,
-          typename DimensionType>
-constexpr GKO_ATTRIBUTES std::enable_if_t<iter == N, ValueType>
-mult_dim_upwards(const dim<N, DimensionType> &)
-{
-    return 1;
-}
-
-template <typename ValueType, size_type iter, size_type N,
-          typename DimensionType>
-constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), ValueType>
-mult_dim_upwards(const dim<N, DimensionType> &size)
-{
-    return size[iter] * mult_dim_upwards<ValueType, iter + 1>(size);
-}
-
-
-template <typename ValueType, size_type iter = 1, size_type N,
-          typename DimensionType, typename... Args>
-constexpr GKO_ATTRIBUTES
-    std::enable_if_t<N == 0 || (iter == N && iter == sizeof...(Args) + 1),
-                     std::array<ValueType, N == 0 ? 0 : N - 1>>
-    compute_default_stride_array(const dim<N, DimensionType> &, Args &&... args)
-{
-    return {{std::forward<Args>(args)...}};
-}
-
-template <typename ValueType, size_type iter = 1, size_type N,
-          typename DimensionType, typename... Args>
-constexpr GKO_ATTRIBUTES std::enable_if_t<
-    (iter < N) && (iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
-compute_default_stride_array(const dim<N, DimensionType> &size, Args &&... args)
-{
-    return compute_default_stride_array<ValueType, iter + 1>(
-        size, std::forward<Args>(args)...,
-        mult_dim_upwards<ValueType, iter>(size));
-}
-
-
-}  // namespace detail
-
-
-/**
- * Computes the default stride array from a given size, assuming there is no
- * padding.
- *
- * Example: dim<4> size={2, 3, 5, 7} results in a return value of:
- * std::array<ValueType, 3> = {3*5*7, 5*7, 7}
- *
- * @tparam ValueType  value type of the values in the returned array
- *
- * @tparam dimensions  number of dimensions in the given size
- *
- * @tparam DimensionType  value type of the stored size
- *
- * @returns an std::array<ValueType, dimensions - 1> with the stride
- *          information.
- */
-template <typename ValueType, size_type dimensions, typename DimensionType>
-constexpr GKO_ATTRIBUTES
-    std::array<ValueType, (dimensions > 0 ? dimensions - 1 : 0)>
-    compute_default_stride_array(const dim<dimensions, DimensionType> &size)
-{
-    return detail::compute_default_stride_array<ValueType>(size);
-}
-
-
 }  // namespace gko
 
 
diff --git a/include/ginkgo/core/base/range_accessors.hpp b/include/ginkgo/core/base/range_accessors.hpp
index c4abdde4e45..142d5e931b8 100644
--- a/include/ginkgo/core/base/range_accessors.hpp
+++ b/include/ginkgo/core/base/range_accessors.hpp
@@ -35,10 +35,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <array>
-#include <type_traits>
 
 
-#include <ginkgo/core/base/dim.hpp>
 #include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/base/types.hpp>
 
@@ -52,233 +50,6 @@ namespace gko {
 namespace accessor {
 
 
-namespace detail {
-
-
-/**
- * This helper runs from first to last dimension in order to compute the index.
- * The index is computed like this:
- * indices: x1, x2, x3, ...
- * compute(stride, x1, x2, x3) -> x1 * stride[0] + x2 * stride[1] + x3
- */
-template <typename ValueType, size_type total_dim, size_type current_iter = 1>
-struct row_major_helper_s {
-    static_assert(total_dim >= 1, "Dimensionality must be >= 1");
-    static_assert(current_iter < total_dim, "Iteration must be < total_dim!");
-
-    static constexpr size_type dim_idx{current_iter - 1};
-
-    template <typename FirstType, typename... Indices>
-    static constexpr GKO_ATTRIBUTES ValueType
-    compute(const std::array<ValueType, total_dim> &size,
-            const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
-                &stride,
-            FirstType first, Indices &&... idxs)
-    {
-        return GKO_ASSERT(first < size[dim_idx]),
-               first * stride[dim_idx] +
-                   row_major_helper_s<ValueType, total_dim, current_iter + 1>::
-                       compute(size, stride, std::forward<Indices>(idxs)...);
-    }
-};
-
-template <typename ValueType, size_type total_dim>
-struct row_major_helper_s<ValueType, total_dim, total_dim> {
-    template <typename FirstType>
-    static constexpr GKO_ATTRIBUTES ValueType
-    compute(const std::array<ValueType, total_dim> &size,
-            const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>,
-            FirstType first)
-    {
-        return GKO_ASSERT(first < size[total_dim - 1]), first;
-    }
-};
-
-
-/**
- * Computes the storage index for the given indices with respect to the given
- * stride array for row-major access
- *
- * @param size  the multi-dimensional sizes of the range of values
- * @param stride  the stride array
- * @param idxs  the multi-dimensional indices of the desired entry
- */
-template <typename ValueType, size_type total_dim, typename... Indices>
-constexpr GKO_ATTRIBUTES ValueType compute_storage_index(
-    const std::array<ValueType, total_dim> &size,
-    const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)> &stride,
-    Indices &&... idxs)
-{
-    return row_major_helper_s<ValueType, total_dim>::compute(
-        size, stride, std::forward<Indices>(idxs)...);
-}
-
-
-template <size_type iter, typename ValueType, size_type N>
-constexpr GKO_ATTRIBUTES std::enable_if_t<iter == N, int> spans_in_size(
-    const std::array<ValueType, N> &)
-{
-    return 0;
-}
-
-template <size_type iter, typename ValueType, size_type N, typename First,
-          typename... Remaining>
-constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), int> spans_in_size(
-    const std::array<ValueType, N> &size, First first,
-    Remaining &&... remaining)
-{
-    static_assert(sizeof...(Remaining) + 1 == N - iter,
-                  "Number of remaining spans must be equal to N - iter");
-    return GKO_ASSERT(span{first}.is_valid()),
-           GKO_ASSERT(span{first} <= span{size[iter]}),
-           spans_in_size<iter + 1>(size, std::forward<Remaining>(remaining)...);
-}
-
-
-template <typename ValueType, size_type N, typename... Spans>
-constexpr GKO_ATTRIBUTES int validate_spans(
-    const std::array<ValueType, N> &size, Spans &&... spans)
-{
-    return detail::spans_in_size<0>(size, std::forward<Spans>(spans)...);
-}
-
-
-template <bool has_span, typename... Args>
-struct are_span_compatible_impl
-    : public std::integral_constant<bool, has_span> {};
-
-template <bool has_span, typename First, typename... Args>
-struct are_span_compatible_impl<has_span, First, Args...>
-    : public std::conditional<
-          std::is_integral<std::decay_t<First>>::value ||
-              std::is_same<std::decay_t<First>, span>::value,
-          are_span_compatible_impl<
-              has_span || std::is_same<std::decay_t<First>, span>::value,
-              Args...>,
-          std::false_type>::type {};
-
-
-/**
- * Evaluates if at least one type of Args is a gko::span and the others either
- * also gko::span or fulfill std::is_integral
- */
-template <typename... Args>
-using are_span_compatible = are_span_compatible_impl<false, Args...>;
-
-
-template <typename ValueType, size_type N, size_type current, typename... Dims>
-constexpr GKO_ATTRIBUTES
-    std::enable_if_t<(current == N), std::array<ValueType, N>>
-    to_array_impl(const dim<N> &, Dims &&... dims)
-{
-    static_assert(sizeof...(Dims) == N,
-                  "Number of arguments must match dimensionality!");
-    return {{std::forward<Dims>(dims)...}};
-}
-
-
-template <typename ValueType, size_type N, size_type current, typename... Dims>
-constexpr GKO_ATTRIBUTES
-    std::enable_if_t<(current < N), std::array<ValueType, N>>
-    to_array_impl(const dim<N> &size, Dims &&... dims)
-{
-    return to_array_impl<ValueType, N, current + 1>(
-        size, std::forward<Dims>(dims)..., size[current]);
-}
-
-
-template <typename ValueType, size_type N>
-constexpr GKO_ATTRIBUTES std::array<ValueType, N> to_array(const dim<N> &size)
-{
-    return to_array_impl<ValueType, N, 0>(size);
-}
-
-
-template <size_type iter, typename ValueType, size_type N>
-constexpr GKO_ATTRIBUTES std::enable_if_t<iter == N, ValueType>
-mult_dim_upwards_impl(const std::array<ValueType, N> &)
-{
-    return 1;
-}
-
-template <size_type iter, typename ValueType, size_type N>
-constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N), ValueType>
-mult_dim_upwards_impl(const std::array<ValueType, N> &size)
-{
-    return size[iter] * mult_dim_upwards_impl<iter + 1>(size);
-}
-
-
-template <size_type iter = 1, typename ValueType, size_type N, typename... Args>
-constexpr GKO_ATTRIBUTES
-    std::enable_if_t<N == 0 || (iter == N && iter == sizeof...(Args) + 1),
-                     std::array<ValueType, N == 0 ? 0 : N - 1>>
-    compute_default_stride_array_impl(const std::array<ValueType, N> &,
-                                      Args &&... args)
-{
-    return {{std::forward<Args>(args)...}};
-}
-
-template <size_type iter = 1, typename ValueType, size_type N, typename... Args>
-constexpr GKO_ATTRIBUTES std::enable_if_t<
-    (iter < N) && (iter == sizeof...(Args) + 1), std::array<ValueType, N - 1>>
-compute_default_stride_array_impl(const std::array<ValueType, N> &size,
-                                  Args &&... args)
-{
-    return compute_default_stride_array_impl<iter + 1>(
-        size, std::forward<Args>(args)..., mult_dim_upwards_impl<iter>(size));
-}
-
-
-template <typename ValueType, size_type dimensions>
-constexpr GKO_ATTRIBUTES
-    std::array<ValueType, (dimensions > 0 ? dimensions - 1 : 0)>
-    compute_default_stride_array(const std::array<ValueType, dimensions> &size)
-{
-    return compute_default_stride_array_impl(size);
-}
-
-
-template <size_type iter, typename ValueType, size_type N, typename Callable,
-          typename... Indices>
-GKO_ATTRIBUTES std::enable_if_t<iter == N> multidim_for_each_impl(
-    const std::array<ValueType, N> &, Callable callable, Indices &&... indices)
-{
-    static_assert(iter == sizeof...(Indices),
-                  "Number arguments must match current iteration!");
-    callable(std::forward<Indices>(indices)...);
-}
-
-template <size_type iter, typename ValueType, size_type N, typename Callable,
-          typename... Indices>
-GKO_ATTRIBUTES std::enable_if_t<(iter < N)> multidim_for_each_impl(
-    const std::array<ValueType, N> &size, Callable &&callable,
-    Indices &&... indices)
-{
-    static_assert(iter == sizeof...(Indices),
-                  "Number arguments must match current iteration!");
-    for (size_type i = 0; i < size[iter]; ++i) {
-        multidim_for_each_impl<iter + 1>(size, std::forward<Callable>(callable),
-                                         std::forward<Indices>(indices)..., i);
-    }
-}
-
-
-/**
- * Creates a recursive for-loop for each dimension and calls dest(indices...) =
- * source(indices...)
- */
-template <typename ValueType, size_type N, typename Callable>
-GKO_ATTRIBUTES void multidim_for_each(const std::array<ValueType, N> &size,
-                                      Callable &&callable)
-{
-    multidim_for_each_impl<0>(size, std::forward<Callable>(callable));
-}
-
-
-}  // namespace detail
-
-
 /**
  * A row_major accessor is a bridge between a range and the row-major memory
  * layout.
@@ -288,20 +59,19 @@ GKO_ATTRIBUTES void multidim_for_each(const std::array<ValueType, N> &size,
  * constructor parameters for this class to the range (it will forward it to
  * this class).
  *
- * @warning For backward compatability reasons, a specialization is provided
- *          for dimensionality == 2.
+ * @warning The current implementation is incomplete, and only allows for
+ *          2-dimensional ranges.
  *
  * @tparam ValueType  type of values this accessor returns
- * @tparam Dimensionality  number of dimensions of this accessor
+ * @tparam Dimensionality  number of dimensions of this accessor (has to be 2)
  */
 template <typename ValueType, size_type Dimensionality>
 class row_major {
 public:
     friend class range<row_major>;
 
-    static_assert(Dimensionality != 0,
-                  "This accessor does not support a dimensionality of 0!");
-    static constexpr size_type dimensionality = Dimensionality;
+    static_assert(Dimensionality == 2,
+                  "This accessor is only implemented for matrices");
 
     /**
      * Type of values returned by the accessor.
@@ -313,183 +83,11 @@ class row_major {
      */
     using data_type = value_type *;
 
-    /**
-     * Number of dimensions of the accessor.
-     */
-
-    using const_accessor = row_major<const ValueType, Dimensionality>;
-    using stride_type = std::array<const size_type, dimensionality - 1>;
-    using length_type = std::array<const size_type, dimensionality>;
-
-protected:
-    /**
-     * Creates a row_major accessor.
-     *
-     * @param data  pointer to the block of memory containing the data
-     * @param lengths size / length of the accesses of each dimension
-     * @param stride  distance (in elements) between starting positions of
-     *                the dimensions (i.e.
-     *                `x_1 * stride_1 + x_2 * stride_2 * ... + x_n`
-     *                points to the element at (x_1, x_2, ..., x_n))
-     */
-    constexpr GKO_ATTRIBUTES explicit row_major(data_type data,
-                                                dim<dimensionality> size,
-                                                stride_type stride)
-        : data{data},
-          lengths(detail::to_array<const size_type>(size)),
-          stride(stride)
-    {}
-
-    /**
-     * Creates a row_major accessor with a default stride (assumes no padding)
-     *
-     * @param data  pointer to the block of memory containing the data
-     * @param lengths size / length of the accesses of each dimension
-     */
-    constexpr GKO_ATTRIBUTES explicit row_major(data_type data,
-                                                dim<dimensionality> size)
-        : data{data},
-          lengths(detail::to_array<const size_type>(size)),
-          stride(detail::compute_default_stride_array(lengths))
-    {}
-
-public:
-    /**
-     * Creates a row_major range which contains a read-only version of the
-     * current accessor.
-     *
-     * @returns  a row major range which is read-only.
-     */
-    constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
-    {
-        // TODO Remove this functionality all together (if requested)
-        return range<const_accessor>(data, lengths, stride);
-    }
-
-    /**
-     * Returns the data element at the specified indices
-     *
-     * @param row  row index
-     * @param col  column index
-     *
-     * @return data element at (indices...)
-     */
-    template <typename... Indices>
-    constexpr GKO_ATTRIBUTES
-        std::enable_if_t<are_all_integral<Indices...>::value, value_type &>
-        operator()(Indices &&... indices) const
-    {
-        return data[detail::compute_storage_index(
-            lengths, stride, std::forward<Indices>(indices)...)];
-    }
-
-    /**
-     * Returns the sub-range spanning the range (x1_span, x2_span, ...)
-     *
-     * @param rows  row span
-     * @param cols  column span
-     *
-     * @return sub-range spanning the given spans
-     */
-    template <typename... SpanTypes>
-    constexpr GKO_ATTRIBUTES std::enable_if_t<
-        detail::are_span_compatible<SpanTypes...>::value, range<row_major>>
-    operator()(SpanTypes... spans) const
-    {
-        return detail::validate_spans(lengths, spans...),
-               range<row_major>{
-                   data + detail::compute_storage_index(lengths, stride,
-                                                        (span{spans}.begin)...),
-                   dim<dimensionality>{
-                       (span{spans}.end - span{spans}.begin)...},
-                   stride};
-    }
-
-    /**
-     * Returns the length in dimension `dimension`.
-     *
-     * @param dimension  a dimension index
-     *
-     * @return length in dimension `dimension`
-     */
-    constexpr GKO_ATTRIBUTES size_type length(size_type dimension) const
-    {
-        return lengths[dimension];
-    }
-
-    /**
-     * Copies data from another accessor
-     *
-     * @warning Do not use this function since it is not optimized for a
-     *          specific executor. It will always be performed sequentially.
-     *          Please write an optimized version (adjusted to the architecture)
-     *          by iterating through the values yourself.
-     *
-     * @tparam OtherAccessor  type of the other accessor
-     *
-     * @param other  other accessor
-     */
-    template <typename OtherAccessor>
-    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
-    {
-        detail::multidim_for_each(lengths, [this, &other](auto... indices) {
-            (*this)(indices...) = other(indices...);
-        });
-    }
-
-    /**
-     * Reference to the underlying data.
-     */
-    const data_type data;
-
-    /**
-     * An array of dimension sizes.
-     */
-    const length_type lengths;
-
-    /**
-     * Distance between consecutive rows for each dimension (except the first).
-     */
-    const stride_type stride;
-};
-
-
-/**
- * A row_major accessor is a bridge between a range and the row-major memory
- * layout.
- *
- * You should never try to explicitly create an instance of this accessor.
- * Instead, supply it as a template parameter to a range, and pass the
- * constructor parameters for this class to the range (it will forward it to
- * this class).
- *
- * @note  This is the original implementation, which is now the specialization
- *        for Dimensionality = 2.
- *
- * @tparam ValueType  type of values this accessor returns
- */
-template <typename ValueType>
-class row_major<ValueType, 2> {
-public:
-    friend class range<row_major>;
-
     /**
      * Number of dimensions of the accessor.
      */
     static constexpr size_type dimensionality = 2;
 
-    /**
-     * Type of values returned by the accessor.
-     */
-    using value_type = ValueType;
-
-    /**
-     * Type of underlying data storage.
-     */
-    using data_type = value_type *;
-
-    using const_accessor = row_major<const ValueType, dimensionality>;
-
 protected:
     /**
      * Creates a row_major accessor.
@@ -508,48 +106,7 @@ class row_major<ValueType, 2> {
         : data{data}, lengths{num_rows, num_cols}, stride{stride}
     {}
 
-    /**
-     * Creates a row_major accessor.
-     *
-     * @param data  pointer to the block of memory containing the data
-     * @param lengths size / length of the accesses of each dimension
-     * @param stride  distance (in elements) between starting positions of
-     *                consecutive rows (i.e. `data + i * stride` points to
-     *                the `i`-th row)
-     */
-    constexpr GKO_ATTRIBUTES explicit row_major(data_type data,
-                                                dim<dimensionality> size,
-                                                size_type stride)
-        : data{data},
-          lengths(detail::to_array<const size_type>(size)),
-          stride(stride)
-    {}
-
-    /**
-     * Creates a row_major accessor with a default stride (assumes no padding)
-     *
-     * @param data  pointer to the block of memory containing the data
-     * @param lengths size / length of the accesses of each dimension
-     */
-    constexpr GKO_ATTRIBUTES explicit row_major(data_type data,
-                                                dim<dimensionality> size)
-        : data{data},
-          lengths(detail::to_array<const size_type>(size)),
-          stride{size[1]}
-    {}
-
 public:
-    /**
-     * Creates a row_major range which contains a read-only version of the
-     * current accessor.
-     *
-     * @returns  a row major range which is read-only.
-     */
-    constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
-    {
-        return range<const_accessor>{data, lengths[0], lengths[1], stride};
-    }
-
     /**
      * Returns the data element at position (row, col)
      *
@@ -635,303 +192,6 @@ class row_major<ValueType, 2> {
 };
 
 
-namespace detail {
-/**
- * Namespace for helper functions and structs for
- * the block column major accessor.
- */
-namespace blk_col_major {
-
-
-/**
- * Runs from first to last dimension in order to compute the index.
- *
- * The index is computed like this:
- * indices: x1, x2, x3, ..., xn
- * compute(stride, x1, x2, x3, ..., x(n-1), xn) ->
- *  x1 * stride[0] + x2 * stride[1] + ...
- *    + x(n-2) * stride[n-3] + x(n-1) + xn * stride[n-2]
- * Note that swap of the last two strides, making this 'block column major'.
- */
-template <typename ValueType, size_type total_dim, size_type current_iter = 1>
-struct index_helper_s {
-    static_assert(total_dim >= 1, "Dimensionality must be >= 1");
-    static_assert(current_iter <= total_dim, "Iteration must be <= total_dim!");
-
-    static constexpr size_type dim_idx{current_iter - 1};
-
-    template <typename FirstType, typename... Indices>
-    static constexpr GKO_ATTRIBUTES ValueType
-    compute(const std::array<ValueType, total_dim> &size,
-            const std::array<ValueType, (total_dim > 0 ? total_dim - 1 : 0)>
-                &stride,
-            FirstType first, Indices &&... idxs)
-    {
-        if (current_iter == total_dim - 1) {
-            return GKO_ASSERT(first < size[dim_idx]),
-                   first +
-                       index_helper_s<ValueType, total_dim, current_iter + 1>::
-                           compute(size, stride,
-                                   std::forward<Indices>(idxs)...);
-        }
-
-        return GKO_ASSERT(first < size[dim_idx]),
-               first * stride[dim_idx] +
-                   index_helper_s<ValueType, total_dim, current_iter + 1>::
-                       compute(size, stride, std::forward<Indices>(idxs)...);
-    }
-};
-
-template <typename ValueType, size_type total_dim>
-struct index_helper_s<ValueType, total_dim, total_dim> {
-    static_assert(total_dim >= 2, "Dimensionality must be >= 2");
-
-    static constexpr size_type dim_idx{total_dim - 1};
-
-    template <typename FirstType>
-    static constexpr GKO_ATTRIBUTES ValueType
-    compute(const std::array<ValueType, total_dim> &size,
-            const std::array<ValueType, (total_dim > 1 ? total_dim - 1 : 0)>
-                &stride,
-            FirstType first)
-    {
-        return GKO_ASSERT(first < size[total_dim - 1]),
-               first * stride[dim_idx - 1];
-    }
-};
-
-/**
- * Computes the flat storage index for block-column-major access.
- *
- * @param size  the multi-dimensional sizes of the range of values
- * @param stride  the stride array
- * @param idxs  the multi-dimensional indices of the desired entry
- */
-template <typename ValueType, size_type total_dim, typename... Indices>
-constexpr GKO_ATTRIBUTES ValueType compute_index(
-    const std::array<ValueType, total_dim> &size,
-    const std::array<ValueType, (total_dim > 0 ? total_dim - 1 : 0)> &stride,
-    Indices &&... idxs)
-{
-    return index_helper_s<ValueType, total_dim>::compute(
-        size, stride, std::forward<Indices>(idxs)...);
-}
-
-
-template <size_type iter = 1, typename ValueType, size_type N, typename... Args>
-constexpr GKO_ATTRIBUTES
-    std::enable_if_t<(iter == N - 1) && (iter == sizeof...(Args) + 1),
-                     std::array<ValueType, N - 1>>
-    default_stride_array_impl(const std::array<ValueType, N> &size,
-                              Args &&... args)
-{
-    return {{std::forward<Args>(args)..., size[N - 2]}};
-}
-
-template <size_type iter = 1, typename ValueType, size_type N, typename... Args>
-constexpr GKO_ATTRIBUTES std::enable_if_t<(iter < N - 1 || iter == N) &&
-                                              (iter == sizeof...(Args) + 1),
-                                          std::array<ValueType, N - 1>>
-default_stride_array_impl(const std::array<ValueType, N> &size, Args &&... args)
-{
-    return default_stride_array_impl<iter + 1>(
-        size, std::forward<Args>(args)...,
-        detail::mult_dim_upwards_impl<iter>(size));
-}
-
-template <typename ValueType, size_type dimensions>
-constexpr GKO_ATTRIBUTES
-    std::array<ValueType, (dimensions > 0 ? dimensions - 1 : 0)>
-    default_stride_array(const std::array<ValueType, dimensions> &size)
-{
-    return default_stride_array_impl(size);
-}
-
-
-}  // namespace blk_col_major
-}  // namespace detail
-
-
-/**
- * A bridge between a range and a block-column-major memory layout.
- *
- * Only the innermost two dimensions are regarded as defining
- * a column-major matrix, and the rest of the dimensions are treated
- * identically to \ref row_major.
- *
- * You should not try to explicitly create an instance of this accessor.
- * Instead, supply it as a template parameter to a range, and pass the
- * constructor parameters for this class to the range (it will forward it to
- * this class).
- *
- * @tparam ValueType  type of values this accessor returns
- * @tparam Dimensionality  number of dimensions of this accessor
- */
-template <typename ValueType, size_type Dimensionality>
-class block_col_major {
-public:
-    friend class range<block_col_major>;
-
-    static_assert(Dimensionality != 0,
-                  "This accessor does not support a dimensionality of 0!");
-    static_assert(Dimensionality != 1,
-                  "Please use row_major accessor for 1D ranges.");
-
-    /**
-     * Number of dimensions of the accessor.
-     */
-    static constexpr size_type dimensionality = Dimensionality;
-
-    /**
-     * Type of values returned by the accessor.
-     */
-    using value_type = ValueType;
-
-    /**
-     * Type of underlying data storage.
-     */
-    using data_type = value_type *;
-
-    using const_accessor = block_col_major<const ValueType, Dimensionality>;
-    using stride_type = std::array<const size_type, dimensionality - 1>;
-    using length_type = std::array<const size_type, dimensionality>;
-
-protected:
-    /**
-     * Creates a block_col_major accessor.
-     *
-     * @param data  pointer to the block of memory containing the data
-     * @param lengths size / length of the accesses of each dimension
-     * @param stride  distance (in elements) between starting positions of
-     *                the dimensions (i.e.
-     *   `x_1 * stride_1 + x_2 * stride_2 * ... + x_(n-1) + x_n * stride_(n-1)`
-     *                points to the element at (x_1, x_2, ..., x_n))
-     */
-    constexpr GKO_ATTRIBUTES explicit block_col_major(data_type data,
-                                                      dim<dimensionality> size,
-                                                      stride_type stride)
-        : data{data},
-          lengths(detail::to_array<const size_type>(size)),
-          stride(stride)
-    {}
-
-    /**
-     * Creates a block_col_major accessor with a default stride
-     * (assumes no padding)
-     *
-     * @param data  pointer to the block of memory containing the data
-     * @param lengths size / length of the accesses of each dimension
-     */
-    constexpr GKO_ATTRIBUTES explicit block_col_major(data_type data,
-                                                      dim<dimensionality> size)
-        : data{data},
-          lengths(detail::to_array<const size_type>(size)),
-          stride(detail::blk_col_major::default_stride_array(lengths))
-    {}
-
-public:
-    /**
-     * Creates a block_col_major range which contains a read-only version of
-     * the current accessor.
-     *
-     * @returns  a block column major range which is read-only.
-     */
-    constexpr GKO_ATTRIBUTES range<const_accessor> to_const() const
-    {
-        // TODO Remove this functionality all together (if requested)
-        return range<const_accessor>(data, lengths, stride);
-    }
-
-    /**
-     * Returns the data element at the specified indices
-     *
-     * @param row  row index
-     * @param col  column index
-     *
-     * @return data element at (indices...)
-     */
-    template <typename... Indices>
-    constexpr GKO_ATTRIBUTES
-        std::enable_if_t<are_all_integral<Indices...>::value, value_type &>
-        operator()(Indices &&... indices) const
-    {
-        return data[detail::blk_col_major::compute_index(
-            lengths, stride, std::forward<Indices>(indices)...)];
-    }
-
-    /**
-     * Returns the sub-range spanning the range (x1_span, x2_span, ...)
-     *
-     * @param rows  row span
-     * @param cols  column span
-     *
-     * @return sub-range spanning the given spans
-     */
-    template <typename... SpanTypes>
-    constexpr GKO_ATTRIBUTES
-        std::enable_if_t<detail::are_span_compatible<SpanTypes...>::value,
-                         range<block_col_major>>
-        operator()(SpanTypes... spans) const
-    {
-        return detail::validate_spans(lengths, spans...),
-               range<block_col_major>{
-                   data + detail::blk_col_major::compute_index(
-                              lengths, stride, (span{spans}.begin)...),
-                   dim<dimensionality>{
-                       (span{spans}.end - span{spans}.begin)...},
-                   stride};
-    }
-
-    /**
-     * Returns the length in dimension `dimension`.
-     *
-     * @param dimension  a dimension index
-     *
-     * @return length in dimension `dimension`
-     */
-    constexpr GKO_ATTRIBUTES size_type length(size_type dimension) const
-    {
-        return lengths[dimension];
-    }
-
-    /**
-     * Copies data from another accessor
-     *
-     * @warning Do not use this function since it is not optimized for a
-     *          specific executor. It will always be performed sequentially.
-     *          Please write an optimized version (adjusted to the architecture)
-     *          by iterating through the values yourself.
-     *
-     * @tparam OtherAccessor  type of the other accessor
-     *
-     * @param other  other accessor
-     */
-    template <typename OtherAccessor>
-    GKO_ATTRIBUTES void copy_from(const OtherAccessor &other) const
-    {
-        detail::multidim_for_each(lengths, [this, &other](auto... indices) {
-            (*this)(indices...) = other(indices...);
-        });
-    }
-
-    /**
-     * Reference to the underlying data.
-     */
-    const data_type data;
-
-    /**
-     * An array of dimension sizes.
-     */
-    const length_type lengths;
-
-    /**
-     * Distance between consecutive 'layers' for each dimension
-     * (except the second, for which it is 1).
-     */
-    const stride_type stride;
-};
-
-
 }  // namespace accessor
 }  // namespace gko
 

From 248d1b92673a9d18eec3be0aec72902df5926585 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Tue, 9 Mar 2021 11:35:44 +0100
Subject: [PATCH 0494/1008] Inside accessor, include header locally

Inside the accessor folder, header are now included without the
`accessor/` prefix.

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 accessor/accessor_helper.hpp          |  4 ++--
 accessor/accessor_references.hpp      |  2 +-
 accessor/index_span.hpp               |  2 +-
 accessor/range.hpp                    |  2 +-
 accessor/reduced_row_major.hpp        | 10 +++++-----
 accessor/scaled_reduced_row_major.hpp | 10 +++++-----
 6 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/accessor/accessor_helper.hpp b/accessor/accessor_helper.hpp
index ae29995f3ce..1d90004d8f8 100644
--- a/accessor/accessor_helper.hpp
+++ b/accessor/accessor_helper.hpp
@@ -41,8 +41,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <utility>
 
 
-#include "accessor/index_span.hpp"
-#include "accessor/utils.hpp"
+#include "index_span.hpp"
+#include "utils.hpp"
 
 
 namespace gko {
diff --git a/accessor/accessor_references.hpp b/accessor/accessor_references.hpp
index 3d9bd1bd136..35cd426e9e0 100644
--- a/accessor/accessor_references.hpp
+++ b/accessor/accessor_references.hpp
@@ -38,7 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <utility>
 
 
-#include "accessor/utils.hpp"
+#include "utils.hpp"
 
 
 // CUDA TOOLKIT < 11 does not support constexpr in combination with
diff --git a/accessor/index_span.hpp b/accessor/index_span.hpp
index 91522ed667b..6feb6007dad 100644
--- a/accessor/index_span.hpp
+++ b/accessor/index_span.hpp
@@ -33,7 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifndef GKO_ACCESSOR_INDEX_SPAN_HPP_
 #define GKO_ACCESSOR_INDEX_SPAN_HPP_
 
-#include "accessor/utils.hpp"
+#include "utils.hpp"
 
 
 namespace gko {
diff --git a/accessor/range.hpp b/accessor/range.hpp
index 258923464b1..be73d8dbf87 100644
--- a/accessor/range.hpp
+++ b/accessor/range.hpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <utility>
 
 
-#include "accessor/utils.hpp"
+#include "utils.hpp"
 
 
 namespace gko {
diff --git a/accessor/reduced_row_major.hpp b/accessor/reduced_row_major.hpp
index 772607a1be7..2b875c95a6b 100644
--- a/accessor/reduced_row_major.hpp
+++ b/accessor/reduced_row_major.hpp
@@ -40,11 +40,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <utility>
 
 
-#include "accessor/accessor_helper.hpp"
-#include "accessor/accessor_references.hpp"
-#include "accessor/index_span.hpp"
-#include "accessor/range.hpp"
-#include "accessor/utils.hpp"
+#include "accessor_helper.hpp"
+#include "accessor_references.hpp"
+#include "index_span.hpp"
+#include "range.hpp"
+#include "utils.hpp"
 
 
 namespace gko {
diff --git a/accessor/scaled_reduced_row_major.hpp b/accessor/scaled_reduced_row_major.hpp
index 9b605577865..c0711542fc0 100644
--- a/accessor/scaled_reduced_row_major.hpp
+++ b/accessor/scaled_reduced_row_major.hpp
@@ -39,11 +39,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <utility>
 
 
-#include "accessor/accessor_helper.hpp"
-#include "accessor/accessor_references.hpp"
-#include "accessor/index_span.hpp"
-#include "accessor/range.hpp"
-#include "accessor/utils.hpp"
+#include "accessor_helper.hpp"
+#include "accessor_references.hpp"
+#include "index_span.hpp"
+#include "range.hpp"
+#include "utils.hpp"
 
 
 namespace gko {

From 9150dc652b8cd4b2e70c1c5c8344bad675d24a45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 10 Mar 2021 19:32:44 +0100
Subject: [PATCH 0495/1008] Make stride&dim-array value-type non-const

Changes `std::array<const size_type, dim>` to
`std::array<size_type, dim>` in order to have an easier user-interface
(and const-ness is preserved because all length_types and stride_types
are const themselves).

Additionally, add a dim_type in {scaled_,}reduced_row_major accessors.
---
 accessor/block_col_major.hpp           |  4 +-
 accessor/reduced_row_major.hpp         | 27 ++++++--------
 accessor/row_major.hpp                 | 11 +++---
 accessor/scaled_reduced_row_major.hpp  | 30 +++++++--------
 core/test/accessor/block_col_major.cpp | 38 ++-----------------
 core/test/accessor/row_major.cpp       | 51 +++-----------------------
 6 files changed, 42 insertions(+), 119 deletions(-)

diff --git a/accessor/block_col_major.hpp b/accessor/block_col_major.hpp
index 275dc387f43..d9522d31662 100644
--- a/accessor/block_col_major.hpp
+++ b/accessor/block_col_major.hpp
@@ -85,8 +85,8 @@ class block_col_major {
     using data_type = value_type *;
 
     using const_accessor = block_col_major<const ValueType, Dimensionality>;
-    using stride_type = std::array<const size_type, dimensionality - 1>;
-    using length_type = std::array<const size_type, dimensionality>;
+    using stride_type = std::array<size_type, dimensionality - 1>;
+    using length_type = std::array<size_type, dimensionality>;
 
 protected:
     /**
diff --git a/accessor/reduced_row_major.hpp b/accessor/reduced_row_major.hpp
index 2b875c95a6b..8a0cf248d2e 100644
--- a/accessor/reduced_row_major.hpp
+++ b/accessor/reduced_row_major.hpp
@@ -90,6 +90,7 @@ class reduced_row_major {
     friend class range<reduced_row_major>;
 
 protected:
+    using dim_type = std::array<size_type, dimensionality>;
     using storage_stride_type = std::array<size_type, dimensionality - 1>;
     using reference_type =
         reference_class::reduced_storage<arithmetic_type, storage_type>;
@@ -103,9 +104,9 @@ class reduced_row_major {
      * @param storage  pointer to the block of memory containing the storage
      * @param stride  stride array used for memory accesses
      */
-    constexpr GKO_ACC_ATTRIBUTES reduced_row_major(
-        std::array<size_type, dimensionality> size, storage_type *storage,
-        storage_stride_type stride)
+    constexpr GKO_ACC_ATTRIBUTES reduced_row_major(dim_type size,
+                                                   storage_type *storage,
+                                                   storage_stride_type stride)
         : size_(size), storage_{storage}, stride_(stride)
     {}
 
@@ -119,9 +120,9 @@ class reduced_row_major {
      * @param strides  strides used for memory accesses
      */
     template <typename... Strides>
-    constexpr GKO_ACC_ATTRIBUTES reduced_row_major(
-        std::array<size_type, dimensionality> size, storage_type *storage,
-        Strides &&... strides)
+    constexpr GKO_ACC_ATTRIBUTES reduced_row_major(dim_type size,
+                                                   storage_type *storage,
+                                                   Strides &&... strides)
         : reduced_row_major{
               size, storage,
               storage_stride_type{{std::forward<Strides>(strides)...}}}
@@ -137,8 +138,8 @@ class reduced_row_major {
      * @param storage  pointer to the block of memory containing the storage
      * @param size  multidimensional size of the memory
      */
-    constexpr GKO_ACC_ATTRIBUTES reduced_row_major(
-        std::array<size_type, dimensionality> size, storage_type *storage)
+    constexpr GKO_ACC_ATTRIBUTES reduced_row_major(dim_type size,
+                                                   storage_type *storage)
         : reduced_row_major{
               size, storage,
               helper::compute_default_row_major_stride_array<size_type>(size)}
@@ -229,7 +230,7 @@ class reduced_row_major {
     {
         return helper::validate_index_spans(size_, spans...),
                range<reduced_row_major>{
-                   std::array<size_type, dimensionality>{
+                   dim_type{
                        (index_span{spans}.end - index_span{spans}.begin)...},
                    storage_ + compute_index((index_span{spans}.begin)...),
                    stride_};
@@ -240,11 +241,7 @@ class reduced_row_major {
      *
      * @returns the size of the accessor
      */
-    constexpr GKO_ACC_ATTRIBUTES std::array<size_type, dimensionality>
-    get_size() const
-    {
-        return size_;
-    }
+    constexpr GKO_ACC_ATTRIBUTES dim_type get_size() const { return size_; }
 
     /**
      * Returns a pointer to a stride array of size dimensionality - 1
@@ -286,7 +283,7 @@ class reduced_row_major {
     }
 
 private:
-    const std::array<size_type, dimensionality> size_;
+    const dim_type size_;
     storage_type *storage_;
     const storage_stride_type stride_;
 };
diff --git a/accessor/row_major.hpp b/accessor/row_major.hpp
index 1facb6d4b33..b1515c4e742 100644
--- a/accessor/row_major.hpp
+++ b/accessor/row_major.hpp
@@ -83,8 +83,8 @@ class row_major {
      */
 
     using const_accessor = row_major<const ValueType, Dimensionality>;
-    using stride_type = std::array<const size_type, dimensionality - 1>;
-    using length_type = std::array<const size_type, dimensionality>;
+    using stride_type = std::array<size_type, dimensionality - 1>;
+    using length_type = std::array<size_type, dimensionality>;
 
 protected:
     /**
@@ -112,10 +112,9 @@ class row_major {
      */
     constexpr GKO_ACC_ATTRIBUTES explicit row_major(data_type data,
                                                     length_type size)
-        : row_major{
-              data, size,
-              helper::compute_default_row_major_stride_array<const size_type>(
-                  size)}
+        : row_major{data, size,
+                    helper::compute_default_row_major_stride_array<
+                        typename stride_type::value_type>(size)}
     {}
 
 public:
diff --git a/accessor/scaled_reduced_row_major.hpp b/accessor/scaled_reduced_row_major.hpp
index c0711542fc0..213050d5697 100644
--- a/accessor/scaled_reduced_row_major.hpp
+++ b/accessor/scaled_reduced_row_major.hpp
@@ -193,6 +193,7 @@ class scaled_reduced_row_major
     static constexpr size_type scalar_stride_dim{
         scalar_dim == 0 ? 0 : (scalar_dim - 1)};
 
+    using dim_type = std::array<size_type, dimensionality>;
     using storage_stride_type = std::array<size_type, dimensionality - 1>;
     using scalar_stride_type = std::array<size_type, scalar_stride_dim>;
     using reference_type =
@@ -211,7 +212,7 @@ class scaled_reduced_row_major
      * @param scalar_stride  stride array used for memory accesses to scalar
      */
     constexpr GKO_ACC_ATTRIBUTES scaled_reduced_row_major(
-        std::array<size_type, dimensionality> size, storage_type *storage,
+        dim_type size, storage_type *storage,
         storage_stride_type storage_stride, scalar_type *scalar,
         scalar_stride_type scalar_stride)
         : size_(size),
@@ -233,13 +234,13 @@ class scaled_reduced_row_major
      *                values.
      */
     constexpr GKO_ACC_ATTRIBUTES scaled_reduced_row_major(
-        std::array<size_type, dimensionality> size, storage_type *storage,
-        storage_stride_type stride, scalar_type *scalar)
+        dim_type size, storage_type *storage, storage_stride_type stride,
+        scalar_type *scalar)
         : scaled_reduced_row_major{
               size, storage, stride, scalar,
               helper::compute_default_masked_row_major_stride_array<
-                  size_type, scalar_mask, scalar_stride_dim, dimensionality>(
-                  size)}
+                  typename scalar_stride_type::value_type, scalar_mask,
+                  scalar_stride_dim, dimensionality>(size)}
     {}
 
     /**
@@ -251,12 +252,13 @@ class scaled_reduced_row_major
      * @param scalar  pointer to the block of memory containing the scalar
      *                values.
      */
-    constexpr GKO_ACC_ATTRIBUTES scaled_reduced_row_major(
-        std::array<size_type, dimensionality> size, storage_type *storage,
-        scalar_type *scalar)
+    constexpr GKO_ACC_ATTRIBUTES scaled_reduced_row_major(dim_type size,
+                                                          storage_type *storage,
+                                                          scalar_type *scalar)
         : scaled_reduced_row_major{
               size, storage,
-              helper::compute_default_row_major_stride_array<size_type>(size),
+              helper::compute_default_row_major_stride_array<
+                  typename storage_stride_type::value_type>(size),
               scalar}
     {}
 
@@ -384,7 +386,7 @@ class scaled_reduced_row_major
     {
         return helper::validate_index_spans(size_, spans...),
                range<scaled_reduced_row_major>{
-                   std::array<size_type, dimensionality>{
+                   dim_type{
                        (index_span{spans}.end - index_span{spans}.begin)...},
                    storage_ + compute_index((index_span{spans}.begin)...),
                    storage_stride_,
@@ -398,11 +400,7 @@ class scaled_reduced_row_major
      *
      * @returns the size of the accessor
      */
-    constexpr GKO_ACC_ATTRIBUTES std::array<size_type, dimensionality>
-    get_size() const
-    {
-        return size_;
-    }
+    constexpr GKO_ACC_ATTRIBUTES dim_type get_size() const { return size_; }
 
     /**
      * Returns a const reference to the storage stride array of size
@@ -504,7 +502,7 @@ class scaled_reduced_row_major
 
 
 private:
-    const std::array<size_type, dimensionality> size_;
+    const dim_type size_;
     storage_type *storage_;
     const storage_stride_type storage_stride_;
     scalar_type *scalar_;
diff --git a/core/test/accessor/block_col_major.cpp b/core/test/accessor/block_col_major.cpp
index b64ddf5ff99..cb911c81743 100644
--- a/core/test/accessor/block_col_major.cpp
+++ b/core/test/accessor/block_col_major.cpp
@@ -81,19 +81,18 @@ class BlockColMajorAccessor3d : public ::testing::Test {
         */
     };
     // clang-format on
-    const std::array<const gko::acc::size_type, dimensionality> dim1{2, 3, 4};
-    const std::array<const gko::acc::size_type, dimensionality> dim2{2, 2, 3};
+    const std::array<gko::acc::size_type, dimensionality> dim1{2, 3, 4};
+    const std::array<gko::acc::size_type, dimensionality> dim2{2, 2, 3};
     blk_col_major_range default_r{data, dim1};
     blk_col_major_range custom_r{
-        data, dim2,
-        std::array<const gko::acc::size_type, dimensionality - 1>{12, 3}};
+        data, dim2, std::array<gko::acc::size_type, dimensionality - 1>{12, 3}};
 };
 
 
 TEST_F(BlockColMajorAccessor3d, ComputesCorrectStride)
 {
     auto range_stride = default_r.get_accessor().stride;
-    auto check_stride = std::array<const gko::acc::size_type, 2>{12, 3};
+    auto check_stride = std::array<gko::acc::size_type, 2>{12, 3};
 
     ASSERT_EQ(range_stride, check_stride);
 }
@@ -154,34 +153,5 @@ TEST_F(BlockColMajorAccessor3d, CanCreateColumnVector)
     EXPECT_EQ(subr(1, 0, 0), 28);
 }
 
-/*
-TEST_F(BlockColMajorAccessor3d, CanAssignValues)
-{
-    default_r(1, 1, 1) = default_r(0, 0, 0);
-
-    EXPECT_EQ(data[16], 1);
-}
-
-
-TEST_F(BlockColMajorAccessor3d, CanAssignSubranges)
-{
-    default_r(1u, span{0, 2}, span{0, 3}) =
-        custom_r(0u, span{0, 2}, span{0, 3});
-
-    EXPECT_EQ(data[12], 1);
-    EXPECT_EQ(data[15], 2);
-    EXPECT_EQ(data[18], -1);
-    EXPECT_EQ(data[21], 24);
-    EXPECT_EQ(data[13], 3);
-    EXPECT_EQ(data[16], 4);
-    EXPECT_EQ(data[19], -2);
-    EXPECT_EQ(data[22], 28);
-    EXPECT_EQ(data[14], 29);
-    EXPECT_EQ(data[17], 30);
-    EXPECT_EQ(data[20], 31);
-    EXPECT_EQ(data[23], 32);
-}
-*/
-
 
 }  // namespace
diff --git a/core/test/accessor/row_major.cpp b/core/test/accessor/row_major.cpp
index fcc3f118972..c9d8a765dd9 100644
--- a/core/test/accessor/row_major.cpp
+++ b/core/test/accessor/row_major.cpp
@@ -54,8 +54,8 @@ namespace {
 class RowMajorAccessor : public ::testing::Test {
 protected:
     using span = gko::acc::index_span;
-    using dim_type = std::array<const gko::acc::size_type, 2>;
-    using stride_type = std::array<const gko::acc::size_type, 1>;
+    using dim_type = std::array<gko::acc::size_type, 2>;
+    using stride_type = std::array<gko::acc::size_type, 1>;
 
     using row_major_int_range = gko::acc::range<gko::acc::row_major<int, 2>>;
 
@@ -156,24 +156,6 @@ TEST_F(RowMajorAccessor, CanAssignValues)
 }
 
 
-/*
-TEST_F(RowMajorAccessor, CanAssignSubranges)
-{
-    r(0, span{0, 2}) = r(1, span{0, 2});
-
-    EXPECT_EQ(data[0], 3);
-    EXPECT_EQ(data[1], 4);
-    EXPECT_EQ(data[2], -1);
-    EXPECT_EQ(data[3], 3);
-    EXPECT_EQ(data[4], 4);
-    EXPECT_EQ(data[5], -2);
-    EXPECT_EQ(data[6], 5);
-    EXPECT_EQ(data[7], 6);
-    EXPECT_EQ(data[8], -3);
-}
-*/
-
-
 class RowMajorAccessor3d : public ::testing::Test {
 protected:
     using span = gko::acc::index_span;
@@ -193,12 +175,11 @@ class RowMajorAccessor3d : public ::testing::Test {
         29, 30, 31, 32
     };
     // clang-format on
-    const std::array<const gko::acc::size_type, dimensionality> dim1{2, 3, 4};
-    const std::array<const gko::acc::size_type, dimensionality> dim2{2, 2, 3};
+    const std::array<gko::acc::size_type, dimensionality> dim1{2, 3, 4};
+    const std::array<gko::acc::size_type, dimensionality> dim2{2, 2, 3};
     row_major_int_range default_r{data, dim1};
     row_major_int_range custom_r{
-        data, dim2,
-        std::array<const gko::size_type, dimensionality - 1>{12, 4}};
+        data, dim2, std::array<gko::size_type, dimensionality - 1>{12, 4}};
 };
 
 
@@ -265,26 +246,4 @@ TEST_F(RowMajorAccessor3d, CanAssignValues)
 }
 
 
-/*
-TEST_F(RowMajorAccessor3d, CanAssignSubranges)
-{
-    default_r(1u, span{0, 2}, span{0, 3}) =
-        custom_r(0u, span{0, 2}, span{0, 3});
-
-    EXPECT_EQ(data[12], 1);
-    EXPECT_EQ(data[13], 2);
-    EXPECT_EQ(data[14], -1);
-    EXPECT_EQ(data[15], 24);
-    EXPECT_EQ(data[16], 3);
-    EXPECT_EQ(data[17], 4);
-    EXPECT_EQ(data[18], -2);
-    EXPECT_EQ(data[19], 28);
-    EXPECT_EQ(data[20], 29);
-    EXPECT_EQ(data[21], 30);
-    EXPECT_EQ(data[22], 31);
-    EXPECT_EQ(data[23], 32);
-}
-*/
-
-
 }  // namespace

From 8c4d3383f265a5248d2cf132b9844290e3064d76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 11 Mar 2021 10:16:21 +0100
Subject: [PATCH 0496/1008] Prevent narrowing casting in row_major accessor

---
 core/test/accessor/block_col_major.cpp |  4 ++--
 core/test/accessor/row_major.cpp       | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/core/test/accessor/block_col_major.cpp b/core/test/accessor/block_col_major.cpp
index cb911c81743..2697aaf9e0f 100644
--- a/core/test/accessor/block_col_major.cpp
+++ b/core/test/accessor/block_col_major.cpp
@@ -127,7 +127,7 @@ TEST_F(BlockColMajorAccessor3d, CanWriteData)
 
 TEST_F(BlockColMajorAccessor3d, CanCreateSubrange)
 {
-    auto subr = custom_r(span{0, 2}, span{1, 2}, span{1, 3});
+    auto subr = custom_r(span{0u, 2u}, span{1u, 2u}, span{1u, 3u});
 
     EXPECT_EQ(subr(0, 0, 0), 4);
     EXPECT_EQ(subr(0, 0, 1), -2);
@@ -138,7 +138,7 @@ TEST_F(BlockColMajorAccessor3d, CanCreateSubrange)
 
 TEST_F(BlockColMajorAccessor3d, CanCreateRowVector)
 {
-    auto subr = default_r(1u, 2u, span{0, 2});
+    auto subr = default_r(1u, 2u, span{0u, 2u});
 
     EXPECT_EQ(subr(0, 0, 0), 29);
     EXPECT_EQ(subr(0, 0, 1), 30);
diff --git a/core/test/accessor/row_major.cpp b/core/test/accessor/row_major.cpp
index c9d8a765dd9..1b6e901c03b 100644
--- a/core/test/accessor/row_major.cpp
+++ b/core/test/accessor/row_major.cpp
@@ -120,7 +120,7 @@ TEST_F(RowMajorAccessor, CanWriteData)
 
 TEST_F(RowMajorAccessor, CanCreateSubrange)
 {
-    auto subr = r(span{1, 3}, span{0, 2});
+    auto subr = r(span{1u, 3u}, span{0u, 2u});
 
     EXPECT_EQ(subr(0, 0), 3);
     EXPECT_EQ(subr(0, 1), 4);
@@ -131,7 +131,7 @@ TEST_F(RowMajorAccessor, CanCreateSubrange)
 
 TEST_F(RowMajorAccessor, CanCreateRowVector)
 {
-    auto subr = r(2, span{0, 2});
+    auto subr = r(2u, span{0u, 2u});
 
     EXPECT_EQ(subr(0, 0), 5);
     EXPECT_EQ(subr(0, 1), 6);
@@ -140,7 +140,7 @@ TEST_F(RowMajorAccessor, CanCreateRowVector)
 
 TEST_F(RowMajorAccessor, CanCreateColumnVector)
 {
-    auto subr = r(span{0, 3}, 0);
+    auto subr = r(span{0u, 3u}, 0u);
 
     EXPECT_EQ(subr(0, 0), 1);
     EXPECT_EQ(subr(1, 0), 3);
@@ -211,7 +211,7 @@ TEST_F(RowMajorAccessor3d, CanWriteData)
 
 TEST_F(RowMajorAccessor3d, CanCreateSubrange)
 {
-    auto subr = custom_r(span{0, 2}, span{1, 2}, span{1, 3});
+    auto subr = custom_r(span{0u, 2u}, span{1u, 2u}, span{1u, 3u});
 
     EXPECT_EQ(subr(0, 0, 0), 4);
     EXPECT_EQ(subr(0, 0, 1), -2);
@@ -222,7 +222,7 @@ TEST_F(RowMajorAccessor3d, CanCreateSubrange)
 
 TEST_F(RowMajorAccessor3d, CanCreateRowVector)
 {
-    auto subr = default_r(1u, 2u, span{0, 2});
+    auto subr = default_r(1u, 2u, span{0u, 2u});
 
     EXPECT_EQ(subr(0, 0, 0), 29);
     EXPECT_EQ(subr(0, 0, 1), 30);

From 11e93a7e3180ee209c7e6134a9c089ad1171fb0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Fri, 12 Mar 2021 10:13:29 +0100
Subject: [PATCH 0497/1008] Remove `copy_from` functionality from Accessors

This step was forgotten previously.
Additionally, `scaled_reduced_row_major` now statically rejects complex
types (since they don't work).

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
---
 accessor/accessor_references.hpp      |  6 ++----
 accessor/block_col_major.hpp          | 20 --------------------
 accessor/reduced_row_major.hpp        | 20 --------------------
 accessor/row_major.hpp                | 20 --------------------
 accessor/scaled_reduced_row_major.hpp | 26 +++-----------------------
 accessor/utils.hpp                    | 24 ++++++++++++++++++++++++
 6 files changed, 29 insertions(+), 87 deletions(-)

diff --git a/accessor/accessor_references.hpp b/accessor/accessor_references.hpp
index 35cd426e9e0..19c2ddbc49a 100644
--- a/accessor/accessor_references.hpp
+++ b/accessor/accessor_references.hpp
@@ -223,7 +223,7 @@ namespace reference_class {
  * conversion between both formats is done with a simple static_cast.
  *
  * Copying this reference is disabled, but move construction is possible to
- * allow for an additional layer (like gko::range).
+ * allow for an additional layer (like gko::acc::range).
  * The assignment operator only works for an rvalue reference (&&) to
  * prevent accidentally copying the reference and working on a reference.
  *
@@ -331,7 +331,6 @@ template <typename ArithmeticType, typename StorageType>
 constexpr remove_complex_t<ArithmeticType> abs(
     const reduced_storage<ArithmeticType, StorageType> &ref)
 {
-    // For some reason necessary to get the correct abs function
     using std::abs;
     return abs(static_cast<ArithmeticType>(ref));
 }
@@ -345,7 +344,7 @@ constexpr remove_complex_t<ArithmeticType> abs(
  * scalar before casting to the StorageType).
  *
  * Copying this reference is disabled, but move construction is possible to
- * allow for an additional layer (like gko::range).
+ * allow for an additional layer (like gko::acc::range).
  * The assignment operator only works for an rvalue reference (&&) to
  * prevent accidentally copying and working on the reference.
  *
@@ -455,7 +454,6 @@ template <typename ArithmeticType, typename StorageType>
 constexpr remove_complex_t<ArithmeticType> abs(
     const scaled_reduced_storage<ArithmeticType, StorageType> &ref)
 {
-    // For some reason necessary to get the correct abs function
     using std::abs;
     return abs(static_cast<ArithmeticType>(ref));
 }
diff --git a/accessor/block_col_major.hpp b/accessor/block_col_major.hpp
index d9522d31662..defa9cb924e 100644
--- a/accessor/block_col_major.hpp
+++ b/accessor/block_col_major.hpp
@@ -184,26 +184,6 @@ class block_col_major {
         return lengths[dimension];
     }
 
-    /**
-     * Copies data from another accessor
-     *
-     * @warning Do not use this function since it is not optimized for a
-     *          specific executor. It will always be performed sequentially.
-     *          Please write an optimized version (adjusted to the architecture)
-     *          by iterating through the values yourself.
-     *
-     * @tparam OtherAccessor  type of the other accessor
-     *
-     * @param other  other accessor
-     */
-    template <typename OtherAccessor>
-    GKO_ACC_ATTRIBUTES void copy_from(const OtherAccessor &other) const
-    {
-        helper::multidim_for_each(lengths, [this, &other](auto... indices) {
-            (*this)(indices...) = other(indices...);
-        });
-    }
-
     /**
      * Reference to the underlying data.
      */
diff --git a/accessor/reduced_row_major.hpp b/accessor/reduced_row_major.hpp
index 8a0cf248d2e..ae0f88ddab4 100644
--- a/accessor/reduced_row_major.hpp
+++ b/accessor/reduced_row_major.hpp
@@ -176,26 +176,6 @@ class reduced_row_major {
         return dimension < dimensionality ? size_[dimension] : 1;
     }
 
-    /**
-     * Copies data from another accessor
-     *
-     * @warning Do not use this function since it is not optimized for a
-     *          specific executor. It will always be performed sequentially.
-     *          Please write an optimized version (adjusted to the architecture)
-     *          by iterating through the values yourself.
-     *
-     * @tparam OtherAccessor  type of the other accessor
-     *
-     * @param other  other accessor
-     */
-    template <typename OtherAccessor>
-    GKO_ACC_ATTRIBUTES void copy_from(const OtherAccessor &other) const
-    {
-        helper::multidim_for_each(size_, [this, &other](auto... indices) {
-            (*this)(indices...) = other(indices...);
-        });
-    }
-
     /**
      * Returns the stored value for the given indices. If the storage is const,
      * a value is returned, otherwise, a reference is returned.
diff --git a/accessor/row_major.hpp b/accessor/row_major.hpp
index b1515c4e742..d0aa18e53f7 100644
--- a/accessor/row_major.hpp
+++ b/accessor/row_major.hpp
@@ -182,26 +182,6 @@ class row_major {
         return lengths[dimension];
     }
 
-    /**
-     * Copies data from another accessor
-     *
-     * @warning Do not use this function since it is not optimized for a
-     *          specific executor. It will always be performed sequentially.
-     *          Please write an optimized version (adjusted to the
-     * architecture) by iterating through the values yourself.
-     *
-     * @tparam OtherAccessor  type of the other accessor
-     *
-     * @param other  other accessor
-     */
-    template <typename OtherAccessor>
-    GKO_ACC_ATTRIBUTES void copy_from(const OtherAccessor &other) const
-    {
-        helper::multidim_for_each(lengths, [this, &other](auto... indices) {
-            (*this)(indices...) = other(indices...);
-        });
-    }
-
     /**
      * Reference to the underlying data.
      */
diff --git a/accessor/scaled_reduced_row_major.hpp b/accessor/scaled_reduced_row_major.hpp
index 213050d5697..cbb61244e96 100644
--- a/accessor/scaled_reduced_row_major.hpp
+++ b/accessor/scaled_reduced_row_major.hpp
@@ -177,6 +177,9 @@ class scaled_reduced_row_major
         scaled_reduced_row_major<dimensionality, arithmetic_type,
                                  const storage_type, ScalarMask>;
 
+    static_assert(!is_complex<ArithmeticType>::value &&
+                      !is_complex<StorageType>::value,
+                  "Both arithmetic and storage type must not be complex!");
     static_assert(Dimensionality >= 1,
                   "Dimensionality must be a positive number!");
     static_assert(dimensionality <= 32,
@@ -329,29 +332,6 @@ class scaled_reduced_row_major
         return dimension < dimensionality ? size_[dimension] : 1;
     }
 
-    /**
-     * Copies data from another accessor
-     *
-     * @warning Do not use this function since it is not optimized for a
-     *          specific executor. It will always be performed sequentially.
-     *          Please write an optimized version (adjusted to the architecture)
-     *          by iterating through the values yourself.
-     *
-     * @tparam OtherAccessor  type of the other accessor
-     *
-     * @param other  other accessor
-     */
-    template <typename OtherAccessor>
-    GKO_ACC_ATTRIBUTES void copy_from(const OtherAccessor &other) const
-    {
-        helper::multidim_for_each(size_, [this, &other](auto... indices) {
-            // especially inefficient if mask has not all bits set
-            this->write_scalar_masked(other.read_scalar_masked(indices...),
-                                      indices...);
-            (*this)(indices...) = other(indices...);
-        });
-    }
-
     /**
      * Returns the stored value for the given indices. If the storage is const,
      * a value is returned, otherwise, a reference is returned.
diff --git a/accessor/utils.hpp b/accessor/utils.hpp
index 4ce433bf259..b8234a36c65 100644
--- a/accessor/utils.hpp
+++ b/accessor/utils.hpp
@@ -118,6 +118,26 @@ struct remove_complex_impl<thrust::complex<T>> {
 #endif
 
 
+template <typename T>
+struct is_complex_impl {
+    static constexpr bool value{false};
+};
+
+
+template <typename T>
+struct is_complex_impl<std::complex<T>> {
+    static constexpr bool value{true};
+};
+
+
+#if defined(__CUDACC__) || defined(__HIPCC__)
+template <typename T>
+struct is_complex_impl<thrust::complex<T>> {
+    static constexpr bool value{true};
+};
+#endif
+
+
 }  // namespace detail
 
 
@@ -125,6 +145,10 @@ template <typename T>
 using remove_complex_t = typename detail::remove_complex_impl<T>::type;
 
 
+template <typename T>
+using is_complex = typename detail::is_complex_impl<T>;
+
+
 /**
  * Evaluates if all template arguments Args fulfill std::is_integral. If that is
  * the case, this class inherits from `std::true_type`, otherwise, it inherits

From 10a970c8425755b8f7e9a85051a675d762ea692f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Sat, 13 Mar 2021 08:10:17 +0100
Subject: [PATCH 0498/1008] Resolve compiler error with GCC 5.4

Simply always using double braces ({{}}) for std::array initialization
resolved the issue.
---
 core/test/accessor/block_col_major.cpp |  9 +++++----
 core/test/accessor/row_major.cpp       | 12 ++++++------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/core/test/accessor/block_col_major.cpp b/core/test/accessor/block_col_major.cpp
index 2697aaf9e0f..0ffa070ba53 100644
--- a/core/test/accessor/block_col_major.cpp
+++ b/core/test/accessor/block_col_major.cpp
@@ -81,18 +81,19 @@ class BlockColMajorAccessor3d : public ::testing::Test {
         */
     };
     // clang-format on
-    const std::array<gko::acc::size_type, dimensionality> dim1{2, 3, 4};
-    const std::array<gko::acc::size_type, dimensionality> dim2{2, 2, 3};
+    const std::array<gko::acc::size_type, dimensionality> dim1{{2, 3, 4}};
+    const std::array<gko::acc::size_type, dimensionality> dim2{{2, 2, 3}};
     blk_col_major_range default_r{data, dim1};
     blk_col_major_range custom_r{
-        data, dim2, std::array<gko::acc::size_type, dimensionality - 1>{12, 3}};
+        data, dim2,
+        std::array<gko::acc::size_type, dimensionality - 1>{{12, 3}}};
 };
 
 
 TEST_F(BlockColMajorAccessor3d, ComputesCorrectStride)
 {
     auto range_stride = default_r.get_accessor().stride;
-    auto check_stride = std::array<gko::acc::size_type, 2>{12, 3};
+    auto check_stride = std::array<gko::acc::size_type, 2>{{12, 3}};
 
     ASSERT_EQ(range_stride, check_stride);
 }
diff --git a/core/test/accessor/row_major.cpp b/core/test/accessor/row_major.cpp
index 1b6e901c03b..ef8b67b66f0 100644
--- a/core/test/accessor/row_major.cpp
+++ b/core/test/accessor/row_major.cpp
@@ -66,13 +66,13 @@ class RowMajorAccessor : public ::testing::Test {
         5, 6, -3
     };
     // clang-format on
-    row_major_int_range r{data, dim_type{3u, 2u}, stride_type{3u}};
+    row_major_int_range r{data, dim_type{{3u, 2u}}, stride_type{{3u}}};
 };
 
 
 TEST_F(RowMajorAccessor, CanCreateWithDim)
 {
-    row_major_int_range r2{data, dim_type{3, 2}, stride_type{3u}};
+    row_major_int_range r2{data, dim_type{{3, 2}}, stride_type{{3u}}};
 
     EXPECT_EQ(r2(0, 0), 1);
     EXPECT_EQ(r2(0, 1), 2);
@@ -85,7 +85,7 @@ TEST_F(RowMajorAccessor, CanCreateWithDim)
 
 TEST_F(RowMajorAccessor, CanCreateDefaultStride)
 {
-    row_major_int_range r2{data, dim_type{3, 3}};
+    row_major_int_range r2{data, dim_type{{3, 3}}};
 
     EXPECT_EQ(r2(0, 0), 1);
     EXPECT_EQ(r2(0, 1), 2);
@@ -175,11 +175,11 @@ class RowMajorAccessor3d : public ::testing::Test {
         29, 30, 31, 32
     };
     // clang-format on
-    const std::array<gko::acc::size_type, dimensionality> dim1{2, 3, 4};
-    const std::array<gko::acc::size_type, dimensionality> dim2{2, 2, 3};
+    const std::array<gko::acc::size_type, dimensionality> dim1{{2, 3, 4}};
+    const std::array<gko::acc::size_type, dimensionality> dim2{{2, 2, 3}};
     row_major_int_range default_r{data, dim1};
     row_major_int_range custom_r{
-        data, dim2, std::array<gko::size_type, dimensionality - 1>{12, 4}};
+        data, dim2, std::array<gko::size_type, dimensionality - 1>{{12, 4}}};
 };
 
 

From 1fa276a08e648c3e4bcb495eccb54718f88e5dad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Sat, 13 Mar 2021 16:20:47 +0100
Subject: [PATCH 0499/1008] Change Accessor constructor order

Now, the size is always first in the constructor, and the data pointer
is followed by the corresponding stride (if existing).
---
 accessor/block_col_major.hpp           | 28 +++++++++---------
 accessor/reduced_row_major.hpp         |  2 +-
 accessor/row_major.hpp                 | 40 +++++++++++++-------------
 accessor/scaled_reduced_row_major.hpp  |  4 +--
 accessor/utils.hpp                     |  5 ++--
 core/test/accessor/block_col_major.cpp |  4 +--
 core/test/accessor/row_major.cpp       | 21 +++-----------
 7 files changed, 45 insertions(+), 59 deletions(-)

diff --git a/accessor/block_col_major.hpp b/accessor/block_col_major.hpp
index defa9cb924e..6495180ba2f 100644
--- a/accessor/block_col_major.hpp
+++ b/accessor/block_col_major.hpp
@@ -99,10 +99,10 @@ class block_col_major {
      *   `x_1 * stride_1 + x_2 * stride_2 * ... + x_(n-1) + x_n * stride_(n-1)`
      *                points to the element at (x_1, x_2, ..., x_n))
      */
-    constexpr GKO_ACC_ATTRIBUTES explicit block_col_major(data_type data,
-                                                          length_type size,
+    constexpr GKO_ACC_ATTRIBUTES explicit block_col_major(length_type size,
+                                                          data_type data,
                                                           stride_type stride)
-        : data{data}, lengths(size), stride(stride)
+        : lengths(size), data{data}, stride(stride)
     {}
 
     /**
@@ -112,10 +112,10 @@ class block_col_major {
      * @param data  pointer to the block of memory containing the data
      * @param lengths size / length of the accesses of each dimension
      */
-    constexpr GKO_ACC_ATTRIBUTES explicit block_col_major(data_type data,
-                                                          length_type size)
-        : data{data},
-          lengths(size),
+    constexpr GKO_ACC_ATTRIBUTES explicit block_col_major(length_type size,
+                                                          data_type data)
+        : lengths(size),
+          data{data},
           stride(helper::blk_col_major::default_stride_array(lengths))
     {}
 
@@ -129,7 +129,7 @@ class block_col_major {
     constexpr GKO_ACC_ATTRIBUTES range<const_accessor> to_const() const
     {
         // TODO Remove this functionality all together (if requested)
-        return range<const_accessor>(data, lengths, stride);
+        return range<const_accessor>(lengths, data, stride);
     }
 
     /**
@@ -165,10 +165,10 @@ class block_col_major {
     {
         return helper::validate_index_spans(lengths, spans...),
                range<block_col_major>{
-                   data + helper::blk_col_major::compute_index(
-                              lengths, stride, (index_span{spans}.begin)...),
                    length_type{
                        (index_span{spans}.end - index_span{spans}.begin)...},
+                   data + helper::blk_col_major::compute_index(
+                              lengths, stride, (index_span{spans}.begin)...),
                    stride};
     }
 
@@ -185,14 +185,14 @@ class block_col_major {
     }
 
     /**
-     * Reference to the underlying data.
+     * An array of dimension sizes.
      */
-    const data_type data;
+    const length_type lengths;
 
     /**
-     * An array of dimension sizes.
+     * Reference to the underlying data.
      */
-    const length_type lengths;
+    const data_type data;
 
     /**
      * Distance between consecutive 'layers' for each dimension
diff --git a/accessor/reduced_row_major.hpp b/accessor/reduced_row_major.hpp
index ae0f88ddab4..2d271fbd340 100644
--- a/accessor/reduced_row_major.hpp
+++ b/accessor/reduced_row_major.hpp
@@ -264,7 +264,7 @@ class reduced_row_major {
 
 private:
     const dim_type size_;
-    storage_type *storage_;
+    storage_type *const storage_;
     const storage_stride_type stride_;
 };
 
diff --git a/accessor/row_major.hpp b/accessor/row_major.hpp
index d0aa18e53f7..3aed9fe685f 100644
--- a/accessor/row_major.hpp
+++ b/accessor/row_major.hpp
@@ -66,6 +66,10 @@ class row_major {
 
     static_assert(Dimensionality != 0,
                   "This accessor does not support a dimensionality of 0!");
+
+    /**
+     * Number of dimensions of the accessor.
+     */
     static constexpr size_type dimensionality = Dimensionality;
 
     /**
@@ -78,41 +82,37 @@ class row_major {
      */
     using data_type = value_type *;
 
-    /**
-     * Number of dimensions of the accessor.
-     */
-
     using const_accessor = row_major<const ValueType, Dimensionality>;
-    using stride_type = std::array<size_type, dimensionality - 1>;
     using length_type = std::array<size_type, dimensionality>;
+    using stride_type = std::array<size_type, dimensionality - 1>;
 
 protected:
     /**
      * Creates a row_major accessor.
      *
-     * @param data  pointer to the block of memory containing the data
      * @param lengths size / length of the accesses of each dimension
+     * @param data  pointer to the block of memory containing the data
      * @param stride  distance (in elements) between starting positions of
      *                the dimensions (i.e.
      *                `x_1 * stride_1 + x_2 * stride_2 * ... + x_n`
      *                points to the element at (x_1, x_2, ..., x_n))
      */
-    constexpr GKO_ACC_ATTRIBUTES explicit row_major(data_type data,
-                                                    length_type size,
+    constexpr GKO_ACC_ATTRIBUTES explicit row_major(length_type size,
+                                                    data_type data,
                                                     stride_type stride)
-        : data{data}, lengths(size), stride(stride)
+        : lengths(size), data{data}, stride(stride)
     {}
 
     /**
      * Creates a row_major accessor with a default stride (assumes no
      * padding)
      *
-     * @param data  pointer to the block of memory containing the data
      * @param lengths size / length of the accesses of each dimension
+     * @param data  pointer to the block of memory containing the data
      */
-    constexpr GKO_ACC_ATTRIBUTES explicit row_major(data_type data,
-                                                    length_type size)
-        : row_major{data, size,
+    constexpr GKO_ACC_ATTRIBUTES explicit row_major(length_type size,
+                                                    data_type data)
+        : row_major{size, data,
                     helper::compute_default_row_major_stride_array<
                         typename stride_type::value_type>(size)}
     {}
@@ -127,7 +127,7 @@ class row_major {
     constexpr GKO_ACC_ATTRIBUTES range<const_accessor> to_const() const
     {
         // TODO Remove this functionality all together (if requested)
-        return range<const_accessor>(data, lengths, stride);
+        return range<const_accessor>(lengths, data, stride);
     }
 
     /**
@@ -163,10 +163,10 @@ class row_major {
     {
         return helper::validate_index_spans(lengths, spans...),
                range<row_major>{
-                   data + helper::compute_row_major_index(
-                              lengths, stride, (index_span{spans}.begin)...),
                    length_type{
                        (index_span{spans}.end - index_span{spans}.begin)...},
+                   data + helper::compute_row_major_index(
+                              lengths, stride, (index_span{spans}.begin)...),
                    stride};
     }
 
@@ -183,14 +183,14 @@ class row_major {
     }
 
     /**
-     * Reference to the underlying data.
+     * An array of dimension sizes.
      */
-    const data_type data;
+    const length_type lengths;
 
     /**
-     * An array of dimension sizes.
+     * Reference to the underlying data.
      */
-    const length_type lengths;
+    const data_type data;
 
     /**
      * Distance between consecutive rows for each dimension (except the
diff --git a/accessor/scaled_reduced_row_major.hpp b/accessor/scaled_reduced_row_major.hpp
index cbb61244e96..356a116fb0a 100644
--- a/accessor/scaled_reduced_row_major.hpp
+++ b/accessor/scaled_reduced_row_major.hpp
@@ -483,9 +483,9 @@ class scaled_reduced_row_major
 
 private:
     const dim_type size_;
-    storage_type *storage_;
+    storage_type *const storage_;
     const storage_stride_type storage_stride_;
-    scalar_type *scalar_;
+    scalar_type *const scalar_;
     const scalar_stride_type scalar_stride_;
 };
 
diff --git a/accessor/utils.hpp b/accessor/utils.hpp
index b8234a36c65..79386536ca2 100644
--- a/accessor/utils.hpp
+++ b/accessor/utils.hpp
@@ -162,9 +162,8 @@ struct are_all_integral : public std::true_type {};
 
 template <typename First, typename... Args>
 struct are_all_integral<First, Args...>
-    : public std::conditional<std::is_integral<std::decay_t<First>>::value,
-                              are_all_integral<Args...>,
-                              std::false_type>::type {};
+    : public std::conditional_t<std::is_integral<std::decay_t<First>>::value,
+                                are_all_integral<Args...>, std::false_type> {};
 
 
 }  // namespace acc
diff --git a/core/test/accessor/block_col_major.cpp b/core/test/accessor/block_col_major.cpp
index 0ffa070ba53..265d615d0bc 100644
--- a/core/test/accessor/block_col_major.cpp
+++ b/core/test/accessor/block_col_major.cpp
@@ -83,9 +83,9 @@ class BlockColMajorAccessor3d : public ::testing::Test {
     // clang-format on
     const std::array<gko::acc::size_type, dimensionality> dim1{{2, 3, 4}};
     const std::array<gko::acc::size_type, dimensionality> dim2{{2, 2, 3}};
-    blk_col_major_range default_r{data, dim1};
+    blk_col_major_range default_r{dim1, data};
     blk_col_major_range custom_r{
-        data, dim2,
+        dim2, data,
         std::array<gko::acc::size_type, dimensionality - 1>{{12, 3}}};
 };
 
diff --git a/core/test/accessor/row_major.cpp b/core/test/accessor/row_major.cpp
index ef8b67b66f0..ba432741bc8 100644
--- a/core/test/accessor/row_major.cpp
+++ b/core/test/accessor/row_major.cpp
@@ -66,26 +66,13 @@ class RowMajorAccessor : public ::testing::Test {
         5, 6, -3
     };
     // clang-format on
-    row_major_int_range r{data, dim_type{{3u, 2u}}, stride_type{{3u}}};
+    row_major_int_range r{dim_type{{3u, 2u}}, data, stride_type{{3u}}};
 };
 
 
-TEST_F(RowMajorAccessor, CanCreateWithDim)
-{
-    row_major_int_range r2{data, dim_type{{3, 2}}, stride_type{{3u}}};
-
-    EXPECT_EQ(r2(0, 0), 1);
-    EXPECT_EQ(r2(0, 1), 2);
-    EXPECT_EQ(r2(1, 0), 3);
-    EXPECT_EQ(r2(1, 1), 4);
-    EXPECT_EQ(r2(2, 0), 5);
-    EXPECT_EQ(r2(2, 1), 6);
-}
-
-
 TEST_F(RowMajorAccessor, CanCreateDefaultStride)
 {
-    row_major_int_range r2{data, dim_type{{3, 3}}};
+    row_major_int_range r2{dim_type{{3, 3}}, data};
 
     EXPECT_EQ(r2(0, 0), 1);
     EXPECT_EQ(r2(0, 1), 2);
@@ -177,9 +164,9 @@ class RowMajorAccessor3d : public ::testing::Test {
     // clang-format on
     const std::array<gko::acc::size_type, dimensionality> dim1{{2, 3, 4}};
     const std::array<gko::acc::size_type, dimensionality> dim2{{2, 2, 3}};
-    row_major_int_range default_r{data, dim1};
+    row_major_int_range default_r{dim1, data};
     row_major_int_range custom_r{
-        data, dim2, std::array<gko::size_type, dimensionality - 1>{{12, 4}}};
+        dim2, data, std::array<gko::size_type, dimensionality - 1>{{12, 4}}};
 };
 
 

From 39bc01dc88675643994f6e71f6fb7d06bb7a2d2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Mon, 15 Mar 2021 16:08:30 +0100
Subject: [PATCH 0500/1008] Accessor tests no longer have dependency on Ginkgo

The accessor tests now make sure that no dependency (on public header)
exists inside the accessor headers by only having the source directory
of Ginkgo as an include directory (and no longer linking against
Ginkgo).
Only one test (`reduced_row_major_ginkgo`) still depends on Ginkgo since
it uses `gko::half`.
---
 core/test/accessor/CMakeLists.txt             |  32 +-
 core/test/accessor/block_col_major.cpp        |   6 +-
 core/test/accessor/reduced_row_major.cpp      | 410 +---------------
 .../accessor/reduced_row_major_ginkgo.cpp     | 445 ++++++++++++++++++
 core/test/accessor/row_major.cpp              |  12 +-
 .../accessor/scaled_reduced_row_major.cpp     |  42 +-
 6 files changed, 510 insertions(+), 437 deletions(-)
 create mode 100644 core/test/accessor/reduced_row_major_ginkgo.cpp

diff --git a/core/test/accessor/CMakeLists.txt b/core/test/accessor/CMakeLists.txt
index b83e171b19a..9fb216ad00c 100644
--- a/core/test/accessor/CMakeLists.txt
+++ b/core/test/accessor/CMakeLists.txt
@@ -1,7 +1,27 @@
-ginkgo_create_test(index_span)
-ginkgo_create_test(range)
+# Creates a test that only adds `Ginkgo_SOURCE_DIR` to the include directories
+# to make sure the accessors are truly independent of Ginkgo and can be used
+# as a header-only library.
+function(create_accessor_test test_name)
+    file(RELATIVE_PATH REL_BINARY_DIR
+        ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+    string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
+    add_executable("${TEST_TARGET_NAME}" "${test_name}.cpp")
+    target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_14)
+    target_include_directories("${TEST_TARGET_NAME}"
+        PRIVATE
+        "${Ginkgo_SOURCE_DIR}"
+        )
+    set_target_properties("${TEST_TARGET_NAME}" PROPERTIES
+        OUTPUT_NAME "${test_name}")
+    target_link_libraries("${TEST_TARGET_NAME}" PRIVATE GTest::Main GTest::GTest ${ARGN})
+    add_test(NAME "${REL_BINARY_DIR}/${test_name}" COMMAND "${TEST_TARGET_NAME}")
+endfunction(create_accessor_test)
+
+create_accessor_test(index_span)
+create_accessor_test(range)
 # Accessor tests:
-ginkgo_create_test(block_col_major)
-ginkgo_create_test(reduced_row_major)
-ginkgo_create_test(row_major)
-ginkgo_create_test(scaled_reduced_row_major)
+create_accessor_test(block_col_major)
+create_accessor_test(reduced_row_major)
+ginkgo_create_test(reduced_row_major_ginkgo) # needs Ginkgo for gko::half
+create_accessor_test(row_major)
+create_accessor_test(scaled_reduced_row_major)
diff --git a/core/test/accessor/block_col_major.cpp b/core/test/accessor/block_col_major.cpp
index 265d615d0bc..7a0572c04c5 100644
--- a/core/test/accessor/block_col_major.cpp
+++ b/core/test/accessor/block_col_major.cpp
@@ -38,13 +38,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
-
-
 #include "accessor/block_col_major.hpp"
 #include "accessor/index_span.hpp"
-#include "core/test/utils.hpp"
+#include "accessor/range.hpp"
 
 
 namespace {
diff --git a/core/test/accessor/reduced_row_major.cpp b/core/test/accessor/reduced_row_major.cpp
index 745c7e09b1d..c60b085908e 100644
--- a/core/test/accessor/reduced_row_major.cpp
+++ b/core/test/accessor/reduced_row_major.cpp
@@ -31,6 +31,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #include <array>
+#include <cmath>
+#include <limits>
 #include <tuple>
 #include <type_traits>
 
@@ -38,410 +40,26 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
-
-
 #include "accessor/index_span.hpp"
+#include "accessor/range.hpp"
 #include "accessor/reduced_row_major.hpp"
-#include "core/base/extended_float.hpp"
-#include "core/test/utils.hpp"
+#include "accessor/utils.hpp"
 
 
 namespace {
 
 
-template <typename ArithmeticStorageType>
-class ReducedStorage3d : public ::testing::Test {
-protected:
-    using ar_type =
-        typename std::tuple_element<0, decltype(ArithmeticStorageType{})>::type;
-    using st_type =
-        typename std::tuple_element<1, decltype(ArithmeticStorageType{})>::type;
-    using rcar_type = gko::remove_complex<ar_type>;
-    static constexpr rcar_type delta{std::is_same<ar_type, st_type>::value
-                                         ? 0
-                                         : ::r<st_type, ar_type>::value};
-
-    // Type for `check_accessor_correctness` to forward the indices
-    using t = std::tuple<int, int, int>;
-    using i_span = gko::acc::index_span;
-
-    using accessor = gko::acc::reduced_row_major<3, ar_type, st_type>;
-    using const_accessor =
-        gko::acc::reduced_row_major<3, ar_type, const st_type>;
-
-    using reduced_storage = gko::acc::range<accessor>;
-    using const_reduced_storage = gko::acc::range<const_accessor>;
-
-    const std::array<gko::size_type, 3> size{{4u, 3u, 2u}};
-    static constexpr gko::size_type data_elements{4 * 3 * 2};
-    // clang-format off
-    st_type data[data_elements] {
-        // 0, y, z
-        1.0, 2.01,
-        -1.02, 3.03,
-        4.04, -2.05,
-        // 1, y, z
-        5.06, 6.07,
-        2.08, 3.09,
-        -1.1, -9.11,
-        // 2, y, z
-        -2.12, 2.13,
-        0.14, 15.15,
-        -9.16, 8.17,
-        // 3, y, z
-        7.18, -6.19,
-        5.2, -4.21,
-        3.22, -2.23
-    };
-    // clang-format on
-    reduced_storage r{size, data};
-    const_reduced_storage cr{size, data};
-
-    // Casts val first to `st_type`, then to `ar_type` in order to be allowed
-    // to test for equality
-    template <typename T>
-    static ar_type c_st_ar(T val)
-    {
-        return static_cast<ar_type>(static_cast<st_type>(val));
-    }
-
-    template <typename Accessor>
-    void check_accessor_correctness(const Accessor &a,
-                                    std::tuple<int, int, int> ignore = t(99, 99,
-                                                                         99))
-    {
-        // Test for equality is fine here since they should not be modified
-        // clang-format off
-        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), c_st_ar(1.0));     }
-        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), c_st_ar(2.01));    }
-        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), c_st_ar(-1.02));   }
-        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), c_st_ar(3.03));    }
-        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), c_st_ar(4.04));    }
-        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), c_st_ar(-2.05));   }
-        if (ignore != t(1, 0, 0)) { EXPECT_EQ(a(1, 0, 0), c_st_ar(5.06));    }
-        if (ignore != t(1, 0, 1)) { EXPECT_EQ(a(1, 0, 1), c_st_ar(6.07));    }
-        if (ignore != t(1, 1, 0)) { EXPECT_EQ(a(1, 1, 0), c_st_ar(2.08));    }
-        if (ignore != t(1, 1, 1)) { EXPECT_EQ(a(1, 1, 1), c_st_ar(3.09));    }
-        if (ignore != t(1, 2, 0)) { EXPECT_EQ(a(1, 2, 0), c_st_ar(-1.1));    }
-        if (ignore != t(1, 2, 1)) { EXPECT_EQ(a(1, 2, 1), c_st_ar(-9.11));   }
-        if (ignore != t(2, 0, 0)) { EXPECT_EQ(a(2, 0, 0), c_st_ar(-2.12));   }
-        if (ignore != t(2, 0, 1)) { EXPECT_EQ(a(2, 0, 1), c_st_ar(2.13));    }
-        if (ignore != t(2, 1, 0)) { EXPECT_EQ(a(2, 1, 0), c_st_ar(0.14));    }
-        if (ignore != t(2, 1, 1)) { EXPECT_EQ(a(2, 1, 1), c_st_ar(15.15));   }
-        if (ignore != t(2, 2, 0)) { EXPECT_EQ(a(2, 2, 0), c_st_ar(-9.16));   }
-        if (ignore != t(2, 2, 1)) { EXPECT_EQ(a(2, 2, 1), c_st_ar(8.17));    }
-        if (ignore != t(3, 0, 0)) { EXPECT_EQ(a(3, 0, 0), c_st_ar(7.18));    }
-        if (ignore != t(3, 0, 1)) { EXPECT_EQ(a(3, 0, 1), c_st_ar(-6.19));   }
-        if (ignore != t(3, 1, 0)) { EXPECT_EQ(a(3, 1, 0), c_st_ar(5.2));     }
-        if (ignore != t(3, 1, 1)) { EXPECT_EQ(a(3, 1, 1), c_st_ar(-4.21));   }
-        if (ignore != t(3, 2, 0)) { EXPECT_EQ(a(3, 2, 0), c_st_ar(3.22));    }
-        if (ignore != t(3, 2, 1)) { EXPECT_EQ(a(3, 2, 1), c_st_ar(-2.23));   }
-        // clang-format on
-    }
-};
-
-using ReducedStorage3dTypes =
-    ::testing::Types<std::tuple<double, double>, std::tuple<double, float>,
-                     std::tuple<float, float>, std::tuple<double, gko::half>,
-                     std::tuple<float, gko::half>,
-                     std::tuple<std::complex<double>, std::complex<double>>,
-                     std::tuple<std::complex<double>, std::complex<float>>,
-                     std::tuple<std::complex<float>, std::complex<float>>>;
-
-TYPED_TEST_SUITE(ReducedStorage3d, ReducedStorage3dTypes);
-
-
-TYPED_TEST(ReducedStorage3d, CorrectLengths)
-{
-    EXPECT_EQ(this->r.length(0), this->size[0]);
-    EXPECT_EQ(this->r.length(1), this->size[1]);
-    EXPECT_EQ(this->r.length(2), this->size[2]);
-    EXPECT_EQ(this->r.length(3), 1);
-    EXPECT_EQ(this->r->get_size(), this->size);
-}
-
-
-TYPED_TEST(ReducedStorage3d, CorrectStride)
-{
-    EXPECT_EQ(this->r->get_stride()[0], this->size[1] * this->size[2]);
-    EXPECT_EQ(this->r->get_stride().at(0), this->size[1] * this->size[2]);
-    EXPECT_EQ(this->r->get_stride()[1], this->size[2]);
-    EXPECT_EQ(this->r->get_stride().at(1), this->size[2]);
-}
-
-
-TYPED_TEST(ReducedStorage3d, CorrectStorage)
-{
-    EXPECT_EQ(this->r->get_stored_data(), this->data);
-    EXPECT_EQ(this->r->get_const_storage(), this->data);
-}
-
-
-TYPED_TEST(ReducedStorage3d, CanReadData)
-{
-    this->check_accessor_correctness(this->r);
-    this->check_accessor_correctness(this->cr);
-}
-
-
-TYPED_TEST(ReducedStorage3d, CanImplicitlyConvertToConst)
-{
-    using const_reduced_storage = typename TestFixture::const_reduced_storage;
-
-    const_reduced_storage const_rs = this->r->to_const();
-    const_reduced_storage const_rs2 = this->cr;
-
-    this->check_accessor_correctness(const_rs);
-    this->check_accessor_correctness(const_rs2);
-}
-
-
-TYPED_TEST(ReducedStorage3d, ToConstWorks)
-{
-    using const_reduced_storage = typename TestFixture::const_reduced_storage;
-
-    auto cr2 = this->r->to_const();
-
-    static_assert(std::is_same<decltype(cr2), const_reduced_storage>::value,
-                  "Types must be equal!");
-    this->check_accessor_correctness(cr2);
-}
-
-
-TYPED_TEST(ReducedStorage3d, CanCreateWithStride)
-{
-    using reduced_storage = typename TestFixture::reduced_storage;
-    using ar_type = typename TestFixture::ar_type;
-    auto size = std::array<gko::size_type, 3>{{2, 2, 2}};
-    auto stride = std::array<gko::size_type, 2>{{12, 2}};
-
-    auto range = reduced_storage{size, this->data, stride};
-    range(1, 1, 0) = ar_type{2.};
-
-    EXPECT_EQ(range(0, 0, 0), this->c_st_ar(1.0));
-    EXPECT_EQ(range(0, 0, 1), this->c_st_ar(2.01));
-    EXPECT_EQ(range(0, 1, 0), this->c_st_ar(-1.02));
-    EXPECT_EQ(range(0, 1, 1), this->c_st_ar(3.03));
-    EXPECT_EQ(range(1, 0, 0), this->c_st_ar(-2.12));
-    EXPECT_EQ(range(1, 0, 1), this->c_st_ar(2.13));
-    EXPECT_EQ(range(1, 1, 0), this->c_st_ar(2.));
-    EXPECT_EQ(range(1, 1, 1), this->c_st_ar(15.15));
-}
-
-
-TYPED_TEST(ReducedStorage3d, CanWriteData)
-{
-    using t = typename TestFixture::t;
-
-    this->r(0, 1, 0) = 100.25;
-
-    this->check_accessor_correctness(this->r, t(0, 1, 0));
-    EXPECT_EQ(this->r(0, 1, 0), this->c_st_ar(100.25));
-}
-
-
-TYPED_TEST(ReducedStorage3d, Assignment)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type expected = 1.2;
-
-    this->r(0, 0, 1) = expected;
-
-    this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_EQ(this->r(0, 0, 1), this->c_st_ar(expected));
-}
-
-
-TYPED_TEST(ReducedStorage3d, Assignment2)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type expected = -1.02;
-
-    this->r(0, 0, 1) = this->r(0, 1, 0);
-
-    this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_EQ(this->r(0, 0, 1), this->c_st_ar(expected));
-}
-
-
-TYPED_TEST(ReducedStorage3d, Addition)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    using gko::abs;
-    const ar_type expected = 1.2 + 2.01;
-
-    ar_type result = this->r(0, 0, 1) + ar_type{1.2};
-    this->r(0, 0, 1) += 1.2;
-
-    this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(abs(this->r(0, 0, 1)), abs(expected), TestFixture::delta);
-    EXPECT_NEAR(abs(result), abs(expected), TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Addition2)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    using gko::abs;
-    const ar_type expected = 2.01 + -1.02;
-
-    auto result = this->r(0, 0, 1) + this->r(0, 1, 0);
-    this->r(0, 0, 1) += this->r(0, 1, 0);
-
-    this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(abs(this->r(0, 0, 1)), abs(this->c_st_ar(expected)),
-                TestFixture::delta);
-    EXPECT_NEAR(abs(result), abs(this->c_st_ar(expected)), TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Subtraction)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    using gko::abs;
-    const ar_type expected = -2.23 - 1;
-
-    auto result = this->r(3, 2, 1) - ar_type{1.};
-    this->r(3, 2, 1) -= 1;
-
-    this->check_accessor_correctness(this->r, t(3, 2, 1));
-    EXPECT_NEAR(abs(this->r(3, 2, 1)), abs(this->c_st_ar(expected)),
-                TestFixture::delta);
-    EXPECT_NEAR(abs(result), abs(this->c_st_ar(expected)), TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Subtraction2)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    using gko::abs;
-    const ar_type expected = 3.22 - -2.23;
-
-    auto result = this->cr(3, 2, 0) - this->r(3, 2, 1);
-    this->r(3, 2, 0) -= this->r(3, 2, 1);
-
-    this->check_accessor_correctness(this->r, t(3, 2, 0));
-    EXPECT_NEAR(abs(this->r(3, 2, 0)), abs(this->c_st_ar(expected)),
-                TestFixture::delta);
-    EXPECT_NEAR(abs(result), abs(this->c_st_ar(expected)), TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Multiplication)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type expected = 1 * 2;
-
-    auto result = this->r(0, 0, 0) * ar_type{2.};
-    this->r(0, 0, 0) *= 2;
-
-    this->check_accessor_correctness(this->r, t(0, 0, 0));
-    EXPECT_EQ(this->r(0, 0, 0), this->c_st_ar(expected));
-    EXPECT_EQ(result, this->c_st_ar(expected));
-}
-
-
-TYPED_TEST(ReducedStorage3d, Multiplication2)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    using gko::abs;
-    const ar_type expected = 2.01 * 3.03;
-
-    auto result = this->r(0, 0, 1) * this->cr(0, 1, 1);
-    this->r(0, 0, 1) *= this->r(0, 1, 1);
-
-    this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(abs(this->r(0, 0, 1)), abs(expected), TestFixture::delta);
-    EXPECT_NEAR(abs(result), abs(expected), TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Division)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    using gko::abs;
-    const ar_type expected = 2.01 / 2.0;
-
-    auto result = this->cr(0, 0, 1) / ar_type{2.};
-    this->r(0, 0, 1) /= 2.;
-
-    this->check_accessor_correctness(this->r, t(0, 0, 1));
-    EXPECT_NEAR(abs(this->r(0, 0, 1)), abs(expected), TestFixture::delta);
-    EXPECT_NEAR(abs(result), abs(expected), TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, Division2)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    using gko::abs;
-    const ar_type expected = 5.06 / 4.04;
-
-    auto result = this->r(1, 0, 0) / this->cr(0, 2, 0);
-    this->r(1, 0, 0) /= this->r(0, 2, 0);
-
-    this->check_accessor_correctness(this->r, t(1, 0, 0));
-    EXPECT_NEAR(abs(this->r(1, 0, 0)), abs(expected), TestFixture::delta);
-    EXPECT_NEAR(abs(result), abs(expected), TestFixture::delta);
-}
-
-
-TYPED_TEST(ReducedStorage3d, UnaryMinus)
-{
-    using t = typename TestFixture::t;
-    using ar_type = typename TestFixture::ar_type;
-    const ar_type neg_expected = this->r(2, 0, 0);
-    const ar_type expected = -neg_expected;
-
-    auto result = -this->r(2, 0, 0);
-
-    this->check_accessor_correctness(this->r);
-    EXPECT_EQ(result, expected);
-}
-
-
-TYPED_TEST(ReducedStorage3d, CanCreateSubrange)
-{
-    using i_span = typename TestFixture::i_span;
-    auto subr = this->r(i_span{1u, 3u}, i_span{0u, 2u}, 0u);
-
-    EXPECT_EQ(subr(0, 0, 0), this->c_st_ar(5.06));
-    EXPECT_EQ(subr(0, 1, 0), this->c_st_ar(2.08));
-    EXPECT_EQ(subr(1, 0, 0), this->c_st_ar(-2.12));
-    EXPECT_EQ(subr(1, 1, 0), this->c_st_ar(0.14));
-}
-
-
-TYPED_TEST(ReducedStorage3d, CanCreateSubrange2)
-{
-    using i_span = typename TestFixture::i_span;
-    auto subr = this->cr(i_span{1u, 3u}, i_span{0u, 2u}, i_span{0u, 1u});
-
-    EXPECT_EQ(subr(0, 0, 0), this->c_st_ar(5.06));
-    EXPECT_EQ(subr(0, 1, 0), this->c_st_ar(2.08));
-    EXPECT_EQ(subr(1, 0, 0), this->c_st_ar(-2.12));
-    EXPECT_EQ(subr(1, 1, 0), this->c_st_ar(0.14));
-}
-
-
+/**
+ * This test makes sure reduced_row_major works independent of Ginkgo and with
+ * dimensionalities 1 and 2.
+ */
 class ReducedStorageXd : public ::testing::Test {
 protected:
     using ar_type = double;
     using st_type = float;
-    using size_type = gko::size_type;
-    static constexpr ar_type delta{::r<st_type>::value};
+    using size_type = gko::acc::size_type;
+    static constexpr ar_type delta{std::numeric_limits<st_type>::epsilon() *
+                                   1e1};
 
     using accessor1d = gko::acc::reduced_row_major<1, ar_type, st_type>;
     using accessor2d = gko::acc::reduced_row_major<2, ar_type, st_type>;
@@ -463,9 +81,9 @@ class ReducedStorageXd : public ::testing::Test {
 
     const std::array<size_type, 0> stride0{{}};
     const std::array<size_type, 1> stride1{{4}};
-    const std::array<gko::size_type, 1> size_1d{{8u}};
-    const std::array<gko::size_type, 2> size_2d{{2u, 4u}};
-    static constexpr gko::size_type data_elements{8};
+    const std::array<gko::acc::size_type, 1> size_1d{{8u}};
+    const std::array<gko::acc::size_type, 2> size_2d{{2u, 4u}};
+    static constexpr gko::acc::size_type data_elements{8};
     st_type data[data_elements]{1.1f, 2.2f, 3.3f, 4.4f,
                                 5.5f, 6.6f, 7.7f, -8.8f};
     reduced_storage1d r1{size_1d, data};
diff --git a/core/test/accessor/reduced_row_major_ginkgo.cpp b/core/test/accessor/reduced_row_major_ginkgo.cpp
new file mode 100644
index 00000000000..c64c2f4cefe
--- /dev/null
+++ b/core/test/accessor/reduced_row_major_ginkgo.cpp
@@ -0,0 +1,445 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <array>
+#include <cmath>
+#include <limits>
+#include <tuple>
+#include <type_traits>
+
+
+#include <gtest/gtest.h>
+
+
+#include "accessor/index_span.hpp"
+#include "accessor/range.hpp"
+#include "accessor/reduced_row_major.hpp"
+#include "accessor/utils.hpp"
+#include "core/base/extended_float.hpp"  // necessary for gko::half
+
+
+namespace {
+
+
+/**
+ * This test makes sure reduced_row_major works properly with various types.
+ * Note that this tests has a dependency on Ginkgo because of gko::half.
+ */
+template <typename ArithmeticStorageType>
+class ReducedStorage3d : public ::testing::Test {
+protected:
+    using ar_type =
+        typename std::tuple_element<0, decltype(ArithmeticStorageType{})>::type;
+    using st_type =
+        typename std::tuple_element<1, decltype(ArithmeticStorageType{})>::type;
+    using rcar_type = gko::acc::remove_complex_t<ar_type>;
+    static constexpr rcar_type delta{
+        std::is_same<ar_type, st_type>::value
+            ? 0
+            : std::numeric_limits<
+                  gko::acc::remove_complex_t<st_type>>::epsilon() *
+                  1e1};
+
+    // Type for `check_accessor_correctness` to forward the indices
+    using t = std::tuple<int, int, int>;
+    using i_span = gko::acc::index_span;
+
+    using accessor = gko::acc::reduced_row_major<3, ar_type, st_type>;
+    using const_accessor =
+        gko::acc::reduced_row_major<3, ar_type, const st_type>;
+
+    using reduced_storage = gko::acc::range<accessor>;
+    using const_reduced_storage = gko::acc::range<const_accessor>;
+
+    const std::array<gko::acc::size_type, 3> size{{4u, 3u, 2u}};
+    static constexpr gko::acc::size_type data_elements{4 * 3 * 2};
+    // clang-format off
+    st_type data[data_elements] {
+        // 0, y, z
+        1.0, 2.01,
+        -1.02, 3.03,
+        4.04, -2.05,
+        // 1, y, z
+        5.06, 6.07,
+        2.08, 3.09,
+        -1.1, -9.11,
+        // 2, y, z
+        -2.12, 2.13,
+        0.14, 15.15,
+        -9.16, 8.17,
+        // 3, y, z
+        7.18, -6.19,
+        5.2, -4.21,
+        3.22, -2.23
+    };
+    // clang-format on
+    reduced_storage r{size, data};
+    const_reduced_storage cr{size, data};
+
+    // Casts val first to `st_type`, then to `ar_type` in order to be allowed
+    // to test for equality
+    template <typename T>
+    static ar_type c_st_ar(T val)
+    {
+        return static_cast<ar_type>(static_cast<st_type>(val));
+    }
+
+    template <typename Accessor>
+    void check_accessor_correctness(const Accessor &a,
+                                    std::tuple<int, int, int> ignore = t(99, 99,
+                                                                         99))
+    {
+        // Test for equality is fine here since they should not be modified
+        // clang-format off
+        if (ignore != t(0, 0, 0)) { EXPECT_EQ(a(0, 0, 0), c_st_ar(1.0));     }
+        if (ignore != t(0, 0, 1)) { EXPECT_EQ(a(0, 0, 1), c_st_ar(2.01));    }
+        if (ignore != t(0, 1, 0)) { EXPECT_EQ(a(0, 1, 0), c_st_ar(-1.02));   }
+        if (ignore != t(0, 1, 1)) { EXPECT_EQ(a(0, 1, 1), c_st_ar(3.03));    }
+        if (ignore != t(0, 2, 0)) { EXPECT_EQ(a(0, 2, 0), c_st_ar(4.04));    }
+        if (ignore != t(0, 2, 1)) { EXPECT_EQ(a(0, 2, 1), c_st_ar(-2.05));   }
+        if (ignore != t(1, 0, 0)) { EXPECT_EQ(a(1, 0, 0), c_st_ar(5.06));    }
+        if (ignore != t(1, 0, 1)) { EXPECT_EQ(a(1, 0, 1), c_st_ar(6.07));    }
+        if (ignore != t(1, 1, 0)) { EXPECT_EQ(a(1, 1, 0), c_st_ar(2.08));    }
+        if (ignore != t(1, 1, 1)) { EXPECT_EQ(a(1, 1, 1), c_st_ar(3.09));    }
+        if (ignore != t(1, 2, 0)) { EXPECT_EQ(a(1, 2, 0), c_st_ar(-1.1));    }
+        if (ignore != t(1, 2, 1)) { EXPECT_EQ(a(1, 2, 1), c_st_ar(-9.11));   }
+        if (ignore != t(2, 0, 0)) { EXPECT_EQ(a(2, 0, 0), c_st_ar(-2.12));   }
+        if (ignore != t(2, 0, 1)) { EXPECT_EQ(a(2, 0, 1), c_st_ar(2.13));    }
+        if (ignore != t(2, 1, 0)) { EXPECT_EQ(a(2, 1, 0), c_st_ar(0.14));    }
+        if (ignore != t(2, 1, 1)) { EXPECT_EQ(a(2, 1, 1), c_st_ar(15.15));   }
+        if (ignore != t(2, 2, 0)) { EXPECT_EQ(a(2, 2, 0), c_st_ar(-9.16));   }
+        if (ignore != t(2, 2, 1)) { EXPECT_EQ(a(2, 2, 1), c_st_ar(8.17));    }
+        if (ignore != t(3, 0, 0)) { EXPECT_EQ(a(3, 0, 0), c_st_ar(7.18));    }
+        if (ignore != t(3, 0, 1)) { EXPECT_EQ(a(3, 0, 1), c_st_ar(-6.19));   }
+        if (ignore != t(3, 1, 0)) { EXPECT_EQ(a(3, 1, 0), c_st_ar(5.2));     }
+        if (ignore != t(3, 1, 1)) { EXPECT_EQ(a(3, 1, 1), c_st_ar(-4.21));   }
+        if (ignore != t(3, 2, 0)) { EXPECT_EQ(a(3, 2, 0), c_st_ar(3.22));    }
+        if (ignore != t(3, 2, 1)) { EXPECT_EQ(a(3, 2, 1), c_st_ar(-2.23));   }
+        // clang-format on
+    }
+};
+
+using ReducedStorage3dTypes =
+    ::testing::Types<std::tuple<double, double>, std::tuple<double, float>,
+                     std::tuple<float, float>, std::tuple<double, gko::half>,
+                     std::tuple<float, gko::half>,
+                     std::tuple<std::complex<double>, std::complex<double>>,
+                     std::tuple<std::complex<double>, std::complex<float>>,
+                     std::tuple<std::complex<float>, std::complex<float>>>;
+
+TYPED_TEST_SUITE(ReducedStorage3d, ReducedStorage3dTypes);
+
+
+TYPED_TEST(ReducedStorage3d, CorrectLengths)
+{
+    EXPECT_EQ(this->r.length(0), this->size[0]);
+    EXPECT_EQ(this->r.length(1), this->size[1]);
+    EXPECT_EQ(this->r.length(2), this->size[2]);
+    EXPECT_EQ(this->r.length(3), 1);
+    EXPECT_EQ(this->r->get_size(), this->size);
+}
+
+
+TYPED_TEST(ReducedStorage3d, CorrectStride)
+{
+    EXPECT_EQ(this->r->get_stride()[0], this->size[1] * this->size[2]);
+    EXPECT_EQ(this->r->get_stride().at(0), this->size[1] * this->size[2]);
+    EXPECT_EQ(this->r->get_stride()[1], this->size[2]);
+    EXPECT_EQ(this->r->get_stride().at(1), this->size[2]);
+}
+
+
+TYPED_TEST(ReducedStorage3d, CorrectStorage)
+{
+    EXPECT_EQ(this->r->get_stored_data(), this->data);
+    EXPECT_EQ(this->r->get_const_storage(), this->data);
+}
+
+
+TYPED_TEST(ReducedStorage3d, CanReadData)
+{
+    this->check_accessor_correctness(this->r);
+    this->check_accessor_correctness(this->cr);
+}
+
+
+TYPED_TEST(ReducedStorage3d, CanImplicitlyConvertToConst)
+{
+    using const_reduced_storage = typename TestFixture::const_reduced_storage;
+
+    const_reduced_storage const_rs = this->r->to_const();
+    const_reduced_storage const_rs2 = this->cr;
+
+    this->check_accessor_correctness(const_rs);
+    this->check_accessor_correctness(const_rs2);
+}
+
+
+TYPED_TEST(ReducedStorage3d, ToConstWorks)
+{
+    using const_reduced_storage = typename TestFixture::const_reduced_storage;
+
+    auto cr2 = this->r->to_const();
+
+    static_assert(std::is_same<decltype(cr2), const_reduced_storage>::value,
+                  "Types must be equal!");
+    this->check_accessor_correctness(cr2);
+}
+
+
+TYPED_TEST(ReducedStorage3d, CanCreateWithStride)
+{
+    using reduced_storage = typename TestFixture::reduced_storage;
+    using ar_type = typename TestFixture::ar_type;
+    auto size = std::array<gko::acc::size_type, 3>{{2, 2, 2}};
+    auto stride = std::array<gko::acc::size_type, 2>{{12, 2}};
+
+    auto range = reduced_storage{size, this->data, stride};
+    range(1, 1, 0) = ar_type{2.};
+
+    EXPECT_EQ(range(0, 0, 0), this->c_st_ar(1.0));
+    EXPECT_EQ(range(0, 0, 1), this->c_st_ar(2.01));
+    EXPECT_EQ(range(0, 1, 0), this->c_st_ar(-1.02));
+    EXPECT_EQ(range(0, 1, 1), this->c_st_ar(3.03));
+    EXPECT_EQ(range(1, 0, 0), this->c_st_ar(-2.12));
+    EXPECT_EQ(range(1, 0, 1), this->c_st_ar(2.13));
+    EXPECT_EQ(range(1, 1, 0), this->c_st_ar(2.));
+    EXPECT_EQ(range(1, 1, 1), this->c_st_ar(15.15));
+}
+
+
+TYPED_TEST(ReducedStorage3d, CanWriteData)
+{
+    using t = typename TestFixture::t;
+
+    this->r(0, 1, 0) = 100.25;
+
+    this->check_accessor_correctness(this->r, t(0, 1, 0));
+    EXPECT_EQ(this->r(0, 1, 0), this->c_st_ar(100.25));
+}
+
+
+TYPED_TEST(ReducedStorage3d, Assignment)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = 1.2;
+
+    this->r(0, 0, 1) = expected;
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_EQ(this->r(0, 0, 1), this->c_st_ar(expected));
+}
+
+
+TYPED_TEST(ReducedStorage3d, Assignment2)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = -1.02;
+
+    this->r(0, 0, 1) = this->r(0, 1, 0);
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_EQ(this->r(0, 0, 1), this->c_st_ar(expected));
+}
+
+
+TYPED_TEST(ReducedStorage3d, Addition)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    using std::abs;
+    const ar_type expected = 1.2 + 2.01;
+
+    ar_type result = this->r(0, 0, 1) + ar_type{1.2};
+    this->r(0, 0, 1) += 1.2;
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(abs(this->r(0, 0, 1)), abs(expected), TestFixture::delta);
+    EXPECT_NEAR(abs(result), abs(expected), TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Addition2)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    using std::abs;
+    const ar_type expected = 2.01 + -1.02;
+
+    auto result = this->r(0, 0, 1) + this->r(0, 1, 0);
+    this->r(0, 0, 1) += this->r(0, 1, 0);
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(abs(this->r(0, 0, 1)), abs(this->c_st_ar(expected)),
+                TestFixture::delta);
+    EXPECT_NEAR(abs(result), abs(this->c_st_ar(expected)), TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Subtraction)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    using std::abs;
+    const ar_type expected = -2.23 - 1;
+
+    auto result = this->r(3, 2, 1) - ar_type{1.};
+    this->r(3, 2, 1) -= 1;
+
+    this->check_accessor_correctness(this->r, t(3, 2, 1));
+    EXPECT_NEAR(abs(this->r(3, 2, 1)), abs(this->c_st_ar(expected)),
+                TestFixture::delta);
+    EXPECT_NEAR(abs(result), abs(this->c_st_ar(expected)), TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Subtraction2)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    using std::abs;
+    const ar_type expected = 3.22 - -2.23;
+
+    auto result = this->cr(3, 2, 0) - this->r(3, 2, 1);
+    this->r(3, 2, 0) -= this->r(3, 2, 1);
+
+    this->check_accessor_correctness(this->r, t(3, 2, 0));
+    EXPECT_NEAR(abs(this->r(3, 2, 0)), abs(this->c_st_ar(expected)),
+                TestFixture::delta);
+    EXPECT_NEAR(abs(result), abs(this->c_st_ar(expected)), TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Multiplication)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type expected = 1 * 2;
+
+    auto result = this->r(0, 0, 0) * ar_type{2.};
+    this->r(0, 0, 0) *= 2;
+
+    this->check_accessor_correctness(this->r, t(0, 0, 0));
+    EXPECT_EQ(this->r(0, 0, 0), this->c_st_ar(expected));
+    EXPECT_EQ(result, this->c_st_ar(expected));
+}
+
+
+TYPED_TEST(ReducedStorage3d, Multiplication2)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    using std::abs;
+    const ar_type expected = 2.01 * 3.03;
+
+    auto result = this->r(0, 0, 1) * this->cr(0, 1, 1);
+    this->r(0, 0, 1) *= this->r(0, 1, 1);
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(abs(this->r(0, 0, 1)), abs(expected), TestFixture::delta);
+    EXPECT_NEAR(abs(result), abs(expected), TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Division)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    using std::abs;
+    const ar_type expected = 2.01 / 2.0;
+
+    auto result = this->cr(0, 0, 1) / ar_type{2.};
+    this->r(0, 0, 1) /= 2.;
+
+    this->check_accessor_correctness(this->r, t(0, 0, 1));
+    EXPECT_NEAR(abs(this->r(0, 0, 1)), abs(expected), TestFixture::delta);
+    EXPECT_NEAR(abs(result), abs(expected), TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, Division2)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    using std::abs;
+    const ar_type expected = 5.06 / 4.04;
+
+    auto result = this->r(1, 0, 0) / this->cr(0, 2, 0);
+    this->r(1, 0, 0) /= this->r(0, 2, 0);
+
+    this->check_accessor_correctness(this->r, t(1, 0, 0));
+    EXPECT_NEAR(abs(this->r(1, 0, 0)), abs(expected), TestFixture::delta);
+    EXPECT_NEAR(abs(result), abs(expected), TestFixture::delta);
+}
+
+
+TYPED_TEST(ReducedStorage3d, UnaryMinus)
+{
+    using t = typename TestFixture::t;
+    using ar_type = typename TestFixture::ar_type;
+    const ar_type neg_expected = this->r(2, 0, 0);
+    const ar_type expected = -neg_expected;
+
+    auto result = -this->r(2, 0, 0);
+
+    this->check_accessor_correctness(this->r);
+    EXPECT_EQ(result, expected);
+}
+
+
+TYPED_TEST(ReducedStorage3d, CanCreateSubrange)
+{
+    using i_span = typename TestFixture::i_span;
+    auto subr = this->r(i_span{1u, 3u}, i_span{0u, 2u}, 0u);
+
+    EXPECT_EQ(subr(0, 0, 0), this->c_st_ar(5.06));
+    EXPECT_EQ(subr(0, 1, 0), this->c_st_ar(2.08));
+    EXPECT_EQ(subr(1, 0, 0), this->c_st_ar(-2.12));
+    EXPECT_EQ(subr(1, 1, 0), this->c_st_ar(0.14));
+}
+
+
+TYPED_TEST(ReducedStorage3d, CanCreateSubrange2)
+{
+    using i_span = typename TestFixture::i_span;
+    auto subr = this->cr(i_span{1u, 3u}, i_span{0u, 2u}, i_span{0u, 1u});
+
+    EXPECT_EQ(subr(0, 0, 0), this->c_st_ar(5.06));
+    EXPECT_EQ(subr(0, 1, 0), this->c_st_ar(2.08));
+    EXPECT_EQ(subr(1, 0, 0), this->c_st_ar(-2.12));
+    EXPECT_EQ(subr(1, 1, 0), this->c_st_ar(0.14));
+}
+
+
+}  // namespace
diff --git a/core/test/accessor/row_major.cpp b/core/test/accessor/row_major.cpp
index ba432741bc8..dd85904b634 100644
--- a/core/test/accessor/row_major.cpp
+++ b/core/test/accessor/row_major.cpp
@@ -38,14 +38,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
-
-
 #include "accessor/index_span.hpp"
+#include "accessor/range.hpp"
 #include "accessor/row_major.hpp"
-#include "core/base/extended_float.hpp"
-#include "core/test/utils.hpp"
 
 
 namespace {
@@ -146,7 +141,7 @@ TEST_F(RowMajorAccessor, CanAssignValues)
 class RowMajorAccessor3d : public ::testing::Test {
 protected:
     using span = gko::acc::index_span;
-    static constexpr gko::size_type dimensionality{3};
+    static constexpr gko::acc::size_type dimensionality{3};
 
     using row_major_int_range =
         gko::acc::range<gko::acc::row_major<int, dimensionality>>;
@@ -166,7 +161,8 @@ class RowMajorAccessor3d : public ::testing::Test {
     const std::array<gko::acc::size_type, dimensionality> dim2{{2, 2, 3}};
     row_major_int_range default_r{dim1, data};
     row_major_int_range custom_r{
-        dim2, data, std::array<gko::size_type, dimensionality - 1>{{12, 4}}};
+        dim2, data,
+        std::array<gko::acc::size_type, dimensionality - 1>{{12, 4}}};
 };
 
 
diff --git a/core/test/accessor/scaled_reduced_row_major.cpp b/core/test/accessor/scaled_reduced_row_major.cpp
index 7d8f95c981a..2cf14d5732b 100644
--- a/core/test/accessor/scaled_reduced_row_major.cpp
+++ b/core/test/accessor/scaled_reduced_row_major.cpp
@@ -31,6 +31,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #include <array>
+#include <cinttypes>
+#include <limits>
 #include <tuple>
 #include <type_traits>
 
@@ -38,14 +40,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
-
-
 #include "accessor/index_span.hpp"
+#include "accessor/range.hpp"
 #include "accessor/scaled_reduced_row_major.hpp"
-#include "core/base/extended_float.hpp"
-#include "core/test/utils.hpp"
 
 
 namespace {
@@ -62,7 +59,8 @@ class ScaledReducedStorage3d : public ::testing::Test {
     using t = std::tuple<int, int, int>;
     using i_span = gko::acc::index_span;
 
-    static constexpr ar_type delta{::r<ar_type>::value};
+    static constexpr ar_type delta{std::numeric_limits<ar_type>::epsilon() *
+                                   1e1};
 
     using accessor =
         gko::acc::scaled_reduced_row_major<3, ar_type, st_type, 0b0101>;
@@ -72,9 +70,9 @@ class ScaledReducedStorage3d : public ::testing::Test {
     using reduced_storage = gko::acc::range<accessor>;
     using const_reduced_storage = gko::acc::range<const_accessor>;
 
-    const std::array<gko::size_type, 3> size{{1u, 4u, 2u}};
-    static constexpr gko::size_type data_elements{8};
-    static constexpr gko::size_type scalar_elements{8};
+    const std::array<gko::acc::size_type, 3> size{{1u, 4u, 2u}};
+    static constexpr gko::acc::size_type data_elements{8};
+    static constexpr gko::acc::size_type scalar_elements{8};
     // clang-format off
     st_type data[8]{
         10, 11,
@@ -86,8 +84,8 @@ class ScaledReducedStorage3d : public ::testing::Test {
         1., 2., 3., 4., 5., 6., 7., 8.
     };
     // clang-format on
-    const std::array<gko::size_type, 2> storage_stride{{8, 2}};
-    const std::array<gko::size_type, 1> scalar_stride{{2}};
+    const std::array<gko::acc::size_type, 2> storage_stride{{8, 2}};
+    const std::array<gko::acc::size_type, 1> scalar_stride{{2}};
     reduced_storage r{size, data, storage_stride, scalar, scalar_stride};
     const_reduced_storage cr{size, data, scalar};
 
@@ -112,9 +110,9 @@ class ScaledReducedStorage3d : public ::testing::Test {
 };
 
 using ScaledReducedStorage3dTypes = ::testing::Types<
-    std::tuple<double, gko::int64>, std::tuple<double, gko::int32>,
-    std::tuple<double, gko::int16>, std::tuple<float, gko::int32>,
-    std::tuple<float, gko::int16>>;
+    std::tuple<double, std::int64_t>, std::tuple<double, std::int32_t>,
+    std::tuple<double, std::int16_t>, std::tuple<float, std::int32_t>,
+    std::tuple<float, std::int16_t>>;
 
 TYPED_TEST_SUITE(ScaledReducedStorage3d, ScaledReducedStorage3dTypes);
 
@@ -184,9 +182,9 @@ TYPED_TEST(ScaledReducedStorage3d, CanCreateWithStride)
 {
     using reduced_storage = typename TestFixture::reduced_storage;
     using ar_type = typename TestFixture::ar_type;
-    std::array<gko::size_type, 3> size{{2, 1, 2}};
-    std::array<gko::size_type, 2> stride_storage{{5, 2}};
-    std::array<gko::size_type, 1> stride_scalar{{4}};
+    std::array<gko::acc::size_type, 3> size{{2, 1, 2}};
+    std::array<gko::acc::size_type, 2> stride_storage{{5, 2}};
+    std::array<gko::acc::size_type, 1> stride_scalar{{4}};
 
     reduced_storage range{size, this->data, stride_storage, this->scalar,
                           stride_scalar};
@@ -390,7 +388,7 @@ class ScaledReducedStorageXd : public ::testing::Test {
 protected:
     using ar_type = double;
     using st_type = int;
-    using size_type = gko::size_type;
+    using size_type = gko::acc::size_type;
     static constexpr ar_type delta{0.1};
 
     using accessor1d =
@@ -416,10 +414,10 @@ class ScaledReducedStorageXd : public ::testing::Test {
     const std::array<size_type, 0> stride0{{}};
     const std::array<size_type, 1> stride1{{4}};
     const std::array<size_type, 1> stride_sc{{5}};
-    const std::array<gko::size_type, 1> size_1d{{8u}};
-    const std::array<gko::size_type, 2> size_2d{{2u, 2u}};
+    const std::array<gko::acc::size_type, 1> size_1d{{8u}};
+    const std::array<gko::acc::size_type, 2> size_2d{{2u, 2u}};
 
-    static constexpr gko::size_type data_elements{8};
+    static constexpr gko::acc::size_type data_elements{8};
     st_type data[data_elements]{10, 22, 32, 44, 54, 66, 76, -88};
     ar_type scalar[data_elements]{1e0,  5e-1, 1e-1, 5e-2,
                                   1e-2, 5e-3, 1e-3, 5e-4};

From c49a2e2e373cd2e2593c6b4456509eafa891d4cd Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 10 Mar 2021 10:16:05 +0100
Subject: [PATCH 0501/1008] Give `create_submatrix` a virtual behavior.

This is useful for users to be able to derive from Dense in order to
specify the behavior of our solvers (i.e., GMRES).
---
 include/ginkgo/core/matrix/dense.hpp | 40 +++++++++++++++++++---------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 336bb401574..41db97591a5 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -510,19 +510,7 @@ class Dense
                                             const span &columns,
                                             const size_type stride)
     {
-        row_major_range range_this{this->get_values(), this->get_size()[0],
-                                   this->get_size()[1], this->get_stride()};
-        auto range_result = range_this(rows, columns);
-        // TODO: can result in HUGE padding - which will be copied with the
-        // vector
-        return Dense::create(
-            this->get_executor(),
-            dim<2>{range_result.length(0), range_result.length(1)},
-            Array<ValueType>::view(
-                this->get_executor(),
-                range_result.length(0) * range_this.length(1) - columns.begin,
-                range_result->data),
-            stride);
+        return this->create_submatrix_impl(rows, columns, stride);
     }
 
     /**
@@ -682,6 +670,32 @@ class Dense
      */
     virtual void compute_norm2_impl(LinOp *result) const;
 
+    /**
+     * @copydoc create_submatrix(const span, const span, const size_type)
+     *
+     * @note  Other implementations of dense should override this function
+     *        instead of create_submatrix(const span, const span, const
+     *        size_type).
+     */
+    virtual std::unique_ptr<Dense> create_submatrix_impl(const span &rows,
+                                                         const span &columns,
+                                                         const size_type stride)
+    {
+        row_major_range range_this{this->get_values(), this->get_size()[0],
+                                   this->get_size()[1], this->get_stride()};
+        auto range_result = range_this(rows, columns);
+        // TODO: can result in HUGE padding - which will be copied with the
+        // vector
+        return Dense::create(
+            this->get_executor(),
+            dim<2>{range_result.length(0), range_result.length(1)},
+            Array<ValueType>::view(
+                this->get_executor(),
+                range_result.length(0) * range_this.length(1) - columns.begin,
+                range_result->data),
+            stride);
+    }
+
     void apply_impl(const LinOp *b, LinOp *x) const override;
 
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,

From a14d3e6e138a500359d07cdc2ca7b70c0779351d Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 10 Mar 2021 21:51:48 +0100
Subject: [PATCH 0502/1008] Add a new create_with_type_of and use it in GMRES.

---
 core/solver/gmres.cpp                |  7 ++--
 include/ginkgo/core/matrix/dense.hpp | 57 ++++++++++++++++++++++++++--
 2 files changed, 57 insertions(+), 7 deletions(-)

diff --git a/core/solver/gmres.cpp b/core/solver/gmres.cpp
index a5727cdb151..2337b937cb3 100644
--- a/core/solver/gmres.cpp
+++ b/core/solver/gmres.cpp
@@ -107,9 +107,10 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     auto dense_b = as<const Vector>(b);
     auto dense_x = as<Vector>(x);
     auto residual = Vector::create_with_config_of(dense_b);
-    auto krylov_bases = Vector::create(
-        exec, dim<2>{system_matrix_->get_size()[1] * (krylov_dim_ + 1),
-                     dense_b->get_size()[1]});
+    auto krylov_bases = Vector::create_with_type_of(
+        dense_b, exec,
+        dim<2>{system_matrix_->get_size()[1] * (krylov_dim_ + 1),
+               dense_b->get_size()[1]});
     std::shared_ptr<matrix::Dense<ValueType>> preconditioned_vector =
         Vector::create_with_config_of(dense_b);
     auto hessenberg = Vector::create(
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 41db97591a5..f80cabbb6bc 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -161,6 +161,41 @@ class Dense
         return (*other).create_with_same_config();
     }
 
+    /**
+     * Creates a Dense matrix with the same type and executor as another Dense
+     * matrix but a different size.
+     *
+     * @param other  The other matrix whose type we target.
+     * @param exec  The executor of the new matrix.
+     * @param size  The size of the new matrix.
+     * @param stride  The stride of the new matrix.
+     *
+     * @returns a Dense matrix with the type of other.
+     */
+    static std::unique_ptr<Dense> create_with_type_of(
+        const Dense *other, std::shared_ptr<const Executor> exec,
+        const dim<2> &size = dim<2>{})
+    {
+        // See create_with_config_of()
+        return (*other).create_with_type_of_impl(exec, size, size[1]);
+    }
+
+    /**
+     * @copydoc create_with_type_of(const Dense*, std::shared_ptr<const
+     * Executor>, const dim<2>)
+     *
+     * @param stride  The stride of the new matrix.
+     *
+     * @note This is an overload which allows full parameter specification.
+     */
+    static std::unique_ptr<Dense> create_with_type_of(
+        const Dense *other, std::shared_ptr<const Executor> exec,
+        const dim<2> &size, size_type stride)
+    {
+        // See create_with_config_of()
+        return (*other).create_with_type_of_impl(exec, size, stride);
+    }
+
     friend class Dense<next_precision<ValueType>>;
 
     void convert_to(Dense<next_precision<ValueType>> *result) const override;
@@ -638,6 +673,20 @@ class Dense
                              this->get_stride());
     }
 
+    /**
+     * Creates a Dense matrix with the same type as the callers matrix.
+     *
+     * @param size  size of the matrix
+     *
+     * @returns a Dense matrix with the same type as the caller.
+     */
+    virtual std::unique_ptr<Dense> create_with_type_of_impl(
+        std::shared_ptr<const Executor> exec, const dim<2> &size,
+        size_type stride) const
+    {
+        return Dense::create(exec, size, stride);
+    }
+
     /**
      * @copydoc scale(const LinOp *)
      *
@@ -745,7 +794,7 @@ class Dense
 template <typename Matrix, typename... TArgs>
 std::unique_ptr<Matrix> initialize(
     size_type stride, std::initializer_list<typename Matrix::value_type> vals,
-    std::shared_ptr<const Executor> exec, TArgs &&... create_args)
+    std::shared_ptr<const Executor> exec, TArgs &&...create_args)
 {
     using dense = matrix::Dense<typename Matrix::value_type>;
     size_type num_rows = vals.size();
@@ -784,7 +833,7 @@ std::unique_ptr<Matrix> initialize(
 template <typename Matrix, typename... TArgs>
 std::unique_ptr<Matrix> initialize(
     std::initializer_list<typename Matrix::value_type> vals,
-    std::shared_ptr<const Executor> exec, TArgs &&... create_args)
+    std::shared_ptr<const Executor> exec, TArgs &&...create_args)
 {
     return initialize<Matrix>(1, vals, std::move(exec),
                               std::forward<TArgs>(create_args)...);
@@ -817,7 +866,7 @@ std::unique_ptr<Matrix> initialize(
     size_type stride,
     std::initializer_list<std::initializer_list<typename Matrix::value_type>>
         vals,
-    std::shared_ptr<const Executor> exec, TArgs &&... create_args)
+    std::shared_ptr<const Executor> exec, TArgs &&...create_args)
 {
     using dense = matrix::Dense<typename Matrix::value_type>;
     size_type num_rows = vals.size();
@@ -865,7 +914,7 @@ template <typename Matrix, typename... TArgs>
 std::unique_ptr<Matrix> initialize(
     std::initializer_list<std::initializer_list<typename Matrix::value_type>>
         vals,
-    std::shared_ptr<const Executor> exec, TArgs &&... create_args)
+    std::shared_ptr<const Executor> exec, TArgs &&...create_args)
 {
     return initialize<Matrix>(vals.size() > 0 ? begin(vals)->size() : 0, vals,
                               std::move(exec),

From b149d4d8b2542bbb029b82e439a5c53469ef620d Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Wed, 10 Mar 2021 21:05:37 +0000
Subject: [PATCH 0503/1008] Format files

Co-authored-by: tcojean <tcojean@users.noreply.github.com>
---
 include/ginkgo/core/matrix/dense.hpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index f80cabbb6bc..01b41b5bff5 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -794,7 +794,7 @@ class Dense
 template <typename Matrix, typename... TArgs>
 std::unique_ptr<Matrix> initialize(
     size_type stride, std::initializer_list<typename Matrix::value_type> vals,
-    std::shared_ptr<const Executor> exec, TArgs &&...create_args)
+    std::shared_ptr<const Executor> exec, TArgs &&... create_args)
 {
     using dense = matrix::Dense<typename Matrix::value_type>;
     size_type num_rows = vals.size();
@@ -833,7 +833,7 @@ std::unique_ptr<Matrix> initialize(
 template <typename Matrix, typename... TArgs>
 std::unique_ptr<Matrix> initialize(
     std::initializer_list<typename Matrix::value_type> vals,
-    std::shared_ptr<const Executor> exec, TArgs &&...create_args)
+    std::shared_ptr<const Executor> exec, TArgs &&... create_args)
 {
     return initialize<Matrix>(1, vals, std::move(exec),
                               std::forward<TArgs>(create_args)...);
@@ -866,7 +866,7 @@ std::unique_ptr<Matrix> initialize(
     size_type stride,
     std::initializer_list<std::initializer_list<typename Matrix::value_type>>
         vals,
-    std::shared_ptr<const Executor> exec, TArgs &&...create_args)
+    std::shared_ptr<const Executor> exec, TArgs &&... create_args)
 {
     using dense = matrix::Dense<typename Matrix::value_type>;
     size_type num_rows = vals.size();
@@ -914,7 +914,7 @@ template <typename Matrix, typename... TArgs>
 std::unique_ptr<Matrix> initialize(
     std::initializer_list<std::initializer_list<typename Matrix::value_type>>
         vals,
-    std::shared_ptr<const Executor> exec, TArgs &&...create_args)
+    std::shared_ptr<const Executor> exec, TArgs &&... create_args)
 {
     return initialize<Matrix>(vals.size() > 0 ? begin(vals)->size() : 0, vals,
                               std::move(exec),

From d7ff0d5d1609fb137ee778784279da9bdca67c76 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 3 Dec 2020 07:40:17 +0100
Subject: [PATCH 0504/1008] First working reference of general isai

---
 core/device_hooks/common_kernels.inc.cpp      |     7 +
 core/preconditioner/isai.cpp                  |    41 +-
 core/preconditioner/isai_kernels.hpp          |    24 +-
 cuda/preconditioner/isai_kernels.cu           |    11 +
 dpcpp/preconditioner/isai_kernels.dp.cpp      |    11 +
 hip/preconditioner/isai_kernels.hip.cpp       |    11 +
 include/ginkgo/core/preconditioner/isai.hpp   |    15 +-
 matrices/CMakeLists.txt                       |     4 +
 matrices/test/1138_bus_isai.mtx               |  4057 +++++++
 matrices/test/a_large.mtx                     |  5052 ++++++++
 matrices/test/a_large_inv.mtx                 | 10002 ++++++++++++++++
 omp/preconditioner/isai_kernels.cpp           |    15 +-
 reference/preconditioner/isai_kernels.cpp     |   107 +-
 .../test/preconditioner/isai_kernels.cpp      |    46 +-
 14 files changed, 19382 insertions(+), 21 deletions(-)
 create mode 100644 matrices/test/1138_bus_isai.mtx
 create mode 100644 matrices/test/a_large.mtx
 create mode 100644 matrices/test/a_large_inv.mtx

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 408d6e4dbf0..ff25b9f85f7 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -1025,6 +1025,13 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_TRI_INVERSE_KERNEL);
 
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_ISAI_GENERATE_GENERAL_INVERSE_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_GENERATE_GENERAL_INVERSE_KERNEL);
+
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/preconditioner/isai.cpp b/core/preconditioner/isai.cpp
index 222eaf28835..163ab58b921 100644
--- a/core/preconditioner/isai.cpp
+++ b/core/preconditioner/isai.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <functional>
+#include <iostream>
 #include <memory>
 #include <type_traits>
 
@@ -42,8 +43,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/preconditioner/jacobi.hpp>
+#include <ginkgo/core/solver/gmres.hpp>
 #include <ginkgo/core/solver/lower_trs.hpp>
 #include <ginkgo/core/solver/upper_trs.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
 
 
 #include "core/base/utils.hpp"
@@ -56,6 +61,8 @@ namespace isai {
 
 
 GKO_REGISTER_OPERATION(generate_tri_inverse, isai::generate_tri_inverse);
+GKO_REGISTER_OPERATION(generate_general_inverse,
+                       isai::generate_general_inverse);
 GKO_REGISTER_OPERATION(generate_excess_system, isai::generate_excess_system);
 GKO_REGISTER_OPERATION(scatter_excess_solution, isai::scatter_excess_solution);
 
@@ -113,12 +120,15 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
     using Dense = matrix::Dense<ValueType>;
     using LowerTrs = solver::LowerTrs<ValueType, IndexType>;
     using UpperTrs = solver::UpperTrs<ValueType, IndexType>;
+    using Gmres = solver::Gmres<ValueType>;
+    using bj = preconditioner::Jacobi<ValueType, IndexType>;
     GKO_ASSERT_IS_SQUARE_MATRIX(input);
     auto exec = this->get_executor();
     auto to_invert = convert_to_with_sorting<Csr>(exec, input, skip_sorting);
     auto inverted = extend_sparsity(exec, to_invert, power);
     auto num_rows = inverted->get_size()[0];
     auto is_lower = IsaiType == isai_type::lower;
+    auto is_general = IsaiType == isai_type::general;
 
     // This stores the beginning of the RHS for the sparse block associated with
     // each row of inverted_l
@@ -127,9 +137,15 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
     // system of excess blocks
     Array<IndexType> excess_row_ptrs_full{exec, num_rows + 1};
 
-    exec->run(isai::make_generate_tri_inverse(
-        lend(to_invert), lend(inverted), excess_block_ptrs.get_data(),
-        excess_row_ptrs_full.get_data(), is_lower));
+    if (is_general) {
+        exec->run(isai::make_generate_general_inverse(
+            lend(to_invert), lend(inverted), excess_block_ptrs.get_data(),
+            excess_row_ptrs_full.get_data()));
+    } else {
+        exec->run(isai::make_generate_tri_inverse(
+            lend(to_invert), lend(inverted), excess_block_ptrs.get_data(),
+            excess_row_ptrs_full.get_data(), is_lower));
+    }
 
     auto excess_dim =
         exec->copy_val_to_host(excess_block_ptrs.get_const_data() + num_rows);
@@ -148,7 +164,20 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
             lend(excess_rhs)));
         // solve it after transposing
         std::unique_ptr<LinOpFactory> trs_factory;
-        if (is_lower) {
+        if (is_general) {
+            trs_factory =
+                Gmres::build()
+                    .with_preconditioner(
+                        bj::build().with_max_block_size(32u).on(exec))
+                    .with_criteria(
+                        gko::stop::Iteration::build()
+                            .with_max_iters(excess_dim)
+                            .on(exec),
+                        gko::stop::ResidualNormReduction<ValueType>::build()
+                            .with_reduction_factor(1e-10)
+                            .on(exec))
+                    .on(exec);
+        } else if (is_lower) {
             trs_factory = UpperTrs::build().on(exec);
         } else {
             trs_factory = LowerTrs::build().on(exec);
@@ -200,6 +229,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_LOWER_ISAI);
     class Isai<isai_type::upper, ValueType, IndexType>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_UPPER_ISAI);
 
+#define GKO_DECLARE_GENERAL_ISAI(ValueType, IndexType) \
+    class Isai<isai_type::general, ValueType, IndexType>
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_GENERAL_ISAI);
+
 
 }  // namespace preconditioner
 }  // namespace gko
diff --git a/core/preconditioner/isai_kernels.hpp b/core/preconditioner/isai_kernels.hpp
index 44d25b50b7a..5c67c4ca7f7 100644
--- a/core/preconditioner/isai_kernels.hpp
+++ b/core/preconditioner/isai_kernels.hpp
@@ -51,6 +51,14 @@ namespace kernels {
                               IndexType *excess_rhs_ptrs,                     \
                               IndexType *excess_nz_ptrs, bool lower)
 
+
+#define GKO_DECLARE_ISAI_GENERATE_GENERAL_INVERSE_KERNEL(ValueType, IndexType) \
+    void generate_general_inverse(                                             \
+        std::shared_ptr<const DefaultExecutor> exec,                           \
+        const matrix::Csr<ValueType, IndexType> *input,                        \
+        matrix::Csr<ValueType, IndexType> *inverse,                            \
+        IndexType *excess_rhs_ptrs, IndexType *excess_nz_ptrs)
+
 #define GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL(ValueType, IndexType) \
     void generate_excess_system(                                             \
         std::shared_ptr<const DefaultExecutor> exec,                         \
@@ -67,13 +75,15 @@ namespace kernels {
         const matrix::Dense<ValueType> *excess_solution,                      \
         matrix::Csr<ValueType, IndexType> *inverse)
 
-#define GKO_DECLARE_ALL_AS_TEMPLATES                                      \
-    constexpr auto row_size_limit = 32;                                   \
-    template <typename ValueType, typename IndexType>                     \
-    GKO_DECLARE_ISAI_GENERATE_TRI_INVERSE_KERNEL(ValueType, IndexType);   \
-    template <typename ValueType, typename IndexType>                     \
-    GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL(ValueType, IndexType); \
-    template <typename ValueType, typename IndexType>                     \
+#define GKO_DECLARE_ALL_AS_TEMPLATES                                        \
+    constexpr auto row_size_limit = 32;                                     \
+    template <typename ValueType, typename IndexType>                       \
+    GKO_DECLARE_ISAI_GENERATE_TRI_INVERSE_KERNEL(ValueType, IndexType);     \
+    template <typename ValueType, typename IndexType>                       \
+    GKO_DECLARE_ISAI_GENERATE_GENERAL_INVERSE_KERNEL(ValueType, IndexType); \
+    template <typename ValueType, typename IndexType>                       \
+    GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL(ValueType, IndexType);   \
+    template <typename ValueType, typename IndexType>                       \
     GKO_DECLARE_ISAI_SCATTER_EXCESS_SOLUTION_KERNEL(ValueType, IndexType)
 
 
diff --git a/cuda/preconditioner/isai_kernels.cu b/cuda/preconditioner/isai_kernels.cu
index e3e75b01eb5..c33a37255d5 100644
--- a/cuda/preconditioner/isai_kernels.cu
+++ b/cuda/preconditioner/isai_kernels.cu
@@ -107,6 +107,17 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_TRI_INVERSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
+                              const matrix::Csr<ValueType, IndexType> *input,
+                              matrix::Csr<ValueType, IndexType> *inverse,
+                              IndexType *excess_rhs_ptrs,
+                              IndexType *excess_nz_ptrs) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_GENERATE_GENERAL_INVERSE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void generate_excess_system(std::shared_ptr<const DefaultExecutor> exec,
                             const matrix::Csr<ValueType, IndexType> *input,
diff --git a/dpcpp/preconditioner/isai_kernels.dp.cpp b/dpcpp/preconditioner/isai_kernels.dp.cpp
index bbad36ea9ae..ac51807b839 100644
--- a/dpcpp/preconditioner/isai_kernels.dp.cpp
+++ b/dpcpp/preconditioner/isai_kernels.dp.cpp
@@ -86,6 +86,17 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_TRI_INVERSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
+                              const matrix::Csr<ValueType, IndexType> *input,
+                              matrix::Csr<ValueType, IndexType> *inverse,
+                              IndexType *excess_rhs_ptrs,
+                              IndexType *excess_nz_ptrs) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_GENERATE_GENERAL_INVERSE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void generate_excess_system(
     std::shared_ptr<const DefaultExecutor>,
diff --git a/hip/preconditioner/isai_kernels.hip.cpp b/hip/preconditioner/isai_kernels.hip.cpp
index 840d5e3c1ed..66591644624 100644
--- a/hip/preconditioner/isai_kernels.hip.cpp
+++ b/hip/preconditioner/isai_kernels.hip.cpp
@@ -110,6 +110,17 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_TRI_INVERSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
+                              const matrix::Csr<ValueType, IndexType> *input,
+                              matrix::Csr<ValueType, IndexType> *inverse,
+                              IndexType *excess_rhs_ptrs,
+                              IndexType *excess_nz_ptrs) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_GENERATE_GENERAL_INVERSE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void generate_excess_system(std::shared_ptr<const DefaultExecutor> exec,
                             const matrix::Csr<ValueType, IndexType> *input,
diff --git a/include/ginkgo/core/preconditioner/isai.hpp b/include/ginkgo/core/preconditioner/isai.hpp
index d88e1487514..2f8c63f4a8f 100644
--- a/include/ginkgo/core/preconditioner/isai.hpp
+++ b/include/ginkgo/core/preconditioner/isai.hpp
@@ -57,10 +57,10 @@ namespace preconditioner {
 /**
  * This enum lists the types of the ISAI preconditioner.
  *
- * ISAI can either generate a lower triangular matrix, or an upper triangular
- * matrix.
+ * ISAI can either generate a general matrix, a lower triangular matrix, or an
+ * upper triangular matrix.
  */
-enum struct isai_type { lower, upper };
+enum struct isai_type { lower, upper, general };
 
 /**
  * The Incomplete Sparse Approximate Inverse (ISAI) Preconditioner generates
@@ -101,6 +101,12 @@ class Isai : public EnableLinOp<Isai<IsaiType, ValueType, IndexType>>,
     friend class Isai<IsaiType == isai_type::lower ? isai_type::upper
                                                    : isai_type::lower,
                       ValueType, IndexType>;
+    friend class Isai<IsaiType == isai_type::upper ? isai_type::general
+                                                   : isai_type::upper,
+                      ValueType, IndexType>;
+    friend class Isai<IsaiType == isai_type::general ? isai_type::lower
+                                                     : isai_type::general,
+                      ValueType, IndexType>;
 
 public:
     using value_type = ValueType;
@@ -210,6 +216,9 @@ using LowerIsai = Isai<isai_type::lower, ValueType, IndexType>;
 template <typename ValueType = default_precision, typename IndexType = int32>
 using UpperIsai = Isai<isai_type::upper, ValueType, IndexType>;
 
+template <typename ValueType = default_precision, typename IndexType = int32>
+using GeneralIsai = Isai<isai_type::general, ValueType, IndexType>;
+
 
 }  // namespace preconditioner
 }  // namespace gko
diff --git a/matrices/CMakeLists.txt b/matrices/CMakeLists.txt
index 0281555c01f..987bcd8610e 100644
--- a/matrices/CMakeLists.txt
+++ b/matrices/CMakeLists.txt
@@ -1,6 +1,8 @@
 configure_file("${Ginkgo_SOURCE_DIR}/matrices/config.hpp.in"
                "${Ginkgo_BINARY_DIR}/matrices/config.hpp" @ONLY)
 
+configure_file("test/1138_bus.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/1138_bus.mtx")
+configure_file("test/1138_bus_isai.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/1138_bus_isai.mtx")
 configure_file("test/ani1.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/ani1.mtx")
 configure_file("test/ani4.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/ani4.mtx")
 configure_file("test/isai_l.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_l.mtx")
@@ -14,3 +16,5 @@ configure_file("test/isai_u_excess_rhs.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/
 configure_file("test/isai_u_inv.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_u_inv.mtx")
 configure_file("test/isai_u_inv_partial.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_u_inv_partial.mtx")
 configure_file("test/1138_bus.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/1138_bus.mtx")
+configure_file("test/1138_bus_isai.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/1138_bus_isai.mtx")
+configure_file("test/a_large.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/a_large_inv.mtx")
diff --git a/matrices/test/1138_bus_isai.mtx b/matrices/test/1138_bus_isai.mtx
new file mode 100644
index 00000000000..149b760de47
--- /dev/null
+++ b/matrices/test/1138_bus_isai.mtx
@@ -0,0 +1,4057 @@
+%%MatrixMarket matrix coordinate real general
+% Generated 02-Dec-2020
+1138 1138 4054
+1 1  0.000681248559728779
+5 1  0.0005808732298176177
+563 1  0.000212507925078226
+2 2  0.1646520939128359
+10 2  0.08829684779764413
+563 2  0.03430164097698579
+3 3  0.02368699304892136
+11 3  0.005428961236945246
+34 3  0.01654454994702224
+35 3  0.007524132779723459
+104 3  0.01465705708874505
+475 3  0.004162471172391967
+4 4  0.04969379938881761
+7 4  0.03633745842841155
+27 4  0.02184528402080151
+101 4  0.04351750798021442
+102 4  0.002200311597615059
+103 4  0.002024472965514462
+1 5  0.0004423161544732949
+5 5  0.09500354946491268
+9 5  0.06728586226654351
+6 6  0.01418255460250985
+7 6  0.008626258966422079
+37 6  0.00996281784281251
+98 6  0.003376716717528236
+103 6  0.001590414343092176
+4 7  0.0332912632013526
+6 7  0.003875002233369217
+7 7  0.04589386701745581
+37 7  0.003861797540635052
+101 7  0.02960334268887455
+102 7  0.001938285938040816
+103 7  0.001793458292838763
+8 8  0.04400823191772331
+26 8  0.03175128681408829
+35 8  0.001206291042366941
+724 8  0.02087036911031598
+5 9  0.06450037559155145
+9 9  0.1918465562161385
+10 9  0.03293478661597532
+104 9  0.01561182863185781
+2 10  0.09822255098000247
+9 10  0.03361856756880777
+10 10  0.2368581714352888
+104 10  0.01906906345546891
+3 11  0.007676642745194343
+11 11  0.04289566629121028
+12 11  0.03853988857270334
+38 11  0.01348474775485342
+566 11  0.005959754499732399
+11 12  0.04289566629121028
+12 12  0.8458398174495795
+13 13  0.1764161925612767
+34 13  0.02562879633506981
+104 13  0.05594095653293755
+14 14  0.1618331449176782
+413 14  0.1087319346343822
+15 15  0.0007333571923797343
+16 15  0.0006002108600772536
+17 15  0.0006001434823037491
+18 15  0.0006001101106035339
+19 15  0.0006001101106035339
+411 15  0.002421932178618019
+15 16  0.0007333571923797343
+16 16  0.03320020727407764
+15 17  0.0007333571923797342
+17 17  0.03340014873030459
+15 18  0.0007333571923797347
+18 18  0.0335001066232039
+15 19  0.0007333571923797346
+19 19  0.0335001066232039
+20 20  0.02254269487639738
+21 20  0.03141327078401259
+37 20  6.343154797014689e-05
+102 20  5.15051584496288e-05
+20 21  0.02246119165317312
+21 21  0.03158827800283255
+22 21  0.0001745256626092879
+23 21  0.0001745250869321762
+24 21  0.0001745249232151727
+25 21  0.0001745243528732199
+21 22  0.03158827800283255
+22 22  0.1922745283135893
+21 23  0.03158827800283255
+23 23  0.1929745247784521
+21 24  0.03158827800283255
+24 24  0.1931745305202153
+21 25  0.03158827800283256
+25 25  0.1938745473256959
+8 26  0.03777862672178795
+26 26  0.2560217339241357
+35 26  0.005523321771863168
+4 27  0.01520623868387142
+27 27  0.3629987113101796
+28 27  0.1481510233754166
+29 27  0.1496009394939
+30 27  0.1260534920598905
+101 27  0.02197002425399606
+27 28  0.3629987113101797
+28 28  0.5694511133229858
+27 29  0.3629987113101797
+29 29  0.5596010050939105
+27 30  0.3629987113101796
+30 30  0.9661538336446894
+31 31  0.7124824145930962
+32 31  0.2802711061386238
+100 31  0.4701005157283245
+31 32  0.2802711061386238
+32 32  0.7124824145930961
+100 32  0.4701005157283243
+33 33  1.54424888493338
+100 33  0.4701009138396892
+3 34  0.01509919387408505
+13 34  0.01378015416852058
+34 34  0.03338970933666868
+104 34  0.0221443077423441
+553 34  0.004106497666312521
+3 35  3.921975939515009e-05
+8 35  4.519451563167247e-06
+26 35  1.739198459517821e-05
+35 35  0.03170024838405466
+104 35  3.103370868239583e-05
+710 35  0.03273816950425548
+36 36  0.04150786749482464
+711 36  0.05122840648749439
+6 37  0.01026128366544737
+7 37  0.008842239420246943
+20 37  0.00516101332813253
+37 37  0.01974040638964612
+102 37  0.0002585417122540082
+11 38  0.004243390622179626
+38 38  0.04941690164381931
+39 38  0.01612111598013823
+98 38  0.02238184966731691
+412 38  0.0007044908029997024
+38 39  0.04717912607627697
+39 39  0.07713052564334147
+99 39  0.01578007328107746
+40 40  0.005143151806086547
+41 40  0.0006196286733581442
+43 40  0.0001249975866733072
+45 40  0.002842486466533443
+40 41  0.0006583388216107648
+41 41  0.00120386063722436
+42 41  0.001041758265432484
+44 41  0.003642069331333222
+41 42  0.001203860637224361
+42 42  0.03554175498793279
+40 43  6.453490793065965e-05
+43 43  0.000556413788835265
+146 43  0.0004242938958185286
+41 44  9.260468919143784e-05
+44 44  0.004321975814517058
+486 44  0.001075993616377296
+40 45  0.002776391303475186
+45 45  0.01090537626521374
+49 45  0.001916483833115058
+46 46  0.0001981994155170788
+48 46  0.004386756576850353
+47 47  0.009579804180749558
+48 47  0.004386756576850353
+46 48  9.819941551707885e-05
+47 48  4.980469536953095e-05
+48 48  0.004386756576850353
+54 48  7.58791172932267e-06
+506 48  9.554351794912995e-05
+45 49  0.00206377128625444
+49 49  0.004164667647750025
+50 49  0.003979634966680848
+53 49  0.0008210134931751628
+49 50  0.0005330899213154762
+50 50  0.009078169026546061
+51 50  0.001041758265432484
+52 50  0.005032316597876657
+50 51  0.009078169026546059
+51 51  0.03554175498793279
+50 52  0.00865262990209776
+52 52  0.005971756341896436
+125 52  0.0002005294381465794
+49 53  0.0002823367797531123
+53 53  0.002204759276128122
+54 53  0.001758725464323372
+55 53  0.0006510001315020267
+48 54  0.0001757527218732435
+53 54  0.001608911051569371
+54 54  0.001952656877995909
+425 54  0.0001782023010670741
+445 54  0.0004330963840954397
+447 54  0.0001686303552894172
+53 55  0.002204759276128122
+55 55  0.02065100013150203
+56 56  0.03419970248216579
+63 56  0.02475599872252582
+57 57  0.01193358366230999
+66 57  0.005058753420015482
+58 58  0.06866513048422875
+70 58  0.02734989316262069
+59 59  0.04808928304500903
+67 59  0.008619277873156228
+60 60  0.01500170873047119
+68 60  0.007282230447790528
+61 61  0.01231214403251011
+130 61  0.005211665741062992
+62 62  0.02721688832821186
+129 62  0.009437459104730673
+56 63  9.969876526538767e-05
+63 63  0.02475599872252582
+64 63  2.724952471630168e-05
+65 63  0.01357137341224411
+63 64  0.004682953000407078
+64 64  0.009073295444798643
+71 64  0.005165563902222893
+131 64  0.0005524194669433561
+226 64  0.004689890351087489
+63 65  0.02471664981601872
+65 65  0.01365217736954061
+1095 65  3.883843633452087e-05
+57 66  0.003033587578308271
+66 66  0.005058753420015482
+67 66  0.001952437716718268
+68 66  0.0008766812759457492
+74 66  0.0004073635549340625
+183 66  0.001647334407114804
+226 66  0.001121354719249025
+59 67  0.006989282634009012
+66 67  0.003087639593553936
+67 67  0.008619277873156226
+68 67  0.001842789313165553
+69 67  0.001730381415443986
+60 68  0.005401711802470204
+66 68  0.0008395044991809331
+67 68  0.001077745041146966
+68 68  0.007282230447790526
+74 68  0.0008387559963791313
+90 68  0.001876116163389953
+201 68  0.001384717904813421
+67 69  0.0006919705072952755
+69 69  0.01891597685855978
+70 69  0.02515420225999527
+58 70  0.004765121218727402
+69 70  0.01471609394135156
+70 70  0.02734989316262069
+182 70  0.0006001253916848435
+196 70  0.00453850096372118
+1063 70  0.001442528314324188
+1065 70  0.002690675432880693
+1074 70  0.0005121697815175987
+64 71  0.003851870716564107
+71 71  0.02012983346115188
+72 71  0.01270007153835384
+71 72  0.01316905019369886
+72 72  0.02206897680904131
+73 72  0.01259999910540006
+121 72  0.003019727780727348
+72 73  0.02206897680904131
+73 73  0.1926000171054019
+66 74  0.0005329625712261697
+68 74  0.000974800732172311
+74 74  0.004890705457093017
+75 74  0.00242993093417575
+76 74  0.0007676610503459754
+78 74  0.0005574058088179555
+80 74  9.69225330900054e-05
+119 74  0.004088019008035104
+74 75  0.0002845256030805037
+75 75  0.02319509933140409
+204 75  0.02085818934229955
+810 75  0.0005258285665422862
+918 75  3.934955365255447e-05
+74 76  0.001037701798146781
+76 76  0.00966764264537184
+77 76  0.0006119916750053705
+78 76  0.00138301136005799
+127 76  0.007161720479276878
+76 77  5.771726981147143e-05
+77 77  0.003003800094158384
+1050 77  0.007289380632363749
+74 78  0.000783918194840543
+76 78  0.001438871260933114
+78 78  0.005215784653489311
+79 78  0.001970130754865572
+452 78  0.002641515655392799
+78 79  0.0008400855043495494
+79 79  0.005692436756110305
+416 79  0.0003493865114599682
+417 79  0.0003493865114599682
+454 79  0.005345796734007223
+74 80  0.000117754927592846
+80 80  0.002357720452614393
+81 80  0.0004214550496419059
+447 80  0.00119460968269267
+452 80  0.00102476250511989
+80 81  0.0003767445408468322
+81 81  0.004126795740775822
+448 81  0.003560755475402297
+450 81  0.002413355897637983
+464 81  0.00198538023394493
+82 82  0.2031781421616329
+94 82  0.005877230870353861
+83 83  0.06023791606499645
+94 83  0.005877230870353861
+84 84  0.07384992176116395
+95 84  0.02345972240229082
+85 85  0.003511431738654743
+87 85  0.000274155628548272
+126 85  0.002413922750527005
+86 86  0.0009997761495309598
+87 86  0.000512804588313572
+241 86  4.153351929177897e-05
+251 86  7.990980763886454e-05
+253 86  5.864093787230469e-05
+267 86  9.167342355770771e-05
+269 86  0.0001263536623349897
+283 86  9.917851390286249e-05
+291 86  1.362704561258246e-05
+293 86  3.881479722896776e-05
+294 86  3.881479722896776e-05
+302 86  0.0001480064718381182
+315 86  3.384921395448833e-05
+85 87  0.000113811913945984
+86 87  0.0009397325049656807
+87 87  0.0006981830367452577
+88 87  0.0001933333555022248
+87 88  0.0006981830367452578
+88 88  0.006993334443502398
+89 89  0.01911121455439772
+105 89  0.001572078869026403
+115 89  0.006410797694367932
+68 90  0.003632263288868323
+90 90  0.01678437110764967
+91 90  0.002271116795289418
+90 91  0.0008454971866149284
+91 91  0.004531217722525695
+92 91  0.001252506965202451
+108 91  0.000117130168114432
+115 91  0.0005814566817775351
+116 91  0.002542576566909739
+91 92  0.002864311958139353
+92 92  0.004936936923557877
+93 92  0.00102670435528675
+92 93  0.0007680432699654916
+93 93  0.002790932198967058
+94 93  0.0002705038701784743
+252 93  0.001831311572033205
+264 93  0.0001725932363595291
+271 93  0.001830903937413845
+288 93  0.0009171516086301031
+292 93  7.740390116339735e-05
+293 93  0.0002784684200391615
+294 93  0.0002784684200391615
+322 93  0.0008189522813544738
+323 93  0.0008958687596585449
+324 93  0.001018921685938732
+325 93  0.0005412463670111131
+326 93  0.0004396871639683349
+82 94  0.002178126885631672
+83 94  0.002237918384996351
+93 94  0.0004197548455136497
+94 94  0.005877230870353861
+96 94  0.0006014854638730493
+264 94  0.002981655476056899
+318 94  0.001726861983328051
+325 94  0.001843822482836495
+84 95  0.01694991385206285
+95 95  0.02345972240229082
+97 95  0.01000194443151547
+262 95  0.001081741513370692
+268 95  0.001309286249033044
+300 95  0.01224182050238317
+94 96  2.815075665825954e-06
+96 96  0.05814496801316306
+145 96  3.486767270367365e-05
+704 96  5.343053643953835e-05
+705 96  0.04366812227074233
+95 97  0.01657416089691347
+97 97  0.09017195729895189
+268 97  0.003511584060424524
+6 98  0.002699058108612025
+38 98  0.04001245422074155
+98 98  0.03625435372215891
+39 99  0.01829630006968984
+99 99  0.3484725119868282
+413 99  0.08293939894190504
+31 100  0.02589698982759715
+32 100  0.02589698982759714
+33 100  0.02494899047914339
+100 100  0.4701009138396891
+725 100  0.1297783102504183
+732 100  0.02606143467207535
+4 101  0.04351750798021441
+7 101  0.03146673606395294
+27 101  0.04625854089554216
+101 101  0.06795839341338922
+102 101  0.002357305413903345
+103 101  0.001981258340451572
+4 102  0.00226806858354301
+7 102  0.002260114569845941
+20 102  0.00656515156698897
+37 102  0.0003134712708337844
+101 102  0.002478689241000047
+102 102  0.02143503851955079
+103 102  0.002916762025562342
+4 103  3.516029982941045e-05
+6 103  5.394683124865673e-05
+7 103  6.135624943939899e-05
+101 103  3.660089136278378e-05
+102 103  6.599489002472127e-05
+103 103  0.004353068895739493
+478 103  0.006988090195758781
+3 104  0.01108975206631132
+9 104  0.0130220612456521
+10 104  0.01558227882171362
+13 104  0.04433392865653338
+34 104  0.02291223342746111
+35 104  0.00297114489493154
+104 104  0.06777084336288788
+89 105  0.001740249051467288
+105 105  0.002681032103705445
+106 105  0.002446875219224727
+109 105  0.00152502346974839
+105 106  0.002681032103705445
+106 106  0.0852469073456372
+107 107  0.0321238999874595
+108 107  0.001118221622047723
+91 108  0.0003034887245227763
+107 108  0.00102389917885948
+108 108  0.001118221622047723
+110 108  0.005977812477686725
+105 109  9.575115764068471e-05
+109 109  0.001732513235867602
+1029 109  0.001800437082327916
+108 110  9.262729407914173e-05
+110 110  0.006633544499293281
+111 110  7.026746840454583e-05
+328 110  0.001025861980529139
+367 110  7.974311888092716e-06
+370 110  3.42578322976917e-05
+110 111  2.776996845054393e-05
+111 111  0.01810571453131894
+367 111  1.172896986395923e-05
+368 111  1.33465520430845e-05
+701 111  0.0175592625109741
+112 112  0.01945153812352913
+113 112  0.0001184342802206041
+370 112  0.0008214897265262109
+686 112  0.03030190795848136
+112 113  1.437578878937231e-05
+113 113  0.004816208745748132
+114 113  7.561234209610165e-06
+115 113  3.927836046327332e-05
+690 113  0.01345565713064452
+113 114  3.669489994361314e-06
+114 114  0.009956712162072143
+1009 114  0.04910283858341127
+89 115  0.006736718553298537
+91 115  0.001430179841959822
+113 115  0.001598359618194295
+115 115  0.01549889922448875
+91 116  0.00371063263793522
+116 116  0.004592291404769712
+117 116  0.002351999823129614
+126 116  0.0006199724630645687
+116 117  0.004592291404769712
+117 117  0.09365202082213446
+118 118  0.1855881563310711
+119 118  0.008339490136900786
+74 119  0.004041394315700626
+118 119  0.004488151278380998
+119 119  0.008339490136900786
+123 119  0.001620151684609045
+120 120  0.01476311046052306
+121 120  0.0107703647853945
+72 121  0.001929758935051333
+120 121  0.006863111803522836
+121 121  0.01077036478539451
+122 121  0.002548602877220247
+504 121  0.001250671036248352
+121 122  0.004999626318088899
+122 122  0.009791044169788148
+123 122  0.002874053735863524
+124 122  0.003514192229568888
+119 123  0.002542899633968689
+122 123  0.003541517835239608
+123 123  0.01083896068944554
+125 123  0.001426130766622482
+122 124  0.004136408596181764
+124 124  0.01298484058332833
+125 124  0.001429305675588127
+127 124  0.004625210265810006
+52 125  0.0006736960274828053
+123 125  0.001113280266628655
+124 125  0.001168064733086433
+125 125  0.004277960792596333
+126 125  0.001246570300217467
+129 125  0.002200956663281992
+546 125  0.002101600612698724
+85 126  0.00195816897631273
+116 126  0.0006532948685811201
+125 126  0.000985780856494245
+126 126  0.003974648534154331
+129 126  0.001600414958890015
+76 127  0.006324626157357197
+124 127  0.003640608709090833
+127 127  0.01900076624522866
+128 127  0.01259999910540006
+127 128  0.01900076624522866
+128 128  0.1926000171054019
+62 129  0.007516887816011841
+125 129  0.001983893339846202
+126 129  0.001824214172754806
+129 129  0.009437459104730673
+61 130  0.004312144032510114
+130 130  0.005211665741062992
+133 130  0.001981334031159003
+173 130  0.001605342590001883
+213 130  0.001180718901054115
+64 131  0.0001624261208669008
+131 131  0.005496545062188923
+132 131  0.0009318855979024285
+133 131  0.004428602031975427
+136 131  0.0004868534760858688
+131 132  0.004091401135323778
+132 132  0.01111310085665062
+144 132  0.0026122901933604
+742 132  0.005001234744832591
+130 133  0.0007770379028807413
+131 133  0.004601876635986089
+133 133  0.005162154580345246
+134 133  0.001103488348470674
+133 134  0.005162154580345246
+134 134  0.08070351413887902
+135 135  0.007083093093560105
+136 135  3.061241490243482e-05
+139 135  1.251456639358159e-05
+740 135  0.01220595106371744
+131 136  0.001991894285428722
+135 136  0.001805455974750392
+136 136  0.01206700091963003
+141 136  0.005533561397417803
+137 137  0.02542334705796045
+139 137  0.01773192434110749
+141 137  0.003666823554414284
+761 137  0.008300342531490581
+138 138  0.009554387072796334
+141 138  0.00192335291309995
+877 138  0.004200650278359249
+882 138  0.006429359699931656
+135 139  0.0003734270498003733
+137 139  0.02045478170890633
+139 139  0.02643524506790336
+140 139  0.004906745044645386
+139 140  0.007462880033783114
+140 140  0.03438206097580358
+829 140  0.0241085653279941
+830 140  0.01931602694553701
+136 141  0.004873967154048279
+137 141  0.007363871512563584
+138 141  0.002927858761459856
+141 141  0.0144587676069406
+142 142  0.03396012133167858
+258 142  0.006958215837690046
+366 142  0.02123498536303899
+143 143  0.05253409842956076
+144 143  3.737609144862155e-05
+743 143  3.273053455976305e-06
+745 143  9.981171885921397e-05
+826 143  9.985431255797791e-05
+132 144  4.891684363463288e-06
+143 144  6.555676654472382e-05
+144 144  0.05984537169173331
+827 144  0.05913660555884374
+96 145  4.201106087792323e-05
+145 145  0.04826252211210828
+318 145  1.067292711318153e-05
+703 145  0.04543389368468301
+43 146  0.0003318959174329431
+146 146  0.0004296855130198788
+147 146  0.0002299996113007038
+146 147  9.765578935825544e-05
+147 147  0.0005699989687018715
+148 147  0.0003399995920004894
+147 148  0.0005699989687018715
+148 148  0.0004399995920004894
+149 149  0.1950750307588969
+166 149  0.004690290407168253
+150 150  0.02113688073170932
+173 150  0.01105797213481282
+151 151  0.09999112334144708
+175 151  0.04515284729470483
+152 152  0.1513934392593726
+180 152  0.1377601741913947
+153 153  0.03175075850354254
+183 153  0.004694917869209315
+154 154  0.07405283545133394
+207 154  0.01947995046815832
+155 155  0.03535344476653352
+198 155  0.0120529435161583
+156 156  0.01418318500048101
+213 156  0.009776534223750005
+157 157  0.03488709778969355
+219 157  0.07970002502433311
+158 158  0.0429710619638005
+226 158  0.01033652162936393
+159 159  0.1300474347284174
+182 159  0.02582330109764804
+205 159  0.04327252845072649
+160 160  0.05632744717310522
+172 160  0.009265365306001163
+233 160  0.01902225262340747
+161 161  0.09954531215457496
+222 161  0.02117189618271392
+743 161  0.01855455525395318
+162 162  0.01773475729671951
+163 162  0.006188773869795133
+169 162  0.005918776339923625
+209 162  0.004652548248564229
+223 162  0.007001645490851817
+232 162  0.007743569014220319
+162 163  0.005600105892468669
+163 163  0.01175250816271155
+186 163  0.003466894397092407
+201 163  0.0007462290367470655
+227 163  0.005113251910474695
+164 164  0.1950750307588969
+166 164  0.004690290407168253
+165 165  0.01263086671907289
+166 165  0.002415242397962376
+167 165  0.007188095238868617
+175 165  0.008221118227580022
+194 165  0.00420253278866024
+149 166  0.002975028107916812
+164 166  0.002975028107916812
+165 166  0.002242673048908967
+166 166  0.004690290407168252
+192 166  0.001758970263234734
+201 166  0.0006130890945648758
+218 166  0.001794184059995515
+165 167  0.009103303946035184
+167 167  0.03958993596969194
+175 167  0.02668698276114242
+168 168  0.06188391972552811
+180 168  0.08632167453688958
+209 168  0.05182362792372523
+162 169  0.007978392672829883
+169 169  0.04490731321160264
+187 169  0.009683898410151553
+170 170  0.05081137663511889
+188 170  0.02772165088265999
+194 170  0.004291213627925625
+171 171  0.02889329972402089
+178 171  0.01396633877726771
+217 171  0.005344323434275283
+160 172  0.008434040716645134
+172 172  0.01978366406952758
+173 172  0.007038639571083761
+222 172  0.003195671592411879
+755 172  0.01741054636210458
+130 173  0.001309510177350962
+150 173  0.007536880405309312
+172 173  0.004082521658941286
+173 173  0.01105797213481282
+184 173  0.001195525828115229
+174 174  0.02562829565967372
+211 174  0.005163235251720969
+216 174  0.01107325596818595
+151 175  0.02479111010624477
+165 175  0.005575741172997473
+167 175  0.01429176827631403
+175 175  0.04515284729470483
+179 175  0.01750756250668328
+176 176  0.008984621483198505
+177 176  0.008177446769356011
+178 176  0.000450560209875999
+207 176  0.01893741669949776
+176 177  0.00854422857710486
+177 177  0.01156068280854432
+207 177  0.01914175455590986
+171 178  0.01289567718151429
+176 178  0.002137805667154547
+178 178  0.0184856475961986
+214 178  0.002843242276535233
+217 178  0.004637007287978548
+175 179  0.02257642364735242
+179 179  0.0794408057444532
+203 179  0.05227064700289084
+152 180  0.05529346453366597
+168 180  0.03547210065369603
+180 180  0.1377601741913947
+209 180  0.03375560917442493
+181 181  0.04586758406187812
+183 181  0.004694917869209315
+70 182  0.002203629418820364
+159 182  0.01906576840335792
+182 182  0.02731018070796857
+183 182  0.002285523332612573
+205 182  0.02291377111241266
+221 182  0.01231422127431991
+66 183  0.00198936893203292
+153 183  0.002750759663542482
+181 183  0.002667587517877839
+182 183  0.002054350932803569
+183 183  0.004694917869209314
+184 183  0.001483132404887844
+201 183  0.0007039906531452232
+226 183  0.001362122911699713
+173 184  0.002205101854106055
+183 184  0.002241791644646623
+184 184  0.006434427254287113
+193 184  0.001031844286264287
+213 184  0.001051750772216008
+185 185  0.1340543473292866
+187 185  0.01760303090452317
+163 186  0.004331406867578962
+186 186  0.01087798201420141
+187 186  0.00581183269954516
+218 186  0.000982312064760316
+169 187  0.005890678929749005
+185 187  0.01005434534528653
+186 187  0.004215331338600365
+187 187  0.01760303090452318
+188 187  0.01017180807117036
+206 187  0.004624127568199378
+209 187  0.002703002287660673
+170 188  0.02206627924713095
+187 188  0.009958416228474147
+188 188  0.03685814527478884
+189 189  0.02954103618486679
+211 189  0.005664173249041018
+216 189  0.01071149211435926
+190 190  0.03159357819327018
+199 190  0.005311312024586768
+191 191  0.009445785482787799
+195 191  0.0004502057870270782
+197 191  0.0005086844560418416
+198 191  0.01164907246328365
+211 191  0.0003693306761868612
+166 192  0.0006075150998615923
+192 192  0.004524666526593245
+199 192  0.004293832269904215
+218 192  0.000424290709225597
+220 192  0.0003463798370771271
+184 193  0.001078657395417853
+193 193  0.004787816875317665
+198 193  0.002957330836441481
+199 193  0.002287519410832948
+208 193  0.001435616760642699
+165 194  0.004568275972415812
+170 194  0.004828852161815346
+194 194  0.01731147984315577
+216 194  0.008041774074406987
+191 195  0.004848045337789377
+195 195  0.03498427278244425
+211 195  0.01464965714420834
+70 196  0.02734989316262069
+196 196  0.1979384941173614
+191 197  0.00381815547328514
+197 197  0.03964947646544643
+212 197  0.02429704973565863
+155 198  0.0008534480440332083
+191 198  0.008740976039508384
+193 198  0.0002416673538872769
+198 198  0.0120529435161583
+190 199  0.001293577890270186
+192 199  0.003742240897167284
+193 199  0.0006318968140394233
+199 199  0.005311312024586768
+200 200  0.02748486987715647
+201 200  0.005820445495695883
+202 200  0.01773913959853929
+68 201  0.002118960243498572
+163 201  0.0009296743429296432
+166 201  0.0005753632929967714
+183 201  0.0009555583818203816
+200 201  0.006982651521205821
+201 201  0.007971729938329305
+218 201  0.0004355378658362216
+200 202  0.02128938213263513
+202 202  0.06573358102509876
+203 202  0.02356510680135571
+179 203  0.05677404204502257
+202 203  0.01875568339317256
+203 203  0.1045412940057817
+75 204  0.02319509933140409
+204 204  0.02240818931904955
+159 205  0.03333421775909028
+182 205  0.02439128258971602
+205 205  0.06288011275318212
+221 205  0.02412258480011461
+187 206  0.008101277506822279
+206 206  0.05717854079752385
+216 206  0.007989818687928404
+154 207  0.001152829473533445
+176 207  0.007815294389350934
+177 207  0.007560512175752126
+207 207  0.01947995046815831
+208 207  0.0008795896148575726
+224 207  0.000841955811051376
+193 208  0.001502930945889371
+207 208  0.01336505385036552
+208 208  0.009184331049542322
+162 209  0.001617378304335601
+168 209  0.04940069981255372
+180 209  0.07830404315129502
+187 209  0.001145957563087521
+209 209  0.05463987152426567
+210 210  0.006249332648497629
+215 210  0.0005600000143360003
+217 210  0.006930191938837317
+218 210  0.0002160026615936735
+174 211  0.00387420779295125
+189 211  0.004015027831902545
+191 211  0.002965586123163986
+195 211  0.01092363099366728
+211 211  0.02049648264457055
+212 211  0.01011154365328455
+197 212  0.02020517110238134
+211 212  0.01005157956517059
+212 212  0.04078177123942444
+130 213  0.0007959490277139882
+156 213  0.007283183275480579
+184 213  0.0004712384717928327
+213 213  0.009776534223750006
+504 213  0.001738948328903979
+178 214  0.004517214619620368
+214 214  0.01622365692881036
+217 214  0.006823204022623214
+210 215  0.00624933264849763
+215 215  0.08976001856793986
+174 216  0.006553651674182944
+189 216  0.005988927043812819
+194 216  0.005015865201539273
+206 216  0.003936888199328748
+216 216  0.01480200462145097
+218 216  0.0005964987786257558
+171 217  0.0003116442438405972
+178 217  0.0003729658367616034
+210 217  0.005974326108660732
+214 217  0.0004934479622916903
+217 217  0.007471819595453229
+166 218  0.001986996839654225
+186 218  0.0009447898958461862
+192 218  0.001359040340707622
+201 218  0.0005166651827864568
+210 218  0.001699405350262223
+216 218  0.0009162982296245775
+218 218  0.003260381416208405
+157 219  0.008887098829693506
+219 219  0.07970002502433311
+220 219  0.0111064592940345
+230 219  0.008021278205120287
+192 220  0.0006585461906468347
+219 220  0.06810000565232581
+220 220  0.01945203998592646
+182 221  0.01651721084133032
+205 221  0.02969407488294984
+221 221  0.06867077576023006
+222 221  0.01491787694894059
+225 221  0.01123305742296696
+161 222  0.005446791021970473
+172 222  0.00105734554889798
+221 222  0.00637057849548162
+222 222  0.04383877564319283
+225 222  0.01396029352073445
+233 222  0.01942492709578699
+743 222  0.001879645362456986
+162 223  0.009537252323750592
+223 223  0.03415770639099423
+232 223  0.02758515654850143
+207 224  0.01192367197024488
+224 224  0.06469263445835478
+1026 224  0.03321290269845339
+221 225  0.01386785543446704
+222 225  0.04035840791539649
+225 225  0.04714710373548588
+64 226  0.00397169308634345
+66 226  0.001311688119627266
+158 226  0.007271056251799585
+183 226  0.001319381244459979
+226 226  0.01033652162936393
+163 227  0.005437727636921652
+227 227  0.01880472598998021
+228 227  0.0153691797619795
+227 228  0.008894606690303017
+228 228  0.02860375113104207
+229 228  0.01432604004185496
+231 228  0.006460191377533867
+232 228  0.006731179414366265
+228 229  0.02860375113104208
+229 229  0.05052603924545497
+219 230  0.07970002502433313
+230 230  0.04602127744512029
+228 231  0.01614024570043577
+231 231  0.04065984863525216
+232 231  0.02258201670108759
+162 232  0.007076967383965091
+223 232  0.01850795004374896
+228 232  0.007523901756069395
+231 232  0.01010298983422199
+232 232  0.03354013420975288
+160 233  0.01051584785838124
+222 233  0.03565445488843226
+233 233  0.03577858589187031
+234 234  0.03955722421188783
+236 234  0.006862052954427557
+307 234  0.007259879494695027
+235 235  0.003140124125252748
+236 235  0.0004389713839516723
+243 235  0.0004756282247415277
+270 235  0.004992824107355747
+272 235  0.0001122322297658058
+298 235  0.0001990208158249584
+299 235  0.0001486121962474559
+234 236  0.006381155396333391
+235 236  0.002351339758579689
+236 236  0.01375420155716404
+286 236  0.01217676770136721
+237 237  0.03222175536937757
+261 237  0.0003348049688261235
+287 237  0.009571047657404731
+309 237  0.01516760558629039
+366 237  0.001668651248794341
+702 237  0.006099666530470593
+238 238  0.006725363712814524
+239 238  0.005075621570773962
+270 238  0.005223798051246839
+238 239  0.00480148246229583
+239 239  0.02149615133522103
+270 239  0.005005489406137049
+281 239  0.01767034896712746
+240 240  0.2035999856933056
+259 240  0.1456999329463718
+86 241  0.0001067215740544099
+241 241  0.001374005697288065
+242 241  0.000290634802869228
+246 241  0.0003293421132907725
+253 241  0.0002236550707773679
+257 241  0.00018363870715457
+266 241  0.0003256955939914388
+275 241  0.0001784559916947189
+279 241  0.000277406849575917
+289 241  3.90223036584587e-05
+291 241  0.0004288607304773129
+292 241  0.0004293326002310656
+293 241  0.0002146019610854077
+294 241  0.0002146019610854077
+299 241  0.0001125580564708786
+310 241  0.0002296081873569976
+321 241  0.0001848292649575446
+327 241  0.000217208135447652
+241 242  0.000808297801446638
+242 242  0.004532535764806162
+244 242  0.003313942588638787
+245 242  0.003202739628222241
+235 243  0.003140124125252748
+243 243  0.06897562890974154
+242 244  0.002185807861869108
+244 244  0.01825603325724795
+272 244  0.01335998716340609
+242 245  0.003423511021214106
+245 245  0.01391310681875108
+298 245  0.008020761167864319
+241 246  0.001264940490760748
+246 246  0.009919386287566598
+310 246  0.004009935755031833
+247 247  0.005023952688409826
+248 247  0.003966809978134316
+255 247  0.005010061754098129
+247 248  0.003966809978134316
+248 248  0.005387345736558113
+255 248  0.00501006175409813
+249 249  0.007500025410094483
+282 249  0.009274861044283244
+250 250  0.001037632973403639
+257 250  0.0004719634104043619
+263 250  0.0005671173776828162
+289 250  0.0003975775806438073
+317 250  0.002139645988580356
+86 251  0.0006808741046427244
+251 251  0.005700731409424105
+267 251  0.00160832709644289
+93 252  0.001274125410371841
+252 252  0.008520372883448564
+310 252  0.001354434476283895
+313 252  0.006532344249316064
+86 253  0.0004431746214548092
+241 253  0.0008720044259764096
+253 253  0.005209857898399626
+254 254  0.004570573683192871
+255 254  0.002719747585571147
+285 254  0.002529730361499031
+247 255  0.002738238376246548
+248 255  0.002738238376246548
+254 255  0.001046081849369832
+255 255  0.005010061754098129
+258 255  0.0004093827040350337
+260 255  0.0004328004023325395
+263 255  0.0004754744011226133
+256 256  0.001232406118703863
+290 256  9.920001162465413e-05
+317 256  0.002847350536171632
+321 256  4.800531971530007e-05
+241 257  0.0006870027744477309
+250 257  0.0005188164306696431
+257 257  0.005727800591376988
+142 258  0.002795122930221372
+255 258  0.0006016801592703452
+258 258  0.01185005493123235
+261 258  0.007180367412454548
+327 258  0.002163824479980224
+240 259  0.0772999797319453
+259 259  0.1456999329463718
+278 259  0.03649842322627914
+255 260  0.001887197908379108
+260 260  0.005476034471244295
+289 260  0.002339197128309662
+237 261  0.0006931321003947503
+258 261  0.0115951437562709
+261 261  0.01466532263097838
+95 262  0.001482503382548444
+262 262  0.007602063438084359
+268 262  0.001251321377377948
+276 262  0.005841372405788837
+314 262  0.00554978090398742
+250 263  0.0006179533834917097
+255 263  0.002026362612105835
+263 263  0.008093427000217779
+93 264  0.000354136026332319
+94 264  0.003676606037609885
+264 264  0.005771035521286595
+311 264  0.003436992392496144
+265 265  0.01353634804014666
+276 265  0.01681685927716855
+300 265  0.0004775947571177971
+241 266  0.0006870028486440326
+266 266  0.007618978768022231
+307 266  0.007244028793220687
+86 267  0.0005156871929242583
+251 267  0.001412234574967097
+267 267  0.002393931278004583
+282 267  0.002374827321004211
+284 267  0.0007365249787451072
+302 267  0.0003004103132447092
+95 268  0.001702030013936146
+97 268  0.002631196757562732
+262 268  0.001303432303117434
+268 268  0.008268326912744399
+270 268  0.001795483912932649
+301 268  0.005225882324093172
+86 269  0.0005713007156371889
+269 269  0.003096547135627085
+284 269  0.00230476431274994
+235 270  0.002710273407661008
+238 270  0.000458126294856638
+239 270  0.0004640441205187209
+268 270  0.0001922668138947943
+270 270  0.005285106527012406
+275 270  0.0002113384082833111
+279 270  8.993314613729047e-05
+280 270  0.0004370867227569277
+318 270  0.0001073915958531392
+320 270  0.0004346249362171608
+93 271  0.002790932198967059
+271 271  0.01698090408891385
+235 272  0.000409799999122878
+244 272  0.01587353887113617
+272 272  0.02580387309143066
+273 273  0.001349989596112143
+274 273  0.000503999894361622
+303 273  0.0008608688510705149
+315 273  0.0001976934209945761
+273 274  0.0006749947980560714
+274 274  0.002153999910861622
+312 274  0.0006749947980560717
+241 275  0.0007045491408197031
+270 275  0.002575061532223329
+275 275  0.03834457886044484
+262 276  0.0004471802022402564
+265 276  0.01274009099906998
+276 276  0.01711241729020651
+277 277  0.003198786015875058
+285 277  9.459459818979985e-05
+289 277  0.003688662932945043
+259 278  0.06839997436369301
+278 278  0.05883674104228977
+309 278  0.01425623729688931
+241 279  0.001090380609736002
+270 279  0.001090962316495103
+279 279  0.02523641459730683
+270 280  0.005285106527012405
+280 280  0.03453709043965733
+239 281  0.02149615133522104
+281 281  0.3941704193726406
+249 282  0.006900025530094459
+267 282  0.0001915144758113623
+282 282  0.009274861044283244
+86 283  0.0006469139243108581
+283 283  0.004861132435203108
+292 283  0.0005251426696648901
+267 284  0.0006492017337374208
+269 284  0.002256805282478487
+284 284  0.005377784133941225
+254 285  0.003055907096497613
+277 285  0.001060062860034962
+285 285  0.005533784625049275
+236 286  0.01375420155716404
+286 286  0.1354767689343672
+237 287  0.03222175536937756
+287 287  0.06227105340170536
+93 288  0.001431018977059318
+288 288  0.00859043633331266
+291 288  0.0007238304331497447
+241 289  7.832280424198652e-06
+250 289  2.344813670225498e-05
+260 289  0.0001237458794768352
+277 289  0.00303598150753357
+289 289  0.003752813945477064
+256 290  0.001232406118703862
+290 290  0.01919920035542466
+86 291  0.0001460544564823494
+241 291  0.00121448759160345
+288 291  0.0005649416684895149
+291 291  0.001485508491030063
+93 292  9.446234339670815e-05
+241 292  0.001216875821722435
+283 292  0.0003913835229850364
+292 292  0.001487904392461122
+86 293  0.0003070354999625437
+93 293  0.0004563009241027801
+241 293  0.0008244542958433231
+293 293  0.004449376350181462
+86 294  0.0003070354999625437
+93 294  0.0004563009241027802
+241 294  0.0008244542958433231
+294 294  0.004449376350181462
+295 295  0.05362723832332691
+298 295  0.03278050112440601
+296 296  0.06730000928300155
+297 296  0.09566479307024195
+296 297  0.05740000918400153
+297 297  0.09566479307024194
+299 297  0.003600738758589758
+235 298  0.0005407355532442388
+245 298  0.004375546175952416
+295 298  0.02752723406902621
+298 298  0.032780501124406
+235 299  0.0009266725405849079
+241 299  0.0004189407119360687
+297 299  0.03826478388624038
+299 299  0.02447775247024812
+95 300  0.006497362701985984
+265 300  0.001943683324790106
+300 300  0.02765274863876077
+696 300  0.01942765355520232
+268 301  0.003633052017465452
+301 301  0.01226715775579435
+314 301  0.006721899241447137
+86 302  0.0009332426037138488
+267 302  0.0004080974533793484
+302 302  0.00141966384181147
+273 303  0.0006749946090576037
+303 303  0.001560868330027445
+312 303  0.0006749946090576038
+304 304  0.001321669685786581
+305 304  0.0005647556482983224
+315 304  0.0002403568633063155
+329 304  0.001557329223669231
+304 305  0.0008701094535217735
+305 305  0.002662907123752241
+315 305  0.0005312731733141023
+306 306  0.04073726346751776
+322 306  0.01559199226187695
+234 307  0.004188410290156808
+266 307  0.00681226259005297
+307 307  0.01448805758644137
+308 308  0.02743015306702811
+323 308  0.01574015380460722
+237 309  0.02663405607802391
+278 309  0.01020310695239171
+309 309  0.02687107284802318
+241 310  0.0008569213532082748
+246 310  0.003896441702978306
+252 310  0.002946901561498048
+310 310  0.01815241623544752
+264 311  0.003786423989447812
+311 311  0.01002270676069404
+326 311  0.005609450089855228
+274 312  0.0005039998943616222
+303 312  0.0008608688510705152
+312 312  0.001349989596112143
+315 312  0.0001976934209945761
+252 313  0.008520372883448564
+313 313  0.01393234617331656
+262 314  0.003768543249850233
+301 314  0.006186000684788312
+314 314  0.0119904185721572
+86 315  0.0001480011448277059
+273 315  0.0002220497300629913
+304 315  0.0002640679800789112
+305 315  0.0003788469527306226
+312 315  0.0002220497300629914
+315 315  0.0009217135543932941
+316 315  0.0006692163067055284
+319 315  0.0001202068991403766
+323 315  0.0002912602966097636
+315 316  0.0008255077159919701
+316 316  0.00350353108752257
+322 316  0.001627447357956611
+250 317  7.981794239769917e-05
+256 317  0.001137605998066949
+317 317  0.002885220298302715
+94 318  0.003830006121181885
+145 318  0.004197780816982526
+270 318  0.001069446635194767
+318 318  0.007241474523312593
+704 318  0.002910117509878809
+315 319  9.320699702466988e-05
+319 319  0.001624041366835758
+329 319  0.002125863158142157
+270 320  0.005285106527012405
+320 320  0.06153463281811816
+241 321  0.0007188891695774862
+256 321  0.000587603034797964
+321 321  0.006031830114480963
+93 322  0.0008338102842363536
+306 322  0.01433726494591769
+316 322  0.001223778329705182
+322 322  0.01559199226187695
+93 323  0.0006597623295049258
+308 323  0.01513015412482802
+315 323  0.0001954187319347493
+323 323  0.01574015380460722
+93 324  0.001854479797952634
+324 324  0.02267436172583527
+325 324  0.006249018262464586
+93 325  0.001046340009120363
+94 325  0.003810628139594725
+324 325  0.005462594994192058
+325 325  0.0111747181380052
+93 326  0.0005388617210688591
+311 326  0.008087563722523863
+326 326  0.01631169261893439
+241 327  0.0007219351393211412
+258 327  0.005623753984343173
+327 327  0.007025770805446348
+110 328  0.00663354449929328
+328 328  0.001125861980529139
+304 329  0.0005947513586039614
+319 329  0.0008932226713696184
+329 329  0.002365022787349113
+330 330  0.4508999260647044
+335 330  0.04019070351291964
+331 331  1.13959995889597
+336 331  0.02757474132279732
+332 332  0.6038571511990494
+337 332  0.08960947178327849
+333 333  0.4221833813416651
+350 333  0.05805941947495961
+334 334  0.7287011258206326
+350 334  0.0580594194749596
+330 335  0.03590000283969022
+335 335  0.04019070351291964
+337 335  0.08247489171225011
+331 336  0.004599999188468143
+336 336  0.02757474132279732
+337 336  0.0892477453094401
+332 337  0.003857271199025383
+335 337  0.004290700827265587
+336 337  0.02297474664702812
+337 337  0.08960947178327851
+338 337  0.001382415283490295
+339 337  0.001401553638641592
+340 337  0.001916275896135684
+337 338  0.0431214456624362
+338 338  0.06631833798880571
+340 338  0.03235529912919195
+337 339  0.04340098495573665
+339 339  0.06583550982543664
+340 339  0.03217226483513082
+337 340  0.03141018516658689
+338 340  0.0170770140952318
+339 340  0.01695572224445799
+340 340  0.0400235117674176
+341 340  0.01192433400923805
+342 340  0.009597517881589177
+340 341  0.0237955848811607
+341 341  0.06132809944511584
+342 341  0.02044571043904978
+340 342  0.01349025747156711
+341 342  0.01440128559122832
+342 342  0.02733470968649056
+343 342  0.006169401047183919
+344 342  0.00685149489186951
+345 342  0.002943366891871184
+370 342  0.007948261086098257
+342 343  0.01024323849138421
+343 343  0.07252885060496569
+346 343  0.01095153750804309
+342 344  0.01041925911497252
+344 344  0.04267383459784683
+346 344  0.01083874881310861
+342 345  0.004326216135614722
+345 345  0.04658231233880137
+346 345  0.01474293786201604
+343 346  0.006699358171649296
+344 346  0.007239024315376075
+345 346  0.01018761693750956
+346 346  0.01751500625278065
+347 346  0.004304953982006397
+349 346  0.005079138881795001
+367 346  0.001454172054025146
+346 347  0.004520592701407972
+347 347  0.03133577584027234
+348 347  0.02376018555322075
+347 348  0.0160394581106115
+348 348  0.03202605464930185
+349 348  0.01227714605981959
+346 349  0.004338408876136321
+348 349  0.01479350505925204
+349 349  0.02515075220638215
+350 349  0.01685942591684934
+333 350  0.03888330567823023
+334 350  0.0372013180575793
+349 350  0.02155036496873836
+350 350  0.05805941947495961
+351 351  0.1645907649718108
+366 351  0.05143990597485315
+352 352  0.1677415812221914
+353 352  0.03358407801109305
+408 352  0.08348439138250735
+409 352  0.08953611253826632
+352 353  0.05204079709633678
+353 353  0.09468589729298324
+363 353  0.00507415031316362
+382 353  0.05110789051702095
+354 354  0.04974317454374558
+382 354  0.02346918356886678
+383 354  0.07540334827104343
+355 355  0.4702425988200105
+375 355  0.04948828803447112
+356 356  0.08030998219707833
+383 356  0.03771054601479147
+398 356  0.008409579326427436
+357 357  0.7757864328798555
+394 357  0.0232032710660538
+395 357  0.002015548461802727
+358 358  0.03518415525707855
+479 358  0.02263656719864041
+359 359  0.07364375832559185
+474 359  0.0241285481784699
+360 360  0.1640699887011138
+715 360  0.05282962784979241
+361 361  0.06454489079530285
+479 361  0.02263664722922551
+480 361  0.06357691756419355
+362 362  0.09851134347060447
+403 362  0.08750001093750137
+714 362  0.01353091383749242
+353 363  0.003738764096769265
+363 363  0.02266531374200563
+364 363  0.01142688560381418
+365 363  0.01142688560381418
+373 363  0.01142688560381418
+374 363  0.01142688560381418
+388 363  0.006396484670544655
+395 363  0.0004965090437304651
+407 363  0.001322349211133573
+474 363  0.001436918225075881
+363 364  0.02266531374200564
+364 364  0.1207268836036242
+363 365  0.02266531374200564
+365 365  0.1207268836036242
+142 366  0.01228964334324267
+237 366  0.003082084361368065
+351 366  0.02899076003597058
+366 366  0.05143990597485316
+699 366  0.02218577128663575
+110 367  0.0002331220826489727
+111 367  0.001255943206821175
+346 367  0.001051486466295456
+367 367  0.008080325719593129
+368 367  0.004420797871254525
+372 367  0.004268611151296343
+111 368  0.002433220001569482
+367 368  0.007516582890387917
+368 368  0.01304211179365907
+369 369  0.01969999893700024
+371 369  0.0120000012000012
+110 370  0.002160868153680853
+112 370  0.004177756554428322
+342 370  0.0125595769289109
+370 370  0.02874308722730365
+369 371  0.005999998800000238
+371 371  0.0120000012000012
+372 371  0.005473931316300327
+367 372  0.004040164071845544
+371 372  0.006000002400000958
+372 372  0.007871272995179882
+363 373  0.02266531374200563
+373 373  0.1207268836036242
+363 374  0.02266531374200564
+374 374  0.1207268836036242
+355 375  0.008342647273315443
+375 375  0.04948828803447112
+376 375  0.00833252688360645
+377 375  0.00833252688360645
+378 375  0.00833252688360645
+379 375  0.008328380168034728
+380 375  0.008328380168034728
+381 375  0.008328380168034728
+389 375  0.02759343872382701
+404 375  0.003940407904002059
+375 376  0.04948828803447112
+376 376  0.5035325530301679
+375 377  0.04948828803447112
+377 377  0.5035325530301679
+375 378  0.04948828803447112
+378 378  0.5035325530301679
+375 379  0.04948828803447113
+379 379  0.518628273209177
+375 380  0.04948828803447113
+380 380  0.518628273209177
+375 381  0.04948828803447113
+381 381  0.518628273209177
+353 382  0.05317839646125243
+354 382  0.02180593855871351
+382 382  0.1097010539683369
+354 383  0.01344667433749287
+356 383  0.005093798483285893
+383 383  0.09592540978204692
+384 383  0.01313663464931629
+385 383  0.01311747358717988
+386 383  0.01475695225845563
+387 383  0.01475695225845563
+392 383  0.008411227964363893
+383 384  0.0959254097820469
+384 384  0.1359366267409968
+383 385  0.0959254097820469
+385 385  0.1376174815551804
+383 386  0.09592540978204692
+386 386  0.07535695286445564
+383 387  0.09592540978204692
+387 387  0.07535695286445564
+363 388  0.008457230857481606
+388 388  0.01764812610154917
+389 388  0.007778731056151564
+390 388  0.01149923997498271
+391 388  0.01149923997498271
+395 388  0.0003965626793025834
+401 388  0.005326143243121655
+405 388  0.004713555305320748
+375 389  0.04065610206645004
+388 389  0.003149666430399245
+389 389  0.03182056075270515
+388 390  0.01764812610154917
+390 390  0.1527992366826928
+388 391  0.01764812610154917
+391 391  0.1527992366826928
+383 392  0.06413472600198364
+392 392  0.1017812758283331
+395 392  0.0006679728610330234
+393 393  0.03800594424409709
+474 393  0.01206427408923495
+479 393  0.0113182835993202
+357 394  0.0232032710660538
+394 394  0.05152222998854333
+395 394  0.002015548484634019
+357 395  0.001306203030861274
+363 395  0.0005949409627688583
+388 395  0.0002496897142986828
+392 395  0.0004470416577322448
+394 395  0.001306203045657397
+395 395  0.002015548444148886
+398 395  0.001345957292266748
+399 395  0.001411655520140569
+405 395  0.0007991848413365207
+406 395  0.001999646519169777
+407 395  0.001290913678155762
+474 395  0.0007708309133194762
+396 396  0.01039879170798621
+406 396  0.002963024354424753
+716 396  0.006742598457066043
+717 396  0.006742597597317331
+397 397  0.05829027397126174
+716 397  0.06097941825040581
+717 397  0.06090304313760567
+356 398  0.007833661897924957
+395 398  0.001818946290217446
+398 398  0.01385715250380462
+395 399  0.002015548444148885
+399 399  0.01431165549434057
+400 400  0.2445987775061038
+473 400  0.2598993828713796
+388 401  0.005708671014284298
+401 401  0.05319211368992818
+402 401  0.04745813290972838
+714 401  0.01064967860759701
+401 402  0.05319211368992818
+402 402  0.1718581206687696
+362 403  0.09851134347060446
+403 403  0.1780999911867057
+375 404  0.01848783121482114
+404 404  0.2280199623643019
+409 404  0.1250424020850327
+388 405  0.007565032031499309
+395 405  0.001334830084583861
+405 405  0.0565161430836392
+395 406  9.016385884124244e-05
+396 406  8.496382333889849e-05
+406 406  0.003316966354672551
+479 406  5.123168037249279e-05
+483 406  0.00162477055619419
+716 406  9.525755527473e-05
+717 406  9.525754619047622e-05
+363 407  0.002968326301297488
+395 407  0.001852195046613556
+407 407  0.01128987599010727
+352 408  0.1677415812221914
+408 408  0.4242843101358066
+352 409  0.1097324145119118
+404 409  0.07885493658968357
+409 409  0.1996142975217009
+410 410  0.000643158433004506
+411 410  0.002569533326060479
+412 410  7.446159408519452e-05
+731 410  5.883559546889036e-05
+15 411  9.835716465245902e-05
+410 411  0.0005477497400358428
+411 411  0.002609486999747394
+476 411  5.813859319170489e-05
+477 411  2.578446271091461e-05
+486 411  4.939008671568841e-06
+38 412  0.002478469536115102
+410 412  0.0004617144309759917
+412 412  0.009429739081657456
+413 412  0.009174104651984229
+478 412  0.0002219056184528127
+709 412  0.0002243443947259024
+724 412  0.0027358543241002
+14 413  0.08463315757847611
+99 413  0.03363341832610426
+412 413  0.003588183179913349
+413 413  0.1087319346343822
+414 414  0.006364255318889591
+431 414  0.005136411098556414
+433 414  0.0006330696963460082
+415 415  0.006364255318889591
+432 415  0.005136411098556414
+433 415  0.0006330696963460082
+79 416  0.0007559194782724966
+416 416  0.004101785720452792
+433 416  0.0005578334325645057
+462 416  0.003576021470224139
+79 417  0.0007559194782724967
+417 417  0.004101785720452792
+433 417  0.0005578334325645057
+462 417  0.003576021470224139
+418 418  0.0570517923715762
+419 418  0.03347999916567841
+545 418  0.01782518940492027
+418 419  0.0570517923715762
+419 419  0.1399800032659285
+420 420  0.001500000825000486
+421 420  0.0005958668376097395
+422 420  0.0005863636523820253
+470 420  0.002424268549138517
+420 421  0.001500000825000485
+421 421  0.02639586678600974
+420 422  0.001500000825000486
+422 422  0.08708639089989062
+423 423  0.003405062645742318
+425 423  0.002541029068390307
+516 423  0.0002515099372990493
+424 424  0.005664338462523589
+466 424  0.003927015826493891
+516 424  0.0005202826270102751
+54 425  0.0002565787489259389
+423 425  0.001978088041133092
+425 425  0.00293037999291745
+426 425  0.0005316534119392825
+427 425  0.001148886486005569
+466 425  0.0009056643417850977
+425 426  0.0004018409226975211
+426 426  0.006550786131435199
+437 426  0.007597187751206623
+519 426  0.0004513917399590606
+425 427  0.002930379992917452
+427 427  0.02924888539010561
+428 428  0.006299997570601267
+429 428  0.003559485227246435
+430 428  0.003718820571593135
+431 428  0.00287392452846315
+432 428  0.00287392452846315
+428 429  0.006299997570601268
+429 429  0.04355948522724643
+428 430  0.006299997570601269
+430 430  0.03071881841159331
+414 431  0.004141815825411193
+428 431  0.002199998464400812
+431 431  0.007209826826433458
+415 432  0.004141815825411193
+428 432  0.002199998464400812
+432 432  0.007209826826433458
+414 433  0.0009698489962126803
+415 433  0.0009698489962126803
+416 433  0.0006250712085439606
+417 433  0.0006250712085439606
+433 433  0.002201356340862928
+434 433  0.00175445638940735
+510 433  0.0007153148545362604
+564 433  0.000732509572838798
+433 434  0.002201356340862928
+434 434  0.03295445838620748
+435 435  0.01234020240462855
+510 435  0.002368518978697939
+436 436  0.04798992551016811
+534 436  0.002753567122893018
+426 437  0.005431400011357155
+437 437  0.00896202923169572
+438 437  0.001989799887815083
+439 437  0.001094884934097482
+441 437  0.001865309750819055
+458 437  0.0004901539131952915
+437 438  0.008962029231695721
+438 438  0.02388980010681508
+437 439  0.002532569114931716
+439 439  0.09645913792039387
+440 439  0.0921454312912984
+460 439  0.007379838453911518
+439 440  0.09645913792039387
+440 440  0.1816854552343008
+437 441  0.002392209988767325
+441 441  0.08436532418832167
+442 441  0.08250001443750257
+441 442  0.08436532418832167
+442 442  0.1125400126951827
+443 443  0.0009259435004858887
+444 443  0.0003236289735023835
+470 443  0.001707252245567811
+506 443  1.839401095112365e-05
+516 443  0.000332744496572827
+443 444  5.144132035192436e-05
+444 444  0.007153934619588723
+458 444  0.008552595425710553
+54 445  0.001274589000506678
+445 445  0.004367579048575067
+446 445  0.003973963702530854
+447 445  0.0004751264904632971
+445 446  0.004367579048575068
+446 446  0.0253739646013309
+54 447  0.0002977541475090077
+80 447  0.0008969608333565075
+445 447  0.000285066183339261
+447 447  0.001648620842923621
+449 447  0.0002365034127268017
+537 447  0.0008366494699104182
+81 448  0.003867353289951346
+448 448  0.009085811687866626
+464 448  0.003452828236499709
+447 449  5.846174222491055e-05
+449 449  0.004026019638989517
+450 449  0.005572822551903635
+81 450  0.0003295690503611483
+449 450  0.003665339259806753
+450 450  0.005777704586613097
+451 450  0.0004596774439698244
+450 451  0.005777704586613096
+451 451  0.04725968043917002
+78 452  0.002219578534304645
+80 452  0.0009738189492165476
+452 452  0.006869084548990908
+453 452  0.005023880261239764
+452 453  0.006869084548990908
+453 453  0.03112388451554046
+79 454  0.005197036156924244
+454 454  0.007995797303423095
+455 454  0.002650000144425008
+454 455  0.007995797303423095
+455 455  0.03044999475122606
+456 456  0.04341440055575247
+457 456  0.01229999289060411
+564 456  0.01602933825183329
+456 457  0.04341440055575248
+457 457  0.05350000080100564
+437 458  0.0002009059434134133
+444 458  0.006755714948519285
+458 458  0.009055686456956213
+459 458  0.000766289156101921
+460 458  0.0006713497988630232
+458 459  0.009055686456956211
+459 459  0.05426628567860215
+439 460  0.008640191627167487
+458 460  0.00443565641705256
+460 460  0.03510182685402592
+461 460  0.03339335231856724
+460 461  0.03510182685402592
+461 461  0.07939335323856726
+416 462  0.001848944552278542
+417 462  0.001848944552278542
+462 462  0.005826020253284005
+463 462  0.002249999668125049
+462 463  0.005826020253284005
+463 463  0.02284999937972506
+81 464  0.001779445449218795
+448 464  0.002849339920859814
+464 464  0.01526227908144768
+465 464  0.01079433379213668
+545 464  0.005691657248232669
+464 465  0.004427225641076556
+465 465  0.03465230862706979
+968 465  0.02538569746428179
+424 466  0.00291083170620776
+425 466  0.001072372864304547
+466 466  0.005415480073738198
+467 466  0.002745132358752783
+466 467  0.005415480073738198
+467 467  0.0228451338863529
+468 468  0.05510550055005413
+544 468  0.05510495451728012
+469 469  0.01341557922769634
+543 469  0.01855183921053081
+420 470  0.0009000006030003422
+443 470  0.0003703773335264287
+470 470  0.002497462050535406
+471 471  0.02675026445067476
+472 471  0.02144476375697679
+478 471  0.002480272980294884
+481 471  0.002283100356600413
+492 471  0.001339230588855801
+707 471  0.002265563440073605
+708 471  0.007899110979947414
+471 472  0.02675026445067475
+472 472  0.06197810141386614
+400 473  0.2444987775061038
+473 473  0.2598993828713796
+474 473  9.876610596768615e-06
+359 474  0.01974838971630254
+363 474  0.002047480310664833
+393 474  0.008859454214705957
+395 474  0.0007630072433894486
+473 474  0.01538178618237745
+474 474  0.02412854817846989
+3 475  0.00646191477577647
+475 475  0.03324810431463584
+710 475  0.01261217684683017
+724 475  0.008319987404669415
+411 476  0.001570265838304289
+476 476  0.01396026233454226
+482 476  0.004080436981661773
+411 477  0.0007799517638505359
+477 477  0.004427955706412391
+478 477  0.004935991219056525
+103 478  0.004269418185236161
+412 478  3.016846928798592e-06
+471 478  4.662970586208979e-05
+477 478  6.965301185643493e-05
+478 478  0.007040263277160317
+708 478  5.374029868745428e-05
+709 478  3.456002432602921e-05
+358 479  0.01388415670547844
+361 479  0.009950702953563957
+393 479  0.005152425593486999
+406 479  0.0006682906745437504
+479 479  0.0226365671986404
+480 479  0.009950703698200726
+714 479  0.001880325451835653
+726 479  0.0009429661915102201
+361 480  0.06357691756419355
+479 480  0.02263664892318421
+480 480  0.06459064614023635
+471 481  0.002756565785961451
+481 481  0.06462635410894584
+484 481  0.02938317987358151
+485 481  0.01849891953488641
+491 481  0.009825378992632576
+492 481  0.002256316524900598
+733 481  0.04196993889449727
+476 482  0.004066407369324664
+482 482  0.01024598771848416
+483 482  0.001219574850379582
+406 483  0.003291052606449368
+482 483  8.004678286288545e-05
+483 483  0.001720823741935278
+481 484  0.02868022868001192
+484 484  0.07505287813943308
+709 484  0.01851248638185964
+733 484  0.04135267929782272
+481 485  0.02921838436352269
+485 485  0.09868051329820485
+491 485  0.03150328057877917
+44 486  0.004278633259597338
+411 486  1.285232732433574e-05
+486 486  0.001165659392410865
+731 486  4.049114586387005e-05
+487 487  0.2020136631043835
+491 487  0.04305408281593034
+488 488  0.2020136631043835
+491 488  0.04305408281593034
+489 489  0.1435997065319451
+492 489  0.01230949843529913
+490 490  0.1413997019505066
+492 490  0.01230949843529913
+481 491  0.01035168481130299
+485 491  0.02100428797037725
+487 491  0.02791365135263274
+488 491  0.02791365135263274
+491 491  0.04305408281593034
+492 491  0.005165474208960842
+734 491  0.001227046330290453
+471 492  0.00197146061259278
+481 492  4.011458626650895e-05
+489 492  9.97041641950374e-05
+490 492  9.970524279656442e-05
+491 492  7.592198581198728e-05
+492 492  0.01230949843529913
+707 492  0.02198305875060876
+493 493  0.08909940016095232
+707 493  0.02206309641910328
+494 494  0.01406253887620965
+505 494  0.008005776682719633
+495 495  0.02879369946723359
+513 495  0.01884335699129711
+496 496  0.007899758913639471
+521 496  0.006694920963544377
+497 497  0.01599663587803108
+522 497  0.007871670338691458
+498 498  0.05419663997109909
+523 498  0.02455560794957828
+499 499  0.02320378893096543
+500 499  0.020850000302325
+819 499  0.003414041754791358
+914 499  0.003526835757809051
+499 500  0.02320378893096543
+500 500  0.04624999948952503
+501 501  0.005232158939923239
+502 501  0.001696692336767111
+505 501  0.0007509631309112995
+507 501  0.00117070888026396
+521 501  0.001443561506118148
+531 501  0.001109607069418476
+536 501  0.0004217156152754402
+501 502  0.001437352830834991
+502 502  0.01165239098808167
+607 502  0.004360314754718347
+609 502  0.004956038990736548
+613 502  0.004951278957397824
+937 502  0.001037580302783396
+503 503  0.01101054586613354
+507 503  0.001507956907669392
+513 503  0.008614977101122299
+553 503  0.003328225977473673
+121 504  0.002803923428298168
+213 504  0.003668071549299035
+504 504  0.006674656586546838
+505 504  0.0029178786038972
+494 505  0.005862537564209443
+501 505  0.0003951008735467709
+504 505  0.001629808978278572
+505 505  0.008005776682719633
+521 505  0.001041376548755184
+526 505  0.0009648404691351401
+578 505  0.001840518415168956
+48 506  0.004319782666390631
+443 506  1.41365439410511e-05
+506 506  0.0001928389218433751
+501 507  0.001087041669452453
+503 507  0.001598049399893468
+507 507  0.004269494702603494
+508 507  0.0008134383940814002
+509 507  0.0003821908291541762
+515 507  0.0003975920977657642
+521 507  0.001564552251530617
+522 507  0.001289983641602014
+547 507  0.001329975337819727
+507 508  0.0001938459936990633
+508 508  0.01998888769357894
+619 508  0.01088941397533659
+639 508  0.003848848222262508
+781 508  0.002544443119803931
+507 509  0.0002755253431618543
+509 509  0.002031718522070935
+511 509  0.0004600728778631732
+515 509  0.0002486502206999728
+520 509  0.0006292261443290039
+534 509  0.001588740474260125
+433 510  0.0001512389025817893
+435 510  0.0003902026376535502
+510 510  0.002368518978697939
+520 510  0.002168860850370775
+509 511  0.0008247230633282589
+511 511  0.0347073296207404
+914 511  0.00292315189268783
+926 511  0.0147935377324369
+512 512  0.01048181821270249
+520 512  0.002410711473413276
+495 513  0.007193701195233452
+503 513  0.003300996193872722
+513 513  0.01884335699129712
+514 513  0.002037287546642802
+540 513  0.007031714890658067
+513 514  0.006240776994745908
+514 514  0.04024397712110585
+790 514  0.01834818795016163
+795 514  0.006109223041858325
+507 515  0.0003005067021361314
+509 515  0.0002553864953828061
+515 515  0.002489282294551514
+516 515  0.001146166513105429
+517 515  0.0007143738360434264
+518 515  0.001492624019293777
+536 515  0.0002630923132246919
+547 515  0.001107137165248834
+423 516  0.0003179471424787562
+424 516  0.0005289066356661023
+443 516  0.0003063147971697974
+515 516  0.001200921043734124
+516 516  0.001892943084271573
+515 517  0.0009350600513675123
+517 517  0.04753688459810971
+993 517  0.03571088281389261
+995 517  0.007727232829020826
+515 518  0.002489282294551514
+518 518  0.05649261851929432
+426 519  0.0005724029120355825
+519 519  0.0297707375762613
+957 519  0.00870215574754522
+967 519  0.02411319225780486
+509 520  0.0001779735487087412
+510 520  0.002074764408768619
+512 520  0.0003818181117024921
+520 520  0.002410711473413276
+496 521  0.004599758880639473
+501 521  0.0004416368365997204
+505 521  0.0006854506463116265
+507 521  0.0005714536312007762
+521 521  0.006694920963544378
+522 521  0.001672368794195314
+497 522  0.006396638950030098
+507 522  0.0008867299146804817
+521 522  0.002918329156480371
+522 522  0.007871670338691456
+523 522  0.001131618119980197
+498 523  0.004396636783898884
+522 523  0.0003817679132325544
+523 523  0.02455560794957828
+582 523  0.01699786774829901
+770 523  0.004096106648807382
+779 523  0.002460291457037252
+802 523  0.0005044474659523555
+524 524  0.0326225767152272
+533 524  0.01617785284301497
+985 524  0.003533427920856026
+1001 524  0.001745248035995898
+525 525  0.02442130355584616
+568 525  0.02917419578216094
+505 526  0.001964562380145534
+526 526  0.01603127693278412
+527 526  0.003271154589289226
+531 526  0.002707873419341401
+526 527  0.002297473511344216
+527 527  0.01093644448317669
+921 527  0.000366389537306514
+933 527  0.001763794600860524
+1068 527  0.004132599271484757
+1073 527  0.00310282109399263
+1074 527  0.006403765665934716
+1124 527  0.00625677626434674
+528 528  0.009626573465522413
+529 528  0.007131985958546047
+809 528  0.00239329992538613
+819 528  0.003106803164457794
+528 529  0.009626573465522413
+529 529  0.01223198677454618
+530 530  0.01691088535944298
+578 530  0.001074968070443172
+583 530  0.009764865690953977
+780 530  0.001219361054567015
+797 530  0.001686463163686368
+803 530  0.03152768681269896
+501 531  0.001090658453345587
+526 531  0.002566388662344346
+531 531  0.005944823775258955
+532 531  0.000821515577234781
+534 531  0.001014190985465075
+531 532  0.0006049555513227042
+532 532  0.003121950329375685
+554 532  0.0003352071636419773
+647 532  0.002073185558188997
+652 532  0.001352902530582362
+663 532  0.0003592411960136898
+668 532  0.0006129106476266529
+672 532  0.0001780754800275389
+673 532  0.001168464042819279
+677 532  0.0004358201556169567
+908 532  0.0002543322005931329
+912 532  0.0007881034826923823
+918 532  8.244961498032527e-05
+1047 532  0.0004211391056336174
+524 533  0.03262257671522721
+533 533  0.02247785120501539
+436 534  0.00138993100896746
+509 534  0.001443241409684484
+531 534  0.0006032764585047403
+534 534  0.002753567122893018
+535 534  0.0008767570510231159
+536 534  0.0002023630937857494
+542 534  0.0003069070984018251
+534 535  0.001576354145301813
+535 535  0.03027165038959069
+885 535  0.007837157227708233
+501 536  0.0003208138436881328
+515 536  0.0002874163965662608
+534 536  0.0002668946311501982
+536 536  0.002244959367557183
+537 536  0.001280692349708539
+538 536  0.0003790346554514227
+539 536  0.001469989041819689
+542 536  0.0004736553092091702
+447 537  0.0008891438110608148
+536 537  0.001034194784351591
+537 537  0.002146826463178067
+536 538  0.0004481344646009161
+538 538  0.01252220930778559
+809 538  0.004569856816044243
+914 538  0.0008492315451505555
+948 538  0.003857866148405762
+953 538  0.004022640943472898
+964 538  0.006819073731973737
+536 539  0.002244959367557183
+539 539  0.05646998354182024
+513 540  0.01461065560185121
+540 540  0.01759083286364846
+800 540  0.007934058510710686
+541 541  0.07154939909954779
+543 541  0.01855183921053081
+534 542  0.0004935388767098761
+536 542  0.0005775217137079195
+542 542  0.005015830848531875
+543 542  0.003098506385171987
+545 542  0.001956527433207467
+546 542  0.002676682454067786
+469 543  0.01331557922769634
+541 543  9.939266904720062e-05
+542 543  2.136670957503959e-05
+543 543  0.01855183921053081
+544 543  9.944401188488195e-05
+468 544  0.05500550055005413
+543 544  3.366940913367631e-05
+544 544  0.05510495451728012
+418 545  0.0235717924167142
+464 545  0.004425982967227826
+542 545  0.001488900773344342
+545 545  0.02342882527243381
+125 546  0.002250491105321284
+542 546  0.00237717115582201
+546 546  0.007634814497046847
+507 547  0.001631377753674887
+515 547  0.001787902010367797
+547 547  0.01241499285070165
+548 548  0.14323571522039
+551 548  0.02570126355718328
+549 549  0.1601038244618143
+563 549  0.05468868855029431
+550 550  0.04008590449537204
+566 550  0.0166719539021383
+548 551  0.01733571909810991
+551 551  0.02570126355718328
+552 551  0.01351756037875005
+555 551  0.005883563599726211
+566 551  0.007220063785949487
+551 552  0.01282274308683993
+552 552  0.05065452995193308
+558 552  0.001734104471257444
+560 552  0.03370717340059064
+34 553  0.006470328250772428
+503 553  0.003731423161817809
+553 553  0.01755231084499826
+561 553  0.007796010687171485
+710 553  0.0024289841157409
+532 554  0.0003193514752011467
+554 554  0.004919122257832556
+574 554  0.003873064794127629
+909 554  0.006235565611699458
+551 555  0.007502647259244349
+555 555  0.02094549304621092
+556 555  0.004504535885850671
+557 555  0.006887885052325974
+564 555  0.003881871867848462
+566 555  0.004016775969250857
+555 556  0.002261041531412913
+556 556  0.01626992155748077
+559 556  0.009951085482870572
+579 556  0.002753261862873228
+621 556  0.005541852802385738
+903 556  0.004529974196489342
+555 557  0.007155585803720392
+557 557  0.04708679258737491
+558 557  0.01330519675132981
+552 558  0.001476850571846759
+557 558  0.00754602971094295
+558 558  0.02020927228120075
+979 558  0.0002514980660136494
+980 558  0.008062883380278824
+986 558  0.006386762196060229
+989 558  0.005052587455907618
+991 558  0.0111793945366686
+998 558  0.00274502194402396
+1000 558  0.01176072660591546
+556 559  0.0146677560682016
+559 559  0.02267773670558225
+579 559  0.004402776932694109
+552 560  0.03683233072325013
+560 560  0.08116779366381811
+980 560  0.01832009613707559
+553 561  0.002339164686951676
+561 561  0.01982387379969139
+562 561  0.01338310636271287
+566 561  0.003220808187600286
+561 562  0.01865589822730168
+562 562  0.01986967192275656
+563 562  0.003222104057952865
+1 563  0.0001226117406941805
+2 563  0.02963423489898008
+549 563  0.04290382160213425
+562 563  0.005279528895186774
+563 563  0.05468868855029431
+567 563  0.01496037139568757
+433 564  0.0004111320681232224
+456 564  0.03111442297343562
+555 564  0.00202233327150202
+564 564  0.02081480258697662
+565 565  0.3305629158751743
+567 565  0.14066430109025
+11 566  0.00346861203074192
+550 566  0.01178590525947201
+551 566  0.005745114858221226
+555 566  0.002506462523915269
+561 566  0.006393320866839186
+566 566  0.0166719539021383
+567 566  0.004953107941387935
+563 567  0.01465646690876327
+565 567  0.1312628974000626
+566 567  0.005583646470765764
+567 567  0.1406643010902501
+525 568  0.01172130396224615
+568 568  0.02917419578216094
+773 568  0.003444012454111461
+795 568  0.006804073324986602
+569 569  0.06519322693617369
+785 569  0.01630843556546118
+570 570  0.08233568498998933
+579 570  0.0163349833646956
+571 571  0.09496350902083774
+776 571  0.04186788389119319
+572 572  0.1158570824502996
+783 572  0.05218409839323126
+573 573  0.07019182874608747
+796 573  0.02708953246662872
+554 574  0.004919122257832556
+574 574  0.01457306524352765
+575 575  0.03866767333357445
+581 575  0.04073951232804236
+576 576  0.05391458588463812
+582 576  0.01203398810588504
+784 576  0.02152420862206806
+577 577  0.00306672227778948
+786 577  0.006696394088334022
+802 577  0.0003043306524726053
+1090 577  0.0012464868831832
+505 578  0.003736133464812645
+530 578  0.00551489235021096
+578 578  0.0250893961158919
+780 578  0.00822075652969664
+556 579  0.003078063323239608
+559 579  0.003339361928438725
+570 579  0.01433567410998759
+579 579  0.0163349833646956
+580 579  0.00250431771032046
+782 579  0.003661767172755128
+783 579  0.005459279435238132
+895 579  0.0096135007675579
+927 579  0.0003409602645951949
+579 580  0.002843774085809601
+580 580  0.07255118833952573
+587 580  0.3430016474367272
+575 581  0.03616767333357445
+581 581  0.04073951232804236
+776 581  0.0009849149617359499
+796 581  0.0005239312254705801
+797 581  0.0002610114068673552
+799 581  0.0006397011125999043
+523 582  0.02299408921831852
+576 582  0.003428484013456784
+582 582  0.02145789393602984
+530 583  0.011403793616007
+583 583  0.04368942257989474
+780 583  0.01112424271842943
+803 583  0.01862651384398668
+584 584  0.02882155982257682
+585 584  0.01809279493917329
+1068 584  0.002213107301306411
+584 585  0.01079774525545556
+585 585  0.02070474183883917
+607 585  0.005046300734798805
+613 585  0.006111085326771086
+659 585  0.0004391739052375558
+586 586  0.4962892537424193
+597 586  0.1592822846412858
+580 587  0.01501616578679236
+587 587  0.4153019406146015
+598 587  0.01896030773337864
+614 587  0.03582205893731713
+588 588  0.1881752688478352
+595 588  0.0962969913992603
+597 588  0.07862545232220013
+589 589  0.2668000019223201
+591 589  0.1729536603888307
+590 590  0.4243994434473364
+595 590  0.07111659791198265
+616 590  0.2269998631190825
+589 591  0.10219999803776
+591 591  0.1729536603888307
+601 591  0.05639671338781434
+592 592  0.2170000006832015
+611 592  0.216699852301541
+593 593  0.1442520613678445
+604 593  0.1113998696564958
+594 594  0.01923999996047211
+600 594  0.02988530359073937
+588 595  0.1006498175293541
+590 595  0.1973997147573786
+595 595  0.1901687731682625
+596 596  0.05495189909556643
+597 596  0.03491193073271593
+603 596  0.01287992830377254
+607 596  0.007692079568584614
+586 597  0.102289243498419
+588 597  0.07546209895054574
+596 597  0.02158904963424665
+597 597  0.1592822846412858
+587 598  0.4153019406146015
+598 598  0.08996029282338179
+599 599  0.227200021676803
+1105 599  0.06992316227266118
+594 600  0.003540000572772093
+600 600  0.02988530359073937
+601 600  0.001243234856412491
+782 600  0.01042509323263464
+591 601  0.07075366235107063
+600 601  0.005882652852335237
+601 601  0.09141338326355179
+602 601  0.05355695061435049
+633 601  0.05066003695649697
+601 602  0.0470338638274957
+602 602  0.2441567076406213
+640 602  0.1905998080659934
+596 603  0.007255886343007262
+603 603  0.04378116350077813
+604 603  0.01095904454988537
+605 603  0.008973606895093062
+1092 603  0.02627123108084698
+593 604  0.04935208120194032
+603 604  0.006876957730302293
+604 604  0.1113998696564958
+612 604  0.04355999646815548
+615 604  0.02384372426385743
+603 605  0.01534211174266245
+605 605  0.04024699409656525
+607 605  0.01026772171243339
+606 606  0.2914000093530803
+610 606  0.362499508159066
+502 607  0.003759678237179507
+585 607  0.007798185119838794
+596 607  0.005713337516057005
+605 607  0.007918174627592513
+607 607  0.01580689215892814
+608 608  0.1748852365409521
+610 608  0.3381995414014211
+1101 608  0.006975330367616711
+502 609  0.006325416646161063
+609 609  0.05431870625589665
+1068 609  0.008019987326824395
+606 610  0.02429999963064
+608 610  0.1603014493503626
+610 610  0.3624995081590661
+592 611  0.08479999158784084
+611 611  0.216699852301541
+615 611  0.05901208119236167
+604 612  0.1113998696564958
+612 612  0.1710600006119056
+502 613  0.005190609684796298
+585 613  0.01148171860232741
+613 613  0.02189807970972138
+587 614  0.4153019406146015
+614 614  0.06152205618741742
+604 615  0.04359349658694239
+611 615  0.1318999054541437
+615 615  0.09686554554992785
+590 616  0.4243994434473363
+616 616  0.3625998680549227
+617 617  0.03020000296200128
+1123 617  0.0193500141544857
+618 618  0.04081223932712202
+629 618  0.05720285282046129
+639 618  0.01144819822827145
+508 619  0.01943499047608837
+619 619  0.01645120916664465
+620 619  0.00423534490524011
+619 620  0.01160192153660821
+620 620  0.1528439410437316
+632 620  0.05409203383921641
+556 621  0.005339324584916143
+621 621  0.01853167930014241
+624 621  0.02018511896003668
+1061 621  0.0006930360661798945
+622 622  0.2035583301907608
+625 622  0.1485785083616206
+641 622  0.04781225452114995
+623 623  0.07281282240639529
+639 623  0.01016929916811019
+642 623  0.03595827612770093
+621 624  0.01467654563738238
+624 624  0.03158510792992999
+638 624  0.01139999587320149
+622 625  0.126835008401068
+625 625  0.1735592372972785
+635 625  0.03281961919387635
+626 626  0.1811769747237068
+634 626  0.1514803924195199
+635 626  0.02945609591899677
+627 627  0.07505490672951293
+639 627  0.0180560389540825
+628 628  0.1056265999772952
+631 628  0.06273914382313427
+642 628  0.01887234760978006
+618 629  8.035635299263623e-05
+629 629  0.1563077836874048
+637 629  0.007997888557420911
+643 629  0.0001995971929049984
+630 630  0.0452388865598593
+631 630  0.01732372432840822
+635 630  0.01358448800769288
+796 630  0.01709806713001748
+628 631  0.08962578652385089
+630 631  0.006852996695493958
+631 631  0.08139971357245947
+620 632  0.02988999796057287
+632 632  0.1835072346350969
+634 632  0.1463196343743539
+641 632  0.05489971304307033
+601 633  0.09141338326355179
+633 633  0.289960036597547
+626 634  0.08183991251708694
+632 634  0.1006147586707503
+634 634  0.2289201683860113
+625 635  0.05995614861954387
+626 635  0.02789077922630471
+630 635  0.01052498461289881
+635 635  0.08707533504085817
+636 635  0.05089120870851743
+635 636  0.05423502137229106
+636 636  0.1899255736897406
+1094 636  0.1090050735602148
+629 637  0.1563077836874048
+637 637  0.008197888557420912
+624 638  0.03158510792992999
+638 638  0.05480000464000327
+508 639  0.01127332807615319
+618 639  0.0071224794614393
+623 639  0.00753993170476529
+627 639  0.01185492745910613
+639 639  0.0180560389540825
+602 640  0.2441567076406213
+640 640  0.3925998100859934
+622 641  0.1025854457209622
+632 641  0.09102675771027169
+641 641  0.3321852897814107
+623 642  0.03267696811303935
+628 642  0.05822342782450567
+642 642  0.08232357311481474
+629 643  0.1563077836874049
+643 643  0.008899599367905542
+644 644  0.01323657303124626
+652 644  0.00239707312597985
+662 644  0.005025584021822942
+645 645  0.09630003872258575
+650 645  0.06664279696435015
+1060 645  0.03730000622537103
+646 646  0.05391285880092912
+664 646  0.08453966978632144
+1051 646  0.01116364723718039
+532 647  0.002678834768336893
+647 647  0.009630532952425559
+1041 647  0.001817468630679752
+648 648  0.1346148415060411
+678 648  0.03513684926359105
+1042 648  0.01540000117040009
+649 649  0.01285948428552433
+672 649  0.00150499110598391
+1055 649  0.003523063866085778
+645 650  0.05900002589511072
+650 650  0.08012313786808076
+653 650  0.02613589321014604
+651 651  0.03797353961441741
+670 651  0.0094971866657186
+1045 651  0.01648149447995612
+532 652  0.0008217221959411972
+644 652  0.0001796923453236811
+652 652  0.003788921731632382
+653 652  7.321716281347581e-05
+663 652  0.001287482605270614
+666 652  5.249537174985109e-05
+668 652  0.0002007198694312743
+676 652  0.0001112784764502267
+1054 652  0.0002529543631071838
+650 653  0.0424459222855708
+652 653  0.0005258686939950927
+653 653  0.06747685422597233
+1048 653  0.04675158312548357
+654 654  0.01260807957932831
+677 654  0.001756059504498253
+1057 654  0.01380321171526099
+655 655  0.008904081060294673
+657 655  0.001946776773363627
+665 655  0.006751113816256559
+656 656  0.2619335792700578
+678 656  0.04737909658209963
+655 657  0.003816035551306431
+657 657  0.007981783418949059
+1055 657  0.003639829735503001
+658 658  0.006260198176426003
+671 658  0.004278479241092181
+672 658  0.001881896729747303
+585 659  0.001037181977881552
+659 659  0.01624023749937267
+677 659  0.007736078590405019
+660 660  0.02790850835976976
+1050 660  0.003412792197284643
+1058 660  0.003312140679385912
+661 661  0.004244637672831487
+677 661  0.008144044717630895
+644 662  0.004723423708553365
+662 662  0.01368075840798956
+672 662  0.002165886092294924
+532 663  0.0008529469044980209
+652 663  0.003755853105115885
+663 663  0.001570809468248279
+646 664  0.02736117789762836
+664 664  0.09770634942975852
+1056 664  0.02240998246274745
+655 665  0.006545383899953773
+665 665  0.008928890811550128
+682 665  0.001639777210896398
+652 666  0.0005900779526785411
+666 666  0.002212746930117028
+681 666  0.008486672214830335
+667 667  0.2585466311633726
+1051 667  0.08284242548864497
+532 668  0.001477443487970514
+652 668  0.003094491599300593
+668 668  0.004822294611486507
+669 669  0.006485560610471147
+679 669  0.001783783119839592
+1058 669  0.005162877238538627
+651 670  0.008303904399083862
+670 670  0.022494378267569
+1121 670  0.007875400787643165
+658 671  0.004458019194828498
+671 671  0.009697889314677979
+679 671  0.003081080429668798
+532 672  0.000137844322866883
+649 672  0.0008297043679315003
+658 672  0.001682971459881668
+662 672  0.001282984835809149
+672 672  0.003367603842288365
+673 672  0.0007493645012146655
+682 672  0.001174687797646172
+1049 672  0.001045870538080219
+1050 672  0.0003985892779680757
+1052 672  0.000145612193105728
+1058 672  0.0001982815878775159
+1121 672  0.001711425586384149
+532 673  0.001852560443682052
+672 673  0.001534848059850153
+673 673  0.01851598774861257
+674 674  0.02402232318843044
+675 674  0.01983441626664977
+1046 674  0.0008145987699775598
+674 675  0.02192459877049487
+675 675  0.02094228360985793
+1050 675  0.0006403606362552
+652 676  0.00040774836406457
+676 676  0.02454558279094245
+1043 676  0.02101172610586178
+532 677  9.327352359497873e-05
+654 677  0.0005312645856911045
+659 677  0.001073514741321412
+661 677  0.002244637672831488
+677 677  0.008144044717630895
+678 677  0.0002178685533486336
+1041 677  0.0009028602314610852
+1059 677  0.001393437897625826
+648 678  0.1192148219765571
+656 678  0.01443357679505776
+677 678  0.0004829145628808069
+678 678  0.04737909658209963
+669 679  0.003607352095920667
+671 679  0.004303795505879453
+679 679  0.01070269818390264
+680 680  0.01488008007666763
+1047 680  0.008686049844786263
+1057 680  0.009766514009451666
+666 681  0.0007112400557228433
+681 681  0.0100521751695015
+1053 681  0.001839884244596412
+665 682  0.003872222529723286
+672 682  0.001907161977891775
+682 682  0.006190158470591963
+683 683  0.7542499839577274
+689 683  0.1675635901773625
+684 684  0.6462017354236403
+689 684  0.1675635901773624
+685 685  0.5666997770669278
+691 685  0.03896641487219615
+112 686  0.01691740047537703
+686 686  0.03200357666390856
+687 686  0.003224099031167913
+688 686  0.001468278890354747
+686 687  0.03200357666390856
+687 687  0.05772408077367402
+686 688  0.01098878080263125
+688 688  0.01972804813157901
+689 688  0.01037615544709978
+694 688  0.0006466060581027378
+699 688  0.006856167006648931
+700 688  0.01003878005679547
+702 688  0.008631739500549513
+683 689  0.1209501944032475
+684 689  0.1262017562236396
+688 689  0.008834200967155087
+689 689  0.1675635901773625
+693 689  0.008839385710618451
+113 690  0.004731551820574414
+690 690  0.0135101390863665
+691 690  1.910253637326516e-05
+694 690  8.776830950330473e-05
+697 690  9.836441838633725e-05
+685 691  9.96103731587792e-05
+690 691  6.714139024044764e-06
+691 691  0.03896641487219615
+692 691  0.02446183953033295
+695 691  6.486467039382385e-05
+691 692  0.03896641487219615
+692 692  0.02456183953033295
+689 693  0.00899196597043592
+693 693  0.04497383722385961
+695 693  0.01805898984373897
+688 694  0.0008725144849825894
+690 694  0.01159370575005668
+694 694  0.005538997701469825
+695 694  0.001862968432341841
+691 695  0.01535552727743594
+693 695  0.01772287716298745
+694 695  0.001173482907586958
+695 695  0.01908304387364602
+300 696  0.02765274863876076
+696 696  0.03352765471140241
+690 697  0.007282564889747137
+697 697  0.07919833333209852
+698 697  0.0790999689137122
+697 698  0.07919833333209851
+698 698  0.1715999758512127
+366 699  0.0290746602309459
+688 699  0.0065709769404977
+699 699  0.06056313922562951
+700 699  0.02592637862647972
+688 700  0.01289503718154808
+699 700  0.03474841209665323
+700 700  0.08346718596140983
+111 701  0.01810571453131894
+701 701  0.01765926251097409
+237 702  0.01774120600256697
+688 702  0.008865841466589953
+702 702  0.02692143333247395
+145 703  0.04826252211210828
+703 703  0.045533893684683
+96 704  0.03023901502527092
+318 704  0.003475455478190859
+704 704  0.04932222886487166
+96 705  0.05814496801316305
+705 705  0.04376812227074232
+706 706  0.09059946824940528
+707 706  0.02206309641910327
+471 707  0.001971734187777073
+492 707  0.01222749729421697
+493 707  9.943932093508234e-05
+706 707  9.943747939481412e-05
+707 707  0.02206309641910328
+726 707  8.901260673467379e-05
+471 708  0.008749316084421591
+478 708  0.00385403264175548
+708 708  0.02183021431262304
+709 708  0.01588296265025516
+412 709  0.0001372999532777806
+478 709  0.002915407954387878
+484 709  0.01809121810037291
+708 709  0.01595480699813724
+709 709  0.03858749505769376
+35 710  0.03164543848459934
+475 710  4.077470760506248e-05
+553 710  8.822741065009899e-06
+710 710  0.03279808035444827
+36 711  0.04140786749482465
+711 711  0.05122840648749439
+724 711  3.257546437679202e-05
+734 711  3.334112418002822e-05
+712 712  0.1036341168626075
+714 712  0.02659885386924947
+713 713  0.102451101173955
+714 713  0.02659885386924947
+362 714  0.0110113325331031
+401 714  0.006063475711334444
+479 714  0.001862046232769363
+712 714  0.01093414680469787
+713 714  0.01095108653395264
+714 714  0.02659885386924947
+715 714  0.01295964108648662
+734 714  0.006067940674193537
+360 715  0.03986999193031365
+714 715  0.02013516471650269
+715 715  0.05282962784979239
+396 716  0.0001525314567501726
+397 716  0.05687887705929573
+406 716  7.515059705383282e-05
+716 716  0.06230139190235773
+717 716  0.05956303589124799
+396 717  0.0001607844820818761
+397 717  0.05680250246813857
+406 717  7.921688284150947e-05
+716 717  0.05956303589124798
+717 717  0.06233878377404831
+718 718  0.07592753070763514
+724 718  0.02428091280675981
+719 719  0.07592753070763514
+724 719  0.02428091280675981
+720 720  0.07592753070763514
+724 720  0.02428091280675981
+721 721  0.07592753070763514
+724 721  0.02428091280675981
+722 722  0.07592753070763514
+724 722  0.02428091280675981
+723 723  0.07592753070763514
+724 723  0.02428091280675981
+8 724  0.00729895583966801
+412 724  0.0007568832425986043
+475 724  0.002321034707383153
+711 724  0.004416349974273951
+718 724  0.007027539733533965
+719 724  0.007027539733533965
+720 724  0.007027539733533965
+721 724  0.007027539733533965
+722 724  0.007027539733533965
+723 724  0.007027539733533965
+724 724  0.02428091280675981
+725 724  0.002143533457852622
+734 724  0.001029094237052859
+100 725  0.4420009004047503
+724 725  0.001451380251119414
+725 725  0.1485693521752023
+479 726  0.002518552400552489
+707 726  0.01960835004028389
+726 726  0.02226920500663214
+727 727  0.04992612389972566
+728 727  0.008628578138534657
+731 727  0.007934854346086633
+727 728  0.008628578138534659
+728 728  0.04992612389972566
+731 728  0.007934854346086633
+729 729  0.05027930639610439
+730 729  0.006884220921467736
+731 729  0.00793485578510947
+729 730  0.006884220921467736
+730 730  0.05010714813185277
+731 730  0.007934855788114785
+410 731  0.000226631396720429
+486 731  0.0002811707081937405
+727 731  0.005847356851518764
+728 731  0.005847356851518764
+729 731  0.005898732982906709
+730 731  0.005898732985140845
+731 731  0.007934856577050509
+100 732  0.4701009138396891
+732 732  0.4481614073622071
+481 733  0.0489007983645527
+484 733  0.04936253445714679
+733 733  0.1040838584588347
+491 734  0.001902048294567494
+711 734  0.01491036806433065
+714 734  0.01429300183540573
+724 734  0.003394614054289708
+734 734  0.02590216027347546
+735 735  0.282986614115017
+743 735  0.03133287048374728
+736 736  0.2084587990179075
+742 736  0.02359174876849497
+737 737  0.1142704624700489
+743 737  0.03133287048374727
+738 738  0.05350832800144269
+742 738  0.02359174876849497
+739 739  0.1587735744516123
+744 739  0.01050801099509594
+135 740  0.006999705600738508
+740 740  0.0122428008299788
+741 740  5.084327844827839e-05
+742 740  2.195292780969384e-05
+744 740  8.37443760462877e-05
+740 741  0.00536070452350935
+741 741  0.01769844564454164
+749 741  0.02757154403341738
+759 741  0.008167400845581754
+132 742  0.002449442968067719
+736 742  0.01085880897694698
+738 742  0.01390833544624129
+740 742  0.001186602214348167
+742 742  0.02359174876849497
+743 742  0.01161345848853242
+143 743  0.001685886278921233
+161 743  0.009654893296614786
+222 743  0.003801822966059999
+735 743  0.0162865921656052
+737 743  0.01827049319003907
+742 743  0.01303947249535167
+743 743  0.03133287048374727
+746 743  0.007456783090147935
+739 744  0.008773581951611889
+740 744  0.00818368763764968
+744 744  0.01050801099509594
+756 744  0.00624219593045746
+143 745  0.05251031125856063
+745 745  0.0001997690630162436
+746 745  5.443332446614393e-05
+743 746  0.01477283492930067
+745 746  0.0001055818613217538
+746 746  0.1201887575476681
+747 747  0.1137999985660004
+748 747  0.006497952592681879
+758 747  0.06429999807100012
+747 748  0.04950000049500017
+748 748  0.03472892247216531
+757 748  0.01095125431616056
+741 749  0.004869002663980502
+749 749  0.06077146545370619
+753 749  0.03319995803525303
+750 750  0.1760001371201187
+751 750  0.03709999985901999
+759 750  0.05615115347388746
+750 751  0.1760001371201187
+751 751  0.14580000003294
+752 752  0.08853550051648364
+757 752  0.01388901769340084
+762 752  0.09800003488801237
+749 753  0.06077146545370619
+753 753  0.04579995793445303
+754 754  0.04067547190252845
+757 754  0.008107751683430028
+765 754  0.02349999530000094
+172 755  0.0054127681906925
+755 755  0.1236106235125528
+764 755  0.1062000660564412
+744 756  0.00572802183530221
+756 756  0.02259272477553183
+757 756  0.008816608457747651
+748 757  0.006961940094299847
+752 757  0.008964694168437832
+754 757  0.01717548063876465
+756 757  0.006236111437314269
+757 757  0.01938184701185934
+747 758  0.1137999985660004
+758 758  0.07909999753820013
+741 759  0.003730752082826072
+750 759  0.1389001269547093
+759 759  0.07531589289586132
+760 760  0.2081461918044692
+761 760  0.1097553914168045
+137 761  0.007653563226806053
+760 761  0.04784618713973914
+761 761  0.1097553914168045
+766 761  0.03100293782511568
+767 761  0.05225039853991487
+752 762  0.04193588603608223
+762 762  0.3458000662099363
+763 762  0.247800031321924
+762 763  0.3458000662099363
+763 763  0.4708000279769241
+755 764  0.1236106235125528
+764 764  0.1462000660564412
+754 765  0.04067547190252845
+765 765  0.04039999462400097
+761 766  0.07358093914497393
+766 766  0.1778670202028769
+768 766  0.09229249471260238
+761 767  0.1097553914168045
+767 767  0.1772503985399149
+766 768  0.04659723444191144
+768 768  0.2800209153583046
+769 768  0.1710364191267975
+772 768  0.1867106290410218
+768 769  0.2800209153583046
+769 769  0.5010364224267975
+523 770  9.022817695390103e-06
+770 770  0.2758305566139244
+823 770  0.2717391304347622
+771 771  0.4970000468600144
+772 771  0.4707107434108653
+768 772  0.1200089783317938
+771 772  0.2840000613440133
+772 772  0.4707107434108653
+568 773  0.007216515414597397
+773 773  0.03860036777063307
+774 773  0.03050467172946603
+795 773  0.003078064250781432
+801 773  0.02752810323413095
+802 773  0.01055000487474257
+773 774  0.03860036777063306
+774 774  0.1393046773000263
+775 775  0.02475995563029326
+776 775  0.03946389099049266
+781 775  0.008931098939211742
+571 776  0.007463498083336377
+581 776  0.002860098888363572
+775 776  0.01392562505952267
+776 776  0.04186788389119318
+781 776  0.006429672691714456
+796 776  0.0005027490914261444
+797 776  0.0003611741410102067
+1127 776  0.008813281121105767
+777 777  0.05035968024085257
+793 777  3.205968249284482e-05
+795 777  0.003314333795221247
+800 777  0.02124147823020587
+778 778  0.07615170046455932
+779 778  0.05249478425065181
+523 779  0.005412151467848236
+778 779  0.03980169204953238
+779 779  0.05249478425065182
+780 779  0.001664032361912951
+787 779  0.01174434114204541
+530 780  0.00603495599069889
+578 780  0.007002957890466868
+583 780  0.01154369993991829
+779 780  0.003450925956451911
+780 780  0.02471680581339951
+797 780  0.0008544879407238196
+949 780  0.01765563128606545
+508 781  0.01146360041979076
+775 781  0.008476554867154679
+776 781  0.01729372494402667
+781 781  0.03199980571972852
+579 782  0.001242824328283301
+600 782  0.0266921211252419
+782 782  0.01299225430463768
+783 782  0.002497153980881631
+572 783  0.04855707787389923
+579 783  0.004065955224199574
+782 783  0.005479659480382699
+783 783  0.05218409839323125
+576 784  0.02248353062551777
+784 784  0.04900984729621592
+802 784  0.01781250393151464
+569 785  0.01539322374897347
+785 785  0.01630843556546118
+786 785  0.00244909751411912
+788 785  0.009024882712692419
+796 785  0.002172894144601377
+797 785  0.001883805647797623
+802 785  0.001631591515025325
+577 786  0.001820374019788474
+785 786  0.0003355912309700669
+786 786  0.007810535605601336
+1128 786  0.001256443987098833
+779 787  0.0278694280628256
+787 787  0.06193981259668287
+949 787  0.03062649908765396
+785 788  0.001423850369810863
+788 788  0.04366344866518873
+789 788  0.003244330702822663
+796 788  0.001417989115834743
+1132 788  0.03456571633990545
+788 789  0.02026741147010662
+789 789  0.2227504892140893
+1094 789  0.1548669885689344
+514 790  0.008049377898348555
+790 790  0.04021974288364626
+791 790  0.02849311321453606
+795 790  0.005888485352821266
+790 791  0.04021974288364626
+791 791  0.05889311029613634
+792 792  0.1060696407972351
+799 792  0.02136027936016361
+1110 792  0.06618131916828844
+777 793  1.96640627202086e-05
+793 793  0.000199737723037119
+794 793  1.966406272020861e-05
+825 793  0.1278772378516793
+793 794  3.205968249284482e-05
+794 794  0.05035968024085258
+795 794  0.003314333795221247
+800 794  0.02124147823020587
+514 795  0.003908683502678743
+568 795  0.0183075963632282
+773 795  0.003952548376903416
+777 795  0.002283122643552819
+790 795  0.008587741971265909
+794 795  0.002283122643552819
+795 795  0.01391925619636464
+573 796  0.01619183306608712
+581 796  0.004965198315835116
+630 796  0.009912532758592284
+776 796  0.001367608629910389
+785 796  0.001370086204695089
+788 796  0.005667081198775904
+796 796  0.02708953246662873
+799 796  0.01044035674673248
+530 797  0.001481789283049012
+581 797  0.002910354294163291
+776 797  0.001407640288568322
+780 797  0.0006392119489119368
+785 797  0.001671815930448331
+797 797  0.01455474950347107
+798 797  0.005444037293553388
+803 797  0.002866477861579545
+805 797  0.01129413035828043
+1119 797  0.01216965786219506
+797 798  0.00648105243726644
+798 798  0.1213169982646069
+1114 798  0.1191860529109802
+581 799  0.009174938259299364
+792 799  0.02850771979805629
+796 799  0.01569177080538385
+799 799  0.03824900208037592
+540 800  0.007439342985845224
+777 800  0.01453102841056699
+794 800  0.01453102841056699
+800 800  0.03532124833562421
+773 801  0.03860036777063306
+801 801  0.2046281153300618
+523 802  9.988409646243638e-06
+577 802  2.727871631739158e-06
+773 802  5.568289864440538e-05
+784 802  7.111133649763519e-05
+785 802  7.371817460457246e-06
+802 802  0.03055429622450982
+1137 802  0.02144542140253185
+530 803  0.01387652166513078
+583 803  0.008982169690104893
+797 803  0.001577656853569151
+803 803  0.03468004196715042
+804 803  0.004085715700539266
+803 804  0.03468004196715042
+804 804  0.02608571834053958
+797 805  0.002008366969679479
+805 805  0.06439410099711014
+1138 805  0.05309997663601025
+806 806  0.207954249930627
+821 806  0.01349798555051323
+807 807  0.2834670560549181
+887 807  0.05157441519278387
+808 808  0.07936053260516683
+897 808  0.01853138087461078
+528 809  0.002673045141394566
+538 809  0.002090950387613578
+809 809  0.01432627788588647
+920 809  0.0111156191121955
+75 810  0.002786154243605049
+810 810  0.0771960043327003
+811 810  0.07512290106614418
+918 810  0.00149192614687986
+810 811  0.0771960043327003
+811 811  0.1131229003061442
+812 812  0.004265268361812286
+909 812  0.009867355183892385
+813 813  0.01954027070213277
+885 813  0.0002534121778379453
+940 813  0.0002653496560293624
+1122 813  0.02067157886613462
+814 814  0.02830913006385971
+815 814  0.02726288784865825
+909 814  0.001532553570697825
+918 814  0.0006148682080056179
+814 815  0.02830913006385971
+815 815  0.03616288393265997
+816 816  0.0205079850325633
+817 816  0.02592216973460077
+927 816  0.002526022369244229
+816 817  0.006986669161115896
+817 817  0.03700904488470814
+818 817  0.01147516453090905
+907 817  0.007306804789957272
+817 818  0.03700904488470813
+818 818  0.03047516415090905
+499 819  0.001633853262516083
+528 819  0.002375281559655103
+819 819  0.01243302892843611
+820 819  0.009258016284850647
+819 820  0.01243302892843611
+820 820  0.01355801701585077
+806 821  0.006954234654625862
+821 821  0.01349798555051323
+888 821  0.005213022865744636
+897 821  0.01431746242031418
+921 821  0.000428045644658355
+822 822  0.02743114645547259
+880 822  0.01784563242422923
+885 822  0.003519434405543951
+935 822  0.004741858572388725
+770 823  0.2758305566139244
+823 823  0.2718391304347622
+824 824  0.0002
+825 824  0.1279772378516793
+793 825  9.986886151855952e-05
+824 825  0.0001
+825 825  0.1279772378516793
+143 826  0.05253409842956077
+826 826  0.0001998543125579779
+144 827  0.05984537169173331
+827 827  0.05923660555884373
+828 828  0.140745281895787
+837 828  0.1087872108014386
+838 828  0.11131169334795
+874 828  0.02298795218084524
+140 829  0.01678060669874037
+829 829  0.05892250351958103
+833 829  0.01675812161640692
+839 829  0.03519316295384665
+140 830  0.01350701571735669
+830 830  0.0626056986503162
+833 830  0.02221119696364045
+840 830  0.03817050159970662
+831 831  0.09305093509008874
+841 831  0.0887732863426738
+869 831  0.01202349830062934
+832 832  0.05913984646017876
+842 832  0.05473001410392463
+869 832  0.01693877374903241
+829 833  0.02300743176882817
+830 833  0.03035354762947573
+833 833  0.06536502674121955
+843 833  0.04558641451909066
+834 834  0.1207221329635831
+844 834  0.01784999839528515
+874 834  0.05326836003683566
+835 835  0.1964786383859093
+845 835  0.0103799999655384
+862 835  0.2318995796605594
+836 836  0.1645374755238136
+846 836  0.1603498963017221
+869 836  0.01100699591087288
+828 837  0.140745281895787
+837 837  0.4029089236604105
+828 838  0.05428581750927225
+838 838  0.3068234234587666
+859 838  0.1931462814608735
+860 838  0.0890078097400584
+829 839  0.05892250351958103
+839 839  0.1377102653694572
+830 840  0.06260569865031622
+840 840  0.1427752113849228
+831 841  0.09305093509008876
+841 841  0.1858420993949347
+832 842  0.05913984646017875
+842 842  0.2071890254987111
+833 843  0.06536502674121954
+843 843  0.3655864145190906
+834 844  0.1207221329635831
+844 844  0.1197220397765233
+835 845  0.1964786383859094
+845 845  0.6253800983655542
+836 846  0.1645374755238136
+846 846  0.3072780402113742
+847 847  0.2786824971151751
+848 847  0.2155859923608576
+847 848  0.1270749109141337
+848 848  0.2155859923608576
+873 848  0.04308522320295062
+849 849  0.2519376152696942
+854 849  0.2125924910206246
+863 849  0.03136559425336743
+864 849  0.06611785644859572
+850 850  0.4578375436506227
+855 850  0.3977900376667386
+873 850  0.02207303230236381
+851 851  1.339820067239498
+856 851  0.8617599205802059
+858 851  0.2524202499316482
+852 852  0.8044515099515441
+857 852  0.8003495927020922
+869 852  0.00977792134478263
+853 853  0.1997879065771989
+876 853  0.02846993447409274
+849 854  0.2519376152696942
+854 854  0.5300727643838391
+850 855  0.4578375436506226
+855 855  0.551494168895637
+851 856  1.339820067239498
+856 856  2.103759888288207
+852 857  0.8044515099515441
+857 857  1.391349724495121
+851 858  0.4780601466592918
+858 858  0.4275616832566078
+873 858  0.05093981980382919
+838 859  0.3068234234587666
+859 859  0.3431462739608739
+838 860  0.1314178953496573
+860 860  1.730707208662083
+861 860  1.641699428146838
+860 861  1.730707208662083
+861 861  2.871699473656839
+835 862  0.1860986461471867
+862 862  0.2358136012299892
+863 862  0.005533066463074931
+849 863  0.05606416253102824
+862 863  0.1198884856378247
+863 863  0.1047338399469532
+869 863  0.006193732174918723
+849 864  0.02262491299310699
+864 864  0.2207916982974485
+865 864  0.1898203160887905
+869 864  0.002563392832267098
+864 865  0.2207916982974485
+865 865  0.237620320868791
+866 866  0.01875702130669307
+869 866  0.02150182985676903
+877 866  0.001007776018379551
+867 867  0.2115002250448753
+878 867  0.02592199514873633
+868 868  0.01744672745892444
+869 868  0.01959989422406455
+871 868  0.0068615973335874
+877 868  0.0009167252276101201
+831 869  0.004277652544827698
+832 869  0.004409832356254128
+836 869  0.004187581908004852
+852 869  0.004101920532422533
+863 869  0.001215503664746272
+864 869  0.002627741759233773
+866 869  0.005137420205803314
+868 869  0.004843553303722979
+869 869  0.02301948565780342
+870 869  0.004817232857402497
+869 870  0.01956887879942859
+870 870  0.01758799859188702
+871 870  0.006963658849763308
+877 870  0.0009171799280344383
+868 871  0.008711000927404886
+870 871  0.008806465925631649
+871 871  0.07746257464992416
+872 872  0.2955921648824598
+882 872  0.01085731727556362
+848 873  0.0885111376844486
+850 873  0.06004750598388407
+858 873  0.04666098678618785
+873 873  0.07919857786041065
+882 873  0.003790852019042492
+828 874  0.009899123140458117
+834 874  0.1028721419133682
+874 874  0.06260203573068722
+878 874  0.00200964117441003
+875 875  0.02610119176195234
+876 875  0.02406994346643991
+882 875  0.00167798373137963
+853 876  0.004399999056640202
+875 876  0.02552634753594662
+876 876  0.02846993447409274
+138 877  0.003970322372452619
+866 877  0.000616616334862829
+868 877  0.0005801374163867748
+870 877  0.0005781860057615934
+877 877  0.01528572934070789
+878 877  0.009795278982989467
+883 877  0.007776205169885589
+867 878  0.0167931044691808
+874 878  0.004932776594236208
+877 878  0.01286758974603248
+878 878  0.02592199514873632
+879 879  0.08062506292350546
+882 879  0.01085731727556362
+822 880  0.02743114645547259
+880 880  0.03074563239842923
+881 881  0.06942598125955839
+882 881  0.01085731727556362
+138 882  0.005884359642788956
+872 882  0.004852738930012023
+873 882  0.002529834727550404
+875 882  0.005175855710690167
+879 882  0.005095049939896242
+881 882  0.005155985578502097
+882 882  0.01085731727556362
+877 883  0.01528572934070789
+883 883  0.07228621615594047
+884 884  0.01219073310470677
+937 884  0.004303038652622925
+1123 884  0.008750007704381021
+535 885  0.007233055798534703
+813 885  0.003244327383216069
+822 885  0.01048696357442695
+885 885  0.01833154993187189
+914 885  0.001804879929133101
+918 885  0.000488965400960133
+886 886  0.03069892868008986
+895 886  0.01719693345925267
+927 886  0.001698690413162683
+937 886  0.003524474624462636
+807 887  0.03746707721091631
+887 887  0.05157441519278387
+896 887  0.01671919608471206
+897 887  0.01558270319732567
+821 888  0.008886870264124687
+888 888  0.03412437670656033
+924 888  0.01139155529348311
+889 889  0.006273538653384737
+917 889  0.007669741488049601
+890 890  0.03010860349890948
+891 890  0.02888211763686513
+908 890  0.00100399961211994
+917 890  0.001995384105343122
+890 891  0.03010860349890949
+891 891  0.04688211619686524
+892 892  0.0135442301691227
+893 892  0.009337962721732261
+899 892  0.003924408800834282
+927 892  0.001772475961012305
+892 893  0.0135442301691227
+893 893  0.02203796231533228
+894 894  0.03109700724541354
+921 894  2.582112257562263e-05
+936 894  3.076147148241834e-05
+1126 894  0.02687449610320036
+579 895  0.0006121947765618673
+886 895  0.0008377126422259026
+895 895  0.0445774089228085
+1125 895  0.03176872369152559
+887 896  0.02022441085999184
+896 896  0.02908660993572379
+897 896  0.01756854754445944
+808 897  0.004360513855162132
+821 897  0.006393579454644784
+887 897  0.005024413380044375
+896 897  0.004682932898399041
+897 897  0.01853138087461078
+913 897  0.004214060454520724
+917 897  0.0001246309721587329
+944 897  0.001291008453868203
+898 898  0.01390583537581733
+914 898  0.002339404868253041
+926 898  0.03301375113484432
+892 899  0.004423419621225885
+899 899  0.01072030581613085
+900 899  0.00580740597577429
+908 899  0.003736709993607725
+899 900  0.01072030581613085
+900 900  0.03350740830257448
+901 901  0.04025952661594935
+913 901  2.051856118270129e-05
+1129 901  0.05793036044128706
+902 902  0.007179126556803894
+906 902  0.008718922408816254
+556 903  0.009709993774629458
+903 903  0.04134997198816331
+987 903  0.01760391604673835
+904 904  0.008324641033037612
+905 904  0.004486544181019496
+939 904  0.001614068474841553
+944 904  0.001567099158187506
+904 905  0.00832464103303761
+905 905  0.008386543518019608
+902 906  0.004679126556803894
+906 906  0.008718922408816254
+922 906  0.0009008540972751597
+939 906  0.001285271122284525
+817 907  0.02506607113613655
+907 907  0.02583935001166341
+923 907  0.01025670433048331
+532 908  0.000292499155021064
+890 908  0.001802761665018293
+899 908  0.005558445265695928
+908 908  0.007472042486884206
+912 908  0.004241646413774165
+917 908  0.0004098903584962235
+927 908  0.0008411096511385652
+554 909  0.001161123334805374
+812 909  0.002265268361812286
+814 909  0.0006960590837098462
+909 909  0.009867355183892385
+910 909  0.001341614853501025
+909 910  0.009867355183892385
+910 910  0.006441615669501156
+911 911  0.02373877031515357
+912 911  0.0193501715378347
+532 912  0.0003638722938778599
+908 912  0.00170285213314922
+911 912  0.01613876757915258
+912 912  0.0193501715378347
+897 913  0.01543164539779126
+901 913  0.006734186126803873
+913 913  0.0189181233883837
+499 914  0.003073724976294847
+511 914  0.001299784730098261
+538 914  0.0004843898066251472
+885 914  0.0009543904130624683
+898 914  0.001677157446132573
+914 914  0.01284379458486224
+935 914  0.00336528949679306
+938 914  0.009775855279802575
+963 914  0.002440084069826731
+915 915  0.01887799841495944
+926 915  0.04036616462179661
+916 916  0.07175018072235473
+991 916  0.08332942064266868
+889 917  0.004373538691384737
+890 917  0.0009288340417040546
+897 917  9.679046066541613e-05
+908 917  0.0001062611591180948
+917 917  0.007669741488049602
+918 917  0.001087296062716872
+930 917  0.001177767946046547
+75 918  0.0004042110443881656
+532 918  8.617349612622082e-05
+810 918  0.002892375746077174
+814 918  0.001645278510677814
+885 918  0.0003812733587880181
+917 918  0.00399285953916823
+918 918  0.006772170020769837
+919 918  0.004045709292940789
+1130 918  0.004753753432982463
+918 919  0.005101115055405544
+919 919  0.01789744077783699
+940 919  0.002093954937998422
+809 920  0.01432627788588647
+920 920  0.02001561519619722
+527 921  0.0004790384345277779
+821 921  0.0006699763470850662
+894 921  0.006281809831012578
+921 921  0.01039280083437153
+923 921  0.005918681287578552
+924 921  0.007852331567483059
+927 921  0.0003791656025174885
+933 921  0.007942247375364437
+937 921  0.0004434451345446738
+906 922  1.532545376228809e-05
+922 922  0.03134760977659256
+930 922  1.964497769671266e-05
+1133 922  0.02956830277942088
+907 923  0.01163573038280261
+921 923  0.005712813591026454
+923 923  0.03178362780410424
+888 924  0.005044455744378779
+921 924  0.003787213122968066
+924 924  0.03334620142643027
+925 924  0.02447248338440741
+924 925  0.03334620142643027
+925 925  0.0501724806345077
+511 926  0.001786855673403372
+898 926  0.006429255755478865
+915 926  0.009177999966959196
+926 926  0.04036616462179662
+579 927  0.0004036607956703073
+816 927  0.003022111914234659
+886 927  0.001480757391267769
+892 927  0.003828100910598467
+908 927  0.00108344732297468
+921 927  0.0004091673134867418
+927 927  0.008432103183035408
+928 927  0.007761229553793364
+937 927  0.0005819387348275373
+927 928  0.0003033149954706448
+928 928  0.02858876260732213
+929 928  0.02198087224497245
+937 928  8.319842440449401e-05
+928 929  0.02858876260732213
+929 929  0.02298087234497246
+917 930  0.003020783387677176
+922 930  0.002737003750895941
+930 930  0.01218165554242968
+931 930  0.01121403728330975
+930 931  0.01218165554242968
+931 931  0.02161403686730977
+932 932  0.05544300641025474
+949 932  0.06528762072500392
+527 933  0.0003892861058939532
+921 933  0.001340716667282153
+933 933  0.0405505325267202
+934 933  0.01790551973456857
+942 933  0.006204209829354447
+933 934  0.0405505325267202
+934 934  0.02550552247056955
+822 935  0.008825044625412099
+914 935  0.003974990604581501
+935 935  0.0208356103168027
+947 935  0.01676870142002394
+894 936  3.798426637685468e-05
+936 936  0.02516388418998334
+937 936  2.572287432225631e-05
+1134 936  0.02213858755811324
+502 937  0.0009308510188288048
+884 937  0.006972812286093009
+886 937  0.002350373142658572
+921 937  0.0003563332503868298
+927 937  0.0004390825271683525
+928 937  0.001454238349828426
+936 937  0.004142765908838729
+937 937  0.007855074867766076
+914 938  0.01284379458486224
+938 938  0.02247585487340259
+904 939  0.004121910053910494
+906 939  0.004401785634565286
+939 939  0.004022831149886777
+813 940  0.00347990398411743
+919 940  0.002181675206045969
+940 940  0.008486027778783729
+954 940  0.02356803958395574
+941 941  0.01029728919443866
+954 941  0.03366802948354273
+933 942  0.04055053252672019
+942 942  0.04430420861015448
+943 943  0.03547542869756949
+944 943  0.02456126604654757
+897 944  0.001906494093816954
+904 944  0.002053343370467679
+943 944  0.0230754284991695
+944 944  0.02456126604654758
+945 945  0.3703205212490625
+961 945  0.07895487623482626
+946 946  0.1435483647575645
+965 946  0.07549099879624292
+935 947  0.0208356103168027
+947 947  0.04546870601202467
+538 948  0.002804809006850223
+948 948  0.01636669233558338
+952 948  0.00172207408614324
+955 948  0.003156881991610024
+957 948  0.003757570867133571
+958 948  0.008999295718421303
+964 948  0.006541884879626206
+780 949  0.003527688672448212
+787 949  0.005347845230501524
+932 949  0.04274300681665472
+949 949  0.06528762072500392
+950 950  0.04775829521461967
+951 950  0.005944919849030421
+955 950  0.01945547812334616
+956 950  0.02488946588201796
+950 951  0.00784997609134226
+951 951  0.07990794709529128
+965 951  0.02881505253248289
+969 951  0.03902640933381142
+975 951  0.05118966836875728
+948 952  0.002698388830761903
+952 952  0.01479698781172081
+960 952  0.01090739773548476
+538 953  0.003984543621843501
+953 953  0.040622894066532
+955 953  0.02192433237461329
+940 954  0.001590537130025061
+941 954  0.007797289194438665
+954 954  0.03366802948354273
+959 954  0.002003968137467726
+948 955  0.00257676859139966
+950 955  0.02471561834984847
+953 955  0.01320429647841619
+955 955  0.03215645693183091
+950 956  0.04775829521461966
+956 956  0.07763945467264532
+519 957  0.006617203945576319
+948 957  0.003373200286415561
+957 957  0.01085292652420506
+958 957  0.004146243831491631
+967 957  0.006984944464507416
+973 957  0.005441157826682218
+948 958  0.01198427246308062
+957 958  0.006150678702683945
+958 958  0.0262500424958768
+954 959  0.03366802948354273
+959 959  0.03620396355466834
+952 960  0.002078228296554667
+960 960  0.01306073009393931
+973 960  0.009100314148404377
+945 961  0.06942049919309083
+961 961  0.07895487623482628
+962 961  0.01107401732878868
+963 961  0.01240845552503707
+968 961  0.01278211235887942
+961 962  0.01113834918658851
+962 962  0.03239343287296422
+966 962  0.05558833395112791
+914 963  0.00475588551202848
+961 963  0.01455928390694178
+963 963  0.109594631764525
+972 963  0.0344279565194329
+538 964  0.007663509157317909
+948 964  0.01011227924199457
+964 964  0.02908047787220006
+946 965  0.04854840275754927
+951 965  0.02527183983034641
+965 965  0.07549099879624292
+968 965  0.006830486233631824
+969 965  0.03315505825299782
+962 966  0.008794989478961807
+966 966  0.06471830236682524
+971 966  0.02405427785548025
+519 967  0.02420168409037794
+957 967  0.009219474515477024
+967 967  0.03108168731236846
+465 968  0.0292007332579842
+961 968  0.006408805799786195
+965 968  0.005748762411683016
+968 968  0.03575835231461739
+951 969  0.04024834751929114
+965 969  0.03898717876164977
+969 969  0.08650732152412599
+970 969  0.05034520448082387
+969 970  0.08650732152412598
+970 970  0.206145210245424
+966 971  0.05802100395703609
+971 971  0.03301916393837059
+972 971  0.008000498722194598
+963 972  0.03263965785558136
+971 972  0.02318534091005487
+972 972  0.07731177900044259
+957 973  0.00178502070848479
+960 973  0.01091258272770016
+973 973  0.01058729410486097
+974 974  0.005628963244658351
+983 974  0.1086511317434904
+951 975  0.07990794709529131
+975 975  0.1894896681751373
+976 976  0.04614585742380604
+977 976  0.01717180389799948
+989 976  0.003108965668848982
+1001 976  0.007670095474669817
+976 977  0.04614585742380604
+977 977  0.03087180403499949
+978 978  0.1104675903104573
+989 978  0.03614724738382001
+558 979  0.0004365638331055172
+979 979  0.005648560866193328
+983 979  0.1063039902882975
+558 980  0.01402526600493269
+560 980  0.02483721564579974
+980 980  0.06713808799758358
+981 981  0.1469555797277405
+982 981  0.04529334233161062
+984 981  0.004942230026113261
+998 981  0.001645866478449362
+981 982  0.1469555797277405
+982 982  0.05049334212361063
+974 983  0.004628963144658341
+979 983  0.0009677247747013971
+983 983  0.1086511317434904
+986 983  0.0003454427303952864
+981 984  0.1469555797277405
+984 984  0.08794224911611766
+524 985  0.0194821966293754
+985 985  0.0177674274029123
+987 985  0.01442955339440103
+989 985  0.002613842741848019
+558 986  0.01234799869920265
+983 986  0.0422645733970262
+986 986  0.0545669678465453
+903 987  0.006150852892462986
+985 987  0.005754132954665552
+987 987  0.04366120217567673
+999 987  0.02823254127879861
+988 988  0.0703815354647411
+993 988  0.03730135475680615
+995 988  0.02554130579815113
+558 989  0.005963673325780709
+976 989  0.01064338744602779
+978 989  0.02706760649005415
+985 989  0.001928213920829602
+989 989  0.03614724738382001
+990 989  0.02324053681363782
+989 990  0.03614724738382002
+990 990  0.1925405523553793
+558 991  0.002836182056199043
+916 991  0.05955018174715463
+991 991  0.08332942064266867
+992 991  0.01043544731946699
+991 992  0.08332942064266868
+992 992  0.351435484488471
+517 993  0.02094209948697999
+988 993  0.02973511206453267
+993 993  0.0758972289969638
+994 993  0.03940774261258517
+993 994  0.07589722899696379
+994 994  0.287707756567046
+517 995  0.007091936783918074
+988 995  0.03186466058113482
+995 995  0.05022594702691818
+997 995  0.02717975237002637
+998 995  0.005974140230181695
+996 996  0.07392341485610768
+997 996  0.08974725993761802
+995 997  0.004585791838812838
+996 997  0.06122341526250766
+997 997  0.08974725993761802
+998 997  0.003811334241688877
+558 998  0.005222985023611815
+981 998  0.04725587006527477
+995 998  0.007828477265006325
+997 998  0.02960125181002267
+998 998  0.03517921881546875
+987 999  0.04366120217567673
+999 999  0.05353254236669866
+558 1000  0.02020927228120075
+1000 1000  0.06646071358731856
+524 1001  0.009024166385107798
+976 1001  0.03338083585573622
+1001 1001  0.02143906598486781
+1002 1002  0.1530376512051281
+1027 1002  0.04890312047371916
+1003 1003  0.2775329080501215
+1035 1003  0.0398555112954956
+1004 1004  0.3780323893372676
+1032 1004  0.1949847397260055
+1005 1005  0.1082314321005827
+1039 1005  0.1680475462134245
+1006 1006  0.01456908479846732
+1007 1006  0.008272877762390409
+1013 1006  0.0004742016329170211
+1019 1006  0.005274052141693458
+1021 1006  0.01259781893617213
+1030 1006  0.002790901981039412
+1035 1006  0.006870177131705836
+1006 1007  0.01456908479846732
+1007 1007  0.2492728541443928
+1008 1008  0.06478174580619263
+1023 1008  0.03498680201478815
+114 1009  0.009864303368119599
+1009 1009  0.04914030404925041
+1010 1009  9.908728126971404e-05
+1018 1009  9.928028189277891e-05
+1019 1009  3.924588614079684e-05
+1021 1009  1.198810312921487e-05
+1022 1009  9.963595870278421e-05
+1040 1009  9.910342467343425e-05
+1009 1010  0.04914030404925041
+1010 1010  0.6380991357692732
+1011 1011  0.1340527400490329
+1040 1011  0.3087982449657204
+1012 1012  0.1928267714401703
+1040 1012  0.3087982449657204
+1006 1013  0.0005505362745983842
+1013 1013  0.002088289199623778
+1023 1013  0.000704722254208448
+1029 1013  0.001133946644822423
+1030 1013  0.001069522852624345
+1037 1013  0.00158754249057476
+1038 1013  0.001548746405940372
+1039 1013  0.001347259410153927
+1014 1014  0.07411818248563802
+1039 1014  0.1680475462134245
+1015 1015  0.02899497950935656
+1029 1015  0.001867119870546095
+1016 1016  0.3242352619217244
+1024 1016  0.08739717026919569
+1034 1016  0.1018617498183449
+1017 1017  0.01364533734131167
+1029 1017  0.001867119870546095
+1009 1018  0.04914030404925041
+1018 1018  0.04729928179229283
+1006 1019  0.002534378931226484
+1009 1019  0.005135408450538448
+1019 1019  0.06290630625337562
+1033 1019  0.05959685114077312
+1020 1020  0.04540867316590868
+1021 1020  0.02850856165564928
+1006 1021  0.008521466014239722
+1009 1021  0.002208122412865323
+1020 1021  0.0168086691047081
+1021 1021  0.02850856165564928
+1009 1022  0.04914030404925041
+1022 1022  0.01759963508370283
+1008 1023  0.02383173839424129
+1013 1023  0.0004741401945795187
+1023 1023  0.03498680201478815
+1024 1023  0.01301110531223596
+1025 1023  0.01310171325372753
+1027 1023  0.009255085251495557
+1016 1024  0.08012700982029282
+1023 1024  0.02216859829078505
+1024 1024  0.12385292413206
+1034 1024  0.04805462776246333
+1023 1025  0.02182076347541613
+1025 1025  0.1182796277398342
+1028 1025  0.06436771072073712
+224 1026  0.02784781750176332
+1026 1026  0.0856222440183788
+1027 1026  0.02785210075284012
+1002 1027  0.03283765764784778
+1023 1027  0.01133624863257827
+1026 1027  0.02123398773966306
+1027 1027  0.04890312047371916
+1028 1027  0.01487890511457545
+1035 1027  0.005038909520881754
+1025 1028  0.06719660013050638
+1027 1028  0.02112044787749787
+1028 1028  0.1710477574258891
+109 1029  0.001640197973334421
+1013 1029  8.984131584140556e-05
+1015 1029  9.498274615620217e-05
+1017 1029  9.533770716166349e-05
+1029 1029  0.001867119870546095
+1006 1030  0.004028544329405392
+1013 1030  0.001329752259689344
+1030 1030  0.02291734456706735
+1031 1030  0.02132198753418528
+1030 1031  0.02291734456706735
+1031 1031  0.1281219747609068
+1004 1032  0.06143241973086459
+1032 1032  0.1949847397260054
+1034 1032  0.04755648988733892
+1036 1032  0.06621961895120788
+1019 1033  0.06290630625337562
+1033 1033  0.08259685160077312
+1016 1034  0.08087820711463256
+1024 1034  0.04161731290577602
+1032 1034  0.1178978918302327
+1034 1034  0.1694309738728105
+1003 1035  0.03273288885779993
+1006 1035  0.00906555707875277
+1027 1035  0.006921300747113141
+1035 1035  0.03985551129549559
+1036 1035  0.01406671708523453
+1032 1036  0.1414436236506199
+1035 1036  0.01094398259724915
+1036 1036  0.1189266545286848
+1013 1037  0.002088289199623779
+1037 1037  0.01003754215257477
+1013 1038  0.002088289199623778
+1038 1038  0.01129874664969038
+1005 1039  0.01223146282057288
+1013 1039  0.0001359099065269315
+1014 1039  0.06091818322483799
+1039 1039  0.1680475462134245
+1009 1040  0.003166993616150146
+1011 1040  0.1097527380564327
+1012 1040  0.02252675389926848
+1040 1040  0.3087982449657204
+647 1041  0.00211815186033333
+677 1041  0.006352832047296143
+1041 1041  0.01280489855845344
+648 1042  0.1346148415060412
+1042 1042  0.05959999020880281
+676 1043  0.02099220839108384
+1043 1043  0.0235456119389732
+1050 1043  0.001061593669875734
+1044 1044  0.01953284291359033
+1050 1044  0.004808909708187654
+1053 1044  0.0009333318458468682
+651 1045  0.01135347889926522
+1045 1045  0.02852471223568966
+1047 1045  0.0136828836464119
+674 1046  0.006078327173654243
+1046 1046  0.01539853994035298
+1058 1046  0.004627828397646399
+532 1047  0.0003353970026791279
+680 1047  0.006468049092205441
+1045 1047  0.01306116223849304
+1047 1047  0.01951864936678242
+653 1048  0.06747685422597234
+1048 1048  0.1410516175449961
+672 1049  0.002035482083724407
+1049 1049  0.01620981673588805
+1054 1049  0.001228456864079547
+77 1050  0.002818165291851867
+660 1050  8.921216502885686e-05
+672 1050  2.199635620334956e-05
+675 1050  0.0001212842738146832
+1043 1050  0.0001150546681760679
+1044 1050  0.0001242421068177792
+1050 1050  0.007333160652441726
+1051 1050  7.318527869573656e-05
+1052 1050  0.0001789413638665126
+646 1051  0.01379407300037537
+667 1051  0.07654661478337116
+1050 1051  0.001860352432717382
+1051 1051  0.08284242548864497
+1056 1051  0.008845188484153853
+672 1052  0.0003031399703474054
+1050 1052  0.006750424310811922
+1052 1052  0.004755973049646703
+681 1053  0.009893731009039716
+1044 1053  0.0003078730259311737
+1053 1053  0.002711408796219971
+652 1054  0.003513028472106132
+1049 1054  0.001180326255607597
+1054 1054  0.003105538986180574
+649 1055  0.002379546354139961
+657 1055  0.003208034688420189
+1055 1055  0.006369699498349443
+1056 1055  0.0007005113099094189
+1058 1055  0.002463583646116846
+664 1056  0.07747445087143598
+1051 1056  0.008009619941229586
+1055 1056  0.0007031089868624823
+1056 1056  0.04550319147949711
+654 1057  0.009028005369760611
+680 1057  0.00422520761440772
+1057 1057  0.01759772702108079
+660 1058  0.001025992365644372
+669 1058  0.004061202683427989
+672 1058  0.0001255217263496964
+1046 1058  0.001538686487980197
+1055 1058  0.001272974247058685
+1058 1058  0.006195453305791682
+677 1059  0.006710970770683924
+1059 1059  0.00540176796751563
+1061 1059  0.003722756958330339
+645 1060  0.09630003872258575
+1060 1060  0.2217000229688926
+621 1061  0.001472716437497991
+1059 1061  0.004972488621553087
+1061 1061  0.02115615872712003
+1062 1062  0.2341655735855472
+1072 1062  0.1162877909424825
+1085 1062  0.06285934904949669
+70 1063  0.008835416720388525
+1063 1063  0.06879257815797121
+1074 1063  0.01257019301063638
+1064 1064  0.09558121278142065
+1075 1064  0.06872176597681982
+1087 1064  0.08699974274180078
+70 1065  0.007987341514557739
+1065 1065  0.03683082955771718
+1077 1065  0.01738783694805822
+1080 1065  0.02885701993604127
+1066 1066  0.08354112319531962
+1068 1066  0.01754317028550596
+1067 1067  0.6093000007470154
+1071 1067  0.3191999064789874
+527 1068  0.004002944190316053
+584 1068  0.001971461686100167
+609 1068  0.005233474827431618
+1066 1068  0.01214111177131779
+1068 1068  0.01754317028550596
+1069 1068  0.007746642417860882
+1092 1068  0.004772379710180124
+1068 1069  0.00767112910888773
+1069 1069  0.03449046817851736
+1086 1069  0.01735646029827682
+1070 1070  0.04596477571514566
+1077 1070  0.02834363723623809
+1113 1070  0.006517913137856713
+1067 1071  0.2325999534334893
+1071 1071  0.3191999064789874
+1088 1071  0.05346508689448282
+1062 1072  0.1402982214385716
+1072 1072  0.1766767678621581
+1075 1072  0.03780296798847921
+527 1073  0.00440645267392877
+1073 1073  0.01038692848789122
+1074 1073  0.01613453602886953
+70 1074  0.0005668180592062543
+527 1074  0.003184584054973412
+1063 1074  0.002271271139130907
+1073 1074  0.005649908198300721
+1074 1074  0.01795532976096158
+1075 1074  0.00150624221967368
+1081 1074  0.004345906141204313
+1086 1074  0.002124743680218939
+1064 1075  0.0438614934803819
+1072 1075  0.02973684838987709
+1074 1075  0.003613116021377893
+1075 1075  0.0943049705701732
+1082 1075  0.04819177008901647
+1076 1076  0.06928100909223454
+1077 1076  0.03712459534923577
+1065 1077  0.006682988407393145
+1070 1077  0.01181738248805265
+1076 1077  0.01578101256973432
+1077 1077  0.03712459534923578
+1078 1077  0.01138352888222821
+1080 1077  0.008012332589582512
+1088 1077  0.01216920620468427
+1095 1077  0.001405596490798533
+1077 1078  0.01134203691997565
+1078 1078  0.09920350325368112
+1084 1078  0.07602758889145694
+1085 1078  0.02229766610317457
+1079 1079  0.5569999767705016
+1117 1079  0.4062000099527207
+1065 1080  0.03013617731697666
+1077 1080  0.02177060522800103
+1080 1080  0.04803317226533439
+1074 1081  0.01795532976096158
+1081 1081  0.01604590552110435
+1075 1082  0.09430497057017322
+1082 1082  0.2808917483781085
+1083 1083  0.2356994624912304
+1087 1083  0.320699001915097
+1078 1084  0.09920350325368112
+1084 1084  0.1326276033810607
+1062 1085  0.1093208345195239
+1078 1085  0.05289007811225765
+1085 1085  0.1839042021651781
+1069 1086  0.01937193492239081
+1074 1086  0.007870529582573086
+1086 1086  0.03084341347749183
+1064 1087  0.0008110412166272464
+1083 1087  0.2336994624912304
+1087 1087  0.3206990019150971
+1071 1088  0.08659997318864784
+1077 1088  0.02705256246998841
+1088 1088  0.08647804876668065
+1089 1089  0.175385221436495
+1106 1089  0.3345991612267034
+1112 1089  0.05932034722361794
+577 1090  0.002950053428144085
+1090 1090  0.009394280978755781
+1091 1090  0.004722190797352253
+1090 1091  0.00301704566221149
+1091 1091  0.1241261391882426
+1093 1091  0.05290591248431776
+1108 1091  0.09624717815304551
+603 1092  0.03228744752564772
+1068 1092  0.004605546273785277
+1092 1092  0.04806122677715817
+1091 1093  0.04706724122373802
+1093 1093  0.1365035818923146
+1098 1093  0.1760001983522047
+636 1094  0.08973768722904406
+789 1094  0.1175033656420125
+1094 1094  0.2890245363533768
+65 1095  0.001594004151586668
+1077 1095  0.001285236114547839
+1095 1095  0.02442936296644875
+1096 1095  0.005457825659281279
+1100 1095  0.005428194353509556
+1105 1095  0.008323146412997286
+1113 1095  0.0177243412163399
+1095 1096  0.006571476261938634
+1096 1096  0.1676258499647094
+1103 1096  0.1120325288688221
+1107 1096  0.07057737800241455
+1112 1096  0.03152811618948742
+1097 1097  0.09398628385474324
+1100 1097  0.100828987509327
+1093 1098  0.08006460919351722
+1098 1098  0.2835002660235161
+1099 1098  0.1075000298312583
+1098 1099  0.2835002660235161
+1099 1099  0.2600000309750083
+1095 1100  0.001616899010265263
+1097 1100  0.08388628375374325
+1100 1100  0.1008289875093271
+1101 1100  0.004294985380093464
+608 1101  0.03965791615114651
+1100 1101  0.03540650853845503
+1101 1101  0.1040558554143926
+1120 1101  0.08482403139628016
+1102 1102  0.4157234555618753
+1112 1102  0.242425508064694
+1096 1103  0.1676258499647094
+1103 1103  0.2439325312562122
+1104 1104  0.06082765961706203
+1118 1104  0.04340000876680176
+1119 1104  0.01303685650800481
+599 1105  0.06160001227072245
+1095 1105  0.01780591591317898
+1105 1105  0.06992316227266117
+1089 1106  0.1062263922626756
+1106 1106  0.442999068132983
+1109 1106  0.1083999794473639
+1096 1107  0.1676258499647094
+1107 1107  0.4981774214465789
+1091 1108  0.1241261391882425
+1108 1108  0.3304471685508459
+1106 1109  0.442999068132983
+1109 1109  0.2748999796138639
+792 1110  0.07996731119167409
+1110 1110  0.1498851900458026
+1114 1110  0.05287448270566146
+1111 1111  0.1491908478242933
+1119 1111  0.02613407252659092
+1089 1112  0.05425915809165077
+1096 1112  0.03725181995953954
+1102 1112  0.1947234615288752
+1112 1112  0.242425508064694
+1070 1113  0.003187338465558733
+1095 1113  0.02273535552515969
+1113 1113  0.02755674135707064
+798 1114  0.04773250616985211
+1110 1114  0.02582091659767817
+1114 1114  0.2148611000891958
+1115 1114  0.09198220549045465
+1116 1114  0.07683700765342699
+1114 1115  0.2148611000891958
+1115 1115  0.3200822100068347
+1114 1116  0.2148611000891957
+1116 1116  0.5230369016363323
+1079 1117  0.22449999921425
+1117 1117  0.4062000099527207
+1120 1117  0.1199664810159885
+1104 1118  0.06082765961706205
+1118 1118  0.08660000531080202
+797 1119  0.009049035677297361
+1104 1119  0.01742765085026024
+1111 1119  0.01369085148279315
+1119 1119  0.02613407252659092
+1101 1120  0.05750994309795388
+1117 1120  0.1817000107384707
+1120 1120  0.2009664109761925
+670 1121  0.007632021197925194
+672 1121  0.002225025024112705
+1121 1121  0.01007956103754005
+813 1122  0.01900208511800962
+1122 1122  0.02164773439817868
+1136 1122  0.001299999317500262
+617 1123  0.01060000366760126
+884 1123  0.007911867973441501
+1123 1123  0.0193500141544857
+527 1124  0.007766460318378678
+1124 1124  0.03560130775451262
+1135 1124  0.01237345993813856
+895 1125  0.0445774089228085
+1125 1125  0.03396872347152561
+894 1126  0.03109700724541354
+1126 1126  0.02697449610320036
+776 1127  0.04186788389119319
+1127 1127  0.04011328287390587
+786 1128  0.007810535605601336
+1128 1128  0.003256443987098833
+901 1129  0.04019593664417826
+1129 1129  0.05799327281272631
+1135 1129  6.743832417317901e-05
+918 1130  0.006772170020769838
+1130 1130  0.04925375365548246
+1131 1131  0.0868999904950067
+1136 1131  0.04719998265580642
+788 1132  0.04366344866518873
+1132 1132  0.04036571610790546
+922 1133  0.03134760977659256
+1133 1133  0.02966830277942088
+936 1134  0.02516388418998334
+1134 1134  0.02223858755811324
+1124 1135  0.01439541418489085
+1129 1135  0.03454367341598973
+1135 1135  0.04268843333067113
+1122 1136  0.01021354363665724
+1131 1136  0.04589998393500563
+1136 1136  0.04719998265580641
+802 1137  0.03055429622450982
+1137 1137  0.02154542140253185
+805 1138  0.06439410099711014
+1138 1138  0.06159998088601238
diff --git a/matrices/test/a_large.mtx b/matrices/test/a_large.mtx
new file mode 100644
index 00000000000..02225c91f92
--- /dev/null
+++ b/matrices/test/a_large.mtx
@@ -0,0 +1,5052 @@
+%%MatrixMarket matrix array real symmetric
+100 100
+ 9.507387586206898
+ 16.9262875862069
+-0.975640275862069
+-9.527639310344828
+-17.43514482758621
+ 1.018235379310345
+ 0.7068437931034484
+ 0.544773048275862
+ 0.04764781034482759
+-0.1706930896551724
+-0.1029656551724138
+-0.06021632206896552
+-0.002943987172413793
+ 0.008551583448275862
+ 0.007157647586206897
+-0.0004222307034482758
+-0.0006059579793103449
+-0.0008743339999999999
+-1.039474275862069e-05
+-0.0002568787586206896
+-0.0003108931034482759
+ 0
+-7.272933103448276e-05
+-0.0001269617379310345
+ 0
+-2.302761103448276e-05
+-3.457650965517242e-05
+ 0
+-1.218220137931034e-05
+-1.494798827586207e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 35.75833655172414
+-1.688903379310345
+-16.72711310344827
+-35.02702137931035
+ 1.771132137931034
+-0.3314247655172414
+-0.2669652344827586
+-0.002670817103448276
+ 0.06519995931034483
+ 0.03932364551724138
+ 0.02300669448275862
+ 0.008551583448275862
+-0.02059450068965517
+-0.01767217379310345
+-0.0006059579793103449
+ 0.0002803287724137931
+ 0.001517457793103448
+-0.0002568787586206896
+ 0.0002676622965517241
+ 0.0005217609586206897
+-7.272933103448276e-05
+ 3.796375793103449e-05
+ 0.0001646504482758621
+-2.302761103448276e-05
+ 1.623803310344828e-05
+ 4.686651172413793e-05
+-1.218220137931034e-05
+ 1.863814551724138e-05
+ 3.459778e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.6378757172413794
+ 1.056251931034483
+ 1.986631793103448
+-0.8257926896551724
+ 0.2961380896551724
+ 0.0139804075862069
+ 0.3849934827586207
+-0.3467231310344828
+-0.2091442206896552
+-0.1223217310344828
+ 0.007157647586206897
+-0.01767217379310345
+-0.01427218413793103
+-0.0008743339999999999
+ 0.001517457793103448
+ 0.001418165586206896
+-0.0003108931034482759
+ 0.0005217609586206897
+ 0.0004680165862068966
+-0.0001269617379310345
+ 0.0001646504482758621
+ 0.0002310678620689655
+-3.457650965517242e-05
+ 4.686651172413793e-05
+ 5.539589724137931e-05
+-1.494798827586207e-05
+ 3.459778e-05
+ 3.289396965517241e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 25.37018689655172
+ 0.03815196137931035
+-11.4157724137931
+-15.96946482758621
+ 16.45557586206897
+ 10.17765310344828
+ 0.02039901793103448
+ 0.03895268275862069
+ 0.01341664
+ 0.3451321586206896
+ 0.1764193931034483
+ 0.1143378620689655
+-0.0004587904827586207
+-0.0003019009586206896
+-0.000847974
+ 0.0001612334206896552
+-0.0003495643793103448
+-0.0006524910896551724
+ 4.129113241379311e-05
+-7.059000689655173e-05
+-0.0002015320344827586
+ 1.349831379310345e-05
+-2.608321448275862e-05
+-5.585081931034483e-05
+ 0
+-1.983652965517241e-05
+-3.200872620689655e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 57.50793517241379
+ 10.55127793103448
+ 17.58907793103448
+-22.60832068965517
+-12.56414965517241
+-0.4182823655172414
+ 0.2225919724137931
+-0.03660212482758621
+ 0.2081843931034483
+ 0.1064208827586207
+ 0.06896635172413793
+-0.0003019009586206896
+-0.0007321437931034484
+ 0.0005469224551724138
+-0.0003495643793103448
+ 5.688953448275862e-05
+ 0.0005623283724137931
+-7.059000689655173e-05
+-2.251513862068965e-05
+ 0.0001300750965517241
+-2.608321448275862e-05
+ 0
+ 4.492035655172414e-05
+-1.983652965517241e-05
+ 1.342995172413793e-05
+ 4.503403724137932e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 9.437159310344827
+ 9.480863448275862
+-12.05305724137931
+-8.601750344827586
+ 0.7408009655172414
+-0.3254658827586207
+ 0.08089086206896552
+ 0.1217593172413793
+ 0.0622367475862069
+ 0.04034112068965517
+-0.000847974
+ 0.0005469224551724138
+ 0.000609310448275862
+-0.0006524910896551724
+ 0.0005623283724137931
+ 0.0008052513103448275
+-0.0002015320344827586
+ 0.0001300750965517241
+ 0.0003032804068965517
+-5.585081931034483e-05
+ 4.492035655172414e-05
+ 7.725388965517241e-05
+-3.200872620689655e-05
+ 4.503403724137932e-05
+ 5.818863517241379e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 18.47544620689655
+-15.36471862068966
+-17.99068137931035
+-2.696934344827586
+-1.413146551724138
+ 7.988540689655172
+-0.6945817931034483
+-0.5530225862068966
+ 0.09409567586206896
+ 0.3679345655172414
+ 0.1812813655172414
+ 0.1330178413793103
+-0.0004751480896551724
+-0.0003164226827586207
+-0.0008712627586206896
+ 4.162283310344828e-05
+-0.0001280162689655172
+-0.0006400958137931035
+ 2.724029793103448e-05
+-6.291741172413793e-05
+-0.0001816629586206897
+-1.494309931034483e-05
+-4.940948068965517e-05
+-9.557138620689655e-05
+ 0
+-1.656822482758621e-05
+-2.765900206896552e-05
+ 0
+-8.080044827586207e-06
+-1.095650551724138e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 26.94837103448276
+ 2.974075379310345
+-1.330644827586207
+-4.051198
+ 9.581779310344828
+-0.3850140275862069
+ 0.1554554965517241
+-0.704422
+ 0.1880755448275862
+ 0.09266883448275862
+ 0.06799222344827587
+-0.0003164226827586207
+-0.0006935188965517242
+ 0.0006210286965517241
+-0.0001280162689655172
+-0.0001892681103448276
+ 0.000284692524137931
+-6.291741172413793e-05
+-2.070887655172414e-05
+ 0.0001251815931034483
+-4.940948068965517e-05
+ 2.416879172413793e-05
+ 0.0001406100896551724
+-1.656822482758621e-05
+ 1.150021172413793e-05
+ 4.124594275862069e-05
+-8.080044827586207e-06
+ 1.361720206896552e-05
+ 2.951842413793103e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 48.22368413793104
+ 7.882573103448276
+ 9.493863448275862
+-39.74624068965517
+-0.2371099724137931
+-0.08082382068965517
+-0.1446659310344828
+ 0.1218918068965517
+ 0.0600538924137931
+ 0.04407042827586207
+-0.0008712627586206896
+ 0.0006210286965517241
+ 0.0007909042068965517
+-0.0006400958137931035
+ 0.000284692524137931
+ 0.001177266482758621
+-0.0001816629586206897
+ 0.0001251815931034483
+ 0.0002973719862068966
+-9.557138620689655e-05
+ 0.0001406100896551724
+ 0.0002234498896551724
+-2.765900206896552e-05
+ 4.124594275862069e-05
+ 5.564861034482758e-05
+-1.095650551724138e-05
+ 2.951842413793103e-05
+ 3.18748248275862e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 15.58613724137931
+-14.6339
+-17.76058620689655
+-12.1547724137931
+ 16.54367517241379
+ 9.712164137931035
+-0.7461117931034483
+-0.308096724137931
+-0.2552290827586207
+ 0.3760677793103449
+ 0.1616321793103448
+ 0.1301900965517241
+-0.0005005506620689655
+-0.0002814506068965517
+-0.0008073446896551724
+ 0.0001293575793103448
+-0.0003224790206896552
+-0.0006272998000000001
+-4.998646896551724e-05
+-0.0001947931655172414
+-0.0002682746068965517
+ 0
+-5.18163124137931e-05
+-6.491219999999999e-05
+-1.21305e-05
+-1.964566413793103e-05
+-2.043121655172414e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 27.98288620689655
+ 4.112758
+ 16.56426068965517
+-24.41440413793104
+-13.05385034482759
+-0.5589071310344828
+ 0.2047835862068965
+-0.08497348275862068
+ 0.1852888413793103
+ 0.07963971034482759
+ 0.06414277103448275
+-0.0002814506068965517
+-0.0007446749655172413
+ 0.0005298609379310345
+-0.0003224790206896552
+ 4.388975103448276e-05
+ 0.0005496528965517241
+-0.0001947931655172414
+ 0.0002077372965517241
+ 0.0004991324068965517
+-5.18163124137931e-05
+ 5.851311448275862e-05
+ 0.0001158903862068966
+-1.964566413793103e-05
+ 4.757615862068966e-05
+ 6.821318482758621e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 49.60891586206897
+ 9.742552413793103
+-13.04671103448276
+-9.648690344827587
+ 0.04352490413793103
+-0.695332275862069
+-0.1590438413793104
+ 0.1359581103448276
+ 0.05843221655172414
+ 0.04707018827586208
+-0.0008073446896551724
+ 0.0005298609379310345
+ 0.0005905069172413793
+-0.0006272998000000001
+ 0.0005496528965517241
+ 0.0008305266206896552
+-0.0002682746068965517
+ 0.0004991324068965517
+ 0.000532731075862069
+-6.491219999999999e-05
+ 0.0001158903862068966
+ 0.0001111821586206897
+-2.043121655172414e-05
+ 6.821318482758621e-05
+ 5.292552482758621e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 16.02089379310345
+-14.31042965517241
+-18.78710758620689
+-2.894899931034483
+-1.735033034482759
+ 9.111203448275862
+-0.7516644827586206
+-0.4936843103448276
+ 0.01360813586206897
+ 0.3822822068965517
+ 0.1545104068965517
+ 0.1480570896551724
+-0.0004364188
+-0.000321631675862069
+-0.0009492923448275862
+-0.0004946472275862069
+-0.0004124728344827587
+-0.0006992057931034482
+-5.768837517241379e-05
+-0.0001635274206896552
+-0.0002360507931034483
+-5.698394896551724e-05
+-4.544111448275862e-05
+-5.163884344827587e-05
+-1.212235793103448e-05
+-1.815835103448276e-05
+-1.920040068965517e-05
+-7.625637241379311e-06
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 28.45061931034483
+ 3.774955379310345
+-1.716966827586207
+-4.298854413793103
+ 9.849167586206896
+-0.3011831241379311
+ 0.2433643172413793
+-0.7025911724137931
+ 0.1643025862068966
+ 0.06641086413793103
+ 0.06363304137931035
+-0.000321631675862069
+-0.000687496572413793
+ 0.0006485134896551724
+-0.0004124728344827587
+ 0.0002017957862068965
+ 0.001505337862068966
+-0.0001635274206896552
+ 0.0001759725793103448
+ 0.0004587468068965517
+-4.544111448275862e-05
+ 0.0001536152620689655
+ 0.0002499883586206897
+-1.815835103448276e-05
+ 4.507379862068966e-05
+ 6.605859103448276e-05
+ 0
+ 2.645245724137931e-05
+ 3.18563475862069e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 48.99697103448276
+ 9.102015172413793
+ 9.840553103448276
+-39.01642137931034
+-0.2548845310344828
+-0.05982489586206897
+-0.1680335103448276
+ 0.132341724137931
+ 0.05348842827586207
+ 0.0512578
+-0.0009492923448275862
+ 0.0006485134896551724
+ 0.001006845103448276
+-0.0006992057931034482
+ 0.001505337862068966
+ 0.001865532275862069
+-0.0002360507931034483
+ 0.0004587468068965517
+ 0.0005203627655172414
+-5.163884344827587e-05
+ 0.0002499883586206897
+ 0.0002177119448275862
+-1.920040068965517e-05
+ 6.605859103448276e-05
+ 5.244864413793103e-05
+ 0
+ 3.18563475862069e-05
+ 2.126724551724138e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 14.38731862068965
+-13.32846620689655
+-18.41909586206896
+-10.41646275862069
+ 15.54403517241379
+ 9.428503448275862
+-0.8028775172413793
+-0.2431268413793103
+-0.2870848275862069
+ 0.3909963517241379
+ 0.1420076413793104
+ 0.1467314
+-0.001137678068965517
+ 0.004599143655172414
+ 0.004705009379310345
+-0.000214330924137931
+-0.0004988466965517241
+-0.0004825145448275862
+-0.0001402028965517241
+-8.782599310344828e-05
+-6.930295172413793e-05
+-2.690888068965517e-05
+-3.668687034482759e-05
+-2.839123103448276e-05
+-1.357759931034483e-05
+ 0
+ 0
+-1.056255931034483e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 29.31237310344827
+ 4.709881241379311
+ 15.56635172413793
+-25.4683475862069
+-13.95292965517241
+-0.5074189586206896
+ 0.2689451793103448
+-0.05748361172413793
+ 0.1580329931034483
+ 0.05739964482758621
+ 0.05930520896551725
+ 0.004599143655172414
+-0.01532399034482759
+-0.01590466413793103
+-0.0004988466965517241
+ 0.0008880568965517242
+ 0.001252216413793104
+-8.782599310344828e-05
+ 0.0005389707379310345
+ 0.0005447447586206896
+-3.668687034482759e-05
+ 0.0001315631655172414
+ 0.0001288906
+ 0
+ 6.055195241379311e-05
+ 5.131481517241379e-05
+ 0
+ 3.90733e-05
+ 4.276421103448276e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 49.72536137931034
+ 9.432560689655173
+-13.94951586206897
+-10.32325103448276
+-0.01740205448275862
+-0.6835017379310344
+-0.205150275862069
+ 0.1514317448275862
+ 0.05499816275862069
+ 0.05683020413793103
+ 0.004705009379310345
+-0.01590466413793103
+-0.01604782275862069
+-0.0004825145448275862
+ 0.001252216413793104
+ 0.0008046601379310344
+-6.930295172413793e-05
+ 0.0005447447586206896
+ 0.0002784732
+-2.839123103448276e-05
+ 0.0001288906
+ 6.475428482758621e-05
+ 0
+ 5.131481517241379e-05
+ 2.146366551724138e-05
+ 0
+ 4.276421103448276e-05
+ 2.558881724137931e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 14.57719862068966
+-12.69538896551724
+-19.20994551724138
+-3.454179586206896
+-2.41915475862069
+ 9.660764137931034
+-0.04239575655172414
+-0.5028410827586207
+ 0.670439075862069
+-0.1464690620689655
+ 0.1695998827586207
+-0.3686785379310345
+-0.001028798068965517
+ 0.003280225241379311
+ 0.003606045793103448
+-0.0006812032068965517
+ 2.989333379310345e-05
+ 2.493868e-05
+-0.0001392868827586207
+-8.673175862068966e-05
+-6.994188965517242e-05
+-4.774673310344828e-05
+ 9.000826896551725e-06
+ 0
+-2.55520324137931e-05
+ 3.123000965517241e-05
+ 2.804268689655172e-05
+ 0
+ 1.550586413793103e-05
+ 1.203971862068966e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 30.26506620689655
+ 3.197392896551724
+-2.489333103448276
+-5.28189648275862
+ 11.20735931034483
+ 0.02861035103448276
+ 0.3081422482758621
+-0.4085184
+-0.05319866344827586
+ 0.0615965724137931
+-0.1339024827586207
+ 0.003280225241379311
+-0.01158284068965517
+-0.01263172896551724
+ 2.989333379310345e-05
+ 0.001456756344827586
+ 0.001784068413793103
+-8.673175862068966e-05
+ 0.0005243722068965517
+ 0.0005441620827586206
+ 9.000826896551725e-06
+ 0.0002030405034482759
+ 0.0001759869310344828
+ 3.123000965517241e-05
+ 0.0001052479655172414
+ 0.0001238030482758621
+ 1.550586413793103e-05
+ 3.979123448275862e-05
+ 3.929856137931034e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 48.3293
+ 9.761829655172413
+ 11.28652275862069
+-37.70450068965517
+-0.003006942689655173
+-0.04476122689655172
+ 0.06036403517241379
+-0.05496545172413794
+ 0.06364721103448276
+-0.1383565103448276
+ 0.003606045793103448
+-0.01263172896551724
+-0.01397869310344828
+ 2.493868e-05
+ 0.001784068413793103
+ 0.0008068340000000001
+-6.994188965517242e-05
+ 0.0005441620827586206
+ 0.0002883744344827586
+ 0
+ 0.0001759869310344828
+ 7.526785517241378e-05
+ 2.804268689655172e-05
+ 0.0001238030482758621
+ 7.854210344827586e-05
+ 1.203971862068966e-05
+ 3.929856137931034e-05
+ 1.96929524137931e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 14.86223103448276
+-11.17081172413793
+-19.21202551724138
+-11.00837724137931
+ 14.75145172413793
+ 9.317006206896551
+ 0.6111585034482758
+-0.4778164413793103
+ 0.09426643448275861
+ 0.06417459172413793
+-0.07060804827586208
+ 0.1382952689655172
+-0.0009058560689655172
+ 0.0001809489034482759
+ 8.133559999999999e-05
+-0.0002433233172413793
+-7.387113103448276e-05
+-3.677954689655172e-05
+-6.299328758620689e-05
+ 3.772604413793103e-05
+ 1.70590648275862e-05
+-3.035420551724138e-05
+ 7.19205724137931e-05
+ 4.475804965517241e-05
+ 0
+ 2.684367931034483e-05
+ 1.480232620689655e-05
+ 0
+ 1.725162965517241e-05
+ 1.196528827586207e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 29.14838965517242
+ 4.680886965517241
+ 13.50934275862069
+-23.79191862068966
+-15.22718551724138
+ 0.4433124827586207
+-0.3410086
+ 0.03337290827586207
+-0.07430629655172415
+ 0.08176048275862069
+-0.1601356068965517
+ 0.0001809489034482759
+ 0.0009656779310344828
+ 0.0008490512413793103
+-7.387113103448276e-05
+ 0.001171721172413793
+ 0.000706394275862069
+ 3.772604413793103e-05
+ 0.0003570491172413793
+ 0.0001914632344827586
+ 7.19205724137931e-05
+ 0.0002159170344827586
+ 0.0001653752
+ 2.684367931034483e-05
+ 6.496945517241379e-05
+ 4.364480344827586e-05
+ 1.725162965517241e-05
+ 2.721767793103448e-05
+ 2.411594068965517e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 50.58806
+ 9.825377241379311
+-15.78971931034483
+-12.93243517241379
+-0.08230637931034482
+ 0.05650040827586207
+ 0.03659139379310344
+ 0.1615299793103448
+-0.1777294137931034
+ 0.3481057172413793
+ 8.133559999999999e-05
+ 0.0008490512413793103
+-0.000541549351724138
+-3.677954689655172e-05
+ 0.000706394275862069
+ 0.000104503475862069
+ 1.70590648275862e-05
+ 0.0001914632344827586
+ 2.022333103448276e-05
+ 4.475804965517241e-05
+ 0.0001653752
+ 5.309790068965517e-05
+ 1.480232620689655e-05
+ 4.364480344827586e-05
+ 9.88770551724138e-06
+ 1.196528827586207e-05
+ 2.411594068965517e-05
+ 9.173352413793103e-06
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 27.99440620689655
+ 6.141295034482758
+-7.052193103448276
+-17.42896275862069
+-20.07475655172414
+-2.72642151724138
+ 0.3300255172413794
+ 0.4689026275862069
+-0.3185072137931034
+ 0.07104062068965518
+-0.08248120689655172
+ 0.158435924137931
+-0.0009192224137931034
+ 0.0001491706206896551
+ 6.659398896551724e-05
+-0.0001718162068965517
+ 0.0003320503655172414
+ 0.0001337059586206897
+ 0
+ 0.0003385045034482758
+ 0.0002045532482758621
+ 1.037986206896552e-05
+ 9.246960689655173e-05
+ 4.853807103448276e-05
+ 1.500210551724138e-05
+ 4.542782965517241e-05
+ 3.137863379310345e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 51.26644965517242
+ 19.80913862068966
+-20.15692965517242
+-27.34780689655172
+-4.329715034482758
+-0.1478756
+-0.3744408275862069
+ 0.346112151724138
+-0.07816238620689656
+ 0.09075457931034484
+-0.1743255517241379
+ 0.0001491706206896551
+ 0.0007678878620689656
+ 0.000759063448275862
+ 0.0003320503655172414
+ 0.001215018068965517
+ 0.0005888076965517241
+ 0.0003385045034482758
+ 0.0005864146275862069
+ 0.0004490578896551724
+ 9.246960689655173e-05
+ 0.0001486279172413793
+ 9.688464827586206e-05
+ 4.542782965517241e-05
+ 4.971801310344827e-05
+ 4.558090896551724e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 15.00886689655172
+-2.558856137931035
+-4.304900275862069
+-1.68203124137931
+-0.6411026068965517
+ 0.1216404068965517
+-0.6591376965517242
+ 0.1530909310344828
+-0.1777519862068966
+ 0.3414386551724138
+ 6.659398896551724e-05
+ 0.000759063448275862
+-0.0005934290620689654
+ 0.0001337059586206897
+ 0.0005888076965517241
+-1.012690620689655e-05
+ 0.0002045532482758621
+ 0.0004490578896551724
+ 0.0001146514413793103
+ 4.853807103448276e-05
+ 9.688464827586206e-05
+ 1.490917034482759e-05
+ 3.137863379310345e-05
+ 4.558090896551724e-05
+ 1.521340413793103e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 27.42867517241379
+ 6.824471103448276
+-7.748226896551724
+-10.67058068965517
+ 12.94137517241379
+ 11.21792413793103
+ 0.3471452137931034
+-0.1415545448275862
+-0.6336685586206897
+ 0.05694206758620689
+-0.07166853103448276
+ 0.1569107862068966
+-0.0009082393793103449
+ 0.0001632459310344828
+ 7.624994482758621e-05
+-0.0003991188482758621
+ 0.0008013735172413793
+ 0.0005871277448275862
+-9.112732413793104e-06
+ 0.0003249579724137931
+ 0.0001970324965517241
+ 3.969214275862069e-05
+ 0.0001481983448275862
+ 0.0001203855862068965
+ 1.439358137931034e-05
+ 4.868494689655172e-05
+ 3.170703448275862e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 49.47399655172413
+ 20.68361655172414
+ 12.94960620689655
+-21.41140413793103
+-16.67916827586207
+ 0.4608146137931034
+-0.3662505586206897
+ 0.1190646620689655
+-0.06611073655172414
+ 0.08321379310344827
+-0.1821868
+ 0.0001632459310344828
+ 0.000958783172413793
+ 0.0008787396551724138
+ 0.0008013735172413793
+ 0.001480246344827586
+ 0.001593285724137931
+ 0.0003249579724137931
+ 0.0005786194137931033
+ 0.0004438317103448276
+ 0.0001481983448275862
+ 0.0001603169931034483
+ 0.0001789662
+ 4.868494689655172e-05
+ 5.703008137931035e-05
+ 4.849746551724138e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 16.64129862068966
+ 11.21379793103448
+-16.67795034482759
+-14.11864551724138
+-0.2949161172413793
+ 0.3409930620689655
+-0.6500570689655173
+ 0.1269904137931034
+-0.1598408068965517
+ 0.3499573241379311
+ 7.624994482758621e-05
+ 0.0008787396551724138
+-0.0005119362965517242
+ 0.0005871277448275862
+ 0.001593285724137931
+ 0.0004729021448275862
+ 0.0001970324965517241
+ 0.0004438317103448276
+ 0.000115737675862069
+ 0.0001203855862068965
+ 0.0001789662
+ 8.538359310344828e-05
+ 3.170703448275862e-05
+ 4.849746551724138e-05
+ 1.414919448275862e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 27.22900344827586
+ 6.557267448275862
+-7.12523448275862
+-17.06198275862069
+-19.96249655172414
+-3.043849448275862
+ 0.3662494620689655
+ 0.4647026689655173
+-0.3127578896551724
+ 0.04409045655172414
+-0.06542255724137931
+ 0.1356757448275862
+-0.008463653793103447
+-0.01230105586206896
+-0.008697089655172413
+ 0.0003488920137931034
+ 0.001096986413793103
+ 0.0006112221172413793
+ 0.0002515918068965518
+ 0.0003869946206896552
+ 0.0003147866482758621
+ 6.702879517241379e-05
+ 0.000113354875862069
+ 7.041461379310344e-05
+ 2.99074324137931e-05
+ 3.766646482758621e-05
+ 2.986818896551724e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 49.63927655172414
+ 21.34938689655172
+-19.95168551724138
+-27.45460689655172
+-5.093884275862069
+-0.168285724137931
+-0.3665937241379311
+ 0.3623362689655172
+-0.05549097310344828
+ 0.08234735172413793
+-0.1707736137931034
+-0.01230105586206896
+-0.01698540689655172
+-0.01221655517241379
+ 0.001096986413793103
+ 0.00112049524137931
+ 0.0008628804137931035
+ 0.0003869946206896552
+ 0.0002486358896551724
+ 0.0003135849310344827
+ 0.000113354875862069
+ 9.037946206896551e-05
+ 7.803943448275861e-05
+ 3.766646482758621e-05
+ 2.144046827586207e-05
+ 2.669914068965517e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 16.99647103448276
+-3.079335310344828
+-5.068160137931034
+-1.74207075862069
+-0.5665745172413793
+ 0.1052798344827586
+-0.6932204137931034
+ 0.1214911862068965
+-0.1802892068965517
+ 0.3738951310344827
+-0.008697089655172413
+-0.01221655517241379
+-0.008343778620689656
+ 0.0006112221172413793
+ 0.0008628804137931035
+ 5.262615586206897e-05
+ 0.0003147866482758621
+ 0.0003135849310344827
+ 0.0001181922551724138
+ 7.041461379310344e-05
+ 7.803943448275861e-05
+ 1.322706e-05
+ 2.986818896551724e-05
+ 2.669914068965517e-05
+ 8.941974482758621e-06
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 28.51891724137931
+ 5.362052344827586
+-7.082658620689655
+-12.08045586206896
+ 14.42987310344828
+ 10.71194344827586
+ 0.3591143931034483
+-0.2183090344827586
+-0.2353391172413793
+ 0.0742888
+-0.03219096551724138
+-0.1039314965517241
+-0.008253270344827587
+-0.01235218896551724
+-0.008644932413793103
+ 0.0006632136
+ 0.001168275310344828
+ 0.001245567517241379
+ 0.0002696775448275862
+ 0.0004502065448275862
+ 0.0003344944
+ 0.0001045902206896552
+ 0.0001209796482758621
+ 0.0001164946137931034
+ 3.476353517241379e-05
+ 4.548105448275862e-05
+ 3.265688620689655e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 50.77792275862069
+ 20.26526206896552
+ 14.39168068965517
+-22.70572827586207
+-15.30736965517241
+ 0.5154355517241379
+-0.3199984344827586
+-0.3093172551724138
+-0.1102442413793104
+ 0.04777490551724138
+ 0.1542258551724138
+-0.01235218896551724
+-0.01749564551724138
+-0.01246429586206897
+ 0.001168275310344828
+ 0.0004245849448275862
+ 0.001124829931034483
+ 0.0004502065448275862
+ 0.0003198911172413794
+ 0.0003536663724137931
+ 0.0001209796482758621
+ 5.55952248275862e-05
+ 9.526921379310345e-05
+ 4.548105448275862e-05
+ 2.793027517241379e-05
+ 3.029557310344827e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 14.02249655172414
+ 10.83402275862069
+-15.36702620689655
+-11.64634689655172
+-0.392058048275862
+ 0.2478738137931034
+ 0.2162618137931034
+ 0.2286299034482759
+-0.09908226896551724
+-0.3198399172413793
+-0.008644932413793103
+-0.01246429586206897
+-0.008409587586206897
+ 0.001245567517241379
+ 0.001124829931034483
+ 0.000568803724137931
+ 0.0003344944
+ 0.0003536663724137931
+ 0.0001066476620689655
+ 0.0001164946137931034
+ 9.526921379310345e-05
+ 4.839572689655173e-05
+ 3.265688620689655e-05
+ 3.029557310344827e-05
+ 7.491126206896552e-06
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 28.75004275862069
+ 5.261753241379311
+-5.826562000000001
+-16.62624068965517
+-18.99846827586207
+-4.512158551724138
+-0.344132475862069
+-0.3504368275862069
+ 0.2809605862068966
+ 0.1571879034482759
+-0.07046330344827587
+-0.2208336068965517
+ 0.0008877453793103449
+ 0.0008368364827586207
+ 0.001068074206896552
+ 0.0008354328965517241
+ 0.0007347057931034483
+ 0.0005565399862068966
+ 0.0002394993862068966
+ 0.0001618842344827586
+ 0.0001703296206896552
+ 7.553464137931035e-05
+ 6.388358827586206e-05
+ 4.614627034482758e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 50.08102896551724
+ 21.05045793103448
+-20.35861448275862
+-27.41952965517241
+-5.600416413793104
+ 0.3417453655172414
+ 0.3442165724137931
+-0.2805089172413793
+-0.06812185310344827
+ 0.0305424475862069
+ 0.09570018620689656
+ 0.0008368364827586207
+-0.0005001923034482758
+ 0.0005018145931034482
+ 0.0007347057931034483
+ 0.0002291414482758621
+ 0.0003724166482758621
+ 0.0001618842344827586
+ 2.417760689655172e-05
+ 9.127979999999999e-05
+ 6.388358827586206e-05
+ 2.22185475862069e-05
+ 3.07463e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 15.29933517241379
+-4.054567448275862
+-5.848794827586207
+-3.518657379310345
+-0.3392276896551724
+-0.3293673379310345
+ 0.2833854827586207
+-0.2198943379310345
+ 0.09856970344827586
+ 0.3089375931034483
+ 0.001068074206896552
+ 0.0005018145931034482
+-0.0002528827103448276
+ 0.0005565399862068966
+ 0.0003724166482758621
+ 1.960679172413793e-05
+ 0.0001703296206896552
+ 9.127979999999999e-05
+ 3.346594137931035e-05
+ 4.614627034482758e-05
+ 3.07463e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 41.26645793103449
+ 11.71601793103448
+ 23.96258482758621
+-24.69156620689655
+ 7.888358620689655
+-19.71257448275862
+-0.5042100620689655
+ 0.305772
+ 0.2620930344827586
+ 0.1706331379310345
+-0.08232462758620689
+-0.2070680689655172
+-0.0007370531034482759
+ 0.000115941724137931
+ 0.0001232883655172414
+ 0.0009418784827586207
+ 0.0004766905655172414
+ 0.0007300750344827586
+ 0.0003046984827586207
+ 0.0002172056068965517
+ 0.0001947314965517241
+ 8.746699310344828e-05
+ 5.088073793103448e-05
+ 5.98050875862069e-05
+ 3.196303448275862e-05
+ 2.270557310344827e-05
+ 1.874114689655172e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 32.21862206896552
+-0.6992486896551724
+ 7.83942
+-4.731418896551724
+ 7.022748965517241
+-0.2675940827586207
+ 0.3040735862068966
+-0.654512124137931
+-0.07649017241379309
+ 0.03690955310344828
+ 0.09281838620689654
+ 0.000115941724137931
+-0.0008997435862068965
+ 6.410590068965518e-05
+ 0.0004766905655172414
+-6.668131724137931e-05
+ 0.0002904492965517241
+ 0.0002172056068965517
+ 4.442569724137931e-05
+ 0.0001103360965517241
+ 5.088073793103448e-05
+ 0
+ 2.806993724137931e-05
+ 2.270557310344827e-05
+ 0
+ 1.071784551724138e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 20.46245793103449
+-19.64142689655172
+ 6.897234482758621
+-16.96584482758621
+ 0.5102879517241379
+-0.2525369034482758
+-0.5837368551724138
+-0.2397231034482759
+ 0.1156549034482759
+ 0.2909176551724138
+ 0.0001232883655172414
+ 6.410590068965518e-05
+-0.0008918611034482759
+ 0.0007300750344827586
+ 0.0002904492965517241
+ 0.0001885161103448276
+ 0.0001947314965517241
+ 0.0001103360965517241
+ 2.027603517241379e-05
+ 5.98050875862069e-05
+ 2.806993724137931e-05
+ 1.205575034482759e-05
+ 1.874114689655172e-05
+ 1.071784551724138e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 39.70973724137932
+ 11.64188137931034
+ 23.20121034482759
+-14.44379379310345
+-19.3131951724138
+-3.770309034482759
+-0.4267884137931034
+-0.2623940482758621
+ 0.4500655931034483
+ 0.1833113241379311
+-0.1042441379310345
+-0.2247420413793103
+ 0.0008846102758620689
+ 0.001073520827586207
+ 0.0009708670344827586
+ 0.0007283088275862069
+ 0.0007603287586206896
+ 0.0004211680068965517
+ 0.0002336683862068965
+ 0.0001845348551724138
+ 0.0001483925310344828
+ 6.814547517241379e-05
+ 6.26750675862069e-05
+ 3.593858689655172e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 34.08062482758621
+-2.185023517241379
+-19.27420275862069
+-29.88585103448276
+-4.253730344827586
+ 0.2480019448275862
+ 0.3068736275862069
+-0.2391401724137931
+-0.08844256551724138
+ 0.05030061862068966
+ 0.1084266344827586
+ 0.001073520827586207
+-0.000230091124137931
+ 0.0005898598827586206
+ 0.0007603287586206896
+ 0.0003517980620689655
+ 0.0003296053379310345
+ 0.0001845348551724138
+ 5.224562206896552e-05
+ 9.2407e-05
+ 6.26750675862069e-05
+ 2.669968137931035e-05
+ 2.596936620689655e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 19.18058413793104
+-3.821402620689655
+-4.248102206896552
+-1.879992482758621
+ 0.2823904620689655
+-0.6905208965517242
+-0.4230164965517241
+-0.2224524827586207
+ 0.1264979655172414
+ 0.2727380689655173
+ 0.0009708670344827586
+ 0.0005898598827586206
+-0.0003488625310344827
+ 0.0004211680068965517
+ 0.0003296053379310345
+-6.065434068965518e-05
+ 0.0001483925310344828
+ 9.2407e-05
+ 1.164110551724138e-05
+ 3.593858689655172e-05
+ 2.596936620689655e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 39.97770620689655
+ 16.00375448275862
+ 24.41433862068966
+-24.61211379310345
+ 3.157319862068965
+-21.11712620689655
+-0.5135459103448275
+ 0.2944088827586207
+ 0.3293224965517241
+ 0.1740081379310345
+-0.09890520689655172
+-0.2194575517241379
+-0.0002878525103448276
+ 0.0004253782068965517
+ 0.0003152464275862069
+ 0.000961910551724138
+ 0.0005948667103448275
+ 0.0006643046413793103
+ 0.000276731551724138
+ 0.0002156788620689655
+ 0.0001520564620689655
+ 9.504140689655173e-05
+ 5.637414620689656e-05
+ 5.644005793103448e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 32.26302344827587
+ 1.598364068965517
+ 3.200061724137931
+-2.387942137931034
+ 3.377357655172414
+-0.2803164965517241
+ 0.1679142206896552
+-0.7365567586206897
+-0.09895373103448275
+ 0.05625045793103448
+ 0.1247944965517241
+ 0.0004253782068965517
+-0.0006825552137931035
+ 0.0002012651517241379
+ 0.0005948667103448275
+ 2.604740689655172e-05
+ 0.0003226469034482758
+ 0.0002156788620689655
+ 6.281593310344827e-05
+ 9.432062068965518e-05
+ 5.637414620689656e-05
+ 0
+ 2.725414482758621e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 21.63931793103448
+-21.14256482758621
+ 3.367158689655173
+-19.16534206896552
+ 0.5466808482758621
+-0.3123450827586207
+-0.4833615793103448
+-0.2133366482758621
+ 0.121254675862069
+ 0.2690665172413793
+ 0.0003152464275862069
+ 0.0002012651517241379
+-0.0008049746896551723
+ 0.0006643046413793103
+ 0.0003226469034482758
+ 9.743713793103448e-05
+ 0.0001520564620689655
+ 9.432062068965518e-05
+ 0
+ 5.644005793103448e-05
+ 2.725414482758621e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 36.71210413793104
+ 15.00731379310345
+ 21.37378068965517
+-11.48783448275862
+-18.18312689655173
+-0.6187589310344828
+-0.4275524413793104
+-0.2375524965517241
+ 0.5376466620689655
+ 0.1645674551724138
+-0.1177548896551724
+-0.207630475862069
+ 0.000534795248275862
+ 0.0009635873103448277
+ 0.0006488769655172413
+ 0.0007428291724137931
+ 0.0007811626206896552
+ 0.0003274485103448276
+ 0.0002916772206896552
+ 0.0002172657793103448
+ 0.0001489709034482759
+ 7.902735862068965e-05
+ 6.46544924137931e-05
+ 3.248020482758621e-05
+ 7.279691034482758e-05
+ 5.773974275862069e-05
+ 1.068440137931035e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 35.55195931034483
+-3.208719448275862
+-18.13119655172414
+-33.47425586206896
+ 0.5467823517241379
+ 0.2452738551724138
+ 0.1433811586206896
+-0.3083971379310345
+-0.09353907586206897
+ 0.0669365724137931
+ 0.1180107793103448
+ 0.0009635873103448277
+-0.0002802572068965517
+ 0.0004300957655172414
+ 0.0007811626206896552
+ 0.0003849674275862069
+ 0.0002609254137931034
+ 0.0002172657793103448
+ 5.803105241379311e-05
+ 8.903956551724138e-05
+ 6.46544924137931e-05
+ 2.338928551724138e-05
+ 2.138392137931034e-05
+ 5.773974275862069e-05
+ 2.134804551724138e-05
+ 6.93670275862069e-06
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 20.7228275862069
+-0.5602410206896551
+ 0.5306131586206897
+-1.180323310344828
+ 0.2532805655172414
+-0.7614670344827585
+-0.3214810206896552
+-0.2075511034482759
+ 0.1485066482758621
+ 0.2618704965517241
+ 0.0006488769655172413
+ 0.0004300957655172414
+-0.0006293239793103448
+ 0.0003274485103448276
+ 0.0002609254137931034
+-0.0001281179931034483
+ 0.0001489709034482759
+ 8.903956551724138e-05
+-1.077624620689655e-05
+ 3.248020482758621e-05
+ 2.138392137931034e-05
+-8.434317931034483e-06
+ 1.068440137931035e-05
+ 6.93670275862069e-06
+-1.485471379310345e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 39.09078689655173
+ 18.0134475862069
+ 21.00863586206897
+-26.82440482758621
+ 0.2096531517241379
+-21.00078689655172
+-0.436236524137931
+ 0.2887346827586207
+ 0.2985287862068965
+ 0.1577044137931034
+-0.1135808827586207
+-0.2068983862068965
+ 0.0006173338344827586
+ 0.0009557809655172414
+ 0.0006057397724137932
+ 0.001079071586206897
+ 0.0005846182344827586
+ 0.0006278839793103448
+ 0.0002897988965517242
+ 0.0001880457517241379
+ 0.0001307994965517241
+ 0.0002780536482758621
+ 0.0001761840413793103
+ 4.408282482758621e-05
+ 6.877851517241379e-05
+ 4.71944475862069e-05
+ 9.000057931034483e-06
+ 4.093239724137931e-05
+ 2.941326965517241e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 35.01032344827586
+-0.5720494482758621
+ 0.277766075862069
+-1.821079310344828
+ 0.8460544827586207
+-0.2871088413793104
+ 0.1024304551724138
+-0.7457581379310344
+-0.1128439172413793
+ 0.08127762068965516
+ 0.1480395517241379
+ 0.0009557809655172414
+-0.0003251076
+ 0.0003767155034482759
+ 0.0005846182344827586
+-1.161522206896552e-05
+ 0.0002729514896551724
+ 0.0001880457517241379
+ 3.474878551724138e-05
+ 6.93374e-05
+ 0.0001761840413793103
+ 3.735283517241379e-05
+ 2.327339724137931e-05
+ 4.71944475862069e-05
+ 1.295211931034483e-05
+ 0
+ 2.941326965517241e-05
+ 9.082691034482758e-06
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 19.40809655172414
+-20.98043793103448
+ 0.8944150344827586
+-17.58073862068965
+ 0.5484192827586207
+-0.3632626896551724
+-0.3784126344827586
+-0.198971875862069
+ 0.1432973793103448
+ 0.2610469862068965
+ 0.0006057397724137932
+ 0.0003767155034482759
+-0.0006807646965517241
+ 0.0006278839793103448
+ 0.0002729514896551724
+ 2.739578896551724e-05
+ 0.0001307994965517241
+ 6.93374e-05
+-1.670526e-05
+ 4.408282482758621e-05
+ 2.327339724137931e-05
+-4.983899172413794e-05
+ 9.000057931034483e-06
+ 0
+-1.298688551724138e-05
+ 0
+ 0
+-8.26132827586207e-06
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 39.51894344827586
+ 18.10416275862069
+ 19.23597310344827
+-11.98347655172414
+-18.37222758620689
+ 1.208118965517241
+-0.4390358689655173
+-0.1971178137931034
+ 0.5734394068965517
+ 0.1482538137931035
+-0.1296261931034483
+-0.1946123655172414
+-0.0002534092275862069
+ 0.0003941712965517241
+ 0.0002608382689655172
+ 0.0009432808275862069
+ 0.0007786280689655173
+ 0.000322861551724138
+ 0.0007446821379310345
+ 0.0005897355517241379
+-8.19135448275862e-06
+ 0.0001600283586206897
+ 0.0001317981655172414
+ 0
+ 7.549633793103447e-05
+ 6.492634620689655e-05
+-1.879009103448276e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 34.39949724137931
+-3.782671655172414
+-18.32586827586207
+-32.62923034482758
+ 3.708423103448276
+ 0.2845623103448276
+ 0.04263611103448276
+-0.3720880482758621
+-0.1067748482758621
+ 0.09336334482758621
+ 0.1401585103448276
+ 0.0003941712965517241
+-0.0007335142068965518
+ 0.0001465506896551724
+ 0.0007786280689655173
+ 0.0002589762206896552
+ 0.0002107840275862069
+ 0.0005897355517241379
+ 0.0002025847103448276
+ 0
+ 0.0001317981655172414
+ 5.210622344827586e-05
+ 0
+ 6.492634620689655e-05
+ 2.707442e-05
+-1.304721793103448e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 19.50487655172414
+ 1.28791075862069
+ 3.708052689655172
+-1.515905862068965
+ 0.2356376206896552
+-0.8098664137931034
+-0.3123390413793103
+-0.1944987103448276
+ 0.170058075862069
+ 0.2553275586206897
+ 0.0002608382689655172
+ 0.0001465506896551724
+-0.0008579970344827587
+ 0.000322861551724138
+ 0.0002107840275862069
+-0.0001619539034482759
+-8.19135448275862e-06
+ 0
+-0.0001753333034482758
+ 0
+ 0
+-3.632199034482759e-05
+-1.879009103448276e-05
+-1.304721793103448e-05
+-1.42321124137931e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 40.32677379310344
+ 18.00858689655172
+ 19.02413793103448
+-27.59739172413793
+ 0.2294763517241379
+-20.8323875862069
+-0.3704433931034483
+ 0.295502151724138
+ 0.2498367448275862
+ 0.1192459379310345
+-0.1091170137931034
+-0.1969563793103448
+ 0.00116865475862069
+ 0.0009750331034482758
+ 0.0007151143448275862
+ 0.002222929103448276
+ 0.001406143448275862
+ 4.38623275862069e-05
+ 0.0006952891724137931
+ 0.0004547175586206896
+ 0
+ 0.0003038548896551724
+ 0.0002129599586206896
+-7.628791034482758e-05
+ 7.717841379310345e-05
+ 5.712122413793104e-05
+-1.416778275862069e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 34.36774413793103
+-3.741732206896552
+ 0.3077754206896551
+-1.773873931034483
+ 0.9010237931034483
+-0.2929204689655172
+ 0.1203369655172414
+-0.7459804137931034
+-0.1042619034482759
+ 0.09540952413793104
+ 0.1722068137931035
+ 0.0009750331034482758
+-0.0004360101724137931
+ 0.0003376502344827586
+ 0.001406143448275862
+-7.544960689655172e-05
+ 2.080023034482759e-05
+ 0.0004547175586206896
+ 0.0001010907862068966
+ 0
+ 0.0002129599586206896
+ 5.709424206896552e-05
+-4.396853586206897e-05
+ 5.712122413793104e-05
+ 1.83094475862069e-05
+-8.637616551724137e-06
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 18.68428
+-20.7764275862069
+ 0.9826528965517242
+-16.80606620689655
+ 0.4832175724137931
+-0.3860178482758621
+-0.3305954965517242
+-0.1565350620689655
+ 0.1432360827586207
+ 0.2585536344827586
+ 0.0007151143448275862
+ 0.0003376502344827586
+-0.0006487399103448276
+ 4.38623275862069e-05
+ 2.080023034482759e-05
+-0.0007416114482758621
+ 0
+ 0
+-0.0001449413517241379
+-7.628791034482758e-05
+-4.396853586206897e-05
+-4.989349379310344e-05
+-1.416778275862069e-05
+-8.637616551724137e-06
+-1.43726924137931e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 43.45843655172414
+ 19.42105655172414
+ 18.4400675862069
+-15.15111793103448
+-19.72130689655172
+ 1.902663379310345
+-0.4225253586206896
+-0.1727341379310345
+ 0.5524875655172414
+ 0.09701246896551724
+-0.1162924206896552
+-0.1673862206896552
+-0.03036607931034483
+-0.01978472620689655
+ 0.007370637931034483
+ 0.001452079379310345
+ 0.001276916275862069
+-0.0003467628689655172
+ 0.0005001192068965518
+ 0.000458532124137931
+-0.0002778349931034483
+ 0.0001322399034482759
+ 0.0001210269379310345
+-5.397381379310345e-05
+ 4.183055379310345e-05
+ 4.256381448275862e-05
+-2.686746137931035e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 31.20419655172414
+-4.343999655172413
+-19.67653172413793
+-29.38343448275862
+ 4.196778827586207
+ 0.3156872551724138
+ 0.02547222551724138
+-0.439704924137931
+-0.08877268965517242
+ 0.1064178620689655
+ 0.1531680275862069
+-0.01978472620689655
+-0.01172929586206897
+ 0.004675749103448276
+ 0.001276916275862069
+ 0.0004374010206896552
+-0.0002353573448275862
+ 0.000458532124137931
+ 0.0001799789931034483
+-0.0001972880896551724
+ 0.0001210269379310345
+ 5.304250689655173e-05
+-3.912995379310345e-05
+ 4.256381448275862e-05
+ 2.045877103448276e-05
+-2.095607310344828e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 18.72144
+ 2.066965655172414
+ 4.175957379310344
+-1.630147517241379
+ 0.1071159931034483
+-0.8186847586206897
+-0.3642535862068966
+-0.1602326896551724
+ 0.1920778827586207
+ 0.2764750827586207
+ 0.007370637931034483
+ 0.004675749103448276
+-0.000920375448275862
+-0.0003467628689655172
+-0.0002353573448275862
+-0.0003653528137931034
+-0.0002778349931034483
+-0.0001972880896551724
+-2.607678e-05
+-5.397381379310345e-05
+-3.912995379310345e-05
+-1.724845448275862e-05
+-2.686746137931035e-05
+-2.095607310344828e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 44.10202482758621
+ 14.75510551724138
+ 17.5031124137931
+-28.52798137931034
+ 5.137234068965518
+-19.80478965517241
+-0.02972575448275862
+ 0.02505569310344827
+-0.04004924482758621
+-0.1441821586206897
+ 0.1415222137931035
+-0.02162762620689655
+-0.02169661172413793
+-0.01120900344827586
+ 0.004080462068965517
+ 0.001647644689655172
+ 0.001321036551724138
+-0.001117773103448276
+ 0.0005876572344827586
+ 0.0004546790827586207
+-0.0002559836620689655
+ 0.0001550013586206897
+ 0.0001380339931034483
+-0.0001082641724137931
+ 5.047456620689655e-05
+ 4.450631034482759e-05
+-2.682253517241379e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 31.85608620689655
+-7.790184827586207
+ 5.244314689655172
+-2.904788896551724
+ 4.347835655172414
+-0.6341364413793104
+ 0.5758511655172414
+-0.4973707379310345
+ 0.172837524137931
+-0.1696489172413793
+ 0.02592599517241379
+-0.01120900344827586
+-0.005797224206896552
+ 0.002108578206896552
+ 0.001321036551724138
+ 9.034236551724138e-05
+-0.0006386535310344828
+ 0.0004546790827586207
+ 0.0001334188689655172
+-0.0001582761655172414
+ 0.0001380339931034483
+ 4.975547448275862e-05
+-7.459131034482759e-05
+ 4.450631034482759e-05
+ 1.76193275862069e-05
+-1.869100275862069e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 16.77203379310345
+-19.5946448275862
+ 4.375358068965517
+-14.9889724137931
+-0.1228040137931034
+ 0.1142649862068966
+-0.07264891034482758
+ 0.2487791172413793
+-0.2441914620689655
+ 0.03731467379310345
+ 0.004080462068965517
+ 0.002108578206896552
+-0.0007726566896551724
+-0.001117773103448276
+-0.0006386535310344828
+-0.0001240553103448276
+-0.0002559836620689655
+-0.0001582761655172414
+-5.860064275862069e-05
+-0.0001082641724137931
+-7.459131034482759e-05
+ 1.315795172413793e-05
+-2.682253517241379e-05
+-1.869100275862069e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 51.14874551724137
+ 15.45131793103448
+ 16.93451655172414
+-22.51253724137931
+-20.79194275862069
+ 2.60916924137931
+ 0.08399590344827586
+-0.07117767586206897
+ 0.01422971793103448
+ 0.2102414689655172
+-0.2574469034482759
+ 0.02414373862068966
+ 0.0001922512827586207
+ 0.0009383610344827586
+-0.001028225103448276
+ 0.0005989798413793103
+ 0.0007233198620689655
+-0.0005540993517241379
+ 0.0001241261379310345
+ 0.0001704768
+-0.0001609140413793104
+ 5.075241724137931e-05
+ 6.178616000000001e-05
+-4.589591448275862e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 26.23579172413793
+-6.549084137931034
+-20.24790344827586
+-23.33974551724138
+ 2.769525448275862
+-0.0659439
+ 0.1738010068965517
+ 0.005887093931034483
+-0.2063625517241379
+ 0.2526987931034483
+-0.02369749517241379
+ 0.0009383610344827586
+-7.446316551724139e-05
+-0.0008924297241379311
+ 0.0007233198620689655
+ 0.0003571539310344828
+-0.0004691640206896552
+ 0.0001704768
+ 9.39471103448276e-05
+-0.0001473003724137931
+ 6.178616000000001e-05
+ 3.472178068965517e-05
+-4.032660275862069e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 17.06144827586207
+ 2.266118344827586
+ 1.523120551724138
+-1.948163586206896
+ 0.1550830551724138
+ 0.8867825517241379
+ 0.1736130206896552
+ 0.03153576689655173
+-0.03861616482758621
+ 0.003622695448275862
+-0.001028225103448276
+-0.0008924297241379311
+ 8.90069724137931e-05
+-0.0005540993517241379
+-0.0004691640206896552
+ 0.0001041154965517241
+-0.0001609140413793104
+-0.0001473003724137931
+ 7.693126896551724e-05
+-4.589591448275862e-05
+-4.032660275862069e-05
+ 1.038856e-05
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 27.39055862068965
+ 22.58968344827586
+-13.10365793103448
+-4.452152482758621
+-2.047016827586207
+ 10.6623324137931
+-0.4173785793103448
+ 0.5297550206896552
+ 0.5515515034482759
+ 0.1896115931034483
+-0.2334113448275862
+-0.01761619103448276
+-0.0005717950965517241
+ 0.0003060482068965517
+-0.0003171232827586207
+ 0.0003598600827586207
+ 0.0005471162068965518
+-0.0006653002482758621
+ 0.0001740269862068966
+ 0.0002120172275862069
+-0.0001917802068965517
+ 3.705769448275862e-05
+ 5.577371862068965e-05
+-5.703476275862069e-05
+ 1.114411862068966e-05
+ 4.449771517241379e-05
+-4.088304275862069e-05
+ 0
+ 0
+ 0
+ 0
+ 25.32444620689655
+-7.190448965517241
+-2.053906344827586
+-2.013893517241379
+ 5.004171310344828
+ 0.4477848206896552
+-0.5346223862068966
+ 0.491654275862069
+-0.2321850344827586
+ 0.2858218344827586
+ 0.02157230827586207
+ 0.0003060482068965517
+-0.0007128908275862068
+-0.0002523390896551724
+ 0.0005471162068965518
+ 0.0002325003310344828
+-0.0005923560620689656
+ 0.0002120172275862069
+ 0.0001057113517241379
+-0.000163355524137931
+ 5.577371862068965e-05
+ 3.215973862068966e-05
+-5.458520620689655e-05
+ 4.449771517241379e-05
+ 4.634963379310345e-05
+-6.013830137931034e-05
+ 0
+ 0
+ 0
+ 0
+ 41.67192
+ 10.81293379310345
+ 4.871001310344828
+-40.0434275862069
+-0.05709302827586207
+ 0.07734135862068967
+ 0.2322138551724138
+ 0.02177473103448276
+-0.02680413931034483
+-0.002021704068965517
+-0.0003171232827586207
+-0.0002523390896551724
+-0.000694945724137931
+-0.0006653002482758621
+-0.0005923560620689656
+ 0.0004656841862068965
+-0.0001917802068965517
+-0.000163355524137931
+ 7.288139310344828e-05
+-5.703476275862069e-05
+-5.458520620689655e-05
+ 3.460066413793103e-05
+-4.088304275862069e-05
+-6.013830137931034e-05
+ 3.614704068965518e-05
+ 0
+ 0
+ 0
+ 0
+ 31.28482896551724
+ 22.67200413793103
+-11.3238324137931
+-26.40783034482759
+-21.17729931034483
+ 0.1017780827586207
+-0.3558900827586207
+ 0.4388752689655173
+ 0.03333126896551724
+ 0.2029007034482758
+-0.2528391034482759
+-0.01990916482758621
+ 0.0003171184344827586
+ 0.00112734675862069
+-0.0009300303448275862
+ 0.0005882853931034484
+ 0.0007279755172413793
+-0.0004262355793103448
+ 0.0001283214137931034
+ 0.0001918923586206897
+-0.0001460472413793103
+ 3.628917310344827e-05
+ 0.0001477496620689655
+-0.0001027058275862069
+ 0
+ 3.762899103448276e-05
+-3.091286e-05
+ 0
+ 20.49952482758621
+-5.517259379310344
+-21.16524137931034
+-17.80257862068965
+ 0.02278027655172414
+ 0.4534075586206897
+-0.524190048275862
+-0.0304271648275862
+-0.2497697724137931
+ 0.311247248275862
+ 0.02450902620689655
+ 0.00112734675862069
+ 0.0001820861379310345
+-0.000875998275862069
+ 0.0007279755172413793
+ 0.0004109818827586207
+-0.0003774773172413793
+ 0.0001918923586206897
+ 0.0001276895034482759
+-0.0001458063517241379
+ 0.0001477496620689655
+ 0.0001777992896551724
+-0.0001630586413793103
+ 3.762899103448276e-05
+ 4.900443517241379e-05
+-5.391214e-05
+ 0
+ 41.56537862068966
+ 0.171111
+-0.004346238689655172
+-2.056418068965517
+ 0.5248021172413794
+ 0.4753244275862069
+ 0.3376739931034483
+-0.01885135172413793
+ 0.02349198620689655
+ 0.001852110482758621
+-0.0009300303448275862
+-0.000875998275862069
+-0.000157088
+-0.0004262355793103448
+-0.0003774773172413793
+-1.270338275862069e-05
+-0.0001460472413793103
+-0.0001458063517241379
+ 4.708471586206896e-05
+-0.0001027058275862069
+-0.0001630586413793103
+ 5.657433931034483e-05
+-3.091286e-05
+-5.391214e-05
+ 2.766891172413793e-05
+ 0
+ 31.82766068965517
+ 24.94355379310345
+-11.38292482758621
+-4.798616620689655
+-4.347339310344828
+ 10.78249379310345
+-0.4060966965517241
+ 0.3893542137931034
+ 0.6586543655172413
+ 0.181095324137931
+-0.2184234275862069
+-0.05618209862068965
+-0.0001991724
+ 0.0006770993448275862
+-0.0005171988206896552
+ 0.0003368152689655173
+ 0.0006365474827586206
+-0.0006031668482758621
+-7.782328965517242e-06
+ 0.0003898951655172414
+-0.0003244248965517242
+-1.111703655172414e-05
+ 7.840250344827585e-05
+-7.542615172413794e-05
+ 0
+ 23.85843931034483
+-11.71251103448276
+-4.468799862068965
+-4.845028689655172
+ 11.5023275862069
+ 0.4992299310344828
+-0.6860933724137932
+ 0.290327724137931
+-0.2256665586206897
+ 0.2721866827586207
+ 0.07001067586206897
+ 0.0006770993448275862
+-0.0003251973034482758
+-0.0004712996137931034
+ 0.0006365474827586206
+ 0.0004105081448275862
+-0.0006390865379310345
+ 0.0003898951655172414
+ 0.0005645707310344826
+-0.0006405520068965518
+ 7.840250344827585e-05
+ 0.0001246207586206897
+-0.0001650521034482759
+ 3.568677586206896e-05
+ 38.35983172413793
+ 10.83331931034483
+ 11.49231724137931
+-36.48879586206896
+ 0.03749725586206896
+-0.1074435310344828
+ 0.318266324137931
+-0.01776978620689655
+ 0.02143389586206897
+ 0.00551445351724138
+-0.0005171988206896552
+-0.0004712996137931034
+-0.0005822081379310345
+-0.0006031668482758621
+-0.0006390865379310345
+ 0.000341624
+-0.0003244248965517242
+-0.0006405520068965518
+ 0.0003277401034482759
+-7.542615172413794e-05
+-0.0001650521034482759
+ 0.0001118416413793103
+-3.015415724137931e-05
+ 32.11008344827586
+ 24.49169862068966
+-6.97196827586207
+-26.71608275862069
+-20.63384965517241
+-4.448274344827587
+-0.3598160206896552
+ 0.4345241310344827
+ 0.1119097586206897
+ 0.1920598068965517
+-0.2455766068965517
+-0.06692743724137931
+-0.000127723
+ 0.0008526115172413793
+-0.0005681574
+-0.0003432196413793103
+ 0.001015391586206897
+-0.0006073388896551725
+-5.916006965517241e-05
+ 0.0003274015379310345
+-0.000254043924137931
+ 1.771352e-05
+ 22.47626896551724
+-8.386455862068965
+-20.65826551724138
+-16.77413862068965
+-3.189389862068965
+ 0.3393084689655173
+-0.6062219862068965
+-0.2091215793103448
+-0.2316466620689655
+ 0.2962004275862069
+ 0.08072387586206897
+ 0.0008526115172413793
+-2.439149310344828e-05
+-0.0006036224827586207
+ 0.001015391586206897
+ 0.001426153379310345
+-0.001334670413793103
+ 0.0003274015379310345
+ 0.0006083681655172414
+-0.0006217567172413794
+ 0.0001290459448275862
+ 38.79810965517241
+-4.375698344827586
+-3.247298896551724
+-2.743457793103449
+ 0.6136763586206896
+ 0.3006891931034483
+ 0.3584232896551724
+-0.05958306413793104
+ 0.07618717931034483
+ 0.02076491586206897
+-0.0005681574
+-0.0006036224827586207
+-0.0005279914620689656
+-0.0006073388896551725
+-0.001334670413793103
+-6.94988e-06
+-0.000254043924137931
+-0.0006217567172413794
+ 0.0002895178275862069
+-8.758220689655173e-05
+ 31.64360689655172
+ 25.77338
+-5.525074620689654
+-4.347569310344827
+-5.55770724137931
+ 9.257432413793103
+-0.3196056827586207
+ 0.2661246206896552
+ 0.723968827586207
+ 0.1688882413793103
+-0.2086634482758621
+-0.09015952413793103
+ 2.06359675862069e-05
+-0.008449997931034483
+ 0.00697492551724138
+-0.000433861
+ 0.0003170682896551724
+-0.0003178751931034482
+-5.09637e-05
+ 26.92778896551724
+-12.75989586206897
+-5.647741448275863
+-9.191789655172412
+ 15.98697724137931
+ 0.4519723379310345
+-0.6449695999999999
+ 0.1306479931034483
+-0.2159480620689655
+ 0.2668143379310345
+ 0.1152842344827586
+-0.008449997931034483
+-0.03631898689655173
+ 0.03153816896551724
+ 0.0003170682896551724
+ 0.001028768344827586
+-0.001532291793103448
+ 0.0002567813724137931
+ 35.08112965517241
+ 9.268364137931034
+ 15.95688689655172
+-32.66338827586207
+ 0.1342069310344828
+-0.2533898620689655
+ 0.304689475862069
+-0.05885251793103449
+ 0.07271531034482759
+ 0.03142007034482758
+ 0.00697492551724138
+ 0.03153816896551724
+-0.02414350827586207
+-0.0003178751931034482
+-0.001532291793103448
+ 0.001036574
+-0.0002167911034482759
+ 32.89063862068966
+ 25.24023103448276
+-4.625261310344827
+-28.08210620689655
+-19.92846413793104
+-5.223497034482759
+-0.2298624413793103
+-0.1301227172413793
+-0.1272063034482759
+ 0.1159782896551724
+ 0.2055630068965517
+ 0.148612124137931
+ 0.0009842978620689656
+-0.004301679586206896
+ 0.003520158275862069
+-0.0003976991379310345
+ 25.4544675862069
+-12.69912965517242
+-20.05793517241379
+-15.4712924137931
+-3.449189103448276
+ 0.1557159034482758
+ 0.06717949448275862
+ 0.07351046206896551
+-0.1432978206896552
+-0.2539802758620689
+-0.1836112068965517
+-0.004301679586206896
+-0.03349073379310345
+ 0.02864431103448276
+ 0.0008653670344827586
+ 34.94141724137931
+-5.263151172413793
+-3.490086689655172
+-2.929016068965518
+ 0.6740874068965517
+ 0.4716791724137931
+ 0.4273861448275862
+-0.06191695448275862
+-0.1097388827586207
+-0.07933599310344827
+ 0.003520158275862069
+ 0.02864431103448276
+-0.02192773862068965
+-0.0005837836344827587
+ 31.07277172413793
+ 23.38896413793103
+-1.325949517241379
+-2.868642482758621
+-3.715712068965517
+ 7.188745517241379
+ 0.08750422068965517
+-0.1753217517241379
+-0.8371608275862069
+ 0.0929403172413793
+ 0.1567172965517241
+ 0.09608602758620689
+-0.0005966441172413793
+ 28.52652137931035
+-14.32528551724138
+-3.742328827586207
+-12.56512275862069
+ 17.57024137931034
+-0.08640257241379309
+ 0.03999789586206896
+ 0.4441856275862069
+ 0.1647217034482759
+ 0.2777685034482759
+ 0.1703047724137931
+-0.001111297448275862
+ 34.52513034482759
+ 5.740124620689655
+ 17.56744068965517
+-31.97023310344828
+-0.03360908551724138
+-0.0129896524137931
+ 0.09078313793103449
+ 0.1190839034482759
+ 0.2008087310344828
+ 0.1231223724137931
+ 0.001019690482758621
+ 30.68476689655173
+-15.74472206896552
+-0.434137875862069
+-27.64366206896552
+ 19.5838675862069
+-5.996231655172414
+ 0.0919669448275862
+ 0.02833621655172414
+ 0.0236381
+ 0.1230282
+ 28.37990551724138
+-22.70207724137931
+ 19.65233793103448
+-15.79130275862069
+ 4.96072227586207
+-0.1700706827586207
+ 0.1814652551724138
+ 0.08812537931034482
+ 0.2074673310344828
+ 35.28006965517241
+-5.964822275862069
+ 5.020082206896552
+-3.339309862068965
+-0.83906
+ 0.4133789724137931
+ 0.1631252
+ 0.1272022620689655
+ 29.71888896551724
+-17.88467793103448
+ 3.4985
+-1.943112137931034
+-1.513113310344828
+ 1.970103724137931
+-0.06623523862068965
+ 27.81602
+-22.6129475862069
+-1.415206965517241
+-12.13759172413793
+ 17.27531448275862
+-0.08042344137931033
+ 37.25177241379311
+ 3.421183310344827
+ 17.30402068965517
+-33.93656689655172
+-0.05085664206896551
+ 30.09676758620689
+ 21.83503103448276
+-5.07211875862069
+-27.88579034482758
+ 28.78493862068966
+-19.79339517241379
+-20.4170724137931
+ 36.01380620689655
+ 2.407097931034483
+ 29.40254551724138
diff --git a/matrices/test/a_large_inv.mtx b/matrices/test/a_large_inv.mtx
new file mode 100644
index 00000000000..de77780b131
--- /dev/null
+++ b/matrices/test/a_large_inv.mtx
@@ -0,0 +1,10002 @@
+%%MatrixMarket matrix array real general
+100 100
+ 1.728797896003273
+ 0.02984235769025842
+ 0.17264373825528
+ 0.5413353040529991
+ 0.5987019788418766
+-0.06133662620669985
+-0.1658007574034648
+ 0.0536623875994405
+-0.1118504505468461
+ 0.2139852414077251
+ 0.1202820547507289
+-0.01573780533860078
+-0.1987557120472836
+-0.1315742535354934
+-0.1135950304025646
+ 0.08388398560957668
+-0.02086145661352059
+-0.01851434162863788
+-0.1144460311415288
+-0.1218978668673911
+-0.09296642509642464
+ 0
+-0.06084404244882342
+-0.02226165544957438
+ 0
+-0.1291249595230563
+-0.0004196580305559638
+ 0
+-0.04430486835167831
+-0.06511379180223759
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.03887151572381558
+ 1.371007594485612
+-1.479019543911773
+ 0.1778047148344253
+ 1.342701307435045
+-0.4256722598504703
+ 0.123038733822573
+ 0.9416446789152163
+ 0.2177767962670925
+-0.03894099315167787
+ 0.2439441233070143
+ 0.01884046185662314
+-0.007859346012856027
+ 0.132699957053182
+ 0.2917819688986858
+-0.1452918485984369
+-0.06492839827498387
+ 0.2081036600352797
+-0.07821264142267455
+-0.007955956211488757
+ 0.2345703532460748
+-0.1727784772586945
+ 0.0163173965863426
+ 0.2168467739495741
+-0.1317134734345977
+ 0.01934085786173204
+ 0.1521977308030799
+-0.08940483444452595
+ 0.04607383935768904
+ 0.1275886534960688
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1722073848720678
+-1.479019543911773
+ 6.65516202914633
+-0.3020820244240092
+-1.481331388834674
+ 2.189858250081084
+-0.8193637514436026
+-0.5961021358922119
+ 0.2230931832456023
+-0.01662380009911164
+ 1.016039769902676
+ 0.7685343284549412
+-0.7646946564513161
+ 0.9024738997372448
+ 0.4697439576553524
+-0.01989337125783583
+ 1.009131607356144
+ 0.6728675495624005
+-0.7196004484102022
+ 0.6952833748147713
+ 0.7377810413154007
+-0.2832735456156473
+ 0.5333953942212142
+ 0.8210181222994786
+-0.4616852075456391
+ 0.4423602556727359
+ 0.8641382184607118
+-0.4932300960594916
+ 0.4678470373828674
+ 0.6349279318761307
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.5459337335217194
+ 0.1776908041109713
+-0.3026039885609523
+ 1.543868655944742
+-0.252839810942398
+ 0.7528298111523792
+ 1.575715845005781
+-0.05389825549197692
+ 0.3826598416223621
+ 0.4193681185798917
+-0.2527725796753442
+ 0.09285980891848172
+ 0.5584224487378291
+-0.08641992960391033
+-0.1375068321001848
+-0.01160008728730358
+-0.1677015394419983
+-0.2923207796930615
+ 0.2280761856726538
+-0.07675909360805622
+-0.2839983127426551
+-0.02318437276961686
+-0.1144601999021718
+-0.3576860987098101
+ 0.06067679887465378
+-0.06790891255162879
+-0.3338285752306802
+ 0
+-0.1492957277824273
+-0.1921416847217516
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.6060244418308103
+ 1.342110355024742
+-1.479136340522196
+-0.2552054953313185
+ 1.809240256447562
+-0.8175005286752326
+-0.6534701740466956
+ 1.028664618010117
+ 0.03421872709418972
+-0.1528811156244744
+ 0.3872105636966044
+-0.01229435187183393
+-0.3562792037882532
+ 0.09284810037687727
+ 0.3338990384174029
+-0.1287811587045025
+-0.0213525846007072
+ 0.3571849071192184
+-0.2433514049189923
+-0.03487105645953558
+ 0.3426848862370973
+-0.1846258648903151
+ 0.04294773452474232
+ 0.3901279918355072
+-0.1724827889892474
+ 0
+ 0.3175705458307803
+-0.09462562618834963
+ 0.1003553428222384
+ 0.1963046485834295
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.03194112332014314
+-0.4256722598504705
+ 2.189858250081083
+ 0.7532808353254768
+-0.8131199586084821
+ 2.331486928414987
+ 0.695705758923945
+-0.1113994548569543
+ 1.302081319250873
+-0.2750484870171973
+ 0.1605788551421328
+ 1.153023531012888
+-0.2702212049549417
+ 0.5168597428298998
+ 0.9779101244456775
+-0.5728708894753742
+ 0.4415532294803036
+ 0.8820861700818785
+-0.6056287796724231
+ 0.5887243251705573
+ 0.9399966434156231
+-0.6885107808810331
+ 0.5188258590164004
+ 0.9081590830119071
+-0.6304687541962376
+ 0.6339851353738079
+ 0.8291935180145759
+-0.5457797200917337
+ 0.4945844168919606
+ 0.7090239317474276
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.1548643115252611
+ 0.1283187909078604
+-0.7669449247029111
+ 1.556663158128347
+-0.6333271761689756
+ 0.7384719889110311
+ 2.339382943676118
+-0.0001065160212148971
+ 0.5963851331960092
+ 0.6279942725411847
+-0.3917583190580677
+ 0.1440964444239836
+ 1.107453695318306
+-0.01182009945557227
+-0.07834501956743929
+ 0.0679419614666439
+-0.216910530225636
+-0.3756743311685913
+ 0.5068196422106329
+-0.03115694802541443
+-0.3405316603725888
+-0.005912912673474375
+-0.1328527664382396
+-0.5131425550769596
+ 0.08970623321647432
+-0.02963979113236682
+-0.5151693908271608
+ 0.04450598014532498
+-0.2005617295510455
+-0.2360223686931227
+ 0
+ 0.02799878699771987
+ 0.01670351103262895
+ 0
+ 0.01056026036814161
+ 0.03473662988491884
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.06214300453208339
+ 0.9354852416155403
+-0.6333655451455816
+-0.04346800791317604
+ 1.018207739014784
+-0.1491290059282507
+ 0.004651349919173486
+ 0.9870805986122776
+ 0.06031732304393399
+-0.073017074168649
+ 0.5545683806305945
+-0.05712703096020352
+-0.07322619228097385
+ 0.4178672334807771
+ 0.2461506631216264
+-0.1069772029720746
+ 0.1454387741222299
+ 0.1697745221805722
+-0.1123382964660199
+ 0.1200871335217258
+ 0.2704647273009727
+-0.1605811585681689
+ 0.06919442011509791
+ 0.2443465497469365
+-0.1519811881416389
+ 0.04714624903266704
+ 0.206313637046412
+-0.108691526802571
+ 0.076689334485619
+ 0.1753116438520862
+ 0.02689836434210402
+-0.0499349231320889
+-0.03516699228438441
+ 0.01492532321949855
+-0.04902275905416193
+-0.02040062918325565
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.06928468604523898
+ 0.1924734443062584
+ 0.07983315677017265
+ 0.4221812948475769
+-0.001472128358754511
+ 1.148343406143939
+ 0.6124067137020126
+ 0.06031732304393445
+ 1.562542007797302
+-0.1345247359877685
+-0.3022678978807723
+ 1.308010045684956
+-0.005149807322970538
+ 0.07856245850045104
+ 1.227724606229933
+-0.5088031662896678
+ 0.09587377758830851
+ 1.103840998333154
+-0.45438055368298
+ 0.3952985138911458
+ 1.004650543045323
+-0.7166608227284996
+ 0.4560670023299445
+ 0.961512627260656
+-0.6359055324610381
+ 0.6162410698345857
+ 0.7852422454121107
+-0.4697863994653487
+ 0.4723806623419002
+ 0.6974564398587709
+-0.06038582796186161
+ 0.1424683568228679
+ 0.1020060601252538
+-0.03394496616657218
+ 0.1148410759531169
+ 0.09582944384407521
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1953030452416966
+-0.02965729095479014
+ 0.04116522802299643
+ 0.401300639949551
+-0.1371158539648848
+-0.2145046436500752
+ 0.6270693227535503
+-0.07513200669510263
+-0.1412279181340248
+ 1.039563025633924
+-0.01179072206987729
+-0.04267945842198871
+ 0.7509757370003032
+-0.192729615872789
+-0.2570731096430172
+ 0.6312286042010815
+-0.09259807634738997
+-0.2594933056221668
+ 0.4511089701796764
+-0.2042910507868694
+-0.4238955024968028
+ 0.3251012104447056
+-0.2014920740048466
+-0.4587661671841043
+ 0.2209483534798725
+-0.2557027259182781
+-0.4576766001492489
+ 0.1313842251948475
+-0.242224942273893
+-0.3023196843119589
+ 0
+-0.01790472480393308
+-0.0111019205050632
+-0.01918367849360801
+-0.03323219985452572
+ 0.003962150317520989
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1165555251590301
+ 0.2333189149174887
+ 0.9385856196542924
+-0.2295801810968944
+ 0.3632710645090986
+ 0.09109638795382748
+-0.3815018697566478
+ 0.5545683806305947
+-0.3022678978807726
+-0.006903580998730262
+ 1.235247886043665
+-0.06544717903866899
+-0.3940861174553016
+ 0.979664455568662
+ 0.0536619717834175
+ 0.05001162286366274
+ 0.7256359149525634
+ 0.09879975328867614
+-0.3172820093146727
+ 0.4279442699028732
+ 0.3045347633422155
+-0.07702523243352963
+ 0.2238763132387307
+ 0.3146624381356733
+-0.1892413218863546
+ 0.08237855119203366
+ 0.3760905370538894
+-0.2267293795598638
+ 0.1701778837107313
+ 0.2805875773989385
+-0.07226225483033642
+ 0.01015574299629869
+ 0.08422307232681754
+-0.06659886878599451
+ 0.01100800859911553
+ 0.04196913255281343
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.01795378104051825
+-0.007225052258570033
+ 0.6134912728331405
+ 0.1364541865059864
+-0.05300715862378665
+ 0.9922806221515482
+ 0.1620644139475202
+-0.05712703096020362
+ 1.308010045684956
+-0.03491916990466497
+-0.06544717903866892
+ 1.276304377245051
+-0.158888876164147
+ 0.171723591885009
+ 1.152924647695593
+-0.3563809069216154
+ 0.2598826412742355
+ 1.120245072661743
+-0.5187424910153317
+ 0.434162604005829
+ 1.002486227576209
+-0.6317743329737705
+ 0.4817480247581714
+ 0.9972127273418497
+-0.621396080289389
+ 0.5814427918308293
+ 0.8439335692507614
+-0.4827632948384918
+ 0.4895571926735098
+ 0.7135399982605599
+-0.08864306353318552
+ 0.1547337046567904
+ 0.1314634352111773
+-0.06012774631348366
+ 0.1267007380308792
+ 0.1092456137591291
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.2028529906191391
+ 0.007663472295111349
+-0.6332636843171476
+ 0.5169315264040835
+-0.3173450425053083
+-0.1558395382070967
+ 1.084285714379934
+-0.06547137228543966
+ 0.0266780815608822
+ 0.7332098444522026
+-0.3763956247787405
+-0.1242155162118755
+ 1.42477130620432
+-0.03322272634792538
+-0.09065313938675719
+ 0.583702139638498
+-0.179268237092648
+-0.3263652957012251
+ 0.8887839228231826
+-0.05040105165255948
+-0.3636072270334467
+ 0.251104179776346
+-0.1632242422562436
+-0.5741749931001503
+ 0.2191171181716481
+-0.1131319404941427
+-0.6338434442048085
+ 0.2186396947882962
+-0.2385042704777418
+-0.2787172937017617
+ 0.0241845147187668
+ 0.03321249096412943
+-0.02467524025443448
+ 0.02944868388425967
+-0.006436125630076017
+ 0.06009197189737728
+-0.003245039320949559
+ 0.04922592897893927
+ 0.008010121409948457
+ 0.0006142484464413057
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.1190409236343763
+ 0.1084272001081703
+ 0.7370875781802806
+-0.03752296711683519
+ 0.05014544785049641
+ 0.3485883830073245
+ 0.03635450627617183
+ 0.3978215801543334
+-0.01599259575027843
+-0.1443407574962302
+ 0.9399017447290055
+ 0.0728162523890938
+-0.004198784179322817
+ 1.183763445144277
+ 0.03796101115389522
+-0.02204022590066332
+ 0.8662030509332055
+-0.041441850117764
+-0.1277487150329576
+ 0.6775746606136939
+ 0.2776381400790454
+-0.1336264042768787
+ 0.3307961044653278
+ 0.1791594328316971
+-0.2076840021274133
+ 0.2111524615864875
+ 0.2694634888489146
+-0.2137605212156669
+ 0.211739270514561
+ 0.305333076343939
+-0.06821800749424599
+ 0.04200875764846114
+ 0.09444219322272285
+-0.05352595431136706
+ 0.0333862654504598
+ 0.08280807300772564
+-0.00155562349434738
+-0.0005608038065905657
+ 0.0104663531700386
+ 0
+-0.01917990935633506
+ 0.004972618268719043
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.05949396790223082
+ 0.2381452539354652
+ 0.1241473464989136
+-0.03505002145825253
+ 0.2434348488090495
+ 0.6119931142589278
+ 0.02299079749113648
+ 0.2030434068413912
+ 1.020090053099957
+-0.1520906576471463
+-0.02909870804578817
+ 0.9370444413445618
+-0.02010789563185395
+ 0.03796101115389554
+ 1.389470640568509
+-0.301887395571093
+-0.06443411895985911
+ 1.286845987404684
+-0.327244019465183
+ 0.2128376419210355
+ 1.135011008647634
+-0.6355450551505332
+ 0.3581562159574024
+ 1.102084151619423
+-0.6030131505239967
+ 0.5457420732008397
+ 0.8282210684499527
+-0.4105454067502672
+ 0.4333463722074636
+ 0.7128885865568467
+-0.07840272599139589
+ 0.1794793119502504
+ 0.1165379462304959
+-0.03243975956639109
+ 0.1366803716927953
+ 0.1201622894958054
+-0.008013311414311202
+ 0.07973197662838594
+ 0.001455861826000812
+ 0
+ 0.04942737172755984
+-0.003232228806017471
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.04288288583548893
+-0.1275823476592937
+ 0.09520990641469353
+-0.04525584431676136
+-0.1026072408943112
+-0.4555158460192354
+ 0.04877161811158949
+-0.1000791701098848
+-0.4801744998766186
+ 0.6163727248135266
+ 0.06547496050276826
+-0.3253821433735916
+ 0.5837321121074017
+-0.05649376313981743
+-0.3853678674690451
+ 1.111793352200932
+-0.007016878293356101
+-0.2509982105265462
+ 0.7405632811729503
+-0.2679354327736296
+-0.3935233872615596
+ 0.6690609770018828
+-0.2697159678407115
+-0.418780115297808
+ 0.4120920212671315
+-0.3822617102277461
+-0.4694934670160535
+ 0.2416772952251766
+-0.2764992133374727
+-0.3327147458932168
+ 0.007086003287936471
+-0.08314533451961652
+-0.05915500916687756
+-0.02108410666249156
+-0.08123465042208253
+-0.03458427866845559
+-0.01104559642171011
+-0.02715566418836943
+-0.01970075401570389
+-0.01221492520747035
+ 0
+ 0
+-0.004182834181807198
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.008586647879725181
+-0.1043746040160939
+ 0.7337453112200211
+-0.08524890832254815
+-0.09157725806465791
+ 0.1618409121016899
+-0.1360532630982197
+ 0.1100730720110193
+-0.07223967154837777
+-0.00813909272777652
+ 0.656002571163308
+ 0.08450402846880221
+-0.1260499304720408
+ 0.8633728414990512
+-0.06919246494425754
+ 0.05735426662063778
+ 1.250405452011807
+ 0.06941011798457683
+-0.3281666058208408
+ 1.033954317926505
+ 0.2310481517374129
+-0.1119678690565057
+ 0.6428638595482884
+ 0.1889142834494066
+-0.2688892224744744
+ 0.3872078268630255
+ 0.3956009372076191
+-0.318496924172954
+ 0.4283497863903946
+ 0.4120288148412199
+-0.1845584111608842
+ 0.2304008514330043
+ 0.2768063676751319
+-0.156481290145249
+ 0.221853719553514
+ 0.2081821264553344
+-0.08106148318824526
+ 0.1909056705806461
+ 0.1364519728536071
+ 0
+ 0.1518813587012469
+ 0.09217426351956572
+ 0
+ 0.09937521863646119
+ 0.07247926279690005
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.02736364077998522
+ 0.1505447830470834
+ 0.2919011441364225
+-0.1785190786675974
+ 0.2557899124476684
+ 0.4844789082867951
+-0.264162179779117
+ 0.1219514357167006
+ 0.8736931089032107
+-0.1432283675429876
+ 0.006876932388039775
+ 0.8809945387259679
+-0.2487471141260569
+-0.04433745720435731
+ 1.280840429526397
+-0.1583166984914722
+ 0.06941011798457618
+ 1.343742426952984
+-0.4157980148590785
+ 0.2488222424827422
+ 1.125546526553535
+-0.5349439256841086
+ 0.4122274787833529
+ 1.151359305754395
+-0.5765248462063814
+ 0.5291943984086531
+ 0.9059705022382259
+-0.4344950435328591
+ 0.4813056922292966
+ 0.7319151274866598
+-0.1158910151709987
+ 0.2058682596470976
+ 0.1603313623656453
+-0.07283221029039584
+ 0.1721284970720283
+ 0.1335088512738512
+-0.03188576105323897
+ 0.1177394848145217
+ 0.0291078890182462
+ 0
+ 0.09841018123171189
+ 0.01995229046132583
+ 0
+ 0.02851279632809474
+ 0.02025487642521403
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.14153889361185
+-0.04321574012647134
+-0.4628328408146838
+ 0.1478503655606767
+-0.1722147715711354
+-0.3586192331804536
+ 0.4349888241298457
+-0.0840824531047962
+-0.3210461068877711
+ 0.379611709010909
+-0.2633628403843943
+-0.3800978573166322
+ 0.8441705033140686
+-0.1163475922428522
+-0.2975212464546924
+ 0.6874901272467245
+-0.3115325731332943
+-0.3871523286800285
+ 1.271753902834939
+-0.09263094152560079
+-0.2801621242890019
+ 0.5634946382796335
+-0.1888012589524561
+-0.5037348700069839
+ 0.4948960949556851
+-0.1270868851604219
+-0.6238482306812283
+ 0.4634960870392485
+-0.1870090626447458
+-0.2974903341501801
+ 0.1484081652916892
+ 0.03690599704033456
+-0.1044653167655952
+ 0.1094868076547757
+ 0.0326782154076562
+ 0.0313567679189074
+ 0.01089871015855057
+ 0.08020822792812557
+-0.01406774858140106
+ 0.01714520464612563
+ 0.0655160447080527
+ 0
+-0.007246397200108536
+ 0.03720546106832864
+ 0.03256630890642381
+ 0
+ 0.03860317434701433
+ 0.04443899322419608
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.08248869558429915
+-0.06344390113652257
+ 0.3381025720201776
+ 0.03090283727281561
+-0.1207675369734728
+ 0.1984864361560299
+ 0.08690718213108196
+ 0.06210759202598081
+ 0.1131808783440845
+-0.06736683660315472
+ 0.3189227728518929
+ 0.1425854784492348
+ 0.0566150887246646
+ 0.6394735026746209
+ 0.1258819661040617
+-0.139391716119275
+ 0.9750205365247968
+ 0.1631556139827516
+-0.08027883357766392
+ 1.199323101350634
+ 0.1748841550995589
+-0.2264323494499089
+ 0.8448921089913984
+ 0.05164318429480112
+-0.2763978495471368
+ 0.659799326894971
+ 0.2543762148276674
+-0.215977451469077
+ 0.5781115119931736
+ 0.4237519130831551
+-0.1090455242003818
+ 0.3670918486607246
+ 0.2613765759679356
+-0.08520974791008415
+ 0.3483011979509406
+ 0.2728740774881688
+-0.07933235597927898
+ 0.2986842473697608
+ 0.174981020181665
+ 0.01974040893957393
+ 0.2345220151283462
+ 0.1293567188251034
+ 0.004794239864378021
+ 0.1388787394529855
+ 0.11345116514523
+ 0.01016066744437013
+ 0.0315821084993036
+ 0.04232804490935031
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.04262793679452875
+ 0.1697740756715866
+ 0.3239074915348768
+-0.1589955150043625
+ 0.2261309862723243
+ 0.5007066948691458
+-0.214948498176686
+ 0.2115410913437121
+ 0.7211998935662339
+-0.2834845192518015
+ 0.1977665592046763
+ 0.7105719830712188
+-0.2629846125158853
+ 0.2427681453222948
+ 1.046240369308911
+-0.269129398416762
+ 0.1781872122967659
+ 1.03874274392891
+-0.2769693797170344
+ 0.1818467501316744
+ 1.348877464355057
+-0.4710685462064164
+ 0.1827804089940346
+ 1.298790609075718
+-0.489235520201789
+ 0.3659945487891929
+ 0.9889729466144815
+-0.3787569007976286
+ 0.3429169426648138
+ 0.762384171560255
+-0.1020077718117634
+ 0.1276450699166033
+ 0.1560144816201697
+-0.05974206683086447
+ 0.1024301574385172
+ 0.1101813862095128
+-0.01836366264038309
+ 0.07641275888249587
+-0.01761614764163003
+ 0
+ 0.050498265002489
+-0.03382818752642417
+-0.01253571506931467
+ 0.01296025523973141
+-0.005708212878175765
+-0.00981215029774337
+-0.004490409692104846
+-0.01715525507410304
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.1242102189239938
+-0.006164997849505758
+-0.07153461814739005
+-0.09444773040108892
+-0.387943428575583
+-0.09883882221569014
+-0.1212536944142565
+-0.5443499233026309
+ 0.2446342882844404
+-0.005420177622913896
+-0.45014992349797
+ 0.1805459963951284
+-0.1149731824589926
+-0.5793268006821727
+ 0.5943780316125726
+-0.07606922043140596
+-0.4764497926559779
+ 0.5512217345426982
+-0.2226561596695373
+-0.4692054844105334
+ 1.147985092990181
+-0.3185462804425448
+-0.3501449776185102
+ 0.8920211634493967
+-0.5449794367210983
+-0.2407310953139373
+ 0.4723421378334628
+-0.1986732196713777
+-0.4188640731699343
+ 0.1544747855487867
+-0.1511790483388062
+-0.07192253244542443
+-0.003725697090364132
+-0.0220913989993934
+-0.1468526543626148
+-0.02817327979646708
+-0.05190672898051355
+-0.006325251931142729
+-0.07525145457458117
+ 0.001697678887603216
+-0.003297495636010923
+-0.03246162326438269
+ 0.01013976787993824
+-0.05182845289631759
+ 0
+ 0.01359443233396007
+-0.0172570418314328
+ 0
+-0.02362951566813379
+-0.06285636906706796
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.000541594919091487
+-0.06028000599526159
+ 0.06056598570524592
+ 0.02953064924547113
+-0.06923471480436504
+-0.01891049181226457
+ 0.0268767202425183
+-0.01562155165351305
+ 0.03650851380746929
+-0.00237832052322528
+ 0.07073903656649719
+ 0.05143750940466155
+ 0.0001469834523781262
+ 0.2686749881238329
+ 0.2116796187687435
+-0.06683812836001574
+ 0.5455583762437547
+ 0.2678847282549483
+-0.167042125792995
+ 0.8247502624594285
+ 0.1551014523158158
+-0.3117922666534864
+ 1.120619590522661
+ 0.2284693806323327
+-0.3131000654955992
+ 1.03714362591117
+ 0.3160604958877395
+-0.1596079439909592
+ 0.9010620280244863
+ 0.4324641730719244
+-0.06454934401311917
+ 0.679922410035844
+ 0.2963524652781443
+-0.05129522211593864
+ 0.6632663480410756
+ 0.3423284178957403
+-0.1107026062549143
+ 0.5782401777699622
+ 0.2616483819084243
+ 0.053649067557817
+ 0.4944944436523461
+ 0.2346875580793441
+ 0.009412604728342261
+ 0.2663830511196768
+ 0.1985776610535804
+ 0.09353364223764664
+ 0.1321153188119533
+ 0.145623535922112
+ 0.1356279678672704
+ 0.08179006836250678
+ 0.1360644250839272
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.02042759737388438
+ 0.14674773697916
+ 0.3571674467316939
+-0.2151635263421847
+ 0.2584936211261787
+ 0.4203232824335663
+-0.3717874500107469
+ 0.1762759358398916
+ 0.6311658301753896
+-0.2964611356429263
+ 0.1909165061130751
+ 0.657225660683243
+-0.4531867001845071
+ 0.1325526769425752
+ 0.9883817750068494
+-0.2692533998896378
+ 0.1150545990637859
+ 1.038203565016419
+-0.4927170003660811
+ 0.03960593138417274
+ 1.282143346625596
+-0.3445153039163922
+ 0.2284693806323329
+ 1.405078700186056
+-0.3982457865178985
+ 0.3676514691999223
+ 1.118642037342427
+-0.3632353790080362
+ 0.4366531833183378
+ 0.7381958054401957
+-0.09148445746430431
+ 0.1757834343565333
+ 0.1812147279427315
+-0.08259945569807613
+ 0.1860477709150544
+ 0.08695824951478025
+-0.04007203193791484
+ 0.1310902263239609
+ 0.01224425975682899
+-0.01354391461917368
+ 0.116064418412012
+-0.0026660135676226
+-0.01844037342594609
+ 0.04677930902538735
+-0.001305595979754572
+ 0.01605838430327047
+ 0.0201649811777727
+-0.002160049138104471
+ 0.04090210676561704
+ 0.01895924652034242
+ 0.02537674547112137
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0794175080944723
+-0.1311047519457179
+-0.01295134131576296
+-0.07559746867709317
+-0.3031321226577994
+-0.0167250101736162
+-0.1183792408956641
+-0.4927659850693444
+ 0.1401190848852171
+-0.1237143398676662
+-0.4692096403777944
+ 0.181418878334746
+-0.1981866369376882
+-0.5736157357492635
+ 0.3698033299851458
+-0.2536101970829642
+-0.54766228665201
+ 0.4892305369514527
+-0.2814526191847183
+-0.4947047532446273
+ 0.8928505090759026
+-0.327688421200874
+-0.4104621621980526
+ 1.172250670502255
+-0.2568301162154136
+-0.2639273236043994
+ 0.9350520824684292
+-0.04176686612980885
+-0.5277923034840782
+ 0.5539539180715677
+-0.01357334334667475
+-0.2143289852765108
+ 0.2814394709243907
+ 0.1849574665556867
+-0.1895270004545035
+ 0.07465277417886505
+ 0.03581851864555304
+-0.04109727173380481
+-0.0243041892802158
+ 0.1156761352354747
+ 0.02699474300995193
+ 0
+ 0.07672900850356405
+-0.02789348757478719
+ 0.03521145645185004
+ 0.08216413320042737
+ 0.05323392699582487
+ 0.07677504731300376
+ 0.02637277036688178
+ 0.009008450568351388
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.002449031357407452
+ 0.01063698771912406
+-0.03951029141370768
+ 0.04462750193803677
+ 0
+ 0.07884910728914267
+ 0.08536650861610245
+ 0.009475804678243856
+ 0.1719868342658235
+-0.0592479879569783
+-0.05064675381903107
+ 0.1270869714173288
+ 0.05426840529149266
+ 0.1462614491971853
+ 0.3931622801149885
+-0.1561196057453254
+ 0.2771648063729535
+ 0.3807812949163497
+-0.1168393618182533
+ 0.6321531872584453
+ 0.351925566307628
+-0.5426936855547794
+ 1.039259775053663
+ 0.3868741132933126
+-0.249435243964555
+ 1.382629698050187
+ 0.164181109541688
+ 0.1644820122200089
+ 1.041360421163825
+ 0.3248672077864385
+ 0.2344663252314359
+ 0.8490498640002748
+ 0.05749267565835372
+ 0.2083467709566488
+ 0.8280925821901266
+ 0.284587855665233
+-0.005561163388316134
+ 0.6603409125030985
+ 0.1756532610454609
+ 0.1246051167405868
+ 0.5836584482046749
+ 0.2436892195356578
+ 0.04788885432278017
+ 0.303611716253695
+ 0.226297349852271
+ 0.1299879455460961
+ 0.1829834806585196
+ 0.219223667683021
+ 0.2145616808252643
+ 0.1285160975686417
+ 0.2223254411952814
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.02564316574837235
+ 0.08371517747481154
+ 0.3671095605303641
+-0.1785310249831468
+ 0.1726927191428491
+ 0.3358462516059161
+-0.3742901407073999
+ 0.1400230321552189
+ 0.4677158663396579
+-0.3003473106705227
+ 0.2504525718945521
+ 0.5149115656093095
+-0.5207969019211087
+ 0.225424055730968
+ 0.724717977710952
+-0.3327583709666753
+ 0.3250258488867132
+ 0.802156701576207
+-0.6121335172994702
+ 0.2464418374782392
+ 0.9735928182597602
+-0.2373303887862142
+ 0.3160604958877394
+ 1.118642037342427
+-0.2516947707803998
+ 0.1485335638520558
+ 1.326520892579401
+-0.4194655541706243
+ 0.3882982498164393
+ 0.8056399717916394
+-0.169328788594763
+ 0.07635187279153749
+ 0.4080553581712917
+-0.2026603906857792
+ 0.1524294050383565
+ 0.1424308151616571
+-0.1029091784701195
+ 0.1128569017500071
+ 0.1113983791458489
+-0.06641747451247863
+ 0.1075531590313645
+ 0.01672476748665855
+-0.04518782745843523
+ 0.06231481418275322
+-0.01307968723936047
+ 0.001484739039221722
+ 0.0149712962056828
+-0.03882964864888243
+ 0.004865321932817059
+ 0.002299773128322388
+-0.02438294464646183
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.01264389783262389
+-0.05793713725717359
+ 0
+-0.01261579684494852
+-0.05006292113497802
+ 0.0585927585072927
+-0.07340538142494876
+-0.2427215358456318
+ 0.05006634523565472
+-0.1627117507788533
+-0.2745570272739167
+ 0.248306407115128
+-0.2015951238560101
+-0.3928287134451823
+ 0.1994708994320764
+-0.3127636264823894
+-0.435859056825597
+ 0.4806186549419238
+-0.2198408442070814
+-0.396148696405932
+ 0.4692238185331468
+-0.1708442202474906
+-0.3978663273927521
+ 0.930432539854766
+ 0.1633731603840306
+-0.4518732412182115
+ 1.338716501551583
+-0.109031231306197
+-0.6330931974218327
+ 1.03719425959278
+ 0.07510046575744928
+-0.5609581255009206
+ 0.7608218415125024
+ 0.2272286665406628
+-0.2749021307511318
+ 0.3251220532744498
+-0.01054680707135332
+-0.2404015440217943
+ 0.1379036234693202
+ 0.1040209196000656
+-0.02880779552312983
+ 0.09936456905726107
+ 0.07028003508995186
+-0.005608917362226716
+ 0.07066448103953958
+ 0.1035993647771786
+ 0.1154205139081911
+ 0.1300829939152364
+ 0.07033727032220777
+ 0.0957129161933688
+-0.006814261345230066
+ 0.01461585745164496
+ 0.01685837742394549
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.00507231121252326
+-0.0334372389620036
+-0.04791691334223844
+ 0.01170073395245119
+-0.02837974723460622
+-0.08167688837353176
+-0.02096849855094491
+-0.01884910557216194
+-0.002243854020911905
+-0.01702519522643522
+-0.003216907348460758
+ 0.002354916082877636
+-0.04933345379441739
+ 0.1424011224620444
+ 0.2714259768856127
+-0.04444749129920447
+ 0.3168241265818476
+ 0.3201753443272889
+-0.1607803624895273
+ 0.5540687088944444
+ 0.3088202382126047
+-0.1903857556737995
+ 0.892707884352013
+ 0.4319656203695874
+-0.02496391921617681
+ 1.025673265149956
+ 0.3843395400397565
+-0.0957245301338806
+ 1.115399867562503
+ 0.3250369308001381
+-0.0580146160246742
+ 0.7725722156722602
+ 0.2751758081130411
+-0.1189509965679552
+ 0.8204677432931591
+ 0.2759983002322761
+-0.1726312421635642
+ 0.671089556740378
+ 0.3087886059414974
+-0.00732436305052842
+ 0.5890268703308188
+ 0.2855033144045582
+-0.01636585270555224
+ 0.3041971618836315
+ 0.1841881095436645
+ 0.1017556489695763
+ 0.1712187687886567
+ 0.1698424586463704
+ 0.1856870823553884
+ 0.09359926680510965
+ 0.1429455848671907
+ 0.004824792404718843
+-0.005658603507934124
+-0.007727438637937199
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.008535350870329075
+ 0.03436218398211274
+ 0.03300232988791075
+-0.02027486122495344
+ 0.03178501900563035
+ 0.08271910037477599
+-0.04871758158701333
+ 0.09119210773342638
+ 0.3041839049927936
+-0.105265983370549
+ 0.1203613808526095
+ 0.3051735990987435
+-0.1597839883176589
+ 0.2472992998471992
+ 0.57717032207967
+-0.1816918812299629
+ 0.326090566059919
+ 0.5986739697440269
+-0.2805222441630954
+ 0.4044831103778653
+ 0.7362684071941031
+-0.4076954569175315
+ 0.4293152906834561
+ 0.7349017313331035
+-0.5059247775394392
+ 0.3123106119090459
+ 0.8021560939257572
+-0.6162684314566595
+ 0.3250369308001382
+ 1.293736911368689
+-0.589592335703418
+ 0.2064173428704563
+ 0.7008293193476045
+-0.4120124783025362
+ 0.08829825319868181
+ 0.6240411861437604
+-0.2561903173351185
+ 0.3393547819130854
+ 0.2489666379681048
+-0.008171132063165762
+ 0.1989965810118738
+ 0.04823495701052179
+-0.1179784331019245
+ 0.1280089048537619
+ 0.1661022791808202
+ 0.02743474471261938
+ 0.01410851755710262
+ 0.0004534935036338368
+-0.03657562872520889
+ 0.03150917830501523
+ 0.09186978877707268
+ 0.04874654950035142
+-0.009922681089159321
+-0.001540662677976581
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.04839081910879701
+-0.1533724932760199
+ 0
+-0.1118204749205744
+-0.1775596682999272
+ 0.1761206679997549
+-0.1947787595015865
+-0.3530094722364744
+ 0.1254834104858047
+-0.3028670759341462
+-0.3949753342902071
+ 0.3155324567704582
+-0.2557705668944926
+-0.3728770395367336
+ 0.3185994351178071
+-0.2148313474663342
+-0.3770105515802546
+ 0.6966402185250694
+ 0.06665706026241587
+-0.4218132488437272
+ 1.155253332564081
+-0.2058683201688108
+-0.7897293304092443
+ 1.369406074416903
+ 0.1416940980492992
+-0.586057126944088
+ 1.119398095610936
+ 0.3033668431366793
+-0.4494675578834791
+ 0.5850225104651052
+-0.07562429183869759
+-0.3205173600664772
+ 0.2847469616478119
+ 0.1292144542323355
+-0.03878532080699099
+ 0.2562539010354006
+ 0.08758852186278157
+-0.03488152623822612
+ 0.1578961302862293
+ 0.1589641944723265
+ 0.1548821325166659
+ 0.277038831532137
+ 0.1407671241510433
+ 0.1547446621772466
+ 0.1295460786819906
+ 0.1233035768468524
+ 0.1152643564635355
+ 0.1478203601705918
+ 0.08887262811179811
+ 0.09873564049668686
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.01902953342090278
+ 0.01965841442259662
+ 0.1265945697270666
+-0.05271710049486282
+ 0.0156000894311955
+ 0.1195974737452692
+-0.03985217833185092
+ 0.1458744688078508
+ 0.4028504995681779
+-0.08453875539682189
+ 0.266716064571811
+ 0.4273501790758238
+-0.2023532170425824
+ 0.496196564324543
+ 0.4792055362272562
+-0.3658521136050059
+ 0.8153197991738401
+ 0.555192480497449
+-0.2298297073024665
+ 0.9973792973863952
+ 0.4186410884421446
+-0.1078548800629369
+ 0.9264223390558074
+ 0.4655437864680848
+ 0.1419039825326285
+ 0.9760557505033376
+ 0.02348302557898171
+ 0.2159329769513006
+ 0.8947765913058501
+ 0.1695663675719657
+ 0.04018824386105823
+ 0.760896030546356
+ 0.04350966705869708
+ 0.2034999769190248
+ 0.6619029126435076
+ 0.1600087757042891
+ 0.07117794424532692
+ 0.3332861870356167
+ 0.191628537733264
+ 0.1878675687767862
+ 0.2000059207841889
+ 0.212349004592466
+ 0.2651969823448151
+ 0.151318764645697
+ 0.2676484228302623
+ 0.117981415875424
+ 0.05537361282438801
+ 0.07948171773334457
+ 0.09794739602244382
+ 0.06053278115034896
+ 0.09083288071455702
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.09085332971282412
+ 0.05978363834149637
+ 0.1976865172651971
+-0.09248954946676832
+ 0.1310720502943713
+ 0.2186767233416529
+-0.2219380118863173
+ 0.2430475176972734
+ 0.4332635647390597
+-0.1819976043662894
+ 0.3868716088431143
+ 0.4851023405236292
+-0.3553980578422949
+ 0.4375482080027094
+ 0.534099102090706
+-0.3031934907684789
+ 0.4757017083576593
+ 0.5888193191986006
+-0.4315917860144173
+ 0.2545633777336691
+ 0.7748629380741596
+-0.74056496876128
+ 0.4661925117741225
+ 0.9831970926278412
+-0.5859497207534007
+ 0.02348302557898186
+ 1.010034188861924
+-0.5514792260655391
+ 0.07176293729208492
+ 0.6288809895411019
+-0.3204162216073807
+ 0.1955755093766477
+ 0.5241038244185691
+-0.1519546590028275
+ 0.1443463848507031
+ 0.2017677476041637
+-0.1035429774029542
+ 0.1144045941462229
+ 0.1394107103279224
+-0.004987839176053102
+ 0.02784290326117664
+-0.01667008654217432
+-0.01178931601840801
+ 0.02031677497261586
+-0.008695668815126454
+ 0.02738007463661901
+ 0.005161092704699138
+-0.02636065386541896
+ 0.005232614805629174
+ 0.0003673640906699979
+-0.02757862720253619
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.001265977516501284
+ 0.03841741295073133
+-0.04167332592492751
+-0.03617242351261972
+ 0.01833298368512788
+ 0.1292223108262921
+-0.134512876634508
+-0.2126540722713442
+ 0.04367669117746803
+-0.2271476182104787
+-0.2573014505804312
+ 0.2294318389980446
+-0.2161703510702842
+-0.2468676013148349
+ 0.1276389836558024
+-0.184624307353196
+-0.2934556693105454
+ 0.3647325033474627
+ 0.05647371273761841
+-0.389847489213109
+ 0.831701014264612
+-0.2467186393092987
+-0.5596066617683856
+ 1.107939548920055
+ 0.2049785850400485
+-0.5520003423998263
+ 1.349994034068507
+ 0.01594024446347822
+-0.4037910347393177
+ 0.8744131947608462
+-0.168312936086927
+-0.5246896962887845
+ 0.6187434293443713
+ 0.0110481573045226
+-0.1866475546951414
+ 0.4400441847359451
+ 0.01636340988356278
+ 0.01647118869278042
+ 0.2939800564746345
+ 0.122925080808936
+ 0.1641436371137526
+ 0.3784159065625461
+ 0.2101487442760755
+ 0.2942825807882382
+ 0.2708836080892262
+ 0.2032320349592514
+ 0.1929589394140589
+ 0.2493267945655633
+ 0.1852512168495825
+ 0.2122091500517266
+ 0.1011804975007311
+ 0.07617270255156854
+ 0.06859950746247327
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.04156408495747906
+-0.009413625457485824
+-0.001908283063673741
+-0.03235881826432552
+-0.01912073681995256
+-0.005161219162513972
+-0.03710536463098445
+ 0.1147134982393787
+ 0.3169170472819148
+-0.03477140841636103
+ 0.2329182713081895
+ 0.3504828100457987
+-0.1805411288983153
+ 0.4722354378299338
+ 0.4149042510315208
+-0.2177799641469858
+ 0.7860510563573369
+ 0.5298793691209585
+-0.00146954936280167
+ 0.9608602312416559
+ 0.4630042454320833
+ 0.06438258933790733
+ 0.9589069020648562
+ 0.323559482727886
+ 0.297493086276174
+ 0.8883624746781769
+ 0.07063402817654904
+ 0.01601772212142599
+ 1.124250615451674
+ 0.05367882220159861
+-0.1636647019013005
+ 0.8212916398268981
+ 0.1808195442953317
+-0.0522399159418384
+ 0.7571368822850639
+ 0.2425011442534682
+-0.05416903697572524
+ 0.3895979198219028
+ 0.1026458763609263
+ 0.0887247336091338
+ 0.2320358988610352
+ 0.1779157731134778
+ 0.1931832532340975
+ 0.09404431662211243
+ 0.1307975875509816
+ 0.01598463245592503
+ 0.0008938890564368197
+ 0.0208553825839959
+ 0.0286119173042573
+-0.0123545294229296
+ 0.003550855019223907
+-0.07594704938390318
+-0.05555336112393602
+-0.04280905561114338
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.05280121217906358
+ 0.01611164118581453
+ 0.0588164594406216
+-0.002376085117951147
+-0.0001332900161797361
+ 0.04477317774386023
+-0.02613038538384484
+ 0.1529686768209856
+ 0.2861874624630084
+-0.08453225471387218
+ 0.2374998272222094
+ 0.300351105386283
+-0.1157673564726861
+ 0.369339086596075
+ 0.3402080156109828
+-0.3063546997027202
+ 0.4504112135651073
+ 0.3542410843144292
+-0.3266370098824081
+ 0.4062691109762042
+ 0.3826169155415374
+-0.4006723478592644
+ 0.3922022223778848
+ 0.8057576920889964
+-0.4479905953291173
+ 0.1659315626636352
+ 0.6263400361692182
+-0.4035361818630668
+ 0.05367882220159864
+ 1.095056112031682
+-0.4017927895565523
+ 0.2990987608263373
+ 0.6330248657581778
+-0.1210708033139963
+ 0.1591644075315025
+ 0.3741413486116172
+-0.173983232253032
+ 0.1264646757089649
+ 0.4272285722412449
+-0.006095417404622631
+ 0.01966898116992491
+ 0.1480953928172449
+-0.01034737477959695
+ 0.06301598982388827
+ 0.1884658178456484
+ 0.04735876098662492
+-0.003559905775143694
+ 0.001406703478946554
+ 0.007980584208621525
+ 0.02340521899745435
+ 0.01084599375532393
+ 0.05085265874132012
+ 0.02241093161219722
+-0.01823643489931556
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.07047834705937615
+-0.09178706197740315
+-0.1806119380374749
+ 0.05456253554578702
+-0.1546714601038524
+-0.2004609409211657
+ 0.1278275884832036
+-0.2400102719891815
+-0.2003581215661663
+ 0.1187267439239399
+-0.2684820656957791
+-0.2499258598031065
+ 0.1484161868230236
+-0.1846933219836615
+-0.289851037221567
+ 0.3952477069065902
+-0.3164493254144762
+-0.409400953561785
+ 0.5865325480397021
+ 0.004528568266388114
+-0.3442701615241253
+ 0.8860126061705046
+-0.1804651557630956
+-0.4170074581586109
+ 0.9715111115018301
+-0.09930961048312786
+-0.4138630715505275
+ 0.8992944725994563
+ 0.005479947954454054
+-0.2127794659124976
+ 0.6845921001123547
+-0.0328318429313325
+ 0.01014330942520348
+ 0.5028789222182671
+ 0.1079018464744724
+ 0.1238393787781744
+ 0.6047787355178867
+ 0.2833830324963177
+ 0.3568437547234754
+ 0.5181889214216189
+ 0.3287812859570573
+ 0.2760945251530408
+ 0.4539800982125043
+ 0.3167270060101302
+ 0.3397138539933505
+ 0.3442626432166768
+ 0.2472769794305177
+ 0.2111906874721716
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.04298876770760442
+ 0.1656470384699898
+ 0.3904131316702136
+-0.03876741165644142
+ 0.2882441359656608
+ 0.4262032947733492
+-0.2501579815982157
+ 0.507824039754495
+ 0.5635436308488173
+-0.333240829143735
+ 0.7833772983512389
+ 0.6385678788801107
+-0.2549717350477951
+ 0.88020440469228
+ 0.5721652738467851
+-0.2629926430688403
+ 0.8889896402117655
+ 0.6931102887066188
+-0.1223631714145813
+ 0.8179976769420301
+ 0.256493081680243
+-0.1966193768492383
+ 0.8778998191378252
+ 0.3466950312014125
+-0.09930961048312804
+ 1.166101233710299
+-0.1022447850215565
+ 0.349388866526547
+ 0.8863972041398483
+-0.1091549209839937
+-0.01509858598318699
+ 0.4585686208661375
+ 0.1649514882150246
+ 0.2876760850561434
+ 0.2113634488339027
+ 0.1088910294240312
+ 0.2169181726250745
+ 0.1723118323520176
+ 0.3450819124993047
+ 0.2043784438760465
+ 0.02838233140458319
+ 0.07296110985661074
+ 0.1015756094393812
+ 0.08590034136615242
+ 0.174655909313396
+ 0.09030187281811035
+ 0.01721589616120555
+ 0.04171456418284304
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0664571702085886
+ 0.1160844668278462
+ 0.2007015135261183
+-0.05901834347390401
+ 0.2192033654981966
+ 0.2349169686261112
+-0.10994791091446
+ 0.2867397539468095
+ 0.1645276200973362
+-0.145593858161492
+ 0.3613346796038783
+ 0.2324676783500399
+-0.1691474485829494
+ 0.2762574066522671
+ 0.3145470737627009
+-0.3493995457132565
+ 0.4111865560286941
+ 0.4128247715750158
+-0.3475729498472833
+ 0.06491935023780998
+ 0.5538388461817163
+-0.5395113709675047
+ 0.2050751790009213
+ 0.6544238059103082
+-0.4137940267940007
+-0.1018910180244136
+ 1.155143704743835
+-0.4282431662149232
+-0.01153586971065928
+ 0.8586971293827956
+-0.0004571308712585015
+-0.009554151875950522
+ 0.3384747347116007
+-0.08653918059664382
+ 0.03973832550075845
+ 0.2123053236210994
+ 0.1773577029439647
+ 0.02446852649066788
+-0.05114228946083501
+ 0.02439128398862156
+ 0.08029086587355641
+ 0.01096301783063296
+ 0.1078027163665591
+ 0.0126108751553812
+-0.1057352043449342
+ 0.05438455361609832
+ 0.0859524006153511
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0001699056108734376
+ 0
+ 0
+-0.004845422436498823
+ 0
+ 0
+ 0.03399893669537405
+-0.1056916697545231
+ 0
+ 4.7667677835957e-05
+-0.08770235130704315
+-0.05334950106615528
+-0.06397520539438785
+-0.04215911776267812
+-0.101808841511791
+ 0.1009373800403707
+-0.1313086962195549
+-0.05695079254041767
+ 0.2245086168773425
+ 0.1212501317311327
+-0.1606046239137854
+ 0.5491881176714973
+-0.09492975913657767
+-0.1391286460416374
+ 0.796856859568104
+ 0.2782919869846736
+-0.4415554563133402
+ 1.338507026803915
+-0.006101728093102918
+-0.3694143305002582
+ 0.8419031866141915
+-0.1362069043714237
+ 0.1538130050320617
+ 0.7521759108703118
+-0.03252378404421097
+ 0.06236571915920732
+ 0.724096331771299
+ 0.3186576822930837
+ 0.5399178516334415
+ 0.6909952386757811
+ 0.3272039314646736
+ 0.2734923805563407
+ 0.5140286515670152
+ 0.4069854291422868
+ 0.460730325508649
+ 0.4811926834678659
+ 0.3164939498209386
+ 0.2524609503865199
+ 0.03348305747586157
+ 0.01006239810646883
+ 0.06114029164162799
+ 0.05998271611024859
+ 0.001338483124748731
+ 0.04912046494745304
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1085209650644184
+ 0.2646503201001987
+ 0
+ 0.1893213711214356
+ 0.2879498444872255
+-0.2009640081238166
+ 0.4281626342432941
+ 0.4371005594569758
+-0.2497621878560982
+ 0.6582663112126301
+ 0.5121264071906843
+-0.1198711906948955
+ 0.7609765910572841
+ 0.4690654412758804
+-0.1002461739395601
+ 0.7597955489260516
+ 0.4848546445494995
+ 0.08179759019639699
+ 0.6962877284141019
+ 0.1932658811817052
+-0.02601846651379103
+ 0.7928230325256173
+ 0.1956867648120076
+-0.01974060903410628
+ 0.8703654432154465
+-0.01153007544255585
+ 0.006890590757748697
+ 0.9202205051060298
+-0.06717963058617565
+-0.1445989570500295
+ 0.575840729915004
+-0.01296695494868298
+ 0.1196604190466994
+ 0.3359278697242619
+ 0.07123512734906096
+ 0.1111616962883617
+ 0.1505492904973262
+ 0.1541501267023
+ 0.08306503260253775
+ 0.02635720620932545
+ 0.04028263096832157
+ 0.04508749306113925
+ 0.01264880321662966
+ 0.07283055569823375
+-0.008432530220948482
+-0.03960507746621952
+ 0.006995478512006049
+-0.02999149844018921
+ 0
+-0.03567874045648146
+-0.04545898708063957
+ 0
+-0.02861614831903463
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.04538962554952197
+ 0.107675357276117
+ 0
+ 0.111020873424774
+ 0.1300026401557825
+ 0
+ 0.1528508858404347
+ 0.01930116286217853
+-0.09065371405532635
+ 0.2523975010440672
+ 0.09691874330793647
+-0.04556300805076537
+ 0.2533479249342087
+ 0.1125150892678504
+-0.09161080357497643
+ 0.3026773580806075
+ 0.1258730297624595
+-0.07210696085809201
+ 0.1504064575730214
+ 0.2160374141783687
+-0.210229843258113
+ 0.2421341169464289
+ 0.3840374135064942
+-0.2407508953585975
+-0.1212009403905005
+ 0.8450783671315125
+-0.3686232122822629
+-0.0730486425349951
+ 1.199841296675042
+ 0.05627145492703167
+-0.1502797204523957
+ 0.6439265534811736
+-0.1539626700947748
+-0.01572214341397699
+ 0.590138327794575
+ 0.2769255751858083
+ 0.02277585740376173
+ 0.1974274506083979
+ 0.02719029413563755
+ 0.1191379909843359
+ 0.186067932597198
+ 0.1968526304773224
+ 0.05577548221950061
+-0.0346332570234083
+ 0.07862022965309619
+ 0.1332554542835316
+ 0.04151166294257791
+ 0.07065376766691027
+ 0.01330490650104134
+-0.001424895750481332
+ 0.05467880139161527
+ 0.03265791488082924
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.002430880387288702
+ 0
+ 0
+ 0.06454060691221508
+-0.1292188585389263
+-0.1004557555017825
+ 0.07956807420147674
+-0.163476278549359
+-0.1471784757546287
+ 0
+-0.155581094189555
+-0.1552122239158223
+ 0.111763290253138
+-0.1686926129688248
+-0.2120500253743202
+ 0.2077599741049015
+-0.02875513373862118
+-0.1286681257596934
+ 0.3881560288067106
+-0.1132014361728064
+-0.20343337531731
+ 0.5988616393792288
+-0.08326724808269938
+-0.0283286815398887
+ 0.84189417184656
+-0.1679450780630685
+ 0.05210781861366457
+ 1.053934307197499
+-0.2149148452640063
+-0.2188116524965337
+ 0.6247584856736758
+ 0.0726125804710796
+-0.02726755463868002
+ 0.9091944714062363
+ 0.2614490815361376
+ 0.04573100887074542
+ 0.617990864224173
+ 0.4321554257708766
+ 0.2190480994108906
+ 0.5990112445827741
+ 0.3346493942569108
+ 0.2201763503838938
+ 0.4323478926743816
+ 0.3753685693181499
+ 0.2903969145719552
+ 0.1130964268164823
+ 0.01599698482811726
+ 0.06237844831106225
+ 0.08358413439215608
+ 0.05437458637583435
+ 0.09450120627341284
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.00385948709995182
+ 0.01102549788002184
+-0.1011736071615975
+ 0.2151046886620052
+ 0.2118138107836209
+-0.110362891478852
+ 0.3138744827558007
+ 0.2577026943498817
+-0.03269933606194544
+ 0.3545843845759463
+ 0.250317993917445
+-0.04688563857617225
+ 0.374334603075552
+ 0.2728148247678721
+ 0.04454045659904322
+ 0.3309847399048657
+ 0.1392600549785949
+-0.02435385942559752
+ 0.3992693326882689
+ 0.1417330484974973
+-0.07617530889133121
+ 0.433779724645776
+-0.01262688887782614
+-0.1355960721621477
+ 0.5764119807149598
+-0.150204486574641
+-0.2115590438374359
+ 0.8227805803246482
+ 0.03565526401580574
+ 0.09959669123005294
+ 0.6055311575571114
+-0.06323542097002967
+ 0.03237569588621741
+ 0.4044293691814657
+ 0.1409239382676068
+ 0.2169644685830129
+ 0.2262215144232321
+ 0.03053394200815901
+ 0.1098233393981265
+ 0.1416087735957518
+ 0.1459160346128377
+ 0.1637332794437391
+ 0.05389734031672933
+ 0.05637226947255175
+ 0.08983151834421278
+ 0.07929666985483162
+ 0.1093773294191723
+ 0.1183986603232397
+ 0.04125733401916568
+ 0.0810053017584091
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.03357859734504264
+ 0.01757022011177168
+ 0.003466294787839106
+ 0.09012183619175529
+ 0.04498644338178364
+-0.1270881823980927
+ 0.1986035380639484
+ 0.1050948660621651
+-0.08652273754340772
+ 0.2182676091372718
+ 0.08708086379790275
+-0.08401719911772791
+ 0.1859975288705404
+ 0.2260281556458444
+-0.07157284011235411
+ 0.1789343395376638
+ 0.1544717121816253
+-0.02227166602906977
+ 0.1027072284127585
+ 0.4374506104438563
+-0.03738097746780947
+ 0.13824317104179
+ 0.3357125090881536
+ 0.1595183493815381
+-0.01345327289208848
+ 0.6431489891510632
+-0.2204083970959405
+ 0.04012221821402379
+ 1.131173049424324
+-0.001547524760757846
+-0.03626361392210729
+ 0.6537740384547847
+ 0.02970973521581357
+ 0.1997024288685219
+ 0.8039367998884733
+ 0.1796254752049998
+ 0.07839428679467762
+ 0.2546391258434929
+ 0.1286535968751069
+ 0.2027236752481019
+ 0.2939620048875812
+ 0.2175859097828638
+ 0.1067480338313124
+ 0.02043925297491477
+ 0.0345177236688381
+ 0.04682865105230121
+ 0.06561290556047709
+ 0.1057925742925759
+-0.007824032709891963
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.000502391620047372
+ 0.03876819663916783
+ 0
+ 0.04010299772346695
+ 0.03741174531763849
+-0.05682410902506225
+ 0.05552716396535357
+ 0.02495393276570052
+-0.01329593994229536
+ 0.03869325690384304
+ 0.05288230309136639
+ 0.07839257873930496
+ 0.1411697975875877
+ 0.01213158323838526
+ 0.2050742917246025
+ 0.07311102212931213
+ 0.004699870741759046
+ 0.3783075488527717
+ 0.2364397064377172
+-0.08623781279439267
+ 0.7241398117343247
+ 0.1223190911164313
+-0.1444531337831893
+ 0.5992605856935499
+ 0.09641554967922984
+-0.006368029179466047
+ 0.8561087888882241
+-0.02789752018723617
+-0.1406959768195478
+ 0.7941927096812281
+ 0.1751902243149739
+ 0.3187749850732784
+ 0.7557548161844081
+ 0.1725075338558075
+ 0.1166314730395731
+ 0.5338221869728753
+ 0.2986534005313589
+ 0.362922199615957
+ 0.4819691275051016
+ 0.2537080265098239
+ 0.1941432443278794
+ 0.1040307738022147
+ 0.08100174390035271
+ 0.1500731817934467
+ 0.1425662561266299
+ 0.05166344353127891
+ 0.09853516106283801
+ 0.02595958692636278
+ 0.01503903900873669
+ 0.05984287335593675
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.05189900256332961
+ 0.1287215894679653
+ 0.1184074643896142
+-0.05800677777040057
+ 0.1716248024134196
+ 0.1302582241231164
+-0.002738623158752699
+ 0.2012206066581327
+ 0.1275021243846876
+ 0.01756256547024469
+ 0.2263675274348328
+ 0.114790620481475
+ 0.09847980583555489
+ 0.196553234845746
+ 0.06311811843547423
+ 0.07085555819409999
+ 0.2470033972931183
+ 0.04163497264003076
+ 0.04783646143460571
+ 0.2074216786174097
+ 0.04670323668927027
+-0.04772698785693049
+ 0.3487901810855591
+-0.007852099124050351
+ 0.05943531125856037
+ 0.6096430338580853
+-0.03381564629962561
+-0.03083635370846946
+ 0.6898366202010748
+-0.06752850136627633
+ 0.08570459413546069
+ 0.5214766738523933
+-0.0535196593937517
+ 0.1623117700791414
+ 0.4282550764717329
+ 0.04221342789562775
+ 0.1437264067358977
+ 0.1979746554503337
+ 0.05089487784798567
+ 0.1643564892624744
+ 0.1451469641075834
+ 0.1000567460984291
+ 0.09621898399373256
+ 0.04630248372237988
+ 0.06921294798292373
+ 0.09899229269744673
+ 0.03807295978199458
+ 0.08684454228003274
+-0.01489086855714524
+ 0
+-0.02999980104815088
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.007075195322054841
+ 0.02738735802773061
+-0.01733476427916956
+-0.07964262769454435
+ 0.1396552230003691
+ 0.05926499270543043
+-0.0124322540564469
+ 0.1774823010979733
+ 0.03884015940899863
+ 0.03933965076011223
+ 0.1438670087138226
+ 0.04127192103064878
+ 0.09877135869400641
+ 0.2100403618493151
+ 0.0175839514134552
+ 0.11366423864029
+ 0.1822319353402183
+ 0.1656083357545077
+ 0.06180512795436149
+ 0.1007214298370139
+ 0.2125030722543799
+ 0.05868687236796716
+ 0.08101439906305402
+ 0.5959287411961318
+-0.03019991302685349
+-0.05608966975373837
+ 0.6590642178954187
+-0.1444314859509232
+-0.06705199105552033
+ 1.132775562530928
+ 0.2291602937990059
+-0.08413284959657472
+ 0.8464007681568506
+ 0.006415080853831236
+ 0.001705210503621362
+ 0.6220054582715917
+ 0.2502210105932713
+ 0.05739069933816202
+ 0.3016084166295029
+ 0.07957038912786035
+ 0.1093860447751527
+ 0.1893505693532784
+ 0.1002111933069782
+ 0.01280701817027201
+ 0.003780374164554648
+ 0.05297034512980268
+ 0.05327073070199739
+ 0
+ 0.05809859641459913
+-0.007718466434749396
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.01548737248812964
+-0.006256316910406804
+-0.01766452893334486
+ 0.02601443039933989
+-0.01141011042074277
+ 0.0153112257785901
+ 0.03584114746026149
+-0.02393866766908594
+ 0.08723380862035857
+ 0.08705159189360627
+-0.002224284169062851
+ 0.1498089053989104
+ 0.08625699187094234
+-0.02654418169243006
+ 0.2735142696755783
+ 0.05866609619823041
+ 0.1205344509541377
+ 0.4353095262476621
+ 0.02584384186749775
+ 0.206570253352424
+ 0.6348045371416045
+-0.07896376175673758
+-0.07117512932599609
+ 0.5463912306157316
+-0.03199165789435739
+ 0.1198077969977105
+ 1.161441590407976
+ 0.0813318026957859
+-0.1770399721456406
+ 0.6706248448529601
+ 0.3424286663160775
+ 0.09823052791434819
+ 0.7377361371850927
+ 0.234786907587239
+-0.01372008184180629
+ 0.4387882315534741
+ 0.3865142228958408
+ 0.2240134945194698
+ 0.2426054559749924
+ 0.05490494407309857
+ 0.02046701572660201
+ 0.1593539093187775
+ 0.1389423956389921
+ 0.157014142313091
+ 0.1101250207754176
+ 0.004835609033077067
+ 0.05852183642727603
+ 0.08188734813336776
+ 0.008606837565868072
+ 0.03784996162062325
+ 0.03694595663118083
+ 0.02388848376320487
+ 0.02686588913184961
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.002844274986378605
+-0.00188993012563363
+ 0.007738809779907678
+ 0.007591977958832679
+-0.006065401752788766
+ 0.009938725357020313
+ 0.004739655543235359
+ 0.04271992245037386
+ 0.03867029196142101
+ 0.006531392625821528
+ 0.003339269564115307
+ 0.01883202564494896
+ 0.0454416792865907
+ 0.02336531206556094
+ 0.0366282963654051
+ 0.04291183056202293
+ 0.04116233208144585
+ 0.00101844246155105
+ 0.09184711420209318
+ 0.08630246183606134
+-0.02570264048004044
+ 0.05330648805999988
+ 0.3146548779410448
+ 0.1021551393888595
+-0.009846203091272253
+ 0.435472810441282
+-0.161506729908956
+ 0.08018365727695444
+ 1.001215192569035
+ 0.09359832618602337
+ 0.2820144790759239
+ 0.8754264561707625
+-0.1086810328846377
+ 0.1132605184805106
+ 0.5927284183845087
+ 0.08085067602115915
+ 0.3867833525246124
+ 0.4024844086036321
+ 0.05833743060812938
+ 0.1457877632318591
+ 0.1648253938231204
+ 0.1685258044833242
+ 0.2716797203445901
+ 0.06803924330202783
+ 0.1422836890139411
+ 0.1085831682369393
+ 0.05503839237163627
+ 0.1019338909307314
+ 0.1394152478505082
+ 0.06818513443803768
+ 0.04055897799669581
+ 0.09392200512087777
+ 0.02337336949809378
+ 0.009686731587931619
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.06783193840490404
+ 0.129577989223027
+ 0.08345162017492355
+-0.05119333463445028
+ 0.1645939824183541
+ 0.05719929501774401
+ 0.003422138668432176
+ 0.1070377833082572
+ 0.1108085172124105
+ 0.06779728706460261
+ 0.2304846242685355
+ 0.02574180195632471
+ 0.1620172548007228
+ 0.1247791594087073
+ 0.1961383811404037
+ 0.1648294251860192
+ 0.2672877541612674
+-0.034393843652069
+ 0.3903060449090177
+ 0.1453632782979608
+ 0.1839257150088484
+-0.0881614787627971
+ 0.09606777396035704
+ 0.7422094976712401
+ 0.1949453288291831
+-0.1020166327860796
+ 0.8052337001634613
+-0.1800941292092771
+ 0.09359832618602326
+ 1.44421799508394
+ 0.1325504766499633
+-0.124162675286915
+ 0.62716933965087
+ 0.02305466536222749
+ 0.1937323830642648
+ 0.743556607901823
+ 0.1726401646265668
+ 0.00243459822989726
+ 0.1547275333142177
+-0.03797284511449882
+ 0.06889670828931663
+ 0.1777119521700858
+ 0.07347703114765824
+-0.03458270083885566
+-0.04897397584694433
+-0.006465196303931979
+ 0.01886736357608327
+ 0.03214342683901935
+-0.05741383601396096
+ 0.008568494127223819
+-0.03393388442068972
+-0.01844153602843652
+-0.01842870152401585
+-0.02863851742975522
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.02554469844355501
+ 0.01093192619465817
+ 0.03263513747779486
+ 0.02280288764845734
+ 0.09017486063392247
+ 0.035894417551042
+ 0.05260363896847221
+ 0.07521148092628134
+ 0.05312964043478007
+ 0.1265187816160211
+ 0.1382466401751607
+ 0.01472936406164443
+ 0.3486737496769006
+ 0.1212279369519054
+-0.001144349249707701
+ 0.2901614257391781
+ 0.1457747348783731
+ 0.06949616881542359
+ 0.4847489616807867
+ 0.0698750482649092
+-0.07330029993295929
+ 0.6618905017578627
+ 0.2708603721743124
+ 0.1638877412459054
+ 1.005053477595991
+ 0.06809129543204773
+-0.004187962474546293
+ 0.7058441500038859
+ 0.1576551090737128
+ 0.3333629761755108
+ 0.6488534780213168
+ 0.1361053050385518
+ 0.1428753500433556
+ 0.274915166860708
+ 0.1002769657690162
+ 0.2431804298510564
+ 0.3014929491629826
+ 0.07085313692086945
+ 0.1480467413205259
+ 0.1410596576101275
+ 0.09183441001980457
+ 0.1755913638111835
+ 0.1174185904423381
+ 0.1074742386075741
+ 0.04091262321196795
+ 0.09325512455723538
+ 0.03400061154270452
+ 0.02254680678432682
+ 0.0495775949969639
+ 0.05039492715638241
+ 0.001338535593531882
+ 0.04721851103231973
+ 0.01479416264985075
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.02719085066971012
+ 0.05039040521446116
+ 0.003069326800424954
+ 0.03829703597379716
+-0.005988614088832385
+-0.004305540622714232
+ 0.09418394149417418
+ 0.02431955618130308
+-0.01080134870398796
+ 0.1168309705283516
+-0.01074946836358617
+ 0.05885424620590181
+ 0.1305585507719302
+ 0.02699263806741721
+ 0.08988321139318869
+ 0.2382441698402682
+ 0.1792966249737566
+ 0.01882767696749361
+ 0.01595988235750899
+ 0.3688787610865161
+-0.05265026825612562
+ 0.35699815423812
+ 0.88287566503563
+-0.09933848239951538
+ 0.09723732952363413
+ 1.045416501622531
+-0.05140177661392307
+ 0.1361542442909363
+ 0.6466494403347459
+-0.1169684256893111
+ 0.2757756573702421
+ 0.5542643308925466
+ 0.1063918631621967
+ 0.1358956492751874
+ 0.1452414681993598
+ 0.04778434550561733
+ 0.2012134793972644
+ 0.1116623666045253
+ 0.1535833144092785
+ 0.1014559797302281
+ 0.009317896682828012
+ 0.04402775324976108
+ 0.1298005541460214
+ 0.01694100505231139
+ 0.04183312190060656
+ 0.06852278025977013
+ 0.02040820771576903
+ 0.01314608838117919
+-0.02127114799750985
+-0.02472711777334497
+ 0
+-0.0227161756279232
+-0.005409324727473289
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.03359567242628882
+ 0.0009416906988700777
+-0.03301721531828659
+ 0.08722351073027727
+ 0.1256464758223607
+-0.01402953052340551
+ 0.1138125029679342
+ 0.08618612671757166
+ 0.01601099956517457
+ 0.09541907061710526
+ 0.07447224605501546
+ 0.01005259271384147
+ 0.1433900074217745
+ 0.09239845346131699
+ 0.1797950906641503
+ 0.08478135391631929
+ 0.02257006789980576
+ 0.2466568960084592
+ 0.01818868273767285
+ 0.01693962433124771
+ 0.5958923530485694
+ 0.1162042056026564
+-0.1003079024749304
+ 0.6579539990509564
+-0.001923924440046667
+-0.0594363392914588
+ 1.052041173101753
+ 0.2432922166141848
+-0.1248474394877834
+ 0.7305152105314757
+-0.008677432188295912
+-0.02091702330619525
+ 0.6220984119226953
+ 0.1230393123724863
+-0.03954570066228136
+ 0.2699228490102467
+-0.00375649745536433
+ 0.04650962316355828
+ 0.1949238000983556
+ 0.08366138428735601
+-0.002196128160013387
+ 0.04181013311074262
+ 0.0446667113514202
+ 0.0009845415427509263
+-0.008661884794539915
+-0.007403207049832998
+ 0.04724851925862013
+-0.01622937854974075
+ 0.05885015592345631
+ 0
+-0.02653233251812281
+ 0
+ 0
+-0.0313074328233286
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.04443532576353969
+ 0.07987153936466021
+ 0.0133144908537148
+ 0.06164600267163708
+ 0.08890593628674756
+ 0.02347393420801132
+ 0.1056967008524369
+ 0.0786163543230274
+ 0.0674238465963372
+ 0.2149843653471525
+ 0.08638930540713126
+ 0.1615018982691128
+ 0.2985822266575806
+ 0.05786774619143967
+ 0.04454312992528011
+ 0.2827374360702383
+ 0.0547697773496413
+ 0.1695079076212276
+ 0.7231832705709893
+ 0.09820842399807266
+ 0.04937925227258344
+ 0.7091669859631984
+ 0.1072346971559732
+ 0.2391228009461467
+ 0.9524070828574946
+ 0.03256140082895305
+-0.07264633325348854
+ 0.55080682690334
+ 0.2533353853705229
+ 0.1018126264342643
+ 0.445369858677944
+ 0.03310284519518947
+-0.08835940604004454
+ 0.2967259102815245
+ 0.1678079210559351
+ 0.1120530835601668
+ 0.2178500932454669
+ 0.02583670347915453
+ 0.04125982180005652
+ 0.1940027694837068
+ 0.07113400998985361
+ 0.02632739649220729
+ 0.1307089447874926
+ 0.07643676101022792
+ 0.01005116657762971
+ 0.1051753821762121
+ 0.05630278943435253
+ 0
+ 0.0727682356843497
+ 0.05967919799796139
+-0.02671286238206237
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.01624572879463861
+-0.02735457553214491
+-0.005388627837351252
+ 0.05973791180183613
+ 0.001801183835953749
+ 0.020652900860863
+ 0.08324318030025669
+ 0.02539044818442687
+ 0.004710846938777422
+ 0.2016208916291706
+ 0.0003640686112277541
+ 0.02859546678308026
+ 0.1355774517421767
+ 0.08952512916281873
+ 0.1209517671115299
+ 0.1323404098083534
+ 0.1368539961899758
+ 0.0005781157360242165
+ 0.2521512796015071
+ 0.6046856600277906
+ 0.2189095993748628
+ 0.2006652488781268
+ 0.6530815661382141
+-0.1135984551911603
+ 0.06935643654433438
+ 1.00532787199056
+ 0.03535648364861795
+ 0.3471011939069352
+ 0.7952080265797263
+-0.1570396501954827
+ 0.01168043310260427
+ 0.4251950406084689
+ 0.03298793335876931
+ 0.2914671585570769
+ 0.2255720806488196
+ 0.03549319748342191
+ 0.09873387907567742
+ 0.09450430253559645
+ 0.1036872476143574
+ 0.1578440663132309
+ 0.06114867060485678
+ 0.0401418043332418
+ 0.1149408895057952
+ 0.02154756827161327
+ 0
+ 0.04063956890685241
+-0.0001247623525372813
+ 0
+ 0.03197869231522771
+ 0.01763249728380177
+-0.01565430430043834
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.05220692012735471
+ 0.1253788605357762
+ 0.01229466546377449
+ 0.09079581850175844
+ 0.05892721307849445
+ 0.0491210503746056
+ 0.104632982227878
+ 0.1368473250615786
+-0.06015497188019826
+ 0.2930897309447529
+ 0.1278547450344229
+-0.03123375722562478
+ 0.0566230375063962
+ 0.1240616644336497
+ 0.2696773159006211
+ 0.2555387844416537
+ 0.02758017598526624
+ 0.2806335400603563
+ 0.02597370083105147
+ 0.1115742319417055
+ 0.7955976300775014
+ 0.3572605617794724
+-0.1113305855428926
+ 0.7358120792042415
+-0.05214475270774152
+ 0.03668653008576597
+ 1.254781381222356
+ 0.127994052522152
+-0.1615571479698871
+ 0.682748798013776
+-0.07657979485747604
+ 0.05491073694347617
+ 0.7011625939312749
+ 0.01825866907591683
+-0.06681456339805611
+ 0.2513558963933228
+ 0.002842422286866354
+ 0.08880671609258875
+ 0.2244979107681655
+-0.03423249329050257
+ 0.05632343773017986
+ 0.01351592283849615
+-0.04344709806894392
+ 0
+ 0.0002735438450783366
+ 0
+ 0
+-0.02551307852177572
+-0.02537277202089864
+-0.05133185997823112
+-0.003098647262690865
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.002999268739130802
+ 0.01323048343016893
+ 0.03097662882462609
+ 0.01957871134796739
+ 0.03783404040762516
+ 0.003539791561315337
+ 0.1674458338494071
+ 0.07030321649356418
+ 0.02976044388683035
+ 0.1109069667492609
+ 0.1014659947163603
+ 0.08216976133492207
+ 0.2236817865781759
+ 0.08302883631038403
+ 0.01248251944768962
+ 0.4757412668439323
+ 0.4012393432199489
+ 0.2415110584065674
+ 0.7113721312990908
+ 0.2811316997552983
+ 0.02658793970352381
+ 0.6041342499741148
+ 0.3369271848283455
+ 0.1419634675013042
+ 0.9601304554553071
+ 0.100983226394245
+-0.005865955587796385
+ 0.514252672301677
+-0.003027593411506199
+ 0.2457352071059557
+ 0.5727051175042291
+-0.05883341501980788
+ 0.1127159735222335
+ 0.3395445366044306
+ 0.09742602547335345
+ 0.2769490405800476
+ 0.2615990573783599
+ 0.2036001411839041
+ 0.05319003934855725
+ 0.2285654230469713
+ 0.03948048163846814
+ 0.01226500713979149
+ 0.1147379547652279
+ 0.1025572216529628
+ 0
+ 0.117744458865304
+ 0.04330552873553968
+-0.05973413307192126
+ 0.03294391440895734
+ 0.04868910947746517
+-0.03406133765767734
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.079438940220537
+ 0.03507773355349241
+ 0.005415152370955367
+ 0.1129211517608453
+ 0.02615209301099764
+ 0.04551620548637231
+ 0.1937946780520654
+-0.003942514568841414
+ 0.1076259235729761
+ 0.2391440824264427
+ 0.04693419032649682
+ 0.06853665531242351
+ 0.1381995010699643
+ 0.1176433114407272
+ 0.07550222283765198
+ 0.415864251506007
+ 0.4153786154608805
+ 0.04606105087996236
+ 0.1786389759294371
+ 0.5649587853986004
+ 0.004163438568025271
+ 0.2895516423334366
+ 0.7942479411513815
+-0.1437920941410486
+ 0.1009832263942448
+ 0.9102231757316858
+-0.03147865701739942
+ 0.001352361556547442
+ 0.4341978203221831
+-0.1389024765201054
+ 0.1294875464652517
+ 0.3721282619202069
+ 0.09418580962100823
+ 0.05367029646388479
+ 0.04891810797132594
+-0.0134197760949454
+ 0.1428802322141913
+-0.01721146247314263
+ 0.04320651234409259
+ 0.06075985079125625
+ 0.04982287850053625
+ 0.01287079476422323
+ 0.0427022475021921
+-0.02356580305379023
+ 0
+ 0.002317309702867352
+ 0.03098975383448471
+ 0.003071777146847332
+-0.01879743840385957
+-0.02826806021171154
+ 0.01716112459159353
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.008692154897039589
+-0.007202037451541151
+-0.03630768997496613
+-0.007183576537927175
+-0.03680673853316485
+ 0
+ 0.04690237874870501
+ 0.02936919933211182
+ 0.02448140162092663
+ 0.1006740129810938
+ 0.003144777181998707
+ 0.01710537269905281
+ 0.06345066720220653
+ 0.06160486506885934
+ 0.1453034093664816
+ 0.3029975792341374
+ 0.1000124643530668
+ 0.2146140225117773
+ 0.200341431317792
+ 0.1401893519329894
+ 0.6564756469208477
+ 0.1619605079455435
+-0.1505903240220231
+ 0.7085839460119159
+ 0.01123885279133603
+-0.01939884048686782
+ 1.337805882791866
+ 0.1544141455550043
+-0.1646631978092379
+ 0.9385684144724438
+-0.07650868372102781
+-0.01326276584079039
+ 0.8161746787054426
+ 0.1812990293450166
+ 0.009986092402126789
+ 0.382915770202775
+ 0.1107386144479993
+ 0.003246143315376855
+ 0.1841345072967435
+-0.07919364067533503
+ 0.0296682494809771
+ 0.1356434127687079
+ 0
+ 0
+ 0.009165905033973215
+-0.1054779771495195
+-0.0476328411044768
+ 0.01981822023362148
+-0.04831804109343124
+-0.03110514781220774
+ 0.0009437336067497957
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.09157794961322638
+ 0.04552403767991522
+ 0.08301970314630944
+ 0.1261595837220865
+ 0.05017685277129666
+ 0.02632516756527461
+ 0.09460463889830099
+ 0.06318522766956154
+ 0.104797775015939
+ 0.4627293331006117
+ 0.2274053953948121
+ 0.1083934116897574
+ 0.4896219062755338
+ 0.2422904944011025
+ 0.2285137224955471
+ 0.6357808913245143
+ 0.1133618741619285
+ 0.03475126970816138
+ 0.6549649110910292
+ 0.1094948693541356
+ 0.2236573295074248
+ 0.7552606158887939
+-0.128385235158096
+-0.1401579396572372
+ 0.5296801163074781
+ 0.06105477046768913
+ 0.03771564878079885
+ 0.4440031397010384
+-0.01780954695783034
+-0.01939578623755676
+ 0.3966991621601017
+ 0.1333360605970911
+-0.07799568175789308
+ 0.2957397591753587
+ 0.1565410931515668
+-0.1108610600518473
+ 0.2460760425241152
+ 0.1597246545719099
+-0.08874082911100292
+ 0.1855908033273366
+ 0.1656452068494594
+-0.1322777529016692
+ 0.1399312304596164
+ 0.1184973242261769
+-0.07510263492719821
+ 0.1112634359128789
+ 0.100048306294601
+-0.006573942478301531
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1708283735131918
+ 0
+ 0.03084266122920517
+ 0.1222787486485877
+ 0.05116937041229231
+ 0.08374035806868128
+ 0.1447751689317241
+ 0.03700001207238644
+ 0.04374488378930881
+ 0.2159563796563574
+ 0.2265500651337879
+ 0.1969255485162078
+ 0.2584285474079688
+ 0.2191670239103262
+ 0.03583522255544845
+ 0.1629698984801329
+ 0.5082227271618474
+ 0.1521520144867269
+ 0.1058810339123703
+ 0.5129812368297004
+-0.1111604965895913
+-0.1229912018376137
+ 0.7965867676866533
+ 0.01327840202944289
+ 0.1018792065650133
+ 0.6201750289874151
+-0.09747480532566091
+-0.09438279372380087
+ 0.3059316766167737
+ 0.003892881252238821
+ 0.1097149363450821
+ 0.1412091349493155
+-0.0314688123575766
+ 0.1015824543477299
+ 0.0999937643982426
+-0.05252011010158934
+ 0.121178267867465
+ 0.02182586869951869
+-0.04751090163830349
+ 0.09136850420447547
+ 0.05170251431820073
+-0.04234802553344709
+ 0.07554589467207116
+ 0.0117487281910596
+-0.01922473944012164
+ 0.06476651504095
+ 0.03705780053141568
+-0.03784493576771158
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1265347592415239
+ 0.05425906569598938
+-0.05238025674499402
+ 0.06126230652541365
+ 0.06121239681667477
+ 0.08165039135881287
+ 0.2154898586468702
+ 0.07810142557599957
+ 0.01535738202146116
+ 0.305281451035008
+ 0.3174990530958363
+ 0.3846070198598388
+ 0.4804505130402825
+ 0.1808561682370068
+ 0.386306851096104
+ 0.107942447064052
+ 0.1343048699829552
+ 0.8456987187667494
+ 0.3975052382619448
+-0.03630781259826539
+ 1.027193595964667
+-0.1435536134149933
+ 0.005627202359721035
+ 1.463171117983109
+-0.003649852196285947
+-0.164327513865978
+ 0.9260584302479572
+ 0.06092937459733453
+ 0.1701144844353229
+ 0.7866476689512349
+-0.04902214802098032
+ 0.106848259234997
+ 0.3594214540719029
+-0.1556749774938593
+-0.1184068458396076
+ 0.2975187963964289
+-0.1613923208125013
+-0.06802324585917598
+ 0.1206512851112414
+-0.1866821807964986
+-0.2027372577020332
+ 0.1088607965439639
+-0.1809126562060383
+-0.1167201576823752
+ 0.0495238261074523
+-0.1382587520633114
+-0.1253891235267075
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.06803339458393921
+ 0.02422844465526069
+ 0.01856191782771133
+ 0.02066586144105373
+ 0.03815942725243979
+ 0.02125088896650612
+ 0.09576580272538603
+ 0.03591858329991948
+ 0.03892906397732858
+ 0.3882234849360811
+ 0.361566910837868
+ 0.2474765077134578
+ 0.5410027484593787
+ 0.3116429187193587
+ 0.1134366714859339
+ 0.4981761023535856
+ 0.3971112756003632
+ 0.1408675275058989
+ 0.7270243253435944
+ 0.2445407296646462
+ 0.004520077091457604
+ 0.5224128737576819
+ 0.09042195845428597
+ 0.006740736561519788
+ 0.9001069562712208
+-0.1641388208757147
+-0.0512512779644147
+ 0.5846459873751356
+-0.03841450824275358
+ 0.2340464266398725
+ 0.4715652862257556
+ 0.1551404467557803
+-0.01517038230725697
+ 0.3940288479974343
+ 0.04171286462181721
+-0.0678132892709727
+ 0.2446181568485518
+ 0.142959164693537
+-0.07991247141075764
+ 0.2109211831088837
+ 0.1089783215667557
+-0.132676806026182
+ 0.1180470124832955
+ 0.1233547270083578
+-0.07835812928047305
+ 0.098122860146846
+ 0.07854563920195906
+-0.03513306311768607
+ 0.02028419195678683
+ 0.0323674302530685
+-0.006481089338727093
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1541133646853098
+ 0
+ 0.07845830170999107
+ 0.1819620588456227
+ 0.04247435967856406
+ 0.07055157078365093
+ 0.1129105167724161
+ 0.04700766089124378
+ 0.08215822150761465
+ 0.2754759425319723
+ 0.1157118145174139
+ 0.05779103839311615
+ 0.1982486933477656
+ 0.1755805777095331
+ 0.1069082798396495
+ 0.2763836947052577
+ 0.2903506956979493
+ 0.01428449203683548
+ 0.02712793495177207
+ 0.4366403329755467
+ 0.02985029070275028
+ 0.06008857786350736
+ 0.6175020127242437
+-0.1547988819435056
+-0.1586622806738245
+ 0.8168791731264279
+ 0.03092276609661944
+-0.1657238703204828
+ 0.3020909672918872
+-0.1562158421033308
+ 0.08098432672061398
+ 0.1085519928743939
+-0.02631409595175094
+ 0.02831985990382107
+ 0.1871791246690147
+-0.03789199430926578
+ 0.1250414531275982
+ 0.02958052084297675
+-0.03148712248256605
+ 0.05827454660478151
+ 0.09095447707992224
+-0.02565181951488819
+ 0.08074967223916892
+-0.0008170243079767171
+-0.01211335566312875
+ 0.06112736897509567
+ 0.04697456393388257
+-0.01261955834096525
+-0.02691099548112465
+-0.03297168302641813
+ 0.001470861292652247
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.03688148131567702
+-0.0267720987685314
+ 0
+ 0.03639904792240925
+-0.03398460946321153
+ 0
+ 0.06468020612079307
+ 0.04697448170028686
+ 0
+ 0.4480803490342942
+ 0.2529712419053997
+ 0.1020273446687904
+ 0.3619686594467425
+ 0.2819406116005387
+ 0.324421163878916
+ 0.3145312388812192
+ 0.08082430611975532
+ 0.3577872266298659
+ 0.2431881495190129
+ 0.1927445149338224
+ 0.9023802817074618
+ 0.05422244979104625
+-0.1008984446828327
+ 0.9087493574072456
+-0.05883815836097019
+ 0.03276024258417103
+ 1.435086927019949
+ 0.2231804417766227
+-0.0362526639981683
+ 0.911127878026996
+ 0.05210079115333111
+-0.08651075890227146
+ 0.7566679792686167
+-0.2471114755381569
+-0.1140522830154616
+ 0.6702863053203578
+-0.2111480676839372
+-0.1397589324741141
+ 0.3293420789741844
+-0.3555969078505137
+-0.2475816429754485
+ 0.2762236302320865
+-0.3203076784390017
+-0.2256455665351642
+ 0.1191916143121416
+-0.2926627301113837
+-0.2076324179190413
+ 0.07214354189492099
+-0.1662501171877677
+-0.125109935273284
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.02034030732903353
+ 0.01528549933217357
+ 0.01695440911906193
+ 0.3768407030719007
+ 0.2760340890992067
+ 0.1474640523713866
+ 0.4249064197981335
+ 0.2930718474413159
+ 0.2047041514929455
+ 0.4675682673126724
+ 0.2670061341481466
+ 0.1476610715331994
+ 0.5557522051170745
+ 0.2182097319246716
+ 0.2939712600212151
+ 0.5042131906796383
+-0.06600253974671041
+ 0.1358809671214941
+ 0.6631636965438602
+-0.1352112243139887
+ 0.2812439551465704
+ 0.7399412236649834
+-0.1318532028508151
+ 0.03443698427063484
+ 0.6554948259813107
+ 0.1045939820561339
+-0.2192512233973622
+ 0.4099076888723338
+ 0.1948530223027911
+-0.2922859752074644
+ 0.3466088412993964
+ 0.214512507938758
+-0.2426549385190052
+ 0.2576354966848444
+ 0.2788445761370781
+-0.2621272958771947
+ 0.2349529697997613
+ 0.2098994503025846
+-0.1490244025553265
+ 0.2198532163001285
+ 0.2112974963800464
+-0.09154331080040978
+ 0.1701345206449615
+ 0.1265033941618106
+-0.06136379907128364
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.01031870911745981
+ 0
+-0.008413550240444139
+ 0.1194507031271152
+ 0.1030750055648655
+ 0.1267352200567672
+ 0.23179374736713
+ 0.05324217397980707
+ 0.04850298263580325
+ 0.1348955095050006
+ 0.1624427290375364
+ 0.1846432063927164
+ 0.1992716954939747
+ 0.1143592882577742
+ 0.05655156282250186
+ 0.005521427563701259
+ 0.3227862968575912
+ 0.2144948648868425
+ 0.001059650563141007
+ 0.3142628855105483
+-0.009209656182879354
+-0.1293595647859125
+ 0.7082897786953049
+ 0.2113045879361458
+-0.004433393050499546
+ 0.6500360098324366
+-0.1453981954434912
+ 0.1443397297947534
+ 0.2116903452644331
+-0.1727219024114052
+ 0.1218935237211743
+ 0.2143009968844558
+-0.1741329008436715
+ 0.2199969987781595
+ 0.07846483638265238
+-0.1857377245073424
+ 0.1532965479507176
+ 0.1154167049031322
+-0.09975617876109724
+ 0.1952985230738583
+ 0.07477234736712299
+-0.1038778315328607
+ 0.1245006625601844
+ 0.09526824750991748
+-0.0252070029892215
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1284618686437821
+ 0.05521170870634963
+ 0
+ 0.4120924666847358
+ 0.3621108518667479
+ 0.2570087534760207
+ 0.5263364278263232
+ 0.2682890292812654
+ 0.1764606269265142
+ 0.3112958101839003
+ 0.2232743461899808
+ 0.389744787129073
+ 0.5192763186211899
+ 0.1405195295515248
+ 0.4951499490254671
+ 0.05674381276352737
+ 0.04511648589043775
+ 0.821284880688384
+ 0.304017687668088
+-0.1348309869814445
+ 0.947516886152169
+ 0.03275003430923586
+ 0.2065049079003671
+ 1.307492322728107
+-0.2134332507122198
+ 0.135581469736577
+ 1.020524406000875
+-0.2293441931197029
+-0.3209823793215572
+ 0.9557843783805313
+-0.3479733615238613
+-0.1868319302951993
+ 0.5190514550388394
+-0.3741533746247315
+-0.4875017789296014
+ 0.4129002656084802
+-0.461744058393222
+-0.3129817225326378
+ 0.1919141107238721
+-0.424361391176049
+-0.383846034551928
+ 0.1232251607610359
+-0.333641498665852
+-0.2203966007762082
+ 0.06825855975303714
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.331452983510104
+ 0.2314813602174839
+ 0.1305539024584947
+ 0.3759033727803379
+ 0.2696012443798124
+ 0.1967942817392815
+ 0.4140843354570928
+ 0.2832786082158
+ 0.1312267240579232
+ 0.4306250577154188
+ 0.2825074210313763
+ 0.2494243393449228
+ 0.3931931973808098
+ 0.1114612002319112
+ 0.08982572037850751
+ 0.4844497493463748
+ 0.09037997003060334
+ 0.216556121942579
+ 0.5586380686070097
+-0.06595614536184118
+-0.05655909953745271
+ 0.7846589378293016
+-0.1120836127267642
+-0.1622528825787964
+ 0.4587512301085379
+ 0.1483188364051098
+-0.2350498599921291
+ 0.4729539297066008
+ 0.07726539061120415
+-0.1880674991152078
+ 0.2638285378909403
+ 0.2622980616733094
+-0.1997793445197367
+ 0.2834911366758052
+ 0.1497133682402081
+-0.1264981867567899
+ 0.2035767732330088
+ 0.2154027787895148
+-0.07475132478855791
+ 0.1630172712749669
+ 0.103585098601759
+-0.06746205495890244
+-0.007877007090405253
+ 0.007806358408923162
+-0.01476610066826817
+-0.007131337480064155
+ 0.01526970778703906
+ 0.005166313779687873
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1332093955173815
+ 0.1174064095903005
+ 0.1178926134185491
+ 0.2459826100840443
+ 0.05737880738885095
+ 0.04259705087912995
+ 0.1692321108441049
+ 0.1192237030276236
+ 0.1624752916523937
+ 0.2829763200148874
+ 0.04462492402711615
+ 0.06520783013960736
+ 0.1113982749104689
+ 0.1417579351833835
+ 0.1990804869311112
+ 0.1513530491538063
+ 0.1043925212877098
+ 0.01875790306682611
+ 0.03255527768919382
+ 0.605572291812799
+ 0.2485883482849533
+-0.1120836127267642
+ 0.8453622009796675
+-0.1526163536259003
+ 0.1099259472701737
+ 0.2797376877892378
+-0.1763815885336019
+ 0.0244651689122269
+ 0.3552378913952989
+-0.1946018350184972
+ 0.2142951240075644
+ 0.1021294695380173
+-0.2228066321818947
+ 0.1149821514736872
+ 0.1737634971766651
+-0.1165636789549033
+ 0.2084595511667217
+ 0.07287335249328573
+-0.1170007013195659
+ 0.127649346982713
+ 0.1121197734448574
+-0.0222982750542847
+ 0.008031227415344868
+-0.005609180030771992
+ 0.009673485941977401
+ 0.006213640105032329
+-0.01308533299646755
+-0.00590620150003087
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.2545307829258172
+ 0.1785030073118203
+ 0.09122514207051007
+ 0.3215971888443164
+ 0.1681778345399439
+ 0.1020474664043229
+ 0.2518054512415106
+ 0.06121379008318012
+ 0.08996310148640849
+ 0.2694612922250535
+ 0.1454866002639575
+ 0.2133933175578625
+ 0.112324777969876
+ 0.05135402676872157
+ 0.2641036627475473
+ 0.1526353089326875
+ 0.04038475243822453
+ 0.5925005479218425
+-0.02172870821282149
+-0.02804915331841235
+ 0.7681102082037131
+-0.1622528825787964
+-0.1526163536259001
+ 1.456439023093136
+-0.1995906289810631
+-0.200327634028785
+ 1.443889069291943
+-0.2024550460456235
+-0.2148641207510449
+ 0.8662574344810691
+-0.4462747787047125
+-0.4823646840254157
+ 0.6902669328439639
+-0.457101457513516
+-0.3824995358978957
+ 0.2885521473987115
+-0.5305190273671501
+-0.4231455145864221
+ 0.2064465931549481
+-0.4592537011890912
+-0.3369701122215291
+ 0.07736076349186174
+-0.07101677417873505
+-0.07086917169787142
+ 0.01152782657511707
+-0.02781418378381701
+-0.01671693730458874
+ 0.001295418978920348
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.03044667703302059
+ 0.04403206740566608
+ 0.03582681873067981
+ 0.1970191390004093
+ 0.06593882888838465
+ 0.05121909634722503
+ 0.1727167742643328
+ 0.1289474301708956
+ 0.005675658529694199
+ 0.2554431885446493
+ 0.1010532392087211
+-0.01520252151990854
+ 0.2223286466276709
+ 0.07682658191327009
+-0.0213312754754193
+ 0.3435854388741006
+ 0.01016484014168444
+-0.07013381010260193
+ 0.2677130921354686
+ 0.05745042261581606
+-0.02781759349216432
+ 0.4516209049819952
+ 0.1033442120768516
+-0.1885592732195571
+ 0.8624168095341098
+ 0.05815875988743358
+-0.1003785745015452
+ 0.6747289496893929
+ 0.2388143876842166
+-0.01763113134369359
+ 0.5813307949485805
+ 0.1542528888277588
+-0.08864440555020732
+ 0.4392701743706457
+ 0.2711436118932059
+-0.08100843721981642
+ 0.3665817567499925
+ 0.1978001931191953
+-0.09663519785199656
+ 0.2489451381663269
+ 0.2076306050245115
+-0.06458040387065019
+ 0.06223447083982179
+ 0.09654846249174477
+-0.04745917281149225
+ 0.004128465618274113
+ 0.02759014681847567
+-0.01940120982411387
+ 0
+ 0.002571358937249166
+-0.03168361259661954
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.02045731958508273
+-0.0426108728790597
+-0.03643801670881111
+ 0.02336418679529378
+-0.00916601005182391
+ 0.02071903558911519
+ 0.06274543143817313
+-0.002924352033261558
+ 0
+ 0.002789101460044177
+ 0.05483318995158447
+ 0.02367999641957973
+ 0.08501030187761978
+ 0.06398991331186546
+-0.06205071004502502
+-0.02054685434728548
+ 0.168235757915879
+-4.160383083549289e-05
+ 0.07759499499269781
+ 0.1386032591445443
+-0.1685482912921846
+ 0.1392035408523171
+ 0.2733818140414618
+-0.1893349023083959
+ 0.05478142647510953
+ 0.7296436958498829
+-0.1640454835615678
+ 0.2922242014780934
+ 0.4191623102468908
+-0.09669166224617004
+ 0.1647746628607176
+ 0.4644214008258962
+-0.1229713029242185
+ 0.3152173190357898
+ 0.2354343774441369
+-0.1051610027511401
+ 0.2474173938675175
+ 0.2648560507450335
+-0.09130072499282633
+ 0.2585952782363763
+ 0.1303092040020581
+-0.1001186584580273
+ 0.03654170883451871
+ 0.03571621537178682
+-0.07066499281361863
+ 0.01880113440246318
+ 0.03955211569560431
+ 0.01051078006649249
+-0.02479764712976456
+ 0.03555235082365083
+ 0.004232550382972015
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.147393540957853
+ 0.1030014784110124
+ 0.04396007297685398
+ 0.2406148190047568
+ 0.1008854153494592
+ 0.05835339870557083
+ 0.1692339638181312
+ 0
+ 0.03157191947671363
+ 0.1816538216969741
+ 0.0952808709023624
+ 0.130136541247062
+ 0.06290448543264195
+ 0.03179279991833318
+ 0.1913243599463574
+ 0.09101977113375354
+ 0.0272590546908461
+ 0.499030767882946
+-0.09343446827403874
+-0.05510117358776137
+ 0.7059991521038377
+-0.2186949145192734
+-0.1687443964906593
+ 1.429767567023642
+-0.09290031529819788
+-0.1635042463347606
+ 1.493650836801028
+-0.1084163194590233
+-0.1602163117017198
+ 0.935098246786198
+-0.3648069207726756
+-0.465283844396313
+ 0.7410489691933031
+-0.3871314685413987
+-0.3411754518775544
+ 0.3053286620649369
+-0.4817930649568226
+-0.4040404216395288
+ 0.2056944131762076
+-0.4206227557262404
+-0.3077282197880447
+ 0.06859066072643273
+-0.0517525855098025
+-0.04528452114517724
+-0.006114100394247388
+-0.02437959763494164
+-0.008015490269529054
+-0.00325449393741661
+-0.002822866834812281
+ 0.004720887190653955
+-0.007448239739027688
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1495069098706701
+ 0.05672342586476095
+ 0.06326824538365942
+ 0.1523235075401624
+ 0.1065843268946623
+ 0
+ 0.172075302285517
+ 0.1304235596401615
+ 0
+ 0.1917693372876988
+ 0.116806175198413
+-0.06688564545124541
+ 0.2295052043559954
+ 0.1243845633915553
+-0.05738909956635813
+ 0.2283013689066606
+ 0.04567986907550232
+-0.140801474742453
+ 0.4869921278408665
+ 0.03138217138102461
+-0.1846983895904171
+ 0.6908989710273516
+ 0.2978475242577998
+-0.1105323004527429
+ 0.9160735215941407
+ 0.009275220906341393
+ 0.008415652920233822
+ 0.5744515840727835
+ 0.2241412662908211
+-0.03759065826253016
+ 0.618506140462265
+ 0.1151242468959312
+-0.07598030182617588
+ 0.3682208717140836
+ 0.2332661220362948
+-0.05705732911713544
+ 0.3165930419810411
+ 0.150569774740758
+-0.1070557787461441
+ 0.03671349648149667
+ 0.07846500669693569
+-0.07798789784126166
+-0.003063613635782227
+ 0.05496503781948593
+ 0.01190322534810254
+-0.02225239499278948
+ 0.0294467433290907
+-0.002896058024706844
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.06787543634648979
+-0.007922532719397701
+ 0
+ 0.07224149959422066
+ 0.01628660405329422
+ 0
+ 0.08940671913108297
+ 0.01067747206902605
+ 0
+ 0.1094758809671548
+ 0.009906136467646987
+-0.005284173045837082
+ 0.106034390137575
+ 0.02365763360792631
+-0.01599401007453661
+ 0.1128254962999722
+ 0.1448878857788443
+-0.02857597816196143
+ 0.08827685244659778
+ 0.3544702182330313
+-0.2042435690694162
+ 0.252261379662941
+ 0.426384158372172
+-0.1641054033464274
+ 0.01462906987761201
+ 0.7114321001018025
+-0.1284561393429695
+ 0.1896061738720026
+ 0.3913651895532327
+-0.1850344098865425
+ 0.1138834046962507
+ 0.4379285184690964
+-0.1116092113003505
+ 0.2598280936990828
+ 0.231698748000693
+-0.1422928112606244
+ 0.1916233923365911
+ 0.2107336418254812
+-0.05215167466849729
+ 0.07613813335148018
+ 0.06787811574482296
+-0.03974355426023836
+ 0.03009932157078149
+ 0.01023879429914638
+-0.02824708460007883
+ 0.0004413930041132366
+ 0.004695022247142875
+-0.03263143495326951
+ 0.0001356725217021724
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.06746540205821781
+ 0
+ 0.004660098126320933
+ 0
+ 0
+-0.03400585992974392
+ 0
+ 0
+-0.00383259980343694
+-0.02250286167721027
+-0.002161532276535457
+ 0.01575819143221054
+-0.02907326899432235
+ 0.01212308568433054
+ 0.1895485619551283
+-0.1226883870996998
+-0.09326396534840108
+ 0.2969888027886603
+-0.203070996085234
+-0.2084683532099987
+ 0.8491159347773153
+-0.03795460427638125
+-0.1029532224716381
+ 0.9361086421453015
+-0.004859332291959456
+-0.1366820902232274
+ 1.039545518119511
+-0.1423920866635132
+-0.2978412287866122
+ 0.9314103195780626
+-0.1341947865413598
+-0.2033139875709718
+ 0.4814115349324639
+-0.3234138829586117
+-0.2987621449144857
+ 0.3398491208950851
+-0.2663595805599724
+-0.218809251959778
+ 0.09930675826233686
+-0.06283131870902126
+-0.07465836167523256
+ 0.01649009102387712
+-0.01937046663801414
+-0.01674868775790473
+ 0.01540444473919237
+ 0.001637604171341718
+-0.006643948276569291
+ 0.01307958466415525
+-0.004706298102512472
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.02898332685159531
+-0.004354525751479183
+ 0
+ 0.01073294877213779
+ 0.02054120346484029
+ 0.00766263125374445
+ 0.07750579671108329
+ 0.0209812241121359
+-0.04437515537557073
+ 0.05469265585623201
+ 0.04570283948423293
+-0.02717713368640161
+ 0.124139486972122
+ 0.0270227710293308
+-0.1303019253528366
+ 0.08822367049810927
+ 0.1115488243306284
+-0.1032915558562808
+ 0.2777750758706082
+ 0.2172107771469001
+-0.432084772402383
+ 0.5924968144533665
+ 0.1681169525737155
+-0.3638298217685866
+ 0.5702169891773118
+ 0.1846199178710025
+-0.1298422859076117
+ 1.059513761324535
+ 0.09211030447859657
+-0.01561395720374273
+ 0.8328064555130887
+ 0.3013936581390753
+ 0.1380564963950855
+ 0.7295128267437798
+ 0.1726908115653597
+ 0.07277149818462429
+ 0.5244058902226941
+ 0.3333975662538784
+ 0.01675687435379709
+ 0.181107270320755
+ 0.1549508763568679
+-0.008530430925992738
+ 0.06272183180044247
+ 0.0008406654098609593
+ 0.001633124323519533
+ 0.01760002016703443
+-0.04042577350348711
+-0.0220268789514094
+-0.01230305286134971
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.029796989501046
+ 0.01262140391350271
+ 0
+ 0.00723388273839362
+-0.01181623564439983
+-0.004183642603085919
+-0.01599818818410756
+ 0.01991640978253419
+ 0.01088331179673357
+ 0.05341456696656432
+-0.0004452594257657127
+-0.06572776881713002
+ 0.01739101993365819
+ 0.0600678336403731
+-0.04239910763554258
+ 0.1261800612079269
+-0.01547226261023061
+-0.241911545576015
+ 0.2659940079745977
+ 0.1033728904602602
+-0.459230242551262
+ 0.1541391861533491
+ 0.4689377301211789
+-0.4633386739359197
+ 0.225649421614781
+ 0.3878329299325556
+-0.2938448965939899
+ 0.09211030447859683
+ 0.9708577987049175
+-0.1481766715720259
+ 0.3845927635994222
+ 0.5365648946386252
+-0.0140227677526953
+ 0.2787504288775663
+ 0.5868615893379269
+ 0.008401388994637974
+ 0.4208893883015439
+ 0.2852520001586972
+-0.100147079319616
+ 0.07595542983556139
+ 0.0967515987019547
+-0.1196203560626718
+ 0.04685344137364201
+ 0.111879553778612
+ 0.03996962885610333
+-0.04493882927320793
+ 0.0718126586103678
+ 0.01516675067158226
+ 0.00442481602400676
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.01364922125150793
+-0.06168013200811229
+-0.03332808481473472
+-0.03179895379705183
+-0.05946985482088529
+-0.02870114808324508
+-0.01506534345543211
+-0.08054957189767846
+-0.02644018080428226
+ 0.005022645584397655
+-0.08773719750652052
+-0.005321508687478919
+ 0.1497959221195863
+-0.1557730606003737
+-0.1212421329707608
+ 0.2261366528506384
+-0.2053959969322332
+-0.2288378683429283
+ 0.6693464590200435
+-0.09822776084638159
+-0.1272576023759546
+ 0.7393206440908646
+-0.04129601313005547
+-0.1856822851133047
+ 0.9248706331998275
+-0.0190445903121543
+-0.1474053557430128
+ 0.960854310895684
+ 0.0255655931356564
+-0.1117809823517434
+ 0.5908934247810864
+-0.2033273452812217
+-0.2015578029659007
+ 0.4391325679340776
+-0.1257456371243098
+-0.1377714170476583
+ 0.1262668249509752
+-0.02160870665757127
+-0.04702441958410075
+ 0.01619576889441505
+ 0.006114257682597852
+-0.002599705846445055
+ 0.03595083950880592
+ 0.0002560624773800675
+-0.008048999357267861
+ 0.02467708834887031
+-0.007345749081730969
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.03453671773665813
+ 0.03910091372558489
+-0.0217806128277353
+ 0.0477906010433775
+ 0.04755003491357004
+-0.05405441524404941
+ 0.07210361065460598
+ 0.08027638082584458
+-0.1085848691249564
+ 0.09823219565098124
+ 0.05911885016714628
+-0.1853960383774744
+ 0.3351769800074288
+ 0.136961043905307
+-0.4429367468435427
+ 0.4736552030171357
+ 0.3343014484436261
+-0.3865849329842753
+ 0.6436495881088224
+ 0.1230713235555615
+-0.1322848505602867
+ 0.8506478586062033
+ 0.3983631136526723
+ 0.01735955245994484
+ 1.090253706406084
+ 0.06018620866675699
+ 0.09648076688640531
+ 0.7225225573456515
+ 0.2419012883851915
+ 0.1049370355865295
+ 0.6774825512361102
+ 0.198269638975059
+-0.0850370609663811
+ 0.1625622160911344
+ 0.1033606303177067
+-0.0876968881265851
+ 0.07842558875981412
+ 0.04458542871302893
+ 0.06594416238081342
+-0.01239943441183437
+ 0.01285892772532419
+ 0.04248895205805455
+-0.006474458738017678
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.01656734268912792
+-0.01330540983132488
+-0.0158340745473802
+ 0.05002760966816803
+-0.006844885889453874
+-0.02131395206575877
+ 0.06766036162347627
+-0.01021495802314147
+-0.03705388133153394
+ 0.0850110315346234
+ 0.03069935193815565
+-0.08459657090925513
+ 0.1721273368845858
+ 0.1794700505476242
+-0.353217751715753
+ 0.2931693705226576
+ 0.2487268179161962
+-0.3404054617989499
+ 0.1322897613814507
+ 0.4449798635008817
+-0.1996429441262241
+ 0.3135021327272443
+ 0.5457765710033901
+-0.1170398104201297
+ 0.06018620866675708
+ 0.8330306675055654
+-0.01293293869645117
+ 0.2548416597106281
+ 0.4849491599242081
+-0.09939157061882915
+ 0.203029194702317
+ 0.4362851592397721
+-0.02230013923919606
+ 0.09719587897266509
+ 0.1809439895872292
+-0.06246631186116594
+ 0.0155668417244723
+ 0.07243469262560916
+-0.05530761199471974
+-0.02148768441240798
+ 0.02288959401604958
+-0.0789276915420626
+-0.00281705548223805
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.01137930166471226
+ 0.001037527197513009
+-0.01498848595762328
+-0.06537707692533822
+-0.02522561765713401
+-0.01738032674510652
+-0.05002821610234268
+-0.008209218232031981
+ 0.0323843603031537
+-0.06641794539566535
+-0.04827689964571867
+ 0.02643811742099644
+-0.1342325771285902
+-0.1199933013742983
+ 0.2514795559897838
+-0.09249576141705464
+-0.1160301522966605
+ 0.3041021298489935
+-0.09090656952784495
+-0.1180285786701245
+ 0.4797224005001612
+ 0.1289584840756142
+-0.02003429285797051
+ 0.5951711866781099
+ 0.096480766886405
+-0.01293293869645114
+ 0.8796047273164596
+ 0.02630746744865203
+-0.08958764164238489
+ 0.7914593694877105
+ 0.06744749237726214
+ 0.01155287875537524
+ 0.3773729733294982
+ 0.03643122195779108
+-0.09169194334474876
+ 0.2066787305019734
+ 0.08036297616610467
+-0.06326117909986063
+ 0.1066184158581806
+ 0.06604709169946164
+-0.09419458776918326
+ 0.08002955108649515
+-0.009560191069422888
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.00289646067159027
+ 0.007004205646794325
+ 0.01097261793677344
+ 0.05835945527251087
+ 0.02441189412637596
+-0.06962733829505056
+ 0.06377324381296934
+ 0.07760600273692224
+-0.1072923759363541
+ 0.2799744852344632
+ 0.2491230160149703
+-0.5507550787824206
+ 0.4320214424362562
+ 0.2626601769388605
+-0.4899612607894883
+ 0.3990383035440956
+ 0.2775912621611886
+-0.327168609905263
+ 0.7591586516310525
+ 0.2926308919829664
+-0.2183413748921876
+ 0.7306493656271388
+ 0.2635506750694763
+ 0.02167717882508768
+ 1.158746498413555
+ 0.1478167154941652
+ 0.1235994016606926
+ 0.8892095826798591
+ 0.3623817065773147
+ 0.191371907717232
+ 0.4647060879430394
+ 0.1395793463321499
+ 0.1391891223910622
+ 0.2781057541541126
+-0.1241896330588512
+ 0.07452981763489898
+ 0.1427072443872709
+-0.166328765809533
+ 0.04019863106016888
+ 0.01231301866858489
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.02550294589223547
+ 0.001607106795463471
+-0.02115740875305558
+ 0.03505662763607797
+ 0.02657711641646523
+-0.01359133484068593
+ 0.08784071128997431
+-0.008435442323719618
+-0.1392706928919042
+ 0.2410808856179767
+ 0.0899077674592946
+-0.3975385604402012
+ 0.2211494487333924
+ 0.2793101584905084
+-0.4064000290127994
+ 0.2567391738675522
+ 0.2429682485562932
+-0.3010442231383564
+ 0.1908285306598604
+ 0.5978717604761639
+-0.2116232335066464
+ 0.2455135258435132
+ 0.4889257966413527
+-0.09192652359485722
+ 0.1478167154941651
+ 0.8637209324822654
+ 0.09116528720254731
+ 0.3591403950408739
+ 0.4780342787926764
+ 0.013380249633847
+ 0.04493628699216265
+ 0.2359559909974446
+-0.05657591797366269
+ 0.002535017223999032
+ 0.2183589475976805
+ 0.06667791693847054
+-0.07885800375324803
+ 0.107943619281646
+ 0.005320783729045826
+-0.002102138210538136
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0859297511444953
+-0.03123487347377921
+ 0
+-0.06470958525160081
+-0.004412039415424872
+ 0.05083465351914572
+-0.06679426264548287
+-0.05789061758710524
+ 0.01201533480566358
+-0.1096599758492784
+-0.1266609628787409
+ 0.183669996416383
+-0.1368700292049179
+-0.117235027743614
+ 0.2271303908593813
+-0.09862214436666189
+-0.1675955368616276
+ 0.3551839797591633
+ 0.04010616227191219
+-0.01828079737070288
+ 0.4611292242512056
+ 0.0819897343764793
+-0.1183644758991115
+ 0.8028086387135387
+ 0.1064183790713736
+ 0.07558353681337865
+ 0.9086777064362186
+ 0.2126081863809646
+ 0.08296946165210906
+ 0.4910900132866513
+ 0.1004737517927588
+-0.05893816208283639
+ 0.2890034634284118
+ 0.1367784144699386
+-0.0439561873339182
+ 0.1748266996629207
+ 0.08755388638592414
+-0.1092755988256249
+ 0.1255203211715755
+-0.003107820609141417
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0453111887730545
+ 0.04129788849889209
+-0.06551099215757458
+ 0.07952260679332711
+ 0.04446407719901178
+-0.1507094131146911
+ 0.3162443773764748
+ 0.1939424030884327
+-0.5544829550120121
+ 0.3708925797358065
+ 0.3329221024495854
+-0.4962545170836462
+ 0.4262529366054946
+ 0.2496156953742299
+-0.3170144615274276
+ 0.6117748604449134
+ 0.4963845938892379
+-0.1846941594067151
+ 0.754178502771673
+ 0.2548001230201909
+ 0.03497855111625389
+ 0.9508196856856667
+ 0.4107921595326413
+ 0.2036767496268742
+ 1.081807205394564
+ 0.1563780569083378
+ 0.04575373491781462
+ 0.444836848050662
+ 0.0005912814978173479
+-0.009846928983868677
+ 0.3489336818221916
+-0.05080052533646509
+ 0.1861936130971906
+ 0.1120518768328002
+-0.07007882752566297
+ 0.1546995329133125
+ 0.04354176769896598
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.02527598457295166
+-0.001768986990454072
+ 0.003317002883114171
+ 0.03297913006185288
+ 0.01366326070349741
+-0.05024524905011145
+ 0.1744398657406253
+ 0.1375670423938269
+-0.3503753904261871
+ 0.2836541174461448
+ 0.1804375239952578
+-0.3593902219941618
+ 0.2247152646836893
+ 0.2529834491179146
+-0.2528701667015872
+ 0.3955041754621541
+ 0.3366960001340097
+-0.1793120645196947
+ 0.2508099372678701
+ 0.4732614839537085
+-0.01147735761987027
+ 0.4033712994296682
+ 0.5126473409959805
+ 0.07734868580508403
+ 0.1563780569083378
+ 0.7584303713220008
+ 0.1341456930030979
+ 0.04591759290117975
+ 0.4317919472154008
+ 0.0452619231902515
+-0.117448049551387
+ 0.1606536980076543
+-0.08827796000712447
+-0.07967053621374606
+ 0.02069569454556858
+-0.1533521039855621
+-0.05358611208257169
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.01800901377956207
+ 0.005175041449258899
+ 0
+ 0.02768126599643208
+ 0.01276640171338106
+-0.05155951291841206
+-0.06220051317288586
+-0.02810571770655953
+ 0.05369650562020231
+-0.08147770996232603
+-0.1299112381239295
+ 0.1104653641416167
+-0.1502282885662586
+-0.07680847635831173
+ 0.1265807640043945
+-0.02161268361288356
+-0.1316963322025976
+ 0.1551205441412535
+-0.1175432753186116
+-0.04499884669811427
+ 0.3918430353650929
+ 0.1620710605802327
+-0.009325979607410011
+ 0.4971655863234344
+ 0.03819551445357049
+ 0.1284578264977487
+ 0.8234683122488623
+ 0.2639297909576087
+-0.0405905357444304
+ 0.7017729860270119
+ 0.2595312815709759
+-0.2090611843105151
+ 0.193723871678607
+ 0.2661962875888019
+-0.2936619798966802
+ 0.1352003871330183
+ 0.03406732666513703
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1941375998894964
+ 0.1530312978676857
+-0.3634514165525199
+ 0.2386496998375353
+ 0.1590283983884435
+-0.2984634891109203
+ 0.1802792467321077
+ 0.1896081751947142
+-0.2294668606929837
+ 0.3342218634277369
+ 0.2185164886551508
+-0.1764409599232029
+ 0.310562298436409
+ 0.2115967919095705
+-0.0432079492395778
+ 0.613967449886055
+ 0.1604405073759448
+ 0.04095631180024667
+ 0.560260893012
+ 0.1284297882436596
+ 0.2360151217341434
+ 0.7911017093314525
+-0.03628312733163946
+ 0.2820413843886657
+ 0.62050943143644
+-0.3099380207357701
+ 0.09779757469535742
+ 0.3750664810887352
+-0.2851086978588858
+ 0.09089457391122706
+ 0.1458609083193354
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1694476251128947
+ 0.1282700605617779
+-0.3148267322943111
+ 0.2549056349957726
+ 0.1822275523933939
+-0.3354018885712621
+ 0.2492183390519549
+ 0.2012637738269327
+-0.2702076707145339
+ 0.3361539745230698
+ 0.268328036367336
+-0.2295929670769196
+ 0.2788575284115253
+ 0.3164264639471738
+-0.1859194963287544
+ 0.3148759186469441
+ 0.3739507610034065
+-0.1283164502812031
+ 0.1348806607770446
+ 0.5277291195618026
+-0.07370015885157938
+-0.0362831273316396
+ 0.6094694673517912
+ 0.05033539349637208
+-0.2241716300896296
+ 0.3292246131175449
+-0.001327696551146982
+-0.1631244840796518
+ 0.1462610708547624
+-0.0880716826554595
+-0.05084222313566003
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.08525837716883324
+-0.03857345103301743
+ 0.1069307706764438
+-0.1198504957561022
+-0.1603205548378792
+ 0.1591472052471194
+-0.1815226001647322
+-0.1146622438877439
+ 0.1279688132761465
+-0.1102612328810432
+-0.2241473340834236
+ 0.1200887740708188
+-0.1897406562988194
+-0.1396021454401372
+ 0.260604739205378
+ 0.03923376155985381
+-0.1387737861614789
+ 0.3275782836358209
+-0.08514958147343689
+-0.00767068754183741
+ 0.7215900091107786
+ 0.2820413843886656
+ 0.05033539349637216
+ 0.7730567567488885
+ 0.2518931547043468
+-0.16584535022333
+ 0.2287007728240014
+ 0.2735891883985518
+-0.2657407115706805
+ 0.1630804698926822
+ 0.05758270964011017
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.07763710847443513
+ 0.05987160169092721
+-0.1511978893737019
+ 0.07231101738229902
+ 0.05573865439435653
+-0.1024358635594179
+ 0.04415913393260545
+ 0.06431678991150819
+-0.07203472188893269
+ 0.1091147384461191
+ 0.09142020979320496
+-0.04192506280389351
+ 0.1245066736253398
+ 0.05086153584988306
+ 0.05584547433912342
+ 0.3247158699508952
+ 0.03878918938047023
+ 0.11828287850483
+ 0.3858534576002275
+-0.09127922678833718
+ 0.2506840721731568
+ 0.6205094314364399
+-0.2241716300896294
+ 0.2518931547043469
+ 0.6983538139068247
+-0.1609922250691227
+ 0.1716318115067201
+ 0.3823553951302308
+-0.1515622134131721
+ 0.1533940149729053
+ 0.2438919615712385
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.05021469962704999
+ 0.0208471471346901
+-0.08288829368617638
+ 0.05837725040380706
+ 0.0994590858395821
+-0.124743075415675
+ 0.1284581338612232
+ 0.05912605744743366
+-0.09427041337908477
+ 0.06923896918067633
+ 0.1854752097024152
+-0.07371581265774065
+ 0.1152230356974949
+ 0.1253388040979826
+-0.09993378287707406
+-0.0557810640086056
+ 0.2761999416394266
+-0.06990500583508999
+ 0.001683379216170666
+ 0.1972640704656796
+-0.2228487297498827
+-0.3099380207357703
+ 0.3292246131175449
+-0.1658453502233301
+-0.1609922250691228
+ 0.6680804932146648
+ 0.2141622513371803
+-0.2617433031212513
+ 0.4582751949587536
+ 0.1016550286884798
+ 0.06311272365368008
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0289777634485793
+ 0.01285486683690303
+-0.05039018279692284
+-0.03724955946653333
+ 0.003835147600739779
+ 0.006518743288640109
+ 0.00329623280387221
+-0.03881636856538331
+ 0.02420963780642721
+-0.01053096306178367
+ 0.03427800736273548
+ 0.04608199590334629
+ 0.05785913100893915
+-0.06301454853342996
+ 0.1122054283320986
+ 0.06536617451567721
+ 0.06165557043280655
+ 0.1792443778623837
+ 0.1800465025360133
+-0.09338105895639466
+ 0.1950920993561757
+ 0.09779757469535734
+-0.001327696551146852
+ 0.2287007728240014
+ 0.17163181150672
+ 0.2141622513371803
+ 0.5708554264117766
+-0.02109506277249117
+ 0.1095634955232319
+ 0.4801043219564997
+ 0.01777180131348255
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.04566348062562483
+ 0.0379275287162404
+-0.05775740904643421
+-0.01564349048741974
+ 0.02556320812416608
+-0.01750943907816571
+-0.0726109741676576
+ 0.02347266155426717
+-0.04680129780172409
+-0.04293234524718757
+ 0.07918732842620622
+ 0.1130638194934263
+-0.1028214206699394
+ 0.09735208451741834
+ 0.08953428778153637
+-0.09553929030576268
+ 0.2726669512078172
+ 0.3735568212132534
+-0.1660144523261445
+ 0.2753282873139399
+ 0.3823929710457864
+-0.2630281908177792
+-0.02080771010652737
+ 0.6040614072541741
+-0.2829509625541405
+-0.01184841718572921
+ 0.3758800563151173
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.001530123680504339
+ 0.06138209634332941
+-0.04921490598370785
+ 0.07616886901758949
+ 0.02129357460706422
+-0.04254970430045962
+-0.006864181738730189
+ 0.1090170817378964
+-0.0407050960119527
+ 0.05011986493583097
+ 0.04639031323453931
+-0.1117051256277355
+-0.1339306856731421
+ 0.1377928742575085
+-0.1209253686340196
+-0.0444857792359557
+ 0.03762937336203414
+-0.3013742648064107
+-0.2850945261040383
+ 0.1471761679486571
+-0.2667134438995792
+-0.1517965786804467
+ 0.4592619000447985
+ 0.1098297252243717
+-0.2828904127971424
+ 0.5530827656728311
+ 0.1507590581644274
+ 0.10591244040782
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.07046125287820829
+-0.02124888069013417
+ 0.05327929423378012
+-0.02907097440307503
+-0.06117285486879783
+ 0.05500178371705559
+-0.05818668072887905
+-0.01516496070557581
+ 0.06403230485631434
+ 0.01119657081226274
+-0.10618270092943
+ 0.09950733362002433
+ 0.005784351463719437
+-0.01962209562624816
+ 0.1400758122708756
+ 0.1288314483647297
+-0.172884710641278
+ 0.1409219079030258
+ 0.08905688434501095
+-0.0897601322625443
+ 0.1633839788563913
+ 0.1526600485696558
+ 0.1013858056460186
+ 0.480256586823653
+-0.01259449546210411
+ 0.1507590581644274
+ 0.499968975340181
+ 0.05332535222879701
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.003482419948602756
+-0.007868364638015095
+-0.01945375472515987
+ 0.007015681720708732
+-0.009616795547391123
+-0.01221821489264071
+-0.001976620367168366
+-0.01018529306494946
+ 0.009526661447888158
+-0.001990096912532073
+-0.003390217053039036
+ 0.04176437415070538
+-0.05436884487422029
+ 0.0344871376772762
+ 0.1454791381551594
+-0.05186778918641916
+ 0.05816285063651524
+ 0.2440875287260171
+ 0.06267899271701106
+ 0.01778222922550598
+ 0.3759966969856117
+ 0.1057756726063925
+ 0.05350522057245403
+ 0.4599302829374405
diff --git a/omp/preconditioner/isai_kernels.cpp b/omp/preconditioner/isai_kernels.cpp
index a64724b260e..424d50424d9 100644
--- a/omp/preconditioner/isai_kernels.cpp
+++ b/omp/preconditioner/isai_kernels.cpp
@@ -151,7 +151,7 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                 }
 
                 // solve dense triangular system
-                trs_solve(trisystem, rhs);
+                trs_solve(trisystem, rhs, row);
 
                 // write triangular solution to inverse
                 for (size_type i = 0; i < i_size; ++i) {
@@ -198,7 +198,7 @@ void generate_tri_inverse(std::shared_ptr<const DefaultExecutor> exec,
 {
     auto trs_solve =
         [lower](const range<accessor::row_major<ValueType, 2>> trisystem,
-                ValueType *rhs) {
+                ValueType *rhs, const IndexType) {
             const IndexType size = trisystem.length(0);
             if (size <= 0) {
                 return;
@@ -240,6 +240,17 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_TRI_INVERSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
+                              const matrix::Csr<ValueType, IndexType> *mtx,
+                              matrix::Csr<ValueType, IndexType> *inverse_mtx,
+                              IndexType *excess_rhs_ptrs,
+                              IndexType *excess_nz_ptrs) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_GENERATE_GENERAL_INVERSE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
                             const matrix::Csr<ValueType, IndexType> *input,
diff --git a/reference/preconditioner/isai_kernels.cpp b/reference/preconditioner/isai_kernels.cpp
index 89f52a4bf16..e2a07bc487c 100644
--- a/reference/preconditioner/isai_kernels.cpp
+++ b/reference/preconditioner/isai_kernels.cpp
@@ -32,8 +32,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/preconditioner/isai_kernels.hpp"
 
-
 #include <algorithm>
+#include <iostream>
 #include <memory>
 
 
@@ -81,7 +81,7 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                       const matrix::Csr<ValueType, IndexType> *mtx,
                       matrix::Csr<ValueType, IndexType> *inverse_mtx,
                       IndexType *excess_rhs_ptrs, IndexType *excess_nz_ptrs,
-                      Callable trs_solve)
+                      Callable trs_solve, bool general = false)
 {
     /*
     Consider: aiM := inverse_mtx; M := mtx
@@ -130,6 +130,7 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                 trisystem_ptr, static_cast<size_type>(i_size),
                 static_cast<size_type>(i_size), static_cast<size_type>(i_size));
             std::fill_n(trisystem_ptr, i_size * i_size, zero<ValueType>());
+            IndexType k = zero<IndexType>();
 
             for (size_type i = 0; i < i_size; ++i) {
                 const auto col = i_cols[i_begin + i];
@@ -138,12 +139,13 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                 forall_matching(
                     m_cols + m_begin, m_size, i_cols + i_begin, i_size,
                     [&](IndexType, IndexType m_idx, IndexType i_idx) {
+                        if (m_cols[m_idx + m_begin] < row && col == row) k++;
                         trisystem(i, i_idx) = m_vals[m_idx + m_begin];
                     });
             }
 
             // solve dense triangular system
-            trs_solve(trisystem, rhs);
+            trs_solve(trisystem, rhs, k);
 
             // write triangular solution to inverse
             for (size_type i = 0; i < i_size; ++i) {
@@ -186,7 +188,7 @@ void generate_tri_inverse(std::shared_ptr<const DefaultExecutor> exec,
 {
     auto trs_solve =
         [lower](const range<accessor::row_major<ValueType, 2>> trisystem,
-                ValueType *rhs) {
+                ValueType *rhs, const IndexType) {
             const IndexType size = trisystem.length(0);
             if (size <= 0) {
                 return;
@@ -228,6 +230,103 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_TRI_INVERSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+inline IndexType choose_pivot(IndexType block_size, const ValueType *block,
+                              size_type stride)
+{
+    IndexType cp = 0;
+    for (IndexType i = 1; i < block_size; ++i) {
+        if (abs(block[cp * stride]) < abs(block[i * stride])) {
+            cp = i;
+        }
+    }
+    return cp;
+}
+
+
+template <typename ValueType, typename IndexType>
+inline void swap_rows(IndexType row1, IndexType row2, IndexType block_size,
+                      ValueType *block, size_type stride)
+{
+    using std::swap;
+    for (IndexType i = 0; i < block_size; ++i) {
+        swap(block[row1 * stride + i], block[row2 * stride + i]);
+    }
+}
+
+
+template <typename ValueType, typename IndexType>
+void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
+                              const matrix::Csr<ValueType, IndexType> *mtx,
+                              matrix::Csr<ValueType, IndexType> *inverse_mtx,
+                              IndexType *excess_rhs_ptrs,
+                              IndexType *excess_nz_ptrs)
+{
+    using std::swap;
+    auto general_solve =
+        [](const range<accessor::row_major<ValueType, 2>> system,
+           ValueType *rhs, const IndexType k) {
+            const IndexType size = system.length(0);
+            if (size <= 0) {
+                return;
+            }
+            // RHS is the identity: zero everywhere except for the diagonal
+            // entry
+            std::fill_n(rhs, size, zero<ValueType>());
+            rhs[k] = one<ValueType>();
+
+            // fill in transposed system
+            ValueType *transposed_system = new ValueType[size * size];
+            for (auto row = 0; row < size; row++) {
+                for (auto col = 0; col < size; col++) {
+                    transposed_system[row * size + col] = system(col, row);
+                }
+            }
+
+            // solve transposed system
+            std::vector<IndexType> perm(size, {});
+            std::iota(begin(perm), end(perm), IndexType{0});
+            for (IndexType col = 0; col < size; col++) {
+                const auto row =
+                    choose_pivot(size - col,
+                                 transposed_system + col * (size + 1), size) +
+                    col;
+                swap_rows(col, row, size, transposed_system, size);
+                swap(perm[row], perm[col]);
+                swap(rhs[row], rhs[col]);
+
+                const ValueType d = transposed_system[col * size + col];
+                for (size_type j = col; j < size; j++) {
+                    transposed_system[col * size + j] /= d;
+                }
+                rhs[col] /= d;
+
+                for (size_type i = col + 1; i < size; i++) {
+                    auto scal = transposed_system[i * size + col];
+                    rhs[i] -= scal * rhs[col];
+                    for (size_type j = col; j < size; j++) {
+                        transposed_system[i * size + j] -=
+                            transposed_system[col * size + j] * scal;
+                    }
+                }
+            }
+
+            for (size_type i = size - 1; i >= 1; i--) {
+                for (size_type j = i; j >= 1; j--) {
+                    rhs[j - 1] -=
+                        transposed_system[(j - 1) * size + i] * rhs[i];
+                }
+            }
+        };
+
+    generic_generate(exec, mtx, inverse_mtx, excess_rhs_ptrs, excess_nz_ptrs,
+                     general_solve, true);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_GENERATE_GENERAL_INVERSE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
                             const matrix::Csr<ValueType, IndexType> *input,
diff --git a/reference/test/preconditioner/isai_kernels.cpp b/reference/test/preconditioner/isai_kernels.cpp
index fe139517bfc..070e824389c 100644
--- a/reference/test/preconditioner/isai_kernels.cpp
+++ b/reference/test/preconditioner/isai_kernels.cpp
@@ -66,6 +66,8 @@ class Isai : public ::testing::Test {
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using LowerIsai = gko::preconditioner::LowerIsai<value_type, index_type>;
     using UpperIsai = gko::preconditioner::UpperIsai<value_type, index_type>;
+    using GeneralIsai =
+        gko::preconditioner::GeneralIsai<value_type, index_type>;
     using Mtx = gko::matrix::Csr<value_type, index_type>;
     using Dense = gko::matrix::Dense<value_type>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
@@ -136,10 +138,20 @@ class Isai : public ::testing::Test {
                                     I<value_type>{-.5, .125, .3125, .09375, .25,
                                                   .125, -.0625, -.5, .25, .5},
                                     I<index_type>{0, 1, 2, 3, 1, 2, 3, 2, 3, 3},
-                                    I<index_type>{0, 4, 7, 9, 10})}
+                                    I<index_type>{0, 4, 7, 9, 10})},
+          a_sparse{Csr::create(exec, gko::dim<2>{4, 4},
+                               I<value_type>{1, 4, 1, 4, 2, 1, 2, 1, 1},
+                               I<index_type>{0, 3, 0, 1, 3, 1, 2, 2, 3},
+                               I<index_type>{0, 2, 5, 7, 9})},
+          a_sparse_inv{Csr::create(
+              exec, gko::dim<2>{4, 4},
+              I<value_type>{1, -4, -.25, .25, .5, -.125, .5, -.5, 1},
+              I<index_type>{0, 3, 0, 1, 3, 1, 2, 2, 3},
+              I<index_type>{0, 2, 5, 7, 9})}
     {
         lower_isai_factory = LowerIsai::build().on(exec);
         upper_isai_factory = UpperIsai::build().on(exec);
+        general_isai_factory = GeneralIsai::build().on(exec);
         l_dense->convert_to(lend(l_csr));
         l_dense_inv->convert_to(lend(l_csr_inv));
         u_dense->convert_to(lend(u_csr));
@@ -154,6 +166,8 @@ class Isai : public ::testing::Test {
         u_csr_longrow_e_rhs = read<Dense>("isai_u_excess_rhs.mtx");
         u_csr_longrow_inv_partial = read<Csr>("isai_u_inv_partial.mtx");
         u_csr_longrow_inv = read<Csr>("isai_u_inv.mtx");
+        a_csr_large = read<Csr>("a_large.mtx");
+        a_csr_large_inv = read<Csr>("a_large_inv.mtx");
     }
 
     template <typename ReadMtx>
@@ -190,6 +204,7 @@ class Isai : public ::testing::Test {
     std::shared_ptr<const gko::ReferenceExecutor> exec;
     std::unique_ptr<typename LowerIsai::Factory> lower_isai_factory;
     std::unique_ptr<typename UpperIsai::Factory> upper_isai_factory;
+    std::unique_ptr<typename GeneralIsai::Factory> general_isai_factory;
     std::shared_ptr<Dense> l_dense;
     std::shared_ptr<Dense> l_dense_inv;
     std::shared_ptr<Dense> u_dense;
@@ -219,6 +234,10 @@ class Isai : public ::testing::Test {
     std::shared_ptr<Csr> u_s_unsorted;
     std::shared_ptr<Csr> u_sparse_inv;
     std::shared_ptr<Csr> u_sparse_inv2;
+    std::shared_ptr<Csr> a_sparse;
+    std::shared_ptr<Csr> a_sparse_inv;
+    std::shared_ptr<Csr> a_csr_large;
+    std::shared_ptr<Csr> a_csr_large_inv;
 };
 
 TYPED_TEST_SUITE(Isai, gko::test::ValueIndexTypes);
@@ -648,6 +667,19 @@ TYPED_TEST(Isai, KernelScatterExcessSolution)
 }
 
 
+TYPED_TEST(Isai, ReturnsCorrectInverseLargeA)
+{
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    const auto isai = this->general_isai_factory->generate(this->a_csr_large);
+
+    auto a_inv = isai->get_approximate_inverse();
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(a_inv, this->a_csr_large_inv);
+    GKO_ASSERT_MTX_NEAR(a_inv, this->a_csr_large_inv, 1e-10);
+}
+
+
 TYPED_TEST(Isai, ReturnsCorrectInverseL)
 {
     using Csr = typename TestFixture::Csr;
@@ -760,6 +792,18 @@ TYPED_TEST(Isai, ReturnsCorrectInverseUWithU3)
 }
 
 
+TYPED_TEST(Isai, ReturnsCorrectInverseAWithA)
+{
+    using value_type = typename TestFixture::value_type;
+    auto isai = this->general_isai_factory->generate(this->a_sparse);
+
+    auto a_inv = isai->get_approximate_inverse();
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(a_inv, this->a_sparse_inv);
+    GKO_ASSERT_MTX_NEAR(a_inv, this->a_sparse_inv, r<value_type>::value);
+}
+
+
 TYPED_TEST(Isai, GeneratesWithUnsortedCsr)
 {
     using Csr = typename TestFixture::Csr;

From fea9ac7153ca39709abeb0bb29bb0c4948dc2bfc Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 3 Dec 2020 08:05:31 +0100
Subject: [PATCH 0505/1008] Add general isai to benchmarks

---
 benchmark/preconditioner/preconditioner.cpp | 7 ++++++-
 benchmark/utils/preconditioners.hpp         | 7 +++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index 9ec10b1ee4c..32ab65c933b 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -110,8 +110,13 @@ std::string encode_parameters(const char *precond_name)
                  << '-' << FLAGS_isai_power;
              return oss.str();
          }},
-        {"ilu-isai", [] {
+        {"ilu-isai",
+         [] {
              return std::string{"ilu-isai-"} + std::to_string(FLAGS_isai_power);
+         }},
+        {"general-isai", [] {
+             return std::string{"general-isai-"} +
+                    std::to_string(FLAGS_isai_power);
          }}};
     if (encoder.find(precond_name) == encoder.end()) {
         return precond_name;
diff --git a/benchmark/utils/preconditioners.hpp b/benchmark/utils/preconditioners.hpp
index ab58461f284..d926d2631db 100644
--- a/benchmark/utils/preconditioners.hpp
+++ b/benchmark/utils/preconditioners.hpp
@@ -305,6 +305,13 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_u_solver_factory(uisai)
                  .on(exec);
          }},
+        {"general-isai",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             return isai =
+                        gko::share(gko::preconditioner::GeneralIsai<>::build()
+                                       .with_sparsity_power(FLAGS_isai_power)
+                                       .on(exec));
+         }},
         {"overhead", [](std::shared_ptr<const gko::Executor> exec) {
              return gko::Overhead<etype>::build()
                  .with_criteria(gko::stop::ResidualNorm<etype>::build()

From 36ccd2cec32ae9e1a8480385e995cbf0a9f25190 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Mon, 7 Dec 2020 13:38:32 +0100
Subject: [PATCH 0506/1008] First cuda version

---
 benchmark/utils/preconditioners.hpp           |    7 +-
 common/components/warp_blas.hpp.inc           |   44 +
 common/preconditioner/isai_kernels.hpp.inc    |   89 +-
 core/preconditioner/isai.cpp                  |    2 +-
 cuda/preconditioner/isai_kernels.cu           |   18 +-
 cuda/test/preconditioner/isai_kernels.cpp     |  116 +-
 matrices/test/a_large.mtx                     | 9498 +++++++++++++----
 matrices/test/a_large_inv.mtx                 | 8792 +++++++--------
 omp/preconditioner/isai_kernels.cpp           |   92 +-
 omp/test/preconditioner/isai_kernels.cpp      |  116 +-
 reference/preconditioner/isai_kernels.cpp     |   39 +-
 .../test/preconditioner/isai_kernels.cpp      |    2 +-
 12 files changed, 12087 insertions(+), 6728 deletions(-)

diff --git a/benchmark/utils/preconditioners.hpp b/benchmark/utils/preconditioners.hpp
index d926d2631db..ade7472c040 100644
--- a/benchmark/utils/preconditioners.hpp
+++ b/benchmark/utils/preconditioners.hpp
@@ -307,10 +307,9 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
          }},
         {"general-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
-             return isai =
-                        gko::share(gko::preconditioner::GeneralIsai<>::build()
-                                       .with_sparsity_power(FLAGS_isai_power)
-                                       .on(exec));
+             return gko::preconditioner::GeneralIsai<>::build()
+                 .with_sparsity_power(FLAGS_isai_power)
+                 .on(exec);
          }},
         {"overhead", [](std::shared_ptr<const gko::Executor> exec) {
              return gko::Overhead<etype>::build()
diff --git a/common/components/warp_blas.hpp.inc b/common/components/warp_blas.hpp.inc
index 128fa124d77..0dce40e5147 100644
--- a/common/components/warp_blas.hpp.inc
+++ b/common/components/warp_blas.hpp.inc
@@ -83,6 +83,50 @@ __device__ __forceinline__ void apply_gauss_jordan_transform(
 }
 
 
+/**
+ * @internal
+ *
+ * Applies a Gauss-Jordan transformation (single step of Gauss-Jordan
+ * elimination) to a `max_problem_size`-by-`max_problem_size` matrix using the
+ * thread group `group. Each thread contributes one `row` of the matrix, and the
+ * routine uses warp shuffles to exchange data between rows. The transform is
+ * performed by using the `key_row`-th row and `key_col`-th column of the
+ * matrix.
+ */
+template <
+    int max_problem_size, typename Group, typename ValueType,
+    typename = std::enable_if_t<group::is_communicator_group<Group>::value>>
+__device__ __forceinline__ void apply_gauss_jordan_transform_with_rhs(
+    const Group &__restrict__ group, int32 key_row, int32 key_col,
+    ValueType *__restrict__ row, ValueType *__restrict__ rhs,
+    bool &__restrict__ status)
+{
+    auto key_col_elem = group.shfl(row[key_col], key_row);
+    auto key_rhs_elem = rhs[key_row] / key_col_elem;
+    if (key_col_elem == zero<ValueType>()) {
+        // TODO: implement error handling for GPUs to be able to properly
+        //       report it here
+        status = false;
+        return;
+    }
+    if (group.thread_rank() == key_row) {
+        key_col_elem = one<ValueType>() / key_col_elem;
+    } else {
+        key_col_elem = -row[key_col] / key_col_elem;
+    }
+#pragma unroll
+    for (int32 i = 0; i < max_problem_size; ++i) {
+        const auto key_row_elem = group.shfl(row[i], key_row);
+        if (group.thread_rank() == key_row) {
+            rhs[i] += key_rhs_elem * key_row_elem;
+            row[i] = zero<ValueType>();
+        }
+        row[i] += key_col_elem * key_row_elem;
+    }
+    row[key_col] = key_col_elem;
+}
+
+
 /**
  * @internal
  *
diff --git a/common/preconditioner/isai_kernels.hpp.inc b/common/preconditioner/isai_kernels.hpp.inc
index 1a71a6fbebf..61a871f7ad1 100644
--- a/common/preconditioner/isai_kernels.hpp.inc
+++ b/common/preconditioner/isai_kernels.hpp.inc
@@ -113,6 +113,8 @@ __forceinline__ __device__ void generic_generate(
 
         subwarp.sync();
 
+        index_type k = 0;
+
         for (IndexType nz = 0; nz < i_row_size; ++nz) {
             auto col = i_col_idxs[i_row_begin + nz];
             auto m_row_begin = m_row_ptrs[col];
@@ -125,6 +127,8 @@ __forceinline__ __device__ void generic_generate(
                 [&](IndexType, IndexType m_idx, IndexType i_idx,
                     config::lane_mask_type, bool valid) {
                     if (valid) {
+                        if (m_cols[m_idx + m_row_begin] < row && col == row)
+                            k++;
                         trisystem(nz, i_idx) = m_values[m_row_begin + m_idx];
                     }
                 });
@@ -140,7 +144,7 @@ __forceinline__ __device__ void generic_generate(
             local_row[i] = trisystem(i, local_id);
         }
 
-        const auto rhs = trs_solve(i_row_size, local_row, subwarp);
+        const auto rhs = trs_solve(i_row_size, local_row, subwarp, k);
 
         // Write back:
         if (local_id < i_row_size) {
@@ -167,27 +171,27 @@ __global__ __launch_bounds__(default_block_size) void generate_l_inverse(
     IndexType *__restrict__ excess_rhs_sizes,
     IndexType *__restrict__ excess_nnz)
 {
-    auto trs_solve = [](IndexType num_elems,
-                        const ValueType *__restrict__ local_row,
-                        group::thread_block_tile<subwarp_size> &subwarp) {
-        const int local_id = subwarp.thread_rank();
-        ValueType rhs =
-            local_id == num_elems - 1 ? one<ValueType>() : zero<ValueType>();
-        // Solve Triangular system
-        for (int d_col = num_elems - 1; d_col >= 0; --d_col) {
-            const auto elem = local_row[d_col];
-            if (d_col == local_id) {
-                rhs /= elem;
-            }
+    auto trs_solve =
+        [](IndexType num_elems, const ValueType *__restrict__ local_row,
+           group::thread_block_tile<subwarp_size> &subwarp, size_type) {
+            const int local_id = subwarp.thread_rank();
+            ValueType rhs = local_id == num_elems - 1 ? one<ValueType>()
+                                                      : zero<ValueType>();
+            // Solve Triangular system
+            for (int d_col = num_elems - 1; d_col >= 0; --d_col) {
+                const auto elem = local_row[d_col];
+                if (d_col == local_id) {
+                    rhs /= elem;
+                }
 
-            const ValueType bot = subwarp.shfl(rhs, d_col);
-            if (local_id < d_col) {
-                rhs -= bot * elem;
+                const ValueType bot = subwarp.shfl(rhs, d_col);
+                if (local_id < d_col) {
+                    rhs -= bot * elem;
+                }
             }
-        }
 
-        return rhs;
-    };
+            return rhs;
+        };
 
     generic_generate<subwarp_size, subwarps_per_block>(
         num_rows, m_row_ptrs, m_col_idxs, m_values, i_row_ptrs, i_col_idxs,
@@ -208,7 +212,8 @@ __global__ __launch_bounds__(default_block_size) void generate_u_inverse(
 {
     auto trs_solve = [](IndexType num_elems,
                         const ValueType *__restrict__ local_row,
-                        group::thread_block_tile<subwarp_size> &subwarp) {
+                        group::thread_block_tile<subwarp_size> &subwarp,
+                        size_type) {
         const int local_id = subwarp.thread_rank();
         ValueType rhs = local_id == 0 ? one<ValueType>() : zero<ValueType>();
         // Solve Triangular system
@@ -233,6 +238,50 @@ __global__ __launch_bounds__(default_block_size) void generate_u_inverse(
 }
 
 
+template <int subwarp_size, int subwarps_per_block, typename ValueType,
+          typename IndexType>
+__global__ __launch_bounds__(default_block_size) void generate_general_inverse(
+    IndexType num_rows, const IndexType *__restrict__ m_row_ptrs,
+    const IndexType *__restrict__ m_col_idxs,
+    const ValueType *__restrict__ m_values,
+    const IndexType *__restrict__ i_row_ptrs,
+    const IndexType *__restrict__ i_col_idxs, ValueType *__restrict__ i_values,
+    IndexType *__restrict__ excess_rhs_sizes,
+    IndexType *__restrict__ excess_nnz)
+{
+    auto general_solve =
+        [](IndexType num_elems, const ValueType *__restrict__ local_row,
+           group::thread_block_tile<subwarp_size> &subwarp, size_type k) {
+            __shared__ ValueType rhs[subwarp_size];
+            const int local_id = subwarp.thread_rank();
+            ValueType rhs[local_id] =
+                local_id == k ? one<ValueType>() : zero<ValueType>();
+
+            auto pivoted = subwarp.thread_rank() >= num_rows;
+            auto status = true;
+            for (size_type i = 0; i < num_rows; i++) {
+                const auto piv = choose_pivot(subwarp, row[i], pivoted);
+                if (local_id == piv) {
+                    perm = i;
+                    pivoted = true;
+                }
+                if (local_id == i) {
+                    trans_perm = piv;
+                }
+
+                apply_gauss_jordan_transform_with_rhs<max_problem_size>(
+                    subwarp, piv, i, row, rhs, status);
+            }
+
+            return rhs[local_id];
+        };
+
+    generic_generate<subwarp_size, subwarps_per_block>(
+        num_rows, m_row_ptrs, m_col_idxs, m_values, i_row_ptrs, i_col_idxs,
+        i_values, excess_rhs_sizes, excess_nnz, general_solve);
+}
+
+
 template <int subwarp_size, typename ValueType, typename IndexType>
 __global__ __launch_bounds__(default_block_size) void generate_excess_system(
     IndexType num_rows, const IndexType *__restrict__ m_row_ptrs,
diff --git a/core/preconditioner/isai.cpp b/core/preconditioner/isai.cpp
index 163ab58b921..ee423c97cc3 100644
--- a/core/preconditioner/isai.cpp
+++ b/core/preconditioner/isai.cpp
@@ -174,7 +174,7 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
                             .with_max_iters(excess_dim)
                             .on(exec),
                         gko::stop::ResidualNormReduction<ValueType>::build()
-                            .with_reduction_factor(1e-10)
+                            .with_reduction_factor(1e-6)
                             .on(exec))
                     .on(exec);
         } else if (is_lower) {
diff --git a/cuda/preconditioner/isai_kernels.cu b/cuda/preconditioner/isai_kernels.cu
index c33a37255d5..2e622768497 100644
--- a/cuda/preconditioner/isai_kernels.cu
+++ b/cuda/preconditioner/isai_kernels.cu
@@ -66,6 +66,7 @@ constexpr int subwarps_per_block{2};
 constexpr int default_block_size{subwarps_per_block * subwarp_size};
 
 
+#include "common/components/warp_blas.hpp.inc"
 #include "common/preconditioner/isai_kernels.hpp.inc"
 
 
@@ -112,7 +113,22 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
                               const matrix::Csr<ValueType, IndexType> *input,
                               matrix::Csr<ValueType, IndexType> *inverse,
                               IndexType *excess_rhs_ptrs,
-                              IndexType *excess_nz_ptrs) GKO_NOT_IMPLEMENTED;
+                              IndexType *excess_nz_ptrs)
+{
+    const auto num_rows = input->get_size()[0];
+
+    const dim3 block(default_block_size, 1, 1);
+    const dim3 grid(ceildiv(num_rows, block.x / subwarp_size), 1, 1);
+    kernel::generate_general_inverse<subwarp_size, subwarps_per_block>
+        <<<grid, block>>>(
+            static_cast<IndexType>(num_rows), input->get_const_row_ptrs(),
+            input->get_const_col_idxs(),
+            as_cuda_type(input->get_const_values()), inverse->get_row_ptrs(),
+            inverse->get_col_idxs(), as_cuda_type(inverse->get_values()),
+            excess_rhs_ptrs, excess_nz_ptrs);
+    components::prefix_sum(exec, excess_rhs_ptrs, num_rows + 1);
+    components::prefix_sum(exec, excess_nz_ptrs, num_rows + 1);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_GENERAL_INVERSE_KERNEL);
diff --git a/cuda/test/preconditioner/isai_kernels.cpp b/cuda/test/preconditioner/isai_kernels.cpp
index e5673bf4bbd..f53ad8f9d5f 100644
--- a/cuda/test/preconditioner/isai_kernels.cpp
+++ b/cuda/test/preconditioner/isai_kernels.cpp
@@ -53,7 +53,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
-enum struct matrix_type { lower, upper };
+enum struct matrix_type { lower, upper, general };
 class Isai : public ::testing::Test {
 protected:
     using value_type = double;
@@ -91,9 +91,14 @@ class Isai : public ::testing::Test {
         auto nz_dist = std::uniform_int_distribution<index_type>(1, row_limit);
         auto val_dist = std::uniform_real_distribution<value_type>(-1., 1.);
         mtx = Csr::create(ref);
-        mtx = gko::test::generate_random_triangular_matrix<Csr>(
-            n, n, true, for_lower_tm, nz_dist, val_dist, rand_engine, ref,
-            gko::dim<2>{n, n});
+        if (type == matrix_type::general) {
+            mtx = gko::test::generate_random_matrix<Csr>(
+                n, n, nz_dist, val_dist, rand_engine, ref, gko::dim<2>{n, n});
+        } else {
+            mtx = gko::test::generate_random_triangular_matrix<Csr>(
+                n, n, true, for_lower_tm, nz_dist, val_dist, rand_engine, ref,
+                gko::dim<2>{n, n});
+        }
         inverse = clone_allocations(mtx.get());
 
         d_mtx = Csr::create(cuda);
@@ -162,6 +167,29 @@ TEST_F(Isai, CudaIsaiGenerateUinverseShortIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, OmpIsaiGenerateAinverseShortIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 615, 31);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::Array<index_type> da1(omp, num_rows + 1);
+    auto da2 = da1;
+
+    gko::kernels::reference::isai::generate_tri_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
+    gko::kernels::omp::isai::generate_tri_inverse(
+        omp, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
+        false);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, r<value_type>::value);
+    GKO_ASSERT_ARRAY_EQ(a1, da1);
+    GKO_ASSERT_ARRAY_EQ(a2, da2);
+    ASSERT_EQ(a1.get_const_data()[num_rows], 0);
+}
+
+
 TEST_F(Isai, CudaIsaiGenerateLinverseLongIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 554, 64);
@@ -208,6 +236,28 @@ TEST_F(Isai, CudaIsaiGenerateUinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, OmpIsaiGenerateAinverseLongIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 695, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::Array<index_type> da1(omp, num_rows + 1);
+    auto da2 = da1;
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+    gko::kernels::omp::isai::generate_general_inverse(
+        omp, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, r<value_type>::value);
+    GKO_ASSERT_ARRAY_EQ(a1, da1);
+    GKO_ASSERT_ARRAY_EQ(a2, da2);
+    ASSERT_GT(a1.get_const_data()[num_rows], 0);
+}
+
+
 TEST_F(Isai, CudaIsaiGenerateExcessLinverseLongIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 518, 40);
@@ -270,6 +320,37 @@ TEST_F(Isai, CudaIsaiGenerateExcessUinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, OmpIsaiGenerateExcessAinverseLongIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 100, 51);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+    gko::Array<index_type> da1(omp, a1);
+    gko::Array<index_type> da2(omp, a2);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_nnz = a2.get_data()[num_rows];
+    auto excess = Csr::create(ref, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    auto dexcess = Csr::create(omp, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto de_rhs = Dense::create(omp, gko::dim<2>(e_dim, 1));
+
+    gko::kernels::reference::isai::generate_excess_system(
+        ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
+        excess.get(), e_rhs.get());
+    gko::kernels::omp::isai::generate_excess_system(
+        omp, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
+        da2.get_const_data(), dexcess.get(), de_rhs.get());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
+    GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
 TEST_F(Isai, CudaIsaiScatterExcessSolutionLIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 572, 52);
@@ -323,4 +404,31 @@ TEST_F(Isai, CudaIsaiScatterExcessSolutionUIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, OmpIsaiScatterExcessSolutionAIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 702, 45);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+    gko::Array<index_type> da1(omp, a1);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(omp);
+    de_rhs->copy_from(lend(e_rhs));
+    // overwrite -1 values with inverse
+    d_inverse->copy_from(lend(inverse));
+
+    gko::kernels::reference::isai::scatter_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get());
+    gko::kernels::omp::isai::scatter_excess_solution(
+        omp, da1.get_const_data(), de_rhs.get(), d_inverse.get());
+
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
 }  // namespace
diff --git a/matrices/test/a_large.mtx b/matrices/test/a_large.mtx
index 02225c91f92..47de5369594 100644
--- a/matrices/test/a_large.mtx
+++ b/matrices/test/a_large.mtx
@@ -1,35 +1,11 @@
-%%MatrixMarket matrix array real symmetric
+%%MatrixMarket matrix array real general
 100 100
- 9.507387586206898
- 16.9262875862069
--0.975640275862069
--9.527639310344828
--17.43514482758621
- 1.018235379310345
- 0.7068437931034484
- 0.544773048275862
- 0.04764781034482759
--0.1706930896551724
--0.1029656551724138
--0.06021632206896552
--0.002943987172413793
- 0.008551583448275862
- 0.007157647586206897
--0.0004222307034482758
--0.0006059579793103449
--0.0008743339999999999
--1.039474275862069e-05
--0.0002568787586206896
--0.0003108931034482759
+ 1
+ 0.0352221
  0
--7.272933103448276e-05
--0.0001269617379310345
+-0.126265
  0
--2.302761103448276e-05
--3.457650965517242e-05
  0
--1.218220137931034e-05
--1.494798827586207e-05
  0
  0
  0
@@ -39,7 +15,9 @@
  0
  0
  0
+ 0.955826
  0
+ 0.448404
  0
  0
  0
@@ -53,101 +31,123 @@
  0
  0
  0
+ 0.588998
  0
  0
  0
+-0.301462
  0
  0
  0
+ 0.287703
  0
  0
  0
+ 0.25185
  0
+ 0.49784
  0
+ 0.547594
+ 0.85276
+ 0.328431
  0
  0
  0
+-0.436379
+ 0.960822
  0
  0
+ 0.55008
  0
  0
  0
+ 0.996657
  0
  0
  0
+-0.07184500000000001
  0
  0
  0
  0
+-0.130761
  0
+ 0.667564
+ 0.655325
  0
+ 0.238939
+-0.184559
+-0.5358270000000001
+ 0.0949749
+-0.489252
+ 0.0186587
  0
+ 0.607581
+-0.408656
  0
  0
+-0.384484
+ 0.575102
  0
+-0.000766162
  0
  0
  0
+-0.539473
  0
  0
  0
  0
  0
+-0.71202
  0
+ 1
  0
  0
  0
  0
  0
  0
- 35.75833655172414
--1.688903379310345
--16.72711310344827
--35.02702137931035
- 1.771132137931034
--0.3314247655172414
--0.2669652344827586
--0.002670817103448276
- 0.06519995931034483
- 0.03932364551724138
- 0.02300669448275862
- 0.008551583448275862
--0.02059450068965517
--0.01767217379310345
--0.0006059579793103449
- 0.0002803287724137931
- 0.001517457793103448
--0.0002568787586206896
- 0.0002676622965517241
- 0.0005217609586206897
--7.272933103448276e-05
- 3.796375793103449e-05
- 0.0001646504482758621
--2.302761103448276e-05
- 1.623803310344828e-05
- 4.686651172413793e-05
--1.218220137931034e-05
- 1.863814551724138e-05
- 3.459778e-05
  0
+ 0.479209
  0
  0
  0
  0
+ 0.762845
  0
+ 0.403298
+ 0.64392
  0
+-0.457379
  0
  0
  0
  0
+-0.0277301
+-0.698778
  0
+-0.2764
  0
  0
+ 0.766326
  0
  0
+-0.403154
  0
+-0.0430981
+-0.496861
  0
+-0.885747
  0
+ 0.747283
+-0.0575297
+-0.59034
+-0.542502
+-0.0261681
+-0.187808
+ 0.820244
+ 0.403374
  0
  0
  0
@@ -155,103 +155,107 @@
  0
  0
  0
+-0.8516820000000001
  0
  0
  0
+-0.81643
  0
  0
+-0.763212
  0
  0
  0
  0
  0
+-0.955558
+ 0.8234089999999999
  0
  0
+ 0.115741
  0
  0
  0
  0
  0
+-0.63423
  0
  0
  0
  0
  0
  0
+-0.635006
+ 0.154736
+ 0.21157
  0
  0
+ 0.115656
  0
  0
  0
+ 0.477922
  0
  0
  0
+-0.125002
  0
  0
+-0.326679
+ 0.1046859999999999
  0
  0
+-0.728782
  0
+ 0.677805
  0
+ 0.333973
  0
  0
+ 0.72497
  0
  0
+-0.94803
  0
+-0.266554
  0
  0
- 0.6378757172413794
- 1.056251931034483
- 1.986631793103448
--0.8257926896551724
- 0.2961380896551724
- 0.0139804075862069
- 0.3849934827586207
--0.3467231310344828
--0.2091442206896552
--0.1223217310344828
- 0.007157647586206897
--0.01767217379310345
--0.01427218413793103
--0.0008743339999999999
- 0.001517457793103448
- 0.001418165586206896
--0.0003108931034482759
- 0.0005217609586206897
- 0.0004680165862068966
--0.0001269617379310345
- 0.0001646504482758621
- 0.0002310678620689655
--3.457650965517242e-05
- 4.686651172413793e-05
- 5.539589724137931e-05
--1.494798827586207e-05
- 3.459778e-05
- 3.289396965517241e-05
  0
+-0.550107
  0
  0
  0
  0
+-0.777374
  0
  0
  0
  0
+-0.343653
  0
  0
  0
  0
  0
+ 0.975595
+-0.253923
  0
  0
+-0.727818
  0
+ 0.813276
  0
  0
  0
  0
  0
+-0.873077
  0
  0
  0
+-0.494212
+ 0.183104
+-0.820892
  0
  0
  0
@@ -263,11 +267,16 @@
  0
  0
  0
+ 0.39238
  0
  0
+-0.286842
  0
+-0.920755
  0
+ 0.573967
  0
+ 0.545054
  0
  0
  0
@@ -284,49 +293,33 @@
  0
  0
  0
+ 0.0808275
  0
+-0.630044
  0
  0
+ 0.46123
+-0.77565
  0
  0
  0
+ 1
  0
  0
  0
  0
+-0.577415
  0
  0
  0
- 25.37018689655172
- 0.03815196137931035
--11.4157724137931
--15.96946482758621
- 16.45557586206897
- 10.17765310344828
- 0.02039901793103448
- 0.03895268275862069
- 0.01341664
- 0.3451321586206896
- 0.1764193931034483
- 0.1143378620689655
--0.0004587904827586207
--0.0003019009586206896
--0.000847974
- 0.0001612334206896552
--0.0003495643793103448
--0.0006524910896551724
- 4.129113241379311e-05
--7.059000689655173e-05
--0.0002015320344827586
- 1.349831379310345e-05
--2.608321448275862e-05
--5.585081931034483e-05
  0
--1.983652965517241e-05
--3.200872620689655e-05
  0
  0
+-0.790793
  0
+ 0.319043
+ 0.505891
+ 0.333113
  0
  0
  0
@@ -349,12 +342,19 @@
  0
  0
  0
+-0.295016
  0
+ 0.586125
  0
  0
  0
+-0.284215
+ 0.475873
+-0.127442
  0
+-0.5929720000000001
  0
+-0.573142
  0
  0
  0
@@ -375,60 +375,59 @@
  0
  0
  0
+-0.39057
  0
  0
  0
  0
+ 0.416972
+-0.776501
+-0.612899
  0
  0
+-0.966733
+-0.413408
+ 0.426742
  0
+ 0.139594
+ 0.47202
  0
+-0.0425343
+ 0.0217776
  0
  0
+-0.37716
  0
+-0.81123
+ 0.689438
  0
  0
  0
  0
+ 1
  0
  0
  0
+-0.600885
  0
- 57.50793517241379
- 10.55127793103448
- 17.58907793103448
--22.60832068965517
--12.56414965517241
--0.4182823655172414
- 0.2225919724137931
--0.03660212482758621
- 0.2081843931034483
- 0.1064208827586207
- 0.06896635172413793
--0.0003019009586206896
--0.0007321437931034484
- 0.0005469224551724138
--0.0003495643793103448
- 5.688953448275862e-05
- 0.0005623283724137931
--7.059000689655173e-05
--2.251513862068965e-05
- 0.0001300750965517241
--2.608321448275862e-05
  0
- 4.492035655172414e-05
--1.983652965517241e-05
- 1.342995172413793e-05
- 4.503403724137932e-05
+-0.9369960000000001
+-0.913012
  0
  0
  0
  0
+ 0.0549891
+ 0.789766
+ 0.155261
+-0.626726
+ 0.438602
  0
  0
  0
  0
  0
+ 0.912946
  0
  0
  0
@@ -436,13 +435,18 @@
  0
  0
  0
+-0.820889
  0
  0
  0
  0
  0
  0
+-0.232923
+-0.379572
  0
+-0.881664
+ 0.177548
  0
  0
  0
@@ -451,9 +455,13 @@
  0
  0
  0
+ 0.8491919999999999
  0
+ 0.820483
+-0.0380151
  0
  0
+-0.370664
  0
  0
  0
@@ -464,57 +472,47 @@
  0
  0
  0
+ 0.425507
  0
  0
+-0.33833
+ 0.905755
  0
+ 0.449572
  0
  0
+ 0.000147088
  0
  0
  0
+ 0.423966
  0
+-0.8492420000000001
  0
  0
+ 0.174948
  0
  0
  0
  0
  0
+ 0.389784
  0
+-0.293117
  0
  0
+-0.663598
  0
  0
+-0.598401
+ 1
  0
  0
  0
  0
  0
- 9.437159310344827
- 9.480863448275862
--12.05305724137931
--8.601750344827586
- 0.7408009655172414
--0.3254658827586207
- 0.08089086206896552
- 0.1217593172413793
- 0.0622367475862069
- 0.04034112068965517
--0.000847974
- 0.0005469224551724138
- 0.000609310448275862
--0.0006524910896551724
- 0.0005623283724137931
- 0.0008052513103448275
--0.0002015320344827586
- 0.0001300750965517241
- 0.0003032804068965517
--5.585081931034483e-05
- 4.492035655172414e-05
- 7.725388965517241e-05
--3.200872620689655e-05
- 4.503403724137932e-05
- 5.818863517241379e-05
+-0.284778
+-0.259316
  0
  0
  0
@@ -524,12 +522,16 @@
  0
  0
  0
+ 0.225826
  0
  0
  0
+-0.418982
+ 0.648581
  0
  0
  0
+-0.6842510000000001
  0
  0
  0
@@ -537,9 +539,15 @@
  0
  0
  0
+ 0.341594
  0
+ 0.620384
+ 0.172187
+-0.427182
  0
+-0.810075
  0
+-0.484587
  0
  0
  0
@@ -549,20 +557,25 @@
  0
  0
  0
+ 0.298056
  0
  0
  0
+-0.492891
  0
+-0.796257
  0
  0
  0
  0
+-0.284484
  0
  0
  0
  0
  0
  0
+ 0.911418
  0
  0
  0
@@ -575,58 +588,38 @@
  0
  0
  0
+ 0.967934
  0
  0
  0
  0
+-0.834484
  0
  0
  0
  0
  0
  0
- 18.47544620689655
--15.36471862068966
--17.99068137931035
--2.696934344827586
--1.413146551724138
- 7.988540689655172
--0.6945817931034483
--0.5530225862068966
- 0.09409567586206896
- 0.3679345655172414
- 0.1812813655172414
- 0.1330178413793103
--0.0004751480896551724
--0.0003164226827586207
--0.0008712627586206896
- 4.162283310344828e-05
--0.0001280162689655172
--0.0006400958137931035
- 2.724029793103448e-05
--6.291741172413793e-05
--0.0001816629586206897
--1.494309931034483e-05
--4.940948068965517e-05
--9.557138620689655e-05
  0
--1.656822482758621e-05
--2.765900206896552e-05
  0
--8.080044827586207e-06
--1.095650551724138e-05
+ 0.27267
  0
  0
  0
+ 0.99203039
  0
  0
+-0.646975
  0
+-0.533131
  0
  0
  0
  0
  0
+ 0.233432
  0
+ 0.546378
  0
  0
  0
@@ -638,24 +631,32 @@
  0
  0
  0
+-0.497845
  0
+ 0.527705
  0
  0
+ 0.487902
+ 0.821416
  0
  0
  0
+-0.152157
  0
+-0.206447
  0
  0
  0
  0
  0
+ 0.746186
  0
  0
  0
  0
  0
  0
+ 0.128726
  0
  0
  0
@@ -667,8 +668,11 @@
  0
  0
  0
+-0.985511
  0
  0
+ 0.52516
+ 0.240263
  0
  0
  0
@@ -679,62 +683,52 @@
  0
  0
  0
- 26.94837103448276
- 2.974075379310345
--1.330644827586207
--4.051198
- 9.581779310344828
--0.3850140275862069
- 0.1554554965517241
--0.704422
- 0.1880755448275862
- 0.09266883448275862
- 0.06799222344827587
--0.0003164226827586207
--0.0006935188965517242
- 0.0006210286965517241
--0.0001280162689655172
--0.0001892681103448276
- 0.000284692524137931
--6.291741172413793e-05
--2.070887655172414e-05
- 0.0001251815931034483
--4.940948068965517e-05
- 2.416879172413793e-05
- 0.0001406100896551724
--1.656822482758621e-05
- 1.150021172413793e-05
- 4.124594275862069e-05
--8.080044827586207e-06
- 1.361720206896552e-05
- 2.951842413793103e-05
  0
+ 0.547479
  0
  0
+-0.656579
  0
  0
+-0.879771
  0
  0
  0
  0
+ 0.166345
  0
  0
  0
+ 0.611135
  0
  0
  0
  0
  0
+-0.439705
  0
+ 1
+ 0.259374
+ 0.5889450000000001
  0
+ 0.0143551
  0
  0
  0
+-0.763266
+ 0.825677
+ 0.733819
+-0.758653
+-0.0877497
  0
+ 0.0547954
+ 0.937111
  0
  0
  0
+ 0.485806
  0
+-0.615699
  0
  0
  0
@@ -742,170 +736,164 @@
  0
  0
  0
+-0.661019
+-0.321536
  0
  0
+-0.822585
  0
  0
  0
+ 0.640329
  0
  0
  0
  0
+ 0.5626910000000001
+-0.8737200000000001
  0
  0
  0
+-0.337115
  0
  0
  0
+ 0.627238
  0
  0
  0
+-0.566686
+-0.639248
  0
  0
+-0.0899401
  0
  0
  0
  0
  0
+ 0.648264
  0
  0
  0
  0
  0
- 48.22368413793104
- 7.882573103448276
- 9.493863448275862
--39.74624068965517
--0.2371099724137931
--0.08082382068965517
--0.1446659310344828
- 0.1218918068965517
- 0.0600538924137931
- 0.04407042827586207
--0.0008712627586206896
- 0.0006210286965517241
- 0.0007909042068965517
--0.0006400958137931035
- 0.000284692524137931
- 0.001177266482758621
--0.0001816629586206897
- 0.0001251815931034483
- 0.0002973719862068966
--9.557138620689655e-05
- 0.0001406100896551724
- 0.0002234498896551724
--2.765900206896552e-05
- 4.124594275862069e-05
- 5.564861034482758e-05
--1.095650551724138e-05
- 2.951842413793103e-05
- 3.18748248275862e-05
  0
  0
+-0.170385
  0
  0
  0
  0
+ 0.891823
  0
+-0.591547
  0
  0
+ 0.467363
  0
  0
  0
  0
+-0.25665
  0
  0
+ 0.874235
+ 0.6489279999999999
  0
  0
  0
  0
+ 0.568695
  0
  0
  0
+ 1
  0
  0
+-0.852568
  0
  0
  0
+ 0.688815
  0
+-0.868268
+ 0.71848
+ 0.874924
  0
+-0.6378200000000001
+ 0.793312
  0
  0
  0
  0
  0
+-0.772224
+-0.994532
  0
+-0.996234
+-0.491159
  0
  0
  0
  0
  0
+ 0.875526
  0
  0
  0
  0
  0
+ 0.373084
  0
  0
  0
  0
+-0.742054
  0
+ 0.876749
  0
  0
  0
  0
  0
  0
+-0.569966
  0
  0
  0
  0
+ 0.876426
  0
+-0.30766
  0
  0
+-0.174341
  0
- 15.58613724137931
--14.6339
--17.76058620689655
--12.1547724137931
- 16.54367517241379
- 9.712164137931035
--0.7461117931034483
--0.308096724137931
--0.2552290827586207
- 0.3760677793103449
- 0.1616321793103448
- 0.1301900965517241
--0.0005005506620689655
--0.0002814506068965517
--0.0008073446896551724
- 0.0001293575793103448
--0.0003224790206896552
--0.0006272998000000001
--4.998646896551724e-05
--0.0001947931655172414
--0.0002682746068965517
+-0.133573
  0
--5.18163124137931e-05
--6.491219999999999e-05
--1.21305e-05
--1.964566413793103e-05
--2.043121655172414e-05
  0
  0
  0
+ 0.649197
+ 0.920309
  0
  0
+ 0.525194
+ 0.263471
  0
+ 0.7017409999999999
  0
  0
  0
  0
  0
  0
+ 0.614198
  0
  0
  0
+ 0.879014
  0
  0
  0
@@ -913,23 +901,36 @@
  0
  0
  0
+ 0.624846
+ 0.691658
  0
  0
  0
  0
  0
+ 0.283198
+ 0.314876
  0
+ 0.972719
  0
  0
+ 0.389011
  0
  0
  0
+-0.928426
+ 0.19006
  0
+-0.114557
  0
  0
+-0.346923
+ 0.329702
  0
+ 0.355238
  0
  0
+ 0.11432
  0
  0
  0
@@ -941,58 +942,44 @@
  0
  0
  0
+ 0.634883
+ 0.933555
  0
  0
  0
  0
  0
+-0.0350987
  0
  0
  0
  0
+-0.827219
+ 0.787873
+ 0.230721
  0
  0
+ 0.143212
+-0.160876
  0
  0
  0
- 27.98288620689655
- 4.112758
- 16.56426068965517
--24.41440413793104
--13.05385034482759
--0.5589071310344828
- 0.2047835862068965
--0.08497348275862068
- 0.1852888413793103
- 0.07963971034482759
- 0.06414277103448275
--0.0002814506068965517
--0.0007446749655172413
- 0.0005298609379310345
--0.0003224790206896552
- 4.388975103448276e-05
- 0.0005496528965517241
--0.0001947931655172414
- 0.0002077372965517241
- 0.0004991324068965517
--5.18163124137931e-05
- 5.851311448275862e-05
- 0.0001158903862068966
--1.964566413793103e-05
- 4.757615862068966e-05
- 6.821318482758621e-05
  0
  0
  0
  0
+ 0.573569
  0
  0
  0
  0
+-0.0518283
  0
  0
  0
  0
+ 0.763983
+-0.288771
  0
  0
  0
@@ -1000,26 +987,35 @@
  0
  0
  0
+ 0.220261
+ 0.00302326
  0
+ 0.678679
  0
  0
+ 0.789704
  0
+ 0.0130651
  0
  0
  0
  0
  0
+-0.980996
  0
+-0.741042
  0
  0
  0
  0
  0
  0
+ 1
  0
  0
  0
  0
+ 0.0606227
  0
  0
  0
@@ -1029,47 +1025,31 @@
  0
  0
  0
+ 0.797913
+ 0.750262
  0
+ 0.752896
  0
+-0.832524
  0
+ 0.358872
  0
  0
+-0.0340256
  0
  0
+-0.487125
  0
  0
+-0.915477
  0
  0
  0
+-0.5155960000000001
  0
  0
  0
  0
- 49.60891586206897
- 9.742552413793103
--13.04671103448276
--9.648690344827587
- 0.04352490413793103
--0.695332275862069
--0.1590438413793104
- 0.1359581103448276
- 0.05843221655172414
- 0.04707018827586208
--0.0008073446896551724
- 0.0005298609379310345
- 0.0005905069172413793
--0.0006272998000000001
- 0.0005496528965517241
- 0.0008305266206896552
--0.0002682746068965517
- 0.0004991324068965517
- 0.000532731075862069
--6.491219999999999e-05
- 0.0001158903862068966
- 0.0001111821586206897
--2.043121655172414e-05
- 6.821318482758621e-05
- 5.292552482758621e-05
  0
  0
  0
@@ -1082,11 +1062,13 @@
  0
  0
  0
+ 0.5317229999999999
  0
  0
  0
  0
  0
+-0.5104300000000001
  0
  0
  0
@@ -1103,9 +1085,11 @@
  0
  0
  0
+-0.72778
  0
  0
  0
+-0.720365
  0
  0
  0
@@ -1114,54 +1098,39 @@
  0
  0
  0
+-0.835952
  0
  0
  0
+ 0.018154
  0
+-0.749382
  0
+ 0.444569
+-0.452357
  0
+-0.967843
  0
+ 1
  0
  0
+ 0.898672
  0
  0
+-0.91964
+-0.958815
  0
  0
  0
+-0.102141
  0
+-0.8037029999999999
  0
  0
+-0.53889
  0
  0
  0
- 16.02089379310345
--14.31042965517241
--18.78710758620689
--2.894899931034483
--1.735033034482759
- 9.111203448275862
--0.7516644827586206
--0.4936843103448276
- 0.01360813586206897
- 0.3822822068965517
- 0.1545104068965517
- 0.1480570896551724
--0.0004364188
--0.000321631675862069
--0.0009492923448275862
--0.0004946472275862069
--0.0004124728344827587
--0.0006992057931034482
--5.768837517241379e-05
--0.0001635274206896552
--0.0002360507931034483
--5.698394896551724e-05
--4.544111448275862e-05
--5.163884344827587e-05
--1.212235793103448e-05
--1.815835103448276e-05
--1.920040068965517e-05
--7.625637241379311e-06
  0
  0
  0
@@ -1170,22 +1139,29 @@
  0
  0
  0
+ 0.5618339999999999
  0
+-0.848769
+-0.581632
  0
  0
  0
+ 0.651222
  0
  0
+ 0.827912
  0
  0
  0
  0
+ 0.215715
  0
  0
  0
  0
  0
  0
+ 0.448614
  0
  0
  0
@@ -1193,69 +1169,60 @@
  0
  0
  0
+ 0.873539
  0
  0
  0
+ 0.240421
  0
  0
  0
  0
+-0.112234
+-0.289582
  0
  0
+-0.159832
+-0.750553
  0
  0
  0
  0
  0
+-0.804454
  0
  0
  0
+-0.332287
+ 0.94617
  0
+-0.116449
  0
  0
  0
  0
  0
+-0.0808903
  0
+-0.5601080000000001
  0
  0
  0
  0
+-0.828614
  0
- 28.45061931034483
- 3.774955379310345
--1.716966827586207
--4.298854413793103
- 9.849167586206896
--0.3011831241379311
- 0.2433643172413793
--0.7025911724137931
- 0.1643025862068966
- 0.06641086413793103
- 0.06363304137931035
--0.000321631675862069
--0.000687496572413793
- 0.0006485134896551724
--0.0004124728344827587
- 0.0002017957862068965
- 0.001505337862068966
--0.0001635274206896552
- 0.0001759725793103448
- 0.0004587468068965517
--4.544111448275862e-05
- 0.0001536152620689655
- 0.0002499883586206897
--1.815835103448276e-05
- 4.507379862068966e-05
- 6.605859103448276e-05
+ 0.850285
+ 1
  0
- 2.645245724137931e-05
- 3.18563475862069e-05
  0
  0
+-0.559895
  0
  0
  0
+ 0.143098
+-0.21878
+-0.805248
  0
  0
  0
@@ -1265,20 +1232,26 @@
  0
  0
  0
+-0.574879
  0
  0
  0
  0
  0
  0
+ 0.228353
+-0.650577
  0
+-0.660778
  0
  0
  0
  0
  0
  0
+-0.253299
  0
+ 0.39971
  0
  0
  0
@@ -1288,16 +1261,22 @@
  0
  0
  0
+-0.987043
+-0.7430060000000001
+ 0.86563
  0
  0
  0
  0
+-0.518626
  0
  0
  0
  0
  0
  0
+-0.470546
+-0.664445
  0
  0
  0
@@ -1305,47 +1284,28 @@
  0
  0
  0
+ 0.444533
+ 0.0268659
+ 0.132374
  0
  0
+ 0.570704
  0
  0
- 48.99697103448276
- 9.102015172413793
- 9.840553103448276
--39.01642137931034
--0.2548845310344828
--0.05982489586206897
--0.1680335103448276
- 0.132341724137931
- 0.05348842827586207
- 0.0512578
--0.0009492923448275862
- 0.0006485134896551724
- 0.001006845103448276
--0.0006992057931034482
- 0.001505337862068966
- 0.001865532275862069
--0.0002360507931034483
- 0.0004587468068965517
- 0.0005203627655172414
--5.163884344827587e-05
- 0.0002499883586206897
- 0.0002177119448275862
--1.920040068965517e-05
- 6.605859103448276e-05
- 5.244864413793103e-05
+-0.0331565
  0
- 3.18563475862069e-05
- 2.126724551724138e-05
  0
+ 0.36033
  0
  0
+ 0.819901
  0
  0
  0
  0
  0
  0
+-0.773919
  0
  0
  0
@@ -1353,10 +1313,15 @@
  0
  0
  0
+ 1
  0
  0
  0
+ 0.842892
+ 0.890227
  0
+-0.151101
+-0.576108
  0
  0
  0
@@ -1374,12 +1339,18 @@
  0
  0
  0
+ 0.981993
+ 0.906771
+ 0.264393
  0
  0
  0
  0
+-0.626338
  0
  0
+-0.561011
+ 0.869959
  0
  0
  0
@@ -1395,41 +1366,20 @@
  0
  0
  0
- 14.38731862068965
--13.32846620689655
--18.41909586206896
--10.41646275862069
- 15.54403517241379
- 9.428503448275862
--0.8028775172413793
--0.2431268413793103
--0.2870848275862069
- 0.3909963517241379
- 0.1420076413793104
- 0.1467314
--0.001137678068965517
- 0.004599143655172414
- 0.004705009379310345
--0.000214330924137931
--0.0004988466965517241
--0.0004825145448275862
--0.0001402028965517241
--8.782599310344828e-05
--6.930295172413793e-05
--2.690888068965517e-05
--3.668687034482759e-05
--2.839123103448276e-05
--1.357759931034483e-05
  0
  0
--1.056255931034483e-05
  0
  0
  0
+ 0.009727080000000001
+ 0.270429
+ 0.800198
  0
  0
+-0.612209
  0
  0
+ 0.08075060000000001
  0
  0
  0
@@ -1439,8 +1389,13 @@
  0
  0
  0
+-0.416234
+-0.262502
  0
+-0.907022
+ 0.27517
  0
+-0.341087
  0
  0
  0
@@ -1448,21 +1403,34 @@
  0
  0
  0
+ 0.553862
+-0.98084
+-0.217547
  0
+ 0.342659
  0
+ 0.308032
  0
  0
+-0.08850769999999999
+-0.28468
+ 1
  0
  0
  0
  0
  0
  0
+-0.969376
  0
  0
  0
+-0.753979
  0
  0
+ 0.827613
+-0.525857
+ 0.621262
  0
  0
  0
@@ -1474,84 +1442,83 @@
  0
  0
  0
+ 0.758388
  0
+ 0.179235
  0
+-0.967231
+-0.388903
  0
+-0.457423
+-0.268806
  0
  0
  0
- 29.31237310344827
- 4.709881241379311
- 15.56635172413793
--25.4683475862069
--13.95292965517241
--0.5074189586206896
- 0.2689451793103448
--0.05748361172413793
- 0.1580329931034483
- 0.05739964482758621
- 0.05930520896551725
- 0.004599143655172414
--0.01532399034482759
--0.01590466413793103
--0.0004988466965517241
- 0.0008880568965517242
- 0.001252216413793104
--8.782599310344828e-05
- 0.0005389707379310345
- 0.0005447447586206896
--3.668687034482759e-05
- 0.0001315631655172414
- 0.0001288906
  0
- 6.055195241379311e-05
- 5.131481517241379e-05
+-0.0795442
  0
- 3.90733e-05
- 4.276421103448276e-05
  0
  0
  0
  0
+-0.921382
  0
  0
  0
+-0.534448
  0
  0
  0
  0
+-0.0177686
  0
  0
+ 0.519825
  0
  0
  0
  0
  0
+ 0.572676
  0
  0
  0
+-0.136917
  0
+ 0.23601
  0
+-0.330842
  0
+ 0.37795
+ 0.75462
  0
+ 0.882412
+ 0.028373
  0
+ 0.177926
  0
  0
+-0.876395
  0
  0
  0
+-0.950016
  0
  0
  0
  0
  0
  0
+-0.996262
  0
  0
  0
  0
  0
+ 1.875563
  0
+ 0.960957
+ 0.787272
  0
  0
  0
@@ -1559,63 +1526,48 @@
  0
  0
  0
+ 0.159258
  0
+-0.0819941
  0
  0
  0
  0
- 49.72536137931034
- 9.432560689655173
--13.94951586206897
--10.32325103448276
--0.01740205448275862
--0.6835017379310344
--0.205150275862069
- 0.1514317448275862
- 0.05499816275862069
- 0.05683020413793103
- 0.004705009379310345
--0.01590466413793103
--0.01604782275862069
--0.0004825145448275862
- 0.001252216413793104
- 0.0008046601379310344
--6.930295172413793e-05
- 0.0005447447586206896
- 0.0002784732
--2.839123103448276e-05
- 0.0001288906
- 6.475428482758621e-05
+ 0.675694
  0
- 5.131481517241379e-05
- 2.146366551724138e-05
  0
- 4.276421103448276e-05
- 2.558881724137931e-05
  0
  0
  0
+-0.585519
  0
  0
  0
  0
+ 0.421643
  0
+ 0.920991
  0
+-0.120211
+ 0.455017
  0
  0
  0
  0
  0
+-0.534284
  0
  0
  0
  0
  0
  0
+-0.520883
  0
  0
  0
  0
+ 0.971579
  0
  0
  0
@@ -1626,63 +1578,55 @@
  0
  0
  0
+-0.729364
  0
  0
+ 0.371877
+-0.905417
  0
+ 0.354084
  0
+-0.367703
+ 0.0162526
  0
  0
+ 0.591507
  0
  0
+ 0.578126
  0
+-0.55881
  0
  0
+-0.9078349999999999
  0
  0
+ 0.458132
  0
+ 0.915848
  0
  0
  0
  0
  0
+-0.942116
  0
  0
- 14.57719862068966
--12.69538896551724
--19.20994551724138
--3.454179586206896
--2.41915475862069
- 9.660764137931034
--0.04239575655172414
--0.5028410827586207
- 0.670439075862069
--0.1464690620689655
- 0.1695998827586207
--0.3686785379310345
--0.001028798068965517
- 0.003280225241379311
- 0.003606045793103448
--0.0006812032068965517
- 2.989333379310345e-05
- 2.493868e-05
--0.0001392868827586207
--8.673175862068966e-05
--6.994188965517242e-05
--4.774673310344828e-05
- 9.000826896551725e-06
  0
--2.55520324137931e-05
- 3.123000965517241e-05
- 2.804268689655172e-05
+ 0.59726
+ 0.728936
  0
- 1.550586413793103e-05
- 1.203971862068966e-05
+ 1.819113
  0
  0
+-0.170334
+-0.226999
  0
  0
  0
  0
+-0.508684
+ 0.42599
  0
  0
  0
@@ -1697,6 +1641,7 @@
  0
  0
  0
+-0.30392
  0
  0
  0
@@ -1716,6 +1661,8 @@
  0
  0
  0
+ 0.637889
+ 0.464512
  0
  0
  0
@@ -1727,47 +1674,28 @@
  0
  0
  0
+-0.5139860000000001
  0
  0
- 30.26506620689655
- 3.197392896551724
--2.489333103448276
--5.28189648275862
- 11.20735931034483
- 0.02861035103448276
- 0.3081422482758621
--0.4085184
--0.05319866344827586
- 0.0615965724137931
--0.1339024827586207
- 0.003280225241379311
--0.01158284068965517
--0.01263172896551724
- 2.989333379310345e-05
- 0.001456756344827586
- 0.001784068413793103
--8.673175862068966e-05
- 0.0005243722068965517
- 0.0005441620827586206
- 9.000826896551725e-06
- 0.0002030405034482759
- 0.0001759869310344828
- 3.123000965517241e-05
- 0.0001052479655172414
- 0.0001238030482758621
- 1.550586413793103e-05
- 3.979123448275862e-05
- 3.929856137931034e-05
  0
  0
+ 0.701715
+ 0.698952
  0
  0
+-0.10664
+-0.6580510000000001
  0
  0
  0
  0
+-0.21736
+-0.614479
  0
+-0.0332876
+ 0.0608625
  0
+-0.880495
  0
  0
  0
@@ -1776,6 +1704,7 @@
  0
  0
  0
+-0.855899
  0
  0
  0
@@ -1785,14 +1714,19 @@
  0
  0
  0
+-0.338955
  0
  0
+ 1
  0
  0
  0
+-0.88388
+ 0.900593
  0
  0
  0
+ 0.259993
  0
  0
  0
@@ -1803,55 +1737,38 @@
  0
  0
  0
+-0.00559104
  0
  0
  0
+ 0.903313
+-0.434739
+-0.0707097
+-0.298429
  0
+ 0.771146
  0
  0
  0
- 48.3293
- 9.761829655172413
- 11.28652275862069
--37.70450068965517
--0.003006942689655173
--0.04476122689655172
- 0.06036403517241379
--0.05496545172413794
- 0.06364721103448276
--0.1383565103448276
- 0.003606045793103448
--0.01263172896551724
--0.01397869310344828
- 2.493868e-05
- 0.001784068413793103
- 0.0008068340000000001
--6.994188965517242e-05
- 0.0005441620827586206
- 0.0002883744344827586
  0
- 0.0001759869310344828
- 7.526785517241378e-05
- 2.804268689655172e-05
- 0.0001238030482758621
- 7.854210344827586e-05
- 1.203971862068966e-05
- 3.929856137931034e-05
- 1.96929524137931e-05
  0
+-0.199954
  0
  0
  0
  0
+ 0.00153506
  0
  0
  0
+ 0.5148740000000001
  0
  0
  0
  0
  0
  0
+-0.802113
  0
  0
  0
@@ -1861,19 +1778,28 @@
  0
  0
  0
+-0.450203
+-0.630819
+ 0.00152143
  0
  0
  0
  0
  0
+-0.932658
  0
  0
+ 0.232096
  0
  0
+ 0.627546
  0
  0
  0
+-0.899024
  0
+ 0.239356
+ 0.363986
  0
  0
  0
@@ -1885,49 +1811,27 @@
  0
  0
  0
+-0.877309
  0
  0
  0
+ 0.767697
+ 0.5294180000000001
  0
+ 1
  0
- 14.86223103448276
--11.17081172413793
--19.21202551724138
--11.00837724137931
- 14.75145172413793
- 9.317006206896551
- 0.6111585034482758
--0.4778164413793103
- 0.09426643448275861
- 0.06417459172413793
--0.07060804827586208
- 0.1382952689655172
--0.0009058560689655172
- 0.0001809489034482759
- 8.133559999999999e-05
--0.0002433233172413793
--7.387113103448276e-05
--3.677954689655172e-05
--6.299328758620689e-05
- 3.772604413793103e-05
- 1.70590648275862e-05
--3.035420551724138e-05
- 7.19205724137931e-05
- 4.475804965517241e-05
  0
- 2.684367931034483e-05
- 1.480232620689655e-05
  0
- 1.725162965517241e-05
- 1.196528827586207e-05
  0
  0
  0
+ 0.705021
  0
  0
  0
  0
  0
+ 0.452926
  0
  0
  0
@@ -1935,11 +1839,15 @@
  0
  0
  0
+ 0.0161528
  0
  0
  0
  0
+ 0.26745
  0
+-0.07983609999999999
+ 0.894411
  0
  0
  0
@@ -1947,72 +1855,56 @@
  0
  0
  0
+-0.77813
  0
  0
  0
  0
  0
  0
+ 0.588775
  0
  0
  0
  0
  0
+-0.38006
  0
  0
  0
  0
+-0.9967240000000001
  0
  0
  0
+ 0.300447
  0
+-0.32854
  0
+-0.271298
  0
  0
- 29.14838965517242
- 4.680886965517241
- 13.50934275862069
--23.79191862068966
--15.22718551724138
- 0.4433124827586207
--0.3410086
- 0.03337290827586207
--0.07430629655172415
- 0.08176048275862069
--0.1601356068965517
- 0.0001809489034482759
- 0.0009656779310344828
- 0.0008490512413793103
--7.387113103448276e-05
- 0.001171721172413793
- 0.000706394275862069
- 3.772604413793103e-05
- 0.0003570491172413793
- 0.0001914632344827586
- 7.19205724137931e-05
- 0.0002159170344827586
- 0.0001653752
- 2.684367931034483e-05
- 6.496945517241379e-05
- 4.364480344827586e-05
- 1.725162965517241e-05
- 2.721767793103448e-05
- 2.411594068965517e-05
  0
+-0.0891188
  0
  0
  0
  0
+ 0.0918489
  0
  0
  0
  0
  0
+-0.679741
  0
  0
+-0.966585
  0
  0
  0
+ 0.0990072
+ 0.217815
  0
  0
  0
@@ -2023,7 +1915,14 @@
  0
  0
  0
+-0.0444764
+ 0.313269
+-0.943724
+-0.483973
+ 1
  0
+ 0.645783
+-0.803159
  0
  0
  0
@@ -2032,49 +1931,32 @@
  0
  0
  0
+-0.97432
  0
+-0.819059
  0
  0
  0
+ 0.900874
  0
  0
+-0.9302240000000001
+-0.386968
+-0.498171
  0
+ 0.657479
  0
  0
+-0.311184
  0
+ 0.0330274
  0
  0
  0
  0
+-0.697004
+ 0.785085
  0
- 50.58806
- 9.825377241379311
--15.78971931034483
--12.93243517241379
--0.08230637931034482
- 0.05650040827586207
- 0.03659139379310344
- 0.1615299793103448
--0.1777294137931034
- 0.3481057172413793
- 8.133559999999999e-05
- 0.0008490512413793103
--0.000541549351724138
--3.677954689655172e-05
- 0.000706394275862069
- 0.000104503475862069
- 1.70590648275862e-05
- 0.0001914632344827586
- 2.022333103448276e-05
- 4.475804965517241e-05
- 0.0001653752
- 5.309790068965517e-05
- 1.480232620689655e-05
- 4.364480344827586e-05
- 9.88770551724138e-06
- 1.196528827586207e-05
- 2.411594068965517e-05
- 9.173352413793103e-06
  0
  0
  0
@@ -2085,72 +1967,67 @@
  0
  0
  0
+ 0.194681
  0
  0
+ 0.863267
  0
+-0.592132
  0
  0
  0
  0
  0
  0
+ 0.9262049999999999
  0
  0
  0
+-0.974857
  0
  0
  0
+ 0.08011310000000001
  0
  0
+ 0.961995
  0
  0
  0
  0
+ 0.69303
+-0.435409
  0
+-0.235911
+ 0.50975
  0
  0
  0
  0
  0
  0
+-0.825164
  0
  0
  0
+ 0.711273
+ 0.996322
+ 0.0373364
  0
  0
  0
  0
+ 0.307555
+-0.48443
  0
+ 1
+ 0.0760803
  0
  0
  0
- 27.99440620689655
- 6.141295034482758
--7.052193103448276
--17.42896275862069
--20.07475655172414
--2.72642151724138
- 0.3300255172413794
- 0.4689026275862069
--0.3185072137931034
- 0.07104062068965518
--0.08248120689655172
- 0.158435924137931
--0.0009192224137931034
- 0.0001491706206896551
- 6.659398896551724e-05
--0.0001718162068965517
- 0.0003320503655172414
- 0.0001337059586206897
+ 0.929615
  0
- 0.0003385045034482758
- 0.0002045532482758621
- 1.037986206896552e-05
- 9.246960689655173e-05
- 4.853807103448276e-05
- 1.500210551724138e-05
- 4.542782965517241e-05
- 3.137863379310345e-05
+ 0.134809
  0
  0
  0
@@ -2160,9 +2037,11 @@
  0
  0
  0
+ 0.380073
  0
  0
  0
+-0.0615359
  0
  0
  0
@@ -2174,6 +2053,7 @@
  0
  0
  0
+-0.466221
  0
  0
  0
@@ -2182,61 +2062,49 @@
  0
  0
  0
+ 0.208577
  0
  0
  0
  0
  0
  0
+ 0.514005
  0
  0
+ 0.393893
  0
  0
  0
+ 0.535456
  0
+ 0.15096
  0
  0
  0
+ 0.918121
+ 0.433264
  0
+ 0.35857
  0
  0
- 51.26644965517242
- 19.80913862068966
--20.15692965517242
--27.34780689655172
--4.329715034482758
--0.1478756
--0.3744408275862069
- 0.346112151724138
--0.07816238620689656
- 0.09075457931034484
--0.1743255517241379
- 0.0001491706206896551
- 0.0007678878620689656
- 0.000759063448275862
- 0.0003320503655172414
- 0.001215018068965517
- 0.0005888076965517241
- 0.0003385045034482758
- 0.0005864146275862069
- 0.0004490578896551724
- 9.246960689655173e-05
- 0.0001486279172413793
- 9.688464827586206e-05
- 4.542782965517241e-05
- 4.971801310344827e-05
- 4.558090896551724e-05
+ 0.334598
  0
  0
  0
+-0.560098
  0
  0
  0
+-0.267865
+-0.534967
+-0.864568
  0
  0
  0
  0
  0
+-0.55958
  0
  0
  0
@@ -2245,19 +2113,25 @@
  0
  0
  0
+-0.283961
  0
  0
+ 0.367275
  0
  0
  0
  0
+ 1
  0
  0
  0
  0
+ 0.388918
+ 0.925996
  0
  0
  0
+ 0.31666
  0
  0
  0
@@ -2268,47 +2142,30 @@
  0
  0
  0
+ 0.820912
+ 0.0569179
  0
+ 0.609494
  0
+ 0.246628
  0
+-0.466481
  0
  0
  0
  0
- 15.00886689655172
--2.558856137931035
--4.304900275862069
--1.68203124137931
--0.6411026068965517
- 0.1216404068965517
--0.6591376965517242
- 0.1530909310344828
--0.1777519862068966
- 0.3414386551724138
- 6.659398896551724e-05
- 0.000759063448275862
--0.0005934290620689654
- 0.0001337059586206897
- 0.0005888076965517241
--1.012690620689655e-05
- 0.0002045532482758621
- 0.0004490578896551724
- 0.0001146514413793103
- 4.853807103448276e-05
- 9.688464827586206e-05
- 1.490917034482759e-05
- 3.137863379310345e-05
- 4.558090896551724e-05
- 1.521340413793103e-05
  0
+-0.6463989999999999
  0
  0
  0
  0
  0
  0
+ 0.96239
  0
  0
+-0.43408
  0
  0
  0
@@ -2321,19 +2178,31 @@
  0
  0
  0
+ 0.943537
+ 0.703053
  0
  0
  0
+-0.326245
+-0.224257
  0
  0
  0
  0
+ 0.179575
+-0.196652
+ 0.530014
+ 0.472888
  0
  0
  0
+-0.0528497
  0
+ 0.372112
+-0.565865
  0
  0
+-0.695865
  0
  0
  0
@@ -2349,42 +2218,19 @@
  0
  0
  0
- 27.42867517241379
- 6.824471103448276
--7.748226896551724
--10.67058068965517
- 12.94137517241379
- 11.21792413793103
- 0.3471452137931034
--0.1415545448275862
--0.6336685586206897
- 0.05694206758620689
--0.07166853103448276
- 0.1569107862068966
--0.0009082393793103449
- 0.0001632459310344828
- 7.624994482758621e-05
--0.0003991188482758621
- 0.0008013735172413793
- 0.0005871277448275862
--9.112732413793104e-06
- 0.0003249579724137931
- 0.0001970324965517241
- 3.969214275862069e-05
- 0.0001481983448275862
- 0.0001203855862068965
- 1.439358137931034e-05
- 4.868494689655172e-05
- 3.170703448275862e-05
  0
  0
+-0.528593
  0
+ 1
  0
  0
+-0.752672
  0
  0
  0
  0
+ 0.985171
  0
  0
  0
@@ -2400,6 +2246,7 @@
  0
  0
  0
+ 0.568941
  0
  0
  0
@@ -2408,63 +2255,51 @@
  0
  0
  0
+-0.931409
+ 0.00905803
  0
  0
  0
  0
+ 0.0712531
  0
  0
  0
+-0.5998559999999999
  0
+ 0.025817
  0
  0
  0
  0
  0
  0
- 49.47399655172413
- 20.68361655172414
- 12.94960620689655
--21.41140413793103
--16.67916827586207
- 0.4608146137931034
--0.3662505586206897
- 0.1190646620689655
--0.06611073655172414
- 0.08321379310344827
--0.1821868
- 0.0001632459310344828
- 0.000958783172413793
- 0.0008787396551724138
- 0.0008013735172413793
- 0.001480246344827586
- 0.001593285724137931
- 0.0003249579724137931
- 0.0005786194137931033
- 0.0004438317103448276
- 0.0001481983448275862
- 0.0001603169931034483
- 0.0001789662
- 4.868494689655172e-05
- 5.703008137931035e-05
- 4.849746551724138e-05
  0
+ 0.682302
+-0.165467
  0
+ 0.598773
  0
  0
  0
+ 0.442903
  0
  0
+-0.786183
  0
  0
  0
  0
+ 0.0281454
  0
  0
+ 0.755406
  0
  0
+-0.024673
  0
  0
+ 0.0667312
  0
  0
  0
@@ -2480,51 +2315,33 @@
  0
  0
  0
+-0.864811
  0
  0
  0
  0
  0
  0
+-0.947692
+ 0.770323
  0
  0
  0
  0
+ 0.421239
  0
+-0.9019
  0
  0
  0
- 16.64129862068966
- 11.21379793103448
--16.67795034482759
--14.11864551724138
--0.2949161172413793
- 0.3409930620689655
--0.6500570689655173
- 0.1269904137931034
--0.1598408068965517
- 0.3499573241379311
- 7.624994482758621e-05
- 0.0008787396551724138
--0.0005119362965517242
- 0.0005871277448275862
- 0.001593285724137931
- 0.0004729021448275862
- 0.0001970324965517241
- 0.0004438317103448276
- 0.000115737675862069
- 0.0001203855862068965
- 0.0001789662
- 8.538359310344828e-05
- 3.170703448275862e-05
- 4.849746551724138e-05
- 1.414919448275862e-05
  0
  0
  0
+-0.788506
  0
  0
  0
+-0.73806
  0
  0
  0
@@ -2538,8 +2355,11 @@
  0
  0
  0
+ 0.39598
+ 0.714214
  0
  0
+-0.680734
  0
  0
  0
@@ -2547,16 +2367,23 @@
  0
  0
  0
+ 0.234227
+-0.9890409999999999
  0
  0
  0
+ 0.662945
+ 0.923506
  0
+ 0.454497
+-0.574734
  0
  0
  0
  0
  0
  0
+ 0.310312
  0
  0
  0
@@ -2565,34 +2392,8 @@
  0
  0
  0
- 27.22900344827586
- 6.557267448275862
--7.12523448275862
--17.06198275862069
--19.96249655172414
--3.043849448275862
- 0.3662494620689655
- 0.4647026689655173
--0.3127578896551724
- 0.04409045655172414
--0.06542255724137931
- 0.1356757448275862
--0.008463653793103447
--0.01230105586206896
--0.008697089655172413
- 0.0003488920137931034
- 0.001096986413793103
- 0.0006112221172413793
- 0.0002515918068965518
- 0.0003869946206896552
- 0.0003147866482758621
- 6.702879517241379e-05
- 0.000113354875862069
- 7.041461379310344e-05
- 2.99074324137931e-05
- 3.766646482758621e-05
- 2.986818896551724e-05
  0
+ 0.520643
  0
  0
  0
@@ -2603,12 +2404,19 @@
  0
  0
  0
+ 0.981698
+ 0.692644
+-0.828054
  0
  0
+-0.159717
  0
+-0.198578
  0
  0
  0
+ 0.856366
+ 0.960822
  0
  0
  0
@@ -2616,69 +2424,63 @@
  0
  0
  0
+ 1
  0
+ 0.454944
  0
  0
  0
+ 0.562352
  0
+ 0.331572
  0
  0
  0
  0
+ 0.858436
  0
+ 0.624214
  0
  0
+-0.257371
  0
+-0.202928
+ 0.1742
+-0.226991
  0
+ 0.146579
  0
  0
  0
  0
+-0.886428
  0
- 49.63927655172414
- 21.34938689655172
--19.95168551724138
--27.45460689655172
--5.093884275862069
--0.168285724137931
--0.3665937241379311
- 0.3623362689655172
--0.05549097310344828
- 0.08234735172413793
--0.1707736137931034
--0.01230105586206896
--0.01698540689655172
--0.01221655517241379
- 0.001096986413793103
- 0.00112049524137931
- 0.0008628804137931035
- 0.0003869946206896552
- 0.0002486358896551724
- 0.0003135849310344827
- 0.000113354875862069
- 9.037946206896551e-05
- 7.803943448275861e-05
- 3.766646482758621e-05
- 2.144046827586207e-05
- 2.669914068965517e-05
  0
  0
  0
  0
  0
  0
+ 0.325708
  0
+-0.339013
  0
  0
  0
+-0.215203
  0
+ 0.771929
  0
  0
+-0.277257
  0
  0
  0
+-0.286208
  0
+ 0.147887
  0
+ 0.952027
  0
  0
  0
@@ -2688,47 +2490,29 @@
  0
  0
  0
+ 0.6573870000000001
  0
+ 0.09863619999999999
  0
  0
  0
  0
  0
+-0.314632
  0
  0
  0
  0
  0
+-0.566458
+-0.584975
  0
+-0.8089229999999999
  0
+-0.103068
  0
  0
  0
- 16.99647103448276
--3.079335310344828
--5.068160137931034
--1.74207075862069
--0.5665745172413793
- 0.1052798344827586
--0.6932204137931034
- 0.1214911862068965
--0.1802892068965517
- 0.3738951310344827
--0.008697089655172413
--0.01221655517241379
--0.008343778620689656
- 0.0006112221172413793
- 0.0008628804137931035
- 5.262615586206897e-05
- 0.0003147866482758621
- 0.0003135849310344827
- 0.0001181922551724138
- 7.041461379310344e-05
- 7.803943448275861e-05
- 1.322706e-05
- 2.986818896551724e-05
- 2.669914068965517e-05
- 8.941974482758621e-06
  0
  0
  0
@@ -2738,8 +2522,10 @@
  0
  0
  0
+-0.283349
  0
  0
+ 1
  0
  0
  0
@@ -2747,15 +2533,20 @@
  0
  0
  0
+ 0.772267
  0
  0
  0
+ 0.531232
  0
  0
  0
+ 0.0602593
  0
+-0.818275
  0
  0
+ 0.640503
  0
  0
  0
@@ -2770,67 +2561,53 @@
  0
  0
  0
+-0.452958
  0
  0
- 28.51891724137931
- 5.362052344827586
--7.082658620689655
--12.08045586206896
- 14.42987310344828
- 10.71194344827586
- 0.3591143931034483
--0.2183090344827586
--0.2353391172413793
- 0.0742888
--0.03219096551724138
--0.1039314965517241
--0.008253270344827587
--0.01235218896551724
--0.008644932413793103
- 0.0006632136
- 0.001168275310344828
- 0.001245567517241379
- 0.0002696775448275862
- 0.0004502065448275862
- 0.0003344944
- 0.0001045902206896552
- 0.0001209796482758621
- 0.0001164946137931034
- 3.476353517241379e-05
- 4.548105448275862e-05
- 3.265688620689655e-05
  0
+ 0.227352
  0
  0
  0
  0
  0
  0
+-0.263366
  0
  0
  0
  0
  0
  0
+ 0.130876
  0
  0
  0
+-0.282325
  0
  0
  0
+-0.397617
  0
+ 0.340223
  0
+-0.723695
+-0.989039
  0
  0
  0
  0
  0
+ 0.618592
+ 0.186969
  0
  0
  0
+ 0.864703
  0
  0
  0
+-0.987061
  0
  0
  0
@@ -2839,40 +2616,18 @@
  0
  0
  0
- 50.77792275862069
- 20.26526206896552
- 14.39168068965517
--22.70572827586207
--15.30736965517241
- 0.5154355517241379
--0.3199984344827586
--0.3093172551724138
--0.1102442413793104
- 0.04777490551724138
- 0.1542258551724138
--0.01235218896551724
--0.01749564551724138
--0.01246429586206897
- 0.001168275310344828
- 0.0004245849448275862
- 0.001124829931034483
- 0.0004502065448275862
- 0.0003198911172413794
- 0.0003536663724137931
- 0.0001209796482758621
- 5.55952248275862e-05
- 9.526921379310345e-05
- 4.548105448275862e-05
- 2.793027517241379e-05
- 3.029557310344827e-05
  0
  0
  0
  0
  0
+-0.451151
+-0.0770106
  0
  0
  0
+ 0.142685
+ 0.406207
  0
  0
  0
@@ -2880,8 +2635,10 @@
  0
  0
  0
+-0.530766
  0
  0
+-0.00318492
  0
  0
  0
@@ -2889,54 +2646,51 @@
  0
  0
  0
+-0.248891
  0
+-0.975061
+ 0.421405
  0
  0
+ 0.180812
  0
+-0.952772
  0
  0
  0
  0
+ 0.0837316
  0
+ 0.742048
+-0.223487
+-0.10158
  0
+-0.419303
  0
  0
+-0.306625
+ 0.779993
  0
  0
  0
  0
- 14.02249655172414
- 10.83402275862069
--15.36702620689655
--11.64634689655172
--0.392058048275862
- 0.2478738137931034
- 0.2162618137931034
- 0.2286299034482759
--0.09908226896551724
--0.3198399172413793
--0.008644932413793103
--0.01246429586206897
--0.008409587586206897
- 0.001245567517241379
- 0.001124829931034483
- 0.000568803724137931
- 0.0003344944
- 0.0003536663724137931
- 0.0001066476620689655
- 0.0001164946137931034
- 9.526921379310345e-05
- 4.839572689655173e-05
- 3.265688620689655e-05
- 3.029557310344827e-05
- 7.491126206896552e-06
  0
  0
+-0.119679
+ 0.700197
+-0.112062
  0
+ 0.296891
  0
+-0.775325
+-0.702605
  0
+-0.486123
  0
+-0.176686
  0
+-0.686523
+-0.367065
  0
  0
  0
@@ -2947,8 +2701,10 @@
  0
  0
  0
+-0.555269
  0
  0
+ 0.548756
  0
  0
  0
@@ -2960,61 +2716,50 @@
  0
  0
  0
+-0.485037
  0
  0
  0
+-0.559457
  0
  0
  0
  0
+ 0.113706
  0
+ 1
  0
  0
- 28.75004275862069
- 5.261753241379311
--5.826562000000001
--16.62624068965517
--18.99846827586207
--4.512158551724138
--0.344132475862069
--0.3504368275862069
- 0.2809605862068966
- 0.1571879034482759
--0.07046330344827587
--0.2208336068965517
- 0.0008877453793103449
- 0.0008368364827586207
- 0.001068074206896552
- 0.0008354328965517241
- 0.0007347057931034483
- 0.0005565399862068966
- 0.0002394993862068966
- 0.0001618842344827586
- 0.0001703296206896552
- 7.553464137931035e-05
- 6.388358827586206e-05
- 4.614627034482758e-05
+ 0.100412
  0
  0
  0
  0
+ 0.636466
  0
  0
+ 0.07075720000000001
+ 0.376973
  0
  0
+-0.211204
  0
+ 0.09748080000000001
  0
  0
  0
  0
+-0.110624
  0
  0
  0
  0
+ 0.0647851
  0
  0
  0
  0
+ 0.446437
  0
  0
  0
@@ -3027,46 +2772,39 @@
  0
  0
  0
+-0.176045
  0
  0
+-0.987692
+ 0.922387
  0
+ 0.0398782
+-0.73672
  0
  0
+-0.471647
  0
  0
- 50.08102896551724
- 21.05045793103448
--20.35861448275862
--27.41952965517241
--5.600416413793104
- 0.3417453655172414
- 0.3442165724137931
--0.2805089172413793
--0.06812185310344827
- 0.0305424475862069
- 0.09570018620689656
- 0.0008368364827586207
--0.0005001923034482758
- 0.0005018145931034482
- 0.0007347057931034483
- 0.0002291414482758621
- 0.0003724166482758621
- 0.0001618842344827586
- 2.417760689655172e-05
- 9.127979999999999e-05
- 6.388358827586206e-05
- 2.22185475862069e-05
- 3.07463e-05
  0
+-0.207676
  0
+-0.303541
+-0.776406
+ 0.692349
+-0.752883
  0
  0
  0
  0
+ 0.6495610000000001
  0
+ 0.306261
  0
  0
+-0.745659
  0
+-0.425891
+-0.489333
  0
  0
  0
@@ -3078,12 +2816,19 @@
  0
  0
  0
+-0.438058
+-0.249839
+-0.600756
+-0.0477695
  0
  0
  0
  0
  0
+-0.875088
+ 0.9595
  0
+ 1
  0
  0
  0
@@ -3093,48 +2838,36 @@
  0
  0
  0
+-0.332186
  0
  0
  0
+-0.458934
+-0.953472
  0
- 15.29933517241379
--4.054567448275862
--5.848794827586207
--3.518657379310345
--0.3392276896551724
--0.3293673379310345
- 0.2833854827586207
--0.2198943379310345
- 0.09856970344827586
- 0.3089375931034483
- 0.001068074206896552
- 0.0005018145931034482
--0.0002528827103448276
- 0.0005565399862068966
- 0.0003724166482758621
- 1.960679172413793e-05
- 0.0001703296206896552
- 9.127979999999999e-05
- 3.346594137931035e-05
- 4.614627034482758e-05
- 3.07463e-05
  0
  0
  0
  0
+-0.5413
  0
  0
  0
  0
  0
+ 0.0190589
  0
  0
  0
+ 0.32949
  0
  0
+-0.364322
  0
+-0.890822
  0
  0
+ 0.514602
  0
  0
  0
@@ -3143,71 +2876,65 @@
  0
  0
  0
+ 0.298092
  0
  0
  0
  0
+ 0.510348
  0
+-0.622988
  0
  0
  0
  0
  0
  0
+-0.181659
  0
+ 0.24513
  0
  0
  0
+-0.498219
  0
- 41.26645793103449
- 11.71601793103448
- 23.96258482758621
--24.69156620689655
- 7.888358620689655
--19.71257448275862
--0.5042100620689655
- 0.305772
- 0.2620930344827586
- 0.1706331379310345
--0.08232462758620689
--0.2070680689655172
--0.0007370531034482759
- 0.000115941724137931
- 0.0001232883655172414
- 0.0009418784827586207
- 0.0004766905655172414
- 0.0007300750344827586
- 0.0003046984827586207
- 0.0002172056068965517
- 0.0001947314965517241
- 8.746699310344828e-05
- 5.088073793103448e-05
- 5.98050875862069e-05
- 3.196303448275862e-05
- 2.270557310344827e-05
- 1.874114689655172e-05
+ 0.492542
  0
  0
+ 0.0283554
  0
+ 0.329472
+-0.172077
  0
  0
+ 0.559774
  0
  0
  0
+-0.40646
  0
  0
+-0.0810903
  0
  0
  0
  0
+-0.027825
+ 0.8294899999999999
+ 0.32648
  0
  0
  0
  0
+-0.794948
  0
  0
+ 1
  0
  0
+-0.607853
+ 0.307702
+ 0.740138
  0
  0
  0
@@ -3216,36 +2943,16 @@
  0
  0
  0
+-0.626233
+-0.128633
  0
+ 0.52036
+ 0.87114
  0
  0
  0
- 32.21862206896552
--0.6992486896551724
- 7.83942
--4.731418896551724
- 7.022748965517241
--0.2675940827586207
- 0.3040735862068966
--0.654512124137931
--0.07649017241379309
- 0.03690955310344828
- 0.09281838620689654
- 0.000115941724137931
--0.0008997435862068965
- 6.410590068965518e-05
- 0.0004766905655172414
--6.668131724137931e-05
- 0.0002904492965517241
- 0.0002172056068965517
- 4.442569724137931e-05
- 0.0001103360965517241
- 5.088073793103448e-05
  0
- 2.806993724137931e-05
- 2.270557310344827e-05
  0
- 1.071784551724138e-05
  0
  0
  0
@@ -3254,11 +2961,13 @@
  0
  0
  0
+-0.649494
  0
  0
  0
  0
  0
+-0.765899
  0
  0
  0
@@ -3266,13 +2975,21 @@
  0
  0
  0
+-6.29984e-05
+-0.813014
  0
+-0.277215
+ 0.769831
  0
+-0.983227
  0
+ 0.389997
  0
  0
  0
+-0.6775679999999999
  0
+-0.45033
  0
  0
  0
@@ -3280,55 +2997,52 @@
  0
  0
  0
- 20.46245793103449
--19.64142689655172
- 6.897234482758621
--16.96584482758621
- 0.5102879517241379
--0.2525369034482758
--0.5837368551724138
--0.2397231034482759
- 0.1156549034482759
- 0.2909176551724138
- 0.0001232883655172414
- 6.410590068965518e-05
--0.0008918611034482759
- 0.0007300750344827586
- 0.0002904492965517241
- 0.0001885161103448276
- 0.0001947314965517241
- 0.0001103360965517241
- 2.027603517241379e-05
- 5.98050875862069e-05
- 2.806993724137931e-05
- 1.205575034482759e-05
- 1.874114689655172e-05
- 1.071784551724138e-05
  0
+ 0.258725
+ 0.429784
  0
+ 0.0654044
  0
+-0.601945
+-0.0694507
  0
  0
+ 0.528713
  0
  0
  0
+ 0.706446
+-0.145461
  0
  0
+ 0.556621
  0
+ 0.537152
+-0.931454
+-0.857105
  0
+ 0.41443
  0
  0
+ 0.690913
+-0.898316
+ 0.444102
  0
  0
  0
+ 1
  0
  0
  0
  0
+ 0.563904
  0
  0
  0
  0
+-0.439682
+-0.861575
+ 0.265489
  0
  0
  0
@@ -3338,37 +3052,15 @@
  0
  0
  0
+-0.9785740000000001
  0
- 39.70973724137932
- 11.64188137931034
- 23.20121034482759
--14.44379379310345
--19.3131951724138
--3.770309034482759
--0.4267884137931034
--0.2623940482758621
- 0.4500655931034483
- 0.1833113241379311
--0.1042441379310345
--0.2247420413793103
- 0.0008846102758620689
- 0.001073520827586207
- 0.0009708670344827586
- 0.0007283088275862069
- 0.0007603287586206896
- 0.0004211680068965517
- 0.0002336683862068965
- 0.0001845348551724138
- 0.0001483925310344828
- 6.814547517241379e-05
- 6.26750675862069e-05
- 3.593858689655172e-05
  0
  0
  0
  0
  0
  0
+ 0.567659
  0
  0
  0
@@ -3379,117 +3071,101 @@
  0
  0
  0
+-0.597462
  0
+-0.96797
  0
  0
  0
  0
  0
+-0.99492
  0
  0
  0
  0
+-0.750054
+ 0.504466
  0
  0
  0
+-0.378227
+ 0.870286
  0
  0
+-0.877214
  0
  0
  0
- 34.08062482758621
--2.185023517241379
--19.27420275862069
--29.88585103448276
--4.253730344827586
- 0.2480019448275862
- 0.3068736275862069
--0.2391401724137931
--0.08844256551724138
- 0.05030061862068966
- 0.1084266344827586
- 0.001073520827586207
--0.000230091124137931
- 0.0005898598827586206
- 0.0007603287586206896
- 0.0003517980620689655
- 0.0003296053379310345
- 0.0001845348551724138
- 5.224562206896552e-05
- 9.2407e-05
- 6.26750675862069e-05
- 2.669968137931035e-05
- 2.596936620689655e-05
  0
  0
  0
  0
+-0.847492
  0
  0
  0
  0
  0
+ 0.333287
  0
  0
  0
+ 0.968925
  0
  0
  0
+ 0.763413
  0
+ 0.494125
  0
  0
+ 0.790668
+-0.502057
  0
  0
  0
+-0.00518237
  0
+ 0.973527
  0
  0
  0
+ 1
  0
+ 0.561007
  0
  0
  0
+-0.072712
  0
  0
  0
  0
  0
- 19.18058413793104
--3.821402620689655
--4.248102206896552
--1.879992482758621
- 0.2823904620689655
--0.6905208965517242
--0.4230164965517241
--0.2224524827586207
- 0.1264979655172414
- 0.2727380689655173
- 0.0009708670344827586
- 0.0005898598827586206
--0.0003488625310344827
- 0.0004211680068965517
- 0.0003296053379310345
--6.065434068965518e-05
- 0.0001483925310344828
- 9.2407e-05
- 1.164110551724138e-05
- 3.593858689655172e-05
- 2.596936620689655e-05
  0
  0
+ 0.73028
+-0.839825
  0
  0
+-0.256166
  0
  0
+-0.0106132
+ 0.82038
  0
  0
  0
  0
  0
+-0.996431
  0
+ 0.281412
  0
  0
  0
+ 0.351482
  0
  0
  0
@@ -3497,44 +3173,30 @@
  0
  0
  0
+-0.110093
  0
  0
+-0.501891
  0
+-0.655143
+ 0.560348
  0
  0
+ 0.700191
  0
+ 0.612231
+-0.187004
  0
+-0.807525
  0
  0
  0
  0
  0
+-0.918936
  0
- 39.97770620689655
- 16.00375448275862
- 24.41433862068966
--24.61211379310345
- 3.157319862068965
--21.11712620689655
--0.5135459103448275
- 0.2944088827586207
- 0.3293224965517241
- 0.1740081379310345
--0.09890520689655172
--0.2194575517241379
--0.0002878525103448276
- 0.0004253782068965517
- 0.0003152464275862069
- 0.000961910551724138
- 0.0005948667103448275
- 0.0006643046413793103
- 0.000276731551724138
- 0.0002156788620689655
- 0.0001520564620689655
- 9.504140689655173e-05
- 5.637414620689656e-05
- 5.644005793103448e-05
  0
+ 0.197198
  0
  0
  0
@@ -3546,54 +3208,42 @@
  0
  0
  0
+ 0.76811
  0
+-0.464659
  0
  0
+ 0.92374
  0
  0
  0
  0
  0
  0
+-0.729592
+ 0.936947
  0
  0
  0
+ 0.822943
  0
  0
  0
  0
  0
  0
+ 1
  0
- 32.26302344827587
- 1.598364068965517
- 3.200061724137931
--2.387942137931034
- 3.377357655172414
--0.2803164965517241
- 0.1679142206896552
--0.7365567586206897
--0.09895373103448275
- 0.05625045793103448
- 0.1247944965517241
- 0.0004253782068965517
--0.0006825552137931035
- 0.0002012651517241379
- 0.0005948667103448275
- 2.604740689655172e-05
- 0.0003226469034482758
- 0.0002156788620689655
- 6.281593310344827e-05
- 9.432062068965518e-05
- 5.637414620689656e-05
  0
- 2.725414482758621e-05
+-0.151991
  0
  0
  0
  0
+-0.731541
  0
  0
+-0.566369
  0
  0
  0
@@ -3609,8 +3259,11 @@
  0
  0
  0
+ 0.538155
  0
+-0.437678
  0
+ 0.587591
  0
  0
  0
@@ -3619,46 +3272,41 @@
  0
  0
  0
- 21.63931793103448
--21.14256482758621
- 3.367158689655173
--19.16534206896552
- 0.5466808482758621
--0.3123450827586207
--0.4833615793103448
--0.2133366482758621
- 0.121254675862069
- 0.2690665172413793
- 0.0003152464275862069
- 0.0002012651517241379
--0.0008049746896551723
- 0.0006643046413793103
- 0.0003226469034482758
- 9.743713793103448e-05
- 0.0001520564620689655
- 9.432062068965518e-05
+ 0.265715
  0
- 5.644005793103448e-05
- 2.725414482758621e-05
  0
  0
+-0.226608
  0
+-0.577827
  0
+-0.420319
  0
+ 0.277472
+ 0.9434090000000001
+-0.688326
  0
  0
  0
  0
+-0.574515
  0
  0
+ 0.916211
  0
  0
  0
+-0.180097
  0
  0
  0
  0
+-0.220395
  0
+ 0.402006
+ 0.368038
+-0.454009
+ 0.798282
  0
  0
  0
@@ -3666,39 +3314,15 @@
  0
  0
  0
+ 0.200078
+-0.279707
  0
  0
+-0.793923
  0
  0
  0
  0
- 36.71210413793104
- 15.00731379310345
- 21.37378068965517
--11.48783448275862
--18.18312689655173
--0.6187589310344828
--0.4275524413793104
--0.2375524965517241
- 0.5376466620689655
- 0.1645674551724138
--0.1177548896551724
--0.207630475862069
- 0.000534795248275862
- 0.0009635873103448277
- 0.0006488769655172413
- 0.0007428291724137931
- 0.0007811626206896552
- 0.0003274485103448276
- 0.0002916772206896552
- 0.0002172657793103448
- 0.0001489709034482759
- 7.902735862068965e-05
- 6.46544924137931e-05
- 3.248020482758621e-05
- 7.279691034482758e-05
- 5.773974275862069e-05
- 1.068440137931035e-05
  0
  0
  0
@@ -3709,98 +3333,64 @@
  0
  0
  0
+ 1
  0
  0
  0
  0
  0
+ 0.6072650000000001
  0
  0
+ 0.644375
  0
  0
  0
  0
+ 0.487367
  0
+ 0.65925
  0
+ 0.781865
+-0.528809
  0
  0
- 35.55195931034483
--3.208719448275862
--18.13119655172414
--33.47425586206896
- 0.5467823517241379
- 0.2452738551724138
- 0.1433811586206896
--0.3083971379310345
--0.09353907586206897
- 0.0669365724137931
- 0.1180107793103448
- 0.0009635873103448277
--0.0002802572068965517
- 0.0004300957655172414
- 0.0007811626206896552
- 0.0003849674275862069
- 0.0002609254137931034
- 0.0002172657793103448
- 5.803105241379311e-05
- 8.903956551724138e-05
- 6.46544924137931e-05
- 2.338928551724138e-05
- 2.138392137931034e-05
- 5.773974275862069e-05
- 2.134804551724138e-05
- 6.93670275862069e-06
  0
  0
+ 0.0355046
  0
+-0.802964
  0
  0
+ 0.211056
+-0.835107
  0
  0
  0
+-0.359129
  0
  0
  0
  0
+-0.957183
  0
  0
  0
+-0.674527
  0
  0
  0
  0
  0
+ 0.520807
  0
+ 0.597821
  0
  0
  0
  0
- 20.7228275862069
--0.5602410206896551
- 0.5306131586206897
--1.180323310344828
- 0.2532805655172414
--0.7614670344827585
--0.3214810206896552
--0.2075511034482759
- 0.1485066482758621
- 0.2618704965517241
- 0.0006488769655172413
- 0.0004300957655172414
--0.0006293239793103448
- 0.0003274485103448276
- 0.0002609254137931034
--0.0001281179931034483
- 0.0001489709034482759
- 8.903956551724138e-05
--1.077624620689655e-05
- 3.248020482758621e-05
- 2.138392137931034e-05
--8.434317931034483e-06
- 1.068440137931035e-05
- 6.93670275862069e-06
--1.485471379310345e-05
  0
+ 0.986897
  0
  0
  0
@@ -3811,49 +3401,26 @@
  0
  0
  0
+-0.268008
+-0.0367526
+ 0.581768
+ 0.526045
  0
  0
  0
  0
  0
  0
+-0.230316
  0
  0
  0
  0
  0
+-0.655722
  0
  0
  0
- 39.09078689655173
- 18.0134475862069
- 21.00863586206897
--26.82440482758621
- 0.2096531517241379
--21.00078689655172
--0.436236524137931
- 0.2887346827586207
- 0.2985287862068965
- 0.1577044137931034
--0.1135808827586207
--0.2068983862068965
- 0.0006173338344827586
- 0.0009557809655172414
- 0.0006057397724137932
- 0.001079071586206897
- 0.0005846182344827586
- 0.0006278839793103448
- 0.0002897988965517242
- 0.0001880457517241379
- 0.0001307994965517241
- 0.0002780536482758621
- 0.0001761840413793103
- 4.408282482758621e-05
- 6.877851517241379e-05
- 4.71944475862069e-05
- 9.000057931034483e-06
- 4.093239724137931e-05
- 2.941326965517241e-05
  0
  0
  0
@@ -3863,8 +3430,11 @@
  0
  0
  0
+-0.687659
  0
+ 0.0559371
  0
+ 1
  0
  0
  0
@@ -3874,129 +3444,71 @@
  0
  0
  0
- 35.01032344827586
--0.5720494482758621
- 0.277766075862069
--1.821079310344828
- 0.8460544827586207
--0.2871088413793104
- 0.1024304551724138
--0.7457581379310344
--0.1128439172413793
- 0.08127762068965516
- 0.1480395517241379
- 0.0009557809655172414
--0.0003251076
- 0.0003767155034482759
- 0.0005846182344827586
--1.161522206896552e-05
- 0.0002729514896551724
- 0.0001880457517241379
- 3.474878551724138e-05
- 6.93374e-05
- 0.0001761840413793103
- 3.735283517241379e-05
- 2.327339724137931e-05
- 4.71944475862069e-05
- 1.295211931034483e-05
  0
- 2.941326965517241e-05
- 9.082691034482758e-06
  0
+ 0.501389
+ 0.859791
  0
+ 0.383719
  0
+ 0.5194800000000001
  0
  0
  0
+ 0.405414
  0
  0
  0
  0
  0
+-0.794355
  0
+-0.378824
  0
  0
  0
+ 0.678763
  0
  0
  0
  0
+ 0.659741
  0
- 19.40809655172414
--20.98043793103448
- 0.8944150344827586
--17.58073862068965
- 0.5484192827586207
--0.3632626896551724
--0.3784126344827586
--0.198971875862069
- 0.1432973793103448
- 0.2610469862068965
- 0.0006057397724137932
- 0.0003767155034482759
--0.0006807646965517241
- 0.0006278839793103448
- 0.0002729514896551724
- 2.739578896551724e-05
- 0.0001307994965517241
- 6.93374e-05
--1.670526e-05
- 4.408282482758621e-05
- 2.327339724137931e-05
--4.983899172413794e-05
- 9.000057931034483e-06
+ 0.276424
  0
--1.298688551724138e-05
  0
  0
--8.26132827586207e-06
+-0.9015840000000001
+ 0.587325
  0
  0
  0
  0
  0
  0
+ 0.687698
  0
  0
+-0.665892
+-0.475804
  0
  0
+ 0.943867
  0
  0
+ 0.6572
  0
+ 0.230882
  0
  0
+ 0.451489
  0
  0
  0
  0
- 39.51894344827586
- 18.10416275862069
- 19.23597310344827
--11.98347655172414
--18.37222758620689
- 1.208118965517241
--0.4390358689655173
--0.1971178137931034
- 0.5734394068965517
- 0.1482538137931035
--0.1296261931034483
--0.1946123655172414
--0.0002534092275862069
- 0.0003941712965517241
- 0.0002608382689655172
- 0.0009432808275862069
- 0.0007786280689655173
- 0.000322861551724138
- 0.0007446821379310345
- 0.0005897355517241379
--8.19135448275862e-06
- 0.0001600283586206897
- 0.0001317981655172414
  0
- 7.549633793103447e-05
- 6.492634620689655e-05
--1.879009103448276e-05
  0
+-0.0718162
  0
  0
  0
@@ -4006,43 +3518,30 @@
  0
  0
  0
+-0.416515
+ 0.881108
  0
  0
  0
  0
+ 0.341995
+ 0.406477
+ 0.562663
  0
  0
  0
+ 0.0987937
+ 0.847928
  0
  0
- 34.39949724137931
--3.782671655172414
--18.32586827586207
--32.62923034482758
- 3.708423103448276
- 0.2845623103448276
- 0.04263611103448276
--0.3720880482758621
--0.1067748482758621
- 0.09336334482758621
- 0.1401585103448276
- 0.0003941712965517241
--0.0007335142068965518
- 0.0001465506896551724
- 0.0007786280689655173
- 0.0002589762206896552
- 0.0002107840275862069
- 0.0005897355517241379
- 0.0002025847103448276
  0
- 0.0001317981655172414
- 5.210622344827586e-05
+ 1
  0
- 6.492634620689655e-05
- 2.707442e-05
--1.304721793103448e-05
+-0.49318
  0
+-0.266143
  0
+-0.000486019
  0
  0
  0
@@ -4060,128 +3559,70 @@
  0
  0
  0
- 19.50487655172414
- 1.28791075862069
- 3.708052689655172
--1.515905862068965
- 0.2356376206896552
--0.8098664137931034
--0.3123390413793103
--0.1944987103448276
- 0.170058075862069
- 0.2553275586206897
- 0.0002608382689655172
- 0.0001465506896551724
--0.0008579970344827587
- 0.000322861551724138
- 0.0002107840275862069
--0.0001619539034482759
--8.19135448275862e-06
  0
--0.0001753333034482758
  0
  0
--3.632199034482759e-05
--1.879009103448276e-05
--1.304721793103448e-05
--1.42321124137931e-05
  0
  0
  0
+-0.216729
  0
  0
+ 0.920589
  0
  0
  0
+ 0.976143
  0
  0
+ 0.171682
  0
  0
  0
+-0.668292
  0
  0
  0
  0
  0
  0
- 40.32677379310344
- 18.00858689655172
- 19.02413793103448
--27.59739172413793
- 0.2294763517241379
--20.8323875862069
--0.3704433931034483
- 0.295502151724138
- 0.2498367448275862
- 0.1192459379310345
--0.1091170137931034
--0.1969563793103448
- 0.00116865475862069
- 0.0009750331034482758
- 0.0007151143448275862
- 0.002222929103448276
- 0.001406143448275862
- 4.38623275862069e-05
- 0.0006952891724137931
- 0.0004547175586206896
  0
- 0.0003038548896551724
- 0.0002129599586206896
--7.628791034482758e-05
- 7.717841379310345e-05
- 5.712122413793104e-05
--1.416778275862069e-05
  0
  0
  0
  0
+ 0.282186
  0
  0
+ 0.8157450000000001
  0
  0
  0
+-0.411406
  0
  0
  0
  0
  0
+ 0.933329
  0
  0
- 34.36774413793103
--3.741732206896552
- 0.3077754206896551
--1.773873931034483
- 0.9010237931034483
--0.2929204689655172
- 0.1203369655172414
--0.7459804137931034
--0.1042619034482759
- 0.09540952413793104
- 0.1722068137931035
- 0.0009750331034482758
--0.0004360101724137931
- 0.0003376502344827586
- 0.001406143448275862
--7.544960689655172e-05
- 2.080023034482759e-05
- 0.0004547175586206896
- 0.0001010907862068966
+-0.0211041
  0
- 0.0002129599586206896
- 5.709424206896552e-05
--4.396853586206897e-05
- 5.712122413793104e-05
- 1.83094475862069e-05
--8.637616551724137e-06
  0
  0
+-0.7948
  0
+ 0.592166
  0
  0
  0
+ 0.77739
  0
+-0.632004
  0
  0
+ 0.567619
  0
  0
  0
@@ -4189,36 +3630,21 @@
  0
  0
  0
- 18.68428
--20.7764275862069
- 0.9826528965517242
--16.80606620689655
- 0.4832175724137931
--0.3860178482758621
--0.3305954965517242
--0.1565350620689655
- 0.1432360827586207
- 0.2585536344827586
- 0.0007151143448275862
- 0.0003376502344827586
--0.0006487399103448276
- 4.38623275862069e-05
- 2.080023034482759e-05
--0.0007416114482758621
+ 0.233368
  0
  0
--0.0001449413517241379
--7.628791034482758e-05
--4.396853586206897e-05
--4.989349379310344e-05
--1.416778275862069e-05
--8.637616551724137e-06
--1.43726924137931e-05
+-0.501713
  0
+-0.697545
+ 1
  0
+ 0.716468
  0
+ 0.30715
  0
+-0.15332
  0
+ 0.449269
  0
  0
  0
@@ -4229,39 +3655,15 @@
  0
  0
  0
+ 0.202787
  0
- 43.45843655172414
- 19.42105655172414
- 18.4400675862069
--15.15111793103448
--19.72130689655172
- 1.902663379310345
--0.4225253586206896
--0.1727341379310345
- 0.5524875655172414
- 0.09701246896551724
--0.1162924206896552
--0.1673862206896552
--0.03036607931034483
--0.01978472620689655
- 0.007370637931034483
- 0.001452079379310345
- 0.001276916275862069
--0.0003467628689655172
- 0.0005001192068965518
- 0.000458532124137931
--0.0002778349931034483
- 0.0001322399034482759
- 0.0001210269379310345
--5.397381379310345e-05
- 4.183055379310345e-05
- 4.256381448275862e-05
--2.686746137931035e-05
  0
  0
  0
  0
  0
+ 0.38851
+-0.923689
  0
  0
  0
@@ -4270,183 +3672,81 @@
  0
  0
  0
- 31.20419655172414
--4.343999655172413
--19.67653172413793
--29.38343448275862
- 4.196778827586207
- 0.3156872551724138
- 0.02547222551724138
--0.439704924137931
--0.08877268965517242
- 0.1064178620689655
- 0.1531680275862069
--0.01978472620689655
--0.01172929586206897
- 0.004675749103448276
- 0.001276916275862069
- 0.0004374010206896552
--0.0002353573448275862
- 0.000458532124137931
- 0.0001799789931034483
--0.0001972880896551724
- 0.0001210269379310345
- 5.304250689655173e-05
--3.912995379310345e-05
- 4.256381448275862e-05
- 2.045877103448276e-05
--2.095607310344828e-05
  0
+ 0.903624
  0
+ 0.0713196
+ 0.343009
  0
  0
+-0.9609259999999999
  0
  0
  0
+ 0.226351
+-0.882984
  0
  0
  0
  0
+ 0.926118
  0
  0
- 18.72144
- 2.066965655172414
- 4.175957379310344
--1.630147517241379
- 0.1071159931034483
--0.8186847586206897
--0.3642535862068966
--0.1602326896551724
- 0.1920778827586207
- 0.2764750827586207
- 0.007370637931034483
- 0.004675749103448276
--0.000920375448275862
--0.0003467628689655172
--0.0002353573448275862
--0.0003653528137931034
--0.0002778349931034483
--0.0001972880896551724
--2.607678e-05
--5.397381379310345e-05
--3.912995379310345e-05
--1.724845448275862e-05
--2.686746137931035e-05
--2.095607310344828e-05
  0
+-0.222101
  0
  0
+-0.930119
  0
+-0.980831
+ 0.0348414
  0
+ 0.233082
+ 0.8291849999999999
+ 0.0511106
+ 0.760646
  0
  0
  0
  0
+ 0.9417489999999999
  0
  0
  0
+-0.49931
+-0.244322
+ 0.698949
  0
+-0.968606
  0
- 44.10202482758621
- 14.75510551724138
- 17.5031124137931
--28.52798137931034
- 5.137234068965518
--19.80478965517241
--0.02972575448275862
- 0.02505569310344827
--0.04004924482758621
--0.1441821586206897
- 0.1415222137931035
--0.02162762620689655
--0.02169661172413793
--0.01120900344827586
- 0.004080462068965517
- 0.001647644689655172
- 0.001321036551724138
--0.001117773103448276
- 0.0005876572344827586
- 0.0004546790827586207
--0.0002559836620689655
- 0.0001550013586206897
- 0.0001380339931034483
--0.0001082641724137931
- 5.047456620689655e-05
- 4.450631034482759e-05
--2.682253517241379e-05
  0
  0
+-0.233887
  0
  0
+-0.0975314
+ 0.218762
  0
  0
  0
  0
  0
  0
- 31.85608620689655
--7.790184827586207
- 5.244314689655172
--2.904788896551724
- 4.347835655172414
--0.6341364413793104
- 0.5758511655172414
--0.4973707379310345
- 0.172837524137931
--0.1696489172413793
- 0.02592599517241379
--0.01120900344827586
--0.005797224206896552
- 0.002108578206896552
- 0.001321036551724138
- 9.034236551724138e-05
--0.0006386535310344828
- 0.0004546790827586207
- 0.0001334188689655172
--0.0001582761655172414
- 0.0001380339931034483
- 4.975547448275862e-05
--7.459131034482759e-05
- 4.450631034482759e-05
- 1.76193275862069e-05
--1.869100275862069e-05
  0
  0
  0
  0
  0
+ 1
  0
  0
  0
  0
  0
- 16.77203379310345
--19.5946448275862
- 4.375358068965517
--14.9889724137931
--0.1228040137931034
- 0.1142649862068966
--0.07264891034482758
- 0.2487791172413793
--0.2441914620689655
- 0.03731467379310345
- 0.004080462068965517
- 0.002108578206896552
--0.0007726566896551724
--0.001117773103448276
--0.0006386535310344828
--0.0001240553103448276
--0.0002559836620689655
--0.0001582761655172414
--5.860064275862069e-05
--0.0001082641724137931
--7.459131034482759e-05
- 1.315795172413793e-05
--2.682253517241379e-05
--1.869100275862069e-05
  0
  0
  0
+ 0.9227070000000001
  0
  0
  0
@@ -4455,30 +3755,7 @@
  0
  0
  0
- 51.14874551724137
- 15.45131793103448
- 16.93451655172414
--22.51253724137931
--20.79194275862069
- 2.60916924137931
- 0.08399590344827586
--0.07117767586206897
- 0.01422971793103448
- 0.2102414689655172
--0.2574469034482759
- 0.02414373862068966
- 0.0001922512827586207
- 0.0009383610344827586
--0.001028225103448276
- 0.0005989798413793103
- 0.0007233198620689655
--0.0005540993517241379
- 0.0001241261379310345
- 0.0001704768
--0.0001609140413793104
- 5.075241724137931e-05
- 6.178616000000001e-05
--4.589591448275862e-05
+-0.957744
  0
  0
  0
@@ -4489,564 +3766,6237 @@
  0
  0
  0
- 26.23579172413793
--6.549084137931034
--20.24790344827586
--23.33974551724138
- 2.769525448275862
--0.0659439
- 0.1738010068965517
- 0.005887093931034483
--0.2063625517241379
- 0.2526987931034483
--0.02369749517241379
- 0.0009383610344827586
--7.446316551724139e-05
--0.0008924297241379311
- 0.0007233198620689655
- 0.0003571539310344828
--0.0004691640206896552
- 0.0001704768
- 9.39471103448276e-05
--0.0001473003724137931
- 6.178616000000001e-05
- 3.472178068965517e-05
--4.032660275862069e-05
  0
  0
+ 0.328535
  0
  0
+-0.913172
  0
  0
  0
  0
  0
  0
- 17.06144827586207
- 2.266118344827586
- 1.523120551724138
--1.948163586206896
- 0.1550830551724138
- 0.8867825517241379
- 0.1736130206896552
- 0.03153576689655173
--0.03861616482758621
- 0.003622695448275862
--0.001028225103448276
--0.0008924297241379311
- 8.90069724137931e-05
--0.0005540993517241379
--0.0004691640206896552
- 0.0001041154965517241
--0.0001609140413793104
--0.0001473003724137931
- 7.693126896551724e-05
--4.589591448275862e-05
--4.032660275862069e-05
- 1.038856e-05
- 0
- 0
  0
+-0.475135
  0
- 0
- 0
- 0
- 0
- 0
- 0
- 27.39055862068965
- 22.58968344827586
--13.10365793103448
--4.452152482758621
--2.047016827586207
- 10.6623324137931
--0.4173785793103448
- 0.5297550206896552
- 0.5515515034482759
- 0.1896115931034483
--0.2334113448275862
--0.01761619103448276
--0.0005717950965517241
- 0.0003060482068965517
--0.0003171232827586207
- 0.0003598600827586207
- 0.0005471162068965518
--0.0006653002482758621
- 0.0001740269862068966
- 0.0002120172275862069
--0.0001917802068965517
- 3.705769448275862e-05
- 5.577371862068965e-05
--5.703476275862069e-05
- 1.114411862068966e-05
- 4.449771517241379e-05
--4.088304275862069e-05
- 0
- 0
- 0
- 0
- 25.32444620689655
--7.190448965517241
--2.053906344827586
--2.013893517241379
- 5.004171310344828
- 0.4477848206896552
--0.5346223862068966
- 0.491654275862069
--0.2321850344827586
- 0.2858218344827586
- 0.02157230827586207
- 0.0003060482068965517
--0.0007128908275862068
--0.0002523390896551724
- 0.0005471162068965518
- 0.0002325003310344828
--0.0005923560620689656
- 0.0002120172275862069
- 0.0001057113517241379
--0.000163355524137931
- 5.577371862068965e-05
- 3.215973862068966e-05
--5.458520620689655e-05
- 4.449771517241379e-05
- 4.634963379310345e-05
--6.013830137931034e-05
- 0
- 0
- 0
- 0
- 41.67192
- 10.81293379310345
- 4.871001310344828
--40.0434275862069
--0.05709302827586207
- 0.07734135862068967
- 0.2322138551724138
- 0.02177473103448276
--0.02680413931034483
--0.002021704068965517
--0.0003171232827586207
--0.0002523390896551724
--0.000694945724137931
--0.0006653002482758621
--0.0005923560620689656
- 0.0004656841862068965
--0.0001917802068965517
--0.000163355524137931
- 7.288139310344828e-05
--5.703476275862069e-05
--5.458520620689655e-05
- 3.460066413793103e-05
--4.088304275862069e-05
--6.013830137931034e-05
- 3.614704068965518e-05
- 0
- 0
- 0
- 0
- 31.28482896551724
- 22.67200413793103
--11.3238324137931
--26.40783034482759
--21.17729931034483
- 0.1017780827586207
--0.3558900827586207
- 0.4388752689655173
- 0.03333126896551724
- 0.2029007034482758
--0.2528391034482759
--0.01990916482758621
- 0.0003171184344827586
- 0.00112734675862069
--0.0009300303448275862
- 0.0005882853931034484
- 0.0007279755172413793
--0.0004262355793103448
- 0.0001283214137931034
- 0.0001918923586206897
--0.0001460472413793103
- 3.628917310344827e-05
- 0.0001477496620689655
--0.0001027058275862069
- 0
- 3.762899103448276e-05
--3.091286e-05
- 0
- 20.49952482758621
--5.517259379310344
--21.16524137931034
--17.80257862068965
- 0.02278027655172414
- 0.4534075586206897
--0.524190048275862
--0.0304271648275862
--0.2497697724137931
- 0.311247248275862
- 0.02450902620689655
- 0.00112734675862069
- 0.0001820861379310345
--0.000875998275862069
- 0.0007279755172413793
- 0.0004109818827586207
--0.0003774773172413793
- 0.0001918923586206897
- 0.0001276895034482759
--0.0001458063517241379
- 0.0001477496620689655
- 0.0001777992896551724
--0.0001630586413793103
- 3.762899103448276e-05
- 4.900443517241379e-05
--5.391214e-05
- 0
- 41.56537862068966
- 0.171111
--0.004346238689655172
--2.056418068965517
- 0.5248021172413794
- 0.4753244275862069
- 0.3376739931034483
--0.01885135172413793
- 0.02349198620689655
- 0.001852110482758621
--0.0009300303448275862
--0.000875998275862069
--0.000157088
--0.0004262355793103448
--0.0003774773172413793
--1.270338275862069e-05
--0.0001460472413793103
--0.0001458063517241379
- 4.708471586206896e-05
--0.0001027058275862069
--0.0001630586413793103
- 5.657433931034483e-05
--3.091286e-05
--5.391214e-05
- 2.766891172413793e-05
- 0
- 31.82766068965517
- 24.94355379310345
--11.38292482758621
--4.798616620689655
--4.347339310344828
- 10.78249379310345
--0.4060966965517241
- 0.3893542137931034
- 0.6586543655172413
- 0.181095324137931
--0.2184234275862069
--0.05618209862068965
--0.0001991724
- 0.0006770993448275862
--0.0005171988206896552
- 0.0003368152689655173
- 0.0006365474827586206
--0.0006031668482758621
--7.782328965517242e-06
- 0.0003898951655172414
--0.0003244248965517242
--1.111703655172414e-05
- 7.840250344827585e-05
--7.542615172413794e-05
- 0
- 23.85843931034483
--11.71251103448276
--4.468799862068965
--4.845028689655172
- 11.5023275862069
- 0.4992299310344828
--0.6860933724137932
- 0.290327724137931
--0.2256665586206897
- 0.2721866827586207
- 0.07001067586206897
- 0.0006770993448275862
--0.0003251973034482758
--0.0004712996137931034
- 0.0006365474827586206
- 0.0004105081448275862
--0.0006390865379310345
- 0.0003898951655172414
- 0.0005645707310344826
--0.0006405520068965518
- 7.840250344827585e-05
- 0.0001246207586206897
--0.0001650521034482759
- 3.568677586206896e-05
- 38.35983172413793
- 10.83331931034483
- 11.49231724137931
--36.48879586206896
- 0.03749725586206896
--0.1074435310344828
- 0.318266324137931
--0.01776978620689655
- 0.02143389586206897
- 0.00551445351724138
--0.0005171988206896552
--0.0004712996137931034
--0.0005822081379310345
--0.0006031668482758621
--0.0006390865379310345
- 0.000341624
--0.0003244248965517242
--0.0006405520068965518
- 0.0003277401034482759
--7.542615172413794e-05
--0.0001650521034482759
- 0.0001118416413793103
--3.015415724137931e-05
- 32.11008344827586
- 24.49169862068966
--6.97196827586207
--26.71608275862069
--20.63384965517241
--4.448274344827587
--0.3598160206896552
- 0.4345241310344827
- 0.1119097586206897
- 0.1920598068965517
--0.2455766068965517
--0.06692743724137931
--0.000127723
- 0.0008526115172413793
--0.0005681574
--0.0003432196413793103
- 0.001015391586206897
--0.0006073388896551725
--5.916006965517241e-05
- 0.0003274015379310345
--0.000254043924137931
- 1.771352e-05
- 22.47626896551724
--8.386455862068965
--20.65826551724138
--16.77413862068965
--3.189389862068965
- 0.3393084689655173
--0.6062219862068965
--0.2091215793103448
--0.2316466620689655
- 0.2962004275862069
- 0.08072387586206897
- 0.0008526115172413793
--2.439149310344828e-05
--0.0006036224827586207
- 0.001015391586206897
- 0.001426153379310345
--0.001334670413793103
- 0.0003274015379310345
- 0.0006083681655172414
--0.0006217567172413794
- 0.0001290459448275862
- 38.79810965517241
--4.375698344827586
--3.247298896551724
--2.743457793103449
- 0.6136763586206896
- 0.3006891931034483
- 0.3584232896551724
--0.05958306413793104
- 0.07618717931034483
- 0.02076491586206897
--0.0005681574
--0.0006036224827586207
--0.0005279914620689656
--0.0006073388896551725
--0.001334670413793103
--6.94988e-06
--0.000254043924137931
--0.0006217567172413794
- 0.0002895178275862069
--8.758220689655173e-05
- 31.64360689655172
- 25.77338
--5.525074620689654
--4.347569310344827
--5.55770724137931
- 9.257432413793103
--0.3196056827586207
- 0.2661246206896552
- 0.723968827586207
- 0.1688882413793103
--0.2086634482758621
--0.09015952413793103
- 2.06359675862069e-05
--0.008449997931034483
- 0.00697492551724138
--0.000433861
- 0.0003170682896551724
--0.0003178751931034482
--5.09637e-05
- 26.92778896551724
--12.75989586206897
--5.647741448275863
--9.191789655172412
- 15.98697724137931
- 0.4519723379310345
--0.6449695999999999
- 0.1306479931034483
--0.2159480620689655
- 0.2668143379310345
- 0.1152842344827586
--0.008449997931034483
--0.03631898689655173
- 0.03153816896551724
- 0.0003170682896551724
- 0.001028768344827586
--0.001532291793103448
- 0.0002567813724137931
- 35.08112965517241
- 9.268364137931034
- 15.95688689655172
--32.66338827586207
- 0.1342069310344828
--0.2533898620689655
- 0.304689475862069
--0.05885251793103449
- 0.07271531034482759
- 0.03142007034482758
- 0.00697492551724138
- 0.03153816896551724
--0.02414350827586207
--0.0003178751931034482
--0.001532291793103448
- 0.001036574
--0.0002167911034482759
- 32.89063862068966
- 25.24023103448276
--4.625261310344827
--28.08210620689655
--19.92846413793104
--5.223497034482759
--0.2298624413793103
--0.1301227172413793
--0.1272063034482759
- 0.1159782896551724
- 0.2055630068965517
- 0.148612124137931
- 0.0009842978620689656
--0.004301679586206896
- 0.003520158275862069
--0.0003976991379310345
- 25.4544675862069
--12.69912965517242
--20.05793517241379
--15.4712924137931
--3.449189103448276
- 0.1557159034482758
- 0.06717949448275862
- 0.07351046206896551
--0.1432978206896552
--0.2539802758620689
--0.1836112068965517
--0.004301679586206896
--0.03349073379310345
- 0.02864431103448276
- 0.0008653670344827586
- 34.94141724137931
--5.263151172413793
--3.490086689655172
--2.929016068965518
- 0.6740874068965517
- 0.4716791724137931
- 0.4273861448275862
--0.06191695448275862
--0.1097388827586207
--0.07933599310344827
- 0.003520158275862069
- 0.02864431103448276
--0.02192773862068965
--0.0005837836344827587
- 31.07277172413793
- 23.38896413793103
--1.325949517241379
--2.868642482758621
--3.715712068965517
- 7.188745517241379
- 0.08750422068965517
--0.1753217517241379
--0.8371608275862069
- 0.0929403172413793
- 0.1567172965517241
- 0.09608602758620689
--0.0005966441172413793
- 28.52652137931035
--14.32528551724138
--3.742328827586207
--12.56512275862069
- 17.57024137931034
--0.08640257241379309
- 0.03999789586206896
- 0.4441856275862069
- 0.1647217034482759
- 0.2777685034482759
- 0.1703047724137931
--0.001111297448275862
- 34.52513034482759
- 5.740124620689655
- 17.56744068965517
--31.97023310344828
--0.03360908551724138
--0.0129896524137931
- 0.09078313793103449
- 0.1190839034482759
- 0.2008087310344828
- 0.1231223724137931
- 0.001019690482758621
- 30.68476689655173
--15.74472206896552
--0.434137875862069
--27.64366206896552
- 19.5838675862069
--5.996231655172414
- 0.0919669448275862
- 0.02833621655172414
- 0.0236381
- 0.1230282
- 28.37990551724138
--22.70207724137931
- 19.65233793103448
--15.79130275862069
- 4.96072227586207
--0.1700706827586207
- 0.1814652551724138
- 0.08812537931034482
- 0.2074673310344828
- 35.28006965517241
--5.964822275862069
- 5.020082206896552
--3.339309862068965
--0.83906
- 0.4133789724137931
- 0.1631252
- 0.1272022620689655
- 29.71888896551724
--17.88467793103448
- 3.4985
--1.943112137931034
--1.513113310344828
- 1.970103724137931
--0.06623523862068965
- 27.81602
--22.6129475862069
--1.415206965517241
--12.13759172413793
- 17.27531448275862
--0.08042344137931033
- 37.25177241379311
- 3.421183310344827
- 17.30402068965517
--33.93656689655172
--0.05085664206896551
- 30.09676758620689
- 21.83503103448276
--5.07211875862069
--27.88579034482758
- 28.78493862068966
--19.79339517241379
--20.4170724137931
- 36.01380620689655
- 2.407097931034483
- 29.40254551724138
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.282902
+ 0
+ 0
+ 0
+ 0
+ 0.669243
+ 0
+ 0.497747
+ 0.844847
+ 0
+ 0
+-0.60248
+ 0
+ 0
+-0.199366
+ 0
+ 0.536739
+ 0
+ 0
+ 0
+-0.863662
+-0.812129
+ 0.362183
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.231178
+-0.0229049
+ 0
+ 0.7116130000000001
+ 0.863457
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.688089
+-0.902513
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.56503
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0.495041
+-0.474025
+ 0
+ 0
+ 0.52991
+ 0
+-0.0892505
+ 0
+ 0
+ 0
+ 0
+-0.384599
+ 0.0117166
+-0.955082
+ 0
+-0.357397
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.798405
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.70269
+ 0
+ 0
+ 0
+ 0.00308692
+ 0
+ 0
+ 0
+ 0
+-0.861565
+ 0
+ 0
+-0.377789
+-0.5775439999999999
+ 0
+-0.183317
+ 0
+ 0
+ 0
+ 0
+ 0.203103
+-0.614461
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.20096
+ 0.62612
+ 0
+-0.679671
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.625292
+ 0
+ 0
+ 0
+-0.993743
+ 0
+ 0
+ 0
+ 0
+ 0.0312606
+ 0
+ 0
+ 0
+ 0
+-0.504444
+ 0
+-0.09743880000000001
+ 0
+ 1
+-0.183638
+-0.758983
+ 0.61515
+-0.137712
+ 0
+-0.114525
+-0.130596
+ 0
+ 0
+ 0.792162
+ 0
+ 0
+ 0.175183
+ 0.905067
+-0.0993045
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.667387
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.38125
+ 0
+-0.913071
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.235465
+ 0
+ 0
+ 0
+ 0
+-0.782763
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.925799
+-0.455384
+ 0.92018
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.198667
+-0.560118
+ 0
+ 0.762945
+ 0
+ 0
+ 0
+ 0.0173914
+ 0
+ 0.76628
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.293363
+ 0
+ 0
+ 0
+-0.230035
+ 1
+ 0
+ 0.864734
+ 0.914411
+ 0.117799
+ 0
+ 0
+ 0
+ 0
+ 0.546054
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.530733
+ 0.6806489999999999
+ 0
+ 0
+ 0
+ 0
+ 0.623245
+ 0
+ 0
+ 0
+ 0.08471339999999999
+ 0
+ 0.885162
+ 0
+ 0.199607
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.655425
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.815068
+-0.291773
+ 0
+ 0
+ 0
+-0.278294
+ 0
+ 0
+ 0.07877049999999999
+ 0.356073
+ 0.982938
+ 0.176067
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.010633
+ 0
+ 0
+ 0
+ 0.49793
+ 0
+ 0
+-0.376356
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.199793
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.06593309999999999
+-0.541268
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1.302394
+ 0
+ 0
+ 0.845684
+ 0
+ 0
+-0.484673
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.430443
+ 0.368101
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.17098
+ 0
+ 0
+ 0.6455959999999999
+ 0
+ 0
+ 0
+ 0
+ 0.7934870000000001
+ 0
+-0.441221
+ 0
+ 0.82097
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.111124
+ 0
+-0.166148
+ 0
+-0.218475
+-0.131328
+ 0
+-0.411184
+ 0
+ 0.267467
+ 0
+ 0
+ 0
+-0.585544
+ 0
+ 0
+ 0.319084
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.376134
+ 0
+-0.192926
+ 0
+ 0.0767085
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.754015
+ 0
+ 0
+-0.642196
+ 0
+ 0
+-0.236775
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0508256
+-0.868807
+ 1
+ 0
+ 0.02112
+ 0
+ 0.530556
+ 0.116396
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.6908339999999999
+ 0
+-0.978752
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.79189
+ 0
+ 0
+ 0
+ 0
+ 0.166072
+ 0
+-0.497046
+ 0
+ 0
+ 0.792318
+ 0
+ 0
+ 0
+ 0.168629
+-0.319897
+ 0
+-0.736453
+ 0
+ 0
+ 0
+ 0
+-0.725367
+-0.054949
+ 0.751924
+ 0
+ 0
+ 0.673337
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.30665
+ 0
+ 0
+ 0.86015
+ 0
+-0.876984
+ 0.230109
+ 0
+ 0
+ 0.630907
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.49027
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.527546
+ 0
+ 0.154843
+ 0
+ 0
+ 0.618815
+ 1
+ 0
+ 0
+ 0
+ 0.327075
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0416737
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.474633
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.93632
+-0.424238
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.597711
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.65785
+-0.457688
+ 0.5479540000000001
+ 0
+ 0.482152
+ 0
+ 0
+-0.906761
+ 0
+ 0
+-0.502023
+ 0.326552
+ 0
+ 0
+ 0
+-0.323529
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.388644
+ 0
+ 0
+ 0.871507
+ 0
+ 0
+ 0.643944
+ 0.64539
+ 0
+ 0
+ 0
+-0.443978
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.809685
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.721132
+ 0
+ 0
+ 1.460584
+ 0
+ 0.388158
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.108809
+ 0
+ 0
+ 0
+-0.483398
+ 0
+ 0.708294
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.480344
+-0.169195
+ 0
+ 0
+-0.6234690000000001
+ 0.515229
+-0.958591
+-0.722326
+ 0.0716487
+ 0
+-0.27721
+ 0.648833
+ 0.883103
+ 0.22843
+ 0
+ 0
+-0.994491
+-0.552121
+ 0.103506
+ 0
+-0.0893031
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.384351
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.828334
+ 0
+ 0
+-0.607252
+ 0
+-0.5787910000000001
+ 0
+ 0
+-0.900639
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.274741
+ 0
+ 0
+ 0.0566341
+ 0
+ 0
+ 1.549332
+ 0
+-0.741055
+ 0
+ 0.321289
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.146525
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.429723
+ 0
+ 0
+ 0.535212
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.680748
+ 0.480954
+ 0.973766
+ 0
+ 0.0883703
+-0.5017740000000001
+ 0
+ 0
+ 0.0290042
+ 0
+ 0
+-0.349416
+-0.118985
+ 0
+-0.584266
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0299071
+ 0
+ 0
+ 0.471451
+ 0
+ 0
+ 0.333558
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.682487
+ 0
+ 0
+ 0.87783
+ 0
+ 0
+ 0.826115
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.296235
+ 0.314654
+ 0.386811
+ 0
+ 0
+ 0
+ 0
+ 1
+-0.3923
+ 0
+-0.6305500000000001
+ 0
+-0.548669
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.5796289999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.166999
+ 0
+ 0
+ 0
+-0.19651
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.00813035
+ 0.466892
+-0.469971
+ 0
+-0.185959
+-0.0194231
+ 0.288058
+ 0
+ 0.164142
+-0.903722
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.431219
+ 0
+ 0
+ 0.199979
+-0.254834
+ 0
+-0.04411
+ 0.260507
+ 0
+-0.462448
+ 0
+ 0
+ 0
+ 0
+-0.230016
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.021241
+ 0.123359
+ 0
+ 0.355183
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.735237
+ 0
+ 0
+ 0
+ 0
+ 0.320242
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.225627
+ 0
+ 0
+ 0
+ 0.052578
+ 0
+ 1.857809
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.111731
+ 0.58955
+ 0
+ 0.554756
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.979145
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.626348
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.358521
+-0.140944
+ 0
+ 0
+-0.726899
+ 0
+ 0.200139
+ 0
+ 0
+ 0
+ 0
+ 0.686053
+-0.657202
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.914959
+ 0
+-0.54144
+-0.190381
+ 0
+-0.104641
+ 0
+-0.694403
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.5385450000000001
+ 0
+ 0
+ 0.564571
+ 0
+ 0
+ 0
+-0.68738
+ 0
+ 0
+ 0
+ 0.684455
+ 0
+ 0.340214
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.108828
+ 0
+-0.131478
+ 0
+ 0
+ 0
+ 0
+ 0.838798
+ 0
+ 0
+-0.354007
+ 0
+ 0
+ 1
+ 0.701021
+-0.849293
+ 0
+ 0
+ 0
+ 0
+-0.196869
+ 0
+ 0
+ 0
+-0.715937
+ 0
+ 0.857105
+ 0
+ 0
+ 0
+ 0.262442
+ 0.303702
+ 0
+ 0
+-0.336013
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.903759
+-0.991656
+-0.5859799999999999
+ 0
+ 0
+ 0
+ 0
+ 0.847888
+-0.792662
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.504401
+-0.577071
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0703101
+ 0
+-0.81279
+ 0
+ 0
+-0.0746636
+ 0
+-0.5602200000000001
+-0.430017
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.487336
+-0.649077
+ 0
+-0.106342
+ 0
+ 0
+-0.223582
+ 0
+-0.113644
+ 0
+ 0.307304
+-0.848152
+ 0
+ 0
+-0.111501
+ 0
+ 0.753867
+ 0.869327
+ 0
+ 0
+-0.865273
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.482212
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.536209
+ 0
+ 0
+ 0
+ 0
+ 0.0232492
+ 0
+ 0
+ 0
+-0.536707
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.840183
+ 0
+ 0.279136
+-0.0719142
+ 0
+ 0.470608
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.07160039999999999
+ 0
+ 0
+ 0
+-0.326723
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.849594
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.781134
+ 0.0505574
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.558619
+ 0
+ 0
+ 0
+ 0
+-0.239596
+ 0
+ 0
+ 0
+ 0
+-0.48385
+ 0.292055
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.819106
+ 0
+ 0.293568
+-0.0414015
+ 0
+-0.643734
+ 0.433746
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0.284906
+ 0
+ 0
+ 0.578257
+ 0
+ 0.0798566
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.961184
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.098804
+ 0.796017
+ 0
+ 0
+ 0
+ 0.648452
+-0.132243
+ 0.286178
+ 0.281413
+ 0.56848
+ 0
+-0.718452
+ 0
+ 0
+ 0
+ 0
+ 0.935816
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.372333
+ 0
+ 0.268971
+-0.247968
+ 0
+ 0
+ 0
+ 0
+ 0.8371960000000001
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0509942
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.441172
+ 0
+-0.81114
+ 0.399556
+ 0
+ 0
+ 0
+ 0.124196
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.17363
+ 0
+ 0.269626
+ 0
+-0.343679
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.7109799999999999
+ 0
+-0.931864
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.971546
+ 0.8814689999999999
+-0.590248
+ 0
+ 0
+ 0.837853
+ 0
+ 0.00621434
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.27041
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.147777
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.06390469999999999
+ 0
+ 0
+ 0.853331
+ 0
+-0.9840680000000001
+ 0
+ 0.533101
+ 0
+-0.695331
+ 0
+ 0
+ 0
+-0.248138
+ 0
+ 0
+ 0
+-0.428246
+ 0
+-0.17748
+ 0
+ 0.557453
+ 0
+ 0.564822
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.679042
+ 0
+ 0
+ 0.0218119
+ 0
+ 0.985649
+ 0
+-0.86148
+ 0
+ 0
+ 0.628686
+ 0.754839
+-0.876493
+ 0
+ 0
+ 0.710156
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0.0239994
+-0.356992
+ 0
+ 0
+ 0
+ 0.465411
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.144247
+ 0
+-0.177605
+ 0
+ 0.5092989999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.31462
+ 0.889902
+ 0
+ 0
+-0.6141760000000001
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.129879
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.307658
+ 0
+ 0
+-0.0384306
+-0.222658
+-0.203847
+ 0
+ 0
+-0.646328
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.761641
+ 0
+-0.334986
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.872873
+ 0
+ 0
+ 0.137286
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.97914
+ 0
+ 0
+-0.572688
+-0.309827
+ 0
+ 0
+-0.64724
+ 0
+ 0.451524
+ 0
+ 0.302897
+ 0
+ 0
+ 0
+ 0
+ 1
+-0.62837
+ 0.9483009999999999
+ 0
+-0.99048
+ 0
+ 0
+ 0
+ 0.176295
+ 0.327433
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.211642
+ 0
+ 0
+ 0.450837
+ 0
+ 0.0451532
+-0.624559
+-0.0263607
+ 0
+ 0
+ 0
+ 0.488915
+-0.391069
+ 0
+ 0.521818
+ 0
+ 0.838289
+ 0.09629450000000001
+ 0
+ 0.642062
+ 0
+ 0
+ 0.07720630000000001
+ 0.17289
+ 0
+ 0
+ 0
+-0.0262979
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.349681
+ 0
+ 0
+-0.377493
+ 0
+ 0.75235
+ 0.81458
+ 0
+ 0
+ 0
+ 0.046767
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.908263
+ 0
+ 0.938391
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.594685
+ 0
+ 0.726817
+ 0
+ 0
+ 0.322359
+ 0.52391
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.9013600000000001
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+-0.50035
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.451309
+-0.73406
+ 0
+-0.9554589999999999
+ 0
+ 0
+-0.822056
+-0.639111
+ 0
+ 0
+-0.0482668
+ 0.0465299
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.68206
+ 0
+ 0
+ 0
+ 0
+ 0.800989
+ 0
+-0.396441
+ 0.789783
+ 0
+-0.20036
+ 0.254096
+ 0
+-0.188165
+-0.0659429
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.9553199999999999
+ 0
+ 0
+ 0
+-0.252561
+ 0
+ 0
+ 0.661473
+ 0
+-0.93027
+ 0
+ 0
+ 0
+-0.999474
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.527715
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.982822
+-0.866102
+ 0.47442
+ 0
+ 0
+ 0
+ 0
+ 0.412638
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.793788
+ 0
+ 0.429709
+-0.6330249999999999
+-0.319963
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.702937
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.294167
+ 0
+ 0.682449
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0568795
+ 0
+ 0.334295
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.703967
+ 0.412607
+ 0
+-0.381381
+ 0
+ 0.559341
+ 0
+-0.473474
+ 0
+ 0
+ 0
+ 0
+ 0.738256
+ 0
+ 0
+ 0
+ 0
+-0.0574905
+ 0.158766
+ 0
+ 0
+ 0
+ 0.695005
+ 0
+ 0.241035
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0695915
+ 0
+ 0
+ 0.706532
+ 0
+ 0.907703
+ 0
+ 0.305903
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.252473
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.551775
+ 0.198146
+ 0
+-0.157677
+-0.824238
+ 0
+ 0.176356
+-0.863127
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.523977
+-0.154902
+ 0.58417
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.857947
+ 0
+ 0
+ 0
+ 0
+ 0.7618740000000001
+-0.803
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.932068
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.148857
+ 0
+ 0
+ 0
+-0.619658
+ 0.922966
+ 0
+ 0
+ 0
+ 0.9714429999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.214076
+ 0
+ 0
+ 0.256854
+ 0
+-0.929716
+ 0
+ 0
+ 0
+ 0.645442
+-0.855593
+-0.592538
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.941651
+ 0.864139
+ 1
+ 0.86504
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.9216839999999999
+ 0
+ 0
+ 0.438863
+ 0
+ 0.7542410000000001
+ 0
+ 0
+ 0
+-0.451558
+-0.425642
+ 0
+ 0
+ 0.287925
+ 0
+ 0
+ 0
+ 0
+ 0.311348
+ 0
+-0.0589609
+ 0
+ 0
+ 0
+ 0
+-0.431072
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.484832
+-0.0887162
+ 0
+ 0
+-0.759043
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.183402
+ 0.7576850000000001
+ 0
+ 0
+ 0.690393
+ 0.112233
+ 0.9530380000000001
+ 0
+ 0
+ 0
+ 0
+ 0.904833
+ 0
+ 0
+-0.652309
+ 0.354304
+ 0
+-0.931638
+-0.897598
+ 0.258748
+-0.380534
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+-0.700336
+ 0
+ 0
+-0.775682
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.789168
+ 0
+ 0
+-0.531953
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.827277
+-0.484324
+ 0.21164
+-0.515428
+-0.0275617
+ 0
+-0.952427
+ 0
+-0.190931
+ 0
+ 0
+ 0.0494318
+-0.375571
+ 0
+ 0.353396
+ 0
+ 0
+-0.278438
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.422178
+ 0.0393633
+ 0
+ 0
+ 0
+ 0
+-0.796573
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.994356
+-0.664761
+ 0
+ 0
+ 0
+ 0.15448
+ 0
+ 0
+-0.0262744
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.44675
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0590007
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0.9615050000000001
+ 0
+ 0
+ 0.363568
+-0.386461
+ 0
+ 0
+ 0
+ 0
+-0.101935
+ 0
+ 0
+ 0
+ 0.161245
+ 0.603179
+ 0
+-0.524204
+-0.501036
+ 0.189373
+ 0
+ 0
+ 0.562928
+-0.550678
+ 0
+ 0
+-0.77328
+ 0
+-0.365856
+ 0.569609
+ 0
+-0.769385
+-0.0912419
+ 0
+-0.842261
+ 0
+ 0
+-0.196155
+ 0.195635
+ 0.973052
+ 0
+ 0
+ 0
+ 0
+-0.379927
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.828494
+ 0
+ 0
+ 0
+ 0
+-0.671508
+-0.114764
+ 0
+ 0
+-0.601373
+ 0
+-0.926522
+-0.07070949999999999
+ 0
+ 0
+ 0.731953
+ 0
+ 0
+-0.968479
+-0.486401
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.637302
+-0.730666
+ 0
+ 0
+-0.58639
+ 0.836741
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.244236
+ 0
+-0.520509
+ 0
+ 0.884535
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.269352
+ 0
+ 0
+-0.800805
+ 0
+ 0
+ 0
+ 0.9159040000000001
+-0.393839
+-0.648084
+ 0
+ 0
+ 0.436883
+ 0
+ 0.190425
+ 0
+ 0.905228
+ 0
+ 0
+ 0
+ 0.382932
+ 0.736825
+ 0
+ 0
+-0.394712
+ 0
+ 0
+ 0
+ 0
+-0.632061
+ 0
+ 0
+-0.0550014
+ 0
+ 0.321859
+ 0
+-0.606473
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.277427
+ 0.206675
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.07083299999999999
+ 0
+ 0
+ 0.53715
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.94216
+ 0
+ 0
+ 0
+ 0
+-0.979495
+ 0
+ 0
+ 0
+-0.0696567
+ 0
+ 0.874463
+ 0
+ 0
+-0.493464
+ 0
+ 0
+ 0
+ 0
+-0.551921
+-0.972986
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0413357
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.1557
+ 0
+-0.0225334
+-0.5547569999999999
+ 0
+ 0
+ 0.460477
+ 0.057137
+ 0
+ 0
+ 0
+-0.232163
+ 0
+-0.790698
+ 0
+-0.23347
+ 0.8356
+ 0
+ 0
+ 0.502562
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.00316352
+ 0
+ 0.494408
+ 0
+-0.112819
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.433862
+ 0
+ 0.550571
+ 0
+ 0
+-0.294804
+ 0
+ 0
+-0.790473
+ 0
+ 0
+ 0
+ 0.278551
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.7813369999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.8251230000000001
+-0.6881080000000001
+ 0
+-0.615936
+ 0
+ 0
+ 0.460652
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.561659
+ 0
+ 0
+ 1
+-0.233085
+ 0
+-0.567922
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.128106
+-0.066507
+ 0
+ 0
+ 0
+-0.693791
+ 0.390723
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.8136330000000001
+ 0
+ 0.128001
+ 0
+ 0
+ 0
+ 0.749609
+ 0
+ 0
+ 0
+ 0
+-0.973876
+ 0
+ 0
+ 0
+-0.573386
+ 0
+ 0.505124
+ 0
+ 0
+ 0.301199
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.381741
+ 0
+ 0.165357
+ 0
+ 0
+ 0.741671
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.7546580000000001
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0895673
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.774469
+ 0
+ 0
+-0.200003
+ 0
+ 0
+-0.875549
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.300597
+ 0
+ 0
+ 0
+-0.0424351
+ 0.0350317
+ 0
+-0.68727
+ 1
+ 0
+ 0.327365
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.94607
+ 0
+-0.835302
+ 0
+-0.447948
+ 0
+ 0
+-0.154438
+ 0
+ 0
+ 0
+-0.951099
+-0.650054
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.478363
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.409591
+ 0
+ 0.0498791
+-0.867816
+ 0
+ 0
+ 0
+ 0
+ 0.765087
+ 0
+-0.0401508
+ 0
+ 0.492045
+ 0
+ 0
+ 0
+ 0
+-0.469223
+-0.264128
+ 0
+ 0
+-0.231793
+ 0
+ 0.227703
+ 0.442429
+ 0
+ 0
+ 0
+ 0
+-0.117015
+ 0.8653690000000001
+-0.93642
+-0.08699990000000001
+ 0
+ 0
+ 0
+-0.928704
+ 0
+-0.114269
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.9006
+ 0
+ 0.030293
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0.668677
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.636721
+ 0
+ 0
+ 0.06731570000000001
+-0.64747
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.770371
+ 0
+ 0
+ 0.809311
+ 0
+-0.384487
+-0.169245
+ 0
+ 0
+ 0.165835
+-0.169889
+ 0
+ 0
+ 0
+-0.8056449999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.5241479999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.8078650000000001
+-0.564747
+-0.862371
+ 0
+ 0
+-0.494426
+-0.66308
+ 0
+ 0
+-0.555857
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.470346
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.769811
+ 0
+ 0
+ 0
+ 0.580275
+-0.110297
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.982152
+ 0
+ 0
+ 0
+ 0.839656
+ 0
+ 1.42352
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.344797
+ 0
+ 0
+ 0
+-0.203743
+ 0
+-0.665435
+ 0
+ 0
+ 0
+ 0
+ 0.145982
+ 0
+-0.704364
+ 0
+ 0
+-0.777087
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.168221
+ 0
+ 0
+ 0
+ 0.359237
+ 0
+ 0
+ 0
+ 0.238345
+ 0
+-0.215334
+ 0
+ 0
+ 0
+ 0.0736675
+ 0
+ 0.228351
+ 0
+ 0
+-0.0586232
+-0.574757
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.481291
+ 0
+-0.500246
+ 0
+ 0
+-0.897141
+ 0
+ 0
+ 0
+ 0.506937
+-0.767756
+ 0
+ 0.561381
+ 0
+ 0.8435279999999999
+ 0
+ 0
+ 0
+ 0
+-0.0544575
+-0.324954
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.652217
+ 0
+ 0
+ 0.449998
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.230641
+ 0.175307
+ 0
+-0.854996
+ 0
+-0.580019
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.398972
+ 0
+-0.228622
+-0.843621
+ 0.298253
+ 0.744174
+-0.619254
+ 0
+ 0
+ 0.649302
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.7530250000000001
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.503371
+ 0
+-0.07167999999999999
+ 0
+ 0
+ 0
+-0.0286198
+ 0
+ 0.328736
+ 0
+-0.676026
+ 0
+ 0
+ 0
+ 0
+ 0.688148
+ 0
+ 0
+ 0
+ 0
+-0.108486
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.942459
+-0.705596
+ 0
+-0.0189803
+ 0
+ 0.247192
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.964525
+ 0
+ 0.660018
+ 0
+ 0
+ 0
+ 0
+-0.882556
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+-0.0247446
+ 0
+ 0
+ 0
+ 0
+-0.526309
+ 0
+ 0
+ 0.108938
+ 0
+ 0
+-0.0344695
+ 0.321215
+ 0
+ 0
+ 0.231956
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.611652
+ 0
+ 0
+ 0
+ 0
+-0.521822
+ 0.7905
+ 0
+ 0
+ 0.718824
+ 0
+-0.143417
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0686484
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.201948
+ 0.78129
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.417287
+ 0
+ 0
+ 0.504935
+ 0.902868
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0861237
+ 0.813936
+ 0
+ 0
+ 0
+-0.09783940000000001
+ 0
+ 0
+-0.311002
+-0.606554
+ 0
+ 0
+ 0.640389
+ 0
+-0.9259810000000001
+-0.797533
+ 0.745912
+ 0.0630105
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.711036
+ 0
+ 0
+-0.787053
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0491289
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.835087
+ 0
+ 0
+ 0.540913
+ 0.111028
+ 0
+ 0
+ 0
+-0.451531
+ 0
+-0.106614
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.6152570000000001
+ 0
+ 0
+-0.287537
+ 0
+-0.22707
+ 0
+-0.196099
+ 0
+-0.823563
+ 0.318277
+ 0
+ 0
+-0.9555709999999999
+ 0
+-0.0111826
+ 0
+ 0
+ 0
+ 0
+-0.0273952
+ 0.646208
+ 0
+-0.00414264
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.896778
+ 0
+ 0
+ 0
+ 0
+ 0.08436399999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.46007
+ 0
+-0.979048
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.645428
+ 0
+ 0.427368
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.231126
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.647655
+ 0
+-0.569102
+ 0.623787
+ 0.324656
+ 0
+ 0
+-0.10013
+ 0
+ 0
+ 0
+ 0
+-0.808626
+ 0.835592
+ 0
+ 0
+-0.773088
+ 0.253501
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0647218
+ 0
+ 0
+ 0
+ 0
+ 0.7913
+ 0
+ 0
+ 0
+ 0
+-0.363972
+ 0
+ 0.272026
+ 0
+-0.260402
+ 0
+ 0
+ 0
+ 0
+-0.951564
+ 0
+ 0
+ 0
+-0.383666
+ 0
+ 0
+ 0.269108
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.608417
+ 0
+ 0
+ 0
+ 0.364889
+ 0
+ 0
+-0.544045
+-0.874072
+ 0
+ 0
+ 0
+ 0
+ 0.806578
+-0.196817
+ 0
+ 0
+ 0
+ 0.628158
+ 0
+ 0
+ 0
+ 0
+-0.124168
+-0.625259
+ 0
+ 1
+ 0
+ 0.821933
+ 0
+ 0
+ 0
+-0.725438
+ 0
+ 0
+-0.42745
+-0.345529
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.392244
+-0.195797
+ 0.809964
+ 0
+ 0
+ 0
+-0.452078
+ 0
+ 0.753362
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.391069
+-0.601471
+ 0
+ 0
+-0.272779
+-0.843699
+ 0.938086
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.223789
+ 0
+ 0
+ 0
+-0.0906565
+ 0.792938
+ 0
+ 0
+ 0
+ 0
+-0.466337
+ 0
+-0.212949
+ 0
+ 0
+-0.767822
+ 0
+ 0
+ 0
+ 0
+-0.0578849
+ 0
+ 0.932304
+ 0
+-0.248808
+ 0
+ 0.546924
+-0.291615
+ 0.940302
+-0.535679
+-0.8729980000000001
+-0.0907119
+ 0.180582
+-0.62288
+-0.439856
+ 0
+-0.735379
+ 0
+ 0
+ 0
+ 0.257706
+ 0
+ 0
+ 0
+ 0.104182
+ 0
+ 0.683724
+ 0
+ 0
+ 0
+-0.50883
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0.379411
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.709603
+-0.330541
+ 0.17339
+ 0
+ 0
+ 0
+-0.610659
+ 0.470957
+-0.699522
+ 0.410749
+ 0
+ 0
+-0.38613
+ 0
+ 0
+-0.662214
+ 0
+-0.190436
+ 0.686275
+ 0
+ 0
+ 0
+-0.808785
+ 0.714234
+-0.659617
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.826365
+ 0
+ 0
+ 0
+-0.736697
+-0.928829
+ 0
+ 0
+ 0.941635
+ 0
+ 0
+ 0
+-0.0520551
+ 0
+ 0
+ 0
+-0.411873
+ 0
+ 0
+-0.377701
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.8972290000000001
+-0.260501
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.009926620000000001
+ 0
+ 0
+ 0.956938
+ 0
+-0.529857
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.159394
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0.38418
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0165425
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.936323
+-0.311156
+-0.905285
+ 0
+-0.921131
+ 0.242099
+ 0
+ 0
+-0.607228
+ 0
+ 0
+ 0.561025
+ 0
+ 0
+-0.995801
+ 0
+-0.698597
+ 0
+ 0
+ 0
+ 0.0698281
+ 0
+ 0.512233
+ 0.694503
+ 0.233714
+ 0.922603
+ 0
+ 0.0385542
+-0.981787
+ 0
+ 0
+ 0
+ 0
+ 0.287984
+ 0
+ 0
+ 0
+ 0
+-0.612909
+-0.175915
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.408939
+-0.425077
+ 0
+ 0
+ 0
+ 0.210489
+ 0
+ 0
+ 0
+ 0.573742
+ 0
+ 0
+ 0
+ 0
+-0.0777447
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.256117
+ 0
+ 0
+ 0.886971
+-0.562997
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0.153193
+ 0.321326
+ 0
+ 0
+ 0.07524359999999999
+ 0
+ 0
+-0.584502
+ 0
+ 0
+-0.27177
+ 0
+ 0
+ 0
+-0.588381
+-0.44643
+ 0
+ 0
+ 0.369778
+ 0
+-0.8797199999999999
+ 0
+ 0
+-0.178052
+-0.0389853
+-0.225794
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.59096
+ 0
+-0.723281
+ 0
+ 0
+ 0
+ 0
+-0.851383
+ 0
+ 0
+-0.11226
+ 0.709249
+ 0
+ 0
+ 0
+ 0
+ 0.14424
+-0.680826
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.155768
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.486373
+ 0.589572
+ 0
+ 0
+ 0
+ 0
+ 0.893585
+ 0
+ 0
+ 0.15804
+ 0
+ 0.56112
+ 0
+ 0
+-0.299121
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0.7186709999999999
+-0.381432
+ 0
+ 0
+ 0.0266486
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.582473
+-0.439777
+-0.929901
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.525309
+ 0
+ 0
+-0.953654
+ 0
+ 0.516549
+ 0
+ 0
+ 0
+ 0
+-0.759543
+ 0
+ 0
+-0.579677
+ 0
+ 0
+ 0.323576
+ 0
+ 0.622624
+-0.567655
+-0.382691
+ 0.963982
+ 0.237792
+ 0.843202
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.417184
+ 0
+ 0
+ 0.182926
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.245849
+ 0
+ 0.920666
+ 0.720187
+ 0.749987
+ 0
+-0.608874
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.996817
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.787933
+-0.486203
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.931391
+ 0
+-0.475301
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.311361
+ 0
+-0.851772
+ 0
+ 0
+ 0
+ 0
+-0.393754
+-0.364588
+-0.239975
+ 0
+ 0
+ 0
+ 0
+ 0.0842064
+ 0
+ 0.576852
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.653818
+ 0
+ 0
+ 0
+ 0.7866880000000001
+ 0
+ 0
+-0.63191
+ 0.886781
+ 0
+ 0
+ 0
+ 0.85899
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.175979
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.162419
+ 0.976518
+ 0
+ 0.802856
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.78316
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.74036
+ 0
+ 0
+ 0
+ 0.004632999999999998
+ 0
+ 0
+-0.0223174
+ 0
+ 0
+-0.146539
+ 0
+-0.905285
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.449744
+ 0
+ 0
+ 0
+-0.955958
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.537241
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.9094640000000001
+-0.87996
+ 0
+-0.700531
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0952012
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0350508
+ 0
+ 0
+-0.239566
+ 0
+ 0.122597
+ 0.941122
+ 0
+ 0
+-0.08898780000000001
+ 0
+ 0
+ 0
+ 0.03288
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.857707
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.729566
+ 0
+ 0
+ 1
+ 0.991129
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.72195
+ 0
+ 0
+-0.435701
+ 0.427308
+ 0
+-0.826875
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.773828
+ 0
+ 0.0933237
+ 0
+ 0
+ 0.662411
+-0.0641785
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0350553
+ 0
+ 0
+ 0
+ 0
+-0.652898
+-0.343929
+ 0
+ 0
+ 0
+ 0
+ 0.687895
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.72954
+ 0
+-0.00692189
+ 0
+ 0.656344
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.9322009999999999
+ 0
+ 0
+ 0
+ 0
+ 0.940073
+ 0
+ 0
+ 0
+ 0.414453
+ 0
+ 0
+ 0
+ 0
+-0.9405019999999999
+ 0
+ 0.988143
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.764223
+ 0
+ 1
+ 0
+ 0
+-0.839417
+ 0
+ 0
+ 0
+-0.680931
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.09761549999999999
+ 0.5863390000000001
+ 0
+ 0.365913
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.294686
+ 0
+ 0
+ 0.128318
+ 0.771414
+ 0.374154
+ 0
+ 0
+ 0
+ 0
+-0.58684
+-0.161994
+ 0
+ 0
+ 0
+-0.353274
+ 0.770666
+-0.916142
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.596986
+ 0.363281
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.240488
+ 0
+ 0
+ 0
+ 0
+ 0.835852
+-0.499483
+-0.98917
+ 0.983189
+ 0.8422460000000001
+ 0
+ 0
+-0.252077
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.349788
+ 0
+ 0
+ 0
+ 0.202314
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.114781
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.221866
+ 1
+ 0.00523072
+ 0
+ 0
+ 0.575969
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.434297
+-0.300445
+ 0
+-0.92587
+ 0.645008
+ 0
+ 0.997601
+ 0
+ 0
+ 0.705715
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.155757
+ 0
+-0.790951
+ 0
+ 0.660185
+ 0.533357
+ 0
+ 0
+-0.367672
+ 0
+ 0
+ 0
+ 0
+-0.877645
+ 0
+ 0.7255819999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.788188
+-0.237091
+ 0
+ 0
+ 0
+ 0.317481
+ 0.165559
+ 0.982955
+-0.0286885
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.203245
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.746632
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.945618
+ 0.556732
+ 0
+ 0
+ 0.0982452
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0.561302
+ 0
+ 0
+ 0
+ 0
+-0.206327
+ 0
+ 0
+ 0.841022
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.126896
+ 0
+ 0
+ 0.709178
+ 0.992294
+ 0.5909759999999999
+ 0.671619
+ 0.168266
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.19823
+ 0
+ 0
+ 0
+ 0
+-0.424976
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.323275
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.351516
+ 0.829498
+ 0
+ 0
+-0.575054
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.104291
+ 0
+ 0
+-0.857495
+ 0
+ 0
+ 0.833725
+ 0
+ 0
+ 1
+-0.609754
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.905613
+-0.757395
+-0.889359
+ 0
+-0.0988322
+ 0
+ 0.707666
+ 0.567767
+ 0
+ 0.641566
+ 0.8926269999999999
+ 0.574334
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.811997
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.860195
+ 0
+ 0
+ 0
+ 0.636025
+-0.263735
+ 0.832127
+-0.394369
+ 0
+ 0
+-0.125291
+ 0
+ 0.283666
+ 0
+ 0
+ 0.227194
+-0.199703
+ 0
+ 0
+ 0
+ 0.118292
+ 0
+ 0
+ 0
+ 0
+ 0.492129
+ 0
+-0.0154354
+ 0
+ 0.563103
+ 0.240693
+-0.0331547
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.863059
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.828261
+ 0
+ 0
+ 0
+ 0.181547
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.848399
+ 0
+ 0
+ 0
+-0.987278
+ 0.279914
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.554204
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.787458
+ 0
+ 0
+-0.8421419999999999
+ 0
+ 0
+-0.237401
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.118921
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.237383
+ 0
+-0.231609
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.760876
+ 0
+ 0
+-0.0962519
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.22447
+ 0
+ 0
+ 0.430901
+-0.210851
+ 0
+ 0.592892
+ 0.254826
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.452211
+ 0
+-0.917371
+ 0
+ 0
+ 0.308908
+-0.973656
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.00114599
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.883929
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.580774
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0.890725
+-0.597643
+ 0
+ 0
+ 0.86953
+ 0
+ 0
+-0.68729
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.9751
+-0.149455
+ 0
+ 0.436949
+ 0
+ 0
+ 0
+ 0
+ 0.585487
+ 0.894138
+ 0
+ 0.747988
+-0.366513
+ 0.249258
+ 0
+-0.5717100000000001
+-0.311366
+ 0
+ 0
+ 0.943872
+ 0
+ 0.9739409999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0667751
+ 0
+-0.497347
+ 0
+ 0
+ 0
+-0.112055
+ 0
+-0.935571
+ 0
+-0.985701
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.603685
+ 0
+ 0
+ 0.553101
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.525407
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.236973
+ 0
+-0.400151
+ 0
+-0.51612
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0.130847
+ 0
+ 0
+ 0
+ 0.459564
+ 0
+-0.503699
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.9217689999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.454757
+ 0
+ 0
+ 0
+ 0.649541
+ 0
+ 0
+ 0
+ 0.07609059999999999
+ 0
+ 0.691545
+ 0
+ 0.348616
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.269245
+ 0
+ 0
+ 0
+ 0.8483309999999999
+ 0.326492
+ 0
+-0.110276
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.643369
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.854979
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.775751
+ 0
+ 0
+ 0
+-0.668401
+ 0
+ 0
+ 0
+ 0
+ 0.209881
+ 0
+ 0.0614556
+ 0.9025879999999999
+ 1.927103
+ 0
+-0.00522418
+ 0
+ 0.996584
+-0.964189
+ 0
+ 0
+ 0
+ 0
+ 0.443105
+ 0
+ 0
+-0.497132
+ 0
+ 0
+ 0.871127
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.900002
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.498691
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.82908
+ 0
+-0.816675
+-0.0974974
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.656752
+ 0
+ 0.203061
+-0.0371787
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.914176
+ 0
+ 0
+-0.133199
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.287786
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.153057
+ 0.658984
+ 0
+-0.530613
+ 0
+ 0
+ 0.487696
+ 1
+-0.741605
+ 0
+-0.954681
+ 0.7059
+ 0
+ 0
+-0.9710839999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.438876
+ 0.526448
+-0.411051
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.513471
+-0.175503
+-0.0788451
+-0.901339
+ 0
+ 0
+ 0.958033
+ 0
+ 0
+ 0
+ 0.509942
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.311675
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0221582
+ 0
+-0.551868
+ 0
+-0.44429
+ 0
+ 0
+ 0
+ 0
+ 0.0831853
+ 0
+ 0
+-0.466401
+ 0
+ 0
+ 0.197286
+ 0
+ 0
+ 0
+-0.930013
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.152819
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.445506
+-0.596526
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1.45265
+ 0
+ 0.189004
+ 0
+ 0
+-0.902206
+-0.551132
+ 0
+ 0.590998
+ 0.920715
+ 0
+-0.350707
+ 0.530287
+ 0
+ 0
+ 0
+-0.119325
+ 0.791968
+-0.695116
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.639161
+ 0
+-0.412946
+-0.46124
+ 0
+ 0
+ 0
+ 0.0343029
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.915203
+ 0
+ 0
+-0.9513470000000001
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.301
+ 0.361656
+ 0
+ 0
+ 0.64212
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.897021
+ 0
+-0.195809
+ 0
+ 0.428701
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.74202
+ 0
+-0.0132017
+ 0
+ 0
+-0.171258
+ 0
+ 1
+ 0
+-0.977328
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.334471
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.996248
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.980129
+ 0
+ 0.259116
+ 0
+-0.645209
+ 0
+ 0
+ 0
+ 0
+ 0.694214
+-0.921286
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.671543
+ 0
+ 0
+-0.191846
+ 0
+ 0
+ 0.5371899999999999
+ 0
+ 0
+ 0.61043
+ 0.106201
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.6407620000000001
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.471755
+ 0
+ 0
+ 0
+-0.952646
+ 0
+-0.914968
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.238788
+ 0.202896
+ 0.153444
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.38162
+ 0
+-0.464698
+ 0
+-0.596566
+ 0
+ 0
+ 0
+ 1.91427
+ 0
+ 0
+ 0.10542
+ 0
+ 0
+ 0
+-0.471454
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.118783
+ 0
+ 0.511216
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.405567
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.119054
+-0.461136
+ 0
+ 0.170682
+ 0
+ 0
+ 0
+ 0
+-0.8219649999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.287439
+ 0.676252
+ 0
+ 0
+ 0
+ 0
+-0.0837363
+ 0
+-0.980019
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.936594
+ 0
+ 0
+ 0
+ 0
+-0.191702
+-0.832917
+ 0
+-0.576425
+ 0
+ 0
+ 0
+ 0
+-0.5559539999999999
+-0.503017
+ 0
+ 0
+-0.771043
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.577082
+ 0
+ 0
+-0.0510836
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.981663
+-0.487206
+ 0
+ 0.115087
+-0.915847
+ 0
+ 0
+ 0.380454
+ 0
+-0.370018
+ 0
+ 0
+ 0.431913
+ 0
+ 0
+ 0
+-0.896953
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0402722
+ 0
+ 0
+ 0.9433859999999999
+ 0
+ 0
+-0.112292
+ 0
+-0.269927
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.72212
+ 0
+ 0
+ 0.115987
+ 0
+ 0
+ 0
+ 0
+-0.246633
+ 0
+ 0.421037
+ 0
+ 0
+-0.481999
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.995146
+ 0
+ 0
+-0.888074
+ 0
+ 0.466047
+ 0
+-0.8479179999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.496357
+-0.246731
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0651752
+ 0.439534
+ 0
+ 0.87233
+ 0.927475
+ 0
+ 0
+ 0
+ 0.48891
+-0.34085
+ 0
+-0.918233
+ 0
+-0.550022
+-0.314738
+ 0
+ 0
+-0.526939
+ 0.454792
+ 0
+-0.144511
+ 0.652778
+ 0.338969
+ 0
+-0.266674
+ 0
+ 0
+ 0.493049
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.109431
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0539152
+-0.266303
+ 0
+ 0.8637359999999999
+ 0
+ 0
+ 0
+-0.313799
+ 0
+ 0
+ 0
+ 0.11005
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.117717
+ 0
+ 0
+ 0
+ 0.904555
+ 0
+-0.276677
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.812263
+ 0
+ 0.5696020000000001
+ 0
+-0.363594
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.664364
+ 0.91682
+ 0
+ 0
+ 0
+ 1
+ 0.914829
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.893093
+ 0
+ 0
+-0.478477
+-0.749803
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.973556
+ 0
+ 0.620466
+ 0
+ 0
+ 0
+ 0.316724
+ 0.841707
+ 0
+ 0
+ 0
+ 0.448781
+-0.543077
+ 0
+ 0
+ 0
+ 0
+ 0.302511
+ 0
+-0.10084
+ 0
+ 0
+ 0
+ 0
+-0.748879
+ 0
+ 0
+ 0
+ 0.162529
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.232837
+ 0.465888
+-0.310108
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.790501
+ 0
+-0.307767
+ 0
+-0.0885382
+ 0
+ 0
+ 0.240963
+ 0
+ 0
+ 0
+ 0
+ 0.341789
+ 0
+-0.446857
+ 0
+ 0
+ 0.818532
+ 0
+ 0
+ 0.131666
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.566569
+ 0.904648
+ 0
+ 0
+ 0.394674
+ 0
+-0.0333986
+ 0
+-0.511354
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0.20747
+-0.694747
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.510774
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.7771439999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.868241
+ 0
+ 0
+ 0.0478857
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.6987640000000001
+ 0
+ 0
+ 0
+ 0
+ 0.126435
+-0.809482
+ 0
+ 0
+ 0
+ 0.998844
+ 0
+-0.203976
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.96052
+ 0.9342780000000001
+ 0.728096
+ 0
+ 0
+-0.43435
+ 0
+ 0
+ 0
+ 0.320126
+ 0
+ 0
+-0.324934
+-0.0915899
+-0.155267
+ 0.0514749
+ 0
+ 0
+-0.562667
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.369753
+ 0.0957861
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0.6669389999999999
+ 0
+-0.268673
+ 0
+ 0
+-0.7887420000000001
+-0.842994
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.20721
+ 0
+ 0
+ 0.25425
+ 0.75204
+ 0.809456
+ 0.579693
+-0.2701
+ 0
+ 0
+-0.211509
+ 0.196984
+ 0.989522
+ 0.290667
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.529594
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0966627
+ 0
+-0.261063
+ 0
+-0.649928
+-0.852185
+-0.807771
+ 0.59355
+ 0.429805
+ 0
+ 0
+ 0
+ 0
+ 0.158546
+-0.195617
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.188405
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.275189
+ 0
+ 0.929624
+ 0
+ 0
+ 0
+ 0
+-0.976168
+ 0
+-0.267403
+ 0
+ 0.0969252
+ 0
+ 0
+ 0.945228
+ 0.882784
+ 0.347699
+ 0
+ 0.497427
+ 0
+ 0
+ 0
+ 0
+ 0.177359
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.366913
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.383278
+ 0
+ 0.284306
+ 0
+ 0
+ 0
+ 0.124199
+ 0
+-0.252784
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0688125
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.954908
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.146999
+ 0.499096
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.689511
+ 0
+ 0
+-0.0515379
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.98959
+ 0.526482
+ 0
+-0.421562
+ 0
+ 0
+-0.378078
+ 0
+ 0
+-0.595798
+ 0
+ 0.212354
+ 0
+ 0
+ 0.915579
+-0.00151776
+ 0
+ 0
+ 1
+ 0
+ 0.714744
+-0.105761
+ 0
+ 0
+-0.265856
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.35725
+-0.565294
+ 0.18039
+-0.544199
+ 0.766061
+ 0
+ 0
+-0.47727
+ 0
+-0.0274381
+ 0.296647
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0781284
+ 0.932547
+ 0
+ 0
+-0.462555
+ 0
+ 0
+ 0
+ 0
+-0.08383939999999999
+ 0
+ 0.407766
+ 0
+-0.757333
+ 0.526645
+ 0
+ 0
+ 0.76839
+ 0
+-0.416188
+ 0
+ 0
+ 0
+ 0.385427
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.6073499999999999
+ 0
+-0.885592
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.29486
+-0.00479562
+ 0
+ 0
+-0.7513339999999999
+-0.485451
+ 0
+ 0
+ 0
+-0.0364946
+ 0
+ 0.409823
+ 0.385125
+ 0
+ 0.326716
+ 0
+-0.645059
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+-0.533578
+ 0
+ 0
+-0.141391
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.193076
+ 0
+ 0
+ 0
+-0.5870919999999999
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.311298
+ 0
+ 0
+ 0
+ 0.280015
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0600836
+ 0
+ 0
+ 0
+-0.299407
+ 0.976213
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.251438
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.462446
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.609112
+-0.997952
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.6219789999999999
+ 0.50054
+ 0
+-0.612312
+ 0
+ 0.0358533
+ 0
+ 0.968217
+ 0.801179
+-0.446137
+ 0
+ 0
+ 0
+ 0
+ 0.440971
+ 0.0694135
+ 0
+ 0.871193
+ 0
+ 0
+ 0
+ 0
+ 1.112607
+ 0.5077700000000001
+ 0
+-0.223485
+ 0
+ 0
+-0.510635
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.160727
+ 0
+ 0.414484
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.461221
+ 0
+ 0.931423
+ 0
+ 0
+ 0
+ 0
+-0.454996
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.355027
+ 0
+-0.606612
+ 0
+ 0.7776149999999999
+ 0.368264
+-0.478208
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.253271
+ 0.345292
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.164102
+ 0
+ 0
+ 0
+ 0.530361
+ 0
+ 0
+ 0
+ 0
+-0.837771
+ 0
+ 0.355711
+ 0
+-0.129631
+ 0
+ 0
+ 0
+-0.203876
+ 0
+-0.551889
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.545945
+-0.896469
+ 0
+ 0
+ 1
diff --git a/matrices/test/a_large_inv.mtx b/matrices/test/a_large_inv.mtx
index de77780b131..2ffdaea4944 100644
--- a/matrices/test/a_large_inv.mtx
+++ b/matrices/test/a_large_inv.mtx
@@ -1,35 +1,11 @@
 %%MatrixMarket matrix array real general
 100 100
- 1.728797896003273
- 0.02984235769025842
- 0.17264373825528
- 0.5413353040529991
- 0.5987019788418766
--0.06133662620669985
--0.1658007574034648
- 0.0536623875994405
--0.1118504505468461
- 0.2139852414077251
- 0.1202820547507289
--0.01573780533860078
--0.1987557120472836
--0.1315742535354934
--0.1135950304025646
- 0.08388398560957668
--0.02086145661352059
--0.01851434162863788
--0.1144460311415288
--0.1218978668673911
--0.09296642509642464
+ 1
+-0.4948484826962766
  0
--0.06084404244882342
--0.02226165544957438
+ 1.296448009330129
  0
--0.1291249595230563
--0.0004196580305559638
  0
--0.04430486835167831
--0.06511379180223759
  0
  0
  0
@@ -39,7 +15,9 @@
  0
  0
  0
+-4.392038891786191
  0
+-0.4777284225073569
  0
  0
  0
@@ -53,102 +31,123 @@
  0
  0
  0
+-0.2716119556444772
  0
  0
  0
+ 30.85062637605465
  0
  0
  0
+ 1.65101796609703
  0
  0
  0
+ 2.135572018880382
  0
+-0.6574524552277107
  0
+ 8.678569356519546
+-1.244734604837189
+-0.09718816849192378
  0
  0
  0
+ 1.412909069015802
+ 0.3824057381749257
  0
  0
+ 0.2031984273747871
  0
  0
  0
+-0.3270471083429252
  0
  0
  0
+ 1.947282594671632
  0
  0
  0
  0
+-0.8410213348839369
  0
+-7.615637091260189
+ 0.3877739650231421
  0
+ 1.929363012934077
+ 6.877497641159541
+ 0.2261957338256072
+-0.04340579661659678
+-0.4423782345077397
+-1.32753749229893
  0
+-1.154171675452101
+ 0.1534844417644058
  0
  0
+ 42.76136386500359
+-26.84790352949593
  0
+-1.597985438747375
  0
  0
  0
+-0.5028987458742963
  0
  0
  0
  0
  0
+-0.2641224237401552
  0
+ 0.2273263532487166
  0
  0
  0
  0
  0
  0
- 0.03887151572381558
- 1.371007594485612
--1.479019543911773
- 0.1778047148344253
- 1.342701307435045
--0.4256722598504703
- 0.123038733822573
- 0.9416446789152163
- 0.2177767962670925
--0.03894099315167787
- 0.2439441233070143
- 0.01884046185662314
--0.007859346012856027
- 0.132699957053182
- 0.2917819688986858
--0.1452918485984369
--0.06492839827498387
- 0.2081036600352797
--0.07821264142267455
--0.007955956211488757
- 0.2345703532460748
--0.1727784772586945
- 0.0163173965863426
- 0.2168467739495741
--0.1317134734345977
- 0.01934085786173204
- 0.1521977308030799
--0.08940483444452595
- 0.04607383935768904
- 0.1275886534960688
  0
+-2.647905742240299
  0
  0
  0
  0
+-10.08295973932505
  0
+ 0.02927473708957342
+-0.1170888798804651
  0
+ 0.3094913797252903
  0
  0
  0
  0
+ 0.6843029897075475
+-0.5979120488850802
  0
+-3.930579890760858
  0
  0
+-0.423912919727431
  0
  0
+ 0.04235351624994581
  0
+-57.52832090042818
+ 0.4605407742850633
  0
+ 0.3215523795124502
  0
+-0.7708840587272257
+-0.6803459748153848
+ 0.7329247708861701
+-0.3270807982536441
+-0.5647553557286519
+ 0.7909079631328433
+-0.1197192169001813
+ 2.773195902836251
  0
  0
  0
@@ -156,105 +155,107 @@
  0
  0
  0
+ 0.5247090966189472
  0
  0
  0
+-0.0250845239064978
  0
  0
+-1.882603388163359
  0
  0
  0
  0
  0
+ 0.5652794611179659
+-0.3441786394575287
  0
  0
+-18.63425959211918
  0
  0
  0
  0
  0
+ 0.2253955976162801
  0
  0
  0
  0
  0
  0
+-1.405934967755039
+-21.95019197849467
+ 14.79062519524239
  0
  0
+ 1.200830472811673
  0
  0
  0
+-0.1906769198228738
  0
  0
  0
+-0.5643508305294765
  0
  0
+-0.1412489694348874
+ 1.932208055387784
  0
  0
+ 1.303569311480962
  0
+-0.6442238459405463
  0
+ 1.492049865865577
  0
  0
+-2.581107111180139
  0
  0
+ 1.051959442697243
  0
+ 0.4041162692018464
  0
  0
- 0.1722073848720678
--1.479019543911773
- 6.65516202914633
--0.3020820244240092
--1.481331388834674
- 2.189858250081084
--0.8193637514436026
--0.5961021358922119
- 0.2230931832456023
--0.01662380009911164
- 1.016039769902676
- 0.7685343284549412
--0.7646946564513161
- 0.9024738997372448
- 0.4697439576553524
--0.01989337125783583
- 1.009131607356144
- 0.6728675495624005
--0.7196004484102022
- 0.6952833748147713
- 0.7377810413154007
--0.2832735456156473
- 0.5333953942212142
- 0.8210181222994786
--0.4616852075456391
- 0.4423602556727359
- 0.8641382184607118
--0.4932300960594916
- 0.4678470373828674
- 0.6349279318761307
  0
+-0.5006050847692073
  0
  0
  0
  0
+ 0.6897551534076837
  0
  0
  0
  0
+ 4.540132606636557
  0
  0
  0
  0
  0
+ 0.9915679283395605
+-4.182277033993933
  0
  0
+-1.673207551170892
  0
+ 1.567325040936841
  0
  0
  0
  0
  0
+-1.517490314266554
  0
  0
  0
+ 8.03005129786718
+-0.7776521127117286
+ 0.1434829948132886
  0
  0
  0
@@ -266,11 +267,16 @@
  0
  0
  0
+ 1.707144867273901
  0
  0
+ 3.627874724999257
  0
+ 2.903683424227111
  0
+-8.063528603745542
  0
+-0.4287250896794856
  0
  0
  0
@@ -287,52 +293,33 @@
  0
  0
  0
+ 0.340974238036757
  0
+-0.8145243484136816
  0
  0
+-0.4174159434134531
+ 1.03986445855464
  0
  0
  0
+ 0.36754363051727
  0
  0
  0
  0
+-1.730944174585986
  0
  0
  0
- 0.5459337335217194
- 0.1776908041109713
--0.3026039885609523
- 1.543868655944742
--0.252839810942398
- 0.7528298111523792
- 1.575715845005781
--0.05389825549197692
- 0.3826598416223621
- 0.4193681185798917
--0.2527725796753442
- 0.09285980891848172
- 0.5584224487378291
--0.08641992960391033
--0.1375068321001848
--0.01160008728730358
--0.1677015394419983
--0.2923207796930615
- 0.2280761856726538
--0.07675909360805622
--0.2839983127426551
--0.02318437276961686
--0.1144601999021718
--0.3576860987098101
- 0.06067679887465378
--0.06790891255162879
--0.3338285752306802
  0
--0.1492957277824273
--0.1921416847217516
  0
  0
+ 0.45371565793248
  0
+ 0.02347849783908041
+-0.4260938993976161
+-0.5767700646170179
  0
  0
  0
@@ -355,12 +342,19 @@
  0
  0
  0
+ 0.8860281611756293
  0
+ 1.095606325216979
  0
  0
  0
+ 0.220166619770703
+-0.07542487524145412
+ 0.3948087161896018
  0
+-1.619179205712842
  0
+ 0.3286476679514239
  0
  0
  0
@@ -381,64 +375,59 @@
  0
  0
  0
+-0.5543790209854219
  0
  0
  0
  0
+-0.7396935249967006
+-0.00354028811544854
+ 0.4918595753930014
  0
  0
+ 0.3660310995604132
+-19.23194275997228
+ 15.82680653523076
  0
+ 0.4359570591948023
+-6.098988867740374
  0
+-0.864668198143852
+ 1.026297632537591
  0
  0
+-0.5063974162849202
  0
+-0.9795577741375692
+-1.013294620712399
  0
  0
  0
  0
+ 0.1951293129872262
  0
  0
  0
+-3.016694233463486
  0
- 0.6060244418308103
- 1.342110355024742
--1.479136340522196
--0.2552054953313185
- 1.809240256447562
--0.8175005286752326
--0.6534701740466956
- 1.028664618010117
- 0.03421872709418972
--0.1528811156244744
- 0.3872105636966044
--0.01229435187183393
--0.3562792037882532
- 0.09284810037687727
- 0.3338990384174029
--0.1287811587045025
--0.0213525846007072
- 0.3571849071192184
--0.2433514049189923
--0.03487105645953558
- 0.3426848862370973
--0.1846258648903151
- 0.04294773452474232
- 0.3901279918355072
--0.1724827889892474
  0
- 0.3175705458307803
--0.09462562618834963
- 0.1003553428222384
- 0.1963046485834295
+ 2.379472440700222
+ 1.117893531154898
  0
  0
  0
  0
+ 0.1837803056569752
+-0.3894980338215976
+ 0.2069584579918223
+ 0.6281900908829987
+ 0.1235166319345552
  0
  0
  0
  0
  0
+ 0.6505454939560723
  0
  0
  0
@@ -446,13 +435,18 @@
  0
  0
  0
+-7.242189493265922
  0
  0
  0
  0
  0
  0
+-0.4094411779876041
+ 2.009464200151736
  0
+ 0.001819232267203992
+-0.6722293257606738
  0
  0
  0
@@ -461,9 +455,13 @@
  0
  0
  0
+-1.050081901839461
  0
+-2.412064627243687
+ 1.03350463502369
  0
  0
+ 0.09259686676252599
  0
  0
  0
@@ -474,62 +472,47 @@
  0
  0
  0
+ 5.200930375149771
  0
  0
+-4.206547801675418
+-0.9599040568340262
  0
+ 0.4770814643127358
  0
  0
+ 0.01879090725614984
  0
  0
  0
+-0.05538293578006107
  0
+-0.3820411879721848
  0
  0
+-7.242164375844034
  0
  0
  0
  0
  0
+-0.1063927576234104
  0
+-0.6532878735852709
  0
  0
+ 0.1727225659870178
  0
  0
+-0.3456510655767607
+ 0.508453460284442
  0
  0
  0
  0
  0
--0.03194112332014314
--0.4256722598504705
- 2.189858250081083
- 0.7532808353254768
--0.8131199586084821
- 2.331486928414987
- 0.695705758923945
--0.1113994548569543
- 1.302081319250873
--0.2750484870171973
- 0.1605788551421328
- 1.153023531012888
--0.2702212049549417
- 0.5168597428298998
- 0.9779101244456775
--0.5728708894753742
- 0.4415532294803036
- 0.8820861700818785
--0.6056287796724231
- 0.5887243251705573
- 0.9399966434156231
--0.6885107808810331
- 0.5188258590164004
- 0.9081590830119071
--0.6304687541962376
- 0.6339851353738079
- 0.8291935180145759
--0.5457797200917337
- 0.4945844168919606
- 0.7090239317474276
+ 3.478180969756633
+ 0.3458732325164959
  0
  0
  0
@@ -539,12 +522,16 @@
  0
  0
  0
+ 0.4482468265599172
  0
  0
  0
+ 0.7182405125550337
+ 0.6780568842099182
  0
  0
  0
+-0.1213867687830573
  0
  0
  0
@@ -552,9 +539,15 @@
  0
  0
  0
+-2.256340174664548
  0
+-1.850687897344088
+-0.4348365590933105
+-1.528621907997535
  0
+-0.3100312974456319
  0
+ 0.8330264350366573
  0
  0
  0
@@ -564,20 +557,25 @@
  0
  0
  0
+-1.978876415637884
  0
  0
  0
+ 0.2712240954623512
  0
+ 70.6967998182616
  0
  0
  0
  0
+ 0.3869011085429613
  0
  0
  0
  0
  0
  0
+ 0.5675110897172608
  0
  0
  0
@@ -590,64 +588,38 @@
  0
  0
  0
+ 0.02052967032341324
  0
  0
  0
  0
+-3.152871910382966
  0
  0
  0
  0
  0
  0
--0.1548643115252611
- 0.1283187909078604
--0.7669449247029111
- 1.556663158128347
--0.6333271761689756
- 0.7384719889110311
- 2.339382943676118
--0.0001065160212148971
- 0.5963851331960092
- 0.6279942725411847
--0.3917583190580677
- 0.1440964444239836
- 1.107453695318306
--0.01182009945557227
--0.07834501956743929
- 0.0679419614666439
--0.216910530225636
--0.3756743311685913
- 0.5068196422106329
--0.03115694802541443
--0.3405316603725888
--0.005912912673474375
--0.1328527664382396
--0.5131425550769596
- 0.08970623321647432
--0.02963979113236682
--0.5151693908271608
- 0.04450598014532498
--0.2005617295510455
--0.2360223686931227
  0
- 0.02799878699771987
- 0.01670351103262895
  0
- 0.01056026036814161
- 0.03473662988491884
+-1.467308411651212
  0
  0
  0
+ 2.101269310386782
  0
  0
+-2.133477085544608
  0
+ 1.598779636234988
  0
  0
  0
  0
  0
+-0.1261932360304374
  0
+ 0.08323682421254082
  0
  0
  0
@@ -659,24 +631,32 @@
  0
  0
  0
+-1.259328499242096
  0
+ 0.8991528855085426
  0
  0
+-0.4558697758587361
+-0.5888695098119464
  0
  0
  0
+-3.941184768668226
  0
+ 2.346510335375143
  0
  0
  0
  0
  0
+ 2.644633728916071
  0
  0
  0
  0
  0
  0
+ 2.319588045455526
  0
  0
  0
@@ -688,8 +668,11 @@
  0
  0
  0
+-0.5947318450327681
  0
  0
+-0.5641262393301952
+-49.01905511013528
  0
  0
  0
@@ -700,69 +683,52 @@
  0
  0
  0
- 0.06214300453208339
- 0.9354852416155403
--0.6333655451455816
--0.04346800791317604
- 1.018207739014784
--0.1491290059282507
- 0.004651349919173486
- 0.9870805986122776
- 0.06031732304393399
--0.073017074168649
- 0.5545683806305945
--0.05712703096020352
--0.07322619228097385
- 0.4178672334807771
- 0.2461506631216264
--0.1069772029720746
- 0.1454387741222299
- 0.1697745221805722
--0.1123382964660199
- 0.1200871335217258
- 0.2704647273009727
--0.1605811585681689
- 0.06919442011509791
- 0.2443465497469365
--0.1519811881416389
- 0.04714624903266704
- 0.206313637046412
--0.108691526802571
- 0.076689334485619
- 0.1753116438520862
- 0.02689836434210402
--0.0499349231320889
--0.03516699228438441
- 0.01492532321949855
--0.04902275905416193
--0.02040062918325565
  0
+-1.019067849743785
  0
  0
+-25.87429713932424
  0
  0
+-4.733526222719125
  0
  0
  0
  0
+ 0.1129166470711533
  0
  0
  0
+-0.486174107161233
  0
  0
  0
  0
  0
+ 0.2949077000010569
  0
+-0.1501207485736518
+ 3.849371709284814
+-0.8341451729489401
  0
+ 3.893782413817492
  0
  0
  0
+ 6.792977813286039
+ 0.957872567205309
+-1.003057656973179
+-0.09274640941984065
+ 0.1023420489263638
  0
+ 0.1269699646354275
+ 2.226011329308089
  0
  0
  0
+ 3.416015959932167
  0
+ 0.8943163457222969
  0
  0
  0
@@ -770,187 +736,164 @@
  0
  0
  0
+ 0.6126989280244138
+-0.3246516947542515
  0
  0
+ 0.2754250040898726
  0
  0
  0
+-0.4854122586429255
  0
  0
  0
  0
+-0.0107142540790587
+ 0.8737200000000001
  0
  0
  0
+ 0.3096091746462638
  0
  0
  0
+-0.4151811537546787
  0
  0
  0
+-0.01298821617283267
+-400.0484843923324
  0
  0
+ 0.02756057559993267
  0
  0
  0
  0
  0
+ 22.33316635771502
  0
  0
  0
  0
  0
--0.06928468604523898
- 0.1924734443062584
- 0.07983315677017265
- 0.4221812948475769
--0.001472128358754511
- 1.148343406143939
- 0.6124067137020126
- 0.06031732304393445
- 1.562542007797302
--0.1345247359877685
--0.3022678978807723
- 1.308010045684956
--0.005149807322970538
- 0.07856245850045104
- 1.227724606229933
--0.5088031662896678
- 0.09587377758830851
- 1.103840998333154
--0.45438055368298
- 0.3952985138911458
- 1.004650543045323
--0.7166608227284996
- 0.4560670023299445
- 0.961512627260656
--0.6359055324610381
- 0.6162410698345857
- 0.7852422454121107
--0.4697863994653487
- 0.4723806623419002
- 0.6974564398587709
--0.06038582796186161
- 0.1424683568228679
- 0.1020060601252538
--0.03394496616657218
- 0.1148410759531169
- 0.09582944384407521
  0
  0
+ 0.6176388782446097
  0
  0
  0
  0
+-0.9326696330902033
  0
+ 21.48472691506654
  0
  0
+ 1.360929014254493
  0
  0
  0
  0
+-2.739198557525646
  0
  0
+-0.6394060477436144
+ 0.06064892640321599
  0
  0
  0
  0
+-1.5375885980933
  0
  0
  0
+-2.530034642065866
  0
  0
+-2.355718205752761
  0
  0
  0
+-8.457960636943049
  0
+-0.5334555035837583
+ 0.5950392780592704
+-0.05802908947544636
  0
+ 0.6565447973856923
+-4.817067231874906
  0
  0
  0
  0
  0
+ 0.7777063759657887
+ 1.608808545691914
  0
+ 0.3641645896625149
+ 1.142688497470754
  0
  0
  0
  0
  0
+-0.6935767511801533
  0
  0
  0
  0
  0
+ 0.7893690091084308
  0
  0
  0
  0
+-0.4838670856424674
  0
+ 1.430208426431188
  0
  0
  0
  0
  0
  0
+ 0.1091998636600013
  0
  0
  0
  0
+ 173.0159784009199
  0
+ 3.087783837032923
  0
  0
+-0.5207456317995715
  0
- 0.1953030452416966
--0.02965729095479014
- 0.04116522802299643
- 0.401300639949551
--0.1371158539648848
--0.2145046436500752
- 0.6270693227535503
--0.07513200669510263
--0.1412279181340248
- 1.039563025633924
--0.01179072206987729
--0.04267945842198871
- 0.7509757370003032
--0.192729615872789
--0.2570731096430172
- 0.6312286042010815
--0.09259807634738997
--0.2594933056221668
- 0.4511089701796764
--0.2042910507868694
--0.4238955024968028
- 0.3251012104447056
--0.2014920740048466
--0.4587661671841043
- 0.2209483534798725
--0.2557027259182781
--0.4576766001492489
- 0.1313842251948475
--0.242224942273893
--0.3023196843119589
+ 0.3663560963011827
  0
--0.01790472480393308
--0.0111019205050632
--0.01918367849360801
--0.03323219985452572
- 0.003962150317520989
  0
  0
  0
+ 1.810199877551485
+ 1.161339867225544
  0
  0
+-0.09183729758336448
+ 0.08277385931196803
  0
+ 0.1647219817208061
  0
  0
  0
  0
  0
  0
+ 1.469826937942972
  0
  0
  0
+-0.1370330482960102
  0
  0
  0
@@ -958,23 +901,36 @@
  0
  0
  0
+ 0.1115021919180884
+-0.9328718935886083
  0
  0
  0
  0
  0
+ 0.1748324697248587
+ 0.03063409864837402
  0
+-4.337358926109053
  0
  0
+ 19.01044390856138
  0
  0
  0
+ 0.2130732686517817
+ 0.1060226766717321
  0
+-0.1395624958206959
  0
  0
+-0.2528936532027943
+-0.07134587355046787
  0
+ 2.357417556612328
  0
  0
+ 0.9075786106248219
  0
  0
  0
@@ -986,68 +942,44 @@
  0
  0
  0
+ 0.007320220569743884
+ 3.869578589517917
  0
  0
  0
  0
  0
+ 0.3686263703879875
  0
  0
  0
  0
+-0.2136046797528302
+ 0.6314622736417405
+ 3.36378582607192
  0
  0
+ 0.05303212482830985
+ 0.6835556807052111
  0
  0
  0
- 0.1165555251590301
- 0.2333189149174887
- 0.9385856196542924
--0.2295801810968944
- 0.3632710645090986
- 0.09109638795382748
--0.3815018697566478
- 0.5545683806305947
--0.3022678978807726
--0.006903580998730262
- 1.235247886043665
--0.06544717903866899
--0.3940861174553016
- 0.979664455568662
- 0.0536619717834175
- 0.05001162286366274
- 0.7256359149525634
- 0.09879975328867614
--0.3172820093146727
- 0.4279442699028732
- 0.3045347633422155
--0.07702523243352963
- 0.2238763132387307
- 0.3146624381356733
--0.1892413218863546
- 0.08237855119203366
- 0.3760905370538894
--0.2267293795598638
- 0.1701778837107313
- 0.2805875773989385
--0.07226225483033642
- 0.01015574299629869
- 0.08422307232681754
--0.06659886878599451
- 0.01100800859911553
- 0.04196913255281343
  0
  0
  0
  0
+-0.1889168859470041
  0
  0
  0
  0
+ 0.1304055813943983
  0
  0
  0
  0
+-0.4606790875902882
+-5.165448856705915
  0
  0
  0
@@ -1055,26 +987,35 @@
  0
  0
  0
+-7.955877087780217
+ 0.8864487651172198
  0
+ 13.06568060533946
  0
  0
+-0.3643418680580818
  0
+ 1.395506666819056
  0
  0
  0
  0
  0
+ 0.07538296328271762
  0
+ 1.650302015917521
  0
  0
  0
  0
  0
  0
+ 1.046842999315985
  0
  0
  0
  0
+ 9.532433213272679
  0
  0
  0
@@ -1084,58 +1025,31 @@
  0
  0
  0
+-0.370851982588047
+-3.05343392917343
  0
+-0.8727223392322785
  0
+-0.4777306179856324
  0
+ 0.1580887059847316
  0
  0
+-166.0021304612256
  0
  0
+ 0.7460368757827937
  0
  0
+-3.614862978247329
  0
  0
  0
+ 1.029643528805812
  0
  0
  0
  0
- 0.01795378104051825
--0.007225052258570033
- 0.6134912728331405
- 0.1364541865059864
--0.05300715862378665
- 0.9922806221515482
- 0.1620644139475202
--0.05712703096020362
- 1.308010045684956
--0.03491916990466497
--0.06544717903866892
- 1.276304377245051
--0.158888876164147
- 0.171723591885009
- 1.152924647695593
--0.3563809069216154
- 0.2598826412742355
- 1.120245072661743
--0.5187424910153317
- 0.434162604005829
- 1.002486227576209
--0.6317743329737705
- 0.4817480247581714
- 0.9972127273418497
--0.621396080289389
- 0.5814427918308293
- 0.8439335692507614
--0.4827632948384918
- 0.4895571926735098
- 0.7135399982605599
--0.08864306353318552
- 0.1547337046567904
- 0.1314634352111773
--0.06012774631348366
- 0.1267007380308792
- 0.1092456137591291
  0
  0
  0
@@ -1148,11 +1062,13 @@
  0
  0
  0
+ 1.147807401687179
  0
  0
  0
  0
  0
+ 0.4298236228686947
  0
  0
  0
@@ -1169,9 +1085,11 @@
  0
  0
  0
+-1.298674552549088
  0
  0
  0
+-0.7980067591331992
  0
  0
  0
@@ -1180,66 +1098,39 @@
  0
  0
  0
+ 0.6197771008673595
  0
  0
  0
+-1.21147243167711
  0
+-0.2605982511793086
  0
+ 1.051060954927249
+ 0.9065964908196933
  0
+-1.407909251466448
  0
+-0.5657270027934864
  0
  0
+-10.25518588630423
  0
  0
+ 0.4265223222861988
+-0.2754317183010266
  0
  0
  0
+ 0.3395498820441549
  0
+ 0.02958761136812027
  0
  0
+ 1.344603747980827
  0
  0
  0
--0.2028529906191391
- 0.007663472295111349
--0.6332636843171476
- 0.5169315264040835
--0.3173450425053083
--0.1558395382070967
- 1.084285714379934
--0.06547137228543966
- 0.0266780815608822
- 0.7332098444522026
--0.3763956247787405
--0.1242155162118755
- 1.42477130620432
--0.03322272634792538
--0.09065313938675719
- 0.583702139638498
--0.179268237092648
--0.3263652957012251
- 0.8887839228231826
--0.05040105165255948
--0.3636072270334467
- 0.251104179776346
--0.1632242422562436
--0.5741749931001503
- 0.2191171181716481
--0.1131319404941427
--0.6338434442048085
- 0.2186396947882962
--0.2385042704777418
--0.2787172937017617
- 0.0241845147187668
- 0.03321249096412943
--0.02467524025443448
- 0.02944868388425967
--0.006436125630076017
- 0.06009197189737728
--0.003245039320949559
- 0.04922592897893927
- 0.008010121409948457
- 0.0006142484464413057
  0
  0
  0
@@ -1248,22 +1139,29 @@
  0
  0
  0
+-0.5601827267188916
  0
+-5.383513440182356
+ 0.8733396775619311
  0
  0
  0
+ 0.4317085672447314
  0
  0
+ 0.6453818134063676
  0
  0
  0
  0
+ 0.5200073830189876
  0
  0
  0
  0
  0
  0
+-0.733086422217669
  0
  0
  0
@@ -1271,82 +1169,60 @@
  0
  0
  0
+-0.4432295124436218
  0
  0
  0
+ 0.8648766690509214
  0
  0
  0
  0
+ 0.2540722356705414
+ 3.47068006348652
  0
  0
+ 0.3316352829003929
+ 0.1298033646181902
  0
  0
  0
  0
  0
+-0.6559784090830778
  0
  0
  0
+-2.461317725460241
+-0.2634313360096167
  0
+-0.9057946753403153
  0
  0
  0
  0
  0
+ 3.869235242037691
  0
+ 0.7681260485630639
  0
  0
  0
  0
+ 0.2297889799585331
  0
--0.1190409236343763
- 0.1084272001081703
- 0.7370875781802806
--0.03752296711683519
- 0.05014544785049641
- 0.3485883830073245
- 0.03635450627617183
- 0.3978215801543334
--0.01599259575027843
--0.1443407574962302
- 0.9399017447290055
- 0.0728162523890938
--0.004198784179322817
- 1.183763445144277
- 0.03796101115389522
--0.02204022590066332
- 0.8662030509332055
--0.041441850117764
--0.1277487150329576
- 0.6775746606136939
- 0.2776381400790454
--0.1336264042768787
- 0.3307961044653278
- 0.1791594328316971
--0.2076840021274133
- 0.2111524615864875
- 0.2694634888489146
--0.2137605212156669
- 0.211739270514561
- 0.305333076343939
--0.06821800749424599
- 0.04200875764846114
- 0.09444219322272285
--0.05352595431136706
- 0.0333862654504598
- 0.08280807300772564
--0.00155562349434738
--0.0005608038065905657
- 0.0104663531700386
+ 1.039605046317225
+ 0.1294372838807519
  0
--0.01917990935633506
- 0.004972618268719043
  0
  0
+-2.27514102465816
  0
  0
  0
+ 2.049062068073663
+ 0.1516796642115219
+-3.916866957852078
  0
  0
  0
@@ -1356,20 +1232,26 @@
  0
  0
  0
+ 0.5722260646438212
  0
  0
  0
  0
  0
  0
+-4.710243718168056
+-0.4870266055543358
  0
+-0.3880125804125346
  0
  0
  0
  0
  0
  0
+ 0.8363423534198683
  0
+ 3.056976067612954
  0
  0
  0
@@ -1379,16 +1261,22 @@
  0
  0
  0
+ 0.2441738162347041
+ 0.0357834428473006
+ 1800.314538517002
  0
  0
  0
  0
+-0.3261478742741906
  0
  0
  0
  0
  0
  0
+-3.574838458869212
+ 0.388967096014158
  0
  0
  0
@@ -1396,61 +1284,28 @@
  0
  0
  0
+ 0.5120150047030911
+-0.4429076713605919
+ 6.567417800467082
  0
  0
+ 1.033403610626028
  0
  0
--0.05949396790223082
- 0.2381452539354652
- 0.1241473464989136
--0.03505002145825253
- 0.2434348488090495
- 0.6119931142589278
- 0.02299079749113648
- 0.2030434068413912
- 1.020090053099957
--0.1520906576471463
--0.02909870804578817
- 0.9370444413445618
--0.02010789563185395
- 0.03796101115389554
- 1.389470640568509
--0.301887395571093
--0.06443411895985911
- 1.286845987404684
--0.327244019465183
- 0.2128376419210355
- 1.135011008647634
--0.6355450551505332
- 0.3581562159574024
- 1.102084151619423
--0.6030131505239967
- 0.5457420732008397
- 0.8282210684499527
--0.4105454067502672
- 0.4333463722074636
- 0.7128885865568467
--0.07840272599139589
- 0.1794793119502504
- 0.1165379462304959
--0.03243975956639109
- 0.1366803716927953
- 0.1201622894958054
--0.008013311414311202
- 0.07973197662838594
- 0.001455861826000812
+-0.1201168089399312
  0
- 0.04942737172755984
--0.003232228806017471
  0
+ 1.167161858379611
  0
  0
+-0.6958161223076397
  0
  0
  0
  0
  0
  0
+ 0.4509149317811544
  0
  0
  0
@@ -1458,10 +1313,15 @@
  0
  0
  0
+ 0.4792757943874765
  0
  0
  0
+ 0.4916110309394393
+ 0.1866444401082296
  0
+-0.6845593898010638
+-0.9052032264783701
  0
  0
  0
@@ -1479,12 +1339,18 @@
  0
  0
  0
+ 4.895441817161111
+-0.7209198947190997
+ 4.054781566780847
  0
  0
  0
  0
+ 2.528980634318294
  0
  0
+-0.5629867656968802
+-0.869959
  0
  0
  0
@@ -1500,56 +1366,20 @@
  0
  0
  0
- 0.04288288583548893
--0.1275823476592937
- 0.09520990641469353
--0.04525584431676136
--0.1026072408943112
--0.4555158460192354
- 0.04877161811158949
--0.1000791701098848
--0.4801744998766186
- 0.6163727248135266
- 0.06547496050276826
--0.3253821433735916
- 0.5837321121074017
--0.05649376313981743
--0.3853678674690451
- 1.111793352200932
--0.007016878293356101
--0.2509982105265462
- 0.7405632811729503
--0.2679354327736296
--0.3935233872615596
- 0.6690609770018828
--0.2697159678407115
--0.418780115297808
- 0.4120920212671315
--0.3822617102277461
--0.4694934670160535
- 0.2416772952251766
--0.2764992133374727
--0.3327147458932168
- 0.007086003287936471
--0.08314533451961652
--0.05915500916687756
--0.02108410666249156
--0.08123465042208253
--0.03458427866845559
--0.01104559642171011
--0.02715566418836943
--0.01970075401570389
--0.01221492520747035
  0
  0
--0.004182834181807198
  0
  0
  0
+ 0.4006144142924327
+-58.8212544358888
+-0.5573049702096061
  0
  0
+ 2.268010772363684
  0
  0
+-4.23775299517115
  0
  0
  0
@@ -1559,8 +1389,13 @@
  0
  0
  0
+-0.7038484816810545
+ 12.42976177283253
  0
+ 0.2598511143545055
+-4.706375917378906
  0
+-1.069792924031905
  0
  0
  0
@@ -1568,21 +1403,34 @@
  0
  0
  0
+ 0.5284818446523596
+-0.2317575652818614
+ 0.6049576045814707
  0
+ 0.0936934825598751
  0
+ 0.6686083357134646
  0
  0
+ 1.14143632433362
+ 0.3503199830054031
+ 12.02281668097733
  0
  0
  0
  0
  0
  0
+ 0.4204897221502805
  0
  0
  0
+ 0.002161286711168726
  0
  0
+-1.823200927539675
+ 0.8488044341158772
+ 0.1650376508635884
  0
  0
  0
@@ -1594,100 +1442,83 @@
  0
  0
  0
+-0.1493032180801644
  0
+ 0.3915997924850128
  0
+-0.3376551130159838
+ 2.451876485774099
  0
+-1.023854922625665
+-0.4969114947662052
  0
  0
  0
--0.008586647879725181
--0.1043746040160939
- 0.7337453112200211
--0.08524890832254815
--0.09157725806465791
- 0.1618409121016899
--0.1360532630982197
- 0.1100730720110193
--0.07223967154837777
--0.00813909272777652
- 0.656002571163308
- 0.08450402846880221
--0.1260499304720408
- 0.8633728414990512
--0.06919246494425754
- 0.05735426662063778
- 1.250405452011807
- 0.06941011798457683
--0.3281666058208408
- 1.033954317926505
- 0.2310481517374129
--0.1119678690565057
- 0.6428638595482884
- 0.1889142834494066
--0.2688892224744744
- 0.3872078268630255
- 0.3956009372076191
--0.318496924172954
- 0.4283497863903946
- 0.4120288148412199
--0.1845584111608842
- 0.2304008514330043
- 0.2768063676751319
--0.156481290145249
- 0.221853719553514
- 0.2081821264553344
--0.08106148318824526
- 0.1909056705806461
- 0.1364519728536071
  0
- 0.1518813587012469
- 0.09217426351956572
+-1.311156244368104
  0
- 0.09937521863646119
- 0.07247926279690005
  0
  0
  0
  0
+ 1.110968516904319
  0
  0
  0
+ 0.5397651888879642
  0
  0
  0
  0
+-2.333073839187608
  0
  0
+-0.07552426206784429
  0
  0
  0
  0
  0
+ 1.631147964603093
  0
  0
  0
+-0.04125382538654197
  0
+ 2.298055590191642
  0
+-4.54636147842368
  0
+ 0.7098816805381168
+ 15.93716172348917
  0
+ 0.9963795629337603
+ 0.1127232320023574
  0
+ 1.132306670131363
  0
  0
+-0.2663574698292075
  0
  0
  0
+ 0.5157919358723543
  0
  0
  0
  0
  0
  0
+ 0.08604829012617518
  0
  0
  0
  0
  0
+ 1.959178936605638
  0
+-0.2193510087756213
+ 0.3442300108980625
  0
  0
  0
@@ -1695,80 +1526,48 @@
  0
  0
  0
+ 1.021071125534755
  0
+ 0.1616977240316401
  0
  0
  0
  0
- 0.02736364077998522
- 0.1505447830470834
- 0.2919011441364225
--0.1785190786675974
- 0.2557899124476684
- 0.4844789082867951
--0.264162179779117
- 0.1219514357167006
- 0.8736931089032107
--0.1432283675429876
- 0.006876932388039775
- 0.8809945387259679
--0.2487471141260569
--0.04433745720435731
- 1.280840429526397
--0.1583166984914722
- 0.06941011798457618
- 1.343742426952984
--0.4157980148590785
- 0.2488222424827422
- 1.125546526553535
--0.5349439256841086
- 0.4122274787833529
- 1.151359305754395
--0.5765248462063814
- 0.5291943984086531
- 0.9059705022382259
--0.4344950435328591
- 0.4813056922292966
- 0.7319151274866598
--0.1158910151709987
- 0.2058682596470976
- 0.1603313623656453
--0.07283221029039584
- 0.1721284970720283
- 0.1335088512738512
--0.03188576105323897
- 0.1177394848145217
- 0.0291078890182462
+-0.0171414009818532
  0
- 0.09841018123171189
- 0.01995229046132583
  0
- 0.02851279632809474
- 0.02025487642521403
  0
  0
  0
+ 1.172187603070537
  0
  0
  0
  0
+ 0.2850175388286725
  0
+ 0.9533132192993321
  0
+ 0.1099414890806674
+-0.3960661468695982
  0
  0
  0
  0
  0
+ 0.124839982277769
  0
  0
  0
  0
  0
  0
+ 1.192359374479515
  0
  0
  0
  0
+ 0.2995921695389488
  0
  0
  0
@@ -1779,81 +1578,55 @@
  0
  0
  0
+-0.6321210847120607
  0
  0
+ 0.03207155215416499
+ 0.09953458596614896
  0
+-0.1534286874362401
  0
+-19.06913382158813
+ 12.669590025082
  0
  0
+-0.7052964622086155
  0
  0
+ 2.120555140213639
  0
+ 0.1013005251328395
  0
  0
+-0.392197925481916
  0
  0
+ 0.1343005612800253
  0
+-0.469051036244502
  0
  0
  0
  0
  0
+-0.5502879597543631
  0
  0
--0.14153889361185
--0.04321574012647134
--0.4628328408146838
- 0.1478503655606767
--0.1722147715711354
--0.3586192331804536
- 0.4349888241298457
--0.0840824531047962
--0.3210461068877711
- 0.379611709010909
--0.2633628403843943
--0.3800978573166322
- 0.8441705033140686
--0.1163475922428522
--0.2975212464546924
- 0.6874901272467245
--0.3115325731332943
--0.3871523286800285
- 1.271753902834939
--0.09263094152560079
--0.2801621242890019
- 0.5634946382796335
--0.1888012589524561
--0.5037348700069839
- 0.4948960949556851
--0.1270868851604219
--0.6238482306812283
- 0.4634960870392485
--0.1870090626447458
--0.2974903341501801
- 0.1484081652916892
- 0.03690599704033456
--0.1044653167655952
- 0.1094868076547757
- 0.0326782154076562
- 0.0313567679189074
- 0.01089871015855057
- 0.08020822792812557
--0.01406774858140106
- 0.01714520464612563
- 0.0655160447080527
  0
--0.007246397200108536
- 0.03720546106832864
- 0.03256630890642381
+-0.3295471965149562
+ 0.1238485530478013
  0
- 0.03860317434701433
- 0.04443899322419608
+ 0.9944594786317867
  0
  0
+ 0.4348181220265078
+ 0.5403324023640561
  0
  0
  0
  0
+ 0.2799417821159156
+-0.4267116644470155
  0
  0
  0
@@ -1868,6 +1641,7 @@
  0
  0
  0
+-0.4639162264180705
  0
  0
  0
@@ -1887,6 +1661,8 @@
  0
  0
  0
+-0.2968261984942903
+ 0.5335368053088048
  0
  0
  0
@@ -1898,66 +1674,28 @@
  0
  0
  0
+-0.2482748823190064
  0
  0
--0.08248869558429915
--0.06344390113652257
- 0.3381025720201776
- 0.03090283727281561
--0.1207675369734728
- 0.1984864361560299
- 0.08690718213108196
- 0.06210759202598081
- 0.1131808783440845
--0.06736683660315472
- 0.3189227728518929
- 0.1425854784492348
- 0.0566150887246646
- 0.6394735026746209
- 0.1258819661040617
--0.139391716119275
- 0.9750205365247968
- 0.1631556139827516
--0.08027883357766392
- 1.199323101350634
- 0.1748841550995589
--0.2264323494499089
- 0.8448921089913984
- 0.05164318429480112
--0.2763978495471368
- 0.659799326894971
- 0.2543762148276674
--0.215977451469077
- 0.5781115119931736
- 0.4237519130831551
--0.1090455242003818
- 0.3670918486607246
- 0.2613765759679356
--0.08520974791008415
- 0.3483011979509406
- 0.2728740774881688
--0.07933235597927898
- 0.2986842473697608
- 0.174981020181665
- 0.01974040893957393
- 0.2345220151283462
- 0.1293567188251034
- 0.004794239864378021
- 0.1388787394529855
- 0.11345116514523
- 0.01016066744437013
- 0.0315821084993036
- 0.04232804490935031
  0
  0
+-3.97041981155859
+-0.1022738659823332
  0
  0
+-0.788896168670442
+ 0.1639749493614056
  0
  0
  0
  0
+-0.5768520561570339
+-4.520986271159193
  0
+-0.2453367041880312
+ 1.441044276131385
  0
+ 0.5254545614648408
  0
  0
  0
@@ -1966,6 +1704,7 @@
  0
  0
  0
+ 0.0766667666262323
  0
  0
  0
@@ -1975,14 +1714,19 @@
  0
  0
  0
+ 4.026465606766232
  0
  0
+ 0.4197329531154022
  0
  0
  0
+ 0.3067694585786604
+ 0.8314371431735575
  0
  0
  0
+-1.591369706346736
  0
  0
  0
@@ -1993,75 +1737,38 @@
  0
  0
  0
+ 0.1259687511468373
  0
  0
  0
+-2.53333376357498
+-0.8602950463753484
+ 1.873620524096339
+-0.6718870234947631
  0
+-2.18839000074766
  0
  0
  0
--0.04262793679452875
- 0.1697740756715866
- 0.3239074915348768
--0.1589955150043625
- 0.2261309862723243
- 0.5007066948691458
--0.214948498176686
- 0.2115410913437121
- 0.7211998935662339
--0.2834845192518015
- 0.1977665592046763
- 0.7105719830712188
--0.2629846125158853
- 0.2427681453222948
- 1.046240369308911
--0.269129398416762
- 0.1781872122967659
- 1.03874274392891
--0.2769693797170344
- 0.1818467501316744
- 1.348877464355057
--0.4710685462064164
- 0.1827804089940346
- 1.298790609075718
--0.489235520201789
- 0.3659945487891929
- 0.9889729466144815
--0.3787569007976286
- 0.3429169426648138
- 0.762384171560255
--0.1020077718117634
- 0.1276450699166033
- 0.1560144816201697
--0.05974206683086447
- 0.1024301574385172
- 0.1101813862095128
--0.01836366264038309
- 0.07641275888249587
--0.01761614764163003
  0
- 0.050498265002489
--0.03382818752642417
--0.01253571506931467
- 0.01296025523973141
--0.005708212878175765
--0.00981215029774337
--0.004490409692104846
--0.01715525507410304
  0
+ 2.692732290771431
  0
  0
  0
  0
+-3.291065831162801
  0
  0
  0
+-0.2329586474529969
  0
  0
  0
  0
  0
  0
+ 0.6135631081273585
  0
  0
  0
@@ -2071,19 +1778,28 @@
  0
  0
  0
+ 0.5614754230564437
+ 3.84506897839564
+ 0.1401832707562678
  0
  0
  0
  0
  0
+-3.972707924920157
  0
  0
+ 0.6897960995450576
  0
  0
+-0.1781238882142194
  0
  0
  0
+-0.750994549808589
  0
+ 0.09100871905332755
+ 0.7426334212224136
  0
  0
  0
@@ -2095,69 +1811,27 @@
  0
  0
  0
+-1.004562434639328
  0
  0
  0
+ 0.3078110601987896
+-2.417182399732592
  0
+ 0.08347068258672727
  0
  0
--0.1242102189239938
--0.006164997849505758
--0.07153461814739005
--0.09444773040108892
--0.387943428575583
--0.09883882221569014
--0.1212536944142565
--0.5443499233026309
- 0.2446342882844404
--0.005420177622913896
--0.45014992349797
- 0.1805459963951284
--0.1149731824589926
--0.5793268006821727
- 0.5943780316125726
--0.07606922043140596
--0.4764497926559779
- 0.5512217345426982
--0.2226561596695373
--0.4692054844105334
- 1.147985092990181
--0.3185462804425448
--0.3501449776185102
- 0.8920211634493967
--0.5449794367210983
--0.2407310953139373
- 0.4723421378334628
--0.1986732196713777
--0.4188640731699343
- 0.1544747855487867
--0.1511790483388062
--0.07192253244542443
--0.003725697090364132
--0.0220913989993934
--0.1468526543626148
--0.02817327979646708
--0.05190672898051355
--0.006325251931142729
--0.07525145457458117
- 0.001697678887603216
--0.003297495636010923
--0.03246162326438269
- 0.01013976787993824
--0.05182845289631759
  0
- 0.01359443233396007
--0.0172570418314328
  0
--0.02362951566813379
--0.06285636906706796
  0
  0
+-0.830317302176479
  0
  0
  0
  0
  0
+ 0.0434567563767247
  0
  0
  0
@@ -2165,11 +1839,15 @@
  0
  0
  0
+ 0.5386706195832376
  0
  0
  0
  0
+-0.6353862693723572
  0
+ 1.975136071000745
+ 2.857816894329415
  0
  0
  0
@@ -2177,94 +1855,56 @@
  0
  0
  0
+ 0.5281194313481445
  0
  0
  0
  0
  0
  0
+-1.479904902831114
  0
  0
  0
  0
  0
+ 0.5122525097374786
  0
  0
  0
  0
+ 0.909151913488855
  0
  0
  0
+ 0.2549383504009166
  0
+ 5.962302297903852
  0
+ 0.1150226024425775
  0
  0
  0
- 0.000541594919091487
--0.06028000599526159
- 0.06056598570524592
- 0.02953064924547113
--0.06923471480436504
--0.01891049181226457
- 0.0268767202425183
--0.01562155165351305
- 0.03650851380746929
--0.00237832052322528
- 0.07073903656649719
- 0.05143750940466155
- 0.0001469834523781262
- 0.2686749881238329
- 0.2116796187687435
--0.06683812836001574
- 0.5455583762437547
- 0.2678847282549483
--0.167042125792995
- 0.8247502624594285
- 0.1551014523158158
--0.3117922666534864
- 1.120619590522661
- 0.2284693806323327
--0.3131000654955992
- 1.03714362591117
- 0.3160604958877395
--0.1596079439909592
- 0.9010620280244863
- 0.4324641730719244
--0.06454934401311917
- 0.679922410035844
- 0.2963524652781443
--0.05129522211593864
- 0.6632663480410756
- 0.3423284178957403
--0.1107026062549143
- 0.5782401777699622
- 0.2616483819084243
- 0.053649067557817
- 0.4944944436523461
- 0.2346875580793441
- 0.009412604728342261
- 0.2663830511196768
- 0.1985776610535804
- 0.09353364223764664
- 0.1321153188119533
- 0.145623535922112
- 0.1356279678672704
- 0.08179006836250678
- 0.1360644250839272
+-1.01380208086729
  0
  0
  0
  0
+ 10.38949477672419
  0
  0
  0
  0
  0
+-0.1001156568581912
  0
  0
+ 0.1232150685288419
  0
  0
  0
+ 0.2799933920423779
+ 0.4745785889503421
  0
  0
  0
@@ -2275,7 +1915,14 @@
  0
  0
  0
+ 4.003537089391028
+ 3.15331633496073
+ 0.8949266268054354
+ 0.1905354497413648
+-0.0006176766367530246
  0
+ 0.6998256194512208
+ 6.866552066980055
  0
  0
  0
@@ -2284,73 +1931,33 @@
  0
  0
  0
+ 0.3247196818172799
  0
+-0.0755298726932041
  0
  0
  0
+ 0.7798741956678945
  0
  0
+ 0.249729711097544
+ 0.5055081102563672
+-1.384472230871766
  0
+-0.8024384886334274
  0
  0
+-1.898073624868862
  0
+ 1.611688473972974
  0
  0
  0
  0
+-0.08014407571295187
+ 0.2564102587943313
  0
  0
- 0.02042759737388438
- 0.14674773697916
- 0.3571674467316939
--0.2151635263421847
- 0.2584936211261787
- 0.4203232824335663
--0.3717874500107469
- 0.1762759358398916
- 0.6311658301753896
--0.2964611356429263
- 0.1909165061130751
- 0.657225660683243
--0.4531867001845071
- 0.1325526769425752
- 0.9883817750068494
--0.2692533998896378
- 0.1150545990637859
- 1.038203565016419
--0.4927170003660811
- 0.03960593138417274
- 1.282143346625596
--0.3445153039163922
- 0.2284693806323329
- 1.405078700186056
--0.3982457865178985
- 0.3676514691999223
- 1.118642037342427
--0.3632353790080362
- 0.4366531833183378
- 0.7381958054401957
--0.09148445746430431
- 0.1757834343565333
- 0.1812147279427315
--0.08259945569807613
- 0.1860477709150544
- 0.08695824951478025
--0.04007203193791484
- 0.1310902263239609
- 0.01224425975682899
--0.01354391461917368
- 0.116064418412012
--0.0026660135676226
--0.01844037342594609
- 0.04677930902538735
--0.001305595979754572
- 0.01605838430327047
- 0.0201649811777727
--0.002160049138104471
- 0.04090210676561704
- 0.01895924652034242
- 0.02537674547112137
  0
  0
  0
@@ -2360,97 +1967,69 @@
  0
  0
  0
+-1.707010158667659
  0
  0
+-4.147202888281389
  0
+ 12.12437272012778
  0
  0
  0
  0
  0
  0
+-0.5048087565002763
  0
  0
  0
+-1.195299323854342
  0
  0
  0
+-5.717476628165155
  0
  0
+-13.07512077460665
  0
  0
  0
  0
+ 0.7365736071424029
+ 1.213005555043753
  0
+ 0.02262669554090805
+ 1.074670827640683
  0
  0
  0
  0
  0
  0
+ 0.3632162325690264
  0
  0
  0
+-0.8456143335581298
+-3.039487674094362
+-0.06850905281137523
  0
  0
  0
  0
+-0.09019236364727108
+ 0.2803226839540793
  0
+ 0.6176066036883436
+-0.6031715112642002
  0
  0
  0
+-0.1918418039784893
  0
+-1.02759238737448
  0
--0.0794175080944723
--0.1311047519457179
--0.01295134131576296
--0.07559746867709317
--0.3031321226577994
--0.0167250101736162
--0.1183792408956641
--0.4927659850693444
- 0.1401190848852171
--0.1237143398676662
--0.4692096403777944
- 0.181418878334746
--0.1981866369376882
--0.5736157357492635
- 0.3698033299851458
--0.2536101970829642
--0.54766228665201
- 0.4892305369514527
--0.2814526191847183
--0.4947047532446273
- 0.8928505090759026
--0.327688421200874
--0.4104621621980526
- 1.172250670502255
--0.2568301162154136
--0.2639273236043994
- 0.9350520824684292
--0.04176686612980885
--0.5277923034840782
- 0.5539539180715677
--0.01357334334667475
--0.2143289852765108
- 0.2814394709243907
- 0.1849574665556867
--0.1895270004545035
- 0.07465277417886505
- 0.03581851864555304
--0.04109727173380481
--0.0243041892802158
- 0.1156761352354747
- 0.02699474300995193
  0
- 0.07672900850356405
--0.02789348757478719
- 0.03521145645185004
- 0.08216413320042737
- 0.05323392699582487
- 0.07677504731300376
- 0.02637277036688178
- 0.009008450568351388
  0
  0
  0
@@ -2458,9 +2037,11 @@
  0
  0
  0
+ 0.6898055200332753
  0
  0
  0
+-0.002090446308802089
  0
  0
  0
@@ -2472,6 +2053,7 @@
  0
  0
  0
+ 0.5036854825681361
  0
  0
  0
@@ -2480,85 +2062,49 @@
  0
  0
  0
+-1.441462578054818
  0
  0
  0
  0
  0
  0
+ 0.04095241481847776
  0
  0
+-4.023403973420407
  0
  0
  0
+-0.0312725987949565
  0
+-0.05170692748138132
  0
  0
  0
+-0.2713606554296591
+-0.2876361587391115
  0
+ 1.092790819907702
  0
  0
+ 0.1073041127787464
  0
  0
--0.002449031357407452
- 0.01063698771912406
--0.03951029141370768
- 0.04462750193803677
  0
- 0.07884910728914267
- 0.08536650861610245
- 0.009475804678243856
- 0.1719868342658235
--0.0592479879569783
--0.05064675381903107
- 0.1270869714173288
- 0.05426840529149266
- 0.1462614491971853
- 0.3931622801149885
--0.1561196057453254
- 0.2771648063729535
- 0.3807812949163497
--0.1168393618182533
- 0.6321531872584453
- 0.351925566307628
--0.5426936855547794
- 1.039259775053663
- 0.3868741132933126
--0.249435243964555
- 1.382629698050187
- 0.164181109541688
- 0.1644820122200089
- 1.041360421163825
- 0.3248672077864385
- 0.2344663252314359
- 0.8490498640002748
- 0.05749267565835372
- 0.2083467709566488
- 0.8280925821901266
- 0.284587855665233
--0.005561163388316134
- 0.6603409125030985
- 0.1756532610454609
- 0.1246051167405868
- 0.5836584482046749
- 0.2436892195356578
- 0.04788885432278017
- 0.303611716253695
- 0.226297349852271
- 0.1299879455460961
- 0.1829834806585196
- 0.219223667683021
- 0.2145616808252643
- 0.1285160975686417
- 0.2223254411952814
+-1.032859866554303
  0
  0
  0
+ 2.396043537578052
+ 0.1729037309633073
+ 0.2517973552198874
  0
  0
  0
  0
  0
+-0.1761251945560189
  0
  0
  0
@@ -2567,19 +2113,25 @@
  0
  0
  0
+ 0.3904653621899105
  0
  0
+-2.285331188687838
  0
  0
  0
  0
+ 0.341166587958904
  0
  0
  0
  0
+-1.317661374659714
+-1.888728240243611
  0
  0
  0
+-0.1590686564583614
  0
  0
  0
@@ -2590,73 +2142,30 @@
  0
  0
  0
+-1.045272475604685
+ 0.3332897234069875
  0
+ 0.2766839566984409
  0
+ 1.491454116190605
  0
+-0.6229288469666586
  0
  0
  0
  0
  0
+-0.5349138460144993
  0
  0
- 0.02564316574837235
- 0.08371517747481154
- 0.3671095605303641
--0.1785310249831468
- 0.1726927191428491
- 0.3358462516059161
--0.3742901407073999
- 0.1400230321552189
- 0.4677158663396579
--0.3003473106705227
- 0.2504525718945521
- 0.5149115656093095
--0.5207969019211087
- 0.225424055730968
- 0.724717977710952
--0.3327583709666753
- 0.3250258488867132
- 0.802156701576207
--0.6121335172994702
- 0.2464418374782392
- 0.9735928182597602
--0.2373303887862142
- 0.3160604958877394
- 1.118642037342427
--0.2516947707803998
- 0.1485335638520558
- 1.326520892579401
--0.4194655541706243
- 0.3882982498164393
- 0.8056399717916394
--0.169328788594763
- 0.07635187279153749
- 0.4080553581712917
--0.2026603906857792
- 0.1524294050383565
- 0.1424308151616571
--0.1029091784701195
- 0.1128569017500071
- 0.1113983791458489
--0.06641747451247863
- 0.1075531590313645
- 0.01672476748665855
--0.04518782745843523
- 0.06231481418275322
--0.01307968723936047
- 0.001484739039221722
- 0.0149712962056828
--0.03882964864888243
- 0.004865321932817059
- 0.002299773128322388
--0.02438294464646183
  0
  0
  0
  0
+ 0.7375191633215031
  0
  0
+ 1.867545208133053
  0
  0
  0
@@ -2669,19 +2178,31 @@
  0
  0
  0
+ 0.8029776017065059
+ 3.276661772557552
  0
  0
  0
+ 0.2585107656953937
+ 0.2248831206011652
  0
  0
  0
  0
+-0.6230664190614899
+ 13.72963719186322
+-0.5371409201423661
+-0.1359288264417897
  0
  0
  0
+-0.3238934751092037
  0
+ 0.1457135237481716
+-0.2274651302089386
  0
  0
+-1.356599501123854
  0
  0
  0
@@ -2699,65 +2220,17 @@
  0
  0
  0
+-1.580531963550888
  0
+ 1.238621220298611
  0
--0.01264389783262389
--0.05793713725717359
  0
--0.01261579684494852
--0.05006292113497802
- 0.0585927585072927
--0.07340538142494876
--0.2427215358456318
- 0.05006634523565472
--0.1627117507788533
--0.2745570272739167
- 0.248306407115128
--0.2015951238560101
--0.3928287134451823
- 0.1994708994320764
--0.3127636264823894
--0.435859056825597
- 0.4806186549419238
--0.2198408442070814
--0.396148696405932
- 0.4692238185331468
--0.1708442202474906
--0.3978663273927521
- 0.930432539854766
- 0.1633731603840306
--0.4518732412182115
- 1.338716501551583
--0.109031231306197
--0.6330931974218327
- 1.03719425959278
- 0.07510046575744928
--0.5609581255009206
- 0.7608218415125024
- 0.2272286665406628
--0.2749021307511318
- 0.3251220532744498
--0.01054680707135332
--0.2404015440217943
- 0.1379036234693202
- 0.1040209196000656
--0.02880779552312983
- 0.09936456905726107
- 0.07028003508995186
--0.005608917362226716
- 0.07066448103953958
- 0.1035993647771786
- 0.1154205139081911
- 0.1300829939152364
- 0.07033727032220777
- 0.0957129161933688
--0.006814261345230066
- 0.01461585745164496
- 0.01685837742394549
+-0.5561997416574285
  0
  0
  0
  0
+-1.197365597246516
  0
  0
  0
@@ -2773,6 +2246,7 @@
  0
  0
  0
+-0.6447357651011012
  0
  0
  0
@@ -2781,14 +2255,19 @@
  0
  0
  0
+-0.2122003804682439
+-1.365058699690163
  0
  0
  0
  0
+ 0.7625308218974119
  0
  0
  0
+-0.1708390348749355
  0
+ 0.7115274676497322
  0
  0
  0
@@ -2796,76 +2275,31 @@
  0
  0
  0
+ 1.642731265592378
+ 0.9061106277271693
  0
+ 0.07950280276876927
  0
  0
  0
- 0.00507231121252326
--0.0334372389620036
--0.04791691334223844
- 0.01170073395245119
--0.02837974723460622
--0.08167688837353176
--0.02096849855094491
--0.01884910557216194
--0.002243854020911905
--0.01702519522643522
--0.003216907348460758
- 0.002354916082877636
--0.04933345379441739
- 0.1424011224620444
- 0.2714259768856127
--0.04444749129920447
- 0.3168241265818476
- 0.3201753443272889
--0.1607803624895273
- 0.5540687088944444
- 0.3088202382126047
--0.1903857556737995
- 0.892707884352013
- 0.4319656203695874
--0.02496391921617681
- 1.025673265149956
- 0.3843395400397565
--0.0957245301338806
- 1.115399867562503
- 0.3250369308001381
--0.0580146160246742
- 0.7725722156722602
- 0.2751758081130411
--0.1189509965679552
- 0.8204677432931591
- 0.2759983002322761
--0.1726312421635642
- 0.671089556740378
- 0.3087886059414974
--0.00732436305052842
- 0.5890268703308188
- 0.2855033144045582
--0.01636585270555224
- 0.3041971618836315
- 0.1841881095436645
- 0.1017556489695763
- 0.1712187687886567
- 0.1698424586463704
- 0.1856870823553884
- 0.09359926680510965
- 0.1429455848671907
- 0.004824792404718843
--0.005658603507934124
--0.007727438637937199
+-0.8161224232435611
  0
  0
+ 0.0777517547926225
  0
  0
  0
  0
+ 3.190560560494386
  0
  0
+ 0.4257272849592115
  0
  0
+ 1.298382971719918
  0
  0
+-0.06693507794509926
  0
  0
  0
@@ -2881,80 +2315,33 @@
  0
  0
  0
+ 6.686058137806026
  0
  0
  0
  0
  0
  0
+ 6.534198600584574
+ 0.4283040837376961
  0
  0
  0
  0
+ 0.7632253630999627
  0
+-2.597117510450818
  0
  0
  0
  0
  0
  0
+-1.44428176956472
  0
  0
--0.008535350870329075
- 0.03436218398211274
- 0.03300232988791075
--0.02027486122495344
- 0.03178501900563035
- 0.08271910037477599
--0.04871758158701333
- 0.09119210773342638
- 0.3041839049927936
--0.105265983370549
- 0.1203613808526095
- 0.3051735990987435
--0.1597839883176589
- 0.2472992998471992
- 0.57717032207967
--0.1816918812299629
- 0.326090566059919
- 0.5986739697440269
--0.2805222441630954
- 0.4044831103778653
- 0.7362684071941031
--0.4076954569175315
- 0.4293152906834561
- 0.7349017313331035
--0.5059247775394392
- 0.3123106119090459
- 0.8021560939257572
--0.6162684314566595
- 0.3250369308001382
- 1.293736911368689
--0.589592335703418
- 0.2064173428704563
- 0.7008293193476045
--0.4120124783025362
- 0.08829825319868181
- 0.6240411861437604
--0.2561903173351185
- 0.3393547819130854
- 0.2489666379681048
--0.008171132063165762
- 0.1989965810118738
- 0.04823495701052179
--0.1179784331019245
- 0.1280089048537619
- 0.1661022791808202
- 0.02743474471261938
- 0.01410851755710262
- 0.0004534935036338368
--0.03657562872520889
- 0.03150917830501523
- 0.09186978877707268
- 0.04874654950035142
--0.009922681089159321
--0.001540662677976581
  0
+-10.17809731785903
  0
  0
  0
@@ -2968,8 +2355,11 @@
  0
  0
  0
+-1.602868955570228
+-1.227778724536436
  0
  0
+ 0.2025456011660076
  0
  0
  0
@@ -2977,16 +2367,23 @@
  0
  0
  0
+ 0.4068888188936426
+ 1.106371853952238
  0
  0
  0
+-2.120467525891301
+ 0.5141030641987713
  0
+ 4.142019259648222
+ 7.273339910131452
  0
  0
  0
  0
  0
  0
+-3.136164028709288
  0
  0
  0
@@ -2996,6 +2393,7 @@
  0
  0
  0
+-1.122811483000477
  0
  0
  0
@@ -3006,61 +2404,19 @@
  0
  0
  0
+-1.01079784158729
+-0.8977837949331151
+ 2.38257492145663
  0
--0.04839081910879701
--0.1533724932760199
  0
--0.1118204749205744
--0.1775596682999272
- 0.1761206679997549
--0.1947787595015865
--0.3530094722364744
- 0.1254834104858047
--0.3028670759341462
--0.3949753342902071
- 0.3155324567704582
--0.2557705668944926
--0.3728770395367336
- 0.3185994351178071
--0.2148313474663342
--0.3770105515802546
- 0.6966402185250694
- 0.06665706026241587
--0.4218132488437272
- 1.155253332564081
--0.2058683201688108
--0.7897293304092443
- 1.369406074416903
- 0.1416940980492992
--0.586057126944088
- 1.119398095610936
- 0.3033668431366793
--0.4494675578834791
- 0.5850225104651052
--0.07562429183869759
--0.3205173600664772
- 0.2847469616478119
- 0.1292144542323355
--0.03878532080699099
- 0.2562539010354006
- 0.08758852186278157
--0.03488152623822612
- 0.1578961302862293
- 0.1589641944723265
- 0.1548821325166659
- 0.277038831532137
- 0.1407671241510433
- 0.1547446621772466
- 0.1295460786819906
- 0.1233035768468524
- 0.1152643564635355
- 0.1478203601705918
- 0.08887262811179811
- 0.09873564049668686
+-2.450953936898679
  0
+-3.804722013238095
  0
  0
  0
+-2.677738173995096
+-1.114097186608559
  0
  0
  0
@@ -3068,24 +2424,36 @@
  0
  0
  0
+ 0.9119876984302767
  0
+-4.547523983843604
  0
  0
  0
+ 0.9856465720838319
  0
+-0.08544754690441386
  0
  0
  0
  0
+-0.7248436876542571
  0
+ 2.083453547864012
  0
  0
+ 2.05621618028899
  0
+-1.806520415415923
+ 0.5240081916689401
+-1.597634193064217
  0
+-0.753583892912139
  0
  0
  0
  0
+ 1.568013441543157
  0
  0
  0
@@ -3093,70 +2461,27 @@
  0
  0
  0
+ 0.411676874911288
  0
+ 542.3095441932002
  0
  0
  0
+-0.2806667362458613
  0
+-0.4596673336940951
  0
  0
+-55.91275459166305
  0
  0
  0
+ 0.2615308601081318
  0
+ 0.01745095050356674
  0
+-1.951103771473351
  0
--0.01902953342090278
- 0.01965841442259662
- 0.1265945697270666
--0.05271710049486282
- 0.0156000894311955
- 0.1195974737452692
--0.03985217833185092
- 0.1458744688078508
- 0.4028504995681779
--0.08453875539682189
- 0.266716064571811
- 0.4273501790758238
--0.2023532170425824
- 0.496196564324543
- 0.4792055362272562
--0.3658521136050059
- 0.8153197991738401
- 0.555192480497449
--0.2298297073024665
- 0.9973792973863952
- 0.4186410884421446
--0.1078548800629369
- 0.9264223390558074
- 0.4655437864680848
- 0.1419039825326285
- 0.9760557505033376
- 0.02348302557898171
- 0.2159329769513006
- 0.8947765913058501
- 0.1695663675719657
- 0.04018824386105823
- 0.760896030546356
- 0.04350966705869708
- 0.2034999769190248
- 0.6619029126435076
- 0.1600087757042891
- 0.07117794424532692
- 0.3332861870356167
- 0.191628537733264
- 0.1878675687767862
- 0.2000059207841889
- 0.212349004592466
- 0.2651969823448151
- 0.151318764645697
- 0.2676484228302623
- 0.117981415875424
- 0.05537361282438801
- 0.07948171773334457
- 0.09794739602244382
- 0.06053278115034896
- 0.09083288071455702
  0
  0
  0
@@ -3165,19 +2490,26 @@
  0
  0
  0
+ 12.8201633486507
  0
+ 0.06986337401001477
  0
  0
  0
  0
  0
+ 0.3880142143477058
  0
  0
  0
  0
  0
+-0.1406845691956707
+ 0.5238083351969455
  0
+ 1.250487214783859
  0
+-1.047394944053435
  0
  0
  0
@@ -3190,8 +2522,10 @@
  0
  0
  0
+-2.100392104780831
  0
  0
+ 0.5306088782791798
  0
  0
  0
@@ -3199,66 +2533,20 @@
  0
  0
  0
+-0.2237997665467135
  0
  0
  0
+-0.7921971919259403
  0
  0
  0
+ 0.1077330002485517
  0
--0.09085332971282412
- 0.05978363834149637
- 0.1976865172651971
--0.09248954946676832
- 0.1310720502943713
- 0.2186767233416529
--0.2219380118863173
- 0.2430475176972734
- 0.4332635647390597
--0.1819976043662894
- 0.3868716088431143
- 0.4851023405236292
--0.3553980578422949
- 0.4375482080027094
- 0.534099102090706
--0.3031934907684789
- 0.4757017083576593
- 0.5888193191986006
--0.4315917860144173
- 0.2545633777336691
- 0.7748629380741596
--0.74056496876128
- 0.4661925117741225
- 0.9831970926278412
--0.5859497207534007
- 0.02348302557898186
- 1.010034188861924
--0.5514792260655391
- 0.07176293729208492
- 0.6288809895411019
--0.3204162216073807
- 0.1955755093766477
- 0.5241038244185691
--0.1519546590028275
- 0.1443463848507031
- 0.2017677476041637
--0.1035429774029542
- 0.1144045941462229
- 0.1394107103279224
--0.004987839176053102
- 0.02784290326117664
--0.01667008654217432
--0.01178931601840801
- 0.02031677497261586
--0.008695668815126454
- 0.02738007463661901
- 0.005161092704699138
--0.02636065386541896
- 0.005232614805629174
- 0.0003673640906699979
--0.02757862720253619
+-1.033306352064682
  0
  0
+-0.6695380972749733
  0
  0
  0
@@ -3273,93 +2561,53 @@
  0
  0
  0
+ 0.8876839636503929
  0
  0
  0
+-1.616700855389318
  0
  0
  0
  0
  0
  0
+-15.66585963240408
  0
  0
  0
  0
  0
  0
+-2.842510394280135
  0
  0
  0
+ 0.9789331801497174
  0
  0
  0
+ 15.4017244676939
  0
+ 1.146598787637898
  0
+-0.2488544716360347
+ 0.335477097389436
  0
  0
  0
  0
  0
+-0.6383860299826728
+-0.4307519605943652
  0
  0
  0
+ 1.02332944709298
  0
  0
  0
--0.001265977516501284
- 0.03841741295073133
--0.04167332592492751
--0.03617242351261972
- 0.01833298368512788
- 0.1292223108262921
--0.134512876634508
--0.2126540722713442
- 0.04367669117746803
--0.2271476182104787
--0.2573014505804312
- 0.2294318389980446
--0.2161703510702842
--0.2468676013148349
- 0.1276389836558024
--0.184624307353196
--0.2934556693105454
- 0.3647325033474627
- 0.05647371273761841
--0.389847489213109
- 0.831701014264612
--0.2467186393092987
--0.5596066617683856
- 1.107939548920055
- 0.2049785850400485
--0.5520003423998263
- 1.349994034068507
- 0.01594024446347822
--0.4037910347393177
- 0.8744131947608462
--0.168312936086927
--0.5246896962887845
- 0.6187434293443713
- 0.0110481573045226
--0.1866475546951414
- 0.4400441847359451
- 0.01636340988356278
- 0.01647118869278042
- 0.2939800564746345
- 0.122925080808936
- 0.1641436371137526
- 0.3784159065625461
- 0.2101487442760755
- 0.2942825807882382
- 0.2708836080892262
- 0.2032320349592514
- 0.1929589394140589
- 0.2493267945655633
- 0.1852512168495825
- 0.2122091500517266
- 0.1011804975007311
- 0.07617270255156854
- 0.06859950746247327
+ 1.808733717244716
  0
  0
  0
@@ -3373,9 +2621,13 @@
  0
  0
  0
+ 0.2269330074689746
+ 3.644798009441442
  0
  0
  0
+ 3.323368896859614
+ 8.432947160682593
  0
  0
  0
@@ -3383,8 +2635,10 @@
  0
  0
  0
+ 26.0543122138953
  0
  0
+-0.3370453272049372
  0
  0
  0
@@ -3392,83 +2646,51 @@
  0
  0
  0
+ 1.792979375879805
  0
+ 0.4356985506684224
+ 1.141915365443236
  0
  0
+-8.885452225220575
  0
+ 0.2329663371220614
  0
  0
  0
  0
+ 0.6453172183777433
  0
+-0.1525216471591352
+-1.51206935745585
+-1409.789566489944
  0
+ 0.3505410578373315
  0
  0
+ 0.3050177145532988
+-0.1350556291701849
  0
  0
--0.04156408495747906
--0.009413625457485824
--0.001908283063673741
--0.03235881826432552
--0.01912073681995256
--0.005161219162513972
--0.03710536463098445
- 0.1147134982393787
- 0.3169170472819148
--0.03477140841636103
- 0.2329182713081895
- 0.3504828100457987
--0.1805411288983153
- 0.4722354378299338
- 0.4149042510315208
--0.2177799641469858
- 0.7860510563573369
- 0.5298793691209585
--0.00146954936280167
- 0.9608602312416559
- 0.4630042454320833
- 0.06438258933790733
- 0.9589069020648562
- 0.323559482727886
- 0.297493086276174
- 0.8883624746781769
- 0.07063402817654904
- 0.01601772212142599
- 1.124250615451674
- 0.05367882220159861
--0.1636647019013005
- 0.8212916398268981
- 0.1808195442953317
--0.0522399159418384
- 0.7571368822850639
- 0.2425011442534682
--0.05416903697572524
- 0.3895979198219028
- 0.1026458763609263
- 0.0887247336091338
- 0.2320358988610352
- 0.1779157731134778
- 0.1931832532340975
- 0.09404431662211243
- 0.1307975875509816
- 0.01598463245592503
- 0.0008938890564368197
- 0.0208553825839959
- 0.0286119173042573
--0.0123545294229296
- 0.003550855019223907
--0.07594704938390318
--0.05555336112393602
--0.04280905561114338
  0
  0
  0
  0
+ 4.624672898948029
+ 0.0794809461888861
+ 1.952800830662676
  0
+ 0.6428929470641352
  0
+-0.9071519888527591
+-0.4334517725109382
  0
+-1.868546803418864
  0
+-31.28972221674916
  0
+ 0.1617582371559916
+-23.34228519031611
  0
  0
  0
@@ -3479,8 +2701,10 @@
  0
  0
  0
+ 0.1013378252601311
  0
  0
+ 0.3791736198855424
  0
  0
  0
@@ -3492,91 +2716,50 @@
  0
  0
  0
+ 2.306185018306861
  0
  0
  0
+ 2.439969276914827
  0
  0
  0
  0
+-1.017845358926976
  0
+-0.8766257649530759
  0
  0
+ 0.6198060744371063
  0
  0
  0
  0
- 0.05280121217906358
- 0.01611164118581453
- 0.0588164594406216
--0.002376085117951147
--0.0001332900161797361
- 0.04477317774386023
--0.02613038538384484
- 0.1529686768209856
- 0.2861874624630084
--0.08453225471387218
- 0.2374998272222094
- 0.300351105386283
--0.1157673564726861
- 0.369339086596075
- 0.3402080156109828
--0.3063546997027202
- 0.4504112135651073
- 0.3542410843144292
--0.3266370098824081
- 0.4062691109762042
- 0.3826169155415374
--0.4006723478592644
- 0.3922022223778848
- 0.8057576920889964
--0.4479905953291173
- 0.1659315626636352
- 0.6263400361692182
--0.4035361818630668
- 0.05367882220159864
- 1.095056112031682
--0.4017927895565523
- 0.2990987608263373
- 0.6330248657581778
--0.1210708033139963
- 0.1591644075315025
- 0.3741413486116172
--0.173983232253032
- 0.1264646757089649
- 0.4272285722412449
--0.006095417404622631
- 0.01966898116992491
- 0.1480953928172449
--0.01034737477959695
- 0.06301598982388827
- 0.1884658178456484
- 0.04735876098662492
--0.003559905775143694
- 0.001406703478946554
- 0.007980584208621525
- 0.02340521899745435
- 0.01084599375532393
- 0.05085265874132012
- 0.02241093161219722
--0.01823643489931556
+-14.54122390283368
  0
  0
+-0.05900668080087645
+-0.8608481304516736
  0
  0
+ 0.9584519544372639
  0
+-0.7192688654953781
  0
  0
  0
  0
+ 0.2230597948668599
  0
  0
  0
  0
+ 0.3731040983273882
  0
  0
  0
  0
+-0.151137228704379
  0
  0
  0
@@ -3589,88 +2772,63 @@
  0
  0
  0
+-50.71934548153453
  0
  0
+ 3.126452264542789
+ 1.132158344845541
  0
+-0.8863528929198792
+-1.260035660258273
  0
  0
+-0.202847099264026
  0
  0
  0
+-19.33863969788858
  0
+ 0.1232261434340604
+-0.2432360087096834
+-2.900892525561199
+-0.2935257595444203
  0
  0
  0
  0
+ 0.7918980316239734
  0
+-0.03017587666352048
  0
  0
+ 0.2295632897630081
  0
+-0.539455340779404
+-0.04261836612506047
  0
  0
  0
  0
  0
  0
- 0.07047834705937615
--0.09178706197740315
--0.1806119380374749
- 0.05456253554578702
--0.1546714601038524
--0.2004609409211657
- 0.1278275884832036
--0.2400102719891815
--0.2003581215661663
- 0.1187267439239399
--0.2684820656957791
--0.2499258598031065
- 0.1484161868230236
--0.1846933219836615
--0.289851037221567
- 0.3952477069065902
--0.3164493254144762
--0.409400953561785
- 0.5865325480397021
- 0.004528568266388114
--0.3442701615241253
- 0.8860126061705046
--0.1804651557630956
--0.4170074581586109
- 0.9715111115018301
--0.09930961048312786
--0.4138630715505275
- 0.8992944725994563
- 0.005479947954454054
--0.2127794659124976
- 0.6845921001123547
--0.0328318429313325
- 0.01014330942520348
- 0.5028789222182671
- 0.1079018464744724
- 0.1238393787781744
- 0.6047787355178867
- 0.2833830324963177
- 0.3568437547234754
- 0.5181889214216189
- 0.3287812859570573
- 0.2760945251530408
- 0.4539800982125043
- 0.3167270060101302
- 0.3397138539933505
- 0.3442626432166768
- 0.2472769794305177
- 0.2111906874721716
  0
  0
  0
  0
  0
+ 0.5057226779971445
+-0.4651331674899222
+ 0.3562731727024678
+ 0.4220906535398433
  0
  0
  0
  0
  0
+-0.2237781519202437
+ 4.044868941894766
  0
+ 1.146134109648113
  0
  0
  0
@@ -3680,27 +2838,36 @@
  0
  0
  0
+ 0.1099205728262046
  0
  0
  0
+-0.1172635807505568
+-5.169188607650955
  0
  0
  0
  0
  0
+ 0.5290078414157005
  0
  0
  0
  0
  0
+-0.1694184955169638
  0
  0
  0
+ 0.09428222406861927
  0
  0
+-2.340675973451509
  0
+ 2.003271014808549
  0
  0
+-0.04067697176415717
  0
  0
  0
@@ -3709,92 +2876,65 @@
  0
  0
  0
+-0.1945619307040214
  0
  0
  0
--0.04298876770760442
- 0.1656470384699898
- 0.3904131316702136
--0.03876741165644142
- 0.2882441359656608
- 0.4262032947733492
--0.2501579815982157
- 0.507824039754495
- 0.5635436308488173
--0.333240829143735
- 0.7833772983512389
- 0.6385678788801107
--0.2549717350477951
- 0.88020440469228
- 0.5721652738467851
--0.2629926430688403
- 0.8889896402117655
- 0.6931102887066188
--0.1223631714145813
- 0.8179976769420301
- 0.256493081680243
--0.1966193768492383
- 0.8778998191378252
- 0.3466950312014125
--0.09930961048312804
- 1.166101233710299
--0.1022447850215565
- 0.349388866526547
- 0.8863972041398483
--0.1091549209839937
--0.01509858598318699
- 0.4585686208661375
- 0.1649514882150246
- 0.2876760850561434
- 0.2113634488339027
- 0.1088910294240312
- 0.2169181726250745
- 0.1723118323520176
- 0.3450819124993047
- 0.2043784438760465
- 0.02838233140458319
- 0.07296110985661074
- 0.1015756094393812
- 0.08590034136615242
- 0.174655909313396
- 0.09030187281811035
- 0.01721589616120555
- 0.04171456418284304
  0
+ 0.04424807833576058
  0
+ 0.3873983960135145
  0
  0
  0
  0
  0
  0
+ 9.501309606987693
  0
+-1.065701617640357
  0
  0
  0
+-0.3761764396197418
  0
+-2.760565873221028
  0
  0
+-0.06407669225881736
  0
+ 0.6954083896979436
+-0.1211157828422568
  0
  0
+ 1.28056896227939
  0
  0
  0
+-2.09853157883936
  0
  0
+ 14.1865481906395
  0
  0
  0
  0
+-0.2356606444263802
+ 2.108683011342688
+ 0.6105938534629533
  0
  0
  0
  0
+ 0.7913328821246927
  0
  0
+ 1.068829806203787
  0
  0
+ 0.8299173235166551
+ 0.01235111812086723
+-0.82768062659
  0
  0
  0
@@ -3803,7 +2943,11 @@
  0
  0
  0
+-5.720237772032373
+ 0.9379845676334215
  0
+-0.1543293607798029
+ 8.116947803426786
  0
  0
  0
@@ -3812,63 +2956,18 @@
  0
  0
  0
--0.0664571702085886
- 0.1160844668278462
- 0.2007015135261183
--0.05901834347390401
- 0.2192033654981966
- 0.2349169686261112
--0.10994791091446
- 0.2867397539468095
- 0.1645276200973362
--0.145593858161492
- 0.3613346796038783
- 0.2324676783500399
--0.1691474485829494
- 0.2762574066522671
- 0.3145470737627009
--0.3493995457132565
- 0.4111865560286941
- 0.4128247715750158
--0.3475729498472833
- 0.06491935023780998
- 0.5538388461817163
--0.5395113709675047
- 0.2050751790009213
- 0.6544238059103082
--0.4137940267940007
--0.1018910180244136
- 1.155143704743835
--0.4282431662149232
--0.01153586971065928
- 0.8586971293827956
--0.0004571308712585015
--0.009554151875950522
- 0.3384747347116007
--0.08653918059664382
- 0.03973832550075845
- 0.2123053236210994
- 0.1773577029439647
- 0.02446852649066788
--0.05114228946083501
- 0.02439128398862156
- 0.08029086587355641
- 0.01096301783063296
- 0.1078027163665591
- 0.0126108751553812
--0.1057352043449342
- 0.05438455361609832
- 0.0859524006153511
  0
  0
  0
  0
  0
+ 0.5402320287781748
  0
  0
  0
  0
  0
+ 0.9462562549611485
  0
  0
  0
@@ -3876,13 +2975,21 @@
  0
  0
  0
+ 3.605026999972954
+ 3.411480067968635
  0
+ 1.06255084451497
+-7.515831098707253
  0
+ 0.05142596470185394
  0
+ 0.06005195016876662
  0
  0
  0
+-33.83369201179862
  0
+ 0.8047221152599965
  0
  0
  0
@@ -3891,81 +2998,53 @@
  0
  0
  0
+-1.34918003604524
+ 1.230615596733249
  0
+ 0.10979142632346
  0
+ 0.8822449650583322
+ 0.6958841073665485
  0
  0
+-0.479429441083319
  0
  0
  0
+ 2.915848331189997
+-0.997872836697216
  0
  0
+ 3.263639045218301
  0
+-0.01478244422614588
+-0.284994270492743
+-0.6130636084263046
  0
+-0.6018543320135842
  0
  0
+-0.588598019619618
+ 0.05684034822488467
+-0.03336069582699805
  0
  0
  0
+-0.07024435978230432
  0
  0
  0
  0
+-49.35641065012955
  0
- 0.0001699056108734376
  0
  0
--0.004845422436498823
  0
+-0.7054576595914014
+ 1.617963110749013
+-2.576357596878969
  0
- 0.03399893669537405
--0.1056916697545231
  0
- 4.7667677835957e-05
--0.08770235130704315
--0.05334950106615528
--0.06397520539438785
--0.04215911776267812
--0.101808841511791
- 0.1009373800403707
--0.1313086962195549
--0.05695079254041767
- 0.2245086168773425
- 0.1212501317311327
--0.1606046239137854
- 0.5491881176714973
--0.09492975913657767
--0.1391286460416374
- 0.796856859568104
- 0.2782919869846736
--0.4415554563133402
- 1.338507026803915
--0.006101728093102918
--0.3694143305002582
- 0.8419031866141915
--0.1362069043714237
- 0.1538130050320617
- 0.7521759108703118
--0.03252378404421097
- 0.06236571915920732
- 0.724096331771299
- 0.3186576822930837
- 0.5399178516334415
- 0.6909952386757811
- 0.3272039314646736
- 0.2734923805563407
- 0.5140286515670152
- 0.4069854291422868
- 0.460730325508649
- 0.4811926834678659
- 0.3164939498209386
- 0.2524609503865199
- 0.03348305747586157
- 0.01006239810646883
- 0.06114029164162799
- 0.05998271611024859
- 0.001338483124748731
- 0.04912046494745304
  0
  0
  0
@@ -3973,6 +3052,7 @@
  0
  0
  0
+ 3.653806778054516
  0
  0
  0
@@ -3980,6 +3060,7 @@
  0
  0
  0
+-0.4162968482907972
  0
  0
  0
@@ -3990,21 +3071,29 @@
  0
  0
  0
+-0.4814608464156824
  0
+ 0.04550265498495074
  0
  0
  0
  0
  0
+-3.085196207941759
  0
  0
  0
  0
+-0.09064188528540848
+-0.7649917791956307
  0
  0
  0
+ 0.279797907572765
+-9.348341835984616
  0
  0
+-0.3281683959664261
  0
  0
  0
@@ -4012,81 +3101,43 @@
  0
  0
  0
+ 0.1328475559642141
  0
- 0.1085209650644184
- 0.2646503201001987
  0
- 0.1893213711214356
- 0.2879498444872255
--0.2009640081238166
- 0.4281626342432941
- 0.4371005594569758
--0.2497621878560982
- 0.6582663112126301
- 0.5121264071906843
--0.1198711906948955
- 0.7609765910572841
- 0.4690654412758804
--0.1002461739395601
- 0.7597955489260516
- 0.4848546445494995
- 0.08179759019639699
- 0.6962877284141019
- 0.1932658811817052
--0.02601846651379103
- 0.7928230325256173
- 0.1956867648120076
--0.01974060903410628
- 0.8703654432154465
--0.01153007544255585
- 0.006890590757748697
- 0.9202205051060298
--0.06717963058617565
--0.1445989570500295
- 0.575840729915004
--0.01296695494868298
- 0.1196604190466994
- 0.3359278697242619
- 0.07123512734906096
- 0.1111616962883617
- 0.1505492904973262
- 0.1541501267023
- 0.08306503260253775
- 0.02635720620932545
- 0.04028263096832157
- 0.04508749306113925
- 0.01264880321662966
- 0.07283055569823375
--0.008432530220948482
--0.03960507746621952
- 0.006995478512006049
--0.02999149844018921
  0
--0.03567874045648146
--0.04545898708063957
  0
--0.02861614831903463
  0
+-1.949932391098605
  0
  0
  0
+ 0.1374794555695837
  0
  0
  0
+-3.091451623378846
  0
+-0.07764732804850226
  0
  0
+-2.434024165245729
+ 0.5500645539322153
  0
  0
  0
+-0.5079753852948696
  0
+-0.5529859236827438
  0
  0
  0
+ 0.2015881697347675
  0
+ 0.2579673115713021
  0
  0
  0
+ 0.2600781412163046
  0
  0
  0
@@ -4094,90 +3145,58 @@
  0
  0
  0
+-0.8522602730893853
+ 0.2733801271893656
  0
  0
+ 0.6998551871651082
  0
  0
+-1.11552369486787
+-0.5084088295849895
  0
  0
  0
  0
  0
+ 0.3404993295374376
  0
+-0.5911308977441611
  0
  0
  0
+-0.6417675201399902
  0
  0
  0
  0
  0
  0
- 0.04538962554952197
- 0.107675357276117
  0
- 0.111020873424774
- 0.1300026401557825
+-0.9005153291632695
  0
- 0.1528508858404347
- 0.01930116286217853
--0.09065371405532635
- 0.2523975010440672
- 0.09691874330793647
--0.04556300805076537
- 0.2533479249342087
- 0.1125150892678504
--0.09161080357497643
- 0.3026773580806075
- 0.1258730297624595
--0.07210696085809201
- 0.1504064575730214
- 0.2160374141783687
--0.210229843258113
- 0.2421341169464289
- 0.3840374135064942
--0.2407508953585975
--0.1212009403905005
- 0.8450783671315125
--0.3686232122822629
--0.0730486425349951
- 1.199841296675042
- 0.05627145492703167
--0.1502797204523957
- 0.6439265534811736
--0.1539626700947748
--0.01572214341397699
- 0.590138327794575
- 0.2769255751858083
- 0.02277585740376173
- 0.1974274506083979
- 0.02719029413563755
- 0.1191379909843359
- 0.186067932597198
- 0.1968526304773224
- 0.05577548221950061
--0.0346332570234083
- 0.07862022965309619
- 0.1332554542835316
- 0.04151166294257791
- 0.07065376766691027
- 0.01330490650104134
--0.001424895750481332
- 0.05467880139161527
- 0.03265791488082924
  0
+-1.562914129206301
  0
+ 1.141381319782258
+ 3.700306510854067
  0
  0
+ 0.4915320792043331
  0
+-0.6203587452825666
+-0.2413972162031621
  0
+-15.0488841819318
  0
  0
  0
  0
  0
+ 0.9224970963993336
  0
  0
+ 0.03554728629349915
  0
  0
  0
@@ -4189,83 +3208,45 @@
  0
  0
  0
+ 2.7953185796338
  0
+-0.1305284316301769
  0
  0
+-0.3507941553620811
  0
  0
  0
  0
  0
  0
+ 1.700791840747238
+-0.4547107736140834
  0
  0
  0
+ 0.2844474796749299
  0
  0
  0
  0
  0
  0
+ 0.2577043505012904
  0
  0
+-37.9205099652173
  0
  0
  0
  0
+ 1.643176969120298
  0
  0
--0.002430880387288702
+-3.442089160337546
  0
  0
- 0.06454060691221508
--0.1292188585389263
--0.1004557555017825
- 0.07956807420147674
--0.163476278549359
--0.1471784757546287
  0
--0.155581094189555
--0.1552122239158223
- 0.111763290253138
--0.1686926129688248
--0.2120500253743202
- 0.2077599741049015
--0.02875513373862118
--0.1286681257596934
- 0.3881560288067106
--0.1132014361728064
--0.20343337531731
- 0.5988616393792288
--0.08326724808269938
--0.0283286815398887
- 0.84189417184656
--0.1679450780630685
- 0.05210781861366457
- 1.053934307197499
--0.2149148452640063
--0.2188116524965337
- 0.6247584856736758
- 0.0726125804710796
--0.02726755463868002
- 0.9091944714062363
- 0.2614490815361376
- 0.04573100887074542
- 0.617990864224173
- 0.4321554257708766
- 0.2190480994108906
- 0.5990112445827741
- 0.3346493942569108
- 0.2201763503838938
- 0.4323478926743816
- 0.3753685693181499
- 0.2903969145719552
- 0.1130964268164823
- 0.01599698482811726
- 0.06237844831106225
- 0.08358413439215608
- 0.05437458637583435
- 0.09450120627341284
  0
  0
  0
@@ -4278,8 +3259,11 @@
  0
  0
  0
+ 0.3685738200902941
  0
+-1.191014929440095
  0
+ 34.17961336962895
  0
  0
  0
@@ -4288,26 +3272,41 @@
  0
  0
  0
+ 65.78378332041297
  0
  0
  0
+-1.209185515163843
  0
+-0.1564896293723537
  0
+ 0.5388282285966564
  0
+ 1.314037662228941
+-1.7449328203312
+ 0.1496937119721336
  0
  0
  0
  0
+-0.2451416154803616
  0
  0
+-1.469872392683623
  0
  0
  0
+ 3.151159804730952
  0
  0
  0
  0
+ 0.05959525184340657
  0
+ 1.117510162860467
+-0.1044855073737362
+ 0.0122456496221879
+-2.607625983237394
  0
  0
  0
@@ -4315,58 +3314,11 @@
  0
  0
  0
+-10.34941174115949
+ 0.8967368490486325
  0
- 0.00385948709995182
- 0.01102549788002184
--0.1011736071615975
- 0.2151046886620052
- 0.2118138107836209
--0.110362891478852
- 0.3138744827558007
- 0.2577026943498817
--0.03269933606194544
- 0.3545843845759463
- 0.250317993917445
--0.04688563857617225
- 0.374334603075552
- 0.2728148247678721
- 0.04454045659904322
- 0.3309847399048657
- 0.1392600549785949
--0.02435385942559752
- 0.3992693326882689
- 0.1417330484974973
--0.07617530889133121
- 0.433779724645776
--0.01262688887782614
--0.1355960721621477
- 0.5764119807149598
--0.150204486574641
--0.2115590438374359
- 0.8227805803246482
- 0.03565526401580574
- 0.09959669123005294
- 0.6055311575571114
--0.06323542097002967
- 0.03237569588621741
- 0.4044293691814657
- 0.1409239382676068
- 0.2169644685830129
- 0.2262215144232321
- 0.03053394200815901
- 0.1098233393981265
- 0.1416087735957518
- 0.1459160346128377
- 0.1637332794437391
- 0.05389734031672933
- 0.05637226947255175
- 0.08983151834421278
- 0.07929666985483162
- 0.1093773294191723
- 0.1183986603232397
- 0.04125733401916568
- 0.0810053017584091
  0
+-0.1929814802386746
  0
  0
  0
@@ -4381,96 +3333,64 @@
  0
  0
  0
+-0.1009095097079969
  0
  0
  0
  0
  0
+-0.274255218503406
  0
  0
+-1.604126413868497
  0
  0
  0
  0
+ 5.100859334768233
  0
+-1.013938327312231
  0
+ 8.692407638247211
+ 3.428578974872518
  0
  0
  0
  0
+ 3.228719022055706
  0
+-0.3617959480624034
  0
  0
+-0.5215164558164206
+ 150.1666332247709
  0
  0
  0
+ 0.03031841122373824
  0
  0
  0
  0
+ 37.73823907986427
  0
  0
  0
+-2.652614939130339
  0
  0
  0
  0
  0
- 0.03357859734504264
- 0.01757022011177168
- 0.003466294787839106
- 0.09012183619175529
- 0.04498644338178364
--0.1270881823980927
- 0.1986035380639484
- 0.1050948660621651
--0.08652273754340772
- 0.2182676091372718
- 0.08708086379790275
--0.08401719911772791
- 0.1859975288705404
- 0.2260281556458444
--0.07157284011235411
- 0.1789343395376638
- 0.1544717121816253
--0.02227166602906977
- 0.1027072284127585
- 0.4374506104438563
--0.03738097746780947
- 0.13824317104179
- 0.3357125090881536
- 0.1595183493815381
--0.01345327289208848
- 0.6431489891510632
--0.2204083970959405
- 0.04012221821402379
- 1.131173049424324
--0.001547524760757846
--0.03626361392210729
- 0.6537740384547847
- 0.02970973521581357
- 0.1997024288685219
- 0.8039367998884733
- 0.1796254752049998
- 0.07839428679467762
- 0.2546391258434929
- 0.1286535968751069
- 0.2027236752481019
- 0.2939620048875812
- 0.2175859097828638
- 0.1067480338313124
- 0.02043925297491477
- 0.0345177236688381
- 0.04682865105230121
- 0.06561290556047709
- 0.1057925742925759
--0.007824032709891963
+ 0.04166819568493204
  0
+ 0.6341729050087129
  0
  0
  0
  0
  0
+ 12.91755519939191
  0
  0
  0
@@ -4481,17 +3401,23 @@
  0
  0
  0
+ 0.05197119971391739
+-2.009655228073888
+-0.2684851004861274
+ 0.9814531190789848
  0
  0
  0
  0
  0
  0
+ 6.165396362398449
  0
  0
  0
  0
  0
+ 0.6579664638625362
  0
  0
  0
@@ -4504,8 +3430,11 @@
  0
  0
  0
+-1.609027750727198
  0
+-0.2530273342511696
  0
+ 1
  0
  0
  0
@@ -4517,98 +3446,69 @@
  0
  0
  0
+ 2.245974095225942
+-2.290949007689282
  0
+ 0.2973402786019308
  0
- 0.000502391620047372
- 0.03876819663916783
+ 4.932880860261193
  0
- 0.04010299772346695
- 0.03741174531763849
--0.05682410902506225
- 0.05552716396535357
- 0.02495393276570052
--0.01329593994229536
- 0.03869325690384304
- 0.05288230309136639
- 0.07839257873930496
- 0.1411697975875877
- 0.01213158323838526
- 0.2050742917246025
- 0.07311102212931213
- 0.004699870741759046
- 0.3783075488527717
- 0.2364397064377172
--0.08623781279439267
- 0.7241398117343247
- 0.1223190911164313
--0.1444531337831893
- 0.5992605856935499
- 0.09641554967922984
--0.006368029179466047
- 0.8561087888882241
--0.02789752018723617
--0.1406959768195478
- 0.7941927096812281
- 0.1751902243149739
- 0.3187749850732784
- 0.7557548161844081
- 0.1725075338558075
- 0.1166314730395731
- 0.5338221869728753
- 0.2986534005313589
- 0.362922199615957
- 0.4819691275051016
- 0.2537080265098239
- 0.1941432443278794
- 0.1040307738022147
- 0.08100174390035271
- 0.1500731817934467
- 0.1425662561266299
- 0.05166344353127891
- 0.09853516106283801
- 0.02595958692636278
- 0.01503903900873669
- 0.05984287335593675
  0
  0
+ 0.2928714732373925
  0
  0
  0
  0
  0
+-0.4675944151435886
  0
+ 602.89717581152
  0
  0
  0
+-0.7281191587822087
  0
  0
  0
  0
+ 5.153170124854154
  0
+-0.5123852401406641
  0
  0
  0
+-0.1687009411677568
+ 9.537578806713723
  0
  0
  0
  0
  0
  0
+-10.33485045262229
  0
  0
+ 0.01934872541733585
+-25.76730925047929
  0
  0
+ 1.361219151604215
  0
  0
+-1.116170521769804
  0
+ 1.800949273495845
  0
  0
+-0.0421006732079543
  0
  0
  0
  0
  0
  0
+-0.1803091143140475
  0
  0
  0
@@ -4618,69 +3518,30 @@
  0
  0
  0
--0.05189900256332961
- 0.1287215894679653
- 0.1184074643896142
--0.05800677777040057
- 0.1716248024134196
- 0.1302582241231164
--0.002738623158752699
- 0.2012206066581327
- 0.1275021243846876
- 0.01756256547024469
- 0.2263675274348328
- 0.114790620481475
- 0.09847980583555489
- 0.196553234845746
- 0.06311811843547423
- 0.07085555819409999
- 0.2470033972931183
- 0.04163497264003076
- 0.04783646143460571
- 0.2074216786174097
- 0.04670323668927027
--0.04772698785693049
- 0.3487901810855591
--0.007852099124050351
- 0.05943531125856037
- 0.6096430338580853
--0.03381564629962561
--0.03083635370846946
- 0.6898366202010748
--0.06752850136627633
- 0.08570459413546069
- 0.5214766738523933
--0.0535196593937517
- 0.1623117700791414
- 0.4282550764717329
- 0.04221342789562775
- 0.1437264067358977
- 0.1979746554503337
- 0.05089487784798567
- 0.1643564892624744
- 0.1451469641075834
- 0.1000567460984291
- 0.09621898399373256
- 0.04630248372237988
- 0.06921294798292373
- 0.09899229269744673
- 0.03807295978199458
- 0.08684454228003274
--0.01489086855714524
+-0.3253613664587537
+-0.009215314014496436
  0
--0.02999980104815088
  0
  0
  0
+-0.5081353632205899
+ 0.7435051959204299
+-0.1608909576166679
  0
  0
  0
+-0.09973737425051146
+-0.4127920551071492
  0
  0
  0
+-0.5187614628318575
  0
+ 0.08412796197652496
  0
+ 0.02373324620914603
  0
+-2.065695977833619
  0
  0
  0
@@ -4704,98 +3565,64 @@
  0
  0
  0
+ 1.746843346031052
  0
  0
+ 0.07234842936872278
  0
  0
  0
+ 12.98711339268042
  0
  0
+-0.8327248767664689
  0
  0
  0
+-1.304034691999358
  0
  0
  0
  0
--0.007075195322054841
- 0.02738735802773061
--0.01733476427916956
--0.07964262769454435
- 0.1396552230003691
- 0.05926499270543043
--0.0124322540564469
- 0.1774823010979733
- 0.03884015940899863
- 0.03933965076011223
- 0.1438670087138226
- 0.04127192103064878
- 0.09877135869400641
- 0.2100403618493151
- 0.0175839514134552
- 0.11366423864029
- 0.1822319353402183
- 0.1656083357545077
- 0.06180512795436149
- 0.1007214298370139
- 0.2125030722543799
- 0.05868687236796716
- 0.08101439906305402
- 0.5959287411961318
--0.03019991302685349
--0.05608966975373837
- 0.6590642178954187
--0.1444314859509232
--0.06705199105552033
- 1.132775562530928
- 0.2291602937990059
--0.08413284959657472
- 0.8464007681568506
- 0.006415080853831236
- 0.001705210503621362
- 0.6220054582715917
- 0.2502210105932713
- 0.05739069933816202
- 0.3016084166295029
- 0.07957038912786035
- 0.1093860447751527
- 0.1893505693532784
- 0.1002111933069782
- 0.01280701817027201
- 0.003780374164554648
- 0.05297034512980268
- 0.05327073070199739
  0
- 0.05809859641459913
--0.007718466434749396
  0
  0
  0
  0
  0
  0
+ 0.3158695360166293
  0
  0
+-0.3064826996869727
  0
  0
  0
+ 0.3264531986312823
  0
  0
  0
  0
  0
+-0.948388578906028
  0
  0
+-2.31353448980116
  0
  0
  0
+-8.389335067261657
  0
+-0.9057254001297296
  0
  0
  0
+-0.3537752724027108
  0
+ 0.05041777327414723
  0
  0
+ 6.584194758138024
  0
  0
  0
@@ -4803,13 +3630,21 @@
  0
  0
  0
+ 0.5301036759676395
  0
  0
+-0.1796234893658151
  0
+-82.07160959215015
+ 0.9269006307298486
  0
+-0.2749138667511867
  0
+ 0.3305303060241508
  0
+ 0.9472033389611979
  0
+ 0.0004586254447066839
  0
  0
  0
@@ -4820,65 +3655,15 @@
  0
  0
  0
+-1.876684153816676
  0
  0
- 0.01548737248812964
--0.006256316910406804
--0.01766452893334486
- 0.02601443039933989
--0.01141011042074277
- 0.0153112257785901
- 0.03584114746026149
--0.02393866766908594
- 0.08723380862035857
- 0.08705159189360627
--0.002224284169062851
- 0.1498089053989104
- 0.08625699187094234
--0.02654418169243006
- 0.2735142696755783
- 0.05866609619823041
- 0.1205344509541377
- 0.4353095262476621
- 0.02584384186749775
- 0.206570253352424
- 0.6348045371416045
--0.07896376175673758
--0.07117512932599609
- 0.5463912306157316
--0.03199165789435739
- 0.1198077969977105
- 1.161441590407976
- 0.0813318026957859
--0.1770399721456406
- 0.6706248448529601
- 0.3424286663160775
- 0.09823052791434819
- 0.7377361371850927
- 0.234786907587239
--0.01372008184180629
- 0.4387882315534741
- 0.3865142228958408
- 0.2240134945194698
- 0.2426054559749924
- 0.05490494407309857
- 0.02046701572660201
- 0.1593539093187775
- 0.1389423956389921
- 0.157014142313091
- 0.1101250207754176
- 0.004835609033077067
- 0.05852183642727603
- 0.08188734813336776
- 0.008606837565868072
- 0.03784996162062325
- 0.03694595663118083
- 0.02388848376320487
- 0.02686588913184961
  0
  0
  0
  0
+ 0.7082223740804381
+ 219.7111859928096
  0
  0
  0
@@ -4888,93 +3673,62 @@
  0
  0
  0
+ 1.232904486380742
  0
+ 3.955415075927579
+-0.5904487530897623
  0
  0
+-6.987361287626547
  0
  0
  0
+-0.4174540658481559
+ 0.4890908934687008
  0
  0
  0
  0
+-3.056270919776234
  0
  0
  0
+-0.01669853102136176
  0
  0
+ 0.6034720457213459
  0
+ 0.9237654194086627
+ 0.0357838423722235
  0
+-0.01058585067925166
+-0.39037337588858
+ 0.6964320196044104
+-0.1288030817416514
  0
  0
  0
  0
+ 0.5539395094188051
  0
  0
  0
+ 0.4162228479499686
+-8.715412755845247
+ 4.655536481113301
  0
+ 0.008871884562407466
  0
  0
  0
+-0.01441495808858097
  0
  0
+ 0.1106769325196798
+-0.03422990431458062
  0
  0
  0
--0.002844274986378605
--0.00188993012563363
- 0.007738809779907678
- 0.007591977958832679
--0.006065401752788766
- 0.009938725357020313
- 0.004739655543235359
- 0.04271992245037386
- 0.03867029196142101
- 0.006531392625821528
- 0.003339269564115307
- 0.01883202564494896
- 0.0454416792865907
- 0.02336531206556094
- 0.0366282963654051
- 0.04291183056202293
- 0.04116233208144585
- 0.00101844246155105
- 0.09184711420209318
- 0.08630246183606134
--0.02570264048004044
- 0.05330648805999988
- 0.3146548779410448
- 0.1021551393888595
--0.009846203091272253
- 0.435472810441282
--0.161506729908956
- 0.08018365727695444
- 1.001215192569035
- 0.09359832618602337
- 0.2820144790759239
- 0.8754264561707625
--0.1086810328846377
- 0.1132605184805106
- 0.5927284183845087
- 0.08085067602115915
- 0.3867833525246124
- 0.4024844086036321
- 0.05833743060812938
- 0.1457877632318591
- 0.1648253938231204
- 0.1685258044833242
- 0.2716797203445901
- 0.06803924330202783
- 0.1422836890139411
- 0.1085831682369393
- 0.05503839237163627
- 0.1019338909307314
- 0.1394152478505082
- 0.06818513443803768
- 0.04055897799669581
- 0.09392200512087777
- 0.02337336949809378
- 0.009686731587931619
  0
  0
  0
@@ -4983,6 +3737,7 @@
  0
  0
  0
+ 0.0963709151375082
  0
  0
  0
@@ -4991,6 +3746,7 @@
  0
  0
  0
+ 0.6403290165668492
  0
  0
  0
@@ -4999,6 +3755,7 @@
  0
  0
  0
+-0.06534584460111437
  0
  0
  0
@@ -5011,8 +3768,10 @@
  0
  0
  0
+ 1.242470294670795
  0
  0
+ 0.4513265813109331
  0
  0
  0
@@ -5020,166 +3779,86 @@
  0
  0
  0
+ 6.26455324966978
  0
--0.06783193840490404
- 0.129577989223027
- 0.08345162017492355
--0.05119333463445028
- 0.1645939824183541
- 0.05719929501774401
- 0.003422138668432176
- 0.1070377833082572
- 0.1108085172124105
- 0.06779728706460261
- 0.2304846242685355
- 0.02574180195632471
- 0.1620172548007228
- 0.1247791594087073
- 0.1961383811404037
- 0.1648294251860192
- 0.2672877541612674
--0.034393843652069
- 0.3903060449090177
- 0.1453632782979608
- 0.1839257150088484
--0.0881614787627971
- 0.09606777396035704
- 0.7422094976712401
- 0.1949453288291831
--0.1020166327860796
- 0.8052337001634613
--0.1800941292092771
- 0.09359832618602326
- 1.44421799508394
- 0.1325504766499633
--0.124162675286915
- 0.62716933965087
- 0.02305466536222749
- 0.1937323830642648
- 0.743556607901823
- 0.1726401646265668
- 0.00243459822989726
- 0.1547275333142177
--0.03797284511449882
- 0.06889670828931663
- 0.1777119521700858
- 0.07347703114765824
--0.03458270083885566
--0.04897397584694433
--0.006465196303931979
- 0.01886736357608327
- 0.03214342683901935
--0.05741383601396096
- 0.008568494127223819
--0.03393388442068972
--0.01844153602843652
--0.01842870152401585
--0.02863851742975522
  0
  0
  0
  0
  0
  0
+-5.788749463935768
  0
  0
  0
  0
+ 0.2479831626428473
  0
+-0.1579424199805311
+-0.4050262624552801
  0
  0
+ 0.1615823852871679
  0
  0
+ 0.05708742947241674
  0
+ 2.063244692768679
  0
  0
  0
+ 1.787123785044046
+ 3.480412896111618
+-1.090884040621945
  0
  0
  0
  0
  0
+-0.6307949013629948
+ 0.7085196209530149
  0
+ 0.06958179356518558
+ 0.5663128212347056
  0
  0
  0
  0
  0
+ 0.6991291777479244
+ 5.119831962698358
  0
  0
  0
  0
  0
  0
+ 0.1285654986482149
  0
  0
  0
  0
+ 0.5365078826409398
  0
+-0.3120053728133674
+-4.668198793561861
  0
  0
+ 0.009461634588692909
  0
+ 1.368372431442259
  0
  0
  0
  0
+ 5.390334257178868
+-5.166687097847665
+ 1.11785039509098
  0
+ 1.083071602025184
  0
  0
  0
--0.02554469844355501
- 0.01093192619465817
- 0.03263513747779486
- 0.02280288764845734
- 0.09017486063392247
- 0.035894417551042
- 0.05260363896847221
- 0.07521148092628134
- 0.05312964043478007
- 0.1265187816160211
- 0.1382466401751607
- 0.01472936406164443
- 0.3486737496769006
- 0.1212279369519054
--0.001144349249707701
- 0.2901614257391781
- 0.1457747348783731
- 0.06949616881542359
- 0.4847489616807867
- 0.0698750482649092
--0.07330029993295929
- 0.6618905017578627
- 0.2708603721743124
- 0.1638877412459054
- 1.005053477595991
- 0.06809129543204773
--0.004187962474546293
- 0.7058441500038859
- 0.1576551090737128
- 0.3333629761755108
- 0.6488534780213168
- 0.1361053050385518
- 0.1428753500433556
- 0.274915166860708
- 0.1002769657690162
- 0.2431804298510564
- 0.3014929491629826
- 0.07085313692086945
- 0.1480467413205259
- 0.1410596576101275
- 0.09183441001980457
- 0.1755913638111835
- 0.1174185904423381
- 0.1074742386075741
- 0.04091262321196795
- 0.09325512455723538
- 0.03400061154270452
- 0.02254680678432682
- 0.0495775949969639
- 0.05039492715638241
- 0.001338535593531882
- 0.04721851103231973
- 0.01479416264985075
  0
  0
  0
@@ -5196,6 +3875,7 @@
  0
  0
  0
+ 1.735155644270017
  0
  0
  0
@@ -5203,103 +3883,78 @@
  0
  0
  0
+ 0.2936589374981971
  0
  0
  0
+-8.426052593557538
  0
  0
  0
  0
+-0.5195776095653704
  0
  0
+-5.148133256963701
+-1.330135598232709
  0
+-0.3089028100362089
  0
  0
  0
  0
+-1.104083242407448
+-0.04490339340203948
  0
  0
  0
  0
  0
  0
+-5.294652116349429
+ 1.679224457856224
  0
+-3.290894143570063
  0
  0
  0
- 0.02719085066971012
- 0.05039040521446116
- 0.003069326800424954
- 0.03829703597379716
--0.005988614088832385
--0.004305540622714232
- 0.09418394149417418
- 0.02431955618130308
--0.01080134870398796
- 0.1168309705283516
--0.01074946836358617
- 0.05885424620590181
- 0.1305585507719302
- 0.02699263806741721
- 0.08988321139318869
- 0.2382441698402682
- 0.1792966249737566
- 0.01882767696749361
- 0.01595988235750899
- 0.3688787610865161
--0.05265026825612562
- 0.35699815423812
- 0.88287566503563
--0.09933848239951538
- 0.09723732952363413
- 1.045416501622531
--0.05140177661392307
- 0.1361542442909363
- 0.6466494403347459
--0.1169684256893111
- 0.2757756573702421
- 0.5542643308925466
- 0.1063918631621967
- 0.1358956492751874
- 0.1452414681993598
- 0.04778434550561733
- 0.2012134793972644
- 0.1116623666045253
- 0.1535833144092785
- 0.1014559797302281
- 0.009317896682828012
- 0.04402775324976108
- 0.1298005541460214
- 0.01694100505231139
- 0.04183312190060656
- 0.06852278025977013
- 0.02040820771576903
- 0.01314608838117919
--0.02127114799750985
--0.02472711777334497
  0
--0.0227161756279232
--0.005409324727473289
  0
  0
+ 0.3890749762819402
  0
  0
  0
+ 0.56924912365979
  0
  0
  0
  0
+-0.8030827361913189
  0
  0
  0
  0
+ 30.86277274861568
  0
+-0.3398319673253343
  0
+-1.082868037605125
+ 0.5136061678322544
+ 0.1554647180658579
+ 0.3655265200143383
+-1.716250939567565
  0
+ 0.8634319462190599
+-2.910130310413088
  0
  0
+-0.1588162077829562
  0
  0
+-0.1664590718275648
+-0.9777961710251044
+-1.372045042252581
  0
  0
  0
@@ -5309,6 +3964,7 @@
  0
  0
  0
+-1.480309092581737
  0
  0
  0
@@ -5323,76 +3979,32 @@
  0
  0
  0
+-3.058032400395982
  0
+ 0.412187606704143
  0
  0
  0
- 0.03359567242628882
- 0.0009416906988700777
--0.03301721531828659
- 0.08722351073027727
- 0.1256464758223607
--0.01402953052340551
- 0.1138125029679342
- 0.08618612671757166
- 0.01601099956517457
- 0.09541907061710526
- 0.07447224605501546
- 0.01005259271384147
- 0.1433900074217745
- 0.09239845346131699
- 0.1797950906641503
- 0.08478135391631929
- 0.02257006789980576
- 0.2466568960084592
- 0.01818868273767285
- 0.01693962433124771
- 0.5958923530485694
- 0.1162042056026564
--0.1003079024749304
- 0.6579539990509564
--0.001923924440046667
--0.0594363392914588
- 1.052041173101753
- 0.2432922166141848
--0.1248474394877834
- 0.7305152105314757
--0.008677432188295912
--0.02091702330619525
- 0.6220984119226953
- 0.1230393123724863
--0.03954570066228136
- 0.2699228490102467
--0.00375649745536433
- 0.04650962316355828
- 0.1949238000983556
- 0.08366138428735601
--0.002196128160013387
- 0.04181013311074262
- 0.0446667113514202
- 0.0009845415427509263
--0.008661884794539915
--0.007403207049832998
- 0.04724851925862013
--0.01622937854974075
- 0.05885015592345631
  0
--0.02653233251812281
  0
  0
--0.0313074328233286
  0
  0
+ 2.431230191709969
  0
  0
  0
  0
+ 0.1962122215985665
  0
  0
  0
  0
  0
  0
+ 2.381168023501953
+ 0.345316073883669
+ 0.1478247275791482
  0
  0
  0
@@ -5401,11 +4013,16 @@
  0
  0
  0
+ 0.5428631983297003
+-3.985532383422589
  0
+ 0.2761110408253554
  0
  0
  0
+ 1.26683255652973
  0
+ 1.970861260534998
  0
  0
  0
@@ -5418,94 +4035,65 @@
  0
  0
  0
+ 54.94854029156146
  0
  0
  0
+ 0.1094666893077373
+ 0.5977085566998216
  0
+ 1.147521819902617
+ 1.102300705123151
+-0.5638834550146347
  0
  0
  0
  0
+ 0.5224691566975731
  0
  0
  0
  0
- 0.04443532576353969
- 0.07987153936466021
- 0.0133144908537148
- 0.06164600267163708
- 0.08890593628674756
- 0.02347393420801132
- 0.1056967008524369
- 0.0786163543230274
- 0.0674238465963372
- 0.2149843653471525
- 0.08638930540713126
- 0.1615018982691128
- 0.2985822266575806
- 0.05786774619143967
- 0.04454312992528011
- 0.2827374360702383
- 0.0547697773496413
- 0.1695079076212276
- 0.7231832705709893
- 0.09820842399807266
- 0.04937925227258344
- 0.7091669859631984
- 0.1072346971559732
- 0.2391228009461467
- 0.9524070828574946
- 0.03256140082895305
--0.07264633325348854
- 0.55080682690334
- 0.2533353853705229
- 0.1018126264342643
- 0.445369858677944
- 0.03310284519518947
--0.08835940604004454
- 0.2967259102815245
- 0.1678079210559351
- 0.1120530835601668
- 0.2178500932454669
- 0.02583670347915453
- 0.04125982180005652
- 0.1940027694837068
- 0.07113400998985361
- 0.02632739649220729
- 0.1307089447874926
- 0.07643676101022792
- 0.01005116657762971
- 0.1051753821762121
- 0.05630278943435253
  0
- 0.0727682356843497
- 0.05967919799796139
--0.02671286238206237
+ 0.9170826350335268
+-1.192903940179488
  0
  0
  0
  0
+-0.72405612721989
  0
  0
  0
+-0.4943184072537858
  0
+-1.645398441249078
  0
+-0.01183162605145718
  0
  0
  0
  0
  0
+-2.445782008272828
  0
  0
  0
  0
  0
  0
+ 1.371977203329604
+-0.6237686085114352
  0
  0
  0
+ 4.896294638225456
  0
  0
+-10.50101090239492
+-0.2186035533671767
+ 0.2534126808841761
+ 0.5499854069795571
  0
  0
  0
@@ -5516,71 +4104,28 @@
  0
  0
  0
+-0.3877747613890682
  0
  0
  0
+ 2.85659521023488
  0
  0
+ 0.4744709679304063
  0
  0
  0
  0
  0
  0
+-0.03521565215270906
  0
  0
  0
--0.01624572879463861
--0.02735457553214491
--0.005388627837351252
- 0.05973791180183613
- 0.001801183835953749
- 0.020652900860863
- 0.08324318030025669
- 0.02539044818442687
- 0.004710846938777422
- 0.2016208916291706
- 0.0003640686112277541
- 0.02859546678308026
- 0.1355774517421767
- 0.08952512916281873
- 0.1209517671115299
- 0.1323404098083534
- 0.1368539961899758
- 0.0005781157360242165
- 0.2521512796015071
- 0.6046856600277906
- 0.2189095993748628
- 0.2006652488781268
- 0.6530815661382141
--0.1135984551911603
- 0.06935643654433438
- 1.00532787199056
- 0.03535648364861795
- 0.3471011939069352
- 0.7952080265797263
--0.1570396501954827
- 0.01168043310260427
- 0.4251950406084689
- 0.03298793335876931
- 0.2914671585570769
- 0.2255720806488196
- 0.03549319748342191
- 0.09873387907567742
- 0.09450430253559645
- 0.1036872476143574
- 0.1578440663132309
- 0.06114867060485678
- 0.0401418043332418
- 0.1149408895057952
- 0.02154756827161327
  0
- 0.04063956890685241
--0.0001247623525372813
  0
- 0.03197869231522771
- 0.01763249728380177
--0.01565430430043834
+-0.03761895482880985
+ 0.5030174792937585
  0
  0
  0
@@ -5596,10 +4141,13 @@
  0
  0
  0
+-0.7649682274545092
  0
  0
+-0.6314170877432317
  0
  0
+ 1.414638374891361
  0
  0
  0
@@ -5607,20 +4155,27 @@
  0
  0
  0
+-0.9802015092735455
+ 0.6106684731414381
  0
  0
  0
  0
  0
  0
+-83.68089381156405
  0
  0
+-0.5000248436886791
  0
  0
  0
  0
+-0.2917034400370256
  0
+ 0.02833986746370941
  0
+-0.02198728912753643
  0
  0
  0
@@ -5628,63 +4183,23 @@
  0
  0
  0
+-1.037430618094088
  0
+-0.6271655935409088
  0
- 0.05220692012735471
- 0.1253788605357762
- 0.01229466546377449
- 0.09079581850175844
- 0.05892721307849445
- 0.0491210503746056
- 0.104632982227878
- 0.1368473250615786
--0.06015497188019826
- 0.2930897309447529
- 0.1278547450344229
--0.03123375722562478
- 0.0566230375063962
- 0.1240616644336497
- 0.2696773159006211
- 0.2555387844416537
- 0.02758017598526624
- 0.2806335400603563
- 0.02597370083105147
- 0.1115742319417055
- 0.7955976300775014
- 0.3572605617794724
--0.1113305855428926
- 0.7358120792042415
--0.05214475270774152
- 0.03668653008576597
- 1.254781381222356
- 0.127994052522152
--0.1615571479698871
- 0.682748798013776
--0.07657979485747604
- 0.05491073694347617
- 0.7011625939312749
- 0.01825866907591683
--0.06681456339805611
- 0.2513558963933228
- 0.002842422286866354
- 0.08880671609258875
- 0.2244979107681655
--0.03423249329050257
- 0.05632343773017986
- 0.01351592283849615
--0.04344709806894392
+-4.48442327538069
+-0.9411955094116317
  0
- 0.0002735438450783366
+-1.352367282837229
  0
+ 0.09489073595255178
  0
--0.02551307852177572
--0.02537277202089864
--0.05133185997823112
--0.003098647262690865
  0
  0
+-1.071999794444188
  0
  0
+ 0.5806143921209845
  0
  0
  0
@@ -5700,17 +4215,23 @@
  0
  0
  0
+ 10.19197372794769
  0
+-0.2976378075102782
  0
+-0.06131668053255376
  0
  0
  0
  0
  0
+ 0.01518827655417578
  0
  0
+ 0.7986434496062542
  0
  0
+-0.03367291104978264
  0
  0
  0
@@ -5719,8 +4240,14 @@
  0
  0
  0
+-0.06159434527439328
+-4.48056600905538
+-0.4783667106655162
  0
+ 0.006856996042966583
  0
+-0.2760056254784384
+ 1.187423511285556
  0
  0
  0
@@ -5728,62 +4255,14 @@
  0
  0
  0
+-0.583636276826058
  0
+-2.824780882837426
  0
  0
  0
  0
- 0.002999268739130802
- 0.01323048343016893
- 0.03097662882462609
- 0.01957871134796739
- 0.03783404040762516
- 0.003539791561315337
- 0.1674458338494071
- 0.07030321649356418
- 0.02976044388683035
- 0.1109069667492609
- 0.1014659947163603
- 0.08216976133492207
- 0.2236817865781759
- 0.08302883631038403
- 0.01248251944768962
- 0.4757412668439323
- 0.4012393432199489
- 0.2415110584065674
- 0.7113721312990908
- 0.2811316997552983
- 0.02658793970352381
- 0.6041342499741148
- 0.3369271848283455
- 0.1419634675013042
- 0.9601304554553071
- 0.100983226394245
--0.005865955587796385
- 0.514252672301677
--0.003027593411506199
- 0.2457352071059557
- 0.5727051175042291
--0.05883341501980788
- 0.1127159735222335
- 0.3395445366044306
- 0.09742602547335345
- 0.2769490405800476
- 0.2615990573783599
- 0.2036001411839041
- 0.05319003934855725
- 0.2285654230469713
- 0.03948048163846814
- 0.01226500713979149
- 0.1147379547652279
- 0.1025572216529628
  0
- 0.117744458865304
- 0.04330552873553968
--0.05973413307192126
- 0.03294391440895734
- 0.04868910947746517
--0.03406133765767734
  0
  0
  0
@@ -5793,23 +4272,34 @@
  0
  0
  0
+ 1.341150956270354
  0
  0
  0
  0
+-0.305214348732556
  0
+-1.460547106503598
  0
  0
+ 0.03979194706420293
  0
  0
  0
+-24.43565266627769
+-8.731850316765762
  0
+ 0.68467785353886
  0
  0
  0
  0
+ 0.6510141304728614
+-0.9656320399240599
+-0.9428798816256519
  0
  0
+ 0.7666305900589855
  0
  0
  0
@@ -5822,87 +4312,49 @@
  0
  0
  0
+-0.9160052618132869
  0
  0
+-2.574692290497155
  0
+ 0.01734075019931757
+ 0.08169627984271563
  0
  0
+ 0.1816372145101804
  0
  0
  0
  0
  0
  0
- 0.079438940220537
- 0.03507773355349241
- 0.005415152370955367
- 0.1129211517608453
- 0.02615209301099764
- 0.04551620548637231
- 0.1937946780520654
--0.003942514568841414
- 0.1076259235729761
- 0.2391440824264427
- 0.04693419032649682
- 0.06853665531242351
- 0.1381995010699643
- 0.1176433114407272
- 0.07550222283765198
- 0.415864251506007
- 0.4153786154608805
- 0.04606105087996236
- 0.1786389759294371
- 0.5649587853986004
- 0.004163438568025271
- 0.2895516423334366
- 0.7942479411513815
--0.1437920941410486
- 0.1009832263942448
- 0.9102231757316858
--0.03147865701739942
- 0.001352361556547442
- 0.4341978203221831
--0.1389024765201054
- 0.1294875464652517
- 0.3721282619202069
- 0.09418580962100823
- 0.05367029646388479
- 0.04891810797132594
--0.0134197760949454
- 0.1428802322141913
--0.01721146247314263
- 0.04320651234409259
- 0.06075985079125625
- 0.04982287850053625
- 0.01287079476422323
- 0.0427022475021921
--0.02356580305379023
  0
- 0.002317309702867352
- 0.03098975383448471
- 0.003071777146847332
--0.01879743840385957
--0.02826806021171154
- 0.01716112459159353
  0
  0
+-0.3318538701646884
  0
  0
  0
  0
  0
+-0.6682147255182748
  0
+-1.820104118158794
  0
  0
+-0.443970510083133
+-2.826135182952575
  0
  0
  0
+ 1.702319917467265
  0
  0
  0
  0
  0
  0
+-0.3213122755903199
  0
  0
  0
@@ -5910,6 +4362,7 @@
  0
  0
  0
+-1.927500521753644
  0
  0
  0
@@ -5922,84 +4375,52 @@
  0
  0
  0
+ 5.411455074560193
+-0.2342183531798989
  0
  0
  0
  0
  0
+-0.6318248474477726
  0
  0
  0
  0
  0
+-0.1769519223283303
+ 1.821838312750941
+ 5.224111206540555
  0
- 0.008692154897039589
--0.007202037451541151
--0.03630768997496613
--0.007183576537927175
--0.03680673853316485
+-0.01964347025582856
  0
- 0.04690237874870501
- 0.02936919933211182
- 0.02448140162092663
- 0.1006740129810938
- 0.003144777181998707
- 0.01710537269905281
- 0.06345066720220653
- 0.06160486506885934
- 0.1453034093664816
- 0.3029975792341374
- 0.1000124643530668
- 0.2146140225117773
- 0.200341431317792
- 0.1401893519329894
- 0.6564756469208477
- 0.1619605079455435
--0.1505903240220231
- 0.7085839460119159
- 0.01123885279133603
--0.01939884048686782
- 1.337805882791866
- 0.1544141455550043
--0.1646631978092379
- 0.9385684144724438
--0.07650868372102781
--0.01326276584079039
- 0.8161746787054426
- 0.1812990293450166
- 0.009986092402126789
- 0.382915770202775
- 0.1107386144479993
- 0.003246143315376855
- 0.1841345072967435
--0.07919364067533503
- 0.0296682494809771
- 0.1356434127687079
  0
+ 0.01152253189111746
  0
- 0.009165905033973215
--0.1054779771495195
--0.0476328411044768
- 0.01981822023362148
--0.04831804109343124
--0.03110514781220774
- 0.0009437336067497957
  0
+-0.2274561707001618
+ 0.852190804066514
  0
  0
  0
+ 0.758036792984357
  0
  0
  0
  0
  0
+ 1.258717191131011
  0
  0
+ 0.9246362091225441
  0
  0
+-1.654708278897577
+-0.8131999456747966
  0
  0
  0
+-1.264601729396739
  0
  0
  0
@@ -6012,6 +4433,7 @@
  0
  0
  0
+-0.04789157230466286
  0
  0
  0
@@ -6019,9 +4441,12 @@
  0
  0
  0
+-3.117351241839914
  0
  0
+ 0.0724003112956593
  0
+ 1.18695192246064
  0
  0
  0
@@ -6030,74 +4455,44 @@
  0
  0
  0
+ 0.4020776578494258
  0
  0
  0
+ 0.01876909665613024
  0
+-0.3447187017682403
  0
  0
  0
  0
  0
- 0.09157794961322638
- 0.04552403767991522
- 0.08301970314630944
- 0.1261595837220865
- 0.05017685277129666
- 0.02632516756527461
- 0.09460463889830099
- 0.06318522766956154
- 0.104797775015939
- 0.4627293331006117
- 0.2274053953948121
- 0.1083934116897574
- 0.4896219062755338
- 0.2422904944011025
- 0.2285137224955471
- 0.6357808913245143
- 0.1133618741619285
- 0.03475126970816138
- 0.6549649110910292
- 0.1094948693541356
- 0.2236573295074248
- 0.7552606158887939
--0.128385235158096
--0.1401579396572372
- 0.5296801163074781
- 0.06105477046768913
- 0.03771564878079885
- 0.4440031397010384
--0.01780954695783034
--0.01939578623755676
- 0.3966991621601017
- 0.1333360605970911
--0.07799568175789308
- 0.2957397591753587
- 0.1565410931515668
--0.1108610600518473
- 0.2460760425241152
- 0.1597246545719099
--0.08874082911100292
- 0.1855908033273366
- 0.1656452068494594
--0.1322777529016692
- 0.1399312304596164
- 0.1184973242261769
--0.07510263492719821
- 0.1112634359128789
- 0.100048306294601
--0.006573942478301531
  0
  0
  0
  0
  0
+ 24.27177041070852
+ 0.04543245448876172
  0
  0
+-1.642622336416251
+-0.1489725394229043
+-0.9639190351065976
+ 3.932904465769799
+-0.3936251935545277
  0
+ 0.3050672664198578
+ 0.6657382075375293
+ 0.1706300753569768
+ 0.6309672243517473
  0
  0
+ 0.6638182119807017
+-0.7678884039489431
+ 5.527532594305081
  0
+ 0.06516542548503328
  0
  0
  0
@@ -6114,17 +4509,22 @@
  0
  0
  0
+-0.1079001471518526
  0
  0
  0
  0
  0
  0
+-0.6390072184953827
  0
  0
+ 0.08230627887095554
  0
+ 0.2524038344166655
  0
  0
+ 0.5901264848880631
  0
  0
  0
@@ -6139,59 +4539,17 @@
  0
  0
  0
- 0.1708283735131918
+-1.592932141628379
  0
- 0.03084266122920517
- 0.1222787486485877
- 0.05116937041229231
- 0.08374035806868128
- 0.1447751689317241
- 0.03700001207238644
- 0.04374488378930881
- 0.2159563796563574
- 0.2265500651337879
- 0.1969255485162078
- 0.2584285474079688
- 0.2191670239103262
- 0.03583522255544845
- 0.1629698984801329
- 0.5082227271618474
- 0.1521520144867269
- 0.1058810339123703
- 0.5129812368297004
--0.1111604965895913
--0.1229912018376137
- 0.7965867676866533
- 0.01327840202944289
- 0.1018792065650133
- 0.6201750289874151
--0.09747480532566091
--0.09438279372380087
- 0.3059316766167737
- 0.003892881252238821
- 0.1097149363450821
- 0.1412091349493155
--0.0314688123575766
- 0.1015824543477299
- 0.0999937643982426
--0.05252011010158934
- 0.121178267867465
- 0.02182586869951869
--0.04751090163830349
- 0.09136850420447547
- 0.05170251431820073
--0.04234802553344709
- 0.07554589467207116
- 0.0117487281910596
--0.01922473944012164
- 0.06476651504095
- 0.03705780053141568
--0.03784493576771158
  0
+-0.2002107320659327
  0
  0
+ 0.49217988619839
  0
+-0.9770889058731814
  0
+-0.3761865859180392
  0
  0
  0
@@ -6203,6 +4561,7 @@
  0
  0
  0
+ 0.3809423449100742
  0
  0
  0
@@ -6210,8 +4569,10 @@
  0
  0
  0
+-0.1148424525478181
  0
  0
+-46.09452490846677
  0
  0
  0
@@ -6220,12 +4581,21 @@
  0
  0
  0
+ 0.01712053051836848
+-0.1140391791814996
+-0.3815727396385838
  0
+-0.2773350156950293
+-21.66070748961982
  0
  0
+ 0.4876125814851835
  0
  0
+ 0.143318774708148
+-2.11160798315818
  0
+ 0.5083569444320168
  0
  0
  0
@@ -6239,83 +4609,49 @@
  0
  0
  0
- 0.1265347592415239
- 0.05425906569598938
--0.05238025674499402
- 0.06126230652541365
- 0.06121239681667477
- 0.08165039135881287
- 0.2154898586468702
- 0.07810142557599957
- 0.01535738202146116
- 0.305281451035008
- 0.3174990530958363
- 0.3846070198598388
- 0.4804505130402825
- 0.1808561682370068
- 0.386306851096104
- 0.107942447064052
- 0.1343048699829552
- 0.8456987187667494
- 0.3975052382619448
--0.03630781259826539
- 1.027193595964667
--0.1435536134149933
- 0.005627202359721035
- 1.463171117983109
--0.003649852196285947
--0.164327513865978
- 0.9260584302479572
- 0.06092937459733453
- 0.1701144844353229
- 0.7866476689512349
--0.04902214802098032
- 0.106848259234997
- 0.3594214540719029
--0.1556749774938593
--0.1184068458396076
- 0.2975187963964289
--0.1613923208125013
--0.06802324585917598
- 0.1206512851112414
--0.1866821807964986
--0.2027372577020332
- 0.1088607965439639
--0.1809126562060383
--0.1167201576823752
- 0.0495238261074523
--0.1382587520633114
--0.1253891235267075
  0
  0
  0
  0
  0
  0
+-1.144156151339128
  0
  0
+ 0.2262120526582969
  0
  0
+-0.1280217046158115
  0
  0
  0
  0
  0
+ 2.538586369849767
  0
  0
+ 0.1714805058385587
  0
  0
+ 0.06412106785558182
  0
  0
  0
  0
  0
+ 3.404923219073047
+ 0.2047409565802319
+-0.4045499586070819
  0
  0
  0
  0
+-0.4556019264659864
+-2.949453895052602
  0
+ 1.348940523513324
  0
+-12.26457385106148
  0
  0
  0
@@ -6325,88 +4661,55 @@
  0
  0
  0
+-0.4620615090745949
  0
  0
  0
  0
  0
  0
+-0.6401980174732465
  0
  0
  0
+-3.719508623116252
  0
  0
  0
  0
  0
- 0.06803339458393921
- 0.02422844465526069
- 0.01856191782771133
- 0.02066586144105373
- 0.03815942725243979
- 0.02125088896650612
- 0.09576580272538603
- 0.03591858329991948
- 0.03892906397732858
- 0.3882234849360811
- 0.361566910837868
- 0.2474765077134578
- 0.5410027484593787
- 0.3116429187193587
- 0.1134366714859339
- 0.4981761023535856
- 0.3971112756003632
- 0.1408675275058989
- 0.7270243253435944
- 0.2445407296646462
- 0.004520077091457604
- 0.5224128737576819
- 0.09042195845428597
- 0.006740736561519788
- 0.9001069562712208
--0.1641388208757147
--0.0512512779644147
- 0.5846459873751356
--0.03841450824275358
- 0.2340464266398725
- 0.4715652862257556
- 0.1551404467557803
--0.01517038230725697
- 0.3940288479974343
- 0.04171286462181721
--0.0678132892709727
- 0.2446181568485518
- 0.142959164693537
--0.07991247141075764
- 0.2109211831088837
- 0.1089783215667557
--0.132676806026182
- 0.1180470124832955
- 0.1233547270083578
--0.07835812928047305
- 0.098122860146846
- 0.07854563920195906
--0.03513306311768607
- 0.02028419195678683
- 0.0323674302530685
--0.006481089338727093
  0
+ 1.391054589611359
+-1.060120782195125
+ 0.124489228101295
  0
+-1.277807772817144
+-0.3785078509987684
+ 1.615512736160592
  0
+ 2.983426606619536
+ 0.2498211090350095
  0
  0
  0
  0
  0
  0
+-0.4494010090887177
  0
  0
+-0.553629785237019
+-0.4910801410284669
  0
+ 0.1414210220266133
+ 1.327731171059285
  0
+ 0.3299007143983764
  0
  0
  0
  0
+ 0.4610314657365753
  0
  0
  0
@@ -6414,7 +4717,10 @@
  0
  0
  0
+ 0.3605192714167013
+ 0.1574398355427645
  0
+ 0.4414981133992893
  0
  0
  0
@@ -6424,76 +4730,34 @@
  0
  0
  0
+ 0.604519617646983
  0
  0
  0
  0
+ 15.71616248102042
  0
  0
  0
  0
  0
+-0.8570158402800939
  0
  0
  0
+-0.2308523535528031
  0
+ 0.9312076455997745
  0
  0
- 0.1541133646853098
  0
- 0.07845830170999107
- 0.1819620588456227
- 0.04247435967856406
- 0.07055157078365093
- 0.1129105167724161
- 0.04700766089124378
- 0.08215822150761465
- 0.2754759425319723
- 0.1157118145174139
- 0.05779103839311615
- 0.1982486933477656
- 0.1755805777095331
- 0.1069082798396495
- 0.2763836947052577
- 0.2903506956979493
- 0.01428449203683548
- 0.02712793495177207
- 0.4366403329755467
- 0.02985029070275028
- 0.06008857786350736
- 0.6175020127242437
--0.1547988819435056
--0.1586622806738245
- 0.8168791731264279
- 0.03092276609661944
--0.1657238703204828
- 0.3020909672918872
--0.1562158421033308
- 0.08098432672061398
- 0.1085519928743939
--0.02631409595175094
- 0.02831985990382107
- 0.1871791246690147
--0.03789199430926578
- 0.1250414531275982
- 0.02958052084297675
--0.03148712248256605
- 0.05827454660478151
- 0.09095447707992224
--0.02565181951488819
- 0.08074967223916892
--0.0008170243079767171
--0.01211335566312875
- 0.06112736897509567
- 0.04697456393388257
--0.01261955834096525
--0.02691099548112465
--0.03297168302641813
- 0.001470861292652247
  0
  0
  0
+ 0.1142824710512821
+-0.3016200134040187
  0
+-1.504735389983186
  0
  0
  0
@@ -6504,24 +4768,32 @@
  0
  0
  0
+-0.1818099598120365
  0
  0
  0
  0
  0
  0
+-1.307131931736971
  0
  0
  0
  0
  0
+-0.03280698962104209
+ 0.1534141657150162
  0
  0
+-0.355929941480607
  0
+ 2.303379083974311
  0
  0
  0
  0
+-0.05997064613475289
+ 0.1434539202411265
  0
  0
  0
@@ -6530,107 +4802,88 @@
  0
  0
  0
+ 1.335357227039381
  0
+ 0.9608986316800635
+ 1.454978763436731
  0
+-0.3595861940762782
  0
+ 0.5215739104603703
  0
  0
  0
  0
  0
+-5.116953504081511
  0
--0.03688148131567702
--0.0267720987685314
  0
- 0.03639904792240925
--0.03398460946321153
+-0.542440371743971
  0
- 0.06468020612079307
- 0.04697448170028686
  0
- 0.4480803490342942
- 0.2529712419053997
- 0.1020273446687904
- 0.3619686594467425
- 0.2819406116005387
- 0.324421163878916
- 0.3145312388812192
- 0.08082430611975532
- 0.3577872266298659
- 0.2431881495190129
- 0.1927445149338224
- 0.9023802817074618
- 0.05422244979104625
--0.1008984446828327
- 0.9087493574072456
--0.05883815836097019
- 0.03276024258417103
- 1.435086927019949
- 0.2231804417766227
--0.0362526639981683
- 0.911127878026996
- 0.05210079115333111
--0.08651075890227146
- 0.7566679792686167
--0.2471114755381569
--0.1140522830154616
- 0.6702863053203578
--0.2111480676839372
--0.1397589324741141
- 0.3293420789741844
--0.3555969078505137
--0.2475816429754485
- 0.2762236302320865
--0.3203076784390017
--0.2256455665351642
- 0.1191916143121416
--0.2926627301113837
--0.2076324179190413
- 0.07214354189492099
--0.1662501171877677
--0.125109935273284
  0
+ 3.378092631477278
  0
  0
  0
+-3.092927332443712
  0
+-1.499543838081713
  0
  0
  0
  0
  0
  0
+ 4.244285915322096
  0
+ 1.294317179651072
  0
  0
  0
  0
+-1.920793020928811
  0
  0
+-0.05796143522638493
  0
  0
+ 0.7718124813687072
+ 0.378543256984148
+ 0.9407199345199999
  0
  0
  0
  0
+-0.4630423730520387
  0
  0
  0
+-0.2866962385734112
  0
+ 0.2471609267439212
  0
  0
  0
+ 1.283099748800791
+ 0.2019717296192572
  0
  0
+ 0.2419837008434579
  0
  0
  0
  0
  0
+-2.177334970191569
+-4.507575454151721
+ 0.2955140797052224
  0
  0
  0
  0
+ 1.264358720487449
+ 0.6421248144713366
  0
  0
  0
@@ -6645,51 +4898,8 @@
  0
  0
  0
- 0.02034030732903353
- 0.01528549933217357
- 0.01695440911906193
- 0.3768407030719007
- 0.2760340890992067
- 0.1474640523713866
- 0.4249064197981335
- 0.2930718474413159
- 0.2047041514929455
- 0.4675682673126724
- 0.2670061341481466
- 0.1476610715331994
- 0.5557522051170745
- 0.2182097319246716
- 0.2939712600212151
- 0.5042131906796383
--0.06600253974671041
- 0.1358809671214941
- 0.6631636965438602
--0.1352112243139887
- 0.2812439551465704
- 0.7399412236649834
--0.1318532028508151
- 0.03443698427063484
- 0.6554948259813107
- 0.1045939820561339
--0.2192512233973622
- 0.4099076888723338
- 0.1948530223027911
--0.2922859752074644
- 0.3466088412993964
- 0.214512507938758
--0.2426549385190052
- 0.2576354966848444
- 0.2788445761370781
--0.2621272958771947
- 0.2349529697997613
- 0.2098994503025846
--0.1490244025553265
- 0.2198532163001285
- 0.2112974963800464
--0.09154331080040978
- 0.1701345206449615
- 0.1265033941618106
--0.06136379907128364
+ 0.2436580984798275
+ 0.3088979486766624
  0
  0
  0
@@ -6697,31 +4907,49 @@
  0
  0
  0
+-1.477428805404112
  0
+-0.1844266021370999
  0
  0
+ 0.3015771534286898
  0
+ 30.36939921604455
+ 2.906418889523707
  0
  0
  0
  0
  0
+-0.03386395734253489
+ 1.367062689079665
  0
+-0.1727500633466222
  0
  0
+ 0.8404688065083183
  0
+ 0.5969833145766199
  0
+-0.3039023428231714
+ 0.6690994566672736
  0
  0
+-33.24776044589527
  0
+-0.6764901176807574
+-0.4777261115121123
  0
  0
+ 2.798605136083226
  0
  0
  0
  0
  0
+ 4.387563205112637
  0
+-0.06254301978189696
  0
  0
  0
@@ -6731,13 +4959,16 @@
  0
  0
  0
+ 0.0565358672666474
  0
  0
  0
  0
+-0.05303174136908768
  0
  0
  0
+-1.077420061381049
  0
  0
  0
@@ -6745,51 +4976,7 @@
  0
  0
  0
- 0.01031870911745981
  0
--0.008413550240444139
- 0.1194507031271152
- 0.1030750055648655
- 0.1267352200567672
- 0.23179374736713
- 0.05324217397980707
- 0.04850298263580325
- 0.1348955095050006
- 0.1624427290375364
- 0.1846432063927164
- 0.1992716954939747
- 0.1143592882577742
- 0.05655156282250186
- 0.005521427563701259
- 0.3227862968575912
- 0.2144948648868425
- 0.001059650563141007
- 0.3142628855105483
--0.009209656182879354
--0.1293595647859125
- 0.7082897786953049
- 0.2113045879361458
--0.004433393050499546
- 0.6500360098324366
--0.1453981954434912
- 0.1443397297947534
- 0.2116903452644331
--0.1727219024114052
- 0.1218935237211743
- 0.2143009968844558
--0.1741329008436715
- 0.2199969987781595
- 0.07846483638265238
--0.1857377245073424
- 0.1532965479507176
- 0.1154167049031322
--0.09975617876109724
- 0.1952985230738583
- 0.07477234736712299
--0.1038778315328607
- 0.1245006625601844
- 0.09526824750991748
--0.0252070029892215
  0
  0
  0
@@ -6800,104 +4987,83 @@
  0
  0
  0
+ 4.544024030800431
  0
+ 0.4828010302581907
+ 14.1985954759888
  0
+-0.2484741314133392
  0
  0
  0
  0
  0
+-0.6266574399135667
  0
  0
  0
+-1.221340551550764
  0
  0
  0
  0
  0
+ 2.663412203768888
  0
  0
  0
  0
  0
  0
+ 2.616864407153919
+ 1.368445147290512
  0
  0
  0
  0
  0
+ 3.168933025649172
  0
  0
  0
  0
+-0.1787839750809836
  0
  0
  0
  0
+-0.957708462875143
+ 0.0270243089554006
  0
  0
  0
  0
  0
+-5.149044968509322
  0
+-5.402452150174298
+ 0.1983931860798278
  0
+-0.09079878744664462
+ 1.100856195238638
  0
  0
  0
  0
- 0.1284618686437821
- 0.05521170870634963
+ 1
  0
- 0.4120924666847358
- 0.3621108518667479
- 0.2570087534760207
- 0.5263364278263232
- 0.2682890292812654
- 0.1764606269265142
- 0.3112958101839003
- 0.2232743461899808
- 0.389744787129073
- 0.5192763186211899
- 0.1405195295515248
- 0.4951499490254671
- 0.05674381276352737
- 0.04511648589043775
- 0.821284880688384
- 0.304017687668088
--0.1348309869814445
- 0.947516886152169
- 0.03275003430923586
- 0.2065049079003671
- 1.307492322728107
--0.2134332507122198
- 0.135581469736577
- 1.020524406000875
--0.2293441931197029
--0.3209823793215572
- 0.9557843783805313
--0.3479733615238613
--0.1868319302951993
- 0.5190514550388394
--0.3741533746247315
--0.4875017789296014
- 0.4129002656084802
--0.461744058393222
--0.3129817225326378
- 0.1919141107238721
--0.424361391176049
--0.383846034551928
- 0.1232251607610359
--0.333641498665852
--0.2203966007762082
- 0.06825855975303714
+-1.757475422789677
  0
  0
+-1.907909824196584
  0
+-2.56843807568624
  0
  0
  0
  0
  0
+ 106.3659753096595
  0
  0
  0
@@ -6906,14 +5072,23 @@
  0
  0
  0
+ 49.8297037410713
+-0.04222112138305432
  0
  0
  0
+-0.00334437238175774
+-1.322833368577879
+ 2.001883137412201
+-0.1653185367017596
+ 0.07546371529819744
  0
+ 0.751813476182292
  0
  0
  0
  0
+ 0.8273234033979112
  0
  0
  0
@@ -6921,17 +5096,22 @@
  0
  0
  0
+-0.2206799510374065
  0
+ 0.007428173634858678
+ 0.7711147909247463
  0
  0
  0
  0
+-0.262301884897598
  0
  0
  0
  0
  0
  0
+ 5.187711830460596
  0
  0
  0
@@ -6939,70 +5119,29 @@
  0
  0
  0
+-0.3701308921519661
  0
+-0.1492866141614466
+ 4.377637052009972
  0
  0
  0
+-1.518495570290538
  0
  0
  0
  0
  0
- 0.331452983510104
- 0.2314813602174839
- 0.1305539024584947
- 0.3759033727803379
- 0.2696012443798124
- 0.1967942817392815
- 0.4140843354570928
- 0.2832786082158
- 0.1312267240579232
- 0.4306250577154188
- 0.2825074210313763
- 0.2494243393449228
- 0.3931931973808098
- 0.1114612002319112
- 0.08982572037850751
- 0.4844497493463748
- 0.09037997003060334
- 0.216556121942579
- 0.5586380686070097
--0.06595614536184118
--0.05655909953745271
- 0.7846589378293016
--0.1120836127267642
--0.1622528825787964
- 0.4587512301085379
- 0.1483188364051098
--0.2350498599921291
- 0.4729539297066008
- 0.07726539061120415
--0.1880674991152078
- 0.2638285378909403
- 0.2622980616733094
--0.1997793445197367
- 0.2834911366758052
- 0.1497133682402081
--0.1264981867567899
- 0.2035767732330088
- 0.2154027787895148
--0.07475132478855791
- 0.1630172712749669
- 0.103585098601759
--0.06746205495890244
--0.007877007090405253
- 0.007806358408923162
--0.01476610066826817
--0.007131337480064155
- 0.01526970778703906
- 0.005166313779687873
  0
  0
  0
  0
  0
+-1.175533190552626
  0
+ 10.7973331717612
  0
+ 0.2016212642457929
  0
  0
  0
@@ -7012,13 +5151,16 @@
  0
  0
  0
+-7.647984958127028
  0
  0
  0
  0
  0
  0
+-0.4235250360581704
  0
+ 1.605262962814493
  0
  0
  0
@@ -7029,15 +5171,21 @@
  0
  0
  0
+-0.151846068350995
+ 10.58198051778713
+-0.1202164155190914
  0
  0
+-0.6630880124698003
  0
+-0.437324480773051
  0
  0
  0
  0
  0
  0
+ 0.7188183917221673
  0
  0
  0
@@ -7048,96 +5196,71 @@
  0
  0
  0
- 0.1332093955173815
- 0.1174064095903005
- 0.1178926134185491
- 0.2459826100840443
- 0.05737880738885095
- 0.04259705087912995
- 0.1692321108441049
- 0.1192237030276236
- 0.1624752916523937
- 0.2829763200148874
- 0.04462492402711615
- 0.06520783013960736
- 0.1113982749104689
- 0.1417579351833835
- 0.1990804869311112
- 0.1513530491538063
- 0.1043925212877098
- 0.01875790306682611
- 0.03255527768919382
- 0.605572291812799
- 0.2485883482849533
--0.1120836127267642
- 0.8453622009796675
--0.1526163536259003
- 0.1099259472701737
- 0.2797376877892378
--0.1763815885336019
- 0.0244651689122269
- 0.3552378913952989
--0.1946018350184972
- 0.2142951240075644
- 0.1021294695380173
--0.2228066321818947
- 0.1149821514736872
- 0.1737634971766651
--0.1165636789549033
- 0.2084595511667217
- 0.07287335249328573
--0.1170007013195659
- 0.127649346982713
- 0.1121197734448574
--0.0222982750542847
- 0.008031227415344868
--0.005609180030771992
- 0.009673485941977401
- 0.006213640105032329
--0.01308533299646755
--0.00590620150003087
  0
  0
+-0.7294810885553434
  0
  0
  0
  0
  0
+ 0.5834380696791688
  0
  0
+-0.622311548892383
  0
+ 0.4679999608322835
  0
+ 6.218088963448483
  0
+ 0.5174412364094663
  0
  0
  0
+ 1.190296287934517
  0
  0
  0
+-0.944888635275562
  0
+-0.1436819972123027
  0
+ 0.337555768794067
  0
+ 2.980798379797518
  0
  0
  0
  0
  0
  0
+ 0.679042
  0
  0
+ 1.061680232170477
  0
+ 14.84914148054109
  0
+-0.3225700738404698
  0
  0
+-0.4426196314308238
+-1.488239021591834
+ 2.044096960925146
  0
  0
+ 1.683276556969745
  0
  0
+ 2.17543388273639
  0
  0
+-0.1933722845744081
+ 3.849942822218599
  0
  0
  0
+ 0.1790811567979584
  0
  0
  0
@@ -7145,70 +5268,29 @@
  0
  0
  0
+-0.06976734777921635
  0
+ 0.6407531719287224
  0
+ 32.96630448688174
  0
- 0.2545307829258172
- 0.1785030073118203
- 0.09122514207051007
- 0.3215971888443164
- 0.1681778345399439
- 0.1020474664043229
- 0.2518054512415106
- 0.06121379008318012
- 0.08996310148640849
- 0.2694612922250535
- 0.1454866002639575
- 0.2133933175578625
- 0.112324777969876
- 0.05135402676872157
- 0.2641036627475473
- 0.1526353089326875
- 0.04038475243822453
- 0.5925005479218425
--0.02172870821282149
--0.02804915331841235
- 0.7681102082037131
--0.1622528825787964
--0.1526163536259001
- 1.456439023093136
--0.1995906289810631
--0.200327634028785
- 1.443889069291943
--0.2024550460456235
--0.2148641207510449
- 0.8662574344810691
--0.4462747787047125
--0.4823646840254157
- 0.6902669328439639
--0.457101457513516
--0.3824995358978957
- 0.2885521473987115
--0.5305190273671501
--0.4231455145864221
- 0.2064465931549481
--0.4592537011890912
--0.3369701122215291
- 0.07736076349186174
--0.07101677417873505
--0.07086917169787142
- 0.01152782657511707
--0.02781418378381701
--0.01671693730458874
- 0.001295418978920348
  0
  0
  0
  0
  0
  0
+-0.4996389426133815
+-0.5177067631142696
  0
  0
+-0.3169862634712258
  0
  0
  0
  0
  0
+-12.54141721772795
  0
  0
  0
@@ -7217,10 +5299,15 @@
  0
  0
  0
+-0.2062309163815745
  0
  0
+ 1.474014777533001
+ 1.636324138104011
+-0.6774017509836604
  0
  0
+-0.1396950781612241
  0
  0
  0
@@ -7229,15 +5316,19 @@
  0
  0
  0
+ 1.655890226504994
  0
+-0.09105491107406645
  0
  0
  0
  0
  0
  0
+ 0.3772795213330301
  0
  0
+ 0.7129656141037942
  0
  0
  0
@@ -7246,70 +5337,32 @@
  0
  0
  0
+-0.1995738237571416
  0
  0
- 0.03044667703302059
- 0.04403206740566608
- 0.03582681873067981
- 0.1970191390004093
- 0.06593882888838465
- 0.05121909634722503
- 0.1727167742643328
- 0.1289474301708956
- 0.005675658529694199
- 0.2554431885446493
- 0.1010532392087211
--0.01520252151990854
- 0.2223286466276709
- 0.07682658191327009
--0.0213312754754193
- 0.3435854388741006
- 0.01016484014168444
--0.07013381010260193
- 0.2677130921354686
- 0.05745042261581606
--0.02781759349216432
- 0.4516209049819952
- 0.1033442120768516
--0.1885592732195571
- 0.8624168095341098
- 0.05815875988743358
--0.1003785745015452
- 0.6747289496893929
- 0.2388143876842166
--0.01763113134369359
- 0.5813307949485805
- 0.1542528888277588
--0.08864440555020732
- 0.4392701743706457
- 0.2711436118932059
--0.08100843721981642
- 0.3665817567499925
- 0.1978001931191953
--0.09663519785199656
- 0.2489451381663269
- 0.2076306050245115
--0.06458040387065019
- 0.06223447083982179
- 0.09654846249174477
--0.04745917281149225
- 0.004128465618274113
- 0.02759014681847567
--0.01940120982411387
+ 0.4881741898924112
+-3.635485830889141
  0
- 0.002571358937249166
--0.03168361259661954
  0
+-1.219804931750709
  0
+ 1.745960394621586
  0
+-0.6708065717577664
  0
  0
  0
  0
+ 1.080357775750419
+ 0.8831513213634841
+ 1.429880283468479
  0
+-4.076266171684704
  0
  0
  0
+ 0.4797442279782531
+ 0.06307150790587217
  0
  0
  0
@@ -7323,87 +5376,58 @@
  0
  0
  0
+ 2.286884079536372
  0
  0
+ 4.292003067546896
  0
+-0.0688495314067514
+-0.533026798842115
+ 1.506111833622525
  0
  0
  0
+ 2.297532616184889
+ 0.2884256723077435
  0
+-6.437370892793756
  0
+-0.8361437906884199
+-0.5712534775192925
  0
+-1.402624039953607
  0
  0
+-0.1359252093563885
+ 1.242344184805322
  0
  0
  0
+-0.1141738798020996
  0
  0
  0
  0
  0
+-1.158618660065336
  0
  0
+-2.089177813925037
  0
+-8.07367477977005
+-1.0150323711742
  0
  0
  0
--0.02045731958508273
--0.0426108728790597
--0.03643801670881111
- 0.02336418679529378
--0.00916601005182391
- 0.02071903558911519
- 0.06274543143817313
--0.002924352033261558
+-0.07056341033340689
  0
- 0.002789101460044177
- 0.05483318995158447
- 0.02367999641957973
- 0.08501030187761978
- 0.06398991331186546
--0.06205071004502502
--0.02054685434728548
- 0.168235757915879
--4.160383083549289e-05
- 0.07759499499269781
- 0.1386032591445443
--0.1685482912921846
- 0.1392035408523171
- 0.2733818140414618
--0.1893349023083959
- 0.05478142647510953
- 0.7296436958498829
--0.1640454835615678
- 0.2922242014780934
- 0.4191623102468908
--0.09669166224617004
- 0.1647746628607176
- 0.4644214008258962
--0.1229713029242185
- 0.3152173190357898
- 0.2354343774441369
--0.1051610027511401
- 0.2474173938675175
- 0.2648560507450335
--0.09130072499282633
- 0.2585952782363763
- 0.1303092040020581
--0.1001186584580273
- 0.03654170883451871
- 0.03571621537178682
--0.07066499281361863
- 0.01880113440246318
- 0.03955211569560431
- 0.01051078006649249
--0.02479764712976456
- 0.03555235082365083
- 0.004232550382972015
  0
  0
  0
  0
+-0.5251837843361902
  0
+-0.6512360211885267
  0
  0
  0
@@ -7411,31 +5435,45 @@
  0
  0
  0
+-26.20235897003792
  0
+-0.4322824983983252
  0
  0
+-0.7648821736851593
+ 5.006095070641486
  0
  0
  0
  0
  0
+-4.337173836387355
  0
  0
  0
  0
  0
  0
+ 0.5268338988565754
+-0.009571158761364785
  0
  0
  0
  0
  0
  0
+ 0.4653791124973645
+ 25.64567120728595
  0
+-0.6496123292638682
  0
  0
+-0.4555035775337348
+ 0.1955280019478218
  0
  0
+ 42.88987330209684
+ 0.03244860746650249
  0
  0
  0
@@ -7444,63 +5482,21 @@
  0
  0
  0
+-1.576908553841815
  0
  0
  0
  0
- 0.147393540957853
- 0.1030014784110124
- 0.04396007297685398
- 0.2406148190047568
- 0.1008854153494592
- 0.05835339870557083
- 0.1692339638181312
+-2.873816010261588
  0
- 0.03157191947671363
- 0.1816538216969741
- 0.0952808709023624
- 0.130136541247062
- 0.06290448543264195
- 0.03179279991833318
- 0.1913243599463574
- 0.09101977113375354
- 0.0272590546908461
- 0.499030767882946
--0.09343446827403874
--0.05510117358776137
- 0.7059991521038377
--0.2186949145192734
--0.1687443964906593
- 1.429767567023642
--0.09290031529819788
--0.1635042463347606
- 1.493650836801028
--0.1084163194590233
--0.1602163117017198
- 0.935098246786198
--0.3648069207726756
--0.465283844396313
- 0.7410489691933031
--0.3871314685413987
--0.3411754518775544
- 0.3053286620649369
--0.4817930649568226
--0.4040404216395288
- 0.2056944131762076
--0.4206227557262404
--0.3077282197880447
- 0.06859066072643273
--0.0517525855098025
--0.04528452114517724
--0.006114100394247388
--0.02437959763494164
--0.008015490269529054
--0.00325449393741661
--0.002822866834812281
- 0.004720887190653955
--0.007448239739027688
+ 0.6951557638967627
+-2.626277292607718
  0
+-1.029532206148845
+-2.453966340890945
  0
+ 0.72016515394283
+ 0.8694012728502445
  0
  0
  0
@@ -7516,15 +5512,20 @@
  0
  0
  0
+-2.68909022957995
  0
  0
  0
+-0.7475340420147444
  0
  0
+-0.2091073641131919
  0
+-0.4709622115437511
  0
  0
  0
+-0.4948605582522377
  0
  0
  0
@@ -7532,76 +5533,40 @@
  0
  0
  0
+-0.07391519327902506
  0
  0
  0
  0
  0
+ 4.193747856539138
+-0.5501610235721783
+ 10.19376200848029
  0
  0
  0
  0
+ 0.3168698940830827
  0
  0
  0
  0
  0
  0
+ 0.2406850922411721
  0
+ 0.6376083595888241
+-1.365491324533382
+ 2.856514672954081
  0
  0
  0
- 0.1495069098706701
- 0.05672342586476095
- 0.06326824538365942
- 0.1523235075401624
- 0.1065843268946623
  0
- 0.172075302285517
- 0.1304235596401615
  0
- 0.1917693372876988
- 0.116806175198413
--0.06688564545124541
- 0.2295052043559954
- 0.1243845633915553
--0.05738909956635813
- 0.2283013689066606
- 0.04567986907550232
--0.140801474742453
- 0.4869921278408665
- 0.03138217138102461
--0.1846983895904171
- 0.6908989710273516
- 0.2978475242577998
--0.1105323004527429
- 0.9160735215941407
- 0.009275220906341393
- 0.008415652920233822
- 0.5744515840727835
- 0.2241412662908211
--0.03759065826253016
- 0.618506140462265
- 0.1151242468959312
--0.07598030182617588
- 0.3682208717140836
- 0.2332661220362948
--0.05705732911713544
- 0.3165930419810411
- 0.150569774740758
--0.1070557787461441
- 0.03671349648149667
- 0.07846500669693569
--0.07798789784126166
--0.003063613635782227
- 0.05496503781948593
- 0.01190322534810254
--0.02225239499278948
- 0.0294467433290907
--0.002896058024706844
  0
  0
  0
+ 1.053328123822466
  0
  0
  0
@@ -7609,7 +5574,9 @@
  0
  0
  0
+ 0.1642622543035108
  0
+-4.680972400503813
  0
  0
  0
@@ -7623,7 +5590,9 @@
  0
  0
  0
+ 4.073044168382923
  0
+ 0.1880083009902825
  0
  0
  0
@@ -7635,81 +5604,49 @@
  0
  0
  0
+ 0.1207846509117613
+-0.03317074813271751
  0
+-0.7055982138911948
  0
+-1.83713974293803
  0
+ 3.037225158734759
  0
  0
  0
  0
+-0.2120732212399692
  0
  0
  0
  0
+ 0.3103350139921735
+-5.740764089657944
  0
  0
  0
+-2.179136981551416
  0
+-1.305844019981313
  0
- 0.06787543634648979
--0.007922532719397701
  0
- 0.07224149959422066
- 0.01628660405329422
  0
- 0.08940671913108297
- 0.01067747206902605
  0
- 0.1094758809671548
- 0.009906136467646987
--0.005284173045837082
- 0.106034390137575
- 0.02365763360792631
--0.01599401007453661
- 0.1128254962999722
- 0.1448878857788443
--0.02857597816196143
- 0.08827685244659778
- 0.3544702182330313
--0.2042435690694162
- 0.252261379662941
- 0.426384158372172
--0.1641054033464274
- 0.01462906987761201
- 0.7114321001018025
--0.1284561393429695
- 0.1896061738720026
- 0.3913651895532327
--0.1850344098865425
- 0.1138834046962507
- 0.4379285184690964
--0.1116092113003505
- 0.2598280936990828
- 0.231698748000693
--0.1422928112606244
- 0.1916233923365911
- 0.2107336418254812
--0.05215167466849729
- 0.07613813335148018
- 0.06787811574482296
--0.03974355426023836
- 0.03009932157078149
- 0.01023879429914638
--0.02824708460007883
- 0.0004413930041132366
- 0.004695022247142875
--0.03263143495326951
- 0.0001356725217021724
  0
  0
  0
  0
  0
  0
+-0.4317067648539093
  0
  0
+ 2.265242158370023
  0
+-0.7794917298629519
  0
+-0.2080790047502954
  0
  0
  0
@@ -7718,6 +5655,8 @@
  0
  0
  0
+-0.404070778515178
+-0.3200287270469684
  0
  0
  0
@@ -7736,13 +5675,22 @@
  0
  0
  0
+-4.266969719763726
+-3.406553559740435
  0
+ 0.0134461421595354
+ 4.397741852193009
  0
+ 0.02270262847050936
+ 1.236312816624057
  0
  0
  0
  0
  0
+-0.8438687874368702
+-0.7304010128337621
+ 4.213672406176546
  0
  0
  0
@@ -7751,70 +5699,34 @@
  0
  0
  0
- 0.06746540205821781
+-0.2418304053586636
  0
- 0.004660098126320933
  0
  0
--0.03400585992974392
  0
+-0.2657663734219002
+ 0.5014011568476776
  0
--0.00383259980343694
--0.02250286167721027
--0.002161532276535457
- 0.01575819143221054
--0.02907326899432235
- 0.01212308568433054
- 0.1895485619551283
--0.1226883870996998
--0.09326396534840108
- 0.2969888027886603
--0.203070996085234
--0.2084683532099987
- 0.8491159347773153
--0.03795460427638125
--0.1029532224716381
- 0.9361086421453015
--0.004859332291959456
--0.1366820902232274
- 1.039545518119511
--0.1423920866635132
--0.2978412287866122
- 0.9314103195780626
--0.1341947865413598
--0.2033139875709718
- 0.4814115349324639
--0.3234138829586117
--0.2987621449144857
- 0.3398491208950851
--0.2663595805599724
--0.218809251959778
- 0.09930675826233686
--0.06283131870902126
--0.07465836167523256
- 0.01649009102387712
--0.01937046663801414
--0.01674868775790473
- 0.01540444473919237
- 0.001637604171341718
--0.006643948276569291
- 0.01307958466415525
--0.004706298102512472
  0
  0
  0
  0
+-2.013417788718923
  0
  0
  0
  0
  0
+-0.06507152503028568
  0
  0
  0
+ 0.6912662714336949
+ 0.1965927372220691
  0
  0
  0
+-1.651384514001629
  0
  0
  0
@@ -7822,12 +5734,18 @@
  0
  0
  0
+ 0.214076
  0
  0
+ 0.5563402540667817
  0
+-0.8867741572195472
  0
  0
  0
+-2.656741289449448
+ 0.4450668633801069
+ 0.385689412005946
  0
  0
  0
@@ -7837,6 +5755,10 @@
  0
  0
  0
+ 0.9436266645675534
+ 2.933814531024174
+-0.9512901478130462
+-1.309898643573616
  0
  0
  0
@@ -7851,74 +5773,35 @@
  0
  0
  0
- 0.02898332685159531
--0.004354525751479183
  0
- 0.01073294877213779
- 0.02054120346484029
- 0.00766263125374445
- 0.07750579671108329
- 0.0209812241121359
--0.04437515537557073
- 0.05469265585623201
- 0.04570283948423293
--0.02717713368640161
- 0.124139486972122
- 0.0270227710293308
--0.1303019253528366
- 0.08822367049810927
- 0.1115488243306284
--0.1032915558562808
- 0.2777750758706082
- 0.2172107771469001
--0.432084772402383
- 0.5924968144533665
- 0.1681169525737155
--0.3638298217685866
- 0.5702169891773118
- 0.1846199178710025
--0.1298422859076117
- 1.059513761324535
- 0.09211030447859657
--0.01561395720374273
- 0.8328064555130887
- 0.3013936581390753
- 0.1380564963950855
- 0.7295128267437798
- 0.1726908115653597
- 0.07277149818462429
- 0.5244058902226941
- 0.3333975662538784
- 0.01675687435379709
- 0.181107270320755
- 0.1549508763568679
--0.008530430925992738
- 0.06272183180044247
- 0.0008406654098609593
- 0.001633124323519533
- 0.01760002016703443
--0.04042577350348711
--0.0220268789514094
--0.01230305286134971
  0
  0
+-0.03962719147094711
  0
  0
+-4.978615272453554
  0
+-0.05027296967367353
  0
  0
  0
+ 0.8500691125051955
+-12.61368270895732
  0
  0
+ 1.087858072226194
  0
  0
  0
  0
+ 0.377904365182381
  0
+-0.6686515407945873
  0
  0
  0
  0
+ 0.1862664661629329
  0
  0
  0
@@ -7933,73 +5816,40 @@
  0
  0
  0
+-0.01466587516723143
+ 0.06610711171668869
  0
  0
+-2.123751921100425
  0
  0
  0
  0
  0
+ 0.1075345371825239
+-1.509641178057292
  0
  0
+-0.8967074716443297
+-0.2793541539661331
+-0.7809135848512985
  0
  0
  0
  0
+-0.6139494500559899
  0
  0
+-0.06244034836215474
+ 0.2024876122386019
  0
+ 0.31508191962413
+ 0.7363768068547064
+ 0.1525291625640584
+ 1.066734274264562
  0
  0
--0.029796989501046
- 0.01262140391350271
  0
- 0.00723388273839362
--0.01181623564439983
--0.004183642603085919
--0.01599818818410756
- 0.01991640978253419
- 0.01088331179673357
- 0.05341456696656432
--0.0004452594257657127
--0.06572776881713002
- 0.01739101993365819
- 0.0600678336403731
--0.04239910763554258
- 0.1261800612079269
--0.01547226261023061
--0.241911545576015
- 0.2659940079745977
- 0.1033728904602602
--0.459230242551262
- 0.1541391861533491
- 0.4689377301211789
--0.4633386739359197
- 0.225649421614781
- 0.3878329299325556
--0.2938448965939899
- 0.09211030447859683
- 0.9708577987049175
--0.1481766715720259
- 0.3845927635994222
- 0.5365648946386252
--0.0140227677526953
- 0.2787504288775663
- 0.5868615893379269
- 0.008401388994637974
- 0.4208893883015439
- 0.2852520001586972
--0.100147079319616
- 0.07595542983556139
- 0.0967515987019547
--0.1196203560626718
- 0.04685344137364201
- 0.111879553778612
- 0.03996962885610333
--0.04493882927320793
- 0.0718126586103678
- 0.01516675067158226
- 0.00442481602400676
  0
  0
  0
@@ -8008,39 +5858,58 @@
  0
  0
  0
+ 0.6218400864802324
  0
+ 0.8920004254831011
  0
  0
+ 396.1839217634359
  0
  0
  0
  0
  0
  0
+-1.199253263921576
  0
  0
+-0.04047875704426624
  0
  0
  0
  0
  0
+ 0.2802717458726153
+-0.4781398339414029
+ 0.2075733873419544
+ 0.2120988866516076
+ 0.2259316723165299
  0
+ 1.855703386599502
  0
+ 48.45482065331772
  0
  0
+ 1.925846335556874
+-0.3825298684997945
  0
+ 2.734965965930728
  0
  0
+ 0.1722425982306823
  0
  0
  0
  0
  0
  0
+ 0.5452342913157545
+-0.1426201569301026
  0
  0
  0
  0
+ 1.777092371423943
  0
  0
  0
@@ -8048,64 +5917,22 @@
  0
  0
  0
+-0.03376619410157096
+-0.3269307626883812
  0
  0
  0
+ 0.6768557113363313
  0
  0
--0.01364922125150793
--0.06168013200811229
--0.03332808481473472
--0.03179895379705183
--0.05946985482088529
--0.02870114808324508
--0.01506534345543211
--0.08054957189767846
--0.02644018080428226
- 0.005022645584397655
--0.08773719750652052
--0.005321508687478919
- 0.1497959221195863
--0.1557730606003737
--0.1212421329707608
- 0.2261366528506384
--0.2053959969322332
--0.2288378683429283
- 0.6693464590200435
--0.09822776084638159
--0.1272576023759546
- 0.7393206440908646
--0.04129601313005547
--0.1856822851133047
- 0.9248706331998275
--0.0190445903121543
--0.1474053557430128
- 0.960854310895684
- 0.0255655931356564
--0.1117809823517434
- 0.5908934247810864
--0.2033273452812217
--0.2015578029659007
- 0.4391325679340776
--0.1257456371243098
--0.1377714170476583
- 0.1262668249509752
--0.02160870665757127
--0.04702441958410075
- 0.01619576889441505
- 0.006114257682597852
--0.002599705846445055
- 0.03595083950880592
- 0.0002560624773800675
--0.008048999357267861
- 0.02467708834887031
--0.007345749081730969
+-0.5302064943646121
  0
  0
  0
  0
  0
  0
+-0.1151294185864515
  0
  0
  0
@@ -8116,6 +5943,7 @@
  0
  0
  0
+ 4.89587606110833
  0
  0
  0
@@ -8131,90 +5959,78 @@
  0
  0
  0
+ 0.1242991952552595
  0
  0
+-0.8921944704667315
  0
  0
+-0.2079497209821426
+ 0.118424235762753
  0
  0
  0
  0
+ 0.02805149213992744
  0
  0
  0
+-1.432378239368848
+-2.371776843647885
  0
+ 0.344804503493937
+ 3.395089135255176
+ 0.9326537240259527
  0
  0
+ 0.3618483504009797
+-0.2204529230128786
  0
  0
+-5.095600934668537
  0
+-0.1751302818073968
+ 24.77215816884107
  0
+ 0.1017087239890091
+-3.378619827176517
  0
+ 0.5450043410439027
  0
  0
+-0.06792314582765882
+-0.8227741754688692
+-0.973052
  0
  0
  0
  0
+-0.2761773945576845
  0
- 0.03453671773665813
- 0.03910091372558489
--0.0217806128277353
- 0.0477906010433775
- 0.04755003491357004
--0.05405441524404941
- 0.07210361065460598
- 0.08027638082584458
--0.1085848691249564
- 0.09823219565098124
- 0.05911885016714628
--0.1853960383774744
- 0.3351769800074288
- 0.136961043905307
--0.4429367468435427
- 0.4736552030171357
- 0.3343014484436261
--0.3865849329842753
- 0.6436495881088224
- 0.1230713235555615
--0.1322848505602867
- 0.8506478586062033
- 0.3983631136526723
- 0.01735955245994484
- 1.090253706406084
- 0.06018620866675699
- 0.09648076688640531
- 0.7225225573456515
- 0.2419012883851915
- 0.1049370355865295
- 0.6774825512361102
- 0.198269638975059
--0.0850370609663811
- 0.1625622160911344
- 0.1033606303177067
--0.0876968881265851
- 0.07842558875981412
- 0.04458542871302893
- 0.06594416238081342
--0.01239943441183437
- 0.01285892772532419
- 0.04248895205805455
--0.006474458738017678
  0
  0
  0
  0
+ 1.475796554910678
  0
  0
  0
  0
+ 0.5250068188840999
+ 0.8463708086907529
  0
  0
+ 1.77875381612637
  0
+ 8.964172785359493
+-0.5306670663521423
  0
  0
+-0.4611627973513845
  0
  0
+ 1.035139221882437
+-0.277849018543929
  0
  0
  0
@@ -8224,8 +6040,12 @@
  0
  0
  0
+-0.08095671647350679
+ 4.267559020070598
  0
  0
+ 0.6157705599681257
+-0.3452226986141149
  0
  0
  0
@@ -8233,10 +6053,14 @@
  0
  0
  0
+-2.032109619024405
  0
+-1.799909961530498
  0
+ 1.996614876342541
  0
  0
+ 1.042071053613529
  0
  0
  0
@@ -8251,83 +6075,59 @@
  0
  0
  0
+ 3.493243119263601
  0
  0
+-0.7519455731142475
  0
  0
  0
+ 1.861670472347809
+-1.302549556633801
+ 0.1164549396633313
  0
- 0.01656734268912792
--0.01330540983132488
--0.0158340745473802
- 0.05002760966816803
--0.006844885889453874
--0.02131395206575877
- 0.06766036162347627
--0.01021495802314147
--0.03705388133153394
- 0.0850110315346234
- 0.03069935193815565
--0.08459657090925513
- 0.1721273368845858
- 0.1794700505476242
--0.353217751715753
- 0.2931693705226576
- 0.2487268179161962
--0.3404054617989499
- 0.1322897613814507
- 0.4449798635008817
--0.1996429441262241
- 0.3135021327272443
- 0.5457765710033901
--0.1170398104201297
- 0.06018620866675708
- 0.8330306675055654
--0.01293293869645117
- 0.2548416597106281
- 0.4849491599242081
--0.09939157061882915
- 0.203029194702317
- 0.4362851592397721
--0.02230013923919606
- 0.09719587897266509
- 0.1809439895872292
--0.06246631186116594
- 0.0155668417244723
- 0.07243469262560916
--0.05530761199471974
--0.02148768441240798
- 0.02288959401604958
--0.0789276915420626
--0.00281705548223805
  0
+-1.923512129505668
  0
+ 2.474447041467192
  0
+-1.040751374820962
  0
  0
  0
+ 0.9780301992400041
+ 3.320521217754515
  0
  0
+ 2.161081574861941
  0
  0
  0
  0
+-0.06872316014507636
  0
  0
+-0.1999009360692771
  0
+ 0.1152876944927467
  0
+ 1.677081838440446
  0
  0
  0
  0
  0
+ 0.6103558037264367
+-0.2243037856522242
  0
  0
  0
  0
  0
+ 0.5731714028165047
  0
  0
+-2.453389315249321
  0
  0
  0
@@ -8335,71 +6135,36 @@
  0
  0
  0
+ 4.937771034501405
  0
  0
  0
  0
+-0.8628481741011198
  0
  0
  0
+ 0.6275581044259833
  0
+-0.6103050445663323
  0
  0
+ 0.9506517744974721
  0
  0
  0
  0
+ 0.07146383477637612
+ 0.2033276276424675
  0
  0
  0
  0
  0
+ 0.3866797756698009
  0
  0
  0
- 0.01137930166471226
- 0.001037527197513009
--0.01498848595762328
--0.06537707692533822
--0.02522561765713401
--0.01738032674510652
--0.05002821610234268
--0.008209218232031981
- 0.0323843603031537
--0.06641794539566535
--0.04827689964571867
- 0.02643811742099644
--0.1342325771285902
--0.1199933013742983
- 0.2514795559897838
--0.09249576141705464
--0.1160301522966605
- 0.3041021298489935
--0.09090656952784495
--0.1180285786701245
- 0.4797224005001612
- 0.1289584840756142
--0.02003429285797051
- 0.5951711866781099
- 0.096480766886405
--0.01293293869645114
- 0.8796047273164596
- 0.02630746744865203
--0.08958764164238489
- 0.7914593694877105
- 0.06744749237726214
- 0.01155287875537524
- 0.3773729733294982
- 0.03643122195779108
--0.09169194334474876
- 0.2066787305019734
- 0.08036297616610467
--0.06326117909986063
- 0.1066184158581806
- 0.06604709169946164
--0.09419458776918326
- 0.08002955108649515
--0.009560191069422888
  0
  0
  0
@@ -8408,22 +6173,33 @@
  0
  0
  0
+ 0.1258547169606876
  0
  0
  0
  0
  0
  0
+ 0.7447751662012354
  0
+ 0.8908618409796408
+ 0.4933084106488362
  0
  0
+-5.039090765171005
+ 8.30841404518636
  0
  0
  0
+-0.3605358565255673
  0
+-1.554953303792497
  0
+ 0.2431225017316495
+ 0.1114327694170067
  0
  0
+ 0.2176529555530883
  0
  0
  0
@@ -8431,21 +6207,29 @@
  0
  0
  0
+ 1.243761507650353
  0
+-1.138724018110861
  0
+-0.1884231426986675
  0
  0
  0
  0
  0
+-0.3379309618996783
  0
+ 0.2485915017880706
  0
  0
+-5.343306933654838
  0
  0
+ 0.001917725227957101
  0
  0
  0
+-0.57272516917422
  0
  0
  0
@@ -8453,54 +6237,19 @@
  0
  0
  0
+-1.950962399877542
  0
  0
  0
  0
  0
+-0.3539993266894119
+ 0.1863721694763848
  0
+ 0.09202470134014996
  0
- 0.00289646067159027
- 0.007004205646794325
- 0.01097261793677344
- 0.05835945527251087
- 0.02441189412637596
--0.06962733829505056
- 0.06377324381296934
- 0.07760600273692224
--0.1072923759363541
- 0.2799744852344632
- 0.2491230160149703
--0.5507550787824206
- 0.4320214424362562
- 0.2626601769388605
--0.4899612607894883
- 0.3990383035440956
- 0.2775912621611886
--0.327168609905263
- 0.7591586516310525
- 0.2926308919829664
--0.2183413748921876
- 0.7306493656271388
- 0.2635506750694763
- 0.02167717882508768
- 1.158746498413555
- 0.1478167154941652
- 0.1235994016606926
- 0.8892095826798591
- 0.3623817065773147
- 0.191371907717232
- 0.4647060879430394
- 0.1395793463321499
- 0.1391891223910622
- 0.2781057541541126
--0.1241896330588512
- 0.07452981763489898
- 0.1427072443872709
--0.166328765809533
- 0.04019863106016888
- 0.01231301866858489
  0
+ 0.3212898449005449
  0
  0
  0
@@ -8510,9 +6259,13 @@
  0
  0
  0
+ 0.3621747592009364
  0
  0
+-0.1414438098336342
+ 157.5397344752773
  0
+-0.09046742395702388
  0
  0
  0
@@ -8522,89 +6275,65 @@
  0
  0
  0
+ 3.862837820137139
+ 0.1924558131841986
  0
  0
  0
+-0.2840026206298199
+ 0.1327272769633906
  0
  0
  0
  0
  0
+ 33.09580110175312
  0
+ 1.106718461462165
  0
  0
  0
+ 1.251918154311314
  0
  0
  0
  0
+-0.5332273353948972
  0
  0
  0
+-1.455788933358241
  0
+-0.1005810698833533
  0
  0
+-0.7048634047882905
  0
  0
  0
  0
  0
+-0.1829592210252697
  0
+ 1.554891259764205
  0
  0
+-0.5622586756351996
  0
  0
  0
  0
  0
  0
+-0.02155014855168353
  0
  0
- 0.02550294589223547
- 0.001607106795463471
--0.02115740875305558
- 0.03505662763607797
- 0.02657711641646523
--0.01359133484068593
- 0.08784071128997431
--0.008435442323719618
--0.1392706928919042
- 0.2410808856179767
- 0.0899077674592946
--0.3975385604402012
- 0.2211494487333924
- 0.2793101584905084
--0.4064000290127994
- 0.2567391738675522
- 0.2429682485562932
--0.3010442231383564
- 0.1908285306598604
- 0.5978717604761639
--0.2116232335066464
- 0.2455135258435132
- 0.4889257966413527
--0.09192652359485722
- 0.1478167154941651
- 0.8637209324822654
- 0.09116528720254731
- 0.3591403950408739
- 0.4780342787926764
- 0.013380249633847
- 0.04493628699216265
- 0.2359559909974446
--0.05657591797366269
- 0.002535017223999032
- 0.2183589475976805
- 0.06667791693847054
--0.07885800375324803
- 0.107943619281646
- 0.005320783729045826
--0.002102138210538136
  0
  0
  0
  0
  0
+-0.05517346460039634
  0
  0
  0
@@ -8613,21 +6342,30 @@
  0
  0
  0
+-0.6481684344216544
  0
  0
+ 0.009127147076078621
  0
  0
+ 0.8920297763357408
  0
  0
  0
  0
  0
  0
+-1.891229456968384
  0
  0
  0
+-0.3717262537806227
+-0.03650552052887308
  0
+ 1.101498666534914
+ 181.1975663423919
  0
+-0.4884495217296688
  0
  0
  0
@@ -8637,18 +6375,25 @@
  0
  0
  0
+ 3.574127663089739
  0
+ 0.1860349011542316
  0
+-0.01108354956990108
  0
  0
+-0.5589239617229802
  0
  0
  0
+-15.98727046132908
+-6.139599379559069
  0
  0
  0
  0
  0
+ 0.3564056231718538
  0
  0
  0
@@ -8660,71 +6405,49 @@
  0
  0
  0
--0.0859297511444953
--0.03123487347377921
  0
--0.06470958525160081
--0.004412039415424872
- 0.05083465351914572
--0.06679426264548287
--0.05789061758710524
- 0.01201533480566358
--0.1096599758492784
--0.1266609628787409
- 0.183669996416383
--0.1368700292049179
--0.117235027743614
- 0.2271303908593813
--0.09862214436666189
--0.1675955368616276
- 0.3551839797591633
- 0.04010616227191219
--0.01828079737070288
- 0.4611292242512056
- 0.0819897343764793
--0.1183644758991115
- 0.8028086387135387
- 0.1064183790713736
- 0.07558353681337865
- 0.9086777064362186
- 0.2126081863809646
- 0.08296946165210906
- 0.4910900132866513
- 0.1004737517927588
--0.05893816208283639
- 0.2890034634284118
- 0.1367784144699386
--0.0439561873339182
- 0.1748266996629207
- 0.08755388638592414
--0.1092755988256249
- 0.1255203211715755
--0.003107820609141417
  0
  0
  0
+ 0.01873966260724039
  0
+ 9.207082663592216
+-1.462609353137826
  0
  0
  0
  0
+-0.3084022201824273
  0
+-0.001494477889351077
  0
+-0.639959639338643
  0
  0
  0
  0
+ 6.680483493497896
+ 5.553240032126599
  0
  0
+ 1.332858851534272
  0
+ 0.1131389989281115
+ 0.05052433436715395
  0
  0
  0
  0
+-0.08032570098366452
+-4.306342631424013
+-0.3311656692546475
+ 5.18545434333079
  0
  0
  0
+-1.35938512410737
  0
+-12.06264393151567
  0
  0
  0
@@ -8734,14 +6457,18 @@
  0
  0
  0
+ 1.774260054200211
  0
+ 0.6190595835642895
  0
  0
  0
  0
+ 2.281013599138365
  0
  0
  0
+-1.272424626221362
  0
  0
  0
@@ -8749,8 +6476,11 @@
  0
  0
  0
+ 4.324605713861138
  0
  0
+-6.345603615790674
+ 0.4697904996479123
  0
  0
  0
@@ -8759,66 +6489,43 @@
  0
  0
  0
+ 0.2796381362266612
  0
  0
+-0.06583423808647407
  0
+-0.1708671119388621
+ 1.446100117702695
  0
- 0.0453111887730545
- 0.04129788849889209
--0.06551099215757458
- 0.07952260679332711
- 0.04446407719901178
--0.1507094131146911
- 0.3162443773764748
- 0.1939424030884327
--0.5544829550120121
- 0.3708925797358065
- 0.3329221024495854
--0.4962545170836462
- 0.4262529366054946
- 0.2496156953742299
--0.3170144615274276
- 0.6117748604449134
- 0.4963845938892379
--0.1846941594067151
- 0.754178502771673
- 0.2548001230201909
- 0.03497855111625389
- 0.9508196856856667
- 0.4107921595326413
- 0.2036767496268742
- 1.081807205394564
- 0.1563780569083378
- 0.04575373491781462
- 0.444836848050662
- 0.0005912814978173479
--0.009846928983868677
- 0.3489336818221916
--0.05080052533646509
- 0.1861936130971906
- 0.1120518768328002
--0.07007882752566297
- 0.1546995329133125
- 0.04354176769896598
  0
+-0.709420994721505
+ 0.8312792208980893
  0
  0
  0
+ 1.150495133643803
  0
  0
  0
  0
  0
+ 0.4247023662659065
  0
  0
  0
  0
  0
  0
+-1.994140389323243
+ 0.08451970938897084
+ 0.2445618119961789
  0
  0
+ 0.1861879473977577
+ 0.8255679381271304
  0
  0
+-0.4972849136882072
  0
  0
  0
@@ -8831,14 +6538,18 @@
  0
  0
  0
+ 0.1892641816600949
  0
  0
  0
  0
  0
+-0.4696475831889678
  0
  0
  0
+-0.7161722386197744
+-0.4773806413906373
  0
  0
  0
@@ -8848,64 +6559,36 @@
  0
  0
  0
+ 0.6855326207390812
  0
  0
  0
+ 189.4490562795383
  0
+ 0.2294168823791598
  0
  0
  0
  0
  0
  0
+ 15.01173079019572
  0
  0
  0
+ 3.023849232116733
  0
+ 0.02400723546902379
  0
- 0.02527598457295166
--0.001768986990454072
- 0.003317002883114171
- 0.03297913006185288
- 0.01366326070349741
--0.05024524905011145
- 0.1744398657406253
- 0.1375670423938269
--0.3503753904261871
- 0.2836541174461448
- 0.1804375239952578
--0.3593902219941618
- 0.2247152646836893
- 0.2529834491179146
--0.2528701667015872
- 0.3955041754621541
- 0.3366960001340097
--0.1793120645196947
- 0.2508099372678701
- 0.4732614839537085
--0.01147735761987027
- 0.4033712994296682
- 0.5126473409959805
- 0.07734868580508403
- 0.1563780569083378
- 0.7584303713220008
- 0.1341456930030979
- 0.04591759290117975
- 0.4317919472154008
- 0.0452619231902515
--0.117448049551387
- 0.1606536980076543
--0.08827796000712447
--0.07967053621374606
- 0.02069569454556858
--0.1533521039855621
--0.05358611208257169
  0
  0
  0
+ 0.1428853246752037
  0
+-0.1258080192744591
  0
  0
+ 0.977392616352955
  0
  0
  0
@@ -8916,36 +6599,53 @@
  0
  0
  0
+-1.159163815357885
  0
  0
  0
+ 1.830819091338398
  0
  0
  0
+-1.224993910620706
  0
+ 1.12737388736329
  0
  0
  0
+-0.3417665236641663
  0
+ 1.363948077662064
  0
  0
+-0.3579474117229764
+-0.5209770793433826
  0
  0
  0
  0
  0
+ 0.3483799281354589
  0
+ 0.7390603694655263
  0
  0
+ 1.79402507837949
  0
  0
  0
+-0.506937
+-98.38946250084041
  0
+ 1.789555367686995
  0
+-2.901378745215094
  0
  0
  0
  0
+-0.3616770619378998
+-0.3024026847600035
  0
  0
  0
@@ -8955,51 +6655,18 @@
  0
  0
  0
+-0.3260923676104936
  0
  0
+ 0.03540338342917591
  0
  0
  0
  0
  0
  0
- 0.01800901377956207
- 0.005175041449258899
  0
- 0.02768126599643208
- 0.01276640171338106
--0.05155951291841206
--0.06220051317288586
--0.02810571770655953
- 0.05369650562020231
--0.08147770996232603
--0.1299112381239295
- 0.1104653641416167
--0.1502282885662586
--0.07680847635831173
- 0.1265807640043945
--0.02161268361288356
--0.1316963322025976
- 0.1551205441412535
--0.1175432753186116
--0.04499884669811427
- 0.3918430353650929
- 0.1620710605802327
--0.009325979607410011
- 0.4971655863234344
- 0.03819551445357049
- 0.1284578264977487
- 0.8234683122488623
- 0.2639297909576087
--0.0405905357444304
- 0.7017729860270119
- 0.2595312815709759
--0.2090611843105151
- 0.193723871678607
- 0.2661962875888019
--0.2936619798966802
- 0.1352003871330183
- 0.03406732666513703
+ 0.3064325656735168
  0
  0
  0
@@ -9008,8 +6675,12 @@
  0
  0
  0
+ 1.080346199490051
+ 3.530698360660324
  0
+-0.5223943446877712
  0
+ 1.157378358175996
  0
  0
  0
@@ -9019,9 +6690,16 @@
  0
  0
  0
+-4.268159133035052
  0
+-0.4528358491068142
+-4.582341386826024
+-0.2679088348068341
+ 2.057257979795762
+ 3.73870859223568
  0
  0
+ 0.4044245983857004
  0
  0
  0
@@ -9031,32 +6709,44 @@
  0
  0
  0
+-1.200286852455682
  0
  0
  0
  0
  0
+ 2.142147343874559
  0
+ 0.5176775834478452
  0
  0
  0
+ 0.3600616894812302
  0
+ 0.2672811945379825
  0
+ 0.1663354456056821
  0
  0
  0
  0
+ 1.322223433879568
  0
  0
  0
  0
+-19.4930336369178
  0
  0
  0
  0
  0
+ 3.561528539945717
+-0.1164984481431163
  0
+ 0.475143755650158
  0
+-5.677337589744861
  0
  0
  0
@@ -9064,70 +6754,54 @@
  0
  0
  0
+ 0.1383495046941607
  0
+ 3.72338460736351
  0
  0
  0
  0
- 0.1941375998894964
- 0.1530312978676857
--0.3634514165525199
- 0.2386496998375353
- 0.1590283983884435
--0.2984634891109203
- 0.1802792467321077
- 0.1896081751947142
--0.2294668606929837
- 0.3342218634277369
- 0.2185164886551508
--0.1764409599232029
- 0.310562298436409
- 0.2115967919095705
--0.0432079492395778
- 0.613967449886055
- 0.1604405073759448
- 0.04095631180024667
- 0.560260893012
- 0.1284297882436596
- 0.2360151217341434
- 0.7911017093314525
--0.03628312733163946
- 0.2820413843886657
- 0.62050943143644
--0.3099380207357701
- 0.09779757469535742
- 0.3750664810887352
--0.2851086978588858
- 0.09089457391122706
- 0.1458609083193354
+ 1.029503712841592
  0
  0
  0
  0
  0
+ 0.5526033179262732
+-0.2922509856369307
  0
  0
  0
  0
+ 0.02192571142437446
  0
  0
+-2.275862499445515
  0
  0
+-5.956539869302246
+ 0.5577919710366753
  0
  0
+ 0.9149456986467562
  0
  0
  0
  0
  0
  0
+ 12.85421515349177
  0
  0
  0
  0
+ 0.06855743856753654
+-0.891803654255144
  0
  0
+ 1.216191370381009
  0
+-0.2525162088130185
  0
  0
  0
@@ -9135,31 +6809,47 @@
  0
  0
  0
+-0.119819991477916
  0
  0
  0
  0
  0
+ 2.429682620515801
+-1.305779481269262
  0
  0
  0
  0
  0
+-1.273832601871327
  0
  0
+-0.4722256149873065
+ 0.1694239131550563
  0
  0
  0
  0
  0
+ 0.4909029282327266
+ 0.2809566080112832
  0
  0
  0
+-0.5049177529917077
  0
  0
+-0.7131852463338735
+-2.280203274456158
  0
  0
+-0.7076217123728984
  0
+-2.782264996918608
+-1.097141367893753
+-0.9468800068107297
+ 1.108086285509248
  0
  0
  0
@@ -9168,44 +6858,17 @@
  0
  0
  0
+-0.6030397649317234
  0
- 0.1694476251128947
- 0.1282700605617779
--0.3148267322943111
- 0.2549056349957726
- 0.1822275523933939
--0.3354018885712621
- 0.2492183390519549
- 0.2012637738269327
--0.2702076707145339
- 0.3361539745230698
- 0.268328036367336
--0.2295929670769196
- 0.2788575284115253
- 0.3164264639471738
--0.1859194963287544
- 0.3148759186469441
- 0.3739507610034065
--0.1283164502812031
- 0.1348806607770446
- 0.5277291195618026
--0.07370015885157938
--0.0362831273316396
- 0.6094694673517912
- 0.05033539349637208
--0.2241716300896296
- 0.3292246131175449
--0.001327696551146982
--0.1631244840796518
- 0.1462610708547624
--0.0880716826554595
--0.05084222313566003
  0
+-0.1146452475178463
  0
  0
  0
  0
  0
+-0.1962433153909341
+ 0.1991815419698557
  0
  0
  0
@@ -9216,31 +6879,47 @@
  0
  0
  0
+ 6.049189216433742
  0
  0
+ 0.09982078399404117
+-1.495186107409558
  0
  0
  0
+-4.321278976118168
  0
+ 0.4774583279972811
  0
  0
  0
  0
  0
  0
+-0.3662508736483057
  0
  0
+-0.03181285309435417
  0
+ 0.4913051667564982
  0
+-0.1657547275193968
  0
+ 0.4006004973474178
+-0.4370423396175648
  0
  0
+ 0.7448841387304859
  0
+-2.512065448579293
  0
  0
  0
  0
+ 0.1834078614552195
+-0.5340080834170147
  0
+ 0.6592265199085
  0
  0
  0
@@ -9249,10 +6928,12 @@
  0
  0
  0
+ 0.2906414221067094
  0
  0
  0
  0
+ 0.1585261181629194
  0
  0
  0
@@ -9262,44 +6943,15 @@
  0
  0
  0
+-1.282585577585656
  0
+-0.587196279763097
  0
  0
  0
  0
  0
  0
--0.08525837716883324
--0.03857345103301743
- 0.1069307706764438
--0.1198504957561022
--0.1603205548378792
- 0.1591472052471194
--0.1815226001647322
--0.1146622438877439
- 0.1279688132761465
--0.1102612328810432
--0.2241473340834236
- 0.1200887740708188
--0.1897406562988194
--0.1396021454401372
- 0.260604739205378
- 0.03923376155985381
--0.1387737861614789
- 0.3275782836358209
--0.08514958147343689
--0.00767068754183741
- 0.7215900091107786
- 0.2820413843886656
- 0.05033539349637216
- 0.7730567567488885
- 0.2518931547043468
--0.16584535022333
- 0.2287007728240014
- 0.2735891883985518
--0.2657407115706805
- 0.1630804698926822
- 0.05758270964011017
  0
  0
  0
@@ -9307,13 +6959,17 @@
  0
  0
  0
+ 0.240278976404221
  0
+ 0.2102593351291137
  0
  0
  0
  0
  0
+ 0.1132484587356649
  0
+ 0.5660472430497463
  0
  0
  0
@@ -9322,15 +6978,24 @@
  0
  0
  0
+-0.5320305596335215
  0
+ 0.08950015573930774
+-0.36066996993926
+-0.9469273217143011
  0
  0
+-0.9597485208035038
  0
  0
  0
  0
+ 17.45460058280481
+-0.4584672381372454
  0
  0
+ 0.4487193183395825
+ 2.509000915737849
  0
  0
  0
@@ -9341,25 +7006,33 @@
  0
  0
  0
+-1.060832891790064
  0
  0
  0
  0
+ 9.257999137863095
  0
  0
  0
  0
+-0.7243493585571089
  0
+ 0.03529861642450902
  0
+-0.197077341093214
  0
  0
  0
  0
+ 0.8954072727507604
  0
  0
  0
+ 0.7698355532451218
  0
  0
+ 0.4023051624638282
  0
  0
  0
@@ -9369,72 +7042,60 @@
  0
  0
  0
- 0.07763710847443513
- 0.05987160169092721
--0.1511978893737019
- 0.07231101738229902
- 0.05573865439435653
--0.1024358635594179
- 0.04415913393260545
- 0.06431678991150819
--0.07203472188893269
- 0.1091147384461191
- 0.09142020979320496
--0.04192506280389351
- 0.1245066736253398
- 0.05086153584988306
- 0.05584547433912342
- 0.3247158699508952
- 0.03878918938047023
- 0.11828287850483
- 0.3858534576002275
--0.09127922678833718
- 0.2506840721731568
- 0.6205094314364399
--0.2241716300896294
- 0.2518931547043469
- 0.6983538139068247
--0.1609922250691227
- 0.1716318115067201
- 0.3823553951302308
--0.1515622134131721
- 0.1533940149729053
- 0.2438919615712385
  0
  0
+-0.09399833638321142
  0
  0
  0
+-1.262600486902211
  0
  0
+-4.160847976544219
+ 1.932346836413282
  0
  0
  0
  0
+-0.104816819255623
+ 0.5714514098133883
  0
  0
  0
+ 0.04443077418036852
  0
  0
  0
  0
+ 0.5585053993437328
+-0.5454601487542008
  0
+ 1.324030976848829
  0
+ 6.820272608146636
  0
  0
  0
+ 3.378335188162803
  0
  0
+ 0.1774125573432427
+-0.1296176597501693
  0
  0
  0
  0
  0
  0
+-34.08189520105155
+ 0.4836512065261352
+-0.03366934059915522
  0
  0
  0
+ 0.9030043282988861
  0
+ 0.3403170399982783
  0
  0
  0
@@ -9442,114 +7103,134 @@
  0
  0
  0
+ 0.6418261333430858
+ 0.5930125797702109
  0
  0
+-0.2653658892281105
+-2.125054195110796
+ 1.305277886752463
  0
  0
  0
  0
  0
+ 5.902457582414605
  0
  0
  0
+-0.2640713338373876
+-1.86205456848709
  0
  0
  0
  0
+-0.06446372278254772
  0
+ 2.458353208397256
  0
  0
+-1.056033522128867
  0
  0
  0
  0
+-95.25352685374895
  0
+-0.4401007194651089
  0
+ 10.54678914248936
  0
+-1.187019882929189
+-3.260297671414309
+-4.702470568084649
+ 0.6662402585162309
+ 0.2644209906260584
+-3.259923607441013
+-3.76578016858871
+ 0.03267157280521491
+-2.131586777350046
  0
+-8.901261451467073
  0
  0
- 0.05021469962704999
- 0.0208471471346901
--0.08288829368617638
- 0.05837725040380706
- 0.0994590858395821
--0.124743075415675
- 0.1284581338612232
- 0.05912605744743366
--0.09427041337908477
- 0.06923896918067633
- 0.1854752097024152
--0.07371581265774065
- 0.1152230356974949
- 0.1253388040979826
--0.09993378287707406
--0.0557810640086056
- 0.2761999416394266
--0.06990500583508999
- 0.001683379216170666
- 0.1972640704656796
--0.2228487297498827
--0.3099380207357703
- 0.3292246131175449
--0.1658453502233301
--0.1609922250691228
- 0.6680804932146648
- 0.2141622513371803
--0.2617433031212513
- 0.4582751949587536
- 0.1016550286884798
- 0.06311272365368008
  0
+-0.9781881107013581
  0
  0
  0
+ 0.0355610885400901
  0
+ 0.8186365257853162
  0
  0
  0
+ 0.7195530488757154
  0
  0
  0
  0
  0
+ 0.2751899949960999
  0
  0
+-0.2396524683759211
  0
  0
  0
  0
  0
  0
+-0.06359004984786285
+ 0.3971337000403037
+ 1.62571743762704
  0
  0
  0
+ 25.65173295550441
+ 1.049947484834283
+ 1.013097891999924
+-6.833222268699098
  0
  0
+-0.838017445959459
  0
  0
+-0.007615529050443952
  0
+-1.586085318688383
+-1.205832076872552
  0
  0
  0
+ 0.2744803035698759
+-0.3635936868167469
+-0.08332371375716639
  0
  0
  0
  0
  0
+-3.541811802794103
  0
  0
  0
+ 6.843791675518111
+-1.749448469749449
  0
  0
+-0.06740889074050756
  0
  0
  0
+ 0.9163098612251073
  0
  0
  0
+ 0.4270001411141601
  0
  0
+-0.08503909842688996
  0
  0
  0
@@ -9559,6 +7240,8 @@
  0
  0
  0
+-0.6910793657521033
+ 1.736013302379081
  0
  0
  0
@@ -9569,42 +7252,14 @@
  0
  0
  0
- 0.0289777634485793
- 0.01285486683690303
--0.05039018279692284
--0.03724955946653333
- 0.003835147600739779
- 0.006518743288640109
- 0.00329623280387221
--0.03881636856538331
- 0.02420963780642721
--0.01053096306178367
- 0.03427800736273548
- 0.04608199590334629
- 0.05785913100893915
--0.06301454853342996
- 0.1122054283320986
- 0.06536617451567721
- 0.06165557043280655
- 0.1792443778623837
- 0.1800465025360133
--0.09338105895639466
- 0.1950920993561757
- 0.09779757469535734
--0.001327696551146852
- 0.2287007728240014
- 0.17163181150672
- 0.2141622513371803
- 0.5708554264117766
--0.02109506277249117
- 0.1095634955232319
- 0.4801043219564997
- 0.01777180131348255
  0
  0
+-0.4136598313956726
  0
  0
+ 0.6812318574308043
  0
+-0.06412392360262623
  0
  0
  0
@@ -9613,17 +7268,21 @@
  0
  0
  0
+ 0.2376019273086113
  0
  0
  0
+-5.395073155760457
  0
  0
+ 1.257826547723061
  0
  0
  0
  0
  0
  0
+-0.0903793043103544
  0
  0
  0
@@ -9634,27 +7293,46 @@
  0
  0
  0
+ 0.5242906044798107
+-0.01742342683510349
+ 0.5231441833401839
  0
+ 0.9252180529033944
+-0.08404744906999868
  0
  0
+ 0.1982211295659624
  0
  0
+-0.7134922556759923
  0
  0
+ 0.3497113364974072
  0
+-1.92262627016739
  0
  0
  0
+ 0.3166988911249562
  0
+-0.01537370047486854
+-0.5507332504717685
+-0.9548093407857391
+ 0.1371119018936899
  0
+-0.3250900736454639
+ 0.2874218466869001
  0
  0
  0
  0
+ 0.5359569272752117
  0
  0
  0
  0
+ 0.1352042087298128
+-0.2128278143785792
  0
  0
  0
@@ -9662,44 +7340,22 @@
  0
  0
  0
+ 0.06891972170365385
+-0.7542786953052235
  0
  0
  0
+-0.2645897690731321
  0
  0
  0
+ 2.128438526907922
  0
  0
  0
  0
+-0.24271016935905
  0
--0.04566348062562483
- 0.0379275287162404
--0.05775740904643421
--0.01564349048741974
- 0.02556320812416608
--0.01750943907816571
--0.0726109741676576
- 0.02347266155426717
--0.04680129780172409
--0.04293234524718757
- 0.07918732842620622
- 0.1130638194934263
--0.1028214206699394
- 0.09735208451741834
- 0.08953428778153637
--0.09553929030576268
- 0.2726669512078172
- 0.3735568212132534
--0.1660144523261445
- 0.2753282873139399
- 0.3823929710457864
--0.2630281908177792
--0.02080771010652737
- 0.6040614072541741
--0.2829509625541405
--0.01184841718572921
- 0.3758800563151173
  0
  0
  0
@@ -9712,47 +7368,71 @@
  0
  0
  0
+ 0.254861765794941
  0
  0
+-0.4076076943047222
+ 27.36762208367834
+-0.24613440449224
  0
  0
  0
  0
+ 0.4608058213821488
+ 0.7567050110937079
  0
  0
+-0.4317127994858896
  0
  0
+-0.9325312804208741
  0
  0
+-0.6789861663591337
  0
  0
  0
+ 0.0295067305371026
+ 0.7551902107307704
  0
  0
+ 0.5513385229268563
  0
+ 0.5985994011157888
  0
  0
+ 0.2576260871928632
+-0.1940614202945517
+ 0.5455890301580043
  0
  0
  0
  0
  0
+-1.836129011512891
  0
+-4.753161586852461
  0
  0
  0
  0
+ 0.78745677826563
  0
  0
+ 0.3264239362936548
+ 0.8990131744412597
  0
  0
  0
  0
+-0.1144378344686671
+ 3.407851160953469
  0
  0
  0
  0
  0
+-0.06560238686281092
  0
  0
  0
@@ -9760,46 +7440,24 @@
  0
  0
  0
+-0.2630762849451005
+ 1.686505625919402
  0
  0
  0
  0
+ 0.7590373052622864
  0
  0
+ 1.148711079483203
  0
+ 14.04204720243525
  0
  0
+-0.06882562324756232
  0
  0
  0
--0.001530123680504339
- 0.06138209634332941
--0.04921490598370785
- 0.07616886901758949
- 0.02129357460706422
--0.04254970430045962
--0.006864181738730189
- 0.1090170817378964
--0.0407050960119527
- 0.05011986493583097
- 0.04639031323453931
--0.1117051256277355
--0.1339306856731421
- 0.1377928742575085
--0.1209253686340196
--0.0444857792359557
- 0.03762937336203414
--0.3013742648064107
--0.2850945261040383
- 0.1471761679486571
--0.2667134438995792
--0.1517965786804467
- 0.4592619000447985
- 0.1098297252243717
--0.2828904127971424
- 0.5530827656728311
- 0.1507590581644274
- 0.10591244040782
  0
  0
  0
@@ -9816,17 +7474,24 @@
  0
  0
  0
+ 0.4531083418190276
  0
  0
  0
+ 1.473570981887633
+-5.604186362510557
  0
  0
+ 0.3155913226765344
  0
  0
  0
  0
  0
  0
+-1.908889933764057
+-18.51673226379179
+ 0.6112558741953972
  0
  0
  0
@@ -9834,18 +7499,30 @@
  0
  0
  0
+-0.0848859884866955
  0
  0
+ 3.768590045274363
  0
+ 0.7733918309026047
  0
  0
  0
  0
+-0.3954332005105799
  0
  0
+-0.6673923776817171
  0
  0
+ 2.25911534939347
  0
+ 0.660611056293277
+-0.1812213554674956
+-0.04527142827977262
+ 2.006619764943415
+-0.1279086760711655
+-1.22903327886986
  0
  0
  0
@@ -9853,16 +7530,23 @@
  0
  0
  0
+ 1.208386377003379
  0
  0
+ 0.2446803420797232
  0
  0
  0
  0
  0
  0
+-0.725799469530091
  0
+-0.7765848593991672
+ 2.873455108604292
+-0.07617763534179756
  0
+ 1.226367682399656
  0
  0
  0
@@ -9871,39 +7555,14 @@
  0
  0
  0
+ 1.573847901789428
  0
--0.07046125287820829
--0.02124888069013417
- 0.05327929423378012
--0.02907097440307503
--0.06117285486879783
- 0.05500178371705559
--0.05818668072887905
--0.01516496070557581
- 0.06403230485631434
- 0.01119657081226274
--0.10618270092943
- 0.09950733362002433
- 0.005784351463719437
--0.01962209562624816
- 0.1400758122708756
- 0.1288314483647297
--0.172884710641278
- 0.1409219079030258
- 0.08905688434501095
--0.0897601322625443
- 0.1633839788563913
- 0.1526600485696558
- 0.1013858056460186
- 0.480256586823653
--0.01259449546210411
- 0.1507590581644274
- 0.499968975340181
- 0.05332535222879701
  0
  0
  0
  0
+-0.06649162841140518
+ 1.107584462866992
  0
  0
  0
@@ -9911,38 +7570,53 @@
  0
  0
  0
+ 4.244401079135958
  0
+-17.87513163308068
  0
  0
+-1.544149723335144
  0
  0
  0
  0
  0
  0
+ 1.298819764872114
  0
+-0.008282862618161422
  0
  0
  0
  0
+-0.4698645726960388
+-8.480466952694655
+-0.4173397491301559
  0
  0
  0
  0
+-0.7716645856393914
  0
+-0.2465528837629267
  0
  0
  0
  0
  0
+ 0.3482376100922915
  0
  0
  0
+ 3.880617777440751
  0
  0
+-2.005286131021399
+-1.303448776542771
  0
  0
  0
+-3.4193362880441
  0
  0
  0
@@ -9957,6 +7631,7 @@
  0
  0
  0
+-1.059385994677692
  0
  0
  0
@@ -9971,32 +7646,2357 @@
  0
  0
  0
+ 1.389749416279727
+ 13.40284755787324
  0
+ 0.2379509723991778
  0
  0
  0
  0
- 0.003482419948602756
--0.007868364638015095
--0.01945375472515987
- 0.007015681720708732
--0.009616795547391123
--0.01221821489264071
--0.001976620367168366
--0.01018529306494946
- 0.009526661447888158
--0.001990096912532073
--0.003390217053039036
- 0.04176437415070538
--0.05436884487422029
- 0.0344871376772762
- 0.1454791381551594
--0.05186778918641916
- 0.05816285063651524
- 0.2440875287260171
- 0.06267899271701106
- 0.01778222922550598
- 0.3759966969856117
- 0.1057756726063925
- 0.05350522057245403
- 0.4599302829374405
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.04365241008911763
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 2.219815557691363
+ 0
+ 0
+ 0
+ 2.535174794619011
+ 0
+ 0
+ 1.352789106934568
+ 0
+ 0
+ 1.586202951336093
+ 0
+ 0.5172361930188167
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.8573618771372511
+ 0
+ 0
+ 0
+-0.3706547554105325
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-1.906339951884944
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1371823955417805
+ 0.8217015084447183
+ 0
+ 0.7043108672951162
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.0952012
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.07786373751854546
+ 0
+ 0
+-0.04768267017453302
+ 0
+-0.354295538146551
+-1.871363032703216
+ 0
+ 0
+ 1.926458816072536
+ 0
+ 0
+ 0
+-0.03552216366667377
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1.371364430695413
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.2153633109984628
+ 0
+ 0
+ 0.4223110646342952
+ 1.430937908948884
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.08570651009198732
+ 0
+ 0
+ 1.704105254682967
+ 0.4572188951355719
+ 0
+-0.174391801024029
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-1.780016805708248
+ 0
+-0.5033127810949863
+ 0
+ 0
+ 3.348341099969325
+-0.1413143117226115
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 2.601973607696656
+ 0
+ 0
+ 0
+ 0
+-0.01096936254525321
+-0.411747449414281
+ 0
+ 0
+ 0
+ 0
+ 0.1911840919183682
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.02507437889959847
+ 0
+-1.920502223064878
+ 0
+ 1.485265811146348
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.1270227091236665
+ 0
+ 0
+ 0
+ 0
+ 0.9319905498951758
+ 0
+ 0
+ 0
+ 0.06355531248568379
+ 0
+ 0
+ 0
+ 0
+ 631.8970732533545
+ 0
+-0.06941261366326315
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.9745960818564167
+ 0
+-0.1319399145673239
+ 0
+ 0
+-0.1712060015170429
+ 0
+ 0
+ 0
+ 0.2199338815009368
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.2951937985353986
+ 1.681449903106215
+ 0
+-0.1874407716775586
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.09141767654559446
+ 0
+ 0
+-0.4518509907644195
+-0.1468542372417183
+-1.075467361334804
+ 0
+ 0
+ 0
+ 0
+-1.955437134594413
+-0.6900778960186711
+ 0
+ 0
+ 0
+-1.449028368203642
+-0.5433146573681179
+-0.4085400611332851
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.2012560825672974
+ 0.4787618884403176
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-8.631168098229734
+ 0
+ 0
+ 0
+ 0
+-0.8104835496027143
+-1.646720809557628
+ 0.8497248224834173
+ 1.103496050793216
+-0.2870479812674109
+ 0
+ 0
+ 3.759113878885802
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.1337480208931563
+ 0
+ 0
+ 0
+-0.3528623797300455
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.0531751278621955
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.4197638202775679
+-1.162312655274553
+-0.2186815719856952
+ 0
+ 0
+ 0.5768419056861613
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.9876302547108818
+ 9.178415714553781
+ 0
+ 0.486443770275648
+-1.074427393258677
+ 0
+-0.7190429399116103
+ 0
+ 0
+-0.004654251984783326
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.08279607355872896
+ 0
+ 2.428085286670281
+ 0
+-0.290905267655203
+ 0.1784003316654723
+ 0
+ 0
+ 0.3937972051871597
+ 0
+ 0
+ 0
+ 0
+ 2.041969844109952
+ 0
+-0.6632670925089514
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.04730653973583668
+ 0.7340408428608161
+ 0
+ 0
+ 0
+ 0.5631018585208183
+ 0.9022015166062511
+-0.4946213370473764
+ 3.116402896865955
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.4753854557031529
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.5776629841543006
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.4693467331486799
+ 0.5092722521912728
+ 0
+ 0
+-0.5751418088375293
+ 0
+ 1.110180300043974
+ 0
+ 0
+ 0
+-0.2052398029192016
+ 0
+ 0
+ 0
+ 0
+ 0.3441802634664664
+ 0
+ 0
+-0.06054885994923487
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 14.14678012942133
+ 0
+ 0
+-0.3774029308910528
+-0.9181606392989132
+-0.4971518532410875
+ 2.060768053484165
+-0.3705989448248495
+ 0
+ 0
+ 0
+ 0
+ 0
+-1.47720358563491
+ 0
+ 0
+ 0
+ 0
+ 0.4942864830129788
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-5.980874489912058
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.7527716678327022
+ 2.817949083040197
+ 0
+ 0
+-0.4396265411868724
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-48.58940723722201
+ 0
+ 0
+-0.3877076367092464
+ 0
+ 0
+-0.652800681008756
+ 0
+ 0
+ 0.2261910205188926
+ 0.6653039312906668
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.6725639123466175
+-0.9328676887049783
+ 12.82686028696979
+ 0
+-0.1825224701986439
+ 0
+-0.2895567290213468
+-0.5793400426562111
+ 0
+-0.6640461395506864
+-0.2270508046043917
+-0.6412166121034713
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1126563621331092
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-1.330630854295177
+ 0
+ 0
+ 0
+ 0.1647035387110507
+ 0.1524236052870568
+ 0.1265520227358675
+-4.496978640717864
+ 0
+ 0
+ 0.05794659470297663
+ 0
+ 0.982493550693461
+ 0
+ 0
+ 0.6567604474650559
+ 0.01440349345695066
+ 0
+ 0
+ 0
+ 11.04811591081289
+ 0
+ 0
+ 0
+ 0
+-0.7645032681110474
+ 0
+ 1.719948735368507
+ 0
+-0.3079470598149933
+ 1.085667939008959
+ 0.3483428365886299
+ 0
+ 0
+ 0
+ 0
+ 0
+-2.930424196970102
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.3606349932057697
+ 0
+ 0
+ 0
+-1.957148493090422
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-2.008276092090168
+ 0
+ 0
+ 0
+-0.00749535856390822
+-0.02428039420676276
+ 0
+ 0.3930395680367711
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 14.39737910881446
+ 0
+ 0
+ 0
+ 0
+ 0
+-1.292935206190298
+ 0
+ 0
+-0.0248949634914246
+ 0
+ 0
+-1.125024319777534
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.4927965488776805
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1869230936462014
+ 0
+-0.6630828683624668
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.4594092525851398
+ 0
+ 0
+-0.3202097255038425
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.22447
+ 0
+ 0
+ 0.5910169655671389
+-0.4836597271883145
+ 0
+-0.5531825210060894
+ 3.721493044549171
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 2.112397477340145
+ 0
+-4.7721984738587
+ 0
+ 0
+-0.5550571596401548
+-0.566879658670128
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.01808279498347775
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.3606017229961584
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.07680287577569114
+ 0
+ 0.1295568785771593
+ 0
+ 0
+ 0
+-11.95421298269813
+-0.604529538866787
+ 0
+ 0
+-0.3184720155498334
+ 0
+ 0
+ 0.1793451921501975
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1.556018466402506
+-0.4667126353644137
+ 0
+-0.4886251029536635
+ 0
+ 0
+ 0
+ 0
+-20.5762341697062
+ 0.9054584447145799
+ 0
+-0.4335941386675467
+-0.231865506519269
+ 0.09267561124885154
+ 0
+-0.3224450001651057
+ 1.946623869033397
+ 0
+ 0
+ 0.1813040482362534
+ 0
+-1.615596155779889
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.2805840358472876
+ 0
+ 0.1752426257870506
+ 0
+ 0
+ 0
+ 0.4245792310373214
+ 0
+ 0.5652421411245472
+ 0
+ 4.342363857508019
+ 0
+ 0
+ 0
+ 0
+ 0
+-1.267729134291342
+ 0
+ 0
+ 0.5097122755503997
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 2.269051532751651
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.7906434423290707
+ 0
+ 1.211667301441654
+ 0
+ 0.5667500477593946
+ 0
+ 0
+ 0
+ 0.2352951235203545
+ 0
+ 18.02523074688196
+ 0
+ 0
+ 0
+ 6.869350820322008
+ 0
+-0.02239742062022338
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.6942810817858166
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.6717730520651083
+ 0
+ 0
+ 0
+ 0.9053323109763444
+ 0
+ 0
+ 0
+ 0.8083143391330183
+ 0
+ 18.95307669364243
+ 0
+-0.05881962894534767
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.2314600321165206
+ 0
+ 0
+ 0
+-0.1563748024625857
+-0.001188087501363268
+ 0
+ 0.6718259486609706
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.09505914867643134
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1068405874333421
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4.569792430389595
+ 0
+ 0
+ 0
+ 4.993089268138056
+ 0
+ 0
+ 0
+ 0
+ 0.01474832937443083
+ 0
+ 0.2728264573828585
+-0.08423510041166489
+-0.2920139791774037
+ 0
+ 2.294522220982419
+ 0
+-0.4169884839112438
+-11.38550988688505
+ 0
+ 0
+ 0
+ 0
+ 0.5073898139195817
+ 0
+ 0
+-0.646334730261419
+ 0
+ 0
+-0.1016224973622556
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.4915017578748171
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.5427839248691851
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.1717333411874349
+ 0
+ 2.713951885471826
+ 1.996453627306153
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.2831911043270032
+ 0
+-0.001542827498603446
+-0.2603547531531951
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.987637150004415
+ 0
+ 0
+ 0.262065744936163
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-1.47447264974626
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.15333572570218
+ 0.3094455719060705
+ 0
+-0.5829498577817496
+ 0
+ 0
+ 0.4609346394178296
+ 4.328021312376961
+-1.805795280283172
+ 0
+ 2.764523895521742
+ 12.15008450166943
+ 0
+ 0
+-1.739598619461891
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.008670708193809626
+ 3.003029141912023
+-0.09571484050607278
+ 0
+ 0
+ 0
+ 0
+ 0
+ 2.998238679478707
+ 0.1328472941945366
+-0.2909476355962676
+-0.174970057795891
+ 0
+ 0
+ 0.1910965456168723
+ 0
+ 0
+ 0
+-0.2727953979361796
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.1409298714812179
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.2476230023872065
+ 0
+-1.269752782652631
+ 0
+ 0.03740787628133018
+ 0
+ 0
+ 0
+ 0
+ 1.937298800300633
+ 0
+ 0
+-0.7649201907844544
+ 0
+ 0
+ 0.07867804647648058
+ 0
+ 0
+ 0
+ 0.4108565224750267
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-1.509400641752963
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3.09388010063306
+ 0.7269515533548975
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-2.683452264523992
+ 0
+ 0.1610736998352869
+ 0
+ 0
+ 0.4530002040490377
+ 0.0940423820304188
+ 0
+-0.2523436531574056
+-0.3146854565835709
+ 0
+-0.5438896078024471
+-0.5481997387936354
+ 0
+ 0
+ 0
+-0.3529512841306343
+ 0.01819211316772903
+ 0.3534341354990801
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.400508770205981
+ 0
+-0.02529498154724328
+ 0.01190877399766866
+ 0
+ 0
+ 0
+ 0.5985093361405944
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.720375393357375
+ 0
+ 0
+ 0.1274464096544585
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.003653887756514014
+-0.005073849378708361
+ 0
+ 0
+-0.8802391626152068
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.3218661687166185
+ 0
+-0.4904032546922169
+ 0
+ 1.72659614623605
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.2264047399303686
+ 0
+-0.8208743070956491
+ 0
+ 0
+-9.51534427481794
+ 0
+ 0.4213718633385596
+ 0
+-19.72682087323344
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.4694729102048924
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.5258119271459143
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3.820621745385362
+ 0
+ 0.1390174491737094
+ 0
+-0.3664113363388742
+ 0
+ 0
+ 0
+ 0
+-0.3885587216893042
+-0.1203394912747273
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.06414053528425237
+ 0
+ 0
+ 0.09289294530186366
+ 0
+ 0
+-0.8466475210571507
+ 0
+ 0
+-0.4678784812654621
+-2.808785268303473
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.6286916829564756
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.2855360076757998
+ 0
+ 0
+ 0
+-50.37032056490291
+ 0
+-0.8602511248033244
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-4.210390682638181
+ 0.06211066398950435
+-0.03632024552548954
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.01138833579768669
+ 0
+ 0.1220142412439677
+ 0
+-0.02347313472194225
+ 0
+ 0
+ 0
+-0.01237103396233737
+ 0
+ 0
+-0.08965046314421378
+ 0
+ 0
+ 0
+ 1.080724675852843
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.6212490534855771
+ 0
+-0.3399176535743508
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.03679826422258359
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.3195751411836361
+ 0.9154179302658173
+ 0
+-0.1150711320732581
+ 0
+ 0
+ 0
+ 0
+ 0.346131493789016
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.7576600189486671
+-0.824844165181285
+ 0
+ 0
+ 0
+ 0
+ 0.14962600744346
+ 0
+-2.748732521186326
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.5081557889752605
+ 0
+ 0
+ 0
+ 0
+ 0.2552804322082619
+-0.2745290629953129
+ 0
+ 3.122896313380039
+ 0
+ 0
+ 0
+ 0
+-2.920923705424083
+ 2.985171570816766
+ 0
+ 0
+ 0.5476907474663835
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-3.506693439210517
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.2345569520482285
+ 0
+ 0
+ 0.1476134575083644
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-18.85881257776057
+ 10.06109775574651
+ 0
+-0.5793006882210047
+ 0.2629820711089522
+ 0
+ 0
+-0.2742277127815957
+ 0
+ 0.131355853707775
+ 0
+ 0
+ 0.04771138396277713
+ 0
+ 0
+ 0
+-1.336706366462532
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.1949107366043034
+ 0
+ 0
+ 1.825493279082812
+ 0
+ 0
+-0.4385088336266119
+ 0
+-0.2743317451076697
+ 0
+ 0
+ 0
+ 0
+ 0
+ 5.442286337540354
+ 0
+ 0
+ 1.404220068296073
+ 0
+ 0
+ 0
+ 0
+-0.2579837173997316
+ 0
+-0.4450387478341629
+ 0
+ 0
+-1.670108626031611
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 5.798267914669685
+ 0
+ 0
+ 0.05658932251961987
+ 0
+ 0.4172779499635791
+ 0
+ 0.105300931833048
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.7484888061118221
+ 2.041850891755478
+ 0.7330715541422896
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1635745536362639
+-0.1783447111432833
+ 0
+-0.01494429797658691
+-1.036642363669109
+ 0
+ 0
+ 0
+-0.9786186554979159
+ 1.475192028122727
+ 0
+ 0.1415519025282671
+ 0
+-1.117981491915489
+ 0.8107534405518425
+ 0
+ 0
+-3.891722344659192
+-0.3299595865831923
+ 0
+ 0.03353894469956772
+-0.20972172091688
+ 0.2909925331783702
+ 0
+-4.374493439557915
+ 0
+ 0
+-0.1630634619639877
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.2856117907298797
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-2.88741370390883
+-0.09463873033654958
+ 0
+ 0.7989153733960048
+ 0
+ 0
+ 0
+ 0.1951193739747824
+ 0
+ 0
+ 0
+ 0.08351546533483424
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 68.37257976990995
+ 0
+ 0
+ 0
+-1.137612020508681
+ 0
+-0.1958510587693432
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.06938557191253593
+ 0
+-0.222839558492001
+ 0
+ 0.09291263597164898
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-3.745044048189886
+-0.1313473536250102
+ 0
+ 0
+ 0
+ 0.3458383157974664
+-1.544738243051305
+ 0
+ 0
+ 0
+ 0
+ 0
+-1.510079153083887
+ 0
+ 0
+ 0.171879670687663
+ 0.3878037114504239
+ 0
+ 0
+ 0
+ 0
+ 0
+ 2.567010692908583
+ 0
+-1.305291396721716
+ 0
+ 0
+ 0
+-0.9411906007959661
+ 0.5434008860134193
+ 0
+ 0
+ 0
+ 0.6226650835463924
+ 0.1482446671978628
+ 0
+ 0
+ 0
+ 0
+-1.455503868024161
+ 0
+ 0.1942377676290563
+ 0
+ 0
+ 0
+ 0
+-0.07492512892754151
+ 0
+ 0
+ 0
+ 0.9714305088747539
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.07904651089037804
+ 0.9157694617406652
+-0.1641815028685157
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.1254313797317232
+ 0
+-0.6905436226153474
+ 0
+-0.08360246413693795
+ 0
+ 0
+-0.5880038786799398
+ 0
+ 0
+ 0
+ 0
+ 0.6022766110942244
+ 0
+ 0.5148146000418764
+ 0
+ 0
+-20.48252707563957
+ 0
+ 0
+ 0.5575512912831021
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.01835850798101779
+-0.2644850972603794
+ 0
+ 0
+-0.09468032394924522
+ 0
+-3.102589977467122
+ 0
+ 0.3600538373706629
+ 0
+ 0
+ 0
+-1.034585556269956
+ 0
+ 1.039162471692296
+ 1.404015979975387
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.2834220750958322
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.06284581408869482
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.8100900574924416
+ 0
+ 0
+ 0.1207606143828642
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-13.01195159386746
+ 0
+ 0
+ 0
+ 0
+ 0.06609523597093495
+ 0.6788148252064654
+ 0
+ 0
+ 0
+-0.2245540166061343
+ 0
+-1.877669744282497
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.1043226421995354
+-0.1541676423463312
+-0.3083832389853993
+ 0
+ 0
+ 26.75044384818589
+ 0
+ 0
+ 0
+ 4.680978823377932
+ 0
+ 0
+-2.467146937828286
+-0.6034157196865881
+ 0.2456847071708288
+ 0.3648953118737308
+ 0
+ 0
+-0.7165533886314798
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.4146478223033953
+ 2.67153547865388
+ 0.4066037119393711
+ 0
+ 0
+ 0
+ 0
+-0.3044243086619978
+ 0
+ 0.05376234688607281
+ 0
+ 0
+ 0.08850532670384344
+ 0.06749428120635347
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.5114454170290706
+ 0
+ 0
+ 4.755777436485564
+-1.240394922245889
+ 0.008546368411881411
+ 0.3339486258385727
+ 0.3961449252235861
+ 0
+ 0
+-4.760115984646152
+ 0.9460435769805294
+ 0.06738988249241178
+-0.1241293530803617
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-3.941528010506792
+ 0
+ 0
+ 0
+ 0
+ 0
+-1.095864235879698
+ 0
+-1.643754348478478
+ 0
+-1.119532545273484
+-2.967482676626235
+-0.5254091503576083
+ 0.516258261247043
+ 1.582259202190788
+ 0
+ 0
+ 0
+ 0
+-0.5232137123210626
+-0.8578576545155802
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-4966.139203211767
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 7.810402306959599
+ 0
+-1.141048704919943
+ 0
+ 0
+ 0
+ 0
+-0.4974314046107617
+ 0
+ 0.02832075084311592
+ 0
+-1.004514081789116
+ 0
+ 0
+-62.26077547658917
+ 23.45803782680646
+ 0.04071433235873094
+ 0
+-0.2740057032473357
+ 0
+ 0
+ 0
+ 0
+-0.6634096867034206
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.05567752536680285
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4.476528587143142
+ 0
+-0.3102269985689736
+ 0
+ 0
+ 0
+-0.2120672325605232
+ 0
+ 0.1082684195035498
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.407383006921814
+ 0
+ 0
+ 0
+ 0
+ 0
+-1.802572821686524
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.261498978802622
+ 0.144742328702142
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.05436200145226892
+ 0
+ 0
+ 0.03123923729849884
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.5493282349642721
+-3.56605055765311
+ 0
+ 0.1189964332247844
+ 0
+ 0
+ 0.2712761050398067
+ 0
+ 0
+ 0.9649887849336986
+ 0
+-1.662202733759401
+ 0
+ 0
+-0.3417780740030298
+-0.825490096558209
+ 0
+ 0
+ 0.8613713427803134
+ 0
+ 0.102663973308054
+-0.118950085071405
+ 0
+ 0
+-0.3367690776431803
+ 0
+ 0
+ 0
+ 0
+ 0
+ 5.139357298646494
+-0.8646777626735802
+-0.1888400086466105
+-8.087815225955227
+-0.7858696596423445
+ 0
+ 0
+ 6.367438364151895
+ 0
+ 0.7140798553081277
+-0.008810645504535789
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1.483097156676205
+ 2.091303289750183
+ 0
+ 0
+ 0.321893877144759
+ 0
+ 0
+ 0
+ 0
+ 13.57001687135731
+ 0
+ 0.5451595807807043
+ 0
+ 1.501350698141659
+-0.06782249716495321
+ 0
+ 0
+ 6.512865467491789
+ 0
+ 0.1265142267536407
+ 0
+ 0
+ 0
+-2.164126884731795
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-1493.381090410497
+ 0
+-0.5528447891372515
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.4878802784931899
+ 29.39776799459366
+ 0
+ 0
+-6.246840660823246
+ 1.837422099503762
+ 0
+ 0
+ 0
+ 0.5438964907160825
+ 0
+ 1.809629679819767
+-0.2088189637961629
+ 0
+ 0.2371944576606236
+ 0
+-4.576313887759413
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1.004436994198865
+ 0.427447789557515
+ 0
+ 0
+ 0.04345421782712366
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-3.785231348831975
+ 0
+ 0
+ 0
+ 3.484380229783484
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.3581986366645341
+ 0
+ 0
+ 0
+ 0.03318181775015192
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-74.78907765408408
+ 0
+ 0
+ 0
+ 1.944787624214348
+ 0.09929293016417996
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.116408775050883
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.2090617426284433
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.01712180883962215
+ 0.113613167796337
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1.689511721776201
+-0.4182393753066492
+ 0
+-1.154405689010111
+ 0
+ 0.1473834929579108
+ 0
+ 0.1434874561955266
+-0.05794164273893747
+-0.01094366369749314
+ 0
+ 0
+ 0
+ 0
+ 3.888280276242147
+ 0.8505678665757467
+ 0
+ 2.402259310185026
+ 0
+ 0
+ 0
+ 0
+-0.4618068812711437
+-0.1093509337748507
+ 0
+-0.1471360397257355
+ 0
+ 0
+-0.1344705267155689
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-4.025016515700098
+ 0
+-0.330133694982991
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+-0.5026319300955789
+ 0
+ 0.2922854120271939
+ 0
+ 0
+ 0
+ 0
+ 41.91848303091628
+ 0
+ 0
+ 0
+ 0
+ 0
+-3.645117646421415
+ 0
+ 0.935149590315432
+ 0
+ 0.2157329735790116
+ 1.888239187045813
+ 3.215581067843307
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.2716732938603457
+ 0.02792966108123637
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0.009161683183842015
+ 0
+ 0
+ 0
+ 34.84420319112557
+ 0
+ 0
+ 0
+ 0
+ 0.3537999629297381
+ 0
+ 1.222501112809391
+ 0
+ 0.04303850878046313
+ 0
+ 0
+ 0
+-0.4616341244191867
+ 0
+-10.73295200880771
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1.251021320332088
+-0.1752269560196747
+ 0
+ 0
+ 0.1440088296602519
diff --git a/omp/preconditioner/isai_kernels.cpp b/omp/preconditioner/isai_kernels.cpp
index 424d50424d9..192af7089c2 100644
--- a/omp/preconditioner/isai_kernels.cpp
+++ b/omp/preconditioner/isai_kernels.cpp
@@ -138,6 +138,7 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                     static_cast<size_type>(i_size),
                     static_cast<size_type>(i_size));
                 std::fill_n(trisystem_ptr, i_size * i_size, zero<ValueType>());
+                IndexType k = zero<IndexType>();
 
                 for (size_type i = 0; i < i_size; ++i) {
                     const auto col = i_cols[i_begin + i];
@@ -146,12 +147,14 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                     forall_matching(
                         m_cols + m_begin, m_size, i_cols + i_begin, i_size,
                         [&](IndexType, IndexType m_idx, IndexType i_idx) {
+                            if (m_cols[m_idx + m_begin] < row && col == row)
+                                k++;
                             trisystem(i, i_idx) = m_vals[m_idx + m_begin];
                         });
                 }
 
                 // solve dense triangular system
-                trs_solve(trisystem, rhs, row);
+                trs_solve(trisystem, rhs, k);
 
                 // write triangular solution to inverse
                 for (size_type i = 0; i < i_size; ++i) {
@@ -240,12 +243,97 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_TRI_INVERSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+inline IndexType choose_pivot(IndexType block_size, const ValueType *block,
+                              size_type stride)
+{
+    IndexType cp = 0;
+    for (IndexType i = 1; i < block_size; ++i) {
+        if (abs(block[cp * stride]) < abs(block[i * stride])) {
+            cp = i;
+        }
+    }
+    return cp;
+}
+
+
+template <typename ValueType, typename IndexType>
+inline void swap_rows(IndexType row1, IndexType row2, IndexType block_size,
+                      ValueType *block, size_type stride)
+{
+    using std::swap;
+    for (IndexType i = 0; i < block_size; ++i) {
+        swap(block[row1 * stride + i], block[row2 * stride + i]);
+    }
+}
+
+
 template <typename ValueType, typename IndexType>
 void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
                               const matrix::Csr<ValueType, IndexType> *mtx,
                               matrix::Csr<ValueType, IndexType> *inverse_mtx,
                               IndexType *excess_rhs_ptrs,
-                              IndexType *excess_nz_ptrs) GKO_NOT_IMPLEMENTED;
+                              IndexType *excess_nz_ptrs)
+{
+    using std::swap;
+    auto general_solve =
+        [](const range<accessor::row_major<ValueType, 2>> system,
+           ValueType *rhs, const IndexType k) {
+            const IndexType size = system.length(0);
+            if (size <= 0) {
+                return;
+            }
+            // RHS is the identity: zero everywhere except for the diagonal
+            // entry
+            std::fill_n(rhs, size, zero<ValueType>());
+            rhs[k] = one<ValueType>();
+
+            // fill in transposed system
+            ValueType *transposed_system = new ValueType[size * size];
+            for (auto row = 0; row < size; row++) {
+                for (auto col = 0; col < size; col++) {
+                    transposed_system[row * size + col] = system(col, row);
+                }
+            }
+
+            // solve transposed system
+            std::vector<IndexType> perm(size, {});
+            std::iota(begin(perm), end(perm), IndexType{0});
+            for (IndexType col = 0; col < size; col++) {
+                const auto row =
+                    choose_pivot(size - col,
+                                 transposed_system + col * (size + 1), size) +
+                    col;
+                swap_rows(col, row, size, transposed_system, size);
+                swap(perm[row], perm[col]);
+                swap(rhs[row], rhs[col]);
+
+                const auto d = transposed_system[col * size + col];
+
+                for (IndexType i = 0; i < size; ++i) {
+                    transposed_system[i * size + col] /= -d;
+                }
+
+                transposed_system[col * size + col] = zero<ValueType>();
+                for (IndexType i = 0; i < size; ++i) {
+                    for (IndexType j = 0; j < size; ++j) {
+                        transposed_system[i * size + j] +=
+                            transposed_system[i * size + col] *
+                            transposed_system[col * size + j];
+                    }
+                    rhs[i] += rhs[col] * transposed_system[i * size + col];
+                }
+                for (IndexType j = 0; j < size; ++j) {
+                    transposed_system[col * size + j] /= d;
+                }
+                rhs[col] /= d;
+                transposed_system[col * size + col] = one<ValueType>() / d;
+            }
+        };
+
+    generic_generate(exec, mtx, inverse_mtx, excess_rhs_ptrs, excess_nz_ptrs,
+                     general_solve);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_GENERAL_INVERSE_KERNEL);
diff --git a/omp/test/preconditioner/isai_kernels.cpp b/omp/test/preconditioner/isai_kernels.cpp
index f0fa382a9d1..cf2d905aae4 100644
--- a/omp/test/preconditioner/isai_kernels.cpp
+++ b/omp/test/preconditioner/isai_kernels.cpp
@@ -52,7 +52,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
-enum struct matrix_type { lower, upper };
+enum struct matrix_type { lower, upper, general };
 class Isai : public ::testing::Test {
 protected:
     using value_type = double;
@@ -89,9 +89,14 @@ class Isai : public ::testing::Test {
         auto nz_dist = std::uniform_int_distribution<index_type>(1, row_limit);
         auto val_dist = std::uniform_real_distribution<value_type>(-1., 1.);
         mtx = Csr::create(ref);
-        mtx = gko::test::generate_random_triangular_matrix<Csr>(
-            n, n, true, for_lower_tm, nz_dist, val_dist, rand_engine, ref,
-            gko::dim<2>{n, n});
+        if (type == matrix_type::general) {
+            mtx = gko::test::generate_random_matrix<Csr>(
+                n, n, nz_dist, val_dist, rand_engine, ref, gko::dim<2>{n, n});
+        } else {
+            mtx = gko::test::generate_random_triangular_matrix<Csr>(
+                n, n, true, for_lower_tm, nz_dist, val_dist, rand_engine, ref,
+                gko::dim<2>{n, n});
+        }
         inverse = clone_allocations(mtx.get());
 
         d_mtx = Csr::create(omp);
@@ -160,6 +165,29 @@ TEST_F(Isai, OmpIsaiGenerateUinverseShortIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, OmpIsaiGenerateAinverseShortIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 615, 31);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::Array<index_type> da1(omp, num_rows + 1);
+    auto da2 = da1;
+
+    gko::kernels::reference::isai::generate_tri_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
+    gko::kernels::omp::isai::generate_tri_inverse(
+        omp, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
+        false);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, r<value_type>::value);
+    GKO_ASSERT_ARRAY_EQ(a1, da1);
+    GKO_ASSERT_ARRAY_EQ(a2, da2);
+    ASSERT_EQ(a1.get_const_data()[num_rows], 0);
+}
+
+
 TEST_F(Isai, OmpIsaiGenerateLinverseLongIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 554, 64);
@@ -206,6 +234,28 @@ TEST_F(Isai, OmpIsaiGenerateUinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, OmpIsaiGenerateAinverseLongIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 695, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::Array<index_type> da1(omp, num_rows + 1);
+    auto da2 = da1;
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+    gko::kernels::omp::isai::generate_general_inverse(
+        omp, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, r<value_type>::value);
+    GKO_ASSERT_ARRAY_EQ(a1, da1);
+    GKO_ASSERT_ARRAY_EQ(a2, da2);
+    ASSERT_GT(a1.get_const_data()[num_rows], 0);
+}
+
+
 TEST_F(Isai, OmpIsaiGenerateExcessLinverseLongIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 518, 40);
@@ -268,6 +318,37 @@ TEST_F(Isai, OmpIsaiGenerateExcessUinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, OmpIsaiGenerateExcessAinverseLongIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 100, 51);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+    gko::Array<index_type> da1(omp, a1);
+    gko::Array<index_type> da2(omp, a2);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_nnz = a2.get_data()[num_rows];
+    auto excess = Csr::create(ref, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    auto dexcess = Csr::create(omp, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto de_rhs = Dense::create(omp, gko::dim<2>(e_dim, 1));
+
+    gko::kernels::reference::isai::generate_excess_system(
+        ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
+        excess.get(), e_rhs.get());
+    gko::kernels::omp::isai::generate_excess_system(
+        omp, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
+        da2.get_const_data(), dexcess.get(), de_rhs.get());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
+    GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
 TEST_F(Isai, OmpIsaiScatterExcessSolutionLIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 572, 52);
@@ -321,4 +402,31 @@ TEST_F(Isai, OmpIsaiScatterExcessSolutionUIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, OmpIsaiScatterExcessSolutionAIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 702, 45);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+    gko::Array<index_type> da1(omp, a1);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(omp);
+    de_rhs->copy_from(lend(e_rhs));
+    // overwrite -1 values with inverse
+    d_inverse->copy_from(lend(inverse));
+
+    gko::kernels::reference::isai::scatter_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get());
+    gko::kernels::omp::isai::scatter_excess_solution(
+        omp, da1.get_const_data(), de_rhs.get(), d_inverse.get());
+
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
 }  // namespace
diff --git a/reference/preconditioner/isai_kernels.cpp b/reference/preconditioner/isai_kernels.cpp
index e2a07bc487c..6949ce0bc4f 100644
--- a/reference/preconditioner/isai_kernels.cpp
+++ b/reference/preconditioner/isai_kernels.cpp
@@ -81,7 +81,7 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                       const matrix::Csr<ValueType, IndexType> *mtx,
                       matrix::Csr<ValueType, IndexType> *inverse_mtx,
                       IndexType *excess_rhs_ptrs, IndexType *excess_nz_ptrs,
-                      Callable trs_solve, bool general = false)
+                      Callable trs_solve)
 {
     /*
     Consider: aiM := inverse_mtx; M := mtx
@@ -284,43 +284,40 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
             }
 
             // solve transposed system
-            std::vector<IndexType> perm(size, {});
-            std::iota(begin(perm), end(perm), IndexType{0});
             for (IndexType col = 0; col < size; col++) {
                 const auto row =
                     choose_pivot(size - col,
                                  transposed_system + col * (size + 1), size) +
                     col;
                 swap_rows(col, row, size, transposed_system, size);
-                swap(perm[row], perm[col]);
                 swap(rhs[row], rhs[col]);
 
-                const ValueType d = transposed_system[col * size + col];
-                for (size_type j = col; j < size; j++) {
-                    transposed_system[col * size + j] /= d;
+                const auto d = transposed_system[col * size + col];
+
+                for (IndexType i = 0; i < size; ++i) {
+                    transposed_system[i * size + col] /= -d;
                 }
-                rhs[col] /= d;
 
-                for (size_type i = col + 1; i < size; i++) {
-                    auto scal = transposed_system[i * size + col];
-                    rhs[i] -= scal * rhs[col];
-                    for (size_type j = col; j < size; j++) {
-                        transposed_system[i * size + j] -=
-                            transposed_system[col * size + j] * scal;
+                transposed_system[col * size + col] = zero<ValueType>();
+                const auto rhs_key_val = rhs[col];
+                for (IndexType i = 0; i < size; ++i) {
+                    const auto scal = transposed_system[i * size + col];
+                    for (IndexType j = 0; j < size; ++j) {
+                        transposed_system[i * size + j] +=
+                            scal * transposed_system[col * size + j];
                     }
+                    rhs[i] += rhs_key_val * scal;
                 }
-            }
-
-            for (size_type i = size - 1; i >= 1; i--) {
-                for (size_type j = i; j >= 1; j--) {
-                    rhs[j - 1] -=
-                        transposed_system[(j - 1) * size + i] * rhs[i];
+                for (IndexType j = 0; j < size; ++j) {
+                    transposed_system[col * size + j] /= d;
                 }
+                rhs[col] /= d;
+                transposed_system[col * size + col] = one<ValueType>() / d;
             }
         };
 
     generic_generate(exec, mtx, inverse_mtx, excess_rhs_ptrs, excess_nz_ptrs,
-                     general_solve, true);
+                     general_solve);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
diff --git a/reference/test/preconditioner/isai_kernels.cpp b/reference/test/preconditioner/isai_kernels.cpp
index 070e824389c..a2e2f15be09 100644
--- a/reference/test/preconditioner/isai_kernels.cpp
+++ b/reference/test/preconditioner/isai_kernels.cpp
@@ -676,7 +676,7 @@ TYPED_TEST(Isai, ReturnsCorrectInverseLargeA)
     auto a_inv = isai->get_approximate_inverse();
 
     GKO_ASSERT_MTX_EQ_SPARSITY(a_inv, this->a_csr_large_inv);
-    GKO_ASSERT_MTX_NEAR(a_inv, this->a_csr_large_inv, 1e-10);
+    GKO_ASSERT_MTX_NEAR(a_inv, this->a_csr_large_inv, r<value_type>::value);
 }
 
 

From 6e0d6a2c6f3dbf82a05f87276eb8526e3cce82e9 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Tue, 8 Dec 2020 08:00:08 +0100
Subject: [PATCH 0507/1008] Make cuda implementation equivalent to ref

---
 common/components/warp_blas.hpp.inc        |  6 ++-
 common/preconditioner/isai_kernels.hpp.inc | 60 ++++++++++-----------
 cuda/preconditioner/isai_kernels.cu        |  1 +
 cuda/test/preconditioner/isai_kernels.cpp  | 63 ++++++++++++----------
 omp/test/preconditioner/isai_kernels.cpp   |  9 ++--
 5 files changed, 74 insertions(+), 65 deletions(-)

diff --git a/common/components/warp_blas.hpp.inc b/common/components/warp_blas.hpp.inc
index 0dce40e5147..d810a972ae4 100644
--- a/common/components/warp_blas.hpp.inc
+++ b/common/components/warp_blas.hpp.inc
@@ -102,7 +102,7 @@ __device__ __forceinline__ void apply_gauss_jordan_transform_with_rhs(
     bool &__restrict__ status)
 {
     auto key_col_elem = group.shfl(row[key_col], key_row);
-    auto key_rhs_elem = rhs[key_row] / key_col_elem;
+    auto key_rhs_elem = group.shfl(rhs[0], key_row);
     if (key_col_elem == zero<ValueType>()) {
         // TODO: implement error handling for GPUs to be able to properly
         //       report it here
@@ -111,16 +111,18 @@ __device__ __forceinline__ void apply_gauss_jordan_transform_with_rhs(
     }
     if (group.thread_rank() == key_row) {
         key_col_elem = one<ValueType>() / key_col_elem;
+        rhs[0] = key_rhs_elem * key_col_elem;
     } else {
         key_col_elem = -row[key_col] / key_col_elem;
+        rhs[0] += key_rhs_elem * key_col_elem;
     }
 #pragma unroll
     for (int32 i = 0; i < max_problem_size; ++i) {
         const auto key_row_elem = group.shfl(row[i], key_row);
         if (group.thread_rank() == key_row) {
-            rhs[i] += key_rhs_elem * key_row_elem;
             row[i] = zero<ValueType>();
         }
+        // rhs[0] += key_rhs_elem * key_row_elem;
         row[i] += key_col_elem * key_row_elem;
     }
     row[key_col] = key_col_elem;
diff --git a/common/preconditioner/isai_kernels.hpp.inc b/common/preconditioner/isai_kernels.hpp.inc
index 61a871f7ad1..7bd2e17e154 100644
--- a/common/preconditioner/isai_kernels.hpp.inc
+++ b/common/preconditioner/isai_kernels.hpp.inc
@@ -54,7 +54,7 @@ __forceinline__ __device__ void generic_generate(
 {
     static_assert(subwarp_size >= row_size_limit, "incompatible subwarp_size");
     const auto row = thread::get_subwarp_id_flat<subwarp_size, IndexType>();
-
+    __shared__ size_type k[subwarps_per_block];
     if (row >= num_rows) {
         return;
     }
@@ -92,6 +92,7 @@ __forceinline__ __device__ void generic_generate(
         if (local_id == 0) {
             excess_rhs_sizes[row] = 0;
             excess_nnz[row] = 0;
+            k[row % subwarps_per_block] = 0;
         }
 
         // subwarp_size^2 storage per subwarp
@@ -113,8 +114,6 @@ __forceinline__ __device__ void generic_generate(
 
         subwarp.sync();
 
-        index_type k = 0;
-
         for (IndexType nz = 0; nz < i_row_size; ++nz) {
             auto col = i_col_idxs[i_row_begin + nz];
             auto m_row_begin = m_row_ptrs[col];
@@ -127,8 +126,9 @@ __forceinline__ __device__ void generic_generate(
                 [&](IndexType, IndexType m_idx, IndexType i_idx,
                     config::lane_mask_type, bool valid) {
                     if (valid) {
-                        if (m_cols[m_idx + m_row_begin] < row && col == row)
-                            k++;
+                        if (m_col_idxs[m_row_begin + m_idx] < row && col == row)
+                            atomic_add(&k[row % subwarps_per_block],
+                                       size_type{1});
                         trisystem(nz, i_idx) = m_values[m_row_begin + m_idx];
                     }
                 });
@@ -144,7 +144,8 @@ __forceinline__ __device__ void generic_generate(
             local_row[i] = trisystem(i, local_id);
         }
 
-        const auto rhs = trs_solve(i_row_size, local_row, subwarp, k);
+        const auto rhs = trs_solve(i_row_size, local_row, subwarp,
+                                   k[row % subwarps_per_block]);
 
         // Write back:
         if (local_id < i_row_size) {
@@ -249,32 +250,31 @@ __global__ __launch_bounds__(default_block_size) void generate_general_inverse(
     IndexType *__restrict__ excess_rhs_sizes,
     IndexType *__restrict__ excess_nnz)
 {
-    auto general_solve =
-        [](IndexType num_elems, const ValueType *__restrict__ local_row,
-           group::thread_block_tile<subwarp_size> &subwarp, size_type k) {
-            __shared__ ValueType rhs[subwarp_size];
-            const int local_id = subwarp.thread_rank();
-            ValueType rhs[local_id] =
-                local_id == k ? one<ValueType>() : zero<ValueType>();
-
-            auto pivoted = subwarp.thread_rank() >= num_rows;
-            auto status = true;
-            for (size_type i = 0; i < num_rows; i++) {
-                const auto piv = choose_pivot(subwarp, row[i], pivoted);
-                if (local_id == piv) {
-                    perm = i;
-                    pivoted = true;
-                }
-                if (local_id == i) {
-                    trans_perm = piv;
-                }
-
-                apply_gauss_jordan_transform_with_rhs<max_problem_size>(
-                    subwarp, piv, i, row, rhs, status);
+    auto general_solve = [](IndexType num_elems,
+                            ValueType *__restrict__ local_row,
+                            group::thread_block_tile<subwarp_size> &subwarp,
+                            size_type k) {
+        const int local_id = subwarp.thread_rank();
+        ValueType rhs = local_id == k ? one<ValueType>() : zero<ValueType>();
+        size_type perm = local_id;
+        auto pivoted = subwarp.thread_rank() >= num_elems;
+        auto status = true;
+        for (size_type i = 0; i < num_elems; i++) {
+            const auto piv = choose_pivot(subwarp, local_row[i], pivoted);
+            if (local_id == piv) {
+                // perm = i;
+                pivoted = true;
+            }
+            if (local_id == i) {
+                perm = piv;
             }
 
-            return rhs[local_id];
-        };
+            apply_gauss_jordan_transform_with_rhs<subwarp_size>(
+                subwarp, piv, i, local_row, &rhs, status);
+        }
+
+        return subwarp.shfl(rhs, perm);
+    };
 
     generic_generate<subwarp_size, subwarps_per_block>(
         num_rows, m_row_ptrs, m_col_idxs, m_values, i_row_ptrs, i_col_idxs,
diff --git a/cuda/preconditioner/isai_kernels.cu b/cuda/preconditioner/isai_kernels.cu
index 2e622768497..bcb757aa935 100644
--- a/cuda/preconditioner/isai_kernels.cu
+++ b/cuda/preconditioner/isai_kernels.cu
@@ -66,6 +66,7 @@ constexpr int subwarps_per_block{2};
 constexpr int default_block_size{subwarps_per_block * subwarp_size};
 
 
+#include "common/components/atomic.hpp.inc"
 #include "common/components/warp_blas.hpp.inc"
 #include "common/preconditioner/isai_kernels.hpp.inc"
 
diff --git a/cuda/test/preconditioner/isai_kernels.cpp b/cuda/test/preconditioner/isai_kernels.cpp
index f53ad8f9d5f..5f18dd2d0d7 100644
--- a/cuda/test/preconditioner/isai_kernels.cpp
+++ b/cuda/test/preconditioner/isai_kernels.cpp
@@ -33,9 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/preconditioner/isai.hpp>
 
 
+#include <iostream>
 #include <random>
 
-
 #include <gtest/gtest.h>
 
 
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/identity.hpp>
 
 
 #include "core/preconditioner/isai_kernels.hpp"
@@ -92,8 +93,15 @@ class Isai : public ::testing::Test {
         auto val_dist = std::uniform_real_distribution<value_type>(-1., 1.);
         mtx = Csr::create(ref);
         if (type == matrix_type::general) {
-            mtx = gko::test::generate_random_matrix<Csr>(
-                n, n, nz_dist, val_dist, rand_engine, ref, gko::dim<2>{n, n});
+            auto lower_mtx = gko::test::generate_random_triangular_matrix<Csr>(
+                n, n, true, true, nz_dist, val_dist, rand_engine, ref,
+                gko::dim<2>{n, n});
+            mtx = gko::test::generate_random_triangular_matrix<Csr>(
+                n, n, false, false, nz_dist, val_dist, rand_engine, ref,
+                gko::dim<2>{n, n});
+            auto one_op = gko::initialize<Dense>({gko::one<value_type>()}, ref);
+            auto id = gko::matrix::Identity<>::create(ref, gko::dim<2>{n, n});
+            lower_mtx->apply(one_op.get(), id.get(), one_op.get(), mtx.get());
         } else {
             mtx = gko::test::generate_random_triangular_matrix<Csr>(
                 n, n, true, for_lower_tm, nz_dist, val_dist, rand_engine, ref,
@@ -167,20 +175,19 @@ TEST_F(Isai, CudaIsaiGenerateUinverseShortIsEquivalentToRef)
 }
 
 
-TEST_F(Isai, OmpIsaiGenerateAinverseShortIsEquivalentToRef)
+TEST_F(Isai, CudaIsaiGenerateAinverseShortIsEquivalentToRef)
 {
-    initialize_data(matrix_type::general, 615, 31);
+    initialize_data(matrix_type::general, 615, 15);
     const auto num_rows = mtx->get_size()[0];
     gko::Array<index_type> a1(ref, num_rows + 1);
     auto a2 = a1;
-    gko::Array<index_type> da1(omp, num_rows + 1);
+    gko::Array<index_type> da1(cuda, num_rows + 1);
     auto da2 = da1;
 
-    gko::kernels::reference::isai::generate_tri_inverse(
-        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
-    gko::kernels::omp::isai::generate_tri_inverse(
-        omp, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
-        false);
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+    gko::kernels::cuda::isai::generate_general_inverse(
+        cuda, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data());
 
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, r<value_type>::value);
@@ -236,22 +243,22 @@ TEST_F(Isai, CudaIsaiGenerateUinverseLongIsEquivalentToRef)
 }
 
 
-TEST_F(Isai, OmpIsaiGenerateAinverseLongIsEquivalentToRef)
+TEST_F(Isai, CudaIsaiGenerateAinverseLongIsEquivalentToRef)
 {
     initialize_data(matrix_type::general, 695, 64);
     const auto num_rows = mtx->get_size()[0];
     gko::Array<index_type> a1(ref, num_rows + 1);
     auto a2 = a1;
-    gko::Array<index_type> da1(omp, num_rows + 1);
+    gko::Array<index_type> da1(cuda, num_rows + 1);
     auto da2 = da1;
 
     gko::kernels::reference::isai::generate_general_inverse(
         ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
-    gko::kernels::omp::isai::generate_general_inverse(
-        omp, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data());
+    gko::kernels::cuda::isai::generate_general_inverse(
+        cuda, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data());
 
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
-    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 10 * r<value_type>::value);
     GKO_ASSERT_ARRAY_EQ(a1, da1);
     GKO_ASSERT_ARRAY_EQ(a2, da2);
     ASSERT_GT(a1.get_const_data()[num_rows], 0);
@@ -320,7 +327,7 @@ TEST_F(Isai, CudaIsaiGenerateExcessUinverseLongIsEquivalentToRef)
 }
 
 
-TEST_F(Isai, OmpIsaiGenerateExcessAinverseLongIsEquivalentToRef)
+TEST_F(Isai, CudaIsaiGenerateExcessAinverseLongIsEquivalentToRef)
 {
     initialize_data(matrix_type::general, 100, 51);
     const auto num_rows = mtx->get_size()[0];
@@ -328,20 +335,20 @@ TEST_F(Isai, OmpIsaiGenerateExcessAinverseLongIsEquivalentToRef)
     auto a2 = a1;
     gko::kernels::reference::isai::generate_general_inverse(
         ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
-    gko::Array<index_type> da1(omp, a1);
-    gko::Array<index_type> da2(omp, a2);
+    gko::Array<index_type> da1(cuda, a1);
+    gko::Array<index_type> da2(cuda, a2);
     auto e_dim = a1.get_data()[num_rows];
     auto e_nnz = a2.get_data()[num_rows];
     auto excess = Csr::create(ref, gko::dim<2>(e_dim, e_dim), e_nnz);
     auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
-    auto dexcess = Csr::create(omp, gko::dim<2>(e_dim, e_dim), e_nnz);
-    auto de_rhs = Dense::create(omp, gko::dim<2>(e_dim, 1));
+    auto dexcess = Csr::create(cuda, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto de_rhs = Dense::create(cuda, gko::dim<2>(e_dim, 1));
 
     gko::kernels::reference::isai::generate_excess_system(
         ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
         excess.get(), e_rhs.get());
-    gko::kernels::omp::isai::generate_excess_system(
-        omp, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
+    gko::kernels::cuda::isai::generate_excess_system(
+        cuda, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
         da2.get_const_data(), dexcess.get(), de_rhs.get());
 
     GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
@@ -404,7 +411,7 @@ TEST_F(Isai, CudaIsaiScatterExcessSolutionUIsEquivalentToRef)
 }
 
 
-TEST_F(Isai, OmpIsaiScatterExcessSolutionAIsEquivalentToRef)
+TEST_F(Isai, CudaIsaiScatterExcessSolutionAIsEquivalentToRef)
 {
     initialize_data(matrix_type::general, 702, 45);
     const auto num_rows = mtx->get_size()[0];
@@ -412,19 +419,19 @@ TEST_F(Isai, OmpIsaiScatterExcessSolutionAIsEquivalentToRef)
     auto a2 = a1;
     gko::kernels::reference::isai::generate_general_inverse(
         ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
-    gko::Array<index_type> da1(omp, a1);
+    gko::Array<index_type> da1(cuda, a1);
     auto e_dim = a1.get_data()[num_rows];
     auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
     std::fill_n(e_rhs->get_values(), e_dim, 123456);
-    auto de_rhs = Dense::create(omp);
+    auto de_rhs = Dense::create(cuda);
     de_rhs->copy_from(lend(e_rhs));
     // overwrite -1 values with inverse
     d_inverse->copy_from(lend(inverse));
 
     gko::kernels::reference::isai::scatter_excess_solution(
         ref, a1.get_const_data(), e_rhs.get(), inverse.get());
-    gko::kernels::omp::isai::scatter_excess_solution(
-        omp, da1.get_const_data(), de_rhs.get(), d_inverse.get());
+    gko::kernels::cuda::isai::scatter_excess_solution(
+        cuda, da1.get_const_data(), de_rhs.get(), d_inverse.get());
 
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
     ASSERT_GT(e_dim, 0);
diff --git a/omp/test/preconditioner/isai_kernels.cpp b/omp/test/preconditioner/isai_kernels.cpp
index cf2d905aae4..879f17c1198 100644
--- a/omp/test/preconditioner/isai_kernels.cpp
+++ b/omp/test/preconditioner/isai_kernels.cpp
@@ -174,11 +174,10 @@ TEST_F(Isai, OmpIsaiGenerateAinverseShortIsEquivalentToRef)
     gko::Array<index_type> da1(omp, num_rows + 1);
     auto da2 = da1;
 
-    gko::kernels::reference::isai::generate_tri_inverse(
-        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
-    gko::kernels::omp::isai::generate_tri_inverse(
-        omp, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
-        false);
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+    gko::kernels::omp::isai::generate_general_inverse(
+        omp, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data());
 
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, r<value_type>::value);

From 823ddf83788b0ced5f70c10d082007a537931df5 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Wed, 16 Dec 2020 07:16:32 +0100
Subject: [PATCH 0508/1008] initialize initial guess for gmres

---
 core/preconditioner/isai.cpp  |     5 +-
 matrices/CMakeLists.txt       |     3 +-
 matrices/test/a_large.mtx     | 10418 ++------------------------------
 matrices/test/a_large_inv.mtx | 10418 ++------------------------------
 4 files changed, 837 insertions(+), 20007 deletions(-)

diff --git a/core/preconditioner/isai.cpp b/core/preconditioner/isai.cpp
index ee423c97cc3..7fb25b084c3 100644
--- a/core/preconditioner/isai.cpp
+++ b/core/preconditioner/isai.cpp
@@ -173,10 +173,11 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
                         gko::stop::Iteration::build()
                             .with_max_iters(excess_dim)
                             .on(exec),
-                        gko::stop::ResidualNormReduction<ValueType>::build()
-                            .with_reduction_factor(1e-6)
+                        gko::stop::AbsoluteResidualNorm<ValueType>::build()
+                            .with_tolerance(1e-6)
                             .on(exec))
                     .on(exec);
+            excess_solution->copy_from(excess_rhs.get());
         } else if (is_lower) {
             trs_factory = UpperTrs::build().on(exec);
         } else {
diff --git a/matrices/CMakeLists.txt b/matrices/CMakeLists.txt
index 987bcd8610e..76a30c4f80b 100644
--- a/matrices/CMakeLists.txt
+++ b/matrices/CMakeLists.txt
@@ -17,4 +17,5 @@ configure_file("test/isai_u_inv.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_u_
 configure_file("test/isai_u_inv_partial.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_u_inv_partial.mtx")
 configure_file("test/1138_bus.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/1138_bus.mtx")
 configure_file("test/1138_bus_isai.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/1138_bus_isai.mtx")
-configure_file("test/a_large.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/a_large_inv.mtx")
+configure_file("test/a_large.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/a_large.mtx")
+configure_file("test/a_large_inv.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/a_large_inv.mtx")
diff --git a/matrices/test/a_large.mtx b/matrices/test/a_large.mtx
index 47de5369594..06722f8ba51 100644
--- a/matrices/test/a_large.mtx
+++ b/matrices/test/a_large.mtx
@@ -1,10002 +1,416 @@
-%%MatrixMarket matrix array real general
-100 100
- 1
- 0.0352221
- 0
--0.126265
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.955826
- 0
- 0.448404
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.588998
- 0
- 0
- 0
--0.301462
- 0
- 0
- 0
- 0.287703
- 0
- 0
- 0
- 0.25185
- 0
- 0.49784
- 0
- 0.547594
- 0.85276
- 0.328431
- 0
- 0
- 0
--0.436379
- 0.960822
- 0
- 0
- 0.55008
- 0
- 0
- 0
- 0.996657
- 0
- 0
- 0
--0.07184500000000001
- 0
- 0
- 0
- 0
--0.130761
- 0
- 0.667564
- 0.655325
- 0
- 0.238939
--0.184559
--0.5358270000000001
- 0.0949749
--0.489252
- 0.0186587
- 0
- 0.607581
--0.408656
- 0
- 0
--0.384484
- 0.575102
- 0
--0.000766162
- 0
- 0
- 0
--0.539473
- 0
- 0
- 0
- 0
- 0
--0.71202
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.479209
- 0
- 0
- 0
- 0
- 0.762845
- 0
- 0.403298
- 0.64392
- 0
--0.457379
- 0
- 0
- 0
- 0
--0.0277301
--0.698778
- 0
--0.2764
- 0
- 0
- 0.766326
- 0
- 0
--0.403154
- 0
--0.0430981
--0.496861
- 0
--0.885747
- 0
- 0.747283
--0.0575297
--0.59034
--0.542502
--0.0261681
--0.187808
- 0.820244
- 0.403374
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.8516820000000001
- 0
- 0
- 0
--0.81643
- 0
- 0
--0.763212
- 0
- 0
- 0
- 0
- 0
--0.955558
- 0.8234089999999999
- 0
- 0
- 0.115741
- 0
- 0
- 0
- 0
- 0
--0.63423
- 0
- 0
- 0
- 0
- 0
- 0
--0.635006
- 0.154736
- 0.21157
- 0
- 0
- 0.115656
- 0
- 0
- 0
- 0.477922
- 0
- 0
- 0
--0.125002
- 0
- 0
--0.326679
- 0.1046859999999999
- 0
- 0
--0.728782
- 0
- 0.677805
- 0
- 0.333973
- 0
- 0
- 0.72497
- 0
- 0
--0.94803
- 0
--0.266554
- 0
- 0
- 0
--0.550107
- 0
- 0
- 0
- 0
--0.777374
- 0
- 0
- 0
- 0
--0.343653
- 0
- 0
- 0
- 0
- 0
- 0.975595
--0.253923
- 0
- 0
--0.727818
- 0
- 0.813276
- 0
- 0
- 0
- 0
- 0
--0.873077
- 0
- 0
- 0
--0.494212
- 0.183104
--0.820892
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.39238
- 0
- 0
--0.286842
- 0
--0.920755
- 0
- 0.573967
- 0
- 0.545054
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0808275
- 0
--0.630044
- 0
- 0
- 0.46123
--0.77565
- 0
- 0
- 0
- 1
- 0
- 0
- 0
- 0
--0.577415
- 0
- 0
- 0
- 0
- 0
- 0
--0.790793
- 0
- 0.319043
- 0.505891
- 0.333113
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.295016
- 0
- 0.586125
- 0
- 0
- 0
--0.284215
- 0.475873
--0.127442
- 0
--0.5929720000000001
- 0
--0.573142
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.39057
- 0
- 0
- 0
- 0
- 0.416972
--0.776501
--0.612899
- 0
- 0
--0.966733
--0.413408
- 0.426742
- 0
- 0.139594
- 0.47202
- 0
--0.0425343
- 0.0217776
- 0
- 0
--0.37716
- 0
--0.81123
- 0.689438
- 0
- 0
- 0
- 0
- 1
- 0
- 0
- 0
--0.600885
- 0
- 0
--0.9369960000000001
--0.913012
- 0
- 0
- 0
- 0
- 0.0549891
- 0.789766
- 0.155261
--0.626726
- 0.438602
- 0
- 0
- 0
- 0
- 0
- 0.912946
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.820889
- 0
- 0
- 0
- 0
- 0
- 0
--0.232923
--0.379572
- 0
--0.881664
- 0.177548
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.8491919999999999
- 0
- 0.820483
--0.0380151
- 0
- 0
--0.370664
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.425507
- 0
- 0
--0.33833
- 0.905755
- 0
- 0.449572
- 0
- 0
- 0.000147088
- 0
- 0
- 0
- 0.423966
- 0
--0.8492420000000001
- 0
- 0
- 0.174948
- 0
- 0
- 0
- 0
- 0
- 0.389784
- 0
--0.293117
- 0
- 0
--0.663598
- 0
- 0
--0.598401
- 1
- 0
- 0
- 0
- 0
- 0
--0.284778
--0.259316
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.225826
- 0
- 0
- 0
--0.418982
- 0.648581
- 0
- 0
- 0
--0.6842510000000001
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.341594
- 0
- 0.620384
- 0.172187
--0.427182
- 0
--0.810075
- 0
--0.484587
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.298056
- 0
- 0
- 0
--0.492891
- 0
--0.796257
- 0
- 0
- 0
- 0
--0.284484
- 0
- 0
- 0
- 0
- 0
- 0
- 0.911418
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.967934
- 0
- 0
- 0
- 0
--0.834484
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.27267
- 0
- 0
- 0
- 0.99203039
- 0
- 0
--0.646975
- 0
--0.533131
- 0
- 0
- 0
- 0
- 0
- 0.233432
- 0
- 0.546378
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.497845
- 0
- 0.527705
- 0
- 0
- 0.487902
- 0.821416
- 0
- 0
- 0
--0.152157
- 0
--0.206447
- 0
- 0
- 0
- 0
- 0
- 0.746186
- 0
- 0
- 0
- 0
- 0
- 0
- 0.128726
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.985511
- 0
- 0
- 0.52516
- 0.240263
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.547479
- 0
- 0
--0.656579
- 0
- 0
--0.879771
- 0
- 0
- 0
- 0
- 0.166345
- 0
- 0
- 0
- 0.611135
- 0
- 0
- 0
- 0
- 0
--0.439705
- 0
- 1
- 0.259374
- 0.5889450000000001
- 0
- 0.0143551
- 0
- 0
- 0
--0.763266
- 0.825677
- 0.733819
--0.758653
--0.0877497
- 0
- 0.0547954
- 0.937111
- 0
- 0
- 0
- 0.485806
- 0
--0.615699
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.661019
--0.321536
- 0
- 0
--0.822585
- 0
- 0
- 0
- 0.640329
- 0
- 0
- 0
- 0
- 0.5626910000000001
--0.8737200000000001
- 0
- 0
- 0
--0.337115
- 0
- 0
- 0
- 0.627238
- 0
- 0
- 0
--0.566686
--0.639248
- 0
- 0
--0.0899401
- 0
- 0
- 0
- 0
- 0
- 0.648264
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.170385
- 0
- 0
- 0
- 0
- 0.891823
- 0
--0.591547
- 0
- 0
- 0.467363
- 0
- 0
- 0
- 0
--0.25665
- 0
- 0
- 0.874235
- 0.6489279999999999
- 0
- 0
- 0
- 0
- 0.568695
- 0
- 0
- 0
- 1
- 0
- 0
--0.852568
- 0
- 0
- 0
- 0.688815
- 0
--0.868268
- 0.71848
- 0.874924
- 0
--0.6378200000000001
- 0.793312
- 0
- 0
- 0
- 0
- 0
--0.772224
--0.994532
- 0
--0.996234
--0.491159
- 0
- 0
- 0
- 0
- 0
- 0.875526
- 0
- 0
- 0
- 0
- 0
- 0.373084
- 0
- 0
- 0
- 0
--0.742054
- 0
- 0.876749
- 0
- 0
- 0
- 0
- 0
- 0
--0.569966
- 0
- 0
- 0
- 0
- 0.876426
- 0
--0.30766
- 0
- 0
--0.174341
- 0
--0.133573
- 0
- 0
- 0
- 0
- 0.649197
- 0.920309
- 0
- 0
- 0.525194
- 0.263471
- 0
- 0.7017409999999999
- 0
- 0
- 0
- 0
- 0
- 0
- 0.614198
- 0
- 0
- 0
- 0.879014
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.624846
- 0.691658
- 0
- 0
- 0
- 0
- 0
- 0.283198
- 0.314876
- 0
- 0.972719
- 0
- 0
- 0.389011
- 0
- 0
- 0
--0.928426
- 0.19006
- 0
--0.114557
- 0
- 0
--0.346923
- 0.329702
- 0
- 0.355238
- 0
- 0
- 0.11432
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.634883
- 0.933555
- 0
- 0
- 0
- 0
- 0
--0.0350987
- 0
- 0
- 0
- 0
--0.827219
- 0.787873
- 0.230721
- 0
- 0
- 0.143212
--0.160876
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.573569
- 0
- 0
- 0
- 0
--0.0518283
- 0
- 0
- 0
- 0
- 0.763983
--0.288771
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.220261
- 0.00302326
- 0
- 0.678679
- 0
- 0
- 0.789704
- 0
- 0.0130651
- 0
- 0
- 0
- 0
- 0
--0.980996
- 0
--0.741042
- 0
- 0
- 0
- 0
- 0
- 0
- 1
- 0
- 0
- 0
- 0
- 0.0606227
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.797913
- 0.750262
- 0
- 0.752896
- 0
--0.832524
- 0
- 0.358872
- 0
- 0
--0.0340256
- 0
- 0
--0.487125
- 0
- 0
--0.915477
- 0
- 0
- 0
--0.5155960000000001
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5317229999999999
- 0
- 0
- 0
- 0
- 0
--0.5104300000000001
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.72778
- 0
- 0
- 0
--0.720365
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.835952
- 0
- 0
- 0
- 0.018154
- 0
--0.749382
- 0
- 0.444569
--0.452357
- 0
--0.967843
- 0
- 1
- 0
- 0
- 0.898672
- 0
- 0
--0.91964
--0.958815
- 0
- 0
- 0
--0.102141
- 0
--0.8037029999999999
- 0
- 0
--0.53889
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5618339999999999
- 0
--0.848769
--0.581632
- 0
- 0
- 0
- 0.651222
- 0
- 0
- 0.827912
- 0
- 0
- 0
- 0
- 0.215715
- 0
- 0
- 0
- 0
- 0
- 0
- 0.448614
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.873539
- 0
- 0
- 0
- 0.240421
- 0
- 0
- 0
- 0
--0.112234
--0.289582
- 0
- 0
--0.159832
--0.750553
- 0
- 0
- 0
- 0
- 0
--0.804454
- 0
- 0
- 0
--0.332287
- 0.94617
- 0
--0.116449
- 0
- 0
- 0
- 0
- 0
--0.0808903
- 0
--0.5601080000000001
- 0
- 0
- 0
- 0
--0.828614
- 0
- 0.850285
- 1
- 0
- 0
- 0
--0.559895
- 0
- 0
- 0
- 0.143098
--0.21878
--0.805248
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.574879
- 0
- 0
- 0
- 0
- 0
- 0
- 0.228353
--0.650577
- 0
--0.660778
- 0
- 0
- 0
- 0
- 0
- 0
--0.253299
- 0
- 0.39971
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.987043
--0.7430060000000001
- 0.86563
- 0
- 0
- 0
- 0
--0.518626
- 0
- 0
- 0
- 0
- 0
- 0
--0.470546
--0.664445
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.444533
- 0.0268659
- 0.132374
- 0
- 0
- 0.570704
- 0
- 0
--0.0331565
- 0
- 0
- 0.36033
- 0
- 0
- 0.819901
- 0
- 0
- 0
- 0
- 0
- 0
--0.773919
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1
- 0
- 0
- 0
- 0.842892
- 0.890227
- 0
--0.151101
--0.576108
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.981993
- 0.906771
- 0.264393
- 0
- 0
- 0
- 0
--0.626338
- 0
- 0
--0.561011
- 0.869959
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.009727080000000001
- 0.270429
- 0.800198
- 0
- 0
--0.612209
- 0
- 0
- 0.08075060000000001
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.416234
--0.262502
- 0
--0.907022
- 0.27517
- 0
--0.341087
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.553862
--0.98084
--0.217547
- 0
- 0.342659
- 0
- 0.308032
- 0
- 0
--0.08850769999999999
--0.28468
- 1
- 0
- 0
- 0
- 0
- 0
- 0
--0.969376
- 0
- 0
- 0
--0.753979
- 0
- 0
- 0.827613
--0.525857
- 0.621262
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.758388
- 0
- 0.179235
- 0
--0.967231
--0.388903
- 0
--0.457423
--0.268806
- 0
- 0
- 0
- 0
--0.0795442
- 0
- 0
- 0
- 0
- 0
--0.921382
- 0
- 0
- 0
--0.534448
- 0
- 0
- 0
- 0
--0.0177686
- 0
- 0
- 0.519825
- 0
- 0
- 0
- 0
- 0
- 0.572676
- 0
- 0
- 0
--0.136917
- 0
- 0.23601
- 0
--0.330842
- 0
- 0.37795
- 0.75462
- 0
- 0.882412
- 0.028373
- 0
- 0.177926
- 0
- 0
--0.876395
- 0
- 0
- 0
--0.950016
- 0
- 0
- 0
- 0
- 0
- 0
--0.996262
- 0
- 0
- 0
- 0
- 0
- 1.875563
- 0
- 0.960957
- 0.787272
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.159258
- 0
--0.0819941
- 0
- 0
- 0
- 0
- 0.675694
- 0
- 0
- 0
- 0
- 0
--0.585519
- 0
- 0
- 0
- 0
- 0.421643
- 0
- 0.920991
- 0
--0.120211
- 0.455017
- 0
- 0
- 0
- 0
- 0
--0.534284
- 0
- 0
- 0
- 0
- 0
- 0
--0.520883
- 0
- 0
- 0
- 0
- 0.971579
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.729364
- 0
- 0
- 0.371877
--0.905417
- 0
- 0.354084
- 0
--0.367703
- 0.0162526
- 0
- 0
- 0.591507
- 0
- 0
- 0.578126
- 0
--0.55881
- 0
- 0
--0.9078349999999999
- 0
- 0
- 0.458132
- 0
- 0.915848
- 0
- 0
- 0
- 0
- 0
--0.942116
- 0
- 0
- 0
- 0.59726
- 0.728936
- 0
- 1.819113
- 0
- 0
--0.170334
--0.226999
- 0
- 0
- 0
- 0
--0.508684
- 0.42599
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.30392
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.637889
- 0.464512
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.5139860000000001
- 0
- 0
- 0
- 0
- 0.701715
- 0.698952
- 0
- 0
--0.10664
--0.6580510000000001
- 0
- 0
- 0
- 0
--0.21736
--0.614479
- 0
--0.0332876
- 0.0608625
- 0
--0.880495
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.855899
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.338955
- 0
- 0
- 1
- 0
- 0
- 0
--0.88388
- 0.900593
- 0
- 0
- 0
- 0.259993
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.00559104
- 0
- 0
- 0
- 0.903313
--0.434739
--0.0707097
--0.298429
- 0
- 0.771146
- 0
- 0
- 0
- 0
- 0
--0.199954
- 0
- 0
- 0
- 0
- 0.00153506
- 0
- 0
- 0
- 0.5148740000000001
- 0
- 0
- 0
- 0
- 0
- 0
--0.802113
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.450203
--0.630819
- 0.00152143
- 0
- 0
- 0
- 0
- 0
--0.932658
- 0
- 0
- 0.232096
- 0
- 0
- 0.627546
- 0
- 0
- 0
--0.899024
- 0
- 0.239356
- 0.363986
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.877309
- 0
- 0
- 0
- 0.767697
- 0.5294180000000001
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0.705021
- 0
- 0
- 0
- 0
- 0
- 0.452926
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0161528
- 0
- 0
- 0
- 0
- 0.26745
- 0
--0.07983609999999999
- 0.894411
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.77813
- 0
- 0
- 0
- 0
- 0
- 0
- 0.588775
- 0
- 0
- 0
- 0
- 0
--0.38006
- 0
- 0
- 0
- 0
--0.9967240000000001
- 0
- 0
- 0
- 0.300447
- 0
--0.32854
- 0
--0.271298
- 0
- 0
- 0
--0.0891188
- 0
- 0
- 0
- 0
- 0.0918489
- 0
- 0
- 0
- 0
- 0
--0.679741
- 0
- 0
--0.966585
- 0
- 0
- 0
- 0.0990072
- 0.217815
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.0444764
- 0.313269
--0.943724
--0.483973
- 1
- 0
- 0.645783
--0.803159
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.97432
- 0
--0.819059
- 0
- 0
- 0
- 0.900874
- 0
- 0
--0.9302240000000001
--0.386968
--0.498171
- 0
- 0.657479
- 0
- 0
--0.311184
- 0
- 0.0330274
- 0
- 0
- 0
- 0
--0.697004
- 0.785085
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.194681
- 0
- 0
- 0.863267
- 0
--0.592132
- 0
- 0
- 0
- 0
- 0
- 0
- 0.9262049999999999
- 0
- 0
- 0
--0.974857
- 0
- 0
- 0
- 0.08011310000000001
- 0
- 0
- 0.961995
- 0
- 0
- 0
- 0
- 0.69303
--0.435409
- 0
--0.235911
- 0.50975
- 0
- 0
- 0
- 0
- 0
- 0
--0.825164
- 0
- 0
- 0
- 0.711273
- 0.996322
- 0.0373364
- 0
- 0
- 0
- 0
- 0.307555
--0.48443
- 0
- 1
- 0.0760803
- 0
- 0
- 0
- 0.929615
- 0
- 0.134809
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.380073
- 0
- 0
- 0
--0.0615359
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.466221
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.208577
- 0
- 0
- 0
- 0
- 0
- 0
- 0.514005
- 0
- 0
- 0.393893
- 0
- 0
- 0
- 0.535456
- 0
- 0.15096
- 0
- 0
- 0
- 0.918121
- 0.433264
- 0
- 0.35857
- 0
- 0
- 0.334598
- 0
- 0
- 0
--0.560098
- 0
- 0
- 0
--0.267865
--0.534967
--0.864568
- 0
- 0
- 0
- 0
- 0
--0.55958
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.283961
- 0
- 0
- 0.367275
- 0
- 0
- 0
- 0
- 1
- 0
- 0
- 0
- 0
- 0.388918
- 0.925996
- 0
- 0
- 0
- 0.31666
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.820912
- 0.0569179
- 0
- 0.609494
- 0
- 0.246628
- 0
--0.466481
- 0
- 0
- 0
- 0
- 0
--0.6463989999999999
- 0
- 0
- 0
- 0
- 0
- 0
- 0.96239
- 0
- 0
--0.43408
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.943537
- 0.703053
- 0
- 0
- 0
--0.326245
--0.224257
- 0
- 0
- 0
- 0
- 0.179575
--0.196652
- 0.530014
- 0.472888
- 0
- 0
- 0
--0.0528497
- 0
- 0.372112
--0.565865
- 0
- 0
--0.695865
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.528593
- 0
- 1
- 0
- 0
--0.752672
- 0
- 0
- 0
- 0
- 0.985171
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.568941
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.931409
- 0.00905803
- 0
- 0
- 0
- 0
- 0.0712531
- 0
- 0
- 0
--0.5998559999999999
- 0
- 0.025817
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.682302
--0.165467
- 0
- 0.598773
- 0
- 0
- 0
- 0.442903
- 0
- 0
--0.786183
- 0
- 0
- 0
- 0
- 0.0281454
- 0
- 0
- 0.755406
- 0
- 0
--0.024673
- 0
- 0
- 0.0667312
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.864811
- 0
- 0
- 0
- 0
- 0
- 0
--0.947692
- 0.770323
- 0
- 0
- 0
- 0
- 0.421239
- 0
--0.9019
- 0
- 0
- 0
- 0
- 0
- 0
--0.788506
- 0
- 0
- 0
--0.73806
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.39598
- 0.714214
- 0
- 0
--0.680734
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.234227
--0.9890409999999999
- 0
- 0
- 0
- 0.662945
- 0.923506
- 0
- 0.454497
--0.574734
- 0
- 0
- 0
- 0
- 0
- 0
- 0.310312
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.520643
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.981698
- 0.692644
--0.828054
- 0
- 0
--0.159717
- 0
--0.198578
- 0
- 0
- 0
- 0.856366
- 0.960822
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1
- 0
- 0.454944
- 0
- 0
- 0
- 0.562352
- 0
- 0.331572
- 0
- 0
- 0
- 0
- 0.858436
- 0
- 0.624214
- 0
- 0
--0.257371
- 0
--0.202928
- 0.1742
--0.226991
- 0
- 0.146579
- 0
- 0
- 0
- 0
--0.886428
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.325708
- 0
--0.339013
- 0
- 0
- 0
--0.215203
- 0
- 0.771929
- 0
- 0
--0.277257
- 0
- 0
- 0
--0.286208
- 0
- 0.147887
- 0
- 0.952027
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.6573870000000001
- 0
- 0.09863619999999999
- 0
- 0
- 0
- 0
- 0
--0.314632
- 0
- 0
- 0
- 0
- 0
--0.566458
--0.584975
- 0
--0.8089229999999999
- 0
--0.103068
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.283349
- 0
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.772267
- 0
- 0
- 0
- 0.531232
- 0
- 0
- 0
- 0.0602593
- 0
--0.818275
- 0
- 0
- 0.640503
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.452958
- 0
- 0
- 0
- 0.227352
- 0
- 0
- 0
- 0
- 0
- 0
--0.263366
- 0
- 0
- 0
- 0
- 0
- 0
- 0.130876
- 0
- 0
- 0
--0.282325
- 0
- 0
- 0
--0.397617
- 0
- 0.340223
- 0
--0.723695
--0.989039
- 0
- 0
- 0
- 0
- 0
- 0.618592
- 0.186969
- 0
- 0
- 0
- 0.864703
- 0
- 0
- 0
--0.987061
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.451151
--0.0770106
- 0
- 0
- 0
- 0.142685
- 0.406207
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.530766
- 0
- 0
--0.00318492
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.248891
- 0
--0.975061
- 0.421405
- 0
- 0
- 0.180812
- 0
--0.952772
- 0
- 0
- 0
- 0
- 0.0837316
- 0
- 0.742048
--0.223487
--0.10158
- 0
--0.419303
- 0
- 0
--0.306625
- 0.779993
- 0
- 0
- 0
- 0
- 0
- 0
--0.119679
- 0.700197
--0.112062
- 0
- 0.296891
- 0
--0.775325
--0.702605
- 0
--0.486123
- 0
--0.176686
- 0
--0.686523
--0.367065
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.555269
- 0
- 0
- 0.548756
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.485037
- 0
- 0
- 0
--0.559457
- 0
- 0
- 0
- 0
- 0.113706
- 0
- 1
- 0
- 0
- 0.100412
- 0
- 0
- 0
- 0
- 0.636466
- 0
- 0
- 0.07075720000000001
- 0.376973
- 0
- 0
--0.211204
- 0
- 0.09748080000000001
- 0
- 0
- 0
- 0
--0.110624
- 0
- 0
- 0
- 0
- 0.0647851
- 0
- 0
- 0
- 0
- 0.446437
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.176045
- 0
- 0
--0.987692
- 0.922387
- 0
- 0.0398782
--0.73672
- 0
- 0
--0.471647
- 0
- 0
- 0
--0.207676
- 0
--0.303541
--0.776406
- 0.692349
--0.752883
- 0
- 0
- 0
- 0
- 0.6495610000000001
- 0
- 0.306261
- 0
- 0
--0.745659
- 0
--0.425891
--0.489333
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.438058
--0.249839
--0.600756
--0.0477695
- 0
- 0
- 0
- 0
- 0
--0.875088
- 0.9595
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.332186
- 0
- 0
- 0
--0.458934
--0.953472
- 0
- 0
- 0
- 0
- 0
--0.5413
- 0
- 0
- 0
- 0
- 0
- 0.0190589
- 0
- 0
- 0
- 0.32949
- 0
- 0
--0.364322
- 0
--0.890822
- 0
- 0
- 0.514602
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.298092
- 0
- 0
- 0
- 0
- 0.510348
- 0
--0.622988
- 0
- 0
- 0
- 0
- 0
- 0
--0.181659
- 0
- 0.24513
- 0
- 0
- 0
--0.498219
- 0
- 0.492542
- 0
- 0
- 0.0283554
- 0
- 0.329472
--0.172077
- 0
- 0
- 0.559774
- 0
- 0
- 0
--0.40646
- 0
- 0
--0.0810903
- 0
- 0
- 0
- 0
--0.027825
- 0.8294899999999999
- 0.32648
- 0
- 0
- 0
- 0
--0.794948
- 0
- 0
- 1
- 0
- 0
--0.607853
- 0.307702
- 0.740138
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.626233
--0.128633
- 0
- 0.52036
- 0.87114
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.649494
- 0
- 0
- 0
- 0
- 0
--0.765899
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--6.29984e-05
--0.813014
- 0
--0.277215
- 0.769831
- 0
--0.983227
- 0
- 0.389997
- 0
- 0
- 0
--0.6775679999999999
- 0
--0.45033
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.258725
- 0.429784
- 0
- 0.0654044
- 0
--0.601945
--0.0694507
- 0
- 0
- 0.528713
- 0
- 0
- 0
- 0.706446
--0.145461
- 0
- 0
- 0.556621
- 0
- 0.537152
--0.931454
--0.857105
- 0
- 0.41443
- 0
- 0
- 0.690913
--0.898316
- 0.444102
- 0
- 0
- 0
- 1
- 0
- 0
- 0
- 0
- 0.563904
- 0
- 0
- 0
- 0
--0.439682
--0.861575
- 0.265489
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.9785740000000001
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.567659
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.597462
- 0
--0.96797
- 0
- 0
- 0
- 0
- 0
--0.99492
- 0
- 0
- 0
- 0
--0.750054
- 0.504466
- 0
- 0
- 0
--0.378227
- 0.870286
- 0
- 0
--0.877214
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.847492
- 0
- 0
- 0
- 0
- 0
- 0.333287
- 0
- 0
- 0
- 0.968925
- 0
- 0
- 0
- 0.763413
- 0
- 0.494125
- 0
- 0
- 0.790668
--0.502057
- 0
- 0
- 0
--0.00518237
- 0
- 0.973527
- 0
- 0
- 0
- 1
- 0
- 0.561007
- 0
- 0
- 0
--0.072712
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.73028
--0.839825
- 0
- 0
--0.256166
- 0
- 0
--0.0106132
- 0.82038
- 0
- 0
- 0
- 0
- 0
--0.996431
- 0
- 0.281412
- 0
- 0
- 0
- 0.351482
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.110093
- 0
- 0
--0.501891
- 0
--0.655143
- 0.560348
- 0
- 0
- 0.700191
- 0
- 0.612231
--0.187004
- 0
--0.807525
- 0
- 0
- 0
- 0
- 0
--0.918936
- 0
- 0
- 0.197198
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.76811
- 0
--0.464659
- 0
- 0
- 0.92374
- 0
- 0
- 0
- 0
- 0
- 0
--0.729592
- 0.936947
- 0
- 0
- 0
- 0.822943
- 0
- 0
- 0
- 0
- 0
- 0
- 1
- 0
- 0
--0.151991
- 0
- 0
- 0
- 0
--0.731541
- 0
- 0
--0.566369
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.538155
- 0
--0.437678
- 0
- 0.587591
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.265715
- 0
- 0
- 0
--0.226608
- 0
--0.577827
- 0
--0.420319
- 0
- 0.277472
- 0.9434090000000001
--0.688326
- 0
- 0
- 0
- 0
--0.574515
- 0
- 0
- 0.916211
- 0
- 0
- 0
--0.180097
- 0
- 0
- 0
- 0
--0.220395
- 0
- 0.402006
- 0.368038
--0.454009
- 0.798282
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.200078
--0.279707
- 0
- 0
--0.793923
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0.6072650000000001
- 0
- 0
- 0.644375
- 0
- 0
- 0
- 0
- 0.487367
- 0
- 0.65925
- 0
- 0.781865
--0.528809
- 0
- 0
- 0
- 0
- 0.0355046
- 0
--0.802964
- 0
- 0
- 0.211056
--0.835107
- 0
- 0
- 0
--0.359129
- 0
- 0
- 0
- 0
--0.957183
- 0
- 0
- 0
--0.674527
- 0
- 0
- 0
- 0
- 0
- 0.520807
- 0
- 0.597821
- 0
- 0
- 0
- 0
- 0
- 0.986897
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.268008
--0.0367526
- 0.581768
- 0.526045
- 0
- 0
- 0
- 0
- 0
- 0
--0.230316
- 0
- 0
- 0
- 0
- 0
--0.655722
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.687659
- 0
- 0.0559371
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.501389
- 0.859791
- 0
- 0.383719
- 0
- 0.5194800000000001
- 0
- 0
- 0
- 0.405414
- 0
- 0
- 0
- 0
- 0
--0.794355
- 0
--0.378824
- 0
- 0
- 0
- 0.678763
- 0
- 0
- 0
- 0
- 0.659741
- 0
- 0.276424
- 0
- 0
- 0
--0.9015840000000001
- 0.587325
- 0
- 0
- 0
- 0
- 0
- 0
- 0.687698
- 0
- 0
--0.665892
--0.475804
- 0
- 0
- 0.943867
- 0
- 0
- 0.6572
- 0
- 0.230882
- 0
- 0
- 0.451489
- 0
- 0
- 0
- 0
- 0
- 0
--0.0718162
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.416515
- 0.881108
- 0
- 0
- 0
- 0
- 0.341995
- 0.406477
- 0.562663
- 0
- 0
- 0
- 0.0987937
- 0.847928
- 0
- 0
- 0
- 1
- 0
--0.49318
- 0
--0.266143
- 0
--0.000486019
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.216729
- 0
- 0
- 0.920589
- 0
- 0
- 0
- 0.976143
- 0
- 0
- 0.171682
- 0
- 0
- 0
--0.668292
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.282186
- 0
- 0
- 0.8157450000000001
- 0
- 0
- 0
--0.411406
- 0
- 0
- 0
- 0
- 0
- 0.933329
- 0
- 0
--0.0211041
- 0
- 0
- 0
--0.7948
- 0
- 0.592166
- 0
- 0
- 0
- 0.77739
- 0
--0.632004
- 0
- 0
- 0.567619
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.233368
- 0
- 0
--0.501713
- 0
--0.697545
- 1
- 0
- 0.716468
- 0
- 0.30715
- 0
--0.15332
- 0
- 0.449269
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.202787
- 0
- 0
- 0
- 0
- 0
- 0
- 0.38851
--0.923689
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.903624
- 0
- 0.0713196
- 0.343009
- 0
- 0
--0.9609259999999999
- 0
- 0
- 0
- 0.226351
--0.882984
- 0
- 0
- 0
- 0
- 0.926118
- 0
- 0
- 0
--0.222101
- 0
- 0
--0.930119
- 0
--0.980831
- 0.0348414
- 0
- 0.233082
- 0.8291849999999999
- 0.0511106
- 0.760646
- 0
- 0
- 0
- 0
- 0.9417489999999999
- 0
- 0
- 0
--0.49931
--0.244322
- 0.698949
- 0
--0.968606
- 0
- 0
- 0
--0.233887
- 0
- 0
--0.0975314
- 0.218762
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.9227070000000001
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.957744
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.328535
- 0
- 0
--0.913172
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.475135
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.282902
- 0
- 0
- 0
- 0
- 0.669243
- 0
- 0.497747
- 0.844847
- 0
- 0
--0.60248
- 0
- 0
--0.199366
- 0
- 0.536739
- 0
- 0
- 0
--0.863662
--0.812129
- 0.362183
- 0
- 0
- 0
- 0
- 0
- 0.231178
--0.0229049
- 0
- 0.7116130000000001
- 0.863457
- 0
- 0
- 0
- 0
- 0
- 0.688089
--0.902513
- 0
- 0
- 0
- 0
- 0
- 0
--0.56503
- 0
- 0
- 0
- 0
- 1
- 0
- 0.495041
--0.474025
- 0
- 0
- 0.52991
- 0
--0.0892505
- 0
- 0
- 0
- 0
--0.384599
- 0.0117166
--0.955082
- 0
--0.357397
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.798405
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.70269
- 0
- 0
- 0
- 0.00308692
- 0
- 0
- 0
- 0
--0.861565
- 0
- 0
--0.377789
--0.5775439999999999
- 0
--0.183317
- 0
- 0
- 0
- 0
- 0.203103
--0.614461
- 0
- 0
- 0
- 0
- 0
- 0
- 0.20096
- 0.62612
- 0
--0.679671
- 0
- 0
- 0
- 0
- 0
- 0
- 0.625292
- 0
- 0
- 0
--0.993743
- 0
- 0
- 0
- 0
- 0.0312606
- 0
- 0
- 0
- 0
--0.504444
- 0
--0.09743880000000001
- 0
- 1
--0.183638
--0.758983
- 0.61515
--0.137712
- 0
--0.114525
--0.130596
- 0
- 0
- 0.792162
- 0
- 0
- 0.175183
- 0.905067
--0.0993045
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.667387
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.38125
- 0
--0.913071
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.235465
- 0
- 0
- 0
- 0
--0.782763
- 0
- 0
- 0
- 0
- 0
- 0
- 0.925799
--0.455384
- 0.92018
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.198667
--0.560118
- 0
- 0.762945
- 0
- 0
- 0
- 0.0173914
- 0
- 0.76628
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.293363
- 0
- 0
- 0
--0.230035
- 1
- 0
- 0.864734
- 0.914411
- 0.117799
- 0
- 0
- 0
- 0
- 0.546054
- 0
- 0
- 0
- 0
- 0
- 0.530733
- 0.6806489999999999
- 0
- 0
- 0
- 0
- 0.623245
- 0
- 0
- 0
- 0.08471339999999999
- 0
- 0.885162
- 0
- 0.199607
- 0
- 0
- 0
- 0
- 0
--0.655425
- 0
- 0
- 0
- 0
- 0
- 0
--0.815068
--0.291773
- 0
- 0
- 0
--0.278294
- 0
- 0
- 0.07877049999999999
- 0.356073
- 0.982938
- 0.176067
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.010633
- 0
- 0
- 0
- 0.49793
- 0
- 0
--0.376356
- 0
- 0
- 0
- 0
- 0
- 0
- 0.199793
- 0
- 0
- 0
- 0
- 0
- 0.06593309999999999
--0.541268
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.302394
- 0
- 0
- 0.845684
- 0
- 0
--0.484673
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.430443
- 0.368101
- 0
- 0
- 0
- 0
- 0
- 0
--0.17098
- 0
- 0
- 0.6455959999999999
- 0
- 0
- 0
- 0
- 0.7934870000000001
- 0
--0.441221
- 0
- 0.82097
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.111124
- 0
--0.166148
- 0
--0.218475
--0.131328
- 0
--0.411184
- 0
- 0.267467
- 0
- 0
- 0
--0.585544
- 0
- 0
- 0.319084
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.376134
- 0
--0.192926
- 0
- 0.0767085
- 0
- 0
- 0
- 0
- 0
- 0.754015
- 0
- 0
--0.642196
- 0
- 0
--0.236775
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0508256
--0.868807
- 1
- 0
- 0.02112
- 0
- 0.530556
- 0.116396
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.6908339999999999
- 0
--0.978752
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.79189
- 0
- 0
- 0
- 0
- 0.166072
- 0
--0.497046
- 0
- 0
- 0.792318
- 0
- 0
- 0
- 0.168629
--0.319897
- 0
--0.736453
- 0
- 0
- 0
- 0
--0.725367
--0.054949
- 0.751924
- 0
- 0
- 0.673337
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.30665
- 0
- 0
- 0.86015
- 0
--0.876984
- 0.230109
- 0
- 0
- 0.630907
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.49027
- 0
- 0
- 0
- 0
- 0
- 0.527546
- 0
- 0.154843
- 0
- 0
- 0.618815
- 1
- 0
- 0
- 0
- 0.327075
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0416737
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.474633
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.93632
--0.424238
- 0
- 0
- 0
- 0
- 0
- 0.597711
- 0
- 0
- 0
- 0
- 0
- 0.65785
--0.457688
- 0.5479540000000001
- 0
- 0.482152
- 0
- 0
--0.906761
- 0
- 0
--0.502023
- 0.326552
- 0
- 0
- 0
--0.323529
- 0
- 0
- 0
- 0
- 0
- 0.388644
- 0
- 0
- 0.871507
- 0
- 0
- 0.643944
- 0.64539
- 0
- 0
- 0
--0.443978
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.809685
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.721132
- 0
- 0
- 1.460584
- 0
- 0.388158
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.108809
- 0
- 0
- 0
--0.483398
- 0
- 0.708294
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.480344
--0.169195
- 0
- 0
--0.6234690000000001
- 0.515229
--0.958591
--0.722326
- 0.0716487
- 0
--0.27721
- 0.648833
- 0.883103
- 0.22843
- 0
- 0
--0.994491
--0.552121
- 0.103506
- 0
--0.0893031
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.384351
- 0
- 0
- 0
- 0
- 0
- 0
- 0.828334
- 0
- 0
--0.607252
- 0
--0.5787910000000001
- 0
- 0
--0.900639
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.274741
- 0
- 0
- 0.0566341
- 0
- 0
- 1.549332
- 0
--0.741055
- 0
- 0.321289
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.146525
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.429723
- 0
- 0
- 0.535212
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.680748
- 0.480954
- 0.973766
- 0
- 0.0883703
--0.5017740000000001
- 0
- 0
- 0.0290042
- 0
- 0
--0.349416
--0.118985
- 0
--0.584266
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0299071
- 0
- 0
- 0.471451
- 0
- 0
- 0.333558
- 0
- 0
- 0
- 0
- 0
- 0.682487
- 0
- 0
- 0.87783
- 0
- 0
- 0.826115
- 0
- 0
- 0
- 0
- 0
- 0.296235
- 0.314654
- 0.386811
- 0
- 0
- 0
- 0
- 1
--0.3923
- 0
--0.6305500000000001
- 0
--0.548669
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5796289999999999
- 0
- 0
- 0
- 0
- 0
- 0
--0.166999
- 0
- 0
- 0
--0.19651
- 0
- 0
- 0
- 0
- 0
- 0
--0.00813035
- 0.466892
--0.469971
- 0
--0.185959
--0.0194231
- 0.288058
- 0
- 0.164142
--0.903722
- 0
- 0
- 0
- 0
- 0
- 0
--0.431219
- 0
- 0
- 0.199979
--0.254834
- 0
--0.04411
- 0.260507
- 0
--0.462448
- 0
- 0
- 0
- 0
--0.230016
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.021241
- 0.123359
- 0
- 0.355183
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.735237
- 0
- 0
- 0
- 0
- 0.320242
- 0
- 0
- 0
- 0
- 0
--0.225627
- 0
- 0
- 0
- 0.052578
- 0
- 1.857809
- 0
- 0
- 0
- 0
- 0
- 0
--0.111731
- 0.58955
- 0
- 0.554756
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.979145
- 0
- 0
- 0
- 0
- 0
- 0
--0.626348
- 0
- 0
- 0
- 0
- 0
- 0.358521
--0.140944
- 0
- 0
--0.726899
- 0
- 0.200139
- 0
- 0
- 0
- 0
- 0.686053
--0.657202
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.914959
- 0
--0.54144
--0.190381
- 0
--0.104641
- 0
--0.694403
- 0
- 0
- 0
- 0
- 0
- 0.5385450000000001
- 0
- 0
- 0.564571
- 0
- 0
- 0
--0.68738
- 0
- 0
- 0
- 0.684455
- 0
- 0.340214
- 0
- 0
- 0
- 0
- 0
- 0
--0.108828
- 0
--0.131478
- 0
- 0
- 0
- 0
- 0.838798
- 0
- 0
--0.354007
- 0
- 0
- 1
- 0.701021
--0.849293
- 0
- 0
- 0
- 0
--0.196869
- 0
- 0
- 0
--0.715937
- 0
- 0.857105
- 0
- 0
- 0
- 0.262442
- 0.303702
- 0
- 0
--0.336013
- 0
- 0
- 0
- 0
- 0
--0.903759
--0.991656
--0.5859799999999999
- 0
- 0
- 0
- 0
- 0.847888
--0.792662
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.504401
--0.577071
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0703101
- 0
--0.81279
- 0
- 0
--0.0746636
- 0
--0.5602200000000001
--0.430017
- 0
- 0
- 0
- 0
- 0
--0.487336
--0.649077
- 0
--0.106342
- 0
- 0
--0.223582
- 0
--0.113644
- 0
- 0.307304
--0.848152
- 0
- 0
--0.111501
- 0
- 0.753867
- 0.869327
- 0
- 0
--0.865273
- 0
- 0
- 0
- 0
- 0
--0.482212
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.536209
- 0
- 0
- 0
- 0
- 0.0232492
- 0
- 0
- 0
--0.536707
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.840183
- 0
- 0.279136
--0.0719142
- 0
- 0.470608
- 0
- 0
- 0
- 0
- 0
--0.07160039999999999
- 0
- 0
- 0
--0.326723
- 0
- 0
- 0
- 0
- 0
- 0.849594
- 0
- 0
- 0
- 0
- 0
- 0
- 0.781134
- 0.0505574
- 0
- 0
- 0
- 0
- 0
- 0.558619
- 0
- 0
- 0
- 0
--0.239596
- 0
- 0
- 0
- 0
--0.48385
- 0.292055
- 0
- 0
- 0
- 0
- 0
--0.819106
- 0
- 0.293568
--0.0414015
- 0
--0.643734
- 0.433746
- 0
- 0
- 0
- 0
- 1
- 0
- 0.284906
- 0
- 0
- 0.578257
- 0
- 0.0798566
- 0
- 0
- 0
- 0
- 0
--0.961184
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.098804
- 0.796017
- 0
- 0
- 0
- 0.648452
--0.132243
- 0.286178
- 0.281413
- 0.56848
- 0
--0.718452
- 0
- 0
- 0
- 0
- 0.935816
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.372333
- 0
- 0.268971
--0.247968
- 0
- 0
- 0
- 0
- 0.8371960000000001
- 0
- 0
- 0
- 0
- 0
- 0
--0.0509942
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.441172
- 0
--0.81114
- 0.399556
- 0
- 0
- 0
- 0.124196
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.17363
- 0
- 0.269626
- 0
--0.343679
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
--0.7109799999999999
- 0
--0.931864
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.971546
- 0.8814689999999999
--0.590248
- 0
- 0
- 0.837853
- 0
- 0.00621434
- 0
- 0
- 0
- 0
- 0
- 0
--0.27041
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.147777
- 0
- 0
- 0
- 0
- 0
- 0.06390469999999999
- 0
- 0
- 0.853331
- 0
--0.9840680000000001
- 0
- 0.533101
- 0
--0.695331
- 0
- 0
- 0
--0.248138
- 0
- 0
- 0
--0.428246
- 0
--0.17748
- 0
- 0.557453
- 0
- 0.564822
- 0
- 0
- 0
- 0
- 0
- 0
--0.679042
- 0
- 0
- 0.0218119
- 0
- 0.985649
- 0
--0.86148
- 0
- 0
- 0.628686
- 0.754839
--0.876493
- 0
- 0
- 0.710156
- 0
- 0
- 1
- 0
- 0
- 0.0239994
--0.356992
- 0
- 0
- 0
- 0.465411
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.144247
- 0
--0.177605
- 0
- 0.5092989999999999
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.31462
- 0.889902
- 0
- 0
--0.6141760000000001
- 0
- 0
- 0
- 0
- 0
--0.129879
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.307658
- 0
- 0
--0.0384306
--0.222658
--0.203847
- 0
- 0
--0.646328
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.761641
- 0
--0.334986
- 0
- 0
- 0
- 0
- 0
- 0
--0.872873
- 0
- 0
- 0.137286
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.97914
- 0
- 0
--0.572688
--0.309827
- 0
- 0
--0.64724
- 0
- 0.451524
- 0
- 0.302897
- 0
- 0
- 0
- 0
- 1
--0.62837
- 0.9483009999999999
- 0
--0.99048
- 0
- 0
- 0
- 0.176295
- 0.327433
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.211642
- 0
- 0
- 0.450837
- 0
- 0.0451532
--0.624559
--0.0263607
- 0
- 0
- 0
- 0.488915
--0.391069
- 0
- 0.521818
- 0
- 0.838289
- 0.09629450000000001
- 0
- 0.642062
- 0
- 0
- 0.07720630000000001
- 0.17289
- 0
- 0
- 0
--0.0262979
- 0
- 0
- 0
- 0
- 0
- 0.349681
- 0
- 0
--0.377493
- 0
- 0.75235
- 0.81458
- 0
- 0
- 0
- 0.046767
- 0
- 0
- 0
- 0
- 0
--0.908263
- 0
- 0.938391
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.594685
- 0
- 0.726817
- 0
- 0
- 0.322359
- 0.52391
- 0
- 0
- 0
- 0
- 0
- 0.9013600000000001
- 0
- 0
- 0
- 0
- 0
- 0
- 1
--0.50035
- 0
- 0
- 0
- 0
- 0
- 0
- 0.451309
--0.73406
- 0
--0.9554589999999999
- 0
- 0
--0.822056
--0.639111
- 0
- 0
--0.0482668
- 0.0465299
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.68206
- 0
- 0
- 0
- 0
- 0.800989
- 0
--0.396441
- 0.789783
- 0
--0.20036
- 0.254096
- 0
--0.188165
--0.0659429
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.9553199999999999
- 0
- 0
- 0
--0.252561
- 0
- 0
- 0.661473
- 0
--0.93027
- 0
- 0
- 0
--0.999474
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.527715
- 0
- 0
- 0
- 0
- 0
- 0.982822
--0.866102
- 0.47442
- 0
- 0
- 0
- 0
- 0.412638
- 0
- 0
- 0
- 0
- 0
- 0
- 0.793788
- 0
- 0.429709
--0.6330249999999999
--0.319963
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.702937
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.294167
- 0
- 0.682449
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.0568795
- 0
- 0.334295
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.703967
- 0.412607
- 0
--0.381381
- 0
- 0.559341
- 0
--0.473474
- 0
- 0
- 0
- 0
- 0.738256
- 0
- 0
- 0
- 0
--0.0574905
- 0.158766
- 0
- 0
- 0
- 0.695005
- 0
- 0.241035
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0695915
- 0
- 0
- 0.706532
- 0
- 0.907703
- 0
- 0.305903
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.252473
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.551775
- 0.198146
- 0
--0.157677
--0.824238
- 0
- 0.176356
--0.863127
- 0
- 0
- 0
- 0
- 0
--0.523977
--0.154902
- 0.58417
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.857947
- 0
- 0
- 0
- 0
- 0.7618740000000001
--0.803
- 0
- 0
- 0
- 0
- 0
- 0.932068
- 0
- 0
- 0
- 0
- 0
- 0.148857
- 0
- 0
- 0
--0.619658
- 0.922966
- 0
- 0
- 0
- 0.9714429999999999
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.214076
- 0
- 0
- 0.256854
- 0
--0.929716
- 0
- 0
- 0
- 0.645442
--0.855593
--0.592538
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.941651
- 0.864139
- 1
- 0.86504
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.9216839999999999
- 0
- 0
- 0.438863
- 0
- 0.7542410000000001
- 0
- 0
- 0
--0.451558
--0.425642
- 0
- 0
- 0.287925
- 0
- 0
- 0
- 0
- 0.311348
- 0
--0.0589609
- 0
- 0
- 0
- 0
--0.431072
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.484832
--0.0887162
- 0
- 0
--0.759043
- 0
- 0
- 0
- 0
- 0
- 0.183402
- 0.7576850000000001
- 0
- 0
- 0.690393
- 0.112233
- 0.9530380000000001
- 0
- 0
- 0
- 0
- 0.904833
- 0
- 0
--0.652309
- 0.354304
- 0
--0.931638
--0.897598
- 0.258748
--0.380534
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1
- 0
--0.700336
- 0
- 0
--0.775682
- 0
- 0
- 0
- 0
- 0
- 0
- 0.789168
- 0
- 0
--0.531953
- 0
- 0
- 0
- 0
- 0
- 0.827277
--0.484324
- 0.21164
--0.515428
--0.0275617
- 0
--0.952427
- 0
--0.190931
- 0
- 0
- 0.0494318
--0.375571
- 0
- 0.353396
- 0
- 0
--0.278438
- 0
- 0
- 0
- 0
- 0
- 0
--0.422178
- 0.0393633
- 0
- 0
- 0
- 0
--0.796573
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.994356
--0.664761
- 0
- 0
- 0
- 0.15448
- 0
- 0
--0.0262744
- 0
- 0
- 0
- 0
- 0
- 0
- 0.44675
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.0590007
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1
- 0
- 0
- 0.9615050000000001
- 0
- 0
- 0.363568
--0.386461
- 0
- 0
- 0
- 0
--0.101935
- 0
- 0
- 0
- 0.161245
- 0.603179
- 0
--0.524204
--0.501036
- 0.189373
- 0
- 0
- 0.562928
--0.550678
- 0
- 0
--0.77328
- 0
--0.365856
- 0.569609
- 0
--0.769385
--0.0912419
- 0
--0.842261
- 0
- 0
--0.196155
- 0.195635
- 0.973052
- 0
- 0
- 0
- 0
--0.379927
- 0
- 0
- 0
- 0
- 0
- 0.828494
- 0
- 0
- 0
- 0
--0.671508
--0.114764
- 0
- 0
--0.601373
- 0
--0.926522
--0.07070949999999999
- 0
- 0
- 0.731953
- 0
- 0
--0.968479
--0.486401
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.637302
--0.730666
- 0
- 0
--0.58639
- 0.836741
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.244236
- 0
--0.520509
- 0
- 0.884535
- 0
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.269352
- 0
- 0
--0.800805
- 0
- 0
- 0
- 0.9159040000000001
--0.393839
--0.648084
- 0
- 0
- 0.436883
- 0
- 0.190425
- 0
- 0.905228
- 0
- 0
- 0
- 0.382932
- 0.736825
- 0
- 0
--0.394712
- 0
- 0
- 0
- 0
--0.632061
- 0
- 0
--0.0550014
- 0
- 0.321859
- 0
--0.606473
- 0
- 0
- 0
- 0
- 0
- 0.277427
- 0.206675
- 0
- 0
- 0
- 0
- 0
--0.07083299999999999
- 0
- 0
- 0.53715
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.94216
- 0
- 0
- 0
- 0
--0.979495
- 0
- 0
- 0
--0.0696567
- 0
- 0.874463
- 0
- 0
--0.493464
- 0
- 0
- 0
- 0
--0.551921
--0.972986
- 0
- 0
- 0
- 0
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0413357
- 0
- 0
- 0
- 0
- 0
- 0
--0.1557
- 0
--0.0225334
--0.5547569999999999
- 0
- 0
- 0.460477
- 0.057137
- 0
- 0
- 0
--0.232163
- 0
--0.790698
- 0
--0.23347
- 0.8356
- 0
- 0
- 0.502562
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.00316352
- 0
- 0.494408
- 0
--0.112819
- 0
- 0
- 0
- 0
- 0
- 0.433862
- 0
- 0.550571
- 0
- 0
--0.294804
- 0
- 0
--0.790473
- 0
- 0
- 0
- 0.278551
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.7813369999999999
- 0
- 0
- 0
- 0
- 0
--0.8251230000000001
--0.6881080000000001
- 0
--0.615936
- 0
- 0
- 0.460652
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.561659
- 0
- 0
- 1
--0.233085
- 0
--0.567922
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.128106
--0.066507
- 0
- 0
- 0
--0.693791
- 0.390723
- 0
- 0
- 0
- 0
- 0
- 0.8136330000000001
- 0
- 0.128001
- 0
- 0
- 0
- 0.749609
- 0
- 0
- 0
- 0
--0.973876
- 0
- 0
- 0
--0.573386
- 0
- 0.505124
- 0
- 0
- 0.301199
- 0
- 0
- 0
- 0
- 0
- 0.381741
- 0
- 0.165357
- 0
- 0
- 0.741671
- 0
- 0
- 0
- 0
- 0
- 0
- 0.7546580000000001
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.0895673
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.774469
- 0
- 0
--0.200003
- 0
- 0
--0.875549
- 0
- 0
- 0
- 0
- 0
- 0
--0.300597
- 0
- 0
- 0
--0.0424351
- 0.0350317
- 0
--0.68727
- 1
- 0
- 0.327365
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.94607
- 0
--0.835302
- 0
--0.447948
- 0
- 0
--0.154438
- 0
- 0
- 0
--0.951099
--0.650054
- 0
- 0
- 0
- 0
- 0
- 0.478363
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.409591
- 0
- 0.0498791
--0.867816
- 0
- 0
- 0
- 0
- 0.765087
- 0
--0.0401508
- 0
- 0.492045
- 0
- 0
- 0
- 0
--0.469223
--0.264128
- 0
- 0
--0.231793
- 0
- 0.227703
- 0.442429
- 0
- 0
- 0
- 0
--0.117015
- 0.8653690000000001
--0.93642
--0.08699990000000001
- 0
- 0
- 0
--0.928704
- 0
--0.114269
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.9006
- 0
- 0.030293
- 0
- 0
- 0
- 0
- 1
- 0
- 0
- 0
- 0.668677
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.636721
- 0
- 0
- 0.06731570000000001
--0.64747
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.770371
- 0
- 0
- 0.809311
- 0
--0.384487
--0.169245
- 0
- 0
- 0.165835
--0.169889
- 0
- 0
- 0
--0.8056449999999999
- 0
- 0
- 0
- 0
- 0
--0.5241479999999999
- 0
- 0
- 0
- 0
- 0
- 0
--0.8078650000000001
--0.564747
--0.862371
- 0
- 0
--0.494426
--0.66308
- 0
- 0
--0.555857
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.470346
- 0
- 0
- 0
- 0
- 0
--0.769811
- 0
- 0
- 0
- 0.580275
--0.110297
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.982152
- 0
- 0
- 0
- 0.839656
- 0
- 1.42352
- 0
- 0
- 0
- 0
- 0
- 0
- 0.344797
- 0
- 0
- 0
--0.203743
- 0
--0.665435
- 0
- 0
- 0
- 0
- 0.145982
- 0
--0.704364
- 0
- 0
--0.777087
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.168221
- 0
- 0
- 0
- 0.359237
- 0
- 0
- 0
- 0.238345
- 0
--0.215334
- 0
- 0
- 0
- 0.0736675
- 0
- 0.228351
- 0
- 0
--0.0586232
--0.574757
- 0
- 0
- 0
- 0
- 0
--0.481291
- 0
--0.500246
- 0
- 0
--0.897141
- 0
- 0
- 0
- 0.506937
--0.767756
- 0
- 0.561381
- 0
- 0.8435279999999999
- 0
- 0
- 0
- 0
--0.0544575
--0.324954
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.652217
- 0
- 0
- 0.449998
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.230641
- 0.175307
- 0
--0.854996
- 0
--0.580019
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.398972
- 0
--0.228622
--0.843621
- 0.298253
- 0.744174
--0.619254
- 0
- 0
- 0.649302
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.7530250000000001
- 0
- 0
- 0
- 0
- 0
- 0.503371
- 0
--0.07167999999999999
- 0
- 0
- 0
--0.0286198
- 0
- 0.328736
- 0
--0.676026
- 0
- 0
- 0
- 0
- 0.688148
- 0
- 0
- 0
- 0
--0.108486
- 0
- 0
- 0
- 0
- 0
--0.942459
--0.705596
- 0
--0.0189803
- 0
- 0.247192
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.964525
- 0
- 0.660018
- 0
- 0
- 0
- 0
--0.882556
- 0
- 0
- 0
- 0
- 0
- 1
--0.0247446
- 0
- 0
- 0
- 0
--0.526309
- 0
- 0
- 0.108938
- 0
- 0
--0.0344695
- 0.321215
- 0
- 0
- 0.231956
- 0
- 0
- 0
- 0
- 0
- 0
- 0.611652
- 0
- 0
- 0
- 0
--0.521822
- 0.7905
- 0
- 0
- 0.718824
- 0
--0.143417
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.0686484
- 0
- 0
- 0
- 0
- 0
--0.201948
- 0.78129
- 0
- 0
- 0
- 0
- 0
--0.417287
- 0
- 0
- 0.504935
- 0.902868
- 0
- 0
- 0
- 0
- 0
- 0.0861237
- 0.813936
- 0
- 0
- 0
--0.09783940000000001
- 0
- 0
--0.311002
--0.606554
- 0
- 0
- 0.640389
- 0
--0.9259810000000001
--0.797533
- 0.745912
- 0.0630105
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.711036
- 0
- 0
--0.787053
- 0
- 0
- 0
- 0
- 0
--0.0491289
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.835087
- 0
- 0
- 0.540913
- 0.111028
- 0
- 0
- 0
--0.451531
- 0
--0.106614
- 0
- 0
- 0
- 0
- 0
- 0
- 0.6152570000000001
- 0
- 0
--0.287537
- 0
--0.22707
- 0
--0.196099
- 0
--0.823563
- 0.318277
- 0
- 0
--0.9555709999999999
- 0
--0.0111826
- 0
- 0
- 0
- 0
--0.0273952
- 0.646208
- 0
--0.00414264
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.896778
- 0
- 0
- 0
- 0
- 0.08436399999999999
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.46007
- 0
--0.979048
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.645428
- 0
- 0.427368
- 0
- 0
- 0
- 0
- 0
- 0.231126
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.647655
- 0
--0.569102
- 0.623787
- 0.324656
- 0
- 0
--0.10013
- 0
- 0
- 0
- 0
--0.808626
- 0.835592
- 0
- 0
--0.773088
- 0.253501
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.0647218
- 0
- 0
- 0
- 0
- 0.7913
- 0
- 0
- 0
- 0
--0.363972
- 0
- 0.272026
- 0
--0.260402
- 0
- 0
- 0
- 0
--0.951564
- 0
- 0
- 0
--0.383666
- 0
- 0
- 0.269108
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.608417
- 0
- 0
- 0
- 0.364889
- 0
- 0
--0.544045
--0.874072
- 0
- 0
- 0
- 0
- 0.806578
--0.196817
- 0
- 0
- 0
- 0.628158
- 0
- 0
- 0
- 0
--0.124168
--0.625259
- 0
- 1
- 0
- 0.821933
- 0
- 0
- 0
--0.725438
- 0
- 0
--0.42745
--0.345529
- 0
- 0
- 0
- 0
- 0
- 0
--0.392244
--0.195797
- 0.809964
- 0
- 0
- 0
--0.452078
- 0
- 0.753362
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.391069
--0.601471
- 0
- 0
--0.272779
--0.843699
- 0.938086
- 0
- 0
- 0
- 0
- 0
--0.223789
- 0
- 0
- 0
--0.0906565
- 0.792938
- 0
- 0
- 0
- 0
--0.466337
- 0
--0.212949
- 0
- 0
--0.767822
- 0
- 0
- 0
- 0
--0.0578849
- 0
- 0.932304
- 0
--0.248808
- 0
- 0.546924
--0.291615
- 0.940302
--0.535679
--0.8729980000000001
--0.0907119
- 0.180582
--0.62288
--0.439856
- 0
--0.735379
- 0
- 0
- 0
- 0.257706
- 0
- 0
- 0
- 0.104182
- 0
- 0.683724
- 0
- 0
- 0
--0.50883
- 0
- 0
- 0
- 0
- 0
- 1
- 0
- 0
- 0.379411
- 0
- 0
- 0
- 0
- 0
- 0
- 0.709603
--0.330541
- 0.17339
- 0
- 0
- 0
--0.610659
- 0.470957
--0.699522
- 0.410749
- 0
- 0
--0.38613
- 0
- 0
--0.662214
- 0
--0.190436
- 0.686275
- 0
- 0
- 0
--0.808785
- 0.714234
--0.659617
- 0
- 0
- 0
- 0
- 0
- 0.826365
- 0
- 0
- 0
--0.736697
--0.928829
- 0
- 0
- 0.941635
- 0
- 0
- 0
--0.0520551
- 0
- 0
- 0
--0.411873
- 0
- 0
--0.377701
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.8972290000000001
--0.260501
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.009926620000000001
- 0
- 0
- 0.956938
- 0
--0.529857
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.159394
- 0
- 0
- 0
- 1
- 0
- 0
- 0.38418
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0165425
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.936323
--0.311156
--0.905285
- 0
--0.921131
- 0.242099
- 0
- 0
--0.607228
- 0
- 0
- 0.561025
- 0
- 0
--0.995801
- 0
--0.698597
- 0
- 0
- 0
- 0.0698281
- 0
- 0.512233
- 0.694503
- 0.233714
- 0.922603
- 0
- 0.0385542
--0.981787
- 0
- 0
- 0
- 0
- 0.287984
- 0
- 0
- 0
- 0
--0.612909
--0.175915
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.408939
--0.425077
- 0
- 0
- 0
- 0.210489
- 0
- 0
- 0
- 0.573742
- 0
- 0
- 0
- 0
--0.0777447
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.256117
- 0
- 0
- 0.886971
--0.562997
- 1
- 0
- 0
- 0
- 0
- 0.153193
- 0.321326
- 0
- 0
- 0.07524359999999999
- 0
- 0
--0.584502
- 0
- 0
--0.27177
- 0
- 0
- 0
--0.588381
--0.44643
- 0
- 0
- 0.369778
- 0
--0.8797199999999999
- 0
- 0
--0.178052
--0.0389853
--0.225794
- 0
- 0
- 0
- 0
- 0
--0.59096
- 0
--0.723281
- 0
- 0
- 0
- 0
--0.851383
- 0
- 0
--0.11226
- 0.709249
- 0
- 0
- 0
- 0
- 0.14424
--0.680826
- 0
- 0
- 0
- 0
- 0
--0.155768
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.486373
- 0.589572
- 0
- 0
- 0
- 0
- 0.893585
- 0
- 0
- 0.15804
- 0
- 0.56112
- 0
- 0
--0.299121
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1
- 0
- 0
- 0
- 0.7186709999999999
--0.381432
- 0
- 0
- 0.0266486
- 0
- 0
- 0
- 0
- 0
- 0
--0.582473
--0.439777
--0.929901
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.525309
- 0
- 0
--0.953654
- 0
- 0.516549
- 0
- 0
- 0
- 0
--0.759543
- 0
- 0
--0.579677
- 0
- 0
- 0.323576
- 0
- 0.622624
--0.567655
--0.382691
- 0.963982
- 0.237792
- 0.843202
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.417184
- 0
- 0
- 0.182926
- 0
- 0
- 0
- 0
- 0
- 0
- 0.245849
- 0
- 0.920666
- 0.720187
- 0.749987
- 0
--0.608874
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.996817
- 0
- 0
- 0
- 0
- 0
--0.787933
--0.486203
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.931391
- 0
--0.475301
- 0
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0.311361
- 0
--0.851772
- 0
- 0
- 0
- 0
--0.393754
--0.364588
--0.239975
- 0
- 0
- 0
- 0
- 0.0842064
- 0
- 0.576852
- 0
- 0
- 0
- 0
- 0
--0.653818
- 0
- 0
- 0
- 0.7866880000000001
- 0
- 0
--0.63191
- 0.886781
- 0
- 0
- 0
- 0.85899
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.175979
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.162419
- 0.976518
- 0
- 0.802856
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.78316
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.74036
- 0
- 0
- 0
- 0.004632999999999998
- 0
- 0
--0.0223174
- 0
- 0
--0.146539
- 0
--0.905285
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.449744
- 0
- 0
- 0
--0.955958
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.537241
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.9094640000000001
--0.87996
- 0
--0.700531
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.0952012
- 0
- 0
- 0
- 0
- 0
--0.0350508
- 0
- 0
--0.239566
- 0
- 0.122597
- 0.941122
- 0
- 0
--0.08898780000000001
- 0
- 0
- 0
- 0.03288
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.857707
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.729566
- 0
- 0
- 1
- 0.991129
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.72195
- 0
- 0
--0.435701
- 0.427308
- 0
--0.826875
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.773828
- 0
- 0.0933237
- 0
- 0
- 0.662411
--0.0641785
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0350553
- 0
- 0
- 0
- 0
--0.652898
--0.343929
- 0
- 0
- 0
- 0
- 0.687895
- 0
- 0
- 0
- 0
- 0
- 0.72954
- 0
--0.00692189
- 0
- 0.656344
- 0
- 0
- 0
- 0
- 0
--0.9322009999999999
- 0
- 0
- 0
- 0
- 0.940073
- 0
- 0
- 0
- 0.414453
- 0
- 0
- 0
- 0
--0.9405019999999999
- 0
- 0.988143
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.764223
- 0
- 1
- 0
- 0
--0.839417
- 0
- 0
- 0
--0.680931
- 0
- 0
- 0
- 0
- 0
- 0
--0.09761549999999999
- 0.5863390000000001
- 0
- 0.365913
- 0
- 0
- 0
- 0
- 0
--0.294686
- 0
- 0
- 0.128318
- 0.771414
- 0.374154
- 0
- 0
- 0
- 0
--0.58684
--0.161994
- 0
- 0
- 0
--0.353274
- 0.770666
--0.916142
- 0
- 0
- 0
- 0
- 0
- 0
--0.596986
- 0.363281
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.240488
- 0
- 0
- 0
- 0
- 0.835852
--0.499483
--0.98917
- 0.983189
- 0.8422460000000001
- 0
- 0
--0.252077
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.349788
- 0
- 0
- 0
- 0.202314
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.114781
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.221866
- 1
- 0.00523072
- 0
- 0
- 0.575969
- 0
- 0
- 0
- 0
- 0
- 0.434297
--0.300445
- 0
--0.92587
- 0.645008
- 0
- 0.997601
- 0
- 0
- 0.705715
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.155757
- 0
--0.790951
- 0
- 0.660185
- 0.533357
- 0
- 0
--0.367672
- 0
- 0
- 0
- 0
--0.877645
- 0
- 0.7255819999999999
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.788188
--0.237091
- 0
- 0
- 0
- 0.317481
- 0.165559
- 0.982955
--0.0286885
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.203245
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.746632
- 0
- 0
- 0
- 0
- 0
- 0.945618
- 0.556732
- 0
- 0
- 0.0982452
- 0
- 1
- 0
- 0
- 0
- 0.561302
- 0
- 0
- 0
- 0
--0.206327
- 0
- 0
- 0.841022
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.126896
- 0
- 0
- 0.709178
- 0.992294
- 0.5909759999999999
- 0.671619
- 0.168266
- 0
- 0
- 0
- 0
- 0
--0.19823
- 0
- 0
- 0
- 0
--0.424976
- 0
- 0
- 0
- 0
- 0
- 0
- 0.323275
- 0
- 0
- 0
- 0
- 0
- 0.351516
- 0.829498
- 0
- 0
--0.575054
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.104291
- 0
- 0
--0.857495
- 0
- 0
- 0.833725
- 0
- 0
- 1
--0.609754
- 0
- 0
- 0
- 0
- 0
- 0.905613
--0.757395
--0.889359
- 0
--0.0988322
- 0
- 0.707666
- 0.567767
- 0
- 0.641566
- 0.8926269999999999
- 0.574334
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.811997
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.860195
- 0
- 0
- 0
- 0.636025
--0.263735
- 0.832127
--0.394369
- 0
- 0
--0.125291
- 0
- 0.283666
- 0
- 0
- 0.227194
--0.199703
- 0
- 0
- 0
- 0.118292
- 0
- 0
- 0
- 0
- 0.492129
- 0
--0.0154354
- 0
- 0.563103
- 0.240693
--0.0331547
- 0
- 0
- 0
- 0
- 0
- 0.863059
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.828261
- 0
- 0
- 0
- 0.181547
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.848399
- 0
- 0
- 0
--0.987278
- 0.279914
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.554204
- 0
- 0
- 0
- 0
- 0
- 0.787458
- 0
- 0
--0.8421419999999999
- 0
- 0
--0.237401
- 0
- 0
- 0
- 0
- 0
- 0
- 0.118921
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.237383
- 0
--0.231609
- 0
- 0
- 0
- 0
- 0
- 0.760876
- 0
- 0
--0.0962519
- 0
- 0
- 0
- 0
- 0
--0.22447
- 0
- 0
- 0.430901
--0.210851
- 0
- 0.592892
- 0.254826
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.452211
- 0
--0.917371
- 0
- 0
- 0.308908
--0.973656
- 0
- 0
- 0
- 0
- 0
- 0
- 0.00114599
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.883929
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.580774
- 0
- 1
- 0
- 0
- 0
- 0.890725
--0.597643
- 0
- 0
- 0.86953
- 0
- 0
--0.68729
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.9751
--0.149455
- 0
- 0.436949
- 0
- 0
- 0
- 0
- 0.585487
- 0.894138
- 0
- 0.747988
--0.366513
- 0.249258
- 0
--0.5717100000000001
--0.311366
- 0
- 0
- 0.943872
- 0
- 0.9739409999999999
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.0667751
- 0
--0.497347
- 0
- 0
- 0
--0.112055
- 0
--0.935571
- 0
--0.985701
- 0
- 0
- 0
- 0
- 0
- 0.603685
- 0
- 0
- 0.553101
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.525407
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.236973
- 0
--0.400151
- 0
--0.51612
- 0
- 0
- 0
- 1
- 0
- 0.130847
- 0
- 0
- 0
- 0.459564
- 0
--0.503699
- 0
- 0
- 0
- 0
- 0
--0.9217689999999999
- 0
- 0
- 0
- 0
- 0
--0.454757
- 0
- 0
- 0
- 0.649541
- 0
- 0
- 0
- 0.07609059999999999
- 0
- 0.691545
- 0
- 0.348616
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.269245
- 0
- 0
- 0
- 0.8483309999999999
- 0.326492
- 0
--0.110276
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.643369
- 0
- 0
- 0
- 0
- 0
- 0.854979
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.775751
- 0
- 0
- 0
--0.668401
- 0
- 0
- 0
- 0
- 0.209881
- 0
- 0.0614556
- 0.9025879999999999
- 1.927103
- 0
--0.00522418
- 0
- 0.996584
--0.964189
- 0
- 0
- 0
- 0
- 0.443105
- 0
- 0
--0.497132
- 0
- 0
- 0.871127
- 0
- 0
- 0
- 0
- 0
- 0
--0.900002
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.498691
- 0
- 0
- 0
- 0
- 0
--0.82908
- 0
--0.816675
--0.0974974
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.656752
- 0
- 0.203061
--0.0371787
- 0
- 0
- 0
- 0
- 0
--0.914176
- 0
- 0
--0.133199
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.287786
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.153057
- 0.658984
- 0
--0.530613
- 0
- 0
- 0.487696
- 1
--0.741605
- 0
--0.954681
- 0.7059
- 0
- 0
--0.9710839999999999
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.438876
- 0.526448
--0.411051
- 0
- 0
- 0
- 0
- 0
--0.513471
--0.175503
--0.0788451
--0.901339
- 0
- 0
- 0.958033
- 0
- 0
- 0
- 0.509942
- 0
- 0
- 0
- 0
- 0
- 0.311675
- 0
- 0
- 0
- 0
- 0
--0.0221582
- 0
--0.551868
- 0
--0.44429
- 0
- 0
- 0
- 0
- 0.0831853
- 0
- 0
--0.466401
- 0
- 0
- 0.197286
- 0
- 0
- 0
--0.930013
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.152819
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.445506
--0.596526
- 0
- 0
- 0
- 0
- 0
- 0
- 1.45265
- 0
- 0.189004
- 0
- 0
--0.902206
--0.551132
- 0
- 0.590998
- 0.920715
- 0
--0.350707
- 0.530287
- 0
- 0
- 0
--0.119325
- 0.791968
--0.695116
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.639161
- 0
--0.412946
--0.46124
- 0
- 0
- 0
- 0.0343029
- 0
- 0
- 0
- 0
- 0
- 0.915203
- 0
- 0
--0.9513470000000001
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.301
- 0.361656
- 0
- 0
- 0.64212
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.897021
- 0
--0.195809
- 0
- 0.428701
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.74202
- 0
--0.0132017
- 0
- 0
--0.171258
- 0
- 1
- 0
--0.977328
- 0
- 0
- 0
- 0
- 0
--0.334471
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.996248
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.980129
- 0
- 0.259116
- 0
--0.645209
- 0
- 0
- 0
- 0
- 0.694214
--0.921286
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.671543
- 0
- 0
--0.191846
- 0
- 0
- 0.5371899999999999
- 0
- 0
- 0.61043
- 0.106201
- 0
- 0
- 0
- 0
- 0
--0.6407620000000001
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.471755
- 0
- 0
- 0
--0.952646
- 0
--0.914968
- 0
- 0
- 0
- 0
- 0
- 0
--0.238788
- 0.202896
- 0.153444
- 0
- 0
- 0
- 0
- 0
- 0
--0.38162
- 0
--0.464698
- 0
--0.596566
- 0
- 0
- 0
- 1.91427
- 0
- 0
- 0.10542
- 0
- 0
- 0
--0.471454
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.118783
- 0
- 0.511216
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.405567
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.119054
--0.461136
- 0
- 0.170682
- 0
- 0
- 0
- 0
--0.8219649999999999
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.287439
- 0.676252
- 0
- 0
- 0
- 0
--0.0837363
- 0
--0.980019
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.936594
- 0
- 0
- 0
- 0
--0.191702
--0.832917
- 0
--0.576425
- 0
- 0
- 0
- 0
--0.5559539999999999
--0.503017
- 0
- 0
--0.771043
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1
- 0
- 0
- 0
- 0
- 0
--0.577082
- 0
- 0
--0.0510836
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.981663
--0.487206
- 0
- 0.115087
--0.915847
- 0
- 0
- 0.380454
- 0
--0.370018
- 0
- 0
- 0.431913
- 0
- 0
- 0
--0.896953
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0402722
- 0
- 0
- 0.9433859999999999
- 0
- 0
--0.112292
- 0
--0.269927
- 0
- 0
- 0
- 0
- 0
- 0.72212
- 0
- 0
- 0.115987
- 0
- 0
- 0
- 0
--0.246633
- 0
- 0.421037
- 0
- 0
--0.481999
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.995146
- 0
- 0
--0.888074
- 0
- 0.466047
- 0
--0.8479179999999999
- 0
- 0
- 0
- 0
- 0
--0.496357
--0.246731
- 1
- 0
- 0
- 0
- 0
- 0
- 0
--0.0651752
- 0.439534
- 0
- 0.87233
- 0.927475
- 0
- 0
- 0
- 0.48891
--0.34085
- 0
--0.918233
- 0
--0.550022
--0.314738
- 0
- 0
--0.526939
- 0.454792
- 0
--0.144511
- 0.652778
- 0.338969
- 0
--0.266674
- 0
- 0
- 0.493049
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.109431
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0539152
--0.266303
- 0
- 0.8637359999999999
- 0
- 0
- 0
--0.313799
- 0
- 0
- 0
- 0.11005
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.117717
- 0
- 0
- 0
- 0.904555
- 0
--0.276677
- 0
- 0
- 0
- 0
- 0
- 0
- 0.812263
- 0
- 0.5696020000000001
- 0
--0.363594
- 0
- 0
- 0
- 0
- 0
- 0
- 0.664364
- 0.91682
- 0
- 0
- 0
- 1
- 0.914829
- 0
- 0
- 0
- 0
- 0
--0.893093
- 0
- 0
--0.478477
--0.749803
- 0
- 0
- 0
- 0
- 0
--0.973556
- 0
- 0.620466
- 0
- 0
- 0
- 0.316724
- 0.841707
- 0
- 0
- 0
- 0.448781
--0.543077
- 0
- 0
- 0
- 0
- 0.302511
- 0
--0.10084
- 0
- 0
- 0
- 0
--0.748879
- 0
- 0
- 0
- 0.162529
- 0
- 0
- 0
- 0
- 0
--0.232837
- 0.465888
--0.310108
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.790501
- 0
--0.307767
- 0
--0.0885382
- 0
- 0
- 0.240963
- 0
- 0
- 0
- 0
- 0.341789
- 0
--0.446857
- 0
- 0
- 0.818532
- 0
- 0
- 0.131666
- 0
- 0
- 0
- 0
- 0
--0.566569
- 0.904648
- 0
- 0
- 0.394674
- 0
--0.0333986
- 0
--0.511354
- 0
- 0
- 0
- 1
- 0
- 0.20747
--0.694747
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.510774
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.7771439999999999
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.868241
- 0
- 0
- 0.0478857
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.6987640000000001
- 0
- 0
- 0
- 0
- 0.126435
--0.809482
- 0
- 0
- 0
- 0.998844
- 0
--0.203976
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.96052
- 0.9342780000000001
- 0.728096
- 0
- 0
--0.43435
- 0
- 0
- 0
- 0.320126
- 0
- 0
--0.324934
--0.0915899
--0.155267
- 0.0514749
- 0
- 0
--0.562667
- 0
- 0
- 0
- 0
- 0
- 0
- 0.369753
- 0.0957861
- 1
- 0
- 0
- 0
- 0
- 0.6669389999999999
- 0
--0.268673
- 0
- 0
--0.7887420000000001
--0.842994
- 0
- 0
- 0
- 0
- 0
- 0
- 0.20721
- 0
- 0
- 0.25425
- 0.75204
- 0.809456
- 0.579693
--0.2701
- 0
- 0
--0.211509
- 0.196984
- 0.989522
- 0.290667
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.529594
- 0
- 0
- 0
- 0
- 0
- 0.0966627
- 0
--0.261063
- 0
--0.649928
--0.852185
--0.807771
- 0.59355
- 0.429805
- 0
- 0
- 0
- 0
- 0.158546
--0.195617
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.188405
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.275189
- 0
- 0.929624
- 0
- 0
- 0
- 0
--0.976168
- 0
--0.267403
- 0
- 0.0969252
- 0
- 0
- 0.945228
- 0.882784
- 0.347699
- 0
- 0.497427
- 0
- 0
- 0
- 0
- 0.177359
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.366913
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.383278
- 0
- 0.284306
- 0
- 0
- 0
- 0.124199
- 0
--0.252784
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.0688125
- 0
- 0
- 0
- 0
- 0
- 0.954908
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.146999
- 0.499096
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.689511
- 0
- 0
--0.0515379
- 0
- 0
- 0
- 0
- 0
- 0
- 0.98959
- 0.526482
- 0
--0.421562
- 0
- 0
--0.378078
- 0
- 0
--0.595798
- 0
- 0.212354
- 0
- 0
- 0.915579
--0.00151776
- 0
- 0
- 1
- 0
- 0.714744
--0.105761
- 0
- 0
--0.265856
- 0
- 0
- 0
- 0
- 0
- 0.35725
--0.565294
- 0.18039
--0.544199
- 0.766061
- 0
- 0
--0.47727
- 0
--0.0274381
- 0.296647
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.0781284
- 0.932547
- 0
- 0
--0.462555
- 0
- 0
- 0
- 0
--0.08383939999999999
- 0
- 0.407766
- 0
--0.757333
- 0.526645
- 0
- 0
- 0.76839
- 0
--0.416188
- 0
- 0
- 0
- 0.385427
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.6073499999999999
- 0
--0.885592
- 0
- 0
- 0
- 0
- 0
- 0.29486
--0.00479562
- 0
- 0
--0.7513339999999999
--0.485451
- 0
- 0
- 0
--0.0364946
- 0
- 0.409823
- 0.385125
- 0
- 0.326716
- 0
--0.645059
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1
--0.533578
- 0
- 0
--0.141391
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.193076
- 0
- 0
- 0
--0.5870919999999999
- 0
- 0
- 0
- 0
- 0
- 0.311298
- 0
- 0
- 0
- 0.280015
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.0600836
- 0
- 0
- 0
--0.299407
- 0.976213
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.251438
- 0
- 0
- 0
- 0
- 0
- 0
--0.462446
- 0
- 0
- 0
- 0
- 0
- 0.609112
--0.997952
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.6219789999999999
- 0.50054
- 0
--0.612312
- 0
- 0.0358533
- 0
- 0.968217
- 0.801179
--0.446137
- 0
- 0
- 0
- 0
- 0.440971
- 0.0694135
- 0
- 0.871193
- 0
- 0
- 0
- 0
- 1.112607
- 0.5077700000000001
- 0
--0.223485
- 0
- 0
--0.510635
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.160727
- 0
- 0.414484
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.461221
- 0
- 0.931423
- 0
- 0
- 0
- 0
--0.454996
- 0
- 0
- 0
- 0
- 0
--0.355027
- 0
--0.606612
- 0
- 0.7776149999999999
- 0.368264
--0.478208
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.253271
- 0.345292
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.164102
- 0
- 0
- 0
- 0.530361
- 0
- 0
- 0
- 0
--0.837771
- 0
- 0.355711
- 0
--0.129631
- 0
- 0
- 0
--0.203876
- 0
--0.551889
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.545945
--0.896469
- 0
- 0
- 1
+%%MatrixMarket matrix coordinate real general
+% Generated 15-Dec-2020
+100 100 413
+1 1  1
+2 1  2
+36 1  1
+2 2  1
+3 2  2
+36 2  1
+1 3  3
+3 3  1
+4 3  2
+36 3  1
+2 4  3
+4 4  1
+5 4  2
+36 4  1
+3 5  3
+5 5  1
+6 5  2
+36 5  1
+4 6  3
+6 6  1
+7 6  2
+36 6  1
+5 7  3
+7 7  1
+8 7  2
+36 7  1
+6 8  3
+8 8  1
+9 8  2
+36 8  1
+7 9  3
+9 9  1
+10 9  2
+36 9  1
+8 10  3
+10 10  1
+11 10  2
+36 10  1
+9 11  3
+11 11  1
+12 11  2
+36 11  1
+10 12  3
+12 12  1
+13 12  2
+36 12  1
+11 13  3
+13 13  1
+14 13  2
+36 13  1
+12 14  3
+14 14  1
+15 14  2
+36 14  1
+13 15  3
+15 15  1
+16 15  2
+36 15  1
+14 16  3
+16 16  1
+17 16  2
+36 16  1
+15 17  4
+17 17  1
+18 17  2
+36 17  1
+15 18  4
+16 18  3
+18 18  1
+19 18  2
+36 18  1
+15 19  4
+17 19  3
+19 19  1
+20 19  2
+36 19  1
+15 20  4
+18 20  3
+20 20  1
+21 20  2
+36 20  1
+15 21  4
+19 21  3
+21 21  1
+22 21  2
+36 21  1
+15 22  4
+20 22  3
+22 22  1
+23 22  2
+36 22  1
+15 23  4
+21 23  3
+23 23  1
+24 23  2
+36 23  1
+15 24  4
+22 24  3
+24 24  1
+25 24  2
+36 24  1
+15 25  4
+23 25  3
+25 25  1
+26 25  2
+36 25  1
+15 26  4
+24 26  3
+26 26  1
+27 26  2
+36 26  1
+15 27  4
+25 27  3
+27 27  1
+28 27  2
+36 27  1
+15 28  4
+26 28  3
+28 28  1
+29 28  2
+36 28  1
+15 29  4
+27 29  3
+29 29  1
+30 29  2
+36 29  1
+15 30  4
+28 30  3
+30 30  1
+31 30  2
+36 30  1
+15 31  4
+29 31  3
+31 31  1
+32 31  2
+36 31  1
+15 32  4
+30 32  3
+32 32  1
+33 32  2
+36 32  1
+15 33  4
+31 33  3
+33 33  1
+34 33  2
+36 33  1
+15 34  4
+32 34  3
+34 34  1
+35 34  2
+15 35  4
+33 35  3
+35 35  1
+36 35  2
+15 36  4
+34 36  3
+36 36  1
+37 36  2
+15 37  4
+35 37  3
+37 37  1
+38 37  2
+15 38  4
+36 38  3
+38 38  1
+39 38  2
+15 39  4
+37 39  3
+39 39  1
+40 39  2
+15 40  4
+38 40  3
+40 40  1
+41 40  2
+15 41  4
+39 41  3
+41 41  1
+42 41  2
+15 42  4
+40 42  3
+42 42  1
+43 42  2
+15 43  4
+41 43  3
+43 43  1
+44 43  2
+15 44  4
+42 44  3
+44 44  1
+45 44  2
+15 45  4
+43 45  3
+45 45  1
+46 45  2
+15 46  4
+44 46  3
+46 46  1
+47 46  2
+15 47  4
+45 47  3
+47 47  1
+48 47  2
+15 48  4
+46 48  3
+48 48  1
+49 48  2
+15 49  4
+47 49  3
+49 49  1
+50 49  2
+15 50  4
+48 50  3
+50 50  1
+51 50  2
+15 51  4
+49 51  3
+51 51  1
+52 51  2
+15 52  4
+50 52  3
+52 52  1
+53 52  2
+15 53  4
+51 53  3
+53 53  1
+54 53  2
+15 54  4
+52 54  3
+54 54  1
+55 54  2
+15 55  4
+53 55  3
+55 55  1
+56 55  2
+15 56  4
+54 56  3
+56 56  1
+57 56  2
+15 57  4
+55 57  3
+57 57  1
+58 57  2
+15 58  4
+56 58  3
+58 58  1
+59 58  2
+15 59  4
+57 59  3
+59 59  1
+60 59  2
+15 60  4
+58 60  3
+60 60  1
+61 60  2
+15 61  4
+59 61  3
+61 61  1
+62 61  2
+15 62  4
+60 62  3
+62 62  1
+63 62  2
+15 63  4
+61 63  3
+63 63  1
+64 63  2
+15 64  4
+62 64  3
+64 64  1
+65 64  2
+15 65  4
+63 65  3
+65 65  1
+66 65  2
+15 66  4
+64 66  3
+66 66  1
+67 66  2
+15 67  4
+65 67  3
+67 67  1
+68 67  2
+15 68  4
+66 68  3
+68 68  1
+69 68  2
+15 69  4
+67 69  3
+69 69  1
+70 69  2
+15 70  4
+68 70  3
+70 70  1
+71 70  2
+15 71  4
+69 71  3
+71 71  1
+72 71  2
+15 72  4
+70 72  3
+72 72  1
+73 72  2
+15 73  4
+71 73  3
+73 73  1
+74 73  2
+15 74  4
+72 74  3
+74 74  1
+75 74  2
+15 75  4
+73 75  3
+75 75  1
+76 75  2
+15 76  4
+74 76  3
+76 76  1
+77 76  2
+15 77  4
+75 77  3
+77 77  1
+78 77  2
+15 78  4
+76 78  3
+78 78  1
+79 78  2
+15 79  4
+77 79  3
+79 79  1
+80 79  2
+15 80  4
+78 80  3
+80 80  1
+81 80  2
+15 81  4
+79 81  3
+81 81  1
+82 81  2
+15 82  4
+80 82  3
+82 82  1
+83 82  2
+15 83  4
+81 83  3
+83 83  1
+84 83  2
+15 84  4
+82 84  3
+84 84  1
+85 84  2
+15 85  4
+83 85  3
+85 85  1
+86 85  2
+15 86  4
+84 86  3
+86 86  1
+87 86  2
+15 87  4
+85 87  3
+87 87  1
+88 87  2
+15 88  4
+86 88  3
+88 88  1
+89 88  2
+15 89  4
+87 89  3
+89 89  1
+90 89  2
+15 90  4
+88 90  3
+90 90  1
+91 90  2
+15 91  4
+89 91  3
+91 91  1
+92 91  2
+15 92  4
+90 92  3
+92 92  1
+93 92  2
+15 93  4
+91 93  3
+93 93  1
+94 93  2
+15 94  4
+92 94  3
+94 94  1
+95 94  2
+15 95  4
+93 95  3
+95 95  1
+96 95  2
+15 96  4
+94 96  3
+96 96  1
+97 96  2
+15 97  4
+95 97  3
+97 97  1
+98 97  2
+15 98  4
+96 98  3
+98 98  1
+99 98  2
+15 99  4
+97 99  3
+99 99  1
+100 99  2
+15 100  4
+98 100  3
+100 100  1
diff --git a/matrices/test/a_large_inv.mtx b/matrices/test/a_large_inv.mtx
index 2ffdaea4944..ffb33d4f9bd 100644
--- a/matrices/test/a_large_inv.mtx
+++ b/matrices/test/a_large_inv.mtx
@@ -1,10002 +1,416 @@
-%%MatrixMarket matrix array real general
-100 100
- 1
--0.4948484826962766
- 0
- 1.296448009330129
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--4.392038891786191
- 0
--0.4777284225073569
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.2716119556444772
- 0
- 0
- 0
- 30.85062637605465
- 0
- 0
- 0
- 1.65101796609703
- 0
- 0
- 0
- 2.135572018880382
- 0
--0.6574524552277107
- 0
- 8.678569356519546
--1.244734604837189
--0.09718816849192378
- 0
- 0
- 0
- 1.412909069015802
- 0.3824057381749257
- 0
- 0
- 0.2031984273747871
- 0
- 0
- 0
--0.3270471083429252
- 0
- 0
- 0
- 1.947282594671632
- 0
- 0
- 0
- 0
--0.8410213348839369
- 0
--7.615637091260189
- 0.3877739650231421
- 0
- 1.929363012934077
- 6.877497641159541
- 0.2261957338256072
--0.04340579661659678
--0.4423782345077397
--1.32753749229893
- 0
--1.154171675452101
- 0.1534844417644058
- 0
- 0
- 42.76136386500359
--26.84790352949593
- 0
--1.597985438747375
- 0
- 0
- 0
--0.5028987458742963
- 0
- 0
- 0
- 0
- 0
--0.2641224237401552
- 0
- 0.2273263532487166
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--2.647905742240299
- 0
- 0
- 0
- 0
--10.08295973932505
- 0
- 0.02927473708957342
--0.1170888798804651
- 0
- 0.3094913797252903
- 0
- 0
- 0
- 0
- 0.6843029897075475
--0.5979120488850802
- 0
--3.930579890760858
- 0
- 0
--0.423912919727431
- 0
- 0
- 0.04235351624994581
- 0
--57.52832090042818
- 0.4605407742850633
- 0
- 0.3215523795124502
- 0
--0.7708840587272257
--0.6803459748153848
- 0.7329247708861701
--0.3270807982536441
--0.5647553557286519
- 0.7909079631328433
--0.1197192169001813
- 2.773195902836251
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5247090966189472
- 0
- 0
- 0
--0.0250845239064978
- 0
- 0
--1.882603388163359
- 0
- 0
- 0
- 0
- 0
- 0.5652794611179659
--0.3441786394575287
- 0
- 0
--18.63425959211918
- 0
- 0
- 0
- 0
- 0
- 0.2253955976162801
- 0
- 0
- 0
- 0
- 0
- 0
--1.405934967755039
--21.95019197849467
- 14.79062519524239
- 0
- 0
- 1.200830472811673
- 0
- 0
- 0
--0.1906769198228738
- 0
- 0
- 0
--0.5643508305294765
- 0
- 0
--0.1412489694348874
- 1.932208055387784
- 0
- 0
- 1.303569311480962
- 0
--0.6442238459405463
- 0
- 1.492049865865577
- 0
- 0
--2.581107111180139
- 0
- 0
- 1.051959442697243
- 0
- 0.4041162692018464
- 0
- 0
- 0
--0.5006050847692073
- 0
- 0
- 0
- 0
- 0.6897551534076837
- 0
- 0
- 0
- 0
- 4.540132606636557
- 0
- 0
- 0
- 0
- 0
- 0.9915679283395605
--4.182277033993933
- 0
- 0
--1.673207551170892
- 0
- 1.567325040936841
- 0
- 0
- 0
- 0
- 0
--1.517490314266554
- 0
- 0
- 0
- 8.03005129786718
--0.7776521127117286
- 0.1434829948132886
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.707144867273901
- 0
- 0
- 3.627874724999257
- 0
- 2.903683424227111
- 0
--8.063528603745542
- 0
--0.4287250896794856
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.340974238036757
- 0
--0.8145243484136816
- 0
- 0
--0.4174159434134531
- 1.03986445855464
- 0
- 0
- 0
- 0.36754363051727
- 0
- 0
- 0
- 0
--1.730944174585986
- 0
- 0
- 0
- 0
- 0
- 0
- 0.45371565793248
- 0
- 0.02347849783908041
--0.4260938993976161
--0.5767700646170179
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.8860281611756293
- 0
- 1.095606325216979
- 0
- 0
- 0
- 0.220166619770703
--0.07542487524145412
- 0.3948087161896018
- 0
--1.619179205712842
- 0
- 0.3286476679514239
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.5543790209854219
- 0
- 0
- 0
- 0
--0.7396935249967006
--0.00354028811544854
- 0.4918595753930014
- 0
- 0
- 0.3660310995604132
--19.23194275997228
- 15.82680653523076
- 0
- 0.4359570591948023
--6.098988867740374
- 0
--0.864668198143852
- 1.026297632537591
- 0
- 0
--0.5063974162849202
- 0
--0.9795577741375692
--1.013294620712399
- 0
- 0
- 0
- 0
- 0.1951293129872262
- 0
- 0
- 0
--3.016694233463486
- 0
- 0
- 2.379472440700222
- 1.117893531154898
- 0
- 0
- 0
- 0
- 0.1837803056569752
--0.3894980338215976
- 0.2069584579918223
- 0.6281900908829987
- 0.1235166319345552
- 0
- 0
- 0
- 0
- 0
- 0.6505454939560723
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--7.242189493265922
- 0
- 0
- 0
- 0
- 0
- 0
--0.4094411779876041
- 2.009464200151736
- 0
- 0.001819232267203992
--0.6722293257606738
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.050081901839461
- 0
--2.412064627243687
- 1.03350463502369
- 0
- 0
- 0.09259686676252599
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 5.200930375149771
- 0
- 0
--4.206547801675418
--0.9599040568340262
- 0
- 0.4770814643127358
- 0
- 0
- 0.01879090725614984
- 0
- 0
- 0
--0.05538293578006107
- 0
--0.3820411879721848
- 0
- 0
--7.242164375844034
- 0
- 0
- 0
- 0
- 0
--0.1063927576234104
- 0
--0.6532878735852709
- 0
- 0
- 0.1727225659870178
- 0
- 0
--0.3456510655767607
- 0.508453460284442
- 0
- 0
- 0
- 0
- 0
- 3.478180969756633
- 0.3458732325164959
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.4482468265599172
- 0
- 0
- 0
- 0.7182405125550337
- 0.6780568842099182
- 0
- 0
- 0
--0.1213867687830573
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--2.256340174664548
- 0
--1.850687897344088
--0.4348365590933105
--1.528621907997535
- 0
--0.3100312974456319
- 0
- 0.8330264350366573
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.978876415637884
- 0
- 0
- 0
- 0.2712240954623512
- 0
- 70.6967998182616
- 0
- 0
- 0
- 0
- 0.3869011085429613
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5675110897172608
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.02052967032341324
- 0
- 0
- 0
- 0
--3.152871910382966
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.467308411651212
- 0
- 0
- 0
- 2.101269310386782
- 0
- 0
--2.133477085544608
- 0
- 1.598779636234988
- 0
- 0
- 0
- 0
- 0
--0.1261932360304374
- 0
- 0.08323682421254082
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.259328499242096
- 0
- 0.8991528855085426
- 0
- 0
--0.4558697758587361
--0.5888695098119464
- 0
- 0
- 0
--3.941184768668226
- 0
- 2.346510335375143
- 0
- 0
- 0
- 0
- 0
- 2.644633728916071
- 0
- 0
- 0
- 0
- 0
- 0
- 2.319588045455526
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.5947318450327681
- 0
- 0
--0.5641262393301952
--49.01905511013528
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.019067849743785
- 0
- 0
--25.87429713932424
- 0
- 0
--4.733526222719125
- 0
- 0
- 0
- 0
- 0.1129166470711533
- 0
- 0
- 0
--0.486174107161233
- 0
- 0
- 0
- 0
- 0
- 0.2949077000010569
- 0
--0.1501207485736518
- 3.849371709284814
--0.8341451729489401
- 0
- 3.893782413817492
- 0
- 0
- 0
- 6.792977813286039
- 0.957872567205309
--1.003057656973179
--0.09274640941984065
- 0.1023420489263638
- 0
- 0.1269699646354275
- 2.226011329308089
- 0
- 0
- 0
- 3.416015959932167
- 0
- 0.8943163457222969
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.6126989280244138
--0.3246516947542515
- 0
- 0
- 0.2754250040898726
- 0
- 0
- 0
--0.4854122586429255
- 0
- 0
- 0
- 0
--0.0107142540790587
- 0.8737200000000001
- 0
- 0
- 0
- 0.3096091746462638
- 0
- 0
- 0
--0.4151811537546787
- 0
- 0
- 0
--0.01298821617283267
--400.0484843923324
- 0
- 0
- 0.02756057559993267
- 0
- 0
- 0
- 0
- 0
- 22.33316635771502
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.6176388782446097
- 0
- 0
- 0
- 0
--0.9326696330902033
- 0
- 21.48472691506654
- 0
- 0
- 1.360929014254493
- 0
- 0
- 0
- 0
--2.739198557525646
- 0
- 0
--0.6394060477436144
- 0.06064892640321599
- 0
- 0
- 0
- 0
--1.5375885980933
- 0
- 0
- 0
--2.530034642065866
- 0
- 0
--2.355718205752761
- 0
- 0
- 0
--8.457960636943049
- 0
--0.5334555035837583
- 0.5950392780592704
--0.05802908947544636
- 0
- 0.6565447973856923
--4.817067231874906
- 0
- 0
- 0
- 0
- 0
- 0.7777063759657887
- 1.608808545691914
- 0
- 0.3641645896625149
- 1.142688497470754
- 0
- 0
- 0
- 0
- 0
--0.6935767511801533
- 0
- 0
- 0
- 0
- 0
- 0.7893690091084308
- 0
- 0
- 0
- 0
--0.4838670856424674
- 0
- 1.430208426431188
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1091998636600013
- 0
- 0
- 0
- 0
- 173.0159784009199
- 0
- 3.087783837032923
- 0
- 0
--0.5207456317995715
- 0
- 0.3663560963011827
- 0
- 0
- 0
- 0
- 1.810199877551485
- 1.161339867225544
- 0
- 0
--0.09183729758336448
- 0.08277385931196803
- 0
- 0.1647219817208061
- 0
- 0
- 0
- 0
- 0
- 0
- 1.469826937942972
- 0
- 0
- 0
--0.1370330482960102
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1115021919180884
--0.9328718935886083
- 0
- 0
- 0
- 0
- 0
- 0.1748324697248587
- 0.03063409864837402
- 0
--4.337358926109053
- 0
- 0
- 19.01044390856138
- 0
- 0
- 0
- 0.2130732686517817
- 0.1060226766717321
- 0
--0.1395624958206959
- 0
- 0
--0.2528936532027943
--0.07134587355046787
- 0
- 2.357417556612328
- 0
- 0
- 0.9075786106248219
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.007320220569743884
- 3.869578589517917
- 0
- 0
- 0
- 0
- 0
- 0.3686263703879875
- 0
- 0
- 0
- 0
--0.2136046797528302
- 0.6314622736417405
- 3.36378582607192
- 0
- 0
- 0.05303212482830985
- 0.6835556807052111
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.1889168859470041
- 0
- 0
- 0
- 0
- 0.1304055813943983
- 0
- 0
- 0
- 0
--0.4606790875902882
--5.165448856705915
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--7.955877087780217
- 0.8864487651172198
- 0
- 13.06568060533946
- 0
- 0
--0.3643418680580818
- 0
- 1.395506666819056
- 0
- 0
- 0
- 0
- 0
- 0.07538296328271762
- 0
- 1.650302015917521
- 0
- 0
- 0
- 0
- 0
- 0
- 1.046842999315985
- 0
- 0
- 0
- 0
- 9.532433213272679
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.370851982588047
--3.05343392917343
- 0
--0.8727223392322785
- 0
--0.4777306179856324
- 0
- 0.1580887059847316
- 0
- 0
--166.0021304612256
- 0
- 0
- 0.7460368757827937
- 0
- 0
--3.614862978247329
- 0
- 0
- 0
- 1.029643528805812
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.147807401687179
- 0
- 0
- 0
- 0
- 0
- 0.4298236228686947
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.298674552549088
- 0
- 0
- 0
--0.7980067591331992
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.6197771008673595
- 0
- 0
- 0
--1.21147243167711
- 0
--0.2605982511793086
- 0
- 1.051060954927249
- 0.9065964908196933
- 0
--1.407909251466448
- 0
--0.5657270027934864
- 0
- 0
--10.25518588630423
- 0
- 0
- 0.4265223222861988
--0.2754317183010266
- 0
- 0
- 0
- 0.3395498820441549
- 0
- 0.02958761136812027
- 0
- 0
- 1.344603747980827
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.5601827267188916
- 0
--5.383513440182356
- 0.8733396775619311
- 0
- 0
- 0
- 0.4317085672447314
- 0
- 0
- 0.6453818134063676
- 0
- 0
- 0
- 0
- 0.5200073830189876
- 0
- 0
- 0
- 0
- 0
- 0
--0.733086422217669
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.4432295124436218
- 0
- 0
- 0
- 0.8648766690509214
- 0
- 0
- 0
- 0
- 0.2540722356705414
- 3.47068006348652
- 0
- 0
- 0.3316352829003929
- 0.1298033646181902
- 0
- 0
- 0
- 0
- 0
--0.6559784090830778
- 0
- 0
- 0
--2.461317725460241
--0.2634313360096167
- 0
--0.9057946753403153
- 0
- 0
- 0
- 0
- 0
- 3.869235242037691
- 0
- 0.7681260485630639
- 0
- 0
- 0
- 0
- 0.2297889799585331
- 0
- 1.039605046317225
- 0.1294372838807519
- 0
- 0
- 0
--2.27514102465816
- 0
- 0
- 0
- 2.049062068073663
- 0.1516796642115219
--3.916866957852078
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5722260646438212
- 0
- 0
- 0
- 0
- 0
- 0
--4.710243718168056
--0.4870266055543358
- 0
--0.3880125804125346
- 0
- 0
- 0
- 0
- 0
- 0
- 0.8363423534198683
- 0
- 3.056976067612954
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2441738162347041
- 0.0357834428473006
- 1800.314538517002
- 0
- 0
- 0
- 0
--0.3261478742741906
- 0
- 0
- 0
- 0
- 0
- 0
--3.574838458869212
- 0.388967096014158
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5120150047030911
--0.4429076713605919
- 6.567417800467082
- 0
- 0
- 1.033403610626028
- 0
- 0
--0.1201168089399312
- 0
- 0
- 1.167161858379611
- 0
- 0
--0.6958161223076397
- 0
- 0
- 0
- 0
- 0
- 0
- 0.4509149317811544
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.4792757943874765
- 0
- 0
- 0
- 0.4916110309394393
- 0.1866444401082296
- 0
--0.6845593898010638
--0.9052032264783701
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 4.895441817161111
--0.7209198947190997
- 4.054781566780847
- 0
- 0
- 0
- 0
- 2.528980634318294
- 0
- 0
--0.5629867656968802
--0.869959
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.4006144142924327
--58.8212544358888
--0.5573049702096061
- 0
- 0
- 2.268010772363684
- 0
- 0
--4.23775299517115
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.7038484816810545
- 12.42976177283253
- 0
- 0.2598511143545055
--4.706375917378906
- 0
--1.069792924031905
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5284818446523596
--0.2317575652818614
- 0.6049576045814707
- 0
- 0.0936934825598751
- 0
- 0.6686083357134646
- 0
- 0
- 1.14143632433362
- 0.3503199830054031
- 12.02281668097733
- 0
- 0
- 0
- 0
- 0
- 0
- 0.4204897221502805
- 0
- 0
- 0
- 0.002161286711168726
- 0
- 0
--1.823200927539675
- 0.8488044341158772
- 0.1650376508635884
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.1493032180801644
- 0
- 0.3915997924850128
- 0
--0.3376551130159838
- 2.451876485774099
- 0
--1.023854922625665
--0.4969114947662052
- 0
- 0
- 0
- 0
--1.311156244368104
- 0
- 0
- 0
- 0
- 0
- 1.110968516904319
- 0
- 0
- 0
- 0.5397651888879642
- 0
- 0
- 0
- 0
--2.333073839187608
- 0
- 0
--0.07552426206784429
- 0
- 0
- 0
- 0
- 0
- 1.631147964603093
- 0
- 0
- 0
--0.04125382538654197
- 0
- 2.298055590191642
- 0
--4.54636147842368
- 0
- 0.7098816805381168
- 15.93716172348917
- 0
- 0.9963795629337603
- 0.1127232320023574
- 0
- 1.132306670131363
- 0
- 0
--0.2663574698292075
- 0
- 0
- 0
- 0.5157919358723543
- 0
- 0
- 0
- 0
- 0
- 0
- 0.08604829012617518
- 0
- 0
- 0
- 0
- 0
- 1.959178936605638
- 0
--0.2193510087756213
- 0.3442300108980625
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.021071125534755
- 0
- 0.1616977240316401
- 0
- 0
- 0
- 0
--0.0171414009818532
- 0
- 0
- 0
- 0
- 0
- 1.172187603070537
- 0
- 0
- 0
- 0
- 0.2850175388286725
- 0
- 0.9533132192993321
- 0
- 0.1099414890806674
--0.3960661468695982
- 0
- 0
- 0
- 0
- 0
- 0.124839982277769
- 0
- 0
- 0
- 0
- 0
- 0
- 1.192359374479515
- 0
- 0
- 0
- 0
- 0.2995921695389488
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.6321210847120607
- 0
- 0
- 0.03207155215416499
- 0.09953458596614896
- 0
--0.1534286874362401
- 0
--19.06913382158813
- 12.669590025082
- 0
- 0
--0.7052964622086155
- 0
- 0
- 2.120555140213639
- 0
- 0.1013005251328395
- 0
- 0
--0.392197925481916
- 0
- 0
- 0.1343005612800253
- 0
--0.469051036244502
- 0
- 0
- 0
- 0
- 0
--0.5502879597543631
- 0
- 0
- 0
--0.3295471965149562
- 0.1238485530478013
- 0
- 0.9944594786317867
- 0
- 0
- 0.4348181220265078
- 0.5403324023640561
- 0
- 0
- 0
- 0
- 0.2799417821159156
--0.4267116644470155
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.4639162264180705
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.2968261984942903
- 0.5335368053088048
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.2482748823190064
- 0
- 0
- 0
- 0
--3.97041981155859
--0.1022738659823332
- 0
- 0
--0.788896168670442
- 0.1639749493614056
- 0
- 0
- 0
- 0
--0.5768520561570339
--4.520986271159193
- 0
--0.2453367041880312
- 1.441044276131385
- 0
- 0.5254545614648408
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0766667666262323
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 4.026465606766232
- 0
- 0
- 0.4197329531154022
- 0
- 0
- 0
- 0.3067694585786604
- 0.8314371431735575
- 0
- 0
- 0
--1.591369706346736
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1259687511468373
- 0
- 0
- 0
--2.53333376357498
--0.8602950463753484
- 1.873620524096339
--0.6718870234947631
- 0
--2.18839000074766
- 0
- 0
- 0
- 0
- 0
- 2.692732290771431
- 0
- 0
- 0
- 0
--3.291065831162801
- 0
- 0
- 0
--0.2329586474529969
- 0
- 0
- 0
- 0
- 0
- 0
- 0.6135631081273585
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5614754230564437
- 3.84506897839564
- 0.1401832707562678
- 0
- 0
- 0
- 0
- 0
--3.972707924920157
- 0
- 0
- 0.6897960995450576
- 0
- 0
--0.1781238882142194
- 0
- 0
- 0
--0.750994549808589
- 0
- 0.09100871905332755
- 0.7426334212224136
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.004562434639328
- 0
- 0
- 0
- 0.3078110601987896
--2.417182399732592
- 0
- 0.08347068258672727
- 0
- 0
- 0
- 0
- 0
- 0
--0.830317302176479
- 0
- 0
- 0
- 0
- 0
- 0.0434567563767247
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5386706195832376
- 0
- 0
- 0
- 0
--0.6353862693723572
- 0
- 1.975136071000745
- 2.857816894329415
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5281194313481445
- 0
- 0
- 0
- 0
- 0
- 0
--1.479904902831114
- 0
- 0
- 0
- 0
- 0
- 0.5122525097374786
- 0
- 0
- 0
- 0
- 0.909151913488855
- 0
- 0
- 0
- 0.2549383504009166
- 0
- 5.962302297903852
- 0
- 0.1150226024425775
- 0
- 0
- 0
--1.01380208086729
- 0
- 0
- 0
- 0
- 10.38949477672419
- 0
- 0
- 0
- 0
- 0
--0.1001156568581912
- 0
- 0
- 0.1232150685288419
- 0
- 0
- 0
- 0.2799933920423779
- 0.4745785889503421
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 4.003537089391028
- 3.15331633496073
- 0.8949266268054354
- 0.1905354497413648
--0.0006176766367530246
- 0
- 0.6998256194512208
- 6.866552066980055
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.3247196818172799
- 0
--0.0755298726932041
- 0
- 0
- 0
- 0.7798741956678945
- 0
- 0
- 0.249729711097544
- 0.5055081102563672
--1.384472230871766
- 0
--0.8024384886334274
- 0
- 0
--1.898073624868862
- 0
- 1.611688473972974
- 0
- 0
- 0
- 0
--0.08014407571295187
- 0.2564102587943313
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.707010158667659
- 0
- 0
--4.147202888281389
- 0
- 12.12437272012778
- 0
- 0
- 0
- 0
- 0
- 0
--0.5048087565002763
- 0
- 0
- 0
--1.195299323854342
- 0
- 0
- 0
--5.717476628165155
- 0
- 0
--13.07512077460665
- 0
- 0
- 0
- 0
- 0.7365736071424029
- 1.213005555043753
- 0
- 0.02262669554090805
- 1.074670827640683
- 0
- 0
- 0
- 0
- 0
- 0
- 0.3632162325690264
- 0
- 0
- 0
--0.8456143335581298
--3.039487674094362
--0.06850905281137523
- 0
- 0
- 0
- 0
--0.09019236364727108
- 0.2803226839540793
- 0
- 0.6176066036883436
--0.6031715112642002
- 0
- 0
- 0
--0.1918418039784893
- 0
--1.02759238737448
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.6898055200332753
- 0
- 0
- 0
--0.002090446308802089
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5036854825681361
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.441462578054818
- 0
- 0
- 0
- 0
- 0
- 0
- 0.04095241481847776
- 0
- 0
--4.023403973420407
- 0
- 0
- 0
--0.0312725987949565
- 0
--0.05170692748138132
- 0
- 0
- 0
--0.2713606554296591
--0.2876361587391115
- 0
- 1.092790819907702
- 0
- 0
- 0.1073041127787464
- 0
- 0
- 0
--1.032859866554303
- 0
- 0
- 0
- 2.396043537578052
- 0.1729037309633073
- 0.2517973552198874
- 0
- 0
- 0
- 0
- 0
--0.1761251945560189
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.3904653621899105
- 0
- 0
--2.285331188687838
- 0
- 0
- 0
- 0
- 0.341166587958904
- 0
- 0
- 0
- 0
--1.317661374659714
--1.888728240243611
- 0
- 0
- 0
--0.1590686564583614
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.045272475604685
- 0.3332897234069875
- 0
- 0.2766839566984409
- 0
- 1.491454116190605
- 0
--0.6229288469666586
- 0
- 0
- 0
- 0
- 0
--0.5349138460144993
- 0
- 0
- 0
- 0
- 0
- 0
- 0.7375191633215031
- 0
- 0
- 1.867545208133053
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.8029776017065059
- 3.276661772557552
- 0
- 0
- 0
- 0.2585107656953937
- 0.2248831206011652
- 0
- 0
- 0
- 0
--0.6230664190614899
- 13.72963719186322
--0.5371409201423661
--0.1359288264417897
- 0
- 0
- 0
--0.3238934751092037
- 0
- 0.1457135237481716
--0.2274651302089386
- 0
- 0
--1.356599501123854
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.580531963550888
- 0
- 1.238621220298611
- 0
- 0
--0.5561997416574285
- 0
- 0
- 0
- 0
--1.197365597246516
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.6447357651011012
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.2122003804682439
--1.365058699690163
- 0
- 0
- 0
- 0
- 0.7625308218974119
- 0
- 0
- 0
--0.1708390348749355
- 0
- 0.7115274676497322
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.642731265592378
- 0.9061106277271693
- 0
- 0.07950280276876927
- 0
- 0
- 0
--0.8161224232435611
- 0
- 0
- 0.0777517547926225
- 0
- 0
- 0
- 0
- 3.190560560494386
- 0
- 0
- 0.4257272849592115
- 0
- 0
- 1.298382971719918
- 0
- 0
--0.06693507794509926
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 6.686058137806026
- 0
- 0
- 0
- 0
- 0
- 0
- 6.534198600584574
- 0.4283040837376961
- 0
- 0
- 0
- 0
- 0.7632253630999627
- 0
--2.597117510450818
- 0
- 0
- 0
- 0
- 0
- 0
--1.44428176956472
- 0
- 0
- 0
--10.17809731785903
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.602868955570228
--1.227778724536436
- 0
- 0
- 0.2025456011660076
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.4068888188936426
- 1.106371853952238
- 0
- 0
- 0
--2.120467525891301
- 0.5141030641987713
- 0
- 4.142019259648222
- 7.273339910131452
- 0
- 0
- 0
- 0
- 0
- 0
--3.136164028709288
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.122811483000477
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.01079784158729
--0.8977837949331151
- 2.38257492145663
- 0
- 0
--2.450953936898679
- 0
--3.804722013238095
- 0
- 0
- 0
--2.677738173995096
--1.114097186608559
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.9119876984302767
- 0
--4.547523983843604
- 0
- 0
- 0
- 0.9856465720838319
- 0
--0.08544754690441386
- 0
- 0
- 0
- 0
--0.7248436876542571
- 0
- 2.083453547864012
- 0
- 0
- 2.05621618028899
- 0
--1.806520415415923
- 0.5240081916689401
--1.597634193064217
- 0
--0.753583892912139
- 0
- 0
- 0
- 0
- 1.568013441543157
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.411676874911288
- 0
- 542.3095441932002
- 0
- 0
- 0
--0.2806667362458613
- 0
--0.4596673336940951
- 0
- 0
--55.91275459166305
- 0
- 0
- 0
- 0.2615308601081318
- 0
- 0.01745095050356674
- 0
--1.951103771473351
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 12.8201633486507
- 0
- 0.06986337401001477
- 0
- 0
- 0
- 0
- 0
- 0.3880142143477058
- 0
- 0
- 0
- 0
- 0
--0.1406845691956707
- 0.5238083351969455
- 0
- 1.250487214783859
- 0
--1.047394944053435
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--2.100392104780831
- 0
- 0
- 0.5306088782791798
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.2237997665467135
- 0
- 0
- 0
--0.7921971919259403
- 0
- 0
- 0
- 0.1077330002485517
- 0
--1.033306352064682
- 0
- 0
--0.6695380972749733
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.8876839636503929
- 0
- 0
- 0
--1.616700855389318
- 0
- 0
- 0
- 0
- 0
- 0
--15.66585963240408
- 0
- 0
- 0
- 0
- 0
- 0
--2.842510394280135
- 0
- 0
- 0
- 0.9789331801497174
- 0
- 0
- 0
- 15.4017244676939
- 0
- 1.146598787637898
- 0
--0.2488544716360347
- 0.335477097389436
- 0
- 0
- 0
- 0
- 0
--0.6383860299826728
--0.4307519605943652
- 0
- 0
- 0
- 1.02332944709298
- 0
- 0
- 0
- 1.808733717244716
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2269330074689746
- 3.644798009441442
- 0
- 0
- 0
- 3.323368896859614
- 8.432947160682593
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 26.0543122138953
- 0
- 0
--0.3370453272049372
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.792979375879805
- 0
- 0.4356985506684224
- 1.141915365443236
- 0
- 0
--8.885452225220575
- 0
- 0.2329663371220614
- 0
- 0
- 0
- 0
- 0.6453172183777433
- 0
--0.1525216471591352
--1.51206935745585
--1409.789566489944
- 0
- 0.3505410578373315
- 0
- 0
- 0.3050177145532988
--0.1350556291701849
- 0
- 0
- 0
- 0
- 0
- 0
- 4.624672898948029
- 0.0794809461888861
- 1.952800830662676
- 0
- 0.6428929470641352
- 0
--0.9071519888527591
--0.4334517725109382
- 0
--1.868546803418864
- 0
--31.28972221674916
- 0
- 0.1617582371559916
--23.34228519031611
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1013378252601311
- 0
- 0
- 0.3791736198855424
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 2.306185018306861
- 0
- 0
- 0
- 2.439969276914827
- 0
- 0
- 0
- 0
--1.017845358926976
- 0
--0.8766257649530759
- 0
- 0
- 0.6198060744371063
- 0
- 0
- 0
- 0
--14.54122390283368
- 0
- 0
--0.05900668080087645
--0.8608481304516736
- 0
- 0
- 0.9584519544372639
- 0
--0.7192688654953781
- 0
- 0
- 0
- 0
- 0.2230597948668599
- 0
- 0
- 0
- 0
- 0.3731040983273882
- 0
- 0
- 0
- 0
--0.151137228704379
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--50.71934548153453
- 0
- 0
- 3.126452264542789
- 1.132158344845541
- 0
--0.8863528929198792
--1.260035660258273
- 0
- 0
--0.202847099264026
- 0
- 0
- 0
--19.33863969788858
- 0
- 0.1232261434340604
--0.2432360087096834
--2.900892525561199
--0.2935257595444203
- 0
- 0
- 0
- 0
- 0.7918980316239734
- 0
--0.03017587666352048
- 0
- 0
- 0.2295632897630081
- 0
--0.539455340779404
--0.04261836612506047
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5057226779971445
--0.4651331674899222
- 0.3562731727024678
- 0.4220906535398433
- 0
- 0
- 0
- 0
- 0
--0.2237781519202437
- 4.044868941894766
- 0
- 1.146134109648113
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1099205728262046
- 0
- 0
- 0
--0.1172635807505568
--5.169188607650955
- 0
- 0
- 0
- 0
- 0
- 0.5290078414157005
- 0
- 0
- 0
- 0
- 0
--0.1694184955169638
- 0
- 0
- 0
- 0.09428222406861927
- 0
- 0
--2.340675973451509
- 0
- 2.003271014808549
- 0
- 0
--0.04067697176415717
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.1945619307040214
- 0
- 0
- 0
- 0
- 0.04424807833576058
- 0
- 0.3873983960135145
- 0
- 0
- 0
- 0
- 0
- 0
- 9.501309606987693
- 0
--1.065701617640357
- 0
- 0
- 0
--0.3761764396197418
- 0
--2.760565873221028
- 0
- 0
--0.06407669225881736
- 0
- 0.6954083896979436
--0.1211157828422568
- 0
- 0
- 1.28056896227939
- 0
- 0
- 0
--2.09853157883936
- 0
- 0
- 14.1865481906395
- 0
- 0
- 0
- 0
--0.2356606444263802
- 2.108683011342688
- 0.6105938534629533
- 0
- 0
- 0
- 0
- 0.7913328821246927
- 0
- 0
- 1.068829806203787
- 0
- 0
- 0.8299173235166551
- 0.01235111812086723
--0.82768062659
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--5.720237772032373
- 0.9379845676334215
- 0
--0.1543293607798029
- 8.116947803426786
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5402320287781748
- 0
- 0
- 0
- 0
- 0
- 0.9462562549611485
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 3.605026999972954
- 3.411480067968635
- 0
- 1.06255084451497
--7.515831098707253
- 0
- 0.05142596470185394
- 0
- 0.06005195016876662
- 0
- 0
- 0
--33.83369201179862
- 0
- 0.8047221152599965
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.34918003604524
- 1.230615596733249
- 0
- 0.10979142632346
- 0
- 0.8822449650583322
- 0.6958841073665485
- 0
- 0
--0.479429441083319
- 0
- 0
- 0
- 2.915848331189997
--0.997872836697216
- 0
- 0
- 3.263639045218301
- 0
--0.01478244422614588
--0.284994270492743
--0.6130636084263046
- 0
--0.6018543320135842
- 0
- 0
--0.588598019619618
- 0.05684034822488467
--0.03336069582699805
- 0
- 0
- 0
--0.07024435978230432
- 0
- 0
- 0
- 0
--49.35641065012955
- 0
- 0
- 0
- 0
--0.7054576595914014
- 1.617963110749013
--2.576357596878969
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 3.653806778054516
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.4162968482907972
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.4814608464156824
- 0
- 0.04550265498495074
- 0
- 0
- 0
- 0
- 0
--3.085196207941759
- 0
- 0
- 0
- 0
--0.09064188528540848
--0.7649917791956307
- 0
- 0
- 0
- 0.279797907572765
--9.348341835984616
- 0
- 0
--0.3281683959664261
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1328475559642141
- 0
- 0
- 0
- 0
- 0
--1.949932391098605
- 0
- 0
- 0
- 0.1374794555695837
- 0
- 0
- 0
--3.091451623378846
- 0
--0.07764732804850226
- 0
- 0
--2.434024165245729
- 0.5500645539322153
- 0
- 0
- 0
--0.5079753852948696
- 0
--0.5529859236827438
- 0
- 0
- 0
- 0.2015881697347675
- 0
- 0.2579673115713021
- 0
- 0
- 0
- 0.2600781412163046
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.8522602730893853
- 0.2733801271893656
- 0
- 0
- 0.6998551871651082
- 0
- 0
--1.11552369486787
--0.5084088295849895
- 0
- 0
- 0
- 0
- 0
- 0.3404993295374376
- 0
--0.5911308977441611
- 0
- 0
- 0
--0.6417675201399902
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.9005153291632695
- 0
- 0
--1.562914129206301
- 0
- 1.141381319782258
- 3.700306510854067
- 0
- 0
- 0.4915320792043331
- 0
--0.6203587452825666
--0.2413972162031621
- 0
--15.0488841819318
- 0
- 0
- 0
- 0
- 0
- 0.9224970963993336
- 0
- 0
- 0.03554728629349915
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 2.7953185796338
- 0
--0.1305284316301769
- 0
- 0
--0.3507941553620811
- 0
- 0
- 0
- 0
- 0
- 0
- 1.700791840747238
--0.4547107736140834
- 0
- 0
- 0
- 0.2844474796749299
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2577043505012904
- 0
- 0
--37.9205099652173
- 0
- 0
- 0
- 0
- 1.643176969120298
- 0
- 0
--3.442089160337546
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.3685738200902941
- 0
--1.191014929440095
- 0
- 34.17961336962895
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 65.78378332041297
- 0
- 0
- 0
--1.209185515163843
- 0
--0.1564896293723537
- 0
- 0.5388282285966564
- 0
- 1.314037662228941
--1.7449328203312
- 0.1496937119721336
- 0
- 0
- 0
- 0
--0.2451416154803616
- 0
- 0
--1.469872392683623
- 0
- 0
- 0
- 3.151159804730952
- 0
- 0
- 0
- 0
- 0.05959525184340657
- 0
- 1.117510162860467
--0.1044855073737362
- 0.0122456496221879
--2.607625983237394
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--10.34941174115949
- 0.8967368490486325
- 0
- 0
--0.1929814802386746
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.1009095097079969
- 0
- 0
- 0
- 0
- 0
--0.274255218503406
- 0
- 0
--1.604126413868497
- 0
- 0
- 0
- 0
- 5.100859334768233
- 0
--1.013938327312231
- 0
- 8.692407638247211
- 3.428578974872518
- 0
- 0
- 0
- 0
- 3.228719022055706
- 0
--0.3617959480624034
- 0
- 0
--0.5215164558164206
- 150.1666332247709
- 0
- 0
- 0
- 0.03031841122373824
- 0
- 0
- 0
- 0
- 37.73823907986427
- 0
- 0
- 0
--2.652614939130339
- 0
- 0
- 0
- 0
- 0
- 0.04166819568493204
- 0
- 0.6341729050087129
- 0
- 0
- 0
- 0
- 0
- 12.91755519939191
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.05197119971391739
--2.009655228073888
--0.2684851004861274
- 0.9814531190789848
- 0
- 0
- 0
- 0
- 0
- 0
- 6.165396362398449
- 0
- 0
- 0
- 0
- 0
- 0.6579664638625362
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.609027750727198
- 0
--0.2530273342511696
- 0
- 1
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 2.245974095225942
--2.290949007689282
- 0
- 0.2973402786019308
- 0
- 4.932880860261193
- 0
- 0
- 0
- 0.2928714732373925
- 0
- 0
- 0
- 0
- 0
--0.4675944151435886
- 0
- 602.89717581152
- 0
- 0
- 0
--0.7281191587822087
- 0
- 0
- 0
- 0
- 5.153170124854154
- 0
--0.5123852401406641
- 0
- 0
- 0
--0.1687009411677568
- 9.537578806713723
- 0
- 0
- 0
- 0
- 0
- 0
--10.33485045262229
- 0
- 0
- 0.01934872541733585
--25.76730925047929
- 0
- 0
- 1.361219151604215
- 0
- 0
--1.116170521769804
- 0
- 1.800949273495845
- 0
- 0
--0.0421006732079543
- 0
- 0
- 0
- 0
- 0
- 0
--0.1803091143140475
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.3253613664587537
--0.009215314014496436
- 0
- 0
- 0
- 0
--0.5081353632205899
- 0.7435051959204299
--0.1608909576166679
- 0
- 0
- 0
--0.09973737425051146
--0.4127920551071492
- 0
- 0
- 0
--0.5187614628318575
- 0
- 0.08412796197652496
- 0
- 0.02373324620914603
- 0
--2.065695977833619
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.746843346031052
- 0
- 0
- 0.07234842936872278
- 0
- 0
- 0
- 12.98711339268042
- 0
- 0
--0.8327248767664689
- 0
- 0
- 0
--1.304034691999358
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.3158695360166293
- 0
- 0
--0.3064826996869727
- 0
- 0
- 0
- 0.3264531986312823
- 0
- 0
- 0
- 0
- 0
--0.948388578906028
- 0
- 0
--2.31353448980116
- 0
- 0
- 0
--8.389335067261657
- 0
--0.9057254001297296
- 0
- 0
- 0
--0.3537752724027108
- 0
- 0.05041777327414723
- 0
- 0
- 6.584194758138024
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5301036759676395
- 0
- 0
--0.1796234893658151
- 0
--82.07160959215015
- 0.9269006307298486
- 0
--0.2749138667511867
- 0
- 0.3305303060241508
- 0
- 0.9472033389611979
- 0
- 0.0004586254447066839
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.876684153816676
- 0
- 0
- 0
- 0
- 0
- 0
- 0.7082223740804381
- 219.7111859928096
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.232904486380742
- 0
- 3.955415075927579
--0.5904487530897623
- 0
- 0
--6.987361287626547
- 0
- 0
- 0
--0.4174540658481559
- 0.4890908934687008
- 0
- 0
- 0
- 0
--3.056270919776234
- 0
- 0
- 0
--0.01669853102136176
- 0
- 0
- 0.6034720457213459
- 0
- 0.9237654194086627
- 0.0357838423722235
- 0
--0.01058585067925166
--0.39037337588858
- 0.6964320196044104
--0.1288030817416514
- 0
- 0
- 0
- 0
- 0.5539395094188051
- 0
- 0
- 0
- 0.4162228479499686
--8.715412755845247
- 4.655536481113301
- 0
- 0.008871884562407466
- 0
- 0
- 0
--0.01441495808858097
- 0
- 0
- 0.1106769325196798
--0.03422990431458062
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0963709151375082
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.6403290165668492
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.06534584460111437
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.242470294670795
- 0
- 0
- 0.4513265813109331
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 6.26455324966978
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--5.788749463935768
- 0
- 0
- 0
- 0
- 0.2479831626428473
- 0
--0.1579424199805311
--0.4050262624552801
- 0
- 0
- 0.1615823852871679
- 0
- 0
- 0.05708742947241674
- 0
- 2.063244692768679
- 0
- 0
- 0
- 1.787123785044046
- 3.480412896111618
--1.090884040621945
- 0
- 0
- 0
- 0
- 0
--0.6307949013629948
- 0.7085196209530149
- 0
- 0.06958179356518558
- 0.5663128212347056
- 0
- 0
- 0
- 0
- 0
- 0.6991291777479244
- 5.119831962698358
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1285654986482149
- 0
- 0
- 0
- 0
- 0.5365078826409398
- 0
--0.3120053728133674
--4.668198793561861
- 0
- 0
- 0.009461634588692909
- 0
- 1.368372431442259
- 0
- 0
- 0
- 0
- 5.390334257178868
--5.166687097847665
- 1.11785039509098
- 0
- 1.083071602025184
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.735155644270017
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2936589374981971
- 0
- 0
- 0
--8.426052593557538
- 0
- 0
- 0
- 0
--0.5195776095653704
- 0
- 0
--5.148133256963701
--1.330135598232709
- 0
--0.3089028100362089
- 0
- 0
- 0
- 0
--1.104083242407448
--0.04490339340203948
- 0
- 0
- 0
- 0
- 0
- 0
--5.294652116349429
- 1.679224457856224
- 0
--3.290894143570063
- 0
- 0
- 0
- 0
- 0
- 0
- 0.3890749762819402
- 0
- 0
- 0
- 0.56924912365979
- 0
- 0
- 0
- 0
--0.8030827361913189
- 0
- 0
- 0
- 0
- 30.86277274861568
- 0
--0.3398319673253343
- 0
--1.082868037605125
- 0.5136061678322544
- 0.1554647180658579
- 0.3655265200143383
--1.716250939567565
- 0
- 0.8634319462190599
--2.910130310413088
- 0
- 0
--0.1588162077829562
- 0
- 0
--0.1664590718275648
--0.9777961710251044
--1.372045042252581
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.480309092581737
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--3.058032400395982
- 0
- 0.412187606704143
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 2.431230191709969
- 0
- 0
- 0
- 0
- 0.1962122215985665
- 0
- 0
- 0
- 0
- 0
- 0
- 2.381168023501953
- 0.345316073883669
- 0.1478247275791482
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5428631983297003
--3.985532383422589
- 0
- 0.2761110408253554
- 0
- 0
- 0
- 1.26683255652973
- 0
- 1.970861260534998
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 54.94854029156146
- 0
- 0
- 0
- 0.1094666893077373
- 0.5977085566998216
- 0
- 1.147521819902617
- 1.102300705123151
--0.5638834550146347
- 0
- 0
- 0
- 0
- 0.5224691566975731
- 0
- 0
- 0
- 0
- 0
- 0.9170826350335268
--1.192903940179488
- 0
- 0
- 0
- 0
--0.72405612721989
- 0
- 0
- 0
--0.4943184072537858
- 0
--1.645398441249078
- 0
--0.01183162605145718
- 0
- 0
- 0
- 0
- 0
--2.445782008272828
- 0
- 0
- 0
- 0
- 0
- 0
- 1.371977203329604
--0.6237686085114352
- 0
- 0
- 0
- 4.896294638225456
- 0
- 0
--10.50101090239492
--0.2186035533671767
- 0.2534126808841761
- 0.5499854069795571
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.3877747613890682
- 0
- 0
- 0
- 2.85659521023488
- 0
- 0
- 0.4744709679304063
- 0
- 0
- 0
- 0
- 0
- 0
--0.03521565215270906
- 0
- 0
- 0
- 0
- 0
--0.03761895482880985
- 0.5030174792937585
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.7649682274545092
- 0
- 0
--0.6314170877432317
- 0
- 0
- 1.414638374891361
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.9802015092735455
- 0.6106684731414381
- 0
- 0
- 0
- 0
- 0
- 0
--83.68089381156405
- 0
- 0
--0.5000248436886791
- 0
- 0
- 0
- 0
--0.2917034400370256
- 0
- 0.02833986746370941
- 0
--0.02198728912753643
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.037430618094088
- 0
--0.6271655935409088
- 0
--4.48442327538069
--0.9411955094116317
- 0
--1.352367282837229
- 0
- 0.09489073595255178
- 0
- 0
- 0
--1.071999794444188
- 0
- 0
- 0.5806143921209845
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 10.19197372794769
- 0
--0.2976378075102782
- 0
--0.06131668053255376
- 0
- 0
- 0
- 0
- 0
- 0.01518827655417578
- 0
- 0
- 0.7986434496062542
- 0
- 0
--0.03367291104978264
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.06159434527439328
--4.48056600905538
--0.4783667106655162
- 0
- 0.006856996042966583
- 0
--0.2760056254784384
- 1.187423511285556
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.583636276826058
- 0
--2.824780882837426
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.341150956270354
- 0
- 0
- 0
- 0
--0.305214348732556
- 0
--1.460547106503598
- 0
- 0
- 0.03979194706420293
- 0
- 0
- 0
--24.43565266627769
--8.731850316765762
- 0
- 0.68467785353886
- 0
- 0
- 0
- 0
- 0.6510141304728614
--0.9656320399240599
--0.9428798816256519
- 0
- 0
- 0.7666305900589855
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.9160052618132869
- 0
- 0
--2.574692290497155
- 0
- 0.01734075019931757
- 0.08169627984271563
- 0
- 0
- 0.1816372145101804
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.3318538701646884
- 0
- 0
- 0
- 0
- 0
--0.6682147255182748
- 0
--1.820104118158794
- 0
- 0
--0.443970510083133
--2.826135182952575
- 0
- 0
- 0
- 1.702319917467265
- 0
- 0
- 0
- 0
- 0
- 0
--0.3213122755903199
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.927500521753644
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 5.411455074560193
--0.2342183531798989
- 0
- 0
- 0
- 0
- 0
--0.6318248474477726
- 0
- 0
- 0
- 0
- 0
--0.1769519223283303
- 1.821838312750941
- 5.224111206540555
- 0
--0.01964347025582856
- 0
- 0
- 0.01152253189111746
- 0
- 0
--0.2274561707001618
- 0.852190804066514
- 0
- 0
- 0
- 0.758036792984357
- 0
- 0
- 0
- 0
- 0
- 1.258717191131011
- 0
- 0
- 0.9246362091225441
- 0
- 0
--1.654708278897577
--0.8131999456747966
- 0
- 0
- 0
--1.264601729396739
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.04789157230466286
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--3.117351241839914
- 0
- 0
- 0.0724003112956593
- 0
- 1.18695192246064
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.4020776578494258
- 0
- 0
- 0
- 0.01876909665613024
- 0
--0.3447187017682403
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 24.27177041070852
- 0.04543245448876172
- 0
- 0
--1.642622336416251
--0.1489725394229043
--0.9639190351065976
- 3.932904465769799
--0.3936251935545277
- 0
- 0.3050672664198578
- 0.6657382075375293
- 0.1706300753569768
- 0.6309672243517473
- 0
- 0
- 0.6638182119807017
--0.7678884039489431
- 5.527532594305081
- 0
- 0.06516542548503328
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.1079001471518526
- 0
- 0
- 0
- 0
- 0
- 0
--0.6390072184953827
- 0
- 0
- 0.08230627887095554
- 0
- 0.2524038344166655
- 0
- 0
- 0.5901264848880631
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.592932141628379
- 0
- 0
--0.2002107320659327
- 0
- 0
- 0.49217988619839
- 0
--0.9770889058731814
- 0
--0.3761865859180392
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.3809423449100742
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.1148424525478181
- 0
- 0
--46.09452490846677
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.01712053051836848
--0.1140391791814996
--0.3815727396385838
- 0
--0.2773350156950293
--21.66070748961982
- 0
- 0
- 0.4876125814851835
- 0
- 0
- 0.143318774708148
--2.11160798315818
- 0
- 0.5083569444320168
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.144156151339128
- 0
- 0
- 0.2262120526582969
- 0
- 0
--0.1280217046158115
- 0
- 0
- 0
- 0
- 0
- 2.538586369849767
- 0
- 0
- 0.1714805058385587
- 0
- 0
- 0.06412106785558182
- 0
- 0
- 0
- 0
- 0
- 3.404923219073047
- 0.2047409565802319
--0.4045499586070819
- 0
- 0
- 0
- 0
--0.4556019264659864
--2.949453895052602
- 0
- 1.348940523513324
- 0
--12.26457385106148
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.4620615090745949
- 0
- 0
- 0
- 0
- 0
- 0
--0.6401980174732465
- 0
- 0
- 0
--3.719508623116252
- 0
- 0
- 0
- 0
- 0
- 0
- 1.391054589611359
--1.060120782195125
- 0.124489228101295
- 0
--1.277807772817144
--0.3785078509987684
- 1.615512736160592
- 0
- 2.983426606619536
- 0.2498211090350095
- 0
- 0
- 0
- 0
- 0
- 0
--0.4494010090887177
- 0
- 0
--0.553629785237019
--0.4910801410284669
- 0
- 0.1414210220266133
- 1.327731171059285
- 0
- 0.3299007143983764
- 0
- 0
- 0
- 0
- 0.4610314657365753
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.3605192714167013
- 0.1574398355427645
- 0
- 0.4414981133992893
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.604519617646983
- 0
- 0
- 0
- 0
- 15.71616248102042
- 0
- 0
- 0
- 0
- 0
--0.8570158402800939
- 0
- 0
- 0
--0.2308523535528031
- 0
- 0.9312076455997745
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1142824710512821
--0.3016200134040187
- 0
--1.504735389983186
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.1818099598120365
- 0
- 0
- 0
- 0
- 0
- 0
--1.307131931736971
- 0
- 0
- 0
- 0
- 0
--0.03280698962104209
- 0.1534141657150162
- 0
- 0
--0.355929941480607
- 0
- 2.303379083974311
- 0
- 0
- 0
- 0
--0.05997064613475289
- 0.1434539202411265
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.335357227039381
- 0
- 0.9608986316800635
- 1.454978763436731
- 0
--0.3595861940762782
- 0
- 0.5215739104603703
- 0
- 0
- 0
- 0
- 0
--5.116953504081511
- 0
- 0
--0.542440371743971
- 0
- 0
- 0
- 3.378092631477278
- 0
- 0
- 0
--3.092927332443712
- 0
--1.499543838081713
- 0
- 0
- 0
- 0
- 0
- 0
- 4.244285915322096
- 0
- 1.294317179651072
- 0
- 0
- 0
- 0
--1.920793020928811
- 0
- 0
--0.05796143522638493
- 0
- 0
- 0.7718124813687072
- 0.378543256984148
- 0.9407199345199999
- 0
- 0
- 0
- 0
--0.4630423730520387
- 0
- 0
- 0
--0.2866962385734112
- 0
- 0.2471609267439212
- 0
- 0
- 0
- 1.283099748800791
- 0.2019717296192572
- 0
- 0
- 0.2419837008434579
- 0
- 0
- 0
- 0
- 0
--2.177334970191569
--4.507575454151721
- 0.2955140797052224
- 0
- 0
- 0
- 0
- 1.264358720487449
- 0.6421248144713366
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2436580984798275
- 0.3088979486766624
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.477428805404112
- 0
--0.1844266021370999
- 0
- 0
- 0.3015771534286898
- 0
- 30.36939921604455
- 2.906418889523707
- 0
- 0
- 0
- 0
- 0
--0.03386395734253489
- 1.367062689079665
- 0
--0.1727500633466222
- 0
- 0
- 0.8404688065083183
- 0
- 0.5969833145766199
- 0
--0.3039023428231714
- 0.6690994566672736
- 0
- 0
--33.24776044589527
- 0
--0.6764901176807574
--0.4777261115121123
- 0
- 0
- 2.798605136083226
- 0
- 0
- 0
- 0
- 0
- 4.387563205112637
- 0
--0.06254301978189696
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0565358672666474
- 0
- 0
- 0
- 0
--0.05303174136908768
- 0
- 0
- 0
--1.077420061381049
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 4.544024030800431
- 0
- 0.4828010302581907
- 14.1985954759888
- 0
--0.2484741314133392
- 0
- 0
- 0
- 0
- 0
--0.6266574399135667
- 0
- 0
- 0
--1.221340551550764
- 0
- 0
- 0
- 0
- 0
- 2.663412203768888
- 0
- 0
- 0
- 0
- 0
- 0
- 2.616864407153919
- 1.368445147290512
- 0
- 0
- 0
- 0
- 0
- 3.168933025649172
- 0
- 0
- 0
- 0
--0.1787839750809836
- 0
- 0
- 0
- 0
--0.957708462875143
- 0.0270243089554006
- 0
- 0
- 0
- 0
- 0
--5.149044968509322
- 0
--5.402452150174298
- 0.1983931860798278
- 0
--0.09079878744664462
- 1.100856195238638
- 0
- 0
- 0
- 0
- 1
- 0
--1.757475422789677
- 0
- 0
--1.907909824196584
- 0
--2.56843807568624
- 0
- 0
- 0
- 0
- 0
- 106.3659753096595
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 49.8297037410713
--0.04222112138305432
- 0
- 0
- 0
--0.00334437238175774
--1.322833368577879
- 2.001883137412201
--0.1653185367017596
- 0.07546371529819744
- 0
- 0.751813476182292
- 0
- 0
- 0
- 0
- 0.8273234033979112
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.2206799510374065
- 0
- 0.007428173634858678
- 0.7711147909247463
- 0
- 0
- 0
- 0
--0.262301884897598
- 0
- 0
- 0
- 0
- 0
- 0
- 5.187711830460596
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.3701308921519661
- 0
--0.1492866141614466
- 4.377637052009972
- 0
- 0
- 0
--1.518495570290538
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.175533190552626
- 0
- 10.7973331717612
- 0
- 0.2016212642457929
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--7.647984958127028
- 0
- 0
- 0
- 0
- 0
- 0
--0.4235250360581704
- 0
- 1.605262962814493
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.151846068350995
- 10.58198051778713
--0.1202164155190914
- 0
- 0
--0.6630880124698003
- 0
--0.437324480773051
- 0
- 0
- 0
- 0
- 0
- 0
- 0.7188183917221673
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.7294810885553434
- 0
- 0
- 0
- 0
- 0
- 0.5834380696791688
- 0
- 0
--0.622311548892383
- 0
- 0.4679999608322835
- 0
- 6.218088963448483
- 0
- 0.5174412364094663
- 0
- 0
- 0
- 1.190296287934517
- 0
- 0
- 0
--0.944888635275562
- 0
--0.1436819972123027
- 0
- 0.337555768794067
- 0
- 2.980798379797518
- 0
- 0
- 0
- 0
- 0
- 0
- 0.679042
- 0
- 0
- 1.061680232170477
- 0
- 14.84914148054109
- 0
--0.3225700738404698
- 0
- 0
--0.4426196314308238
--1.488239021591834
- 2.044096960925146
- 0
- 0
- 1.683276556969745
- 0
- 0
- 2.17543388273639
- 0
- 0
--0.1933722845744081
- 3.849942822218599
- 0
- 0
- 0
- 0.1790811567979584
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.06976734777921635
- 0
- 0.6407531719287224
- 0
- 32.96630448688174
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.4996389426133815
--0.5177067631142696
- 0
- 0
--0.3169862634712258
- 0
- 0
- 0
- 0
- 0
--12.54141721772795
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.2062309163815745
- 0
- 0
- 1.474014777533001
- 1.636324138104011
--0.6774017509836604
- 0
- 0
--0.1396950781612241
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.655890226504994
- 0
--0.09105491107406645
- 0
- 0
- 0
- 0
- 0
- 0
- 0.3772795213330301
- 0
- 0
- 0.7129656141037942
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.1995738237571416
- 0
- 0
- 0.4881741898924112
--3.635485830889141
- 0
- 0
--1.219804931750709
- 0
- 1.745960394621586
- 0
--0.6708065717577664
- 0
- 0
- 0
- 0
- 1.080357775750419
- 0.8831513213634841
- 1.429880283468479
- 0
--4.076266171684704
- 0
- 0
- 0
- 0.4797442279782531
- 0.06307150790587217
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 2.286884079536372
- 0
- 0
- 4.292003067546896
- 0
--0.0688495314067514
--0.533026798842115
- 1.506111833622525
- 0
- 0
- 0
- 2.297532616184889
- 0.2884256723077435
- 0
--6.437370892793756
- 0
--0.8361437906884199
--0.5712534775192925
- 0
--1.402624039953607
- 0
- 0
--0.1359252093563885
- 1.242344184805322
- 0
- 0
- 0
--0.1141738798020996
- 0
- 0
- 0
- 0
- 0
--1.158618660065336
- 0
- 0
--2.089177813925037
- 0
--8.07367477977005
--1.0150323711742
- 0
- 0
- 0
--0.07056341033340689
- 0
- 0
- 0
- 0
- 0
--0.5251837843361902
- 0
--0.6512360211885267
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--26.20235897003792
- 0
--0.4322824983983252
- 0
- 0
--0.7648821736851593
- 5.006095070641486
- 0
- 0
- 0
- 0
- 0
--4.337173836387355
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5268338988565754
--0.009571158761364785
- 0
- 0
- 0
- 0
- 0
- 0
- 0.4653791124973645
- 25.64567120728595
- 0
--0.6496123292638682
- 0
- 0
--0.4555035775337348
- 0.1955280019478218
- 0
- 0
- 42.88987330209684
- 0.03244860746650249
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.576908553841815
- 0
- 0
- 0
- 0
--2.873816010261588
- 0
- 0.6951557638967627
--2.626277292607718
- 0
--1.029532206148845
--2.453966340890945
- 0
- 0.72016515394283
- 0.8694012728502445
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--2.68909022957995
- 0
- 0
- 0
--0.7475340420147444
- 0
- 0
--0.2091073641131919
- 0
--0.4709622115437511
- 0
- 0
- 0
--0.4948605582522377
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.07391519327902506
- 0
- 0
- 0
- 0
- 0
- 4.193747856539138
--0.5501610235721783
- 10.19376200848029
- 0
- 0
- 0
- 0
- 0.3168698940830827
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2406850922411721
- 0
- 0.6376083595888241
--1.365491324533382
- 2.856514672954081
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.053328123822466
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1642622543035108
- 0
--4.680972400503813
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 4.073044168382923
- 0
- 0.1880083009902825
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1207846509117613
--0.03317074813271751
- 0
--0.7055982138911948
- 0
--1.83713974293803
- 0
- 3.037225158734759
- 0
- 0
- 0
- 0
--0.2120732212399692
- 0
- 0
- 0
- 0
- 0.3103350139921735
--5.740764089657944
- 0
- 0
- 0
--2.179136981551416
- 0
--1.305844019981313
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.4317067648539093
- 0
- 0
- 2.265242158370023
- 0
--0.7794917298629519
- 0
--0.2080790047502954
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.404070778515178
--0.3200287270469684
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--4.266969719763726
--3.406553559740435
- 0
- 0.0134461421595354
- 4.397741852193009
- 0
- 0.02270262847050936
- 1.236312816624057
- 0
- 0
- 0
- 0
- 0
--0.8438687874368702
--0.7304010128337621
- 4.213672406176546
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.2418304053586636
- 0
- 0
- 0
- 0
--0.2657663734219002
- 0.5014011568476776
- 0
- 0
- 0
- 0
- 0
--2.013417788718923
- 0
- 0
- 0
- 0
- 0
--0.06507152503028568
- 0
- 0
- 0
- 0.6912662714336949
- 0.1965927372220691
- 0
- 0
- 0
--1.651384514001629
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.214076
- 0
- 0
- 0.5563402540667817
- 0
--0.8867741572195472
- 0
- 0
- 0
--2.656741289449448
- 0.4450668633801069
- 0.385689412005946
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.9436266645675534
- 2.933814531024174
--0.9512901478130462
--1.309898643573616
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.03962719147094711
- 0
- 0
--4.978615272453554
- 0
--0.05027296967367353
- 0
- 0
- 0
- 0.8500691125051955
--12.61368270895732
- 0
- 0
- 1.087858072226194
- 0
- 0
- 0
- 0
- 0.377904365182381
- 0
--0.6686515407945873
- 0
- 0
- 0
- 0
- 0.1862664661629329
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.01466587516723143
- 0.06610711171668869
- 0
- 0
--2.123751921100425
- 0
- 0
- 0
- 0
- 0
- 0.1075345371825239
--1.509641178057292
- 0
- 0
--0.8967074716443297
--0.2793541539661331
--0.7809135848512985
- 0
- 0
- 0
- 0
--0.6139494500559899
- 0
- 0
--0.06244034836215474
- 0.2024876122386019
- 0
- 0.31508191962413
- 0.7363768068547064
- 0.1525291625640584
- 1.066734274264562
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.6218400864802324
- 0
- 0.8920004254831011
- 0
- 0
- 396.1839217634359
- 0
- 0
- 0
- 0
- 0
- 0
--1.199253263921576
- 0
- 0
--0.04047875704426624
- 0
- 0
- 0
- 0
- 0
- 0.2802717458726153
--0.4781398339414029
- 0.2075733873419544
- 0.2120988866516076
- 0.2259316723165299
- 0
- 1.855703386599502
- 0
- 48.45482065331772
- 0
- 0
- 1.925846335556874
--0.3825298684997945
- 0
- 2.734965965930728
- 0
- 0
- 0.1722425982306823
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5452342913157545
--0.1426201569301026
- 0
- 0
- 0
- 0
- 1.777092371423943
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.03376619410157096
--0.3269307626883812
- 0
- 0
- 0
- 0.6768557113363313
- 0
- 0
--0.5302064943646121
- 0
- 0
- 0
- 0
- 0
- 0
--0.1151294185864515
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 4.89587606110833
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1242991952552595
- 0
- 0
--0.8921944704667315
- 0
- 0
--0.2079497209821426
- 0.118424235762753
- 0
- 0
- 0
- 0
- 0.02805149213992744
- 0
- 0
- 0
--1.432378239368848
--2.371776843647885
- 0
- 0.344804503493937
- 3.395089135255176
- 0.9326537240259527
- 0
- 0
- 0.3618483504009797
--0.2204529230128786
- 0
- 0
--5.095600934668537
- 0
--0.1751302818073968
- 24.77215816884107
- 0
- 0.1017087239890091
--3.378619827176517
- 0
- 0.5450043410439027
- 0
- 0
--0.06792314582765882
--0.8227741754688692
--0.973052
- 0
- 0
- 0
- 0
--0.2761773945576845
- 0
- 0
- 0
- 0
- 0
- 1.475796554910678
- 0
- 0
- 0
- 0
- 0.5250068188840999
- 0.8463708086907529
- 0
- 0
- 1.77875381612637
- 0
- 8.964172785359493
--0.5306670663521423
- 0
- 0
--0.4611627973513845
- 0
- 0
- 1.035139221882437
--0.277849018543929
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.08095671647350679
- 4.267559020070598
- 0
- 0
- 0.6157705599681257
--0.3452226986141149
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--2.032109619024405
- 0
--1.799909961530498
- 0
- 1.996614876342541
- 0
- 0
- 1.042071053613529
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 3.493243119263601
- 0
- 0
--0.7519455731142475
- 0
- 0
- 0
- 1.861670472347809
--1.302549556633801
- 0.1164549396633313
- 0
- 0
--1.923512129505668
- 0
- 2.474447041467192
- 0
--1.040751374820962
- 0
- 0
- 0
- 0.9780301992400041
- 3.320521217754515
- 0
- 0
- 2.161081574861941
- 0
- 0
- 0
- 0
--0.06872316014507636
- 0
- 0
--0.1999009360692771
- 0
- 0.1152876944927467
- 0
- 1.677081838440446
- 0
- 0
- 0
- 0
- 0
- 0.6103558037264367
--0.2243037856522242
- 0
- 0
- 0
- 0
- 0
- 0.5731714028165047
- 0
- 0
--2.453389315249321
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 4.937771034501405
- 0
- 0
- 0
- 0
--0.8628481741011198
- 0
- 0
- 0
- 0.6275581044259833
- 0
--0.6103050445663323
- 0
- 0
- 0.9506517744974721
- 0
- 0
- 0
- 0
- 0.07146383477637612
- 0.2033276276424675
- 0
- 0
- 0
- 0
- 0
- 0.3866797756698009
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1258547169606876
- 0
- 0
- 0
- 0
- 0
- 0
- 0.7447751662012354
- 0
- 0.8908618409796408
- 0.4933084106488362
- 0
- 0
--5.039090765171005
- 8.30841404518636
- 0
- 0
- 0
--0.3605358565255673
- 0
--1.554953303792497
- 0
- 0.2431225017316495
- 0.1114327694170067
- 0
- 0
- 0.2176529555530883
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.243761507650353
- 0
--1.138724018110861
- 0
--0.1884231426986675
- 0
- 0
- 0
- 0
- 0
--0.3379309618996783
- 0
- 0.2485915017880706
- 0
- 0
--5.343306933654838
- 0
- 0
- 0.001917725227957101
- 0
- 0
- 0
--0.57272516917422
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.950962399877542
- 0
- 0
- 0
- 0
- 0
--0.3539993266894119
- 0.1863721694763848
- 0
- 0.09202470134014996
- 0
- 0
- 0.3212898449005449
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.3621747592009364
- 0
- 0
--0.1414438098336342
- 157.5397344752773
- 0
--0.09046742395702388
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 3.862837820137139
- 0.1924558131841986
- 0
- 0
- 0
--0.2840026206298199
- 0.1327272769633906
- 0
- 0
- 0
- 0
- 0
- 33.09580110175312
- 0
- 1.106718461462165
- 0
- 0
- 0
- 1.251918154311314
- 0
- 0
- 0
- 0
--0.5332273353948972
- 0
- 0
- 0
--1.455788933358241
- 0
--0.1005810698833533
- 0
- 0
--0.7048634047882905
- 0
- 0
- 0
- 0
- 0
--0.1829592210252697
- 0
- 1.554891259764205
- 0
- 0
--0.5622586756351996
- 0
- 0
- 0
- 0
- 0
- 0
--0.02155014855168353
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.05517346460039634
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.6481684344216544
- 0
- 0
- 0.009127147076078621
- 0
- 0
- 0.8920297763357408
- 0
- 0
- 0
- 0
- 0
- 0
--1.891229456968384
- 0
- 0
- 0
--0.3717262537806227
--0.03650552052887308
- 0
- 1.101498666534914
- 181.1975663423919
- 0
--0.4884495217296688
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 3.574127663089739
- 0
- 0.1860349011542316
- 0
--0.01108354956990108
- 0
- 0
--0.5589239617229802
- 0
- 0
- 0
--15.98727046132908
--6.139599379559069
- 0
- 0
- 0
- 0
- 0
- 0.3564056231718538
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.01873966260724039
- 0
- 9.207082663592216
--1.462609353137826
- 0
- 0
- 0
- 0
--0.3084022201824273
- 0
--0.001494477889351077
- 0
--0.639959639338643
- 0
- 0
- 0
- 0
- 6.680483493497896
- 5.553240032126599
- 0
- 0
- 1.332858851534272
- 0
- 0.1131389989281115
- 0.05052433436715395
- 0
- 0
- 0
- 0
--0.08032570098366452
--4.306342631424013
--0.3311656692546475
- 5.18545434333079
- 0
- 0
- 0
--1.35938512410737
- 0
--12.06264393151567
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.774260054200211
- 0
- 0.6190595835642895
- 0
- 0
- 0
- 0
- 2.281013599138365
- 0
- 0
- 0
--1.272424626221362
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 4.324605713861138
- 0
- 0
--6.345603615790674
- 0.4697904996479123
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2796381362266612
- 0
- 0
--0.06583423808647407
- 0
--0.1708671119388621
- 1.446100117702695
- 0
- 0
--0.709420994721505
- 0.8312792208980893
- 0
- 0
- 0
- 1.150495133643803
- 0
- 0
- 0
- 0
- 0
- 0.4247023662659065
- 0
- 0
- 0
- 0
- 0
- 0
--1.994140389323243
- 0.08451970938897084
- 0.2445618119961789
- 0
- 0
- 0.1861879473977577
- 0.8255679381271304
- 0
- 0
--0.4972849136882072
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1892641816600949
- 0
- 0
- 0
- 0
- 0
--0.4696475831889678
- 0
- 0
- 0
--0.7161722386197744
--0.4773806413906373
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.6855326207390812
- 0
- 0
- 0
- 189.4490562795383
- 0
- 0.2294168823791598
- 0
- 0
- 0
- 0
- 0
- 0
- 15.01173079019572
- 0
- 0
- 0
- 3.023849232116733
- 0
- 0.02400723546902379
- 0
- 0
- 0
- 0
- 0.1428853246752037
- 0
--0.1258080192744591
- 0
- 0
- 0.977392616352955
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.159163815357885
- 0
- 0
- 0
- 1.830819091338398
- 0
- 0
- 0
--1.224993910620706
- 0
- 1.12737388736329
- 0
- 0
- 0
--0.3417665236641663
- 0
- 1.363948077662064
- 0
- 0
--0.3579474117229764
--0.5209770793433826
- 0
- 0
- 0
- 0
- 0
- 0.3483799281354589
- 0
- 0.7390603694655263
- 0
- 0
- 1.79402507837949
- 0
- 0
- 0
--0.506937
--98.38946250084041
- 0
- 1.789555367686995
- 0
--2.901378745215094
- 0
- 0
- 0
- 0
--0.3616770619378998
--0.3024026847600035
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.3260923676104936
- 0
- 0
- 0.03540338342917591
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.3064325656735168
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.080346199490051
- 3.530698360660324
- 0
--0.5223943446877712
- 0
- 1.157378358175996
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--4.268159133035052
- 0
--0.4528358491068142
--4.582341386826024
--0.2679088348068341
- 2.057257979795762
- 3.73870859223568
- 0
- 0
- 0.4044245983857004
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.200286852455682
- 0
- 0
- 0
- 0
- 0
- 2.142147343874559
- 0
- 0.5176775834478452
- 0
- 0
- 0
- 0.3600616894812302
- 0
- 0.2672811945379825
- 0
- 0.1663354456056821
- 0
- 0
- 0
- 0
- 1.322223433879568
- 0
- 0
- 0
- 0
--19.4930336369178
- 0
- 0
- 0
- 0
- 0
- 3.561528539945717
--0.1164984481431163
- 0
- 0.475143755650158
- 0
--5.677337589744861
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1383495046941607
- 0
- 3.72338460736351
- 0
- 0
- 0
- 0
- 1.029503712841592
- 0
- 0
- 0
- 0
- 0
- 0.5526033179262732
--0.2922509856369307
- 0
- 0
- 0
- 0
- 0.02192571142437446
- 0
- 0
--2.275862499445515
- 0
- 0
--5.956539869302246
- 0.5577919710366753
- 0
- 0
- 0.9149456986467562
- 0
- 0
- 0
- 0
- 0
- 0
- 12.85421515349177
- 0
- 0
- 0
- 0
- 0.06855743856753654
--0.891803654255144
- 0
- 0
- 1.216191370381009
- 0
--0.2525162088130185
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.119819991477916
- 0
- 0
- 0
- 0
- 0
- 2.429682620515801
--1.305779481269262
- 0
- 0
- 0
- 0
- 0
--1.273832601871327
- 0
- 0
--0.4722256149873065
- 0.1694239131550563
- 0
- 0
- 0
- 0
- 0
- 0.4909029282327266
- 0.2809566080112832
- 0
- 0
- 0
--0.5049177529917077
- 0
- 0
--0.7131852463338735
--2.280203274456158
- 0
- 0
--0.7076217123728984
- 0
--2.782264996918608
--1.097141367893753
--0.9468800068107297
- 1.108086285509248
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.6030397649317234
- 0
- 0
--0.1146452475178463
- 0
- 0
- 0
- 0
- 0
--0.1962433153909341
- 0.1991815419698557
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 6.049189216433742
- 0
- 0
- 0.09982078399404117
--1.495186107409558
- 0
- 0
- 0
--4.321278976118168
- 0
- 0.4774583279972811
- 0
- 0
- 0
- 0
- 0
- 0
--0.3662508736483057
- 0
- 0
--0.03181285309435417
- 0
- 0.4913051667564982
- 0
--0.1657547275193968
- 0
- 0.4006004973474178
--0.4370423396175648
- 0
- 0
- 0.7448841387304859
- 0
--2.512065448579293
- 0
- 0
- 0
- 0
- 0.1834078614552195
--0.5340080834170147
- 0
- 0.6592265199085
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2906414221067094
- 0
- 0
- 0
- 0
- 0.1585261181629194
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.282585577585656
- 0
--0.587196279763097
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.240278976404221
- 0
- 0.2102593351291137
- 0
- 0
- 0
- 0
- 0
- 0.1132484587356649
- 0
- 0.5660472430497463
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.5320305596335215
- 0
- 0.08950015573930774
--0.36066996993926
--0.9469273217143011
- 0
- 0
--0.9597485208035038
- 0
- 0
- 0
- 0
- 17.45460058280481
--0.4584672381372454
- 0
- 0
- 0.4487193183395825
- 2.509000915737849
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.060832891790064
- 0
- 0
- 0
- 0
- 9.257999137863095
- 0
- 0
- 0
- 0
--0.7243493585571089
- 0
- 0.03529861642450902
- 0
--0.197077341093214
- 0
- 0
- 0
- 0
- 0.8954072727507604
- 0
- 0
- 0
- 0.7698355532451218
- 0
- 0
- 0.4023051624638282
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.09399833638321142
- 0
- 0
- 0
--1.262600486902211
- 0
- 0
--4.160847976544219
- 1.932346836413282
- 0
- 0
- 0
- 0
--0.104816819255623
- 0.5714514098133883
- 0
- 0
- 0
- 0.04443077418036852
- 0
- 0
- 0
- 0
- 0.5585053993437328
--0.5454601487542008
- 0
- 1.324030976848829
- 0
- 6.820272608146636
- 0
- 0
- 0
- 3.378335188162803
- 0
- 0
- 0.1774125573432427
--0.1296176597501693
- 0
- 0
- 0
- 0
- 0
- 0
--34.08189520105155
- 0.4836512065261352
--0.03366934059915522
- 0
- 0
- 0
- 0.9030043282988861
- 0
- 0.3403170399982783
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.6418261333430858
- 0.5930125797702109
- 0
- 0
--0.2653658892281105
--2.125054195110796
- 1.305277886752463
- 0
- 0
- 0
- 0
- 0
- 5.902457582414605
- 0
- 0
- 0
--0.2640713338373876
--1.86205456848709
- 0
- 0
- 0
- 0
--0.06446372278254772
- 0
- 2.458353208397256
- 0
- 0
--1.056033522128867
- 0
- 0
- 0
- 0
--95.25352685374895
- 0
--0.4401007194651089
- 0
- 10.54678914248936
- 0
--1.187019882929189
--3.260297671414309
--4.702470568084649
- 0.6662402585162309
- 0.2644209906260584
--3.259923607441013
--3.76578016858871
- 0.03267157280521491
--2.131586777350046
- 0
--8.901261451467073
- 0
- 0
- 0
--0.9781881107013581
- 0
- 0
- 0
- 0.0355610885400901
- 0
- 0.8186365257853162
- 0
- 0
- 0
- 0.7195530488757154
- 0
- 0
- 0
- 0
- 0
- 0.2751899949960999
- 0
- 0
--0.2396524683759211
- 0
- 0
- 0
- 0
- 0
- 0
--0.06359004984786285
- 0.3971337000403037
- 1.62571743762704
- 0
- 0
- 0
- 25.65173295550441
- 1.049947484834283
- 1.013097891999924
--6.833222268699098
- 0
- 0
--0.838017445959459
- 0
- 0
--0.007615529050443952
- 0
--1.586085318688383
--1.205832076872552
- 0
- 0
- 0
- 0.2744803035698759
--0.3635936868167469
--0.08332371375716639
- 0
- 0
- 0
- 0
- 0
--3.541811802794103
- 0
- 0
- 0
- 6.843791675518111
--1.749448469749449
- 0
- 0
--0.06740889074050756
- 0
- 0
- 0
- 0.9163098612251073
- 0
- 0
- 0
- 0.4270001411141601
- 0
- 0
--0.08503909842688996
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.6910793657521033
- 1.736013302379081
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.4136598313956726
- 0
- 0
- 0.6812318574308043
- 0
--0.06412392360262623
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2376019273086113
- 0
- 0
- 0
--5.395073155760457
- 0
- 0
- 1.257826547723061
- 0
- 0
- 0
- 0
- 0
- 0
--0.0903793043103544
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5242906044798107
--0.01742342683510349
- 0.5231441833401839
- 0
- 0.9252180529033944
--0.08404744906999868
- 0
- 0
- 0.1982211295659624
- 0
- 0
--0.7134922556759923
- 0
- 0
- 0.3497113364974072
- 0
--1.92262627016739
- 0
- 0
- 0
- 0.3166988911249562
- 0
--0.01537370047486854
--0.5507332504717685
--0.9548093407857391
- 0.1371119018936899
- 0
--0.3250900736454639
- 0.2874218466869001
- 0
- 0
- 0
- 0
- 0.5359569272752117
- 0
- 0
- 0
- 0
- 0.1352042087298128
--0.2128278143785792
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.06891972170365385
--0.7542786953052235
- 0
- 0
- 0
--0.2645897690731321
- 0
- 0
- 0
- 2.128438526907922
- 0
- 0
- 0
- 0
--0.24271016935905
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.254861765794941
- 0
- 0
--0.4076076943047222
- 27.36762208367834
--0.24613440449224
- 0
- 0
- 0
- 0
- 0.4608058213821488
- 0.7567050110937079
- 0
- 0
--0.4317127994858896
- 0
- 0
--0.9325312804208741
- 0
- 0
--0.6789861663591337
- 0
- 0
- 0
- 0.0295067305371026
- 0.7551902107307704
- 0
- 0
- 0.5513385229268563
- 0
- 0.5985994011157888
- 0
- 0
- 0.2576260871928632
--0.1940614202945517
- 0.5455890301580043
- 0
- 0
- 0
- 0
- 0
--1.836129011512891
- 0
--4.753161586852461
- 0
- 0
- 0
- 0
- 0.78745677826563
- 0
- 0
- 0.3264239362936548
- 0.8990131744412597
- 0
- 0
- 0
- 0
--0.1144378344686671
- 3.407851160953469
- 0
- 0
- 0
- 0
- 0
--0.06560238686281092
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.2630762849451005
- 1.686505625919402
- 0
- 0
- 0
- 0
- 0.7590373052622864
- 0
- 0
- 1.148711079483203
- 0
- 14.04204720243525
- 0
- 0
--0.06882562324756232
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.4531083418190276
- 0
- 0
- 0
- 1.473570981887633
--5.604186362510557
- 0
- 0
- 0.3155913226765344
- 0
- 0
- 0
- 0
- 0
- 0
--1.908889933764057
--18.51673226379179
- 0.6112558741953972
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.0848859884866955
- 0
- 0
- 3.768590045274363
- 0
- 0.7733918309026047
- 0
- 0
- 0
- 0
--0.3954332005105799
- 0
- 0
--0.6673923776817171
- 0
- 0
- 2.25911534939347
- 0
- 0.660611056293277
--0.1812213554674956
--0.04527142827977262
- 2.006619764943415
--0.1279086760711655
--1.22903327886986
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.208386377003379
- 0
- 0
- 0.2446803420797232
- 0
- 0
- 0
- 0
- 0
- 0
--0.725799469530091
- 0
--0.7765848593991672
- 2.873455108604292
--0.07617763534179756
- 0
- 1.226367682399656
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.573847901789428
- 0
- 0
- 0
- 0
- 0
--0.06649162841140518
- 1.107584462866992
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 4.244401079135958
- 0
--17.87513163308068
- 0
- 0
--1.544149723335144
- 0
- 0
- 0
- 0
- 0
- 0
- 1.298819764872114
- 0
--0.008282862618161422
- 0
- 0
- 0
- 0
--0.4698645726960388
--8.480466952694655
--0.4173397491301559
- 0
- 0
- 0
- 0
--0.7716645856393914
- 0
--0.2465528837629267
- 0
- 0
- 0
- 0
- 0
- 0.3482376100922915
- 0
- 0
- 0
- 3.880617777440751
- 0
- 0
--2.005286131021399
--1.303448776542771
- 0
- 0
- 0
--3.4193362880441
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.059385994677692
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.389749416279727
- 13.40284755787324
- 0
- 0.2379509723991778
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.04365241008911763
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 2.219815557691363
- 0
- 0
- 0
- 2.535174794619011
- 0
- 0
- 1.352789106934568
- 0
- 0
- 1.586202951336093
- 0
- 0.5172361930188167
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.8573618771372511
- 0
- 0
- 0
--0.3706547554105325
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.906339951884944
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1371823955417805
- 0.8217015084447183
- 0
- 0.7043108672951162
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.0952012
- 0
- 0
- 0
- 0
- 0
--0.07786373751854546
- 0
- 0
--0.04768267017453302
- 0
--0.354295538146551
--1.871363032703216
- 0
- 0
- 1.926458816072536
- 0
- 0
- 0
--0.03552216366667377
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.371364430695413
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.2153633109984628
- 0
- 0
- 0.4223110646342952
- 1.430937908948884
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.08570651009198732
- 0
- 0
- 1.704105254682967
- 0.4572188951355719
- 0
--0.174391801024029
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.780016805708248
- 0
--0.5033127810949863
- 0
- 0
- 3.348341099969325
--0.1413143117226115
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 2.601973607696656
- 0
- 0
- 0
- 0
--0.01096936254525321
--0.411747449414281
- 0
- 0
- 0
- 0
- 0.1911840919183682
- 0
- 0
- 0
- 0
- 0
- 0.02507437889959847
- 0
--1.920502223064878
- 0
- 1.485265811146348
- 0
- 0
- 0
- 0
- 0
--0.1270227091236665
- 0
- 0
- 0
- 0
- 0.9319905498951758
- 0
- 0
- 0
- 0.06355531248568379
- 0
- 0
- 0
- 0
- 631.8970732533545
- 0
--0.06941261366326315
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.9745960818564167
- 0
--0.1319399145673239
- 0
- 0
--0.1712060015170429
- 0
- 0
- 0
- 0.2199338815009368
- 0
- 0
- 0
- 0
- 0
- 0
--0.2951937985353986
- 1.681449903106215
- 0
--0.1874407716775586
- 0
- 0
- 0
- 0
- 0
--0.09141767654559446
- 0
- 0
--0.4518509907644195
--0.1468542372417183
--1.075467361334804
- 0
- 0
- 0
- 0
--1.955437134594413
--0.6900778960186711
- 0
- 0
- 0
--1.449028368203642
--0.5433146573681179
--0.4085400611332851
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2012560825672974
- 0.4787618884403176
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--8.631168098229734
- 0
- 0
- 0
- 0
--0.8104835496027143
--1.646720809557628
- 0.8497248224834173
- 1.103496050793216
--0.2870479812674109
- 0
- 0
- 3.759113878885802
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.1337480208931563
- 0
- 0
- 0
--0.3528623797300455
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.0531751278621955
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.4197638202775679
--1.162312655274553
--0.2186815719856952
- 0
- 0
- 0.5768419056861613
- 0
- 0
- 0
- 0
- 0
--0.9876302547108818
- 9.178415714553781
- 0
- 0.486443770275648
--1.074427393258677
- 0
--0.7190429399116103
- 0
- 0
--0.004654251984783326
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.08279607355872896
- 0
- 2.428085286670281
- 0
--0.290905267655203
- 0.1784003316654723
- 0
- 0
- 0.3937972051871597
- 0
- 0
- 0
- 0
- 2.041969844109952
- 0
--0.6632670925089514
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.04730653973583668
- 0.7340408428608161
- 0
- 0
- 0
- 0.5631018585208183
- 0.9022015166062511
--0.4946213370473764
- 3.116402896865955
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.4753854557031529
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.5776629841543006
- 0
- 0
- 0
- 0
- 0
--0.4693467331486799
- 0.5092722521912728
- 0
- 0
--0.5751418088375293
- 0
- 1.110180300043974
- 0
- 0
- 0
--0.2052398029192016
- 0
- 0
- 0
- 0
- 0.3441802634664664
- 0
- 0
--0.06054885994923487
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 14.14678012942133
- 0
- 0
--0.3774029308910528
--0.9181606392989132
--0.4971518532410875
- 2.060768053484165
--0.3705989448248495
- 0
- 0
- 0
- 0
- 0
--1.47720358563491
- 0
- 0
- 0
- 0
- 0.4942864830129788
- 0
- 0
- 0
- 0
- 0
- 0
--5.980874489912058
- 0
- 0
- 0
- 0
- 0
- 0.7527716678327022
- 2.817949083040197
- 0
- 0
--0.4396265411868724
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--48.58940723722201
- 0
- 0
--0.3877076367092464
- 0
- 0
--0.652800681008756
- 0
- 0
- 0.2261910205188926
- 0.6653039312906668
- 0
- 0
- 0
- 0
- 0
--0.6725639123466175
--0.9328676887049783
- 12.82686028696979
- 0
--0.1825224701986439
- 0
--0.2895567290213468
--0.5793400426562111
- 0
--0.6640461395506864
--0.2270508046043917
--0.6412166121034713
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1126563621331092
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.330630854295177
- 0
- 0
- 0
- 0.1647035387110507
- 0.1524236052870568
- 0.1265520227358675
--4.496978640717864
- 0
- 0
- 0.05794659470297663
- 0
- 0.982493550693461
- 0
- 0
- 0.6567604474650559
- 0.01440349345695066
- 0
- 0
- 0
- 11.04811591081289
- 0
- 0
- 0
- 0
--0.7645032681110474
- 0
- 1.719948735368507
- 0
--0.3079470598149933
- 1.085667939008959
- 0.3483428365886299
- 0
- 0
- 0
- 0
- 0
--2.930424196970102
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.3606349932057697
- 0
- 0
- 0
--1.957148493090422
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--2.008276092090168
- 0
- 0
- 0
--0.00749535856390822
--0.02428039420676276
- 0
- 0.3930395680367711
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 14.39737910881446
- 0
- 0
- 0
- 0
- 0
--1.292935206190298
- 0
- 0
--0.0248949634914246
- 0
- 0
--1.125024319777534
- 0
- 0
- 0
- 0
- 0
- 0
--0.4927965488776805
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1869230936462014
- 0
--0.6630828683624668
- 0
- 0
- 0
- 0
- 0
--0.4594092525851398
- 0
- 0
--0.3202097255038425
- 0
- 0
- 0
- 0
- 0
- 0.22447
- 0
- 0
- 0.5910169655671389
--0.4836597271883145
- 0
--0.5531825210060894
- 3.721493044549171
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 2.112397477340145
- 0
--4.7721984738587
- 0
- 0
--0.5550571596401548
--0.566879658670128
- 0
- 0
- 0
- 0
- 0
- 0
- 0.01808279498347775
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.3606017229961584
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.07680287577569114
- 0
- 0.1295568785771593
- 0
- 0
- 0
--11.95421298269813
--0.604529538866787
- 0
- 0
--0.3184720155498334
- 0
- 0
- 0.1793451921501975
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.556018466402506
--0.4667126353644137
- 0
--0.4886251029536635
- 0
- 0
- 0
- 0
--20.5762341697062
- 0.9054584447145799
- 0
--0.4335941386675467
--0.231865506519269
- 0.09267561124885154
- 0
--0.3224450001651057
- 1.946623869033397
- 0
- 0
- 0.1813040482362534
- 0
--1.615596155779889
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.2805840358472876
- 0
- 0.1752426257870506
- 0
- 0
- 0
- 0.4245792310373214
- 0
- 0.5652421411245472
- 0
- 4.342363857508019
- 0
- 0
- 0
- 0
- 0
--1.267729134291342
- 0
- 0
- 0.5097122755503997
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 2.269051532751651
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.7906434423290707
- 0
- 1.211667301441654
- 0
- 0.5667500477593946
- 0
- 0
- 0
- 0.2352951235203545
- 0
- 18.02523074688196
- 0
- 0
- 0
- 6.869350820322008
- 0
--0.02239742062022338
- 0
- 0
- 0
- 0
- 0
- 0.6942810817858166
- 0
- 0
- 0
- 0
- 0
- 0.6717730520651083
- 0
- 0
- 0
- 0.9053323109763444
- 0
- 0
- 0
- 0.8083143391330183
- 0
- 18.95307669364243
- 0
--0.05881962894534767
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2314600321165206
- 0
- 0
- 0
--0.1563748024625857
--0.001188087501363268
- 0
- 0.6718259486609706
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.09505914867643134
- 0
- 0
- 0
- 0
- 0
- 0.1068405874333421
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 4.569792430389595
- 0
- 0
- 0
- 4.993089268138056
- 0
- 0
- 0
- 0
- 0.01474832937443083
- 0
- 0.2728264573828585
--0.08423510041166489
--0.2920139791774037
- 0
- 2.294522220982419
- 0
--0.4169884839112438
--11.38550988688505
- 0
- 0
- 0
- 0
- 0.5073898139195817
- 0
- 0
--0.646334730261419
- 0
- 0
--0.1016224973622556
- 0
- 0
- 0
- 0
- 0
- 0
--0.4915017578748171
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.5427839248691851
- 0
- 0
- 0
- 0
- 0
--0.1717333411874349
- 0
- 2.713951885471826
- 1.996453627306153
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2831911043270032
- 0
--0.001542827498603446
--0.2603547531531951
- 0
- 0
- 0
- 0
- 0
- 0.987637150004415
- 0
- 0
- 0.262065744936163
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.47447264974626
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.15333572570218
- 0.3094455719060705
- 0
--0.5829498577817496
- 0
- 0
- 0.4609346394178296
- 4.328021312376961
--1.805795280283172
- 0
- 2.764523895521742
- 12.15008450166943
- 0
- 0
--1.739598619461891
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.008670708193809626
- 3.003029141912023
--0.09571484050607278
- 0
- 0
- 0
- 0
- 0
- 2.998238679478707
- 0.1328472941945366
--0.2909476355962676
--0.174970057795891
- 0
- 0
- 0.1910965456168723
- 0
- 0
- 0
--0.2727953979361796
- 0
- 0
- 0
- 0
- 0
--0.1409298714812179
- 0
- 0
- 0
- 0
- 0
--0.2476230023872065
- 0
--1.269752782652631
- 0
- 0.03740787628133018
- 0
- 0
- 0
- 0
- 1.937298800300633
- 0
- 0
--0.7649201907844544
- 0
- 0
- 0.07867804647648058
- 0
- 0
- 0
- 0.4108565224750267
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1.509400641752963
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 3.09388010063306
- 0.7269515533548975
- 0
- 0
- 0
- 0
- 0
- 0
--2.683452264523992
- 0
- 0.1610736998352869
- 0
- 0
- 0.4530002040490377
- 0.0940423820304188
- 0
--0.2523436531574056
--0.3146854565835709
- 0
--0.5438896078024471
--0.5481997387936354
- 0
- 0
- 0
--0.3529512841306343
- 0.01819211316772903
- 0.3534341354990801
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.400508770205981
- 0
--0.02529498154724328
- 0.01190877399766866
- 0
- 0
- 0
- 0.5985093361405944
- 0
- 0
- 0
- 0
- 0
--0.720375393357375
- 0
- 0
- 0.1274464096544585
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.003653887756514014
--0.005073849378708361
- 0
- 0
--0.8802391626152068
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.3218661687166185
- 0
--0.4904032546922169
- 0
- 1.72659614623605
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2264047399303686
- 0
--0.8208743070956491
- 0
- 0
--9.51534427481794
- 0
- 0.4213718633385596
- 0
--19.72682087323344
- 0
- 0
- 0
- 0
- 0
- 0.4694729102048924
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5258119271459143
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 3.820621745385362
- 0
- 0.1390174491737094
- 0
--0.3664113363388742
- 0
- 0
- 0
- 0
--0.3885587216893042
--0.1203394912747273
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.06414053528425237
- 0
- 0
- 0.09289294530186366
- 0
- 0
--0.8466475210571507
- 0
- 0
--0.4678784812654621
--2.808785268303473
- 0
- 0
- 0
- 0
- 0
- 0.6286916829564756
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2855360076757998
- 0
- 0
- 0
--50.37032056490291
- 0
--0.8602511248033244
- 0
- 0
- 0
- 0
- 0
- 0
--4.210390682638181
- 0.06211066398950435
--0.03632024552548954
- 0
- 0
- 0
- 0
- 0
- 0
--0.01138833579768669
- 0
- 0.1220142412439677
- 0
--0.02347313472194225
- 0
- 0
- 0
--0.01237103396233737
- 0
- 0
--0.08965046314421378
- 0
- 0
- 0
- 1.080724675852843
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.6212490534855771
- 0
--0.3399176535743508
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.03679826422258359
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.3195751411836361
- 0.9154179302658173
- 0
--0.1150711320732581
- 0
- 0
- 0
- 0
- 0.346131493789016
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.7576600189486671
--0.824844165181285
- 0
- 0
- 0
- 0
- 0.14962600744346
- 0
--2.748732521186326
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.5081557889752605
- 0
- 0
- 0
- 0
- 0.2552804322082619
--0.2745290629953129
- 0
- 3.122896313380039
- 0
- 0
- 0
- 0
--2.920923705424083
- 2.985171570816766
- 0
- 0
- 0.5476907474663835
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--3.506693439210517
- 0
- 0
- 0
- 0
- 0
--0.2345569520482285
- 0
- 0
- 0.1476134575083644
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--18.85881257776057
- 10.06109775574651
- 0
--0.5793006882210047
- 0.2629820711089522
- 0
- 0
--0.2742277127815957
- 0
- 0.131355853707775
- 0
- 0
- 0.04771138396277713
- 0
- 0
- 0
--1.336706366462532
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.1949107366043034
- 0
- 0
- 1.825493279082812
- 0
- 0
--0.4385088336266119
- 0
--0.2743317451076697
- 0
- 0
- 0
- 0
- 0
- 5.442286337540354
- 0
- 0
- 1.404220068296073
- 0
- 0
- 0
- 0
--0.2579837173997316
- 0
--0.4450387478341629
- 0
- 0
--1.670108626031611
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 5.798267914669685
- 0
- 0
- 0.05658932251961987
- 0
- 0.4172779499635791
- 0
- 0.105300931833048
- 0
- 0
- 0
- 0
- 0
--0.7484888061118221
- 2.041850891755478
- 0.7330715541422896
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1635745536362639
--0.1783447111432833
- 0
--0.01494429797658691
--1.036642363669109
- 0
- 0
- 0
--0.9786186554979159
- 1.475192028122727
- 0
- 0.1415519025282671
- 0
--1.117981491915489
- 0.8107534405518425
- 0
- 0
--3.891722344659192
--0.3299595865831923
- 0
- 0.03353894469956772
--0.20972172091688
- 0.2909925331783702
- 0
--4.374493439557915
- 0
- 0
--0.1630634619639877
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.2856117907298797
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--2.88741370390883
--0.09463873033654958
- 0
- 0.7989153733960048
- 0
- 0
- 0
- 0.1951193739747824
- 0
- 0
- 0
- 0.08351546533483424
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 68.37257976990995
- 0
- 0
- 0
--1.137612020508681
- 0
--0.1958510587693432
- 0
- 0
- 0
- 0
- 0
- 0
--0.06938557191253593
- 0
--0.222839558492001
- 0
- 0.09291263597164898
- 0
- 0
- 0
- 0
- 0
- 0
--3.745044048189886
--0.1313473536250102
- 0
- 0
- 0
- 0.3458383157974664
--1.544738243051305
- 0
- 0
- 0
- 0
- 0
--1.510079153083887
- 0
- 0
- 0.171879670687663
- 0.3878037114504239
- 0
- 0
- 0
- 0
- 0
- 2.567010692908583
- 0
--1.305291396721716
- 0
- 0
- 0
--0.9411906007959661
- 0.5434008860134193
- 0
- 0
- 0
- 0.6226650835463924
- 0.1482446671978628
- 0
- 0
- 0
- 0
--1.455503868024161
- 0
- 0.1942377676290563
- 0
- 0
- 0
- 0
--0.07492512892754151
- 0
- 0
- 0
- 0.9714305088747539
- 0
- 0
- 0
- 0
- 0
--0.07904651089037804
- 0.9157694617406652
--0.1641815028685157
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.1254313797317232
- 0
--0.6905436226153474
- 0
--0.08360246413693795
- 0
- 0
--0.5880038786799398
- 0
- 0
- 0
- 0
- 0.6022766110942244
- 0
- 0.5148146000418764
- 0
- 0
--20.48252707563957
- 0
- 0
- 0.5575512912831021
- 0
- 0
- 0
- 0
- 0
--0.01835850798101779
--0.2644850972603794
- 0
- 0
--0.09468032394924522
- 0
--3.102589977467122
- 0
- 0.3600538373706629
- 0
- 0
- 0
--1.034585556269956
- 0
- 1.039162471692296
- 1.404015979975387
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2834220750958322
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.06284581408869482
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.8100900574924416
- 0
- 0
- 0.1207606143828642
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--13.01195159386746
- 0
- 0
- 0
- 0
- 0.06609523597093495
- 0.6788148252064654
- 0
- 0
- 0
--0.2245540166061343
- 0
--1.877669744282497
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.1043226421995354
--0.1541676423463312
--0.3083832389853993
- 0
- 0
- 26.75044384818589
- 0
- 0
- 0
- 4.680978823377932
- 0
- 0
--2.467146937828286
--0.6034157196865881
- 0.2456847071708288
- 0.3648953118737308
- 0
- 0
--0.7165533886314798
- 0
- 0
- 0
- 0
- 0
- 0
- 0.4146478223033953
- 2.67153547865388
- 0.4066037119393711
- 0
- 0
- 0
- 0
--0.3044243086619978
- 0
- 0.05376234688607281
- 0
- 0
- 0.08850532670384344
- 0.06749428120635347
- 0
- 0
- 0
- 0
- 0
- 0
--0.5114454170290706
- 0
- 0
- 4.755777436485564
--1.240394922245889
- 0.008546368411881411
- 0.3339486258385727
- 0.3961449252235861
- 0
- 0
--4.760115984646152
- 0.9460435769805294
- 0.06738988249241178
--0.1241293530803617
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--3.941528010506792
- 0
- 0
- 0
- 0
- 0
--1.095864235879698
- 0
--1.643754348478478
- 0
--1.119532545273484
--2.967482676626235
--0.5254091503576083
- 0.516258261247043
- 1.582259202190788
- 0
- 0
- 0
- 0
--0.5232137123210626
--0.8578576545155802
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--4966.139203211767
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 7.810402306959599
- 0
--1.141048704919943
- 0
- 0
- 0
- 0
--0.4974314046107617
- 0
- 0.02832075084311592
- 0
--1.004514081789116
- 0
- 0
--62.26077547658917
- 23.45803782680646
- 0.04071433235873094
- 0
--0.2740057032473357
- 0
- 0
- 0
- 0
--0.6634096867034206
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.05567752536680285
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 4.476528587143142
- 0
--0.3102269985689736
- 0
- 0
- 0
--0.2120672325605232
- 0
- 0.1082684195035498
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.407383006921814
- 0
- 0
- 0
- 0
- 0
--1.802572821686524
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.261498978802622
- 0.144742328702142
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.05436200145226892
- 0
- 0
- 0.03123923729849884
- 0
- 0
- 0
- 0
- 0
- 0
- 0.5493282349642721
--3.56605055765311
- 0
- 0.1189964332247844
- 0
- 0
- 0.2712761050398067
- 0
- 0
- 0.9649887849336986
- 0
--1.662202733759401
- 0
- 0
--0.3417780740030298
--0.825490096558209
- 0
- 0
- 0.8613713427803134
- 0
- 0.102663973308054
--0.118950085071405
- 0
- 0
--0.3367690776431803
- 0
- 0
- 0
- 0
- 0
- 5.139357298646494
--0.8646777626735802
--0.1888400086466105
--8.087815225955227
--0.7858696596423445
- 0
- 0
- 6.367438364151895
- 0
- 0.7140798553081277
--0.008810645504535789
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.483097156676205
- 2.091303289750183
- 0
- 0
- 0.321893877144759
- 0
- 0
- 0
- 0
- 13.57001687135731
- 0
- 0.5451595807807043
- 0
- 1.501350698141659
--0.06782249716495321
- 0
- 0
- 6.512865467491789
- 0
- 0.1265142267536407
- 0
- 0
- 0
--2.164126884731795
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--1493.381090410497
- 0
--0.5528447891372515
- 0
- 0
- 0
- 0
- 0
--0.4878802784931899
- 29.39776799459366
- 0
- 0
--6.246840660823246
- 1.837422099503762
- 0
- 0
- 0
- 0.5438964907160825
- 0
- 1.809629679819767
--0.2088189637961629
- 0
- 0.2371944576606236
- 0
--4.576313887759413
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.004436994198865
- 0.427447789557515
- 0
- 0
- 0.04345421782712366
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--3.785231348831975
- 0
- 0
- 0
- 3.484380229783484
- 0
- 0
- 0
- 0
- 0
--0.3581986366645341
- 0
- 0
- 0
- 0.03318181775015192
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--74.78907765408408
- 0
- 0
- 0
- 1.944787624214348
- 0.09929293016417996
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.116408775050883
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2090617426284433
- 0
- 0
- 0
- 0
- 0
- 0.01712180883962215
- 0.113613167796337
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.689511721776201
--0.4182393753066492
- 0
--1.154405689010111
- 0
- 0.1473834929579108
- 0
- 0.1434874561955266
--0.05794164273893747
--0.01094366369749314
- 0
- 0
- 0
- 0
- 3.888280276242147
- 0.8505678665757467
- 0
- 2.402259310185026
- 0
- 0
- 0
- 0
--0.4618068812711437
--0.1093509337748507
- 0
--0.1471360397257355
- 0
- 0
--0.1344705267155689
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--4.025016515700098
- 0
--0.330133694982991
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
--0.5026319300955789
- 0
- 0.2922854120271939
- 0
- 0
- 0
- 0
- 41.91848303091628
- 0
- 0
- 0
- 0
- 0
--3.645117646421415
- 0
- 0.935149590315432
- 0
- 0.2157329735790116
- 1.888239187045813
- 3.215581067843307
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.2716732938603457
- 0.02792966108123637
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 0.009161683183842015
- 0
- 0
- 0
- 34.84420319112557
- 0
- 0
- 0
- 0
- 0.3537999629297381
- 0
- 1.222501112809391
- 0
- 0.04303850878046313
- 0
- 0
- 0
--0.4616341244191867
- 0
--10.73295200880771
- 0
- 0
- 0
- 0
- 0
- 0
- 0
- 1.251021320332088
--0.1752269560196747
- 0
- 0
- 0.1440088296602519
+%%MatrixMarket matrix coordinate real general
+% Generated 15-Dec-2020
+100 100 413
+1 1  1
+2 1 -2
+36 1 -0.1213541159864248
+2 2  1
+3 2 -2
+36 2 -0.2234076325883467
+1 3 -3
+3 3  1
+4 3 -2
+36 3 -0.1723808742873859
+2 4 -3
+4 4  1
+5 4 -2
+36 4 -0.01586307945822912
+3 5 -3
+5 5  1
+6 5 -2
+36 5  0.05895829803007624
+4 6 -3
+6 6  1
+7 6 -2
+36 6 -0.05499252816551889
+5 7 -3
+7 7  1
+8 7 -2
+36 7 -0.2327938073114557
+6 8 -3
+8 8  1
+9 8 -2
+36 8 -0.256125233970946
+7 9 -3
+9 9  1
+10 9 -2
+36 9 -0.07353328134780768
+8 10 -3
+10 10  1
+11 10 -2
+36 10  0.1018726610595282
+9 11 -3
+11 11  1
+12 11 -2
+36 11  0.04916682984509524
+10 12 -3
+12 12  1
+13 12 -2
+36 12 -0.1983681834823948
+11 13 -3
+13 13  1
+14 13 -2
+36 13 -0.3377095904296545
+12 14 -3
+14 14  1
+15 14 -1.724373893059409
+36 14 -0.1889801401343747
+13 15 -3
+15 15  0.7243738930594099
+16 15 -2
+36 15  0.1079576547092199
+14 16 -3
+16 16  1
+17 16 -2
+36 16  0.1685008677083124
+15 17 -0.679031449792187
+17 17  1
+18 17 -2
+36 17 -0.08486491423415454
+15 18 -1.247045114693007
+16 18  5
+18 18  1
+19 18 -2
+36 18 -0.4575675428829226
+15 19 -0.9630382822425891
+17 19 -3
+19 19  1
+20 19 -2
+36 19 -0.5239675301210069
+15 20 -0.08649452377957778
+18 20 -3
+20 20  1
+21 20 -2
+36 20 -0.1107188635882647
+15 21  0.327254094340226
+19 21 -3
+21 21  1
+22 21 -2
+36 21  0.2417107461185162
+15 22 -0.3056304633953435
+20 22 -3
+22 22  1
+23 22 -2
+36 22  0.1650959221222523
+15 23 -1.304003822222047
+21 23 -3
+23 23  1
+24 23 -2
+36 23 -0.416469665678729
+15 24 -1.42544006998843
+22 24 -3
+24 24  1
+25 24 -2
+36 24 -0.6543312863384098
+15 25 -0.4153951095018817
+23 25 -3
+25 25  1
+26 25 -2
+36 25 -0.4204782400141739
+15 26  0.5771424484949292
+24 26 -3
+26 26  1
+27 26 -2
+36 26  0.3349436185251811
+15 27  0.2630280411460528
+25 27 -3
+27 27  1
+28 27 -2
+36 27  0.3140251202450244
+15 28 -1.094982195909327
+26 28 -3
+28 28  1
+29 28 -2
+36 28 -0.02629520010125064
+15 29 -1.904783414376831
+27 29 -3
+29 29  1
+30 29 -2
+36 29 -0.9892678277371458
+15 30 -1.02871119411978
+28 30 -3
+30 30  1
+31 30 -2
+36 30 -0.4764037664989635
+15 31  0.570268051334746
+29 31 -3
+31 31  1
+32 31 -2
+36 31 -0.2223553165986417
+15 32  0.9854802563087663
+30 32 -3
+32 32  1
+33 32 -2
+36 32  1.09507939990504
+15 33 -0.5362341765638339
+31 33 -3
+33 33  1
+34 33  1
+36 33 -0.3329340502040744
+15 34 -2.173845828309308
+32 34 -3
+34 34  1
+35 34 -2
+15 35 -1.840045256427322
+33 35 -3
+35 35  1
+36 35 -0.569367230550896
+15 36  0.2756261069405881
+34 36 -3
+36 36  0.568169381163119
+37 36 -2
+15 37  1.674207902874854
+35 37 -3
+37 37  1
+38 37 -2
+15 38  0.4742161470847417
+36 38 -2.136338762326237
+38 38  1
+39 38 -2
+15 39 -2.099295020072068
+37 39 -3
+39 39  1
+40 39 -2
+15 40 -2.91041213039506
+38 40 -3
+40 40  1
+41 40 -2
+15 41 -0.7048659415484002
+39 41 -3
+41 41  1
+42 41 -2
+15 42  2.052627714763479
+40 42 -3
+42 42  1
+43 42 -2
+15 43  1.890556552092056
+41 43 -3
+43 43  1
+44 43 -2
+15 44 -1.336727149842232
+42 44 -3
+44 44  1
+45 44 -2
+15 45 -3.859325783342909
+43 45 -3
+45 45  1
+46 45 -2
+15 46 -2.354919722585437
+44 46 -3
+46 46  1
+47 46 -2
+15 47  1.733802799937269
+45 47 -3
+47 47  1
+48 47 -2
+15 48  3.473339488926904
+46 48 -3
+48 48  1
+49 48 -2
+15 49  0.3469620532959092
+47 49 -3
+49 49  1
+50 49 -2
+15 50 -4.222933012672698
+48 50 -3
+50 50  1
+51 50 -2
+15 51 -4.54729051317281
+49 51 -3
+51 51  1
+52 51 -2
+15 52  0.3044543905237369
+50 52 -3
+52 52  1
+53 52 -2
+15 53  4.733424537628359
+51 53 -3
+53 53  1
+54 53 -2
+15 54  3.005475714826209
+52 54 -3
+54 54  1
+55 54 -2
+15 55 -3.40816722931753
+53 55 -3
+55 55  1
+56 55 -2
+15 56 -6.844800977902615
+54 56 -3
+56 56  1
+57 56 -2
+15 57 -2.534560869406824
+55 57 -3
+57 57  1
+58 57 -2
+15 58  4.930783492560888
+56 58 -3
+58 58  1
+59 58 -2
+15 59  6.353061934454685
+57 59 -3
+59 59  1
+60 59 -2
+15 60 -0.8234374492359253
+58 60 -3
+60 60  1
+61 60 -2
+15 61 -8.433204300342153
+59 61 -3
+61 61  1
+62 61 -2
+15 62 -6.761738537629796
+60 62 -3
+62 62  1
+63 62 -2
+15 63  3.167277656549965
+61 63 -3
+63 63  1
+64 63 -2
+15 64  9.61741983611941
+62 64 -3
+64 64  1
+65 64 -2
+15 65  3.885150102266174
+63 65 -3
+65 65  1
+66 65 -2
+15 66 -8.14223932207687
+64 66 -3
+66 66  1
+67 66 -2
+15 67 -11.8037578792595
+65 67 -3
+67 67  1
+68 67 -2
+15 68 -1.374593999888337
+66 68 -3
+68 68  1
+69 68 -2
+15 69  11.45190819694065
+67 69 -3
+69 69  1
+70 69 -2
+15 70  10.5309349343001
+68 70 -3
+70 70  1
+71 70 -2
+15 71 -4.652324253436384
+69 71 -3
+71 71  1
+72 71 -2
+15 72 -16.30044795481162
+70 72 -3
+72 72  1
+73 72 -2
+15 73 -9.094926210163164
+71 73 -3
+73 73  1
+74 73 -2
+15 74  10.07720169911733
+72 74 -3
+74 74  1
+75 74 -2
+15 75  17.96332329653994
+73 75 -3
+75 75  1
+76 75 -2
+15 76  3.211979880855939
+74 76 -3
+76 76  1
+77 76 -2
+15 77 -18.17054027522279
+75 77 -3
+77 77  1
+78 77 -2
+15 78 -19.30846259331732
+76 78 -3
+78 78  1
+79 78 -2
+15 79  3.387513689255936
+77 79 -3
+79 79  1
+80 79 -2
+15 80  24.11330578208741
+78 80 -3
+80 80  1
+81 80 -2
+15 81  15.45729321281348
+79 81 -3
+81 81  1
+82 81 -2
+15 82 -14.25866492640947
+80 82 -3
+82 82  1
+83 82 -2
+15 83 -30.48937399604517
+81 83 -3
+83 83  1
+84 83 -2
+15 84 -9.390000607316445
+82 84 -3
+84 84  1
+85 84 -2
+15 85  24.63424990715358
+83 85 -3
+85 85  1
+86 85 -2
+15 86  31.96818825437212
+84 86 -3
+86 86  1
+87 86 -2
+15 87 -3.34784100233019
+85 87 -3
+87 87  1
+88 87 -2
+15 88 -36.72620214568408
+86 88 -3
+88 88  1
+89 88 -2
+15 89 -31.03792909483495
+87 89 -3
+89 89  1
+90 89 -2
+15 90  19.09197826479394
+88 90 -3
+90 90  1
+91 90 -2
+15 91  44.09456630001039
+89 91 -3
+91 91  1
+92 91 -2
+15 92  23.06086270612836
+90 92 -3
+92 92  1
+93 92 -2
+15 93 -41.61714653637393
+91 93 -3
+93 93  1
+94 93 -2
+15 94 -46.78202396794738
+92 94 -3
+94 94  1
+95 94 -2
+15 95 -12.64902986133766
+93 95 -3
+95 95  1
+96 95 -2
+15 96  67.30148694911092
+94 96 -3
+96 96  1
+97 96 -2
+15 97  35.07354469124684
+95 97 -3
+97 97  1
+98 97 -2
+15 98 -0.01197533973573717
+96 98 -3
+98 98  1
+99 98 -2
+15 99 -102.3949905399174
+97 99 -3
+99 99  1
+100 99 -2
+15 100 -2.861569553030421
+98 100 -3
+100 100  1

From 99a79e8030b2449b95d5d742903da023a6bbdefe Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Wed, 16 Dec 2020 08:22:45 +0100
Subject: [PATCH 0509/1008] cascading excess solution, cuda

---
 common/preconditioner/isai_kernels.hpp.inc    |  36 +++---
 core/preconditioner/isai.cpp                  | 106 +++++++++++-------
 core/preconditioner/isai_kernels.hpp          |   6 +-
 cuda/preconditioner/isai_kernels.cu           |  14 ++-
 cuda/test/preconditioner/isai_kernels.cpp     | 102 +++++++++++++++--
 dpcpp/preconditioner/isai_kernels.dp.cpp      |  26 +++--
 hip/preconditioner/isai_kernels.hip.cpp       |   8 +-
 omp/preconditioner/isai_kernels.cpp           |  18 +--
 omp/test/preconditioner/isai_kernels.cpp      |  24 ++--
 reference/preconditioner/isai_kernels.cpp     |  18 +--
 .../test/preconditioner/isai_kernels.cpp      |   9 +-
 11 files changed, 250 insertions(+), 117 deletions(-)

diff --git a/common/preconditioner/isai_kernels.hpp.inc b/common/preconditioner/isai_kernels.hpp.inc
index 7bd2e17e154..335ffb5b2ed 100644
--- a/common/preconditioner/isai_kernels.hpp.inc
+++ b/common/preconditioner/isai_kernels.hpp.inc
@@ -293,11 +293,13 @@ __global__ __launch_bounds__(default_block_size) void generate_excess_system(
     const IndexType *__restrict__ excess_nz_ptrs,
     IndexType *__restrict__ excess_row_ptrs,
     IndexType *__restrict__ excess_col_idxs,
-    ValueType *__restrict__ excess_values, ValueType *__restrict__ excess_rhs)
+    ValueType *__restrict__ excess_values, ValueType *__restrict__ excess_rhs,
+    const size_type e_start, const size_type e_end)
 {
-    const auto row = thread::get_subwarp_id_flat<subwarp_size, IndexType>();
+    const auto row =
+        thread::get_subwarp_id_flat<subwarp_size, IndexType>() + e_start;
 
-    if (row >= num_rows) {
+    if (row >= e_end) {
         return;
     }
 
@@ -309,7 +311,7 @@ __global__ __launch_bounds__(default_block_size) void generate_excess_system(
     const int local_id = subwarp.thread_rank();
     const auto prefix_mask = (config::lane_mask_type{1} << local_id) - 1;
 
-    if (row == 0 && local_id == 0) {
+    if (row == e_start && local_id == 0) {
         excess_row_ptrs[0] = 0;
     }
 
@@ -320,6 +322,9 @@ __global__ __launch_bounds__(default_block_size) void generate_excess_system(
     auto excess_rhs_begin = excess_rhs_ptrs[row];
     auto excess_nz_begin = excess_nz_ptrs[row];
 
+    auto out_nz_begin = excess_nz_begin - excess_nz_ptrs[e_start];
+    auto out_ptrs_begin = excess_rhs_begin - excess_rhs_ptrs[e_start];
+
     // defer long rows: store their nnz and number of matches
     for (IndexType nz = 0; nz < i_row_size; ++nz) {
         auto col = i_col_idxs[i_row_begin + nz];
@@ -335,33 +340,35 @@ __global__ __launch_bounds__(default_block_size) void generate_excess_system(
                 // trisystem(nz, i_idx) = m_values[m_row_begin + m_idx]
                 // only in sparse :)
                 if (valid) {
-                    auto nz = excess_nz_begin + popcnt(mask & prefix_mask);
-                    excess_col_idxs[nz] = excess_rhs_begin + i_idx;
+                    auto nz = out_nz_begin + popcnt(mask & prefix_mask);
+                    excess_col_idxs[nz] = out_ptrs_begin + i_idx;
                     excess_values[nz] = m_values[m_row_begin + m_idx];
                 }
-                excess_nz_begin += popcnt(mask);
+                out_nz_begin += popcnt(mask);
             });
         if (local_id == 0) {
             // build right-hand side: 1 for diagonal entry, 0 else
-            excess_rhs[excess_rhs_begin + nz] =
+            excess_rhs[out_ptrs_begin + nz] =
                 row == col ? one<ValueType>() : zero<ValueType>();
             // store row pointers
-            excess_row_ptrs[excess_rhs_begin + nz + 1] = excess_nz_begin;
+            excess_row_ptrs[out_ptrs_begin + nz + 1] = out_nz_begin;
         }
     }
 }
 
-
 template <int subwarp_size, typename ValueType, typename IndexType>
 __global__ __launch_bounds__(default_block_size) void copy_excess_solution(
     IndexType num_rows, const IndexType *__restrict__ i_row_ptrs,
     const IndexType *__restrict__ excess_rhs_ptrs,
     const ValueType *__restrict__ excess_solution,
-    ValueType *__restrict__ i_values)
+    ValueType *__restrict__ i_values, const size_type e_start,
+    const size_type e_end)
 {
-    const auto row = thread::get_subwarp_id_flat<subwarp_size, IndexType>();
+    const auto excess_row =
+        thread::get_subwarp_id_flat<subwarp_size, IndexType>();
+    const auto row = excess_row + e_start;
 
-    if (row >= num_rows) {
+    if (row >= e_end) {
         return;
     }
 
@@ -376,7 +383,8 @@ __global__ __launch_bounds__(default_block_size) void copy_excess_solution(
     if (excess_size > 0) {
         // copy the values for this row
         for (IndexType nz = local_id; nz < excess_size; nz += subwarp_size) {
-            i_values[nz + i_row_begin] = excess_solution[nz + excess_begin];
+            i_values[nz + i_row_begin] =
+                excess_solution[nz + excess_begin - excess_rhs_ptrs[e_start]];
         }
     }
 }
diff --git a/core/preconditioner/isai.cpp b/core/preconditioner/isai.cpp
index 7fb25b084c3..884888ca93d 100644
--- a/core/preconditioner/isai.cpp
+++ b/core/preconditioner/isai.cpp
@@ -147,48 +147,76 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
             excess_row_ptrs_full.get_data(), is_lower));
     }
 
-    auto excess_dim =
+    auto total_excess_dim =
         exec->copy_val_to_host(excess_block_ptrs.get_const_data() + num_rows);
     // if we had long rows:
-    if (excess_dim > 0) {
-        // build the excess sparse triangular system
-        auto excess_nnz = exec->copy_val_to_host(
-            excess_row_ptrs_full.get_const_data() + num_rows);
-        auto excess_system =
-            Csr::create(exec, dim<2>(excess_dim, excess_dim), excess_nnz);
-        auto excess_rhs = Dense::create(exec, dim<2>(excess_dim, 1));
-        auto excess_solution = Dense::create(exec, dim<2>(excess_dim, 1));
-        exec->run(isai::make_generate_excess_system(
-            lend(to_invert), lend(inverted), excess_block_ptrs.get_const_data(),
-            excess_row_ptrs_full.get_const_data(), lend(excess_system),
-            lend(excess_rhs)));
-        // solve it after transposing
-        std::unique_ptr<LinOpFactory> trs_factory;
-        if (is_general) {
-            trs_factory =
-                Gmres::build()
-                    .with_preconditioner(
-                        bj::build().with_max_block_size(32u).on(exec))
-                    .with_criteria(
-                        gko::stop::Iteration::build()
-                            .with_max_iters(excess_dim)
-                            .on(exec),
-                        gko::stop::AbsoluteResidualNorm<ValueType>::build()
-                            .with_tolerance(1e-6)
-                            .on(exec))
-                    .on(exec);
-            excess_solution->copy_from(excess_rhs.get());
-        } else if (is_lower) {
-            trs_factory = UpperTrs::build().on(exec);
-        } else {
-            trs_factory = LowerTrs::build().on(exec);
+    if (total_excess_dim > 0) {
+        bool done = false;
+        size_type block = 0;
+        while (true) {
+            // build the excess sparse triangular system
+            size_type excess_dim = 0;
+            size_type excess_start = block;
+            const auto block_offset = exec->copy_val_to_host(
+                excess_block_ptrs.get_const_data() + block);
+            const auto nnz_offset = exec->copy_val_to_host(
+                excess_row_ptrs_full.get_const_data() + block);
+            while (excess_dim < 2048 && block < num_rows) {
+                block++;
+                excess_dim = exec->copy_val_to_host(
+                                 excess_block_ptrs.get_const_data() + block) -
+                             block_offset;
+            }
+            if (excess_dim == 0) break;
+            auto excess_nnz =
+                exec->copy_val_to_host(excess_row_ptrs_full.get_const_data() +
+                                       block) -
+                nnz_offset;
+            auto excess_system =
+                Csr::create(exec, dim<2>(excess_dim, excess_dim), excess_nnz);
+            excess_system->set_strategy(
+                std::make_shared<typename Csr::classical>());
+            auto excess_rhs = Dense::create(exec, dim<2>(excess_dim, 1));
+            auto excess_solution = Dense::create(exec, dim<2>(excess_dim, 1));
+            exec->run(isai::make_generate_excess_system(
+                lend(to_invert), lend(inverted),
+                excess_block_ptrs.get_const_data(),
+                excess_row_ptrs_full.get_const_data(), lend(excess_system),
+                lend(excess_rhs), excess_start, block));
+            // solve it after transposing
+            auto system_copy = Csr::create(exec->get_master());
+            system_copy->copy_from(excess_system.get());
+            auto rhs_copy = Dense::create(exec->get_master());
+            rhs_copy->copy_from(excess_rhs.get());
+            std::unique_ptr<LinOpFactory> trs_factory;
+            if (is_general) {
+                trs_factory =
+                    Gmres::build()
+                        .with_preconditioner(
+                            bj::build().with_max_block_size(32u).on(exec))
+                        .with_criteria(
+                            gko::stop::Iteration::build()
+                                .with_max_iters(excess_dim)
+                                .on(exec),
+                            gko::stop::RelativeResidualNorm<ValueType>::build()
+                                .with_tolerance(1e-6)
+                                .on(exec))
+                        .on(exec);
+                excess_solution->copy_from(excess_rhs.get());
+                auto sol_copy = Dense::create(exec->get_master());
+                sol_copy->copy_from(excess_solution.get());
+            } else if (is_lower) {
+                trs_factory = UpperTrs::build().on(exec);
+            } else {
+                trs_factory = LowerTrs::build().on(exec);
+            }
+            trs_factory->generate(share(excess_system->transpose()))
+                ->apply(lend(excess_rhs), lend(excess_solution));
+            // and copy the results back to the original ISAI
+            exec->run(isai::make_scatter_excess_solution(
+                excess_block_ptrs.get_const_data(), lend(excess_solution),
+                lend(inverted), excess_start, block));
         }
-        trs_factory->generate(share(excess_system->transpose()))
-            ->apply(lend(excess_rhs), lend(excess_solution));
-        // and copy the results back to the original ISAI
-        exec->run(isai::make_scatter_excess_solution(
-            excess_block_ptrs.get_const_data(), lend(excess_solution),
-            lend(inverted)));
     }
 
     approximate_inverse_ = std::move(inverted);
diff --git a/core/preconditioner/isai_kernels.hpp b/core/preconditioner/isai_kernels.hpp
index 5c67c4ca7f7..db92adad459 100644
--- a/core/preconditioner/isai_kernels.hpp
+++ b/core/preconditioner/isai_kernels.hpp
@@ -66,14 +66,16 @@ namespace kernels {
         const matrix::Csr<ValueType, IndexType> *inverse,                    \
         const IndexType *excess_rhs_ptrs, const IndexType *excess_nz_ptrs,   \
         matrix::Csr<ValueType, IndexType> *excess_system,                    \
-        matrix::Dense<ValueType> *excess_rhs)
+        matrix::Dense<ValueType> *excess_rhs, const size_type excess_start,  \
+        const size_type excess_end)
 
 #define GKO_DECLARE_ISAI_SCATTER_EXCESS_SOLUTION_KERNEL(ValueType, IndexType) \
     void scatter_excess_solution(                                             \
         std::shared_ptr<const DefaultExecutor> exec,                          \
         const IndexType *excess_rhs_ptrs,                                     \
         const matrix::Dense<ValueType> *excess_solution,                      \
-        matrix::Csr<ValueType, IndexType> *inverse)
+        matrix::Csr<ValueType, IndexType> *inverse, const size_type e_start,  \
+        const size_type e_end)
 
 #define GKO_DECLARE_ALL_AS_TEMPLATES                                        \
     constexpr auto row_size_limit = 32;                                     \
diff --git a/cuda/preconditioner/isai_kernels.cu b/cuda/preconditioner/isai_kernels.cu
index bcb757aa935..fdfc1808ac9 100644
--- a/cuda/preconditioner/isai_kernels.cu
+++ b/cuda/preconditioner/isai_kernels.cu
@@ -142,12 +142,13 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor> exec,
                             const IndexType *excess_rhs_ptrs,
                             const IndexType *excess_nz_ptrs,
                             matrix::Csr<ValueType, IndexType> *excess_system,
-                            matrix::Dense<ValueType> *excess_rhs)
+                            matrix::Dense<ValueType> *excess_rhs,
+                            const size_type e_start, const size_type e_end)
 {
     const auto num_rows = input->get_size()[0];
 
     const dim3 block(default_block_size, 1, 1);
-    const dim3 grid(ceildiv(num_rows, block.x / subwarp_size), 1, 1);
+    const dim3 grid(ceildiv(e_end - e_start, block.x / subwarp_size), 1, 1);
     kernel::generate_excess_system<subwarp_size><<<grid, block>>>(
         static_cast<IndexType>(num_rows), input->get_const_row_ptrs(),
         input->get_const_col_idxs(), as_cuda_type(input->get_const_values()),
@@ -155,7 +156,7 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor> exec,
         excess_rhs_ptrs, excess_nz_ptrs, excess_system->get_row_ptrs(),
         excess_system->get_col_idxs(),
         as_cuda_type(excess_system->get_values()),
-        as_cuda_type(excess_rhs->get_values()));
+        as_cuda_type(excess_rhs->get_values()), e_start, e_end);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -166,16 +167,17 @@ template <typename ValueType, typename IndexType>
 void scatter_excess_solution(std::shared_ptr<const DefaultExecutor> exec,
                              const IndexType *excess_rhs_ptrs,
                              const matrix::Dense<ValueType> *excess_solution,
-                             matrix::Csr<ValueType, IndexType> *inverse)
+                             matrix::Csr<ValueType, IndexType> *inverse,
+                             const size_type e_start, const size_type e_end)
 {
     const auto num_rows = inverse->get_size()[0];
 
     const dim3 block(default_block_size, 1, 1);
-    const dim3 grid(ceildiv(num_rows, block.x / subwarp_size), 1, 1);
+    const dim3 grid(ceildiv(e_end - e_start, block.x / subwarp_size), 1, 1);
     kernel::copy_excess_solution<subwarp_size><<<grid, block>>>(
         static_cast<IndexType>(num_rows), inverse->get_const_row_ptrs(),
         excess_rhs_ptrs, as_cuda_type(excess_solution->get_const_values()),
-        as_cuda_type(inverse->get_values()));
+        as_cuda_type(inverse->get_values()), e_start, e_end);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
diff --git a/cuda/test/preconditioner/isai_kernels.cpp b/cuda/test/preconditioner/isai_kernels.cpp
index 5f18dd2d0d7..35d914774ac 100644
--- a/cuda/test/preconditioner/isai_kernels.cpp
+++ b/cuda/test/preconditioner/isai_kernels.cpp
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/mtx_io.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
@@ -49,6 +50,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/preconditioner/isai_kernels.hpp"
 #include "cuda/base/config.hpp"
 #include "cuda/test/utils.hpp"
+#include "matrices/config.hpp"
 
 
 namespace {
@@ -115,6 +117,21 @@ class Isai : public ::testing::Test {
         d_inverse->copy_from(inverse.get());
     }
 
+    template <typename ReadMtx>
+    std::unique_ptr<ReadMtx> read(const char *name)
+    {
+        std::ifstream mtxstream{std::string{gko::matrices::location_isai_mtxs} +
+                                name};
+        auto result = gko::read<ReadMtx>(mtxstream, ref);
+        // to avoid removing 0s, the matrices store 12345 instead
+        for (gko::size_type i = 0; i < result->get_num_stored_elements(); ++i) {
+            auto &val = result->get_values()[i];
+            if (val == static_cast<value_type>(12345.0)) {
+                val = 0;
+            }
+        }
+        return std::move(result);
+    }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::CudaExecutor> cuda;
@@ -284,10 +301,10 @@ TEST_F(Isai, CudaIsaiGenerateExcessLinverseLongIsEquivalentToRef)
 
     gko::kernels::reference::isai::generate_excess_system(
         ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
-        excess.get(), e_rhs.get());
+        excess.get(), e_rhs.get(), 0, num_rows);
     gko::kernels::cuda::isai::generate_excess_system(
         cuda, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
-        da2.get_const_data(), dexcess.get(), de_rhs.get());
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
     GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
@@ -315,10 +332,10 @@ TEST_F(Isai, CudaIsaiGenerateExcessUinverseLongIsEquivalentToRef)
 
     gko::kernels::reference::isai::generate_excess_system(
         ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
-        excess.get(), e_rhs.get());
+        excess.get(), e_rhs.get(), 0, num_rows);
     gko::kernels::cuda::isai::generate_excess_system(
         cuda, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
-        da2.get_const_data(), dexcess.get(), de_rhs.get());
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
     GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
@@ -346,10 +363,10 @@ TEST_F(Isai, CudaIsaiGenerateExcessAinverseLongIsEquivalentToRef)
 
     gko::kernels::reference::isai::generate_excess_system(
         ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
-        excess.get(), e_rhs.get());
+        excess.get(), e_rhs.get(), 0, num_rows);
     gko::kernels::cuda::isai::generate_excess_system(
         cuda, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
-        da2.get_const_data(), dexcess.get(), de_rhs.get());
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
     GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
@@ -375,9 +392,9 @@ TEST_F(Isai, CudaIsaiScatterExcessSolutionLIsEquivalentToRef)
     d_inverse->copy_from(lend(inverse));
 
     gko::kernels::reference::isai::scatter_excess_solution(
-        ref, a1.get_const_data(), e_rhs.get(), inverse.get());
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 0, num_rows);
     gko::kernels::cuda::isai::scatter_excess_solution(
-        cuda, da1.get_const_data(), de_rhs.get(), d_inverse.get());
+        cuda, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
     ASSERT_GT(e_dim, 0);
@@ -402,9 +419,9 @@ TEST_F(Isai, CudaIsaiScatterExcessSolutionUIsEquivalentToRef)
     d_inverse->copy_from(lend(inverse));
 
     gko::kernels::reference::isai::scatter_excess_solution(
-        ref, a1.get_const_data(), e_rhs.get(), inverse.get());
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 0, num_rows);
     gko::kernels::cuda::isai::scatter_excess_solution(
-        cuda, da1.get_const_data(), de_rhs.get(), d_inverse.get());
+        cuda, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
     ASSERT_GT(e_dim, 0);
@@ -429,13 +446,74 @@ TEST_F(Isai, CudaIsaiScatterExcessSolutionAIsEquivalentToRef)
     d_inverse->copy_from(lend(inverse));
 
     gko::kernels::reference::isai::scatter_excess_solution(
-        ref, a1.get_const_data(), e_rhs.get(), inverse.get());
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 0, num_rows);
     gko::kernels::cuda::isai::scatter_excess_solution(
-        cuda, da1.get_const_data(), de_rhs.get(), d_inverse.get());
+        cuda, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
     ASSERT_GT(e_dim, 0);
 }
 
 
+TEST_F(Isai, CudaLInverseIsEquivalentToRef)
+{
+    using LowerIsai = gko::preconditioner::LowerIsai<value_type, index_type>;
+    auto mtx = read<Csr>("isai_l.mtx");
+    auto d_mtx = Csr::create(cuda);
+    d_mtx->copy_from(mtx.get());
+    auto isai_factory = LowerIsai::build().on(ref);
+    auto d_isai_factory = LowerIsai::build().on(cuda);
+
+    auto isai = isai_factory->generate(gko::share(mtx));
+    auto d_isai = d_isai_factory->generate(gko::share(d_mtx));
+
+    auto inv = isai->get_approximate_inverse();
+    auto d_inv = d_isai->get_approximate_inverse();
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inv, d_inv);
+    GKO_ASSERT_MTX_NEAR(inv, d_inv, 1e-14);
+}
+
+
+TEST_F(Isai, CudaUInverseIsEquivalentToRef)
+{
+    using UpperIsai = gko::preconditioner::UpperIsai<value_type, index_type>;
+    auto mtx = read<Csr>("isai_u.mtx");
+    auto d_mtx = Csr::create(cuda);
+    d_mtx->copy_from(mtx.get());
+    auto isai_factory = UpperIsai::build().on(ref);
+    auto d_isai_factory = UpperIsai::build().on(cuda);
+
+    auto isai = isai_factory->generate(gko::share(mtx));
+    auto d_isai = d_isai_factory->generate(gko::share(d_mtx));
+
+    auto inv = isai->get_approximate_inverse();
+    auto d_inv = d_isai->get_approximate_inverse();
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inv, d_inv);
+    GKO_ASSERT_MTX_NEAR(inv, d_inv, 1e-14);
+}
+
+
+TEST_F(Isai, CudaAInverseIsEquivalentToRef)
+{
+    using GeneralIsai =
+        gko::preconditioner::GeneralIsai<value_type, index_type>;
+    auto mtx = read<Csr>("a_large.mtx");
+    auto d_mtx = Csr::create(cuda);
+    d_mtx->copy_from(mtx.get());
+    auto isai_factory = GeneralIsai::build().on(ref);
+    auto d_isai_factory = GeneralIsai::build().on(cuda);
+
+    auto isai = isai_factory->generate(gko::share(mtx));
+    auto d_isai = d_isai_factory->generate(gko::share(d_mtx));
+
+    auto inv = isai->get_approximate_inverse();
+    auto d_inv = d_isai->get_approximate_inverse();
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inv, d_inv);
+    GKO_ASSERT_MTX_NEAR(inv, d_inv, 1e-14);
+}
+
+
 }  // namespace
diff --git a/dpcpp/preconditioner/isai_kernels.dp.cpp b/dpcpp/preconditioner/isai_kernels.dp.cpp
index ac51807b839..34a2a525542 100644
--- a/dpcpp/preconditioner/isai_kernels.dp.cpp
+++ b/dpcpp/preconditioner/isai_kernels.dp.cpp
@@ -98,23 +98,27 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void generate_excess_system(
-    std::shared_ptr<const DefaultExecutor>,
-    const matrix::Csr<ValueType, IndexType> *input,
-    const matrix::Csr<ValueType, IndexType> *inverse,
-    const IndexType *excess_rhs_ptrs, const IndexType *excess_nz_ptrs,
-    matrix::Csr<ValueType, IndexType> *excess_system,
-    matrix::Dense<ValueType> *excess_rhs) GKO_NOT_IMPLEMENTED;
+void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
+                            const matrix::Csr<ValueType, IndexType> *input,
+                            const matrix::Csr<ValueType, IndexType> *inverse,
+                            const IndexType *excess_rhs_ptrs,
+                            const IndexType *excess_nz_ptrs,
+                            matrix::Csr<ValueType, IndexType> *excess_system,
+                            matrix::Dense<ValueType> *excess_rhs,
+                            const size_type e_start,
+                            const size_type e_end) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void scatter_excess_solution(
-    std::shared_ptr<const DefaultExecutor>, const IndexType *excess_block_ptrs,
-    const matrix::Dense<ValueType> *excess_solution,
-    matrix::Csr<ValueType, IndexType> *inverse) GKO_NOT_IMPLEMENTED;
+void scatter_excess_solution(std::shared_ptr<const DefaultExecutor>,
+                             const IndexType *excess_block_ptrs,
+                             const matrix::Dense<ValueType> *excess_solution,
+                             matrix::Csr<ValueType, IndexType> *inverse,
+                             const size_type e_start,
+                             const size_type e_end) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_SCATTER_EXCESS_SOLUTION_KERNEL);
diff --git a/hip/preconditioner/isai_kernels.hip.cpp b/hip/preconditioner/isai_kernels.hip.cpp
index 66591644624..c25c8e2672d 100644
--- a/hip/preconditioner/isai_kernels.hip.cpp
+++ b/hip/preconditioner/isai_kernels.hip.cpp
@@ -128,7 +128,8 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor> exec,
                             const IndexType *excess_rhs_ptrs,
                             const IndexType *excess_nz_ptrs,
                             matrix::Csr<ValueType, IndexType> *excess_system,
-                            matrix::Dense<ValueType> *excess_rhs)
+                            matrix::Dense<ValueType> *excess_rhs,
+                            const size_type e_start, const size_type e_end)
 {
     const auto num_rows = input->get_size()[0];
 
@@ -153,9 +154,10 @@ template <typename ValueType, typename IndexType>
 void scatter_excess_solution(std::shared_ptr<const DefaultExecutor> exec,
                              const IndexType *excess_rhs_ptrs,
                              const matrix::Dense<ValueType> *excess_solution,
-                             matrix::Csr<ValueType, IndexType> *inverse)
+                             matrix::Csr<ValueType, IndexType> *inverse,
+                             const size_type e_start, const size_type e_end)
 {
-    const auto num_rows = inverse->get_size()[0];
+    // const auto num_rows = inverse->get_size()[0];
 
     const dim3 block(default_block_size, 1, 1);
     const dim3 grid(ceildiv(num_rows, block.x / subwarp_size), 1, 1);
diff --git a/omp/preconditioner/isai_kernels.cpp b/omp/preconditioner/isai_kernels.cpp
index 192af7089c2..d9a7e97f76b 100644
--- a/omp/preconditioner/isai_kernels.cpp
+++ b/omp/preconditioner/isai_kernels.cpp
@@ -346,7 +346,8 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
                             const IndexType *excess_rhs_ptrs,
                             const IndexType *excess_nz_ptrs,
                             matrix::Csr<ValueType, IndexType> *excess_system,
-                            matrix::Dense<ValueType> *excess_rhs)
+                            matrix::Dense<ValueType> *excess_rhs,
+                            const size_type e_start, const size_type e_end)
 {
     const auto num_rows = input->get_size()[0];
     const auto m_row_ptrs = input->get_const_row_ptrs();
@@ -361,7 +362,7 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
     auto e_rhs = excess_rhs->get_values();
 
 #pragma omp parallel for
-    for (size_type row = 0; row < num_rows; ++row) {
+    for (size_type row = e_start; row < e_end; ++row) {
         const auto i_begin = i_row_ptrs[row];
         const auto i_size = i_row_ptrs[row + 1] - i_begin;
         // first row index of the sparse block in the excess system
@@ -406,16 +407,19 @@ template <typename ValueType, typename IndexType>
 void scatter_excess_solution(std::shared_ptr<const DefaultExecutor>,
                              const IndexType *excess_block_ptrs,
                              const matrix::Dense<ValueType> *excess_solution,
-                             matrix::Csr<ValueType, IndexType> *inverse)
+                             matrix::Csr<ValueType, IndexType> *inverse,
+                             const size_type e_start, const size_type e_end)
 {
-    const auto num_rows = inverse->get_size()[0];
     auto excess_values = excess_solution->get_const_values();
     auto values = inverse->get_values();
     auto row_ptrs = inverse->get_const_row_ptrs();
+    auto offset = excess_block_ptrs[e_start];
 #pragma omp parallel for
-    for (size_type row = 0; row < num_rows; ++row) {
-        const auto excess_begin = excess_values + excess_block_ptrs[row];
-        const auto excess_end = excess_values + excess_block_ptrs[row + 1];
+    for (size_type row = e_start; row < e_end; ++row) {
+        const auto excess_begin =
+            excess_values + excess_block_ptrs[row] - offset;
+        const auto excess_end =
+            excess_values + excess_block_ptrs[row + 1] - offset;
         auto values_begin = values + row_ptrs[row];
         std::copy(excess_begin, excess_end, values_begin);
     }
diff --git a/omp/test/preconditioner/isai_kernels.cpp b/omp/test/preconditioner/isai_kernels.cpp
index 879f17c1198..7423721f903 100644
--- a/omp/test/preconditioner/isai_kernels.cpp
+++ b/omp/test/preconditioner/isai_kernels.cpp
@@ -274,10 +274,10 @@ TEST_F(Isai, OmpIsaiGenerateExcessLinverseLongIsEquivalentToRef)
 
     gko::kernels::reference::isai::generate_excess_system(
         ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
-        excess.get(), e_rhs.get());
+        excess.get(), e_rhs.get(), 0, num_rows);
     gko::kernels::omp::isai::generate_excess_system(
         omp, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
-        da2.get_const_data(), dexcess.get(), de_rhs.get());
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
     GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
@@ -305,10 +305,10 @@ TEST_F(Isai, OmpIsaiGenerateExcessUinverseLongIsEquivalentToRef)
 
     gko::kernels::reference::isai::generate_excess_system(
         ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
-        excess.get(), e_rhs.get());
+        excess.get(), e_rhs.get(), 0, num_rows);
     gko::kernels::omp::isai::generate_excess_system(
         omp, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
-        da2.get_const_data(), dexcess.get(), de_rhs.get());
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
     GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
@@ -336,10 +336,10 @@ TEST_F(Isai, OmpIsaiGenerateExcessAinverseLongIsEquivalentToRef)
 
     gko::kernels::reference::isai::generate_excess_system(
         ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
-        excess.get(), e_rhs.get());
+        excess.get(), e_rhs.get(), 0, num_rows);
     gko::kernels::omp::isai::generate_excess_system(
         omp, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
-        da2.get_const_data(), dexcess.get(), de_rhs.get());
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
     GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
@@ -365,9 +365,9 @@ TEST_F(Isai, OmpIsaiScatterExcessSolutionLIsEquivalentToRef)
     d_inverse->copy_from(lend(inverse));
 
     gko::kernels::reference::isai::scatter_excess_solution(
-        ref, a1.get_const_data(), e_rhs.get(), inverse.get());
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 0, num_rows);
     gko::kernels::omp::isai::scatter_excess_solution(
-        omp, da1.get_const_data(), de_rhs.get(), d_inverse.get());
+        omp, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
     ASSERT_GT(e_dim, 0);
@@ -392,9 +392,9 @@ TEST_F(Isai, OmpIsaiScatterExcessSolutionUIsEquivalentToRef)
     d_inverse->copy_from(lend(inverse));
 
     gko::kernels::reference::isai::scatter_excess_solution(
-        ref, a1.get_const_data(), e_rhs.get(), inverse.get());
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 0, num_rows);
     gko::kernels::omp::isai::scatter_excess_solution(
-        omp, da1.get_const_data(), de_rhs.get(), d_inverse.get());
+        omp, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
     ASSERT_GT(e_dim, 0);
@@ -419,9 +419,9 @@ TEST_F(Isai, OmpIsaiScatterExcessSolutionAIsEquivalentToRef)
     d_inverse->copy_from(lend(inverse));
 
     gko::kernels::reference::isai::scatter_excess_solution(
-        ref, a1.get_const_data(), e_rhs.get(), inverse.get());
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 0, num_rows);
     gko::kernels::omp::isai::scatter_excess_solution(
-        omp, da1.get_const_data(), de_rhs.get(), d_inverse.get());
+        omp, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
     ASSERT_GT(e_dim, 0);
diff --git a/reference/preconditioner/isai_kernels.cpp b/reference/preconditioner/isai_kernels.cpp
index 6949ce0bc4f..e44a4911308 100644
--- a/reference/preconditioner/isai_kernels.cpp
+++ b/reference/preconditioner/isai_kernels.cpp
@@ -330,7 +330,8 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
                             const matrix::Csr<ValueType, IndexType> *inverse,
                             const IndexType *, const IndexType *,
                             matrix::Csr<ValueType, IndexType> *excess_system,
-                            matrix::Dense<ValueType> *excess_rhs)
+                            matrix::Dense<ValueType> *excess_rhs,
+                            const size_type e_start, const size_type e_end)
 {
     const auto num_rows = input->get_size()[0];
     const auto m_row_ptrs = input->get_const_row_ptrs();
@@ -346,7 +347,7 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
     IndexType e_block_begin{};
     IndexType e_nz{};
 
-    for (size_type row = 0; row < num_rows; ++row) {
+    for (size_type row = e_start; row < e_end; ++row) {
         const auto i_begin = i_row_ptrs[row];
         const auto i_size = i_row_ptrs[row + 1] - i_begin;
 
@@ -388,15 +389,18 @@ template <typename ValueType, typename IndexType>
 void scatter_excess_solution(std::shared_ptr<const DefaultExecutor>,
                              const IndexType *excess_block_ptrs,
                              const matrix::Dense<ValueType> *excess_solution,
-                             matrix::Csr<ValueType, IndexType> *inverse)
+                             matrix::Csr<ValueType, IndexType> *inverse,
+                             const size_type e_start, const size_type e_end)
 {
-    const auto num_rows = inverse->get_size()[0];
     auto excess_values = excess_solution->get_const_values();
     auto values = inverse->get_values();
     auto row_ptrs = inverse->get_const_row_ptrs();
-    for (size_type row = 0; row < num_rows; ++row) {
-        const auto excess_begin = excess_values + excess_block_ptrs[row];
-        const auto excess_end = excess_values + excess_block_ptrs[row + 1];
+    auto offset = excess_block_ptrs[e_start];
+    for (size_type row = e_start; row < e_end; ++row) {
+        const auto excess_begin =
+            excess_values + excess_block_ptrs[row] - offset;
+        const auto excess_end =
+            excess_values + excess_block_ptrs[row + 1] - offset;
         auto values_begin = values + row_ptrs[row];
         std::copy(excess_begin, excess_end, values_begin);
     }
diff --git a/reference/test/preconditioner/isai_kernels.cpp b/reference/test/preconditioner/isai_kernels.cpp
index a2e2f15be09..e95da3d4dc5 100644
--- a/reference/test/preconditioner/isai_kernels.cpp
+++ b/reference/test/preconditioner/isai_kernels.cpp
@@ -437,7 +437,7 @@ TYPED_TEST(Isai, KernelGenerateExcessLLongrow)
     gko::kernels::reference::isai::generate_excess_system(
         this->exec, lend(this->l_csr_longrow), lend(this->l_csr_longrow),
         a1.get_const_data(), a2.get_const_data(), lend(result),
-        lend(result_rhs));
+        lend(result_rhs), 0, num_rows);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(result, this->l_csr_longrow_e);
     GKO_ASSERT_MTX_NEAR(result, this->l_csr_longrow_e, 0);
@@ -633,7 +633,7 @@ TYPED_TEST(Isai, KernelGenerateExcessULongrow)
     gko::kernels::reference::isai::generate_excess_system(
         this->exec, lend(this->u_csr_longrow), lend(this->u_csr_longrow),
         a1.get_const_data(), a2.get_const_data(), lend(result),
-        lend(result_rhs));
+        lend(result_rhs), 0, num_rows);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(result, this->u_csr_longrow_e);
     GKO_ASSERT_MTX_NEAR(result, this->u_csr_longrow_e, 0);
@@ -661,7 +661,7 @@ TYPED_TEST(Isai, KernelScatterExcessSolution)
                              I<value_type>{11, 12, 13, 14, 15, 16, 17}, 1);
 
     gko::kernels::reference::isai::scatter_excess_solution(
-        this->exec, ptrs.get_const_data(), sol.get(), mtx.get());
+        this->exec, ptrs.get_const_data(), sol.get(), mtx.get(), 0, 6);
 
     GKO_ASSERT_MTX_NEAR(mtx, expect, 0);
 }
@@ -676,7 +676,8 @@ TYPED_TEST(Isai, ReturnsCorrectInverseLargeA)
     auto a_inv = isai->get_approximate_inverse();
 
     GKO_ASSERT_MTX_EQ_SPARSITY(a_inv, this->a_csr_large_inv);
-    GKO_ASSERT_MTX_NEAR(a_inv, this->a_csr_large_inv, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(a_inv, this->a_csr_large_inv,
+                        10 * r<value_type>::value);
 }
 
 

From fd7527806f4ae310b44d63e435f23373c013204d Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Tue, 5 Jan 2021 16:36:28 +0100
Subject: [PATCH 0510/1008] Write tests for general ISAI

---
 core/preconditioner/isai.cpp                  |     7 +-
 core/test/preconditioner/isai.cpp             |    62 +
 cuda/test/preconditioner/isai_kernels.cpp     |    83 +-
 hip/preconditioner/isai_kernels.hip.cpp       |    26 +-
 hip/test/preconditioner/isai_kernels.hip.cpp  |   122 +-
 include/ginkgo/core/preconditioner/isai.hpp   |    35 +-
 matrices/CMakeLists.txt                       |     7 +-
 matrices/test/{a_large.mtx => isai_a.mtx}     |     0
 matrices/test/isai_a_excess.mtx               | 14886 ++++++++++++++++
 matrices/test/isai_a_excess_rhs.mtx           |   124 +
 .../test/{a_large_inv.mtx => isai_a_inv.mtx}  |     0
 matrices/test/isai_a_inv_partial.mtx          |   416 +
 omp/test/preconditioner/isai_kernels.cpp      |    11 +-
 reference/preconditioner/isai_kernels.cpp     |     1 -
 .../test/preconditioner/isai_kernels.cpp      |   274 +-
 15 files changed, 15935 insertions(+), 119 deletions(-)
 rename matrices/test/{a_large.mtx => isai_a.mtx} (100%)
 create mode 100644 matrices/test/isai_a_excess.mtx
 create mode 100644 matrices/test/isai_a_excess_rhs.mtx
 rename matrices/test/{a_large_inv.mtx => isai_a_inv.mtx} (100%)
 create mode 100644 matrices/test/isai_a_inv_partial.mtx

diff --git a/core/preconditioner/isai.cpp b/core/preconditioner/isai.cpp
index 884888ca93d..52693e84750 100644
--- a/core/preconditioner/isai.cpp
+++ b/core/preconditioner/isai.cpp
@@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <functional>
-#include <iostream>
 #include <memory>
 #include <type_traits>
 
@@ -115,7 +114,8 @@ std::shared_ptr<Csr> extend_sparsity(std::shared_ptr<const Executor> &exec,
 
 template <isai_type IsaiType, typename ValueType, typename IndexType>
 void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
-    std::shared_ptr<const LinOp> input, bool skip_sorting, int power)
+    std::shared_ptr<const LinOp> input, bool skip_sorting, int power,
+    int excess_limit)
 {
     using Dense = matrix::Dense<ValueType>;
     using LowerTrs = solver::LowerTrs<ValueType, IndexType>;
@@ -129,6 +129,7 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
     auto num_rows = inverted->get_size()[0];
     auto is_lower = IsaiType == isai_type::lower;
     auto is_general = IsaiType == isai_type::general;
+    auto excess_lim = excess_limit == 0 ? num_rows : excess_limit;
 
     // This stores the beginning of the RHS for the sparse block associated with
     // each row of inverted_l
@@ -161,7 +162,7 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
                 excess_block_ptrs.get_const_data() + block);
             const auto nnz_offset = exec->copy_val_to_host(
                 excess_row_ptrs_full.get_const_data() + block);
-            while (excess_dim < 2048 && block < num_rows) {
+            while (excess_dim < excess_lim && block < num_rows) {
                 block++;
                 excess_dim = exec->copy_val_to_host(
                                  excess_block_ptrs.get_const_data() + block) -
diff --git a/core/test/preconditioner/isai.cpp b/core/test/preconditioner/isai.cpp
index 350055c6b79..823e8e66a0b 100644
--- a/core/test/preconditioner/isai.cpp
+++ b/core/test/preconditioner/isai.cpp
@@ -71,17 +71,21 @@ class IsaiFactory : public ::testing::Test {
         typename std::tuple_element<0, decltype(ValueIndexType())>::type;
     using index_type =
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using GeneralIsai =
+        gko::preconditioner::GeneralIsai<value_type, index_type>;
     using LowerIsai = gko::preconditioner::LowerIsai<value_type, index_type>;
     using UpperIsai = gko::preconditioner::UpperIsai<value_type, index_type>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
 
     IsaiFactory()
         : exec(gko::ReferenceExecutor::create()),
+          general_isai_factory(GeneralIsai::build().on(exec)),
           lower_isai_factory(LowerIsai::build().on(exec)),
           upper_isai_factory(UpperIsai::build().on(exec))
     {}
 
     std::shared_ptr<const gko::Executor> exec;
+    std::unique_ptr<typename GeneralIsai::Factory> general_isai_factory;
     std::unique_ptr<typename LowerIsai::Factory> lower_isai_factory;
     std::unique_ptr<typename UpperIsai::Factory> upper_isai_factory;
 };
@@ -91,6 +95,7 @@ TYPED_TEST_SUITE(IsaiFactory, gko::test::ValueIndexTypes);
 
 TYPED_TEST(IsaiFactory, KnowsItsExecutor)
 {
+    ASSERT_EQ(this->general_isai_factory->get_executor(), this->exec);
     ASSERT_EQ(this->lower_isai_factory->get_executor(), this->exec);
     ASSERT_EQ(this->upper_isai_factory->get_executor(), this->exec);
 }
@@ -98,14 +103,18 @@ TYPED_TEST(IsaiFactory, KnowsItsExecutor)
 
 TYPED_TEST(IsaiFactory, SetsSkipSortingCorrectly)
 {
+    using GeneralIsai = typename TestFixture::GeneralIsai;
     using LowerIsai = typename TestFixture::LowerIsai;
     using UpperIsai = typename TestFixture::UpperIsai;
 
+    auto a_isai_factory =
+        GeneralIsai::build().with_skip_sorting(true).on(this->exec);
     auto l_isai_factory =
         LowerIsai::build().with_skip_sorting(true).on(this->exec);
     auto u_isai_factory =
         UpperIsai::build().with_skip_sorting(true).on(this->exec);
 
+    ASSERT_EQ(a_isai_factory->get_parameters().skip_sorting, true);
     ASSERT_EQ(l_isai_factory->get_parameters().skip_sorting, true);
     ASSERT_EQ(u_isai_factory->get_parameters().skip_sorting, true);
 }
@@ -113,6 +122,7 @@ TYPED_TEST(IsaiFactory, SetsSkipSortingCorrectly)
 
 TYPED_TEST(IsaiFactory, SetsDefaultSkipSortingCorrectly)
 {
+    ASSERT_EQ(this->general_isai_factory->get_parameters().skip_sorting, false);
     ASSERT_EQ(this->lower_isai_factory->get_parameters().skip_sorting, false);
     ASSERT_EQ(this->upper_isai_factory->get_parameters().skip_sorting, false);
 }
@@ -120,14 +130,18 @@ TYPED_TEST(IsaiFactory, SetsDefaultSkipSortingCorrectly)
 
 TYPED_TEST(IsaiFactory, SetsSparsityPowerCorrectly)
 {
+    using GeneralIsai = typename TestFixture::GeneralIsai;
     using LowerIsai = typename TestFixture::LowerIsai;
     using UpperIsai = typename TestFixture::UpperIsai;
 
+    auto a_isai_factory =
+        GeneralIsai::build().with_sparsity_power(2).on(this->exec);
     auto l_isai_factory =
         LowerIsai::build().with_sparsity_power(2).on(this->exec);
     auto u_isai_factory =
         UpperIsai::build().with_sparsity_power(2).on(this->exec);
 
+    ASSERT_EQ(a_isai_factory->get_parameters().sparsity_power, 2);
     ASSERT_EQ(l_isai_factory->get_parameters().sparsity_power, 2);
     ASSERT_EQ(u_isai_factory->get_parameters().sparsity_power, 2);
 }
@@ -135,11 +149,49 @@ TYPED_TEST(IsaiFactory, SetsSparsityPowerCorrectly)
 
 TYPED_TEST(IsaiFactory, SetsDefaultSparsityPowerCorrectly)
 {
+    ASSERT_EQ(this->general_isai_factory->get_parameters().sparsity_power, 1);
     ASSERT_EQ(this->lower_isai_factory->get_parameters().sparsity_power, 1);
     ASSERT_EQ(this->upper_isai_factory->get_parameters().sparsity_power, 1);
 }
 
 
+TYPED_TEST(IsaiFactory, SetsExcessLimitCorrectly)
+{
+    using GeneralIsai = typename TestFixture::GeneralIsai;
+    using LowerIsai = typename TestFixture::LowerIsai;
+    using UpperIsai = typename TestFixture::UpperIsai;
+
+    auto a_isai_factory =
+        GeneralIsai::build().with_excess_limit(1024).on(this->exec);
+    auto l_isai_factory =
+        LowerIsai::build().with_excess_limit(1024).on(this->exec);
+    auto u_isai_factory =
+        UpperIsai::build().with_excess_limit(1024).on(this->exec);
+
+    ASSERT_EQ(a_isai_factory->get_parameters().excess_limit, 1024);
+    ASSERT_EQ(l_isai_factory->get_parameters().excess_limit, 1024);
+    ASSERT_EQ(u_isai_factory->get_parameters().excess_limit, 1024);
+}
+
+
+TYPED_TEST(IsaiFactory, SetsDefaultExcessLimitCorrectly)
+{
+    ASSERT_EQ(this->general_isai_factory->get_parameters().excess_limit, 0);
+    ASSERT_EQ(this->lower_isai_factory->get_parameters().excess_limit, 0);
+    ASSERT_EQ(this->upper_isai_factory->get_parameters().excess_limit, 0);
+}
+
+
+TYPED_TEST(IsaiFactory, ThrowsWrongDimensionA)
+{
+    using Csr = typename TestFixture::Csr;
+    auto mtx = Csr::create(this->exec, gko::dim<2>{1, 2}, 1);
+
+    ASSERT_THROW(this->general_isai_factory->generate(gko::share(mtx)),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(IsaiFactory, ThrowsWrongDimensionL)
 {
     using Csr = typename TestFixture::Csr;
@@ -160,6 +212,16 @@ TYPED_TEST(IsaiFactory, ThrowsWrongDimensionU)
 }
 
 
+TYPED_TEST(IsaiFactory, ThrowsNoConversionCsrA)
+{
+    using Csr = typename TestFixture::Csr;
+    auto mtx = DummyOperator::create(this->exec, gko::dim<2>{2, 2});
+
+    ASSERT_THROW(this->general_isai_factory->generate(gko::share(mtx)),
+                 gko::NotSupported);
+}
+
+
 TYPED_TEST(IsaiFactory, ThrowsNoConversionCsrL)
 {
     using Csr = typename TestFixture::Csr;
diff --git a/cuda/test/preconditioner/isai_kernels.cpp b/cuda/test/preconditioner/isai_kernels.cpp
index 35d914774ac..4c891666ea8 100644
--- a/cuda/test/preconditioner/isai_kernels.cpp
+++ b/cuda/test/preconditioner/isai_kernels.cpp
@@ -33,7 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/preconditioner/isai.hpp>
 
 
-#include <iostream>
 #include <random>
 
 #include <gtest/gtest.h>
@@ -95,15 +94,10 @@ class Isai : public ::testing::Test {
         auto val_dist = std::uniform_real_distribution<value_type>(-1., 1.);
         mtx = Csr::create(ref);
         if (type == matrix_type::general) {
-            auto lower_mtx = gko::test::generate_random_triangular_matrix<Csr>(
-                n, n, true, true, nz_dist, val_dist, rand_engine, ref,
-                gko::dim<2>{n, n});
-            mtx = gko::test::generate_random_triangular_matrix<Csr>(
-                n, n, false, false, nz_dist, val_dist, rand_engine, ref,
-                gko::dim<2>{n, n});
-            auto one_op = gko::initialize<Dense>({gko::one<value_type>()}, ref);
-            auto id = gko::matrix::Identity<>::create(ref, gko::dim<2>{n, n});
-            lower_mtx->apply(one_op.get(), id.get(), one_op.get(), mtx.get());
+            auto dense_mtx = gko::test::generate_random_matrix<Dense>(
+                n, n, nz_dist, val_dist, rand_engine, ref, gko::dim<2>{n, n});
+            ensure_diagonal(dense_mtx.get());
+            mtx->copy_from(dense_mtx.get());
         } else {
             mtx = gko::test::generate_random_triangular_matrix<Csr>(
                 n, n, true, for_lower_tm, nz_dist, val_dist, rand_engine, ref,
@@ -133,6 +127,14 @@ class Isai : public ::testing::Test {
         return std::move(result);
     }
 
+    void ensure_diagonal(Dense *mtx)
+    {
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            mtx->at(i, i) = gko::one<Dense::value_type>();
+        }
+    }
+
+
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::CudaExecutor> cuda;
 
@@ -455,65 +457,4 @@ TEST_F(Isai, CudaIsaiScatterExcessSolutionAIsEquivalentToRef)
 }
 
 
-TEST_F(Isai, CudaLInverseIsEquivalentToRef)
-{
-    using LowerIsai = gko::preconditioner::LowerIsai<value_type, index_type>;
-    auto mtx = read<Csr>("isai_l.mtx");
-    auto d_mtx = Csr::create(cuda);
-    d_mtx->copy_from(mtx.get());
-    auto isai_factory = LowerIsai::build().on(ref);
-    auto d_isai_factory = LowerIsai::build().on(cuda);
-
-    auto isai = isai_factory->generate(gko::share(mtx));
-    auto d_isai = d_isai_factory->generate(gko::share(d_mtx));
-
-    auto inv = isai->get_approximate_inverse();
-    auto d_inv = d_isai->get_approximate_inverse();
-
-    GKO_ASSERT_MTX_EQ_SPARSITY(inv, d_inv);
-    GKO_ASSERT_MTX_NEAR(inv, d_inv, 1e-14);
-}
-
-
-TEST_F(Isai, CudaUInverseIsEquivalentToRef)
-{
-    using UpperIsai = gko::preconditioner::UpperIsai<value_type, index_type>;
-    auto mtx = read<Csr>("isai_u.mtx");
-    auto d_mtx = Csr::create(cuda);
-    d_mtx->copy_from(mtx.get());
-    auto isai_factory = UpperIsai::build().on(ref);
-    auto d_isai_factory = UpperIsai::build().on(cuda);
-
-    auto isai = isai_factory->generate(gko::share(mtx));
-    auto d_isai = d_isai_factory->generate(gko::share(d_mtx));
-
-    auto inv = isai->get_approximate_inverse();
-    auto d_inv = d_isai->get_approximate_inverse();
-
-    GKO_ASSERT_MTX_EQ_SPARSITY(inv, d_inv);
-    GKO_ASSERT_MTX_NEAR(inv, d_inv, 1e-14);
-}
-
-
-TEST_F(Isai, CudaAInverseIsEquivalentToRef)
-{
-    using GeneralIsai =
-        gko::preconditioner::GeneralIsai<value_type, index_type>;
-    auto mtx = read<Csr>("a_large.mtx");
-    auto d_mtx = Csr::create(cuda);
-    d_mtx->copy_from(mtx.get());
-    auto isai_factory = GeneralIsai::build().on(ref);
-    auto d_isai_factory = GeneralIsai::build().on(cuda);
-
-    auto isai = isai_factory->generate(gko::share(mtx));
-    auto d_isai = d_isai_factory->generate(gko::share(d_mtx));
-
-    auto inv = isai->get_approximate_inverse();
-    auto d_inv = d_isai->get_approximate_inverse();
-
-    GKO_ASSERT_MTX_EQ_SPARSITY(inv, d_inv);
-    GKO_ASSERT_MTX_NEAR(inv, d_inv, 1e-14);
-}
-
-
 }  // namespace
diff --git a/hip/preconditioner/isai_kernels.hip.cpp b/hip/preconditioner/isai_kernels.hip.cpp
index c25c8e2672d..216b361071f 100644
--- a/hip/preconditioner/isai_kernels.hip.cpp
+++ b/hip/preconditioner/isai_kernels.hip.cpp
@@ -115,7 +115,23 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
                               const matrix::Csr<ValueType, IndexType> *input,
                               matrix::Csr<ValueType, IndexType> *inverse,
                               IndexType *excess_rhs_ptrs,
-                              IndexType *excess_nz_ptrs) GKO_NOT_IMPLEMENTED;
+                              IndexType *excess_nz_ptrs)
+{
+    const auto num_rows = input->get_size()[0];
+
+    const dim3 block(default_block_size, 1, 1);
+    const dim3 grid(ceildiv(num_rows, block.x / subwarp_size), 1, 1);
+    hipLaunchKernelGGL(
+        HIP_KERNEL_NAME(
+            kernel::generate_general_inverse<subwarp_size, subwarps_per_block>),
+        grid, block, 0, 0, static_cast<IndexType>(num_rows),
+        input->get_const_row_ptrs(), input->get_const_col_idxs(),
+        as_hip_type(input->get_const_values()), inverse->get_row_ptrs(),
+        inverse->get_col_idxs(), as_hip_type(inverse->get_values()),
+        excess_rhs_ptrs, excess_nz_ptrs);
+    components::prefix_sum(exec, excess_rhs_ptrs, num_rows + 1);
+    components::prefix_sum(exec, excess_nz_ptrs, num_rows + 1);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_GENERAL_INVERSE_KERNEL);
@@ -134,7 +150,7 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor> exec,
     const auto num_rows = input->get_size()[0];
 
     const dim3 block(default_block_size, 1, 1);
-    const dim3 grid(ceildiv(num_rows, block.x / subwarp_size), 1, 1);
+    const dim3 grid(ceildiv(e_end - e_start, block.x / subwarp_size), 1, 1);
     hipLaunchKernelGGL(
         HIP_KERNEL_NAME(kernel::generate_excess_system<subwarp_size>), grid,
         block, 0, 0, static_cast<IndexType>(num_rows),
@@ -143,7 +159,7 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor> exec,
         inverse->get_const_col_idxs(), excess_rhs_ptrs, excess_nz_ptrs,
         excess_system->get_row_ptrs(), excess_system->get_col_idxs(),
         as_hip_type(excess_system->get_values()),
-        as_hip_type(excess_rhs->get_values()));
+        as_hip_type(excess_rhs->get_values()), e_start, e_end);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -160,13 +176,13 @@ void scatter_excess_solution(std::shared_ptr<const DefaultExecutor> exec,
     // const auto num_rows = inverse->get_size()[0];
 
     const dim3 block(default_block_size, 1, 1);
-    const dim3 grid(ceildiv(num_rows, block.x / subwarp_size), 1, 1);
+    const dim3 grid(ceildiv(e_end - e_start, block.x / subwarp_size), 1, 1);
     hipLaunchKernelGGL(
         HIP_KERNEL_NAME(kernel::copy_excess_solution<subwarp_size>), grid,
         block, 0, 0, static_cast<IndexType>(num_rows),
         inverse->get_const_row_ptrs(), excess_rhs_ptrs,
         as_hip_type(excess_solution->get_const_values()),
-        as_hip_type(inverse->get_values()));
+        as_hip_type(inverse->get_values()), e_start, e_end);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
diff --git a/hip/test/preconditioner/isai_kernels.hip.cpp b/hip/test/preconditioner/isai_kernels.hip.cpp
index 7db20b3e22c..dd640ac20d6 100644
--- a/hip/test/preconditioner/isai_kernels.hip.cpp
+++ b/hip/test/preconditioner/isai_kernels.hip.cpp
@@ -91,9 +91,16 @@ class Isai : public ::testing::Test {
         auto nz_dist = std::uniform_int_distribution<index_type>(1, row_limit);
         auto val_dist = std::uniform_real_distribution<value_type>(-1., 1.);
         mtx = Csr::create(ref);
-        mtx = gko::test::generate_random_triangular_matrix<Csr>(
-            n, n, true, for_lower_tm, nz_dist, val_dist, rand_engine, ref,
-            gko::dim<2>{n, n});
+        if (type == matrix_type::general) {
+            auto dense_mtx = gko::test::generate_random_matrix<Dense>(
+                n, n, nz_dist, val_dist, rand_engine, ref, gko::dim<2>{n, n});
+            ensure_diagonal(dense_mtx.get());
+            mtx->copy_from(dense_mtx.get());
+        } else {
+            mtx = gko::test::generate_random_triangular_matrix<Csr>(
+                n, n, true, for_lower_tm, nz_dist, val_dist, rand_engine, ref,
+                gko::dim<2>{n, n});
+        }
         inverse = clone_allocations(mtx.get());
 
         d_mtx = Csr::create(hip);
@@ -102,6 +109,13 @@ class Isai : public ::testing::Test {
         d_inverse->copy_from(inverse.get());
     }
 
+    void ensure_diagonal(Dense *mtx)
+    {
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            mtx->at(i, i) = gko::one<Dense::value_type>();
+        }
+    }
+
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::HipExecutor> hip;
@@ -162,6 +176,28 @@ TEST_F(Isai, HipIsaiGenerateUinverseShortIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, HipIsaiGenerateAinverseShortIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 615, 15);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::Array<index_type> da1(hip, num_rows + 1);
+    auto da2 = da1;
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+    gko::kernels::hip::isai::generate_general_inverse(
+        hip, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, r<value_type>::value);
+    GKO_ASSERT_ARRAY_EQ(a1, da1);
+    GKO_ASSERT_ARRAY_EQ(a2, da2);
+    ASSERT_EQ(a1.get_const_data()[num_rows], 0);
+}
+
+
 TEST_F(Isai, HipIsaiGenerateLinverseLongIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 554, 64);
@@ -208,6 +244,28 @@ TEST_F(Isai, HipIsaiGenerateUinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, HipIsaiGenerateAinverseLongIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 695, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::Array<index_type> da1(hip, num_rows + 1);
+    auto da2 = da1;
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+    gko::kernels::hip::isai::generate_general_inverse(
+        hip, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 10 * r<value_type>::value);
+    GKO_ASSERT_ARRAY_EQ(a1, da1);
+    GKO_ASSERT_ARRAY_EQ(a2, da2);
+    ASSERT_GT(a1.get_const_data()[num_rows], 0);
+}
+
+
 TEST_F(Isai, HipIsaiGenerateExcessLinverseLongIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 518, 40);
@@ -270,6 +328,37 @@ TEST_F(Isai, HipIsaiGenerateExcessUinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, HipIsaiGenerateExcessAinverseLongIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 100, 51);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+    gko::Array<index_type> da1(hip, a1);
+    gko::Array<index_type> da2(hip, a2);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_nnz = a2.get_data()[num_rows];
+    auto excess = Csr::create(ref, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    auto dexcess = Csr::create(hip, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto de_rhs = Dense::create(hip, gko::dim<2>(e_dim, 1));
+
+    gko::kernels::reference::isai::generate_excess_system(
+        ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
+        excess.get(), e_rhs.get(), 0, num_rows);
+    gko::kernels::hip::isai::generate_excess_system(
+        hip, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 0, num_rows);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
+    GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
 TEST_F(Isai, HipIsaiScatterExcessSolutionLIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 572, 52);
@@ -323,4 +412,31 @@ TEST_F(Isai, HipIsaiScatterExcessSolutionUIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, HipIsaiScatterExcessSolutionAIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 702, 45);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+    gko::Array<index_type> da1(hip, a1);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(hip);
+    de_rhs->copy_from(lend(e_rhs));
+    // overwrite -1 values with inverse
+    d_inverse->copy_from(lend(inverse));
+
+    gko::kernels::reference::isai::scatter_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 0, num_rows);
+    gko::kernels::hip::isai::scatter_excess_solution(
+        hip, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 0, num_rows);
+
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
 }  // namespace
diff --git a/include/ginkgo/core/preconditioner/isai.hpp b/include/ginkgo/core/preconditioner/isai.hpp
index 2f8c63f4a8f..590a54711f7 100644
--- a/include/ginkgo/core/preconditioner/isai.hpp
+++ b/include/ginkgo/core/preconditioner/isai.hpp
@@ -98,21 +98,18 @@ class Isai : public EnableLinOp<Isai<IsaiType, ValueType, IndexType>>,
              public Transposable {
     friend class EnableLinOp<Isai>;
     friend class EnablePolymorphicObject<Isai, LinOp>;
-    friend class Isai<IsaiType == isai_type::lower ? isai_type::upper
-                                                   : isai_type::lower,
-                      ValueType, IndexType>;
-    friend class Isai<IsaiType == isai_type::upper ? isai_type::general
-                                                   : isai_type::upper,
-                      ValueType, IndexType>;
-    friend class Isai<IsaiType == isai_type::general ? isai_type::lower
-                                                     : isai_type::general,
-                      ValueType, IndexType>;
+    friend class Isai<isai_type::general, ValueType, IndexType>;
+    friend class Isai<isai_type::lower, ValueType, IndexType>;
+    friend class Isai<isai_type::upper, ValueType, IndexType>;
 
 public:
     using value_type = ValueType;
     using index_type = IndexType;
     using transposed_type =
-        Isai<IsaiType == isai_type::lower ? isai_type::upper : isai_type::lower,
+        Isai<IsaiType == isai_type::general
+                 ? isai_type::general
+                 : IsaiType == isai_type::lower ? isai_type::upper
+                                                : isai_type::lower,
              ValueType, IndexType>;
     using Csr = matrix::Csr<ValueType, IndexType>;
     static constexpr isai_type type{IsaiType};
@@ -149,6 +146,19 @@ class Isai : public EnableLinOp<Isai<IsaiType, ValueType, IndexType>>,
          * Must be at least 1, default value 1.
          */
         int GKO_FACTORY_PARAMETER_SCALAR(sparsity_power, 1);
+
+        /**
+         * @brief Size limit for the excess system.
+         *
+         * For rows with more than 32 nonzero entries, the algorithm builds up
+         * an excess system which is solved with sparse triangular solves (for
+         * upper or lower ISAI) or GMRES (for general ISAI). If this parameter
+         * is set to some m > 0, the excess system is solved as soon
+         * as its size supersedes m. This is repeated until the complete excess
+         * solution has been computed.
+         * Must be at least 0, default value 0.
+         */
+        int GKO_FACTORY_PARAMETER_SCALAR(excess_limit, 0);
     };
 
     GKO_ENABLE_LIN_OP_FACTORY(Isai, parameters, Factory);
@@ -177,7 +187,8 @@ class Isai : public EnableLinOp<Isai<IsaiType, ValueType, IndexType>>,
     {
         const auto skip_sorting = parameters_.skip_sorting;
         const auto power = parameters_.sparsity_power;
-        generate_inverse(system_matrix, skip_sorting, power);
+        const auto excess_limit = parameters_.excess_limit;
+        generate_inverse(system_matrix, skip_sorting, power, excess_limit);
     }
 
     void apply_impl(const LinOp *b, LinOp *x) const override
@@ -203,7 +214,7 @@ class Isai : public EnableLinOp<Isai<IsaiType, ValueType, IndexType>>,
      *                      be skipped.
      */
     void generate_inverse(std::shared_ptr<const LinOp> to_invert,
-                          bool skip_sorting, int power);
+                          bool skip_sorting, int power, int excess_limit);
 
 private:
     std::shared_ptr<Csr> approximate_inverse_;
diff --git a/matrices/CMakeLists.txt b/matrices/CMakeLists.txt
index 76a30c4f80b..bf16f7bbee0 100644
--- a/matrices/CMakeLists.txt
+++ b/matrices/CMakeLists.txt
@@ -17,5 +17,8 @@ configure_file("test/isai_u_inv.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_u_
 configure_file("test/isai_u_inv_partial.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_u_inv_partial.mtx")
 configure_file("test/1138_bus.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/1138_bus.mtx")
 configure_file("test/1138_bus_isai.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/1138_bus_isai.mtx")
-configure_file("test/a_large.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/a_large.mtx")
-configure_file("test/a_large_inv.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/a_large_inv.mtx")
+configure_file("test/isai_a.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_a.mtx")
+configure_file("test/isai_a_excess.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_a_excess.mtx")
+configure_file("test/isai_a_excess_rhs.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_a_excess_rhs.mtx")
+configure_file("test/isai_a_inv.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_a_inv.mtx")
+configure_file("test/isai_a_inv_partial.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_a_inv_partial.mtx")
diff --git a/matrices/test/a_large.mtx b/matrices/test/isai_a.mtx
similarity index 100%
rename from matrices/test/a_large.mtx
rename to matrices/test/isai_a.mtx
diff --git a/matrices/test/isai_a_excess.mtx b/matrices/test/isai_a_excess.mtx
new file mode 100644
index 00000000000..91bf6e02fa2
--- /dev/null
+++ b/matrices/test/isai_a_excess.mtx
@@ -0,0 +1,14886 @@
+%%MatrixMarket matrix array real general
+122 122
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 2
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 0
+ 1
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 1
+ 2
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 3
+ 1
diff --git a/matrices/test/isai_a_excess_rhs.mtx b/matrices/test/isai_a_excess_rhs.mtx
new file mode 100644
index 00000000000..98684a9b26c
--- /dev/null
+++ b/matrices/test/isai_a_excess_rhs.mtx
@@ -0,0 +1,124 @@
+%%MatrixMarket matrix array real general
+122 1
+ 0
+ 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
+ 0
diff --git a/matrices/test/a_large_inv.mtx b/matrices/test/isai_a_inv.mtx
similarity index 100%
rename from matrices/test/a_large_inv.mtx
rename to matrices/test/isai_a_inv.mtx
diff --git a/matrices/test/isai_a_inv_partial.mtx b/matrices/test/isai_a_inv_partial.mtx
new file mode 100644
index 00000000000..f401ed2896f
--- /dev/null
+++ b/matrices/test/isai_a_inv_partial.mtx
@@ -0,0 +1,416 @@
+%%MatrixMarket matrix coordinate real general
+% Generated 05-Jan-2021
+100 100 413
+1 1  1
+2 1 -2
+36 1 -1
+2 2  1
+3 2 -2
+36 2 -1
+1 3 -3
+3 3  1
+4 3 -2
+36 3 -1
+2 4 -3
+4 4  1
+5 4 -2
+36 4 -1
+3 5 -3
+5 5  1
+6 5 -2
+36 5 -1
+4 6 -3
+6 6  1
+7 6 -2
+36 6 -1
+5 7 -3
+7 7  1
+8 7 -2
+36 7 -1
+6 8 -3
+8 8  1
+9 8 -2
+36 8 -1
+7 9 -3
+9 9  1
+10 9 -2
+36 9 -1
+8 10 -3
+10 10  1
+11 10 -2
+36 10 -1
+9 11 -3
+11 11  1
+12 11 -2
+36 11 -1
+10 12 -3
+12 12  1
+13 12 -2
+36 12 -1
+11 13 -3
+13 13  1
+14 13 -2
+36 13 -1
+12 14 -3
+14 14  1
+15 14 -1
+36 14 -1
+13 15 -3
+15 15 -1
+16 15 -2
+36 15 -1
+14 16 -3
+16 16  1
+17 16 -2
+36 16 -1
+15 17 -1
+17 17  1
+18 17 -2
+36 17 -1
+15 18 -1
+16 18  5
+18 18  1
+19 18 -2
+36 18 -1
+15 19 -1
+17 19 -3
+19 19  1
+20 19 -2
+36 19 -1
+15 20 -1
+18 20 -3
+20 20  1
+21 20 -2
+36 20 -1
+15 21 -1
+19 21 -3
+21 21  1
+22 21 -2
+36 21 -1
+15 22 -1
+20 22 -3
+22 22  1
+23 22 -2
+36 22 -1
+15 23 -1
+21 23 -3
+23 23  1
+24 23 -2
+36 23 -1
+15 24 -1
+22 24 -3
+24 24  1
+25 24 -2
+36 24 -1
+15 25 -1
+23 25 -3
+25 25  1
+26 25 -2
+36 25 -1
+15 26 -1
+24 26 -3
+26 26  1
+27 26 -2
+36 26 -1
+15 27 -1
+25 27 -3
+27 27  1
+28 27 -2
+36 27 -1
+15 28 -1
+26 28 -3
+28 28  1
+29 28 -2
+36 28 -1
+15 29 -1
+27 29 -3
+29 29  1
+30 29 -2
+36 29 -1
+15 30 -1
+28 30 -3
+30 30  1
+31 30 -2
+36 30 -1
+15 31 -1
+29 31 -3
+31 31  1
+32 31 -2
+36 31 -1
+15 32 -1
+30 32 -3
+32 32  1
+33 32 -2
+36 32 -1
+15 33 -1
+31 33 -3
+33 33  1
+34 33  1
+36 33 -1
+15 34 -1
+32 34 -3
+34 34  1
+35 34 -2
+15 35 -1
+33 35 -3
+35 35  1
+36 35 -1
+15 36 -1
+34 36 -3
+36 36 -1
+37 36 -2
+15 37 -1
+35 37 -3
+37 37  1
+38 37 -2
+15 38 -1
+36 38 -1
+38 38  1
+39 38 -2
+15 39 -1
+37 39 -3
+39 39  1
+40 39 -2
+15 40 -1
+38 40 -3
+40 40  1
+41 40 -2
+15 41 -1
+39 41 -3
+41 41  1
+42 41 -2
+15 42 -1
+40 42 -3
+42 42  1
+43 42 -2
+15 43 -1
+41 43 -3
+43 43  1
+44 43 -2
+15 44 -1
+42 44 -3
+44 44  1
+45 44 -2
+15 45 -1
+43 45 -3
+45 45  1
+46 45 -2
+15 46 -1
+44 46 -3
+46 46  1
+47 46 -2
+15 47 -1
+45 47 -3
+47 47  1
+48 47 -2
+15 48 -1
+46 48 -3
+48 48  1
+49 48 -2
+15 49 -1
+47 49 -3
+49 49  1
+50 49 -2
+15 50 -1
+48 50 -3
+50 50  1
+51 50 -2
+15 51 -1
+49 51 -3
+51 51  1
+52 51 -2
+15 52 -1
+50 52 -3
+52 52  1
+53 52 -2
+15 53 -1
+51 53 -3
+53 53  1
+54 53 -2
+15 54 -1
+52 54 -3
+54 54  1
+55 54 -2
+15 55 -1
+53 55 -3
+55 55  1
+56 55 -2
+15 56 -1
+54 56 -3
+56 56  1
+57 56 -2
+15 57 -1
+55 57 -3
+57 57  1
+58 57 -2
+15 58 -1
+56 58 -3
+58 58  1
+59 58 -2
+15 59 -1
+57 59 -3
+59 59  1
+60 59 -2
+15 60 -1
+58 60 -3
+60 60  1
+61 60 -2
+15 61 -1
+59 61 -3
+61 61  1
+62 61 -2
+15 62 -1
+60 62 -3
+62 62  1
+63 62 -2
+15 63 -1
+61 63 -3
+63 63  1
+64 63 -2
+15 64 -1
+62 64 -3
+64 64  1
+65 64 -2
+15 65 -1
+63 65 -3
+65 65  1
+66 65 -2
+15 66 -1
+64 66 -3
+66 66  1
+67 66 -2
+15 67 -1
+65 67 -3
+67 67  1
+68 67 -2
+15 68 -1
+66 68 -3
+68 68  1
+69 68 -2
+15 69 -1
+67 69 -3
+69 69  1
+70 69 -2
+15 70 -1
+68 70 -3
+70 70  1
+71 70 -2
+15 71 -1
+69 71 -3
+71 71  1
+72 71 -2
+15 72 -1
+70 72 -3
+72 72  1
+73 72 -2
+15 73 -1
+71 73 -3
+73 73  1
+74 73 -2
+15 74 -1
+72 74 -3
+74 74  1
+75 74 -2
+15 75 -1
+73 75 -3
+75 75  1
+76 75 -2
+15 76 -1
+74 76 -3
+76 76  1
+77 76 -2
+15 77 -1
+75 77 -3
+77 77  1
+78 77 -2
+15 78 -1
+76 78 -3
+78 78  1
+79 78 -2
+15 79 -1
+77 79 -3
+79 79  1
+80 79 -2
+15 80 -1
+78 80 -3
+80 80  1
+81 80 -2
+15 81 -1
+79 81 -3
+81 81  1
+82 81 -2
+15 82 -1
+80 82 -3
+82 82  1
+83 82 -2
+15 83 -1
+81 83 -3
+83 83  1
+84 83 -2
+15 84 -1
+82 84 -3
+84 84  1
+85 84 -2
+15 85 -1
+83 85 -3
+85 85  1
+86 85 -2
+15 86 -1
+84 86 -3
+86 86  1
+87 86 -2
+15 87 -1
+85 87 -3
+87 87  1
+88 87 -2
+15 88 -1
+86 88 -3
+88 88  1
+89 88 -2
+15 89 -1
+87 89 -3
+89 89  1
+90 89 -2
+15 90 -1
+88 90 -3
+90 90  1
+91 90 -2
+15 91 -1
+89 91 -3
+91 91  1
+92 91 -2
+15 92 -1
+90 92 -3
+92 92  1
+93 92 -2
+15 93 -1
+91 93 -3
+93 93  1
+94 93 -2
+15 94 -1
+92 94 -3
+94 94  1
+95 94 -2
+15 95 -1
+93 95 -3
+95 95  1
+96 95 -2
+15 96 -1
+94 96 -3
+96 96  1
+97 96 -2
+15 97 -1
+95 97 -3
+97 97  1
+98 97 -2
+15 98 -1
+96 98 -3
+98 98  1
+99 98 -2
+15 99 -1
+97 99 -3
+99 99  1
+100 99 -2
+15 100 -1
+98 100 -3
+100 100  1
diff --git a/omp/test/preconditioner/isai_kernels.cpp b/omp/test/preconditioner/isai_kernels.cpp
index 7423721f903..76c413e30b0 100644
--- a/omp/test/preconditioner/isai_kernels.cpp
+++ b/omp/test/preconditioner/isai_kernels.cpp
@@ -90,8 +90,10 @@ class Isai : public ::testing::Test {
         auto val_dist = std::uniform_real_distribution<value_type>(-1., 1.);
         mtx = Csr::create(ref);
         if (type == matrix_type::general) {
-            mtx = gko::test::generate_random_matrix<Csr>(
+            auto dense_mtx = gko::test::generate_random_matrix<Dense>(
                 n, n, nz_dist, val_dist, rand_engine, ref, gko::dim<2>{n, n});
+            ensure_diagonal(dense_mtx.get());
+            mtx->copy_from(dense_mtx.get());
         } else {
             mtx = gko::test::generate_random_triangular_matrix<Csr>(
                 n, n, true, for_lower_tm, nz_dist, val_dist, rand_engine, ref,
@@ -105,6 +107,13 @@ class Isai : public ::testing::Test {
         d_inverse->copy_from(inverse.get());
     }
 
+    void ensure_diagonal(Dense *mtx)
+    {
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            mtx->at(i, i) = gko::one<Dense::value_type>();
+        }
+    }
+
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::OmpExecutor> omp;
diff --git a/reference/preconditioner/isai_kernels.cpp b/reference/preconditioner/isai_kernels.cpp
index e44a4911308..80ce8f4910b 100644
--- a/reference/preconditioner/isai_kernels.cpp
+++ b/reference/preconditioner/isai_kernels.cpp
@@ -33,7 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/preconditioner/isai_kernels.hpp"
 
 #include <algorithm>
-#include <iostream>
 #include <memory>
 
 
diff --git a/reference/test/preconditioner/isai_kernels.cpp b/reference/test/preconditioner/isai_kernels.cpp
index e95da3d4dc5..6f3b501ae0a 100644
--- a/reference/test/preconditioner/isai_kernels.cpp
+++ b/reference/test/preconditioner/isai_kernels.cpp
@@ -74,6 +74,12 @@ class Isai : public ::testing::Test {
 
     Isai()
         : exec{gko::ReferenceExecutor::create()},
+          a_dense{gko::initialize<Dense>({{2, 1, 2}, {1, -2, 3}, {-1, 1, 1}},
+                                         exec)},
+          a_dense_inv{gko::initialize<Dense>({{0.3125, -0.0625, -0.4375},
+                                              {0.25, -0.25, 0.25},
+                                              {0.0625, 0.1875, 0.3125}},
+                                             exec)},
           l_dense{gko::initialize<Dense>(
               {{2., 0., 0.}, {1., -2., 0.}, {-1., 1., -1.}}, exec)},
           l_dense_inv{gko::initialize<Dense>(
@@ -82,6 +88,8 @@ class Isai : public ::testing::Test {
               {{4., 1., -1.}, {0., -2., 4.}, {0., 0., 8.}}, exec)},
           u_dense_inv{gko::initialize<Dense>(
               {{.25, .125, -0.03125}, {0., -.5, .25}, {0., 0., .125}}, exec)},
+          a_csr{Csr::create(exec)},
+          a_csr_inv{Csr::create(exec)},
           l_csr{Csr::create(exec)},
           l_csr_inv{Csr::create(exec)},
           u_csr{Csr::create(exec)},
@@ -143,6 +151,10 @@ class Isai : public ::testing::Test {
                                I<value_type>{1, 4, 1, 4, 2, 1, 2, 1, 1},
                                I<index_type>{0, 3, 0, 1, 3, 1, 2, 2, 3},
                                I<index_type>{0, 2, 5, 7, 9})},
+          a_s_unsorted{Csr::create(exec, gko::dim<2>{4, 4},
+                                   I<value_type>{4, 1, 1, 2, 4, 2, 1, 1, 1},
+                                   I<index_type>{3, 0, 0, 3, 1, 2, 1, 2, 3},
+                                   I<index_type>{0, 2, 5, 7, 9})},
           a_sparse_inv{Csr::create(
               exec, gko::dim<2>{4, 4},
               I<value_type>{1, -4, -.25, .25, .5, -.125, .5, -.5, 1},
@@ -152,6 +164,8 @@ class Isai : public ::testing::Test {
         lower_isai_factory = LowerIsai::build().on(exec);
         upper_isai_factory = UpperIsai::build().on(exec);
         general_isai_factory = GeneralIsai::build().on(exec);
+        a_dense->convert_to(lend(a_csr));
+        a_dense_inv->convert_to(lend(a_csr_inv));
         l_dense->convert_to(lend(l_csr));
         l_dense_inv->convert_to(lend(l_csr_inv));
         u_dense->convert_to(lend(u_csr));
@@ -166,8 +180,11 @@ class Isai : public ::testing::Test {
         u_csr_longrow_e_rhs = read<Dense>("isai_u_excess_rhs.mtx");
         u_csr_longrow_inv_partial = read<Csr>("isai_u_inv_partial.mtx");
         u_csr_longrow_inv = read<Csr>("isai_u_inv.mtx");
-        a_csr_large = read<Csr>("a_large.mtx");
-        a_csr_large_inv = read<Csr>("a_large_inv.mtx");
+        a_csr_longrow = read<Csr>("isai_a.mtx");
+        a_csr_longrow_e = read<Csr>("isai_a_excess.mtx");
+        a_csr_longrow_e_rhs = read<Dense>("isai_a_excess_rhs.mtx");
+        a_csr_longrow_inv_partial = read<Csr>("isai_a_inv_partial.mtx");
+        a_csr_longrow_inv = read<Csr>("isai_a_inv.mtx");
     }
 
     template <typename ReadMtx>
@@ -205,10 +222,14 @@ class Isai : public ::testing::Test {
     std::unique_ptr<typename LowerIsai::Factory> lower_isai_factory;
     std::unique_ptr<typename UpperIsai::Factory> upper_isai_factory;
     std::unique_ptr<typename GeneralIsai::Factory> general_isai_factory;
+    std::shared_ptr<Dense> a_dense;
+    std::shared_ptr<Dense> a_dense_inv;
     std::shared_ptr<Dense> l_dense;
     std::shared_ptr<Dense> l_dense_inv;
     std::shared_ptr<Dense> u_dense;
     std::shared_ptr<Dense> u_dense_inv;
+    std::shared_ptr<Csr> a_csr;
+    std::shared_ptr<Csr> a_csr_inv;
     std::shared_ptr<Csr> l_csr;
     std::shared_ptr<Csr> l_csr_inv;
     std::shared_ptr<Csr> l_csr_longrow;
@@ -235,14 +256,158 @@ class Isai : public ::testing::Test {
     std::shared_ptr<Csr> u_sparse_inv;
     std::shared_ptr<Csr> u_sparse_inv2;
     std::shared_ptr<Csr> a_sparse;
+    std::shared_ptr<Csr> a_s_unsorted;
     std::shared_ptr<Csr> a_sparse_inv;
-    std::shared_ptr<Csr> a_csr_large;
-    std::shared_ptr<Csr> a_csr_large_inv;
+    std::shared_ptr<Csr> a_csr_longrow;
+    std::shared_ptr<Csr> a_csr_longrow_e;
+    std::shared_ptr<Dense> a_csr_longrow_e_rhs;
+    std::shared_ptr<Csr> a_csr_longrow_inv_partial;
+    std::shared_ptr<Csr> a_csr_longrow_inv;
 };
 
 TYPED_TEST_SUITE(Isai, gko::test::ValueIndexTypes);
 
 
+TYPED_TEST(Isai, KernelGenerateA)
+{
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto result = this->clone_allocations(lend(this->a_csr));
+    auto num_rows = result->get_size()[0];
+    gko::Array<index_type> a1(this->exec, num_rows + 1);
+    gko::Array<index_type> a2(this->exec, num_rows + 1);
+    // zero-filled array
+    gko::Array<index_type> zeros(this->exec, num_rows + 1);
+    std::fill_n(zeros.get_data(), num_rows + 1, 0);
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        this->exec, lend(this->a_csr), lend(result), a1.get_data(),
+        a2.get_data());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(result, this->a_csr_inv);
+    GKO_ASSERT_MTX_NEAR(result, this->a_csr_inv, r<value_type>::value);
+    // no row above the size limit -> zero array
+    GKO_ASSERT_ARRAY_EQ(a1, zeros);
+    GKO_ASSERT_ARRAY_EQ(a2, zeros);
+}
+
+
+TYPED_TEST(Isai, KernelGenerateA2)
+{
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto a_transpose = this->transpose(lend(this->a_csr));
+    auto result = this->clone_allocations(lend(a_transpose));
+    auto num_rows = result->get_size()[0];
+    gko::Array<index_type> a1(this->exec, num_rows + 1);
+    gko::Array<index_type> a2(this->exec, num_rows + 1);
+    // zero-filled array
+    gko::Array<index_type> zeros(this->exec, num_rows + 1);
+    std::fill_n(zeros.get_data(), num_rows + 1, 0);
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        this->exec, lend(a_transpose), lend(result), a1.get_data(),
+        a2.get_data());
+
+    const auto expected = this->transpose(lend(this->a_csr_inv));
+    GKO_ASSERT_MTX_EQ_SPARSITY(result, expected);
+    GKO_ASSERT_MTX_NEAR(result, expected, r<value_type>::value);
+    // no row above the size limit -> zero array
+    GKO_ASSERT_ARRAY_EQ(a1, zeros);
+    GKO_ASSERT_ARRAY_EQ(a2, zeros);
+}
+
+
+TYPED_TEST(Isai, KernelGenerateAsparse)
+{
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto result = this->clone_allocations(lend(this->a_sparse));
+    auto num_rows = result->get_size()[0];
+    gko::Array<index_type> a1(this->exec, num_rows + 1);
+    gko::Array<index_type> a2(this->exec, num_rows + 1);
+    // zero-filled array
+    gko::Array<index_type> zeros(this->exec, num_rows + 1);
+    std::fill_n(zeros.get_data(), num_rows + 1, 0);
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        this->exec, lend(this->a_sparse), lend(result), a1.get_data(),
+        a2.get_data());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(result, this->a_sparse_inv);
+    GKO_ASSERT_MTX_NEAR(result, this->a_sparse_inv, r<value_type>::value);
+    // no row above the size limit -> zero array
+    GKO_ASSERT_ARRAY_EQ(a1, zeros);
+    GKO_ASSERT_ARRAY_EQ(a2, zeros);
+}
+
+
+TYPED_TEST(Isai, KernelGenerateALongrow)
+{
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto result = this->clone_allocations(lend(this->a_csr_longrow));
+    auto num_rows = result->get_size()[0];
+    gko::Array<index_type> a1(this->exec, num_rows + 1);
+    gko::Array<index_type> a2(this->exec, num_rows + 1);
+    // zero-filled array
+    gko::Array<index_type> zeros(this->exec, num_rows + 1);
+    std::fill_n(zeros.get_data(), num_rows + 1, 0);
+    // only the 32nd row has some excess storage
+    auto a1_expect = zeros;
+    std::fill_n(a1_expect.get_data() + 15, 21, 86);
+    std::fill_n(a1_expect.get_data() + 36, 65, 122);
+    auto a2_expect = zeros;
+    std::fill_n(a2_expect.get_data() + 15, 21, 355);
+    std::fill_n(a2_expect.get_data() + 36, 65, 509);
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        this->exec, lend(this->a_csr_longrow), lend(result), a1.get_data(),
+        a2.get_data());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(result, this->a_csr_longrow_inv_partial);
+    GKO_ASSERT_MTX_NEAR(result, this->a_csr_longrow_inv_partial,
+                        r<value_type>::value);
+    // no row above the size limit -> zero array
+    GKO_ASSERT_ARRAY_EQ(a1, a1_expect);
+    GKO_ASSERT_ARRAY_EQ(a2, a2_expect);
+}
+
+
+TYPED_TEST(Isai, KernelGenerateExcessALongrow)
+{
+    using Csr = typename TestFixture::Csr;
+    using Dense = typename TestFixture::Dense;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto num_rows = this->a_csr_longrow->get_size()[0];
+    gko::Array<index_type> zeros(this->exec, num_rows + 1);
+    std::fill_n(zeros.get_data(), num_rows + 1, 0);
+    // only the 32nd row has some excess storage
+    auto a1 = zeros;
+    std::fill_n(a1.get_data() + 15, 21, 86);
+    std::fill_n(a1.get_data() + 36, 65, 122);
+    auto a2 = zeros;
+    std::fill_n(a2.get_data() + 15, 21, 355);
+    std::fill_n(a2.get_data() + 36, 65, 509);
+    auto result = Csr::create(this->exec, gko::dim<2>(122, 122), 509);
+    auto result_rhs = Dense::create(this->exec, gko::dim<2>(122, 1));
+
+    gko::kernels::reference::isai::generate_excess_system(
+        this->exec, lend(this->a_csr_longrow), lend(this->a_csr_longrow),
+        a1.get_const_data(), a2.get_const_data(), lend(result),
+        lend(result_rhs), 0, num_rows);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(result, this->a_csr_longrow_e);
+    GKO_ASSERT_MTX_NEAR(result, this->a_csr_longrow_e, 0);
+    GKO_ASSERT_MTX_NEAR(result_rhs, this->a_csr_longrow_e_rhs, 0);
+}
+
+
 TYPED_TEST(Isai, KernelGenerateL1)
 {
     using Csr = typename TestFixture::Csr;
@@ -667,16 +832,29 @@ TYPED_TEST(Isai, KernelScatterExcessSolution)
 }
 
 
-TYPED_TEST(Isai, ReturnsCorrectInverseLargeA)
+TYPED_TEST(Isai, ReturnsCorrectInverseA)
+{
+    using value_type = typename TestFixture::value_type;
+    const auto isai = this->general_isai_factory->generate(this->a_sparse);
+
+    auto l_inv = isai->get_approximate_inverse();
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(l_inv, this->a_sparse_inv);
+    GKO_ASSERT_MTX_NEAR(l_inv, this->a_sparse_inv, r<value_type>::value);
+}
+
+
+TYPED_TEST(Isai, ReturnsCorrectInverseALongrow)
 {
-    using Csr = typename TestFixture::Csr;
     using value_type = typename TestFixture::value_type;
-    const auto isai = this->general_isai_factory->generate(this->a_csr_large);
+    const auto isai = this->general_isai_factory->generate(this->a_csr_longrow);
 
     auto a_inv = isai->get_approximate_inverse();
 
-    GKO_ASSERT_MTX_EQ_SPARSITY(a_inv, this->a_csr_large_inv);
-    GKO_ASSERT_MTX_NEAR(a_inv, this->a_csr_large_inv,
+    GKO_ASSERT_MTX_EQ_SPARSITY(a_inv, this->a_csr_longrow_inv);
+    // need to reduce precision due to general ISAI using GMRES instead of
+    // direct solve
+    GKO_ASSERT_MTX_NEAR(a_inv, this->a_csr_longrow_inv,
                         10 * r<value_type>::value);
 }
 
@@ -793,33 +971,39 @@ TYPED_TEST(Isai, ReturnsCorrectInverseUWithU3)
 }
 
 
-TYPED_TEST(Isai, ReturnsCorrectInverseAWithA)
-{
-    using value_type = typename TestFixture::value_type;
-    auto isai = this->general_isai_factory->generate(this->a_sparse);
-
-    auto a_inv = isai->get_approximate_inverse();
-
-    GKO_ASSERT_MTX_EQ_SPARSITY(a_inv, this->a_sparse_inv);
-    GKO_ASSERT_MTX_NEAR(a_inv, this->a_sparse_inv, r<value_type>::value);
-}
-
-
 TYPED_TEST(Isai, GeneratesWithUnsortedCsr)
 {
     using Csr = typename TestFixture::Csr;
     using T = typename TestFixture::value_type;
 
+    const auto a_isai =
+        this->general_isai_factory->generate(this->a_s_unsorted);
     const auto l_isai = this->lower_isai_factory->generate(this->l_s_unsorted);
     const auto u_isai = this->upper_isai_factory->generate(this->u_s_unsorted);
+    auto a_inv = a_isai->get_approximate_inverse();
     auto l_inv = l_isai->get_approximate_inverse();
     auto u_inv = u_isai->get_approximate_inverse();
 
+    GKO_ASSERT_MTX_NEAR(a_inv, this->a_sparse_inv, r<T>::value);
     GKO_ASSERT_MTX_NEAR(l_inv, this->l_sparse_inv, r<T>::value);
     GKO_ASSERT_MTX_NEAR(u_inv, this->u_sparse_inv, r<T>::value);
 }
 
 
+TYPED_TEST(Isai, ApplyWithAMtx)
+{
+    using Dense = typename TestFixture::Dense;
+    using T = typename TestFixture::value_type;
+    const auto vec = gko::initialize<Dense>({18., 16., 12.}, this->exec);
+    auto result = Dense::create_with_config_of(lend(vec));
+    const auto a_isai = this->general_isai_factory->generate(this->a_dense);
+
+    a_isai->apply(lend(vec), lend(result));
+
+    GKO_ASSERT_MTX_NEAR(result, l({-0.625, 3.5, 7.875}), r<T>::value);
+}
+
+
 TYPED_TEST(Isai, ApplyWithLMtx)
 {
     using Dense = typename TestFixture::Dense;
@@ -848,6 +1032,22 @@ TYPED_TEST(Isai, ApplyWithUMtx)
 }
 
 
+TYPED_TEST(Isai, AdvancedApplyAMtx)
+{
+    using Dense = typename TestFixture::Dense;
+    using T = typename TestFixture::value_type;
+    const auto alpha = gko::initialize<Dense>({3.}, this->exec);
+    const auto beta = gko::initialize<Dense>({-4.}, this->exec);
+    const auto vec = gko::initialize<Dense>({18., 16., 12}, this->exec);
+    auto result = gko::initialize<Dense>({2., -3., 1.}, this->exec);
+    const auto a_isai = this->general_isai_factory->generate(this->a_dense);
+
+    a_isai->apply(lend(alpha), lend(vec), lend(beta), lend(result));
+
+    GKO_ASSERT_MTX_NEAR(result, l({-9.875, 22.5, 19.625}), r<T>::value);
+}
+
+
 TYPED_TEST(Isai, AdvancedApplyLMtx)
 {
     using Dense = typename TestFixture::Dense;
@@ -902,6 +1102,22 @@ TYPED_TEST(Isai, UseWithIluPreconditioner)
 }
 
 
+TYPED_TEST(Isai, ReturnsTransposedCorrectInverseA)
+{
+    using GeneralIsai = typename TestFixture::GeneralIsai;
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    const auto isai = this->general_isai_factory->generate(this->a_sparse);
+
+    auto a_inv = gko::as<Csr>(gko::as<GeneralIsai>(isai->transpose())
+                                  ->get_approximate_inverse()
+                                  ->transpose());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(a_inv, this->a_sparse_inv);
+    GKO_ASSERT_MTX_NEAR(a_inv, this->a_sparse_inv, r<value_type>::value);
+}
+
+
 TYPED_TEST(Isai, ReturnsTransposedCorrectInverseL)
 {
     using UpperIsai = typename TestFixture::UpperIsai;
@@ -934,6 +1150,22 @@ TYPED_TEST(Isai, ReturnsTransposedCorrectInverseU)
 }
 
 
+TYPED_TEST(Isai, ReturnsConjTransposedCorrectInverseA)
+{
+    using GeneralIsai = typename TestFixture::GeneralIsai;
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    const auto isai = this->general_isai_factory->generate(this->a_sparse);
+
+    auto a_inv = gko::as<Csr>(gko::as<GeneralIsai>(isai->conj_transpose())
+                                  ->get_approximate_inverse()
+                                  ->conj_transpose());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(a_inv, this->a_sparse_inv);
+    GKO_ASSERT_MTX_NEAR(a_inv, this->a_sparse_inv, r<value_type>::value);
+}
+
+
 TYPED_TEST(Isai, ReturnsConjTransposedCorrectInverseL)
 {
     using UpperIsai = typename TestFixture::UpperIsai;

From d85bb43c11ce7dc9f62844afc0126a75ed84203e Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Tue, 5 Jan 2021 17:10:59 +0100
Subject: [PATCH 0511/1008] Fix hip tests

---
 hip/preconditioner/isai_kernels.hip.cpp      |  4 +++-
 hip/test/preconditioner/isai_kernels.hip.cpp | 18 +++++++++---------
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/hip/preconditioner/isai_kernels.hip.cpp b/hip/preconditioner/isai_kernels.hip.cpp
index 216b361071f..d5af2162a3d 100644
--- a/hip/preconditioner/isai_kernels.hip.cpp
+++ b/hip/preconditioner/isai_kernels.hip.cpp
@@ -69,6 +69,8 @@ constexpr int subwarps_per_block{2};
 constexpr int default_block_size{subwarps_per_block * subwarp_size};
 
 
+#include "common/components/atomic.hpp.inc"
+#include "common/components/warp_blas.hpp.inc"
 #include "common/preconditioner/isai_kernels.hpp.inc"
 
 
@@ -173,7 +175,7 @@ void scatter_excess_solution(std::shared_ptr<const DefaultExecutor> exec,
                              matrix::Csr<ValueType, IndexType> *inverse,
                              const size_type e_start, const size_type e_end)
 {
-    // const auto num_rows = inverse->get_size()[0];
+    const auto num_rows = inverse->get_size()[0];
 
     const dim3 block(default_block_size, 1, 1);
     const dim3 grid(ceildiv(e_end - e_start, block.x / subwarp_size), 1, 1);
diff --git a/hip/test/preconditioner/isai_kernels.hip.cpp b/hip/test/preconditioner/isai_kernels.hip.cpp
index dd640ac20d6..31f1ccd2ba7 100644
--- a/hip/test/preconditioner/isai_kernels.hip.cpp
+++ b/hip/test/preconditioner/isai_kernels.hip.cpp
@@ -53,7 +53,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
-enum struct matrix_type { lower, upper };
+enum struct matrix_type { lower, upper, general };
 class Isai : public ::testing::Test {
 protected:
     using value_type = double;
@@ -285,10 +285,10 @@ TEST_F(Isai, HipIsaiGenerateExcessLinverseLongIsEquivalentToRef)
 
     gko::kernels::reference::isai::generate_excess_system(
         ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
-        excess.get(), e_rhs.get());
+        excess.get(), e_rhs.get(), 0, num_rows);
     gko::kernels::hip::isai::generate_excess_system(
         hip, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
-        da2.get_const_data(), dexcess.get(), de_rhs.get());
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
     GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
@@ -316,10 +316,10 @@ TEST_F(Isai, HipIsaiGenerateExcessUinverseLongIsEquivalentToRef)
 
     gko::kernels::reference::isai::generate_excess_system(
         ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
-        excess.get(), e_rhs.get());
+        excess.get(), e_rhs.get(), 0, num_rows);
     gko::kernels::hip::isai::generate_excess_system(
         hip, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
-        da2.get_const_data(), dexcess.get(), de_rhs.get());
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
     GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
@@ -376,9 +376,9 @@ TEST_F(Isai, HipIsaiScatterExcessSolutionLIsEquivalentToRef)
     d_inverse->copy_from(lend(inverse));
 
     gko::kernels::reference::isai::scatter_excess_solution(
-        ref, a1.get_const_data(), e_rhs.get(), inverse.get());
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 0, num_rows);
     gko::kernels::hip::isai::scatter_excess_solution(
-        hip, da1.get_const_data(), de_rhs.get(), d_inverse.get());
+        hip, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
     ASSERT_GT(e_dim, 0);
@@ -403,9 +403,9 @@ TEST_F(Isai, HipIsaiScatterExcessSolutionUIsEquivalentToRef)
     d_inverse->copy_from(lend(inverse));
 
     gko::kernels::reference::isai::scatter_excess_solution(
-        ref, a1.get_const_data(), e_rhs.get(), inverse.get());
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 0, num_rows);
     gko::kernels::hip::isai::scatter_excess_solution(
-        hip, da1.get_const_data(), de_rhs.get(), d_inverse.get());
+        hip, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 0, num_rows);
 
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
     ASSERT_GT(e_dim, 0);

From e9471cdfb4c017a1f84ad918ccefc8fb627dc70c Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Tue, 5 Jan 2021 18:27:55 +0100
Subject: [PATCH 0512/1008] adjust variable names to general setting

---
 common/preconditioner/isai_kernels.hpp.inc |   41 +-
 matrices/CMakeLists.txt                    |    3 -
 matrices/test/1138_bus_isai.mtx            | 4057 --------------------
 omp/preconditioner/isai_kernels.cpp        |   34 +-
 reference/preconditioner/isai_kernels.cpp  |   34 +-
 5 files changed, 60 insertions(+), 4109 deletions(-)
 delete mode 100644 matrices/test/1138_bus_isai.mtx

diff --git a/common/preconditioner/isai_kernels.hpp.inc b/common/preconditioner/isai_kernels.hpp.inc
index 335ffb5b2ed..8d7a9733b8a 100644
--- a/common/preconditioner/isai_kernels.hpp.inc
+++ b/common/preconditioner/isai_kernels.hpp.inc
@@ -50,11 +50,14 @@ __forceinline__ __device__ void generic_generate(
     const IndexType *__restrict__ i_row_ptrs,
     const IndexType *__restrict__ i_col_idxs, ValueType *__restrict__ i_values,
     IndexType *__restrict__ excess_rhs_sizes,
-    IndexType *__restrict__ excess_nnz, Callable trs_solve)
+    IndexType *__restrict__ excess_nnz, Callable direct_solve)
 {
     static_assert(subwarp_size >= row_size_limit, "incompatible subwarp_size");
     const auto row = thread::get_subwarp_id_flat<subwarp_size, IndexType>();
-    __shared__ size_type k[subwarps_per_block];
+
+    // For general ISAI, the index of the one in the rhs depends on the number
+    // of nonzeros in the lower half of the matrix of the according row.
+    __shared__ size_type rhs_one_idxs[subwarps_per_block];
     if (row >= num_rows) {
         return;
     }
@@ -92,7 +95,7 @@ __forceinline__ __device__ void generic_generate(
         if (local_id == 0) {
             excess_rhs_sizes[row] = 0;
             excess_nnz[row] = 0;
-            k[row % subwarps_per_block] = 0;
+            rhs_one_idxs[row % subwarps_per_block] = 0;
         }
 
         // subwarp_size^2 storage per subwarp
@@ -100,16 +103,16 @@ __forceinline__ __device__ void generic_generate(
                                                      subwarps_per_block>
             storage;
 
-        auto trisystem_ptr = storage + (threadIdx.x / subwarp_size) *
-                                           subwarp_size * subwarp_size;
+        auto dense_system_ptr = storage + (threadIdx.x / subwarp_size) *
+                                              subwarp_size * subwarp_size;
         // row-major accessor
-        auto trisystem = [&](IndexType row, IndexType col) -> ValueType & {
-            return trisystem_ptr[row * subwarp_size + col];
+        auto dense_system = [&](IndexType row, IndexType col) -> ValueType & {
+            return dense_system_ptr[row * subwarp_size + col];
         };
 
 #pragma unroll
         for (int i = 0; i < subwarp_size; ++i) {
-            trisystem(i, local_id) = zero<ValueType>();
+            dense_system(i, local_id) = zero<ValueType>();
         }
 
         subwarp.sync();
@@ -127,25 +130,25 @@ __forceinline__ __device__ void generic_generate(
                     config::lane_mask_type, bool valid) {
                     if (valid) {
                         if (m_col_idxs[m_row_begin + m_idx] < row && col == row)
-                            atomic_add(&k[row % subwarps_per_block],
+                            atomic_add(&rhs_one_idxs[row % subwarps_per_block],
                                        size_type{1});
-                        trisystem(nz, i_idx) = m_values[m_row_begin + m_idx];
+                        dense_system(nz, i_idx) = m_values[m_row_begin + m_idx];
                     }
                 });
         }
 
         subwarp.sync();
 
-        // Now, read a full col of `trisystem` into local registers, which will
-        // be row elements after this (implicit) transpose
+        // Now, read a full col of `dense_system` into local registers, which
+        // will be row elements after this (implicit) transpose
         ValueType local_row[subwarp_size];
 #pragma unroll
         for (int i = 0; i < subwarp_size; ++i) {
-            local_row[i] = trisystem(i, local_id);
+            local_row[i] = dense_system(i, local_id);
         }
 
-        const auto rhs = trs_solve(i_row_size, local_row, subwarp,
-                                   k[row % subwarps_per_block]);
+        const auto rhs = direct_solve(i_row_size, local_row, subwarp,
+                                      rhs_one_idxs[row % subwarps_per_block]);
 
         // Write back:
         if (local_id < i_row_size) {
@@ -253,16 +256,16 @@ __global__ __launch_bounds__(default_block_size) void generate_general_inverse(
     auto general_solve = [](IndexType num_elems,
                             ValueType *__restrict__ local_row,
                             group::thread_block_tile<subwarp_size> &subwarp,
-                            size_type k) {
+                            size_type rhs_one_idx) {
         const int local_id = subwarp.thread_rank();
-        ValueType rhs = local_id == k ? one<ValueType>() : zero<ValueType>();
+        ValueType rhs =
+            local_id == rhs_one_idx ? one<ValueType>() : zero<ValueType>();
         size_type perm = local_id;
         auto pivoted = subwarp.thread_rank() >= num_elems;
         auto status = true;
         for (size_type i = 0; i < num_elems; i++) {
             const auto piv = choose_pivot(subwarp, local_row[i], pivoted);
             if (local_id == piv) {
-                // perm = i;
                 pivoted = true;
             }
             if (local_id == i) {
@@ -337,7 +340,7 @@ __global__ __launch_bounds__(default_block_size) void generate_excess_system(
             i_row_size, subwarp,
             [&](IndexType col, IndexType m_idx, IndexType i_idx,
                 config::lane_mask_type mask, bool valid) {
-                // trisystem(nz, i_idx) = m_values[m_row_begin + m_idx]
+                // dense_system(nz, i_idx) = m_values[m_row_begin + m_idx]
                 // only in sparse :)
                 if (valid) {
                     auto nz = out_nz_begin + popcnt(mask & prefix_mask);
diff --git a/matrices/CMakeLists.txt b/matrices/CMakeLists.txt
index bf16f7bbee0..f49ba06b66a 100644
--- a/matrices/CMakeLists.txt
+++ b/matrices/CMakeLists.txt
@@ -1,8 +1,6 @@
 configure_file("${Ginkgo_SOURCE_DIR}/matrices/config.hpp.in"
                "${Ginkgo_BINARY_DIR}/matrices/config.hpp" @ONLY)
 
-configure_file("test/1138_bus.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/1138_bus.mtx")
-configure_file("test/1138_bus_isai.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/1138_bus_isai.mtx")
 configure_file("test/ani1.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/ani1.mtx")
 configure_file("test/ani4.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/ani4.mtx")
 configure_file("test/isai_l.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_l.mtx")
@@ -16,7 +14,6 @@ configure_file("test/isai_u_excess_rhs.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/
 configure_file("test/isai_u_inv.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_u_inv.mtx")
 configure_file("test/isai_u_inv_partial.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_u_inv_partial.mtx")
 configure_file("test/1138_bus.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/1138_bus.mtx")
-configure_file("test/1138_bus_isai.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/1138_bus_isai.mtx")
 configure_file("test/isai_a.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_a.mtx")
 configure_file("test/isai_a_excess.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_a_excess.mtx")
 configure_file("test/isai_a_excess_rhs.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_a_excess_rhs.mtx")
diff --git a/matrices/test/1138_bus_isai.mtx b/matrices/test/1138_bus_isai.mtx
deleted file mode 100644
index 149b760de47..00000000000
--- a/matrices/test/1138_bus_isai.mtx
+++ /dev/null
@@ -1,4057 +0,0 @@
-%%MatrixMarket matrix coordinate real general
-% Generated 02-Dec-2020
-1138 1138 4054
-1 1  0.000681248559728779
-5 1  0.0005808732298176177
-563 1  0.000212507925078226
-2 2  0.1646520939128359
-10 2  0.08829684779764413
-563 2  0.03430164097698579
-3 3  0.02368699304892136
-11 3  0.005428961236945246
-34 3  0.01654454994702224
-35 3  0.007524132779723459
-104 3  0.01465705708874505
-475 3  0.004162471172391967
-4 4  0.04969379938881761
-7 4  0.03633745842841155
-27 4  0.02184528402080151
-101 4  0.04351750798021442
-102 4  0.002200311597615059
-103 4  0.002024472965514462
-1 5  0.0004423161544732949
-5 5  0.09500354946491268
-9 5  0.06728586226654351
-6 6  0.01418255460250985
-7 6  0.008626258966422079
-37 6  0.00996281784281251
-98 6  0.003376716717528236
-103 6  0.001590414343092176
-4 7  0.0332912632013526
-6 7  0.003875002233369217
-7 7  0.04589386701745581
-37 7  0.003861797540635052
-101 7  0.02960334268887455
-102 7  0.001938285938040816
-103 7  0.001793458292838763
-8 8  0.04400823191772331
-26 8  0.03175128681408829
-35 8  0.001206291042366941
-724 8  0.02087036911031598
-5 9  0.06450037559155145
-9 9  0.1918465562161385
-10 9  0.03293478661597532
-104 9  0.01561182863185781
-2 10  0.09822255098000247
-9 10  0.03361856756880777
-10 10  0.2368581714352888
-104 10  0.01906906345546891
-3 11  0.007676642745194343
-11 11  0.04289566629121028
-12 11  0.03853988857270334
-38 11  0.01348474775485342
-566 11  0.005959754499732399
-11 12  0.04289566629121028
-12 12  0.8458398174495795
-13 13  0.1764161925612767
-34 13  0.02562879633506981
-104 13  0.05594095653293755
-14 14  0.1618331449176782
-413 14  0.1087319346343822
-15 15  0.0007333571923797343
-16 15  0.0006002108600772536
-17 15  0.0006001434823037491
-18 15  0.0006001101106035339
-19 15  0.0006001101106035339
-411 15  0.002421932178618019
-15 16  0.0007333571923797343
-16 16  0.03320020727407764
-15 17  0.0007333571923797342
-17 17  0.03340014873030459
-15 18  0.0007333571923797347
-18 18  0.0335001066232039
-15 19  0.0007333571923797346
-19 19  0.0335001066232039
-20 20  0.02254269487639738
-21 20  0.03141327078401259
-37 20  6.343154797014689e-05
-102 20  5.15051584496288e-05
-20 21  0.02246119165317312
-21 21  0.03158827800283255
-22 21  0.0001745256626092879
-23 21  0.0001745250869321762
-24 21  0.0001745249232151727
-25 21  0.0001745243528732199
-21 22  0.03158827800283255
-22 22  0.1922745283135893
-21 23  0.03158827800283255
-23 23  0.1929745247784521
-21 24  0.03158827800283255
-24 24  0.1931745305202153
-21 25  0.03158827800283256
-25 25  0.1938745473256959
-8 26  0.03777862672178795
-26 26  0.2560217339241357
-35 26  0.005523321771863168
-4 27  0.01520623868387142
-27 27  0.3629987113101796
-28 27  0.1481510233754166
-29 27  0.1496009394939
-30 27  0.1260534920598905
-101 27  0.02197002425399606
-27 28  0.3629987113101797
-28 28  0.5694511133229858
-27 29  0.3629987113101797
-29 29  0.5596010050939105
-27 30  0.3629987113101796
-30 30  0.9661538336446894
-31 31  0.7124824145930962
-32 31  0.2802711061386238
-100 31  0.4701005157283245
-31 32  0.2802711061386238
-32 32  0.7124824145930961
-100 32  0.4701005157283243
-33 33  1.54424888493338
-100 33  0.4701009138396892
-3 34  0.01509919387408505
-13 34  0.01378015416852058
-34 34  0.03338970933666868
-104 34  0.0221443077423441
-553 34  0.004106497666312521
-3 35  3.921975939515009e-05
-8 35  4.519451563167247e-06
-26 35  1.739198459517821e-05
-35 35  0.03170024838405466
-104 35  3.103370868239583e-05
-710 35  0.03273816950425548
-36 36  0.04150786749482464
-711 36  0.05122840648749439
-6 37  0.01026128366544737
-7 37  0.008842239420246943
-20 37  0.00516101332813253
-37 37  0.01974040638964612
-102 37  0.0002585417122540082
-11 38  0.004243390622179626
-38 38  0.04941690164381931
-39 38  0.01612111598013823
-98 38  0.02238184966731691
-412 38  0.0007044908029997024
-38 39  0.04717912607627697
-39 39  0.07713052564334147
-99 39  0.01578007328107746
-40 40  0.005143151806086547
-41 40  0.0006196286733581442
-43 40  0.0001249975866733072
-45 40  0.002842486466533443
-40 41  0.0006583388216107648
-41 41  0.00120386063722436
-42 41  0.001041758265432484
-44 41  0.003642069331333222
-41 42  0.001203860637224361
-42 42  0.03554175498793279
-40 43  6.453490793065965e-05
-43 43  0.000556413788835265
-146 43  0.0004242938958185286
-41 44  9.260468919143784e-05
-44 44  0.004321975814517058
-486 44  0.001075993616377296
-40 45  0.002776391303475186
-45 45  0.01090537626521374
-49 45  0.001916483833115058
-46 46  0.0001981994155170788
-48 46  0.004386756576850353
-47 47  0.009579804180749558
-48 47  0.004386756576850353
-46 48  9.819941551707885e-05
-47 48  4.980469536953095e-05
-48 48  0.004386756576850353
-54 48  7.58791172932267e-06
-506 48  9.554351794912995e-05
-45 49  0.00206377128625444
-49 49  0.004164667647750025
-50 49  0.003979634966680848
-53 49  0.0008210134931751628
-49 50  0.0005330899213154762
-50 50  0.009078169026546061
-51 50  0.001041758265432484
-52 50  0.005032316597876657
-50 51  0.009078169026546059
-51 51  0.03554175498793279
-50 52  0.00865262990209776
-52 52  0.005971756341896436
-125 52  0.0002005294381465794
-49 53  0.0002823367797531123
-53 53  0.002204759276128122
-54 53  0.001758725464323372
-55 53  0.0006510001315020267
-48 54  0.0001757527218732435
-53 54  0.001608911051569371
-54 54  0.001952656877995909
-425 54  0.0001782023010670741
-445 54  0.0004330963840954397
-447 54  0.0001686303552894172
-53 55  0.002204759276128122
-55 55  0.02065100013150203
-56 56  0.03419970248216579
-63 56  0.02475599872252582
-57 57  0.01193358366230999
-66 57  0.005058753420015482
-58 58  0.06866513048422875
-70 58  0.02734989316262069
-59 59  0.04808928304500903
-67 59  0.008619277873156228
-60 60  0.01500170873047119
-68 60  0.007282230447790528
-61 61  0.01231214403251011
-130 61  0.005211665741062992
-62 62  0.02721688832821186
-129 62  0.009437459104730673
-56 63  9.969876526538767e-05
-63 63  0.02475599872252582
-64 63  2.724952471630168e-05
-65 63  0.01357137341224411
-63 64  0.004682953000407078
-64 64  0.009073295444798643
-71 64  0.005165563902222893
-131 64  0.0005524194669433561
-226 64  0.004689890351087489
-63 65  0.02471664981601872
-65 65  0.01365217736954061
-1095 65  3.883843633452087e-05
-57 66  0.003033587578308271
-66 66  0.005058753420015482
-67 66  0.001952437716718268
-68 66  0.0008766812759457492
-74 66  0.0004073635549340625
-183 66  0.001647334407114804
-226 66  0.001121354719249025
-59 67  0.006989282634009012
-66 67  0.003087639593553936
-67 67  0.008619277873156226
-68 67  0.001842789313165553
-69 67  0.001730381415443986
-60 68  0.005401711802470204
-66 68  0.0008395044991809331
-67 68  0.001077745041146966
-68 68  0.007282230447790526
-74 68  0.0008387559963791313
-90 68  0.001876116163389953
-201 68  0.001384717904813421
-67 69  0.0006919705072952755
-69 69  0.01891597685855978
-70 69  0.02515420225999527
-58 70  0.004765121218727402
-69 70  0.01471609394135156
-70 70  0.02734989316262069
-182 70  0.0006001253916848435
-196 70  0.00453850096372118
-1063 70  0.001442528314324188
-1065 70  0.002690675432880693
-1074 70  0.0005121697815175987
-64 71  0.003851870716564107
-71 71  0.02012983346115188
-72 71  0.01270007153835384
-71 72  0.01316905019369886
-72 72  0.02206897680904131
-73 72  0.01259999910540006
-121 72  0.003019727780727348
-72 73  0.02206897680904131
-73 73  0.1926000171054019
-66 74  0.0005329625712261697
-68 74  0.000974800732172311
-74 74  0.004890705457093017
-75 74  0.00242993093417575
-76 74  0.0007676610503459754
-78 74  0.0005574058088179555
-80 74  9.69225330900054e-05
-119 74  0.004088019008035104
-74 75  0.0002845256030805037
-75 75  0.02319509933140409
-204 75  0.02085818934229955
-810 75  0.0005258285665422862
-918 75  3.934955365255447e-05
-74 76  0.001037701798146781
-76 76  0.00966764264537184
-77 76  0.0006119916750053705
-78 76  0.00138301136005799
-127 76  0.007161720479276878
-76 77  5.771726981147143e-05
-77 77  0.003003800094158384
-1050 77  0.007289380632363749
-74 78  0.000783918194840543
-76 78  0.001438871260933114
-78 78  0.005215784653489311
-79 78  0.001970130754865572
-452 78  0.002641515655392799
-78 79  0.0008400855043495494
-79 79  0.005692436756110305
-416 79  0.0003493865114599682
-417 79  0.0003493865114599682
-454 79  0.005345796734007223
-74 80  0.000117754927592846
-80 80  0.002357720452614393
-81 80  0.0004214550496419059
-447 80  0.00119460968269267
-452 80  0.00102476250511989
-80 81  0.0003767445408468322
-81 81  0.004126795740775822
-448 81  0.003560755475402297
-450 81  0.002413355897637983
-464 81  0.00198538023394493
-82 82  0.2031781421616329
-94 82  0.005877230870353861
-83 83  0.06023791606499645
-94 83  0.005877230870353861
-84 84  0.07384992176116395
-95 84  0.02345972240229082
-85 85  0.003511431738654743
-87 85  0.000274155628548272
-126 85  0.002413922750527005
-86 86  0.0009997761495309598
-87 86  0.000512804588313572
-241 86  4.153351929177897e-05
-251 86  7.990980763886454e-05
-253 86  5.864093787230469e-05
-267 86  9.167342355770771e-05
-269 86  0.0001263536623349897
-283 86  9.917851390286249e-05
-291 86  1.362704561258246e-05
-293 86  3.881479722896776e-05
-294 86  3.881479722896776e-05
-302 86  0.0001480064718381182
-315 86  3.384921395448833e-05
-85 87  0.000113811913945984
-86 87  0.0009397325049656807
-87 87  0.0006981830367452577
-88 87  0.0001933333555022248
-87 88  0.0006981830367452578
-88 88  0.006993334443502398
-89 89  0.01911121455439772
-105 89  0.001572078869026403
-115 89  0.006410797694367932
-68 90  0.003632263288868323
-90 90  0.01678437110764967
-91 90  0.002271116795289418
-90 91  0.0008454971866149284
-91 91  0.004531217722525695
-92 91  0.001252506965202451
-108 91  0.000117130168114432
-115 91  0.0005814566817775351
-116 91  0.002542576566909739
-91 92  0.002864311958139353
-92 92  0.004936936923557877
-93 92  0.00102670435528675
-92 93  0.0007680432699654916
-93 93  0.002790932198967058
-94 93  0.0002705038701784743
-252 93  0.001831311572033205
-264 93  0.0001725932363595291
-271 93  0.001830903937413845
-288 93  0.0009171516086301031
-292 93  7.740390116339735e-05
-293 93  0.0002784684200391615
-294 93  0.0002784684200391615
-322 93  0.0008189522813544738
-323 93  0.0008958687596585449
-324 93  0.001018921685938732
-325 93  0.0005412463670111131
-326 93  0.0004396871639683349
-82 94  0.002178126885631672
-83 94  0.002237918384996351
-93 94  0.0004197548455136497
-94 94  0.005877230870353861
-96 94  0.0006014854638730493
-264 94  0.002981655476056899
-318 94  0.001726861983328051
-325 94  0.001843822482836495
-84 95  0.01694991385206285
-95 95  0.02345972240229082
-97 95  0.01000194443151547
-262 95  0.001081741513370692
-268 95  0.001309286249033044
-300 95  0.01224182050238317
-94 96  2.815075665825954e-06
-96 96  0.05814496801316306
-145 96  3.486767270367365e-05
-704 96  5.343053643953835e-05
-705 96  0.04366812227074233
-95 97  0.01657416089691347
-97 97  0.09017195729895189
-268 97  0.003511584060424524
-6 98  0.002699058108612025
-38 98  0.04001245422074155
-98 98  0.03625435372215891
-39 99  0.01829630006968984
-99 99  0.3484725119868282
-413 99  0.08293939894190504
-31 100  0.02589698982759715
-32 100  0.02589698982759714
-33 100  0.02494899047914339
-100 100  0.4701009138396891
-725 100  0.1297783102504183
-732 100  0.02606143467207535
-4 101  0.04351750798021441
-7 101  0.03146673606395294
-27 101  0.04625854089554216
-101 101  0.06795839341338922
-102 101  0.002357305413903345
-103 101  0.001981258340451572
-4 102  0.00226806858354301
-7 102  0.002260114569845941
-20 102  0.00656515156698897
-37 102  0.0003134712708337844
-101 102  0.002478689241000047
-102 102  0.02143503851955079
-103 102  0.002916762025562342
-4 103  3.516029982941045e-05
-6 103  5.394683124865673e-05
-7 103  6.135624943939899e-05
-101 103  3.660089136278378e-05
-102 103  6.599489002472127e-05
-103 103  0.004353068895739493
-478 103  0.006988090195758781
-3 104  0.01108975206631132
-9 104  0.0130220612456521
-10 104  0.01558227882171362
-13 104  0.04433392865653338
-34 104  0.02291223342746111
-35 104  0.00297114489493154
-104 104  0.06777084336288788
-89 105  0.001740249051467288
-105 105  0.002681032103705445
-106 105  0.002446875219224727
-109 105  0.00152502346974839
-105 106  0.002681032103705445
-106 106  0.0852469073456372
-107 107  0.0321238999874595
-108 107  0.001118221622047723
-91 108  0.0003034887245227763
-107 108  0.00102389917885948
-108 108  0.001118221622047723
-110 108  0.005977812477686725
-105 109  9.575115764068471e-05
-109 109  0.001732513235867602
-1029 109  0.001800437082327916
-108 110  9.262729407914173e-05
-110 110  0.006633544499293281
-111 110  7.026746840454583e-05
-328 110  0.001025861980529139
-367 110  7.974311888092716e-06
-370 110  3.42578322976917e-05
-110 111  2.776996845054393e-05
-111 111  0.01810571453131894
-367 111  1.172896986395923e-05
-368 111  1.33465520430845e-05
-701 111  0.0175592625109741
-112 112  0.01945153812352913
-113 112  0.0001184342802206041
-370 112  0.0008214897265262109
-686 112  0.03030190795848136
-112 113  1.437578878937231e-05
-113 113  0.004816208745748132
-114 113  7.561234209610165e-06
-115 113  3.927836046327332e-05
-690 113  0.01345565713064452
-113 114  3.669489994361314e-06
-114 114  0.009956712162072143
-1009 114  0.04910283858341127
-89 115  0.006736718553298537
-91 115  0.001430179841959822
-113 115  0.001598359618194295
-115 115  0.01549889922448875
-91 116  0.00371063263793522
-116 116  0.004592291404769712
-117 116  0.002351999823129614
-126 116  0.0006199724630645687
-116 117  0.004592291404769712
-117 117  0.09365202082213446
-118 118  0.1855881563310711
-119 118  0.008339490136900786
-74 119  0.004041394315700626
-118 119  0.004488151278380998
-119 119  0.008339490136900786
-123 119  0.001620151684609045
-120 120  0.01476311046052306
-121 120  0.0107703647853945
-72 121  0.001929758935051333
-120 121  0.006863111803522836
-121 121  0.01077036478539451
-122 121  0.002548602877220247
-504 121  0.001250671036248352
-121 122  0.004999626318088899
-122 122  0.009791044169788148
-123 122  0.002874053735863524
-124 122  0.003514192229568888
-119 123  0.002542899633968689
-122 123  0.003541517835239608
-123 123  0.01083896068944554
-125 123  0.001426130766622482
-122 124  0.004136408596181764
-124 124  0.01298484058332833
-125 124  0.001429305675588127
-127 124  0.004625210265810006
-52 125  0.0006736960274828053
-123 125  0.001113280266628655
-124 125  0.001168064733086433
-125 125  0.004277960792596333
-126 125  0.001246570300217467
-129 125  0.002200956663281992
-546 125  0.002101600612698724
-85 126  0.00195816897631273
-116 126  0.0006532948685811201
-125 126  0.000985780856494245
-126 126  0.003974648534154331
-129 126  0.001600414958890015
-76 127  0.006324626157357197
-124 127  0.003640608709090833
-127 127  0.01900076624522866
-128 127  0.01259999910540006
-127 128  0.01900076624522866
-128 128  0.1926000171054019
-62 129  0.007516887816011841
-125 129  0.001983893339846202
-126 129  0.001824214172754806
-129 129  0.009437459104730673
-61 130  0.004312144032510114
-130 130  0.005211665741062992
-133 130  0.001981334031159003
-173 130  0.001605342590001883
-213 130  0.001180718901054115
-64 131  0.0001624261208669008
-131 131  0.005496545062188923
-132 131  0.0009318855979024285
-133 131  0.004428602031975427
-136 131  0.0004868534760858688
-131 132  0.004091401135323778
-132 132  0.01111310085665062
-144 132  0.0026122901933604
-742 132  0.005001234744832591
-130 133  0.0007770379028807413
-131 133  0.004601876635986089
-133 133  0.005162154580345246
-134 133  0.001103488348470674
-133 134  0.005162154580345246
-134 134  0.08070351413887902
-135 135  0.007083093093560105
-136 135  3.061241490243482e-05
-139 135  1.251456639358159e-05
-740 135  0.01220595106371744
-131 136  0.001991894285428722
-135 136  0.001805455974750392
-136 136  0.01206700091963003
-141 136  0.005533561397417803
-137 137  0.02542334705796045
-139 137  0.01773192434110749
-141 137  0.003666823554414284
-761 137  0.008300342531490581
-138 138  0.009554387072796334
-141 138  0.00192335291309995
-877 138  0.004200650278359249
-882 138  0.006429359699931656
-135 139  0.0003734270498003733
-137 139  0.02045478170890633
-139 139  0.02643524506790336
-140 139  0.004906745044645386
-139 140  0.007462880033783114
-140 140  0.03438206097580358
-829 140  0.0241085653279941
-830 140  0.01931602694553701
-136 141  0.004873967154048279
-137 141  0.007363871512563584
-138 141  0.002927858761459856
-141 141  0.0144587676069406
-142 142  0.03396012133167858
-258 142  0.006958215837690046
-366 142  0.02123498536303899
-143 143  0.05253409842956076
-144 143  3.737609144862155e-05
-743 143  3.273053455976305e-06
-745 143  9.981171885921397e-05
-826 143  9.985431255797791e-05
-132 144  4.891684363463288e-06
-143 144  6.555676654472382e-05
-144 144  0.05984537169173331
-827 144  0.05913660555884374
-96 145  4.201106087792323e-05
-145 145  0.04826252211210828
-318 145  1.067292711318153e-05
-703 145  0.04543389368468301
-43 146  0.0003318959174329431
-146 146  0.0004296855130198788
-147 146  0.0002299996113007038
-146 147  9.765578935825544e-05
-147 147  0.0005699989687018715
-148 147  0.0003399995920004894
-147 148  0.0005699989687018715
-148 148  0.0004399995920004894
-149 149  0.1950750307588969
-166 149  0.004690290407168253
-150 150  0.02113688073170932
-173 150  0.01105797213481282
-151 151  0.09999112334144708
-175 151  0.04515284729470483
-152 152  0.1513934392593726
-180 152  0.1377601741913947
-153 153  0.03175075850354254
-183 153  0.004694917869209315
-154 154  0.07405283545133394
-207 154  0.01947995046815832
-155 155  0.03535344476653352
-198 155  0.0120529435161583
-156 156  0.01418318500048101
-213 156  0.009776534223750005
-157 157  0.03488709778969355
-219 157  0.07970002502433311
-158 158  0.0429710619638005
-226 158  0.01033652162936393
-159 159  0.1300474347284174
-182 159  0.02582330109764804
-205 159  0.04327252845072649
-160 160  0.05632744717310522
-172 160  0.009265365306001163
-233 160  0.01902225262340747
-161 161  0.09954531215457496
-222 161  0.02117189618271392
-743 161  0.01855455525395318
-162 162  0.01773475729671951
-163 162  0.006188773869795133
-169 162  0.005918776339923625
-209 162  0.004652548248564229
-223 162  0.007001645490851817
-232 162  0.007743569014220319
-162 163  0.005600105892468669
-163 163  0.01175250816271155
-186 163  0.003466894397092407
-201 163  0.0007462290367470655
-227 163  0.005113251910474695
-164 164  0.1950750307588969
-166 164  0.004690290407168253
-165 165  0.01263086671907289
-166 165  0.002415242397962376
-167 165  0.007188095238868617
-175 165  0.008221118227580022
-194 165  0.00420253278866024
-149 166  0.002975028107916812
-164 166  0.002975028107916812
-165 166  0.002242673048908967
-166 166  0.004690290407168252
-192 166  0.001758970263234734
-201 166  0.0006130890945648758
-218 166  0.001794184059995515
-165 167  0.009103303946035184
-167 167  0.03958993596969194
-175 167  0.02668698276114242
-168 168  0.06188391972552811
-180 168  0.08632167453688958
-209 168  0.05182362792372523
-162 169  0.007978392672829883
-169 169  0.04490731321160264
-187 169  0.009683898410151553
-170 170  0.05081137663511889
-188 170  0.02772165088265999
-194 170  0.004291213627925625
-171 171  0.02889329972402089
-178 171  0.01396633877726771
-217 171  0.005344323434275283
-160 172  0.008434040716645134
-172 172  0.01978366406952758
-173 172  0.007038639571083761
-222 172  0.003195671592411879
-755 172  0.01741054636210458
-130 173  0.001309510177350962
-150 173  0.007536880405309312
-172 173  0.004082521658941286
-173 173  0.01105797213481282
-184 173  0.001195525828115229
-174 174  0.02562829565967372
-211 174  0.005163235251720969
-216 174  0.01107325596818595
-151 175  0.02479111010624477
-165 175  0.005575741172997473
-167 175  0.01429176827631403
-175 175  0.04515284729470483
-179 175  0.01750756250668328
-176 176  0.008984621483198505
-177 176  0.008177446769356011
-178 176  0.000450560209875999
-207 176  0.01893741669949776
-176 177  0.00854422857710486
-177 177  0.01156068280854432
-207 177  0.01914175455590986
-171 178  0.01289567718151429
-176 178  0.002137805667154547
-178 178  0.0184856475961986
-214 178  0.002843242276535233
-217 178  0.004637007287978548
-175 179  0.02257642364735242
-179 179  0.0794408057444532
-203 179  0.05227064700289084
-152 180  0.05529346453366597
-168 180  0.03547210065369603
-180 180  0.1377601741913947
-209 180  0.03375560917442493
-181 181  0.04586758406187812
-183 181  0.004694917869209315
-70 182  0.002203629418820364
-159 182  0.01906576840335792
-182 182  0.02731018070796857
-183 182  0.002285523332612573
-205 182  0.02291377111241266
-221 182  0.01231422127431991
-66 183  0.00198936893203292
-153 183  0.002750759663542482
-181 183  0.002667587517877839
-182 183  0.002054350932803569
-183 183  0.004694917869209314
-184 183  0.001483132404887844
-201 183  0.0007039906531452232
-226 183  0.001362122911699713
-173 184  0.002205101854106055
-183 184  0.002241791644646623
-184 184  0.006434427254287113
-193 184  0.001031844286264287
-213 184  0.001051750772216008
-185 185  0.1340543473292866
-187 185  0.01760303090452317
-163 186  0.004331406867578962
-186 186  0.01087798201420141
-187 186  0.00581183269954516
-218 186  0.000982312064760316
-169 187  0.005890678929749005
-185 187  0.01005434534528653
-186 187  0.004215331338600365
-187 187  0.01760303090452318
-188 187  0.01017180807117036
-206 187  0.004624127568199378
-209 187  0.002703002287660673
-170 188  0.02206627924713095
-187 188  0.009958416228474147
-188 188  0.03685814527478884
-189 189  0.02954103618486679
-211 189  0.005664173249041018
-216 189  0.01071149211435926
-190 190  0.03159357819327018
-199 190  0.005311312024586768
-191 191  0.009445785482787799
-195 191  0.0004502057870270782
-197 191  0.0005086844560418416
-198 191  0.01164907246328365
-211 191  0.0003693306761868612
-166 192  0.0006075150998615923
-192 192  0.004524666526593245
-199 192  0.004293832269904215
-218 192  0.000424290709225597
-220 192  0.0003463798370771271
-184 193  0.001078657395417853
-193 193  0.004787816875317665
-198 193  0.002957330836441481
-199 193  0.002287519410832948
-208 193  0.001435616760642699
-165 194  0.004568275972415812
-170 194  0.004828852161815346
-194 194  0.01731147984315577
-216 194  0.008041774074406987
-191 195  0.004848045337789377
-195 195  0.03498427278244425
-211 195  0.01464965714420834
-70 196  0.02734989316262069
-196 196  0.1979384941173614
-191 197  0.00381815547328514
-197 197  0.03964947646544643
-212 197  0.02429704973565863
-155 198  0.0008534480440332083
-191 198  0.008740976039508384
-193 198  0.0002416673538872769
-198 198  0.0120529435161583
-190 199  0.001293577890270186
-192 199  0.003742240897167284
-193 199  0.0006318968140394233
-199 199  0.005311312024586768
-200 200  0.02748486987715647
-201 200  0.005820445495695883
-202 200  0.01773913959853929
-68 201  0.002118960243498572
-163 201  0.0009296743429296432
-166 201  0.0005753632929967714
-183 201  0.0009555583818203816
-200 201  0.006982651521205821
-201 201  0.007971729938329305
-218 201  0.0004355378658362216
-200 202  0.02128938213263513
-202 202  0.06573358102509876
-203 202  0.02356510680135571
-179 203  0.05677404204502257
-202 203  0.01875568339317256
-203 203  0.1045412940057817
-75 204  0.02319509933140409
-204 204  0.02240818931904955
-159 205  0.03333421775909028
-182 205  0.02439128258971602
-205 205  0.06288011275318212
-221 205  0.02412258480011461
-187 206  0.008101277506822279
-206 206  0.05717854079752385
-216 206  0.007989818687928404
-154 207  0.001152829473533445
-176 207  0.007815294389350934
-177 207  0.007560512175752126
-207 207  0.01947995046815831
-208 207  0.0008795896148575726
-224 207  0.000841955811051376
-193 208  0.001502930945889371
-207 208  0.01336505385036552
-208 208  0.009184331049542322
-162 209  0.001617378304335601
-168 209  0.04940069981255372
-180 209  0.07830404315129502
-187 209  0.001145957563087521
-209 209  0.05463987152426567
-210 210  0.006249332648497629
-215 210  0.0005600000143360003
-217 210  0.006930191938837317
-218 210  0.0002160026615936735
-174 211  0.00387420779295125
-189 211  0.004015027831902545
-191 211  0.002965586123163986
-195 211  0.01092363099366728
-211 211  0.02049648264457055
-212 211  0.01011154365328455
-197 212  0.02020517110238134
-211 212  0.01005157956517059
-212 212  0.04078177123942444
-130 213  0.0007959490277139882
-156 213  0.007283183275480579
-184 213  0.0004712384717928327
-213 213  0.009776534223750006
-504 213  0.001738948328903979
-178 214  0.004517214619620368
-214 214  0.01622365692881036
-217 214  0.006823204022623214
-210 215  0.00624933264849763
-215 215  0.08976001856793986
-174 216  0.006553651674182944
-189 216  0.005988927043812819
-194 216  0.005015865201539273
-206 216  0.003936888199328748
-216 216  0.01480200462145097
-218 216  0.0005964987786257558
-171 217  0.0003116442438405972
-178 217  0.0003729658367616034
-210 217  0.005974326108660732
-214 217  0.0004934479622916903
-217 217  0.007471819595453229
-166 218  0.001986996839654225
-186 218  0.0009447898958461862
-192 218  0.001359040340707622
-201 218  0.0005166651827864568
-210 218  0.001699405350262223
-216 218  0.0009162982296245775
-218 218  0.003260381416208405
-157 219  0.008887098829693506
-219 219  0.07970002502433311
-220 219  0.0111064592940345
-230 219  0.008021278205120287
-192 220  0.0006585461906468347
-219 220  0.06810000565232581
-220 220  0.01945203998592646
-182 221  0.01651721084133032
-205 221  0.02969407488294984
-221 221  0.06867077576023006
-222 221  0.01491787694894059
-225 221  0.01123305742296696
-161 222  0.005446791021970473
-172 222  0.00105734554889798
-221 222  0.00637057849548162
-222 222  0.04383877564319283
-225 222  0.01396029352073445
-233 222  0.01942492709578699
-743 222  0.001879645362456986
-162 223  0.009537252323750592
-223 223  0.03415770639099423
-232 223  0.02758515654850143
-207 224  0.01192367197024488
-224 224  0.06469263445835478
-1026 224  0.03321290269845339
-221 225  0.01386785543446704
-222 225  0.04035840791539649
-225 225  0.04714710373548588
-64 226  0.00397169308634345
-66 226  0.001311688119627266
-158 226  0.007271056251799585
-183 226  0.001319381244459979
-226 226  0.01033652162936393
-163 227  0.005437727636921652
-227 227  0.01880472598998021
-228 227  0.0153691797619795
-227 228  0.008894606690303017
-228 228  0.02860375113104207
-229 228  0.01432604004185496
-231 228  0.006460191377533867
-232 228  0.006731179414366265
-228 229  0.02860375113104208
-229 229  0.05052603924545497
-219 230  0.07970002502433313
-230 230  0.04602127744512029
-228 231  0.01614024570043577
-231 231  0.04065984863525216
-232 231  0.02258201670108759
-162 232  0.007076967383965091
-223 232  0.01850795004374896
-228 232  0.007523901756069395
-231 232  0.01010298983422199
-232 232  0.03354013420975288
-160 233  0.01051584785838124
-222 233  0.03565445488843226
-233 233  0.03577858589187031
-234 234  0.03955722421188783
-236 234  0.006862052954427557
-307 234  0.007259879494695027
-235 235  0.003140124125252748
-236 235  0.0004389713839516723
-243 235  0.0004756282247415277
-270 235  0.004992824107355747
-272 235  0.0001122322297658058
-298 235  0.0001990208158249584
-299 235  0.0001486121962474559
-234 236  0.006381155396333391
-235 236  0.002351339758579689
-236 236  0.01375420155716404
-286 236  0.01217676770136721
-237 237  0.03222175536937757
-261 237  0.0003348049688261235
-287 237  0.009571047657404731
-309 237  0.01516760558629039
-366 237  0.001668651248794341
-702 237  0.006099666530470593
-238 238  0.006725363712814524
-239 238  0.005075621570773962
-270 238  0.005223798051246839
-238 239  0.00480148246229583
-239 239  0.02149615133522103
-270 239  0.005005489406137049
-281 239  0.01767034896712746
-240 240  0.2035999856933056
-259 240  0.1456999329463718
-86 241  0.0001067215740544099
-241 241  0.001374005697288065
-242 241  0.000290634802869228
-246 241  0.0003293421132907725
-253 241  0.0002236550707773679
-257 241  0.00018363870715457
-266 241  0.0003256955939914388
-275 241  0.0001784559916947189
-279 241  0.000277406849575917
-289 241  3.90223036584587e-05
-291 241  0.0004288607304773129
-292 241  0.0004293326002310656
-293 241  0.0002146019610854077
-294 241  0.0002146019610854077
-299 241  0.0001125580564708786
-310 241  0.0002296081873569976
-321 241  0.0001848292649575446
-327 241  0.000217208135447652
-241 242  0.000808297801446638
-242 242  0.004532535764806162
-244 242  0.003313942588638787
-245 242  0.003202739628222241
-235 243  0.003140124125252748
-243 243  0.06897562890974154
-242 244  0.002185807861869108
-244 244  0.01825603325724795
-272 244  0.01335998716340609
-242 245  0.003423511021214106
-245 245  0.01391310681875108
-298 245  0.008020761167864319
-241 246  0.001264940490760748
-246 246  0.009919386287566598
-310 246  0.004009935755031833
-247 247  0.005023952688409826
-248 247  0.003966809978134316
-255 247  0.005010061754098129
-247 248  0.003966809978134316
-248 248  0.005387345736558113
-255 248  0.00501006175409813
-249 249  0.007500025410094483
-282 249  0.009274861044283244
-250 250  0.001037632973403639
-257 250  0.0004719634104043619
-263 250  0.0005671173776828162
-289 250  0.0003975775806438073
-317 250  0.002139645988580356
-86 251  0.0006808741046427244
-251 251  0.005700731409424105
-267 251  0.00160832709644289
-93 252  0.001274125410371841
-252 252  0.008520372883448564
-310 252  0.001354434476283895
-313 252  0.006532344249316064
-86 253  0.0004431746214548092
-241 253  0.0008720044259764096
-253 253  0.005209857898399626
-254 254  0.004570573683192871
-255 254  0.002719747585571147
-285 254  0.002529730361499031
-247 255  0.002738238376246548
-248 255  0.002738238376246548
-254 255  0.001046081849369832
-255 255  0.005010061754098129
-258 255  0.0004093827040350337
-260 255  0.0004328004023325395
-263 255  0.0004754744011226133
-256 256  0.001232406118703863
-290 256  9.920001162465413e-05
-317 256  0.002847350536171632
-321 256  4.800531971530007e-05
-241 257  0.0006870027744477309
-250 257  0.0005188164306696431
-257 257  0.005727800591376988
-142 258  0.002795122930221372
-255 258  0.0006016801592703452
-258 258  0.01185005493123235
-261 258  0.007180367412454548
-327 258  0.002163824479980224
-240 259  0.0772999797319453
-259 259  0.1456999329463718
-278 259  0.03649842322627914
-255 260  0.001887197908379108
-260 260  0.005476034471244295
-289 260  0.002339197128309662
-237 261  0.0006931321003947503
-258 261  0.0115951437562709
-261 261  0.01466532263097838
-95 262  0.001482503382548444
-262 262  0.007602063438084359
-268 262  0.001251321377377948
-276 262  0.005841372405788837
-314 262  0.00554978090398742
-250 263  0.0006179533834917097
-255 263  0.002026362612105835
-263 263  0.008093427000217779
-93 264  0.000354136026332319
-94 264  0.003676606037609885
-264 264  0.005771035521286595
-311 264  0.003436992392496144
-265 265  0.01353634804014666
-276 265  0.01681685927716855
-300 265  0.0004775947571177971
-241 266  0.0006870028486440326
-266 266  0.007618978768022231
-307 266  0.007244028793220687
-86 267  0.0005156871929242583
-251 267  0.001412234574967097
-267 267  0.002393931278004583
-282 267  0.002374827321004211
-284 267  0.0007365249787451072
-302 267  0.0003004103132447092
-95 268  0.001702030013936146
-97 268  0.002631196757562732
-262 268  0.001303432303117434
-268 268  0.008268326912744399
-270 268  0.001795483912932649
-301 268  0.005225882324093172
-86 269  0.0005713007156371889
-269 269  0.003096547135627085
-284 269  0.00230476431274994
-235 270  0.002710273407661008
-238 270  0.000458126294856638
-239 270  0.0004640441205187209
-268 270  0.0001922668138947943
-270 270  0.005285106527012406
-275 270  0.0002113384082833111
-279 270  8.993314613729047e-05
-280 270  0.0004370867227569277
-318 270  0.0001073915958531392
-320 270  0.0004346249362171608
-93 271  0.002790932198967059
-271 271  0.01698090408891385
-235 272  0.000409799999122878
-244 272  0.01587353887113617
-272 272  0.02580387309143066
-273 273  0.001349989596112143
-274 273  0.000503999894361622
-303 273  0.0008608688510705149
-315 273  0.0001976934209945761
-273 274  0.0006749947980560714
-274 274  0.002153999910861622
-312 274  0.0006749947980560717
-241 275  0.0007045491408197031
-270 275  0.002575061532223329
-275 275  0.03834457886044484
-262 276  0.0004471802022402564
-265 276  0.01274009099906998
-276 276  0.01711241729020651
-277 277  0.003198786015875058
-285 277  9.459459818979985e-05
-289 277  0.003688662932945043
-259 278  0.06839997436369301
-278 278  0.05883674104228977
-309 278  0.01425623729688931
-241 279  0.001090380609736002
-270 279  0.001090962316495103
-279 279  0.02523641459730683
-270 280  0.005285106527012405
-280 280  0.03453709043965733
-239 281  0.02149615133522104
-281 281  0.3941704193726406
-249 282  0.006900025530094459
-267 282  0.0001915144758113623
-282 282  0.009274861044283244
-86 283  0.0006469139243108581
-283 283  0.004861132435203108
-292 283  0.0005251426696648901
-267 284  0.0006492017337374208
-269 284  0.002256805282478487
-284 284  0.005377784133941225
-254 285  0.003055907096497613
-277 285  0.001060062860034962
-285 285  0.005533784625049275
-236 286  0.01375420155716404
-286 286  0.1354767689343672
-237 287  0.03222175536937756
-287 287  0.06227105340170536
-93 288  0.001431018977059318
-288 288  0.00859043633331266
-291 288  0.0007238304331497447
-241 289  7.832280424198652e-06
-250 289  2.344813670225498e-05
-260 289  0.0001237458794768352
-277 289  0.00303598150753357
-289 289  0.003752813945477064
-256 290  0.001232406118703862
-290 290  0.01919920035542466
-86 291  0.0001460544564823494
-241 291  0.00121448759160345
-288 291  0.0005649416684895149
-291 291  0.001485508491030063
-93 292  9.446234339670815e-05
-241 292  0.001216875821722435
-283 292  0.0003913835229850364
-292 292  0.001487904392461122
-86 293  0.0003070354999625437
-93 293  0.0004563009241027801
-241 293  0.0008244542958433231
-293 293  0.004449376350181462
-86 294  0.0003070354999625437
-93 294  0.0004563009241027802
-241 294  0.0008244542958433231
-294 294  0.004449376350181462
-295 295  0.05362723832332691
-298 295  0.03278050112440601
-296 296  0.06730000928300155
-297 296  0.09566479307024195
-296 297  0.05740000918400153
-297 297  0.09566479307024194
-299 297  0.003600738758589758
-235 298  0.0005407355532442388
-245 298  0.004375546175952416
-295 298  0.02752723406902621
-298 298  0.032780501124406
-235 299  0.0009266725405849079
-241 299  0.0004189407119360687
-297 299  0.03826478388624038
-299 299  0.02447775247024812
-95 300  0.006497362701985984
-265 300  0.001943683324790106
-300 300  0.02765274863876077
-696 300  0.01942765355520232
-268 301  0.003633052017465452
-301 301  0.01226715775579435
-314 301  0.006721899241447137
-86 302  0.0009332426037138488
-267 302  0.0004080974533793484
-302 302  0.00141966384181147
-273 303  0.0006749946090576037
-303 303  0.001560868330027445
-312 303  0.0006749946090576038
-304 304  0.001321669685786581
-305 304  0.0005647556482983224
-315 304  0.0002403568633063155
-329 304  0.001557329223669231
-304 305  0.0008701094535217735
-305 305  0.002662907123752241
-315 305  0.0005312731733141023
-306 306  0.04073726346751776
-322 306  0.01559199226187695
-234 307  0.004188410290156808
-266 307  0.00681226259005297
-307 307  0.01448805758644137
-308 308  0.02743015306702811
-323 308  0.01574015380460722
-237 309  0.02663405607802391
-278 309  0.01020310695239171
-309 309  0.02687107284802318
-241 310  0.0008569213532082748
-246 310  0.003896441702978306
-252 310  0.002946901561498048
-310 310  0.01815241623544752
-264 311  0.003786423989447812
-311 311  0.01002270676069404
-326 311  0.005609450089855228
-274 312  0.0005039998943616222
-303 312  0.0008608688510705152
-312 312  0.001349989596112143
-315 312  0.0001976934209945761
-252 313  0.008520372883448564
-313 313  0.01393234617331656
-262 314  0.003768543249850233
-301 314  0.006186000684788312
-314 314  0.0119904185721572
-86 315  0.0001480011448277059
-273 315  0.0002220497300629913
-304 315  0.0002640679800789112
-305 315  0.0003788469527306226
-312 315  0.0002220497300629914
-315 315  0.0009217135543932941
-316 315  0.0006692163067055284
-319 315  0.0001202068991403766
-323 315  0.0002912602966097636
-315 316  0.0008255077159919701
-316 316  0.00350353108752257
-322 316  0.001627447357956611
-250 317  7.981794239769917e-05
-256 317  0.001137605998066949
-317 317  0.002885220298302715
-94 318  0.003830006121181885
-145 318  0.004197780816982526
-270 318  0.001069446635194767
-318 318  0.007241474523312593
-704 318  0.002910117509878809
-315 319  9.320699702466988e-05
-319 319  0.001624041366835758
-329 319  0.002125863158142157
-270 320  0.005285106527012405
-320 320  0.06153463281811816
-241 321  0.0007188891695774862
-256 321  0.000587603034797964
-321 321  0.006031830114480963
-93 322  0.0008338102842363536
-306 322  0.01433726494591769
-316 322  0.001223778329705182
-322 322  0.01559199226187695
-93 323  0.0006597623295049258
-308 323  0.01513015412482802
-315 323  0.0001954187319347493
-323 323  0.01574015380460722
-93 324  0.001854479797952634
-324 324  0.02267436172583527
-325 324  0.006249018262464586
-93 325  0.001046340009120363
-94 325  0.003810628139594725
-324 325  0.005462594994192058
-325 325  0.0111747181380052
-93 326  0.0005388617210688591
-311 326  0.008087563722523863
-326 326  0.01631169261893439
-241 327  0.0007219351393211412
-258 327  0.005623753984343173
-327 327  0.007025770805446348
-110 328  0.00663354449929328
-328 328  0.001125861980529139
-304 329  0.0005947513586039614
-319 329  0.0008932226713696184
-329 329  0.002365022787349113
-330 330  0.4508999260647044
-335 330  0.04019070351291964
-331 331  1.13959995889597
-336 331  0.02757474132279732
-332 332  0.6038571511990494
-337 332  0.08960947178327849
-333 333  0.4221833813416651
-350 333  0.05805941947495961
-334 334  0.7287011258206326
-350 334  0.0580594194749596
-330 335  0.03590000283969022
-335 335  0.04019070351291964
-337 335  0.08247489171225011
-331 336  0.004599999188468143
-336 336  0.02757474132279732
-337 336  0.0892477453094401
-332 337  0.003857271199025383
-335 337  0.004290700827265587
-336 337  0.02297474664702812
-337 337  0.08960947178327851
-338 337  0.001382415283490295
-339 337  0.001401553638641592
-340 337  0.001916275896135684
-337 338  0.0431214456624362
-338 338  0.06631833798880571
-340 338  0.03235529912919195
-337 339  0.04340098495573665
-339 339  0.06583550982543664
-340 339  0.03217226483513082
-337 340  0.03141018516658689
-338 340  0.0170770140952318
-339 340  0.01695572224445799
-340 340  0.0400235117674176
-341 340  0.01192433400923805
-342 340  0.009597517881589177
-340 341  0.0237955848811607
-341 341  0.06132809944511584
-342 341  0.02044571043904978
-340 342  0.01349025747156711
-341 342  0.01440128559122832
-342 342  0.02733470968649056
-343 342  0.006169401047183919
-344 342  0.00685149489186951
-345 342  0.002943366891871184
-370 342  0.007948261086098257
-342 343  0.01024323849138421
-343 343  0.07252885060496569
-346 343  0.01095153750804309
-342 344  0.01041925911497252
-344 344  0.04267383459784683
-346 344  0.01083874881310861
-342 345  0.004326216135614722
-345 345  0.04658231233880137
-346 345  0.01474293786201604
-343 346  0.006699358171649296
-344 346  0.007239024315376075
-345 346  0.01018761693750956
-346 346  0.01751500625278065
-347 346  0.004304953982006397
-349 346  0.005079138881795001
-367 346  0.001454172054025146
-346 347  0.004520592701407972
-347 347  0.03133577584027234
-348 347  0.02376018555322075
-347 348  0.0160394581106115
-348 348  0.03202605464930185
-349 348  0.01227714605981959
-346 349  0.004338408876136321
-348 349  0.01479350505925204
-349 349  0.02515075220638215
-350 349  0.01685942591684934
-333 350  0.03888330567823023
-334 350  0.0372013180575793
-349 350  0.02155036496873836
-350 350  0.05805941947495961
-351 351  0.1645907649718108
-366 351  0.05143990597485315
-352 352  0.1677415812221914
-353 352  0.03358407801109305
-408 352  0.08348439138250735
-409 352  0.08953611253826632
-352 353  0.05204079709633678
-353 353  0.09468589729298324
-363 353  0.00507415031316362
-382 353  0.05110789051702095
-354 354  0.04974317454374558
-382 354  0.02346918356886678
-383 354  0.07540334827104343
-355 355  0.4702425988200105
-375 355  0.04948828803447112
-356 356  0.08030998219707833
-383 356  0.03771054601479147
-398 356  0.008409579326427436
-357 357  0.7757864328798555
-394 357  0.0232032710660538
-395 357  0.002015548461802727
-358 358  0.03518415525707855
-479 358  0.02263656719864041
-359 359  0.07364375832559185
-474 359  0.0241285481784699
-360 360  0.1640699887011138
-715 360  0.05282962784979241
-361 361  0.06454489079530285
-479 361  0.02263664722922551
-480 361  0.06357691756419355
-362 362  0.09851134347060447
-403 362  0.08750001093750137
-714 362  0.01353091383749242
-353 363  0.003738764096769265
-363 363  0.02266531374200563
-364 363  0.01142688560381418
-365 363  0.01142688560381418
-373 363  0.01142688560381418
-374 363  0.01142688560381418
-388 363  0.006396484670544655
-395 363  0.0004965090437304651
-407 363  0.001322349211133573
-474 363  0.001436918225075881
-363 364  0.02266531374200564
-364 364  0.1207268836036242
-363 365  0.02266531374200564
-365 365  0.1207268836036242
-142 366  0.01228964334324267
-237 366  0.003082084361368065
-351 366  0.02899076003597058
-366 366  0.05143990597485316
-699 366  0.02218577128663575
-110 367  0.0002331220826489727
-111 367  0.001255943206821175
-346 367  0.001051486466295456
-367 367  0.008080325719593129
-368 367  0.004420797871254525
-372 367  0.004268611151296343
-111 368  0.002433220001569482
-367 368  0.007516582890387917
-368 368  0.01304211179365907
-369 369  0.01969999893700024
-371 369  0.0120000012000012
-110 370  0.002160868153680853
-112 370  0.004177756554428322
-342 370  0.0125595769289109
-370 370  0.02874308722730365
-369 371  0.005999998800000238
-371 371  0.0120000012000012
-372 371  0.005473931316300327
-367 372  0.004040164071845544
-371 372  0.006000002400000958
-372 372  0.007871272995179882
-363 373  0.02266531374200563
-373 373  0.1207268836036242
-363 374  0.02266531374200564
-374 374  0.1207268836036242
-355 375  0.008342647273315443
-375 375  0.04948828803447112
-376 375  0.00833252688360645
-377 375  0.00833252688360645
-378 375  0.00833252688360645
-379 375  0.008328380168034728
-380 375  0.008328380168034728
-381 375  0.008328380168034728
-389 375  0.02759343872382701
-404 375  0.003940407904002059
-375 376  0.04948828803447112
-376 376  0.5035325530301679
-375 377  0.04948828803447112
-377 377  0.5035325530301679
-375 378  0.04948828803447112
-378 378  0.5035325530301679
-375 379  0.04948828803447113
-379 379  0.518628273209177
-375 380  0.04948828803447113
-380 380  0.518628273209177
-375 381  0.04948828803447113
-381 381  0.518628273209177
-353 382  0.05317839646125243
-354 382  0.02180593855871351
-382 382  0.1097010539683369
-354 383  0.01344667433749287
-356 383  0.005093798483285893
-383 383  0.09592540978204692
-384 383  0.01313663464931629
-385 383  0.01311747358717988
-386 383  0.01475695225845563
-387 383  0.01475695225845563
-392 383  0.008411227964363893
-383 384  0.0959254097820469
-384 384  0.1359366267409968
-383 385  0.0959254097820469
-385 385  0.1376174815551804
-383 386  0.09592540978204692
-386 386  0.07535695286445564
-383 387  0.09592540978204692
-387 387  0.07535695286445564
-363 388  0.008457230857481606
-388 388  0.01764812610154917
-389 388  0.007778731056151564
-390 388  0.01149923997498271
-391 388  0.01149923997498271
-395 388  0.0003965626793025834
-401 388  0.005326143243121655
-405 388  0.004713555305320748
-375 389  0.04065610206645004
-388 389  0.003149666430399245
-389 389  0.03182056075270515
-388 390  0.01764812610154917
-390 390  0.1527992366826928
-388 391  0.01764812610154917
-391 391  0.1527992366826928
-383 392  0.06413472600198364
-392 392  0.1017812758283331
-395 392  0.0006679728610330234
-393 393  0.03800594424409709
-474 393  0.01206427408923495
-479 393  0.0113182835993202
-357 394  0.0232032710660538
-394 394  0.05152222998854333
-395 394  0.002015548484634019
-357 395  0.001306203030861274
-363 395  0.0005949409627688583
-388 395  0.0002496897142986828
-392 395  0.0004470416577322448
-394 395  0.001306203045657397
-395 395  0.002015548444148886
-398 395  0.001345957292266748
-399 395  0.001411655520140569
-405 395  0.0007991848413365207
-406 395  0.001999646519169777
-407 395  0.001290913678155762
-474 395  0.0007708309133194762
-396 396  0.01039879170798621
-406 396  0.002963024354424753
-716 396  0.006742598457066043
-717 396  0.006742597597317331
-397 397  0.05829027397126174
-716 397  0.06097941825040581
-717 397  0.06090304313760567
-356 398  0.007833661897924957
-395 398  0.001818946290217446
-398 398  0.01385715250380462
-395 399  0.002015548444148885
-399 399  0.01431165549434057
-400 400  0.2445987775061038
-473 400  0.2598993828713796
-388 401  0.005708671014284298
-401 401  0.05319211368992818
-402 401  0.04745813290972838
-714 401  0.01064967860759701
-401 402  0.05319211368992818
-402 402  0.1718581206687696
-362 403  0.09851134347060446
-403 403  0.1780999911867057
-375 404  0.01848783121482114
-404 404  0.2280199623643019
-409 404  0.1250424020850327
-388 405  0.007565032031499309
-395 405  0.001334830084583861
-405 405  0.0565161430836392
-395 406  9.016385884124244e-05
-396 406  8.496382333889849e-05
-406 406  0.003316966354672551
-479 406  5.123168037249279e-05
-483 406  0.00162477055619419
-716 406  9.525755527473e-05
-717 406  9.525754619047622e-05
-363 407  0.002968326301297488
-395 407  0.001852195046613556
-407 407  0.01128987599010727
-352 408  0.1677415812221914
-408 408  0.4242843101358066
-352 409  0.1097324145119118
-404 409  0.07885493658968357
-409 409  0.1996142975217009
-410 410  0.000643158433004506
-411 410  0.002569533326060479
-412 410  7.446159408519452e-05
-731 410  5.883559546889036e-05
-15 411  9.835716465245902e-05
-410 411  0.0005477497400358428
-411 411  0.002609486999747394
-476 411  5.813859319170489e-05
-477 411  2.578446271091461e-05
-486 411  4.939008671568841e-06
-38 412  0.002478469536115102
-410 412  0.0004617144309759917
-412 412  0.009429739081657456
-413 412  0.009174104651984229
-478 412  0.0002219056184528127
-709 412  0.0002243443947259024
-724 412  0.0027358543241002
-14 413  0.08463315757847611
-99 413  0.03363341832610426
-412 413  0.003588183179913349
-413 413  0.1087319346343822
-414 414  0.006364255318889591
-431 414  0.005136411098556414
-433 414  0.0006330696963460082
-415 415  0.006364255318889591
-432 415  0.005136411098556414
-433 415  0.0006330696963460082
-79 416  0.0007559194782724966
-416 416  0.004101785720452792
-433 416  0.0005578334325645057
-462 416  0.003576021470224139
-79 417  0.0007559194782724967
-417 417  0.004101785720452792
-433 417  0.0005578334325645057
-462 417  0.003576021470224139
-418 418  0.0570517923715762
-419 418  0.03347999916567841
-545 418  0.01782518940492027
-418 419  0.0570517923715762
-419 419  0.1399800032659285
-420 420  0.001500000825000486
-421 420  0.0005958668376097395
-422 420  0.0005863636523820253
-470 420  0.002424268549138517
-420 421  0.001500000825000485
-421 421  0.02639586678600974
-420 422  0.001500000825000486
-422 422  0.08708639089989062
-423 423  0.003405062645742318
-425 423  0.002541029068390307
-516 423  0.0002515099372990493
-424 424  0.005664338462523589
-466 424  0.003927015826493891
-516 424  0.0005202826270102751
-54 425  0.0002565787489259389
-423 425  0.001978088041133092
-425 425  0.00293037999291745
-426 425  0.0005316534119392825
-427 425  0.001148886486005569
-466 425  0.0009056643417850977
-425 426  0.0004018409226975211
-426 426  0.006550786131435199
-437 426  0.007597187751206623
-519 426  0.0004513917399590606
-425 427  0.002930379992917452
-427 427  0.02924888539010561
-428 428  0.006299997570601267
-429 428  0.003559485227246435
-430 428  0.003718820571593135
-431 428  0.00287392452846315
-432 428  0.00287392452846315
-428 429  0.006299997570601268
-429 429  0.04355948522724643
-428 430  0.006299997570601269
-430 430  0.03071881841159331
-414 431  0.004141815825411193
-428 431  0.002199998464400812
-431 431  0.007209826826433458
-415 432  0.004141815825411193
-428 432  0.002199998464400812
-432 432  0.007209826826433458
-414 433  0.0009698489962126803
-415 433  0.0009698489962126803
-416 433  0.0006250712085439606
-417 433  0.0006250712085439606
-433 433  0.002201356340862928
-434 433  0.00175445638940735
-510 433  0.0007153148545362604
-564 433  0.000732509572838798
-433 434  0.002201356340862928
-434 434  0.03295445838620748
-435 435  0.01234020240462855
-510 435  0.002368518978697939
-436 436  0.04798992551016811
-534 436  0.002753567122893018
-426 437  0.005431400011357155
-437 437  0.00896202923169572
-438 437  0.001989799887815083
-439 437  0.001094884934097482
-441 437  0.001865309750819055
-458 437  0.0004901539131952915
-437 438  0.008962029231695721
-438 438  0.02388980010681508
-437 439  0.002532569114931716
-439 439  0.09645913792039387
-440 439  0.0921454312912984
-460 439  0.007379838453911518
-439 440  0.09645913792039387
-440 440  0.1816854552343008
-437 441  0.002392209988767325
-441 441  0.08436532418832167
-442 441  0.08250001443750257
-441 442  0.08436532418832167
-442 442  0.1125400126951827
-443 443  0.0009259435004858887
-444 443  0.0003236289735023835
-470 443  0.001707252245567811
-506 443  1.839401095112365e-05
-516 443  0.000332744496572827
-443 444  5.144132035192436e-05
-444 444  0.007153934619588723
-458 444  0.008552595425710553
-54 445  0.001274589000506678
-445 445  0.004367579048575067
-446 445  0.003973963702530854
-447 445  0.0004751264904632971
-445 446  0.004367579048575068
-446 446  0.0253739646013309
-54 447  0.0002977541475090077
-80 447  0.0008969608333565075
-445 447  0.000285066183339261
-447 447  0.001648620842923621
-449 447  0.0002365034127268017
-537 447  0.0008366494699104182
-81 448  0.003867353289951346
-448 448  0.009085811687866626
-464 448  0.003452828236499709
-447 449  5.846174222491055e-05
-449 449  0.004026019638989517
-450 449  0.005572822551903635
-81 450  0.0003295690503611483
-449 450  0.003665339259806753
-450 450  0.005777704586613097
-451 450  0.0004596774439698244
-450 451  0.005777704586613096
-451 451  0.04725968043917002
-78 452  0.002219578534304645
-80 452  0.0009738189492165476
-452 452  0.006869084548990908
-453 452  0.005023880261239764
-452 453  0.006869084548990908
-453 453  0.03112388451554046
-79 454  0.005197036156924244
-454 454  0.007995797303423095
-455 454  0.002650000144425008
-454 455  0.007995797303423095
-455 455  0.03044999475122606
-456 456  0.04341440055575247
-457 456  0.01229999289060411
-564 456  0.01602933825183329
-456 457  0.04341440055575248
-457 457  0.05350000080100564
-437 458  0.0002009059434134133
-444 458  0.006755714948519285
-458 458  0.009055686456956213
-459 458  0.000766289156101921
-460 458  0.0006713497988630232
-458 459  0.009055686456956211
-459 459  0.05426628567860215
-439 460  0.008640191627167487
-458 460  0.00443565641705256
-460 460  0.03510182685402592
-461 460  0.03339335231856724
-460 461  0.03510182685402592
-461 461  0.07939335323856726
-416 462  0.001848944552278542
-417 462  0.001848944552278542
-462 462  0.005826020253284005
-463 462  0.002249999668125049
-462 463  0.005826020253284005
-463 463  0.02284999937972506
-81 464  0.001779445449218795
-448 464  0.002849339920859814
-464 464  0.01526227908144768
-465 464  0.01079433379213668
-545 464  0.005691657248232669
-464 465  0.004427225641076556
-465 465  0.03465230862706979
-968 465  0.02538569746428179
-424 466  0.00291083170620776
-425 466  0.001072372864304547
-466 466  0.005415480073738198
-467 466  0.002745132358752783
-466 467  0.005415480073738198
-467 467  0.0228451338863529
-468 468  0.05510550055005413
-544 468  0.05510495451728012
-469 469  0.01341557922769634
-543 469  0.01855183921053081
-420 470  0.0009000006030003422
-443 470  0.0003703773335264287
-470 470  0.002497462050535406
-471 471  0.02675026445067476
-472 471  0.02144476375697679
-478 471  0.002480272980294884
-481 471  0.002283100356600413
-492 471  0.001339230588855801
-707 471  0.002265563440073605
-708 471  0.007899110979947414
-471 472  0.02675026445067475
-472 472  0.06197810141386614
-400 473  0.2444987775061038
-473 473  0.2598993828713796
-474 473  9.876610596768615e-06
-359 474  0.01974838971630254
-363 474  0.002047480310664833
-393 474  0.008859454214705957
-395 474  0.0007630072433894486
-473 474  0.01538178618237745
-474 474  0.02412854817846989
-3 475  0.00646191477577647
-475 475  0.03324810431463584
-710 475  0.01261217684683017
-724 475  0.008319987404669415
-411 476  0.001570265838304289
-476 476  0.01396026233454226
-482 476  0.004080436981661773
-411 477  0.0007799517638505359
-477 477  0.004427955706412391
-478 477  0.004935991219056525
-103 478  0.004269418185236161
-412 478  3.016846928798592e-06
-471 478  4.662970586208979e-05
-477 478  6.965301185643493e-05
-478 478  0.007040263277160317
-708 478  5.374029868745428e-05
-709 478  3.456002432602921e-05
-358 479  0.01388415670547844
-361 479  0.009950702953563957
-393 479  0.005152425593486999
-406 479  0.0006682906745437504
-479 479  0.0226365671986404
-480 479  0.009950703698200726
-714 479  0.001880325451835653
-726 479  0.0009429661915102201
-361 480  0.06357691756419355
-479 480  0.02263664892318421
-480 480  0.06459064614023635
-471 481  0.002756565785961451
-481 481  0.06462635410894584
-484 481  0.02938317987358151
-485 481  0.01849891953488641
-491 481  0.009825378992632576
-492 481  0.002256316524900598
-733 481  0.04196993889449727
-476 482  0.004066407369324664
-482 482  0.01024598771848416
-483 482  0.001219574850379582
-406 483  0.003291052606449368
-482 483  8.004678286288545e-05
-483 483  0.001720823741935278
-481 484  0.02868022868001192
-484 484  0.07505287813943308
-709 484  0.01851248638185964
-733 484  0.04135267929782272
-481 485  0.02921838436352269
-485 485  0.09868051329820485
-491 485  0.03150328057877917
-44 486  0.004278633259597338
-411 486  1.285232732433574e-05
-486 486  0.001165659392410865
-731 486  4.049114586387005e-05
-487 487  0.2020136631043835
-491 487  0.04305408281593034
-488 488  0.2020136631043835
-491 488  0.04305408281593034
-489 489  0.1435997065319451
-492 489  0.01230949843529913
-490 490  0.1413997019505066
-492 490  0.01230949843529913
-481 491  0.01035168481130299
-485 491  0.02100428797037725
-487 491  0.02791365135263274
-488 491  0.02791365135263274
-491 491  0.04305408281593034
-492 491  0.005165474208960842
-734 491  0.001227046330290453
-471 492  0.00197146061259278
-481 492  4.011458626650895e-05
-489 492  9.97041641950374e-05
-490 492  9.970524279656442e-05
-491 492  7.592198581198728e-05
-492 492  0.01230949843529913
-707 492  0.02198305875060876
-493 493  0.08909940016095232
-707 493  0.02206309641910328
-494 494  0.01406253887620965
-505 494  0.008005776682719633
-495 495  0.02879369946723359
-513 495  0.01884335699129711
-496 496  0.007899758913639471
-521 496  0.006694920963544377
-497 497  0.01599663587803108
-522 497  0.007871670338691458
-498 498  0.05419663997109909
-523 498  0.02455560794957828
-499 499  0.02320378893096543
-500 499  0.020850000302325
-819 499  0.003414041754791358
-914 499  0.003526835757809051
-499 500  0.02320378893096543
-500 500  0.04624999948952503
-501 501  0.005232158939923239
-502 501  0.001696692336767111
-505 501  0.0007509631309112995
-507 501  0.00117070888026396
-521 501  0.001443561506118148
-531 501  0.001109607069418476
-536 501  0.0004217156152754402
-501 502  0.001437352830834991
-502 502  0.01165239098808167
-607 502  0.004360314754718347
-609 502  0.004956038990736548
-613 502  0.004951278957397824
-937 502  0.001037580302783396
-503 503  0.01101054586613354
-507 503  0.001507956907669392
-513 503  0.008614977101122299
-553 503  0.003328225977473673
-121 504  0.002803923428298168
-213 504  0.003668071549299035
-504 504  0.006674656586546838
-505 504  0.0029178786038972
-494 505  0.005862537564209443
-501 505  0.0003951008735467709
-504 505  0.001629808978278572
-505 505  0.008005776682719633
-521 505  0.001041376548755184
-526 505  0.0009648404691351401
-578 505  0.001840518415168956
-48 506  0.004319782666390631
-443 506  1.41365439410511e-05
-506 506  0.0001928389218433751
-501 507  0.001087041669452453
-503 507  0.001598049399893468
-507 507  0.004269494702603494
-508 507  0.0008134383940814002
-509 507  0.0003821908291541762
-515 507  0.0003975920977657642
-521 507  0.001564552251530617
-522 507  0.001289983641602014
-547 507  0.001329975337819727
-507 508  0.0001938459936990633
-508 508  0.01998888769357894
-619 508  0.01088941397533659
-639 508  0.003848848222262508
-781 508  0.002544443119803931
-507 509  0.0002755253431618543
-509 509  0.002031718522070935
-511 509  0.0004600728778631732
-515 509  0.0002486502206999728
-520 509  0.0006292261443290039
-534 509  0.001588740474260125
-433 510  0.0001512389025817893
-435 510  0.0003902026376535502
-510 510  0.002368518978697939
-520 510  0.002168860850370775
-509 511  0.0008247230633282589
-511 511  0.0347073296207404
-914 511  0.00292315189268783
-926 511  0.0147935377324369
-512 512  0.01048181821270249
-520 512  0.002410711473413276
-495 513  0.007193701195233452
-503 513  0.003300996193872722
-513 513  0.01884335699129712
-514 513  0.002037287546642802
-540 513  0.007031714890658067
-513 514  0.006240776994745908
-514 514  0.04024397712110585
-790 514  0.01834818795016163
-795 514  0.006109223041858325
-507 515  0.0003005067021361314
-509 515  0.0002553864953828061
-515 515  0.002489282294551514
-516 515  0.001146166513105429
-517 515  0.0007143738360434264
-518 515  0.001492624019293777
-536 515  0.0002630923132246919
-547 515  0.001107137165248834
-423 516  0.0003179471424787562
-424 516  0.0005289066356661023
-443 516  0.0003063147971697974
-515 516  0.001200921043734124
-516 516  0.001892943084271573
-515 517  0.0009350600513675123
-517 517  0.04753688459810971
-993 517  0.03571088281389261
-995 517  0.007727232829020826
-515 518  0.002489282294551514
-518 518  0.05649261851929432
-426 519  0.0005724029120355825
-519 519  0.0297707375762613
-957 519  0.00870215574754522
-967 519  0.02411319225780486
-509 520  0.0001779735487087412
-510 520  0.002074764408768619
-512 520  0.0003818181117024921
-520 520  0.002410711473413276
-496 521  0.004599758880639473
-501 521  0.0004416368365997204
-505 521  0.0006854506463116265
-507 521  0.0005714536312007762
-521 521  0.006694920963544378
-522 521  0.001672368794195314
-497 522  0.006396638950030098
-507 522  0.0008867299146804817
-521 522  0.002918329156480371
-522 522  0.007871670338691456
-523 522  0.001131618119980197
-498 523  0.004396636783898884
-522 523  0.0003817679132325544
-523 523  0.02455560794957828
-582 523  0.01699786774829901
-770 523  0.004096106648807382
-779 523  0.002460291457037252
-802 523  0.0005044474659523555
-524 524  0.0326225767152272
-533 524  0.01617785284301497
-985 524  0.003533427920856026
-1001 524  0.001745248035995898
-525 525  0.02442130355584616
-568 525  0.02917419578216094
-505 526  0.001964562380145534
-526 526  0.01603127693278412
-527 526  0.003271154589289226
-531 526  0.002707873419341401
-526 527  0.002297473511344216
-527 527  0.01093644448317669
-921 527  0.000366389537306514
-933 527  0.001763794600860524
-1068 527  0.004132599271484757
-1073 527  0.00310282109399263
-1074 527  0.006403765665934716
-1124 527  0.00625677626434674
-528 528  0.009626573465522413
-529 528  0.007131985958546047
-809 528  0.00239329992538613
-819 528  0.003106803164457794
-528 529  0.009626573465522413
-529 529  0.01223198677454618
-530 530  0.01691088535944298
-578 530  0.001074968070443172
-583 530  0.009764865690953977
-780 530  0.001219361054567015
-797 530  0.001686463163686368
-803 530  0.03152768681269896
-501 531  0.001090658453345587
-526 531  0.002566388662344346
-531 531  0.005944823775258955
-532 531  0.000821515577234781
-534 531  0.001014190985465075
-531 532  0.0006049555513227042
-532 532  0.003121950329375685
-554 532  0.0003352071636419773
-647 532  0.002073185558188997
-652 532  0.001352902530582362
-663 532  0.0003592411960136898
-668 532  0.0006129106476266529
-672 532  0.0001780754800275389
-673 532  0.001168464042819279
-677 532  0.0004358201556169567
-908 532  0.0002543322005931329
-912 532  0.0007881034826923823
-918 532  8.244961498032527e-05
-1047 532  0.0004211391056336174
-524 533  0.03262257671522721
-533 533  0.02247785120501539
-436 534  0.00138993100896746
-509 534  0.001443241409684484
-531 534  0.0006032764585047403
-534 534  0.002753567122893018
-535 534  0.0008767570510231159
-536 534  0.0002023630937857494
-542 534  0.0003069070984018251
-534 535  0.001576354145301813
-535 535  0.03027165038959069
-885 535  0.007837157227708233
-501 536  0.0003208138436881328
-515 536  0.0002874163965662608
-534 536  0.0002668946311501982
-536 536  0.002244959367557183
-537 536  0.001280692349708539
-538 536  0.0003790346554514227
-539 536  0.001469989041819689
-542 536  0.0004736553092091702
-447 537  0.0008891438110608148
-536 537  0.001034194784351591
-537 537  0.002146826463178067
-536 538  0.0004481344646009161
-538 538  0.01252220930778559
-809 538  0.004569856816044243
-914 538  0.0008492315451505555
-948 538  0.003857866148405762
-953 538  0.004022640943472898
-964 538  0.006819073731973737
-536 539  0.002244959367557183
-539 539  0.05646998354182024
-513 540  0.01461065560185121
-540 540  0.01759083286364846
-800 540  0.007934058510710686
-541 541  0.07154939909954779
-543 541  0.01855183921053081
-534 542  0.0004935388767098761
-536 542  0.0005775217137079195
-542 542  0.005015830848531875
-543 542  0.003098506385171987
-545 542  0.001956527433207467
-546 542  0.002676682454067786
-469 543  0.01331557922769634
-541 543  9.939266904720062e-05
-542 543  2.136670957503959e-05
-543 543  0.01855183921053081
-544 543  9.944401188488195e-05
-468 544  0.05500550055005413
-543 544  3.366940913367631e-05
-544 544  0.05510495451728012
-418 545  0.0235717924167142
-464 545  0.004425982967227826
-542 545  0.001488900773344342
-545 545  0.02342882527243381
-125 546  0.002250491105321284
-542 546  0.00237717115582201
-546 546  0.007634814497046847
-507 547  0.001631377753674887
-515 547  0.001787902010367797
-547 547  0.01241499285070165
-548 548  0.14323571522039
-551 548  0.02570126355718328
-549 549  0.1601038244618143
-563 549  0.05468868855029431
-550 550  0.04008590449537204
-566 550  0.0166719539021383
-548 551  0.01733571909810991
-551 551  0.02570126355718328
-552 551  0.01351756037875005
-555 551  0.005883563599726211
-566 551  0.007220063785949487
-551 552  0.01282274308683993
-552 552  0.05065452995193308
-558 552  0.001734104471257444
-560 552  0.03370717340059064
-34 553  0.006470328250772428
-503 553  0.003731423161817809
-553 553  0.01755231084499826
-561 553  0.007796010687171485
-710 553  0.0024289841157409
-532 554  0.0003193514752011467
-554 554  0.004919122257832556
-574 554  0.003873064794127629
-909 554  0.006235565611699458
-551 555  0.007502647259244349
-555 555  0.02094549304621092
-556 555  0.004504535885850671
-557 555  0.006887885052325974
-564 555  0.003881871867848462
-566 555  0.004016775969250857
-555 556  0.002261041531412913
-556 556  0.01626992155748077
-559 556  0.009951085482870572
-579 556  0.002753261862873228
-621 556  0.005541852802385738
-903 556  0.004529974196489342
-555 557  0.007155585803720392
-557 557  0.04708679258737491
-558 557  0.01330519675132981
-552 558  0.001476850571846759
-557 558  0.00754602971094295
-558 558  0.02020927228120075
-979 558  0.0002514980660136494
-980 558  0.008062883380278824
-986 558  0.006386762196060229
-989 558  0.005052587455907618
-991 558  0.0111793945366686
-998 558  0.00274502194402396
-1000 558  0.01176072660591546
-556 559  0.0146677560682016
-559 559  0.02267773670558225
-579 559  0.004402776932694109
-552 560  0.03683233072325013
-560 560  0.08116779366381811
-980 560  0.01832009613707559
-553 561  0.002339164686951676
-561 561  0.01982387379969139
-562 561  0.01338310636271287
-566 561  0.003220808187600286
-561 562  0.01865589822730168
-562 562  0.01986967192275656
-563 562  0.003222104057952865
-1 563  0.0001226117406941805
-2 563  0.02963423489898008
-549 563  0.04290382160213425
-562 563  0.005279528895186774
-563 563  0.05468868855029431
-567 563  0.01496037139568757
-433 564  0.0004111320681232224
-456 564  0.03111442297343562
-555 564  0.00202233327150202
-564 564  0.02081480258697662
-565 565  0.3305629158751743
-567 565  0.14066430109025
-11 566  0.00346861203074192
-550 566  0.01178590525947201
-551 566  0.005745114858221226
-555 566  0.002506462523915269
-561 566  0.006393320866839186
-566 566  0.0166719539021383
-567 566  0.004953107941387935
-563 567  0.01465646690876327
-565 567  0.1312628974000626
-566 567  0.005583646470765764
-567 567  0.1406643010902501
-525 568  0.01172130396224615
-568 568  0.02917419578216094
-773 568  0.003444012454111461
-795 568  0.006804073324986602
-569 569  0.06519322693617369
-785 569  0.01630843556546118
-570 570  0.08233568498998933
-579 570  0.0163349833646956
-571 571  0.09496350902083774
-776 571  0.04186788389119319
-572 572  0.1158570824502996
-783 572  0.05218409839323126
-573 573  0.07019182874608747
-796 573  0.02708953246662872
-554 574  0.004919122257832556
-574 574  0.01457306524352765
-575 575  0.03866767333357445
-581 575  0.04073951232804236
-576 576  0.05391458588463812
-582 576  0.01203398810588504
-784 576  0.02152420862206806
-577 577  0.00306672227778948
-786 577  0.006696394088334022
-802 577  0.0003043306524726053
-1090 577  0.0012464868831832
-505 578  0.003736133464812645
-530 578  0.00551489235021096
-578 578  0.0250893961158919
-780 578  0.00822075652969664
-556 579  0.003078063323239608
-559 579  0.003339361928438725
-570 579  0.01433567410998759
-579 579  0.0163349833646956
-580 579  0.00250431771032046
-782 579  0.003661767172755128
-783 579  0.005459279435238132
-895 579  0.0096135007675579
-927 579  0.0003409602645951949
-579 580  0.002843774085809601
-580 580  0.07255118833952573
-587 580  0.3430016474367272
-575 581  0.03616767333357445
-581 581  0.04073951232804236
-776 581  0.0009849149617359499
-796 581  0.0005239312254705801
-797 581  0.0002610114068673552
-799 581  0.0006397011125999043
-523 582  0.02299408921831852
-576 582  0.003428484013456784
-582 582  0.02145789393602984
-530 583  0.011403793616007
-583 583  0.04368942257989474
-780 583  0.01112424271842943
-803 583  0.01862651384398668
-584 584  0.02882155982257682
-585 584  0.01809279493917329
-1068 584  0.002213107301306411
-584 585  0.01079774525545556
-585 585  0.02070474183883917
-607 585  0.005046300734798805
-613 585  0.006111085326771086
-659 585  0.0004391739052375558
-586 586  0.4962892537424193
-597 586  0.1592822846412858
-580 587  0.01501616578679236
-587 587  0.4153019406146015
-598 587  0.01896030773337864
-614 587  0.03582205893731713
-588 588  0.1881752688478352
-595 588  0.0962969913992603
-597 588  0.07862545232220013
-589 589  0.2668000019223201
-591 589  0.1729536603888307
-590 590  0.4243994434473364
-595 590  0.07111659791198265
-616 590  0.2269998631190825
-589 591  0.10219999803776
-591 591  0.1729536603888307
-601 591  0.05639671338781434
-592 592  0.2170000006832015
-611 592  0.216699852301541
-593 593  0.1442520613678445
-604 593  0.1113998696564958
-594 594  0.01923999996047211
-600 594  0.02988530359073937
-588 595  0.1006498175293541
-590 595  0.1973997147573786
-595 595  0.1901687731682625
-596 596  0.05495189909556643
-597 596  0.03491193073271593
-603 596  0.01287992830377254
-607 596  0.007692079568584614
-586 597  0.102289243498419
-588 597  0.07546209895054574
-596 597  0.02158904963424665
-597 597  0.1592822846412858
-587 598  0.4153019406146015
-598 598  0.08996029282338179
-599 599  0.227200021676803
-1105 599  0.06992316227266118
-594 600  0.003540000572772093
-600 600  0.02988530359073937
-601 600  0.001243234856412491
-782 600  0.01042509323263464
-591 601  0.07075366235107063
-600 601  0.005882652852335237
-601 601  0.09141338326355179
-602 601  0.05355695061435049
-633 601  0.05066003695649697
-601 602  0.0470338638274957
-602 602  0.2441567076406213
-640 602  0.1905998080659934
-596 603  0.007255886343007262
-603 603  0.04378116350077813
-604 603  0.01095904454988537
-605 603  0.008973606895093062
-1092 603  0.02627123108084698
-593 604  0.04935208120194032
-603 604  0.006876957730302293
-604 604  0.1113998696564958
-612 604  0.04355999646815548
-615 604  0.02384372426385743
-603 605  0.01534211174266245
-605 605  0.04024699409656525
-607 605  0.01026772171243339
-606 606  0.2914000093530803
-610 606  0.362499508159066
-502 607  0.003759678237179507
-585 607  0.007798185119838794
-596 607  0.005713337516057005
-605 607  0.007918174627592513
-607 607  0.01580689215892814
-608 608  0.1748852365409521
-610 608  0.3381995414014211
-1101 608  0.006975330367616711
-502 609  0.006325416646161063
-609 609  0.05431870625589665
-1068 609  0.008019987326824395
-606 610  0.02429999963064
-608 610  0.1603014493503626
-610 610  0.3624995081590661
-592 611  0.08479999158784084
-611 611  0.216699852301541
-615 611  0.05901208119236167
-604 612  0.1113998696564958
-612 612  0.1710600006119056
-502 613  0.005190609684796298
-585 613  0.01148171860232741
-613 613  0.02189807970972138
-587 614  0.4153019406146015
-614 614  0.06152205618741742
-604 615  0.04359349658694239
-611 615  0.1318999054541437
-615 615  0.09686554554992785
-590 616  0.4243994434473363
-616 616  0.3625998680549227
-617 617  0.03020000296200128
-1123 617  0.0193500141544857
-618 618  0.04081223932712202
-629 618  0.05720285282046129
-639 618  0.01144819822827145
-508 619  0.01943499047608837
-619 619  0.01645120916664465
-620 619  0.00423534490524011
-619 620  0.01160192153660821
-620 620  0.1528439410437316
-632 620  0.05409203383921641
-556 621  0.005339324584916143
-621 621  0.01853167930014241
-624 621  0.02018511896003668
-1061 621  0.0006930360661798945
-622 622  0.2035583301907608
-625 622  0.1485785083616206
-641 622  0.04781225452114995
-623 623  0.07281282240639529
-639 623  0.01016929916811019
-642 623  0.03595827612770093
-621 624  0.01467654563738238
-624 624  0.03158510792992999
-638 624  0.01139999587320149
-622 625  0.126835008401068
-625 625  0.1735592372972785
-635 625  0.03281961919387635
-626 626  0.1811769747237068
-634 626  0.1514803924195199
-635 626  0.02945609591899677
-627 627  0.07505490672951293
-639 627  0.0180560389540825
-628 628  0.1056265999772952
-631 628  0.06273914382313427
-642 628  0.01887234760978006
-618 629  8.035635299263623e-05
-629 629  0.1563077836874048
-637 629  0.007997888557420911
-643 629  0.0001995971929049984
-630 630  0.0452388865598593
-631 630  0.01732372432840822
-635 630  0.01358448800769288
-796 630  0.01709806713001748
-628 631  0.08962578652385089
-630 631  0.006852996695493958
-631 631  0.08139971357245947
-620 632  0.02988999796057287
-632 632  0.1835072346350969
-634 632  0.1463196343743539
-641 632  0.05489971304307033
-601 633  0.09141338326355179
-633 633  0.289960036597547
-626 634  0.08183991251708694
-632 634  0.1006147586707503
-634 634  0.2289201683860113
-625 635  0.05995614861954387
-626 635  0.02789077922630471
-630 635  0.01052498461289881
-635 635  0.08707533504085817
-636 635  0.05089120870851743
-635 636  0.05423502137229106
-636 636  0.1899255736897406
-1094 636  0.1090050735602148
-629 637  0.1563077836874048
-637 637  0.008197888557420912
-624 638  0.03158510792992999
-638 638  0.05480000464000327
-508 639  0.01127332807615319
-618 639  0.0071224794614393
-623 639  0.00753993170476529
-627 639  0.01185492745910613
-639 639  0.0180560389540825
-602 640  0.2441567076406213
-640 640  0.3925998100859934
-622 641  0.1025854457209622
-632 641  0.09102675771027169
-641 641  0.3321852897814107
-623 642  0.03267696811303935
-628 642  0.05822342782450567
-642 642  0.08232357311481474
-629 643  0.1563077836874049
-643 643  0.008899599367905542
-644 644  0.01323657303124626
-652 644  0.00239707312597985
-662 644  0.005025584021822942
-645 645  0.09630003872258575
-650 645  0.06664279696435015
-1060 645  0.03730000622537103
-646 646  0.05391285880092912
-664 646  0.08453966978632144
-1051 646  0.01116364723718039
-532 647  0.002678834768336893
-647 647  0.009630532952425559
-1041 647  0.001817468630679752
-648 648  0.1346148415060411
-678 648  0.03513684926359105
-1042 648  0.01540000117040009
-649 649  0.01285948428552433
-672 649  0.00150499110598391
-1055 649  0.003523063866085778
-645 650  0.05900002589511072
-650 650  0.08012313786808076
-653 650  0.02613589321014604
-651 651  0.03797353961441741
-670 651  0.0094971866657186
-1045 651  0.01648149447995612
-532 652  0.0008217221959411972
-644 652  0.0001796923453236811
-652 652  0.003788921731632382
-653 652  7.321716281347581e-05
-663 652  0.001287482605270614
-666 652  5.249537174985109e-05
-668 652  0.0002007198694312743
-676 652  0.0001112784764502267
-1054 652  0.0002529543631071838
-650 653  0.0424459222855708
-652 653  0.0005258686939950927
-653 653  0.06747685422597233
-1048 653  0.04675158312548357
-654 654  0.01260807957932831
-677 654  0.001756059504498253
-1057 654  0.01380321171526099
-655 655  0.008904081060294673
-657 655  0.001946776773363627
-665 655  0.006751113816256559
-656 656  0.2619335792700578
-678 656  0.04737909658209963
-655 657  0.003816035551306431
-657 657  0.007981783418949059
-1055 657  0.003639829735503001
-658 658  0.006260198176426003
-671 658  0.004278479241092181
-672 658  0.001881896729747303
-585 659  0.001037181977881552
-659 659  0.01624023749937267
-677 659  0.007736078590405019
-660 660  0.02790850835976976
-1050 660  0.003412792197284643
-1058 660  0.003312140679385912
-661 661  0.004244637672831487
-677 661  0.008144044717630895
-644 662  0.004723423708553365
-662 662  0.01368075840798956
-672 662  0.002165886092294924
-532 663  0.0008529469044980209
-652 663  0.003755853105115885
-663 663  0.001570809468248279
-646 664  0.02736117789762836
-664 664  0.09770634942975852
-1056 664  0.02240998246274745
-655 665  0.006545383899953773
-665 665  0.008928890811550128
-682 665  0.001639777210896398
-652 666  0.0005900779526785411
-666 666  0.002212746930117028
-681 666  0.008486672214830335
-667 667  0.2585466311633726
-1051 667  0.08284242548864497
-532 668  0.001477443487970514
-652 668  0.003094491599300593
-668 668  0.004822294611486507
-669 669  0.006485560610471147
-679 669  0.001783783119839592
-1058 669  0.005162877238538627
-651 670  0.008303904399083862
-670 670  0.022494378267569
-1121 670  0.007875400787643165
-658 671  0.004458019194828498
-671 671  0.009697889314677979
-679 671  0.003081080429668798
-532 672  0.000137844322866883
-649 672  0.0008297043679315003
-658 672  0.001682971459881668
-662 672  0.001282984835809149
-672 672  0.003367603842288365
-673 672  0.0007493645012146655
-682 672  0.001174687797646172
-1049 672  0.001045870538080219
-1050 672  0.0003985892779680757
-1052 672  0.000145612193105728
-1058 672  0.0001982815878775159
-1121 672  0.001711425586384149
-532 673  0.001852560443682052
-672 673  0.001534848059850153
-673 673  0.01851598774861257
-674 674  0.02402232318843044
-675 674  0.01983441626664977
-1046 674  0.0008145987699775598
-674 675  0.02192459877049487
-675 675  0.02094228360985793
-1050 675  0.0006403606362552
-652 676  0.00040774836406457
-676 676  0.02454558279094245
-1043 676  0.02101172610586178
-532 677  9.327352359497873e-05
-654 677  0.0005312645856911045
-659 677  0.001073514741321412
-661 677  0.002244637672831488
-677 677  0.008144044717630895
-678 677  0.0002178685533486336
-1041 677  0.0009028602314610852
-1059 677  0.001393437897625826
-648 678  0.1192148219765571
-656 678  0.01443357679505776
-677 678  0.0004829145628808069
-678 678  0.04737909658209963
-669 679  0.003607352095920667
-671 679  0.004303795505879453
-679 679  0.01070269818390264
-680 680  0.01488008007666763
-1047 680  0.008686049844786263
-1057 680  0.009766514009451666
-666 681  0.0007112400557228433
-681 681  0.0100521751695015
-1053 681  0.001839884244596412
-665 682  0.003872222529723286
-672 682  0.001907161977891775
-682 682  0.006190158470591963
-683 683  0.7542499839577274
-689 683  0.1675635901773625
-684 684  0.6462017354236403
-689 684  0.1675635901773624
-685 685  0.5666997770669278
-691 685  0.03896641487219615
-112 686  0.01691740047537703
-686 686  0.03200357666390856
-687 686  0.003224099031167913
-688 686  0.001468278890354747
-686 687  0.03200357666390856
-687 687  0.05772408077367402
-686 688  0.01098878080263125
-688 688  0.01972804813157901
-689 688  0.01037615544709978
-694 688  0.0006466060581027378
-699 688  0.006856167006648931
-700 688  0.01003878005679547
-702 688  0.008631739500549513
-683 689  0.1209501944032475
-684 689  0.1262017562236396
-688 689  0.008834200967155087
-689 689  0.1675635901773625
-693 689  0.008839385710618451
-113 690  0.004731551820574414
-690 690  0.0135101390863665
-691 690  1.910253637326516e-05
-694 690  8.776830950330473e-05
-697 690  9.836441838633725e-05
-685 691  9.96103731587792e-05
-690 691  6.714139024044764e-06
-691 691  0.03896641487219615
-692 691  0.02446183953033295
-695 691  6.486467039382385e-05
-691 692  0.03896641487219615
-692 692  0.02456183953033295
-689 693  0.00899196597043592
-693 693  0.04497383722385961
-695 693  0.01805898984373897
-688 694  0.0008725144849825894
-690 694  0.01159370575005668
-694 694  0.005538997701469825
-695 694  0.001862968432341841
-691 695  0.01535552727743594
-693 695  0.01772287716298745
-694 695  0.001173482907586958
-695 695  0.01908304387364602
-300 696  0.02765274863876076
-696 696  0.03352765471140241
-690 697  0.007282564889747137
-697 697  0.07919833333209852
-698 697  0.0790999689137122
-697 698  0.07919833333209851
-698 698  0.1715999758512127
-366 699  0.0290746602309459
-688 699  0.0065709769404977
-699 699  0.06056313922562951
-700 699  0.02592637862647972
-688 700  0.01289503718154808
-699 700  0.03474841209665323
-700 700  0.08346718596140983
-111 701  0.01810571453131894
-701 701  0.01765926251097409
-237 702  0.01774120600256697
-688 702  0.008865841466589953
-702 702  0.02692143333247395
-145 703  0.04826252211210828
-703 703  0.045533893684683
-96 704  0.03023901502527092
-318 704  0.003475455478190859
-704 704  0.04932222886487166
-96 705  0.05814496801316305
-705 705  0.04376812227074232
-706 706  0.09059946824940528
-707 706  0.02206309641910327
-471 707  0.001971734187777073
-492 707  0.01222749729421697
-493 707  9.943932093508234e-05
-706 707  9.943747939481412e-05
-707 707  0.02206309641910328
-726 707  8.901260673467379e-05
-471 708  0.008749316084421591
-478 708  0.00385403264175548
-708 708  0.02183021431262304
-709 708  0.01588296265025516
-412 709  0.0001372999532777806
-478 709  0.002915407954387878
-484 709  0.01809121810037291
-708 709  0.01595480699813724
-709 709  0.03858749505769376
-35 710  0.03164543848459934
-475 710  4.077470760506248e-05
-553 710  8.822741065009899e-06
-710 710  0.03279808035444827
-36 711  0.04140786749482465
-711 711  0.05122840648749439
-724 711  3.257546437679202e-05
-734 711  3.334112418002822e-05
-712 712  0.1036341168626075
-714 712  0.02659885386924947
-713 713  0.102451101173955
-714 713  0.02659885386924947
-362 714  0.0110113325331031
-401 714  0.006063475711334444
-479 714  0.001862046232769363
-712 714  0.01093414680469787
-713 714  0.01095108653395264
-714 714  0.02659885386924947
-715 714  0.01295964108648662
-734 714  0.006067940674193537
-360 715  0.03986999193031365
-714 715  0.02013516471650269
-715 715  0.05282962784979239
-396 716  0.0001525314567501726
-397 716  0.05687887705929573
-406 716  7.515059705383282e-05
-716 716  0.06230139190235773
-717 716  0.05956303589124799
-396 717  0.0001607844820818761
-397 717  0.05680250246813857
-406 717  7.921688284150947e-05
-716 717  0.05956303589124798
-717 717  0.06233878377404831
-718 718  0.07592753070763514
-724 718  0.02428091280675981
-719 719  0.07592753070763514
-724 719  0.02428091280675981
-720 720  0.07592753070763514
-724 720  0.02428091280675981
-721 721  0.07592753070763514
-724 721  0.02428091280675981
-722 722  0.07592753070763514
-724 722  0.02428091280675981
-723 723  0.07592753070763514
-724 723  0.02428091280675981
-8 724  0.00729895583966801
-412 724  0.0007568832425986043
-475 724  0.002321034707383153
-711 724  0.004416349974273951
-718 724  0.007027539733533965
-719 724  0.007027539733533965
-720 724  0.007027539733533965
-721 724  0.007027539733533965
-722 724  0.007027539733533965
-723 724  0.007027539733533965
-724 724  0.02428091280675981
-725 724  0.002143533457852622
-734 724  0.001029094237052859
-100 725  0.4420009004047503
-724 725  0.001451380251119414
-725 725  0.1485693521752023
-479 726  0.002518552400552489
-707 726  0.01960835004028389
-726 726  0.02226920500663214
-727 727  0.04992612389972566
-728 727  0.008628578138534657
-731 727  0.007934854346086633
-727 728  0.008628578138534659
-728 728  0.04992612389972566
-731 728  0.007934854346086633
-729 729  0.05027930639610439
-730 729  0.006884220921467736
-731 729  0.00793485578510947
-729 730  0.006884220921467736
-730 730  0.05010714813185277
-731 730  0.007934855788114785
-410 731  0.000226631396720429
-486 731  0.0002811707081937405
-727 731  0.005847356851518764
-728 731  0.005847356851518764
-729 731  0.005898732982906709
-730 731  0.005898732985140845
-731 731  0.007934856577050509
-100 732  0.4701009138396891
-732 732  0.4481614073622071
-481 733  0.0489007983645527
-484 733  0.04936253445714679
-733 733  0.1040838584588347
-491 734  0.001902048294567494
-711 734  0.01491036806433065
-714 734  0.01429300183540573
-724 734  0.003394614054289708
-734 734  0.02590216027347546
-735 735  0.282986614115017
-743 735  0.03133287048374728
-736 736  0.2084587990179075
-742 736  0.02359174876849497
-737 737  0.1142704624700489
-743 737  0.03133287048374727
-738 738  0.05350832800144269
-742 738  0.02359174876849497
-739 739  0.1587735744516123
-744 739  0.01050801099509594
-135 740  0.006999705600738508
-740 740  0.0122428008299788
-741 740  5.084327844827839e-05
-742 740  2.195292780969384e-05
-744 740  8.37443760462877e-05
-740 741  0.00536070452350935
-741 741  0.01769844564454164
-749 741  0.02757154403341738
-759 741  0.008167400845581754
-132 742  0.002449442968067719
-736 742  0.01085880897694698
-738 742  0.01390833544624129
-740 742  0.001186602214348167
-742 742  0.02359174876849497
-743 742  0.01161345848853242
-143 743  0.001685886278921233
-161 743  0.009654893296614786
-222 743  0.003801822966059999
-735 743  0.0162865921656052
-737 743  0.01827049319003907
-742 743  0.01303947249535167
-743 743  0.03133287048374727
-746 743  0.007456783090147935
-739 744  0.008773581951611889
-740 744  0.00818368763764968
-744 744  0.01050801099509594
-756 744  0.00624219593045746
-143 745  0.05251031125856063
-745 745  0.0001997690630162436
-746 745  5.443332446614393e-05
-743 746  0.01477283492930067
-745 746  0.0001055818613217538
-746 746  0.1201887575476681
-747 747  0.1137999985660004
-748 747  0.006497952592681879
-758 747  0.06429999807100012
-747 748  0.04950000049500017
-748 748  0.03472892247216531
-757 748  0.01095125431616056
-741 749  0.004869002663980502
-749 749  0.06077146545370619
-753 749  0.03319995803525303
-750 750  0.1760001371201187
-751 750  0.03709999985901999
-759 750  0.05615115347388746
-750 751  0.1760001371201187
-751 751  0.14580000003294
-752 752  0.08853550051648364
-757 752  0.01388901769340084
-762 752  0.09800003488801237
-749 753  0.06077146545370619
-753 753  0.04579995793445303
-754 754  0.04067547190252845
-757 754  0.008107751683430028
-765 754  0.02349999530000094
-172 755  0.0054127681906925
-755 755  0.1236106235125528
-764 755  0.1062000660564412
-744 756  0.00572802183530221
-756 756  0.02259272477553183
-757 756  0.008816608457747651
-748 757  0.006961940094299847
-752 757  0.008964694168437832
-754 757  0.01717548063876465
-756 757  0.006236111437314269
-757 757  0.01938184701185934
-747 758  0.1137999985660004
-758 758  0.07909999753820013
-741 759  0.003730752082826072
-750 759  0.1389001269547093
-759 759  0.07531589289586132
-760 760  0.2081461918044692
-761 760  0.1097553914168045
-137 761  0.007653563226806053
-760 761  0.04784618713973914
-761 761  0.1097553914168045
-766 761  0.03100293782511568
-767 761  0.05225039853991487
-752 762  0.04193588603608223
-762 762  0.3458000662099363
-763 762  0.247800031321924
-762 763  0.3458000662099363
-763 763  0.4708000279769241
-755 764  0.1236106235125528
-764 764  0.1462000660564412
-754 765  0.04067547190252845
-765 765  0.04039999462400097
-761 766  0.07358093914497393
-766 766  0.1778670202028769
-768 766  0.09229249471260238
-761 767  0.1097553914168045
-767 767  0.1772503985399149
-766 768  0.04659723444191144
-768 768  0.2800209153583046
-769 768  0.1710364191267975
-772 768  0.1867106290410218
-768 769  0.2800209153583046
-769 769  0.5010364224267975
-523 770  9.022817695390103e-06
-770 770  0.2758305566139244
-823 770  0.2717391304347622
-771 771  0.4970000468600144
-772 771  0.4707107434108653
-768 772  0.1200089783317938
-771 772  0.2840000613440133
-772 772  0.4707107434108653
-568 773  0.007216515414597397
-773 773  0.03860036777063307
-774 773  0.03050467172946603
-795 773  0.003078064250781432
-801 773  0.02752810323413095
-802 773  0.01055000487474257
-773 774  0.03860036777063306
-774 774  0.1393046773000263
-775 775  0.02475995563029326
-776 775  0.03946389099049266
-781 775  0.008931098939211742
-571 776  0.007463498083336377
-581 776  0.002860098888363572
-775 776  0.01392562505952267
-776 776  0.04186788389119318
-781 776  0.006429672691714456
-796 776  0.0005027490914261444
-797 776  0.0003611741410102067
-1127 776  0.008813281121105767
-777 777  0.05035968024085257
-793 777  3.205968249284482e-05
-795 777  0.003314333795221247
-800 777  0.02124147823020587
-778 778  0.07615170046455932
-779 778  0.05249478425065181
-523 779  0.005412151467848236
-778 779  0.03980169204953238
-779 779  0.05249478425065182
-780 779  0.001664032361912951
-787 779  0.01174434114204541
-530 780  0.00603495599069889
-578 780  0.007002957890466868
-583 780  0.01154369993991829
-779 780  0.003450925956451911
-780 780  0.02471680581339951
-797 780  0.0008544879407238196
-949 780  0.01765563128606545
-508 781  0.01146360041979076
-775 781  0.008476554867154679
-776 781  0.01729372494402667
-781 781  0.03199980571972852
-579 782  0.001242824328283301
-600 782  0.0266921211252419
-782 782  0.01299225430463768
-783 782  0.002497153980881631
-572 783  0.04855707787389923
-579 783  0.004065955224199574
-782 783  0.005479659480382699
-783 783  0.05218409839323125
-576 784  0.02248353062551777
-784 784  0.04900984729621592
-802 784  0.01781250393151464
-569 785  0.01539322374897347
-785 785  0.01630843556546118
-786 785  0.00244909751411912
-788 785  0.009024882712692419
-796 785  0.002172894144601377
-797 785  0.001883805647797623
-802 785  0.001631591515025325
-577 786  0.001820374019788474
-785 786  0.0003355912309700669
-786 786  0.007810535605601336
-1128 786  0.001256443987098833
-779 787  0.0278694280628256
-787 787  0.06193981259668287
-949 787  0.03062649908765396
-785 788  0.001423850369810863
-788 788  0.04366344866518873
-789 788  0.003244330702822663
-796 788  0.001417989115834743
-1132 788  0.03456571633990545
-788 789  0.02026741147010662
-789 789  0.2227504892140893
-1094 789  0.1548669885689344
-514 790  0.008049377898348555
-790 790  0.04021974288364626
-791 790  0.02849311321453606
-795 790  0.005888485352821266
-790 791  0.04021974288364626
-791 791  0.05889311029613634
-792 792  0.1060696407972351
-799 792  0.02136027936016361
-1110 792  0.06618131916828844
-777 793  1.96640627202086e-05
-793 793  0.000199737723037119
-794 793  1.966406272020861e-05
-825 793  0.1278772378516793
-793 794  3.205968249284482e-05
-794 794  0.05035968024085258
-795 794  0.003314333795221247
-800 794  0.02124147823020587
-514 795  0.003908683502678743
-568 795  0.0183075963632282
-773 795  0.003952548376903416
-777 795  0.002283122643552819
-790 795  0.008587741971265909
-794 795  0.002283122643552819
-795 795  0.01391925619636464
-573 796  0.01619183306608712
-581 796  0.004965198315835116
-630 796  0.009912532758592284
-776 796  0.001367608629910389
-785 796  0.001370086204695089
-788 796  0.005667081198775904
-796 796  0.02708953246662873
-799 796  0.01044035674673248
-530 797  0.001481789283049012
-581 797  0.002910354294163291
-776 797  0.001407640288568322
-780 797  0.0006392119489119368
-785 797  0.001671815930448331
-797 797  0.01455474950347107
-798 797  0.005444037293553388
-803 797  0.002866477861579545
-805 797  0.01129413035828043
-1119 797  0.01216965786219506
-797 798  0.00648105243726644
-798 798  0.1213169982646069
-1114 798  0.1191860529109802
-581 799  0.009174938259299364
-792 799  0.02850771979805629
-796 799  0.01569177080538385
-799 799  0.03824900208037592
-540 800  0.007439342985845224
-777 800  0.01453102841056699
-794 800  0.01453102841056699
-800 800  0.03532124833562421
-773 801  0.03860036777063306
-801 801  0.2046281153300618
-523 802  9.988409646243638e-06
-577 802  2.727871631739158e-06
-773 802  5.568289864440538e-05
-784 802  7.111133649763519e-05
-785 802  7.371817460457246e-06
-802 802  0.03055429622450982
-1137 802  0.02144542140253185
-530 803  0.01387652166513078
-583 803  0.008982169690104893
-797 803  0.001577656853569151
-803 803  0.03468004196715042
-804 803  0.004085715700539266
-803 804  0.03468004196715042
-804 804  0.02608571834053958
-797 805  0.002008366969679479
-805 805  0.06439410099711014
-1138 805  0.05309997663601025
-806 806  0.207954249930627
-821 806  0.01349798555051323
-807 807  0.2834670560549181
-887 807  0.05157441519278387
-808 808  0.07936053260516683
-897 808  0.01853138087461078
-528 809  0.002673045141394566
-538 809  0.002090950387613578
-809 809  0.01432627788588647
-920 809  0.0111156191121955
-75 810  0.002786154243605049
-810 810  0.0771960043327003
-811 810  0.07512290106614418
-918 810  0.00149192614687986
-810 811  0.0771960043327003
-811 811  0.1131229003061442
-812 812  0.004265268361812286
-909 812  0.009867355183892385
-813 813  0.01954027070213277
-885 813  0.0002534121778379453
-940 813  0.0002653496560293624
-1122 813  0.02067157886613462
-814 814  0.02830913006385971
-815 814  0.02726288784865825
-909 814  0.001532553570697825
-918 814  0.0006148682080056179
-814 815  0.02830913006385971
-815 815  0.03616288393265997
-816 816  0.0205079850325633
-817 816  0.02592216973460077
-927 816  0.002526022369244229
-816 817  0.006986669161115896
-817 817  0.03700904488470814
-818 817  0.01147516453090905
-907 817  0.007306804789957272
-817 818  0.03700904488470813
-818 818  0.03047516415090905
-499 819  0.001633853262516083
-528 819  0.002375281559655103
-819 819  0.01243302892843611
-820 819  0.009258016284850647
-819 820  0.01243302892843611
-820 820  0.01355801701585077
-806 821  0.006954234654625862
-821 821  0.01349798555051323
-888 821  0.005213022865744636
-897 821  0.01431746242031418
-921 821  0.000428045644658355
-822 822  0.02743114645547259
-880 822  0.01784563242422923
-885 822  0.003519434405543951
-935 822  0.004741858572388725
-770 823  0.2758305566139244
-823 823  0.2718391304347622
-824 824  0.0002
-825 824  0.1279772378516793
-793 825  9.986886151855952e-05
-824 825  0.0001
-825 825  0.1279772378516793
-143 826  0.05253409842956077
-826 826  0.0001998543125579779
-144 827  0.05984537169173331
-827 827  0.05923660555884373
-828 828  0.140745281895787
-837 828  0.1087872108014386
-838 828  0.11131169334795
-874 828  0.02298795218084524
-140 829  0.01678060669874037
-829 829  0.05892250351958103
-833 829  0.01675812161640692
-839 829  0.03519316295384665
-140 830  0.01350701571735669
-830 830  0.0626056986503162
-833 830  0.02221119696364045
-840 830  0.03817050159970662
-831 831  0.09305093509008874
-841 831  0.0887732863426738
-869 831  0.01202349830062934
-832 832  0.05913984646017876
-842 832  0.05473001410392463
-869 832  0.01693877374903241
-829 833  0.02300743176882817
-830 833  0.03035354762947573
-833 833  0.06536502674121955
-843 833  0.04558641451909066
-834 834  0.1207221329635831
-844 834  0.01784999839528515
-874 834  0.05326836003683566
-835 835  0.1964786383859093
-845 835  0.0103799999655384
-862 835  0.2318995796605594
-836 836  0.1645374755238136
-846 836  0.1603498963017221
-869 836  0.01100699591087288
-828 837  0.140745281895787
-837 837  0.4029089236604105
-828 838  0.05428581750927225
-838 838  0.3068234234587666
-859 838  0.1931462814608735
-860 838  0.0890078097400584
-829 839  0.05892250351958103
-839 839  0.1377102653694572
-830 840  0.06260569865031622
-840 840  0.1427752113849228
-831 841  0.09305093509008876
-841 841  0.1858420993949347
-832 842  0.05913984646017875
-842 842  0.2071890254987111
-833 843  0.06536502674121954
-843 843  0.3655864145190906
-834 844  0.1207221329635831
-844 844  0.1197220397765233
-835 845  0.1964786383859094
-845 845  0.6253800983655542
-836 846  0.1645374755238136
-846 846  0.3072780402113742
-847 847  0.2786824971151751
-848 847  0.2155859923608576
-847 848  0.1270749109141337
-848 848  0.2155859923608576
-873 848  0.04308522320295062
-849 849  0.2519376152696942
-854 849  0.2125924910206246
-863 849  0.03136559425336743
-864 849  0.06611785644859572
-850 850  0.4578375436506227
-855 850  0.3977900376667386
-873 850  0.02207303230236381
-851 851  1.339820067239498
-856 851  0.8617599205802059
-858 851  0.2524202499316482
-852 852  0.8044515099515441
-857 852  0.8003495927020922
-869 852  0.00977792134478263
-853 853  0.1997879065771989
-876 853  0.02846993447409274
-849 854  0.2519376152696942
-854 854  0.5300727643838391
-850 855  0.4578375436506226
-855 855  0.551494168895637
-851 856  1.339820067239498
-856 856  2.103759888288207
-852 857  0.8044515099515441
-857 857  1.391349724495121
-851 858  0.4780601466592918
-858 858  0.4275616832566078
-873 858  0.05093981980382919
-838 859  0.3068234234587666
-859 859  0.3431462739608739
-838 860  0.1314178953496573
-860 860  1.730707208662083
-861 860  1.641699428146838
-860 861  1.730707208662083
-861 861  2.871699473656839
-835 862  0.1860986461471867
-862 862  0.2358136012299892
-863 862  0.005533066463074931
-849 863  0.05606416253102824
-862 863  0.1198884856378247
-863 863  0.1047338399469532
-869 863  0.006193732174918723
-849 864  0.02262491299310699
-864 864  0.2207916982974485
-865 864  0.1898203160887905
-869 864  0.002563392832267098
-864 865  0.2207916982974485
-865 865  0.237620320868791
-866 866  0.01875702130669307
-869 866  0.02150182985676903
-877 866  0.001007776018379551
-867 867  0.2115002250448753
-878 867  0.02592199514873633
-868 868  0.01744672745892444
-869 868  0.01959989422406455
-871 868  0.0068615973335874
-877 868  0.0009167252276101201
-831 869  0.004277652544827698
-832 869  0.004409832356254128
-836 869  0.004187581908004852
-852 869  0.004101920532422533
-863 869  0.001215503664746272
-864 869  0.002627741759233773
-866 869  0.005137420205803314
-868 869  0.004843553303722979
-869 869  0.02301948565780342
-870 869  0.004817232857402497
-869 870  0.01956887879942859
-870 870  0.01758799859188702
-871 870  0.006963658849763308
-877 870  0.0009171799280344383
-868 871  0.008711000927404886
-870 871  0.008806465925631649
-871 871  0.07746257464992416
-872 872  0.2955921648824598
-882 872  0.01085731727556362
-848 873  0.0885111376844486
-850 873  0.06004750598388407
-858 873  0.04666098678618785
-873 873  0.07919857786041065
-882 873  0.003790852019042492
-828 874  0.009899123140458117
-834 874  0.1028721419133682
-874 874  0.06260203573068722
-878 874  0.00200964117441003
-875 875  0.02610119176195234
-876 875  0.02406994346643991
-882 875  0.00167798373137963
-853 876  0.004399999056640202
-875 876  0.02552634753594662
-876 876  0.02846993447409274
-138 877  0.003970322372452619
-866 877  0.000616616334862829
-868 877  0.0005801374163867748
-870 877  0.0005781860057615934
-877 877  0.01528572934070789
-878 877  0.009795278982989467
-883 877  0.007776205169885589
-867 878  0.0167931044691808
-874 878  0.004932776594236208
-877 878  0.01286758974603248
-878 878  0.02592199514873632
-879 879  0.08062506292350546
-882 879  0.01085731727556362
-822 880  0.02743114645547259
-880 880  0.03074563239842923
-881 881  0.06942598125955839
-882 881  0.01085731727556362
-138 882  0.005884359642788956
-872 882  0.004852738930012023
-873 882  0.002529834727550404
-875 882  0.005175855710690167
-879 882  0.005095049939896242
-881 882  0.005155985578502097
-882 882  0.01085731727556362
-877 883  0.01528572934070789
-883 883  0.07228621615594047
-884 884  0.01219073310470677
-937 884  0.004303038652622925
-1123 884  0.008750007704381021
-535 885  0.007233055798534703
-813 885  0.003244327383216069
-822 885  0.01048696357442695
-885 885  0.01833154993187189
-914 885  0.001804879929133101
-918 885  0.000488965400960133
-886 886  0.03069892868008986
-895 886  0.01719693345925267
-927 886  0.001698690413162683
-937 886  0.003524474624462636
-807 887  0.03746707721091631
-887 887  0.05157441519278387
-896 887  0.01671919608471206
-897 887  0.01558270319732567
-821 888  0.008886870264124687
-888 888  0.03412437670656033
-924 888  0.01139155529348311
-889 889  0.006273538653384737
-917 889  0.007669741488049601
-890 890  0.03010860349890948
-891 890  0.02888211763686513
-908 890  0.00100399961211994
-917 890  0.001995384105343122
-890 891  0.03010860349890949
-891 891  0.04688211619686524
-892 892  0.0135442301691227
-893 892  0.009337962721732261
-899 892  0.003924408800834282
-927 892  0.001772475961012305
-892 893  0.0135442301691227
-893 893  0.02203796231533228
-894 894  0.03109700724541354
-921 894  2.582112257562263e-05
-936 894  3.076147148241834e-05
-1126 894  0.02687449610320036
-579 895  0.0006121947765618673
-886 895  0.0008377126422259026
-895 895  0.0445774089228085
-1125 895  0.03176872369152559
-887 896  0.02022441085999184
-896 896  0.02908660993572379
-897 896  0.01756854754445944
-808 897  0.004360513855162132
-821 897  0.006393579454644784
-887 897  0.005024413380044375
-896 897  0.004682932898399041
-897 897  0.01853138087461078
-913 897  0.004214060454520724
-917 897  0.0001246309721587329
-944 897  0.001291008453868203
-898 898  0.01390583537581733
-914 898  0.002339404868253041
-926 898  0.03301375113484432
-892 899  0.004423419621225885
-899 899  0.01072030581613085
-900 899  0.00580740597577429
-908 899  0.003736709993607725
-899 900  0.01072030581613085
-900 900  0.03350740830257448
-901 901  0.04025952661594935
-913 901  2.051856118270129e-05
-1129 901  0.05793036044128706
-902 902  0.007179126556803894
-906 902  0.008718922408816254
-556 903  0.009709993774629458
-903 903  0.04134997198816331
-987 903  0.01760391604673835
-904 904  0.008324641033037612
-905 904  0.004486544181019496
-939 904  0.001614068474841553
-944 904  0.001567099158187506
-904 905  0.00832464103303761
-905 905  0.008386543518019608
-902 906  0.004679126556803894
-906 906  0.008718922408816254
-922 906  0.0009008540972751597
-939 906  0.001285271122284525
-817 907  0.02506607113613655
-907 907  0.02583935001166341
-923 907  0.01025670433048331
-532 908  0.000292499155021064
-890 908  0.001802761665018293
-899 908  0.005558445265695928
-908 908  0.007472042486884206
-912 908  0.004241646413774165
-917 908  0.0004098903584962235
-927 908  0.0008411096511385652
-554 909  0.001161123334805374
-812 909  0.002265268361812286
-814 909  0.0006960590837098462
-909 909  0.009867355183892385
-910 909  0.001341614853501025
-909 910  0.009867355183892385
-910 910  0.006441615669501156
-911 911  0.02373877031515357
-912 911  0.0193501715378347
-532 912  0.0003638722938778599
-908 912  0.00170285213314922
-911 912  0.01613876757915258
-912 912  0.0193501715378347
-897 913  0.01543164539779126
-901 913  0.006734186126803873
-913 913  0.0189181233883837
-499 914  0.003073724976294847
-511 914  0.001299784730098261
-538 914  0.0004843898066251472
-885 914  0.0009543904130624683
-898 914  0.001677157446132573
-914 914  0.01284379458486224
-935 914  0.00336528949679306
-938 914  0.009775855279802575
-963 914  0.002440084069826731
-915 915  0.01887799841495944
-926 915  0.04036616462179661
-916 916  0.07175018072235473
-991 916  0.08332942064266868
-889 917  0.004373538691384737
-890 917  0.0009288340417040546
-897 917  9.679046066541613e-05
-908 917  0.0001062611591180948
-917 917  0.007669741488049602
-918 917  0.001087296062716872
-930 917  0.001177767946046547
-75 918  0.0004042110443881656
-532 918  8.617349612622082e-05
-810 918  0.002892375746077174
-814 918  0.001645278510677814
-885 918  0.0003812733587880181
-917 918  0.00399285953916823
-918 918  0.006772170020769837
-919 918  0.004045709292940789
-1130 918  0.004753753432982463
-918 919  0.005101115055405544
-919 919  0.01789744077783699
-940 919  0.002093954937998422
-809 920  0.01432627788588647
-920 920  0.02001561519619722
-527 921  0.0004790384345277779
-821 921  0.0006699763470850662
-894 921  0.006281809831012578
-921 921  0.01039280083437153
-923 921  0.005918681287578552
-924 921  0.007852331567483059
-927 921  0.0003791656025174885
-933 921  0.007942247375364437
-937 921  0.0004434451345446738
-906 922  1.532545376228809e-05
-922 922  0.03134760977659256
-930 922  1.964497769671266e-05
-1133 922  0.02956830277942088
-907 923  0.01163573038280261
-921 923  0.005712813591026454
-923 923  0.03178362780410424
-888 924  0.005044455744378779
-921 924  0.003787213122968066
-924 924  0.03334620142643027
-925 924  0.02447248338440741
-924 925  0.03334620142643027
-925 925  0.0501724806345077
-511 926  0.001786855673403372
-898 926  0.006429255755478865
-915 926  0.009177999966959196
-926 926  0.04036616462179662
-579 927  0.0004036607956703073
-816 927  0.003022111914234659
-886 927  0.001480757391267769
-892 927  0.003828100910598467
-908 927  0.00108344732297468
-921 927  0.0004091673134867418
-927 927  0.008432103183035408
-928 927  0.007761229553793364
-937 927  0.0005819387348275373
-927 928  0.0003033149954706448
-928 928  0.02858876260732213
-929 928  0.02198087224497245
-937 928  8.319842440449401e-05
-928 929  0.02858876260732213
-929 929  0.02298087234497246
-917 930  0.003020783387677176
-922 930  0.002737003750895941
-930 930  0.01218165554242968
-931 930  0.01121403728330975
-930 931  0.01218165554242968
-931 931  0.02161403686730977
-932 932  0.05544300641025474
-949 932  0.06528762072500392
-527 933  0.0003892861058939532
-921 933  0.001340716667282153
-933 933  0.0405505325267202
-934 933  0.01790551973456857
-942 933  0.006204209829354447
-933 934  0.0405505325267202
-934 934  0.02550552247056955
-822 935  0.008825044625412099
-914 935  0.003974990604581501
-935 935  0.0208356103168027
-947 935  0.01676870142002394
-894 936  3.798426637685468e-05
-936 936  0.02516388418998334
-937 936  2.572287432225631e-05
-1134 936  0.02213858755811324
-502 937  0.0009308510188288048
-884 937  0.006972812286093009
-886 937  0.002350373142658572
-921 937  0.0003563332503868298
-927 937  0.0004390825271683525
-928 937  0.001454238349828426
-936 937  0.004142765908838729
-937 937  0.007855074867766076
-914 938  0.01284379458486224
-938 938  0.02247585487340259
-904 939  0.004121910053910494
-906 939  0.004401785634565286
-939 939  0.004022831149886777
-813 940  0.00347990398411743
-919 940  0.002181675206045969
-940 940  0.008486027778783729
-954 940  0.02356803958395574
-941 941  0.01029728919443866
-954 941  0.03366802948354273
-933 942  0.04055053252672019
-942 942  0.04430420861015448
-943 943  0.03547542869756949
-944 943  0.02456126604654757
-897 944  0.001906494093816954
-904 944  0.002053343370467679
-943 944  0.0230754284991695
-944 944  0.02456126604654758
-945 945  0.3703205212490625
-961 945  0.07895487623482626
-946 946  0.1435483647575645
-965 946  0.07549099879624292
-935 947  0.0208356103168027
-947 947  0.04546870601202467
-538 948  0.002804809006850223
-948 948  0.01636669233558338
-952 948  0.00172207408614324
-955 948  0.003156881991610024
-957 948  0.003757570867133571
-958 948  0.008999295718421303
-964 948  0.006541884879626206
-780 949  0.003527688672448212
-787 949  0.005347845230501524
-932 949  0.04274300681665472
-949 949  0.06528762072500392
-950 950  0.04775829521461967
-951 950  0.005944919849030421
-955 950  0.01945547812334616
-956 950  0.02488946588201796
-950 951  0.00784997609134226
-951 951  0.07990794709529128
-965 951  0.02881505253248289
-969 951  0.03902640933381142
-975 951  0.05118966836875728
-948 952  0.002698388830761903
-952 952  0.01479698781172081
-960 952  0.01090739773548476
-538 953  0.003984543621843501
-953 953  0.040622894066532
-955 953  0.02192433237461329
-940 954  0.001590537130025061
-941 954  0.007797289194438665
-954 954  0.03366802948354273
-959 954  0.002003968137467726
-948 955  0.00257676859139966
-950 955  0.02471561834984847
-953 955  0.01320429647841619
-955 955  0.03215645693183091
-950 956  0.04775829521461966
-956 956  0.07763945467264532
-519 957  0.006617203945576319
-948 957  0.003373200286415561
-957 957  0.01085292652420506
-958 957  0.004146243831491631
-967 957  0.006984944464507416
-973 957  0.005441157826682218
-948 958  0.01198427246308062
-957 958  0.006150678702683945
-958 958  0.0262500424958768
-954 959  0.03366802948354273
-959 959  0.03620396355466834
-952 960  0.002078228296554667
-960 960  0.01306073009393931
-973 960  0.009100314148404377
-945 961  0.06942049919309083
-961 961  0.07895487623482628
-962 961  0.01107401732878868
-963 961  0.01240845552503707
-968 961  0.01278211235887942
-961 962  0.01113834918658851
-962 962  0.03239343287296422
-966 962  0.05558833395112791
-914 963  0.00475588551202848
-961 963  0.01455928390694178
-963 963  0.109594631764525
-972 963  0.0344279565194329
-538 964  0.007663509157317909
-948 964  0.01011227924199457
-964 964  0.02908047787220006
-946 965  0.04854840275754927
-951 965  0.02527183983034641
-965 965  0.07549099879624292
-968 965  0.006830486233631824
-969 965  0.03315505825299782
-962 966  0.008794989478961807
-966 966  0.06471830236682524
-971 966  0.02405427785548025
-519 967  0.02420168409037794
-957 967  0.009219474515477024
-967 967  0.03108168731236846
-465 968  0.0292007332579842
-961 968  0.006408805799786195
-965 968  0.005748762411683016
-968 968  0.03575835231461739
-951 969  0.04024834751929114
-965 969  0.03898717876164977
-969 969  0.08650732152412599
-970 969  0.05034520448082387
-969 970  0.08650732152412598
-970 970  0.206145210245424
-966 971  0.05802100395703609
-971 971  0.03301916393837059
-972 971  0.008000498722194598
-963 972  0.03263965785558136
-971 972  0.02318534091005487
-972 972  0.07731177900044259
-957 973  0.00178502070848479
-960 973  0.01091258272770016
-973 973  0.01058729410486097
-974 974  0.005628963244658351
-983 974  0.1086511317434904
-951 975  0.07990794709529131
-975 975  0.1894896681751373
-976 976  0.04614585742380604
-977 976  0.01717180389799948
-989 976  0.003108965668848982
-1001 976  0.007670095474669817
-976 977  0.04614585742380604
-977 977  0.03087180403499949
-978 978  0.1104675903104573
-989 978  0.03614724738382001
-558 979  0.0004365638331055172
-979 979  0.005648560866193328
-983 979  0.1063039902882975
-558 980  0.01402526600493269
-560 980  0.02483721564579974
-980 980  0.06713808799758358
-981 981  0.1469555797277405
-982 981  0.04529334233161062
-984 981  0.004942230026113261
-998 981  0.001645866478449362
-981 982  0.1469555797277405
-982 982  0.05049334212361063
-974 983  0.004628963144658341
-979 983  0.0009677247747013971
-983 983  0.1086511317434904
-986 983  0.0003454427303952864
-981 984  0.1469555797277405
-984 984  0.08794224911611766
-524 985  0.0194821966293754
-985 985  0.0177674274029123
-987 985  0.01442955339440103
-989 985  0.002613842741848019
-558 986  0.01234799869920265
-983 986  0.0422645733970262
-986 986  0.0545669678465453
-903 987  0.006150852892462986
-985 987  0.005754132954665552
-987 987  0.04366120217567673
-999 987  0.02823254127879861
-988 988  0.0703815354647411
-993 988  0.03730135475680615
-995 988  0.02554130579815113
-558 989  0.005963673325780709
-976 989  0.01064338744602779
-978 989  0.02706760649005415
-985 989  0.001928213920829602
-989 989  0.03614724738382001
-990 989  0.02324053681363782
-989 990  0.03614724738382002
-990 990  0.1925405523553793
-558 991  0.002836182056199043
-916 991  0.05955018174715463
-991 991  0.08332942064266867
-992 991  0.01043544731946699
-991 992  0.08332942064266868
-992 992  0.351435484488471
-517 993  0.02094209948697999
-988 993  0.02973511206453267
-993 993  0.0758972289969638
-994 993  0.03940774261258517
-993 994  0.07589722899696379
-994 994  0.287707756567046
-517 995  0.007091936783918074
-988 995  0.03186466058113482
-995 995  0.05022594702691818
-997 995  0.02717975237002637
-998 995  0.005974140230181695
-996 996  0.07392341485610768
-997 996  0.08974725993761802
-995 997  0.004585791838812838
-996 997  0.06122341526250766
-997 997  0.08974725993761802
-998 997  0.003811334241688877
-558 998  0.005222985023611815
-981 998  0.04725587006527477
-995 998  0.007828477265006325
-997 998  0.02960125181002267
-998 998  0.03517921881546875
-987 999  0.04366120217567673
-999 999  0.05353254236669866
-558 1000  0.02020927228120075
-1000 1000  0.06646071358731856
-524 1001  0.009024166385107798
-976 1001  0.03338083585573622
-1001 1001  0.02143906598486781
-1002 1002  0.1530376512051281
-1027 1002  0.04890312047371916
-1003 1003  0.2775329080501215
-1035 1003  0.0398555112954956
-1004 1004  0.3780323893372676
-1032 1004  0.1949847397260055
-1005 1005  0.1082314321005827
-1039 1005  0.1680475462134245
-1006 1006  0.01456908479846732
-1007 1006  0.008272877762390409
-1013 1006  0.0004742016329170211
-1019 1006  0.005274052141693458
-1021 1006  0.01259781893617213
-1030 1006  0.002790901981039412
-1035 1006  0.006870177131705836
-1006 1007  0.01456908479846732
-1007 1007  0.2492728541443928
-1008 1008  0.06478174580619263
-1023 1008  0.03498680201478815
-114 1009  0.009864303368119599
-1009 1009  0.04914030404925041
-1010 1009  9.908728126971404e-05
-1018 1009  9.928028189277891e-05
-1019 1009  3.924588614079684e-05
-1021 1009  1.198810312921487e-05
-1022 1009  9.963595870278421e-05
-1040 1009  9.910342467343425e-05
-1009 1010  0.04914030404925041
-1010 1010  0.6380991357692732
-1011 1011  0.1340527400490329
-1040 1011  0.3087982449657204
-1012 1012  0.1928267714401703
-1040 1012  0.3087982449657204
-1006 1013  0.0005505362745983842
-1013 1013  0.002088289199623778
-1023 1013  0.000704722254208448
-1029 1013  0.001133946644822423
-1030 1013  0.001069522852624345
-1037 1013  0.00158754249057476
-1038 1013  0.001548746405940372
-1039 1013  0.001347259410153927
-1014 1014  0.07411818248563802
-1039 1014  0.1680475462134245
-1015 1015  0.02899497950935656
-1029 1015  0.001867119870546095
-1016 1016  0.3242352619217244
-1024 1016  0.08739717026919569
-1034 1016  0.1018617498183449
-1017 1017  0.01364533734131167
-1029 1017  0.001867119870546095
-1009 1018  0.04914030404925041
-1018 1018  0.04729928179229283
-1006 1019  0.002534378931226484
-1009 1019  0.005135408450538448
-1019 1019  0.06290630625337562
-1033 1019  0.05959685114077312
-1020 1020  0.04540867316590868
-1021 1020  0.02850856165564928
-1006 1021  0.008521466014239722
-1009 1021  0.002208122412865323
-1020 1021  0.0168086691047081
-1021 1021  0.02850856165564928
-1009 1022  0.04914030404925041
-1022 1022  0.01759963508370283
-1008 1023  0.02383173839424129
-1013 1023  0.0004741401945795187
-1023 1023  0.03498680201478815
-1024 1023  0.01301110531223596
-1025 1023  0.01310171325372753
-1027 1023  0.009255085251495557
-1016 1024  0.08012700982029282
-1023 1024  0.02216859829078505
-1024 1024  0.12385292413206
-1034 1024  0.04805462776246333
-1023 1025  0.02182076347541613
-1025 1025  0.1182796277398342
-1028 1025  0.06436771072073712
-224 1026  0.02784781750176332
-1026 1026  0.0856222440183788
-1027 1026  0.02785210075284012
-1002 1027  0.03283765764784778
-1023 1027  0.01133624863257827
-1026 1027  0.02123398773966306
-1027 1027  0.04890312047371916
-1028 1027  0.01487890511457545
-1035 1027  0.005038909520881754
-1025 1028  0.06719660013050638
-1027 1028  0.02112044787749787
-1028 1028  0.1710477574258891
-109 1029  0.001640197973334421
-1013 1029  8.984131584140556e-05
-1015 1029  9.498274615620217e-05
-1017 1029  9.533770716166349e-05
-1029 1029  0.001867119870546095
-1006 1030  0.004028544329405392
-1013 1030  0.001329752259689344
-1030 1030  0.02291734456706735
-1031 1030  0.02132198753418528
-1030 1031  0.02291734456706735
-1031 1031  0.1281219747609068
-1004 1032  0.06143241973086459
-1032 1032  0.1949847397260054
-1034 1032  0.04755648988733892
-1036 1032  0.06621961895120788
-1019 1033  0.06290630625337562
-1033 1033  0.08259685160077312
-1016 1034  0.08087820711463256
-1024 1034  0.04161731290577602
-1032 1034  0.1178978918302327
-1034 1034  0.1694309738728105
-1003 1035  0.03273288885779993
-1006 1035  0.00906555707875277
-1027 1035  0.006921300747113141
-1035 1035  0.03985551129549559
-1036 1035  0.01406671708523453
-1032 1036  0.1414436236506199
-1035 1036  0.01094398259724915
-1036 1036  0.1189266545286848
-1013 1037  0.002088289199623779
-1037 1037  0.01003754215257477
-1013 1038  0.002088289199623778
-1038 1038  0.01129874664969038
-1005 1039  0.01223146282057288
-1013 1039  0.0001359099065269315
-1014 1039  0.06091818322483799
-1039 1039  0.1680475462134245
-1009 1040  0.003166993616150146
-1011 1040  0.1097527380564327
-1012 1040  0.02252675389926848
-1040 1040  0.3087982449657204
-647 1041  0.00211815186033333
-677 1041  0.006352832047296143
-1041 1041  0.01280489855845344
-648 1042  0.1346148415060412
-1042 1042  0.05959999020880281
-676 1043  0.02099220839108384
-1043 1043  0.0235456119389732
-1050 1043  0.001061593669875734
-1044 1044  0.01953284291359033
-1050 1044  0.004808909708187654
-1053 1044  0.0009333318458468682
-651 1045  0.01135347889926522
-1045 1045  0.02852471223568966
-1047 1045  0.0136828836464119
-674 1046  0.006078327173654243
-1046 1046  0.01539853994035298
-1058 1046  0.004627828397646399
-532 1047  0.0003353970026791279
-680 1047  0.006468049092205441
-1045 1047  0.01306116223849304
-1047 1047  0.01951864936678242
-653 1048  0.06747685422597234
-1048 1048  0.1410516175449961
-672 1049  0.002035482083724407
-1049 1049  0.01620981673588805
-1054 1049  0.001228456864079547
-77 1050  0.002818165291851867
-660 1050  8.921216502885686e-05
-672 1050  2.199635620334956e-05
-675 1050  0.0001212842738146832
-1043 1050  0.0001150546681760679
-1044 1050  0.0001242421068177792
-1050 1050  0.007333160652441726
-1051 1050  7.318527869573656e-05
-1052 1050  0.0001789413638665126
-646 1051  0.01379407300037537
-667 1051  0.07654661478337116
-1050 1051  0.001860352432717382
-1051 1051  0.08284242548864497
-1056 1051  0.008845188484153853
-672 1052  0.0003031399703474054
-1050 1052  0.006750424310811922
-1052 1052  0.004755973049646703
-681 1053  0.009893731009039716
-1044 1053  0.0003078730259311737
-1053 1053  0.002711408796219971
-652 1054  0.003513028472106132
-1049 1054  0.001180326255607597
-1054 1054  0.003105538986180574
-649 1055  0.002379546354139961
-657 1055  0.003208034688420189
-1055 1055  0.006369699498349443
-1056 1055  0.0007005113099094189
-1058 1055  0.002463583646116846
-664 1056  0.07747445087143598
-1051 1056  0.008009619941229586
-1055 1056  0.0007031089868624823
-1056 1056  0.04550319147949711
-654 1057  0.009028005369760611
-680 1057  0.00422520761440772
-1057 1057  0.01759772702108079
-660 1058  0.001025992365644372
-669 1058  0.004061202683427989
-672 1058  0.0001255217263496964
-1046 1058  0.001538686487980197
-1055 1058  0.001272974247058685
-1058 1058  0.006195453305791682
-677 1059  0.006710970770683924
-1059 1059  0.00540176796751563
-1061 1059  0.003722756958330339
-645 1060  0.09630003872258575
-1060 1060  0.2217000229688926
-621 1061  0.001472716437497991
-1059 1061  0.004972488621553087
-1061 1061  0.02115615872712003
-1062 1062  0.2341655735855472
-1072 1062  0.1162877909424825
-1085 1062  0.06285934904949669
-70 1063  0.008835416720388525
-1063 1063  0.06879257815797121
-1074 1063  0.01257019301063638
-1064 1064  0.09558121278142065
-1075 1064  0.06872176597681982
-1087 1064  0.08699974274180078
-70 1065  0.007987341514557739
-1065 1065  0.03683082955771718
-1077 1065  0.01738783694805822
-1080 1065  0.02885701993604127
-1066 1066  0.08354112319531962
-1068 1066  0.01754317028550596
-1067 1067  0.6093000007470154
-1071 1067  0.3191999064789874
-527 1068  0.004002944190316053
-584 1068  0.001971461686100167
-609 1068  0.005233474827431618
-1066 1068  0.01214111177131779
-1068 1068  0.01754317028550596
-1069 1068  0.007746642417860882
-1092 1068  0.004772379710180124
-1068 1069  0.00767112910888773
-1069 1069  0.03449046817851736
-1086 1069  0.01735646029827682
-1070 1070  0.04596477571514566
-1077 1070  0.02834363723623809
-1113 1070  0.006517913137856713
-1067 1071  0.2325999534334893
-1071 1071  0.3191999064789874
-1088 1071  0.05346508689448282
-1062 1072  0.1402982214385716
-1072 1072  0.1766767678621581
-1075 1072  0.03780296798847921
-527 1073  0.00440645267392877
-1073 1073  0.01038692848789122
-1074 1073  0.01613453602886953
-70 1074  0.0005668180592062543
-527 1074  0.003184584054973412
-1063 1074  0.002271271139130907
-1073 1074  0.005649908198300721
-1074 1074  0.01795532976096158
-1075 1074  0.00150624221967368
-1081 1074  0.004345906141204313
-1086 1074  0.002124743680218939
-1064 1075  0.0438614934803819
-1072 1075  0.02973684838987709
-1074 1075  0.003613116021377893
-1075 1075  0.0943049705701732
-1082 1075  0.04819177008901647
-1076 1076  0.06928100909223454
-1077 1076  0.03712459534923577
-1065 1077  0.006682988407393145
-1070 1077  0.01181738248805265
-1076 1077  0.01578101256973432
-1077 1077  0.03712459534923578
-1078 1077  0.01138352888222821
-1080 1077  0.008012332589582512
-1088 1077  0.01216920620468427
-1095 1077  0.001405596490798533
-1077 1078  0.01134203691997565
-1078 1078  0.09920350325368112
-1084 1078  0.07602758889145694
-1085 1078  0.02229766610317457
-1079 1079  0.5569999767705016
-1117 1079  0.4062000099527207
-1065 1080  0.03013617731697666
-1077 1080  0.02177060522800103
-1080 1080  0.04803317226533439
-1074 1081  0.01795532976096158
-1081 1081  0.01604590552110435
-1075 1082  0.09430497057017322
-1082 1082  0.2808917483781085
-1083 1083  0.2356994624912304
-1087 1083  0.320699001915097
-1078 1084  0.09920350325368112
-1084 1084  0.1326276033810607
-1062 1085  0.1093208345195239
-1078 1085  0.05289007811225765
-1085 1085  0.1839042021651781
-1069 1086  0.01937193492239081
-1074 1086  0.007870529582573086
-1086 1086  0.03084341347749183
-1064 1087  0.0008110412166272464
-1083 1087  0.2336994624912304
-1087 1087  0.3206990019150971
-1071 1088  0.08659997318864784
-1077 1088  0.02705256246998841
-1088 1088  0.08647804876668065
-1089 1089  0.175385221436495
-1106 1089  0.3345991612267034
-1112 1089  0.05932034722361794
-577 1090  0.002950053428144085
-1090 1090  0.009394280978755781
-1091 1090  0.004722190797352253
-1090 1091  0.00301704566221149
-1091 1091  0.1241261391882426
-1093 1091  0.05290591248431776
-1108 1091  0.09624717815304551
-603 1092  0.03228744752564772
-1068 1092  0.004605546273785277
-1092 1092  0.04806122677715817
-1091 1093  0.04706724122373802
-1093 1093  0.1365035818923146
-1098 1093  0.1760001983522047
-636 1094  0.08973768722904406
-789 1094  0.1175033656420125
-1094 1094  0.2890245363533768
-65 1095  0.001594004151586668
-1077 1095  0.001285236114547839
-1095 1095  0.02442936296644875
-1096 1095  0.005457825659281279
-1100 1095  0.005428194353509556
-1105 1095  0.008323146412997286
-1113 1095  0.0177243412163399
-1095 1096  0.006571476261938634
-1096 1096  0.1676258499647094
-1103 1096  0.1120325288688221
-1107 1096  0.07057737800241455
-1112 1096  0.03152811618948742
-1097 1097  0.09398628385474324
-1100 1097  0.100828987509327
-1093 1098  0.08006460919351722
-1098 1098  0.2835002660235161
-1099 1098  0.1075000298312583
-1098 1099  0.2835002660235161
-1099 1099  0.2600000309750083
-1095 1100  0.001616899010265263
-1097 1100  0.08388628375374325
-1100 1100  0.1008289875093271
-1101 1100  0.004294985380093464
-608 1101  0.03965791615114651
-1100 1101  0.03540650853845503
-1101 1101  0.1040558554143926
-1120 1101  0.08482403139628016
-1102 1102  0.4157234555618753
-1112 1102  0.242425508064694
-1096 1103  0.1676258499647094
-1103 1103  0.2439325312562122
-1104 1104  0.06082765961706203
-1118 1104  0.04340000876680176
-1119 1104  0.01303685650800481
-599 1105  0.06160001227072245
-1095 1105  0.01780591591317898
-1105 1105  0.06992316227266117
-1089 1106  0.1062263922626756
-1106 1106  0.442999068132983
-1109 1106  0.1083999794473639
-1096 1107  0.1676258499647094
-1107 1107  0.4981774214465789
-1091 1108  0.1241261391882425
-1108 1108  0.3304471685508459
-1106 1109  0.442999068132983
-1109 1109  0.2748999796138639
-792 1110  0.07996731119167409
-1110 1110  0.1498851900458026
-1114 1110  0.05287448270566146
-1111 1111  0.1491908478242933
-1119 1111  0.02613407252659092
-1089 1112  0.05425915809165077
-1096 1112  0.03725181995953954
-1102 1112  0.1947234615288752
-1112 1112  0.242425508064694
-1070 1113  0.003187338465558733
-1095 1113  0.02273535552515969
-1113 1113  0.02755674135707064
-798 1114  0.04773250616985211
-1110 1114  0.02582091659767817
-1114 1114  0.2148611000891958
-1115 1114  0.09198220549045465
-1116 1114  0.07683700765342699
-1114 1115  0.2148611000891958
-1115 1115  0.3200822100068347
-1114 1116  0.2148611000891957
-1116 1116  0.5230369016363323
-1079 1117  0.22449999921425
-1117 1117  0.4062000099527207
-1120 1117  0.1199664810159885
-1104 1118  0.06082765961706205
-1118 1118  0.08660000531080202
-797 1119  0.009049035677297361
-1104 1119  0.01742765085026024
-1111 1119  0.01369085148279315
-1119 1119  0.02613407252659092
-1101 1120  0.05750994309795388
-1117 1120  0.1817000107384707
-1120 1120  0.2009664109761925
-670 1121  0.007632021197925194
-672 1121  0.002225025024112705
-1121 1121  0.01007956103754005
-813 1122  0.01900208511800962
-1122 1122  0.02164773439817868
-1136 1122  0.001299999317500262
-617 1123  0.01060000366760126
-884 1123  0.007911867973441501
-1123 1123  0.0193500141544857
-527 1124  0.007766460318378678
-1124 1124  0.03560130775451262
-1135 1124  0.01237345993813856
-895 1125  0.0445774089228085
-1125 1125  0.03396872347152561
-894 1126  0.03109700724541354
-1126 1126  0.02697449610320036
-776 1127  0.04186788389119319
-1127 1127  0.04011328287390587
-786 1128  0.007810535605601336
-1128 1128  0.003256443987098833
-901 1129  0.04019593664417826
-1129 1129  0.05799327281272631
-1135 1129  6.743832417317901e-05
-918 1130  0.006772170020769838
-1130 1130  0.04925375365548246
-1131 1131  0.0868999904950067
-1136 1131  0.04719998265580642
-788 1132  0.04366344866518873
-1132 1132  0.04036571610790546
-922 1133  0.03134760977659256
-1133 1133  0.02966830277942088
-936 1134  0.02516388418998334
-1134 1134  0.02223858755811324
-1124 1135  0.01439541418489085
-1129 1135  0.03454367341598973
-1135 1135  0.04268843333067113
-1122 1136  0.01021354363665724
-1131 1136  0.04589998393500563
-1136 1136  0.04719998265580641
-802 1137  0.03055429622450982
-1137 1137  0.02154542140253185
-805 1138  0.06439410099711014
-1138 1138  0.06159998088601238
diff --git a/omp/preconditioner/isai_kernels.cpp b/omp/preconditioner/isai_kernels.cpp
index d9a7e97f76b..d1482c64074 100644
--- a/omp/preconditioner/isai_kernels.cpp
+++ b/omp/preconditioner/isai_kernels.cpp
@@ -85,7 +85,7 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                       const matrix::Csr<ValueType, IndexType> *mtx,
                       matrix::Csr<ValueType, IndexType> *inverse_mtx,
                       IndexType *excess_rhs_ptrs, IndexType *excess_nz_ptrs,
-                      Callable trs_solve)
+                      Callable direct_solve)
 {
     /*
     Consider: aiM := inverse_mtx; M := mtx
@@ -110,10 +110,10 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
     auto i_vals = inverse_mtx->get_values();
 
     auto num_threads = static_cast<size_type>(omp_get_max_threads());
-    // RHS for local trisystem
+    // RHS for local dense system
     gko::Array<ValueType> rhs_array{exec, row_size_limit * num_threads};
-    // memory for dense trisystem
-    gko::Array<ValueType> trisystem_array{
+    // memory for dense dense system
+    gko::Array<ValueType> dense_system_array{
         exec, row_size_limit * row_size_limit * num_threads};
 
 #pragma omp parallel
@@ -121,8 +121,8 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
         auto thread_num = static_cast<size_type>(omp_get_thread_num());
 
         auto rhs = rhs_array.get_data() + thread_num * row_size_limit;
-        auto trisystem_ptr = trisystem_array.get_data() +
-                             thread_num * row_size_limit * row_size_limit;
+        auto dense_system_ptr = dense_system_array.get_data() +
+                                thread_num * row_size_limit * row_size_limit;
 
 #pragma omp for
         for (size_type row = 0; row < num_rows; ++row) {
@@ -133,12 +133,16 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                 // short rows: treat directly as dense system
                 excess_rhs_ptrs[row] = 0;
                 excess_nz_ptrs[row] = 0;
-                auto trisystem = range<accessor::row_major<ValueType, 2>>(
-                    trisystem_ptr, static_cast<size_type>(i_size),
+                auto dense_system = range<accessor::row_major<ValueType, 2>>(
+                    dense_system_ptr, static_cast<size_type>(i_size),
                     static_cast<size_type>(i_size),
                     static_cast<size_type>(i_size));
-                std::fill_n(trisystem_ptr, i_size * i_size, zero<ValueType>());
-                IndexType k = zero<IndexType>();
+                std::fill_n(dense_system_ptr, i_size * i_size,
+                            zero<ValueType>());
+                // For general ISAI, the index of the one in the rhs depends on
+                // the number of nonzeros in the lower half of the matrix of the
+                // according row.
+                IndexType rhs_one_idx = zero<IndexType>();
 
                 for (size_type i = 0; i < i_size; ++i) {
                     const auto col = i_cols[i_begin + i];
@@ -148,13 +152,13 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                         m_cols + m_begin, m_size, i_cols + i_begin, i_size,
                         [&](IndexType, IndexType m_idx, IndexType i_idx) {
                             if (m_cols[m_idx + m_begin] < row && col == row)
-                                k++;
-                            trisystem(i, i_idx) = m_vals[m_idx + m_begin];
+                                rhs_one_idx++;
+                            dense_system(i, i_idx) = m_vals[m_idx + m_begin];
                         });
                 }
 
                 // solve dense triangular system
-                trs_solve(trisystem, rhs, k);
+                direct_solve(dense_system, rhs, rhs_one_idx);
 
                 // write triangular solution to inverse
                 for (size_type i = 0; i < i_size; ++i) {
@@ -278,7 +282,7 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
     using std::swap;
     auto general_solve =
         [](const range<accessor::row_major<ValueType, 2>> system,
-           ValueType *rhs, const IndexType k) {
+           ValueType *rhs, const IndexType rhs_one_idx) {
             const IndexType size = system.length(0);
             if (size <= 0) {
                 return;
@@ -286,7 +290,7 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
             // RHS is the identity: zero everywhere except for the diagonal
             // entry
             std::fill_n(rhs, size, zero<ValueType>());
-            rhs[k] = one<ValueType>();
+            rhs[rhs_one_idx] = one<ValueType>();
 
             // fill in transposed system
             ValueType *transposed_system = new ValueType[size * size];
diff --git a/reference/preconditioner/isai_kernels.cpp b/reference/preconditioner/isai_kernels.cpp
index 80ce8f4910b..1003cd0ba41 100644
--- a/reference/preconditioner/isai_kernels.cpp
+++ b/reference/preconditioner/isai_kernels.cpp
@@ -80,7 +80,7 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                       const matrix::Csr<ValueType, IndexType> *mtx,
                       matrix::Csr<ValueType, IndexType> *inverse_mtx,
                       IndexType *excess_rhs_ptrs, IndexType *excess_nz_ptrs,
-                      Callable trs_solve)
+                      Callable direct_solve)
 {
     /*
     Consider: aiM := inverse_mtx; M := mtx
@@ -103,13 +103,13 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
     const auto i_row_ptrs = inverse_mtx->get_const_row_ptrs();
     const auto i_cols = inverse_mtx->get_const_col_idxs();
     auto i_vals = inverse_mtx->get_values();
-    // RHS for local trisystem
+    // RHS for local dense system
     gko::Array<ValueType> rhs_array{exec, row_size_limit};
     auto rhs = rhs_array.get_data();
-    // memory for dense trisystem
-    gko::Array<ValueType> trisystem_array{exec,
-                                          row_size_limit * row_size_limit};
-    auto trisystem_ptr = trisystem_array.get_data();
+    // memory for dense dense system
+    gko::Array<ValueType> dense_system_array{exec,
+                                             row_size_limit * row_size_limit};
+    auto dense_system_ptr = dense_system_array.get_data();
     // stores the next free index in the excess rhs/solution
     IndexType excess_rhs_begin{};
     // stores the next free non-zero index in the excess system
@@ -125,11 +125,14 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
             // short rows: treat directly as dense system
             // we need this ugly workaround to get rid of a few
             // warnings and compilation issues
-            auto trisystem = range<accessor::row_major<ValueType, 2>>(
-                trisystem_ptr, static_cast<size_type>(i_size),
+            auto dense_system = range<accessor::row_major<ValueType, 2>>(
+                dense_system_ptr, static_cast<size_type>(i_size),
                 static_cast<size_type>(i_size), static_cast<size_type>(i_size));
-            std::fill_n(trisystem_ptr, i_size * i_size, zero<ValueType>());
-            IndexType k = zero<IndexType>();
+            std::fill_n(dense_system_ptr, i_size * i_size, zero<ValueType>());
+            // For general ISAI, the index of the one in the rhs depends on
+            // the number of nonzeros in the lower half of the matrix of the
+            // according row.
+            IndexType rhs_one_idx = zero<IndexType>();
 
             for (size_type i = 0; i < i_size; ++i) {
                 const auto col = i_cols[i_begin + i];
@@ -138,13 +141,14 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                 forall_matching(
                     m_cols + m_begin, m_size, i_cols + i_begin, i_size,
                     [&](IndexType, IndexType m_idx, IndexType i_idx) {
-                        if (m_cols[m_idx + m_begin] < row && col == row) k++;
-                        trisystem(i, i_idx) = m_vals[m_idx + m_begin];
+                        if (m_cols[m_idx + m_begin] < row && col == row)
+                            rhs_one_idx++;
+                        dense_system(i, i_idx) = m_vals[m_idx + m_begin];
                     });
             }
 
             // solve dense triangular system
-            trs_solve(trisystem, rhs, k);
+            direct_solve(dense_system, rhs, rhs_one_idx);
 
             // write triangular solution to inverse
             for (size_type i = 0; i < i_size; ++i) {
@@ -264,7 +268,7 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
     using std::swap;
     auto general_solve =
         [](const range<accessor::row_major<ValueType, 2>> system,
-           ValueType *rhs, const IndexType k) {
+           ValueType *rhs, const IndexType rhs_one_idx) {
             const IndexType size = system.length(0);
             if (size <= 0) {
                 return;
@@ -272,7 +276,7 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
             // RHS is the identity: zero everywhere except for the diagonal
             // entry
             std::fill_n(rhs, size, zero<ValueType>());
-            rhs[k] = one<ValueType>();
+            rhs[rhs_one_idx] = one<ValueType>();
 
             // fill in transposed system
             ValueType *transposed_system = new ValueType[size * size];

From 8354b460427df95840dc9758c3170e191113d244 Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Tue, 5 Jan 2021 17:34:24 +0000
Subject: [PATCH 0513/1008] Format files

Co-authored-by: fritzgoebel <fritzgoebel@users.noreply.github.com>
---
 cuda/test/preconditioner/isai_kernels.cpp | 1 +
 reference/preconditioner/isai_kernels.cpp | 1 +
 2 files changed, 2 insertions(+)

diff --git a/cuda/test/preconditioner/isai_kernels.cpp b/cuda/test/preconditioner/isai_kernels.cpp
index 4c891666ea8..2dfc528103d 100644
--- a/cuda/test/preconditioner/isai_kernels.cpp
+++ b/cuda/test/preconditioner/isai_kernels.cpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <random>
 
+
 #include <gtest/gtest.h>
 
 
diff --git a/reference/preconditioner/isai_kernels.cpp b/reference/preconditioner/isai_kernels.cpp
index 1003cd0ba41..01ba055b76c 100644
--- a/reference/preconditioner/isai_kernels.cpp
+++ b/reference/preconditioner/isai_kernels.cpp
@@ -32,6 +32,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/preconditioner/isai_kernels.hpp"
 
+
 #include <algorithm>
 #include <memory>
 

From 3327179093d932ed2d15adb3ef1da759bd435ad9 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Tue, 5 Jan 2021 18:40:55 +0100
Subject: [PATCH 0514/1008] Adjust tolerance type in gmres

---
 core/preconditioner/isai.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/preconditioner/isai.cpp b/core/preconditioner/isai.cpp
index 52693e84750..18d693a8f25 100644
--- a/core/preconditioner/isai.cpp
+++ b/core/preconditioner/isai.cpp
@@ -200,7 +200,7 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
                                 .with_max_iters(excess_dim)
                                 .on(exec),
                             gko::stop::RelativeResidualNorm<ValueType>::build()
-                                .with_tolerance(1e-6)
+                                .with_tolerance(remove_complex<ValueType>{1e-6})
                                 .on(exec))
                         .on(exec);
                 excess_solution->copy_from(excess_rhs.get());

From 86731af6c1e0afac19895964a0b777b961a6b635 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Wed, 20 Jan 2021 16:17:20 +0100
Subject: [PATCH 0515/1008] Add Spd Isai

---
 benchmark/utils/preconditioners.hpp           |   6 +
 common/preconditioner/isai_kernels.hpp.inc    |  63 +-
 core/device_hooks/common_kernels.inc.cpp      |   6 +
 core/preconditioner/isai.cpp                  |  72 +-
 core/preconditioner/isai_kernels.hpp          |  18 +-
 cuda/preconditioner/isai_kernels.cu           |  25 +-
 cuda/test/preconditioner/isai_kernels.cpp     |  39 +-
 dpcpp/preconditioner/isai_kernels.dp.cpp      |  22 +-
 hip/preconditioner/isai_kernels.hip.cpp       |  26 +-
 hip/test/preconditioner/isai_kernels.hip.cpp  |  39 +-
 include/ginkgo/core/preconditioner/isai.hpp   |  73 +-
 matrices/CMakeLists.txt                       |   5 +
 matrices/test/isai_spd.mtx                    | 510 +++++++++++++
 matrices/test/isai_spd_excess.mtx             | 668 ++++++++++++++++++
 matrices/test/isai_spd_excess_rhs.mtx         |  38 +
 matrices/test/isai_spd_inv.mtx                | 510 +++++++++++++
 matrices/test/isai_spd_inv_partial.mtx        | 510 +++++++++++++
 omp/preconditioner/isai_kernels.cpp           |  47 +-
 omp/test/preconditioner/isai_kernels.cpp      |  39 +-
 reference/preconditioner/isai_kernels.cpp     |  46 +-
 .../test/preconditioner/isai_kernels.cpp      | 307 +++++++-
 21 files changed, 2953 insertions(+), 116 deletions(-)
 create mode 100644 matrices/test/isai_spd.mtx
 create mode 100644 matrices/test/isai_spd_excess.mtx
 create mode 100644 matrices/test/isai_spd_excess_rhs.mtx
 create mode 100644 matrices/test/isai_spd_inv.mtx
 create mode 100644 matrices/test/isai_spd_inv_partial.mtx

diff --git a/benchmark/utils/preconditioners.hpp b/benchmark/utils/preconditioners.hpp
index ade7472c040..6af0d2d7a12 100644
--- a/benchmark/utils/preconditioners.hpp
+++ b/benchmark/utils/preconditioners.hpp
@@ -311,6 +311,12 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
                  .with_sparsity_power(FLAGS_isai_power)
                  .on(exec);
          }},
+        {"spd-isai",
+         [](std::shared_ptr<const gko::Executor> exec) {
+             return gko::preconditioner::SpdIsai<>::build()
+                 .with_sparsity_power(FLAGS_isai_power)
+                 .on(exec);
+         }},
         {"overhead", [](std::shared_ptr<const gko::Executor> exec) {
              return gko::Overhead<etype>::build()
                  .with_criteria(gko::stop::ResidualNorm<etype>::build()
diff --git a/common/preconditioner/isai_kernels.hpp.inc b/common/preconditioner/isai_kernels.hpp.inc
index 8d7a9733b8a..3fb91135bea 100644
--- a/common/preconditioner/isai_kernels.hpp.inc
+++ b/common/preconditioner/isai_kernels.hpp.inc
@@ -55,9 +55,6 @@ __forceinline__ __device__ void generic_generate(
     static_assert(subwarp_size >= row_size_limit, "incompatible subwarp_size");
     const auto row = thread::get_subwarp_id_flat<subwarp_size, IndexType>();
 
-    // For general ISAI, the index of the one in the rhs depends on the number
-    // of nonzeros in the lower half of the matrix of the according row.
-    __shared__ size_type rhs_one_idxs[subwarps_per_block];
     if (row >= num_rows) {
         return;
     }
@@ -117,6 +114,7 @@ __forceinline__ __device__ void generic_generate(
 
         subwarp.sync();
 
+        IndexType rhs_one_idx{};
         for (IndexType nz = 0; nz < i_row_size; ++nz) {
             auto col = i_col_idxs[i_row_begin + nz];
             auto m_row_begin = m_row_ptrs[col];
@@ -128,10 +126,10 @@ __forceinline__ __device__ void generic_generate(
                 i_row_size, subwarp,
                 [&](IndexType, IndexType m_idx, IndexType i_idx,
                     config::lane_mask_type, bool valid) {
+                    rhs_one_idx += popcnt(subwarp.ballot(
+                        valid && m_col_idxs[m_row_begin + m_idx] < row &&
+                        col == row));
                     if (valid) {
-                        if (m_col_idxs[m_row_begin + m_idx] < row && col == row)
-                            atomic_add(&rhs_one_idxs[row % subwarps_per_block],
-                                       size_type{1});
                         dense_system(nz, i_idx) = m_values[m_row_begin + m_idx];
                     }
                 });
@@ -147,8 +145,8 @@ __forceinline__ __device__ void generic_generate(
             local_row[i] = dense_system(i, local_id);
         }
 
-        const auto rhs = direct_solve(i_row_size, local_row, subwarp,
-                                      rhs_one_idxs[row % subwarps_per_block]);
+        const auto rhs =
+            direct_solve(i_row_size, local_row, subwarp, rhs_one_idx);
 
         // Write back:
         if (local_id < i_row_size) {
@@ -251,12 +249,12 @@ __global__ __launch_bounds__(default_block_size) void generate_general_inverse(
     const IndexType *__restrict__ i_row_ptrs,
     const IndexType *__restrict__ i_col_idxs, ValueType *__restrict__ i_values,
     IndexType *__restrict__ excess_rhs_sizes,
-    IndexType *__restrict__ excess_nnz)
+    IndexType *__restrict__ excess_nnz, bool spd)
 {
-    auto general_solve = [](IndexType num_elems,
-                            ValueType *__restrict__ local_row,
-                            group::thread_block_tile<subwarp_size> &subwarp,
-                            size_type rhs_one_idx) {
+    auto general_solve = [spd](IndexType num_elems,
+                               ValueType *__restrict__ local_row,
+                               group::thread_block_tile<subwarp_size> &subwarp,
+                               size_type rhs_one_idx) {
         const int local_id = subwarp.thread_rank();
         ValueType rhs =
             local_id == rhs_one_idx ? one<ValueType>() : zero<ValueType>();
@@ -276,7 +274,13 @@ __global__ __launch_bounds__(default_block_size) void generate_general_inverse(
                 subwarp, piv, i, local_row, &rhs, status);
         }
 
-        return subwarp.shfl(rhs, perm);
+        ValueType sol = subwarp.shfl(rhs, perm);
+
+        if (spd) {
+            sol /= sqrt(subwarp.shfl(rhs, num_elems - 1));
+        }
+
+        return sol;
     };
 
     generic_generate<subwarp_size, subwarps_per_block>(
@@ -297,7 +301,7 @@ __global__ __launch_bounds__(default_block_size) void generate_excess_system(
     IndexType *__restrict__ excess_row_ptrs,
     IndexType *__restrict__ excess_col_idxs,
     ValueType *__restrict__ excess_values, ValueType *__restrict__ excess_rhs,
-    const size_type e_start, const size_type e_end)
+    size_type e_start, size_type e_end)
 {
     const auto row =
         thread::get_subwarp_id_flat<subwarp_size, IndexType>() + e_start;
@@ -359,13 +363,38 @@ __global__ __launch_bounds__(default_block_size) void generate_excess_system(
     }
 }
 
+template <int subwarp_size, typename ValueType, typename IndexType>
+__global__ __launch_bounds__(default_block_size) void scale_excess_solution(
+    const IndexType *__restrict__ excess_block_ptrs,
+    ValueType *__restrict__ excess_solution, size_type e_start, size_type e_end)
+{
+    const auto warp_id = thread::get_subwarp_id_flat<subwarp_size, IndexType>();
+    auto subwarp =
+        group::tiled_partition<subwarp_size>(group::this_thread_block());
+    const int local_id = subwarp.thread_rank();
+
+    if (warp_id >= e_end - e_start) {
+        return;
+    }
+
+    const IndexType offset = excess_block_ptrs[e_start];
+    const IndexType block_begin = excess_block_ptrs[warp_id] - offset;
+    const IndexType block_end = excess_block_ptrs[warp_id + 1] - offset;
+    const ValueType scal =
+        one<ValueType>() / sqrt(excess_solution[block_end - 1]);
+
+    for (size_type i = block_begin + local_id; i < block_end;
+         i += subwarp_size) {
+        excess_solution[i] *= scal;
+    }
+}
+
 template <int subwarp_size, typename ValueType, typename IndexType>
 __global__ __launch_bounds__(default_block_size) void copy_excess_solution(
     IndexType num_rows, const IndexType *__restrict__ i_row_ptrs,
     const IndexType *__restrict__ excess_rhs_ptrs,
     const ValueType *__restrict__ excess_solution,
-    ValueType *__restrict__ i_values, const size_type e_start,
-    const size_type e_end)
+    ValueType *__restrict__ i_values, size_type e_start, size_type e_end)
 {
     const auto excess_row =
         thread::get_subwarp_id_flat<subwarp_size, IndexType>();
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index ff25b9f85f7..eceae46dd8b 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -1038,6 +1038,12 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL);
 
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_ISAI_SCALE_EXCESS_SOLUTION_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_SCALE_EXCESS_SOLUTION_KERNEL);
+
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_ISAI_SCATTER_EXCESS_SOLUTION_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/preconditioner/isai.cpp b/core/preconditioner/isai.cpp
index 18d693a8f25..6fa5f7c7aee 100644
--- a/core/preconditioner/isai.cpp
+++ b/core/preconditioner/isai.cpp
@@ -51,6 +51,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/base/utils.hpp"
+#include "core/factorization/factorization_kernels.hpp"
 #include "core/preconditioner/isai_kernels.hpp"
 
 
@@ -63,7 +64,11 @@ GKO_REGISTER_OPERATION(generate_tri_inverse, isai::generate_tri_inverse);
 GKO_REGISTER_OPERATION(generate_general_inverse,
                        isai::generate_general_inverse);
 GKO_REGISTER_OPERATION(generate_excess_system, isai::generate_excess_system);
+GKO_REGISTER_OPERATION(scale_excess_solution, isai::scale_excess_solution);
 GKO_REGISTER_OPERATION(scatter_excess_solution, isai::scatter_excess_solution);
+GKO_REGISTER_OPERATION(initialize_row_ptrs_l,
+                       factorization::initialize_row_ptrs_l);
+GKO_REGISTER_OPERATION(initialize_l, factorization::initialize_l);
 
 
 }  // namespace isai
@@ -115,7 +120,7 @@ std::shared_ptr<Csr> extend_sparsity(std::shared_ptr<const Executor> &exec,
 template <isai_type IsaiType, typename ValueType, typename IndexType>
 void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
     std::shared_ptr<const LinOp> input, bool skip_sorting, int power,
-    int excess_limit)
+    IndexType excess_limit)
 {
     using Dense = matrix::Dense<ValueType>;
     using LowerTrs = solver::LowerTrs<ValueType, IndexType>;
@@ -124,12 +129,39 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
     using bj = preconditioner::Jacobi<ValueType, IndexType>;
     GKO_ASSERT_IS_SQUARE_MATRIX(input);
     auto exec = this->get_executor();
-    auto to_invert = convert_to_with_sorting<Csr>(exec, input, skip_sorting);
-    auto inverted = extend_sparsity(exec, to_invert, power);
-    auto num_rows = inverted->get_size()[0];
     auto is_lower = IsaiType == isai_type::lower;
     auto is_general = IsaiType == isai_type::general;
-    auto excess_lim = excess_limit == 0 ? num_rows : excess_limit;
+    auto is_spd = IsaiType == isai_type::spd;
+    auto to_invert = convert_to_with_sorting<Csr>(exec, input, skip_sorting);
+    auto num_rows = to_invert->get_size()[0];
+    std::shared_ptr<Csr> inverted;
+    if (!is_spd) {
+        inverted = extend_sparsity(exec, to_invert, power);
+    } else {
+        // Extract lower triangular part: compute non-zeros
+        Array<IndexType> inverted_row_ptrs{exec, num_rows + 1};
+        exec->run(isai::make_initialize_row_ptrs_l(
+            to_invert.get(), inverted_row_ptrs.get_data()));
+
+        // Get nnz from device memory
+        auto inverted_nnz = static_cast<size_type>(
+            exec->copy_val_to_host(inverted_row_ptrs.get_data() + num_rows));
+
+        // Init arrays
+        Array<IndexType> inverted_col_idxs{exec, inverted_nnz};
+        Array<ValueType> inverted_vals{exec, inverted_nnz};
+        std::shared_ptr<Csr> inverted_base = Csr::create(
+            exec, dim<2>{num_rows, num_rows}, std::move(inverted_vals),
+            std::move(inverted_col_idxs), std::move(inverted_row_ptrs));
+
+        // Extract lower factor: columns and values
+        exec->run(isai::make_initialize_l(to_invert.get(), inverted_base.get(),
+                                          false));
+
+        inverted = power == 1
+                       ? std::move(inverted_base)
+                       : extend_sparsity<Csr>(exec, inverted_base, power);
+    }
 
     // This stores the beginning of the RHS for the sparse block associated with
     // each row of inverted_l
@@ -138,10 +170,10 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
     // system of excess blocks
     Array<IndexType> excess_row_ptrs_full{exec, num_rows + 1};
 
-    if (is_general) {
+    if (is_general || is_spd) {
         exec->run(isai::make_generate_general_inverse(
             lend(to_invert), lend(inverted), excess_block_ptrs.get_data(),
-            excess_row_ptrs_full.get_data()));
+            excess_row_ptrs_full.get_data(), is_spd));
     } else {
         exec->run(isai::make_generate_tri_inverse(
             lend(to_invert), lend(inverted), excess_block_ptrs.get_data(),
@@ -150,6 +182,7 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
 
     auto total_excess_dim =
         exec->copy_val_to_host(excess_block_ptrs.get_const_data() + num_rows);
+    auto excess_lim = excess_limit == 0 ? total_excess_dim : excess_limit;
     // if we had long rows:
     if (total_excess_dim > 0) {
         bool done = false;
@@ -190,7 +223,7 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
             auto rhs_copy = Dense::create(exec->get_master());
             rhs_copy->copy_from(excess_rhs.get());
             std::unique_ptr<LinOpFactory> trs_factory;
-            if (is_general) {
+            if (is_general || is_spd) {
                 trs_factory =
                     Gmres::build()
                         .with_preconditioner(
@@ -213,6 +246,11 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
             }
             trs_factory->generate(share(excess_system->transpose()))
                 ->apply(lend(excess_rhs), lend(excess_solution));
+            if (is_spd) {
+                exec->run(isai::make_scale_excess_solution(
+                    excess_block_ptrs.get_const_data(), lend(excess_solution),
+                    excess_start, block));
+            }
             // and copy the results back to the original ISAI
             exec->run(isai::make_scatter_excess_solution(
                 excess_block_ptrs.get_const_data(), lend(excess_solution),
@@ -227,11 +265,16 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
 template <isai_type IsaiType, typename ValueType, typename IndexType>
 std::unique_ptr<LinOp> Isai<IsaiType, ValueType, IndexType>::transpose() const
 {
+    auto is_spd = IsaiType == isai_type::spd;
+    if (is_spd) {
+        return this->clone();
+    }
+
     std::unique_ptr<transposed_type> transp{
         new transposed_type{this->get_executor()}};
     transp->set_size(gko::transpose(this->get_size()));
     transp->approximate_inverse_ =
-        share(as<Csr>(this->get_approximate_inverse()->transpose()));
+        share(as<Csr>(this->get_approximate_inverse())->transpose());
 
     return std::move(transp);
 }
@@ -241,11 +284,16 @@ template <isai_type IsaiType, typename ValueType, typename IndexType>
 std::unique_ptr<LinOp> Isai<IsaiType, ValueType, IndexType>::conj_transpose()
     const
 {
+    auto is_spd = IsaiType == isai_type::spd;
+    if (is_spd) {
+        return this->clone();
+    }
+
     std::unique_ptr<transposed_type> transp{
         new transposed_type{this->get_executor()}};
     transp->set_size(gko::transpose(this->get_size()));
     transp->approximate_inverse_ =
-        share(as<Csr>(this->get_approximate_inverse()->conj_transpose()));
+        share(as<Csr>(this->get_approximate_inverse())->conj_transpose());
 
     return std::move(transp);
 }
@@ -263,6 +311,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_UPPER_ISAI);
     class Isai<isai_type::general, ValueType, IndexType>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_GENERAL_ISAI);
 
+#define GKO_DECLARE_SPD_ISAI(ValueType, IndexType) \
+    class Isai<isai_type::spd, ValueType, IndexType>
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_SPD_ISAI);
+
 
 }  // namespace preconditioner
 }  // namespace gko
diff --git a/core/preconditioner/isai_kernels.hpp b/core/preconditioner/isai_kernels.hpp
index db92adad459..c9c71f6efe9 100644
--- a/core/preconditioner/isai_kernels.hpp
+++ b/core/preconditioner/isai_kernels.hpp
@@ -57,7 +57,7 @@ namespace kernels {
         std::shared_ptr<const DefaultExecutor> exec,                           \
         const matrix::Csr<ValueType, IndexType> *input,                        \
         matrix::Csr<ValueType, IndexType> *inverse,                            \
-        IndexType *excess_rhs_ptrs, IndexType *excess_nz_ptrs)
+        IndexType *excess_rhs_ptrs, IndexType *excess_nz_ptrs, bool spd)
 
 #define GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL(ValueType, IndexType) \
     void generate_excess_system(                                             \
@@ -66,16 +66,22 @@ namespace kernels {
         const matrix::Csr<ValueType, IndexType> *inverse,                    \
         const IndexType *excess_rhs_ptrs, const IndexType *excess_nz_ptrs,   \
         matrix::Csr<ValueType, IndexType> *excess_system,                    \
-        matrix::Dense<ValueType> *excess_rhs, const size_type excess_start,  \
-        const size_type excess_end)
+        matrix::Dense<ValueType> *excess_rhs, size_type e_start,             \
+        size_type e_end)
+
+#define GKO_DECLARE_ISAI_SCALE_EXCESS_SOLUTION_KERNEL(ValueType, IndexType) \
+    void scale_excess_solution(std::shared_ptr<const DefaultExecutor> exec, \
+                               const IndexType *excess_block_ptrs,          \
+                               matrix::Dense<ValueType> *excess_solution,   \
+                               size_type e_start, size_type e_end)
 
 #define GKO_DECLARE_ISAI_SCATTER_EXCESS_SOLUTION_KERNEL(ValueType, IndexType) \
     void scatter_excess_solution(                                             \
         std::shared_ptr<const DefaultExecutor> exec,                          \
         const IndexType *excess_rhs_ptrs,                                     \
         const matrix::Dense<ValueType> *excess_solution,                      \
-        matrix::Csr<ValueType, IndexType> *inverse, const size_type e_start,  \
-        const size_type e_end)
+        matrix::Csr<ValueType, IndexType> *inverse, size_type e_start,        \
+        size_type e_end)
 
 #define GKO_DECLARE_ALL_AS_TEMPLATES                                        \
     constexpr auto row_size_limit = 32;                                     \
@@ -86,6 +92,8 @@ namespace kernels {
     template <typename ValueType, typename IndexType>                       \
     GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL(ValueType, IndexType);   \
     template <typename ValueType, typename IndexType>                       \
+    GKO_DECLARE_ISAI_SCALE_EXCESS_SOLUTION_KERNEL(ValueType, IndexType);    \
+    template <typename ValueType, typename IndexType>                       \
     GKO_DECLARE_ISAI_SCATTER_EXCESS_SOLUTION_KERNEL(ValueType, IndexType)
 
 
diff --git a/cuda/preconditioner/isai_kernels.cu b/cuda/preconditioner/isai_kernels.cu
index fdfc1808ac9..527dd79f1d4 100644
--- a/cuda/preconditioner/isai_kernels.cu
+++ b/cuda/preconditioner/isai_kernels.cu
@@ -114,7 +114,7 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
                               const matrix::Csr<ValueType, IndexType> *input,
                               matrix::Csr<ValueType, IndexType> *inverse,
                               IndexType *excess_rhs_ptrs,
-                              IndexType *excess_nz_ptrs)
+                              IndexType *excess_nz_ptrs, bool spd)
 {
     const auto num_rows = input->get_size()[0];
 
@@ -126,7 +126,7 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
             input->get_const_col_idxs(),
             as_cuda_type(input->get_const_values()), inverse->get_row_ptrs(),
             inverse->get_col_idxs(), as_cuda_type(inverse->get_values()),
-            excess_rhs_ptrs, excess_nz_ptrs);
+            excess_rhs_ptrs, excess_nz_ptrs, spd);
     components::prefix_sum(exec, excess_rhs_ptrs, num_rows + 1);
     components::prefix_sum(exec, excess_nz_ptrs, num_rows + 1);
 }
@@ -143,7 +143,7 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor> exec,
                             const IndexType *excess_nz_ptrs,
                             matrix::Csr<ValueType, IndexType> *excess_system,
                             matrix::Dense<ValueType> *excess_rhs,
-                            const size_type e_start, const size_type e_end)
+                            size_type e_start, size_type e_end)
 {
     const auto num_rows = input->get_size()[0];
 
@@ -163,12 +163,29 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void scale_excess_solution(std::shared_ptr<const DefaultExecutor>,
+                           const IndexType *excess_block_ptrs,
+                           matrix::Dense<ValueType> *excess_solution,
+                           size_type e_start, size_type e_end)
+{
+    const dim3 block(default_block_size, 1, 1);
+    const dim3 grid(ceildiv(e_end - e_start, block.x / subwarp_size), 1, 1);
+    kernel::scale_excess_solution<subwarp_size><<<grid, block>>>(
+        excess_block_ptrs, as_cuda_type(excess_solution->get_values()), e_start,
+        e_end);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_SCALE_EXCESS_SOLUTION_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void scatter_excess_solution(std::shared_ptr<const DefaultExecutor> exec,
                              const IndexType *excess_rhs_ptrs,
                              const matrix::Dense<ValueType> *excess_solution,
                              matrix::Csr<ValueType, IndexType> *inverse,
-                             const size_type e_start, const size_type e_end)
+                             size_type e_start, size_type e_end)
 {
     const auto num_rows = inverse->get_size()[0];
 
diff --git a/cuda/test/preconditioner/isai_kernels.cpp b/cuda/test/preconditioner/isai_kernels.cpp
index 2dfc528103d..ca9f4f3f1d9 100644
--- a/cuda/test/preconditioner/isai_kernels.cpp
+++ b/cuda/test/preconditioner/isai_kernels.cpp
@@ -205,9 +205,10 @@ TEST_F(Isai, CudaIsaiGenerateAinverseShortIsEquivalentToRef)
     auto da2 = da1;
 
     gko::kernels::reference::isai::generate_general_inverse(
-        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
     gko::kernels::cuda::isai::generate_general_inverse(
-        cuda, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data());
+        cuda, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
+        false);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, r<value_type>::value);
@@ -273,9 +274,10 @@ TEST_F(Isai, CudaIsaiGenerateAinverseLongIsEquivalentToRef)
     auto da2 = da1;
 
     gko::kernels::reference::isai::generate_general_inverse(
-        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
     gko::kernels::cuda::isai::generate_general_inverse(
-        cuda, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data());
+        cuda, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
+        false);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 10 * r<value_type>::value);
@@ -354,7 +356,7 @@ TEST_F(Isai, CudaIsaiGenerateExcessAinverseLongIsEquivalentToRef)
     gko::Array<index_type> a1(ref, num_rows + 1);
     auto a2 = a1;
     gko::kernels::reference::isai::generate_general_inverse(
-        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
     gko::Array<index_type> da1(cuda, a1);
     gko::Array<index_type> da2(cuda, a2);
     auto e_dim = a1.get_data()[num_rows];
@@ -378,6 +380,31 @@ TEST_F(Isai, CudaIsaiGenerateExcessAinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, CudaIsaiScaleExcessSolutionIsEquivalentToRef)
+{
+    initialize_data(matrix_type::lower, 572, 52);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_tri_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::Array<index_type> da1(cuda, a1);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(cuda);
+    de_rhs->copy_from(lend(e_rhs));
+    d_inverse->copy_from(lend(inverse));
+
+    gko::kernels::reference::isai::scale_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), 0, num_rows);
+    gko::kernels::cuda::isai::scale_excess_solution(cuda, da1.get_const_data(),
+                                                    de_rhs.get(), 0, num_rows);
+
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+}
+
+
 TEST_F(Isai, CudaIsaiScatterExcessSolutionLIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 572, 52);
@@ -438,7 +465,7 @@ TEST_F(Isai, CudaIsaiScatterExcessSolutionAIsEquivalentToRef)
     gko::Array<index_type> a1(ref, num_rows + 1);
     auto a2 = a1;
     gko::kernels::reference::isai::generate_general_inverse(
-        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
     gko::Array<index_type> da1(cuda, a1);
     auto e_dim = a1.get_data()[num_rows];
     auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
diff --git a/dpcpp/preconditioner/isai_kernels.dp.cpp b/dpcpp/preconditioner/isai_kernels.dp.cpp
index 34a2a525542..33cbe044fc9 100644
--- a/dpcpp/preconditioner/isai_kernels.dp.cpp
+++ b/dpcpp/preconditioner/isai_kernels.dp.cpp
@@ -91,7 +91,8 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
                               const matrix::Csr<ValueType, IndexType> *input,
                               matrix::Csr<ValueType, IndexType> *inverse,
                               IndexType *excess_rhs_ptrs,
-                              IndexType *excess_nz_ptrs) GKO_NOT_IMPLEMENTED;
+                              IndexType *excess_nz_ptrs,
+                              bool spd) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_GENERAL_INVERSE_KERNEL);
@@ -105,20 +106,31 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
                             const IndexType *excess_nz_ptrs,
                             matrix::Csr<ValueType, IndexType> *excess_system,
                             matrix::Dense<ValueType> *excess_rhs,
-                            const size_type e_start,
-                            const size_type e_end) GKO_NOT_IMPLEMENTED;
+                            size_type e_start,
+                            size_type e_end) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void scale_excess_solution(std::shared_ptr<const DefaultExecutor>,
+                           const IndexType *excess_block_ptrs,
+                           matrix::Dense<ValueType> *excess_solution,
+                           size_type e_start,
+                           size_type e_end) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_SCALE_EXCESS_SOLUTION_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void scatter_excess_solution(std::shared_ptr<const DefaultExecutor>,
                              const IndexType *excess_block_ptrs,
                              const matrix::Dense<ValueType> *excess_solution,
                              matrix::Csr<ValueType, IndexType> *inverse,
-                             const size_type e_start,
-                             const size_type e_end) GKO_NOT_IMPLEMENTED;
+                             size_type e_start,
+                             size_type e_end) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_SCATTER_EXCESS_SOLUTION_KERNEL);
diff --git a/hip/preconditioner/isai_kernels.hip.cpp b/hip/preconditioner/isai_kernels.hip.cpp
index d5af2162a3d..5b1c9ffa07a 100644
--- a/hip/preconditioner/isai_kernels.hip.cpp
+++ b/hip/preconditioner/isai_kernels.hip.cpp
@@ -117,7 +117,7 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
                               const matrix::Csr<ValueType, IndexType> *input,
                               matrix::Csr<ValueType, IndexType> *inverse,
                               IndexType *excess_rhs_ptrs,
-                              IndexType *excess_nz_ptrs)
+                              IndexType *excess_nz_ptrs, bool spd)
 {
     const auto num_rows = input->get_size()[0];
 
@@ -130,7 +130,7 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
         input->get_const_row_ptrs(), input->get_const_col_idxs(),
         as_hip_type(input->get_const_values()), inverse->get_row_ptrs(),
         inverse->get_col_idxs(), as_hip_type(inverse->get_values()),
-        excess_rhs_ptrs, excess_nz_ptrs);
+        excess_rhs_ptrs, excess_nz_ptrs, spd);
     components::prefix_sum(exec, excess_rhs_ptrs, num_rows + 1);
     components::prefix_sum(exec, excess_nz_ptrs, num_rows + 1);
 }
@@ -147,7 +147,7 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor> exec,
                             const IndexType *excess_nz_ptrs,
                             matrix::Csr<ValueType, IndexType> *excess_system,
                             matrix::Dense<ValueType> *excess_rhs,
-                            const size_type e_start, const size_type e_end)
+                            size_type e_start, size_type e_end)
 {
     const auto num_rows = input->get_size()[0];
 
@@ -168,12 +168,30 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void scale_excess_solution(std::shared_ptr<const DefaultExecutor>,
+                           const IndexType *excess_block_ptrs,
+                           matrix::Dense<ValueType> *excess_solution,
+                           size_type e_start, size_type e_end)
+{
+    const dim3 block(default_block_size, 1, 1);
+    const dim3 grid(ceildiv(e_end - e_start, block.x / subwarp_size), 1, 1);
+    hipLaunchKernelGGL(
+        HIP_KERNEL_NAME(kernel::scale_excess_solution<subwarp_size>), grid,
+        block, 0, 0, excess_block_ptrs,
+        as_hip_type(excess_solution->get_values()), e_start, e_end);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_SCALE_EXCESS_SOLUTION_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void scatter_excess_solution(std::shared_ptr<const DefaultExecutor> exec,
                              const IndexType *excess_rhs_ptrs,
                              const matrix::Dense<ValueType> *excess_solution,
                              matrix::Csr<ValueType, IndexType> *inverse,
-                             const size_type e_start, const size_type e_end)
+                             size_type e_start, size_type e_end)
 {
     const auto num_rows = inverse->get_size()[0];
 
diff --git a/hip/test/preconditioner/isai_kernels.hip.cpp b/hip/test/preconditioner/isai_kernels.hip.cpp
index 31f1ccd2ba7..adf5ad87789 100644
--- a/hip/test/preconditioner/isai_kernels.hip.cpp
+++ b/hip/test/preconditioner/isai_kernels.hip.cpp
@@ -186,9 +186,10 @@ TEST_F(Isai, HipIsaiGenerateAinverseShortIsEquivalentToRef)
     auto da2 = da1;
 
     gko::kernels::reference::isai::generate_general_inverse(
-        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
     gko::kernels::hip::isai::generate_general_inverse(
-        hip, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data());
+        hip, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
+        false);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, r<value_type>::value);
@@ -254,9 +255,10 @@ TEST_F(Isai, HipIsaiGenerateAinverseLongIsEquivalentToRef)
     auto da2 = da1;
 
     gko::kernels::reference::isai::generate_general_inverse(
-        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
     gko::kernels::hip::isai::generate_general_inverse(
-        hip, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data());
+        hip, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
+        false);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 10 * r<value_type>::value);
@@ -335,7 +337,7 @@ TEST_F(Isai, HipIsaiGenerateExcessAinverseLongIsEquivalentToRef)
     gko::Array<index_type> a1(ref, num_rows + 1);
     auto a2 = a1;
     gko::kernels::reference::isai::generate_general_inverse(
-        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
     gko::Array<index_type> da1(hip, a1);
     gko::Array<index_type> da2(hip, a2);
     auto e_dim = a1.get_data()[num_rows];
@@ -359,6 +361,31 @@ TEST_F(Isai, HipIsaiGenerateExcessAinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, HipIsaiScaleExcessSolutionIsEquivalentToRef)
+{
+    initialize_data(matrix_type::lower, 572, 52);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_tri_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::Array<index_type> da1(hip, a1);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(hip);
+    de_rhs->copy_from(lend(e_rhs));
+    d_inverse->copy_from(lend(inverse));
+
+    gko::kernels::reference::isai::scale_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), 0, num_rows);
+    gko::kernels::hip::isai::scale_excess_solution(hip, da1.get_const_data(),
+                                                   de_rhs.get(), 0, num_rows);
+
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+}
+
+
 TEST_F(Isai, HipIsaiScatterExcessSolutionLIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 572, 52);
@@ -419,7 +446,7 @@ TEST_F(Isai, HipIsaiScatterExcessSolutionAIsEquivalentToRef)
     gko::Array<index_type> a1(ref, num_rows + 1);
     auto a2 = a1;
     gko::kernels::reference::isai::generate_general_inverse(
-        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
     gko::Array<index_type> da1(hip, a1);
     auto e_dim = a1.get_data()[num_rows];
     auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
diff --git a/include/ginkgo/core/preconditioner/isai.hpp b/include/ginkgo/core/preconditioner/isai.hpp
index 590a54711f7..ca0a5f1f75a 100644
--- a/include/ginkgo/core/preconditioner/isai.hpp
+++ b/include/ginkgo/core/preconditioner/isai.hpp
@@ -57,22 +57,30 @@ namespace preconditioner {
 /**
  * This enum lists the types of the ISAI preconditioner.
  *
- * ISAI can either generate a general matrix, a lower triangular matrix, or an
- * upper triangular matrix.
+ * ISAI can either be generated for a general square matrix, a lower triangular
+ * matrix, an upper triangular matrix or a spd matrix.
  */
-enum struct isai_type { lower, upper, general };
+enum struct isai_type { lower, upper, general, spd };
 
 /**
  * The Incomplete Sparse Approximate Inverse (ISAI) Preconditioner generates
- * an approximate inverse matrix for a given lower triangular matrix L or upper
- * triangular matrix U.
+ * an approximate inverse matrix for a given square matrix A, lower triangular
+ * matrix L, upper triangular matrix U or symmetric positive (spd) matrix B.
  *
- * Using the preconditioner computes $aiU * x$ or $aiL * x$ (depending on the
- * type of the Isai) for a given vector x (may have multiple right hand sides).
- * aiU and aiL are the approximate inverses for U and L respectively.
+ * Using the preconditioner computes $aiA * x$, $aiU * x$, $aiL * x$ or $aiC^T *
+ * aiC * x$ (depending on the type of the Isai) for a given vector x (may have
+ * multiple right hand sides). aiA, aiU and aiL are the approximate inverses for
+ * A, U and L respectively. aiC is an approximation to C, the exact Cholesky
+ * factor of B (This is commonly referred to as a Factorized Sparse Approximate
+ * Inverse, short FSPAI).
  *
- * The sparsity pattern used for the approximate inverse is the same as
- * the sparsity pattern of the respective triangular matrix.
+ * The sparsity pattern used for the approximate inverse of A, L and U is the
+ * same as the sparsity pattern of the respective matrix. For B, the sparsity
+ * pattern used for the approximate inverse is the same as the sparsity pattern
+ * of the lower triangular half of B.
+ *
+ * Note that, except for the spd case, for a matrix A generally
+ * ISAI(A)^T != ISAI(A^T).
  *
  * For more details on the algorithm, see the paper
  * <a href="https://doi.org/10.1016/j.parco.2017.10.003">
@@ -84,8 +92,9 @@ enum struct isai_type { lower, upper, general };
  *       matrix. If there are more than `width` elements per row, the remaining
  *       elements will be ignored.
  *
- * @tparam IsaiType  determines if the ISAI is generated for a lower triangular
- *                   matrix or an upper triangular matrix
+ * @tparam IsaiType  determines if the ISAI is generated for a general square
+ * matrix, a lower triangular matrix, an upper triangular matrix or an spd
+ * matrix
  * @tparam ValueType  precision of matrix elements
  * @tparam IndexType  precision of matrix indexes
  *
@@ -101,6 +110,7 @@ class Isai : public EnableLinOp<Isai<IsaiType, ValueType, IndexType>>,
     friend class Isai<isai_type::general, ValueType, IndexType>;
     friend class Isai<isai_type::lower, ValueType, IndexType>;
     friend class Isai<isai_type::upper, ValueType, IndexType>;
+    friend class Isai<isai_type::spd, ValueType, IndexType>;
 
 public:
     using value_type = ValueType;
@@ -108,21 +118,29 @@ class Isai : public EnableLinOp<Isai<IsaiType, ValueType, IndexType>>,
     using transposed_type =
         Isai<IsaiType == isai_type::general
                  ? isai_type::general
-                 : IsaiType == isai_type::lower ? isai_type::upper
-                                                : isai_type::lower,
+                 : IsaiType == isai_type::spd
+                       ? isai_type::spd
+                       : IsaiType == isai_type::lower ? isai_type::upper
+                                                      : isai_type::lower,
              ValueType, IndexType>;
+    using Comp = Composition<ValueType>;
     using Csr = matrix::Csr<ValueType, IndexType>;
+    using Dense = matrix::Dense<ValueType>;
     static constexpr isai_type type{IsaiType};
 
     /**
-     * Returns the approximate inverse of the given matrix (either L or U,
-     * depending on the template parameter IsaiType).
+     * Returns the approximate inverse of the given matrix (either a CSR matrix
+     * for IsaiType general, upper or lower or a composition of two CSR matrices
+     * for IsaiType spd).
      *
      * @returns the generated approximate inverse
      */
-    std::shared_ptr<const Csr> get_approximate_inverse() const
+    std::shared_ptr<const typename std::conditional<IsaiType == isai_type::spd,
+                                                    Comp, Csr>::type>
+    get_approximate_inverse() const
     {
-        return approximate_inverse_;
+        return as<typename std::conditional<IsaiType == isai_type::spd, Comp,
+                                            Csr>::type>(approximate_inverse_);
     }
 
     GKO_CREATE_FACTORY_PARAMETERS(parameters, Factory)
@@ -158,7 +176,7 @@ class Isai : public EnableLinOp<Isai<IsaiType, ValueType, IndexType>>,
          * solution has been computed.
          * Must be at least 0, default value 0.
          */
-        int GKO_FACTORY_PARAMETER_SCALAR(excess_limit, 0);
+        index_type GKO_FACTORY_PARAMETER_SCALAR(excess_limit, 0);
     };
 
     GKO_ENABLE_LIN_OP_FACTORY(Isai, parameters, Factory);
@@ -177,8 +195,7 @@ class Isai : public EnableLinOp<Isai<IsaiType, ValueType, IndexType>>,
      * Creates an Isai preconditioner from a matrix using an Isai::Factory.
      *
      * @param factory  the factory to use to create the preconditoner
-     * @param factors  Composition<ValueType> of a lower triangular and an
-     *                 upper triangular matrix (L and U)
+     * @param system_matrix  the matrix for which an ISAI is to be computed
      */
     explicit Isai(const Factory *factory,
                   std::shared_ptr<const LinOp> system_matrix)
@@ -189,6 +206,12 @@ class Isai : public EnableLinOp<Isai<IsaiType, ValueType, IndexType>>,
         const auto power = parameters_.sparsity_power;
         const auto excess_limit = parameters_.excess_limit;
         generate_inverse(system_matrix, skip_sorting, power, excess_limit);
+        if (IsaiType == isai_type::spd) {
+            auto inv = share(as<Csr>(approximate_inverse_));
+            auto inv_transp = share(inv->conj_transpose());
+            approximate_inverse_ =
+                Composition<ValueType>::create(inv_transp, inv);
+        }
     }
 
     void apply_impl(const LinOp *b, LinOp *x) const override
@@ -214,10 +237,11 @@ class Isai : public EnableLinOp<Isai<IsaiType, ValueType, IndexType>>,
      *                      be skipped.
      */
     void generate_inverse(std::shared_ptr<const LinOp> to_invert,
-                          bool skip_sorting, int power, int excess_limit);
+                          bool skip_sorting, int power,
+                          index_type excess_limit);
 
 private:
-    std::shared_ptr<Csr> approximate_inverse_;
+    std::shared_ptr<LinOp> approximate_inverse_;
 };
 
 
@@ -230,6 +254,9 @@ using UpperIsai = Isai<isai_type::upper, ValueType, IndexType>;
 template <typename ValueType = default_precision, typename IndexType = int32>
 using GeneralIsai = Isai<isai_type::general, ValueType, IndexType>;
 
+template <typename ValueType = default_precision, typename IndexType = int32>
+using SpdIsai = Isai<isai_type::spd, ValueType, IndexType>;
+
 
 }  // namespace preconditioner
 }  // namespace gko
diff --git a/matrices/CMakeLists.txt b/matrices/CMakeLists.txt
index f49ba06b66a..60baca23ca4 100644
--- a/matrices/CMakeLists.txt
+++ b/matrices/CMakeLists.txt
@@ -19,3 +19,8 @@ configure_file("test/isai_a_excess.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai
 configure_file("test/isai_a_excess_rhs.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_a_excess_rhs.mtx")
 configure_file("test/isai_a_inv.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_a_inv.mtx")
 configure_file("test/isai_a_inv_partial.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_a_inv_partial.mtx")
+configure_file("test/isai_spd.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_spd.mtx")
+configure_file("test/isai_spd_excess.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_spd_excess.mtx")
+configure_file("test/isai_spd_excess_rhs.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_spd_excess_rhs.mtx")
+configure_file("test/isai_spd_inv.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_spd_inv.mtx")
+configure_file("test/isai_spd_inv_partial.mtx" "${Ginkgo_BINARY_DIR}/matrices/test/isai_spd_inv_partial.mtx")
diff --git a/matrices/test/isai_spd.mtx b/matrices/test/isai_spd.mtx
new file mode 100644
index 00000000000..63e5c4b2665
--- /dev/null
+++ b/matrices/test/isai_spd.mtx
@@ -0,0 +1,510 @@
+%%MatrixMarket matrix coordinate real symmetric
+% Generated 25-Feb-2021
+100 100 507
+1 1  10
+2 1  2
+3 1  3
+4 1  6
+36 1  4
+2 2  14
+3 2  2
+4 2  3
+5 2  6
+36 2  6
+3 3  14
+4 3  2
+5 3  3
+6 3  6
+36 3  6
+4 4  14
+5 4  2
+6 4  3
+7 4  6
+36 4  6
+5 5  14
+6 5  2
+7 5  3
+8 5  6
+36 5  6
+6 6  14
+7 6  2
+8 6  3
+9 6  6
+36 6  6
+7 7  14
+8 7  2
+9 7  3
+10 7  6
+36 7  6
+8 8  14
+9 8  2
+10 8  3
+11 8  6
+36 8  6
+9 9  14
+10 9  2
+11 9  3
+12 9  6
+36 9  6
+10 10  14
+11 10  2
+12 10  3
+13 10  6
+36 10  6
+11 11  14
+12 11  2
+13 11  3
+14 11  6
+36 11  6
+12 12  14
+13 12  2
+14 12  3
+15 12  6
+36 12  6
+13 13  14
+14 13  2
+15 13  3
+16 13  6
+36 13  6
+14 14  14
+15 14  2
+16 14  3
+17 14  6
+36 14  6
+15 15  1349
+16 15  14
+17 15  16
+18 15  24
+19 15  24
+20 15  24
+21 15  24
+22 15  24
+23 15  24
+24 15  24
+25 15  24
+26 15  24
+27 15  24
+28 15  24
+29 15  24
+30 15  24
+31 15  24
+32 15  24
+33 15  24
+34 15  24
+35 15  24
+36 15  95
+37 15  24
+38 15  24
+39 15  24
+40 15  24
+41 15  24
+42 15  24
+43 15  24
+44 15  24
+45 15  24
+46 15  24
+47 15  24
+48 15  24
+49 15  24
+50 15  24
+51 15  24
+52 15  24
+53 15  24
+54 15  24
+55 15  24
+56 15  24
+57 15  24
+58 15  24
+59 15  24
+60 15  24
+61 15  24
+62 15  24
+63 15  24
+64 15  24
+65 15  24
+66 15  24
+67 15  24
+68 15  24
+69 15  24
+70 15  24
+71 15  24
+72 15  24
+73 15  24
+74 15  24
+75 15  24
+76 15  24
+77 15  24
+78 15  24
+79 15  24
+80 15  24
+81 15  24
+82 15  24
+83 15  24
+84 15  24
+85 15  24
+86 15  24
+87 15  24
+88 15  24
+89 15  24
+90 15  24
+91 15  24
+92 15  24
+93 15  24
+94 15  24
+95 15  24
+96 15  24
+97 15  24
+98 15  24
+99 15  12
+100 15  12
+16 16  14
+17 16  2
+18 16  3
+19 16  6
+36 16  6
+17 17  14
+18 17  2
+19 17  3
+20 17  6
+36 17  6
+18 18  14
+19 18  2
+20 18  3
+21 18  6
+36 18  6
+19 19  14
+20 19  2
+21 19  3
+22 19  6
+36 19  6
+20 20  14
+21 20  2
+22 20  3
+23 20  6
+36 20  6
+21 21  14
+22 21  2
+23 21  3
+24 21  6
+36 21  6
+22 22  14
+23 22  2
+24 22  3
+25 22  6
+36 22  6
+23 23  14
+24 23  2
+25 23  3
+26 23  6
+36 23  6
+24 24  14
+25 24  2
+26 24  3
+27 24  6
+36 24  6
+25 25  14
+26 25  2
+27 25  3
+28 25  6
+36 25  6
+26 26  14
+27 26  2
+28 26  3
+29 26  6
+36 26  6
+27 27  14
+28 27  2
+29 27  3
+30 27  6
+36 27  6
+28 28  14
+29 28  2
+30 28  3
+31 28  6
+36 28  6
+29 29  14
+30 29  2
+31 29  3
+32 29  6
+36 29  6
+30 30  14
+31 30  2
+32 30  3
+33 30  6
+36 30  6
+31 31  14
+32 31  2
+33 31  3
+34 31  6
+36 31  6
+32 32  14
+33 32  2
+34 32  3
+35 32  6
+36 32  3
+33 33  14
+34 33  2
+35 33  3
+36 33  9
+34 34  14
+35 34  2
+36 34  5
+37 34  6
+35 35  14
+36 35  2
+37 35  3
+38 35  6
+36 36  47
+37 36  2
+38 36  3
+39 36  6
+37 37  14
+38 37  2
+39 37  3
+40 37  6
+38 38  14
+39 38  2
+40 38  3
+41 38  6
+39 39  14
+40 39  2
+41 39  3
+42 39  6
+40 40  14
+41 40  2
+42 40  3
+43 40  6
+41 41  14
+42 41  2
+43 41  3
+44 41  6
+42 42  14
+43 42  2
+44 42  3
+45 42  6
+43 43  14
+44 43  2
+45 43  3
+46 43  6
+44 44  14
+45 44  2
+46 44  3
+47 44  6
+45 45  14
+46 45  2
+47 45  3
+48 45  6
+46 46  14
+47 46  2
+48 46  3
+49 46  6
+47 47  14
+48 47  2
+49 47  3
+50 47  6
+48 48  14
+49 48  2
+50 48  3
+51 48  6
+49 49  14
+50 49  2
+51 49  3
+52 49  6
+50 50  14
+51 50  2
+52 50  3
+53 50  6
+51 51  14
+52 51  2
+53 51  3
+54 51  6
+52 52  14
+53 52  2
+54 52  3
+55 52  6
+53 53  14
+54 53  2
+55 53  3
+56 53  6
+54 54  14
+55 54  2
+56 54  3
+57 54  6
+55 55  14
+56 55  2
+57 55  3
+58 55  6
+56 56  14
+57 56  2
+58 56  3
+59 56  6
+57 57  14
+58 57  2
+59 57  3
+60 57  6
+58 58  14
+59 58  2
+60 58  3
+61 58  6
+59 59  14
+60 59  2
+61 59  3
+62 59  6
+60 60  14
+61 60  2
+62 60  3
+63 60  6
+61 61  14
+62 61  2
+63 61  3
+64 61  6
+62 62  14
+63 62  2
+64 62  3
+65 62  6
+63 63  14
+64 63  2
+65 63  3
+66 63  6
+64 64  14
+65 64  2
+66 64  3
+67 64  6
+65 65  14
+66 65  2
+67 65  3
+68 65  6
+66 66  14
+67 66  2
+68 66  3
+69 66  6
+67 67  14
+68 67  2
+69 67  3
+70 67  6
+68 68  14
+69 68  2
+70 68  3
+71 68  6
+69 69  14
+70 69  2
+71 69  3
+72 69  6
+70 70  14
+71 70  2
+72 70  3
+73 70  6
+71 71  14
+72 71  2
+73 71  3
+74 71  6
+72 72  14
+73 72  2
+74 72  3
+75 72  6
+73 73  14
+74 73  2
+75 73  3
+76 73  6
+74 74  14
+75 74  2
+76 74  3
+77 74  6
+75 75  14
+76 75  2
+77 75  3
+78 75  6
+76 76  14
+77 76  2
+78 76  3
+79 76  6
+77 77  14
+78 77  2
+79 77  3
+80 77  6
+78 78  14
+79 78  2
+80 78  3
+81 78  6
+79 79  14
+80 79  2
+81 79  3
+82 79  6
+80 80  14
+81 80  2
+82 80  3
+83 80  6
+81 81  14
+82 81  2
+83 81  3
+84 81  6
+82 82  14
+83 82  2
+84 82  3
+85 82  6
+83 83  14
+84 83  2
+85 83  3
+86 83  6
+84 84  14
+85 84  2
+86 84  3
+87 84  6
+85 85  14
+86 85  2
+87 85  3
+88 85  6
+86 86  14
+87 86  2
+88 86  3
+89 86  6
+87 87  14
+88 87  2
+89 87  3
+90 87  6
+88 88  14
+89 88  2
+90 88  3
+91 88  6
+89 89  14
+90 89  2
+91 89  3
+92 89  6
+90 90  14
+91 90  2
+92 90  3
+93 90  6
+91 91  14
+92 91  2
+93 91  3
+94 91  6
+92 92  14
+93 92  2
+94 92  3
+95 92  6
+93 93  14
+94 93  2
+95 93  3
+96 93  6
+94 94  14
+95 94  2
+96 94  3
+97 94  6
+95 95  14
+96 95  2
+97 95  3
+98 95  6
+96 96  14
+97 96  2
+98 96  3
+99 96  6
+97 97  14
+98 97  2
+99 97  3
+100 97  6
+98 98  14
+99 98  2
+100 98  3
+99 99  5
+100 99  2
+100 100  5
diff --git a/matrices/test/isai_spd_excess.mtx b/matrices/test/isai_spd_excess.mtx
new file mode 100644
index 00000000000..1e3a410baa3
--- /dev/null
+++ b/matrices/test/isai_spd_excess.mtx
@@ -0,0 +1,668 @@
+%%MatrixMarket matrix array real symmetric
+36 36
+ 10
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 4
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 1349
+ 14
+ 16
+ 24
+ 24
+ 24
+ 24
+ 24
+ 24
+ 24
+ 24
+ 24
+ 24
+ 24
+ 24
+ 24
+ 24
+ 24
+ 24
+ 24
+ 24
+ 95
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 0
+ 6
+ 14
+ 2
+ 3
+ 6
+ 3
+ 14
+ 2
+ 3
+ 9
+ 14
+ 2
+ 5
+ 14
+ 2
+ 47
diff --git a/matrices/test/isai_spd_excess_rhs.mtx b/matrices/test/isai_spd_excess_rhs.mtx
new file mode 100644
index 00000000000..3555cb6d355
--- /dev/null
+++ b/matrices/test/isai_spd_excess_rhs.mtx
@@ -0,0 +1,38 @@
+%%MatrixMarket matrix array real general
+36 1
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 0
+ 1
diff --git a/matrices/test/isai_spd_inv.mtx b/matrices/test/isai_spd_inv.mtx
new file mode 100644
index 00000000000..cf81713fc21
--- /dev/null
+++ b/matrices/test/isai_spd_inv.mtx
@@ -0,0 +1,510 @@
+%%MatrixMarket matrix coordinate real general
+% Generated 25-Feb-2021
+100 100 507
+1 1  0.3162277660168379
+2 1 -0.05423261445466404
+3 1 -0.07762671183167356
+4 1 -0.1798172116961022
+36 1 -0.07538896690562906
+2 2  0.2711630722733202
+3 2 -0.02859931488535342
+4 2 -0.04148258448038794
+5 2 -0.1199466995852642
+36 2 -0.1149179864341074
+3 3  0.2778219160291475
+4 3 -0.0003560736865269385
+5 3 -0.0457222567726007
+6 3 -0.1199466995852642
+36 3 -0.08394600557844488
+4 4  0.3137009178302299
+5 4 -0.01068831976502354
+6 4 -0.0457222567726007
+7 4 -0.1199466995852642
+36 4 -0.01673770132079616
+5 5  0.3004605445056617
+6 5 -0.01068831976502354
+7 5 -0.0457222567726007
+8 5 -0.1199466995852642
+36 5 -0.004374990144814157
+6 6  0.3004605445056617
+7 6 -0.01068831976502354
+8 6 -0.0457222567726007
+9 6 -0.1199466995852642
+36 6 -0.05435624585292672
+7 7  0.3004605445056617
+8 7 -0.01068831976502354
+9 7 -0.0457222567726007
+10 7 -0.1199466995852642
+36 7 -0.1054764152740641
+8 8  0.3004605445056617
+9 8 -0.01068831976502354
+10 8 -0.0457222567726007
+11 8 -0.1199466995852642
+36 8 -0.09046197705665167
+9 9  0.3004605445056617
+10 9 -0.01068831976502354
+11 9 -0.0457222567726007
+12 9 -0.1199466995852642
+36 9 -0.03300324641586369
+10 10  0.3004605445056617
+11 10 -0.01068831976502354
+12 10 -0.0457222567726007
+13 10 -0.1199466995852642
+36 10 -0.004186270435676823
+11 11  0.3004605445056617
+12 11 -0.01068831976502354
+13 11 -0.0457222567726007
+14 11 -0.1199466995852642
+36 11 -0.04311338466154185
+12 12  0.3004605445056617
+13 12 -0.01068831976502354
+14 12 -0.0457222567726007
+15 12 -0.01088092680025987
+36 12 -0.09678933002429982
+13 13  0.3004605445056617
+14 13 -0.01068831976502354
+15 13 -0.004147680017920843
+16 13 -0.1220568030985163
+36 13 -0.09975522312256663
+14 14  0.3004605445056617
+15 14 -0.0009695875366568199
+16 14 -0.04683756674630794
+17 14 -0.1246849031957135
+36 14 -0.04289982379003091
+15 15  0.02725618297490839
+16 15 -0.002791295581157084
+17 15 -0.003222116241547108
+18 15 -0.004090132505647694
+19 15 -0.003544813992424168
+20 15 -0.003116190754480626
+21 15 -0.002379174771699148
+22 15 -0.002379174771699148
+23 15 -0.002379174771699148
+24 15 -0.002379174771699148
+25 15 -0.002379174771699148
+26 15 -0.002379174771699148
+27 15 -0.002379174771699148
+28 15 -0.002379174771699148
+29 15 -0.002379174771699148
+30 15 -0.002379174771699148
+31 15 -0.002379174771699148
+32 15 -0.002379174771699148
+33 15 -0.002379174771699148
+34 15 -0.002379174771699148
+35 15 -0.002379174771699148
+36 15 -0.003986533799367643
+37 15 -0.003006395816161536
+38 15 -0.002353633281329151
+39 15 -0.001660066651943648
+40 15 -0.002379174771699148
+41 15 -0.002379174771699148
+42 15 -0.002379174771699148
+43 15 -0.002379174771699148
+44 15 -0.002379174771699148
+45 15 -0.002379174771699148
+46 15 -0.002379174771699148
+47 15 -0.002379174771699148
+48 15 -0.002379174771699148
+49 15 -0.002379174771699148
+50 15 -0.002379174771699148
+51 15 -0.002379174771699148
+52 15 -0.002379174771699148
+53 15 -0.002379174771699148
+54 15 -0.002379174771699148
+55 15 -0.002379174771699148
+56 15 -0.002379174771699148
+57 15 -0.002379174771699148
+58 15 -0.002379174771699148
+59 15 -0.002379174771699148
+60 15 -0.002379174771699148
+61 15 -0.002379174771699148
+62 15 -0.002379174771699148
+63 15 -0.002379174771699148
+64 15 -0.002379174771699148
+65 15 -0.002379174771699148
+66 15 -0.002379174771699148
+67 15 -0.002379174771699148
+68 15 -0.002379174771699148
+69 15 -0.002379174771699148
+70 15 -0.002379174771699148
+71 15 -0.002379174771699148
+72 15 -0.002379174771699148
+73 15 -0.002379174771699148
+74 15 -0.002379174771699148
+75 15 -0.002379174771699148
+76 15 -0.002379174771699148
+77 15 -0.002379174771699148
+78 15 -0.002379174771699148
+79 15 -0.002379174771699148
+80 15 -0.002379174771699148
+81 15 -0.002379174771699148
+82 15 -0.002379174771699148
+83 15 -0.002379174771699148
+84 15 -0.002379174771699148
+85 15 -0.002379174771699148
+86 15 -0.002379174771699148
+87 15 -0.002379174771699148
+88 15 -0.002379174771699148
+89 15 -0.002379174771699148
+90 15 -0.002379174771699148
+91 15 -0.002379174771699148
+92 15 -0.002379174771699148
+93 15 -0.002379174771699148
+94 15 -0.002379174771699148
+95 15 -0.002379174771699148
+96 15 -0.002379174771699148
+97 15 -0.002379174771699148
+98 15 -0.002379174771699148
+99 15  0.001153567856303423
+100 15  0.001104007209054193
+16 16  0.3018073772692194
+17 16 -0.01268050939273546
+18 16 -0.05162748887230814
+19 16 -0.1189213753455324
+36 16 -0.02693599842259033
+17 17  0.2983457342335482
+18 17 -0.02773689750204802
+19 17 -0.04307115771971171
+20 17 -0.1186748059339512
+36 17 -0.0943412432771171
+18 18  0.2785068314318259
+19 18 -0.005557000879184963
+20 18 -0.04155970386656466
+21 18 -0.1173933022977732
+36 18 -0.1096709274668464
+19 19  0.3028899948013952
+20 19 -0.006477654665450831
+21 19 -0.04265714275977608
+22 19 -0.1173933022977732
+36 19 -0.05435721161820799
+20 20  0.3023097844794148
+21 20 -0.00774711670277443
+22 20 -0.04265714275977608
+23 20 -0.1173933022977732
+36 20  0.01237692936104656
+21 21  0.3015270103862467
+22 21 -0.00774711670277443
+23 21 -0.04265714275977608
+24 21 -0.1173933022977732
+36 21 -0.005306814530161682
+22 22  0.3015270103862467
+23 22 -0.00774711670277443
+24 22 -0.04265714275977608
+25 22 -0.1173933022977732
+36 22 -0.08202647764470203
+23 23  0.3015270103862467
+24 23 -0.00774711670277443
+25 23 -0.04265714275977608
+26 23 -0.1173933022977732
+36 23 -0.1368528787421111
+24 24  0.3015270103862467
+25 24 -0.00774711670277443
+26 24 -0.04265714275977608
+27 24 -0.1173933022977732
+36 24 -0.07717627547008193
+25 25  0.3015270103862467
+26 25 -0.00774711670277443
+27 25 -0.04265714275977608
+28 25 -0.1173933022977732
+36 25  0.007487232152437592
+26 26  0.3015270103862467
+27 26 -0.00774711670277443
+28 26 -0.04265714275977608
+29 26 -0.1173933022977732
+36 26  0.040872498482319
+27 27  0.3015270103862467
+28 27 -0.00774711670277443
+29 27 -0.04265714275977608
+30 27 -0.1173933022977732
+36 27 -0.06896766904391485
+28 28  0.3015270103862467
+29 28 -0.00774711670277443
+30 28 -0.04265714275977608
+31 28 -0.1173933022977732
+36 28 -0.1384835292707354
+29 29  0.3015270103862467
+30 29 -0.00774711670277443
+31 29 -0.04265714275977608
+32 29 -0.1173933022977732
+36 29 -0.1482442344977491
+30 30  0.3015270103862467
+31 30 -0.00774711670277443
+32 30 -0.04265714275977608
+33 30 -0.1173933022977732
+36 30  0.02296501655932748
+31 31  0.3015270103862467
+32 31 -0.00774711670277443
+33 31 -0.04265714275977608
+34 31 -0.1173933022977732
+36 31  0.03807488169032599
+32 32  0.3015270103862467
+33 32 -0.00774711670277443
+34 32 -0.04265714275977608
+35 32 -0.1173933022977732
+36 32  0.04093194847767558
+33 33  0.3015270103862467
+34 33 -0.00774711670277443
+35 33 -0.04265714275977608
+36 33 -0.2171697930441952
+34 34  0.3015270103862467
+35 34 -0.00774711670277443
+36 34 -0.1070011673634044
+37 34 -0.1208176272753754
+35 35  0.3015270103862467
+36 35  0.003400135777561141
+37 35 -0.04338802548453834
+38 35 -0.1201237830553024
+36 36  0.3339974666461543
+37 36  0.007934110567880069
+38 36 -0.008683931085510691
+39 36 -0.02902465189347272
+37 37  0.3017842965954681
+38 37 -0.0116130459854018
+39 37 -0.05007537137962354
+40 37 -0.1173933022977732
+38 38  0.2984045269422267
+39 38 -0.02412350819046598
+40 38 -0.04265714275977608
+41 38 -0.1173933022977732
+39 39  0.2823977063764183
+40 39 -0.00774711670277443
+41 39 -0.04265714275977608
+42 39 -0.1173933022977732
+40 40  0.3015270103862467
+41 40 -0.00774711670277443
+42 40 -0.04265714275977608
+43 40 -0.1173933022977732
+41 41  0.3015270103862467
+42 41 -0.00774711670277443
+43 41 -0.04265714275977608
+44 41 -0.1173933022977732
+42 42  0.3015270103862467
+43 42 -0.00774711670277443
+44 42 -0.04265714275977608
+45 42 -0.1173933022977732
+43 43  0.3015270103862467
+44 43 -0.00774711670277443
+45 43 -0.04265714275977608
+46 43 -0.1173933022977732
+44 44  0.3015270103862467
+45 44 -0.00774711670277443
+46 44 -0.04265714275977608
+47 44 -0.1173933022977732
+45 45  0.3015270103862467
+46 45 -0.00774711670277443
+47 45 -0.04265714275977608
+48 45 -0.1173933022977732
+46 46  0.3015270103862467
+47 46 -0.00774711670277443
+48 46 -0.04265714275977608
+49 46 -0.1173933022977732
+47 47  0.3015270103862467
+48 47 -0.00774711670277443
+49 47 -0.04265714275977608
+50 47 -0.1173933022977732
+48 48  0.3015270103862467
+49 48 -0.00774711670277443
+50 48 -0.04265714275977608
+51 48 -0.1173933022977732
+49 49  0.3015270103862467
+50 49 -0.00774711670277443
+51 49 -0.04265714275977608
+52 49 -0.1173933022977732
+50 50  0.3015270103862467
+51 50 -0.00774711670277443
+52 50 -0.04265714275977608
+53 50 -0.1173933022977732
+51 51  0.3015270103862467
+52 51 -0.00774711670277443
+53 51 -0.04265714275977608
+54 51 -0.1173933022977732
+52 52  0.3015270103862467
+53 52 -0.00774711670277443
+54 52 -0.04265714275977608
+55 52 -0.1173933022977732
+53 53  0.3015270103862467
+54 53 -0.00774711670277443
+55 53 -0.04265714275977608
+56 53 -0.1173933022977732
+54 54  0.3015270103862467
+55 54 -0.00774711670277443
+56 54 -0.04265714275977608
+57 54 -0.1173933022977732
+55 55  0.3015270103862467
+56 55 -0.00774711670277443
+57 55 -0.04265714275977608
+58 55 -0.1173933022977732
+56 56  0.3015270103862467
+57 56 -0.00774711670277443
+58 56 -0.04265714275977608
+59 56 -0.1173933022977732
+57 57  0.3015270103862467
+58 57 -0.00774711670277443
+59 57 -0.04265714275977608
+60 57 -0.1173933022977732
+58 58  0.3015270103862467
+59 58 -0.00774711670277443
+60 58 -0.04265714275977608
+61 58 -0.1173933022977732
+59 59  0.3015270103862467
+60 59 -0.00774711670277443
+61 59 -0.04265714275977608
+62 59 -0.1173933022977732
+60 60  0.3015270103862467
+61 60 -0.00774711670277443
+62 60 -0.04265714275977608
+63 60 -0.1173933022977732
+61 61  0.3015270103862467
+62 61 -0.00774711670277443
+63 61 -0.04265714275977608
+64 61 -0.1173933022977732
+62 62  0.3015270103862467
+63 62 -0.00774711670277443
+64 62 -0.04265714275977608
+65 62 -0.1173933022977732
+63 63  0.3015270103862467
+64 63 -0.00774711670277443
+65 63 -0.04265714275977608
+66 63 -0.1173933022977732
+64 64  0.3015270103862467
+65 64 -0.00774711670277443
+66 64 -0.04265714275977608
+67 64 -0.1173933022977732
+65 65  0.3015270103862467
+66 65 -0.00774711670277443
+67 65 -0.04265714275977608
+68 65 -0.1173933022977732
+66 66  0.3015270103862467
+67 66 -0.00774711670277443
+68 66 -0.04265714275977608
+69 66 -0.1173933022977732
+67 67  0.3015270103862467
+68 67 -0.00774711670277443
+69 67 -0.04265714275977608
+70 67 -0.1173933022977732
+68 68  0.3015270103862467
+69 68 -0.00774711670277443
+70 68 -0.04265714275977608
+71 68 -0.1173933022977732
+69 69  0.3015270103862467
+70 69 -0.00774711670277443
+71 69 -0.04265714275977608
+72 69 -0.1173933022977732
+70 70  0.3015270103862467
+71 70 -0.00774711670277443
+72 70 -0.04265714275977608
+73 70 -0.1173933022977732
+71 71  0.3015270103862467
+72 71 -0.00774711670277443
+73 71 -0.04265714275977608
+74 71 -0.1173933022977732
+72 72  0.3015270103862467
+73 72 -0.00774711670277443
+74 72 -0.04265714275977608
+75 72 -0.1173933022977732
+73 73  0.3015270103862467
+74 73 -0.00774711670277443
+75 73 -0.04265714275977608
+76 73 -0.1173933022977732
+74 74  0.3015270103862467
+75 74 -0.00774711670277443
+76 74 -0.04265714275977608
+77 74 -0.1173933022977732
+75 75  0.3015270103862467
+76 75 -0.00774711670277443
+77 75 -0.04265714275977608
+78 75 -0.1173933022977732
+76 76  0.3015270103862467
+77 76 -0.00774711670277443
+78 76 -0.04265714275977608
+79 76 -0.1173933022977732
+77 77  0.3015270103862467
+78 77 -0.00774711670277443
+79 77 -0.04265714275977608
+80 77 -0.1173933022977732
+78 78  0.3015270103862467
+79 78 -0.00774711670277443
+80 78 -0.04265714275977608
+81 78 -0.1173933022977732
+79 79  0.3015270103862467
+80 79 -0.00774711670277443
+81 79 -0.04265714275977608
+82 79 -0.1173933022977732
+80 80  0.3015270103862467
+81 80 -0.00774711670277443
+82 80 -0.04265714275977608
+83 80 -0.1173933022977732
+81 81  0.3015270103862467
+82 81 -0.00774711670277443
+83 81 -0.04265714275977608
+84 81 -0.1173933022977732
+82 82  0.3015270103862467
+83 82 -0.00774711670277443
+84 82 -0.04265714275977608
+85 82 -0.1173933022977732
+83 83  0.3015270103862467
+84 83 -0.00774711670277443
+85 83 -0.04265714275977608
+86 83 -0.1173933022977732
+84 84  0.3015270103862467
+85 84 -0.00774711670277443
+86 84 -0.04265714275977608
+87 84 -0.1173933022977732
+85 85  0.3015270103862467
+86 85 -0.00774711670277443
+87 85 -0.04265714275977608
+88 85 -0.1173933022977732
+86 86  0.3015270103862467
+87 86 -0.00774711670277443
+88 86 -0.04265714275977608
+89 86 -0.1173933022977732
+87 87  0.3015270103862467
+88 87 -0.00774711670277443
+89 87 -0.04265714275977608
+90 87 -0.1173933022977732
+88 88  0.3015270103862467
+89 88 -0.00774711670277443
+90 88 -0.04265714275977608
+91 88 -0.1173933022977732
+89 89  0.3015270103862467
+90 89 -0.00774711670277443
+91 89 -0.04265714275977608
+92 89 -0.1173933022977732
+90 90  0.3015270103862467
+91 90 -0.00774711670277443
+92 90 -0.04265714275977608
+93 90 -0.1173933022977732
+91 91  0.3015270103862467
+92 91 -0.00774711670277443
+93 91 -0.04265714275977608
+94 91 -0.1173933022977732
+92 92  0.3015270103862467
+93 92 -0.00774711670277443
+94 92 -0.04265714275977608
+95 92 -0.1173933022977732
+93 93  0.3015270103862467
+94 93 -0.00774711670277443
+95 93 -0.04265714275977608
+96 93 -0.1173933022977732
+94 94  0.3015270103862467
+95 94 -0.00774711670277443
+96 94 -0.04265714275977608
+97 94 -0.1173933022977732
+95 95  0.3015270103862467
+96 95 -0.00774711670277443
+97 95 -0.04265714275977608
+98 95 -0.1173933022977732
+96 96  0.3015270103862467
+97 96 -0.00774711670277443
+98 96 -0.04265714275977608
+99 96 -0.278672717928978
+97 97  0.3015270103862467
+98 97 -0.00774711670277443
+99 97 -0.1072409551079743
+100 97 -0.2704350793462527
+98 98  0.3015270103862467
+99 98 -0.02614797503373225
+100 98 -0.1019634419302533
+99 99  0.6944430429619259
+100 99 -0.07979747629324176
+100 100  0.7004857084284116
diff --git a/matrices/test/isai_spd_inv_partial.mtx b/matrices/test/isai_spd_inv_partial.mtx
new file mode 100644
index 00000000000..cbe488c8b49
--- /dev/null
+++ b/matrices/test/isai_spd_inv_partial.mtx
@@ -0,0 +1,510 @@
+%%MatrixMarket matrix coordinate real general
+% Generated 25-Feb-2021
+100 100 507
+1 1  0.3162277660168379
+2 1 -0.05423261445466404
+3 1 -0.07762671183167356
+4 1 -0.1798172116961022
+36 1 -1
+2 2  0.2711630722733202
+3 2 -0.02859931488535342
+4 2 -0.04148258448038794
+5 2 -0.1199466995852642
+36 2 -1
+3 3  0.2778219160291475
+4 3 -0.0003560736865269385
+5 3 -0.0457222567726007
+6 3 -0.1199466995852642
+36 3 -1
+4 4  0.3137009178302299
+5 4 -0.01068831976502354
+6 4 -0.0457222567726007
+7 4 -0.1199466995852642
+36 4 -1
+5 5  0.3004605445056617
+6 5 -0.01068831976502354
+7 5 -0.0457222567726007
+8 5 -0.1199466995852642
+36 5 -1
+6 6  0.3004605445056617
+7 6 -0.01068831976502354
+8 6 -0.0457222567726007
+9 6 -0.1199466995852642
+36 6 -1
+7 7  0.3004605445056617
+8 7 -0.01068831976502354
+9 7 -0.0457222567726007
+10 7 -0.1199466995852642
+36 7 -1
+8 8  0.3004605445056617
+9 8 -0.01068831976502354
+10 8 -0.0457222567726007
+11 8 -0.1199466995852642
+36 8 -1
+9 9  0.3004605445056617
+10 9 -0.01068831976502354
+11 9 -0.0457222567726007
+12 9 -0.1199466995852642
+36 9 -1
+10 10  0.3004605445056617
+11 10 -0.01068831976502354
+12 10 -0.0457222567726007
+13 10 -0.1199466995852642
+36 10 -1
+11 11  0.3004605445056617
+12 11 -0.01068831976502354
+13 11 -0.0457222567726007
+14 11 -0.1199466995852642
+36 11 -1
+12 12  0.3004605445056617
+13 12 -0.01068831976502354
+14 12 -0.0457222567726007
+15 12 -0.01088092680025987
+36 12 -1
+13 13  0.3004605445056617
+14 13 -0.01068831976502354
+15 13 -0.004147680017920843
+16 13 -0.1220568030985163
+36 13 -1
+14 14  0.3004605445056617
+15 14 -0.0009695875366568199
+16 14 -0.04683756674630794
+17 14 -0.1246849031957135
+36 14 -1
+15 15  0.02725618297490839
+16 15 -0.002791295581157084
+17 15 -0.003222116241547108
+18 15 -0.004090132505647694
+19 15 -0.003544813992424168
+20 15 -0.003116190754480626
+21 15 -0.002379174771699148
+22 15 -0.002379174771699148
+23 15 -0.002379174771699148
+24 15 -0.002379174771699148
+25 15 -0.002379174771699148
+26 15 -0.002379174771699148
+27 15 -0.002379174771699148
+28 15 -0.002379174771699148
+29 15 -0.002379174771699148
+30 15 -0.002379174771699148
+31 15 -0.002379174771699148
+32 15 -0.002379174771699148
+33 15 -0.002379174771699148
+34 15 -0.002379174771699148
+35 15 -0.002379174771699148
+36 15 -1
+37 15 -0.003006395816161536
+38 15 -0.002353633281329151
+39 15 -0.001660066651943648
+40 15 -0.002379174771699148
+41 15 -0.002379174771699148
+42 15 -0.002379174771699148
+43 15 -0.002379174771699148
+44 15 -0.002379174771699148
+45 15 -0.002379174771699148
+46 15 -0.002379174771699148
+47 15 -0.002379174771699148
+48 15 -0.002379174771699148
+49 15 -0.002379174771699148
+50 15 -0.002379174771699148
+51 15 -0.002379174771699148
+52 15 -0.002379174771699148
+53 15 -0.002379174771699148
+54 15 -0.002379174771699148
+55 15 -0.002379174771699148
+56 15 -0.002379174771699148
+57 15 -0.002379174771699148
+58 15 -0.002379174771699148
+59 15 -0.002379174771699148
+60 15 -0.002379174771699148
+61 15 -0.002379174771699148
+62 15 -0.002379174771699148
+63 15 -0.002379174771699148
+64 15 -0.002379174771699148
+65 15 -0.002379174771699148
+66 15 -0.002379174771699148
+67 15 -0.002379174771699148
+68 15 -0.002379174771699148
+69 15 -0.002379174771699148
+70 15 -0.002379174771699148
+71 15 -0.002379174771699148
+72 15 -0.002379174771699148
+73 15 -0.002379174771699148
+74 15 -0.002379174771699148
+75 15 -0.002379174771699148
+76 15 -0.002379174771699148
+77 15 -0.002379174771699148
+78 15 -0.002379174771699148
+79 15 -0.002379174771699148
+80 15 -0.002379174771699148
+81 15 -0.002379174771699148
+82 15 -0.002379174771699148
+83 15 -0.002379174771699148
+84 15 -0.002379174771699148
+85 15 -0.002379174771699148
+86 15 -0.002379174771699148
+87 15 -0.002379174771699148
+88 15 -0.002379174771699148
+89 15 -0.002379174771699148
+90 15 -0.002379174771699148
+91 15 -0.002379174771699148
+92 15 -0.002379174771699148
+93 15 -0.002379174771699148
+94 15 -0.002379174771699148
+95 15 -0.002379174771699148
+96 15 -0.002379174771699148
+97 15 -0.002379174771699148
+98 15 -0.002379174771699148
+99 15  0.001153567856303423
+100 15  0.001104007209054193
+16 16  0.3018073772692194
+17 16 -0.01268050939273546
+18 16 -0.05162748887230814
+19 16 -0.1189213753455324
+36 16 -1
+17 17  0.2983457342335482
+18 17 -0.02773689750204802
+19 17 -0.04307115771971171
+20 17 -0.1186748059339512
+36 17 -1
+18 18  0.2785068314318259
+19 18 -0.005557000879184963
+20 18 -0.04155970386656466
+21 18 -0.1173933022977732
+36 18 -1
+19 19  0.3028899948013952
+20 19 -0.006477654665450831
+21 19 -0.04265714275977608
+22 19 -0.1173933022977732
+36 19 -1
+20 20  0.3023097844794148
+21 20 -0.00774711670277443
+22 20 -0.04265714275977608
+23 20 -0.1173933022977732
+36 20 -1
+21 21  0.3015270103862467
+22 21 -0.00774711670277443
+23 21 -0.04265714275977608
+24 21 -0.1173933022977732
+36 21 -1
+22 22  0.3015270103862467
+23 22 -0.00774711670277443
+24 22 -0.04265714275977608
+25 22 -0.1173933022977732
+36 22 -1
+23 23  0.3015270103862467
+24 23 -0.00774711670277443
+25 23 -0.04265714275977608
+26 23 -0.1173933022977732
+36 23 -1
+24 24  0.3015270103862467
+25 24 -0.00774711670277443
+26 24 -0.04265714275977608
+27 24 -0.1173933022977732
+36 24 -1
+25 25  0.3015270103862467
+26 25 -0.00774711670277443
+27 25 -0.04265714275977608
+28 25 -0.1173933022977732
+36 25 -1
+26 26  0.3015270103862467
+27 26 -0.00774711670277443
+28 26 -0.04265714275977608
+29 26 -0.1173933022977732
+36 26 -1
+27 27  0.3015270103862467
+28 27 -0.00774711670277443
+29 27 -0.04265714275977608
+30 27 -0.1173933022977732
+36 27 -1
+28 28  0.3015270103862467
+29 28 -0.00774711670277443
+30 28 -0.04265714275977608
+31 28 -0.1173933022977732
+36 28 -1
+29 29  0.3015270103862467
+30 29 -0.00774711670277443
+31 29 -0.04265714275977608
+32 29 -0.1173933022977732
+36 29 -1
+30 30  0.3015270103862467
+31 30 -0.00774711670277443
+32 30 -0.04265714275977608
+33 30 -0.1173933022977732
+36 30 -1
+31 31  0.3015270103862467
+32 31 -0.00774711670277443
+33 31 -0.04265714275977608
+34 31 -0.1173933022977732
+36 31 -1
+32 32  0.3015270103862467
+33 32 -0.00774711670277443
+34 32 -0.04265714275977608
+35 32 -0.1173933022977732
+36 32 -1
+33 33  0.3015270103862467
+34 33 -0.00774711670277443
+35 33 -0.04265714275977608
+36 33 -1
+34 34  0.3015270103862467
+35 34 -0.00774711670277443
+36 34 -1
+37 34 -0.1208176272753754
+35 35  0.3015270103862467
+36 35 -1
+37 35 -0.04338802548453834
+38 35 -0.1201237830553024
+36 36 -1
+37 36  0.007934110567880069
+38 36 -0.008683931085510691
+39 36 -0.02902465189347272
+37 37  0.3017842965954681
+38 37 -0.0116130459854018
+39 37 -0.05007537137962354
+40 37 -0.1173933022977732
+38 38  0.2984045269422267
+39 38 -0.02412350819046598
+40 38 -0.04265714275977608
+41 38 -0.1173933022977732
+39 39  0.2823977063764183
+40 39 -0.00774711670277443
+41 39 -0.04265714275977608
+42 39 -0.1173933022977732
+40 40  0.3015270103862467
+41 40 -0.00774711670277443
+42 40 -0.04265714275977608
+43 40 -0.1173933022977732
+41 41  0.3015270103862467
+42 41 -0.00774711670277443
+43 41 -0.04265714275977608
+44 41 -0.1173933022977732
+42 42  0.3015270103862467
+43 42 -0.00774711670277443
+44 42 -0.04265714275977608
+45 42 -0.1173933022977732
+43 43  0.3015270103862467
+44 43 -0.00774711670277443
+45 43 -0.04265714275977608
+46 43 -0.1173933022977732
+44 44  0.3015270103862467
+45 44 -0.00774711670277443
+46 44 -0.04265714275977608
+47 44 -0.1173933022977732
+45 45  0.3015270103862467
+46 45 -0.00774711670277443
+47 45 -0.04265714275977608
+48 45 -0.1173933022977732
+46 46  0.3015270103862467
+47 46 -0.00774711670277443
+48 46 -0.04265714275977608
+49 46 -0.1173933022977732
+47 47  0.3015270103862467
+48 47 -0.00774711670277443
+49 47 -0.04265714275977608
+50 47 -0.1173933022977732
+48 48  0.3015270103862467
+49 48 -0.00774711670277443
+50 48 -0.04265714275977608
+51 48 -0.1173933022977732
+49 49  0.3015270103862467
+50 49 -0.00774711670277443
+51 49 -0.04265714275977608
+52 49 -0.1173933022977732
+50 50  0.3015270103862467
+51 50 -0.00774711670277443
+52 50 -0.04265714275977608
+53 50 -0.1173933022977732
+51 51  0.3015270103862467
+52 51 -0.00774711670277443
+53 51 -0.04265714275977608
+54 51 -0.1173933022977732
+52 52  0.3015270103862467
+53 52 -0.00774711670277443
+54 52 -0.04265714275977608
+55 52 -0.1173933022977732
+53 53  0.3015270103862467
+54 53 -0.00774711670277443
+55 53 -0.04265714275977608
+56 53 -0.1173933022977732
+54 54  0.3015270103862467
+55 54 -0.00774711670277443
+56 54 -0.04265714275977608
+57 54 -0.1173933022977732
+55 55  0.3015270103862467
+56 55 -0.00774711670277443
+57 55 -0.04265714275977608
+58 55 -0.1173933022977732
+56 56  0.3015270103862467
+57 56 -0.00774711670277443
+58 56 -0.04265714275977608
+59 56 -0.1173933022977732
+57 57  0.3015270103862467
+58 57 -0.00774711670277443
+59 57 -0.04265714275977608
+60 57 -0.1173933022977732
+58 58  0.3015270103862467
+59 58 -0.00774711670277443
+60 58 -0.04265714275977608
+61 58 -0.1173933022977732
+59 59  0.3015270103862467
+60 59 -0.00774711670277443
+61 59 -0.04265714275977608
+62 59 -0.1173933022977732
+60 60  0.3015270103862467
+61 60 -0.00774711670277443
+62 60 -0.04265714275977608
+63 60 -0.1173933022977732
+61 61  0.3015270103862467
+62 61 -0.00774711670277443
+63 61 -0.04265714275977608
+64 61 -0.1173933022977732
+62 62  0.3015270103862467
+63 62 -0.00774711670277443
+64 62 -0.04265714275977608
+65 62 -0.1173933022977732
+63 63  0.3015270103862467
+64 63 -0.00774711670277443
+65 63 -0.04265714275977608
+66 63 -0.1173933022977732
+64 64  0.3015270103862467
+65 64 -0.00774711670277443
+66 64 -0.04265714275977608
+67 64 -0.1173933022977732
+65 65  0.3015270103862467
+66 65 -0.00774711670277443
+67 65 -0.04265714275977608
+68 65 -0.1173933022977732
+66 66  0.3015270103862467
+67 66 -0.00774711670277443
+68 66 -0.04265714275977608
+69 66 -0.1173933022977732
+67 67  0.3015270103862467
+68 67 -0.00774711670277443
+69 67 -0.04265714275977608
+70 67 -0.1173933022977732
+68 68  0.3015270103862467
+69 68 -0.00774711670277443
+70 68 -0.04265714275977608
+71 68 -0.1173933022977732
+69 69  0.3015270103862467
+70 69 -0.00774711670277443
+71 69 -0.04265714275977608
+72 69 -0.1173933022977732
+70 70  0.3015270103862467
+71 70 -0.00774711670277443
+72 70 -0.04265714275977608
+73 70 -0.1173933022977732
+71 71  0.3015270103862467
+72 71 -0.00774711670277443
+73 71 -0.04265714275977608
+74 71 -0.1173933022977732
+72 72  0.3015270103862467
+73 72 -0.00774711670277443
+74 72 -0.04265714275977608
+75 72 -0.1173933022977732
+73 73  0.3015270103862467
+74 73 -0.00774711670277443
+75 73 -0.04265714275977608
+76 73 -0.1173933022977732
+74 74  0.3015270103862467
+75 74 -0.00774711670277443
+76 74 -0.04265714275977608
+77 74 -0.1173933022977732
+75 75  0.3015270103862467
+76 75 -0.00774711670277443
+77 75 -0.04265714275977608
+78 75 -0.1173933022977732
+76 76  0.3015270103862467
+77 76 -0.00774711670277443
+78 76 -0.04265714275977608
+79 76 -0.1173933022977732
+77 77  0.3015270103862467
+78 77 -0.00774711670277443
+79 77 -0.04265714275977608
+80 77 -0.1173933022977732
+78 78  0.3015270103862467
+79 78 -0.00774711670277443
+80 78 -0.04265714275977608
+81 78 -0.1173933022977732
+79 79  0.3015270103862467
+80 79 -0.00774711670277443
+81 79 -0.04265714275977608
+82 79 -0.1173933022977732
+80 80  0.3015270103862467
+81 80 -0.00774711670277443
+82 80 -0.04265714275977608
+83 80 -0.1173933022977732
+81 81  0.3015270103862467
+82 81 -0.00774711670277443
+83 81 -0.04265714275977608
+84 81 -0.1173933022977732
+82 82  0.3015270103862467
+83 82 -0.00774711670277443
+84 82 -0.04265714275977608
+85 82 -0.1173933022977732
+83 83  0.3015270103862467
+84 83 -0.00774711670277443
+85 83 -0.04265714275977608
+86 83 -0.1173933022977732
+84 84  0.3015270103862467
+85 84 -0.00774711670277443
+86 84 -0.04265714275977608
+87 84 -0.1173933022977732
+85 85  0.3015270103862467
+86 85 -0.00774711670277443
+87 85 -0.04265714275977608
+88 85 -0.1173933022977732
+86 86  0.3015270103862467
+87 86 -0.00774711670277443
+88 86 -0.04265714275977608
+89 86 -0.1173933022977732
+87 87  0.3015270103862467
+88 87 -0.00774711670277443
+89 87 -0.04265714275977608
+90 87 -0.1173933022977732
+88 88  0.3015270103862467
+89 88 -0.00774711670277443
+90 88 -0.04265714275977608
+91 88 -0.1173933022977732
+89 89  0.3015270103862467
+90 89 -0.00774711670277443
+91 89 -0.04265714275977608
+92 89 -0.1173933022977732
+90 90  0.3015270103862467
+91 90 -0.00774711670277443
+92 90 -0.04265714275977608
+93 90 -0.1173933022977732
+91 91  0.3015270103862467
+92 91 -0.00774711670277443
+93 91 -0.04265714275977608
+94 91 -0.1173933022977732
+92 92  0.3015270103862467
+93 92 -0.00774711670277443
+94 92 -0.04265714275977608
+95 92 -0.1173933022977732
+93 93  0.3015270103862467
+94 93 -0.00774711670277443
+95 93 -0.04265714275977608
+96 93 -0.1173933022977732
+94 94  0.3015270103862467
+95 94 -0.00774711670277443
+96 94 -0.04265714275977608
+97 94 -0.1173933022977732
+95 95  0.3015270103862467
+96 95 -0.00774711670277443
+97 95 -0.04265714275977608
+98 95 -0.1173933022977732
+96 96  0.3015270103862467
+97 96 -0.00774711670277443
+98 96 -0.04265714275977608
+99 96 -0.278672717928978
+97 97  0.3015270103862467
+98 97 -0.00774711670277443
+99 97 -0.1072409551079743
+100 97 -0.2704350793462527
+98 98  0.3015270103862467
+99 98 -0.02614797503373225
+100 98 -0.1019634419302533
+99 99  0.6944430429619259
+100 99 -0.07979747629324176
+100 100  0.7004857084284116
diff --git a/omp/preconditioner/isai_kernels.cpp b/omp/preconditioner/isai_kernels.cpp
index d1482c64074..a33654bd33e 100644
--- a/omp/preconditioner/isai_kernels.cpp
+++ b/omp/preconditioner/isai_kernels.cpp
@@ -151,8 +151,9 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                     forall_matching(
                         m_cols + m_begin, m_size, i_cols + i_begin, i_size,
                         [&](IndexType, IndexType m_idx, IndexType i_idx) {
-                            if (m_cols[m_idx + m_begin] < row && col == row)
+                            if (m_cols[m_idx + m_begin] < row && col == row) {
                                 rhs_one_idx++;
+                            }
                             dense_system(i, i_idx) = m_vals[m_idx + m_begin];
                         });
                 }
@@ -277,12 +278,12 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
                               const matrix::Csr<ValueType, IndexType> *mtx,
                               matrix::Csr<ValueType, IndexType> *inverse_mtx,
                               IndexType *excess_rhs_ptrs,
-                              IndexType *excess_nz_ptrs)
+                              IndexType *excess_nz_ptrs, bool spd)
 {
     using std::swap;
     auto general_solve =
-        [](const range<accessor::row_major<ValueType, 2>> system,
-           ValueType *rhs, const IndexType rhs_one_idx) {
+        [spd](const range<accessor::row_major<ValueType, 2>> system,
+              ValueType *rhs, const IndexType rhs_one_idx) {
             const IndexType size = system.length(0);
             if (size <= 0) {
                 return;
@@ -301,15 +302,12 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
             }
 
             // solve transposed system
-            std::vector<IndexType> perm(size, {});
-            std::iota(begin(perm), end(perm), IndexType{0});
             for (IndexType col = 0; col < size; col++) {
                 const auto row =
                     choose_pivot(size - col,
                                  transposed_system + col * (size + 1), size) +
                     col;
                 swap_rows(col, row, size, transposed_system, size);
-                swap(perm[row], perm[col]);
                 swap(rhs[row], rhs[col]);
 
                 const auto d = transposed_system[col * size + col];
@@ -333,6 +331,13 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
                 rhs[col] /= d;
                 transposed_system[col * size + col] = one<ValueType>() / d;
             }
+
+            if (spd) {
+                const auto scal = one<ValueType>() / sqrt(rhs[size - 1]);
+                for (IndexType row = 0; row < size; row++) {
+                    rhs[row] *= scal;
+                }
+            }
         };
 
     generic_generate(exec, mtx, inverse_mtx, excess_rhs_ptrs, excess_nz_ptrs,
@@ -351,7 +356,7 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
                             const IndexType *excess_nz_ptrs,
                             matrix::Csr<ValueType, IndexType> *excess_system,
                             matrix::Dense<ValueType> *excess_rhs,
-                            const size_type e_start, const size_type e_end)
+                            size_type e_start, size_type e_end)
 {
     const auto num_rows = input->get_size()[0];
     const auto m_row_ptrs = input->get_const_row_ptrs();
@@ -407,12 +412,36 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void scale_excess_solution(std::shared_ptr<const DefaultExecutor>,
+                           const IndexType *excess_block_ptrs,
+                           matrix::Dense<ValueType> *excess_solution,
+                           size_type e_start, size_type e_end)
+{
+    auto excess_values = excess_solution->get_values();
+    auto offset = excess_block_ptrs[e_start];
+#pragma omp parallel for
+    for (size_type row = e_start; row < e_end; ++row) {
+        IndexType block_start = excess_block_ptrs[row] - offset;
+        IndexType block_end = excess_block_ptrs[row + 1] - offset;
+        const ValueType scal =
+            one<ValueType>() / sqrt(excess_values[block_end - 1]);
+        for (size_type i = block_start; i < block_end; i++) {
+            excess_values[i] *= scal;
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_SCALE_EXCESS_SOLUTION_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void scatter_excess_solution(std::shared_ptr<const DefaultExecutor>,
                              const IndexType *excess_block_ptrs,
                              const matrix::Dense<ValueType> *excess_solution,
                              matrix::Csr<ValueType, IndexType> *inverse,
-                             const size_type e_start, const size_type e_end)
+                             size_type e_start, size_type e_end)
 {
     auto excess_values = excess_solution->get_const_values();
     auto values = inverse->get_values();
diff --git a/omp/test/preconditioner/isai_kernels.cpp b/omp/test/preconditioner/isai_kernels.cpp
index 76c413e30b0..c42a1f08096 100644
--- a/omp/test/preconditioner/isai_kernels.cpp
+++ b/omp/test/preconditioner/isai_kernels.cpp
@@ -184,9 +184,10 @@ TEST_F(Isai, OmpIsaiGenerateAinverseShortIsEquivalentToRef)
     auto da2 = da1;
 
     gko::kernels::reference::isai::generate_general_inverse(
-        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
     gko::kernels::omp::isai::generate_general_inverse(
-        omp, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data());
+        omp, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
+        false);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, r<value_type>::value);
@@ -252,9 +253,10 @@ TEST_F(Isai, OmpIsaiGenerateAinverseLongIsEquivalentToRef)
     auto da2 = da1;
 
     gko::kernels::reference::isai::generate_general_inverse(
-        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
     gko::kernels::omp::isai::generate_general_inverse(
-        omp, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data());
+        omp, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
+        false);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
     GKO_ASSERT_MTX_NEAR(inverse, d_inverse, r<value_type>::value);
@@ -333,7 +335,7 @@ TEST_F(Isai, OmpIsaiGenerateExcessAinverseLongIsEquivalentToRef)
     gko::Array<index_type> a1(ref, num_rows + 1);
     auto a2 = a1;
     gko::kernels::reference::isai::generate_general_inverse(
-        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
     gko::Array<index_type> da1(omp, a1);
     gko::Array<index_type> da2(omp, a2);
     auto e_dim = a1.get_data()[num_rows];
@@ -357,6 +359,31 @@ TEST_F(Isai, OmpIsaiGenerateExcessAinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, OmpIsaiScaleExcessSolutionIsEquivalentToRef)
+{
+    initialize_data(matrix_type::lower, 572, 52);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_tri_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::Array<index_type> da1(omp, a1);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(omp);
+    de_rhs->copy_from(lend(e_rhs));
+    d_inverse->copy_from(lend(inverse));
+
+    gko::kernels::reference::isai::scale_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), 0, num_rows);
+    gko::kernels::omp::isai::scale_excess_solution(omp, da1.get_const_data(),
+                                                   de_rhs.get(), 0, num_rows);
+
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+}
+
+
 TEST_F(Isai, OmpIsaiScatterExcessSolutionLIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 572, 52);
@@ -417,7 +444,7 @@ TEST_F(Isai, OmpIsaiScatterExcessSolutionAIsEquivalentToRef)
     gko::Array<index_type> a1(ref, num_rows + 1);
     auto a2 = a1;
     gko::kernels::reference::isai::generate_general_inverse(
-        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data());
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
     gko::Array<index_type> da1(omp, a1);
     auto e_dim = a1.get_data()[num_rows];
     auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
diff --git a/reference/preconditioner/isai_kernels.cpp b/reference/preconditioner/isai_kernels.cpp
index 01ba055b76c..8e3d0fe9327 100644
--- a/reference/preconditioner/isai_kernels.cpp
+++ b/reference/preconditioner/isai_kernels.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
+#include <iostream>
 #include <memory>
 
 
@@ -142,8 +143,9 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                 forall_matching(
                     m_cols + m_begin, m_size, i_cols + i_begin, i_size,
                     [&](IndexType, IndexType m_idx, IndexType i_idx) {
-                        if (m_cols[m_idx + m_begin] < row && col == row)
+                        if (m_cols[m_idx + m_begin] < row && col == row) {
                             rhs_one_idx++;
+                        }
                         dense_system(i, i_idx) = m_vals[m_idx + m_begin];
                     });
             }
@@ -264,12 +266,12 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
                               const matrix::Csr<ValueType, IndexType> *mtx,
                               matrix::Csr<ValueType, IndexType> *inverse_mtx,
                               IndexType *excess_rhs_ptrs,
-                              IndexType *excess_nz_ptrs)
+                              IndexType *excess_nz_ptrs, bool spd)
 {
     using std::swap;
     auto general_solve =
-        [](const range<accessor::row_major<ValueType, 2>> system,
-           ValueType *rhs, const IndexType rhs_one_idx) {
+        [spd](const range<accessor::row_major<ValueType, 2>> system,
+              ValueType *rhs, const IndexType rhs_one_idx) {
             const IndexType size = system.length(0);
             if (size <= 0) {
                 return;
@@ -318,6 +320,13 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
                 rhs[col] /= d;
                 transposed_system[col * size + col] = one<ValueType>() / d;
             }
+
+            if (spd) {
+                const auto scal = one<ValueType>() / sqrt(rhs[size - 1]);
+                for (IndexType row = 0; row < size; row++) {
+                    rhs[row] *= scal;
+                }
+            }
         };
 
     generic_generate(exec, mtx, inverse_mtx, excess_rhs_ptrs, excess_nz_ptrs,
@@ -335,7 +344,7 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
                             const IndexType *, const IndexType *,
                             matrix::Csr<ValueType, IndexType> *excess_system,
                             matrix::Dense<ValueType> *excess_rhs,
-                            const size_type e_start, const size_type e_end)
+                            size_type e_start, size_type e_end)
 {
     const auto num_rows = input->get_size()[0];
     const auto m_row_ptrs = input->get_const_row_ptrs();
@@ -389,12 +398,37 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ISAI_GENERATE_EXCESS_SYSTEM_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void scale_excess_solution(std::shared_ptr<const DefaultExecutor>,
+                           const IndexType *excess_block_ptrs,
+                           matrix::Dense<ValueType> *excess_solution,
+                           size_type e_start, size_type e_end)
+{
+    auto excess_values = excess_solution->get_values();
+    IndexType block_start = 0;
+    IndexType block_end = 0;
+    auto offset = excess_block_ptrs[e_start];
+    for (size_type row = e_start; row < e_end; ++row) {
+        block_start = excess_block_ptrs[row] - offset;
+        block_end = excess_block_ptrs[row + 1] - offset;
+        const ValueType scal =
+            one<ValueType>() / sqrt(excess_values[block_end - 1]);
+        for (size_type i = block_start; i < block_end; i++) {
+            excess_values[i] *= scal;
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ISAI_SCALE_EXCESS_SOLUTION_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void scatter_excess_solution(std::shared_ptr<const DefaultExecutor>,
                              const IndexType *excess_block_ptrs,
                              const matrix::Dense<ValueType> *excess_solution,
                              matrix::Csr<ValueType, IndexType> *inverse,
-                             const size_type e_start, const size_type e_end)
+                             size_type e_start, size_type e_end)
 {
     auto excess_values = excess_solution->get_const_values();
     auto values = inverse->get_values();
diff --git a/reference/test/preconditioner/isai_kernels.cpp b/reference/test/preconditioner/isai_kernels.cpp
index 6f3b501ae0a..eeb3e9f20bf 100644
--- a/reference/test/preconditioner/isai_kernels.cpp
+++ b/reference/test/preconditioner/isai_kernels.cpp
@@ -68,6 +68,7 @@ class Isai : public ::testing::Test {
     using UpperIsai = gko::preconditioner::UpperIsai<value_type, index_type>;
     using GeneralIsai =
         gko::preconditioner::GeneralIsai<value_type, index_type>;
+    using SpdIsai = gko::preconditioner::SpdIsai<value_type, index_type>;
     using Mtx = gko::matrix::Csr<value_type, index_type>;
     using Dense = gko::matrix::Dense<value_type>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
@@ -88,12 +89,19 @@ class Isai : public ::testing::Test {
               {{4., 1., -1.}, {0., -2., 4.}, {0., 0., 8.}}, exec)},
           u_dense_inv{gko::initialize<Dense>(
               {{.25, .125, -0.03125}, {0., -.5, .25}, {0., 0., .125}}, exec)},
+          spd_dense{gko::initialize<Dense>(
+              {{.0625, -.0625, .25}, {-.0625, .3125, -.5}, {.25, -.5, 2.25}},
+              exec)},
+          spd_dense_inv{gko::initialize<Dense>(
+              {{4., 0., 0.}, {2., 2., 0.}, {-3., 1., 1.}}, exec)},
           a_csr{Csr::create(exec)},
           a_csr_inv{Csr::create(exec)},
           l_csr{Csr::create(exec)},
           l_csr_inv{Csr::create(exec)},
           u_csr{Csr::create(exec)},
           u_csr_inv{Csr::create(exec)},
+          spd_csr{Csr::create(exec)},
+          spd_csr_inv{Csr::create(exec)},
           l_sparse{Csr::create(exec, gko::dim<2>(4, 4),
                                I<value_type>{-1., 2., 4., 5., -4., 8., -8.},
                                I<index_type>{0, 0, 1, 1, 2, 2, 3},
@@ -147,29 +155,45 @@ class Isai : public ::testing::Test {
                                                   .125, -.0625, -.5, .25, .5},
                                     I<index_type>{0, 1, 2, 3, 1, 2, 3, 2, 3, 3},
                                     I<index_type>{0, 4, 7, 9, 10})},
-          a_sparse{Csr::create(exec, gko::dim<2>{4, 4},
-                               I<value_type>{1, 4, 1, 4, 2, 1, 2, 1, 1},
-                               I<index_type>{0, 3, 0, 1, 3, 1, 2, 2, 3},
-                               I<index_type>{0, 2, 5, 7, 9})},
-          a_s_unsorted{Csr::create(exec, gko::dim<2>{4, 4},
-                                   I<value_type>{4, 1, 1, 2, 4, 2, 1, 1, 1},
-                                   I<index_type>{3, 0, 0, 3, 1, 2, 1, 2, 3},
-                                   I<index_type>{0, 2, 5, 7, 9})},
+          a_sparse{
+              Csr::create(exec, gko::dim<2>{4, 4},
+                          I<value_type>{1., 4., 1., 4., 2., 1., 2., 1., 1.},
+                          I<index_type>{0, 3, 0, 1, 3, 1, 2, 2, 3},
+                          I<index_type>{0, 2, 5, 7, 9})},
+          a_s_unsorted{
+              Csr::create(exec, gko::dim<2>{4, 4},
+                          I<value_type>{4., 1., 1., 2., 4., 2., 1., 1., 1.},
+                          I<index_type>{3, 0, 0, 3, 1, 2, 1, 2, 3},
+                          I<index_type>{0, 2, 5, 7, 9})},
           a_sparse_inv{Csr::create(
               exec, gko::dim<2>{4, 4},
-              I<value_type>{1, -4, -.25, .25, .5, -.125, .5, -.5, 1},
+              I<value_type>{1., -4, -.25, .25, .5, -.125, .5, -.5, 1},
               I<index_type>{0, 3, 0, 1, 3, 1, 2, 2, 3},
-              I<index_type>{0, 2, 5, 7, 9})}
+              I<index_type>{0, 2, 5, 7, 9})},
+          spd_sparse{Csr::create(exec, gko::dim<2>{4, 4},
+                                 I<value_type>{.25, -.25, -.25, .5, 4., 4.},
+                                 I<index_type>{0, 1, 0, 1, 2, 3},
+                                 I<index_type>{0, 2, 4, 5, 6})},
+          spd_s_unsorted{Csr::create(exec, gko::dim<2>{4, 4},
+                                     I<value_type>{-.25, .25, .5, -.25, 4., 4.},
+                                     I<index_type>{1, 0, 1, 0, 2, 3},
+                                     I<index_type>{0, 2, 4, 5, 6})},
+          spd_sparse_inv{Csr::create(
+              exec, gko::dim<2>{4, 4}, I<value_type>{2., 2., 2., .5, .5},
+              I<index_type>{0, 0, 1, 2, 3}, I<index_type>{0, 1, 3, 4, 5})}
     {
         lower_isai_factory = LowerIsai::build().on(exec);
         upper_isai_factory = UpperIsai::build().on(exec);
         general_isai_factory = GeneralIsai::build().on(exec);
+        spd_isai_factory = SpdIsai::build().on(exec);
         a_dense->convert_to(lend(a_csr));
         a_dense_inv->convert_to(lend(a_csr_inv));
         l_dense->convert_to(lend(l_csr));
         l_dense_inv->convert_to(lend(l_csr_inv));
         u_dense->convert_to(lend(u_csr));
         u_dense_inv->convert_to(lend(u_csr_inv));
+        spd_dense->convert_to(lend(spd_csr));
+        spd_dense_inv->convert_to(lend(spd_csr_inv));
         l_csr_longrow = read<Csr>("isai_l.mtx");
         l_csr_longrow_e = read<Csr>("isai_l_excess.mtx");
         l_csr_longrow_e_rhs = read<Dense>("isai_l_excess_rhs.mtx");
@@ -185,6 +209,11 @@ class Isai : public ::testing::Test {
         a_csr_longrow_e_rhs = read<Dense>("isai_a_excess_rhs.mtx");
         a_csr_longrow_inv_partial = read<Csr>("isai_a_inv_partial.mtx");
         a_csr_longrow_inv = read<Csr>("isai_a_inv.mtx");
+        spd_csr_longrow = read<Csr>("isai_spd.mtx");
+        spd_csr_longrow_e = read<Csr>("isai_spd_excess.mtx");
+        spd_csr_longrow_e_rhs = read<Dense>("isai_spd_excess_rhs.mtx");
+        spd_csr_longrow_inv_partial = read<Csr>("isai_spd_inv_partial.mtx");
+        spd_csr_longrow_inv = read<Csr>("isai_spd_inv.mtx");
     }
 
     template <typename ReadMtx>
@@ -222,12 +251,15 @@ class Isai : public ::testing::Test {
     std::unique_ptr<typename LowerIsai::Factory> lower_isai_factory;
     std::unique_ptr<typename UpperIsai::Factory> upper_isai_factory;
     std::unique_ptr<typename GeneralIsai::Factory> general_isai_factory;
+    std::unique_ptr<typename SpdIsai::Factory> spd_isai_factory;
     std::shared_ptr<Dense> a_dense;
     std::shared_ptr<Dense> a_dense_inv;
     std::shared_ptr<Dense> l_dense;
     std::shared_ptr<Dense> l_dense_inv;
     std::shared_ptr<Dense> u_dense;
     std::shared_ptr<Dense> u_dense_inv;
+    std::shared_ptr<Dense> spd_dense;
+    std::shared_ptr<Dense> spd_dense_inv;
     std::shared_ptr<Csr> a_csr;
     std::shared_ptr<Csr> a_csr_inv;
     std::shared_ptr<Csr> l_csr;
@@ -263,6 +295,16 @@ class Isai : public ::testing::Test {
     std::shared_ptr<Dense> a_csr_longrow_e_rhs;
     std::shared_ptr<Csr> a_csr_longrow_inv_partial;
     std::shared_ptr<Csr> a_csr_longrow_inv;
+    std::shared_ptr<Csr> spd_csr;
+    std::shared_ptr<Csr> spd_csr_inv;
+    std::shared_ptr<Csr> spd_csr_longrow;
+    std::shared_ptr<Csr> spd_csr_longrow_e;
+    std::shared_ptr<Dense> spd_csr_longrow_e_rhs;
+    std::shared_ptr<Csr> spd_csr_longrow_inv_partial;
+    std::shared_ptr<Csr> spd_csr_longrow_inv;
+    std::shared_ptr<Csr> spd_sparse;
+    std::shared_ptr<Csr> spd_s_unsorted;
+    std::shared_ptr<Csr> spd_sparse_inv;
 };
 
 TYPED_TEST_SUITE(Isai, gko::test::ValueIndexTypes);
@@ -283,7 +325,7 @@ TYPED_TEST(Isai, KernelGenerateA)
 
     gko::kernels::reference::isai::generate_general_inverse(
         this->exec, lend(this->a_csr), lend(result), a1.get_data(),
-        a2.get_data());
+        a2.get_data(), false);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(result, this->a_csr_inv);
     GKO_ASSERT_MTX_NEAR(result, this->a_csr_inv, r<value_type>::value);
@@ -309,7 +351,7 @@ TYPED_TEST(Isai, KernelGenerateA2)
 
     gko::kernels::reference::isai::generate_general_inverse(
         this->exec, lend(a_transpose), lend(result), a1.get_data(),
-        a2.get_data());
+        a2.get_data(), false);
 
     const auto expected = this->transpose(lend(this->a_csr_inv));
     GKO_ASSERT_MTX_EQ_SPARSITY(result, expected);
@@ -335,7 +377,7 @@ TYPED_TEST(Isai, KernelGenerateAsparse)
 
     gko::kernels::reference::isai::generate_general_inverse(
         this->exec, lend(this->a_sparse), lend(result), a1.get_data(),
-        a2.get_data());
+        a2.get_data(), false);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(result, this->a_sparse_inv);
     GKO_ASSERT_MTX_NEAR(result, this->a_sparse_inv, r<value_type>::value);
@@ -367,7 +409,7 @@ TYPED_TEST(Isai, KernelGenerateALongrow)
 
     gko::kernels::reference::isai::generate_general_inverse(
         this->exec, lend(this->a_csr_longrow), lend(result), a1.get_data(),
-        a2.get_data());
+        a2.get_data(), false);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(result, this->a_csr_longrow_inv_partial);
     GKO_ASSERT_MTX_NEAR(result, this->a_csr_longrow_inv_partial,
@@ -806,6 +848,115 @@ TYPED_TEST(Isai, KernelGenerateExcessULongrow)
 }
 
 
+TYPED_TEST(Isai, KernelGenerateSpd)
+{
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto result = this->clone_allocations(lend(this->spd_csr_inv));
+    auto num_rows = result->get_size()[0];
+    gko::Array<index_type> a1(this->exec, num_rows + 1);
+    gko::Array<index_type> a2(this->exec, num_rows + 1);
+    // zero-filled array
+    gko::Array<index_type> zeros(this->exec, num_rows + 1);
+    std::fill_n(zeros.get_data(), num_rows + 1, 0);
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        this->exec, lend(this->spd_csr), lend(result), a1.get_data(),
+        a2.get_data(), true);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(result, this->spd_csr_inv);
+    GKO_ASSERT_MTX_NEAR(result, this->spd_csr_inv, r<value_type>::value);
+    // no row above the size limit -> zero array
+    GKO_ASSERT_ARRAY_EQ(a1, zeros);
+    GKO_ASSERT_ARRAY_EQ(a2, zeros);
+}
+
+
+TYPED_TEST(Isai, KernelGenerateSpdsparse)
+{
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto result = this->clone_allocations(lend(this->spd_sparse_inv));
+    auto num_rows = result->get_size()[0];
+    gko::Array<index_type> a1(this->exec, num_rows + 1);
+    gko::Array<index_type> a2(this->exec, num_rows + 1);
+    // zero-filled array
+    gko::Array<index_type> zeros(this->exec, num_rows + 1);
+    std::fill_n(zeros.get_data(), num_rows + 1, 0);
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        this->exec, lend(this->spd_sparse), lend(result), a1.get_data(),
+        a2.get_data(), true);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(result, this->spd_sparse_inv);
+    GKO_ASSERT_MTX_NEAR(result, this->spd_sparse_inv, r<value_type>::value);
+    // no row above the size limit -> zero array
+    GKO_ASSERT_ARRAY_EQ(a1, zeros);
+    GKO_ASSERT_ARRAY_EQ(a2, zeros);
+}
+
+
+TYPED_TEST(Isai, KernelGenerateSpdLongrow)
+{
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto result = this->clone_allocations(lend(this->spd_csr_longrow_inv));
+    auto num_rows = result->get_size()[0];
+    gko::Array<index_type> a1(this->exec, num_rows + 1);
+    gko::Array<index_type> a2(this->exec, num_rows + 1);
+    // zero-filled array
+    gko::Array<index_type> zeros(this->exec, num_rows + 1);
+    std::fill_n(zeros.get_data(), num_rows + 1, 0);
+    // only the 36th row has some excess storage
+    auto a1_expect = zeros;
+    std::fill_n(a1_expect.get_data() + 36, 65, 36);
+    auto a2_expect = zeros;
+    std::fill_n(a2_expect.get_data() + 36, 65, 338);
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        this->exec, lend(this->spd_csr_longrow), lend(result), a1.get_data(),
+        a2.get_data(), true);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(result, this->spd_csr_longrow_inv_partial);
+    GKO_ASSERT_MTX_NEAR(result, this->spd_csr_longrow_inv_partial,
+                        r<value_type>::value);
+    // no row above the size limit -> zero array
+    GKO_ASSERT_ARRAY_EQ(a1, a1_expect);
+    GKO_ASSERT_ARRAY_EQ(a2, a2_expect);
+}
+
+
+TYPED_TEST(Isai, KernelGenerateExcessSpdLongrow)
+{
+    using Csr = typename TestFixture::Csr;
+    using Dense = typename TestFixture::Dense;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    auto num_rows = this->spd_csr_longrow->get_size()[0];
+    gko::Array<index_type> zeros(this->exec, num_rows + 1);
+    std::fill_n(zeros.get_data(), num_rows + 1, 0);
+    // only the 36th row has some excess storage
+    auto a1 = zeros;
+    std::fill_n(a1.get_data() + 36, 65, 36);
+    auto a2 = zeros;
+    std::fill_n(a2.get_data() + 36, 65, 338);
+    auto result = Csr::create(this->exec, gko::dim<2>(36, 36), 338);
+    auto result_rhs = Dense::create(this->exec, gko::dim<2>(36, 1));
+
+    gko::kernels::reference::isai::generate_excess_system(
+        this->exec, lend(this->spd_csr_longrow),
+        lend(this->spd_csr_longrow_inv_partial), a1.get_const_data(),
+        a2.get_const_data(), lend(result), lend(result_rhs), 0, num_rows);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(result, this->spd_csr_longrow_e);
+    GKO_ASSERT_MTX_NEAR(result, this->spd_csr_longrow_e, 0);
+    GKO_ASSERT_MTX_NEAR(result_rhs, this->spd_csr_longrow_e_rhs, 0);
+}
+
+
 TYPED_TEST(Isai, KernelScatterExcessSolution)
 {
     using Csr = typename TestFixture::Csr;
@@ -855,7 +1006,7 @@ TYPED_TEST(Isai, ReturnsCorrectInverseALongrow)
     // need to reduce precision due to general ISAI using GMRES instead of
     // direct solve
     GKO_ASSERT_MTX_NEAR(a_inv, this->a_csr_longrow_inv,
-                        10 * r<value_type>::value);
+                        20 * r<value_type>::value);
 }
 
 
@@ -971,6 +1122,48 @@ TYPED_TEST(Isai, ReturnsCorrectInverseUWithU3)
 }
 
 
+TYPED_TEST(Isai, ReturnsCorrectInverseSpd)
+{
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    const auto isai = this->spd_isai_factory->generate(this->spd_sparse);
+    const auto expected_transpose =
+        gko::as<Csr>(this->spd_sparse_inv->transpose());
+
+    // In the spd case, the approximate inverse is a composition of L^T and L.
+    const auto composition = isai->get_approximate_inverse()->get_operators();
+    const auto lower_t = gko::as<Csr>(composition[0]);
+    const auto lower = gko::as<Csr>(composition[1]);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(lower, this->spd_sparse_inv);
+    GKO_ASSERT_MTX_EQ_SPARSITY(lower_t, expected_transpose);
+    GKO_ASSERT_MTX_NEAR(lower, this->spd_sparse_inv, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(lower_t, expected_transpose, r<value_type>::value);
+}
+
+
+TYPED_TEST(Isai, ReturnsCorrectInverseSpdLongrow)
+{
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    const auto isai = this->spd_isai_factory->generate(this->spd_csr_longrow);
+    const auto expected_transpose =
+        gko::as<Csr>(this->spd_csr_longrow_inv->transpose());
+
+    const auto composition = isai->get_approximate_inverse()->get_operators();
+    const auto lower_t = gko::as<Csr>(composition[0]);
+    const auto lower = gko::as<Csr>(composition[1]);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(lower, this->spd_csr_longrow_inv);
+    GKO_ASSERT_MTX_EQ_SPARSITY(lower_t, expected_transpose);
+    // need to reduce precision due to spd ISAI using GMRES instead of
+    // direct solve
+    GKO_ASSERT_MTX_NEAR(lower, this->spd_csr_longrow_inv,
+                        10 * r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(lower_t, expected_transpose, 10 * r<value_type>::value);
+}
+
+
 TYPED_TEST(Isai, GeneratesWithUnsortedCsr)
 {
     using Csr = typename TestFixture::Csr;
@@ -980,13 +1173,23 @@ TYPED_TEST(Isai, GeneratesWithUnsortedCsr)
         this->general_isai_factory->generate(this->a_s_unsorted);
     const auto l_isai = this->lower_isai_factory->generate(this->l_s_unsorted);
     const auto u_isai = this->upper_isai_factory->generate(this->u_s_unsorted);
+    const auto spd_isai =
+        this->spd_isai_factory->generate(this->spd_s_unsorted);
     auto a_inv = a_isai->get_approximate_inverse();
     auto l_inv = l_isai->get_approximate_inverse();
     auto u_inv = u_isai->get_approximate_inverse();
+    auto spd_l_t_inv =
+        gko::as<Csr>(spd_isai->get_approximate_inverse()->get_operators()[0]);
+    auto spd_l_inv =
+        gko::as<Csr>(spd_isai->get_approximate_inverse()->get_operators()[1]);
 
     GKO_ASSERT_MTX_NEAR(a_inv, this->a_sparse_inv, r<T>::value);
     GKO_ASSERT_MTX_NEAR(l_inv, this->l_sparse_inv, r<T>::value);
     GKO_ASSERT_MTX_NEAR(u_inv, this->u_sparse_inv, r<T>::value);
+    GKO_ASSERT_MTX_NEAR(spd_l_inv, this->spd_sparse_inv, r<T>::value);
+    GKO_ASSERT_MTX_NEAR(spd_l_t_inv,
+                        gko::as<Csr>(this->spd_sparse_inv->transpose()),
+                        r<T>::value);
 }
 
 
@@ -1032,6 +1235,20 @@ TYPED_TEST(Isai, ApplyWithUMtx)
 }
 
 
+TYPED_TEST(Isai, ApplyWithSpdComposition)
+{
+    using Dense = typename TestFixture::Dense;
+    using T = typename TestFixture::value_type;
+    const auto vec = gko::initialize<Dense>({18., 16., 12.}, this->exec);
+    auto result = Dense::create_with_config_of(lend(vec));
+    const auto spd_isai = this->spd_isai_factory->generate(this->spd_dense);
+
+    spd_isai->apply(lend(vec), lend(result));
+
+    GKO_ASSERT_MTX_NEAR(result, l({502., 110., -26.}), r<T>::value);
+}
+
+
 TYPED_TEST(Isai, AdvancedApplyAMtx)
 {
     using Dense = typename TestFixture::Dense;
@@ -1080,6 +1297,22 @@ TYPED_TEST(Isai, AdvancedApplyUMtx)
 }
 
 
+TYPED_TEST(Isai, AdvancedApplySpdComposition)
+{
+    using Dense = typename TestFixture::Dense;
+    using T = typename TestFixture::value_type;
+    const auto alpha = gko::initialize<Dense>({3.}, this->exec);
+    const auto beta = gko::initialize<Dense>({-4.}, this->exec);
+    const auto vec = gko::initialize<Dense>({18., 16., 12}, this->exec);
+    auto result = gko::initialize<Dense>({2., -3., 1.}, this->exec);
+    const auto spd_isai = this->spd_isai_factory->generate(this->spd_dense);
+
+    spd_isai->apply(lend(alpha), lend(vec), lend(beta), lend(result));
+
+    GKO_ASSERT_MTX_NEAR(result, l({1498., 342., -82.}), r<T>::value);
+}
+
+
 TYPED_TEST(Isai, UseWithIluPreconditioner)
 {
     using Dense = typename TestFixture::Dense;
@@ -1150,6 +1383,28 @@ TYPED_TEST(Isai, ReturnsTransposedCorrectInverseU)
 }
 
 
+TYPED_TEST(Isai, ReturnsTransposedCorrectInverseSpd)
+{
+    using SpdIsai = typename TestFixture::SpdIsai;
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    const auto isai = this->spd_isai_factory->generate(this->spd_sparse);
+
+    auto transpose =
+        gko::as<SpdIsai>(isai->transpose())->get_approximate_inverse();
+    auto lower_t = gko::as<Csr>(transpose->get_operators()[0]);
+    auto lower = gko::as<Csr>(transpose->get_operators()[1]);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(lower, this->spd_sparse_inv);
+    GKO_ASSERT_MTX_EQ_SPARSITY(lower_t,
+                               gko::as<Csr>(this->spd_sparse_inv->transpose()));
+    GKO_ASSERT_MTX_NEAR(lower, this->spd_sparse_inv, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(lower_t,
+                        gko::as<Csr>(this->spd_sparse_inv->transpose()),
+                        r<value_type>::value);
+}
+
+
 TYPED_TEST(Isai, ReturnsConjTransposedCorrectInverseA)
 {
     using GeneralIsai = typename TestFixture::GeneralIsai;
@@ -1198,4 +1453,26 @@ TYPED_TEST(Isai, ReturnsConjTransposedCorrectInverseU)
 }
 
 
+TYPED_TEST(Isai, ReturnsConjTransposedCorrectInverseSpd)
+{
+    using SpdIsai = typename TestFixture::SpdIsai;
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    const auto isai = this->spd_isai_factory->generate(this->spd_sparse);
+
+    auto transpose =
+        gko::as<SpdIsai>(isai->conj_transpose())->get_approximate_inverse();
+    auto lower_t = gko::as<Csr>(transpose->get_operators()[0]);
+    auto lower = gko::as<Csr>(transpose->get_operators()[1]);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(lower, this->spd_sparse_inv);
+    GKO_ASSERT_MTX_EQ_SPARSITY(
+        lower_t, gko::as<Csr>(this->spd_sparse_inv->conj_transpose()));
+    GKO_ASSERT_MTX_NEAR(lower, this->spd_sparse_inv, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(lower_t,
+                        gko::as<Csr>(this->spd_sparse_inv->conj_transpose()),
+                        r<value_type>::value);
+}
+
+
 }  // namespace

From 38fa81462b951880e08b741bcca67d01afdf791c Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 4 Mar 2021 11:57:43 +0100
Subject: [PATCH 0516/1008] Make Excess Solver Factory configurable

---
 common/preconditioner/isai_kernels.hpp.inc    |   1 -
 core/preconditioner/isai.cpp                  |  18 +--
 core/test/preconditioner/isai.cpp             |  95 ++++++++++++++++
 include/ginkgo/core/preconditioner/isai.hpp   |   6 +
 .../test/preconditioner/isai_kernels.cpp      | 106 ++++++++++++++++++
 5 files changed, 217 insertions(+), 9 deletions(-)

diff --git a/common/preconditioner/isai_kernels.hpp.inc b/common/preconditioner/isai_kernels.hpp.inc
index 3fb91135bea..1ce99948cbe 100644
--- a/common/preconditioner/isai_kernels.hpp.inc
+++ b/common/preconditioner/isai_kernels.hpp.inc
@@ -92,7 +92,6 @@ __forceinline__ __device__ void generic_generate(
         if (local_id == 0) {
             excess_rhs_sizes[row] = 0;
             excess_nnz[row] = 0;
-            rhs_one_idxs[row % subwarps_per_block] = 0;
         }
 
         // subwarp_size^2 storage per subwarp
diff --git a/core/preconditioner/isai.cpp b/core/preconditioner/isai.cpp
index 6fa5f7c7aee..7663f0b6bf2 100644
--- a/core/preconditioner/isai.cpp
+++ b/core/preconditioner/isai.cpp
@@ -222,9 +222,13 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
             system_copy->copy_from(excess_system.get());
             auto rhs_copy = Dense::create(exec->get_master());
             rhs_copy->copy_from(excess_rhs.get());
-            std::unique_ptr<LinOpFactory> trs_factory;
-            if (is_general || is_spd) {
-                trs_factory =
+            std::shared_ptr<LinOpFactory> excess_solver_factory;
+            if (parameters_.excess_solver_factory) {
+                excess_solver_factory =
+                    share(parameters_.excess_solver_factory);
+                excess_solution->copy_from(excess_rhs.get());
+            } else if (is_general || is_spd) {
+                excess_solver_factory =
                     Gmres::build()
                         .with_preconditioner(
                             bj::build().with_max_block_size(32u).on(exec))
@@ -237,14 +241,12 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
                                 .on(exec))
                         .on(exec);
                 excess_solution->copy_from(excess_rhs.get());
-                auto sol_copy = Dense::create(exec->get_master());
-                sol_copy->copy_from(excess_solution.get());
             } else if (is_lower) {
-                trs_factory = UpperTrs::build().on(exec);
+                excess_solver_factory = UpperTrs::build().on(exec);
             } else {
-                trs_factory = LowerTrs::build().on(exec);
+                excess_solver_factory = LowerTrs::build().on(exec);
             }
-            trs_factory->generate(share(excess_system->transpose()))
+            excess_solver_factory->generate(share(excess_system->transpose()))
                 ->apply(lend(excess_rhs), lend(excess_solution));
             if (is_spd) {
                 exec->run(isai::make_scale_excess_solution(
diff --git a/core/test/preconditioner/isai.cpp b/core/test/preconditioner/isai.cpp
index 823e8e66a0b..202dd33bde8 100644
--- a/core/test/preconditioner/isai.cpp
+++ b/core/test/preconditioner/isai.cpp
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/solver/bicgstab.hpp>
 
 
 #include "core/test/utils.hpp"
@@ -71,21 +72,27 @@ class IsaiFactory : public ::testing::Test {
         typename std::tuple_element<0, decltype(ValueIndexType())>::type;
     using index_type =
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using excess_solver_type = gko::solver::Bicgstab<value_type>;
     using GeneralIsai =
         gko::preconditioner::GeneralIsai<value_type, index_type>;
+    using SpdIsai = gko::preconditioner::SpdIsai<value_type, index_type>;
     using LowerIsai = gko::preconditioner::LowerIsai<value_type, index_type>;
     using UpperIsai = gko::preconditioner::UpperIsai<value_type, index_type>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
 
     IsaiFactory()
         : exec(gko::ReferenceExecutor::create()),
+          excess_solver_factory(excess_solver_type::build().on(exec)),
           general_isai_factory(GeneralIsai::build().on(exec)),
+          spd_isai_factory(SpdIsai::build().on(exec)),
           lower_isai_factory(LowerIsai::build().on(exec)),
           upper_isai_factory(UpperIsai::build().on(exec))
     {}
 
     std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<typename excess_solver_type::Factory> excess_solver_factory;
     std::unique_ptr<typename GeneralIsai::Factory> general_isai_factory;
+    std::unique_ptr<typename SpdIsai::Factory> spd_isai_factory;
     std::unique_ptr<typename LowerIsai::Factory> lower_isai_factory;
     std::unique_ptr<typename UpperIsai::Factory> upper_isai_factory;
 };
@@ -96,6 +103,7 @@ TYPED_TEST_SUITE(IsaiFactory, gko::test::ValueIndexTypes);
 TYPED_TEST(IsaiFactory, KnowsItsExecutor)
 {
     ASSERT_EQ(this->general_isai_factory->get_executor(), this->exec);
+    ASSERT_EQ(this->spd_isai_factory->get_executor(), this->exec);
     ASSERT_EQ(this->lower_isai_factory->get_executor(), this->exec);
     ASSERT_EQ(this->upper_isai_factory->get_executor(), this->exec);
 }
@@ -104,17 +112,21 @@ TYPED_TEST(IsaiFactory, KnowsItsExecutor)
 TYPED_TEST(IsaiFactory, SetsSkipSortingCorrectly)
 {
     using GeneralIsai = typename TestFixture::GeneralIsai;
+    using SpdIsai = typename TestFixture::SpdIsai;
     using LowerIsai = typename TestFixture::LowerIsai;
     using UpperIsai = typename TestFixture::UpperIsai;
 
     auto a_isai_factory =
         GeneralIsai::build().with_skip_sorting(true).on(this->exec);
+    auto spd_isai_factory =
+        SpdIsai::build().with_skip_sorting(true).on(this->exec);
     auto l_isai_factory =
         LowerIsai::build().with_skip_sorting(true).on(this->exec);
     auto u_isai_factory =
         UpperIsai::build().with_skip_sorting(true).on(this->exec);
 
     ASSERT_EQ(a_isai_factory->get_parameters().skip_sorting, true);
+    ASSERT_EQ(spd_isai_factory->get_parameters().skip_sorting, true);
     ASSERT_EQ(l_isai_factory->get_parameters().skip_sorting, true);
     ASSERT_EQ(u_isai_factory->get_parameters().skip_sorting, true);
 }
@@ -123,6 +135,7 @@ TYPED_TEST(IsaiFactory, SetsSkipSortingCorrectly)
 TYPED_TEST(IsaiFactory, SetsDefaultSkipSortingCorrectly)
 {
     ASSERT_EQ(this->general_isai_factory->get_parameters().skip_sorting, false);
+    ASSERT_EQ(this->spd_isai_factory->get_parameters().skip_sorting, false);
     ASSERT_EQ(this->lower_isai_factory->get_parameters().skip_sorting, false);
     ASSERT_EQ(this->upper_isai_factory->get_parameters().skip_sorting, false);
 }
@@ -131,17 +144,21 @@ TYPED_TEST(IsaiFactory, SetsDefaultSkipSortingCorrectly)
 TYPED_TEST(IsaiFactory, SetsSparsityPowerCorrectly)
 {
     using GeneralIsai = typename TestFixture::GeneralIsai;
+    using SpdIsai = typename TestFixture::SpdIsai;
     using LowerIsai = typename TestFixture::LowerIsai;
     using UpperIsai = typename TestFixture::UpperIsai;
 
     auto a_isai_factory =
         GeneralIsai::build().with_sparsity_power(2).on(this->exec);
+    auto spd_isai_factory =
+        SpdIsai::build().with_sparsity_power(2).on(this->exec);
     auto l_isai_factory =
         LowerIsai::build().with_sparsity_power(2).on(this->exec);
     auto u_isai_factory =
         UpperIsai::build().with_sparsity_power(2).on(this->exec);
 
     ASSERT_EQ(a_isai_factory->get_parameters().sparsity_power, 2);
+    ASSERT_EQ(spd_isai_factory->get_parameters().sparsity_power, 2);
     ASSERT_EQ(l_isai_factory->get_parameters().sparsity_power, 2);
     ASSERT_EQ(u_isai_factory->get_parameters().sparsity_power, 2);
 }
@@ -150,6 +167,7 @@ TYPED_TEST(IsaiFactory, SetsSparsityPowerCorrectly)
 TYPED_TEST(IsaiFactory, SetsDefaultSparsityPowerCorrectly)
 {
     ASSERT_EQ(this->general_isai_factory->get_parameters().sparsity_power, 1);
+    ASSERT_EQ(this->spd_isai_factory->get_parameters().sparsity_power, 1);
     ASSERT_EQ(this->lower_isai_factory->get_parameters().sparsity_power, 1);
     ASSERT_EQ(this->upper_isai_factory->get_parameters().sparsity_power, 1);
 }
@@ -158,17 +176,21 @@ TYPED_TEST(IsaiFactory, SetsDefaultSparsityPowerCorrectly)
 TYPED_TEST(IsaiFactory, SetsExcessLimitCorrectly)
 {
     using GeneralIsai = typename TestFixture::GeneralIsai;
+    using SpdIsai = typename TestFixture::SpdIsai;
     using LowerIsai = typename TestFixture::LowerIsai;
     using UpperIsai = typename TestFixture::UpperIsai;
 
     auto a_isai_factory =
         GeneralIsai::build().with_excess_limit(1024).on(this->exec);
+    auto spd_isai_factory =
+        SpdIsai::build().with_excess_limit(1024).on(this->exec);
     auto l_isai_factory =
         LowerIsai::build().with_excess_limit(1024).on(this->exec);
     auto u_isai_factory =
         UpperIsai::build().with_excess_limit(1024).on(this->exec);
 
     ASSERT_EQ(a_isai_factory->get_parameters().excess_limit, 1024);
+    ASSERT_EQ(spd_isai_factory->get_parameters().excess_limit, 1024);
     ASSERT_EQ(l_isai_factory->get_parameters().excess_limit, 1024);
     ASSERT_EQ(u_isai_factory->get_parameters().excess_limit, 1024);
 }
@@ -177,11 +199,64 @@ TYPED_TEST(IsaiFactory, SetsExcessLimitCorrectly)
 TYPED_TEST(IsaiFactory, SetsDefaultExcessLimitCorrectly)
 {
     ASSERT_EQ(this->general_isai_factory->get_parameters().excess_limit, 0);
+    ASSERT_EQ(this->spd_isai_factory->get_parameters().excess_limit, 0);
     ASSERT_EQ(this->lower_isai_factory->get_parameters().excess_limit, 0);
     ASSERT_EQ(this->upper_isai_factory->get_parameters().excess_limit, 0);
 }
 
 
+TYPED_TEST(IsaiFactory, CanSetExcessSolverFactoryA)
+{
+    using GeneralIsai = typename TestFixture::GeneralIsai;
+    auto general_isai_factory =
+        GeneralIsai::build()
+            .with_excess_solver_factory(this->excess_solver_factory)
+            .on(this->exec);
+
+    ASSERT_EQ(general_isai_factory->get_parameters().excess_solver_factory,
+              this->excess_solver_factory);
+}
+
+
+TYPED_TEST(IsaiFactory, CanSetExcessSolverFactorySpd)
+{
+    using SpdIsai = typename TestFixture::SpdIsai;
+    auto spd_isai_factory =
+        SpdIsai::build()
+            .with_excess_solver_factory(this->excess_solver_factory)
+            .on(this->exec);
+
+    ASSERT_EQ(spd_isai_factory->get_parameters().excess_solver_factory,
+              this->excess_solver_factory);
+}
+
+
+TYPED_TEST(IsaiFactory, CanSetExcessSolverFactoryL)
+{
+    using LowerIsai = typename TestFixture::LowerIsai;
+    auto lower_isai_factory =
+        LowerIsai::build()
+            .with_excess_solver_factory(this->excess_solver_factory)
+            .on(this->exec);
+
+    ASSERT_EQ(lower_isai_factory->get_parameters().excess_solver_factory,
+              this->excess_solver_factory);
+}
+
+
+TYPED_TEST(IsaiFactory, CanSetExcessSolverFactoryU)
+{
+    using UpperIsai = typename TestFixture::UpperIsai;
+    auto upper_isai_factory =
+        UpperIsai::build()
+            .with_excess_solver_factory(this->excess_solver_factory)
+            .on(this->exec);
+
+    ASSERT_EQ(upper_isai_factory->get_parameters().excess_solver_factory,
+              this->excess_solver_factory);
+}
+
+
 TYPED_TEST(IsaiFactory, ThrowsWrongDimensionA)
 {
     using Csr = typename TestFixture::Csr;
@@ -192,6 +267,16 @@ TYPED_TEST(IsaiFactory, ThrowsWrongDimensionA)
 }
 
 
+TYPED_TEST(IsaiFactory, ThrowsWrongDimensionSpd)
+{
+    using Csr = typename TestFixture::Csr;
+    auto mtx = Csr::create(this->exec, gko::dim<2>{1, 2}, 1);
+
+    ASSERT_THROW(this->spd_isai_factory->generate(gko::share(mtx)),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(IsaiFactory, ThrowsWrongDimensionL)
 {
     using Csr = typename TestFixture::Csr;
@@ -222,6 +307,16 @@ TYPED_TEST(IsaiFactory, ThrowsNoConversionCsrA)
 }
 
 
+TYPED_TEST(IsaiFactory, ThrowsNoConversionCsrSpd)
+{
+    using Csr = typename TestFixture::Csr;
+    auto mtx = DummyOperator::create(this->exec, gko::dim<2>{2, 2});
+
+    ASSERT_THROW(this->spd_isai_factory->generate(gko::share(mtx)),
+                 gko::NotSupported);
+}
+
+
 TYPED_TEST(IsaiFactory, ThrowsNoConversionCsrL)
 {
     using Csr = typename TestFixture::Csr;
diff --git a/include/ginkgo/core/preconditioner/isai.hpp b/include/ginkgo/core/preconditioner/isai.hpp
index ca0a5f1f75a..ba86bf9996a 100644
--- a/include/ginkgo/core/preconditioner/isai.hpp
+++ b/include/ginkgo/core/preconditioner/isai.hpp
@@ -177,6 +177,12 @@ class Isai : public EnableLinOp<Isai<IsaiType, ValueType, IndexType>>,
          * Must be at least 0, default value 0.
          */
         index_type GKO_FACTORY_PARAMETER_SCALAR(excess_limit, 0);
+
+        /**
+         * Factory for the Excess System solver
+         */
+        std::shared_ptr<LinOpFactory> GKO_FACTORY_PARAMETER_SCALAR(
+            excess_solver_factory, nullptr);
     };
 
     GKO_ENABLE_LIN_OP_FACTORY(Isai, parameters, Factory);
diff --git a/reference/test/preconditioner/isai_kernels.cpp b/reference/test/preconditioner/isai_kernels.cpp
index eeb3e9f20bf..3273a4c783f 100644
--- a/reference/test/preconditioner/isai_kernels.cpp
+++ b/reference/test/preconditioner/isai_kernels.cpp
@@ -47,6 +47,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/preconditioner/ilu.hpp>
+#include <ginkgo/core/preconditioner/jacobi.hpp>
+#include <ginkgo/core/solver/gmres.hpp>
 
 
 #include "core/preconditioner/isai_kernels.hpp"
@@ -64,6 +66,8 @@ class Isai : public ::testing::Test {
         typename std::tuple_element<0, decltype(ValueIndexType())>::type;
     using index_type =
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using excess_solver_type = gko::solver::Gmres<value_type>;
+    using bj = gko::preconditioner::Jacobi<value_type, index_type>;
     using LowerIsai = gko::preconditioner::LowerIsai<value_type, index_type>;
     using UpperIsai = gko::preconditioner::UpperIsai<value_type, index_type>;
     using GeneralIsai =
@@ -75,6 +79,17 @@ class Isai : public ::testing::Test {
 
     Isai()
         : exec{gko::ReferenceExecutor::create()},
+          excess_solver_factory(
+              excess_solver_type::build()
+                  .with_preconditioner(
+                      bj::build().with_max_block_size(16u).on(exec))
+                  .with_criteria(
+                      gko::stop::Iteration::build().with_max_iters(1000u).on(
+                          exec),
+                      gko::stop::RelativeResidualNorm<value_type>::build()
+                          .with_tolerance(gko::remove_complex<value_type>{1e-6})
+                          .on(exec))
+                  .on(exec)),
           a_dense{gko::initialize<Dense>({{2, 1, 2}, {1, -2, 3}, {-1, 1, 1}},
                                          exec)},
           a_dense_inv{gko::initialize<Dense>({{0.3125, -0.0625, -0.4375},
@@ -248,6 +263,7 @@ class Isai : public ::testing::Test {
     }
 
     std::shared_ptr<const gko::ReferenceExecutor> exec;
+    std::shared_ptr<typename excess_solver_type::Factory> excess_solver_factory;
     std::unique_ptr<typename LowerIsai::Factory> lower_isai_factory;
     std::unique_ptr<typename UpperIsai::Factory> upper_isai_factory;
     std::unique_ptr<typename GeneralIsai::Factory> general_isai_factory;
@@ -1010,6 +1026,26 @@ TYPED_TEST(Isai, ReturnsCorrectInverseALongrow)
 }
 
 
+TYPED_TEST(Isai, ReturnsCorrectInverseALongrowWithExcessSolver)
+{
+    using value_type = typename TestFixture::value_type;
+    using GeneralIsai = typename TestFixture::GeneralIsai;
+    auto general_isai_factory =
+        GeneralIsai::build()
+            .with_excess_solver_factory(this->excess_solver_factory)
+            .on(this->exec);
+    const auto isai = general_isai_factory->generate(this->a_csr_longrow);
+
+    auto a_inv = isai->get_approximate_inverse();
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(a_inv, this->a_csr_longrow_inv);
+    // need to drastically reduce precision due to using different excess solver
+    // factory.
+    GKO_ASSERT_MTX_NEAR(a_inv, this->a_csr_longrow_inv,
+                        2e4 * r<value_type>::value);
+}
+
+
 TYPED_TEST(Isai, ReturnsCorrectInverseL)
 {
     using Csr = typename TestFixture::Csr;
@@ -1036,6 +1072,27 @@ TYPED_TEST(Isai, ReturnsCorrectInverseLLongrow)
 }
 
 
+TYPED_TEST(Isai, ReturnsCorrectInverseLLongrowWithExcessSolver)
+{
+    using Csr = typename TestFixture::Csr;
+    using LowerIsai = typename TestFixture::LowerIsai;
+    using value_type = typename TestFixture::value_type;
+    auto lower_isai_factory =
+        LowerIsai::build()
+            .with_excess_solver_factory(this->excess_solver_factory)
+            .on(this->exec);
+    const auto isai = lower_isai_factory->generate(this->l_csr_longrow);
+
+    auto l_inv = isai->get_approximate_inverse();
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(l_inv, this->l_csr_longrow_inv);
+    // need to drastically reduce precision due to using different excess solver
+    // factory.
+    GKO_ASSERT_MTX_NEAR(l_inv, this->l_csr_longrow_inv,
+                        1e3 * r<value_type>::value);
+}
+
+
 TYPED_TEST(Isai, ReturnsCorrectInverseU)
 {
     using Csr = typename TestFixture::Csr;
@@ -1062,6 +1119,27 @@ TYPED_TEST(Isai, ReturnsCorrectInverseULongrow)
 }
 
 
+TYPED_TEST(Isai, ReturnsCorrectInverseULongrowWithExcessSolver)
+{
+    using Csr = typename TestFixture::Csr;
+    using UpperIsai = typename TestFixture::UpperIsai;
+    using value_type = typename TestFixture::value_type;
+    auto upper_isai_factory =
+        UpperIsai::build()
+            .with_excess_solver_factory(this->excess_solver_factory)
+            .on(this->exec);
+    const auto isai = upper_isai_factory->generate(this->u_csr_longrow);
+
+    auto u_inv = isai->get_approximate_inverse();
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(u_inv, this->u_csr_longrow_inv);
+    // need to drastically reduce precision due to using different excess solver
+    // factory.
+    GKO_ASSERT_MTX_NEAR(u_inv, this->u_csr_longrow_inv,
+                        1e3 * r<value_type>::value);
+}
+
+
 TYPED_TEST(Isai, ReturnsCorrectInverseLWithL2)
 {
     using value_type = typename TestFixture::value_type;
@@ -1164,6 +1242,34 @@ TYPED_TEST(Isai, ReturnsCorrectInverseSpdLongrow)
 }
 
 
+TYPED_TEST(Isai, ReturnsCorrectInverseSpdLongrowWithExcessSolver)
+{
+    using Csr = typename TestFixture::Csr;
+    using SpdIsai = typename TestFixture::SpdIsai;
+    using value_type = typename TestFixture::value_type;
+    const auto expected_transpose =
+        gko::as<Csr>(this->spd_csr_longrow_inv->transpose());
+    auto spd_isai_factory =
+        SpdIsai::build()
+            .with_excess_solver_factory(this->excess_solver_factory)
+            .on(this->exec);
+    const auto isai = spd_isai_factory->generate(this->spd_csr_longrow);
+
+    const auto composition = isai->get_approximate_inverse()->get_operators();
+    const auto lower_t = gko::as<Csr>(composition[0]);
+    const auto lower = gko::as<Csr>(composition[1]);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(lower, this->spd_csr_longrow_inv);
+    GKO_ASSERT_MTX_EQ_SPARSITY(lower_t, expected_transpose);
+    // need to drastically reduce precision due to using different excess solver
+    // factory.
+    GKO_ASSERT_MTX_NEAR(lower, this->spd_csr_longrow_inv,
+                        1e3 * r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(lower_t, expected_transpose,
+                        1e3 * r<value_type>::value);
+}
+
+
 TYPED_TEST(Isai, GeneratesWithUnsortedCsr)
 {
     using Csr = typename TestFixture::Csr;

From 55f2da2b017db0ddc0ca012e7c2c4533fde7ff57 Mon Sep 17 00:00:00 2001
From: fritzgoebel <fritz.goebel@kit.edu>
Date: Tue, 16 Mar 2021 09:27:31 +0100
Subject: [PATCH 0517/1008] Apply suggestions from code review

Co-authored-by: Yuhsiang M. Tsai <19565938+yhmtsai@users.noreply.github.com>
---
 benchmark/preconditioner/preconditioner.cpp   |   6 +-
 common/components/warp_blas.hpp.inc           |   2 +
 common/preconditioner/isai_kernels.hpp.inc    |  14 +-
 core/preconditioner/isai.cpp                  |  43 ++--
 core/preconditioner/isai_kernels.hpp          |   1 -
 core/test/preconditioner/isai.cpp             |  24 +--
 core/test/utils/matrix_generator.hpp          |  46 ++++
 cuda/test/preconditioner/isai_kernels.cpp     | 200 +++++++++++++++++-
 hip/test/preconditioner/isai_kernels.hip.cpp  | 200 +++++++++++++++++-
 include/ginkgo/core/preconditioner/isai.hpp   |   8 +-
 omp/preconditioner/isai_kernels.cpp           | 120 +++++------
 omp/test/preconditioner/isai_kernels.cpp      | 200 +++++++++++++++++-
 reference/preconditioner/isai_kernels.cpp     | 127 ++++++-----
 .../test/preconditioner/isai_kernels.cpp      |   2 -
 14 files changed, 813 insertions(+), 180 deletions(-)

diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index 32ab65c933b..f86011632d5 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -114,9 +114,13 @@ std::string encode_parameters(const char *precond_name)
          [] {
              return std::string{"ilu-isai-"} + std::to_string(FLAGS_isai_power);
          }},
-        {"general-isai", [] {
+        {"general-isai",
+         [] {
              return std::string{"general-isai-"} +
                     std::to_string(FLAGS_isai_power);
+         }},
+        {"spd-isai", [] {
+             return std::string{"spd-isai-"} + std::to_string(FLAGS_isai_power);
          }}};
     if (encoder.find(precond_name) == encoder.end()) {
         return precond_name;
diff --git a/common/components/warp_blas.hpp.inc b/common/components/warp_blas.hpp.inc
index d810a972ae4..6c7f608511b 100644
--- a/common/components/warp_blas.hpp.inc
+++ b/common/components/warp_blas.hpp.inc
@@ -92,6 +92,8 @@ __device__ __forceinline__ void apply_gauss_jordan_transform(
  * routine uses warp shuffles to exchange data between rows. The transform is
  * performed by using the `key_row`-th row and `key_col`-th column of the
  * matrix.
+ * Works with one right hand side vector `rhs` which can be directly worked on
+ * when solving Ax = rhs without the need of storing the inverse of A.
  */
 template <
     int max_problem_size, typename Group, typename ValueType,
diff --git a/common/preconditioner/isai_kernels.hpp.inc b/common/preconditioner/isai_kernels.hpp.inc
index 1ce99948cbe..d2ee667e7a4 100644
--- a/common/preconditioner/isai_kernels.hpp.inc
+++ b/common/preconditioner/isai_kernels.hpp.inc
@@ -276,7 +276,8 @@ __global__ __launch_bounds__(default_block_size) void generate_general_inverse(
         ValueType sol = subwarp.shfl(rhs, perm);
 
         if (spd) {
-            sol /= sqrt(subwarp.shfl(rhs, num_elems - 1));
+            auto diag = subwarp.shfl(sol, num_elems - 1);
+            sol /= sqrt(diag);
         }
 
         return sol;
@@ -371,16 +372,17 @@ __global__ __launch_bounds__(default_block_size) void scale_excess_solution(
     auto subwarp =
         group::tiled_partition<subwarp_size>(group::this_thread_block());
     const int local_id = subwarp.thread_rank();
+    const auto row = warp_id + e_start;
 
-    if (warp_id >= e_end - e_start) {
+    if (row >= e_end) {
         return;
     }
 
     const IndexType offset = excess_block_ptrs[e_start];
-    const IndexType block_begin = excess_block_ptrs[warp_id] - offset;
-    const IndexType block_end = excess_block_ptrs[warp_id + 1] - offset;
-    const ValueType scal =
-        one<ValueType>() / sqrt(excess_solution[block_end - 1]);
+    const IndexType block_begin = excess_block_ptrs[row] - offset;
+    const IndexType block_end = excess_block_ptrs[row + 1] - offset;
+    const auto diag = excess_solution[block_end - 1];
+    const ValueType scal = one<ValueType>() / sqrt(diag);
 
     for (size_type i = block_begin + local_id; i < block_end;
          i += subwarp_size) {
diff --git a/core/preconditioner/isai.cpp b/core/preconditioner/isai.cpp
index 7663f0b6bf2..7247b9a04e7 100644
--- a/core/preconditioner/isai.cpp
+++ b/core/preconditioner/isai.cpp
@@ -126,7 +126,7 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
     using LowerTrs = solver::LowerTrs<ValueType, IndexType>;
     using UpperTrs = solver::UpperTrs<ValueType, IndexType>;
     using Gmres = solver::Gmres<ValueType>;
-    using bj = preconditioner::Jacobi<ValueType, IndexType>;
+    using Bj = preconditioner::Jacobi<ValueType, IndexType>;
     GKO_ASSERT_IS_SQUARE_MATRIX(input);
     auto exec = this->get_executor();
     auto is_lower = IsaiType == isai_type::lower;
@@ -150,9 +150,9 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
         // Init arrays
         Array<IndexType> inverted_col_idxs{exec, inverted_nnz};
         Array<ValueType> inverted_vals{exec, inverted_nnz};
-        std::shared_ptr<Csr> inverted_base = Csr::create(
+        auto inverted_base = share(Csr::create(
             exec, dim<2>{num_rows, num_rows}, std::move(inverted_vals),
-            std::move(inverted_col_idxs), std::move(inverted_row_ptrs));
+            std::move(inverted_col_idxs), std::move(inverted_row_ptrs)));
 
         // Extract lower factor: columns and values
         exec->run(isai::make_initialize_l(to_invert.get(), inverted_base.get(),
@@ -180,32 +180,33 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
             excess_row_ptrs_full.get_data(), is_lower));
     }
 
-    auto total_excess_dim =
-        exec->copy_val_to_host(excess_block_ptrs.get_const_data() + num_rows);
+    auto host_excess_block_ptrs_array =
+        Array<IndexType>(exec->get_master(), excess_block_ptrs);
+    auto host_excess_block_ptrs = host_excess_block_ptrs_array.get_const_data();
+    auto host_excess_row_ptrs_full_array =
+        Array<IndexType>(exec->get_master(), excess_row_ptrs_full);
+    auto host_excess_row_ptrs_full =
+        host_excess_row_ptrs_full_array.get_const_data();
+    auto total_excess_dim = host_excess_block_ptrs[num_rows];
     auto excess_lim = excess_limit == 0 ? total_excess_dim : excess_limit;
     // if we had long rows:
     if (total_excess_dim > 0) {
         bool done = false;
         size_type block = 0;
-        while (true) {
+        while (block < num_rows) {
             // build the excess sparse triangular system
-            size_type excess_dim = 0;
+            size_type excess_dim;
             size_type excess_start = block;
-            const auto block_offset = exec->copy_val_to_host(
-                excess_block_ptrs.get_const_data() + block);
-            const auto nnz_offset = exec->copy_val_to_host(
-                excess_row_ptrs_full.get_const_data() + block);
-            while (excess_dim < excess_lim && block < num_rows) {
+            const auto block_offset = host_excess_block_ptrs[block];
+            const auto nnz_offset = host_excess_row_ptrs_full[block];
+            for (excess_dim = 0; excess_dim < excess_lim && block < num_rows;
+                 excess_dim = host_excess_block_ptrs[block] - block_offset) {
                 block++;
-                excess_dim = exec->copy_val_to_host(
-                                 excess_block_ptrs.get_const_data() + block) -
-                             block_offset;
             }
-            if (excess_dim == 0) break;
-            auto excess_nnz =
-                exec->copy_val_to_host(excess_row_ptrs_full.get_const_data() +
-                                       block) -
-                nnz_offset;
+            if (excess_dim == 0) {
+                break;
+            }
+            auto excess_nnz = host_excess_row_ptrs_full[block] - nnz_offset;
             auto excess_system =
                 Csr::create(exec, dim<2>(excess_dim, excess_dim), excess_nnz);
             excess_system->set_strategy(
@@ -231,7 +232,7 @@ void Isai<IsaiType, ValueType, IndexType>::generate_inverse(
                 excess_solver_factory =
                     Gmres::build()
                         .with_preconditioner(
-                            bj::build().with_max_block_size(32u).on(exec))
+                            Bj::build().with_max_block_size(32u).on(exec))
                         .with_criteria(
                             gko::stop::Iteration::build()
                                 .with_max_iters(excess_dim)
diff --git a/core/preconditioner/isai_kernels.hpp b/core/preconditioner/isai_kernels.hpp
index c9c71f6efe9..f85a74910da 100644
--- a/core/preconditioner/isai_kernels.hpp
+++ b/core/preconditioner/isai_kernels.hpp
@@ -51,7 +51,6 @@ namespace kernels {
                               IndexType *excess_rhs_ptrs,                     \
                               IndexType *excess_nz_ptrs, bool lower)
 
-
 #define GKO_DECLARE_ISAI_GENERATE_GENERAL_INVERSE_KERNEL(ValueType, IndexType) \
     void generate_general_inverse(                                             \
         std::shared_ptr<const DefaultExecutor> exec,                           \
diff --git a/core/test/preconditioner/isai.cpp b/core/test/preconditioner/isai.cpp
index 202dd33bde8..fc8f6cee1dd 100644
--- a/core/test/preconditioner/isai.cpp
+++ b/core/test/preconditioner/isai.cpp
@@ -181,27 +181,27 @@ TYPED_TEST(IsaiFactory, SetsExcessLimitCorrectly)
     using UpperIsai = typename TestFixture::UpperIsai;
 
     auto a_isai_factory =
-        GeneralIsai::build().with_excess_limit(1024).on(this->exec);
+        GeneralIsai::build().with_excess_limit(1024u).on(this->exec);
     auto spd_isai_factory =
-        SpdIsai::build().with_excess_limit(1024).on(this->exec);
+        SpdIsai::build().with_excess_limit(1024u).on(this->exec);
     auto l_isai_factory =
-        LowerIsai::build().with_excess_limit(1024).on(this->exec);
+        LowerIsai::build().with_excess_limit(1024u).on(this->exec);
     auto u_isai_factory =
-        UpperIsai::build().with_excess_limit(1024).on(this->exec);
+        UpperIsai::build().with_excess_limit(1024u).on(this->exec);
 
-    ASSERT_EQ(a_isai_factory->get_parameters().excess_limit, 1024);
-    ASSERT_EQ(spd_isai_factory->get_parameters().excess_limit, 1024);
-    ASSERT_EQ(l_isai_factory->get_parameters().excess_limit, 1024);
-    ASSERT_EQ(u_isai_factory->get_parameters().excess_limit, 1024);
+    ASSERT_EQ(a_isai_factory->get_parameters().excess_limit, 1024u);
+    ASSERT_EQ(spd_isai_factory->get_parameters().excess_limit, 1024u);
+    ASSERT_EQ(l_isai_factory->get_parameters().excess_limit, 1024u);
+    ASSERT_EQ(u_isai_factory->get_parameters().excess_limit, 1024u);
 }
 
 
 TYPED_TEST(IsaiFactory, SetsDefaultExcessLimitCorrectly)
 {
-    ASSERT_EQ(this->general_isai_factory->get_parameters().excess_limit, 0);
-    ASSERT_EQ(this->spd_isai_factory->get_parameters().excess_limit, 0);
-    ASSERT_EQ(this->lower_isai_factory->get_parameters().excess_limit, 0);
-    ASSERT_EQ(this->upper_isai_factory->get_parameters().excess_limit, 0);
+    ASSERT_EQ(this->general_isai_factory->get_parameters().excess_limit, 0u);
+    ASSERT_EQ(this->spd_isai_factory->get_parameters().excess_limit, 0u);
+    ASSERT_EQ(this->lower_isai_factory->get_parameters().excess_limit, 0u);
+    ASSERT_EQ(this->upper_isai_factory->get_parameters().excess_limit, 0u);
 }
 
 
diff --git a/core/test/utils/matrix_generator.hpp b/core/test/utils/matrix_generator.hpp
index 6d235a4f5eb..c3f9b2fb50d 100644
--- a/core/test/utils/matrix_generator.hpp
+++ b/core/test/utils/matrix_generator.hpp
@@ -340,6 +340,52 @@ std::unique_ptr<MatrixType> generate_random_upper_triangular_matrix(
 }
 
 
+/**
+ * Generates a random square band matrix.
+ *
+ * @tparam MatrixType  type of matrix to generate (matrix::Dense must implement
+ *                     the interface `ConvertibleTo<MatrixType>`)
+ * @tparam ValueDistribution  type of value distribution
+ * @tparam Engine  type of random engine
+ * @tparam MatrixArgs  the arguments from the matrix to be forwarded.
+ *
+ * @param size  number of rows and columns
+ * @param lower_bandwidth number of nonzeros in each row left of the main
+ * diagonal
+ * @param upper_bandwidth number of nonzeros in each row right of the main
+ * diagonal
+ * @param value_dist  distribution of matrix values
+ * @param engine  a random engine
+ * @param exec  executor where the matrix should be allocated
+ * @param args  additional arguments for the matrix constructor
+ */
+template <typename MatrixType = matrix::Dense<>, typename ValueDistribution,
+          typename Engine, typename... MatrixArgs>
+std::unique_ptr<MatrixType> generate_random_band_matrix(
+    size_type size, size_type lower_bandwidth, size_type upper_bandwidth,
+    ValueDistribution &&value_dist, Engine &&engine,
+    std::shared_ptr<const Executor> exec, MatrixArgs &&... args)
+{
+    using value_type = typename MatrixType::value_type;
+    using index_type = typename MatrixType::index_type;
+
+    matrix_data<value_type, index_type> data{gko::dim<2>{size, size}, {}};
+
+    for (size_type row = 0; row < size; ++row) {
+        for (size_type col = row < lower_bandwidth ? 0 : row - lower_bandwidth;
+             col <= std::min(row + upper_bandwidth, size - 1); col++) {
+            auto val = detail::get_rand_value<value_type>(value_dist, engine);
+            data.nonzeros.emplace_back(row, col, val);
+        }
+    }
+
+    // convert to the correct matrix type
+    auto result = MatrixType::create(exec, std::forward<MatrixArgs>(args)...);
+    result->read(data);
+    return result;
+}
+
+
 }  // namespace test
 }  // namespace gko
 
diff --git a/cuda/test/preconditioner/isai_kernels.cpp b/cuda/test/preconditioner/isai_kernels.cpp
index ca9f4f3f1d9..a8600039d7e 100644
--- a/cuda/test/preconditioner/isai_kernels.cpp
+++ b/cuda/test/preconditioner/isai_kernels.cpp
@@ -56,7 +56,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
-enum struct matrix_type { lower, upper, general };
+enum struct matrix_type { lower, upper, general, spd };
 class Isai : public ::testing::Test {
 protected:
     using value_type = double;
@@ -99,6 +99,14 @@ class Isai : public ::testing::Test {
                 n, n, nz_dist, val_dist, rand_engine, ref, gko::dim<2>{n, n});
             ensure_diagonal(dense_mtx.get());
             mtx->copy_from(dense_mtx.get());
+        } else if (type == matrix_type::spd) {
+            auto dense_mtx = gko::test::generate_random_band_matrix<Dense>(
+                n, row_limit / 4, row_limit / 4, val_dist, rand_engine, ref,
+                gko::dim<2>{n, n});
+            auto transp = gko::as<Dense>(dense_mtx->transpose());
+            auto spd_mtx = Dense::create(ref, gko::dim<2>{n, n});
+            dense_mtx->apply(transp.get(), spd_mtx.get());
+            mtx->copy_from(spd_mtx.get());
         } else {
             mtx = gko::test::generate_random_triangular_matrix<Csr>(
                 n, n, true, for_lower_tm, nz_dist, val_dist, rand_engine, ref,
@@ -218,6 +226,29 @@ TEST_F(Isai, CudaIsaiGenerateAinverseShortIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, CudaIsaiGenerateSpdinverseShortIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 15);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::Array<index_type> da1(cuda, num_rows + 1);
+    auto da2 = da1;
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::kernels::cuda::isai::generate_general_inverse(
+        cuda, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
+        true);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 10 * r<value_type>::value);
+    GKO_ASSERT_ARRAY_EQ(a1, da1);
+    GKO_ASSERT_ARRAY_EQ(a2, da2);
+    ASSERT_EQ(a1.get_const_data()[num_rows], 0);
+}
+
+
 TEST_F(Isai, CudaIsaiGenerateLinverseLongIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 554, 64);
@@ -287,6 +318,29 @@ TEST_F(Isai, CudaIsaiGenerateAinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, CudaIsaiGenerateSpdinverseLongIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::Array<index_type> da1(cuda, num_rows + 1);
+    auto da2 = da1;
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
+    gko::kernels::cuda::isai::generate_general_inverse(
+        cuda, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
+        false);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 10 * r<value_type>::value);
+    GKO_ASSERT_ARRAY_EQ(a1, da1);
+    GKO_ASSERT_ARRAY_EQ(a2, da2);
+    ASSERT_GT(a1.get_const_data()[num_rows], 0);
+}
+
+
 TEST_F(Isai, CudaIsaiGenerateExcessLinverseLongIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 518, 40);
@@ -380,13 +434,75 @@ TEST_F(Isai, CudaIsaiGenerateExcessAinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, CudaIsaiGenerateExcessSpdinverseLongIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::Array<index_type> da1(cuda, a1);
+    gko::Array<index_type> da2(cuda, a2);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_nnz = a2.get_data()[num_rows];
+    auto excess = Csr::create(ref, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    auto dexcess = Csr::create(cuda, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto de_rhs = Dense::create(cuda, gko::dim<2>(e_dim, 1));
+
+    gko::kernels::reference::isai::generate_excess_system(
+        ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
+        excess.get(), e_rhs.get(), 0, num_rows);
+    gko::kernels::cuda::isai::generate_excess_system(
+        cuda, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 0, num_rows);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
+    GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
+TEST_F(Isai, CudaIsaiGeneratePartialExcessIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
+    gko::Array<index_type> da1(cuda, a1);
+    gko::Array<index_type> da2(cuda, a2);
+    auto e_dim = a1.get_data()[10] - a1.get_data()[5];
+    auto e_nnz = a2.get_data()[10] - a2.get_data()[5];
+    auto excess = Csr::create(ref, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    auto dexcess = Csr::create(cuda, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto de_rhs = Dense::create(cuda, gko::dim<2>(e_dim, 1));
+
+    gko::kernels::reference::isai::generate_excess_system(
+        ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
+        excess.get(), e_rhs.get(), 5u, 10u);
+    gko::kernels::cuda::isai::generate_excess_system(
+        cuda, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 5u, 10u);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
+    GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
 TEST_F(Isai, CudaIsaiScaleExcessSolutionIsEquivalentToRef)
 {
-    initialize_data(matrix_type::lower, 572, 52);
+    initialize_data(matrix_type::spd, 100, 64);
     const auto num_rows = mtx->get_size()[0];
     gko::Array<index_type> a1(ref, num_rows + 1);
     auto a2 = a1;
-    gko::kernels::reference::isai::generate_tri_inverse(
+    gko::kernels::reference::isai::generate_general_inverse(
         ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
     gko::Array<index_type> da1(cuda, a1);
     auto e_dim = a1.get_data()[num_rows];
@@ -405,6 +521,30 @@ TEST_F(Isai, CudaIsaiScaleExcessSolutionIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, CudaIsaiScalePartialExcessSolutionIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::Array<index_type> da1(cuda, a1);
+    auto e_dim = a1.get_data()[10] - a1.get_data()[5];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(cuda);
+    de_rhs->copy_from(lend(e_rhs));
+
+    gko::kernels::reference::isai::scale_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), 5u, 10u);
+    gko::kernels::cuda::isai::scale_excess_solution(cuda, da1.get_const_data(),
+                                                    de_rhs.get(), 5u, 10u);
+
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+}
+
+
 TEST_F(Isai, CudaIsaiScatterExcessSolutionLIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 572, 52);
@@ -485,4 +625,58 @@ TEST_F(Isai, CudaIsaiScatterExcessSolutionAIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, CudaIsaiScatterExcessSolutionSpdIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
+    gko::Array<index_type> da1(cuda, a1);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(cuda);
+    de_rhs->copy_from(lend(e_rhs));
+    // overwrite -1 values with inverse
+    d_inverse->copy_from(lend(inverse));
+
+    gko::kernels::reference::isai::scatter_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 0, num_rows);
+    gko::kernels::cuda::isai::scatter_excess_solution(
+        cuda, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 0, num_rows);
+
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
+TEST_F(Isai, CudaIsaiScatterPartialExcessSolutionIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::Array<index_type> da1(cuda, a1);
+    auto e_dim = a1.get_data()[10] - a1.get_data()[5];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(cuda);
+    de_rhs->copy_from(lend(e_rhs));
+    // overwrite -1 values with inverse
+    d_inverse->copy_from(lend(inverse));
+
+    gko::kernels::reference::isai::scatter_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 5u, 10u);
+    gko::kernels::cuda::isai::scatter_excess_solution(
+        cuda, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 5u, 10u);
+
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
 }  // namespace
diff --git a/hip/test/preconditioner/isai_kernels.hip.cpp b/hip/test/preconditioner/isai_kernels.hip.cpp
index adf5ad87789..e4052cc022c 100644
--- a/hip/test/preconditioner/isai_kernels.hip.cpp
+++ b/hip/test/preconditioner/isai_kernels.hip.cpp
@@ -53,7 +53,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
-enum struct matrix_type { lower, upper, general };
+enum struct matrix_type { lower, upper, general, spd };
 class Isai : public ::testing::Test {
 protected:
     using value_type = double;
@@ -96,6 +96,14 @@ class Isai : public ::testing::Test {
                 n, n, nz_dist, val_dist, rand_engine, ref, gko::dim<2>{n, n});
             ensure_diagonal(dense_mtx.get());
             mtx->copy_from(dense_mtx.get());
+        } else if (type == matrix_type::spd) {
+            auto dense_mtx = gko::test::generate_random_band_matrix<Dense>(
+                n, row_limit / 4, row_limit / 4, val_dist, rand_engine, ref,
+                gko::dim<2>{n, n});
+            auto transp = gko::as<Dense>(dense_mtx->transpose());
+            auto spd_mtx = Dense::create(ref, gko::dim<2>{n, n});
+            dense_mtx->apply(transp.get(), spd_mtx.get());
+            mtx->copy_from(spd_mtx.get());
         } else {
             mtx = gko::test::generate_random_triangular_matrix<Csr>(
                 n, n, true, for_lower_tm, nz_dist, val_dist, rand_engine, ref,
@@ -199,6 +207,29 @@ TEST_F(Isai, HipIsaiGenerateAinverseShortIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, HipIsaiGenerateSpdinverseShortIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 15);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::Array<index_type> da1(hip, num_rows + 1);
+    auto da2 = da1;
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::kernels::hip::isai::generate_general_inverse(
+        hip, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
+        true);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 10 * r<value_type>::value);
+    GKO_ASSERT_ARRAY_EQ(a1, da1);
+    GKO_ASSERT_ARRAY_EQ(a2, da2);
+    ASSERT_EQ(a1.get_const_data()[num_rows], 0);
+}
+
+
 TEST_F(Isai, HipIsaiGenerateLinverseLongIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 554, 64);
@@ -268,6 +299,29 @@ TEST_F(Isai, HipIsaiGenerateAinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, HipIsaiGenerateSpdinverseLongIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::Array<index_type> da1(hip, num_rows + 1);
+    auto da2 = da1;
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::kernels::hip::isai::generate_general_inverse(
+        hip, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
+        true);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 10 * r<value_type>::value);
+    GKO_ASSERT_ARRAY_EQ(a1, da1);
+    GKO_ASSERT_ARRAY_EQ(a2, da2);
+    ASSERT_GT(a1.get_const_data()[num_rows], 0);
+}
+
+
 TEST_F(Isai, HipIsaiGenerateExcessLinverseLongIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 518, 40);
@@ -361,13 +415,75 @@ TEST_F(Isai, HipIsaiGenerateExcessAinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, HipIsaiGenerateExcessSpdinverseLongIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::Array<index_type> da1(hip, a1);
+    gko::Array<index_type> da2(hip, a2);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_nnz = a2.get_data()[num_rows];
+    auto excess = Csr::create(ref, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    auto dexcess = Csr::create(hip, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto de_rhs = Dense::create(hip, gko::dim<2>(e_dim, 1));
+
+    gko::kernels::reference::isai::generate_excess_system(
+        ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
+        excess.get(), e_rhs.get(), 0, num_rows);
+    gko::kernels::hip::isai::generate_excess_system(
+        hip, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 0, num_rows);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
+    GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
+TEST_F(Isai, HipIsaiGeneratePartialExcessIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
+    gko::Array<index_type> da1(hip, a1);
+    gko::Array<index_type> da2(hip, a2);
+    auto e_dim = a1.get_data()[10] - a1.get_data()[5];
+    auto e_nnz = a2.get_data()[10] - a2.get_data()[5];
+    auto excess = Csr::create(ref, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    auto dexcess = Csr::create(hip, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto de_rhs = Dense::create(hip, gko::dim<2>(e_dim, 1));
+
+    gko::kernels::reference::isai::generate_excess_system(
+        ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
+        excess.get(), e_rhs.get(), 5u, 10u);
+    gko::kernels::hip::isai::generate_excess_system(
+        hip, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 5u, 10u);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
+    GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
 TEST_F(Isai, HipIsaiScaleExcessSolutionIsEquivalentToRef)
 {
-    initialize_data(matrix_type::lower, 572, 52);
+    initialize_data(matrix_type::spd, 100, 64);
     const auto num_rows = mtx->get_size()[0];
     gko::Array<index_type> a1(ref, num_rows + 1);
     auto a2 = a1;
-    gko::kernels::reference::isai::generate_tri_inverse(
+    gko::kernels::reference::isai::generate_general_inverse(
         ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
     gko::Array<index_type> da1(hip, a1);
     auto e_dim = a1.get_data()[num_rows];
@@ -386,6 +502,30 @@ TEST_F(Isai, HipIsaiScaleExcessSolutionIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, HipIsaiScalePartialExcessSolutionIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::Array<index_type> da1(hip, a1);
+    auto e_dim = a1.get_data()[10] - a1.get_data()[5];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(hip);
+    de_rhs->copy_from(lend(e_rhs));
+
+    gko::kernels::reference::isai::scale_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), 5u, 10u);
+    gko::kernels::hip::isai::scale_excess_solution(hip, da1.get_const_data(),
+                                                   de_rhs.get(), 5u, 10u);
+
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+}
+
+
 TEST_F(Isai, HipIsaiScatterExcessSolutionLIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 572, 52);
@@ -466,4 +606,58 @@ TEST_F(Isai, HipIsaiScatterExcessSolutionAIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, HipIsaiScatterExcessSolutionSpdIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::Array<index_type> da1(hip, a1);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(hip);
+    de_rhs->copy_from(lend(e_rhs));
+    // overwrite -1 values with inverse
+    d_inverse->copy_from(lend(inverse));
+
+    gko::kernels::reference::isai::scatter_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 0, num_rows);
+    gko::kernels::hip::isai::scatter_excess_solution(
+        hip, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 0, num_rows);
+
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
+TEST_F(Isai, HipIsaiScatterPartialExcessSolutionIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::Array<index_type> da1(hip, a1);
+    auto e_dim = a1.get_data()[10] - a1.get_data()[5];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(hip);
+    de_rhs->copy_from(lend(e_rhs));
+    // overwrite -1 values with inverse
+    d_inverse->copy_from(lend(inverse));
+
+    gko::kernels::reference::isai::scatter_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 5u, 10u);
+    gko::kernels::hip::isai::scatter_excess_solution(
+        hip, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 5u, 10u);
+
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
 }  // namespace
diff --git a/include/ginkgo/core/preconditioner/isai.hpp b/include/ginkgo/core/preconditioner/isai.hpp
index ba86bf9996a..4266c203bf4 100644
--- a/include/ginkgo/core/preconditioner/isai.hpp
+++ b/include/ginkgo/core/preconditioner/isai.hpp
@@ -58,7 +58,7 @@ namespace preconditioner {
  * This enum lists the types of the ISAI preconditioner.
  *
  * ISAI can either be generated for a general square matrix, a lower triangular
- * matrix, an upper triangular matrix or a spd matrix.
+ * matrix, an upper triangular matrix or an spd matrix.
  */
 enum struct isai_type { lower, upper, general, spd };
 
@@ -93,8 +93,8 @@ enum struct isai_type { lower, upper, general, spd };
  *       elements will be ignored.
  *
  * @tparam IsaiType  determines if the ISAI is generated for a general square
- * matrix, a lower triangular matrix, an upper triangular matrix or an spd
- * matrix
+ *         matrix, a lower triangular matrix, an upper triangular matrix or an
+ *         spd matrix
  * @tparam ValueType  precision of matrix elements
  * @tparam IndexType  precision of matrix indexes
  *
@@ -176,7 +176,7 @@ class Isai : public EnableLinOp<Isai<IsaiType, ValueType, IndexType>>,
          * solution has been computed.
          * Must be at least 0, default value 0.
          */
-        index_type GKO_FACTORY_PARAMETER_SCALAR(excess_limit, 0);
+        size_type GKO_FACTORY_PARAMETER_SCALAR(excess_limit, 0u);
 
         /**
          * Factory for the Excess System solver
diff --git a/omp/preconditioner/isai_kernels.cpp b/omp/preconditioner/isai_kernels.cpp
index a33654bd33e..7cac20afab1 100644
--- a/omp/preconditioner/isai_kernels.cpp
+++ b/omp/preconditioner/isai_kernels.cpp
@@ -85,7 +85,7 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                       const matrix::Csr<ValueType, IndexType> *mtx,
                       matrix::Csr<ValueType, IndexType> *inverse_mtx,
                       IndexType *excess_rhs_ptrs, IndexType *excess_nz_ptrs,
-                      Callable direct_solve)
+                      Callable direct_solve, bool tri)
 {
     /*
     Consider: aiM := inverse_mtx; M := mtx
@@ -154,7 +154,13 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                             if (m_cols[m_idx + m_begin] < row && col == row) {
                                 rhs_one_idx++;
                             }
-                            dense_system(i, i_idx) = m_vals[m_idx + m_begin];
+                            if (tri) {
+                                dense_system(i, i_idx) =
+                                    m_vals[m_idx + m_begin];
+                            } else {
+                                dense_system(i_idx, i) =
+                                    m_vals[m_idx + m_begin];
+                            }
                         });
                 }
 
@@ -241,7 +247,7 @@ void generate_tri_inverse(std::shared_ptr<const DefaultExecutor> exec,
         };
 
     generic_generate(exec, mtx, inverse_mtx, excess_rhs_ptrs, excess_nz_ptrs,
-                     trs_solve);
+                     trs_solve, true);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -281,67 +287,61 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
                               IndexType *excess_nz_ptrs, bool spd)
 {
     using std::swap;
-    auto general_solve =
-        [spd](const range<accessor::row_major<ValueType, 2>> system,
-              ValueType *rhs, const IndexType rhs_one_idx) {
-            const IndexType size = system.length(0);
-            if (size <= 0) {
-                return;
-            }
-            // RHS is the identity: zero everywhere except for the diagonal
-            // entry
-            std::fill_n(rhs, size, zero<ValueType>());
-            rhs[rhs_one_idx] = one<ValueType>();
-
-            // fill in transposed system
-            ValueType *transposed_system = new ValueType[size * size];
-            for (auto row = 0; row < size; row++) {
-                for (auto col = 0; col < size; col++) {
-                    transposed_system[row * size + col] = system(col, row);
-                }
+    auto general_solve = [spd](const range<accessor::row_major<ValueType, 2>>
+                                   transposed_system_range,
+                               ValueType *rhs, const IndexType rhs_one_idx) {
+        const IndexType size = transposed_system_range.length(0);
+        if (size <= 0) {
+            return;
+        }
+        // RHS is the identity: zero everywhere except for the diagonal
+        // entry
+        std::fill_n(rhs, size, zero<ValueType>());
+        rhs[rhs_one_idx] = one<ValueType>();
+
+        auto transposed_system = transposed_system_range.get_accessor().data;
+
+        // solve transposed system
+        for (IndexType col = 0; col < size; col++) {
+            const auto row =
+                choose_pivot(size - col, transposed_system + col * (size + 1),
+                             size) +
+                col;
+            swap_rows(col, row, size, transposed_system, size);
+            swap(rhs[row], rhs[col]);
+
+            const auto d = transposed_system[col * size + col];
+
+            for (IndexType i = 0; i < size; ++i) {
+                transposed_system[i * size + col] /= -d;
             }
 
-            // solve transposed system
-            for (IndexType col = 0; col < size; col++) {
-                const auto row =
-                    choose_pivot(size - col,
-                                 transposed_system + col * (size + 1), size) +
-                    col;
-                swap_rows(col, row, size, transposed_system, size);
-                swap(rhs[row], rhs[col]);
-
-                const auto d = transposed_system[col * size + col];
-
-                for (IndexType i = 0; i < size; ++i) {
-                    transposed_system[i * size + col] /= -d;
-                }
-
-                transposed_system[col * size + col] = zero<ValueType>();
-                for (IndexType i = 0; i < size; ++i) {
-                    for (IndexType j = 0; j < size; ++j) {
-                        transposed_system[i * size + j] +=
-                            transposed_system[i * size + col] *
-                            transposed_system[col * size + j];
-                    }
-                    rhs[i] += rhs[col] * transposed_system[i * size + col];
-                }
+            transposed_system[col * size + col] = zero<ValueType>();
+            for (IndexType i = 0; i < size; ++i) {
                 for (IndexType j = 0; j < size; ++j) {
-                    transposed_system[col * size + j] /= d;
+                    transposed_system[i * size + j] +=
+                        transposed_system[i * size + col] *
+                        transposed_system[col * size + j];
                 }
-                rhs[col] /= d;
-                transposed_system[col * size + col] = one<ValueType>() / d;
+                rhs[i] += rhs[col] * transposed_system[i * size + col];
             }
+            for (IndexType j = 0; j < size; ++j) {
+                transposed_system[col * size + j] /= d;
+            }
+            rhs[col] /= d;
+            transposed_system[col * size + col] = one<ValueType>() / d;
+        }
 
-            if (spd) {
-                const auto scal = one<ValueType>() / sqrt(rhs[size - 1]);
-                for (IndexType row = 0; row < size; row++) {
-                    rhs[row] *= scal;
-                }
+        if (spd) {
+            const auto scal = one<ValueType>() / sqrt(rhs[size - 1]);
+            for (IndexType row = 0; row < size; row++) {
+                rhs[row] *= scal;
             }
-        };
+        }
+    };
 
     generic_generate(exec, mtx, inverse_mtx, excess_rhs_ptrs, excess_nz_ptrs,
-                     general_solve);
+                     general_solve, false);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -375,9 +375,9 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
         const auto i_begin = i_row_ptrs[row];
         const auto i_size = i_row_ptrs[row + 1] - i_begin;
         // first row index of the sparse block in the excess system
-        auto e_begin = excess_rhs_ptrs[row];
+        auto e_begin = excess_rhs_ptrs[row] - excess_rhs_ptrs[e_start];
         // first non-zero index in the sparse block
-        auto e_nz = excess_nz_ptrs[row];
+        auto e_nz = excess_nz_ptrs[row] - excess_nz_ptrs[e_start];
 
         if (i_size > row_size_limit) {
             // count non-zeros and dimension in the excess system
@@ -405,7 +405,7 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
             }
         }
     }
-    e_row_ptrs[e_dim] = excess_nz_ptrs[num_rows];
+    e_row_ptrs[e_dim] = excess_nz_ptrs[e_end] - excess_nz_ptrs[e_start];
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -424,8 +424,8 @@ void scale_excess_solution(std::shared_ptr<const DefaultExecutor>,
     for (size_type row = e_start; row < e_end; ++row) {
         IndexType block_start = excess_block_ptrs[row] - offset;
         IndexType block_end = excess_block_ptrs[row + 1] - offset;
-        const ValueType scal =
-            one<ValueType>() / sqrt(excess_values[block_end - 1]);
+        const ValueType diag = excess_values[block_end - 1];
+        const ValueType scal = one<ValueType>() / sqrt(diag);
         for (size_type i = block_start; i < block_end; i++) {
             excess_values[i] *= scal;
         }
diff --git a/omp/test/preconditioner/isai_kernels.cpp b/omp/test/preconditioner/isai_kernels.cpp
index c42a1f08096..58c52d9d74a 100644
--- a/omp/test/preconditioner/isai_kernels.cpp
+++ b/omp/test/preconditioner/isai_kernels.cpp
@@ -52,7 +52,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
-enum struct matrix_type { lower, upper, general };
+enum struct matrix_type { lower, upper, general, spd };
 class Isai : public ::testing::Test {
 protected:
     using value_type = double;
@@ -94,6 +94,14 @@ class Isai : public ::testing::Test {
                 n, n, nz_dist, val_dist, rand_engine, ref, gko::dim<2>{n, n});
             ensure_diagonal(dense_mtx.get());
             mtx->copy_from(dense_mtx.get());
+        } else if (type == matrix_type::spd) {
+            auto dense_mtx = gko::test::generate_random_band_matrix<Dense>(
+                n, row_limit / 4, row_limit / 4, val_dist, rand_engine, ref,
+                gko::dim<2>{n, n});
+            auto transp = gko::as<Dense>(dense_mtx->transpose());
+            auto spd_mtx = Dense::create(ref, gko::dim<2>{n, n});
+            dense_mtx->apply(transp.get(), spd_mtx.get());
+            mtx->copy_from(spd_mtx.get());
         } else {
             mtx = gko::test::generate_random_triangular_matrix<Csr>(
                 n, n, true, for_lower_tm, nz_dist, val_dist, rand_engine, ref,
@@ -197,6 +205,29 @@ TEST_F(Isai, OmpIsaiGenerateAinverseShortIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, OmpIsaiGenerateSpdinverseShortIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 15);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::Array<index_type> da1(omp, num_rows + 1);
+    auto da2 = da1;
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::kernels::omp::isai::generate_general_inverse(
+        omp, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
+        true);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, r<value_type>::value);
+    GKO_ASSERT_ARRAY_EQ(a1, da1);
+    GKO_ASSERT_ARRAY_EQ(a2, da2);
+    ASSERT_EQ(a1.get_const_data()[num_rows], 0);
+}
+
+
 TEST_F(Isai, OmpIsaiGenerateLinverseLongIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 554, 64);
@@ -266,6 +297,29 @@ TEST_F(Isai, OmpIsaiGenerateAinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, OmpIsaiGenerateSpdinverseLongIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::Array<index_type> da1(omp, num_rows + 1);
+    auto da2 = da1;
+
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::kernels::omp::isai::generate_general_inverse(
+        omp, d_mtx.get(), d_inverse.get(), da1.get_data(), da2.get_data(),
+        true);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, r<value_type>::value);
+    GKO_ASSERT_ARRAY_EQ(a1, da1);
+    GKO_ASSERT_ARRAY_EQ(a2, da2);
+    ASSERT_GT(a1.get_const_data()[num_rows], 0);
+}
+
+
 TEST_F(Isai, OmpIsaiGenerateExcessLinverseLongIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 518, 40);
@@ -359,13 +413,75 @@ TEST_F(Isai, OmpIsaiGenerateExcessAinverseLongIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, OmpIsaiGenerateExcessSpdinverseLongIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::Array<index_type> da1(omp, a1);
+    gko::Array<index_type> da2(omp, a2);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_nnz = a2.get_data()[num_rows];
+    auto excess = Csr::create(ref, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    auto dexcess = Csr::create(omp, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto de_rhs = Dense::create(omp, gko::dim<2>(e_dim, 1));
+
+    gko::kernels::reference::isai::generate_excess_system(
+        ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
+        excess.get(), e_rhs.get(), 0, num_rows);
+    gko::kernels::omp::isai::generate_excess_system(
+        omp, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 0, num_rows);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
+    GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
+TEST_F(Isai, OmpIsaiGeneratePartialExcessIsEquivalentToRef)
+{
+    initialize_data(matrix_type::general, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), false);
+    gko::Array<index_type> da1(omp, a1);
+    gko::Array<index_type> da2(omp, a2);
+    auto e_dim = a1.get_data()[10] - a1.get_data()[5];
+    auto e_nnz = a2.get_data()[10] - a2.get_data()[5];
+    auto excess = Csr::create(ref, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    auto dexcess = Csr::create(omp, gko::dim<2>(e_dim, e_dim), e_nnz);
+    auto de_rhs = Dense::create(omp, gko::dim<2>(e_dim, 1));
+
+    gko::kernels::reference::isai::generate_excess_system(
+        ref, mtx.get(), inverse.get(), a1.get_const_data(), a2.get_const_data(),
+        excess.get(), e_rhs.get(), 5u, 10u);
+    gko::kernels::omp::isai::generate_excess_system(
+        omp, d_mtx.get(), d_inverse.get(), da1.get_const_data(),
+        da2.get_const_data(), dexcess.get(), de_rhs.get(), 5u, 10u);
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(excess, dexcess);
+    GKO_ASSERT_MTX_NEAR(excess, dexcess, 0);
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
 TEST_F(Isai, OmpIsaiScaleExcessSolutionIsEquivalentToRef)
 {
-    initialize_data(matrix_type::lower, 572, 52);
+    initialize_data(matrix_type::spd, 100, 64);
     const auto num_rows = mtx->get_size()[0];
     gko::Array<index_type> a1(ref, num_rows + 1);
     auto a2 = a1;
-    gko::kernels::reference::isai::generate_tri_inverse(
+    gko::kernels::reference::isai::generate_general_inverse(
         ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
     gko::Array<index_type> da1(omp, a1);
     auto e_dim = a1.get_data()[num_rows];
@@ -384,6 +500,30 @@ TEST_F(Isai, OmpIsaiScaleExcessSolutionIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, OmpIsaiScalePartialExcessSolutionIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::Array<index_type> da1(omp, a1);
+    auto e_dim = a1.get_data()[10] - a1.get_data()[5];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(omp);
+    de_rhs->copy_from(lend(e_rhs));
+
+    gko::kernels::reference::isai::scale_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), 5u, 10u);
+    gko::kernels::omp::isai::scale_excess_solution(omp, da1.get_const_data(),
+                                                   de_rhs.get(), 5u, 10u);
+
+    GKO_ASSERT_MTX_NEAR(e_rhs, de_rhs, 0);
+}
+
+
 TEST_F(Isai, OmpIsaiScatterExcessSolutionLIsEquivalentToRef)
 {
     initialize_data(matrix_type::lower, 572, 52);
@@ -464,4 +604,58 @@ TEST_F(Isai, OmpIsaiScatterExcessSolutionAIsEquivalentToRef)
 }
 
 
+TEST_F(Isai, OmpIsaiScatterExcessSolutionSpdIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::Array<index_type> da1(omp, a1);
+    auto e_dim = a1.get_data()[num_rows];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(omp);
+    de_rhs->copy_from(lend(e_rhs));
+    // overwrite -1 values with inverse
+    d_inverse->copy_from(lend(inverse));
+
+    gko::kernels::reference::isai::scatter_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 0, num_rows);
+    gko::kernels::omp::isai::scatter_excess_solution(
+        omp, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 0, num_rows);
+
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
+TEST_F(Isai, OmpIsaiScatterPartialExcessSolutionIsEquivalentToRef)
+{
+    initialize_data(matrix_type::spd, 100, 64);
+    const auto num_rows = mtx->get_size()[0];
+    gko::Array<index_type> a1(ref, num_rows + 1);
+    auto a2 = a1;
+    gko::kernels::reference::isai::generate_general_inverse(
+        ref, mtx.get(), inverse.get(), a1.get_data(), a2.get_data(), true);
+    gko::Array<index_type> da1(omp, a1);
+    auto e_dim = a1.get_data()[10] - a1.get_data()[5];
+    auto e_rhs = Dense::create(ref, gko::dim<2>(e_dim, 1));
+    std::fill_n(e_rhs->get_values(), e_dim, 123456);
+    auto de_rhs = Dense::create(omp);
+    de_rhs->copy_from(lend(e_rhs));
+    // overwrite -1 values with inverse
+    d_inverse->copy_from(lend(inverse));
+
+    gko::kernels::reference::isai::scatter_excess_solution(
+        ref, a1.get_const_data(), e_rhs.get(), inverse.get(), 5u, 10u);
+    gko::kernels::omp::isai::scatter_excess_solution(
+        omp, da1.get_const_data(), de_rhs.get(), d_inverse.get(), 5u, 10u);
+
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 0);
+    ASSERT_GT(e_dim, 0);
+}
+
+
 }  // namespace
diff --git a/reference/preconditioner/isai_kernels.cpp b/reference/preconditioner/isai_kernels.cpp
index 8e3d0fe9327..5e0dccbea09 100644
--- a/reference/preconditioner/isai_kernels.cpp
+++ b/reference/preconditioner/isai_kernels.cpp
@@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
-#include <iostream>
 #include <memory>
 
 
@@ -82,7 +81,7 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                       const matrix::Csr<ValueType, IndexType> *mtx,
                       matrix::Csr<ValueType, IndexType> *inverse_mtx,
                       IndexType *excess_rhs_ptrs, IndexType *excess_nz_ptrs,
-                      Callable direct_solve)
+                      Callable direct_solve, bool tri)
 {
     /*
     Consider: aiM := inverse_mtx; M := mtx
@@ -146,7 +145,11 @@ void generic_generate(std::shared_ptr<const DefaultExecutor> exec,
                         if (m_cols[m_idx + m_begin] < row && col == row) {
                             rhs_one_idx++;
                         }
-                        dense_system(i, i_idx) = m_vals[m_idx + m_begin];
+                        if (tri) {
+                            dense_system(i, i_idx) = m_vals[m_idx + m_begin];
+                        } else {
+                            dense_system(i_idx, i) = m_vals[m_idx + m_begin];
+                        }
                     });
             }
 
@@ -229,7 +232,7 @@ void generate_tri_inverse(std::shared_ptr<const DefaultExecutor> exec,
         };
 
     generic_generate(exec, mtx, inverse_mtx, excess_rhs_ptrs, excess_nz_ptrs,
-                     trs_solve);
+                     trs_solve, true);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -269,68 +272,62 @@ void generate_general_inverse(std::shared_ptr<const DefaultExecutor> exec,
                               IndexType *excess_nz_ptrs, bool spd)
 {
     using std::swap;
-    auto general_solve =
-        [spd](const range<accessor::row_major<ValueType, 2>> system,
-              ValueType *rhs, const IndexType rhs_one_idx) {
-            const IndexType size = system.length(0);
-            if (size <= 0) {
-                return;
-            }
-            // RHS is the identity: zero everywhere except for the diagonal
-            // entry
-            std::fill_n(rhs, size, zero<ValueType>());
-            rhs[rhs_one_idx] = one<ValueType>();
-
-            // fill in transposed system
-            ValueType *transposed_system = new ValueType[size * size];
-            for (auto row = 0; row < size; row++) {
-                for (auto col = 0; col < size; col++) {
-                    transposed_system[row * size + col] = system(col, row);
-                }
+    auto general_solve = [spd](const range<accessor::row_major<ValueType, 2>>
+                                   transposed_system_range,
+                               ValueType *rhs, const IndexType rhs_one_idx) {
+        const IndexType size = transposed_system_range.length(0);
+        if (size <= 0) {
+            return;
+        }
+        // RHS is the identity: zero everywhere except for the diagonal
+        // entry
+        std::fill_n(rhs, size, zero<ValueType>());
+        rhs[rhs_one_idx] = one<ValueType>();
+
+        auto transposed_system = transposed_system_range.get_accessor().data;
+
+        // solve transposed system
+        for (IndexType col = 0; col < size; col++) {
+            const auto row =
+                choose_pivot(size - col, transposed_system + col * (size + 1),
+                             size) +
+                col;
+            swap_rows(col, row, size, transposed_system, size);
+            swap(rhs[row], rhs[col]);
+
+            const auto d = transposed_system[col * size + col];
+
+            for (IndexType i = 0; i < size; ++i) {
+                transposed_system[i * size + col] /= -d;
             }
 
-            // solve transposed system
-            for (IndexType col = 0; col < size; col++) {
-                const auto row =
-                    choose_pivot(size - col,
-                                 transposed_system + col * (size + 1), size) +
-                    col;
-                swap_rows(col, row, size, transposed_system, size);
-                swap(rhs[row], rhs[col]);
-
-                const auto d = transposed_system[col * size + col];
-
-                for (IndexType i = 0; i < size; ++i) {
-                    transposed_system[i * size + col] /= -d;
-                }
-
-                transposed_system[col * size + col] = zero<ValueType>();
-                const auto rhs_key_val = rhs[col];
-                for (IndexType i = 0; i < size; ++i) {
-                    const auto scal = transposed_system[i * size + col];
-                    for (IndexType j = 0; j < size; ++j) {
-                        transposed_system[i * size + j] +=
-                            scal * transposed_system[col * size + j];
-                    }
-                    rhs[i] += rhs_key_val * scal;
-                }
+            transposed_system[col * size + col] = zero<ValueType>();
+            const auto rhs_key_val = rhs[col];
+            for (IndexType i = 0; i < size; ++i) {
+                const auto scal = transposed_system[i * size + col];
                 for (IndexType j = 0; j < size; ++j) {
-                    transposed_system[col * size + j] /= d;
+                    transposed_system[i * size + j] +=
+                        scal * transposed_system[col * size + j];
                 }
-                rhs[col] /= d;
-                transposed_system[col * size + col] = one<ValueType>() / d;
+                rhs[i] += rhs_key_val * scal;
             }
+            for (IndexType j = 0; j < size; ++j) {
+                transposed_system[col * size + j] /= d;
+            }
+            rhs[col] /= d;
+            transposed_system[col * size + col] = one<ValueType>() / d;
+        }
 
-            if (spd) {
-                const auto scal = one<ValueType>() / sqrt(rhs[size - 1]);
-                for (IndexType row = 0; row < size; row++) {
-                    rhs[row] *= scal;
-                }
+        if (spd) {
+            const auto scal = one<ValueType>() / sqrt(rhs[size - 1]);
+            for (IndexType row = 0; row < size; row++) {
+                rhs[row] *= scal;
             }
-        };
+        }
+    };
 
     generic_generate(exec, mtx, inverse_mtx, excess_rhs_ptrs, excess_nz_ptrs,
-                     general_solve);
+                     general_solve, false);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -341,7 +338,8 @@ template <typename ValueType, typename IndexType>
 void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
                             const matrix::Csr<ValueType, IndexType> *input,
                             const matrix::Csr<ValueType, IndexType> *inverse,
-                            const IndexType *, const IndexType *,
+                            const IndexType *excess_rhs_ptrs,
+                            const IndexType *excess_nz_ptrs,
                             matrix::Csr<ValueType, IndexType> *excess_system,
                             matrix::Dense<ValueType> *excess_rhs,
                             size_type e_start, size_type e_end)
@@ -357,18 +355,20 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
     auto e_cols = excess_system->get_col_idxs();
     auto e_vals = excess_system->get_values();
     auto e_rhs = excess_rhs->get_values();
-    IndexType e_block_begin{};
-    IndexType e_nz{};
 
     for (size_type row = e_start; row < e_end; ++row) {
         const auto i_begin = i_row_ptrs[row];
         const auto i_size = i_row_ptrs[row + 1] - i_begin;
+        // first row index of the sparse block in the excess system
+        auto e_begin = excess_rhs_ptrs[row] - excess_rhs_ptrs[e_start];
+        // first non-zero index in the sparse block
+        auto e_nz = excess_nz_ptrs[row] - excess_nz_ptrs[e_start];
 
         if (i_size > row_size_limit) {
             // count non-zeros and dimension in the excess system
             for (size_type i = 0; i < i_size; ++i) {
                 // current row in the excess system
-                const auto e_row = e_block_begin + i;
+                const auto e_row = e_begin + i;
                 const auto col = i_cols[i_begin + i];
                 const auto m_begin = m_row_ptrs[col];
                 const auto m_size = m_row_ptrs[col + 1] - m_begin;
@@ -383,15 +383,14 @@ void generate_excess_system(std::shared_ptr<const DefaultExecutor>,
                     [&](IndexType, IndexType m_idx, IndexType i_idx) {
                         // trisystem(i, i_idx) = m_vals[m_idx + m_begin]
                         // just in sparse
-                        e_cols[e_nz] = i_idx + e_block_begin;
+                        e_cols[e_nz] = i_idx + e_begin;
                         e_vals[e_nz] = m_vals[m_idx + m_begin];
                         ++e_nz;
                     });
             }
-            e_block_begin += i_size;
         }
     }
-    e_row_ptrs[e_dim] = e_nz;
+    e_row_ptrs[e_dim] = excess_nz_ptrs[e_end] - excess_nz_ptrs[e_start];
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
diff --git a/reference/test/preconditioner/isai_kernels.cpp b/reference/test/preconditioner/isai_kernels.cpp
index 3273a4c783f..5a79f064577 100644
--- a/reference/test/preconditioner/isai_kernels.cpp
+++ b/reference/test/preconditioner/isai_kernels.cpp
@@ -430,7 +430,6 @@ TYPED_TEST(Isai, KernelGenerateALongrow)
     GKO_ASSERT_MTX_EQ_SPARSITY(result, this->a_csr_longrow_inv_partial);
     GKO_ASSERT_MTX_NEAR(result, this->a_csr_longrow_inv_partial,
                         r<value_type>::value);
-    // no row above the size limit -> zero array
     GKO_ASSERT_ARRAY_EQ(a1, a1_expect);
     GKO_ASSERT_ARRAY_EQ(a2, a2_expect);
 }
@@ -939,7 +938,6 @@ TYPED_TEST(Isai, KernelGenerateSpdLongrow)
     GKO_ASSERT_MTX_EQ_SPARSITY(result, this->spd_csr_longrow_inv_partial);
     GKO_ASSERT_MTX_NEAR(result, this->spd_csr_longrow_inv_partial,
                         r<value_type>::value);
-    // no row above the size limit -> zero array
     GKO_ASSERT_ARRAY_EQ(a1, a1_expect);
     GKO_ASSERT_ARRAY_EQ(a2, a2_expect);
 }

From 9b0ed227793f8c0bf17d01e4988a6c62debb5088 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 18 Mar 2021 09:22:53 +0100
Subject: [PATCH 0518/1008] Code Review

---
 include/ginkgo/core/preconditioner/isai.hpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/include/ginkgo/core/preconditioner/isai.hpp b/include/ginkgo/core/preconditioner/isai.hpp
index 4266c203bf4..d3ff87d7b10 100644
--- a/include/ginkgo/core/preconditioner/isai.hpp
+++ b/include/ginkgo/core/preconditioner/isai.hpp
@@ -179,7 +179,10 @@ class Isai : public EnableLinOp<Isai<IsaiType, ValueType, IndexType>>,
         size_type GKO_FACTORY_PARAMETER_SCALAR(excess_limit, 0u);
 
         /**
-         * Factory for the Excess System solver
+         * @brief Factory for the Excess System solver.
+         *
+         * Defaults to using a triangular solver for upper and lower ISAI and
+         * to Block-Jacobi preconditioned GMRES for general and spd ISAI.
          */
         std::shared_ptr<LinOpFactory> GKO_FACTORY_PARAMETER_SCALAR(
             excess_solver_factory, nullptr);

From 0569d1a36d8430aee2c0151a0311b2e4fcf87ab6 Mon Sep 17 00:00:00 2001
From: fritzgoebel <fritz.goebel@kit.edu>
Date: Thu, 18 Mar 2021 09:25:40 +0100
Subject: [PATCH 0519/1008] Apply suggestions from code review

Co-authored-by: Yuhsiang M. Tsai <19565938+yhmtsai@users.noreply.github.com>
---
 cuda/test/preconditioner/isai_kernels.cpp    | 2 ++
 hip/test/preconditioner/isai_kernels.hip.cpp | 2 ++
 omp/test/preconditioner/isai_kernels.cpp     | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/cuda/test/preconditioner/isai_kernels.cpp b/cuda/test/preconditioner/isai_kernels.cpp
index a8600039d7e..f099e3f503b 100644
--- a/cuda/test/preconditioner/isai_kernels.cpp
+++ b/cuda/test/preconditioner/isai_kernels.cpp
@@ -57,6 +57,8 @@ namespace {
 
 
 enum struct matrix_type { lower, upper, general, spd };
+
+
 class Isai : public ::testing::Test {
 protected:
     using value_type = double;
diff --git a/hip/test/preconditioner/isai_kernels.hip.cpp b/hip/test/preconditioner/isai_kernels.hip.cpp
index e4052cc022c..18470c3d77c 100644
--- a/hip/test/preconditioner/isai_kernels.hip.cpp
+++ b/hip/test/preconditioner/isai_kernels.hip.cpp
@@ -54,6 +54,8 @@ namespace {
 
 
 enum struct matrix_type { lower, upper, general, spd };
+
+
 class Isai : public ::testing::Test {
 protected:
     using value_type = double;
diff --git a/omp/test/preconditioner/isai_kernels.cpp b/omp/test/preconditioner/isai_kernels.cpp
index 58c52d9d74a..d1f7ba0243b 100644
--- a/omp/test/preconditioner/isai_kernels.cpp
+++ b/omp/test/preconditioner/isai_kernels.cpp
@@ -53,6 +53,8 @@ namespace {
 
 
 enum struct matrix_type { lower, upper, general, spd };
+
+
 class Isai : public ::testing::Test {
 protected:
     using value_type = double;

From 36a7b9e409d8036b7adcdce19d4f70a92c4df8b4 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Fri, 19 Mar 2021 08:42:31 +0100
Subject: [PATCH 0520/1008] remove wrong comment

---
 reference/test/preconditioner/isai_kernels.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/reference/test/preconditioner/isai_kernels.cpp b/reference/test/preconditioner/isai_kernels.cpp
index 5a79f064577..c501f956cfc 100644
--- a/reference/test/preconditioner/isai_kernels.cpp
+++ b/reference/test/preconditioner/isai_kernels.cpp
@@ -629,7 +629,6 @@ TYPED_TEST(Isai, KernelGenerateLLongrow)
     GKO_ASSERT_MTX_EQ_SPARSITY(result, this->l_csr_longrow_inv_partial);
     GKO_ASSERT_MTX_NEAR(result, this->l_csr_longrow_inv_partial,
                         r<value_type>::value);
-    // no row above the size limit -> zero array
     GKO_ASSERT_ARRAY_EQ(a1, a1_expect);
     GKO_ASSERT_ARRAY_EQ(a2, a2_expect);
 }
@@ -829,7 +828,6 @@ TYPED_TEST(Isai, KernelGenerateULongrow)
     GKO_ASSERT_MTX_EQ_SPARSITY(result, this->u_csr_longrow_inv_partial);
     GKO_ASSERT_MTX_NEAR(result, this->u_csr_longrow_inv_partial,
                         r<value_type>::value);
-    // no row above the size limit -> zero array
     GKO_ASSERT_ARRAY_EQ(a1, a1_expect);
     GKO_ASSERT_ARRAY_EQ(a2, a2_expect);
 }

From 58a205fc143fcc4b4740495639dd8ded275bd32a Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 17 Mar 2021 22:44:40 +0800
Subject: [PATCH 0521/1008] pass the use_lib_linops variable to next function

---
 benchmark/CMakeLists.txt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
index 0ad53594fd4..9e0e1f118ef 100644
--- a/benchmark/CMakeLists.txt
+++ b/benchmark/CMakeLists.txt
@@ -96,13 +96,13 @@ endfunction(ginkgo_add_single_benchmark_executable)
 # All remaining arguments will be treated as source files
 function(ginkgo_add_typed_benchmark_executables name use_lib_linops)
     ginkgo_add_single_benchmark_executable(
-        "${name}" use_lib_linops "GKO_BENCHMARK_USE_DOUBLE_PRECISION" ${ARGN})
+        "${name}" "${use_lib_linops}" "GKO_BENCHMARK_USE_DOUBLE_PRECISION" ${ARGN})
     ginkgo_add_single_benchmark_executable(
-        "${name}_single" use_lib_linops "GKO_BENCHMARK_USE_SINGLE_PRECISION" ${ARGN})
+        "${name}_single" "${use_lib_linops}" "GKO_BENCHMARK_USE_SINGLE_PRECISION" ${ARGN})
     ginkgo_add_single_benchmark_executable(
-        "${name}_dcomplex" use_lib_linops "GKO_BENCHMARK_USE_DOUBLE_COMPLEX_PRECISION" ${ARGN})
+        "${name}_dcomplex" "${use_lib_linops}" "GKO_BENCHMARK_USE_DOUBLE_COMPLEX_PRECISION" ${ARGN})
     ginkgo_add_single_benchmark_executable(
-        "${name}_scomplex" use_lib_linops "GKO_BENCHMARK_USE_SINGLE_COMPLEX_PRECISION" ${ARGN})
+        "${name}_scomplex" "${use_lib_linops}" "GKO_BENCHMARK_USE_SINGLE_COMPLEX_PRECISION" ${ARGN})
 endfunction(ginkgo_add_typed_benchmark_executables)
 
 

From 98541064932d95ae322bddaefaaa2c580ba74664 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 23 Mar 2021 09:05:28 +0100
Subject: [PATCH 0522/1008] Add a publications dropdown in doc

---
 doc/DoxygenLayout.xml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/DoxygenLayout.xml b/doc/DoxygenLayout.xml
index 268f8348145..b31ab5bfc0b 100644
--- a/doc/DoxygenLayout.xml
+++ b/doc/DoxygenLayout.xml
@@ -5,7 +5,11 @@
     <tab type="mainpage" visible="yes" title=""/>
     <tab type="user" visible="yes" title="Tutorial" url="https://github.com/ginkgo-project/ginkgo/wiki/Tutorial:-Building-a-Poisson-Solver" />
     <tab type="user" visible="yes" title="Examples" url="@ref Examples" />
-    <tab type="user" visible="yes" title="Citing Ginkgo" url="@ref citing_ginkgo" />
+    <tab type="user" visible="yes" title="Publications" url="@ref citing_ginkgo" >
+      <tab type="user" visible="yes" title="Citing Ginkgo" url="@ref citing_ginkgo" />
+      <tab type="user" visible="yes" title="Ginkgo White Paper" url="https://www.arxiv-vanity.com/papers/2006.16852/" />
+      <tab type="user" visible="yes" title="Slides and Outreach" url="https://github.com/ginkgo-project/outreach" />
+    </tab>
     <tab type="user" visible="yes" title="Contributing To Ginkgo" url="@ref contributing_guidelines" />
     <tab type="user" visible="yes" title="Using Ginkgo" url="@ref install_ginkgo">
       <tab type="user" visible="yes" title="Installing Ginkgo" url="@ref install_ginkgo" />

From aab5148f6dcb1b93bb22a2a43587492409fe97e1 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Fri, 26 Feb 2021 05:30:59 +0100
Subject: [PATCH 0523/1008] HIP overhaul, Hwloc and test_install fixes.

On hwloc
________

There were a few issues with the CMake hwloc setup.
+ hwloc was always built even when a valid version already existed on
the system.
+ The FindHWLOC module had a few issues in terms of organization, search
paths as well as variable management. This has been fixed.
+ hwloc is a public dependency of Ginkgo but was not installed when we
use the one from our build directory, that would create errors with
missing libraries in some cases.
+ The `HWLOC_XMLFILE` should not be managed by Ginkgo, hwloc already
takes care of it, if the environment variable `HWLOC_XMLFILE` is set, it
uses that as base, otherwise it uses the machine topology it can
detect.
+ The core/devices which are always compiled were moved into their own
root directory. This better reflects the new library that it creates as
well as the fact all libraries depend on it.

On test install
________

Test install was missing a HIP version which I added. In addition, the
commands were not properly triggered thus only the base (non CUDA
version) was tested, that is now fixed.

On RPATH management
_________

For HIP and HWLOC, the INSTALL_RPATH property is now properly managed.
Since all libraries require HWLOC, this setting is added directly into
`ginkgo_install_libraries`. For HIP, the setting is added only when
actually building HIP.

On HIP
________

This part saw a lot of changes.

+ ROCm < 3.5 support is dropped as it is now old and all HPC clusters
always update the ROCm version since they know the ecosystem is somewhat
unstable.
+ Instead of linking against `hip::device` in `ginkgo_hip`, I link
against `libamdhip64.so` publicly.
+ In general, the purpose of this rework is to depend more on CMake
transitive properties such that we remove special cases. Thanks to this
and the previous point as well, it is possible to remove entirely the
`hip_helpers.cmake` file and remove all of the bans we were doing.
+ Thanks to relying on transitive properties, there is no need for the
`special_linkage` test case anymore.
+ Instead of building the `RPATH_FOR_HIP` chain, in Ginkgo everything
uses the `CXX` language for linking, which properly takes into account
RPATH information unlike when liking with the `HIP` language. For users,
a simple string `GINKGO_INSTALL_RPATH_FOR_HIP` is provided which is a
linking flag to be given to the `HIP` language linker after installing
Ginkgo.
+ In general, there is also better management or relying more on the
defaults for some environment variables and HIP variables.

Open issues
________

An annoying issue (still), although it's mitigated thanks to linking
publicly against `libamdhip64.so` which doesn't create inconvenience to
the user, is that we have to include `hip/hip_runtime.hpp` in our public
headers (to find `__forceinline__` as well as `HIP_DEVICE_COMPILE`, see
https://github.com/ginkgo-project/ginkgo/blob/develop/include/ginkgo/core/base/types.hpp#L46-L85.
In addition, the `GKO_ASSERT` cannot be reworked as well otherwise it
creates mysterious issues in the `check_headers` test, see:
https://gitlab.com/ginkgo-project/ginkgo-public-ci/-/jobs/1073846520
---
 CMakeLists.txt                                |  56 +--
 benchmark/CMakeLists.txt                      |  12 -
 cmake/GinkgoConfig.cmake.in                   |  14 +-
 cmake/Modules/FindHWLOC.cmake                 |  54 ++-
 cmake/create_test.cmake                       |  63 +--
 cmake/get_info.cmake                          |   1 -
 cmake/hip_helpers.cmake                       |  30 --
 cmake/information_helpers.cmake               |  10 +-
 cmake/install_helpers.cmake                   |  37 +-
 core/CMakeLists.txt                           |   9 +-
 core/device_hooks/CMakeLists.txt              |  15 +-
 {core/devices => devices}/CMakeLists.txt      |   2 +-
 {core/devices => devices}/cuda/CMakeLists.txt |   0
 {core/devices => devices}/cuda/executor.cpp   |   0
 .../devices => devices}/dpcpp/CMakeLists.txt  |   0
 {core/devices => devices}/dpcpp/executor.cpp  |   0
 {core/devices => devices}/hip/CMakeLists.txt  |   0
 {core/devices => devices}/hip/executor.cpp    |   0
 .../devices => devices}/machine_topology.cpp  |   1 -
 {core/devices => devices}/omp/CMakeLists.txt  |   0
 {core/devices => devices}/omp/executor.cpp    |   0
 .../reference/CMakeLists.txt                  |   0
 {core/devices => devices}/reference/dummy.cpp |   0
 dpcpp/CMakeLists.txt                          |   2 +-
 hip/CMakeLists.txt                            | 128 ++----
 hip/get_info.cmake                            |   5 +-
 hip/test/factorization/CMakeLists.txt         |   4 +-
 hip/test/preconditioner/CMakeLists.txt        |   2 +-
 hip/test/solver/CMakeLists.txt                |  22 +-
 hip/test/stop/CMakeLists.txt                  |   2 +-
 include/ginkgo/config.hpp.in                  |   5 +-
 omp/CMakeLists.txt                            |   8 +-
 reference/CMakeLists.txt                      |   2 +-
 test_exportbuild/CMakeLists.txt               |  15 -
 test_install/CMakeLists.txt                   |  34 +-
 test_install/test_install.hip.cpp             | 387 ++++++++++++++++++
 third_party/CMakeLists.txt                    |   2 +-
 third_party/hwloc/CMakeLists.txt              |  10 +-
 38 files changed, 590 insertions(+), 342 deletions(-)
 delete mode 100644 cmake/hip_helpers.cmake
 rename {core/devices => devices}/CMakeLists.txt (93%)
 rename {core/devices => devices}/cuda/CMakeLists.txt (100%)
 rename {core/devices => devices}/cuda/executor.cpp (100%)
 rename {core/devices => devices}/dpcpp/CMakeLists.txt (100%)
 rename {core/devices => devices}/dpcpp/executor.cpp (100%)
 rename {core/devices => devices}/hip/CMakeLists.txt (100%)
 rename {core/devices => devices}/hip/executor.cpp (100%)
 rename {core/devices => devices}/machine_topology.cpp (99%)
 rename {core/devices => devices}/omp/CMakeLists.txt (100%)
 rename {core/devices => devices}/omp/executor.cpp (100%)
 rename {core/devices => devices}/reference/CMakeLists.txt (100%)
 rename {core/devices => devices}/reference/dummy.cpp (100%)
 create mode 100644 test_install/test_install.hip.cpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 49a6d20f9b4..9027c2e466a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -65,8 +65,6 @@ set(GINKGO_HIP_COMPILER_FLAGS "" CACHE STRING
     "Set the required HIP compiler flags. Current default is an empty string.")
 set(GINKGO_HIP_NVCC_COMPILER_FLAGS "" CACHE STRING
     "Set the required HIP nvcc compiler flags. Current default is an empty string.")
-set(GINKGO_HIP_HCC_COMPILER_FLAGS "" CACHE STRING
-    "Set the required HIP HCC compiler flags. Current default is an empty string.")
 set(GINKGO_HIP_CLANG_COMPILER_FLAGS "" CACHE STRING
     "Set the required HIP CLANG compiler flags. Current default is an empty string.")
 set(GINKGO_HIP_AMDGPU "" CACHE STRING
@@ -191,12 +189,22 @@ if(GINKGO_BUILD_HIP)
     endif()
 endif()
 
-configure_file(${Ginkgo_SOURCE_DIR}/include/ginkgo/config.hpp.in
-    ${Ginkgo_BINARY_DIR}/include/ginkgo/config.hpp @ONLY)
+
+# Try to find the third party packages before using our subdirectories
+include(cmake/package_helpers.cmake)
+ginkgo_find_package(GTest "GTest::GTest;GTest::Main" FALSE 1.8.1)
+ginkgo_find_package(gflags gflags FALSE 2.2.2)
+ginkgo_find_package(RapidJSON rapidjson TRUE 1.1.0)
+if(GINKGO_HAVE_HWLOC)
+    ginkgo_find_package(HWLOC hwloc FALSE 2.1)
+    if (NOT HWLOC_FOUND)
+        set(GINKGO_USE_EXTERNAL_HWLOC 0)
+    endif()
+endif()
+add_subdirectory(third_party)    # Third-party tools and libraries
 
 # Load CMake helpers
 include(cmake/build_helpers.cmake)
-include(cmake/hip_helpers.cmake)
 include(cmake/install_helpers.cmake)
 include(cmake/windows_helpers.cmake)
 
@@ -214,40 +222,30 @@ if(MSVC)
     endif()
 endif()
 
-# Try to find the third party packages before using our subdirectories
-include(cmake/package_helpers.cmake)
-ginkgo_find_package(GTest "GTest::GTest;GTest::Main" FALSE 1.8.1)
-ginkgo_find_package(gflags gflags FALSE 2.2.2)
-ginkgo_find_package(RapidJSON rapidjson TRUE 1.1.0)
-if(GINKGO_HAVE_HWLOC)
-    ginkgo_find_package(HWLOC hwloc FALSE 2.1)
-endif()
-set(GKO_HWLOC_XMLFILE "$ENV{HWLOC_XMLFILE}")
-message(STATUS "HWLOC XML file set to: ${GKO_HWLOC_XMLFILE}")
-add_subdirectory(third_party)    # Third-party tools and libraries
+configure_file(${Ginkgo_SOURCE_DIR}/include/ginkgo/config.hpp.in
+    ${Ginkgo_BINARY_DIR}/include/ginkgo/config.hpp @ONLY)
 
 # Ginkgo core libraries
 # Needs to be first in order for `CMAKE_CUDA_DEVICE_LINK_EXECUTABLE` to be
 # propagated to the other parts of Ginkgo in case of building as static libraries
+add_subdirectory(devices)        # Basic device functionalities. Always compiled.
 if(GINKGO_BUILD_CUDA)
     add_subdirectory(cuda)       # High-performance kernels for NVIDIA GPUs
 endif()
-add_subdirectory(core)           # Core Ginkgo types and top-level functions
-add_subdirectory(include)        # Public API self-contained check
 if (GINKGO_BUILD_REFERENCE)
     add_subdirectory(reference)  # Reference kernel implementations
 endif()
-if (GINKGO_BUILD_OMP)
-    add_subdirectory(omp)        # High-performance omp kernels
+if(GINKGO_BUILD_HIP)
+    add_subdirectory(hip)        # High-performance kernels for AMD or NVIDIA GPUs
 endif()
 if (GINKGO_BUILD_DPCPP)
     add_subdirectory(dpcpp)        # High-performance DPC++ kernels
 endif()
-# HIP needs to be last because it builds the GINKGO_RPATH_FOR_HIP variable
-# which needs to know the `ginkgo` target.
-if(GINKGO_BUILD_HIP)
-    add_subdirectory(hip)        # High-performance kernels for AMD or NVIDIA GPUs
+if (GINKGO_BUILD_OMP)
+    add_subdirectory(omp)        # High-performance omp kernels
 endif()
+add_subdirectory(core)           # Core Ginkgo types and top-level functions
+add_subdirectory(include)        # Public API self-contained check
 
 # Non core directories and targets
 if(GINKGO_BUILD_EXAMPLES)
@@ -322,20 +320,22 @@ if(MSVC)
     set(GINKGO_TEST_INSTALL_COMMAND "${Ginkgo_BINARY_DIR}/test_install/$<CONFIG>/test_install")
     set(GINKGO_TEST_EXPORTBUILD_COMMAND "${Ginkgo_BINARY_DIR}/test_exportbuild/$<CONFIG>/test_exportbuild")
     if(GINKGO_BUILD_CUDA)
-        set(GINKGO_TEST_INSTALL_COMMAND "${GINKGO_TEST_INSTALL_COMMAND}" "${Ginkgo_BINARY_DIR}/test_install/$<CONFIG>/test_install_cuda")
+        set(GINKGO_TEST_INSTALL_CUDA_COMMAND "${Ginkgo_BINARY_DIR}/test_install/$<CONFIG>/test_install_cuda")
     endif()
 else()
     set(GINKGO_TEST_INSTALL_COMMAND "${Ginkgo_BINARY_DIR}/test_install/test_install")
     set(GINKGO_TEST_EXPORTBUILD_COMMAND "${Ginkgo_BINARY_DIR}/test_exportbuild/test_exportbuild")
     if(GINKGO_BUILD_CUDA)
-        set(GINKGO_TEST_INSTALL_COMMAND "${GINKGO_TEST_INSTALL_COMMAND}" "${Ginkgo_BINARY_DIR}/test_install/test_install_cuda")
+        set(GINKGO_TEST_INSTALL_CUDA_COMMAND "${Ginkgo_BINARY_DIR}/test_install/test_install_cuda")
+    endif()
+    if(GINKGO_BUILD_HIP)
+        set(GINKGO_TEST_INSTALL_HIP_COMMAND "${Ginkgo_BINARY_DIR}/test_install/test_install_hip")
     endif()
 endif()
 add_custom_target(test_install
     COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} -H${Ginkgo_SOURCE_DIR}/test_install
     -B${Ginkgo_BINARY_DIR}/test_install
     -DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX}/${GINKGO_INSTALL_CONFIG_DIR}
-    -DHWLOC_DIR=${HWLOC_DIR}
     -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
     -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
     # `--config cfg` is ignored by single-configuration generator.
@@ -343,6 +343,8 @@ add_custom_target(test_install
     # single-configuration generator.
     COMMAND ${CMAKE_COMMAND} --build ${Ginkgo_BINARY_DIR}/test_install --config $<CONFIG>
     COMMAND ${GINKGO_TEST_INSTALL_COMMAND}
+    COMMAND ${GINKGO_TEST_INSTALL_CUDA_COMMAND}
+    COMMAND ${GINKGO_TEST_INSTALL_HIP_COMMAND}
     COMMENT "Running a test on the installed binaries. This requires running `(sudo) make install` first.")
 
 add_custom_target(test_exportbuild
diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
index 9e0e1f118ef..af09ee108af 100644
--- a/benchmark/CMakeLists.txt
+++ b/benchmark/CMakeLists.txt
@@ -45,18 +45,6 @@ function(ginkgo_benchmark_hipsp_linops name)
         ${HSA_HEADER} ${HIP_INCLUDE_DIRS}
         ${HIPBLAS_INCLUDE_DIRS} ${HIPSPARSE_INCLUDE_DIRS})
 
-    if(GINKGO_HIP_PLATFORM MATCHES "hcc")
-        ginkgo_hip_ban_link_hcflag(hcc::hccrt)
-        ginkgo_hip_ban_link_hcflag(hcc::hc_am)
-        ginkgo_hip_ban_link_hcflag(hcc::mcwamp)
-        ginkgo_hip_ban_compile_hcflag(hcc::hccrt)
-        ginkgo_hip_ban_compile_hcflag(hcc::hc_am)
-        ginkgo_hip_ban_compile_hcflag(hcc::mcwamp)
-        ginkgo_hip_clang_ban_hip_device_flags()
-        target_link_libraries("${name}" hip::device)
-    else()
-        target_link_libraries("${name}" ${HIP_CUDA_LIBRARIES})
-    endif()
     target_link_libraries("${name}" ${HIPSPARSE_LIBRARIES})
 endfunction()
 
diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 51a2e8a7249..460b626ded3 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -107,6 +107,7 @@ set(GINKGO_INSTALL_LIBRARY_DIR "${GINKGO_INSTALL_PREFIX}/@GINKGO_INSTALL_LIBRARY
 set(GINKGO_INSTALL_PKGCONFIG_DIR "${GINKGO_INSTALL_PREFIX}/@GINKGO_INSTALL_PKGCONFIG_DIR@")
 set(GINKGO_INSTALL_CONFIG_DIR "${GINKGO_INSTALL_PREFIX}/@GINKGO_INSTALL_CONFIG_DIR@")
 set(GINKGO_INSTALL_MODULE_DIR "${GINKGO_INSTALL_PREFIX}/@GINKGO_INSTALL_MODULE_DIR@")
+set(GINKGO_INSTALL_RPATH_FOR_HIP "-Wl,-rpath,${GINKGO_INSTALL_LIBRARY_DIR}")
 
 # Forward Ginkgo's MODULE PATH and the PREFIX PATH for HIP and more
 list(APPEND CMAKE_MODULE_PATH "@CMAKE_MODULE_PATH@" "${GINKGO_INSTALL_MODULE_DIR}")
@@ -139,10 +140,6 @@ set(GINKGO_OPENMP_LIBRARIES @OpenMP_CXX_LIBRARIES@)
 
 set(GINKGO_OPENMP_FLAGS "@OpenMP_CXX_FLAGS@")
 
-# Provide useful HIP helper functions
-if(GINKGO_BUILD_HIP)
-    include(${CMAKE_CURRENT_LIST_DIR}/hip_helpers.cmake)
-endif()
 if (WIN32 OR CYGWIN)
     include(${CMAKE_CURRENT_LIST_DIR}/windows_helpers.cmake)
 endif()
@@ -181,15 +178,6 @@ if((NOT GINKGO_BUILD_SHARED_LIBS) AND GINKGO_BUILD_HIP)
     find_package(hiprand REQUIRED)
     find_package(hipsparse REQUIRED)
     find_package(rocrand REQUIRED)
-    if(GINKGO_HIP_PLATFORM MATCHES "hcc")
-        ginkgo_hip_ban_link_hcflag(hcc::hccrt)
-        ginkgo_hip_ban_link_hcflag(hcc::hc_am)
-        ginkgo_hip_ban_link_hcflag(hcc::mcwamp)
-        ginkgo_hip_ban_compile_hcflag(hcc::hccrt)
-        ginkgo_hip_ban_compile_hcflag(hcc::hc_am)
-        ginkgo_hip_ban_compile_hcflag(hcc::mcwamp)
-    endif()
 endif()
 
-
 include(${CMAKE_CURRENT_LIST_DIR}/GinkgoTargets.cmake)
diff --git a/cmake/Modules/FindHWLOC.cmake b/cmake/Modules/FindHWLOC.cmake
index e0d146bafe4..4d6eddff5f2 100644
--- a/cmake/Modules/FindHWLOC.cmake
+++ b/cmake/Modules/FindHWLOC.cmake
@@ -40,21 +40,18 @@ include(hwloc_helpers)
 find_path(HWLOC_INCLUDE_DIRS
     NAMES "hwloc.h"
     HINTS ${HWLOC_DIR} $ENV{HWLOC_DIR}
-    PATH_SUFFIXES include src/include
+    PATH_SUFFIXES include
     DOC "Find the hwloc.h main header"
     )
 
 find_library(HWLOC_LIBRARIES "hwloc"
     HINTS ${HWLOC_DIR} $ENV{HWLOC_DIR}
-    PATH_SUFFIXES hwloc/lib lib lib64
+    PATH_SUFFIXES lib lib64
     DOC "Find the hwloc library"
     )
 
 if (HWLOC_INCLUDE_DIRS)
-    unset(HWLOC_FOUND CACHE)
-    set(HWLOC_FOUND 1)
-
-    # Find the version of hwloc found
+        # Find the version of hwloc found
     if(NOT HWLOC_VERSION)
         file(READ "${HWLOC_INCLUDE_DIRS}/hwloc.h"
             HEADER_CONTENTS LIMIT 16384)
@@ -68,25 +65,26 @@ if (HWLOC_INCLUDE_DIRS)
         get_dec_from_hex("${HEX_PATCH}" DEC_PATCH)
         set(HWLOC_VERSION "${DEC_MAJOR}.${DEC_MINOR}.${DEC_PATCH}" CACHE STRING "HWLOC version")
     endif()
+endif()
 
-    if (NOT HWLOC_FIND_QUIETLY)
-        if (HWLOC_FOUND AND HWLOC_LIBRARIES)
-            message(STATUS "Looking for HWLOC - found version ${HWLOC_VERSION}")
-        else()
-            message(STATUS "${Magenta}Looking for HWLOC - not found"
-                "\n   Please check that your environment variable HWLOC_DIR"
-                "\n   has been set properly.${ColourReset}")
-        endif()
+if (NOT HWLOC_FIND_QUIETLY)
+    if (HWLOC_INCLUDE_DIRS AND HWLOC_LIBRARIES)
+        message(STATUS "Looking for HWLOC - found version ${HWLOC_VERSION}")
+    else()
+        message(STATUS "${Magenta}Looking for HWLOC - not found"
+            "\n   Please check that your environment variable HWLOC_DIR"
+            "\n   has been set properly.${ColourReset}")
+        unset(HWLOC_LIBRARIES CACHE)
+        unset(HWLOC_INCLUDE_DIRS CACHE)
     endif()
 endif()
 
-# check a function to validate the find
-if(HWLOC_FOUND AND HWLOC_LIBRARIES)
-
-    # set required libraries for link
+# check a function to validate what was found
+if(HWLOC_INCLUDE_DIRS AND HWLOC_LIBRARIES)
+    # set required libraries for linking
     ginkgo_set_required_test_lib_link(HWLOC)
 
-    # test link
+    # test linking
     unset(HWLOC_WORKS CACHE)
     include(CheckFunctionExists)
     check_function_exists(hwloc_topology_init HWLOC_WORKS)
@@ -108,18 +106,19 @@ if(HWLOC_FOUND AND HWLOC_LIBRARIES)
     string(SUBSTRING "${HWLOC_VERSION}" 0 3 HWLOC_VERSION)
     if(HWLOC_VERSION VERSION_LESS HWLOC_FIND_VERSION)
         message(STATUS "Required version ${HWLOC_FIND_VERSION}, but found version ${HWLOC_VERSION}")
-        set(HWLOC_FOUND 0)
-        unset(HWLOC_LIBRARIES)
-        unset(HWLOC_INCLUDE_DIRS)
+        unset(HWLOC_FOUND CACHE)
+        unset(HWLOC_LIBRARIES CACHE)
+        unset(HWLOC_INCLUDE_DIRS CACHE)
     else()
+        unset(HWLOC_FOUND CACHE)
+        set(HWLOC_FOUND 1)
         include(FindPackageHandleStandardArgs)
         find_package_handle_standard_args(HWLOC
             REQUIRED_VARS HWLOC_LIBRARIES HWLOC_INCLUDE_DIRS HWLOC_WORKS
             VERSION_VAR HWLOC_VERSION)
         mark_as_advanced(HWLOC_INCLUDE_DIRS HWLOC_LIBRARIES HWLOC_VERSION HWLOC_WORKS)
     endif()
-
-endif(HWLOC_FOUND AND HWLOC_LIBRARIES)
+endif(HWLOC_INCLUDE_DIRS AND HWLOC_LIBRARIES)
 
 if(HWLOC_FOUND)
     add_library(hwloc SHARED IMPORTED GLOBAL)
@@ -127,10 +126,3 @@ if(HWLOC_FOUND)
     set_target_properties(hwloc PROPERTIES INTERFACE_LINK_LIBRARIES ${HWLOC_LIBRARIES})
     set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${HWLOC_INCLUDE_DIRS})
 endif()
-
-if(HWLOC_FIND_REQUIRED AND NOT HWLOC_FOUND)
-    unset(HWLOC_LIBRARIES)
-    unset(HWLOC_INCLUDE_DIRS)
-    message(SEND_ERROR "HWLOC could not be found. A version mismatch could have occured. The version found was ${HWLOC_VERSION}.\n"
-        "Hints where hwloc is installed can be given in the HWLOC_DIR variable. Current HWLOC_DIR: ${HWLOC_DIR}")
-endif()
diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index 022de4bf61b..dda929ca4f7 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -101,29 +101,6 @@ function(ginkgo_create_cuda_test test_name)
     add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
 endfunction(ginkgo_create_cuda_test)
 
-function(ginkgo_create_hip_test_special_linkage test_name)
-    # use gcc to compile but use hip to link
-    file(RELATIVE_PATH REL_BINARY_DIR
-        ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
-    string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
-    add_executable(${TEST_TARGET_NAME} ${test_name}.cpp)
-    target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_14)
-    # Fix the missing metadata when building static library.
-    if(GINKGO_HIP_PLATFORM MATCHES "hcc" AND NOT BUILD_SHARED_LIBS)
-        set_target_properties(${TEST_TARGET_NAME} PROPERTIES LINKER_LANGUAGE HIP)
-    endif()
-    target_include_directories("${TEST_TARGET_NAME}"
-        PRIVATE
-        "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
-        )
-    set_target_properties(${TEST_TARGET_NAME} PROPERTIES
-        OUTPUT_NAME ${test_name})
-    if (GINKGO_CHECK_CIRCULAR_DEPS)
-        target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
-    endif()
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
-    add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
-endfunction(ginkgo_create_hip_test_special_linkage)
 
 function(ginkgo_create_hip_test test_name)
     file(RELATIVE_PATH REL_BINARY_DIR
@@ -131,25 +108,16 @@ function(ginkgo_create_hip_test test_name)
     string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
 
     set_source_files_properties(${test_name}.hip.cpp PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
+    hip_add_executable(${TEST_TARGET_NAME} ${test_name}.hip.cpp
+        # With how HIP works, to prevent redifinition we need to comment these.
+        # They might need to be used later though.
+        #HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS}
+        #NVCC_OPTIONS  ${GINKGO_HIP_NVCC_OPTIONS}
+        #CLANG_OPTIONS ${GINKGO_HIP_CLANG_OPTIONS}
+        )
 
-    if (HIP_VERSION GREATER_EQUAL "3.5")
-        hip_add_executable(${TEST_TARGET_NAME} ${test_name}.hip.cpp
-            HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS}
-            NVCC_OPTIONS  ${GINKGO_HIP_NVCC_OPTIONS}
-            HCC_OPTIONS ${GINKGO_HIP_HCC_OPTIONS}
-            CLANG_OPTIONS ${GINKGO_HIP_CLANG_OPTIONS})
-    else()
-        hip_add_executable(${TEST_TARGET_NAME} ${test_name}.hip.cpp
-            HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS}
-            NVCC_OPTIONS  ${GINKGO_HIP_NVCC_OPTIONS}
-            HCC_OPTIONS ${GINKGO_HIP_HCC_OPTIONS})
-    endif()
-    target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_14)
-
-    # Let's really not use nvcc for linking here
-    if (GINKGO_HIP_PLATFORM MATCHES "nvcc")
-        set_target_properties(${TEST_TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
-    endif()
+    # Let's use a normal compiler for linking
+    set_target_properties(${TEST_TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
 
     target_include_directories("${TEST_TARGET_NAME}"
         PRIVATE
@@ -157,7 +125,7 @@ function(ginkgo_create_hip_test test_name)
         # Only `math` requires it so far, but it's much easier
         # to put these this way.
         ${GINKGO_HIP_THRUST_PATH}
-        # Only `exception_helpers` requires thess so far, but it's much easier
+        # Only `exception_helpers` requires these so far, but it's much easier
         # to put these this way.
         ${HIPBLAS_INCLUDE_DIRS}
         ${hiprand_INCLUDE_DIRS}
@@ -166,18 +134,7 @@ function(ginkgo_create_hip_test test_name)
     set_target_properties(${TEST_TARGET_NAME} PROPERTIES
         OUTPUT_NAME ${test_name})
 
-    # Pass in the `--amdgpu-target` flags if asked
-    if(GINKGO_HIP_AMDGPU AND GINKGO_HIP_PLATFORM MATCHES "hcc")
-        foreach(target ${GINKGO_HIP_AMDGPU})
-            target_link_libraries(${TEST_TARGET_NAME} PRIVATE --amdgpu-target=${target})
-        endforeach()
-    endif()
-
-    # GINKGO_RPATH_FOR_HIP needs to be populated before calling this for the linker to include
-    # our libraries path into the executable's runpath.
     if(BUILD_SHARED_LIBS)
-        target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_RPATH_FOR_HIP}")
-
         if (GINKGO_CHECK_CIRCULAR_DEPS)
             target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
         endif()
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index c44367afe6d..d60ef65e6f1 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -178,7 +178,6 @@ endif()
 ginkgo_print_variable(${minimal_log} "GINKGO_BUILD_HWLOC")
 ginkgo_print_variable(${detailed_log} "GINKGO_BUILD_HWLOC")
 ginkgo_print_variable(${detailed_log} "HWLOC_VERSION")
-ginkgo_print_variable(${detailed_log} "GKO_HWLOC_XMLFILE")
 ginkgo_print_variable(${detailed_log} "HWLOC_LIBRARIES")
 ginkgo_print_variable(${detailed_log} "HWLOC_INCLUDE_DIRS")
 
diff --git a/cmake/hip_helpers.cmake b/cmake/hip_helpers.cmake
deleted file mode 100644
index c296ffc1228..00000000000
--- a/cmake/hip_helpers.cmake
+++ /dev/null
@@ -1,30 +0,0 @@
-macro(ginkgo_hip_ban_link_hcflag target)
-    if(TARGET ${target})
-        get_target_property(GINKGO_TARGET_ILL ${target} INTERFACE_LINK_LIBRARIES)
-        string(REPLACE "-hc " "" GINKGO_TARGET_NEW_ILL "${GINKGO_TARGET_ILL}")
-        set_target_properties(${target} PROPERTIES INTERFACE_LINK_LIBRARIES "${GINKGO_TARGET_NEW_ILL}")
-    endif()
-endmacro()
-
-macro(ginkgo_hip_ban_compile_hcflag target)
-    if(TARGET ${target})
-        get_target_property(GINKGO_TARGET_ILL ${target} INTERFACE_COMPILE_OPTIONS)
-        string(REPLACE "-hc" "" GINKGO_TARGET_NEW_ILL "${GINKGO_TARGET_ILL}")
-        set_target_properties(${target} PROPERTIES INTERFACE_COMPILE_OPTIONS "${GINKGO_TARGET_NEW_ILL}")
-    endif()
-endmacro()
-
-macro(ginkgo_hip_clang_ban_hip_device_flags)
-    if (GINKGO_HIP_VERSION VERSION_GREATER_EQUAL "3.5")
-        # Compile options somehow add hip-clang specific flags. Wipe them.
-        # Currently, the flags wiped out should be:
-        # -x;hip;--hip-device-lib-path=/opt/rocm/lib;--cuda-gpu-arch=gfx900;
-        # --cuda-gpu-arch=gfx906
-        set_target_properties(hip::device PROPERTIES INTERFACE_COMPILE_OPTIONS "")
-        # In addition, link libraries have a similar problem. We only keep
-        # `hip::host`. Currently, the flags should be:
-        # hip::host;--hip-device-lib-path=/opt/rocm/lib;--hip-link;
-        # --cuda-gpu-arch=gfx900;--cuda-gpu-arch=gfx906
-        set_target_properties(hip::device PROPERTIES INTERFACE_LINK_LIBRARIES "hip::host")
-    endif()
-endmacro()
diff --git a/cmake/information_helpers.cmake b/cmake/information_helpers.cmake
index e128fb5869a..e4f9033be50 100644
--- a/cmake/information_helpers.cmake
+++ b/cmake/information_helpers.cmake
@@ -1,11 +1,11 @@
 macro(ginkgo_interface_information)
     get_target_property(GINKGO_INTERFACE_LINK_LIBRARIES ginkgo INTERFACE_LINK_LIBRARIES)
-		set(GINKGO_INTERFACE_LINK_FLAGS "-L${CMAKE_INSTALL_PREFIX}/${GINKGO_INSTALL_LIBRARY_DIR} -lginkgo")
-		set(GINKGO_INTERFACE_CXX_FLAGS "-I${CMAKE_INSTALL_PREFIX}/${GINKGO_INSTALL_INCLUDE_DIR}")
+    set(GINKGO_INTERFACE_LINK_FLAGS "-L${CMAKE_INSTALL_PREFIX}/${GINKGO_INSTALL_LIBRARY_DIR} -lginkgo")
+    set(GINKGO_INTERFACE_CXX_FLAGS "-I${CMAKE_INSTALL_PREFIX}/${GINKGO_INSTALL_INCLUDE_DIR}")
 
-		foreach(_libs IN LISTS GINKGO_INTERFACE_LINK_LIBRARIES)
-			  set(GINKGO_INTERFACE_LINK_FLAGS "${GINKGO_INTERFACE_LINK_FLAGS} -l${_libs}")
-		endforeach()
+    foreach(_libs IN LISTS GINKGO_INTERFACE_LINK_LIBRARIES)
+        set(GINKGO_INTERFACE_LINK_FLAGS "${GINKGO_INTERFACE_LINK_FLAGS} -l${_libs}")
+    endforeach()
 endmacro(ginkgo_interface_information)
 
 macro(ginkgo_git_information)
diff --git a/cmake/install_helpers.cmake b/cmake/install_helpers.cmake
index 74ae6f8b6b2..7be6608e117 100644
--- a/cmake/install_helpers.cmake
+++ b/cmake/install_helpers.cmake
@@ -9,7 +9,19 @@ set(GINKGO_INSTALL_CONFIG_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Ginkgo")
 set(GINKGO_INSTALL_MODULE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Ginkgo/Modules")
 
 function(ginkgo_install_library name subdir)
-
+    if (BUILD_SHARED_LIBS)
+        set (HWLOC_LIB_PATH "")
+        if  (GINKGO_HAVE_HWLOC AND GINKGO_USE_EXTERNAL_HWLOC)
+            get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY)
+        endif()
+        if (APPLE)
+            set(ORIGIN_OR_LOADER_PATH "@loader_path")
+        else()
+            set(ORIGIN_OR_LOADER_PATH "$ORIGIN")
+        endif()
+        set_property(TARGET "${name}" PROPERTY INSTALL_RPATH
+        "${ORIGIN_OR_LOADER_PATH}" "${HWLOC_LIB_PATH}" "${ARGN}")
+    endif()
     if (WIN32 OR CYGWIN)
         # dll is considered as runtime
         install(TARGETS "${name}"
@@ -49,6 +61,21 @@ function(ginkgo_install)
             )
     endif()
 
+    if  (GINKGO_HAVE_HWLOC AND NOT GINKGO_USE_EXTERNAL_HWLOC)
+        get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY)
+        file(GLOB HWLOC_LIBS "${HWLOC_LIB_PATH}/libhwloc*")
+        install(FILES ${HWLOC_LIBS}
+            DESTINATION "${GINKGO_INSTALL_LIBRARY_DIR}"
+            )
+        # We only use hwloc and not netloc
+        install(DIRECTORY "${HWLOC_INCLUDE_DIRS}/hwloc"
+            DESTINATION "${GINKGO_INSTALL_INCLUDE_DIR}"
+            )
+        install(FILES "${HWLOC_INCLUDE_DIRS}/hwloc.h"
+            DESTINATION "${GINKGO_INSTALL_INCLUDE_DIR}"
+            )
+    endif()
+
     # export targets
     export(EXPORT Ginkgo
         NAMESPACE Ginkgo::
@@ -66,7 +93,7 @@ function(ginkgo_install)
         "${Ginkgo_BINARY_DIR}/GinkgoConfig.cmake"
         INSTALL_DESTINATION "${GINKGO_INSTALL_CONFIG_DIR}"
         )
-    set(HELPERS "hip_helpers.cmake" "windows_helpers.cmake")
+    set(HELPERS "windows_helpers.cmake")
     foreach (helper ${HELPERS})
         configure_file(${Ginkgo_SOURCE_DIR}/cmake/${helper}
             ${Ginkgo_BINARY_DIR}/${helper} COPYONLY)
@@ -82,12 +109,6 @@ function(ginkgo_install)
             DESTINATION "${GINKGO_INSTALL_CONFIG_DIR}"
             )
     endif()
-    if (GINKGO_BUILD_HIP)
-        install(FILES
-            "${Ginkgo_SOURCE_DIR}/cmake/hip_helpers.cmake"
-            DESTINATION "${GINKGO_INSTALL_CONFIG_DIR}"
-            )
-    endif()
     install(EXPORT Ginkgo
         NAMESPACE Ginkgo::
         FILE GinkgoTargets.cmake
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index a4f8594249f..eb73259c87f 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -1,4 +1,3 @@
-add_subdirectory(devices)  # basic device functionalities, always compiled
 add_subdirectory(device_hooks)  # placeholders for disabled modules
 
 add_library(ginkgo "")
@@ -69,8 +68,14 @@ if (GINKGO_HAVE_PAPI_SDE)
     target_link_libraries(ginkgo PUBLIC PAPI::PAPI)
 endif()
 
+# Since we have a public dependency on HIP, this dependency appears here as well
+set(RPATH_ADD_HIP_PATH "")
+if (GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc")
+    set(RPATH_ADD_HIP_PATH "${HIP_PATH}/lib")
+endif()
+
 ginkgo_default_includes(ginkgo)
-ginkgo_install_library(ginkgo core)
+ginkgo_install_library(ginkgo core "${RPATH_ADD_HIP_PATH}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo)
diff --git a/core/device_hooks/CMakeLists.txt b/core/device_hooks/CMakeLists.txt
index ff96f6e19f9..c653337ff3b 100644
--- a/core/device_hooks/CMakeLists.txt
+++ b/core/device_hooks/CMakeLists.txt
@@ -2,7 +2,7 @@ if(NOT GINKGO_BUILD_CUDA)
     add_library(ginkgo_cuda
         $<TARGET_OBJECTS:ginkgo_cuda_device>
         cuda_hooks.cpp)
-    target_link_libraries(ginkgo_cuda PUBLIC ginkgo_core_device)
+    target_link_libraries(ginkgo_cuda PRIVATE ginkgo_core_device)
     ginkgo_compile_features(ginkgo_cuda)
     ginkgo_default_includes(ginkgo_cuda)
     ginkgo_install_library(ginkgo_cuda cuda)
@@ -12,7 +12,7 @@ if (NOT GINKGO_BUILD_DPCPP)
     add_library(ginkgo_dpcpp
         $<TARGET_OBJECTS:ginkgo_dpcpp_device>
         dpcpp_hooks.cpp)
-    target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_core_device)
+    target_link_libraries(ginkgo_dpcpp PRIVATE ginkgo_core_device)
     ginkgo_compile_features(ginkgo_dpcpp)
     ginkgo_default_includes(ginkgo_dpcpp)
     ginkgo_install_library(ginkgo_dpcpp dpcpp)
@@ -22,7 +22,7 @@ if(NOT GINKGO_BUILD_HIP)
     add_library(ginkgo_hip
         $<TARGET_OBJECTS:ginkgo_hip_device>
         hip_hooks.cpp)
-    target_link_libraries(ginkgo_hip PUBLIC ginkgo_core_device)
+    target_link_libraries(ginkgo_hip PRIVATE ginkgo_core_device)
     ginkgo_compile_features(ginkgo_hip)
     ginkgo_default_includes(ginkgo_hip)
     ginkgo_install_library(ginkgo_hip hip)
@@ -33,9 +33,10 @@ if (NOT GINKGO_BUILD_OMP)
         $<TARGET_OBJECTS:ginkgo_omp_device>
         omp_hooks.cpp)
     ginkgo_compile_features(ginkgo_omp)
-    target_link_libraries(ginkgo_omp PUBLIC ginkgo_core_device ginkgo_cuda)
-    target_link_libraries(ginkgo_omp PUBLIC ginkgo_hip)
-    target_link_libraries(ginkgo_omp PUBLIC ginkgo_dpcpp)
+    target_link_libraries(ginkgo_omp PRIVATE ginkgo_cuda)
+    target_link_libraries(ginkgo_omp PRIVATE ginkgo_hip)
+    target_link_libraries(ginkgo_omp PRIVATE ginkgo_dpcpp)
+    target_link_libraries(ginkgo_omp PRIVATE ginkgo_core_device)
     ginkgo_default_includes(ginkgo_omp)
     ginkgo_install_library(ginkgo_omp omp)
 endif()
@@ -44,7 +45,7 @@ if (NOT GINKGO_BUILD_REFERENCE)
     add_library(ginkgo_reference
         $<TARGET_OBJECTS:ginkgo_reference_device>
         reference_hooks.cpp)
-    target_link_libraries(ginkgo_reference PUBLIC ginkgo_core_device)
+    target_link_libraries(ginkgo_reference PRIVATE ginkgo_core_device)
     ginkgo_compile_features(ginkgo_reference)
     ginkgo_default_includes(ginkgo_reference)
     ginkgo_install_library(ginkgo_reference reference)
diff --git a/core/devices/CMakeLists.txt b/devices/CMakeLists.txt
similarity index 93%
rename from core/devices/CMakeLists.txt
rename to devices/CMakeLists.txt
index 4502b7f9b93..4d6e018f80b 100644
--- a/core/devices/CMakeLists.txt
+++ b/devices/CMakeLists.txt
@@ -17,7 +17,7 @@ function(ginkgo_add_library name)
 endfunction()
 
 ginkgo_add_library(ginkgo_core_device machine_topology.cpp)
-ginkgo_install_library(ginkgo_core_device core/devices)
+ginkgo_install_library(ginkgo_core_device devices)
 
 add_subdirectory(cuda)
 add_subdirectory(dpcpp)
diff --git a/core/devices/cuda/CMakeLists.txt b/devices/cuda/CMakeLists.txt
similarity index 100%
rename from core/devices/cuda/CMakeLists.txt
rename to devices/cuda/CMakeLists.txt
diff --git a/core/devices/cuda/executor.cpp b/devices/cuda/executor.cpp
similarity index 100%
rename from core/devices/cuda/executor.cpp
rename to devices/cuda/executor.cpp
diff --git a/core/devices/dpcpp/CMakeLists.txt b/devices/dpcpp/CMakeLists.txt
similarity index 100%
rename from core/devices/dpcpp/CMakeLists.txt
rename to devices/dpcpp/CMakeLists.txt
diff --git a/core/devices/dpcpp/executor.cpp b/devices/dpcpp/executor.cpp
similarity index 100%
rename from core/devices/dpcpp/executor.cpp
rename to devices/dpcpp/executor.cpp
diff --git a/core/devices/hip/CMakeLists.txt b/devices/hip/CMakeLists.txt
similarity index 100%
rename from core/devices/hip/CMakeLists.txt
rename to devices/hip/CMakeLists.txt
diff --git a/core/devices/hip/executor.cpp b/devices/hip/executor.cpp
similarity index 100%
rename from core/devices/hip/executor.cpp
rename to devices/hip/executor.cpp
diff --git a/core/devices/machine_topology.cpp b/devices/machine_topology.cpp
similarity index 99%
rename from core/devices/machine_topology.cpp
rename to devices/machine_topology.cpp
index cc7a86f51d4..4028ad31dae 100644
--- a/core/devices/machine_topology.cpp
+++ b/devices/machine_topology.cpp
@@ -69,7 +69,6 @@ hwloc_topology *init_topology()
                                    HWLOC_TYPE_FILTER_KEEP_NONE);
     hwloc_topology_set_type_filter(tmp, HWLOC_OBJ_OS_DEVICE,
                                    HWLOC_TYPE_FILTER_KEEP_IMPORTANT);
-    hwloc_topology_set_xml(tmp, GKO_HWLOC_XMLFILE);
     hwloc_topology_load(tmp);
 
     return tmp;
diff --git a/core/devices/omp/CMakeLists.txt b/devices/omp/CMakeLists.txt
similarity index 100%
rename from core/devices/omp/CMakeLists.txt
rename to devices/omp/CMakeLists.txt
diff --git a/core/devices/omp/executor.cpp b/devices/omp/executor.cpp
similarity index 100%
rename from core/devices/omp/executor.cpp
rename to devices/omp/executor.cpp
diff --git a/core/devices/reference/CMakeLists.txt b/devices/reference/CMakeLists.txt
similarity index 100%
rename from core/devices/reference/CMakeLists.txt
rename to devices/reference/CMakeLists.txt
diff --git a/core/devices/reference/dummy.cpp b/devices/reference/dummy.cpp
similarity index 100%
rename from core/devices/reference/dummy.cpp
rename to devices/reference/dummy.cpp
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 9e340ba33fe..f9a226e8bcf 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -54,7 +54,7 @@ set(GINKGO_DPCPP_FLAGS ${GINKGO_COMPILER_FLAGS} -fsycl -fsycl-device-lib=libc)
 set(GINKGO_DPCPP_FLAGS ${GINKGO_DPCPP_FLAGS} PARENT_SCOPE)
 target_compile_options(ginkgo_dpcpp PRIVATE "${GINKGO_DPCPP_FLAGS}")
 target_compile_features(ginkgo_dpcpp PRIVATE cxx_std_17)
-target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_core_device)
+target_link_libraries(ginkgo_dpcpp PRIVATE ginkgo_core_device)
 
 ginkgo_default_includes(ginkgo_dpcpp)
 ginkgo_install_library(ginkgo_dpcpp dpcpp)
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 4e34da9466f..f6bbeaf12e2 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -44,23 +44,6 @@ if(NOT DEFINED HIPSPARSE_PATH)
     endif()
 endif()
 
-## Both the definition of `HCC_PATH` and `HIP_HIPCC_CMAKE_LINKER_HELPER` are required
-## before including `FindHIP`, as these are essential but not defined in the beginning
-## of the `FindHIP` file itself. Not defining these currently results in:
-## 1. Without `HCC_PATH`: the `hcc` backend not working properly if it is wrongly set,
-##    if it is not set, popentially all compilation could fail.
-## 2. Without `HIP_HIPCC_CMAKE_LINKER_HELPER` two compilations are required, since
-##    `FindHIP` defines this only in macro calls, which we call much later on after
-##    including the file itself.
-if(NOT DEFINED HCC_PATH)
-    if(DEFINED ENV{HCC_PATH})
-        set(HCC_PATH $ENV{HCC_PATH} CACHE PATH "Path to which HCC has been installed")
-    else()
-        set(HCC_PATH "${ROCM_PATH}/hcc" CACHE PATH "Path to which HCC has been installed")
-    endif()
-    set(HCC_HOME "${HCC_PATH}")
-endif()
-
 if(NOT DEFINED HIP_CLANG_PATH)
     if(NOT DEFINED ENV{HIP_CLANG_PATH})
         set(HIP_CLANG_PATH "${ROCM_PATH}/llvm/bin" CACHE PATH "Path to which HIP compatible clang binaries have been installed")
@@ -122,18 +105,11 @@ if (GINKGO_HIP_PLATFORM MATCHES "nvcc") # ensure ENV{CUDA_PATH} is set by the us
      endif()
 endif()
 
-if (GINKGO_HIP_PLATFORM STREQUAL "hcc")
-    # This is required by hipblas/hipsparse in the case where the platform is hcc.
-    # For nvcc platform, these aren't required and only cause trouble.
-    list(APPEND CMAKE_PREFIX_PATH
-        "${HIP_PATH}/lib/cmake"
-        "${HIP_PATH}/../lib/cmake" # hopefully catches all extra HIP dependencies, e.g. hcc
-    )
-endif()
-
-
 ## Setup all CMAKE variables to find HIP and its dependencies
 list(APPEND CMAKE_MODULE_PATH "${HIP_PATH}/cmake")
+if (GINKGO_HIP_PLATFORM STREQUAL "hcc")
+    list(APPEND CMAKE_PREFIX_PATH "${HIP_PATH}/lib/cmake")
+endif()
 list(APPEND CMAKE_PREFIX_PATH
     "${HIPBLAS_PATH}/lib/cmake"
     "${HIPRAND_PATH}/lib/cmake"
@@ -144,8 +120,7 @@ list(APPEND CMAKE_PREFIX_PATH
 set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" PARENT_SCOPE)
 set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH}" PARENT_SCOPE)
 
-# setting the default flags like CMAKE_{LANG}_FLAGS_{TYPE}
-# the setting is copied from the default CMAKE_CXX_FLAGS_{TYPE}
+# NOTE: without this, HIP jacobi build takes a *very* long time
 set(HIP_HIPCC_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}" CACHE STRING "Flags used by the HIPCC compiler during DEBUG builds")
 set(HIP_HIPCC_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}" CACHE STRING "Flags used by the HIPCC compiler during MINSIZEREL builds")
 set(HIP_HIPCC_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}" CACHE STRING "Flags used by the HIPCC compiler during RELEASE builds")
@@ -221,11 +196,11 @@ if (GINKGO_HIP_PLATFORM MATCHES "nvcc")
         unset(CMAKE_CUDA_HOST_COMPILER CACHE)
     endif()
     if (CMAKE_CUDA_HOST_COMPILER)
-        set(GINKGO_HIP_CUDA_HOST_COMPILER "-ccbin=${CMAKE_CUDA_HOST_COMPILER}")
+        list(APPEND GINKGO_HIP_NVCC_ADDITIONAL_FLAGS "-ccbin=${CMAKE_CUDA_HOST_COMPILER}")
     endif()
 
     # Remove false positive CUDA warnings when calling one<T>() and zero<T>()
-    set(GINKGO_HIP_NVCC_ADDITIONAL_FLAGS --expt-relaxed-constexpr)
+    list(APPEND GINKGO_HIP_NVCC_ADDITIONAL_FLAGS "--expt-relaxed-constexpr")
 
     if (GINKGO_HIP_PLATFROM MATCHES "nvcc" AND CMAKE_CUDA_COMPILER_VERSION
             MATCHES "9.2" AND CMAKE_CUDA_HOST_COMPILER MATCHES ".*clang.*" )
@@ -251,87 +226,62 @@ if(GINKGO_HIP_AMDGPU)
     endforeach()
 endif()
 
-set(GINKGO_HIPCC_OPTIONS ${GINKGO_HIP_COMPILER_FLAGS})
+set(GINKGO_HIPCC_OPTIONS ${GINKGO_HIP_COMPILER_FLAGS} "-std=c++14")
 set(GINKGO_HIP_NVCC_OPTIONS ${GINKGO_HIP_NVCC_COMPILER_FLAGS} ${GINKGO_HIP_NVCC_ARCH} ${GINKGO_HIP_NVCC_ADDITIONAL_FLAGS})
-set(GINKGO_HIP_HCC_OPTIONS ${GINKGO_HIP_HCC_COMPILER_FLAGS} ${GINKGO_AMD_ARCH_FLAGS})
 set(GINKGO_HIP_CLANG_OPTIONS ${GINKGO_HIP_CLANG_COMPILER_FLAGS} ${GINKGO_AMD_ARCH_FLAGS})
 
 set_source_files_properties(${GINKGO_HIP_SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
-if (GINKGO_HIP_VERSION VERSION_GREATER_EQUAL "3.5")
-    hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_hip_device> ${GINKGO_HIP_SOURCES}
-        HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS} "-std=c++14"
-        HCC_OPTIONS ${GINKGO_HIP_HCC_OPTIONS}
-        CLANG_OPTIONS ${GINKGO_HIP_CLANG_OPTIONS}
-        NVCC_OPTIONS ${GINKGO_HIP_NVCC_OPTIONS} ${GINKGO_HIP_CUDA_HOST_COMPILER}
-        ${GINKGO_STATIC_OR_SHARED})
-else()
-    hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_hip_device> ${GINKGO_HIP_SOURCES}
-        HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS} "-std=c++14"
-        HCC_OPTIONS ${GINKGO_HIP_HCC_OPTIONS}
-        NVCC_OPTIONS ${GINKGO_HIP_NVCC_OPTIONS} ${GINKGO_HIP_CUDA_HOST_COMPILER}
-        ${GINKGO_STATIC_OR_SHARED})
-endif()
+hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_hip_device> ${GINKGO_HIP_SOURCES}
+    HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS}
+    CLANG_OPTIONS ${GINKGO_HIP_CLANG_OPTIONS}
+    NVCC_OPTIONS ${GINKGO_HIP_NVCC_OPTIONS}
+    ${GINKGO_STATIC_OR_SHARED})
+
+target_include_directories(ginkgo_hip
+    PUBLIC
+        ${HIP_INCLUDE_DIRS}
+    PRIVATE
+        ${GINKGO_HIP_THRUST_PATH}
+        ${HIPBLAS_INCLUDE_DIRS}
+        ${hiprand_INCLUDE_DIRS}
+        ${HIPSPARSE_INCLUDE_DIRS}
+        $<BUILD_INTERFACE:${ROCPRIM_INCLUDE_DIRS}>)
 
+target_link_libraries(ginkgo_hip PRIVATE ginkgo_core_device)
+target_link_libraries(ginkgo_hip PRIVATE roc::hipblas roc::hipsparse hip::hiprand roc::rocrand)
 
 target_compile_options(ginkgo_hip PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${GINKGO_COMPILER_FLAGS}>)
-if(GINKGO_WITH_CLANG_TIDY AND GINKGO_CLANG_TIDY_PATH)
-    set_property(TARGET ginkgo_hip PROPERTY CXX_CLANG_TIDY "${GINKGO_CLANG_TIDY_PATH};-checks=*")
-endif()
-if(GINKGO_WITH_IWYU AND GINKGO_IWYU_PATH)
-    set_property(TARGET ginkgo_hip PROPERTY CXX_INCLUDE_WHAT_YOU_USE ${GINKGO_IWYU_PATH})
-endif()
 
 if(GINKGO_HIP_PLATFORM MATCHES "hcc")
-    # Fix the exception thrown bug with `hcc` backend and shared libraries
-    set_target_properties(ginkgo_hip PROPERTIES LINKER_LANGUAGE HIP)
-
-    # Ban `-hc` flag as INTERFACE_LINK_LIBRARIES since that is propagated when building
-    # a static library, and it's definitely not a known option to any compiler.
-    ginkgo_hip_ban_link_hcflag(hcc::hccrt)
-
-    if (NOT BUILD_SHARED_LIBS)
-        # Do not let hip::device flags propagate to executables which don't
-        # directly use HIP
-        ginkgo_hip_clang_ban_hip_device_flags()
+    find_package(hip REQUIRED)
+    # To save a bit of pain, we directly link against the `library` instead of
+    # linking against the target.
+    get_target_property(HIP_LIBAMDHIP64_CONFIGURATIONS hip::amdhip64 IMPORTED_CONFIGURATIONS)
+    if (CMAKE_BUILD_TYPE MATCHES "[Dd][Ee][Bb][Uu][Gg]" AND HIP_LIBAMDHIP64_CONFIGURATIONS MATCHES "[Dd][Ee][Bb][Uu][Gg]")
+        get_target_property(HIP_LIBAMDHIP64_LIBRARIES hip::amdhip64 IMPORTED_LOCATION_DEBUG)
+    else()
+        get_target_property(HIP_LIBAMDHIP64_LIBRARIES hip::amdhip64 IMPORTED_LOCATION_RELEASE)
     endif()
-    target_link_libraries(ginkgo_hip PRIVATE hip::device)
+    target_link_libraries(ginkgo_hip PUBLIC ${HIP_LIBAMDHIP64_LIBRARIES})
 elseif(GINKGO_HIP_PLATFORM MATCHES "nvcc")
     find_package(CUDA 9.2 REQUIRED)
     target_link_libraries(ginkgo_hip PUBLIC ${CUDA_LIBRARIES})
-    set(HIP_CUDA_LIBRARIES ${CUDA_LIBRARIES} PARENT_SCOPE)
 endif()
 
-target_link_libraries(ginkgo_hip PRIVATE roc::hipblas roc::hipsparse hip::hiprand roc::rocrand)
-
-target_link_libraries(ginkgo_hip PUBLIC ginkgo_core_device)
-
-target_include_directories(ginkgo_hip
-    PUBLIC
-        ${HIP_INCLUDE_DIRS}
-    PRIVATE
-        ${GINKGO_HIP_THRUST_PATH}
-        ${HIPBLAS_INCLUDE_DIRS}
-        ${hiprand_INCLUDE_DIRS}
-        ${HIPSPARSE_INCLUDE_DIRS}
-        $<BUILD_INTERFACE:${ROCPRIM_INCLUDE_DIRS}>)
+set(RPATH_ADD_HIP_PATH "")
+if (GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc")
+    set(RPATH_ADD_HIP_PATH "${HIP_PATH}/lib")
+endif()
 
 ginkgo_compile_features(ginkgo_hip)
 ginkgo_default_includes(ginkgo_hip)
-ginkgo_install_library(ginkgo_hip hip)
+ginkgo_install_library(ginkgo_hip hip "${RPATH_ADD_HIP_PATH}" "${HIPBLAS_PATH}/lib"
+    "${HIPRAND_PATH}/lib" "${HIPSPARSE_PATH}/lib" "${ROCRAND_PATH}/lib")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo_hip)
 endif()
 
 if(GINKGO_BUILD_TESTS)
-  # Here, we go through all of Ginkgo's dependencies to build a `-Wl,-rpath` string since for
-  # some reason `hipcc` through CMake does not have rpath settings unlike the other compilers.
-    get_target_property(GINKGO_LINK_LIBRARIES ginkgo LINK_LIBRARIES)
-    set(GINKGO_RPATH_FOR_HIP "-Wl,-rpath,$<TARGET_FILE_DIR:ginkgo>")
-    foreach(target ${GINKGO_LINK_LIBRARIES})
-        if("${target}" MATCHES "^ginkgo")
-            set(GINKGO_RPATH_FOR_HIP "${GINKGO_RPATH_FOR_HIP}:$<TARGET_FILE_DIR:${target}>")
-        endif()
-    endforeach()
     add_subdirectory(test)
 endif()
diff --git a/hip/get_info.cmake b/hip/get_info.cmake
index f93a8acac79..1610ac0eee4 100644
--- a/hip/get_info.cmake
+++ b/hip/get_info.cmake
@@ -1,15 +1,14 @@
 ginkgo_print_module_header(${detailed_log} "HIP")
 ginkgo_print_foreach_variable(${detailed_log}
     "GINKGO_HIPCONFIG_PATH;GINKGO_HIP_AMDGPU"
-    "GINKGO_HIP_HCC_COMPILER_FLAGS;GINKGO_HIP_NVCC_COMPILER_FLAGS"
+    "GINKGO_HIP_CLANG_COMPILER_FLAGS;GINKGO_HIP_NVCC_COMPILER_FLAGS"
     "GINKGO_HIP_THRUST_PATH;GINKGO_AMD_ARCH_FLAGS")
 ginkgo_print_module_footer(${detailed_log} "HIP variables:")
 ginkgo_print_foreach_variable(${detailed_log}
     "HIP_VERSION;HIP_COMPILER;HIP_PATH;ROCM_PATH"
-    "HIP_PLATFORM;HIP_ROOT_DIR;HCC_PATH;HIP_RUNTIME;HIPBLAS_PATH;HIPSPARSE_PATH"
+    "HIP_PLATFORM;HIP_ROOT_DIR;HIP_RUNTIME;HIPBLAS_PATH;HIPSPARSE_PATH"
     "HIPRAND_PATH;ROCRAND_PATH;HIP_CLANG_INCLUDE_PATH;HIP_CLANG_PATH"
     "HIP_HIPCC_EXECUTABLE;HIP_HIPCONFIG_EXECUTABLE;HIP_HOST_COMPILATION_CPP")
-ginkgo_print_flags(${detailed_log} "HIP_HCC_FLAGS")
 ginkgo_print_flags(${detailed_log} "HIP_HIPCC_FLAGS")
 ginkgo_print_flags(${detailed_log} "HIP_NVCC_FLAGS")
 ginkgo_print_flags(${detailed_log} "HIP_CLANG_FLAGS")
diff --git a/hip/test/factorization/CMakeLists.txt b/hip/test/factorization/CMakeLists.txt
index 5eb375638c3..998601d6b64 100644
--- a/hip/test/factorization/CMakeLists.txt
+++ b/hip/test/factorization/CMakeLists.txt
@@ -1,5 +1,5 @@
-ginkgo_create_hip_test_special_linkage(ic_kernels)
-ginkgo_create_hip_test_special_linkage(ilu_kernels)
+ginkgo_create_test(ic_kernels)
+ginkgo_create_test(ilu_kernels)
 ginkgo_create_hip_test(par_ic_kernels)
 ginkgo_create_hip_test(par_ict_kernels)
 ginkgo_create_hip_test(par_ilu_kernels)
diff --git a/hip/test/preconditioner/CMakeLists.txt b/hip/test/preconditioner/CMakeLists.txt
index 6f974174421..6769092fde5 100644
--- a/hip/test/preconditioner/CMakeLists.txt
+++ b/hip/test/preconditioner/CMakeLists.txt
@@ -1,2 +1,2 @@
-ginkgo_create_hip_test_special_linkage(jacobi_kernels)
+ginkgo_create_test(jacobi_kernels)
 ginkgo_create_hip_test(isai_kernels)
diff --git a/hip/test/solver/CMakeLists.txt b/hip/test/solver/CMakeLists.txt
index 1e36230d874..a69af95bd22 100644
--- a/hip/test/solver/CMakeLists.txt
+++ b/hip/test/solver/CMakeLists.txt
@@ -1,11 +1,11 @@
-ginkgo_create_hip_test_special_linkage(bicg_kernels)
-ginkgo_create_hip_test_special_linkage(bicgstab_kernels)
-ginkgo_create_hip_test_special_linkage(cg_kernels)
-ginkgo_create_hip_test_special_linkage(cgs_kernels)
-ginkgo_create_hip_test_special_linkage(fcg_kernels)
-ginkgo_create_hip_test_special_linkage(gmres_kernels)
-ginkgo_create_hip_test_special_linkage(cb_gmres_kernels)
-ginkgo_create_hip_test_special_linkage(idr_kernels)
-ginkgo_create_hip_test_special_linkage(ir_kernels)
-ginkgo_create_hip_test_special_linkage(lower_trs_kernels)
-ginkgo_create_hip_test_special_linkage(upper_trs_kernels)
+ginkgo_create_test(bicg_kernels)
+ginkgo_create_test(bicgstab_kernels)
+ginkgo_create_test(cg_kernels)
+ginkgo_create_test(cgs_kernels)
+ginkgo_create_test(fcg_kernels)
+ginkgo_create_test(gmres_kernels)
+ginkgo_create_test(cb_gmres_kernels)
+ginkgo_create_test(idr_kernels)
+ginkgo_create_test(ir_kernels)
+ginkgo_create_test(lower_trs_kernels)
+ginkgo_create_test(upper_trs_kernels)
diff --git a/hip/test/stop/CMakeLists.txt b/hip/test/stop/CMakeLists.txt
index 844f7037768..eb3fa564d29 100644
--- a/hip/test/stop/CMakeLists.txt
+++ b/hip/test/stop/CMakeLists.txt
@@ -1,2 +1,2 @@
 ginkgo_create_hip_test(criterion_kernels)
-ginkgo_create_hip_test_special_linkage(residual_norm_kernels)
+ginkgo_create_test(residual_norm_kernels)
diff --git a/include/ginkgo/config.hpp.in b/include/ginkgo/config.hpp.in
index e41862634f9..edef91e6d9e 100644
--- a/include/ginkgo/config.hpp.in
+++ b/include/ginkgo/config.hpp.in
@@ -77,14 +77,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_HAVE_PAPI_SDE @GINKGO_HAVE_PAPI_SDE@
 // clang-format on
 
+
 /* Is HWLOC available ? */
 // clang-format off
 #define GKO_HAVE_HWLOC @GINKGO_HAVE_HWLOC@
 // clang-format on
 
-/* Set the system provided XML config file*/
-// clang-format off
-#define GKO_HWLOC_XMLFILE "@GKO_HWLOC_XMLFILE@"
-// clang-format on
 
 #endif  // GKO_INCLUDE_CONFIG_H
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index 4fd677d5b5c..1c2e2407a30 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -43,7 +43,6 @@ target_sources(ginkgo_omp
 
 ginkgo_compile_features(ginkgo_omp)
 
-target_link_libraries(ginkgo_omp PUBLIC ginkgo_core_device)
 target_link_libraries(ginkgo_omp PRIVATE "${OpenMP_CXX_LIBRARIES}")
 target_include_directories(ginkgo_omp PRIVATE "${OpenMP_CXX_INCLUDE_DIRS}")
 # We first separate the arguments, otherwise, the target_compile_options adds it as a string
@@ -53,11 +52,12 @@ target_compile_options(ginkgo_omp PRIVATE "${OpenMP_SEP_FLAGS}")
 target_compile_options(ginkgo_omp PRIVATE "${GINKGO_COMPILER_FLAGS}")
 
 # Need to link against ginkgo_cuda for the `raw_copy_to(CudaExecutor ...)` method
-target_link_libraries(ginkgo_omp PUBLIC ginkgo_cuda)
+target_link_libraries(ginkgo_omp PRIVATE ginkgo_cuda)
 # Need to link against ginkgo_hip for the `raw_copy_to(HipExecutor ...)` method
-target_link_libraries(ginkgo_omp PUBLIC ginkgo_hip)
+target_link_libraries(ginkgo_omp PRIVATE ginkgo_hip)
 # Need to link against ginkgo_dpcpp for the `raw_copy_to(DpcppExecutor ...)` method
-target_link_libraries(ginkgo_omp PUBLIC ginkgo_dpcpp)
+target_link_libraries(ginkgo_omp PRIVATE ginkgo_dpcpp)
+target_link_libraries(ginkgo_omp PRIVATE ginkgo_core_device)
 
 ginkgo_default_includes(ginkgo_omp)
 ginkgo_install_library(ginkgo_omp omp)
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index b17b75ca8ba..fbcd833f626 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -39,7 +39,7 @@ target_sources(ginkgo_reference
     stop/criterion_kernels.cpp
     stop/residual_norm_kernels.cpp)
 
-target_link_libraries(ginkgo_reference PUBLIC ginkgo_core_device)
+target_link_libraries(ginkgo_reference PRIVATE ginkgo_core_device)
 ginkgo_compile_features(ginkgo_reference)
 ginkgo_default_includes(ginkgo_reference)
 ginkgo_install_library(ginkgo_reference reference)
diff --git a/test_exportbuild/CMakeLists.txt b/test_exportbuild/CMakeLists.txt
index 8cfb100263e..fd8ec8afc48 100644
--- a/test_exportbuild/CMakeLists.txt
+++ b/test_exportbuild/CMakeLists.txt
@@ -16,18 +16,3 @@ endif()
 add_executable(test_exportbuild ../test_install/test_install.cpp)
 target_compile_features(test_exportbuild PUBLIC cxx_std_14)
 target_link_libraries(test_exportbuild PRIVATE Ginkgo::ginkgo)
-
-if(GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc"
-        AND GINKGO_HIP_VERSION VERSION_GREATER_EQUAL "3.5"
-        AND NOT GINKGO_BUILD_SHARED_LIBS)
-    # Compile options somehow add hip-clang specific flags. Wipe them.
-    # Currently, the flags wiped out should be:
-    # -x;hip;--hip-device-lib-path=/opt/rocm/lib;--cuda-gpu-arch=gfx900;
-    # --cuda-gpu-arch=gfx906
-    set_target_properties(hip::device PROPERTIES INTERFACE_COMPILE_OPTIONS "")
-    # In addition, link libraries have a similar problem. We only keep
-    # `hip::host`. Currently, the flags should be:
-    # hip::host;--hip-device-lib-path=/opt/rocm/lib;--hip-link;
-    # --cuda-gpu-arch=gfx900;--cuda-gpu-arch=gfx906
-    set_target_properties(hip::device PROPERTIES INTERFACE_LINK_LIBRARIES "hip::host")
-endif()
diff --git a/test_install/CMakeLists.txt b/test_install/CMakeLists.txt
index 88d41bec448..6396b6ba88b 100644
--- a/test_install/CMakeLists.txt
+++ b/test_install/CMakeLists.txt
@@ -37,17 +37,25 @@ if(GINKGO_BUILD_CUDA)
     target_link_libraries(test_install_cuda PRIVATE Ginkgo::ginkgo)
 endif()
 
-if(GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc"
-        AND GINKGO_HIP_VERSION VERSION_GREATER_EQUAL "3.5"
-        AND NOT GINKGO_BUILD_SHARED_LIBS)
-    # Compile options somehow add hip-clang specific flags. Wipe them.
-    # Currently, the flags wiped out should be:
-    # -x;hip;--hip-device-lib-path=/opt/rocm/lib;--cuda-gpu-arch=gfx900;
-    # --cuda-gpu-arch=gfx906
-    set_target_properties(hip::device PROPERTIES INTERFACE_COMPILE_OPTIONS "")
-    # In addition, link libraries have a similar problem. We only keep
-    # `hip::host`. Currently, the flags should be:
-    # hip::host;--hip-device-lib-path=/opt/rocm/lib;--hip-link;
-    # --cuda-gpu-arch=gfx900;--cuda-gpu-arch=gfx906
-    set_target_properties(hip::device PROPERTIES INTERFACE_LINK_LIBRARIES "hip::host")
+if(GINKGO_BUILD_HIP)
+    find_package(HIP REQUIRED)
+
+    set_source_files_properties(test_install.hip.cpp PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
+    # The library was compiled with `-fPIC` when in shared library mode
+    if (GINKGO_BUILD_SHARED_LIBS)
+        set (GINKGO_PIC_OPTION "-fPIC")
+        set (GINKGO_CUDA_PIC_OPTION "-Xcompiler '-fPIC'")
+    endif()
+    hip_add_executable(test_install_hip test_install.hip.cpp
+        HIPCC_OPTIONS "-std=c++14"
+        CLANG_OPTIONS "${GINKGO_PIC_OPTION}"
+        NVCC_OPTIONS "${GINKGO_CUDA_PIC_OPTION}")
+
+    target_link_libraries(test_install_hip PRIVATE Ginkgo::ginkgo)
+
+    # If we always link with CXX there is no RPATH issue
+    set_target_properties(test_install_hip PROPERTIES LINKER_LANGUAGE CXX)
+
+    # Instead of using CXX, it's possible to instead use
+    # target_link_libraries(test_install_hip PRIVATE ${GINKGO_INSTALL_RPATH_FOR_HIP})
 endif()
diff --git a/test_install/test_install.hip.cpp b/test_install/test_install.hip.cpp
new file mode 100644
index 00000000000..79f33c5692c
--- /dev/null
+++ b/test_install/test_install.hip.cpp
@@ -0,0 +1,387 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/ginkgo.hpp>
+
+
+#include <hip/hip_runtime.h>
+
+
+#include <chrono>
+#include <iostream>
+#include <map>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+
+// core/base/polymorphic_object.hpp
+class PolymorphicObjectTest : public gko::PolymorphicObject {};
+
+
+int main(int, char **)
+{
+    auto refExec = gko::ReferenceExecutor::create();
+    auto hipExec = gko::HipExecutor::create(0, refExec);
+    // core/base/abstract_factory.hpp
+    {
+        using type1 = int;
+        using type2 = double;
+        static_assert(
+            std::is_same<
+                gko::AbstractFactory<type1, type2>::abstract_product_type,
+                type1>::value,
+            "abstract_factory.hpp not included properly!");
+    }
+
+    // core/base/array.hpp
+    {
+        using type1 = int;
+        using ArrayType = gko::Array<type1>;
+        ArrayType{};
+    }
+
+    // core/base/combination.hpp
+    {
+        using type1 = int;
+        static_assert(
+            std::is_same<gko::Combination<type1>::value_type, type1>::value,
+            "combination.hpp not included properly!");
+    }
+
+    // core/base/composition.hpp
+    {
+        using type1 = int;
+        static_assert(
+            std::is_same<gko::Composition<type1>::value_type, type1>::value,
+            "composition.hpp not included properly");
+    }
+
+    // core/base/dim.hpp
+    {
+        using type1 = int;
+        gko::dim<3, type1>{4, 4, 4};
+    }
+
+    // core/base/exception.hpp
+    {
+        gko::Error(std::string("file"), 12,
+                   std::string("Test for an error class."));
+    }
+
+    // core/base/exception_helpers.hpp
+    {
+        auto test = gko::dim<2>{3};
+        GKO_ASSERT_IS_SQUARE_MATRIX(test);
+    }
+
+    // core/base/executor.hpp
+    {
+        gko::ReferenceExecutor::create();
+    }
+
+    // core/base/math.hpp
+    {
+        using testType = double;
+        static_assert(gko::is_complex<testType>() == false,
+                      "math.hpp not included properly!");
+    }
+
+    // core/base/matrix_data.hpp
+    {
+        gko::matrix_data<>{};
+    }
+
+    // core/base/mtx_io.hpp
+    {
+        static_assert(gko::layout_type::array != gko::layout_type::coordinate,
+                      "mtx_io.hpp not included properly!");
+    }
+
+    // core/base/name_demangling.hpp
+    {
+        auto testVar = 3.0;
+        gko::name_demangling::get_static_type(testVar);
+    }
+
+
+    // core/base/polymorphic_object.hpp
+    {
+        gko::PolymorphicObject *test;
+        (void)test;  // silence unused variable warning
+    }
+
+    // core/base/range.hpp
+    {
+        gko::span{12};
+    }
+
+    // core/base/range_accessors.hpp
+    {
+        auto testVar = 12;
+        gko::range<gko::accessor::row_major<decltype(testVar), 2>>(&testVar, 1u,
+                                                                   1u, 1u);
+    }
+
+    // core/base/perturbation.hpp
+    {
+        using type1 = int;
+        static_assert(
+            std::is_same<gko::Perturbation<type1>::value_type, type1>::value,
+            "perturbation.hpp not included properly");
+    }
+
+    // core/base/std_extensions.hpp
+    {
+        static_assert(std::is_same<gko::xstd::void_t<double>, void>::value,
+                      "std_extensions.hpp not included properly!");
+    }
+
+    // core/base/types.hpp
+    {
+        static_assert(gko::size_type{12} == 12,
+                      "types.hpp not included properly");
+    }
+
+    // core/base/utils.hpp
+    {
+        gko::null_deleter<double>{};
+    }
+
+    // core/base/version.hpp
+    {
+        gko::version_info::get().header_version;
+    }
+
+    // core/factorization/par_ilu.hpp
+    {
+        gko::factorization::ParIlu<>::build().on(hipExec);
+    }
+
+    // core/log/convergence.hpp
+    {
+        gko::log::Convergence<>::create(hipExec);
+    }
+
+    // core/log/record.hpp
+    {
+        gko::log::executor_data{};
+    }
+
+    // core/log/stream.hpp
+    {
+        gko::log::Stream<>::create(hipExec);
+    }
+
+#if GKO_HAVE_PAPI_SDE
+    // core/log/papi.hpp
+    {
+        gko::log::Papi<>::create(hipExec);
+    }
+#endif  // GKO_HAVE_PAPI_SDE
+
+    // core/matrix/coo.hpp
+    {
+        using Mtx = gko::matrix::Coo<>;
+        Mtx::create(hipExec, gko::dim<2>{2, 2}, 2);
+    }
+
+    // core/matrix/csr.hpp
+    {
+        using Mtx = gko::matrix::Csr<>;
+        Mtx::create(hipExec, gko::dim<2>{2, 2}, 2,
+                    std::make_shared<Mtx::load_balance>(2));
+    }
+
+    // core/matrix/dense.hpp
+    {
+        using Mtx = gko::matrix::Dense<>;
+        Mtx::create(hipExec, gko::dim<2>{2, 2});
+    }
+
+    // core/matrix/ell.hpp
+    {
+        using Mtx = gko::matrix::Ell<>;
+        Mtx::create(hipExec, gko::dim<2>{2, 2}, 2);
+    }
+
+    // core/matrix/hybrid.hpp
+    {
+        using Mtx = gko::matrix::Hybrid<>;
+        Mtx::create(hipExec, gko::dim<2>{2, 2}, 2, 2, 1);
+    }
+
+    // core/matrix/identity.hpp
+    {
+        using Mtx = gko::matrix::Identity<>;
+        Mtx::create(hipExec);
+    }
+
+    // core/matrix/permutation.hpp
+    {
+        using Mtx = gko::matrix::Permutation<>;
+        Mtx::create(hipExec, gko::dim<2>{2, 2});
+    }
+
+    // core/matrix/sellp.hpp
+    {
+        using Mtx = gko::matrix::Sellp<>;
+        Mtx::create(hipExec, gko::dim<2>{2, 2}, 2);
+    }
+
+    // core/matrix/sparsity_csr.hpp
+    {
+        using Mtx = gko::matrix::SparsityCsr<>;
+        Mtx::create(hipExec, gko::dim<2>{2, 2});
+    }
+
+    // core/preconditioner/ilu.hpp
+    {
+        gko::preconditioner::Ilu<>::build().on(hipExec);
+    }
+
+    // core/preconditioner/jacobi.hpp
+    {
+        using Bj = gko::preconditioner::Jacobi<>;
+        Bj::build().with_max_block_size(1u).on(hipExec);
+    }
+
+    // core/solver/bicgstab.hpp
+    {
+        using Solver = gko::solver::Bicgstab<>;
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(hipExec))
+            .on(hipExec);
+    }
+
+    // core/solver/cg.hpp
+    {
+        using Solver = gko::solver::Cg<>;
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(hipExec))
+            .on(hipExec);
+    }
+
+    // core/solver/cgs.hpp
+    {
+        using Solver = gko::solver::Cgs<>;
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(hipExec))
+            .on(hipExec);
+    }
+
+    // core/solver/fcg.hpp
+    {
+        using Solver = gko::solver::Fcg<>;
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(hipExec))
+            .on(hipExec);
+    }
+
+    // core/solver/gmres.hpp
+    {
+        using Solver = gko::solver::Gmres<>;
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(hipExec))
+            .on(hipExec);
+    }
+
+    // core/solver/ir.hpp
+    {
+        using Solver = gko::solver::Ir<>;
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(hipExec))
+            .on(hipExec);
+    }
+
+    // core/solver/lower_trs.hpp
+    {
+        using Solver = gko::solver::LowerTrs<>;
+        Solver::build().on(hipExec);
+    }
+
+    // core/stop/
+    {
+        // iteration.hpp
+        auto iteration =
+            gko::stop::Iteration::build().with_max_iters(1u).on(hipExec);
+
+        // time.hpp
+        auto time = gko::stop::Time::build()
+                        .with_time_limit(std::chrono::milliseconds(10))
+                        .on(hipExec);
+
+        // residual_norm.hpp
+        gko::stop::ResidualNormReduction<>::build()
+            .with_reduction_factor(1e-10)
+            .on(hipExec);
+
+        gko::stop::ResidualNorm<>::build()
+            .with_reduction_factor(1e-10)
+            .with_baseline(gko::stop::mode::absolute)
+            .on(hipExec);
+
+        gko::stop::ImplicitResidualNorm<>::build()
+            .with_reduction_factor(1e-10)
+            .with_baseline(gko::stop::mode::absolute)
+            .on(hipExec);
+
+        gko::stop::RelativeResidualNorm<>::build().with_tolerance(1e-10).on(
+            hipExec);
+
+        gko::stop::AbsoluteResidualNorm<>::build().with_tolerance(1e-10).on(
+            hipExec);
+
+        // stopping_status.hpp
+        gko::stopping_status{};
+
+        // combined.hpp
+        auto combined =
+            gko::stop::Combined::build()
+                .with_criteria(std::move(time), std::move(iteration))
+                .on(hipExec);
+    }
+
+    std::cout
+        << "test_install_hip: the Ginkgo installation was correctly detected "
+           "and is complete."
+        << std::endl;
+
+    return 0;
+}
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index 71cd76defa6..507d12b3c18 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -13,7 +13,7 @@ if(GINKGO_BUILD_TESTS)
     ginkgo_add_subdirectory(GTest gtest)
 endif()
 
-if(GINKGO_HAVE_HWLOC)
+if(GINKGO_HAVE_HWLOC AND NOT GINKGO_USE_EXTERNAL_HWLOC)
     ginkgo_add_subdirectory(HWLOC hwloc)
 endif()
 
diff --git a/third_party/hwloc/CMakeLists.txt b/third_party/hwloc/CMakeLists.txt
index 9bf34b460a7..be1e057c91c 100644
--- a/third_party/hwloc/CMakeLists.txt
+++ b/third_party/hwloc/CMakeLists.txt
@@ -21,14 +21,14 @@ else()
 endif()
 
 if(MSVC OR WIN32)
+    add_library(hwloc STATIC IMPORTED GLOBAL)
     set(HWLOC_LIBRARIES "${TPL_HWLOC_PATH}/hwloc/hwloc-win32-build-2.3.0/lib/libhwloc.a" CACHE FILEPATH "The path to HWLOC library libhwloc.so" FORCE)
     set(HWLOC_INCLUDE_DIRS "${TPL_HWLOC_PATH}/hwloc/hwloc-win32-build-2.3.0/include" CACHE PATH "The directory containing the hwloc header, hwloc.h" FORCE)
 else()
     add_library(hwloc SHARED IMPORTED GLOBAL)
     add_dependencies(hwloc hwloc_external )
-    set(HWLOC_DIR "${TPL_HWLOC_PATH}" CACHE FILEPATH "The path to HWLOC")
     file(MAKE_DIRECTORY ${TPL_HWLOC_PATH}/lib/)
-    file(GLOB HWLOC_LIBS "${TPL_HWLOC_PATH}/build/hwloc/.libs/libhwloc*")
+    file(GLOB HWLOC_LIBS "${TPL_HWLOC_PATH}/build/hwloc/.libs/libhwloc.so*")
     configure_file("${TPL_HWLOC_PATH}/build/include/hwloc/autogen/config.h" "${TPL_HWLOC_PATH}/src/include/hwloc/autogen/config.h" COPYONLY)
     foreach(lib ${HWLOC_LIBS})
         get_filename_component(lib_name ${lib} NAME)
@@ -36,7 +36,7 @@ else()
     endforeach()
     set(HWLOC_LIBRARIES "${TPL_HWLOC_PATH}/lib/libhwloc.so" CACHE FILEPATH "The path to HWLOC library libhwloc.so" FORCE)
     set(HWLOC_INCLUDE_DIRS "${TPL_HWLOC_PATH}/src/include" CACHE PATH "The directory containing the hwloc header, hwloc.h" FORCE)
-    set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION ${HWLOC_LIBRARIES})
-    set_target_properties(hwloc PROPERTIES INTERFACE_LINK_LIBRARIES ${HWLOC_LIBRARIES})
-    set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${HWLOC_INCLUDE_DIRS}")
 endif()
+set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION ${HWLOC_LIBRARIES})
+set_target_properties(hwloc PROPERTIES INTERFACE_LINK_LIBRARIES ${HWLOC_LIBRARIES})
+set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${HWLOC_INCLUDE_DIRS}")

From fb4fe2cce7daf6b43b0fe43bac02521daa133a50 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 10 Mar 2021 09:19:12 +0100
Subject: [PATCH 0524/1008] Review update.

+ Change behavior and name of the ginkgo_core_device library (to
  ginkgo_device).
+ Update the documentation in INSTALL.md and README.md and in
  cmake/install_helpers.cmake.
+ Fix some code style issues.

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 INSTALL.md                        | 34 +++++++++++++++++++------------
 README.md                         |  8 ++++----
 cmake/Modules/FindHWLOC.cmake     |  2 +-
 cmake/install_helpers.cmake       |  7 +++++++
 core/CMakeLists.txt               |  2 +-
 core/device_hooks/CMakeLists.txt  | 10 ++++-----
 cuda/CMakeLists.txt               |  2 +-
 devices/CMakeLists.txt            |  4 ++--
 dpcpp/CMakeLists.txt              |  2 +-
 hip/CMakeLists.txt                |  6 ++++--
 omp/CMakeLists.txt                |  2 +-
 reference/CMakeLists.txt          |  2 +-
 test_install/test_install.cpp     |  1 -
 test_install/test_install.hip.cpp |  1 -
 test_install/test_install_cuda.cu |  1 -
 15 files changed, 49 insertions(+), 35 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index 7849e1c40d9..c2cb93aa37c 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -192,14 +192,21 @@ Ginkgo's HIP backend adds a dependency to the following packages:
 All HIP installation paths can be configured through the use of environment
 variables or CMake variables. This way of configuring the paths is currently
 imposed by the `HIP` tool suite. The variables are the following:
-+ CMake `-DHIP_PATH=` or  environment `export HIP_PATH=`: sets the `HIP`
-  installation path. The default value is `/opt/rocm/hip`.
-+ CMake `-DHIPBLAS_PATH=` or  environment `export HIPBLAS_PATH=`: sets the
-  `hipBLAS` installation path. The default value is `/opt/rocm/hipblas`.
-+ CMake `-DHIPSPARSE_PATH=` or  environment `export HIPSPARSE_PATH=`: sets the
-  `hipSPARSE` installation path. The default value is `/opt/rocm/hipsparse`.
-+ CMake `-DHCC_PATH=` or  environment `export HCC_PATH=`: sets the `HCC`
-  installation path, for AMD backends. The default value is `/opt/rocm/hcc`.
++ CMake `-DROCM_PATH=` or environment `export ROCM_PATH=`: sets the `ROCM`
+  installation path. The default value is `/opt/rocm/`.
++ CMake `-DHIP_CLANG_PATH` or environment `export HIP_CLANG_PATH=`: sets the
+  `HIP` compatible `clang` binary path. The default value is
+  `${ROCM_PATH}/llvm/bin`.
++ CMake `-DHIP_PATH=` or environment `export HIP_PATH=`: sets the `HIP`
+  installation path. The default value is `${ROCM_PATH}/hip`.
++ CMake `-DHIPBLAS_PATH=` or environment `export HIPBLAS_PATH=`: sets the
+  `hipBLAS` installation path. The default value is `${ROCM_PATH}/hipblas`.
++ CMake `-DHIPSPARSE_PATH=` or environment `export HIPSPARSE_PATH=`: sets the
+  `hipSPARSE` installation path. The default value is `${ROCM_PATH}/hipsparse`.
++ CMake `-DROCRAND_PATH=` or environment `export ROCRAND_PATH=`: sets the
+  `rocRAND` installation path. The default value is `${ROCM_PATH}/rocrand`.
++ CMake `-DHIPRAND_PATH=` or environment `export HIPRAND_PATH=`: sets the
+  `hipRAND` installation path. The default value is `${ROCM_PATH}/hiprand`.
 + environment `export CUDA_PATH=`: where `hipcc` can find `CUDA` if it is not in
   the default `/usr/local/cuda` path.
 
@@ -216,9 +223,9 @@ export HIP_PLATFORM=nvcc
 ```
 
 When using `HIP_PLATFORM=hcc`, note that two `HIP` compilers can be set, the old
-`hcc` or since ROCm 3.5, `clang`. On newer installations, `clang` should be set
-by default, but if encountering any problem try to manually set the `HIP`
-compiler to `clang`:
+`hcc` or since ROCm 3.5, `clang`. Ginkgo is only compatible with the `clang`
+based installations. Although this setting should be automatically done, it is
+also possible to manually set the `HIP` compiler to `clang`:
 ```
 export HIP_COMPILER=clang
 ```
@@ -227,7 +234,6 @@ export HIP_COMPILER=clang
 Platform specific compilation flags can be given through the following
 CMake variables:
 + `-DGINKGO_HIP_COMPILER_FLAGS=`: compilation flags given to all platforms.
-+ `-DGINKGO_HIP_HCC_COMPILER_FLAGS=`: compilation flags given to AMD platforms.
 + `-DGINKGO_HIP_NVCC_COMPILER_FLAGS=`: compilation flags given to NVIDIA platforms.
 + `-DGINKGO_HIP_CLANG_COMPILER_FLAGS=`: compilation flags given to AMD clang compiler.
 
@@ -257,7 +263,9 @@ packages `GTEST`, `GFLAGS`, `RAPIDJSON` and `CAS`, it is possible to force
 Ginkgo to try to use an external version of a package. For this, Ginkgo provides
 two ways to find packages. To rely on the CMake `find_package` command, use the
 CMake option `-DGINKGO_USE_EXTERNAL_<package>=ON`. `HWLOC` works the opposite
-way, Ginkgo always looks for the system's `hwloc` first.
+way, Ginkgo always looks for the system's `hwloc` first. In addition, when
+installing Ginkgo, `hwloc` will be installed as well into the Ginkgo directory
+if there were no system `hwloc` detected.
 
 Note that, if the external packages were not installed to the default location,
 the CMake option `-DCMAKE_PREFIX_PATH=<path-list>` needs to be set to the
diff --git a/README.md b/README.md
index 3012dc5dd7b..ef045569249 100644
--- a/README.md
+++ b/README.md
@@ -61,10 +61,10 @@ following:
 
 The Ginkgo HIP module has the following __additional__ requirements:
 
-* _ROCm 2.8+_
-*    the HIP, hipBLAS and hipSPARSE packages compiled with either:
-    * _AMD_ backend
-    * _CUDA 9.0+_ backend. When using CUDA 10+, _cmake 3.12.2+_ is required.
+* _ROCm 3.5+_
+*    the HIP, hipBLAS, hipSPARSE, hip/rocRAND packages compiled with either:
+    * _AMD_ backend (using the `clang` compiler)
+    * _CUDA 9.2+_ backend. When using CUDA 10+, _cmake 3.12.2+_ is required.
 
 ### Windows
 
diff --git a/cmake/Modules/FindHWLOC.cmake b/cmake/Modules/FindHWLOC.cmake
index 4d6eddff5f2..7f8bb2ba36d 100644
--- a/cmake/Modules/FindHWLOC.cmake
+++ b/cmake/Modules/FindHWLOC.cmake
@@ -51,7 +51,7 @@ find_library(HWLOC_LIBRARIES "hwloc"
     )
 
 if (HWLOC_INCLUDE_DIRS)
-        # Find the version of hwloc found
+    # Find the version of hwloc found
     if(NOT HWLOC_VERSION)
         file(READ "${HWLOC_INCLUDE_DIRS}/hwloc.h"
             HEADER_CONTENTS LIMIT 16384)
diff --git a/cmake/install_helpers.cmake b/cmake/install_helpers.cmake
index 7be6608e117..4060c720d10 100644
--- a/cmake/install_helpers.cmake
+++ b/cmake/install_helpers.cmake
@@ -9,6 +9,13 @@ set(GINKGO_INSTALL_CONFIG_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Ginkgo")
 set(GINKGO_INSTALL_MODULE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Ginkgo/Modules")
 
 function(ginkgo_install_library name subdir)
+    # This setting sets the directory of the library as the `INSTALL_RPATH`.
+    # This is required for `libginkgo.so` to find its dependencies such as
+    # `libginkgo_omp.so` etc. Note that with this setting it's not possible on
+    # Linux to control which Ginkgo libraries are found and where: if all the
+    # libraries are present in the directory, these will be used first and
+    # foremost and the environment variable `LD_LIBRARY_PATH` will have no
+    # effect.
     if (BUILD_SHARED_LIBS)
         set (HWLOC_LIB_PATH "")
         if  (GINKGO_HAVE_HWLOC AND GINKGO_USE_EXTERNAL_HWLOC)
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index eb73259c87f..9b85b0324c5 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -62,7 +62,7 @@ target_compile_options(ginkgo PRIVATE "${GINKGO_COMPILER_FLAGS}")
 # regardless of whether it is installed or added as a subdirectory
 add_library(Ginkgo::ginkgo ALIAS ginkgo)
 target_link_libraries(ginkgo
-    PUBLIC ginkgo_core_device ginkgo_omp ginkgo_cuda ginkgo_reference ginkgo_hip ginkgo_dpcpp)
+    PUBLIC ginkgo_device ginkgo_omp ginkgo_cuda ginkgo_reference ginkgo_hip ginkgo_dpcpp)
 # The PAPI dependency needs to be exposed to the user.
 if (GINKGO_HAVE_PAPI_SDE)
     target_link_libraries(ginkgo PUBLIC PAPI::PAPI)
diff --git a/core/device_hooks/CMakeLists.txt b/core/device_hooks/CMakeLists.txt
index c653337ff3b..698391de20a 100644
--- a/core/device_hooks/CMakeLists.txt
+++ b/core/device_hooks/CMakeLists.txt
@@ -2,7 +2,7 @@ if(NOT GINKGO_BUILD_CUDA)
     add_library(ginkgo_cuda
         $<TARGET_OBJECTS:ginkgo_cuda_device>
         cuda_hooks.cpp)
-    target_link_libraries(ginkgo_cuda PRIVATE ginkgo_core_device)
+    target_link_libraries(ginkgo_cuda PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_cuda)
     ginkgo_default_includes(ginkgo_cuda)
     ginkgo_install_library(ginkgo_cuda cuda)
@@ -12,7 +12,7 @@ if (NOT GINKGO_BUILD_DPCPP)
     add_library(ginkgo_dpcpp
         $<TARGET_OBJECTS:ginkgo_dpcpp_device>
         dpcpp_hooks.cpp)
-    target_link_libraries(ginkgo_dpcpp PRIVATE ginkgo_core_device)
+    target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_dpcpp)
     ginkgo_default_includes(ginkgo_dpcpp)
     ginkgo_install_library(ginkgo_dpcpp dpcpp)
@@ -22,7 +22,7 @@ if(NOT GINKGO_BUILD_HIP)
     add_library(ginkgo_hip
         $<TARGET_OBJECTS:ginkgo_hip_device>
         hip_hooks.cpp)
-    target_link_libraries(ginkgo_hip PRIVATE ginkgo_core_device)
+    target_link_libraries(ginkgo_hip PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_hip)
     ginkgo_default_includes(ginkgo_hip)
     ginkgo_install_library(ginkgo_hip hip)
@@ -36,7 +36,7 @@ if (NOT GINKGO_BUILD_OMP)
     target_link_libraries(ginkgo_omp PRIVATE ginkgo_cuda)
     target_link_libraries(ginkgo_omp PRIVATE ginkgo_hip)
     target_link_libraries(ginkgo_omp PRIVATE ginkgo_dpcpp)
-    target_link_libraries(ginkgo_omp PRIVATE ginkgo_core_device)
+    target_link_libraries(ginkgo_omp PUBLIC ginkgo_device)
     ginkgo_default_includes(ginkgo_omp)
     ginkgo_install_library(ginkgo_omp omp)
 endif()
@@ -45,7 +45,7 @@ if (NOT GINKGO_BUILD_REFERENCE)
     add_library(ginkgo_reference
         $<TARGET_OBJECTS:ginkgo_reference_device>
         reference_hooks.cpp)
-    target_link_libraries(ginkgo_reference PRIVATE ginkgo_core_device)
+    target_link_libraries(ginkgo_reference PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_reference)
     ginkgo_default_includes(ginkgo_reference)
     ginkgo_install_library(ginkgo_reference reference)
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 5e4505d8187..8d96766371b 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -160,7 +160,7 @@ ginkgo_compile_features(ginkgo_cuda)
 target_include_directories(ginkgo_cuda
     SYSTEM PRIVATE ${CUDA_INCLUDE_DIRS})
 target_link_libraries(ginkgo_cuda PRIVATE ${CUDA_RUNTIME_LIBS} ${CUBLAS} ${CUSPARSE} ${CURAND})
-target_link_libraries(ginkgo_cuda PRIVATE ginkgo_core_device)
+target_link_libraries(ginkgo_cuda PUBLIC ginkgo_device)
 target_compile_options(ginkgo_cuda
         PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_ARCH_FLAGS}>")
 
diff --git a/devices/CMakeLists.txt b/devices/CMakeLists.txt
index 4d6e018f80b..733a10d1fa8 100644
--- a/devices/CMakeLists.txt
+++ b/devices/CMakeLists.txt
@@ -16,8 +16,8 @@ function(ginkgo_add_library name)
     set_target_properties(${name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endfunction()
 
-ginkgo_add_library(ginkgo_core_device machine_topology.cpp)
-ginkgo_install_library(ginkgo_core_device devices)
+ginkgo_add_library(ginkgo_device machine_topology.cpp)
+ginkgo_install_library(ginkgo_device devices)
 
 add_subdirectory(cuda)
 add_subdirectory(dpcpp)
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index f9a226e8bcf..89600b4aa6b 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -54,7 +54,7 @@ set(GINKGO_DPCPP_FLAGS ${GINKGO_COMPILER_FLAGS} -fsycl -fsycl-device-lib=libc)
 set(GINKGO_DPCPP_FLAGS ${GINKGO_DPCPP_FLAGS} PARENT_SCOPE)
 target_compile_options(ginkgo_dpcpp PRIVATE "${GINKGO_DPCPP_FLAGS}")
 target_compile_features(ginkgo_dpcpp PRIVATE cxx_std_17)
-target_link_libraries(ginkgo_dpcpp PRIVATE ginkgo_core_device)
+target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
 
 ginkgo_default_includes(ginkgo_dpcpp)
 ginkgo_install_library(ginkgo_dpcpp dpcpp)
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index f6bbeaf12e2..1500596ff31 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -120,7 +120,9 @@ list(APPEND CMAKE_PREFIX_PATH
 set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" PARENT_SCOPE)
 set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH}" PARENT_SCOPE)
 
-# NOTE: without this, HIP jacobi build takes a *very* long time
+# NOTE: without this, HIP jacobi build takes a *very* long time. The reason for
+# that is that these variables are seemingly empty by default, thus there is no
+# proper optimization applied to the HIP builds otherwise.
 set(HIP_HIPCC_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}" CACHE STRING "Flags used by the HIPCC compiler during DEBUG builds")
 set(HIP_HIPCC_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}" CACHE STRING "Flags used by the HIPCC compiler during MINSIZEREL builds")
 set(HIP_HIPCC_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}" CACHE STRING "Flags used by the HIPCC compiler during RELEASE builds")
@@ -247,7 +249,7 @@ target_include_directories(ginkgo_hip
         ${HIPSPARSE_INCLUDE_DIRS}
         $<BUILD_INTERFACE:${ROCPRIM_INCLUDE_DIRS}>)
 
-target_link_libraries(ginkgo_hip PRIVATE ginkgo_core_device)
+target_link_libraries(ginkgo_hip PUBLIC ginkgo_device)
 target_link_libraries(ginkgo_hip PRIVATE roc::hipblas roc::hipsparse hip::hiprand roc::rocrand)
 
 target_compile_options(ginkgo_hip PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${GINKGO_COMPILER_FLAGS}>)
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index 1c2e2407a30..64349b24c6b 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -57,7 +57,7 @@ target_link_libraries(ginkgo_omp PRIVATE ginkgo_cuda)
 target_link_libraries(ginkgo_omp PRIVATE ginkgo_hip)
 # Need to link against ginkgo_dpcpp for the `raw_copy_to(DpcppExecutor ...)` method
 target_link_libraries(ginkgo_omp PRIVATE ginkgo_dpcpp)
-target_link_libraries(ginkgo_omp PRIVATE ginkgo_core_device)
+target_link_libraries(ginkgo_omp PUBLIC ginkgo_device)
 
 ginkgo_default_includes(ginkgo_omp)
 ginkgo_install_library(ginkgo_omp omp)
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index fbcd833f626..d379dac167e 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -39,7 +39,7 @@ target_sources(ginkgo_reference
     stop/criterion_kernels.cpp
     stop/residual_norm_kernels.cpp)
 
-target_link_libraries(ginkgo_reference PRIVATE ginkgo_core_device)
+target_link_libraries(ginkgo_reference PUBLIC ginkgo_device)
 ginkgo_compile_features(ginkgo_reference)
 ginkgo_default_includes(ginkgo_reference)
 ginkgo_install_library(ginkgo_reference reference)
diff --git a/test_install/test_install.cpp b/test_install/test_install.cpp
index e15518d5acc..f55fef3fd0a 100644
--- a/test_install/test_install.cpp
+++ b/test_install/test_install.cpp
@@ -129,7 +129,6 @@ int main(int, char **)
         auto test = gko::name_demangling::get_static_type(testVar);
     }
 
-
     // core/base/polymorphic_object.hpp
     {
         auto test = gko::layout_type::array;
diff --git a/test_install/test_install.hip.cpp b/test_install/test_install.hip.cpp
index 79f33c5692c..65708b0cffa 100644
--- a/test_install/test_install.hip.cpp
+++ b/test_install/test_install.hip.cpp
@@ -134,7 +134,6 @@ int main(int, char **)
         gko::name_demangling::get_static_type(testVar);
     }
 
-
     // core/base/polymorphic_object.hpp
     {
         gko::PolymorphicObject *test;
diff --git a/test_install/test_install_cuda.cu b/test_install/test_install_cuda.cu
index 5158e3f00fa..1c404236110 100644
--- a/test_install/test_install_cuda.cu
+++ b/test_install/test_install_cuda.cu
@@ -131,7 +131,6 @@ int main(int, char **)
         gko::name_demangling::get_static_type(testVar);
     }
 
-
     // core/base/polymorphic_object.hpp
     {
         gko::PolymorphicObject *test;

From b0fb791d0e3c80808c5e0f270227c1d4cccaf656 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Thu, 11 Mar 2021 16:50:16 +0100
Subject: [PATCH 0525/1008] Pass the HIP options to tests without the NVCC bug

---
 cmake/create_test.cmake | 27 ++++++++++++++++++++-------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index dda929ca4f7..817891b68ed 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -108,13 +108,26 @@ function(ginkgo_create_hip_test test_name)
     string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
 
     set_source_files_properties(${test_name}.hip.cpp PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
-    hip_add_executable(${TEST_TARGET_NAME} ${test_name}.hip.cpp
-        # With how HIP works, to prevent redifinition we need to comment these.
-        # They might need to be used later though.
-        #HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS}
-        #NVCC_OPTIONS  ${GINKGO_HIP_NVCC_OPTIONS}
-        #CLANG_OPTIONS ${GINKGO_HIP_CLANG_OPTIONS}
-        )
+
+    # NOTE: With how HIP works, passing the flags `HIPCC_OPTIONS` etc. here
+    # creates a redefinition of all flags. This creates some issues with `nvcc`
+    # (particularly the flags in `HIPCC_OPTIONS`), but `clang` is fine with the
+    # redefinitions.
+    if (GINKGO_HIP_PLATFORM STREQUAL "nvcc")
+        hip_add_executable(${TEST_TARGET_NAME} ${test_name}.hip.cpp
+            # If `FindHIP.cmake`, namely `HIP_PARSE_HIPCC_OPTIONS` macro and
+            # call gets fixed, uncomment this.
+            # HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS}
+            # NVCC_OPTIONS  ${GINKGO_HIP_NVCC_OPTIONS}
+            # CLANG_OPTIONS ${GINKGO_HIP_CLANG_OPTIONS}
+            )
+    else() # hcc/clang
+        hip_add_executable(${TEST_TARGET_NAME} ${test_name}.hip.cpp
+            HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS}
+            NVCC_OPTIONS  ${GINKGO_HIP_NVCC_OPTIONS}
+            CLANG_OPTIONS ${GINKGO_HIP_CLANG_OPTIONS}
+            )
+    endif()
 
     # Let's use a normal compiler for linking
     set_target_properties(${TEST_TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)

From 278dbb212ef15c4339410e3c04f55c16dd36f79a Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Mon, 22 Mar 2021 20:51:46 +0100
Subject: [PATCH 0526/1008] Add three new options to control RPATH behavior.

---
 CMakeLists.txt              |  3 +++
 INSTALL.md                  |  9 +++++++++
 cmake/install_helpers.cmake | 39 +++++++++++++++++++++----------------
 core/CMakeLists.txt         | 12 ++++++++----
 cuda/CMakeLists.txt         |  4 +++-
 hip/CMakeLists.txt          | 16 ++++++++++-----
 6 files changed, 56 insertions(+), 27 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9027c2e466a..7b14f0a08d4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -72,6 +72,9 @@ set(GINKGO_HIP_AMDGPU "" CACHE STRING
 option(GINKGO_JACOBI_FULL_OPTIMIZATIONS "Use all the optimizations for the CUDA Jacobi algorithm" OFF)
 option(BUILD_SHARED_LIBS "Build shared (.so, .dylib, .dll) libraries" ON)
 option(GINKGO_BUILD_HWLOC "Build Ginkgo with HWLOC. Default is ON. If a system HWLOC is not found, then we try to build it ourselves. Switch this OFF to disable HWLOC." ON)
+option(GINKGO_INSTALL_RPATH "Set the RPATH when installing its libraries." ON)
+option(GINKGO_INSTALL_RPATH_ORIGIN "Add $ORIGIN (Linux) or @loader_path (MacOS) to the installation RPATH." ON)
+option(GINKGO_INSTALL_RPATH_DEPENDENCIES "Add dependencies to the installation RPATH." OFF)
 
 set(GINKGO_CIRCULAR_DEPS_FLAGS "-Wl,--no-undefined")
 
diff --git a/INSTALL.md b/INSTALL.md
index c2cb93aa37c..b015b7bf020 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -65,6 +65,12 @@ Ginkgo adds the following additional switches to control what is being built:
 *   `-DGINKGO_VERBOSE_LEVEL=integer` sets the verbosity of Ginkgo.
     * `0` disables all output in the main libraries,
     * `1` enables a few important messages related to unexpected behavior (default).
+*   `GINKGO_INSTALL_RPATH` sets the RPATH when installing the Ginkgo libraries.
+    The default is `ON`.
+*   `GINKGO_INSTALL_RPATH_ORIGIN` adds $ORIGIN (Linux) or @loader_path (MacOS)
+    to the installation RPATH. The default is `ON`.
+*   `GINKGO_INSTALL_RPATH_DEPENDENCIES` adds the dependencies to the
+    installation RPATH. The default is `OFF`.
 *   `-DCMAKE_INSTALL_PREFIX=path` sets the installation path for `make install`.
     The default value is usually something like `/usr/local`.
 *   `-DCMAKE_BUILD_TYPE=type` specifies which configuration will be used for
@@ -285,6 +291,9 @@ format):
 When applicable (e.g. for `GTest` libraries), a `;` separated list can be given
 to the `TPL_<package>_{LIBRARIES|INCLUDE_DIRS}` variables.
 
+Note that for convenience, the options `GINKGO_INSTALL_RPATH[_.*]` can be used
+to make bind the Ginkgo shared libraries to the path of its dependencies.
+
 ### Installing Ginkgo
 
 To install Ginkgo into the specified folder, execute the following command in
diff --git a/cmake/install_helpers.cmake b/cmake/install_helpers.cmake
index 4060c720d10..a831af54ebc 100644
--- a/cmake/install_helpers.cmake
+++ b/cmake/install_helpers.cmake
@@ -8,27 +8,32 @@ set(GINKGO_INSTALL_PKGCONFIG_DIR "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 set(GINKGO_INSTALL_CONFIG_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Ginkgo")
 set(GINKGO_INSTALL_MODULE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Ginkgo/Modules")
 
-function(ginkgo_install_library name subdir)
-    # This setting sets the directory of the library as the `INSTALL_RPATH`.
-    # This is required for `libginkgo.so` to find its dependencies such as
-    # `libginkgo_omp.so` etc. Note that with this setting it's not possible on
-    # Linux to control which Ginkgo libraries are found and where: if all the
-    # libraries are present in the directory, these will be used first and
-    # foremost and the environment variable `LD_LIBRARY_PATH` will have no
-    # effect.
+function(ginkgo_add_install_rpath name subdir)
     if (BUILD_SHARED_LIBS)
-        set (HWLOC_LIB_PATH "")
-        if  (GINKGO_HAVE_HWLOC AND GINKGO_USE_EXTERNAL_HWLOC)
-            get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY)
+        if (GINKGO_INSTALL_RPATH_ORIGIN)
+            if (APPLE)
+                set(ORIGIN_OR_LOADER_PATH "@loader_path")
+            else()
+                set(ORIGIN_OR_LOADER_PATH "$ORIGIN")
+            endif()
+        endif()
+        if (GINKGO_INSTALL_RPATH_DEPENDENCIES)
+            set(RPATH_DEPENDENCIES "${ARGN}")
+            if  (GINKGO_HAVE_HWLOC AND GINKGO_USE_EXTERNAL_HWLOC)
+                get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY)
+                list(APPEND RPATH_DEPENDENCIES "${HWLOC_LIBRARIES}")
+            endif()
         endif()
-        if (APPLE)
-            set(ORIGIN_OR_LOADER_PATH "@loader_path")
-        else()
-            set(ORIGIN_OR_LOADER_PATH "$ORIGIN")
+        if (GINKGO_INSTALL_RPATH)
+            set_property(TARGET "${name}" PROPERTY INSTALL_RPATH
+                "${ORIGIN_OR_LOADER_PATH}" "${RPATH_DEPENDENCIES}")
         endif()
-        set_property(TARGET "${name}" PROPERTY INSTALL_RPATH
-        "${ORIGIN_OR_LOADER_PATH}" "${HWLOC_LIB_PATH}" "${ARGN}")
     endif()
+endfunction()
+
+function(ginkgo_install_library name subdir)
+    ginkgo_add_install_rpath("${name}" "${subdir}" "${ARGN}")
+
     if (WIN32 OR CYGWIN)
         # dll is considered as runtime
         install(TARGETS "${name}"
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 9b85b0324c5..93c878e6983 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -64,18 +64,22 @@ add_library(Ginkgo::ginkgo ALIAS ginkgo)
 target_link_libraries(ginkgo
     PUBLIC ginkgo_device ginkgo_omp ginkgo_cuda ginkgo_reference ginkgo_hip ginkgo_dpcpp)
 # The PAPI dependency needs to be exposed to the user.
+set(GKO_RPATH_ADDITIONS "")
 if (GINKGO_HAVE_PAPI_SDE)
     target_link_libraries(ginkgo PUBLIC PAPI::PAPI)
+    list(GET PAPI_LIBRARIES 0 PAPI_FIRST_LIB)
+    get_filename_component(GKO_PAPI_LIBDIR "${PAPI_FIRST_LIB}" DIRECTORY)
+    list(APPEND GKO_RPATH_ADDITIONS "${GKO_PAPI_LIBDIR}")
 endif()
 
-# Since we have a public dependency on HIP, this dependency appears here as well
-set(RPATH_ADD_HIP_PATH "")
+# Since we have a public dependency on HIP, this dependency appears
+# here as well
 if (GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc")
-    set(RPATH_ADD_HIP_PATH "${HIP_PATH}/lib")
+    list(APPEND GKO_RPATH_ADDITIONS "${HIP_PATH}/lib")
 endif()
 
 ginkgo_default_includes(ginkgo)
-ginkgo_install_library(ginkgo core "${RPATH_ADD_HIP_PATH}")
+ginkgo_install_library(ginkgo core "${GKO_RPATH_ADDITIONS}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo)
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 8d96766371b..4748dfdf278 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -163,9 +163,11 @@ target_link_libraries(ginkgo_cuda PRIVATE ${CUDA_RUNTIME_LIBS} ${CUBLAS} ${CUSPA
 target_link_libraries(ginkgo_cuda PUBLIC ginkgo_device)
 target_compile_options(ginkgo_cuda
         PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_ARCH_FLAGS}>")
+list(GET CUDA_RUNTIME_LIBS 0 CUDA_FIRST_LIB)
+get_filename_component(GKO_CUDA_LIBDIR "${CUDA_FIRST_LIB}" DIRECTORY)
 
 ginkgo_default_includes(ginkgo_cuda)
-ginkgo_install_library(ginkgo_cuda cuda)
+ginkgo_install_library(ginkgo_cuda cuda "${GKO_CUDA_LIBDIR}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo_cuda)
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 1500596ff31..3c349a5a699 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -270,15 +270,21 @@ elseif(GINKGO_HIP_PLATFORM MATCHES "nvcc")
     target_link_libraries(ginkgo_hip PUBLIC ${CUDA_LIBRARIES})
 endif()
 
-set(RPATH_ADD_HIP_PATH "")
-if (GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc")
-    set(RPATH_ADD_HIP_PATH "${HIP_PATH}/lib")
+# Try to find everything in /opt/rocm/lib first.
+set(GKO_HIP_RPATH "${ROCM_PATH}/lib" )
+if (GINKGO_HIP_PLATFORM MATCHES "nvcc")
+    list(GET CUDA_LIBRARIES 0 CUDA_FIRST_LIB)
+    get_filename_component(GKO_CUDA_LIBDIR "${CUDA_FIRST_LIB}" DIRECTORY)
+    list(APPEND GKO_HIP_RPATH "${GKO_CUDA_LIBDIR}")
+else()
+    list(APPEND GKO_HIP_RPATH "${HIP_PATH}/lib")
 endif()
+list(APPEND GKO_HIP_RPATH "${HIPBLAS_PATH}/lib" "${HIPRAND_PATH}/lib"
+    "${HIPSPARSE_PATH}/lib" "${ROCRAND_PATH}/lib")
 
 ginkgo_compile_features(ginkgo_hip)
 ginkgo_default_includes(ginkgo_hip)
-ginkgo_install_library(ginkgo_hip hip "${RPATH_ADD_HIP_PATH}" "${HIPBLAS_PATH}/lib"
-    "${HIPRAND_PATH}/lib" "${HIPSPARSE_PATH}/lib" "${ROCRAND_PATH}/lib")
+ginkgo_install_library(ginkgo_hip hip "${GKO_HIP_RPATH}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo_hip)

From 87162b1d1537c6cb4c8ba27731552a820eae0290 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 24 Mar 2021 11:10:31 +0100
Subject: [PATCH 0527/1008] Review update.

+ Add documentation to the new RPATH macro in install_helpers.
+ Remove the usage of `subdir` in `ginkgo_install_library` and the new
  `ginkgo_add_install_rpath` since this is unused.
+ Some minor improvements to other documentations.

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 INSTALL.md                       |  5 +--
 cmake/create_test.cmake          |  5 ++-
 cmake/install_helpers.cmake      | 56 ++++++++++++++++++++------------
 core/CMakeLists.txt              |  2 +-
 core/device_hooks/CMakeLists.txt | 10 +++---
 cuda/CMakeLists.txt              |  2 +-
 devices/CMakeLists.txt           |  2 +-
 dpcpp/CMakeLists.txt             |  2 +-
 hip/CMakeLists.txt               |  2 +-
 omp/CMakeLists.txt               |  2 +-
 reference/CMakeLists.txt         |  2 +-
 11 files changed, 53 insertions(+), 37 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index b015b7bf020..110006a2c91 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -65,8 +65,9 @@ Ginkgo adds the following additional switches to control what is being built:
 *   `-DGINKGO_VERBOSE_LEVEL=integer` sets the verbosity of Ginkgo.
     * `0` disables all output in the main libraries,
     * `1` enables a few important messages related to unexpected behavior (default).
-*   `GINKGO_INSTALL_RPATH` sets the RPATH when installing the Ginkgo libraries.
-    The default is `ON`.
+*   `GINKGO_INSTALL_RPATH` allows setting any RPATH information when installing
+    the Ginkgo libraries. If this is `OFF`, the behavior is the same as if all
+    other RPATH flags are set to `OFF` as well. The default is `ON`.
 *   `GINKGO_INSTALL_RPATH_ORIGIN` adds $ORIGIN (Linux) or @loader_path (MacOS)
     to the installation RPATH. The default is `ON`.
 *   `GINKGO_INSTALL_RPATH_DEPENDENCIES` adds the dependencies to the
diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index 817891b68ed..8621765c965 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -110,9 +110,8 @@ function(ginkgo_create_hip_test test_name)
     set_source_files_properties(${test_name}.hip.cpp PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
 
     # NOTE: With how HIP works, passing the flags `HIPCC_OPTIONS` etc. here
-    # creates a redefinition of all flags. This creates some issues with `nvcc`
-    # (particularly the flags in `HIPCC_OPTIONS`), but `clang` is fine with the
-    # redefinitions.
+    # creates a redefinition of all flags. This creates some issues with `nvcc`,
+    # but `clang` seems fine with the redefinitions.
     if (GINKGO_HIP_PLATFORM STREQUAL "nvcc")
         hip_add_executable(${TEST_TARGET_NAME} ${test_name}.hip.cpp
             # If `FindHIP.cmake`, namely `HIP_PARSE_HIPCC_OPTIONS` macro and
diff --git a/cmake/install_helpers.cmake b/cmake/install_helpers.cmake
index a831af54ebc..2ee22f39f92 100644
--- a/cmake/install_helpers.cmake
+++ b/cmake/install_helpers.cmake
@@ -8,31 +8,47 @@ set(GINKGO_INSTALL_PKGCONFIG_DIR "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 set(GINKGO_INSTALL_CONFIG_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Ginkgo")
 set(GINKGO_INSTALL_MODULE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Ginkgo/Modules")
 
-function(ginkgo_add_install_rpath name subdir)
-    if (BUILD_SHARED_LIBS)
-        if (GINKGO_INSTALL_RPATH_ORIGIN)
-            if (APPLE)
-                set(ORIGIN_OR_LOADER_PATH "@loader_path")
-            else()
-                set(ORIGIN_OR_LOADER_PATH "$ORIGIN")
-            endif()
+# This function adds the correct RPATH properties to a Ginkgo target.
+#
+# The behavior depends on three options GINKGO_INSTALL_RPATH[*] variables. It
+# does the following:
+#
+# 1. GINKGO_INSTALL_RPATH : If this flag is not set, no RPATH information is
+#    added.
+# 2. GINKGO_INSTALL_RPATH_ORIGIN : Allows adding the library directory to the
+#    RPATH.
+# 3. GINKGO_INSTALL_RPATH_DEPENDENCIES : Allows adding any extra paths to the
+#    RPATH.
+#
+# @param name  the name of the target
+# @param ARGN  any external dependencies path to be added
+function(ginkgo_add_install_rpath name)
+    if (GINKGO_INSTALL_RPATH_ORIGIN)
+        if (APPLE)
+            set(ORIGIN_OR_LOADER_PATH "@loader_path")
+        else()
+            set(ORIGIN_OR_LOADER_PATH "$ORIGIN")
         endif()
-        if (GINKGO_INSTALL_RPATH_DEPENDENCIES)
-            set(RPATH_DEPENDENCIES "${ARGN}")
-            if  (GINKGO_HAVE_HWLOC AND GINKGO_USE_EXTERNAL_HWLOC)
-                get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY)
-                list(APPEND RPATH_DEPENDENCIES "${HWLOC_LIBRARIES}")
-            endif()
-        endif()
-        if (GINKGO_INSTALL_RPATH)
-            set_property(TARGET "${name}" PROPERTY INSTALL_RPATH
-                "${ORIGIN_OR_LOADER_PATH}" "${RPATH_DEPENDENCIES}")
+    endif()
+    if (GINKGO_INSTALL_RPATH_DEPENDENCIES)
+        set(RPATH_DEPENDENCIES "${ARGN}")
+        if  (GINKGO_HAVE_HWLOC AND GINKGO_USE_EXTERNAL_HWLOC)
+            get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY)
+            list(APPEND RPATH_DEPENDENCIES "${HWLOC_LIBRARIES}")
         endif()
     endif()
+    if (GINKGO_INSTALL_RPATH)
+        set_property(TARGET "${name}" PROPERTY INSTALL_RPATH
+            "${ORIGIN_OR_LOADER_PATH}" "${RPATH_DEPENDENCIES}")
+    endif()
 endfunction()
 
-function(ginkgo_install_library name subdir)
-    ginkgo_add_install_rpath("${name}" "${subdir}" "${ARGN}")
+# Handles installation settings for a Ginkgo library.
+#
+# @param name  the name of the Ginkgo library target
+# @param ARGN  this should contain any external dependency's library PATH
+function(ginkgo_install_library name)
+    ginkgo_add_install_rpath("${name}" "${ARGN}")
 
     if (WIN32 OR CYGWIN)
         # dll is considered as runtime
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 93c878e6983..754727381da 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -79,7 +79,7 @@ if (GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc")
 endif()
 
 ginkgo_default_includes(ginkgo)
-ginkgo_install_library(ginkgo core "${GKO_RPATH_ADDITIONS}")
+ginkgo_install_library(ginkgo "${GKO_RPATH_ADDITIONS}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo)
diff --git a/core/device_hooks/CMakeLists.txt b/core/device_hooks/CMakeLists.txt
index 698391de20a..fcb370a81a0 100644
--- a/core/device_hooks/CMakeLists.txt
+++ b/core/device_hooks/CMakeLists.txt
@@ -5,7 +5,7 @@ if(NOT GINKGO_BUILD_CUDA)
     target_link_libraries(ginkgo_cuda PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_cuda)
     ginkgo_default_includes(ginkgo_cuda)
-    ginkgo_install_library(ginkgo_cuda cuda)
+    ginkgo_install_library(ginkgo_cuda)
 endif()
 
 if (NOT GINKGO_BUILD_DPCPP)
@@ -15,7 +15,7 @@ if (NOT GINKGO_BUILD_DPCPP)
     target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_dpcpp)
     ginkgo_default_includes(ginkgo_dpcpp)
-    ginkgo_install_library(ginkgo_dpcpp dpcpp)
+    ginkgo_install_library(ginkgo_dpcpp)
 endif()
 
 if(NOT GINKGO_BUILD_HIP)
@@ -25,7 +25,7 @@ if(NOT GINKGO_BUILD_HIP)
     target_link_libraries(ginkgo_hip PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_hip)
     ginkgo_default_includes(ginkgo_hip)
-    ginkgo_install_library(ginkgo_hip hip)
+    ginkgo_install_library(ginkgo_hip)
 endif()
 
 if (NOT GINKGO_BUILD_OMP)
@@ -38,7 +38,7 @@ if (NOT GINKGO_BUILD_OMP)
     target_link_libraries(ginkgo_omp PRIVATE ginkgo_dpcpp)
     target_link_libraries(ginkgo_omp PUBLIC ginkgo_device)
     ginkgo_default_includes(ginkgo_omp)
-    ginkgo_install_library(ginkgo_omp omp)
+    ginkgo_install_library(ginkgo_omp)
 endif()
 
 if (NOT GINKGO_BUILD_REFERENCE)
@@ -48,5 +48,5 @@ if (NOT GINKGO_BUILD_REFERENCE)
     target_link_libraries(ginkgo_reference PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_reference)
     ginkgo_default_includes(ginkgo_reference)
-    ginkgo_install_library(ginkgo_reference reference)
+    ginkgo_install_library(ginkgo_reference)
 endif()
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 4748dfdf278..a4ded75abed 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -167,7 +167,7 @@ list(GET CUDA_RUNTIME_LIBS 0 CUDA_FIRST_LIB)
 get_filename_component(GKO_CUDA_LIBDIR "${CUDA_FIRST_LIB}" DIRECTORY)
 
 ginkgo_default_includes(ginkgo_cuda)
-ginkgo_install_library(ginkgo_cuda cuda "${GKO_CUDA_LIBDIR}")
+ginkgo_install_library(ginkgo_cuda "${GKO_CUDA_LIBDIR}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo_cuda)
diff --git a/devices/CMakeLists.txt b/devices/CMakeLists.txt
index 733a10d1fa8..0c0e80febaf 100644
--- a/devices/CMakeLists.txt
+++ b/devices/CMakeLists.txt
@@ -17,7 +17,7 @@ function(ginkgo_add_library name)
 endfunction()
 
 ginkgo_add_library(ginkgo_device machine_topology.cpp)
-ginkgo_install_library(ginkgo_device devices)
+ginkgo_install_library(ginkgo_device)
 
 add_subdirectory(cuda)
 add_subdirectory(dpcpp)
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 89600b4aa6b..186da7f8b41 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -57,7 +57,7 @@ target_compile_features(ginkgo_dpcpp PRIVATE cxx_std_17)
 target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
 
 ginkgo_default_includes(ginkgo_dpcpp)
-ginkgo_install_library(ginkgo_dpcpp dpcpp)
+ginkgo_install_library(ginkgo_dpcpp)
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo_dpcpp)
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 3c349a5a699..14aefba1a0d 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -284,7 +284,7 @@ list(APPEND GKO_HIP_RPATH "${HIPBLAS_PATH}/lib" "${HIPRAND_PATH}/lib"
 
 ginkgo_compile_features(ginkgo_hip)
 ginkgo_default_includes(ginkgo_hip)
-ginkgo_install_library(ginkgo_hip hip "${GKO_HIP_RPATH}")
+ginkgo_install_library(ginkgo_hip "${GKO_HIP_RPATH}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo_hip)
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index 64349b24c6b..80c6262c30c 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -60,7 +60,7 @@ target_link_libraries(ginkgo_omp PRIVATE ginkgo_dpcpp)
 target_link_libraries(ginkgo_omp PUBLIC ginkgo_device)
 
 ginkgo_default_includes(ginkgo_omp)
-ginkgo_install_library(ginkgo_omp omp)
+ginkgo_install_library(ginkgo_omp)
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
     ginkgo_check_headers(ginkgo_omp)
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index d379dac167e..573b555506a 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -42,7 +42,7 @@ target_sources(ginkgo_reference
 target_link_libraries(ginkgo_reference PUBLIC ginkgo_device)
 ginkgo_compile_features(ginkgo_reference)
 ginkgo_default_includes(ginkgo_reference)
-ginkgo_install_library(ginkgo_reference reference)
+ginkgo_install_library(ginkgo_reference)
 target_compile_options(ginkgo_reference PRIVATE "${GINKGO_COMPILER_FLAGS}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)

From 14cbd4aa8f359ebe9f944508b2296b67d7dca837 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 24 Mar 2021 12:19:18 +0100
Subject: [PATCH 0528/1008] add ccache support for CMake

---
 CMakeLists.txt | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7b14f0a08d4..564b5c71931 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -39,6 +39,7 @@ option(GINKGO_EXPORT_BUILD_DIR
     OFF)
 option(GINKGO_WITH_CLANG_TIDY "Make Ginkgo call `clang-tidy` to find programming issues." OFF)
 option(GINKGO_WITH_IWYU "Make Ginkgo call `iwyu` (Include What You Use) to find include issues." OFF)
+option(GINKGO_WITH_CCACHE "Use ccache if available to speed up C++ and CUDA rebuilds by caching compilations." ON)
 option(GINKGO_CHECK_CIRCULAR_DEPS
     "Enable compile-time checks detecting circular dependencies between libraries and non-self-sufficient headers."
     OFF)
@@ -78,6 +79,17 @@ option(GINKGO_INSTALL_RPATH_DEPENDENCIES "Add dependencies to the installation R
 
 set(GINKGO_CIRCULAR_DEPS_FLAGS "-Wl,--no-undefined")
 
+# Use ccache as compilation launcher
+if(GINKGO_WITH_CCACHE)
+    find_program(CCACHE_PROGRAM ccache)
+    if(CCACHE_PROGRAM)
+        set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
+        if(GINKGO_BUILD_CUDA)
+            set(CMAKE_CUDA_COMPILER_LAUNCHER "${CCACHE_PROGRAM}")
+        endif()
+    endif()
+endif()
+
 if(GINKGO_BENCHMARK_ENABLE_TUNING)
     # In this state, the tests and examples cannot be compiled without extra
     # complexity/intrusiveness, so we simply disable them.

From 3a0b5d9c38901078e264fc07124384e2f7a3bba6 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Fri, 6 Nov 2020 13:04:22 +0100
Subject: [PATCH 0529/1008] [tests fail] copied files for fixed block size BCSR
 format, core tests pass

---
 core/CMakeLists.txt                           |    2 +
 core/components/fixed_block.hpp               |  233 +++
 core/device_hooks/common_kernels.inc.cpp      |  138 ++
 core/matrix/fbcsr.cpp                         |  698 ++++++++
 core/matrix/fbcsr_builder.hpp                 |   94 +
 core/matrix/fbcsr_kernels.hpp                 |  275 +++
 core/test/matrix/CMakeLists.txt               |    2 +
 core/test/matrix/fbcsr.cpp                    |  378 ++++
 core/test/matrix/fbcsr_builder.cpp            |  120 ++
 cuda/CMakeLists.txt                           |    1 +
 cuda/matrix/fbcsr_kernels.cu                  | 1446 +++++++++++++++
 cuda/test/matrix/CMakeLists.txt               |    1 +
 cuda/test/matrix/fbcsr_kernels.cpp            |  883 ++++++++++
 hip/CMakeLists.txt                            |    1 +
 hip/matrix/fbcsr_kernels.hip.cpp              | 1263 ++++++++++++++
 hip/test/matrix/CMakeLists.txt                |    1 +
 hip/test/matrix/fbcsr_kernels.hip.cpp         |  866 +++++++++
 include/ginkgo/core/matrix/fbcsr.hpp          |  569 ++++++
 .../ginkgo/core/matrix/matrix_strategies.hpp  |  503 ++++++
 include/ginkgo/core/matrix/sparsity_csr.hpp   |    4 +
 include/ginkgo/ginkgo.hpp                     |    2 +
 omp/CMakeLists.txt                            |    1 +
 omp/components/fbcsr_spgeam.hpp               |   31 +
 omp/matrix/fbcsr_kernels.cpp                  |  884 ++++++++++
 omp/test/matrix/CMakeLists.txt                |    1 +
 omp/test/matrix/fbcsr_kernels.cpp             |  662 +++++++
 reference/CMakeLists.txt                      |    1 +
 reference/components/fbcsr_spgeam.hpp         |   31 +
 reference/matrix/fbcsr_kernels.cpp            |  968 ++++++++++
 reference/test/matrix/CMakeLists.txt          |    1 +
 reference/test/matrix/fbcsr_kernels.cpp       | 1550 +++++++++++++++++
 31 files changed, 11610 insertions(+)
 create mode 100644 core/components/fixed_block.hpp
 create mode 100644 core/matrix/fbcsr.cpp
 create mode 100644 core/matrix/fbcsr_builder.hpp
 create mode 100644 core/matrix/fbcsr_kernels.hpp
 create mode 100644 core/test/matrix/fbcsr.cpp
 create mode 100644 core/test/matrix/fbcsr_builder.cpp
 create mode 100644 cuda/matrix/fbcsr_kernels.cu
 create mode 100644 cuda/test/matrix/fbcsr_kernels.cpp
 create mode 100644 hip/matrix/fbcsr_kernels.hip.cpp
 create mode 100644 hip/test/matrix/fbcsr_kernels.hip.cpp
 create mode 100644 include/ginkgo/core/matrix/fbcsr.hpp
 create mode 100644 include/ginkgo/core/matrix/matrix_strategies.hpp
 create mode 100644 omp/components/fbcsr_spgeam.hpp
 create mode 100644 omp/matrix/fbcsr_kernels.cpp
 create mode 100644 omp/test/matrix/fbcsr_kernels.cpp
 create mode 100644 reference/components/fbcsr_spgeam.hpp
 create mode 100644 reference/matrix/fbcsr_kernels.cpp
 create mode 100644 reference/test/matrix/fbcsr_kernels.cpp

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 754727381da..4e1a7086148 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -22,9 +22,11 @@ target_sources(ginkgo
     log/stream.cpp
     matrix/coo.cpp
     matrix/csr.cpp
+    matrix/fbcsr.cpp
     matrix/dense.cpp
     matrix/diagonal.cpp
     matrix/ell.cpp
+    matrix/fbcsr.cpp
     matrix/hybrid.cpp
     matrix/identity.cpp
     matrix/permutation.cpp
diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
new file mode 100644
index 00000000000..526cdb25909
--- /dev/null
+++ b/core/components/fixed_block.hpp
@@ -0,0 +1,233 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_COMPONENTS_FIXED_BLOCK_HPP_
+#define GKO_CORE_COMPONENTS_FIXED_BLOCK_HPP_
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/math.hpp>
+
+#include <type_traits>
+
+namespace gko {
+namespace blockutils {
+
+
+/// Error that denotes issues between block sizes and matrix dimensions
+template <typename IndexType>
+class BlockSizeError : public Error {
+public:
+    BlockSizeError(const std::string &file, const int line,
+                   const int block_size, const IndexType size)
+        : Error(file, line,
+                " block size = " + std::to_string(block_size) +
+                    ", size = " + std::to_string(size))
+    {}
+};
+
+/// Error that denotes issues between block sizes and matrix dimensions
+template <typename IndexType>
+class BlockReadError : public Error {
+public:
+    BlockReadError(const std::string &file, const int line,
+                   const std::string &msg)
+        : Error(file, line, msg)
+    {}
+};
+
+template <typename IndexType>
+int getNumFixedBlocks(const int block_size, const IndexType size)
+{
+    if (size % block_size != 0)
+        throw BlockSizeError<IndexType>(__FILE__, __LINE__, block_size, size);
+    return size / block_size;
+}
+
+/// A dense block of values with compile-time constant dimensions
+/** The blocks are stored row-major. However, in future,
+ *  a layout template parameter can be added if needed.
+ *
+ * The primary use is to reinterpret subsets of entries in a big array as
+ *  small dense blocks.
+ */
+template <typename ValueType, int nrows, int ncols>
+class FixedBlock final {
+    static_assert(nrows > 0, "Requires positive number of rows!");
+    static_assert(ncols > 0, "Requires positive number of columns!");
+
+public:
+    using value_type = ValueType;
+
+    value_type &at(const int row, const int col)
+    {
+        return vals[row * ncols + col];
+    }
+
+    const value_type &at(const int row, const int col) const
+    {
+        return vals[row * ncols + col];
+    }
+
+    value_type &operator()(const int row, const int col)
+    {
+        return at(row, col);
+    }
+
+    const value_type &operator()(const int row, const int col) const
+    {
+        return at(row, col);
+    }
+
+private:
+    ValueType vals[nrows * ncols];
+};
+
+
+/// Two-dimensional square block
+// template <typename ValueType, int size>
+// using FixedBlock = FixedRectangularBlock<ValueType, size, size>;
+
+/// Fixed-size column vector
+// template <typename ValueType, int size>
+// using FixedSegment = FixedRectangularBlock<ValueType, size, 1>;
+
+/// A lightweight dynamic block type for the host space
+template <typename ValueType>
+class DenseBlock final {
+public:
+    using value_type = ValueType;
+
+    DenseBlock() : nrows_{0}, ncols_{0}, vals_{nullptr} {}
+
+    DenseBlock(const int num_rows, const int num_cols)
+        : nrows_{num_rows},
+          ncols_{num_cols},
+          vals_{new value_type[num_rows * num_cols]}
+    {}
+
+    ~DenseBlock() { delete[] vals_; }
+
+    value_type &at(const int row, const int col)
+    {
+        return vals_[row * ncols_ + col];
+    }
+
+    const value_type &at(const int row, const int col) const
+    {
+        return vals_[row * ncols_ + col];
+    }
+
+    value_type &operator()(const int row, const int col)
+    {
+        return at(row, col);
+    }
+
+    const value_type &operator()(const int row, const int col) const
+    {
+        return at(row, col);
+    }
+
+    int size() const { return nrows_ * ncols_; }
+
+    void resize(const int nrows, const int ncols)
+    {
+        nrows_ = nrows;
+        ncols_ = ncols;
+        delete[] vals_;
+        vals_ = new value_type[nrows_ * ncols_];
+    }
+
+    void zero()
+    {
+        for (int i = 0; i < nrows_ * ncols_; i++)
+            // vals_[i] = gko::zero<value_type>();
+            vals_[i] = static_cast<value_type>(0);
+    }
+
+private:
+    int nrows_;
+    int ncols_;
+    value_type *vals_;
+};
+
+/// A view into a an array of dense of dense blocks of some runtime-defined size
+template <typename ValueType, typename IndexType = int32>
+class DenseBlocksView final {
+public:
+    using value_type = ValueType;
+    using index_type = IndexType;
+
+    /**
+     * @param buffer Segment of memory to be interpreted as an array of 2D
+     * blocks
+     * @param num_rows Number of rows in each block
+     * @param num_cols Number of columns in each block
+     */
+    DenseBlocksView(ValueType *const buffer, const int num_rows,
+                    const int num_cols)
+        : nrows_{num_rows}, ncols_{num_cols}, vals_{buffer}
+    {}
+
+    value_type &at(const index_type block, const int row, const int col)
+    {
+        return vals_[block * nrows_ * ncols_ + row * ncols_ + col];
+    }
+
+    const typename std::remove_const<value_type>::type &at(
+        const index_type block, const int row, const int col) const
+    {
+        return vals_[block * nrows_ * ncols_ + row * ncols_ + col];
+    }
+
+    value_type &operator()(const index_type block, const int row, const int col)
+    {
+        return at(block, row, col);
+    }
+
+    const typename std::remove_const<value_type>::type &operator()(
+        const index_type block, const int row, const int col) const
+    {
+        return at(block, row, col);
+    }
+
+private:
+    int nrows_;  ///< Number of rows in each block
+    int ncols_;  ///< Number of columns in each block
+    value_type *vals_;
+};
+
+
+}  // namespace blockutils
+}  // namespace gko
+
+#endif
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index eceae46dd8b..43f4d90d8c0 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -49,6 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/dense_kernels.hpp"
 #include "core/matrix/diagonal_kernels.hpp"
 #include "core/matrix/ell_kernels.hpp"
+#include "core/matrix/fbcsr_kernels.hpp"
 #include "core/matrix/hybrid_kernels.hpp"
 #include "core/matrix/sellp_kernels.hpp"
 #include "core/matrix/sparsity_csr_kernels.hpp"
@@ -803,6 +804,143 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_EXTRACT_DIAGONAL);
 }  // namespace csr
 
 
+// TODO (script:fbcsr): adapt this block as needed
+namespace fbcsr {
+
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_SPMV_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPMV_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_SPGEMM_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEMM_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_SPGEAM_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEAM_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL);
+
+
+}  // namespace fbcsr
+
+
 namespace coo {
 
 
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
new file mode 100644
index 00000000000..e362b3b466f
--- /dev/null
+++ b/core/matrix/fbcsr.cpp
@@ -0,0 +1,698 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/matrix/fbcsr.hpp>
+
+
+#include <limits>
+#include <map>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/utils.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/ell.hpp>
+#include <ginkgo/core/matrix/identity.hpp>
+#include <ginkgo/core/matrix/sellp.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+
+
+#include "core/components/absolute_array.hpp"
+#include "core/components/fill_array.hpp"
+#include "core/components/fixed_block.hpp"
+#include "core/matrix/fbcsr_kernels.hpp"
+
+
+namespace gko {
+namespace matrix {
+namespace fbcsr {
+
+
+GKO_REGISTER_OPERATION(spmv, fbcsr::spmv);
+// GKO_REGISTER_OPERATION(advanced_spmv, fbcsr::advanced_spmv);
+GKO_REGISTER_OPERATION(spgemm, fbcsr::spgemm);
+// GKO_REGISTER_OPERATION(advanced_spgemm, fbcsr::advanced_spgemm);
+GKO_REGISTER_OPERATION(spgeam, fbcsr::spgeam);
+GKO_REGISTER_OPERATION(convert_to_coo, fbcsr::convert_to_coo);
+GKO_REGISTER_OPERATION(convert_to_dense, fbcsr::convert_to_dense);
+// GKO_REGISTER_OPERATION(convert_to_sellp, fbcsr::convert_to_sellp);
+GKO_REGISTER_OPERATION(calculate_total_cols, fbcsr::calculate_total_cols);
+// GKO_REGISTER_OPERATION(convert_to_ell, fbcsr::convert_to_ell);
+// GKO_REGISTER_OPERATION(convert_to_hybrid, fbcsr::convert_to_hybrid);
+GKO_REGISTER_OPERATION(transpose, fbcsr::transpose);
+GKO_REGISTER_OPERATION(conj_transpose, fbcsr::conj_transpose);
+GKO_REGISTER_OPERATION(row_permute, fbcsr::row_permute);
+GKO_REGISTER_OPERATION(column_permute, fbcsr::column_permute);
+GKO_REGISTER_OPERATION(inverse_row_permute, fbcsr::inverse_row_permute);
+GKO_REGISTER_OPERATION(inverse_column_permute, fbcsr::inverse_column_permute);
+GKO_REGISTER_OPERATION(calculate_max_nnz_per_row,
+                       fbcsr::calculate_max_nnz_per_row);
+GKO_REGISTER_OPERATION(calculate_nonzeros_per_row,
+                       fbcsr::calculate_nonzeros_per_row);
+GKO_REGISTER_OPERATION(sort_by_column_index, fbcsr::sort_by_column_index);
+GKO_REGISTER_OPERATION(is_sorted_by_column_index,
+                       fbcsr::is_sorted_by_column_index);
+GKO_REGISTER_OPERATION(extract_diagonal, fbcsr::extract_diagonal);
+GKO_REGISTER_OPERATION(fill_array, components::fill_array);
+GKO_REGISTER_OPERATION(inplace_absolute_array,
+                       components::inplace_absolute_array);
+GKO_REGISTER_OPERATION(outplace_absolute_array,
+                       components::outplace_absolute_array);
+
+
+}  // namespace fbcsr
+
+
+template <typename ValueType, typename IndexType>
+Fbcsr<ValueType, IndexType>::Fbcsr(std::shared_ptr<const Executor> exec,
+                                   const dim<2> &size, size_type num_nonzeros,
+                                   int block_size,
+                                   std::shared_ptr<strategy_type> strategy)
+    : EnableLinOp<Fbcsr>(exec, size),
+      bs_{block_size},
+      values_(exec, num_nonzeros),
+      col_idxs_(exec, gko::blockutils::getNumFixedBlocks(
+                          block_size * block_size, num_nonzeros)),
+      row_ptrs_(exec,
+                gko::blockutils::getNumFixedBlocks(block_size, size[0]) + 1),
+      startrow_(exec, strategy->calc_size(num_nonzeros)),
+      strategy_(strategy->copy())
+{
+    if (size[0] % bs_ != 0)
+        throw gko::BadDimension(__FILE__, __LINE__, __func__, "construct",
+                                size[0], size[1],
+                                "block size does not divide the dim 0!");
+    if (size[1] % bs_ != 0)
+        throw gko::BadDimension(__FILE__, __LINE__, __func__, "construct",
+                                size[0], size[1],
+                                "block size does not divide the dim 1!");
+    if (num_nonzeros % (bs_ * bs_) != 0)
+        throw gko::BadDimension(__FILE__, __LINE__, __func__, "construct",
+                                size[0], size[1],
+                                "block size^2 does not divide NNZ!");
+}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const b,
+                                             LinOp *const x) const
+{
+    // TODO (script:fbcsr): change the code imported from matrix/csr if needed
+    using Dense = Dense<ValueType>;
+    using TFbcsr = Fbcsr<ValueType, IndexType>;
+    if (auto b_fbcsr = dynamic_cast<const TFbcsr *>(b)) {
+        // if b is a FBCSR matrix, we compute a SpGeMM
+        throw /*::gko::*/ NotImplemented(__FILE__, __LINE__,
+                                         "SpGeMM for Fbcsr");
+        auto x_fbcsr = as<TFbcsr>(x);
+        this->get_executor()->run(fbcsr::make_spgemm(this, b_fbcsr, x_fbcsr));
+    } else {
+        // otherwise we assume that b is dense and compute a SpMV/SpMM
+        this->get_executor()->run(
+            fbcsr::make_spmv(this, as<Dense>(b), as<Dense>(x)));
+    }
+}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
+                                             const LinOp *beta, LinOp *x) const
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Dense = Dense<ValueType>;
+//    using TFbcsr = Fbcsr<ValueType, IndexType>;
+//    if (auto b_fbcsr = dynamic_cast<const TFbcsr *>(b)) {
+//        // if b is a FBCSR matrix, we compute a SpGeMM
+//        auto x_fbcsr = as<TFbcsr>(x);
+//        auto x_copy = x_fbcsr->clone();
+//        this->get_executor()->run(
+//            fbcsr::make_advanced_spgemm(as<Dense>(alpha), this, b_fbcsr,
+//                                      as<Dense>(beta), x_copy.get(),
+//                                      x_fbcsr));
+//    } else if (dynamic_cast<const Identity<ValueType> *>(b)) {
+//        // if b is an identity matrix, we compute an SpGEAM
+//        auto x_fbcsr = as<TFbcsr>(x);
+//        auto x_copy = x_fbcsr->clone();
+//        this->get_executor()->run(fbcsr::make_spgeam(
+//            as<Dense>(alpha), this, as<Dense>(beta), lend(x_copy), x_fbcsr));
+//    } else {
+//        // otherwise we assume that b is dense and compute a SpMV/SpMM
+//        this->get_executor()->run(
+//            fbcsr::make_advanced_spmv(as<Dense>(alpha), this, as<Dense>(b),
+//                                    as<Dense>(beta), as<Dense>(x)));
+//    }
+//}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::convert_to(
+    Fbcsr<next_precision<ValueType>, IndexType> *const result) const
+{
+    result->values_ = this->values_;
+    result->col_idxs_ = this->col_idxs_;
+    result->row_ptrs_ = this->row_ptrs_;
+    result->set_size(this->get_size());
+    result->bs_ = this->bs_;
+    convert_strategy_helper(result);
+}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::move_to(
+    Fbcsr<next_precision<ValueType>, IndexType> *result)
+{
+    this->convert_to(result);
+}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::convert_to(
+    Coo<ValueType, IndexType> *result) const GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto exec = this->get_executor();
+//    auto tmp = Coo<ValueType, IndexType>::create(
+//        exec, this->get_size(), this->get_num_stored_elements());
+//    tmp->values_ = this->values_;
+//    tmp->col_idxs_ = this->col_idxs_;
+//    exec->run(fbcsr::make_convert_to_coo(this, tmp.get()));
+//    tmp->move_to(result);
+//}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::move_to(Coo<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    this->convert_to(result);
+//}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::convert_to(Dense<ValueType> *result) const
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto exec = this->get_executor();
+//    auto tmp = Dense<ValueType>::create(exec, this->get_size());
+//    exec->run(fbcsr::make_convert_to_dense(this, tmp.get()));
+//    tmp->move_to(result);
+//}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::move_to(Dense<ValueType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    this->convert_to(result);
+//}
+
+
+// template <typename ValueType, typename IndexType>
+// void Fbcsr<ValueType, IndexType>::convert_to(
+//     Hybrid<ValueType, IndexType> *result) const
+// GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto exec = this->get_executor();
+//    Array<size_type> row_nnz(exec, this->get_size()[0]);
+//
+//    size_type ell_lim = zero<size_type>();
+//    size_type coo_lim = zero<size_type>();
+//    result->get_strategy()->compute_hybrid_config(row_nnz, &ell_lim,
+//    &coo_lim); const auto max_nnz_per_row =
+//        std::max(result->get_ell_num_stored_elements_per_row(), ell_lim);
+//    const auto stride = std::max(result->get_ell_stride(),
+//    this->get_size()[0]); const auto coo_nnz =
+//        std::max(result->get_coo_num_stored_elements(), coo_lim);
+//    auto tmp = Hybrid<ValueType, IndexType>::create(
+//        exec, this->get_size(), max_nnz_per_row, stride, coo_nnz,
+//        result->get_strategy());
+//    exec->run(fbcsr::make_convert_to_hybrid(this, tmp.get()));
+//    tmp->move_to(result);
+//}
+
+
+// template <typename ValueType, typename IndexType>
+// void Fbcsr<ValueType, IndexType>::move_to(Hybrid<ValueType, IndexType>
+// *result) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    this->convert_to(result);
+//}
+
+
+// template <typename ValueType, typename IndexType>
+// void Fbcsr<ValueType, IndexType>::convert_to(
+//     Sellp<ValueType, IndexType> *result) const
+// GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto exec = this->get_executor();
+//    const auto stride_factor = (result->get_stride_factor() == 0)
+//                                   ? default_stride_factor
+//                                   : result->get_stride_factor();
+//    const auto slice_size = (result->get_slice_size() == 0)
+//                                ? default_slice_size
+//                                : result->get_slice_size();
+//    size_type total_cols = 0;
+//    exec->run(fbcsr::make_calculate_total_cols(this, &total_cols,
+//    stride_factor,
+//                                             slice_size));
+//    auto tmp = Sellp<ValueType, IndexType>::create(
+//        exec, this->get_size(), slice_size, stride_factor, total_cols);
+//    exec->run(fbcsr::make_convert_to_sellp(this, tmp.get()));
+//    tmp->move_to(result);
+//}
+
+
+// template <typename ValueType, typename IndexType>
+// void Fbcsr<ValueType, IndexType>::move_to(Sellp<ValueType, IndexType>
+// *result) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    this->convert_to(result);
+//}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::convert_to(
+    SparsityCsr<ValueType, IndexType> *result) const
+{
+    auto exec = this->get_executor();
+    auto tmp = SparsityCsr<ValueType, IndexType>::create(
+        exec, this->get_size(), this->get_num_stored_elements());
+    tmp->col_idxs_ = this->col_idxs_;
+    tmp->row_ptrs_ = this->row_ptrs_;
+    // if (result->value_.get_data()) {
+    //     tmp->value_ = result->value_;
+    // } else {
+    tmp->value_ = gko::Array<ValueType>(exec, {one<ValueType>()});
+    // }
+    tmp->move_to(result);
+}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::move_to(
+    SparsityCsr<ValueType, IndexType> *result)
+{
+    this->convert_to(result);
+}
+
+
+// template <typename ValueType, typename IndexType>
+// void Fbcsr<ValueType, IndexType>::convert_to(
+//     Ell<ValueType, IndexType> *result) const
+// GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto exec = this->get_executor();
+//    size_type max_nnz_per_row;
+//    exec->run(fbcsr::make_calculate_max_nnz_per_row(this, &max_nnz_per_row));
+//    auto tmp = Ell<ValueType, IndexType>::create(exec, this->get_size(),
+//                                                 max_nnz_per_row);
+//    exec->run(fbcsr::make_convert_to_ell(this, tmp.get()));
+//    tmp->move_to(result);
+//}
+
+
+// template <typename ValueType, typename IndexType>
+// void Fbcsr<ValueType, IndexType>::move_to(Ell<ValueType, IndexType> *result)
+// GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    this->convert_to(result);
+//}
+
+
+/* Within blocks, the storage order is row-major.
+ * Currently, this implementation is sequential and has complexity O(n log n)
+ * assuming nnz = O(n).
+ * Can this be changed to a parallel O(n) implementation?
+ */
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
+{
+    if (data.nonzeros.size() > std::numeric_limits<index_type>::max())
+        throw std::range_error(std::string("file: ") + __FILE__ + ":" +
+                               std::to_string(__LINE__) +
+                               ": List of nonzeros is too big!");
+
+    const index_type nnz = static_cast<index_type>(data.nonzeros.size());
+
+    const int bs = this->bs_;
+    // GKO_ASSERT_EQ(nnz%(this->bs_*this->bs_), 0);
+
+    using Blk_t = blockutils::DenseBlock<value_type>;
+
+    struct FbEntry {
+        index_type block_row;
+        index_type block_column;
+    };
+
+    struct FbLess {
+        bool operator()(const FbEntry &a, const FbEntry &b) const
+        {
+            if (a.block_row != b.block_row)
+                return a.block_row < b.block_row;
+            else
+                return a.block_column < b.block_column;
+        }
+    };
+
+    auto create_block_set = [nnz, bs](const mat_data &data) {
+        std::map<FbEntry, Blk_t, FbLess> blocks;
+        for (index_type inz = 0; inz < nnz; inz++) {
+            const index_type row = data.nonzeros[inz].row;
+            const index_type col = data.nonzeros[inz].column;
+            const value_type val = data.nonzeros[inz].value;
+
+            const int localrow = static_cast<int>(row % bs);
+            const int localcol = static_cast<int>(col % bs);
+            const index_type blockrow = row / bs;
+            const index_type blockcol = col / bs;
+
+            // const typename std::map<FbEntry,Blk_t,FbLess>::iterator it
+            //     = blocks.find(FbEntry{row/bs, col/bs,
+            //     DenseBlock<value_type>()});
+            Blk_t &nnzblk = blocks[{blockrow, blockcol}];
+            if (nnzblk.size() == 0) {
+                nnzblk.resize(bs, bs);
+                nnzblk.zero();
+                nnzblk(localrow, localcol) = val;
+            } else {
+                if (nnzblk(localrow, localcol) != gko::zero<value_type>())
+                    throw Error(__FILE__, __LINE__,
+                                "Error in reading fixed block CSR matrix!");
+                nnzblk(localrow, localcol) = val;
+            }
+        }
+        return blocks;
+    };
+
+    const std::map<FbEntry, Blk_t, FbLess> blocks = create_block_set(data);
+
+    auto tmp = Fbcsr::create(this->get_executor()->get_master(), data.size,
+                             blocks.size() * bs * bs, bs, this->get_strategy());
+
+    tmp->row_ptrs_.get_data()[0] = 0;
+    index_type cur_brow = 0, cur_bnz = 0,
+               cur_bcol = blocks.begin()->first.block_column;
+    const index_type num_brows = data.size[0] / bs;
+
+    gko::blockutils::DenseBlocksView<value_type, index_type> values(
+        tmp->values_.get_data(), bs, bs);
+
+    for (auto it = blocks.begin(); it != blocks.end(); it++) {
+        if (cur_brow >= num_brows)
+            throw gko::OutOfBoundsError(__FILE__, __LINE__, cur_brow,
+                                        num_brows);
+
+        // set block-column index and block values
+        tmp->col_idxs_.get_data()[cur_bnz] = it->first.block_column;
+        // vals
+        for (int ibr = 0; ibr < bs; ibr++)
+            for (int jbr = 0; jbr < bs; jbr++)
+                values(cur_bnz, ibr, jbr) = it->second(ibr, jbr);
+
+        if (it->first.block_row > cur_brow) {
+            tmp->row_ptrs_.get_data()[++cur_brow] = cur_bnz;
+        } else {
+            assert(cur_brow == it->first.block_row);
+            assert(cur_bcol <= it->first.block_column);
+        }
+
+        cur_bcol = it->first.block_column;
+        cur_bnz++;
+    }
+
+    tmp->row_ptrs_.get_data()[++cur_brow] =
+        static_cast<index_type>(blocks.size());
+    assert(cur_brow == tmp->get_size()[0] / bs);
+
+    tmp->make_srow();
+    tmp->move_to(this);
+}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::write(mat_data &data) const
+{
+    std::unique_ptr<const LinOp> op{};
+    const Fbcsr *tmp{};
+    if (this->get_executor()->get_master() != this->get_executor()) {
+        op = this->clone(this->get_executor()->get_master());
+        tmp = static_cast<const Fbcsr *>(op.get());
+    } else {
+        tmp = this;
+    }
+
+    data = {tmp->get_size(), {}};
+
+    const gko::blockutils::DenseBlocksView<const value_type, index_type>
+        vblocks(tmp->values_.get_const_data(), bs_, bs_);
+
+    for (size_type brow = 0; brow < tmp->get_size()[0] / bs_; ++brow) {
+        const auto start = tmp->row_ptrs_.get_const_data()[brow];
+        const auto end = tmp->row_ptrs_.get_const_data()[brow + 1];
+
+        for (auto inz = start; inz < end; ++inz) {
+            for (int ib = 0; ib < bs_; ib++) {
+                const auto row = brow * bs_ + ib;
+                for (int jb = 0; jb < bs_; jb++) {
+                    const auto col =
+                        tmp->col_idxs_.get_const_data()[inz] * bs_ + jb;
+                    const auto val = vblocks(inz, ib, jb);
+                    data.nonzeros.emplace_back(row, col, val);
+                }
+            }
+        }
+    }
+}
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::transpose() const
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto exec = this->get_executor();
+//    auto trans_cpy =
+//        Fbcsr::create(exec, gko::transpose(this->get_size()),
+//                    this->get_num_stored_elements(), this->get_strategy());
+//
+//    exec->run(fbcsr::make_transpose(this, trans_cpy.get()));
+//    trans_cpy->make_srow();
+//    return std::move(trans_cpy);
+//}
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::conj_transpose() const
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto exec = this->get_executor();
+//    auto trans_cpy =
+//        Fbcsr::create(exec, gko::transpose(this->get_size()),
+//                    this->get_num_stored_elements(), this->get_strategy());
+//
+//    exec->run(fbcsr::make_conj_transpose(this, trans_cpy.get()));
+//    trans_cpy->make_srow();
+//    return std::move(trans_cpy);
+//}
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::row_permute(
+    const Array<IndexType> *permutation_indices) const GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+//    auto exec = this->get_executor();
+//    auto permute_cpy =
+//        Fbcsr::create(exec, this->get_size(), this->get_num_stored_elements(),
+//                    this->get_strategy());
+//
+//    exec->run(
+//        fbcsr::make_row_permute(permutation_indices, this,
+//        permute_cpy.get()));
+//    permute_cpy->make_srow();
+//    return std::move(permute_cpy);
+//}
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::column_permute(
+    const Array<IndexType> *permutation_indices) const GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
+//    auto exec = this->get_executor();
+//    auto permute_cpy =
+//        Fbcsr::create(exec, this->get_size(), this->get_num_stored_elements(),
+//                    this->get_strategy());
+//
+//    exec->run(
+//        fbcsr::make_column_permute(permutation_indices, this,
+//        permute_cpy.get()));
+//    permute_cpy->make_srow();
+//    return std::move(permute_cpy);
+//}
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::inverse_row_permute(
+    const Array<IndexType> *inverse_permutation_indices) const
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    GKO_ASSERT_EQ(inverse_permutation_indices->get_num_elems(),
+//                  this->get_size()[0]);
+//    auto exec = this->get_executor();
+//    auto inverse_permute_cpy =
+//        Fbcsr::create(exec, this->get_size(), this->get_num_stored_elements(),
+//                    this->get_strategy());
+//
+//    exec->run(fbcsr::make_inverse_row_permute(inverse_permutation_indices,
+//    this,
+//                                            inverse_permute_cpy.get()));
+//    inverse_permute_cpy->make_srow();
+//    return std::move(inverse_permute_cpy);
+//}
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::inverse_column_permute(
+    const Array<IndexType> *inverse_permutation_indices) const
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    GKO_ASSERT_EQ(inverse_permutation_indices->get_num_elems(),
+//                  this->get_size()[1]);
+//    auto exec = this->get_executor();
+//    auto inverse_permute_cpy =
+//        Fbcsr::create(exec, this->get_size(), this->get_num_stored_elements(),
+//                    this->get_strategy());
+//
+//    exec->run(fbcsr::make_inverse_column_permute(
+//        inverse_permutation_indices, this, inverse_permute_cpy.get()));
+//    inverse_permute_cpy->make_srow();
+//    return std::move(inverse_permute_cpy);
+//}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::sort_by_column_index() GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto exec = this->get_executor();
+//    exec->run(fbcsr::make_sort_by_column_index(this));
+//}
+
+
+template <typename ValueType, typename IndexType>
+bool Fbcsr<ValueType, IndexType>::is_sorted_by_column_index() const
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto exec = this->get_executor();
+//    bool is_sorted;
+//    exec->run(fbcsr::make_is_sorted_by_column_index(this, &is_sorted));
+//    return is_sorted;
+//}
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<Diagonal<ValueType>>
+Fbcsr<ValueType, IndexType>::extract_diagonal() const GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto exec = this->get_executor();
+//
+//    const auto diag_size = std::min(this->get_size()[0], this->get_size()[1]);
+//    auto diag = Diagonal<ValueType>::create(exec, diag_size);
+//    exec->run(fbcsr::make_fill_array(diag->get_values(), diag->get_size()[0],
+//                                   zero<ValueType>()));
+//    exec->run(fbcsr::make_extract_diagonal(this, lend(diag)));
+//    return diag;
+//}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::compute_absolute_inplace()
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto exec = this->get_executor();
+//
+//    exec->run(fbcsr::make_inplace_absolute_array(
+//        this->get_values(), this->get_num_stored_elements()));
+//}
+
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<typename Fbcsr<ValueType, IndexType>::absolute_type>
+Fbcsr<ValueType, IndexType>::compute_absolute() const GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto exec = this->get_executor();
+//
+//    auto abs_fbcsr = absolute_type::create(exec, this->get_size(),
+//                                         this->get_num_stored_elements());
+//
+//    abs_fbcsr->col_idxs_ = col_idxs_;
+//    abs_fbcsr->row_ptrs_ = row_ptrs_;
+//    exec->run(fbcsr::make_outplace_absolute_array(this->get_const_values(),
+//                                                this->get_num_stored_elements(),
+//                                                abs_fbcsr->get_values()));
+//
+//    convert_strategy_helper(abs_fbcsr.get());
+//    return abs_fbcsr;
+//}
+
+
+#define GKO_DECLARE_FBCSR_MATRIX(ValueType, IndexType) \
+    class Fbcsr<ValueType, IndexType>
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_MATRIX);
+
+
+}  // namespace matrix
+}  // namespace gko
diff --git a/core/matrix/fbcsr_builder.hpp b/core/matrix/fbcsr_builder.hpp
new file mode 100644
index 00000000000..54b6ece53f9
--- /dev/null
+++ b/core/matrix/fbcsr_builder.hpp
@@ -0,0 +1,94 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_MATRIX_FBCSR_BUILDER_HPP_
+#define GKO_CORE_MATRIX_FBCSR_BUILDER_HPP_
+
+
+#include <ginkgo/core/matrix/fbcsr.hpp>
+
+
+namespace gko {
+namespace matrix {
+
+
+/**
+ * @internal
+ *
+ * Allows intrusive access to the arrays stored within a @ref Fbcsr matrix.
+ *
+ * @tparam ValueType  the value type of the matrix
+ * @tparam IndexType  the index type of the matrix
+ */
+template <typename ValueType = default_precision, typename IndexType = int32>
+class FbcsrBuilder {
+public:
+    /**
+     * Returns the column index array of the CSR matrix.
+     */
+    Array<IndexType> &get_col_idx_array() { return matrix_->col_idxs_; }
+
+    /**
+     * Returns the value array of the CSR matrix.
+     */
+    Array<ValueType> &get_value_array() { return matrix_->values_; }
+
+    /// Returns the (uniform) block size
+    int get_block_size() const { return matrix_->bs_; }
+
+    /**
+     * Initializes a CsrBuilder from an existing CSR matrix.
+     */
+    explicit FbcsrBuilder(Fbcsr<ValueType, IndexType> *const matrix)
+        : matrix_{matrix}
+    {}
+
+    /**
+     * Updates the internal matrix data structures at destruction.
+     */
+    ~FbcsrBuilder() { matrix_->make_srow(); }
+
+    // make this type non-movable
+    FbcsrBuilder(const FbcsrBuilder &) = delete;
+    FbcsrBuilder(FbcsrBuilder &&) = delete;
+    FbcsrBuilder &operator=(const FbcsrBuilder &) = delete;
+    FbcsrBuilder &operator=(FbcsrBuilder &&) = delete;
+
+private:
+    Fbcsr<ValueType, IndexType> *matrix_;
+};
+
+
+}  // namespace matrix
+}  // namespace gko
+
+#endif  // GKO_CORE_MATRIX_FBCSR_BUILDER_HPP_
diff --git a/core/matrix/fbcsr_kernels.hpp b/core/matrix/fbcsr_kernels.hpp
new file mode 100644
index 00000000000..715bb1192ce
--- /dev/null
+++ b/core/matrix/fbcsr_kernels.hpp
@@ -0,0 +1,275 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_MATRIX_FBCSR_KERNELS_HPP_
+#define GKO_CORE_MATRIX_FBCSR_KERNELS_HPP_
+
+
+#include <ginkgo/core/matrix/fbcsr.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+#include <ginkgo/core/matrix/ell.hpp>
+#include <ginkgo/core/matrix/hybrid.hpp>
+#include <ginkgo/core/matrix/sellp.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+
+
+namespace gko {
+namespace kernels {
+
+
+#define GKO_DECLARE_FBCSR_SPMV_KERNEL(ValueType, IndexType) \
+    void spmv(std::shared_ptr<const DefaultExecutor> exec,  \
+              const matrix::Fbcsr<ValueType, IndexType> *a, \
+              const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+
+#define GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL(ValueType, IndexType) \
+    void advanced_spmv(std::shared_ptr<const DefaultExecutor> exec,  \
+                       const matrix::Dense<ValueType> *alpha,        \
+                       const matrix::Fbcsr<ValueType, IndexType> *a, \
+                       const matrix::Dense<ValueType> *b,            \
+                       const matrix::Dense<ValueType> *beta,         \
+                       matrix::Dense<ValueType> *c)
+
+#define GKO_DECLARE_FBCSR_SPGEMM_KERNEL(ValueType, IndexType) \
+    void spgemm(std::shared_ptr<const DefaultExecutor> exec,  \
+                const matrix::Fbcsr<ValueType, IndexType> *a, \
+                const matrix::Fbcsr<ValueType, IndexType> *b, \
+                matrix::Fbcsr<ValueType, IndexType> *c)
+
+#define GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL(ValueType, IndexType) \
+    void advanced_spgemm(std::shared_ptr<const DefaultExecutor> exec,  \
+                         const matrix::Dense<ValueType> *alpha,        \
+                         const matrix::Fbcsr<ValueType, IndexType> *a, \
+                         const matrix::Fbcsr<ValueType, IndexType> *b, \
+                         const matrix::Dense<ValueType> *beta,         \
+                         const matrix::Fbcsr<ValueType, IndexType> *d, \
+                         matrix::Fbcsr<ValueType, IndexType> *c)
+
+#define GKO_DECLARE_FBCSR_SPGEAM_KERNEL(ValueType, IndexType) \
+    void spgeam(std::shared_ptr<const DefaultExecutor> exec,  \
+                const matrix::Dense<ValueType> *alpha,        \
+                const matrix::Fbcsr<ValueType, IndexType> *a, \
+                const matrix::Dense<ValueType> *beta,         \
+                const matrix::Fbcsr<ValueType, IndexType> *b, \
+                matrix::Fbcsr<ValueType, IndexType> *c)
+
+#define GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType)      \
+    void convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,       \
+                          const matrix::Fbcsr<ValueType, IndexType> *source, \
+                          matrix::Dense<ValueType> *result)
+
+#define GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL(ValueType, IndexType)      \
+    void convert_to_coo(std::shared_ptr<const DefaultExecutor> exec,       \
+                        const matrix::Fbcsr<ValueType, IndexType> *source, \
+                        matrix::Coo<ValueType, IndexType> *result)
+
+#define GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL(ValueType, IndexType)      \
+    void convert_to_ell(std::shared_ptr<const DefaultExecutor> exec,       \
+                        const matrix::Fbcsr<ValueType, IndexType> *source, \
+                        matrix::Ell<ValueType, IndexType> *result)
+
+#define GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL(ValueType, IndexType)      \
+    void convert_to_hybrid(std::shared_ptr<const DefaultExecutor> exec,       \
+                           const matrix::Fbcsr<ValueType, IndexType> *source, \
+                           matrix::Hybrid<ValueType, IndexType> *result)
+
+#define GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL(ValueType, IndexType)      \
+    void convert_to_sellp(std::shared_ptr<const DefaultExecutor> exec,       \
+                          const matrix::Fbcsr<ValueType, IndexType> *source, \
+                          matrix::Sellp<ValueType, IndexType> *result)
+
+#define GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL(ValueType, IndexType)   \
+    void calculate_total_cols(                                                \
+        std::shared_ptr<const DefaultExecutor> exec,                          \
+        const matrix::Fbcsr<ValueType, IndexType> *source, size_type *result, \
+        size_type stride_factor, size_type slice_size)
+
+#define GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL(ValueType, IndexType)    \
+    void transpose(std::shared_ptr<const DefaultExecutor> exec,     \
+                   const matrix::Fbcsr<ValueType, IndexType> *orig, \
+                   matrix::Fbcsr<ValueType, IndexType> *trans)
+
+#define GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL(ValueType, IndexType)    \
+    void conj_transpose(std::shared_ptr<const DefaultExecutor> exec,     \
+                        const matrix::Fbcsr<ValueType, IndexType> *orig, \
+                        matrix::Fbcsr<ValueType, IndexType> *trans)
+
+#define GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL(ValueType, IndexType)    \
+    void row_permute(std::shared_ptr<const DefaultExecutor> exec,     \
+                     const Array<IndexType> *permutation_indices,     \
+                     const matrix::Fbcsr<ValueType, IndexType> *orig, \
+                     matrix::Fbcsr<ValueType, IndexType> *row_permuted)
+
+#define GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)    \
+    void column_permute(std::shared_ptr<const DefaultExecutor> exec,     \
+                        const Array<IndexType> *permutation_indices,     \
+                        const matrix::Fbcsr<ValueType, IndexType> *orig, \
+                        matrix::Fbcsr<ValueType, IndexType> *column_permuted)
+
+#define GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType) \
+    void inverse_row_permute(                                              \
+        std::shared_ptr<const DefaultExecutor> exec,                       \
+        const Array<IndexType> *permutation_indices,                       \
+        const matrix::Fbcsr<ValueType, IndexType> *orig,                   \
+        matrix::Fbcsr<ValueType, IndexType> *row_permuted)
+
+#define GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType) \
+    void inverse_column_permute(                                              \
+        std::shared_ptr<const DefaultExecutor> exec,                          \
+        const Array<IndexType> *permutation_indices,                          \
+        const matrix::Fbcsr<ValueType, IndexType> *orig,                      \
+        matrix::Fbcsr<ValueType, IndexType> *column_permuted)
+
+#define GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL(ValueType, \
+                                                           IndexType) \
+    void calculate_max_nnz_per_row(                                   \
+        std::shared_ptr<const DefaultExecutor> exec,                  \
+        const matrix::Fbcsr<ValueType, IndexType> *source, size_type *result)
+
+#define GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL(ValueType, \
+                                                            IndexType) \
+    void calculate_nonzeros_per_row(                                   \
+        std::shared_ptr<const DefaultExecutor> exec,                   \
+        const matrix::Fbcsr<ValueType, IndexType> *source,             \
+        Array<size_type> *result)
+
+#define GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX(ValueType, IndexType)       \
+    void sort_by_column_index(std::shared_ptr<const DefaultExecutor> exec, \
+                              matrix::Fbcsr<ValueType, IndexType> *to_sort)
+
+#define GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX(ValueType, IndexType) \
+    void is_sorted_by_column_index(                                       \
+        std::shared_ptr<const DefaultExecutor> exec,                      \
+        const matrix::Fbcsr<ValueType, IndexType> *to_check, bool *is_sorted)
+
+#define GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL(ValueType, IndexType)           \
+    void extract_diagonal(std::shared_ptr<const DefaultExecutor> exec,     \
+                          const matrix::Fbcsr<ValueType, IndexType> *orig, \
+                          matrix::Diagonal<ValueType> *diag)
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES                                           \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_SPMV_KERNEL(ValueType, IndexType);                       \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL(ValueType, IndexType);              \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_SPGEMM_KERNEL(ValueType, IndexType);                     \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL(ValueType, IndexType);            \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_SPGEAM_KERNEL(ValueType, IndexType);                     \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType);           \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL(ValueType, IndexType);             \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL(ValueType, IndexType);           \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL(ValueType, IndexType);          \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL(ValueType, IndexType);             \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL(ValueType, IndexType);       \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL(ValueType, IndexType);                  \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL(ValueType, IndexType);             \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL(ValueType, IndexType);                \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL(ValueType, IndexType);             \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType);        \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType);     \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL(ValueType, IndexType);  \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL(ValueType, IndexType); \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX(ValueType, IndexType);              \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX(ValueType, IndexType);         \
+    template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL(ValueType, IndexType)
+
+
+namespace omp {
+namespace fbcsr {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace fbcsr
+}  // namespace omp
+
+
+namespace cuda {
+namespace fbcsr {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace fbcsr
+}  // namespace cuda
+
+
+namespace reference {
+namespace fbcsr {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace fbcsr
+}  // namespace reference
+
+
+namespace hip {
+namespace fbcsr {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace fbcsr
+}  // namespace hip
+
+
+#undef GKO_DECLARE_ALL_AS_TEMPLATES
+
+
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_CORE_MATRIX_FBCSR_KERNELS_HPP_
diff --git a/core/test/matrix/CMakeLists.txt b/core/test/matrix/CMakeLists.txt
index ac36ed4f7ff..64b3b3ed593 100644
--- a/core/test/matrix/CMakeLists.txt
+++ b/core/test/matrix/CMakeLists.txt
@@ -5,6 +5,8 @@ ginkgo_create_test(csr_builder)
 ginkgo_create_test(dense)
 ginkgo_create_test(diagonal)
 ginkgo_create_test(ell)
+ginkgo_create_test(fbcsr)
+ginkgo_create_test(fbcsr_builder)
 ginkgo_create_test(hybrid)
 ginkgo_create_test(identity)
 ginkgo_create_test(permutation)
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
new file mode 100644
index 00000000000..dd456ef7316
--- /dev/null
+++ b/core/test/matrix/fbcsr.cpp
@@ -0,0 +1,378 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/matrix/fbcsr.hpp>
+#include <ginkgo/core/matrix/matrix_strategies.hpp>
+
+
+#include <gtest/gtest.h>
+#include <iostream>
+#include <limits>
+
+
+#include "core/components/fixed_block.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+namespace matstr = gko::matrix::matrix_strategy;
+
+
+template <typename ValueIndexType>
+class Fbcsr : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using Mtx = gko::matrix::Fbcsr<value_type, index_type>;
+
+    Fbcsr()
+        : exec(gko::ReferenceExecutor::create()),
+          mtx(Mtx::create(exec, gko::dim<2>{6, 12}, 36, 3,
+                          std::make_shared<matstr::classical<Mtx>>()))
+    {
+        const int bs = 3;
+        value_type *const v = mtx->get_values();
+        index_type *const c = mtx->get_col_idxs();
+        index_type *const r = mtx->get_row_ptrs();
+        index_type *const s = mtx->get_srow();
+        r[0] = 0;
+        r[1] = 2;
+        r[2] = 4;
+        c[0] = 1;
+        c[1] = 3;
+        c[2] = 0;
+        c[3] = 2;
+
+        gko::blockutils::DenseBlocksView<value_type, index_type> vals(v, bs,
+                                                                      bs);
+
+        if (mtx->get_size()[0] % bs != 0)
+            throw gko::BadDimension(__FILE__, __LINE__, __func__, "test fbcsr",
+                                    mtx->get_size()[0], mtx->get_size()[1],
+                                    "block size does not divide the size!");
+
+        for (index_type ibrow = 0; ibrow < mtx->get_size()[0] / bs; ibrow++) {
+            const index_type *const browptr = mtx->get_row_ptrs();
+            for (index_type inz = browptr[ibrow]; inz < browptr[ibrow + 1];
+                 inz++) {
+                const index_type bcolind = mtx->get_col_idxs()[inz];
+                const value_type base = (ibrow + 1) * (bcolind + 1);
+                for (int ival = 0; ival < bs; ival++)
+                    for (int jval = 0; jval < bs; jval++)
+                        vals(inz, ival, jval) =
+                            base + static_cast<gko::remove_complex<value_type>>(
+                                       ival * bs + jval);
+            }
+        }
+
+        // Some of the entries are set to zero
+        vals(0, 2, 0) = gko::zero<value_type>();
+        vals(0, 2, 2) = gko::zero<value_type>();
+        vals(3, 0, 0) = gko::zero<value_type>();
+
+        for (index_type is = 0; is < mtx->get_num_srow_elements(); is++)
+            s[is] = 0;
+
+        // backup for move tests
+        orig_size = mtx->get_size();
+        orig_rowptrs.resize(3);
+        orig_colinds.resize(4);
+        orig_vals.resize(36);
+        for (index_type i = 0; i < 3; i++) orig_rowptrs[i] = r[i];
+        for (index_type i = 0; i < 4; i++) orig_colinds[i] = c[i];
+        for (index_type i = 0; i < 36; i++) orig_vals[i] = v[i];
+    }
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::unique_ptr<Mtx> mtx;
+
+    gko::dim<2> orig_size;
+    std::vector<value_type> orig_vals;
+    std::vector<index_type> orig_rowptrs;
+    std::vector<index_type> orig_colinds;
+
+    void assert_equal_to_original_mtx(const Mtx *m)
+    {
+        auto v = m->get_const_values();
+        auto c = m->get_const_col_idxs();
+        auto r = m->get_const_row_ptrs();
+        auto s = m->get_const_srow();
+
+        const int bs = 3;
+
+        ASSERT_EQ(m->get_size(), orig_size);
+        ASSERT_EQ(m->get_num_stored_elements(), orig_vals.size());
+        ASSERT_EQ(m->get_block_size(), bs);
+
+
+        for (index_type irow = 0; irow < orig_size[0] / bs; irow++) {
+            const index_type *const rowptr = &orig_rowptrs[0];
+            ASSERT_EQ(r[irow], rowptr[irow]);
+
+            for (index_type inz = rowptr[irow]; inz < rowptr[irow + 1]; inz++) {
+                ASSERT_EQ(c[inz], orig_colinds[inz]);
+
+                for (int i = 0; i < bs * bs; i++) {
+                    // ASSERT_LT(gko::abs(v[inz*bs*bs + i] -
+                    // mtx->get_values()[inz*bs*bs + i]),
+                    //           std::numeric_limits<gko::remove_complex<value_type>>::epsilon());
+                    ASSERT_EQ(v[inz * bs * bs + i],
+                              orig_vals[inz * bs * bs + i]);
+                }
+            }
+        }
+
+        ASSERT_EQ(m->get_num_srow_elements(), 0);
+        // for(index_type is = 0; is < mtx->get_num_srow_elements(); is++)
+        //     ASSERT_EQ(s[is], 0);
+    }
+
+    void assert_empty(const Mtx *m)
+    {
+        ASSERT_EQ(m->get_size(), gko::dim<2>(0, 0));
+        ASSERT_EQ(m->get_num_stored_elements(), 0);
+        ASSERT_EQ(m->get_block_size(), 1);
+        ASSERT_EQ(m->get_const_values(), nullptr);
+        ASSERT_EQ(m->get_const_col_idxs(), nullptr);
+        ASSERT_NE(m->get_const_row_ptrs(), nullptr);
+        ASSERT_EQ(m->get_const_srow(), nullptr);
+    }
+};
+
+TYPED_TEST_CASE(Fbcsr, gko::test::ValueIndexTypes);
+
+
+TYPED_TEST(Fbcsr, KnowsItsSize)
+{
+    ASSERT_EQ(this->mtx->get_size(), gko::dim<2>(6, 12));
+    ASSERT_EQ(this->mtx->get_block_size(), 3);
+    ASSERT_EQ(this->mtx->get_num_stored_elements(), 36);
+}
+
+
+TYPED_TEST(Fbcsr, ContainsCorrectData)
+{
+    this->assert_equal_to_original_mtx(this->mtx.get());
+}
+
+
+TYPED_TEST(Fbcsr, CanBeEmpty)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto mtx = Mtx::create(this->exec);
+
+    this->assert_empty(mtx.get());
+}
+
+
+TYPED_TEST(Fbcsr, CanBeCreatedFromExistingData)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+
+    constexpr int bs = 3;
+    constexpr index_type nbrows = 2;
+    constexpr index_type nbcols = 4;
+    constexpr index_type bnnz = 4;
+    value_type values[bnnz * bs * bs];
+    index_type col_idxs[] = {1, 3, 0, 2};
+    index_type row_ptrs[] = {0, 2, 4};
+
+    gko::blockutils::DenseBlocksView<value_type, index_type> vals(values, bs,
+                                                                  bs);
+
+    for (index_type ibrow = 0; ibrow < nbrows; ibrow++) {
+        for (index_type inz = row_ptrs[ibrow]; inz < row_ptrs[ibrow + 1];
+             inz++) {
+            const index_type bcolind = col_idxs[inz];
+            const value_type base = (ibrow + 1) * (bcolind + 1);
+            for (int ival = 0; ival < bs; ival++)
+                for (int jval = 0; jval < bs; jval++)
+                    vals(inz, ival, jval) =
+                        base + static_cast<gko::remove_complex<value_type>>(
+                                   ival * bs + jval);
+        }
+    }
+
+    // Some of the entries are set to zero
+    vals(0, 2, 0) = gko::zero<value_type>();
+    vals(0, 2, 2) = gko::zero<value_type>();
+    vals(3, 0, 0) = gko::zero<value_type>();
+
+    auto mtx = gko::matrix::Fbcsr<value_type, index_type>::create(
+        this->exec, gko::dim<2>{nbrows * bs, nbcols * bs}, bs,
+        gko::Array<value_type>::view(this->exec, bnnz * bs * bs, values),
+        gko::Array<index_type>::view(this->exec, bnnz, col_idxs),
+        gko::Array<index_type>::view(this->exec, nbrows + 1, row_ptrs),
+        std::make_shared<matstr::classical<Mtx>>());
+
+    ASSERT_EQ(mtx->get_num_srow_elements(), 0);
+    ASSERT_EQ(mtx->get_const_values(), values);
+    ASSERT_EQ(mtx->get_const_col_idxs(), col_idxs);
+    ASSERT_EQ(mtx->get_const_row_ptrs(), row_ptrs);
+    // ASSERT_EQ(mtx->get_const_srow()[0], 0);
+}
+
+
+TYPED_TEST(Fbcsr, CanBeCopied)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto copy = Mtx::create(this->exec);
+
+    copy->copy_from(this->mtx.get());
+
+    this->assert_equal_to_original_mtx(this->mtx.get());
+    this->mtx->get_values()[1] = 3.0;
+    this->assert_equal_to_original_mtx(copy.get());
+}
+
+
+TYPED_TEST(Fbcsr, CanBeMoved)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto copy = Mtx::create(this->exec);
+
+    copy->copy_from(std::move(this->mtx));
+
+    this->assert_equal_to_original_mtx(copy.get());
+}
+
+
+TYPED_TEST(Fbcsr, CanBeCloned)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto clone = this->mtx->clone();
+
+    this->assert_equal_to_original_mtx(this->mtx.get());
+    this->mtx->get_values()[1] = 5.0;
+    this->assert_equal_to_original_mtx(dynamic_cast<Mtx *>(clone.get()));
+}
+
+
+TYPED_TEST(Fbcsr, CanBeCleared)
+{
+    this->mtx->clear();
+
+    this->assert_empty(this->mtx.get());
+}
+
+
+TYPED_TEST(Fbcsr, CanBeReadFromMatrixData)
+{
+    // TODO (script:fbcsr): change the code imported from matrix/csr if needed
+    using Mtx = typename TestFixture::Mtx;
+    auto m =
+        Mtx::create(this->exec, std::make_shared<matstr::classical<Mtx>>());
+    m->set_block_size(3);
+
+    // Assuming row-major blocks
+    m->read(
+        {{6, 12}, {{0, 3, 2.0},   {0, 4, 3.0},  {0, 5, 4.0},  {1, 3, 5.0},
+                   {1, 4, 6.0},   {1, 5, 7.0},  {2, 4, 9.0},
+
+                   {0, 9, 4.0},   {0, 10, 5.0}, {0, 11, 6.0}, {1, 9, 7.0},
+                   {1, 10, 8.0},  {1, 11, 9.0}, {2, 9, 10.0}, {2, 10, 11.0},
+                   {2, 11, 12.0},
+
+                   {3, 0, 2.0},   {3, 1, 3.0},  {3, 2, 4.0},  {4, 0, 5.0},
+                   {4, 1, 6.0},   {4, 2, 7.0},  {5, 0, 8.0},  {5, 1, 9.0},
+                   {5, 2, 10.0},
+
+                   {3, 7, 7.0},   {3, 8, 8.0},  {4, 6, 9.0},  {4, 7, 10.0},
+                   {4, 8, 11.0},  {5, 6, 12.0}, {5, 7, 13.0}, {5, 8, 14.0}}});
+
+    this->assert_equal_to_original_mtx(m.get());
+}
+
+
+TYPED_TEST(Fbcsr, GeneratesCorrectMatrixData)
+{
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using tpl = typename gko::matrix_data<value_type, index_type>::nonzero_type;
+    gko::matrix_data<value_type, index_type> data;
+
+    this->mtx->write(data);
+    data.ensure_row_major_order();
+
+    ASSERT_EQ(data.size, gko::dim<2>(6, 12));
+    ASSERT_EQ(data.nonzeros.size(), 36);
+    EXPECT_EQ(data.nonzeros[0], tpl(0, 3, value_type{2.0}));
+    EXPECT_EQ(data.nonzeros[1], tpl(0, 4, value_type{3.0}));
+    EXPECT_EQ(data.nonzeros[2], tpl(0, 5, value_type{4.0}));
+    EXPECT_EQ(data.nonzeros[3], tpl(0, 9, value_type{4.0}));
+    EXPECT_EQ(data.nonzeros[4], tpl(0, 10, value_type{5.0}));
+    EXPECT_EQ(data.nonzeros[5], tpl(0, 11, value_type{6.0}));
+
+    EXPECT_EQ(data.nonzeros[6], tpl(1, 3, value_type{5.0}));
+    EXPECT_EQ(data.nonzeros[7], tpl(1, 4, value_type{6.0}));
+    EXPECT_EQ(data.nonzeros[8], tpl(1, 5, value_type{7.0}));
+    EXPECT_EQ(data.nonzeros[9], tpl(1, 9, value_type{7.0}));
+    EXPECT_EQ(data.nonzeros[10], tpl(1, 10, value_type{8.0}));
+    EXPECT_EQ(data.nonzeros[11], tpl(1, 11, value_type{9.0}));
+
+    EXPECT_EQ(data.nonzeros[12], tpl(2, 3, value_type{0.0}));
+    EXPECT_EQ(data.nonzeros[13], tpl(2, 4, value_type{9.0}));
+    EXPECT_EQ(data.nonzeros[14], tpl(2, 5, value_type{0.0}));
+    EXPECT_EQ(data.nonzeros[15], tpl(2, 9, value_type{10.0}));
+    EXPECT_EQ(data.nonzeros[16], tpl(2, 10, value_type{11.0}));
+    EXPECT_EQ(data.nonzeros[17], tpl(2, 11, value_type{12.0}));
+
+    EXPECT_EQ(data.nonzeros[18], tpl(3, 0, value_type{2.0}));
+    EXPECT_EQ(data.nonzeros[19], tpl(3, 1, value_type{3.0}));
+    EXPECT_EQ(data.nonzeros[20], tpl(3, 2, value_type{4.0}));
+    EXPECT_EQ(data.nonzeros[21], tpl(3, 6, value_type{0.0}));
+    EXPECT_EQ(data.nonzeros[22], tpl(3, 7, value_type{7.0}));
+    EXPECT_EQ(data.nonzeros[23], tpl(3, 8, value_type{8.0}));
+
+    EXPECT_EQ(data.nonzeros[24], tpl(4, 0, value_type{5.0}));
+    EXPECT_EQ(data.nonzeros[25], tpl(4, 1, value_type{6.0}));
+    EXPECT_EQ(data.nonzeros[26], tpl(4, 2, value_type{7.0}));
+    EXPECT_EQ(data.nonzeros[27], tpl(4, 6, value_type{9.0}));
+    EXPECT_EQ(data.nonzeros[28], tpl(4, 7, value_type{10.0}));
+    EXPECT_EQ(data.nonzeros[29], tpl(4, 8, value_type{11.0}));
+
+    EXPECT_EQ(data.nonzeros[30], tpl(5, 0, value_type{8.0}));
+    EXPECT_EQ(data.nonzeros[31], tpl(5, 1, value_type{9.0}));
+    EXPECT_EQ(data.nonzeros[32], tpl(5, 2, value_type{10.0}));
+    EXPECT_EQ(data.nonzeros[33], tpl(5, 6, value_type{12.0}));
+    EXPECT_EQ(data.nonzeros[34], tpl(5, 7, value_type{13.0}));
+    EXPECT_EQ(data.nonzeros[35], tpl(5, 8, value_type{14.0}));
+}
+
+
+}  // namespace
diff --git a/core/test/matrix/fbcsr_builder.cpp b/core/test/matrix/fbcsr_builder.cpp
new file mode 100644
index 00000000000..31ccc2145bc
--- /dev/null
+++ b/core/test/matrix/fbcsr_builder.cpp
@@ -0,0 +1,120 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/fbcsr_builder.hpp"
+
+
+#include <memory>
+
+
+#include <gtest/gtest.h>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+template <typename ValueIndexType>
+class FbcsrBuilder : public ::testing::Test {
+public:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using Mtx = gko::matrix::Fbcsr<value_type, index_type>;
+
+protected:
+    FbcsrBuilder()
+        : exec(gko::ReferenceExecutor::create()),
+          mtx(Mtx::create(exec, gko::dim<2>{2, 3}, 4))
+    {}
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::unique_ptr<Mtx> mtx;
+};
+
+TYPED_TEST_CASE(FbcsrBuilder, gko::test::ValueIndexTypes);
+
+
+TYPED_TEST(FbcsrBuilder, ReturnsCorrectArrays)
+{
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    gko::matrix::FbcsrBuilder<value_type, index_type> builder{this->mtx.get()};
+
+    auto builder_col_idxs = builder.get_col_idx_array().get_data();
+    auto builder_values = builder.get_value_array().get_data();
+    auto ref_col_idxs = this->mtx->get_col_idxs();
+    auto ref_values = this->mtx->get_values();
+
+    ASSERT_EQ(builder_col_idxs, ref_col_idxs);
+    ASSERT_EQ(builder_values, ref_values);
+}
+
+
+TYPED_TEST(FbcsrBuilder, UpdatesSrowOnDestruction)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    struct mock_strategy
+        : public gko::matrix::matrix_strategy::strategy_type<Mtx> {
+        virtual void process(const gko::Array<index_type> &,
+                             gko::Array<index_type> *) override
+        {
+            *was_called = true;
+        }
+
+        virtual int64_t calc_size(const int64_t nnz) override { return 0; }
+
+        virtual std::shared_ptr<typename Mtx::strategy_type> copy() override
+        {
+            return std::make_shared<mock_strategy>(*was_called);
+        }
+
+        mock_strategy(bool &flag) : Mtx::strategy_type(""), was_called(&flag) {}
+
+        bool *was_called;
+    };
+    bool was_called{};
+    this->mtx->set_strategy(std::make_shared<mock_strategy>(was_called));
+    was_called = false;
+
+    gko::matrix::FbcsrBuilder<value_type, index_type>{this->mtx.get()};
+
+    ASSERT_TRUE(was_called);
+}
+
+
+}  // namespace
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index a4ded75abed..842f4b75259 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -98,6 +98,7 @@ target_sources(ginkgo_cuda
     matrix/dense_kernels.cu
     matrix/diagonal_kernels.cu
     matrix/ell_kernels.cu
+    matrix/fbcsr_kernels.cu
     matrix/hybrid_kernels.cu
     matrix/sellp_kernels.cu
     matrix/sparsity_csr_kernels.cu
diff --git a/cuda/matrix/fbcsr_kernels.cu b/cuda/matrix/fbcsr_kernels.cu
new file mode 100644
index 00000000000..5d67bcc90d3
--- /dev/null
+++ b/cuda/matrix/fbcsr_kernels.cu
@@ -0,0 +1,1446 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/fbcsr_kernels.hpp"
+
+
+#include <algorithm>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/ell.hpp>
+#include <ginkgo/core/matrix/hybrid.hpp>
+#include <ginkgo/core/matrix/sellp.hpp>
+
+
+#include "core/components/fill_array.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/dense_kernels.hpp"
+#include "core/matrix/fbcsr_builder.hpp"
+#include "core/synthesizer/implementation_selection.hpp"
+#include "cuda/base/config.hpp"
+#include "cuda/base/cusparse_bindings.hpp"
+#include "cuda/base/math.hpp"
+#include "cuda/base/pointer_mode_guard.hpp"
+#include "cuda/base/types.hpp"
+#include "cuda/components/atomic.cuh"
+#include "cuda/components/cooperative_groups.cuh"
+#include "cuda/components/intrinsics.cuh"
+#include "cuda/components/merging.cuh"
+#include "cuda/components/reduction.cuh"
+#include "cuda/components/segment_scan.cuh"
+#include "cuda/components/thread_ids.cuh"
+#include "cuda/components/uninitialized_array.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+/**
+ * @brief The fixed-size block compressed sparse row matrix format namespace.
+ *
+ * @ingroup fbcsr
+ */
+namespace fbcsr {
+
+
+constexpr int default_block_size = 512;
+constexpr int warps_in_block = 4;
+constexpr int spmv_block_size = warps_in_block * config::warp_size;
+constexpr int wsize = config::warp_size;
+constexpr int classical_overweight = 32;
+
+
+/**
+ * A compile-time list of the number items per threads for which spmv kernel
+ * should be compiled.
+ */
+using compiled_kernels = syn::value_list<int, 3, 4, 6, 7, 8, 12, 14>;
+
+using classical_kernels =
+    syn::value_list<int, config::warp_size, 32, 16, 8, 4, 2, 1>;
+
+using spgeam_kernels =
+    syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
+
+
+namespace host_kernel {
+
+
+template <int items_per_thread, typename ValueType, typename IndexType>
+void merge_path_spmv(
+    syn::value_list<int, items_per_thread>,
+    std::shared_ptr<const CudaExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *a,
+    const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c,
+    const matrix::Dense<ValueType> *alpha = nullptr,
+    const matrix::Dense<ValueType> *beta = nullptr) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const IndexType total = a->get_size()[0] + a->get_num_stored_elements();
+//    const IndexType grid_num =
+//        ceildiv(total, spmv_block_size * items_per_thread);
+//    const dim3 grid(grid_num);
+//    const dim3 block(spmv_block_size);
+//    Array<IndexType> row_out(exec, grid_num);
+//    Array<ValueType> val_out(exec, grid_num);
+//
+//    for (IndexType column_id = 0; column_id < b->get_size()[1]; column_id++) {
+//        if (alpha == nullptr && beta == nullptr) {
+//            const auto b_vals = b->get_const_values() + column_id;
+//            auto c_vals = c->get_values() + column_id;
+//            kernel::abstract_merge_path_spmv<items_per_thread>
+//                <<<grid, block, 0, 0>>>(
+//                    static_cast<IndexType>(a->get_size()[0]),
+//                    as_cuda_type(a->get_const_values()),
+//                    a->get_const_col_idxs(),
+//                    as_cuda_type(a->get_const_row_ptrs()),
+//                    as_cuda_type(a->get_const_srow()), as_cuda_type(b_vals),
+//                    b->get_stride(), as_cuda_type(c_vals), c->get_stride(),
+//                    as_cuda_type(row_out.get_data()),
+//                    as_cuda_type(val_out.get_data()));
+//            kernel::abstract_reduce<<<1, spmv_block_size>>>(
+//                grid_num, as_cuda_type(val_out.get_data()),
+//                as_cuda_type(row_out.get_data()), as_cuda_type(c_vals),
+//                c->get_stride());
+//
+//        } else if (alpha != nullptr && beta != nullptr) {
+//            const auto b_vals = b->get_const_values() + column_id;
+//            auto c_vals = c->get_values() + column_id;
+//            kernel::abstract_merge_path_spmv<items_per_thread>
+//                <<<grid, block, 0, 0>>>(
+//                    static_cast<IndexType>(a->get_size()[0]),
+//                    as_cuda_type(alpha->get_const_values()),
+//                    as_cuda_type(a->get_const_values()),
+//                    a->get_const_col_idxs(),
+//                    as_cuda_type(a->get_const_row_ptrs()),
+//                    as_cuda_type(a->get_const_srow()), as_cuda_type(b_vals),
+//                    b->get_stride(), as_cuda_type(beta->get_const_values()),
+//                    as_cuda_type(c_vals), c->get_stride(),
+//                    as_cuda_type(row_out.get_data()),
+//                    as_cuda_type(val_out.get_data()));
+//            kernel::abstract_reduce<<<1, spmv_block_size>>>(
+//                grid_num, as_cuda_type(val_out.get_data()),
+//                as_cuda_type(row_out.get_data()),
+//                as_cuda_type(alpha->get_const_values()), as_cuda_type(c_vals),
+//                c->get_stride());
+//        } else {
+//            GKO_KERNEL_NOT_FOUND;
+//        }
+//    }
+//}
+
+GKO_ENABLE_IMPLEMENTATION_SELECTION(select_merge_path_spmv, merge_path_spmv);
+
+
+template <typename ValueType, typename IndexType>
+int compute_items_per_thread(std::shared_ptr<const CudaExecutor> exec)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const int version =
+//        (exec->get_major_version() << 4) + exec->get_minor_version();
+//    // The num_item is decided to make the occupancy 100%
+//    // TODO: Extend this list when new GPU is released
+//    //       Tune this parameter
+//    // 128 threads/block the number of items per threads
+//    // 3.0 3.5: 6
+//    // 3.7: 14
+//    // 5.0, 5.3, 6.0, 6.2: 8
+//    // 5.2, 6.1, 7.0: 12
+//    int num_item = 6;
+//    switch (version) {
+//    case 0x50:
+//    case 0x53:
+//    case 0x60:
+//    case 0x62:
+//        num_item = 8;
+//        break;
+//    case 0x52:
+//    case 0x61:
+//    case 0x70:
+//        num_item = 12;
+//        break;
+//    case 0x37:
+//        num_item = 14;
+//    }
+//    // Ensure that the following is satisfied:
+//    // sizeof(IndexType) + sizeof(ValueType)
+//    // <= items_per_thread * sizeof(IndexType)
+//    constexpr int minimal_num =
+//        ceildiv(sizeof(IndexType) + sizeof(ValueType), sizeof(IndexType));
+//    int items_per_thread = num_item * 4 / sizeof(IndexType);
+//    return std::max(minimal_num, items_per_thread);
+//}
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+void classical_spmv(
+    syn::value_list<int, subwarp_size>,
+    std::shared_ptr<const CudaExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *a,
+    const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c,
+    const matrix::Dense<ValueType> *alpha = nullptr,
+    const matrix::Dense<ValueType> *beta = nullptr) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto nwarps = exec->get_num_warps_per_sm() *
+//                        exec->get_num_multiprocessor() * classical_overweight;
+//    const auto gridx =
+//        std::min(ceildiv(a->get_size()[0], spmv_block_size / subwarp_size),
+//                 int64(nwarps / warps_in_block));
+//    const dim3 grid(gridx, b->get_size()[1]);
+//    const dim3 block(spmv_block_size);
+//
+//    if (alpha == nullptr && beta == nullptr) {
+//        kernel::abstract_classical_spmv<subwarp_size><<<grid, block, 0, 0>>>(
+//            a->get_size()[0], as_cuda_type(a->get_const_values()),
+//            a->get_const_col_idxs(), as_cuda_type(a->get_const_row_ptrs()),
+//            as_cuda_type(b->get_const_values()), b->get_stride(),
+//            as_cuda_type(c->get_values()), c->get_stride());
+//
+//    } else if (alpha != nullptr && beta != nullptr) {
+//        kernel::abstract_classical_spmv<subwarp_size><<<grid, block, 0, 0>>>(
+//            a->get_size()[0], as_cuda_type(alpha->get_const_values()),
+//            as_cuda_type(a->get_const_values()), a->get_const_col_idxs(),
+//            as_cuda_type(a->get_const_row_ptrs()),
+//            as_cuda_type(b->get_const_values()), b->get_stride(),
+//            as_cuda_type(beta->get_const_values()),
+//            as_cuda_type(c->get_values()), c->get_stride());
+//    } else {
+//        GKO_KERNEL_NOT_FOUND;
+//    }
+//}
+
+GKO_ENABLE_IMPLEMENTATION_SELECTION(select_classical_spmv, classical_spmv);
+
+
+}  // namespace host_kernel
+
+
+template <typename ValueType, typename IndexType>
+void spmv(std::shared_ptr<const CudaExecutor> exec,
+          const matrix::Fbcsr<ValueType, IndexType> *a,
+          const matrix::Dense<ValueType> *b,
+          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    if (a->get_strategy()->get_name() == "load_balance") {
+//        components::fill_array(exec, c->get_values(),
+//                               c->get_num_stored_elements(),
+//                               zero<ValueType>());
+//        const IndexType nwarps = a->get_num_srow_elements();
+//        if (nwarps > 0) {
+//            const dim3 fbcsr_block(config::warp_size, warps_in_block, 1);
+//            const dim3 fbcsr_grid(ceildiv(nwarps, warps_in_block),
+//                                b->get_size()[1]);
+//            kernel::abstract_spmv<<<fbcsr_grid, fbcsr_block>>>(
+//                nwarps, static_cast<IndexType>(a->get_size()[0]),
+//                as_cuda_type(a->get_const_values()), a->get_const_col_idxs(),
+//                as_cuda_type(a->get_const_row_ptrs()),
+//                as_cuda_type(a->get_const_srow()),
+//                as_cuda_type(b->get_const_values()),
+//                as_cuda_type(b->get_stride()), as_cuda_type(c->get_values()),
+//                as_cuda_type(c->get_stride()));
+//        } else {
+//            GKO_NOT_SUPPORTED(nwarps);
+//        }
+//    } else if (a->get_strategy()->get_name() == "merge_path") {
+//        int items_per_thread =
+//            host_kernel::compute_items_per_thread<ValueType, IndexType>(exec);
+//        host_kernel::select_merge_path_spmv(
+//            compiled_kernels(),
+//            [&items_per_thread](int compiled_info) {
+//                return items_per_thread == compiled_info;
+//            },
+//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c);
+//    } else if (a->get_strategy()->get_name() == "classical") {
+//        IndexType max_length_per_row = 0;
+//        using Tfbcsr = matrix::Fbcsr<ValueType, IndexType>;
+//        if (auto strategy =
+//                std::dynamic_pointer_cast<const typename Tfbcsr::classical>(
+//                    a->get_strategy())) {
+//            max_length_per_row = strategy->get_max_length_per_row();
+//        } else if (auto strategy = std::dynamic_pointer_cast<
+//                       const typename Tfbcsr::automatical>(a->get_strategy()))
+//                       {
+//            max_length_per_row = strategy->get_max_length_per_row();
+//        } else {
+//            GKO_NOT_SUPPORTED(a->get_strategy());
+//        }
+//        host_kernel::select_classical_spmv(
+//            classical_kernels(),
+//            [&max_length_per_row](int compiled_info) {
+//                return max_length_per_row >= compiled_info;
+//            },
+//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c);
+//    } else if (a->get_strategy()->get_name() == "sparselib" ||
+//               a->get_strategy()->get_name() == "cusparse") {
+//        if (cusparse::is_supported<ValueType, IndexType>::value) {
+//            // TODO: add implementation for int64 and multiple RHS
+//            auto handle = exec->get_cusparse_handle();
+//            {
+//                cusparse::pointer_mode_guard pm_guard(handle);
+//                const auto alpha = one<ValueType>();
+//                const auto beta = zero<ValueType>();
+//                // TODO: add implementation for int64 and multiple RHS
+//                if (b->get_stride() != 1 || c->get_stride() != 1)
+//                    GKO_NOT_IMPLEMENTED;
+//
+//#if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
+//                auto descr = cusparse::create_mat_descr();
+//                auto row_ptrs = a->get_const_row_ptrs();
+//                auto col_idxs = a->get_const_col_idxs();
+//                cusparse::spmv(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
+//                               a->get_size()[0], a->get_size()[1],
+//                               a->get_num_stored_elements(), &alpha, descr,
+//                               a->get_const_values(), row_ptrs, col_idxs,
+//                               b->get_const_values(), &beta, c->get_values());
+//
+//                cusparse::destroy(descr);
+//#else  // CUDA_VERSION >= 11000
+//                cusparseOperation_t trans = CUSPARSE_OPERATION_NON_TRANSPOSE;
+//                cusparseSpMVAlg_t alg = CUSPARSE_FBCSRMV_ALG1;
+//                auto row_ptrs =
+//                    const_cast<IndexType *>(a->get_const_row_ptrs());
+//                auto col_idxs =
+//                    const_cast<IndexType *>(a->get_const_col_idxs());
+//                auto values = const_cast<ValueType *>(a->get_const_values());
+//                auto mat = cusparse::create_fbcsr(
+//                    a->get_size()[0], a->get_size()[1],
+//                    a->get_num_stored_elements(), row_ptrs, col_idxs, values);
+//                auto b_val = const_cast<ValueType *>(b->get_const_values());
+//                auto c_val = c->get_values();
+//                auto vecb =
+//                    cusparse::create_dnvec(b->get_num_stored_elements(),
+//                    b_val);
+//                auto vecc =
+//                    cusparse::create_dnvec(c->get_num_stored_elements(),
+//                    c_val);
+//                size_type buffer_size = 0;
+//                cusparse::spmv_buffersize<ValueType>(handle, trans, &alpha,
+//                mat,
+//                                                     vecb, &beta, vecc, alg,
+//                                                     &buffer_size);
+//
+//                gko::Array<char> buffer_array(exec, buffer_size);
+//                auto buffer = buffer_array.get_data();
+//                cusparse::spmv<ValueType>(handle, trans, &alpha, mat, vecb,
+//                                          &beta, vecc, alg, buffer);
+//                cusparse::destroy(vecb);
+//                cusparse::destroy(vecc);
+//                cusparse::destroy(mat);
+//#endif
+//            }
+//        } else {
+//            GKO_NOT_IMPLEMENTED;
+//        }
+//    } else {
+//        GKO_NOT_IMPLEMENTED;
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spmv(std::shared_ptr<const CudaExecutor> exec,
+                   const matrix::Dense<ValueType> *alpha,
+                   const matrix::Fbcsr<ValueType, IndexType> *a,
+                   const matrix::Dense<ValueType> *b,
+                   const matrix::Dense<ValueType> *beta,
+                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    if (a->get_strategy()->get_name() == "load_balance") {
+//        dense::scale(exec, beta, c);
+//
+//        const IndexType nwarps = a->get_num_srow_elements();
+//
+//        if (nwarps > 0) {
+//            const dim3 fbcsr_block(config::warp_size, warps_in_block, 1);
+//            const dim3 fbcsr_grid(ceildiv(nwarps, warps_in_block),
+//                                b->get_size()[1]);
+//            kernel::abstract_spmv<<<fbcsr_grid, fbcsr_block>>>(
+//                nwarps, static_cast<IndexType>(a->get_size()[0]),
+//                as_cuda_type(alpha->get_const_values()),
+//                as_cuda_type(a->get_const_values()), a->get_const_col_idxs(),
+//                as_cuda_type(a->get_const_row_ptrs()),
+//                as_cuda_type(a->get_const_srow()),
+//                as_cuda_type(b->get_const_values()),
+//                as_cuda_type(b->get_stride()), as_cuda_type(c->get_values()),
+//                as_cuda_type(c->get_stride()));
+//        } else {
+//            GKO_NOT_SUPPORTED(nwarps);
+//        }
+//    } else if (a->get_strategy()->get_name() == "sparselib" ||
+//               a->get_strategy()->get_name() == "cusparse") {
+//        if (cusparse::is_supported<ValueType, IndexType>::value) {
+//            // TODO: add implementation for int64 and multiple RHS
+//            if (b->get_stride() != 1 || c->get_stride() != 1)
+//                GKO_NOT_IMPLEMENTED;
+//
+//#if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
+//            auto descr = cusparse::create_mat_descr();
+//            auto row_ptrs = a->get_const_row_ptrs();
+//            auto col_idxs = a->get_const_col_idxs();
+//            cusparse::spmv(exec->get_cusparse_handle(),
+//                           CUSPARSE_OPERATION_NON_TRANSPOSE, a->get_size()[0],
+//                           a->get_size()[1], a->get_num_stored_elements(),
+//                           alpha->get_const_values(), descr,
+//                           a->get_const_values(), row_ptrs, col_idxs,
+//                           b->get_const_values(), beta->get_const_values(),
+//                           c->get_values());
+//
+//            cusparse::destroy(descr);
+//#else  // CUDA_VERSION >= 11000
+//            cusparseOperation_t trans = CUSPARSE_OPERATION_NON_TRANSPOSE;
+//            cusparseSpMVAlg_t alg = CUSPARSE_FBCSRMV_ALG1;
+//            auto row_ptrs = const_cast<IndexType *>(a->get_const_row_ptrs());
+//            auto col_idxs = const_cast<IndexType *>(a->get_const_col_idxs());
+//            auto values = const_cast<ValueType *>(a->get_const_values());
+//            auto mat = cusparse::create_fbcsr(a->get_size()[0],
+//            a->get_size()[1],
+//                                            a->get_num_stored_elements(),
+//                                            row_ptrs, col_idxs, values);
+//            auto b_val = const_cast<ValueType *>(b->get_const_values());
+//            auto c_val = c->get_values();
+//            auto vecb =
+//                cusparse::create_dnvec(b->get_num_stored_elements(), b_val);
+//            auto vecc =
+//                cusparse::create_dnvec(c->get_num_stored_elements(), c_val);
+//            size_type buffer_size = 0;
+//            cusparse::spmv_buffersize<ValueType>(
+//                exec->get_cusparse_handle(), trans, alpha->get_const_values(),
+//                mat, vecb, beta->get_const_values(), vecc, alg, &buffer_size);
+//            gko::Array<char> buffer_array(exec, buffer_size);
+//            auto buffer = buffer_array.get_data();
+//            cusparse::spmv<ValueType>(
+//                exec->get_cusparse_handle(), trans, alpha->get_const_values(),
+//                mat, vecb, beta->get_const_values(), vecc, alg, buffer);
+//            cusparse::destroy(vecb);
+//            cusparse::destroy(vecc);
+//            cusparse::destroy(mat);
+//#endif
+//        } else {
+//            GKO_NOT_IMPLEMENTED;
+//        }
+//    } else if (a->get_strategy()->get_name() == "classical") {
+//        IndexType max_length_per_row = 0;
+//        using Tfbcsr = matrix::Fbcsr<ValueType, IndexType>;
+//        if (auto strategy =
+//                std::dynamic_pointer_cast<const typename Tfbcsr::classical>(
+//                    a->get_strategy())) {
+//            max_length_per_row = strategy->get_max_length_per_row();
+//        } else if (auto strategy = std::dynamic_pointer_cast<
+//                       const typename Tfbcsr::automatical>(a->get_strategy()))
+//                       {
+//            max_length_per_row = strategy->get_max_length_per_row();
+//        } else {
+//            GKO_NOT_SUPPORTED(a->get_strategy());
+//        }
+//        host_kernel::select_classical_spmv(
+//            classical_kernels(),
+//            [&max_length_per_row](int compiled_info) {
+//                return max_length_per_row >= compiled_info;
+//            },
+//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c, alpha,
+//            beta);
+//    } else if (a->get_strategy()->get_name() == "merge_path") {
+//        int items_per_thread =
+//            host_kernel::compute_items_per_thread<ValueType, IndexType>(exec);
+//        host_kernel::select_merge_path_spmv(
+//            compiled_kernels(),
+//            [&items_per_thread](int compiled_info) {
+//                return items_per_thread == compiled_info;
+//            },
+//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c, alpha,
+//            beta);
+//    } else {
+//        GKO_NOT_IMPLEMENTED;
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void spgemm(std::shared_ptr<const CudaExecutor> exec,
+            const matrix::Fbcsr<ValueType, IndexType> *a,
+            const matrix::Fbcsr<ValueType, IndexType> *b,
+            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto a_nnz = IndexType(a->get_num_stored_elements());
+//    auto a_vals = a->get_const_values();
+//    auto a_row_ptrs = a->get_const_row_ptrs();
+//    auto a_col_idxs = a->get_const_col_idxs();
+//    auto b_vals = b->get_const_values();
+//    auto b_row_ptrs = b->get_const_row_ptrs();
+//    auto b_col_idxs = b->get_const_col_idxs();
+//    auto c_row_ptrs = c->get_row_ptrs();
+//
+//    if (cusparse::is_supported<ValueType, IndexType>::value) {
+//        auto handle = exec->get_cusparse_handle();
+//        cusparse::pointer_mode_guard pm_guard(handle);
+//
+//        auto alpha = one<ValueType>();
+//        auto a_nnz = static_cast<IndexType>(a->get_num_stored_elements());
+//        auto b_nnz = static_cast<IndexType>(b->get_num_stored_elements());
+//        auto null_value = static_cast<ValueType *>(nullptr);
+//        auto null_index = static_cast<IndexType *>(nullptr);
+//        auto zero_nnz = IndexType{};
+//        auto m = IndexType(a->get_size()[0]);
+//        auto n = IndexType(b->get_size()[1]);
+//        auto k = IndexType(a->get_size()[1]);
+//        matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
+//        auto &c_col_idxs_array = c_builder.get_col_idx_array();
+//        auto &c_vals_array = c_builder.get_value_array();
+//
+//#if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
+//        auto a_descr = cusparse::create_mat_descr();
+//        auto b_descr = cusparse::create_mat_descr();
+//        auto c_descr = cusparse::create_mat_descr();
+//        auto d_descr = cusparse::create_mat_descr();
+//        auto info = cusparse::create_spgemm_info();
+//        // allocate buffer
+//        size_type buffer_size{};
+//        cusparse::spgemm_buffer_size(
+//            handle, m, n, k, &alpha, a_descr, a_nnz, a_row_ptrs, a_col_idxs,
+//            b_descr, b_nnz, b_row_ptrs, b_col_idxs, null_value, d_descr,
+//            zero_nnz, null_index, null_index, info, buffer_size);
+//        Array<char> buffer_array(exec, buffer_size);
+//        auto buffer = buffer_array.get_data();
+//
+//        // count nnz
+//        IndexType c_nnz{};
+//        cusparse::spgemm_nnz(handle, m, n, k, a_descr, a_nnz, a_row_ptrs,
+//                             a_col_idxs, b_descr, b_nnz, b_row_ptrs,
+//                             b_col_idxs, d_descr, zero_nnz, null_index,
+//                             null_index, c_descr, c_row_ptrs, &c_nnz, info,
+//                             buffer);
+//
+//        // accumulate non-zeros
+//        c_col_idxs_array.resize_and_reset(c_nnz);
+//        c_vals_array.resize_and_reset(c_nnz);
+//        auto c_col_idxs = c_col_idxs_array.get_data();
+//        auto c_vals = c_vals_array.get_data();
+//        cusparse::spgemm(handle, m, n, k, &alpha, a_descr, a_nnz, a_vals,
+//                         a_row_ptrs, a_col_idxs, b_descr, b_nnz, b_vals,
+//                         b_row_ptrs, b_col_idxs, null_value, d_descr,
+//                         zero_nnz, null_value, null_index, null_index,
+//                         c_descr, c_vals, c_row_ptrs, c_col_idxs, info,
+//                         buffer);
+//
+//        cusparse::destroy(info);
+//        cusparse::destroy(d_descr);
+//        cusparse::destroy(c_descr);
+//        cusparse::destroy(b_descr);
+//        cusparse::destroy(a_descr);
+//
+//#else   // CUDA_VERSION >= 11000
+//        const auto beta = zero<ValueType>();
+//        auto spgemm_descr = cusparse::create_spgemm_descr();
+//        auto a_descr = cusparse::create_fbcsr(m, k, a_nnz,
+//                                            const_cast<IndexType
+//                                            *>(a_row_ptrs),
+//                                            const_cast<IndexType
+//                                            *>(a_col_idxs),
+//                                            const_cast<ValueType *>(a_vals));
+//        auto b_descr = cusparse::create_fbcsr(k, n, b_nnz,
+//                                            const_cast<IndexType
+//                                            *>(b_row_ptrs),
+//                                            const_cast<IndexType
+//                                            *>(b_col_idxs),
+//                                            const_cast<ValueType *>(b_vals));
+//        auto c_descr = cusparse::create_fbcsr(m, n, zero_nnz, null_index,
+//                                            null_index, null_value);
+//
+//        // estimate work
+//        size_type buffer1_size{};
+//        cusparse::spgemm_work_estimation(handle, &alpha, a_descr, b_descr,
+//                                         &beta, c_descr, spgemm_descr,
+//                                         buffer1_size, nullptr);
+//        Array<char> buffer1{exec, buffer1_size};
+//        cusparse::spgemm_work_estimation(handle, &alpha, a_descr, b_descr,
+//                                         &beta, c_descr, spgemm_descr,
+//                                         buffer1_size, buffer1.get_data());
+//
+//        // compute spgemm
+//        size_type buffer2_size{};
+//        cusparse::spgemm_compute(handle, &alpha, a_descr, b_descr, &beta,
+//                                 c_descr, spgemm_descr, buffer1.get_data(),
+//                                 buffer2_size, nullptr);
+//        Array<char> buffer2{exec, buffer2_size};
+//        cusparse::spgemm_compute(handle, &alpha, a_descr, b_descr, &beta,
+//                                 c_descr, spgemm_descr, buffer1.get_data(),
+//                                 buffer2_size, buffer2.get_data());
+//
+//        // copy data to result
+//        auto c_nnz = cusparse::sparse_matrix_nnz(c_descr);
+//        c_col_idxs_array.resize_and_reset(c_nnz);
+//        c_vals_array.resize_and_reset(c_nnz);
+//        cusparse::fbcsr_set_pointers(c_descr, c_row_ptrs,
+//                                   c_col_idxs_array.get_data(),
+//                                   c_vals_array.get_data());
+//
+//        cusparse::spgemm_copy(handle, &alpha, a_descr, b_descr, &beta,
+//        c_descr,
+//                              spgemm_descr);
+//
+//        cusparse::destroy(c_descr);
+//        cusparse::destroy(b_descr);
+//        cusparse::destroy(a_descr);
+//        cusparse::destroy(spgemm_descr);
+//#endif  // CUDA_VERSION >= 11000
+//    } else {
+//        GKO_NOT_IMPLEMENTED;
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEMM_KERNEL);
+
+
+namespace {
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+void spgeam(syn::value_list<int, subwarp_size>,
+            std::shared_ptr<const DefaultExecutor> exec, const ValueType *alpha,
+            const IndexType *a_row_ptrs, const IndexType *a_col_idxs,
+            const ValueType *a_vals, const ValueType *beta,
+            const IndexType *b_row_ptrs, const IndexType *b_col_idxs,
+            const ValueType *b_vals,
+            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto m = static_cast<IndexType>(c->get_size()[0]);
+//    auto c_row_ptrs = c->get_row_ptrs();
+//    // count nnz for alpha * A + beta * B
+//    auto subwarps_per_block = default_block_size / subwarp_size;
+//    auto num_blocks = ceildiv(m, subwarps_per_block);
+//    kernel::spgeam_nnz<subwarp_size><<<num_blocks, default_block_size>>>(
+//        a_row_ptrs, a_col_idxs, b_row_ptrs, b_col_idxs, m, c_row_ptrs);
+//
+//    // build row pointers
+//    components::prefix_sum(exec, c_row_ptrs, m + 1);
+//
+//    // accumulate non-zeros for alpha * A + beta * B
+//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
+//    auto c_nnz = exec->copy_val_to_host(c_row_ptrs + m);
+//    c_builder.get_col_idx_array().resize_and_reset(c_nnz);
+//    c_builder.get_value_array().resize_and_reset(c_nnz);
+//    auto c_col_idxs = c->get_col_idxs();
+//    auto c_vals = c->get_values();
+//    kernel::spgeam<subwarp_size><<<num_blocks, default_block_size>>>(
+//        as_cuda_type(alpha), a_row_ptrs, a_col_idxs, as_cuda_type(a_vals),
+//        as_cuda_type(beta), b_row_ptrs, b_col_idxs, as_cuda_type(b_vals), m,
+//        c_row_ptrs, c_col_idxs, as_cuda_type(c_vals));
+//}
+
+GKO_ENABLE_IMPLEMENTATION_SELECTION(select_spgeam, spgeam);
+
+
+}  // namespace
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spgemm(std::shared_ptr<const CudaExecutor> exec,
+                     const matrix::Dense<ValueType> *alpha,
+                     const matrix::Fbcsr<ValueType, IndexType> *a,
+                     const matrix::Fbcsr<ValueType, IndexType> *b,
+                     const matrix::Dense<ValueType> *beta,
+                     const matrix::Fbcsr<ValueType, IndexType> *d,
+                     matrix::Fbcsr<ValueType, IndexType> *c)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    if (cusparse::is_supported<ValueType, IndexType>::value) {
+//        auto handle = exec->get_cusparse_handle();
+//        cusparse::pointer_mode_guard pm_guard(handle);
+//
+//        auto valpha = exec->copy_val_to_host(alpha->get_const_values());
+//        auto a_nnz = IndexType(a->get_num_stored_elements());
+//        auto a_vals = a->get_const_values();
+//        auto a_row_ptrs = a->get_const_row_ptrs();
+//        auto a_col_idxs = a->get_const_col_idxs();
+//        auto b_nnz = IndexType(b->get_num_stored_elements());
+//        auto b_vals = b->get_const_values();
+//        auto b_row_ptrs = b->get_const_row_ptrs();
+//        auto b_col_idxs = b->get_const_col_idxs();
+//        auto vbeta = exec->copy_val_to_host(beta->get_const_values());
+//        auto d_nnz = IndexType(d->get_num_stored_elements());
+//        auto d_vals = d->get_const_values();
+//        auto d_row_ptrs = d->get_const_row_ptrs();
+//        auto d_col_idxs = d->get_const_col_idxs();
+//        auto m = IndexType(a->get_size()[0]);
+//        auto n = IndexType(b->get_size()[1]);
+//        auto k = IndexType(a->get_size()[1]);
+//        auto c_row_ptrs = c->get_row_ptrs();
+//
+//#if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
+//        matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
+//        auto &c_col_idxs_array = c_builder.get_col_idx_array();
+//        auto &c_vals_array = c_builder.get_value_array();
+//        auto a_descr = cusparse::create_mat_descr();
+//        auto b_descr = cusparse::create_mat_descr();
+//        auto c_descr = cusparse::create_mat_descr();
+//        auto d_descr = cusparse::create_mat_descr();
+//        auto info = cusparse::create_spgemm_info();
+//        // allocate buffer
+//        size_type buffer_size{};
+//        cusparse::spgemm_buffer_size(
+//            handle, m, n, k, &valpha, a_descr, a_nnz, a_row_ptrs, a_col_idxs,
+//            b_descr, b_nnz, b_row_ptrs, b_col_idxs, &vbeta, d_descr, d_nnz,
+//            d_row_ptrs, d_col_idxs, info, buffer_size);
+//        Array<char> buffer_array(exec, buffer_size);
+//        auto buffer = buffer_array.get_data();
+//
+//        // count nnz
+//        IndexType c_nnz{};
+//        cusparse::spgemm_nnz(handle, m, n, k, a_descr, a_nnz, a_row_ptrs,
+//                             a_col_idxs, b_descr, b_nnz, b_row_ptrs,
+//                             b_col_idxs, d_descr, d_nnz, d_row_ptrs,
+//                             d_col_idxs, c_descr, c_row_ptrs, &c_nnz, info,
+//                             buffer);
+//
+//        // accumulate non-zeros
+//        c_col_idxs_array.resize_and_reset(c_nnz);
+//        c_vals_array.resize_and_reset(c_nnz);
+//        auto c_col_idxs = c_col_idxs_array.get_data();
+//        auto c_vals = c_vals_array.get_data();
+//        cusparse::spgemm(handle, m, n, k, &valpha, a_descr, a_nnz, a_vals,
+//                         a_row_ptrs, a_col_idxs, b_descr, b_nnz, b_vals,
+//                         b_row_ptrs, b_col_idxs, &vbeta, d_descr, d_nnz,
+//                         d_vals, d_row_ptrs, d_col_idxs, c_descr, c_vals,
+//                         c_row_ptrs, c_col_idxs, info, buffer);
+//
+//        cusparse::destroy(info);
+//        cusparse::destroy(d_descr);
+//        cusparse::destroy(c_descr);
+//        cusparse::destroy(b_descr);
+//        cusparse::destroy(a_descr);
+//#else   // CUDA_VERSION >= 11000
+//        auto null_value = static_cast<ValueType *>(nullptr);
+//        auto null_index = static_cast<IndexType *>(nullptr);
+//        auto one_val = one<ValueType>();
+//        auto zero_val = zero<ValueType>();
+//        auto zero_nnz = IndexType{};
+//        auto spgemm_descr = cusparse::create_spgemm_descr();
+//        auto a_descr = cusparse::create_fbcsr(m, k, a_nnz,
+//                                            const_cast<IndexType
+//                                            *>(a_row_ptrs),
+//                                            const_cast<IndexType
+//                                            *>(a_col_idxs),
+//                                            const_cast<ValueType *>(a_vals));
+//        auto b_descr = cusparse::create_fbcsr(k, n, b_nnz,
+//                                            const_cast<IndexType
+//                                            *>(b_row_ptrs),
+//                                            const_cast<IndexType
+//                                            *>(b_col_idxs),
+//                                            const_cast<ValueType *>(b_vals));
+//        auto c_descr = cusparse::create_fbcsr(m, n, zero_nnz, null_index,
+//                                            null_index, null_value);
+//
+//        // estimate work
+//        size_type buffer1_size{};
+//        cusparse::spgemm_work_estimation(handle, &one_val, a_descr, b_descr,
+//                                         &zero_val, c_descr, spgemm_descr,
+//                                         buffer1_size, nullptr);
+//        Array<char> buffer1{exec, buffer1_size};
+//        cusparse::spgemm_work_estimation(handle, &one_val, a_descr, b_descr,
+//                                         &zero_val, c_descr, spgemm_descr,
+//                                         buffer1_size, buffer1.get_data());
+//
+//        // compute spgemm
+//        size_type buffer2_size{};
+//        cusparse::spgemm_compute(handle, &one_val, a_descr, b_descr,
+//        &zero_val,
+//                                 c_descr, spgemm_descr, buffer1.get_data(),
+//                                 buffer2_size, nullptr);
+//        Array<char> buffer2{exec, buffer2_size};
+//        cusparse::spgemm_compute(handle, &one_val, a_descr, b_descr,
+//        &zero_val,
+//                                 c_descr, spgemm_descr, buffer1.get_data(),
+//                                 buffer2_size, buffer2.get_data());
+//
+//        // write result to temporary storage
+//        auto c_tmp_nnz = cusparse::sparse_matrix_nnz(c_descr);
+//        Array<IndexType> c_tmp_row_ptrs_array(exec, m + 1);
+//        Array<IndexType> c_tmp_col_idxs_array(exec, c_tmp_nnz);
+//        Array<ValueType> c_tmp_vals_array(exec, c_tmp_nnz);
+//        cusparse::fbcsr_set_pointers(c_descr, c_tmp_row_ptrs_array.get_data(),
+//                                   c_tmp_col_idxs_array.get_data(),
+//                                   c_tmp_vals_array.get_data());
+//
+//        cusparse::spgemm_copy(handle, &one_val, a_descr, b_descr, &zero_val,
+//                              c_descr, spgemm_descr);
+//
+//        cusparse::destroy(c_descr);
+//        cusparse::destroy(b_descr);
+//        cusparse::destroy(a_descr);
+//        cusparse::destroy(spgemm_descr);
+//
+//        auto spgeam_total_nnz = c_tmp_nnz + d->get_num_stored_elements();
+//        auto nnz_per_row = spgeam_total_nnz / m;
+//        select_spgeam(
+//            spgeam_kernels(),
+//            [&](int compiled_subwarp_size) {
+//                return compiled_subwarp_size >= nnz_per_row ||
+//                       compiled_subwarp_size == config::warp_size;
+//            },
+//            syn::value_list<int>(), syn::type_list<>(), exec,
+//            alpha->get_const_values(), c_tmp_row_ptrs_array.get_const_data(),
+//            c_tmp_col_idxs_array.get_const_data(),
+//            c_tmp_vals_array.get_const_data(), beta->get_const_values(),
+//            d_row_ptrs, d_col_idxs, d_vals, c);
+//#endif  // CUDA_VERSION >= 11000
+//    } else {
+//        GKO_NOT_IMPLEMENTED;
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void spgeam(std::shared_ptr<const DefaultExecutor> exec,
+            const matrix::Dense<ValueType> *alpha,
+            const matrix::Fbcsr<ValueType, IndexType> *a,
+            const matrix::Dense<ValueType> *beta,
+            const matrix::Fbcsr<ValueType, IndexType> *b,
+            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto total_nnz =
+//        a->get_num_stored_elements() + b->get_num_stored_elements();
+//    auto nnz_per_row = total_nnz / a->get_size()[0];
+//    select_spgeam(
+//        spgeam_kernels(),
+//        [&](int compiled_subwarp_size) {
+//            return compiled_subwarp_size >= nnz_per_row ||
+//                   compiled_subwarp_size == config::warp_size;
+//        },
+//        syn::value_list<int>(), syn::type_list<>(), exec,
+//        alpha->get_const_values(), a->get_const_row_ptrs(),
+//        a->get_const_col_idxs(), a->get_const_values(),
+//        beta->get_const_values(), b->get_const_row_ptrs(),
+//        b->get_const_col_idxs(), b->get_const_values(), c);
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEAM_KERNEL);
+
+
+template <typename IndexType>
+void convert_row_ptrs_to_idxs(std::shared_ptr<const CudaExecutor> exec,
+                              const IndexType *ptrs, size_type num_rows,
+                              IndexType *idxs) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto grid_dim = ceildiv(num_rows, default_block_size);
+//
+//    kernel::convert_row_ptrs_to_idxs<<<grid_dim, default_block_size>>>(
+//        num_rows, as_cuda_type(ptrs), as_cuda_type(idxs));
+//}
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_coo(std::shared_ptr<const CudaExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *source,
+                    matrix::Coo<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = result->get_size()[0];
+//
+//    auto row_idxs = result->get_row_idxs();
+//    const auto source_row_ptrs = source->get_const_row_ptrs();
+//
+//    convert_row_ptrs_to_idxs(exec, source_row_ptrs, num_rows, row_idxs);
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_dense(std::shared_ptr<const CudaExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *source,
+                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto num_rows = result->get_size()[0];
+//    const auto num_cols = result->get_size()[1];
+//    const auto stride = result->get_stride();
+//    const auto row_ptrs = source->get_const_row_ptrs();
+//    const auto col_idxs = source->get_const_col_idxs();
+//    const auto vals = source->get_const_values();
+//
+//    const dim3 block_size(config::warp_size,
+//                          config::max_block_size / config::warp_size, 1);
+//    const dim3 init_grid_dim(ceildiv(num_cols, block_size.x),
+//                             ceildiv(num_rows, block_size.y), 1);
+//    kernel::initialize_zero_dense<<<init_grid_dim, block_size>>>(
+//        num_rows, num_cols, stride, as_cuda_type(result->get_values()));
+//
+//    auto grid_dim = ceildiv(num_rows, default_block_size);
+//    kernel::fill_in_dense<<<grid_dim, default_block_size>>>(
+//        num_rows, as_cuda_type(row_ptrs), as_cuda_type(col_idxs),
+//        as_cuda_type(vals), stride, as_cuda_type(result->get_values()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_sellp(std::shared_ptr<const CudaExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *source,
+                      matrix::Sellp<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto num_rows = result->get_size()[0];
+//    const auto num_cols = result->get_size()[1];
+//
+//    auto result_values = result->get_values();
+//    auto result_col_idxs = result->get_col_idxs();
+//    auto slice_lengths = result->get_slice_lengths();
+//    auto slice_sets = result->get_slice_sets();
+//
+//    const auto slice_size = (result->get_slice_size() == 0)
+//                                ? matrix::default_slice_size
+//                                : result->get_slice_size();
+//    const auto stride_factor = (result->get_stride_factor() == 0)
+//                                   ? matrix::default_stride_factor
+//                                   : result->get_stride_factor();
+//    const int slice_num = ceildiv(num_rows, slice_size);
+//
+//    const auto source_values = source->get_const_values();
+//    const auto source_row_ptrs = source->get_const_row_ptrs();
+//    const auto source_col_idxs = source->get_const_col_idxs();
+//
+//    auto nnz_per_row = Array<size_type>(exec, num_rows);
+//    auto grid_dim = ceildiv(num_rows, default_block_size);
+//
+//    if (grid_dim > 0) {
+//        kernel::calculate_nnz_per_row<<<grid_dim, default_block_size>>>(
+//            num_rows, as_cuda_type(source_row_ptrs),
+//            as_cuda_type(nnz_per_row.get_data()));
+//    }
+//
+//    grid_dim = slice_num;
+//
+//    if (grid_dim > 0) {
+//        kernel::calculate_slice_lengths<<<grid_dim, config::warp_size>>>(
+//            num_rows, slice_size, stride_factor,
+//            as_cuda_type(nnz_per_row.get_const_data()),
+//            as_cuda_type(slice_lengths), as_cuda_type(slice_sets));
+//    }
+//
+//    components::prefix_sum(exec, slice_sets, slice_num + 1);
+//
+//    grid_dim = ceildiv(num_rows, default_block_size);
+//    if (grid_dim > 0) {
+//        kernel::fill_in_sellp<<<grid_dim, default_block_size>>>(
+//            num_rows, slice_size, as_cuda_type(source_values),
+//            as_cuda_type(source_row_ptrs), as_cuda_type(source_col_idxs),
+//            as_cuda_type(slice_lengths), as_cuda_type(slice_sets),
+//            as_cuda_type(result_col_idxs), as_cuda_type(result_values));
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_ell(std::shared_ptr<const CudaExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *source,
+                    matrix::Ell<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto source_values = source->get_const_values();
+//    const auto source_row_ptrs = source->get_const_row_ptrs();
+//    const auto source_col_idxs = source->get_const_col_idxs();
+//
+//    auto result_values = result->get_values();
+//    auto result_col_idxs = result->get_col_idxs();
+//    const auto stride = result->get_stride();
+//    const auto max_nnz_per_row = result->get_num_stored_elements_per_row();
+//    const auto num_rows = result->get_size()[0];
+//    const auto num_cols = result->get_size()[1];
+//
+//    const auto init_grid_dim =
+//        ceildiv(max_nnz_per_row * num_rows, default_block_size);
+//
+//    kernel::initialize_zero_ell<<<init_grid_dim, default_block_size>>>(
+//        max_nnz_per_row, stride, as_cuda_type(result_values),
+//        as_cuda_type(result_col_idxs));
+//
+//    const auto grid_dim =
+//        ceildiv(num_rows * config::warp_size, default_block_size);
+//
+//    kernel::fill_in_ell<<<grid_dim, default_block_size>>>(
+//        num_rows, stride, as_cuda_type(source_values),
+//        as_cuda_type(source_row_ptrs), as_cuda_type(source_col_idxs),
+//        as_cuda_type(result_values), as_cuda_type(result_col_idxs));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_total_cols(std::shared_ptr<const CudaExecutor> exec,
+                          const matrix::Fbcsr<ValueType, IndexType> *source,
+                          size_type *result, size_type stride_factor,
+                          size_type slice_size) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto num_rows = source->get_size()[0];
+//
+//    if (num_rows == 0) {
+//        *result = 0;
+//        return;
+//    }
+//
+//    const auto slice_num = ceildiv(num_rows, slice_size);
+//    const auto row_ptrs = source->get_const_row_ptrs();
+//
+//    auto nnz_per_row = Array<size_type>(exec, num_rows);
+//    auto grid_dim = ceildiv(num_rows, default_block_size);
+//
+//    kernel::calculate_nnz_per_row<<<grid_dim, default_block_size>>>(
+//        num_rows, as_cuda_type(row_ptrs),
+//        as_cuda_type(nnz_per_row.get_data()));
+//
+//    grid_dim = ceildiv(slice_num * config::warp_size, default_block_size);
+//    auto max_nnz_per_slice = Array<size_type>(exec, slice_num);
+//
+//    kernel::reduce_max_nnz_per_slice<<<grid_dim, default_block_size>>>(
+//        num_rows, slice_size, stride_factor,
+//        as_cuda_type(nnz_per_row.get_const_data()),
+//        as_cuda_type(max_nnz_per_slice.get_data()));
+//
+//    grid_dim = ceildiv(slice_num, default_block_size);
+//    auto block_results = Array<size_type>(exec, grid_dim);
+//
+//    kernel::reduce_total_cols<<<grid_dim, default_block_size>>>(
+//        slice_num, as_cuda_type(max_nnz_per_slice.get_const_data()),
+//        as_cuda_type(block_results.get_data()));
+//
+//    auto d_result = Array<size_type>(exec, 1);
+//
+//    kernel::reduce_total_cols<<<1, default_block_size>>>(
+//        grid_dim, as_cuda_type(block_results.get_const_data()),
+//        as_cuda_type(d_result.get_data()));
+//
+//    *result = exec->copy_val_to_host(d_result.get_const_data());
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void transpose(std::shared_ptr<const CudaExecutor> exec,
+               const matrix::Fbcsr<ValueType, IndexType> *orig,
+               matrix::Fbcsr<ValueType, IndexType> *trans) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    if (cusparse::is_supported<ValueType, IndexType>::value) {
+//#if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
+//        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
+//        cusparseIndexBase_t idxBase = CUSPARSE_INDEX_BASE_ZERO;
+//
+//        cusparse::transpose(
+//            exec->get_cusparse_handle(), orig->get_size()[0],
+//            orig->get_size()[1], orig->get_num_stored_elements(),
+//            orig->get_const_values(), orig->get_const_row_ptrs(),
+//            orig->get_const_col_idxs(), trans->get_values(),
+//            trans->get_row_ptrs(), trans->get_col_idxs(), copyValues,
+//            idxBase);
+//#else  // CUDA_VERSION >= 11000
+//        cudaDataType_t cu_value =
+//            gko::kernels::cuda::cuda_data_type<ValueType>();
+//        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
+//        cusparseIndexBase_t idxBase = CUSPARSE_INDEX_BASE_ZERO;
+//        cusparseFbcsr2CscAlg_t alg = CUSPARSE_FBCSR2CSC_ALG1;
+//        size_type buffer_size = 0;
+//        cusparse::transpose_buffersize(
+//            exec->get_cusparse_handle(), orig->get_size()[0],
+//            orig->get_size()[1], orig->get_num_stored_elements(),
+//            orig->get_const_values(), orig->get_const_row_ptrs(),
+//            orig->get_const_col_idxs(), trans->get_values(),
+//            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value,
+//            copyValues, idxBase, alg, &buffer_size);
+//        Array<char> buffer_array(exec, buffer_size);
+//        auto buffer = buffer_array.get_data();
+//        cusparse::transpose(
+//            exec->get_cusparse_handle(), orig->get_size()[0],
+//            orig->get_size()[1], orig->get_num_stored_elements(),
+//            orig->get_const_values(), orig->get_const_row_ptrs(),
+//            orig->get_const_col_idxs(), trans->get_values(),
+//            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value,
+//            copyValues, idxBase, alg, buffer);
+//#endif
+//    } else {
+//        GKO_NOT_IMPLEMENTED;
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void conj_transpose(std::shared_ptr<const CudaExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *orig,
+                    matrix::Fbcsr<ValueType, IndexType> *trans)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    if (cusparse::is_supported<ValueType, IndexType>::value) {
+//        const dim3 block_size(default_block_size, 1, 1);
+//        const dim3 grid_size(
+//            ceildiv(trans->get_num_stored_elements(), block_size.x), 1, 1);
+//
+//#if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
+//        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
+//        cusparseIndexBase_t idxBase = CUSPARSE_INDEX_BASE_ZERO;
+//
+//        cusparse::transpose(
+//            exec->get_cusparse_handle(), orig->get_size()[0],
+//            orig->get_size()[1], orig->get_num_stored_elements(),
+//            orig->get_const_values(), orig->get_const_row_ptrs(),
+//            orig->get_const_col_idxs(), trans->get_values(),
+//            trans->get_row_ptrs(), trans->get_col_idxs(), copyValues,
+//            idxBase);
+//#else  // CUDA_VERSION >= 11000
+//        cudaDataType_t cu_value =
+//            gko::kernels::cuda::cuda_data_type<ValueType>();
+//        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
+//        cusparseIndexBase_t idxBase = CUSPARSE_INDEX_BASE_ZERO;
+//        cusparseFbcsr2CscAlg_t alg = CUSPARSE_FBCSR2CSC_ALG1;
+//        size_type buffer_size = 0;
+//        cusparse::transpose_buffersize(
+//            exec->get_cusparse_handle(), orig->get_size()[0],
+//            orig->get_size()[1], orig->get_num_stored_elements(),
+//            orig->get_const_values(), orig->get_const_row_ptrs(),
+//            orig->get_const_col_idxs(), trans->get_values(),
+//            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value,
+//            copyValues, idxBase, alg, &buffer_size);
+//        Array<char> buffer_array(exec, buffer_size);
+//        auto buffer = buffer_array.get_data();
+//        cusparse::transpose(
+//            exec->get_cusparse_handle(), orig->get_size()[0],
+//            orig->get_size()[1], orig->get_num_stored_elements(),
+//            orig->get_const_values(), orig->get_const_row_ptrs(),
+//            orig->get_const_col_idxs(), trans->get_values(),
+//            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value,
+//            copyValues, idxBase, alg, buffer);
+//#endif
+//
+//        conjugate_kernel<<<grid_size, block_size, 0, 0>>>(
+//            trans->get_num_stored_elements(),
+//            as_cuda_type(trans->get_values()));
+//    } else {
+//        GKO_NOT_IMPLEMENTED;
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void row_permute(std::shared_ptr<const CudaExecutor> exec,
+                 const Array<IndexType> *permutation_indices,
+                 const matrix::Fbcsr<ValueType, IndexType> *orig,
+                 matrix::Fbcsr<ValueType, IndexType> *row_permuted)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void column_permute(std::shared_ptr<const CudaExecutor> exec,
+                    const Array<IndexType> *permutation_indices,
+                    const matrix::Fbcsr<ValueType, IndexType> *orig,
+                    matrix::Fbcsr<ValueType, IndexType> *column_permuted)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_row_permute(std::shared_ptr<const CudaExecutor> exec,
+                         const Array<IndexType> *permutation_indices,
+                         const matrix::Fbcsr<ValueType, IndexType> *orig,
+                         matrix::Fbcsr<ValueType, IndexType> *row_permuted)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_column_permute(
+    std::shared_ptr<const CudaExecutor> exec,
+    const Array<IndexType> *permutation_indices,
+    const matrix::Fbcsr<ValueType, IndexType> *orig,
+    matrix::Fbcsr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_max_nnz_per_row(
+    std::shared_ptr<const CudaExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *source,
+    size_type *result) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto num_rows = source->get_size()[0];
+//
+//    auto nnz_per_row = Array<size_type>(exec, num_rows);
+//    auto block_results = Array<size_type>(exec, default_block_size);
+//    auto d_result = Array<size_type>(exec, 1);
+//
+//    const auto grid_dim = ceildiv(num_rows, default_block_size);
+//    kernel::calculate_nnz_per_row<<<grid_dim, default_block_size>>>(
+//        num_rows, as_cuda_type(source->get_const_row_ptrs()),
+//        as_cuda_type(nnz_per_row.get_data()));
+//
+//    const auto n = ceildiv(num_rows, default_block_size);
+//    const auto reduce_dim = n <= default_block_size ? n : default_block_size;
+//    kernel::reduce_max_nnz<<<reduce_dim, default_block_size>>>(
+//        num_rows, as_cuda_type(nnz_per_row.get_const_data()),
+//        as_cuda_type(block_results.get_data()));
+//
+//    kernel::reduce_max_nnz<<<1, default_block_size>>>(
+//        reduce_dim, as_cuda_type(block_results.get_const_data()),
+//        as_cuda_type(d_result.get_data()));
+//
+//    *result = exec->copy_val_to_host(d_result.get_const_data());
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_hybrid(std::shared_ptr<const CudaExecutor> exec,
+                       const matrix::Fbcsr<ValueType, IndexType> *source,
+                       matrix::Hybrid<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto ell_val = result->get_ell_values();
+//    auto ell_col = result->get_ell_col_idxs();
+//    auto coo_val = result->get_coo_values();
+//    auto coo_col = result->get_coo_col_idxs();
+//    auto coo_row = result->get_coo_row_idxs();
+//    const auto stride = result->get_ell_stride();
+//    const auto max_nnz_per_row =
+//    result->get_ell_num_stored_elements_per_row(); const auto num_rows =
+//    result->get_size()[0]; const auto coo_num_stored_elements =
+//    result->get_coo_num_stored_elements(); auto grid_dim =
+//    ceildiv(max_nnz_per_row * num_rows, default_block_size);
+//
+//    kernel::initialize_zero_ell<<<grid_dim, default_block_size>>>(
+//        max_nnz_per_row, stride, as_cuda_type(ell_val),
+//        as_cuda_type(ell_col));
+//
+//    grid_dim = ceildiv(num_rows, default_block_size);
+//    auto coo_offset = Array<size_type>(exec, num_rows);
+//    kernel::calculate_hybrid_coo_row_nnz<<<grid_dim, default_block_size>>>(
+//        num_rows, max_nnz_per_row, as_cuda_type(source->get_const_row_ptrs()),
+//        as_cuda_type(coo_offset.get_data()));
+//
+//    components::prefix_sum(exec, coo_offset.get_data(), num_rows);
+//
+//    grid_dim = ceildiv(num_rows * config::warp_size, default_block_size);
+//    kernel::fill_in_hybrid<<<grid_dim, default_block_size>>>(
+//        num_rows, stride, max_nnz_per_row,
+//        as_cuda_type(source->get_const_values()),
+//        as_cuda_type(source->get_const_row_ptrs()),
+//        as_cuda_type(source->get_const_col_idxs()),
+//        as_cuda_type(coo_offset.get_const_data()), as_cuda_type(ell_val),
+//        as_cuda_type(ell_col), as_cuda_type(coo_val), as_cuda_type(coo_col),
+//        as_cuda_type(coo_row));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_nonzeros_per_row(
+    std::shared_ptr<const CudaExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *source,
+    Array<size_type> *result) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto num_rows = source->get_size()[0];
+//    auto row_ptrs = source->get_const_row_ptrs();
+//    auto grid_dim = ceildiv(num_rows, default_block_size);
+//
+//    kernel::calculate_nnz_per_row<<<grid_dim, default_block_size>>>(
+//        num_rows, as_cuda_type(row_ptrs), as_cuda_type(result->get_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void sort_by_column_index(std::shared_ptr<const CudaExecutor> exec,
+                          matrix::Fbcsr<ValueType, IndexType> *to_sort)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    if (cusparse::is_supported<ValueType, IndexType>::value) {
+//        auto handle = exec->get_cusparse_handle();
+//        auto descr = cusparse::create_mat_descr();
+//        auto m = IndexType(to_sort->get_size()[0]);
+//        auto n = IndexType(to_sort->get_size()[1]);
+//        auto nnz = IndexType(to_sort->get_num_stored_elements());
+//        auto row_ptrs = to_sort->get_const_row_ptrs();
+//        auto col_idxs = to_sort->get_col_idxs();
+//        auto vals = to_sort->get_values();
+//
+//        // copy values
+//        Array<ValueType> tmp_vals_array(exec, nnz);
+//        exec->copy(nnz, vals, tmp_vals_array.get_data());
+//        auto tmp_vals = tmp_vals_array.get_const_data();
+//
+//        // init identity permutation
+//        Array<IndexType> permutation_array(exec, nnz);
+//        auto permutation = permutation_array.get_data();
+//        cusparse::create_identity_permutation(handle, nnz, permutation);
+//
+//        // allocate buffer
+//        size_type buffer_size{};
+//        cusparse::fbcsrsort_buffer_size(handle, m, n, nnz, row_ptrs, col_idxs,
+//                                      buffer_size);
+//        Array<char> buffer_array{exec, buffer_size};
+//        auto buffer = buffer_array.get_data();
+//
+//        // sort column indices
+//        cusparse::fbcsrsort(handle, m, n, nnz, descr, row_ptrs, col_idxs,
+//                          permutation, buffer);
+//
+//        // sort values
+//#if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
+//        cusparse::gather(handle, nnz, tmp_vals, vals, permutation);
+//#else  // CUDA_VERSION >= 11000
+//        auto val_vec = cusparse::create_spvec(nnz, nnz, permutation, vals);
+//        auto tmp_vec =
+//            cusparse::create_dnvec(nnz, const_cast<ValueType *>(tmp_vals));
+//        cusparse::gather(handle, tmp_vec, val_vec);
+//#endif
+//
+//        cusparse::destroy(descr);
+//    } else {
+//        GKO_NOT_IMPLEMENTED;
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
+
+
+template <typename ValueType, typename IndexType>
+void is_sorted_by_column_index(
+    std::shared_ptr<const CudaExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *to_check,
+    bool *is_sorted) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    *is_sorted = true;
+//    auto cpu_array = Array<bool>::view(exec->get_master(), 1, is_sorted);
+//    auto gpu_array = Array<bool>{exec, cpu_array};
+//    auto block_size = default_block_size;
+//    auto num_rows = static_cast<IndexType>(to_check->get_size()[0]);
+//    auto num_blocks = ceildiv(num_rows, block_size);
+//    kernel::check_unsorted<<<num_blocks, block_size>>>(
+//        to_check->get_const_row_ptrs(), to_check->get_const_col_idxs(),
+//        num_rows, gpu_array.get_data());
+//    cpu_array = gpu_array;
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
+
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(std::shared_ptr<const CudaExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *orig,
+                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto nnz = orig->get_num_stored_elements();
+//    const auto diag_size = diag->get_size()[0];
+//    const auto num_blocks =
+//        ceildiv(config::warp_size * diag_size, default_block_size);
+//
+//    const auto orig_values = orig->get_const_values();
+//    const auto orig_row_ptrs = orig->get_const_row_ptrs();
+//    const auto orig_col_idxs = orig->get_const_col_idxs();
+//    auto diag_values = diag->get_values();
+//
+//    kernel::extract_diagonal<<<num_blocks, default_block_size>>>(
+//        diag_size, nnz, as_cuda_type(orig_values),
+//        as_cuda_type(orig_row_ptrs), as_cuda_type(orig_col_idxs),
+//        as_cuda_type(diag_values));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL);
+
+
+}  // namespace fbcsr
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
diff --git a/cuda/test/matrix/CMakeLists.txt b/cuda/test/matrix/CMakeLists.txt
index 65ce218ac71..5be841b3d00 100644
--- a/cuda/test/matrix/CMakeLists.txt
+++ b/cuda/test/matrix/CMakeLists.txt
@@ -3,5 +3,6 @@ ginkgo_create_test(csr_kernels)
 ginkgo_create_test(dense_kernels)
 ginkgo_create_test(diagonal_kernels)
 ginkgo_create_test(ell_kernels)
+ginkgo_create_test(fbcsr_kernels)
 ginkgo_create_test(hybrid_kernels)
 ginkgo_create_test(sellp_kernels)
diff --git a/cuda/test/matrix/fbcsr_kernels.cpp b/cuda/test/matrix/fbcsr_kernels.cpp
new file mode 100644
index 00000000000..043412c6402
--- /dev/null
+++ b/cuda/test/matrix/fbcsr_kernels.cpp
@@ -0,0 +1,883 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/matrix/fbcsr.hpp>
+
+
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+#include <ginkgo/core/matrix/ell.hpp>
+#include <ginkgo/core/matrix/hybrid.hpp>
+#include <ginkgo/core/matrix/identity.hpp>
+#include <ginkgo/core/matrix/sellp.hpp>
+#include <ginkgo/core/matrix/sparsity_fbcsr.hpp>
+
+
+#include "core/matrix/fbcsr_kernels.hpp"
+#include "cuda/test/utils.hpp"
+
+
+namespace {
+
+
+class Fbcsr : public ::testing::Test {
+protected:
+    using Vec = gko::matrix::Dense<>;
+    using Mtx = gko::matrix::Fbcsr<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
+    using ComplexMtx = gko::matrix::Fbcsr<std::complex<double>>;
+
+    Fbcsr() : mtx_size(532, 231), rand_engine(42) {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        cuda = gko::CudaExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (cuda != nullptr) {
+            ASSERT_NO_THROW(cuda->synchronize());
+        }
+    }
+
+    template <typename MtxType>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row)
+    {
+        return gko::test::generate_random_matrix<MtxType>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(min_nnz_row, num_cols),
+            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    void set_up_apply_data(std::shared_ptr<Mtx::strategy_type> strategy,
+                           int num_vectors = 1)
+    {
+        mtx = Mtx::create(ref, strategy);
+        mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[1], 1));
+        square_mtx = Mtx::create(ref, strategy);
+        square_mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[0], 1));
+        expected = gen_mtx<Vec>(mtx_size[0], num_vectors, 1);
+        y = gen_mtx<Vec>(mtx_size[1], num_vectors, 1);
+        alpha = gko::initialize<Vec>({2.0}, ref);
+        beta = gko::initialize<Vec>({-1.0}, ref);
+        dmtx = Mtx::create(cuda, strategy);
+        dmtx->copy_from(mtx.get());
+        square_dmtx = Mtx::create(cuda, strategy);
+        square_dmtx->copy_from(square_mtx.get());
+        dresult = Vec::create(cuda);
+        dresult->copy_from(expected.get());
+        dy = Vec::create(cuda);
+        dy->copy_from(y.get());
+        dalpha = Vec::create(cuda);
+        dalpha->copy_from(alpha.get());
+        dbeta = Vec::create(cuda);
+        dbeta->copy_from(beta.get());
+    }
+
+    void set_up_apply_complex_data(
+        std::shared_ptr<ComplexMtx::strategy_type> strategy)
+    {
+        complex_mtx = ComplexMtx::create(ref, strategy);
+        complex_mtx->copy_from(
+            gen_mtx<ComplexVec>(mtx_size[0], mtx_size[1], 1));
+        complex_dmtx = ComplexMtx::create(cuda, strategy);
+        complex_dmtx->copy_from(complex_mtx.get());
+    }
+
+    struct matrix_pair {
+        std::unique_ptr<Mtx> ref;
+        std::unique_ptr<Mtx> cuda;
+    };
+
+    matrix_pair gen_unsorted_mtx()
+    {
+        constexpr int min_nnz_per_row = 2;  // Must be at least 2
+        auto local_mtx_ref =
+            gen_mtx<Mtx>(mtx_size[0], mtx_size[1], min_nnz_per_row);
+        for (size_t row = 0; row < mtx_size[0]; ++row) {
+            const auto row_ptrs = local_mtx_ref->get_const_row_ptrs();
+            const auto start_row = row_ptrs[row];
+            auto col_idx = local_mtx_ref->get_col_idxs() + start_row;
+            auto vals = local_mtx_ref->get_values() + start_row;
+            const auto nnz_in_this_row = row_ptrs[row + 1] - row_ptrs[row];
+            auto swap_idx_dist =
+                std::uniform_int_distribution<>(0, nnz_in_this_row - 1);
+            // shuffle `nnz_in_this_row / 2` times
+            for (size_t perm = 0; perm < nnz_in_this_row; perm += 2) {
+                const auto idx1 = swap_idx_dist(rand_engine);
+                const auto idx2 = swap_idx_dist(rand_engine);
+                std::swap(col_idx[idx1], col_idx[idx2]);
+                std::swap(vals[idx1], vals[idx2]);
+            }
+        }
+        auto local_mtx_cuda = Mtx::create(cuda);
+        local_mtx_cuda->copy_from(local_mtx_ref.get());
+
+        return {std::move(local_mtx_ref), std::move(local_mtx_cuda)};
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::CudaExecutor> cuda;
+
+    const gko::dim<2> mtx_size;
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Mtx> mtx;
+    std::unique_ptr<ComplexMtx> complex_mtx;
+    std::unique_ptr<Mtx> square_mtx;
+    std::unique_ptr<Vec> expected;
+    std::unique_ptr<Vec> y;
+    std::unique_ptr<Vec> alpha;
+    std::unique_ptr<Vec> beta;
+
+    std::unique_ptr<Mtx> dmtx;
+    std::unique_ptr<ComplexMtx> complex_dmtx;
+    std::unique_ptr<Mtx> square_dmtx;
+    std::unique_ptr<Vec> dresult;
+    std::unique_ptr<Vec> dy;
+    std::unique_ptr<Vec> dalpha;
+    std::unique_ptr<Vec> dbeta;
+};
+
+
+TEST_F(Fbcsr, StrategyAfterCopyIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::load_balance>(cuda));
+//
+//    ASSERT_EQ(mtx->get_strategy()->get_name(),
+//              dmtx->get_strategy()->get_name());
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithLoadBalance)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::load_balance>(cuda));
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithLoadBalance)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::load_balance>(cuda));
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithCusparse)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithCusparse)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithMergePath)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::merge_path>());
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithMergePath)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::merge_path>());
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithClassical)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::classical>());
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithClassical)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::classical>());
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithAutomatical)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>(cuda));
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyToDenseMatrixIsEquivalentToRefWithLoadBalance)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::load_balance>(cuda), 3);
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithLoadBalance)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::load_balance>(cuda), 3);
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyToDenseMatrixIsEquivalentToRefWithClassical)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::classical>(), 3);
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithClassical)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::classical>(), 3);
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyToDenseMatrixIsEquivalentToRefWithMergePath)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::merge_path>(), 3);
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithMergePath)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::merge_path>(), 3);
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyToFbcsrMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>());
+//    auto trans = mtx->transpose();
+//    auto d_trans = dmtx->transpose();
+//
+//    mtx->apply(alpha.get(), trans.get(), beta.get(), square_mtx.get());
+//    dmtx->apply(dalpha.get(), d_trans.get(), dbeta.get(), square_dmtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
+//    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+//    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyToFbcsrMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>());
+//    auto trans = mtx->transpose();
+//    auto d_trans = dmtx->transpose();
+//
+//    mtx->apply(trans.get(), square_mtx.get());
+//    dmtx->apply(d_trans.get(), square_dmtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
+//    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+//    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>());
+//    auto a = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
+//    auto b = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
+//    auto da = Mtx::create(cuda);
+//    auto db = Mtx::create(cuda);
+//    da->copy_from(a.get());
+//    db->copy_from(b.get());
+//    auto id = gko::matrix::Identity<Mtx::value_type>::create(ref,
+//    mtx_size[1]); auto did =
+//        gko::matrix::Identity<Mtx::value_type>::create(cuda, mtx_size[1]);
+//
+//    a->apply(alpha.get(), id.get(), beta.get(), b.get());
+//    da->apply(dalpha.get(), did.get(), dbeta.get(), db.get());
+//
+//    GKO_ASSERT_MTX_NEAR(b, db, 1e-14);
+//    GKO_ASSERT_MTX_EQ_SPARSITY(b, db);
+//    ASSERT_TRUE(db->is_sorted_by_column_index());
+//}
+
+
+TEST_F(Fbcsr, TransposeIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>(cuda));
+//
+//    auto trans = mtx->transpose();
+//    auto d_trans = dmtx->transpose();
+//
+//    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(d_trans.get()),
+//                        static_cast<Mtx *>(trans.get()), 0.0);
+//    ASSERT_TRUE(static_cast<Mtx
+//    *>(d_trans.get())->is_sorted_by_column_index());
+//}
+
+
+TEST_F(Fbcsr, ConjugateTransposeIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(cuda));
+//
+//    auto trans = complex_mtx->conj_transpose();
+//    auto d_trans = complex_dmtx->conj_transpose();
+//
+//    GKO_ASSERT_MTX_NEAR(static_cast<ComplexMtx *>(d_trans.get()),
+//                        static_cast<ComplexMtx *>(trans.get()), 0.0);
+//    ASSERT_TRUE(
+//        static_cast<ComplexMtx
+//        *>(d_trans.get())->is_sorted_by_column_index());
+//}
+
+
+TEST_F(Fbcsr, ConvertToDenseIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto dense_mtx = gko::matrix::Dense<>::create(ref);
+//    auto ddense_mtx = gko::matrix::Dense<>::create(cuda);
+//
+//    mtx->convert_to(dense_mtx.get());
+//    dmtx->convert_to(ddense_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToDenseIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto dense_mtx = gko::matrix::Dense<>::create(ref);
+//    auto ddense_mtx = gko::matrix::Dense<>::create(cuda);
+//
+//    mtx->move_to(dense_mtx.get());
+//    dmtx->move_to(ddense_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, ConvertToEllIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto ell_mtx = gko::matrix::Ell<>::create(ref);
+//    auto dell_mtx = gko::matrix::Ell<>::create(cuda);
+//
+//    mtx->convert_to(ell_mtx.get());
+//    dmtx->convert_to(dell_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(ell_mtx.get(), dell_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToEllIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto ell_mtx = gko::matrix::Ell<>::create(ref);
+//    auto dell_mtx = gko::matrix::Ell<>::create(cuda);
+//
+//    mtx->move_to(ell_mtx.get());
+//    dmtx->move_to(dell_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(ell_mtx.get(), dell_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, ConvertToSparsityFbcsrIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(ref);
+//    auto d_sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(cuda);
+//
+//    mtx->convert_to(sparsity_mtx.get());
+//    dmtx->convert_to(d_sparsity_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(sparsity_mtx.get(), d_sparsity_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToSparsityFbcsrIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(ref);
+//    auto d_sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(cuda);
+//
+//    mtx->move_to(sparsity_mtx.get());
+//    dmtx->move_to(d_sparsity_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(sparsity_mtx.get(), d_sparsity_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, CalculateMaxNnzPerRowIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    gko::size_type max_nnz_per_row;
+//    gko::size_type dmax_nnz_per_row;
+//
+//    gko::kernels::reference::fbcsr::calculate_max_nnz_per_row(ref, mtx.get(),
+//                                                            &max_nnz_per_row);
+//    gko::kernels::cuda::fbcsr::calculate_max_nnz_per_row(cuda, dmtx.get(),
+//                                                       &dmax_nnz_per_row);
+//
+//    ASSERT_EQ(max_nnz_per_row, dmax_nnz_per_row);
+//}
+
+
+TEST_F(Fbcsr, ConvertToCooIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto coo_mtx = gko::matrix::Coo<>::create(ref);
+//    auto dcoo_mtx = gko::matrix::Coo<>::create(cuda);
+//
+//    mtx->convert_to(coo_mtx.get());
+//    dmtx->convert_to(dcoo_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToCooIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto coo_mtx = gko::matrix::Coo<>::create(ref);
+//    auto dcoo_mtx = gko::matrix::Coo<>::create(cuda);
+//
+//    mtx->move_to(coo_mtx.get());
+//    dmtx->move_to(dcoo_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, ConvertToSellpIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
+//    auto dsellp_mtx = gko::matrix::Sellp<>::create(cuda);
+//
+//    mtx->convert_to(sellp_mtx.get());
+//    dmtx->convert_to(dsellp_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(sellp_mtx.get(), dsellp_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToSellpIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
+//    auto dsellp_mtx = gko::matrix::Sellp<>::create(cuda);
+//
+//    mtx->move_to(sellp_mtx.get());
+//    dmtx->move_to(dsellp_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(sellp_mtx.get(), dsellp_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, ConvertsEmptyToSellp)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto dempty_mtx = Mtx::create(cuda);
+//    auto dsellp_mtx = gko::matrix::Sellp<>::create(cuda);
+//
+//    dempty_mtx->convert_to(dsellp_mtx.get());
+//
+//    ASSERT_EQ(cuda->copy_val_to_host(dsellp_mtx->get_const_slice_sets()), 0);
+//    ASSERT_FALSE(dsellp_mtx->get_size());
+//}
+
+
+TEST_F(Fbcsr, CalculateTotalColsIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    gko::size_type total_cols;
+//    gko::size_type dtotal_cols;
+//
+//    gko::kernels::reference::fbcsr::calculate_total_cols(
+//        ref, mtx.get(), &total_cols, 2, gko::matrix::default_slice_size);
+//    gko::kernels::cuda::fbcsr::calculate_total_cols(
+//        cuda, dmtx.get(), &dtotal_cols, 2, gko::matrix::default_slice_size);
+//
+//    ASSERT_EQ(total_cols, dtotal_cols);
+//}
+
+
+TEST_F(Fbcsr, CalculatesNonzerosPerRow)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    gko::Array<gko::size_type> row_nnz(ref, mtx->get_size()[0]);
+//    gko::Array<gko::size_type> drow_nnz(cuda, dmtx->get_size()[0]);
+//
+//    gko::kernels::reference::fbcsr::calculate_nonzeros_per_row(ref, mtx.get(),
+//                                                             &row_nnz);
+//    gko::kernels::cuda::fbcsr::calculate_nonzeros_per_row(cuda, dmtx.get(),
+//                                                        &drow_nnz);
+//
+//    GKO_ASSERT_ARRAY_EQ(row_nnz, drow_nnz);
+//}
+
+
+TEST_F(Fbcsr, ConvertToHybridIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Hybrid_type = gko::matrix::Hybrid<>;
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto hybrid_mtx = Hybrid_type::create(
+//        ref, std::make_shared<Hybrid_type::column_limit>(2));
+//    auto dhybrid_mtx = Hybrid_type::create(
+//        cuda, std::make_shared<Hybrid_type::column_limit>(2));
+//
+//    mtx->convert_to(hybrid_mtx.get());
+//    dmtx->convert_to(dhybrid_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToHybridIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Hybrid_type = gko::matrix::Hybrid<>;
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto hybrid_mtx = Hybrid_type::create(
+//        ref, std::make_shared<Hybrid_type::column_limit>(2));
+//    auto dhybrid_mtx = Hybrid_type::create(
+//        cuda, std::make_shared<Hybrid_type::column_limit>(2));
+//
+//    mtx->move_to(hybrid_mtx.get());
+//    dmtx->move_to(dhybrid_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, RecognizeSortedMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>());
+//    bool is_sorted_cuda{};
+//    bool is_sorted_ref{};
+//
+//    is_sorted_ref = mtx->is_sorted_by_column_index();
+//    is_sorted_cuda = dmtx->is_sorted_by_column_index();
+//
+//    ASSERT_EQ(is_sorted_ref, is_sorted_cuda);
+//}
+
+
+TEST_F(Fbcsr, RecognizeUnsortedMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto uns_mtx = gen_unsorted_mtx();
+//    bool is_sorted_cuda{};
+//    bool is_sorted_ref{};
+//
+//    is_sorted_ref = uns_mtx.ref->is_sorted_by_column_index();
+//    is_sorted_cuda = uns_mtx.cuda->is_sorted_by_column_index();
+//
+//    ASSERT_EQ(is_sorted_ref, is_sorted_cuda);
+//}
+
+
+TEST_F(Fbcsr, SortSortedMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>());
+//
+//    mtx->sort_by_column_index();
+//    dmtx->sort_by_column_index();
+//
+//    // Values must be unchanged, therefore, tolerance is `0`
+//    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 0);
+//}
+
+
+TEST_F(Fbcsr, SortUnsortedMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto uns_mtx = gen_unsorted_mtx();
+//
+//    uns_mtx.ref->sort_by_column_index();
+//    uns_mtx.cuda->sort_by_column_index();
+//
+//    // Values must be unchanged, therefore, tolerance is `0`
+//    GKO_ASSERT_MTX_NEAR(uns_mtx.ref, uns_mtx.cuda, 0);
+//}
+
+
+TEST_F(Fbcsr, OneAutomaticalWorksWithDifferentMatrices)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto automatical = std::make_shared<Mtx::automatical>();
+//    auto row_len_limit = std::max(automatical->nvidia_row_len_limit,
+//                                  automatical->amd_row_len_limit);
+//    auto load_balance_mtx = Mtx::create(ref);
+//    auto classical_mtx = Mtx::create(ref);
+//    load_balance_mtx->copy_from(
+//        gen_mtx<Vec>(1, row_len_limit + 1000, row_len_limit + 1));
+//    classical_mtx->copy_from(gen_mtx<Vec>(50, 50, 1));
+//    auto load_balance_mtx_d = Mtx::create(cuda);
+//    auto classical_mtx_d = Mtx::create(cuda);
+//    load_balance_mtx_d->copy_from(load_balance_mtx.get());
+//    classical_mtx_d->copy_from(classical_mtx.get());
+//
+//    load_balance_mtx_d->set_strategy(automatical);
+//    classical_mtx_d->set_strategy(automatical);
+//
+//    EXPECT_EQ("load_balance", load_balance_mtx_d->get_strategy()->get_name());
+//    EXPECT_EQ("classical", classical_mtx_d->get_strategy()->get_name());
+//    ASSERT_NE(load_balance_mtx_d->get_strategy().get(),
+//              classical_mtx_d->get_strategy().get());
+//}
+
+
+TEST_F(Fbcsr, ExtractDiagonalIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>());
+//
+//    auto diag = mtx->extract_diagonal();
+//    auto ddiag = dmtx->extract_diagonal();
+//
+//    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+//}
+
+
+TEST_F(Fbcsr, InplaceAbsoluteMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>(cuda));
+//
+//    mtx->compute_absolute_inplace();
+//    dmtx->compute_absolute_inplace();
+//
+//    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, OutplaceAbsoluteMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>(cuda));
+//
+//    auto abs_mtx = mtx->compute_absolute();
+//    auto dabs_mtx = dmtx->compute_absolute();
+//
+//    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(cuda));
+//
+//    complex_mtx->compute_absolute_inplace();
+//    complex_dmtx->compute_absolute_inplace();
+//
+//    GKO_ASSERT_MTX_NEAR(complex_mtx, complex_dmtx, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(cuda));
+//
+//    auto abs_mtx = complex_mtx->compute_absolute();
+//    auto dabs_mtx = complex_dmtx->compute_absolute();
+//
+//    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+//}
+
+
+}  // namespace
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 14aefba1a0d..2b9ed806938 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -166,6 +166,7 @@ set(GINKGO_HIP_SOURCES
     matrix/dense_kernels.hip.cpp
     matrix/diagonal_kernels.hip.cpp
     matrix/ell_kernels.hip.cpp
+    matrix/fbcsr_kernels.hip.cpp
     matrix/hybrid_kernels.hip.cpp
     matrix/sellp_kernels.hip.cpp
     matrix/sparsity_csr_kernels.hip.cpp
diff --git a/hip/matrix/fbcsr_kernels.hip.cpp b/hip/matrix/fbcsr_kernels.hip.cpp
new file mode 100644
index 00000000000..5263e535e7a
--- /dev/null
+++ b/hip/matrix/fbcsr_kernels.hip.cpp
@@ -0,0 +1,1263 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/fbcsr_kernels.hpp"
+
+
+#include <algorithm>
+
+
+#include <hip/hip_runtime.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/ell.hpp>
+#include <ginkgo/core/matrix/hybrid.hpp>
+#include <ginkgo/core/matrix/sellp.hpp>
+
+
+#include "core/components/fill_array.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/dense_kernels.hpp"
+#include "core/matrix/fbcsr_builder.hpp"
+#include "core/synthesizer/implementation_selection.hpp"
+#include "hip/base/config.hip.hpp"
+#include "hip/base/hipsparse_bindings.hip.hpp"
+#include "hip/base/math.hip.hpp"
+#include "hip/base/pointer_mode_guard.hip.hpp"
+#include "hip/base/types.hip.hpp"
+#include "hip/components/atomic.hip.hpp"
+#include "hip/components/cooperative_groups.hip.hpp"
+#include "hip/components/intrinsics.hip.hpp"
+#include "hip/components/merging.hip.hpp"
+#include "hip/components/reduction.hip.hpp"
+#include "hip/components/segment_scan.hip.hpp"
+#include "hip/components/thread_ids.hip.hpp"
+#include "hip/components/uninitialized_array.hip.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace hip {
+/**
+ * @brief The Compressed sparse row matrix format namespace.
+ *
+ * @ingroup fbcsr
+ */
+namespace fbcsr {
+
+
+constexpr int default_block_size = 512;
+constexpr int warps_in_block = 4;
+constexpr int spmv_block_size = warps_in_block * config::warp_size;
+constexpr int wsize = config::warp_size;
+constexpr int classical_overweight = 32;
+
+
+/**
+ * A compile-time list of the number items per threads for which spmv kernel
+ * should be compiled.
+ */
+using compiled_kernels = syn::value_list<int, 3, 4, 6, 7, 8, 12, 14>;
+
+using classical_kernels =
+    syn::value_list<int, config::warp_size, 32, 16, 8, 4, 2, 1>;
+
+using spgeam_kernels =
+    syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
+
+
+namespace host_kernel {
+
+
+template <int items_per_thread, typename ValueType, typename IndexType>
+void merge_path_spmv(
+    syn::value_list<int, items_per_thread>,
+    std::shared_ptr<const HipExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *a,
+    const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c,
+    const matrix::Dense<ValueType> *alpha = nullptr,
+    const matrix::Dense<ValueType> *beta = nullptr) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const IndexType total = a->get_size()[0] + a->get_num_stored_elements();
+//    const IndexType grid_num =
+//        ceildiv(total, spmv_block_size * items_per_thread);
+//    const dim3 grid(grid_num);
+//    const dim3 block(spmv_block_size);
+//    Array<IndexType> row_out(exec, grid_num);
+//    Array<ValueType> val_out(exec, grid_num);
+//
+//    for (IndexType column_id = 0; column_id < b->get_size()[1]; column_id++) {
+//        if (alpha == nullptr && beta == nullptr) {
+//            const auto b_vals = b->get_const_values() + column_id;
+//            auto c_vals = c->get_values() + column_id;
+//            hipLaunchKernelGGL(
+//                HIP_KERNEL_NAME(
+//                    kernel::abstract_merge_path_spmv<items_per_thread>),
+//                dim3(grid), dim3(block), 0, 0,
+//                static_cast<IndexType>(a->get_size()[0]),
+//                as_hip_type(a->get_const_values()), a->get_const_col_idxs(),
+//                as_hip_type(a->get_const_row_ptrs()),
+//                as_hip_type(a->get_const_srow()), as_hip_type(b_vals),
+//                b->get_stride(), as_hip_type(c_vals), c->get_stride(),
+//                as_hip_type(row_out.get_data()),
+//                as_hip_type(val_out.get_data()));
+//            hipLaunchKernelGGL(kernel::abstract_reduce, dim3(1),
+//                               dim3(spmv_block_size), 0, 0, grid_num,
+//                               as_hip_type(val_out.get_data()),
+//                               as_hip_type(row_out.get_data()),
+//                               as_hip_type(c_vals), c->get_stride());
+//
+//        } else if (alpha != nullptr && beta != nullptr) {
+//            const auto b_vals = b->get_const_values() + column_id;
+//            auto c_vals = c->get_values() + column_id;
+//            hipLaunchKernelGGL(
+//                HIP_KERNEL_NAME(
+//                    kernel::abstract_merge_path_spmv<items_per_thread>),
+//                dim3(grid), dim3(block), 0, 0,
+//                static_cast<IndexType>(a->get_size()[0]),
+//                as_hip_type(alpha->get_const_values()),
+//                as_hip_type(a->get_const_values()), a->get_const_col_idxs(),
+//                as_hip_type(a->get_const_row_ptrs()),
+//                as_hip_type(a->get_const_srow()), as_hip_type(b_vals),
+//                b->get_stride(), as_hip_type(beta->get_const_values()),
+//                as_hip_type(c_vals), c->get_stride(),
+//                as_hip_type(row_out.get_data()),
+//                as_hip_type(val_out.get_data()));
+//            hipLaunchKernelGGL(kernel::abstract_reduce, dim3(1),
+//                               dim3(spmv_block_size), 0, 0, grid_num,
+//                               as_hip_type(val_out.get_data()),
+//                               as_hip_type(row_out.get_data()),
+//                               as_hip_type(alpha->get_const_values()),
+//                               as_hip_type(c_vals), c->get_stride());
+//        } else {
+//            GKO_KERNEL_NOT_FOUND;
+//        }
+//    }
+//}
+
+GKO_ENABLE_IMPLEMENTATION_SELECTION(select_merge_path_spmv, merge_path_spmv);
+
+
+template <typename ValueType, typename IndexType>
+int compute_items_per_thread(std::shared_ptr<const HipExecutor> exec)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//#if GINKGO_HIP_PLATFORM_NVCC
+//
+//
+//    const int version =
+//        (exec->get_major_version() << 4) + exec->get_minor_version();
+//    // The num_item is decided to make the occupancy 100%
+//    // TODO: Extend this list when new GPU is released
+//    //       Tune this parameter
+//    // 128 threads/block the number of items per threads
+//    // 3.0 3.5: 6
+//    // 3.7: 14
+//    // 5.0, 5.3, 6.0, 6.2: 8
+//    // 5.2, 6.1, 7.0: 12
+//    int num_item = 6;
+//    switch (version) {
+//    case 0x50:
+//    case 0x53:
+//    case 0x60:
+//    case 0x62:
+//        num_item = 8;
+//        break;
+//    case 0x52:
+//    case 0x61:
+//    case 0x70:
+//        num_item = 12;
+//        break;
+//    case 0x37:
+//        num_item = 14;
+//    }
+//
+//
+//#else
+//
+//
+//    // HIP uses the minimal num_item to make the code work correctly.
+//    // TODO: this parameter should be tuned.
+//    int num_item = 6;
+//
+//
+//#endif  // GINKGO_HIP_PLATFORM_NVCC
+//
+//
+//    // Ensure that the following is satisfied:
+//    // sizeof(IndexType) + sizeof(ValueType)
+//    // <= items_per_thread * sizeof(IndexType)
+//    constexpr int minimal_num =
+//        ceildiv(sizeof(IndexType) + sizeof(ValueType), sizeof(IndexType));
+//    int items_per_thread = num_item * 4 / sizeof(IndexType);
+//    return std::max(minimal_num, items_per_thread);
+//}
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+void classical_spmv(
+    syn::value_list<int, subwarp_size>, std::shared_ptr<const HipExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *a,
+    const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c,
+    const matrix::Dense<ValueType> *alpha = nullptr,
+    const matrix::Dense<ValueType> *beta = nullptr) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto nwarps = exec->get_num_warps_per_sm() *
+//                        exec->get_num_multiprocessor() * classical_overweight;
+//    const auto gridx =
+//        std::min(ceildiv(a->get_size()[0], spmv_block_size / subwarp_size),
+//                 int64(nwarps / warps_in_block));
+//    const dim3 grid(gridx, b->get_size()[1]);
+//    const dim3 block(spmv_block_size);
+//
+//    if (alpha == nullptr && beta == nullptr) {
+//        hipLaunchKernelGGL(
+//            HIP_KERNEL_NAME(kernel::abstract_classical_spmv<subwarp_size>),
+//            dim3(grid), dim3(block), 0, 0, a->get_size()[0],
+//            as_hip_type(a->get_const_values()), a->get_const_col_idxs(),
+//            as_hip_type(a->get_const_row_ptrs()),
+//            as_hip_type(b->get_const_values()), b->get_stride(),
+//            as_hip_type(c->get_values()), c->get_stride());
+//
+//    } else if (alpha != nullptr && beta != nullptr) {
+//        hipLaunchKernelGGL(
+//            HIP_KERNEL_NAME(kernel::abstract_classical_spmv<subwarp_size>),
+//            dim3(grid), dim3(block), 0, 0, a->get_size()[0],
+//            as_hip_type(alpha->get_const_values()),
+//            as_hip_type(a->get_const_values()), a->get_const_col_idxs(),
+//            as_hip_type(a->get_const_row_ptrs()),
+//            as_hip_type(b->get_const_values()), b->get_stride(),
+//            as_hip_type(beta->get_const_values()),
+//            as_hip_type(c->get_values()), c->get_stride());
+//    } else {
+//        GKO_KERNEL_NOT_FOUND;
+//    }
+//}
+
+GKO_ENABLE_IMPLEMENTATION_SELECTION(select_classical_spmv, classical_spmv);
+
+
+}  // namespace host_kernel
+
+
+template <typename ValueType, typename IndexType>
+void spmv(std::shared_ptr<const HipExecutor> exec,
+          const matrix::Fbcsr<ValueType, IndexType> *a,
+          const matrix::Dense<ValueType> *b,
+          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    if (a->get_strategy()->get_name() == "load_balance") {
+//        components::fill_array(exec, c->get_values(),
+//                               c->get_num_stored_elements(),
+//                               zero<ValueType>());
+//        const IndexType nwarps = a->get_num_srow_elements();
+//        if (nwarps > 0) {
+//            const dim3 fbcsr_block(config::warp_size, warps_in_block, 1);
+//            const dim3 fbcsr_grid(ceildiv(nwarps, warps_in_block),
+//                                b->get_size()[1]);
+//            hipLaunchKernelGGL(
+//                kernel::abstract_spmv, dim3(fbcsr_grid), dim3(fbcsr_block), 0,
+//                0, nwarps, static_cast<IndexType>(a->get_size()[0]),
+//                as_hip_type(a->get_const_values()), a->get_const_col_idxs(),
+//                as_hip_type(a->get_const_row_ptrs()),
+//                as_hip_type(a->get_const_srow()),
+//                as_hip_type(b->get_const_values()),
+//                as_hip_type(b->get_stride()), as_hip_type(c->get_values()),
+//                as_hip_type(c->get_stride()));
+//        } else {
+//            GKO_NOT_SUPPORTED(nwarps);
+//        }
+//    } else if (a->get_strategy()->get_name() == "merge_path") {
+//        int items_per_thread =
+//            host_kernel::compute_items_per_thread<ValueType, IndexType>(exec);
+//        host_kernel::select_merge_path_spmv(
+//            compiled_kernels(),
+//            [&items_per_thread](int compiled_info) {
+//                return items_per_thread == compiled_info;
+//            },
+//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c);
+//    } else if (a->get_strategy()->get_name() == "classical") {
+//        IndexType max_length_per_row = 0;
+//        using Tfbcsr = matrix::Fbcsr<ValueType, IndexType>;
+//        if (auto strategy =
+//                std::dynamic_pointer_cast<const typename Tfbcsr::classical>(
+//                    a->get_strategy())) {
+//            max_length_per_row = strategy->get_max_length_per_row();
+//        } else if (auto strategy = std::dynamic_pointer_cast<
+//                       const typename Tfbcsr::automatical>(a->get_strategy()))
+//                       {
+//            max_length_per_row = strategy->get_max_length_per_row();
+//        } else {
+//            GKO_NOT_SUPPORTED(a->get_strategy());
+//        }
+//        host_kernel::select_classical_spmv(
+//            classical_kernels(),
+//            [&max_length_per_row](int compiled_info) {
+//                return max_length_per_row >= compiled_info;
+//            },
+//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c);
+//    } else if (a->get_strategy()->get_name() == "sparselib" ||
+//               a->get_strategy()->get_name() == "cusparse") {
+//        if (hipsparse::is_supported<ValueType, IndexType>::value) {
+//            // TODO: add implementation for int64 and multiple RHS
+//            auto handle = exec->get_hipsparse_handle();
+//            auto descr = hipsparse::create_mat_descr();
+//            {
+//                hipsparse::pointer_mode_guard pm_guard(handle);
+//                auto row_ptrs = a->get_const_row_ptrs();
+//                auto col_idxs = a->get_const_col_idxs();
+//                auto alpha = one<ValueType>();
+//                auto beta = zero<ValueType>();
+//                if (b->get_stride() != 1 || c->get_stride() != 1) {
+//                    GKO_NOT_IMPLEMENTED;
+//                }
+//                hipsparse::spmv(handle, HIPSPARSE_OPERATION_NON_TRANSPOSE,
+//                                a->get_size()[0], a->get_size()[1],
+//                                a->get_num_stored_elements(), &alpha, descr,
+//                                a->get_const_values(), row_ptrs, col_idxs,
+//                                b->get_const_values(), &beta,
+//                                c->get_values());
+//            }
+//            hipsparse::destroy(descr);
+//        } else {
+//            GKO_NOT_IMPLEMENTED;
+//        }
+//    } else {
+//        GKO_NOT_IMPLEMENTED;
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spmv(std::shared_ptr<const HipExecutor> exec,
+                   const matrix::Dense<ValueType> *alpha,
+                   const matrix::Fbcsr<ValueType, IndexType> *a,
+                   const matrix::Dense<ValueType> *b,
+                   const matrix::Dense<ValueType> *beta,
+                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    if (a->get_strategy()->get_name() == "load_balance") {
+//        dense::scale(exec, beta, c);
+//
+//        const IndexType nwarps = a->get_num_srow_elements();
+//
+//        if (nwarps > 0) {
+//            const dim3 fbcsr_block(config::warp_size, warps_in_block, 1);
+//            const dim3 fbcsr_grid(ceildiv(nwarps, warps_in_block),
+//                                b->get_size()[1]);
+//            hipLaunchKernelGGL(
+//                kernel::abstract_spmv, dim3(fbcsr_grid), dim3(fbcsr_block), 0,
+//                0, nwarps, static_cast<IndexType>(a->get_size()[0]),
+//                as_hip_type(alpha->get_const_values()),
+//                as_hip_type(a->get_const_values()), a->get_const_col_idxs(),
+//                as_hip_type(a->get_const_row_ptrs()),
+//                as_hip_type(a->get_const_srow()),
+//                as_hip_type(b->get_const_values()),
+//                as_hip_type(b->get_stride()), as_hip_type(c->get_values()),
+//                as_hip_type(c->get_stride()));
+//        } else {
+//            GKO_NOT_SUPPORTED(nwarps);
+//        }
+//    } else if (a->get_strategy()->get_name() == "sparselib" ||
+//               a->get_strategy()->get_name() == "cusparse") {
+//        if (hipsparse::is_supported<ValueType, IndexType>::value) {
+//            // TODO: add implementation for int64 and multiple RHS
+//            auto descr = hipsparse::create_mat_descr();
+//
+//            auto row_ptrs = a->get_const_row_ptrs();
+//            auto col_idxs = a->get_const_col_idxs();
+//
+//            if (b->get_stride() != 1 || c->get_stride() != 1)
+//                GKO_NOT_IMPLEMENTED;
+//
+//            hipsparse::spmv(exec->get_hipsparse_handle(),
+//                            HIPSPARSE_OPERATION_NON_TRANSPOSE,
+//                            a->get_size()[0], a->get_size()[1],
+//                            a->get_num_stored_elements(),
+//                            alpha->get_const_values(), descr,
+//                            a->get_const_values(), row_ptrs, col_idxs,
+//                            b->get_const_values(), beta->get_const_values(),
+//                            c->get_values());
+//
+//            hipsparse::destroy(descr);
+//        } else {
+//            GKO_NOT_IMPLEMENTED;
+//        }
+//    } else if (a->get_strategy()->get_name() == "classical") {
+//        IndexType max_length_per_row = 0;
+//        using Tfbcsr = matrix::Fbcsr<ValueType, IndexType>;
+//        if (auto strategy =
+//                std::dynamic_pointer_cast<const typename Tfbcsr::classical>(
+//                    a->get_strategy())) {
+//            max_length_per_row = strategy->get_max_length_per_row();
+//        } else if (auto strategy = std::dynamic_pointer_cast<
+//                       const typename Tfbcsr::automatical>(a->get_strategy()))
+//                       {
+//            max_length_per_row = strategy->get_max_length_per_row();
+//        } else {
+//            GKO_NOT_SUPPORTED(a->get_strategy());
+//        }
+//        host_kernel::select_classical_spmv(
+//            classical_kernels(),
+//            [&max_length_per_row](int compiled_info) {
+//                return max_length_per_row >= compiled_info;
+//            },
+//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c, alpha,
+//            beta);
+//    } else if (a->get_strategy()->get_name() == "merge_path") {
+//        int items_per_thread =
+//            host_kernel::compute_items_per_thread<ValueType, IndexType>(exec);
+//        host_kernel::select_merge_path_spmv(
+//            compiled_kernels(),
+//            [&items_per_thread](int compiled_info) {
+//                return items_per_thread == compiled_info;
+//            },
+//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c, alpha,
+//            beta);
+//    } else {
+//        GKO_NOT_IMPLEMENTED;
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void spgemm(std::shared_ptr<const HipExecutor> exec,
+            const matrix::Fbcsr<ValueType, IndexType> *a,
+            const matrix::Fbcsr<ValueType, IndexType> *b,
+            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    if (hipsparse::is_supported<ValueType, IndexType>::value) {
+//        auto handle = exec->get_hipsparse_handle();
+//        hipsparse::pointer_mode_guard pm_guard(handle);
+//        auto a_descr = hipsparse::create_mat_descr();
+//        auto b_descr = hipsparse::create_mat_descr();
+//        auto c_descr = hipsparse::create_mat_descr();
+//        auto d_descr = hipsparse::create_mat_descr();
+//        auto info = hipsparse::create_spgemm_info();
+//
+//        auto alpha = one<ValueType>();
+//        auto a_nnz = static_cast<IndexType>(a->get_num_stored_elements());
+//        auto a_vals = a->get_const_values();
+//        auto a_row_ptrs = a->get_const_row_ptrs();
+//        auto a_col_idxs = a->get_const_col_idxs();
+//        auto b_nnz = static_cast<IndexType>(b->get_num_stored_elements());
+//        auto b_vals = b->get_const_values();
+//        auto b_row_ptrs = b->get_const_row_ptrs();
+//        auto b_col_idxs = b->get_const_col_idxs();
+//        auto null_value = static_cast<ValueType *>(nullptr);
+//        auto null_index = static_cast<IndexType *>(nullptr);
+//        auto zero_nnz = IndexType{};
+//        auto m = static_cast<IndexType>(a->get_size()[0]);
+//        auto n = static_cast<IndexType>(b->get_size()[1]);
+//        auto k = static_cast<IndexType>(a->get_size()[1]);
+//        auto c_row_ptrs = c->get_row_ptrs();
+//        matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
+//        auto &c_col_idxs_array = c_builder.get_col_idx_array();
+//        auto &c_vals_array = c_builder.get_value_array();
+//
+//        // allocate buffer
+//        size_type buffer_size{};
+//        hipsparse::spgemm_buffer_size(
+//            handle, m, n, k, &alpha, a_descr, a_nnz, a_row_ptrs, a_col_idxs,
+//            b_descr, b_nnz, b_row_ptrs, b_col_idxs, null_value, d_descr,
+//            zero_nnz, null_index, null_index, info, buffer_size);
+//        Array<char> buffer_array(exec, buffer_size);
+//        auto buffer = buffer_array.get_data();
+//
+//        // count nnz
+//        IndexType c_nnz{};
+//        hipsparse::spgemm_nnz(
+//            handle, m, n, k, a_descr, a_nnz, a_row_ptrs, a_col_idxs, b_descr,
+//            b_nnz, b_row_ptrs, b_col_idxs, d_descr, zero_nnz, null_index,
+//            null_index, c_descr, c_row_ptrs, &c_nnz, info, buffer);
+//
+//        // accumulate non-zeros
+//        c_col_idxs_array.resize_and_reset(c_nnz);
+//        c_vals_array.resize_and_reset(c_nnz);
+//        auto c_col_idxs = c_col_idxs_array.get_data();
+//        auto c_vals = c_vals_array.get_data();
+//        hipsparse::spgemm(handle, m, n, k, &alpha, a_descr, a_nnz, a_vals,
+//                          a_row_ptrs, a_col_idxs, b_descr, b_nnz, b_vals,
+//                          b_row_ptrs, b_col_idxs, null_value, d_descr,
+//                          zero_nnz, null_value, null_index, null_index,
+//                          c_descr, c_vals, c_row_ptrs, c_col_idxs, info,
+//                          buffer);
+//
+//        hipsparse::destroy_spgemm_info(info);
+//        hipsparse::destroy(d_descr);
+//        hipsparse::destroy(c_descr);
+//        hipsparse::destroy(b_descr);
+//        hipsparse::destroy(a_descr);
+//    } else {
+//        GKO_NOT_IMPLEMENTED;
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEMM_KERNEL);
+
+
+namespace {
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+void spgeam(syn::value_list<int, subwarp_size>,
+            std::shared_ptr<const HipExecutor> exec, const ValueType *alpha,
+            const IndexType *a_row_ptrs, const IndexType *a_col_idxs,
+            const ValueType *a_vals, const ValueType *beta,
+            const IndexType *b_row_ptrs, const IndexType *b_col_idxs,
+            const ValueType *b_vals,
+            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto m = static_cast<IndexType>(c->get_size()[0]);
+//    auto c_row_ptrs = c->get_row_ptrs();
+//    // count nnz for alpha * A + beta * B
+//    auto subwarps_per_block = default_block_size / subwarp_size;
+//    auto num_blocks = ceildiv(m, subwarps_per_block);
+//    hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel::spgeam_nnz<subwarp_size>),
+//                       dim3(num_blocks), dim3(default_block_size), 0, 0,
+//                       a_row_ptrs, a_col_idxs, b_row_ptrs, b_col_idxs, m,
+//                       c_row_ptrs);
+//
+//    // build row pointers
+//    components::prefix_sum(exec, c_row_ptrs, m + 1);
+//
+//    // accumulate non-zeros for alpha * A + beta * B
+//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
+//    auto c_nnz = exec->copy_val_to_host(c_row_ptrs + m);
+//    c_builder.get_col_idx_array().resize_and_reset(c_nnz);
+//    c_builder.get_value_array().resize_and_reset(c_nnz);
+//    auto c_col_idxs = c->get_col_idxs();
+//    auto c_vals = c->get_values();
+//    hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel::spgeam<subwarp_size>),
+//                       dim3(num_blocks), dim3(default_block_size), 0, 0,
+//                       as_hip_type(alpha), a_row_ptrs, a_col_idxs,
+//                       as_hip_type(a_vals), as_hip_type(beta), b_row_ptrs,
+//                       b_col_idxs, as_hip_type(b_vals), m, c_row_ptrs,
+//                       c_col_idxs, as_hip_type(c_vals));
+//}
+
+GKO_ENABLE_IMPLEMENTATION_SELECTION(select_spgeam, spgeam);
+
+
+}  // namespace
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spgemm(std::shared_ptr<const HipExecutor> exec,
+                     const matrix::Dense<ValueType> *alpha,
+                     const matrix::Fbcsr<ValueType, IndexType> *a,
+                     const matrix::Fbcsr<ValueType, IndexType> *b,
+                     const matrix::Dense<ValueType> *beta,
+                     const matrix::Fbcsr<ValueType, IndexType> *d,
+                     matrix::Fbcsr<ValueType, IndexType> *c)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    if (hipsparse::is_supported<ValueType, IndexType>::value) {
+//        auto handle = exec->get_hipsparse_handle();
+//        hipsparse::pointer_mode_guard pm_guard(handle);
+//        auto a_descr = hipsparse::create_mat_descr();
+//        auto b_descr = hipsparse::create_mat_descr();
+//        auto c_descr = hipsparse::create_mat_descr();
+//        auto d_descr = hipsparse::create_mat_descr();
+//        auto info = hipsparse::create_spgemm_info();
+//
+//        auto a_nnz = static_cast<IndexType>(a->get_num_stored_elements());
+//        auto a_vals = a->get_const_values();
+//        auto a_row_ptrs = a->get_const_row_ptrs();
+//        auto a_col_idxs = a->get_const_col_idxs();
+//        auto b_nnz = static_cast<IndexType>(b->get_num_stored_elements());
+//        auto b_vals = b->get_const_values();
+//        auto b_row_ptrs = b->get_const_row_ptrs();
+//        auto b_col_idxs = b->get_const_col_idxs();
+//        auto d_vals = d->get_const_values();
+//        auto d_row_ptrs = d->get_const_row_ptrs();
+//        auto d_col_idxs = d->get_const_col_idxs();
+//        auto null_value = static_cast<ValueType *>(nullptr);
+//        auto null_index = static_cast<IndexType *>(nullptr);
+//        auto one_value = one<ValueType>();
+//        auto m = static_cast<IndexType>(a->get_size()[0]);
+//        auto n = static_cast<IndexType>(b->get_size()[1]);
+//        auto k = static_cast<IndexType>(a->get_size()[1]);
+//
+//        // allocate buffer
+//        size_type buffer_size{};
+//        hipsparse::spgemm_buffer_size(
+//            handle, m, n, k, &one_value, a_descr, a_nnz, a_row_ptrs,
+//            a_col_idxs, b_descr, b_nnz, b_row_ptrs, b_col_idxs, null_value,
+//            d_descr, IndexType{}, null_index, null_index, info, buffer_size);
+//        Array<char> buffer_array(exec, buffer_size);
+//        auto buffer = buffer_array.get_data();
+//
+//        // count nnz
+//        Array<IndexType> c_tmp_row_ptrs_array(exec, m + 1);
+//        auto c_tmp_row_ptrs = c_tmp_row_ptrs_array.get_data();
+//        IndexType c_nnz{};
+//        hipsparse::spgemm_nnz(
+//            handle, m, n, k, a_descr, a_nnz, a_row_ptrs, a_col_idxs, b_descr,
+//            b_nnz, b_row_ptrs, b_col_idxs, d_descr, IndexType{}, null_index,
+//            null_index, c_descr, c_tmp_row_ptrs, &c_nnz, info, buffer);
+//
+//        // accumulate non-zeros for A * B
+//        Array<IndexType> c_tmp_col_idxs_array(exec, c_nnz);
+//        Array<ValueType> c_tmp_vals_array(exec, c_nnz);
+//        auto c_tmp_col_idxs = c_tmp_col_idxs_array.get_data();
+//        auto c_tmp_vals = c_tmp_vals_array.get_data();
+//        hipsparse::spgemm(handle, m, n, k, &one_value, a_descr, a_nnz, a_vals,
+//                          a_row_ptrs, a_col_idxs, b_descr, b_nnz, b_vals,
+//                          b_row_ptrs, b_col_idxs, null_value, d_descr,
+//                          IndexType{}, null_value, null_index, null_index,
+//                          c_descr, c_tmp_vals, c_tmp_row_ptrs, c_tmp_col_idxs,
+//                          info, buffer);
+//
+//        // destroy hipsparse context
+//        hipsparse::destroy_spgemm_info(info);
+//        hipsparse::destroy(d_descr);
+//        hipsparse::destroy(c_descr);
+//        hipsparse::destroy(b_descr);
+//        hipsparse::destroy(a_descr);
+//
+//        auto total_nnz = c_nnz + d->get_num_stored_elements();
+//        auto nnz_per_row = total_nnz / m;
+//        select_spgeam(
+//            spgeam_kernels(),
+//            [&](int compiled_subwarp_size) {
+//                return compiled_subwarp_size >= nnz_per_row ||
+//                       compiled_subwarp_size == config::warp_size;
+//            },
+//            syn::value_list<int>(), syn::type_list<>(), exec,
+//            alpha->get_const_values(), c_tmp_row_ptrs, c_tmp_col_idxs,
+//            c_tmp_vals, beta->get_const_values(), d_row_ptrs, d_col_idxs,
+//            d_vals, c);
+//    } else {
+//        GKO_NOT_IMPLEMENTED;
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void spgeam(std::shared_ptr<const DefaultExecutor> exec,
+            const matrix::Dense<ValueType> *alpha,
+            const matrix::Fbcsr<ValueType, IndexType> *a,
+            const matrix::Dense<ValueType> *beta,
+            const matrix::Fbcsr<ValueType, IndexType> *b,
+            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto total_nnz =
+//        a->get_num_stored_elements() + b->get_num_stored_elements();
+//    auto nnz_per_row = total_nnz / a->get_size()[0];
+//    select_spgeam(
+//        spgeam_kernels(),
+//        [&](int compiled_subwarp_size) {
+//            return compiled_subwarp_size >= nnz_per_row ||
+//                   compiled_subwarp_size == config::warp_size;
+//        },
+//        syn::value_list<int>(), syn::type_list<>(), exec,
+//        alpha->get_const_values(), a->get_const_row_ptrs(),
+//        a->get_const_col_idxs(), a->get_const_values(),
+//        beta->get_const_values(), b->get_const_row_ptrs(),
+//        b->get_const_col_idxs(), b->get_const_values(), c);
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEAM_KERNEL);
+
+
+template <typename IndexType>
+void convert_row_ptrs_to_idxs(std::shared_ptr<const HipExecutor> exec,
+                              const IndexType *ptrs, size_type num_rows,
+                              IndexType *idxs) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto grid_dim = ceildiv(num_rows, default_block_size);
+//
+//    hipLaunchKernelGGL(kernel::convert_row_ptrs_to_idxs, dim3(grid_dim),
+//                       dim3(default_block_size), 0, 0, num_rows,
+//                       as_hip_type(ptrs), as_hip_type(idxs));
+//}
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_coo(std::shared_ptr<const HipExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *source,
+                    matrix::Coo<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = result->get_size()[0];
+//
+//    auto row_idxs = result->get_row_idxs();
+//    const auto source_row_ptrs = source->get_const_row_ptrs();
+//
+//    convert_row_ptrs_to_idxs(exec, source_row_ptrs, num_rows, row_idxs);
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_dense(std::shared_ptr<const HipExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *source,
+                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto num_rows = result->get_size()[0];
+//    const auto num_cols = result->get_size()[1];
+//    const auto stride = result->get_stride();
+//    const auto row_ptrs = source->get_const_row_ptrs();
+//    const auto col_idxs = source->get_const_col_idxs();
+//    const auto vals = source->get_const_values();
+//
+//    const dim3 block_size(config::warp_size,
+//                          config::max_block_size / config::warp_size, 1);
+//    const dim3 init_grid_dim(ceildiv(num_cols, block_size.x),
+//                             ceildiv(num_rows, block_size.y), 1);
+//    hipLaunchKernelGGL(kernel::initialize_zero_dense, dim3(init_grid_dim),
+//                       dim3(block_size), 0, 0, num_rows, num_cols, stride,
+//                       as_hip_type(result->get_values()));
+//
+//    auto grid_dim = ceildiv(num_rows, default_block_size);
+//    hipLaunchKernelGGL(
+//        kernel::fill_in_dense, dim3(grid_dim), dim3(default_block_size), 0, 0,
+//        num_rows, as_hip_type(row_ptrs), as_hip_type(col_idxs),
+//        as_hip_type(vals), stride, as_hip_type(result->get_values()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_sellp(std::shared_ptr<const HipExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *source,
+                      matrix::Sellp<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto num_rows = result->get_size()[0];
+//    const auto num_cols = result->get_size()[1];
+//
+//    auto result_values = result->get_values();
+//    auto result_col_idxs = result->get_col_idxs();
+//    auto slice_lengths = result->get_slice_lengths();
+//    auto slice_sets = result->get_slice_sets();
+//
+//    const auto slice_size = (result->get_slice_size() == 0)
+//                                ? matrix::default_slice_size
+//                                : result->get_slice_size();
+//    const auto stride_factor = (result->get_stride_factor() == 0)
+//                                   ? matrix::default_stride_factor
+//                                   : result->get_stride_factor();
+//    const int slice_num = ceildiv(num_rows, slice_size);
+//
+//    const auto source_values = source->get_const_values();
+//    const auto source_row_ptrs = source->get_const_row_ptrs();
+//    const auto source_col_idxs = source->get_const_col_idxs();
+//
+//    auto nnz_per_row = Array<size_type>(exec, num_rows);
+//    auto grid_dim = ceildiv(num_rows, default_block_size);
+//
+//    if (grid_dim > 0) {
+//        hipLaunchKernelGGL(kernel::calculate_nnz_per_row, dim3(grid_dim),
+//                           dim3(default_block_size), 0, 0, num_rows,
+//                           as_hip_type(source_row_ptrs),
+//                           as_hip_type(nnz_per_row.get_data()));
+//    }
+//
+//    grid_dim = slice_num;
+//
+//    if (grid_dim > 0) {
+//        hipLaunchKernelGGL(kernel::calculate_slice_lengths, dim3(grid_dim),
+//                           dim3(config::warp_size), 0, 0, num_rows,
+//                           slice_size, stride_factor,
+//                           as_hip_type(nnz_per_row.get_const_data()),
+//                           as_hip_type(slice_lengths),
+//                           as_hip_type(slice_sets));
+//    }
+//
+//    components::prefix_sum(exec, slice_sets, slice_num + 1);
+//
+//    grid_dim = ceildiv(num_rows, default_block_size);
+//    if (grid_dim > 0) {
+//        hipLaunchKernelGGL(
+//            kernel::fill_in_sellp, dim3(grid_dim), dim3(default_block_size),
+//            0, 0, num_rows, slice_size, as_hip_type(source_values),
+//            as_hip_type(source_row_ptrs), as_hip_type(source_col_idxs),
+//            as_hip_type(slice_lengths), as_hip_type(slice_sets),
+//            as_hip_type(result_col_idxs), as_hip_type(result_values));
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_ell(std::shared_ptr<const HipExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *source,
+                    matrix::Ell<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto source_values = source->get_const_values();
+//    const auto source_row_ptrs = source->get_const_row_ptrs();
+//    const auto source_col_idxs = source->get_const_col_idxs();
+//
+//    auto result_values = result->get_values();
+//    auto result_col_idxs = result->get_col_idxs();
+//    const auto stride = result->get_stride();
+//    const auto max_nnz_per_row = result->get_num_stored_elements_per_row();
+//    const auto num_rows = result->get_size()[0];
+//    const auto num_cols = result->get_size()[1];
+//
+//    const auto init_grid_dim =
+//        ceildiv(max_nnz_per_row * num_rows, default_block_size);
+//
+//    hipLaunchKernelGGL(kernel::initialize_zero_ell, dim3(init_grid_dim),
+//                       dim3(default_block_size), 0, 0, max_nnz_per_row,
+//                       stride, as_hip_type(result_values),
+//                       as_hip_type(result_col_idxs));
+//
+//    const auto grid_dim =
+//        ceildiv(num_rows * config::warp_size, default_block_size);
+//
+//    hipLaunchKernelGGL(kernel::fill_in_ell, dim3(grid_dim),
+//                       dim3(default_block_size), 0, 0, num_rows, stride,
+//                       as_hip_type(source_values),
+//                       as_hip_type(source_row_ptrs),
+//                       as_hip_type(source_col_idxs),
+//                       as_hip_type(result_values),
+//                       as_hip_type(result_col_idxs));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_total_cols(std::shared_ptr<const HipExecutor> exec,
+                          const matrix::Fbcsr<ValueType, IndexType> *source,
+                          size_type *result, size_type stride_factor,
+                          size_type slice_size) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto num_rows = source->get_size()[0];
+//
+//    if (num_rows == 0) {
+//        *result = 0;
+//        return;
+//    }
+//
+//    const auto slice_num = ceildiv(num_rows, slice_size);
+//    const auto row_ptrs = source->get_const_row_ptrs();
+//
+//    auto nnz_per_row = Array<size_type>(exec, num_rows);
+//    auto grid_dim = ceildiv(num_rows, default_block_size);
+//
+//    hipLaunchKernelGGL(kernel::calculate_nnz_per_row, dim3(grid_dim),
+//                       dim3(default_block_size), 0, 0, num_rows,
+//                       as_hip_type(row_ptrs),
+//                       as_hip_type(nnz_per_row.get_data()));
+//
+//    grid_dim = ceildiv(slice_num * config::warp_size, default_block_size);
+//    auto max_nnz_per_slice = Array<size_type>(exec, slice_num);
+//
+//    hipLaunchKernelGGL(kernel::reduce_max_nnz_per_slice, dim3(grid_dim),
+//                       dim3(default_block_size), 0, 0, num_rows, slice_size,
+//                       stride_factor,
+//                       as_hip_type(nnz_per_row.get_const_data()),
+//                       as_hip_type(max_nnz_per_slice.get_data()));
+//
+//    grid_dim = ceildiv(slice_num, default_block_size);
+//    auto block_results = Array<size_type>(exec, grid_dim);
+//
+//    hipLaunchKernelGGL(kernel::reduce_total_cols, dim3(grid_dim),
+//                       dim3(default_block_size), 0, 0, slice_num,
+//                       as_hip_type(max_nnz_per_slice.get_const_data()),
+//                       as_hip_type(block_results.get_data()));
+//
+//    auto d_result = Array<size_type>(exec, 1);
+//
+//    hipLaunchKernelGGL(kernel::reduce_total_cols, dim3(1),
+//                       dim3(default_block_size), 0, 0, grid_dim,
+//                       as_hip_type(block_results.get_const_data()),
+//                       as_hip_type(d_result.get_data()));
+//
+//    *result = exec->copy_val_to_host(d_result.get_const_data());
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void transpose(std::shared_ptr<const HipExecutor> exec,
+               const matrix::Fbcsr<ValueType, IndexType> *orig,
+               matrix::Fbcsr<ValueType, IndexType> *trans) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    if (hipsparse::is_supported<ValueType, IndexType>::value) {
+//        hipsparseAction_t copyValues = HIPSPARSE_ACTION_NUMERIC;
+//        hipsparseIndexBase_t idxBase = HIPSPARSE_INDEX_BASE_ZERO;
+//
+//        hipsparse::transpose(
+//            exec->get_hipsparse_handle(), orig->get_size()[0],
+//            orig->get_size()[1], orig->get_num_stored_elements(),
+//            orig->get_const_values(), orig->get_const_row_ptrs(),
+//            orig->get_const_col_idxs(), trans->get_values(),
+//            trans->get_row_ptrs(), trans->get_col_idxs(), copyValues,
+//            idxBase);
+//    } else {
+//        GKO_NOT_IMPLEMENTED;
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void conj_transpose(std::shared_ptr<const HipExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *orig,
+                    matrix::Fbcsr<ValueType, IndexType> *trans)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    if (hipsparse::is_supported<ValueType, IndexType>::value) {
+//        const dim3 block_size(default_block_size, 1, 1);
+//        const dim3 grid_size(
+//            ceildiv(trans->get_num_stored_elements(), block_size.x), 1, 1);
+//
+//        hipsparseAction_t copyValues = HIPSPARSE_ACTION_NUMERIC;
+//        hipsparseIndexBase_t idxBase = HIPSPARSE_INDEX_BASE_ZERO;
+//
+//        hipsparse::transpose(
+//            exec->get_hipsparse_handle(), orig->get_size()[0],
+//            orig->get_size()[1], orig->get_num_stored_elements(),
+//            orig->get_const_values(), orig->get_const_row_ptrs(),
+//            orig->get_const_col_idxs(), trans->get_values(),
+//            trans->get_col_idxs(), trans->get_row_ptrs(), copyValues,
+//            idxBase);
+//
+//        hipLaunchKernelGGL(conjugate_kernel, dim3(grid_size),
+//        dim3(block_size),
+//                           0, 0, trans->get_num_stored_elements(),
+//                           as_hip_type(trans->get_values()));
+//    } else {
+//        GKO_NOT_IMPLEMENTED;
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void row_permute(std::shared_ptr<const HipExecutor> exec,
+                 const Array<IndexType> *permutation_indices,
+                 const matrix::Fbcsr<ValueType, IndexType> *orig,
+                 matrix::Fbcsr<ValueType, IndexType> *row_permuted)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void column_permute(std::shared_ptr<const HipExecutor> exec,
+                    const Array<IndexType> *permutation_indices,
+                    const matrix::Fbcsr<ValueType, IndexType> *orig,
+                    matrix::Fbcsr<ValueType, IndexType> *column_permuted)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_row_permute(std::shared_ptr<const HipExecutor> exec,
+                         const Array<IndexType> *permutation_indices,
+                         const matrix::Fbcsr<ValueType, IndexType> *orig,
+                         matrix::Fbcsr<ValueType, IndexType> *row_permuted)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_column_permute(
+    std::shared_ptr<const HipExecutor> exec,
+    const Array<IndexType> *permutation_indices,
+    const matrix::Fbcsr<ValueType, IndexType> *orig,
+    matrix::Fbcsr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_max_nnz_per_row(
+    std::shared_ptr<const HipExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *source,
+    size_type *result) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto num_rows = source->get_size()[0];
+//
+//    auto nnz_per_row = Array<size_type>(exec, num_rows);
+//    auto block_results = Array<size_type>(exec, default_block_size);
+//    auto d_result = Array<size_type>(exec, 1);
+//
+//    const auto grid_dim = ceildiv(num_rows, default_block_size);
+//    hipLaunchKernelGGL(kernel::calculate_nnz_per_row, dim3(grid_dim),
+//                       dim3(default_block_size), 0, 0, num_rows,
+//                       as_hip_type(source->get_const_row_ptrs()),
+//                       as_hip_type(nnz_per_row.get_data()));
+//
+//    const auto n = ceildiv(num_rows, default_block_size);
+//    const auto reduce_dim = n <= default_block_size ? n : default_block_size;
+//    hipLaunchKernelGGL(kernel::reduce_max_nnz, dim3(reduce_dim),
+//                       dim3(default_block_size), 0, 0, num_rows,
+//                       as_hip_type(nnz_per_row.get_const_data()),
+//                       as_hip_type(block_results.get_data()));
+//
+//    hipLaunchKernelGGL(kernel::reduce_max_nnz, dim3(1),
+//                       dim3(default_block_size), 0, 0, reduce_dim,
+//                       as_hip_type(block_results.get_const_data()),
+//                       as_hip_type(d_result.get_data()));
+//
+//    *result = exec->copy_val_to_host(d_result.get_const_data());
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_hybrid(std::shared_ptr<const HipExecutor> exec,
+                       const matrix::Fbcsr<ValueType, IndexType> *source,
+                       matrix::Hybrid<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto ell_val = result->get_ell_values();
+//    auto ell_col = result->get_ell_col_idxs();
+//    auto coo_val = result->get_coo_values();
+//    auto coo_col = result->get_coo_col_idxs();
+//    auto coo_row = result->get_coo_row_idxs();
+//    const auto stride = result->get_ell_stride();
+//    const auto max_nnz_per_row =
+//    result->get_ell_num_stored_elements_per_row(); const auto num_rows =
+//    result->get_size()[0]; const auto coo_num_stored_elements =
+//    result->get_coo_num_stored_elements(); auto grid_dim =
+//    ceildiv(max_nnz_per_row * num_rows, default_block_size);
+//
+//    hipLaunchKernelGGL(kernel::initialize_zero_ell, dim3(grid_dim),
+//                       dim3(default_block_size), 0, 0, max_nnz_per_row,
+//                       stride, as_hip_type(ell_val), as_hip_type(ell_col));
+//
+//    grid_dim = ceildiv(num_rows, default_block_size);
+//    auto coo_offset = Array<size_type>(exec, num_rows);
+//    hipLaunchKernelGGL(kernel::calculate_hybrid_coo_row_nnz, dim3(grid_dim),
+//                       dim3(default_block_size), 0, 0, num_rows,
+//                       max_nnz_per_row,
+//                       as_hip_type(source->get_const_row_ptrs()),
+//                       as_hip_type(coo_offset.get_data()));
+//
+//    components::prefix_sum(exec, coo_offset.get_data(), num_rows);
+//
+//    grid_dim = ceildiv(num_rows * config::warp_size, default_block_size);
+//    hipLaunchKernelGGL(kernel::fill_in_hybrid, dim3(grid_dim),
+//                       dim3(default_block_size), 0, 0, num_rows, stride,
+//                       max_nnz_per_row,
+//                       as_hip_type(source->get_const_values()),
+//                       as_hip_type(source->get_const_row_ptrs()),
+//                       as_hip_type(source->get_const_col_idxs()),
+//                       as_hip_type(coo_offset.get_const_data()),
+//                       as_hip_type(ell_val), as_hip_type(ell_col),
+//                       as_hip_type(coo_val), as_hip_type(coo_col),
+//                       as_hip_type(coo_row));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_nonzeros_per_row(
+    std::shared_ptr<const HipExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *source,
+    Array<size_type> *result) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto num_rows = source->get_size()[0];
+//    auto row_ptrs = source->get_const_row_ptrs();
+//    auto grid_dim = ceildiv(num_rows, default_block_size);
+//
+//    hipLaunchKernelGGL(kernel::calculate_nnz_per_row, dim3(grid_dim),
+//                       dim3(default_block_size), 0, 0, num_rows,
+//                       as_hip_type(row_ptrs),
+//                       as_hip_type(result->get_data()));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void sort_by_column_index(std::shared_ptr<const HipExecutor> exec,
+                          matrix::Fbcsr<ValueType, IndexType> *to_sort)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    if (hipsparse::is_supported<ValueType, IndexType>::value) {
+//        auto handle = exec->get_hipsparse_handle();
+//        auto descr = hipsparse::create_mat_descr();
+//        auto m = IndexType(to_sort->get_size()[0]);
+//        auto n = IndexType(to_sort->get_size()[1]);
+//        auto nnz = IndexType(to_sort->get_num_stored_elements());
+//        auto row_ptrs = to_sort->get_const_row_ptrs();
+//        auto col_idxs = to_sort->get_col_idxs();
+//        auto vals = to_sort->get_values();
+//
+//        // copy values
+//        Array<ValueType> tmp_vals_array(exec, nnz);
+//        exec->copy(nnz, vals, tmp_vals_array.get_data());
+//        auto tmp_vals = tmp_vals_array.get_const_data();
+//
+//        // init identity permutation
+//        Array<IndexType> permutation_array(exec, nnz);
+//        auto permutation = permutation_array.get_data();
+//        hipsparse::create_identity_permutation(handle, nnz, permutation);
+//
+//        // allocate buffer
+//        size_type buffer_size{};
+//        hipsparse::fbcsrsort_buffer_size(handle, m, n, nnz, row_ptrs,
+//        col_idxs,
+//                                       buffer_size);
+//        Array<char> buffer_array{exec, buffer_size};
+//        auto buffer = buffer_array.get_data();
+//
+//        // sort column indices
+//        hipsparse::fbcsrsort(handle, m, n, nnz, descr, row_ptrs, col_idxs,
+//                           permutation, buffer);
+//
+//        // sort values
+//        hipsparse::gather(handle, nnz, tmp_vals, vals, permutation);
+//
+//        hipsparse::destroy(descr);
+//    } else {
+//        GKO_NOT_IMPLEMENTED;
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
+
+
+template <typename ValueType, typename IndexType>
+void is_sorted_by_column_index(
+    std::shared_ptr<const HipExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *to_check,
+    bool *is_sorted) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    *is_sorted = true;
+//    auto cpu_array = Array<bool>::view(exec->get_master(), 1, is_sorted);
+//    auto gpu_array = Array<bool>{exec, cpu_array};
+//    auto block_size = default_block_size;
+//    auto num_rows = static_cast<IndexType>(to_check->get_size()[0]);
+//    auto num_blocks = ceildiv(num_rows, block_size);
+//    hipLaunchKernelGGL(
+//        HIP_KERNEL_NAME(kernel::check_unsorted), dim3(num_blocks),
+//        dim3(block_size), 0, 0, to_check->get_const_row_ptrs(),
+//        to_check->get_const_col_idxs(), num_rows, gpu_array.get_data());
+//    cpu_array = gpu_array;
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
+
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(std::shared_ptr<const HipExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *orig,
+                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto nnz = orig->get_num_stored_elements();
+//    const auto diag_size = diag->get_size()[0];
+//    const auto num_blocks =
+//        ceildiv(config::warp_size * diag_size, default_block_size);
+//
+//    const auto orig_values = orig->get_const_values();
+//    const auto orig_row_ptrs = orig->get_const_row_ptrs();
+//    const auto orig_col_idxs = orig->get_const_col_idxs();
+//    auto diag_values = diag->get_values();
+//
+//    hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel::extract_diagonal),
+//                       dim3(num_blocks), dim3(default_block_size), 0, 0,
+//                       diag_size, nnz, as_hip_type(orig_values),
+//                       as_hip_type(orig_row_ptrs), as_hip_type(orig_col_idxs),
+//                       as_hip_type(diag_values));
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL);
+
+
+}  // namespace fbcsr
+}  // namespace hip
+}  // namespace kernels
+}  // namespace gko
diff --git a/hip/test/matrix/CMakeLists.txt b/hip/test/matrix/CMakeLists.txt
index 94e92f08f5c..4a32b5272f7 100644
--- a/hip/test/matrix/CMakeLists.txt
+++ b/hip/test/matrix/CMakeLists.txt
@@ -3,5 +3,6 @@ ginkgo_create_hip_test(csr_kernels)
 ginkgo_create_hip_test(dense_kernels)
 ginkgo_create_hip_test(diagonal_kernels)
 ginkgo_create_hip_test(ell_kernels)
+ginkgo_create_hip_test(fbcsr_kernels)
 ginkgo_create_hip_test(hybrid_kernels)
 ginkgo_create_hip_test(sellp_kernels)
diff --git a/hip/test/matrix/fbcsr_kernels.hip.cpp b/hip/test/matrix/fbcsr_kernels.hip.cpp
new file mode 100644
index 00000000000..79b873cb02d
--- /dev/null
+++ b/hip/test/matrix/fbcsr_kernels.hip.cpp
@@ -0,0 +1,866 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/matrix/fbcsr.hpp>
+
+
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+#include <ginkgo/core/matrix/ell.hpp>
+#include <ginkgo/core/matrix/hybrid.hpp>
+#include <ginkgo/core/matrix/identity.hpp>
+#include <ginkgo/core/matrix/sellp.hpp>
+#include <ginkgo/core/matrix/sparsity_fbcsr.hpp>
+
+
+#include "core/matrix/fbcsr_kernels.hpp"
+#include "hip/test/utils.hip.hpp"
+
+
+namespace {
+
+
+class Fbcsr : public ::testing::Test {
+protected:
+    using Vec = gko::matrix::Dense<>;
+    using Mtx = gko::matrix::Fbcsr<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
+    using ComplexMtx = gko::matrix::Fbcsr<std::complex<double>>;
+
+    Fbcsr() : mtx_size(532, 231), rand_engine(42) {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::HipExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        hip = gko::HipExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (hip != nullptr) {
+            ASSERT_NO_THROW(hip->synchronize());
+        }
+    }
+
+    template <typename MtxType>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row)
+    {
+        return gko::test::generate_random_matrix<MtxType>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(min_nnz_row, num_cols),
+            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    void set_up_apply_data(std::shared_ptr<Mtx::strategy_type> strategy,
+                           int num_vectors = 1)
+    {
+        mtx = Mtx::create(ref, strategy);
+        mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[1], 1));
+        square_mtx = Mtx::create(ref, strategy);
+        square_mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[0], 1));
+        expected = gen_mtx<Vec>(mtx_size[0], num_vectors, 1);
+        y = gen_mtx<Vec>(mtx_size[1], num_vectors, 1);
+        alpha = gko::initialize<Vec>({2.0}, ref);
+        beta = gko::initialize<Vec>({-1.0}, ref);
+        dmtx = Mtx::create(hip, strategy);
+        dmtx->copy_from(mtx.get());
+        square_dmtx = Mtx::create(hip, strategy);
+        square_dmtx->copy_from(square_mtx.get());
+        dresult = Vec::create(hip);
+        dresult->copy_from(expected.get());
+        dy = Vec::create(hip);
+        dy->copy_from(y.get());
+        dalpha = Vec::create(hip);
+        dalpha->copy_from(alpha.get());
+        dbeta = Vec::create(hip);
+        dbeta->copy_from(beta.get());
+    }
+
+    void set_up_apply_complex_data(
+        std::shared_ptr<ComplexMtx::strategy_type> strategy)
+    {
+        complex_mtx = ComplexMtx::create(ref, strategy);
+        complex_mtx->copy_from(
+            gen_mtx<ComplexVec>(mtx_size[0], mtx_size[1], 1));
+        complex_dmtx = ComplexMtx::create(hip, strategy);
+        complex_dmtx->copy_from(complex_mtx.get());
+    }
+
+    struct matrix_pair {
+        std::unique_ptr<Mtx> ref;
+        std::unique_ptr<Mtx> hip;
+    };
+
+    matrix_pair gen_unsorted_mtx()
+    {
+        constexpr int min_nnz_per_row = 2;  // Must be at least 2
+        auto local_mtx_ref =
+            gen_mtx<Mtx>(mtx_size[0], mtx_size[1], min_nnz_per_row);
+        for (size_t row = 0; row < mtx_size[0]; ++row) {
+            const auto row_ptrs = local_mtx_ref->get_const_row_ptrs();
+            const auto start_row = row_ptrs[row];
+            auto col_idx = local_mtx_ref->get_col_idxs() + start_row;
+            auto vals = local_mtx_ref->get_values() + start_row;
+            const auto nnz_in_this_row = row_ptrs[row + 1] - row_ptrs[row];
+            auto swap_idx_dist =
+                std::uniform_int_distribution<>(0, nnz_in_this_row - 1);
+            // shuffle `nnz_in_this_row / 2` times
+            for (size_t perm = 0; perm < nnz_in_this_row; perm += 2) {
+                const auto idx1 = swap_idx_dist(rand_engine);
+                const auto idx2 = swap_idx_dist(rand_engine);
+                std::swap(col_idx[idx1], col_idx[idx2]);
+                std::swap(vals[idx1], vals[idx2]);
+            }
+        }
+        auto local_mtx_hip = Mtx::create(hip);
+        local_mtx_hip->copy_from(local_mtx_ref.get());
+
+        return {std::move(local_mtx_ref), std::move(local_mtx_hip)};
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::HipExecutor> hip;
+
+    const gko::dim<2> mtx_size;
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Mtx> mtx;
+    std::unique_ptr<ComplexMtx> complex_mtx;
+    std::unique_ptr<Mtx> square_mtx;
+    std::unique_ptr<Vec> expected;
+    std::unique_ptr<Vec> y;
+    std::unique_ptr<Vec> alpha;
+    std::unique_ptr<Vec> beta;
+
+    std::unique_ptr<Mtx> dmtx;
+    std::unique_ptr<ComplexMtx> complex_dmtx;
+    std::unique_ptr<Mtx> square_dmtx;
+    std::unique_ptr<Vec> dresult;
+    std::unique_ptr<Vec> dy;
+    std::unique_ptr<Vec> dalpha;
+    std::unique_ptr<Vec> dbeta;
+};
+
+
+TEST_F(Fbcsr, StrategyAfterCopyIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::load_balance>(hip));
+//
+//    ASSERT_EQ(mtx->get_strategy()->get_name(),
+//              dmtx->get_strategy()->get_name());
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithLoadBalance)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::load_balance>(hip));
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithLoadBalance)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::load_balance>(hip));
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithHipsparse)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithHipsparse)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithMergePath)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::merge_path>());
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithMergePath)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::merge_path>());
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithClassical)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::classical>());
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithClassical)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::classical>());
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithAutomatical)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyToDenseMatrixIsEquivalentToRefWithLoadBalance)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::load_balance>(hip), 3);
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithLoadBalance)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::load_balance>(hip), 3);
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyToDenseMatrixIsEquivalentToRefWithClassical)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::classical>(), 3);
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithClassical)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::classical>(), 3);
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyToDenseMatrixIsEquivalentToRefWithMergePath)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::merge_path>(), 3);
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithMergePath)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::merge_path>(), 3);
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyToFbcsrMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+//    auto trans = mtx->transpose();
+//    auto d_trans = dmtx->transpose();
+//
+//    mtx->apply(alpha.get(), trans.get(), beta.get(), square_mtx.get());
+//    dmtx->apply(dalpha.get(), d_trans.get(), dbeta.get(), square_dmtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
+//    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+//    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyToFbcsrMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+//    auto trans = mtx->transpose();
+//    auto d_trans = dmtx->transpose();
+//
+//    mtx->apply(trans.get(), square_mtx.get());
+//    dmtx->apply(d_trans.get(), square_dmtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
+//    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+//    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+//    auto a = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
+//    auto b = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
+//    auto da = Mtx::create(hip);
+//    auto db = Mtx::create(hip);
+//    da->copy_from(a.get());
+//    db->copy_from(b.get());
+//    auto id = gko::matrix::Identity<Mtx::value_type>::create(ref,
+//    mtx_size[1]); auto did =
+//    gko::matrix::Identity<Mtx::value_type>::create(hip, mtx_size[1]);
+//
+//    a->apply(alpha.get(), id.get(), beta.get(), b.get());
+//    da->apply(dalpha.get(), did.get(), dbeta.get(), db.get());
+//
+//    GKO_ASSERT_MTX_NEAR(b, db, 1e-14);
+//    GKO_ASSERT_MTX_EQ_SPARSITY(b, db);
+//    ASSERT_TRUE(db->is_sorted_by_column_index());
+//}
+
+
+TEST_F(Fbcsr, TransposeIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+//
+//    auto trans = mtx->transpose();
+//    auto d_trans = dmtx->transpose();
+//
+//    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(d_trans.get()),
+//                        static_cast<Mtx *>(trans.get()), 0.0);
+//    ASSERT_TRUE(static_cast<Mtx
+//    *>(d_trans.get())->is_sorted_by_column_index());
+//}
+
+
+TEST_F(Fbcsr, ConvertToDenseIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto dense_mtx = gko::matrix::Dense<>::create(ref);
+//    auto ddense_mtx = gko::matrix::Dense<>::create(hip);
+//
+//    mtx->convert_to(dense_mtx.get());
+//    dmtx->convert_to(ddense_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToDenseIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto dense_mtx = gko::matrix::Dense<>::create(ref);
+//    auto ddense_mtx = gko::matrix::Dense<>::create(hip);
+//
+//    mtx->move_to(dense_mtx.get());
+//    dmtx->move_to(ddense_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, ConvertToEllIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto ell_mtx = gko::matrix::Ell<>::create(ref);
+//    auto dell_mtx = gko::matrix::Ell<>::create(hip);
+//
+//    mtx->convert_to(ell_mtx.get());
+//    dmtx->convert_to(dell_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(ell_mtx.get(), dell_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToEllIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto ell_mtx = gko::matrix::Ell<>::create(ref);
+//    auto dell_mtx = gko::matrix::Ell<>::create(hip);
+//
+//    mtx->move_to(ell_mtx.get());
+//    dmtx->move_to(dell_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(ell_mtx.get(), dell_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, ConvertToSparsityFbcsrIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(ref);
+//    auto d_sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(hip);
+//
+//    mtx->convert_to(sparsity_mtx.get());
+//    dmtx->convert_to(d_sparsity_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(sparsity_mtx.get(), d_sparsity_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToSparsityFbcsrIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(ref);
+//    auto d_sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(hip);
+//
+//    mtx->move_to(sparsity_mtx.get());
+//    dmtx->move_to(d_sparsity_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(sparsity_mtx.get(), d_sparsity_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, CalculateMaxNnzPerRowIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    gko::size_type max_nnz_per_row;
+//    gko::size_type dmax_nnz_per_row;
+//
+//    gko::kernels::reference::fbcsr::calculate_max_nnz_per_row(ref, mtx.get(),
+//                                                            &max_nnz_per_row);
+//    gko::kernels::hip::fbcsr::calculate_max_nnz_per_row(hip, dmtx.get(),
+//                                                      &dmax_nnz_per_row);
+//
+//    ASSERT_EQ(max_nnz_per_row, dmax_nnz_per_row);
+//}
+
+
+TEST_F(Fbcsr, ConvertToCooIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto coo_mtx = gko::matrix::Coo<>::create(ref);
+//    auto dcoo_mtx = gko::matrix::Coo<>::create(hip);
+//
+//    mtx->convert_to(coo_mtx.get());
+//    dmtx->convert_to(dcoo_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToCooIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto coo_mtx = gko::matrix::Coo<>::create(ref);
+//    auto dcoo_mtx = gko::matrix::Coo<>::create(hip);
+//
+//    mtx->move_to(coo_mtx.get());
+//    dmtx->move_to(dcoo_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, ConvertToSellpIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
+//    auto dsellp_mtx = gko::matrix::Sellp<>::create(hip);
+//
+//    mtx->convert_to(sellp_mtx.get());
+//    dmtx->convert_to(dsellp_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(sellp_mtx.get(), dsellp_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToSellpIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
+//    auto dsellp_mtx = gko::matrix::Sellp<>::create(hip);
+//
+//    mtx->move_to(sellp_mtx.get());
+//    dmtx->move_to(dsellp_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(sellp_mtx.get(), dsellp_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, ConvertsEmptyToSellp)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto dempty_mtx = Mtx::create(hip);
+//    auto dsellp_mtx = gko::matrix::Sellp<>::create(hip);
+//
+//    dempty_mtx->convert_to(dsellp_mtx.get());
+//
+//    ASSERT_EQ(hip->copy_val_to_host(dsellp_mtx->get_const_slice_sets()), 0);
+//    ASSERT_FALSE(dsellp_mtx->get_size());
+//}
+
+
+TEST_F(Fbcsr, CalculateTotalColsIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    gko::size_type total_cols;
+//    gko::size_type dtotal_cols;
+//
+//    gko::kernels::reference::fbcsr::calculate_total_cols(
+//        ref, mtx.get(), &total_cols, 2, gko::matrix::default_slice_size);
+//    gko::kernels::hip::fbcsr::calculate_total_cols(
+//        hip, dmtx.get(), &dtotal_cols, 2, gko::matrix::default_slice_size);
+//
+//    ASSERT_EQ(total_cols, dtotal_cols);
+//}
+
+
+TEST_F(Fbcsr, CalculatesNonzerosPerRow)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    gko::Array<gko::size_type> row_nnz(ref, mtx->get_size()[0]);
+//    gko::Array<gko::size_type> drow_nnz(hip, dmtx->get_size()[0]);
+//
+//    gko::kernels::reference::fbcsr::calculate_nonzeros_per_row(ref, mtx.get(),
+//                                                             &row_nnz);
+//    gko::kernels::hip::fbcsr::calculate_nonzeros_per_row(hip, dmtx.get(),
+//                                                       &drow_nnz);
+//
+//    GKO_ASSERT_ARRAY_EQ(row_nnz, drow_nnz);
+//}
+
+
+TEST_F(Fbcsr, ConvertToHybridIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Hybrid_type = gko::matrix::Hybrid<>;
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto hybrid_mtx = Hybrid_type::create(
+//        ref, std::make_shared<Hybrid_type::column_limit>(2));
+//    auto dhybrid_mtx = Hybrid_type::create(
+//        hip, std::make_shared<Hybrid_type::column_limit>(2));
+//
+//    mtx->convert_to(hybrid_mtx.get());
+//    dmtx->convert_to(dhybrid_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToHybridIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Hybrid_type = gko::matrix::Hybrid<>;
+//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+//    auto hybrid_mtx = Hybrid_type::create(
+//        ref, std::make_shared<Hybrid_type::column_limit>(2));
+//    auto dhybrid_mtx = Hybrid_type::create(
+//        hip, std::make_shared<Hybrid_type::column_limit>(2));
+//
+//    mtx->move_to(hybrid_mtx.get());
+//    dmtx->move_to(dhybrid_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, RecognizeSortedMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+//    bool is_sorted_hip{};
+//    bool is_sorted_ref{};
+//
+//    is_sorted_ref = mtx->is_sorted_by_column_index();
+//    is_sorted_hip = dmtx->is_sorted_by_column_index();
+//
+//    ASSERT_EQ(is_sorted_ref, is_sorted_hip);
+//}
+
+
+TEST_F(Fbcsr, RecognizeUnsortedMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto uns_mtx = gen_unsorted_mtx();
+//    bool is_sorted_hip{};
+//    bool is_sorted_ref{};
+//
+//    is_sorted_ref = uns_mtx.ref->is_sorted_by_column_index();
+//    is_sorted_hip = uns_mtx.hip->is_sorted_by_column_index();
+//
+//    ASSERT_EQ(is_sorted_ref, is_sorted_hip);
+//}
+
+
+TEST_F(Fbcsr, SortSortedMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+//
+//    mtx->sort_by_column_index();
+//    dmtx->sort_by_column_index();
+//
+//    // Values must be unchanged, therefore, tolerance is `0`
+//    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 0);
+//}
+
+
+TEST_F(Fbcsr, SortUnsortedMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto uns_mtx = gen_unsorted_mtx();
+//
+//    uns_mtx.ref->sort_by_column_index();
+//    uns_mtx.hip->sort_by_column_index();
+//
+//    // Values must be unchanged, therefore, tolerance is `0`
+//    GKO_ASSERT_MTX_NEAR(uns_mtx.ref, uns_mtx.hip, 0);
+//}
+
+
+TEST_F(Fbcsr, OneAutomaticalWorksWithDifferentMatrices)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto automatical = std::make_shared<Mtx::automatical>(hip);
+//    auto row_len_limit = std::max(automatical->nvidia_row_len_limit,
+//                                  automatical->amd_row_len_limit);
+//    auto load_balance_mtx = Mtx::create(ref);
+//    auto classical_mtx = Mtx::create(ref);
+//    load_balance_mtx->copy_from(
+//        gen_mtx<Vec>(1, row_len_limit + 1000, row_len_limit + 1));
+//    classical_mtx->copy_from(gen_mtx<Vec>(50, 50, 1));
+//    auto load_balance_mtx_d = Mtx::create(hip);
+//    auto classical_mtx_d = Mtx::create(hip);
+//    load_balance_mtx_d->copy_from(load_balance_mtx.get());
+//    classical_mtx_d->copy_from(classical_mtx.get());
+//
+//    load_balance_mtx_d->set_strategy(automatical);
+//    classical_mtx_d->set_strategy(automatical);
+//
+//    EXPECT_EQ("load_balance", load_balance_mtx_d->get_strategy()->get_name());
+//    EXPECT_EQ("classical", classical_mtx_d->get_strategy()->get_name());
+//    ASSERT_NE(load_balance_mtx_d->get_strategy().get(),
+//              classical_mtx_d->get_strategy().get());
+//}
+
+
+TEST_F(Fbcsr, ExtractDiagonalIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+//
+//    auto diag = mtx->extract_diagonal();
+//    auto ddiag = dmtx->extract_diagonal();
+//
+//    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+//}
+
+
+TEST_F(Fbcsr, InplaceAbsoluteMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+//
+//    mtx->compute_absolute_inplace();
+//    dmtx->compute_absolute_inplace();
+//
+//    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, OutplaceAbsoluteMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
+//
+//    auto abs_mtx = mtx->compute_absolute();
+//    auto dabs_mtx = dmtx->compute_absolute();
+//
+//    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(hip));
+//
+//    complex_mtx->compute_absolute_inplace();
+//    complex_dmtx->compute_absolute_inplace();
+//
+//    GKO_ASSERT_MTX_NEAR(complex_mtx, complex_dmtx, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(hip));
+//
+//    auto abs_mtx = complex_mtx->compute_absolute();
+//    auto dabs_mtx = complex_dmtx->compute_absolute();
+//
+//    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+//}
+
+
+}  // namespace
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
new file mode 100644
index 00000000000..ac70964dfbc
--- /dev/null
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -0,0 +1,569 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_MATRIX_FBCSR_HPP_
+#define GKO_CORE_MATRIX_FBCSR_HPP_
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/math.hpp>
+
+#include "matrix_strategies.hpp"
+
+
+namespace gko {
+namespace matrix {
+
+
+template <typename ValueType>
+class Dense;
+
+template <typename ValueType, typename IndexType>
+class Coo;
+
+template <typename ValueType, typename IndexType>
+class Ell;
+
+template <typename ValueType, typename IndexType>
+class Hybrid;
+
+template <typename ValueType, typename IndexType>
+class Sellp;
+
+template <typename ValueType, typename IndexType>
+class SparsityCsr;
+
+template <typename ValueType, typename IndexType>
+class Fbcsr;
+
+template <typename ValueType, typename IndexType>
+class FbcsrBuilder;
+
+
+/**
+ * FBCSR is a matrix format which stores only the nonzero coefficients by
+ * compressing each row of the matrix (compressed sparse row format).
+ * However, unlike Csr, each non-zero location stores a small dense block of
+ * entries having a constant size. This reduces the number of integers that need
+ * to be stored in order to refer to a given non-zero entry, and enables
+ * efficient implementation of certain block methods.
+ *
+ * The entries within each dense block are stored row-major.
+ *
+ * @note The total number of rows and the number of columns are expected to be
+ *   divisible by the block size.
+ *
+ * The nonzero elements are stored in a 1D array row-wise, and accompanied
+ * with a row pointer array which stores the starting index of each block-row.
+ * An additional block-column index array is used to identify the block-column
+ * of each nonzero block.
+ *
+ * The Fbcsr LinOp supports different operations:
+ *
+ * ```cpp
+ * matrix::Fbcsr *A, *B, *C;      // matrices
+ * matrix::Dense *b, *x;        // vectors tall-and-skinny matrices
+ * matrix::Dense *alpha, *beta; // scalars of dimension 1x1
+ * matrix::Identity *I;         // identity matrix
+ *
+ * // Applying to Dense matrices computes an SpMV/SpMM product
+ * A->apply(b, x)              // x = A*b
+ * A->apply(alpha, b, beta, x) // x = alpha*A*b + beta*x
+ *
+ * // Applying to Fbcsr matrices computes a SpGEMM product of two sparse
+ * matrices A->apply(B, C)              // C = A*B A->apply(alpha, B, beta, C)
+ * // C = alpha*A*B + beta*C
+ *
+ * // Applying to an Identity matrix computes a SpGEAM sparse matrix addition
+ * A->apply(alpha, I, beta, B) // B = alpha*A + beta*B
+ * ```
+ * Both the SpGEMM and SpGEAM operation require the input matrices to be sorted
+ * by column index, otherwise the algorithms will produce incorrect results.
+ *
+ * @tparam ValueType  precision of matrix elements
+ * @tparam IndexType  precision of matrix indexes
+ *
+ * @ingroup fbcsr
+ * @ingroup mat_formats
+ * @ingroup LinOp
+ */
+template <typename ValueType = default_precision, typename IndexType = int32>
+class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
+              public EnableCreateMethod<Fbcsr<ValueType, IndexType>>,
+              public ConvertibleTo<Fbcsr<next_precision<ValueType>, IndexType>>,
+              public ConvertibleTo<Dense<ValueType>>,
+              public ConvertibleTo<Coo<ValueType, IndexType>>,
+              // public ConvertibleTo<Ell<ValueType, IndexType>>,
+              // public ConvertibleTo<Hybrid<ValueType, IndexType>>,
+              // public ConvertibleTo<Sellp<ValueType, IndexType>>,
+              public ConvertibleTo<SparsityCsr<ValueType, IndexType>>,
+              public DiagonalExtractable<ValueType>,
+              public ReadableFromMatrixData<ValueType, IndexType>,
+              public WritableToMatrixData<ValueType, IndexType>,
+              public Transposable,
+              public Permutable<IndexType>,
+              public EnableAbsoluteComputation<
+                  remove_complex<Fbcsr<ValueType, IndexType>>> {
+    friend class EnableCreateMethod<Fbcsr>;
+    friend class EnablePolymorphicObject<Fbcsr, LinOp>;
+    friend class Coo<ValueType, IndexType>;
+    friend class Dense<ValueType>;
+    // friend class Ell<ValueType, IndexType>;
+    // friend class Hybrid<ValueType, IndexType>;
+    // friend class Sellp<ValueType, IndexType>;
+    friend class SparsityCsr<ValueType, IndexType>;
+    friend class FbcsrBuilder<ValueType, IndexType>;
+    friend class Fbcsr<to_complex<ValueType>, IndexType>;
+
+public:
+    using value_type = ValueType;
+    using index_type = IndexType;
+    using transposed_type = Fbcsr<ValueType, IndexType>;
+    using mat_data = matrix_data<ValueType, IndexType>;
+    using absolute_type = remove_complex<Fbcsr>;
+
+    using strategy_type =
+        matrix_strategy::strategy_type<Fbcsr<value_type, index_type>>;
+
+
+    void convert_to(Fbcsr<ValueType, IndexType> *result) const override
+    {
+        bool same_executor = this->get_executor() == result->get_executor();
+        // NOTE: as soon as strategies are improved, this can be reverted
+        result->values_ = this->values_;
+        result->col_idxs_ = this->col_idxs_;
+        result->row_ptrs_ = this->row_ptrs_;
+        result->startrow_ = this->startrow_;
+        result->set_size(this->get_size());
+        result->bs_ = this->bs_;
+        if (!same_executor) {
+            convert_strategy_helper(result);
+        } else {
+            result->set_strategy(std::move(this->get_strategy()->copy()));
+        }
+        // END NOTE
+    }
+
+    void move_to(Fbcsr<ValueType, IndexType> *result) override
+    {
+        bool same_executor = this->get_executor() == result->get_executor();
+        EnableLinOp<Fbcsr>::move_to(result);
+        if (!same_executor) {
+            matrix_strategy::strategy_rebuild_helper(result);
+        }
+    }
+    friend class Fbcsr<next_precision<ValueType>, IndexType>;
+
+    void convert_to(
+        Fbcsr<next_precision<ValueType>, IndexType> *result) const override;
+
+    void move_to(Fbcsr<next_precision<ValueType>, IndexType> *result) override;
+
+    void convert_to(Dense<ValueType> *other) const override;
+
+    void move_to(Dense<ValueType> *other) override;
+
+    void convert_to(Coo<ValueType, IndexType> *result) const override;
+
+    void move_to(Coo<ValueType, IndexType> *result) override;
+
+    // void convert_to(Ell<ValueType, IndexType> *result) const override;
+
+    // void move_to(Ell<ValueType, IndexType> *result) override;
+
+    // void convert_to(Hybrid<ValueType, IndexType> *result) const override;
+
+    // void move_to(Hybrid<ValueType, IndexType> *result) override;
+
+    // void convert_to(Sellp<ValueType, IndexType> *result) const override;
+
+    // void move_to(Sellp<ValueType, IndexType> *result) override;
+
+    void convert_to(SparsityCsr<ValueType, IndexType> *result) const override;
+
+    void move_to(SparsityCsr<ValueType, IndexType> *result) override;
+
+    /// Convert COO data into block CSR
+    /** @warning Unlike Csr::read, here explicit non-zeros are NOT dropped.
+     */
+    void read(const mat_data &data) override;
+
+    void write(mat_data &data) const override;
+
+    std::unique_ptr<LinOp> transpose() const override;
+
+    std::unique_ptr<LinOp> conj_transpose() const override;
+
+    std::unique_ptr<LinOp> row_permute(
+        const Array<IndexType> *permutation_indices) const override;
+
+    std::unique_ptr<LinOp> column_permute(
+        const Array<IndexType> *permutation_indices) const override;
+
+    std::unique_ptr<LinOp> inverse_row_permute(
+        const Array<IndexType> *inverse_permutation_indices) const override;
+
+    std::unique_ptr<LinOp> inverse_column_permute(
+        const Array<IndexType> *inverse_permutation_indices) const override;
+
+    std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
+
+    std::unique_ptr<absolute_type> compute_absolute() const override;
+
+    void compute_absolute_inplace() override;
+
+    /**
+     * Sorts all (value, col_idx) pairs in each row by column index
+     */
+    void sort_by_column_index();
+
+    /*
+     * Tests if all row entry pairs (value, col_idx) are sorted by column index
+     *
+     * @returns True if all row entry pairs (value, col_idx) are sorted by
+     *          column index
+     */
+    bool is_sorted_by_column_index() const;
+
+    /**
+     * Returns the values of the matrix.
+     *
+     * @return the values of the matrix.
+     */
+    value_type *get_values() noexcept { return values_.get_data(); }
+
+    /// @see Fbcsr::get_const_values()
+    const value_type *get_values() const noexcept
+    {
+        return values_.get_const_data();
+    }
+
+    /**
+     * @copydoc Fbcsr::get_values()
+     *
+     * @note This is the constant version of the function, which can be
+     *       significantly more memory efficient than the non-constant version,
+     *       so always prefer this version.
+     */
+    const value_type *get_const_values() const noexcept
+    {
+        return values_.get_const_data();
+    }
+
+    /**
+     * Returns the column indexes of the matrix.
+     *
+     * @return the column indexes of the matrix.
+     */
+    index_type *get_col_idxs() noexcept { return col_idxs_.get_data(); }
+
+    /// @see Fbcsr::get_const_col_idxs()
+    const index_type *get_col_idxs() const noexcept
+    {
+        return col_idxs_.get_const_data();
+    }
+
+    /**
+     * @copydoc Fbcsr::get_col_idxs()
+     *
+     * @note This is the constant version of the function, which can be
+     *       significantly more memory efficient than the non-constant version,
+     *       so always prefer this version.
+     */
+    const index_type *get_const_col_idxs() const noexcept
+    {
+        return col_idxs_.get_const_data();
+    }
+
+    /**
+     * Returns the row pointers of the matrix.
+     *
+     * @return the row pointers of the matrix.
+     */
+    index_type *get_row_ptrs() noexcept { return row_ptrs_.get_data(); }
+
+    /// @see Fbcsr::get_const_row_ptrs()
+    const index_type *get_row_ptrs() const noexcept
+    {
+        return row_ptrs_.get_const_data();
+    }
+
+    /**
+     * @copydoc Fbcsr::get_row_ptrs()
+     *
+     * @note This is the constant version of the function, which can be
+     *       significantly more memory efficient than the non-constant version,
+     *       so always prefer this version.
+     */
+    const index_type *get_const_row_ptrs() const noexcept
+    {
+        return row_ptrs_.get_const_data();
+    }
+
+    /**
+     * Returns the starting rows.
+     *
+     * @return the starting rows.
+     */
+    index_type *get_srow() noexcept { return startrow_.get_data(); }
+
+    const index_type *get_srow() const noexcept
+    {
+        return startrow_.get_const_data();
+    }
+
+    /**
+     * @copydoc Fbcsr::get_srow()
+     *
+     * @note This is the constant version of the function, which can be
+     *       significantly more memory efficient than the non-constant version,
+     *       so always prefer this version.
+     */
+    const index_type *get_const_srow() const noexcept
+    {
+        return startrow_.get_const_data();
+    }
+
+    /**
+     * Returns the number of the srow stored elements (involved warps)
+     *
+     * @return the number of the srow stored elements (involved warps)
+     */
+    size_type get_num_srow_elements() const noexcept
+    {
+        return startrow_.get_num_elems();
+    }
+
+    /**
+     * Returns the number of elements explicitly stored in the matrix.
+     *
+     * @return the number of elements explicitly stored in the matrix
+     */
+    size_type get_num_stored_elements() const noexcept
+    {
+        return values_.get_num_elems();
+    }
+
+    /** Returns the strategy
+     *
+     * @return the strategy
+     */
+    std::shared_ptr<strategy_type> get_strategy() const noexcept
+    {
+        return strategy_;
+    }
+
+    /**
+     * Set the strategy
+     *
+     * @param strategy the fbcsr strategy
+     */
+    void set_strategy(std::shared_ptr<strategy_type> strategy)
+    {
+        strategy_ = std::move(strategy->copy());
+        this->make_srow();
+    }
+
+    int get_block_size() const { return bs_; }
+
+    void set_block_size(const int block_size) { bs_ = block_size; }
+
+protected:
+    using classical = matrix_strategy::classical<Fbcsr<value_type, index_type>>;
+
+    /**
+     * Creates an uninitialized FBCSR matrix with a block size of 1.
+     *
+     * @param exec  Executor associated to the matrix
+     * @param strategy  the strategy of FBCSR
+     */
+    Fbcsr(std::shared_ptr<const Executor> exec,
+          std::shared_ptr<strategy_type> strategy)
+        : Fbcsr(std::move(exec), dim<2>{}, {}, 1, std::move(strategy))
+    {}
+
+    /**
+     * Creates an uninitialized FBCSR matrix of the specified size.
+     *
+     * @param exec  Executor associated to the matrix
+     * @param size  size of the matrix
+     * @param num_nonzeros  number of nonzeros
+     * @param block_size Size of the small dense square blocks
+     * @param strategy  the strategy of FBCSR
+     */
+    Fbcsr(std::shared_ptr<const Executor> exec, const dim<2> &size = dim<2>{},
+          size_type num_nonzeros = {}, int block_size = 1,
+          std::shared_ptr<strategy_type> strategy =
+              std::make_shared<classical>());
+
+    /**
+     * Creates a FBCSR matrix from already allocated (and initialized) row
+     * pointer, column index and value arrays.
+     *
+     * @tparam ValuesArray  type of `values` array
+     * @tparam ColIdxsArray  type of `col_idxs` array
+     * @tparam RowPtrsArray  type of `row_ptrs` array
+     *
+     * @param exec  Executor associated to the matrix
+     * @param size  size of the matrix
+     * @param block_size
+     * @param values  array of matrix values
+     * @param col_idxs  array of column indexes
+     * @param row_ptrs  array of row pointers
+     *
+     * @note If one of `row_ptrs`, `col_idxs` or `values` is not an rvalue, not
+     *       an array of IndexType, IndexType and ValueType, respectively, or
+     *       is on the wrong executor, an internal copy of that array will be
+     *       created, and the original array data will not be used in the
+     *       matrix.
+     */
+    template <typename ValuesArray, typename ColIdxsArray,
+              typename RowPtrsArray>
+    Fbcsr(
+        std::shared_ptr<const Executor> exec, const dim<2> &size,
+        int block_size, ValuesArray &&values, ColIdxsArray &&col_idxs,
+        RowPtrsArray &&row_ptrs,
+        std::shared_ptr<strategy_type> strategy = std::make_shared<classical>())
+        : EnableLinOp<Fbcsr>(exec, size),
+          bs_{block_size},
+          values_{exec, std::forward<ValuesArray>(values)},
+          col_idxs_{exec, std::forward<ColIdxsArray>(col_idxs)},
+          row_ptrs_{exec, std::forward<RowPtrsArray>(row_ptrs)},
+          startrow_(exec),
+          strategy_(strategy->copy())
+    {
+        GKO_ASSERT_EQ(values_.get_num_elems(),
+                      col_idxs_.get_num_elems() * bs_ * bs_);
+        GKO_ASSERT_EQ(this->get_size()[0] / bs_ + 1, row_ptrs_.get_num_elems());
+        this->make_srow();
+    }
+
+    void apply_impl(const LinOp *b, LinOp *x) const override;
+
+    void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
+                    LinOp *x) const override;
+
+    // TODO clean this up as soon as we improve strategy_type
+    template <typename FbcsrType>
+    void convert_strategy_helper(FbcsrType *result) const
+    {
+        auto strat = this->get_strategy().get();
+        std::shared_ptr<typename matrix_strategy::strategy_type<FbcsrType>>
+            new_strat;
+        using classical = matrix_strategy::classical<FbcsrType>;
+        using load_balance = matrix_strategy::load_balance<FbcsrType>;
+        using automatic = matrix_strategy::automatic<FbcsrType>;
+
+        if (dynamic_cast<classical *>(strat)) {
+            new_strat = std::make_shared<classical>();
+        } else {
+            auto rexec = result->get_executor();
+            auto cuda_exec =
+                std::dynamic_pointer_cast<const CudaExecutor>(rexec);
+            auto hip_exec = std::dynamic_pointer_cast<const HipExecutor>(rexec);
+            auto lb = dynamic_cast<load_balance *>(strat);
+            if (cuda_exec) {
+                if (lb) {
+                    new_strat = std::make_shared<load_balance>(cuda_exec);
+                } else {
+                    new_strat = std::make_shared<automatic>(cuda_exec);
+                }
+            } else if (hip_exec) {
+                if (lb) {
+                    new_strat = std::make_shared<load_balance>(hip_exec);
+                } else {
+                    new_strat = std::make_shared<automatic>(hip_exec);
+                }
+            } else {
+                // Try to preserve this executor's configuration
+                auto this_cuda_exec =
+                    std::dynamic_pointer_cast<const CudaExecutor>(
+                        this->get_executor());
+                auto this_hip_exec =
+                    std::dynamic_pointer_cast<const HipExecutor>(
+                        this->get_executor());
+                if (this_cuda_exec) {
+                    if (lb) {
+                        new_strat =
+                            std::make_shared<load_balance>(this_cuda_exec);
+                    } else {
+                        new_strat = std::make_shared<automatic>(this_cuda_exec);
+                    }
+                } else if (this_hip_exec) {
+                    if (lb) {
+                        new_strat =
+                            std::make_shared<load_balance>(this_hip_exec);
+                    } else {
+                        new_strat = std::make_shared<automatic>(this_hip_exec);
+                    }
+                } else {
+                    // We had a load balance or automatic strategy from a non
+                    // HIP or Cuda executor and are moving to a non HIP or Cuda
+                    // executor.
+                    // FIXME this creates a long delay
+                    if (lb) {
+                        new_strat = std::make_shared<load_balance>();
+                    } else {
+                        new_strat = std::make_shared<automatic>();
+                    }
+                }
+            }
+        }
+        result->set_strategy(new_strat);
+    }
+
+    /**
+     * Computes srow. It should be run after changing any row_ptrs_ value.
+     */
+    void make_srow()
+    {
+        startrow_.resize_and_reset(
+            strategy_->calc_size(values_.get_num_elems() / bs_ / bs_));
+        strategy_->process(row_ptrs_, &startrow_);
+    }
+
+private:
+    int bs_;  ///< Block size
+    Array<value_type> values_;
+    Array<index_type> col_idxs_;
+    Array<index_type> row_ptrs_;
+    Array<index_type> startrow_;
+    std::shared_ptr<strategy_type> strategy_;
+};
+
+
+}  // namespace matrix
+}  // namespace gko
+
+
+#endif  // GKO_CORE_MATRIX_FBCSR_HPP_
diff --git a/include/ginkgo/core/matrix/matrix_strategies.hpp b/include/ginkgo/core/matrix/matrix_strategies.hpp
new file mode 100644
index 00000000000..c774209f7d7
--- /dev/null
+++ b/include/ginkgo/core/matrix/matrix_strategies.hpp
@@ -0,0 +1,503 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_MATRIX_MATRIX_STRATEGY_HPP_
+#define GKO_CORE_MATRIX_MATRIX_STRATEGY_HPP_
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/lin_op.hpp>
+
+
+namespace gko {
+namespace matrix {
+
+
+namespace matrix_strategy {
+
+
+template <typename MtxType>
+class automatic;
+
+/**
+ * strategy_type is to decide how to set the fbcsr algorithm.
+ *
+ * The practical strategy method should inherit strategy_type and implement
+ * its `process`, `calc_size` function and the corresponding device kernel.
+ */
+template <typename MtxType>
+class strategy_type {
+    friend class automatic<MtxType>;
+
+public:
+    using index_type = typename MtxType::index_type;
+
+    /**
+     * Creates a strategy_type.
+     *
+     * @param name  the name of strategy
+     */
+    strategy_type(std::string name) : name_(name) {}
+
+    /**
+     * Returns the name of strategy
+     *
+     * @return the name of strategy
+     */
+    std::string get_name() { return name_; }
+
+    /**
+     * Computes srow according to row pointers.
+     *
+     * @param mtx_row_ptrs  the row pointers of the matrix
+     * @param mtx_srow  the srow of the matrix
+     */
+    virtual void process(const Array<index_type> &mtx_row_ptrs,
+                         Array<index_type> *mtx_srow) = 0;
+
+    /**
+     * Computes the srow size according to the number of nonzeros.
+     *
+     * @param nnz  the number of nonzeros
+     *
+     * @return the size of srow
+     */
+    virtual int64_t calc_size(const int64_t nnz) = 0;
+
+    /**
+     * Copy a strategy. This is a workaround until strategies are revamped,
+     * since strategies like `automatic` do not work when actually shared.
+     */
+    virtual std::shared_ptr<strategy_type> copy() = 0;
+
+protected:
+    void set_name(std::string name) { name_ = name; }
+
+private:
+    std::string name_;
+};
+
+/**
+ * classical is a strategy_type which uses the same number of threads on
+ * each block-row. Classical strategy uses multithreads to calculate on parts of
+ * rows and then do a reduction of these threads results. The number of
+ * threads per row depends on the max number of stored elements per row.
+ */
+template <typename MtxType>
+class classical : public strategy_type<MtxType> {
+public:
+    using index_type = typename strategy_type<MtxType>::index_type;
+
+    /**
+     * Creates a classical strategy.
+     */
+    classical() : strategy_type<MtxType>("classical"), max_length_per_row_(0) {}
+
+    void process(const Array<index_type> &mtx_row_ptrs,
+                 Array<index_type> *mtx_srow) override
+    {
+        auto host_mtx_exec = mtx_row_ptrs.get_executor()->get_master();
+        Array<index_type> row_ptrs_host(host_mtx_exec);
+        const bool is_mtx_on_host{host_mtx_exec == mtx_row_ptrs.get_executor()};
+        const index_type *row_ptrs{};
+        if (is_mtx_on_host) {
+            row_ptrs = mtx_row_ptrs.get_const_data();
+        } else {
+            row_ptrs_host = mtx_row_ptrs;
+            row_ptrs = row_ptrs_host.get_const_data();
+        }
+        auto num_rows = mtx_row_ptrs.get_num_elems() - 1;
+        max_length_per_row_ = 0;
+        for (index_type i = 1; i < num_rows + 1; i++) {
+            max_length_per_row_ =
+                std::max(max_length_per_row_, row_ptrs[i] - row_ptrs[i - 1]);
+        }
+    }
+
+    int64_t calc_size(const int64_t nnz) override { return 0; }
+
+    index_type get_max_length_per_row() const noexcept
+    {
+        return max_length_per_row_;
+    }
+
+    std::shared_ptr<strategy_type<MtxType>> copy() override
+    {
+        return std::make_shared<classical<MtxType>>();
+    }
+
+private:
+    index_type max_length_per_row_;
+};
+
+/**
+ * load_balance is a strategy_type which uses the load balance algorithm.
+ */
+template <typename MtxType>
+class load_balance : public strategy_type<MtxType> {
+public:
+    using index_type = typename strategy_type<MtxType>::index_type;
+
+    /**
+     * Creates a load_balance strategy.
+     */
+    load_balance()
+        : load_balance(std::move(
+              gko::CudaExecutor::create(0, gko::OmpExecutor::create())))
+    {}
+
+    /**
+     * Creates a load_balance strategy with CUDA executor.
+     *
+     * @param exec the CUDA executor
+     */
+    load_balance(std::shared_ptr<const CudaExecutor> exec)
+        : load_balance(exec->get_num_warps(), exec->get_warp_size())
+    {}
+
+    /**
+     * Creates a load_balance strategy with HIP executor.
+     *
+     * @param exec the HIP executor
+     */
+    load_balance(std::shared_ptr<const HipExecutor> exec)
+        : load_balance(exec->get_num_warps(), exec->get_warp_size(), false)
+    {}
+
+    /**
+     * Creates a load_balance strategy with specified parameters
+     *
+     * @param nwarps The number of warps in the executor
+     * @param warp_size The warp size of the executor
+     * @param cuda_params Whether Nvidia-based warp parameters should be used.
+     *
+     * @note The warp_size must be the size of full warp. When using this
+     *       constructor, set_strategy needs to be called with correct
+     *       parameters which is replaced during the conversion.
+     */
+    load_balance(int64_t nwarps, int warp_size = 32, bool cuda_params = true)
+        : strategy_type<MtxType>("load_balance"),
+          nwarps_(nwarps),
+          warp_size_(warp_size),
+          cuda_params_(cuda_params)
+    {}
+
+    void process(const Array<index_type> &mtx_row_ptrs,
+                 Array<index_type> *mtx_srow) override
+    {
+        auto nwarps = mtx_srow->get_num_elems();
+
+        if (nwarps > 0) {
+            auto host_srow_exec = mtx_srow->get_executor()->get_master();
+            auto host_mtx_exec = mtx_row_ptrs.get_executor()->get_master();
+            const bool is_srow_on_host{host_srow_exec ==
+                                       mtx_srow->get_executor()};
+            const bool is_mtx_on_host{host_mtx_exec ==
+                                      mtx_row_ptrs.get_executor()};
+            Array<index_type> row_ptrs_host(host_mtx_exec);
+            Array<index_type> srow_host(host_srow_exec);
+            const index_type *row_ptrs{};
+            index_type *srow{};
+            if (is_srow_on_host) {
+                srow = mtx_srow->get_data();
+            } else {
+                srow_host = *mtx_srow;
+                srow = srow_host.get_data();
+            }
+            if (is_mtx_on_host) {
+                row_ptrs = mtx_row_ptrs.get_const_data();
+            } else {
+                row_ptrs_host = mtx_row_ptrs;
+                row_ptrs = row_ptrs_host.get_const_data();
+            }
+            for (size_type i = 0; i < nwarps; i++) {
+                srow[i] = 0;
+            }
+            const size_type num_rows = mtx_row_ptrs.get_num_elems() - 1;
+            const index_type num_elems = row_ptrs[num_rows];
+            for (size_type i = 0; i < num_rows; i++) {
+                const auto num =
+                    (ceildiv(row_ptrs[i + 1], warp_size_) * nwarps);
+                const auto den = ceildiv(num_elems, warp_size_);
+                auto bucket = ceildiv(num, den);
+                if (bucket < nwarps) {
+                    srow[bucket]++;
+                }
+            }
+            // find starting row for thread i
+            for (size_type i = 1; i < nwarps; i++) {
+                srow[i] += srow[i - 1];
+            }
+            if (!is_srow_on_host) {
+                *mtx_srow = srow_host;
+            }
+        }
+    }
+
+    int64_t calc_size(const int64_t nnz) override
+    {
+        if (warp_size_ > 0) {
+            int multiple = 8;
+            if (nnz >= 2e8) {
+                multiple = 2048;
+            } else if (nnz >= 2e7) {
+                multiple = 512;
+            } else if (nnz >= 2e6) {
+                multiple = 128;
+            } else if (nnz >= 2e5) {
+                multiple = 32;
+            }
+
+#if GINKGO_HIP_PLATFORM_HCC
+            if (!cuda_params_) {
+                multiple = 8;
+                if (nnz >= 1e7) {
+                    multiple = 64;
+                } else if (nnz >= 1e6) {
+                    multiple = 16;
+                }
+            }
+#endif  // GINKGO_HIP_PLATFORM_HCC
+
+            auto nwarps = nwarps_ * multiple;
+            return min(ceildiv(nnz, warp_size_), int64_t(nwarps));
+        } else {
+            return 0;
+        }
+    }
+
+    std::shared_ptr<strategy_type<MtxType>> copy() override
+    {
+        return std::make_shared<load_balance<MtxType>>(nwarps_, warp_size_,
+                                                       cuda_params_);
+    }
+
+private:
+    int64_t nwarps_;
+    int warp_size_;
+    bool cuda_params_;
+};
+
+template <typename MtxType>
+class automatic : public strategy_type<MtxType> {
+public:
+    using index_type = typename strategy_type<MtxType>::index_type;
+
+    /* Use imbalance strategy when the maximum number of nonzero per row is
+     * more than 1024 on NVIDIA hardware */
+    const index_type nvidia_row_len_limit = 1024;
+    /* Use imbalance strategy when the matrix has more more than 1e6 on
+     * NVIDIA hardware */
+    const index_type nvidia_nnz_limit = 1e6;
+    /* Use imbalance strategy when the maximum number of nonzero per row is
+     * more than 768 on AMD hardware */
+    const index_type amd_row_len_limit = 768;
+    /* Use imbalance strategy when the matrix has more more than 1e8 on AMD
+     * hardware */
+    const index_type amd_nnz_limit = 1e8;
+
+    /**
+     * Creates an automatic strategy.
+     */
+    automatic()
+        : automatic(std::move(
+              gko::CudaExecutor::create(0, gko::OmpExecutor::create())))
+    {}
+
+    /**
+     * Creates an automatic strategy with CUDA executor.
+     *
+     * @param exec the CUDA executor
+     */
+    automatic(std::shared_ptr<const CudaExecutor> exec)
+        : automatic(exec->get_num_warps(), exec->get_warp_size())
+    {}
+
+    /**
+     * Creates an automatic strategy with HIP executor.
+     *
+     * @param exec the HIP executor
+     */
+    automatic(std::shared_ptr<const HipExecutor> exec)
+        : automatic(exec->get_num_warps(), exec->get_warp_size(), false)
+    {}
+
+    /**
+     * Creates an automatic strategy with specified parameters
+     *
+     * @param nwarps the number of warps in the executor
+     * @param warp_size the warp size of the executor
+     * @param cuda_strategy  whether the `cuda_strategy` needs to be used.
+     *
+     * @note The warp_size must be the size of full warp. When using this
+     *       constructor, set_strategy needs to be called with correct
+     *       parameters which is replaced during the conversion.
+     */
+    automatic(int64_t nwarps, int warp_size = 32, bool cuda_strategy = true)
+        : strategy_type<MtxType>("automatic"),
+          nwarps_(nwarps),
+          warp_size_(warp_size),
+          cuda_strategy_(cuda_strategy),
+          max_length_per_row_(0)
+    {}
+
+    void process(const Array<index_type> &mtx_row_ptrs,
+                 Array<index_type> *mtx_srow) override
+    {
+        // if the number of stored elements is larger than <nnz_limit> or
+        // the maximum number of stored elements per row is larger than
+        // <row_len_limit>, use load_balance otherwise use classical
+        index_type nnz_limit = nvidia_nnz_limit;
+        index_type row_len_limit = nvidia_row_len_limit;
+#if GINKGO_HIP_PLATFORM_HCC
+        if (!cuda_strategy_) {
+            nnz_limit = amd_nnz_limit;
+            row_len_limit = amd_row_len_limit;
+        }
+#endif  // GINKGO_HIP_PLATFORM_HCC
+        auto host_mtx_exec = mtx_row_ptrs.get_executor()->get_master();
+        const bool is_mtx_on_host{host_mtx_exec == mtx_row_ptrs.get_executor()};
+        Array<index_type> row_ptrs_host(host_mtx_exec);
+        const index_type *row_ptrs{};
+        if (is_mtx_on_host) {
+            row_ptrs = mtx_row_ptrs.get_const_data();
+        } else {
+            row_ptrs_host = mtx_row_ptrs;
+            row_ptrs = row_ptrs_host.get_const_data();
+        }
+        const auto num_rows = mtx_row_ptrs.get_num_elems() - 1;
+        if (row_ptrs[num_rows] > nnz_limit) {
+            load_balance<MtxType> actual_strategy(nwarps_, warp_size_,
+                                                  cuda_strategy_);
+            if (is_mtx_on_host) {
+                actual_strategy.process(mtx_row_ptrs, mtx_srow);
+            } else {
+                actual_strategy.process(row_ptrs_host, mtx_srow);
+            }
+            this->set_name(actual_strategy.get_name());
+        } else {
+            index_type maxnum = 0;
+            for (index_type i = 1; i < num_rows + 1; i++) {
+                maxnum = std::max(maxnum, row_ptrs[i] - row_ptrs[i - 1]);
+            }
+            if (maxnum > row_len_limit) {
+                load_balance<MtxType> actual_strategy(nwarps_, warp_size_,
+                                                      cuda_strategy_);
+                if (is_mtx_on_host) {
+                    actual_strategy.process(mtx_row_ptrs, mtx_srow);
+                } else {
+                    actual_strategy.process(row_ptrs_host, mtx_srow);
+                }
+                this->set_name(actual_strategy.get_name());
+            } else {
+                classical<MtxType> actual_strategy;
+                if (is_mtx_on_host) {
+                    actual_strategy.process(mtx_row_ptrs, mtx_srow);
+                    max_length_per_row_ =
+                        actual_strategy.get_max_length_per_row();
+                } else {
+                    actual_strategy.process(row_ptrs_host, mtx_srow);
+                    max_length_per_row_ =
+                        actual_strategy.get_max_length_per_row();
+                }
+                this->set_name(actual_strategy.get_name());
+            }
+        }
+    }
+
+    int64_t calc_size(const int64_t nnz) override
+    {
+        return std::make_shared<load_balance<MtxType>>(nwarps_, warp_size_,
+                                                       cuda_strategy_)
+            ->calc_size(nnz);
+    }
+
+    index_type get_max_length_per_row() const noexcept
+    {
+        return max_length_per_row_;
+    }
+
+    std::shared_ptr<strategy_type<MtxType>> copy() override
+    {
+        return std::make_shared<automatic<MtxType>>(nwarps_, warp_size_,
+                                                    cuda_strategy_);
+    }
+
+private:
+    int64_t nwarps_;
+    int warp_size_;
+    bool cuda_strategy_;
+    index_type max_length_per_row_;
+};
+
+
+/**
+ * When strategy is load_balance or automatic, rebuild the strategy
+ * according to executor's property.
+ *
+ * @param result  the matrix.
+ */
+template <typename MtxType>
+void strategy_rebuild_helper(MtxType *const result)
+{
+    // TODO (script:fbcsr): change the code imported from matrix/csr if needed
+    // using load_balance = typename Fbcsr<ValueType, IndexType>::load_balance;
+    // using automatic = typename Fbcsr<ValueType, IndexType>::automatic;
+    auto strategy = result->get_strategy();
+    auto executor = result->get_executor();
+    if (std::dynamic_pointer_cast<load_balance<MtxType>>(strategy)) {
+        if (auto exec =
+                std::dynamic_pointer_cast<const HipExecutor>(executor)) {
+            result->set_strategy(std::make_shared<load_balance<MtxType>>(exec));
+        } else if (auto exec = std::dynamic_pointer_cast<const CudaExecutor>(
+                       executor)) {
+            result->set_strategy(std::make_shared<load_balance<MtxType>>(exec));
+        }
+    } else if (std::dynamic_pointer_cast<automatic<MtxType>>(strategy)) {
+        if (auto exec =
+                std::dynamic_pointer_cast<const HipExecutor>(executor)) {
+            result->set_strategy(std::make_shared<automatic<MtxType>>(exec));
+        } else if (auto exec = std::dynamic_pointer_cast<const CudaExecutor>(
+                       executor)) {
+            result->set_strategy(std::make_shared<automatic<MtxType>>(exec));
+        }
+    }
+}
+
+
+}  // namespace matrix_strategy
+
+
+}  // namespace matrix
+}  // namespace gko
+
+#endif
diff --git a/include/ginkgo/core/matrix/sparsity_csr.hpp b/include/ginkgo/core/matrix/sparsity_csr.hpp
index 087c0538b7d..b5f16f5ae3e 100644
--- a/include/ginkgo/core/matrix/sparsity_csr.hpp
+++ b/include/ginkgo/core/matrix/sparsity_csr.hpp
@@ -48,6 +48,9 @@ namespace matrix {
 template <typename ValueType, typename IndexType>
 class Csr;
 
+template <typename ValueType, typename IndexType>
+class Fbcsr;
+
 
 /**
  * SparsityCsr is a matrix format which stores only the sparsity pattern of a
@@ -77,6 +80,7 @@ class SparsityCsr
     friend class EnableCreateMethod<SparsityCsr>;
     friend class EnablePolymorphicObject<SparsityCsr, LinOp>;
     friend class Csr<ValueType, IndexType>;
+    friend class Fbcsr<ValueType, IndexType>;
 
 public:
     using EnableLinOp<SparsityCsr>::convert_to;
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 3ae5bcfdd96..30408b8970b 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -81,8 +81,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/diagonal.hpp>
 #include <ginkgo/core/matrix/ell.hpp>
+#include <ginkgo/core/matrix/fbcsr.hpp>
 #include <ginkgo/core/matrix/hybrid.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
+#include <ginkgo/core/matrix/matrix_strategies.hpp>
 #include <ginkgo/core/matrix/permutation.hpp>
 #include <ginkgo/core/matrix/sellp.hpp>
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index 80c6262c30c..acfe33d3ef6 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -20,6 +20,7 @@ target_sources(ginkgo_omp
     matrix/dense_kernels.cpp
     matrix/diagonal_kernels.cpp
     matrix/ell_kernels.cpp
+    matrix/fbcsr_kernels.cpp
     matrix/hybrid_kernels.cpp
     matrix/sellp_kernels.cpp
     matrix/sparsity_csr_kernels.cpp
diff --git a/omp/components/fbcsr_spgeam.hpp b/omp/components/fbcsr_spgeam.hpp
new file mode 100644
index 00000000000..e4a06532ed3
--- /dev/null
+++ b/omp/components/fbcsr_spgeam.hpp
@@ -0,0 +1,31 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
diff --git a/omp/matrix/fbcsr_kernels.cpp b/omp/matrix/fbcsr_kernels.cpp
new file mode 100644
index 00000000000..3ad97e0e6bc
--- /dev/null
+++ b/omp/matrix/fbcsr_kernels.cpp
@@ -0,0 +1,884 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/fbcsr_kernels.hpp"
+
+
+#include <algorithm>
+#include <numeric>
+#include <utility>
+
+
+#include <omp.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/hybrid.hpp>
+
+
+#include "core/base/allocator.hpp"
+#include "core/base/iterator_factory.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/fbcsr_builder.hpp"
+#include "omp/components/fbcsr_spgeam.hpp"
+#include "omp/components/format_conversion.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+/**
+ * @brief The Compressed sparse row matrix format namespace.
+ *
+ * @ingroup fbcsr
+ */
+namespace fbcsr {
+
+
+template <typename ValueType, typename IndexType>
+void spmv(std::shared_ptr<const OmpExecutor> exec,
+          const matrix::Fbcsr<ValueType, IndexType> *a,
+          const matrix::Dense<ValueType> *b,
+          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto row_ptrs = a->get_const_row_ptrs();
+//    auto col_idxs = a->get_const_col_idxs();
+//    auto vals = a->get_const_values();
+//
+//#pragma omp parallel for
+//    for (size_type row = 0; row < a->get_size()[0]; ++row) {
+//        for (size_type j = 0; j < c->get_size()[1]; ++j) {
+//            c->at(row, j) = zero<ValueType>();
+//        }
+//        for (size_type k = row_ptrs[row];
+//             k < static_cast<size_type>(row_ptrs[row + 1]); ++k) {
+//            auto val = vals[k];
+//            auto col = col_idxs[k];
+//            for (size_type j = 0; j < c->get_size()[1]; ++j) {
+//                c->at(row, j) += val * b->at(col, j);
+//            }
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spmv(std::shared_ptr<const OmpExecutor> exec,
+                   const matrix::Dense<ValueType> *alpha,
+                   const matrix::Fbcsr<ValueType, IndexType> *a,
+                   const matrix::Dense<ValueType> *b,
+                   const matrix::Dense<ValueType> *beta,
+                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto row_ptrs = a->get_const_row_ptrs();
+//    auto col_idxs = a->get_const_col_idxs();
+//    auto vals = a->get_const_values();
+//    auto valpha = alpha->at(0, 0);
+//    auto vbeta = beta->at(0, 0);
+//
+//#pragma omp parallel for
+//    for (size_type row = 0; row < a->get_size()[0]; ++row) {
+//        for (size_type j = 0; j < c->get_size()[1]; ++j) {
+//            c->at(row, j) *= vbeta;
+//        }
+//        for (size_type k = row_ptrs[row];
+//             k < static_cast<size_type>(row_ptrs[row + 1]); ++k) {
+//            auto val = vals[k];
+//            auto col = col_idxs[k];
+//            for (size_type j = 0; j < c->get_size()[1]; ++j) {
+//                c->at(row, j) += valpha * val * b->at(col, j);
+//            }
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_insert_row(unordered_set<IndexType> &cols,
+                       const matrix::Fbcsr<ValueType, IndexType> *c,
+                       size_type row) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto row_ptrs = c->get_const_row_ptrs();
+//    auto col_idxs = c->get_const_col_idxs();
+//    cols.insert(col_idxs + row_ptrs[row], col_idxs + row_ptrs[row + 1]);
+//}
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_insert_row2(unordered_set<IndexType> &cols,
+                        const matrix::Fbcsr<ValueType, IndexType> *a,
+                        const matrix::Fbcsr<ValueType, IndexType> *b,
+                        size_type row) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto a_row_ptrs = a->get_const_row_ptrs();
+//    auto a_col_idxs = a->get_const_col_idxs();
+//    auto b_row_ptrs = b->get_const_row_ptrs();
+//    auto b_col_idxs = b->get_const_col_idxs();
+//    for (size_type a_nz = a_row_ptrs[row];
+//         a_nz < size_type(a_row_ptrs[row + 1]); ++a_nz) {
+//        auto a_col = a_col_idxs[a_nz];
+//        auto b_row = a_col;
+//        cols.insert(b_col_idxs + b_row_ptrs[b_row],
+//                    b_col_idxs + b_row_ptrs[b_row + 1]);
+//    }
+//}
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_accumulate_row(map<IndexType, ValueType> &cols,
+                           const matrix::Fbcsr<ValueType, IndexType> *c,
+                           ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto row_ptrs = c->get_const_row_ptrs();
+//    auto col_idxs = c->get_const_col_idxs();
+//    auto vals = c->get_const_values();
+//    for (size_type c_nz = row_ptrs[row]; c_nz < size_type(row_ptrs[row + 1]);
+//         ++c_nz) {
+//        auto c_col = col_idxs[c_nz];
+//        auto c_val = vals[c_nz];
+//        cols[c_col] += scale * c_val;
+//    }
+//}
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_accumulate_row2(map<IndexType, ValueType> &cols,
+                            const matrix::Fbcsr<ValueType, IndexType> *a,
+                            const matrix::Fbcsr<ValueType, IndexType> *b,
+                            ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto a_row_ptrs = a->get_const_row_ptrs();
+//    auto a_col_idxs = a->get_const_col_idxs();
+//    auto a_vals = a->get_const_values();
+//    auto b_row_ptrs = b->get_const_row_ptrs();
+//    auto b_col_idxs = b->get_const_col_idxs();
+//    auto b_vals = b->get_const_values();
+//    for (size_type a_nz = a_row_ptrs[row];
+//         a_nz < size_type(a_row_ptrs[row + 1]); ++a_nz) {
+//        auto a_col = a_col_idxs[a_nz];
+//        auto a_val = a_vals[a_nz];
+//        auto b_row = a_col;
+//        for (size_type b_nz = b_row_ptrs[b_row];
+//             b_nz < size_type(b_row_ptrs[b_row + 1]); ++b_nz) {
+//            auto b_col = b_col_idxs[b_nz];
+//            auto b_val = b_vals[b_nz];
+//            cols[b_col] += scale * a_val * b_val;
+//        }
+//    }
+//}
+
+
+template <typename ValueType, typename IndexType>
+void spgemm(std::shared_ptr<const OmpExecutor> exec,
+            const matrix::Fbcsr<ValueType, IndexType> *a,
+            const matrix::Fbcsr<ValueType, IndexType> *b,
+            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = a->get_size()[0];
+//
+//    // first sweep: count nnz for each row
+//    auto c_row_ptrs = c->get_row_ptrs();
+//
+//    unordered_set<IndexType> local_col_idxs(exec);
+//#pragma omp parallel for firstprivate(local_col_idxs)
+//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
+//        local_col_idxs.clear();
+//        spgemm_insert_row2(local_col_idxs, a, b, a_row);
+//        c_row_ptrs[a_row] = local_col_idxs.size();
+//    }
+//
+//    // build row pointers
+//    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
+//
+//    // second sweep: accumulate non-zeros
+//    auto new_nnz = c_row_ptrs[num_rows];
+//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
+//    auto &c_col_idxs_array = c_builder.get_col_idx_array();
+//    auto &c_vals_array = c_builder.get_value_array();
+//    c_col_idxs_array.resize_and_reset(new_nnz);
+//    c_vals_array.resize_and_reset(new_nnz);
+//    auto c_col_idxs = c_col_idxs_array.get_data();
+//    auto c_vals = c_vals_array.get_data();
+//
+//    map<IndexType, ValueType> local_row_nzs(exec);
+//#pragma omp parallel for firstprivate(local_row_nzs)
+//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
+//        local_row_nzs.clear();
+//        spgemm_accumulate_row2(local_row_nzs, a, b, one<ValueType>(), a_row);
+//        // store result
+//        auto c_nz = c_row_ptrs[a_row];
+//        for (auto pair : local_row_nzs) {
+//            c_col_idxs[c_nz] = pair.first;
+//            c_vals[c_nz] = pair.second;
+//            ++c_nz;
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEMM_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spgemm(std::shared_ptr<const OmpExecutor> exec,
+                     const matrix::Dense<ValueType> *alpha,
+                     const matrix::Fbcsr<ValueType, IndexType> *a,
+                     const matrix::Fbcsr<ValueType, IndexType> *b,
+                     const matrix::Dense<ValueType> *beta,
+                     const matrix::Fbcsr<ValueType, IndexType> *d,
+                     matrix::Fbcsr<ValueType, IndexType> *c)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = a->get_size()[0];
+//    auto valpha = alpha->at(0, 0);
+//    auto vbeta = beta->at(0, 0);
+//
+//    // first sweep: count nnz for each row
+//    auto c_row_ptrs = c->get_row_ptrs();
+//
+//    unordered_set<IndexType> local_col_idxs(exec);
+//#pragma omp parallel for firstprivate(local_col_idxs)
+//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
+//        local_col_idxs.clear();
+//        spgemm_insert_row(local_col_idxs, d, a_row);
+//        spgemm_insert_row2(local_col_idxs, a, b, a_row);
+//        c_row_ptrs[a_row] = local_col_idxs.size();
+//    }
+//
+//    // build row pointers
+//    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
+//
+//    // second sweep: accumulate non-zeros
+//    auto new_nnz = c_row_ptrs[num_rows];
+//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
+//    auto &c_col_idxs_array = c_builder.get_col_idx_array();
+//    auto &c_vals_array = c_builder.get_value_array();
+//    c_col_idxs_array.resize_and_reset(new_nnz);
+//    c_vals_array.resize_and_reset(new_nnz);
+//    auto c_col_idxs = c_col_idxs_array.get_data();
+//    auto c_vals = c_vals_array.get_data();
+//
+//    map<IndexType, ValueType> local_row_nzs(exec);
+//#pragma omp parallel for firstprivate(local_row_nzs)
+//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
+//        local_row_nzs.clear();
+//        spgemm_accumulate_row(local_row_nzs, d, vbeta, a_row);
+//        spgemm_accumulate_row2(local_row_nzs, a, b, valpha, a_row);
+//        // store result
+//        auto c_nz = c_row_ptrs[a_row];
+//        for (auto pair : local_row_nzs) {
+//            c_col_idxs[c_nz] = pair.first;
+//            c_vals[c_nz] = pair.second;
+//            ++c_nz;
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void spgeam(std::shared_ptr<const OmpExecutor> exec,
+            const matrix::Dense<ValueType> *alpha,
+            const matrix::Fbcsr<ValueType, IndexType> *a,
+            const matrix::Dense<ValueType> *beta,
+            const matrix::Fbcsr<ValueType, IndexType> *b,
+            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = a->get_size()[0];
+//    auto valpha = alpha->at(0, 0);
+//    auto vbeta = beta->at(0, 0);
+//
+//    // first sweep: count nnz for each row
+//    auto c_row_ptrs = c->get_row_ptrs();
+//
+//    abstract_spgeam(
+//        a, b, [](IndexType) { return IndexType{}; },
+//        [](IndexType, IndexType, ValueType, ValueType, IndexType &nnz) {
+//            ++nnz;
+//        },
+//        [&](IndexType row, IndexType nnz) { c_row_ptrs[row] = nnz; });
+//
+//    // build row pointers
+//    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
+//
+//    // second sweep: accumulate non-zeros
+//    auto new_nnz = c_row_ptrs[num_rows];
+//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
+//    auto &c_col_idxs_array = c_builder.get_col_idx_array();
+//    auto &c_vals_array = c_builder.get_value_array();
+//    c_col_idxs_array.resize_and_reset(new_nnz);
+//    c_vals_array.resize_and_reset(new_nnz);
+//    auto c_col_idxs = c_col_idxs_array.get_data();
+//    auto c_vals = c_vals_array.get_data();
+//
+//    abstract_spgeam(
+//        a, b, [&](IndexType row) { return c_row_ptrs[row]; },
+//        [&](IndexType, IndexType col, ValueType a_val, ValueType b_val,
+//            IndexType &nz) {
+//            c_vals[nz] = valpha * a_val + vbeta * b_val;
+//            c_col_idxs[nz] = col;
+//            ++nz;
+//        },
+//        [](IndexType, IndexType) {});
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEAM_KERNEL);
+
+
+template <typename IndexType>
+void convert_row_ptrs_to_idxs(std::shared_ptr<const OmpExecutor> exec,
+                              const IndexType *ptrs, size_type num_rows,
+                              IndexType *idxs) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    convert_ptrs_to_idxs(ptrs, num_rows, idxs);
+//}
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_coo(std::shared_ptr<const OmpExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *source,
+                    matrix::Coo<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = result->get_size()[0];
+//
+//    auto row_idxs = result->get_row_idxs();
+//    const auto source_row_ptrs = source->get_const_row_ptrs();
+//
+//    convert_row_ptrs_to_idxs(exec, source_row_ptrs, num_rows, row_idxs);
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_dense(std::shared_ptr<const OmpExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *source,
+                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = source->get_size()[0];
+//    auto num_cols = source->get_size()[1];
+//    auto row_ptrs = source->get_const_row_ptrs();
+//    auto col_idxs = source->get_const_col_idxs();
+//    auto vals = source->get_const_values();
+//
+//#pragma omp parallel for
+//    for (size_type row = 0; row < num_rows; ++row) {
+//        for (size_type col = 0; col < num_cols; ++col) {
+//            result->at(row, col) = zero<ValueType>();
+//        }
+//        for (size_type i = row_ptrs[row];
+//             i < static_cast<size_type>(row_ptrs[row + 1]); ++i) {
+//            result->at(row, col_idxs[i]) = vals[i];
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_sellp(std::shared_ptr<const OmpExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *source,
+                      matrix::Sellp<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_ell(std::shared_ptr<const OmpExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *source,
+                    matrix::Ell<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL);
+
+
+template <typename ValueType, typename IndexType, typename UnaryOperator>
+inline void convert_fbcsr_to_csc(size_type num_rows, const IndexType *row_ptrs,
+                                 const IndexType *col_idxs,
+                                 const ValueType *fbcsr_vals,
+                                 IndexType *row_idxs, IndexType *col_ptrs,
+                                 ValueType *csc_vals,
+                                 UnaryOperator op) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    for (size_type row = 0; row < num_rows; ++row) {
+//        for (auto i = row_ptrs[row]; i < row_ptrs[row + 1]; ++i) {
+//            const auto dest_idx = col_ptrs[col_idxs[i]]++;
+//            row_idxs[dest_idx] = row;
+//            csc_vals[dest_idx] = op(fbcsr_vals[i]);
+//        }
+//    }
+//}
+
+
+template <typename ValueType, typename IndexType, typename UnaryOperator>
+void transpose_and_transform(std::shared_ptr<const OmpExecutor> exec,
+                             matrix::Fbcsr<ValueType, IndexType> *trans,
+                             const matrix::Fbcsr<ValueType, IndexType> *orig,
+                             UnaryOperator op) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto trans_row_ptrs = trans->get_row_ptrs();
+//    auto orig_row_ptrs = orig->get_const_row_ptrs();
+//    auto trans_col_idxs = trans->get_col_idxs();
+//    auto orig_col_idxs = orig->get_const_col_idxs();
+//    auto trans_vals = trans->get_values();
+//    auto orig_vals = orig->get_const_values();
+//
+//    auto orig_num_cols = orig->get_size()[1];
+//    auto orig_num_rows = orig->get_size()[0];
+//    auto orig_nnz = orig_row_ptrs[orig_num_rows];
+//
+//    trans_row_ptrs[0] = 0;
+//    convert_unsorted_idxs_to_ptrs(orig_col_idxs, orig_nnz, trans_row_ptrs + 1,
+//                                  orig_num_cols);
+//
+//    convert_fbcsr_to_csc(orig_num_rows, orig_row_ptrs, orig_col_idxs,
+//    orig_vals,
+//                       trans_col_idxs, trans_row_ptrs + 1, trans_vals, op);
+//}
+
+
+template <typename ValueType, typename IndexType>
+void transpose(std::shared_ptr<const OmpExecutor> exec,
+               const matrix::Fbcsr<ValueType, IndexType> *orig,
+               matrix::Fbcsr<ValueType, IndexType> *trans) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    transpose_and_transform(exec, trans, orig,
+//                            [](const ValueType x) { return x; });
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void conj_transpose(std::shared_ptr<const OmpExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *orig,
+                    matrix::Fbcsr<ValueType, IndexType> *trans)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    transpose_and_transform(exec, trans, orig,
+//                            [](const ValueType x) { return conj(x); });
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_total_cols(std::shared_ptr<const OmpExecutor> exec,
+                          const matrix::Fbcsr<ValueType, IndexType> *source,
+                          size_type *result, size_type stride_factor,
+                          size_type slice_size) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_max_nnz_per_row(
+    std::shared_ptr<const OmpExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *source,
+    size_type *result) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_hybrid(std::shared_ptr<const OmpExecutor> exec,
+                       const matrix::Fbcsr<ValueType, IndexType> *source,
+                       matrix::Hybrid<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = result->get_size()[0];
+//    auto num_cols = result->get_size()[1];
+//    auto strategy = result->get_strategy();
+//    auto ell_lim = strategy->get_ell_num_stored_elements_per_row();
+//    auto coo_lim = strategy->get_coo_nnz();
+//    auto coo_val = result->get_coo_values();
+//    auto coo_col = result->get_coo_col_idxs();
+//    auto coo_row = result->get_coo_row_idxs();
+//    const auto max_nnz_per_row =
+//    result->get_ell_num_stored_elements_per_row();
+//
+//// Initial Hybrid Matrix
+//#pragma omp parallel for
+//    for (size_type i = 0; i < max_nnz_per_row; i++) {
+//        for (size_type j = 0; j < result->get_ell_stride(); j++) {
+//            result->ell_val_at(j, i) = zero<ValueType>();
+//            result->ell_col_at(j, i) = 0;
+//        }
+//    }
+//
+//    const auto fbcsr_row_ptrs = source->get_const_row_ptrs();
+//    const auto fbcsr_vals = source->get_const_values();
+//    auto coo_offset = Array<IndexType>(exec, num_rows);
+//    auto coo_offset_val = coo_offset.get_data();
+//
+//    coo_offset_val[0] = 0;
+//#pragma omp parallel for
+//    for (size_type i = 1; i < num_rows; i++) {
+//        auto temp = fbcsr_row_ptrs[i] - fbcsr_row_ptrs[i - 1];
+//        coo_offset_val[i] = (temp > max_nnz_per_row) * (temp -
+//        max_nnz_per_row);
+//    }
+//
+//    auto workspace = Array<IndexType>(exec, num_rows);
+//    auto workspace_val = workspace.get_data();
+//    for (size_type i = 1; i < num_rows; i <<= 1) {
+//#pragma omp parallel for
+//        for (size_type j = i; j < num_rows; j++) {
+//            workspace_val[j] = coo_offset_val[j] + coo_offset_val[j - i];
+//        }
+//#pragma omp parallel for
+//        for (size_type j = i; j < num_rows; j++) {
+//            coo_offset_val[j] = workspace_val[j];
+//        }
+//    }
+//
+//#pragma omp parallel for
+//    for (IndexType row = 0; row < num_rows; row++) {
+//        size_type ell_idx = 0;
+//        size_type fbcsr_idx = fbcsr_row_ptrs[row];
+//        size_type coo_idx = coo_offset_val[row];
+//        while (fbcsr_idx < fbcsr_row_ptrs[row + 1]) {
+//            const auto val = fbcsr_vals[fbcsr_idx];
+//            if (ell_idx < ell_lim) {
+//                result->ell_val_at(row, ell_idx) = val;
+//                result->ell_col_at(row, ell_idx) =
+//                    source->get_const_col_idxs()[fbcsr_idx];
+//                ell_idx++;
+//            } else {
+//                coo_val[coo_idx] = val;
+//                coo_col[coo_idx] = source->get_const_col_idxs()[fbcsr_idx];
+//                coo_row[coo_idx] = row;
+//                coo_idx++;
+//            }
+//            fbcsr_idx++;
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void row_permute_impl(std::shared_ptr<const OmpExecutor> exec,
+                      const Array<IndexType> *permutation_indices,
+                      const matrix::Fbcsr<ValueType, IndexType> *orig,
+                      matrix::Fbcsr<ValueType, IndexType> *row_permuted)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto perm = permutation_indices->get_const_data();
+//    auto orig_row_ptrs = orig->get_const_row_ptrs();
+//    auto orig_col_idxs = orig->get_const_col_idxs();
+//    auto orig_vals = orig->get_const_values();
+//    auto rp_row_ptrs = row_permuted->get_row_ptrs();
+//    auto rp_col_idxs = row_permuted->get_col_idxs();
+//    auto rp_vals = row_permuted->get_values();
+//    size_type num_rows = orig->get_size()[0];
+//    size_type num_nnz = orig->get_num_stored_elements();
+//
+//    size_type cur_ptr = 0;
+//    rp_row_ptrs[0] = cur_ptr;
+//    vector<size_type> orig_num_nnz_per_row(num_rows, 0, exec);
+//#pragma omp parallel for
+//    for (size_type row = 0; row < num_rows; ++row) {
+//        orig_num_nnz_per_row[row] = orig_row_ptrs[row + 1] -
+//        orig_row_ptrs[row];
+//    }
+//    for (size_type row = 0; row < num_rows; ++row) {
+//        rp_row_ptrs[row + 1] =
+//            rp_row_ptrs[row] + orig_num_nnz_per_row[perm[row]];
+//    }
+//    rp_row_ptrs[num_rows] = orig_row_ptrs[num_rows];
+//#pragma omp parallel for
+//    for (size_type row = 0; row < num_rows; ++row) {
+//        auto new_row = perm[row];
+//        auto new_k = orig_row_ptrs[new_row];
+//        for (size_type k = rp_row_ptrs[row];
+//             k < size_type(rp_row_ptrs[row + 1]); ++k) {
+//            rp_col_idxs[k] = orig_col_idxs[new_k];
+//            rp_vals[k] = orig_vals[new_k];
+//            new_k++;
+//        }
+//    }
+//}
+
+
+template <typename ValueType, typename IndexType>
+void row_permute(std::shared_ptr<const OmpExecutor> exec,
+                 const Array<IndexType> *permutation_indices,
+                 const matrix::Fbcsr<ValueType, IndexType> *orig,
+                 matrix::Fbcsr<ValueType, IndexType> *row_permuted)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    row_permute_impl(exec, permutation_indices, orig, row_permuted);
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_row_permute(std::shared_ptr<const OmpExecutor> exec,
+                         const Array<IndexType> *permutation_indices,
+                         const matrix::Fbcsr<ValueType, IndexType> *orig,
+                         matrix::Fbcsr<ValueType, IndexType> *row_permuted)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto perm = permutation_indices->get_const_data();
+//    Array<IndexType> inv_perm(*permutation_indices);
+//    auto iperm = inv_perm.get_data();
+//#pragma omp parallel for
+//    for (size_type ind = 0; ind < inv_perm.get_num_elems(); ++ind) {
+//        iperm[perm[ind]] = ind;
+//    }
+//
+//    row_permute_impl(exec, &inv_perm, orig, row_permuted);
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void column_permute_impl(const Array<IndexType> *permutation_indices,
+                         const matrix::Fbcsr<ValueType, IndexType> *orig,
+                         matrix::Fbcsr<ValueType, IndexType> *column_permuted)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto perm = permutation_indices->get_const_data();
+//    auto orig_row_ptrs = orig->get_const_row_ptrs();
+//    auto orig_col_idxs = orig->get_const_col_idxs();
+//    auto orig_vals = orig->get_const_values();
+//    auto cp_row_ptrs = column_permuted->get_row_ptrs();
+//    auto cp_col_idxs = column_permuted->get_col_idxs();
+//    auto cp_vals = column_permuted->get_values();
+//    auto num_nnz = orig->get_num_stored_elements();
+//    size_type num_rows = orig->get_size()[0];
+//    size_type num_cols = orig->get_size()[1];
+//
+//#pragma omp parallel for
+//    for (size_type row = 0; row < num_rows; ++row) {
+//        cp_row_ptrs[row] = orig_row_ptrs[row];
+//        for (size_type k = orig_row_ptrs[row];
+//             k < size_type(orig_row_ptrs[row + 1]); ++k) {
+//            cp_col_idxs[k] = perm[orig_col_idxs[k]];
+//            cp_vals[k] = orig_vals[k];
+//        }
+//    }
+//    cp_row_ptrs[num_rows] = orig_row_ptrs[num_rows];
+//}
+
+
+template <typename ValueType, typename IndexType>
+void column_permute(std::shared_ptr<const OmpExecutor> exec,
+                    const Array<IndexType> *permutation_indices,
+                    const matrix::Fbcsr<ValueType, IndexType> *orig,
+                    matrix::Fbcsr<ValueType, IndexType> *column_permuted)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto perm = permutation_indices->get_const_data();
+//    Array<IndexType> inv_perm(*permutation_indices);
+//    auto iperm = inv_perm.get_data();
+//#pragma omp parallel for
+//    for (size_type ind = 0; ind < inv_perm.get_num_elems(); ++ind) {
+//        iperm[perm[ind]] = ind;
+//    }
+//    column_permute_impl(&inv_perm, orig, column_permuted);
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_column_permute(
+    std::shared_ptr<const OmpExecutor> exec,
+    const Array<IndexType> *permutation_indices,
+    const matrix::Fbcsr<ValueType, IndexType> *orig,
+    matrix::Fbcsr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    column_permute_impl(permutation_indices, orig, column_permuted);
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_nonzeros_per_row(
+    std::shared_ptr<const OmpExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *source,
+    Array<size_type> *result) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto row_ptrs = source->get_const_row_ptrs();
+//    auto row_nnz_val = result->get_data();
+//
+//#pragma omp parallel for
+//    for (size_type i = 0; i < result->get_num_elems(); i++) {
+//        row_nnz_val[i] = row_ptrs[i + 1] - row_ptrs[i];
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void sort_by_column_index(std::shared_ptr<const OmpExecutor> exec,
+                          matrix::Fbcsr<ValueType, IndexType> *to_sort)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto values = to_sort->get_values();
+//    auto row_ptrs = to_sort->get_row_ptrs();
+//    auto col_idxs = to_sort->get_col_idxs();
+//    const auto number_rows = to_sort->get_size()[0];
+//#pragma omp parallel for
+//    for (size_type i = 0; i < number_rows; ++i) {
+//        auto start_row_idx = row_ptrs[i];
+//        auto row_nnz = row_ptrs[i + 1] - start_row_idx;
+//        auto helper = detail::IteratorFactory<IndexType, ValueType>(
+//            col_idxs + start_row_idx, values + start_row_idx, row_nnz);
+//        std::sort(helper.begin(), helper.end());
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
+
+
+template <typename ValueType, typename IndexType>
+void is_sorted_by_column_index(
+    std::shared_ptr<const OmpExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *to_check,
+    bool *is_sorted) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto row_ptrs = to_check->get_const_row_ptrs();
+//    const auto col_idxs = to_check->get_const_col_idxs();
+//    const auto size = to_check->get_size();
+//    bool local_is_sorted = true;
+//#pragma omp parallel for reduction(&& : local_is_sorted)
+//    for (size_type i = 0; i < size[0]; ++i) {
+//        // Skip comparison if any thread detects that it is not sorted
+//        if (local_is_sorted) {
+//            for (auto idx = row_ptrs[i] + 1; idx < row_ptrs[i + 1]; ++idx) {
+//                if (col_idxs[idx - 1] > col_idxs[idx]) {
+//                    local_is_sorted = false;
+//                    break;
+//                }
+//            }
+//        }
+//    }
+//    *is_sorted = local_is_sorted;
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
+
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(std::shared_ptr<const OmpExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *orig,
+                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto row_ptrs = orig->get_const_row_ptrs();
+//    const auto col_idxs = orig->get_const_col_idxs();
+//    const auto values = orig->get_const_values();
+//    const auto diag_size = diag->get_size()[0];
+//    auto diag_values = diag->get_values();
+//
+//#pragma omp parallel for
+//    for (size_type row = 0; row < diag_size; ++row) {
+//        for (size_type idx = row_ptrs[row]; idx < row_ptrs[row + 1]; ++idx) {
+//            if (col_idxs[idx] == row) {
+//                diag_values[row] = values[idx];
+//                break;
+//            }
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL);
+
+
+}  // namespace fbcsr
+}  // namespace omp
+}  // namespace kernels
+}  // namespace gko
diff --git a/omp/test/matrix/CMakeLists.txt b/omp/test/matrix/CMakeLists.txt
index 0c9fd00eb30..ff8ed2d0118 100644
--- a/omp/test/matrix/CMakeLists.txt
+++ b/omp/test/matrix/CMakeLists.txt
@@ -3,6 +3,7 @@ ginkgo_create_test(csr_kernels)
 ginkgo_create_test(dense_kernels)
 ginkgo_create_test(diagonal_kernels)
 ginkgo_create_test(ell_kernels)
+ginkgo_create_test(fbcsr_kernels)
 ginkgo_create_test(hybrid_kernels)
 ginkgo_create_test(sellp_kernels)
 ginkgo_create_test(sparsity_csr_kernels)
diff --git a/omp/test/matrix/fbcsr_kernels.cpp b/omp/test/matrix/fbcsr_kernels.cpp
new file mode 100644
index 00000000000..a1826bfa899
--- /dev/null
+++ b/omp/test/matrix/fbcsr_kernels.cpp
@@ -0,0 +1,662 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/matrix/fbcsr.hpp>
+
+
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+#include <ginkgo/core/matrix/identity.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+
+
+#include "core/matrix/fbcsr_kernels.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class Fbcsr : public ::testing::Test {
+protected:
+    using Arr = gko::Array<int>;
+    using Mtx = gko::matrix::Fbcsr<>;
+    using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
+    using ComplexMtx = gko::matrix::Fbcsr<std::complex<double>>;
+
+    Fbcsr() : mtx_size(532, 231), rand_engine(42) {}
+
+    void SetUp()
+    {
+        ref = gko::ReferenceExecutor::create();
+        omp = gko::OmpExecutor::create();
+    }
+
+    void TearDown()
+    {
+        if (omp != nullptr) {
+            ASSERT_NO_THROW(omp->synchronize());
+        }
+    }
+
+    template <typename MtxType>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row)
+    {
+        return gko::test::generate_random_matrix<MtxType>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(min_nnz_row, num_cols),
+            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    void set_up_apply_data(int num_vectors = 1)
+    {
+        mtx = Mtx::create(ref);
+        mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[1], 1));
+        complex_mtx = ComplexMtx::create(ref);
+        complex_mtx->copy_from(
+            gen_mtx<ComplexVec>(mtx_size[0], mtx_size[1], 1));
+        square_mtx = Mtx::create(ref);
+        square_mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[0], 1));
+        expected = gen_mtx<Vec>(mtx_size[0], num_vectors, 1);
+        y = gen_mtx<Vec>(mtx_size[1], num_vectors, 1);
+        alpha = gko::initialize<Vec>({2.0}, ref);
+        beta = gko::initialize<Vec>({-1.0}, ref);
+        dmtx = Mtx::create(omp);
+        dmtx->copy_from(mtx.get());
+        complex_dmtx = ComplexMtx::create(omp);
+        complex_dmtx->copy_from(complex_mtx.get());
+        square_dmtx = Mtx::create(omp);
+        square_dmtx->copy_from(square_mtx.get());
+        dresult = Vec::create(omp);
+        dresult->copy_from(expected.get());
+        dy = Vec::create(omp);
+        dy->copy_from(y.get());
+        dalpha = Vec::create(omp);
+        dalpha->copy_from(alpha.get());
+        dbeta = Vec::create(omp);
+        dbeta->copy_from(beta.get());
+
+        std::vector<int> tmp(mtx->get_size()[0], 0);
+        auto rng = std::default_random_engine{};
+        std::iota(tmp.begin(), tmp.end(), 0);
+        std::shuffle(tmp.begin(), tmp.end(), rng);
+        std::vector<int> tmp2(mtx->get_size()[1], 0);
+        std::iota(tmp2.begin(), tmp2.end(), 0);
+        std::shuffle(tmp2.begin(), tmp2.end(), rng);
+        rpermute_idxs =
+            std::unique_ptr<Arr>(new Arr{ref, tmp.begin(), tmp.end()});
+        drpermute_idxs =
+            std::unique_ptr<Arr>(new Arr{omp, tmp.begin(), tmp.end()});
+        cpermute_idxs =
+            std::unique_ptr<Arr>(new Arr{ref, tmp2.begin(), tmp2.end()});
+        dcpermute_idxs =
+            std::unique_ptr<Arr>(new Arr{omp, tmp2.begin(), tmp2.end()});
+    }
+
+    struct matrix_pair {
+        std::unique_ptr<Mtx> ref;
+        std::unique_ptr<Mtx> omp;
+    };
+
+    matrix_pair gen_unsorted_mtx()
+    {
+        constexpr int min_nnz_per_row = 2;  // Must be at least 2
+        auto local_mtx_ref =
+            gen_mtx<Mtx>(mtx_size[0], mtx_size[1], min_nnz_per_row);
+        for (size_t row = 0; row < mtx_size[0]; ++row) {
+            const auto row_ptrs = local_mtx_ref->get_const_row_ptrs();
+            const auto start_row = row_ptrs[row];
+            auto col_idx = local_mtx_ref->get_col_idxs() + start_row;
+            auto vals = local_mtx_ref->get_values() + start_row;
+            const auto nnz_in_this_row = row_ptrs[row + 1] - row_ptrs[row];
+            auto swap_idx_dist =
+                std::uniform_int_distribution<>(0, nnz_in_this_row - 1);
+            // shuffle `nnz_in_this_row / 2` times
+            for (size_t perm = 0; perm < nnz_in_this_row; perm += 2) {
+                const auto idx1 = swap_idx_dist(rand_engine);
+                const auto idx2 = swap_idx_dist(rand_engine);
+                std::swap(col_idx[idx1], col_idx[idx2]);
+                std::swap(vals[idx1], vals[idx2]);
+            }
+        }
+        auto local_mtx_omp = Mtx::create(omp);
+        local_mtx_omp->copy_from(local_mtx_ref.get());
+
+        return {std::move(local_mtx_ref), std::move(local_mtx_omp)};
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::OmpExecutor> omp;
+
+    const gko::dim<2> mtx_size;
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Mtx> mtx;
+    std::unique_ptr<ComplexMtx> complex_mtx;
+    std::unique_ptr<Mtx> square_mtx;
+    std::unique_ptr<Vec> expected;
+    std::unique_ptr<Vec> y;
+    std::unique_ptr<Vec> alpha;
+    std::unique_ptr<Vec> beta;
+
+    std::unique_ptr<Mtx> dmtx;
+    std::unique_ptr<ComplexMtx> complex_dmtx;
+    std::unique_ptr<Mtx> square_dmtx;
+    std::unique_ptr<Vec> dresult;
+    std::unique_ptr<Vec> dy;
+    std::unique_ptr<Vec> dalpha;
+    std::unique_ptr<Vec> dbeta;
+    std::unique_ptr<Arr> rpermute_idxs;
+    std::unique_ptr<Arr> drpermute_idxs;
+    std::unique_ptr<Arr> cpermute_idxs;
+    std::unique_ptr<Arr> dcpermute_idxs;
+};
+
+
+TEST_F(Fbcsr, SimpleApplyIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyToDenseMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(3);
+//
+//    mtx->apply(y.get(), expected.get());
+//    dmtx->apply(dy.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyToFbcsrMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    auto trans = mtx->transpose();
+//    auto d_trans = dmtx->transpose();
+//
+//    mtx->apply(alpha.get(), trans.get(), beta.get(), square_mtx.get());
+//    dmtx->apply(dalpha.get(), d_trans.get(), dbeta.get(), square_dmtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
+//    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+//    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
+//}
+
+
+TEST_F(Fbcsr, SimpleApplyToFbcsrMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    auto trans = mtx->transpose();
+//    auto d_trans = dmtx->transpose();
+//
+//    mtx->apply(trans.get(), square_mtx.get());
+//    dmtx->apply(d_trans.get(), square_dmtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
+//    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+//    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    auto a = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
+//    auto b = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
+//    auto da = Mtx::create(omp);
+//    auto db = Mtx::create(omp);
+//    da->copy_from(a.get());
+//    db->copy_from(b.get());
+//    auto id = gko::matrix::Identity<Mtx::value_type>::create(ref,
+//    mtx_size[1]); auto did =
+//    gko::matrix::Identity<Mtx::value_type>::create(omp, mtx_size[1]);
+//
+//    a->apply(alpha.get(), id.get(), beta.get(), b.get());
+//    da->apply(dalpha.get(), did.get(), dbeta.get(), db.get());
+//
+//    GKO_ASSERT_MTX_NEAR(b, db, 1e-14);
+//    GKO_ASSERT_MTX_EQ_SPARSITY(b, db);
+//    ASSERT_TRUE(db->is_sorted_by_column_index());
+//}
+
+
+TEST_F(Fbcsr, AdvancedApplyToDenseMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data(3);
+//
+//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, TransposeIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//
+//    auto trans = mtx->transpose();
+//    auto d_trans = dmtx->transpose();
+//
+//    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(d_trans.get()),
+//                        static_cast<Mtx *>(trans.get()), 0.0);
+//    ASSERT_TRUE(static_cast<Mtx
+//    *>(d_trans.get())->is_sorted_by_column_index());
+//}
+
+
+TEST_F(Fbcsr, ConjugateTransposeIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//
+//    auto trans = complex_mtx->conj_transpose();
+//    auto d_trans = complex_dmtx->conj_transpose();
+//
+//    GKO_ASSERT_MTX_NEAR(static_cast<ComplexMtx *>(d_trans.get()),
+//                        static_cast<ComplexMtx *>(trans.get()), 0.0);
+//    ASSERT_TRUE(
+//        static_cast<ComplexMtx
+//        *>(d_trans.get())->is_sorted_by_column_index());
+//}
+
+
+TEST_F(Fbcsr, ConvertToCooIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    auto coo_mtx = gko::matrix::Coo<>::create(ref);
+//    auto dcoo_mtx = gko::matrix::Coo<>::create(omp);
+//
+//    mtx->convert_to(coo_mtx.get());
+//    dmtx->convert_to(dcoo_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToCooIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    auto coo_mtx = gko::matrix::Coo<>::create(ref);
+//    auto dcoo_mtx = gko::matrix::Coo<>::create(omp);
+//
+//    mtx->move_to(coo_mtx.get());
+//    dmtx->move_to(dcoo_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, ConvertToDenseIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    auto dense_mtx = gko::matrix::Dense<>::create(ref);
+//    auto ddense_mtx = gko::matrix::Dense<>::create(omp);
+//
+//    mtx->convert_to(dense_mtx.get());
+//    dmtx->convert_to(ddense_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(ddense_mtx.get(), dense_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToDenseIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    auto dense_mtx = gko::matrix::Dense<>::create(ref);
+//    auto ddense_mtx = gko::matrix::Dense<>::create(omp);
+//
+//    mtx->move_to(dense_mtx.get());
+//    dmtx->move_to(ddense_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(ddense_mtx.get(), dense_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, ConvertToSparsityFbcsrIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    auto sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(ref);
+//    auto d_sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(omp);
+//
+//    mtx->convert_to(sparsity_mtx.get());
+//    dmtx->convert_to(d_sparsity_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_sparsity_mtx.get(), sparsity_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToSparsityFbcsrIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    auto sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(ref);
+//    auto d_sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(omp);
+//
+//    mtx->move_to(sparsity_mtx.get());
+//    dmtx->move_to(d_sparsity_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(d_sparsity_mtx.get(), sparsity_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, CalculatesNonzerosPerRow)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    gko::Array<gko::size_type> row_nnz(ref, mtx->get_size()[0]);
+//    gko::Array<gko::size_type> drow_nnz(omp, dmtx->get_size()[0]);
+//
+//    gko::kernels::reference::fbcsr::calculate_nonzeros_per_row(ref, mtx.get(),
+//                                                             &row_nnz);
+//    gko::kernels::omp::fbcsr::calculate_nonzeros_per_row(omp, dmtx.get(),
+//                                                       &drow_nnz);
+//
+//    GKO_ASSERT_ARRAY_EQ(row_nnz, drow_nnz);
+//}
+
+
+TEST_F(Fbcsr, ConvertToHybridIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Hybrid_type = gko::matrix::Hybrid<>;
+//    set_up_apply_data();
+//    auto hybrid_mtx = Hybrid_type::create(
+//        ref, std::make_shared<Hybrid_type::column_limit>(2));
+//    auto dhybrid_mtx = Hybrid_type::create(
+//        omp, std::make_shared<Hybrid_type::column_limit>(2));
+//
+//    mtx->convert_to(hybrid_mtx.get());
+//    dmtx->convert_to(dhybrid_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, MoveToHybridIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Hybrid_type = gko::matrix::Hybrid<>;
+//    set_up_apply_data();
+//    auto hybrid_mtx = Hybrid_type::create(
+//        ref, std::make_shared<Hybrid_type::column_limit>(2));
+//    auto dhybrid_mtx = Hybrid_type::create(
+//        omp, std::make_shared<Hybrid_type::column_limit>(2));
+//
+//    mtx->move_to(hybrid_mtx.get());
+//    dmtx->move_to(dhybrid_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
+//}
+
+
+TEST_F(Fbcsr, IsRowPermutable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    auto r_permute = mtx->row_permute(rpermute_idxs.get());
+//    auto dr_permute = dmtx->row_permute(drpermute_idxs.get());
+//
+//    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(r_permute.get()),
+//                        static_cast<Mtx *>(dr_permute.get()), 0);
+//}
+
+
+TEST_F(Fbcsr, IsColPermutable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    auto c_permute = mtx->column_permute(cpermute_idxs.get());
+//    auto dc_permute = dmtx->column_permute(dcpermute_idxs.get());
+//
+//    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(c_permute.get()),
+//                        static_cast<Mtx *>(dc_permute.get()), 0);
+//}
+
+
+TEST_F(Fbcsr, IsInverseRowPermutable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    auto inverse_r_permute = mtx->inverse_row_permute(rpermute_idxs.get());
+//    auto d_inverse_r_permute =
+//    dmtx->inverse_row_permute(drpermute_idxs.get());
+//
+//    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_r_permute.get()),
+//                        static_cast<Mtx *>(d_inverse_r_permute.get()), 0);
+//}
+
+
+TEST_F(Fbcsr, IsInverseColPermutable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    auto inverse_c_permute = mtx->inverse_column_permute(cpermute_idxs.get());
+//    auto d_inverse_c_permute =
+//        dmtx->inverse_column_permute(dcpermute_idxs.get());
+//
+//    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_c_permute.get()),
+//                        static_cast<Mtx *>(d_inverse_c_permute.get()), 0);
+//}
+
+
+TEST_F(Fbcsr, RecognizeSortedMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//    bool is_sorted_omp{};
+//    bool is_sorted_ref{};
+//
+//    is_sorted_ref = mtx->is_sorted_by_column_index();
+//    is_sorted_omp = dmtx->is_sorted_by_column_index();
+//
+//    ASSERT_EQ(is_sorted_ref, is_sorted_omp);
+//}
+
+
+TEST_F(Fbcsr, RecognizeUnsortedMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto uns_mtx = gen_unsorted_mtx();
+//    bool is_sorted_omp{};
+//    bool is_sorted_ref{};
+//
+//    is_sorted_ref = uns_mtx.ref->is_sorted_by_column_index();
+//    is_sorted_omp = uns_mtx.omp->is_sorted_by_column_index();
+//
+//    ASSERT_EQ(is_sorted_ref, is_sorted_omp);
+//}
+
+
+TEST_F(Fbcsr, SortSortedMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//
+//    mtx->sort_by_column_index();
+//    dmtx->sort_by_column_index();
+//
+//    // Values must be unchanged, therefore, tolerance is `0`
+//    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 0);
+//}
+
+
+TEST_F(Fbcsr, SortUnsortedMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto uns_mtx = gen_unsorted_mtx();
+//
+//    uns_mtx.ref->sort_by_column_index();
+//    uns_mtx.omp->sort_by_column_index();
+//
+//    // Values must be unchanged, therefore, tolerance is `0`
+//    GKO_ASSERT_MTX_NEAR(uns_mtx.ref, uns_mtx.omp, 0);
+//}
+
+
+TEST_F(Fbcsr, ExtractDiagonalIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//
+//    auto diag = mtx->extract_diagonal();
+//    auto ddiag = dmtx->extract_diagonal();
+//
+//    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+//}
+
+
+TEST_F(Fbcsr, InplaceAbsoluteMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//
+//    mtx->compute_absolute_inplace();
+//    dmtx->compute_absolute_inplace();
+//
+//    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, OutplaceAbsoluteMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//
+//    auto abs_mtx = mtx->compute_absolute();
+//    auto dabs_mtx = dmtx->compute_absolute();
+//
+//    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//
+//    complex_mtx->compute_absolute_inplace();
+//    complex_dmtx->compute_absolute_inplace();
+//
+//    GKO_ASSERT_MTX_NEAR(complex_mtx, complex_dmtx, 1e-14);
+//}
+
+
+TEST_F(Fbcsr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    set_up_apply_data();
+//
+//    auto abs_mtx = complex_mtx->compute_absolute();
+//    auto dabs_mtx = complex_dmtx->compute_absolute();
+//
+//    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+//}
+
+
+}  // namespace
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index 573b555506a..e84453b8664 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -18,6 +18,7 @@ target_sources(ginkgo_reference
     matrix/dense_kernels.cpp
     matrix/diagonal_kernels.cpp
     matrix/ell_kernels.cpp
+    matrix/fbcsr_kernels.cpp
     matrix/hybrid_kernels.cpp
     matrix/sellp_kernels.cpp
     matrix/sparsity_csr_kernels.cpp
diff --git a/reference/components/fbcsr_spgeam.hpp b/reference/components/fbcsr_spgeam.hpp
new file mode 100644
index 00000000000..e4a06532ed3
--- /dev/null
+++ b/reference/components/fbcsr_spgeam.hpp
@@ -0,0 +1,31 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
new file mode 100644
index 00000000000..af0f19a8947
--- /dev/null
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -0,0 +1,968 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/fbcsr_kernels.hpp"
+
+
+#include <algorithm>
+#include <iterator>
+#include <numeric>
+#include <utility>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/ell.hpp>
+#include <ginkgo/core/matrix/hybrid.hpp>
+#include <ginkgo/core/matrix/sellp.hpp>
+
+
+#include "core/base/allocator.hpp"
+#include "core/base/iterator_factory.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/fbcsr_builder.hpp"
+#include "reference/components/fbcsr_spgeam.hpp"
+#include "reference/components/format_conversion.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace reference {
+/**
+ * @brief The Compressed sparse row matrix format namespace.
+ * @ref Fbcsr
+ * @ingroup fbcsr
+ */
+namespace fbcsr {
+
+
+template <typename ValueType, typename IndexType>
+void spmv(std::shared_ptr<const ReferenceExecutor> exec,
+          const matrix::Fbcsr<ValueType, IndexType> *a,
+          const matrix::Dense<ValueType> *b,
+          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto row_ptrs = a->get_const_row_ptrs();
+//    auto col_idxs = a->get_const_col_idxs();
+//    auto vals = a->get_const_values();
+//
+//    for (size_type row = 0; row < a->get_size()[0]; ++row) {
+//        for (size_type j = 0; j < c->get_size()[1]; ++j) {
+//            c->at(row, j) = zero<ValueType>();
+//        }
+//        for (size_type k = row_ptrs[row];
+//             k < static_cast<size_type>(row_ptrs[row + 1]); ++k) {
+//            auto val = vals[k];
+//            auto col = col_idxs[k];
+//            for (size_type j = 0; j < c->get_size()[1]; ++j) {
+//                c->at(row, j) += val * b->at(col, j);
+//            }
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spmv(std::shared_ptr<const ReferenceExecutor> exec,
+                   const matrix::Dense<ValueType> *alpha,
+                   const matrix::Fbcsr<ValueType, IndexType> *a,
+                   const matrix::Dense<ValueType> *b,
+                   const matrix::Dense<ValueType> *beta,
+                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto row_ptrs = a->get_const_row_ptrs();
+//    auto col_idxs = a->get_const_col_idxs();
+//    auto vals = a->get_const_values();
+//    auto valpha = alpha->at(0, 0);
+//    auto vbeta = beta->at(0, 0);
+//
+//    for (size_type row = 0; row < a->get_size()[0]; ++row) {
+//        for (size_type j = 0; j < c->get_size()[1]; ++j) {
+//            c->at(row, j) *= vbeta;
+//        }
+//        for (size_type k = row_ptrs[row];
+//             k < static_cast<size_type>(row_ptrs[row + 1]); ++k) {
+//            auto val = vals[k];
+//            auto col = col_idxs[k];
+//            for (size_type j = 0; j < c->get_size()[1]; ++j) {
+//                c->at(row, j) += valpha * val * b->at(col, j);
+//            }
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_insert_row(unordered_set<IndexType> &cols,
+                       const matrix::Fbcsr<ValueType, IndexType> *c,
+                       size_type row) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto row_ptrs = c->get_const_row_ptrs();
+//    auto col_idxs = c->get_const_col_idxs();
+//    cols.insert(col_idxs + row_ptrs[row], col_idxs + row_ptrs[row + 1]);
+//}
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_insert_row2(unordered_set<IndexType> &cols,
+                        const matrix::Fbcsr<ValueType, IndexType> *a,
+                        const matrix::Fbcsr<ValueType, IndexType> *b,
+                        size_type row) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto a_row_ptrs = a->get_const_row_ptrs();
+//    auto a_col_idxs = a->get_const_col_idxs();
+//    auto b_row_ptrs = b->get_const_row_ptrs();
+//    auto b_col_idxs = b->get_const_col_idxs();
+//    for (size_type a_nz = a_row_ptrs[row];
+//         a_nz < size_type(a_row_ptrs[row + 1]); ++a_nz) {
+//        auto a_col = a_col_idxs[a_nz];
+//        auto b_row = a_col;
+//        cols.insert(b_col_idxs + b_row_ptrs[b_row],
+//                    b_col_idxs + b_row_ptrs[b_row + 1]);
+//    }
+//}
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_accumulate_row(map<IndexType, ValueType> &cols,
+                           const matrix::Fbcsr<ValueType, IndexType> *c,
+                           ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto row_ptrs = c->get_const_row_ptrs();
+//    auto col_idxs = c->get_const_col_idxs();
+//    auto vals = c->get_const_values();
+//    for (size_type c_nz = row_ptrs[row]; c_nz < size_type(row_ptrs[row + 1]);
+//         ++c_nz) {
+//        auto c_col = col_idxs[c_nz];
+//        auto c_val = vals[c_nz];
+//        cols[c_col] += scale * c_val;
+//    }
+//}
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_accumulate_row2(map<IndexType, ValueType> &cols,
+                            const matrix::Fbcsr<ValueType, IndexType> *a,
+                            const matrix::Fbcsr<ValueType, IndexType> *b,
+                            ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto a_row_ptrs = a->get_const_row_ptrs();
+//    auto a_col_idxs = a->get_const_col_idxs();
+//    auto a_vals = a->get_const_values();
+//    auto b_row_ptrs = b->get_const_row_ptrs();
+//    auto b_col_idxs = b->get_const_col_idxs();
+//    auto b_vals = b->get_const_values();
+//    for (size_type a_nz = a_row_ptrs[row];
+//         a_nz < size_type(a_row_ptrs[row + 1]); ++a_nz) {
+//        auto a_col = a_col_idxs[a_nz];
+//        auto a_val = a_vals[a_nz];
+//        auto b_row = a_col;
+//        for (size_type b_nz = b_row_ptrs[b_row];
+//             b_nz < size_type(b_row_ptrs[b_row + 1]); ++b_nz) {
+//            auto b_col = b_col_idxs[b_nz];
+//            auto b_val = b_vals[b_nz];
+//            cols[b_col] += scale * a_val * b_val;
+//        }
+//    }
+//}
+
+
+template <typename ValueType, typename IndexType>
+void spgemm(std::shared_ptr<const ReferenceExecutor> exec,
+            const matrix::Fbcsr<ValueType, IndexType> *a,
+            const matrix::Fbcsr<ValueType, IndexType> *b,
+            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = a->get_size()[0];
+//
+//    // first sweep: count nnz for each row
+//    auto c_row_ptrs = c->get_row_ptrs();
+//
+//    unordered_set<IndexType> local_col_idxs(exec);
+//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
+//        local_col_idxs.clear();
+//        spgemm_insert_row2(local_col_idxs, a, b, a_row);
+//        c_row_ptrs[a_row] = local_col_idxs.size();
+//    }
+//
+//    // build row pointers
+//    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
+//
+//    // second sweep: accumulate non-zeros
+//    auto new_nnz = c_row_ptrs[num_rows];
+//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
+//    auto &c_col_idxs_array = c_builder.get_col_idx_array();
+//    auto &c_vals_array = c_builder.get_value_array();
+//    c_col_idxs_array.resize_and_reset(new_nnz);
+//    c_vals_array.resize_and_reset(new_nnz);
+//    auto c_col_idxs = c_col_idxs_array.get_data();
+//    auto c_vals = c_vals_array.get_data();
+//
+//    map<IndexType, ValueType> local_row_nzs(exec);
+//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
+//        local_row_nzs.clear();
+//        spgemm_accumulate_row2(local_row_nzs, a, b, one<ValueType>(), a_row);
+//        // store result
+//        auto c_nz = c_row_ptrs[a_row];
+//        for (auto pair : local_row_nzs) {
+//            c_col_idxs[c_nz] = pair.first;
+//            c_vals[c_nz] = pair.second;
+//            ++c_nz;
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEMM_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spgemm(std::shared_ptr<const ReferenceExecutor> exec,
+                     const matrix::Dense<ValueType> *alpha,
+                     const matrix::Fbcsr<ValueType, IndexType> *a,
+                     const matrix::Fbcsr<ValueType, IndexType> *b,
+                     const matrix::Dense<ValueType> *beta,
+                     const matrix::Fbcsr<ValueType, IndexType> *d,
+                     matrix::Fbcsr<ValueType, IndexType> *c)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = a->get_size()[0];
+//    auto valpha = alpha->at(0, 0);
+//    auto vbeta = beta->at(0, 0);
+//
+//    // first sweep: count nnz for each row
+//    auto c_row_ptrs = c->get_row_ptrs();
+//
+//    unordered_set<IndexType> local_col_idxs(exec);
+//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
+//        local_col_idxs.clear();
+//        spgemm_insert_row(local_col_idxs, d, a_row);
+//        spgemm_insert_row2(local_col_idxs, a, b, a_row);
+//        c_row_ptrs[a_row] = local_col_idxs.size();
+//    }
+//
+//    // build row pointers
+//    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
+//
+//    // second sweep: accumulate non-zeros
+//    auto new_nnz = c_row_ptrs[num_rows];
+//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
+//    auto &c_col_idxs_array = c_builder.get_col_idx_array();
+//    auto &c_vals_array = c_builder.get_value_array();
+//    c_col_idxs_array.resize_and_reset(new_nnz);
+//    c_vals_array.resize_and_reset(new_nnz);
+//    auto c_col_idxs = c_col_idxs_array.get_data();
+//    auto c_vals = c_vals_array.get_data();
+//
+//    map<IndexType, ValueType> local_row_nzs(exec);
+//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
+//        local_row_nzs.clear();
+//        spgemm_accumulate_row(local_row_nzs, d, vbeta, a_row);
+//        spgemm_accumulate_row2(local_row_nzs, a, b, valpha, a_row);
+//        // store result
+//        auto c_nz = c_row_ptrs[a_row];
+//        for (auto pair : local_row_nzs) {
+//            c_col_idxs[c_nz] = pair.first;
+//            c_vals[c_nz] = pair.second;
+//            ++c_nz;
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void spgeam(std::shared_ptr<const ReferenceExecutor> exec,
+            const matrix::Dense<ValueType> *alpha,
+            const matrix::Fbcsr<ValueType, IndexType> *a,
+            const matrix::Dense<ValueType> *beta,
+            const matrix::Fbcsr<ValueType, IndexType> *b,
+            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = a->get_size()[0];
+//    auto valpha = alpha->at(0, 0);
+//    auto vbeta = beta->at(0, 0);
+//
+//    // first sweep: count nnz for each row
+//    auto c_row_ptrs = c->get_row_ptrs();
+//
+//    abstract_spgeam(
+//        a, b, [](IndexType) { return IndexType{}; },
+//        [](IndexType, IndexType, ValueType, ValueType, IndexType &nnz) {
+//            ++nnz;
+//        },
+//        [&](IndexType row, IndexType nnz) { c_row_ptrs[row] = nnz; });
+//
+//    // build row pointers
+//    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
+//
+//    // second sweep: accumulate non-zeros
+//    auto new_nnz = c_row_ptrs[num_rows];
+//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
+//    auto &c_col_idxs_array = c_builder.get_col_idx_array();
+//    auto &c_vals_array = c_builder.get_value_array();
+//    c_col_idxs_array.resize_and_reset(new_nnz);
+//    c_vals_array.resize_and_reset(new_nnz);
+//    auto c_col_idxs = c_col_idxs_array.get_data();
+//    auto c_vals = c_vals_array.get_data();
+//
+//    abstract_spgeam(
+//        a, b, [&](IndexType row) { return c_row_ptrs[row]; },
+//        [&](IndexType, IndexType col, ValueType a_val, ValueType b_val,
+//            IndexType &nz) {
+//            c_vals[nz] = valpha * a_val + vbeta * b_val;
+//            c_col_idxs[nz] = col;
+//            ++nz;
+//        },
+//        [](IndexType, IndexType) {});
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEAM_KERNEL);
+
+
+template <typename IndexType>
+void convert_row_ptrs_to_idxs(std::shared_ptr<const ReferenceExecutor> exec,
+                              const IndexType *ptrs, size_type num_rows,
+                              IndexType *idxs) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    convert_ptrs_to_idxs(ptrs, num_rows, idxs);
+//}
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_coo(std::shared_ptr<const ReferenceExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *source,
+                    matrix::Coo<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = result->get_size()[0];
+//
+//    auto row_idxs = result->get_row_idxs();
+//    const auto source_row_ptrs = source->get_const_row_ptrs();
+//
+//    convert_row_ptrs_to_idxs(exec, source_row_ptrs, num_rows, row_idxs);
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL);
+
+template <typename ValueType, typename IndexType>
+void convert_to_dense(std::shared_ptr<const ReferenceExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *source,
+                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = source->get_size()[0];
+//    auto num_cols = source->get_size()[1];
+//    auto row_ptrs = source->get_const_row_ptrs();
+//    auto col_idxs = source->get_const_col_idxs();
+//    auto vals = source->get_const_values();
+//
+//    for (size_type row = 0; row < num_rows; ++row) {
+//        for (size_type col = 0; col < num_cols; ++col) {
+//            result->at(row, col) = zero<ValueType>();
+//        }
+//        for (size_type i = row_ptrs[row];
+//             i < static_cast<size_type>(row_ptrs[row + 1]); ++i) {
+//            result->at(row, col_idxs[i]) = vals[i];
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_sellp(std::shared_ptr<const ReferenceExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *source,
+                      matrix::Sellp<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = result->get_size()[0];
+//    auto num_cols = result->get_size()[1];
+//    auto vals = result->get_values();
+//    auto col_idxs = result->get_col_idxs();
+//    auto slice_lengths = result->get_slice_lengths();
+//    auto slice_sets = result->get_slice_sets();
+//    auto slice_size = (result->get_slice_size() == 0)
+//                          ? matrix::default_slice_size
+//                          : result->get_slice_size();
+//    auto stride_factor = (result->get_stride_factor() == 0)
+//                             ? matrix::default_stride_factor
+//                             : result->get_stride_factor();
+//
+//    const auto source_row_ptrs = source->get_const_row_ptrs();
+//    const auto source_col_idxs = source->get_const_col_idxs();
+//    const auto source_values = source->get_const_values();
+//
+//    int slice_num = ceildiv(num_rows, slice_size);
+//    slice_sets[0] = 0;
+//    for (size_type slice = 0; slice < slice_num; slice++) {
+//        if (slice > 0) {
+//            slice_sets[slice] =
+//                slice_sets[slice - 1] + slice_lengths[slice - 1];
+//        }
+//        slice_lengths[slice] = 0;
+//        for (size_type row = 0; row < slice_size; row++) {
+//            size_type global_row = slice * slice_size + row;
+//            if (global_row >= num_rows) {
+//                break;
+//            }
+//            slice_lengths[slice] =
+//                (slice_lengths[slice] >
+//                 source_row_ptrs[global_row + 1] -
+//                 source_row_ptrs[global_row])
+//                    ? slice_lengths[slice]
+//                    : source_row_ptrs[global_row + 1] -
+//                          source_row_ptrs[global_row];
+//        }
+//        slice_lengths[slice] =
+//            stride_factor * ceildiv(slice_lengths[slice], stride_factor);
+//        for (size_type row = 0; row < slice_size; row++) {
+//            size_type global_row = slice * slice_size + row;
+//            if (global_row >= num_rows) {
+//                break;
+//            }
+//            size_type sellp_ind = slice_sets[slice] * slice_size + row;
+//            for (size_type fbcsr_ind = source_row_ptrs[global_row];
+//                 fbcsr_ind < source_row_ptrs[global_row + 1]; fbcsr_ind++) {
+//                vals[sellp_ind] = source_values[fbcsr_ind];
+//                col_idxs[sellp_ind] = source_col_idxs[fbcsr_ind];
+//                sellp_ind += slice_size;
+//            }
+//            for (size_type i = sellp_ind;
+//                 i <
+//                 (slice_sets[slice] + slice_lengths[slice]) * slice_size +
+//                 row; i += slice_size) {
+//                col_idxs[i] = 0;
+//                vals[i] = zero<ValueType>();
+//            }
+//        }
+//    }
+//    if (slice_num > 0) {
+//        slice_sets[slice_num] =
+//            slice_sets[slice_num - 1] + slice_lengths[slice_num - 1];
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_total_cols(std::shared_ptr<const ReferenceExecutor> exec,
+                          const matrix::Fbcsr<ValueType, IndexType> *source,
+                          size_type *result, size_type stride_factor,
+                          size_type slice_size) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    size_type total_cols = 0;
+//    const auto num_rows = source->get_size()[0];
+//    const auto slice_num = ceildiv(num_rows, slice_size);
+//
+//    const auto row_ptrs = source->get_const_row_ptrs();
+//
+//    for (size_type slice = 0; slice < slice_num; slice++) {
+//        IndexType max_nnz_per_row_in_this_slice = 0;
+//        for (size_type row = 0;
+//             row < slice_size && row + slice * slice_size < num_rows; row++) {
+//            size_type global_row = slice * slice_size + row;
+//            max_nnz_per_row_in_this_slice =
+//                max(row_ptrs[global_row + 1] - row_ptrs[global_row],
+//                    max_nnz_per_row_in_this_slice);
+//        }
+//        total_cols += ceildiv(max_nnz_per_row_in_this_slice, stride_factor) *
+//                      stride_factor;
+//    }
+//
+//    *result = total_cols;
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_ell(std::shared_ptr<const ReferenceExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *source,
+                    matrix::Ell<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto num_rows = source->get_size()[0];
+//    const auto num_cols = source->get_size()[1];
+//    const auto vals = source->get_const_values();
+//    const auto col_idxs = source->get_const_col_idxs();
+//    const auto row_ptrs = source->get_const_row_ptrs();
+//
+//    const auto num_stored_elements_per_row =
+//        result->get_num_stored_elements_per_row();
+//
+//    for (size_type row = 0; row < num_rows; row++) {
+//        for (size_type i = 0; i < num_stored_elements_per_row; i++) {
+//            result->val_at(row, i) = zero<ValueType>();
+//            result->col_at(row, i) = 0;
+//        }
+//        for (size_type col_idx = 0; col_idx < row_ptrs[row + 1] -
+//        row_ptrs[row];
+//             col_idx++) {
+//            result->val_at(row, col_idx) = vals[row_ptrs[row] + col_idx];
+//            result->col_at(row, col_idx) = col_idxs[row_ptrs[row] + col_idx];
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL);
+
+
+template <typename ValueType, typename IndexType, typename UnaryOperator>
+inline void convert_fbcsr_to_csc(size_type num_rows, const IndexType *row_ptrs,
+                                 const IndexType *col_idxs,
+                                 const ValueType *fbcsr_vals,
+                                 IndexType *row_idxs, IndexType *col_ptrs,
+                                 ValueType *csc_vals,
+                                 UnaryOperator op) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    for (size_type row = 0; row < num_rows; ++row) {
+//        for (auto i = row_ptrs[row]; i < row_ptrs[row + 1]; ++i) {
+//            const auto dest_idx = col_ptrs[col_idxs[i]]++;
+//            row_idxs[dest_idx] = row;
+//            csc_vals[dest_idx] = op(fbcsr_vals[i]);
+//        }
+//    }
+//}
+
+
+template <typename ValueType, typename IndexType, typename UnaryOperator>
+void transpose_and_transform(std::shared_ptr<const ReferenceExecutor> exec,
+                             matrix::Fbcsr<ValueType, IndexType> *trans,
+                             const matrix::Fbcsr<ValueType, IndexType> *orig,
+                             UnaryOperator op) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto trans_row_ptrs = trans->get_row_ptrs();
+//    auto orig_row_ptrs = orig->get_const_row_ptrs();
+//    auto trans_col_idxs = trans->get_col_idxs();
+//    auto orig_col_idxs = orig->get_const_col_idxs();
+//    auto trans_vals = trans->get_values();
+//    auto orig_vals = orig->get_const_values();
+//
+//    auto orig_num_cols = orig->get_size()[1];
+//    auto orig_num_rows = orig->get_size()[0];
+//    auto orig_nnz = orig_row_ptrs[orig_num_rows];
+//
+//    trans_row_ptrs[0] = 0;
+//    convert_idxs_to_ptrs(orig_col_idxs, orig_nnz, trans_row_ptrs + 1,
+//                         orig_num_cols);
+//
+//    convert_fbcsr_to_csc(orig_num_rows, orig_row_ptrs, orig_col_idxs,
+//    orig_vals,
+//                       trans_col_idxs, trans_row_ptrs + 1, trans_vals, op);
+//}
+
+
+template <typename ValueType, typename IndexType>
+void transpose(std::shared_ptr<const ReferenceExecutor> exec,
+               const matrix::Fbcsr<ValueType, IndexType> *orig,
+               matrix::Fbcsr<ValueType, IndexType> *trans) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    transpose_and_transform(exec, trans, orig,
+//                            [](const ValueType x) { return x; });
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void conj_transpose(std::shared_ptr<const ReferenceExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *orig,
+                    matrix::Fbcsr<ValueType, IndexType> *trans)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    transpose_and_transform(exec, trans, orig,
+//                            [](const ValueType x) { return conj(x); });
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_max_nnz_per_row(
+    std::shared_ptr<const ReferenceExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *source,
+    size_type *result) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto num_rows = source->get_size()[0];
+//    const auto row_ptrs = source->get_const_row_ptrs();
+//    IndexType max_nnz = 0;
+//
+//    for (size_type i = 0; i < num_rows; i++) {
+//        max_nnz = std::max(row_ptrs[i + 1] - row_ptrs[i], max_nnz);
+//    }
+//
+//    *result = max_nnz;
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_hybrid(std::shared_ptr<const ReferenceExecutor> exec,
+                       const matrix::Fbcsr<ValueType, IndexType> *source,
+                       matrix::Hybrid<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto num_rows = result->get_size()[0];
+//    auto num_cols = result->get_size()[1];
+//    auto strategy = result->get_strategy();
+//    auto ell_lim = strategy->get_ell_num_stored_elements_per_row();
+//    auto coo_lim = strategy->get_coo_nnz();
+//    auto coo_val = result->get_coo_values();
+//    auto coo_col = result->get_coo_col_idxs();
+//    auto coo_row = result->get_coo_row_idxs();
+//
+//    // Initial Hybrid Matrix
+//    for (size_type i = 0; i < result->get_ell_num_stored_elements_per_row();
+//         i++) {
+//        for (size_type j = 0; j < result->get_ell_stride(); j++) {
+//            result->ell_val_at(j, i) = zero<ValueType>();
+//            result->ell_col_at(j, i) = 0;
+//        }
+//    }
+//    for (size_type i = 0; i < result->get_coo_num_stored_elements(); i++) {
+//        coo_val[i] = zero<ValueType>();
+//        coo_col[i] = 0;
+//        coo_row[i] = 0;
+//    }
+//
+//    const auto fbcsr_row_ptrs = source->get_const_row_ptrs();
+//    const auto fbcsr_vals = source->get_const_values();
+//    size_type fbcsr_idx = 0;
+//    size_type coo_idx = 0;
+//    for (IndexType row = 0; row < num_rows; row++) {
+//        size_type ell_idx = 0;
+//        while (fbcsr_idx < fbcsr_row_ptrs[row + 1]) {
+//            const auto val = fbcsr_vals[fbcsr_idx];
+//            if (ell_idx < ell_lim) {
+//                result->ell_val_at(row, ell_idx) = val;
+//                result->ell_col_at(row, ell_idx) =
+//                    source->get_const_col_idxs()[fbcsr_idx];
+//                ell_idx++;
+//            } else {
+//                coo_val[coo_idx] = val;
+//                coo_col[coo_idx] = source->get_const_col_idxs()[fbcsr_idx];
+//                coo_row[coo_idx] = row;
+//                coo_idx++;
+//            }
+//            fbcsr_idx++;
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void row_permute_impl(std::shared_ptr<const ReferenceExecutor> exec,
+                      const Array<IndexType> *permutation_indices,
+                      const matrix::Fbcsr<ValueType, IndexType> *orig,
+                      matrix::Fbcsr<ValueType, IndexType> *row_permuted)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto perm = permutation_indices->get_const_data();
+//    auto orig_row_ptrs = orig->get_const_row_ptrs();
+//    auto orig_col_idxs = orig->get_const_col_idxs();
+//    auto orig_vals = orig->get_const_values();
+//    auto rp_row_ptrs = row_permuted->get_row_ptrs();
+//    auto rp_col_idxs = row_permuted->get_col_idxs();
+//    auto rp_vals = row_permuted->get_values();
+//    size_type num_rows = orig->get_size()[0];
+//    size_type num_nnz = orig->get_num_stored_elements();
+//
+//    size_type cur_ptr = 0;
+//    rp_row_ptrs[0] = cur_ptr;
+//    vector<size_type> orig_num_nnz_per_row(num_rows, 0, exec);
+//    for (size_type row = 0; row < num_rows; ++row) {
+//        orig_num_nnz_per_row[row] = orig_row_ptrs[row + 1] -
+//        orig_row_ptrs[row];
+//    }
+//    for (size_type row = 0; row < num_rows; ++row) {
+//        rp_row_ptrs[row + 1] =
+//            rp_row_ptrs[row] + orig_num_nnz_per_row[perm[row]];
+//    }
+//    rp_row_ptrs[num_rows] = orig_row_ptrs[num_rows];
+//    for (size_type row = 0; row < num_rows; ++row) {
+//        auto new_row = perm[row];
+//        auto new_k = orig_row_ptrs[new_row];
+//        for (size_type k = rp_row_ptrs[row];
+//             k < size_type(rp_row_ptrs[row + 1]); ++k) {
+//            rp_col_idxs[k] = orig_col_idxs[new_k];
+//            rp_vals[k] = orig_vals[new_k];
+//            new_k++;
+//        }
+//    }
+//}
+
+
+template <typename ValueType, typename IndexType>
+void row_permute(std::shared_ptr<const ReferenceExecutor> exec,
+                 const Array<IndexType> *permutation_indices,
+                 const matrix::Fbcsr<ValueType, IndexType> *orig,
+                 matrix::Fbcsr<ValueType, IndexType> *row_permuted)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    row_permute_impl(exec, permutation_indices, orig, row_permuted);
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_row_permute(std::shared_ptr<const ReferenceExecutor> exec,
+                         const Array<IndexType> *permutation_indices,
+                         const matrix::Fbcsr<ValueType, IndexType> *orig,
+                         matrix::Fbcsr<ValueType, IndexType> *row_permuted)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto perm = permutation_indices->get_const_data();
+//    Array<IndexType> inv_perm(*permutation_indices);
+//    auto iperm = inv_perm.get_data();
+//    for (size_type ind = 0; ind < inv_perm.get_num_elems(); ++ind) {
+//        iperm[perm[ind]] = ind;
+//    }
+//
+//    row_permute_impl(exec, &inv_perm, orig, row_permuted);
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void column_permute_impl(const Array<IndexType> *permutation_indices,
+                         const matrix::Fbcsr<ValueType, IndexType> *orig,
+                         matrix::Fbcsr<ValueType, IndexType> *column_permuted)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto perm = permutation_indices->get_const_data();
+//    auto orig_row_ptrs = orig->get_const_row_ptrs();
+//    auto orig_col_idxs = orig->get_const_col_idxs();
+//    auto orig_vals = orig->get_const_values();
+//    auto cp_row_ptrs = column_permuted->get_row_ptrs();
+//    auto cp_col_idxs = column_permuted->get_col_idxs();
+//    auto cp_vals = column_permuted->get_values();
+//    auto num_nnz = orig->get_num_stored_elements();
+//    size_type num_rows = orig->get_size()[0];
+//    size_type num_cols = orig->get_size()[1];
+//
+//    for (size_type row = 0; row < num_rows; ++row) {
+//        cp_row_ptrs[row] = orig_row_ptrs[row];
+//        for (size_type k = orig_row_ptrs[row];
+//             k < size_type(orig_row_ptrs[row + 1]); ++k) {
+//            cp_col_idxs[k] = perm[orig_col_idxs[k]];
+//            cp_vals[k] = orig_vals[k];
+//        }
+//    }
+//    cp_row_ptrs[num_rows] = orig_row_ptrs[num_rows];
+//}
+
+
+template <typename ValueType, typename IndexType>
+void column_permute(std::shared_ptr<const ReferenceExecutor> exec,
+                    const Array<IndexType> *permutation_indices,
+                    const matrix::Fbcsr<ValueType, IndexType> *orig,
+                    matrix::Fbcsr<ValueType, IndexType> *column_permuted)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto perm = permutation_indices->get_const_data();
+//    Array<IndexType> inv_perm(*permutation_indices);
+//    auto iperm = inv_perm.get_data();
+//    for (size_type ind = 0; ind < inv_perm.get_num_elems(); ++ind) {
+//        iperm[perm[ind]] = ind;
+//    }
+//    column_permute_impl(&inv_perm, orig, column_permuted);
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_column_permute(
+    std::shared_ptr<const ReferenceExecutor> exec,
+    const Array<IndexType> *permutation_indices,
+    const matrix::Fbcsr<ValueType, IndexType> *orig,
+    matrix::Fbcsr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    column_permute_impl(permutation_indices, orig, column_permuted);
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_nonzeros_per_row(
+    std::shared_ptr<const ReferenceExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *source,
+    Array<size_type> *result) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto row_ptrs = source->get_const_row_ptrs();
+//    auto row_nnz_val = result->get_data();
+//    for (size_type i = 0; i < result->get_num_elems(); i++) {
+//        row_nnz_val[i] = row_ptrs[i + 1] - row_ptrs[i];
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void sort_by_column_index(std::shared_ptr<const ReferenceExecutor> exec,
+                          matrix::Fbcsr<ValueType, IndexType> *to_sort)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto values = to_sort->get_values();
+//    auto row_ptrs = to_sort->get_row_ptrs();
+//    auto col_idxs = to_sort->get_col_idxs();
+//    const auto number_rows = to_sort->get_size()[0];
+//    for (size_type i = 0; i < number_rows; ++i) {
+//        auto start_row_idx = row_ptrs[i];
+//        auto row_nnz = row_ptrs[i + 1] - start_row_idx;
+//        auto helper = detail::IteratorFactory<IndexType, ValueType>(
+//            col_idxs + start_row_idx, values + start_row_idx, row_nnz);
+//        std::sort(helper.begin(), helper.end());
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
+
+
+template <typename ValueType, typename IndexType>
+void is_sorted_by_column_index(
+    std::shared_ptr<const ReferenceExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *to_check,
+    bool *is_sorted) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto row_ptrs = to_check->get_const_row_ptrs();
+//    const auto col_idxs = to_check->get_const_col_idxs();
+//    const auto size = to_check->get_size();
+//    for (size_type i = 0; i < size[0]; ++i) {
+//        for (auto idx = row_ptrs[i] + 1; idx < row_ptrs[i + 1]; ++idx) {
+//            if (col_idxs[idx - 1] > col_idxs[idx]) {
+//                *is_sorted = false;
+//                return;
+//            }
+//        }
+//    }
+//    *is_sorted = true;
+//    return;
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
+
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(std::shared_ptr<const ReferenceExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *orig,
+                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    const auto row_ptrs = orig->get_const_row_ptrs();
+//    const auto col_idxs = orig->get_const_col_idxs();
+//    const auto values = orig->get_const_values();
+//    const auto diag_size = diag->get_size()[0];
+//    auto diag_values = diag->get_values();
+//
+//    for (size_type row = 0; row < diag_size; ++row) {
+//        for (size_type idx = row_ptrs[row]; idx < row_ptrs[row + 1]; ++idx) {
+//            if (col_idxs[idx] == row) {
+//                diag_values[row] = values[idx];
+//                break;
+//            }
+//        }
+//    }
+//}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL);
+
+
+}  // namespace fbcsr
+}  // namespace reference
+}  // namespace kernels
+}  // namespace gko
diff --git a/reference/test/matrix/CMakeLists.txt b/reference/test/matrix/CMakeLists.txt
index 81298ba86cd..c826a9eaca7 100644
--- a/reference/test/matrix/CMakeLists.txt
+++ b/reference/test/matrix/CMakeLists.txt
@@ -3,6 +3,7 @@ ginkgo_create_test(csr_kernels)
 ginkgo_create_test(dense_kernels)
 ginkgo_create_test(diagonal_kernels)
 ginkgo_create_test(ell_kernels)
+ginkgo_create_test(fbcsr_kernels)
 ginkgo_create_test(hybrid_kernels)
 ginkgo_create_test(identity)
 ginkgo_create_test(permutation)
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
new file mode 100644
index 00000000000..b5fd2162da5
--- /dev/null
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -0,0 +1,1550 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/matrix/fbcsr.hpp>
+
+
+#include <algorithm>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+#include <ginkgo/core/matrix/ell.hpp>
+#include <ginkgo/core/matrix/hybrid.hpp>
+#include <ginkgo/core/matrix/identity.hpp>
+#include <ginkgo/core/matrix/matrix_strategies.hpp>
+#include <ginkgo/core/matrix/sellp.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+
+
+#include "core/matrix/fbcsr_kernels.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+namespace matstr = gko::matrix::matrix_strategy;
+
+
+constexpr int mat_bs = 1;
+
+
+template <typename ValueIndexType>
+class Fbcsr : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using Coo = gko::matrix::Coo<value_type, index_type>;
+    using Mtx = gko::matrix::Fbcsr<value_type, index_type>;
+    using Sellp = gko::matrix::Sellp<value_type, index_type>;
+    using SparsityCsr = gko::matrix::SparsityCsr<value_type, index_type>;
+    using Ell = gko::matrix::Ell<value_type, index_type>;
+    using Hybrid = gko::matrix::Hybrid<value_type, index_type>;
+    using Vec = gko::matrix::Dense<value_type>;
+
+    Fbcsr()
+        : exec(gko::ReferenceExecutor::create()),
+          mtx(Mtx::create(exec, gko::dim<2>{2, 3}, 4, mat_bs,
+                          std::make_shared<matstr::load_balance<Mtx>>(2))),
+          mtx2(Mtx::create(exec, gko::dim<2>{2, 3}, 5, mat_bs,
+                           std::make_shared<matstr::classical<Mtx>>())),
+          mtx3_sorted(Mtx::create(exec, gko::dim<2>(3, 3), 7, mat_bs,
+                                  std::make_shared<matstr::classical<Mtx>>())),
+          mtx3_unsorted(Mtx::create(exec, gko::dim<2>(3, 3), 7, mat_bs,
+                                    std::make_shared<matstr::classical<Mtx>>()))
+    {
+        this->create_mtx(mtx.get());
+        this->create_mtx2(mtx2.get());
+        this->create_mtx3(mtx3_sorted.get(), mtx3_unsorted.get());
+    }
+
+    void create_mtx(Mtx *m)
+    {
+        value_type *v = m->get_values();
+        index_type *c = m->get_col_idxs();
+        index_type *r = m->get_row_ptrs();
+        auto *s = m->get_srow();
+        /*
+         * 1   3   2
+         * 0   5   0
+         */
+        r[0] = 0;
+        r[1] = 3;
+        r[2] = 4;
+        c[0] = 0;
+        c[1] = 1;
+        c[2] = 2;
+        c[3] = 1;
+        v[0] = 1.0;
+        v[1] = 3.0;
+        v[2] = 2.0;
+        v[3] = 5.0;
+        s[0] = 0;
+    }
+
+    void create_mtx2(Mtx *m)
+    {
+        value_type *v = m->get_values();
+        index_type *c = m->get_col_idxs();
+        index_type *r = m->get_row_ptrs();
+        // It keeps an explict zero
+        /*
+         *  1    3   2
+         * {0}   5   0
+         */
+        r[0] = 0;
+        r[1] = 3;
+        r[2] = 5;
+        c[0] = 0;
+        c[1] = 1;
+        c[2] = 2;
+        c[3] = 0;
+        c[4] = 1;
+        v[0] = 1.0;
+        v[1] = 3.0;
+        v[2] = 2.0;
+        v[3] = 0.0;
+        v[4] = 5.0;
+    }
+
+    void create_mtx3(Mtx *sorted, Mtx *unsorted)
+    {
+        auto vals_s = sorted->get_values();
+        auto cols_s = sorted->get_col_idxs();
+        auto rows_s = sorted->get_row_ptrs();
+        auto vals_u = unsorted->get_values();
+        auto cols_u = unsorted->get_col_idxs();
+        auto rows_u = unsorted->get_row_ptrs();
+        /* For both versions (sorted and unsorted), this matrix is stored:
+         * 0  2  1
+         * 3  1  8
+         * 2  0  3
+         * The unsorted matrix will have the (value, column) pair per row not
+         * sorted, which we still consider a valid FBCSR format.
+         */
+        rows_s[0] = 0;
+        rows_s[1] = 2;
+        rows_s[2] = 5;
+        rows_s[3] = 7;
+        rows_u[0] = 0;
+        rows_u[1] = 2;
+        rows_u[2] = 5;
+        rows_u[3] = 7;
+
+        vals_s[0] = 2.;
+        vals_s[1] = 1.;
+        vals_s[2] = 3.;
+        vals_s[3] = 1.;
+        vals_s[4] = 8.;
+        vals_s[5] = 2.;
+        vals_s[6] = 3.;
+        // Each row is stored rotated once to the left
+        vals_u[0] = 1.;
+        vals_u[1] = 2.;
+        vals_u[2] = 1.;
+        vals_u[3] = 8.;
+        vals_u[4] = 3.;
+        vals_u[5] = 3.;
+        vals_u[6] = 2.;
+
+        cols_s[0] = 1;
+        cols_s[1] = 2;
+        cols_s[2] = 0;
+        cols_s[3] = 1;
+        cols_s[4] = 2;
+        cols_s[5] = 0;
+        cols_s[6] = 2;
+        // The same applies for the columns
+        cols_u[0] = 2;
+        cols_u[1] = 1;
+        cols_u[2] = 1;
+        cols_u[3] = 2;
+        cols_u[4] = 0;
+        cols_u[5] = 2;
+        cols_u[6] = 0;
+    }
+
+    void assert_equal_to_mtx(const Coo *m)
+    {
+        auto v = m->get_const_values();
+        auto c = m->get_const_col_idxs();
+        auto r = m->get_const_row_idxs();
+
+        ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
+        ASSERT_EQ(m->get_num_stored_elements(), 4);
+        EXPECT_EQ(r[0], 0);
+        EXPECT_EQ(r[1], 0);
+        EXPECT_EQ(r[2], 0);
+        EXPECT_EQ(r[3], 1);
+        EXPECT_EQ(c[0], 0);
+        EXPECT_EQ(c[1], 1);
+        EXPECT_EQ(c[2], 2);
+        EXPECT_EQ(c[3], 1);
+        EXPECT_EQ(v[0], value_type{1.0});
+        EXPECT_EQ(v[1], value_type{3.0});
+        EXPECT_EQ(v[2], value_type{2.0});
+        EXPECT_EQ(v[3], value_type{5.0});
+    }
+
+    void assert_equal_to_mtx(const Sellp *m)
+    {
+        auto v = m->get_const_values();
+        auto c = m->get_const_col_idxs();
+        auto slice_sets = m->get_const_slice_sets();
+        auto slice_lengths = m->get_const_slice_lengths();
+        auto stride_factor = m->get_stride_factor();
+        auto slice_size = m->get_slice_size();
+
+        ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
+        ASSERT_EQ(stride_factor, 1);
+        ASSERT_EQ(slice_size, 64);
+        EXPECT_EQ(slice_sets[0], 0);
+        EXPECT_EQ(slice_lengths[0], 3);
+        EXPECT_EQ(c[0], 0);
+        EXPECT_EQ(c[1], 1);
+        EXPECT_EQ(c[64], 1);
+        EXPECT_EQ(c[65], 0);
+        EXPECT_EQ(c[128], 2);
+        EXPECT_EQ(c[129], 0);
+        EXPECT_EQ(v[0], value_type{1.0});
+        EXPECT_EQ(v[1], value_type{5.0});
+        EXPECT_EQ(v[64], value_type{3.0});
+        EXPECT_EQ(v[65], value_type{0.0});
+        EXPECT_EQ(v[128], value_type{2.0});
+        EXPECT_EQ(v[129], value_type{0.0});
+    }
+
+    void assert_equal_to_mtx(const SparsityCsr *m)
+    {
+        auto *c = m->get_const_col_idxs();
+        auto *r = m->get_const_row_ptrs();
+
+        ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
+        ASSERT_EQ(m->get_num_nonzeros(), 4);
+        EXPECT_EQ(r[0], 0);
+        EXPECT_EQ(r[1], 3);
+        EXPECT_EQ(r[2], 4);
+        EXPECT_EQ(c[0], 0);
+        EXPECT_EQ(c[1], 1);
+        EXPECT_EQ(c[2], 2);
+        EXPECT_EQ(c[3], 1);
+    }
+
+    void assert_equal_to_mtx(const Ell *m)
+    {
+        auto v = m->get_const_values();
+        auto c = m->get_const_col_idxs();
+
+        ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
+        ASSERT_EQ(m->get_num_stored_elements(), 6);
+        EXPECT_EQ(c[0], 0);
+        EXPECT_EQ(c[1], 1);
+        EXPECT_EQ(c[2], 1);
+        EXPECT_EQ(c[3], 0);
+        EXPECT_EQ(c[4], 2);
+        EXPECT_EQ(c[5], 0);
+        EXPECT_EQ(v[0], value_type{1.0});
+        EXPECT_EQ(v[1], value_type{5.0});
+        EXPECT_EQ(v[2], value_type{3.0});
+        EXPECT_EQ(v[3], value_type{0.0});
+        EXPECT_EQ(v[4], value_type{2.0});
+        EXPECT_EQ(v[5], value_type{0.0});
+    }
+
+    void assert_equal_to_mtx(const Hybrid *m)
+    {
+        auto v = m->get_const_coo_values();
+        auto c = m->get_const_coo_col_idxs();
+        auto r = m->get_const_coo_row_idxs();
+        auto n = m->get_ell_num_stored_elements_per_row();
+        auto p = m->get_ell_stride();
+
+        ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
+        ASSERT_EQ(m->get_ell_num_stored_elements(), 0);
+        ASSERT_EQ(m->get_coo_num_stored_elements(), 4);
+        EXPECT_EQ(n, 0);
+        EXPECT_EQ(p, 2);
+        EXPECT_EQ(r[0], 0);
+        EXPECT_EQ(r[1], 0);
+        EXPECT_EQ(r[2], 0);
+        EXPECT_EQ(r[3], 1);
+        EXPECT_EQ(c[0], 0);
+        EXPECT_EQ(c[1], 1);
+        EXPECT_EQ(c[2], 2);
+        EXPECT_EQ(c[3], 1);
+        EXPECT_EQ(v[0], value_type{1.0});
+        EXPECT_EQ(v[1], value_type{3.0});
+        EXPECT_EQ(v[2], value_type{2.0});
+        EXPECT_EQ(v[3], value_type{5.0});
+    }
+
+    void assert_equal_to_mtx2(const Hybrid *m)
+    {
+        auto v = m->get_const_coo_values();
+        auto c = m->get_const_coo_col_idxs();
+        auto r = m->get_const_coo_row_idxs();
+        auto n = m->get_ell_num_stored_elements_per_row();
+        auto p = m->get_ell_stride();
+        auto ell_v = m->get_const_ell_values();
+        auto ell_c = m->get_const_ell_col_idxs();
+
+        ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
+        // Test Coo values
+        ASSERT_EQ(m->get_coo_num_stored_elements(), 1);
+        EXPECT_EQ(r[0], 0);
+        EXPECT_EQ(c[0], 2);
+        EXPECT_EQ(v[0], value_type{2.0});
+        // Test Ell values
+        ASSERT_EQ(m->get_ell_num_stored_elements(), 4);
+        EXPECT_EQ(n, 2);
+        EXPECT_EQ(p, 2);
+        EXPECT_EQ(ell_v[0], value_type{1});
+        EXPECT_EQ(ell_v[1], value_type{0});
+        EXPECT_EQ(ell_v[2], value_type{3});
+        EXPECT_EQ(ell_v[3], value_type{5});
+        EXPECT_EQ(ell_c[0], 0);
+        EXPECT_EQ(ell_c[1], 0);
+        EXPECT_EQ(ell_c[2], 1);
+        EXPECT_EQ(ell_c[3], 1);
+    }
+
+    std::shared_ptr<const gko::ReferenceExecutor> exec;
+    std::unique_ptr<Mtx> mtx;
+    std::unique_ptr<Mtx> mtx2;
+    std::unique_ptr<Mtx> mtx3_sorted;
+    std::unique_ptr<Mtx> mtx3_unsorted;
+};
+
+TYPED_TEST_CASE(Fbcsr, gko::test::ValueIndexTypes);
+
+
+TYPED_TEST(Fbcsr, AppliesToDenseVector)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Vec = typename TestFixture::Vec;
+//    using T = typename TestFixture::value_type;
+//    auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
+//    auto y = Vec::create(this->exec, gko::dim<2>{2, 1});
+//
+//    this->mtx->apply(x.get(), y.get());
+//
+//    EXPECT_EQ(y->at(0), T{13.0});
+//    EXPECT_EQ(y->at(1), T{5.0});
+//}
+
+
+TYPED_TEST(Fbcsr, AppliesToDenseMatrix)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Vec = typename TestFixture::Vec;
+//    using T = typename TestFixture::value_type;
+//    auto x = gko::initialize<Vec>(
+//        {I<T>{2.0, 3.0}, I<T>{1.0, -1.5}, I<T>{4.0, 2.5}}, this->exec);
+//    auto y = Vec::create(this->exec, gko::dim<2>{2});
+//
+//    this->mtx->apply(x.get(), y.get());
+//
+//    EXPECT_EQ(y->at(0, 0), T{13.0});
+//    EXPECT_EQ(y->at(1, 0), T{5.0});
+//    EXPECT_EQ(y->at(0, 1), T{3.5});
+//    EXPECT_EQ(y->at(1, 1), T{-7.5});
+//}
+
+
+TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseVector)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Vec = typename TestFixture::Vec;
+//    using T = typename TestFixture::value_type;
+//    auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
+//    auto beta = gko::initialize<Vec>({2.0}, this->exec);
+//    auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
+//    auto y = gko::initialize<Vec>({1.0, 2.0}, this->exec);
+//
+//    this->mtx->apply(alpha.get(), x.get(), beta.get(), y.get());
+//
+//    EXPECT_EQ(y->at(0), T{-11.0});
+//    EXPECT_EQ(y->at(1), T{-1.0});
+//}
+
+
+TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseMatrix)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Vec = typename TestFixture::Vec;
+//    using T = typename TestFixture::value_type;
+//    auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
+//    auto beta = gko::initialize<Vec>({2.0}, this->exec);
+//    auto x = gko::initialize<Vec>(
+//        {I<T>{2.0, 3.0}, I<T>{1.0, -1.5}, I<T>{4.0, 2.5}}, this->exec);
+//    auto y =
+//        gko::initialize<Vec>({I<T>{1.0, 0.5}, I<T>{2.0, -1.5}}, this->exec);
+//
+//    this->mtx->apply(alpha.get(), x.get(), beta.get(), y.get());
+//
+//    EXPECT_EQ(y->at(0, 0), T{-11.0});
+//    EXPECT_EQ(y->at(1, 0), T{-1.0});
+//    EXPECT_EQ(y->at(0, 1), T{-2.5});
+//    EXPECT_EQ(y->at(1, 1), T{4.5});
+//}
+
+
+TYPED_TEST(Fbcsr, AppliesToFbcsrMatrix)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using T = typename TestFixture::value_type;
+//    this->mtx->apply(this->mtx3_unsorted.get(), this->mtx2.get());
+//
+//    ASSERT_EQ(this->mtx2->get_size(), gko::dim<2>(2, 3));
+//    ASSERT_EQ(this->mtx2->get_num_stored_elements(), 6);
+//    ASSERT_TRUE(this->mtx2->is_sorted_by_column_index());
+//    auto r = this->mtx2->get_const_row_ptrs();
+//    auto c = this->mtx2->get_const_col_idxs();
+//    auto v = this->mtx2->get_const_values();
+//    // 13  5 31
+//    // 15  5 40
+//    EXPECT_EQ(r[0], 0);
+//    EXPECT_EQ(r[1], 3);
+//    EXPECT_EQ(r[2], 6);
+//    EXPECT_EQ(c[0], 0);
+//    EXPECT_EQ(c[1], 1);
+//    EXPECT_EQ(c[2], 2);
+//    EXPECT_EQ(c[3], 0);
+//    EXPECT_EQ(c[4], 1);
+//    EXPECT_EQ(c[5], 2);
+//    EXPECT_EQ(v[0], T{13});
+//    EXPECT_EQ(v[1], T{5});
+//    EXPECT_EQ(v[2], T{31});
+//    EXPECT_EQ(v[3], T{15});
+//    EXPECT_EQ(v[4], T{5});
+//    EXPECT_EQ(v[5], T{40});
+//}
+
+
+TYPED_TEST(Fbcsr, AppliesLinearCombinationToFbcsrMatrix)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Vec = typename TestFixture::Vec;
+//    using T = typename TestFixture::value_type;
+//    auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
+//    auto beta = gko::initialize<Vec>({2.0}, this->exec);
+//
+//    this->mtx->apply(alpha.get(), this->mtx3_unsorted.get(), beta.get(),
+//                     this->mtx2.get());
+//
+//    ASSERT_EQ(this->mtx2->get_size(), gko::dim<2>(2, 3));
+//    ASSERT_EQ(this->mtx2->get_num_stored_elements(), 6);
+//    ASSERT_TRUE(this->mtx2->is_sorted_by_column_index());
+//    auto r = this->mtx2->get_const_row_ptrs();
+//    auto c = this->mtx2->get_const_col_idxs();
+//    auto v = this->mtx2->get_const_values();
+//    // -11 1 -27
+//    // -15 5 -40
+//    EXPECT_EQ(r[0], 0);
+//    EXPECT_EQ(r[1], 3);
+//    EXPECT_EQ(r[2], 6);
+//    EXPECT_EQ(c[0], 0);
+//    EXPECT_EQ(c[1], 1);
+//    EXPECT_EQ(c[2], 2);
+//    EXPECT_EQ(c[3], 0);
+//    EXPECT_EQ(c[4], 1);
+//    EXPECT_EQ(c[5], 2);
+//    EXPECT_EQ(v[0], T{-11});
+//    EXPECT_EQ(v[1], T{1});
+//    EXPECT_EQ(v[2], T{-27});
+//    EXPECT_EQ(v[3], T{-15});
+//    EXPECT_EQ(v[4], T{5});
+//    EXPECT_EQ(v[5], T{-40});
+//}
+
+
+TYPED_TEST(Fbcsr, AppliesLinearCombinationToIdentityMatrix)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using T = typename TestFixture::value_type;
+//    using Vec = typename TestFixture::Vec;
+//    using Mtx = typename TestFixture::Mtx;
+//    auto alpha = gko::initialize<Vec>({-3.0}, this->exec);
+//    auto beta = gko::initialize<Vec>({2.0}, this->exec);
+//    auto a = gko::initialize<Mtx>(
+//        {I<T>{2.0, 0.0, 3.0}, I<T>{0.0, 1.0, -1.5}, I<T>{0.0, -2.0, 0.0},
+//         I<T>{5.0, 0.0, 0.0}, I<T>{1.0, 0.0, 4.0}, I<T>{2.0, -2.0, 0.0},
+//         I<T>{0.0, 0.0, 0.0}},
+//        this->exec);
+//    auto b = gko::initialize<Mtx>(
+//        {I<T>{2.0, -2.0, 0.0}, I<T>{1.0, 0.0, 4.0}, I<T>{2.0, 0.0, 3.0},
+//         I<T>{0.0, 1.0, -1.5}, I<T>{1.0, 0.0, 0.0}, I<T>{0.0, 0.0, 0.0},
+//         I<T>{0.0, 0.0, 0.0}},
+//        this->exec);
+//    auto expect = gko::initialize<Mtx>(
+//        {I<T>{-2.0, -4.0, -9.0}, I<T>{2.0, -3.0, 12.5}, I<T>{4.0, 6.0, 6.0},
+//         I<T>{-15.0, 2.0, -3.0}, I<T>{-1.0, 0.0, -12.0}, I<T>{-6.0, 6.0, 0.0},
+//         I<T>{0.0, 0.0, 0.0}},
+//        this->exec);
+//    auto id = gko::matrix::Identity<T>::create(this->exec, a->get_size()[1]);
+//
+//    a->apply(gko::lend(alpha), gko::lend(id), gko::lend(beta), gko::lend(b));
+//
+//    GKO_ASSERT_MTX_NEAR(b, expect, r<T>::value);
+//    GKO_ASSERT_MTX_EQ_SPARSITY(b, expect);
+//    ASSERT_TRUE(b->is_sorted_by_column_index());
+//}
+
+
+TYPED_TEST(Fbcsr, ApplyFailsOnWrongInnerDimension)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Vec = typename TestFixture::Vec;
+//    auto x = Vec::create(this->exec, gko::dim<2>{2});
+//    auto y = Vec::create(this->exec, gko::dim<2>{2});
+//
+//    ASSERT_THROW(this->mtx->apply(x.get(), y.get()), gko::DimensionMismatch);
+//}
+
+
+TYPED_TEST(Fbcsr, ApplyFailsOnWrongNumberOfRows)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Vec = typename TestFixture::Vec;
+//    auto x = Vec::create(this->exec, gko::dim<2>{3, 2});
+//    auto y = Vec::create(this->exec, gko::dim<2>{3, 2});
+//
+//    ASSERT_THROW(this->mtx->apply(x.get(), y.get()), gko::DimensionMismatch);
+//}
+
+
+TYPED_TEST(Fbcsr, ApplyFailsOnWrongNumberOfCols)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Vec = typename TestFixture::Vec;
+//    auto x = Vec::create(this->exec, gko::dim<2>{3});
+//    auto y = Vec::create(this->exec, gko::dim<2>{2});
+//
+//    ASSERT_THROW(this->mtx->apply(x.get(), y.get()), gko::DimensionMismatch);
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsToPrecision)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using IndexType = typename TestFixture::index_type;
+//    using OtherType = typename gko::next_precision<ValueType>;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using OtherFbcsr = gko::matrix::Fbcsr<OtherType, IndexType>;
+//    auto tmp = OtherFbcsr::create(this->exec);
+//    auto res = Fbcsr::create(this->exec);
+//    // If OtherType is more precise: 0, otherwise r
+//    auto residual = r<OtherType>::value < r<ValueType>::value
+//                        ? gko::remove_complex<ValueType>{0}
+//                        : gko::remove_complex<ValueType>{r<OtherType>::value};
+//
+//    // use mtx2 as mtx's strategy would involve creating a CudaExecutor
+//    this->mtx2->convert_to(tmp.get());
+//    tmp->convert_to(res.get());
+//
+//    GKO_ASSERT_MTX_NEAR(this->mtx2, res, residual);
+//    ASSERT_EQ(typeid(*this->mtx2->get_strategy()),
+//              typeid(*res->get_strategy()));
+//}
+
+
+TYPED_TEST(Fbcsr, MovesToPrecision)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using IndexType = typename TestFixture::index_type;
+//    using OtherType = typename gko::next_precision<ValueType>;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using OtherFbcsr = gko::matrix::Fbcsr<OtherType, IndexType>;
+//    auto tmp = OtherFbcsr::create(this->exec);
+//    auto res = Fbcsr::create(this->exec);
+//    // If OtherType is more precise: 0, otherwise r
+//    auto residual = r<OtherType>::value < r<ValueType>::value
+//                        ? gko::remove_complex<ValueType>{0}
+//                        : gko::remove_complex<ValueType>{r<OtherType>::value};
+//
+//    // use mtx2 as mtx's strategy would involve creating a CudaExecutor
+//    this->mtx2->move_to(tmp.get());
+//    tmp->move_to(res.get());
+//
+//    GKO_ASSERT_MTX_NEAR(this->mtx2, res, residual);
+//    ASSERT_EQ(typeid(*this->mtx2->get_strategy()),
+//              typeid(*res->get_strategy()));
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsToDense)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Dense = typename TestFixture::Vec;
+//    auto dense_mtx = Dense::create(this->mtx->get_executor());
+//    auto dense_other = gko::initialize<Dense>(
+//        4, {{1.0, 3.0, 2.0}, {0.0, 5.0, 0.0}}, this->exec);
+//
+//    this->mtx->convert_to(dense_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dense_mtx, dense_other, 0.0);
+//}
+
+
+TYPED_TEST(Fbcsr, MovesToDense)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Dense = typename TestFixture::Vec;
+//    auto dense_mtx = Dense::create(this->mtx->get_executor());
+//    auto dense_other = gko::initialize<Dense>(
+//        4, {{1.0, 3.0, 2.0}, {0.0, 5.0, 0.0}}, this->exec);
+//
+//    this->mtx->move_to(dense_mtx.get());
+//
+//    GKO_ASSERT_MTX_NEAR(dense_mtx, dense_other, 0.0);
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsToCoo)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Coo = typename TestFixture::Coo;
+//    auto coo_mtx = Coo::create(this->mtx->get_executor());
+//
+//    this->mtx->convert_to(coo_mtx.get());
+//
+//    this->assert_equal_to_mtx(coo_mtx.get());
+//}
+
+
+TYPED_TEST(Fbcsr, MovesToCoo)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Coo = typename TestFixture::Coo;
+//    auto coo_mtx = Coo::create(this->mtx->get_executor());
+//
+//    this->mtx->move_to(coo_mtx.get());
+//
+//    this->assert_equal_to_mtx(coo_mtx.get());
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsToSellp)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Sellp = typename TestFixture::Sellp;
+//    auto sellp_mtx = Sellp::create(this->mtx->get_executor());
+//
+//    this->mtx->convert_to(sellp_mtx.get());
+//
+//    this->assert_equal_to_mtx(sellp_mtx.get());
+//}
+
+
+TYPED_TEST(Fbcsr, MovesToSellp)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Sellp = typename TestFixture::Sellp;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    auto sellp_mtx = Sellp::create(this->mtx->get_executor());
+//    auto fbcsr_ref = Fbcsr::create(this->mtx->get_executor());
+//
+//    fbcsr_ref->copy_from(this->mtx.get());
+//    fbcsr_ref->move_to(sellp_mtx.get());
+//
+//    this->assert_equal_to_mtx(sellp_mtx.get());
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsToSparsityFbcsr)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using SparsityFbcsr = typename TestFixture::SparsityFbcsr;
+//    auto sparsity_mtx = SparsityFbcsr::create(this->mtx->get_executor());
+//
+//    this->mtx->convert_to(sparsity_mtx.get());
+//
+//    this->assert_equal_to_mtx(sparsity_mtx.get());
+//}
+
+
+TYPED_TEST(Fbcsr, MovesToSparsityFbcsr)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using SparsityFbcsr = typename TestFixture::SparsityFbcsr;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    auto sparsity_mtx = SparsityFbcsr::create(this->mtx->get_executor());
+//    auto fbcsr_ref = Fbcsr::create(this->mtx->get_executor());
+//
+//    fbcsr_ref->copy_from(this->mtx.get());
+//    fbcsr_ref->move_to(sparsity_mtx.get());
+//
+//    this->assert_equal_to_mtx(sparsity_mtx.get());
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsToHybridAutomatically)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Hybrid = typename TestFixture::Hybrid;
+//    auto hybrid_mtx = Hybrid::create(this->mtx->get_executor());
+//
+//    this->mtx->convert_to(hybrid_mtx.get());
+//
+//    this->assert_equal_to_mtx(hybrid_mtx.get());
+//}
+
+
+TYPED_TEST(Fbcsr, MovesToHybridAutomatically)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Hybrid = typename TestFixture::Hybrid;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    auto hybrid_mtx = Hybrid::create(this->mtx->get_executor());
+//    auto fbcsr_ref = Fbcsr::create(this->mtx->get_executor());
+//
+//    fbcsr_ref->copy_from(this->mtx.get());
+//    fbcsr_ref->move_to(hybrid_mtx.get());
+//
+//    this->assert_equal_to_mtx(hybrid_mtx.get());
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsToHybridByColumn2)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Hybrid = typename TestFixture::Hybrid;
+//    auto hybrid_mtx =
+//        Hybrid::create(this->mtx2->get_executor(),
+//                       std::make_shared<typename Hybrid::column_limit>(2));
+//
+//    this->mtx2->convert_to(hybrid_mtx.get());
+//
+//    this->assert_equal_to_mtx2(hybrid_mtx.get());
+//}
+
+
+TYPED_TEST(Fbcsr, MovesToHybridByColumn2)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Hybrid = typename TestFixture::Hybrid;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    auto hybrid_mtx =
+//        Hybrid::create(this->mtx2->get_executor(),
+//                       std::make_shared<typename Hybrid::column_limit>(2));
+//    auto fbcsr_ref = Fbcsr::create(this->mtx2->get_executor());
+//
+//    fbcsr_ref->copy_from(this->mtx2.get());
+//    fbcsr_ref->move_to(hybrid_mtx.get());
+//
+//    this->assert_equal_to_mtx2(hybrid_mtx.get());
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsEmptyToPrecision)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using IndexType = typename TestFixture::index_type;
+//    using OtherType = typename gko::next_precision<ValueType>;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using OtherFbcsr = gko::matrix::Fbcsr<OtherType, IndexType>;
+//    auto empty = OtherFbcsr::create(this->exec);
+//    empty->get_row_ptrs()[0] = 0;
+//    auto res = Fbcsr::create(this->exec);
+//
+//    empty->convert_to(res.get());
+//
+//    ASSERT_EQ(res->get_num_stored_elements(), 0);
+//    ASSERT_EQ(*res->get_const_row_ptrs(), 0);
+//    ASSERT_FALSE(res->get_size());
+//}
+
+
+TYPED_TEST(Fbcsr, MovesEmptyToPrecision)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using IndexType = typename TestFixture::index_type;
+//    using OtherType = typename gko::next_precision<ValueType>;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using OtherFbcsr = gko::matrix::Fbcsr<OtherType, IndexType>;
+//    auto empty = OtherFbcsr::create(this->exec);
+//    empty->get_row_ptrs()[0] = 0;
+//    auto res = Fbcsr::create(this->exec);
+//
+//    empty->move_to(res.get());
+//
+//    ASSERT_EQ(res->get_num_stored_elements(), 0);
+//    ASSERT_EQ(*res->get_const_row_ptrs(), 0);
+//    ASSERT_FALSE(res->get_size());
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsEmptyToDense)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using Dense = gko::matrix::Dense<ValueType>;
+//    auto empty = Fbcsr::create(this->exec);
+//    auto res = Dense::create(this->exec);
+//
+//    empty->convert_to(res.get());
+//
+//    ASSERT_FALSE(res->get_size());
+//}
+
+
+TYPED_TEST(Fbcsr, MovesEmptyToDense)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using Dense = gko::matrix::Dense<ValueType>;
+//    auto empty = Fbcsr::create(this->exec);
+//    auto res = Dense::create(this->exec);
+//
+//    empty->move_to(res.get());
+//
+//    ASSERT_FALSE(res->get_size());
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsEmptyToCoo)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using IndexType = typename TestFixture::index_type;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using Coo = gko::matrix::Coo<ValueType, IndexType>;
+//    auto empty = Fbcsr::create(this->exec);
+//    auto res = Coo::create(this->exec);
+//
+//    empty->convert_to(res.get());
+//
+//    ASSERT_EQ(res->get_num_stored_elements(), 0);
+//    ASSERT_FALSE(res->get_size());
+//}
+
+
+TYPED_TEST(Fbcsr, MovesEmptyToCoo)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using IndexType = typename TestFixture::index_type;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using Coo = gko::matrix::Coo<ValueType, IndexType>;
+//    auto empty = Fbcsr::create(this->exec);
+//    auto res = Coo::create(this->exec);
+//
+//    empty->move_to(res.get());
+//
+//    ASSERT_EQ(res->get_num_stored_elements(), 0);
+//    ASSERT_FALSE(res->get_size());
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsEmptyToEll)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using IndexType = typename TestFixture::index_type;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using Ell = gko::matrix::Ell<ValueType, IndexType>;
+//    auto empty = Fbcsr::create(this->exec);
+//    auto res = Ell::create(this->exec);
+//
+//    empty->convert_to(res.get());
+//
+//    ASSERT_EQ(res->get_num_stored_elements(), 0);
+//    ASSERT_FALSE(res->get_size());
+//}
+
+
+TYPED_TEST(Fbcsr, MovesEmptyToEll)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using IndexType = typename TestFixture::index_type;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using Ell = gko::matrix::Ell<ValueType, IndexType>;
+//    auto empty = Fbcsr::create(this->exec);
+//    auto res = Ell::create(this->exec);
+//
+//    empty->move_to(res.get());
+//
+//    ASSERT_EQ(res->get_num_stored_elements(), 0);
+//    ASSERT_FALSE(res->get_size());
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsEmptyToSellp)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using IndexType = typename TestFixture::index_type;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using Sellp = gko::matrix::Sellp<ValueType, IndexType>;
+//    auto empty = Fbcsr::create(this->exec);
+//    auto res = Sellp::create(this->exec);
+//
+//    empty->convert_to(res.get());
+//
+//    ASSERT_EQ(res->get_num_stored_elements(), 0);
+//    ASSERT_EQ(*res->get_const_slice_sets(), 0);
+//    ASSERT_FALSE(res->get_size());
+//}
+
+
+TYPED_TEST(Fbcsr, MovesEmptyToSellp)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using IndexType = typename TestFixture::index_type;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using Sellp = gko::matrix::Sellp<ValueType, IndexType>;
+//    auto empty = Fbcsr::create(this->exec);
+//    auto res = Sellp::create(this->exec);
+//
+//    empty->move_to(res.get());
+//
+//    ASSERT_EQ(res->get_num_stored_elements(), 0);
+//    ASSERT_EQ(*res->get_const_slice_sets(), 0);
+//    ASSERT_FALSE(res->get_size());
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsEmptyToSparsityFbcsr)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using IndexType = typename TestFixture::index_type;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using SparsityFbcsr = gko::matrix::SparsityFbcsr<ValueType, IndexType>;
+//    auto empty = Fbcsr::create(this->exec);
+//    empty->get_row_ptrs()[0] = 0;
+//    auto res = SparsityFbcsr::create(this->exec);
+//
+//    empty->convert_to(res.get());
+//
+//    ASSERT_EQ(res->get_num_nonzeros(), 0);
+//    ASSERT_EQ(*res->get_const_row_ptrs(), 0);
+//}
+
+
+TYPED_TEST(Fbcsr, MovesEmptyToSparsityFbcsr)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using IndexType = typename TestFixture::index_type;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using SparsityFbcsr = gko::matrix::SparsityFbcsr<ValueType, IndexType>;
+//    auto empty = Fbcsr::create(this->exec);
+//    empty->get_row_ptrs()[0] = 0;
+//    auto res = SparsityFbcsr::create(this->exec);
+//
+//    empty->move_to(res.get());
+//
+//    ASSERT_EQ(res->get_num_nonzeros(), 0);
+//    ASSERT_EQ(*res->get_const_row_ptrs(), 0);
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsEmptyToHybrid)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using IndexType = typename TestFixture::index_type;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using Hybrid = gko::matrix::Hybrid<ValueType, IndexType>;
+//    auto empty = Fbcsr::create(this->exec);
+//    auto res = Hybrid::create(this->exec);
+//
+//    empty->convert_to(res.get());
+//
+//    ASSERT_EQ(res->get_num_stored_elements(), 0);
+//    ASSERT_FALSE(res->get_size());
+//}
+
+
+TYPED_TEST(Fbcsr, MovesEmptyToHybrid)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using ValueType = typename TestFixture::value_type;
+//    using IndexType = typename TestFixture::index_type;
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using Hybrid = gko::matrix::Hybrid<ValueType, IndexType>;
+//    auto empty = Fbcsr::create(this->exec);
+//    auto res = Hybrid::create(this->exec);
+//
+//    empty->move_to(res.get());
+//
+//    ASSERT_EQ(res->get_num_stored_elements(), 0);
+//    ASSERT_FALSE(res->get_size());
+//}
+
+
+TYPED_TEST(Fbcsr, CalculatesNonzerosPerRow)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    gko::Array<gko::size_type> row_nnz(this->exec, this->mtx->get_size()[0]);
+//
+//    gko::kernels::reference::fbcsr::calculate_nonzeros_per_row(
+//        this->exec, this->mtx.get(), &row_nnz);
+//
+//    auto row_nnz_val = row_nnz.get_data();
+//    ASSERT_EQ(row_nnz_val[0], 3);
+//    ASSERT_EQ(row_nnz_val[1], 1);
+//}
+
+
+TYPED_TEST(Fbcsr, CalculatesTotalCols)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    gko::size_type total_cols;
+//    gko::size_type stride_factor = gko::matrix::default_stride_factor;
+//    gko::size_type slice_size = gko::matrix::default_slice_size;
+//
+//    gko::kernels::reference::fbcsr::calculate_total_cols(
+//        this->exec, this->mtx.get(), &total_cols, stride_factor, slice_size);
+//
+//    ASSERT_EQ(total_cols, 3);
+//}
+
+
+TYPED_TEST(Fbcsr, ConvertsToEll)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Ell = typename TestFixture::Ell;
+//    using Dense = typename TestFixture::Vec;
+//    auto ell_mtx = Ell::create(this->mtx->get_executor());
+//    auto dense_mtx = Dense::create(this->mtx->get_executor());
+//    auto ref_dense_mtx = Dense::create(this->mtx->get_executor());
+//
+//    this->mtx->convert_to(ell_mtx.get());
+//
+//    this->assert_equal_to_mtx(ell_mtx.get());
+//}
+
+
+TYPED_TEST(Fbcsr, MovesToEll)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Ell = typename TestFixture::Ell;
+//    using Dense = typename TestFixture::Vec;
+//    auto ell_mtx = Ell::create(this->mtx->get_executor());
+//    auto dense_mtx = Dense::create(this->mtx->get_executor());
+//    auto ref_dense_mtx = Dense::create(this->mtx->get_executor());
+//
+//    this->mtx->move_to(ell_mtx.get());
+//
+//    this->assert_equal_to_mtx(ell_mtx.get());
+//}
+
+
+TYPED_TEST(Fbcsr, SquareMtxIsTransposable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Fbcsr = typename TestFixture::Mtx;
+//    // clang-format off
+//    auto mtx2 = gko::initialize<Fbcsr>(
+//                {{1.0, 3.0, 2.0},
+//                 {0.0, 5.0, 0.0},
+//                 {0.0, 1.5, 2.0}}, this->exec);
+//    // clang-format on
+//
+//    auto trans = mtx2->transpose();
+//    auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
+//
+//    // clang-format off
+//    GKO_ASSERT_MTX_NEAR(trans_as_fbcsr,
+//                    l({{1.0, 0.0, 0.0},
+//                       {3.0, 5.0, 1.5},
+//                       {2.0, 0.0, 2.0}}), 0.0);
+//    // clang-format on
+//}
+
+
+TYPED_TEST(Fbcsr, NonSquareMtxIsTransposable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Fbcsr = typename TestFixture::Mtx;
+//    auto trans = this->mtx->transpose();
+//    auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
+//
+//    // clang-format off
+//    GKO_ASSERT_MTX_NEAR(trans_as_fbcsr,
+//                    l({{1.0, 0.0},
+//                       {3.0, 5.0},
+//                       {2.0, 0.0}}), 0.0);
+//    // clang-format on
+//}
+
+
+TYPED_TEST(Fbcsr, SquareMatrixIsRowPermutable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using index_type = typename TestFixture::index_type;
+//    // clang-format off
+//    auto p_mtx = gko::initialize<Fbcsr>({{1.0, 3.0, 2.0},
+//                                       {0.0, 5.0, 0.0},
+//                                       {0.0, 1.5, 2.0}}, this->exec);
+//    // clang-format on
+//    gko::Array<index_type> permute_idxs{this->exec, {1, 2, 0}};
+//
+//    auto row_permute = p_mtx->row_permute(&permute_idxs);
+//
+//    auto row_permute_fbcsr = static_cast<Fbcsr *>(row_permute.get());
+//    // clang-format off
+//    GKO_ASSERT_MTX_NEAR(row_permute_fbcsr,
+//                        l({{0.0, 5.0, 0.0},
+//                           {0.0, 1.5, 2.0},
+//                           {1.0, 3.0, 2.0}}),
+//                        0.0);
+//    // clang-format on
+//}
+
+
+TYPED_TEST(Fbcsr, NonSquareMatrixIsRowPermutable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using index_type = typename TestFixture::index_type;
+//    // clang-format off
+//    auto p_mtx = gko::initialize<Fbcsr>({{1.0, 3.0, 2.0},
+//                                       {0.0, 5.0, 0.0}}, this->exec);
+//    // clang-format on
+//    gko::Array<index_type> permute_idxs{this->exec, {1, 0}};
+//
+//    auto row_permute = p_mtx->row_permute(&permute_idxs);
+//
+//    auto row_permute_fbcsr = static_cast<Fbcsr *>(row_permute.get());
+//    // clang-format off
+//    GKO_ASSERT_MTX_NEAR(row_permute_fbcsr,
+//                        l({{0.0, 5.0, 0.0},
+//                           {1.0, 3.0, 2.0}}),
+//                        0.0);
+//    // clang-format on
+//}
+
+
+TYPED_TEST(Fbcsr, SquareMatrixIsColPermutable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using index_type = typename TestFixture::index_type;
+//    // clang-format off
+//    auto p_mtx = gko::initialize<Fbcsr>({{1.0, 3.0, 2.0},
+//                                       {0.0, 5.0, 0.0},
+//                                       {0.0, 1.5, 2.0}}, this->exec);
+//    // clang-format on
+//    gko::Array<index_type> permute_idxs{this->exec, {1, 2, 0}};
+//
+//    auto c_permute = p_mtx->column_permute(&permute_idxs);
+//
+//    auto c_permute_fbcsr = static_cast<Fbcsr *>(c_permute.get());
+//    // clang-format off
+//    GKO_ASSERT_MTX_NEAR(c_permute_fbcsr,
+//                        l({{3.0, 2.0, 1.0},
+//                           {5.0, 0.0, 0.0},
+//                           {1.5, 2.0, 0.0}}),
+//                        0.0);
+//    // clang-format on
+//}
+
+
+TYPED_TEST(Fbcsr, NonSquareMatrixIsColPermutable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using index_type = typename TestFixture::index_type;
+//    // clang-format off
+//    auto p_mtx = gko::initialize<Fbcsr>({{1.0, 0.0, 2.0},
+//                                       {0.0, 5.0, 0.0}}, this->exec);
+//    // clang-format on
+//    gko::Array<index_type> permute_idxs{this->exec, {1, 2, 0}};
+//
+//    auto c_permute = p_mtx->column_permute(&permute_idxs);
+//
+//    auto c_permute_fbcsr = static_cast<Fbcsr *>(c_permute.get());
+//    // clang-format off
+//    GKO_ASSERT_MTX_NEAR(c_permute_fbcsr,
+//                        l({{0.0, 2.0, 1.0},
+//                           {5.0, 0.0, 0.0}}),
+//                        0.0);
+//    // clang-format on
+//}
+
+
+TYPED_TEST(Fbcsr, SquareMatrixIsInverseRowPermutable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using index_type = typename TestFixture::index_type;
+//    // clang-format off
+//    auto inverse_p_mtx = gko::initialize<Fbcsr>({{1.0, 3.0, 2.0},
+//                                               {0.0, 5.0, 0.0},
+//                                               {0.0, 1.5, 2.0}}, this->exec);
+//    // clang-format on
+//    gko::Array<index_type> inverse_permute_idxs{this->exec, {1, 2, 0}};
+//
+//    auto inverse_row_permute =
+//        inverse_p_mtx->inverse_row_permute(&inverse_permute_idxs);
+//
+//    auto inverse_row_permute_fbcsr =
+//        static_cast<Fbcsr *>(inverse_row_permute.get());
+//    // clang-format off
+//    GKO_ASSERT_MTX_NEAR(inverse_row_permute_fbcsr,
+//                        l({{0.0, 1.5, 2.0},
+//                           {1.0, 3.0, 2.0},
+//                           {0.0, 5.0, 0.0}}),
+//                        0.0);
+//    // clang-format on
+//}
+
+
+TYPED_TEST(Fbcsr, NonSquareMatrixIsInverseRowPermutable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using index_type = typename TestFixture::index_type;
+//    // clang-format off
+//    auto inverse_p_mtx = gko::initialize<Fbcsr>({{1.0, 3.0, 2.0},
+//                                               {0.0, 5.0, 0.0}}, this->exec);
+//    // clang-format on
+//    gko::Array<index_type> inverse_permute_idxs{this->exec, {1, 0}};
+//
+//    auto inverse_row_permute =
+//        inverse_p_mtx->inverse_row_permute(&inverse_permute_idxs);
+//
+//    auto inverse_row_permute_fbcsr =
+//        static_cast<Fbcsr *>(inverse_row_permute.get());
+//    // clang-format off
+//    GKO_ASSERT_MTX_NEAR(inverse_row_permute_fbcsr,
+//                        l({{0.0, 5.0, 0.0},
+//                           {1.0, 3.0, 2.0}}),
+//                        0.0);
+//    // clang-format on
+//}
+
+
+TYPED_TEST(Fbcsr, SquareMatrixIsInverseColPermutable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using index_type = typename TestFixture::index_type;
+//    // clang-format off
+//    auto inverse_p_mtx = gko::initialize<Fbcsr>({{1.0, 3.0, 2.0},
+//                                               {0.0, 5.0, 0.0},
+//                                               {0.0, 1.5, 2.0}}, this->exec);
+//    // clang-format on
+//    gko::Array<index_type> inverse_permute_idxs{this->exec, {1, 2, 0}};
+//
+//    auto inverse_c_permute =
+//        inverse_p_mtx->inverse_column_permute(&inverse_permute_idxs);
+//
+//    auto inverse_c_permute_fbcsr = static_cast<Fbcsr
+//    *>(inverse_c_permute.get());
+//    // clang-format off
+//    GKO_ASSERT_MTX_NEAR(inverse_c_permute_fbcsr,
+//                        l({{2.0, 1.0, 3.0},
+//                           {0.0, 0.0, 5.0},
+//                           {2.0, 0.0, 1.5}}),
+//                        0.0);
+//    // clang-format on
+//}
+
+
+TYPED_TEST(Fbcsr, NonSquareMatrixIsInverseColPermutable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using index_type = typename TestFixture::index_type;
+//    // clang-format off
+//    auto inverse_p_mtx = gko::initialize<Fbcsr>({{1.0, 3.0, 2.0},
+//                                              {0.0, 5.0, 0.0}}, this->exec);
+//    // clang-format on
+//    gko::Array<index_type> inverse_permute_idxs{this->exec, {1, 2, 0}};
+//
+//    auto inverse_c_permute =
+//        inverse_p_mtx->inverse_column_permute(&inverse_permute_idxs);
+//
+//    auto inverse_c_permute_fbcsr = static_cast<Fbcsr
+//    *>(inverse_c_permute.get());
+//    // clang-format off
+//    GKO_ASSERT_MTX_NEAR(inverse_c_permute_fbcsr,
+//                        l({{2.0, 1.0, 3.0},
+//                           {0.0, 0.0, 5.0}}),
+//                        0.0);
+//    // clang-format on
+//}
+
+
+TYPED_TEST(Fbcsr, RecognizeSortedMatrix)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    ASSERT_TRUE(this->mtx->is_sorted_by_column_index());
+//    ASSERT_TRUE(this->mtx2->is_sorted_by_column_index());
+//    ASSERT_TRUE(this->mtx3_sorted->is_sorted_by_column_index());
+//}
+
+
+TYPED_TEST(Fbcsr, RecognizeUnsortedMatrix)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    ASSERT_FALSE(this->mtx3_unsorted->is_sorted_by_column_index());
+//}
+
+
+TYPED_TEST(Fbcsr, SortSortedMatrix)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto matrix = this->mtx3_sorted->clone();
+//
+//    matrix->sort_by_column_index();
+//
+//    GKO_ASSERT_MTX_NEAR(matrix, this->mtx3_sorted, 0.0);
+//}
+
+
+TYPED_TEST(Fbcsr, SortUnsortedMatrix)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto matrix = this->mtx3_unsorted->clone();
+//
+//    matrix->sort_by_column_index();
+//
+//    GKO_ASSERT_MTX_NEAR(matrix, this->mtx3_sorted, 0.0);
+//}
+
+
+TYPED_TEST(Fbcsr, ExtractsDiagonal)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using T = typename TestFixture::value_type;
+//    auto matrix = this->mtx3_unsorted->clone();
+//    auto diag = matrix->extract_diagonal();
+//
+//    ASSERT_EQ(diag->get_size()[0], 3);
+//    ASSERT_EQ(diag->get_size()[1], 3);
+//    ASSERT_EQ(diag->get_values()[0], T{0.});
+//    ASSERT_EQ(diag->get_values()[1], T{1.});
+//    ASSERT_EQ(diag->get_values()[2], T{3.});
+//}
+
+
+TYPED_TEST(Fbcsr, InplaceAbsolute)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    auto mtx = gko::initialize<Mtx>(
+//        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
+//
+//    mtx->compute_absolute_inplace();
+//
+//    GKO_ASSERT_MTX_NEAR(
+//        mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+//}
+
+
+TYPED_TEST(Fbcsr, OutplaceAbsolute)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    auto mtx = gko::initialize<Mtx>(
+//        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
+//
+//    auto abs_mtx = mtx->compute_absolute();
+//
+//    GKO_ASSERT_MTX_NEAR(
+//        abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
+//    ASSERT_EQ(mtx->get_strategy()->get_name(),
+//              abs_mtx->get_strategy()->get_name());
+//}
+
+
+template <typename ValueIndexType>
+class FbcsrComplex : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using Mtx = gko::matrix::Fbcsr<value_type, index_type>;
+};
+
+TYPED_TEST_CASE(FbcsrComplex, gko::test::ComplexValueIndexTypes);
+
+
+TYPED_TEST(FbcsrComplex, MtxIsConjugateTransposable)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Fbcsr = typename TestFixture::Mtx;
+//    using T = typename TestFixture::value_type;
+//
+//    auto exec = gko::ReferenceExecutor::create();
+//    // clang-format off
+//    auto mtx2 = gko::initialize<Fbcsr>(
+//        {{T{1.0, 2.0}, T{3.0, 0.0}, T{2.0, 0.0}},
+//         {T{0.0, 0.0}, T{5.0, - 3.5}, T{0.0,0.0}},
+//         {T{0.0, 0.0}, T{0.0, 1.5}, T{2.0,0.0}}}, exec);
+//    // clang-format on
+//
+//    auto trans = mtx2->conj_transpose();
+//    auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
+//
+//    // clang-format off
+//    GKO_ASSERT_MTX_NEAR(trans_as_fbcsr,
+//                        l({{T{1.0, - 2.0}, T{0.0, 0.0}, T{0.0, 0.0}},
+//                           {T{3.0, 0.0}, T{5.0, 3.5}, T{0.0, - 1.5}},
+//                           {T{2.0, 0.0}, T{0.0, 0.0}, T{2.0 + 0.0}}}), 0.0);
+//    // clang-format on
+//}
+
+
+TYPED_TEST(FbcsrComplex, InplaceAbsolute)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using T = typename TestFixture::value_type;
+//    using index_type = typename TestFixture::index_type;
+//    auto exec = gko::ReferenceExecutor::create();
+//    // clang-format off
+//    auto mtx = gko::initialize<Mtx>(
+//        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+//         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+//         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+//    // clang-format on
+//
+//    mtx->compute_absolute_inplace();
+//
+//    GKO_ASSERT_MTX_NEAR(
+//        mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+//}
+
+
+TYPED_TEST(FbcsrComplex, OutplaceAbsolute)
+GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    using Mtx = typename TestFixture::Mtx;
+//    using T = typename TestFixture::value_type;
+//    using index_type = typename TestFixture::index_type;
+//    auto exec = gko::ReferenceExecutor::create();
+//    // clang-format off
+//    auto mtx = gko::initialize<Mtx>(
+//        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+//         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+//         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
+//    // clang-format on
+//
+//    auto abs_mtx = mtx->compute_absolute();
+//
+//    GKO_ASSERT_MTX_NEAR(
+//        abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+//    ASSERT_EQ(mtx->get_strategy()->get_name(),
+//              abs_mtx->get_strategy()->get_name());
+//}
+
+
+}  // namespace

From 0efe9d7c4db2afb4c475b92150076d6692abb83b Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Fri, 6 Nov 2020 16:19:38 +0100
Subject: [PATCH 0530/1008] [tests fail] fixed cuda build issue and enabled
 DPCPP kernels (unimplemented) for fbcsr

---
 core/matrix/fbcsr_kernels.hpp      | 9 +++++++++
 cuda/test/matrix/fbcsr_kernels.cpp | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/core/matrix/fbcsr_kernels.hpp b/core/matrix/fbcsr_kernels.hpp
index 715bb1192ce..a57b2af3c82 100644
--- a/core/matrix/fbcsr_kernels.hpp
+++ b/core/matrix/fbcsr_kernels.hpp
@@ -265,6 +265,15 @@ GKO_DECLARE_ALL_AS_TEMPLATES;
 }  // namespace hip
 
 
+namespace dpcpp {
+namespace fbcsr {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace fbcsr
+}  // namespace dpcpp
+
+
 #undef GKO_DECLARE_ALL_AS_TEMPLATES
 
 
diff --git a/cuda/test/matrix/fbcsr_kernels.cpp b/cuda/test/matrix/fbcsr_kernels.cpp
index 043412c6402..72bc9c149f4 100644
--- a/cuda/test/matrix/fbcsr_kernels.cpp
+++ b/cuda/test/matrix/fbcsr_kernels.cpp
@@ -48,7 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/hybrid.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
 #include <ginkgo/core/matrix/sellp.hpp>
-#include <ginkgo/core/matrix/sparsity_fbcsr.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
 #include "core/matrix/fbcsr_kernels.hpp"

From 850a43d05dfe4ce5b56a8b3f2413807b4bf3863b Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Tue, 10 Nov 2020 17:31:56 +0100
Subject: [PATCH 0531/1008] moved fbcsr test matrix generation to separate
 class

---
 core/components/fixed_block.hpp      |  16 +-
 core/matrix/fbcsr.cpp                |  21 ++
 core/test/matrix/CMakeLists.txt      |   9 +-
 core/test/matrix/fbcsr.cpp           | 230 +++++++++------------
 core/test/matrix/fbcsr_sample.cpp    | 286 +++++++++++++++++++++++++++
 core/test/matrix/fbcsr_sample.hpp    |  83 ++++++++
 include/ginkgo/core/matrix/fbcsr.hpp |   8 +
 7 files changed, 500 insertions(+), 153 deletions(-)
 create mode 100644 core/test/matrix/fbcsr_sample.cpp
 create mode 100644 core/test/matrix/fbcsr_sample.hpp

diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
index 526cdb25909..f2ff770cafb 100644
--- a/core/components/fixed_block.hpp
+++ b/core/components/fixed_block.hpp
@@ -66,7 +66,7 @@ class BlockReadError : public Error {
 };
 
 template <typename IndexType>
-int getNumFixedBlocks(const int block_size, const IndexType size)
+IndexType getNumFixedBlocks(const int block_size, const IndexType size)
 {
     if (size % block_size != 0)
         throw BlockSizeError<IndexType>(__FILE__, __LINE__, block_size, size);
@@ -113,14 +113,6 @@ class FixedBlock final {
 };
 
 
-/// Two-dimensional square block
-// template <typename ValueType, int size>
-// using FixedBlock = FixedRectangularBlock<ValueType, size, size>;
-
-/// Fixed-size column vector
-// template <typename ValueType, int size>
-// using FixedSegment = FixedRectangularBlock<ValueType, size, 1>;
-
 /// A lightweight dynamic block type for the host space
 template <typename ValueType>
 class DenseBlock final {
@@ -180,7 +172,11 @@ class DenseBlock final {
     value_type *vals_;
 };
 
-/// A view into a an array of dense of dense blocks of some runtime-defined size
+/// A view into a an array of dense blocks of some runtime-defined size
+/** Note that accessing BSR values using this type of view abstracts away the
+ * storage layout within the individual blocks, as long as all blocks use the
+ * same layout. For now, row-major blocks are assumed.
+ */
 template <typename ValueType, typename IndexType = int32>
 class DenseBlocksView final {
 public:
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index e362b3b466f..1ef12627602 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -244,6 +244,27 @@ void Fbcsr<ValueType, IndexType>::move_to(Dense<ValueType> *result)
 //}
 
 
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::convert_to(
+    Csr<ValueType, IndexType> *result) const GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    auto exec = this->get_executor();
+//    auto tmp = Dense<ValueType>::create(exec, this->get_size());
+//    exec->run(fbcsr::make_convert_to_csr(this, tmp.get()));
+//    tmp->move_to(result);
+//}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::move_to(Csr<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+//{
+// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+//    this->convert_to(result);
+//}
+
+
 // template <typename ValueType, typename IndexType>
 // void Fbcsr<ValueType, IndexType>::convert_to(
 //     Hybrid<ValueType, IndexType> *result) const
diff --git a/core/test/matrix/CMakeLists.txt b/core/test/matrix/CMakeLists.txt
index 64b3b3ed593..22a8d0828c6 100644
--- a/core/test/matrix/CMakeLists.txt
+++ b/core/test/matrix/CMakeLists.txt
@@ -5,7 +5,14 @@ ginkgo_create_test(csr_builder)
 ginkgo_create_test(dense)
 ginkgo_create_test(diagonal)
 ginkgo_create_test(ell)
-ginkgo_create_test(fbcsr)
+add_library(test_fbcsr_sample fbcsr_sample.cpp)
+target_include_directories(test_fbcsr_sample
+  PRIVATE
+  "$<BUILD_INTERFACE:${Ginkgo_SOURCE_DIR}/include>"
+  "$<BUILD_INTERFACE:${Ginkgo_SOURCE_DIR}>"
+  "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}/include>"
+  )
+ginkgo_create_test(fbcsr test_fbcsr_sample)
 ginkgo_create_test(fbcsr_builder)
 ginkgo_create_test(hybrid)
 ginkgo_create_test(identity)
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index dd456ef7316..893e12b0df6 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/components/fixed_block.hpp"
+#include "core/test/matrix/fbcsr_sample.hpp"
 #include "core/test/utils.hpp"
 
 
@@ -60,63 +61,26 @@ class Fbcsr : public ::testing::Test {
 
     Fbcsr()
         : exec(gko::ReferenceExecutor::create()),
-          mtx(Mtx::create(exec, gko::dim<2>{6, 12}, 36, 3,
-                          std::make_shared<matstr::classical<Mtx>>()))
+          fbsample(
+              std::static_pointer_cast<const gko::ReferenceExecutor>(exec)),
+          mtx(fbsample.generate_fbcsr())
     {
-        const int bs = 3;
-        value_type *const v = mtx->get_values();
-        index_type *const c = mtx->get_col_idxs();
-        index_type *const r = mtx->get_row_ptrs();
-        index_type *const s = mtx->get_srow();
-        r[0] = 0;
-        r[1] = 2;
-        r[2] = 4;
-        c[0] = 1;
-        c[1] = 3;
-        c[2] = 0;
-        c[3] = 2;
-
-        gko::blockutils::DenseBlocksView<value_type, index_type> vals(v, bs,
-                                                                      bs);
-
-        if (mtx->get_size()[0] % bs != 0)
-            throw gko::BadDimension(__FILE__, __LINE__, __func__, "test fbcsr",
-                                    mtx->get_size()[0], mtx->get_size()[1],
-                                    "block size does not divide the size!");
-
-        for (index_type ibrow = 0; ibrow < mtx->get_size()[0] / bs; ibrow++) {
-            const index_type *const browptr = mtx->get_row_ptrs();
-            for (index_type inz = browptr[ibrow]; inz < browptr[ibrow + 1];
-                 inz++) {
-                const index_type bcolind = mtx->get_col_idxs()[inz];
-                const value_type base = (ibrow + 1) * (bcolind + 1);
-                for (int ival = 0; ival < bs; ival++)
-                    for (int jval = 0; jval < bs; jval++)
-                        vals(inz, ival, jval) =
-                            base + static_cast<gko::remove_complex<value_type>>(
-                                       ival * bs + jval);
-            }
-        }
-
-        // Some of the entries are set to zero
-        vals(0, 2, 0) = gko::zero<value_type>();
-        vals(0, 2, 2) = gko::zero<value_type>();
-        vals(3, 0, 0) = gko::zero<value_type>();
-
-        for (index_type is = 0; is < mtx->get_num_srow_elements(); is++)
-            s[is] = 0;
-
         // backup for move tests
+        const value_type *const v = mtx->get_values();
+        const index_type *const c = mtx->get_col_idxs();
+        const index_type *const r = mtx->get_row_ptrs();
         orig_size = mtx->get_size();
-        orig_rowptrs.resize(3);
-        orig_colinds.resize(4);
-        orig_vals.resize(36);
-        for (index_type i = 0; i < 3; i++) orig_rowptrs[i] = r[i];
-        for (index_type i = 0; i < 4; i++) orig_colinds[i] = c[i];
-        for (index_type i = 0; i < 36; i++) orig_vals[i] = v[i];
+        orig_rowptrs.resize(fbsample.nbrows + 1);
+        orig_colinds.resize(fbsample.nbnz);
+        orig_vals.resize(fbsample.nnz);
+        for (index_type i = 0; i < fbsample.nbrows + 1; i++)
+            orig_rowptrs[i] = r[i];
+        for (index_type i = 0; i < fbsample.nbnz; i++) orig_colinds[i] = c[i];
+        for (index_type i = 0; i < fbsample.nnz; i++) orig_vals[i] = v[i];
     }
 
     std::shared_ptr<const gko::Executor> exec;
+    const gko::testing::FbcsrSample<value_type, index_type> fbsample;
     std::unique_ptr<Mtx> mtx;
 
     gko::dim<2> orig_size;
@@ -156,8 +120,6 @@ class Fbcsr : public ::testing::Test {
         }
 
         ASSERT_EQ(m->get_num_srow_elements(), 0);
-        // for(index_type is = 0; is < mtx->get_num_srow_elements(); is++)
-        //     ASSERT_EQ(s[is], 0);
     }
 
     void assert_empty(const Mtx *m)
@@ -175,6 +137,60 @@ class Fbcsr : public ::testing::Test {
 TYPED_TEST_CASE(Fbcsr, gko::test::ValueIndexTypes);
 
 
+TYPED_TEST(Fbcsr, SampleGeneratorIsCorrect)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+
+    std::unique_ptr<const Mtx> fbmtx = this->fbsample.generate_fbcsr();
+    std::unique_ptr<const Csr> csmtx = this->fbsample.generate_csr();
+    std::unique_ptr<const Dense> dmtx = this->fbsample.generate_dense();
+    const int bs = this->fbsample.bs;
+    ASSERT_EQ(bs, fbmtx->get_block_size());
+
+    gko::blockutils::DenseBlocksView<const value_type, index_type> fbvals(
+        fbmtx->get_const_values(), bs, bs);
+
+    for (index_type ibrow = 0; ibrow < this->fbsample.nbrows; ibrow++) {
+        const index_type *const browptr = fbmtx->get_row_ptrs();
+        const index_type numblocksbrow = browptr[ibrow + 1] - browptr[ibrow];
+        for (index_type irow = ibrow * bs; irow < ibrow * bs + bs; irow++) {
+            const index_type rowstart =
+                browptr[ibrow] * bs * bs +
+                (irow - ibrow * bs) * numblocksbrow * bs;
+            ASSERT_EQ(csmtx->get_const_row_ptrs()[irow], rowstart);
+        }
+
+        const index_type *const bcolinds = fbmtx->get_col_idxs();
+
+        for (index_type ibnz = browptr[ibrow]; ibnz < browptr[ibrow + 1];
+             ibnz++) {
+            const index_type bcol = bcolinds[ibnz];
+            const index_type blkoffset_frombrowstart = ibnz - browptr[ibrow];
+
+            for (int ib = 0; ib < bs; ib++) {
+                const index_type row = ibrow * bs + ib;
+                const index_type inz_rowstart =
+                    csmtx->get_const_row_ptrs()[row] +
+                    blkoffset_frombrowstart * bs;
+
+                for (int jb = 0; jb < bs; jb++) {
+                    const index_type col = bcol * bs + jb;
+                    const index_type inz = inz_rowstart + jb;
+                    ASSERT_EQ(col, csmtx->get_const_col_idxs()[inz]);
+                    ASSERT_EQ(fbvals(ibnz, ib, jb),
+                              csmtx->get_const_values()[inz]);
+                    ASSERT_EQ(fbvals(ibnz, ib, jb), dmtx->at(row, col));
+                }
+            }
+        }
+    }
+}
+
+
 TYPED_TEST(Fbcsr, KnowsItsSize)
 {
     ASSERT_EQ(this->mtx->get_size(), gko::dim<2>(6, 12));
@@ -203,35 +219,16 @@ TYPED_TEST(Fbcsr, CanBeCreatedFromExistingData)
     using Mtx = typename TestFixture::Mtx;
     using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
+    using size_type = gko::size_type;
 
-    constexpr int bs = 3;
-    constexpr index_type nbrows = 2;
-    constexpr index_type nbcols = 4;
-    constexpr index_type bnnz = 4;
-    value_type values[bnnz * bs * bs];
-    index_type col_idxs[] = {1, 3, 0, 2};
-    index_type row_ptrs[] = {0, 2, 4};
-
-    gko::blockutils::DenseBlocksView<value_type, index_type> vals(values, bs,
-                                                                  bs);
-
-    for (index_type ibrow = 0; ibrow < nbrows; ibrow++) {
-        for (index_type inz = row_ptrs[ibrow]; inz < row_ptrs[ibrow + 1];
-             inz++) {
-            const index_type bcolind = col_idxs[inz];
-            const value_type base = (ibrow + 1) * (bcolind + 1);
-            for (int ival = 0; ival < bs; ival++)
-                for (int jval = 0; jval < bs; jval++)
-                    vals(inz, ival, jval) =
-                        base + static_cast<gko::remove_complex<value_type>>(
-                                   ival * bs + jval);
-        }
-    }
-
-    // Some of the entries are set to zero
-    vals(0, 2, 0) = gko::zero<value_type>();
-    vals(0, 2, 2) = gko::zero<value_type>();
-    vals(3, 0, 0) = gko::zero<value_type>();
+    const int bs = this->fbsample.bs;
+    const size_type nbrows = this->fbsample.nbrows;
+    const size_type nbcols = this->fbsample.nbcols;
+    const size_type bnnz = this->fbsample.nbnz;
+    std::unique_ptr<Mtx> refmat = this->fbsample.generate_fbcsr();
+    value_type *const values = refmat->get_values();
+    index_type *const col_idxs = refmat->get_col_idxs();
+    index_type *const row_ptrs = refmat->get_row_ptrs();
 
     auto mtx = gko::matrix::Fbcsr<value_type, index_type>::create(
         this->exec, gko::dim<2>{nbrows * bs, nbcols * bs}, bs,
@@ -244,7 +241,6 @@ TYPED_TEST(Fbcsr, CanBeCreatedFromExistingData)
     ASSERT_EQ(mtx->get_const_values(), values);
     ASSERT_EQ(mtx->get_const_col_idxs(), col_idxs);
     ASSERT_EQ(mtx->get_const_row_ptrs(), row_ptrs);
-    // ASSERT_EQ(mtx->get_const_srow()[0], 0);
 }
 
 
@@ -293,27 +289,12 @@ TYPED_TEST(Fbcsr, CanBeCleared)
 
 TYPED_TEST(Fbcsr, CanBeReadFromMatrixData)
 {
-    // TODO (script:fbcsr): change the code imported from matrix/csr if needed
     using Mtx = typename TestFixture::Mtx;
     auto m =
         Mtx::create(this->exec, std::make_shared<matstr::classical<Mtx>>());
-    m->set_block_size(3);
-
-    // Assuming row-major blocks
-    m->read(
-        {{6, 12}, {{0, 3, 2.0},   {0, 4, 3.0},  {0, 5, 4.0},  {1, 3, 5.0},
-                   {1, 4, 6.0},   {1, 5, 7.0},  {2, 4, 9.0},
+    m->set_block_size(this->fbsample.bs);
 
-                   {0, 9, 4.0},   {0, 10, 5.0}, {0, 11, 6.0}, {1, 9, 7.0},
-                   {1, 10, 8.0},  {1, 11, 9.0}, {2, 9, 10.0}, {2, 10, 11.0},
-                   {2, 11, 12.0},
-
-                   {3, 0, 2.0},   {3, 1, 3.0},  {3, 2, 4.0},  {4, 0, 5.0},
-                   {4, 1, 6.0},   {4, 2, 7.0},  {5, 0, 8.0},  {5, 1, 9.0},
-                   {5, 2, 10.0},
-
-                   {3, 7, 7.0},   {3, 8, 8.0},  {4, 6, 9.0},  {4, 7, 10.0},
-                   {4, 8, 11.0},  {5, 6, 12.0}, {5, 7, 13.0}, {5, 8, 14.0}}});
+    m->read(this->fbsample.generate_matrix_data());
 
     this->assert_equal_to_original_mtx(m.get());
 }
@@ -329,49 +310,14 @@ TYPED_TEST(Fbcsr, GeneratesCorrectMatrixData)
     this->mtx->write(data);
     data.ensure_row_major_order();
 
-    ASSERT_EQ(data.size, gko::dim<2>(6, 12));
-    ASSERT_EQ(data.nonzeros.size(), 36);
-    EXPECT_EQ(data.nonzeros[0], tpl(0, 3, value_type{2.0}));
-    EXPECT_EQ(data.nonzeros[1], tpl(0, 4, value_type{3.0}));
-    EXPECT_EQ(data.nonzeros[2], tpl(0, 5, value_type{4.0}));
-    EXPECT_EQ(data.nonzeros[3], tpl(0, 9, value_type{4.0}));
-    EXPECT_EQ(data.nonzeros[4], tpl(0, 10, value_type{5.0}));
-    EXPECT_EQ(data.nonzeros[5], tpl(0, 11, value_type{6.0}));
-
-    EXPECT_EQ(data.nonzeros[6], tpl(1, 3, value_type{5.0}));
-    EXPECT_EQ(data.nonzeros[7], tpl(1, 4, value_type{6.0}));
-    EXPECT_EQ(data.nonzeros[8], tpl(1, 5, value_type{7.0}));
-    EXPECT_EQ(data.nonzeros[9], tpl(1, 9, value_type{7.0}));
-    EXPECT_EQ(data.nonzeros[10], tpl(1, 10, value_type{8.0}));
-    EXPECT_EQ(data.nonzeros[11], tpl(1, 11, value_type{9.0}));
-
-    EXPECT_EQ(data.nonzeros[12], tpl(2, 3, value_type{0.0}));
-    EXPECT_EQ(data.nonzeros[13], tpl(2, 4, value_type{9.0}));
-    EXPECT_EQ(data.nonzeros[14], tpl(2, 5, value_type{0.0}));
-    EXPECT_EQ(data.nonzeros[15], tpl(2, 9, value_type{10.0}));
-    EXPECT_EQ(data.nonzeros[16], tpl(2, 10, value_type{11.0}));
-    EXPECT_EQ(data.nonzeros[17], tpl(2, 11, value_type{12.0}));
-
-    EXPECT_EQ(data.nonzeros[18], tpl(3, 0, value_type{2.0}));
-    EXPECT_EQ(data.nonzeros[19], tpl(3, 1, value_type{3.0}));
-    EXPECT_EQ(data.nonzeros[20], tpl(3, 2, value_type{4.0}));
-    EXPECT_EQ(data.nonzeros[21], tpl(3, 6, value_type{0.0}));
-    EXPECT_EQ(data.nonzeros[22], tpl(3, 7, value_type{7.0}));
-    EXPECT_EQ(data.nonzeros[23], tpl(3, 8, value_type{8.0}));
-
-    EXPECT_EQ(data.nonzeros[24], tpl(4, 0, value_type{5.0}));
-    EXPECT_EQ(data.nonzeros[25], tpl(4, 1, value_type{6.0}));
-    EXPECT_EQ(data.nonzeros[26], tpl(4, 2, value_type{7.0}));
-    EXPECT_EQ(data.nonzeros[27], tpl(4, 6, value_type{9.0}));
-    EXPECT_EQ(data.nonzeros[28], tpl(4, 7, value_type{10.0}));
-    EXPECT_EQ(data.nonzeros[29], tpl(4, 8, value_type{11.0}));
-
-    EXPECT_EQ(data.nonzeros[30], tpl(5, 0, value_type{8.0}));
-    EXPECT_EQ(data.nonzeros[31], tpl(5, 1, value_type{9.0}));
-    EXPECT_EQ(data.nonzeros[32], tpl(5, 2, value_type{10.0}));
-    EXPECT_EQ(data.nonzeros[33], tpl(5, 6, value_type{12.0}));
-    EXPECT_EQ(data.nonzeros[34], tpl(5, 7, value_type{13.0}));
-    EXPECT_EQ(data.nonzeros[35], tpl(5, 8, value_type{14.0}));
+    gko::matrix_data<value_type, index_type> refdata =
+        this->fbsample.generate_matrix_data_with_explicit_zeros();
+    refdata.ensure_row_major_order();
+
+    ASSERT_EQ(data.size, refdata.size);
+    ASSERT_EQ(data.nonzeros.size(), refdata.nonzeros.size());
+    for (size_t i = 0; i < data.nonzeros.size(); i++)
+        ASSERT_EQ(data.nonzeros[i], refdata.nonzeros[i]);
 }
 
 
diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
new file mode 100644
index 00000000000..27944dd6873
--- /dev/null
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -0,0 +1,286 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+
+#include <ginkgo/core/matrix/matrix_strategies.hpp>
+
+#include "core/components/fixed_block.hpp"
+#include "fbcsr_sample.hpp"
+
+namespace gko {
+namespace testing {
+
+
+namespace matstr = gko::matrix::matrix_strategy;
+
+template <typename ValueType, typename IndexType>
+FbcsrSample<ValueType, IndexType>::FbcsrSample(
+    const std::shared_ptr<const gko::ReferenceExecutor> rexec)
+    : nrows{6},
+      ncols{12},
+      nnz{36},
+      nbrows{2},
+      nbcols{4},
+      nbnz{4},
+      bs{3},
+      exec(rexec)
+{}
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
+FbcsrSample<ValueType, IndexType>::generate_fbcsr() const
+{
+    std::unique_ptr<Fbcsr> mtx =
+        Fbcsr::create(exec,
+                      gko::dim<2>{static_cast<size_type>(nrows),
+                                  static_cast<size_type>(ncols)},
+                      nnz, bs, std::make_shared<matstr::classical<Fbcsr>>());
+
+    value_type *const v = mtx->get_values();
+    index_type *const c = mtx->get_col_idxs();
+    index_type *const r = mtx->get_row_ptrs();
+    index_type *const s = mtx->get_srow();
+    r[0] = 0;
+    r[1] = 2;
+    r[2] = 4;
+    c[0] = 1;
+    c[1] = 3;
+    c[2] = 0;
+    c[3] = 2;
+
+    gko::blockutils::DenseBlocksView<value_type, index_type> vals(v, bs, bs);
+
+    if (mtx->get_size()[0] % bs != 0)
+        throw gko::BadDimension(__FILE__, __LINE__, __func__, "test fbcsr",
+                                mtx->get_size()[0], mtx->get_size()[1],
+                                "block size does not divide the size!");
+
+    for (index_type ibrow = 0; ibrow < mtx->get_size()[0] / bs; ibrow++) {
+        const index_type *const browptr = mtx->get_row_ptrs();
+        for (index_type inz = browptr[ibrow]; inz < browptr[ibrow + 1]; inz++) {
+            const index_type bcolind = mtx->get_col_idxs()[inz];
+            const value_type base = (ibrow + 1) * (bcolind + 1);
+            for (int ival = 0; ival < bs; ival++)
+                for (int jval = 0; jval < bs; jval++)
+                    vals(inz, ival, jval) =
+                        base + static_cast<gko::remove_complex<value_type>>(
+                                   ival * bs + jval);
+        }
+    }
+
+    // Some of the entries are set to zero
+    vals(0, 2, 0) = gko::zero<value_type>();
+    vals(0, 2, 2) = gko::zero<value_type>();
+    vals(3, 0, 0) = gko::zero<value_type>();
+
+    for (index_type is = 0; is < mtx->get_num_srow_elements(); is++) s[is] = 0;
+
+    return mtx;
+}
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Csr<ValueType, IndexType>>
+FbcsrSample<ValueType, IndexType>::generate_csr() const
+{
+    std::unique_ptr<Csr> csrm =
+        Csr::create(exec, gko::dim<2>{nrows, ncols}, nnz,
+                    std::make_shared<typename Csr::classical>());
+    index_type *const csrrow = csrm->get_row_ptrs();
+    index_type *const csrcols = csrm->get_col_idxs();
+    value_type *const csrvals = csrm->get_values();
+
+    csrrow[0] = 0;
+    csrrow[1] = 6;
+    csrrow[2] = 12;
+    csrrow[3] = 18;
+    csrrow[4] = 24;
+    csrrow[5] = 30;
+    csrrow[6] = 36;
+
+    csrcols[0] = 3;
+    csrcols[1] = 4;
+    csrcols[2] = 5;
+    csrcols[6] = 3;
+    csrcols[7] = 4;
+    csrcols[8] = 5;
+    csrcols[12] = 3;
+    csrcols[13] = 4;
+    csrcols[14] = 5;
+
+    csrcols[3] = 9;
+    csrcols[4] = 10;
+    csrcols[5] = 11;
+    csrcols[9] = 9;
+    csrcols[10] = 10;
+    csrcols[11] = 11;
+    csrcols[15] = 9;
+    csrcols[16] = 10;
+    csrcols[17] = 11;
+
+    csrcols[18] = 0;
+    csrcols[19] = 1;
+    csrcols[20] = 2;
+    csrcols[24] = 0;
+    csrcols[25] = 1;
+    csrcols[26] = 2;
+    csrcols[30] = 0;
+    csrcols[31] = 1;
+    csrcols[32] = 2;
+
+    csrcols[21] = 6;
+    csrcols[22] = 7;
+    csrcols[23] = 8;
+    csrcols[27] = 6;
+    csrcols[28] = 7;
+    csrcols[29] = 8;
+    csrcols[33] = 6;
+    csrcols[34] = 7;
+    csrcols[35] = 8;
+
+    // values
+    csrvals[0] = 2;
+    csrvals[1] = 3;
+    csrvals[2] = 4;
+    csrvals[6] = 5;
+    csrvals[7] = 6;
+    csrvals[8] = 7;
+    csrvals[12] = 0;
+    csrvals[13] = 9;
+    csrvals[14] = 0;
+
+    csrvals[3] = 4;
+    csrvals[4] = 5;
+    csrvals[5] = 6;
+    csrvals[9] = 7;
+    csrvals[10] = 8;
+    csrvals[11] = 9;
+    csrvals[15] = 10;
+    csrvals[16] = 11;
+    csrvals[17] = 12;
+
+    csrvals[18] = 2;
+    csrvals[19] = 3;
+    csrvals[20] = 4;
+    csrvals[24] = 5;
+    csrvals[25] = 6;
+    csrvals[26] = 7;
+    csrvals[30] = 8;
+    csrvals[31] = 9;
+    csrvals[32] = 10;
+
+    csrvals[21] = 0;
+    csrvals[22] = 7;
+    csrvals[23] = 8;
+    csrvals[27] = 9;
+    csrvals[28] = 10;
+    csrvals[29] = 11;
+    csrvals[33] = 12;
+    csrvals[34] = 13;
+    csrvals[35] = 14;
+
+    return csrm;
+}
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Dense<ValueType>>
+FbcsrSample<ValueType, IndexType>::generate_dense() const
+{
+    std::unique_ptr<Dense> densem =
+        Dense::create(exec, gko::dim<2>(nrows, ncols));
+
+    for (size_type irow = 0; irow < densem->get_size()[0]; irow++)
+        for (size_type jcol = 0; jcol < densem->get_size()[1]; jcol++) {
+            densem->at(irow, jcol) = 0;
+            if (irow < 3 && jcol >= 3 && jcol < 6)
+                densem->at(irow, jcol) = 2.0 + irow * bs + jcol - 3;
+            if (irow < 3 && jcol >= 9)
+                densem->at(irow, jcol) = 4.0 + irow * bs + jcol - 9;
+            if (irow >= 3 && jcol < 3)
+                densem->at(irow, jcol) = 2.0 + (irow - 3) * bs + jcol;
+            if (irow >= 3 && jcol >= 6 && jcol < 9)
+                densem->at(irow, jcol) = 6.0 + (irow - 3) * bs + jcol - 6;
+        }
+
+    densem->at(2, 3) = densem->at(2, 5) = densem->at(3, 6) = 0.0;
+
+    return densem;
+}
+
+// Assuming row-major blocks
+template <typename ValueType, typename IndexType>
+gko::matrix_data<ValueType, IndexType>
+FbcsrSample<ValueType, IndexType>::generate_matrix_data() const
+{
+    return MatData(
+        {{6, 12}, {{0, 3, 2.0},   {0, 4, 3.0},  {0, 5, 4.0},  {1, 3, 5.0},
+                   {1, 4, 6.0},   {1, 5, 7.0},  {2, 4, 9.0},
+
+                   {0, 9, 4.0},   {0, 10, 5.0}, {0, 11, 6.0}, {1, 9, 7.0},
+                   {1, 10, 8.0},  {1, 11, 9.0}, {2, 9, 10.0}, {2, 10, 11.0},
+                   {2, 11, 12.0},
+
+                   {3, 0, 2.0},   {3, 1, 3.0},  {3, 2, 4.0},  {4, 0, 5.0},
+                   {4, 1, 6.0},   {4, 2, 7.0},  {5, 0, 8.0},  {5, 1, 9.0},
+                   {5, 2, 10.0},
+
+                   {3, 7, 7.0},   {3, 8, 8.0},  {4, 6, 9.0},  {4, 7, 10.0},
+                   {4, 8, 11.0},  {5, 6, 12.0}, {5, 7, 13.0}, {5, 8, 14.0}}});
+}
+
+// Assuming row-major blocks
+template <typename ValueType, typename IndexType>
+gko::matrix_data<ValueType, IndexType> FbcsrSample<
+    ValueType, IndexType>::generate_matrix_data_with_explicit_zeros() const
+{
+    return MatData({{6, 12}, {{0, 3, 2.0},  {0, 4, 3.0},   {0, 5, 4.0},
+                              {1, 3, 5.0},  {1, 4, 6.0},   {1, 5, 7.0},
+                              {2, 3, 0.0},  {2, 4, 9.0},   {2, 5, 0.0},
+
+                              {0, 9, 4.0},  {0, 10, 5.0},  {0, 11, 6.0},
+                              {1, 9, 7.0},  {1, 10, 8.0},  {1, 11, 9.0},
+                              {2, 9, 10.0}, {2, 10, 11.0}, {2, 11, 12.0},
+
+                              {3, 0, 2.0},  {3, 1, 3.0},   {3, 2, 4.0},
+                              {4, 0, 5.0},  {4, 1, 6.0},   {4, 2, 7.0},
+                              {5, 0, 8.0},  {5, 1, 9.0},   {5, 2, 10.0},
+
+                              {3, 6, 0.0},  {3, 7, 7.0},   {3, 8, 8.0},
+                              {4, 6, 9.0},  {4, 7, 10.0},  {4, 8, 11.0},
+                              {5, 6, 12.0}, {5, 7, 13.0},  {5, 8, 14.0}}});
+}
+
+#define GKO_DECLARE_FBCSR_TEST_SAMPLE(ValueType, IndexType) \
+    class FbcsrSample<ValueType, IndexType>
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_TEST_SAMPLE);
+
+}  // namespace testing
+}  // namespace gko
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
new file mode 100644
index 00000000000..d19037cddcd
--- /dev/null
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -0,0 +1,83 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+
+#ifndef GKO_CORE_MATRIX_TEST_FBCSR_SAMPLE_HPP
+#define GKO_CORE_MATRIX_TEST_FBCSR_SAMPLE_HPP
+
+#include <ginkgo/core/base/matrix_data.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/fbcsr.hpp>
+
+namespace gko {
+namespace testing {
+
+
+/// Generates the same sample block CSR matrix in different formats
+template <typename ValueType, typename IndexType>
+class FbcsrSample {
+public:
+    using value_type = ValueType;
+    using index_type = IndexType;
+    using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+    using MatData = gko::matrix_data<value_type, index_type>;
+
+    FbcsrSample(std::shared_ptr<const gko::ReferenceExecutor> exec);
+
+    std::unique_ptr<Fbcsr> generate_fbcsr() const;
+
+    /// Generates CSR matrix equal to the BSR matrix. Keeps explicit zeros.
+    std::unique_ptr<Csr> generate_csr() const;
+
+    std::unique_ptr<Dense> generate_dense() const;
+
+    MatData generate_matrix_data() const;
+
+    MatData generate_matrix_data_with_explicit_zeros() const;
+
+    const size_type nrows;
+    const size_type ncols;
+    const size_type nnz;
+    const size_type nbrows;
+    const size_type nbcols;
+    const size_type nbnz;
+    const int bs;
+    const std::shared_ptr<const gko::Executor> exec;
+};
+
+}  // namespace testing
+}  // namespace gko
+
+#endif
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index ac70964dfbc..bfbab84eabe 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -48,6 +48,9 @@ namespace matrix {
 template <typename ValueType>
 class Dense;
 
+template <typename ValueType, typename IndexType>
+class Csr;
+
 template <typename ValueType, typename IndexType>
 class Coo;
 
@@ -122,6 +125,7 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
               public EnableCreateMethod<Fbcsr<ValueType, IndexType>>,
               public ConvertibleTo<Fbcsr<next_precision<ValueType>, IndexType>>,
               public ConvertibleTo<Dense<ValueType>>,
+              public ConvertibleTo<Csr<ValueType, IndexType>>,
               public ConvertibleTo<Coo<ValueType, IndexType>>,
               // public ConvertibleTo<Ell<ValueType, IndexType>>,
               // public ConvertibleTo<Hybrid<ValueType, IndexType>>,
@@ -193,6 +197,10 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
 
     void move_to(Dense<ValueType> *other) override;
 
+    void convert_to(Csr<ValueType, IndexType> *result) const override;
+
+    void move_to(Csr<ValueType, IndexType> *result) override;
+
     void convert_to(Coo<ValueType, IndexType> *result) const override;
 
     void move_to(Coo<ValueType, IndexType> *result) override;

From f5b2006fa60f73f5c506cc8e1d8906a550a48a45 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 11 Nov 2020 16:03:00 +0100
Subject: [PATCH 0532/1008] [tests fail] added conversion to dense and csr on
 reference

---
 core/device_hooks/common_kernels.inc.cpp |   6 +
 core/matrix/fbcsr.cpp                    |  48 +-
 core/matrix/fbcsr_kernels.hpp            |   8 +
 core/test/matrix/fbcsr_sample.cpp        |  36 ++
 core/test/matrix/fbcsr_sample.hpp        |   9 +
 cuda/matrix/fbcsr_kernels.cu             |  11 +
 hip/matrix/fbcsr_kernels.hip.cpp         |  11 +
 omp/matrix/fbcsr_kernels.cpp             |  11 +
 reference/matrix/fbcsr_kernels.cpp       | 135 ++++--
 reference/test/matrix/CMakeLists.txt     |   2 +-
 reference/test/matrix/fbcsr_kernels.cpp  | 532 ++++++++---------------
 11 files changed, 389 insertions(+), 420 deletions(-)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 43f4d90d8c0..f66c749fd73 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -841,6 +841,12 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
 
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL);
+
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 1ef12627602..0d00e70bcc5 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -67,6 +67,7 @@ GKO_REGISTER_OPERATION(spgemm, fbcsr::spgemm);
 // GKO_REGISTER_OPERATION(advanced_spgemm, fbcsr::advanced_spgemm);
 GKO_REGISTER_OPERATION(spgeam, fbcsr::spgeam);
 GKO_REGISTER_OPERATION(convert_to_coo, fbcsr::convert_to_coo);
+GKO_REGISTER_OPERATION(convert_to_csr, fbcsr::convert_to_csr);
 GKO_REGISTER_OPERATION(convert_to_dense, fbcsr::convert_to_dense);
 // GKO_REGISTER_OPERATION(convert_to_sellp, fbcsr::convert_to_sellp);
 GKO_REGISTER_OPERATION(calculate_total_cols, fbcsr::calculate_total_cols);
@@ -225,44 +226,39 @@ void Fbcsr<ValueType, IndexType>::move_to(Coo<ValueType, IndexType> *result)
 
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(Dense<ValueType> *result) const
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto exec = this->get_executor();
-//    auto tmp = Dense<ValueType>::create(exec, this->get_size());
-//    exec->run(fbcsr::make_convert_to_dense(this, tmp.get()));
-//    tmp->move_to(result);
-//}
+{
+    auto exec = this->get_executor();
+    auto tmp = Dense<ValueType>::create(exec, this->get_size());
+    exec->run(fbcsr::make_convert_to_dense(this, tmp.get()));
+    tmp->move_to(result);
+}
 
 
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::move_to(Dense<ValueType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    this->convert_to(result);
-//}
+{
+    this->convert_to(result);
+}
 
 
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(
-    Csr<ValueType, IndexType> *result) const GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto exec = this->get_executor();
-//    auto tmp = Dense<ValueType>::create(exec, this->get_size());
-//    exec->run(fbcsr::make_convert_to_csr(this, tmp.get()));
-//    tmp->move_to(result);
-//}
+    Csr<ValueType, IndexType> *result) const
+{
+    auto exec = this->get_executor();
+    auto tmp = Csr<ValueType, IndexType>::create(
+        exec, this->get_size(), this->get_num_stored_elements(),
+        result->get_strategy());
+    exec->run(fbcsr::make_convert_to_csr(this, tmp.get()));
+    tmp->move_to(result);
+}
 
 
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::move_to(Csr<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    this->convert_to(result);
-//}
+{
+    this->convert_to(result);
+}
 
 
 // template <typename ValueType, typename IndexType>
diff --git a/core/matrix/fbcsr_kernels.hpp b/core/matrix/fbcsr_kernels.hpp
index a57b2af3c82..6fa130b11c9 100644
--- a/core/matrix/fbcsr_kernels.hpp
+++ b/core/matrix/fbcsr_kernels.hpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/diagonal.hpp>
 #include <ginkgo/core/matrix/ell.hpp>
@@ -93,6 +94,11 @@ namespace kernels {
                           const matrix::Fbcsr<ValueType, IndexType> *source, \
                           matrix::Dense<ValueType> *result)
 
+#define GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL(ValueType, IndexType)      \
+    void convert_to_csr(std::shared_ptr<const DefaultExecutor> exec,       \
+                        const matrix::Fbcsr<ValueType, IndexType> *source, \
+                        matrix::Csr<ValueType, IndexType> *result)
+
 #define GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL(ValueType, IndexType)      \
     void convert_to_coo(std::shared_ptr<const DefaultExecutor> exec,       \
                         const matrix::Fbcsr<ValueType, IndexType> *source, \
@@ -196,6 +202,8 @@ namespace kernels {
     template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType);           \
     template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL(ValueType, IndexType);             \
+    template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL(ValueType, IndexType);             \
     template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL(ValueType, IndexType);           \
diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
index 27944dd6873..a221cd3381e 100644
--- a/core/test/matrix/fbcsr_sample.cpp
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -33,6 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/matrix/matrix_strategies.hpp>
 
+#include <algorithm>
+
 #include "core/components/fixed_block.hpp"
 #include "fbcsr_sample.hpp"
 
@@ -278,6 +280,40 @@ gko::matrix_data<ValueType, IndexType> FbcsrSample<
                               {5, 6, 12.0}, {5, 7, 13.0},  {5, 8, 14.0}}});
 }
 
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Coo<ValueType, IndexType>>
+FbcsrSample<ValueType, IndexType>::generate_coo() const
+{
+    gko::matrix_data<ValueType, IndexType> mdata =
+        generate_matrix_data_with_explicit_zeros();
+
+    using nztype =
+        typename gko::matrix_data<ValueType, IndexType>::nonzero_type;
+    std::sort(mdata.nonzeros.begin(), mdata.nonzeros.end(),
+              [](const nztype &a, const nztype &b) {
+                  if (a.row < b.row)
+                      return true;
+                  else if (a.row > b.row)
+                      return false;
+                  else if (a.column < b.column)
+                      return true;
+                  else
+                      return false;
+              });
+
+    gko::Array<IndexType> rowidx(exec, nnz);
+    gko::Array<IndexType> colidx(exec, nnz);
+    gko::Array<ValueType> values(exec, nnz);
+    for (size_t i = 0; i < mdata.nonzeros.size(); i++) {
+        rowidx.get_data()[i] = mdata.nonzeros[i].row;
+        colidx.get_data()[i] = mdata.nonzeros[i].column;
+        values.get_data()[i] = mdata.nonzeros[i].value;
+    }
+    auto mat =
+        Coo::create(exec, gko::dim<2>{nrows, ncols}, values, colidx, rowidx);
+    return mat;
+}
+
 #define GKO_DECLARE_FBCSR_TEST_SAMPLE(ValueType, IndexType) \
     class FbcsrSample<ValueType, IndexType>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_TEST_SAMPLE);
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index d19037cddcd..7fe2fd3cb32 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_MATRIX_TEST_FBCSR_SAMPLE_HPP
 
 #include <ginkgo/core/base/matrix_data.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/fbcsr.hpp>
@@ -44,6 +45,8 @@ namespace testing {
 
 
 /// Generates the same sample block CSR matrix in different formats
+/** This currently a 6 x 12 matrix with 3x3 blocks.
+ */
 template <typename ValueType, typename IndexType>
 class FbcsrSample {
 public:
@@ -51,6 +54,7 @@ class FbcsrSample {
     using index_type = IndexType;
     using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
+    using Coo = gko::matrix::Coo<value_type, index_type>;
     using Dense = gko::matrix::Dense<value_type>;
     using MatData = gko::matrix_data<value_type, index_type>;
 
@@ -63,6 +67,11 @@ class FbcsrSample {
 
     std::unique_ptr<Dense> generate_dense() const;
 
+    /// Returns the matrix in COO format keeping explicit nonzeros
+    /** The nonzeros are sorted by row and column.
+     */
+    std::unique_ptr<Coo> generate_coo() const;
+
     MatData generate_matrix_data() const;
 
     MatData generate_matrix_data_with_explicit_zeros() const;
diff --git a/cuda/matrix/fbcsr_kernels.cu b/cuda/matrix/fbcsr_kernels.cu
index 5d67bcc90d3..968399ff8ed 100644
--- a/cuda/matrix/fbcsr_kernels.cu
+++ b/cuda/matrix/fbcsr_kernels.cu
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/ell.hpp>
 #include <ginkgo/core/matrix/hybrid.hpp>
@@ -928,6 +929,16 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void convert_to_csr(const std::shared_ptr<const CudaExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *const source,
+                    matrix::Csr<ValueType, IndexType> *const result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void convert_to_sellp(std::shared_ptr<const CudaExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *source,
diff --git a/hip/matrix/fbcsr_kernels.hip.cpp b/hip/matrix/fbcsr_kernels.hip.cpp
index 5263e535e7a..da425238ab0 100644
--- a/hip/matrix/fbcsr_kernels.hip.cpp
+++ b/hip/matrix/fbcsr_kernels.hip.cpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/ell.hpp>
 #include <ginkgo/core/matrix/hybrid.hpp>
@@ -777,6 +778,16 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void convert_to_csr(const std::shared_ptr<const HipExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *const source,
+                    matrix::Csr<ValueType, IndexType> *const result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void convert_to_sellp(std::shared_ptr<const HipExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *source,
diff --git a/omp/matrix/fbcsr_kernels.cpp b/omp/matrix/fbcsr_kernels.cpp
index 3ad97e0e6bc..d30d8cd3b16 100644
--- a/omp/matrix/fbcsr_kernels.cpp
+++ b/omp/matrix/fbcsr_kernels.cpp
@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/hybrid.hpp>
 
@@ -430,6 +431,16 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void convert_to_csr(const std::shared_ptr<const OmpExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *const source,
+                    matrix::Csr<ValueType, IndexType> *const result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void convert_to_sellp(std::shared_ptr<const OmpExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *source,
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index af0f19a8947..fe6bac8efd3 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -51,6 +51,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/base/allocator.hpp"
 #include "core/base/iterator_factory.hpp"
+#include "core/components/fixed_block.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/fbcsr_builder.hpp"
 #include "reference/components/fbcsr_spgeam.hpp"
@@ -367,14 +368,13 @@ void spgeam(std::shared_ptr<const ReferenceExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEAM_KERNEL);
 
 
-template <typename IndexType>
-void convert_row_ptrs_to_idxs(std::shared_ptr<const ReferenceExecutor> exec,
-                              const IndexType *ptrs, size_type num_rows,
-                              IndexType *idxs) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+// template <typename IndexType>
+// void convert_row_ptrs_to_idxs(std::shared_ptr<const ReferenceExecutor> exec,
+//                               const IndexType *ptrs, size_type num_rows,
+//                               IndexType *idxs)
+// {
 //    convert_ptrs_to_idxs(ptrs, num_rows, idxs);
-//}
+// }
 
 
 template <typename ValueType, typename IndexType>
@@ -382,46 +382,107 @@ void convert_to_coo(std::shared_ptr<const ReferenceExecutor> exec,
                     const matrix::Fbcsr<ValueType, IndexType> *source,
                     matrix::Coo<ValueType, IndexType> *result)
     GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = result->get_size()[0];
-//
-//    auto row_idxs = result->get_row_idxs();
-//    const auto source_row_ptrs = source->get_const_row_ptrs();
-//
-//    convert_row_ptrs_to_idxs(exec, source_row_ptrs, num_rows, row_idxs);
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL);
 
 template <typename ValueType, typename IndexType>
-void convert_to_dense(std::shared_ptr<const ReferenceExecutor> exec,
-                      const matrix::Fbcsr<ValueType, IndexType> *source,
-                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = source->get_size()[0];
-//    auto num_cols = source->get_size()[1];
-//    auto row_ptrs = source->get_const_row_ptrs();
-//    auto col_idxs = source->get_const_col_idxs();
-//    auto vals = source->get_const_values();
-//
-//    for (size_type row = 0; row < num_rows; ++row) {
-//        for (size_type col = 0; col < num_cols; ++col) {
-//            result->at(row, col) = zero<ValueType>();
-//        }
-//        for (size_type i = row_ptrs[row];
-//             i < static_cast<size_type>(row_ptrs[row + 1]); ++i) {
-//            result->at(row, col_idxs[i]) = vals[i];
-//        }
-//    }
-//}
+void convert_to_dense(const std::shared_ptr<const ReferenceExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *const source,
+                      matrix::Dense<ValueType> *const result)
+{
+    const int bs = source->get_block_size();
+    const size_type nbrows =
+        gko::blockutils::getNumFixedBlocks(bs, source->get_size()[0]);
+    const size_type nbcols =
+        gko::blockutils::getNumFixedBlocks(bs, source->get_size()[1]);
+    const IndexType *const row_ptrs = source->get_const_row_ptrs();
+    const IndexType *const col_idxs = source->get_const_col_idxs();
+    const ValueType *const vals = source->get_const_values();
+
+    const gko::blockutils::DenseBlocksView<const ValueType, IndexType> values(
+        vals, bs, bs);
+
+    for (size_type brow = 0; brow < nbrows; ++brow) {
+        for (size_type bcol = 0; bcol < nbcols; ++bcol) {
+            for (int ib = 0; ib < bs; ib++)
+                for (int jb = 0; jb < bs; jb++)
+                    result->at(brow * bs + ib, bcol * bs + jb) =
+                        zero<ValueType>();
+        }
+        for (IndexType ibnz = row_ptrs[brow]; ibnz < row_ptrs[brow + 1];
+             ++ibnz) {
+            for (int ib = 0; ib < bs; ib++) {
+                const IndexType row = brow * bs + ib;
+                for (int jb = 0; jb < bs; jb++)
+                    result->at(row, col_idxs[ibnz] * bs + jb) =
+                        values(ibnz, ib, jb);
+            }
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
 
 
+template <typename ValueType, typename IndexType>
+void convert_to_csr(const std::shared_ptr<const ReferenceExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *const source,
+                    matrix::Csr<ValueType, IndexType> *const result)
+{
+    const int bs = source->get_block_size();
+    const size_type nbrows =
+        gko::blockutils::getNumFixedBlocks(bs, source->get_size()[0]);
+    const size_type nbcols =
+        gko::blockutils::getNumFixedBlocks(bs, source->get_size()[1]);
+    const IndexType *const browptrs = source->get_const_row_ptrs();
+    const IndexType *const bcolinds = source->get_const_col_idxs();
+    const ValueType *const bvals = source->get_const_values();
+
+    assert(nbrows * bs == result->get_size()[0]);
+    assert(nbcols * bs == result->get_size()[1]);
+    assert(source->get_num_stored_elements() ==
+           result->get_num_stored_elements());
+
+    IndexType *const row_ptrs = result->get_row_ptrs();
+    IndexType *const col_idxs = result->get_col_idxs();
+    ValueType *const vals = result->get_values();
+
+    const gko::blockutils::DenseBlocksView<const ValueType, IndexType> bvalues(
+        bvals, bs, bs);
+
+    for (size_type brow = 0; brow < nbrows; ++brow) {
+        const IndexType nz_browstart = browptrs[brow] * bs * bs;
+        const IndexType numblocks_brow = browptrs[brow + 1] - browptrs[brow];
+        for (int ib = 0; ib < bs; ib++)
+            row_ptrs[brow * bs + ib] = nz_browstart + numblocks_brow * bs * ib;
+
+        for (IndexType ibnz = browptrs[brow]; ibnz < browptrs[brow + 1];
+             ++ibnz) {
+            const IndexType bcol = bcolinds[ibnz];
+
+            for (int ib = 0; ib < bs; ib++) {
+                const IndexType row = brow * bs + ib;
+                const IndexType inz_blockstart =
+                    row_ptrs[row] + (ibnz - browptrs[brow]) * bs;
+
+                for (int jb = 0; jb < bs; jb++) {
+                    const IndexType inz = inz_blockstart + jb;
+                    vals[inz] = bvalues(ibnz, ib, jb);
+                    col_idxs[inz] = bcol * bs + jb;
+                }
+            }
+        }
+    }
+
+    row_ptrs[source->get_size()[0]] = source->get_num_stored_elements();
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void convert_to_sellp(std::shared_ptr<const ReferenceExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *source,
diff --git a/reference/test/matrix/CMakeLists.txt b/reference/test/matrix/CMakeLists.txt
index c826a9eaca7..cb4d2b0217a 100644
--- a/reference/test/matrix/CMakeLists.txt
+++ b/reference/test/matrix/CMakeLists.txt
@@ -3,7 +3,7 @@ ginkgo_create_test(csr_kernels)
 ginkgo_create_test(dense_kernels)
 ginkgo_create_test(diagonal_kernels)
 ginkgo_create_test(ell_kernels)
-ginkgo_create_test(fbcsr_kernels)
+ginkgo_create_test(fbcsr_kernels test_fbcsr_sample)
 ginkgo_create_test(hybrid_kernels)
 ginkgo_create_test(identity)
 ginkgo_create_test(permutation)
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index b5fd2162da5..8fb53f4e3b2 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -55,6 +55,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/matrix/fbcsr_kernels.hpp"
+#include "core/test/matrix/fbcsr_sample.hpp"
 #include "core/test/utils.hpp"
 
 
@@ -74,8 +75,10 @@ class Fbcsr : public ::testing::Test {
         typename std::tuple_element<0, decltype(ValueIndexType())>::type;
     using index_type =
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
-    using Coo = gko::matrix::Coo<value_type, index_type>;
     using Mtx = gko::matrix::Fbcsr<value_type, index_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    using Coo = gko::matrix::Coo<value_type, index_type>;
+    using Dense = gko::matrix::Dense<value_type>;
     using Sellp = gko::matrix::Sellp<value_type, index_type>;
     using SparsityCsr = gko::matrix::SparsityCsr<value_type, index_type>;
     using Ell = gko::matrix::Ell<value_type, index_type>;
@@ -84,174 +87,63 @@ class Fbcsr : public ::testing::Test {
 
     Fbcsr()
         : exec(gko::ReferenceExecutor::create()),
-          mtx(Mtx::create(exec, gko::dim<2>{2, 3}, 4, mat_bs,
-                          std::make_shared<matstr::load_balance<Mtx>>(2))),
-          mtx2(Mtx::create(exec, gko::dim<2>{2, 3}, 5, mat_bs,
-                           std::make_shared<matstr::classical<Mtx>>())),
-          mtx3_sorted(Mtx::create(exec, gko::dim<2>(3, 3), 7, mat_bs,
-                                  std::make_shared<matstr::classical<Mtx>>())),
-          mtx3_unsorted(Mtx::create(exec, gko::dim<2>(3, 3), 7, mat_bs,
-                                    std::make_shared<matstr::classical<Mtx>>()))
-    {
-        this->create_mtx(mtx.get());
-        this->create_mtx2(mtx2.get());
-        this->create_mtx3(mtx3_sorted.get(), mtx3_unsorted.get());
-    }
-
-    void create_mtx(Mtx *m)
+          fbsample(exec),
+          mtx(fbsample.generate_fbcsr()),
+          refmtx(fbsample.generate_fbcsr()),
+          refcsrmtx(fbsample.generate_csr()),
+          refdenmtx(fbsample.generate_dense()),
+          refcoomtx(fbsample.generate_coo())
+    {}
+
+    // void create_mtx3(Mtx *sorted, Mtx *unsorted)
+    // {
+    //     /* For both versions (sorted and unsorted), this matrix is stored:
+    //      * 0  2  1
+    //      * 3  1  8
+    //      * 2  0  3
+    //      * The unsorted matrix will have the (value, column) pair per row not
+    //      * sorted, which we still consider a valid FBCSR format.
+    //      */
+    // }
+
+    void assert_equal_to_mtx(const Csr *const m)
     {
-        value_type *v = m->get_values();
-        index_type *c = m->get_col_idxs();
-        index_type *r = m->get_row_ptrs();
-        auto *s = m->get_srow();
-        /*
-         * 1   3   2
-         * 0   5   0
-         */
-        r[0] = 0;
-        r[1] = 3;
-        r[2] = 4;
-        c[0] = 0;
-        c[1] = 1;
-        c[2] = 2;
-        c[3] = 1;
-        v[0] = 1.0;
-        v[1] = 3.0;
-        v[2] = 2.0;
-        v[3] = 5.0;
-        s[0] = 0;
+        ASSERT_EQ(m->get_size(), refcsrmtx->get_size());
+        ASSERT_EQ(m->get_num_stored_elements(),
+                  refcsrmtx->get_num_stored_elements());
+        for (index_type i = 0; i < m->get_size()[0] + 1; i++)
+            ASSERT_EQ(m->get_const_row_ptrs()[i],
+                      refcsrmtx->get_const_row_ptrs()[i]);
+        for (index_type i = 0; i < m->get_num_stored_elements(); i++) {
+            ASSERT_EQ(m->get_const_col_idxs()[i],
+                      refcsrmtx->get_const_col_idxs()[i]);
+            ASSERT_EQ(m->get_const_values()[i],
+                      refcsrmtx->get_const_values()[i]);
+        }
     }
 
-    void create_mtx2(Mtx *m)
-    {
-        value_type *v = m->get_values();
-        index_type *c = m->get_col_idxs();
-        index_type *r = m->get_row_ptrs();
-        // It keeps an explict zero
-        /*
-         *  1    3   2
-         * {0}   5   0
-         */
-        r[0] = 0;
-        r[1] = 3;
-        r[2] = 5;
-        c[0] = 0;
-        c[1] = 1;
-        c[2] = 2;
-        c[3] = 0;
-        c[4] = 1;
-        v[0] = 1.0;
-        v[1] = 3.0;
-        v[2] = 2.0;
-        v[3] = 0.0;
-        v[4] = 5.0;
-    }
-
-    void create_mtx3(Mtx *sorted, Mtx *unsorted)
-    {
-        auto vals_s = sorted->get_values();
-        auto cols_s = sorted->get_col_idxs();
-        auto rows_s = sorted->get_row_ptrs();
-        auto vals_u = unsorted->get_values();
-        auto cols_u = unsorted->get_col_idxs();
-        auto rows_u = unsorted->get_row_ptrs();
-        /* For both versions (sorted and unsorted), this matrix is stored:
-         * 0  2  1
-         * 3  1  8
-         * 2  0  3
-         * The unsorted matrix will have the (value, column) pair per row not
-         * sorted, which we still consider a valid FBCSR format.
-         */
-        rows_s[0] = 0;
-        rows_s[1] = 2;
-        rows_s[2] = 5;
-        rows_s[3] = 7;
-        rows_u[0] = 0;
-        rows_u[1] = 2;
-        rows_u[2] = 5;
-        rows_u[3] = 7;
-
-        vals_s[0] = 2.;
-        vals_s[1] = 1.;
-        vals_s[2] = 3.;
-        vals_s[3] = 1.;
-        vals_s[4] = 8.;
-        vals_s[5] = 2.;
-        vals_s[6] = 3.;
-        // Each row is stored rotated once to the left
-        vals_u[0] = 1.;
-        vals_u[1] = 2.;
-        vals_u[2] = 1.;
-        vals_u[3] = 8.;
-        vals_u[4] = 3.;
-        vals_u[5] = 3.;
-        vals_u[6] = 2.;
-
-        cols_s[0] = 1;
-        cols_s[1] = 2;
-        cols_s[2] = 0;
-        cols_s[3] = 1;
-        cols_s[4] = 2;
-        cols_s[5] = 0;
-        cols_s[6] = 2;
-        // The same applies for the columns
-        cols_u[0] = 2;
-        cols_u[1] = 1;
-        cols_u[2] = 1;
-        cols_u[3] = 2;
-        cols_u[4] = 0;
-        cols_u[5] = 2;
-        cols_u[6] = 0;
-    }
+    // void assert_equal_to_mtx(const Dense *const m)
+    // {
+    //     ASSERT_EQ(m->get_size(), refdenmtx->get_size());
+    //     ASSERT_EQ(m->get_num_stored_elements(),
+    //     refdenmtx->get_num_stored_elements()); for(index_type i = 0; i <
+    //     m->get_size()[0]; i++)
+    //         for(index_type j = 0; j < m->get_size()[1]; j++)
+    //             ASSERT_EQ(m->at(i,j), refdenmtx->at(i,j));
+    // }
 
     void assert_equal_to_mtx(const Coo *m)
     {
-        auto v = m->get_const_values();
-        auto c = m->get_const_col_idxs();
-        auto r = m->get_const_row_idxs();
-
-        ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
-        ASSERT_EQ(m->get_num_stored_elements(), 4);
-        EXPECT_EQ(r[0], 0);
-        EXPECT_EQ(r[1], 0);
-        EXPECT_EQ(r[2], 0);
-        EXPECT_EQ(r[3], 1);
-        EXPECT_EQ(c[0], 0);
-        EXPECT_EQ(c[1], 1);
-        EXPECT_EQ(c[2], 2);
-        EXPECT_EQ(c[3], 1);
-        EXPECT_EQ(v[0], value_type{1.0});
-        EXPECT_EQ(v[1], value_type{3.0});
-        EXPECT_EQ(v[2], value_type{2.0});
-        EXPECT_EQ(v[3], value_type{5.0});
-    }
-
-    void assert_equal_to_mtx(const Sellp *m)
-    {
-        auto v = m->get_const_values();
-        auto c = m->get_const_col_idxs();
-        auto slice_sets = m->get_const_slice_sets();
-        auto slice_lengths = m->get_const_slice_lengths();
-        auto stride_factor = m->get_stride_factor();
-        auto slice_size = m->get_slice_size();
-
-        ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
-        ASSERT_EQ(stride_factor, 1);
-        ASSERT_EQ(slice_size, 64);
-        EXPECT_EQ(slice_sets[0], 0);
-        EXPECT_EQ(slice_lengths[0], 3);
-        EXPECT_EQ(c[0], 0);
-        EXPECT_EQ(c[1], 1);
-        EXPECT_EQ(c[64], 1);
-        EXPECT_EQ(c[65], 0);
-        EXPECT_EQ(c[128], 2);
-        EXPECT_EQ(c[129], 0);
-        EXPECT_EQ(v[0], value_type{1.0});
-        EXPECT_EQ(v[1], value_type{5.0});
-        EXPECT_EQ(v[64], value_type{3.0});
-        EXPECT_EQ(v[65], value_type{0.0});
-        EXPECT_EQ(v[128], value_type{2.0});
-        EXPECT_EQ(v[129], value_type{0.0});
+        ASSERT_EQ(m->get_size(), refcoomtx->get_size());
+        ASSERT_EQ(m->get_num_stored_elements(),
+                  refcoomtx->get_num_stored_elements());
+        for (index_type i = 0; i < m->get_num_stored_elements(); i++) {
+            ASSERT_EQ(m->get_const_row_idxs()[i],
+                      refcoomtx->get_const_row_idxs[i]);
+            ASSERT_EQ(m->get_const_col_idxs()[i],
+                      refcoomtx->get_const_col_idxs[i]);
+            ASSERT_EQ(m->get_const_values()[i], refcoomtx->get_const_values[i]);
+        }
     }
 
     void assert_equal_to_mtx(const SparsityCsr *m)
@@ -270,89 +162,13 @@ class Fbcsr : public ::testing::Test {
         EXPECT_EQ(c[3], 1);
     }
 
-    void assert_equal_to_mtx(const Ell *m)
-    {
-        auto v = m->get_const_values();
-        auto c = m->get_const_col_idxs();
-
-        ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
-        ASSERT_EQ(m->get_num_stored_elements(), 6);
-        EXPECT_EQ(c[0], 0);
-        EXPECT_EQ(c[1], 1);
-        EXPECT_EQ(c[2], 1);
-        EXPECT_EQ(c[3], 0);
-        EXPECT_EQ(c[4], 2);
-        EXPECT_EQ(c[5], 0);
-        EXPECT_EQ(v[0], value_type{1.0});
-        EXPECT_EQ(v[1], value_type{5.0});
-        EXPECT_EQ(v[2], value_type{3.0});
-        EXPECT_EQ(v[3], value_type{0.0});
-        EXPECT_EQ(v[4], value_type{2.0});
-        EXPECT_EQ(v[5], value_type{0.0});
-    }
-
-    void assert_equal_to_mtx(const Hybrid *m)
-    {
-        auto v = m->get_const_coo_values();
-        auto c = m->get_const_coo_col_idxs();
-        auto r = m->get_const_coo_row_idxs();
-        auto n = m->get_ell_num_stored_elements_per_row();
-        auto p = m->get_ell_stride();
-
-        ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
-        ASSERT_EQ(m->get_ell_num_stored_elements(), 0);
-        ASSERT_EQ(m->get_coo_num_stored_elements(), 4);
-        EXPECT_EQ(n, 0);
-        EXPECT_EQ(p, 2);
-        EXPECT_EQ(r[0], 0);
-        EXPECT_EQ(r[1], 0);
-        EXPECT_EQ(r[2], 0);
-        EXPECT_EQ(r[3], 1);
-        EXPECT_EQ(c[0], 0);
-        EXPECT_EQ(c[1], 1);
-        EXPECT_EQ(c[2], 2);
-        EXPECT_EQ(c[3], 1);
-        EXPECT_EQ(v[0], value_type{1.0});
-        EXPECT_EQ(v[1], value_type{3.0});
-        EXPECT_EQ(v[2], value_type{2.0});
-        EXPECT_EQ(v[3], value_type{5.0});
-    }
-
-    void assert_equal_to_mtx2(const Hybrid *m)
-    {
-        auto v = m->get_const_coo_values();
-        auto c = m->get_const_coo_col_idxs();
-        auto r = m->get_const_coo_row_idxs();
-        auto n = m->get_ell_num_stored_elements_per_row();
-        auto p = m->get_ell_stride();
-        auto ell_v = m->get_const_ell_values();
-        auto ell_c = m->get_const_ell_col_idxs();
-
-        ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
-        // Test Coo values
-        ASSERT_EQ(m->get_coo_num_stored_elements(), 1);
-        EXPECT_EQ(r[0], 0);
-        EXPECT_EQ(c[0], 2);
-        EXPECT_EQ(v[0], value_type{2.0});
-        // Test Ell values
-        ASSERT_EQ(m->get_ell_num_stored_elements(), 4);
-        EXPECT_EQ(n, 2);
-        EXPECT_EQ(p, 2);
-        EXPECT_EQ(ell_v[0], value_type{1});
-        EXPECT_EQ(ell_v[1], value_type{0});
-        EXPECT_EQ(ell_v[2], value_type{3});
-        EXPECT_EQ(ell_v[3], value_type{5});
-        EXPECT_EQ(ell_c[0], 0);
-        EXPECT_EQ(ell_c[1], 0);
-        EXPECT_EQ(ell_c[2], 1);
-        EXPECT_EQ(ell_c[3], 1);
-    }
-
     std::shared_ptr<const gko::ReferenceExecutor> exec;
+    const gko::testing::FbcsrSample<value_type, index_type> fbsample;
     std::unique_ptr<Mtx> mtx;
-    std::unique_ptr<Mtx> mtx2;
-    std::unique_ptr<Mtx> mtx3_sorted;
-    std::unique_ptr<Mtx> mtx3_unsorted;
+    const std::unique_ptr<const Mtx> refmtx;
+    const std::unique_ptr<const Csr> refcsrmtx;
+    const std::unique_ptr<const Dense> refdenmtx;
+    const std::unique_ptr<const Coo> refcoomtx;
 };
 
 TYPED_TEST_CASE(Fbcsr, gko::test::ValueIndexTypes);
@@ -627,63 +443,75 @@ GKO_NOT_IMPLEMENTED;
 
 
 TYPED_TEST(Fbcsr, ConvertsToDense)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Dense = typename TestFixture::Vec;
-//    auto dense_mtx = Dense::create(this->mtx->get_executor());
-//    auto dense_other = gko::initialize<Dense>(
-//        4, {{1.0, 3.0, 2.0}, {0.0, 5.0, 0.0}}, this->exec);
-//
-//    this->mtx->convert_to(dense_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dense_mtx, dense_other, 0.0);
-//}
+{
+    using Dense = typename TestFixture::Dense;
+    auto dense_mtx = Dense::create(this->mtx->get_executor());
+
+    this->mtx->convert_to(dense_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(dense_mtx, this->refdenmtx, 0.0);
+}
 
 
 TYPED_TEST(Fbcsr, MovesToDense)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Dense = typename TestFixture::Vec;
-//    auto dense_mtx = Dense::create(this->mtx->get_executor());
-//    auto dense_other = gko::initialize<Dense>(
-//        4, {{1.0, 3.0, 2.0}, {0.0, 5.0, 0.0}}, this->exec);
-//
-//    this->mtx->move_to(dense_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dense_mtx, dense_other, 0.0);
-//}
+{
+    using Dense = typename TestFixture::Dense;
+    auto dense_mtx = Dense::create(this->mtx->get_executor());
 
+    this->mtx->move_to(dense_mtx.get());
 
-TYPED_TEST(Fbcsr, ConvertsToCoo)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+    GKO_ASSERT_MTX_NEAR(dense_mtx, this->refdenmtx, 0.0);
+}
+
+
+TYPED_TEST(Fbcsr, ConvertsToCsr)
+{
+    using Csr = typename TestFixture::Csr;
+    auto csr_mtx = Csr::create(this->mtx->get_executor(),
+                               std::make_shared<typename Csr::classical>());
+
+    this->mtx->convert_to(csr_mtx.get());
+
+    this->assert_equal_to_mtx(csr_mtx.get());
+}
+
+
+TYPED_TEST(Fbcsr, MovesToCsr)
+{
+    using Csr = typename TestFixture::Csr;
+    auto csr_mtx = Csr::create(this->mtx->get_executor(),
+                               std::make_shared<typename Csr::classical>());
+
+    this->mtx->move_to(csr_mtx.get());
+
+    this->assert_equal_to_mtx(csr_mtx.get());
+}
+
+
+// TYPED_TEST(Fbcsr, ConvertsToCoo)
+// {
 //    using Coo = typename TestFixture::Coo;
 //    auto coo_mtx = Coo::create(this->mtx->get_executor());
-//
+
 //    this->mtx->convert_to(coo_mtx.get());
-//
+
 //    this->assert_equal_to_mtx(coo_mtx.get());
-//}
+// }
 
 
-TYPED_TEST(Fbcsr, MovesToCoo)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+// TYPED_TEST(Fbcsr, MovesToCoo)
+// {
 //    using Coo = typename TestFixture::Coo;
 //    auto coo_mtx = Coo::create(this->mtx->get_executor());
-//
+
 //    this->mtx->move_to(coo_mtx.get());
-//
+
 //    this->assert_equal_to_mtx(coo_mtx.get());
-//}
+// }
 
 
-TYPED_TEST(Fbcsr, ConvertsToSellp)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, ConvertsToSellp)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Sellp = typename TestFixture::Sellp;
@@ -695,8 +523,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, MovesToSellp)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, MovesToSellp)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Sellp = typename TestFixture::Sellp;
@@ -711,12 +539,12 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, ConvertsToSparsityFbcsr)
+TYPED_TEST(Fbcsr, ConvertsToSparsityCsr)
 GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using SparsityFbcsr = typename TestFixture::SparsityFbcsr;
-//    auto sparsity_mtx = SparsityFbcsr::create(this->mtx->get_executor());
+//    auto sparsity_mtx = SparsityCsr::create(this->mtx->get_executor());
 //
 //    this->mtx->convert_to(sparsity_mtx.get());
 //
@@ -724,13 +552,13 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, MovesToSparsityFbcsr)
+TYPED_TEST(Fbcsr, MovesToSparsityCsr)
 GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using SparsityFbcsr = typename TestFixture::SparsityFbcsr;
 //    using Fbcsr = typename TestFixture::Mtx;
-//    auto sparsity_mtx = SparsityFbcsr::create(this->mtx->get_executor());
+//    auto sparsity_mtx = SparsityCsr::create(this->mtx->get_executor());
 //    auto fbcsr_ref = Fbcsr::create(this->mtx->get_executor());
 //
 //    fbcsr_ref->copy_from(this->mtx.get());
@@ -740,8 +568,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, ConvertsToHybridAutomatically)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, ConvertsToHybridAutomatically)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Hybrid = typename TestFixture::Hybrid;
@@ -753,8 +581,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, MovesToHybridAutomatically)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, MovesToHybridAutomatically)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Hybrid = typename TestFixture::Hybrid;
@@ -769,8 +597,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, ConvertsToHybridByColumn2)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, ConvertsToHybridByColumn2)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Hybrid = typename TestFixture::Hybrid;
@@ -784,8 +612,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, MovesToHybridByColumn2)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, MovesToHybridByColumn2)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Hybrid = typename TestFixture::Hybrid;
@@ -845,75 +673,67 @@ GKO_NOT_IMPLEMENTED;
 
 
 TYPED_TEST(Fbcsr, ConvertsEmptyToDense)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using ValueType = typename TestFixture::value_type;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using Dense = gko::matrix::Dense<ValueType>;
-//    auto empty = Fbcsr::create(this->exec);
-//    auto res = Dense::create(this->exec);
-//
-//    empty->convert_to(res.get());
-//
-//    ASSERT_FALSE(res->get_size());
-//}
+{
+    using ValueType = typename TestFixture::value_type;
+    using Fbcsr = typename TestFixture::Mtx;
+    using Dense = typename TestFixture::Dense;
+    auto empty = Fbcsr::create(this->exec);
+    auto res = Dense::create(this->exec);
+
+    empty->convert_to(res.get());
+
+    ASSERT_FALSE(res->get_size());
+}
 
 
 TYPED_TEST(Fbcsr, MovesEmptyToDense)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using ValueType = typename TestFixture::value_type;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using Dense = gko::matrix::Dense<ValueType>;
-//    auto empty = Fbcsr::create(this->exec);
-//    auto res = Dense::create(this->exec);
-//
-//    empty->move_to(res.get());
-//
-//    ASSERT_FALSE(res->get_size());
-//}
+{
+    using ValueType = typename TestFixture::value_type;
+    using Fbcsr = typename TestFixture::Mtx;
+    using Dense = typename TestFixture::Dense;
+    auto empty = Fbcsr::create(this->exec);
+    auto res = Dense::create(this->exec);
 
+    empty->move_to(res.get());
 
-TYPED_TEST(Fbcsr, ConvertsEmptyToCoo)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+    ASSERT_FALSE(res->get_size());
+}
+
+
+// TYPED_TEST(Fbcsr, ConvertsEmptyToCoo)
+// {
 //    using ValueType = typename TestFixture::value_type;
 //    using IndexType = typename TestFixture::index_type;
 //    using Fbcsr = typename TestFixture::Mtx;
 //    using Coo = gko::matrix::Coo<ValueType, IndexType>;
 //    auto empty = Fbcsr::create(this->exec);
 //    auto res = Coo::create(this->exec);
-//
+
 //    empty->convert_to(res.get());
-//
+
 //    ASSERT_EQ(res->get_num_stored_elements(), 0);
 //    ASSERT_FALSE(res->get_size());
-//}
+// }
 
 
-TYPED_TEST(Fbcsr, MovesEmptyToCoo)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
+// TYPED_TEST(Fbcsr, MovesEmptyToCoo)
+// {
 //    using ValueType = typename TestFixture::value_type;
 //    using IndexType = typename TestFixture::index_type;
 //    using Fbcsr = typename TestFixture::Mtx;
 //    using Coo = gko::matrix::Coo<ValueType, IndexType>;
 //    auto empty = Fbcsr::create(this->exec);
 //    auto res = Coo::create(this->exec);
-//
+
 //    empty->move_to(res.get());
-//
+
 //    ASSERT_EQ(res->get_num_stored_elements(), 0);
 //    ASSERT_FALSE(res->get_size());
-//}
+// }
 
 
-TYPED_TEST(Fbcsr, ConvertsEmptyToEll)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, ConvertsEmptyToEll)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using ValueType = typename TestFixture::value_type;
@@ -930,8 +750,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, MovesEmptyToEll)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, MovesEmptyToEll)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using ValueType = typename TestFixture::value_type;
@@ -948,8 +768,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, ConvertsEmptyToSellp)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, ConvertsEmptyToSellp)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using ValueType = typename TestFixture::value_type;
@@ -967,8 +787,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, MovesEmptyToSellp)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, MovesEmptyToSellp)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using ValueType = typename TestFixture::value_type;
@@ -986,7 +806,7 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, ConvertsEmptyToSparsityFbcsr)
+TYPED_TEST(Fbcsr, ConvertsEmptyToSparsityCsr)
 GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
@@ -1005,7 +825,7 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, MovesEmptyToSparsityFbcsr)
+TYPED_TEST(Fbcsr, MovesEmptyToSparsityCsr)
 GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
@@ -1024,8 +844,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, ConvertsEmptyToHybrid)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, ConvertsEmptyToHybrid)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using ValueType = typename TestFixture::value_type;
@@ -1042,8 +862,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, MovesEmptyToHybrid)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, MovesEmptyToHybrid)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using ValueType = typename TestFixture::value_type;
@@ -1090,8 +910,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, ConvertsToEll)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, ConvertsToEll)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Ell = typename TestFixture::Ell;
@@ -1106,8 +926,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, MovesToEll)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, MovesToEll)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Ell = typename TestFixture::Ell;

From 585f8f19c59f7238e4a3f4545a4a4c5a7d672920 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Thu, 12 Nov 2020 10:47:14 +0100
Subject: [PATCH 0533/1008] [tests fail] conversion of FBCSR to sparsity CSR
 now works

---
 core/matrix/fbcsr.cpp                   | 110 +++++++++++++++++++-
 core/test/matrix/fbcsr_sample.cpp       |  14 +++
 core/test/matrix/fbcsr_sample.hpp       |   4 +
 include/ginkgo/core/matrix/fbcsr.hpp    | 111 ++------------------
 reference/test/matrix/fbcsr_kernels.cpp | 131 +++++++++++-------------
 5 files changed, 192 insertions(+), 178 deletions(-)

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 0d00e70bcc5..e4d88aa9717 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -179,6 +179,38 @@ void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
 //}
 
 
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::convert_to(
+    Fbcsr<ValueType, IndexType> *result) const
+{
+    bool same_executor = this->get_executor() == result->get_executor();
+    // NOTE: as soon as strategies are improved, this can be reverted
+    result->values_ = this->values_;
+    result->col_idxs_ = this->col_idxs_;
+    result->row_ptrs_ = this->row_ptrs_;
+    result->startrow_ = this->startrow_;
+    result->set_size(this->get_size());
+    result->bs_ = this->bs_;
+    if (!same_executor) {
+        convert_strategy_helper(result);
+    } else {
+        result->set_strategy(std::move(this->get_strategy()->copy()));
+    }
+    // END NOTE
+}
+
+
+template <typename ValueType, typename IndexType>
+void Fbcsr<ValueType, IndexType>::move_to(Fbcsr<ValueType, IndexType> *result)
+{
+    bool same_executor = this->get_executor() == result->get_executor();
+    EnableLinOp<Fbcsr>::move_to(result);
+    if (!same_executor) {
+        matrix_strategy::strategy_rebuild_helper(result);
+    }
+}
+
+
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(
     Fbcsr<next_precision<ValueType>, IndexType> *const result) const
@@ -332,16 +364,17 @@ template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(
     SparsityCsr<ValueType, IndexType> *result) const
 {
+    using gko::blockutils::getNumFixedBlocks;
     auto exec = this->get_executor();
     auto tmp = SparsityCsr<ValueType, IndexType>::create(
-        exec, this->get_size(), this->get_num_stored_elements());
+        exec,
+        gko::dim<2>{getNumFixedBlocks(bs_, this->get_size()[0]),
+                    getNumFixedBlocks(bs_, this->get_size()[1])},
+        getNumFixedBlocks(bs_ * bs_, this->get_num_stored_elements()));
+
     tmp->col_idxs_ = this->col_idxs_;
     tmp->row_ptrs_ = this->row_ptrs_;
-    // if (result->value_.get_data()) {
-    //     tmp->value_ = result->value_;
-    // } else {
     tmp->value_ = gko::Array<ValueType>(exec, {one<ValueType>()});
-    // }
     tmp->move_to(result);
 }
 
@@ -706,6 +739,73 @@ Fbcsr<ValueType, IndexType>::compute_absolute() const GKO_NOT_IMPLEMENTED;
 //}
 
 
+// TODO clean this up as soon as we improve strategy_type
+template <typename ValueType, typename IndexType>
+template <typename FbcsrType>
+void Fbcsr<ValueType, IndexType>::convert_strategy_helper(
+    FbcsrType *result) const
+{
+    auto strat = this->get_strategy().get();
+    std::shared_ptr<typename matrix_strategy::strategy_type<FbcsrType>>
+        new_strat;
+    using classical = matrix_strategy::classical<FbcsrType>;
+    using load_balance = matrix_strategy::load_balance<FbcsrType>;
+    using automatic = matrix_strategy::automatic<FbcsrType>;
+
+    if (dynamic_cast<classical *>(strat)) {
+        new_strat = std::make_shared<classical>();
+    } else {
+        auto rexec = result->get_executor();
+        auto cuda_exec = std::dynamic_pointer_cast<const CudaExecutor>(rexec);
+        auto hip_exec = std::dynamic_pointer_cast<const HipExecutor>(rexec);
+        auto lb = dynamic_cast<load_balance *>(strat);
+        if (cuda_exec) {
+            if (lb) {
+                new_strat = std::make_shared<load_balance>(cuda_exec);
+            } else {
+                new_strat = std::make_shared<automatic>(cuda_exec);
+            }
+        } else if (hip_exec) {
+            if (lb) {
+                new_strat = std::make_shared<load_balance>(hip_exec);
+            } else {
+                new_strat = std::make_shared<automatic>(hip_exec);
+            }
+        } else {
+            // Try to preserve this executor's configuration
+            auto this_cuda_exec = std::dynamic_pointer_cast<const CudaExecutor>(
+                this->get_executor());
+            auto this_hip_exec = std::dynamic_pointer_cast<const HipExecutor>(
+                this->get_executor());
+            if (this_cuda_exec) {
+                if (lb) {
+                    new_strat = std::make_shared<load_balance>(this_cuda_exec);
+                } else {
+                    new_strat = std::make_shared<automatic>(this_cuda_exec);
+                }
+            } else if (this_hip_exec) {
+                if (lb) {
+                    new_strat = std::make_shared<load_balance>(this_hip_exec);
+                } else {
+                    new_strat = std::make_shared<automatic>(this_hip_exec);
+                }
+            } else {
+                // We had a load balance or automatic strategy from a non
+                // HIP or Cuda executor and are moving to a non HIP or Cuda
+                // executor.
+                // FIXME this creates a long delay
+                if (lb) {
+                    new_strat = std::make_shared<load_balance>();
+                } else {
+                    new_strat = std::make_shared<automatic>();
+                }
+            }
+        }
+    }
+    result->set_strategy(new_strat);
+}
+
+
 #define GKO_DECLARE_FBCSR_MATRIX(ValueType, IndexType) \
     class Fbcsr<ValueType, IndexType>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_MATRIX);
diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
index a221cd3381e..035cdcefca8 100644
--- a/core/test/matrix/fbcsr_sample.cpp
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -314,6 +314,20 @@ FbcsrSample<ValueType, IndexType>::generate_coo() const
     return mat;
 }
 
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::SparsityCsr<ValueType, IndexType>>
+FbcsrSample<ValueType, IndexType>::generate_sparsity_csr() const
+{
+    gko::Array<IndexType> colids(exec, nbnz);
+    gko::Array<IndexType> rowptrs(exec, nbrows + 1);
+    const std::unique_ptr<const Fbcsr> fbmat = generate_fbcsr();
+    for (index_type i = 0; i < nbrows + 1; i++)
+        rowptrs.get_data()[i] = fbmat->get_row_ptrs()[i];
+    for (index_type i = 0; i < nbnz; i++)
+        colids.get_data()[i] = fbmat->get_col_idxs()[i];
+    return SparCsr::create(exec, gko::dim<2>{nbrows, nbcols}, colids, rowptrs);
+}
+
 #define GKO_DECLARE_FBCSR_TEST_SAMPLE(ValueType, IndexType) \
     class FbcsrSample<ValueType, IndexType>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_TEST_SAMPLE);
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 7fe2fd3cb32..5df96e6d64f 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/fbcsr.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 namespace gko {
 namespace testing {
@@ -57,6 +58,7 @@ class FbcsrSample {
     using Coo = gko::matrix::Coo<value_type, index_type>;
     using Dense = gko::matrix::Dense<value_type>;
     using MatData = gko::matrix_data<value_type, index_type>;
+    using SparCsr = gko::matrix::SparsityCsr<value_type, index_type>;
 
     FbcsrSample(std::shared_ptr<const gko::ReferenceExecutor> exec);
 
@@ -72,6 +74,8 @@ class FbcsrSample {
      */
     std::unique_ptr<Coo> generate_coo() const;
 
+    std::unique_ptr<SparCsr> generate_sparsity_csr() const;
+
     MatData generate_matrix_data() const;
 
     MatData generate_matrix_data_with_explicit_zeros() const;
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index bfbab84eabe..836169ba0c3 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -160,32 +160,10 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
         matrix_strategy::strategy_type<Fbcsr<value_type, index_type>>;
 
 
-    void convert_to(Fbcsr<ValueType, IndexType> *result) const override
-    {
-        bool same_executor = this->get_executor() == result->get_executor();
-        // NOTE: as soon as strategies are improved, this can be reverted
-        result->values_ = this->values_;
-        result->col_idxs_ = this->col_idxs_;
-        result->row_ptrs_ = this->row_ptrs_;
-        result->startrow_ = this->startrow_;
-        result->set_size(this->get_size());
-        result->bs_ = this->bs_;
-        if (!same_executor) {
-            convert_strategy_helper(result);
-        } else {
-            result->set_strategy(std::move(this->get_strategy()->copy()));
-        }
-        // END NOTE
-    }
+    void convert_to(Fbcsr<ValueType, IndexType> *result) const override;
+
+    void move_to(Fbcsr<ValueType, IndexType> *result) override;
 
-    void move_to(Fbcsr<ValueType, IndexType> *result) override
-    {
-        bool same_executor = this->get_executor() == result->get_executor();
-        EnableLinOp<Fbcsr>::move_to(result);
-        if (!same_executor) {
-            matrix_strategy::strategy_rebuild_helper(result);
-        }
-    }
     friend class Fbcsr<next_precision<ValueType>, IndexType>;
 
     void convert_to(
@@ -205,18 +183,10 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
 
     void move_to(Coo<ValueType, IndexType> *result) override;
 
-    // void convert_to(Ell<ValueType, IndexType> *result) const override;
-
-    // void move_to(Ell<ValueType, IndexType> *result) override;
-
-    // void convert_to(Hybrid<ValueType, IndexType> *result) const override;
-
-    // void move_to(Hybrid<ValueType, IndexType> *result) override;
-
-    // void convert_to(Sellp<ValueType, IndexType> *result) const override;
-
-    // void move_to(Sellp<ValueType, IndexType> *result) override;
-
+    /// Get the block sparsity pattern in CSR-like format
+    /** Note that the actual non-zero values are never copied;
+     * the result always has a value array of size 1 with the value 1.
+     */
     void convert_to(SparsityCsr<ValueType, IndexType> *result) const override;
 
     void move_to(SparsityCsr<ValueType, IndexType> *result) override;
@@ -483,72 +453,7 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
 
     // TODO clean this up as soon as we improve strategy_type
     template <typename FbcsrType>
-    void convert_strategy_helper(FbcsrType *result) const
-    {
-        auto strat = this->get_strategy().get();
-        std::shared_ptr<typename matrix_strategy::strategy_type<FbcsrType>>
-            new_strat;
-        using classical = matrix_strategy::classical<FbcsrType>;
-        using load_balance = matrix_strategy::load_balance<FbcsrType>;
-        using automatic = matrix_strategy::automatic<FbcsrType>;
-
-        if (dynamic_cast<classical *>(strat)) {
-            new_strat = std::make_shared<classical>();
-        } else {
-            auto rexec = result->get_executor();
-            auto cuda_exec =
-                std::dynamic_pointer_cast<const CudaExecutor>(rexec);
-            auto hip_exec = std::dynamic_pointer_cast<const HipExecutor>(rexec);
-            auto lb = dynamic_cast<load_balance *>(strat);
-            if (cuda_exec) {
-                if (lb) {
-                    new_strat = std::make_shared<load_balance>(cuda_exec);
-                } else {
-                    new_strat = std::make_shared<automatic>(cuda_exec);
-                }
-            } else if (hip_exec) {
-                if (lb) {
-                    new_strat = std::make_shared<load_balance>(hip_exec);
-                } else {
-                    new_strat = std::make_shared<automatic>(hip_exec);
-                }
-            } else {
-                // Try to preserve this executor's configuration
-                auto this_cuda_exec =
-                    std::dynamic_pointer_cast<const CudaExecutor>(
-                        this->get_executor());
-                auto this_hip_exec =
-                    std::dynamic_pointer_cast<const HipExecutor>(
-                        this->get_executor());
-                if (this_cuda_exec) {
-                    if (lb) {
-                        new_strat =
-                            std::make_shared<load_balance>(this_cuda_exec);
-                    } else {
-                        new_strat = std::make_shared<automatic>(this_cuda_exec);
-                    }
-                } else if (this_hip_exec) {
-                    if (lb) {
-                        new_strat =
-                            std::make_shared<load_balance>(this_hip_exec);
-                    } else {
-                        new_strat = std::make_shared<automatic>(this_hip_exec);
-                    }
-                } else {
-                    // We had a load balance or automatic strategy from a non
-                    // HIP or Cuda executor and are moving to a non HIP or Cuda
-                    // executor.
-                    // FIXME this creates a long delay
-                    if (lb) {
-                        new_strat = std::make_shared<load_balance>();
-                    } else {
-                        new_strat = std::make_shared<automatic>();
-                    }
-                }
-            }
-        }
-        result->set_strategy(new_strat);
-    }
+    void convert_strategy_helper(FbcsrType *result) const;
 
     /**
      * Computes srow. It should be run after changing any row_ptrs_ value.
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 8fb53f4e3b2..f20fe456b29 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -80,7 +80,7 @@ class Fbcsr : public ::testing::Test {
     using Coo = gko::matrix::Coo<value_type, index_type>;
     using Dense = gko::matrix::Dense<value_type>;
     using Sellp = gko::matrix::Sellp<value_type, index_type>;
-    using SparsityCsr = gko::matrix::SparsityCsr<value_type, index_type>;
+    using SparCsr = gko::matrix::SparsityCsr<value_type, index_type>;
     using Ell = gko::matrix::Ell<value_type, index_type>;
     using Hybrid = gko::matrix::Hybrid<value_type, index_type>;
     using Vec = gko::matrix::Dense<value_type>;
@@ -92,7 +92,8 @@ class Fbcsr : public ::testing::Test {
           refmtx(fbsample.generate_fbcsr()),
           refcsrmtx(fbsample.generate_csr()),
           refdenmtx(fbsample.generate_dense()),
-          refcoomtx(fbsample.generate_coo())
+          refcoomtx(fbsample.generate_coo()),
+          refspcmtx(fbsample.generate_sparsity_csr())
     {}
 
     // void create_mtx3(Mtx *sorted, Mtx *unsorted)
@@ -146,20 +147,17 @@ class Fbcsr : public ::testing::Test {
         }
     }
 
-    void assert_equal_to_mtx(const SparsityCsr *m)
+    void assert_equal_to_mtx(const SparCsr *m)
     {
-        auto *c = m->get_const_col_idxs();
-        auto *r = m->get_const_row_ptrs();
-
-        ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3));
-        ASSERT_EQ(m->get_num_nonzeros(), 4);
-        EXPECT_EQ(r[0], 0);
-        EXPECT_EQ(r[1], 3);
-        EXPECT_EQ(r[2], 4);
-        EXPECT_EQ(c[0], 0);
-        EXPECT_EQ(c[1], 1);
-        EXPECT_EQ(c[2], 2);
-        EXPECT_EQ(c[3], 1);
+        ASSERT_EQ(m->get_size(), refspcmtx->get_size());
+        ASSERT_EQ(m->get_num_nonzeros(), refspcmtx->get_num_nonzeros());
+        for (index_type i = 0; i < m->get_size()[0] + 1; i++)
+            ASSERT_EQ(m->get_const_row_ptrs()[i],
+                      refspcmtx->get_const_row_ptrs()[i]);
+        for (index_type i = 0; i < m->get_num_nonzeros(); i++) {
+            ASSERT_EQ(m->get_const_col_idxs()[i],
+                      refspcmtx->get_const_col_idxs()[i]);
+        }
     }
 
     std::shared_ptr<const gko::ReferenceExecutor> exec;
@@ -169,6 +167,7 @@ class Fbcsr : public ::testing::Test {
     const std::unique_ptr<const Csr> refcsrmtx;
     const std::unique_ptr<const Dense> refdenmtx;
     const std::unique_ptr<const Coo> refcoomtx;
+    const std::unique_ptr<const SparCsr> refspcmtx;
 };
 
 TYPED_TEST_CASE(Fbcsr, gko::test::ValueIndexTypes);
@@ -540,32 +539,28 @@ TYPED_TEST(Fbcsr, MovesToCsr)
 
 
 TYPED_TEST(Fbcsr, ConvertsToSparsityCsr)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using SparsityFbcsr = typename TestFixture::SparsityFbcsr;
-//    auto sparsity_mtx = SparsityCsr::create(this->mtx->get_executor());
-//
-//    this->mtx->convert_to(sparsity_mtx.get());
-//
-//    this->assert_equal_to_mtx(sparsity_mtx.get());
-//}
+{
+    using SparsityCsr = typename TestFixture::SparCsr;
+    auto sparsity_mtx = SparsityCsr::create(this->mtx->get_executor());
+
+    this->mtx->convert_to(sparsity_mtx.get());
+
+    this->assert_equal_to_mtx(sparsity_mtx.get());
+}
 
 
 TYPED_TEST(Fbcsr, MovesToSparsityCsr)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using SparsityFbcsr = typename TestFixture::SparsityFbcsr;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    auto sparsity_mtx = SparsityCsr::create(this->mtx->get_executor());
-//    auto fbcsr_ref = Fbcsr::create(this->mtx->get_executor());
-//
-//    fbcsr_ref->copy_from(this->mtx.get());
-//    fbcsr_ref->move_to(sparsity_mtx.get());
-//
-//    this->assert_equal_to_mtx(sparsity_mtx.get());
-//}
+{
+    using SparsityCsr = typename TestFixture::SparCsr;
+    using Fbcsr = typename TestFixture::Mtx;
+    auto sparsity_mtx = SparsityCsr::create(this->mtx->get_executor());
+    // auto fbcsr_ref = Fbcsr::create(this->mtx->get_executor());
+
+    // fbcsr_ref->copy_from(this->mtx.get());
+    this->mtx->move_to(sparsity_mtx.get());
+
+    this->assert_equal_to_mtx(sparsity_mtx.get());
+}
 
 
 // TYPED_TEST(Fbcsr, ConvertsToHybridAutomatically)
@@ -807,41 +802,37 @@ TYPED_TEST(Fbcsr, MovesEmptyToDense)
 
 
 TYPED_TEST(Fbcsr, ConvertsEmptyToSparsityCsr)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using ValueType = typename TestFixture::value_type;
-//    using IndexType = typename TestFixture::index_type;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using SparsityFbcsr = gko::matrix::SparsityFbcsr<ValueType, IndexType>;
-//    auto empty = Fbcsr::create(this->exec);
-//    empty->get_row_ptrs()[0] = 0;
-//    auto res = SparsityFbcsr::create(this->exec);
-//
-//    empty->convert_to(res.get());
-//
-//    ASSERT_EQ(res->get_num_nonzeros(), 0);
-//    ASSERT_EQ(*res->get_const_row_ptrs(), 0);
-//}
+{
+    using ValueType = typename TestFixture::value_type;
+    using IndexType = typename TestFixture::index_type;
+    using Fbcsr = typename TestFixture::Mtx;
+    using SparCsr = typename TestFixture::SparCsr;
+    auto empty = Fbcsr::create(this->exec);
+    empty->get_row_ptrs()[0] = 0;
+    auto res = SparCsr::create(this->exec);
+
+    empty->convert_to(res.get());
+
+    ASSERT_EQ(res->get_num_nonzeros(), 0);
+    ASSERT_EQ(*res->get_const_row_ptrs(), 0);
+}
 
 
 TYPED_TEST(Fbcsr, MovesEmptyToSparsityCsr)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using ValueType = typename TestFixture::value_type;
-//    using IndexType = typename TestFixture::index_type;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using SparsityFbcsr = gko::matrix::SparsityFbcsr<ValueType, IndexType>;
-//    auto empty = Fbcsr::create(this->exec);
-//    empty->get_row_ptrs()[0] = 0;
-//    auto res = SparsityFbcsr::create(this->exec);
-//
-//    empty->move_to(res.get());
-//
-//    ASSERT_EQ(res->get_num_nonzeros(), 0);
-//    ASSERT_EQ(*res->get_const_row_ptrs(), 0);
-//}
+{
+    using ValueType = typename TestFixture::value_type;
+    using IndexType = typename TestFixture::index_type;
+    using Fbcsr = typename TestFixture::Mtx;
+    using SparCsr = typename TestFixture::SparCsr;
+    auto empty = Fbcsr::create(this->exec);
+    empty->get_row_ptrs()[0] = 0;
+    auto res = SparCsr::create(this->exec);
+
+    empty->move_to(res.get());
+
+    ASSERT_EQ(res->get_num_nonzeros(), 0);
+    ASSERT_EQ(*res->get_const_row_ptrs(), 0);
+}
 
 
 // TYPED_TEST(Fbcsr, ConvertsEmptyToHybrid)

From 325a5e14c7a578e001d7016431052b7cde705eb3 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Thu, 12 Nov 2020 11:34:20 +0100
Subject: [PATCH 0534/1008] [tests fail] enabled tests for fbcsr precision
 conversion

---
 reference/test/matrix/fbcsr_kernels.cpp | 154 +++++++++++-------------
 1 file changed, 72 insertions(+), 82 deletions(-)

diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index f20fe456b29..dbba82dc3e4 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -390,55 +390,49 @@ GKO_NOT_IMPLEMENTED;
 
 
 TYPED_TEST(Fbcsr, ConvertsToPrecision)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using ValueType = typename TestFixture::value_type;
-//    using IndexType = typename TestFixture::index_type;
-//    using OtherType = typename gko::next_precision<ValueType>;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using OtherFbcsr = gko::matrix::Fbcsr<OtherType, IndexType>;
-//    auto tmp = OtherFbcsr::create(this->exec);
-//    auto res = Fbcsr::create(this->exec);
-//    // If OtherType is more precise: 0, otherwise r
-//    auto residual = r<OtherType>::value < r<ValueType>::value
-//                        ? gko::remove_complex<ValueType>{0}
-//                        : gko::remove_complex<ValueType>{r<OtherType>::value};
-//
-//    // use mtx2 as mtx's strategy would involve creating a CudaExecutor
-//    this->mtx2->convert_to(tmp.get());
-//    tmp->convert_to(res.get());
-//
-//    GKO_ASSERT_MTX_NEAR(this->mtx2, res, residual);
-//    ASSERT_EQ(typeid(*this->mtx2->get_strategy()),
-//              typeid(*res->get_strategy()));
-//}
+{
+    using ValueType = typename TestFixture::value_type;
+    using IndexType = typename TestFixture::index_type;
+    using OtherType = typename gko::next_precision<ValueType>;
+    using Fbcsr = typename TestFixture::Mtx;
+    using OtherFbcsr = gko::matrix::Fbcsr<OtherType, IndexType>;
+    auto tmp = OtherFbcsr::create(this->exec);
+    auto res = Fbcsr::create(this->exec);
+    // If OtherType is more precise: 0, otherwise r
+    auto residual = r<OtherType>::value < r<ValueType>::value
+                        ? gko::remove_complex<ValueType>{0}
+                        : gko::remove_complex<ValueType>{r<OtherType>::value};
+
+    this->mtx->convert_to(tmp.get());
+    tmp->convert_to(res.get());
+
+    GKO_ASSERT_MTX_NEAR(this->mtx, res, residual);
+    // ASSERT_EQ(typeid(*this->mtx->get_strategy()),
+    //           typeid(*res->get_strategy()));
+}
 
 
 TYPED_TEST(Fbcsr, MovesToPrecision)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using ValueType = typename TestFixture::value_type;
-//    using IndexType = typename TestFixture::index_type;
-//    using OtherType = typename gko::next_precision<ValueType>;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using OtherFbcsr = gko::matrix::Fbcsr<OtherType, IndexType>;
-//    auto tmp = OtherFbcsr::create(this->exec);
-//    auto res = Fbcsr::create(this->exec);
-//    // If OtherType is more precise: 0, otherwise r
-//    auto residual = r<OtherType>::value < r<ValueType>::value
-//                        ? gko::remove_complex<ValueType>{0}
-//                        : gko::remove_complex<ValueType>{r<OtherType>::value};
-//
-//    // use mtx2 as mtx's strategy would involve creating a CudaExecutor
-//    this->mtx2->move_to(tmp.get());
-//    tmp->move_to(res.get());
-//
-//    GKO_ASSERT_MTX_NEAR(this->mtx2, res, residual);
-//    ASSERT_EQ(typeid(*this->mtx2->get_strategy()),
-//              typeid(*res->get_strategy()));
-//}
+{
+    using ValueType = typename TestFixture::value_type;
+    using IndexType = typename TestFixture::index_type;
+    using OtherType = typename gko::next_precision<ValueType>;
+    using Fbcsr = typename TestFixture::Mtx;
+    using OtherFbcsr = gko::matrix::Fbcsr<OtherType, IndexType>;
+    auto tmp = OtherFbcsr::create(this->exec);
+    auto res = Fbcsr::create(this->exec);
+    // If OtherType is more precise: 0, otherwise r
+    auto residual = r<OtherType>::value < r<ValueType>::value
+                        ? gko::remove_complex<ValueType>{0}
+                        : gko::remove_complex<ValueType>{r<OtherType>::value};
+
+    this->mtx->move_to(tmp.get());
+    tmp->move_to(res.get());
+
+    GKO_ASSERT_MTX_NEAR(this->mtx, res, residual);
+    // ASSERT_EQ(typeid(*this->mtx->get_strategy()),
+    //           typeid(*res->get_strategy()));
+}
 
 
 TYPED_TEST(Fbcsr, ConvertsToDense)
@@ -626,45 +620,41 @@ TYPED_TEST(Fbcsr, MovesToSparsityCsr)
 
 
 TYPED_TEST(Fbcsr, ConvertsEmptyToPrecision)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using ValueType = typename TestFixture::value_type;
-//    using IndexType = typename TestFixture::index_type;
-//    using OtherType = typename gko::next_precision<ValueType>;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using OtherFbcsr = gko::matrix::Fbcsr<OtherType, IndexType>;
-//    auto empty = OtherFbcsr::create(this->exec);
-//    empty->get_row_ptrs()[0] = 0;
-//    auto res = Fbcsr::create(this->exec);
-//
-//    empty->convert_to(res.get());
-//
-//    ASSERT_EQ(res->get_num_stored_elements(), 0);
-//    ASSERT_EQ(*res->get_const_row_ptrs(), 0);
-//    ASSERT_FALSE(res->get_size());
-//}
+{
+    using ValueType = typename TestFixture::value_type;
+    using IndexType = typename TestFixture::index_type;
+    using OtherType = typename gko::next_precision<ValueType>;
+    using Fbcsr = typename TestFixture::Mtx;
+    using OtherFbcsr = gko::matrix::Fbcsr<OtherType, IndexType>;
+    auto empty = OtherFbcsr::create(this->exec);
+    empty->get_row_ptrs()[0] = 0;
+    auto res = Fbcsr::create(this->exec);
+
+    empty->convert_to(res.get());
+
+    ASSERT_EQ(res->get_num_stored_elements(), 0);
+    ASSERT_EQ(*res->get_const_row_ptrs(), 0);
+    ASSERT_FALSE(res->get_size());
+}
 
 
 TYPED_TEST(Fbcsr, MovesEmptyToPrecision)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using ValueType = typename TestFixture::value_type;
-//    using IndexType = typename TestFixture::index_type;
-//    using OtherType = typename gko::next_precision<ValueType>;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using OtherFbcsr = gko::matrix::Fbcsr<OtherType, IndexType>;
-//    auto empty = OtherFbcsr::create(this->exec);
-//    empty->get_row_ptrs()[0] = 0;
-//    auto res = Fbcsr::create(this->exec);
-//
-//    empty->move_to(res.get());
-//
-//    ASSERT_EQ(res->get_num_stored_elements(), 0);
-//    ASSERT_EQ(*res->get_const_row_ptrs(), 0);
-//    ASSERT_FALSE(res->get_size());
-//}
+{
+    using ValueType = typename TestFixture::value_type;
+    using IndexType = typename TestFixture::index_type;
+    using OtherType = typename gko::next_precision<ValueType>;
+    using Fbcsr = typename TestFixture::Mtx;
+    using OtherFbcsr = gko::matrix::Fbcsr<OtherType, IndexType>;
+    auto empty = OtherFbcsr::create(this->exec);
+    empty->get_row_ptrs()[0] = 0;
+    auto res = Fbcsr::create(this->exec);
+
+    empty->move_to(res.get());
+
+    ASSERT_EQ(res->get_num_stored_elements(), 0);
+    ASSERT_EQ(*res->get_const_row_ptrs(), 0);
+    ASSERT_FALSE(res->get_size());
+}
 
 
 TYPED_TEST(Fbcsr, ConvertsEmptyToDense)

From e1e30a78fbfc76eacc2d871478165802f33beefe Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Fri, 13 Nov 2020 17:16:47 +0100
Subject: [PATCH 0535/1008] [tests fail] ref spmv and generalized spmv work and
 are tested

---
 core/matrix/fbcsr.cpp                   |  77 +++----
 core/test/matrix/fbcsr_sample.cpp       | 234 ++++++++++++++++---
 core/test/matrix/fbcsr_sample.hpp       |  61 +++++
 reference/matrix/fbcsr_kernels.cpp      | 234 +++++++++++--------
 reference/test/matrix/fbcsr_kernels.cpp | 294 +++++++++++++++---------
 5 files changed, 628 insertions(+), 272 deletions(-)

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index e4d88aa9717..68defa46f4b 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -62,9 +62,9 @@ namespace fbcsr {
 
 
 GKO_REGISTER_OPERATION(spmv, fbcsr::spmv);
-// GKO_REGISTER_OPERATION(advanced_spmv, fbcsr::advanced_spmv);
+GKO_REGISTER_OPERATION(advanced_spmv, fbcsr::advanced_spmv);
 GKO_REGISTER_OPERATION(spgemm, fbcsr::spgemm);
-// GKO_REGISTER_OPERATION(advanced_spgemm, fbcsr::advanced_spgemm);
+GKO_REGISTER_OPERATION(advanced_spgemm, fbcsr::advanced_spgemm);
 GKO_REGISTER_OPERATION(spgeam, fbcsr::spgeam);
 GKO_REGISTER_OPERATION(convert_to_coo, fbcsr::convert_to_coo);
 GKO_REGISTER_OPERATION(convert_to_csr, fbcsr::convert_to_csr);
@@ -131,7 +131,6 @@ template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const b,
                                              LinOp *const x) const
 {
-    // TODO (script:fbcsr): change the code imported from matrix/csr if needed
     using Dense = Dense<ValueType>;
     using TFbcsr = Fbcsr<ValueType, IndexType>;
     if (auto b_fbcsr = dynamic_cast<const TFbcsr *>(b)) {
@@ -151,32 +150,29 @@ void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const b,
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                              const LinOp *beta, LinOp *x) const
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Dense = Dense<ValueType>;
-//    using TFbcsr = Fbcsr<ValueType, IndexType>;
-//    if (auto b_fbcsr = dynamic_cast<const TFbcsr *>(b)) {
-//        // if b is a FBCSR matrix, we compute a SpGeMM
-//        auto x_fbcsr = as<TFbcsr>(x);
-//        auto x_copy = x_fbcsr->clone();
-//        this->get_executor()->run(
-//            fbcsr::make_advanced_spgemm(as<Dense>(alpha), this, b_fbcsr,
-//                                      as<Dense>(beta), x_copy.get(),
-//                                      x_fbcsr));
-//    } else if (dynamic_cast<const Identity<ValueType> *>(b)) {
-//        // if b is an identity matrix, we compute an SpGEAM
-//        auto x_fbcsr = as<TFbcsr>(x);
-//        auto x_copy = x_fbcsr->clone();
-//        this->get_executor()->run(fbcsr::make_spgeam(
-//            as<Dense>(alpha), this, as<Dense>(beta), lend(x_copy), x_fbcsr));
-//    } else {
-//        // otherwise we assume that b is dense and compute a SpMV/SpMM
-//        this->get_executor()->run(
-//            fbcsr::make_advanced_spmv(as<Dense>(alpha), this, as<Dense>(b),
-//                                    as<Dense>(beta), as<Dense>(x)));
-//    }
-//}
+{
+    using Dense = Dense<ValueType>;
+    using TFbcsr = Fbcsr<ValueType, IndexType>;
+    if (auto b_fbcsr = dynamic_cast<const TFbcsr *>(b)) {
+        // if b is a FBCSR matrix, we compute a SpGeMM
+        auto x_fbcsr = as<TFbcsr>(x);
+        auto x_copy = x_fbcsr->clone();
+        this->get_executor()->run(fbcsr::make_advanced_spgemm(
+            as<Dense>(alpha), this, b_fbcsr, as<Dense>(beta), x_copy.get(),
+            x_fbcsr));
+    } else if (dynamic_cast<const Identity<ValueType> *>(b)) {
+        // if b is an identity matrix, we compute an SpGEAM
+        auto x_fbcsr = as<TFbcsr>(x);
+        auto x_copy = x_fbcsr->clone();
+        this->get_executor()->run(fbcsr::make_spgeam(
+            as<Dense>(alpha), this, as<Dense>(beta), lend(x_copy), x_fbcsr));
+    } else {
+        // otherwise we assume that b is dense and compute a SpMV/SpMM
+        this->get_executor()->run(
+            fbcsr::make_advanced_spmv(as<Dense>(alpha), this, as<Dense>(b),
+                                      as<Dense>(beta), as<Dense>(x)));
+    }
+}
 
 
 template <typename ValueType, typename IndexType>
@@ -692,18 +688,17 @@ bool Fbcsr<ValueType, IndexType>::is_sorted_by_column_index() const
 
 template <typename ValueType, typename IndexType>
 std::unique_ptr<Diagonal<ValueType>>
-Fbcsr<ValueType, IndexType>::extract_diagonal() const GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto exec = this->get_executor();
-//
-//    const auto diag_size = std::min(this->get_size()[0], this->get_size()[1]);
-//    auto diag = Diagonal<ValueType>::create(exec, diag_size);
-//    exec->run(fbcsr::make_fill_array(diag->get_values(), diag->get_size()[0],
-//                                   zero<ValueType>()));
-//    exec->run(fbcsr::make_extract_diagonal(this, lend(diag)));
-//    return diag;
-//}
+Fbcsr<ValueType, IndexType>::extract_diagonal() const
+{
+    auto exec = this->get_executor();
+
+    const auto diag_size = std::min(this->get_size()[0], this->get_size()[1]);
+    auto diag = Diagonal<ValueType>::create(exec, diag_size);
+    exec->run(fbcsr::make_fill_array(diag->get_values(), diag->get_size()[0],
+                                     zero<ValueType>()));
+    exec->run(fbcsr::make_extract_diagonal(this, lend(diag)));
+    return diag;
+}
 
 
 template <typename ValueType, typename IndexType>
diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
index 035cdcefca8..f8d8136e23f 100644
--- a/core/test/matrix/fbcsr_sample.cpp
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -38,6 +38,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/fixed_block.hpp"
 #include "fbcsr_sample.hpp"
 
+#define FBCSR_TEST_OFFSET 0.000011118888
+
 namespace gko {
 namespace testing {
 
@@ -104,6 +106,9 @@ FbcsrSample<ValueType, IndexType>::generate_fbcsr() const
     vals(0, 2, 2) = gko::zero<value_type>();
     vals(3, 0, 0) = gko::zero<value_type>();
 
+    v[34] += FBCSR_TEST_OFFSET;
+    v[35] += FBCSR_TEST_OFFSET;
+
     for (index_type is = 0; is < mtx->get_num_srow_elements(); is++) s[is] = 0;
 
     return mtx;
@@ -209,6 +214,9 @@ FbcsrSample<ValueType, IndexType>::generate_csr() const
     csrvals[34] = 13;
     csrvals[35] = 14;
 
+    csrvals[34] += FBCSR_TEST_OFFSET;
+    csrvals[35] += FBCSR_TEST_OFFSET;
+
     return csrm;
 }
 
@@ -233,6 +241,8 @@ FbcsrSample<ValueType, IndexType>::generate_dense() const
         }
 
     densem->at(2, 3) = densem->at(2, 5) = densem->at(3, 6) = 0.0;
+    densem->at(5, 7) += FBCSR_TEST_OFFSET;
+    densem->at(5, 8) += FBCSR_TEST_OFFSET;
 
     return densem;
 }
@@ -242,20 +252,43 @@ template <typename ValueType, typename IndexType>
 gko::matrix_data<ValueType, IndexType>
 FbcsrSample<ValueType, IndexType>::generate_matrix_data() const
 {
-    return MatData(
-        {{6, 12}, {{0, 3, 2.0},   {0, 4, 3.0},  {0, 5, 4.0},  {1, 3, 5.0},
-                   {1, 4, 6.0},   {1, 5, 7.0},  {2, 4, 9.0},
-
-                   {0, 9, 4.0},   {0, 10, 5.0}, {0, 11, 6.0}, {1, 9, 7.0},
-                   {1, 10, 8.0},  {1, 11, 9.0}, {2, 9, 10.0}, {2, 10, 11.0},
-                   {2, 11, 12.0},
-
-                   {3, 0, 2.0},   {3, 1, 3.0},  {3, 2, 4.0},  {4, 0, 5.0},
-                   {4, 1, 6.0},   {4, 2, 7.0},  {5, 0, 8.0},  {5, 1, 9.0},
-                   {5, 2, 10.0},
-
-                   {3, 7, 7.0},   {3, 8, 8.0},  {4, 6, 9.0},  {4, 7, 10.0},
-                   {4, 8, 11.0},  {5, 6, 12.0}, {5, 7, 13.0}, {5, 8, 14.0}}});
+    return MatData({{6, 12},
+                    {{0, 3, 2.0},
+                     {0, 4, 3.0},
+                     {0, 5, 4.0},
+                     {1, 3, 5.0},
+                     {1, 4, 6.0},
+                     {1, 5, 7.0},
+                     {2, 4, 9.0},
+
+                     {0, 9, 4.0},
+                     {0, 10, 5.0},
+                     {0, 11, 6.0},
+                     {1, 9, 7.0},
+                     {1, 10, 8.0},
+                     {1, 11, 9.0},
+                     {2, 9, 10.0},
+                     {2, 10, 11.0},
+                     {2, 11, 12.0},
+
+                     {3, 0, 2.0},
+                     {3, 1, 3.0},
+                     {3, 2, 4.0},
+                     {4, 0, 5.0},
+                     {4, 1, 6.0},
+                     {4, 2, 7.0},
+                     {5, 0, 8.0},
+                     {5, 1, 9.0},
+                     {5, 2, 10.0},
+
+                     {3, 7, 7.0},
+                     {3, 8, 8.0},
+                     {4, 6, 9.0},
+                     {4, 7, 10.0},
+                     {4, 8, 11.0},
+                     {5, 6, 12.0},
+                     {5, 7, 13.0 + FBCSR_TEST_OFFSET},
+                     {5, 8, 14.0 + FBCSR_TEST_OFFSET}}});
 }
 
 // Assuming row-major blocks
@@ -263,21 +296,46 @@ template <typename ValueType, typename IndexType>
 gko::matrix_data<ValueType, IndexType> FbcsrSample<
     ValueType, IndexType>::generate_matrix_data_with_explicit_zeros() const
 {
-    return MatData({{6, 12}, {{0, 3, 2.0},  {0, 4, 3.0},   {0, 5, 4.0},
-                              {1, 3, 5.0},  {1, 4, 6.0},   {1, 5, 7.0},
-                              {2, 3, 0.0},  {2, 4, 9.0},   {2, 5, 0.0},
-
-                              {0, 9, 4.0},  {0, 10, 5.0},  {0, 11, 6.0},
-                              {1, 9, 7.0},  {1, 10, 8.0},  {1, 11, 9.0},
-                              {2, 9, 10.0}, {2, 10, 11.0}, {2, 11, 12.0},
-
-                              {3, 0, 2.0},  {3, 1, 3.0},   {3, 2, 4.0},
-                              {4, 0, 5.0},  {4, 1, 6.0},   {4, 2, 7.0},
-                              {5, 0, 8.0},  {5, 1, 9.0},   {5, 2, 10.0},
-
-                              {3, 6, 0.0},  {3, 7, 7.0},   {3, 8, 8.0},
-                              {4, 6, 9.0},  {4, 7, 10.0},  {4, 8, 11.0},
-                              {5, 6, 12.0}, {5, 7, 13.0},  {5, 8, 14.0}}});
+    return MatData({{6, 12},
+                    {{0, 3, 2.0},
+                     {0, 4, 3.0},
+                     {0, 5, 4.0},
+                     {1, 3, 5.0},
+                     {1, 4, 6.0},
+                     {1, 5, 7.0},
+                     {2, 3, 0.0},
+                     {2, 4, 9.0},
+                     {2, 5, 0.0},
+
+                     {0, 9, 4.0},
+                     {0, 10, 5.0},
+                     {0, 11, 6.0},
+                     {1, 9, 7.0},
+                     {1, 10, 8.0},
+                     {1, 11, 9.0},
+                     {2, 9, 10.0},
+                     {2, 10, 11.0},
+                     {2, 11, 12.0},
+
+                     {3, 0, 2.0},
+                     {3, 1, 3.0},
+                     {3, 2, 4.0},
+                     {4, 0, 5.0},
+                     {4, 1, 6.0},
+                     {4, 2, 7.0},
+                     {5, 0, 8.0},
+                     {5, 1, 9.0},
+                     {5, 2, 10.0},
+
+                     {3, 6, 0.0},
+                     {3, 7, 7.0},
+                     {3, 8, 8.0},
+                     {4, 6, 9.0},
+                     {4, 7, 10.0},
+                     {4, 8, 11.0},
+                     {5, 6, 12.0},
+                     {5, 7, 13.0 + FBCSR_TEST_OFFSET},
+                     {5, 8, 14.0 + FBCSR_TEST_OFFSET}}});
 }
 
 template <typename ValueType, typename IndexType>
@@ -328,9 +386,127 @@ FbcsrSample<ValueType, IndexType>::generate_sparsity_csr() const
     return SparCsr::create(exec, gko::dim<2>{nbrows, nbcols}, colids, rowptrs);
 }
 
+template <typename ValueType, typename IndexType>
+gko::Array<IndexType> FbcsrSample<ValueType, IndexType>::getNonzerosPerRow()
+    const
+{
+    return gko::Array<index_type>(exec, {6, 6, 6, 6, 6, 6});
+}
+
 #define GKO_DECLARE_FBCSR_TEST_SAMPLE(ValueType, IndexType) \
     class FbcsrSample<ValueType, IndexType>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_TEST_SAMPLE);
 
+
+template <typename ValueType, typename IndexType>
+FbcsrSample2<ValueType, IndexType>::FbcsrSample2(
+    const std::shared_ptr<const gko::ReferenceExecutor> rexec)
+    : nrows{6},
+      ncols{8},
+      nnz{16},
+      nbrows{3},
+      nbcols{4},
+      nbnz{4},
+      bs{2},
+      exec(rexec)
+{}
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
+FbcsrSample2<ValueType, IndexType>::generate_fbcsr() const
+{
+    std::unique_ptr<Fbcsr> mtx =
+        Fbcsr::create(exec,
+                      gko::dim<2>{static_cast<size_type>(nrows),
+                                  static_cast<size_type>(ncols)},
+                      nnz, bs, std::make_shared<matstr::classical<Fbcsr>>());
+
+    value_type *const v = mtx->get_values();
+    index_type *const c = mtx->get_col_idxs();
+    index_type *const r = mtx->get_row_ptrs();
+    index_type *const s = mtx->get_srow();
+    r[0] = 0;
+    r[1] = 1;
+    r[2] = 3;
+    r[3] = 4;
+    c[0] = 0;
+    c[1] = 0;
+    c[2] = 3;
+    c[3] = 2;
+
+    for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
+
+    v[0] = 1;
+    v[1] = 2;
+    v[2] = 3;
+    v[3] = 0;
+    v[10] = 0;
+    v[11] = 0;
+    v[12] = -12;
+    v[13] = -1;
+    v[14] = -2;
+    v[15] = -11;
+
+    for (index_type is = 0; is < mtx->get_num_srow_elements(); is++) s[is] = 0;
+
+    return mtx;
+}
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Diagonal<ValueType>>
+FbcsrSample2<ValueType, IndexType>::extract_diagonal() const
+{
+    gko::Array<ValueType> dvals(exec, nrows);
+    ValueType *const dv = dvals.get_data();
+    dv[0] = 1;
+    dv[1] = 0;
+    dv[2] = 0;
+    dv[3] = 0;
+    dv[4] = -12;
+    dv[5] = -11;
+    return Diagonal::create(exec, nrows, dvals);
+}
+
+template <typename ValueType, typename IndexType>
+gko::Array<IndexType> FbcsrSample2<ValueType, IndexType>::getNonzerosPerRow()
+    const
+{
+    return gko::Array<index_type>(exec, {2, 2, 4, 4, 2, 2});
+}
+
+template <typename ValueType, typename IndexType>
+void FbcsrSample2<ValueType, IndexType>::apply(
+    const gko::matrix::Dense<ValueType> *const x,
+    gko::matrix::Dense<ValueType> *const y) const
+{
+    if (x->get_size()[0] != ncols)
+        throw gko::BadDimension(__FILE__, __LINE__, __func__, "spmv", nrows,
+                                ncols, "");
+    if (y->get_size()[0] != nrows)
+        throw gko::BadDimension(__FILE__, __LINE__, __func__, "spmv", nrows,
+                                ncols, "");
+    if (x->get_size()[1] != y->get_size()[1])
+        throw gko::BadDimension(__FILE__, __LINE__, __func__, "spmv", nrows,
+                                ncols, "");
+
+    const ValueType defv = sct(0.15 + FBCSR_TEST_OFFSET);
+
+    // ValueType *const yvals = y->get_data();
+    // const ValueType *const xvals = x->get_const_data();
+    for (index_type k = 0; k < x->get_size()[1]; k++) {
+        y->at(0, k) = sct(1.0) * x->at(0, k) + sct(2.0) * x->at(1, k);
+        y->at(1, k) = sct(3.0) * x->at(0, k);
+        y->at(2, k) =
+            defv * (x->at(0, k) + x->at(1, k) + x->at(6, k) + x->at(7, k));
+        y->at(3, k) = defv * (x->at(0, k) + x->at(1, k));
+        y->at(4, k) = sct(-12.0) * x->at(4, k) - x->at(5, k);
+        y->at(5, k) = sct(-2.0) * x->at(4, k) + sct(-11.0) * x->at(5, k);
+    }
+}
+
+#define GKO_DECLARE_FBCSR_TEST_SAMPLE_2(ValueType, IndexType) \
+    class FbcsrSample2<ValueType, IndexType>
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_TEST_SAMPLE_2);
+
 }  // namespace testing
 }  // namespace gko
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 5df96e6d64f..05b1a2f76ae 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
 #include <ginkgo/core/matrix/fbcsr.hpp>
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
@@ -47,6 +48,7 @@ namespace testing {
 
 /// Generates the same sample block CSR matrix in different formats
 /** This currently a 6 x 12 matrix with 3x3 blocks.
+ * Assumes that the layout within each block is row-major.
  */
 template <typename ValueType, typename IndexType>
 class FbcsrSample {
@@ -80,6 +82,65 @@ class FbcsrSample {
 
     MatData generate_matrix_data_with_explicit_zeros() const;
 
+    /// Returns an array containing number of stored values in each row
+    ///  (not block-row)
+    gko::Array<index_type> getNonzerosPerRow() const;
+
+    const size_type nrows;
+    const size_type ncols;
+    const size_type nnz;
+    const size_type nbrows;
+    const size_type nbcols;
+    const size_type nbnz;
+    const int bs;
+    const std::shared_ptr<const gko::Executor> exec;
+};
+
+/// Generates the a sample block CSR matrix in different formats
+/** This currently a 6 x 8 matrix with 2x2 blocks.
+ */
+template <typename ValueType, typename IndexType>
+class FbcsrSample2 {
+public:
+    using value_type = ValueType;
+    using index_type = IndexType;
+    using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    using Coo = gko::matrix::Coo<value_type, index_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+    using MatData = gko::matrix_data<value_type, index_type>;
+    using SparCsr = gko::matrix::SparsityCsr<value_type, index_type>;
+    using Diagonal = gko::matrix::Diagonal<value_type>;
+
+    FbcsrSample2(std::shared_ptr<const gko::ReferenceExecutor> exec);
+
+    std::unique_ptr<Fbcsr> generate_fbcsr() const;
+
+    std::unique_ptr<Diagonal> extract_diagonal() const;
+
+    void apply(const Dense *x, Dense *y) const;
+
+    // std::unique_ptr<Csr> generate_csr() const;
+
+    // std::unique_ptr<Dense> generate_dense() const;
+
+    // std::unique_ptr<Coo> generate_coo() const;
+
+    // std::unique_ptr<SparCsr> generate_sparsity_csr() const;
+
+    // MatData generate_matrix_data() const;
+
+    // MatData generate_matrix_data_with_explicit_zeros() const;
+
+    gko::Array<index_type> getNonzerosPerRow() const;
+
+    template <typename U>
+    inline constexpr ValueType sct(U u) const
+    {
+        return static_cast<ValueType>(u);
+    }
+
+
     const size_type nrows;
     const size_type ncols;
     const size_type nnz;
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index fe6bac8efd3..938f146e4a0 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -70,63 +70,87 @@ namespace fbcsr {
 
 
 template <typename ValueType, typename IndexType>
-void spmv(std::shared_ptr<const ReferenceExecutor> exec,
-          const matrix::Fbcsr<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b,
-          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto row_ptrs = a->get_const_row_ptrs();
-//    auto col_idxs = a->get_const_col_idxs();
-//    auto vals = a->get_const_values();
-//
-//    for (size_type row = 0; row < a->get_size()[0]; ++row) {
-//        for (size_type j = 0; j < c->get_size()[1]; ++j) {
-//            c->at(row, j) = zero<ValueType>();
-//        }
-//        for (size_type k = row_ptrs[row];
-//             k < static_cast<size_type>(row_ptrs[row + 1]); ++k) {
-//            auto val = vals[k];
-//            auto col = col_idxs[k];
-//            for (size_type j = 0; j < c->get_size()[1]; ++j) {
-//                c->at(row, j) += val * b->at(col, j);
-//            }
-//        }
-//    }
-//}
+void spmv(const std::shared_ptr<const ReferenceExecutor> exec,
+          const matrix::Fbcsr<ValueType, IndexType> *const a,
+          const matrix::Dense<ValueType> *const b,
+          matrix::Dense<ValueType> *const c)
+{
+    const int bs = a->get_block_size();
+    const IndexType nvecs = static_cast<IndexType>(b->get_size()[1]);
+    const IndexType nbrows =
+        gko::blockutils::getNumFixedBlocks(bs, a->get_size()[0]);
+    auto row_ptrs = a->get_const_row_ptrs();
+    auto col_idxs = a->get_const_col_idxs();
+    auto vals = a->get_const_values();
+    const gko::blockutils::DenseBlocksView<const ValueType, IndexType> avalues(
+        vals, bs, bs);
+
+    ValueType *const cvals = c->get_values();
+
+    for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
+        const IndexType crowblkend = (ibrow + 1) * bs * nvecs;
+        for (IndexType i = ibrow * bs * nvecs; i < crowblkend; i++)
+            cvals[i] = zero<ValueType>();
+
+        for (IndexType inz = row_ptrs[ibrow]; inz < row_ptrs[ibrow + 1];
+             ++inz) {
+            for (int ib = 0; ib < bs; ib++) {
+                const IndexType row = ibrow * bs + ib;
+                for (int jb = 0; jb < bs; jb++) {
+                    const auto val = avalues(inz, ib, jb);
+                    const auto col = col_idxs[inz] * bs + jb;
+                    for (size_type j = 0; j < nvecs; ++j)
+                        c->at(row, j) += val * b->at(col, j);
+                }
+            }
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPMV_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void advanced_spmv(std::shared_ptr<const ReferenceExecutor> exec,
-                   const matrix::Dense<ValueType> *alpha,
-                   const matrix::Fbcsr<ValueType, IndexType> *a,
-                   const matrix::Dense<ValueType> *b,
-                   const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto row_ptrs = a->get_const_row_ptrs();
-//    auto col_idxs = a->get_const_col_idxs();
-//    auto vals = a->get_const_values();
-//    auto valpha = alpha->at(0, 0);
-//    auto vbeta = beta->at(0, 0);
-//
-//    for (size_type row = 0; row < a->get_size()[0]; ++row) {
-//        for (size_type j = 0; j < c->get_size()[1]; ++j) {
-//            c->at(row, j) *= vbeta;
-//        }
-//        for (size_type k = row_ptrs[row];
-//             k < static_cast<size_type>(row_ptrs[row + 1]); ++k) {
-//            auto val = vals[k];
-//            auto col = col_idxs[k];
-//            for (size_type j = 0; j < c->get_size()[1]; ++j) {
-//                c->at(row, j) += valpha * val * b->at(col, j);
-//            }
-//        }
-//    }
-//}
+void advanced_spmv(const std::shared_ptr<const ReferenceExecutor> exec,
+                   const matrix::Dense<ValueType> *const alpha,
+                   const matrix::Fbcsr<ValueType, IndexType> *const a,
+                   const matrix::Dense<ValueType> *const b,
+                   const matrix::Dense<ValueType> *const beta,
+                   matrix::Dense<ValueType> *const c)
+{
+    const int bs = a->get_block_size();
+    const IndexType nvecs = static_cast<IndexType>(b->get_size()[1]);
+    const IndexType nbrows =
+        gko::blockutils::getNumFixedBlocks(bs, a->get_size()[0]);
+    auto row_ptrs = a->get_const_row_ptrs();
+    auto col_idxs = a->get_const_col_idxs();
+    auto vals = a->get_const_values();
+    auto valpha = alpha->at(0, 0);
+    auto vbeta = beta->at(0, 0);
+    const gko::blockutils::DenseBlocksView<const ValueType, IndexType> avalues(
+        vals, bs, bs);
+
+    ValueType *const cvals = c->get_values();
+
+    for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
+        const IndexType crowblkend = (ibrow + 1) * bs * nvecs;
+        for (IndexType i = ibrow * bs * nvecs; i < crowblkend; i++)
+            cvals[i] *= vbeta;
+
+        for (IndexType inz = row_ptrs[ibrow]; inz < row_ptrs[ibrow + 1];
+             ++inz) {
+            for (int ib = 0; ib < bs; ib++) {
+                const IndexType row = ibrow * bs + ib;
+                for (int jb = 0; jb < bs; jb++) {
+                    const auto val = avalues(inz, ib, jb);
+                    const auto col = col_idxs[inz] * bs + jb;
+                    for (size_type j = 0; j < nvecs; ++j)
+                        c->at(row, j) += valpha * val * b->at(col, j);
+                }
+            }
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
@@ -933,16 +957,18 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void calculate_nonzeros_per_row(
     std::shared_ptr<const ReferenceExecutor> exec,
-    const matrix::Fbcsr<ValueType, IndexType> *source,
-    Array<size_type> *result) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto row_ptrs = source->get_const_row_ptrs();
-//    auto row_nnz_val = result->get_data();
-//    for (size_type i = 0; i < result->get_num_elems(); i++) {
-//        row_nnz_val[i] = row_ptrs[i + 1] - row_ptrs[i];
-//    }
-//}
+    const matrix::Fbcsr<ValueType, IndexType> *source, Array<size_type> *result)
+{
+    const auto row_ptrs = source->get_const_row_ptrs();
+    auto row_nnz_val = result->get_data();
+    const int bs = source->get_block_size();
+    assert(result->get_num_elems() == source->get_size()[0]);
+
+    for (size_type i = 0; i < result->get_num_elems(); i++) {
+        const size_type ibrow = i / bs;
+        row_nnz_val[i] = (row_ptrs[ibrow + 1] - row_ptrs[ibrow]) * bs;
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
@@ -974,24 +1000,26 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void is_sorted_by_column_index(
     std::shared_ptr<const ReferenceExecutor> exec,
-    const matrix::Fbcsr<ValueType, IndexType> *to_check,
-    bool *is_sorted) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto row_ptrs = to_check->get_const_row_ptrs();
-//    const auto col_idxs = to_check->get_const_col_idxs();
-//    const auto size = to_check->get_size();
-//    for (size_type i = 0; i < size[0]; ++i) {
-//        for (auto idx = row_ptrs[i] + 1; idx < row_ptrs[i + 1]; ++idx) {
-//            if (col_idxs[idx - 1] > col_idxs[idx]) {
-//                *is_sorted = false;
-//                return;
-//            }
-//        }
-//    }
-//    *is_sorted = true;
-//    return;
-//}
+    const matrix::Fbcsr<ValueType, IndexType> *const to_check,
+    bool *const is_sorted)
+{
+    const auto row_ptrs = to_check->get_const_row_ptrs();
+    const auto col_idxs = to_check->get_const_col_idxs();
+    const auto size = to_check->get_size();
+    const int bs = to_check->get_block_size();
+    const size_type nbrows = gko::blockutils::getNumFixedBlocks(bs, size[0]);
+
+    for (size_type i = 0; i < nbrows; ++i) {
+        for (auto idx = row_ptrs[i] + 1; idx < row_ptrs[i + 1]; ++idx) {
+            if (col_idxs[idx - 1] > col_idxs[idx]) {
+                *is_sorted = false;
+                return;
+            }
+        }
+    }
+    *is_sorted = true;
+    return;
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
@@ -999,25 +1027,33 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const ReferenceExecutor> exec,
-                      const matrix::Fbcsr<ValueType, IndexType> *orig,
-                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto row_ptrs = orig->get_const_row_ptrs();
-//    const auto col_idxs = orig->get_const_col_idxs();
-//    const auto values = orig->get_const_values();
-//    const auto diag_size = diag->get_size()[0];
-//    auto diag_values = diag->get_values();
-//
-//    for (size_type row = 0; row < diag_size; ++row) {
-//        for (size_type idx = row_ptrs[row]; idx < row_ptrs[row + 1]; ++idx) {
-//            if (col_idxs[idx] == row) {
-//                diag_values[row] = values[idx];
-//                break;
-//            }
-//        }
-//    }
-//}
+                      const matrix::Fbcsr<ValueType, IndexType> *const orig,
+                      matrix::Diagonal<ValueType> *const diag)
+{
+    const auto row_ptrs = orig->get_const_row_ptrs();
+    const auto col_idxs = orig->get_const_col_idxs();
+    const auto values = orig->get_const_values();
+    const int bs = orig->get_block_size();
+    const size_type diag_size = diag->get_size()[0];
+    const size_type nbrows =
+        gko::blockutils::getNumFixedBlocks(bs, orig->get_size()[0]);
+    auto diag_values = diag->get_values();
+    assert(diag_size == orig->get_size()[0]);
+
+    const gko::blockutils::DenseBlocksView<const ValueType, IndexType> vblocks(
+        values, bs, bs);
+
+    for (size_type ibrow = 0; ibrow < nbrows; ++ibrow) {
+        for (size_type idx = row_ptrs[ibrow]; idx < row_ptrs[ibrow + 1];
+             ++idx) {
+            if (col_idxs[idx] == ibrow) {
+                for (int ib = 0; ib < bs; ib++)
+                    diag_values[ibrow * bs + ib] = vblocks(idx, ib, ib);
+                break;
+            }
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL);
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index dbba82dc3e4..4ed6bad3b8c 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -83,17 +83,21 @@ class Fbcsr : public ::testing::Test {
     using SparCsr = gko::matrix::SparsityCsr<value_type, index_type>;
     using Ell = gko::matrix::Ell<value_type, index_type>;
     using Hybrid = gko::matrix::Hybrid<value_type, index_type>;
+    using Diag = gko::matrix::Diagonal<value_type>;
     using Vec = gko::matrix::Dense<value_type>;
 
     Fbcsr()
         : exec(gko::ReferenceExecutor::create()),
           fbsample(exec),
+          fbsample2(exec),
           mtx(fbsample.generate_fbcsr()),
           refmtx(fbsample.generate_fbcsr()),
           refcsrmtx(fbsample.generate_csr()),
           refdenmtx(fbsample.generate_dense()),
           refcoomtx(fbsample.generate_coo()),
-          refspcmtx(fbsample.generate_sparsity_csr())
+          refspcmtx(fbsample.generate_sparsity_csr()),
+          mtx2(fbsample2.generate_fbcsr()),
+          m2diag(fbsample2.extract_diagonal())
     {}
 
     // void create_mtx3(Mtx *sorted, Mtx *unsorted)
@@ -162,90 +166,174 @@ class Fbcsr : public ::testing::Test {
 
     std::shared_ptr<const gko::ReferenceExecutor> exec;
     const gko::testing::FbcsrSample<value_type, index_type> fbsample;
+    const gko::testing::FbcsrSample2<value_type, index_type> fbsample2;
     std::unique_ptr<Mtx> mtx;
     const std::unique_ptr<const Mtx> refmtx;
     const std::unique_ptr<const Csr> refcsrmtx;
     const std::unique_ptr<const Dense> refdenmtx;
     const std::unique_ptr<const Coo> refcoomtx;
     const std::unique_ptr<const SparCsr> refspcmtx;
+    const std::unique_ptr<const Mtx> mtx2;
+    const std::unique_ptr<const Diag> m2diag;
 };
 
 TYPED_TEST_CASE(Fbcsr, gko::test::ValueIndexTypes);
 
 
+template <typename T>
+constexpr T get_some_number()
+{
+    return static_cast<T>(1.2);
+}
+
+template <typename T>
+constexpr typename std::enable_if_t<gko::is_complex<T>> get_some_number()
+{
+    using RT = gko::remove_complex<T>;
+    return {static_cast<RT>(1.2), static_cast<RT>(3.4)};
+}
+
+
 TYPED_TEST(Fbcsr, AppliesToDenseVector)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Vec = typename TestFixture::Vec;
-//    using T = typename TestFixture::value_type;
-//    auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
-//    auto y = Vec::create(this->exec, gko::dim<2>{2, 1});
-//
-//    this->mtx->apply(x.get(), y.get());
-//
-//    EXPECT_EQ(y->at(0), T{13.0});
-//    EXPECT_EQ(y->at(1), T{5.0});
-//}
+{
+    using Vec = typename TestFixture::Vec;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+
+    const index_type nrows = this->mtx2->get_size()[0];
+    const index_type ncols = this->mtx2->get_size()[1];
+    auto x = Vec::create(this->exec, gko::dim<2>{(gko::size_type)ncols, 1});
+    T *const xvals = x->get_values();
+    for (index_type i = 0; i < ncols; i++)
+        // xvals[i] = std::log(static_cast<T>(static_cast<float>((i+1)^2)));
+        xvals[i] = (i + 1.0) * (i + 1.0);
+    auto y = Vec::create(this->exec, gko::dim<2>{(gko::size_type)nrows, 1});
+    auto yref = Vec::create(this->exec, gko::dim<2>{(gko::size_type)nrows, 1});
+
+    this->mtx2->apply(x.get(), y.get());
+
+    this->fbsample2.apply(x.get(), yref.get());
+
+    const double tolerance =
+        std::numeric_limits<gko::remove_complex<T>>::epsilon();
+    GKO_ASSERT_MTX_NEAR(y, yref, tolerance);
+}
 
 
 TYPED_TEST(Fbcsr, AppliesToDenseMatrix)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Vec = typename TestFixture::Vec;
-//    using T = typename TestFixture::value_type;
-//    auto x = gko::initialize<Vec>(
-//        {I<T>{2.0, 3.0}, I<T>{1.0, -1.5}, I<T>{4.0, 2.5}}, this->exec);
-//    auto y = Vec::create(this->exec, gko::dim<2>{2});
-//
-//    this->mtx->apply(x.get(), y.get());
-//
-//    EXPECT_EQ(y->at(0, 0), T{13.0});
-//    EXPECT_EQ(y->at(1, 0), T{5.0});
-//    EXPECT_EQ(y->at(0, 1), T{3.5});
-//    EXPECT_EQ(y->at(1, 1), T{-7.5});
-//}
+{
+    using Vec = typename TestFixture::Vec;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+
+    const gko::size_type nrows = this->mtx2->get_size()[0];
+    const gko::size_type ncols = this->mtx2->get_size()[1];
+    const gko::size_type nvecs = 3;
+    auto x = Vec::create(this->exec, gko::dim<2>{ncols, nvecs});
+
+    for (index_type i = 0; i < ncols; i++)
+        for (index_type j = 0; j < nvecs; j++)
+            x->at(i, j) = (static_cast<T>(3.0 * i) + get_some_number<T>()) /
+                          static_cast<T>(j + 1.0);
+    auto y = Vec::create(this->exec, gko::dim<2>{nrows, nvecs});
+    auto yref = Vec::create(this->exec, gko::dim<2>{nrows, nvecs});
+
+    this->mtx2->apply(x.get(), y.get());
+
+    this->fbsample2.apply(x.get(), yref.get());
+
+    const double tolerance =
+        std::numeric_limits<gko::remove_complex<T>>::epsilon();
+    GKO_ASSERT_MTX_NEAR(y, yref, tolerance);
+}
 
 
 TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseVector)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Vec = typename TestFixture::Vec;
-//    using T = typename TestFixture::value_type;
-//    auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
-//    auto beta = gko::initialize<Vec>({2.0}, this->exec);
-//    auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
-//    auto y = gko::initialize<Vec>({1.0, 2.0}, this->exec);
-//
-//    this->mtx->apply(alpha.get(), x.get(), beta.get(), y.get());
-//
-//    EXPECT_EQ(y->at(0), T{-11.0});
-//    EXPECT_EQ(y->at(1), T{-1.0});
-//}
+{
+    using Vec = typename TestFixture::Vec;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+
+    constexpr T alphav = -1.0;
+    constexpr T betav = 2.0;
+    auto alpha = gko::initialize<Vec>({alphav}, this->exec);
+    auto beta = gko::initialize<Vec>({betav}, this->exec);
+
+    const gko::size_type nrows = this->mtx2->get_size()[0];
+    const gko::size_type ncols = this->mtx2->get_size()[1];
+    auto x = Vec::create(this->exec, gko::dim<2>{ncols, 1});
+    auto y = Vec::create(this->exec, gko::dim<2>{nrows, 1});
+
+    for (index_type i = 0; i < ncols; i++) {
+        // xvals[i] = std::log(static_cast<T>(static_cast<float>((i+1)^2)));
+        x->at(i, 0) = (i + 1.0) * (i + 1.0);
+    }
+    for (index_type i = 0; i < nrows; i++) {
+        y->at(i, 0) = static_cast<T>(std::sin(2 * 3.14 * (i + 0.1) / nrows)) +
+                      get_some_number<T>();
+    }
+
+    auto yref = Vec::create(this->exec, gko::dim<2>{nrows, 1});
+    yref = y->clone();
+
+    this->mtx2->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    auto prod = Vec::create(this->exec, gko::dim<2>{nrows, 1});
+    this->fbsample2.apply(x.get(), prod.get());
+
+    yref->scale(beta.get());
+    yref->add_scaled(alpha.get(), prod.get());
+
+    const double tolerance =
+        std::numeric_limits<gko::remove_complex<T>>::epsilon();
+    GKO_ASSERT_MTX_NEAR(y, yref, tolerance);
+}
 
 
 TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseMatrix)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Vec = typename TestFixture::Vec;
-//    using T = typename TestFixture::value_type;
-//    auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
-//    auto beta = gko::initialize<Vec>({2.0}, this->exec);
-//    auto x = gko::initialize<Vec>(
-//        {I<T>{2.0, 3.0}, I<T>{1.0, -1.5}, I<T>{4.0, 2.5}}, this->exec);
-//    auto y =
-//        gko::initialize<Vec>({I<T>{1.0, 0.5}, I<T>{2.0, -1.5}}, this->exec);
-//
-//    this->mtx->apply(alpha.get(), x.get(), beta.get(), y.get());
-//
-//    EXPECT_EQ(y->at(0, 0), T{-11.0});
-//    EXPECT_EQ(y->at(1, 0), T{-1.0});
-//    EXPECT_EQ(y->at(0, 1), T{-2.5});
-//    EXPECT_EQ(y->at(1, 1), T{4.5});
-//}
+{
+    using Vec = typename TestFixture::Vec;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+
+    constexpr T alphav = -1.0;
+    constexpr T betav = 2.0;
+    auto alpha = gko::initialize<Vec>({alphav}, this->exec);
+    auto beta = gko::initialize<Vec>({betav}, this->exec);
+
+    const gko::size_type nrows = this->mtx2->get_size()[0];
+    const gko::size_type ncols = this->mtx2->get_size()[1];
+    const gko::size_type nvecs = 3;
+    auto x = Vec::create(this->exec, gko::dim<2>{ncols, nvecs});
+    auto y = Vec::create(this->exec, gko::dim<2>{nrows, nvecs});
+
+    for (index_type i = 0; i < ncols; i++)
+        for (index_type j = 0; j < nvecs; j++) {
+            // xvals[i] = std::log(static_cast<T>(static_cast<float>((i+1)^2)));
+            x->at(i, j) = (i + 1.0) / (j + 1.0);
+        }
+    for (index_type i = 0; i < nrows; i++)
+        for (index_type j = 0; j < nvecs; j++) {
+            y->at(i, j) =
+                static_cast<T>(std::sin(2 * 3.14 * (i + j + 0.1) / nrows)) +
+                get_some_number<T>();
+        }
+
+    auto yref = Vec::create(this->exec, gko::dim<2>{nrows, nvecs});
+    yref = y->clone();
+
+    this->mtx2->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    auto prod = Vec::create(this->exec, gko::dim<2>{nrows, nvecs});
+    this->fbsample2.apply(x.get(), prod.get());
+
+    yref->scale(beta.get());
+    yref->add_scaled(alpha.get(), prod.get());
+
+    const double tolerance =
+        std::numeric_limits<gko::remove_complex<T>>::epsilon();
+    GKO_ASSERT_MTX_NEAR(y, yref, tolerance);
+}
 
 
 TYPED_TEST(Fbcsr, AppliesToFbcsrMatrix)
@@ -862,22 +950,24 @@ TYPED_TEST(Fbcsr, MovesEmptyToSparsityCsr)
 
 
 TYPED_TEST(Fbcsr, CalculatesNonzerosPerRow)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    gko::Array<gko::size_type> row_nnz(this->exec, this->mtx->get_size()[0]);
-//
-//    gko::kernels::reference::fbcsr::calculate_nonzeros_per_row(
-//        this->exec, this->mtx.get(), &row_nnz);
-//
-//    auto row_nnz_val = row_nnz.get_data();
-//    ASSERT_EQ(row_nnz_val[0], 3);
-//    ASSERT_EQ(row_nnz_val[1], 1);
-//}
+{
+    using IndexType = typename TestFixture::index_type;
 
+    gko::Array<gko::size_type> row_nnz(this->exec, this->mtx2->get_size()[0]);
 
-TYPED_TEST(Fbcsr, CalculatesTotalCols)
-GKO_NOT_IMPLEMENTED;
+    gko::kernels::reference::fbcsr ::calculate_nonzeros_per_row(
+        this->exec, this->mtx2.get(), &row_nnz);
+
+    auto row_nnz_val = row_nnz.get_data();
+    const gko::Array<IndexType> refrnnz = this->fbsample2.getNonzerosPerRow();
+    ASSERT_EQ(row_nnz.get_num_elems(), refrnnz.get_num_elems());
+    for (gko::size_type i = 0; i < this->mtx2->get_size()[0]; i++)
+        ASSERT_EQ(row_nnz_val[i], refrnnz.get_const_data()[i]);
+}
+
+
+// TYPED_TEST(Fbcsr, CalculatesTotalCols)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    gko::size_type total_cols;
@@ -1173,21 +1263,22 @@ GKO_NOT_IMPLEMENTED;
 
 
 TYPED_TEST(Fbcsr, RecognizeSortedMatrix)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    ASSERT_TRUE(this->mtx->is_sorted_by_column_index());
-//    ASSERT_TRUE(this->mtx2->is_sorted_by_column_index());
-//    ASSERT_TRUE(this->mtx3_sorted->is_sorted_by_column_index());
-//}
+{
+    ASSERT_TRUE(this->mtx->is_sorted_by_column_index());
+}
 
 
 TYPED_TEST(Fbcsr, RecognizeUnsortedMatrix)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    ASSERT_FALSE(this->mtx3_unsorted->is_sorted_by_column_index());
-//}
+{
+    using Fbcsr = typename TestFixture::Mtx;
+    using index_type = typename TestFixture::index_type;
+
+    // auto cpmat = Fbcsr::create(this->exec, this->mtx->get_strategy());
+    auto cpmat = this->mtx->clone();
+    index_type *const colinds = cpmat->get_col_idxs();
+    std::swap(colinds[0], colinds[1]);
+    ASSERT_FALSE(cpmat->is_sorted_by_column_index());
+}
 
 
 TYPED_TEST(Fbcsr, SortSortedMatrix)
@@ -1215,19 +1306,16 @@ GKO_NOT_IMPLEMENTED;
 
 
 TYPED_TEST(Fbcsr, ExtractsDiagonal)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using T = typename TestFixture::value_type;
-//    auto matrix = this->mtx3_unsorted->clone();
-//    auto diag = matrix->extract_diagonal();
-//
-//    ASSERT_EQ(diag->get_size()[0], 3);
-//    ASSERT_EQ(diag->get_size()[1], 3);
-//    ASSERT_EQ(diag->get_values()[0], T{0.});
-//    ASSERT_EQ(diag->get_values()[1], T{1.});
-//    ASSERT_EQ(diag->get_values()[2], T{3.});
-//}
+{
+    using T = typename TestFixture::value_type;
+    auto matrix = this->mtx2->clone();
+    auto diag = matrix->extract_diagonal();
+
+    ASSERT_EQ(this->m2diag->get_size(), diag->get_size());
+    for (gko::size_type i = 0; i < this->m2diag->get_size()[0]; i++)
+        ASSERT_EQ(this->m2diag->get_const_values()[i],
+                  diag->get_const_values()[i]);
+}
 
 
 TYPED_TEST(Fbcsr, InplaceAbsolute)

From 540047ef1d8ae328c83695bad21118872a12e774 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@mail.mcgill.ca>
Date: Sun, 15 Nov 2020 23:24:10 +0100
Subject: [PATCH 0536/1008] [tests fail] added fbcsr transpose and test for
 square matrix

---
 core/matrix/fbcsr.cpp                   |  22 +++--
 core/test/matrix/fbcsr_sample.cpp       |  77 +++++++++++++++++
 core/test/matrix/fbcsr_sample.hpp       |  38 ++++++---
 reference/matrix/fbcsr_kernels.cpp      | 108 +++++++++++++-----------
 reference/test/matrix/fbcsr_kernels.cpp |  97 +++++++++------------
 5 files changed, 215 insertions(+), 127 deletions(-)

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 68defa46f4b..8b239617991 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -556,18 +556,16 @@ void Fbcsr<ValueType, IndexType>::write(mat_data &data) const
 
 template <typename ValueType, typename IndexType>
 std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::transpose() const
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto exec = this->get_executor();
-//    auto trans_cpy =
-//        Fbcsr::create(exec, gko::transpose(this->get_size()),
-//                    this->get_num_stored_elements(), this->get_strategy());
-//
-//    exec->run(fbcsr::make_transpose(this, trans_cpy.get()));
-//    trans_cpy->make_srow();
-//    return std::move(trans_cpy);
-//}
+{
+    auto exec = this->get_executor();
+    auto trans_cpy = Fbcsr::create(exec, gko::transpose(this->get_size()),
+                                   this->get_num_stored_elements(), bs_,
+                                   this->get_strategy());
+
+    exec->run(fbcsr::make_transpose(this, trans_cpy.get()));
+    trans_cpy->make_srow();
+    return std::move(trans_cpy);
+}
 
 
 template <typename ValueType, typename IndexType>
diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
index f8d8136e23f..aa8299d33c8 100644
--- a/core/test/matrix/fbcsr_sample.cpp
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -508,5 +508,82 @@ void FbcsrSample2<ValueType, IndexType>::apply(
     class FbcsrSample2<ValueType, IndexType>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_TEST_SAMPLE_2);
 
+
+template <typename ValueType, typename IndexType>
+FbcsrSampleSquare<ValueType, IndexType>::FbcsrSampleSquare(
+    const std::shared_ptr<const gko::ReferenceExecutor> rexec)
+    : nrows{4},
+      ncols{4},
+      nnz{8},
+      nbrows{2},
+      nbcols{2},
+      nbnz{2},
+      bs{2},
+      exec(rexec)
+{}
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
+FbcsrSampleSquare<ValueType, IndexType>::generate_fbcsr() const
+{
+    std::unique_ptr<Fbcsr> mtx =
+        Fbcsr::create(exec,
+                      gko::dim<2>{static_cast<size_type>(nrows),
+                                  static_cast<size_type>(ncols)},
+                      nnz, bs, std::make_shared<matstr::classical<Fbcsr>>());
+
+    value_type *const v = mtx->get_values();
+    index_type *const c = mtx->get_col_idxs();
+    index_type *const r = mtx->get_row_ptrs();
+    index_type *const s = mtx->get_srow();
+    r[0] = 0;
+    r[1] = 1;
+    r[2] = 2;
+    c[0] = 1;
+    c[1] = 1;
+
+    for (IndexType i = 0; i < nnz; i++) v[i] = i;
+
+    return mtx;
+}
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
+FbcsrSampleSquare<ValueType, IndexType>::generate_transpose_fbcsr() const
+{
+    std::unique_ptr<Fbcsr> mtx =
+        Fbcsr::create(exec,
+                      gko::dim<2>{static_cast<size_type>(nrows),
+                                  static_cast<size_type>(ncols)},
+                      nnz, bs, std::make_shared<matstr::classical<Fbcsr>>());
+
+    value_type *const v = mtx->get_values();
+    index_type *const c = mtx->get_col_idxs();
+    index_type *const r = mtx->get_row_ptrs();
+    index_type *const s = mtx->get_srow();
+    r[0] = 0;
+    r[1] = 0;
+    r[2] = 2;
+    c[0] = 0;
+    c[1] = 1;
+
+    gko::blockutils::DenseBlocksView<value_type, index_type> vals(v, bs, bs);
+    vals(0, 0, 0) = 0;
+    vals(0, 0, 1) = 2;
+    vals(0, 1, 0) = 1;
+    vals(0, 1, 1) = 3;
+    vals(1, 0, 0) = 4;
+    vals(1, 0, 1) = 6;
+    vals(1, 1, 0) = 5;
+    vals(1, 1, 1) = 7;
+
+    return mtx;
+}
+
+#define GKO_DECLARE_FBCSR_TEST_SAMPLE_SQUARE(ValueType, IndexType) \
+    class FbcsrSampleSquare<ValueType, IndexType>
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_TEST_SAMPLE_SQUARE);
+
 }  // namespace testing
 }  // namespace gko
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 05b1a2f76ae..a0990887477 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -120,18 +120,6 @@ class FbcsrSample2 {
 
     void apply(const Dense *x, Dense *y) const;
 
-    // std::unique_ptr<Csr> generate_csr() const;
-
-    // std::unique_ptr<Dense> generate_dense() const;
-
-    // std::unique_ptr<Coo> generate_coo() const;
-
-    // std::unique_ptr<SparCsr> generate_sparsity_csr() const;
-
-    // MatData generate_matrix_data() const;
-
-    // MatData generate_matrix_data_with_explicit_zeros() const;
-
     gko::Array<index_type> getNonzerosPerRow() const;
 
     template <typename U>
@@ -151,6 +139,32 @@ class FbcsrSample2 {
     const std::shared_ptr<const gko::Executor> exec;
 };
 
+/// Generates the a sample block CSR square matrix in different formats
+/** This currently a 4 x 4 matrix with 2x2 blocks.
+ */
+template <typename ValueType, typename IndexType>
+class FbcsrSampleSquare {
+public:
+    using value_type = ValueType;
+    using index_type = IndexType;
+    using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
+
+    FbcsrSampleSquare(std::shared_ptr<const gko::ReferenceExecutor> exec);
+
+    std::unique_ptr<Fbcsr> generate_fbcsr() const;
+
+    std::unique_ptr<Fbcsr> generate_transpose_fbcsr() const;
+
+    const size_type nrows;
+    const size_type ncols;
+    const size_type nnz;
+    const size_type nbrows;
+    const size_type nbcols;
+    const size_type nbnz;
+    const int bs;
+    const std::shared_ptr<const gko::Executor> exec;
+};
+
 }  // namespace testing
 }  // namespace gko
 
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index 938f146e4a0..fe1ec87061e 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -652,62 +652,74 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL);
 
 
-template <typename ValueType, typename IndexType, typename UnaryOperator>
-inline void convert_fbcsr_to_csc(size_type num_rows, const IndexType *row_ptrs,
-                                 const IndexType *col_idxs,
-                                 const ValueType *fbcsr_vals,
-                                 IndexType *row_idxs, IndexType *col_ptrs,
-                                 ValueType *csc_vals,
-                                 UnaryOperator op) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    for (size_type row = 0; row < num_rows; ++row) {
-//        for (auto i = row_ptrs[row]; i < row_ptrs[row + 1]; ++i) {
-//            const auto dest_idx = col_ptrs[col_idxs[i]]++;
-//            row_idxs[dest_idx] = row;
-//            csc_vals[dest_idx] = op(fbcsr_vals[i]);
-//        }
-//    }
-//}
+template <typename ValueType, typename IndexType, typename UnaryOperator,
+          bool transpose_blocks>
+void convert_fbcsr_to_fbcsc(const size_type num_blk_rows, const int blksz,
+                            const IndexType *const row_ptrs,
+                            const IndexType *const col_idxs,
+                            const ValueType *const fbcsr_vals,
+                            IndexType *const row_idxs,
+                            IndexType *const col_ptrs,
+                            ValueType *const csc_vals, UnaryOperator op)
+{
+    const gko::blockutils::DenseBlocksView<const ValueType, IndexType> rvalues(
+        fbcsr_vals, blksz, blksz);
+    gko::blockutils::DenseBlocksView<ValueType, IndexType> cvalues(
+        csc_vals, blksz, blksz);
+    for (size_type brow = 0; brow < num_blk_rows; ++brow) {
+        for (auto i = row_ptrs[brow]; i < row_ptrs[brow + 1]; ++i) {
+            const auto dest_idx = col_ptrs[col_idxs[i]]++;
+            row_idxs[dest_idx] = brow;
+            for (int ib = 0; ib < blksz; ib++)
+                for (int jb = 0; jb < blksz; jb++)
+                    // csc_vals[dest_idx] = op(fbcsr_vals[i]);
+                    cvalues(dest_idx, ib, jb) =
+                        op(transpose_blocks ? rvalues(i, jb, ib)
+                                            : rvalues(i, ib, jb));
+        }
+    }
+}
 
 
 template <typename ValueType, typename IndexType, typename UnaryOperator>
-void transpose_and_transform(std::shared_ptr<const ReferenceExecutor> exec,
-                             matrix::Fbcsr<ValueType, IndexType> *trans,
-                             const matrix::Fbcsr<ValueType, IndexType> *orig,
-                             UnaryOperator op) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto trans_row_ptrs = trans->get_row_ptrs();
-//    auto orig_row_ptrs = orig->get_const_row_ptrs();
-//    auto trans_col_idxs = trans->get_col_idxs();
-//    auto orig_col_idxs = orig->get_const_col_idxs();
-//    auto trans_vals = trans->get_values();
-//    auto orig_vals = orig->get_const_values();
-//
-//    auto orig_num_cols = orig->get_size()[1];
-//    auto orig_num_rows = orig->get_size()[0];
-//    auto orig_nnz = orig_row_ptrs[orig_num_rows];
-//
-//    trans_row_ptrs[0] = 0;
-//    convert_idxs_to_ptrs(orig_col_idxs, orig_nnz, trans_row_ptrs + 1,
-//                         orig_num_cols);
-//
-//    convert_fbcsr_to_csc(orig_num_rows, orig_row_ptrs, orig_col_idxs,
-//    orig_vals,
-//                       trans_col_idxs, trans_row_ptrs + 1, trans_vals, op);
-//}
+void transpose_and_transform(
+    const std::shared_ptr<const ReferenceExecutor> exec,
+    matrix::Fbcsr<ValueType, IndexType> *const trans,
+    const matrix::Fbcsr<ValueType, IndexType> *const orig, UnaryOperator op)
+{
+    const int bs = orig->get_block_size();
+    auto trans_row_ptrs = trans->get_row_ptrs();
+    auto orig_row_ptrs = orig->get_const_row_ptrs();
+    auto trans_col_idxs = trans->get_col_idxs();
+    auto orig_col_idxs = orig->get_const_col_idxs();
+    auto trans_vals = trans->get_values();
+    auto orig_vals = orig->get_const_values();
+
+    auto orig_num_cols = orig->get_size()[1];
+    const size_type nbcols =
+        gko::blockutils::getNumFixedBlocks(bs, orig_num_cols);
+    auto orig_num_rows = orig->get_size()[0];
+    const size_type nbrows =
+        gko::blockutils::getNumFixedBlocks(bs, orig_num_rows);
+    auto orig_nbnz = orig_row_ptrs[nbrows];
+
+    trans_row_ptrs[0] = 0;
+    convert_idxs_to_ptrs(orig_col_idxs, orig_nbnz, trans_row_ptrs + 1, nbcols);
+
+    convert_fbcsr_to_fbcsc<ValueType, IndexType, UnaryOperator, true>(
+        nbrows, bs, orig_row_ptrs, orig_col_idxs, orig_vals, trans_col_idxs,
+        trans_row_ptrs + 1, trans_vals, op);
+}
 
 
 template <typename ValueType, typename IndexType>
 void transpose(std::shared_ptr<const ReferenceExecutor> exec,
                const matrix::Fbcsr<ValueType, IndexType> *orig,
-               matrix::Fbcsr<ValueType, IndexType> *trans) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    transpose_and_transform(exec, trans, orig,
-//                            [](const ValueType x) { return x; });
-//}
+               matrix::Fbcsr<ValueType, IndexType> *trans)
+{
+    transpose_and_transform(exec, trans, orig,
+                            [](const ValueType x) { return x; });
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL);
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 4ed6bad3b8c..50407efb317 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -90,6 +90,7 @@ class Fbcsr : public ::testing::Test {
         : exec(gko::ReferenceExecutor::create()),
           fbsample(exec),
           fbsample2(exec),
+          fbsamplesquare(exec),
           mtx(fbsample.generate_fbcsr()),
           refmtx(fbsample.generate_fbcsr()),
           refcsrmtx(fbsample.generate_csr()),
@@ -97,7 +98,8 @@ class Fbcsr : public ::testing::Test {
           refcoomtx(fbsample.generate_coo()),
           refspcmtx(fbsample.generate_sparsity_csr()),
           mtx2(fbsample2.generate_fbcsr()),
-          m2diag(fbsample2.extract_diagonal())
+          m2diag(fbsample2.extract_diagonal()),
+          mtxsq(fbsamplesquare.generate_fbcsr())
     {}
 
     // void create_mtx3(Mtx *sorted, Mtx *unsorted)
@@ -167,6 +169,8 @@ class Fbcsr : public ::testing::Test {
     std::shared_ptr<const gko::ReferenceExecutor> exec;
     const gko::testing::FbcsrSample<value_type, index_type> fbsample;
     const gko::testing::FbcsrSample2<value_type, index_type> fbsample2;
+    const gko::testing::FbcsrSampleSquare<value_type, index_type>
+        fbsamplesquare;
     std::unique_ptr<Mtx> mtx;
     const std::unique_ptr<const Mtx> refmtx;
     const std::unique_ptr<const Csr> refcsrmtx;
@@ -175,6 +179,7 @@ class Fbcsr : public ::testing::Test {
     const std::unique_ptr<const SparCsr> refspcmtx;
     const std::unique_ptr<const Mtx> mtx2;
     const std::unique_ptr<const Diag> m2diag;
+    const std::unique_ptr<const Mtx> mtxsq;
 };
 
 TYPED_TEST_CASE(Fbcsr, gko::test::ValueIndexTypes);
@@ -336,8 +341,8 @@ TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseMatrix)
 }
 
 
-TYPED_TEST(Fbcsr, AppliesToFbcsrMatrix)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, AppliesToFbcsrMatrix)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using T = typename TestFixture::value_type;
@@ -369,8 +374,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, AppliesLinearCombinationToFbcsrMatrix)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, AppliesLinearCombinationToFbcsrMatrix)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Vec = typename TestFixture::Vec;
@@ -407,8 +412,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, AppliesLinearCombinationToIdentityMatrix)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, AppliesLinearCombinationToIdentityMatrix)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using T = typename TestFixture::value_type;
@@ -442,39 +447,33 @@ GKO_NOT_IMPLEMENTED;
 
 
 TYPED_TEST(Fbcsr, ApplyFailsOnWrongInnerDimension)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Vec = typename TestFixture::Vec;
-//    auto x = Vec::create(this->exec, gko::dim<2>{2});
-//    auto y = Vec::create(this->exec, gko::dim<2>{2});
-//
-//    ASSERT_THROW(this->mtx->apply(x.get(), y.get()), gko::DimensionMismatch);
-//}
+{
+    using Vec = typename TestFixture::Vec;
+    auto x = Vec::create(this->exec, gko::dim<2>{2});
+    auto y = Vec::create(this->exec, gko::dim<2>{this->fbsample.nrows});
+
+    ASSERT_THROW(this->mtx->apply(x.get(), y.get()), gko::DimensionMismatch);
+}
 
 
 TYPED_TEST(Fbcsr, ApplyFailsOnWrongNumberOfRows)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Vec = typename TestFixture::Vec;
-//    auto x = Vec::create(this->exec, gko::dim<2>{3, 2});
-//    auto y = Vec::create(this->exec, gko::dim<2>{3, 2});
-//
-//    ASSERT_THROW(this->mtx->apply(x.get(), y.get()), gko::DimensionMismatch);
-//}
+{
+    using Vec = typename TestFixture::Vec;
+    auto x = Vec::create(this->exec, gko::dim<2>{this->fbsample.ncols, 2});
+    auto y = Vec::create(this->exec, gko::dim<2>{3, 2});
+
+    ASSERT_THROW(this->mtx->apply(x.get(), y.get()), gko::DimensionMismatch);
+}
 
 
 TYPED_TEST(Fbcsr, ApplyFailsOnWrongNumberOfCols)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Vec = typename TestFixture::Vec;
-//    auto x = Vec::create(this->exec, gko::dim<2>{3});
-//    auto y = Vec::create(this->exec, gko::dim<2>{2});
-//
-//    ASSERT_THROW(this->mtx->apply(x.get(), y.get()), gko::DimensionMismatch);
-//}
+{
+    using Vec = typename TestFixture::Vec;
+    auto x = Vec::create(this->exec, gko::dim<2>{this->fbsample.ncols, 3});
+    auto y = Vec::create(this->exec, gko::dim<2>{this->fbsample.nrows, 2});
+
+    ASSERT_THROW(this->mtx->apply(x.get(), y.get()), gko::DimensionMismatch);
+}
 
 
 TYPED_TEST(Fbcsr, ConvertsToPrecision)
@@ -1014,27 +1013,15 @@ TYPED_TEST(Fbcsr, CalculatesNonzerosPerRow)
 
 
 TYPED_TEST(Fbcsr, SquareMtxIsTransposable)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Fbcsr = typename TestFixture::Mtx;
-//    // clang-format off
-//    auto mtx2 = gko::initialize<Fbcsr>(
-//                {{1.0, 3.0, 2.0},
-//                 {0.0, 5.0, 0.0},
-//                 {0.0, 1.5, 2.0}}, this->exec);
-//    // clang-format on
-//
-//    auto trans = mtx2->transpose();
-//    auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
-//
-//    // clang-format off
-//    GKO_ASSERT_MTX_NEAR(trans_as_fbcsr,
-//                    l({{1.0, 0.0, 0.0},
-//                       {3.0, 5.0, 1.5},
-//                       {2.0, 0.0, 2.0}}), 0.0);
-//    // clang-format on
-//}
+{
+    using Fbcsr = typename TestFixture::Mtx;
+    auto reftmtx = this->fbsamplesquare.generate_transpose_fbcsr();
+
+    auto trans = this->mtxsq->transpose();
+    auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
+
+    GKO_ASSERT_MTX_NEAR(trans_as_fbcsr, reftmtx, 0.0);
+}
 
 
 TYPED_TEST(Fbcsr, NonSquareMtxIsTransposable)

From 0092ced1ad36271adec08a8ba12c60cf527bbefa Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Mon, 16 Nov 2020 15:25:53 +0100
Subject: [PATCH 0537/1008] [tests fail] enabled fbcsr non-square transpose
 test

---
 core/test/matrix/fbcsr_sample.cpp       | 42 +++++++++++++++++++++++++
 core/test/matrix/fbcsr_sample.hpp       |  2 ++
 reference/test/matrix/fbcsr_kernels.cpp | 23 ++++++--------
 3 files changed, 53 insertions(+), 14 deletions(-)

diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
index aa8299d33c8..d0cbe649efe 100644
--- a/core/test/matrix/fbcsr_sample.cpp
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -452,6 +452,48 @@ FbcsrSample2<ValueType, IndexType>::generate_fbcsr() const
     return mtx;
 }
 
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
+FbcsrSample2<ValueType, IndexType>::generate_transpose_fbcsr() const
+{
+    std::unique_ptr<Fbcsr> mtx =
+        Fbcsr::create(exec,
+                      gko::dim<2>{static_cast<size_type>(ncols),
+                                  static_cast<size_type>(nrows)},
+                      nnz, bs, std::make_shared<matstr::classical<Fbcsr>>());
+
+    value_type *const v = mtx->get_values();
+    index_type *const c = mtx->get_col_idxs();
+    index_type *const r = mtx->get_row_ptrs();
+    index_type *const s = mtx->get_srow();
+    r[0] = 0;
+    r[1] = 2;
+    r[2] = 2;
+    r[3] = 3;
+    r[4] = 4;
+    c[0] = 0;
+    c[1] = 1;
+    c[2] = 2;
+    c[3] = 1;
+
+    for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
+
+    v[0] = 1;
+    v[1] = 3;
+    v[2] = 2;
+    v[3] = 0;
+    v[8] = -12;
+    v[9] = -2;
+    v[10] = -1;
+    v[11] = -11;
+    v[13] = 0;
+    v[15] = 0;
+
+    for (index_type is = 0; is < mtx->get_num_srow_elements(); is++) s[is] = 0;
+
+    return mtx;
+}
+
 template <typename ValueType, typename IndexType>
 std::unique_ptr<gko::matrix::Diagonal<ValueType>>
 FbcsrSample2<ValueType, IndexType>::extract_diagonal() const
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index a0990887477..48a68343d94 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -116,6 +116,8 @@ class FbcsrSample2 {
 
     std::unique_ptr<Fbcsr> generate_fbcsr() const;
 
+    std::unique_ptr<Fbcsr> generate_transpose_fbcsr() const;
+
     std::unique_ptr<Diagonal> extract_diagonal() const;
 
     void apply(const Dense *x, Dense *y) const;
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 50407efb317..229fcb20e26 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -1025,20 +1025,15 @@ TYPED_TEST(Fbcsr, SquareMtxIsTransposable)
 
 
 TYPED_TEST(Fbcsr, NonSquareMtxIsTransposable)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Fbcsr = typename TestFixture::Mtx;
-//    auto trans = this->mtx->transpose();
-//    auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
-//
-//    // clang-format off
-//    GKO_ASSERT_MTX_NEAR(trans_as_fbcsr,
-//                    l({{1.0, 0.0},
-//                       {3.0, 5.0},
-//                       {2.0, 0.0}}), 0.0);
-//    // clang-format on
-//}
+{
+    using Fbcsr = typename TestFixture::Mtx;
+    auto reftmtx = this->fbsample2.generate_transpose_fbcsr();
+
+    auto trans = this->mtx2->transpose();
+    auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
+
+    GKO_ASSERT_MTX_NEAR(trans_as_fbcsr, reftmtx, 0.0);
+}
 
 
 TYPED_TEST(Fbcsr, SquareMatrixIsRowPermutable)

From 3fbb9e97ca16411aaca568d2f6ea52200e196c66 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 18 Nov 2020 18:29:41 +0100
Subject: [PATCH 0538/1008] [tests fail] fbcsr absolute value tests pass

---
 core/matrix/fbcsr.cpp                   |  62 ++++-----
 core/test/matrix/fbcsr_sample.cpp       | 174 ++++++++++++++++++++++--
 core/test/matrix/fbcsr_sample.hpp       |  39 ++++++
 reference/test/matrix/fbcsr_kernels.cpp | 136 +++++++++---------
 4 files changed, 298 insertions(+), 113 deletions(-)

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 8b239617991..d94aa5d6e81 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -674,14 +674,12 @@ void Fbcsr<ValueType, IndexType>::sort_by_column_index() GKO_NOT_IMPLEMENTED;
 
 template <typename ValueType, typename IndexType>
 bool Fbcsr<ValueType, IndexType>::is_sorted_by_column_index() const
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto exec = this->get_executor();
-//    bool is_sorted;
-//    exec->run(fbcsr::make_is_sorted_by_column_index(this, &is_sorted));
-//    return is_sorted;
-//}
+{
+    auto exec = this->get_executor();
+    bool is_sorted;
+    exec->run(fbcsr::make_is_sorted_by_column_index(this, &is_sorted));
+    return is_sorted;
+}
 
 
 template <typename ValueType, typename IndexType>
@@ -701,35 +699,33 @@ Fbcsr<ValueType, IndexType>::extract_diagonal() const
 
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::compute_absolute_inplace()
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto exec = this->get_executor();
-//
-//    exec->run(fbcsr::make_inplace_absolute_array(
-//        this->get_values(), this->get_num_stored_elements()));
-//}
+{
+    auto exec = this->get_executor();
+
+    exec->run(fbcsr::make_inplace_absolute_array(
+        this->get_values(), this->get_num_stored_elements()));
+}
 
 
 template <typename ValueType, typename IndexType>
 std::unique_ptr<typename Fbcsr<ValueType, IndexType>::absolute_type>
-Fbcsr<ValueType, IndexType>::compute_absolute() const GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto exec = this->get_executor();
-//
-//    auto abs_fbcsr = absolute_type::create(exec, this->get_size(),
-//                                         this->get_num_stored_elements());
-//
-//    abs_fbcsr->col_idxs_ = col_idxs_;
-//    abs_fbcsr->row_ptrs_ = row_ptrs_;
-//    exec->run(fbcsr::make_outplace_absolute_array(this->get_const_values(),
-//                                                this->get_num_stored_elements(),
-//                                                abs_fbcsr->get_values()));
-//
-//    convert_strategy_helper(abs_fbcsr.get());
-//    return abs_fbcsr;
-//}
+Fbcsr<ValueType, IndexType>::compute_absolute() const
+{
+    auto exec = this->get_executor();
+
+    auto abs_fbcsr = absolute_type::create(exec, this->get_size(),
+                                           this->get_num_stored_elements(),
+                                           this->get_block_size());
+
+    abs_fbcsr->col_idxs_ = col_idxs_;
+    abs_fbcsr->row_ptrs_ = row_ptrs_;
+    exec->run(fbcsr::make_outplace_absolute_array(
+        this->get_const_values(), this->get_num_stored_elements(),
+        abs_fbcsr->get_values()));
+
+    convert_strategy_helper(abs_fbcsr.get());
+    return abs_fbcsr;
+}
 
 
 // TODO clean this up as soon as we improve strategy_type
diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
index d0cbe649efe..6e7815f96d2 100644
--- a/core/test/matrix/fbcsr_sample.cpp
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -40,12 +40,56 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #define FBCSR_TEST_OFFSET 0.000011118888
 
+#define FBCSR_TEST_C_MAG 0.1 + FBCSR_TEST_OFFSET
+#define FBCSR_TEST_IMAGINARY \
+    sct(std::complex<remove_complex<ValueType>>(0, FBCSR_TEST_C_MAG))
+
 namespace gko {
 namespace testing {
 
 
 namespace matstr = gko::matrix::matrix_strategy;
 
+/// Generates a copu of the given matrix with a different scalar type
+/** \tparam AbsValueType The scalar type of the output matrix
+ */
+template <typename FbcsrType, typename AbsValueType>
+static std::unique_ptr<
+    gko::matrix::Fbcsr<AbsValueType, typename FbcsrType::index_type>>
+generate_acopy_impl(const FbcsrType *const mat)
+{
+    // using AbsValueType = typename gko::remove_complex<ValueType>;
+    using index_type = typename FbcsrType::index_type;
+    using value_type = typename FbcsrType::value_type;
+    using AbsFbcsr = gko::matrix::Fbcsr<AbsValueType, index_type>;
+    using classical = matstr::classical<AbsFbcsr>;
+
+    std::shared_ptr<const ReferenceExecutor> exec =
+        std::dynamic_pointer_cast<const ReferenceExecutor>(mat->get_executor());
+
+    std::unique_ptr<AbsFbcsr> amat =
+        AbsFbcsr::create(exec, mat->get_size(), mat->get_num_stored_elements(),
+                         mat->get_block_size(), std::make_shared<classical>());
+
+    const index_type *const colidxs = mat->get_col_idxs();
+    const index_type *const rowptrs = mat->get_row_ptrs();
+    index_type *const acolidxs = amat->get_col_idxs();
+    index_type *const arowptrs = amat->get_row_ptrs();
+    // blockutils
+    for (index_type i = 0;
+         i < mat->get_num_stored_elements() /
+                 (mat->get_block_size() * mat->get_block_size());
+         i++)
+        acolidxs[i] = colidxs[i];
+    // blockutils
+    for (index_type i = 0; i < mat->get_size()[0] / mat->get_block_size() + 1;
+         i++)
+        arowptrs[i] = rowptrs[i];
+
+    return amat;
+}
+
+
 template <typename ValueType, typename IndexType>
 FbcsrSample<ValueType, IndexType>::FbcsrSample(
     const std::shared_ptr<const gko::ReferenceExecutor> rexec)
@@ -88,6 +132,7 @@ FbcsrSample<ValueType, IndexType>::generate_fbcsr() const
                                 mtx->get_size()[0], mtx->get_size()[1],
                                 "block size does not divide the size!");
 
+    // blockutils
     for (index_type ibrow = 0; ibrow < mtx->get_size()[0] / bs; ibrow++) {
         const index_type *const browptr = mtx->get_row_ptrs();
         for (index_type inz = browptr[ibrow]; inz < browptr[ibrow + 1]; inz++) {
@@ -106,14 +151,70 @@ FbcsrSample<ValueType, IndexType>::generate_fbcsr() const
     vals(0, 2, 2) = gko::zero<value_type>();
     vals(3, 0, 0) = gko::zero<value_type>();
 
-    v[34] += FBCSR_TEST_OFFSET;
-    v[35] += FBCSR_TEST_OFFSET;
+    v[34] += FBCSR_TEST_IMAGINARY;
+    v[35] += FBCSR_TEST_IMAGINARY;
 
     for (index_type is = 0; is < mtx->get_num_srow_elements(); is++) s[is] = 0;
 
     return mtx;
 }
 
+template <typename ValueType, typename IndexType>
+template <typename FbcsrType>
+void FbcsrSample<ValueType, IndexType>::correct_abs_for_complex_values(
+    FbcsrType *const amat) const
+{
+    using out_value_type = typename FbcsrType::value_type;
+    using outreal_type = remove_complex<out_value_type>;
+    out_value_type *const avals = amat->get_values();
+    if (is_complex<ValueType>()) {
+        auto mo = static_cast<outreal_type>(FBCSR_TEST_C_MAG);
+        avals[34] = sqrt(pow(static_cast<outreal_type>(13.0), 2) +
+                         pow(static_cast<outreal_type>(mo), 2));
+        avals[35] = sqrt(pow(static_cast<outreal_type>(14.0), 2) +
+                         pow(static_cast<outreal_type>(mo), 2));
+    }
+}
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Fbcsr<remove_complex<ValueType>, IndexType>>
+FbcsrSample<ValueType, IndexType>::generate_abs_fbcsr_abstype() const
+{
+    using AbsValueType = typename gko::remove_complex<ValueType>;
+    using AbsFbcsr = gko::matrix::Fbcsr<AbsValueType, IndexType>;
+
+    const std::unique_ptr<const Fbcsr> mat = generate_fbcsr();
+    std::unique_ptr<AbsFbcsr> amat =
+        generate_acopy_impl<Fbcsr, AbsValueType>(mat.get());
+
+    AbsValueType *const avals = amat->get_values();
+    const ValueType *const vals = mat->get_values();
+    for (IndexType i = 0; i < amat->get_num_stored_elements(); i++)
+        avals[i] = abs(vals[i]);
+
+    correct_abs_for_complex_values(amat.get());
+
+    return amat;
+}
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
+FbcsrSample<ValueType, IndexType>::generate_abs_fbcsr() const
+{
+    const std::unique_ptr<const Fbcsr> mat = generate_fbcsr();
+    std::unique_ptr<Fbcsr> amat =
+        generate_acopy_impl<Fbcsr, ValueType>(mat.get());
+
+    ValueType *const avals = amat->get_values();
+    const ValueType *const vals = mat->get_values();
+    for (IndexType i = 0; i < amat->get_num_stored_elements(); i++)
+        avals[i] = abs(vals[i]);
+
+    correct_abs_for_complex_values(amat.get());
+
+    return amat;
+}
+
 template <typename ValueType, typename IndexType>
 std::unique_ptr<gko::matrix::Csr<ValueType, IndexType>>
 FbcsrSample<ValueType, IndexType>::generate_csr() const
@@ -214,8 +315,8 @@ FbcsrSample<ValueType, IndexType>::generate_csr() const
     csrvals[34] = 13;
     csrvals[35] = 14;
 
-    csrvals[34] += FBCSR_TEST_OFFSET;
-    csrvals[35] += FBCSR_TEST_OFFSET;
+    csrvals[34] += FBCSR_TEST_IMAGINARY;
+    csrvals[35] += FBCSR_TEST_IMAGINARY;
 
     return csrm;
 }
@@ -241,8 +342,8 @@ FbcsrSample<ValueType, IndexType>::generate_dense() const
         }
 
     densem->at(2, 3) = densem->at(2, 5) = densem->at(3, 6) = 0.0;
-    densem->at(5, 7) += FBCSR_TEST_OFFSET;
-    densem->at(5, 8) += FBCSR_TEST_OFFSET;
+    densem->at(5, 7) += FBCSR_TEST_IMAGINARY;
+    densem->at(5, 8) += FBCSR_TEST_IMAGINARY;
 
     return densem;
 }
@@ -287,8 +388,8 @@ FbcsrSample<ValueType, IndexType>::generate_matrix_data() const
                      {4, 7, 10.0},
                      {4, 8, 11.0},
                      {5, 6, 12.0},
-                     {5, 7, 13.0 + FBCSR_TEST_OFFSET},
-                     {5, 8, 14.0 + FBCSR_TEST_OFFSET}}});
+                     {5, 7, sct(13.0) + FBCSR_TEST_IMAGINARY},
+                     {5, 8, sct(14.0) + FBCSR_TEST_IMAGINARY}}});
 }
 
 // Assuming row-major blocks
@@ -334,8 +435,8 @@ gko::matrix_data<ValueType, IndexType> FbcsrSample<
                      {4, 7, 10.0},
                      {4, 8, 11.0},
                      {5, 6, 12.0},
-                     {5, 7, 13.0 + FBCSR_TEST_OFFSET},
-                     {5, 8, 14.0 + FBCSR_TEST_OFFSET}}});
+                     {5, 7, sct(13.0) + FBCSR_TEST_IMAGINARY},
+                     {5, 8, sct(14.0) + FBCSR_TEST_IMAGINARY}}});
 }
 
 template <typename ValueType, typename IndexType>
@@ -452,6 +553,59 @@ FbcsrSample2<ValueType, IndexType>::generate_fbcsr() const
     return mtx;
 }
 
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Fbcsr<remove_complex<ValueType>, IndexType>>
+FbcsrSample2<ValueType, IndexType>::generate_abs_fbcsr_abstype() const
+{
+    using AbsValueType = typename gko::remove_complex<ValueType>;
+    using AbsFbcsr = gko::matrix::Fbcsr<AbsValueType, IndexType>;
+
+    const std::unique_ptr<const Fbcsr> mat = generate_fbcsr();
+    std::unique_ptr<AbsFbcsr> amat =
+        generate_acopy_impl<Fbcsr, AbsValueType>(mat.get());
+
+    AbsValueType *const v = amat->get_values();
+
+    for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
+    v[0] = 1;
+    v[1] = 2;
+    v[2] = 3;
+    v[3] = 0;
+    v[10] = 0;
+    v[11] = 0;
+    v[12] = 12;
+    v[13] = 1;
+    v[14] = 2;
+    v[15] = 11;
+
+    return amat;
+}
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
+FbcsrSample2<ValueType, IndexType>::generate_abs_fbcsr() const
+{
+    const std::unique_ptr<const Fbcsr> mat = generate_fbcsr();
+    std::unique_ptr<Fbcsr> amat =
+        generate_acopy_impl<Fbcsr, ValueType>(mat.get());
+
+    ValueType *const v = amat->get_values();
+
+    for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
+    v[0] = 1.0;
+    v[1] = 2.0;
+    v[2] = 3.0;
+    v[3] = 0.0;
+    v[10] = 0.0;
+    v[11] = 0.0;
+    v[12] = 12.0;
+    v[13] = 1.0;
+    v[14] = 2.0;
+    v[15] = 11.0;
+
+    return amat;
+}
+
 template <typename ValueType, typename IndexType>
 std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
 FbcsrSample2<ValueType, IndexType>::generate_transpose_fbcsr() const
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 48a68343d94..51208c3dfb7 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -55,6 +55,7 @@ class FbcsrSample {
 public:
     using value_type = ValueType;
     using index_type = IndexType;
+    using absvalue_type = remove_complex<value_type>;
     using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
     using Coo = gko::matrix::Coo<value_type, index_type>;
@@ -86,6 +87,12 @@ class FbcsrSample {
     ///  (not block-row)
     gko::Array<index_type> getNonzerosPerRow() const;
 
+    std::unique_ptr<Fbcsr> generate_abs_fbcsr() const;
+
+    std::unique_ptr<gko::matrix::Fbcsr<remove_complex<value_type>, index_type>>
+    generate_abs_fbcsr_abstype() const;
+
+
     const size_type nrows;
     const size_type ncols;
     const size_type nnz;
@@ -94,6 +101,32 @@ class FbcsrSample {
     const size_type nbnz;
     const int bs;
     const std::shared_ptr<const gko::Executor> exec;
+
+private:
+    // template <typename AbsValueType>
+    // void
+    // correct_abs_for_complex(gko::matrix::Fbcsr<AbsValueType,IndexType> *amat)
+    //     const;
+    template <typename FbcsrType>
+    void correct_abs_for_complex_values(FbcsrType *const mat) const;
+
+    /// Enables complex data to be used for complex instantiations...
+    template <typename U>
+    constexpr std::enable_if_t<!is_complex<U>() || is_complex<ValueType>(),
+                               ValueType>
+    sct(U u) const
+    {
+        return static_cast<ValueType>(u);
+    }
+
+    /// ... while ignoring imaginary parts for real instantiations
+    template <typename U>
+    constexpr std::enable_if_t<is_complex<U>() && !is_complex<ValueType>(),
+                               ValueType>
+    sct(U u) const
+    {
+        return static_cast<ValueType>(u.real());
+    }
 };
 
 /// Generates the a sample block CSR matrix in different formats
@@ -124,6 +157,12 @@ class FbcsrSample2 {
 
     gko::Array<index_type> getNonzerosPerRow() const;
 
+    std::unique_ptr<Fbcsr> generate_abs_fbcsr() const;
+
+    std::unique_ptr<gko::matrix::Fbcsr<remove_complex<value_type>, index_type>>
+    generate_abs_fbcsr_abstype() const;
+
+    /// Enables use of literals to instantiate value data
     template <typename U>
     inline constexpr ValueType sct(U u) const
     {
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 229fcb20e26..5185e03b5e1 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
+#include <iostream>
 
 
 #include <gtest/gtest.h>
@@ -65,9 +66,6 @@ namespace {
 namespace matstr = gko::matrix::matrix_strategy;
 
 
-constexpr int mat_bs = 1;
-
-
 template <typename ValueIndexType>
 class Fbcsr : public ::testing::Test {
 protected:
@@ -192,7 +190,7 @@ constexpr T get_some_number()
 }
 
 template <typename T>
-constexpr typename std::enable_if_t<gko::is_complex<T>> get_some_number()
+constexpr typename std::enable_if_t<gko::is_complex<T>, T> get_some_number()
 {
     using RT = gko::remove_complex<T>;
     return {static_cast<RT>(1.2), static_cast<RT>(3.4)};
@@ -1301,35 +1299,35 @@ TYPED_TEST(Fbcsr, ExtractsDiagonal)
 
 
 TYPED_TEST(Fbcsr, InplaceAbsolute)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    auto mtx = gko::initialize<Mtx>(
-//        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
-//
-//    mtx->compute_absolute_inplace();
-//
-//    GKO_ASSERT_MTX_NEAR(
-//        mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto mtx = this->fbsample2.generate_fbcsr();
+    const std::unique_ptr<const Mtx> refabs =
+        this->fbsample2.generate_abs_fbcsr();
+
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    const value_type *const refvals = refabs->get_const_values();
+
+    mtx->compute_absolute_inplace();
+
+    GKO_ASSERT_MTX_NEAR(mtx, refabs, 0.0);
+}
 
 
 TYPED_TEST(Fbcsr, OutplaceAbsolute)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    auto mtx = gko::initialize<Mtx>(
-//        {{1.0, 2.0, -2.0}, {3.0, -5.0, 0.0}, {0.0, 1.0, -1.5}}, this->exec);
-//
-//    auto abs_mtx = mtx->compute_absolute();
-//
-//    GKO_ASSERT_MTX_NEAR(
-//        abs_mtx, l({{1.0, 2.0, 2.0}, {3.0, 5.0, 0.0}, {0.0, 1.0, 1.5}}), 0.0);
-//    ASSERT_EQ(mtx->get_strategy()->get_name(),
-//              abs_mtx->get_strategy()->get_name());
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using AbsMtx = typename gko::remove_complex<typename TestFixture::Mtx>;
+
+    auto mtx = this->fbsample2.generate_fbcsr();
+    const std::unique_ptr<const AbsMtx> refabs =
+        this->fbsample2.generate_abs_fbcsr_abstype();
+
+    auto abs_mtx = mtx->compute_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx, refabs, 0.0);
+}
 
 
 template <typename ValueIndexType>
@@ -1373,49 +1371,47 @@ GKO_NOT_IMPLEMENTED;
 
 
 TYPED_TEST(FbcsrComplex, InplaceAbsolute)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using T = typename TestFixture::value_type;
-//    using index_type = typename TestFixture::index_type;
-//    auto exec = gko::ReferenceExecutor::create();
-//    // clang-format off
-//    auto mtx = gko::initialize<Mtx>(
-//        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
-//         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
-//         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
-//    // clang-format on
-//
-//    mtx->compute_absolute_inplace();
-//
-//    GKO_ASSERT_MTX_NEAR(
-//        mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    gko::testing::FbcsrSample<value_type, index_type> fbsample(
+        gko::ReferenceExecutor::create());
+    auto mtx = fbsample.generate_fbcsr();
+
+    std::cout << " Generated fbcsr: " << mtx->get_values()[34] << ", "
+              << mtx->get_values()[35] << std::endl;
+
+    const std::unique_ptr<const Mtx> refabs = fbsample.generate_abs_fbcsr();
+
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    const value_type *const refvals = refabs->get_const_values();
+
+    mtx->compute_absolute_inplace();
+
+    GKO_ASSERT_MTX_NEAR(mtx, refabs, 0.0);
+}
 
 
 TYPED_TEST(FbcsrComplex, OutplaceAbsolute)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Mtx = typename TestFixture::Mtx;
-//    using T = typename TestFixture::value_type;
-//    using index_type = typename TestFixture::index_type;
-//    auto exec = gko::ReferenceExecutor::create();
-//    // clang-format off
-//    auto mtx = gko::initialize<Mtx>(
-//        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
-//         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
-//         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
-//    // clang-format on
-//
-//    auto abs_mtx = mtx->compute_absolute();
-//
-//    GKO_ASSERT_MTX_NEAR(
-//        abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
-//    ASSERT_EQ(mtx->get_strategy()->get_name(),
-//              abs_mtx->get_strategy()->get_name());
-//}
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using AbsMtx = typename gko::remove_complex<typename TestFixture::Mtx>;
+
+    gko::testing::FbcsrSample<value_type, index_type> fbsample(
+        gko::ReferenceExecutor::create());
+
+    auto mtx = fbsample.generate_fbcsr();
+    const std::unique_ptr<const AbsMtx> refabs =
+        fbsample.generate_abs_fbcsr_abstype();
+
+    auto abs_mtx = mtx->compute_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx, refabs, 0.0);
+}
 
 
 }  // namespace

From 0f5507eeed0253b8188e63194bd69074e67bf688 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 18 Nov 2020 19:46:56 +0100
Subject: [PATCH 0539/1008] [tests fail] moved block utils to public interface;
 Fbcsr can now return number of block-rows and -columns

---
 core/components/fixed_block.hpp         | 30 ----------
 core/matrix/fbcsr.cpp                   | 31 +++--------
 include/ginkgo/core/base/blockutils.hpp | 73 +++++++++++++++++++++++++
 include/ginkgo/core/matrix/fbcsr.hpp    | 14 ++++-
 include/ginkgo/ginkgo.hpp               |  1 +
 reference/matrix/fbcsr_kernels.cpp      | 23 ++++----
 6 files changed, 106 insertions(+), 66 deletions(-)
 create mode 100644 include/ginkgo/core/base/blockutils.hpp

diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
index f2ff770cafb..8b6c433dfff 100644
--- a/core/components/fixed_block.hpp
+++ b/core/components/fixed_block.hpp
@@ -43,36 +43,6 @@ namespace gko {
 namespace blockutils {
 
 
-/// Error that denotes issues between block sizes and matrix dimensions
-template <typename IndexType>
-class BlockSizeError : public Error {
-public:
-    BlockSizeError(const std::string &file, const int line,
-                   const int block_size, const IndexType size)
-        : Error(file, line,
-                " block size = " + std::to_string(block_size) +
-                    ", size = " + std::to_string(size))
-    {}
-};
-
-/// Error that denotes issues between block sizes and matrix dimensions
-template <typename IndexType>
-class BlockReadError : public Error {
-public:
-    BlockReadError(const std::string &file, const int line,
-                   const std::string &msg)
-        : Error(file, line, msg)
-    {}
-};
-
-template <typename IndexType>
-IndexType getNumFixedBlocks(const int block_size, const IndexType size)
-{
-    if (size % block_size != 0)
-        throw BlockSizeError<IndexType>(__FILE__, __LINE__, block_size, size);
-    return size / block_size;
-}
-
 /// A dense block of values with compile-time constant dimensions
 /** The blocks are stored row-major. However, in future,
  *  a layout template parameter can be added if needed.
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index d94aa5d6e81..0153c3b77ae 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -104,27 +104,14 @@ Fbcsr<ValueType, IndexType>::Fbcsr(std::shared_ptr<const Executor> exec,
                                    std::shared_ptr<strategy_type> strategy)
     : EnableLinOp<Fbcsr>(exec, size),
       bs_{block_size},
+      nbcols_{gko::blockutils::getNumBlocks(block_size, size[1])},
       values_(exec, num_nonzeros),
-      col_idxs_(exec, gko::blockutils::getNumFixedBlocks(
-                          block_size * block_size, num_nonzeros)),
-      row_ptrs_(exec,
-                gko::blockutils::getNumFixedBlocks(block_size, size[0]) + 1),
+      col_idxs_(exec, gko::blockutils::getNumBlocks(block_size * block_size,
+                                                    num_nonzeros)),
+      row_ptrs_(exec, gko::blockutils::getNumBlocks(block_size, size[0]) + 1),
       startrow_(exec, strategy->calc_size(num_nonzeros)),
       strategy_(strategy->copy())
-{
-    if (size[0] % bs_ != 0)
-        throw gko::BadDimension(__FILE__, __LINE__, __func__, "construct",
-                                size[0], size[1],
-                                "block size does not divide the dim 0!");
-    if (size[1] % bs_ != 0)
-        throw gko::BadDimension(__FILE__, __LINE__, __func__, "construct",
-                                size[0], size[1],
-                                "block size does not divide the dim 1!");
-    if (num_nonzeros % (bs_ * bs_) != 0)
-        throw gko::BadDimension(__FILE__, __LINE__, __func__, "construct",
-                                size[0], size[1],
-                                "block size^2 does not divide NNZ!");
-}
+{}
 
 
 template <typename ValueType, typename IndexType>
@@ -360,13 +347,13 @@ template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(
     SparsityCsr<ValueType, IndexType> *result) const
 {
-    using gko::blockutils::getNumFixedBlocks;
+    using gko::blockutils::getNumBlocks;
     auto exec = this->get_executor();
     auto tmp = SparsityCsr<ValueType, IndexType>::create(
         exec,
-        gko::dim<2>{getNumFixedBlocks(bs_, this->get_size()[0]),
-                    getNumFixedBlocks(bs_, this->get_size()[1])},
-        getNumFixedBlocks(bs_ * bs_, this->get_num_stored_elements()));
+        gko::dim<2>{getNumBlocks(bs_, this->get_size()[0]),
+                    getNumBlocks(bs_, this->get_size()[1])},
+        getNumBlocks(bs_ * bs_, this->get_num_stored_elements()));
 
     tmp->col_idxs_ = this->col_idxs_;
     tmp->row_ptrs_ = this->row_ptrs_;
diff --git a/include/ginkgo/core/base/blockutils.hpp b/include/ginkgo/core/base/blockutils.hpp
new file mode 100644
index 00000000000..4bd4d3d040d
--- /dev/null
+++ b/include/ginkgo/core/base/blockutils.hpp
@@ -0,0 +1,73 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+
+#ifndef GINKGO_CORE_BASE_BLOCKUTILS_HPP_
+#define GINKGO_CORE_BASE_BLOCKUTILS_HPP_
+
+
+#include <ginkgo/core/base/exception.hpp>
+
+
+namespace gko {
+namespace blockutils {
+
+
+/// Error that denotes issues between block sizes and matrix dimensions
+template <typename IndexType>
+class BlockSizeError : public Error {
+public:
+    BlockSizeError(const std::string &file, const int line,
+                   const int block_size, const IndexType size)
+        : Error(file, line,
+                " block size = " + std::to_string(block_size) +
+                    ", size = " + std::to_string(size))
+    {}
+};
+
+
+/** Returns the quotient of the second arg divided by the first
+ * but throws when they don't divide
+ */
+template <typename IndexType>
+IndexType getNumBlocks(const int block_size, const IndexType size)
+{
+    if (size % block_size != 0)
+        throw BlockSizeError<IndexType>(__FILE__, __LINE__, block_size, size);
+    return size / block_size;
+}
+
+
+}  // namespace blockutils
+}  // namespace gko
+
+#endif
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index 836169ba0c3..141c1a3c3eb 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/blockutils.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/base/math.hpp>
 
@@ -376,6 +377,13 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
 
     void set_block_size(const int block_size) { bs_ = block_size; }
 
+    index_type get_num_block_rows() const
+    {
+        return row_ptrs_.get_num_elems() - 1;
+    }
+
+    index_type get_num_block_cols() const { return nbcols_; }
+
 protected:
     using classical = matrix_strategy::classical<Fbcsr<value_type, index_type>>;
 
@@ -434,6 +442,7 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
         std::shared_ptr<strategy_type> strategy = std::make_shared<classical>())
         : EnableLinOp<Fbcsr>(exec, size),
           bs_{block_size},
+          nbcols_{gko::blockutils::getNumBlocks(block_size, size[1])},
           values_{exec, std::forward<ValuesArray>(values)},
           col_idxs_{exec, std::forward<ColIdxsArray>(col_idxs)},
           row_ptrs_{exec, std::forward<RowPtrsArray>(row_ptrs)},
@@ -461,12 +470,13 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     void make_srow()
     {
         startrow_.resize_and_reset(
-            strategy_->calc_size(values_.get_num_elems() / bs_ / bs_));
+            strategy_->calc_size(col_idxs_.get_num_elems()));
         strategy_->process(row_ptrs_, &startrow_);
     }
 
 private:
-    int bs_;  ///< Block size
+    int bs_;            ///< Block size
+    size_type nbcols_;  ///< Number of block-columns
     Array<value_type> values_;
     Array<index_type> col_idxs_;
     Array<index_type> row_ptrs_;
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 30408b8970b..541980beaa3 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/abstract_factory.hpp>
 #include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/blockutils.hpp>
 #include <ginkgo/core/base/combination.hpp>
 #include <ginkgo/core/base/composition.hpp>
 #include <ginkgo/core/base/dim.hpp>
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index fe1ec87061e..9601df09d1d 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/blockutils.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/matrix/coo.hpp>
@@ -78,7 +79,7 @@ void spmv(const std::shared_ptr<const ReferenceExecutor> exec,
     const int bs = a->get_block_size();
     const IndexType nvecs = static_cast<IndexType>(b->get_size()[1]);
     const IndexType nbrows =
-        gko::blockutils::getNumFixedBlocks(bs, a->get_size()[0]);
+        gko::blockutils::getNumBlocks(bs, a->get_size()[0]);
     auto row_ptrs = a->get_const_row_ptrs();
     auto col_idxs = a->get_const_col_idxs();
     auto vals = a->get_const_values();
@@ -121,7 +122,7 @@ void advanced_spmv(const std::shared_ptr<const ReferenceExecutor> exec,
     const int bs = a->get_block_size();
     const IndexType nvecs = static_cast<IndexType>(b->get_size()[1]);
     const IndexType nbrows =
-        gko::blockutils::getNumFixedBlocks(bs, a->get_size()[0]);
+        gko::blockutils::getNumBlocks(bs, a->get_size()[0]);
     auto row_ptrs = a->get_const_row_ptrs();
     auto col_idxs = a->get_const_col_idxs();
     auto vals = a->get_const_values();
@@ -417,9 +418,9 @@ void convert_to_dense(const std::shared_ptr<const ReferenceExecutor> exec,
 {
     const int bs = source->get_block_size();
     const size_type nbrows =
-        gko::blockutils::getNumFixedBlocks(bs, source->get_size()[0]);
+        gko::blockutils::getNumBlocks(bs, source->get_size()[0]);
     const size_type nbcols =
-        gko::blockutils::getNumFixedBlocks(bs, source->get_size()[1]);
+        gko::blockutils::getNumBlocks(bs, source->get_size()[1]);
     const IndexType *const row_ptrs = source->get_const_row_ptrs();
     const IndexType *const col_idxs = source->get_const_col_idxs();
     const ValueType *const vals = source->get_const_values();
@@ -457,9 +458,9 @@ void convert_to_csr(const std::shared_ptr<const ReferenceExecutor> exec,
 {
     const int bs = source->get_block_size();
     const size_type nbrows =
-        gko::blockutils::getNumFixedBlocks(bs, source->get_size()[0]);
+        gko::blockutils::getNumBlocks(bs, source->get_size()[0]);
     const size_type nbcols =
-        gko::blockutils::getNumFixedBlocks(bs, source->get_size()[1]);
+        gko::blockutils::getNumBlocks(bs, source->get_size()[1]);
     const IndexType *const browptrs = source->get_const_row_ptrs();
     const IndexType *const bcolinds = source->get_const_col_idxs();
     const ValueType *const bvals = source->get_const_values();
@@ -696,11 +697,9 @@ void transpose_and_transform(
     auto orig_vals = orig->get_const_values();
 
     auto orig_num_cols = orig->get_size()[1];
-    const size_type nbcols =
-        gko::blockutils::getNumFixedBlocks(bs, orig_num_cols);
+    const size_type nbcols = gko::blockutils::getNumBlocks(bs, orig_num_cols);
     auto orig_num_rows = orig->get_size()[0];
-    const size_type nbrows =
-        gko::blockutils::getNumFixedBlocks(bs, orig_num_rows);
+    const size_type nbrows = gko::blockutils::getNumBlocks(bs, orig_num_rows);
     auto orig_nbnz = orig_row_ptrs[nbrows];
 
     trans_row_ptrs[0] = 0;
@@ -1019,7 +1018,7 @@ void is_sorted_by_column_index(
     const auto col_idxs = to_check->get_const_col_idxs();
     const auto size = to_check->get_size();
     const int bs = to_check->get_block_size();
-    const size_type nbrows = gko::blockutils::getNumFixedBlocks(bs, size[0]);
+    const size_type nbrows = gko::blockutils::getNumBlocks(bs, size[0]);
 
     for (size_type i = 0; i < nbrows; ++i) {
         for (auto idx = row_ptrs[i] + 1; idx < row_ptrs[i + 1]; ++idx) {
@@ -1048,7 +1047,7 @@ void extract_diagonal(std::shared_ptr<const ReferenceExecutor> exec,
     const int bs = orig->get_block_size();
     const size_type diag_size = diag->get_size()[0];
     const size_type nbrows =
-        gko::blockutils::getNumFixedBlocks(bs, orig->get_size()[0]);
+        gko::blockutils::getNumBlocks(bs, orig->get_size()[0]);
     auto diag_values = diag->get_values();
     assert(diag_size == orig->get_size()[0]);
 

From 5b3745467ac266cff48e6f30badd1d8654ee4865 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Thu, 19 Nov 2020 17:28:04 +0100
Subject: [PATCH 0540/1008] [tests fail] conjugate transpose works for fbcsr,
 disabled other tests

---
 core/matrix/fbcsr.cpp                   |  22 +++--
 core/test/matrix/fbcsr_sample.cpp       | 104 ++++++++++++++++++++++
 core/test/matrix/fbcsr_sample.hpp       |  40 ++++++++-
 cuda/test/matrix/CMakeLists.txt         |   2 +-
 hip/test/matrix/CMakeLists.txt          |   2 +-
 omp/test/matrix/CMakeLists.txt          |   2 +-
 reference/matrix/fbcsr_kernels.cpp      |  66 ++++++--------
 reference/test/matrix/fbcsr_kernels.cpp | 113 +++++++++---------------
 8 files changed, 223 insertions(+), 128 deletions(-)

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 0153c3b77ae..7d2d94209aa 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -557,18 +557,16 @@ std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::transpose() const
 
 template <typename ValueType, typename IndexType>
 std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::conj_transpose() const
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto exec = this->get_executor();
-//    auto trans_cpy =
-//        Fbcsr::create(exec, gko::transpose(this->get_size()),
-//                    this->get_num_stored_elements(), this->get_strategy());
-//
-//    exec->run(fbcsr::make_conj_transpose(this, trans_cpy.get()));
-//    trans_cpy->make_srow();
-//    return std::move(trans_cpy);
-//}
+{
+    auto exec = this->get_executor();
+    auto trans_cpy = Fbcsr::create(exec, gko::transpose(this->get_size()),
+                                   this->get_num_stored_elements(), bs_,
+                                   this->get_strategy());
+
+    exec->run(fbcsr::make_conj_transpose(this, trans_cpy.get()));
+    trans_cpy->make_srow();
+    return std::move(trans_cpy);
+}
 
 
 template <typename ValueType, typename IndexType>
diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
index 6e7815f96d2..b48562c2cf1 100644
--- a/core/test/matrix/fbcsr_sample.cpp
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -781,5 +781,109 @@ FbcsrSampleSquare<ValueType, IndexType>::generate_transpose_fbcsr() const
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_TEST_SAMPLE_SQUARE);
 
+
+template <typename ValueType, typename IndexType>
+FbcsrSampleComplex<ValueType, IndexType>::FbcsrSampleComplex(
+    const std::shared_ptr<const gko::ReferenceExecutor> rexec)
+    : nrows{6},
+      ncols{8},
+      nnz{16},
+      nbrows{3},
+      nbcols{4},
+      nbnz{4},
+      bs{2},
+      exec(rexec)
+{}
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
+FbcsrSampleComplex<ValueType, IndexType>::generate_fbcsr() const
+{
+    std::unique_ptr<Fbcsr> mtx =
+        Fbcsr::create(exec,
+                      gko::dim<2>{static_cast<size_type>(nrows),
+                                  static_cast<size_type>(ncols)},
+                      nnz, bs, std::make_shared<matstr::classical<Fbcsr>>());
+
+    value_type *const v = mtx->get_values();
+    index_type *const c = mtx->get_col_idxs();
+    index_type *const r = mtx->get_row_ptrs();
+    index_type *const s = mtx->get_srow();
+    r[0] = 0;
+    r[1] = 1;
+    r[2] = 3;
+    r[3] = 4;
+    c[0] = 0;
+    c[1] = 0;
+    c[2] = 3;
+    c[3] = 2;
+
+    for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
+
+    using namespace std::complex_literals;
+    v[0] = 1.0 + 1.15i;
+    v[1] = 2.0 + 2.15i;
+    v[2] = 3.0 - 3.15i;
+    v[3] = 0.0 - 0.15i;
+    v[10] = 0.0;
+    v[11] = 0.0;
+    v[12] = -12.0 + 12.15i;
+    v[13] = -1.0 + 1.15i;
+    v[14] = -2.0 - 2.15i;
+    v[15] = -11.0 - 11.15i;
+
+    for (index_type is = 0; is < mtx->get_num_srow_elements(); is++) s[is] = 0;
+
+    return mtx;
+}
+
+template <typename ValueType, typename IndexType>
+std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
+FbcsrSampleComplex<ValueType, IndexType>::generate_conjtranspose_fbcsr() const
+{
+    std::unique_ptr<Fbcsr> mtx =
+        Fbcsr::create(exec,
+                      gko::dim<2>{static_cast<size_type>(ncols),
+                                  static_cast<size_type>(nrows)},
+                      nnz, bs, std::make_shared<matstr::classical<Fbcsr>>());
+
+    value_type *const v = mtx->get_values();
+    index_type *const c = mtx->get_col_idxs();
+    index_type *const r = mtx->get_row_ptrs();
+    index_type *const s = mtx->get_srow();
+    r[0] = 0;
+    r[1] = 2;
+    r[2] = 2;
+    r[3] = 3;
+    r[4] = 4;
+    c[0] = 0;
+    c[1] = 1;
+    c[2] = 2;
+    c[3] = 1;
+
+    for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
+
+    using namespace std::complex_literals;
+    v[0] = 1.0 - 1.15i;
+    v[1] = 3.0 + 3.15i;
+    v[2] = 2.0 - 2.15i;
+    v[3] = 0.0 + 0.15i;
+    v[8] = -12.0 - 12.15i;
+    v[9] = -2.0 + 2.15i;
+    v[10] = -1.0 - 1.15i;
+    v[11] = -11.0 + 11.15i;
+    v[13] = 0;
+    v[15] = 0;
+
+    for (index_type is = 0; is < mtx->get_num_srow_elements(); is++) s[is] = 0;
+
+    return mtx;
+}
+
+template class FbcsrSampleComplex<std::complex<float>, int>;
+template class FbcsrSampleComplex<std::complex<double>, int>;
+template class FbcsrSampleComplex<std::complex<float>, long>;
+template class FbcsrSampleComplex<std::complex<double>, long>;
+
 }  // namespace testing
 }  // namespace gko
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 51208c3dfb7..8fe2b18497f 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -49,6 +49,7 @@ namespace testing {
 /// Generates the same sample block CSR matrix in different formats
 /** This currently a 6 x 12 matrix with 3x3 blocks.
  * Assumes that the layout within each block is row-major.
+ * Generates complex data when instantiated with a complex value type.
  */
 template <typename ValueType, typename IndexType>
 class FbcsrSample {
@@ -103,10 +104,6 @@ class FbcsrSample {
     const std::shared_ptr<const gko::Executor> exec;
 
 private:
-    // template <typename AbsValueType>
-    // void
-    // correct_abs_for_complex(gko::matrix::Fbcsr<AbsValueType,IndexType> *amat)
-    //     const;
     template <typename FbcsrType>
     void correct_abs_for_complex_values(FbcsrType *const mat) const;
 
@@ -206,6 +203,41 @@ class FbcsrSampleSquare {
     const std::shared_ptr<const gko::Executor> exec;
 };
 
+/// Generates the a sample block CSR matrix with complex values
+/** This currently a 6 x 8 matrix with 2x2 blocks.
+ */
+template <typename ValueType, typename IndexType>
+class FbcsrSampleComplex {
+public:
+    using value_type = ValueType;
+    using index_type = IndexType;
+    using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    using Coo = gko::matrix::Coo<value_type, index_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+    using MatData = gko::matrix_data<value_type, index_type>;
+    using SparCsr = gko::matrix::SparsityCsr<value_type, index_type>;
+    using Diagonal = gko::matrix::Diagonal<value_type>;
+
+    static_assert(is_complex<ValueType>(), "Only for complex types!");
+
+    FbcsrSampleComplex(std::shared_ptr<const gko::ReferenceExecutor> exec);
+
+    std::unique_ptr<Fbcsr> generate_fbcsr() const;
+
+    std::unique_ptr<Fbcsr> generate_conjtranspose_fbcsr() const;
+
+
+    const size_type nrows;
+    const size_type ncols;
+    const size_type nnz;
+    const size_type nbrows;
+    const size_type nbcols;
+    const size_type nbnz;
+    const int bs;
+    const std::shared_ptr<const gko::Executor> exec;
+};
+
 }  // namespace testing
 }  // namespace gko
 
diff --git a/cuda/test/matrix/CMakeLists.txt b/cuda/test/matrix/CMakeLists.txt
index 5be841b3d00..9d40716bea0 100644
--- a/cuda/test/matrix/CMakeLists.txt
+++ b/cuda/test/matrix/CMakeLists.txt
@@ -3,6 +3,6 @@ ginkgo_create_test(csr_kernels)
 ginkgo_create_test(dense_kernels)
 ginkgo_create_test(diagonal_kernels)
 ginkgo_create_test(ell_kernels)
-ginkgo_create_test(fbcsr_kernels)
+#ginkgo_create_test(fbcsr_kernels)
 ginkgo_create_test(hybrid_kernels)
 ginkgo_create_test(sellp_kernels)
diff --git a/hip/test/matrix/CMakeLists.txt b/hip/test/matrix/CMakeLists.txt
index 4a32b5272f7..95d058e8069 100644
--- a/hip/test/matrix/CMakeLists.txt
+++ b/hip/test/matrix/CMakeLists.txt
@@ -3,6 +3,6 @@ ginkgo_create_hip_test(csr_kernels)
 ginkgo_create_hip_test(dense_kernels)
 ginkgo_create_hip_test(diagonal_kernels)
 ginkgo_create_hip_test(ell_kernels)
-ginkgo_create_hip_test(fbcsr_kernels)
+#ginkgo_create_hip_test(fbcsr_kernels)
 ginkgo_create_hip_test(hybrid_kernels)
 ginkgo_create_hip_test(sellp_kernels)
diff --git a/omp/test/matrix/CMakeLists.txt b/omp/test/matrix/CMakeLists.txt
index ff8ed2d0118..3c61fd32243 100644
--- a/omp/test/matrix/CMakeLists.txt
+++ b/omp/test/matrix/CMakeLists.txt
@@ -3,7 +3,7 @@ ginkgo_create_test(csr_kernels)
 ginkgo_create_test(dense_kernels)
 ginkgo_create_test(diagonal_kernels)
 ginkgo_create_test(ell_kernels)
-ginkgo_create_test(fbcsr_kernels)
+#ginkgo_create_test(fbcsr_kernels)
 ginkgo_create_test(hybrid_kernels)
 ginkgo_create_test(sellp_kernels)
 ginkgo_create_test(sparsity_csr_kernels)
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index 9601df09d1d..151661a875b 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -78,8 +78,7 @@ void spmv(const std::shared_ptr<const ReferenceExecutor> exec,
 {
     const int bs = a->get_block_size();
     const IndexType nvecs = static_cast<IndexType>(b->get_size()[1]);
-    const IndexType nbrows =
-        gko::blockutils::getNumBlocks(bs, a->get_size()[0]);
+    const IndexType nbrows = a->get_num_block_rows();
     auto row_ptrs = a->get_const_row_ptrs();
     auto col_idxs = a->get_const_col_idxs();
     auto vals = a->get_const_values();
@@ -121,8 +120,7 @@ void advanced_spmv(const std::shared_ptr<const ReferenceExecutor> exec,
 {
     const int bs = a->get_block_size();
     const IndexType nvecs = static_cast<IndexType>(b->get_size()[1]);
-    const IndexType nbrows =
-        gko::blockutils::getNumBlocks(bs, a->get_size()[0]);
+    const IndexType nbrows = a->get_num_block_rows();
     auto row_ptrs = a->get_const_row_ptrs();
     auto col_idxs = a->get_const_col_idxs();
     auto vals = a->get_const_values();
@@ -417,10 +415,8 @@ void convert_to_dense(const std::shared_ptr<const ReferenceExecutor> exec,
                       matrix::Dense<ValueType> *const result)
 {
     const int bs = source->get_block_size();
-    const size_type nbrows =
-        gko::blockutils::getNumBlocks(bs, source->get_size()[0]);
-    const size_type nbcols =
-        gko::blockutils::getNumBlocks(bs, source->get_size()[1]);
+    const size_type nbrows = source->get_num_block_rows();
+    const size_type nbcols = source->get_num_block_cols();
     const IndexType *const row_ptrs = source->get_const_row_ptrs();
     const IndexType *const col_idxs = source->get_const_col_idxs();
     const ValueType *const vals = source->get_const_values();
@@ -457,10 +453,8 @@ void convert_to_csr(const std::shared_ptr<const ReferenceExecutor> exec,
                     matrix::Csr<ValueType, IndexType> *const result)
 {
     const int bs = source->get_block_size();
-    const size_type nbrows =
-        gko::blockutils::getNumBlocks(bs, source->get_size()[0]);
-    const size_type nbcols =
-        gko::blockutils::getNumBlocks(bs, source->get_size()[1]);
+    const size_type nbrows = source->get_num_block_rows();
+    const size_type nbcols = source->get_num_block_cols();
     const IndexType *const browptrs = source->get_const_row_ptrs();
     const IndexType *const bcolinds = source->get_const_col_idxs();
     const ValueType *const bvals = source->get_const_values();
@@ -697,9 +691,9 @@ void transpose_and_transform(
     auto orig_vals = orig->get_const_values();
 
     auto orig_num_cols = orig->get_size()[1];
-    const size_type nbcols = gko::blockutils::getNumBlocks(bs, orig_num_cols);
+    const size_type nbcols = orig->get_num_block_cols();
     auto orig_num_rows = orig->get_size()[0];
-    const size_type nbrows = gko::blockutils::getNumBlocks(bs, orig_num_rows);
+    const size_type nbrows = orig->get_num_block_rows();
     auto orig_nbnz = orig_row_ptrs[nbrows];
 
     trans_row_ptrs[0] = 0;
@@ -728,12 +722,10 @@ template <typename ValueType, typename IndexType>
 void conj_transpose(std::shared_ptr<const ReferenceExecutor> exec,
                     const matrix::Fbcsr<ValueType, IndexType> *orig,
                     matrix::Fbcsr<ValueType, IndexType> *trans)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    transpose_and_transform(exec, trans, orig,
-//                            [](const ValueType x) { return conj(x); });
-//}
+{
+    transpose_and_transform(exec, trans, orig,
+                            [](const ValueType x) { return conj(x); });
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL);
@@ -742,20 +734,21 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void calculate_max_nnz_per_row(
     std::shared_ptr<const ReferenceExecutor> exec,
-    const matrix::Fbcsr<ValueType, IndexType> *source,
-    size_type *result) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto num_rows = source->get_size()[0];
-//    const auto row_ptrs = source->get_const_row_ptrs();
-//    IndexType max_nnz = 0;
-//
-//    for (size_type i = 0; i < num_rows; i++) {
-//        max_nnz = std::max(row_ptrs[i + 1] - row_ptrs[i], max_nnz);
-//    }
-//
-//    *result = max_nnz;
-//}
+    const matrix::Fbcsr<ValueType, IndexType> *source, size_type *const result)
+{
+    const auto num_rows = source->get_size()[0];
+    const auto row_ptrs = source->get_const_row_ptrs();
+    const int bs = source->get_block_size();
+    IndexType max_nnz = 0;
+
+    for (size_type i = 0; i < num_rows; i++) {
+        const size_type ibrow = i / bs;
+        max_nnz =
+            std::max((row_ptrs[ibrow + 1] - row_ptrs[ibrow]) * bs, max_nnz);
+    }
+
+    *result = max_nnz;
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
@@ -1018,7 +1011,7 @@ void is_sorted_by_column_index(
     const auto col_idxs = to_check->get_const_col_idxs();
     const auto size = to_check->get_size();
     const int bs = to_check->get_block_size();
-    const size_type nbrows = gko::blockutils::getNumBlocks(bs, size[0]);
+    const size_type nbrows = to_check->get_num_block_rows();
 
     for (size_type i = 0; i < nbrows; ++i) {
         for (auto idx = row_ptrs[i] + 1; idx < row_ptrs[i + 1]; ++idx) {
@@ -1046,8 +1039,7 @@ void extract_diagonal(std::shared_ptr<const ReferenceExecutor> exec,
     const auto values = orig->get_const_values();
     const int bs = orig->get_block_size();
     const size_type diag_size = diag->get_size()[0];
-    const size_type nbrows =
-        gko::blockutils::getNumBlocks(bs, orig->get_size()[0]);
+    const size_type nbrows = orig->get_num_block_rows();
     auto diag_values = diag->get_values();
     assert(diag_size == orig->get_size()[0]);
 
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 5185e03b5e1..4b75c240236 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -100,17 +100,6 @@ class Fbcsr : public ::testing::Test {
           mtxsq(fbsamplesquare.generate_fbcsr())
     {}
 
-    // void create_mtx3(Mtx *sorted, Mtx *unsorted)
-    // {
-    //     /* For both versions (sorted and unsorted), this matrix is stored:
-    //      * 0  2  1
-    //      * 3  1  8
-    //      * 2  0  3
-    //      * The unsorted matrix will have the (value, column) pair per row not
-    //      * sorted, which we still consider a valid FBCSR format.
-    //      */
-    // }
-
     void assert_equal_to_mtx(const Csr *const m)
     {
         ASSERT_EQ(m->get_size(), refcsrmtx->get_size());
@@ -127,16 +116,6 @@ class Fbcsr : public ::testing::Test {
         }
     }
 
-    // void assert_equal_to_mtx(const Dense *const m)
-    // {
-    //     ASSERT_EQ(m->get_size(), refdenmtx->get_size());
-    //     ASSERT_EQ(m->get_num_stored_elements(),
-    //     refdenmtx->get_num_stored_elements()); for(index_type i = 0; i <
-    //     m->get_size()[0]; i++)
-    //         for(index_type j = 0; j < m->get_size()[1]; j++)
-    //             ASSERT_EQ(m->at(i,j), refdenmtx->at(i,j));
-    // }
-
     void assert_equal_to_mtx(const Coo *m)
     {
         ASSERT_EQ(m->get_size(), refcoomtx->get_size());
@@ -1034,8 +1013,8 @@ TYPED_TEST(Fbcsr, NonSquareMtxIsTransposable)
 }
 
 
-TYPED_TEST(Fbcsr, SquareMatrixIsRowPermutable)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, SquareMatrixIsRowPermutable)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Fbcsr = typename TestFixture::Mtx;
@@ -1060,8 +1039,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, NonSquareMatrixIsRowPermutable)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, NonSquareMatrixIsRowPermutable)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Fbcsr = typename TestFixture::Mtx;
@@ -1084,8 +1063,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, SquareMatrixIsColPermutable)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, SquareMatrixIsColPermutable)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Fbcsr = typename TestFixture::Mtx;
@@ -1110,8 +1089,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, NonSquareMatrixIsColPermutable)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, NonSquareMatrixIsColPermutable)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Fbcsr = typename TestFixture::Mtx;
@@ -1134,8 +1113,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, SquareMatrixIsInverseRowPermutable)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, SquareMatrixIsInverseRowPermutable)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Fbcsr = typename TestFixture::Mtx;
@@ -1162,8 +1141,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, NonSquareMatrixIsInverseRowPermutable)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, NonSquareMatrixIsInverseRowPermutable)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Fbcsr = typename TestFixture::Mtx;
@@ -1188,8 +1167,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, SquareMatrixIsInverseColPermutable)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, SquareMatrixIsInverseColPermutable)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Fbcsr = typename TestFixture::Mtx;
@@ -1216,8 +1195,8 @@ GKO_NOT_IMPLEMENTED;
 //}
 
 
-TYPED_TEST(Fbcsr, NonSquareMatrixIsInverseColPermutable)
-GKO_NOT_IMPLEMENTED;
+// TYPED_TEST(Fbcsr, NonSquareMatrixIsInverseColPermutable)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    using Fbcsr = typename TestFixture::Mtx;
@@ -1261,20 +1240,18 @@ TYPED_TEST(Fbcsr, RecognizeUnsortedMatrix)
 }
 
 
-TYPED_TEST(Fbcsr, SortSortedMatrix)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto matrix = this->mtx3_sorted->clone();
-//
+// TYPED_TEST(Fbcsr, SortSortedMatrix)
+// {
+//    auto matrix = this->mtx->clone();
+
 //    matrix->sort_by_column_index();
-//
-//    GKO_ASSERT_MTX_NEAR(matrix, this->mtx3_sorted, 0.0);
-//}
 
+//    GKO_ASSERT_MTX_NEAR(matrix, this->mtx, 0.0);
+// }
 
-TYPED_TEST(Fbcsr, SortUnsortedMatrix)
-GKO_NOT_IMPLEMENTED;
+
+// TYPED_TEST(Fbcsr, SortUnsortedMatrix)
+// GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    auto matrix = this->mtx3_unsorted->clone();
@@ -1344,30 +1321,22 @@ TYPED_TEST_CASE(FbcsrComplex, gko::test::ComplexValueIndexTypes);
 
 
 TYPED_TEST(FbcsrComplex, MtxIsConjugateTransposable)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using T = typename TestFixture::value_type;
-//
-//    auto exec = gko::ReferenceExecutor::create();
-//    // clang-format off
-//    auto mtx2 = gko::initialize<Fbcsr>(
-//        {{T{1.0, 2.0}, T{3.0, 0.0}, T{2.0, 0.0}},
-//         {T{0.0, 0.0}, T{5.0, - 3.5}, T{0.0,0.0}},
-//         {T{0.0, 0.0}, T{0.0, 1.5}, T{2.0,0.0}}}, exec);
-//    // clang-format on
-//
-//    auto trans = mtx2->conj_transpose();
-//    auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
-//
-//    // clang-format off
-//    GKO_ASSERT_MTX_NEAR(trans_as_fbcsr,
-//                        l({{T{1.0, - 2.0}, T{0.0, 0.0}, T{0.0, 0.0}},
-//                           {T{3.0, 0.0}, T{5.0, 3.5}, T{0.0, - 1.5}},
-//                           {T{2.0, 0.0}, T{0.0, 0.0}, T{2.0 + 0.0}}}), 0.0);
-//    // clang-format on
-//}
+{
+    using Fbcsr = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+
+    auto exec = gko::ReferenceExecutor::create();
+    gko::testing::FbcsrSampleComplex<T, index_type> csample(exec);
+    std::unique_ptr<const Fbcsr> mtx = csample.generate_fbcsr();
+    std::unique_ptr<const Fbcsr> reftrans =
+        csample.generate_conjtranspose_fbcsr();
+
+    auto trans = mtx->conj_transpose();
+    auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
+
+    GKO_ASSERT_MTX_NEAR(trans_as_fbcsr, reftrans, 0.0);
+}
 
 
 TYPED_TEST(FbcsrComplex, InplaceAbsolute)

From 4d871ad6eab4521b0d295a5bb3d20504609ad05d Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Fri, 20 Nov 2020 17:32:36 +0100
Subject: [PATCH 0541/1008] [builds fail] changed fbcsr permutation interfaces
 to match csr

---
 core/matrix/fbcsr_kernels.hpp      |  14 +-
 dpcpp/CMakeLists.txt               |   1 +
 dpcpp/matrix/fbcsr_kernels.dp.cpp  | 351 +++++++++++++++++++++++++++++
 reference/matrix/fbcsr_kernels.cpp |  37 ++-
 4 files changed, 373 insertions(+), 30 deletions(-)
 create mode 100644 dpcpp/matrix/fbcsr_kernels.dp.cpp

diff --git a/core/matrix/fbcsr_kernels.hpp b/core/matrix/fbcsr_kernels.hpp
index 6fa130b11c9..11d76888859 100644
--- a/core/matrix/fbcsr_kernels.hpp
+++ b/core/matrix/fbcsr_kernels.hpp
@@ -137,27 +137,21 @@ namespace kernels {
 
 #define GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL(ValueType, IndexType)    \
     void row_permute(std::shared_ptr<const DefaultExecutor> exec,     \
-                     const Array<IndexType> *permutation_indices,     \
+                     const IndexType *permutation_indices,            \
                      const matrix::Fbcsr<ValueType, IndexType> *orig, \
                      matrix::Fbcsr<ValueType, IndexType> *row_permuted)
 
-#define GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)    \
-    void column_permute(std::shared_ptr<const DefaultExecutor> exec,     \
-                        const Array<IndexType> *permutation_indices,     \
-                        const matrix::Fbcsr<ValueType, IndexType> *orig, \
-                        matrix::Fbcsr<ValueType, IndexType> *column_permuted)
-
 #define GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType) \
     void inverse_row_permute(                                              \
         std::shared_ptr<const DefaultExecutor> exec,                       \
-        const Array<IndexType> *permutation_indices,                       \
+        const IndexType *permutation_indices,                              \
         const matrix::Fbcsr<ValueType, IndexType> *orig,                   \
         matrix::Fbcsr<ValueType, IndexType> *row_permuted)
 
 #define GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType) \
     void inverse_column_permute(                                              \
         std::shared_ptr<const DefaultExecutor> exec,                          \
-        const Array<IndexType> *permutation_indices,                          \
+        const IndexType > *permutation_indices,                               \
         const matrix::Fbcsr<ValueType, IndexType> *orig,                      \
         matrix::Fbcsr<ValueType, IndexType> *column_permuted)
 
@@ -220,8 +214,6 @@ namespace kernels {
     template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL(ValueType, IndexType);                \
     template <typename ValueType, typename IndexType>                          \
-    GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL(ValueType, IndexType);             \
-    template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType);        \
     template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType);     \
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 186da7f8b41..c52c77e109b 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -24,6 +24,7 @@ target_sources(ginkgo_dpcpp
     factorization/par_ilut_kernels.dp.cpp
     matrix/coo_kernels.dp.cpp
     matrix/csr_kernels.dp.cpp
+    matrix/fbcsr_kernels.dp.cpp
     matrix/dense_kernels.dp.cpp
     matrix/diagonal_kernels.dp.cpp
     matrix/ell_kernels.dp.cpp
diff --git a/dpcpp/matrix/fbcsr_kernels.dp.cpp b/dpcpp/matrix/fbcsr_kernels.dp.cpp
new file mode 100644
index 00000000000..6375aea4223
--- /dev/null
+++ b/dpcpp/matrix/fbcsr_kernels.dp.cpp
@@ -0,0 +1,351 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/fbcsr_kernels.hpp"
+
+
+#include <algorithm>
+#include <numeric>
+#include <utility>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/hybrid.hpp>
+
+
+#include "core/base/allocator.hpp"
+#include "core/base/iterator_factory.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/fbcsr_builder.hpp"
+#include "dpcpp/components/format_conversion.dp.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The Compressed sparse row matrix format namespace.
+ *
+ * @ingroup fbcsr
+ */
+namespace fbcsr {
+
+
+template <typename ValueType, typename IndexType>
+void spmv(std::shared_ptr<const DpcppExecutor> exec,
+          const matrix::Fbcsr<ValueType, IndexType> *a,
+          const matrix::Dense<ValueType> *b,
+          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
+                   const matrix::Dense<ValueType> *alpha,
+                   const matrix::Fbcsr<ValueType, IndexType> *a,
+                   const matrix::Dense<ValueType> *b,
+                   const matrix::Dense<ValueType> *beta,
+                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_insert_row(unordered_set<IndexType> &cols,
+                       const matrix::Fbcsr<ValueType, IndexType> *c,
+                       size_type row) GKO_NOT_IMPLEMENTED;
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_insert_row2(unordered_set<IndexType> &cols,
+                        const matrix::Fbcsr<ValueType, IndexType> *a,
+                        const matrix::Fbcsr<ValueType, IndexType> *b,
+                        size_type row) GKO_NOT_IMPLEMENTED;
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_accumulate_row(map<IndexType, ValueType> &cols,
+                           const matrix::Fbcsr<ValueType, IndexType> *c,
+                           ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
+
+
+template <typename ValueType, typename IndexType>
+void spgemm_accumulate_row2(map<IndexType, ValueType> &cols,
+                            const matrix::Fbcsr<ValueType, IndexType> *a,
+                            const matrix::Fbcsr<ValueType, IndexType> *b,
+                            ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
+
+
+template <typename ValueType, typename IndexType>
+void spgemm(std::shared_ptr<const DpcppExecutor> exec,
+            const matrix::Fbcsr<ValueType, IndexType> *a,
+            const matrix::Fbcsr<ValueType, IndexType> *b,
+            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEMM_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spgemm(std::shared_ptr<const DpcppExecutor> exec,
+                     const matrix::Dense<ValueType> *alpha,
+                     const matrix::Fbcsr<ValueType, IndexType> *a,
+                     const matrix::Fbcsr<ValueType, IndexType> *b,
+                     const matrix::Dense<ValueType> *beta,
+                     const matrix::Fbcsr<ValueType, IndexType> *d,
+                     matrix::Fbcsr<ValueType, IndexType> *c)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void spgeam(std::shared_ptr<const DpcppExecutor> exec,
+            const matrix::Dense<ValueType> *alpha,
+            const matrix::Fbcsr<ValueType, IndexType> *a,
+            const matrix::Dense<ValueType> *beta,
+            const matrix::Fbcsr<ValueType, IndexType> *b,
+            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEAM_KERNEL);
+
+
+template <typename IndexType>
+void convert_row_ptrs_to_idxs(std::shared_ptr<const DpcppExecutor> exec,
+                              const IndexType *ptrs, size_type num_rows,
+                              IndexType *idxs) GKO_NOT_IMPLEMENTED;
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_coo(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *source,
+                    matrix::Coo<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *source,
+                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_csr(const std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *const source,
+                    matrix::Csr<ValueType, IndexType> *const result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *source,
+                      matrix::Sellp<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_ell(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *source,
+                    matrix::Ell<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL);
+
+
+template <typename ValueType, typename IndexType, typename UnaryOperator>
+inline void convert_fbcsr_to_csc(size_type num_rows, const IndexType *row_ptrs,
+                                 const IndexType *col_idxs,
+                                 const ValueType *fbcsr_vals,
+                                 IndexType *row_idxs, IndexType *col_ptrs,
+                                 ValueType *csc_vals,
+                                 UnaryOperator op) GKO_NOT_IMPLEMENTED;
+
+
+template <typename ValueType, typename IndexType, typename UnaryOperator>
+void transpose_and_transform(std::shared_ptr<const DpcppExecutor> exec,
+                             matrix::Fbcsr<ValueType, IndexType> *trans,
+                             const matrix::Fbcsr<ValueType, IndexType> *orig,
+                             UnaryOperator op) GKO_NOT_IMPLEMENTED;
+
+
+template <typename ValueType, typename IndexType>
+void transpose(std::shared_ptr<const DpcppExecutor> exec,
+               const matrix::Fbcsr<ValueType, IndexType> *orig,
+               matrix::Fbcsr<ValueType, IndexType> *trans) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Fbcsr<ValueType, IndexType> *orig,
+                    matrix::Fbcsr<ValueType, IndexType> *trans)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
+                          const matrix::Fbcsr<ValueType, IndexType> *source,
+                          size_type *result, size_type stride_factor,
+                          size_type slice_size) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_max_nnz_per_row(
+    std::shared_ptr<const DpcppExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *source,
+    size_type *result) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_hybrid(std::shared_ptr<const DpcppExecutor> exec,
+                       const matrix::Fbcsr<ValueType, IndexType> *source,
+                       matrix::Hybrid<ValueType, IndexType> *result)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL);
+
+
+// template <typename IndexType>
+// void invert_permutation(std::shared_ptr<const DpcppExecutor> exec,
+//                         size_type size, const IndexType *permutation_indices,
+//                         IndexType *inv_permutation) GKO_NOT_IMPLEMENTED;
+
+// GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void row_permute(
+    std::shared_ptr<const DpcppExecutor> exec, const Array<IndexType> *perm,
+    const matrix::Fbcsr<ValueType, IndexType> *orig,
+    matrix::Fbcsr<ValueType, IndexType> *row_permuted) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_row_permute(
+    std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
+    const matrix::Fbcsr<ValueType, IndexType> *orig,
+    matrix::Fbcsr<ValueType, IndexType> *row_permuted) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_column_permute(
+    std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
+    const matrix::Fbcsr<ValueType, IndexType> *orig,
+    matrix::Fbcsr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void calculate_nonzeros_per_row(
+    std::shared_ptr<const DpcppExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *source,
+    Array<size_type> *result) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void sort_by_column_index(std::shared_ptr<const DpcppExecutor> exec,
+                          matrix::Fbcsr<ValueType, IndexType> *to_sort)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
+
+
+template <typename ValueType, typename IndexType>
+void is_sorted_by_column_index(
+    std::shared_ptr<const DpcppExecutor> exec,
+    const matrix::Fbcsr<ValueType, IndexType> *to_check,
+    bool *is_sorted) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
+
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Fbcsr<ValueType, IndexType> *orig,
+                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL);
+
+
+}  // namespace fbcsr
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index 151661a875b..042357d0429 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -707,8 +707,8 @@ void transpose_and_transform(
 
 template <typename ValueType, typename IndexType>
 void transpose(std::shared_ptr<const ReferenceExecutor> exec,
-               const matrix::Fbcsr<ValueType, IndexType> *orig,
-               matrix::Fbcsr<ValueType, IndexType> *trans)
+               const matrix::Fbcsr<ValueType, IndexType> *const orig,
+               matrix::Fbcsr<ValueType, IndexType> *const trans)
 {
     transpose_and_transform(exec, trans, orig,
                             [](const ValueType x) { return x; });
@@ -720,8 +720,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void conj_transpose(std::shared_ptr<const ReferenceExecutor> exec,
-                    const matrix::Fbcsr<ValueType, IndexType> *orig,
-                    matrix::Fbcsr<ValueType, IndexType> *trans)
+                    const matrix::Fbcsr<ValueType, IndexType> *const orig,
+                    matrix::Fbcsr<ValueType, IndexType> *const trans)
 {
     transpose_and_transform(exec, trans, orig,
                             [](const ValueType x) { return conj(x); });
@@ -734,7 +734,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void calculate_max_nnz_per_row(
     std::shared_ptr<const ReferenceExecutor> exec,
-    const matrix::Fbcsr<ValueType, IndexType> *source, size_type *const result)
+    const matrix::Fbcsr<ValueType, IndexType> *const source,
+    size_type *const result)
 {
     const auto num_rows = source->get_size()[0];
     const auto row_ptrs = source->get_const_row_ptrs();
@@ -814,7 +815,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void row_permute_impl(std::shared_ptr<const ReferenceExecutor> exec,
-                      const Array<IndexType> *permutation_indices,
+                      const IndexType *const permutation_indices,
                       const matrix::Fbcsr<ValueType, IndexType> *orig,
                       matrix::Fbcsr<ValueType, IndexType> *row_permuted)
     GKO_NOT_IMPLEMENTED;
@@ -857,7 +858,7 @@ void row_permute_impl(std::shared_ptr<const ReferenceExecutor> exec,
 
 template <typename ValueType, typename IndexType>
 void row_permute(std::shared_ptr<const ReferenceExecutor> exec,
-                 const Array<IndexType> *permutation_indices,
+                 const IndexType *const permutation_indices,
                  const matrix::Fbcsr<ValueType, IndexType> *orig,
                  matrix::Fbcsr<ValueType, IndexType> *row_permuted)
     GKO_NOT_IMPLEMENTED;
@@ -872,7 +873,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void inverse_row_permute(std::shared_ptr<const ReferenceExecutor> exec,
-                         const Array<IndexType> *permutation_indices,
+                         const IndexType *const permutation_indices,
                          const matrix::Fbcsr<ValueType, IndexType> *orig,
                          matrix::Fbcsr<ValueType, IndexType> *row_permuted)
     GKO_NOT_IMPLEMENTED;
@@ -893,10 +894,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void column_permute_impl(const Array<IndexType> *permutation_indices,
-                         const matrix::Fbcsr<ValueType, IndexType> *orig,
-                         matrix::Fbcsr<ValueType, IndexType> *column_permuted)
-    GKO_NOT_IMPLEMENTED;
+void column_permute_impl(const IndexType *const permutation_indices,
+                         const matrix::Fbcsr<ValueType, IndexType> *const orig,
+                         matrix::Fbcsr<ValueType, IndexType>
+                             *const column_permuted) GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    auto perm = permutation_indices->get_const_data();
@@ -924,7 +925,7 @@ void column_permute_impl(const Array<IndexType> *permutation_indices,
 
 template <typename ValueType, typename IndexType>
 void column_permute(std::shared_ptr<const ReferenceExecutor> exec,
-                    const Array<IndexType> *permutation_indices,
+                    const IndexType *permutation_indices,
                     const matrix::Fbcsr<ValueType, IndexType> *orig,
                     matrix::Fbcsr<ValueType, IndexType> *column_permuted)
     GKO_NOT_IMPLEMENTED;
@@ -939,16 +940,14 @@ void column_permute(std::shared_ptr<const ReferenceExecutor> exec,
 //    column_permute_impl(&inv_perm, orig, column_permuted);
 //}
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL);
-
 
 template <typename ValueType, typename IndexType>
 void inverse_column_permute(
     std::shared_ptr<const ReferenceExecutor> exec,
-    const Array<IndexType> *permutation_indices,
-    const matrix::Fbcsr<ValueType, IndexType> *orig,
-    matrix::Fbcsr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
+    const IndexType *const permutation_indices,
+    const matrix::Fbcsr<ValueType, IndexType> *const orig,
+    matrix::Fbcsr<ValueType, IndexType> *const column_permuted)
+    GKO_NOT_IMPLEMENTED;
 //{
 // TODO (script:fbcsr): change the code imported from matrix/csr if needed
 //    column_permute_impl(permutation_indices, orig, column_permuted);

From 32b2e87abcab0e62556c8bf6d5bed44bde5d51bd Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Mon, 23 Nov 2020 11:10:23 +0100
Subject: [PATCH 0542/1008] fixed build on dpcpp

---
 core/device_hooks/common_kernels.inc.cpp | 7 -------
 core/matrix/fbcsr.cpp                    | 1 -
 core/matrix/fbcsr_kernels.hpp            | 2 +-
 dpcpp/matrix/fbcsr_kernels.dp.cpp        | 2 +-
 4 files changed, 2 insertions(+), 10 deletions(-)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index f66c749fd73..3cb06d78763 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -804,7 +804,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_EXTRACT_DIAGONAL);
 }  // namespace csr
 
 
-// TODO (script:fbcsr): adapt this block as needed
 namespace fbcsr {
 
 
@@ -895,12 +894,6 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
 
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL);
-
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 7d2d94209aa..4f5ba607873 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -76,7 +76,6 @@ GKO_REGISTER_OPERATION(calculate_total_cols, fbcsr::calculate_total_cols);
 GKO_REGISTER_OPERATION(transpose, fbcsr::transpose);
 GKO_REGISTER_OPERATION(conj_transpose, fbcsr::conj_transpose);
 GKO_REGISTER_OPERATION(row_permute, fbcsr::row_permute);
-GKO_REGISTER_OPERATION(column_permute, fbcsr::column_permute);
 GKO_REGISTER_OPERATION(inverse_row_permute, fbcsr::inverse_row_permute);
 GKO_REGISTER_OPERATION(inverse_column_permute, fbcsr::inverse_column_permute);
 GKO_REGISTER_OPERATION(calculate_max_nnz_per_row,
diff --git a/core/matrix/fbcsr_kernels.hpp b/core/matrix/fbcsr_kernels.hpp
index 11d76888859..4968acd4713 100644
--- a/core/matrix/fbcsr_kernels.hpp
+++ b/core/matrix/fbcsr_kernels.hpp
@@ -151,7 +151,7 @@ namespace kernels {
 #define GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType) \
     void inverse_column_permute(                                              \
         std::shared_ptr<const DefaultExecutor> exec,                          \
-        const IndexType > *permutation_indices,                               \
+        const IndexType *permutation_indices,                                 \
         const matrix::Fbcsr<ValueType, IndexType> *orig,                      \
         matrix::Fbcsr<ValueType, IndexType> *column_permuted)
 
diff --git a/dpcpp/matrix/fbcsr_kernels.dp.cpp b/dpcpp/matrix/fbcsr_kernels.dp.cpp
index 6375aea4223..749ef7c3d2d 100644
--- a/dpcpp/matrix/fbcsr_kernels.dp.cpp
+++ b/dpcpp/matrix/fbcsr_kernels.dp.cpp
@@ -279,7 +279,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void row_permute(
-    std::shared_ptr<const DpcppExecutor> exec, const Array<IndexType> *perm,
+    std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
     const matrix::Fbcsr<ValueType, IndexType> *orig,
     matrix::Fbcsr<ValueType, IndexType> *row_permuted) GKO_NOT_IMPLEMENTED;
 

From aed601ff259c6d5773fa64f7dd5d70bddb40f119 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Mon, 23 Nov 2020 15:07:14 +0100
Subject: [PATCH 0543/1008] fixed hip and omp builds for the permutation
 kernels' interface change

---
 hip/matrix/fbcsr_kernels.hip.cpp | 17 +++--------------
 omp/matrix/fbcsr_kernels.cpp     | 28 +++-------------------------
 2 files changed, 6 insertions(+), 39 deletions(-)

diff --git a/hip/matrix/fbcsr_kernels.hip.cpp b/hip/matrix/fbcsr_kernels.hip.cpp
index da425238ab0..e6dc61946cb 100644
--- a/hip/matrix/fbcsr_kernels.hip.cpp
+++ b/hip/matrix/fbcsr_kernels.hip.cpp
@@ -1015,7 +1015,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void row_permute(std::shared_ptr<const HipExecutor> exec,
-                 const Array<IndexType> *permutation_indices,
+                 const IndexType *permutation_indices,
                  const matrix::Fbcsr<ValueType, IndexType> *orig,
                  matrix::Fbcsr<ValueType, IndexType> *row_permuted)
     GKO_NOT_IMPLEMENTED;
@@ -1024,20 +1024,9 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void column_permute(std::shared_ptr<const HipExecutor> exec,
-                    const Array<IndexType> *permutation_indices,
-                    const matrix::Fbcsr<ValueType, IndexType> *orig,
-                    matrix::Fbcsr<ValueType, IndexType> *column_permuted)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void inverse_row_permute(std::shared_ptr<const HipExecutor> exec,
-                         const Array<IndexType> *permutation_indices,
+                         const IndexType *permutation_indices,
                          const matrix::Fbcsr<ValueType, IndexType> *orig,
                          matrix::Fbcsr<ValueType, IndexType> *row_permuted)
     GKO_NOT_IMPLEMENTED;
@@ -1049,7 +1038,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void inverse_column_permute(
     std::shared_ptr<const HipExecutor> exec,
-    const Array<IndexType> *permutation_indices,
+    const IndexType *permutation_indices,
     const matrix::Fbcsr<ValueType, IndexType> *orig,
     matrix::Fbcsr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
 
diff --git a/omp/matrix/fbcsr_kernels.cpp b/omp/matrix/fbcsr_kernels.cpp
index d30d8cd3b16..609ab990cdb 100644
--- a/omp/matrix/fbcsr_kernels.cpp
+++ b/omp/matrix/fbcsr_kernels.cpp
@@ -684,7 +684,7 @@ void row_permute_impl(std::shared_ptr<const OmpExecutor> exec,
 
 template <typename ValueType, typename IndexType>
 void row_permute(std::shared_ptr<const OmpExecutor> exec,
-                 const Array<IndexType> *permutation_indices,
+                 const IndexType *permutation_indices,
                  const matrix::Fbcsr<ValueType, IndexType> *orig,
                  matrix::Fbcsr<ValueType, IndexType> *row_permuted)
     GKO_NOT_IMPLEMENTED;
@@ -699,7 +699,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void inverse_row_permute(std::shared_ptr<const OmpExecutor> exec,
-                         const Array<IndexType> *permutation_indices,
+                         const IndexType *permutation_indices,
                          const matrix::Fbcsr<ValueType, IndexType> *orig,
                          matrix::Fbcsr<ValueType, IndexType> *row_permuted)
     GKO_NOT_IMPLEMENTED;
@@ -751,32 +751,10 @@ void column_permute_impl(const Array<IndexType> *permutation_indices,
 //}
 
 
-template <typename ValueType, typename IndexType>
-void column_permute(std::shared_ptr<const OmpExecutor> exec,
-                    const Array<IndexType> *permutation_indices,
-                    const matrix::Fbcsr<ValueType, IndexType> *orig,
-                    matrix::Fbcsr<ValueType, IndexType> *column_permuted)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto perm = permutation_indices->get_const_data();
-//    Array<IndexType> inv_perm(*permutation_indices);
-//    auto iperm = inv_perm.get_data();
-//#pragma omp parallel for
-//    for (size_type ind = 0; ind < inv_perm.get_num_elems(); ++ind) {
-//        iperm[perm[ind]] = ind;
-//    }
-//    column_permute_impl(&inv_perm, orig, column_permuted);
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void inverse_column_permute(
     std::shared_ptr<const OmpExecutor> exec,
-    const Array<IndexType> *permutation_indices,
+    const IndexType *permutation_indices,
     const matrix::Fbcsr<ValueType, IndexType> *orig,
     matrix::Fbcsr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
 //{

From 47d64e0b70e21477e01d9837df7c3fe83285a011 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Mon, 23 Nov 2020 16:11:24 +0100
Subject: [PATCH 0544/1008] fixed cuda build for changes to permutation
 interface

Also cleaned up some commented code etc.
---
 core/CMakeLists.txt                           |   1 -
 core/components/fixed_block.hpp               |   1 -
 core/matrix/fbcsr.cpp                         | 109 +-----------------
 core/test/matrix/fbcsr_sample.cpp             |   8 +-
 core/test/matrix/fbcsr_sample.hpp             |   2 +-
 cuda/matrix/fbcsr_kernels.cu                  |  17 +--
 include/ginkgo/core/matrix/fbcsr.hpp          |  16 +--
 .../ginkgo/core/matrix/matrix_strategies.hpp  |   2 +-
 8 files changed, 20 insertions(+), 136 deletions(-)

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 4e1a7086148..7c67744d69a 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -22,7 +22,6 @@ target_sources(ginkgo
     log/stream.cpp
     matrix/coo.cpp
     matrix/csr.cpp
-    matrix/fbcsr.cpp
     matrix/dense.cpp
     matrix/diagonal.cpp
     matrix/ell.cpp
diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
index 8b6c433dfff..fbd8a259f69 100644
--- a/core/components/fixed_block.hpp
+++ b/core/components/fixed_block.hpp
@@ -132,7 +132,6 @@ class DenseBlock final {
     void zero()
     {
         for (int i = 0; i < nrows_ * ncols_; i++)
-            // vals_[i] = gko::zero<value_type>();
             vals_[i] = static_cast<value_type>(0);
     }
 
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 4f5ba607873..b8ff4558408 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -69,10 +69,7 @@ GKO_REGISTER_OPERATION(spgeam, fbcsr::spgeam);
 GKO_REGISTER_OPERATION(convert_to_coo, fbcsr::convert_to_coo);
 GKO_REGISTER_OPERATION(convert_to_csr, fbcsr::convert_to_csr);
 GKO_REGISTER_OPERATION(convert_to_dense, fbcsr::convert_to_dense);
-// GKO_REGISTER_OPERATION(convert_to_sellp, fbcsr::convert_to_sellp);
 GKO_REGISTER_OPERATION(calculate_total_cols, fbcsr::calculate_total_cols);
-// GKO_REGISTER_OPERATION(convert_to_ell, fbcsr::convert_to_ell);
-// GKO_REGISTER_OPERATION(convert_to_hybrid, fbcsr::convert_to_hybrid);
 GKO_REGISTER_OPERATION(transpose, fbcsr::transpose);
 GKO_REGISTER_OPERATION(conj_transpose, fbcsr::conj_transpose);
 GKO_REGISTER_OPERATION(row_permute, fbcsr::row_permute);
@@ -275,73 +272,6 @@ void Fbcsr<ValueType, IndexType>::move_to(Csr<ValueType, IndexType> *result)
 }
 
 
-// template <typename ValueType, typename IndexType>
-// void Fbcsr<ValueType, IndexType>::convert_to(
-//     Hybrid<ValueType, IndexType> *result) const
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto exec = this->get_executor();
-//    Array<size_type> row_nnz(exec, this->get_size()[0]);
-//
-//    size_type ell_lim = zero<size_type>();
-//    size_type coo_lim = zero<size_type>();
-//    result->get_strategy()->compute_hybrid_config(row_nnz, &ell_lim,
-//    &coo_lim); const auto max_nnz_per_row =
-//        std::max(result->get_ell_num_stored_elements_per_row(), ell_lim);
-//    const auto stride = std::max(result->get_ell_stride(),
-//    this->get_size()[0]); const auto coo_nnz =
-//        std::max(result->get_coo_num_stored_elements(), coo_lim);
-//    auto tmp = Hybrid<ValueType, IndexType>::create(
-//        exec, this->get_size(), max_nnz_per_row, stride, coo_nnz,
-//        result->get_strategy());
-//    exec->run(fbcsr::make_convert_to_hybrid(this, tmp.get()));
-//    tmp->move_to(result);
-//}
-
-
-// template <typename ValueType, typename IndexType>
-// void Fbcsr<ValueType, IndexType>::move_to(Hybrid<ValueType, IndexType>
-// *result) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    this->convert_to(result);
-//}
-
-
-// template <typename ValueType, typename IndexType>
-// void Fbcsr<ValueType, IndexType>::convert_to(
-//     Sellp<ValueType, IndexType> *result) const
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto exec = this->get_executor();
-//    const auto stride_factor = (result->get_stride_factor() == 0)
-//                                   ? default_stride_factor
-//                                   : result->get_stride_factor();
-//    const auto slice_size = (result->get_slice_size() == 0)
-//                                ? default_slice_size
-//                                : result->get_slice_size();
-//    size_type total_cols = 0;
-//    exec->run(fbcsr::make_calculate_total_cols(this, &total_cols,
-//    stride_factor,
-//                                             slice_size));
-//    auto tmp = Sellp<ValueType, IndexType>::create(
-//        exec, this->get_size(), slice_size, stride_factor, total_cols);
-//    exec->run(fbcsr::make_convert_to_sellp(this, tmp.get()));
-//    tmp->move_to(result);
-//}
-
-
-// template <typename ValueType, typename IndexType>
-// void Fbcsr<ValueType, IndexType>::move_to(Sellp<ValueType, IndexType>
-// *result) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    this->convert_to(result);
-//}
-
-
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(
     SparsityCsr<ValueType, IndexType> *result) const
@@ -369,31 +299,6 @@ void Fbcsr<ValueType, IndexType>::move_to(
 }
 
 
-// template <typename ValueType, typename IndexType>
-// void Fbcsr<ValueType, IndexType>::convert_to(
-//     Ell<ValueType, IndexType> *result) const
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto exec = this->get_executor();
-//    size_type max_nnz_per_row;
-//    exec->run(fbcsr::make_calculate_max_nnz_per_row(this, &max_nnz_per_row));
-//    auto tmp = Ell<ValueType, IndexType>::create(exec, this->get_size(),
-//                                                 max_nnz_per_row);
-//    exec->run(fbcsr::make_convert_to_ell(this, tmp.get()));
-//    tmp->move_to(result);
-//}
-
-
-// template <typename ValueType, typename IndexType>
-// void Fbcsr<ValueType, IndexType>::move_to(Ell<ValueType, IndexType> *result)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    this->convert_to(result);
-//}
-
-
 /* Within blocks, the storage order is row-major.
  * Currently, this implementation is sequential and has complexity O(n log n)
  * assuming nnz = O(n).
@@ -410,7 +315,6 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
     const index_type nnz = static_cast<index_type>(data.nonzeros.size());
 
     const int bs = this->bs_;
-    // GKO_ASSERT_EQ(nnz%(this->bs_*this->bs_), 0);
 
     using Blk_t = blockutils::DenseBlock<value_type>;
 
@@ -429,7 +333,7 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
         }
     };
 
-    auto create_block_set = [nnz, bs](const mat_data &data) {
+    auto create_block_map = [nnz, bs](const mat_data &data) {
         std::map<FbEntry, Blk_t, FbLess> blocks;
         for (index_type inz = 0; inz < nnz; inz++) {
             const index_type row = data.nonzeros[inz].row;
@@ -441,9 +345,6 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
             const index_type blockrow = row / bs;
             const index_type blockcol = col / bs;
 
-            // const typename std::map<FbEntry,Blk_t,FbLess>::iterator it
-            //     = blocks.find(FbEntry{row/bs, col/bs,
-            //     DenseBlock<value_type>()});
             Blk_t &nnzblk = blocks[{blockrow, blockcol}];
             if (nnzblk.size() == 0) {
                 nnzblk.resize(bs, bs);
@@ -452,14 +353,14 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
             } else {
                 if (nnzblk(localrow, localcol) != gko::zero<value_type>())
                     throw Error(__FILE__, __LINE__,
-                                "Error in reading fixed block CSR matrix!");
+                                "Error: re-visited the same non-zero!");
                 nnzblk(localrow, localcol) = val;
             }
         }
         return blocks;
     };
 
-    const std::map<FbEntry, Blk_t, FbLess> blocks = create_block_set(data);
+    const std::map<FbEntry, Blk_t, FbLess> blocks = create_block_map(data);
 
     auto tmp = Fbcsr::create(this->get_executor()->get_master(), data.size,
                              blocks.size() * bs * bs, bs, this->get_strategy());
@@ -477,9 +378,8 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
             throw gko::OutOfBoundsError(__FILE__, __LINE__, cur_brow,
                                         num_brows);
 
-        // set block-column index and block values
         tmp->col_idxs_.get_data()[cur_bnz] = it->first.block_column;
-        // vals
+
         for (int ibr = 0; ibr < bs; ibr++)
             for (int jbr = 0; jbr < bs; jbr++)
                 values(cur_bnz, ibr, jbr) = it->second(ibr, jbr);
@@ -497,6 +397,7 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
 
     tmp->row_ptrs_.get_data()[++cur_brow] =
         static_cast<index_type>(blocks.size());
+
     assert(cur_brow == tmp->get_size()[0] / bs);
 
     tmp->make_srow();
diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
index b48562c2cf1..246510db57d 100644
--- a/core/test/matrix/fbcsr_sample.cpp
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -75,13 +75,13 @@ generate_acopy_impl(const FbcsrType *const mat)
     const index_type *const rowptrs = mat->get_row_ptrs();
     index_type *const acolidxs = amat->get_col_idxs();
     index_type *const arowptrs = amat->get_row_ptrs();
-    // blockutils
+
     for (index_type i = 0;
          i < mat->get_num_stored_elements() /
                  (mat->get_block_size() * mat->get_block_size());
          i++)
         acolidxs[i] = colidxs[i];
-    // blockutils
+
     for (index_type i = 0; i < mat->get_size()[0] / mat->get_block_size() + 1;
          i++)
         arowptrs[i] = rowptrs[i];
@@ -132,8 +132,7 @@ FbcsrSample<ValueType, IndexType>::generate_fbcsr() const
                                 mtx->get_size()[0], mtx->get_size()[1],
                                 "block size does not divide the size!");
 
-    // blockutils
-    for (index_type ibrow = 0; ibrow < mtx->get_size()[0] / bs; ibrow++) {
+    for (index_type ibrow = 0; ibrow < mtx->get_num_block_rows(); ibrow++) {
         const index_type *const browptr = mtx->get_row_ptrs();
         for (index_type inz = browptr[ibrow]; inz < browptr[ibrow + 1]; inz++) {
             const index_type bcolind = mtx->get_col_idxs()[inz];
@@ -778,6 +777,7 @@ FbcsrSampleSquare<ValueType, IndexType>::generate_transpose_fbcsr() const
 
 #define GKO_DECLARE_FBCSR_TEST_SAMPLE_SQUARE(ValueType, IndexType) \
     class FbcsrSampleSquare<ValueType, IndexType>
+
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_TEST_SAMPLE_SQUARE);
 
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 8fe2b18497f..5e879964950 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -161,7 +161,7 @@ class FbcsrSample2 {
 
     /// Enables use of literals to instantiate value data
     template <typename U>
-    inline constexpr ValueType sct(U u) const
+    constexpr ValueType sct(U u) const
     {
         return static_cast<ValueType>(u);
     }
diff --git a/cuda/matrix/fbcsr_kernels.cu b/cuda/matrix/fbcsr_kernels.cu
index 968399ff8ed..b53df43ae55 100644
--- a/cuda/matrix/fbcsr_kernels.cu
+++ b/cuda/matrix/fbcsr_kernels.cu
@@ -1203,7 +1203,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void row_permute(std::shared_ptr<const CudaExecutor> exec,
-                 const Array<IndexType> *permutation_indices,
+                 const IndexType *permutation_indices,
                  const matrix::Fbcsr<ValueType, IndexType> *orig,
                  matrix::Fbcsr<ValueType, IndexType> *row_permuted)
     GKO_NOT_IMPLEMENTED;
@@ -1212,20 +1212,9 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void column_permute(std::shared_ptr<const CudaExecutor> exec,
-                    const Array<IndexType> *permutation_indices,
-                    const matrix::Fbcsr<ValueType, IndexType> *orig,
-                    matrix::Fbcsr<ValueType, IndexType> *column_permuted)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_COLUMN_PERMUTE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void inverse_row_permute(std::shared_ptr<const CudaExecutor> exec,
-                         const Array<IndexType> *permutation_indices,
+                         const IndexType *permutation_indices,
                          const matrix::Fbcsr<ValueType, IndexType> *orig,
                          matrix::Fbcsr<ValueType, IndexType> *row_permuted)
     GKO_NOT_IMPLEMENTED;
@@ -1237,7 +1226,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void inverse_column_permute(
     std::shared_ptr<const CudaExecutor> exec,
-    const Array<IndexType> *permutation_indices,
+    const IndexType *permutation_indices,
     const matrix::Fbcsr<ValueType, IndexType> *orig,
     matrix::Fbcsr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
 
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index 141c1a3c3eb..7d5c00cbd02 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -75,14 +75,15 @@ class FbcsrBuilder;
 
 
 /**
- * FBCSR is a matrix format which stores only the nonzero coefficients by
- * compressing each row of the matrix (compressed sparse row format).
+ * FBCSR is a matrix format meant for matrices having a natural block structure
+ * made up of small, dense, disjoint blocks. It is similar to CSR \sa Csr.
  * However, unlike Csr, each non-zero location stores a small dense block of
  * entries having a constant size. This reduces the number of integers that need
  * to be stored in order to refer to a given non-zero entry, and enables
  * efficient implementation of certain block methods.
  *
- * The entries within each dense block are stored row-major.
+ * The block size is expected to be known in advance and passed to the
+ * constructor.
  *
  * @note The total number of rows and the number of columns are expected to be
  *   divisible by the block size.
@@ -112,7 +113,8 @@ class FbcsrBuilder;
  * A->apply(alpha, I, beta, B) // B = alpha*A + beta*B
  * ```
  * Both the SpGEMM and SpGEAM operation require the input matrices to be sorted
- * by column index, otherwise the algorithms will produce incorrect results.
+ * by block-column index, otherwise the algorithms will produce incorrect
+ * results.
  *
  * @tparam ValueType  precision of matrix elements
  * @tparam IndexType  precision of matrix indexes
@@ -128,9 +130,6 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
               public ConvertibleTo<Dense<ValueType>>,
               public ConvertibleTo<Csr<ValueType, IndexType>>,
               public ConvertibleTo<Coo<ValueType, IndexType>>,
-              // public ConvertibleTo<Ell<ValueType, IndexType>>,
-              // public ConvertibleTo<Hybrid<ValueType, IndexType>>,
-              // public ConvertibleTo<Sellp<ValueType, IndexType>>,
               public ConvertibleTo<SparsityCsr<ValueType, IndexType>>,
               public DiagonalExtractable<ValueType>,
               public ReadableFromMatrixData<ValueType, IndexType>,
@@ -143,9 +142,6 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     friend class EnablePolymorphicObject<Fbcsr, LinOp>;
     friend class Coo<ValueType, IndexType>;
     friend class Dense<ValueType>;
-    // friend class Ell<ValueType, IndexType>;
-    // friend class Hybrid<ValueType, IndexType>;
-    // friend class Sellp<ValueType, IndexType>;
     friend class SparsityCsr<ValueType, IndexType>;
     friend class FbcsrBuilder<ValueType, IndexType>;
     friend class Fbcsr<to_complex<ValueType>, IndexType>;
diff --git a/include/ginkgo/core/matrix/matrix_strategies.hpp b/include/ginkgo/core/matrix/matrix_strategies.hpp
index c774209f7d7..47133b3610f 100644
--- a/include/ginkgo/core/matrix/matrix_strategies.hpp
+++ b/include/ginkgo/core/matrix/matrix_strategies.hpp
@@ -49,7 +49,7 @@ template <typename MtxType>
 class automatic;
 
 /**
- * strategy_type is to decide how to set the fbcsr algorithm.
+ * strategy_type is to decide how map the work-items to execution units
  *
  * The practical strategy method should inherit strategy_type and implement
  * its `process`, `calc_size` function and the corresponding device kernel.

From 1b48c898593a9f301f9fcbe066a84c6efcf13c9c Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@mail.mcgill.ca>
Date: Thu, 26 Nov 2020 11:03:48 +0100
Subject: [PATCH 0545/1008] removed commented and dead code, and unnecessary
 includes

---
 core/device_hooks/common_kernels.inc.cpp |   64 --
 core/matrix/fbcsr.cpp                    |  124 +--
 core/matrix/fbcsr_kernels.hpp            |   91 --
 cuda/matrix/fbcsr_kernels.cu             | 1182 ----------------------
 cuda/test/matrix/CMakeLists.txt          |    1 -
 cuda/test/matrix/fbcsr_kernels.cpp       |  883 ----------------
 dpcpp/matrix/fbcsr_kernels.dp.cpp        |  148 ---
 hip/matrix/fbcsr_kernels.hip.cpp         | 1003 ------------------
 hip/test/matrix/CMakeLists.txt           |    1 -
 hip/test/matrix/fbcsr_kernels.hip.cpp    |  866 ----------------
 include/ginkgo/core/matrix/fbcsr.hpp     |   31 -
 omp/components/fbcsr_spgeam.hpp          |   31 -
 omp/matrix/fbcsr_kernels.cpp             |  575 -----------
 omp/test/matrix/CMakeLists.txt           |    1 -
 omp/test/matrix/fbcsr_kernels.cpp        |  662 ------------
 reference/components/fbcsr_spgeam.hpp    |   31 -
 reference/matrix/fbcsr_kernels.cpp       |  556 ----------
 reference/test/matrix/fbcsr_kernels.cpp  |  661 ------------
 18 files changed, 2 insertions(+), 6909 deletions(-)
 delete mode 100644 cuda/test/matrix/fbcsr_kernels.cpp
 delete mode 100644 hip/test/matrix/fbcsr_kernels.hip.cpp
 delete mode 100644 omp/components/fbcsr_spgeam.hpp
 delete mode 100644 omp/test/matrix/fbcsr_kernels.cpp
 delete mode 100644 reference/components/fbcsr_spgeam.hpp

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 3cb06d78763..9cca9e2d26f 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -818,22 +818,6 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
 
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_FBCSR_SPGEMM_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEMM_KERNEL);
-
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL);
-
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_FBCSR_SPGEAM_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEAM_KERNEL);
-
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
@@ -846,36 +830,6 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL);
 
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL);
-
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL);
-
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL);
-
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL);
-
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL);
-
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
@@ -888,24 +842,6 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL);
 
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
-
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL);
-
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL);
-
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index b8ff4558408..54e938d768d 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -63,18 +63,10 @@ namespace fbcsr {
 
 GKO_REGISTER_OPERATION(spmv, fbcsr::spmv);
 GKO_REGISTER_OPERATION(advanced_spmv, fbcsr::advanced_spmv);
-GKO_REGISTER_OPERATION(spgemm, fbcsr::spgemm);
-GKO_REGISTER_OPERATION(advanced_spgemm, fbcsr::advanced_spgemm);
-GKO_REGISTER_OPERATION(spgeam, fbcsr::spgeam);
-GKO_REGISTER_OPERATION(convert_to_coo, fbcsr::convert_to_coo);
 GKO_REGISTER_OPERATION(convert_to_csr, fbcsr::convert_to_csr);
 GKO_REGISTER_OPERATION(convert_to_dense, fbcsr::convert_to_dense);
-GKO_REGISTER_OPERATION(calculate_total_cols, fbcsr::calculate_total_cols);
 GKO_REGISTER_OPERATION(transpose, fbcsr::transpose);
 GKO_REGISTER_OPERATION(conj_transpose, fbcsr::conj_transpose);
-GKO_REGISTER_OPERATION(row_permute, fbcsr::row_permute);
-GKO_REGISTER_OPERATION(inverse_row_permute, fbcsr::inverse_row_permute);
-GKO_REGISTER_OPERATION(inverse_column_permute, fbcsr::inverse_column_permute);
 GKO_REGISTER_OPERATION(calculate_max_nnz_per_row,
                        fbcsr::calculate_max_nnz_per_row);
 GKO_REGISTER_OPERATION(calculate_nonzeros_per_row,
@@ -120,8 +112,6 @@ void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const b,
         // if b is a FBCSR matrix, we compute a SpGeMM
         throw /*::gko::*/ NotImplemented(__FILE__, __LINE__,
                                          "SpGeMM for Fbcsr");
-        auto x_fbcsr = as<TFbcsr>(x);
-        this->get_executor()->run(fbcsr::make_spgemm(this, b_fbcsr, x_fbcsr));
     } else {
         // otherwise we assume that b is dense and compute a SpMV/SpMM
         this->get_executor()->run(
@@ -138,17 +128,10 @@ void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
     using TFbcsr = Fbcsr<ValueType, IndexType>;
     if (auto b_fbcsr = dynamic_cast<const TFbcsr *>(b)) {
         // if b is a FBCSR matrix, we compute a SpGeMM
-        auto x_fbcsr = as<TFbcsr>(x);
-        auto x_copy = x_fbcsr->clone();
-        this->get_executor()->run(fbcsr::make_advanced_spgemm(
-            as<Dense>(alpha), this, b_fbcsr, as<Dense>(beta), x_copy.get(),
-            x_fbcsr));
+        throw NotImplemented(__FILE__, __LINE__, "Adv SpGeMM for Fbcsr");
     } else if (dynamic_cast<const Identity<ValueType> *>(b)) {
         // if b is an identity matrix, we compute an SpGEAM
-        auto x_fbcsr = as<TFbcsr>(x);
-        auto x_copy = x_fbcsr->clone();
-        this->get_executor()->run(fbcsr::make_spgeam(
-            as<Dense>(alpha), this, as<Dense>(beta), lend(x_copy), x_fbcsr));
+        throw NotImplemented(__FILE__, __LINE__, "Adv SpGeMM for Fbcsr");
     } else {
         // otherwise we assume that b is dense and compute a SpMV/SpMM
         this->get_executor()->run(
@@ -211,30 +194,6 @@ void Fbcsr<ValueType, IndexType>::move_to(
 }
 
 
-template <typename ValueType, typename IndexType>
-void Fbcsr<ValueType, IndexType>::convert_to(
-    Coo<ValueType, IndexType> *result) const GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto exec = this->get_executor();
-//    auto tmp = Coo<ValueType, IndexType>::create(
-//        exec, this->get_size(), this->get_num_stored_elements());
-//    tmp->values_ = this->values_;
-//    tmp->col_idxs_ = this->col_idxs_;
-//    exec->run(fbcsr::make_convert_to_coo(this, tmp.get()));
-//    tmp->move_to(result);
-//}
-
-
-template <typename ValueType, typename IndexType>
-void Fbcsr<ValueType, IndexType>::move_to(Coo<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    this->convert_to(result);
-//}
-
-
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(Dense<ValueType> *result) const
 {
@@ -469,85 +428,6 @@ std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::conj_transpose() const
 }
 
 
-template <typename ValueType, typename IndexType>
-std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::row_permute(
-    const Array<IndexType> *permutation_indices) const GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
-//    auto exec = this->get_executor();
-//    auto permute_cpy =
-//        Fbcsr::create(exec, this->get_size(), this->get_num_stored_elements(),
-//                    this->get_strategy());
-//
-//    exec->run(
-//        fbcsr::make_row_permute(permutation_indices, this,
-//        permute_cpy.get()));
-//    permute_cpy->make_srow();
-//    return std::move(permute_cpy);
-//}
-
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::column_permute(
-    const Array<IndexType> *permutation_indices) const GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
-//    auto exec = this->get_executor();
-//    auto permute_cpy =
-//        Fbcsr::create(exec, this->get_size(), this->get_num_stored_elements(),
-//                    this->get_strategy());
-//
-//    exec->run(
-//        fbcsr::make_column_permute(permutation_indices, this,
-//        permute_cpy.get()));
-//    permute_cpy->make_srow();
-//    return std::move(permute_cpy);
-//}
-
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::inverse_row_permute(
-    const Array<IndexType> *inverse_permutation_indices) const
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    GKO_ASSERT_EQ(inverse_permutation_indices->get_num_elems(),
-//                  this->get_size()[0]);
-//    auto exec = this->get_executor();
-//    auto inverse_permute_cpy =
-//        Fbcsr::create(exec, this->get_size(), this->get_num_stored_elements(),
-//                    this->get_strategy());
-//
-//    exec->run(fbcsr::make_inverse_row_permute(inverse_permutation_indices,
-//    this,
-//                                            inverse_permute_cpy.get()));
-//    inverse_permute_cpy->make_srow();
-//    return std::move(inverse_permute_cpy);
-//}
-
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::inverse_column_permute(
-    const Array<IndexType> *inverse_permutation_indices) const
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    GKO_ASSERT_EQ(inverse_permutation_indices->get_num_elems(),
-//                  this->get_size()[1]);
-//    auto exec = this->get_executor();
-//    auto inverse_permute_cpy =
-//        Fbcsr::create(exec, this->get_size(), this->get_num_stored_elements(),
-//                    this->get_strategy());
-//
-//    exec->run(fbcsr::make_inverse_column_permute(
-//        inverse_permutation_indices, this, inverse_permute_cpy.get()));
-//    inverse_permute_cpy->make_srow();
-//    return std::move(inverse_permute_cpy);
-//}
-
-
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::sort_by_column_index() GKO_NOT_IMPLEMENTED;
 //{
diff --git a/core/matrix/fbcsr_kernels.hpp b/core/matrix/fbcsr_kernels.hpp
index 4968acd4713..c7a2747039e 100644
--- a/core/matrix/fbcsr_kernels.hpp
+++ b/core/matrix/fbcsr_kernels.hpp
@@ -66,29 +66,6 @@ namespace kernels {
                        const matrix::Dense<ValueType> *beta,         \
                        matrix::Dense<ValueType> *c)
 
-#define GKO_DECLARE_FBCSR_SPGEMM_KERNEL(ValueType, IndexType) \
-    void spgemm(std::shared_ptr<const DefaultExecutor> exec,  \
-                const matrix::Fbcsr<ValueType, IndexType> *a, \
-                const matrix::Fbcsr<ValueType, IndexType> *b, \
-                matrix::Fbcsr<ValueType, IndexType> *c)
-
-#define GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL(ValueType, IndexType) \
-    void advanced_spgemm(std::shared_ptr<const DefaultExecutor> exec,  \
-                         const matrix::Dense<ValueType> *alpha,        \
-                         const matrix::Fbcsr<ValueType, IndexType> *a, \
-                         const matrix::Fbcsr<ValueType, IndexType> *b, \
-                         const matrix::Dense<ValueType> *beta,         \
-                         const matrix::Fbcsr<ValueType, IndexType> *d, \
-                         matrix::Fbcsr<ValueType, IndexType> *c)
-
-#define GKO_DECLARE_FBCSR_SPGEAM_KERNEL(ValueType, IndexType) \
-    void spgeam(std::shared_ptr<const DefaultExecutor> exec,  \
-                const matrix::Dense<ValueType> *alpha,        \
-                const matrix::Fbcsr<ValueType, IndexType> *a, \
-                const matrix::Dense<ValueType> *beta,         \
-                const matrix::Fbcsr<ValueType, IndexType> *b, \
-                matrix::Fbcsr<ValueType, IndexType> *c)
-
 #define GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType)      \
     void convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,       \
                           const matrix::Fbcsr<ValueType, IndexType> *source, \
@@ -99,32 +76,6 @@ namespace kernels {
                         const matrix::Fbcsr<ValueType, IndexType> *source, \
                         matrix::Csr<ValueType, IndexType> *result)
 
-#define GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL(ValueType, IndexType)      \
-    void convert_to_coo(std::shared_ptr<const DefaultExecutor> exec,       \
-                        const matrix::Fbcsr<ValueType, IndexType> *source, \
-                        matrix::Coo<ValueType, IndexType> *result)
-
-#define GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL(ValueType, IndexType)      \
-    void convert_to_ell(std::shared_ptr<const DefaultExecutor> exec,       \
-                        const matrix::Fbcsr<ValueType, IndexType> *source, \
-                        matrix::Ell<ValueType, IndexType> *result)
-
-#define GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL(ValueType, IndexType)      \
-    void convert_to_hybrid(std::shared_ptr<const DefaultExecutor> exec,       \
-                           const matrix::Fbcsr<ValueType, IndexType> *source, \
-                           matrix::Hybrid<ValueType, IndexType> *result)
-
-#define GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL(ValueType, IndexType)      \
-    void convert_to_sellp(std::shared_ptr<const DefaultExecutor> exec,       \
-                          const matrix::Fbcsr<ValueType, IndexType> *source, \
-                          matrix::Sellp<ValueType, IndexType> *result)
-
-#define GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL(ValueType, IndexType)   \
-    void calculate_total_cols(                                                \
-        std::shared_ptr<const DefaultExecutor> exec,                          \
-        const matrix::Fbcsr<ValueType, IndexType> *source, size_type *result, \
-        size_type stride_factor, size_type slice_size)
-
 #define GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL(ValueType, IndexType)    \
     void transpose(std::shared_ptr<const DefaultExecutor> exec,     \
                    const matrix::Fbcsr<ValueType, IndexType> *orig, \
@@ -135,26 +86,6 @@ namespace kernels {
                         const matrix::Fbcsr<ValueType, IndexType> *orig, \
                         matrix::Fbcsr<ValueType, IndexType> *trans)
 
-#define GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL(ValueType, IndexType)    \
-    void row_permute(std::shared_ptr<const DefaultExecutor> exec,     \
-                     const IndexType *permutation_indices,            \
-                     const matrix::Fbcsr<ValueType, IndexType> *orig, \
-                     matrix::Fbcsr<ValueType, IndexType> *row_permuted)
-
-#define GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType) \
-    void inverse_row_permute(                                              \
-        std::shared_ptr<const DefaultExecutor> exec,                       \
-        const IndexType *permutation_indices,                              \
-        const matrix::Fbcsr<ValueType, IndexType> *orig,                   \
-        matrix::Fbcsr<ValueType, IndexType> *row_permuted)
-
-#define GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType) \
-    void inverse_column_permute(                                              \
-        std::shared_ptr<const DefaultExecutor> exec,                          \
-        const IndexType *permutation_indices,                                 \
-        const matrix::Fbcsr<ValueType, IndexType> *orig,                      \
-        matrix::Fbcsr<ValueType, IndexType> *column_permuted)
-
 #define GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL(ValueType, \
                                                            IndexType) \
     void calculate_max_nnz_per_row(                                   \
@@ -188,36 +119,14 @@ namespace kernels {
     template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL(ValueType, IndexType);              \
     template <typename ValueType, typename IndexType>                          \
-    GKO_DECLARE_FBCSR_SPGEMM_KERNEL(ValueType, IndexType);                     \
-    template <typename ValueType, typename IndexType>                          \
-    GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL(ValueType, IndexType);            \
-    template <typename ValueType, typename IndexType>                          \
-    GKO_DECLARE_FBCSR_SPGEAM_KERNEL(ValueType, IndexType);                     \
-    template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType);           \
     template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL(ValueType, IndexType);             \
     template <typename ValueType, typename IndexType>                          \
-    GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL(ValueType, IndexType);             \
-    template <typename ValueType, typename IndexType>                          \
-    GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL(ValueType, IndexType);           \
-    template <typename ValueType, typename IndexType>                          \
-    GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL(ValueType, IndexType);          \
-    template <typename ValueType, typename IndexType>                          \
-    GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL(ValueType, IndexType);             \
-    template <typename ValueType, typename IndexType>                          \
-    GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL(ValueType, IndexType);       \
-    template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL(ValueType, IndexType);                  \
     template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL(ValueType, IndexType);             \
     template <typename ValueType, typename IndexType>                          \
-    GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL(ValueType, IndexType);                \
-    template <typename ValueType, typename IndexType>                          \
-    GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL(ValueType, IndexType);        \
-    template <typename ValueType, typename IndexType>                          \
-    GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL(ValueType, IndexType);     \
-    template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL(ValueType, IndexType);  \
     template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL(ValueType, IndexType); \
diff --git a/cuda/matrix/fbcsr_kernels.cu b/cuda/matrix/fbcsr_kernels.cu
index b53df43ae55..f2ac5a85a01 100644
--- a/cuda/matrix/fbcsr_kernels.cu
+++ b/cuda/matrix/fbcsr_kernels.cu
@@ -39,12 +39,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/ell.hpp>
-#include <ginkgo/core/matrix/hybrid.hpp>
-#include <ginkgo/core/matrix/sellp.hpp>
 
 
 #include "core/components/fill_array.hpp"
@@ -94,285 +90,12 @@ using compiled_kernels = syn::value_list<int, 3, 4, 6, 7, 8, 12, 14>;
 using classical_kernels =
     syn::value_list<int, config::warp_size, 32, 16, 8, 4, 2, 1>;
 
-using spgeam_kernels =
-    syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
-
-
-namespace host_kernel {
-
-
-template <int items_per_thread, typename ValueType, typename IndexType>
-void merge_path_spmv(
-    syn::value_list<int, items_per_thread>,
-    std::shared_ptr<const CudaExecutor> exec,
-    const matrix::Fbcsr<ValueType, IndexType> *a,
-    const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c,
-    const matrix::Dense<ValueType> *alpha = nullptr,
-    const matrix::Dense<ValueType> *beta = nullptr) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const IndexType total = a->get_size()[0] + a->get_num_stored_elements();
-//    const IndexType grid_num =
-//        ceildiv(total, spmv_block_size * items_per_thread);
-//    const dim3 grid(grid_num);
-//    const dim3 block(spmv_block_size);
-//    Array<IndexType> row_out(exec, grid_num);
-//    Array<ValueType> val_out(exec, grid_num);
-//
-//    for (IndexType column_id = 0; column_id < b->get_size()[1]; column_id++) {
-//        if (alpha == nullptr && beta == nullptr) {
-//            const auto b_vals = b->get_const_values() + column_id;
-//            auto c_vals = c->get_values() + column_id;
-//            kernel::abstract_merge_path_spmv<items_per_thread>
-//                <<<grid, block, 0, 0>>>(
-//                    static_cast<IndexType>(a->get_size()[0]),
-//                    as_cuda_type(a->get_const_values()),
-//                    a->get_const_col_idxs(),
-//                    as_cuda_type(a->get_const_row_ptrs()),
-//                    as_cuda_type(a->get_const_srow()), as_cuda_type(b_vals),
-//                    b->get_stride(), as_cuda_type(c_vals), c->get_stride(),
-//                    as_cuda_type(row_out.get_data()),
-//                    as_cuda_type(val_out.get_data()));
-//            kernel::abstract_reduce<<<1, spmv_block_size>>>(
-//                grid_num, as_cuda_type(val_out.get_data()),
-//                as_cuda_type(row_out.get_data()), as_cuda_type(c_vals),
-//                c->get_stride());
-//
-//        } else if (alpha != nullptr && beta != nullptr) {
-//            const auto b_vals = b->get_const_values() + column_id;
-//            auto c_vals = c->get_values() + column_id;
-//            kernel::abstract_merge_path_spmv<items_per_thread>
-//                <<<grid, block, 0, 0>>>(
-//                    static_cast<IndexType>(a->get_size()[0]),
-//                    as_cuda_type(alpha->get_const_values()),
-//                    as_cuda_type(a->get_const_values()),
-//                    a->get_const_col_idxs(),
-//                    as_cuda_type(a->get_const_row_ptrs()),
-//                    as_cuda_type(a->get_const_srow()), as_cuda_type(b_vals),
-//                    b->get_stride(), as_cuda_type(beta->get_const_values()),
-//                    as_cuda_type(c_vals), c->get_stride(),
-//                    as_cuda_type(row_out.get_data()),
-//                    as_cuda_type(val_out.get_data()));
-//            kernel::abstract_reduce<<<1, spmv_block_size>>>(
-//                grid_num, as_cuda_type(val_out.get_data()),
-//                as_cuda_type(row_out.get_data()),
-//                as_cuda_type(alpha->get_const_values()), as_cuda_type(c_vals),
-//                c->get_stride());
-//        } else {
-//            GKO_KERNEL_NOT_FOUND;
-//        }
-//    }
-//}
-
-GKO_ENABLE_IMPLEMENTATION_SELECTION(select_merge_path_spmv, merge_path_spmv);
-
-
-template <typename ValueType, typename IndexType>
-int compute_items_per_thread(std::shared_ptr<const CudaExecutor> exec)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const int version =
-//        (exec->get_major_version() << 4) + exec->get_minor_version();
-//    // The num_item is decided to make the occupancy 100%
-//    // TODO: Extend this list when new GPU is released
-//    //       Tune this parameter
-//    // 128 threads/block the number of items per threads
-//    // 3.0 3.5: 6
-//    // 3.7: 14
-//    // 5.0, 5.3, 6.0, 6.2: 8
-//    // 5.2, 6.1, 7.0: 12
-//    int num_item = 6;
-//    switch (version) {
-//    case 0x50:
-//    case 0x53:
-//    case 0x60:
-//    case 0x62:
-//        num_item = 8;
-//        break;
-//    case 0x52:
-//    case 0x61:
-//    case 0x70:
-//        num_item = 12;
-//        break;
-//    case 0x37:
-//        num_item = 14;
-//    }
-//    // Ensure that the following is satisfied:
-//    // sizeof(IndexType) + sizeof(ValueType)
-//    // <= items_per_thread * sizeof(IndexType)
-//    constexpr int minimal_num =
-//        ceildiv(sizeof(IndexType) + sizeof(ValueType), sizeof(IndexType));
-//    int items_per_thread = num_item * 4 / sizeof(IndexType);
-//    return std::max(minimal_num, items_per_thread);
-//}
-
-
-template <int subwarp_size, typename ValueType, typename IndexType>
-void classical_spmv(
-    syn::value_list<int, subwarp_size>,
-    std::shared_ptr<const CudaExecutor> exec,
-    const matrix::Fbcsr<ValueType, IndexType> *a,
-    const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c,
-    const matrix::Dense<ValueType> *alpha = nullptr,
-    const matrix::Dense<ValueType> *beta = nullptr) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto nwarps = exec->get_num_warps_per_sm() *
-//                        exec->get_num_multiprocessor() * classical_overweight;
-//    const auto gridx =
-//        std::min(ceildiv(a->get_size()[0], spmv_block_size / subwarp_size),
-//                 int64(nwarps / warps_in_block));
-//    const dim3 grid(gridx, b->get_size()[1]);
-//    const dim3 block(spmv_block_size);
-//
-//    if (alpha == nullptr && beta == nullptr) {
-//        kernel::abstract_classical_spmv<subwarp_size><<<grid, block, 0, 0>>>(
-//            a->get_size()[0], as_cuda_type(a->get_const_values()),
-//            a->get_const_col_idxs(), as_cuda_type(a->get_const_row_ptrs()),
-//            as_cuda_type(b->get_const_values()), b->get_stride(),
-//            as_cuda_type(c->get_values()), c->get_stride());
-//
-//    } else if (alpha != nullptr && beta != nullptr) {
-//        kernel::abstract_classical_spmv<subwarp_size><<<grid, block, 0, 0>>>(
-//            a->get_size()[0], as_cuda_type(alpha->get_const_values()),
-//            as_cuda_type(a->get_const_values()), a->get_const_col_idxs(),
-//            as_cuda_type(a->get_const_row_ptrs()),
-//            as_cuda_type(b->get_const_values()), b->get_stride(),
-//            as_cuda_type(beta->get_const_values()),
-//            as_cuda_type(c->get_values()), c->get_stride());
-//    } else {
-//        GKO_KERNEL_NOT_FOUND;
-//    }
-//}
-
-GKO_ENABLE_IMPLEMENTATION_SELECTION(select_classical_spmv, classical_spmv);
-
-
-}  // namespace host_kernel
-
 
 template <typename ValueType, typename IndexType>
 void spmv(std::shared_ptr<const CudaExecutor> exec,
           const matrix::Fbcsr<ValueType, IndexType> *a,
           const matrix::Dense<ValueType> *b,
           matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    if (a->get_strategy()->get_name() == "load_balance") {
-//        components::fill_array(exec, c->get_values(),
-//                               c->get_num_stored_elements(),
-//                               zero<ValueType>());
-//        const IndexType nwarps = a->get_num_srow_elements();
-//        if (nwarps > 0) {
-//            const dim3 fbcsr_block(config::warp_size, warps_in_block, 1);
-//            const dim3 fbcsr_grid(ceildiv(nwarps, warps_in_block),
-//                                b->get_size()[1]);
-//            kernel::abstract_spmv<<<fbcsr_grid, fbcsr_block>>>(
-//                nwarps, static_cast<IndexType>(a->get_size()[0]),
-//                as_cuda_type(a->get_const_values()), a->get_const_col_idxs(),
-//                as_cuda_type(a->get_const_row_ptrs()),
-//                as_cuda_type(a->get_const_srow()),
-//                as_cuda_type(b->get_const_values()),
-//                as_cuda_type(b->get_stride()), as_cuda_type(c->get_values()),
-//                as_cuda_type(c->get_stride()));
-//        } else {
-//            GKO_NOT_SUPPORTED(nwarps);
-//        }
-//    } else if (a->get_strategy()->get_name() == "merge_path") {
-//        int items_per_thread =
-//            host_kernel::compute_items_per_thread<ValueType, IndexType>(exec);
-//        host_kernel::select_merge_path_spmv(
-//            compiled_kernels(),
-//            [&items_per_thread](int compiled_info) {
-//                return items_per_thread == compiled_info;
-//            },
-//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c);
-//    } else if (a->get_strategy()->get_name() == "classical") {
-//        IndexType max_length_per_row = 0;
-//        using Tfbcsr = matrix::Fbcsr<ValueType, IndexType>;
-//        if (auto strategy =
-//                std::dynamic_pointer_cast<const typename Tfbcsr::classical>(
-//                    a->get_strategy())) {
-//            max_length_per_row = strategy->get_max_length_per_row();
-//        } else if (auto strategy = std::dynamic_pointer_cast<
-//                       const typename Tfbcsr::automatical>(a->get_strategy()))
-//                       {
-//            max_length_per_row = strategy->get_max_length_per_row();
-//        } else {
-//            GKO_NOT_SUPPORTED(a->get_strategy());
-//        }
-//        host_kernel::select_classical_spmv(
-//            classical_kernels(),
-//            [&max_length_per_row](int compiled_info) {
-//                return max_length_per_row >= compiled_info;
-//            },
-//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c);
-//    } else if (a->get_strategy()->get_name() == "sparselib" ||
-//               a->get_strategy()->get_name() == "cusparse") {
-//        if (cusparse::is_supported<ValueType, IndexType>::value) {
-//            // TODO: add implementation for int64 and multiple RHS
-//            auto handle = exec->get_cusparse_handle();
-//            {
-//                cusparse::pointer_mode_guard pm_guard(handle);
-//                const auto alpha = one<ValueType>();
-//                const auto beta = zero<ValueType>();
-//                // TODO: add implementation for int64 and multiple RHS
-//                if (b->get_stride() != 1 || c->get_stride() != 1)
-//                    GKO_NOT_IMPLEMENTED;
-//
-//#if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
-//                auto descr = cusparse::create_mat_descr();
-//                auto row_ptrs = a->get_const_row_ptrs();
-//                auto col_idxs = a->get_const_col_idxs();
-//                cusparse::spmv(handle, CUSPARSE_OPERATION_NON_TRANSPOSE,
-//                               a->get_size()[0], a->get_size()[1],
-//                               a->get_num_stored_elements(), &alpha, descr,
-//                               a->get_const_values(), row_ptrs, col_idxs,
-//                               b->get_const_values(), &beta, c->get_values());
-//
-//                cusparse::destroy(descr);
-//#else  // CUDA_VERSION >= 11000
-//                cusparseOperation_t trans = CUSPARSE_OPERATION_NON_TRANSPOSE;
-//                cusparseSpMVAlg_t alg = CUSPARSE_FBCSRMV_ALG1;
-//                auto row_ptrs =
-//                    const_cast<IndexType *>(a->get_const_row_ptrs());
-//                auto col_idxs =
-//                    const_cast<IndexType *>(a->get_const_col_idxs());
-//                auto values = const_cast<ValueType *>(a->get_const_values());
-//                auto mat = cusparse::create_fbcsr(
-//                    a->get_size()[0], a->get_size()[1],
-//                    a->get_num_stored_elements(), row_ptrs, col_idxs, values);
-//                auto b_val = const_cast<ValueType *>(b->get_const_values());
-//                auto c_val = c->get_values();
-//                auto vecb =
-//                    cusparse::create_dnvec(b->get_num_stored_elements(),
-//                    b_val);
-//                auto vecc =
-//                    cusparse::create_dnvec(c->get_num_stored_elements(),
-//                    c_val);
-//                size_type buffer_size = 0;
-//                cusparse::spmv_buffersize<ValueType>(handle, trans, &alpha,
-//                mat,
-//                                                     vecb, &beta, vecc, alg,
-//                                                     &buffer_size);
-//
-//                gko::Array<char> buffer_array(exec, buffer_size);
-//                auto buffer = buffer_array.get_data();
-//                cusparse::spmv<ValueType>(handle, trans, &alpha, mat, vecb,
-//                                          &beta, vecc, alg, buffer);
-//                cusparse::destroy(vecb);
-//                cusparse::destroy(vecc);
-//                cusparse::destroy(mat);
-//#endif
-//            }
-//        } else {
-//            GKO_NOT_IMPLEMENTED;
-//        }
-//    } else {
-//        GKO_NOT_IMPLEMENTED;
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPMV_KERNEL);
 
@@ -384,489 +107,11 @@ void advanced_spmv(std::shared_ptr<const CudaExecutor> exec,
                    const matrix::Dense<ValueType> *b,
                    const matrix::Dense<ValueType> *beta,
                    matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    if (a->get_strategy()->get_name() == "load_balance") {
-//        dense::scale(exec, beta, c);
-//
-//        const IndexType nwarps = a->get_num_srow_elements();
-//
-//        if (nwarps > 0) {
-//            const dim3 fbcsr_block(config::warp_size, warps_in_block, 1);
-//            const dim3 fbcsr_grid(ceildiv(nwarps, warps_in_block),
-//                                b->get_size()[1]);
-//            kernel::abstract_spmv<<<fbcsr_grid, fbcsr_block>>>(
-//                nwarps, static_cast<IndexType>(a->get_size()[0]),
-//                as_cuda_type(alpha->get_const_values()),
-//                as_cuda_type(a->get_const_values()), a->get_const_col_idxs(),
-//                as_cuda_type(a->get_const_row_ptrs()),
-//                as_cuda_type(a->get_const_srow()),
-//                as_cuda_type(b->get_const_values()),
-//                as_cuda_type(b->get_stride()), as_cuda_type(c->get_values()),
-//                as_cuda_type(c->get_stride()));
-//        } else {
-//            GKO_NOT_SUPPORTED(nwarps);
-//        }
-//    } else if (a->get_strategy()->get_name() == "sparselib" ||
-//               a->get_strategy()->get_name() == "cusparse") {
-//        if (cusparse::is_supported<ValueType, IndexType>::value) {
-//            // TODO: add implementation for int64 and multiple RHS
-//            if (b->get_stride() != 1 || c->get_stride() != 1)
-//                GKO_NOT_IMPLEMENTED;
-//
-//#if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
-//            auto descr = cusparse::create_mat_descr();
-//            auto row_ptrs = a->get_const_row_ptrs();
-//            auto col_idxs = a->get_const_col_idxs();
-//            cusparse::spmv(exec->get_cusparse_handle(),
-//                           CUSPARSE_OPERATION_NON_TRANSPOSE, a->get_size()[0],
-//                           a->get_size()[1], a->get_num_stored_elements(),
-//                           alpha->get_const_values(), descr,
-//                           a->get_const_values(), row_ptrs, col_idxs,
-//                           b->get_const_values(), beta->get_const_values(),
-//                           c->get_values());
-//
-//            cusparse::destroy(descr);
-//#else  // CUDA_VERSION >= 11000
-//            cusparseOperation_t trans = CUSPARSE_OPERATION_NON_TRANSPOSE;
-//            cusparseSpMVAlg_t alg = CUSPARSE_FBCSRMV_ALG1;
-//            auto row_ptrs = const_cast<IndexType *>(a->get_const_row_ptrs());
-//            auto col_idxs = const_cast<IndexType *>(a->get_const_col_idxs());
-//            auto values = const_cast<ValueType *>(a->get_const_values());
-//            auto mat = cusparse::create_fbcsr(a->get_size()[0],
-//            a->get_size()[1],
-//                                            a->get_num_stored_elements(),
-//                                            row_ptrs, col_idxs, values);
-//            auto b_val = const_cast<ValueType *>(b->get_const_values());
-//            auto c_val = c->get_values();
-//            auto vecb =
-//                cusparse::create_dnvec(b->get_num_stored_elements(), b_val);
-//            auto vecc =
-//                cusparse::create_dnvec(c->get_num_stored_elements(), c_val);
-//            size_type buffer_size = 0;
-//            cusparse::spmv_buffersize<ValueType>(
-//                exec->get_cusparse_handle(), trans, alpha->get_const_values(),
-//                mat, vecb, beta->get_const_values(), vecc, alg, &buffer_size);
-//            gko::Array<char> buffer_array(exec, buffer_size);
-//            auto buffer = buffer_array.get_data();
-//            cusparse::spmv<ValueType>(
-//                exec->get_cusparse_handle(), trans, alpha->get_const_values(),
-//                mat, vecb, beta->get_const_values(), vecc, alg, buffer);
-//            cusparse::destroy(vecb);
-//            cusparse::destroy(vecc);
-//            cusparse::destroy(mat);
-//#endif
-//        } else {
-//            GKO_NOT_IMPLEMENTED;
-//        }
-//    } else if (a->get_strategy()->get_name() == "classical") {
-//        IndexType max_length_per_row = 0;
-//        using Tfbcsr = matrix::Fbcsr<ValueType, IndexType>;
-//        if (auto strategy =
-//                std::dynamic_pointer_cast<const typename Tfbcsr::classical>(
-//                    a->get_strategy())) {
-//            max_length_per_row = strategy->get_max_length_per_row();
-//        } else if (auto strategy = std::dynamic_pointer_cast<
-//                       const typename Tfbcsr::automatical>(a->get_strategy()))
-//                       {
-//            max_length_per_row = strategy->get_max_length_per_row();
-//        } else {
-//            GKO_NOT_SUPPORTED(a->get_strategy());
-//        }
-//        host_kernel::select_classical_spmv(
-//            classical_kernels(),
-//            [&max_length_per_row](int compiled_info) {
-//                return max_length_per_row >= compiled_info;
-//            },
-//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c, alpha,
-//            beta);
-//    } else if (a->get_strategy()->get_name() == "merge_path") {
-//        int items_per_thread =
-//            host_kernel::compute_items_per_thread<ValueType, IndexType>(exec);
-//        host_kernel::select_merge_path_spmv(
-//            compiled_kernels(),
-//            [&items_per_thread](int compiled_info) {
-//                return items_per_thread == compiled_info;
-//            },
-//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c, alpha,
-//            beta);
-//    } else {
-//        GKO_NOT_IMPLEMENTED;
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void spgemm(std::shared_ptr<const CudaExecutor> exec,
-            const matrix::Fbcsr<ValueType, IndexType> *a,
-            const matrix::Fbcsr<ValueType, IndexType> *b,
-            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto a_nnz = IndexType(a->get_num_stored_elements());
-//    auto a_vals = a->get_const_values();
-//    auto a_row_ptrs = a->get_const_row_ptrs();
-//    auto a_col_idxs = a->get_const_col_idxs();
-//    auto b_vals = b->get_const_values();
-//    auto b_row_ptrs = b->get_const_row_ptrs();
-//    auto b_col_idxs = b->get_const_col_idxs();
-//    auto c_row_ptrs = c->get_row_ptrs();
-//
-//    if (cusparse::is_supported<ValueType, IndexType>::value) {
-//        auto handle = exec->get_cusparse_handle();
-//        cusparse::pointer_mode_guard pm_guard(handle);
-//
-//        auto alpha = one<ValueType>();
-//        auto a_nnz = static_cast<IndexType>(a->get_num_stored_elements());
-//        auto b_nnz = static_cast<IndexType>(b->get_num_stored_elements());
-//        auto null_value = static_cast<ValueType *>(nullptr);
-//        auto null_index = static_cast<IndexType *>(nullptr);
-//        auto zero_nnz = IndexType{};
-//        auto m = IndexType(a->get_size()[0]);
-//        auto n = IndexType(b->get_size()[1]);
-//        auto k = IndexType(a->get_size()[1]);
-//        matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
-//        auto &c_col_idxs_array = c_builder.get_col_idx_array();
-//        auto &c_vals_array = c_builder.get_value_array();
-//
-//#if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
-//        auto a_descr = cusparse::create_mat_descr();
-//        auto b_descr = cusparse::create_mat_descr();
-//        auto c_descr = cusparse::create_mat_descr();
-//        auto d_descr = cusparse::create_mat_descr();
-//        auto info = cusparse::create_spgemm_info();
-//        // allocate buffer
-//        size_type buffer_size{};
-//        cusparse::spgemm_buffer_size(
-//            handle, m, n, k, &alpha, a_descr, a_nnz, a_row_ptrs, a_col_idxs,
-//            b_descr, b_nnz, b_row_ptrs, b_col_idxs, null_value, d_descr,
-//            zero_nnz, null_index, null_index, info, buffer_size);
-//        Array<char> buffer_array(exec, buffer_size);
-//        auto buffer = buffer_array.get_data();
-//
-//        // count nnz
-//        IndexType c_nnz{};
-//        cusparse::spgemm_nnz(handle, m, n, k, a_descr, a_nnz, a_row_ptrs,
-//                             a_col_idxs, b_descr, b_nnz, b_row_ptrs,
-//                             b_col_idxs, d_descr, zero_nnz, null_index,
-//                             null_index, c_descr, c_row_ptrs, &c_nnz, info,
-//                             buffer);
-//
-//        // accumulate non-zeros
-//        c_col_idxs_array.resize_and_reset(c_nnz);
-//        c_vals_array.resize_and_reset(c_nnz);
-//        auto c_col_idxs = c_col_idxs_array.get_data();
-//        auto c_vals = c_vals_array.get_data();
-//        cusparse::spgemm(handle, m, n, k, &alpha, a_descr, a_nnz, a_vals,
-//                         a_row_ptrs, a_col_idxs, b_descr, b_nnz, b_vals,
-//                         b_row_ptrs, b_col_idxs, null_value, d_descr,
-//                         zero_nnz, null_value, null_index, null_index,
-//                         c_descr, c_vals, c_row_ptrs, c_col_idxs, info,
-//                         buffer);
-//
-//        cusparse::destroy(info);
-//        cusparse::destroy(d_descr);
-//        cusparse::destroy(c_descr);
-//        cusparse::destroy(b_descr);
-//        cusparse::destroy(a_descr);
-//
-//#else   // CUDA_VERSION >= 11000
-//        const auto beta = zero<ValueType>();
-//        auto spgemm_descr = cusparse::create_spgemm_descr();
-//        auto a_descr = cusparse::create_fbcsr(m, k, a_nnz,
-//                                            const_cast<IndexType
-//                                            *>(a_row_ptrs),
-//                                            const_cast<IndexType
-//                                            *>(a_col_idxs),
-//                                            const_cast<ValueType *>(a_vals));
-//        auto b_descr = cusparse::create_fbcsr(k, n, b_nnz,
-//                                            const_cast<IndexType
-//                                            *>(b_row_ptrs),
-//                                            const_cast<IndexType
-//                                            *>(b_col_idxs),
-//                                            const_cast<ValueType *>(b_vals));
-//        auto c_descr = cusparse::create_fbcsr(m, n, zero_nnz, null_index,
-//                                            null_index, null_value);
-//
-//        // estimate work
-//        size_type buffer1_size{};
-//        cusparse::spgemm_work_estimation(handle, &alpha, a_descr, b_descr,
-//                                         &beta, c_descr, spgemm_descr,
-//                                         buffer1_size, nullptr);
-//        Array<char> buffer1{exec, buffer1_size};
-//        cusparse::spgemm_work_estimation(handle, &alpha, a_descr, b_descr,
-//                                         &beta, c_descr, spgemm_descr,
-//                                         buffer1_size, buffer1.get_data());
-//
-//        // compute spgemm
-//        size_type buffer2_size{};
-//        cusparse::spgemm_compute(handle, &alpha, a_descr, b_descr, &beta,
-//                                 c_descr, spgemm_descr, buffer1.get_data(),
-//                                 buffer2_size, nullptr);
-//        Array<char> buffer2{exec, buffer2_size};
-//        cusparse::spgemm_compute(handle, &alpha, a_descr, b_descr, &beta,
-//                                 c_descr, spgemm_descr, buffer1.get_data(),
-//                                 buffer2_size, buffer2.get_data());
-//
-//        // copy data to result
-//        auto c_nnz = cusparse::sparse_matrix_nnz(c_descr);
-//        c_col_idxs_array.resize_and_reset(c_nnz);
-//        c_vals_array.resize_and_reset(c_nnz);
-//        cusparse::fbcsr_set_pointers(c_descr, c_row_ptrs,
-//                                   c_col_idxs_array.get_data(),
-//                                   c_vals_array.get_data());
-//
-//        cusparse::spgemm_copy(handle, &alpha, a_descr, b_descr, &beta,
-//        c_descr,
-//                              spgemm_descr);
-//
-//        cusparse::destroy(c_descr);
-//        cusparse::destroy(b_descr);
-//        cusparse::destroy(a_descr);
-//        cusparse::destroy(spgemm_descr);
-//#endif  // CUDA_VERSION >= 11000
-//    } else {
-//        GKO_NOT_IMPLEMENTED;
-//    }
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEMM_KERNEL);
-
-
-namespace {
-
-
-template <int subwarp_size, typename ValueType, typename IndexType>
-void spgeam(syn::value_list<int, subwarp_size>,
-            std::shared_ptr<const DefaultExecutor> exec, const ValueType *alpha,
-            const IndexType *a_row_ptrs, const IndexType *a_col_idxs,
-            const ValueType *a_vals, const ValueType *beta,
-            const IndexType *b_row_ptrs, const IndexType *b_col_idxs,
-            const ValueType *b_vals,
-            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto m = static_cast<IndexType>(c->get_size()[0]);
-//    auto c_row_ptrs = c->get_row_ptrs();
-//    // count nnz for alpha * A + beta * B
-//    auto subwarps_per_block = default_block_size / subwarp_size;
-//    auto num_blocks = ceildiv(m, subwarps_per_block);
-//    kernel::spgeam_nnz<subwarp_size><<<num_blocks, default_block_size>>>(
-//        a_row_ptrs, a_col_idxs, b_row_ptrs, b_col_idxs, m, c_row_ptrs);
-//
-//    // build row pointers
-//    components::prefix_sum(exec, c_row_ptrs, m + 1);
-//
-//    // accumulate non-zeros for alpha * A + beta * B
-//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
-//    auto c_nnz = exec->copy_val_to_host(c_row_ptrs + m);
-//    c_builder.get_col_idx_array().resize_and_reset(c_nnz);
-//    c_builder.get_value_array().resize_and_reset(c_nnz);
-//    auto c_col_idxs = c->get_col_idxs();
-//    auto c_vals = c->get_values();
-//    kernel::spgeam<subwarp_size><<<num_blocks, default_block_size>>>(
-//        as_cuda_type(alpha), a_row_ptrs, a_col_idxs, as_cuda_type(a_vals),
-//        as_cuda_type(beta), b_row_ptrs, b_col_idxs, as_cuda_type(b_vals), m,
-//        c_row_ptrs, c_col_idxs, as_cuda_type(c_vals));
-//}
-
-GKO_ENABLE_IMPLEMENTATION_SELECTION(select_spgeam, spgeam);
-
-
-}  // namespace
-
-
-template <typename ValueType, typename IndexType>
-void advanced_spgemm(std::shared_ptr<const CudaExecutor> exec,
-                     const matrix::Dense<ValueType> *alpha,
-                     const matrix::Fbcsr<ValueType, IndexType> *a,
-                     const matrix::Fbcsr<ValueType, IndexType> *b,
-                     const matrix::Dense<ValueType> *beta,
-                     const matrix::Fbcsr<ValueType, IndexType> *d,
-                     matrix::Fbcsr<ValueType, IndexType> *c)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    if (cusparse::is_supported<ValueType, IndexType>::value) {
-//        auto handle = exec->get_cusparse_handle();
-//        cusparse::pointer_mode_guard pm_guard(handle);
-//
-//        auto valpha = exec->copy_val_to_host(alpha->get_const_values());
-//        auto a_nnz = IndexType(a->get_num_stored_elements());
-//        auto a_vals = a->get_const_values();
-//        auto a_row_ptrs = a->get_const_row_ptrs();
-//        auto a_col_idxs = a->get_const_col_idxs();
-//        auto b_nnz = IndexType(b->get_num_stored_elements());
-//        auto b_vals = b->get_const_values();
-//        auto b_row_ptrs = b->get_const_row_ptrs();
-//        auto b_col_idxs = b->get_const_col_idxs();
-//        auto vbeta = exec->copy_val_to_host(beta->get_const_values());
-//        auto d_nnz = IndexType(d->get_num_stored_elements());
-//        auto d_vals = d->get_const_values();
-//        auto d_row_ptrs = d->get_const_row_ptrs();
-//        auto d_col_idxs = d->get_const_col_idxs();
-//        auto m = IndexType(a->get_size()[0]);
-//        auto n = IndexType(b->get_size()[1]);
-//        auto k = IndexType(a->get_size()[1]);
-//        auto c_row_ptrs = c->get_row_ptrs();
-//
-//#if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
-//        matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
-//        auto &c_col_idxs_array = c_builder.get_col_idx_array();
-//        auto &c_vals_array = c_builder.get_value_array();
-//        auto a_descr = cusparse::create_mat_descr();
-//        auto b_descr = cusparse::create_mat_descr();
-//        auto c_descr = cusparse::create_mat_descr();
-//        auto d_descr = cusparse::create_mat_descr();
-//        auto info = cusparse::create_spgemm_info();
-//        // allocate buffer
-//        size_type buffer_size{};
-//        cusparse::spgemm_buffer_size(
-//            handle, m, n, k, &valpha, a_descr, a_nnz, a_row_ptrs, a_col_idxs,
-//            b_descr, b_nnz, b_row_ptrs, b_col_idxs, &vbeta, d_descr, d_nnz,
-//            d_row_ptrs, d_col_idxs, info, buffer_size);
-//        Array<char> buffer_array(exec, buffer_size);
-//        auto buffer = buffer_array.get_data();
-//
-//        // count nnz
-//        IndexType c_nnz{};
-//        cusparse::spgemm_nnz(handle, m, n, k, a_descr, a_nnz, a_row_ptrs,
-//                             a_col_idxs, b_descr, b_nnz, b_row_ptrs,
-//                             b_col_idxs, d_descr, d_nnz, d_row_ptrs,
-//                             d_col_idxs, c_descr, c_row_ptrs, &c_nnz, info,
-//                             buffer);
-//
-//        // accumulate non-zeros
-//        c_col_idxs_array.resize_and_reset(c_nnz);
-//        c_vals_array.resize_and_reset(c_nnz);
-//        auto c_col_idxs = c_col_idxs_array.get_data();
-//        auto c_vals = c_vals_array.get_data();
-//        cusparse::spgemm(handle, m, n, k, &valpha, a_descr, a_nnz, a_vals,
-//                         a_row_ptrs, a_col_idxs, b_descr, b_nnz, b_vals,
-//                         b_row_ptrs, b_col_idxs, &vbeta, d_descr, d_nnz,
-//                         d_vals, d_row_ptrs, d_col_idxs, c_descr, c_vals,
-//                         c_row_ptrs, c_col_idxs, info, buffer);
-//
-//        cusparse::destroy(info);
-//        cusparse::destroy(d_descr);
-//        cusparse::destroy(c_descr);
-//        cusparse::destroy(b_descr);
-//        cusparse::destroy(a_descr);
-//#else   // CUDA_VERSION >= 11000
-//        auto null_value = static_cast<ValueType *>(nullptr);
-//        auto null_index = static_cast<IndexType *>(nullptr);
-//        auto one_val = one<ValueType>();
-//        auto zero_val = zero<ValueType>();
-//        auto zero_nnz = IndexType{};
-//        auto spgemm_descr = cusparse::create_spgemm_descr();
-//        auto a_descr = cusparse::create_fbcsr(m, k, a_nnz,
-//                                            const_cast<IndexType
-//                                            *>(a_row_ptrs),
-//                                            const_cast<IndexType
-//                                            *>(a_col_idxs),
-//                                            const_cast<ValueType *>(a_vals));
-//        auto b_descr = cusparse::create_fbcsr(k, n, b_nnz,
-//                                            const_cast<IndexType
-//                                            *>(b_row_ptrs),
-//                                            const_cast<IndexType
-//                                            *>(b_col_idxs),
-//                                            const_cast<ValueType *>(b_vals));
-//        auto c_descr = cusparse::create_fbcsr(m, n, zero_nnz, null_index,
-//                                            null_index, null_value);
-//
-//        // estimate work
-//        size_type buffer1_size{};
-//        cusparse::spgemm_work_estimation(handle, &one_val, a_descr, b_descr,
-//                                         &zero_val, c_descr, spgemm_descr,
-//                                         buffer1_size, nullptr);
-//        Array<char> buffer1{exec, buffer1_size};
-//        cusparse::spgemm_work_estimation(handle, &one_val, a_descr, b_descr,
-//                                         &zero_val, c_descr, spgemm_descr,
-//                                         buffer1_size, buffer1.get_data());
-//
-//        // compute spgemm
-//        size_type buffer2_size{};
-//        cusparse::spgemm_compute(handle, &one_val, a_descr, b_descr,
-//        &zero_val,
-//                                 c_descr, spgemm_descr, buffer1.get_data(),
-//                                 buffer2_size, nullptr);
-//        Array<char> buffer2{exec, buffer2_size};
-//        cusparse::spgemm_compute(handle, &one_val, a_descr, b_descr,
-//        &zero_val,
-//                                 c_descr, spgemm_descr, buffer1.get_data(),
-//                                 buffer2_size, buffer2.get_data());
-//
-//        // write result to temporary storage
-//        auto c_tmp_nnz = cusparse::sparse_matrix_nnz(c_descr);
-//        Array<IndexType> c_tmp_row_ptrs_array(exec, m + 1);
-//        Array<IndexType> c_tmp_col_idxs_array(exec, c_tmp_nnz);
-//        Array<ValueType> c_tmp_vals_array(exec, c_tmp_nnz);
-//        cusparse::fbcsr_set_pointers(c_descr, c_tmp_row_ptrs_array.get_data(),
-//                                   c_tmp_col_idxs_array.get_data(),
-//                                   c_tmp_vals_array.get_data());
-//
-//        cusparse::spgemm_copy(handle, &one_val, a_descr, b_descr, &zero_val,
-//                              c_descr, spgemm_descr);
-//
-//        cusparse::destroy(c_descr);
-//        cusparse::destroy(b_descr);
-//        cusparse::destroy(a_descr);
-//        cusparse::destroy(spgemm_descr);
-//
-//        auto spgeam_total_nnz = c_tmp_nnz + d->get_num_stored_elements();
-//        auto nnz_per_row = spgeam_total_nnz / m;
-//        select_spgeam(
-//            spgeam_kernels(),
-//            [&](int compiled_subwarp_size) {
-//                return compiled_subwarp_size >= nnz_per_row ||
-//                       compiled_subwarp_size == config::warp_size;
-//            },
-//            syn::value_list<int>(), syn::type_list<>(), exec,
-//            alpha->get_const_values(), c_tmp_row_ptrs_array.get_const_data(),
-//            c_tmp_col_idxs_array.get_const_data(),
-//            c_tmp_vals_array.get_const_data(), beta->get_const_values(),
-//            d_row_ptrs, d_col_idxs, d_vals, c);
-//#endif  // CUDA_VERSION >= 11000
-//    } else {
-//        GKO_NOT_IMPLEMENTED;
-//    }
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void spgeam(std::shared_ptr<const DefaultExecutor> exec,
-            const matrix::Dense<ValueType> *alpha,
-            const matrix::Fbcsr<ValueType, IndexType> *a,
-            const matrix::Dense<ValueType> *beta,
-            const matrix::Fbcsr<ValueType, IndexType> *b,
-            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto total_nnz =
-//        a->get_num_stored_elements() + b->get_num_stored_elements();
-//    auto nnz_per_row = total_nnz / a->get_size()[0];
-//    select_spgeam(
-//        spgeam_kernels(),
-//        [&](int compiled_subwarp_size) {
-//            return compiled_subwarp_size >= nnz_per_row ||
-//                   compiled_subwarp_size == config::warp_size;
-//        },
-//        syn::value_list<int>(), syn::type_list<>(), exec,
-//        alpha->get_const_values(), a->get_const_row_ptrs(),
-//        a->get_const_col_idxs(), a->get_const_values(),
-//        beta->get_const_values(), b->get_const_row_ptrs(),
-//        b->get_const_col_idxs(), b->get_const_values(), c);
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEAM_KERNEL);
-
-
 template <typename IndexType>
 void convert_row_ptrs_to_idxs(std::shared_ptr<const CudaExecutor> exec,
                               const IndexType *ptrs, size_type num_rows,
@@ -880,50 +125,10 @@ void convert_row_ptrs_to_idxs(std::shared_ptr<const CudaExecutor> exec,
 //}
 
 
-template <typename ValueType, typename IndexType>
-void convert_to_coo(std::shared_ptr<const CudaExecutor> exec,
-                    const matrix::Fbcsr<ValueType, IndexType> *source,
-                    matrix::Coo<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = result->get_size()[0];
-//
-//    auto row_idxs = result->get_row_idxs();
-//    const auto source_row_ptrs = source->get_const_row_ptrs();
-//
-//    convert_row_ptrs_to_idxs(exec, source_row_ptrs, num_rows, row_idxs);
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const CudaExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *source,
                       matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto num_rows = result->get_size()[0];
-//    const auto num_cols = result->get_size()[1];
-//    const auto stride = result->get_stride();
-//    const auto row_ptrs = source->get_const_row_ptrs();
-//    const auto col_idxs = source->get_const_col_idxs();
-//    const auto vals = source->get_const_values();
-//
-//    const dim3 block_size(config::warp_size,
-//                          config::max_block_size / config::warp_size, 1);
-//    const dim3 init_grid_dim(ceildiv(num_cols, block_size.x),
-//                             ceildiv(num_rows, block_size.y), 1);
-//    kernel::initialize_zero_dense<<<init_grid_dim, block_size>>>(
-//        num_rows, num_cols, stride, as_cuda_type(result->get_values()));
-//
-//    auto grid_dim = ceildiv(num_rows, default_block_size);
-//    kernel::fill_in_dense<<<grid_dim, default_block_size>>>(
-//        num_rows, as_cuda_type(row_ptrs), as_cuda_type(col_idxs),
-//        as_cuda_type(vals), stride, as_cuda_type(result->get_values()));
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
@@ -939,203 +144,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void convert_to_sellp(std::shared_ptr<const CudaExecutor> exec,
-                      const matrix::Fbcsr<ValueType, IndexType> *source,
-                      matrix::Sellp<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto num_rows = result->get_size()[0];
-//    const auto num_cols = result->get_size()[1];
-//
-//    auto result_values = result->get_values();
-//    auto result_col_idxs = result->get_col_idxs();
-//    auto slice_lengths = result->get_slice_lengths();
-//    auto slice_sets = result->get_slice_sets();
-//
-//    const auto slice_size = (result->get_slice_size() == 0)
-//                                ? matrix::default_slice_size
-//                                : result->get_slice_size();
-//    const auto stride_factor = (result->get_stride_factor() == 0)
-//                                   ? matrix::default_stride_factor
-//                                   : result->get_stride_factor();
-//    const int slice_num = ceildiv(num_rows, slice_size);
-//
-//    const auto source_values = source->get_const_values();
-//    const auto source_row_ptrs = source->get_const_row_ptrs();
-//    const auto source_col_idxs = source->get_const_col_idxs();
-//
-//    auto nnz_per_row = Array<size_type>(exec, num_rows);
-//    auto grid_dim = ceildiv(num_rows, default_block_size);
-//
-//    if (grid_dim > 0) {
-//        kernel::calculate_nnz_per_row<<<grid_dim, default_block_size>>>(
-//            num_rows, as_cuda_type(source_row_ptrs),
-//            as_cuda_type(nnz_per_row.get_data()));
-//    }
-//
-//    grid_dim = slice_num;
-//
-//    if (grid_dim > 0) {
-//        kernel::calculate_slice_lengths<<<grid_dim, config::warp_size>>>(
-//            num_rows, slice_size, stride_factor,
-//            as_cuda_type(nnz_per_row.get_const_data()),
-//            as_cuda_type(slice_lengths), as_cuda_type(slice_sets));
-//    }
-//
-//    components::prefix_sum(exec, slice_sets, slice_num + 1);
-//
-//    grid_dim = ceildiv(num_rows, default_block_size);
-//    if (grid_dim > 0) {
-//        kernel::fill_in_sellp<<<grid_dim, default_block_size>>>(
-//            num_rows, slice_size, as_cuda_type(source_values),
-//            as_cuda_type(source_row_ptrs), as_cuda_type(source_col_idxs),
-//            as_cuda_type(slice_lengths), as_cuda_type(slice_sets),
-//            as_cuda_type(result_col_idxs), as_cuda_type(result_values));
-//    }
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void convert_to_ell(std::shared_ptr<const CudaExecutor> exec,
-                    const matrix::Fbcsr<ValueType, IndexType> *source,
-                    matrix::Ell<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto source_values = source->get_const_values();
-//    const auto source_row_ptrs = source->get_const_row_ptrs();
-//    const auto source_col_idxs = source->get_const_col_idxs();
-//
-//    auto result_values = result->get_values();
-//    auto result_col_idxs = result->get_col_idxs();
-//    const auto stride = result->get_stride();
-//    const auto max_nnz_per_row = result->get_num_stored_elements_per_row();
-//    const auto num_rows = result->get_size()[0];
-//    const auto num_cols = result->get_size()[1];
-//
-//    const auto init_grid_dim =
-//        ceildiv(max_nnz_per_row * num_rows, default_block_size);
-//
-//    kernel::initialize_zero_ell<<<init_grid_dim, default_block_size>>>(
-//        max_nnz_per_row, stride, as_cuda_type(result_values),
-//        as_cuda_type(result_col_idxs));
-//
-//    const auto grid_dim =
-//        ceildiv(num_rows * config::warp_size, default_block_size);
-//
-//    kernel::fill_in_ell<<<grid_dim, default_block_size>>>(
-//        num_rows, stride, as_cuda_type(source_values),
-//        as_cuda_type(source_row_ptrs), as_cuda_type(source_col_idxs),
-//        as_cuda_type(result_values), as_cuda_type(result_col_idxs));
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void calculate_total_cols(std::shared_ptr<const CudaExecutor> exec,
-                          const matrix::Fbcsr<ValueType, IndexType> *source,
-                          size_type *result, size_type stride_factor,
-                          size_type slice_size) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto num_rows = source->get_size()[0];
-//
-//    if (num_rows == 0) {
-//        *result = 0;
-//        return;
-//    }
-//
-//    const auto slice_num = ceildiv(num_rows, slice_size);
-//    const auto row_ptrs = source->get_const_row_ptrs();
-//
-//    auto nnz_per_row = Array<size_type>(exec, num_rows);
-//    auto grid_dim = ceildiv(num_rows, default_block_size);
-//
-//    kernel::calculate_nnz_per_row<<<grid_dim, default_block_size>>>(
-//        num_rows, as_cuda_type(row_ptrs),
-//        as_cuda_type(nnz_per_row.get_data()));
-//
-//    grid_dim = ceildiv(slice_num * config::warp_size, default_block_size);
-//    auto max_nnz_per_slice = Array<size_type>(exec, slice_num);
-//
-//    kernel::reduce_max_nnz_per_slice<<<grid_dim, default_block_size>>>(
-//        num_rows, slice_size, stride_factor,
-//        as_cuda_type(nnz_per_row.get_const_data()),
-//        as_cuda_type(max_nnz_per_slice.get_data()));
-//
-//    grid_dim = ceildiv(slice_num, default_block_size);
-//    auto block_results = Array<size_type>(exec, grid_dim);
-//
-//    kernel::reduce_total_cols<<<grid_dim, default_block_size>>>(
-//        slice_num, as_cuda_type(max_nnz_per_slice.get_const_data()),
-//        as_cuda_type(block_results.get_data()));
-//
-//    auto d_result = Array<size_type>(exec, 1);
-//
-//    kernel::reduce_total_cols<<<1, default_block_size>>>(
-//        grid_dim, as_cuda_type(block_results.get_const_data()),
-//        as_cuda_type(d_result.get_data()));
-//
-//    *result = exec->copy_val_to_host(d_result.get_const_data());
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void transpose(std::shared_ptr<const CudaExecutor> exec,
                const matrix::Fbcsr<ValueType, IndexType> *orig,
                matrix::Fbcsr<ValueType, IndexType> *trans) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    if (cusparse::is_supported<ValueType, IndexType>::value) {
-//#if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
-//        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
-//        cusparseIndexBase_t idxBase = CUSPARSE_INDEX_BASE_ZERO;
-//
-//        cusparse::transpose(
-//            exec->get_cusparse_handle(), orig->get_size()[0],
-//            orig->get_size()[1], orig->get_num_stored_elements(),
-//            orig->get_const_values(), orig->get_const_row_ptrs(),
-//            orig->get_const_col_idxs(), trans->get_values(),
-//            trans->get_row_ptrs(), trans->get_col_idxs(), copyValues,
-//            idxBase);
-//#else  // CUDA_VERSION >= 11000
-//        cudaDataType_t cu_value =
-//            gko::kernels::cuda::cuda_data_type<ValueType>();
-//        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
-//        cusparseIndexBase_t idxBase = CUSPARSE_INDEX_BASE_ZERO;
-//        cusparseFbcsr2CscAlg_t alg = CUSPARSE_FBCSR2CSC_ALG1;
-//        size_type buffer_size = 0;
-//        cusparse::transpose_buffersize(
-//            exec->get_cusparse_handle(), orig->get_size()[0],
-//            orig->get_size()[1], orig->get_num_stored_elements(),
-//            orig->get_const_values(), orig->get_const_row_ptrs(),
-//            orig->get_const_col_idxs(), trans->get_values(),
-//            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value,
-//            copyValues, idxBase, alg, &buffer_size);
-//        Array<char> buffer_array(exec, buffer_size);
-//        auto buffer = buffer_array.get_data();
-//        cusparse::transpose(
-//            exec->get_cusparse_handle(), orig->get_size()[0],
-//            orig->get_size()[1], orig->get_num_stored_elements(),
-//            orig->get_const_values(), orig->get_const_row_ptrs(),
-//            orig->get_const_col_idxs(), trans->get_values(),
-//            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value,
-//            copyValues, idxBase, alg, buffer);
-//#endif
-//    } else {
-//        GKO_NOT_IMPLEMENTED;
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL);
@@ -1146,94 +158,11 @@ void conj_transpose(std::shared_ptr<const CudaExecutor> exec,
                     const matrix::Fbcsr<ValueType, IndexType> *orig,
                     matrix::Fbcsr<ValueType, IndexType> *trans)
     GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    if (cusparse::is_supported<ValueType, IndexType>::value) {
-//        const dim3 block_size(default_block_size, 1, 1);
-//        const dim3 grid_size(
-//            ceildiv(trans->get_num_stored_elements(), block_size.x), 1, 1);
-//
-//#if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
-//        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
-//        cusparseIndexBase_t idxBase = CUSPARSE_INDEX_BASE_ZERO;
-//
-//        cusparse::transpose(
-//            exec->get_cusparse_handle(), orig->get_size()[0],
-//            orig->get_size()[1], orig->get_num_stored_elements(),
-//            orig->get_const_values(), orig->get_const_row_ptrs(),
-//            orig->get_const_col_idxs(), trans->get_values(),
-//            trans->get_row_ptrs(), trans->get_col_idxs(), copyValues,
-//            idxBase);
-//#else  // CUDA_VERSION >= 11000
-//        cudaDataType_t cu_value =
-//            gko::kernels::cuda::cuda_data_type<ValueType>();
-//        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
-//        cusparseIndexBase_t idxBase = CUSPARSE_INDEX_BASE_ZERO;
-//        cusparseFbcsr2CscAlg_t alg = CUSPARSE_FBCSR2CSC_ALG1;
-//        size_type buffer_size = 0;
-//        cusparse::transpose_buffersize(
-//            exec->get_cusparse_handle(), orig->get_size()[0],
-//            orig->get_size()[1], orig->get_num_stored_elements(),
-//            orig->get_const_values(), orig->get_const_row_ptrs(),
-//            orig->get_const_col_idxs(), trans->get_values(),
-//            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value,
-//            copyValues, idxBase, alg, &buffer_size);
-//        Array<char> buffer_array(exec, buffer_size);
-//        auto buffer = buffer_array.get_data();
-//        cusparse::transpose(
-//            exec->get_cusparse_handle(), orig->get_size()[0],
-//            orig->get_size()[1], orig->get_num_stored_elements(),
-//            orig->get_const_values(), orig->get_const_row_ptrs(),
-//            orig->get_const_col_idxs(), trans->get_values(),
-//            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value,
-//            copyValues, idxBase, alg, buffer);
-//#endif
-//
-//        conjugate_kernel<<<grid_size, block_size, 0, 0>>>(
-//            trans->get_num_stored_elements(),
-//            as_cuda_type(trans->get_values()));
-//    } else {
-//        GKO_NOT_IMPLEMENTED;
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void row_permute(std::shared_ptr<const CudaExecutor> exec,
-                 const IndexType *permutation_indices,
-                 const matrix::Fbcsr<ValueType, IndexType> *orig,
-                 matrix::Fbcsr<ValueType, IndexType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_row_permute(std::shared_ptr<const CudaExecutor> exec,
-                         const IndexType *permutation_indices,
-                         const matrix::Fbcsr<ValueType, IndexType> *orig,
-                         matrix::Fbcsr<ValueType, IndexType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(
-    std::shared_ptr<const CudaExecutor> exec,
-    const IndexType *permutation_indices,
-    const matrix::Fbcsr<ValueType, IndexType> *orig,
-    matrix::Fbcsr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void calculate_max_nnz_per_row(
     std::shared_ptr<const CudaExecutor> exec,
@@ -1269,52 +198,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void convert_to_hybrid(std::shared_ptr<const CudaExecutor> exec,
-                       const matrix::Fbcsr<ValueType, IndexType> *source,
-                       matrix::Hybrid<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto ell_val = result->get_ell_values();
-//    auto ell_col = result->get_ell_col_idxs();
-//    auto coo_val = result->get_coo_values();
-//    auto coo_col = result->get_coo_col_idxs();
-//    auto coo_row = result->get_coo_row_idxs();
-//    const auto stride = result->get_ell_stride();
-//    const auto max_nnz_per_row =
-//    result->get_ell_num_stored_elements_per_row(); const auto num_rows =
-//    result->get_size()[0]; const auto coo_num_stored_elements =
-//    result->get_coo_num_stored_elements(); auto grid_dim =
-//    ceildiv(max_nnz_per_row * num_rows, default_block_size);
-//
-//    kernel::initialize_zero_ell<<<grid_dim, default_block_size>>>(
-//        max_nnz_per_row, stride, as_cuda_type(ell_val),
-//        as_cuda_type(ell_col));
-//
-//    grid_dim = ceildiv(num_rows, default_block_size);
-//    auto coo_offset = Array<size_type>(exec, num_rows);
-//    kernel::calculate_hybrid_coo_row_nnz<<<grid_dim, default_block_size>>>(
-//        num_rows, max_nnz_per_row, as_cuda_type(source->get_const_row_ptrs()),
-//        as_cuda_type(coo_offset.get_data()));
-//
-//    components::prefix_sum(exec, coo_offset.get_data(), num_rows);
-//
-//    grid_dim = ceildiv(num_rows * config::warp_size, default_block_size);
-//    kernel::fill_in_hybrid<<<grid_dim, default_block_size>>>(
-//        num_rows, stride, max_nnz_per_row,
-//        as_cuda_type(source->get_const_values()),
-//        as_cuda_type(source->get_const_row_ptrs()),
-//        as_cuda_type(source->get_const_col_idxs()),
-//        as_cuda_type(coo_offset.get_const_data()), as_cuda_type(ell_val),
-//        as_cuda_type(ell_col), as_cuda_type(coo_val), as_cuda_type(coo_col),
-//        as_cuda_type(coo_row));
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void calculate_nonzeros_per_row(
     std::shared_ptr<const CudaExecutor> exec,
@@ -1338,54 +221,6 @@ template <typename ValueType, typename IndexType>
 void sort_by_column_index(std::shared_ptr<const CudaExecutor> exec,
                           matrix::Fbcsr<ValueType, IndexType> *to_sort)
     GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    if (cusparse::is_supported<ValueType, IndexType>::value) {
-//        auto handle = exec->get_cusparse_handle();
-//        auto descr = cusparse::create_mat_descr();
-//        auto m = IndexType(to_sort->get_size()[0]);
-//        auto n = IndexType(to_sort->get_size()[1]);
-//        auto nnz = IndexType(to_sort->get_num_stored_elements());
-//        auto row_ptrs = to_sort->get_const_row_ptrs();
-//        auto col_idxs = to_sort->get_col_idxs();
-//        auto vals = to_sort->get_values();
-//
-//        // copy values
-//        Array<ValueType> tmp_vals_array(exec, nnz);
-//        exec->copy(nnz, vals, tmp_vals_array.get_data());
-//        auto tmp_vals = tmp_vals_array.get_const_data();
-//
-//        // init identity permutation
-//        Array<IndexType> permutation_array(exec, nnz);
-//        auto permutation = permutation_array.get_data();
-//        cusparse::create_identity_permutation(handle, nnz, permutation);
-//
-//        // allocate buffer
-//        size_type buffer_size{};
-//        cusparse::fbcsrsort_buffer_size(handle, m, n, nnz, row_ptrs, col_idxs,
-//                                      buffer_size);
-//        Array<char> buffer_array{exec, buffer_size};
-//        auto buffer = buffer_array.get_data();
-//
-//        // sort column indices
-//        cusparse::fbcsrsort(handle, m, n, nnz, descr, row_ptrs, col_idxs,
-//                          permutation, buffer);
-//
-//        // sort values
-//#if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
-//        cusparse::gather(handle, nnz, tmp_vals, vals, permutation);
-//#else  // CUDA_VERSION >= 11000
-//        auto val_vec = cusparse::create_spvec(nnz, nnz, permutation, vals);
-//        auto tmp_vec =
-//            cusparse::create_dnvec(nnz, const_cast<ValueType *>(tmp_vals));
-//        cusparse::gather(handle, tmp_vec, val_vec);
-//#endif
-//
-//        cusparse::destroy(descr);
-//    } else {
-//        GKO_NOT_IMPLEMENTED;
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
@@ -1418,23 +253,6 @@ template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const CudaExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *orig,
                       matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto nnz = orig->get_num_stored_elements();
-//    const auto diag_size = diag->get_size()[0];
-//    const auto num_blocks =
-//        ceildiv(config::warp_size * diag_size, default_block_size);
-//
-//    const auto orig_values = orig->get_const_values();
-//    const auto orig_row_ptrs = orig->get_const_row_ptrs();
-//    const auto orig_col_idxs = orig->get_const_col_idxs();
-//    auto diag_values = diag->get_values();
-//
-//    kernel::extract_diagonal<<<num_blocks, default_block_size>>>(
-//        diag_size, nnz, as_cuda_type(orig_values),
-//        as_cuda_type(orig_row_ptrs), as_cuda_type(orig_col_idxs),
-//        as_cuda_type(diag_values));
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL);
diff --git a/cuda/test/matrix/CMakeLists.txt b/cuda/test/matrix/CMakeLists.txt
index 9d40716bea0..65ce218ac71 100644
--- a/cuda/test/matrix/CMakeLists.txt
+++ b/cuda/test/matrix/CMakeLists.txt
@@ -3,6 +3,5 @@ ginkgo_create_test(csr_kernels)
 ginkgo_create_test(dense_kernels)
 ginkgo_create_test(diagonal_kernels)
 ginkgo_create_test(ell_kernels)
-#ginkgo_create_test(fbcsr_kernels)
 ginkgo_create_test(hybrid_kernels)
 ginkgo_create_test(sellp_kernels)
diff --git a/cuda/test/matrix/fbcsr_kernels.cpp b/cuda/test/matrix/fbcsr_kernels.cpp
deleted file mode 100644
index 72bc9c149f4..00000000000
--- a/cuda/test/matrix/fbcsr_kernels.cpp
+++ /dev/null
@@ -1,883 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/matrix/fbcsr.hpp>
-
-
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/coo.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/diagonal.hpp>
-#include <ginkgo/core/matrix/ell.hpp>
-#include <ginkgo/core/matrix/hybrid.hpp>
-#include <ginkgo/core/matrix/identity.hpp>
-#include <ginkgo/core/matrix/sellp.hpp>
-#include <ginkgo/core/matrix/sparsity_csr.hpp>
-
-
-#include "core/matrix/fbcsr_kernels.hpp"
-#include "cuda/test/utils.hpp"
-
-
-namespace {
-
-
-class Fbcsr : public ::testing::Test {
-protected:
-    using Vec = gko::matrix::Dense<>;
-    using Mtx = gko::matrix::Fbcsr<>;
-    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
-    using ComplexMtx = gko::matrix::Fbcsr<std::complex<double>>;
-
-    Fbcsr() : mtx_size(532, 231), rand_engine(42) {}
-
-    void SetUp()
-    {
-        ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
-        ref = gko::ReferenceExecutor::create();
-        cuda = gko::CudaExecutor::create(0, ref);
-    }
-
-    void TearDown()
-    {
-        if (cuda != nullptr) {
-            ASSERT_NO_THROW(cuda->synchronize());
-        }
-    }
-
-    template <typename MtxType>
-    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
-                                     int min_nnz_row)
-    {
-        return gko::test::generate_random_matrix<MtxType>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(min_nnz_row, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
-    }
-
-    void set_up_apply_data(std::shared_ptr<Mtx::strategy_type> strategy,
-                           int num_vectors = 1)
-    {
-        mtx = Mtx::create(ref, strategy);
-        mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[1], 1));
-        square_mtx = Mtx::create(ref, strategy);
-        square_mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[0], 1));
-        expected = gen_mtx<Vec>(mtx_size[0], num_vectors, 1);
-        y = gen_mtx<Vec>(mtx_size[1], num_vectors, 1);
-        alpha = gko::initialize<Vec>({2.0}, ref);
-        beta = gko::initialize<Vec>({-1.0}, ref);
-        dmtx = Mtx::create(cuda, strategy);
-        dmtx->copy_from(mtx.get());
-        square_dmtx = Mtx::create(cuda, strategy);
-        square_dmtx->copy_from(square_mtx.get());
-        dresult = Vec::create(cuda);
-        dresult->copy_from(expected.get());
-        dy = Vec::create(cuda);
-        dy->copy_from(y.get());
-        dalpha = Vec::create(cuda);
-        dalpha->copy_from(alpha.get());
-        dbeta = Vec::create(cuda);
-        dbeta->copy_from(beta.get());
-    }
-
-    void set_up_apply_complex_data(
-        std::shared_ptr<ComplexMtx::strategy_type> strategy)
-    {
-        complex_mtx = ComplexMtx::create(ref, strategy);
-        complex_mtx->copy_from(
-            gen_mtx<ComplexVec>(mtx_size[0], mtx_size[1], 1));
-        complex_dmtx = ComplexMtx::create(cuda, strategy);
-        complex_dmtx->copy_from(complex_mtx.get());
-    }
-
-    struct matrix_pair {
-        std::unique_ptr<Mtx> ref;
-        std::unique_ptr<Mtx> cuda;
-    };
-
-    matrix_pair gen_unsorted_mtx()
-    {
-        constexpr int min_nnz_per_row = 2;  // Must be at least 2
-        auto local_mtx_ref =
-            gen_mtx<Mtx>(mtx_size[0], mtx_size[1], min_nnz_per_row);
-        for (size_t row = 0; row < mtx_size[0]; ++row) {
-            const auto row_ptrs = local_mtx_ref->get_const_row_ptrs();
-            const auto start_row = row_ptrs[row];
-            auto col_idx = local_mtx_ref->get_col_idxs() + start_row;
-            auto vals = local_mtx_ref->get_values() + start_row;
-            const auto nnz_in_this_row = row_ptrs[row + 1] - row_ptrs[row];
-            auto swap_idx_dist =
-                std::uniform_int_distribution<>(0, nnz_in_this_row - 1);
-            // shuffle `nnz_in_this_row / 2` times
-            for (size_t perm = 0; perm < nnz_in_this_row; perm += 2) {
-                const auto idx1 = swap_idx_dist(rand_engine);
-                const auto idx2 = swap_idx_dist(rand_engine);
-                std::swap(col_idx[idx1], col_idx[idx2]);
-                std::swap(vals[idx1], vals[idx2]);
-            }
-        }
-        auto local_mtx_cuda = Mtx::create(cuda);
-        local_mtx_cuda->copy_from(local_mtx_ref.get());
-
-        return {std::move(local_mtx_ref), std::move(local_mtx_cuda)};
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::CudaExecutor> cuda;
-
-    const gko::dim<2> mtx_size;
-    std::ranlux48 rand_engine;
-
-    std::unique_ptr<Mtx> mtx;
-    std::unique_ptr<ComplexMtx> complex_mtx;
-    std::unique_ptr<Mtx> square_mtx;
-    std::unique_ptr<Vec> expected;
-    std::unique_ptr<Vec> y;
-    std::unique_ptr<Vec> alpha;
-    std::unique_ptr<Vec> beta;
-
-    std::unique_ptr<Mtx> dmtx;
-    std::unique_ptr<ComplexMtx> complex_dmtx;
-    std::unique_ptr<Mtx> square_dmtx;
-    std::unique_ptr<Vec> dresult;
-    std::unique_ptr<Vec> dy;
-    std::unique_ptr<Vec> dalpha;
-    std::unique_ptr<Vec> dbeta;
-};
-
-
-TEST_F(Fbcsr, StrategyAfterCopyIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::load_balance>(cuda));
-//
-//    ASSERT_EQ(mtx->get_strategy()->get_name(),
-//              dmtx->get_strategy()->get_name());
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithLoadBalance)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::load_balance>(cuda));
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithLoadBalance)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::load_balance>(cuda));
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithCusparse)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithCusparse)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithMergePath)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::merge_path>());
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithMergePath)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::merge_path>());
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithClassical)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::classical>());
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithClassical)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::classical>());
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithAutomatical)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>(cuda));
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyToDenseMatrixIsEquivalentToRefWithLoadBalance)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::load_balance>(cuda), 3);
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithLoadBalance)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::load_balance>(cuda), 3);
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyToDenseMatrixIsEquivalentToRefWithClassical)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::classical>(), 3);
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithClassical)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::classical>(), 3);
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyToDenseMatrixIsEquivalentToRefWithMergePath)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::merge_path>(), 3);
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithMergePath)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::merge_path>(), 3);
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyToFbcsrMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>());
-//    auto trans = mtx->transpose();
-//    auto d_trans = dmtx->transpose();
-//
-//    mtx->apply(alpha.get(), trans.get(), beta.get(), square_mtx.get());
-//    dmtx->apply(dalpha.get(), d_trans.get(), dbeta.get(), square_dmtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
-//    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-//    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyToFbcsrMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>());
-//    auto trans = mtx->transpose();
-//    auto d_trans = dmtx->transpose();
-//
-//    mtx->apply(trans.get(), square_mtx.get());
-//    dmtx->apply(d_trans.get(), square_dmtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
-//    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-//    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>());
-//    auto a = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
-//    auto b = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
-//    auto da = Mtx::create(cuda);
-//    auto db = Mtx::create(cuda);
-//    da->copy_from(a.get());
-//    db->copy_from(b.get());
-//    auto id = gko::matrix::Identity<Mtx::value_type>::create(ref,
-//    mtx_size[1]); auto did =
-//        gko::matrix::Identity<Mtx::value_type>::create(cuda, mtx_size[1]);
-//
-//    a->apply(alpha.get(), id.get(), beta.get(), b.get());
-//    da->apply(dalpha.get(), did.get(), dbeta.get(), db.get());
-//
-//    GKO_ASSERT_MTX_NEAR(b, db, 1e-14);
-//    GKO_ASSERT_MTX_EQ_SPARSITY(b, db);
-//    ASSERT_TRUE(db->is_sorted_by_column_index());
-//}
-
-
-TEST_F(Fbcsr, TransposeIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>(cuda));
-//
-//    auto trans = mtx->transpose();
-//    auto d_trans = dmtx->transpose();
-//
-//    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(d_trans.get()),
-//                        static_cast<Mtx *>(trans.get()), 0.0);
-//    ASSERT_TRUE(static_cast<Mtx
-//    *>(d_trans.get())->is_sorted_by_column_index());
-//}
-
-
-TEST_F(Fbcsr, ConjugateTransposeIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(cuda));
-//
-//    auto trans = complex_mtx->conj_transpose();
-//    auto d_trans = complex_dmtx->conj_transpose();
-//
-//    GKO_ASSERT_MTX_NEAR(static_cast<ComplexMtx *>(d_trans.get()),
-//                        static_cast<ComplexMtx *>(trans.get()), 0.0);
-//    ASSERT_TRUE(
-//        static_cast<ComplexMtx
-//        *>(d_trans.get())->is_sorted_by_column_index());
-//}
-
-
-TEST_F(Fbcsr, ConvertToDenseIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto dense_mtx = gko::matrix::Dense<>::create(ref);
-//    auto ddense_mtx = gko::matrix::Dense<>::create(cuda);
-//
-//    mtx->convert_to(dense_mtx.get());
-//    dmtx->convert_to(ddense_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToDenseIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto dense_mtx = gko::matrix::Dense<>::create(ref);
-//    auto ddense_mtx = gko::matrix::Dense<>::create(cuda);
-//
-//    mtx->move_to(dense_mtx.get());
-//    dmtx->move_to(ddense_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, ConvertToEllIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto ell_mtx = gko::matrix::Ell<>::create(ref);
-//    auto dell_mtx = gko::matrix::Ell<>::create(cuda);
-//
-//    mtx->convert_to(ell_mtx.get());
-//    dmtx->convert_to(dell_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(ell_mtx.get(), dell_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToEllIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto ell_mtx = gko::matrix::Ell<>::create(ref);
-//    auto dell_mtx = gko::matrix::Ell<>::create(cuda);
-//
-//    mtx->move_to(ell_mtx.get());
-//    dmtx->move_to(dell_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(ell_mtx.get(), dell_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, ConvertToSparsityFbcsrIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(ref);
-//    auto d_sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(cuda);
-//
-//    mtx->convert_to(sparsity_mtx.get());
-//    dmtx->convert_to(d_sparsity_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(sparsity_mtx.get(), d_sparsity_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToSparsityFbcsrIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(ref);
-//    auto d_sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(cuda);
-//
-//    mtx->move_to(sparsity_mtx.get());
-//    dmtx->move_to(d_sparsity_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(sparsity_mtx.get(), d_sparsity_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, CalculateMaxNnzPerRowIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    gko::size_type max_nnz_per_row;
-//    gko::size_type dmax_nnz_per_row;
-//
-//    gko::kernels::reference::fbcsr::calculate_max_nnz_per_row(ref, mtx.get(),
-//                                                            &max_nnz_per_row);
-//    gko::kernels::cuda::fbcsr::calculate_max_nnz_per_row(cuda, dmtx.get(),
-//                                                       &dmax_nnz_per_row);
-//
-//    ASSERT_EQ(max_nnz_per_row, dmax_nnz_per_row);
-//}
-
-
-TEST_F(Fbcsr, ConvertToCooIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto coo_mtx = gko::matrix::Coo<>::create(ref);
-//    auto dcoo_mtx = gko::matrix::Coo<>::create(cuda);
-//
-//    mtx->convert_to(coo_mtx.get());
-//    dmtx->convert_to(dcoo_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToCooIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto coo_mtx = gko::matrix::Coo<>::create(ref);
-//    auto dcoo_mtx = gko::matrix::Coo<>::create(cuda);
-//
-//    mtx->move_to(coo_mtx.get());
-//    dmtx->move_to(dcoo_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, ConvertToSellpIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
-//    auto dsellp_mtx = gko::matrix::Sellp<>::create(cuda);
-//
-//    mtx->convert_to(sellp_mtx.get());
-//    dmtx->convert_to(dsellp_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(sellp_mtx.get(), dsellp_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToSellpIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
-//    auto dsellp_mtx = gko::matrix::Sellp<>::create(cuda);
-//
-//    mtx->move_to(sellp_mtx.get());
-//    dmtx->move_to(dsellp_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(sellp_mtx.get(), dsellp_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, ConvertsEmptyToSellp)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto dempty_mtx = Mtx::create(cuda);
-//    auto dsellp_mtx = gko::matrix::Sellp<>::create(cuda);
-//
-//    dempty_mtx->convert_to(dsellp_mtx.get());
-//
-//    ASSERT_EQ(cuda->copy_val_to_host(dsellp_mtx->get_const_slice_sets()), 0);
-//    ASSERT_FALSE(dsellp_mtx->get_size());
-//}
-
-
-TEST_F(Fbcsr, CalculateTotalColsIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    gko::size_type total_cols;
-//    gko::size_type dtotal_cols;
-//
-//    gko::kernels::reference::fbcsr::calculate_total_cols(
-//        ref, mtx.get(), &total_cols, 2, gko::matrix::default_slice_size);
-//    gko::kernels::cuda::fbcsr::calculate_total_cols(
-//        cuda, dmtx.get(), &dtotal_cols, 2, gko::matrix::default_slice_size);
-//
-//    ASSERT_EQ(total_cols, dtotal_cols);
-//}
-
-
-TEST_F(Fbcsr, CalculatesNonzerosPerRow)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    gko::Array<gko::size_type> row_nnz(ref, mtx->get_size()[0]);
-//    gko::Array<gko::size_type> drow_nnz(cuda, dmtx->get_size()[0]);
-//
-//    gko::kernels::reference::fbcsr::calculate_nonzeros_per_row(ref, mtx.get(),
-//                                                             &row_nnz);
-//    gko::kernels::cuda::fbcsr::calculate_nonzeros_per_row(cuda, dmtx.get(),
-//                                                        &drow_nnz);
-//
-//    GKO_ASSERT_ARRAY_EQ(row_nnz, drow_nnz);
-//}
-
-
-TEST_F(Fbcsr, ConvertToHybridIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Hybrid_type = gko::matrix::Hybrid<>;
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto hybrid_mtx = Hybrid_type::create(
-//        ref, std::make_shared<Hybrid_type::column_limit>(2));
-//    auto dhybrid_mtx = Hybrid_type::create(
-//        cuda, std::make_shared<Hybrid_type::column_limit>(2));
-//
-//    mtx->convert_to(hybrid_mtx.get());
-//    dmtx->convert_to(dhybrid_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToHybridIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Hybrid_type = gko::matrix::Hybrid<>;
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto hybrid_mtx = Hybrid_type::create(
-//        ref, std::make_shared<Hybrid_type::column_limit>(2));
-//    auto dhybrid_mtx = Hybrid_type::create(
-//        cuda, std::make_shared<Hybrid_type::column_limit>(2));
-//
-//    mtx->move_to(hybrid_mtx.get());
-//    dmtx->move_to(dhybrid_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, RecognizeSortedMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>());
-//    bool is_sorted_cuda{};
-//    bool is_sorted_ref{};
-//
-//    is_sorted_ref = mtx->is_sorted_by_column_index();
-//    is_sorted_cuda = dmtx->is_sorted_by_column_index();
-//
-//    ASSERT_EQ(is_sorted_ref, is_sorted_cuda);
-//}
-
-
-TEST_F(Fbcsr, RecognizeUnsortedMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto uns_mtx = gen_unsorted_mtx();
-//    bool is_sorted_cuda{};
-//    bool is_sorted_ref{};
-//
-//    is_sorted_ref = uns_mtx.ref->is_sorted_by_column_index();
-//    is_sorted_cuda = uns_mtx.cuda->is_sorted_by_column_index();
-//
-//    ASSERT_EQ(is_sorted_ref, is_sorted_cuda);
-//}
-
-
-TEST_F(Fbcsr, SortSortedMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>());
-//
-//    mtx->sort_by_column_index();
-//    dmtx->sort_by_column_index();
-//
-//    // Values must be unchanged, therefore, tolerance is `0`
-//    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 0);
-//}
-
-
-TEST_F(Fbcsr, SortUnsortedMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto uns_mtx = gen_unsorted_mtx();
-//
-//    uns_mtx.ref->sort_by_column_index();
-//    uns_mtx.cuda->sort_by_column_index();
-//
-//    // Values must be unchanged, therefore, tolerance is `0`
-//    GKO_ASSERT_MTX_NEAR(uns_mtx.ref, uns_mtx.cuda, 0);
-//}
-
-
-TEST_F(Fbcsr, OneAutomaticalWorksWithDifferentMatrices)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto automatical = std::make_shared<Mtx::automatical>();
-//    auto row_len_limit = std::max(automatical->nvidia_row_len_limit,
-//                                  automatical->amd_row_len_limit);
-//    auto load_balance_mtx = Mtx::create(ref);
-//    auto classical_mtx = Mtx::create(ref);
-//    load_balance_mtx->copy_from(
-//        gen_mtx<Vec>(1, row_len_limit + 1000, row_len_limit + 1));
-//    classical_mtx->copy_from(gen_mtx<Vec>(50, 50, 1));
-//    auto load_balance_mtx_d = Mtx::create(cuda);
-//    auto classical_mtx_d = Mtx::create(cuda);
-//    load_balance_mtx_d->copy_from(load_balance_mtx.get());
-//    classical_mtx_d->copy_from(classical_mtx.get());
-//
-//    load_balance_mtx_d->set_strategy(automatical);
-//    classical_mtx_d->set_strategy(automatical);
-//
-//    EXPECT_EQ("load_balance", load_balance_mtx_d->get_strategy()->get_name());
-//    EXPECT_EQ("classical", classical_mtx_d->get_strategy()->get_name());
-//    ASSERT_NE(load_balance_mtx_d->get_strategy().get(),
-//              classical_mtx_d->get_strategy().get());
-//}
-
-
-TEST_F(Fbcsr, ExtractDiagonalIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>());
-//
-//    auto diag = mtx->extract_diagonal();
-//    auto ddiag = dmtx->extract_diagonal();
-//
-//    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-//}
-
-
-TEST_F(Fbcsr, InplaceAbsoluteMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>(cuda));
-//
-//    mtx->compute_absolute_inplace();
-//    dmtx->compute_absolute_inplace();
-//
-//    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, OutplaceAbsoluteMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>(cuda));
-//
-//    auto abs_mtx = mtx->compute_absolute();
-//    auto dabs_mtx = dmtx->compute_absolute();
-//
-//    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(cuda));
-//
-//    complex_mtx->compute_absolute_inplace();
-//    complex_dmtx->compute_absolute_inplace();
-//
-//    GKO_ASSERT_MTX_NEAR(complex_mtx, complex_dmtx, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(cuda));
-//
-//    auto abs_mtx = complex_mtx->compute_absolute();
-//    auto dabs_mtx = complex_dmtx->compute_absolute();
-//
-//    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
-//}
-
-
-}  // namespace
diff --git a/dpcpp/matrix/fbcsr_kernels.dp.cpp b/dpcpp/matrix/fbcsr_kernels.dp.cpp
index 749ef7c3d2d..17c44830b9a 100644
--- a/dpcpp/matrix/fbcsr_kernels.dp.cpp
+++ b/dpcpp/matrix/fbcsr_kernels.dp.cpp
@@ -89,82 +89,12 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void spgemm_insert_row(unordered_set<IndexType> &cols,
-                       const matrix::Fbcsr<ValueType, IndexType> *c,
-                       size_type row) GKO_NOT_IMPLEMENTED;
-
-
-template <typename ValueType, typename IndexType>
-void spgemm_insert_row2(unordered_set<IndexType> &cols,
-                        const matrix::Fbcsr<ValueType, IndexType> *a,
-                        const matrix::Fbcsr<ValueType, IndexType> *b,
-                        size_type row) GKO_NOT_IMPLEMENTED;
-
-
-template <typename ValueType, typename IndexType>
-void spgemm_accumulate_row(map<IndexType, ValueType> &cols,
-                           const matrix::Fbcsr<ValueType, IndexType> *c,
-                           ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
-
-
-template <typename ValueType, typename IndexType>
-void spgemm_accumulate_row2(map<IndexType, ValueType> &cols,
-                            const matrix::Fbcsr<ValueType, IndexType> *a,
-                            const matrix::Fbcsr<ValueType, IndexType> *b,
-                            ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
-
-
-template <typename ValueType, typename IndexType>
-void spgemm(std::shared_ptr<const DpcppExecutor> exec,
-            const matrix::Fbcsr<ValueType, IndexType> *a,
-            const matrix::Fbcsr<ValueType, IndexType> *b,
-            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEMM_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void advanced_spgemm(std::shared_ptr<const DpcppExecutor> exec,
-                     const matrix::Dense<ValueType> *alpha,
-                     const matrix::Fbcsr<ValueType, IndexType> *a,
-                     const matrix::Fbcsr<ValueType, IndexType> *b,
-                     const matrix::Dense<ValueType> *beta,
-                     const matrix::Fbcsr<ValueType, IndexType> *d,
-                     matrix::Fbcsr<ValueType, IndexType> *c)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void spgeam(std::shared_ptr<const DpcppExecutor> exec,
-            const matrix::Dense<ValueType> *alpha,
-            const matrix::Fbcsr<ValueType, IndexType> *a,
-            const matrix::Dense<ValueType> *beta,
-            const matrix::Fbcsr<ValueType, IndexType> *b,
-            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEAM_KERNEL);
-
-
 template <typename IndexType>
 void convert_row_ptrs_to_idxs(std::shared_ptr<const DpcppExecutor> exec,
                               const IndexType *ptrs, size_type num_rows,
                               IndexType *idxs) GKO_NOT_IMPLEMENTED;
 
 
-template <typename ValueType, typename IndexType>
-void convert_to_coo(std::shared_ptr<const DpcppExecutor> exec,
-                    const matrix::Fbcsr<ValueType, IndexType> *source,
-                    matrix::Coo<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *source,
@@ -184,26 +114,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
-                      const matrix::Fbcsr<ValueType, IndexType> *source,
-                      matrix::Sellp<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void convert_to_ell(std::shared_ptr<const DpcppExecutor> exec,
-                    const matrix::Fbcsr<ValueType, IndexType> *source,
-                    matrix::Ell<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL);
-
-
 template <typename ValueType, typename IndexType, typename UnaryOperator>
 inline void convert_fbcsr_to_csc(size_type num_rows, const IndexType *row_ptrs,
                                  const IndexType *col_idxs,
@@ -239,16 +149,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
-                          const matrix::Fbcsr<ValueType, IndexType> *source,
-                          size_type *result, size_type stride_factor,
-                          size_type slice_size) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void calculate_max_nnz_per_row(
     std::shared_ptr<const DpcppExecutor> exec,
@@ -259,54 +159,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void convert_to_hybrid(std::shared_ptr<const DpcppExecutor> exec,
-                       const matrix::Fbcsr<ValueType, IndexType> *source,
-                       matrix::Hybrid<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL);
-
-
-// template <typename IndexType>
-// void invert_permutation(std::shared_ptr<const DpcppExecutor> exec,
-//                         size_type size, const IndexType *permutation_indices,
-//                         IndexType *inv_permutation) GKO_NOT_IMPLEMENTED;
-
-// GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void row_permute(
-    std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
-    const matrix::Fbcsr<ValueType, IndexType> *orig,
-    matrix::Fbcsr<ValueType, IndexType> *row_permuted) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_row_permute(
-    std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
-    const matrix::Fbcsr<ValueType, IndexType> *orig,
-    matrix::Fbcsr<ValueType, IndexType> *row_permuted) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(
-    std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
-    const matrix::Fbcsr<ValueType, IndexType> *orig,
-    matrix::Fbcsr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void calculate_nonzeros_per_row(
     std::shared_ptr<const DpcppExecutor> exec,
diff --git a/hip/matrix/fbcsr_kernels.hip.cpp b/hip/matrix/fbcsr_kernels.hip.cpp
index e6dc61946cb..891e644b25c 100644
--- a/hip/matrix/fbcsr_kernels.hip.cpp
+++ b/hip/matrix/fbcsr_kernels.hip.cpp
@@ -42,12 +42,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/ell.hpp>
-#include <ginkgo/core/matrix/hybrid.hpp>
-#include <ginkgo/core/matrix/sellp.hpp>
 
 
 #include "core/components/fill_array.hpp"
@@ -97,274 +93,12 @@ using compiled_kernels = syn::value_list<int, 3, 4, 6, 7, 8, 12, 14>;
 using classical_kernels =
     syn::value_list<int, config::warp_size, 32, 16, 8, 4, 2, 1>;
 
-using spgeam_kernels =
-    syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
-
-
-namespace host_kernel {
-
-
-template <int items_per_thread, typename ValueType, typename IndexType>
-void merge_path_spmv(
-    syn::value_list<int, items_per_thread>,
-    std::shared_ptr<const HipExecutor> exec,
-    const matrix::Fbcsr<ValueType, IndexType> *a,
-    const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c,
-    const matrix::Dense<ValueType> *alpha = nullptr,
-    const matrix::Dense<ValueType> *beta = nullptr) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const IndexType total = a->get_size()[0] + a->get_num_stored_elements();
-//    const IndexType grid_num =
-//        ceildiv(total, spmv_block_size * items_per_thread);
-//    const dim3 grid(grid_num);
-//    const dim3 block(spmv_block_size);
-//    Array<IndexType> row_out(exec, grid_num);
-//    Array<ValueType> val_out(exec, grid_num);
-//
-//    for (IndexType column_id = 0; column_id < b->get_size()[1]; column_id++) {
-//        if (alpha == nullptr && beta == nullptr) {
-//            const auto b_vals = b->get_const_values() + column_id;
-//            auto c_vals = c->get_values() + column_id;
-//            hipLaunchKernelGGL(
-//                HIP_KERNEL_NAME(
-//                    kernel::abstract_merge_path_spmv<items_per_thread>),
-//                dim3(grid), dim3(block), 0, 0,
-//                static_cast<IndexType>(a->get_size()[0]),
-//                as_hip_type(a->get_const_values()), a->get_const_col_idxs(),
-//                as_hip_type(a->get_const_row_ptrs()),
-//                as_hip_type(a->get_const_srow()), as_hip_type(b_vals),
-//                b->get_stride(), as_hip_type(c_vals), c->get_stride(),
-//                as_hip_type(row_out.get_data()),
-//                as_hip_type(val_out.get_data()));
-//            hipLaunchKernelGGL(kernel::abstract_reduce, dim3(1),
-//                               dim3(spmv_block_size), 0, 0, grid_num,
-//                               as_hip_type(val_out.get_data()),
-//                               as_hip_type(row_out.get_data()),
-//                               as_hip_type(c_vals), c->get_stride());
-//
-//        } else if (alpha != nullptr && beta != nullptr) {
-//            const auto b_vals = b->get_const_values() + column_id;
-//            auto c_vals = c->get_values() + column_id;
-//            hipLaunchKernelGGL(
-//                HIP_KERNEL_NAME(
-//                    kernel::abstract_merge_path_spmv<items_per_thread>),
-//                dim3(grid), dim3(block), 0, 0,
-//                static_cast<IndexType>(a->get_size()[0]),
-//                as_hip_type(alpha->get_const_values()),
-//                as_hip_type(a->get_const_values()), a->get_const_col_idxs(),
-//                as_hip_type(a->get_const_row_ptrs()),
-//                as_hip_type(a->get_const_srow()), as_hip_type(b_vals),
-//                b->get_stride(), as_hip_type(beta->get_const_values()),
-//                as_hip_type(c_vals), c->get_stride(),
-//                as_hip_type(row_out.get_data()),
-//                as_hip_type(val_out.get_data()));
-//            hipLaunchKernelGGL(kernel::abstract_reduce, dim3(1),
-//                               dim3(spmv_block_size), 0, 0, grid_num,
-//                               as_hip_type(val_out.get_data()),
-//                               as_hip_type(row_out.get_data()),
-//                               as_hip_type(alpha->get_const_values()),
-//                               as_hip_type(c_vals), c->get_stride());
-//        } else {
-//            GKO_KERNEL_NOT_FOUND;
-//        }
-//    }
-//}
-
-GKO_ENABLE_IMPLEMENTATION_SELECTION(select_merge_path_spmv, merge_path_spmv);
-
-
-template <typename ValueType, typename IndexType>
-int compute_items_per_thread(std::shared_ptr<const HipExecutor> exec)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//#if GINKGO_HIP_PLATFORM_NVCC
-//
-//
-//    const int version =
-//        (exec->get_major_version() << 4) + exec->get_minor_version();
-//    // The num_item is decided to make the occupancy 100%
-//    // TODO: Extend this list when new GPU is released
-//    //       Tune this parameter
-//    // 128 threads/block the number of items per threads
-//    // 3.0 3.5: 6
-//    // 3.7: 14
-//    // 5.0, 5.3, 6.0, 6.2: 8
-//    // 5.2, 6.1, 7.0: 12
-//    int num_item = 6;
-//    switch (version) {
-//    case 0x50:
-//    case 0x53:
-//    case 0x60:
-//    case 0x62:
-//        num_item = 8;
-//        break;
-//    case 0x52:
-//    case 0x61:
-//    case 0x70:
-//        num_item = 12;
-//        break;
-//    case 0x37:
-//        num_item = 14;
-//    }
-//
-//
-//#else
-//
-//
-//    // HIP uses the minimal num_item to make the code work correctly.
-//    // TODO: this parameter should be tuned.
-//    int num_item = 6;
-//
-//
-//#endif  // GINKGO_HIP_PLATFORM_NVCC
-//
-//
-//    // Ensure that the following is satisfied:
-//    // sizeof(IndexType) + sizeof(ValueType)
-//    // <= items_per_thread * sizeof(IndexType)
-//    constexpr int minimal_num =
-//        ceildiv(sizeof(IndexType) + sizeof(ValueType), sizeof(IndexType));
-//    int items_per_thread = num_item * 4 / sizeof(IndexType);
-//    return std::max(minimal_num, items_per_thread);
-//}
-
-
-template <int subwarp_size, typename ValueType, typename IndexType>
-void classical_spmv(
-    syn::value_list<int, subwarp_size>, std::shared_ptr<const HipExecutor> exec,
-    const matrix::Fbcsr<ValueType, IndexType> *a,
-    const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c,
-    const matrix::Dense<ValueType> *alpha = nullptr,
-    const matrix::Dense<ValueType> *beta = nullptr) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto nwarps = exec->get_num_warps_per_sm() *
-//                        exec->get_num_multiprocessor() * classical_overweight;
-//    const auto gridx =
-//        std::min(ceildiv(a->get_size()[0], spmv_block_size / subwarp_size),
-//                 int64(nwarps / warps_in_block));
-//    const dim3 grid(gridx, b->get_size()[1]);
-//    const dim3 block(spmv_block_size);
-//
-//    if (alpha == nullptr && beta == nullptr) {
-//        hipLaunchKernelGGL(
-//            HIP_KERNEL_NAME(kernel::abstract_classical_spmv<subwarp_size>),
-//            dim3(grid), dim3(block), 0, 0, a->get_size()[0],
-//            as_hip_type(a->get_const_values()), a->get_const_col_idxs(),
-//            as_hip_type(a->get_const_row_ptrs()),
-//            as_hip_type(b->get_const_values()), b->get_stride(),
-//            as_hip_type(c->get_values()), c->get_stride());
-//
-//    } else if (alpha != nullptr && beta != nullptr) {
-//        hipLaunchKernelGGL(
-//            HIP_KERNEL_NAME(kernel::abstract_classical_spmv<subwarp_size>),
-//            dim3(grid), dim3(block), 0, 0, a->get_size()[0],
-//            as_hip_type(alpha->get_const_values()),
-//            as_hip_type(a->get_const_values()), a->get_const_col_idxs(),
-//            as_hip_type(a->get_const_row_ptrs()),
-//            as_hip_type(b->get_const_values()), b->get_stride(),
-//            as_hip_type(beta->get_const_values()),
-//            as_hip_type(c->get_values()), c->get_stride());
-//    } else {
-//        GKO_KERNEL_NOT_FOUND;
-//    }
-//}
-
-GKO_ENABLE_IMPLEMENTATION_SELECTION(select_classical_spmv, classical_spmv);
-
-
-}  // namespace host_kernel
-
 
 template <typename ValueType, typename IndexType>
 void spmv(std::shared_ptr<const HipExecutor> exec,
           const matrix::Fbcsr<ValueType, IndexType> *a,
           const matrix::Dense<ValueType> *b,
           matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    if (a->get_strategy()->get_name() == "load_balance") {
-//        components::fill_array(exec, c->get_values(),
-//                               c->get_num_stored_elements(),
-//                               zero<ValueType>());
-//        const IndexType nwarps = a->get_num_srow_elements();
-//        if (nwarps > 0) {
-//            const dim3 fbcsr_block(config::warp_size, warps_in_block, 1);
-//            const dim3 fbcsr_grid(ceildiv(nwarps, warps_in_block),
-//                                b->get_size()[1]);
-//            hipLaunchKernelGGL(
-//                kernel::abstract_spmv, dim3(fbcsr_grid), dim3(fbcsr_block), 0,
-//                0, nwarps, static_cast<IndexType>(a->get_size()[0]),
-//                as_hip_type(a->get_const_values()), a->get_const_col_idxs(),
-//                as_hip_type(a->get_const_row_ptrs()),
-//                as_hip_type(a->get_const_srow()),
-//                as_hip_type(b->get_const_values()),
-//                as_hip_type(b->get_stride()), as_hip_type(c->get_values()),
-//                as_hip_type(c->get_stride()));
-//        } else {
-//            GKO_NOT_SUPPORTED(nwarps);
-//        }
-//    } else if (a->get_strategy()->get_name() == "merge_path") {
-//        int items_per_thread =
-//            host_kernel::compute_items_per_thread<ValueType, IndexType>(exec);
-//        host_kernel::select_merge_path_spmv(
-//            compiled_kernels(),
-//            [&items_per_thread](int compiled_info) {
-//                return items_per_thread == compiled_info;
-//            },
-//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c);
-//    } else if (a->get_strategy()->get_name() == "classical") {
-//        IndexType max_length_per_row = 0;
-//        using Tfbcsr = matrix::Fbcsr<ValueType, IndexType>;
-//        if (auto strategy =
-//                std::dynamic_pointer_cast<const typename Tfbcsr::classical>(
-//                    a->get_strategy())) {
-//            max_length_per_row = strategy->get_max_length_per_row();
-//        } else if (auto strategy = std::dynamic_pointer_cast<
-//                       const typename Tfbcsr::automatical>(a->get_strategy()))
-//                       {
-//            max_length_per_row = strategy->get_max_length_per_row();
-//        } else {
-//            GKO_NOT_SUPPORTED(a->get_strategy());
-//        }
-//        host_kernel::select_classical_spmv(
-//            classical_kernels(),
-//            [&max_length_per_row](int compiled_info) {
-//                return max_length_per_row >= compiled_info;
-//            },
-//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c);
-//    } else if (a->get_strategy()->get_name() == "sparselib" ||
-//               a->get_strategy()->get_name() == "cusparse") {
-//        if (hipsparse::is_supported<ValueType, IndexType>::value) {
-//            // TODO: add implementation for int64 and multiple RHS
-//            auto handle = exec->get_hipsparse_handle();
-//            auto descr = hipsparse::create_mat_descr();
-//            {
-//                hipsparse::pointer_mode_guard pm_guard(handle);
-//                auto row_ptrs = a->get_const_row_ptrs();
-//                auto col_idxs = a->get_const_col_idxs();
-//                auto alpha = one<ValueType>();
-//                auto beta = zero<ValueType>();
-//                if (b->get_stride() != 1 || c->get_stride() != 1) {
-//                    GKO_NOT_IMPLEMENTED;
-//                }
-//                hipsparse::spmv(handle, HIPSPARSE_OPERATION_NON_TRANSPOSE,
-//                                a->get_size()[0], a->get_size()[1],
-//                                a->get_num_stored_elements(), &alpha, descr,
-//                                a->get_const_values(), row_ptrs, col_idxs,
-//                                b->get_const_values(), &beta,
-//                                c->get_values());
-//            }
-//            hipsparse::destroy(descr);
-//        } else {
-//            GKO_NOT_IMPLEMENTED;
-//        }
-//    } else {
-//        GKO_NOT_IMPLEMENTED;
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPMV_KERNEL);
 
@@ -376,403 +110,15 @@ void advanced_spmv(std::shared_ptr<const HipExecutor> exec,
                    const matrix::Dense<ValueType> *b,
                    const matrix::Dense<ValueType> *beta,
                    matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    if (a->get_strategy()->get_name() == "load_balance") {
-//        dense::scale(exec, beta, c);
-//
-//        const IndexType nwarps = a->get_num_srow_elements();
-//
-//        if (nwarps > 0) {
-//            const dim3 fbcsr_block(config::warp_size, warps_in_block, 1);
-//            const dim3 fbcsr_grid(ceildiv(nwarps, warps_in_block),
-//                                b->get_size()[1]);
-//            hipLaunchKernelGGL(
-//                kernel::abstract_spmv, dim3(fbcsr_grid), dim3(fbcsr_block), 0,
-//                0, nwarps, static_cast<IndexType>(a->get_size()[0]),
-//                as_hip_type(alpha->get_const_values()),
-//                as_hip_type(a->get_const_values()), a->get_const_col_idxs(),
-//                as_hip_type(a->get_const_row_ptrs()),
-//                as_hip_type(a->get_const_srow()),
-//                as_hip_type(b->get_const_values()),
-//                as_hip_type(b->get_stride()), as_hip_type(c->get_values()),
-//                as_hip_type(c->get_stride()));
-//        } else {
-//            GKO_NOT_SUPPORTED(nwarps);
-//        }
-//    } else if (a->get_strategy()->get_name() == "sparselib" ||
-//               a->get_strategy()->get_name() == "cusparse") {
-//        if (hipsparse::is_supported<ValueType, IndexType>::value) {
-//            // TODO: add implementation for int64 and multiple RHS
-//            auto descr = hipsparse::create_mat_descr();
-//
-//            auto row_ptrs = a->get_const_row_ptrs();
-//            auto col_idxs = a->get_const_col_idxs();
-//
-//            if (b->get_stride() != 1 || c->get_stride() != 1)
-//                GKO_NOT_IMPLEMENTED;
-//
-//            hipsparse::spmv(exec->get_hipsparse_handle(),
-//                            HIPSPARSE_OPERATION_NON_TRANSPOSE,
-//                            a->get_size()[0], a->get_size()[1],
-//                            a->get_num_stored_elements(),
-//                            alpha->get_const_values(), descr,
-//                            a->get_const_values(), row_ptrs, col_idxs,
-//                            b->get_const_values(), beta->get_const_values(),
-//                            c->get_values());
-//
-//            hipsparse::destroy(descr);
-//        } else {
-//            GKO_NOT_IMPLEMENTED;
-//        }
-//    } else if (a->get_strategy()->get_name() == "classical") {
-//        IndexType max_length_per_row = 0;
-//        using Tfbcsr = matrix::Fbcsr<ValueType, IndexType>;
-//        if (auto strategy =
-//                std::dynamic_pointer_cast<const typename Tfbcsr::classical>(
-//                    a->get_strategy())) {
-//            max_length_per_row = strategy->get_max_length_per_row();
-//        } else if (auto strategy = std::dynamic_pointer_cast<
-//                       const typename Tfbcsr::automatical>(a->get_strategy()))
-//                       {
-//            max_length_per_row = strategy->get_max_length_per_row();
-//        } else {
-//            GKO_NOT_SUPPORTED(a->get_strategy());
-//        }
-//        host_kernel::select_classical_spmv(
-//            classical_kernels(),
-//            [&max_length_per_row](int compiled_info) {
-//                return max_length_per_row >= compiled_info;
-//            },
-//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c, alpha,
-//            beta);
-//    } else if (a->get_strategy()->get_name() == "merge_path") {
-//        int items_per_thread =
-//            host_kernel::compute_items_per_thread<ValueType, IndexType>(exec);
-//        host_kernel::select_merge_path_spmv(
-//            compiled_kernels(),
-//            [&items_per_thread](int compiled_info) {
-//                return items_per_thread == compiled_info;
-//            },
-//            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c, alpha,
-//            beta);
-//    } else {
-//        GKO_NOT_IMPLEMENTED;
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void spgemm(std::shared_ptr<const HipExecutor> exec,
-            const matrix::Fbcsr<ValueType, IndexType> *a,
-            const matrix::Fbcsr<ValueType, IndexType> *b,
-            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    if (hipsparse::is_supported<ValueType, IndexType>::value) {
-//        auto handle = exec->get_hipsparse_handle();
-//        hipsparse::pointer_mode_guard pm_guard(handle);
-//        auto a_descr = hipsparse::create_mat_descr();
-//        auto b_descr = hipsparse::create_mat_descr();
-//        auto c_descr = hipsparse::create_mat_descr();
-//        auto d_descr = hipsparse::create_mat_descr();
-//        auto info = hipsparse::create_spgemm_info();
-//
-//        auto alpha = one<ValueType>();
-//        auto a_nnz = static_cast<IndexType>(a->get_num_stored_elements());
-//        auto a_vals = a->get_const_values();
-//        auto a_row_ptrs = a->get_const_row_ptrs();
-//        auto a_col_idxs = a->get_const_col_idxs();
-//        auto b_nnz = static_cast<IndexType>(b->get_num_stored_elements());
-//        auto b_vals = b->get_const_values();
-//        auto b_row_ptrs = b->get_const_row_ptrs();
-//        auto b_col_idxs = b->get_const_col_idxs();
-//        auto null_value = static_cast<ValueType *>(nullptr);
-//        auto null_index = static_cast<IndexType *>(nullptr);
-//        auto zero_nnz = IndexType{};
-//        auto m = static_cast<IndexType>(a->get_size()[0]);
-//        auto n = static_cast<IndexType>(b->get_size()[1]);
-//        auto k = static_cast<IndexType>(a->get_size()[1]);
-//        auto c_row_ptrs = c->get_row_ptrs();
-//        matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
-//        auto &c_col_idxs_array = c_builder.get_col_idx_array();
-//        auto &c_vals_array = c_builder.get_value_array();
-//
-//        // allocate buffer
-//        size_type buffer_size{};
-//        hipsparse::spgemm_buffer_size(
-//            handle, m, n, k, &alpha, a_descr, a_nnz, a_row_ptrs, a_col_idxs,
-//            b_descr, b_nnz, b_row_ptrs, b_col_idxs, null_value, d_descr,
-//            zero_nnz, null_index, null_index, info, buffer_size);
-//        Array<char> buffer_array(exec, buffer_size);
-//        auto buffer = buffer_array.get_data();
-//
-//        // count nnz
-//        IndexType c_nnz{};
-//        hipsparse::spgemm_nnz(
-//            handle, m, n, k, a_descr, a_nnz, a_row_ptrs, a_col_idxs, b_descr,
-//            b_nnz, b_row_ptrs, b_col_idxs, d_descr, zero_nnz, null_index,
-//            null_index, c_descr, c_row_ptrs, &c_nnz, info, buffer);
-//
-//        // accumulate non-zeros
-//        c_col_idxs_array.resize_and_reset(c_nnz);
-//        c_vals_array.resize_and_reset(c_nnz);
-//        auto c_col_idxs = c_col_idxs_array.get_data();
-//        auto c_vals = c_vals_array.get_data();
-//        hipsparse::spgemm(handle, m, n, k, &alpha, a_descr, a_nnz, a_vals,
-//                          a_row_ptrs, a_col_idxs, b_descr, b_nnz, b_vals,
-//                          b_row_ptrs, b_col_idxs, null_value, d_descr,
-//                          zero_nnz, null_value, null_index, null_index,
-//                          c_descr, c_vals, c_row_ptrs, c_col_idxs, info,
-//                          buffer);
-//
-//        hipsparse::destroy_spgemm_info(info);
-//        hipsparse::destroy(d_descr);
-//        hipsparse::destroy(c_descr);
-//        hipsparse::destroy(b_descr);
-//        hipsparse::destroy(a_descr);
-//    } else {
-//        GKO_NOT_IMPLEMENTED;
-//    }
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEMM_KERNEL);
-
-
-namespace {
-
-
-template <int subwarp_size, typename ValueType, typename IndexType>
-void spgeam(syn::value_list<int, subwarp_size>,
-            std::shared_ptr<const HipExecutor> exec, const ValueType *alpha,
-            const IndexType *a_row_ptrs, const IndexType *a_col_idxs,
-            const ValueType *a_vals, const ValueType *beta,
-            const IndexType *b_row_ptrs, const IndexType *b_col_idxs,
-            const ValueType *b_vals,
-            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto m = static_cast<IndexType>(c->get_size()[0]);
-//    auto c_row_ptrs = c->get_row_ptrs();
-//    // count nnz for alpha * A + beta * B
-//    auto subwarps_per_block = default_block_size / subwarp_size;
-//    auto num_blocks = ceildiv(m, subwarps_per_block);
-//    hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel::spgeam_nnz<subwarp_size>),
-//                       dim3(num_blocks), dim3(default_block_size), 0, 0,
-//                       a_row_ptrs, a_col_idxs, b_row_ptrs, b_col_idxs, m,
-//                       c_row_ptrs);
-//
-//    // build row pointers
-//    components::prefix_sum(exec, c_row_ptrs, m + 1);
-//
-//    // accumulate non-zeros for alpha * A + beta * B
-//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
-//    auto c_nnz = exec->copy_val_to_host(c_row_ptrs + m);
-//    c_builder.get_col_idx_array().resize_and_reset(c_nnz);
-//    c_builder.get_value_array().resize_and_reset(c_nnz);
-//    auto c_col_idxs = c->get_col_idxs();
-//    auto c_vals = c->get_values();
-//    hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel::spgeam<subwarp_size>),
-//                       dim3(num_blocks), dim3(default_block_size), 0, 0,
-//                       as_hip_type(alpha), a_row_ptrs, a_col_idxs,
-//                       as_hip_type(a_vals), as_hip_type(beta), b_row_ptrs,
-//                       b_col_idxs, as_hip_type(b_vals), m, c_row_ptrs,
-//                       c_col_idxs, as_hip_type(c_vals));
-//}
-
-GKO_ENABLE_IMPLEMENTATION_SELECTION(select_spgeam, spgeam);
-
-
-}  // namespace
-
-
-template <typename ValueType, typename IndexType>
-void advanced_spgemm(std::shared_ptr<const HipExecutor> exec,
-                     const matrix::Dense<ValueType> *alpha,
-                     const matrix::Fbcsr<ValueType, IndexType> *a,
-                     const matrix::Fbcsr<ValueType, IndexType> *b,
-                     const matrix::Dense<ValueType> *beta,
-                     const matrix::Fbcsr<ValueType, IndexType> *d,
-                     matrix::Fbcsr<ValueType, IndexType> *c)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    if (hipsparse::is_supported<ValueType, IndexType>::value) {
-//        auto handle = exec->get_hipsparse_handle();
-//        hipsparse::pointer_mode_guard pm_guard(handle);
-//        auto a_descr = hipsparse::create_mat_descr();
-//        auto b_descr = hipsparse::create_mat_descr();
-//        auto c_descr = hipsparse::create_mat_descr();
-//        auto d_descr = hipsparse::create_mat_descr();
-//        auto info = hipsparse::create_spgemm_info();
-//
-//        auto a_nnz = static_cast<IndexType>(a->get_num_stored_elements());
-//        auto a_vals = a->get_const_values();
-//        auto a_row_ptrs = a->get_const_row_ptrs();
-//        auto a_col_idxs = a->get_const_col_idxs();
-//        auto b_nnz = static_cast<IndexType>(b->get_num_stored_elements());
-//        auto b_vals = b->get_const_values();
-//        auto b_row_ptrs = b->get_const_row_ptrs();
-//        auto b_col_idxs = b->get_const_col_idxs();
-//        auto d_vals = d->get_const_values();
-//        auto d_row_ptrs = d->get_const_row_ptrs();
-//        auto d_col_idxs = d->get_const_col_idxs();
-//        auto null_value = static_cast<ValueType *>(nullptr);
-//        auto null_index = static_cast<IndexType *>(nullptr);
-//        auto one_value = one<ValueType>();
-//        auto m = static_cast<IndexType>(a->get_size()[0]);
-//        auto n = static_cast<IndexType>(b->get_size()[1]);
-//        auto k = static_cast<IndexType>(a->get_size()[1]);
-//
-//        // allocate buffer
-//        size_type buffer_size{};
-//        hipsparse::spgemm_buffer_size(
-//            handle, m, n, k, &one_value, a_descr, a_nnz, a_row_ptrs,
-//            a_col_idxs, b_descr, b_nnz, b_row_ptrs, b_col_idxs, null_value,
-//            d_descr, IndexType{}, null_index, null_index, info, buffer_size);
-//        Array<char> buffer_array(exec, buffer_size);
-//        auto buffer = buffer_array.get_data();
-//
-//        // count nnz
-//        Array<IndexType> c_tmp_row_ptrs_array(exec, m + 1);
-//        auto c_tmp_row_ptrs = c_tmp_row_ptrs_array.get_data();
-//        IndexType c_nnz{};
-//        hipsparse::spgemm_nnz(
-//            handle, m, n, k, a_descr, a_nnz, a_row_ptrs, a_col_idxs, b_descr,
-//            b_nnz, b_row_ptrs, b_col_idxs, d_descr, IndexType{}, null_index,
-//            null_index, c_descr, c_tmp_row_ptrs, &c_nnz, info, buffer);
-//
-//        // accumulate non-zeros for A * B
-//        Array<IndexType> c_tmp_col_idxs_array(exec, c_nnz);
-//        Array<ValueType> c_tmp_vals_array(exec, c_nnz);
-//        auto c_tmp_col_idxs = c_tmp_col_idxs_array.get_data();
-//        auto c_tmp_vals = c_tmp_vals_array.get_data();
-//        hipsparse::spgemm(handle, m, n, k, &one_value, a_descr, a_nnz, a_vals,
-//                          a_row_ptrs, a_col_idxs, b_descr, b_nnz, b_vals,
-//                          b_row_ptrs, b_col_idxs, null_value, d_descr,
-//                          IndexType{}, null_value, null_index, null_index,
-//                          c_descr, c_tmp_vals, c_tmp_row_ptrs, c_tmp_col_idxs,
-//                          info, buffer);
-//
-//        // destroy hipsparse context
-//        hipsparse::destroy_spgemm_info(info);
-//        hipsparse::destroy(d_descr);
-//        hipsparse::destroy(c_descr);
-//        hipsparse::destroy(b_descr);
-//        hipsparse::destroy(a_descr);
-//
-//        auto total_nnz = c_nnz + d->get_num_stored_elements();
-//        auto nnz_per_row = total_nnz / m;
-//        select_spgeam(
-//            spgeam_kernels(),
-//            [&](int compiled_subwarp_size) {
-//                return compiled_subwarp_size >= nnz_per_row ||
-//                       compiled_subwarp_size == config::warp_size;
-//            },
-//            syn::value_list<int>(), syn::type_list<>(), exec,
-//            alpha->get_const_values(), c_tmp_row_ptrs, c_tmp_col_idxs,
-//            c_tmp_vals, beta->get_const_values(), d_row_ptrs, d_col_idxs,
-//            d_vals, c);
-//    } else {
-//        GKO_NOT_IMPLEMENTED;
-//    }
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void spgeam(std::shared_ptr<const DefaultExecutor> exec,
-            const matrix::Dense<ValueType> *alpha,
-            const matrix::Fbcsr<ValueType, IndexType> *a,
-            const matrix::Dense<ValueType> *beta,
-            const matrix::Fbcsr<ValueType, IndexType> *b,
-            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto total_nnz =
-//        a->get_num_stored_elements() + b->get_num_stored_elements();
-//    auto nnz_per_row = total_nnz / a->get_size()[0];
-//    select_spgeam(
-//        spgeam_kernels(),
-//        [&](int compiled_subwarp_size) {
-//            return compiled_subwarp_size >= nnz_per_row ||
-//                   compiled_subwarp_size == config::warp_size;
-//        },
-//        syn::value_list<int>(), syn::type_list<>(), exec,
-//        alpha->get_const_values(), a->get_const_row_ptrs(),
-//        a->get_const_col_idxs(), a->get_const_values(),
-//        beta->get_const_values(), b->get_const_row_ptrs(),
-//        b->get_const_col_idxs(), b->get_const_values(), c);
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEAM_KERNEL);
-
-
-template <typename IndexType>
-void convert_row_ptrs_to_idxs(std::shared_ptr<const HipExecutor> exec,
-                              const IndexType *ptrs, size_type num_rows,
-                              IndexType *idxs) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto grid_dim = ceildiv(num_rows, default_block_size);
-//
-//    hipLaunchKernelGGL(kernel::convert_row_ptrs_to_idxs, dim3(grid_dim),
-//                       dim3(default_block_size), 0, 0, num_rows,
-//                       as_hip_type(ptrs), as_hip_type(idxs));
-//}
-
-
-template <typename ValueType, typename IndexType>
-void convert_to_coo(std::shared_ptr<const HipExecutor> exec,
-                    const matrix::Fbcsr<ValueType, IndexType> *source,
-                    matrix::Coo<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = result->get_size()[0];
-//
-//    auto row_idxs = result->get_row_idxs();
-//    const auto source_row_ptrs = source->get_const_row_ptrs();
-//
-//    convert_row_ptrs_to_idxs(exec, source_row_ptrs, num_rows, row_idxs);
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const HipExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *source,
                       matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto num_rows = result->get_size()[0];
-//    const auto num_cols = result->get_size()[1];
-//    const auto stride = result->get_stride();
-//    const auto row_ptrs = source->get_const_row_ptrs();
-//    const auto col_idxs = source->get_const_col_idxs();
-//    const auto vals = source->get_const_values();
-//
-//    const dim3 block_size(config::warp_size,
-//                          config::max_block_size / config::warp_size, 1);
-//    const dim3 init_grid_dim(ceildiv(num_cols, block_size.x),
-//                             ceildiv(num_rows, block_size.y), 1);
-//    hipLaunchKernelGGL(kernel::initialize_zero_dense, dim3(init_grid_dim),
-//                       dim3(block_size), 0, 0, num_rows, num_cols, stride,
-//                       as_hip_type(result->get_values()));
-//
-//    auto grid_dim = ceildiv(num_rows, default_block_size);
-//    hipLaunchKernelGGL(
-//        kernel::fill_in_dense, dim3(grid_dim), dim3(default_block_size), 0, 0,
-//        num_rows, as_hip_type(row_ptrs), as_hip_type(col_idxs),
-//        as_hip_type(vals), stride, as_hip_type(result->get_values()));
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
@@ -788,190 +134,10 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void convert_to_sellp(std::shared_ptr<const HipExecutor> exec,
-                      const matrix::Fbcsr<ValueType, IndexType> *source,
-                      matrix::Sellp<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto num_rows = result->get_size()[0];
-//    const auto num_cols = result->get_size()[1];
-//
-//    auto result_values = result->get_values();
-//    auto result_col_idxs = result->get_col_idxs();
-//    auto slice_lengths = result->get_slice_lengths();
-//    auto slice_sets = result->get_slice_sets();
-//
-//    const auto slice_size = (result->get_slice_size() == 0)
-//                                ? matrix::default_slice_size
-//                                : result->get_slice_size();
-//    const auto stride_factor = (result->get_stride_factor() == 0)
-//                                   ? matrix::default_stride_factor
-//                                   : result->get_stride_factor();
-//    const int slice_num = ceildiv(num_rows, slice_size);
-//
-//    const auto source_values = source->get_const_values();
-//    const auto source_row_ptrs = source->get_const_row_ptrs();
-//    const auto source_col_idxs = source->get_const_col_idxs();
-//
-//    auto nnz_per_row = Array<size_type>(exec, num_rows);
-//    auto grid_dim = ceildiv(num_rows, default_block_size);
-//
-//    if (grid_dim > 0) {
-//        hipLaunchKernelGGL(kernel::calculate_nnz_per_row, dim3(grid_dim),
-//                           dim3(default_block_size), 0, 0, num_rows,
-//                           as_hip_type(source_row_ptrs),
-//                           as_hip_type(nnz_per_row.get_data()));
-//    }
-//
-//    grid_dim = slice_num;
-//
-//    if (grid_dim > 0) {
-//        hipLaunchKernelGGL(kernel::calculate_slice_lengths, dim3(grid_dim),
-//                           dim3(config::warp_size), 0, 0, num_rows,
-//                           slice_size, stride_factor,
-//                           as_hip_type(nnz_per_row.get_const_data()),
-//                           as_hip_type(slice_lengths),
-//                           as_hip_type(slice_sets));
-//    }
-//
-//    components::prefix_sum(exec, slice_sets, slice_num + 1);
-//
-//    grid_dim = ceildiv(num_rows, default_block_size);
-//    if (grid_dim > 0) {
-//        hipLaunchKernelGGL(
-//            kernel::fill_in_sellp, dim3(grid_dim), dim3(default_block_size),
-//            0, 0, num_rows, slice_size, as_hip_type(source_values),
-//            as_hip_type(source_row_ptrs), as_hip_type(source_col_idxs),
-//            as_hip_type(slice_lengths), as_hip_type(slice_sets),
-//            as_hip_type(result_col_idxs), as_hip_type(result_values));
-//    }
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void convert_to_ell(std::shared_ptr<const HipExecutor> exec,
-                    const matrix::Fbcsr<ValueType, IndexType> *source,
-                    matrix::Ell<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto source_values = source->get_const_values();
-//    const auto source_row_ptrs = source->get_const_row_ptrs();
-//    const auto source_col_idxs = source->get_const_col_idxs();
-//
-//    auto result_values = result->get_values();
-//    auto result_col_idxs = result->get_col_idxs();
-//    const auto stride = result->get_stride();
-//    const auto max_nnz_per_row = result->get_num_stored_elements_per_row();
-//    const auto num_rows = result->get_size()[0];
-//    const auto num_cols = result->get_size()[1];
-//
-//    const auto init_grid_dim =
-//        ceildiv(max_nnz_per_row * num_rows, default_block_size);
-//
-//    hipLaunchKernelGGL(kernel::initialize_zero_ell, dim3(init_grid_dim),
-//                       dim3(default_block_size), 0, 0, max_nnz_per_row,
-//                       stride, as_hip_type(result_values),
-//                       as_hip_type(result_col_idxs));
-//
-//    const auto grid_dim =
-//        ceildiv(num_rows * config::warp_size, default_block_size);
-//
-//    hipLaunchKernelGGL(kernel::fill_in_ell, dim3(grid_dim),
-//                       dim3(default_block_size), 0, 0, num_rows, stride,
-//                       as_hip_type(source_values),
-//                       as_hip_type(source_row_ptrs),
-//                       as_hip_type(source_col_idxs),
-//                       as_hip_type(result_values),
-//                       as_hip_type(result_col_idxs));
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void calculate_total_cols(std::shared_ptr<const HipExecutor> exec,
-                          const matrix::Fbcsr<ValueType, IndexType> *source,
-                          size_type *result, size_type stride_factor,
-                          size_type slice_size) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto num_rows = source->get_size()[0];
-//
-//    if (num_rows == 0) {
-//        *result = 0;
-//        return;
-//    }
-//
-//    const auto slice_num = ceildiv(num_rows, slice_size);
-//    const auto row_ptrs = source->get_const_row_ptrs();
-//
-//    auto nnz_per_row = Array<size_type>(exec, num_rows);
-//    auto grid_dim = ceildiv(num_rows, default_block_size);
-//
-//    hipLaunchKernelGGL(kernel::calculate_nnz_per_row, dim3(grid_dim),
-//                       dim3(default_block_size), 0, 0, num_rows,
-//                       as_hip_type(row_ptrs),
-//                       as_hip_type(nnz_per_row.get_data()));
-//
-//    grid_dim = ceildiv(slice_num * config::warp_size, default_block_size);
-//    auto max_nnz_per_slice = Array<size_type>(exec, slice_num);
-//
-//    hipLaunchKernelGGL(kernel::reduce_max_nnz_per_slice, dim3(grid_dim),
-//                       dim3(default_block_size), 0, 0, num_rows, slice_size,
-//                       stride_factor,
-//                       as_hip_type(nnz_per_row.get_const_data()),
-//                       as_hip_type(max_nnz_per_slice.get_data()));
-//
-//    grid_dim = ceildiv(slice_num, default_block_size);
-//    auto block_results = Array<size_type>(exec, grid_dim);
-//
-//    hipLaunchKernelGGL(kernel::reduce_total_cols, dim3(grid_dim),
-//                       dim3(default_block_size), 0, 0, slice_num,
-//                       as_hip_type(max_nnz_per_slice.get_const_data()),
-//                       as_hip_type(block_results.get_data()));
-//
-//    auto d_result = Array<size_type>(exec, 1);
-//
-//    hipLaunchKernelGGL(kernel::reduce_total_cols, dim3(1),
-//                       dim3(default_block_size), 0, 0, grid_dim,
-//                       as_hip_type(block_results.get_const_data()),
-//                       as_hip_type(d_result.get_data()));
-//
-//    *result = exec->copy_val_to_host(d_result.get_const_data());
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void transpose(std::shared_ptr<const HipExecutor> exec,
                const matrix::Fbcsr<ValueType, IndexType> *orig,
                matrix::Fbcsr<ValueType, IndexType> *trans) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    if (hipsparse::is_supported<ValueType, IndexType>::value) {
-//        hipsparseAction_t copyValues = HIPSPARSE_ACTION_NUMERIC;
-//        hipsparseIndexBase_t idxBase = HIPSPARSE_INDEX_BASE_ZERO;
-//
-//        hipsparse::transpose(
-//            exec->get_hipsparse_handle(), orig->get_size()[0],
-//            orig->get_size()[1], orig->get_num_stored_elements(),
-//            orig->get_const_values(), orig->get_const_row_ptrs(),
-//            orig->get_const_col_idxs(), trans->get_values(),
-//            trans->get_row_ptrs(), trans->get_col_idxs(), copyValues,
-//            idxBase);
-//    } else {
-//        GKO_NOT_IMPLEMENTED;
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL);
@@ -982,70 +148,11 @@ void conj_transpose(std::shared_ptr<const HipExecutor> exec,
                     const matrix::Fbcsr<ValueType, IndexType> *orig,
                     matrix::Fbcsr<ValueType, IndexType> *trans)
     GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    if (hipsparse::is_supported<ValueType, IndexType>::value) {
-//        const dim3 block_size(default_block_size, 1, 1);
-//        const dim3 grid_size(
-//            ceildiv(trans->get_num_stored_elements(), block_size.x), 1, 1);
-//
-//        hipsparseAction_t copyValues = HIPSPARSE_ACTION_NUMERIC;
-//        hipsparseIndexBase_t idxBase = HIPSPARSE_INDEX_BASE_ZERO;
-//
-//        hipsparse::transpose(
-//            exec->get_hipsparse_handle(), orig->get_size()[0],
-//            orig->get_size()[1], orig->get_num_stored_elements(),
-//            orig->get_const_values(), orig->get_const_row_ptrs(),
-//            orig->get_const_col_idxs(), trans->get_values(),
-//            trans->get_col_idxs(), trans->get_row_ptrs(), copyValues,
-//            idxBase);
-//
-//        hipLaunchKernelGGL(conjugate_kernel, dim3(grid_size),
-//        dim3(block_size),
-//                           0, 0, trans->get_num_stored_elements(),
-//                           as_hip_type(trans->get_values()));
-//    } else {
-//        GKO_NOT_IMPLEMENTED;
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void row_permute(std::shared_ptr<const HipExecutor> exec,
-                 const IndexType *permutation_indices,
-                 const matrix::Fbcsr<ValueType, IndexType> *orig,
-                 matrix::Fbcsr<ValueType, IndexType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_row_permute(std::shared_ptr<const HipExecutor> exec,
-                         const IndexType *permutation_indices,
-                         const matrix::Fbcsr<ValueType, IndexType> *orig,
-                         matrix::Fbcsr<ValueType, IndexType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(
-    std::shared_ptr<const HipExecutor> exec,
-    const IndexType *permutation_indices,
-    const matrix::Fbcsr<ValueType, IndexType> *orig,
-    matrix::Fbcsr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void calculate_max_nnz_per_row(
     std::shared_ptr<const HipExecutor> exec,
@@ -1084,56 +191,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void convert_to_hybrid(std::shared_ptr<const HipExecutor> exec,
-                       const matrix::Fbcsr<ValueType, IndexType> *source,
-                       matrix::Hybrid<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto ell_val = result->get_ell_values();
-//    auto ell_col = result->get_ell_col_idxs();
-//    auto coo_val = result->get_coo_values();
-//    auto coo_col = result->get_coo_col_idxs();
-//    auto coo_row = result->get_coo_row_idxs();
-//    const auto stride = result->get_ell_stride();
-//    const auto max_nnz_per_row =
-//    result->get_ell_num_stored_elements_per_row(); const auto num_rows =
-//    result->get_size()[0]; const auto coo_num_stored_elements =
-//    result->get_coo_num_stored_elements(); auto grid_dim =
-//    ceildiv(max_nnz_per_row * num_rows, default_block_size);
-//
-//    hipLaunchKernelGGL(kernel::initialize_zero_ell, dim3(grid_dim),
-//                       dim3(default_block_size), 0, 0, max_nnz_per_row,
-//                       stride, as_hip_type(ell_val), as_hip_type(ell_col));
-//
-//    grid_dim = ceildiv(num_rows, default_block_size);
-//    auto coo_offset = Array<size_type>(exec, num_rows);
-//    hipLaunchKernelGGL(kernel::calculate_hybrid_coo_row_nnz, dim3(grid_dim),
-//                       dim3(default_block_size), 0, 0, num_rows,
-//                       max_nnz_per_row,
-//                       as_hip_type(source->get_const_row_ptrs()),
-//                       as_hip_type(coo_offset.get_data()));
-//
-//    components::prefix_sum(exec, coo_offset.get_data(), num_rows);
-//
-//    grid_dim = ceildiv(num_rows * config::warp_size, default_block_size);
-//    hipLaunchKernelGGL(kernel::fill_in_hybrid, dim3(grid_dim),
-//                       dim3(default_block_size), 0, 0, num_rows, stride,
-//                       max_nnz_per_row,
-//                       as_hip_type(source->get_const_values()),
-//                       as_hip_type(source->get_const_row_ptrs()),
-//                       as_hip_type(source->get_const_col_idxs()),
-//                       as_hip_type(coo_offset.get_const_data()),
-//                       as_hip_type(ell_val), as_hip_type(ell_col),
-//                       as_hip_type(coo_val), as_hip_type(coo_col),
-//                       as_hip_type(coo_row));
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void calculate_nonzeros_per_row(
     std::shared_ptr<const HipExecutor> exec,
@@ -1159,48 +216,6 @@ template <typename ValueType, typename IndexType>
 void sort_by_column_index(std::shared_ptr<const HipExecutor> exec,
                           matrix::Fbcsr<ValueType, IndexType> *to_sort)
     GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    if (hipsparse::is_supported<ValueType, IndexType>::value) {
-//        auto handle = exec->get_hipsparse_handle();
-//        auto descr = hipsparse::create_mat_descr();
-//        auto m = IndexType(to_sort->get_size()[0]);
-//        auto n = IndexType(to_sort->get_size()[1]);
-//        auto nnz = IndexType(to_sort->get_num_stored_elements());
-//        auto row_ptrs = to_sort->get_const_row_ptrs();
-//        auto col_idxs = to_sort->get_col_idxs();
-//        auto vals = to_sort->get_values();
-//
-//        // copy values
-//        Array<ValueType> tmp_vals_array(exec, nnz);
-//        exec->copy(nnz, vals, tmp_vals_array.get_data());
-//        auto tmp_vals = tmp_vals_array.get_const_data();
-//
-//        // init identity permutation
-//        Array<IndexType> permutation_array(exec, nnz);
-//        auto permutation = permutation_array.get_data();
-//        hipsparse::create_identity_permutation(handle, nnz, permutation);
-//
-//        // allocate buffer
-//        size_type buffer_size{};
-//        hipsparse::fbcsrsort_buffer_size(handle, m, n, nnz, row_ptrs,
-//        col_idxs,
-//                                       buffer_size);
-//        Array<char> buffer_array{exec, buffer_size};
-//        auto buffer = buffer_array.get_data();
-//
-//        // sort column indices
-//        hipsparse::fbcsrsort(handle, m, n, nnz, descr, row_ptrs, col_idxs,
-//                           permutation, buffer);
-//
-//        // sort values
-//        hipsparse::gather(handle, nnz, tmp_vals, vals, permutation);
-//
-//        hipsparse::destroy(descr);
-//    } else {
-//        GKO_NOT_IMPLEMENTED;
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
@@ -1234,24 +249,6 @@ template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const HipExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *orig,
                       matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto nnz = orig->get_num_stored_elements();
-//    const auto diag_size = diag->get_size()[0];
-//    const auto num_blocks =
-//        ceildiv(config::warp_size * diag_size, default_block_size);
-//
-//    const auto orig_values = orig->get_const_values();
-//    const auto orig_row_ptrs = orig->get_const_row_ptrs();
-//    const auto orig_col_idxs = orig->get_const_col_idxs();
-//    auto diag_values = diag->get_values();
-//
-//    hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel::extract_diagonal),
-//                       dim3(num_blocks), dim3(default_block_size), 0, 0,
-//                       diag_size, nnz, as_hip_type(orig_values),
-//                       as_hip_type(orig_row_ptrs), as_hip_type(orig_col_idxs),
-//                       as_hip_type(diag_values));
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL);
diff --git a/hip/test/matrix/CMakeLists.txt b/hip/test/matrix/CMakeLists.txt
index 95d058e8069..94e92f08f5c 100644
--- a/hip/test/matrix/CMakeLists.txt
+++ b/hip/test/matrix/CMakeLists.txt
@@ -3,6 +3,5 @@ ginkgo_create_hip_test(csr_kernels)
 ginkgo_create_hip_test(dense_kernels)
 ginkgo_create_hip_test(diagonal_kernels)
 ginkgo_create_hip_test(ell_kernels)
-#ginkgo_create_hip_test(fbcsr_kernels)
 ginkgo_create_hip_test(hybrid_kernels)
 ginkgo_create_hip_test(sellp_kernels)
diff --git a/hip/test/matrix/fbcsr_kernels.hip.cpp b/hip/test/matrix/fbcsr_kernels.hip.cpp
deleted file mode 100644
index 79b873cb02d..00000000000
--- a/hip/test/matrix/fbcsr_kernels.hip.cpp
+++ /dev/null
@@ -1,866 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/matrix/fbcsr.hpp>
-
-
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/coo.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/diagonal.hpp>
-#include <ginkgo/core/matrix/ell.hpp>
-#include <ginkgo/core/matrix/hybrid.hpp>
-#include <ginkgo/core/matrix/identity.hpp>
-#include <ginkgo/core/matrix/sellp.hpp>
-#include <ginkgo/core/matrix/sparsity_fbcsr.hpp>
-
-
-#include "core/matrix/fbcsr_kernels.hpp"
-#include "hip/test/utils.hip.hpp"
-
-
-namespace {
-
-
-class Fbcsr : public ::testing::Test {
-protected:
-    using Vec = gko::matrix::Dense<>;
-    using Mtx = gko::matrix::Fbcsr<>;
-    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
-    using ComplexMtx = gko::matrix::Fbcsr<std::complex<double>>;
-
-    Fbcsr() : mtx_size(532, 231), rand_engine(42) {}
-
-    void SetUp()
-    {
-        ASSERT_GT(gko::HipExecutor::get_num_devices(), 0);
-        ref = gko::ReferenceExecutor::create();
-        hip = gko::HipExecutor::create(0, ref);
-    }
-
-    void TearDown()
-    {
-        if (hip != nullptr) {
-            ASSERT_NO_THROW(hip->synchronize());
-        }
-    }
-
-    template <typename MtxType>
-    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
-                                     int min_nnz_row)
-    {
-        return gko::test::generate_random_matrix<MtxType>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(min_nnz_row, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
-    }
-
-    void set_up_apply_data(std::shared_ptr<Mtx::strategy_type> strategy,
-                           int num_vectors = 1)
-    {
-        mtx = Mtx::create(ref, strategy);
-        mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[1], 1));
-        square_mtx = Mtx::create(ref, strategy);
-        square_mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[0], 1));
-        expected = gen_mtx<Vec>(mtx_size[0], num_vectors, 1);
-        y = gen_mtx<Vec>(mtx_size[1], num_vectors, 1);
-        alpha = gko::initialize<Vec>({2.0}, ref);
-        beta = gko::initialize<Vec>({-1.0}, ref);
-        dmtx = Mtx::create(hip, strategy);
-        dmtx->copy_from(mtx.get());
-        square_dmtx = Mtx::create(hip, strategy);
-        square_dmtx->copy_from(square_mtx.get());
-        dresult = Vec::create(hip);
-        dresult->copy_from(expected.get());
-        dy = Vec::create(hip);
-        dy->copy_from(y.get());
-        dalpha = Vec::create(hip);
-        dalpha->copy_from(alpha.get());
-        dbeta = Vec::create(hip);
-        dbeta->copy_from(beta.get());
-    }
-
-    void set_up_apply_complex_data(
-        std::shared_ptr<ComplexMtx::strategy_type> strategy)
-    {
-        complex_mtx = ComplexMtx::create(ref, strategy);
-        complex_mtx->copy_from(
-            gen_mtx<ComplexVec>(mtx_size[0], mtx_size[1], 1));
-        complex_dmtx = ComplexMtx::create(hip, strategy);
-        complex_dmtx->copy_from(complex_mtx.get());
-    }
-
-    struct matrix_pair {
-        std::unique_ptr<Mtx> ref;
-        std::unique_ptr<Mtx> hip;
-    };
-
-    matrix_pair gen_unsorted_mtx()
-    {
-        constexpr int min_nnz_per_row = 2;  // Must be at least 2
-        auto local_mtx_ref =
-            gen_mtx<Mtx>(mtx_size[0], mtx_size[1], min_nnz_per_row);
-        for (size_t row = 0; row < mtx_size[0]; ++row) {
-            const auto row_ptrs = local_mtx_ref->get_const_row_ptrs();
-            const auto start_row = row_ptrs[row];
-            auto col_idx = local_mtx_ref->get_col_idxs() + start_row;
-            auto vals = local_mtx_ref->get_values() + start_row;
-            const auto nnz_in_this_row = row_ptrs[row + 1] - row_ptrs[row];
-            auto swap_idx_dist =
-                std::uniform_int_distribution<>(0, nnz_in_this_row - 1);
-            // shuffle `nnz_in_this_row / 2` times
-            for (size_t perm = 0; perm < nnz_in_this_row; perm += 2) {
-                const auto idx1 = swap_idx_dist(rand_engine);
-                const auto idx2 = swap_idx_dist(rand_engine);
-                std::swap(col_idx[idx1], col_idx[idx2]);
-                std::swap(vals[idx1], vals[idx2]);
-            }
-        }
-        auto local_mtx_hip = Mtx::create(hip);
-        local_mtx_hip->copy_from(local_mtx_ref.get());
-
-        return {std::move(local_mtx_ref), std::move(local_mtx_hip)};
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::HipExecutor> hip;
-
-    const gko::dim<2> mtx_size;
-    std::ranlux48 rand_engine;
-
-    std::unique_ptr<Mtx> mtx;
-    std::unique_ptr<ComplexMtx> complex_mtx;
-    std::unique_ptr<Mtx> square_mtx;
-    std::unique_ptr<Vec> expected;
-    std::unique_ptr<Vec> y;
-    std::unique_ptr<Vec> alpha;
-    std::unique_ptr<Vec> beta;
-
-    std::unique_ptr<Mtx> dmtx;
-    std::unique_ptr<ComplexMtx> complex_dmtx;
-    std::unique_ptr<Mtx> square_dmtx;
-    std::unique_ptr<Vec> dresult;
-    std::unique_ptr<Vec> dy;
-    std::unique_ptr<Vec> dalpha;
-    std::unique_ptr<Vec> dbeta;
-};
-
-
-TEST_F(Fbcsr, StrategyAfterCopyIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::load_balance>(hip));
-//
-//    ASSERT_EQ(mtx->get_strategy()->get_name(),
-//              dmtx->get_strategy()->get_name());
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithLoadBalance)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::load_balance>(hip));
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithLoadBalance)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::load_balance>(hip));
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithHipsparse)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithHipsparse)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithMergePath)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::merge_path>());
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithMergePath)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::merge_path>());
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithClassical)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::classical>());
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRefWithClassical)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::classical>());
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyIsEquivalentToRefWithAutomatical)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyToDenseMatrixIsEquivalentToRefWithLoadBalance)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::load_balance>(hip), 3);
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithLoadBalance)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::load_balance>(hip), 3);
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyToDenseMatrixIsEquivalentToRefWithClassical)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::classical>(), 3);
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithClassical)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::classical>(), 3);
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyToDenseMatrixIsEquivalentToRefWithMergePath)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::merge_path>(), 3);
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithMergePath)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::merge_path>(), 3);
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyToFbcsrMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
-//    auto trans = mtx->transpose();
-//    auto d_trans = dmtx->transpose();
-//
-//    mtx->apply(alpha.get(), trans.get(), beta.get(), square_mtx.get());
-//    dmtx->apply(dalpha.get(), d_trans.get(), dbeta.get(), square_dmtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
-//    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-//    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyToFbcsrMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
-//    auto trans = mtx->transpose();
-//    auto d_trans = dmtx->transpose();
-//
-//    mtx->apply(trans.get(), square_mtx.get());
-//    dmtx->apply(d_trans.get(), square_dmtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
-//    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-//    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
-//    auto a = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
-//    auto b = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
-//    auto da = Mtx::create(hip);
-//    auto db = Mtx::create(hip);
-//    da->copy_from(a.get());
-//    db->copy_from(b.get());
-//    auto id = gko::matrix::Identity<Mtx::value_type>::create(ref,
-//    mtx_size[1]); auto did =
-//    gko::matrix::Identity<Mtx::value_type>::create(hip, mtx_size[1]);
-//
-//    a->apply(alpha.get(), id.get(), beta.get(), b.get());
-//    da->apply(dalpha.get(), did.get(), dbeta.get(), db.get());
-//
-//    GKO_ASSERT_MTX_NEAR(b, db, 1e-14);
-//    GKO_ASSERT_MTX_EQ_SPARSITY(b, db);
-//    ASSERT_TRUE(db->is_sorted_by_column_index());
-//}
-
-
-TEST_F(Fbcsr, TransposeIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
-//
-//    auto trans = mtx->transpose();
-//    auto d_trans = dmtx->transpose();
-//
-//    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(d_trans.get()),
-//                        static_cast<Mtx *>(trans.get()), 0.0);
-//    ASSERT_TRUE(static_cast<Mtx
-//    *>(d_trans.get())->is_sorted_by_column_index());
-//}
-
-
-TEST_F(Fbcsr, ConvertToDenseIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto dense_mtx = gko::matrix::Dense<>::create(ref);
-//    auto ddense_mtx = gko::matrix::Dense<>::create(hip);
-//
-//    mtx->convert_to(dense_mtx.get());
-//    dmtx->convert_to(ddense_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToDenseIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto dense_mtx = gko::matrix::Dense<>::create(ref);
-//    auto ddense_mtx = gko::matrix::Dense<>::create(hip);
-//
-//    mtx->move_to(dense_mtx.get());
-//    dmtx->move_to(ddense_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, ConvertToEllIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto ell_mtx = gko::matrix::Ell<>::create(ref);
-//    auto dell_mtx = gko::matrix::Ell<>::create(hip);
-//
-//    mtx->convert_to(ell_mtx.get());
-//    dmtx->convert_to(dell_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(ell_mtx.get(), dell_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToEllIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto ell_mtx = gko::matrix::Ell<>::create(ref);
-//    auto dell_mtx = gko::matrix::Ell<>::create(hip);
-//
-//    mtx->move_to(ell_mtx.get());
-//    dmtx->move_to(dell_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(ell_mtx.get(), dell_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, ConvertToSparsityFbcsrIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(ref);
-//    auto d_sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(hip);
-//
-//    mtx->convert_to(sparsity_mtx.get());
-//    dmtx->convert_to(d_sparsity_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(sparsity_mtx.get(), d_sparsity_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToSparsityFbcsrIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(ref);
-//    auto d_sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(hip);
-//
-//    mtx->move_to(sparsity_mtx.get());
-//    dmtx->move_to(d_sparsity_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(sparsity_mtx.get(), d_sparsity_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, CalculateMaxNnzPerRowIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    gko::size_type max_nnz_per_row;
-//    gko::size_type dmax_nnz_per_row;
-//
-//    gko::kernels::reference::fbcsr::calculate_max_nnz_per_row(ref, mtx.get(),
-//                                                            &max_nnz_per_row);
-//    gko::kernels::hip::fbcsr::calculate_max_nnz_per_row(hip, dmtx.get(),
-//                                                      &dmax_nnz_per_row);
-//
-//    ASSERT_EQ(max_nnz_per_row, dmax_nnz_per_row);
-//}
-
-
-TEST_F(Fbcsr, ConvertToCooIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto coo_mtx = gko::matrix::Coo<>::create(ref);
-//    auto dcoo_mtx = gko::matrix::Coo<>::create(hip);
-//
-//    mtx->convert_to(coo_mtx.get());
-//    dmtx->convert_to(dcoo_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToCooIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto coo_mtx = gko::matrix::Coo<>::create(ref);
-//    auto dcoo_mtx = gko::matrix::Coo<>::create(hip);
-//
-//    mtx->move_to(coo_mtx.get());
-//    dmtx->move_to(dcoo_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, ConvertToSellpIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
-//    auto dsellp_mtx = gko::matrix::Sellp<>::create(hip);
-//
-//    mtx->convert_to(sellp_mtx.get());
-//    dmtx->convert_to(dsellp_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(sellp_mtx.get(), dsellp_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToSellpIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
-//    auto dsellp_mtx = gko::matrix::Sellp<>::create(hip);
-//
-//    mtx->move_to(sellp_mtx.get());
-//    dmtx->move_to(dsellp_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(sellp_mtx.get(), dsellp_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, ConvertsEmptyToSellp)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto dempty_mtx = Mtx::create(hip);
-//    auto dsellp_mtx = gko::matrix::Sellp<>::create(hip);
-//
-//    dempty_mtx->convert_to(dsellp_mtx.get());
-//
-//    ASSERT_EQ(hip->copy_val_to_host(dsellp_mtx->get_const_slice_sets()), 0);
-//    ASSERT_FALSE(dsellp_mtx->get_size());
-//}
-
-
-TEST_F(Fbcsr, CalculateTotalColsIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    gko::size_type total_cols;
-//    gko::size_type dtotal_cols;
-//
-//    gko::kernels::reference::fbcsr::calculate_total_cols(
-//        ref, mtx.get(), &total_cols, 2, gko::matrix::default_slice_size);
-//    gko::kernels::hip::fbcsr::calculate_total_cols(
-//        hip, dmtx.get(), &dtotal_cols, 2, gko::matrix::default_slice_size);
-//
-//    ASSERT_EQ(total_cols, dtotal_cols);
-//}
-
-
-TEST_F(Fbcsr, CalculatesNonzerosPerRow)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    gko::Array<gko::size_type> row_nnz(ref, mtx->get_size()[0]);
-//    gko::Array<gko::size_type> drow_nnz(hip, dmtx->get_size()[0]);
-//
-//    gko::kernels::reference::fbcsr::calculate_nonzeros_per_row(ref, mtx.get(),
-//                                                             &row_nnz);
-//    gko::kernels::hip::fbcsr::calculate_nonzeros_per_row(hip, dmtx.get(),
-//                                                       &drow_nnz);
-//
-//    GKO_ASSERT_ARRAY_EQ(row_nnz, drow_nnz);
-//}
-
-
-TEST_F(Fbcsr, ConvertToHybridIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Hybrid_type = gko::matrix::Hybrid<>;
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto hybrid_mtx = Hybrid_type::create(
-//        ref, std::make_shared<Hybrid_type::column_limit>(2));
-//    auto dhybrid_mtx = Hybrid_type::create(
-//        hip, std::make_shared<Hybrid_type::column_limit>(2));
-//
-//    mtx->convert_to(hybrid_mtx.get());
-//    dmtx->convert_to(dhybrid_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToHybridIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Hybrid_type = gko::matrix::Hybrid<>;
-//    set_up_apply_data(std::make_shared<Mtx::sparselib>());
-//    auto hybrid_mtx = Hybrid_type::create(
-//        ref, std::make_shared<Hybrid_type::column_limit>(2));
-//    auto dhybrid_mtx = Hybrid_type::create(
-//        hip, std::make_shared<Hybrid_type::column_limit>(2));
-//
-//    mtx->move_to(hybrid_mtx.get());
-//    dmtx->move_to(dhybrid_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, RecognizeSortedMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
-//    bool is_sorted_hip{};
-//    bool is_sorted_ref{};
-//
-//    is_sorted_ref = mtx->is_sorted_by_column_index();
-//    is_sorted_hip = dmtx->is_sorted_by_column_index();
-//
-//    ASSERT_EQ(is_sorted_ref, is_sorted_hip);
-//}
-
-
-TEST_F(Fbcsr, RecognizeUnsortedMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto uns_mtx = gen_unsorted_mtx();
-//    bool is_sorted_hip{};
-//    bool is_sorted_ref{};
-//
-//    is_sorted_ref = uns_mtx.ref->is_sorted_by_column_index();
-//    is_sorted_hip = uns_mtx.hip->is_sorted_by_column_index();
-//
-//    ASSERT_EQ(is_sorted_ref, is_sorted_hip);
-//}
-
-
-TEST_F(Fbcsr, SortSortedMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
-//
-//    mtx->sort_by_column_index();
-//    dmtx->sort_by_column_index();
-//
-//    // Values must be unchanged, therefore, tolerance is `0`
-//    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 0);
-//}
-
-
-TEST_F(Fbcsr, SortUnsortedMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto uns_mtx = gen_unsorted_mtx();
-//
-//    uns_mtx.ref->sort_by_column_index();
-//    uns_mtx.hip->sort_by_column_index();
-//
-//    // Values must be unchanged, therefore, tolerance is `0`
-//    GKO_ASSERT_MTX_NEAR(uns_mtx.ref, uns_mtx.hip, 0);
-//}
-
-
-TEST_F(Fbcsr, OneAutomaticalWorksWithDifferentMatrices)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto automatical = std::make_shared<Mtx::automatical>(hip);
-//    auto row_len_limit = std::max(automatical->nvidia_row_len_limit,
-//                                  automatical->amd_row_len_limit);
-//    auto load_balance_mtx = Mtx::create(ref);
-//    auto classical_mtx = Mtx::create(ref);
-//    load_balance_mtx->copy_from(
-//        gen_mtx<Vec>(1, row_len_limit + 1000, row_len_limit + 1));
-//    classical_mtx->copy_from(gen_mtx<Vec>(50, 50, 1));
-//    auto load_balance_mtx_d = Mtx::create(hip);
-//    auto classical_mtx_d = Mtx::create(hip);
-//    load_balance_mtx_d->copy_from(load_balance_mtx.get());
-//    classical_mtx_d->copy_from(classical_mtx.get());
-//
-//    load_balance_mtx_d->set_strategy(automatical);
-//    classical_mtx_d->set_strategy(automatical);
-//
-//    EXPECT_EQ("load_balance", load_balance_mtx_d->get_strategy()->get_name());
-//    EXPECT_EQ("classical", classical_mtx_d->get_strategy()->get_name());
-//    ASSERT_NE(load_balance_mtx_d->get_strategy().get(),
-//              classical_mtx_d->get_strategy().get());
-//}
-
-
-TEST_F(Fbcsr, ExtractDiagonalIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
-//
-//    auto diag = mtx->extract_diagonal();
-//    auto ddiag = dmtx->extract_diagonal();
-//
-//    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-//}
-
-
-TEST_F(Fbcsr, InplaceAbsoluteMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
-//
-//    mtx->compute_absolute_inplace();
-//    dmtx->compute_absolute_inplace();
-//
-//    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, OutplaceAbsoluteMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
-//
-//    auto abs_mtx = mtx->compute_absolute();
-//    auto dabs_mtx = dmtx->compute_absolute();
-//
-//    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(hip));
-//
-//    complex_mtx->compute_absolute_inplace();
-//    complex_dmtx->compute_absolute_inplace();
-//
-//    GKO_ASSERT_MTX_NEAR(complex_mtx, complex_dmtx, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(hip));
-//
-//    auto abs_mtx = complex_mtx->compute_absolute();
-//    auto dabs_mtx = complex_dmtx->compute_absolute();
-//
-//    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
-//}
-
-
-}  // namespace
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index 7d5c00cbd02..33e18a99794 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -52,18 +52,6 @@ class Dense;
 template <typename ValueType, typename IndexType>
 class Csr;
 
-template <typename ValueType, typename IndexType>
-class Coo;
-
-template <typename ValueType, typename IndexType>
-class Ell;
-
-template <typename ValueType, typename IndexType>
-class Hybrid;
-
-template <typename ValueType, typename IndexType>
-class Sellp;
-
 template <typename ValueType, typename IndexType>
 class SparsityCsr;
 
@@ -129,18 +117,15 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
               public ConvertibleTo<Fbcsr<next_precision<ValueType>, IndexType>>,
               public ConvertibleTo<Dense<ValueType>>,
               public ConvertibleTo<Csr<ValueType, IndexType>>,
-              public ConvertibleTo<Coo<ValueType, IndexType>>,
               public ConvertibleTo<SparsityCsr<ValueType, IndexType>>,
               public DiagonalExtractable<ValueType>,
               public ReadableFromMatrixData<ValueType, IndexType>,
               public WritableToMatrixData<ValueType, IndexType>,
               public Transposable,
-              public Permutable<IndexType>,
               public EnableAbsoluteComputation<
                   remove_complex<Fbcsr<ValueType, IndexType>>> {
     friend class EnableCreateMethod<Fbcsr>;
     friend class EnablePolymorphicObject<Fbcsr, LinOp>;
-    friend class Coo<ValueType, IndexType>;
     friend class Dense<ValueType>;
     friend class SparsityCsr<ValueType, IndexType>;
     friend class FbcsrBuilder<ValueType, IndexType>;
@@ -176,10 +161,6 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
 
     void move_to(Csr<ValueType, IndexType> *result) override;
 
-    void convert_to(Coo<ValueType, IndexType> *result) const override;
-
-    void move_to(Coo<ValueType, IndexType> *result) override;
-
     /// Get the block sparsity pattern in CSR-like format
     /** Note that the actual non-zero values are never copied;
      * the result always has a value array of size 1 with the value 1.
@@ -199,18 +180,6 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
 
     std::unique_ptr<LinOp> conj_transpose() const override;
 
-    std::unique_ptr<LinOp> row_permute(
-        const Array<IndexType> *permutation_indices) const override;
-
-    std::unique_ptr<LinOp> column_permute(
-        const Array<IndexType> *permutation_indices) const override;
-
-    std::unique_ptr<LinOp> inverse_row_permute(
-        const Array<IndexType> *inverse_permutation_indices) const override;
-
-    std::unique_ptr<LinOp> inverse_column_permute(
-        const Array<IndexType> *inverse_permutation_indices) const override;
-
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
     std::unique_ptr<absolute_type> compute_absolute() const override;
diff --git a/omp/components/fbcsr_spgeam.hpp b/omp/components/fbcsr_spgeam.hpp
deleted file mode 100644
index e4a06532ed3..00000000000
--- a/omp/components/fbcsr_spgeam.hpp
+++ /dev/null
@@ -1,31 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
diff --git a/omp/matrix/fbcsr_kernels.cpp b/omp/matrix/fbcsr_kernels.cpp
index 609ab990cdb..a38b3c9bda7 100644
--- a/omp/matrix/fbcsr_kernels.cpp
+++ b/omp/matrix/fbcsr_kernels.cpp
@@ -44,17 +44,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/hybrid.hpp>
 
 
 #include "core/base/allocator.hpp"
 #include "core/base/iterator_factory.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/fbcsr_builder.hpp"
-#include "omp/components/fbcsr_spgeam.hpp"
 #include "omp/components/format_conversion.hpp"
 
 
@@ -74,27 +70,6 @@ void spmv(std::shared_ptr<const OmpExecutor> exec,
           const matrix::Fbcsr<ValueType, IndexType> *a,
           const matrix::Dense<ValueType> *b,
           matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto row_ptrs = a->get_const_row_ptrs();
-//    auto col_idxs = a->get_const_col_idxs();
-//    auto vals = a->get_const_values();
-//
-//#pragma omp parallel for
-//    for (size_type row = 0; row < a->get_size()[0]; ++row) {
-//        for (size_type j = 0; j < c->get_size()[1]; ++j) {
-//            c->at(row, j) = zero<ValueType>();
-//        }
-//        for (size_type k = row_ptrs[row];
-//             k < static_cast<size_type>(row_ptrs[row + 1]); ++k) {
-//            auto val = vals[k];
-//            auto col = col_idxs[k];
-//            for (size_type j = 0; j < c->get_size()[1]; ++j) {
-//                c->at(row, j) += val * b->at(col, j);
-//            }
-//        }
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPMV_KERNEL);
 
@@ -106,29 +81,6 @@ void advanced_spmv(std::shared_ptr<const OmpExecutor> exec,
                    const matrix::Dense<ValueType> *b,
                    const matrix::Dense<ValueType> *beta,
                    matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto row_ptrs = a->get_const_row_ptrs();
-//    auto col_idxs = a->get_const_col_idxs();
-//    auto vals = a->get_const_values();
-//    auto valpha = alpha->at(0, 0);
-//    auto vbeta = beta->at(0, 0);
-//
-//#pragma omp parallel for
-//    for (size_type row = 0; row < a->get_size()[0]; ++row) {
-//        for (size_type j = 0; j < c->get_size()[1]; ++j) {
-//            c->at(row, j) *= vbeta;
-//        }
-//        for (size_type k = row_ptrs[row];
-//             k < static_cast<size_type>(row_ptrs[row + 1]); ++k) {
-//            auto val = vals[k];
-//            auto col = col_idxs[k];
-//            for (size_type j = 0; j < c->get_size()[1]; ++j) {
-//                c->at(row, j) += valpha * val * b->at(col, j);
-//            }
-//        }
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
@@ -167,213 +119,6 @@ void spgemm_insert_row2(unordered_set<IndexType> &cols,
 //}
 
 
-template <typename ValueType, typename IndexType>
-void spgemm_accumulate_row(map<IndexType, ValueType> &cols,
-                           const matrix::Fbcsr<ValueType, IndexType> *c,
-                           ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto row_ptrs = c->get_const_row_ptrs();
-//    auto col_idxs = c->get_const_col_idxs();
-//    auto vals = c->get_const_values();
-//    for (size_type c_nz = row_ptrs[row]; c_nz < size_type(row_ptrs[row + 1]);
-//         ++c_nz) {
-//        auto c_col = col_idxs[c_nz];
-//        auto c_val = vals[c_nz];
-//        cols[c_col] += scale * c_val;
-//    }
-//}
-
-
-template <typename ValueType, typename IndexType>
-void spgemm_accumulate_row2(map<IndexType, ValueType> &cols,
-                            const matrix::Fbcsr<ValueType, IndexType> *a,
-                            const matrix::Fbcsr<ValueType, IndexType> *b,
-                            ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto a_row_ptrs = a->get_const_row_ptrs();
-//    auto a_col_idxs = a->get_const_col_idxs();
-//    auto a_vals = a->get_const_values();
-//    auto b_row_ptrs = b->get_const_row_ptrs();
-//    auto b_col_idxs = b->get_const_col_idxs();
-//    auto b_vals = b->get_const_values();
-//    for (size_type a_nz = a_row_ptrs[row];
-//         a_nz < size_type(a_row_ptrs[row + 1]); ++a_nz) {
-//        auto a_col = a_col_idxs[a_nz];
-//        auto a_val = a_vals[a_nz];
-//        auto b_row = a_col;
-//        for (size_type b_nz = b_row_ptrs[b_row];
-//             b_nz < size_type(b_row_ptrs[b_row + 1]); ++b_nz) {
-//            auto b_col = b_col_idxs[b_nz];
-//            auto b_val = b_vals[b_nz];
-//            cols[b_col] += scale * a_val * b_val;
-//        }
-//    }
-//}
-
-
-template <typename ValueType, typename IndexType>
-void spgemm(std::shared_ptr<const OmpExecutor> exec,
-            const matrix::Fbcsr<ValueType, IndexType> *a,
-            const matrix::Fbcsr<ValueType, IndexType> *b,
-            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = a->get_size()[0];
-//
-//    // first sweep: count nnz for each row
-//    auto c_row_ptrs = c->get_row_ptrs();
-//
-//    unordered_set<IndexType> local_col_idxs(exec);
-//#pragma omp parallel for firstprivate(local_col_idxs)
-//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
-//        local_col_idxs.clear();
-//        spgemm_insert_row2(local_col_idxs, a, b, a_row);
-//        c_row_ptrs[a_row] = local_col_idxs.size();
-//    }
-//
-//    // build row pointers
-//    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
-//
-//    // second sweep: accumulate non-zeros
-//    auto new_nnz = c_row_ptrs[num_rows];
-//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
-//    auto &c_col_idxs_array = c_builder.get_col_idx_array();
-//    auto &c_vals_array = c_builder.get_value_array();
-//    c_col_idxs_array.resize_and_reset(new_nnz);
-//    c_vals_array.resize_and_reset(new_nnz);
-//    auto c_col_idxs = c_col_idxs_array.get_data();
-//    auto c_vals = c_vals_array.get_data();
-//
-//    map<IndexType, ValueType> local_row_nzs(exec);
-//#pragma omp parallel for firstprivate(local_row_nzs)
-//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
-//        local_row_nzs.clear();
-//        spgemm_accumulate_row2(local_row_nzs, a, b, one<ValueType>(), a_row);
-//        // store result
-//        auto c_nz = c_row_ptrs[a_row];
-//        for (auto pair : local_row_nzs) {
-//            c_col_idxs[c_nz] = pair.first;
-//            c_vals[c_nz] = pair.second;
-//            ++c_nz;
-//        }
-//    }
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEMM_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void advanced_spgemm(std::shared_ptr<const OmpExecutor> exec,
-                     const matrix::Dense<ValueType> *alpha,
-                     const matrix::Fbcsr<ValueType, IndexType> *a,
-                     const matrix::Fbcsr<ValueType, IndexType> *b,
-                     const matrix::Dense<ValueType> *beta,
-                     const matrix::Fbcsr<ValueType, IndexType> *d,
-                     matrix::Fbcsr<ValueType, IndexType> *c)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = a->get_size()[0];
-//    auto valpha = alpha->at(0, 0);
-//    auto vbeta = beta->at(0, 0);
-//
-//    // first sweep: count nnz for each row
-//    auto c_row_ptrs = c->get_row_ptrs();
-//
-//    unordered_set<IndexType> local_col_idxs(exec);
-//#pragma omp parallel for firstprivate(local_col_idxs)
-//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
-//        local_col_idxs.clear();
-//        spgemm_insert_row(local_col_idxs, d, a_row);
-//        spgemm_insert_row2(local_col_idxs, a, b, a_row);
-//        c_row_ptrs[a_row] = local_col_idxs.size();
-//    }
-//
-//    // build row pointers
-//    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
-//
-//    // second sweep: accumulate non-zeros
-//    auto new_nnz = c_row_ptrs[num_rows];
-//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
-//    auto &c_col_idxs_array = c_builder.get_col_idx_array();
-//    auto &c_vals_array = c_builder.get_value_array();
-//    c_col_idxs_array.resize_and_reset(new_nnz);
-//    c_vals_array.resize_and_reset(new_nnz);
-//    auto c_col_idxs = c_col_idxs_array.get_data();
-//    auto c_vals = c_vals_array.get_data();
-//
-//    map<IndexType, ValueType> local_row_nzs(exec);
-//#pragma omp parallel for firstprivate(local_row_nzs)
-//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
-//        local_row_nzs.clear();
-//        spgemm_accumulate_row(local_row_nzs, d, vbeta, a_row);
-//        spgemm_accumulate_row2(local_row_nzs, a, b, valpha, a_row);
-//        // store result
-//        auto c_nz = c_row_ptrs[a_row];
-//        for (auto pair : local_row_nzs) {
-//            c_col_idxs[c_nz] = pair.first;
-//            c_vals[c_nz] = pair.second;
-//            ++c_nz;
-//        }
-//    }
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void spgeam(std::shared_ptr<const OmpExecutor> exec,
-            const matrix::Dense<ValueType> *alpha,
-            const matrix::Fbcsr<ValueType, IndexType> *a,
-            const matrix::Dense<ValueType> *beta,
-            const matrix::Fbcsr<ValueType, IndexType> *b,
-            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = a->get_size()[0];
-//    auto valpha = alpha->at(0, 0);
-//    auto vbeta = beta->at(0, 0);
-//
-//    // first sweep: count nnz for each row
-//    auto c_row_ptrs = c->get_row_ptrs();
-//
-//    abstract_spgeam(
-//        a, b, [](IndexType) { return IndexType{}; },
-//        [](IndexType, IndexType, ValueType, ValueType, IndexType &nnz) {
-//            ++nnz;
-//        },
-//        [&](IndexType row, IndexType nnz) { c_row_ptrs[row] = nnz; });
-//
-//    // build row pointers
-//    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
-//
-//    // second sweep: accumulate non-zeros
-//    auto new_nnz = c_row_ptrs[num_rows];
-//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
-//    auto &c_col_idxs_array = c_builder.get_col_idx_array();
-//    auto &c_vals_array = c_builder.get_value_array();
-//    c_col_idxs_array.resize_and_reset(new_nnz);
-//    c_vals_array.resize_and_reset(new_nnz);
-//    auto c_col_idxs = c_col_idxs_array.get_data();
-//    auto c_vals = c_vals_array.get_data();
-//
-//    abstract_spgeam(
-//        a, b, [&](IndexType row) { return c_row_ptrs[row]; },
-//        [&](IndexType, IndexType col, ValueType a_val, ValueType b_val,
-//            IndexType &nz) {
-//            c_vals[nz] = valpha * a_val + vbeta * b_val;
-//            c_col_idxs[nz] = col;
-//            ++nz;
-//        },
-//        [](IndexType, IndexType) {});
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEAM_KERNEL);
-
-
 template <typename IndexType>
 void convert_row_ptrs_to_idxs(std::shared_ptr<const OmpExecutor> exec,
                               const IndexType *ptrs, size_type num_rows,
@@ -384,48 +129,10 @@ void convert_row_ptrs_to_idxs(std::shared_ptr<const OmpExecutor> exec,
 //}
 
 
-template <typename ValueType, typename IndexType>
-void convert_to_coo(std::shared_ptr<const OmpExecutor> exec,
-                    const matrix::Fbcsr<ValueType, IndexType> *source,
-                    matrix::Coo<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = result->get_size()[0];
-//
-//    auto row_idxs = result->get_row_idxs();
-//    const auto source_row_ptrs = source->get_const_row_ptrs();
-//
-//    convert_row_ptrs_to_idxs(exec, source_row_ptrs, num_rows, row_idxs);
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const OmpExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *source,
                       matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = source->get_size()[0];
-//    auto num_cols = source->get_size()[1];
-//    auto row_ptrs = source->get_const_row_ptrs();
-//    auto col_idxs = source->get_const_col_idxs();
-//    auto vals = source->get_const_values();
-//
-//#pragma omp parallel for
-//    for (size_type row = 0; row < num_rows; ++row) {
-//        for (size_type col = 0; col < num_cols; ++col) {
-//            result->at(row, col) = zero<ValueType>();
-//        }
-//        for (size_type i = row_ptrs[row];
-//             i < static_cast<size_type>(row_ptrs[row + 1]); ++i) {
-//            result->at(row, col_idxs[i]) = vals[i];
-//        }
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
@@ -441,26 +148,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void convert_to_sellp(std::shared_ptr<const OmpExecutor> exec,
-                      const matrix::Fbcsr<ValueType, IndexType> *source,
-                      matrix::Sellp<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void convert_to_ell(std::shared_ptr<const OmpExecutor> exec,
-                    const matrix::Fbcsr<ValueType, IndexType> *source,
-                    matrix::Ell<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL);
-
-
 template <typename ValueType, typename IndexType, typename UnaryOperator>
 inline void convert_fbcsr_to_csc(size_type num_rows, const IndexType *row_ptrs,
                                  const IndexType *col_idxs,
@@ -468,16 +155,6 @@ inline void convert_fbcsr_to_csc(size_type num_rows, const IndexType *row_ptrs,
                                  IndexType *row_idxs, IndexType *col_ptrs,
                                  ValueType *csc_vals,
                                  UnaryOperator op) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    for (size_type row = 0; row < num_rows; ++row) {
-//        for (auto i = row_ptrs[row]; i < row_ptrs[row + 1]; ++i) {
-//            const auto dest_idx = col_ptrs[col_idxs[i]]++;
-//            row_idxs[dest_idx] = row;
-//            csc_vals[dest_idx] = op(fbcsr_vals[i]);
-//        }
-//    }
-//}
 
 
 template <typename ValueType, typename IndexType, typename UnaryOperator>
@@ -537,16 +214,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void calculate_total_cols(std::shared_ptr<const OmpExecutor> exec,
-                          const matrix::Fbcsr<ValueType, IndexType> *source,
-                          size_type *result, size_type stride_factor,
-                          size_type slice_size) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void calculate_max_nnz_per_row(
     std::shared_ptr<const OmpExecutor> exec,
@@ -557,215 +224,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void convert_to_hybrid(std::shared_ptr<const OmpExecutor> exec,
-                       const matrix::Fbcsr<ValueType, IndexType> *source,
-                       matrix::Hybrid<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = result->get_size()[0];
-//    auto num_cols = result->get_size()[1];
-//    auto strategy = result->get_strategy();
-//    auto ell_lim = strategy->get_ell_num_stored_elements_per_row();
-//    auto coo_lim = strategy->get_coo_nnz();
-//    auto coo_val = result->get_coo_values();
-//    auto coo_col = result->get_coo_col_idxs();
-//    auto coo_row = result->get_coo_row_idxs();
-//    const auto max_nnz_per_row =
-//    result->get_ell_num_stored_elements_per_row();
-//
-//// Initial Hybrid Matrix
-//#pragma omp parallel for
-//    for (size_type i = 0; i < max_nnz_per_row; i++) {
-//        for (size_type j = 0; j < result->get_ell_stride(); j++) {
-//            result->ell_val_at(j, i) = zero<ValueType>();
-//            result->ell_col_at(j, i) = 0;
-//        }
-//    }
-//
-//    const auto fbcsr_row_ptrs = source->get_const_row_ptrs();
-//    const auto fbcsr_vals = source->get_const_values();
-//    auto coo_offset = Array<IndexType>(exec, num_rows);
-//    auto coo_offset_val = coo_offset.get_data();
-//
-//    coo_offset_val[0] = 0;
-//#pragma omp parallel for
-//    for (size_type i = 1; i < num_rows; i++) {
-//        auto temp = fbcsr_row_ptrs[i] - fbcsr_row_ptrs[i - 1];
-//        coo_offset_val[i] = (temp > max_nnz_per_row) * (temp -
-//        max_nnz_per_row);
-//    }
-//
-//    auto workspace = Array<IndexType>(exec, num_rows);
-//    auto workspace_val = workspace.get_data();
-//    for (size_type i = 1; i < num_rows; i <<= 1) {
-//#pragma omp parallel for
-//        for (size_type j = i; j < num_rows; j++) {
-//            workspace_val[j] = coo_offset_val[j] + coo_offset_val[j - i];
-//        }
-//#pragma omp parallel for
-//        for (size_type j = i; j < num_rows; j++) {
-//            coo_offset_val[j] = workspace_val[j];
-//        }
-//    }
-//
-//#pragma omp parallel for
-//    for (IndexType row = 0; row < num_rows; row++) {
-//        size_type ell_idx = 0;
-//        size_type fbcsr_idx = fbcsr_row_ptrs[row];
-//        size_type coo_idx = coo_offset_val[row];
-//        while (fbcsr_idx < fbcsr_row_ptrs[row + 1]) {
-//            const auto val = fbcsr_vals[fbcsr_idx];
-//            if (ell_idx < ell_lim) {
-//                result->ell_val_at(row, ell_idx) = val;
-//                result->ell_col_at(row, ell_idx) =
-//                    source->get_const_col_idxs()[fbcsr_idx];
-//                ell_idx++;
-//            } else {
-//                coo_val[coo_idx] = val;
-//                coo_col[coo_idx] = source->get_const_col_idxs()[fbcsr_idx];
-//                coo_row[coo_idx] = row;
-//                coo_idx++;
-//            }
-//            fbcsr_idx++;
-//        }
-//    }
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void row_permute_impl(std::shared_ptr<const OmpExecutor> exec,
-                      const Array<IndexType> *permutation_indices,
-                      const matrix::Fbcsr<ValueType, IndexType> *orig,
-                      matrix::Fbcsr<ValueType, IndexType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto perm = permutation_indices->get_const_data();
-//    auto orig_row_ptrs = orig->get_const_row_ptrs();
-//    auto orig_col_idxs = orig->get_const_col_idxs();
-//    auto orig_vals = orig->get_const_values();
-//    auto rp_row_ptrs = row_permuted->get_row_ptrs();
-//    auto rp_col_idxs = row_permuted->get_col_idxs();
-//    auto rp_vals = row_permuted->get_values();
-//    size_type num_rows = orig->get_size()[0];
-//    size_type num_nnz = orig->get_num_stored_elements();
-//
-//    size_type cur_ptr = 0;
-//    rp_row_ptrs[0] = cur_ptr;
-//    vector<size_type> orig_num_nnz_per_row(num_rows, 0, exec);
-//#pragma omp parallel for
-//    for (size_type row = 0; row < num_rows; ++row) {
-//        orig_num_nnz_per_row[row] = orig_row_ptrs[row + 1] -
-//        orig_row_ptrs[row];
-//    }
-//    for (size_type row = 0; row < num_rows; ++row) {
-//        rp_row_ptrs[row + 1] =
-//            rp_row_ptrs[row] + orig_num_nnz_per_row[perm[row]];
-//    }
-//    rp_row_ptrs[num_rows] = orig_row_ptrs[num_rows];
-//#pragma omp parallel for
-//    for (size_type row = 0; row < num_rows; ++row) {
-//        auto new_row = perm[row];
-//        auto new_k = orig_row_ptrs[new_row];
-//        for (size_type k = rp_row_ptrs[row];
-//             k < size_type(rp_row_ptrs[row + 1]); ++k) {
-//            rp_col_idxs[k] = orig_col_idxs[new_k];
-//            rp_vals[k] = orig_vals[new_k];
-//            new_k++;
-//        }
-//    }
-//}
-
-
-template <typename ValueType, typename IndexType>
-void row_permute(std::shared_ptr<const OmpExecutor> exec,
-                 const IndexType *permutation_indices,
-                 const matrix::Fbcsr<ValueType, IndexType> *orig,
-                 matrix::Fbcsr<ValueType, IndexType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    row_permute_impl(exec, permutation_indices, orig, row_permuted);
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_row_permute(std::shared_ptr<const OmpExecutor> exec,
-                         const IndexType *permutation_indices,
-                         const matrix::Fbcsr<ValueType, IndexType> *orig,
-                         matrix::Fbcsr<ValueType, IndexType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto perm = permutation_indices->get_const_data();
-//    Array<IndexType> inv_perm(*permutation_indices);
-//    auto iperm = inv_perm.get_data();
-//#pragma omp parallel for
-//    for (size_type ind = 0; ind < inv_perm.get_num_elems(); ++ind) {
-//        iperm[perm[ind]] = ind;
-//    }
-//
-//    row_permute_impl(exec, &inv_perm, orig, row_permuted);
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void column_permute_impl(const Array<IndexType> *permutation_indices,
-                         const matrix::Fbcsr<ValueType, IndexType> *orig,
-                         matrix::Fbcsr<ValueType, IndexType> *column_permuted)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto perm = permutation_indices->get_const_data();
-//    auto orig_row_ptrs = orig->get_const_row_ptrs();
-//    auto orig_col_idxs = orig->get_const_col_idxs();
-//    auto orig_vals = orig->get_const_values();
-//    auto cp_row_ptrs = column_permuted->get_row_ptrs();
-//    auto cp_col_idxs = column_permuted->get_col_idxs();
-//    auto cp_vals = column_permuted->get_values();
-//    auto num_nnz = orig->get_num_stored_elements();
-//    size_type num_rows = orig->get_size()[0];
-//    size_type num_cols = orig->get_size()[1];
-//
-//#pragma omp parallel for
-//    for (size_type row = 0; row < num_rows; ++row) {
-//        cp_row_ptrs[row] = orig_row_ptrs[row];
-//        for (size_type k = orig_row_ptrs[row];
-//             k < size_type(orig_row_ptrs[row + 1]); ++k) {
-//            cp_col_idxs[k] = perm[orig_col_idxs[k]];
-//            cp_vals[k] = orig_vals[k];
-//        }
-//    }
-//    cp_row_ptrs[num_rows] = orig_row_ptrs[num_rows];
-//}
-
-
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(
-    std::shared_ptr<const OmpExecutor> exec,
-    const IndexType *permutation_indices,
-    const matrix::Fbcsr<ValueType, IndexType> *orig,
-    matrix::Fbcsr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    column_permute_impl(permutation_indices, orig, column_permuted);
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void calculate_nonzeros_per_row(
     std::shared_ptr<const OmpExecutor> exec,
@@ -790,21 +248,6 @@ template <typename ValueType, typename IndexType>
 void sort_by_column_index(std::shared_ptr<const OmpExecutor> exec,
                           matrix::Fbcsr<ValueType, IndexType> *to_sort)
     GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto values = to_sort->get_values();
-//    auto row_ptrs = to_sort->get_row_ptrs();
-//    auto col_idxs = to_sort->get_col_idxs();
-//    const auto number_rows = to_sort->get_size()[0];
-//#pragma omp parallel for
-//    for (size_type i = 0; i < number_rows; ++i) {
-//        auto start_row_idx = row_ptrs[i];
-//        auto row_nnz = row_ptrs[i + 1] - start_row_idx;
-//        auto helper = detail::IteratorFactory<IndexType, ValueType>(
-//            col_idxs + start_row_idx, values + start_row_idx, row_nnz);
-//        std::sort(helper.begin(), helper.end());
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
@@ -844,24 +287,6 @@ template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const OmpExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *orig,
                       matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto row_ptrs = orig->get_const_row_ptrs();
-//    const auto col_idxs = orig->get_const_col_idxs();
-//    const auto values = orig->get_const_values();
-//    const auto diag_size = diag->get_size()[0];
-//    auto diag_values = diag->get_values();
-//
-//#pragma omp parallel for
-//    for (size_type row = 0; row < diag_size; ++row) {
-//        for (size_type idx = row_ptrs[row]; idx < row_ptrs[row + 1]; ++idx) {
-//            if (col_idxs[idx] == row) {
-//                diag_values[row] = values[idx];
-//                break;
-//            }
-//        }
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL);
diff --git a/omp/test/matrix/CMakeLists.txt b/omp/test/matrix/CMakeLists.txt
index 3c61fd32243..0c9fd00eb30 100644
--- a/omp/test/matrix/CMakeLists.txt
+++ b/omp/test/matrix/CMakeLists.txt
@@ -3,7 +3,6 @@ ginkgo_create_test(csr_kernels)
 ginkgo_create_test(dense_kernels)
 ginkgo_create_test(diagonal_kernels)
 ginkgo_create_test(ell_kernels)
-#ginkgo_create_test(fbcsr_kernels)
 ginkgo_create_test(hybrid_kernels)
 ginkgo_create_test(sellp_kernels)
 ginkgo_create_test(sparsity_csr_kernels)
diff --git a/omp/test/matrix/fbcsr_kernels.cpp b/omp/test/matrix/fbcsr_kernels.cpp
deleted file mode 100644
index a1826bfa899..00000000000
--- a/omp/test/matrix/fbcsr_kernels.cpp
+++ /dev/null
@@ -1,662 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/matrix/fbcsr.hpp>
-
-
-#include <algorithm>
-#include <numeric>
-#include <random>
-#include <vector>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/coo.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/diagonal.hpp>
-#include <ginkgo/core/matrix/identity.hpp>
-#include <ginkgo/core/matrix/sparsity_csr.hpp>
-
-
-#include "core/matrix/fbcsr_kernels.hpp"
-#include "core/test/utils.hpp"
-
-
-namespace {
-
-
-class Fbcsr : public ::testing::Test {
-protected:
-    using Arr = gko::Array<int>;
-    using Mtx = gko::matrix::Fbcsr<>;
-    using Vec = gko::matrix::Dense<>;
-    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
-    using ComplexMtx = gko::matrix::Fbcsr<std::complex<double>>;
-
-    Fbcsr() : mtx_size(532, 231), rand_engine(42) {}
-
-    void SetUp()
-    {
-        ref = gko::ReferenceExecutor::create();
-        omp = gko::OmpExecutor::create();
-    }
-
-    void TearDown()
-    {
-        if (omp != nullptr) {
-            ASSERT_NO_THROW(omp->synchronize());
-        }
-    }
-
-    template <typename MtxType>
-    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
-                                     int min_nnz_row)
-    {
-        return gko::test::generate_random_matrix<MtxType>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(min_nnz_row, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
-    }
-
-    void set_up_apply_data(int num_vectors = 1)
-    {
-        mtx = Mtx::create(ref);
-        mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[1], 1));
-        complex_mtx = ComplexMtx::create(ref);
-        complex_mtx->copy_from(
-            gen_mtx<ComplexVec>(mtx_size[0], mtx_size[1], 1));
-        square_mtx = Mtx::create(ref);
-        square_mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[0], 1));
-        expected = gen_mtx<Vec>(mtx_size[0], num_vectors, 1);
-        y = gen_mtx<Vec>(mtx_size[1], num_vectors, 1);
-        alpha = gko::initialize<Vec>({2.0}, ref);
-        beta = gko::initialize<Vec>({-1.0}, ref);
-        dmtx = Mtx::create(omp);
-        dmtx->copy_from(mtx.get());
-        complex_dmtx = ComplexMtx::create(omp);
-        complex_dmtx->copy_from(complex_mtx.get());
-        square_dmtx = Mtx::create(omp);
-        square_dmtx->copy_from(square_mtx.get());
-        dresult = Vec::create(omp);
-        dresult->copy_from(expected.get());
-        dy = Vec::create(omp);
-        dy->copy_from(y.get());
-        dalpha = Vec::create(omp);
-        dalpha->copy_from(alpha.get());
-        dbeta = Vec::create(omp);
-        dbeta->copy_from(beta.get());
-
-        std::vector<int> tmp(mtx->get_size()[0], 0);
-        auto rng = std::default_random_engine{};
-        std::iota(tmp.begin(), tmp.end(), 0);
-        std::shuffle(tmp.begin(), tmp.end(), rng);
-        std::vector<int> tmp2(mtx->get_size()[1], 0);
-        std::iota(tmp2.begin(), tmp2.end(), 0);
-        std::shuffle(tmp2.begin(), tmp2.end(), rng);
-        rpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{ref, tmp.begin(), tmp.end()});
-        drpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{omp, tmp.begin(), tmp.end()});
-        cpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{ref, tmp2.begin(), tmp2.end()});
-        dcpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{omp, tmp2.begin(), tmp2.end()});
-    }
-
-    struct matrix_pair {
-        std::unique_ptr<Mtx> ref;
-        std::unique_ptr<Mtx> omp;
-    };
-
-    matrix_pair gen_unsorted_mtx()
-    {
-        constexpr int min_nnz_per_row = 2;  // Must be at least 2
-        auto local_mtx_ref =
-            gen_mtx<Mtx>(mtx_size[0], mtx_size[1], min_nnz_per_row);
-        for (size_t row = 0; row < mtx_size[0]; ++row) {
-            const auto row_ptrs = local_mtx_ref->get_const_row_ptrs();
-            const auto start_row = row_ptrs[row];
-            auto col_idx = local_mtx_ref->get_col_idxs() + start_row;
-            auto vals = local_mtx_ref->get_values() + start_row;
-            const auto nnz_in_this_row = row_ptrs[row + 1] - row_ptrs[row];
-            auto swap_idx_dist =
-                std::uniform_int_distribution<>(0, nnz_in_this_row - 1);
-            // shuffle `nnz_in_this_row / 2` times
-            for (size_t perm = 0; perm < nnz_in_this_row; perm += 2) {
-                const auto idx1 = swap_idx_dist(rand_engine);
-                const auto idx2 = swap_idx_dist(rand_engine);
-                std::swap(col_idx[idx1], col_idx[idx2]);
-                std::swap(vals[idx1], vals[idx2]);
-            }
-        }
-        auto local_mtx_omp = Mtx::create(omp);
-        local_mtx_omp->copy_from(local_mtx_ref.get());
-
-        return {std::move(local_mtx_ref), std::move(local_mtx_omp)};
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::OmpExecutor> omp;
-
-    const gko::dim<2> mtx_size;
-    std::ranlux48 rand_engine;
-
-    std::unique_ptr<Mtx> mtx;
-    std::unique_ptr<ComplexMtx> complex_mtx;
-    std::unique_ptr<Mtx> square_mtx;
-    std::unique_ptr<Vec> expected;
-    std::unique_ptr<Vec> y;
-    std::unique_ptr<Vec> alpha;
-    std::unique_ptr<Vec> beta;
-
-    std::unique_ptr<Mtx> dmtx;
-    std::unique_ptr<ComplexMtx> complex_dmtx;
-    std::unique_ptr<Mtx> square_dmtx;
-    std::unique_ptr<Vec> dresult;
-    std::unique_ptr<Vec> dy;
-    std::unique_ptr<Vec> dalpha;
-    std::unique_ptr<Vec> dbeta;
-    std::unique_ptr<Arr> rpermute_idxs;
-    std::unique_ptr<Arr> drpermute_idxs;
-    std::unique_ptr<Arr> cpermute_idxs;
-    std::unique_ptr<Arr> dcpermute_idxs;
-};
-
-
-TEST_F(Fbcsr, SimpleApplyIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyToDenseMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(3);
-//
-//    mtx->apply(y.get(), expected.get());
-//    dmtx->apply(dy.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyToFbcsrMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    auto trans = mtx->transpose();
-//    auto d_trans = dmtx->transpose();
-//
-//    mtx->apply(alpha.get(), trans.get(), beta.get(), square_mtx.get());
-//    dmtx->apply(dalpha.get(), d_trans.get(), dbeta.get(), square_dmtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
-//    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-//    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
-//}
-
-
-TEST_F(Fbcsr, SimpleApplyToFbcsrMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    auto trans = mtx->transpose();
-//    auto d_trans = dmtx->transpose();
-//
-//    mtx->apply(trans.get(), square_mtx.get());
-//    dmtx->apply(d_trans.get(), square_dmtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
-//    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-//    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    auto a = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
-//    auto b = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
-//    auto da = Mtx::create(omp);
-//    auto db = Mtx::create(omp);
-//    da->copy_from(a.get());
-//    db->copy_from(b.get());
-//    auto id = gko::matrix::Identity<Mtx::value_type>::create(ref,
-//    mtx_size[1]); auto did =
-//    gko::matrix::Identity<Mtx::value_type>::create(omp, mtx_size[1]);
-//
-//    a->apply(alpha.get(), id.get(), beta.get(), b.get());
-//    da->apply(dalpha.get(), did.get(), dbeta.get(), db.get());
-//
-//    GKO_ASSERT_MTX_NEAR(b, db, 1e-14);
-//    GKO_ASSERT_MTX_EQ_SPARSITY(b, db);
-//    ASSERT_TRUE(db->is_sorted_by_column_index());
-//}
-
-
-TEST_F(Fbcsr, AdvancedApplyToDenseMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data(3);
-//
-//    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-//    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
-//
-//    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, TransposeIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//
-//    auto trans = mtx->transpose();
-//    auto d_trans = dmtx->transpose();
-//
-//    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(d_trans.get()),
-//                        static_cast<Mtx *>(trans.get()), 0.0);
-//    ASSERT_TRUE(static_cast<Mtx
-//    *>(d_trans.get())->is_sorted_by_column_index());
-//}
-
-
-TEST_F(Fbcsr, ConjugateTransposeIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//
-//    auto trans = complex_mtx->conj_transpose();
-//    auto d_trans = complex_dmtx->conj_transpose();
-//
-//    GKO_ASSERT_MTX_NEAR(static_cast<ComplexMtx *>(d_trans.get()),
-//                        static_cast<ComplexMtx *>(trans.get()), 0.0);
-//    ASSERT_TRUE(
-//        static_cast<ComplexMtx
-//        *>(d_trans.get())->is_sorted_by_column_index());
-//}
-
-
-TEST_F(Fbcsr, ConvertToCooIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    auto coo_mtx = gko::matrix::Coo<>::create(ref);
-//    auto dcoo_mtx = gko::matrix::Coo<>::create(omp);
-//
-//    mtx->convert_to(coo_mtx.get());
-//    dmtx->convert_to(dcoo_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToCooIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    auto coo_mtx = gko::matrix::Coo<>::create(ref);
-//    auto dcoo_mtx = gko::matrix::Coo<>::create(omp);
-//
-//    mtx->move_to(coo_mtx.get());
-//    dmtx->move_to(dcoo_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, ConvertToDenseIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    auto dense_mtx = gko::matrix::Dense<>::create(ref);
-//    auto ddense_mtx = gko::matrix::Dense<>::create(omp);
-//
-//    mtx->convert_to(dense_mtx.get());
-//    dmtx->convert_to(ddense_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(ddense_mtx.get(), dense_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToDenseIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    auto dense_mtx = gko::matrix::Dense<>::create(ref);
-//    auto ddense_mtx = gko::matrix::Dense<>::create(omp);
-//
-//    mtx->move_to(dense_mtx.get());
-//    dmtx->move_to(ddense_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(ddense_mtx.get(), dense_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, ConvertToSparsityFbcsrIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    auto sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(ref);
-//    auto d_sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(omp);
-//
-//    mtx->convert_to(sparsity_mtx.get());
-//    dmtx->convert_to(d_sparsity_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_sparsity_mtx.get(), sparsity_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToSparsityFbcsrIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    auto sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(ref);
-//    auto d_sparsity_mtx = gko::matrix::SparsityFbcsr<>::create(omp);
-//
-//    mtx->move_to(sparsity_mtx.get());
-//    dmtx->move_to(d_sparsity_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(d_sparsity_mtx.get(), sparsity_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, CalculatesNonzerosPerRow)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    gko::Array<gko::size_type> row_nnz(ref, mtx->get_size()[0]);
-//    gko::Array<gko::size_type> drow_nnz(omp, dmtx->get_size()[0]);
-//
-//    gko::kernels::reference::fbcsr::calculate_nonzeros_per_row(ref, mtx.get(),
-//                                                             &row_nnz);
-//    gko::kernels::omp::fbcsr::calculate_nonzeros_per_row(omp, dmtx.get(),
-//                                                       &drow_nnz);
-//
-//    GKO_ASSERT_ARRAY_EQ(row_nnz, drow_nnz);
-//}
-
-
-TEST_F(Fbcsr, ConvertToHybridIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Hybrid_type = gko::matrix::Hybrid<>;
-//    set_up_apply_data();
-//    auto hybrid_mtx = Hybrid_type::create(
-//        ref, std::make_shared<Hybrid_type::column_limit>(2));
-//    auto dhybrid_mtx = Hybrid_type::create(
-//        omp, std::make_shared<Hybrid_type::column_limit>(2));
-//
-//    mtx->convert_to(hybrid_mtx.get());
-//    dmtx->convert_to(dhybrid_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, MoveToHybridIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Hybrid_type = gko::matrix::Hybrid<>;
-//    set_up_apply_data();
-//    auto hybrid_mtx = Hybrid_type::create(
-//        ref, std::make_shared<Hybrid_type::column_limit>(2));
-//    auto dhybrid_mtx = Hybrid_type::create(
-//        omp, std::make_shared<Hybrid_type::column_limit>(2));
-//
-//    mtx->move_to(hybrid_mtx.get());
-//    dmtx->move_to(dhybrid_mtx.get());
-//
-//    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
-//}
-
-
-TEST_F(Fbcsr, IsRowPermutable)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    auto r_permute = mtx->row_permute(rpermute_idxs.get());
-//    auto dr_permute = dmtx->row_permute(drpermute_idxs.get());
-//
-//    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(r_permute.get()),
-//                        static_cast<Mtx *>(dr_permute.get()), 0);
-//}
-
-
-TEST_F(Fbcsr, IsColPermutable)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    auto c_permute = mtx->column_permute(cpermute_idxs.get());
-//    auto dc_permute = dmtx->column_permute(dcpermute_idxs.get());
-//
-//    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(c_permute.get()),
-//                        static_cast<Mtx *>(dc_permute.get()), 0);
-//}
-
-
-TEST_F(Fbcsr, IsInverseRowPermutable)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    auto inverse_r_permute = mtx->inverse_row_permute(rpermute_idxs.get());
-//    auto d_inverse_r_permute =
-//    dmtx->inverse_row_permute(drpermute_idxs.get());
-//
-//    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_r_permute.get()),
-//                        static_cast<Mtx *>(d_inverse_r_permute.get()), 0);
-//}
-
-
-TEST_F(Fbcsr, IsInverseColPermutable)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    auto inverse_c_permute = mtx->inverse_column_permute(cpermute_idxs.get());
-//    auto d_inverse_c_permute =
-//        dmtx->inverse_column_permute(dcpermute_idxs.get());
-//
-//    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_c_permute.get()),
-//                        static_cast<Mtx *>(d_inverse_c_permute.get()), 0);
-//}
-
-
-TEST_F(Fbcsr, RecognizeSortedMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//    bool is_sorted_omp{};
-//    bool is_sorted_ref{};
-//
-//    is_sorted_ref = mtx->is_sorted_by_column_index();
-//    is_sorted_omp = dmtx->is_sorted_by_column_index();
-//
-//    ASSERT_EQ(is_sorted_ref, is_sorted_omp);
-//}
-
-
-TEST_F(Fbcsr, RecognizeUnsortedMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto uns_mtx = gen_unsorted_mtx();
-//    bool is_sorted_omp{};
-//    bool is_sorted_ref{};
-//
-//    is_sorted_ref = uns_mtx.ref->is_sorted_by_column_index();
-//    is_sorted_omp = uns_mtx.omp->is_sorted_by_column_index();
-//
-//    ASSERT_EQ(is_sorted_ref, is_sorted_omp);
-//}
-
-
-TEST_F(Fbcsr, SortSortedMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//
-//    mtx->sort_by_column_index();
-//    dmtx->sort_by_column_index();
-//
-//    // Values must be unchanged, therefore, tolerance is `0`
-//    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 0);
-//}
-
-
-TEST_F(Fbcsr, SortUnsortedMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto uns_mtx = gen_unsorted_mtx();
-//
-//    uns_mtx.ref->sort_by_column_index();
-//    uns_mtx.omp->sort_by_column_index();
-//
-//    // Values must be unchanged, therefore, tolerance is `0`
-//    GKO_ASSERT_MTX_NEAR(uns_mtx.ref, uns_mtx.omp, 0);
-//}
-
-
-TEST_F(Fbcsr, ExtractDiagonalIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//
-//    auto diag = mtx->extract_diagonal();
-//    auto ddiag = dmtx->extract_diagonal();
-//
-//    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-//}
-
-
-TEST_F(Fbcsr, InplaceAbsoluteMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//
-//    mtx->compute_absolute_inplace();
-//    dmtx->compute_absolute_inplace();
-//
-//    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, OutplaceAbsoluteMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//
-//    auto abs_mtx = mtx->compute_absolute();
-//    auto dabs_mtx = dmtx->compute_absolute();
-//
-//    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//
-//    complex_mtx->compute_absolute_inplace();
-//    complex_dmtx->compute_absolute_inplace();
-//
-//    GKO_ASSERT_MTX_NEAR(complex_mtx, complex_dmtx, 1e-14);
-//}
-
-
-TEST_F(Fbcsr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
-GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    set_up_apply_data();
-//
-//    auto abs_mtx = complex_mtx->compute_absolute();
-//    auto dabs_mtx = complex_dmtx->compute_absolute();
-//
-//    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
-//}
-
-
-}  // namespace
diff --git a/reference/components/fbcsr_spgeam.hpp b/reference/components/fbcsr_spgeam.hpp
deleted file mode 100644
index e4a06532ed3..00000000000
--- a/reference/components/fbcsr_spgeam.hpp
+++ /dev/null
@@ -1,31 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index 042357d0429..8060d777ec2 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -43,11 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/blockutils.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/ell.hpp>
-#include <ginkgo/core/matrix/hybrid.hpp>
-#include <ginkgo/core/matrix/sellp.hpp>
 
 
 #include "core/base/allocator.hpp"
@@ -55,7 +51,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/fixed_block.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/fbcsr_builder.hpp"
-#include "reference/components/fbcsr_spgeam.hpp"
 #include "reference/components/format_conversion.hpp"
 
 
@@ -192,18 +187,6 @@ template <typename ValueType, typename IndexType>
 void spgemm_accumulate_row(map<IndexType, ValueType> &cols,
                            const matrix::Fbcsr<ValueType, IndexType> *c,
                            ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto row_ptrs = c->get_const_row_ptrs();
-//    auto col_idxs = c->get_const_col_idxs();
-//    auto vals = c->get_const_values();
-//    for (size_type c_nz = row_ptrs[row]; c_nz < size_type(row_ptrs[row + 1]);
-//         ++c_nz) {
-//        auto c_col = col_idxs[c_nz];
-//        auto c_val = vals[c_nz];
-//        cols[c_col] += scale * c_val;
-//    }
-//}
 
 
 template <typename ValueType, typename IndexType>
@@ -211,27 +194,6 @@ void spgemm_accumulate_row2(map<IndexType, ValueType> &cols,
                             const matrix::Fbcsr<ValueType, IndexType> *a,
                             const matrix::Fbcsr<ValueType, IndexType> *b,
                             ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto a_row_ptrs = a->get_const_row_ptrs();
-//    auto a_col_idxs = a->get_const_col_idxs();
-//    auto a_vals = a->get_const_values();
-//    auto b_row_ptrs = b->get_const_row_ptrs();
-//    auto b_col_idxs = b->get_const_col_idxs();
-//    auto b_vals = b->get_const_values();
-//    for (size_type a_nz = a_row_ptrs[row];
-//         a_nz < size_type(a_row_ptrs[row + 1]); ++a_nz) {
-//        auto a_col = a_col_idxs[a_nz];
-//        auto a_val = a_vals[a_nz];
-//        auto b_row = a_col;
-//        for (size_type b_nz = b_row_ptrs[b_row];
-//             b_nz < size_type(b_row_ptrs[b_row + 1]); ++b_nz) {
-//            auto b_col = b_col_idxs[b_nz];
-//            auto b_val = b_vals[b_nz];
-//            cols[b_col] += scale * a_val * b_val;
-//        }
-//    }
-//}
 
 
 template <typename ValueType, typename IndexType>
@@ -239,48 +201,6 @@ void spgemm(std::shared_ptr<const ReferenceExecutor> exec,
             const matrix::Fbcsr<ValueType, IndexType> *a,
             const matrix::Fbcsr<ValueType, IndexType> *b,
             matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = a->get_size()[0];
-//
-//    // first sweep: count nnz for each row
-//    auto c_row_ptrs = c->get_row_ptrs();
-//
-//    unordered_set<IndexType> local_col_idxs(exec);
-//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
-//        local_col_idxs.clear();
-//        spgemm_insert_row2(local_col_idxs, a, b, a_row);
-//        c_row_ptrs[a_row] = local_col_idxs.size();
-//    }
-//
-//    // build row pointers
-//    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
-//
-//    // second sweep: accumulate non-zeros
-//    auto new_nnz = c_row_ptrs[num_rows];
-//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
-//    auto &c_col_idxs_array = c_builder.get_col_idx_array();
-//    auto &c_vals_array = c_builder.get_value_array();
-//    c_col_idxs_array.resize_and_reset(new_nnz);
-//    c_vals_array.resize_and_reset(new_nnz);
-//    auto c_col_idxs = c_col_idxs_array.get_data();
-//    auto c_vals = c_vals_array.get_data();
-//
-//    map<IndexType, ValueType> local_row_nzs(exec);
-//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
-//        local_row_nzs.clear();
-//        spgemm_accumulate_row2(local_row_nzs, a, b, one<ValueType>(), a_row);
-//        // store result
-//        auto c_nz = c_row_ptrs[a_row];
-//        for (auto pair : local_row_nzs) {
-//            c_col_idxs[c_nz] = pair.first;
-//            c_vals[c_nz] = pair.second;
-//            ++c_nz;
-//        }
-//    }
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEMM_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -292,122 +212,7 @@ void advanced_spgemm(std::shared_ptr<const ReferenceExecutor> exec,
                      const matrix::Fbcsr<ValueType, IndexType> *d,
                      matrix::Fbcsr<ValueType, IndexType> *c)
     GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = a->get_size()[0];
-//    auto valpha = alpha->at(0, 0);
-//    auto vbeta = beta->at(0, 0);
-//
-//    // first sweep: count nnz for each row
-//    auto c_row_ptrs = c->get_row_ptrs();
-//
-//    unordered_set<IndexType> local_col_idxs(exec);
-//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
-//        local_col_idxs.clear();
-//        spgemm_insert_row(local_col_idxs, d, a_row);
-//        spgemm_insert_row2(local_col_idxs, a, b, a_row);
-//        c_row_ptrs[a_row] = local_col_idxs.size();
-//    }
-//
-//    // build row pointers
-//    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
-//
-//    // second sweep: accumulate non-zeros
-//    auto new_nnz = c_row_ptrs[num_rows];
-//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
-//    auto &c_col_idxs_array = c_builder.get_col_idx_array();
-//    auto &c_vals_array = c_builder.get_value_array();
-//    c_col_idxs_array.resize_and_reset(new_nnz);
-//    c_vals_array.resize_and_reset(new_nnz);
-//    auto c_col_idxs = c_col_idxs_array.get_data();
-//    auto c_vals = c_vals_array.get_data();
-//
-//    map<IndexType, ValueType> local_row_nzs(exec);
-//    for (size_type a_row = 0; a_row < num_rows; ++a_row) {
-//        local_row_nzs.clear();
-//        spgemm_accumulate_row(local_row_nzs, d, vbeta, a_row);
-//        spgemm_accumulate_row2(local_row_nzs, a, b, valpha, a_row);
-//        // store result
-//        auto c_nz = c_row_ptrs[a_row];
-//        for (auto pair : local_row_nzs) {
-//            c_col_idxs[c_nz] = pair.first;
-//            c_vals[c_nz] = pair.second;
-//            ++c_nz;
-//        }
-//    }
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_ADVANCED_SPGEMM_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void spgeam(std::shared_ptr<const ReferenceExecutor> exec,
-            const matrix::Dense<ValueType> *alpha,
-            const matrix::Fbcsr<ValueType, IndexType> *a,
-            const matrix::Dense<ValueType> *beta,
-            const matrix::Fbcsr<ValueType, IndexType> *b,
-            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = a->get_size()[0];
-//    auto valpha = alpha->at(0, 0);
-//    auto vbeta = beta->at(0, 0);
-//
-//    // first sweep: count nnz for each row
-//    auto c_row_ptrs = c->get_row_ptrs();
-//
-//    abstract_spgeam(
-//        a, b, [](IndexType) { return IndexType{}; },
-//        [](IndexType, IndexType, ValueType, ValueType, IndexType &nnz) {
-//            ++nnz;
-//        },
-//        [&](IndexType row, IndexType nnz) { c_row_ptrs[row] = nnz; });
-//
-//    // build row pointers
-//    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
-//
-//    // second sweep: accumulate non-zeros
-//    auto new_nnz = c_row_ptrs[num_rows];
-//    matrix::FbcsrBuilder<ValueType, IndexType> c_builder{c};
-//    auto &c_col_idxs_array = c_builder.get_col_idx_array();
-//    auto &c_vals_array = c_builder.get_value_array();
-//    c_col_idxs_array.resize_and_reset(new_nnz);
-//    c_vals_array.resize_and_reset(new_nnz);
-//    auto c_col_idxs = c_col_idxs_array.get_data();
-//    auto c_vals = c_vals_array.get_data();
-//
-//    abstract_spgeam(
-//        a, b, [&](IndexType row) { return c_row_ptrs[row]; },
-//        [&](IndexType, IndexType col, ValueType a_val, ValueType b_val,
-//            IndexType &nz) {
-//            c_vals[nz] = valpha * a_val + vbeta * b_val;
-//            c_col_idxs[nz] = col;
-//            ++nz;
-//        },
-//        [](IndexType, IndexType) {});
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPGEAM_KERNEL);
-
-
-// template <typename IndexType>
-// void convert_row_ptrs_to_idxs(std::shared_ptr<const ReferenceExecutor> exec,
-//                               const IndexType *ptrs, size_type num_rows,
-//                               IndexType *idxs)
-// {
-//    convert_ptrs_to_idxs(ptrs, num_rows, idxs);
-// }
-
-
-template <typename ValueType, typename IndexType>
-void convert_to_coo(std::shared_ptr<const ReferenceExecutor> exec,
-                    const matrix::Fbcsr<ValueType, IndexType> *source,
-                    matrix::Coo<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_COO_KERNEL);
 
 template <typename ValueType, typename IndexType>
 void convert_to_dense(const std::shared_ptr<const ReferenceExecutor> exec,
@@ -502,151 +307,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void convert_to_sellp(std::shared_ptr<const ReferenceExecutor> exec,
-                      const matrix::Fbcsr<ValueType, IndexType> *source,
-                      matrix::Sellp<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = result->get_size()[0];
-//    auto num_cols = result->get_size()[1];
-//    auto vals = result->get_values();
-//    auto col_idxs = result->get_col_idxs();
-//    auto slice_lengths = result->get_slice_lengths();
-//    auto slice_sets = result->get_slice_sets();
-//    auto slice_size = (result->get_slice_size() == 0)
-//                          ? matrix::default_slice_size
-//                          : result->get_slice_size();
-//    auto stride_factor = (result->get_stride_factor() == 0)
-//                             ? matrix::default_stride_factor
-//                             : result->get_stride_factor();
-//
-//    const auto source_row_ptrs = source->get_const_row_ptrs();
-//    const auto source_col_idxs = source->get_const_col_idxs();
-//    const auto source_values = source->get_const_values();
-//
-//    int slice_num = ceildiv(num_rows, slice_size);
-//    slice_sets[0] = 0;
-//    for (size_type slice = 0; slice < slice_num; slice++) {
-//        if (slice > 0) {
-//            slice_sets[slice] =
-//                slice_sets[slice - 1] + slice_lengths[slice - 1];
-//        }
-//        slice_lengths[slice] = 0;
-//        for (size_type row = 0; row < slice_size; row++) {
-//            size_type global_row = slice * slice_size + row;
-//            if (global_row >= num_rows) {
-//                break;
-//            }
-//            slice_lengths[slice] =
-//                (slice_lengths[slice] >
-//                 source_row_ptrs[global_row + 1] -
-//                 source_row_ptrs[global_row])
-//                    ? slice_lengths[slice]
-//                    : source_row_ptrs[global_row + 1] -
-//                          source_row_ptrs[global_row];
-//        }
-//        slice_lengths[slice] =
-//            stride_factor * ceildiv(slice_lengths[slice], stride_factor);
-//        for (size_type row = 0; row < slice_size; row++) {
-//            size_type global_row = slice * slice_size + row;
-//            if (global_row >= num_rows) {
-//                break;
-//            }
-//            size_type sellp_ind = slice_sets[slice] * slice_size + row;
-//            for (size_type fbcsr_ind = source_row_ptrs[global_row];
-//                 fbcsr_ind < source_row_ptrs[global_row + 1]; fbcsr_ind++) {
-//                vals[sellp_ind] = source_values[fbcsr_ind];
-//                col_idxs[sellp_ind] = source_col_idxs[fbcsr_ind];
-//                sellp_ind += slice_size;
-//            }
-//            for (size_type i = sellp_ind;
-//                 i <
-//                 (slice_sets[slice] + slice_lengths[slice]) * slice_size +
-//                 row; i += slice_size) {
-//                col_idxs[i] = 0;
-//                vals[i] = zero<ValueType>();
-//            }
-//        }
-//    }
-//    if (slice_num > 0) {
-//        slice_sets[slice_num] =
-//            slice_sets[slice_num - 1] + slice_lengths[slice_num - 1];
-//    }
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_SELLP_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void calculate_total_cols(std::shared_ptr<const ReferenceExecutor> exec,
-                          const matrix::Fbcsr<ValueType, IndexType> *source,
-                          size_type *result, size_type stride_factor,
-                          size_type slice_size) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    size_type total_cols = 0;
-//    const auto num_rows = source->get_size()[0];
-//    const auto slice_num = ceildiv(num_rows, slice_size);
-//
-//    const auto row_ptrs = source->get_const_row_ptrs();
-//
-//    for (size_type slice = 0; slice < slice_num; slice++) {
-//        IndexType max_nnz_per_row_in_this_slice = 0;
-//        for (size_type row = 0;
-//             row < slice_size && row + slice * slice_size < num_rows; row++) {
-//            size_type global_row = slice * slice_size + row;
-//            max_nnz_per_row_in_this_slice =
-//                max(row_ptrs[global_row + 1] - row_ptrs[global_row],
-//                    max_nnz_per_row_in_this_slice);
-//        }
-//        total_cols += ceildiv(max_nnz_per_row_in_this_slice, stride_factor) *
-//                      stride_factor;
-//    }
-//
-//    *result = total_cols;
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CALCULATE_TOTAL_COLS_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void convert_to_ell(std::shared_ptr<const ReferenceExecutor> exec,
-                    const matrix::Fbcsr<ValueType, IndexType> *source,
-                    matrix::Ell<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto num_rows = source->get_size()[0];
-//    const auto num_cols = source->get_size()[1];
-//    const auto vals = source->get_const_values();
-//    const auto col_idxs = source->get_const_col_idxs();
-//    const auto row_ptrs = source->get_const_row_ptrs();
-//
-//    const auto num_stored_elements_per_row =
-//        result->get_num_stored_elements_per_row();
-//
-//    for (size_type row = 0; row < num_rows; row++) {
-//        for (size_type i = 0; i < num_stored_elements_per_row; i++) {
-//            result->val_at(row, i) = zero<ValueType>();
-//            result->col_at(row, i) = 0;
-//        }
-//        for (size_type col_idx = 0; col_idx < row_ptrs[row + 1] -
-//        row_ptrs[row];
-//             col_idx++) {
-//            result->val_at(row, col_idx) = vals[row_ptrs[row] + col_idx];
-//            result->col_at(row, col_idx) = col_idxs[row_ptrs[row] + col_idx];
-//        }
-//    }
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_ELL_KERNEL);
-
-
 template <typename ValueType, typename IndexType, typename UnaryOperator,
           bool transpose_blocks>
 void convert_fbcsr_to_fbcsc(const size_type num_blk_rows, const int blksz,
@@ -755,208 +415,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void convert_to_hybrid(std::shared_ptr<const ReferenceExecutor> exec,
-                       const matrix::Fbcsr<ValueType, IndexType> *source,
-                       matrix::Hybrid<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto num_rows = result->get_size()[0];
-//    auto num_cols = result->get_size()[1];
-//    auto strategy = result->get_strategy();
-//    auto ell_lim = strategy->get_ell_num_stored_elements_per_row();
-//    auto coo_lim = strategy->get_coo_nnz();
-//    auto coo_val = result->get_coo_values();
-//    auto coo_col = result->get_coo_col_idxs();
-//    auto coo_row = result->get_coo_row_idxs();
-//
-//    // Initial Hybrid Matrix
-//    for (size_type i = 0; i < result->get_ell_num_stored_elements_per_row();
-//         i++) {
-//        for (size_type j = 0; j < result->get_ell_stride(); j++) {
-//            result->ell_val_at(j, i) = zero<ValueType>();
-//            result->ell_col_at(j, i) = 0;
-//        }
-//    }
-//    for (size_type i = 0; i < result->get_coo_num_stored_elements(); i++) {
-//        coo_val[i] = zero<ValueType>();
-//        coo_col[i] = 0;
-//        coo_row[i] = 0;
-//    }
-//
-//    const auto fbcsr_row_ptrs = source->get_const_row_ptrs();
-//    const auto fbcsr_vals = source->get_const_values();
-//    size_type fbcsr_idx = 0;
-//    size_type coo_idx = 0;
-//    for (IndexType row = 0; row < num_rows; row++) {
-//        size_type ell_idx = 0;
-//        while (fbcsr_idx < fbcsr_row_ptrs[row + 1]) {
-//            const auto val = fbcsr_vals[fbcsr_idx];
-//            if (ell_idx < ell_lim) {
-//                result->ell_val_at(row, ell_idx) = val;
-//                result->ell_col_at(row, ell_idx) =
-//                    source->get_const_col_idxs()[fbcsr_idx];
-//                ell_idx++;
-//            } else {
-//                coo_val[coo_idx] = val;
-//                coo_col[coo_idx] = source->get_const_col_idxs()[fbcsr_idx];
-//                coo_row[coo_idx] = row;
-//                coo_idx++;
-//            }
-//            fbcsr_idx++;
-//        }
-//    }
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_CONVERT_TO_HYBRID_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void row_permute_impl(std::shared_ptr<const ReferenceExecutor> exec,
-                      const IndexType *const permutation_indices,
-                      const matrix::Fbcsr<ValueType, IndexType> *orig,
-                      matrix::Fbcsr<ValueType, IndexType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto perm = permutation_indices->get_const_data();
-//    auto orig_row_ptrs = orig->get_const_row_ptrs();
-//    auto orig_col_idxs = orig->get_const_col_idxs();
-//    auto orig_vals = orig->get_const_values();
-//    auto rp_row_ptrs = row_permuted->get_row_ptrs();
-//    auto rp_col_idxs = row_permuted->get_col_idxs();
-//    auto rp_vals = row_permuted->get_values();
-//    size_type num_rows = orig->get_size()[0];
-//    size_type num_nnz = orig->get_num_stored_elements();
-//
-//    size_type cur_ptr = 0;
-//    rp_row_ptrs[0] = cur_ptr;
-//    vector<size_type> orig_num_nnz_per_row(num_rows, 0, exec);
-//    for (size_type row = 0; row < num_rows; ++row) {
-//        orig_num_nnz_per_row[row] = orig_row_ptrs[row + 1] -
-//        orig_row_ptrs[row];
-//    }
-//    for (size_type row = 0; row < num_rows; ++row) {
-//        rp_row_ptrs[row + 1] =
-//            rp_row_ptrs[row] + orig_num_nnz_per_row[perm[row]];
-//    }
-//    rp_row_ptrs[num_rows] = orig_row_ptrs[num_rows];
-//    for (size_type row = 0; row < num_rows; ++row) {
-//        auto new_row = perm[row];
-//        auto new_k = orig_row_ptrs[new_row];
-//        for (size_type k = rp_row_ptrs[row];
-//             k < size_type(rp_row_ptrs[row + 1]); ++k) {
-//            rp_col_idxs[k] = orig_col_idxs[new_k];
-//            rp_vals[k] = orig_vals[new_k];
-//            new_k++;
-//        }
-//    }
-//}
-
-
-template <typename ValueType, typename IndexType>
-void row_permute(std::shared_ptr<const ReferenceExecutor> exec,
-                 const IndexType *const permutation_indices,
-                 const matrix::Fbcsr<ValueType, IndexType> *orig,
-                 matrix::Fbcsr<ValueType, IndexType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    row_permute_impl(exec, permutation_indices, orig, row_permuted);
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_ROW_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_row_permute(std::shared_ptr<const ReferenceExecutor> exec,
-                         const IndexType *const permutation_indices,
-                         const matrix::Fbcsr<ValueType, IndexType> *orig,
-                         matrix::Fbcsr<ValueType, IndexType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto perm = permutation_indices->get_const_data();
-//    Array<IndexType> inv_perm(*permutation_indices);
-//    auto iperm = inv_perm.get_data();
-//    for (size_type ind = 0; ind < inv_perm.get_num_elems(); ++ind) {
-//        iperm[perm[ind]] = ind;
-//    }
-//
-//    row_permute_impl(exec, &inv_perm, orig, row_permuted);
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_INVERSE_ROW_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void column_permute_impl(const IndexType *const permutation_indices,
-                         const matrix::Fbcsr<ValueType, IndexType> *const orig,
-                         matrix::Fbcsr<ValueType, IndexType>
-                             *const column_permuted) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto perm = permutation_indices->get_const_data();
-//    auto orig_row_ptrs = orig->get_const_row_ptrs();
-//    auto orig_col_idxs = orig->get_const_col_idxs();
-//    auto orig_vals = orig->get_const_values();
-//    auto cp_row_ptrs = column_permuted->get_row_ptrs();
-//    auto cp_col_idxs = column_permuted->get_col_idxs();
-//    auto cp_vals = column_permuted->get_values();
-//    auto num_nnz = orig->get_num_stored_elements();
-//    size_type num_rows = orig->get_size()[0];
-//    size_type num_cols = orig->get_size()[1];
-//
-//    for (size_type row = 0; row < num_rows; ++row) {
-//        cp_row_ptrs[row] = orig_row_ptrs[row];
-//        for (size_type k = orig_row_ptrs[row];
-//             k < size_type(orig_row_ptrs[row + 1]); ++k) {
-//            cp_col_idxs[k] = perm[orig_col_idxs[k]];
-//            cp_vals[k] = orig_vals[k];
-//        }
-//    }
-//    cp_row_ptrs[num_rows] = orig_row_ptrs[num_rows];
-//}
-
-
-template <typename ValueType, typename IndexType>
-void column_permute(std::shared_ptr<const ReferenceExecutor> exec,
-                    const IndexType *permutation_indices,
-                    const matrix::Fbcsr<ValueType, IndexType> *orig,
-                    matrix::Fbcsr<ValueType, IndexType> *column_permuted)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto perm = permutation_indices->get_const_data();
-//    Array<IndexType> inv_perm(*permutation_indices);
-//    auto iperm = inv_perm.get_data();
-//    for (size_type ind = 0; ind < inv_perm.get_num_elems(); ++ind) {
-//        iperm[perm[ind]] = ind;
-//    }
-//    column_permute_impl(&inv_perm, orig, column_permuted);
-//}
-
-
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(
-    std::shared_ptr<const ReferenceExecutor> exec,
-    const IndexType *const permutation_indices,
-    const matrix::Fbcsr<ValueType, IndexType> *const orig,
-    matrix::Fbcsr<ValueType, IndexType> *const column_permuted)
-    GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    column_permute_impl(permutation_indices, orig, column_permuted);
-//}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_INVERSE_COLUMN_PERMUTE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void calculate_nonzeros_per_row(
     std::shared_ptr<const ReferenceExecutor> exec,
@@ -981,20 +439,6 @@ template <typename ValueType, typename IndexType>
 void sort_by_column_index(std::shared_ptr<const ReferenceExecutor> exec,
                           matrix::Fbcsr<ValueType, IndexType> *to_sort)
     GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto values = to_sort->get_values();
-//    auto row_ptrs = to_sort->get_row_ptrs();
-//    auto col_idxs = to_sort->get_col_idxs();
-//    const auto number_rows = to_sort->get_size()[0];
-//    for (size_type i = 0; i < number_rows; ++i) {
-//        auto start_row_idx = row_ptrs[i];
-//        auto row_nnz = row_ptrs[i + 1] - start_row_idx;
-//        auto helper = detail::IteratorFactory<IndexType, ValueType>(
-//            col_idxs + start_row_idx, values + start_row_idx, row_nnz);
-//        std::sort(helper.begin(), helper.end());
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 4b75c240236..367d643af14 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -44,14 +44,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/diagonal.hpp>
-#include <ginkgo/core/matrix/ell.hpp>
-#include <ginkgo/core/matrix/hybrid.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
 #include <ginkgo/core/matrix/matrix_strategies.hpp>
-#include <ginkgo/core/matrix/sellp.hpp>
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
@@ -75,12 +71,8 @@ class Fbcsr : public ::testing::Test {
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using Mtx = gko::matrix::Fbcsr<value_type, index_type>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
-    using Coo = gko::matrix::Coo<value_type, index_type>;
     using Dense = gko::matrix::Dense<value_type>;
-    using Sellp = gko::matrix::Sellp<value_type, index_type>;
     using SparCsr = gko::matrix::SparsityCsr<value_type, index_type>;
-    using Ell = gko::matrix::Ell<value_type, index_type>;
-    using Hybrid = gko::matrix::Hybrid<value_type, index_type>;
     using Diag = gko::matrix::Diagonal<value_type>;
     using Vec = gko::matrix::Dense<value_type>;
 
@@ -93,7 +85,6 @@ class Fbcsr : public ::testing::Test {
           refmtx(fbsample.generate_fbcsr()),
           refcsrmtx(fbsample.generate_csr()),
           refdenmtx(fbsample.generate_dense()),
-          refcoomtx(fbsample.generate_coo()),
           refspcmtx(fbsample.generate_sparsity_csr()),
           mtx2(fbsample2.generate_fbcsr()),
           m2diag(fbsample2.extract_diagonal()),
@@ -116,20 +107,6 @@ class Fbcsr : public ::testing::Test {
         }
     }
 
-    void assert_equal_to_mtx(const Coo *m)
-    {
-        ASSERT_EQ(m->get_size(), refcoomtx->get_size());
-        ASSERT_EQ(m->get_num_stored_elements(),
-                  refcoomtx->get_num_stored_elements());
-        for (index_type i = 0; i < m->get_num_stored_elements(); i++) {
-            ASSERT_EQ(m->get_const_row_idxs()[i],
-                      refcoomtx->get_const_row_idxs[i]);
-            ASSERT_EQ(m->get_const_col_idxs()[i],
-                      refcoomtx->get_const_col_idxs[i]);
-            ASSERT_EQ(m->get_const_values()[i], refcoomtx->get_const_values[i]);
-        }
-    }
-
     void assert_equal_to_mtx(const SparCsr *m)
     {
         ASSERT_EQ(m->get_size(), refspcmtx->get_size());
@@ -152,7 +129,6 @@ class Fbcsr : public ::testing::Test {
     const std::unique_ptr<const Mtx> refmtx;
     const std::unique_ptr<const Csr> refcsrmtx;
     const std::unique_ptr<const Dense> refdenmtx;
-    const std::unique_ptr<const Coo> refcoomtx;
     const std::unique_ptr<const SparCsr> refspcmtx;
     const std::unique_ptr<const Mtx> mtx2;
     const std::unique_ptr<const Diag> m2diag;
@@ -318,111 +294,6 @@ TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseMatrix)
 }
 
 
-// TYPED_TEST(Fbcsr, AppliesToFbcsrMatrix)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using T = typename TestFixture::value_type;
-//    this->mtx->apply(this->mtx3_unsorted.get(), this->mtx2.get());
-//
-//    ASSERT_EQ(this->mtx2->get_size(), gko::dim<2>(2, 3));
-//    ASSERT_EQ(this->mtx2->get_num_stored_elements(), 6);
-//    ASSERT_TRUE(this->mtx2->is_sorted_by_column_index());
-//    auto r = this->mtx2->get_const_row_ptrs();
-//    auto c = this->mtx2->get_const_col_idxs();
-//    auto v = this->mtx2->get_const_values();
-//    // 13  5 31
-//    // 15  5 40
-//    EXPECT_EQ(r[0], 0);
-//    EXPECT_EQ(r[1], 3);
-//    EXPECT_EQ(r[2], 6);
-//    EXPECT_EQ(c[0], 0);
-//    EXPECT_EQ(c[1], 1);
-//    EXPECT_EQ(c[2], 2);
-//    EXPECT_EQ(c[3], 0);
-//    EXPECT_EQ(c[4], 1);
-//    EXPECT_EQ(c[5], 2);
-//    EXPECT_EQ(v[0], T{13});
-//    EXPECT_EQ(v[1], T{5});
-//    EXPECT_EQ(v[2], T{31});
-//    EXPECT_EQ(v[3], T{15});
-//    EXPECT_EQ(v[4], T{5});
-//    EXPECT_EQ(v[5], T{40});
-//}
-
-
-// TYPED_TEST(Fbcsr, AppliesLinearCombinationToFbcsrMatrix)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Vec = typename TestFixture::Vec;
-//    using T = typename TestFixture::value_type;
-//    auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
-//    auto beta = gko::initialize<Vec>({2.0}, this->exec);
-//
-//    this->mtx->apply(alpha.get(), this->mtx3_unsorted.get(), beta.get(),
-//                     this->mtx2.get());
-//
-//    ASSERT_EQ(this->mtx2->get_size(), gko::dim<2>(2, 3));
-//    ASSERT_EQ(this->mtx2->get_num_stored_elements(), 6);
-//    ASSERT_TRUE(this->mtx2->is_sorted_by_column_index());
-//    auto r = this->mtx2->get_const_row_ptrs();
-//    auto c = this->mtx2->get_const_col_idxs();
-//    auto v = this->mtx2->get_const_values();
-//    // -11 1 -27
-//    // -15 5 -40
-//    EXPECT_EQ(r[0], 0);
-//    EXPECT_EQ(r[1], 3);
-//    EXPECT_EQ(r[2], 6);
-//    EXPECT_EQ(c[0], 0);
-//    EXPECT_EQ(c[1], 1);
-//    EXPECT_EQ(c[2], 2);
-//    EXPECT_EQ(c[3], 0);
-//    EXPECT_EQ(c[4], 1);
-//    EXPECT_EQ(c[5], 2);
-//    EXPECT_EQ(v[0], T{-11});
-//    EXPECT_EQ(v[1], T{1});
-//    EXPECT_EQ(v[2], T{-27});
-//    EXPECT_EQ(v[3], T{-15});
-//    EXPECT_EQ(v[4], T{5});
-//    EXPECT_EQ(v[5], T{-40});
-//}
-
-
-// TYPED_TEST(Fbcsr, AppliesLinearCombinationToIdentityMatrix)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using T = typename TestFixture::value_type;
-//    using Vec = typename TestFixture::Vec;
-//    using Mtx = typename TestFixture::Mtx;
-//    auto alpha = gko::initialize<Vec>({-3.0}, this->exec);
-//    auto beta = gko::initialize<Vec>({2.0}, this->exec);
-//    auto a = gko::initialize<Mtx>(
-//        {I<T>{2.0, 0.0, 3.0}, I<T>{0.0, 1.0, -1.5}, I<T>{0.0, -2.0, 0.0},
-//         I<T>{5.0, 0.0, 0.0}, I<T>{1.0, 0.0, 4.0}, I<T>{2.0, -2.0, 0.0},
-//         I<T>{0.0, 0.0, 0.0}},
-//        this->exec);
-//    auto b = gko::initialize<Mtx>(
-//        {I<T>{2.0, -2.0, 0.0}, I<T>{1.0, 0.0, 4.0}, I<T>{2.0, 0.0, 3.0},
-//         I<T>{0.0, 1.0, -1.5}, I<T>{1.0, 0.0, 0.0}, I<T>{0.0, 0.0, 0.0},
-//         I<T>{0.0, 0.0, 0.0}},
-//        this->exec);
-//    auto expect = gko::initialize<Mtx>(
-//        {I<T>{-2.0, -4.0, -9.0}, I<T>{2.0, -3.0, 12.5}, I<T>{4.0, 6.0, 6.0},
-//         I<T>{-15.0, 2.0, -3.0}, I<T>{-1.0, 0.0, -12.0}, I<T>{-6.0, 6.0, 0.0},
-//         I<T>{0.0, 0.0, 0.0}},
-//        this->exec);
-//    auto id = gko::matrix::Identity<T>::create(this->exec, a->get_size()[1]);
-//
-//    a->apply(gko::lend(alpha), gko::lend(id), gko::lend(beta), gko::lend(b));
-//
-//    GKO_ASSERT_MTX_NEAR(b, expect, r<T>::value);
-//    GKO_ASSERT_MTX_EQ_SPARSITY(b, expect);
-//    ASSERT_TRUE(b->is_sorted_by_column_index());
-//}
-
-
 TYPED_TEST(Fbcsr, ApplyFailsOnWrongInnerDimension)
 {
     using Vec = typename TestFixture::Vec;
@@ -545,57 +416,6 @@ TYPED_TEST(Fbcsr, MovesToCsr)
 }
 
 
-// TYPED_TEST(Fbcsr, ConvertsToCoo)
-// {
-//    using Coo = typename TestFixture::Coo;
-//    auto coo_mtx = Coo::create(this->mtx->get_executor());
-
-//    this->mtx->convert_to(coo_mtx.get());
-
-//    this->assert_equal_to_mtx(coo_mtx.get());
-// }
-
-
-// TYPED_TEST(Fbcsr, MovesToCoo)
-// {
-//    using Coo = typename TestFixture::Coo;
-//    auto coo_mtx = Coo::create(this->mtx->get_executor());
-
-//    this->mtx->move_to(coo_mtx.get());
-
-//    this->assert_equal_to_mtx(coo_mtx.get());
-// }
-
-
-// TYPED_TEST(Fbcsr, ConvertsToSellp)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Sellp = typename TestFixture::Sellp;
-//    auto sellp_mtx = Sellp::create(this->mtx->get_executor());
-//
-//    this->mtx->convert_to(sellp_mtx.get());
-//
-//    this->assert_equal_to_mtx(sellp_mtx.get());
-//}
-
-
-// TYPED_TEST(Fbcsr, MovesToSellp)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Sellp = typename TestFixture::Sellp;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    auto sellp_mtx = Sellp::create(this->mtx->get_executor());
-//    auto fbcsr_ref = Fbcsr::create(this->mtx->get_executor());
-//
-//    fbcsr_ref->copy_from(this->mtx.get());
-//    fbcsr_ref->move_to(sellp_mtx.get());
-//
-//    this->assert_equal_to_mtx(sellp_mtx.get());
-//}
-
-
 TYPED_TEST(Fbcsr, ConvertsToSparsityCsr)
 {
     using SparsityCsr = typename TestFixture::SparCsr;
@@ -621,68 +441,6 @@ TYPED_TEST(Fbcsr, MovesToSparsityCsr)
 }
 
 
-// TYPED_TEST(Fbcsr, ConvertsToHybridAutomatically)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Hybrid = typename TestFixture::Hybrid;
-//    auto hybrid_mtx = Hybrid::create(this->mtx->get_executor());
-//
-//    this->mtx->convert_to(hybrid_mtx.get());
-//
-//    this->assert_equal_to_mtx(hybrid_mtx.get());
-//}
-
-
-// TYPED_TEST(Fbcsr, MovesToHybridAutomatically)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Hybrid = typename TestFixture::Hybrid;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    auto hybrid_mtx = Hybrid::create(this->mtx->get_executor());
-//    auto fbcsr_ref = Fbcsr::create(this->mtx->get_executor());
-//
-//    fbcsr_ref->copy_from(this->mtx.get());
-//    fbcsr_ref->move_to(hybrid_mtx.get());
-//
-//    this->assert_equal_to_mtx(hybrid_mtx.get());
-//}
-
-
-// TYPED_TEST(Fbcsr, ConvertsToHybridByColumn2)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Hybrid = typename TestFixture::Hybrid;
-//    auto hybrid_mtx =
-//        Hybrid::create(this->mtx2->get_executor(),
-//                       std::make_shared<typename Hybrid::column_limit>(2));
-//
-//    this->mtx2->convert_to(hybrid_mtx.get());
-//
-//    this->assert_equal_to_mtx2(hybrid_mtx.get());
-//}
-
-
-// TYPED_TEST(Fbcsr, MovesToHybridByColumn2)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Hybrid = typename TestFixture::Hybrid;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    auto hybrid_mtx =
-//        Hybrid::create(this->mtx2->get_executor(),
-//                       std::make_shared<typename Hybrid::column_limit>(2));
-//    auto fbcsr_ref = Fbcsr::create(this->mtx2->get_executor());
-//
-//    fbcsr_ref->copy_from(this->mtx2.get());
-//    fbcsr_ref->move_to(hybrid_mtx.get());
-//
-//    this->assert_equal_to_mtx2(hybrid_mtx.get());
-//}
-
-
 TYPED_TEST(Fbcsr, ConvertsEmptyToPrecision)
 {
     using ValueType = typename TestFixture::value_type;
@@ -749,112 +507,6 @@ TYPED_TEST(Fbcsr, MovesEmptyToDense)
 }
 
 
-// TYPED_TEST(Fbcsr, ConvertsEmptyToCoo)
-// {
-//    using ValueType = typename TestFixture::value_type;
-//    using IndexType = typename TestFixture::index_type;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using Coo = gko::matrix::Coo<ValueType, IndexType>;
-//    auto empty = Fbcsr::create(this->exec);
-//    auto res = Coo::create(this->exec);
-
-//    empty->convert_to(res.get());
-
-//    ASSERT_EQ(res->get_num_stored_elements(), 0);
-//    ASSERT_FALSE(res->get_size());
-// }
-
-
-// TYPED_TEST(Fbcsr, MovesEmptyToCoo)
-// {
-//    using ValueType = typename TestFixture::value_type;
-//    using IndexType = typename TestFixture::index_type;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using Coo = gko::matrix::Coo<ValueType, IndexType>;
-//    auto empty = Fbcsr::create(this->exec);
-//    auto res = Coo::create(this->exec);
-
-//    empty->move_to(res.get());
-
-//    ASSERT_EQ(res->get_num_stored_elements(), 0);
-//    ASSERT_FALSE(res->get_size());
-// }
-
-
-// TYPED_TEST(Fbcsr, ConvertsEmptyToEll)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using ValueType = typename TestFixture::value_type;
-//    using IndexType = typename TestFixture::index_type;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using Ell = gko::matrix::Ell<ValueType, IndexType>;
-//    auto empty = Fbcsr::create(this->exec);
-//    auto res = Ell::create(this->exec);
-//
-//    empty->convert_to(res.get());
-//
-//    ASSERT_EQ(res->get_num_stored_elements(), 0);
-//    ASSERT_FALSE(res->get_size());
-//}
-
-
-// TYPED_TEST(Fbcsr, MovesEmptyToEll)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using ValueType = typename TestFixture::value_type;
-//    using IndexType = typename TestFixture::index_type;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using Ell = gko::matrix::Ell<ValueType, IndexType>;
-//    auto empty = Fbcsr::create(this->exec);
-//    auto res = Ell::create(this->exec);
-//
-//    empty->move_to(res.get());
-//
-//    ASSERT_EQ(res->get_num_stored_elements(), 0);
-//    ASSERT_FALSE(res->get_size());
-//}
-
-
-// TYPED_TEST(Fbcsr, ConvertsEmptyToSellp)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using ValueType = typename TestFixture::value_type;
-//    using IndexType = typename TestFixture::index_type;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using Sellp = gko::matrix::Sellp<ValueType, IndexType>;
-//    auto empty = Fbcsr::create(this->exec);
-//    auto res = Sellp::create(this->exec);
-//
-//    empty->convert_to(res.get());
-//
-//    ASSERT_EQ(res->get_num_stored_elements(), 0);
-//    ASSERT_EQ(*res->get_const_slice_sets(), 0);
-//    ASSERT_FALSE(res->get_size());
-//}
-
-
-// TYPED_TEST(Fbcsr, MovesEmptyToSellp)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using ValueType = typename TestFixture::value_type;
-//    using IndexType = typename TestFixture::index_type;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using Sellp = gko::matrix::Sellp<ValueType, IndexType>;
-//    auto empty = Fbcsr::create(this->exec);
-//    auto res = Sellp::create(this->exec);
-//
-//    empty->move_to(res.get());
-//
-//    ASSERT_EQ(res->get_num_stored_elements(), 0);
-//    ASSERT_EQ(*res->get_const_slice_sets(), 0);
-//    ASSERT_FALSE(res->get_size());
-//}
-
-
 TYPED_TEST(Fbcsr, ConvertsEmptyToSparsityCsr)
 {
     using ValueType = typename TestFixture::value_type;
@@ -889,42 +541,6 @@ TYPED_TEST(Fbcsr, MovesEmptyToSparsityCsr)
 }
 
 
-// TYPED_TEST(Fbcsr, ConvertsEmptyToHybrid)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using ValueType = typename TestFixture::value_type;
-//    using IndexType = typename TestFixture::index_type;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using Hybrid = gko::matrix::Hybrid<ValueType, IndexType>;
-//    auto empty = Fbcsr::create(this->exec);
-//    auto res = Hybrid::create(this->exec);
-//
-//    empty->convert_to(res.get());
-//
-//    ASSERT_EQ(res->get_num_stored_elements(), 0);
-//    ASSERT_FALSE(res->get_size());
-//}
-
-
-// TYPED_TEST(Fbcsr, MovesEmptyToHybrid)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using ValueType = typename TestFixture::value_type;
-//    using IndexType = typename TestFixture::index_type;
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using Hybrid = gko::matrix::Hybrid<ValueType, IndexType>;
-//    auto empty = Fbcsr::create(this->exec);
-//    auto res = Hybrid::create(this->exec);
-//
-//    empty->move_to(res.get());
-//
-//    ASSERT_EQ(res->get_num_stored_elements(), 0);
-//    ASSERT_FALSE(res->get_size());
-//}
-
-
 TYPED_TEST(Fbcsr, CalculatesNonzerosPerRow)
 {
     using IndexType = typename TestFixture::index_type;
@@ -942,53 +558,6 @@ TYPED_TEST(Fbcsr, CalculatesNonzerosPerRow)
 }
 
 
-// TYPED_TEST(Fbcsr, CalculatesTotalCols)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    gko::size_type total_cols;
-//    gko::size_type stride_factor = gko::matrix::default_stride_factor;
-//    gko::size_type slice_size = gko::matrix::default_slice_size;
-//
-//    gko::kernels::reference::fbcsr::calculate_total_cols(
-//        this->exec, this->mtx.get(), &total_cols, stride_factor, slice_size);
-//
-//    ASSERT_EQ(total_cols, 3);
-//}
-
-
-// TYPED_TEST(Fbcsr, ConvertsToEll)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Ell = typename TestFixture::Ell;
-//    using Dense = typename TestFixture::Vec;
-//    auto ell_mtx = Ell::create(this->mtx->get_executor());
-//    auto dense_mtx = Dense::create(this->mtx->get_executor());
-//    auto ref_dense_mtx = Dense::create(this->mtx->get_executor());
-//
-//    this->mtx->convert_to(ell_mtx.get());
-//
-//    this->assert_equal_to_mtx(ell_mtx.get());
-//}
-
-
-// TYPED_TEST(Fbcsr, MovesToEll)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Ell = typename TestFixture::Ell;
-//    using Dense = typename TestFixture::Vec;
-//    auto ell_mtx = Ell::create(this->mtx->get_executor());
-//    auto dense_mtx = Dense::create(this->mtx->get_executor());
-//    auto ref_dense_mtx = Dense::create(this->mtx->get_executor());
-//
-//    this->mtx->move_to(ell_mtx.get());
-//
-//    this->assert_equal_to_mtx(ell_mtx.get());
-//}
-
-
 TYPED_TEST(Fbcsr, SquareMtxIsTransposable)
 {
     using Fbcsr = typename TestFixture::Mtx;
@@ -1013,214 +582,6 @@ TYPED_TEST(Fbcsr, NonSquareMtxIsTransposable)
 }
 
 
-// TYPED_TEST(Fbcsr, SquareMatrixIsRowPermutable)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using index_type = typename TestFixture::index_type;
-//    // clang-format off
-//    auto p_mtx = gko::initialize<Fbcsr>({{1.0, 3.0, 2.0},
-//                                       {0.0, 5.0, 0.0},
-//                                       {0.0, 1.5, 2.0}}, this->exec);
-//    // clang-format on
-//    gko::Array<index_type> permute_idxs{this->exec, {1, 2, 0}};
-//
-//    auto row_permute = p_mtx->row_permute(&permute_idxs);
-//
-//    auto row_permute_fbcsr = static_cast<Fbcsr *>(row_permute.get());
-//    // clang-format off
-//    GKO_ASSERT_MTX_NEAR(row_permute_fbcsr,
-//                        l({{0.0, 5.0, 0.0},
-//                           {0.0, 1.5, 2.0},
-//                           {1.0, 3.0, 2.0}}),
-//                        0.0);
-//    // clang-format on
-//}
-
-
-// TYPED_TEST(Fbcsr, NonSquareMatrixIsRowPermutable)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using index_type = typename TestFixture::index_type;
-//    // clang-format off
-//    auto p_mtx = gko::initialize<Fbcsr>({{1.0, 3.0, 2.0},
-//                                       {0.0, 5.0, 0.0}}, this->exec);
-//    // clang-format on
-//    gko::Array<index_type> permute_idxs{this->exec, {1, 0}};
-//
-//    auto row_permute = p_mtx->row_permute(&permute_idxs);
-//
-//    auto row_permute_fbcsr = static_cast<Fbcsr *>(row_permute.get());
-//    // clang-format off
-//    GKO_ASSERT_MTX_NEAR(row_permute_fbcsr,
-//                        l({{0.0, 5.0, 0.0},
-//                           {1.0, 3.0, 2.0}}),
-//                        0.0);
-//    // clang-format on
-//}
-
-
-// TYPED_TEST(Fbcsr, SquareMatrixIsColPermutable)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using index_type = typename TestFixture::index_type;
-//    // clang-format off
-//    auto p_mtx = gko::initialize<Fbcsr>({{1.0, 3.0, 2.0},
-//                                       {0.0, 5.0, 0.0},
-//                                       {0.0, 1.5, 2.0}}, this->exec);
-//    // clang-format on
-//    gko::Array<index_type> permute_idxs{this->exec, {1, 2, 0}};
-//
-//    auto c_permute = p_mtx->column_permute(&permute_idxs);
-//
-//    auto c_permute_fbcsr = static_cast<Fbcsr *>(c_permute.get());
-//    // clang-format off
-//    GKO_ASSERT_MTX_NEAR(c_permute_fbcsr,
-//                        l({{3.0, 2.0, 1.0},
-//                           {5.0, 0.0, 0.0},
-//                           {1.5, 2.0, 0.0}}),
-//                        0.0);
-//    // clang-format on
-//}
-
-
-// TYPED_TEST(Fbcsr, NonSquareMatrixIsColPermutable)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using index_type = typename TestFixture::index_type;
-//    // clang-format off
-//    auto p_mtx = gko::initialize<Fbcsr>({{1.0, 0.0, 2.0},
-//                                       {0.0, 5.0, 0.0}}, this->exec);
-//    // clang-format on
-//    gko::Array<index_type> permute_idxs{this->exec, {1, 2, 0}};
-//
-//    auto c_permute = p_mtx->column_permute(&permute_idxs);
-//
-//    auto c_permute_fbcsr = static_cast<Fbcsr *>(c_permute.get());
-//    // clang-format off
-//    GKO_ASSERT_MTX_NEAR(c_permute_fbcsr,
-//                        l({{0.0, 2.0, 1.0},
-//                           {5.0, 0.0, 0.0}}),
-//                        0.0);
-//    // clang-format on
-//}
-
-
-// TYPED_TEST(Fbcsr, SquareMatrixIsInverseRowPermutable)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using index_type = typename TestFixture::index_type;
-//    // clang-format off
-//    auto inverse_p_mtx = gko::initialize<Fbcsr>({{1.0, 3.0, 2.0},
-//                                               {0.0, 5.0, 0.0},
-//                                               {0.0, 1.5, 2.0}}, this->exec);
-//    // clang-format on
-//    gko::Array<index_type> inverse_permute_idxs{this->exec, {1, 2, 0}};
-//
-//    auto inverse_row_permute =
-//        inverse_p_mtx->inverse_row_permute(&inverse_permute_idxs);
-//
-//    auto inverse_row_permute_fbcsr =
-//        static_cast<Fbcsr *>(inverse_row_permute.get());
-//    // clang-format off
-//    GKO_ASSERT_MTX_NEAR(inverse_row_permute_fbcsr,
-//                        l({{0.0, 1.5, 2.0},
-//                           {1.0, 3.0, 2.0},
-//                           {0.0, 5.0, 0.0}}),
-//                        0.0);
-//    // clang-format on
-//}
-
-
-// TYPED_TEST(Fbcsr, NonSquareMatrixIsInverseRowPermutable)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using index_type = typename TestFixture::index_type;
-//    // clang-format off
-//    auto inverse_p_mtx = gko::initialize<Fbcsr>({{1.0, 3.0, 2.0},
-//                                               {0.0, 5.0, 0.0}}, this->exec);
-//    // clang-format on
-//    gko::Array<index_type> inverse_permute_idxs{this->exec, {1, 0}};
-//
-//    auto inverse_row_permute =
-//        inverse_p_mtx->inverse_row_permute(&inverse_permute_idxs);
-//
-//    auto inverse_row_permute_fbcsr =
-//        static_cast<Fbcsr *>(inverse_row_permute.get());
-//    // clang-format off
-//    GKO_ASSERT_MTX_NEAR(inverse_row_permute_fbcsr,
-//                        l({{0.0, 5.0, 0.0},
-//                           {1.0, 3.0, 2.0}}),
-//                        0.0);
-//    // clang-format on
-//}
-
-
-// TYPED_TEST(Fbcsr, SquareMatrixIsInverseColPermutable)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using index_type = typename TestFixture::index_type;
-//    // clang-format off
-//    auto inverse_p_mtx = gko::initialize<Fbcsr>({{1.0, 3.0, 2.0},
-//                                               {0.0, 5.0, 0.0},
-//                                               {0.0, 1.5, 2.0}}, this->exec);
-//    // clang-format on
-//    gko::Array<index_type> inverse_permute_idxs{this->exec, {1, 2, 0}};
-//
-//    auto inverse_c_permute =
-//        inverse_p_mtx->inverse_column_permute(&inverse_permute_idxs);
-//
-//    auto inverse_c_permute_fbcsr = static_cast<Fbcsr
-//    *>(inverse_c_permute.get());
-//    // clang-format off
-//    GKO_ASSERT_MTX_NEAR(inverse_c_permute_fbcsr,
-//                        l({{2.0, 1.0, 3.0},
-//                           {0.0, 0.0, 5.0},
-//                           {2.0, 0.0, 1.5}}),
-//                        0.0);
-//    // clang-format on
-//}
-
-
-// TYPED_TEST(Fbcsr, NonSquareMatrixIsInverseColPermutable)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    using Fbcsr = typename TestFixture::Mtx;
-//    using index_type = typename TestFixture::index_type;
-//    // clang-format off
-//    auto inverse_p_mtx = gko::initialize<Fbcsr>({{1.0, 3.0, 2.0},
-//                                              {0.0, 5.0, 0.0}}, this->exec);
-//    // clang-format on
-//    gko::Array<index_type> inverse_permute_idxs{this->exec, {1, 2, 0}};
-//
-//    auto inverse_c_permute =
-//        inverse_p_mtx->inverse_column_permute(&inverse_permute_idxs);
-//
-//    auto inverse_c_permute_fbcsr = static_cast<Fbcsr
-//    *>(inverse_c_permute.get());
-//    // clang-format off
-//    GKO_ASSERT_MTX_NEAR(inverse_c_permute_fbcsr,
-//                        l({{2.0, 1.0, 3.0},
-//                           {0.0, 0.0, 5.0}}),
-//                        0.0);
-//    // clang-format on
-//}
-
-
 TYPED_TEST(Fbcsr, RecognizeSortedMatrix)
 {
     ASSERT_TRUE(this->mtx->is_sorted_by_column_index());
@@ -1240,28 +601,6 @@ TYPED_TEST(Fbcsr, RecognizeUnsortedMatrix)
 }
 
 
-// TYPED_TEST(Fbcsr, SortSortedMatrix)
-// {
-//    auto matrix = this->mtx->clone();
-
-//    matrix->sort_by_column_index();
-
-//    GKO_ASSERT_MTX_NEAR(matrix, this->mtx, 0.0);
-// }
-
-
-// TYPED_TEST(Fbcsr, SortUnsortedMatrix)
-// GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto matrix = this->mtx3_unsorted->clone();
-//
-//    matrix->sort_by_column_index();
-//
-//    GKO_ASSERT_MTX_NEAR(matrix, this->mtx3_sorted, 0.0);
-//}
-
-
 TYPED_TEST(Fbcsr, ExtractsDiagonal)
 {
     using T = typename TestFixture::value_type;

From ef41bcfb20888c92437a43082118924a50fa7578 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Thu, 26 Nov 2020 15:13:19 +0100
Subject: [PATCH 0546/1008] fixed docstrings, made const usage more consistent,
 and other fixes

- removed remaining commented code
- fixed documentation in FbcsrBuilder
- removed unnecessary headers
- improved docstrings for FBCSR sample matrix generation
---
 core/components/fixed_block.hpp         |  13 ++-
 core/matrix/fbcsr.cpp                   |  29 ++++---
 core/matrix/fbcsr_builder.hpp           |  11 ++-
 core/matrix/fbcsr_kernels.hpp           |   4 -
 core/test/matrix/fbcsr_sample.cpp       |   8 +-
 core/test/matrix/fbcsr_sample.hpp       |  71 ++++++++++-----
 cuda/matrix/fbcsr_kernels.cu            |  71 ---------------
 hip/matrix/fbcsr_kernels.hip.cpp        |  72 +--------------
 include/ginkgo/core/base/blockutils.hpp |   9 +-
 include/ginkgo/core/matrix/fbcsr.hpp    |  67 ++++++++------
 omp/matrix/fbcsr_kernels.cpp            | 111 ------------------------
 11 files changed, 138 insertions(+), 328 deletions(-)

diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
index fbd8a259f69..32c6fb8ddda 100644
--- a/core/components/fixed_block.hpp
+++ b/core/components/fixed_block.hpp
@@ -49,6 +49,10 @@ namespace blockutils {
  *
  * The primary use is to reinterpret subsets of entries in a big array as
  *  small dense blocks.
+ *
+ * @tparam ValueType The numeric type of entries of the block
+ * @tparam nrows Number of rows
+ * @tparam ncols Number of columns
  */
 template <typename ValueType, int nrows, int ncols>
 class FixedBlock final {
@@ -83,7 +87,11 @@ class FixedBlock final {
 };
 
 
-/// A lightweight dynamic block type for the host space
+/**
+ * A lightweight dynamic block type for the host space
+ *
+ * @tparam ValueType The numeric type of entries of the block
+ */
 template <typename ValueType>
 class DenseBlock final {
 public:
@@ -145,6 +153,9 @@ class DenseBlock final {
 /** Note that accessing BSR values using this type of view abstracts away the
  * storage layout within the individual blocks, as long as all blocks use the
  * same layout. For now, row-major blocks are assumed.
+ *
+ * @tparam ValueType The numeric type of entries of the block
+ * @tparam IndexType The type of integer used to identify the different blocks
  */
 template <typename ValueType, typename IndexType = int32>
 class DenseBlocksView final {
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 54e938d768d..46fe819d240 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -121,8 +121,10 @@ void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const b,
 
 
 template <typename ValueType, typename IndexType>
-void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
-                                             const LinOp *beta, LinOp *x) const
+void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const alpha,
+                                             const LinOp *const b,
+                                             const LinOp *const beta,
+                                             LinOp *const x) const
 {
     using Dense = Dense<ValueType>;
     using TFbcsr = Fbcsr<ValueType, IndexType>;
@@ -143,7 +145,7 @@ void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
 
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(
-    Fbcsr<ValueType, IndexType> *result) const
+    Fbcsr<ValueType, IndexType> *const result) const
 {
     bool same_executor = this->get_executor() == result->get_executor();
     // NOTE: as soon as strategies are improved, this can be reverted
@@ -163,7 +165,8 @@ void Fbcsr<ValueType, IndexType>::convert_to(
 
 
 template <typename ValueType, typename IndexType>
-void Fbcsr<ValueType, IndexType>::move_to(Fbcsr<ValueType, IndexType> *result)
+void Fbcsr<ValueType, IndexType>::move_to(
+    Fbcsr<ValueType, IndexType> *const result)
 {
     bool same_executor = this->get_executor() == result->get_executor();
     EnableLinOp<Fbcsr>::move_to(result);
@@ -188,14 +191,15 @@ void Fbcsr<ValueType, IndexType>::convert_to(
 
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::move_to(
-    Fbcsr<next_precision<ValueType>, IndexType> *result)
+    Fbcsr<next_precision<ValueType>, IndexType> *const result)
 {
     this->convert_to(result);
 }
 
 
 template <typename ValueType, typename IndexType>
-void Fbcsr<ValueType, IndexType>::convert_to(Dense<ValueType> *result) const
+void Fbcsr<ValueType, IndexType>::convert_to(
+    Dense<ValueType> *const result) const
 {
     auto exec = this->get_executor();
     auto tmp = Dense<ValueType>::create(exec, this->get_size());
@@ -205,7 +209,7 @@ void Fbcsr<ValueType, IndexType>::convert_to(Dense<ValueType> *result) const
 
 
 template <typename ValueType, typename IndexType>
-void Fbcsr<ValueType, IndexType>::move_to(Dense<ValueType> *result)
+void Fbcsr<ValueType, IndexType>::move_to(Dense<ValueType> *const result)
 {
     this->convert_to(result);
 }
@@ -213,7 +217,7 @@ void Fbcsr<ValueType, IndexType>::move_to(Dense<ValueType> *result)
 
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(
-    Csr<ValueType, IndexType> *result) const
+    Csr<ValueType, IndexType> *const result) const
 {
     auto exec = this->get_executor();
     auto tmp = Csr<ValueType, IndexType>::create(
@@ -225,7 +229,8 @@ void Fbcsr<ValueType, IndexType>::convert_to(
 
 
 template <typename ValueType, typename IndexType>
-void Fbcsr<ValueType, IndexType>::move_to(Csr<ValueType, IndexType> *result)
+void Fbcsr<ValueType, IndexType>::move_to(
+    Csr<ValueType, IndexType> *const result)
 {
     this->convert_to(result);
 }
@@ -233,7 +238,7 @@ void Fbcsr<ValueType, IndexType>::move_to(Csr<ValueType, IndexType> *result)
 
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(
-    SparsityCsr<ValueType, IndexType> *result) const
+    SparsityCsr<ValueType, IndexType> *const result) const
 {
     using gko::blockutils::getNumBlocks;
     auto exec = this->get_executor();
@@ -252,7 +257,7 @@ void Fbcsr<ValueType, IndexType>::convert_to(
 
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::move_to(
-    SparsityCsr<ValueType, IndexType> *result)
+    SparsityCsr<ValueType, IndexType> *const result)
 {
     this->convert_to(result);
 }
@@ -497,7 +502,7 @@ Fbcsr<ValueType, IndexType>::compute_absolute() const
 template <typename ValueType, typename IndexType>
 template <typename FbcsrType>
 void Fbcsr<ValueType, IndexType>::convert_strategy_helper(
-    FbcsrType *result) const
+    FbcsrType *const result) const
 {
     auto strat = this->get_strategy().get();
     std::shared_ptr<typename matrix_strategy::strategy_type<FbcsrType>>
diff --git a/core/matrix/fbcsr_builder.hpp b/core/matrix/fbcsr_builder.hpp
index 54b6ece53f9..62096ae27fc 100644
--- a/core/matrix/fbcsr_builder.hpp
+++ b/core/matrix/fbcsr_builder.hpp
@@ -53,20 +53,23 @@ template <typename ValueType = default_precision, typename IndexType = int32>
 class FbcsrBuilder {
 public:
     /**
-     * Returns the column index array of the CSR matrix.
+     * @return The column index array of the matrix.
      */
     Array<IndexType> &get_col_idx_array() { return matrix_->col_idxs_; }
 
     /**
-     * Returns the value array of the CSR matrix.
+     * @return The value array of the matrix.
      */
     Array<ValueType> &get_value_array() { return matrix_->values_; }
 
-    /// Returns the (uniform) block size
+    /**
+     * @return The (uniform) block size
+     */
     int get_block_size() const { return matrix_->bs_; }
 
     /**
-     * Initializes a CsrBuilder from an existing CSR matrix.
+     * @param matrix An existing FBCSR matrix
+     *        for which intrusive access is needed
      */
     explicit FbcsrBuilder(Fbcsr<ValueType, IndexType> *const matrix)
         : matrix_{matrix}
diff --git a/core/matrix/fbcsr_kernels.hpp b/core/matrix/fbcsr_kernels.hpp
index c7a2747039e..ed37c49153d 100644
--- a/core/matrix/fbcsr_kernels.hpp
+++ b/core/matrix/fbcsr_kernels.hpp
@@ -39,13 +39,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/diagonal.hpp>
-#include <ginkgo/core/matrix/ell.hpp>
-#include <ginkgo/core/matrix/hybrid.hpp>
-#include <ginkgo/core/matrix/sellp.hpp>
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
index 246510db57d..72685d99eab 100644
--- a/core/test/matrix/fbcsr_sample.cpp
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -50,15 +50,15 @@ namespace testing {
 
 namespace matstr = gko::matrix::matrix_strategy;
 
-/// Generates a copu of the given matrix with a different scalar type
-/** \tparam AbsValueType The scalar type of the output matrix
+/** Generates a copy of the given matrix with a different scalar type
+ *
+ * \tparam AbsValueType The scalar type of the output matrix
  */
 template <typename FbcsrType, typename AbsValueType>
 static std::unique_ptr<
     gko::matrix::Fbcsr<AbsValueType, typename FbcsrType::index_type>>
 generate_acopy_impl(const FbcsrType *const mat)
 {
-    // using AbsValueType = typename gko::remove_complex<ValueType>;
     using index_type = typename FbcsrType::index_type;
     using value_type = typename FbcsrType::value_type;
     using AbsFbcsr = gko::matrix::Fbcsr<AbsValueType, index_type>;
@@ -686,8 +686,6 @@ void FbcsrSample2<ValueType, IndexType>::apply(
 
     const ValueType defv = sct(0.15 + FBCSR_TEST_OFFSET);
 
-    // ValueType *const yvals = y->get_data();
-    // const ValueType *const xvals = x->get_const_data();
     for (index_type k = 0; k < x->get_size()[1]; k++) {
         y->at(0, k) = sct(1.0) * x->at(0, k) + sct(2.0) * x->at(1, k);
         y->at(1, k) = sct(3.0) * x->at(0, k);
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 5e879964950..e6fe63dc37b 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifndef GKO_CORE_MATRIX_TEST_FBCSR_SAMPLE_HPP
 #define GKO_CORE_MATRIX_TEST_FBCSR_SAMPLE_HPP
 
+
 #include <ginkgo/core/base/matrix_data.hpp>
 #include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
@@ -42,12 +43,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/fbcsr.hpp>
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
+
 namespace gko {
 namespace testing {
 
 
-/// Generates the same sample block CSR matrix in different formats
-/** This currently a 6 x 12 matrix with 3x3 blocks.
+/** Generates the same sample block CSR matrix in different formats
+ *
+ * This currently a 6 x 12 matrix with 3x3 blocks.
  * Assumes that the layout within each block is row-major.
  * Generates complex data when instantiated with a complex value type.
  */
@@ -66,30 +69,64 @@ class FbcsrSample {
 
     FbcsrSample(std::shared_ptr<const gko::ReferenceExecutor> exec);
 
+    /**
+     * @return The sample matrix in FBCSR format
+     */
     std::unique_ptr<Fbcsr> generate_fbcsr() const;
 
-    /// Generates CSR matrix equal to the BSR matrix. Keeps explicit zeros.
+    /**
+     * @return Sample matrix in CSR format
+     *
+     * Keeps explicit zeros.
+     */
     std::unique_ptr<Csr> generate_csr() const;
 
+    /**
+     * @return Sample matrix as dense
+     */
     std::unique_ptr<Dense> generate_dense() const;
 
-    /// Returns the matrix in COO format keeping explicit nonzeros
-    /** The nonzeros are sorted by row and column.
+    /**
+     * @return The matrix in COO format keeping explicit nonzeros
+     *
+     * The nonzeros are sorted by row and column.
      */
     std::unique_ptr<Coo> generate_coo() const;
 
+    /**
+     * @return Sparsity structure of the matrix
+     */
     std::unique_ptr<SparCsr> generate_sparsity_csr() const;
 
+    /**
+     * @return Array of COO triplets that represent the matrix
+     *
+     * @note The order of the triplets assumes the blocks are stored row-major
+     */
     MatData generate_matrix_data() const;
 
+    /**
+     * @return Array of COO triplets that represent the matrix; includes
+     *         explicit zeros
+     *
+     * @note The order of the triplets assumes the blocks are stored row-major
+     */
     MatData generate_matrix_data_with_explicit_zeros() const;
 
-    /// Returns an array containing number of stored values in each row
-    ///  (not block-row)
+    /**
+     *  @return An array containing number of stored values in each row
+     */
     gko::Array<index_type> getNonzerosPerRow() const;
 
+    /**
+     * @return FBCSR matrix with absolute values of respective entries
+     */
     std::unique_ptr<Fbcsr> generate_abs_fbcsr() const;
 
+    /**
+     * @return FBCSR matrix with real scalar type,
+     *         with absolute values of respective entries
+     */
     std::unique_ptr<gko::matrix::Fbcsr<remove_complex<value_type>, index_type>>
     generate_abs_fbcsr_abstype() const;
 
@@ -126,8 +163,9 @@ class FbcsrSample {
     }
 };
 
-/// Generates the a sample block CSR matrix in different formats
-/** This currently a 6 x 8 matrix with 2x2 blocks.
+/**
+ * Generates a sample block CSR matrix in different formats.
+ * 6 x 8 matrix with 2x2 blocks.
  */
 template <typename ValueType, typename IndexType>
 class FbcsrSample2 {
@@ -135,11 +173,7 @@ class FbcsrSample2 {
     using value_type = ValueType;
     using index_type = IndexType;
     using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
-    using Csr = gko::matrix::Csr<value_type, index_type>;
-    using Coo = gko::matrix::Coo<value_type, index_type>;
     using Dense = gko::matrix::Dense<value_type>;
-    using MatData = gko::matrix_data<value_type, index_type>;
-    using SparCsr = gko::matrix::SparsityCsr<value_type, index_type>;
     using Diagonal = gko::matrix::Diagonal<value_type>;
 
     FbcsrSample2(std::shared_ptr<const gko::ReferenceExecutor> exec);
@@ -203,8 +237,9 @@ class FbcsrSampleSquare {
     const std::shared_ptr<const gko::Executor> exec;
 };
 
-/// Generates the a sample block CSR matrix with complex values
-/** This currently a 6 x 8 matrix with 2x2 blocks.
+/**
+ * Generates the a sample block CSR matrix with complex values
+ * This is a 6 x 8 matrix with 2x2 blocks.
  */
 template <typename ValueType, typename IndexType>
 class FbcsrSampleComplex {
@@ -212,12 +247,6 @@ class FbcsrSampleComplex {
     using value_type = ValueType;
     using index_type = IndexType;
     using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
-    using Csr = gko::matrix::Csr<value_type, index_type>;
-    using Coo = gko::matrix::Coo<value_type, index_type>;
-    using Dense = gko::matrix::Dense<value_type>;
-    using MatData = gko::matrix_data<value_type, index_type>;
-    using SparCsr = gko::matrix::SparsityCsr<value_type, index_type>;
-    using Diagonal = gko::matrix::Diagonal<value_type>;
 
     static_assert(is_complex<ValueType>(), "Only for complex types!");
 
diff --git a/cuda/matrix/fbcsr_kernels.cu b/cuda/matrix/fbcsr_kernels.cu
index f2ac5a85a01..7da16b2976d 100644
--- a/cuda/matrix/fbcsr_kernels.cu
+++ b/cuda/matrix/fbcsr_kernels.cu
@@ -43,24 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
-#include "core/components/fill_array.hpp"
-#include "core/components/prefix_sum.hpp"
-#include "core/matrix/dense_kernels.hpp"
-#include "core/matrix/fbcsr_builder.hpp"
-#include "core/synthesizer/implementation_selection.hpp"
 #include "cuda/base/config.hpp"
-#include "cuda/base/cusparse_bindings.hpp"
-#include "cuda/base/math.hpp"
-#include "cuda/base/pointer_mode_guard.hpp"
-#include "cuda/base/types.hpp"
-#include "cuda/components/atomic.cuh"
-#include "cuda/components/cooperative_groups.cuh"
-#include "cuda/components/intrinsics.cuh"
-#include "cuda/components/merging.cuh"
-#include "cuda/components/reduction.cuh"
-#include "cuda/components/segment_scan.cuh"
-#include "cuda/components/thread_ids.cuh"
-#include "cuda/components/uninitialized_array.hpp"
 
 
 namespace gko {
@@ -116,13 +99,6 @@ template <typename IndexType>
 void convert_row_ptrs_to_idxs(std::shared_ptr<const CudaExecutor> exec,
                               const IndexType *ptrs, size_type num_rows,
                               IndexType *idxs) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto grid_dim = ceildiv(num_rows, default_block_size);
-//
-//    kernel::convert_row_ptrs_to_idxs<<<grid_dim, default_block_size>>>(
-//        num_rows, as_cuda_type(ptrs), as_cuda_type(idxs));
-//}
 
 
 template <typename ValueType, typename IndexType>
@@ -168,31 +144,6 @@ void calculate_max_nnz_per_row(
     std::shared_ptr<const CudaExecutor> exec,
     const matrix::Fbcsr<ValueType, IndexType> *source,
     size_type *result) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto num_rows = source->get_size()[0];
-//
-//    auto nnz_per_row = Array<size_type>(exec, num_rows);
-//    auto block_results = Array<size_type>(exec, default_block_size);
-//    auto d_result = Array<size_type>(exec, 1);
-//
-//    const auto grid_dim = ceildiv(num_rows, default_block_size);
-//    kernel::calculate_nnz_per_row<<<grid_dim, default_block_size>>>(
-//        num_rows, as_cuda_type(source->get_const_row_ptrs()),
-//        as_cuda_type(nnz_per_row.get_data()));
-//
-//    const auto n = ceildiv(num_rows, default_block_size);
-//    const auto reduce_dim = n <= default_block_size ? n : default_block_size;
-//    kernel::reduce_max_nnz<<<reduce_dim, default_block_size>>>(
-//        num_rows, as_cuda_type(nnz_per_row.get_const_data()),
-//        as_cuda_type(block_results.get_data()));
-//
-//    kernel::reduce_max_nnz<<<1, default_block_size>>>(
-//        reduce_dim, as_cuda_type(block_results.get_const_data()),
-//        as_cuda_type(d_result.get_data()));
-//
-//    *result = exec->copy_val_to_host(d_result.get_const_data());
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
@@ -203,15 +154,6 @@ void calculate_nonzeros_per_row(
     std::shared_ptr<const CudaExecutor> exec,
     const matrix::Fbcsr<ValueType, IndexType> *source,
     Array<size_type> *result) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto num_rows = source->get_size()[0];
-//    auto row_ptrs = source->get_const_row_ptrs();
-//    auto grid_dim = ceildiv(num_rows, default_block_size);
-//
-//    kernel::calculate_nnz_per_row<<<grid_dim, default_block_size>>>(
-//        num_rows, as_cuda_type(row_ptrs), as_cuda_type(result->get_data()));
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
@@ -231,19 +173,6 @@ void is_sorted_by_column_index(
     std::shared_ptr<const CudaExecutor> exec,
     const matrix::Fbcsr<ValueType, IndexType> *to_check,
     bool *is_sorted) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    *is_sorted = true;
-//    auto cpu_array = Array<bool>::view(exec->get_master(), 1, is_sorted);
-//    auto gpu_array = Array<bool>{exec, cpu_array};
-//    auto block_size = default_block_size;
-//    auto num_rows = static_cast<IndexType>(to_check->get_size()[0]);
-//    auto num_blocks = ceildiv(num_rows, block_size);
-//    kernel::check_unsorted<<<num_blocks, block_size>>>(
-//        to_check->get_const_row_ptrs(), to_check->get_const_col_idxs(),
-//        num_rows, gpu_array.get_data());
-//    cpu_array = gpu_array;
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
diff --git a/hip/matrix/fbcsr_kernels.hip.cpp b/hip/matrix/fbcsr_kernels.hip.cpp
index 891e644b25c..1ce2735d8f6 100644
--- a/hip/matrix/fbcsr_kernels.hip.cpp
+++ b/hip/matrix/fbcsr_kernels.hip.cpp
@@ -46,24 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
-#include "core/components/fill_array.hpp"
-#include "core/components/prefix_sum.hpp"
-#include "core/matrix/dense_kernels.hpp"
-#include "core/matrix/fbcsr_builder.hpp"
-#include "core/synthesizer/implementation_selection.hpp"
-#include "hip/base/config.hip.hpp"
-#include "hip/base/hipsparse_bindings.hip.hpp"
-#include "hip/base/math.hip.hpp"
-#include "hip/base/pointer_mode_guard.hip.hpp"
-#include "hip/base/types.hip.hpp"
-#include "hip/components/atomic.hip.hpp"
-#include "hip/components/cooperative_groups.hip.hpp"
-#include "hip/components/intrinsics.hip.hpp"
-#include "hip/components/merging.hip.hpp"
-#include "hip/components/reduction.hip.hpp"
-#include "hip/components/segment_scan.hip.hpp"
-#include "hip/components/thread_ids.hip.hpp"
-#include "hip/components/uninitialized_array.hip.hpp"
+#include "hip/base/config.hpp"
 
 
 namespace gko {
@@ -158,34 +141,6 @@ void calculate_max_nnz_per_row(
     std::shared_ptr<const HipExecutor> exec,
     const matrix::Fbcsr<ValueType, IndexType> *source,
     size_type *result) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto num_rows = source->get_size()[0];
-//
-//    auto nnz_per_row = Array<size_type>(exec, num_rows);
-//    auto block_results = Array<size_type>(exec, default_block_size);
-//    auto d_result = Array<size_type>(exec, 1);
-//
-//    const auto grid_dim = ceildiv(num_rows, default_block_size);
-//    hipLaunchKernelGGL(kernel::calculate_nnz_per_row, dim3(grid_dim),
-//                       dim3(default_block_size), 0, 0, num_rows,
-//                       as_hip_type(source->get_const_row_ptrs()),
-//                       as_hip_type(nnz_per_row.get_data()));
-//
-//    const auto n = ceildiv(num_rows, default_block_size);
-//    const auto reduce_dim = n <= default_block_size ? n : default_block_size;
-//    hipLaunchKernelGGL(kernel::reduce_max_nnz, dim3(reduce_dim),
-//                       dim3(default_block_size), 0, 0, num_rows,
-//                       as_hip_type(nnz_per_row.get_const_data()),
-//                       as_hip_type(block_results.get_data()));
-//
-//    hipLaunchKernelGGL(kernel::reduce_max_nnz, dim3(1),
-//                       dim3(default_block_size), 0, 0, reduce_dim,
-//                       as_hip_type(block_results.get_const_data()),
-//                       as_hip_type(d_result.get_data()));
-//
-//    *result = exec->copy_val_to_host(d_result.get_const_data());
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
@@ -196,17 +151,6 @@ void calculate_nonzeros_per_row(
     std::shared_ptr<const HipExecutor> exec,
     const matrix::Fbcsr<ValueType, IndexType> *source,
     Array<size_type> *result) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto num_rows = source->get_size()[0];
-//    auto row_ptrs = source->get_const_row_ptrs();
-//    auto grid_dim = ceildiv(num_rows, default_block_size);
-//
-//    hipLaunchKernelGGL(kernel::calculate_nnz_per_row, dim3(grid_dim),
-//                       dim3(default_block_size), 0, 0, num_rows,
-//                       as_hip_type(row_ptrs),
-//                       as_hip_type(result->get_data()));
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
@@ -226,20 +170,6 @@ void is_sorted_by_column_index(
     std::shared_ptr<const HipExecutor> exec,
     const matrix::Fbcsr<ValueType, IndexType> *to_check,
     bool *is_sorted) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    *is_sorted = true;
-//    auto cpu_array = Array<bool>::view(exec->get_master(), 1, is_sorted);
-//    auto gpu_array = Array<bool>{exec, cpu_array};
-//    auto block_size = default_block_size;
-//    auto num_rows = static_cast<IndexType>(to_check->get_size()[0]);
-//    auto num_blocks = ceildiv(num_rows, block_size);
-//    hipLaunchKernelGGL(
-//        HIP_KERNEL_NAME(kernel::check_unsorted), dim3(num_blocks),
-//        dim3(block_size), 0, 0, to_check->get_const_row_ptrs(),
-//        to_check->get_const_col_idxs(), num_rows, gpu_array.get_data());
-//    cpu_array = gpu_array;
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
diff --git a/include/ginkgo/core/base/blockutils.hpp b/include/ginkgo/core/base/blockutils.hpp
index 4bd4d3d040d..c057e1dec04 100644
--- a/include/ginkgo/core/base/blockutils.hpp
+++ b/include/ginkgo/core/base/blockutils.hpp
@@ -55,8 +55,13 @@ class BlockSizeError : public Error {
 };
 
 
-/** Returns the quotient of the second arg divided by the first
- * but throws when they don't divide
+/**
+ * Computes the number of blocks
+ *
+ * @param block_size The size of each block
+ * @param size The total size of some array/vector
+ * @return The quotient of the size divided by the block size
+ *         but throws when they don't divide
  */
 template <typename IndexType>
 IndexType getNumBlocks(const int block_size, const IndexType size)
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index 33e18a99794..30dadb68e54 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -161,16 +161,19 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
 
     void move_to(Csr<ValueType, IndexType> *result) override;
 
-    /// Get the block sparsity pattern in CSR-like format
-    /** Note that the actual non-zero values are never copied;
+    /**
+     * Get the block sparsity pattern in CSR-like format
+     *
+     * Note that the actual non-zero values are never copied;
      * the result always has a value array of size 1 with the value 1.
      */
     void convert_to(SparsityCsr<ValueType, IndexType> *result) const override;
 
     void move_to(SparsityCsr<ValueType, IndexType> *result) override;
 
-    /// Convert COO data into block CSR
-    /** @warning Unlike Csr::read, here explicit non-zeros are NOT dropped.
+    /** Convert COO data into block CSR
+     *
+     * @warning Unlike Csr::read, here explicit non-zeros are NOT dropped.
      */
     void read(const mat_data &data) override;
 
@@ -200,13 +203,13 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     bool is_sorted_by_column_index() const;
 
     /**
-     * Returns the values of the matrix.
-     *
-     * @return the values of the matrix.
+     * @return The values of the matrix.
      */
     value_type *get_values() noexcept { return values_.get_data(); }
 
-    /// @see Fbcsr::get_const_values()
+    /**
+     * @see Fbcsr::get_const_values()
+     */
     const value_type *get_values() const noexcept
     {
         return values_.get_const_data();
@@ -225,13 +228,13 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     }
 
     /**
-     * Returns the column indexes of the matrix.
-     *
-     * @return the column indexes of the matrix.
+     * @return The column indexes of the matrix.
      */
     index_type *get_col_idxs() noexcept { return col_idxs_.get_data(); }
 
-    /// @see Fbcsr::get_const_col_idxs()
+    /**
+     * @see Fbcsr::get_const_col_idxs()
+     */
     const index_type *get_col_idxs() const noexcept
     {
         return col_idxs_.get_const_data();
@@ -250,13 +253,13 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     }
 
     /**
-     * Returns the row pointers of the matrix.
-     *
-     * @return the row pointers of the matrix.
+     * @return The row pointers of the matrix.
      */
     index_type *get_row_ptrs() noexcept { return row_ptrs_.get_data(); }
 
-    /// @see Fbcsr::get_const_row_ptrs()
+    /**
+     * @see Fbcsr::get_const_row_ptrs()
+     */
     const index_type *get_row_ptrs() const noexcept
     {
         return row_ptrs_.get_const_data();
@@ -275,12 +278,13 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     }
 
     /**
-     * Returns the starting rows.
-     *
-     * @return the starting rows.
+     * @return The starting row for each 'team' of threads
      */
     index_type *get_srow() noexcept { return startrow_.get_data(); }
 
+    /**
+     * @see get_const_srow
+     */
     const index_type *get_srow() const noexcept
     {
         return startrow_.get_const_data();
@@ -299,9 +303,7 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     }
 
     /**
-     * Returns the number of the srow stored elements (involved warps)
-     *
-     * @return the number of the srow stored elements (involved warps)
+     * @return The number of the srow stored elements (involved warps)
      */
     size_type get_num_srow_elements() const noexcept
     {
@@ -318,9 +320,8 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
         return values_.get_num_elems();
     }
 
-    /** Returns the strategy
-     *
-     * @return the strategy
+    /**
+     * @return The strategy
      */
     std::shared_ptr<strategy_type> get_strategy() const noexcept
     {
@@ -338,15 +339,29 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
         this->make_srow();
     }
 
+    /**
+     * @return The fixed block size for this matrix
+     */
     int get_block_size() const { return bs_; }
 
+    /**
+     * Set the fixed block size for this matrix
+     *
+     * @param block_size The block size
+     */
     void set_block_size(const int block_size) { bs_ = block_size; }
 
+    /**
+     * @return The number of block-rows in the matrix
+     */
     index_type get_num_block_rows() const
     {
         return row_ptrs_.get_num_elems() - 1;
     }
 
+    /**
+     * @return The number of block-columns in the matrix
+     */
     index_type get_num_block_cols() const { return nbcols_; }
 
 protected:
@@ -430,7 +445,7 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     void convert_strategy_helper(FbcsrType *result) const;
 
     /**
-     * Computes srow. It should be run after changing any row_ptrs_ value.
+     * Computes srow. It should be run after changing any row pointer.
      */
     void make_srow()
     {
diff --git a/omp/matrix/fbcsr_kernels.cpp b/omp/matrix/fbcsr_kernels.cpp
index a38b3c9bda7..fc2e2bc17ba 100644
--- a/omp/matrix/fbcsr_kernels.cpp
+++ b/omp/matrix/fbcsr_kernels.cpp
@@ -47,13 +47,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/csr.hpp>
 
 
-#include "core/base/allocator.hpp"
-#include "core/base/iterator_factory.hpp"
-#include "core/components/prefix_sum.hpp"
-#include "core/matrix/fbcsr_builder.hpp"
-#include "omp/components/format_conversion.hpp"
-
-
 namespace gko {
 namespace kernels {
 namespace omp {
@@ -86,49 +79,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void spgemm_insert_row(unordered_set<IndexType> &cols,
-                       const matrix::Fbcsr<ValueType, IndexType> *c,
-                       size_type row) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto row_ptrs = c->get_const_row_ptrs();
-//    auto col_idxs = c->get_const_col_idxs();
-//    cols.insert(col_idxs + row_ptrs[row], col_idxs + row_ptrs[row + 1]);
-//}
-
-
-template <typename ValueType, typename IndexType>
-void spgemm_insert_row2(unordered_set<IndexType> &cols,
-                        const matrix::Fbcsr<ValueType, IndexType> *a,
-                        const matrix::Fbcsr<ValueType, IndexType> *b,
-                        size_type row) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto a_row_ptrs = a->get_const_row_ptrs();
-//    auto a_col_idxs = a->get_const_col_idxs();
-//    auto b_row_ptrs = b->get_const_row_ptrs();
-//    auto b_col_idxs = b->get_const_col_idxs();
-//    for (size_type a_nz = a_row_ptrs[row];
-//         a_nz < size_type(a_row_ptrs[row + 1]); ++a_nz) {
-//        auto a_col = a_col_idxs[a_nz];
-//        auto b_row = a_col;
-//        cols.insert(b_col_idxs + b_row_ptrs[b_row],
-//                    b_col_idxs + b_row_ptrs[b_row + 1]);
-//    }
-//}
-
-
-template <typename IndexType>
-void convert_row_ptrs_to_idxs(std::shared_ptr<const OmpExecutor> exec,
-                              const IndexType *ptrs, size_type num_rows,
-                              IndexType *idxs) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    convert_ptrs_to_idxs(ptrs, num_rows, idxs);
-//}
-
-
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const OmpExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *source,
@@ -162,38 +112,12 @@ void transpose_and_transform(std::shared_ptr<const OmpExecutor> exec,
                              matrix::Fbcsr<ValueType, IndexType> *trans,
                              const matrix::Fbcsr<ValueType, IndexType> *orig,
                              UnaryOperator op) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto trans_row_ptrs = trans->get_row_ptrs();
-//    auto orig_row_ptrs = orig->get_const_row_ptrs();
-//    auto trans_col_idxs = trans->get_col_idxs();
-//    auto orig_col_idxs = orig->get_const_col_idxs();
-//    auto trans_vals = trans->get_values();
-//    auto orig_vals = orig->get_const_values();
-//
-//    auto orig_num_cols = orig->get_size()[1];
-//    auto orig_num_rows = orig->get_size()[0];
-//    auto orig_nnz = orig_row_ptrs[orig_num_rows];
-//
-//    trans_row_ptrs[0] = 0;
-//    convert_unsorted_idxs_to_ptrs(orig_col_idxs, orig_nnz, trans_row_ptrs + 1,
-//                                  orig_num_cols);
-//
-//    convert_fbcsr_to_csc(orig_num_rows, orig_row_ptrs, orig_col_idxs,
-//    orig_vals,
-//                       trans_col_idxs, trans_row_ptrs + 1, trans_vals, op);
-//}
 
 
 template <typename ValueType, typename IndexType>
 void transpose(std::shared_ptr<const OmpExecutor> exec,
                const matrix::Fbcsr<ValueType, IndexType> *orig,
                matrix::Fbcsr<ValueType, IndexType> *trans) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    transpose_and_transform(exec, trans, orig,
-//                            [](const ValueType x) { return x; });
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL);
@@ -204,11 +128,6 @@ void conj_transpose(std::shared_ptr<const OmpExecutor> exec,
                     const matrix::Fbcsr<ValueType, IndexType> *orig,
                     matrix::Fbcsr<ValueType, IndexType> *trans)
     GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    transpose_and_transform(exec, trans, orig,
-//                            [](const ValueType x) { return conj(x); });
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONJ_TRANSPOSE_KERNEL);
@@ -229,16 +148,6 @@ void calculate_nonzeros_per_row(
     std::shared_ptr<const OmpExecutor> exec,
     const matrix::Fbcsr<ValueType, IndexType> *source,
     Array<size_type> *result) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto row_ptrs = source->get_const_row_ptrs();
-//    auto row_nnz_val = result->get_data();
-//
-//#pragma omp parallel for
-//    for (size_type i = 0; i < result->get_num_elems(); i++) {
-//        row_nnz_val[i] = row_ptrs[i + 1] - row_ptrs[i];
-//    }
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
@@ -258,26 +167,6 @@ void is_sorted_by_column_index(
     std::shared_ptr<const OmpExecutor> exec,
     const matrix::Fbcsr<ValueType, IndexType> *to_check,
     bool *is_sorted) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    const auto row_ptrs = to_check->get_const_row_ptrs();
-//    const auto col_idxs = to_check->get_const_col_idxs();
-//    const auto size = to_check->get_size();
-//    bool local_is_sorted = true;
-//#pragma omp parallel for reduction(&& : local_is_sorted)
-//    for (size_type i = 0; i < size[0]; ++i) {
-//        // Skip comparison if any thread detects that it is not sorted
-//        if (local_is_sorted) {
-//            for (auto idx = row_ptrs[i] + 1; idx < row_ptrs[i + 1]; ++idx) {
-//                if (col_idxs[idx - 1] > col_idxs[idx]) {
-//                    local_is_sorted = false;
-//                    break;
-//                }
-//            }
-//        }
-//    }
-//    *is_sorted = local_is_sorted;
-//}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);

From 2e45701f103a34d92455956cdab734c49ab56d12 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@mail.mcgill.ca>
Date: Fri, 27 Nov 2020 09:58:34 +0100
Subject: [PATCH 0547/1008] fixed header name for fbcsr hip kernels

---
 hip/matrix/fbcsr_kernels.hip.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hip/matrix/fbcsr_kernels.hip.cpp b/hip/matrix/fbcsr_kernels.hip.cpp
index 1ce2735d8f6..5e7e21d1a23 100644
--- a/hip/matrix/fbcsr_kernels.hip.cpp
+++ b/hip/matrix/fbcsr_kernels.hip.cpp
@@ -46,7 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
-#include "hip/base/config.hpp"
+#include "hip/base/config.hip.hpp"
 
 
 namespace gko {

From 6abd124f81bd08057c5d9264347985a2c8607656 Mon Sep 17 00:00:00 2001
From: Aditya <aditya.kashi@yahoo.com>
Date: Fri, 27 Nov 2020 11:01:56 +0100
Subject: [PATCH 0548/1008] Apply suggestions from code review

updated gtest macro

Co-authored-by: tcojean <terry.cojean@kit.edu>
---
 core/test/matrix/fbcsr.cpp              |  2 +-
 core/test/matrix/fbcsr_builder.cpp      |  2 +-
 dpcpp/matrix/fbcsr_kernels.dp.cpp       | 14 --------------
 reference/test/matrix/fbcsr_kernels.cpp |  4 ++--
 4 files changed, 4 insertions(+), 18 deletions(-)

diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index 893e12b0df6..10643276706 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -134,7 +134,7 @@ class Fbcsr : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Fbcsr, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Fbcsr, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(Fbcsr, SampleGeneratorIsCorrect)
diff --git a/core/test/matrix/fbcsr_builder.cpp b/core/test/matrix/fbcsr_builder.cpp
index 31ccc2145bc..60e6d93d6cd 100644
--- a/core/test/matrix/fbcsr_builder.cpp
+++ b/core/test/matrix/fbcsr_builder.cpp
@@ -64,7 +64,7 @@ class FbcsrBuilder : public ::testing::Test {
     std::unique_ptr<Mtx> mtx;
 };
 
-TYPED_TEST_CASE(FbcsrBuilder, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(FbcsrBuilder, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(FbcsrBuilder, ReturnsCorrectArrays)
diff --git a/dpcpp/matrix/fbcsr_kernels.dp.cpp b/dpcpp/matrix/fbcsr_kernels.dp.cpp
index 17c44830b9a..9e02173781f 100644
--- a/dpcpp/matrix/fbcsr_kernels.dp.cpp
+++ b/dpcpp/matrix/fbcsr_kernels.dp.cpp
@@ -33,28 +33,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/fbcsr_kernels.hpp"
 
 
-#include <algorithm>
-#include <numeric>
-#include <utility>
-
-
 #include <CL/sycl.hpp>
 
 
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/hybrid.hpp>
-
-
-#include "core/base/allocator.hpp"
-#include "core/base/iterator_factory.hpp"
-#include "core/components/prefix_sum.hpp"
-#include "core/matrix/fbcsr_builder.hpp"
-#include "dpcpp/components/format_conversion.dp.hpp"
 
 
 namespace gko {
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 367d643af14..9e389655bb9 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -135,7 +135,7 @@ class Fbcsr : public ::testing::Test {
     const std::unique_ptr<const Mtx> mtxsq;
 };
 
-TYPED_TEST_CASE(Fbcsr, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(Fbcsr, gko::test::ValueIndexTypes);
 
 
 template <typename T>
@@ -656,7 +656,7 @@ class FbcsrComplex : public ::testing::Test {
     using Mtx = gko::matrix::Fbcsr<value_type, index_type>;
 };
 
-TYPED_TEST_CASE(FbcsrComplex, gko::test::ComplexValueIndexTypes);
+TYPED_TEST_SUITE(FbcsrComplex, gko::test::ComplexValueIndexTypes);
 
 
 TYPED_TEST(FbcsrComplex, MtxIsConjugateTransposable)

From c9faaa64baf86e58a3589e83df7e68c2430f4335 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Mon, 30 Nov 2020 15:13:31 +0100
Subject: [PATCH 0549/1008] fixed bug in fbcsr apply tests

---
 core/test/matrix/CMakeLists.txt         | 1 +
 reference/test/matrix/fbcsr_kernels.cpp | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/core/test/matrix/CMakeLists.txt b/core/test/matrix/CMakeLists.txt
index 22a8d0828c6..f750c5a506f 100644
--- a/core/test/matrix/CMakeLists.txt
+++ b/core/test/matrix/CMakeLists.txt
@@ -12,6 +12,7 @@ target_include_directories(test_fbcsr_sample
   "$<BUILD_INTERFACE:${Ginkgo_SOURCE_DIR}>"
   "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}/include>"
   )
+set_property(TARGET test_fbcsr_sample PROPERTY CXX_STANDARD 14)
 ginkgo_create_test(fbcsr test_fbcsr_sample)
 ginkgo_create_test(fbcsr_builder)
 ginkgo_create_test(hybrid)
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 9e389655bb9..4a257ca750c 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -139,13 +139,13 @@ TYPED_TEST_SUITE(Fbcsr, gko::test::ValueIndexTypes);
 
 
 template <typename T>
-constexpr T get_some_number()
+constexpr typename std::enable_if_t<!gko::is_complex<T>(), T> get_some_number()
 {
     return static_cast<T>(1.2);
 }
 
 template <typename T>
-constexpr typename std::enable_if_t<gko::is_complex<T>, T> get_some_number()
+constexpr typename std::enable_if_t<gko::is_complex<T>(), T> get_some_number()
 {
     using RT = gko::remove_complex<T>;
     return {static_cast<RT>(1.2), static_cast<RT>(3.4)};

From 7b87706e4fac01566715205e2e9adc8506e66aac Mon Sep 17 00:00:00 2001
From: Aditya <aditya.kashi@yahoo.com>
Date: Mon, 30 Nov 2020 17:32:31 +0100
Subject: [PATCH 0550/1008] Apply suggestions from code review

Co-authored-by: tcojean <terry.cojean@kit.edu>
Co-authored-by: fritzgoebel <fritz.goebel@kit.edu>
---
 core/components/fixed_block.hpp             |  6 +-
 core/device_hooks/common_kernels.inc.cpp    |  6 --
 core/matrix/fbcsr.cpp                       | 39 ++++-------
 core/matrix/fbcsr_kernels.hpp               |  6 --
 core/test/matrix/fbcsr.cpp                  |  3 -
 core/test/matrix/fbcsr_sample.cpp           |  2 +-
 cuda/matrix/fbcsr_kernels.cu                |  9 ---
 dpcpp/matrix/fbcsr_kernels.dp.cpp           |  9 ---
 hip/matrix/fbcsr_kernels.hip.cpp            |  9 ---
 include/ginkgo/core/matrix/fbcsr.hpp        |  5 +-
 include/ginkgo/core/matrix/sparsity_csr.hpp |  1 +
 omp/matrix/fbcsr_kernels.cpp                |  9 ---
 reference/matrix/fbcsr_kernels.cpp          | 77 +--------------------
 reference/test/matrix/fbcsr_kernels.cpp     |  2 +-
 14 files changed, 24 insertions(+), 159 deletions(-)

diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
index 32c6fb8ddda..555f880e10f 100644
--- a/core/components/fixed_block.hpp
+++ b/core/components/fixed_block.hpp
@@ -131,10 +131,12 @@ class DenseBlock final {
 
     void resize(const int nrows, const int ncols)
     {
+        if (nrows * ncols != nrows_ * ncols_) {
+            delete[] vals_;
+            vals_ = new value_type[nrows * ncols];
+        }
         nrows_ = nrows;
         ncols_ = ncols;
-        delete[] vals_;
-        vals_ = new value_type[nrows_ * ncols_];
     }
 
     void zero()
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 9cca9e2d26f..b2818c4959a 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -854,12 +854,6 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
 
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
-
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 46fe819d240..4308cb616e8 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -42,11 +42,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/utils.hpp>
-#include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/ell.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
-#include <ginkgo/core/matrix/sellp.hpp>
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
@@ -71,7 +68,6 @@ GKO_REGISTER_OPERATION(calculate_max_nnz_per_row,
                        fbcsr::calculate_max_nnz_per_row);
 GKO_REGISTER_OPERATION(calculate_nonzeros_per_row,
                        fbcsr::calculate_nonzeros_per_row);
-GKO_REGISTER_OPERATION(sort_by_column_index, fbcsr::sort_by_column_index);
 GKO_REGISTER_OPERATION(is_sorted_by_column_index,
                        fbcsr::is_sorted_by_column_index);
 GKO_REGISTER_OPERATION(extract_diagonal, fbcsr::extract_diagonal);
@@ -92,11 +88,11 @@ Fbcsr<ValueType, IndexType>::Fbcsr(std::shared_ptr<const Executor> exec,
                                    std::shared_ptr<strategy_type> strategy)
     : EnableLinOp<Fbcsr>(exec, size),
       bs_{block_size},
-      nbcols_{gko::blockutils::getNumBlocks(block_size, size[1])},
+      nbcols_{blockutils::getNumBlocks(block_size, size[1])},
       values_(exec, num_nonzeros),
-      col_idxs_(exec, gko::blockutils::getNumBlocks(block_size * block_size,
-                                                    num_nonzeros)),
-      row_ptrs_(exec, gko::blockutils::getNumBlocks(block_size, size[0]) + 1),
+      col_idxs_(exec, blockutils::getNumBlocks(block_size * block_size,
+                                               num_nonzeros)),
+      row_ptrs_(exec, blockutils::getNumBlocks(block_size, size[0]) + 1),
       startrow_(exec, strategy->calc_size(num_nonzeros)),
       strategy_(strategy->copy())
 {}
@@ -107,8 +103,7 @@ void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const b,
                                              LinOp *const x) const
 {
     using Dense = Dense<ValueType>;
-    using TFbcsr = Fbcsr<ValueType, IndexType>;
-    if (auto b_fbcsr = dynamic_cast<const TFbcsr *>(b)) {
+    if (auto b_fbcsr = dynamic_cast<const Fbcsr<ValueType, IndexType> *>(b)) {
         // if b is a FBCSR matrix, we compute a SpGeMM
         throw /*::gko::*/ NotImplemented(__FILE__, __LINE__,
                                          "SpGeMM for Fbcsr");
@@ -127,13 +122,12 @@ void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const alpha,
                                              LinOp *const x) const
 {
     using Dense = Dense<ValueType>;
-    using TFbcsr = Fbcsr<ValueType, IndexType>;
-    if (auto b_fbcsr = dynamic_cast<const TFbcsr *>(b)) {
+    if (auto b_fbcsr = dynamic_cast<const Fbcsr<ValueType, IndexType> *>(b)) {
         // if b is a FBCSR matrix, we compute a SpGeMM
         throw NotImplemented(__FILE__, __LINE__, "Adv SpGeMM for Fbcsr");
     } else if (dynamic_cast<const Identity<ValueType> *>(b)) {
         // if b is an identity matrix, we compute an SpGEAM
-        throw NotImplemented(__FILE__, __LINE__, "Adv SpGeMM for Fbcsr");
+        throw NotImplemented(__FILE__, __LINE__, "Adv SpGEAM for Fbcsr");
     } else {
         // otherwise we assume that b is dense and compute a SpMV/SpMM
         this->get_executor()->run(
@@ -240,7 +234,7 @@ template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(
     SparsityCsr<ValueType, IndexType> *const result) const
 {
-    using gko::blockutils::getNumBlocks;
+    using blockutils::getNumBlocks;
     auto exec = this->get_executor();
     auto tmp = SparsityCsr<ValueType, IndexType>::create(
         exec,
@@ -250,7 +244,7 @@ void Fbcsr<ValueType, IndexType>::convert_to(
 
     tmp->col_idxs_ = this->col_idxs_;
     tmp->row_ptrs_ = this->row_ptrs_;
-    tmp->value_ = gko::Array<ValueType>(exec, {one<ValueType>()});
+    tmp->value_ = Array<ValueType>(exec, {one<ValueType>()});
     tmp->move_to(result);
 }
 
@@ -334,13 +328,11 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
                cur_bcol = blocks.begin()->first.block_column;
     const index_type num_brows = data.size[0] / bs;
 
-    gko::blockutils::DenseBlocksView<value_type, index_type> values(
+    blockutils::DenseBlocksView<value_type, index_type> values(
         tmp->values_.get_data(), bs, bs);
 
     for (auto it = blocks.begin(); it != blocks.end(); it++) {
-        if (cur_brow >= num_brows)
-            throw gko::OutOfBoundsError(__FILE__, __LINE__, cur_brow,
-                                        num_brows);
+        GKO_ENSURE_IN_BOUNDS(cur_brow, num_brows);
 
         tmp->col_idxs_.get_data()[cur_bnz] = it->first.block_column;
 
@@ -383,8 +375,8 @@ void Fbcsr<ValueType, IndexType>::write(mat_data &data) const
 
     data = {tmp->get_size(), {}};
 
-    const gko::blockutils::DenseBlocksView<const value_type, index_type>
-        vblocks(tmp->values_.get_const_data(), bs_, bs_);
+    const blockutils::DenseBlocksView<const value_type, index_type> vblocks(
+        tmp->values_.get_const_data(), bs_, bs_);
 
     for (size_type brow = 0; brow < tmp->get_size()[0] / bs_; ++brow) {
         const auto start = tmp->row_ptrs_.get_const_data()[brow];
@@ -435,11 +427,6 @@ std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::conj_transpose() const
 
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::sort_by_column_index() GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto exec = this->get_executor();
-//    exec->run(fbcsr::make_sort_by_column_index(this));
-//}
 
 
 template <typename ValueType, typename IndexType>
diff --git a/core/matrix/fbcsr_kernels.hpp b/core/matrix/fbcsr_kernels.hpp
index ed37c49153d..af7e527a891 100644
--- a/core/matrix/fbcsr_kernels.hpp
+++ b/core/matrix/fbcsr_kernels.hpp
@@ -95,10 +95,6 @@ namespace kernels {
         const matrix::Fbcsr<ValueType, IndexType> *source,             \
         Array<size_type> *result)
 
-#define GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX(ValueType, IndexType)       \
-    void sort_by_column_index(std::shared_ptr<const DefaultExecutor> exec, \
-                              matrix::Fbcsr<ValueType, IndexType> *to_sort)
-
 #define GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX(ValueType, IndexType) \
     void is_sorted_by_column_index(                                       \
         std::shared_ptr<const DefaultExecutor> exec,                      \
@@ -127,8 +123,6 @@ namespace kernels {
     template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL(ValueType, IndexType); \
     template <typename ValueType, typename IndexType>                          \
-    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX(ValueType, IndexType);              \
-    template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX(ValueType, IndexType);         \
     template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL(ValueType, IndexType)
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index 10643276706..d6257b49ddc 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -110,9 +110,6 @@ class Fbcsr : public ::testing::Test {
                 ASSERT_EQ(c[inz], orig_colinds[inz]);
 
                 for (int i = 0; i < bs * bs; i++) {
-                    // ASSERT_LT(gko::abs(v[inz*bs*bs + i] -
-                    // mtx->get_values()[inz*bs*bs + i]),
-                    //           std::numeric_limits<gko::remove_complex<value_type>>::epsilon());
                     ASSERT_EQ(v[inz * bs * bs + i],
                               orig_vals[inz * bs * bs + i]);
                 }
diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
index 72685d99eab..17879742892 100644
--- a/core/test/matrix/fbcsr_sample.cpp
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <algorithm>
 
 #include "core/components/fixed_block.hpp"
-#include "fbcsr_sample.hpp"
+#include "core/test/matrix/fbcsr_sample.hpp"
 
 #define FBCSR_TEST_OFFSET 0.000011118888
 
diff --git a/cuda/matrix/fbcsr_kernels.cu b/cuda/matrix/fbcsr_kernels.cu
index 7da16b2976d..b8d397ef5fc 100644
--- a/cuda/matrix/fbcsr_kernels.cu
+++ b/cuda/matrix/fbcsr_kernels.cu
@@ -159,15 +159,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void sort_by_column_index(std::shared_ptr<const CudaExecutor> exec,
-                          matrix::Fbcsr<ValueType, IndexType> *to_sort)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
-
-
 template <typename ValueType, typename IndexType>
 void is_sorted_by_column_index(
     std::shared_ptr<const CudaExecutor> exec,
diff --git a/dpcpp/matrix/fbcsr_kernels.dp.cpp b/dpcpp/matrix/fbcsr_kernels.dp.cpp
index 9e02173781f..72a708bc5ff 100644
--- a/dpcpp/matrix/fbcsr_kernels.dp.cpp
+++ b/dpcpp/matrix/fbcsr_kernels.dp.cpp
@@ -155,15 +155,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void sort_by_column_index(std::shared_ptr<const DpcppExecutor> exec,
-                          matrix::Fbcsr<ValueType, IndexType> *to_sort)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
-
-
 template <typename ValueType, typename IndexType>
 void is_sorted_by_column_index(
     std::shared_ptr<const DpcppExecutor> exec,
diff --git a/hip/matrix/fbcsr_kernels.hip.cpp b/hip/matrix/fbcsr_kernels.hip.cpp
index 5e7e21d1a23..ed2eb2667c1 100644
--- a/hip/matrix/fbcsr_kernels.hip.cpp
+++ b/hip/matrix/fbcsr_kernels.hip.cpp
@@ -156,15 +156,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void sort_by_column_index(std::shared_ptr<const HipExecutor> exec,
-                          matrix::Fbcsr<ValueType, IndexType> *to_sort)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
-
-
 template <typename ValueType, typename IndexType>
 void is_sorted_by_column_index(
     std::shared_ptr<const HipExecutor> exec,
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index 30dadb68e54..3c455d7ea5d 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -38,8 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/blockutils.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/base/math.hpp>
-
-#include "matrix_strategies.hpp"
+#include <ginkgo/core/matrix/matrix_strategies.hpp>
 
 
 namespace gko {
@@ -422,7 +421,7 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
         std::shared_ptr<strategy_type> strategy = std::make_shared<classical>())
         : EnableLinOp<Fbcsr>(exec, size),
           bs_{block_size},
-          nbcols_{gko::blockutils::getNumBlocks(block_size, size[1])},
+          nbcols_{blockutils::getNumBlocks(block_size, size[1])},
           values_{exec, std::forward<ValuesArray>(values)},
           col_idxs_{exec, std::forward<ColIdxsArray>(col_idxs)},
           row_ptrs_{exec, std::forward<RowPtrsArray>(row_ptrs)},
diff --git a/include/ginkgo/core/matrix/sparsity_csr.hpp b/include/ginkgo/core/matrix/sparsity_csr.hpp
index b5f16f5ae3e..bce6621dd7d 100644
--- a/include/ginkgo/core/matrix/sparsity_csr.hpp
+++ b/include/ginkgo/core/matrix/sparsity_csr.hpp
@@ -48,6 +48,7 @@ namespace matrix {
 template <typename ValueType, typename IndexType>
 class Csr;
 
+
 template <typename ValueType, typename IndexType>
 class Fbcsr;
 
diff --git a/omp/matrix/fbcsr_kernels.cpp b/omp/matrix/fbcsr_kernels.cpp
index fc2e2bc17ba..442d6f7c107 100644
--- a/omp/matrix/fbcsr_kernels.cpp
+++ b/omp/matrix/fbcsr_kernels.cpp
@@ -153,15 +153,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void sort_by_column_index(std::shared_ptr<const OmpExecutor> exec,
-                          matrix::Fbcsr<ValueType, IndexType> *to_sort)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
-
-
 template <typename ValueType, typename IndexType>
 void is_sorted_by_column_index(
     std::shared_ptr<const OmpExecutor> exec,
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index 8060d777ec2..c59f8a17e52 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -77,7 +77,7 @@ void spmv(const std::shared_ptr<const ReferenceExecutor> exec,
     auto row_ptrs = a->get_const_row_ptrs();
     auto col_idxs = a->get_const_col_idxs();
     auto vals = a->get_const_values();
-    const gko::blockutils::DenseBlocksView<const ValueType, IndexType> avalues(
+    const blockutils::DenseBlocksView<const ValueType, IndexType> avalues(
         vals, bs, bs);
 
     ValueType *const cvals = c->get_values();
@@ -121,7 +121,7 @@ void advanced_spmv(const std::shared_ptr<const ReferenceExecutor> exec,
     auto vals = a->get_const_values();
     auto valpha = alpha->at(0, 0);
     auto vbeta = beta->at(0, 0);
-    const gko::blockutils::DenseBlocksView<const ValueType, IndexType> avalues(
+    const blockutils::DenseBlocksView<const ValueType, IndexType> avalues(
         vals, bs, bs);
 
     ValueType *const cvals = c->get_values();
@@ -150,70 +150,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void spgemm_insert_row(unordered_set<IndexType> &cols,
-                       const matrix::Fbcsr<ValueType, IndexType> *c,
-                       size_type row) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto row_ptrs = c->get_const_row_ptrs();
-//    auto col_idxs = c->get_const_col_idxs();
-//    cols.insert(col_idxs + row_ptrs[row], col_idxs + row_ptrs[row + 1]);
-//}
-
-
-template <typename ValueType, typename IndexType>
-void spgemm_insert_row2(unordered_set<IndexType> &cols,
-                        const matrix::Fbcsr<ValueType, IndexType> *a,
-                        const matrix::Fbcsr<ValueType, IndexType> *b,
-                        size_type row) GKO_NOT_IMPLEMENTED;
-//{
-// TODO (script:fbcsr): change the code imported from matrix/csr if needed
-//    auto a_row_ptrs = a->get_const_row_ptrs();
-//    auto a_col_idxs = a->get_const_col_idxs();
-//    auto b_row_ptrs = b->get_const_row_ptrs();
-//    auto b_col_idxs = b->get_const_col_idxs();
-//    for (size_type a_nz = a_row_ptrs[row];
-//         a_nz < size_type(a_row_ptrs[row + 1]); ++a_nz) {
-//        auto a_col = a_col_idxs[a_nz];
-//        auto b_row = a_col;
-//        cols.insert(b_col_idxs + b_row_ptrs[b_row],
-//                    b_col_idxs + b_row_ptrs[b_row + 1]);
-//    }
-//}
-
-
-template <typename ValueType, typename IndexType>
-void spgemm_accumulate_row(map<IndexType, ValueType> &cols,
-                           const matrix::Fbcsr<ValueType, IndexType> *c,
-                           ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
-
-
-template <typename ValueType, typename IndexType>
-void spgemm_accumulate_row2(map<IndexType, ValueType> &cols,
-                            const matrix::Fbcsr<ValueType, IndexType> *a,
-                            const matrix::Fbcsr<ValueType, IndexType> *b,
-                            ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
-
-
-template <typename ValueType, typename IndexType>
-void spgemm(std::shared_ptr<const ReferenceExecutor> exec,
-            const matrix::Fbcsr<ValueType, IndexType> *a,
-            const matrix::Fbcsr<ValueType, IndexType> *b,
-            matrix::Fbcsr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
-
-
-template <typename ValueType, typename IndexType>
-void advanced_spgemm(std::shared_ptr<const ReferenceExecutor> exec,
-                     const matrix::Dense<ValueType> *alpha,
-                     const matrix::Fbcsr<ValueType, IndexType> *a,
-                     const matrix::Fbcsr<ValueType, IndexType> *b,
-                     const matrix::Dense<ValueType> *beta,
-                     const matrix::Fbcsr<ValueType, IndexType> *d,
-                     matrix::Fbcsr<ValueType, IndexType> *c)
-    GKO_NOT_IMPLEMENTED;
-
-
 template <typename ValueType, typename IndexType>
 void convert_to_dense(const std::shared_ptr<const ReferenceExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *const source,
@@ -435,15 +371,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void sort_by_column_index(std::shared_ptr<const ReferenceExecutor> exec,
-                          matrix::Fbcsr<ValueType, IndexType> *to_sort)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
-
-
 template <typename ValueType, typename IndexType>
 void is_sorted_by_column_index(
     std::shared_ptr<const ReferenceExecutor> exec,
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 4a257ca750c..5cce425a3cb 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -547,7 +547,7 @@ TYPED_TEST(Fbcsr, CalculatesNonzerosPerRow)
 
     gko::Array<gko::size_type> row_nnz(this->exec, this->mtx2->get_size()[0]);
 
-    gko::kernels::reference::fbcsr ::calculate_nonzeros_per_row(
+    gko::kernels::reference::fbcsr::calculate_nonzeros_per_row(
         this->exec, this->mtx2.get(), &row_nnz);
 
     auto row_nnz_val = row_nnz.get_data();

From baf20cb18d303adf8116459867e49ffecd2c4472 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@mail.mcgill.ca>
Date: Mon, 30 Nov 2020 23:25:49 +0100
Subject: [PATCH 0551/1008] attempt to fix MSVC SFINAE issue and Apple linker
 errors

---
 core/test/matrix/CMakeLists.txt   | 3 ++-
 core/test/matrix/fbcsr_sample.hpp | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/core/test/matrix/CMakeLists.txt b/core/test/matrix/CMakeLists.txt
index f750c5a506f..203821ae392 100644
--- a/core/test/matrix/CMakeLists.txt
+++ b/core/test/matrix/CMakeLists.txt
@@ -5,7 +5,7 @@ ginkgo_create_test(csr_builder)
 ginkgo_create_test(dense)
 ginkgo_create_test(diagonal)
 ginkgo_create_test(ell)
-add_library(test_fbcsr_sample fbcsr_sample.cpp)
+add_library(test_fbcsr_sample STATIC fbcsr_sample.cpp)
 target_include_directories(test_fbcsr_sample
   PRIVATE
   "$<BUILD_INTERFACE:${Ginkgo_SOURCE_DIR}/include>"
@@ -13,6 +13,7 @@ target_include_directories(test_fbcsr_sample
   "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}/include>"
   )
 set_property(TARGET test_fbcsr_sample PROPERTY CXX_STANDARD 14)
+set_property(TARGET test_fbcsr_sample PROPERTY CXX_EXTENSIONS OFF)
 ginkgo_create_test(fbcsr test_fbcsr_sample)
 ginkgo_create_test(fbcsr_builder)
 ginkgo_create_test(hybrid)
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index e6fe63dc37b..4d23eb5557b 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -155,11 +155,11 @@ class FbcsrSample {
 
     /// ... while ignoring imaginary parts for real instantiations
     template <typename U>
-    constexpr std::enable_if_t<is_complex<U>() && !is_complex<ValueType>(),
+    constexpr std::enable_if_t<!is_complex<U>() && !is_complex<ValueType>(),
                                ValueType>
-    sct(U u) const
+    sct(std::complex<U> cu) const
     {
-        return static_cast<ValueType>(u.real());
+        return static_cast<ValueType>(cu.real());
     }
 };
 

From 97acb1d733e72cb60a002d833e63be954ecaa645 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Tue, 1 Dec 2020 10:46:57 +0100
Subject: [PATCH 0552/1008] removed constexpr with std complex constructor to
 satisfy older toolchains, added ginkgo as explicit dependency of fbcsr_sample

---
 core/test/matrix/CMakeLists.txt         | 1 +
 reference/test/matrix/fbcsr_kernels.cpp | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/core/test/matrix/CMakeLists.txt b/core/test/matrix/CMakeLists.txt
index 203821ae392..9376b970552 100644
--- a/core/test/matrix/CMakeLists.txt
+++ b/core/test/matrix/CMakeLists.txt
@@ -14,6 +14,7 @@ target_include_directories(test_fbcsr_sample
   )
 set_property(TARGET test_fbcsr_sample PROPERTY CXX_STANDARD 14)
 set_property(TARGET test_fbcsr_sample PROPERTY CXX_EXTENSIONS OFF)
+target_link_libraries(test_fbcsr_sample PRIVATE ginkgo)
 ginkgo_create_test(fbcsr test_fbcsr_sample)
 ginkgo_create_test(fbcsr_builder)
 ginkgo_create_test(hybrid)
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 5cce425a3cb..82fb882c518 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -212,8 +212,8 @@ TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseVector)
     using T = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
 
-    constexpr T alphav = -1.0;
-    constexpr T betav = 2.0;
+    const T alphav = -1.0;
+    const T betav = 2.0;
     auto alpha = gko::initialize<Vec>({alphav}, this->exec);
     auto beta = gko::initialize<Vec>({betav}, this->exec);
 
@@ -254,8 +254,8 @@ TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseMatrix)
     using T = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
 
-    constexpr T alphav = -1.0;
-    constexpr T betav = 2.0;
+    const T alphav = -1.0;
+    const T betav = 2.0;
     auto alpha = gko::initialize<Vec>({alphav}, this->exec);
     auto beta = gko::initialize<Vec>({betav}, this->exec);
 

From 5ab1e0eea4b1a6f24a78cdb7f0b1c413135d3da5 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Tue, 1 Dec 2020 12:22:30 +0100
Subject: [PATCH 0553/1008] fbcsr complex sample class now uses correct index
 types

---
 core/components/fixed_block.hpp                  |  7 ++++---
 core/test/matrix/fbcsr.cpp                       |  8 ++++++--
 core/test/matrix/fbcsr_sample.cpp                | 11 ++++++-----
 core/test/matrix/fbcsr_sample.hpp                |  7 +++----
 include/ginkgo/core/base/blockutils.hpp          |  1 -
 include/ginkgo/core/matrix/fbcsr.hpp             |  6 +++---
 include/ginkgo/core/matrix/matrix_strategies.hpp |  6 +++---
 reference/test/matrix/fbcsr_kernels.cpp          |  6 ++++--
 8 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
index 555f880e10f..14c035fc965 100644
--- a/core/components/fixed_block.hpp
+++ b/core/components/fixed_block.hpp
@@ -34,11 +34,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_COMPONENTS_FIXED_BLOCK_HPP_
 
 
+#include <type_traits>
+
+
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/math.hpp>
 
-#include <type_traits>
-
 namespace gko {
 namespace blockutils {
 
@@ -208,4 +209,4 @@ class DenseBlocksView final {
 }  // namespace blockutils
 }  // namespace gko
 
-#endif
+#endif  // GKO_CORE_COMPONENTS_FIXED_BLOCK_HPP_
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index d6257b49ddc..57b5b09e341 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -31,14 +31,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #include <ginkgo/core/matrix/fbcsr.hpp>
-#include <ginkgo/core/matrix/matrix_strategies.hpp>
 
 
-#include <gtest/gtest.h>
 #include <iostream>
 #include <limits>
 
 
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/matrix/matrix_strategies.hpp>
+
+
 #include "core/components/fixed_block.hpp"
 #include "core/test/matrix/fbcsr_sample.hpp"
 #include "core/test/utils.hpp"
diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
index 17879742892..a5b55a5de3a 100644
--- a/core/test/matrix/fbcsr_sample.cpp
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -30,10 +30,11 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include <algorithm>
+
 
 #include <ginkgo/core/matrix/matrix_strategies.hpp>
 
-#include <algorithm>
 
 #include "core/components/fixed_block.hpp"
 #include "core/test/matrix/fbcsr_sample.hpp"
@@ -878,10 +879,10 @@ FbcsrSampleComplex<ValueType, IndexType>::generate_conjtranspose_fbcsr() const
     return mtx;
 }
 
-template class FbcsrSampleComplex<std::complex<float>, int>;
-template class FbcsrSampleComplex<std::complex<double>, int>;
-template class FbcsrSampleComplex<std::complex<float>, long>;
-template class FbcsrSampleComplex<std::complex<double>, long>;
+template class FbcsrSampleComplex<std::complex<float>, int32>;
+template class FbcsrSampleComplex<std::complex<double>, int32>;
+template class FbcsrSampleComplex<std::complex<float>, int64>;
+template class FbcsrSampleComplex<std::complex<double>, int64>;
 
 }  // namespace testing
 }  // namespace gko
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 4d23eb5557b..12ec1fb310f 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -30,9 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-
-#ifndef GKO_CORE_MATRIX_TEST_FBCSR_SAMPLE_HPP
-#define GKO_CORE_MATRIX_TEST_FBCSR_SAMPLE_HPP
+#ifndef GKO_CORE_TEST_MATRIX_FBCSR_SAMPLE_HPP_
+#define GKO_CORE_TEST_MATRIX_FBCSR_SAMPLE_HPP_
 
 
 #include <ginkgo/core/base/matrix_data.hpp>
@@ -270,4 +269,4 @@ class FbcsrSampleComplex {
 }  // namespace testing
 }  // namespace gko
 
-#endif
+#endif  // GKO_CORE_TEST_MATRIX_FBCSR_SAMPLE_HPP_
diff --git a/include/ginkgo/core/base/blockutils.hpp b/include/ginkgo/core/base/blockutils.hpp
index c057e1dec04..f3664e1d092 100644
--- a/include/ginkgo/core/base/blockutils.hpp
+++ b/include/ginkgo/core/base/blockutils.hpp
@@ -30,7 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-
 #ifndef GINKGO_CORE_BASE_BLOCKUTILS_HPP_
 #define GINKGO_CORE_BASE_BLOCKUTILS_HPP_
 
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index 3c455d7ea5d..dcb3e0fcb6a 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_MATRIX_FBCSR_HPP_
-#define GKO_CORE_MATRIX_FBCSR_HPP_
+#ifndef GKO_PUBLIC_CORE_MATRIX_FBCSR_HPP_
+#define GKO_PUBLIC_CORE_MATRIX_FBCSR_HPP_
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -468,4 +468,4 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
 }  // namespace gko
 
 
-#endif  // GKO_CORE_MATRIX_FBCSR_HPP_
+#endif  // GKO_PUBLIC_CORE_MATRIX_FBCSR_HPP_
diff --git a/include/ginkgo/core/matrix/matrix_strategies.hpp b/include/ginkgo/core/matrix/matrix_strategies.hpp
index 47133b3610f..30dd715cf55 100644
--- a/include/ginkgo/core/matrix/matrix_strategies.hpp
+++ b/include/ginkgo/core/matrix/matrix_strategies.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_MATRIX_MATRIX_STRATEGY_HPP_
-#define GKO_CORE_MATRIX_MATRIX_STRATEGY_HPP_
+#ifndef GKO_PUBLIC_CORE_MATRIX_MATRIX_STRATEGIES_HPP_
+#define GKO_PUBLIC_CORE_MATRIX_MATRIX_STRATEGIES_HPP_
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -500,4 +500,4 @@ void strategy_rebuild_helper(MtxType *const result)
 }  // namespace matrix
 }  // namespace gko
 
-#endif
+#endif  // GKO_PUBLIC_CORE_MATRIX_MATRIX_STRATEGIES_HPP_
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 82fb882c518..cafce276ae1 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -139,13 +139,15 @@ TYPED_TEST_SUITE(Fbcsr, gko::test::ValueIndexTypes);
 
 
 template <typename T>
-constexpr typename std::enable_if_t<!gko::is_complex<T>(), T> get_some_number()
+constexpr typename std::enable_if_t<!gko::is_complex_s<T>::value, T>
+get_some_number()
 {
     return static_cast<T>(1.2);
 }
 
 template <typename T>
-constexpr typename std::enable_if_t<gko::is_complex<T>(), T> get_some_number()
+constexpr typename std::enable_if_t<gko::is_complex_s<T>::value, T>
+get_some_number()
 {
     using RT = gko::remove_complex<T>;
     return {static_cast<RT>(1.2), static_cast<RT>(3.4)};

From c3d304f86488f754cd4ab99adf3f35c1c79c119a Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 2 Dec 2020 15:42:46 +0100
Subject: [PATCH 0554/1008] removed strategies from fbcsr

---
 core/matrix/fbcsr.cpp                         | 105 +---
 core/matrix/fbcsr_builder.hpp                 |   5 +-
 core/test/matrix/fbcsr.cpp                    |  17 +-
 core/test/matrix/fbcsr_builder.cpp            |  36 +-
 core/test/matrix/fbcsr_sample.cpp             |  39 +-
 include/ginkgo/core/matrix/fbcsr.hpp          | 113 +---
 .../ginkgo/core/matrix/matrix_strategies.hpp  | 503 ------------------
 include/ginkgo/ginkgo.hpp                     |   1 -
 reference/test/matrix/fbcsr_kernels.cpp       |   9 -
 9 files changed, 36 insertions(+), 792 deletions(-)
 delete mode 100644 include/ginkgo/core/matrix/matrix_strategies.hpp

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 4308cb616e8..cb7b40702cb 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -82,19 +82,17 @@ GKO_REGISTER_OPERATION(outplace_absolute_array,
 
 
 template <typename ValueType, typename IndexType>
-Fbcsr<ValueType, IndexType>::Fbcsr(std::shared_ptr<const Executor> exec,
-                                   const dim<2> &size, size_type num_nonzeros,
-                                   int block_size,
-                                   std::shared_ptr<strategy_type> strategy)
+Fbcsr<ValueType, IndexType>::Fbcsr(const std::shared_ptr<const Executor> exec,
+                                   const dim<2> &size,
+                                   const size_type num_nonzeros,
+                                   const int block_size)
     : EnableLinOp<Fbcsr>(exec, size),
       bs_{block_size},
       nbcols_{blockutils::getNumBlocks(block_size, size[1])},
       values_(exec, num_nonzeros),
       col_idxs_(exec, blockutils::getNumBlocks(block_size * block_size,
                                                num_nonzeros)),
-      row_ptrs_(exec, blockutils::getNumBlocks(block_size, size[0]) + 1),
-      startrow_(exec, strategy->calc_size(num_nonzeros)),
-      strategy_(strategy->copy())
+      row_ptrs_(exec, blockutils::getNumBlocks(block_size, size[0]) + 1)
 {}
 
 
@@ -141,20 +139,11 @@ template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(
     Fbcsr<ValueType, IndexType> *const result) const
 {
-    bool same_executor = this->get_executor() == result->get_executor();
-    // NOTE: as soon as strategies are improved, this can be reverted
     result->values_ = this->values_;
     result->col_idxs_ = this->col_idxs_;
     result->row_ptrs_ = this->row_ptrs_;
-    result->startrow_ = this->startrow_;
     result->set_size(this->get_size());
     result->bs_ = this->bs_;
-    if (!same_executor) {
-        convert_strategy_helper(result);
-    } else {
-        result->set_strategy(std::move(this->get_strategy()->copy()));
-    }
-    // END NOTE
 }
 
 
@@ -162,11 +151,7 @@ template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::move_to(
     Fbcsr<ValueType, IndexType> *const result)
 {
-    bool same_executor = this->get_executor() == result->get_executor();
     EnableLinOp<Fbcsr>::move_to(result);
-    if (!same_executor) {
-        matrix_strategy::strategy_rebuild_helper(result);
-    }
 }
 
 
@@ -179,7 +164,6 @@ void Fbcsr<ValueType, IndexType>::convert_to(
     result->row_ptrs_ = this->row_ptrs_;
     result->set_size(this->get_size());
     result->bs_ = this->bs_;
-    convert_strategy_helper(result);
 }
 
 
@@ -321,7 +305,7 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
     const std::map<FbEntry, Blk_t, FbLess> blocks = create_block_map(data);
 
     auto tmp = Fbcsr::create(this->get_executor()->get_master(), data.size,
-                             blocks.size() * bs * bs, bs, this->get_strategy());
+                             blocks.size() * bs * bs, bs);
 
     tmp->row_ptrs_.get_data()[0] = 0;
     index_type cur_brow = 0, cur_bnz = 0,
@@ -356,7 +340,6 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
 
     assert(cur_brow == tmp->get_size()[0] / bs);
 
-    tmp->make_srow();
     tmp->move_to(this);
 }
 
@@ -402,11 +385,9 @@ std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::transpose() const
 {
     auto exec = this->get_executor();
     auto trans_cpy = Fbcsr::create(exec, gko::transpose(this->get_size()),
-                                   this->get_num_stored_elements(), bs_,
-                                   this->get_strategy());
+                                   this->get_num_stored_elements(), bs_);
 
     exec->run(fbcsr::make_transpose(this, trans_cpy.get()));
-    trans_cpy->make_srow();
     return std::move(trans_cpy);
 }
 
@@ -416,11 +397,9 @@ std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::conj_transpose() const
 {
     auto exec = this->get_executor();
     auto trans_cpy = Fbcsr::create(exec, gko::transpose(this->get_size()),
-                                   this->get_num_stored_elements(), bs_,
-                                   this->get_strategy());
+                                   this->get_num_stored_elements(), bs_);
 
     exec->run(fbcsr::make_conj_transpose(this, trans_cpy.get()));
-    trans_cpy->make_srow();
     return std::move(trans_cpy);
 }
 
@@ -480,78 +459,10 @@ Fbcsr<ValueType, IndexType>::compute_absolute() const
         this->get_const_values(), this->get_num_stored_elements(),
         abs_fbcsr->get_values()));
 
-    convert_strategy_helper(abs_fbcsr.get());
     return abs_fbcsr;
 }
 
 
-// TODO clean this up as soon as we improve strategy_type
-template <typename ValueType, typename IndexType>
-template <typename FbcsrType>
-void Fbcsr<ValueType, IndexType>::convert_strategy_helper(
-    FbcsrType *const result) const
-{
-    auto strat = this->get_strategy().get();
-    std::shared_ptr<typename matrix_strategy::strategy_type<FbcsrType>>
-        new_strat;
-    using classical = matrix_strategy::classical<FbcsrType>;
-    using load_balance = matrix_strategy::load_balance<FbcsrType>;
-    using automatic = matrix_strategy::automatic<FbcsrType>;
-
-    if (dynamic_cast<classical *>(strat)) {
-        new_strat = std::make_shared<classical>();
-    } else {
-        auto rexec = result->get_executor();
-        auto cuda_exec = std::dynamic_pointer_cast<const CudaExecutor>(rexec);
-        auto hip_exec = std::dynamic_pointer_cast<const HipExecutor>(rexec);
-        auto lb = dynamic_cast<load_balance *>(strat);
-        if (cuda_exec) {
-            if (lb) {
-                new_strat = std::make_shared<load_balance>(cuda_exec);
-            } else {
-                new_strat = std::make_shared<automatic>(cuda_exec);
-            }
-        } else if (hip_exec) {
-            if (lb) {
-                new_strat = std::make_shared<load_balance>(hip_exec);
-            } else {
-                new_strat = std::make_shared<automatic>(hip_exec);
-            }
-        } else {
-            // Try to preserve this executor's configuration
-            auto this_cuda_exec = std::dynamic_pointer_cast<const CudaExecutor>(
-                this->get_executor());
-            auto this_hip_exec = std::dynamic_pointer_cast<const HipExecutor>(
-                this->get_executor());
-            if (this_cuda_exec) {
-                if (lb) {
-                    new_strat = std::make_shared<load_balance>(this_cuda_exec);
-                } else {
-                    new_strat = std::make_shared<automatic>(this_cuda_exec);
-                }
-            } else if (this_hip_exec) {
-                if (lb) {
-                    new_strat = std::make_shared<load_balance>(this_hip_exec);
-                } else {
-                    new_strat = std::make_shared<automatic>(this_hip_exec);
-                }
-            } else {
-                // We had a load balance or automatic strategy from a non
-                // HIP or Cuda executor and are moving to a non HIP or Cuda
-                // executor.
-                // FIXME this creates a long delay
-                if (lb) {
-                    new_strat = std::make_shared<load_balance>();
-                } else {
-                    new_strat = std::make_shared<automatic>();
-                }
-            }
-        }
-    }
-    result->set_strategy(new_strat);
-}
-
-
 #define GKO_DECLARE_FBCSR_MATRIX(ValueType, IndexType) \
     class Fbcsr<ValueType, IndexType>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_MATRIX);
diff --git a/core/matrix/fbcsr_builder.hpp b/core/matrix/fbcsr_builder.hpp
index 62096ae27fc..b99aebe824b 100644
--- a/core/matrix/fbcsr_builder.hpp
+++ b/core/matrix/fbcsr_builder.hpp
@@ -75,10 +75,7 @@ class FbcsrBuilder {
         : matrix_{matrix}
     {}
 
-    /**
-     * Updates the internal matrix data structures at destruction.
-     */
-    ~FbcsrBuilder() { matrix_->make_srow(); }
+    ~FbcsrBuilder() {}
 
     // make this type non-movable
     FbcsrBuilder(const FbcsrBuilder &) = delete;
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index 57b5b09e341..b32dbb76367 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -40,9 +40,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
-#include <ginkgo/core/matrix/matrix_strategies.hpp>
-
-
 #include "core/components/fixed_block.hpp"
 #include "core/test/matrix/fbcsr_sample.hpp"
 #include "core/test/utils.hpp"
@@ -51,9 +48,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
-namespace matstr = gko::matrix::matrix_strategy;
-
-
 template <typename ValueIndexType>
 class Fbcsr : public ::testing::Test {
 protected:
@@ -97,7 +91,6 @@ class Fbcsr : public ::testing::Test {
         auto v = m->get_const_values();
         auto c = m->get_const_col_idxs();
         auto r = m->get_const_row_ptrs();
-        auto s = m->get_const_srow();
 
         const int bs = 3;
 
@@ -119,8 +112,6 @@ class Fbcsr : public ::testing::Test {
                 }
             }
         }
-
-        ASSERT_EQ(m->get_num_srow_elements(), 0);
     }
 
     void assert_empty(const Mtx *m)
@@ -131,7 +122,6 @@ class Fbcsr : public ::testing::Test {
         ASSERT_EQ(m->get_const_values(), nullptr);
         ASSERT_EQ(m->get_const_col_idxs(), nullptr);
         ASSERT_NE(m->get_const_row_ptrs(), nullptr);
-        ASSERT_EQ(m->get_const_srow(), nullptr);
     }
 };
 
@@ -235,10 +225,8 @@ TYPED_TEST(Fbcsr, CanBeCreatedFromExistingData)
         this->exec, gko::dim<2>{nbrows * bs, nbcols * bs}, bs,
         gko::Array<value_type>::view(this->exec, bnnz * bs * bs, values),
         gko::Array<index_type>::view(this->exec, bnnz, col_idxs),
-        gko::Array<index_type>::view(this->exec, nbrows + 1, row_ptrs),
-        std::make_shared<matstr::classical<Mtx>>());
+        gko::Array<index_type>::view(this->exec, nbrows + 1, row_ptrs));
 
-    ASSERT_EQ(mtx->get_num_srow_elements(), 0);
     ASSERT_EQ(mtx->get_const_values(), values);
     ASSERT_EQ(mtx->get_const_col_idxs(), col_idxs);
     ASSERT_EQ(mtx->get_const_row_ptrs(), row_ptrs);
@@ -291,8 +279,7 @@ TYPED_TEST(Fbcsr, CanBeCleared)
 TYPED_TEST(Fbcsr, CanBeReadFromMatrixData)
 {
     using Mtx = typename TestFixture::Mtx;
-    auto m =
-        Mtx::create(this->exec, std::make_shared<matstr::classical<Mtx>>());
+    auto m = Mtx::create(this->exec);
     m->set_block_size(this->fbsample.bs);
 
     m->read(this->fbsample.generate_matrix_data());
diff --git a/core/test/matrix/fbcsr_builder.cpp b/core/test/matrix/fbcsr_builder.cpp
index 60e6d93d6cd..684aa2e324e 100644
--- a/core/test/matrix/fbcsr_builder.cpp
+++ b/core/test/matrix/fbcsr_builder.cpp
@@ -57,7 +57,7 @@ class FbcsrBuilder : public ::testing::Test {
 protected:
     FbcsrBuilder()
         : exec(gko::ReferenceExecutor::create()),
-          mtx(Mtx::create(exec, gko::dim<2>{2, 3}, 4))
+          mtx(Mtx::create(exec, gko::dim<2>{4, 6}, 8, 2))
     {}
 
     std::shared_ptr<const gko::Executor> exec;
@@ -83,38 +83,4 @@ TYPED_TEST(FbcsrBuilder, ReturnsCorrectArrays)
 }
 
 
-TYPED_TEST(FbcsrBuilder, UpdatesSrowOnDestruction)
-{
-    using Mtx = typename TestFixture::Mtx;
-    using value_type = typename TestFixture::value_type;
-    using index_type = typename TestFixture::index_type;
-    struct mock_strategy
-        : public gko::matrix::matrix_strategy::strategy_type<Mtx> {
-        virtual void process(const gko::Array<index_type> &,
-                             gko::Array<index_type> *) override
-        {
-            *was_called = true;
-        }
-
-        virtual int64_t calc_size(const int64_t nnz) override { return 0; }
-
-        virtual std::shared_ptr<typename Mtx::strategy_type> copy() override
-        {
-            return std::make_shared<mock_strategy>(*was_called);
-        }
-
-        mock_strategy(bool &flag) : Mtx::strategy_type(""), was_called(&flag) {}
-
-        bool *was_called;
-    };
-    bool was_called{};
-    this->mtx->set_strategy(std::make_shared<mock_strategy>(was_called));
-    was_called = false;
-
-    gko::matrix::FbcsrBuilder<value_type, index_type>{this->mtx.get()};
-
-    ASSERT_TRUE(was_called);
-}
-
-
 }  // namespace
diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
index a5b55a5de3a..555f2db8627 100644
--- a/core/test/matrix/fbcsr_sample.cpp
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <algorithm>
 
 
-#include <ginkgo/core/matrix/matrix_strategies.hpp>
-
-
 #include "core/components/fixed_block.hpp"
 #include "core/test/matrix/fbcsr_sample.hpp"
 
@@ -49,8 +46,6 @@ namespace gko {
 namespace testing {
 
 
-namespace matstr = gko::matrix::matrix_strategy;
-
 /** Generates a copy of the given matrix with a different scalar type
  *
  * \tparam AbsValueType The scalar type of the output matrix
@@ -63,14 +58,13 @@ generate_acopy_impl(const FbcsrType *const mat)
     using index_type = typename FbcsrType::index_type;
     using value_type = typename FbcsrType::value_type;
     using AbsFbcsr = gko::matrix::Fbcsr<AbsValueType, index_type>;
-    using classical = matstr::classical<AbsFbcsr>;
 
     std::shared_ptr<const ReferenceExecutor> exec =
         std::dynamic_pointer_cast<const ReferenceExecutor>(mat->get_executor());
 
     std::unique_ptr<AbsFbcsr> amat =
         AbsFbcsr::create(exec, mat->get_size(), mat->get_num_stored_elements(),
-                         mat->get_block_size(), std::make_shared<classical>());
+                         mat->get_block_size());
 
     const index_type *const colidxs = mat->get_col_idxs();
     const index_type *const rowptrs = mat->get_row_ptrs();
@@ -112,12 +106,11 @@ FbcsrSample<ValueType, IndexType>::generate_fbcsr() const
         Fbcsr::create(exec,
                       gko::dim<2>{static_cast<size_type>(nrows),
                                   static_cast<size_type>(ncols)},
-                      nnz, bs, std::make_shared<matstr::classical<Fbcsr>>());
+                      nnz, bs);
 
     value_type *const v = mtx->get_values();
     index_type *const c = mtx->get_col_idxs();
     index_type *const r = mtx->get_row_ptrs();
-    index_type *const s = mtx->get_srow();
     r[0] = 0;
     r[1] = 2;
     r[2] = 4;
@@ -154,8 +147,6 @@ FbcsrSample<ValueType, IndexType>::generate_fbcsr() const
     v[34] += FBCSR_TEST_IMAGINARY;
     v[35] += FBCSR_TEST_IMAGINARY;
 
-    for (index_type is = 0; is < mtx->get_num_srow_elements(); is++) s[is] = 0;
-
     return mtx;
 }
 
@@ -520,12 +511,11 @@ FbcsrSample2<ValueType, IndexType>::generate_fbcsr() const
         Fbcsr::create(exec,
                       gko::dim<2>{static_cast<size_type>(nrows),
                                   static_cast<size_type>(ncols)},
-                      nnz, bs, std::make_shared<matstr::classical<Fbcsr>>());
+                      nnz, bs);
 
     value_type *const v = mtx->get_values();
     index_type *const c = mtx->get_col_idxs();
     index_type *const r = mtx->get_row_ptrs();
-    index_type *const s = mtx->get_srow();
     r[0] = 0;
     r[1] = 1;
     r[2] = 3;
@@ -548,8 +538,6 @@ FbcsrSample2<ValueType, IndexType>::generate_fbcsr() const
     v[14] = -2;
     v[15] = -11;
 
-    for (index_type is = 0; is < mtx->get_num_srow_elements(); is++) s[is] = 0;
-
     return mtx;
 }
 
@@ -614,12 +602,11 @@ FbcsrSample2<ValueType, IndexType>::generate_transpose_fbcsr() const
         Fbcsr::create(exec,
                       gko::dim<2>{static_cast<size_type>(ncols),
                                   static_cast<size_type>(nrows)},
-                      nnz, bs, std::make_shared<matstr::classical<Fbcsr>>());
+                      nnz, bs);
 
     value_type *const v = mtx->get_values();
     index_type *const c = mtx->get_col_idxs();
     index_type *const r = mtx->get_row_ptrs();
-    index_type *const s = mtx->get_srow();
     r[0] = 0;
     r[1] = 2;
     r[2] = 2;
@@ -643,8 +630,6 @@ FbcsrSample2<ValueType, IndexType>::generate_transpose_fbcsr() const
     v[13] = 0;
     v[15] = 0;
 
-    for (index_type is = 0; is < mtx->get_num_srow_elements(); is++) s[is] = 0;
-
     return mtx;
 }
 
@@ -724,12 +709,11 @@ FbcsrSampleSquare<ValueType, IndexType>::generate_fbcsr() const
         Fbcsr::create(exec,
                       gko::dim<2>{static_cast<size_type>(nrows),
                                   static_cast<size_type>(ncols)},
-                      nnz, bs, std::make_shared<matstr::classical<Fbcsr>>());
+                      nnz, bs);
 
     value_type *const v = mtx->get_values();
     index_type *const c = mtx->get_col_idxs();
     index_type *const r = mtx->get_row_ptrs();
-    index_type *const s = mtx->get_srow();
     r[0] = 0;
     r[1] = 1;
     r[2] = 2;
@@ -749,12 +733,11 @@ FbcsrSampleSquare<ValueType, IndexType>::generate_transpose_fbcsr() const
         Fbcsr::create(exec,
                       gko::dim<2>{static_cast<size_type>(nrows),
                                   static_cast<size_type>(ncols)},
-                      nnz, bs, std::make_shared<matstr::classical<Fbcsr>>());
+                      nnz, bs);
 
     value_type *const v = mtx->get_values();
     index_type *const c = mtx->get_col_idxs();
     index_type *const r = mtx->get_row_ptrs();
-    index_type *const s = mtx->get_srow();
     r[0] = 0;
     r[1] = 0;
     r[2] = 2;
@@ -802,12 +785,11 @@ FbcsrSampleComplex<ValueType, IndexType>::generate_fbcsr() const
         Fbcsr::create(exec,
                       gko::dim<2>{static_cast<size_type>(nrows),
                                   static_cast<size_type>(ncols)},
-                      nnz, bs, std::make_shared<matstr::classical<Fbcsr>>());
+                      nnz, bs);
 
     value_type *const v = mtx->get_values();
     index_type *const c = mtx->get_col_idxs();
     index_type *const r = mtx->get_row_ptrs();
-    index_type *const s = mtx->get_srow();
     r[0] = 0;
     r[1] = 1;
     r[2] = 3;
@@ -831,8 +813,6 @@ FbcsrSampleComplex<ValueType, IndexType>::generate_fbcsr() const
     v[14] = -2.0 - 2.15i;
     v[15] = -11.0 - 11.15i;
 
-    for (index_type is = 0; is < mtx->get_num_srow_elements(); is++) s[is] = 0;
-
     return mtx;
 }
 
@@ -844,12 +824,11 @@ FbcsrSampleComplex<ValueType, IndexType>::generate_conjtranspose_fbcsr() const
         Fbcsr::create(exec,
                       gko::dim<2>{static_cast<size_type>(ncols),
                                   static_cast<size_type>(nrows)},
-                      nnz, bs, std::make_shared<matstr::classical<Fbcsr>>());
+                      nnz, bs);
 
     value_type *const v = mtx->get_values();
     index_type *const c = mtx->get_col_idxs();
     index_type *const r = mtx->get_row_ptrs();
-    index_type *const s = mtx->get_srow();
     r[0] = 0;
     r[1] = 2;
     r[2] = 2;
@@ -874,8 +853,6 @@ FbcsrSampleComplex<ValueType, IndexType>::generate_conjtranspose_fbcsr() const
     v[13] = 0;
     v[15] = 0;
 
-    for (index_type is = 0; is < mtx->get_num_srow_elements(); is++) s[is] = 0;
-
     return mtx;
 }
 
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index dcb3e0fcb6a..745bd45dbfe 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -38,7 +38,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/blockutils.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/matrix/matrix_strategies.hpp>
 
 
 namespace gko {
@@ -137,9 +136,6 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     using mat_data = matrix_data<ValueType, IndexType>;
     using absolute_type = remove_complex<Fbcsr>;
 
-    using strategy_type =
-        matrix_strategy::strategy_type<Fbcsr<value_type, index_type>>;
-
 
     void convert_to(Fbcsr<ValueType, IndexType> *result) const override;
 
@@ -276,39 +272,6 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
         return row_ptrs_.get_const_data();
     }
 
-    /**
-     * @return The starting row for each 'team' of threads
-     */
-    index_type *get_srow() noexcept { return startrow_.get_data(); }
-
-    /**
-     * @see get_const_srow
-     */
-    const index_type *get_srow() const noexcept
-    {
-        return startrow_.get_const_data();
-    }
-
-    /**
-     * @copydoc Fbcsr::get_srow()
-     *
-     * @note This is the constant version of the function, which can be
-     *       significantly more memory efficient than the non-constant version,
-     *       so always prefer this version.
-     */
-    const index_type *get_const_srow() const noexcept
-    {
-        return startrow_.get_const_data();
-    }
-
-    /**
-     * @return The number of the srow stored elements (involved warps)
-     */
-    size_type get_num_srow_elements() const noexcept
-    {
-        return startrow_.get_num_elems();
-    }
-
     /**
      * Returns the number of elements explicitly stored in the matrix.
      *
@@ -319,25 +282,6 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
         return values_.get_num_elems();
     }
 
-    /**
-     * @return The strategy
-     */
-    std::shared_ptr<strategy_type> get_strategy() const noexcept
-    {
-        return strategy_;
-    }
-
-    /**
-     * Set the strategy
-     *
-     * @param strategy the fbcsr strategy
-     */
-    void set_strategy(std::shared_ptr<strategy_type> strategy)
-    {
-        strategy_ = std::move(strategy->copy());
-        this->make_srow();
-    }
-
     /**
      * @return The fixed block size for this matrix
      */
@@ -364,17 +308,13 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     index_type get_num_block_cols() const { return nbcols_; }
 
 protected:
-    using classical = matrix_strategy::classical<Fbcsr<value_type, index_type>>;
-
     /**
      * Creates an uninitialized FBCSR matrix with a block size of 1.
      *
      * @param exec  Executor associated to the matrix
-     * @param strategy  the strategy of FBCSR
      */
-    Fbcsr(std::shared_ptr<const Executor> exec,
-          std::shared_ptr<strategy_type> strategy)
-        : Fbcsr(std::move(exec), dim<2>{}, {}, 1, std::move(strategy))
+    Fbcsr(std::shared_ptr<const Executor> exec)
+        : Fbcsr(std::move(exec), dim<2>{}, {}, 1)
     {}
 
     /**
@@ -384,12 +324,12 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
      * @param size  size of the matrix
      * @param num_nonzeros  number of nonzeros
      * @param block_size Size of the small dense square blocks
-     * @param strategy  the strategy of FBCSR
      */
-    Fbcsr(std::shared_ptr<const Executor> exec, const dim<2> &size = dim<2>{},
-          size_type num_nonzeros = {}, int block_size = 1,
-          std::shared_ptr<strategy_type> strategy =
-              std::make_shared<classical>());
+    // Fbcsr(std::shared_ptr<const Executor> exec, const dim<2> &size =
+    // dim<2>{},
+    //       size_type num_nonzeros = {}, int block_size = 1);
+    Fbcsr(std::shared_ptr<const Executor> exec, const dim<2> &size,
+          size_type num_nonzeros, int block_size);
 
     /**
      * Creates a FBCSR matrix from already allocated (and initialized) row
@@ -414,24 +354,19 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
      */
     template <typename ValuesArray, typename ColIdxsArray,
               typename RowPtrsArray>
-    Fbcsr(
-        std::shared_ptr<const Executor> exec, const dim<2> &size,
-        int block_size, ValuesArray &&values, ColIdxsArray &&col_idxs,
-        RowPtrsArray &&row_ptrs,
-        std::shared_ptr<strategy_type> strategy = std::make_shared<classical>())
+    Fbcsr(std::shared_ptr<const Executor> exec, const dim<2> &size,
+          int block_size, ValuesArray &&values, ColIdxsArray &&col_idxs,
+          RowPtrsArray &&row_ptrs)
         : EnableLinOp<Fbcsr>(exec, size),
           bs_{block_size},
           nbcols_{blockutils::getNumBlocks(block_size, size[1])},
           values_{exec, std::forward<ValuesArray>(values)},
           col_idxs_{exec, std::forward<ColIdxsArray>(col_idxs)},
-          row_ptrs_{exec, std::forward<RowPtrsArray>(row_ptrs)},
-          startrow_(exec),
-          strategy_(strategy->copy())
+          row_ptrs_{exec, std::forward<RowPtrsArray>(row_ptrs)}
     {
         GKO_ASSERT_EQ(values_.get_num_elems(),
                       col_idxs_.get_num_elems() * bs_ * bs_);
         GKO_ASSERT_EQ(this->get_size()[0] / bs_ + 1, row_ptrs_.get_num_elems());
-        this->make_srow();
     }
 
     void apply_impl(const LinOp *b, LinOp *x) const override;
@@ -439,28 +374,12 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override;
 
-    // TODO clean this up as soon as we improve strategy_type
-    template <typename FbcsrType>
-    void convert_strategy_helper(FbcsrType *result) const;
-
-    /**
-     * Computes srow. It should be run after changing any row pointer.
-     */
-    void make_srow()
-    {
-        startrow_.resize_and_reset(
-            strategy_->calc_size(col_idxs_.get_num_elems()));
-        strategy_->process(row_ptrs_, &startrow_);
-    }
-
 private:
-    int bs_;            ///< Block size
-    size_type nbcols_;  ///< Number of block-columns
-    Array<value_type> values_;
-    Array<index_type> col_idxs_;
-    Array<index_type> row_ptrs_;
-    Array<index_type> startrow_;
-    std::shared_ptr<strategy_type> strategy_;
+    int bs_;                      ///< Block size
+    size_type nbcols_;            ///< Number of block-columns
+    Array<value_type> values_;    ///< Non-zero values of all blocks
+    Array<index_type> col_idxs_;  ///< Block-column indices of all blocks
+    Array<index_type> row_ptrs_;  ///< Block-row pointers into @ref col_idxs_
 };
 
 
diff --git a/include/ginkgo/core/matrix/matrix_strategies.hpp b/include/ginkgo/core/matrix/matrix_strategies.hpp
deleted file mode 100644
index 30dd715cf55..00000000000
--- a/include/ginkgo/core/matrix/matrix_strategies.hpp
+++ /dev/null
@@ -1,503 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GKO_PUBLIC_CORE_MATRIX_MATRIX_STRATEGIES_HPP_
-#define GKO_PUBLIC_CORE_MATRIX_MATRIX_STRATEGIES_HPP_
-
-
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/lin_op.hpp>
-
-
-namespace gko {
-namespace matrix {
-
-
-namespace matrix_strategy {
-
-
-template <typename MtxType>
-class automatic;
-
-/**
- * strategy_type is to decide how map the work-items to execution units
- *
- * The practical strategy method should inherit strategy_type and implement
- * its `process`, `calc_size` function and the corresponding device kernel.
- */
-template <typename MtxType>
-class strategy_type {
-    friend class automatic<MtxType>;
-
-public:
-    using index_type = typename MtxType::index_type;
-
-    /**
-     * Creates a strategy_type.
-     *
-     * @param name  the name of strategy
-     */
-    strategy_type(std::string name) : name_(name) {}
-
-    /**
-     * Returns the name of strategy
-     *
-     * @return the name of strategy
-     */
-    std::string get_name() { return name_; }
-
-    /**
-     * Computes srow according to row pointers.
-     *
-     * @param mtx_row_ptrs  the row pointers of the matrix
-     * @param mtx_srow  the srow of the matrix
-     */
-    virtual void process(const Array<index_type> &mtx_row_ptrs,
-                         Array<index_type> *mtx_srow) = 0;
-
-    /**
-     * Computes the srow size according to the number of nonzeros.
-     *
-     * @param nnz  the number of nonzeros
-     *
-     * @return the size of srow
-     */
-    virtual int64_t calc_size(const int64_t nnz) = 0;
-
-    /**
-     * Copy a strategy. This is a workaround until strategies are revamped,
-     * since strategies like `automatic` do not work when actually shared.
-     */
-    virtual std::shared_ptr<strategy_type> copy() = 0;
-
-protected:
-    void set_name(std::string name) { name_ = name; }
-
-private:
-    std::string name_;
-};
-
-/**
- * classical is a strategy_type which uses the same number of threads on
- * each block-row. Classical strategy uses multithreads to calculate on parts of
- * rows and then do a reduction of these threads results. The number of
- * threads per row depends on the max number of stored elements per row.
- */
-template <typename MtxType>
-class classical : public strategy_type<MtxType> {
-public:
-    using index_type = typename strategy_type<MtxType>::index_type;
-
-    /**
-     * Creates a classical strategy.
-     */
-    classical() : strategy_type<MtxType>("classical"), max_length_per_row_(0) {}
-
-    void process(const Array<index_type> &mtx_row_ptrs,
-                 Array<index_type> *mtx_srow) override
-    {
-        auto host_mtx_exec = mtx_row_ptrs.get_executor()->get_master();
-        Array<index_type> row_ptrs_host(host_mtx_exec);
-        const bool is_mtx_on_host{host_mtx_exec == mtx_row_ptrs.get_executor()};
-        const index_type *row_ptrs{};
-        if (is_mtx_on_host) {
-            row_ptrs = mtx_row_ptrs.get_const_data();
-        } else {
-            row_ptrs_host = mtx_row_ptrs;
-            row_ptrs = row_ptrs_host.get_const_data();
-        }
-        auto num_rows = mtx_row_ptrs.get_num_elems() - 1;
-        max_length_per_row_ = 0;
-        for (index_type i = 1; i < num_rows + 1; i++) {
-            max_length_per_row_ =
-                std::max(max_length_per_row_, row_ptrs[i] - row_ptrs[i - 1]);
-        }
-    }
-
-    int64_t calc_size(const int64_t nnz) override { return 0; }
-
-    index_type get_max_length_per_row() const noexcept
-    {
-        return max_length_per_row_;
-    }
-
-    std::shared_ptr<strategy_type<MtxType>> copy() override
-    {
-        return std::make_shared<classical<MtxType>>();
-    }
-
-private:
-    index_type max_length_per_row_;
-};
-
-/**
- * load_balance is a strategy_type which uses the load balance algorithm.
- */
-template <typename MtxType>
-class load_balance : public strategy_type<MtxType> {
-public:
-    using index_type = typename strategy_type<MtxType>::index_type;
-
-    /**
-     * Creates a load_balance strategy.
-     */
-    load_balance()
-        : load_balance(std::move(
-              gko::CudaExecutor::create(0, gko::OmpExecutor::create())))
-    {}
-
-    /**
-     * Creates a load_balance strategy with CUDA executor.
-     *
-     * @param exec the CUDA executor
-     */
-    load_balance(std::shared_ptr<const CudaExecutor> exec)
-        : load_balance(exec->get_num_warps(), exec->get_warp_size())
-    {}
-
-    /**
-     * Creates a load_balance strategy with HIP executor.
-     *
-     * @param exec the HIP executor
-     */
-    load_balance(std::shared_ptr<const HipExecutor> exec)
-        : load_balance(exec->get_num_warps(), exec->get_warp_size(), false)
-    {}
-
-    /**
-     * Creates a load_balance strategy with specified parameters
-     *
-     * @param nwarps The number of warps in the executor
-     * @param warp_size The warp size of the executor
-     * @param cuda_params Whether Nvidia-based warp parameters should be used.
-     *
-     * @note The warp_size must be the size of full warp. When using this
-     *       constructor, set_strategy needs to be called with correct
-     *       parameters which is replaced during the conversion.
-     */
-    load_balance(int64_t nwarps, int warp_size = 32, bool cuda_params = true)
-        : strategy_type<MtxType>("load_balance"),
-          nwarps_(nwarps),
-          warp_size_(warp_size),
-          cuda_params_(cuda_params)
-    {}
-
-    void process(const Array<index_type> &mtx_row_ptrs,
-                 Array<index_type> *mtx_srow) override
-    {
-        auto nwarps = mtx_srow->get_num_elems();
-
-        if (nwarps > 0) {
-            auto host_srow_exec = mtx_srow->get_executor()->get_master();
-            auto host_mtx_exec = mtx_row_ptrs.get_executor()->get_master();
-            const bool is_srow_on_host{host_srow_exec ==
-                                       mtx_srow->get_executor()};
-            const bool is_mtx_on_host{host_mtx_exec ==
-                                      mtx_row_ptrs.get_executor()};
-            Array<index_type> row_ptrs_host(host_mtx_exec);
-            Array<index_type> srow_host(host_srow_exec);
-            const index_type *row_ptrs{};
-            index_type *srow{};
-            if (is_srow_on_host) {
-                srow = mtx_srow->get_data();
-            } else {
-                srow_host = *mtx_srow;
-                srow = srow_host.get_data();
-            }
-            if (is_mtx_on_host) {
-                row_ptrs = mtx_row_ptrs.get_const_data();
-            } else {
-                row_ptrs_host = mtx_row_ptrs;
-                row_ptrs = row_ptrs_host.get_const_data();
-            }
-            for (size_type i = 0; i < nwarps; i++) {
-                srow[i] = 0;
-            }
-            const size_type num_rows = mtx_row_ptrs.get_num_elems() - 1;
-            const index_type num_elems = row_ptrs[num_rows];
-            for (size_type i = 0; i < num_rows; i++) {
-                const auto num =
-                    (ceildiv(row_ptrs[i + 1], warp_size_) * nwarps);
-                const auto den = ceildiv(num_elems, warp_size_);
-                auto bucket = ceildiv(num, den);
-                if (bucket < nwarps) {
-                    srow[bucket]++;
-                }
-            }
-            // find starting row for thread i
-            for (size_type i = 1; i < nwarps; i++) {
-                srow[i] += srow[i - 1];
-            }
-            if (!is_srow_on_host) {
-                *mtx_srow = srow_host;
-            }
-        }
-    }
-
-    int64_t calc_size(const int64_t nnz) override
-    {
-        if (warp_size_ > 0) {
-            int multiple = 8;
-            if (nnz >= 2e8) {
-                multiple = 2048;
-            } else if (nnz >= 2e7) {
-                multiple = 512;
-            } else if (nnz >= 2e6) {
-                multiple = 128;
-            } else if (nnz >= 2e5) {
-                multiple = 32;
-            }
-
-#if GINKGO_HIP_PLATFORM_HCC
-            if (!cuda_params_) {
-                multiple = 8;
-                if (nnz >= 1e7) {
-                    multiple = 64;
-                } else if (nnz >= 1e6) {
-                    multiple = 16;
-                }
-            }
-#endif  // GINKGO_HIP_PLATFORM_HCC
-
-            auto nwarps = nwarps_ * multiple;
-            return min(ceildiv(nnz, warp_size_), int64_t(nwarps));
-        } else {
-            return 0;
-        }
-    }
-
-    std::shared_ptr<strategy_type<MtxType>> copy() override
-    {
-        return std::make_shared<load_balance<MtxType>>(nwarps_, warp_size_,
-                                                       cuda_params_);
-    }
-
-private:
-    int64_t nwarps_;
-    int warp_size_;
-    bool cuda_params_;
-};
-
-template <typename MtxType>
-class automatic : public strategy_type<MtxType> {
-public:
-    using index_type = typename strategy_type<MtxType>::index_type;
-
-    /* Use imbalance strategy when the maximum number of nonzero per row is
-     * more than 1024 on NVIDIA hardware */
-    const index_type nvidia_row_len_limit = 1024;
-    /* Use imbalance strategy when the matrix has more more than 1e6 on
-     * NVIDIA hardware */
-    const index_type nvidia_nnz_limit = 1e6;
-    /* Use imbalance strategy when the maximum number of nonzero per row is
-     * more than 768 on AMD hardware */
-    const index_type amd_row_len_limit = 768;
-    /* Use imbalance strategy when the matrix has more more than 1e8 on AMD
-     * hardware */
-    const index_type amd_nnz_limit = 1e8;
-
-    /**
-     * Creates an automatic strategy.
-     */
-    automatic()
-        : automatic(std::move(
-              gko::CudaExecutor::create(0, gko::OmpExecutor::create())))
-    {}
-
-    /**
-     * Creates an automatic strategy with CUDA executor.
-     *
-     * @param exec the CUDA executor
-     */
-    automatic(std::shared_ptr<const CudaExecutor> exec)
-        : automatic(exec->get_num_warps(), exec->get_warp_size())
-    {}
-
-    /**
-     * Creates an automatic strategy with HIP executor.
-     *
-     * @param exec the HIP executor
-     */
-    automatic(std::shared_ptr<const HipExecutor> exec)
-        : automatic(exec->get_num_warps(), exec->get_warp_size(), false)
-    {}
-
-    /**
-     * Creates an automatic strategy with specified parameters
-     *
-     * @param nwarps the number of warps in the executor
-     * @param warp_size the warp size of the executor
-     * @param cuda_strategy  whether the `cuda_strategy` needs to be used.
-     *
-     * @note The warp_size must be the size of full warp. When using this
-     *       constructor, set_strategy needs to be called with correct
-     *       parameters which is replaced during the conversion.
-     */
-    automatic(int64_t nwarps, int warp_size = 32, bool cuda_strategy = true)
-        : strategy_type<MtxType>("automatic"),
-          nwarps_(nwarps),
-          warp_size_(warp_size),
-          cuda_strategy_(cuda_strategy),
-          max_length_per_row_(0)
-    {}
-
-    void process(const Array<index_type> &mtx_row_ptrs,
-                 Array<index_type> *mtx_srow) override
-    {
-        // if the number of stored elements is larger than <nnz_limit> or
-        // the maximum number of stored elements per row is larger than
-        // <row_len_limit>, use load_balance otherwise use classical
-        index_type nnz_limit = nvidia_nnz_limit;
-        index_type row_len_limit = nvidia_row_len_limit;
-#if GINKGO_HIP_PLATFORM_HCC
-        if (!cuda_strategy_) {
-            nnz_limit = amd_nnz_limit;
-            row_len_limit = amd_row_len_limit;
-        }
-#endif  // GINKGO_HIP_PLATFORM_HCC
-        auto host_mtx_exec = mtx_row_ptrs.get_executor()->get_master();
-        const bool is_mtx_on_host{host_mtx_exec == mtx_row_ptrs.get_executor()};
-        Array<index_type> row_ptrs_host(host_mtx_exec);
-        const index_type *row_ptrs{};
-        if (is_mtx_on_host) {
-            row_ptrs = mtx_row_ptrs.get_const_data();
-        } else {
-            row_ptrs_host = mtx_row_ptrs;
-            row_ptrs = row_ptrs_host.get_const_data();
-        }
-        const auto num_rows = mtx_row_ptrs.get_num_elems() - 1;
-        if (row_ptrs[num_rows] > nnz_limit) {
-            load_balance<MtxType> actual_strategy(nwarps_, warp_size_,
-                                                  cuda_strategy_);
-            if (is_mtx_on_host) {
-                actual_strategy.process(mtx_row_ptrs, mtx_srow);
-            } else {
-                actual_strategy.process(row_ptrs_host, mtx_srow);
-            }
-            this->set_name(actual_strategy.get_name());
-        } else {
-            index_type maxnum = 0;
-            for (index_type i = 1; i < num_rows + 1; i++) {
-                maxnum = std::max(maxnum, row_ptrs[i] - row_ptrs[i - 1]);
-            }
-            if (maxnum > row_len_limit) {
-                load_balance<MtxType> actual_strategy(nwarps_, warp_size_,
-                                                      cuda_strategy_);
-                if (is_mtx_on_host) {
-                    actual_strategy.process(mtx_row_ptrs, mtx_srow);
-                } else {
-                    actual_strategy.process(row_ptrs_host, mtx_srow);
-                }
-                this->set_name(actual_strategy.get_name());
-            } else {
-                classical<MtxType> actual_strategy;
-                if (is_mtx_on_host) {
-                    actual_strategy.process(mtx_row_ptrs, mtx_srow);
-                    max_length_per_row_ =
-                        actual_strategy.get_max_length_per_row();
-                } else {
-                    actual_strategy.process(row_ptrs_host, mtx_srow);
-                    max_length_per_row_ =
-                        actual_strategy.get_max_length_per_row();
-                }
-                this->set_name(actual_strategy.get_name());
-            }
-        }
-    }
-
-    int64_t calc_size(const int64_t nnz) override
-    {
-        return std::make_shared<load_balance<MtxType>>(nwarps_, warp_size_,
-                                                       cuda_strategy_)
-            ->calc_size(nnz);
-    }
-
-    index_type get_max_length_per_row() const noexcept
-    {
-        return max_length_per_row_;
-    }
-
-    std::shared_ptr<strategy_type<MtxType>> copy() override
-    {
-        return std::make_shared<automatic<MtxType>>(nwarps_, warp_size_,
-                                                    cuda_strategy_);
-    }
-
-private:
-    int64_t nwarps_;
-    int warp_size_;
-    bool cuda_strategy_;
-    index_type max_length_per_row_;
-};
-
-
-/**
- * When strategy is load_balance or automatic, rebuild the strategy
- * according to executor's property.
- *
- * @param result  the matrix.
- */
-template <typename MtxType>
-void strategy_rebuild_helper(MtxType *const result)
-{
-    // TODO (script:fbcsr): change the code imported from matrix/csr if needed
-    // using load_balance = typename Fbcsr<ValueType, IndexType>::load_balance;
-    // using automatic = typename Fbcsr<ValueType, IndexType>::automatic;
-    auto strategy = result->get_strategy();
-    auto executor = result->get_executor();
-    if (std::dynamic_pointer_cast<load_balance<MtxType>>(strategy)) {
-        if (auto exec =
-                std::dynamic_pointer_cast<const HipExecutor>(executor)) {
-            result->set_strategy(std::make_shared<load_balance<MtxType>>(exec));
-        } else if (auto exec = std::dynamic_pointer_cast<const CudaExecutor>(
-                       executor)) {
-            result->set_strategy(std::make_shared<load_balance<MtxType>>(exec));
-        }
-    } else if (std::dynamic_pointer_cast<automatic<MtxType>>(strategy)) {
-        if (auto exec =
-                std::dynamic_pointer_cast<const HipExecutor>(executor)) {
-            result->set_strategy(std::make_shared<automatic<MtxType>>(exec));
-        } else if (auto exec = std::dynamic_pointer_cast<const CudaExecutor>(
-                       executor)) {
-            result->set_strategy(std::make_shared<automatic<MtxType>>(exec));
-        }
-    }
-}
-
-
-}  // namespace matrix_strategy
-
-
-}  // namespace matrix
-}  // namespace gko
-
-#endif  // GKO_PUBLIC_CORE_MATRIX_MATRIX_STRATEGIES_HPP_
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 541980beaa3..38ac3fd861c 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -85,7 +85,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/fbcsr.hpp>
 #include <ginkgo/core/matrix/hybrid.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
-#include <ginkgo/core/matrix/matrix_strategies.hpp>
 #include <ginkgo/core/matrix/permutation.hpp>
 #include <ginkgo/core/matrix/sellp.hpp>
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index cafce276ae1..368ada19b45 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -47,7 +47,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/diagonal.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
-#include <ginkgo/core/matrix/matrix_strategies.hpp>
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
@@ -59,9 +58,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
-namespace matstr = gko::matrix::matrix_strategy;
-
-
 template <typename ValueIndexType>
 class Fbcsr : public ::testing::Test {
 protected:
@@ -344,8 +340,6 @@ TYPED_TEST(Fbcsr, ConvertsToPrecision)
     tmp->convert_to(res.get());
 
     GKO_ASSERT_MTX_NEAR(this->mtx, res, residual);
-    // ASSERT_EQ(typeid(*this->mtx->get_strategy()),
-    //           typeid(*res->get_strategy()));
 }
 
 
@@ -367,8 +361,6 @@ TYPED_TEST(Fbcsr, MovesToPrecision)
     tmp->move_to(res.get());
 
     GKO_ASSERT_MTX_NEAR(this->mtx, res, residual);
-    // ASSERT_EQ(typeid(*this->mtx->get_strategy()),
-    //           typeid(*res->get_strategy()));
 }
 
 
@@ -595,7 +587,6 @@ TYPED_TEST(Fbcsr, RecognizeUnsortedMatrix)
     using Fbcsr = typename TestFixture::Mtx;
     using index_type = typename TestFixture::index_type;
 
-    // auto cpmat = Fbcsr::create(this->exec, this->mtx->get_strategy());
     auto cpmat = this->mtx->clone();
     index_type *const colinds = cpmat->get_col_idxs();
     std::swap(colinds[0], colinds[1]);

From 36853309ce6788bbc11fb58bc5ce55e3cfc50e8e Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 2 Dec 2020 16:26:50 +0100
Subject: [PATCH 0555/1008] tolerance for absolute tests is the respective
 epsilon

---
 reference/test/matrix/fbcsr_kernels.cpp | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 368ada19b45..4feaf5d1e5d 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -620,12 +620,15 @@ TYPED_TEST(Fbcsr, InplaceAbsolute)
 
     mtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx, refabs, 0.0);
+    const gko::remove_complex<value_type> tolerance =
+        std::numeric_limits<gko::remove_complex<value_type>>::epsilon();
+    GKO_ASSERT_MTX_NEAR(mtx, refabs, tolerance);
 }
 
 
 TYPED_TEST(Fbcsr, OutplaceAbsolute)
 {
+    using value_type = typename TestFixture::value_type;
     using Mtx = typename TestFixture::Mtx;
     using AbsMtx = typename gko::remove_complex<typename TestFixture::Mtx>;
 
@@ -635,7 +638,9 @@ TYPED_TEST(Fbcsr, OutplaceAbsolute)
 
     auto abs_mtx = mtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx, refabs, 0.0);
+    const gko::remove_complex<value_type> tolerance =
+        std::numeric_limits<gko::remove_complex<value_type>>::epsilon();
+    GKO_ASSERT_MTX_NEAR(abs_mtx, refabs, tolerance);
 }
 
 
@@ -691,7 +696,9 @@ TYPED_TEST(FbcsrComplex, InplaceAbsolute)
 
     mtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx, refabs, 0.0);
+    const gko::remove_complex<value_type> tolerance =
+        std::numeric_limits<gko::remove_complex<value_type>>::epsilon();
+    GKO_ASSERT_MTX_NEAR(mtx, refabs, tolerance);
 }
 
 
@@ -711,7 +718,9 @@ TYPED_TEST(FbcsrComplex, OutplaceAbsolute)
 
     auto abs_mtx = mtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx, refabs, 0.0);
+    const gko::remove_complex<value_type> tolerance =
+        std::numeric_limits<gko::remove_complex<value_type>>::epsilon();
+    GKO_ASSERT_MTX_NEAR(abs_mtx, refabs, tolerance);
 }
 
 

From 821e2d70ac230d364153dfe5c58c2b6577d6ffcc Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Fri, 4 Dec 2020 19:11:42 +0100
Subject: [PATCH 0556/1008] incorporated Terry's comments about allocation and
 docstrings

---
 core/components/fixed_block.hpp | 51 ++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 23 deletions(-)

diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
index 14c035fc965..c3d21b8a171 100644
--- a/core/components/fixed_block.hpp
+++ b/core/components/fixed_block.hpp
@@ -34,26 +34,31 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_COMPONENTS_FIXED_BLOCK_HPP_
 
 
+#include <algorithm>
 #include <type_traits>
+#include <vector>
 
 
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/math.hpp>
 
+
 namespace gko {
 namespace blockutils {
 
 
-/// A dense block of values with compile-time constant dimensions
-/** The blocks are stored row-major. However, in future,
+/**
+ * @brief A dense block of values with compile-time constant dimensions
+ *
+ * The blocks are interpreted as row-major. However, in the future,
  *  a layout template parameter can be added if needed.
  *
  * The primary use is to reinterpret subsets of entries in a big array as
  *  small dense blocks.
  *
- * @tparam ValueType The numeric type of entries of the block
- * @tparam nrows Number of rows
- * @tparam ncols Number of columns
+ * @tparam ValueType  The numeric type of entries of the block
+ * @tparam nrows  Number of rows
+ * @tparam ncols  Number of columns
  */
 template <typename ValueType, int nrows, int ncols>
 class FixedBlock final {
@@ -89,7 +94,7 @@ class FixedBlock final {
 
 
 /**
- * A lightweight dynamic block type for the host space
+ * A lightweight dynamic block type on the host
  *
  * @tparam ValueType The numeric type of entries of the block
  */
@@ -98,16 +103,16 @@ class DenseBlock final {
 public:
     using value_type = ValueType;
 
-    DenseBlock() : nrows_{0}, ncols_{0}, vals_{nullptr} {}
+    /**
+     * If this default construtor is used, @ref set_executor must be called
+     *  prior to any other member function.
+     */
+    DenseBlock() : nrows_{0}, ncols_{0} {}
 
     DenseBlock(const int num_rows, const int num_cols)
-        : nrows_{num_rows},
-          ncols_{num_cols},
-          vals_{new value_type[num_rows * num_cols]}
+        : nrows_{num_rows}, ncols_{num_cols}, vals_(num_rows * num_cols)
     {}
 
-    ~DenseBlock() { delete[] vals_; }
-
     value_type &at(const int row, const int col)
     {
         return vals_[row * ncols_ + col];
@@ -132,33 +137,32 @@ class DenseBlock final {
 
     void resize(const int nrows, const int ncols)
     {
-        if (nrows * ncols != nrows_ * ncols_) {
-            delete[] vals_;
-            vals_ = new value_type[nrows * ncols];
-        }
+        vals_.resize(nrows * ncols);
         nrows_ = nrows;
         ncols_ = ncols;
     }
 
     void zero()
     {
-        for (int i = 0; i < nrows_ * ncols_; i++)
-            vals_[i] = static_cast<value_type>(0);
+        std::fill(vals_.begin(), vals_.end(), static_cast<value_type>(0));
     }
 
 private:
     int nrows_;
     int ncols_;
-    value_type *vals_;
+    std::vector<value_type> vals_;
 };
 
-/// A view into a an array of dense blocks of some runtime-defined size
-/** Note that accessing BSR values using this type of view abstracts away the
+
+/**
+ * @brief A view into a an array of dense blocks of some runtime-defined size
+ *
+ * Accessing BSR values using this type of view abstracts away the
  * storage layout within the individual blocks, as long as all blocks use the
  * same layout. For now, row-major blocks are assumed.
  *
- * @tparam ValueType The numeric type of entries of the block
- * @tparam IndexType The type of integer used to identify the different blocks
+ * @tparam ValueType  The numeric type of entries of the block
+ * @tparam IndexType  The type of integer used to identify the different blocks
  */
 template <typename ValueType, typename IndexType = int32>
 class DenseBlocksView final {
@@ -209,4 +213,5 @@ class DenseBlocksView final {
 }  // namespace blockutils
 }  // namespace gko
 
+
 #endif  // GKO_CORE_COMPONENTS_FIXED_BLOCK_HPP_

From 2e64b9c4830099e1342061c6aca2284b1165e59c Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Mon, 7 Dec 2020 17:06:28 +0100
Subject: [PATCH 0557/1008] addressed Pratik's comments and some SonarCloud
 suggestions

---
 core/components/fixed_block.hpp         | 10 +---
 core/matrix/fbcsr.cpp                   | 21 +++----
 core/matrix/fbcsr_builder.hpp           |  2 +-
 include/ginkgo/core/matrix/fbcsr.hpp    |  3 -
 reference/matrix/fbcsr_kernels.cpp      | 80 ++++++++++++-------------
 reference/test/matrix/fbcsr_kernels.cpp |  2 -
 6 files changed, 53 insertions(+), 65 deletions(-)

diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
index c3d21b8a171..c41386b9964 100644
--- a/core/components/fixed_block.hpp
+++ b/core/components/fixed_block.hpp
@@ -103,11 +103,7 @@ class DenseBlock final {
 public:
     using value_type = ValueType;
 
-    /**
-     * If this default construtor is used, @ref set_executor must be called
-     *  prior to any other member function.
-     */
-    DenseBlock() : nrows_{0}, ncols_{0} {}
+    DenseBlock() {}
 
     DenseBlock(const int num_rows, const int num_cols)
         : nrows_{num_rows}, ncols_{num_cols}, vals_(num_rows * num_cols)
@@ -148,8 +144,8 @@ class DenseBlock final {
     }
 
 private:
-    int nrows_;
-    int ncols_;
+    int nrows_ = 0;
+    int ncols_ = 0;
     std::vector<value_type> vals_;
 };
 
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index cb7b40702cb..51950dc96fe 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -254,7 +254,7 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
                                std::to_string(__LINE__) +
                                ": List of nonzeros is too big!");
 
-    const index_type nnz = static_cast<index_type>(data.nonzeros.size());
+    const auto nnz = static_cast<index_type>(data.nonzeros.size());
 
     const int bs = this->bs_;
 
@@ -275,15 +275,15 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
         }
     };
 
-    auto create_block_map = [nnz, bs](const mat_data &data) {
+    auto create_block_map = [nnz, bs](const mat_data &mdata) {
         std::map<FbEntry, Blk_t, FbLess> blocks;
         for (index_type inz = 0; inz < nnz; inz++) {
-            const index_type row = data.nonzeros[inz].row;
-            const index_type col = data.nonzeros[inz].column;
-            const value_type val = data.nonzeros[inz].value;
+            const index_type row = mdata.nonzeros[inz].row;
+            const index_type col = mdata.nonzeros[inz].column;
+            const value_type val = mdata.nonzeros[inz].value;
 
-            const int localrow = static_cast<int>(row % bs);
-            const int localcol = static_cast<int>(col % bs);
+            const auto localrow = static_cast<int>(row % bs);
+            const auto localcol = static_cast<int>(col % bs);
             const index_type blockrow = row / bs;
             const index_type blockcol = col / bs;
 
@@ -308,8 +308,9 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
                              blocks.size() * bs * bs, bs);
 
     tmp->row_ptrs_.get_data()[0] = 0;
-    index_type cur_brow = 0, cur_bnz = 0,
-               cur_bcol = blocks.begin()->first.block_column;
+    index_type cur_brow = 0;
+    index_type cur_bnz = 0;
+    index_type cur_bcol = blocks.begin()->first.block_column;
     const index_type num_brows = data.size[0] / bs;
 
     blockutils::DenseBlocksView<value_type, index_type> values(
@@ -361,7 +362,7 @@ void Fbcsr<ValueType, IndexType>::write(mat_data &data) const
     const blockutils::DenseBlocksView<const value_type, index_type> vblocks(
         tmp->values_.get_const_data(), bs_, bs_);
 
-    for (size_type brow = 0; brow < tmp->get_size()[0] / bs_; ++brow) {
+    for (size_type brow = 0; brow < tmp->get_num_block_rows(); ++brow) {
         const auto start = tmp->row_ptrs_.get_const_data()[brow];
         const auto end = tmp->row_ptrs_.get_const_data()[brow + 1];
 
diff --git a/core/matrix/fbcsr_builder.hpp b/core/matrix/fbcsr_builder.hpp
index b99aebe824b..e80d76b3eeb 100644
--- a/core/matrix/fbcsr_builder.hpp
+++ b/core/matrix/fbcsr_builder.hpp
@@ -75,7 +75,7 @@ class FbcsrBuilder {
         : matrix_{matrix}
     {}
 
-    ~FbcsrBuilder() {}
+    ~FbcsrBuilder() = default;
 
     // make this type non-movable
     FbcsrBuilder(const FbcsrBuilder &) = delete;
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index 745bd45dbfe..409a6fc5130 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -325,9 +325,6 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
      * @param num_nonzeros  number of nonzeros
      * @param block_size Size of the small dense square blocks
      */
-    // Fbcsr(std::shared_ptr<const Executor> exec, const dim<2> &size =
-    // dim<2>{},
-    //       size_type num_nonzeros = {}, int block_size = 1);
     Fbcsr(std::shared_ptr<const Executor> exec, const dim<2> &size,
           size_type num_nonzeros, int block_size);
 
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index c59f8a17e52..c3532238db6 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -66,13 +66,13 @@ namespace fbcsr {
 
 
 template <typename ValueType, typename IndexType>
-void spmv(const std::shared_ptr<const ReferenceExecutor> exec,
+void spmv(const std::shared_ptr<const ReferenceExecutor>,
           const matrix::Fbcsr<ValueType, IndexType> *const a,
           const matrix::Dense<ValueType> *const b,
           matrix::Dense<ValueType> *const c)
 {
     const int bs = a->get_block_size();
-    const IndexType nvecs = static_cast<IndexType>(b->get_size()[1]);
+    const auto nvecs = static_cast<IndexType>(b->get_size()[1]);
     const IndexType nbrows = a->get_num_block_rows();
     auto row_ptrs = a->get_const_row_ptrs();
     auto col_idxs = a->get_const_col_idxs();
@@ -80,11 +80,12 @@ void spmv(const std::shared_ptr<const ReferenceExecutor> exec,
     const blockutils::DenseBlocksView<const ValueType, IndexType> avalues(
         vals, bs, bs);
 
-    ValueType *const cvals = c->get_values();
+    auto *const cvals = c->get_values();
 
     for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
-        const IndexType crowblkend = (ibrow + 1) * bs * nvecs;
-        for (IndexType i = ibrow * bs * nvecs; i < crowblkend; i++)
+        // const IndexType crowblkend = (ibrow + 1) * bs * nvecs;
+        for (IndexType i = ibrow * bs * nvecs; i < (ibrow + 1) * bs * nvecs;
+             ++i)
             cvals[i] = zero<ValueType>();
 
         for (IndexType inz = row_ptrs[ibrow]; inz < row_ptrs[ibrow + 1];
@@ -106,7 +107,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPMV_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void advanced_spmv(const std::shared_ptr<const ReferenceExecutor> exec,
+void advanced_spmv(const std::shared_ptr<const ReferenceExecutor>,
                    const matrix::Dense<ValueType> *const alpha,
                    const matrix::Fbcsr<ValueType, IndexType> *const a,
                    const matrix::Dense<ValueType> *const b,
@@ -114,7 +115,7 @@ void advanced_spmv(const std::shared_ptr<const ReferenceExecutor> exec,
                    matrix::Dense<ValueType> *const c)
 {
     const int bs = a->get_block_size();
-    const IndexType nvecs = static_cast<IndexType>(b->get_size()[1]);
+    const auto nvecs = static_cast<IndexType>(b->get_size()[1]);
     const IndexType nbrows = a->get_num_block_rows();
     auto row_ptrs = a->get_const_row_ptrs();
     auto col_idxs = a->get_const_col_idxs();
@@ -124,11 +125,11 @@ void advanced_spmv(const std::shared_ptr<const ReferenceExecutor> exec,
     const blockutils::DenseBlocksView<const ValueType, IndexType> avalues(
         vals, bs, bs);
 
-    ValueType *const cvals = c->get_values();
+    auto *const cvals = c->get_values();
 
     for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
-        const IndexType crowblkend = (ibrow + 1) * bs * nvecs;
-        for (IndexType i = ibrow * bs * nvecs; i < crowblkend; i++)
+        for (IndexType i = ibrow * bs * nvecs; i < (ibrow + 1) * bs * nvecs;
+             ++i)
             cvals[i] *= vbeta;
 
         for (IndexType inz = row_ptrs[ibrow]; inz < row_ptrs[ibrow + 1];
@@ -151,13 +152,13 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void convert_to_dense(const std::shared_ptr<const ReferenceExecutor> exec,
+void convert_to_dense(const std::shared_ptr<const ReferenceExecutor>,
                       const matrix::Fbcsr<ValueType, IndexType> *const source,
                       matrix::Dense<ValueType> *const result)
 {
     const int bs = source->get_block_size();
-    const size_type nbrows = source->get_num_block_rows();
-    const size_type nbcols = source->get_num_block_cols();
+    const IndexType nbrows = source->get_num_block_rows();
+    const IndexType nbcols = source->get_num_block_cols();
     const IndexType *const row_ptrs = source->get_const_row_ptrs();
     const IndexType *const col_idxs = source->get_const_col_idxs();
     const ValueType *const vals = source->get_const_values();
@@ -189,13 +190,13 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void convert_to_csr(const std::shared_ptr<const ReferenceExecutor> exec,
+void convert_to_csr(const std::shared_ptr<const ReferenceExecutor>,
                     const matrix::Fbcsr<ValueType, IndexType> *const source,
                     matrix::Csr<ValueType, IndexType> *const result)
 {
     const int bs = source->get_block_size();
-    const size_type nbrows = source->get_num_block_rows();
-    const size_type nbcols = source->get_num_block_cols();
+    const IndexType nbrows = source->get_num_block_rows();
+    const IndexType nbcols = source->get_num_block_cols();
     const IndexType *const browptrs = source->get_const_row_ptrs();
     const IndexType *const bcolinds = source->get_const_col_idxs();
     const ValueType *const bvals = source->get_const_values();
@@ -236,7 +237,8 @@ void convert_to_csr(const std::shared_ptr<const ReferenceExecutor> exec,
         }
     }
 
-    row_ptrs[source->get_size()[0]] = source->get_num_stored_elements();
+    row_ptrs[source->get_size()[0]] =
+        static_cast<IndexType>(source->get_num_stored_elements());
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -245,7 +247,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType, typename UnaryOperator,
           bool transpose_blocks>
-void convert_fbcsr_to_fbcsc(const size_type num_blk_rows, const int blksz,
+void convert_fbcsr_to_fbcsc(const IndexType num_blk_rows, const int blksz,
                             const IndexType *const row_ptrs,
                             const IndexType *const col_idxs,
                             const ValueType *const fbcsr_vals,
@@ -257,13 +259,13 @@ void convert_fbcsr_to_fbcsc(const size_type num_blk_rows, const int blksz,
         fbcsr_vals, blksz, blksz);
     gko::blockutils::DenseBlocksView<ValueType, IndexType> cvalues(
         csc_vals, blksz, blksz);
-    for (size_type brow = 0; brow < num_blk_rows; ++brow) {
+    for (IndexType brow = 0; brow < num_blk_rows; ++brow) {
         for (auto i = row_ptrs[brow]; i < row_ptrs[brow + 1]; ++i) {
-            const auto dest_idx = col_ptrs[col_idxs[i]]++;
+            const auto dest_idx = col_ptrs[col_idxs[i]];
+            col_ptrs[col_idxs[i]]++;
             row_idxs[dest_idx] = brow;
             for (int ib = 0; ib < blksz; ib++)
                 for (int jb = 0; jb < blksz; jb++)
-                    // csc_vals[dest_idx] = op(fbcsr_vals[i]);
                     cvalues(dest_idx, ib, jb) =
                         op(transpose_blocks ? rvalues(i, jb, ib)
                                             : rvalues(i, ib, jb));
@@ -274,7 +276,6 @@ void convert_fbcsr_to_fbcsc(const size_type num_blk_rows, const int blksz,
 
 template <typename ValueType, typename IndexType, typename UnaryOperator>
 void transpose_and_transform(
-    const std::shared_ptr<const ReferenceExecutor> exec,
     matrix::Fbcsr<ValueType, IndexType> *const trans,
     const matrix::Fbcsr<ValueType, IndexType> *const orig, UnaryOperator op)
 {
@@ -286,10 +287,8 @@ void transpose_and_transform(
     auto trans_vals = trans->get_values();
     auto orig_vals = orig->get_const_values();
 
-    auto orig_num_cols = orig->get_size()[1];
-    const size_type nbcols = orig->get_num_block_cols();
-    auto orig_num_rows = orig->get_size()[0];
-    const size_type nbrows = orig->get_num_block_rows();
+    const IndexType nbcols = orig->get_num_block_cols();
+    const IndexType nbrows = orig->get_num_block_rows();
     auto orig_nbnz = orig_row_ptrs[nbrows];
 
     trans_row_ptrs[0] = 0;
@@ -302,12 +301,11 @@ void transpose_and_transform(
 
 
 template <typename ValueType, typename IndexType>
-void transpose(std::shared_ptr<const ReferenceExecutor> exec,
+void transpose(std::shared_ptr<const ReferenceExecutor>,
                const matrix::Fbcsr<ValueType, IndexType> *const orig,
                matrix::Fbcsr<ValueType, IndexType> *const trans)
 {
-    transpose_and_transform(exec, trans, orig,
-                            [](const ValueType x) { return x; });
+    transpose_and_transform(trans, orig, [](const ValueType x) { return x; });
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -315,11 +313,11 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void conj_transpose(std::shared_ptr<const ReferenceExecutor> exec,
+void conj_transpose(std::shared_ptr<const ReferenceExecutor>,
                     const matrix::Fbcsr<ValueType, IndexType> *const orig,
                     matrix::Fbcsr<ValueType, IndexType> *const trans)
 {
-    transpose_and_transform(exec, trans, orig,
+    transpose_and_transform(trans, orig,
                             [](const ValueType x) { return conj(x); });
 }
 
@@ -329,7 +327,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void calculate_max_nnz_per_row(
-    std::shared_ptr<const ReferenceExecutor> exec,
+    std::shared_ptr<const ReferenceExecutor>,
     const matrix::Fbcsr<ValueType, IndexType> *const source,
     size_type *const result)
 {
@@ -353,7 +351,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void calculate_nonzeros_per_row(
-    std::shared_ptr<const ReferenceExecutor> exec,
+    std::shared_ptr<const ReferenceExecutor>,
     const matrix::Fbcsr<ValueType, IndexType> *source, Array<size_type> *result)
 {
     const auto row_ptrs = source->get_const_row_ptrs();
@@ -373,14 +371,12 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void is_sorted_by_column_index(
-    std::shared_ptr<const ReferenceExecutor> exec,
+    std::shared_ptr<const ReferenceExecutor>,
     const matrix::Fbcsr<ValueType, IndexType> *const to_check,
     bool *const is_sorted)
 {
     const auto row_ptrs = to_check->get_const_row_ptrs();
     const auto col_idxs = to_check->get_const_col_idxs();
-    const auto size = to_check->get_size();
-    const int bs = to_check->get_block_size();
     const size_type nbrows = to_check->get_num_block_rows();
 
     for (size_type i = 0; i < nbrows; ++i) {
@@ -400,7 +396,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void extract_diagonal(std::shared_ptr<const ReferenceExecutor> exec,
+void extract_diagonal(std::shared_ptr<const ReferenceExecutor>,
                       const matrix::Fbcsr<ValueType, IndexType> *const orig,
                       matrix::Diagonal<ValueType> *const diag)
 {
@@ -408,16 +404,16 @@ void extract_diagonal(std::shared_ptr<const ReferenceExecutor> exec,
     const auto col_idxs = orig->get_const_col_idxs();
     const auto values = orig->get_const_values();
     const int bs = orig->get_block_size();
-    const size_type diag_size = diag->get_size()[0];
-    const size_type nbrows = orig->get_num_block_rows();
+    const IndexType nbrows = orig->get_num_block_rows();
     auto diag_values = diag->get_values();
-    assert(diag_size == orig->get_size()[0]);
+
+    assert(diag->get_size()[0] == orig->get_size()[0]);
 
     const gko::blockutils::DenseBlocksView<const ValueType, IndexType> vblocks(
         values, bs, bs);
 
-    for (size_type ibrow = 0; ibrow < nbrows; ++ibrow) {
-        for (size_type idx = row_ptrs[ibrow]; idx < row_ptrs[ibrow + 1];
+    for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
+        for (IndexType idx = row_ptrs[ibrow]; idx < row_ptrs[ibrow + 1];
              ++idx) {
             if (col_idxs[idx] == ibrow) {
                 for (int ib = 0; ib < bs; ib++)
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 4feaf5d1e5d..807c8e967bd 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -426,9 +426,7 @@ TYPED_TEST(Fbcsr, MovesToSparsityCsr)
     using SparsityCsr = typename TestFixture::SparCsr;
     using Fbcsr = typename TestFixture::Mtx;
     auto sparsity_mtx = SparsityCsr::create(this->mtx->get_executor());
-    // auto fbcsr_ref = Fbcsr::create(this->mtx->get_executor());
 
-    // fbcsr_ref->copy_from(this->mtx.get());
     this->mtx->move_to(sparsity_mtx.get());
 
     this->assert_equal_to_mtx(sparsity_mtx.get());

From b2e2ef92494b2d964c4e56a8335b4411122f6f09 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Tue, 8 Dec 2020 11:38:39 +0100
Subject: [PATCH 0558/1008] refactored DenseBlocksView a little and wrote test

---
 core/components/fixed_block.hpp   | 75 ++++---------------------------
 core/matrix/fbcsr.cpp             | 67 ++++++++++++++++++++++++++-
 core/test/matrix/fbcsr.cpp        | 23 ++++++++++
 core/test/matrix/fbcsr_sample.cpp | 20 +++++++++
 core/test/matrix/fbcsr_sample.hpp | 19 ++++++--
 5 files changed, 133 insertions(+), 71 deletions(-)

diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
index c41386b9964..2e9595cdc04 100644
--- a/core/components/fixed_block.hpp
+++ b/core/components/fixed_block.hpp
@@ -34,13 +34,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_COMPONENTS_FIXED_BLOCK_HPP_
 
 
-#include <algorithm>
 #include <type_traits>
-#include <vector>
 
 
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
 
 
 namespace gko {
@@ -93,63 +90,6 @@ class FixedBlock final {
 };
 
 
-/**
- * A lightweight dynamic block type on the host
- *
- * @tparam ValueType The numeric type of entries of the block
- */
-template <typename ValueType>
-class DenseBlock final {
-public:
-    using value_type = ValueType;
-
-    DenseBlock() {}
-
-    DenseBlock(const int num_rows, const int num_cols)
-        : nrows_{num_rows}, ncols_{num_cols}, vals_(num_rows * num_cols)
-    {}
-
-    value_type &at(const int row, const int col)
-    {
-        return vals_[row * ncols_ + col];
-    }
-
-    const value_type &at(const int row, const int col) const
-    {
-        return vals_[row * ncols_ + col];
-    }
-
-    value_type &operator()(const int row, const int col)
-    {
-        return at(row, col);
-    }
-
-    const value_type &operator()(const int row, const int col) const
-    {
-        return at(row, col);
-    }
-
-    int size() const { return nrows_ * ncols_; }
-
-    void resize(const int nrows, const int ncols)
-    {
-        vals_.resize(nrows * ncols);
-        nrows_ = nrows;
-        ncols_ = ncols;
-    }
-
-    void zero()
-    {
-        std::fill(vals_.begin(), vals_.end(), static_cast<value_type>(0));
-    }
-
-private:
-    int nrows_ = 0;
-    int ncols_ = 0;
-    std::vector<value_type> vals_;
-};
-
-
 /**
  * @brief A view into a an array of dense blocks of some runtime-defined size
  *
@@ -174,18 +114,18 @@ class DenseBlocksView final {
      */
     DenseBlocksView(ValueType *const buffer, const int num_rows,
                     const int num_cols)
-        : nrows_{num_rows}, ncols_{num_cols}, vals_{buffer}
+        : nrows{num_rows}, ncols{num_cols}, vals_{buffer}
     {}
 
     value_type &at(const index_type block, const int row, const int col)
     {
-        return vals_[block * nrows_ * ncols_ + row * ncols_ + col];
+        return vals_[block * nrows * ncols + row * ncols + col];
     }
 
     const typename std::remove_const<value_type>::type &at(
         const index_type block, const int row, const int col) const
     {
-        return vals_[block * nrows_ * ncols_ + row * ncols_ + col];
+        return vals_[block * nrows * ncols + row * ncols + col];
     }
 
     value_type &operator()(const index_type block, const int row, const int col)
@@ -199,10 +139,11 @@ class DenseBlocksView final {
         return at(block, row, col);
     }
 
+    const int nrows;  ///< Number of rows in each block
+    const int ncols;  ///< Number of columns in each block
+
 private:
-    int nrows_;  ///< Number of rows in each block
-    int ncols_;  ///< Number of columns in each block
-    value_type *vals_;
+    value_type *const vals_;
 };
 
 
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 51950dc96fe..88b17928d06 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -81,6 +81,71 @@ GKO_REGISTER_OPERATION(outplace_absolute_array,
 }  // namespace fbcsr
 
 
+namespace detail {
+
+
+/**
+ * A lightweight dynamic block type on the host
+ *
+ * @internal Currently used only while reading a FBCSR matrix from matrix_data.
+ *
+ * @tparam ValueType The numeric type of entries of the block
+ */
+template <typename ValueType>
+class DenseBlock final {
+public:
+    using value_type = ValueType;
+
+    DenseBlock() {}
+
+    DenseBlock(const int num_rows, const int num_cols)
+        : nrows_{num_rows}, ncols_{num_cols}, vals_(num_rows * num_cols)
+    {}
+
+    value_type &at(const int row, const int col)
+    {
+        return vals_[row * ncols_ + col];
+    }
+
+    const value_type &at(const int row, const int col) const
+    {
+        return vals_[row * ncols_ + col];
+    }
+
+    value_type &operator()(const int row, const int col)
+    {
+        return at(row, col);
+    }
+
+    const value_type &operator()(const int row, const int col) const
+    {
+        return at(row, col);
+    }
+
+    int size() const { return nrows_ * ncols_; }
+
+    void resize(const int nrows, const int ncols)
+    {
+        vals_.resize(nrows * ncols);
+        nrows_ = nrows;
+        ncols_ = ncols;
+    }
+
+    void zero()
+    {
+        std::fill(vals_.begin(), vals_.end(), static_cast<value_type>(0));
+    }
+
+private:
+    int nrows_ = 0;
+    int ncols_ = 0;
+    std::vector<value_type> vals_;
+};
+
+
+}  // namespace detail
+
+
 template <typename ValueType, typename IndexType>
 Fbcsr<ValueType, IndexType>::Fbcsr(const std::shared_ptr<const Executor> exec,
                                    const dim<2> &size,
@@ -258,7 +323,7 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
 
     const int bs = this->bs_;
 
-    using Blk_t = blockutils::DenseBlock<value_type>;
+    using Blk_t = detail::DenseBlock<value_type>;
 
     struct FbEntry {
         index_type block_row;
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index b32dbb76367..fe0b48ef4b0 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -309,4 +309,27 @@ TYPED_TEST(Fbcsr, GeneratesCorrectMatrixData)
 }
 
 
+TYPED_TEST(Fbcsr, DenseBlocksViewWorksCorrectly)
+{
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using Dbv = gko::blockutils::DenseBlocksView<value_type, index_type>;
+
+    const gko::testing::FbcsrSample2<value_type, index_type> fbsample(
+        std::static_pointer_cast<const gko::ReferenceExecutor>(this->exec));
+
+    auto refmtx = fbsample.generate_fbcsr();
+    const Dbv testdbv(refmtx->get_values(), fbsample.bs, fbsample.bs);
+
+    std::vector<value_type> ref_dbv_array(fbsample.nnz);
+    Dbv refdbv(ref_dbv_array.data(), fbsample.bs, fbsample.bs);
+    fbsample.fill_value_blocks_view(refdbv);
+
+    for (index_type ibz = 0; ibz < fbsample.nbnz; ibz++)
+        for (int i = 0; i < fbsample.bs; ++i)
+            for (int j = 0; j < fbsample.bs; ++j)
+                ASSERT_EQ(testdbv(ibz, i, j), refdbv(ibz, i, j));
+}
+
+
 }  // namespace
diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
index 555f2db8627..22d8a3c26da 100644
--- a/core/test/matrix/fbcsr_sample.cpp
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -541,6 +541,26 @@ FbcsrSample2<ValueType, IndexType>::generate_fbcsr() const
     return mtx;
 }
 
+template <typename ValueType, typename IndexType>
+void FbcsrSample2<ValueType, IndexType>::fill_value_blocks_view(
+    DenseBlocksView &dbv) const
+{
+    dbv(0, 0, 0) = 1.0;
+    dbv(0, 0, 1) = 2.0;
+    dbv(0, 1, 0) = 3.0;
+    dbv(0, 1, 1) = 0.0;
+    for (int i = 0; i < 2; ++i)
+        for (int j = 0; j < 2; ++j) dbv(1, i, j) = 0.15 + FBCSR_TEST_OFFSET;
+    dbv(2, 0, 0) = 0.15 + FBCSR_TEST_OFFSET;
+    dbv(2, 0, 1) = 0.15 + FBCSR_TEST_OFFSET;
+    dbv(2, 1, 0) = 0.0;
+    dbv(2, 1, 1) = 0.0;
+    dbv(3, 0, 0) = -12.0;
+    dbv(3, 0, 1) = -1.0;
+    dbv(3, 1, 0) = -2.0;
+    dbv(3, 1, 1) = -11.0;
+}
+
 template <typename ValueType, typename IndexType>
 std::unique_ptr<gko::matrix::Fbcsr<remove_complex<ValueType>, IndexType>>
 FbcsrSample2<ValueType, IndexType>::generate_abs_fbcsr_abstype() const
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 12ec1fb310f..9b04d9c30cc 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -43,6 +43,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
+#include "core/components/fixed_block.hpp"
+
+
 namespace gko {
 namespace testing {
 
@@ -174,6 +177,8 @@ class FbcsrSample2 {
     using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
     using Dense = gko::matrix::Dense<value_type>;
     using Diagonal = gko::matrix::Diagonal<value_type>;
+    using DenseBlocksView =
+        gko::blockutils::DenseBlocksView<ValueType, IndexType>;
 
     FbcsrSample2(std::shared_ptr<const gko::ReferenceExecutor> exec);
 
@@ -192,6 +197,11 @@ class FbcsrSample2 {
     std::unique_ptr<gko::matrix::Fbcsr<remove_complex<value_type>, index_type>>
     generate_abs_fbcsr_abstype() const;
 
+    /**
+     * Fills a view into a FBCSR values array using the sample matrix's data
+     */
+    void fill_value_blocks_view(DenseBlocksView &dbv) const;
+
     /// Enables use of literals to instantiate value data
     template <typename U>
     constexpr ValueType sct(U u) const
@@ -210,8 +220,10 @@ class FbcsrSample2 {
     const std::shared_ptr<const gko::Executor> exec;
 };
 
-/// Generates the a sample block CSR square matrix in different formats
-/** This currently a 4 x 4 matrix with 2x2 blocks.
+/**
+ * @brief Generates the a sample block CSR square matrix and its transpose
+ *
+ * This currently a 4 x 4 matrix with 2x2 blocks.
  */
 template <typename ValueType, typename IndexType>
 class FbcsrSampleSquare {
@@ -237,7 +249,8 @@ class FbcsrSampleSquare {
 };
 
 /**
- * Generates the a sample block CSR matrix with complex values
+ * @brief Generates a sample block CSR matrix with complex values
+ *
  * This is a 6 x 8 matrix with 2x2 blocks.
  */
 template <typename ValueType, typename IndexType>

From b4914caca1c996b01ede1eb2fb5a26c616ad0a48 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Tue, 8 Dec 2020 16:21:46 +0100
Subject: [PATCH 0559/1008] removed couple of unnecessary functions from fbcsr
 sample generators

---
 core/test/matrix/fbcsr_sample.cpp       | 41 -------------------------
 core/test/matrix/fbcsr_sample.hpp       | 14 ---------
 include/ginkgo/core/matrix/fbcsr.hpp    |  8 ++++-
 reference/test/matrix/fbcsr_kernels.cpp |  8 +++--
 4 files changed, 13 insertions(+), 58 deletions(-)

diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
index 22d8a3c26da..d7fc2a8a237 100644
--- a/core/test/matrix/fbcsr_sample.cpp
+++ b/core/test/matrix/fbcsr_sample.cpp
@@ -430,40 +430,6 @@ gko::matrix_data<ValueType, IndexType> FbcsrSample<
                      {5, 8, sct(14.0) + FBCSR_TEST_IMAGINARY}}});
 }
 
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Coo<ValueType, IndexType>>
-FbcsrSample<ValueType, IndexType>::generate_coo() const
-{
-    gko::matrix_data<ValueType, IndexType> mdata =
-        generate_matrix_data_with_explicit_zeros();
-
-    using nztype =
-        typename gko::matrix_data<ValueType, IndexType>::nonzero_type;
-    std::sort(mdata.nonzeros.begin(), mdata.nonzeros.end(),
-              [](const nztype &a, const nztype &b) {
-                  if (a.row < b.row)
-                      return true;
-                  else if (a.row > b.row)
-                      return false;
-                  else if (a.column < b.column)
-                      return true;
-                  else
-                      return false;
-              });
-
-    gko::Array<IndexType> rowidx(exec, nnz);
-    gko::Array<IndexType> colidx(exec, nnz);
-    gko::Array<ValueType> values(exec, nnz);
-    for (size_t i = 0; i < mdata.nonzeros.size(); i++) {
-        rowidx.get_data()[i] = mdata.nonzeros[i].row;
-        colidx.get_data()[i] = mdata.nonzeros[i].column;
-        values.get_data()[i] = mdata.nonzeros[i].value;
-    }
-    auto mat =
-        Coo::create(exec, gko::dim<2>{nrows, ncols}, values, colidx, rowidx);
-    return mat;
-}
-
 template <typename ValueType, typename IndexType>
 std::unique_ptr<gko::matrix::SparsityCsr<ValueType, IndexType>>
 FbcsrSample<ValueType, IndexType>::generate_sparsity_csr() const
@@ -478,13 +444,6 @@ FbcsrSample<ValueType, IndexType>::generate_sparsity_csr() const
     return SparCsr::create(exec, gko::dim<2>{nbrows, nbcols}, colids, rowptrs);
 }
 
-template <typename ValueType, typename IndexType>
-gko::Array<IndexType> FbcsrSample<ValueType, IndexType>::getNonzerosPerRow()
-    const
-{
-    return gko::Array<index_type>(exec, {6, 6, 6, 6, 6, 6});
-}
-
 #define GKO_DECLARE_FBCSR_TEST_SAMPLE(ValueType, IndexType) \
     class FbcsrSample<ValueType, IndexType>
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_TEST_SAMPLE);
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 9b04d9c30cc..5985e192a5b 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -35,7 +35,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/matrix_data.hpp>
-#include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/diagonal.hpp>
@@ -64,7 +63,6 @@ class FbcsrSample {
     using absvalue_type = remove_complex<value_type>;
     using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
-    using Coo = gko::matrix::Coo<value_type, index_type>;
     using Dense = gko::matrix::Dense<value_type>;
     using MatData = gko::matrix_data<value_type, index_type>;
     using SparCsr = gko::matrix::SparsityCsr<value_type, index_type>;
@@ -88,13 +86,6 @@ class FbcsrSample {
      */
     std::unique_ptr<Dense> generate_dense() const;
 
-    /**
-     * @return The matrix in COO format keeping explicit nonzeros
-     *
-     * The nonzeros are sorted by row and column.
-     */
-    std::unique_ptr<Coo> generate_coo() const;
-
     /**
      * @return Sparsity structure of the matrix
      */
@@ -115,11 +106,6 @@ class FbcsrSample {
      */
     MatData generate_matrix_data_with_explicit_zeros() const;
 
-    /**
-     *  @return An array containing number of stored values in each row
-     */
-    gko::Array<index_type> getNonzerosPerRow() const;
-
     /**
      * @return FBCSR matrix with absolute values of respective entries
      */
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index 409a6fc5130..925e7268749 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -152,6 +152,12 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
 
     void move_to(Dense<ValueType> *other) override;
 
+    /**
+     * Converts the matrix to CSR format
+     *
+     * @note Any explicit zeros in the original matrix are retained
+     * in the converted result.
+     */
     void convert_to(Csr<ValueType, IndexType> *result) const override;
 
     void move_to(Csr<ValueType, IndexType> *result) override;
@@ -159,7 +165,7 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     /**
      * Get the block sparsity pattern in CSR-like format
      *
-     * Note that the actual non-zero values are never copied;
+     * @note The actual non-zero values are never copied;
      * the result always has a value array of size 1 with the value 1.
      */
     void convert_to(SparsityCsr<ValueType, IndexType> *result) const override;
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 807c8e967bd..1a7e00d3f40 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -161,14 +161,18 @@ TYPED_TEST(Fbcsr, AppliesToDenseVector)
     auto x = Vec::create(this->exec, gko::dim<2>{(gko::size_type)ncols, 1});
     T *const xvals = x->get_values();
     for (index_type i = 0; i < ncols; i++)
-        // xvals[i] = std::log(static_cast<T>(static_cast<float>((i+1)^2)));
-        xvals[i] = (i + 1.0) * (i + 1.0);
+        xvals[i] = std::sin(static_cast<T>(static_cast<float>((i + 1) ^ 2)));
     auto y = Vec::create(this->exec, gko::dim<2>{(gko::size_type)nrows, 1});
     auto yref = Vec::create(this->exec, gko::dim<2>{(gko::size_type)nrows, 1});
 
     this->mtx2->apply(x.get(), y.get());
 
     this->fbsample2.apply(x.get(), yref.get());
+    // using Csr = typename TestFixture::Csr;
+    // auto csr_mtx = Csr::create(this->mtx->get_executor(),
+    //                            std::make_shared<typename Csr::classical>());
+    // this->mtx2->convert_to(csr_mtx.get());
+    // csr_mtx->apply(x.get(), yref.get());
 
     const double tolerance =
         std::numeric_limits<gko::remove_complex<T>>::epsilon();

From 09b657069a499c06c7e5fb6e58ffbfa3639e58b7 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 9 Dec 2020 18:17:52 +0100
Subject: [PATCH 0560/1008] FBCSR reference kernel testing now depends mostly
 on comparison with CSR kernels

This enabled reduction in FBCSR sample matrix code and removal of the
cpp file, so the test_fbcsr_sample library is no longer needed.
---
 core/test/matrix/CMakeLists.txt         |  12 +-
 core/test/matrix/fbcsr.cpp              |   3 -
 core/test/matrix/fbcsr_sample.cpp       | 844 ------------------------
 core/test/matrix/fbcsr_sample.hpp       | 619 ++++++++++++++---
 reference/test/matrix/CMakeLists.txt    |   2 +-
 reference/test/matrix/fbcsr_kernels.cpp | 139 ++--
 6 files changed, 623 insertions(+), 996 deletions(-)
 delete mode 100644 core/test/matrix/fbcsr_sample.cpp

diff --git a/core/test/matrix/CMakeLists.txt b/core/test/matrix/CMakeLists.txt
index 9376b970552..64b3b3ed593 100644
--- a/core/test/matrix/CMakeLists.txt
+++ b/core/test/matrix/CMakeLists.txt
@@ -5,17 +5,7 @@ ginkgo_create_test(csr_builder)
 ginkgo_create_test(dense)
 ginkgo_create_test(diagonal)
 ginkgo_create_test(ell)
-add_library(test_fbcsr_sample STATIC fbcsr_sample.cpp)
-target_include_directories(test_fbcsr_sample
-  PRIVATE
-  "$<BUILD_INTERFACE:${Ginkgo_SOURCE_DIR}/include>"
-  "$<BUILD_INTERFACE:${Ginkgo_SOURCE_DIR}>"
-  "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}/include>"
-  )
-set_property(TARGET test_fbcsr_sample PROPERTY CXX_STANDARD 14)
-set_property(TARGET test_fbcsr_sample PROPERTY CXX_EXTENSIONS OFF)
-target_link_libraries(test_fbcsr_sample PRIVATE ginkgo)
-ginkgo_create_test(fbcsr test_fbcsr_sample)
+ginkgo_create_test(fbcsr)
 ginkgo_create_test(fbcsr_builder)
 ginkgo_create_test(hybrid)
 ginkgo_create_test(identity)
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index fe0b48ef4b0..619315e5c98 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -134,11 +134,9 @@ TYPED_TEST(Fbcsr, SampleGeneratorIsCorrect)
     using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
     using Csr = gko::matrix::Csr<value_type, index_type>;
-    using Dense = gko::matrix::Dense<value_type>;
 
     std::unique_ptr<const Mtx> fbmtx = this->fbsample.generate_fbcsr();
     std::unique_ptr<const Csr> csmtx = this->fbsample.generate_csr();
-    std::unique_ptr<const Dense> dmtx = this->fbsample.generate_dense();
     const int bs = this->fbsample.bs;
     ASSERT_EQ(bs, fbmtx->get_block_size());
 
@@ -174,7 +172,6 @@ TYPED_TEST(Fbcsr, SampleGeneratorIsCorrect)
                     ASSERT_EQ(col, csmtx->get_const_col_idxs()[inz]);
                     ASSERT_EQ(fbvals(ibnz, ib, jb),
                               csmtx->get_const_values()[inz]);
-                    ASSERT_EQ(fbvals(ibnz, ib, jb), dmtx->at(row, col));
                 }
             }
         }
diff --git a/core/test/matrix/fbcsr_sample.cpp b/core/test/matrix/fbcsr_sample.cpp
deleted file mode 100644
index d7fc2a8a237..00000000000
--- a/core/test/matrix/fbcsr_sample.cpp
+++ /dev/null
@@ -1,844 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <algorithm>
-
-
-#include "core/components/fixed_block.hpp"
-#include "core/test/matrix/fbcsr_sample.hpp"
-
-#define FBCSR_TEST_OFFSET 0.000011118888
-
-#define FBCSR_TEST_C_MAG 0.1 + FBCSR_TEST_OFFSET
-#define FBCSR_TEST_IMAGINARY \
-    sct(std::complex<remove_complex<ValueType>>(0, FBCSR_TEST_C_MAG))
-
-namespace gko {
-namespace testing {
-
-
-/** Generates a copy of the given matrix with a different scalar type
- *
- * \tparam AbsValueType The scalar type of the output matrix
- */
-template <typename FbcsrType, typename AbsValueType>
-static std::unique_ptr<
-    gko::matrix::Fbcsr<AbsValueType, typename FbcsrType::index_type>>
-generate_acopy_impl(const FbcsrType *const mat)
-{
-    using index_type = typename FbcsrType::index_type;
-    using value_type = typename FbcsrType::value_type;
-    using AbsFbcsr = gko::matrix::Fbcsr<AbsValueType, index_type>;
-
-    std::shared_ptr<const ReferenceExecutor> exec =
-        std::dynamic_pointer_cast<const ReferenceExecutor>(mat->get_executor());
-
-    std::unique_ptr<AbsFbcsr> amat =
-        AbsFbcsr::create(exec, mat->get_size(), mat->get_num_stored_elements(),
-                         mat->get_block_size());
-
-    const index_type *const colidxs = mat->get_col_idxs();
-    const index_type *const rowptrs = mat->get_row_ptrs();
-    index_type *const acolidxs = amat->get_col_idxs();
-    index_type *const arowptrs = amat->get_row_ptrs();
-
-    for (index_type i = 0;
-         i < mat->get_num_stored_elements() /
-                 (mat->get_block_size() * mat->get_block_size());
-         i++)
-        acolidxs[i] = colidxs[i];
-
-    for (index_type i = 0; i < mat->get_size()[0] / mat->get_block_size() + 1;
-         i++)
-        arowptrs[i] = rowptrs[i];
-
-    return amat;
-}
-
-
-template <typename ValueType, typename IndexType>
-FbcsrSample<ValueType, IndexType>::FbcsrSample(
-    const std::shared_ptr<const gko::ReferenceExecutor> rexec)
-    : nrows{6},
-      ncols{12},
-      nnz{36},
-      nbrows{2},
-      nbcols{4},
-      nbnz{4},
-      bs{3},
-      exec(rexec)
-{}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
-FbcsrSample<ValueType, IndexType>::generate_fbcsr() const
-{
-    std::unique_ptr<Fbcsr> mtx =
-        Fbcsr::create(exec,
-                      gko::dim<2>{static_cast<size_type>(nrows),
-                                  static_cast<size_type>(ncols)},
-                      nnz, bs);
-
-    value_type *const v = mtx->get_values();
-    index_type *const c = mtx->get_col_idxs();
-    index_type *const r = mtx->get_row_ptrs();
-    r[0] = 0;
-    r[1] = 2;
-    r[2] = 4;
-    c[0] = 1;
-    c[1] = 3;
-    c[2] = 0;
-    c[3] = 2;
-
-    gko::blockutils::DenseBlocksView<value_type, index_type> vals(v, bs, bs);
-
-    if (mtx->get_size()[0] % bs != 0)
-        throw gko::BadDimension(__FILE__, __LINE__, __func__, "test fbcsr",
-                                mtx->get_size()[0], mtx->get_size()[1],
-                                "block size does not divide the size!");
-
-    for (index_type ibrow = 0; ibrow < mtx->get_num_block_rows(); ibrow++) {
-        const index_type *const browptr = mtx->get_row_ptrs();
-        for (index_type inz = browptr[ibrow]; inz < browptr[ibrow + 1]; inz++) {
-            const index_type bcolind = mtx->get_col_idxs()[inz];
-            const value_type base = (ibrow + 1) * (bcolind + 1);
-            for (int ival = 0; ival < bs; ival++)
-                for (int jval = 0; jval < bs; jval++)
-                    vals(inz, ival, jval) =
-                        base + static_cast<gko::remove_complex<value_type>>(
-                                   ival * bs + jval);
-        }
-    }
-
-    // Some of the entries are set to zero
-    vals(0, 2, 0) = gko::zero<value_type>();
-    vals(0, 2, 2) = gko::zero<value_type>();
-    vals(3, 0, 0) = gko::zero<value_type>();
-
-    v[34] += FBCSR_TEST_IMAGINARY;
-    v[35] += FBCSR_TEST_IMAGINARY;
-
-    return mtx;
-}
-
-template <typename ValueType, typename IndexType>
-template <typename FbcsrType>
-void FbcsrSample<ValueType, IndexType>::correct_abs_for_complex_values(
-    FbcsrType *const amat) const
-{
-    using out_value_type = typename FbcsrType::value_type;
-    using outreal_type = remove_complex<out_value_type>;
-    out_value_type *const avals = amat->get_values();
-    if (is_complex<ValueType>()) {
-        auto mo = static_cast<outreal_type>(FBCSR_TEST_C_MAG);
-        avals[34] = sqrt(pow(static_cast<outreal_type>(13.0), 2) +
-                         pow(static_cast<outreal_type>(mo), 2));
-        avals[35] = sqrt(pow(static_cast<outreal_type>(14.0), 2) +
-                         pow(static_cast<outreal_type>(mo), 2));
-    }
-}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Fbcsr<remove_complex<ValueType>, IndexType>>
-FbcsrSample<ValueType, IndexType>::generate_abs_fbcsr_abstype() const
-{
-    using AbsValueType = typename gko::remove_complex<ValueType>;
-    using AbsFbcsr = gko::matrix::Fbcsr<AbsValueType, IndexType>;
-
-    const std::unique_ptr<const Fbcsr> mat = generate_fbcsr();
-    std::unique_ptr<AbsFbcsr> amat =
-        generate_acopy_impl<Fbcsr, AbsValueType>(mat.get());
-
-    AbsValueType *const avals = amat->get_values();
-    const ValueType *const vals = mat->get_values();
-    for (IndexType i = 0; i < amat->get_num_stored_elements(); i++)
-        avals[i] = abs(vals[i]);
-
-    correct_abs_for_complex_values(amat.get());
-
-    return amat;
-}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
-FbcsrSample<ValueType, IndexType>::generate_abs_fbcsr() const
-{
-    const std::unique_ptr<const Fbcsr> mat = generate_fbcsr();
-    std::unique_ptr<Fbcsr> amat =
-        generate_acopy_impl<Fbcsr, ValueType>(mat.get());
-
-    ValueType *const avals = amat->get_values();
-    const ValueType *const vals = mat->get_values();
-    for (IndexType i = 0; i < amat->get_num_stored_elements(); i++)
-        avals[i] = abs(vals[i]);
-
-    correct_abs_for_complex_values(amat.get());
-
-    return amat;
-}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Csr<ValueType, IndexType>>
-FbcsrSample<ValueType, IndexType>::generate_csr() const
-{
-    std::unique_ptr<Csr> csrm =
-        Csr::create(exec, gko::dim<2>{nrows, ncols}, nnz,
-                    std::make_shared<typename Csr::classical>());
-    index_type *const csrrow = csrm->get_row_ptrs();
-    index_type *const csrcols = csrm->get_col_idxs();
-    value_type *const csrvals = csrm->get_values();
-
-    csrrow[0] = 0;
-    csrrow[1] = 6;
-    csrrow[2] = 12;
-    csrrow[3] = 18;
-    csrrow[4] = 24;
-    csrrow[5] = 30;
-    csrrow[6] = 36;
-
-    csrcols[0] = 3;
-    csrcols[1] = 4;
-    csrcols[2] = 5;
-    csrcols[6] = 3;
-    csrcols[7] = 4;
-    csrcols[8] = 5;
-    csrcols[12] = 3;
-    csrcols[13] = 4;
-    csrcols[14] = 5;
-
-    csrcols[3] = 9;
-    csrcols[4] = 10;
-    csrcols[5] = 11;
-    csrcols[9] = 9;
-    csrcols[10] = 10;
-    csrcols[11] = 11;
-    csrcols[15] = 9;
-    csrcols[16] = 10;
-    csrcols[17] = 11;
-
-    csrcols[18] = 0;
-    csrcols[19] = 1;
-    csrcols[20] = 2;
-    csrcols[24] = 0;
-    csrcols[25] = 1;
-    csrcols[26] = 2;
-    csrcols[30] = 0;
-    csrcols[31] = 1;
-    csrcols[32] = 2;
-
-    csrcols[21] = 6;
-    csrcols[22] = 7;
-    csrcols[23] = 8;
-    csrcols[27] = 6;
-    csrcols[28] = 7;
-    csrcols[29] = 8;
-    csrcols[33] = 6;
-    csrcols[34] = 7;
-    csrcols[35] = 8;
-
-    // values
-    csrvals[0] = 2;
-    csrvals[1] = 3;
-    csrvals[2] = 4;
-    csrvals[6] = 5;
-    csrvals[7] = 6;
-    csrvals[8] = 7;
-    csrvals[12] = 0;
-    csrvals[13] = 9;
-    csrvals[14] = 0;
-
-    csrvals[3] = 4;
-    csrvals[4] = 5;
-    csrvals[5] = 6;
-    csrvals[9] = 7;
-    csrvals[10] = 8;
-    csrvals[11] = 9;
-    csrvals[15] = 10;
-    csrvals[16] = 11;
-    csrvals[17] = 12;
-
-    csrvals[18] = 2;
-    csrvals[19] = 3;
-    csrvals[20] = 4;
-    csrvals[24] = 5;
-    csrvals[25] = 6;
-    csrvals[26] = 7;
-    csrvals[30] = 8;
-    csrvals[31] = 9;
-    csrvals[32] = 10;
-
-    csrvals[21] = 0;
-    csrvals[22] = 7;
-    csrvals[23] = 8;
-    csrvals[27] = 9;
-    csrvals[28] = 10;
-    csrvals[29] = 11;
-    csrvals[33] = 12;
-    csrvals[34] = 13;
-    csrvals[35] = 14;
-
-    csrvals[34] += FBCSR_TEST_IMAGINARY;
-    csrvals[35] += FBCSR_TEST_IMAGINARY;
-
-    return csrm;
-}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Dense<ValueType>>
-FbcsrSample<ValueType, IndexType>::generate_dense() const
-{
-    std::unique_ptr<Dense> densem =
-        Dense::create(exec, gko::dim<2>(nrows, ncols));
-
-    for (size_type irow = 0; irow < densem->get_size()[0]; irow++)
-        for (size_type jcol = 0; jcol < densem->get_size()[1]; jcol++) {
-            densem->at(irow, jcol) = 0;
-            if (irow < 3 && jcol >= 3 && jcol < 6)
-                densem->at(irow, jcol) = 2.0 + irow * bs + jcol - 3;
-            if (irow < 3 && jcol >= 9)
-                densem->at(irow, jcol) = 4.0 + irow * bs + jcol - 9;
-            if (irow >= 3 && jcol < 3)
-                densem->at(irow, jcol) = 2.0 + (irow - 3) * bs + jcol;
-            if (irow >= 3 && jcol >= 6 && jcol < 9)
-                densem->at(irow, jcol) = 6.0 + (irow - 3) * bs + jcol - 6;
-        }
-
-    densem->at(2, 3) = densem->at(2, 5) = densem->at(3, 6) = 0.0;
-    densem->at(5, 7) += FBCSR_TEST_IMAGINARY;
-    densem->at(5, 8) += FBCSR_TEST_IMAGINARY;
-
-    return densem;
-}
-
-// Assuming row-major blocks
-template <typename ValueType, typename IndexType>
-gko::matrix_data<ValueType, IndexType>
-FbcsrSample<ValueType, IndexType>::generate_matrix_data() const
-{
-    return MatData({{6, 12},
-                    {{0, 3, 2.0},
-                     {0, 4, 3.0},
-                     {0, 5, 4.0},
-                     {1, 3, 5.0},
-                     {1, 4, 6.0},
-                     {1, 5, 7.0},
-                     {2, 4, 9.0},
-
-                     {0, 9, 4.0},
-                     {0, 10, 5.0},
-                     {0, 11, 6.0},
-                     {1, 9, 7.0},
-                     {1, 10, 8.0},
-                     {1, 11, 9.0},
-                     {2, 9, 10.0},
-                     {2, 10, 11.0},
-                     {2, 11, 12.0},
-
-                     {3, 0, 2.0},
-                     {3, 1, 3.0},
-                     {3, 2, 4.0},
-                     {4, 0, 5.0},
-                     {4, 1, 6.0},
-                     {4, 2, 7.0},
-                     {5, 0, 8.0},
-                     {5, 1, 9.0},
-                     {5, 2, 10.0},
-
-                     {3, 7, 7.0},
-                     {3, 8, 8.0},
-                     {4, 6, 9.0},
-                     {4, 7, 10.0},
-                     {4, 8, 11.0},
-                     {5, 6, 12.0},
-                     {5, 7, sct(13.0) + FBCSR_TEST_IMAGINARY},
-                     {5, 8, sct(14.0) + FBCSR_TEST_IMAGINARY}}});
-}
-
-// Assuming row-major blocks
-template <typename ValueType, typename IndexType>
-gko::matrix_data<ValueType, IndexType> FbcsrSample<
-    ValueType, IndexType>::generate_matrix_data_with_explicit_zeros() const
-{
-    return MatData({{6, 12},
-                    {{0, 3, 2.0},
-                     {0, 4, 3.0},
-                     {0, 5, 4.0},
-                     {1, 3, 5.0},
-                     {1, 4, 6.0},
-                     {1, 5, 7.0},
-                     {2, 3, 0.0},
-                     {2, 4, 9.0},
-                     {2, 5, 0.0},
-
-                     {0, 9, 4.0},
-                     {0, 10, 5.0},
-                     {0, 11, 6.0},
-                     {1, 9, 7.0},
-                     {1, 10, 8.0},
-                     {1, 11, 9.0},
-                     {2, 9, 10.0},
-                     {2, 10, 11.0},
-                     {2, 11, 12.0},
-
-                     {3, 0, 2.0},
-                     {3, 1, 3.0},
-                     {3, 2, 4.0},
-                     {4, 0, 5.0},
-                     {4, 1, 6.0},
-                     {4, 2, 7.0},
-                     {5, 0, 8.0},
-                     {5, 1, 9.0},
-                     {5, 2, 10.0},
-
-                     {3, 6, 0.0},
-                     {3, 7, 7.0},
-                     {3, 8, 8.0},
-                     {4, 6, 9.0},
-                     {4, 7, 10.0},
-                     {4, 8, 11.0},
-                     {5, 6, 12.0},
-                     {5, 7, sct(13.0) + FBCSR_TEST_IMAGINARY},
-                     {5, 8, sct(14.0) + FBCSR_TEST_IMAGINARY}}});
-}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::SparsityCsr<ValueType, IndexType>>
-FbcsrSample<ValueType, IndexType>::generate_sparsity_csr() const
-{
-    gko::Array<IndexType> colids(exec, nbnz);
-    gko::Array<IndexType> rowptrs(exec, nbrows + 1);
-    const std::unique_ptr<const Fbcsr> fbmat = generate_fbcsr();
-    for (index_type i = 0; i < nbrows + 1; i++)
-        rowptrs.get_data()[i] = fbmat->get_row_ptrs()[i];
-    for (index_type i = 0; i < nbnz; i++)
-        colids.get_data()[i] = fbmat->get_col_idxs()[i];
-    return SparCsr::create(exec, gko::dim<2>{nbrows, nbcols}, colids, rowptrs);
-}
-
-#define GKO_DECLARE_FBCSR_TEST_SAMPLE(ValueType, IndexType) \
-    class FbcsrSample<ValueType, IndexType>
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_TEST_SAMPLE);
-
-
-template <typename ValueType, typename IndexType>
-FbcsrSample2<ValueType, IndexType>::FbcsrSample2(
-    const std::shared_ptr<const gko::ReferenceExecutor> rexec)
-    : nrows{6},
-      ncols{8},
-      nnz{16},
-      nbrows{3},
-      nbcols{4},
-      nbnz{4},
-      bs{2},
-      exec(rexec)
-{}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
-FbcsrSample2<ValueType, IndexType>::generate_fbcsr() const
-{
-    std::unique_ptr<Fbcsr> mtx =
-        Fbcsr::create(exec,
-                      gko::dim<2>{static_cast<size_type>(nrows),
-                                  static_cast<size_type>(ncols)},
-                      nnz, bs);
-
-    value_type *const v = mtx->get_values();
-    index_type *const c = mtx->get_col_idxs();
-    index_type *const r = mtx->get_row_ptrs();
-    r[0] = 0;
-    r[1] = 1;
-    r[2] = 3;
-    r[3] = 4;
-    c[0] = 0;
-    c[1] = 0;
-    c[2] = 3;
-    c[3] = 2;
-
-    for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
-
-    v[0] = 1;
-    v[1] = 2;
-    v[2] = 3;
-    v[3] = 0;
-    v[10] = 0;
-    v[11] = 0;
-    v[12] = -12;
-    v[13] = -1;
-    v[14] = -2;
-    v[15] = -11;
-
-    return mtx;
-}
-
-template <typename ValueType, typename IndexType>
-void FbcsrSample2<ValueType, IndexType>::fill_value_blocks_view(
-    DenseBlocksView &dbv) const
-{
-    dbv(0, 0, 0) = 1.0;
-    dbv(0, 0, 1) = 2.0;
-    dbv(0, 1, 0) = 3.0;
-    dbv(0, 1, 1) = 0.0;
-    for (int i = 0; i < 2; ++i)
-        for (int j = 0; j < 2; ++j) dbv(1, i, j) = 0.15 + FBCSR_TEST_OFFSET;
-    dbv(2, 0, 0) = 0.15 + FBCSR_TEST_OFFSET;
-    dbv(2, 0, 1) = 0.15 + FBCSR_TEST_OFFSET;
-    dbv(2, 1, 0) = 0.0;
-    dbv(2, 1, 1) = 0.0;
-    dbv(3, 0, 0) = -12.0;
-    dbv(3, 0, 1) = -1.0;
-    dbv(3, 1, 0) = -2.0;
-    dbv(3, 1, 1) = -11.0;
-}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Fbcsr<remove_complex<ValueType>, IndexType>>
-FbcsrSample2<ValueType, IndexType>::generate_abs_fbcsr_abstype() const
-{
-    using AbsValueType = typename gko::remove_complex<ValueType>;
-    using AbsFbcsr = gko::matrix::Fbcsr<AbsValueType, IndexType>;
-
-    const std::unique_ptr<const Fbcsr> mat = generate_fbcsr();
-    std::unique_ptr<AbsFbcsr> amat =
-        generate_acopy_impl<Fbcsr, AbsValueType>(mat.get());
-
-    AbsValueType *const v = amat->get_values();
-
-    for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
-    v[0] = 1;
-    v[1] = 2;
-    v[2] = 3;
-    v[3] = 0;
-    v[10] = 0;
-    v[11] = 0;
-    v[12] = 12;
-    v[13] = 1;
-    v[14] = 2;
-    v[15] = 11;
-
-    return amat;
-}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
-FbcsrSample2<ValueType, IndexType>::generate_abs_fbcsr() const
-{
-    const std::unique_ptr<const Fbcsr> mat = generate_fbcsr();
-    std::unique_ptr<Fbcsr> amat =
-        generate_acopy_impl<Fbcsr, ValueType>(mat.get());
-
-    ValueType *const v = amat->get_values();
-
-    for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
-    v[0] = 1.0;
-    v[1] = 2.0;
-    v[2] = 3.0;
-    v[3] = 0.0;
-    v[10] = 0.0;
-    v[11] = 0.0;
-    v[12] = 12.0;
-    v[13] = 1.0;
-    v[14] = 2.0;
-    v[15] = 11.0;
-
-    return amat;
-}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
-FbcsrSample2<ValueType, IndexType>::generate_transpose_fbcsr() const
-{
-    std::unique_ptr<Fbcsr> mtx =
-        Fbcsr::create(exec,
-                      gko::dim<2>{static_cast<size_type>(ncols),
-                                  static_cast<size_type>(nrows)},
-                      nnz, bs);
-
-    value_type *const v = mtx->get_values();
-    index_type *const c = mtx->get_col_idxs();
-    index_type *const r = mtx->get_row_ptrs();
-    r[0] = 0;
-    r[1] = 2;
-    r[2] = 2;
-    r[3] = 3;
-    r[4] = 4;
-    c[0] = 0;
-    c[1] = 1;
-    c[2] = 2;
-    c[3] = 1;
-
-    for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
-
-    v[0] = 1;
-    v[1] = 3;
-    v[2] = 2;
-    v[3] = 0;
-    v[8] = -12;
-    v[9] = -2;
-    v[10] = -1;
-    v[11] = -11;
-    v[13] = 0;
-    v[15] = 0;
-
-    return mtx;
-}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Diagonal<ValueType>>
-FbcsrSample2<ValueType, IndexType>::extract_diagonal() const
-{
-    gko::Array<ValueType> dvals(exec, nrows);
-    ValueType *const dv = dvals.get_data();
-    dv[0] = 1;
-    dv[1] = 0;
-    dv[2] = 0;
-    dv[3] = 0;
-    dv[4] = -12;
-    dv[5] = -11;
-    return Diagonal::create(exec, nrows, dvals);
-}
-
-template <typename ValueType, typename IndexType>
-gko::Array<IndexType> FbcsrSample2<ValueType, IndexType>::getNonzerosPerRow()
-    const
-{
-    return gko::Array<index_type>(exec, {2, 2, 4, 4, 2, 2});
-}
-
-template <typename ValueType, typename IndexType>
-void FbcsrSample2<ValueType, IndexType>::apply(
-    const gko::matrix::Dense<ValueType> *const x,
-    gko::matrix::Dense<ValueType> *const y) const
-{
-    if (x->get_size()[0] != ncols)
-        throw gko::BadDimension(__FILE__, __LINE__, __func__, "spmv", nrows,
-                                ncols, "");
-    if (y->get_size()[0] != nrows)
-        throw gko::BadDimension(__FILE__, __LINE__, __func__, "spmv", nrows,
-                                ncols, "");
-    if (x->get_size()[1] != y->get_size()[1])
-        throw gko::BadDimension(__FILE__, __LINE__, __func__, "spmv", nrows,
-                                ncols, "");
-
-    const ValueType defv = sct(0.15 + FBCSR_TEST_OFFSET);
-
-    for (index_type k = 0; k < x->get_size()[1]; k++) {
-        y->at(0, k) = sct(1.0) * x->at(0, k) + sct(2.0) * x->at(1, k);
-        y->at(1, k) = sct(3.0) * x->at(0, k);
-        y->at(2, k) =
-            defv * (x->at(0, k) + x->at(1, k) + x->at(6, k) + x->at(7, k));
-        y->at(3, k) = defv * (x->at(0, k) + x->at(1, k));
-        y->at(4, k) = sct(-12.0) * x->at(4, k) - x->at(5, k);
-        y->at(5, k) = sct(-2.0) * x->at(4, k) + sct(-11.0) * x->at(5, k);
-    }
-}
-
-#define GKO_DECLARE_FBCSR_TEST_SAMPLE_2(ValueType, IndexType) \
-    class FbcsrSample2<ValueType, IndexType>
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_TEST_SAMPLE_2);
-
-
-template <typename ValueType, typename IndexType>
-FbcsrSampleSquare<ValueType, IndexType>::FbcsrSampleSquare(
-    const std::shared_ptr<const gko::ReferenceExecutor> rexec)
-    : nrows{4},
-      ncols{4},
-      nnz{8},
-      nbrows{2},
-      nbcols{2},
-      nbnz{2},
-      bs{2},
-      exec(rexec)
-{}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
-FbcsrSampleSquare<ValueType, IndexType>::generate_fbcsr() const
-{
-    std::unique_ptr<Fbcsr> mtx =
-        Fbcsr::create(exec,
-                      gko::dim<2>{static_cast<size_type>(nrows),
-                                  static_cast<size_type>(ncols)},
-                      nnz, bs);
-
-    value_type *const v = mtx->get_values();
-    index_type *const c = mtx->get_col_idxs();
-    index_type *const r = mtx->get_row_ptrs();
-    r[0] = 0;
-    r[1] = 1;
-    r[2] = 2;
-    c[0] = 1;
-    c[1] = 1;
-
-    for (IndexType i = 0; i < nnz; i++) v[i] = i;
-
-    return mtx;
-}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
-FbcsrSampleSquare<ValueType, IndexType>::generate_transpose_fbcsr() const
-{
-    std::unique_ptr<Fbcsr> mtx =
-        Fbcsr::create(exec,
-                      gko::dim<2>{static_cast<size_type>(nrows),
-                                  static_cast<size_type>(ncols)},
-                      nnz, bs);
-
-    value_type *const v = mtx->get_values();
-    index_type *const c = mtx->get_col_idxs();
-    index_type *const r = mtx->get_row_ptrs();
-    r[0] = 0;
-    r[1] = 0;
-    r[2] = 2;
-    c[0] = 0;
-    c[1] = 1;
-
-    gko::blockutils::DenseBlocksView<value_type, index_type> vals(v, bs, bs);
-    vals(0, 0, 0) = 0;
-    vals(0, 0, 1) = 2;
-    vals(0, 1, 0) = 1;
-    vals(0, 1, 1) = 3;
-    vals(1, 0, 0) = 4;
-    vals(1, 0, 1) = 6;
-    vals(1, 1, 0) = 5;
-    vals(1, 1, 1) = 7;
-
-    return mtx;
-}
-
-#define GKO_DECLARE_FBCSR_TEST_SAMPLE_SQUARE(ValueType, IndexType) \
-    class FbcsrSampleSquare<ValueType, IndexType>
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_FBCSR_TEST_SAMPLE_SQUARE);
-
-
-template <typename ValueType, typename IndexType>
-FbcsrSampleComplex<ValueType, IndexType>::FbcsrSampleComplex(
-    const std::shared_ptr<const gko::ReferenceExecutor> rexec)
-    : nrows{6},
-      ncols{8},
-      nnz{16},
-      nbrows{3},
-      nbcols{4},
-      nbnz{4},
-      bs{2},
-      exec(rexec)
-{}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
-FbcsrSampleComplex<ValueType, IndexType>::generate_fbcsr() const
-{
-    std::unique_ptr<Fbcsr> mtx =
-        Fbcsr::create(exec,
-                      gko::dim<2>{static_cast<size_type>(nrows),
-                                  static_cast<size_type>(ncols)},
-                      nnz, bs);
-
-    value_type *const v = mtx->get_values();
-    index_type *const c = mtx->get_col_idxs();
-    index_type *const r = mtx->get_row_ptrs();
-    r[0] = 0;
-    r[1] = 1;
-    r[2] = 3;
-    r[3] = 4;
-    c[0] = 0;
-    c[1] = 0;
-    c[2] = 3;
-    c[3] = 2;
-
-    for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
-
-    using namespace std::complex_literals;
-    v[0] = 1.0 + 1.15i;
-    v[1] = 2.0 + 2.15i;
-    v[2] = 3.0 - 3.15i;
-    v[3] = 0.0 - 0.15i;
-    v[10] = 0.0;
-    v[11] = 0.0;
-    v[12] = -12.0 + 12.15i;
-    v[13] = -1.0 + 1.15i;
-    v[14] = -2.0 - 2.15i;
-    v[15] = -11.0 - 11.15i;
-
-    return mtx;
-}
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<gko::matrix::Fbcsr<ValueType, IndexType>>
-FbcsrSampleComplex<ValueType, IndexType>::generate_conjtranspose_fbcsr() const
-{
-    std::unique_ptr<Fbcsr> mtx =
-        Fbcsr::create(exec,
-                      gko::dim<2>{static_cast<size_type>(ncols),
-                                  static_cast<size_type>(nrows)},
-                      nnz, bs);
-
-    value_type *const v = mtx->get_values();
-    index_type *const c = mtx->get_col_idxs();
-    index_type *const r = mtx->get_row_ptrs();
-    r[0] = 0;
-    r[1] = 2;
-    r[2] = 2;
-    r[3] = 3;
-    r[4] = 4;
-    c[0] = 0;
-    c[1] = 1;
-    c[2] = 2;
-    c[3] = 1;
-
-    for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
-
-    using namespace std::complex_literals;
-    v[0] = 1.0 - 1.15i;
-    v[1] = 3.0 + 3.15i;
-    v[2] = 2.0 - 2.15i;
-    v[3] = 0.0 + 0.15i;
-    v[8] = -12.0 - 12.15i;
-    v[9] = -2.0 + 2.15i;
-    v[10] = -1.0 - 1.15i;
-    v[11] = -11.0 + 11.15i;
-    v[13] = 0;
-    v[15] = 0;
-
-    return mtx;
-}
-
-template class FbcsrSampleComplex<std::complex<float>, int32>;
-template class FbcsrSampleComplex<std::complex<double>, int32>;
-template class FbcsrSampleComplex<std::complex<float>, int64>;
-template class FbcsrSampleComplex<std::complex<double>, int64>;
-
-}  // namespace testing
-}  // namespace gko
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 5985e192a5b..082fb5f0342 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -45,6 +45,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/fixed_block.hpp"
 
 
+#define FBCSR_TEST_OFFSET 0.000011118888
+
+#define FBCSR_TEST_C_MAG 0.1 + FBCSR_TEST_OFFSET
+
+#define FBCSR_TEST_IMAGINARY \
+    sct(std::complex<remove_complex<ValueType>>(0, FBCSR_TEST_C_MAG))
+
+
 namespace gko {
 namespace testing {
 
@@ -60,43 +68,246 @@ class FbcsrSample {
 public:
     using value_type = ValueType;
     using index_type = IndexType;
-    using absvalue_type = remove_complex<value_type>;
     using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
-    using Dense = gko::matrix::Dense<value_type>;
     using MatData = gko::matrix_data<value_type, index_type>;
     using SparCsr = gko::matrix::SparsityCsr<value_type, index_type>;
 
-    FbcsrSample(std::shared_ptr<const gko::ReferenceExecutor> exec);
+
+    const size_type nrows = 6;
+    const size_type ncols = 12;
+    const size_type nnz = 36;
+    const size_type nbrows = 2;
+    const size_type nbcols = 4;
+    const size_type nbnz = 4;
+    const int bs = 3;
+    const std::shared_ptr<const gko::Executor> exec;
+
+
+    FbcsrSample(std::shared_ptr<const gko::ReferenceExecutor> rexec)
+        : exec(rexec)
+    {}
 
     /**
      * @return The sample matrix in FBCSR format
      */
-    std::unique_ptr<Fbcsr> generate_fbcsr() const;
+    std::unique_ptr<Fbcsr> generate_fbcsr() const
+    {
+        std::unique_ptr<Fbcsr> mtx =
+            Fbcsr::create(exec,
+                          gko::dim<2>{static_cast<size_type>(nrows),
+                                      static_cast<size_type>(ncols)},
+                          nnz, bs);
+
+        value_type *const v = mtx->get_values();
+        index_type *const c = mtx->get_col_idxs();
+        index_type *const r = mtx->get_row_ptrs();
+        r[0] = 0;
+        r[1] = 2;
+        r[2] = 4;
+        c[0] = 1;
+        c[1] = 3;
+        c[2] = 0;
+        c[3] = 2;
+
+        gko::blockutils::DenseBlocksView<value_type, index_type> vals(v, bs,
+                                                                      bs);
+
+        if (mtx->get_size()[0] % bs != 0)
+            throw gko::BadDimension(__FILE__, __LINE__, __func__, "test fbcsr",
+                                    mtx->get_size()[0], mtx->get_size()[1],
+                                    "block size does not divide the size!");
+
+        for (index_type ibrow = 0; ibrow < mtx->get_num_block_rows(); ibrow++) {
+            const index_type *const browptr = mtx->get_row_ptrs();
+            for (index_type inz = browptr[ibrow]; inz < browptr[ibrow + 1];
+                 inz++) {
+                const index_type bcolind = mtx->get_col_idxs()[inz];
+                const value_type base = (ibrow + 1) * (bcolind + 1);
+                for (int ival = 0; ival < bs; ival++)
+                    for (int jval = 0; jval < bs; jval++)
+                        vals(inz, ival, jval) =
+                            base + static_cast<gko::remove_complex<value_type>>(
+                                       ival * bs + jval);
+            }
+        }
+
+        // Some of the entries are set to zero
+        vals(0, 2, 0) = gko::zero<value_type>();
+        vals(0, 2, 2) = gko::zero<value_type>();
+        vals(3, 0, 0) = gko::zero<value_type>();
+
+        v[34] += FBCSR_TEST_IMAGINARY;
+        v[35] += FBCSR_TEST_IMAGINARY;
+
+        return mtx;
+    }
 
     /**
      * @return Sample matrix in CSR format
      *
      * Keeps explicit zeros.
      */
-    std::unique_ptr<Csr> generate_csr() const;
-
-    /**
-     * @return Sample matrix as dense
-     */
-    std::unique_ptr<Dense> generate_dense() const;
+    std::unique_ptr<Csr> generate_csr() const
+    {
+        std::unique_ptr<Csr> csrm =
+            Csr::create(exec, gko::dim<2>{nrows, ncols}, nnz,
+                        std::make_shared<typename Csr::classical>());
+        index_type *const csrrow = csrm->get_row_ptrs();
+        index_type *const csrcols = csrm->get_col_idxs();
+        value_type *const csrvals = csrm->get_values();
+
+        csrrow[0] = 0;
+        csrrow[1] = 6;
+        csrrow[2] = 12;
+        csrrow[3] = 18;
+        csrrow[4] = 24;
+        csrrow[5] = 30;
+        csrrow[6] = 36;
+
+        csrcols[0] = 3;
+        csrvals[0] = 2;
+        csrcols[1] = 4;
+        csrvals[1] = 3;
+        csrcols[2] = 5;
+        csrvals[2] = 4;
+        csrcols[6] = 3;
+        csrvals[6] = 5;
+        csrcols[7] = 4;
+        csrvals[7] = 6;
+        csrcols[8] = 5;
+        csrvals[8] = 7;
+        csrcols[12] = 3;
+        csrvals[12] = 0;
+        csrcols[13] = 4;
+        csrvals[13] = 9;
+        csrcols[14] = 5;
+        csrvals[14] = 0;
+
+        csrcols[3] = 9;
+        csrvals[3] = 4;
+        csrcols[4] = 10;
+        csrvals[4] = 5;
+        csrcols[5] = 11;
+        csrvals[5] = 6;
+        csrcols[9] = 9;
+        csrvals[9] = 7;
+        csrcols[10] = 10;
+        csrvals[10] = 8;
+        csrcols[11] = 11;
+        csrvals[11] = 9;
+        csrcols[15] = 9;
+        csrvals[15] = 10;
+        csrcols[16] = 10;
+        csrvals[16] = 11;
+        csrcols[17] = 11;
+        csrvals[17] = 12;
+
+        csrcols[18] = 0;
+        csrvals[18] = 2;
+        csrcols[19] = 1;
+        csrvals[19] = 3;
+        csrcols[20] = 2;
+        csrvals[20] = 4;
+        csrcols[24] = 0;
+        csrvals[24] = 5;
+        csrcols[25] = 1;
+        csrvals[25] = 6;
+        csrcols[26] = 2;
+        csrvals[26] = 7;
+        csrcols[30] = 0;
+        csrvals[30] = 8;
+        csrcols[31] = 1;
+        csrvals[31] = 9;
+        csrcols[32] = 2;
+        csrvals[32] = 10;
+
+        csrcols[21] = 6;
+        csrvals[21] = 0;
+        csrcols[22] = 7;
+        csrvals[22] = 7;
+        csrcols[23] = 8;
+        csrvals[23] = 8;
+        csrcols[27] = 6;
+        csrvals[27] = 9;
+        csrcols[28] = 7;
+        csrvals[28] = 10;
+        csrcols[29] = 8;
+        csrvals[29] = 11;
+        csrcols[33] = 6;
+        csrvals[33] = 12;
+        csrcols[34] = 7;
+        csrvals[34] = 13;
+        csrcols[35] = 8;
+        csrvals[35] = 14;
+
+        csrvals[34] += FBCSR_TEST_IMAGINARY;
+        csrvals[35] += FBCSR_TEST_IMAGINARY;
+
+        return csrm;
+    }
 
     /**
      * @return Sparsity structure of the matrix
      */
-    std::unique_ptr<SparCsr> generate_sparsity_csr() const;
+    std::unique_ptr<SparCsr> generate_sparsity_csr() const
+    {
+        gko::Array<IndexType> colids(exec, nbnz);
+        gko::Array<IndexType> rowptrs(exec, nbrows + 1);
+        const std::unique_ptr<const Fbcsr> fbmat = generate_fbcsr();
+        for (index_type i = 0; i < nbrows + 1; i++)
+            rowptrs.get_data()[i] = fbmat->get_row_ptrs()[i];
+        for (index_type i = 0; i < nbnz; i++)
+            colids.get_data()[i] = fbmat->get_col_idxs()[i];
+        return SparCsr::create(exec, gko::dim<2>{nbrows, nbcols}, colids,
+                               rowptrs);
+    }
 
     /**
      * @return Array of COO triplets that represent the matrix
      *
      * @note The order of the triplets assumes the blocks are stored row-major
      */
-    MatData generate_matrix_data() const;
+    MatData generate_matrix_data() const
+    {
+        return MatData({{6, 12},
+                        {{0, 3, 2.0},
+                         {0, 4, 3.0},
+                         {0, 5, 4.0},
+                         {1, 3, 5.0},
+                         {1, 4, 6.0},
+                         {1, 5, 7.0},
+                         {2, 4, 9.0},
+
+                         {0, 9, 4.0},
+                         {0, 10, 5.0},
+                         {0, 11, 6.0},
+                         {1, 9, 7.0},
+                         {1, 10, 8.0},
+                         {1, 11, 9.0},
+                         {2, 9, 10.0},
+                         {2, 10, 11.0},
+                         {2, 11, 12.0},
+
+                         {3, 0, 2.0},
+                         {3, 1, 3.0},
+                         {3, 2, 4.0},
+                         {4, 0, 5.0},
+                         {4, 1, 6.0},
+                         {4, 2, 7.0},
+                         {5, 0, 8.0},
+                         {5, 1, 9.0},
+                         {5, 2, 10.0},
+
+                         {3, 7, 7.0},
+                         {3, 8, 8.0},
+                         {4, 6, 9.0},
+                         {4, 7, 10.0},
+                         {4, 8, 11.0},
+                         {5, 6, 12.0},
+                         {5, 7, sct(13.0) + FBCSR_TEST_IMAGINARY},
+                         {5, 8, sct(14.0) + FBCSR_TEST_IMAGINARY}}});
+    }
 
     /**
      * @return Array of COO triplets that represent the matrix; includes
@@ -104,34 +315,51 @@ class FbcsrSample {
      *
      * @note The order of the triplets assumes the blocks are stored row-major
      */
-    MatData generate_matrix_data_with_explicit_zeros() const;
-
-    /**
-     * @return FBCSR matrix with absolute values of respective entries
-     */
-    std::unique_ptr<Fbcsr> generate_abs_fbcsr() const;
-
-    /**
-     * @return FBCSR matrix with real scalar type,
-     *         with absolute values of respective entries
-     */
-    std::unique_ptr<gko::matrix::Fbcsr<remove_complex<value_type>, index_type>>
-    generate_abs_fbcsr_abstype() const;
-
-
-    const size_type nrows;
-    const size_type ncols;
-    const size_type nnz;
-    const size_type nbrows;
-    const size_type nbcols;
-    const size_type nbnz;
-    const int bs;
-    const std::shared_ptr<const gko::Executor> exec;
+    MatData generate_matrix_data_with_explicit_zeros() const
+    {
+        return MatData({{6, 12},
+                        {{0, 3, 2.0},
+                         {0, 4, 3.0},
+                         {0, 5, 4.0},
+                         {1, 3, 5.0},
+                         {1, 4, 6.0},
+                         {1, 5, 7.0},
+                         {2, 3, 0.0},
+                         {2, 4, 9.0},
+                         {2, 5, 0.0},
+
+                         {0, 9, 4.0},
+                         {0, 10, 5.0},
+                         {0, 11, 6.0},
+                         {1, 9, 7.0},
+                         {1, 10, 8.0},
+                         {1, 11, 9.0},
+                         {2, 9, 10.0},
+                         {2, 10, 11.0},
+                         {2, 11, 12.0},
+
+                         {3, 0, 2.0},
+                         {3, 1, 3.0},
+                         {3, 2, 4.0},
+                         {4, 0, 5.0},
+                         {4, 1, 6.0},
+                         {4, 2, 7.0},
+                         {5, 0, 8.0},
+                         {5, 1, 9.0},
+                         {5, 2, 10.0},
+
+                         {3, 6, 0.0},
+                         {3, 7, 7.0},
+                         {3, 8, 8.0},
+                         {4, 6, 9.0},
+                         {4, 7, 10.0},
+                         {4, 8, 11.0},
+                         {5, 6, 12.0},
+                         {5, 7, sct(13.0) + FBCSR_TEST_IMAGINARY},
+                         {5, 8, sct(14.0) + FBCSR_TEST_IMAGINARY}}});
+    }
 
 private:
-    template <typename FbcsrType>
-    void correct_abs_for_complex_values(FbcsrType *const mat) const;
-
     /// Enables complex data to be used for complex instantiations...
     template <typename U>
     constexpr std::enable_if_t<!is_complex<U>() || is_complex<ValueType>(),
@@ -161,49 +389,159 @@ class FbcsrSample2 {
     using value_type = ValueType;
     using index_type = IndexType;
     using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
-    using Dense = gko::matrix::Dense<value_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
     using Diagonal = gko::matrix::Diagonal<value_type>;
     using DenseBlocksView =
         gko::blockutils::DenseBlocksView<ValueType, IndexType>;
 
-    FbcsrSample2(std::shared_ptr<const gko::ReferenceExecutor> exec);
 
-    std::unique_ptr<Fbcsr> generate_fbcsr() const;
+    const size_type nrows = 6;
+    const size_type ncols = 8;
+    const size_type nnz = 16;
+    const size_type nbrows = 3;
+    const size_type nbcols = 4;
+    const size_type nbnz = 4;
+    const int bs = 2;
+    const std::shared_ptr<const gko::Executor> exec;
+
 
-    std::unique_ptr<Fbcsr> generate_transpose_fbcsr() const;
+    FbcsrSample2(std::shared_ptr<const gko::ReferenceExecutor> rexec)
+        : exec(rexec)
+    {}
 
-    std::unique_ptr<Diagonal> extract_diagonal() const;
+    std::unique_ptr<Fbcsr> generate_fbcsr() const
+    {
+        std::unique_ptr<Fbcsr> mtx =
+            Fbcsr::create(exec,
+                          gko::dim<2>{static_cast<size_type>(nrows),
+                                      static_cast<size_type>(ncols)},
+                          nnz, bs);
+
+        value_type *const v = mtx->get_values();
+        index_type *const c = mtx->get_col_idxs();
+        index_type *const r = mtx->get_row_ptrs();
+        r[0] = 0;
+        r[1] = 1;
+        r[2] = 3;
+        r[3] = 4;
+        c[0] = 0;
+        c[1] = 0;
+        c[2] = 3;
+        c[3] = 2;
+
+        for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
+
+        v[0] = 1;
+        v[1] = 2;
+        v[2] = 3;
+        v[3] = 0;
+        v[10] = 0;
+        v[11] = 0;
+        v[12] = -12;
+        v[13] = -1;
+        v[14] = -2;
+        v[15] = -11;
+
+        return mtx;
+    }
 
-    void apply(const Dense *x, Dense *y) const;
+    std::unique_ptr<Csr> generate_csr() const
+    {
+        std::unique_ptr<Csr> mtx =
+            Csr::create(exec,
+                        gko::dim<2>{static_cast<size_type>(nrows),
+                                    static_cast<size_type>(ncols)},
+                        nnz, std::make_shared<typename Csr::classical>());
+
+        value_type *const v = mtx->get_values();
+        index_type *const c = mtx->get_col_idxs();
+        index_type *const r = mtx->get_row_ptrs();
+        r[0] = 0;
+        r[1] = 2;
+        r[2] = 4;
+        r[3] = 8;
+        r[4] = 12;
+        r[5] = 14;
+        r[6] = 16;
+
+        c[0] = 0;
+        c[1] = 1;
+        c[2] = 0;
+        c[3] = 1;
+        c[4] = 0;
+        c[5] = 1;
+        c[6] = 6;
+        c[7] = 7;
+        c[8] = 0;
+        c[9] = 1;
+        c[10] = 6;
+        c[11] = 7;
+        c[12] = 4;
+        c[13] = 5;
+        c[14] = 4;
+        c[15] = 5;
+
+        for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
+
+        v[0] = 1;
+        v[1] = 2;
+        v[2] = 3;
+        v[3] = 0;
+        v[10] = 0;
+        v[11] = 0;
+        v[12] = -12;
+        v[13] = -1;
+        v[14] = -2;
+        v[15] = -11;
+
+        return mtx;
+    }
 
-    gko::Array<index_type> getNonzerosPerRow() const;
+    std::unique_ptr<Diagonal> extract_diagonal() const
+    {
+        gko::Array<ValueType> dvals(exec, nrows);
+        ValueType *const dv = dvals.get_data();
+        dv[0] = 1;
+        dv[1] = 0;
+        dv[2] = 0;
+        dv[3] = 0;
+        dv[4] = -12;
+        dv[5] = -11;
+        return Diagonal::create(exec, nrows, dvals);
+    }
 
-    std::unique_ptr<Fbcsr> generate_abs_fbcsr() const;
+    gko::Array<index_type> getNonzerosPerRow() const
+    {
+        return gko::Array<index_type>(exec, {2, 2, 4, 4, 2, 2});
+    }
 
-    std::unique_ptr<gko::matrix::Fbcsr<remove_complex<value_type>, index_type>>
-    generate_abs_fbcsr_abstype() const;
+    /// Fills a view into a FBCSR values array using the sample matrix's data
+    void fill_value_blocks_view(DenseBlocksView &dbv) const
+    {
+        dbv(0, 0, 0) = 1.0;
+        dbv(0, 0, 1) = 2.0;
+        dbv(0, 1, 0) = 3.0;
+        dbv(0, 1, 1) = 0.0;
+        for (int i = 0; i < 2; ++i)
+            for (int j = 0; j < 2; ++j) dbv(1, i, j) = 0.15 + FBCSR_TEST_OFFSET;
+        dbv(2, 0, 0) = 0.15 + FBCSR_TEST_OFFSET;
+        dbv(2, 0, 1) = 0.15 + FBCSR_TEST_OFFSET;
+        dbv(2, 1, 0) = 0.0;
+        dbv(2, 1, 1) = 0.0;
+        dbv(3, 0, 0) = -12.0;
+        dbv(3, 0, 1) = -1.0;
+        dbv(3, 1, 0) = -2.0;
+        dbv(3, 1, 1) = -11.0;
+    }
 
-    /**
-     * Fills a view into a FBCSR values array using the sample matrix's data
-     */
-    void fill_value_blocks_view(DenseBlocksView &dbv) const;
 
+private:
     /// Enables use of literals to instantiate value data
     template <typename U>
     constexpr ValueType sct(U u) const
     {
         return static_cast<ValueType>(u);
     }
-
-
-    const size_type nrows;
-    const size_type ncols;
-    const size_type nnz;
-    const size_type nbrows;
-    const size_type nbcols;
-    const size_type nbnz;
-    const int bs;
-    const std::shared_ptr<const gko::Executor> exec;
 };
 
 /**
@@ -218,20 +556,42 @@ class FbcsrSampleSquare {
     using index_type = IndexType;
     using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
 
-    FbcsrSampleSquare(std::shared_ptr<const gko::ReferenceExecutor> exec);
 
-    std::unique_ptr<Fbcsr> generate_fbcsr() const;
+    const size_type nrows = 4;
+    const size_type ncols = 4;
+    const size_type nnz = 8;
+    const size_type nbrows = 2;
+    const size_type nbcols = 2;
+    const size_type nbnz = 2;
+    const int bs = 2;
+    const std::shared_ptr<const gko::Executor> exec;
 
-    std::unique_ptr<Fbcsr> generate_transpose_fbcsr() const;
 
-    const size_type nrows;
-    const size_type ncols;
-    const size_type nnz;
-    const size_type nbrows;
-    const size_type nbcols;
-    const size_type nbnz;
-    const int bs;
-    const std::shared_ptr<const gko::Executor> exec;
+    FbcsrSampleSquare(std::shared_ptr<const gko::ReferenceExecutor> rexec)
+        : exec(rexec)
+    {}
+
+    std::unique_ptr<Fbcsr> generate_fbcsr() const
+    {
+        std::unique_ptr<Fbcsr> mtx =
+            Fbcsr::create(exec,
+                          gko::dim<2>{static_cast<size_type>(nrows),
+                                      static_cast<size_type>(ncols)},
+                          nnz, bs);
+
+        value_type *const v = mtx->get_values();
+        index_type *const c = mtx->get_col_idxs();
+        index_type *const r = mtx->get_row_ptrs();
+        r[0] = 0;
+        r[1] = 1;
+        r[2] = 2;
+        c[0] = 1;
+        c[1] = 1;
+
+        for (IndexType i = 0; i < nnz; i++) v[i] = i;
+
+        return mtx;
+    }
 };
 
 /**
@@ -244,25 +604,116 @@ class FbcsrSampleComplex {
 public:
     using value_type = ValueType;
     using index_type = IndexType;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
     using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
 
+
     static_assert(is_complex<ValueType>(), "Only for complex types!");
 
-    FbcsrSampleComplex(std::shared_ptr<const gko::ReferenceExecutor> exec);
 
-    std::unique_ptr<Fbcsr> generate_fbcsr() const;
+    const size_type nrows = 6;
+    const size_type ncols = 8;
+    const size_type nnz = 16;
+    const size_type nbrows = 3;
+    const size_type nbcols = 4;
+    const size_type nbnz = 4;
+    const int bs = 2;
+    const std::shared_ptr<const gko::Executor> exec;
 
-    std::unique_ptr<Fbcsr> generate_conjtranspose_fbcsr() const;
 
+    FbcsrSampleComplex(std::shared_ptr<const gko::ReferenceExecutor> rexec)
+        : exec(rexec)
+    {}
 
-    const size_type nrows;
-    const size_type ncols;
-    const size_type nnz;
-    const size_type nbrows;
-    const size_type nbcols;
-    const size_type nbnz;
-    const int bs;
-    const std::shared_ptr<const gko::Executor> exec;
+    std::unique_ptr<Fbcsr> generate_fbcsr() const
+    {
+        std::unique_ptr<Fbcsr> mtx =
+            Fbcsr::create(exec,
+                          gko::dim<2>{static_cast<size_type>(nrows),
+                                      static_cast<size_type>(ncols)},
+                          nnz, bs);
+
+        value_type *const v = mtx->get_values();
+        index_type *const c = mtx->get_col_idxs();
+        index_type *const r = mtx->get_row_ptrs();
+        r[0] = 0;
+        r[1] = 1;
+        r[2] = 3;
+        r[3] = 4;
+        c[0] = 0;
+        c[1] = 0;
+        c[2] = 3;
+        c[3] = 2;
+
+        for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
+
+        using namespace std::complex_literals;
+        v[0] = 1.0 + 1.15i;
+        v[1] = 2.0 + 2.15i;
+        v[2] = 3.0 - 3.15i;
+        v[3] = 0.0 - 0.15i;
+        v[10] = 0.0;
+        v[11] = 0.0;
+        v[12] = -12.0 + 12.15i;
+        v[13] = -1.0 + 1.15i;
+        v[14] = -2.0 - 2.15i;
+        v[15] = -11.0 - 11.15i;
+
+        return mtx;
+    }
+
+    std::unique_ptr<Csr> generate_csr() const
+    {
+        std::unique_ptr<Csr> mtx =
+            Csr::create(exec,
+                        gko::dim<2>{static_cast<size_type>(nrows),
+                                    static_cast<size_type>(ncols)},
+                        nnz, std::make_shared<typename Csr::classical>());
+
+        value_type *const v = mtx->get_values();
+        index_type *const c = mtx->get_col_idxs();
+        index_type *const r = mtx->get_row_ptrs();
+        r[0] = 0;
+        r[1] = 2;
+        r[2] = 4;
+        r[3] = 8;
+        r[4] = 12;
+        r[5] = 14;
+        r[6] = 16;
+
+        c[0] = 0;
+        c[1] = 1;
+        c[2] = 0;
+        c[3] = 1;
+        c[4] = 0;
+        c[5] = 1;
+        c[6] = 6;
+        c[7] = 7;
+        c[8] = 0;
+        c[9] = 1;
+        c[10] = 6;
+        c[11] = 7;
+        c[12] = 4;
+        c[13] = 5;
+        c[14] = 4;
+        c[15] = 5;
+
+        for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
+
+        using namespace std::complex_literals;
+        v[0] = 1.0 + 1.15i;
+        v[1] = 2.0 + 2.15i;
+        v[2] = 3.0 - 3.15i;
+        v[3] = 0.0 - 0.15i;
+        v[10] = 0.0;
+        v[11] = 0.0;
+        v[12] = -12.0 + 12.15i;
+        v[13] = -1.0 + 1.15i;
+        v[14] = -2.0 - 2.15i;
+        v[15] = -11.0 - 11.15i;
+
+        return mtx;
+    }
 };
 
 }  // namespace testing
diff --git a/reference/test/matrix/CMakeLists.txt b/reference/test/matrix/CMakeLists.txt
index cb4d2b0217a..c826a9eaca7 100644
--- a/reference/test/matrix/CMakeLists.txt
+++ b/reference/test/matrix/CMakeLists.txt
@@ -3,7 +3,7 @@ ginkgo_create_test(csr_kernels)
 ginkgo_create_test(dense_kernels)
 ginkgo_create_test(diagonal_kernels)
 ginkgo_create_test(ell_kernels)
-ginkgo_create_test(fbcsr_kernels test_fbcsr_sample)
+ginkgo_create_test(fbcsr_kernels)
 ginkgo_create_test(hybrid_kernels)
 ginkgo_create_test(identity)
 ginkgo_create_test(permutation)
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 1a7e00d3f40..92549484030 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -50,6 +50,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
+#include "core/matrix/csr_kernels.hpp"
 #include "core/matrix/fbcsr_kernels.hpp"
 #include "core/test/matrix/fbcsr_sample.hpp"
 #include "core/test/utils.hpp"
@@ -79,8 +80,9 @@ class Fbcsr : public ::testing::Test {
           fbsamplesquare(exec),
           mtx(fbsample.generate_fbcsr()),
           refmtx(fbsample.generate_fbcsr()),
+          ref2mtx(fbsample2.generate_fbcsr()),
           refcsrmtx(fbsample.generate_csr()),
-          refdenmtx(fbsample.generate_dense()),
+          ref2csrmtx(fbsample2.generate_csr()),
           refspcmtx(fbsample.generate_sparsity_csr()),
           mtx2(fbsample2.generate_fbcsr()),
           m2diag(fbsample2.extract_diagonal()),
@@ -123,7 +125,9 @@ class Fbcsr : public ::testing::Test {
         fbsamplesquare;
     std::unique_ptr<Mtx> mtx;
     const std::unique_ptr<const Mtx> refmtx;
+    const std::unique_ptr<const Mtx> ref2mtx;
     const std::unique_ptr<const Csr> refcsrmtx;
+    const std::unique_ptr<const Csr> ref2csrmtx;
     const std::unique_ptr<const Dense> refdenmtx;
     const std::unique_ptr<const SparCsr> refspcmtx;
     const std::unique_ptr<const Mtx> mtx2;
@@ -167,12 +171,11 @@ TYPED_TEST(Fbcsr, AppliesToDenseVector)
 
     this->mtx2->apply(x.get(), y.get());
 
-    this->fbsample2.apply(x.get(), yref.get());
-    // using Csr = typename TestFixture::Csr;
-    // auto csr_mtx = Csr::create(this->mtx->get_executor(),
-    //                            std::make_shared<typename Csr::classical>());
-    // this->mtx2->convert_to(csr_mtx.get());
-    // csr_mtx->apply(x.get(), yref.get());
+    using Csr = typename TestFixture::Csr;
+    auto csr_mtx = Csr::create(this->mtx->get_executor(),
+                               std::make_shared<typename Csr::classical>());
+    this->mtx2->convert_to(csr_mtx.get());
+    csr_mtx->apply(x.get(), yref.get());
 
     const double tolerance =
         std::numeric_limits<gko::remove_complex<T>>::epsilon();
@@ -200,7 +203,7 @@ TYPED_TEST(Fbcsr, AppliesToDenseMatrix)
 
     this->mtx2->apply(x.get(), y.get());
 
-    this->fbsample2.apply(x.get(), yref.get());
+    this->ref2csrmtx->apply(x.get(), yref.get());
 
     const double tolerance =
         std::numeric_limits<gko::remove_complex<T>>::epsilon();
@@ -225,7 +228,6 @@ TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseVector)
     auto y = Vec::create(this->exec, gko::dim<2>{nrows, 1});
 
     for (index_type i = 0; i < ncols; i++) {
-        // xvals[i] = std::log(static_cast<T>(static_cast<float>((i+1)^2)));
         x->at(i, 0) = (i + 1.0) * (i + 1.0);
     }
     for (index_type i = 0; i < nrows; i++) {
@@ -233,16 +235,11 @@ TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseVector)
                       get_some_number<T>();
     }
 
-    auto yref = Vec::create(this->exec, gko::dim<2>{nrows, 1});
-    yref = y->clone();
+    auto yref = y->clone();
 
     this->mtx2->apply(alpha.get(), x.get(), beta.get(), y.get());
 
-    auto prod = Vec::create(this->exec, gko::dim<2>{nrows, 1});
-    this->fbsample2.apply(x.get(), prod.get());
-
-    yref->scale(beta.get());
-    yref->add_scaled(alpha.get(), prod.get());
+    this->ref2csrmtx->apply(alpha.get(), x.get(), beta.get(), yref.get());
 
     const double tolerance =
         std::numeric_limits<gko::remove_complex<T>>::epsilon();
@@ -269,8 +266,8 @@ TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseMatrix)
 
     for (index_type i = 0; i < ncols; i++)
         for (index_type j = 0; j < nvecs; j++) {
-            // xvals[i] = std::log(static_cast<T>(static_cast<float>((i+1)^2)));
-            x->at(i, j) = (i + 1.0) / (j + 1.0);
+            x->at(i, j) =
+                std::log(static_cast<T>(0.1 + static_cast<float>((i + 1) ^ 2)));
         }
     for (index_type i = 0; i < nrows; i++)
         for (index_type j = 0; j < nvecs; j++) {
@@ -279,16 +276,11 @@ TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseMatrix)
                 get_some_number<T>();
         }
 
-    auto yref = Vec::create(this->exec, gko::dim<2>{nrows, nvecs});
-    yref = y->clone();
+    auto yref = y->clone();
 
     this->mtx2->apply(alpha.get(), x.get(), beta.get(), y.get());
 
-    auto prod = Vec::create(this->exec, gko::dim<2>{nrows, nvecs});
-    this->fbsample2.apply(x.get(), prod.get());
-
-    yref->scale(beta.get());
-    yref->add_scaled(alpha.get(), prod.get());
+    this->ref2csrmtx->apply(alpha.get(), x.get(), beta.get(), yref.get());
 
     const double tolerance =
         std::numeric_limits<gko::remove_complex<T>>::epsilon();
@@ -375,7 +367,9 @@ TYPED_TEST(Fbcsr, ConvertsToDense)
 
     this->mtx->convert_to(dense_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(dense_mtx, this->refdenmtx, 0.0);
+    auto refdenmtx = Dense::create(this->mtx->get_executor());
+    this->refcsrmtx->convert_to(refdenmtx.get());
+    GKO_ASSERT_MTX_NEAR(dense_mtx, refdenmtx, 0.0);
 }
 
 
@@ -386,7 +380,9 @@ TYPED_TEST(Fbcsr, MovesToDense)
 
     this->mtx->move_to(dense_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(dense_mtx, this->refdenmtx, 0.0);
+    auto refdenmtx = Dense::create(this->mtx->get_executor());
+    this->refcsrmtx->convert_to(refdenmtx.get());
+    GKO_ASSERT_MTX_NEAR(dense_mtx, refdenmtx, 0.0);
 }
 
 
@@ -395,10 +391,13 @@ TYPED_TEST(Fbcsr, ConvertsToCsr)
     using Csr = typename TestFixture::Csr;
     auto csr_mtx = Csr::create(this->mtx->get_executor(),
                                std::make_shared<typename Csr::classical>());
-
     this->mtx->convert_to(csr_mtx.get());
-
     this->assert_equal_to_mtx(csr_mtx.get());
+
+    auto csr_mtx_2 = Csr::create(this->mtx->get_executor(),
+                                 std::make_shared<typename Csr::classical>());
+    this->ref2mtx->convert_to(csr_mtx_2.get());
+    GKO_ASSERT_MTX_NEAR(csr_mtx_2, this->ref2csrmtx, 0.0);
 }
 
 
@@ -546,9 +545,14 @@ TYPED_TEST(Fbcsr, CalculatesNonzerosPerRow)
     gko::kernels::reference::fbcsr::calculate_nonzeros_per_row(
         this->exec, this->mtx2.get(), &row_nnz);
 
-    auto row_nnz_val = row_nnz.get_data();
-    const gko::Array<IndexType> refrnnz = this->fbsample2.getNonzerosPerRow();
+    // const gko::Array<IndexType> refrnnz =
+    // this->fbsample2.getNonzerosPerRow();
+    gko::Array<gko::size_type> refrnnz(this->exec, this->mtx2->get_size()[0]);
+    gko::kernels::reference::csr ::calculate_nonzeros_per_row(
+        this->exec, this->ref2csrmtx.get(), &refrnnz);
+
     ASSERT_EQ(row_nnz.get_num_elems(), refrnnz.get_num_elems());
+    auto row_nnz_val = row_nnz.get_data();
     for (gko::size_type i = 0; i < this->mtx2->get_size()[0]; i++)
         ASSERT_EQ(row_nnz_val[i], refrnnz.get_const_data()[i]);
 }
@@ -557,24 +561,36 @@ TYPED_TEST(Fbcsr, CalculatesNonzerosPerRow)
 TYPED_TEST(Fbcsr, SquareMtxIsTransposable)
 {
     using Fbcsr = typename TestFixture::Mtx;
-    auto reftmtx = this->fbsamplesquare.generate_transpose_fbcsr();
+    using Csr = typename TestFixture::Csr;
+
+    auto csrmtxsq =
+        Csr::create(this->exec, std::make_shared<typename Csr::classical>());
+    this->mtxsq->convert_to(csrmtxsq.get());
+    std::unique_ptr<const gko::LinOp> reftmtx = csrmtxsq->transpose();
+    auto reftmtx_as_csr = static_cast<const Csr *>(reftmtx.get());
 
     auto trans = this->mtxsq->transpose();
     auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
 
-    GKO_ASSERT_MTX_NEAR(trans_as_fbcsr, reftmtx, 0.0);
+    GKO_ASSERT_MTX_NEAR(trans_as_fbcsr, reftmtx_as_csr, 0.0);
 }
 
 
 TYPED_TEST(Fbcsr, NonSquareMtxIsTransposable)
 {
     using Fbcsr = typename TestFixture::Mtx;
-    auto reftmtx = this->fbsample2.generate_transpose_fbcsr();
+    using Csr = typename TestFixture::Csr;
+
+    auto csrmtx =
+        Csr::create(this->exec, std::make_shared<typename Csr::classical>());
+    this->mtx2->convert_to(csrmtx.get());
+    std::unique_ptr<gko::LinOp> reftmtx = csrmtx->transpose();
+    auto reftmtx_as_csr = static_cast<Csr *>(reftmtx.get());
 
     auto trans = this->mtx2->transpose();
     auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
 
-    GKO_ASSERT_MTX_NEAR(trans_as_fbcsr, reftmtx, 0.0);
+    GKO_ASSERT_MTX_NEAR(trans_as_fbcsr, reftmtx_as_csr, 0.0);
 }
 
 
@@ -612,13 +628,13 @@ TYPED_TEST(Fbcsr, ExtractsDiagonal)
 TYPED_TEST(Fbcsr, InplaceAbsolute)
 {
     using Mtx = typename TestFixture::Mtx;
+    using Csr = typename TestFixture::Csr;
     auto mtx = this->fbsample2.generate_fbcsr();
-    const std::unique_ptr<const Mtx> refabs =
-        this->fbsample2.generate_abs_fbcsr();
+    const std::unique_ptr<Csr> refabs = this->ref2csrmtx->clone();
+    refabs->compute_absolute_inplace();
 
     using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
-    const value_type *const refvals = refabs->get_const_values();
 
     mtx->compute_absolute_inplace();
 
@@ -632,11 +648,12 @@ TYPED_TEST(Fbcsr, OutplaceAbsolute)
 {
     using value_type = typename TestFixture::value_type;
     using Mtx = typename TestFixture::Mtx;
+    using AbsCsr = typename gko::remove_complex<typename TestFixture::Csr>;
     using AbsMtx = typename gko::remove_complex<typename TestFixture::Mtx>;
 
     auto mtx = this->fbsample2.generate_fbcsr();
-    const std::unique_ptr<const AbsMtx> refabs =
-        this->fbsample2.generate_abs_fbcsr_abstype();
+    const std::unique_ptr<const AbsCsr> refabs =
+        this->ref2csrmtx->compute_absolute();
 
     auto abs_mtx = mtx->compute_absolute();
 
@@ -654,13 +671,15 @@ class FbcsrComplex : public ::testing::Test {
     using index_type =
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using Mtx = gko::matrix::Fbcsr<value_type, index_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
 };
 
 TYPED_TEST_SUITE(FbcsrComplex, gko::test::ComplexValueIndexTypes);
 
 
-TYPED_TEST(FbcsrComplex, MtxIsConjugateTransposable)
+TYPED_TEST(FbcsrComplex, ConvertsComplexToCsr)
 {
+    using Csr = typename TestFixture::Csr;
     using Fbcsr = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
@@ -668,9 +687,27 @@ TYPED_TEST(FbcsrComplex, MtxIsConjugateTransposable)
     auto exec = gko::ReferenceExecutor::create();
     gko::testing::FbcsrSampleComplex<T, index_type> csample(exec);
     std::unique_ptr<const Fbcsr> mtx = csample.generate_fbcsr();
-    std::unique_ptr<const Fbcsr> reftrans =
-        csample.generate_conjtranspose_fbcsr();
+    auto csr_mtx =
+        Csr::create(exec, std::make_shared<typename Csr::classical>());
+    mtx->convert_to(csr_mtx.get());
+    GKO_ASSERT_MTX_NEAR(csr_mtx, mtx, 0.0);
+}
+
+
+TYPED_TEST(FbcsrComplex, MtxIsConjugateTransposable)
+{
+    using Csr = typename TestFixture::Csr;
+    using Fbcsr = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
 
+    auto exec = gko::ReferenceExecutor::create();
+    gko::testing::FbcsrSampleComplex<T, index_type> csample(exec);
+    auto csrmtx = csample.generate_csr();
+    auto reftranslinop = csrmtx->conj_transpose();
+    const Csr *const reftrans = static_cast<const Csr *>(reftranslinop.get());
+
+    std::unique_ptr<const Fbcsr> mtx = csample.generate_fbcsr();
     auto trans = mtx->conj_transpose();
     auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
 
@@ -680,27 +717,23 @@ TYPED_TEST(FbcsrComplex, MtxIsConjugateTransposable)
 
 TYPED_TEST(FbcsrComplex, InplaceAbsolute)
 {
+    using Csr = typename TestFixture::Csr;
     using Mtx = typename TestFixture::Mtx;
     using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
     gko::testing::FbcsrSample<value_type, index_type> fbsample(
         gko::ReferenceExecutor::create());
     auto mtx = fbsample.generate_fbcsr();
-
-    std::cout << " Generated fbcsr: " << mtx->get_values()[34] << ", "
-              << mtx->get_values()[35] << std::endl;
-
-    const std::unique_ptr<const Mtx> refabs = fbsample.generate_abs_fbcsr();
+    auto csrmtx = fbsample.generate_csr();
 
     using value_type = typename TestFixture::value_type;
-    using index_type = typename TestFixture::index_type;
-    const value_type *const refvals = refabs->get_const_values();
 
     mtx->compute_absolute_inplace();
+    csrmtx->compute_absolute_inplace();
 
     const gko::remove_complex<value_type> tolerance =
         std::numeric_limits<gko::remove_complex<value_type>>::epsilon();
-    GKO_ASSERT_MTX_NEAR(mtx, refabs, tolerance);
+    GKO_ASSERT_MTX_NEAR(mtx, csrmtx, tolerance);
 }
 
 
@@ -715,10 +748,10 @@ TYPED_TEST(FbcsrComplex, OutplaceAbsolute)
         gko::ReferenceExecutor::create());
 
     auto mtx = fbsample.generate_fbcsr();
-    const std::unique_ptr<const AbsMtx> refabs =
-        fbsample.generate_abs_fbcsr_abstype();
+    auto csrmtx = fbsample.generate_csr();
 
     auto abs_mtx = mtx->compute_absolute();
+    auto refabs = mtx->compute_absolute();
 
     const gko::remove_complex<value_type> tolerance =
         std::numeric_limits<gko::remove_complex<value_type>>::epsilon();

From 7e77653f059279e5d35f0898a15755a124e85113 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Sat, 19 Dec 2020 00:04:33 +0100
Subject: [PATCH 0561/1008] moved fbcsr constructor to header file to fix MSVC
 linker errors

Why do we bother with Windoze builds again?
---
 core/matrix/fbcsr.cpp                | 15 ---------------
 include/ginkgo/core/matrix/fbcsr.hpp | 10 +++++++++-
 2 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 88b17928d06..1075a5396a1 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -146,21 +146,6 @@ class DenseBlock final {
 }  // namespace detail
 
 
-template <typename ValueType, typename IndexType>
-Fbcsr<ValueType, IndexType>::Fbcsr(const std::shared_ptr<const Executor> exec,
-                                   const dim<2> &size,
-                                   const size_type num_nonzeros,
-                                   const int block_size)
-    : EnableLinOp<Fbcsr>(exec, size),
-      bs_{block_size},
-      nbcols_{blockutils::getNumBlocks(block_size, size[1])},
-      values_(exec, num_nonzeros),
-      col_idxs_(exec, blockutils::getNumBlocks(block_size * block_size,
-                                               num_nonzeros)),
-      row_ptrs_(exec, blockutils::getNumBlocks(block_size, size[0]) + 1)
-{}
-
-
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const b,
                                              LinOp *const x) const
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index 925e7268749..a0b0d5e6525 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -332,7 +332,15 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
      * @param block_size Size of the small dense square blocks
      */
     Fbcsr(std::shared_ptr<const Executor> exec, const dim<2> &size,
-          size_type num_nonzeros, int block_size);
+          size_type num_nonzeros, int block_size)
+        : EnableLinOp<Fbcsr>(exec, size),
+          bs_{block_size},
+          nbcols_{blockutils::getNumBlocks(block_size, size[1])},
+          values_(exec, num_nonzeros),
+          col_idxs_(exec, blockutils::getNumBlocks(block_size * block_size,
+                                                   num_nonzeros)),
+          row_ptrs_(exec, blockutils::getNumBlocks(block_size, size[0]) + 1)
+    {}
 
     /**
      * Creates a FBCSR matrix from already allocated (and initialized) row

From 192165c2d830510eb244be9e70fc9d8e3bcd7a72 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Mon, 4 Jan 2021 13:07:28 +0100
Subject: [PATCH 0562/1008] fixed a bug in fbcsr copy, improved tests

---
 core/matrix/fbcsr.cpp      | 2 ++
 core/test/matrix/fbcsr.cpp | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 1075a5396a1..455cc88fee8 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -194,6 +194,7 @@ void Fbcsr<ValueType, IndexType>::convert_to(
     result->row_ptrs_ = this->row_ptrs_;
     result->set_size(this->get_size());
     result->bs_ = this->bs_;
+    result->nbcols_ = this->nbcols_;
 }
 
 
@@ -214,6 +215,7 @@ void Fbcsr<ValueType, IndexType>::convert_to(
     result->row_ptrs_ = this->row_ptrs_;
     result->set_size(this->get_size());
     result->bs_ = this->bs_;
+    result->nbcols_ = this->nbcols_;
 }
 
 
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index 619315e5c98..236105cfb48 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -97,6 +97,8 @@ class Fbcsr : public ::testing::Test {
         ASSERT_EQ(m->get_size(), orig_size);
         ASSERT_EQ(m->get_num_stored_elements(), orig_vals.size());
         ASSERT_EQ(m->get_block_size(), bs);
+        ASSERT_EQ(m->get_num_block_rows(), m->get_size()[0] / bs);
+        ASSERT_EQ(m->get_num_block_cols(), m->get_size()[1] / bs);
 
 
         for (index_type irow = 0; irow < orig_size[0] / bs; irow++) {
@@ -184,6 +186,8 @@ TYPED_TEST(Fbcsr, KnowsItsSize)
     ASSERT_EQ(this->mtx->get_size(), gko::dim<2>(6, 12));
     ASSERT_EQ(this->mtx->get_block_size(), 3);
     ASSERT_EQ(this->mtx->get_num_stored_elements(), 36);
+    ASSERT_EQ(this->mtx->get_num_block_rows(), 2);
+    ASSERT_EQ(this->mtx->get_num_block_cols(), 4);
 }
 
 
@@ -278,6 +282,7 @@ TYPED_TEST(Fbcsr, CanBeReadFromMatrixData)
     using Mtx = typename TestFixture::Mtx;
     auto m = Mtx::create(this->exec);
     m->set_block_size(this->fbsample.bs);
+    ASSERT_EQ(m->get_block_size(), this->fbsample.bs);
 
     m->read(this->fbsample.generate_matrix_data());
 

From ab7e6d64c1df838bc1491377d0877093342765dc Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 13 Jan 2021 11:06:13 +0100
Subject: [PATCH 0563/1008] updated year in headers

---
 core/components/fixed_block.hpp         | 2 +-
 core/matrix/fbcsr.cpp                   | 2 +-
 core/matrix/fbcsr_builder.hpp           | 2 +-
 core/matrix/fbcsr_kernels.hpp           | 2 +-
 core/test/matrix/fbcsr.cpp              | 2 +-
 core/test/matrix/fbcsr_builder.cpp      | 2 +-
 core/test/matrix/fbcsr_sample.hpp       | 2 +-
 cuda/matrix/fbcsr_kernels.cu            | 2 +-
 dpcpp/matrix/fbcsr_kernels.dp.cpp       | 2 +-
 hip/matrix/fbcsr_kernels.hip.cpp        | 2 +-
 include/ginkgo/core/base/blockutils.hpp | 2 +-
 include/ginkgo/core/matrix/fbcsr.hpp    | 2 +-
 omp/matrix/fbcsr_kernels.cpp            | 2 +-
 reference/matrix/fbcsr_kernels.cpp      | 2 +-
 reference/test/matrix/fbcsr_kernels.cpp | 2 +-
 15 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
index 2e9595cdc04..3059ee11bbb 100644
--- a/core/components/fixed_block.hpp
+++ b/core/components/fixed_block.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 455cc88fee8..a0298f35cf7 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/fbcsr_builder.hpp b/core/matrix/fbcsr_builder.hpp
index e80d76b3eeb..22d0bcfacb9 100644
--- a/core/matrix/fbcsr_builder.hpp
+++ b/core/matrix/fbcsr_builder.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/matrix/fbcsr_kernels.hpp b/core/matrix/fbcsr_kernels.hpp
index af7e527a891..332292eb314 100644
--- a/core/matrix/fbcsr_kernels.hpp
+++ b/core/matrix/fbcsr_kernels.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index 236105cfb48..34dce1af828 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/fbcsr_builder.cpp b/core/test/matrix/fbcsr_builder.cpp
index 684aa2e324e..3a4bf358d51 100644
--- a/core/test/matrix/fbcsr_builder.cpp
+++ b/core/test/matrix/fbcsr_builder.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 082fb5f0342..52f8e66acd5 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/cuda/matrix/fbcsr_kernels.cu b/cuda/matrix/fbcsr_kernels.cu
index b8d397ef5fc..b4da9345e48 100644
--- a/cuda/matrix/fbcsr_kernels.cu
+++ b/cuda/matrix/fbcsr_kernels.cu
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/matrix/fbcsr_kernels.dp.cpp b/dpcpp/matrix/fbcsr_kernels.dp.cpp
index 72a708bc5ff..cebba0ed528 100644
--- a/dpcpp/matrix/fbcsr_kernels.dp.cpp
+++ b/dpcpp/matrix/fbcsr_kernels.dp.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/matrix/fbcsr_kernels.hip.cpp b/hip/matrix/fbcsr_kernels.hip.cpp
index ed2eb2667c1..fd30554a469 100644
--- a/hip/matrix/fbcsr_kernels.hip.cpp
+++ b/hip/matrix/fbcsr_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/base/blockutils.hpp b/include/ginkgo/core/base/blockutils.hpp
index f3664e1d092..0638390ad1a 100644
--- a/include/ginkgo/core/base/blockutils.hpp
+++ b/include/ginkgo/core/base/blockutils.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index a0b0d5e6525..7b3cc390769 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/matrix/fbcsr_kernels.cpp b/omp/matrix/fbcsr_kernels.cpp
index 442d6f7c107..15f63ee9794 100644
--- a/omp/matrix/fbcsr_kernels.cpp
+++ b/omp/matrix/fbcsr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index c3532238db6..108697537df 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 92549484030..11c5d1bfa7a 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

From 827bcb483057f7f63520f96673f98d4a4300fb7a Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 13 Jan 2021 11:23:50 +0100
Subject: [PATCH 0564/1008] review comments: removed FixedBlock for now,
 renamed getNumBlocks to get_num_blocks, fixed fbcsr doc block

---
 core/components/fixed_block.hpp         | 46 -------------------------
 core/matrix/fbcsr.cpp                   |  8 ++---
 include/ginkgo/core/base/blockutils.hpp |  2 +-
 include/ginkgo/core/matrix/fbcsr.hpp    | 23 ++++---------
 4 files changed, 12 insertions(+), 67 deletions(-)

diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
index 3059ee11bbb..32e53510f74 100644
--- a/core/components/fixed_block.hpp
+++ b/core/components/fixed_block.hpp
@@ -44,52 +44,6 @@ namespace gko {
 namespace blockutils {
 
 
-/**
- * @brief A dense block of values with compile-time constant dimensions
- *
- * The blocks are interpreted as row-major. However, in the future,
- *  a layout template parameter can be added if needed.
- *
- * The primary use is to reinterpret subsets of entries in a big array as
- *  small dense blocks.
- *
- * @tparam ValueType  The numeric type of entries of the block
- * @tparam nrows  Number of rows
- * @tparam ncols  Number of columns
- */
-template <typename ValueType, int nrows, int ncols>
-class FixedBlock final {
-    static_assert(nrows > 0, "Requires positive number of rows!");
-    static_assert(ncols > 0, "Requires positive number of columns!");
-
-public:
-    using value_type = ValueType;
-
-    value_type &at(const int row, const int col)
-    {
-        return vals[row * ncols + col];
-    }
-
-    const value_type &at(const int row, const int col) const
-    {
-        return vals[row * ncols + col];
-    }
-
-    value_type &operator()(const int row, const int col)
-    {
-        return at(row, col);
-    }
-
-    const value_type &operator()(const int row, const int col) const
-    {
-        return at(row, col);
-    }
-
-private:
-    ValueType vals[nrows * ncols];
-};
-
-
 /**
  * @brief A view into a an array of dense blocks of some runtime-defined size
  *
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index a0298f35cf7..d709639a4da 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -270,13 +270,13 @@ template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(
     SparsityCsr<ValueType, IndexType> *const result) const
 {
-    using blockutils::getNumBlocks;
+    using blockutils::get_num_blocks;
     auto exec = this->get_executor();
     auto tmp = SparsityCsr<ValueType, IndexType>::create(
         exec,
-        gko::dim<2>{getNumBlocks(bs_, this->get_size()[0]),
-                    getNumBlocks(bs_, this->get_size()[1])},
-        getNumBlocks(bs_ * bs_, this->get_num_stored_elements()));
+        gko::dim<2>{get_num_blocks(bs_, this->get_size()[0]),
+                    get_num_blocks(bs_, this->get_size()[1])},
+        get_num_blocks(bs_ * bs_, this->get_num_stored_elements()));
 
     tmp->col_idxs_ = this->col_idxs_;
     tmp->row_ptrs_ = this->row_ptrs_;
diff --git a/include/ginkgo/core/base/blockutils.hpp b/include/ginkgo/core/base/blockutils.hpp
index 0638390ad1a..c0b16ef6ac1 100644
--- a/include/ginkgo/core/base/blockutils.hpp
+++ b/include/ginkgo/core/base/blockutils.hpp
@@ -63,7 +63,7 @@ class BlockSizeError : public Error {
  *         but throws when they don't divide
  */
 template <typename IndexType>
-IndexType getNumBlocks(const int block_size, const IndexType size)
+IndexType get_num_blocks(const int block_size, const IndexType size)
 {
     if (size % block_size != 0)
         throw BlockSizeError<IndexType>(__FILE__, __LINE__, block_size, size);
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index 7b3cc390769..6eb4b061724 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -61,6 +61,8 @@ class FbcsrBuilder;
 
 
 /**
+ * @brief Fixed-block compressed sparse row storage matrix format
+ *
  * FBCSR is a matrix format meant for matrices having a natural block structure
  * made up of small, dense, disjoint blocks. It is similar to CSR \sa Csr.
  * However, unlike Csr, each non-zero location stores a small dense block of
@@ -85,22 +87,11 @@ class FbcsrBuilder;
  * matrix::Fbcsr *A, *B, *C;      // matrices
  * matrix::Dense *b, *x;        // vectors tall-and-skinny matrices
  * matrix::Dense *alpha, *beta; // scalars of dimension 1x1
- * matrix::Identity *I;         // identity matrix
  *
  * // Applying to Dense matrices computes an SpMV/SpMM product
  * A->apply(b, x)              // x = A*b
  * A->apply(alpha, b, beta, x) // x = alpha*A*b + beta*x
- *
- * // Applying to Fbcsr matrices computes a SpGEMM product of two sparse
- * matrices A->apply(B, C)              // C = A*B A->apply(alpha, B, beta, C)
- * // C = alpha*A*B + beta*C
- *
- * // Applying to an Identity matrix computes a SpGEAM sparse matrix addition
- * A->apply(alpha, I, beta, B) // B = alpha*A + beta*B
  * ```
- * Both the SpGEMM and SpGEAM operation require the input matrices to be sorted
- * by block-column index, otherwise the algorithms will produce incorrect
- * results.
  *
  * @tparam ValueType  precision of matrix elements
  * @tparam IndexType  precision of matrix indexes
@@ -335,11 +326,11 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
           size_type num_nonzeros, int block_size)
         : EnableLinOp<Fbcsr>(exec, size),
           bs_{block_size},
-          nbcols_{blockutils::getNumBlocks(block_size, size[1])},
+          nbcols_{blockutils::get_num_blocks(block_size, size[1])},
           values_(exec, num_nonzeros),
-          col_idxs_(exec, blockutils::getNumBlocks(block_size * block_size,
-                                                   num_nonzeros)),
-          row_ptrs_(exec, blockutils::getNumBlocks(block_size, size[0]) + 1)
+          col_idxs_(exec, blockutils::get_num_blocks(block_size * block_size,
+                                                     num_nonzeros)),
+          row_ptrs_(exec, blockutils::get_num_blocks(block_size, size[0]) + 1)
     {}
 
     /**
@@ -370,7 +361,7 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
           RowPtrsArray &&row_ptrs)
         : EnableLinOp<Fbcsr>(exec, size),
           bs_{block_size},
-          nbcols_{blockutils::getNumBlocks(block_size, size[1])},
+          nbcols_{blockutils::get_num_blocks(block_size, size[1])},
           values_{exec, std::forward<ValuesArray>(values)},
           col_idxs_{exec, std::forward<ColIdxsArray>(col_idxs)},
           row_ptrs_{exec, std::forward<RowPtrsArray>(row_ptrs)}

From 2126265a64622a0a4e9bd01ee682bb70c7ad96a6 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 13 Jan 2021 11:43:39 +0100
Subject: [PATCH 0565/1008] first fbcsr constructor now takes optional block
 size

---
 include/ginkgo/core/matrix/fbcsr.hpp    | 8 +++++---
 reference/test/matrix/fbcsr_kernels.cpp | 2 --
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index 6eb4b061724..e783d50e078 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -306,12 +306,14 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
 
 protected:
     /**
-     * Creates an uninitialized FBCSR matrix with a block size of 1.
+     * Creates an uninitialized FBCSR matrix with the given block size.
      *
      * @param exec  Executor associated to the matrix
+     * @param block_size  The desired size of the dense square nonzero blocks;
+     *                    defaults to 1.
      */
-    Fbcsr(std::shared_ptr<const Executor> exec)
-        : Fbcsr(std::move(exec), dim<2>{}, {}, 1)
+    Fbcsr(std::shared_ptr<const Executor> exec, int block_size = 1)
+        : Fbcsr(std::move(exec), dim<2>{}, {}, block_size)
     {}
 
     /**
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 11c5d1bfa7a..8299f36465e 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -545,8 +545,6 @@ TYPED_TEST(Fbcsr, CalculatesNonzerosPerRow)
     gko::kernels::reference::fbcsr::calculate_nonzeros_per_row(
         this->exec, this->mtx2.get(), &row_nnz);
 
-    // const gko::Array<IndexType> refrnnz =
-    // this->fbsample2.getNonzerosPerRow();
     gko::Array<gko::size_type> refrnnz(this->exec, this->mtx2->get_size()[0]);
     gko::kernels::reference::csr ::calculate_nonzeros_per_row(
         this->exec, this->ref2csrmtx.get(), &refrnnz);

From 8fe1a7ba1b5f0cfc8287f8601832f5883030cba7 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 13 Jan 2021 20:31:57 +0100
Subject: [PATCH 0566/1008] Incorporated suggestions from review

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Pratik Nayak <pratik.nayak@kit.edu>
---
 core/components/fixed_block.hpp         |   8 +-
 core/matrix/fbcsr.cpp                   |  32 +--
 core/matrix/fbcsr_builder.hpp           |   4 +-
 core/test/matrix/fbcsr.cpp              |  29 +-
 core/test/matrix/fbcsr_sample.hpp       | 336 +++++-------------------
 hip/matrix/fbcsr_kernels.hip.cpp        |   2 +-
 include/ginkgo/core/base/blockutils.hpp |  21 +-
 include/ginkgo/core/matrix/fbcsr.hpp    |  16 +-
 omp/matrix/fbcsr_kernels.cpp            |   2 +-
 reference/matrix/fbcsr_kernels.cpp      |   5 +-
 reference/test/matrix/fbcsr_kernels.cpp |  28 +-
 11 files changed, 141 insertions(+), 342 deletions(-)

diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
index 32e53510f74..9bc8facc041 100644
--- a/core/components/fixed_block.hpp
+++ b/core/components/fixed_block.hpp
@@ -61,10 +61,10 @@ class DenseBlocksView final {
     using index_type = IndexType;
 
     /**
-     * @param buffer Segment of memory to be interpreted as an array of 2D
-     * blocks
-     * @param num_rows Number of rows in each block
-     * @param num_cols Number of columns in each block
+     * @param buffer  Pointer to the segment of memory to be interpreted as
+     *                an array of 2D blocks
+     * @param num_rows  Number of rows in each block
+     * @param num_cols  Number of columns in each block
      */
     DenseBlocksView(ValueType *const buffer, const int num_rows,
                     const int num_cols)
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index d709639a4da..dbd21047da3 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -89,7 +89,7 @@ namespace detail {
  *
  * @internal Currently used only while reading a FBCSR matrix from matrix_data.
  *
- * @tparam ValueType The numeric type of entries of the block
+ * @tparam ValueType  The numeric type of entries of the block
  */
 template <typename ValueType>
 class DenseBlock final {
@@ -153,8 +153,7 @@ void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const b,
     using Dense = Dense<ValueType>;
     if (auto b_fbcsr = dynamic_cast<const Fbcsr<ValueType, IndexType> *>(b)) {
         // if b is a FBCSR matrix, we compute a SpGeMM
-        throw /*::gko::*/ NotImplemented(__FILE__, __LINE__,
-                                         "SpGeMM for Fbcsr");
+        throw NotImplemented(__FILE__, __LINE__, "SpGeMM for Fbcsr");
     } else {
         // otherwise we assume that b is dense and compute a SpMV/SpMM
         this->get_executor()->run(
@@ -293,24 +292,22 @@ void Fbcsr<ValueType, IndexType>::move_to(
 }
 
 
-/* Within blocks, the storage order is row-major.
- * Currently, this implementation is sequential and has complexity O(n log n)
- * assuming nnz = O(n).
- * Can this be changed to a parallel O(n) implementation?
+/*
+ * Currently, this implementation is sequential and has complexity
+ * O(nnz log(nnz)).
+ * @note Can this be changed to a parallel O(nnz) implementation?
  */
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
 {
-    if (data.nonzeros.size() > std::numeric_limits<index_type>::max())
-        throw std::range_error(std::string("file: ") + __FILE__ + ":" +
-                               std::to_string(__LINE__) +
-                               ": List of nonzeros is too big!");
+    GKO_ENSURE_IN_BOUNDS(data.nonzeros.size(),
+                         std::numeric_limits<index_type>::max());
 
     const auto nnz = static_cast<index_type>(data.nonzeros.size());
 
     const int bs = this->bs_;
 
-    using Blk_t = detail::DenseBlock<value_type>;
+    using Block_t = detail::DenseBlock<value_type>;
 
     struct FbEntry {
         index_type block_row;
@@ -328,7 +325,7 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
     };
 
     auto create_block_map = [nnz, bs](const mat_data &mdata) {
-        std::map<FbEntry, Blk_t, FbLess> blocks;
+        std::map<FbEntry, Block_t, FbLess> blocks;
         for (index_type inz = 0; inz < nnz; inz++) {
             const index_type row = mdata.nonzeros[inz].row;
             const index_type col = mdata.nonzeros[inz].column;
@@ -339,22 +336,21 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
             const index_type blockrow = row / bs;
             const index_type blockcol = col / bs;
 
-            Blk_t &nnzblk = blocks[{blockrow, blockcol}];
+            Block_t &nnzblk = blocks[{blockrow, blockcol}];
             if (nnzblk.size() == 0) {
                 nnzblk.resize(bs, bs);
                 nnzblk.zero();
                 nnzblk(localrow, localcol) = val;
             } else {
-                if (nnzblk(localrow, localcol) != gko::zero<value_type>())
-                    throw Error(__FILE__, __LINE__,
-                                "Error: re-visited the same non-zero!");
+                // If this does not happen, we re-visited a nonzero
+                assert(nnzblk(localrow, localcol) == gko::zero<value_type>());
                 nnzblk(localrow, localcol) = val;
             }
         }
         return blocks;
     };
 
-    const std::map<FbEntry, Blk_t, FbLess> blocks = create_block_map(data);
+    const std::map<FbEntry, Block_t, FbLess> blocks = create_block_map(data);
 
     auto tmp = Fbcsr::create(this->get_executor()->get_master(), data.size,
                              blocks.size() * bs * bs, bs);
diff --git a/core/matrix/fbcsr_builder.hpp b/core/matrix/fbcsr_builder.hpp
index 22d0bcfacb9..df10c2a3a57 100644
--- a/core/matrix/fbcsr_builder.hpp
+++ b/core/matrix/fbcsr_builder.hpp
@@ -68,8 +68,8 @@ class FbcsrBuilder {
     int get_block_size() const { return matrix_->bs_; }
 
     /**
-     * @param matrix An existing FBCSR matrix
-     *        for which intrusive access is needed
+     * @param matrix  An existing FBCSR matrix
+     *                for which intrusive access is needed
      */
     explicit FbcsrBuilder(Fbcsr<ValueType, IndexType> *const matrix)
         : matrix_{matrix}
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index 34dce1af828..a38e1b32695 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -33,7 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/fbcsr.hpp>
 
 
-#include <iostream>
+#include <algorithm>
 #include <limits>
 
 
@@ -71,10 +71,9 @@ class Fbcsr : public ::testing::Test {
         orig_rowptrs.resize(fbsample.nbrows + 1);
         orig_colinds.resize(fbsample.nbnz);
         orig_vals.resize(fbsample.nnz);
-        for (index_type i = 0; i < fbsample.nbrows + 1; i++)
-            orig_rowptrs[i] = r[i];
-        for (index_type i = 0; i < fbsample.nbnz; i++) orig_colinds[i] = c[i];
-        for (index_type i = 0; i < fbsample.nnz; i++) orig_vals[i] = v[i];
+        std::copy(r, r + fbsample.nbrows + 1, orig_rowptrs.data());
+        std::copy(c, c + fbsample.nbnz, orig_colinds.data());
+        std::copy(v, v + fbsample.nnz, orig_vals.data());
     }
 
     std::shared_ptr<const gko::Executor> exec;
@@ -197,6 +196,15 @@ TYPED_TEST(Fbcsr, ContainsCorrectData)
 }
 
 
+TYPED_TEST(Fbcsr, BlockSizeIsSetCorrectly)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto m = Mtx::create(this->exec);
+    m->set_block_size(6);
+    ASSERT_EQ(m->get_block_size(), 6);
+}
+
+
 TYPED_TEST(Fbcsr, CanBeEmpty)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -282,7 +290,6 @@ TYPED_TEST(Fbcsr, CanBeReadFromMatrixData)
     using Mtx = typename TestFixture::Mtx;
     auto m = Mtx::create(this->exec);
     m->set_block_size(this->fbsample.bs);
-    ASSERT_EQ(m->get_block_size(), this->fbsample.bs);
 
     m->read(this->fbsample.generate_matrix_data());
 
@@ -294,16 +301,14 @@ TYPED_TEST(Fbcsr, GeneratesCorrectMatrixData)
 {
     using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
-    using tpl = typename gko::matrix_data<value_type, index_type>::nonzero_type;
-    gko::matrix_data<value_type, index_type> data;
-
-    this->mtx->write(data);
-    data.ensure_row_major_order();
-
     gko::matrix_data<value_type, index_type> refdata =
         this->fbsample.generate_matrix_data_with_explicit_zeros();
     refdata.ensure_row_major_order();
 
+    gko::matrix_data<value_type, index_type> data;
+    this->mtx->write(data);
+    data.ensure_row_major_order();
+
     ASSERT_EQ(data.size, refdata.size);
     ASSERT_EQ(data.nonzeros.size(), refdata.nonzeros.size());
     for (size_t i = 0; i < data.nonzeros.size(); i++)
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 52f8e66acd5..098aef69aa1 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -43,12 +43,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/components/fixed_block.hpp"
+#include "core/test/utils.hpp"
 
 
 #define FBCSR_TEST_OFFSET 0.000011118888
-
 #define FBCSR_TEST_C_MAG 0.1 + FBCSR_TEST_OFFSET
-
 #define FBCSR_TEST_IMAGINARY \
     sct(std::complex<remove_complex<ValueType>>(0, FBCSR_TEST_C_MAG))
 
@@ -150,101 +149,20 @@ class FbcsrSample {
      */
     std::unique_ptr<Csr> generate_csr() const
     {
-        std::unique_ptr<Csr> csrm =
-            Csr::create(exec, gko::dim<2>{nrows, ncols}, nnz,
-                        std::make_shared<typename Csr::classical>());
-        index_type *const csrrow = csrm->get_row_ptrs();
-        index_type *const csrcols = csrm->get_col_idxs();
-        value_type *const csrvals = csrm->get_values();
-
-        csrrow[0] = 0;
-        csrrow[1] = 6;
-        csrrow[2] = 12;
-        csrrow[3] = 18;
-        csrrow[4] = 24;
-        csrrow[5] = 30;
-        csrrow[6] = 36;
-
-        csrcols[0] = 3;
-        csrvals[0] = 2;
-        csrcols[1] = 4;
-        csrvals[1] = 3;
-        csrcols[2] = 5;
-        csrvals[2] = 4;
-        csrcols[6] = 3;
-        csrvals[6] = 5;
-        csrcols[7] = 4;
-        csrvals[7] = 6;
-        csrcols[8] = 5;
-        csrvals[8] = 7;
-        csrcols[12] = 3;
-        csrvals[12] = 0;
-        csrcols[13] = 4;
-        csrvals[13] = 9;
-        csrcols[14] = 5;
-        csrvals[14] = 0;
-
-        csrcols[3] = 9;
-        csrvals[3] = 4;
-        csrcols[4] = 10;
-        csrvals[4] = 5;
-        csrcols[5] = 11;
-        csrvals[5] = 6;
-        csrcols[9] = 9;
-        csrvals[9] = 7;
-        csrcols[10] = 10;
-        csrvals[10] = 8;
-        csrcols[11] = 11;
-        csrvals[11] = 9;
-        csrcols[15] = 9;
-        csrvals[15] = 10;
-        csrcols[16] = 10;
-        csrvals[16] = 11;
-        csrcols[17] = 11;
-        csrvals[17] = 12;
-
-        csrcols[18] = 0;
-        csrvals[18] = 2;
-        csrcols[19] = 1;
-        csrvals[19] = 3;
-        csrcols[20] = 2;
-        csrvals[20] = 4;
-        csrcols[24] = 0;
-        csrvals[24] = 5;
-        csrcols[25] = 1;
-        csrvals[25] = 6;
-        csrcols[26] = 2;
-        csrvals[26] = 7;
-        csrcols[30] = 0;
-        csrvals[30] = 8;
-        csrcols[31] = 1;
-        csrvals[31] = 9;
-        csrcols[32] = 2;
-        csrvals[32] = 10;
-
-        csrcols[21] = 6;
-        csrvals[21] = 0;
-        csrcols[22] = 7;
-        csrvals[22] = 7;
-        csrcols[23] = 8;
-        csrvals[23] = 8;
-        csrcols[27] = 6;
-        csrvals[27] = 9;
-        csrcols[28] = 7;
-        csrvals[28] = 10;
-        csrcols[29] = 8;
-        csrvals[29] = 11;
-        csrcols[33] = 6;
-        csrvals[33] = 12;
-        csrcols[34] = 7;
-        csrvals[34] = 13;
-        csrcols[35] = 8;
-        csrvals[35] = 14;
-
-        csrvals[34] += FBCSR_TEST_IMAGINARY;
-        csrvals[35] += FBCSR_TEST_IMAGINARY;
-
-        return csrm;
+        gko::Array<index_type> csrrow(exec, {0, 6, 12, 18, 24, 30, 36});
+        gko::Array<index_type> csrcols(
+            exec, {3, 4, 5, 9, 10, 11, 3, 4, 5, 9, 10, 11, 3, 4, 5, 9, 10, 11,
+                   0, 1, 2, 6, 7,  8,  0, 1, 2, 6, 7,  8,  0, 1, 2, 6, 7,  8});
+        // clang-format off
+        gko::Array<value_type> csrvals(exec, I<value_type>
+            {2, 3, 4, 4, 5, 6, 5, 6, 7, 7, 8, 9, 0, 9, 0,
+	         10, 11, 12, 2, 3, 4, 0, 7, 8, 5, 6, 7,
+	         9, 10, 11, 8, 9, 10, 12,
+	         sct<value_type>(13.0) + FBCSR_TEST_IMAGINARY,
+	         sct<value_type>(14.0) + FBCSR_TEST_IMAGINARY});
+        // clang-format on
+        return Csr::create(exec, gko::dim<2>{nrows, ncols}, csrvals, csrcols,
+                           csrrow);
     }
 
     /**
@@ -317,46 +235,12 @@ class FbcsrSample {
      */
     MatData generate_matrix_data_with_explicit_zeros() const
     {
-        return MatData({{6, 12},
-                        {{0, 3, 2.0},
-                         {0, 4, 3.0},
-                         {0, 5, 4.0},
-                         {1, 3, 5.0},
-                         {1, 4, 6.0},
-                         {1, 5, 7.0},
-                         {2, 3, 0.0},
-                         {2, 4, 9.0},
-                         {2, 5, 0.0},
-
-                         {0, 9, 4.0},
-                         {0, 10, 5.0},
-                         {0, 11, 6.0},
-                         {1, 9, 7.0},
-                         {1, 10, 8.0},
-                         {1, 11, 9.0},
-                         {2, 9, 10.0},
-                         {2, 10, 11.0},
-                         {2, 11, 12.0},
-
-                         {3, 0, 2.0},
-                         {3, 1, 3.0},
-                         {3, 2, 4.0},
-                         {4, 0, 5.0},
-                         {4, 1, 6.0},
-                         {4, 2, 7.0},
-                         {5, 0, 8.0},
-                         {5, 1, 9.0},
-                         {5, 2, 10.0},
-
-                         {3, 6, 0.0},
-                         {3, 7, 7.0},
-                         {3, 8, 8.0},
-                         {4, 6, 9.0},
-                         {4, 7, 10.0},
-                         {4, 8, 11.0},
-                         {5, 6, 12.0},
-                         {5, 7, sct(13.0) + FBCSR_TEST_IMAGINARY},
-                         {5, 8, sct(14.0) + FBCSR_TEST_IMAGINARY}}});
+        auto mdata = generate_matrix_data();
+        mdata.nonzeros.push_back({2, 3, 0.0});
+        mdata.nonzeros.push_back({2, 5, 0.0});
+        mdata.nonzeros.push_back({3, 6, 0.0});
+        mdata.ensure_row_major_order();
+        return mdata;
     }
 
 private:
@@ -411,24 +295,10 @@ class FbcsrSample2 {
 
     std::unique_ptr<Fbcsr> generate_fbcsr() const
     {
-        std::unique_ptr<Fbcsr> mtx =
-            Fbcsr::create(exec,
-                          gko::dim<2>{static_cast<size_type>(nrows),
-                                      static_cast<size_type>(ncols)},
-                          nnz, bs);
-
-        value_type *const v = mtx->get_values();
-        index_type *const c = mtx->get_col_idxs();
-        index_type *const r = mtx->get_row_ptrs();
-        r[0] = 0;
-        r[1] = 1;
-        r[2] = 3;
-        r[3] = 4;
-        c[0] = 0;
-        c[1] = 0;
-        c[2] = 3;
-        c[3] = 2;
-
+        gko::Array<index_type> r(exec, {0, 1, 3, 4});
+        gko::Array<index_type> c(exec, {0, 0, 3, 2});
+        gko::Array<value_type> vals(exec, nnz);
+        value_type *const v = vals.get_data();
         for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
 
         v[0] = 1;
@@ -442,47 +312,20 @@ class FbcsrSample2 {
         v[14] = -2;
         v[15] = -11;
 
-        return mtx;
+        return Fbcsr::create(exec,
+                             gko::dim<2>{static_cast<size_type>(nrows),
+                                         static_cast<size_type>(ncols)},
+                             bs, vals, c, r);
     }
 
     std::unique_ptr<Csr> generate_csr() const
     {
-        std::unique_ptr<Csr> mtx =
-            Csr::create(exec,
-                        gko::dim<2>{static_cast<size_type>(nrows),
-                                    static_cast<size_type>(ncols)},
-                        nnz, std::make_shared<typename Csr::classical>());
-
-        value_type *const v = mtx->get_values();
-        index_type *const c = mtx->get_col_idxs();
-        index_type *const r = mtx->get_row_ptrs();
-        r[0] = 0;
-        r[1] = 2;
-        r[2] = 4;
-        r[3] = 8;
-        r[4] = 12;
-        r[5] = 14;
-        r[6] = 16;
-
-        c[0] = 0;
-        c[1] = 1;
-        c[2] = 0;
-        c[3] = 1;
-        c[4] = 0;
-        c[5] = 1;
-        c[6] = 6;
-        c[7] = 7;
-        c[8] = 0;
-        c[9] = 1;
-        c[10] = 6;
-        c[11] = 7;
-        c[12] = 4;
-        c[13] = 5;
-        c[14] = 4;
-        c[15] = 5;
-
+        gko::Array<index_type> r(exec, {0, 2, 4, 8, 12, 14, 16});
+        gko::Array<index_type> c(
+            exec, {0, 1, 0, 1, 0, 1, 6, 7, 0, 1, 6, 7, 4, 5, 4, 5});
+        gko::Array<value_type> vals(exec, nnz);
+        value_type *const v = vals.get_data();
         for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
-
         v[0] = 1;
         v[1] = 2;
         v[2] = 3;
@@ -494,19 +337,16 @@ class FbcsrSample2 {
         v[14] = -2;
         v[15] = -11;
 
-        return mtx;
+        return Csr::create(exec,
+                           gko::dim<2>{static_cast<size_type>(nrows),
+                                       static_cast<size_type>(ncols)},
+                           vals, c, r,
+                           std::make_shared<typename Csr::classical>());
     }
 
     std::unique_ptr<Diagonal> extract_diagonal() const
     {
-        gko::Array<ValueType> dvals(exec, nrows);
-        ValueType *const dv = dvals.get_data();
-        dv[0] = 1;
-        dv[1] = 0;
-        dv[2] = 0;
-        dv[3] = 0;
-        dv[4] = -12;
-        dv[5] = -11;
+        gko::Array<ValueType> dvals(exec, {1, 0, 0, 0, -12, -11});
         return Diagonal::create(exec, nrows, dvals);
     }
 
@@ -573,24 +413,16 @@ class FbcsrSampleSquare {
 
     std::unique_ptr<Fbcsr> generate_fbcsr() const
     {
-        std::unique_ptr<Fbcsr> mtx =
-            Fbcsr::create(exec,
-                          gko::dim<2>{static_cast<size_type>(nrows),
-                                      static_cast<size_type>(ncols)},
-                          nnz, bs);
-
-        value_type *const v = mtx->get_values();
-        index_type *const c = mtx->get_col_idxs();
-        index_type *const r = mtx->get_row_ptrs();
-        r[0] = 0;
-        r[1] = 1;
-        r[2] = 2;
-        c[0] = 1;
-        c[1] = 1;
-
+        gko::Array<index_type> c(exec, {1, 1});
+        gko::Array<index_type> r(exec, {0, 1, 2});
+        gko::Array<value_type> vals(exec, nnz);
+        value_type *const v = vals.get_data();
         for (IndexType i = 0; i < nnz; i++) v[i] = i;
 
-        return mtx;
+        return Fbcsr::create(exec,
+                             gko::dim<2>{static_cast<size_type>(nrows),
+                                         static_cast<size_type>(ncols)},
+                             bs, vals, c, r);
     }
 };
 
@@ -627,24 +459,10 @@ class FbcsrSampleComplex {
 
     std::unique_ptr<Fbcsr> generate_fbcsr() const
     {
-        std::unique_ptr<Fbcsr> mtx =
-            Fbcsr::create(exec,
-                          gko::dim<2>{static_cast<size_type>(nrows),
-                                      static_cast<size_type>(ncols)},
-                          nnz, bs);
-
-        value_type *const v = mtx->get_values();
-        index_type *const c = mtx->get_col_idxs();
-        index_type *const r = mtx->get_row_ptrs();
-        r[0] = 0;
-        r[1] = 1;
-        r[2] = 3;
-        r[3] = 4;
-        c[0] = 0;
-        c[1] = 0;
-        c[2] = 3;
-        c[3] = 2;
-
+        gko::Array<index_type> r(exec, {0, 1, 3, 4});
+        gko::Array<index_type> c(exec, {0, 0, 3, 2});
+        gko::Array<value_type> vals(exec, nnz);
+        value_type *const v = vals.get_data();
         for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
 
         using namespace std::complex_literals;
@@ -659,45 +477,19 @@ class FbcsrSampleComplex {
         v[14] = -2.0 - 2.15i;
         v[15] = -11.0 - 11.15i;
 
-        return mtx;
+        return Fbcsr::create(exec,
+                             gko::dim<2>{static_cast<size_type>(nrows),
+                                         static_cast<size_type>(ncols)},
+                             bs, vals, c, r);
     }
 
     std::unique_ptr<Csr> generate_csr() const
     {
-        std::unique_ptr<Csr> mtx =
-            Csr::create(exec,
-                        gko::dim<2>{static_cast<size_type>(nrows),
-                                    static_cast<size_type>(ncols)},
-                        nnz, std::make_shared<typename Csr::classical>());
-
-        value_type *const v = mtx->get_values();
-        index_type *const c = mtx->get_col_idxs();
-        index_type *const r = mtx->get_row_ptrs();
-        r[0] = 0;
-        r[1] = 2;
-        r[2] = 4;
-        r[3] = 8;
-        r[4] = 12;
-        r[5] = 14;
-        r[6] = 16;
-
-        c[0] = 0;
-        c[1] = 1;
-        c[2] = 0;
-        c[3] = 1;
-        c[4] = 0;
-        c[5] = 1;
-        c[6] = 6;
-        c[7] = 7;
-        c[8] = 0;
-        c[9] = 1;
-        c[10] = 6;
-        c[11] = 7;
-        c[12] = 4;
-        c[13] = 5;
-        c[14] = 4;
-        c[15] = 5;
-
+        gko::Array<index_type> r(exec, {0, 2, 4, 8, 12, 14, 16});
+        gko::Array<index_type> c(
+            exec, {0, 1, 0, 1, 0, 1, 6, 7, 0, 1, 6, 7, 4, 5, 4, 5});
+        gko::Array<value_type> vals(exec, nnz);
+        value_type *const v = vals.get_data();
         for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
 
         using namespace std::complex_literals;
@@ -712,7 +504,11 @@ class FbcsrSampleComplex {
         v[14] = -2.0 - 2.15i;
         v[15] = -11.0 - 11.15i;
 
-        return mtx;
+        return Csr::create(exec,
+                           gko::dim<2>{static_cast<size_type>(nrows),
+                                       static_cast<size_type>(ncols)},
+                           vals, c, r,
+                           std::make_shared<typename Csr::classical>());
     }
 };
 
diff --git a/hip/matrix/fbcsr_kernels.hip.cpp b/hip/matrix/fbcsr_kernels.hip.cpp
index fd30554a469..3bf18bc6515 100644
--- a/hip/matrix/fbcsr_kernels.hip.cpp
+++ b/hip/matrix/fbcsr_kernels.hip.cpp
@@ -53,7 +53,7 @@ namespace gko {
 namespace kernels {
 namespace hip {
 /**
- * @brief The Compressed sparse row matrix format namespace.
+ * @brief The fixed-size block compressed sparse row matrix format namespace.
  *
  * @ingroup fbcsr
  */
diff --git a/include/ginkgo/core/base/blockutils.hpp b/include/ginkgo/core/base/blockutils.hpp
index c0b16ef6ac1..20c6ac5162a 100644
--- a/include/ginkgo/core/base/blockutils.hpp
+++ b/include/ginkgo/core/base/blockutils.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GINKGO_CORE_BASE_BLOCKUTILS_HPP_
-#define GINKGO_CORE_BASE_BLOCKUTILS_HPP_
+#ifndef GINKGO_PUBLIC_CORE_BASE_BLOCKUTILS_HPP_
+#define GINKGO_PUBLIC_CORE_BASE_BLOCKUTILS_HPP_
 
 
 #include <ginkgo/core/base/exception.hpp>
@@ -41,7 +41,12 @@ namespace gko {
 namespace blockutils {
 
 
-/// Error that denotes issues between block sizes and matrix dimensions
+/**
+ * Error that denotes issues between block sizes and matrix dimensions
+ *
+ * \tparam IntexType  Type of index used by the linear algebra object that is
+ *                    incompatible with the requried block size.
+ */
 template <typename IndexType>
 class BlockSizeError : public Error {
 public:
@@ -57,10 +62,12 @@ class BlockSizeError : public Error {
 /**
  * Computes the number of blocks
  *
- * @param block_size The size of each block
- * @param size The total size of some array/vector
- * @return The quotient of the size divided by the block size
- *         but throws when they don't divide
+ * @param  block_size The size of each block
+ * @param  size The total size of some array/vector
+ * @return  The number of blocks, ie.,
+ *          quotient of the size divided by the block size.
+ *
+ * @throw BlockSizeError  when block_size does not divide the total size.
  */
 template <typename IndexType>
 IndexType get_num_blocks(const int block_size, const IndexType size)
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index e783d50e078..feda178c6a5 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -163,7 +163,9 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
 
     void move_to(SparsityCsr<ValueType, IndexType> *result) override;
 
-    /** Convert COO data into block CSR
+    /**
+     * Reads a @ref matrix_data into Fbcsr format.
+     * Requires the block size to be set beforehand @sa set_block_size.
      *
      * @warning Unlike Csr::read, here explicit non-zeros are NOT dropped.
      */
@@ -186,7 +188,7 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
      */
     void sort_by_column_index();
 
-    /*
+    /**
      * Tests if all row entry pairs (value, col_idx) are sorted by column index
      *
      * @returns True if all row entry pairs (value, col_idx) are sorted by
@@ -328,7 +330,8 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
           size_type num_nonzeros, int block_size)
         : EnableLinOp<Fbcsr>(exec, size),
           bs_{block_size},
-          nbcols_{blockutils::get_num_blocks(block_size, size[1])},
+          nbcols_{static_cast<index_type>(
+              blockutils::get_num_blocks(block_size, size[1]))},
           values_(exec, num_nonzeros),
           col_idxs_(exec, blockutils::get_num_blocks(block_size * block_size,
                                                      num_nonzeros)),
@@ -345,7 +348,7 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
      *
      * @param exec  Executor associated to the matrix
      * @param size  size of the matrix
-     * @param block_size
+     * @param block_size  Size of the small square dense nonzero blocks
      * @param values  array of matrix values
      * @param col_idxs  array of column indexes
      * @param row_ptrs  array of row pointers
@@ -363,7 +366,8 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
           RowPtrsArray &&row_ptrs)
         : EnableLinOp<Fbcsr>(exec, size),
           bs_{block_size},
-          nbcols_{blockutils::get_num_blocks(block_size, size[1])},
+          nbcols_{static_cast<index_type>(
+              blockutils::get_num_blocks(block_size, size[1]))},
           values_{exec, std::forward<ValuesArray>(values)},
           col_idxs_{exec, std::forward<ColIdxsArray>(col_idxs)},
           row_ptrs_{exec, std::forward<RowPtrsArray>(row_ptrs)}
@@ -380,7 +384,7 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
 
 private:
     int bs_;                      ///< Block size
-    size_type nbcols_;            ///< Number of block-columns
+    index_type nbcols_;           ///< Number of block-columns
     Array<value_type> values_;    ///< Non-zero values of all blocks
     Array<index_type> col_idxs_;  ///< Block-column indices of all blocks
     Array<index_type> row_ptrs_;  ///< Block-row pointers into @ref col_idxs_
diff --git a/omp/matrix/fbcsr_kernels.cpp b/omp/matrix/fbcsr_kernels.cpp
index 15f63ee9794..9bc8a091906 100644
--- a/omp/matrix/fbcsr_kernels.cpp
+++ b/omp/matrix/fbcsr_kernels.cpp
@@ -51,7 +51,7 @@ namespace gko {
 namespace kernels {
 namespace omp {
 /**
- * @brief The Compressed sparse row matrix format namespace.
+ * @brief The fixed-block block compressed sparse row matrix format namespace.
  *
  * @ingroup fbcsr
  */
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index 108697537df..6590c895376 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -58,8 +58,8 @@ namespace gko {
 namespace kernels {
 namespace reference {
 /**
- * @brief The Compressed sparse row matrix format namespace.
- * @ref Fbcsr
+ * @brief The fixed-block compressed sparse row matrix format namespace.
+ *
  * @ingroup fbcsr
  */
 namespace fbcsr {
@@ -83,7 +83,6 @@ void spmv(const std::shared_ptr<const ReferenceExecutor>,
     auto *const cvals = c->get_values();
 
     for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
-        // const IndexType crowblkend = (ibrow + 1) * bs * nvecs;
         for (IndexType i = ibrow * bs * nvecs; i < (ibrow + 1) * bs * nvecs;
              ++i)
             cvals[i] = zero<ValueType>();
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 8299f36465e..e9fff20c1ec 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -539,7 +539,6 @@ TYPED_TEST(Fbcsr, MovesEmptyToSparsityCsr)
 TYPED_TEST(Fbcsr, CalculatesNonzerosPerRow)
 {
     using IndexType = typename TestFixture::index_type;
-
     gko::Array<gko::size_type> row_nnz(this->exec, this->mtx2->get_size()[0]);
 
     gko::kernels::reference::fbcsr::calculate_nonzeros_per_row(
@@ -560,7 +559,6 @@ TYPED_TEST(Fbcsr, SquareMtxIsTransposable)
 {
     using Fbcsr = typename TestFixture::Mtx;
     using Csr = typename TestFixture::Csr;
-
     auto csrmtxsq =
         Csr::create(this->exec, std::make_shared<typename Csr::classical>());
     this->mtxsq->convert_to(csrmtxsq.get());
@@ -578,7 +576,6 @@ TYPED_TEST(Fbcsr, NonSquareMtxIsTransposable)
 {
     using Fbcsr = typename TestFixture::Mtx;
     using Csr = typename TestFixture::Csr;
-
     auto csrmtx =
         Csr::create(this->exec, std::make_shared<typename Csr::classical>());
     this->mtx2->convert_to(csrmtx.get());
@@ -602,10 +599,10 @@ TYPED_TEST(Fbcsr, RecognizeUnsortedMatrix)
 {
     using Fbcsr = typename TestFixture::Mtx;
     using index_type = typename TestFixture::index_type;
-
     auto cpmat = this->mtx->clone();
     index_type *const colinds = cpmat->get_col_idxs();
     std::swap(colinds[0], colinds[1]);
+
     ASSERT_FALSE(cpmat->is_sorted_by_column_index());
 }
 
@@ -614,6 +611,7 @@ TYPED_TEST(Fbcsr, ExtractsDiagonal)
 {
     using T = typename TestFixture::value_type;
     auto matrix = this->mtx2->clone();
+
     auto diag = matrix->extract_diagonal();
 
     ASSERT_EQ(this->m2diag->get_size(), diag->get_size());
@@ -625,15 +623,14 @@ TYPED_TEST(Fbcsr, ExtractsDiagonal)
 
 TYPED_TEST(Fbcsr, InplaceAbsolute)
 {
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
     using Mtx = typename TestFixture::Mtx;
     using Csr = typename TestFixture::Csr;
     auto mtx = this->fbsample2.generate_fbcsr();
     const std::unique_ptr<Csr> refabs = this->ref2csrmtx->clone();
     refabs->compute_absolute_inplace();
 
-    using value_type = typename TestFixture::value_type;
-    using index_type = typename TestFixture::index_type;
-
     mtx->compute_absolute_inplace();
 
     const gko::remove_complex<value_type> tolerance =
@@ -648,7 +645,6 @@ TYPED_TEST(Fbcsr, OutplaceAbsolute)
     using Mtx = typename TestFixture::Mtx;
     using AbsCsr = typename gko::remove_complex<typename TestFixture::Csr>;
     using AbsMtx = typename gko::remove_complex<typename TestFixture::Mtx>;
-
     auto mtx = this->fbsample2.generate_fbcsr();
     const std::unique_ptr<const AbsCsr> refabs =
         this->ref2csrmtx->compute_absolute();
@@ -681,13 +677,14 @@ TYPED_TEST(FbcsrComplex, ConvertsComplexToCsr)
     using Fbcsr = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
-
     auto exec = gko::ReferenceExecutor::create();
     gko::testing::FbcsrSampleComplex<T, index_type> csample(exec);
     std::unique_ptr<const Fbcsr> mtx = csample.generate_fbcsr();
     auto csr_mtx =
         Csr::create(exec, std::make_shared<typename Csr::classical>());
+
     mtx->convert_to(csr_mtx.get());
+
     GKO_ASSERT_MTX_NEAR(csr_mtx, mtx, 0.0);
 }
 
@@ -698,16 +695,15 @@ TYPED_TEST(FbcsrComplex, MtxIsConjugateTransposable)
     using Fbcsr = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
-
     auto exec = gko::ReferenceExecutor::create();
     gko::testing::FbcsrSampleComplex<T, index_type> csample(exec);
     auto csrmtx = csample.generate_csr();
-    auto reftranslinop = csrmtx->conj_transpose();
-    const Csr *const reftrans = static_cast<const Csr *>(reftranslinop.get());
+    auto mtx = csample.generate_fbcsr();
 
-    std::unique_ptr<const Fbcsr> mtx = csample.generate_fbcsr();
+    auto reftranslinop = csrmtx->conj_transpose();
+    auto reftrans = static_cast<const Csr *>(reftranslinop.get());
     auto trans = mtx->conj_transpose();
-    auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
+    auto trans_as_fbcsr = static_cast<const Fbcsr *>(trans.get());
 
     GKO_ASSERT_MTX_NEAR(trans_as_fbcsr, reftrans, 0.0);
 }
@@ -724,8 +720,6 @@ TYPED_TEST(FbcsrComplex, InplaceAbsolute)
     auto mtx = fbsample.generate_fbcsr();
     auto csrmtx = fbsample.generate_csr();
 
-    using value_type = typename TestFixture::value_type;
-
     mtx->compute_absolute_inplace();
     csrmtx->compute_absolute_inplace();
 
@@ -741,10 +735,8 @@ TYPED_TEST(FbcsrComplex, OutplaceAbsolute)
     using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
     using AbsMtx = typename gko::remove_complex<typename TestFixture::Mtx>;
-
     gko::testing::FbcsrSample<value_type, index_type> fbsample(
         gko::ReferenceExecutor::create());
-
     auto mtx = fbsample.generate_fbcsr();
     auto csrmtx = fbsample.generate_csr();
 

From 148225c62048b9a0e05832b85ad73a74599f4fad Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Thu, 14 Jan 2021 16:06:29 +0100
Subject: [PATCH 0567/1008] removed extra getter overloads

---
 core/test/matrix/fbcsr.cpp           |  4 ++--
 core/test/matrix/fbcsr_sample.hpp    |  4 ++--
 include/ginkgo/core/matrix/fbcsr.hpp | 24 ------------------------
 3 files changed, 4 insertions(+), 28 deletions(-)

diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index a38e1b32695..e7244220d17 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -145,7 +145,7 @@ TYPED_TEST(Fbcsr, SampleGeneratorIsCorrect)
         fbmtx->get_const_values(), bs, bs);
 
     for (index_type ibrow = 0; ibrow < this->fbsample.nbrows; ibrow++) {
-        const index_type *const browptr = fbmtx->get_row_ptrs();
+        const index_type *const browptr = fbmtx->get_const_row_ptrs();
         const index_type numblocksbrow = browptr[ibrow + 1] - browptr[ibrow];
         for (index_type irow = ibrow * bs; irow < ibrow * bs + bs; irow++) {
             const index_type rowstart =
@@ -154,7 +154,7 @@ TYPED_TEST(Fbcsr, SampleGeneratorIsCorrect)
             ASSERT_EQ(csmtx->get_const_row_ptrs()[irow], rowstart);
         }
 
-        const index_type *const bcolinds = fbmtx->get_col_idxs();
+        const index_type *const bcolinds = fbmtx->get_const_col_idxs();
 
         for (index_type ibnz = browptr[ibrow]; ibnz < browptr[ibrow + 1];
              ibnz++) {
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 098aef69aa1..ecd0aaff7f6 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -174,9 +174,9 @@ class FbcsrSample {
         gko::Array<IndexType> rowptrs(exec, nbrows + 1);
         const std::unique_ptr<const Fbcsr> fbmat = generate_fbcsr();
         for (index_type i = 0; i < nbrows + 1; i++)
-            rowptrs.get_data()[i] = fbmat->get_row_ptrs()[i];
+            rowptrs.get_data()[i] = fbmat->get_const_row_ptrs()[i];
         for (index_type i = 0; i < nbnz; i++)
-            colids.get_data()[i] = fbmat->get_col_idxs()[i];
+            colids.get_data()[i] = fbmat->get_const_col_idxs()[i];
         return SparCsr::create(exec, gko::dim<2>{nbrows, nbcols}, colids,
                                rowptrs);
     }
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index feda178c6a5..facb8306a24 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -201,14 +201,6 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
      */
     value_type *get_values() noexcept { return values_.get_data(); }
 
-    /**
-     * @see Fbcsr::get_const_values()
-     */
-    const value_type *get_values() const noexcept
-    {
-        return values_.get_const_data();
-    }
-
     /**
      * @copydoc Fbcsr::get_values()
      *
@@ -226,14 +218,6 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
      */
     index_type *get_col_idxs() noexcept { return col_idxs_.get_data(); }
 
-    /**
-     * @see Fbcsr::get_const_col_idxs()
-     */
-    const index_type *get_col_idxs() const noexcept
-    {
-        return col_idxs_.get_const_data();
-    }
-
     /**
      * @copydoc Fbcsr::get_col_idxs()
      *
@@ -251,14 +235,6 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
      */
     index_type *get_row_ptrs() noexcept { return row_ptrs_.get_data(); }
 
-    /**
-     * @see Fbcsr::get_const_row_ptrs()
-     */
-    const index_type *get_row_ptrs() const noexcept
-    {
-        return row_ptrs_.get_const_data();
-    }
-
     /**
      * @copydoc Fbcsr::get_row_ptrs()
      *

From 79809e8454d81f7521e3fb27ad4815e32ca33555 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Thu, 14 Jan 2021 20:16:49 +0100
Subject: [PATCH 0568/1008] incorporated some suggestions from SonarCloud

---
 core/components/fixed_block.hpp      | 11 ++---
 core/matrix/fbcsr.cpp                | 10 +---
 dpcpp/matrix/fbcsr_kernels.dp.cpp    |  2 +-
 include/ginkgo/core/matrix/fbcsr.hpp |  7 ++-
 omp/matrix/fbcsr_kernels.cpp         | 69 +++++++++++++++-------------
 reference/matrix/fbcsr_kernels.cpp   | 12 ++---
 6 files changed, 52 insertions(+), 59 deletions(-)

diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
index 9bc8facc041..889cb2145a2 100644
--- a/core/components/fixed_block.hpp
+++ b/core/components/fixed_block.hpp
@@ -34,9 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_COMPONENTS_FIXED_BLOCK_HPP_
 
 
-#include <type_traits>
-
-
 #include <ginkgo/core/base/types.hpp>
 
 
@@ -76,8 +73,8 @@ class DenseBlocksView final {
         return vals_[block * nrows * ncols + row * ncols + col];
     }
 
-    const typename std::remove_const<value_type>::type &at(
-        const index_type block, const int row, const int col) const
+    const value_type &at(const index_type block, const int row,
+                         const int col) const
     {
         return vals_[block * nrows * ncols + row * ncols + col];
     }
@@ -87,8 +84,8 @@ class DenseBlocksView final {
         return at(block, row, col);
     }
 
-    const typename std::remove_const<value_type>::type &operator()(
-        const index_type block, const int row, const int col) const
+    const value_type &operator()(const index_type block, const int row,
+                                 const int col) const
     {
         return at(block, row, col);
     }
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index dbd21047da3..f0f03224ada 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -96,7 +96,7 @@ class DenseBlock final {
 public:
     using value_type = ValueType;
 
-    DenseBlock() {}
+    DenseBlock() = default;
 
     DenseBlock(const int num_rows, const int num_cols)
         : nrows_{num_rows}, ncols_{num_cols}, vals_(num_rows * num_cols)
@@ -197,14 +197,6 @@ void Fbcsr<ValueType, IndexType>::convert_to(
 }
 
 
-template <typename ValueType, typename IndexType>
-void Fbcsr<ValueType, IndexType>::move_to(
-    Fbcsr<ValueType, IndexType> *const result)
-{
-    EnableLinOp<Fbcsr>::move_to(result);
-}
-
-
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(
     Fbcsr<next_precision<ValueType>, IndexType> *const result) const
diff --git a/dpcpp/matrix/fbcsr_kernels.dp.cpp b/dpcpp/matrix/fbcsr_kernels.dp.cpp
index cebba0ed528..b31bbc9a479 100644
--- a/dpcpp/matrix/fbcsr_kernels.dp.cpp
+++ b/dpcpp/matrix/fbcsr_kernels.dp.cpp
@@ -47,7 +47,7 @@ namespace gko {
 namespace kernels {
 namespace dpcpp {
 /**
- * @brief The Compressed sparse row matrix format namespace.
+ * @brief The fixed-size block compressed sparse row matrix format namespace.
  *
  * @ingroup fbcsr
  */
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index facb8306a24..9e5423395ba 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -130,7 +130,12 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
 
     void convert_to(Fbcsr<ValueType, IndexType> *result) const override;
 
-    void move_to(Fbcsr<ValueType, IndexType> *result) override;
+    /**
+     * For moving to another Fbcsr of the same type, use the default
+     *  implementation provided by EnableLinOp via the
+     *  EnablePolymorphicAssignment mixin.
+     */
+    using EnableLinOp<Fbcsr<ValueType, IndexType>>::move_to;
 
     friend class Fbcsr<next_precision<ValueType>, IndexType>;
 
diff --git a/omp/matrix/fbcsr_kernels.cpp b/omp/matrix/fbcsr_kernels.cpp
index 9bc8a091906..52e8caefb37 100644
--- a/omp/matrix/fbcsr_kernels.cpp
+++ b/omp/matrix/fbcsr_kernels.cpp
@@ -51,7 +51,7 @@ namespace gko {
 namespace kernels {
 namespace omp {
 /**
- * @brief The fixed-block block compressed sparse row matrix format namespace.
+ * @brief The fixed-block compressed sparse row matrix format namespace.
  *
  * @ingroup fbcsr
  */
@@ -60,20 +60,20 @@ namespace fbcsr {
 
 template <typename ValueType, typename IndexType>
 void spmv(std::shared_ptr<const OmpExecutor> exec,
-          const matrix::Fbcsr<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b,
-          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+          const matrix::Fbcsr<ValueType, IndexType> *const a,
+          const matrix::Dense<ValueType> *const b,
+          matrix::Dense<ValueType> *const c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_FBCSR_SPMV_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
 void advanced_spmv(std::shared_ptr<const OmpExecutor> exec,
-                   const matrix::Dense<ValueType> *alpha,
-                   const matrix::Fbcsr<ValueType, IndexType> *a,
-                   const matrix::Dense<ValueType> *b,
-                   const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+                   const matrix::Dense<ValueType> *const alpha,
+                   const matrix::Fbcsr<ValueType, IndexType> *const a,
+                   const matrix::Dense<ValueType> *const b,
+                   const matrix::Dense<ValueType> *const beta,
+                   matrix::Dense<ValueType> *const c) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_ADVANCED_SPMV_KERNEL);
@@ -81,8 +81,9 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const OmpExecutor> exec,
-                      const matrix::Fbcsr<ValueType, IndexType> *source,
-                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
+                      const matrix::Fbcsr<ValueType, IndexType> *const source,
+                      matrix::Dense<ValueType> *const result)
+    GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_DENSE_KERNEL);
@@ -99,25 +100,26 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType, typename UnaryOperator>
-inline void convert_fbcsr_to_csc(size_type num_rows, const IndexType *row_ptrs,
-                                 const IndexType *col_idxs,
-                                 const ValueType *fbcsr_vals,
-                                 IndexType *row_idxs, IndexType *col_ptrs,
-                                 ValueType *csc_vals,
-                                 UnaryOperator op) GKO_NOT_IMPLEMENTED;
+inline void convert_fbcsr_to_csc(
+    size_type num_rows, const IndexType *const row_ptrs,
+    const IndexType *const col_idxs, const ValueType *const fbcsr_vals,
+    IndexType *const row_idxs, IndexType *const col_ptrs,
+    ValueType *const csc_vals, UnaryOperator op) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType, typename UnaryOperator>
-void transpose_and_transform(std::shared_ptr<const OmpExecutor> exec,
-                             matrix::Fbcsr<ValueType, IndexType> *trans,
-                             const matrix::Fbcsr<ValueType, IndexType> *orig,
-                             UnaryOperator op) GKO_NOT_IMPLEMENTED;
+void transpose_and_transform(
+    std::shared_ptr<const OmpExecutor> exec,
+    matrix::Fbcsr<ValueType, IndexType> *const trans,
+    const matrix::Fbcsr<ValueType, IndexType> *const orig,
+    UnaryOperator op) GKO_NOT_IMPLEMENTED;
 
 
 template <typename ValueType, typename IndexType>
 void transpose(std::shared_ptr<const OmpExecutor> exec,
-               const matrix::Fbcsr<ValueType, IndexType> *orig,
-               matrix::Fbcsr<ValueType, IndexType> *trans) GKO_NOT_IMPLEMENTED;
+               const matrix::Fbcsr<ValueType, IndexType> *const orig,
+               matrix::Fbcsr<ValueType, IndexType> *const trans)
+    GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_TRANSPOSE_KERNEL);
@@ -125,8 +127,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void conj_transpose(std::shared_ptr<const OmpExecutor> exec,
-                    const matrix::Fbcsr<ValueType, IndexType> *orig,
-                    matrix::Fbcsr<ValueType, IndexType> *trans)
+                    const matrix::Fbcsr<ValueType, IndexType> *const orig,
+                    matrix::Fbcsr<ValueType, IndexType> *const trans)
     GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -136,8 +138,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void calculate_max_nnz_per_row(
     std::shared_ptr<const OmpExecutor> exec,
-    const matrix::Fbcsr<ValueType, IndexType> *source,
-    size_type *result) GKO_NOT_IMPLEMENTED;
+    const matrix::Fbcsr<ValueType, IndexType> *const source,
+    size_type *const result) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
@@ -146,8 +148,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void calculate_nonzeros_per_row(
     std::shared_ptr<const OmpExecutor> exec,
-    const matrix::Fbcsr<ValueType, IndexType> *source,
-    Array<size_type> *result) GKO_NOT_IMPLEMENTED;
+    const matrix::Fbcsr<ValueType, IndexType> *const source,
+    Array<size_type> *const result) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
@@ -156,8 +158,8 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void is_sorted_by_column_index(
     std::shared_ptr<const OmpExecutor> exec,
-    const matrix::Fbcsr<ValueType, IndexType> *to_check,
-    bool *is_sorted) GKO_NOT_IMPLEMENTED;
+    const matrix::Fbcsr<ValueType, IndexType> *const to_check,
+    bool *const is_sorted) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
@@ -165,8 +167,9 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const OmpExecutor> exec,
-                      const matrix::Fbcsr<ValueType, IndexType> *orig,
-                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
+                      const matrix::Fbcsr<ValueType, IndexType> *const orig,
+                      matrix::Diagonal<ValueType> *const diag)
+    GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL);
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index 6590c895376..441c0b10d28 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -80,12 +80,10 @@ void spmv(const std::shared_ptr<const ReferenceExecutor>,
     const blockutils::DenseBlocksView<const ValueType, IndexType> avalues(
         vals, bs, bs);
 
-    auto *const cvals = c->get_values();
-
     for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
         for (IndexType i = ibrow * bs * nvecs; i < (ibrow + 1) * bs * nvecs;
              ++i)
-            cvals[i] = zero<ValueType>();
+            c->get_values()[i] = zero<ValueType>();
 
         for (IndexType inz = row_ptrs[ibrow]; inz < row_ptrs[ibrow + 1];
              ++inz) {
@@ -124,12 +122,10 @@ void advanced_spmv(const std::shared_ptr<const ReferenceExecutor>,
     const blockutils::DenseBlocksView<const ValueType, IndexType> avalues(
         vals, bs, bs);
 
-    auto *const cvals = c->get_values();
-
     for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
         for (IndexType i = ibrow * bs * nvecs; i < (ibrow + 1) * bs * nvecs;
              ++i)
-            cvals[i] *= vbeta;
+            c->get_values()[i] *= vbeta;
 
         for (IndexType inz = row_ptrs[ibrow]; inz < row_ptrs[ibrow + 1];
              ++inz) {
@@ -165,7 +161,7 @@ void convert_to_dense(const std::shared_ptr<const ReferenceExecutor>,
     const gko::blockutils::DenseBlocksView<const ValueType, IndexType> values(
         vals, bs, bs);
 
-    for (size_type brow = 0; brow < nbrows; ++brow) {
+    for (IndexType brow = 0; brow < nbrows; ++brow) {
         for (size_type bcol = 0; bcol < nbcols; ++bcol) {
             for (int ib = 0; ib < bs; ib++)
                 for (int jb = 0; jb < bs; jb++)
@@ -212,7 +208,7 @@ void convert_to_csr(const std::shared_ptr<const ReferenceExecutor>,
     const gko::blockutils::DenseBlocksView<const ValueType, IndexType> bvalues(
         bvals, bs, bs);
 
-    for (size_type brow = 0; brow < nbrows; ++brow) {
+    for (IndexType brow = 0; brow < nbrows; ++brow) {
         const IndexType nz_browstart = browptrs[brow] * bs * bs;
         const IndexType numblocks_brow = browptrs[brow + 1] - browptrs[brow];
         for (int ib = 0; ib < bs; ib++)

From 85a3f0626be2cddca816e4be2cb0f23c0fe1a66f Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Tue, 19 Jan 2021 11:02:15 +0100
Subject: [PATCH 0569/1008] added test for fbcsr calculate max nnz per row

---
 reference/test/matrix/fbcsr_kernels.cpp | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index e9fff20c1ec..3e1a9eed629 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -555,6 +555,22 @@ TYPED_TEST(Fbcsr, CalculatesNonzerosPerRow)
 }
 
 
+TYPED_TEST(Fbcsr, CalculatesMaxNnzPerRow)
+{
+    using IndexType = typename TestFixture::index_type;
+    gko::size_type max_row_nnz{};
+
+    gko::kernels::reference::fbcsr::calculate_max_nnz_per_row(
+        this->exec, this->mtx2.get(), &max_row_nnz);
+
+    gko::size_type ref_max_row_nnz{};
+    gko::kernels::reference::csr::calculate_max_nnz_per_row(
+        this->exec, this->ref2csrmtx.get(), &ref_max_row_nnz);
+
+    ASSERT_EQ(max_row_nnz, ref_max_row_nnz);
+}
+
+
 TYPED_TEST(Fbcsr, SquareMtxIsTransposable)
 {
     using Fbcsr = typename TestFixture::Mtx;

From f42f2e40b18c7ab050cec67c6b842c7dad6e8907 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Tue, 19 Jan 2021 18:07:03 +0100
Subject: [PATCH 0570/1008] fixed doc-string as suggested in review

---
 core/matrix/fbcsr.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index f0f03224ada..c7d9c6b7b55 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -85,9 +85,10 @@ namespace detail {
 
 
 /**
+ * @internal
  * A lightweight dynamic block type on the host
  *
- * @internal Currently used only while reading a FBCSR matrix from matrix_data.
+ * Currently used only while reading a FBCSR matrix from matrix_data.
  *
  * @tparam ValueType  The numeric type of entries of the block
  */

From db9e55a6661b76daa0698283c51156f595cb008f Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Tue, 19 Jan 2021 21:16:09 +0100
Subject: [PATCH 0571/1008] added reference kernel for sorting fbcsr matrix by
 column

---
 core/device_hooks/common_kernels.inc.cpp |  6 +++
 core/matrix/fbcsr.cpp                    |  7 +++-
 core/matrix/fbcsr_kernels.hpp            |  6 +++
 core/test/matrix/fbcsr_sample.hpp        | 46 ++++++++++++++++++++++
 cuda/matrix/fbcsr_kernels.cu             |  9 +++++
 dpcpp/matrix/fbcsr_kernels.dp.cpp        |  9 +++++
 hip/matrix/fbcsr_kernels.hip.cpp         |  9 +++++
 omp/matrix/fbcsr_kernels.cpp             |  9 +++++
 reference/matrix/fbcsr_kernels.cpp       | 50 ++++++++++++++++++++++++
 reference/test/matrix/fbcsr_kernels.cpp  | 17 ++++++++
 10 files changed, 167 insertions(+), 1 deletion(-)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index b2818c4959a..81d74a33f88 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -860,6 +860,12 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
 
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
+
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index c7d9c6b7b55..341840e215a 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -70,6 +70,7 @@ GKO_REGISTER_OPERATION(calculate_nonzeros_per_row,
                        fbcsr::calculate_nonzeros_per_row);
 GKO_REGISTER_OPERATION(is_sorted_by_column_index,
                        fbcsr::is_sorted_by_column_index);
+GKO_REGISTER_OPERATION(sort_by_column_index, fbcsr::sort_by_column_index);
 GKO_REGISTER_OPERATION(extract_diagonal, fbcsr::extract_diagonal);
 GKO_REGISTER_OPERATION(fill_array, components::fill_array);
 GKO_REGISTER_OPERATION(inplace_absolute_array,
@@ -447,7 +448,11 @@ std::unique_ptr<LinOp> Fbcsr<ValueType, IndexType>::conj_transpose() const
 
 
 template <typename ValueType, typename IndexType>
-void Fbcsr<ValueType, IndexType>::sort_by_column_index() GKO_NOT_IMPLEMENTED;
+void Fbcsr<ValueType, IndexType>::sort_by_column_index()
+{
+    auto exec = this->get_executor();
+    exec->run(fbcsr::make_sort_by_column_index(this));
+}
 
 
 template <typename ValueType, typename IndexType>
diff --git a/core/matrix/fbcsr_kernels.hpp b/core/matrix/fbcsr_kernels.hpp
index 332292eb314..5d2492a3b37 100644
--- a/core/matrix/fbcsr_kernels.hpp
+++ b/core/matrix/fbcsr_kernels.hpp
@@ -95,6 +95,10 @@ namespace kernels {
         const matrix::Fbcsr<ValueType, IndexType> *source,             \
         Array<size_type> *result)
 
+#define GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX(ValueType, IndexType)       \
+    void sort_by_column_index(std::shared_ptr<const DefaultExecutor> exec, \
+                              matrix::Fbcsr<ValueType, IndexType> *to_sort)
+
 #define GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX(ValueType, IndexType) \
     void is_sorted_by_column_index(                                       \
         std::shared_ptr<const DefaultExecutor> exec,                      \
@@ -125,6 +129,8 @@ namespace kernels {
     template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX(ValueType, IndexType);         \
     template <typename ValueType, typename IndexType>                          \
+    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX(ValueType, IndexType);              \
+    template <typename ValueType, typename IndexType>                          \
     GKO_DECLARE_FBCSR_EXTRACT_DIAGONAL(ValueType, IndexType)
 
 
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index ecd0aaff7f6..4f6bcc864f2 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -512,6 +512,52 @@ class FbcsrSampleComplex {
     }
 };
 
+
+/**
+ * Generates a fixed-block CSR matrix with longer and unsorted columns
+ */
+template <typename ValueType, typename IndexType>
+class FbcsrSampleUnsorted {
+public:
+    using value_type = ValueType;
+    using index_type = IndexType;
+    using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
+
+
+    const size_type nbrows = 3;
+    const size_type nbcols = 20;
+    const size_type nbnz = 30;
+    const int bs = 3;
+    const size_type nrows = nbrows * bs;
+    const size_type ncols = nbcols * bs;
+    const size_type nnz = nbnz * bs * bs;
+    const std::shared_ptr<const gko::Executor> exec;
+
+
+    FbcsrSampleUnsorted(std::shared_ptr<const gko::ReferenceExecutor> rexec)
+        : exec(rexec)
+    {}
+
+    std::unique_ptr<Fbcsr> generate_fbcsr() const
+    {
+        gko::Array<index_type> r(exec, {0, 8, 19, 30});
+        gko::Array<index_type> c(
+            exec, {0,  1,  20, 15, 12, 18, 5, 28, 3,  10, 29, 5,  9,  2,  16,
+                   12, 21, 2,  0,  1,  5,  9, 12, 15, 17, 20, 22, 24, 27, 28});
+        gko::Array<value_type> vals(exec, nnz);
+        value_type *const v = vals.get_data();
+        for (IndexType i = 0; i < nnz; i++) {
+            v[i] = static_cast<value_type>(i + 0.15 + FBCSR_TEST_OFFSET);
+        }
+
+        return Fbcsr::create(exec,
+                             gko::dim<2>{static_cast<size_type>(nrows),
+                                         static_cast<size_type>(ncols)},
+                             bs, vals, c, r);
+    }
+};
+
+
 }  // namespace testing
 }  // namespace gko
 
diff --git a/cuda/matrix/fbcsr_kernels.cu b/cuda/matrix/fbcsr_kernels.cu
index b4da9345e48..056c90d57ed 100644
--- a/cuda/matrix/fbcsr_kernels.cu
+++ b/cuda/matrix/fbcsr_kernels.cu
@@ -169,6 +169,15 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
 
 
+template <typename ValueType, typename IndexType>
+void sort_by_column_index(const std::shared_ptr<const CudaExecutor> exec,
+                          matrix::Fbcsr<ValueType, IndexType> *const to_sort)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
+
+
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const CudaExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *orig,
diff --git a/dpcpp/matrix/fbcsr_kernels.dp.cpp b/dpcpp/matrix/fbcsr_kernels.dp.cpp
index b31bbc9a479..32244099d6b 100644
--- a/dpcpp/matrix/fbcsr_kernels.dp.cpp
+++ b/dpcpp/matrix/fbcsr_kernels.dp.cpp
@@ -165,6 +165,15 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
 
 
+template <typename ValueType, typename IndexType>
+void sort_by_column_index(const std::shared_ptr<const DpcppExecutor> exec,
+                          matrix::Fbcsr<ValueType, IndexType> *const to_sort)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
+
+
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *orig,
diff --git a/hip/matrix/fbcsr_kernels.hip.cpp b/hip/matrix/fbcsr_kernels.hip.cpp
index 3bf18bc6515..934b7f270cd 100644
--- a/hip/matrix/fbcsr_kernels.hip.cpp
+++ b/hip/matrix/fbcsr_kernels.hip.cpp
@@ -166,6 +166,15 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
 
 
+template <typename ValueType, typename IndexType>
+void sort_by_column_index(const std::shared_ptr<const HipExecutor> exec,
+                          matrix::Fbcsr<ValueType, IndexType> *const to_sort)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
+
+
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const HipExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *orig,
diff --git a/omp/matrix/fbcsr_kernels.cpp b/omp/matrix/fbcsr_kernels.cpp
index 52e8caefb37..f2747e12e18 100644
--- a/omp/matrix/fbcsr_kernels.cpp
+++ b/omp/matrix/fbcsr_kernels.cpp
@@ -165,6 +165,15 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
 
 
+template <typename ValueType, typename IndexType>
+void sort_by_column_index(const std::shared_ptr<const OmpExecutor> exec,
+                          matrix::Fbcsr<ValueType, IndexType> *const to_sort)
+    GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
+
+
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const OmpExecutor> exec,
                       const matrix::Fbcsr<ValueType, IndexType> *const orig,
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index 441c0b10d28..f56fa70315b 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -390,6 +390,56 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
 
 
+template <int matBlkSz, typename ValueType, typename IndexType>
+static void sort_by_column_index_impl(
+    matrix::Fbcsr<ValueType, IndexType> *const to_sort)
+{
+    auto row_ptrs = to_sort->get_const_row_ptrs();
+    auto col_idxs = to_sort->get_col_idxs();
+    auto values = to_sort->get_values();
+    const auto nbrows = to_sort->get_num_block_rows();
+    constexpr int bs2 = matBlkSz * matBlkSz;
+    for (IndexType i = 0; i < nbrows; ++i) {
+        IndexType *const brow_col_idxs = col_idxs + row_ptrs[i];
+        ValueType *const brow_vals = values + row_ptrs[i] * bs2;
+        const IndexType nbnz_brow = row_ptrs[i + 1] - row_ptrs[i];
+
+        std::vector<IndexType> col_permute(nbnz_brow);
+        std::iota(col_permute.begin(), col_permute.end(), 0);
+        auto helper = detail::IteratorFactory<IndexType, IndexType>(
+            brow_col_idxs, col_permute.data(), nbnz_brow);
+        std::sort(helper.begin(), helper.end());
+
+        std::vector<ValueType> oldvalues(nbnz_brow * bs2);
+        std::copy(brow_vals, brow_vals + nbnz_brow * bs2, oldvalues.begin());
+        for (IndexType ibz = 0; ibz < nbnz_brow; ibz++) {
+            for (int i = 0; i < bs2; i++) {
+                brow_vals[ibz * bs2 + i] =
+                    oldvalues[col_permute[ibz] * bs2 + i];
+            }
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void sort_by_column_index(const std::shared_ptr<const ReferenceExecutor> exec,
+                          matrix::Fbcsr<ValueType, IndexType> *const to_sort)
+{
+    const int bs = to_sort->get_block_size();
+    if (bs == 2)
+        sort_by_column_index_impl<2>(to_sort);
+    else if (bs == 3)
+        sort_by_column_index_impl<3>(to_sort);
+    else if (bs == 4)
+        sort_by_column_index_impl<4>(to_sort);
+    else
+        GKO_NOT_IMPLEMENTED;
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_FBCSR_SORT_BY_COLUMN_INDEX);
+
+
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const ReferenceExecutor>,
                       const matrix::Fbcsr<ValueType, IndexType> *const orig,
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 3e1a9eed629..1d639d0df7f 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -623,6 +623,23 @@ TYPED_TEST(Fbcsr, RecognizeUnsortedMatrix)
 }
 
 
+TYPED_TEST(Fbcsr, SortUnsortedMatrix)
+{
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using Mtx = typename TestFixture::Mtx;
+    const gko::testing::FbcsrSampleUnsorted<value_type, index_type> fbsample(
+        this->exec);
+    auto unsrt_mtx = fbsample.generate_fbcsr();
+    auto srt_mtx = unsrt_mtx->clone();
+
+    srt_mtx->sort_by_column_index();
+
+    GKO_ASSERT_MTX_NEAR(unsrt_mtx, srt_mtx, 0.0);
+    ASSERT_TRUE(srt_mtx->is_sorted_by_column_index());
+}
+
+
 TYPED_TEST(Fbcsr, ExtractsDiagonal)
 {
     using T = typename TestFixture::value_type;

From 153566918161e5835c17570667024e4cd0b3861c Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 20 Jan 2021 01:24:36 +0100
Subject: [PATCH 0572/1008] Addressed code structures issues raised in review

- blockutils.hpp is removed for now, its contents moved to
  pre-existing files
- added braces after control-flow statements
- More public functions in class Fbcsr are marked noexecpt
---
 core/matrix/fbcsr.cpp                   | 22 +++----
 core/test/matrix/fbcsr.cpp              | 12 ++--
 cuda/matrix/fbcsr_kernels.cu            |  7 ---
 include/ginkgo/core/base/blockutils.hpp | 84 -------------------------
 include/ginkgo/core/base/exception.hpp  | 24 +++++++
 include/ginkgo/core/matrix/fbcsr.hpp    | 60 +++++++++++++-----
 include/ginkgo/ginkgo.hpp               |  1 -
 reference/matrix/fbcsr_kernels.cpp      |  1 -
 reference/test/matrix/fbcsr_kernels.cpp |  3 +-
 9 files changed, 90 insertions(+), 124 deletions(-)
 delete mode 100644 include/ginkgo/core/base/blockutils.hpp

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 341840e215a..b9cc0d8d7ae 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -154,7 +154,7 @@ void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const b,
 {
     using Dense = Dense<ValueType>;
     if (auto b_fbcsr = dynamic_cast<const Fbcsr<ValueType, IndexType> *>(b)) {
-        // if b is a FBCSR matrix, we compute a SpGeMM
+        // if b is a FBCSR matrix, we need an SpGeMM
         throw NotImplemented(__FILE__, __LINE__, "SpGeMM for Fbcsr");
     } else {
         // otherwise we assume that b is dense and compute a SpMV/SpMM
@@ -172,10 +172,10 @@ void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const alpha,
 {
     using Dense = Dense<ValueType>;
     if (auto b_fbcsr = dynamic_cast<const Fbcsr<ValueType, IndexType> *>(b)) {
-        // if b is a FBCSR matrix, we compute a SpGeMM
+        // if b is a FBCSR matrix, we need an SpGeMM
         throw NotImplemented(__FILE__, __LINE__, "Adv SpGeMM for Fbcsr");
     } else if (dynamic_cast<const Identity<ValueType> *>(b)) {
-        // if b is an identity matrix, we compute an SpGEAM
+        // if b is an identity matrix, we need an SpGEAM
         throw NotImplemented(__FILE__, __LINE__, "Adv SpGEAM for Fbcsr");
     } else {
         // otherwise we assume that b is dense and compute a SpMV/SpMM
@@ -263,13 +263,12 @@ template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(
     SparsityCsr<ValueType, IndexType> *const result) const
 {
-    using blockutils::get_num_blocks;
     auto exec = this->get_executor();
     auto tmp = SparsityCsr<ValueType, IndexType>::create(
         exec,
-        gko::dim<2>{get_num_blocks(bs_, this->get_size()[0]),
-                    get_num_blocks(bs_, this->get_size()[1])},
-        get_num_blocks(bs_ * bs_, this->get_num_stored_elements()));
+        gko::dim<2>{static_cast<size_type>(this->get_num_block_rows()),
+                    static_cast<size_type>(this->get_num_block_cols())},
+        this->get_num_stored_blocks());
 
     tmp->col_idxs_ = this->col_idxs_;
     tmp->row_ptrs_ = this->row_ptrs_;
@@ -298,7 +297,6 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
                          std::numeric_limits<index_type>::max());
 
     const auto nnz = static_cast<index_type>(data.nonzeros.size());
-
     const int bs = this->bs_;
 
     using Block_t = detail::DenseBlock<value_type>;
@@ -362,11 +360,11 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
         GKO_ENSURE_IN_BOUNDS(cur_brow, num_brows);
 
         tmp->col_idxs_.get_data()[cur_bnz] = it->first.block_column;
-
-        for (int ibr = 0; ibr < bs; ibr++)
-            for (int jbr = 0; jbr < bs; jbr++)
+        for (int ibr = 0; ibr < bs; ibr++) {
+            for (int jbr = 0; jbr < bs; jbr++) {
                 values(cur_bnz, ibr, jbr) = it->second(ibr, jbr);
-
+            }
+        }
         if (it->first.block_row > cur_brow) {
             tmp->row_ptrs_.get_data()[++cur_brow] = cur_bnz;
         } else {
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index e7244220d17..3330ab2237a 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -311,8 +311,9 @@ TYPED_TEST(Fbcsr, GeneratesCorrectMatrixData)
 
     ASSERT_EQ(data.size, refdata.size);
     ASSERT_EQ(data.nonzeros.size(), refdata.nonzeros.size());
-    for (size_t i = 0; i < data.nonzeros.size(); i++)
+    for (size_t i = 0; i < data.nonzeros.size(); i++) {
         ASSERT_EQ(data.nonzeros[i], refdata.nonzeros[i]);
+    }
 }
 
 
@@ -332,10 +333,13 @@ TYPED_TEST(Fbcsr, DenseBlocksViewWorksCorrectly)
     Dbv refdbv(ref_dbv_array.data(), fbsample.bs, fbsample.bs);
     fbsample.fill_value_blocks_view(refdbv);
 
-    for (index_type ibz = 0; ibz < fbsample.nbnz; ibz++)
-        for (int i = 0; i < fbsample.bs; ++i)
-            for (int j = 0; j < fbsample.bs; ++j)
+    for (index_type ibz = 0; ibz < fbsample.nbnz; ibz++) {
+        for (int i = 0; i < fbsample.bs; ++i) {
+            for (int j = 0; j < fbsample.bs; ++j) {
                 ASSERT_EQ(testdbv(ibz, i, j), refdbv(ibz, i, j));
+            }
+        }
+    }
 }
 
 
diff --git a/cuda/matrix/fbcsr_kernels.cu b/cuda/matrix/fbcsr_kernels.cu
index 056c90d57ed..2d3677ea6bb 100644
--- a/cuda/matrix/fbcsr_kernels.cu
+++ b/cuda/matrix/fbcsr_kernels.cu
@@ -57,13 +57,6 @@ namespace cuda {
 namespace fbcsr {
 
 
-constexpr int default_block_size = 512;
-constexpr int warps_in_block = 4;
-constexpr int spmv_block_size = warps_in_block * config::warp_size;
-constexpr int wsize = config::warp_size;
-constexpr int classical_overweight = 32;
-
-
 /**
  * A compile-time list of the number items per threads for which spmv kernel
  * should be compiled.
diff --git a/include/ginkgo/core/base/blockutils.hpp b/include/ginkgo/core/base/blockutils.hpp
deleted file mode 100644
index 20c6ac5162a..00000000000
--- a/include/ginkgo/core/base/blockutils.hpp
+++ /dev/null
@@ -1,84 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GINKGO_PUBLIC_CORE_BASE_BLOCKUTILS_HPP_
-#define GINKGO_PUBLIC_CORE_BASE_BLOCKUTILS_HPP_
-
-
-#include <ginkgo/core/base/exception.hpp>
-
-
-namespace gko {
-namespace blockutils {
-
-
-/**
- * Error that denotes issues between block sizes and matrix dimensions
- *
- * \tparam IntexType  Type of index used by the linear algebra object that is
- *                    incompatible with the requried block size.
- */
-template <typename IndexType>
-class BlockSizeError : public Error {
-public:
-    BlockSizeError(const std::string &file, const int line,
-                   const int block_size, const IndexType size)
-        : Error(file, line,
-                " block size = " + std::to_string(block_size) +
-                    ", size = " + std::to_string(size))
-    {}
-};
-
-
-/**
- * Computes the number of blocks
- *
- * @param  block_size The size of each block
- * @param  size The total size of some array/vector
- * @return  The number of blocks, ie.,
- *          quotient of the size divided by the block size.
- *
- * @throw BlockSizeError  when block_size does not divide the total size.
- */
-template <typename IndexType>
-IndexType get_num_blocks(const int block_size, const IndexType size)
-{
-    if (size % block_size != 0)
-        throw BlockSizeError<IndexType>(__FILE__, __LINE__, block_size, size);
-    return size / block_size;
-}
-
-
-}  // namespace blockutils
-}  // namespace gko
-
-#endif
diff --git a/include/ginkgo/core/base/exception.hpp b/include/ginkgo/core/base/exception.hpp
index 46fc9db93af..e6e4076164c 100644
--- a/include/ginkgo/core/base/exception.hpp
+++ b/include/ginkgo/core/base/exception.hpp
@@ -421,6 +421,30 @@ class BadDimension : public Error {
 };
 
 
+/**
+ * Error that denotes issues between block sizes and matrix dimensions
+ *
+ * \tparam IntexType  Type of index used by the linear algebra object that is
+ *                    incompatible with the requried block size.
+ */
+template <typename IndexType>
+class BlockSizeError : public Error {
+public:
+    /**
+     * @param file  The name of the offending source file
+     * @param line  The source code line number where the error occurred
+     * @param block_size  Size of small dense blocks in a matrix
+     * @param size  The size that is not exactly divided by the block size
+     */
+    BlockSizeError(const std::string &file, const int line,
+                   const int block_size, const IndexType size)
+        : Error(file, line,
+                " block size = " + std::to_string(block_size) +
+                    ", size = " + std::to_string(size))
+    {}
+};
+
+
 /**
  * ValueMismatch is thrown if two values are not equal.
  */
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index 9e5423395ba..ba912d1e034 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -35,7 +35,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/blockutils.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
 #include <ginkgo/core/base/math.hpp>
 
@@ -60,6 +59,32 @@ template <typename ValueType, typename IndexType>
 class FbcsrBuilder;
 
 
+namespace detail {
+
+
+/**
+ * Computes the number of blocks in some array of given size
+ *
+ * @param  block_size The size of each block
+ * @param  size The total size of some array/vector
+ * @return  The number of blocks, ie.,
+ *          quotient of the size divided by the block size.
+ *
+ * @throw BlockSizeError  when block_size does not divide the total size.
+ */
+template <typename IndexType>
+IndexType get_num_blocks(const int block_size, const IndexType size)
+{
+    if (size % block_size != 0) {
+        throw BlockSizeError<IndexType>(__FILE__, __LINE__, block_size, size);
+    }
+    return size / block_size;
+}
+
+
+}  // namespace detail
+
+
 /**
  * @brief Fixed-block compressed sparse row storage matrix format
  *
@@ -189,7 +214,8 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     void compute_absolute_inplace() override;
 
     /**
-     * Sorts all (value, col_idx) pairs in each row by column index
+     * Sorts the values blocks and block-column indices in each row
+     * by column index
      */
     void sort_by_column_index();
 
@@ -253,31 +279,37 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     }
 
     /**
-     * Returns the number of elements explicitly stored in the matrix.
-     *
-     * @return the number of elements explicitly stored in the matrix
+     * @return  The number of elements explicitly stored in the matrix
      */
     size_type get_num_stored_elements() const noexcept
     {
         return values_.get_num_elems();
     }
 
+    /**
+     * @return  The number of non-zero blocks explicitly stored in the matrix
+     */
+    size_type get_num_stored_blocks() const noexcept
+    {
+        return col_idxs_.get_num_elems();
+    }
+
     /**
      * @return The fixed block size for this matrix
      */
-    int get_block_size() const { return bs_; }
+    int get_block_size() const noexcept { return bs_; }
 
     /**
      * Set the fixed block size for this matrix
      *
      * @param block_size The block size
      */
-    void set_block_size(const int block_size) { bs_ = block_size; }
+    void set_block_size(const int block_size) noexcept { bs_ = block_size; }
 
     /**
      * @return The number of block-rows in the matrix
      */
-    index_type get_num_block_rows() const
+    index_type get_num_block_rows() const noexcept
     {
         return row_ptrs_.get_num_elems() - 1;
     }
@@ -285,7 +317,7 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     /**
      * @return The number of block-columns in the matrix
      */
-    index_type get_num_block_cols() const { return nbcols_; }
+    index_type get_num_block_cols() const noexcept { return nbcols_; }
 
 protected:
     /**
@@ -312,11 +344,11 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
         : EnableLinOp<Fbcsr>(exec, size),
           bs_{block_size},
           nbcols_{static_cast<index_type>(
-              blockutils::get_num_blocks(block_size, size[1]))},
+              detail::get_num_blocks(block_size, size[1]))},
           values_(exec, num_nonzeros),
-          col_idxs_(exec, blockutils::get_num_blocks(block_size * block_size,
-                                                     num_nonzeros)),
-          row_ptrs_(exec, blockutils::get_num_blocks(block_size, size[0]) + 1)
+          col_idxs_(exec, detail::get_num_blocks(block_size * block_size,
+                                                 num_nonzeros)),
+          row_ptrs_(exec, detail::get_num_blocks(block_size, size[0]) + 1)
     {}
 
     /**
@@ -348,7 +380,7 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
         : EnableLinOp<Fbcsr>(exec, size),
           bs_{block_size},
           nbcols_{static_cast<index_type>(
-              blockutils::get_num_blocks(block_size, size[1]))},
+              detail::get_num_blocks(block_size, size[1]))},
           values_{exec, std::forward<ValuesArray>(values)},
           col_idxs_{exec, std::forward<ColIdxsArray>(col_idxs)},
           row_ptrs_{exec, std::forward<RowPtrsArray>(row_ptrs)}
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 38ac3fd861c..a2b62ec8671 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -39,7 +39,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/abstract_factory.hpp>
 #include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/blockutils.hpp>
 #include <ginkgo/core/base/combination.hpp>
 #include <ginkgo/core/base/composition.hpp>
 #include <ginkgo/core/base/dim.hpp>
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index f56fa70315b..39ddaa91116 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -40,7 +40,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/blockutils.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 1d639d0df7f..341ed67d690 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -648,9 +648,10 @@ TYPED_TEST(Fbcsr, ExtractsDiagonal)
     auto diag = matrix->extract_diagonal();
 
     ASSERT_EQ(this->m2diag->get_size(), diag->get_size());
-    for (gko::size_type i = 0; i < this->m2diag->get_size()[0]; i++)
+    for (gko::size_type i = 0; i < this->m2diag->get_size()[0]; i++) {
         ASSERT_EQ(this->m2diag->get_const_values()[i],
                   diag->get_const_values()[i]);
+    }
 }
 
 

From 9a98f437e7e5c221a667ac81ee2d763d81735a4a Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Mon, 25 Jan 2021 21:17:47 +0100
Subject: [PATCH 0573/1008] Improved test coverage

Added tests for BlockSizeError, getNumBlocks and especially FbcsrSample
classes.
---
 core/test/base/exception.cpp           |   8 +
 core/test/matrix/fbcsr.cpp             | 268 +++++++++++++++++++------
 cuda/test/matrix/CMakeLists.txt        |   1 +
 cuda/test/matrix/fbcsr_kernels.cpp     |  93 +++++++++
 include/ginkgo/core/base/exception.hpp |   4 +-
 5 files changed, 315 insertions(+), 59 deletions(-)
 create mode 100644 cuda/test/matrix/fbcsr_kernels.cpp

diff --git a/core/test/base/exception.cpp b/core/test/base/exception.cpp
index 2c1b672f1db..bbbe623dc30 100644
--- a/core/test/base/exception.cpp
+++ b/core/test/base/exception.cpp
@@ -148,6 +148,14 @@ TEST(ExceptionClasses, DimensionMismatchReturnsCorrectWhatMessage)
 }
 
 
+TEST(ExceptionClasses, BlockSizeErrorCorrectWhatMessage)
+{
+    gko::BlockSizeError<int> error("test_file.cpp", 243, 3, 20);
+    ASSERT_EQ(std::string("test_file.cpp:243: block size = 3, size = 20"),
+              error.what());
+}
+
+
 TEST(ExceptionClasses, AllocationErrorReturnsCorrectWhatMessage)
 {
     gko::AllocationError error("test_file.cpp", 42, "OMP", 135);
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index 3330ab2237a..c4a7e3c3420 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -48,6 +48,192 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
+template <typename ValueType, typename IndexType>
+void assert_matrices_are_same(
+    const gko::matrix::Fbcsr<ValueType, IndexType> *const bm,
+    const gko::matrix::Csr<ValueType, IndexType> *const cm,
+    const gko::matrix::Diagonal<ValueType> *const diam = nullptr,
+    const gko::matrix_data<ValueType, IndexType> *const md = nullptr)
+{
+    if (cm) {
+        ASSERT_EQ(bm->get_size(), cm->get_size());
+        ASSERT_EQ(bm->get_num_stored_elements(), cm->get_num_stored_elements());
+    }
+    if (md) {
+        ASSERT_EQ(bm->get_size(), md->size);
+        ASSERT_EQ(bm->get_num_stored_elements(), md->nonzeros.size());
+    }
+    if (diam) {
+        const gko::size_type minsize =
+            std::min(bm->get_size()[0], bm->get_size()[1]);
+        ASSERT_EQ(minsize, diam->get_size()[0]);
+        ASSERT_EQ(minsize, diam->get_size()[1]);
+    }
+
+    const IndexType nbrows = bm->get_num_block_rows();
+    const int bs = bm->get_block_size();
+    gko::blockutils::DenseBlocksView<const ValueType, IndexType> fbvals(
+        bm->get_const_values(), bs, bs);
+
+    for (IndexType ibrow = 0; ibrow < nbrows; ibrow++) {
+        const IndexType *const browptr = bm->get_const_row_ptrs();
+        const IndexType numblocksbrow = browptr[ibrow + 1] - browptr[ibrow];
+        for (IndexType irow = ibrow * bs; irow < ibrow * bs + bs; irow++) {
+            const IndexType rowstart = browptr[ibrow] * bs * bs +
+                                       (irow - ibrow * bs) * numblocksbrow * bs;
+            if (cm) {
+                ASSERT_EQ(cm->get_const_row_ptrs()[irow], rowstart);
+            }
+        }
+
+        const IndexType iz_browstart = browptr[ibrow] * bs * bs;
+        const IndexType *const bcolinds = bm->get_const_col_idxs();
+
+        for (IndexType ibnz = browptr[ibrow]; ibnz < browptr[ibrow + 1];
+             ibnz++) {
+            const IndexType bcol = bcolinds[ibnz];
+            const IndexType blkoffset_frombrowstart = ibnz - browptr[ibrow];
+
+            for (int ib = 0; ib < bs; ib++) {
+                const IndexType row = ibrow * bs + ib;
+                const IndexType inz_rowstart =
+                    iz_browstart + ib * numblocksbrow * bs;
+                const IndexType inz_blockstart_row =
+                    inz_rowstart + blkoffset_frombrowstart * bs;
+
+                for (int jb = 0; jb < bs; jb++) {
+                    const IndexType col = bcol * bs + jb;
+                    const IndexType inz = inz_blockstart_row + jb;
+                    if (cm) {
+                        ASSERT_EQ(col, cm->get_const_col_idxs()[inz]);
+                        ASSERT_EQ(fbvals(ibnz, ib, jb),
+                                  cm->get_const_values()[inz]);
+                    }
+                    if (md) {
+                        ASSERT_EQ(row, md->nonzeros[inz].row);
+                        ASSERT_EQ(col, md->nonzeros[inz].column);
+                        ASSERT_EQ(fbvals(ibnz, ib, jb),
+                                  md->nonzeros[inz].value);
+                    }
+                    if (row == col && diam) {
+                        ASSERT_EQ(fbvals(ibnz, ib, jb),
+                                  diam->get_const_values()[row]);
+                    }
+                }
+            }
+        }
+    }
+}
+
+
+template <typename SampleGenerator>
+void check_sample_generator_common(const SampleGenerator sg)
+{
+    auto fbmtx = sg.generate_fbcsr();
+    ASSERT_EQ(fbmtx->get_num_block_rows(), sg.nbrows);
+    ASSERT_EQ(fbmtx->get_num_block_cols(), sg.nbcols);
+    ASSERT_EQ(fbmtx->get_size()[0], sg.nrows);
+    ASSERT_EQ(fbmtx->get_size()[1], sg.ncols);
+    ASSERT_EQ(fbmtx->get_num_stored_blocks(), sg.nbnz);
+    ASSERT_EQ(fbmtx->get_num_stored_elements(), sg.nnz);
+}
+
+
+template <typename ValueIndexType>
+class FbcsrSample : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+
+    FbcsrSample() : ref(gko::ReferenceExecutor::create()) {}
+
+    std::shared_ptr<const gko::ReferenceExecutor> ref;
+};
+
+
+TYPED_TEST_SUITE(FbcsrSample, gko::test::ValueIndexTypes);
+
+
+TYPED_TEST(FbcsrSample, SampleGeneratorsAreCorrect)
+{
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using Mtx = gko::matrix::Fbcsr<value_type, index_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    using MtxData = gko::matrix_data<value_type, index_type>;
+    using Diag = gko::matrix::Diagonal<value_type>;
+    auto ref = this->ref;
+    gko::testing::FbcsrSample<value_type, index_type> fbsample(ref);
+    gko::testing::FbcsrSample2<value_type, index_type> fbsample2(ref);
+
+    std::unique_ptr<const Mtx> fbmtx = fbsample.generate_fbcsr();
+    std::unique_ptr<const Csr> csmtx = fbsample.generate_csr();
+    const MtxData mdata = fbsample.generate_matrix_data_with_explicit_zeros();
+    std::unique_ptr<const Mtx> fbmtx2 = fbsample2.generate_fbcsr();
+    std::unique_ptr<const Csr> csmtx2 = fbsample2.generate_csr();
+    std::unique_ptr<const Diag> diag2 = fbsample2.extract_diagonal();
+    const gko::Array<index_type> nnzperrow = fbsample2.getNonzerosPerRow();
+
+    check_sample_generator_common(fbsample);
+    assert_matrices_are_same(fbmtx.get(), csmtx.get(),
+                             static_cast<const Diag *>(nullptr), &mdata);
+    check_sample_generator_common(fbsample2);
+    assert_matrices_are_same(fbmtx2.get(), csmtx2.get(), diag2.get());
+    for (index_type irow = 0; irow < fbsample2.nrows; irow++) {
+        const index_type *const row_ptrs = csmtx2->get_const_row_ptrs();
+        const index_type num_nnz_row = row_ptrs[irow + 1] - row_ptrs[irow];
+        ASSERT_EQ(nnzperrow.get_const_data()[irow], num_nnz_row);
+        for (index_type iz = row_ptrs[irow]; iz < row_ptrs[irow + 1]; iz++) {
+            const index_type col = csmtx2->get_const_col_idxs()[iz];
+            if (irow == col) {
+                ASSERT_EQ(csmtx2->get_const_values()[iz],
+                          diag2->get_const_values()[irow]);
+            }
+        }
+    }
+    check_sample_generator_common(
+        gko::testing::FbcsrSampleUnsorted<value_type, index_type>(ref));
+    check_sample_generator_common(
+        gko::testing::FbcsrSampleSquare<value_type, index_type>(ref));
+}
+
+
+template <typename ValueIndexType>
+class FbcsrSampleComplex : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+
+    FbcsrSampleComplex() : ref(gko::ReferenceExecutor::create()) {}
+
+    std::shared_ptr<const gko::ReferenceExecutor> ref;
+};
+
+
+TYPED_TEST_SUITE(FbcsrSampleComplex, gko::test::ComplexValueIndexTypes);
+
+
+TYPED_TEST(FbcsrSampleComplex, ComplexSampleGeneratorIsCorrect)
+{
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using Mtx = gko::matrix::Fbcsr<value_type, index_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    auto ref = this->ref;
+    gko::testing::FbcsrSampleComplex<value_type, index_type> fbsample3(ref);
+
+    std::unique_ptr<const Mtx> fbmtx3 = fbsample3.generate_fbcsr();
+    std::unique_ptr<const Csr> csmtx3 = fbsample3.generate_csr();
+
+    check_sample_generator_common(fbsample3);
+    assert_matrices_are_same(fbmtx3.get(), csmtx3.get());
+}
+
+
 template <typename ValueIndexType>
 class Fbcsr : public ::testing::Test {
 protected:
@@ -56,11 +242,12 @@ class Fbcsr : public ::testing::Test {
     using index_type =
         typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using Mtx = gko::matrix::Fbcsr<value_type, index_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    using MtxData = gko::matrix_data<value_type, index_type>;
 
     Fbcsr()
         : exec(gko::ReferenceExecutor::create()),
-          fbsample(
-              std::static_pointer_cast<const gko::ReferenceExecutor>(exec)),
+          fbsample(exec),
           mtx(fbsample.generate_fbcsr())
     {
         // backup for move tests
@@ -76,7 +263,7 @@ class Fbcsr : public ::testing::Test {
         std::copy(v, v + fbsample.nnz, orig_vals.data());
     }
 
-    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<const gko::ReferenceExecutor> exec;
     const gko::testing::FbcsrSample<value_type, index_type> fbsample;
     std::unique_ptr<Mtx> mtx;
 
@@ -91,7 +278,7 @@ class Fbcsr : public ::testing::Test {
         auto c = m->get_const_col_idxs();
         auto r = m->get_const_row_ptrs();
 
-        const int bs = 3;
+        const int bs = fbsample.bs;
 
         ASSERT_EQ(m->get_size(), orig_size);
         ASSERT_EQ(m->get_num_stored_elements(), orig_vals.size());
@@ -99,7 +286,6 @@ class Fbcsr : public ::testing::Test {
         ASSERT_EQ(m->get_num_block_rows(), m->get_size()[0] / bs);
         ASSERT_EQ(m->get_num_block_cols(), m->get_size()[1] / bs);
 
-
         for (index_type irow = 0; irow < orig_size[0] / bs; irow++) {
             const index_type *const rowptr = &orig_rowptrs[0];
             ASSERT_EQ(r[irow], rowptr[irow]);
@@ -129,54 +315,24 @@ class Fbcsr : public ::testing::Test {
 TYPED_TEST_SUITE(Fbcsr, gko::test::ValueIndexTypes);
 
 
-TYPED_TEST(Fbcsr, SampleGeneratorIsCorrect)
+TYPED_TEST(Fbcsr, GetNumBlocksCorrectlyThrows)
 {
-    using Mtx = typename TestFixture::Mtx;
-    using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
-    using Csr = gko::matrix::Csr<value_type, index_type>;
+    const index_type vec_sz = 47;
+    const int blk_sz = 9;
 
-    std::unique_ptr<const Mtx> fbmtx = this->fbsample.generate_fbcsr();
-    std::unique_ptr<const Csr> csmtx = this->fbsample.generate_csr();
-    const int bs = this->fbsample.bs;
-    ASSERT_EQ(bs, fbmtx->get_block_size());
-
-    gko::blockutils::DenseBlocksView<const value_type, index_type> fbvals(
-        fbmtx->get_const_values(), bs, bs);
-
-    for (index_type ibrow = 0; ibrow < this->fbsample.nbrows; ibrow++) {
-        const index_type *const browptr = fbmtx->get_const_row_ptrs();
-        const index_type numblocksbrow = browptr[ibrow + 1] - browptr[ibrow];
-        for (index_type irow = ibrow * bs; irow < ibrow * bs + bs; irow++) {
-            const index_type rowstart =
-                browptr[ibrow] * bs * bs +
-                (irow - ibrow * bs) * numblocksbrow * bs;
-            ASSERT_EQ(csmtx->get_const_row_ptrs()[irow], rowstart);
-        }
-
-        const index_type *const bcolinds = fbmtx->get_const_col_idxs();
+    ASSERT_THROW(gko::matrix::detail::get_num_blocks(blk_sz, vec_sz),
+                 gko::BlockSizeError<index_type>);
+}
 
-        for (index_type ibnz = browptr[ibrow]; ibnz < browptr[ibrow + 1];
-             ibnz++) {
-            const index_type bcol = bcolinds[ibnz];
-            const index_type blkoffset_frombrowstart = ibnz - browptr[ibrow];
 
-            for (int ib = 0; ib < bs; ib++) {
-                const index_type row = ibrow * bs + ib;
-                const index_type inz_rowstart =
-                    csmtx->get_const_row_ptrs()[row] +
-                    blkoffset_frombrowstart * bs;
+TYPED_TEST(Fbcsr, GetNumBlocksWorks)
+{
+    using index_type = typename TestFixture::index_type;
+    const index_type vec_sz = 45;
+    const int blk_sz = 9;
 
-                for (int jb = 0; jb < bs; jb++) {
-                    const index_type col = bcol * bs + jb;
-                    const index_type inz = inz_rowstart + jb;
-                    ASSERT_EQ(col, csmtx->get_const_col_idxs()[inz]);
-                    ASSERT_EQ(fbvals(ibnz, ib, jb),
-                              csmtx->get_const_values()[inz]);
-                }
-            }
-        }
-    }
+    ASSERT_EQ(gko::matrix::detail::get_num_blocks(blk_sz, vec_sz), 5);
 }
 
 
@@ -220,7 +376,6 @@ TYPED_TEST(Fbcsr, CanBeCreatedFromExistingData)
     using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
     using size_type = gko::size_type;
-
     const int bs = this->fbsample.bs;
     const size_type nbrows = this->fbsample.nbrows;
     const size_type nbcols = this->fbsample.nbcols;
@@ -269,6 +424,7 @@ TYPED_TEST(Fbcsr, CanBeMoved)
 TYPED_TEST(Fbcsr, CanBeCloned)
 {
     using Mtx = typename TestFixture::Mtx;
+
     auto clone = this->mtx->clone();
 
     this->assert_equal_to_original_mtx(this->mtx.get());
@@ -301,11 +457,11 @@ TYPED_TEST(Fbcsr, GeneratesCorrectMatrixData)
 {
     using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
-    gko::matrix_data<value_type, index_type> refdata =
-        this->fbsample.generate_matrix_data_with_explicit_zeros();
+    using MtxData = typename TestFixture::MtxData;
+    MtxData refdata = this->fbsample.generate_matrix_data_with_explicit_zeros();
     refdata.ensure_row_major_order();
 
-    gko::matrix_data<value_type, index_type> data;
+    MtxData data;
     this->mtx->write(data);
     data.ensure_row_major_order();
 
@@ -322,17 +478,15 @@ TYPED_TEST(Fbcsr, DenseBlocksViewWorksCorrectly)
     using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
     using Dbv = gko::blockutils::DenseBlocksView<value_type, index_type>;
-
     const gko::testing::FbcsrSample2<value_type, index_type> fbsample(
-        std::static_pointer_cast<const gko::ReferenceExecutor>(this->exec));
-
-    auto refmtx = fbsample.generate_fbcsr();
-    const Dbv testdbv(refmtx->get_values(), fbsample.bs, fbsample.bs);
-
+        this->exec);
     std::vector<value_type> ref_dbv_array(fbsample.nnz);
     Dbv refdbv(ref_dbv_array.data(), fbsample.bs, fbsample.bs);
     fbsample.fill_value_blocks_view(refdbv);
 
+    auto refmtx = fbsample.generate_fbcsr();
+    const Dbv testdbv(refmtx->get_values(), fbsample.bs, fbsample.bs);
+
     for (index_type ibz = 0; ibz < fbsample.nbnz; ibz++) {
         for (int i = 0; i < fbsample.bs; ++i) {
             for (int j = 0; j < fbsample.bs; ++j) {
diff --git a/cuda/test/matrix/CMakeLists.txt b/cuda/test/matrix/CMakeLists.txt
index 65ce218ac71..a1d7ca4a7a0 100644
--- a/cuda/test/matrix/CMakeLists.txt
+++ b/cuda/test/matrix/CMakeLists.txt
@@ -1,5 +1,6 @@
 ginkgo_create_test(coo_kernels)
 ginkgo_create_test(csr_kernels)
+ginkgo_create_test(fbcsr_kernels)
 ginkgo_create_test(dense_kernels)
 ginkgo_create_test(diagonal_kernels)
 ginkgo_create_test(ell_kernels)
diff --git a/cuda/test/matrix/fbcsr_kernels.cpp b/cuda/test/matrix/fbcsr_kernels.cpp
new file mode 100644
index 00000000000..79b455cb21c
--- /dev/null
+++ b/cuda/test/matrix/fbcsr_kernels.cpp
@@ -0,0 +1,93 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/matrix/fbcsr.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+//#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include "core/test/matrix/fbcsr_sample.hpp"
+#include "cuda/test/utils.hpp"
+
+
+namespace {
+
+
+class Fbcsr : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Fbcsr<>;
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        cuda = gko::CudaExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (cuda != nullptr) {
+            ASSERT_NO_THROW(cuda->synchronize());
+        }
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::CudaExecutor> cuda;
+
+    std::unique_ptr<Mtx> mtx;
+};
+
+
+TEST_F(Fbcsr, CanWriteFromMatrixOnDevice)
+{
+    using value_type = Mtx::value_type;
+    using index_type = Mtx::index_type;
+    using MatData = gko::matrix_data<value_type, index_type>;
+    gko::testing::FbcsrSample<value_type, index_type> sample(ref);
+    auto refmat = sample.generate_fbcsr();
+    auto cudamat = Mtx::create(cuda);
+    cudamat->copy_from(gko::lend(refmat));
+
+    MatData refdata, cudadata;
+    refmat->write(refdata);
+    cudamat->write(cudadata);
+
+    ASSERT_TRUE(refdata.nonzeros == cudadata.nonzeros);
+}
+
+
+}  // namespace
diff --git a/include/ginkgo/core/base/exception.hpp b/include/ginkgo/core/base/exception.hpp
index e6e4076164c..7556602f13b 100644
--- a/include/ginkgo/core/base/exception.hpp
+++ b/include/ginkgo/core/base/exception.hpp
@@ -424,7 +424,7 @@ class BadDimension : public Error {
 /**
  * Error that denotes issues between block sizes and matrix dimensions
  *
- * \tparam IntexType  Type of index used by the linear algebra object that is
+ * \tparam IndexType  Type of index used by the linear algebra object that is
  *                    incompatible with the requried block size.
  */
 template <typename IndexType>
@@ -439,7 +439,7 @@ class BlockSizeError : public Error {
     BlockSizeError(const std::string &file, const int line,
                    const int block_size, const IndexType size)
         : Error(file, line,
-                " block size = " + std::to_string(block_size) +
+                "block size = " + std::to_string(block_size) +
                     ", size = " + std::to_string(size))
     {}
 };

From 00d0508d2357ed7ed9ec8e87a51ec8a061ac9570 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Tue, 26 Jan 2021 00:52:02 +0100
Subject: [PATCH 0574/1008] Addressed points raised in review

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 core/matrix/fbcsr.cpp                   | 21 +++------------
 include/ginkgo/core/matrix/fbcsr.hpp    |  9 ++++---
 reference/matrix/fbcsr_kernels.cpp      | 34 +++++++++++++++----------
 reference/test/matrix/fbcsr_kernels.cpp | 27 ++++----------------
 4 files changed, 36 insertions(+), 55 deletions(-)

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index b9cc0d8d7ae..99e6c9e8914 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -155,7 +155,7 @@ void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const b,
     using Dense = Dense<ValueType>;
     if (auto b_fbcsr = dynamic_cast<const Fbcsr<ValueType, IndexType> *>(b)) {
         // if b is a FBCSR matrix, we need an SpGeMM
-        throw NotImplemented(__FILE__, __LINE__, "SpGeMM for Fbcsr");
+        GKO_NOT_SUPPORTED(b_fbcsr);
     } else {
         // otherwise we assume that b is dense and compute a SpMV/SpMM
         this->get_executor()->run(
@@ -173,10 +173,10 @@ void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const alpha,
     using Dense = Dense<ValueType>;
     if (auto b_fbcsr = dynamic_cast<const Fbcsr<ValueType, IndexType> *>(b)) {
         // if b is a FBCSR matrix, we need an SpGeMM
-        throw NotImplemented(__FILE__, __LINE__, "Adv SpGeMM for Fbcsr");
-    } else if (dynamic_cast<const Identity<ValueType> *>(b)) {
+        GKO_NOT_SUPPORTED(b_fbcsr);
+    } else if (auto b_ident = dynamic_cast<const Identity<ValueType> *>(b)) {
         // if b is an identity matrix, we need an SpGEAM
-        throw NotImplemented(__FILE__, __LINE__, "Adv SpGEAM for Fbcsr");
+        GKO_NOT_SUPPORTED(b_ident);
     } else {
         // otherwise we assume that b is dense and compute a SpMV/SpMM
         this->get_executor()->run(
@@ -186,19 +186,6 @@ void Fbcsr<ValueType, IndexType>::apply_impl(const LinOp *const alpha,
 }
 
 
-template <typename ValueType, typename IndexType>
-void Fbcsr<ValueType, IndexType>::convert_to(
-    Fbcsr<ValueType, IndexType> *const result) const
-{
-    result->values_ = this->values_;
-    result->col_idxs_ = this->col_idxs_;
-    result->row_ptrs_ = this->row_ptrs_;
-    result->set_size(this->get_size());
-    result->bs_ = this->bs_;
-    result->nbcols_ = this->nbcols_;
-}
-
-
 template <typename ValueType, typename IndexType>
 void Fbcsr<ValueType, IndexType>::convert_to(
     Fbcsr<next_precision<ValueType>, IndexType> *const result) const
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index ba912d1e034..16a7dd6b726 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -152,9 +152,6 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
     using mat_data = matrix_data<ValueType, IndexType>;
     using absolute_type = remove_complex<Fbcsr>;
 
-
-    void convert_to(Fbcsr<ValueType, IndexType> *result) const override;
-
     /**
      * For moving to another Fbcsr of the same type, use the default
      *  implementation provided by EnableLinOp via the
@@ -162,6 +159,12 @@ class Fbcsr : public EnableLinOp<Fbcsr<ValueType, IndexType>>,
      */
     using EnableLinOp<Fbcsr<ValueType, IndexType>>::move_to;
 
+    /**
+     * For converting (copying) to another Fbcsr of the same type,
+     * use the default implementation provided by EnableLinOp.
+     */
+    using EnableLinOp<Fbcsr<ValueType, IndexType>>::convert_to;
+
     friend class Fbcsr<next_precision<ValueType>, IndexType>;
 
     void convert_to(
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index 39ddaa91116..da7877c2363 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -81,9 +81,9 @@ void spmv(const std::shared_ptr<const ReferenceExecutor>,
 
     for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
         for (IndexType i = ibrow * bs * nvecs; i < (ibrow + 1) * bs * nvecs;
-             ++i)
+             ++i) {
             c->get_values()[i] = zero<ValueType>();
-
+        }
         for (IndexType inz = row_ptrs[ibrow]; inz < row_ptrs[ibrow + 1];
              ++inz) {
             for (int ib = 0; ib < bs; ib++) {
@@ -91,8 +91,9 @@ void spmv(const std::shared_ptr<const ReferenceExecutor>,
                 for (int jb = 0; jb < bs; jb++) {
                     const auto val = avalues(inz, ib, jb);
                     const auto col = col_idxs[inz] * bs + jb;
-                    for (size_type j = 0; j < nvecs; ++j)
+                    for (size_type j = 0; j < nvecs; ++j) {
                         c->at(row, j) += val * b->at(col, j);
+                    }
                 }
             }
         }
@@ -162,18 +163,21 @@ void convert_to_dense(const std::shared_ptr<const ReferenceExecutor>,
 
     for (IndexType brow = 0; brow < nbrows; ++brow) {
         for (size_type bcol = 0; bcol < nbcols; ++bcol) {
-            for (int ib = 0; ib < bs; ib++)
-                for (int jb = 0; jb < bs; jb++)
+            for (int ib = 0; ib < bs; ib++) {
+                for (int jb = 0; jb < bs; jb++) {
                     result->at(brow * bs + ib, bcol * bs + jb) =
                         zero<ValueType>();
+                }
+            }
         }
         for (IndexType ibnz = row_ptrs[brow]; ibnz < row_ptrs[brow + 1];
              ++ibnz) {
             for (int ib = 0; ib < bs; ib++) {
                 const IndexType row = brow * bs + ib;
-                for (int jb = 0; jb < bs; jb++)
+                for (int jb = 0; jb < bs; jb++) {
                     result->at(row, col_idxs[ibnz] * bs + jb) =
                         values(ibnz, ib, jb);
+                }
             }
         }
     }
@@ -258,11 +262,13 @@ void convert_fbcsr_to_fbcsc(const IndexType num_blk_rows, const int blksz,
             const auto dest_idx = col_ptrs[col_idxs[i]];
             col_ptrs[col_idxs[i]]++;
             row_idxs[dest_idx] = brow;
-            for (int ib = 0; ib < blksz; ib++)
-                for (int jb = 0; jb < blksz; jb++)
+            for (int ib = 0; ib < blksz; ib++) {
+                for (int jb = 0; jb < blksz; jb++) {
                     cvalues(dest_idx, ib, jb) =
                         op(transpose_blocks ? rvalues(i, jb, ib)
                                             : rvalues(i, ib, jb));
+                }
+            }
         }
     }
 }
@@ -425,14 +431,15 @@ void sort_by_column_index(const std::shared_ptr<const ReferenceExecutor> exec,
                           matrix::Fbcsr<ValueType, IndexType> *const to_sort)
 {
     const int bs = to_sort->get_block_size();
-    if (bs == 2)
+    if (bs == 2) {
         sort_by_column_index_impl<2>(to_sort);
-    else if (bs == 3)
+    } else if (bs == 3) {
         sort_by_column_index_impl<3>(to_sort);
-    else if (bs == 4)
+    } else if (bs == 4) {
         sort_by_column_index_impl<4>(to_sort);
-    else
+    } else {
         GKO_NOT_IMPLEMENTED;
+    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -460,8 +467,9 @@ void extract_diagonal(std::shared_ptr<const ReferenceExecutor>,
         for (IndexType idx = row_ptrs[ibrow]; idx < row_ptrs[ibrow + 1];
              ++idx) {
             if (col_idxs[idx] == ibrow) {
-                for (int ib = 0; ib < bs; ib++)
+                for (int ib = 0; ib < bs; ib++) {
                     diag_values[ibrow * bs + ib] = vblocks(idx, ib, ib);
+                }
                 break;
             }
         }
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 341ed67d690..2ed4e2be557 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -159,7 +159,6 @@ TYPED_TEST(Fbcsr, AppliesToDenseVector)
     using Vec = typename TestFixture::Vec;
     using T = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
-
     const index_type nrows = this->mtx2->get_size()[0];
     const index_type ncols = this->mtx2->get_size()[1];
     auto x = Vec::create(this->exec, gko::dim<2>{(gko::size_type)ncols, 1});
@@ -168,13 +167,12 @@ TYPED_TEST(Fbcsr, AppliesToDenseVector)
         xvals[i] = std::sin(static_cast<T>(static_cast<float>((i + 1) ^ 2)));
     auto y = Vec::create(this->exec, gko::dim<2>{(gko::size_type)nrows, 1});
     auto yref = Vec::create(this->exec, gko::dim<2>{(gko::size_type)nrows, 1});
-
-    this->mtx2->apply(x.get(), y.get());
-
     using Csr = typename TestFixture::Csr;
     auto csr_mtx = Csr::create(this->mtx->get_executor(),
                                std::make_shared<typename Csr::classical>());
     this->mtx2->convert_to(csr_mtx.get());
+
+    this->mtx2->apply(x.get(), y.get());
     csr_mtx->apply(x.get(), yref.get());
 
     const double tolerance =
@@ -188,12 +186,10 @@ TYPED_TEST(Fbcsr, AppliesToDenseMatrix)
     using Vec = typename TestFixture::Vec;
     using T = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
-
     const gko::size_type nrows = this->mtx2->get_size()[0];
     const gko::size_type ncols = this->mtx2->get_size()[1];
     const gko::size_type nvecs = 3;
     auto x = Vec::create(this->exec, gko::dim<2>{ncols, nvecs});
-
     for (index_type i = 0; i < ncols; i++)
         for (index_type j = 0; j < nvecs; j++)
             x->at(i, j) = (static_cast<T>(3.0 * i) + get_some_number<T>()) /
@@ -202,7 +198,6 @@ TYPED_TEST(Fbcsr, AppliesToDenseMatrix)
     auto yref = Vec::create(this->exec, gko::dim<2>{nrows, nvecs});
 
     this->mtx2->apply(x.get(), y.get());
-
     this->ref2csrmtx->apply(x.get(), yref.get());
 
     const double tolerance =
@@ -216,17 +211,14 @@ TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseVector)
     using Vec = typename TestFixture::Vec;
     using T = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
-
     const T alphav = -1.0;
     const T betav = 2.0;
     auto alpha = gko::initialize<Vec>({alphav}, this->exec);
     auto beta = gko::initialize<Vec>({betav}, this->exec);
-
     const gko::size_type nrows = this->mtx2->get_size()[0];
     const gko::size_type ncols = this->mtx2->get_size()[1];
     auto x = Vec::create(this->exec, gko::dim<2>{ncols, 1});
     auto y = Vec::create(this->exec, gko::dim<2>{nrows, 1});
-
     for (index_type i = 0; i < ncols; i++) {
         x->at(i, 0) = (i + 1.0) * (i + 1.0);
     }
@@ -234,11 +226,9 @@ TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseVector)
         y->at(i, 0) = static_cast<T>(std::sin(2 * 3.14 * (i + 0.1) / nrows)) +
                       get_some_number<T>();
     }
-
     auto yref = y->clone();
 
     this->mtx2->apply(alpha.get(), x.get(), beta.get(), y.get());
-
     this->ref2csrmtx->apply(alpha.get(), x.get(), beta.get(), yref.get());
 
     const double tolerance =
@@ -252,18 +242,15 @@ TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseMatrix)
     using Vec = typename TestFixture::Vec;
     using T = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
-
     const T alphav = -1.0;
     const T betav = 2.0;
     auto alpha = gko::initialize<Vec>({alphav}, this->exec);
     auto beta = gko::initialize<Vec>({betav}, this->exec);
-
     const gko::size_type nrows = this->mtx2->get_size()[0];
     const gko::size_type ncols = this->mtx2->get_size()[1];
     const gko::size_type nvecs = 3;
     auto x = Vec::create(this->exec, gko::dim<2>{ncols, nvecs});
     auto y = Vec::create(this->exec, gko::dim<2>{nrows, nvecs});
-
     for (index_type i = 0; i < ncols; i++)
         for (index_type j = 0; j < nvecs; j++) {
             x->at(i, j) =
@@ -275,11 +262,9 @@ TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseMatrix)
                 static_cast<T>(std::sin(2 * 3.14 * (i + j + 0.1) / nrows)) +
                 get_some_number<T>();
         }
-
     auto yref = y->clone();
 
     this->mtx2->apply(alpha.get(), x.get(), beta.get(), y.get());
-
     this->ref2csrmtx->apply(alpha.get(), x.get(), beta.get(), yref.get());
 
     const double tolerance =
@@ -543,7 +528,6 @@ TYPED_TEST(Fbcsr, CalculatesNonzerosPerRow)
 
     gko::kernels::reference::fbcsr::calculate_nonzeros_per_row(
         this->exec, this->mtx2.get(), &row_nnz);
-
     gko::Array<gko::size_type> refrnnz(this->exec, this->mtx2->get_size()[0]);
     gko::kernels::reference::csr ::calculate_nonzeros_per_row(
         this->exec, this->ref2csrmtx.get(), &refrnnz);
@@ -562,7 +546,6 @@ TYPED_TEST(Fbcsr, CalculatesMaxNnzPerRow)
 
     gko::kernels::reference::fbcsr::calculate_max_nnz_per_row(
         this->exec, this->mtx2.get(), &max_row_nnz);
-
     gko::size_type ref_max_row_nnz{};
     gko::kernels::reference::csr::calculate_max_nnz_per_row(
         this->exec, this->ref2csrmtx.get(), &ref_max_row_nnz);
@@ -578,9 +561,9 @@ TYPED_TEST(Fbcsr, SquareMtxIsTransposable)
     auto csrmtxsq =
         Csr::create(this->exec, std::make_shared<typename Csr::classical>());
     this->mtxsq->convert_to(csrmtxsq.get());
+
     std::unique_ptr<const gko::LinOp> reftmtx = csrmtxsq->transpose();
     auto reftmtx_as_csr = static_cast<const Csr *>(reftmtx.get());
-
     auto trans = this->mtxsq->transpose();
     auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
 
@@ -595,9 +578,9 @@ TYPED_TEST(Fbcsr, NonSquareMtxIsTransposable)
     auto csrmtx =
         Csr::create(this->exec, std::make_shared<typename Csr::classical>());
     this->mtx2->convert_to(csrmtx.get());
+
     std::unique_ptr<gko::LinOp> reftmtx = csrmtx->transpose();
     auto reftmtx_as_csr = static_cast<Csr *>(reftmtx.get());
-
     auto trans = this->mtx2->transpose();
     auto trans_as_fbcsr = static_cast<Fbcsr *>(trans.get());
 
@@ -680,9 +663,9 @@ TYPED_TEST(Fbcsr, OutplaceAbsolute)
     using AbsCsr = typename gko::remove_complex<typename TestFixture::Csr>;
     using AbsMtx = typename gko::remove_complex<typename TestFixture::Mtx>;
     auto mtx = this->fbsample2.generate_fbcsr();
+
     const std::unique_ptr<const AbsCsr> refabs =
         this->ref2csrmtx->compute_absolute();
-
     auto abs_mtx = mtx->compute_absolute();
 
     const gko::remove_complex<value_type> tolerance =

From 9187c466c1c267ce155a3dcff2202b93eff3e24c Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Tue, 26 Jan 2021 12:28:19 +0100
Subject: [PATCH 0575/1008] small dense blocks are now column-major

---
 core/components/fixed_block.hpp         |  6 +++---
 core/matrix/fbcsr.cpp                   |  4 ++--
 core/test/matrix/fbcsr_sample.hpp       | 26 ++++++++++++-------------
 reference/test/matrix/fbcsr_kernels.cpp | 15 +++++++++-----
 4 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
index 889cb2145a2..f6fed3c6a6b 100644
--- a/core/components/fixed_block.hpp
+++ b/core/components/fixed_block.hpp
@@ -46,7 +46,7 @@ namespace blockutils {
  *
  * Accessing BSR values using this type of view abstracts away the
  * storage layout within the individual blocks, as long as all blocks use the
- * same layout. For now, row-major blocks are assumed.
+ * same layout. For now, column-major blocks are assumed.
  *
  * @tparam ValueType  The numeric type of entries of the block
  * @tparam IndexType  The type of integer used to identify the different blocks
@@ -70,13 +70,13 @@ class DenseBlocksView final {
 
     value_type &at(const index_type block, const int row, const int col)
     {
-        return vals_[block * nrows * ncols + row * ncols + col];
+        return vals_[block * nrows * ncols + row + col * nrows];
     }
 
     const value_type &at(const index_type block, const int row,
                          const int col) const
     {
-        return vals_[block * nrows * ncols + row * ncols + col];
+        return vals_[block * nrows * ncols + row + col * nrows];
     }
 
     value_type &operator()(const index_type block, const int row, const int col)
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 99e6c9e8914..38f2766729d 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -106,12 +106,12 @@ class DenseBlock final {
 
     value_type &at(const int row, const int col)
     {
-        return vals_[row * ncols_ + col];
+        return vals_[row + col * nrows_];
     }
 
     const value_type &at(const int row, const int col) const
     {
-        return vals_[row * ncols_ + col];
+        return vals_[row + col * nrows_];
     }
 
     value_type &operator()(const int row, const int col)
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 4f6bcc864f2..107532a44a9 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -136,8 +136,8 @@ class FbcsrSample {
         vals(0, 2, 2) = gko::zero<value_type>();
         vals(3, 0, 0) = gko::zero<value_type>();
 
-        v[34] += FBCSR_TEST_IMAGINARY;
-        v[35] += FBCSR_TEST_IMAGINARY;
+        vals(3, 2, 1) += FBCSR_TEST_IMAGINARY;
+        vals(3, 2, 2) += FBCSR_TEST_IMAGINARY;
 
         return mtx;
     }
@@ -302,14 +302,14 @@ class FbcsrSample2 {
         for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
 
         v[0] = 1;
-        v[1] = 2;
-        v[2] = 3;
+        v[1] = 3;
+        v[2] = 2;
         v[3] = 0;
-        v[10] = 0;
+        v[9] = 0;
         v[11] = 0;
         v[12] = -12;
-        v[13] = -1;
-        v[14] = -2;
+        v[13] = -2;
+        v[14] = -1;
         v[15] = -11;
 
         return Fbcsr::create(exec,
@@ -385,7 +385,7 @@ class FbcsrSample2 {
 };
 
 /**
- * @brief Generates the a sample block CSR square matrix and its transpose
+ * @brief Generates the a sample block CSR square matrix
  *
  * This currently a 4 x 4 matrix with 2x2 blocks.
  */
@@ -467,14 +467,14 @@ class FbcsrSampleComplex {
 
         using namespace std::complex_literals;
         v[0] = 1.0 + 1.15i;
-        v[1] = 2.0 + 2.15i;
-        v[2] = 3.0 - 3.15i;
+        v[2] = 2.0 + 2.15i;
+        v[1] = 3.0 - 3.15i;
         v[3] = 0.0 - 0.15i;
-        v[10] = 0.0;
+        v[9] = 0.0;
         v[11] = 0.0;
         v[12] = -12.0 + 12.15i;
-        v[13] = -1.0 + 1.15i;
-        v[14] = -2.0 - 2.15i;
+        v[14] = -1.0 + 1.15i;
+        v[13] = -2.0 - 2.15i;
         v[15] = -11.0 - 11.15i;
 
         return Fbcsr::create(exec,
diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp
index 2ed4e2be557..96f376fa68c 100644
--- a/reference/test/matrix/fbcsr_kernels.cpp
+++ b/reference/test/matrix/fbcsr_kernels.cpp
@@ -163,8 +163,9 @@ TYPED_TEST(Fbcsr, AppliesToDenseVector)
     const index_type ncols = this->mtx2->get_size()[1];
     auto x = Vec::create(this->exec, gko::dim<2>{(gko::size_type)ncols, 1});
     T *const xvals = x->get_values();
-    for (index_type i = 0; i < ncols; i++)
+    for (index_type i = 0; i < ncols; i++) {
         xvals[i] = std::sin(static_cast<T>(static_cast<float>((i + 1) ^ 2)));
+    }
     auto y = Vec::create(this->exec, gko::dim<2>{(gko::size_type)nrows, 1});
     auto yref = Vec::create(this->exec, gko::dim<2>{(gko::size_type)nrows, 1});
     using Csr = typename TestFixture::Csr;
@@ -190,10 +191,12 @@ TYPED_TEST(Fbcsr, AppliesToDenseMatrix)
     const gko::size_type ncols = this->mtx2->get_size()[1];
     const gko::size_type nvecs = 3;
     auto x = Vec::create(this->exec, gko::dim<2>{ncols, nvecs});
-    for (index_type i = 0; i < ncols; i++)
-        for (index_type j = 0; j < nvecs; j++)
+    for (index_type i = 0; i < ncols; i++) {
+        for (index_type j = 0; j < nvecs; j++) {
             x->at(i, j) = (static_cast<T>(3.0 * i) + get_some_number<T>()) /
                           static_cast<T>(j + 1.0);
+        }
+    }
     auto y = Vec::create(this->exec, gko::dim<2>{nrows, nvecs});
     auto yref = Vec::create(this->exec, gko::dim<2>{nrows, nvecs});
 
@@ -251,17 +254,19 @@ TYPED_TEST(Fbcsr, AppliesLinearCombinationToDenseMatrix)
     const gko::size_type nvecs = 3;
     auto x = Vec::create(this->exec, gko::dim<2>{ncols, nvecs});
     auto y = Vec::create(this->exec, gko::dim<2>{nrows, nvecs});
-    for (index_type i = 0; i < ncols; i++)
+    for (index_type i = 0; i < ncols; i++) {
         for (index_type j = 0; j < nvecs; j++) {
             x->at(i, j) =
                 std::log(static_cast<T>(0.1 + static_cast<float>((i + 1) ^ 2)));
         }
-    for (index_type i = 0; i < nrows; i++)
+    }
+    for (index_type i = 0; i < nrows; i++) {
         for (index_type j = 0; j < nvecs; j++) {
             y->at(i, j) =
                 static_cast<T>(std::sin(2 * 3.14 * (i + j + 0.1) / nrows)) +
                 get_some_number<T>();
         }
+    }
     auto yref = y->clone();
 
     this->mtx2->apply(alpha.get(), x.get(), beta.get(), y.get());

From c8e0feb7b9bd8e0593a8d884d27ae26967af5c7b Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 27 Jan 2021 12:56:32 +0100
Subject: [PATCH 0576/1008] New exception helper for divisibility by block size

Also in fbcsr.hpp, detail::get_num_blocks is now declared inline.
---
 .../ginkgo/core/base/exception_helpers.hpp    | 20 +++++++++++++++++++
 include/ginkgo/core/matrix/fbcsr.hpp          |  6 ++----
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/include/ginkgo/core/base/exception_helpers.hpp b/include/ginkgo/core/base/exception_helpers.hpp
index a10f05896fe..7a34a0835a0 100644
--- a/include/ginkgo/core/base/exception_helpers.hpp
+++ b/include/ginkgo/core/base/exception_helpers.hpp
@@ -567,6 +567,26 @@ inline T ensure_allocated_impl(T ptr, const std::string &file, int line,
                   "semi-colon warnings")
 
 
+/**
+ * Ensures that a given size, typically of a linear algebraic object,
+ * is divisible by a given block size.
+ *
+ * @param _size  A size of a vector or matrix
+ * @param _block_size  Size of small dense blocks that make up
+ *                     the vector or matrix
+ *
+ * @throw BlockSizeError  if _block_size does not divide _size
+ */
+#define GKO_ASSERT_BLOCK_SIZE_CONFORMANT(_size, _block_size)                   \
+    if (_size % _block_size != 0) {                                            \
+        throw BlockSizeError<decltype(_size)>(__FILE__, __LINE__, _block_size, \
+                                              _size);                          \
+    }                                                                          \
+    static_assert(true,                                                        \
+                  "This assert is used to counter the false positive extra "   \
+                  "semi-colon warnings")
+
+
 }  // namespace gko
 
 
diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp
index 16a7dd6b726..38cac854bc5 100644
--- a/include/ginkgo/core/matrix/fbcsr.hpp
+++ b/include/ginkgo/core/matrix/fbcsr.hpp
@@ -73,11 +73,9 @@ namespace detail {
  * @throw BlockSizeError  when block_size does not divide the total size.
  */
 template <typename IndexType>
-IndexType get_num_blocks(const int block_size, const IndexType size)
+inline IndexType get_num_blocks(const int block_size, const IndexType size)
 {
-    if (size % block_size != 0) {
-        throw BlockSizeError<IndexType>(__FILE__, __LINE__, block_size, size);
-    }
+    GKO_ASSERT_BLOCK_SIZE_CONFORMANT(size, block_size);
     return size / block_size;
 }
 

From a0cf52b6dafb05d3caf6129e3a5cfe84422f30d4 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 27 Jan 2021 17:34:50 +0100
Subject: [PATCH 0577/1008] fixed get_num_blocks test

---
 core/test/matrix/fbcsr.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index c4a7e3c3420..f5765bd055a 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -322,7 +322,7 @@ TYPED_TEST(Fbcsr, GetNumBlocksCorrectlyThrows)
     const int blk_sz = 9;
 
     ASSERT_THROW(gko::matrix::detail::get_num_blocks(blk_sz, vec_sz),
-                 gko::BlockSizeError<index_type>);
+                 gko::BlockSizeError<decltype(vec_sz)>);
 }
 
 

From 22e874c0e436c9d936108f0396a554db776a2170 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 3 Feb 2021 16:07:45 +0100
Subject: [PATCH 0578/1008] Fbcsr write now writes in row-major order

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/matrix/fbcsr.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 38f2766729d..6ed3f22eac0 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -393,9 +393,9 @@ void Fbcsr<ValueType, IndexType>::write(mat_data &data) const
         const auto start = tmp->row_ptrs_.get_const_data()[brow];
         const auto end = tmp->row_ptrs_.get_const_data()[brow + 1];
 
-        for (auto inz = start; inz < end; ++inz) {
-            for (int ib = 0; ib < bs_; ib++) {
-                const auto row = brow * bs_ + ib;
+        for (int ib = 0; ib < bs_; ib++) {
+            const auto row = brow * bs_ + ib;
+            for (auto inz = start; inz < end; ++inz) {
                 for (int jb = 0; jb < bs_; jb++) {
                     const auto col =
                         tmp->col_idxs_.get_const_data()[inz] * bs_ + jb;

From 10558dcdab4aedebfcc87ee12ab675d70489a751 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 3 Feb 2021 18:07:03 +0100
Subject: [PATCH 0579/1008] changed macros to variables in fbcsr sample

---
 core/test/matrix/fbcsr_sample.hpp | 40 +++++++++++++++----------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 107532a44a9..d89dfa4839e 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -46,16 +46,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/test/utils.hpp"
 
 
-#define FBCSR_TEST_OFFSET 0.000011118888
-#define FBCSR_TEST_C_MAG 0.1 + FBCSR_TEST_OFFSET
-#define FBCSR_TEST_IMAGINARY \
-    sct(std::complex<remove_complex<ValueType>>(0, FBCSR_TEST_C_MAG))
-
-
 namespace gko {
 namespace testing {
 
 
+constexpr auto fbcsr_test_offset = 0.000011118888;
+
+
 /** Generates the same sample block CSR matrix in different formats
  *
  * This currently a 6 x 12 matrix with 3x3 blocks.
@@ -136,8 +133,8 @@ class FbcsrSample {
         vals(0, 2, 2) = gko::zero<value_type>();
         vals(3, 0, 0) = gko::zero<value_type>();
 
-        vals(3, 2, 1) += FBCSR_TEST_IMAGINARY;
-        vals(3, 2, 2) += FBCSR_TEST_IMAGINARY;
+        vals(3, 2, 1) += fbcsr_test_imaginary;
+        vals(3, 2, 2) += fbcsr_test_imaginary;
 
         return mtx;
     }
@@ -158,8 +155,8 @@ class FbcsrSample {
             {2, 3, 4, 4, 5, 6, 5, 6, 7, 7, 8, 9, 0, 9, 0,
 	         10, 11, 12, 2, 3, 4, 0, 7, 8, 5, 6, 7,
 	         9, 10, 11, 8, 9, 10, 12,
-	         sct<value_type>(13.0) + FBCSR_TEST_IMAGINARY,
-	         sct<value_type>(14.0) + FBCSR_TEST_IMAGINARY});
+	         sct<value_type>(13.0) + fbcsr_test_imaginary,
+	         sct<value_type>(14.0) + fbcsr_test_imaginary});
         // clang-format on
         return Csr::create(exec, gko::dim<2>{nrows, ncols}, csrvals, csrcols,
                            csrrow);
@@ -223,8 +220,8 @@ class FbcsrSample {
                          {4, 7, 10.0},
                          {4, 8, 11.0},
                          {5, 6, 12.0},
-                         {5, 7, sct(13.0) + FBCSR_TEST_IMAGINARY},
-                         {5, 8, sct(14.0) + FBCSR_TEST_IMAGINARY}}});
+                         {5, 7, sct(13.0) + fbcsr_test_imaginary},
+                         {5, 8, sct(14.0) + fbcsr_test_imaginary}}});
     }
 
     /**
@@ -261,6 +258,9 @@ class FbcsrSample {
     {
         return static_cast<ValueType>(cu.real());
     }
+
+    const ValueType fbcsr_test_imaginary = sct(
+        std::complex<remove_complex<ValueType>>(0, 0.1 + fbcsr_test_offset));
 };
 
 /**
@@ -299,7 +299,7 @@ class FbcsrSample2 {
         gko::Array<index_type> c(exec, {0, 0, 3, 2});
         gko::Array<value_type> vals(exec, nnz);
         value_type *const v = vals.get_data();
-        for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
+        for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + fbcsr_test_offset;
 
         v[0] = 1;
         v[1] = 3;
@@ -325,7 +325,7 @@ class FbcsrSample2 {
             exec, {0, 1, 0, 1, 0, 1, 6, 7, 0, 1, 6, 7, 4, 5, 4, 5});
         gko::Array<value_type> vals(exec, nnz);
         value_type *const v = vals.get_data();
-        for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
+        for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + fbcsr_test_offset;
         v[0] = 1;
         v[1] = 2;
         v[2] = 3;
@@ -363,9 +363,9 @@ class FbcsrSample2 {
         dbv(0, 1, 0) = 3.0;
         dbv(0, 1, 1) = 0.0;
         for (int i = 0; i < 2; ++i)
-            for (int j = 0; j < 2; ++j) dbv(1, i, j) = 0.15 + FBCSR_TEST_OFFSET;
-        dbv(2, 0, 0) = 0.15 + FBCSR_TEST_OFFSET;
-        dbv(2, 0, 1) = 0.15 + FBCSR_TEST_OFFSET;
+            for (int j = 0; j < 2; ++j) dbv(1, i, j) = 0.15 + fbcsr_test_offset;
+        dbv(2, 0, 0) = 0.15 + fbcsr_test_offset;
+        dbv(2, 0, 1) = 0.15 + fbcsr_test_offset;
         dbv(2, 1, 0) = 0.0;
         dbv(2, 1, 1) = 0.0;
         dbv(3, 0, 0) = -12.0;
@@ -463,7 +463,7 @@ class FbcsrSampleComplex {
         gko::Array<index_type> c(exec, {0, 0, 3, 2});
         gko::Array<value_type> vals(exec, nnz);
         value_type *const v = vals.get_data();
-        for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
+        for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + fbcsr_test_offset;
 
         using namespace std::complex_literals;
         v[0] = 1.0 + 1.15i;
@@ -490,7 +490,7 @@ class FbcsrSampleComplex {
             exec, {0, 1, 0, 1, 0, 1, 6, 7, 0, 1, 6, 7, 4, 5, 4, 5});
         gko::Array<value_type> vals(exec, nnz);
         value_type *const v = vals.get_data();
-        for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + FBCSR_TEST_OFFSET;
+        for (IndexType i = 0; i < nnz; i++) v[i] = 0.15 + fbcsr_test_offset;
 
         using namespace std::complex_literals;
         v[0] = 1.0 + 1.15i;
@@ -547,7 +547,7 @@ class FbcsrSampleUnsorted {
         gko::Array<value_type> vals(exec, nnz);
         value_type *const v = vals.get_data();
         for (IndexType i = 0; i < nnz; i++) {
-            v[i] = static_cast<value_type>(i + 0.15 + FBCSR_TEST_OFFSET);
+            v[i] = static_cast<value_type>(i + 0.15 + fbcsr_test_offset);
         }
 
         return Fbcsr::create(exec,

From 97f5e4c505ba68372f0edf5421bc94a58c59be9e Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 3 Feb 2021 20:13:14 +0100
Subject: [PATCH 0580/1008] review updates

- removal of unnecessary lines of code
- snake case convention for non-type template param

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 cuda/matrix/fbcsr_kernels.cu       | 10 ----------
 cuda/test/matrix/fbcsr_kernels.cpp |  4 ++--
 hip/matrix/fbcsr_kernels.hip.cpp   | 17 -----------------
 reference/matrix/fbcsr_kernels.cpp |  4 ++--
 4 files changed, 4 insertions(+), 31 deletions(-)

diff --git a/cuda/matrix/fbcsr_kernels.cu b/cuda/matrix/fbcsr_kernels.cu
index 2d3677ea6bb..6f7bc48cf92 100644
--- a/cuda/matrix/fbcsr_kernels.cu
+++ b/cuda/matrix/fbcsr_kernels.cu
@@ -57,16 +57,6 @@ namespace cuda {
 namespace fbcsr {
 
 
-/**
- * A compile-time list of the number items per threads for which spmv kernel
- * should be compiled.
- */
-using compiled_kernels = syn::value_list<int, 3, 4, 6, 7, 8, 12, 14>;
-
-using classical_kernels =
-    syn::value_list<int, config::warp_size, 32, 16, 8, 4, 2, 1>;
-
-
 template <typename ValueType, typename IndexType>
 void spmv(std::shared_ptr<const CudaExecutor> exec,
           const matrix::Fbcsr<ValueType, IndexType> *a,
diff --git a/cuda/test/matrix/fbcsr_kernels.cpp b/cuda/test/matrix/fbcsr_kernels.cpp
index 79b455cb21c..02bfa3adfe1 100644
--- a/cuda/test/matrix/fbcsr_kernels.cpp
+++ b/cuda/test/matrix/fbcsr_kernels.cpp
@@ -36,7 +36,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
-//#include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 
 
@@ -82,7 +81,8 @@ TEST_F(Fbcsr, CanWriteFromMatrixOnDevice)
     auto cudamat = Mtx::create(cuda);
     cudamat->copy_from(gko::lend(refmat));
 
-    MatData refdata, cudadata;
+    MatData refdata;
+    MatData cudadata;
     refmat->write(refdata);
     cudamat->write(cudadata);
 
diff --git a/hip/matrix/fbcsr_kernels.hip.cpp b/hip/matrix/fbcsr_kernels.hip.cpp
index 934b7f270cd..13ca94ad5f7 100644
--- a/hip/matrix/fbcsr_kernels.hip.cpp
+++ b/hip/matrix/fbcsr_kernels.hip.cpp
@@ -60,23 +60,6 @@ namespace hip {
 namespace fbcsr {
 
 
-constexpr int default_block_size = 512;
-constexpr int warps_in_block = 4;
-constexpr int spmv_block_size = warps_in_block * config::warp_size;
-constexpr int wsize = config::warp_size;
-constexpr int classical_overweight = 32;
-
-
-/**
- * A compile-time list of the number items per threads for which spmv kernel
- * should be compiled.
- */
-using compiled_kernels = syn::value_list<int, 3, 4, 6, 7, 8, 12, 14>;
-
-using classical_kernels =
-    syn::value_list<int, config::warp_size, 32, 16, 8, 4, 2, 1>;
-
-
 template <typename ValueType, typename IndexType>
 void spmv(std::shared_ptr<const HipExecutor> exec,
           const matrix::Fbcsr<ValueType, IndexType> *a,
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index da7877c2363..0c5ba27f511 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -395,7 +395,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_IS_SORTED_BY_COLUMN_INDEX);
 
 
-template <int matBlkSz, typename ValueType, typename IndexType>
+template <int mat_blk_sz, typename ValueType, typename IndexType>
 static void sort_by_column_index_impl(
     matrix::Fbcsr<ValueType, IndexType> *const to_sort)
 {
@@ -403,7 +403,7 @@ static void sort_by_column_index_impl(
     auto col_idxs = to_sort->get_col_idxs();
     auto values = to_sort->get_values();
     const auto nbrows = to_sort->get_num_block_rows();
-    constexpr int bs2 = matBlkSz * matBlkSz;
+    constexpr int bs2 = mat_blk_sz * mat_blk_sz;
     for (IndexType i = 0; i < nbrows; ++i) {
         IndexType *const brow_col_idxs = col_idxs + row_ptrs[i];
         ValueType *const brow_vals = values + row_ptrs[i] * bs2;

From 78accb2b5cf964152c132d8eea98887ea6b4c18e Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Thu, 4 Feb 2021 11:19:54 +0100
Subject: [PATCH 0581/1008] added block size check to in fbcsr read, updated
 contributors list

---
 contributors.txt      | 1 +
 core/matrix/fbcsr.cpp | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/contributors.txt b/contributors.txt
index fde5b54d2c3..a7a558499e2 100644
--- a/contributors.txt
+++ b/contributors.txt
@@ -14,6 +14,7 @@ Grützmacher Thomas <thogru.kit@gmx.de> Karlsruhe Institute of Technology
 Heroux Mike <maherou@sandia.gov> Sandia National Laboratories
 Hoemmen Mark <mhoemme@sandia.gov> Sandia National Laboratories
 Holeksa Claudius <mail@keldu.de> Karlsruhe Institute of Technology
+Kashi Aditya <aditya.kashi@kit.edu> Karlsruhe Institute of Technology
 Maier Matthias <matthias@43-1.org> Texas A&M University
 Nayak Pratik <pratik.nayak@kit.edu> Karlsruhe Institute of Technology
 Olenik Gregor <go@hpsim.de> HPSim
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 6ed3f22eac0..f86292154dd 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -338,7 +338,7 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
     index_type cur_brow = 0;
     index_type cur_bnz = 0;
     index_type cur_bcol = blocks.begin()->first.block_column;
-    const index_type num_brows = data.size[0] / bs;
+    const index_type num_brows = detail::get_num_blocks(bs, data.size[0]);
 
     blockutils::DenseBlocksView<value_type, index_type> values(
         tmp->values_.get_data(), bs, bs);

From 656c3727d63009cd8aade2b288bc1a7d7439e340 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Mon, 8 Feb 2021 21:29:21 +0100
Subject: [PATCH 0582/1008] fixed a bug in ref fbcsr transpose

---
 reference/matrix/fbcsr_kernels.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index 0c5ba27f511..2076b13c1b2 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -456,14 +456,16 @@ void extract_diagonal(std::shared_ptr<const ReferenceExecutor>,
     const auto values = orig->get_const_values();
     const int bs = orig->get_block_size();
     const IndexType nbrows = orig->get_num_block_rows();
+    const IndexType nbdim_min =
+        std::min(orig->get_num_block_rows(), orig->get_num_block_cols());
     auto diag_values = diag->get_values();
 
-    assert(diag->get_size()[0] == orig->get_size()[0]);
+    assert(diag->get_size()[0] == nbdim_min * bs);
 
     const gko::blockutils::DenseBlocksView<const ValueType, IndexType> vblocks(
         values, bs, bs);
 
-    for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
+    for (IndexType ibrow = 0; ibrow < nbdim_min; ++ibrow) {
         for (IndexType idx = row_ptrs[ibrow]; idx < row_ptrs[ibrow + 1];
              ++idx) {
             if (col_idxs[idx] == ibrow) {

From fb2550aea1f93fa6bd3bd94375a00493b59a5673 Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Thu, 18 Feb 2021 19:01:26 +0100
Subject: [PATCH 0583/1008] replaced new DenseBlocksView with range and
 col_major accessor

---
 core/components/fixed_block.hpp    | 105 -----------------------------
 core/matrix/fbcsr.cpp              |  14 ++--
 core/test/matrix/fbcsr.cpp         |  30 +--------
 core/test/matrix/fbcsr_sample.hpp  |  26 +------
 reference/matrix/fbcsr_kernels.cpp |  31 ++++-----
 5 files changed, 30 insertions(+), 176 deletions(-)
 delete mode 100644 core/components/fixed_block.hpp

diff --git a/core/components/fixed_block.hpp b/core/components/fixed_block.hpp
deleted file mode 100644
index f6fed3c6a6b..00000000000
--- a/core/components/fixed_block.hpp
+++ /dev/null
@@ -1,105 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GKO_CORE_COMPONENTS_FIXED_BLOCK_HPP_
-#define GKO_CORE_COMPONENTS_FIXED_BLOCK_HPP_
-
-
-#include <ginkgo/core/base/types.hpp>
-
-
-namespace gko {
-namespace blockutils {
-
-
-/**
- * @brief A view into a an array of dense blocks of some runtime-defined size
- *
- * Accessing BSR values using this type of view abstracts away the
- * storage layout within the individual blocks, as long as all blocks use the
- * same layout. For now, column-major blocks are assumed.
- *
- * @tparam ValueType  The numeric type of entries of the block
- * @tparam IndexType  The type of integer used to identify the different blocks
- */
-template <typename ValueType, typename IndexType = int32>
-class DenseBlocksView final {
-public:
-    using value_type = ValueType;
-    using index_type = IndexType;
-
-    /**
-     * @param buffer  Pointer to the segment of memory to be interpreted as
-     *                an array of 2D blocks
-     * @param num_rows  Number of rows in each block
-     * @param num_cols  Number of columns in each block
-     */
-    DenseBlocksView(ValueType *const buffer, const int num_rows,
-                    const int num_cols)
-        : nrows{num_rows}, ncols{num_cols}, vals_{buffer}
-    {}
-
-    value_type &at(const index_type block, const int row, const int col)
-    {
-        return vals_[block * nrows * ncols + row + col * nrows];
-    }
-
-    const value_type &at(const index_type block, const int row,
-                         const int col) const
-    {
-        return vals_[block * nrows * ncols + row + col * nrows];
-    }
-
-    value_type &operator()(const index_type block, const int row, const int col)
-    {
-        return at(block, row, col);
-    }
-
-    const value_type &operator()(const index_type block, const int row,
-                                 const int col) const
-    {
-        return at(block, row, col);
-    }
-
-    const int nrows;  ///< Number of rows in each block
-    const int ncols;  ///< Number of columns in each block
-
-private:
-    value_type *const vals_;
-};
-
-
-}  // namespace blockutils
-}  // namespace gko
-
-
-#endif  // GKO_CORE_COMPONENTS_FIXED_BLOCK_HPP_
diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index f86292154dd..53d86048778 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -49,7 +49,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/components/absolute_array.hpp"
 #include "core/components/fill_array.hpp"
-#include "core/components/fixed_block.hpp"
 #include "core/matrix/fbcsr_kernels.hpp"
 
 
@@ -340,8 +339,10 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
     index_type cur_bcol = blocks.begin()->first.block_column;
     const index_type num_brows = detail::get_num_blocks(bs, data.size[0]);
 
-    blockutils::DenseBlocksView<value_type, index_type> values(
-        tmp->values_.get_data(), bs, bs);
+    // blockutils::DenseBlocksView<value_type, index_type> values(
+    //     tmp->values_.get_data(), bs, bs);
+    range<accessor::col_major<value_type, 3>> values(
+        tmp->values_.get_data(), dim<3>(blocks.size(), bs, bs));
 
     for (auto it = blocks.begin(); it != blocks.end(); it++) {
         GKO_ENSURE_IN_BOUNDS(cur_brow, num_brows);
@@ -386,8 +387,11 @@ void Fbcsr<ValueType, IndexType>::write(mat_data &data) const
 
     data = {tmp->get_size(), {}};
 
-    const blockutils::DenseBlocksView<const value_type, index_type> vblocks(
-        tmp->values_.get_const_data(), bs_, bs_);
+    // const blockutils::DenseBlocksView<const value_type, index_type> vblocks(
+    //     tmp->values_.get_const_data(), bs_, bs_);
+    const size_type nbnz = tmp->get_num_stored_blocks();
+    const range<accessor::col_major<const value_type, 3>> vblocks(
+        tmp->values_.get_const_data(), dim<3>(nbnz, bs_, bs_));
 
     for (size_type brow = 0; brow < tmp->get_num_block_rows(); ++brow) {
         const auto start = tmp->row_ptrs_.get_const_data()[brow];
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index f5765bd055a..3e2757a0050 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -40,7 +40,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
-#include "core/components/fixed_block.hpp"
 #include "core/test/matrix/fbcsr_sample.hpp"
 #include "core/test/utils.hpp"
 
@@ -72,8 +71,9 @@ void assert_matrices_are_same(
 
     const IndexType nbrows = bm->get_num_block_rows();
     const int bs = bm->get_block_size();
-    gko::blockutils::DenseBlocksView<const ValueType, IndexType> fbvals(
-        bm->get_const_values(), bs, bs);
+    const auto nbnz = bm->get_num_stored_blocks();
+    gko::range<gko::accessor::col_major<const ValueType, 3>> fbvals(
+        bm->get_const_values(), gko::dim<3>(nbnz, bs, bs));
 
     for (IndexType ibrow = 0; ibrow < nbrows; ibrow++) {
         const IndexType *const browptr = bm->get_const_row_ptrs();
@@ -473,28 +473,4 @@ TYPED_TEST(Fbcsr, GeneratesCorrectMatrixData)
 }
 
 
-TYPED_TEST(Fbcsr, DenseBlocksViewWorksCorrectly)
-{
-    using value_type = typename TestFixture::value_type;
-    using index_type = typename TestFixture::index_type;
-    using Dbv = gko::blockutils::DenseBlocksView<value_type, index_type>;
-    const gko::testing::FbcsrSample2<value_type, index_type> fbsample(
-        this->exec);
-    std::vector<value_type> ref_dbv_array(fbsample.nnz);
-    Dbv refdbv(ref_dbv_array.data(), fbsample.bs, fbsample.bs);
-    fbsample.fill_value_blocks_view(refdbv);
-
-    auto refmtx = fbsample.generate_fbcsr();
-    const Dbv testdbv(refmtx->get_values(), fbsample.bs, fbsample.bs);
-
-    for (index_type ibz = 0; ibz < fbsample.nbnz; ibz++) {
-        for (int i = 0; i < fbsample.bs; ++i) {
-            for (int j = 0; j < fbsample.bs; ++j) {
-                ASSERT_EQ(testdbv(ibz, i, j), refdbv(ibz, i, j));
-            }
-        }
-    }
-}
-
-
 }  // namespace
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index d89dfa4839e..748e5234361 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -42,7 +42,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
-#include "core/components/fixed_block.hpp"
 #include "core/test/utils.hpp"
 
 
@@ -106,8 +105,8 @@ class FbcsrSample {
         c[2] = 0;
         c[3] = 2;
 
-        gko::blockutils::DenseBlocksView<value_type, index_type> vals(v, bs,
-                                                                      bs);
+        gko::range<gko::accessor::col_major<value_type, 3>> vals(
+            v, gko::dim<3>(nbnz, bs, bs));
 
         if (mtx->get_size()[0] % bs != 0)
             throw gko::BadDimension(__FILE__, __LINE__, __func__, "test fbcsr",
@@ -275,8 +274,6 @@ class FbcsrSample2 {
     using Fbcsr = gko::matrix::Fbcsr<value_type, index_type>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
     using Diagonal = gko::matrix::Diagonal<value_type>;
-    using DenseBlocksView =
-        gko::blockutils::DenseBlocksView<ValueType, IndexType>;
 
 
     const size_type nrows = 6;
@@ -355,25 +352,6 @@ class FbcsrSample2 {
         return gko::Array<index_type>(exec, {2, 2, 4, 4, 2, 2});
     }
 
-    /// Fills a view into a FBCSR values array using the sample matrix's data
-    void fill_value_blocks_view(DenseBlocksView &dbv) const
-    {
-        dbv(0, 0, 0) = 1.0;
-        dbv(0, 0, 1) = 2.0;
-        dbv(0, 1, 0) = 3.0;
-        dbv(0, 1, 1) = 0.0;
-        for (int i = 0; i < 2; ++i)
-            for (int j = 0; j < 2; ++j) dbv(1, i, j) = 0.15 + fbcsr_test_offset;
-        dbv(2, 0, 0) = 0.15 + fbcsr_test_offset;
-        dbv(2, 0, 1) = 0.15 + fbcsr_test_offset;
-        dbv(2, 1, 0) = 0.0;
-        dbv(2, 1, 1) = 0.0;
-        dbv(3, 0, 0) = -12.0;
-        dbv(3, 0, 1) = -1.0;
-        dbv(3, 1, 0) = -2.0;
-        dbv(3, 1, 1) = -11.0;
-    }
-
 
 private:
     /// Enables use of literals to instantiate value data
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index 2076b13c1b2..6bb3563dbe7 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -42,12 +42,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/range_accessors.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
 #include "core/base/allocator.hpp"
 #include "core/base/iterator_factory.hpp"
-#include "core/components/fixed_block.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/fbcsr_builder.hpp"
 #include "reference/components/format_conversion.hpp"
@@ -73,11 +73,12 @@ void spmv(const std::shared_ptr<const ReferenceExecutor>,
     const int bs = a->get_block_size();
     const auto nvecs = static_cast<IndexType>(b->get_size()[1]);
     const IndexType nbrows = a->get_num_block_rows();
+    const size_type nbnz = a->get_num_stored_blocks();
     auto row_ptrs = a->get_const_row_ptrs();
     auto col_idxs = a->get_const_col_idxs();
     auto vals = a->get_const_values();
-    const blockutils::DenseBlocksView<const ValueType, IndexType> avalues(
-        vals, bs, bs);
+    const range<accessor::col_major<const ValueType, 3>> avalues{
+        vals, dim<3>(nbnz, bs, bs)};
 
     for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
         for (IndexType i = ibrow * bs * nvecs; i < (ibrow + 1) * bs * nvecs;
@@ -119,8 +120,8 @@ void advanced_spmv(const std::shared_ptr<const ReferenceExecutor>,
     auto vals = a->get_const_values();
     auto valpha = alpha->at(0, 0);
     auto vbeta = beta->at(0, 0);
-    const blockutils::DenseBlocksView<const ValueType, IndexType> avalues(
-        vals, bs, bs);
+    const range<accessor::col_major<const ValueType, 3>> avalues{
+        vals, dim<3>(a->get_num_stored_blocks(), bs, bs)};
 
     for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
         for (IndexType i = ibrow * bs * nvecs; i < (ibrow + 1) * bs * nvecs;
@@ -158,8 +159,8 @@ void convert_to_dense(const std::shared_ptr<const ReferenceExecutor>,
     const IndexType *const col_idxs = source->get_const_col_idxs();
     const ValueType *const vals = source->get_const_values();
 
-    const gko::blockutils::DenseBlocksView<const ValueType, IndexType> values(
-        vals, bs, bs);
+    const range<accessor::col_major<const ValueType, 3>> values{
+        vals, dim<3>(source->get_num_stored_blocks(), bs, bs)};
 
     for (IndexType brow = 0; brow < nbrows; ++brow) {
         for (size_type bcol = 0; bcol < nbcols; ++bcol) {
@@ -208,8 +209,8 @@ void convert_to_csr(const std::shared_ptr<const ReferenceExecutor>,
     IndexType *const col_idxs = result->get_col_idxs();
     ValueType *const vals = result->get_values();
 
-    const gko::blockutils::DenseBlocksView<const ValueType, IndexType> bvalues(
-        bvals, bs, bs);
+    const range<accessor::col_major<const ValueType, 3>> bvalues{
+        bvals, dim<3>(source->get_num_stored_blocks(), bs, bs)};
 
     for (IndexType brow = 0; brow < nbrows; ++brow) {
         const IndexType nz_browstart = browptrs[brow] * bs * bs;
@@ -253,10 +254,10 @@ void convert_fbcsr_to_fbcsc(const IndexType num_blk_rows, const int blksz,
                             IndexType *const col_ptrs,
                             ValueType *const csc_vals, UnaryOperator op)
 {
-    const gko::blockutils::DenseBlocksView<const ValueType, IndexType> rvalues(
-        fbcsr_vals, blksz, blksz);
-    gko::blockutils::DenseBlocksView<ValueType, IndexType> cvalues(
-        csc_vals, blksz, blksz);
+    const range<accessor::col_major<const ValueType, 3>> rvalues{
+        fbcsr_vals, dim<3>(row_ptrs[num_blk_rows], blksz, blksz)};
+    const range<accessor::col_major<ValueType, 3>> cvalues{
+        csc_vals, dim<3>(row_ptrs[num_blk_rows], blksz, blksz)};
     for (IndexType brow = 0; brow < num_blk_rows; ++brow) {
         for (auto i = row_ptrs[brow]; i < row_ptrs[brow + 1]; ++i) {
             const auto dest_idx = col_ptrs[col_idxs[i]];
@@ -462,8 +463,8 @@ void extract_diagonal(std::shared_ptr<const ReferenceExecutor>,
 
     assert(diag->get_size()[0] == nbdim_min * bs);
 
-    const gko::blockutils::DenseBlocksView<const ValueType, IndexType> vblocks(
-        values, bs, bs);
+    const range<accessor::col_major<const ValueType, 3>> vblocks{
+        values, dim<3>(orig->get_num_stored_blocks(), bs, bs)};
 
     for (IndexType ibrow = 0; ibrow < nbdim_min; ++ibrow) {
         for (IndexType idx = row_ptrs[ibrow]; idx < row_ptrs[ibrow + 1];

From 36c04e28947b56c2f4a5822821099e823d25697f Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 24 Feb 2021 10:50:04 +0100
Subject: [PATCH 0584/1008] rebased onto renamed block_col_major accessor

---
 core/matrix/fbcsr.cpp              |  8 ++------
 core/test/matrix/fbcsr.cpp         |  2 +-
 core/test/matrix/fbcsr_sample.hpp  |  2 +-
 reference/matrix/fbcsr_kernels.cpp | 14 +++++++-------
 4 files changed, 11 insertions(+), 15 deletions(-)

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 53d86048778..1876e421751 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -339,9 +339,7 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
     index_type cur_bcol = blocks.begin()->first.block_column;
     const index_type num_brows = detail::get_num_blocks(bs, data.size[0]);
 
-    // blockutils::DenseBlocksView<value_type, index_type> values(
-    //     tmp->values_.get_data(), bs, bs);
-    range<accessor::col_major<value_type, 3>> values(
+    range<accessor::block_col_major<value_type, 3>> values(
         tmp->values_.get_data(), dim<3>(blocks.size(), bs, bs));
 
     for (auto it = blocks.begin(); it != blocks.end(); it++) {
@@ -387,10 +385,8 @@ void Fbcsr<ValueType, IndexType>::write(mat_data &data) const
 
     data = {tmp->get_size(), {}};
 
-    // const blockutils::DenseBlocksView<const value_type, index_type> vblocks(
-    //     tmp->values_.get_const_data(), bs_, bs_);
     const size_type nbnz = tmp->get_num_stored_blocks();
-    const range<accessor::col_major<const value_type, 3>> vblocks(
+    const range<accessor::block_col_major<const value_type, 3>> vblocks(
         tmp->values_.get_const_data(), dim<3>(nbnz, bs_, bs_));
 
     for (size_type brow = 0; brow < tmp->get_num_block_rows(); ++brow) {
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index 3e2757a0050..abe8f1c1059 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -72,7 +72,7 @@ void assert_matrices_are_same(
     const IndexType nbrows = bm->get_num_block_rows();
     const int bs = bm->get_block_size();
     const auto nbnz = bm->get_num_stored_blocks();
-    gko::range<gko::accessor::col_major<const ValueType, 3>> fbvals(
+    gko::range<gko::accessor::block_col_major<const ValueType, 3>> fbvals(
         bm->get_const_values(), gko::dim<3>(nbnz, bs, bs));
 
     for (IndexType ibrow = 0; ibrow < nbrows; ibrow++) {
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index 748e5234361..aa34981cd06 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -105,7 +105,7 @@ class FbcsrSample {
         c[2] = 0;
         c[3] = 2;
 
-        gko::range<gko::accessor::col_major<value_type, 3>> vals(
+        gko::range<gko::accessor::block_col_major<value_type, 3>> vals(
             v, gko::dim<3>(nbnz, bs, bs));
 
         if (mtx->get_size()[0] % bs != 0)
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index 6bb3563dbe7..a5dbc75d120 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -77,7 +77,7 @@ void spmv(const std::shared_ptr<const ReferenceExecutor>,
     auto row_ptrs = a->get_const_row_ptrs();
     auto col_idxs = a->get_const_col_idxs();
     auto vals = a->get_const_values();
-    const range<accessor::col_major<const ValueType, 3>> avalues{
+    const range<accessor::block_col_major<const ValueType, 3>> avalues{
         vals, dim<3>(nbnz, bs, bs)};
 
     for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
@@ -120,7 +120,7 @@ void advanced_spmv(const std::shared_ptr<const ReferenceExecutor>,
     auto vals = a->get_const_values();
     auto valpha = alpha->at(0, 0);
     auto vbeta = beta->at(0, 0);
-    const range<accessor::col_major<const ValueType, 3>> avalues{
+    const range<accessor::block_col_major<const ValueType, 3>> avalues{
         vals, dim<3>(a->get_num_stored_blocks(), bs, bs)};
 
     for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
@@ -159,7 +159,7 @@ void convert_to_dense(const std::shared_ptr<const ReferenceExecutor>,
     const IndexType *const col_idxs = source->get_const_col_idxs();
     const ValueType *const vals = source->get_const_values();
 
-    const range<accessor::col_major<const ValueType, 3>> values{
+    const range<accessor::block_col_major<const ValueType, 3>> values{
         vals, dim<3>(source->get_num_stored_blocks(), bs, bs)};
 
     for (IndexType brow = 0; brow < nbrows; ++brow) {
@@ -209,7 +209,7 @@ void convert_to_csr(const std::shared_ptr<const ReferenceExecutor>,
     IndexType *const col_idxs = result->get_col_idxs();
     ValueType *const vals = result->get_values();
 
-    const range<accessor::col_major<const ValueType, 3>> bvalues{
+    const range<accessor::block_col_major<const ValueType, 3>> bvalues{
         bvals, dim<3>(source->get_num_stored_blocks(), bs, bs)};
 
     for (IndexType brow = 0; brow < nbrows; ++brow) {
@@ -254,9 +254,9 @@ void convert_fbcsr_to_fbcsc(const IndexType num_blk_rows, const int blksz,
                             IndexType *const col_ptrs,
                             ValueType *const csc_vals, UnaryOperator op)
 {
-    const range<accessor::col_major<const ValueType, 3>> rvalues{
+    const range<accessor::block_col_major<const ValueType, 3>> rvalues{
         fbcsr_vals, dim<3>(row_ptrs[num_blk_rows], blksz, blksz)};
-    const range<accessor::col_major<ValueType, 3>> cvalues{
+    const range<accessor::block_col_major<ValueType, 3>> cvalues{
         csc_vals, dim<3>(row_ptrs[num_blk_rows], blksz, blksz)};
     for (IndexType brow = 0; brow < num_blk_rows; ++brow) {
         for (auto i = row_ptrs[brow]; i < row_ptrs[brow + 1]; ++i) {
@@ -463,7 +463,7 @@ void extract_diagonal(std::shared_ptr<const ReferenceExecutor>,
 
     assert(diag->get_size()[0] == nbdim_min * bs);
 
-    const range<accessor::col_major<const ValueType, 3>> vblocks{
+    const range<accessor::block_col_major<const ValueType, 3>> vblocks{
         values, dim<3>(orig->get_num_stored_blocks(), bs, bs)};
 
     for (IndexType ibrow = 0; ibrow < nbdim_min; ++ibrow) {

From 2409a285b181527c1871375adb3bf244fbe73a6e Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Tue, 23 Mar 2021 01:04:34 +0100
Subject: [PATCH 0585/1008] addressed changes to accessor sub-library

---
 core/matrix/fbcsr.cpp              | 13 +++++---
 core/test/matrix/fbcsr.cpp         | 11 +++++--
 core/test/matrix/fbcsr_sample.hpp  |  8 +++--
 reference/matrix/fbcsr_kernels.cpp | 49 +++++++++++++++++++++---------
 4 files changed, 58 insertions(+), 23 deletions(-)

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 1876e421751..00f93bf6351 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -47,6 +47,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
+#include "accessor/block_col_major.hpp"
+#include "accessor/range.hpp"
 #include "core/components/absolute_array.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/matrix/fbcsr_kernels.hpp"
@@ -339,8 +341,9 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
     index_type cur_bcol = blocks.begin()->first.block_column;
     const index_type num_brows = detail::get_num_blocks(bs, data.size[0]);
 
-    range<accessor::block_col_major<value_type, 3>> values(
-        tmp->values_.get_data(), dim<3>(blocks.size(), bs, bs));
+    acc::range<acc::block_col_major<value_type, 3>> values(
+        std::array<size_type, 3>{blocks.size(), (size_type)bs, (size_type)bs},
+        tmp->values_.get_data());
 
     for (auto it = blocks.begin(); it != blocks.end(); it++) {
         GKO_ENSURE_IN_BOUNDS(cur_brow, num_brows);
@@ -386,8 +389,10 @@ void Fbcsr<ValueType, IndexType>::write(mat_data &data) const
     data = {tmp->get_size(), {}};
 
     const size_type nbnz = tmp->get_num_stored_blocks();
-    const range<accessor::block_col_major<const value_type, 3>> vblocks(
-        tmp->values_.get_const_data(), dim<3>(nbnz, bs_, bs_));
+    const acc::range<acc::block_col_major<const value_type, 3>> vblocks(
+        std::array<size_type, 3>{nbnz, static_cast<size_type>(bs_),
+                                 static_cast<size_type>(bs_)},
+        tmp->values_.get_const_data());
 
     for (size_type brow = 0; brow < tmp->get_num_block_rows(); ++brow) {
         const auto start = tmp->row_ptrs_.get_const_data()[brow];
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index abe8f1c1059..6b59ce1d30d 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -40,6 +40,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
+#include <ginkgo/core/base/types.hpp>
+
+
+#include "accessor/block_col_major.hpp"
+#include "accessor/range.hpp"
 #include "core/test/matrix/fbcsr_sample.hpp"
 #include "core/test/utils.hpp"
 
@@ -72,8 +77,10 @@ void assert_matrices_are_same(
     const IndexType nbrows = bm->get_num_block_rows();
     const int bs = bm->get_block_size();
     const auto nbnz = bm->get_num_stored_blocks();
-    gko::range<gko::accessor::block_col_major<const ValueType, 3>> fbvals(
-        bm->get_const_values(), gko::dim<3>(nbnz, bs, bs));
+    gko::acc::range<gko::acc::block_col_major<const ValueType, 3>> fbvals(
+        std::array<gko::size_type, 3>{nbnz, static_cast<gko::size_type>(bs),
+                                      static_cast<gko::size_type>(bs)},
+        bm->get_const_values());
 
     for (IndexType ibrow = 0; ibrow < nbrows; ibrow++) {
         const IndexType *const browptr = bm->get_const_row_ptrs();
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index aa34981cd06..ac6007e789c 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -42,6 +42,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
+#include "accessor/block_col_major.hpp"
+#include "accessor/range.hpp"
 #include "core/test/utils.hpp"
 
 
@@ -105,8 +107,10 @@ class FbcsrSample {
         c[2] = 0;
         c[3] = 2;
 
-        gko::range<gko::accessor::block_col_major<value_type, 3>> vals(
-            v, gko::dim<3>(nbnz, bs, bs));
+        gko::acc::range<gko::acc::block_col_major<value_type, 3>> vals(
+            std::array<size_type, 3>{nbnz, static_cast<size_type>(bs),
+                                     static_cast<size_type>(bs)},
+            v);
 
         if (mtx->get_size()[0] % bs != 0)
             throw gko::BadDimension(__FILE__, __LINE__, __func__, "test fbcsr",
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index a5dbc75d120..a74753623f9 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -42,10 +42,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/range_accessors.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "accessor/block_col_major.hpp"
+#include "accessor/range.hpp"
 #include "core/base/allocator.hpp"
 #include "core/base/iterator_factory.hpp"
 #include "core/components/prefix_sum.hpp"
@@ -77,8 +78,8 @@ void spmv(const std::shared_ptr<const ReferenceExecutor>,
     auto row_ptrs = a->get_const_row_ptrs();
     auto col_idxs = a->get_const_col_idxs();
     auto vals = a->get_const_values();
-    const range<accessor::block_col_major<const ValueType, 3>> avalues{
-        vals, dim<3>(nbnz, bs, bs)};
+    const acc::range<acc::block_col_major<const ValueType, 3>> avalues{
+        std::array<size_type, 3>{nbnz, (size_type)bs, (size_type)bs}, vals};
 
     for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
         for (IndexType i = ibrow * bs * nvecs; i < (ibrow + 1) * bs * nvecs;
@@ -120,8 +121,11 @@ void advanced_spmv(const std::shared_ptr<const ReferenceExecutor>,
     auto vals = a->get_const_values();
     auto valpha = alpha->at(0, 0);
     auto vbeta = beta->at(0, 0);
-    const range<accessor::block_col_major<const ValueType, 3>> avalues{
-        vals, dim<3>(a->get_num_stored_blocks(), bs, bs)};
+    const acc::range<acc::block_col_major<const ValueType, 3>> avalues{
+        std::array<size_type, 3>{a->get_num_stored_blocks(),
+                                 static_cast<size_type>(bs),
+                                 static_cast<size_type>(bs)},
+        vals};
 
     for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
         for (IndexType i = ibrow * bs * nvecs; i < (ibrow + 1) * bs * nvecs;
@@ -159,8 +163,11 @@ void convert_to_dense(const std::shared_ptr<const ReferenceExecutor>,
     const IndexType *const col_idxs = source->get_const_col_idxs();
     const ValueType *const vals = source->get_const_values();
 
-    const range<accessor::block_col_major<const ValueType, 3>> values{
-        vals, dim<3>(source->get_num_stored_blocks(), bs, bs)};
+    const acc::range<acc::block_col_major<const ValueType, 3>> values{
+        std::array<size_type, 3>{source->get_num_stored_blocks(),
+                                 static_cast<size_type>(bs),
+                                 static_cast<size_type>(bs)},
+        vals};
 
     for (IndexType brow = 0; brow < nbrows; ++brow) {
         for (size_type bcol = 0; bcol < nbcols; ++bcol) {
@@ -209,8 +216,11 @@ void convert_to_csr(const std::shared_ptr<const ReferenceExecutor>,
     IndexType *const col_idxs = result->get_col_idxs();
     ValueType *const vals = result->get_values();
 
-    const range<accessor::block_col_major<const ValueType, 3>> bvalues{
-        bvals, dim<3>(source->get_num_stored_blocks(), bs, bs)};
+    const acc::range<acc::block_col_major<const ValueType, 3>> bvalues{
+        std::array<size_type, 3>{source->get_num_stored_blocks(),
+                                 static_cast<size_type>(bs),
+                                 static_cast<size_type>(bs)},
+        bvals};
 
     for (IndexType brow = 0; brow < nbrows; ++brow) {
         const IndexType nz_browstart = browptrs[brow] * bs * bs;
@@ -254,10 +264,16 @@ void convert_fbcsr_to_fbcsc(const IndexType num_blk_rows, const int blksz,
                             IndexType *const col_ptrs,
                             ValueType *const csc_vals, UnaryOperator op)
 {
-    const range<accessor::block_col_major<const ValueType, 3>> rvalues{
-        fbcsr_vals, dim<3>(row_ptrs[num_blk_rows], blksz, blksz)};
-    const range<accessor::block_col_major<ValueType, 3>> cvalues{
-        csc_vals, dim<3>(row_ptrs[num_blk_rows], blksz, blksz)};
+    const acc::range<acc::block_col_major<const ValueType, 3>> rvalues{
+        std::array<size_type, 3>{static_cast<size_type>(row_ptrs[num_blk_rows]),
+                                 static_cast<size_type>(blksz),
+                                 static_cast<size_type>(blksz)},
+        fbcsr_vals};
+    const acc::range<acc::block_col_major<ValueType, 3>> cvalues{
+        std::array<size_type, 3>{static_cast<size_type>(row_ptrs[num_blk_rows]),
+                                 static_cast<size_type>(blksz),
+                                 static_cast<size_type>(blksz)},
+        csc_vals};
     for (IndexType brow = 0; brow < num_blk_rows; ++brow) {
         for (auto i = row_ptrs[brow]; i < row_ptrs[brow + 1]; ++i) {
             const auto dest_idx = col_ptrs[col_idxs[i]];
@@ -463,8 +479,11 @@ void extract_diagonal(std::shared_ptr<const ReferenceExecutor>,
 
     assert(diag->get_size()[0] == nbdim_min * bs);
 
-    const range<accessor::block_col_major<const ValueType, 3>> vblocks{
-        values, dim<3>(orig->get_num_stored_blocks(), bs, bs)};
+    const acc::range<acc::block_col_major<const ValueType, 3>> vblocks{
+        std::array<size_type, 3>{orig->get_num_stored_blocks(),
+                                 static_cast<size_type>(bs),
+                                 static_cast<size_type>(bs)},
+        values};
 
     for (IndexType ibrow = 0; ibrow < nbdim_min; ++ibrow) {
         for (IndexType idx = row_ptrs[ibrow]; idx < row_ptrs[ibrow + 1];

From 2ef5cea1c0f27bcb8a687a8603869b9b6ae54248 Mon Sep 17 00:00:00 2001
From: Aditya <aditya.kashi@kit.edu>
Date: Tue, 23 Mar 2021 14:43:59 +0100
Subject: [PATCH 0586/1008] Fixed C-style casts
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 core/matrix/fbcsr.cpp              | 3 ++-
 reference/matrix/fbcsr_kernels.cpp | 4 +++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 00f93bf6351..5019a047923 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -342,7 +342,8 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
     const index_type num_brows = detail::get_num_blocks(bs, data.size[0]);
 
     acc::range<acc::block_col_major<value_type, 3>> values(
-        std::array<size_type, 3>{blocks.size(), (size_type)bs, (size_type)bs},
+        std::array<size_type, 3>{blocks.size(), static_cast<size_type>(bs),
+                                 static_cast<size_type>(bs)},
         tmp->values_.get_data());
 
     for (auto it = blocks.begin(); it != blocks.end(); it++) {
diff --git a/reference/matrix/fbcsr_kernels.cpp b/reference/matrix/fbcsr_kernels.cpp
index a74753623f9..35c0bcbbf8b 100644
--- a/reference/matrix/fbcsr_kernels.cpp
+++ b/reference/matrix/fbcsr_kernels.cpp
@@ -79,7 +79,9 @@ void spmv(const std::shared_ptr<const ReferenceExecutor>,
     auto col_idxs = a->get_const_col_idxs();
     auto vals = a->get_const_values();
     const acc::range<acc::block_col_major<const ValueType, 3>> avalues{
-        std::array<size_type, 3>{nbnz, (size_type)bs, (size_type)bs}, vals};
+        std::array<size_type, 3>{nbnz, static_cast<size_type>(bs),
+                                 static_cast<size_type>(bs)},
+        vals};
 
     for (IndexType ibrow = 0; ibrow < nbrows; ++ibrow) {
         for (IndexType i = ibrow * bs * nvecs; i < (ibrow + 1) * bs * nvecs;

From 33059eccf9cd0e14b07153c969d234eacf30da1a Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 30 Mar 2021 13:38:15 +0200
Subject: [PATCH 0587/1008] conversion-based mixed precision SpMV & BLAS

---
 core/base/combination.cpp                     |  34 ++-
 core/base/composition.cpp                     |  37 ++-
 core/base/perturbation.cpp                    |  48 ++--
 core/matrix/coo.cpp                           |  74 ++---
 core/matrix/csr.cpp                           |  37 +--
 core/matrix/dense.cpp                         |  56 ++--
 core/matrix/diagonal.cpp                      |  49 ++--
 core/matrix/ell.cpp                           |  38 +--
 core/matrix/hybrid.cpp                        |  28 +-
 core/matrix/identity.cpp                      |  10 +-
 core/matrix/sellp.cpp                         |  38 +--
 core/matrix/sparsity_csr.cpp                  |  19 +-
 core/preconditioner/jacobi.cpp                |  33 ++-
 core/solver/bicg.cpp                          |  34 ++-
 core/solver/bicgstab.cpp                      |  36 ++-
 core/solver/cb_gmres.cpp                      |  41 ++-
 core/solver/cg.cpp                            |  34 ++-
 core/solver/cgs.cpp                           |  34 ++-
 core/solver/fcg.cpp                           |  33 ++-
 core/solver/gmres.cpp                         |  37 ++-
 core/solver/idr.cpp                           |  49 ++--
 core/solver/ir.cpp                            |  34 ++-
 core/solver/lower_trs.cpp                     |  71 ++---
 core/solver/upper_trs.cpp                     |  71 ++---
 .../ginkgo/core/base/precision_dispatch.hpp   | 232 ++++++++++++++++
 .../ginkgo/core/base/temporary_conversion.hpp | 253 ++++++++++++++++++
 include/ginkgo/core/preconditioner/ic.hpp     |  29 +-
 include/ginkgo/core/preconditioner/ilu.hpp    |  56 ++--
 include/ginkgo/core/solver/bicg.hpp           |   4 +
 include/ginkgo/core/solver/bicgstab.hpp       |   4 +
 include/ginkgo/core/solver/cb_gmres.hpp       |   4 +
 include/ginkgo/core/solver/cg.hpp             |   4 +
 include/ginkgo/core/solver/cgs.hpp            |   4 +
 include/ginkgo/core/solver/fcg.hpp            |   4 +
 include/ginkgo/core/solver/gmres.hpp          |   4 +
 include/ginkgo/core/solver/idr.hpp            |   3 +-
 include/ginkgo/core/solver/ir.hpp             |   3 +
 include/ginkgo/ginkgo.hpp                     |   2 +
 38 files changed, 1122 insertions(+), 459 deletions(-)
 create mode 100644 include/ginkgo/core/base/precision_dispatch.hpp
 create mode 100644 include/ginkgo/core/base/temporary_conversion.hpp

diff --git a/core/base/combination.cpp b/core/base/combination.cpp
index 61bd892af55..5051f43565a 100644
--- a/core/base/combination.cpp
+++ b/core/base/combination.cpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/combination.hpp>
 
 
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
@@ -103,10 +104,16 @@ void Combination<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 {
     initialize_scalars<ValueType>(this->get_executor(), cache_.zero,
                                   cache_.one);
-    operators_[0]->apply(lend(coefficients_[0]), b, lend(cache_.zero), x);
-    for (size_type i = 1; i < operators_.size(); ++i) {
-        operators_[i]->apply(lend(coefficients_[i]), b, lend(cache_.one), x);
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            operators_[0]->apply(lend(coefficients_[0]), dense_b,
+                                 lend(cache_.zero), dense_x);
+            for (size_type i = 1; i < operators_.size(); ++i) {
+                operators_[i]->apply(lend(coefficients_[i]), dense_b,
+                                     lend(cache_.one), dense_x);
+            }
+        },
+        b, x);
 }
 
 
@@ -114,14 +121,17 @@ template <typename ValueType>
 void Combination<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                         const LinOp *beta, LinOp *x) const
 {
-    if (cache_.intermediate_x == nullptr ||
-        cache_.intermediate_x->get_size() != x->get_size()) {
-        cache_.intermediate_x = x->clone();
-    }
-    this->apply_impl(b, lend(cache_.intermediate_x));
-    auto dense_x = as<matrix::Dense<ValueType>>(x);
-    dense_x->scale(beta);
-    dense_x->add_scaled(alpha, lend(cache_.intermediate_x));
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            if (cache_.intermediate_x == nullptr ||
+                cache_.intermediate_x->get_size() != dense_x->get_size()) {
+                cache_.intermediate_x = dense_x->clone();
+            }
+            this->apply_impl(dense_b, lend(cache_.intermediate_x));
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, lend(cache_.intermediate_x));
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/base/composition.cpp b/core/base/composition.cpp
index f5826abb396..d383b0174b6 100644
--- a/core/base/composition.cpp
+++ b/core/base/composition.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <iterator>
 
 
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
@@ -164,12 +165,17 @@ std::unique_ptr<LinOp> Composition<ValueType>::conj_transpose() const
 template <typename ValueType>
 void Composition<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    if (operators_.size() > 1) {
-        operators_[0]->apply(
-            lend(apply_inner_operators(operators_, storage_, b)), x);
-    } else {
-        operators_[0]->apply(b, x);
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            if (operators_.size() > 1) {
+                operators_[0]->apply(
+                    lend(apply_inner_operators(operators_, storage_, dense_b)),
+                    dense_x);
+            } else {
+                operators_[0]->apply(dense_b, dense_x);
+            }
+        },
+        b, x);
 }
 
 
@@ -177,13 +183,18 @@ template <typename ValueType>
 void Composition<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                         const LinOp *beta, LinOp *x) const
 {
-    if (operators_.size() > 1) {
-        operators_[0]->apply(
-            alpha, lend(apply_inner_operators(operators_, storage_, b)), beta,
-            x);
-    } else {
-        operators_[0]->apply(alpha, b, beta, x);
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            if (operators_.size() > 1) {
+                operators_[0]->apply(
+                    dense_alpha,
+                    lend(apply_inner_operators(operators_, storage_, dense_b)),
+                    dense_beta, dense_x);
+            } else {
+                operators_[0]->apply(dense_alpha, dense_b, dense_beta, dense_x);
+            }
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/base/perturbation.cpp b/core/base/perturbation.cpp
index 19ab053d403..c7fead043d6 100644
--- a/core/base/perturbation.cpp
+++ b/core/base/perturbation.cpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/perturbation.hpp>
 
 
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
@@ -46,15 +47,18 @@ void Perturbation<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     // temp = projector * b                 : projector->apply(b, temp)
     // x = b                                : x->copy_from(b)
     // x = 1 * x + scalar * basis * temp    : basis->apply(scalar, temp, 1, x)
-    using vec = gko::matrix::Dense<ValueType>;
-    auto exec = this->get_executor();
-    auto intermediate_size =
-        gko::dim<2>(projector_->get_size()[0], b->get_size()[1]);
-    cache_.allocate(exec, intermediate_size);
-    projector_->apply(b, lend(cache_.intermediate));
-    x->copy_from(b);
-    basis_->apply(lend(scalar_), lend(cache_.intermediate), lend(cache_.one),
-                  x);
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            auto exec = this->get_executor();
+            auto intermediate_size =
+                gko::dim<2>(projector_->get_size()[0], dense_b->get_size()[1]);
+            cache_.allocate(exec, intermediate_size);
+            projector_->apply(dense_b, lend(cache_.intermediate));
+            dense_x->copy_from(dense_b);
+            basis_->apply(lend(scalar_), lend(cache_.intermediate),
+                          lend(cache_.one), dense_x);
+        },
+        b, x);
 }
 
 
@@ -69,18 +73,20 @@ void Perturbation<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
     //                            x->add_scaled(alpha, b)
     // x = x + alpha * scalar * basis * temp
     //                          : basis->apply(alpha * scalar, temp, 1, x)
-    using vec = gko::matrix::Dense<ValueType>;
-    auto exec = this->get_executor();
-    auto intermediate_size =
-        gko::dim<2>(projector_->get_size()[0], b->get_size()[1]);
-    cache_.allocate(exec, intermediate_size);
-    projector_->apply(b, lend(cache_.intermediate));
-    auto vec_x = as<vec>(x);
-    vec_x->scale(beta);
-    vec_x->add_scaled(alpha, b);
-    alpha->apply(lend(scalar_), lend(cache_.alpha_scalar));
-    basis_->apply(lend(cache_.alpha_scalar), lend(cache_.intermediate),
-                  lend(cache_.one), vec_x);
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            auto exec = this->get_executor();
+            auto intermediate_size =
+                gko::dim<2>(projector_->get_size()[0], dense_b->get_size()[1]);
+            cache_.allocate(exec, intermediate_size);
+            projector_->apply(dense_b, lend(cache_.intermediate));
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, dense_b);
+            dense_alpha->apply(lend(scalar_), lend(cache_.alpha_scalar));
+            basis_->apply(lend(cache_.alpha_scalar), lend(cache_.intermediate),
+                          lend(cache_.one), dense_x);
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/matrix/coo.cpp b/core/matrix/coo.cpp
index 55ec75371aa..efa2b160a92 100644
--- a/core/matrix/coo.cpp
+++ b/core/matrix/coo.cpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
@@ -77,17 +78,11 @@ GKO_REGISTER_OPERATION(outplace_absolute_array,
 template <typename ValueType, typename IndexType>
 void Coo<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    using ComplexDense = Dense<to_complex<ValueType>>;
-
-    if (dynamic_cast<const Dense<ValueType> *>(b)) {
-        this->get_executor()->run(coo::make_spmv(this, as<Dense<ValueType>>(b),
-                                                 as<Dense<ValueType>>(x)));
-    } else {
-        auto dense_b = as<ComplexDense>(b);
-        auto dense_x = as<ComplexDense>(x);
-        this->apply(dense_b->create_real_view().get(),
-                    dense_x->create_real_view().get());
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->get_executor()->run(coo::make_spmv(this, dense_b, dense_x));
+        },
+        b, x);
 }
 
 
@@ -95,38 +90,23 @@ template <typename ValueType, typename IndexType>
 void Coo<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                            const LinOp *beta, LinOp *x) const
 {
-    using ComplexDense = Dense<to_complex<ValueType>>;
-    using RealDense = Dense<remove_complex<ValueType>>;
-
-    if (dynamic_cast<const Dense<ValueType> *>(b)) {
-        this->get_executor()->run(coo::make_advanced_spmv(
-            as<Dense<ValueType>>(alpha), this, as<Dense<ValueType>>(b),
-            as<Dense<ValueType>>(beta), as<Dense<ValueType>>(x)));
-    } else {
-        auto dense_b = as<ComplexDense>(b);
-        auto dense_x = as<ComplexDense>(x);
-        auto dense_alpha = as<RealDense>(alpha);
-        auto dense_beta = as<RealDense>(beta);
-        this->apply(dense_alpha, dense_b->create_real_view().get(), dense_beta,
-                    dense_x->create_real_view().get());
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            this->get_executor()->run(coo::make_advanced_spmv(
+                dense_alpha, this, dense_b, dense_beta, dense_x));
+        },
+        alpha, b, beta, x);
 }
 
 
 template <typename ValueType, typename IndexType>
 void Coo<ValueType, IndexType>::apply2_impl(const LinOp *b, LinOp *x) const
 {
-    using ComplexDense = Dense<to_complex<ValueType>>;
-
-    if (dynamic_cast<const Dense<ValueType> *>(b)) {
-        this->get_executor()->run(coo::make_spmv2(this, as<Dense<ValueType>>(b),
-                                                  as<Dense<ValueType>>(x)));
-    } else {
-        auto dense_b = as<ComplexDense>(b);
-        auto dense_x = as<ComplexDense>(x);
-        this->apply2(dense_b->create_real_view().get(),
-                     dense_x->create_real_view().get());
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->get_executor()->run(coo::make_spmv2(this, dense_b, dense_x));
+        },
+        b, x);
 }
 
 
@@ -134,20 +114,12 @@ template <typename ValueType, typename IndexType>
 void Coo<ValueType, IndexType>::apply2_impl(const LinOp *alpha, const LinOp *b,
                                             LinOp *x) const
 {
-    using ComplexDense = Dense<to_complex<ValueType>>;
-    using RealDense = Dense<remove_complex<ValueType>>;
-
-    if (dynamic_cast<const Dense<ValueType> *>(b)) {
-        this->get_executor()->run(coo::make_advanced_spmv2(
-            as<Dense<ValueType>>(alpha), this, as<Dense<ValueType>>(b),
-            as<Dense<ValueType>>(x)));
-    } else {
-        auto dense_b = as<ComplexDense>(b);
-        auto dense_x = as<ComplexDense>(x);
-        auto dense_alpha = as<RealDense>(alpha);
-        this->apply2(dense_alpha, dense_b->create_real_view().get(),
-                     dense_x->create_real_view().get());
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_x) {
+            this->get_executor()->run(
+                coo::make_advanced_spmv2(dense_alpha, this, dense_b, dense_x));
+        },
+        alpha, b, x);
 }
 
 
diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp
index 64ed24d49be..cd09474cf63 100644
--- a/core/matrix/csr.cpp
+++ b/core/matrix/csr.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
@@ -102,16 +103,12 @@ void Csr<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
         auto x_csr = as<TCsr>(x);
         this->get_executor()->run(csr::make_spgemm(this, b_csr, x_csr));
     } else {
-        // otherwise we assume that b is dense and compute a SpMV/SpMM
-        if (dynamic_cast<const Dense<ValueType> *>(b)) {
-            this->get_executor()->run(csr::make_spmv(
-                this, as<Dense<ValueType>>(b), as<Dense<ValueType>>(x)));
-        } else {
-            auto dense_b = as<ComplexDense>(b);
-            auto dense_x = as<ComplexDense>(x);
-            this->apply(dense_b->create_real_view().get(),
-                        dense_x->create_real_view().get());
-        }
+        precision_dispatch_real_complex<ValueType>(
+            [this](auto dense_b, auto dense_x) {
+                this->get_executor()->run(
+                    csr::make_spmv(this, dense_b, dense_x));
+            },
+            b, x);
     }
 }
 
@@ -138,19 +135,13 @@ void Csr<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
             csr::make_spgeam(as<Dense<ValueType>>(alpha), this,
                              as<Dense<ValueType>>(beta), lend(x_copy), x_csr));
     } else {
-        // otherwise we assume that b is dense and compute a SpMV/SpMM
-        if (dynamic_cast<const Dense<ValueType> *>(b)) {
-            this->get_executor()->run(csr::make_advanced_spmv(
-                as<Dense<ValueType>>(alpha), this, as<Dense<ValueType>>(b),
-                as<Dense<ValueType>>(beta), as<Dense<ValueType>>(x)));
-        } else {
-            auto dense_b = as<ComplexDense>(b);
-            auto dense_x = as<ComplexDense>(x);
-            auto dense_alpha = as<RealDense>(alpha);
-            auto dense_beta = as<RealDense>(beta);
-            this->apply(dense_alpha, dense_b->create_real_view().get(),
-                        dense_beta, dense_x->create_real_view().get());
-        }
+        precision_dispatch_real_complex<ValueType>(
+            [this](auto dense_alpha, auto dense_b, auto dense_beta,
+                   auto dense_x) {
+                this->get_executor()->run(csr::make_advanced_spmv(
+                    dense_alpha, this, dense_b, dense_beta, dense_x));
+            },
+            alpha, b, beta, x);
     }
 }
 
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 3a24dedddc5..1fe202fade8 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
@@ -222,15 +223,12 @@ inline void conversion_helper(SparsityCsr<ValueType, IndexType> *result,
 template <typename ValueType>
 void Dense<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    if (dynamic_cast<const Dense<ValueType> *>(b)) {
-        this->get_executor()->run(dense::make_simple_apply(
-            this, as<Dense<ValueType>>(b), as<Dense<ValueType>>(x)));
-    } else {
-        auto dense_b = as<Dense<to_complex<ValueType>>>(b);
-        auto dense_x = as<Dense<to_complex<ValueType>>>(x);
-        this->apply(dense_b->create_real_view().get(),
-                    dense_x->create_real_view().get());
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->get_executor()->run(
+                dense::make_simple_apply(this, dense_b, dense_x));
+        },
+        b, x);
 }
 
 
@@ -238,18 +236,12 @@ template <typename ValueType>
 void Dense<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                   const LinOp *beta, LinOp *x) const
 {
-    if (dynamic_cast<const Dense<ValueType> *>(b)) {
-        this->get_executor()->run(dense::make_apply(
-            as<Dense<ValueType>>(alpha), this, as<Dense<ValueType>>(b),
-            as<Dense<ValueType>>(beta), as<Dense<ValueType>>(x)));
-    } else {
-        auto dense_b = as<Dense<to_complex<ValueType>>>(b);
-        auto dense_x = as<Dense<to_complex<ValueType>>>(x);
-        auto dense_alpha = as<Dense<remove_complex<ValueType>>>(alpha);
-        auto dense_beta = as<Dense<remove_complex<ValueType>>>(beta);
-        this->apply(dense_alpha, dense_b->create_real_view().get(), dense_beta,
-                    dense_x->create_real_view().get());
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            this->get_executor()->run(dense::make_apply(
+                dense_alpha, this, dense_b, dense_beta, dense_x));
+        },
+        alpha, b, beta, x);
 }
 
 
@@ -269,7 +261,8 @@ void Dense<ValueType>::scale_impl(const LinOp *alpha)
         GKO_ASSERT_EQUAL_COLS(this, alpha);
     }
     auto exec = this->get_executor();
-    exec->run(dense::make_scale(as<Dense<ValueType>>(alpha), this));
+    exec->run(dense::make_scale(
+        make_temporary_conversion<ValueType>(alpha).get(), this));
 }
 
 
@@ -288,11 +281,11 @@ void Dense<ValueType>::add_scaled_impl(const LinOp *alpha, const LinOp *b)
         exec->run(dense::make_add_scaled_diag(
             as<Dense<ValueType>>(alpha),
             dynamic_cast<const Diagonal<ValueType> *>(b), this));
-        return;
+    } else {
+        exec->run(dense::make_add_scaled(
+            make_temporary_conversion<ValueType>(alpha).get(),
+            make_temporary_conversion<ValueType>(b).get(), this));
     }
-
-    exec->run(dense::make_add_scaled(as<Dense<ValueType>>(alpha),
-                                     as<Dense<ValueType>>(b), this));
 }
 
 
@@ -302,19 +295,20 @@ void Dense<ValueType>::compute_dot_impl(const LinOp *b, LinOp *result) const
     GKO_ASSERT_EQUAL_DIMENSIONS(this, b);
     GKO_ASSERT_EQUAL_DIMENSIONS(result, dim<2>(1, this->get_size()[1]));
     auto exec = this->get_executor();
-    exec->run(dense::make_compute_dot(this, as<Dense<ValueType>>(b),
-                                      as<Dense<ValueType>>(result)));
+    exec->run(dense::make_compute_dot(
+        this, make_temporary_conversion<ValueType>(b).get(),
+        make_temporary_conversion<ValueType>(result).get()));
 }
 
 
 template <typename ValueType>
 void Dense<ValueType>::compute_norm2_impl(LinOp *result) const
 {
-    using NormVector = Dense<remove_complex<ValueType>>;
     GKO_ASSERT_EQUAL_DIMENSIONS(result, dim<2>(1, this->get_size()[1]));
     auto exec = this->get_executor();
-    exec->run(dense::make_compute_norm2(as<Dense<ValueType>>(this),
-                                        as<NormVector>(result)));
+    exec->run(dense::make_compute_norm2(
+        this,
+        make_temporary_conversion<remove_complex<ValueType>>(result).get()));
 }
 
 
diff --git a/core/matrix/diagonal.cpp b/core/matrix/diagonal.cpp
index 7bca7555ae7..18a4785fb13 100644
--- a/core/matrix/diagonal.cpp
+++ b/core/matrix/diagonal.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
@@ -67,12 +68,8 @@ void Diagonal<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 {
     auto exec = this->get_executor();
 
-    if (dynamic_cast<const Dense<ValueType> *>(b) &&
-        dynamic_cast<Dense<ValueType> *>(x)) {
-        exec->run(diagonal::make_apply_to_dense(this, as<Dense<ValueType>>(b),
-                                                as<Dense<ValueType>>(x)));
-    } else if (dynamic_cast<const Csr<ValueType, int32> *>(b) &&
-               dynamic_cast<Csr<ValueType, int32> *>(x)) {
+    if (dynamic_cast<const Csr<ValueType, int32> *>(b) &&
+        dynamic_cast<Csr<ValueType, int32> *>(x)) {
         exec->run(diagonal::make_apply_to_csr(
             this, as<Csr<ValueType, int32>>(b), as<Csr<ValueType, int32>>(x)));
     } else if (dynamic_cast<const Csr<ValueType, int64> *>(b) &&
@@ -80,7 +77,12 @@ void Diagonal<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         exec->run(diagonal::make_apply_to_csr(
             this, as<Csr<ValueType, int64>>(b), as<Csr<ValueType, int64>>(x)));
     } else {
-        GKO_NOT_IMPLEMENTED;
+        precision_dispatch_real_complex<ValueType>(
+            [this, &exec](auto dense_b, auto dense_x) {
+                exec->run(
+                    diagonal::make_apply_to_dense(this, dense_b, dense_x));
+            },
+            b, x);
     }
 }
 
@@ -90,12 +92,8 @@ void Diagonal<ValueType>::rapply_impl(const LinOp *b, LinOp *x) const
 {
     auto exec = this->get_executor();
 
-    if (dynamic_cast<const Dense<ValueType> *>(b) &&
-        dynamic_cast<Dense<ValueType> *>(x)) {
-        exec->run(diagonal::make_right_apply_to_dense(
-            this, as<Dense<ValueType>>(b), as<Dense<ValueType>>(x)));
-    } else if (dynamic_cast<const Csr<ValueType, int32> *>(b) &&
-               dynamic_cast<Csr<ValueType, int32> *>(x)) {
+    if (dynamic_cast<const Csr<ValueType, int32> *>(b) &&
+        dynamic_cast<Csr<ValueType, int32> *>(x)) {
         exec->run(diagonal::make_right_apply_to_csr(
             this, as<Csr<ValueType, int32>>(b), as<Csr<ValueType, int32>>(x)));
     } else if (dynamic_cast<const Csr<ValueType, int64> *>(b) &&
@@ -103,7 +101,14 @@ void Diagonal<ValueType>::rapply_impl(const LinOp *b, LinOp *x) const
         exec->run(diagonal::make_right_apply_to_csr(
             this, as<Csr<ValueType, int64>>(b), as<Csr<ValueType, int64>>(x)));
     } else {
-        GKO_NOT_IMPLEMENTED;
+        // no real-to-complex conversion, as this would require doubling the
+        // diagonal entries for the complex-to-real columns
+        precision_dispatch<ValueType>(
+            [this, &exec](auto dense_b, auto dense_x) {
+                exec->run(diagonal::make_right_apply_to_dense(this, dense_b,
+                                                              dense_x));
+            },
+            b, x);
     }
 }
 
@@ -112,14 +117,14 @@ template <typename ValueType>
 void Diagonal<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                      const LinOp *beta, LinOp *x) const
 {
-    if (dynamic_cast<const Dense<ValueType> *>(b) &&
-        dynamic_cast<Dense<ValueType> *>(x)) {
-        auto dense_x = as<Dense<ValueType>>(x);
-        dense_x->scale(beta);
-        dense_x->add_scaled(alpha, b);
-    } else {
-        GKO_NOT_IMPLEMENTED;
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            auto x_clone = dense_x->clone();
+            this->apply(dense_b, x_clone.get());
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, x_clone.get());
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/matrix/ell.cpp b/core/matrix/ell.cpp
index f2855383e98..0dc9a950f75 100644
--- a/core/matrix/ell.cpp
+++ b/core/matrix/ell.cpp
@@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
@@ -101,17 +102,11 @@ size_type calculate_max_nnz_per_row(
 template <typename ValueType, typename IndexType>
 void Ell<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    using ComplexDense = Dense<to_complex<ValueType>>;
-
-    if (dynamic_cast<const Dense<ValueType> *>(b)) {
-        this->get_executor()->run(ell::make_spmv(this, as<Dense<ValueType>>(b),
-                                                 as<Dense<ValueType>>(x)));
-    } else {
-        auto dense_b = as<ComplexDense>(b);
-        auto dense_x = as<ComplexDense>(x);
-        this->apply(dense_b->create_real_view().get(),
-                    dense_x->create_real_view().get());
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->get_executor()->run(ell::make_spmv(this, dense_b, dense_x));
+        },
+        b, x);
 }
 
 
@@ -119,21 +114,12 @@ template <typename ValueType, typename IndexType>
 void Ell<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                            const LinOp *beta, LinOp *x) const
 {
-    using ComplexDense = Dense<to_complex<ValueType>>;
-    using RealDense = Dense<remove_complex<ValueType>>;
-
-    if (dynamic_cast<const Dense<ValueType> *>(b)) {
-        this->get_executor()->run(ell::make_advanced_spmv(
-            as<Dense<ValueType>>(alpha), this, as<Dense<ValueType>>(b),
-            as<Dense<ValueType>>(beta), as<Dense<ValueType>>(x)));
-    } else {
-        auto dense_b = as<ComplexDense>(b);
-        auto dense_x = as<ComplexDense>(x);
-        auto dense_alpha = as<RealDense>(alpha);
-        auto dense_beta = as<RealDense>(beta);
-        this->apply(dense_alpha, dense_b->create_real_view().get(), dense_beta,
-                    dense_x->create_real_view().get());
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            this->get_executor()->run(ell::make_advanced_spmv(
+                dense_alpha, this, dense_b, dense_beta, dense_x));
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/matrix/hybrid.cpp b/core/matrix/hybrid.cpp
index cf5cd017ad2..f6cf69f2955 100644
--- a/core/matrix/hybrid.cpp
+++ b/core/matrix/hybrid.cpp
@@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
@@ -101,10 +102,14 @@ void get_each_row_nnz(const matrix_data<ValueType, IndexType> &data,
 template <typename ValueType, typename IndexType>
 void Hybrid<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    auto ell_mtx = this->get_ell();
-    auto coo_mtx = this->get_coo();
-    ell_mtx->apply(b, x);
-    coo_mtx->apply2(b, x);
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            auto ell_mtx = this->get_ell();
+            auto coo_mtx = this->get_coo();
+            ell_mtx->apply(dense_b, dense_x);
+            coo_mtx->apply2(dense_b, dense_x);
+        },
+        b, x);
 }
 
 
@@ -113,10 +118,14 @@ void Hybrid<ValueType, IndexType>::apply_impl(const LinOp *alpha,
                                               const LinOp *b, const LinOp *beta,
                                               LinOp *x) const
 {
-    auto ell_mtx = this->get_ell();
-    auto coo_mtx = this->get_coo();
-    ell_mtx->apply(alpha, b, beta, x);
-    coo_mtx->apply2(alpha, b, x);
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            auto ell_mtx = this->get_ell();
+            auto coo_mtx = this->get_coo();
+            ell_mtx->apply(dense_alpha, dense_b, dense_beta, dense_x);
+            coo_mtx->apply2(dense_alpha, dense_b, dense_x);
+        },
+        alpha, b, beta, x);
 }
 
 
@@ -127,7 +136,8 @@ void Hybrid<ValueType, IndexType>::convert_to(
     this->ell_->convert_to(result->ell_.get());
     this->coo_->convert_to(result->coo_.get());
     // TODO set strategy correctly
-    // There is no way to correctly clone the strategy like in Csr::convert_to
+    // There is no way to correctly clone the strategy like in
+    // Csr::convert_to
     result->set_size(this->get_size());
 }
 
diff --git a/core/matrix/identity.cpp b/core/matrix/identity.cpp
index 95451e2f553..169a4f4ba68 100644
--- a/core/matrix/identity.cpp
+++ b/core/matrix/identity.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
@@ -53,9 +54,12 @@ template <typename ValueType>
 void Identity<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                      const LinOp *beta, LinOp *x) const
 {
-    auto dense_x = as<Dense<ValueType>>(x);
-    dense_x->scale(beta);
-    dense_x->add_scaled(alpha, b);
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, dense_b);
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/matrix/sellp.cpp b/core/matrix/sellp.cpp
index b5f1d9b50e8..07542e3ab40 100644
--- a/core/matrix/sellp.cpp
+++ b/core/matrix/sellp.cpp
@@ -36,6 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
@@ -112,17 +113,11 @@ size_type calculate_total_cols(const matrix_data<ValueType, IndexType> &data,
 template <typename ValueType, typename IndexType>
 void Sellp<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    using ComplexDense = Dense<to_complex<ValueType>>;
-
-    if (dynamic_cast<const Dense<ValueType> *>(b)) {
-        this->get_executor()->run(sellp::make_spmv(
-            this, as<Dense<ValueType>>(b), as<Dense<ValueType>>(x)));
-    } else {
-        auto dense_b = as<ComplexDense>(b);
-        auto dense_x = as<ComplexDense>(x);
-        this->apply(dense_b->create_real_view().get(),
-                    dense_x->create_real_view().get());
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->get_executor()->run(sellp::make_spmv(this, dense_b, dense_x));
+        },
+        b, x);
 }
 
 
@@ -130,21 +125,12 @@ template <typename ValueType, typename IndexType>
 void Sellp<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                              const LinOp *beta, LinOp *x) const
 {
-    using ComplexDense = Dense<to_complex<ValueType>>;
-    using RealDense = Dense<remove_complex<ValueType>>;
-
-    if (dynamic_cast<const Dense<ValueType> *>(b)) {
-        this->get_executor()->run(sellp::make_advanced_spmv(
-            as<Dense<ValueType>>(alpha), this, as<Dense<ValueType>>(b),
-            as<Dense<ValueType>>(beta), as<Dense<ValueType>>(x)));
-    } else {
-        auto dense_b = as<ComplexDense>(b);
-        auto dense_x = as<ComplexDense>(x);
-        auto dense_alpha = as<RealDense>(alpha);
-        auto dense_beta = as<RealDense>(beta);
-        this->apply(dense_alpha, dense_b->create_real_view().get(), dense_beta,
-                    dense_x->create_real_view().get());
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            this->get_executor()->run(sellp::make_advanced_spmv(
+                dense_alpha, this, dense_b, dense_beta, dense_x));
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/matrix/sparsity_csr.cpp b/core/matrix/sparsity_csr.cpp
index b8a8bf5d087..d253792c6a3 100644
--- a/core/matrix/sparsity_csr.cpp
+++ b/core/matrix/sparsity_csr.cpp
@@ -36,6 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
@@ -68,9 +69,12 @@ template <typename ValueType, typename IndexType>
 void SparsityCsr<ValueType, IndexType>::apply_impl(const LinOp *b,
                                                    LinOp *x) const
 {
-    using Dense = Dense<ValueType>;
-    this->get_executor()->run(
-        sparsity_csr::make_spmv(this, as<Dense>(b), as<Dense>(x)));
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->get_executor()->run(
+                sparsity_csr::make_spmv(this, dense_b, dense_x));
+        },
+        b, x);
 }
 
 
@@ -80,9 +84,12 @@ void SparsityCsr<ValueType, IndexType>::apply_impl(const LinOp *alpha,
                                                    const LinOp *beta,
                                                    LinOp *x) const
 {
-    using Dense = Dense<ValueType>;
-    this->get_executor()->run(sparsity_csr::make_advanced_spmv(
-        as<Dense>(alpha), this, as<Dense>(b), as<Dense>(beta), as<Dense>(x)));
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            this->get_executor()->run(sparsity_csr::make_advanced_spmv(
+                dense_alpha, this, dense_b, dense_beta, dense_x));
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index 93abaf6a7fe..5f135b6c010 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
@@ -71,11 +72,14 @@ GKO_REGISTER_OPERATION(initialize_precisions, jacobi::initialize_precisions);
 template <typename ValueType, typename IndexType>
 void Jacobi<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    using dense = matrix::Dense<ValueType>;
-    this->get_executor()->run(jacobi::make_simple_apply(
-        num_blocks_, parameters_.max_block_size, storage_scheme_,
-        parameters_.storage_optimization.block_wise, parameters_.block_pointers,
-        blocks_, as<dense>(b), as<dense>(x)));
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->get_executor()->run(jacobi::make_simple_apply(
+                num_blocks_, parameters_.max_block_size, storage_scheme_,
+                parameters_.storage_optimization.block_wise,
+                parameters_.block_pointers, blocks_, dense_b, dense_x));
+        },
+        b, x);
 }
 
 
@@ -84,12 +88,15 @@ void Jacobi<ValueType, IndexType>::apply_impl(const LinOp *alpha,
                                               const LinOp *b, const LinOp *beta,
                                               LinOp *x) const
 {
-    using dense = matrix::Dense<ValueType>;
-    this->get_executor()->run(jacobi::make_apply(
-        num_blocks_, parameters_.max_block_size, storage_scheme_,
-        parameters_.storage_optimization.block_wise, parameters_.block_pointers,
-        blocks_, as<dense>(alpha), as<dense>(b), as<dense>(beta),
-        as<dense>(x)));
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            this->get_executor()->run(jacobi::make_apply(
+                num_blocks_, parameters_.max_block_size, storage_scheme_,
+                parameters_.storage_optimization.block_wise,
+                parameters_.block_pointers, blocks_, dense_alpha, dense_b,
+                dense_beta, dense_x));
+        },
+        alpha, b, beta, x);
 }
 
 
@@ -220,8 +227,8 @@ void Jacobi<ValueType, IndexType>::generate(const LinOp *system_matrix,
 
     const auto all_block_opt = parameters_.storage_optimization.of_all_blocks;
     auto &precisions = parameters_.storage_optimization.block_wise;
-    // if adaptive version is used, make sure that the precision array is of the
-    // correct size by replicating it multiple times if needed
+    // if adaptive version is used, make sure that the precision array is of
+    // the correct size by replicating it multiple times if needed
     if (parameters_.storage_optimization.is_block_wise ||
         all_block_opt != precision_reduction(0, 0)) {
         if (!parameters_.storage_optimization.is_block_wise) {
diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp
index 300270fd106..642ffb30364 100644
--- a/core/solver/bicg.cpp
+++ b/core/solver/bicg.cpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/name_demangling.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 
 
@@ -109,6 +110,18 @@ std::unique_ptr<LinOp> conj_transpose_with_csr(const LinOp *mtx)
 
 template <typename ValueType>
 void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
+{
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->apply_dense_impl(dense_b, dense_x);
+        },
+        b, x);
+}
+
+
+template <typename ValueType>
+void Bicg<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
+                                       matrix::Dense<ValueType> *dense_x) const
 {
     using std::swap;
     using Vector = matrix::Dense<ValueType>;
@@ -119,8 +132,6 @@ void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     auto one_op = initialize<Vector>({one<ValueType>()}, exec);
     auto neg_one_op = initialize<Vector>({-one<ValueType>()}, exec);
 
-    auto dense_b = as<const Vector>(b);
-    auto dense_x = as<Vector>(x);
     auto r = Vector::create_with_config_of(dense_b);
     auto r2 = Vector::create_with_config_of(dense_b);
     auto z = Vector::create_with_config_of(dense_b);
@@ -179,8 +190,9 @@ void Bicg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     r2->copy_from(r.get());
     // r2 = r
     auto stop_criterion = stop_criterion_factory_->generate(
-        system_matrix_, std::shared_ptr<const LinOp>(b, [](const LinOp *) {}),
-        x, r.get());
+        system_matrix_,
+        std::shared_ptr<const LinOp>(dense_b, [](const LinOp *) {}), dense_x,
+        r.get());
 
     int iter = -1;
 
@@ -234,12 +246,14 @@ template <typename ValueType>
 void Bicg<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                  const LinOp *beta, LinOp *x) const
 {
-    auto dense_x = as<matrix::Dense<ValueType>>(x);
-
-    auto x_clone = dense_x->clone();
-    this->apply(b, x_clone.get());
-    dense_x->scale(beta);
-    dense_x->add_scaled(alpha, x_clone.get());
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            auto x_clone = dense_x->clone();
+            this->apply(dense_b, x_clone.get());
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, x_clone.get());
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/solver/bicgstab.cpp b/core/solver/bicgstab.cpp
index 640af69ad45..8dd89b24607 100644
--- a/core/solver/bicgstab.cpp
+++ b/core/solver/bicgstab.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 
 
@@ -86,6 +87,19 @@ std::unique_ptr<LinOp> Bicgstab<ValueType>::conj_transpose() const
 
 template <typename ValueType>
 void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
+{
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->apply_dense_impl(dense_b, dense_x);
+        },
+        b, x);
+}
+
+
+template <typename ValueType>
+void Bicgstab<ValueType>::apply_dense_impl(
+    const matrix::Dense<ValueType> *dense_b,
+    matrix::Dense<ValueType> *dense_x) const
 {
     using std::swap;
     using Vector = matrix::Dense<ValueType>;
@@ -98,8 +112,6 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     auto one_op = initialize<Vector>({one<ValueType>()}, exec);
     auto neg_one_op = initialize<Vector>({-one<ValueType>()}, exec);
 
-    auto dense_b = as<Vector>(b);
-    auto dense_x = as<Vector>(x);
     auto r = Vector::create_with_config_of(dense_b);
     auto z = Vector::create_with_config_of(dense_b);
     auto y = Vector::create_with_config_of(dense_b);
@@ -132,8 +144,9 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(), r.get());
     auto stop_criterion = stop_criterion_factory_->generate(
-        system_matrix_, std::shared_ptr<const LinOp>(b, [](const LinOp *) {}),
-        x, r.get());
+        system_matrix_,
+        std::shared_ptr<const LinOp>(dense_b, [](const LinOp *) {}), dense_x,
+        r.get());
     rr->copy_from(r.get());
 
     int iter = -1;
@@ -205,18 +218,21 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
             beta.get(), gamma.get(), omega.get(), &stop_status));
         swap(prev_rho, rho);
     }
-}  // namespace solver
+}
 
 
 template <typename ValueType>
 void Bicgstab<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                      const LinOp *beta, LinOp *x) const
 {
-    auto dense_x = as<matrix::Dense<ValueType>>(x);
-    auto x_clone = dense_x->clone();
-    this->apply(b, x_clone.get());
-    dense_x->scale(beta);
-    dense_x->add_scaled(alpha, x_clone.get());
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            auto x_clone = dense_x->clone();
+            this->apply(dense_b, x_clone.get());
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, x_clone.get());
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/solver/cb_gmres.cpp b/core/solver/cb_gmres.cpp
index c7b7384df1a..fb2acca9067 100644
--- a/core/solver/cb_gmres.cpp
+++ b/core/solver/cb_gmres.cpp
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils_helper.hpp>
 #include <ginkgo/core/log/logger.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
@@ -188,8 +189,21 @@ struct helper<std::complex<T>> {
 template <typename ValueType>
 void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    // Current workaround to get a lambda with a template argument (only the
-    // type of `value` matters, the content does not)
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->apply_dense_impl(dense_b, dense_x);
+        },
+        b, x);
+}
+
+
+template <typename ValueType>
+void CbGmres<ValueType>::apply_dense_impl(
+    const matrix::Dense<ValueType> *dense_b,
+    matrix::Dense<ValueType> *dense_x) const
+{
+    // Current workaround to get a lambda with a template argument (only
+    // the type of `value` matters, the content does not)
     auto apply_templated = [&](auto value) {
         using storage_type = decltype(value);
         GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix_);
@@ -207,8 +221,6 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
         auto one_op = initialize<Vector>({one<ValueType>()}, exec);
         auto neg_one_op = initialize<Vector>({-one<ValueType>()}, exec);
 
-        auto dense_b = as<const Vector>(b);
-        auto dense_x = as<Vector>(x);
         auto residual = Vector::create_with_config_of(dense_b);
         /* The dimensions {x, y, z} explained for the krylov_bases:
          * - x: selects the krylov vector (which has krylov_dim + 1 vectors)
@@ -287,8 +299,8 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
         auto stop_criterion = stop_criterion_factory_->generate(
             system_matrix_,
-            std::shared_ptr<const LinOp>(b, [](const LinOp *) {}), x,
-            residual.get());
+            std::shared_ptr<const LinOp>(dense_b, [](const LinOp *) {}),
+            dense_x, residual.get());
 
         int total_iter = -1;
         size_type restart_iter = 0;
@@ -493,15 +505,16 @@ void CbGmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
 template <typename ValueType>
 void CbGmres<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
-                                    const LinOp *residual_norm_collection,
-                                    LinOp *x) const
+                                    const LinOp *beta, LinOp *x) const
 {
-    auto dense_x = as<matrix::Dense<ValueType>>(x);
-
-    auto x_clone = dense_x->clone();
-    this->apply(b, x_clone.get());
-    dense_x->scale(residual_norm_collection);
-    dense_x->add_scaled(alpha, x_clone.get());
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            auto x_clone = dense_x->clone();
+            this->apply(dense_b, x_clone.get());
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, x_clone.get());
+        },
+        alpha, b, beta, x);
 }
 
 #define GKO_DECLARE_CB_GMRES(_type1) class CbGmres<_type1>
diff --git a/core/solver/cg.cpp b/core/solver/cg.cpp
index ba19aa1b1d0..2a4e6c1d8b7 100644
--- a/core/solver/cg.cpp
+++ b/core/solver/cg.cpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/name_demangling.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 
 
@@ -87,6 +88,18 @@ std::unique_ptr<LinOp> Cg<ValueType>::conj_transpose() const
 
 template <typename ValueType>
 void Cg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
+{
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->apply_dense_impl(dense_b, dense_x);
+        },
+        b, x);
+}
+
+
+template <typename ValueType>
+void Cg<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
+                                     matrix::Dense<ValueType> *dense_x) const
 {
     using std::swap;
     using Vector = matrix::Dense<ValueType>;
@@ -98,8 +111,6 @@ void Cg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     auto one_op = initialize<Vector>({one<ValueType>()}, exec);
     auto neg_one_op = initialize<Vector>({-one<ValueType>()}, exec);
 
-    auto dense_b = as<const Vector>(b);
-    auto dense_x = as<Vector>(x);
     auto r = Vector::create_with_config_of(dense_b);
     auto z = Vector::create_with_config_of(dense_b);
     auto p = Vector::create_with_config_of(dense_b);
@@ -124,8 +135,9 @@ void Cg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(), r.get());
     auto stop_criterion = stop_criterion_factory_->generate(
-        system_matrix_, std::shared_ptr<const LinOp>(b, [](const LinOp *) {}),
-        x, r.get());
+        system_matrix_,
+        std::shared_ptr<const LinOp>(dense_b, [](const LinOp *) {}), dense_x,
+        r.get());
 
     int iter = -1;
     /* Memory movement summary:
@@ -173,12 +185,14 @@ template <typename ValueType>
 void Cg<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                const LinOp *beta, LinOp *x) const
 {
-    auto dense_x = as<matrix::Dense<ValueType>>(x);
-
-    auto x_clone = dense_x->clone();
-    this->apply(b, x_clone.get());
-    dense_x->scale(beta);
-    dense_x->add_scaled(alpha, x_clone.get());
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            auto x_clone = dense_x->clone();
+            this->apply(dense_b, x_clone.get());
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, x_clone.get());
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index ffcf800b9d2..e798d593b58 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 
 
@@ -87,11 +88,21 @@ std::unique_ptr<LinOp> Cgs<ValueType>::conj_transpose() const
 
 template <typename ValueType>
 void Cgs<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
+{
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->apply_dense_impl(dense_b, dense_x);
+        },
+        b, x);
+}
+
+
+template <typename ValueType>
+void Cgs<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
+                                      matrix::Dense<ValueType> *dense_x) const
 {
     using std::swap;
     using Vector = matrix::Dense<ValueType>;
-    auto dense_b = as<const Vector>(b);
-    auto dense_x = as<Vector>(x);
 
     constexpr uint8 RelativeStoppingId{1};
 
@@ -133,8 +144,9 @@ void Cgs<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(), r.get());
     auto stop_criterion = stop_criterion_factory_->generate(
-        system_matrix_, std::shared_ptr<const LinOp>(b, [](const LinOp *) {}),
-        x, r.get());
+        system_matrix_,
+        std::shared_ptr<const LinOp>(dense_b, [](const LinOp *) {}), dense_x,
+        r.get());
     r_tld->copy_from(r.get());
 
     int iter = 0;
@@ -194,12 +206,14 @@ template <typename ValueType>
 void Cgs<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                 const LinOp *beta, LinOp *x) const
 {
-    auto dense_x = as<matrix::Dense<ValueType>>(x);
-
-    auto x_clone = dense_x->clone();
-    this->apply(b, x_clone.get());
-    dense_x->scale(beta);
-    dense_x->add_scaled(alpha, x_clone.get());
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            auto x_clone = dense_x->clone();
+            this->apply(dense_b, x_clone.get());
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, x_clone.get());
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/solver/fcg.cpp b/core/solver/fcg.cpp
index bc41dfeca58..0676206b998 100644
--- a/core/solver/fcg.cpp
+++ b/core/solver/fcg.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 
 
@@ -84,11 +85,21 @@ std::unique_ptr<LinOp> Fcg<ValueType>::conj_transpose() const
 
 template <typename ValueType>
 void Fcg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
+{
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->apply_dense_impl(dense_b, dense_x);
+        },
+        b, x);
+}
+
+
+template <typename ValueType>
+void Fcg<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
+                                      matrix::Dense<ValueType> *dense_x) const
 {
     using std::swap;
     using Vector = matrix::Dense<ValueType>;
-    auto dense_b = as<const Vector>(b);
-    auto dense_x = as<Vector>(x);
 
     constexpr uint8 RelativeStoppingId{1};
 
@@ -126,8 +137,9 @@ void Fcg<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
     system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(), r.get());
     auto stop_criterion = stop_criterion_factory_->generate(
-        system_matrix_, std::shared_ptr<const LinOp>(b, [](const LinOp *) {}),
-        x, r.get());
+        system_matrix_,
+        std::shared_ptr<const LinOp>(dense_b, [](const LinOp *) {}), dense_x,
+        r.get());
 
     int iter = -1;
     /* Memory movement summary:
@@ -178,11 +190,14 @@ template <typename ValueType>
 void Fcg<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                 const LinOp *beta, LinOp *x) const
 {
-    auto dense_x = as<matrix::Dense<ValueType>>(x);
-    auto x_clone = dense_x->clone();
-    this->apply(b, x_clone.get());
-    dense_x->scale(beta);
-    dense_x->add_scaled(alpha, x_clone.get());
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            auto x_clone = dense_x->clone();
+            this->apply(dense_b, x_clone.get());
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, x_clone.get());
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/solver/gmres.cpp b/core/solver/gmres.cpp
index 2337b937cb3..9476f339ebe 100644
--- a/core/solver/gmres.cpp
+++ b/core/solver/gmres.cpp
@@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/name_demangling.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
@@ -93,6 +94,18 @@ std::unique_ptr<LinOp> Gmres<ValueType>::conj_transpose() const
 
 template <typename ValueType>
 void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
+{
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->apply_dense_impl(dense_b, dense_x);
+        },
+        b, x);
+}
+
+
+template <typename ValueType>
+void Gmres<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
+                                        matrix::Dense<ValueType> *dense_x) const
 {
     using Vector = matrix::Dense<ValueType>;
     using NormVector = matrix::Dense<remove_complex<ValueType>>;
@@ -104,8 +117,6 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     auto one_op = initialize<Vector>({one<ValueType>()}, exec);
     auto neg_one_op = initialize<Vector>({-one<ValueType>()}, exec);
 
-    auto dense_b = as<const Vector>(b);
-    auto dense_x = as<Vector>(x);
     auto residual = Vector::create_with_config_of(dense_b);
     auto krylov_bases = Vector::create_with_type_of(
         dense_b, exec,
@@ -149,8 +160,9 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     // final_iter_nums = {0, ..., 0}
 
     auto stop_criterion = stop_criterion_factory_->generate(
-        system_matrix_, std::shared_ptr<const LinOp>(b, [](const LinOp *) {}),
-        x, residual.get());
+        system_matrix_,
+        std::shared_ptr<const LinOp>(dense_b, [](const LinOp *) {}), dense_x,
+        residual.get());
 
     int total_iter = -1;
     size_type restart_iter = 0;
@@ -313,15 +325,16 @@ void Gmres<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 
 template <typename ValueType>
 void Gmres<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
-                                  const LinOp *residual_norm_collection,
-                                  LinOp *x) const
+                                  const LinOp *beta, LinOp *x) const
 {
-    auto dense_x = as<matrix::Dense<ValueType>>(x);
-
-    auto x_clone = dense_x->clone();
-    this->apply(b, x_clone.get());
-    dense_x->scale(residual_norm_collection);
-    dense_x->add_scaled(alpha, x_clone.get());
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            auto x_clone = dense_x->clone();
+            this->apply(dense_b, x_clone.get());
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, x_clone.get());
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index 8dcde2e00d9..649ca7ddf0f 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/utils.hpp>
 
 
@@ -88,7 +89,8 @@ std::unique_ptr<LinOp> Idr<ValueType>::conj_transpose() const
 
 template <typename ValueType>
 template <typename SubspaceType>
-void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
+void Idr<ValueType>::iterate(const matrix::Dense<SubspaceType> *dense_b,
+                             matrix::Dense<SubspaceType> *dense_x) const
 {
     using std::swap;
     using Vector = matrix::Dense<SubspaceType>;
@@ -102,9 +104,6 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
         initialize<matrix::Dense<ValueType>>({-one<ValueType>()}, exec);
     auto subspace_neg_one_op = initialize<Vector>({-one<SubspaceType>()}, exec);
 
-    auto dense_b = as<Vector>(b);
-    auto dense_x = as<Vector>(x);
-
     constexpr uint8 RelativeStoppingId{1};
 
     const auto problem_size = system_matrix_->get_size()[0];
@@ -272,18 +271,22 @@ void Idr<ValueType>::iterate(const LinOp *b, LinOp *x) const
 template <typename ValueType>
 void Idr<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    // If ValueType is complex, the subspace matrix P will be complex anyway.
-    if (!is_complex<ValueType>() && complex_subspace_) {
-        auto dense_b = as<matrix::Dense<ValueType>>(b);
-        auto dense_x = as<matrix::Dense<ValueType>>(x);
-        auto complex_b = dense_b->make_complex();
-        auto complex_x = dense_x->make_complex();
-        this->iterate<to_complex<ValueType>>(complex_b.get(), complex_x.get());
-        complex_x->get_real(
-            dynamic_cast<matrix::Dense<remove_complex<ValueType>> *>(dense_x));
-    } else {
-        this->iterate<ValueType>(b, x);
-    }
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            // If ValueType is complex, the subspace matrix P will be complex
+            // anyway.
+            if (!is_complex<ValueType>() && complex_subspace_) {
+                auto complex_b = dense_b->make_complex();
+                auto complex_x = dense_x->make_complex();
+                this->iterate(complex_b.get(), complex_x.get());
+                complex_x->get_real(
+                    dynamic_cast<matrix::Dense<remove_complex<ValueType>> *>(
+                        dense_x));
+            } else {
+                this->iterate(dense_b, dense_x);
+            }
+        },
+        b, x);
 }
 
 
@@ -291,12 +294,14 @@ template <typename ValueType>
 void Idr<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                 const LinOp *beta, LinOp *x) const
 {
-    auto dense_x = as<matrix::Dense<ValueType>>(x);
-
-    auto x_clone = dense_x->clone();
-    this->apply(b, x_clone.get());
-    dense_x->scale(beta);
-    dense_x->add_scaled(alpha, x_clone.get());
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            auto x_clone = dense_x->clone();
+            this->apply(dense_b, x_clone.get());
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, x_clone.get());
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/solver/ir.cpp b/core/solver/ir.cpp
index 0526a701661..a249c660a54 100644
--- a/core/solver/ir.cpp
+++ b/core/solver/ir.cpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/ir.hpp>
 
 
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
@@ -80,6 +81,18 @@ std::unique_ptr<LinOp> Ir<ValueType>::conj_transpose() const
 
 template <typename ValueType>
 void Ir<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
+{
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            this->apply_dense_impl(dense_b, dense_x);
+        },
+        b, x);
+}
+
+
+template <typename ValueType>
+void Ir<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
+                                     matrix::Dense<ValueType> *dense_x) const
 {
     using Vector = matrix::Dense<ValueType>;
     constexpr uint8 relative_stopping_id{1};
@@ -88,8 +101,6 @@ void Ir<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
     auto one_op = initialize<Vector>({one<ValueType>()}, exec);
     auto neg_one_op = initialize<Vector>({-one<ValueType>()}, exec);
 
-    auto dense_b = as<const Vector>(b);
-    auto dense_x = as<Vector>(x);
     auto residual = Vector::create_with_config_of(dense_b);
     auto inner_solution = Vector::create_with_config_of(dense_b);
 
@@ -102,8 +113,9 @@ void Ir<ValueType>::apply_impl(const LinOp *b, LinOp *x) const
                           lend(residual));
 
     auto stop_criterion = stop_criterion_factory_->generate(
-        system_matrix_, std::shared_ptr<const LinOp>(b, [](const LinOp *) {}),
-        x, lend(residual));
+        system_matrix_,
+        std::shared_ptr<const LinOp>(dense_b, [](const LinOp *) {}), dense_x,
+        lend(residual));
 
     int iter = -1;
     while (true) {
@@ -152,12 +164,14 @@ template <typename ValueType>
 void Ir<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                const LinOp *beta, LinOp *x) const
 {
-    auto dense_x = as<matrix::Dense<ValueType>>(x);
-
-    auto x_clone = dense_x->clone();
-    this->apply(b, x_clone.get());
-    dense_x->scale(beta);
-    dense_x->add_scaled(alpha, x_clone.get());
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            auto x_clone = dense_x->clone();
+            this->apply(dense_b, x_clone.get());
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, x_clone.get());
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/solver/lower_trs.cpp b/core/solver/lower_trs.cpp
index 8f5b979a3b9..3fe87356625 100644
--- a/core/solver/lower_trs.cpp
+++ b/core/solver/lower_trs.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/polymorphic_object.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
@@ -101,32 +102,36 @@ void LowerTrs<ValueType, IndexType>::generate()
 template <typename ValueType, typename IndexType>
 void LowerTrs<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    using Vector = matrix::Dense<ValueType>;
-    const auto exec = this->get_executor();
-
-    auto dense_b = as<const Vector>(b);
-    auto dense_x = as<Vector>(x);
-
-    // This kernel checks if a transpose is needed for the multiple rhs case.
-    // Currently only the algorithm for CUDA version <=9.1 needs this
-    // transposition due to the limitation in the cusparse algorithm. The other
-    // executors (omp and reference) do not use the transpose (trans_x and
-    // trans_b) and hence are passed in empty pointers.
-    bool do_transpose = false;
-    std::shared_ptr<Vector> trans_b;
-    std::shared_ptr<Vector> trans_x;
-    this->get_executor()->run(
-        lower_trs::make_should_perform_transpose(do_transpose));
-    if (do_transpose) {
-        trans_b = Vector::create(exec, gko::transpose(dense_b->get_size()));
-        trans_x = Vector::create(exec, gko::transpose(dense_x->get_size()));
-    } else {
-        trans_b = Vector::create(exec);
-        trans_x = Vector::create(exec);
-    }
-    exec->run(lower_trs::make_solve(
-        gko::lend(system_matrix_), gko::lend(this->solve_struct_),
-        gko::lend(trans_b), gko::lend(trans_x), dense_b, dense_x));
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            using Vector = matrix::Dense<ValueType>;
+            const auto exec = this->get_executor();
+
+            // This kernel checks if a transpose is needed for the multiple rhs
+            // case. Currently only the algorithm for CUDA version <=9.1 needs
+            // this transposition due to the limitation in the cusparse
+            // algorithm. The other executors (omp and reference) do not use the
+            // transpose (trans_x and trans_b) and hence are passed in empty
+            // pointers.
+            bool do_transpose = false;
+            std::shared_ptr<Vector> trans_b;
+            std::shared_ptr<Vector> trans_x;
+            this->get_executor()->run(
+                lower_trs::make_should_perform_transpose(do_transpose));
+            if (do_transpose) {
+                trans_b =
+                    Vector::create(exec, gko::transpose(dense_b->get_size()));
+                trans_x =
+                    Vector::create(exec, gko::transpose(dense_x->get_size()));
+            } else {
+                trans_b = Vector::create(exec);
+                trans_x = Vector::create(exec);
+            }
+            exec->run(lower_trs::make_solve(
+                gko::lend(system_matrix_), gko::lend(this->solve_struct_),
+                gko::lend(trans_b), gko::lend(trans_x), dense_b, dense_x));
+        },
+        b, x);
 }
 
 
@@ -136,12 +141,14 @@ void LowerTrs<ValueType, IndexType>::apply_impl(const LinOp *alpha,
                                                 const LinOp *beta,
                                                 LinOp *x) const
 {
-    auto dense_x = as<matrix::Dense<ValueType>>(x);
-
-    auto x_clone = dense_x->clone();
-    this->apply(b, x_clone.get());
-    dense_x->scale(beta);
-    dense_x->add_scaled(alpha, gko::lend(x_clone));
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            auto x_clone = dense_x->clone();
+            this->apply(dense_b, x_clone.get());
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, x_clone.get());
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/core/solver/upper_trs.cpp b/core/solver/upper_trs.cpp
index 7b0fede44ea..bd1c5a9d896 100644
--- a/core/solver/upper_trs.cpp
+++ b/core/solver/upper_trs.cpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/polymorphic_object.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
@@ -101,32 +102,36 @@ void UpperTrs<ValueType, IndexType>::generate()
 template <typename ValueType, typename IndexType>
 void UpperTrs<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    using Vector = matrix::Dense<ValueType>;
-    const auto exec = this->get_executor();
-
-    auto dense_b = as<const Vector>(b);
-    auto dense_x = as<Vector>(x);
-
-    // This kernel checks if a transpose is needed for the multiple rhs case.
-    // Currently only the algorithm for CUDA version <=9.1 needs this
-    // transposition due to the limitation in the cusparse algorithm. The other
-    // executors (omp and reference) do not use the transpose (trans_x and
-    // trans_b) and hence are passed in empty pointers.
-    bool do_transpose = false;
-    std::shared_ptr<Vector> trans_b;
-    std::shared_ptr<Vector> trans_x;
-    this->get_executor()->run(
-        upper_trs::make_should_perform_transpose(do_transpose));
-    if (do_transpose) {
-        trans_b = Vector::create(exec, gko::transpose(dense_b->get_size()));
-        trans_x = Vector::create(exec, gko::transpose(dense_x->get_size()));
-    } else {
-        trans_b = Vector::create(exec);
-        trans_x = Vector::create(exec);
-    }
-    exec->run(upper_trs::make_solve(
-        gko::lend(system_matrix_), gko::lend(this->solve_struct_),
-        gko::lend(trans_b), gko::lend(trans_x), dense_b, dense_x));
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_b, auto dense_x) {
+            using Vector = matrix::Dense<ValueType>;
+            const auto exec = this->get_executor();
+
+            // This kernel checks if a transpose is needed for the multiple rhs
+            // case. Currently only the algorithm for CUDA version <=9.1 needs
+            // this transposition due to the limitation in the cusparse
+            // algorithm. The other executors (omp and reference) do not use the
+            // transpose (trans_x and trans_b) and hence are passed in empty
+            // pointers.
+            bool do_transpose = false;
+            std::shared_ptr<Vector> trans_b;
+            std::shared_ptr<Vector> trans_x;
+            this->get_executor()->run(
+                upper_trs::make_should_perform_transpose(do_transpose));
+            if (do_transpose) {
+                trans_b =
+                    Vector::create(exec, gko::transpose(dense_b->get_size()));
+                trans_x =
+                    Vector::create(exec, gko::transpose(dense_x->get_size()));
+            } else {
+                trans_b = Vector::create(exec);
+                trans_x = Vector::create(exec);
+            }
+            exec->run(upper_trs::make_solve(
+                gko::lend(system_matrix_), gko::lend(this->solve_struct_),
+                gko::lend(trans_b), gko::lend(trans_x), dense_b, dense_x));
+        },
+        b, x);
 }
 
 
@@ -136,12 +141,14 @@ void UpperTrs<ValueType, IndexType>::apply_impl(const LinOp *alpha,
                                                 const LinOp *beta,
                                                 LinOp *x) const
 {
-    auto dense_x = as<matrix::Dense<ValueType>>(x);
-
-    auto x_clone = dense_x->clone();
-    this->apply(b, x_clone.get());
-    dense_x->scale(beta);
-    dense_x->add_scaled(alpha, gko::lend(x_clone));
+    precision_dispatch_real_complex<ValueType>(
+        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+            auto x_clone = dense_x->clone();
+            this->apply(dense_b, x_clone.get());
+            dense_x->scale(dense_beta);
+            dense_x->add_scaled(dense_alpha, x_clone.get());
+        },
+        alpha, b, beta, x);
 }
 
 
diff --git a/include/ginkgo/core/base/precision_dispatch.hpp b/include/ginkgo/core/base/precision_dispatch.hpp
new file mode 100644
index 00000000000..5588d566b41
--- /dev/null
+++ b/include/ginkgo/core/base/precision_dispatch.hpp
@@ -0,0 +1,232 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_PUBLIC_CORE_BASE_PRECISION_DISPATCH_HPP_
+#define GKO_PUBLIC_CORE_BASE_PRECISION_DISPATCH_HPP_
+
+
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/temporary_conversion.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+namespace gko {
+
+
+/**
+ * Convert the given LinOp from matrix::Dense<...> to matrix::Dense<ValueType>.
+ * The conversion tries to convert the input LinOp to all Dense types with value
+ * type recursively reachable by next_precision<...> starting from the ValueType
+ * template parameter. This means that all real-to-real and complex-to-complex
+ * conversions for default precisions are being considered.
+ * If the input matrix is non-const, the contents of the modified converted
+ * object will be converted back to the input matrix when the returned object is
+ * destroyed. This may lead to a loss of precision!
+ *
+ * @param matrix  the input matrix which is supposed to be converted. It is
+ *                wrapped unchanged if it is already of type
+ *                matrix::Dense<ValueType>, otherwise it will be converted to
+ *                this type if possible.
+ *
+ * @returns  a detail::temporary_conversion pointing to the (potentially
+ *           converted) object.
+ *
+ * @throws NotSupported  if the input matrix cannot be converted to
+ *                       matrix::Dense<ValueType>
+ *
+ * @tparam ValueType  the value type into whose associated matrix::Dense type to
+ *                    convert the input LinOp.
+ */
+template <typename ValueType>
+detail::temporary_conversion<matrix::Dense<ValueType>>
+make_temporary_conversion(LinOp *matrix)
+{
+    auto result =
+        detail::temporary_conversion<matrix::Dense<ValueType>>::template create<
+            matrix::Dense<next_precision<ValueType>>>(matrix);
+    if (!result) {
+        GKO_NOT_SUPPORTED(matrix);
+    }
+    return result;
+}
+
+
+/** @copydoc make_temporary_conversion(LinOp*) */
+template <typename ValueType>
+detail::temporary_conversion<const matrix::Dense<ValueType>>
+make_temporary_conversion(const LinOp *matrix)
+{
+    auto result = detail::temporary_conversion<const matrix::Dense<ValueType>>::
+        template create<matrix::Dense<next_precision<ValueType>>>(matrix);
+    if (!result) {
+        GKO_NOT_SUPPORTED(matrix);
+    }
+    return result;
+}
+
+
+/**
+ * Calls the given function with each given argument LinOp temporarily
+ * converted into matrix::Dense<ValueType> as parameters.
+ *
+ * @param fn  the given function. It will be passed one (potentially const)
+ *            matrix::Dense<ValueType>* parameter per parameter in the parameter
+ *            pack `linops`.
+ * @param linops  the given arguments to be converted and passed on to fn.
+ *
+ * @tparam ValueType  the value type to use for the parameters of `fn`.
+ * @tparam Function  the function pointer, lambda or other functor type to call
+ *                   with the converted arguments.
+ * @tparam Args  the argument type list.
+ * */
+template <typename ValueType, typename Function, typename... Args>
+void precision_dispatch(Function fn, Args *... linops)
+{
+    fn(make_temporary_conversion<ValueType>(linops).get()...);
+}
+
+
+/**
+ * Calls the given function with the given LinOps temporarily converted to
+ * matrix::Dense<ValueType>* as parameters.
+ * If ValueType is real and both input vectors are complex, uses
+ * matrix::Dense::get_real_view() to convert them into real matrices after
+ * precision conversion.
+ *
+ * @see precision_dispatch()
+ */
+template <typename ValueType, typename Function>
+void precision_dispatch_real_complex(Function fn, const LinOp *in, LinOp *out)
+{
+    // do we need to convert complex Dense to real Dense?
+    // all real dense vectors are intra-convertible, thus by casting to
+    // ConvertibleTo<matrix::Dense<>>, we can check whether a LinOp is a real
+    // dense matrix:
+    auto complex_to_real =
+        !(is_complex<ValueType>() ||
+          dynamic_cast<const ConvertibleTo<matrix::Dense<>> *>(in));
+    if (complex_to_real) {
+        auto dense_in = make_temporary_conversion<to_complex<ValueType>>(in);
+        auto dense_out = make_temporary_conversion<to_complex<ValueType>>(out);
+        using Dense = matrix::Dense<ValueType>;
+        // These dynamic_casts are only needed to make the code compile
+        // If ValueType is complex, this branch will never be taken
+        // If ValueType is real, the cast is a no-op
+        fn(dynamic_cast<const Dense *>(dense_in->create_real_view().get()),
+           dynamic_cast<Dense *>(dense_out->create_real_view().get()));
+    } else {
+        precision_dispatch<ValueType>(fn, in, out);
+    }
+}
+
+
+/**
+ * Calls the given function with the given LinOps temporarily converted to
+ * matrix::Dense<ValueType>* as parameters.
+ * If ValueType is real and both `in` and `out` are complex, uses
+ * matrix::Dense::get_real_view() to convert them into real matrices after
+ * precision conversion.
+ *
+ * @see precision_dispatch()
+ */
+template <typename ValueType, typename Function>
+void precision_dispatch_real_complex(Function fn, const LinOp *alpha,
+                                     const LinOp *in, LinOp *out)
+{
+    // do we need to convert complex Dense to real Dense?
+    // all real dense vectors are intra-convertible, thus by casting to
+    // ConvertibleTo<matrix::Dense<>>, we can check whether a LinOp is a real
+    // dense matrix:
+    auto complex_to_real =
+        !(is_complex<ValueType>() ||
+          dynamic_cast<const ConvertibleTo<matrix::Dense<>> *>(in));
+    if (complex_to_real) {
+        auto dense_in = make_temporary_conversion<to_complex<ValueType>>(in);
+        auto dense_out = make_temporary_conversion<to_complex<ValueType>>(out);
+        auto dense_alpha = make_temporary_conversion<ValueType>(alpha);
+        using Dense = matrix::Dense<ValueType>;
+        // These dynamic_casts are only needed to make the code compile
+        // If ValueType is complex, this branch will never be taken
+        // If ValueType is real, the cast is a no-op
+        fn(dense_alpha.get(),
+           dynamic_cast<const Dense *>(dense_in->create_real_view().get()),
+           dynamic_cast<Dense *>(dense_out->create_real_view().get()));
+    } else {
+        precision_dispatch<ValueType>(fn, alpha, in, out);
+    }
+}
+
+
+/**
+ * Calls the given function with the given LinOps temporarily converted to
+ * matrix::Dense<ValueType>* as parameters.
+ * If ValueType is real and both `in` and `out` are complex, uses
+ * matrix::Dense::get_real_view() to convert them into real matrices after
+ * precision conversion.
+ *
+ * @see precision_dispatch()
+ */
+template <typename ValueType, typename Function>
+void precision_dispatch_real_complex(Function fn, const LinOp *alpha,
+                                     const LinOp *in, const LinOp *beta,
+                                     LinOp *out)
+{
+    // do we need to convert complex Dense to real Dense?
+    // all real dense vectors are intra-convertible, thus by casting to
+    // ConvertibleTo<matrix::Dense<>>, we can check whether a LinOp is a real
+    // dense matrix:
+    auto complex_to_real =
+        !(is_complex<ValueType>() ||
+          dynamic_cast<const ConvertibleTo<matrix::Dense<>> *>(in));
+    if (complex_to_real) {
+        auto dense_in = make_temporary_conversion<to_complex<ValueType>>(in);
+        auto dense_out = make_temporary_conversion<to_complex<ValueType>>(out);
+        auto dense_alpha = make_temporary_conversion<ValueType>(alpha);
+        auto dense_beta = make_temporary_conversion<ValueType>(beta);
+        using Dense = matrix::Dense<ValueType>;
+        // These dynamic_casts are only needed to make the code compile
+        // If ValueType is complex, this branch will never be taken
+        // If ValueType is real, the cast is a no-op
+        fn(dense_alpha.get(),
+           dynamic_cast<const Dense *>(dense_in->create_real_view().get()),
+           dense_beta.get(),
+           dynamic_cast<Dense *>(dense_out->create_real_view().get()));
+    } else {
+        precision_dispatch<ValueType>(fn, alpha, in, beta, out);
+    }
+}
+
+
+}  // namespace gko
+
+
+#endif  // GKO_PUBLIC_CORE_BASE_PRECISION_DISPATCH_HPP_
diff --git a/include/ginkgo/core/base/temporary_conversion.hpp b/include/ginkgo/core/base/temporary_conversion.hpp
new file mode 100644
index 00000000000..946d8378b2e
--- /dev/null
+++ b/include/ginkgo/core/base/temporary_conversion.hpp
@@ -0,0 +1,253 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_PUBLIC_CORE_BASE_TEMPORARY_CONVERSION_HPP_
+#define GKO_PUBLIC_CORE_BASE_TEMPORARY_CONVERSION_HPP_
+
+
+#include <memory>
+#include <tuple>
+#include <type_traits>
+
+
+#include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/utils.hpp>
+
+
+namespace gko {
+namespace detail {
+
+
+/**
+ * @internal
+ *
+ * A convert_back_deleter is a type of deleter that converts and copies the data
+ * to an internally referenced object before performing the deletion.
+ *
+ * The deleter will use the `convert_to` method to perform the conversion, and
+ * then delete the passed object using the `delete` keyword. This kind of
+ * deleter is useful when temporarily converting an object with the intent of
+ * converting it back once it goes out of scope.
+ *
+ * There is also a specialization for constant objects that does not perform the
+ * conversion, since a constant object couldn't have been changed.
+ *
+ * @tparam CopyType  the type of converted object being deleted
+ * @tparam OrigType  the type of converted object to which the data will be
+ *                   converted back
+ */
+template <typename CopyType, typename OrigType>
+class convert_back_deleter {
+public:
+    using pointer = CopyType *;
+    using original_pointer = OrigType *;
+
+    /**
+     * Creates a new deleter object.
+     *
+     * @param original  the origin object to which the data will be converted
+     *                  back before deletion
+     */
+    convert_back_deleter(original_pointer original) : original_{original} {}
+
+    /**
+     * Deletes the object.
+     *
+     * @param ptr  pointer to the object being deleted
+     */
+    void operator()(pointer ptr) const
+    {
+        ptr->convert_to(original_);
+        delete ptr;
+    }
+
+private:
+    original_pointer original_;
+};
+
+// specialization for constant objects, no need to convert back something that
+// cannot change
+template <typename CopyType, typename OrigType>
+class convert_back_deleter<const CopyType, const OrigType> {
+public:
+    using pointer = const CopyType *;
+    using original_pointer = const OrigType *;
+    convert_back_deleter(original_pointer) {}
+
+    void operator()(pointer ptr) const { delete ptr; }
+};
+
+
+/**
+ * @internal
+ *
+ * Helper type that attempts to statically find the dynamic type of a given
+ * LinOp from a list of ConversionCandidates and, on the first match, converts
+ * it to TargetType with an appropriate .
+ *
+ * @tparam ConversionCandidates  list of potential dynamic types of the input
+ *                               object to be checked.
+ */
+template <typename... ConversionCandidates>
+struct conversion_helper {
+    /** Dispatch convert_impl with the ConversionCandidates list */
+    template <typename TargetType, typename MaybeConstLinOp>
+    static std::unique_ptr<TargetType, std::function<void(TargetType *)>>
+    convert(MaybeConstLinOp *obj)
+    {
+        return convert_impl<TargetType, MaybeConstLinOp,
+                            ConversionCandidates...>(obj);
+    }
+
+    /**
+     * Attempts to cast obj from the first ConversionCandidate and convert it to
+     * TargetType with a matching convert_back_deleter. If the cast fails,
+     * recursively tries the remaining conversion candidates.
+     */
+    template <typename TargetType, typename MaybeConstLinOp,
+              typename FirstCandidate, typename... TrailingCandidates>
+    static std::unique_ptr<TargetType, std::function<void(TargetType *)>>
+    convert_impl(MaybeConstLinOp *obj)
+    {
+        // make candidate_type conditionally const based on whether obj is const
+        using candidate_type =
+            std::conditional_t<std::is_const<MaybeConstLinOp>::value,
+                               const FirstCandidate, FirstCandidate>;
+        candidate_type *cast_obj{};
+        if ((cast_obj = dynamic_cast<candidate_type *>(obj))) {
+            // if the cast is successful, obj is of dynamic type candidate_type
+            // so we can convert from this type to TargetType
+            auto converted = TargetType::create(obj->get_executor());
+            cast_obj->convert_to(converted.get());
+            // Make sure ConvertibleTo<TargetType> is available and symmetric
+            static_assert(
+                std::is_base_of<ConvertibleTo<std::remove_cv_t<TargetType>>,
+                                FirstCandidate>::value,
+                "ConvertibleTo not implemented");
+            static_assert(std::is_base_of<ConvertibleTo<FirstCandidate>,
+                                          TargetType>::value,
+                          "ConvertibleTo not symmetric");
+            return {converted.release(),
+                    convert_back_deleter<TargetType, candidate_type>{cast_obj}};
+        } else {
+            // else try the remaining candidates
+            return conversion_helper<TrailingCandidates...>::template convert<
+                TargetType>(obj);
+        }
+    }
+};
+
+template <>
+struct conversion_helper<> {
+    template <typename T, typename MaybeConstLinOp>
+    static std::unique_ptr<T, std::function<void(T *)>> convert(
+        MaybeConstLinOp *obj)
+    {
+        // return nullptr if no previous candidates matched
+        return {nullptr, null_deleter<T>{}};
+    }
+};
+
+
+/**
+ * A temporary_conversion is a special smart pointer-like object that is
+ * designed to hold an object temporarily converted to another format.
+ *
+ * After the temporary_conversion goes out of scope, the stored object will
+ * be converted back to its original format. This class is optimized to
+ * avoid copies if the object is already in the correct format, in which
+ * case it will just hold a reference to that object, without performing the
+ * conversion.
+ *
+ * @tparam T  the type of object held in the temporary_conversion
+ */
+template <typename T>
+class temporary_conversion {
+public:
+    using value_type = T;
+    using pointer = T *;
+    using lin_op_type =
+        std::conditional_t<std::is_const<T>::value, const LinOp, LinOp>;
+
+    /**
+     * Create a temporary conversion for a non-temporary LinOp.
+     *
+     * @tparam ConversionCandidates  list of potential dynamic types of ptr to
+     *                               try out for converting ptr to type T.
+     */
+    template <typename... ConversionCandidates>
+    static temporary_conversion create(lin_op_type *ptr)
+    {
+        T *cast_ptr{};
+        if ((cast_ptr = dynamic_cast<T *>(ptr))) {
+            return handle_type{cast_ptr, null_deleter<T>{}};
+        } else {
+            return conversion_helper<ConversionCandidates...>::template convert<
+                T>(ptr);
+        }
+    }
+
+    /**
+     * Returns the object held by temporary_conversion.
+     *
+     * @return the object held by temporary_conversion
+     */
+    T *get() const { return handle_.get(); }
+
+    /**
+     * Calls a method on the underlying object.
+     *
+     * @return the underlying object
+     */
+    T *operator->() const { return handle_.get(); }
+
+    /**
+     * Returns if the conversion was successful.
+     */
+    explicit operator bool() { return static_cast<bool>(handle_); }
+
+private:
+    // std::function deleter allows to decide the (type of) deleter at
+    // runtime
+    using handle_type = std::unique_ptr<T, std::function<void(T *)>>;
+
+    temporary_conversion(handle_type handle) : handle_{std::move(handle)} {}
+
+    handle_type handle_;
+};
+
+
+}  // namespace detail
+}  // namespace gko
+
+
+#endif  // GKO_PUBLIC_CORE_BASE_TEMPORARY_CONVERSION_HPP_
diff --git a/include/ginkgo/core/preconditioner/ic.hpp b/include/ginkgo/core/preconditioner/ic.hpp
index ead157415b0..dcc35a9b317 100644
--- a/include/ginkgo/core/preconditioner/ic.hpp
+++ b/include/ginkgo/core/preconditioner/ic.hpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/factorization/par_ic.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
@@ -190,20 +191,30 @@ class Ic : public EnableLinOp<Ic<LSolverType, IndexType>>, public Transposable {
 protected:
     void apply_impl(const LinOp *b, LinOp *x) const override
     {
-        set_cache_to(b);
-        l_solver_->apply(b, cache_.intermediate.get());
-        if (lh_solver_->apply_uses_initial_guess()) {
-            x->copy_from(cache_.intermediate.get());
-        }
-        lh_solver_->apply(cache_.intermediate.get(), x);
+        // take care of real-to-complex apply
+        precision_dispatch_real_complex<value_type>(
+            [&](auto dense_b, auto dense_x) {
+                this->set_cache_to(dense_b);
+                l_solver_->apply(dense_b, cache_.intermediate.get());
+                if (lh_solver_->apply_uses_initial_guess()) {
+                    dense_x->copy_from(cache_.intermediate.get());
+                }
+                lh_solver_->apply(cache_.intermediate.get(), dense_x);
+            },
+            b, x);
     }
 
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override
     {
-        set_cache_to(b);
-        l_solver_->apply(b, cache_.intermediate.get());
-        lh_solver_->apply(alpha, cache_.intermediate.get(), beta, x);
+        precision_dispatch_real_complex<value_type>(
+            [&](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+                this->set_cache_to(dense_b);
+                l_solver_->apply(dense_b, cache_.intermediate.get());
+                lh_solver_->apply(dense_alpha, cache_.intermediate.get(),
+                                  dense_beta, dense_x);
+            },
+            alpha, b, beta, x);
     }
 
     explicit Ic(std::shared_ptr<const Executor> exec)
diff --git a/include/ginkgo/core/preconditioner/ilu.hpp b/include/ginkgo/core/preconditioner/ilu.hpp
index 42272288a22..9eae5447af3 100644
--- a/include/ginkgo/core/preconditioner/ilu.hpp
+++ b/include/ginkgo/core/preconditioner/ilu.hpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/factorization/par_ilu.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
@@ -208,33 +209,44 @@ class Ilu : public EnableLinOp<
 protected:
     void apply_impl(const LinOp *b, LinOp *x) const override
     {
-        set_cache_to(b);
-        if (!ReverseApply) {
-            l_solver_->apply(b, cache_.intermediate.get());
-            if (u_solver_->apply_uses_initial_guess()) {
-                x->copy_from(cache_.intermediate.get());
-            }
-            u_solver_->apply(cache_.intermediate.get(), x);
-        } else {
-            u_solver_->apply(b, cache_.intermediate.get());
-            if (l_solver_->apply_uses_initial_guess()) {
-                x->copy_from(cache_.intermediate.get());
-            }
-            l_solver_->apply(cache_.intermediate.get(), x);
-        }
+        // take care of real-to-complex apply
+        precision_dispatch_real_complex<value_type>(
+            [&](auto dense_b, auto dense_x) {
+                this->set_cache_to(dense_b);
+                if (!ReverseApply) {
+                    l_solver_->apply(dense_b, cache_.intermediate.get());
+                    if (u_solver_->apply_uses_initial_guess()) {
+                        dense_x->copy_from(cache_.intermediate.get());
+                    }
+                    u_solver_->apply(cache_.intermediate.get(), dense_x);
+                } else {
+                    u_solver_->apply(dense_b, cache_.intermediate.get());
+                    if (l_solver_->apply_uses_initial_guess()) {
+                        dense_x->copy_from(cache_.intermediate.get());
+                    }
+                    l_solver_->apply(cache_.intermediate.get(), dense_x);
+                }
+            },
+            b, x);
     }
 
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override
     {
-        set_cache_to(b);
-        if (!ReverseApply) {
-            l_solver_->apply(b, cache_.intermediate.get());
-            u_solver_->apply(alpha, cache_.intermediate.get(), beta, x);
-        } else {
-            u_solver_->apply(b, cache_.intermediate.get());
-            l_solver_->apply(alpha, cache_.intermediate.get(), beta, x);
-        }
+        precision_dispatch_real_complex<value_type>(
+            [&](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+                this->set_cache_to(dense_b);
+                if (!ReverseApply) {
+                    l_solver_->apply(dense_b, cache_.intermediate.get());
+                    u_solver_->apply(dense_alpha, cache_.intermediate.get(),
+                                     dense_beta, dense_x);
+                } else {
+                    u_solver_->apply(dense_b, cache_.intermediate.get());
+                    l_solver_->apply(dense_alpha, cache_.intermediate.get(),
+                                     dense_beta, dense_x);
+                }
+            },
+            alpha, b, beta, x);
     }
 
     explicit Ilu(std::shared_ptr<const Executor> exec)
diff --git a/include/ginkgo/core/solver/bicg.hpp b/include/ginkgo/core/solver/bicg.hpp
index 85a6cbcf6c6..da57e30e732 100644
--- a/include/ginkgo/core/solver/bicg.hpp
+++ b/include/ginkgo/core/solver/bicg.hpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/log/logger.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/criterion.hpp>
@@ -155,6 +156,9 @@ class Bicg : public EnableLinOp<Bicg<ValueType>>,
 protected:
     void apply_impl(const LinOp *b, LinOp *x) const override;
 
+    void apply_dense_impl(const matrix::Dense<ValueType> *b,
+                          matrix::Dense<ValueType> *x) const;
+
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override;
 
diff --git a/include/ginkgo/core/solver/bicgstab.hpp b/include/ginkgo/core/solver/bicgstab.hpp
index a7027356a8d..d842a4fbd31 100644
--- a/include/ginkgo/core/solver/bicgstab.hpp
+++ b/include/ginkgo/core/solver/bicgstab.hpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/log/logger.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/criterion.hpp>
@@ -153,6 +154,9 @@ class Bicgstab : public EnableLinOp<Bicgstab<ValueType>>,
 protected:
     void apply_impl(const LinOp *b, LinOp *x) const override;
 
+    void apply_dense_impl(const matrix::Dense<ValueType> *b,
+                          matrix::Dense<ValueType> *x) const;
+
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override;
 
diff --git a/include/ginkgo/core/solver/cb_gmres.hpp b/include/ginkgo/core/solver/cb_gmres.hpp
index 08af9de1766..ebc85b7e323 100644
--- a/include/ginkgo/core/solver/cb_gmres.hpp
+++ b/include/ginkgo/core/solver/cb_gmres.hpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/log/logger.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/criterion.hpp>
@@ -191,6 +192,9 @@ class CbGmres : public EnableLinOp<CbGmres<ValueType>>,
 protected:
     void apply_impl(const LinOp *b, LinOp *x) const override;
 
+    void apply_dense_impl(const matrix::Dense<ValueType> *b,
+                          matrix::Dense<ValueType> *x) const;
+
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override;
 
diff --git a/include/ginkgo/core/solver/cg.hpp b/include/ginkgo/core/solver/cg.hpp
index b3c6469ed8c..d84df60906d 100644
--- a/include/ginkgo/core/solver/cg.hpp
+++ b/include/ginkgo/core/solver/cg.hpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/log/logger.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/criterion.hpp>
@@ -149,6 +150,9 @@ class Cg : public EnableLinOp<Cg<ValueType>>,
 protected:
     void apply_impl(const LinOp *b, LinOp *x) const override;
 
+    void apply_dense_impl(const matrix::Dense<ValueType> *b,
+                          matrix::Dense<ValueType> *x) const;
+
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override;
 
diff --git a/include/ginkgo/core/solver/cgs.hpp b/include/ginkgo/core/solver/cgs.hpp
index 781032f5081..14282bb6c8f 100644
--- a/include/ginkgo/core/solver/cgs.hpp
+++ b/include/ginkgo/core/solver/cgs.hpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/log/logger.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/criterion.hpp>
@@ -146,6 +147,9 @@ class Cgs : public EnableLinOp<Cgs<ValueType>>,
 protected:
     void apply_impl(const LinOp *b, LinOp *x) const override;
 
+    void apply_dense_impl(const matrix::Dense<ValueType> *b,
+                          matrix::Dense<ValueType> *x) const;
+
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override;
 
diff --git a/include/ginkgo/core/solver/fcg.hpp b/include/ginkgo/core/solver/fcg.hpp
index 570e4ac5033..55ded85f667 100644
--- a/include/ginkgo/core/solver/fcg.hpp
+++ b/include/ginkgo/core/solver/fcg.hpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/log/logger.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/criterion.hpp>
@@ -154,6 +155,9 @@ class Fcg : public EnableLinOp<Fcg<ValueType>>,
 protected:
     void apply_impl(const LinOp *b, LinOp *x) const override;
 
+    void apply_dense_impl(const matrix::Dense<ValueType> *b,
+                          matrix::Dense<ValueType> *x) const;
+
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override;
 
diff --git a/include/ginkgo/core/solver/gmres.hpp b/include/ginkgo/core/solver/gmres.hpp
index 555234a3da1..0f643ece69a 100644
--- a/include/ginkgo/core/solver/gmres.hpp
+++ b/include/ginkgo/core/solver/gmres.hpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/log/logger.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/criterion.hpp>
@@ -168,6 +169,9 @@ class Gmres : public EnableLinOp<Gmres<ValueType>>,
 protected:
     void apply_impl(const LinOp *b, LinOp *x) const override;
 
+    void apply_dense_impl(const matrix::Dense<ValueType> *b,
+                          matrix::Dense<ValueType> *x) const;
+
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override;
 
diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index 939ecfcbb2c..a24ea0db177 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -251,7 +251,8 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
                     LinOp *x) const override;
 
     template <typename SubspaceType>
-    void iterate(const LinOp *b, LinOp *x) const;
+    void iterate(const matrix::Dense<SubspaceType> *dense_b,
+                 matrix::Dense<SubspaceType> *dense_x) const;
 
     explicit Idr(std::shared_ptr<const Executor> exec)
         : EnableLinOp<Idr>(std::move(exec))
diff --git a/include/ginkgo/core/solver/ir.hpp b/include/ginkgo/core/solver/ir.hpp
index 9d572339e3a..16ca3e1d3fa 100644
--- a/include/ginkgo/core/solver/ir.hpp
+++ b/include/ginkgo/core/solver/ir.hpp
@@ -205,6 +205,9 @@ class Ir : public EnableLinOp<Ir<ValueType>>, public Transposable {
 protected:
     void apply_impl(const LinOp *b, LinOp *x) const override;
 
+    void apply_dense_impl(const matrix::Dense<ValueType> *b,
+                          matrix::Dense<ValueType> *x) const;
+
     void apply_impl(const LinOp *alpha, const LinOp *b, const LinOp *beta,
                     LinOp *x) const override;
 
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index a2b62ec8671..a757a5a6072 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -54,10 +54,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/name_demangling.hpp>
 #include <ginkgo/core/base/perturbation.hpp>
 #include <ginkgo/core/base/polymorphic_object.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/base/range.hpp>
 #include <ginkgo/core/base/range_accessors.hpp>
 #include <ginkgo/core/base/std_extensions.hpp>
 #include <ginkgo/core/base/temporary_clone.hpp>
+#include <ginkgo/core/base/temporary_conversion.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/base/utils_helper.hpp>

From 0b5f52e25c8e73c0bb9dbc36621fa4a87333ee57 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 30 Mar 2021 13:18:49 +0200
Subject: [PATCH 0588/1008] add mixed precision tests

---
 core/test/utils.hpp                           |   7 +
 reference/test/base/combination.cpp           | 139 +++++++++++
 reference/test/base/composition.cpp           | 130 +++++++++++
 reference/test/base/perturbation.cpp          | 136 +++++++++++
 reference/test/matrix/coo_kernels.cpp         | 217 ++++++++++++++++--
 reference/test/matrix/csr_kernels.cpp         | 104 ++++++++-
 reference/test/matrix/dense_kernels.cpp       | 186 ++++++++++++++-
 reference/test/matrix/diagonal_kernels.cpp    | 206 +++++++++++++++++
 reference/test/matrix/ell_kernels.cpp         |  98 +++++++-
 reference/test/matrix/hybrid_kernels.cpp      |  99 +++++++-
 reference/test/matrix/identity.cpp            | 109 +++++++++
 reference/test/matrix/sellp_kernels.cpp       |  97 +++++++-
 .../test/matrix/sparsity_csr_kernels.cpp      | 100 ++++++++
 reference/test/preconditioner/ic.cpp          | 133 +++++++++++
 reference/test/preconditioner/ilu.cpp         | 138 +++++++++++
 .../test/preconditioner/jacobi_kernels.cpp    | 134 +++++++++++
 reference/test/solver/bicg_kernels.cpp        | 128 ++++++++++-
 reference/test/solver/bicgstab_kernels.cpp    | 130 ++++++++++-
 reference/test/solver/cb_gmres_kernels.cpp    | 136 ++++++++++-
 reference/test/solver/cg_kernels.cpp          | 124 ++++++++++
 reference/test/solver/cgs_kernels.cpp         | 132 ++++++++++-
 reference/test/solver/fcg_kernels.cpp         | 126 +++++++++-
 reference/test/solver/gmres_kernels.cpp       | 128 +++++++++++
 reference/test/solver/idr_kernels.cpp         | 130 ++++++++++-
 reference/test/solver/ir_kernels.cpp          | 123 ++++++++++
 reference/test/solver/lower_trs_kernels.cpp   | 128 +++++++++++
 reference/test/solver/upper_trs_kernels.cpp   | 128 +++++++++++
 27 files changed, 3374 insertions(+), 72 deletions(-)

diff --git a/core/test/utils.hpp b/core/test/utils.hpp
index 843955d219e..fde94fcc652 100644
--- a/core/test/utils.hpp
+++ b/core/test/utils.hpp
@@ -116,6 +116,13 @@ template <typename Precision, typename OutputType = Precision>
 using r = typename gko::test::reduction_factor<Precision, OutputType>;
 
 
+template <typename Precision1, typename Precision2>
+constexpr double r_mixed()
+{
+    return std::max<double>(r<Precision1>::value, r<Precision2>::value);
+}
+
+
 template <typename T>
 using I = std::initializer_list<T>;
 
diff --git a/reference/test/base/combination.cpp b/reference/test/base/combination.cpp
index b6dba6f7386..5f15fcdd701 100644
--- a/reference/test/base/combination.cpp
+++ b/reference/test/base/combination.cpp
@@ -89,6 +89,71 @@ TYPED_TEST(Combination, AppliesToVector)
 }
 
 
+TYPED_TEST(Combination, AppliesToMixedVector)
+{
+    /*
+        cmb = [ 8 7 ]
+              [ 5 4 ]
+    */
+    using value_type = gko::next_precision<TypeParam>;
+    using Mtx = typename TestFixture::Mtx;
+    auto cmb = gko::Combination<TypeParam>::create(
+        this->coefficients[0], this->operators[0], this->coefficients[1],
+        this->operators[1]);
+    auto x = gko::initialize<Mtx>({1.0, 2.0}, this->exec);
+    auto res = clone(x);
+
+    cmb->apply(lend(x), lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res, l({22.0, 13.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Combination, AppliesToComplexVector)
+{
+    /*
+        cmb = [ 8 7 ]
+              [ 5 4 ]
+    */
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using T = typename Mtx::value_type;
+    auto cmb = gko::Combination<TypeParam>::create(
+        this->coefficients[0], this->operators[0], this->coefficients[1],
+        this->operators[1]);
+    auto x = gko::initialize<Mtx>({T{1.0, -2.0}, T{2.0, -4.0}}, this->exec);
+    auto res = clone(x);
+
+    cmb->apply(lend(x), lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res, l({T{22.0, -44.0}, T{13.0, -26.0}}),
+                        r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Combination, AppliesToMixedComplexVector)
+{
+    /*
+        cmb = [ 8 7 ]
+              [ 5 4 ]
+    */
+    using value_type = gko::to_complex<gko::next_precision<TypeParam>>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto cmb = gko::Combination<TypeParam>::create(
+        this->coefficients[0], this->operators[0], this->coefficients[1],
+        this->operators[1]);
+    auto x = gko::initialize<Mtx>(
+        {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec);
+    auto res = clone(x);
+
+    cmb->apply(lend(x), lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res,
+                        l({value_type{22.0, -44.0}, value_type{13.0, -26.0}}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
 TYPED_TEST(Combination, AppliesLinearCombinationToVector)
 {
     /*
@@ -110,4 +175,78 @@ TYPED_TEST(Combination, AppliesLinearCombinationToVector)
 }
 
 
+TYPED_TEST(Combination, AppliesLinearCombinationToMixedVector)
+{
+    /*
+        cmb = [ 8 7 ]
+              [ 5 4 ]
+    */
+    using value_type = gko::next_precision<TypeParam>;
+    using Mtx = typename TestFixture::Mtx;
+    auto cmb = gko::Combination<TypeParam>::create(
+        this->coefficients[0], this->operators[0], this->coefficients[1],
+        this->operators[1]);
+    auto alpha = gko::initialize<Mtx>({3.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({1.0, 2.0}, this->exec);
+    auto res = clone(x);
+
+    cmb->apply(lend(alpha), lend(x), lend(beta), lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res, l({65.0, 37.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Combination, AppliesLinearCombinationToComplexVector)
+{
+    /*
+        cmb = [ 8 7 ]
+              [ 5 4 ]
+    */
+    using Dense = typename TestFixture::Mtx;
+    using DenseComplex = gko::to_complex<Dense>;
+    using T = typename DenseComplex::value_type;
+    auto cmb = gko::Combination<TypeParam>::create(
+        this->coefficients[0], this->operators[0], this->coefficients[1],
+        this->operators[1]);
+    auto alpha = gko::initialize<Dense>({3.0}, this->exec);
+    auto beta = gko::initialize<Dense>({-1.0}, this->exec);
+    auto x =
+        gko::initialize<DenseComplex>({T{1.0, -2.0}, T{2.0, -4.0}}, this->exec);
+    auto res = clone(x);
+
+    cmb->apply(lend(alpha), lend(x), lend(beta), lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res, l({T{65.0, -130.0}, T{37.0, -74.0}}),
+                        r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Combination, AppliesLinearCombinationToMixedComplexVector)
+{
+    /*
+        cmb = [ 8 7 ]
+              [ 5 4 ]
+    */
+    using MixedDense = gko::matrix::Dense<gko::next_precision<TypeParam>>;
+    using MixedDenseComplex = gko::to_complex<MixedDense>;
+    using value_type = typename MixedDenseComplex::value_type;
+    auto cmb = gko::Combination<TypeParam>::create(
+        this->coefficients[0], this->operators[0], this->coefficients[1],
+        this->operators[1]);
+    auto alpha = gko::initialize<MixedDense>({3.0}, this->exec);
+    auto beta = gko::initialize<MixedDense>({-1.0}, this->exec);
+    auto x = gko::initialize<MixedDenseComplex>(
+        {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec);
+    auto res = clone(x);
+
+    cmb->apply(lend(alpha), lend(x), lend(beta), lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res,
+                        l({value_type{65.0, -130.0}, value_type{37.0, -74.0}}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
 }  // namespace
diff --git a/reference/test/base/composition.cpp b/reference/test/base/composition.cpp
index f48ff60ef4d..5c964f94d49 100644
--- a/reference/test/base/composition.cpp
+++ b/reference/test/base/composition.cpp
@@ -128,6 +128,67 @@ TYPED_TEST(Composition, AppliesSingleToVector)
 }
 
 
+TYPED_TEST(Composition, AppliesSingleToMixedVector)
+{
+    /*
+        cmp = [ -9 -2 ]
+              [ 27 26 ]
+    */
+    using Mtx = gko::matrix::Dense<gko::next_precision<TypeParam>>;
+    using value_type = typename Mtx::value_type;
+    auto cmp = gko::Composition<TypeParam>::create(this->product);
+    auto x = gko::initialize<Mtx>({1.0, 2.0}, this->exec);
+    auto res = clone(x);
+
+    cmp->apply(lend(x), lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res, l({-13.0, 79.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Composition, AppliesSingleToComplexVector)
+{
+    /*
+        cmp = [ -9 -2 ]
+              [ 27 26 ]
+    */
+    using value_type = gko::to_complex<TypeParam>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto cmp = gko::Composition<TypeParam>::create(this->product);
+    auto x = gko::initialize<Mtx>(
+        {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec);
+    auto res = clone(x);
+
+    cmp->apply(lend(x), lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res,
+                        l({value_type{-13.0, 26.0}, value_type{79.0, -158.0}}),
+                        r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Composition, AppliesSingleToMixedComplexVector)
+{
+    /*
+        cmp = [ -9 -2 ]
+              [ 27 26 ]
+    */
+    using value_type = gko::next_precision<gko::to_complex<TypeParam>>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto cmp = gko::Composition<TypeParam>::create(this->product);
+    auto x = gko::initialize<Mtx>(
+        {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec);
+    auto res = clone(x);
+
+    cmp->apply(lend(x), lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res,
+                        l({value_type{-13.0, 26.0}, value_type{79.0, -158.0}}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
 TYPED_TEST(Composition, AppliesSingleLinearCombinationToVector)
 {
     /*
@@ -147,6 +208,75 @@ TYPED_TEST(Composition, AppliesSingleLinearCombinationToVector)
 }
 
 
+TYPED_TEST(Composition, AppliesSingleLinearCombinationToMixedVector)
+{
+    /*
+        cmp = [ -9 -2 ]
+              [ 27 26 ]
+    */
+    using value_type = gko::next_precision<TypeParam>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto cmp = gko::Composition<TypeParam>::create(this->product);
+    auto alpha = gko::initialize<Mtx>({3.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({1.0, 2.0}, this->exec);
+    auto res = clone(x);
+
+    cmp->apply(lend(alpha), lend(x), lend(beta), lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res, l({-40.0, 235.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Composition, AppliesSingleLinearCombinationToComplexVector)
+{
+    /*
+        cmp = [ -9 -2 ]
+              [ 27 26 ]
+    */
+    using Dense = typename TestFixture::Mtx;
+    using DenseComplex = gko::to_complex<Dense>;
+    using value_type = typename DenseComplex::value_type;
+    auto cmp = gko::Composition<TypeParam>::create(this->product);
+    auto alpha = gko::initialize<Dense>({3.0}, this->exec);
+    auto beta = gko::initialize<Dense>({-1.0}, this->exec);
+    auto x = gko::initialize<DenseComplex>(
+        {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec);
+    auto res = clone(x);
+
+    cmp->apply(lend(alpha), lend(x), lend(beta), lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res,
+                        l({value_type{-40.0, 80.0}, value_type{235.0, -470.0}}),
+                        r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Composition, AppliesSingleLinearCombinationToMixedComplexVector)
+{
+    /*
+        cmp = [ -9 -2 ]
+              [ 27 26 ]
+    */
+    using MixedDense = gko::matrix::Dense<gko::next_precision<TypeParam>>;
+    using MixedDenseComplex = gko::to_complex<MixedDense>;
+    using value_type = typename MixedDenseComplex::value_type;
+    auto cmp = gko::Composition<TypeParam>::create(this->product);
+    auto alpha = gko::initialize<MixedDense>({3.0}, this->exec);
+    auto beta = gko::initialize<MixedDense>({-1.0}, this->exec);
+    auto x = gko::initialize<MixedDenseComplex>(
+        {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec);
+    auto res = clone(x);
+
+    cmp->apply(lend(alpha), lend(x), lend(beta), lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res,
+                        l({value_type{-40.0, 80.0}, value_type{235.0, -470.0}}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
 TYPED_TEST(Composition, AppliesToVector)
 {
     /*
diff --git a/reference/test/base/perturbation.cpp b/reference/test/base/perturbation.cpp
index 7f847e825c2..6d13b863a6a 100644
--- a/reference/test/base/perturbation.cpp
+++ b/reference/test/base/perturbation.cpp
@@ -87,6 +87,70 @@ TYPED_TEST(Perturbation, AppliesToVector)
 }
 
 
+TYPED_TEST(Perturbation, AppliesToMixedVector)
+{
+    /*
+        cmp = I + 2 * [ 2 ] * [ 3 2 ]
+                      [ 1 ]
+    */
+    using Mtx = gko::matrix::Dense<gko::next_precision<TypeParam>>;
+    using value_type = typename Mtx::value_type;
+    auto cmp = gko::Perturbation<TypeParam>::create(this->scalar, this->basis,
+                                                    this->projector);
+    auto x = gko::initialize<Mtx>({1.0, 2.0}, this->exec);
+    auto res = Mtx::create_with_config_of(gko::lend(x));
+
+    cmp->apply(gko::lend(x), gko::lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res, l({29.0, 16.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Perturbation, AppliesToComplexVector)
+{
+    /*
+        cmp = I + 2 * [ 2 ] * [ 3 2 ]
+                      [ 1 ]
+    */
+    using value_type = gko::to_complex<TypeParam>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto cmp = gko::Perturbation<TypeParam>::create(this->scalar, this->basis,
+                                                    this->projector);
+    auto x = gko::initialize<Mtx>(
+        {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec);
+    auto res = Mtx::create_with_config_of(gko::lend(x));
+
+    cmp->apply(gko::lend(x), gko::lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res,
+                        l({value_type{29.0, -58.0}, value_type{16.0, -32.0}}),
+                        r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Perturbation, AppliesToMixedComplexVector)
+{
+    /*
+        cmp = I + 2 * [ 2 ] * [ 3 2 ]
+                      [ 1 ]
+    */
+    using value_type = gko::to_complex<gko::next_precision<TypeParam>>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto cmp = gko::Perturbation<TypeParam>::create(this->scalar, this->basis,
+                                                    this->projector);
+    auto x = gko::initialize<Mtx>(
+        {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec);
+    auto res = Mtx::create_with_config_of(gko::lend(x));
+
+    cmp->apply(gko::lend(x), gko::lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res,
+                        l({value_type{29.0, -58.0}, value_type{16.0, -32.0}}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
 TYPED_TEST(Perturbation, AppliesLinearCombinationToVector)
 {
     /*
@@ -107,6 +171,78 @@ TYPED_TEST(Perturbation, AppliesLinearCombinationToVector)
 }
 
 
+TYPED_TEST(Perturbation, AppliesLinearCombinationToMixedVector)
+{
+    /*
+        cmp = I + 2 * [ 2 ] * [ 3 2 ]
+                      [ 1 ]
+    */
+    using value_type = gko::next_precision<TypeParam>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto cmp = gko::Perturbation<TypeParam>::create(this->scalar, this->basis,
+                                                    this->projector);
+    auto alpha = gko::initialize<Mtx>({3.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({1.0, 2.0}, this->exec);
+    auto res = gko::clone(x);
+
+    cmp->apply(gko::lend(alpha), gko::lend(x), gko::lend(beta), gko::lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res, l({86.0, 46.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Perturbation, AppliesLinearCombinationToComplexVector)
+{
+    /*
+        cmp = I + 2 * [ 2 ] * [ 3 2 ]
+                      [ 1 ]
+    */
+    using Dense = typename TestFixture::Mtx;
+    using DenseComplex = gko::to_complex<Dense>;
+    using value_type = typename DenseComplex::value_type;
+    auto cmp = gko::Perturbation<TypeParam>::create(this->scalar, this->basis,
+                                                    this->projector);
+    auto alpha = gko::initialize<Dense>({3.0}, this->exec);
+    auto beta = gko::initialize<Dense>({-1.0}, this->exec);
+    auto x = gko::initialize<DenseComplex>(
+        {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec);
+    auto res = gko::clone(x);
+
+    cmp->apply(gko::lend(alpha), gko::lend(x), gko::lend(beta), gko::lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res,
+                        l({value_type{86.0, -172.0}, value_type{46.0, -92.0}}),
+                        r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Perturbation, AppliesLinearCombinationToMixedComplexVector)
+{
+    /*
+        cmp = I + 2 * [ 2 ] * [ 3 2 ]
+                      [ 1 ]
+    */
+    using MixedDense = gko::matrix::Dense<gko::next_precision<TypeParam>>;
+    using MixedDenseComplex = gko::to_complex<MixedDense>;
+    using value_type = typename MixedDenseComplex::value_type;
+    auto cmp = gko::Perturbation<TypeParam>::create(this->scalar, this->basis,
+                                                    this->projector);
+    auto alpha = gko::initialize<MixedDense>({3.0}, this->exec);
+    auto beta = gko::initialize<MixedDense>({-1.0}, this->exec);
+    auto x = gko::initialize<MixedDenseComplex>(
+        {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec);
+    auto res = gko::clone(x);
+
+    cmp->apply(gko::lend(alpha), gko::lend(x), gko::lend(beta), gko::lend(res));
+
+    GKO_ASSERT_MTX_NEAR(res,
+                        l({value_type{86.0, -172.0}, value_type{46.0, -92.0}}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
 TYPED_TEST(Perturbation, ConstructionByBasisAppliesToVector)
 {
     /*
diff --git a/reference/test/matrix/coo_kernels.cpp b/reference/test/matrix/coo_kernels.cpp
index c728a7ccbda..1993db54843 100644
--- a/reference/test/matrix/coo_kernels.cpp
+++ b/reference/test/matrix/coo_kernels.cpp
@@ -64,6 +64,7 @@ class Coo : public ::testing::Test {
     using Csr = gko::matrix::Csr<value_type, index_type>;
     using Mtx = gko::matrix::Coo<value_type, index_type>;
     using Vec = gko::matrix::Dense<value_type>;
+    using MixedVec = gko::matrix::Dense<gko::next_precision<value_type>>;
 
     Coo() : exec(gko::ReferenceExecutor::create()), mtx(Mtx::create(exec))
     {
@@ -349,6 +350,18 @@ TYPED_TEST(Coo, AppliesToDenseVector)
 }
 
 
+TYPED_TEST(Coo, AppliesToMixedDenseVector)
+{
+    using MixedVec = typename TestFixture::MixedVec;
+    auto x = gko::initialize<MixedVec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = MixedVec::create(this->exec, gko::dim<2>{2, 1});
+
+    this->mtx->apply(x.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({13.0, 5.0}), 0.0);
+}
+
+
 TYPED_TEST(Coo, AppliesToDenseVectorUnsorted)
 {
     using Vec = typename TestFixture::Vec;
@@ -419,6 +432,20 @@ TYPED_TEST(Coo, AppliesLinearCombinationToDenseVector)
 }
 
 
+TYPED_TEST(Coo, AppliesLinearCombinationToMixedDenseVector)
+{
+    using MixedVec = typename TestFixture::MixedVec;
+    auto alpha = gko::initialize<MixedVec>({-1.0}, this->exec);
+    auto beta = gko::initialize<MixedVec>({2.0}, this->exec);
+    auto x = gko::initialize<MixedVec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = gko::initialize<MixedVec>({1.0, 2.0}, this->exec);
+
+    this->mtx->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({-11.0, -1.0}), 0.0);
+}
+
+
 TYPED_TEST(Coo, AppliesLinearCombinationToDenseMatrix)
 {
     using Vec = typename TestFixture::Vec;
@@ -487,6 +514,18 @@ TYPED_TEST(Coo, AppliesAddToDenseVector)
 }
 
 
+TYPED_TEST(Coo, AppliesAddToMixedDenseVector)
+{
+    using MixedVec = typename TestFixture::MixedVec;
+    auto x = gko::initialize<MixedVec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = gko::initialize<MixedVec>({2.0, 1.0}, this->exec);
+
+    this->mtx->apply2(x.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({15.0, 6.0}), 0.0);
+}
+
+
 TYPED_TEST(Coo, AppliesAddToDenseMatrix)
 {
     using Vec = typename TestFixture::Vec;
@@ -524,6 +563,19 @@ TYPED_TEST(Coo, AppliesLinearCombinationAddToDenseVector)
 }
 
 
+TYPED_TEST(Coo, AppliesLinearCombinationAddToMixedDenseVector)
+{
+    using MixedVec = typename TestFixture::MixedVec;
+    auto alpha = gko::initialize<MixedVec>({-1.0}, this->exec);
+    auto x = gko::initialize<MixedVec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = gko::initialize<MixedVec>({1.0, 2.0}, this->exec);
+
+    this->mtx->apply2(alpha.get(), x.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({-12.0, -3.0}), 0.0);
+}
+
+
 TYPED_TEST(Coo, AppliesLinearCombinationAddToDenseMatrix)
 {
     using Vec = typename TestFixture::Vec;
@@ -622,8 +674,7 @@ TYPED_TEST(Coo, AppliesToComplex)
 {
     using value_type = typename TestFixture::value_type;
     using complex_type = gko::to_complex<value_type>;
-    using Mtx = typename TestFixture::Mtx;
-    using Vec = typename gko::matrix::Dense<complex_type>;
+    using Vec = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
     // clang-format off
@@ -644,25 +695,50 @@ TYPED_TEST(Coo, AppliesToComplex)
 }
 
 
+TYPED_TEST(Coo, AppliesToMixedComplex)
+{
+    using mixed_value_type =
+        gko::next_precision<typename TestFixture::value_type>;
+    using mixed_complex_type = gko::to_complex<mixed_value_type>;
+    using Vec = gko::matrix::Dense<mixed_complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}},
+         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec);
+    auto x = Vec::create(exec, gko::dim<2>{2,2});
+    // clang-format on
+
+    this->mtx->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{mixed_complex_type{13.0, 14.0}, mixed_complex_type{19.0, 20.0}},
+           {mixed_complex_type{10.0, 10.0}, mixed_complex_type{15.0, 15.0}}}),
+        0.0);
+}
+
+
 TYPED_TEST(Coo, AdvancedAppliesToComplex)
 {
     using value_type = typename TestFixture::value_type;
     using complex_type = gko::to_complex<value_type>;
-    using Mtx = typename TestFixture::Mtx;
-    using Scal = typename gko::matrix::Dense<value_type>;
-    using Vec = typename gko::matrix::Dense<complex_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+    using DenseComplex = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
     // clang-format off
-    auto b = gko::initialize<Vec>(
+    auto b = gko::initialize<DenseComplex>(
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
          {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
          {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
-    auto x = gko::initialize<Vec>(
+    auto x = gko::initialize<DenseComplex>(
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
          {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
-    auto alpha = gko::initialize<Scal>({-1.0}, this->exec);
-    auto beta = gko::initialize<Scal>({2.0}, this->exec);
+    auto alpha = gko::initialize<Dense>({-1.0}, this->exec);
+    auto beta = gko::initialize<Dense>({2.0}, this->exec);
     // clang-format on
 
     this->mtx->apply(alpha.get(), b.get(), beta.get(), x.get());
@@ -675,13 +751,42 @@ TYPED_TEST(Coo, AdvancedAppliesToComplex)
 }
 
 
+TYPED_TEST(Coo, AdvancedAppliesToMixedComplex)
+{
+    using mixed_value_type =
+        gko::next_precision<typename TestFixture::value_type>;
+    using mixed_complex_type = gko::to_complex<mixed_value_type>;
+    using MixedDense = gko::matrix::Dense<mixed_value_type>;
+    using MixedDenseComplex = gko::matrix::Dense<mixed_complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<MixedDenseComplex>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}},
+         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<MixedDenseComplex>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}}, exec);
+    auto alpha = gko::initialize<MixedDense>({-1.0}, this->exec);
+    auto beta = gko::initialize<MixedDense>({2.0}, this->exec);
+    // clang-format on
+
+    this->mtx->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{mixed_complex_type{-11.0, -14.0}, mixed_complex_type{-15.0, -18.0}},
+           {mixed_complex_type{-6.0, -6.0}, mixed_complex_type{-9.0, -9.0}}}),
+        0.0);
+}
+
+
 TYPED_TEST(Coo, ApplyAddsToComplex)
 {
     using value_type = typename TestFixture::value_type;
     using complex_type = gko::to_complex<value_type>;
-    using Mtx = typename TestFixture::Mtx;
-    using Scal = typename gko::matrix::Dense<value_type>;
-    using Vec = typename gko::matrix::Dense<complex_type>;
+    using Vec = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
     // clang-format off
@@ -692,7 +797,63 @@ TYPED_TEST(Coo, ApplyAddsToComplex)
     auto x = gko::initialize<Vec>(
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
          {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
-    auto alpha = gko::initialize<Scal>({-1.0}, this->exec);
+    // clang-format on
+
+    this->mtx->apply2(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{14.0, 14.0}, complex_type{21.0, 21.0}},
+           {complex_type{12.0, 12.0}, complex_type{18.0, 18.0}}}),
+        0.0);
+}
+
+
+TYPED_TEST(Coo, ApplyAddsToMixedComplex)
+{
+    using mixed_value_type =
+        gko::next_precision<typename TestFixture::value_type>;
+    using mixed_complex_type = gko::to_complex<mixed_value_type>;
+    using MixedVec = gko::matrix::Dense<mixed_complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<MixedVec>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}},
+         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<MixedVec>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}}, exec);
+    // clang-format on
+
+    this->mtx->apply2(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{mixed_complex_type{14.0, 14.0}, mixed_complex_type{21.0, 21.0}},
+           {mixed_complex_type{12.0, 12.0}, mixed_complex_type{18.0, 18.0}}}),
+        0.0);
+}
+
+
+TYPED_TEST(Coo, ApplyAddsScaledToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+    using DenseComplex = gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<DenseComplex>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<DenseComplex>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
+    auto alpha = gko::initialize<Dense>({-1.0}, this->exec);
     // clang-format on
 
     this->mtx->apply2(alpha.get(), b.get(), x.get());
@@ -705,6 +866,36 @@ TYPED_TEST(Coo, ApplyAddsToComplex)
 }
 
 
+TYPED_TEST(Coo, ApplyAddsScaledToMixedComplex)
+{
+    using mixed_value_type =
+        gko::next_precision<typename TestFixture::value_type>;
+    using mixed_complex_type = gko::to_complex<mixed_value_type>;
+    using MixedDense = gko::matrix::Dense<mixed_value_type>;
+    using MixedDenseComplex = gko::matrix::Dense<mixed_complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<MixedDenseComplex>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}},
+         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<MixedDenseComplex>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}}, exec);
+    auto alpha = gko::initialize<MixedDense>({-1.0}, this->exec);
+    // clang-format on
+
+    this->mtx->apply2(alpha.get(), b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{mixed_complex_type{-12.0, -14.0}, mixed_complex_type{-17.0, -19.0}},
+           {mixed_complex_type{-8.0, -8.0}, mixed_complex_type{-12.0, -12.0}}}),
+        0.0);
+}
+
+
 template <typename ValueIndexType>
 class CooComplex : public ::testing::Test {
 protected:
diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp
index 186aa700598..162ec21654b 100644
--- a/reference/test/matrix/csr_kernels.cpp
+++ b/reference/test/matrix/csr_kernels.cpp
@@ -74,6 +74,7 @@ class Csr : public ::testing::Test {
     using Ell = gko::matrix::Ell<value_type, index_type>;
     using Hybrid = gko::matrix::Hybrid<value_type, index_type>;
     using Vec = gko::matrix::Dense<value_type>;
+    using MixedVec = gko::matrix::Dense<gko::next_precision<value_type>>;
 
     Csr()
         : exec(gko::ReferenceExecutor::create()),
@@ -366,6 +367,20 @@ TYPED_TEST(Csr, AppliesToDenseVector)
 }
 
 
+TYPED_TEST(Csr, AppliesToMixedDenseVector)
+{
+    using MixedVec = typename TestFixture::MixedVec;
+    using MixedT = typename MixedVec::value_type;
+    auto x = gko::initialize<MixedVec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = MixedVec::create(this->exec, gko::dim<2>{2, 1});
+
+    this->mtx->apply(x.get(), y.get());
+
+    EXPECT_EQ(y->at(0), MixedT{13.0});
+    EXPECT_EQ(y->at(1), MixedT{5.0});
+}
+
+
 TYPED_TEST(Csr, AppliesToDenseMatrix)
 {
     using Vec = typename TestFixture::Vec;
@@ -399,6 +414,22 @@ TYPED_TEST(Csr, AppliesLinearCombinationToDenseVector)
 }
 
 
+TYPED_TEST(Csr, AppliesLinearCombinationToMixedDenseVector)
+{
+    using MixedVec = typename TestFixture::MixedVec;
+    using MixedT = typename MixedVec::value_type;
+    auto alpha = gko::initialize<MixedVec>({-1.0}, this->exec);
+    auto beta = gko::initialize<MixedVec>({2.0}, this->exec);
+    auto x = gko::initialize<MixedVec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = gko::initialize<MixedVec>({1.0, 2.0}, this->exec);
+
+    this->mtx->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    EXPECT_EQ(y->at(0), MixedT{-11.0});
+    EXPECT_EQ(y->at(1), MixedT{-1.0});
+}
+
+
 TYPED_TEST(Csr, AppliesLinearCombinationToDenseMatrix)
 {
     using Vec = typename TestFixture::Vec;
@@ -1367,8 +1398,7 @@ TYPED_TEST(Csr, AppliesToComplex)
 {
     using value_type = typename TestFixture::value_type;
     using complex_type = gko::to_complex<value_type>;
-    using Mtx = typename TestFixture::Mtx;
-    using Vec = typename gko::matrix::Dense<complex_type>;
+    using Vec = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
     // clang-format off
@@ -1389,25 +1419,50 @@ TYPED_TEST(Csr, AppliesToComplex)
 }
 
 
+TYPED_TEST(Csr, AppliesToMixedComplex)
+{
+    using mixed_value_type =
+        gko::next_precision<typename TestFixture::value_type>;
+    using mixed_complex_type = gko::to_complex<mixed_value_type>;
+    using Vec = gko::matrix::Dense<mixed_complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}},
+         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec);
+    auto x = Vec::create(exec, gko::dim<2>{2,2});
+    // clang-format on
+
+    this->mtx->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{mixed_complex_type{13.0, 14.0}, mixed_complex_type{19.0, 20.0}},
+           {mixed_complex_type{10.0, 10.0}, mixed_complex_type{15.0, 15.0}}}),
+        0.0);
+}
+
+
 TYPED_TEST(Csr, AdvancedAppliesToComplex)
 {
     using value_type = typename TestFixture::value_type;
     using complex_type = gko::to_complex<value_type>;
-    using Mtx = typename TestFixture::Mtx;
-    using Scal = typename gko::matrix::Dense<value_type>;
-    using Vec = typename gko::matrix::Dense<complex_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+    using DenseComplex = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
     // clang-format off
-    auto b = gko::initialize<Vec>(
+    auto b = gko::initialize<DenseComplex>(
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
          {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
          {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
-    auto x = gko::initialize<Vec>(
+    auto x = gko::initialize<DenseComplex>(
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
          {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
-    auto alpha = gko::initialize<Scal>({-1.0}, this->exec);
-    auto beta = gko::initialize<Scal>({2.0}, this->exec);
+    auto alpha = gko::initialize<Dense>({-1.0}, this->exec);
+    auto beta = gko::initialize<Dense>({2.0}, this->exec);
     // clang-format on
 
     this->mtx->apply(alpha.get(), b.get(), beta.get(), x.get());
@@ -1420,6 +1475,37 @@ TYPED_TEST(Csr, AdvancedAppliesToComplex)
 }
 
 
+TYPED_TEST(Csr, AdvancedAppliesToMixedComplex)
+{
+    using mixed_value_type =
+        gko::next_precision<typename TestFixture::value_type>;
+    using mixed_complex_type = gko::to_complex<mixed_value_type>;
+    using MixedDense = gko::matrix::Dense<mixed_value_type>;
+    using MixedDenseComplex = gko::matrix::Dense<mixed_complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<MixedDenseComplex>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}},
+         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<MixedDenseComplex>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}}, exec);
+    auto alpha = gko::initialize<MixedDense>({-1.0}, this->exec);
+    auto beta = gko::initialize<MixedDense>({2.0}, this->exec);
+    // clang-format on
+
+    this->mtx->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{mixed_complex_type{-11.0, -14.0}, mixed_complex_type{-15.0, -18.0}},
+           {mixed_complex_type{-6.0, -6.0}, mixed_complex_type{-9.0, -9.0}}}),
+        0.0);
+}
+
+
 template <typename ValueIndexType>
 class CsrComplex : public ::testing::Test {
 protected:
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index ebe3e132045..8dcfece1f04 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -65,7 +65,9 @@ class Dense : public ::testing::Test {
 protected:
     using value_type = T;
     using Mtx = gko::matrix::Dense<value_type>;
+    using MixedMtx = gko::matrix::Dense<gko::next_precision<value_type>>;
     using ComplexMtx = gko::to_complex<Mtx>;
+    using MixedComplexMtx = gko::to_complex<MixedMtx>;
     using RealMtx = gko::remove_complex<Mtx>;
     Dense()
         : exec(gko::ReferenceExecutor::create()),
@@ -168,6 +170,26 @@ TYPED_TEST(Dense, AppliesToDense)
 }
 
 
+TYPED_TEST(Dense, AppliesToMixedDense)
+{
+    using MixedMtx = typename TestFixture::MixedMtx;
+    using MixedT = typename MixedMtx::value_type;
+    auto mmtx1 = MixedMtx::create(this->exec);
+    auto mmtx3 = MixedMtx::create(this->exec);
+    this->mtx1->convert_to(mmtx1.get());
+    this->mtx3->convert_to(mmtx3.get());
+
+    this->mtx2->apply(mmtx1.get(), mmtx3.get());
+
+    EXPECT_EQ(mmtx3->at(0, 0), MixedT{-0.5});
+    EXPECT_EQ(mmtx3->at(0, 1), MixedT{-0.5});
+    EXPECT_EQ(mmtx3->at(0, 2), MixedT{-0.5});
+    EXPECT_EQ(mmtx3->at(1, 0), MixedT{1.0});
+    EXPECT_EQ(mmtx3->at(1, 1), MixedT{1.0});
+    ASSERT_EQ(mmtx3->at(1, 2), MixedT{1.0});
+}
+
+
 TYPED_TEST(Dense, AppliesLinearCombinationToDense)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -187,6 +209,28 @@ TYPED_TEST(Dense, AppliesLinearCombinationToDense)
 }
 
 
+TYPED_TEST(Dense, AppliesLinearCombinationToMixedDense)
+{
+    using MixedMtx = typename TestFixture::MixedMtx;
+    using MixedT = typename MixedMtx::value_type;
+    auto mmtx1 = MixedMtx::create(this->exec);
+    auto mmtx3 = MixedMtx::create(this->exec);
+    this->mtx1->convert_to(mmtx1.get());
+    this->mtx3->convert_to(mmtx3.get());
+    auto alpha = gko::initialize<MixedMtx>({-1.0}, this->exec);
+    auto beta = gko::initialize<MixedMtx>({2.0}, this->exec);
+
+    this->mtx2->apply(alpha.get(), mmtx1.get(), beta.get(), mmtx3.get());
+
+    EXPECT_EQ(mmtx3->at(0, 0), MixedT{2.5});
+    EXPECT_EQ(mmtx3->at(0, 1), MixedT{4.5});
+    EXPECT_EQ(mmtx3->at(0, 2), MixedT{6.5});
+    EXPECT_EQ(mmtx3->at(1, 0), MixedT{0.0});
+    EXPECT_EQ(mmtx3->at(1, 1), MixedT{2.0});
+    ASSERT_EQ(mmtx3->at(1, 2), MixedT{4.0});
+}
+
+
 TYPED_TEST(Dense, ApplyFailsOnWrongInnerDimension)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -232,6 +276,22 @@ TYPED_TEST(Dense, ScalesData)
 }
 
 
+TYPED_TEST(Dense, ScalesDataMixed)
+{
+    using MixedMtx = typename TestFixture::MixedMtx;
+    using MixedT = typename MixedMtx::value_type;
+    using T = typename TestFixture::value_type;
+    auto alpha = gko::initialize<MixedMtx>({I<MixedT>{2.0, -2.0}}, this->exec);
+
+    this->mtx2->scale(alpha.get());
+
+    EXPECT_EQ(this->mtx2->at(0, 0), T{2.0});
+    EXPECT_EQ(this->mtx2->at(0, 1), T{2.0});
+    EXPECT_EQ(this->mtx2->at(1, 0), T{-4.0});
+    EXPECT_EQ(this->mtx2->at(1, 1), T{-4.0});
+}
+
+
 TYPED_TEST(Dense, ScalesDataWithScalar)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -281,6 +341,25 @@ TYPED_TEST(Dense, AddsScaled)
 }
 
 
+TYPED_TEST(Dense, AddsScaledMixed)
+{
+    using MixedMtx = typename TestFixture::MixedMtx;
+    using T = typename TestFixture::value_type;
+    auto mmtx3 = MixedMtx::create(this->exec);
+    this->mtx3->convert_to(mmtx3.get());
+    auto alpha = gko::initialize<MixedMtx>({{2.0, 1.0, -2.0}}, this->exec);
+
+    this->mtx1->add_scaled(alpha.get(), this->mtx3.get());
+
+    EXPECT_EQ(this->mtx1->at(0, 0), T{3.0});
+    EXPECT_EQ(this->mtx1->at(0, 1), T{4.0});
+    EXPECT_EQ(this->mtx1->at(0, 2), T{-3.0});
+    EXPECT_EQ(this->mtx1->at(1, 0), T{2.5});
+    EXPECT_EQ(this->mtx1->at(1, 1), T{4.0});
+    ASSERT_EQ(this->mtx1->at(1, 2), T{-1.5});
+}
+
+
 TYPED_TEST(Dense, AddsScaledWithScalar)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -338,6 +417,22 @@ TYPED_TEST(Dense, ComputesDot)
 }
 
 
+TYPED_TEST(Dense, ComputesDotMixed)
+{
+    using MixedMtx = typename TestFixture::MixedMtx;
+    using MixedT = typename MixedMtx::value_type;
+    auto mmtx3 = MixedMtx::create(this->exec);
+    this->mtx3->convert_to(mmtx3.get());
+    auto result = MixedMtx::create(this->exec, gko::dim<2>{1, 3});
+
+    this->mtx1->compute_dot(this->mtx3.get(), result.get());
+
+    EXPECT_EQ(result->at(0, 0), MixedT{1.75});
+    EXPECT_EQ(result->at(0, 1), MixedT{7.75});
+    ASSERT_EQ(result->at(0, 2), MixedT{17.75});
+}
+
+
 TYPED_TEST(Dense, ComputesNorm2)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -355,6 +450,24 @@ TYPED_TEST(Dense, ComputesNorm2)
 }
 
 
+TYPED_TEST(Dense, ComputesNorm2Mixed)
+{
+    using MixedMtx = typename TestFixture::MixedMtx;
+    using MixedT = typename MixedMtx::value_type;
+    using MixedT_nc = gko::remove_complex<MixedT>;
+    using MixedNormVector = gko::matrix::Dense<MixedT_nc>;
+    auto mtx(gko::initialize<MixedMtx>(
+        {I<MixedT>{1.0, 0.0}, I<MixedT>{2.0, 3.0}, I<MixedT>{2.0, 4.0}},
+        this->exec));
+    auto result = MixedNormVector::create(this->exec, gko::dim<2>{1, 2});
+
+    mtx->compute_norm2(result.get());
+
+    EXPECT_EQ(result->at(0, 0), MixedT_nc{3.0});
+    EXPECT_EQ(result->at(0, 1), MixedT_nc{5.0});
+}
+
+
 TYPED_TEST(Dense, ComputDotFailsOnWrongInputSize)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -2312,8 +2425,7 @@ TYPED_TEST(Dense, AppliesToComplex)
 {
     using value_type = typename TestFixture::value_type;
     using complex_type = gko::to_complex<value_type>;
-    using Mtx = typename TestFixture::Mtx;
-    using Vec = typename gko::matrix::Dense<complex_type>;
+    using Vec = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
     // clang-format off
@@ -2334,24 +2446,50 @@ TYPED_TEST(Dense, AppliesToComplex)
 }
 
 
+TYPED_TEST(Dense, AppliesToMixedComplex)
+{
+    using mixed_value_type =
+        gko::next_precision<typename TestFixture::value_type>;
+    using mixed_complex_type = gko::to_complex<mixed_value_type>;
+    using Vec = gko::matrix::Dense<mixed_complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}},
+         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec);
+    auto x = Vec::create(exec, gko::dim<2>{2,2});
+    // clang-format on
+
+    this->mtx1->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{mixed_complex_type{14.0, 16.0}, mixed_complex_type{20.0, 22.0}},
+           {mixed_complex_type{17.0, 19.0}, mixed_complex_type{24.5, 26.5}}}),
+        0.0);
+}
+
+
 TYPED_TEST(Dense, AdvancedAppliesToComplex)
 {
     using value_type = typename TestFixture::value_type;
     using complex_type = gko::to_complex<value_type>;
-    using Mtx = typename TestFixture::Mtx;
-    using Vec = typename gko::matrix::Dense<complex_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+    using DenseComplex = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
     // clang-format off
-    auto b = gko::initialize<Vec>(
+    auto b = gko::initialize<DenseComplex>(
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
          {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
          {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
-    auto x = gko::initialize<Vec>(
+    auto x = gko::initialize<DenseComplex>(
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
          {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
-    auto alpha = gko::initialize<Mtx>({-1.0}, this->exec);
-    auto beta = gko::initialize<Mtx>({2.0}, this->exec);
+    auto alpha = gko::initialize<Dense>({-1.0}, this->exec);
+    auto beta = gko::initialize<Dense>({2.0}, this->exec);
     // clang-format on
 
     this->mtx1->apply(alpha.get(), b.get(), beta.get(), x.get());
@@ -2364,6 +2502,38 @@ TYPED_TEST(Dense, AdvancedAppliesToComplex)
 }
 
 
+TYPED_TEST(Dense, AdvancedAppliesToMixedComplex)
+{
+    using mixed_value_type =
+        gko::next_precision<typename TestFixture::value_type>;
+    using mixed_complex_type = gko::to_complex<mixed_value_type>;
+    using MixedDense = gko::matrix::Dense<mixed_value_type>;
+    using MixedDenseComplex = gko::matrix::Dense<mixed_complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<MixedDenseComplex>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}},
+         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<MixedDenseComplex>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}}, exec);
+    auto alpha = gko::initialize<MixedDense>({-1.0}, this->exec);
+    auto beta = gko::initialize<MixedDense>({2.0}, this->exec);
+    // clang-format on
+
+    this->mtx1->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{mixed_complex_type{-12.0, -16.0}, mixed_complex_type{-16.0, -20.0}},
+           {mixed_complex_type{-13.0, -15.0},
+            mixed_complex_type{-18.5, -20.5}}}),
+        0.0);
+}
+
+
 TYPED_TEST(Dense, MakeComplex)
 {
     using T = typename TestFixture::value_type;
diff --git a/reference/test/matrix/diagonal_kernels.cpp b/reference/test/matrix/diagonal_kernels.cpp
index 01db0c6e800..e772f07ebc2 100644
--- a/reference/test/matrix/diagonal_kernels.cpp
+++ b/reference/test/matrix/diagonal_kernels.cpp
@@ -62,6 +62,8 @@ class Diagonal : public ::testing::Test {
     using Csr = gko::matrix::Csr<value_type>;
     using Diag = gko::matrix::Diagonal<value_type>;
     using Dense = gko::matrix::Dense<value_type>;
+    using MixedDense = gko::matrix::Dense<gko::next_precision<value_type>>;
+
     Diagonal()
         : exec(gko::ReferenceExecutor::create()),
           diag1(Diag::create(exec, 2)),
@@ -120,6 +122,27 @@ TYPED_TEST(Diagonal, AppliesToDense)
 }
 
 
+TYPED_TEST(Diagonal, AppliesToMixedDense)
+{
+    using value_type = typename TestFixture::value_type;
+    using MixedDense = typename TestFixture::MixedDense;
+    using mixed_value_type = typename MixedDense::value_type;
+    auto mdense1 = MixedDense::create(this->exec);
+    auto mdense2 = MixedDense::create(this->exec);
+    this->dense1->convert_to(mdense1.get());
+    this->dense2->convert_to(mdense2.get());
+
+    this->diag1->apply(mdense1.get(), mdense2.get());
+
+    EXPECT_EQ(mdense2->at(0, 0), mixed_value_type{2.0});
+    EXPECT_EQ(mdense2->at(0, 1), mixed_value_type{4.0});
+    EXPECT_EQ(mdense2->at(0, 2), mixed_value_type{6.0});
+    EXPECT_EQ(mdense2->at(1, 0), mixed_value_type{4.5});
+    EXPECT_EQ(mdense2->at(1, 1), mixed_value_type{7.5});
+    EXPECT_EQ(mdense2->at(1, 2), mixed_value_type{10.5});
+}
+
+
 TYPED_TEST(Diagonal, RightAppliesToDense)
 {
     using value_type = typename TestFixture::value_type;
@@ -134,6 +157,69 @@ TYPED_TEST(Diagonal, RightAppliesToDense)
 }
 
 
+TYPED_TEST(Diagonal, RightAppliesToMixedDense)
+{
+    using value_type = typename TestFixture::value_type;
+    using MixedDense = typename TestFixture::MixedDense;
+    using mixed_value_type = typename MixedDense::value_type;
+    auto mdense1 = MixedDense::create(this->exec);
+    auto mdense2 = MixedDense::create(this->exec);
+    this->dense1->convert_to(mdense1.get());
+    this->dense2->convert_to(mdense2.get());
+
+    this->diag2->rapply(mdense1.get(), mdense2.get());
+
+    EXPECT_EQ(mdense2->at(0, 0), mixed_value_type{2.0});
+    EXPECT_EQ(mdense2->at(0, 1), mixed_value_type{6.0});
+    EXPECT_EQ(mdense2->at(0, 2), mixed_value_type{12.0});
+    EXPECT_EQ(mdense2->at(1, 0), mixed_value_type{3.0});
+    EXPECT_EQ(mdense2->at(1, 1), mixed_value_type{7.5});
+    EXPECT_EQ(mdense2->at(1, 2), mixed_value_type{14.0});
+}
+
+
+TYPED_TEST(Diagonal, AppliesLinearCombinationToDense)
+{
+    using value_type = typename TestFixture::value_type;
+    using Dense = typename TestFixture::Dense;
+    auto alpha = gko::initialize<Dense>({-1.0}, this->exec);
+    auto beta = gko::initialize<Dense>({2.0}, this->exec);
+
+    this->diag1->apply(alpha.get(), this->dense1.get(), beta.get(),
+                       this->dense2.get());
+
+    EXPECT_EQ(this->dense2->at(0, 0), value_type{0.0});
+    EXPECT_EQ(this->dense2->at(0, 1), value_type{0.0});
+    EXPECT_EQ(this->dense2->at(0, 2), value_type{0.0});
+    EXPECT_EQ(this->dense2->at(1, 0), value_type{-1.5});
+    EXPECT_EQ(this->dense2->at(1, 1), value_type{-2.5});
+    EXPECT_EQ(this->dense2->at(1, 2), value_type{-3.5});
+}
+
+
+TYPED_TEST(Diagonal, AppliesLinearCombinationToMixedDense)
+{
+    using value_type = typename TestFixture::value_type;
+    using MixedDense = typename TestFixture::MixedDense;
+    using mixed_value_type = typename MixedDense::value_type;
+    auto mdense1 = MixedDense::create(this->exec);
+    auto mdense2 = MixedDense::create(this->exec);
+    auto alpha = gko::initialize<MixedDense>({-1.0}, this->exec);
+    auto beta = gko::initialize<MixedDense>({2.0}, this->exec);
+    this->dense1->convert_to(mdense1.get());
+    this->dense2->convert_to(mdense2.get());
+
+    this->diag1->apply(alpha.get(), mdense1.get(), beta.get(), mdense2.get());
+
+    EXPECT_EQ(mdense2->at(0, 0), mixed_value_type{0.0});
+    EXPECT_EQ(mdense2->at(0, 1), mixed_value_type{0.0});
+    EXPECT_EQ(mdense2->at(0, 2), mixed_value_type{0.0});
+    EXPECT_EQ(mdense2->at(1, 0), mixed_value_type{-1.5});
+    EXPECT_EQ(mdense2->at(1, 1), mixed_value_type{-2.5});
+    EXPECT_EQ(mdense2->at(1, 2), mixed_value_type{-3.5});
+}
+
+
 TYPED_TEST(Diagonal, ApplyToDenseFailsForWrongInnerDimensions)
 {
     using value_type = typename TestFixture::value_type;
@@ -381,6 +467,126 @@ TYPED_TEST(Diagonal, OutplaceAbsolute)
 }
 
 
+TYPED_TEST(Diagonal, AppliesToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Vec = gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+    auto dense1 =
+        gko::initialize<Vec>({{complex_type{1.0, 2.0}, complex_type{2.0, 4.0},
+                               complex_type{3.0, 6.0}},
+                              {complex_type{1.5, 3.0}, complex_type{2.5, 5.0},
+                               complex_type{3.5, 7.0}}},
+                             exec);
+    auto dense2 = Vec::create(exec, gko::dim<2>{2, 3});
+
+    this->diag1->apply(dense1.get(), dense2.get());
+
+    GKO_ASSERT_MTX_NEAR(dense2,
+                        l({{complex_type{2.0, 4.0}, complex_type{4.0, 8.0},
+                            complex_type{6.0, 12.0}},
+                           {complex_type{4.5, 9.0}, complex_type{7.5, 15.0},
+                            complex_type{10.5, 21.0}}}),
+                        0.0);
+}
+
+
+TYPED_TEST(Diagonal, AppliesToMixedComplex)
+{
+    using mixed_value_type =
+        gko::next_precision<typename TestFixture::value_type>;
+    using mixed_complex_type = gko::to_complex<mixed_value_type>;
+    using Vec = gko::matrix::Dense<mixed_complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+    auto mdense1 = gko::initialize<Vec>(
+        {{mixed_complex_type{1.0, 2.0}, mixed_complex_type{2.0, 4.0},
+          mixed_complex_type{3.0, 6.0}},
+         {mixed_complex_type{1.5, 3.0}, mixed_complex_type{2.5, 5.0},
+          mixed_complex_type{3.5, 7.0}}},
+        exec);
+    auto mdense2 = Vec::create(exec, gko::dim<2>{2, 3});
+
+    this->diag1->apply(mdense1.get(), mdense2.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        mdense2,
+        l({{mixed_complex_type{2.0, 4.0}, mixed_complex_type{4.0, 8.0},
+            mixed_complex_type{6.0, 12.0}},
+           {mixed_complex_type{4.5, 9.0}, mixed_complex_type{7.5, 15.0},
+            mixed_complex_type{10.5, 21.0}}}),
+        0.0);
+}
+
+
+TYPED_TEST(Diagonal, AppliesLinearCombinationToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Vec = gko::matrix::Dense<complex_type>;
+    using Scalar = gko::matrix::Dense<value_type>;
+    auto exec = gko::ReferenceExecutor::create();
+    auto dense1 =
+        gko::initialize<Vec>({{complex_type{1.0, 2.0}, complex_type{2.0, 4.0},
+                               complex_type{3.0, 6.0}},
+                              {complex_type{1.5, 3.0}, complex_type{2.5, 5.0},
+                               complex_type{3.5, 7.0}}},
+                             exec);
+    auto dense2 =
+        gko::initialize<Vec>({{complex_type{1.0, 2.0}, complex_type{2.0, 4.0},
+                               complex_type{3.0, 6.0}},
+                              {complex_type{1.5, 3.0}, complex_type{2.5, 5.0},
+                               complex_type{3.5, 7.0}}},
+                             exec);
+    auto alpha = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({2.0}, this->exec);
+
+    this->diag1->apply(alpha.get(), dense1.get(), beta.get(), dense2.get());
+
+    GKO_ASSERT_MTX_NEAR(dense2,
+                        l({{complex_type{0.0, 0.0}, complex_type{0.0, 0.0},
+                            complex_type{0.0, 0.0}},
+                           {complex_type{-1.5, -3.0}, complex_type{-2.5, -5.0},
+                            complex_type{-3.5, -7.0}}}),
+                        0.0);
+}
+
+
+TYPED_TEST(Diagonal, AppliesLinearCombinationToMixedComplex)
+{
+    using mixed_value_type =
+        gko::next_precision<typename TestFixture::value_type>;
+    using mixed_complex_type = gko::to_complex<mixed_value_type>;
+    using Vec = gko::matrix::Dense<mixed_complex_type>;
+    using Scalar = gko::matrix::Dense<mixed_value_type>;
+    auto exec = gko::ReferenceExecutor::create();
+    auto dense1 = gko::initialize<Vec>(
+        {{mixed_complex_type{1.0, 2.0}, mixed_complex_type{2.0, 4.0},
+          mixed_complex_type{3.0, 6.0}},
+         {mixed_complex_type{1.5, 3.0}, mixed_complex_type{2.5, 5.0},
+          mixed_complex_type{3.5, 7.0}}},
+        exec);
+    auto dense2 = gko::initialize<Vec>(
+        {{mixed_complex_type{1.0, 2.0}, mixed_complex_type{2.0, 4.0},
+          mixed_complex_type{3.0, 6.0}},
+         {mixed_complex_type{1.5, 3.0}, mixed_complex_type{2.5, 5.0},
+          mixed_complex_type{3.5, 7.0}}},
+        exec);
+    auto alpha = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({2.0}, this->exec);
+
+    this->diag1->apply(alpha.get(), dense1.get(), beta.get(), dense2.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        dense2,
+        l({{mixed_complex_type{0.0, 0.0}, mixed_complex_type{0.0, 0.0},
+            mixed_complex_type{0.0, 0.0}},
+           {mixed_complex_type{-1.5, -3.0}, mixed_complex_type{-2.5, -5.0},
+            mixed_complex_type{-3.5, -7.0}}}),
+        0.0);
+}
+
+
 template <typename ValueType>
 class DiagonalComplex : public ::testing::Test {
 protected:
diff --git a/reference/test/matrix/ell_kernels.cpp b/reference/test/matrix/ell_kernels.cpp
index 91b29752be9..77436ad1287 100644
--- a/reference/test/matrix/ell_kernels.cpp
+++ b/reference/test/matrix/ell_kernels.cpp
@@ -63,6 +63,7 @@ class Ell : public ::testing::Test {
     using Mtx = gko::matrix::Ell<value_type, index_type>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
     using Vec = gko::matrix::Dense<value_type>;
+    using MixedVec = gko::matrix::Dense<gko::next_precision<value_type>>;
 
     Ell()
         : exec(gko::ReferenceExecutor::create()),
@@ -119,6 +120,18 @@ TYPED_TEST(Ell, AppliesToDenseVector)
 }
 
 
+TYPED_TEST(Ell, AppliesToMixedDenseVector)
+{
+    using MixedVec = typename TestFixture::MixedVec;
+    auto x = gko::initialize<MixedVec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = MixedVec::create(this->exec, gko::dim<2>{2, 1});
+
+    this->mtx1->apply(x.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({13.0, 5.0}), 0.0);
+}
+
+
 TYPED_TEST(Ell, AppliesToDenseMatrix)
 {
     using Vec = typename TestFixture::Vec;
@@ -155,6 +168,20 @@ TYPED_TEST(Ell, AppliesLinearCombinationToDenseVector)
 }
 
 
+TYPED_TEST(Ell, AppliesLinearCombinationToMixedDenseVector)
+{
+    using MixedVec = typename TestFixture::MixedVec;
+    auto alpha = gko::initialize<MixedVec>({-1.0}, this->exec);
+    auto beta = gko::initialize<MixedVec>({2.0}, this->exec);
+    auto x = gko::initialize<MixedVec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = gko::initialize<MixedVec>({1.0, 2.0}, this->exec);
+
+    this->mtx1->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({-11.0, -1.0}), 0.0);
+}
+
+
 TYPED_TEST(Ell, AppliesLinearCombinationToDenseMatrix)
 {
     using Vec = typename TestFixture::Vec;
@@ -642,7 +669,7 @@ TYPED_TEST(Ell, AppliesToComplex)
     using value_type = typename TestFixture::value_type;
     using complex_type = gko::to_complex<value_type>;
     using Mtx = typename TestFixture::Mtx;
-    using Vec = typename gko::matrix::Dense<complex_type>;
+    using Vec = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
     // clang-format off
@@ -663,25 +690,51 @@ TYPED_TEST(Ell, AppliesToComplex)
 }
 
 
+TYPED_TEST(Ell, AppliesToMixedComplex)
+{
+    using mixed_value_type =
+        gko::next_precision<typename TestFixture::value_type>;
+    using mixed_complex_type = gko::to_complex<mixed_value_type>;
+    using Vec = gko::matrix::Dense<mixed_complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}},
+         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec);
+    auto x = Vec::create(exec, gko::dim<2>{2,2});
+    // clang-format on
+
+    this->mtx1->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{mixed_complex_type{13.0, 14.0}, mixed_complex_type{19.0, 20.0}},
+           {mixed_complex_type{10.0, 10.0}, mixed_complex_type{15.0, 15.0}}}),
+        0.0);
+}
+
+
 TYPED_TEST(Ell, AdvancedAppliesToComplex)
 {
     using value_type = typename TestFixture::value_type;
     using complex_type = gko::to_complex<value_type>;
     using Mtx = typename TestFixture::Mtx;
-    using Scal = typename gko::matrix::Dense<value_type>;
-    using Vec = typename gko::matrix::Dense<complex_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+    using DenseComplex = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
     // clang-format off
-    auto b = gko::initialize<Vec>(
+    auto b = gko::initialize<DenseComplex>(
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
          {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
          {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
-    auto x = gko::initialize<Vec>(
+    auto x = gko::initialize<DenseComplex>(
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
          {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
-    auto alpha = gko::initialize<Scal>({-1.0}, this->exec);
-    auto beta = gko::initialize<Scal>({2.0}, this->exec);
+    auto alpha = gko::initialize<Dense>({-1.0}, this->exec);
+    auto beta = gko::initialize<Dense>({2.0}, this->exec);
     // clang-format on
 
     this->mtx1->apply(alpha.get(), b.get(), beta.get(), x.get());
@@ -694,6 +747,37 @@ TYPED_TEST(Ell, AdvancedAppliesToComplex)
 }
 
 
+TYPED_TEST(Ell, AdvancedAppliesToMixedComplex)
+{
+    using mixed_value_type =
+        gko::next_precision<typename TestFixture::value_type>;
+    using mixed_complex_type = gko::to_complex<mixed_value_type>;
+    using MixedDense = gko::matrix::Dense<mixed_value_type>;
+    using MixedDenseComplex = gko::matrix::Dense<mixed_complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<MixedDenseComplex>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}},
+         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<MixedDenseComplex>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}}, exec);
+    auto alpha = gko::initialize<MixedDense>({-1.0}, this->exec);
+    auto beta = gko::initialize<MixedDense>({2.0}, this->exec);
+    // clang-format on
+
+    this->mtx1->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{mixed_complex_type{-11.0, -14.0}, mixed_complex_type{-15.0, -18.0}},
+           {mixed_complex_type{-6.0, -6.0}, mixed_complex_type{-9.0, -9.0}}}),
+        0.0);
+}
+
+
 template <typename ValueIndexType>
 class EllComplex : public ::testing::Test {
 protected:
diff --git a/reference/test/matrix/hybrid_kernels.cpp b/reference/test/matrix/hybrid_kernels.cpp
index 59064eeceaa..800ef70b768 100644
--- a/reference/test/matrix/hybrid_kernels.cpp
+++ b/reference/test/matrix/hybrid_kernels.cpp
@@ -64,6 +64,7 @@ class Hybrid : public ::testing::Test {
     using Mtx = gko::matrix::Hybrid<value_type, index_type>;
     using Vec = gko::matrix::Dense<value_type>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
+    using MixedVec = gko::matrix::Dense<gko::next_precision<value_type>>;
 
     Hybrid()
         : exec(gko::ReferenceExecutor::create()),
@@ -145,6 +146,18 @@ TYPED_TEST(Hybrid, AppliesToDenseVector)
 }
 
 
+TYPED_TEST(Hybrid, AppliesToMixedDenseVector)
+{
+    using MixedVec = typename TestFixture::MixedVec;
+    auto x = gko::initialize<MixedVec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = MixedVec::create(this->exec, gko::dim<2>{2, 1});
+
+    this->mtx1->apply(x.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({13.0, 5.0}), 0.0);
+}
+
+
 TYPED_TEST(Hybrid, AppliesToDenseMatrix)
 {
     using Vec = typename TestFixture::Vec;
@@ -181,6 +194,20 @@ TYPED_TEST(Hybrid, AppliesLinearCombinationToDenseVector)
 }
 
 
+TYPED_TEST(Hybrid, AppliesLinearCombinationToMixedDenseVector)
+{
+    using MixedVec = typename TestFixture::MixedVec;
+    auto alpha = gko::initialize<MixedVec>({-1.0}, this->exec);
+    auto beta = gko::initialize<MixedVec>({2.0}, this->exec);
+    auto x = gko::initialize<MixedVec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = gko::initialize<MixedVec>({1.0, 2.0}, this->exec);
+
+    this->mtx1->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({-11.0, -1.0}), 0.0);
+}
+
+
 TYPED_TEST(Hybrid, AppliesLinearCombinationToDenseMatrix)
 {
     using Vec = typename TestFixture::Vec;
@@ -691,7 +718,7 @@ TYPED_TEST(Hybrid, AppliesToComplex)
     using value_type = typename TestFixture::value_type;
     using complex_type = gko::to_complex<value_type>;
     using Mtx = typename TestFixture::Mtx;
-    using Vec = typename gko::matrix::Dense<complex_type>;
+    using Vec = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
     // clang-format off
@@ -712,25 +739,51 @@ TYPED_TEST(Hybrid, AppliesToComplex)
 }
 
 
+TYPED_TEST(Hybrid, AppliesToMixedComplex)
+{
+    using mixed_value_type =
+        gko::next_precision<typename TestFixture::value_type>;
+    using mixed_complex_type = gko::to_complex<mixed_value_type>;
+    using Vec = gko::matrix::Dense<mixed_complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<Vec>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}},
+         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec);
+    auto x = Vec::create(exec, gko::dim<2>{2,2});
+    // clang-format on
+
+    this->mtx1->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{mixed_complex_type{13.0, 14.0}, mixed_complex_type{19.0, 20.0}},
+           {mixed_complex_type{10.0, 10.0}, mixed_complex_type{15.0, 15.0}}}),
+        0.0);
+}
+
+
 TYPED_TEST(Hybrid, AdvancedAppliesToComplex)
 {
     using value_type = typename TestFixture::value_type;
     using complex_type = gko::to_complex<value_type>;
     using Mtx = typename TestFixture::Mtx;
-    using Scal = typename gko::matrix::Dense<value_type>;
-    using Vec = typename gko::matrix::Dense<complex_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+    using DenseComplex = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
     // clang-format off
-    auto b = gko::initialize<Vec>(
+    auto b = gko::initialize<DenseComplex>(
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
          {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
          {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
-    auto x = gko::initialize<Vec>(
+    auto x = gko::initialize<DenseComplex>(
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
          {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
-    auto alpha = gko::initialize<Scal>({-1.0}, this->exec);
-    auto beta = gko::initialize<Scal>({2.0}, this->exec);
+    auto alpha = gko::initialize<Dense>({-1.0}, this->exec);
+    auto beta = gko::initialize<Dense>({2.0}, this->exec);
     // clang-format on
 
     this->mtx1->apply(alpha.get(), b.get(), beta.get(), x.get());
@@ -743,6 +796,38 @@ TYPED_TEST(Hybrid, AdvancedAppliesToComplex)
 }
 
 
+TYPED_TEST(Hybrid, AdvancedAppliesToMixedComplex)
+{
+    using mixed_value_type =
+        gko::next_precision<typename TestFixture::value_type>;
+    using mixed_complex_type = gko::to_complex<mixed_value_type>;
+    using MixedDense = gko::matrix::Dense<mixed_value_type>;
+    using MixedDenseComplex = gko::matrix::Dense<mixed_complex_type>;
+
+    // clang-format off
+    auto b = gko::initialize<MixedDenseComplex>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}},
+         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}},
+         this->exec);
+    auto x = gko::initialize<MixedDenseComplex>(
+        {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}},
+         this->exec);
+    auto alpha = gko::initialize<MixedDense>({-1.0}, this->exec);
+    auto beta = gko::initialize<MixedDense>({2.0}, this->exec);
+    // clang-format on
+
+    this->mtx1->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{mixed_complex_type{-11.0, -14.0}, mixed_complex_type{-15.0, -18.0}},
+           {mixed_complex_type{-6.0, -6.0}, mixed_complex_type{-9.0, -9.0}}}),
+        0.0);
+}
+
+
 template <typename ValueIndexType>
 class HybridComplex : public ::testing::Test {
 protected:
diff --git a/reference/test/matrix/identity.cpp b/reference/test/matrix/identity.cpp
index 690c7b811ce..04e032d4b33 100644
--- a/reference/test/matrix/identity.cpp
+++ b/reference/test/matrix/identity.cpp
@@ -51,6 +51,9 @@ class Identity : public ::testing::Test {
     using value_type = T;
     using Id = gko::matrix::Identity<value_type>;
     using Vec = gko::matrix::Dense<value_type>;
+    using MixedVec = gko::matrix::Dense<gko::next_precision<value_type>>;
+    using ComplexVec = gko::to_complex<Vec>;
+    using MixedComplexVec = gko::to_complex<MixedVec>;
 
     Identity() : exec(gko::ReferenceExecutor::create()) {}
 
@@ -61,6 +64,34 @@ class Identity : public ::testing::Test {
 TYPED_TEST_SUITE(Identity, gko::test::ValueTypes);
 
 
+TYPED_TEST(Identity, AppliesToVector)
+{
+    using Id = typename TestFixture::Id;
+    using Vec = typename TestFixture::Vec;
+    auto identity = Id::create(this->exec, 3);
+    auto x = gko::initialize<Vec>({3.0, -1.0, 2.0}, this->exec);
+    auto b = gko::initialize<Vec>({2.0, 1.0, 5.0}, this->exec);
+
+    identity->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({2.0, 1.0, 5.0}), 0.0);
+}
+
+
+TYPED_TEST(Identity, AppliesToMixedVector)
+{
+    using Id = typename TestFixture::Id;
+    using MixedVec = typename TestFixture::MixedVec;
+    auto identity = Id::create(this->exec, 3);
+    auto x = gko::initialize<MixedVec>({3.0, -1.0, 2.0}, this->exec);
+    auto b = gko::initialize<MixedVec>({2.0, 1.0, 5.0}, this->exec);
+
+    identity->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({2.0, 1.0, 5.0}), 0.0);
+}
+
+
 TYPED_TEST(Identity, AppliesLinearCombinationToVector)
 {
     using Id = typename TestFixture::Id;
@@ -77,6 +108,22 @@ TYPED_TEST(Identity, AppliesLinearCombinationToVector)
 }
 
 
+TYPED_TEST(Identity, AppliesLinearCombinationToMixedVector)
+{
+    using Id = typename TestFixture::Id;
+    using MixedVec = typename TestFixture::MixedVec;
+    auto identity = Id::create(this->exec, 3);
+    auto alpha = gko::initialize<MixedVec>({2.0}, this->exec);
+    auto beta = gko::initialize<MixedVec>({1.0}, this->exec);
+    auto x = gko::initialize<MixedVec>({3.0, -1.0, 2.0}, this->exec);
+    auto b = gko::initialize<MixedVec>({2.0, 1.0, 5.0}, this->exec);
+
+    identity->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({7.0, 1.0, 12.0}), 0.0);
+}
+
+
 TYPED_TEST(Identity, AppliesLinearCombinationToMultipleVectors)
 {
     using Id = typename TestFixture::Id;
@@ -96,4 +143,66 @@ TYPED_TEST(Identity, AppliesLinearCombinationToMultipleVectors)
 }
 
 
+TYPED_TEST(Identity, AppliesToComplex)
+{
+    using Id = typename TestFixture::Id;
+    using ComplexVec = typename TestFixture::ComplexVec;
+    auto identity = Id::create(this->exec, 3);
+    auto x = gko::initialize<ComplexVec>({3.0, -1.0, 2.0}, this->exec);
+    auto b = gko::initialize<ComplexVec>({2.0, 1.0, 5.0}, this->exec);
+
+    identity->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({2.0, 1.0, 5.0}), 0.0);
+}
+
+
+TYPED_TEST(Identity, AppliesToMixedComplex)
+{
+    using Id = typename TestFixture::Id;
+    using MixedComplexVec = typename TestFixture::MixedComplexVec;
+    auto identity = Id::create(this->exec, 3);
+    auto x = gko::initialize<MixedComplexVec>({3.0, -1.0, 2.0}, this->exec);
+    auto b = gko::initialize<MixedComplexVec>({2.0, 1.0, 5.0}, this->exec);
+
+    identity->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({2.0, 1.0, 5.0}), 0.0);
+}
+
+
+TYPED_TEST(Identity, AppliesLinearCombinationToComplex)
+{
+    using Id = typename TestFixture::Id;
+    using Vec = typename TestFixture::Vec;
+    using ComplexVec = typename TestFixture::ComplexVec;
+    auto identity = Id::create(this->exec, 3);
+    auto alpha = gko::initialize<Vec>({2.0}, this->exec);
+    auto beta = gko::initialize<Vec>({1.0}, this->exec);
+    auto x = gko::initialize<ComplexVec>({3.0, -1.0, 2.0}, this->exec);
+    auto b = gko::initialize<ComplexVec>({2.0, 1.0, 5.0}, this->exec);
+
+    identity->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({7.0, 1.0, 12.0}), 0.0);
+}
+
+
+TYPED_TEST(Identity, AppliesLinearCombinationToMixedComplex)
+{
+    using Id = typename TestFixture::Id;
+    using MixedVec = typename TestFixture::MixedVec;
+    using MixedComplexVec = typename TestFixture::MixedComplexVec;
+    auto identity = Id::create(this->exec, 3);
+    auto alpha = gko::initialize<MixedVec>({2.0}, this->exec);
+    auto beta = gko::initialize<MixedVec>({1.0}, this->exec);
+    auto x = gko::initialize<MixedComplexVec>({3.0, -1.0, 2.0}, this->exec);
+    auto b = gko::initialize<MixedComplexVec>({2.0, 1.0, 5.0}, this->exec);
+
+    identity->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({7.0, 1.0, 12.0}), 0.0);
+}
+
+
 }  // namespace
diff --git a/reference/test/matrix/sellp_kernels.cpp b/reference/test/matrix/sellp_kernels.cpp
index f4ea24287ea..9ee2eebd38e 100644
--- a/reference/test/matrix/sellp_kernels.cpp
+++ b/reference/test/matrix/sellp_kernels.cpp
@@ -96,6 +96,18 @@ TYPED_TEST(Sellp, AppliesToDenseVector)
 }
 
 
+TYPED_TEST(Sellp, AppliesToMixedDenseVector)
+{
+    using Vec = typename TestFixture::Vec;
+    auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = Vec::create(this->exec, gko::dim<2>{2, 1});
+
+    this->mtx1->apply(x.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({13.0, 5.0}), 0.0);
+}
+
+
 TYPED_TEST(Sellp, AppliesToDenseMatrix)
 {
     using Vec = typename TestFixture::Vec;
@@ -132,6 +144,20 @@ TYPED_TEST(Sellp, AppliesLinearCombinationToDenseVector)
 }
 
 
+TYPED_TEST(Sellp, AppliesLinearCombinationToMixedDenseVector)
+{
+    using Vec = typename TestFixture::Vec;
+    auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
+    auto beta = gko::initialize<Vec>({2.0}, this->exec);
+    auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = gko::initialize<Vec>({1.0, 2.0}, this->exec);
+
+    this->mtx1->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({-11.0, -1.0}), 0.0);
+}
+
+
 TYPED_TEST(Sellp, AppliesLinearCombinationToDenseMatrix)
 {
     using Vec = typename TestFixture::Vec;
@@ -645,8 +671,7 @@ TYPED_TEST(Sellp, AppliesToComplex)
 {
     using value_type = typename TestFixture::value_type;
     using complex_type = gko::to_complex<value_type>;
-    using Mtx = typename TestFixture::Mtx;
-    using Vec = typename gko::matrix::Dense<complex_type>;
+    using Vec = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
     // clang-format off
@@ -667,13 +692,11 @@ TYPED_TEST(Sellp, AppliesToComplex)
 }
 
 
-TYPED_TEST(Sellp, AdvancedAppliesToComplex)
+TYPED_TEST(Sellp, AppliesToMixedComplex)
 {
     using value_type = typename TestFixture::value_type;
     using complex_type = gko::to_complex<value_type>;
-    using Mtx = typename TestFixture::Mtx;
-    using Scal = typename gko::matrix::Dense<value_type>;
-    using Vec = typename gko::matrix::Dense<complex_type>;
+    using Vec = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
     // clang-format off
@@ -681,11 +704,67 @@ TYPED_TEST(Sellp, AdvancedAppliesToComplex)
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
          {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
          {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
-    auto x = gko::initialize<Vec>(
+    auto x = Vec::create(exec, gko::dim<2>{2,2});
+    // clang-format on
+
+    this->mtx1->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{13.0, 14.0}, complex_type{19.0, 20.0}},
+           {complex_type{10.0, 10.0}, complex_type{15.0, 15.0}}}),
+        0.0);
+}
+
+
+TYPED_TEST(Sellp, AdvancedAppliesToComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+    using DenseComplex = gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<DenseComplex>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<DenseComplex>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
+    auto alpha = gko::initialize<Dense>({-1.0}, this->exec);
+    auto beta = gko::initialize<Dense>({2.0}, this->exec);
+    // clang-format on
+
+    this->mtx1->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{complex_type{-11.0, -14.0}, complex_type{-15.0, -18.0}},
+           {complex_type{-6.0, -6.0}, complex_type{-9.0, -9.0}}}),
+        0.0);
+}
+
+
+TYPED_TEST(Sellp, AdvancedAppliesToMixedComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using MixedDense = gko::matrix::Dense<value_type>;
+    using MixedDenseComplex = gko::matrix::Dense<complex_type>;
+    auto exec = gko::ReferenceExecutor::create();
+
+    // clang-format off
+    auto b = gko::initialize<MixedDenseComplex>(
+        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+    auto x = gko::initialize<MixedDenseComplex>(
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
          {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
-    auto alpha = gko::initialize<Scal>({-1.0}, this->exec);
-    auto beta = gko::initialize<Scal>({2.0}, this->exec);
+    auto alpha = gko::initialize<MixedDense>({-1.0}, this->exec);
+    auto beta = gko::initialize<MixedDense>({2.0}, this->exec);
     // clang-format on
 
     this->mtx1->apply(alpha.get(), b.get(), beta.get(), x.get());
diff --git a/reference/test/matrix/sparsity_csr_kernels.cpp b/reference/test/matrix/sparsity_csr_kernels.cpp
index ae5d5a532c7..95f7123e3cc 100644
--- a/reference/test/matrix/sparsity_csr_kernels.cpp
+++ b/reference/test/matrix/sparsity_csr_kernels.cpp
@@ -174,6 +174,20 @@ TYPED_TEST(SparsityCsr, AppliesToDenseVector)
 }
 
 
+TYPED_TEST(SparsityCsr, AppliesToMixedDenseVector)
+{
+    using Vec = typename TestFixture::Vec;
+    using T = typename TestFixture::value_type;
+    auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = Vec::create(this->exec, gko::dim<2>{2, 1});
+
+    this->mtx->apply(x.get(), y.get());
+
+    EXPECT_EQ(y->at(0), T{7.0});
+    EXPECT_EQ(y->at(1), T{1.0});
+}
+
+
 TYPED_TEST(SparsityCsr, AppliesToDenseMatrix)
 {
     using Vec = typename TestFixture::Vec;
@@ -207,6 +221,22 @@ TYPED_TEST(SparsityCsr, AppliesLinearCombinationToDenseVector)
 }
 
 
+TYPED_TEST(SparsityCsr, AppliesLinearCombinationToMixedDenseVector)
+{
+    using Vec = typename TestFixture::Vec;
+    using T = typename TestFixture::value_type;
+    auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
+    auto beta = gko::initialize<Vec>({2.0}, this->exec);
+    auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = gko::initialize<Vec>({1.0, 2.0}, this->exec);
+
+    this->mtx->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    EXPECT_EQ(y->at(0), T{-5.0});
+    EXPECT_EQ(y->at(1), T{3.0});
+}
+
+
 TYPED_TEST(SparsityCsr, AppliesLinearCombinationToDenseMatrix)
 {
     using Vec = typename TestFixture::Vec;
@@ -227,6 +257,76 @@ TYPED_TEST(SparsityCsr, AppliesLinearCombinationToDenseMatrix)
 }
 
 
+TYPED_TEST(SparsityCsr, AppliesToComplex)
+{
+    using Vec = gko::to_complex<typename TestFixture::Vec>;
+    using T = gko::to_complex<typename TestFixture::value_type>;
+    auto x = gko::initialize<Vec>({T{2.0, 4.0}, T{1.0, 2.0}, T{4.0, 8.0}},
+                                  this->exec);
+    auto y = Vec::create(this->exec, gko::dim<2>{2, 1});
+
+    this->mtx->apply(x.get(), y.get());
+
+    EXPECT_EQ(y->at(0), T(7.0, 14.0));
+    EXPECT_EQ(y->at(1), T(1.0, 2.0));
+}
+
+
+TYPED_TEST(SparsityCsr, AppliesToMixedComplex)
+{
+    using T =
+        gko::next_precision<gko::to_complex<typename TestFixture::value_type>>;
+    using Vec = gko::matrix::Dense<T>;
+    auto x = gko::initialize<Vec>({T{2.0, 4.0}, T{1.0, 2.0}, T{4.0, 8.0}},
+                                  this->exec);
+    auto y = Vec::create(this->exec, gko::dim<2>{2, 1});
+
+    this->mtx->apply(x.get(), y.get());
+
+    EXPECT_EQ(y->at(0), T(7.0, 14.0));
+    EXPECT_EQ(y->at(1), T(1.0, 2.0));
+}
+
+
+TYPED_TEST(SparsityCsr, AppliesLinearCombinationToComplex)
+{
+    using Vec = typename TestFixture::Vec;
+    using ComplexVec = gko::to_complex<Vec>;
+    using T = gko::to_complex<typename TestFixture::value_type>;
+    auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
+    auto beta = gko::initialize<Vec>({2.0}, this->exec);
+    auto x = gko::initialize<ComplexVec>(
+        {T{2.0, 4.0}, T{1.0, 2.0}, T{4.0, 8.0}}, this->exec);
+    auto y =
+        gko::initialize<ComplexVec>({T{1.0, 2.0}, T{2.0, 4.0}}, this->exec);
+
+    this->mtx->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    EXPECT_EQ(y->at(0), T(-5.0, -10.0));
+    EXPECT_EQ(y->at(1), T(3.0, 6.0));
+}
+
+
+TYPED_TEST(SparsityCsr, AppliesLinearCombinationToMixedComplex)
+{
+    using Vec = gko::matrix::Dense<
+        gko::next_precision<typename TestFixture::value_type>>;
+    using ComplexVec = gko::to_complex<Vec>;
+    using T = typename ComplexVec::value_type;
+    auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
+    auto beta = gko::initialize<Vec>({2.0}, this->exec);
+    auto x = gko::initialize<ComplexVec>(
+        {T{2.0, 4.0}, T{1.0, 2.0}, T{4.0, 8.0}}, this->exec);
+    auto y =
+        gko::initialize<ComplexVec>({T{1.0, 2.0}, T{2.0, 4.0}}, this->exec);
+
+    this->mtx->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    EXPECT_EQ(y->at(0), T(-5.0, -10.0));
+    EXPECT_EQ(y->at(1), T(3.0, 6.0));
+}
+
+
 TYPED_TEST(SparsityCsr, ApplyFailsOnWrongInnerDimension)
 {
     using Vec = typename TestFixture::Vec;
diff --git a/reference/test/preconditioner/ic.cpp b/reference/test/preconditioner/ic.cpp
index 24d49d0b5fb..2020427b9f9 100644
--- a/reference/test/preconditioner/ic.cpp
+++ b/reference/test/preconditioner/ic.cpp
@@ -276,6 +276,139 @@ TYPED_TEST(Ic, SolvesSingleRhs)
 }
 
 
+TYPED_TEST(Ic, SolvesSingleRhsMixed)
+{
+    using ic_prec_type = typename TestFixture::ic_prec_type;
+    using T = typename TestFixture::value_type;
+    using Vec = gko::matrix::Dense<gko::next_precision<T>>;
+    const auto b = gko::initialize<Vec>({1.0, 3.0, 6.0}, this->exec);
+    auto x = Vec::create(this->exec, gko::dim<2>{3, 1});
+    auto preconditioner =
+        ic_prec_type::build().on(this->exec)->generate(this->mtx);
+
+    preconditioner->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({3.0, -2.0, 4.0}), this->tol);
+}
+
+
+TYPED_TEST(Ic, SolvesSingleRhsComplex)
+{
+    using ic_prec_type = typename TestFixture::ic_prec_type;
+    using Vec = gko::to_complex<typename TestFixture::Vec>;
+    using T = typename Vec::value_type;
+    const auto b = gko::initialize<Vec>(
+        {T{1.0, 2.0}, T{3.0, 6.0}, T{6.0, 12.0}}, this->exec);
+    auto x = Vec::create(this->exec, gko::dim<2>{3, 1});
+    auto preconditioner =
+        ic_prec_type::build().on(this->exec)->generate(this->mtx);
+
+    preconditioner->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({T{3.0, 6.0}, T{-2.0, -4.0}, T{4.0, 8.0}}),
+                        this->tol);
+}
+
+
+TYPED_TEST(Ic, SolvesSingleRhsComplexMixed)
+{
+    using ic_prec_type = typename TestFixture::ic_prec_type;
+    using Vec = gko::matrix::Dense<
+        gko::next_precision<gko::to_complex<typename TestFixture::value_type>>>;
+    using T = typename Vec::value_type;
+    const auto b = gko::initialize<Vec>(
+        {T{1.0, 2.0}, T{3.0, 6.0}, T{6.0, 12.0}}, this->exec);
+    auto x = Vec::create(this->exec, gko::dim<2>{3, 1});
+    auto preconditioner =
+        ic_prec_type::build().on(this->exec)->generate(this->mtx);
+
+    preconditioner->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({T{3.0, 6.0}, T{-2.0, -4.0}, T{4.0, 8.0}}),
+                        this->tol);
+}
+
+
+TYPED_TEST(Ic, AdvancedSolvesSingleRhs)
+{
+    using ic_prec_type = typename TestFixture::ic_prec_type;
+    using Vec = typename TestFixture::Vec;
+    const auto b = gko::initialize<Vec>({1.0, 3.0, 6.0}, this->exec);
+    const auto alpha = gko::initialize<Vec>({2.0}, this->exec);
+    const auto beta = gko::initialize<Vec>({-1.0}, this->exec);
+    auto x = gko::initialize<Vec>({1.0, 2.0, 3.0}, this->exec);
+    auto preconditioner =
+        ic_prec_type::build().on(this->exec)->generate(this->mtx);
+
+    preconditioner->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({5.0, -6.0, 5.0}), this->tol);
+}
+
+
+TYPED_TEST(Ic, AdvancedSolvesSingleRhsMixed)
+{
+    using ic_prec_type = typename TestFixture::ic_prec_type;
+    using T = typename TestFixture::value_type;
+    using Vec = gko::matrix::Dense<gko::next_precision<T>>;
+    const auto b = gko::initialize<Vec>({1.0, 3.0, 6.0}, this->exec);
+    const auto alpha = gko::initialize<Vec>({2.0}, this->exec);
+    const auto beta = gko::initialize<Vec>({-1.0}, this->exec);
+    auto x = gko::initialize<Vec>({1.0, 2.0, 3.0}, this->exec);
+    auto preconditioner =
+        ic_prec_type::build().on(this->exec)->generate(this->mtx);
+
+    preconditioner->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({5.0, -6.0, 5.0}), this->tol);
+}
+
+
+TYPED_TEST(Ic, AdvancedSolvesSingleRhsComplex)
+{
+    using ic_prec_type = typename TestFixture::ic_prec_type;
+    using Dense = typename TestFixture::Vec;
+    using DenseComplex = gko::to_complex<Dense>;
+    using T = typename DenseComplex::value_type;
+    const auto b = gko::initialize<DenseComplex>(
+        {T{1.0, 2.0}, T{3.0, 6.0}, T{6.0, 12.0}}, this->exec);
+    const auto alpha = gko::initialize<Dense>({2.0}, this->exec);
+    const auto beta = gko::initialize<Dense>({-1.0}, this->exec);
+    auto x = gko::initialize<DenseComplex>(
+        {T{1.0, 2.0}, T{2.0, 4.0}, T{3.0, 6.0}}, this->exec);
+    auto preconditioner =
+        ic_prec_type::build().on(this->exec)->generate(this->mtx);
+
+    preconditioner->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({T{5.0, 10.0}, T{-6.0, -12.0}, T{5.0, 10.0}}),
+                        this->tol);
+}
+
+
+TYPED_TEST(Ic, AdvancedSolvesSingleRhsComplexMixed)
+{
+    using ic_prec_type = typename TestFixture::ic_prec_type;
+    using MixedDense = gko::matrix::Dense<
+        gko::next_precision<typename TestFixture::value_type>>;
+    using MixedDenseComplex = gko::to_complex<MixedDense>;
+    using T = typename MixedDenseComplex::value_type;
+    const auto b = gko::initialize<MixedDenseComplex>(
+        {T{1.0, 2.0}, T{3.0, 6.0}, T{6.0, 12.0}}, this->exec);
+    const auto alpha = gko::initialize<MixedDense>({2.0}, this->exec);
+    const auto beta = gko::initialize<MixedDense>({-1.0}, this->exec);
+    auto x = gko::initialize<MixedDenseComplex>(
+        {T{1.0, 2.0}, T{2.0, 4.0}, T{3.0, 6.0}}, this->exec);
+    auto preconditioner =
+        ic_prec_type::build().on(this->exec)->generate(this->mtx);
+
+    preconditioner->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({T{5.0, 10.0}, T{-6.0, -12.0}, T{5.0, 10.0}}),
+                        this->tol);
+}
+
+
 TYPED_TEST(Ic, SolvesMultipleRhs)
 {
     using ic_prec_type = typename TestFixture::ic_prec_type;
diff --git a/reference/test/preconditioner/ilu.cpp b/reference/test/preconditioner/ilu.cpp
index 3a8713bd686..4f0ae8b3562 100644
--- a/reference/test/preconditioner/ilu.cpp
+++ b/reference/test/preconditioner/ilu.cpp
@@ -357,6 +357,60 @@ TYPED_TEST(Ilu, SolvesSingleRhsWithMtx)
 }
 
 
+TYPED_TEST(Ilu, SolvesSingleRhsWithMixedMtx)
+{
+    using Mtx = gko::matrix::Dense<
+        gko::next_precision<typename TestFixture::value_type>>;
+    const auto b = gko::initialize<Mtx>({1.0, 3.0, 6.0}, this->exec);
+    auto x = Mtx::create(this->exec, gko::dim<2>{3, 1});
+    x->copy_from(b.get());
+
+    auto preconditioner = this->ilu_pre_factory->generate(this->mtx);
+    preconditioner->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x.get(), l({-0.125, 0.25, 1.0}),
+        (r_mixed<TypeParam, typename Mtx::value_type>()) * 1e+1);
+}
+
+
+TYPED_TEST(Ilu, SolvesSingleRhsWithComplexMtx)
+{
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using T = typename Mtx::value_type;
+    const auto b = gko::initialize<Mtx>(
+        {T{1.0, 2.0}, T{3.0, 6.0}, T{6.0, 12.0}}, this->exec);
+    auto x = Mtx::create(this->exec, gko::dim<2>{3, 1});
+    x->copy_from(b.get());
+
+    auto preconditioner = this->ilu_pre_factory->generate(this->mtx);
+    preconditioner->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x.get(),
+                        l({T{-0.125, -0.25}, T{0.25, 0.5}, T{1.0, 2.0}}),
+                        r<TypeParam>::value * 1e+1);
+}
+
+
+TYPED_TEST(Ilu, SolvesSingleRhsWithMixedComplexMtx)
+{
+    using Mtx = gko::matrix::Dense<
+        gko::to_complex<gko::next_precision<typename TestFixture::value_type>>>;
+    using T = typename Mtx::value_type;
+    const auto b = gko::initialize<Mtx>(
+        {T{1.0, 2.0}, T{3.0, 6.0}, T{6.0, 12.0}}, this->exec);
+    auto x = Mtx::create(this->exec, gko::dim<2>{3, 1});
+    x->copy_from(b.get());
+
+    auto preconditioner = this->ilu_pre_factory->generate(this->mtx);
+    preconditioner->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x.get(), l({T{-0.125, -0.25}, T{0.25, 0.5}, T{1.0, 2.0}}),
+        (r_mixed<TypeParam, typename Mtx::value_type>()) * 1e+1);
+}
+
+
 TYPED_TEST(Ilu, SolvesReverseSingleRhs)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -393,6 +447,90 @@ TYPED_TEST(Ilu, SolvesAdvancedSingleRhs)
 }
 
 
+TYPED_TEST(Ilu, SolvesAdvancedSingleRhsMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    const value_type alpha{2.0};
+    const auto alpha_linop = gko::initialize<Mtx>({alpha}, this->exec);
+    const value_type beta{-1};
+    const auto beta_linop = gko::initialize<Mtx>({beta}, this->exec);
+    const auto b = gko::initialize<Mtx>({-3.0, 6.0, 9.0}, this->exec);
+    auto x = gko::initialize<Mtx>({1.0, 2.0, 3.0}, this->exec);
+    auto preconditioner =
+        this->ilu_pre_factory->generate(this->l_u_composition);
+
+    preconditioner->apply(alpha_linop.get(), b.get(), beta_linop.get(),
+                          x.get());
+
+    GKO_ASSERT_MTX_NEAR(x.get(), l({-7.0, 2.0, -1.0}),
+                        (r_mixed<TypeParam, typename Mtx::value_type>()));
+}
+
+
+TYPED_TEST(Ilu, SolvesAdvancedSingleRhsComplex)
+{
+    using value_type = typename TestFixture::value_type;
+    using complex_type = gko::to_complex<value_type>;
+    using Dense = typename TestFixture::Mtx;
+    using DenseComplex = gko::to_complex<Dense>;
+    const value_type alpha{2.0};
+    const auto alpha_linop = gko::initialize<Dense>({alpha}, this->exec);
+    const value_type beta{-1};
+    const auto beta_linop = gko::initialize<Dense>({beta}, this->exec);
+    const auto b = gko::initialize<DenseComplex>(
+        {complex_type{-3.0, 6.0}, complex_type{6.0, -12.0},
+         complex_type{9.0, -18.0}},
+        this->exec);
+    auto x = gko::initialize<DenseComplex>(
+        {complex_type{1.0, -2.0}, complex_type{2.0, -4.0},
+         complex_type{3.0, -6.0}},
+        this->exec);
+    auto preconditioner =
+        this->ilu_pre_factory->generate(this->l_u_composition);
+
+    preconditioner->apply(alpha_linop.get(), b.get(), beta_linop.get(),
+                          x.get());
+
+    GKO_ASSERT_MTX_NEAR(x.get(),
+                        l({complex_type{-7.0, 14.0}, complex_type{2.0, -4.0},
+                           complex_type{-1.0, 2.0}}),
+                        r<TypeParam>::value);
+}
+
+
+TYPED_TEST(Ilu, SolvesAdvancedSingleRhsMixedComplex)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using complex_type = gko::to_complex<value_type>;
+    using MixedDense = gko::matrix::Dense<value_type>;
+    using MixedDenseComplex = gko::to_complex<MixedDense>;
+    const value_type alpha{2.0};
+    const auto alpha_linop = gko::initialize<MixedDense>({alpha}, this->exec);
+    const value_type beta{-1};
+    const auto beta_linop = gko::initialize<MixedDense>({beta}, this->exec);
+    const auto b = gko::initialize<MixedDenseComplex>(
+        {complex_type{-3.0, 6.0}, complex_type{6.0, -12.0},
+         complex_type{9.0, -18.0}},
+        this->exec);
+    auto x = gko::initialize<MixedDenseComplex>(
+        {complex_type{1.0, -2.0}, complex_type{2.0, -4.0},
+         complex_type{3.0, -6.0}},
+        this->exec);
+    auto preconditioner =
+        this->ilu_pre_factory->generate(this->l_u_composition);
+
+    preconditioner->apply(alpha_linop.get(), b.get(), beta_linop.get(),
+                          x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x.get(),
+        l({complex_type{-7.0, 14.0}, complex_type{2.0, -4.0},
+           complex_type{-1.0, 2.0}}),
+        (r_mixed<TypeParam, typename MixedDenseComplex::value_type>()));
+}
+
+
 TYPED_TEST(Ilu, SolvesAdvancedReverseSingleRhs)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/reference/test/preconditioner/jacobi_kernels.cpp b/reference/test/preconditioner/jacobi_kernels.cpp
index 778d0dd467d..f9cda1ad532 100644
--- a/reference/test/preconditioner/jacobi_kernels.cpp
+++ b/reference/test/preconditioner/jacobi_kernels.cpp
@@ -656,6 +656,71 @@ TYPED_TEST(Jacobi, AppliesToVector)
 }
 
 
+TYPED_TEST(Jacobi, AppliesToMixedVector)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Vec = gko::matrix::Dense<value_type>;
+    auto x = gko::initialize<Vec>({1.0, -1.0, 2.0, -2.0, 3.0}, this->exec);
+    auto b = gko::initialize<Vec>({4.0, -1.0, -2.0, 4.0, -1.0}, this->exec);
+    auto bj = this->bj_factory->generate(this->mtx);
+
+    bj->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x, l({1.0, 0.0, 0.0, 1.0, 0.0}),
+        (r_mixed<value_type, typename TestFixture::value_type>()));
+}
+
+
+TYPED_TEST(Jacobi, AppliesToComplexVector)
+{
+    using Vec = gko::to_complex<typename TestFixture::Vec>;
+    using value_type = typename Vec::value_type;
+    auto x = gko::initialize<Vec>(
+        {value_type{1.0, 2.0}, value_type{-1.0, -2.0}, value_type{2.0, 4.0},
+         value_type{-2.0, -4.0}, value_type{3.0, 6.0}},
+        this->exec);
+    auto b = gko::initialize<Vec>(
+        {value_type{4.0, 8.0}, value_type{-1.0, -2.0}, value_type{-2.0, -4.0},
+         value_type{4.0, 8.0}, value_type{-1.0, -2.0}},
+        this->exec);
+    auto bj = this->bj_factory->generate(this->mtx);
+
+    bj->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({value_type{1.0, 2.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0},
+           value_type{1.0, 2.0}, value_type{0.0, 0.0}}),
+        r<value_type>::value);
+}
+
+
+TYPED_TEST(Jacobi, AppliesToMixedComplexVector)
+{
+    using value_type =
+        gko::to_complex<gko::next_precision<typename TestFixture::value_type>>;
+    using Vec = gko::matrix::Dense<value_type>;
+    auto x = gko::initialize<Vec>(
+        {value_type{1.0, 2.0}, value_type{-1.0, -2.0}, value_type{2.0, 4.0},
+         value_type{-2.0, -4.0}, value_type{3.0, 6.0}},
+        this->exec);
+    auto b = gko::initialize<Vec>(
+        {value_type{4.0, 8.0}, value_type{-1.0, -2.0}, value_type{-2.0, -4.0},
+         value_type{4.0, 8.0}, value_type{-1.0, -2.0}},
+        this->exec);
+    auto bj = this->bj_factory->generate(this->mtx);
+
+    bj->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({value_type{1.0, 2.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0},
+           value_type{1.0, 2.0}, value_type{0.0, 0.0}}),
+        (r_mixed<value_type, typename TestFixture::value_type>()));
+}
+
+
 TYPED_TEST(Jacobi, AppliesToVectorWithAdaptivePrecision)
 {
     using Vec = typename TestFixture::Vec;
@@ -794,6 +859,75 @@ TYPED_TEST(Jacobi, AppliesLinearCombinationToVector)
 }
 
 
+TYPED_TEST(Jacobi, AppliesLinearCombinationToMixedVector)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Vec = gko::matrix::Dense<value_type>;
+    auto x = gko::initialize<Vec>({1.0, -1.0, 2.0, -2.0, 3.0}, this->exec);
+    auto b = gko::initialize<Vec>({4.0, -1.0, -2.0, 4.0, -1.0}, this->exec);
+    auto alpha = gko::initialize<Vec>({2.0}, this->exec);
+    auto beta = gko::initialize<Vec>({-1.0}, this->exec);
+    auto bj = this->bj_factory->generate(this->mtx);
+
+    bj->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x, l({1.0, 1.0, -2.0, 4.0, -3.0}),
+        (r_mixed<value_type, typename TestFixture::value_type>()));
+}
+
+
+TYPED_TEST(Jacobi, AppliesLinearCombinationToComplexVector)
+{
+    using Dense = typename TestFixture::Vec;
+    using DenseComplex = gko::to_complex<Dense>;
+    using value_type = typename TestFixture::value_type;
+    using T = gko::to_complex<value_type>;
+    auto x = gko::initialize<DenseComplex>(
+        {T{1.0, 2.0}, T{-1.0, -2.0}, T{2.0, 4.0}, T{-2.0, -4.0}, T{3.0, 6.0}},
+        this->exec);
+    auto b = gko::initialize<DenseComplex>(
+        {T{4.0, 8.0}, T{-1.0, -2.0}, T{-2.0, -4.0}, T{4.0, 8.0}, T{-1.0, -2.0}},
+        this->exec);
+    auto alpha = gko::initialize<Dense>({2.0}, this->exec);
+    auto beta = gko::initialize<Dense>({-1.0}, this->exec);
+    auto bj = this->bj_factory->generate(this->mtx);
+
+    bj->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({T{1.0, 2.0}, T{1.0, 2.0}, T{-2.0, -4.0}, T{4.0, 8.0},
+                           T{-3.0, -6.0}}),
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(Jacobi, AppliesLinearCombinationToMixedComplexVector)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using MixedDense = gko::matrix::Dense<value_type>;
+    using MixedDenseComplex = gko::to_complex<MixedDense>;
+    using T = gko::to_complex<value_type>;
+    auto x = gko::initialize<MixedDenseComplex>(
+        {T{1.0, 2.0}, T{-1.0, -2.0}, T{2.0, 4.0}, T{-2.0, -4.0}, T{3.0, 6.0}},
+        this->exec);
+    auto b = gko::initialize<MixedDenseComplex>(
+        {T{4.0, 8.0}, T{-1.0, -2.0}, T{-2.0, -4.0}, T{4.0, 8.0}, T{-1.0, -2.0}},
+        this->exec);
+    auto alpha = gko::initialize<MixedDense>({2.0}, this->exec);
+    auto beta = gko::initialize<MixedDense>({-1.0}, this->exec);
+    auto bj = this->bj_factory->generate(this->mtx);
+
+    bj->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({T{1.0, 2.0}, T{1.0, 2.0}, T{-2.0, -4.0}, T{4.0, 8.0},
+           T{-3.0, -6.0}}),
+        (r_mixed<value_type, typename TestFixture::value_type>()));
+}
+
+
 TYPED_TEST(Jacobi, AppliesLinearCombinationToVectorWithAdaptivePrecision)
 {
     using Vec = typename TestFixture::Vec;
diff --git a/reference/test/solver/bicg_kernels.cpp b/reference/test/solver/bicg_kernels.cpp
index 3d74bd8a107..bee4ce18ece 100644
--- a/reference/test/solver/bicg_kernels.cpp
+++ b/reference/test/solver/bicg_kernels.cpp
@@ -131,6 +131,64 @@ TYPED_TEST(Bicg, SolvesStencilSystem)
 }
 
 
+TYPED_TEST(Bicg, SolvesStencilSystemMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->bicg_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Bicg, SolvesStencilSystemComplex)
+{
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->bicg_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{3.0, -6.0},
+                           value_type{2.0, -4.0}}),
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(Bicg, SolvesStencilSystemMixedComplex)
+{
+    using value_type =
+        gko::to_complex<gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->bicg_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{3.0, -6.0},
+                           value_type{2.0, -4.0}}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
 TYPED_TEST(Bicg, SolvesMultipleStencilSystems)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -165,6 +223,72 @@ TYPED_TEST(Bicg, SolvesStencilSystemUsingAdvancedApply)
 }
 
 
+TYPED_TEST(Bicg, SolvesStencilSystemUsingAdvancedApplyMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->bicg_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Bicg, SolvesStencilSystemUsingAdvancedApplyComplex)
+{
+    using Scalar = typename TestFixture::Mtx;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->bicg_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.5, -3.0}, value_type{5.0, -10.0},
+                           value_type{2.0, -4.0}}),
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(Bicg, SolvesStencilSystemUsingAdvancedApplyMixedComplex)
+{
+    using Scalar = gko::matrix::Dense<
+        gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->bicg_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.5, -3.0}, value_type{5.0, -10.0},
+                           value_type{2.0, -4.0}}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
 TYPED_TEST(Bicg, SolvesMultipleStencilSystemsUsingAdvancedApply)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -326,8 +450,8 @@ TYPED_TEST(Bicg, SolvesMultipleDenseSystemForDivergenceCheck)
     ASSERT_LE(normC1 / normB1, normS1 / normB1 + r<value_type>::value);
     ASSERT_LE(normC2 / normB2, normS2 / normB2 + r<value_type>::value);
 
-    // Not sure if this is necessary, the assertions above should cover what is
-    // needed.
+    // Not sure if this is necessary, the assertions above should cover what
+    // is needed.
     GKO_ASSERT_MTX_NEAR(xc, mergedRes, r<value_type>::value);
 }
 
diff --git a/reference/test/solver/bicgstab_kernels.cpp b/reference/test/solver/bicgstab_kernels.cpp
index 0a53fd7c88b..d88a527fd47 100644
--- a/reference/test/solver/bicgstab_kernels.cpp
+++ b/reference/test/solver/bicgstab_kernels.cpp
@@ -112,14 +112,71 @@ TYPED_TEST(Bicgstab, SolvesDenseSystem)
 {
     using Mtx = typename TestFixture::Mtx;
     using value_type = typename TestFixture::value_type;
-    auto half_tol = std::sqrt(r<value_type>::value);
     auto solver = this->bicgstab_factory->generate(this->mtx);
     auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
     auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), half_tol);
+    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), r<value_type>::value);
+}
+
+
+TYPED_TEST(Bicgstab, SolvesDenseSystemMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->bicgstab_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Bicgstab, SolvesDenseSystemComplex)
+{
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->bicgstab_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{-4.0, 8.0}, value_type{-1.0, 2.0},
+                           value_type{4.0, -8.0}}),
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(Bicgstab, SolvesDenseSystemMixedComplex)
+{
+    using value_type =
+        gko::to_complex<gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->bicgstab_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{-4.0, 8.0}, value_type{-1.0, 2.0},
+                           value_type{4.0, -8.0}}),
+                        (r_mixed<value_type, TypeParam>()));
 }
 
 
@@ -165,7 +222,6 @@ TYPED_TEST(Bicgstab, SolvesDenseSystemUsingAdvancedApply)
 {
     using Mtx = typename TestFixture::Mtx;
     using value_type = typename TestFixture::value_type;
-    auto half_tol = std::sqrt(r<value_type>::value);
     auto solver = this->bicgstab_factory->generate(this->mtx);
     auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
     auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
@@ -174,8 +230,73 @@ TYPED_TEST(Bicgstab, SolvesDenseSystemUsingAdvancedApply)
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
+    GKO_ASSERT_MTX_NEAR(x, l({-8.5, -3.0, 6.0}), r<value_type>::value);
+}
+
+
+TYPED_TEST(Bicgstab, SolvesDenseSystemUsingAdvancedApplyMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->bicgstab_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({-8.5, -3.0, 6.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Bicgstab, SolvesDenseSystemUsingAdvancedApplyComplex)
+{
+    using Scalar = typename TestFixture::Mtx;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->bicgstab_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{-8.5, 17.0}, value_type{-3.0, 6.0},
+                           value_type{6.0, -12.0}}),
+                        r<value_type>::value);
+}
+
 
-    GKO_ASSERT_MTX_NEAR(x, l({-8.5, -3.0, 6.0}), half_tol);
+TYPED_TEST(Bicgstab, SolvesDenseSystemUsingAdvancedApplyMixedComplex)
+{
+    using Scalar = gko::matrix::Dense<
+        gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->bicgstab_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{-8.5, 17.0}, value_type{-3.0, 6.0},
+                           value_type{6.0, -12.0}}),
+                        (r_mixed<value_type, TypeParam>()));
 }
 
 
@@ -195,7 +316,6 @@ TYPED_TEST(Bicgstab, SolvesMultipleDenseSystemsUsingAdvancedApply)
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
-
     GKO_ASSERT_MTX_NEAR(x, l({{-8.5, 1.0}, {-3.0, 2.0}, {6.0, -5.0}}),
                         half_tol);
 }
diff --git a/reference/test/solver/cb_gmres_kernels.cpp b/reference/test/solver/cb_gmres_kernels.cpp
index ae5e3fa4ec9..b9ac6cfbc53 100644
--- a/reference/test/solver/cb_gmres_kernels.cpp
+++ b/reference/test/solver/cb_gmres_kernels.cpp
@@ -199,6 +199,68 @@ TYPED_TEST(CbGmres, SolvesStencilSystem)
 }
 
 
+TYPED_TEST(CbGmres, SolvesStencilSystemMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->cb_gmres_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x, l({1.0, 3.0, 2.0}),
+        std::max<double>(this->assert_precision(), r<value_type>::value));
+}
+
+
+TYPED_TEST(CbGmres, SolvesStencilSystemComplex)
+{
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->cb_gmres_factory->generate(this->mtx);
+    auto b =
+        gko::initialize<Mtx>({value_type{13.0, -26.0}, value_type{7.0, -14.0},
+                              value_type{1.0, -2.0}},
+                             this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{3.0, -6.0},
+                           value_type{2.0, -4.0}}),
+                        this->assert_precision());
+}
+
+
+TYPED_TEST(CbGmres, SolvesStencilSystemMixedComplex)
+{
+    using value_type =
+        gko::to_complex<gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->cb_gmres_factory->generate(this->mtx);
+    auto b =
+        gko::initialize<Mtx>({value_type{13.0, -26.0}, value_type{7.0, -14.0},
+                              value_type{1.0, -2.0}},
+                             this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({value_type{1.0, -2.0}, value_type{3.0, -6.0},
+           value_type{2.0, -4.0}}),
+        std::max<double>(this->assert_precision(), r<value_type>::value));
+}
+
+
 TYPED_TEST(CbGmres, SolvesStencilSystem2)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -260,6 +322,76 @@ TYPED_TEST(CbGmres, SolvesStencilSystemUsingAdvancedApply)
 }
 
 
+TYPED_TEST(CbGmres, SolvesStencilSystemUsingAdvancedApplyMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->cb_gmres_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x, l({1.5, 5.0, 2.0}),
+        std::max<double>(this->assert_precision(), r<value_type>::value));
+}
+
+
+TYPED_TEST(CbGmres, SolvesStencilSystemUsingAdvancedApplyComplex)
+{
+    using Scalar = typename TestFixture::Mtx;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->cb_gmres_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b =
+        gko::initialize<Mtx>({value_type{13.0, -26.0}, value_type{7.0, -14.0},
+                              value_type{1.0, -2.0}},
+                             this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.5, -3.0}, value_type{5.0, -10.0},
+                           value_type{2.0, -4.0}}),
+                        this->assert_precision());
+}
+
+
+TYPED_TEST(CbGmres, SolvesStencilSystemUsingAdvancedApplyMixedComplex)
+{
+    using Scalar = gko::matrix::Dense<
+        gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->cb_gmres_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b =
+        gko::initialize<Mtx>({value_type{13.0, -26.0}, value_type{7.0, -14.0},
+                              value_type{1.0, -2.0}},
+                             this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({value_type{1.5, -3.0}, value_type{5.0, -10.0},
+           value_type{2.0, -4.0}}),
+        std::max<double>(this->assert_precision(), r<value_type>::value));
+}
+
+
 TYPED_TEST(CbGmres, SolvesMultipleStencilSystemsUsingAdvancedApply)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -390,8 +522,8 @@ TYPED_TEST(CbGmres, SolvesMultipleDenseSystemForDivergenceCheck)
     ASSERT_LE(normC1 / normB1, normS1 / normB1 + this->assert_precision());
     ASSERT_LE(normC2 / normB2, normS2 / normB2 + this->assert_precision());
 
-    // Not sure if this is necessary, the assertions above should cover what is
-    // needed.
+    // Not sure if this is necessary, the assertions above should cover what
+    // is needed.
     GKO_ASSERT_MTX_NEAR(xc, mergedRes, this->assert_precision());
 }
 
diff --git a/reference/test/solver/cg_kernels.cpp b/reference/test/solver/cg_kernels.cpp
index 2eadaa5748a..6c443cfa6a9 100644
--- a/reference/test/solver/cg_kernels.cpp
+++ b/reference/test/solver/cg_kernels.cpp
@@ -126,6 +126,64 @@ TYPED_TEST(Cg, SolvesStencilSystem)
 }
 
 
+TYPED_TEST(Cg, SolvesStencilSystemMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->cg_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Cg, SolvesStencilSystemComplex)
+{
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->cg_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{3.0, -6.0},
+                           value_type{2.0, -4.0}}),
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(Cg, SolvesStencilSystemMixedComplex)
+{
+    using value_type =
+        gko::to_complex<gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->cg_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{3.0, -6.0},
+                           value_type{2.0, -4.0}}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
 TYPED_TEST(Cg, SolvesMultipleStencilSystems)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -160,6 +218,72 @@ TYPED_TEST(Cg, SolvesStencilSystemUsingAdvancedApply)
 }
 
 
+TYPED_TEST(Cg, SolvesStencilSystemUsingAdvancedApplyMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->cg_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Cg, SolvesStencilSystemUsingAdvancedApplyComplex)
+{
+    using Scalar = typename TestFixture::Mtx;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->cg_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.5, -3.0}, value_type{5.0, -10.0},
+                           value_type{2.0, -4.0}}),
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(Cg, SolvesStencilSystemUsingAdvancedApplyMixedComplex)
+{
+    using Scalar = gko::matrix::Dense<
+        gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->cg_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.5, -3.0}, value_type{5.0, -10.0},
+                           value_type{2.0, -4.0}}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
 TYPED_TEST(Cg, SolvesMultipleStencilSystemsUsingAdvancedApply)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/reference/test/solver/cgs_kernels.cpp b/reference/test/solver/cgs_kernels.cpp
index cfe2a2ef10d..6916c0a8a2d 100644
--- a/reference/test/solver/cgs_kernels.cpp
+++ b/reference/test/solver/cgs_kernels.cpp
@@ -113,15 +113,72 @@ TYPED_TEST_SUITE(Cgs, gko::test::ValueTypes);
 TYPED_TEST(Cgs, SolvesDenseSystem)
 {
     using Mtx = typename TestFixture::Mtx;
-    using T = typename TestFixture::value_type;
-    auto half_tol = std::sqrt(r<T>::value);
+    using value_type = typename TestFixture::value_type;
     auto solver = this->cgs_factory->generate(this->mtx);
     auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
     auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), half_tol);
+    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), r<value_type>::value);
+}
+
+
+TYPED_TEST(Cgs, SolvesDenseSystemMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->cgs_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Cgs, SolvesDenseSystemComplex)
+{
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->cgs_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{-4.0, 8.0}, value_type{-1.0, 2.0},
+                           value_type{4.0, -8.0}}),
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(Cgs, SolvesDenseSystemMixedComplex)
+{
+    using value_type =
+        gko::to_complex<gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->cgs_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{-4.0, 8.0}, value_type{-1.0, 2.0},
+                           value_type{4.0, -8.0}}),
+                        (r_mixed<value_type, TypeParam>()));
 }
 
 
@@ -148,7 +205,6 @@ TYPED_TEST(Cgs, SolvesDenseSystemUsingAdvancedApply)
 {
     using Mtx = typename TestFixture::Mtx;
     using value_type = typename TestFixture::value_type;
-    auto half_tol = std::sqrt(r<value_type>::value);
     auto solver = this->cgs_factory->generate(this->mtx);
     auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
     auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
@@ -157,7 +213,73 @@ TYPED_TEST(Cgs, SolvesDenseSystemUsingAdvancedApply)
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({-8.5, -3.0, 6.0}), half_tol);
+    GKO_ASSERT_MTX_NEAR(x, l({-8.5, -3.0, 6.0}), r<value_type>::value * 1e1);
+}
+
+
+TYPED_TEST(Cgs, SolvesDenseSystemUsingAdvancedApplyMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->cgs_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({-8.5, -3.0, 6.0}),
+                        (r_mixed<value_type, TypeParam>()) * 1e1);
+}
+
+
+TYPED_TEST(Cgs, SolvesDenseSystemUsingAdvancedApplyComplex)
+{
+    using Scalar = typename TestFixture::Mtx;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->cgs_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{-8.5, 17.0}, value_type{-3.0, 6.0},
+                           value_type{6.0, -12.0}}),
+                        r<value_type>::value * 1e1);
+}
+
+
+TYPED_TEST(Cgs, SolvesDenseSystemUsingAdvancedApplyMixedComplex)
+{
+    using Scalar = gko::matrix::Dense<
+        gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->cgs_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{-8.5, 17.0}, value_type{-3.0, 6.0},
+                           value_type{6.0, -12.0}}),
+                        (r_mixed<value_type, TypeParam>()) * 1e1);
 }
 
 
diff --git a/reference/test/solver/fcg_kernels.cpp b/reference/test/solver/fcg_kernels.cpp
index e95299b9c98..e374211f91b 100644
--- a/reference/test/solver/fcg_kernels.cpp
+++ b/reference/test/solver/fcg_kernels.cpp
@@ -126,6 +126,64 @@ TYPED_TEST(Fcg, SolvesStencilSystem)
 }
 
 
+TYPED_TEST(Fcg, SolvesStencilSystemMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->fcg_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Fcg, SolvesStencilSystemComplex)
+{
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->fcg_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{3.0, -6.0},
+                           value_type{2.0, -4.0}}),
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(Fcg, SolvesStencilSystemMixedComplex)
+{
+    using value_type =
+        gko::to_complex<gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->fcg_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{3.0, -6.0},
+                           value_type{2.0, -4.0}}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
 TYPED_TEST(Fcg, SolvesMultipleStencilSystems)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -156,7 +214,73 @@ TYPED_TEST(Fcg, SolvesStencilSystemUsingAdvancedApply)
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), r<value_type>::value * 1e1);
+    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), r<value_type>::value);
+}
+
+
+TYPED_TEST(Fcg, SolvesStencilSystemUsingAdvancedApplyMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->fcg_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Fcg, SolvesStencilSystemUsingAdvancedApplyComplex)
+{
+    using Scalar = typename TestFixture::Mtx;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->fcg_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.5, -3.0}, value_type{5.0, -10.0},
+                           value_type{2.0, -4.0}}),
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(Fcg, SolvesStencilSystemUsingAdvancedApplyMixedComplex)
+{
+    using Scalar = gko::matrix::Dense<
+        gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->fcg_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.5, -3.0}, value_type{5.0, -10.0},
+                           value_type{2.0, -4.0}}),
+                        (r_mixed<value_type, TypeParam>()));
 }
 
 
diff --git a/reference/test/solver/gmres_kernels.cpp b/reference/test/solver/gmres_kernels.cpp
index cded0a059ba..f64523aaabb 100644
--- a/reference/test/solver/gmres_kernels.cpp
+++ b/reference/test/solver/gmres_kernels.cpp
@@ -135,6 +135,66 @@ TYPED_TEST(Gmres, SolvesStencilSystem)
 }
 
 
+TYPED_TEST(Gmres, SolvesStencilSystemMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->gmres_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Gmres, SolvesStencilSystemComplex)
+{
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->gmres_factory->generate(this->mtx);
+    auto b =
+        gko::initialize<Mtx>({value_type{13.0, -26.0}, value_type{7.0, -14.0},
+                              value_type{1.0, -2.0}},
+                             this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{3.0, -6.0},
+                           value_type{2.0, -4.0}}),
+                        r<value_type>::value * 1e1);
+}
+
+
+TYPED_TEST(Gmres, SolvesStencilSystemMixedComplex)
+{
+    using value_type =
+        gko::to_complex<gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->gmres_factory->generate(this->mtx);
+    auto b =
+        gko::initialize<Mtx>({value_type{13.0, -26.0}, value_type{7.0, -14.0},
+                              value_type{1.0, -2.0}},
+                             this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{3.0, -6.0},
+                           value_type{2.0, -4.0}}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
 TYPED_TEST(Gmres, SolvesMultipleStencilSystems)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -169,6 +229,74 @@ TYPED_TEST(Gmres, SolvesStencilSystemUsingAdvancedApply)
 }
 
 
+TYPED_TEST(Gmres, SolvesStencilSystemUsingAdvancedApplyMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->gmres_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>({13.0, 7.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
+TYPED_TEST(Gmres, SolvesStencilSystemUsingAdvancedApplyComplex)
+{
+    using Scalar = typename TestFixture::Mtx;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->gmres_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b =
+        gko::initialize<Mtx>({value_type{13.0, -26.0}, value_type{7.0, -14.0},
+                              value_type{1.0, -2.0}},
+                             this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.5, -3.0}, value_type{5.0, -10.0},
+                           value_type{2.0, -4.0}}),
+                        r<value_type>::value * 1e1);
+}
+
+
+TYPED_TEST(Gmres, SolvesStencilSystemUsingAdvancedApplyMixedComplex)
+{
+    using Scalar = gko::matrix::Dense<
+        gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->gmres_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b =
+        gko::initialize<Mtx>({value_type{13.0, -26.0}, value_type{7.0, -14.0},
+                              value_type{1.0, -2.0}},
+                             this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.5, -3.0}, value_type{5.0, -10.0},
+                           value_type{2.0, -4.0}}),
+                        (r_mixed<value_type, TypeParam>()));
+}
+
+
 TYPED_TEST(Gmres, SolvesMultipleStencilSystemsUsingAdvancedApply)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp
index d473a94518e..c90882406d2 100644
--- a/reference/test/solver/idr_kernels.cpp
+++ b/reference/test/solver/idr_kernels.cpp
@@ -102,14 +102,71 @@ TYPED_TEST(Idr, SolvesDenseSystem)
 {
     using Mtx = typename TestFixture::Mtx;
     using value_type = typename TestFixture::value_type;
-    auto half_tol = std::sqrt(r<value_type>::value);
     auto solver = this->idr_factory->generate(this->mtx);
     auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
     auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
 
     solver->apply(b.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), half_tol);
+    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}), r<value_type>::value * 1e1);
+}
+
+
+TYPED_TEST(Idr, SolvesDenseSystemMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->idr_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({-4.0, -1.0, 4.0}),
+                        (r_mixed<value_type, TypeParam>()) * 1e1);
+}
+
+
+TYPED_TEST(Idr, SolvesDenseSystemComplex)
+{
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->idr_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{-4.0, 8.0}, value_type{-1.0, 2.0},
+                           value_type{4.0, -8.0}}),
+                        r<value_type>::value * 1e1);
+}
+
+
+TYPED_TEST(Idr, SolvesDenseSystemMixedComplex)
+{
+    using value_type =
+        gko::to_complex<gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->idr_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{-4.0, 8.0}, value_type{-1.0, 2.0},
+                           value_type{4.0, -8.0}}),
+                        (r_mixed<value_type, TypeParam>()) * 1e1);
 }
 
 
@@ -198,7 +255,6 @@ TYPED_TEST(Idr, SolvesDenseSystemUsingAdvancedApply)
 {
     using Mtx = typename TestFixture::Mtx;
     using value_type = typename TestFixture::value_type;
-    auto half_tol = std::sqrt(r<value_type>::value);
     auto solver = this->idr_factory->generate(this->mtx);
     auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
     auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
@@ -207,7 +263,73 @@ TYPED_TEST(Idr, SolvesDenseSystemUsingAdvancedApply)
 
     solver->apply(alpha.get(), b.get(), beta.get(), x.get());
 
-    GKO_ASSERT_MTX_NEAR(x, l({-8.5, -3.0, 6.0}), half_tol);
+    GKO_ASSERT_MTX_NEAR(x, l({-8.5, -3.0, 6.0}), r<value_type>::value * 1e1);
+}
+
+
+TYPED_TEST(Idr, SolvesDenseSystemUsingAdvancedApplyMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->idr_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>({-1.0, 3.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({-8.5, -3.0, 6.0}),
+                        (r_mixed<value_type, TypeParam>()) * 1e1);
+}
+
+
+TYPED_TEST(Idr, SolvesDenseSystemUsingAdvancedApplyComplex)
+{
+    using Scalar = typename TestFixture::Mtx;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->idr_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{-8.5, 17.0}, value_type{-3.0, 6.0},
+                           value_type{6.0, -12.0}}),
+                        r<value_type>::value * 1e1);
+}
+
+
+TYPED_TEST(Idr, SolvesDenseSystemUsingAdvancedApplyMixedComplex)
+{
+    using Scalar = gko::matrix::Dense<
+        gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->idr_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{-8.5, 17.0}, value_type{-3.0, 6.0},
+                           value_type{6.0, -12.0}}),
+                        (r_mixed<value_type, TypeParam>()) * 1e1);
 }
 
 
diff --git a/reference/test/solver/ir_kernels.cpp b/reference/test/solver/ir_kernels.cpp
index 6ded3841a3d..c4007bc5fe6 100644
--- a/reference/test/solver/ir_kernels.cpp
+++ b/reference/test/solver/ir_kernels.cpp
@@ -97,6 +97,64 @@ TYPED_TEST(Ir, SolvesTriangularSystem)
 }
 
 
+TYPED_TEST(Ir, SolvesTriangularSystemMixed)
+{
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->ir_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>({3.9, 9.0, 2.2}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 3.0, 2.0}),
+                        (r_mixed<value_type, TypeParam>()) * 1e1);
+}
+
+
+TYPED_TEST(Ir, SolvesTriangularSystemComplex)
+{
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->ir_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {value_type{3.9, -7.8}, value_type{9.0, -18.0}, value_type{2.2, -4.4}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{3.0, -6.0},
+                           value_type{2.0, -4.0}}),
+                        r<value_type>::value * 1e1);
+}
+
+
+TYPED_TEST(Ir, SolvesTriangularSystemMixedComplex)
+{
+    using value_type =
+        gko::to_complex<gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto solver = this->ir_factory->generate(this->mtx);
+    auto b = gko::initialize<Mtx>(
+        {value_type{3.9, -7.8}, value_type{9.0, -18.0}, value_type{2.2, -4.4}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{3.0, -6.0},
+                           value_type{2.0, -4.0}}),
+                        (r_mixed<value_type, TypeParam>()) * 1e1);
+}
+
+
 TYPED_TEST(Ir, SolvesTriangularSystemWithIterativeInnerSolver)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -162,6 +220,71 @@ TYPED_TEST(Ir, SolvesTriangularSystemUsingAdvancedApply)
 }
 
 
+TYPED_TEST(Ir, SolvesTriangularSystemUsingAdvancedApplyMixed)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using value_type = typename TestFixture::value_type;
+    auto solver = this->ir_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>({3.9, 9.0, 2.2}, this->exec);
+    auto x = gko::initialize<Mtx>({0.5, 1.0, 2.0}, this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.5, 5.0, 2.0}), r<value_type>::value * 1e1);
+}
+
+
+TYPED_TEST(Ir, SolvesTriangularSystemUsingAdvancedApplyComplex)
+{
+    using Scalar = typename TestFixture::Mtx;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->ir_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {value_type{3.9, -7.8}, value_type{9.0, -18.0}, value_type{2.2, -4.4}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.5, -3.0}, value_type{5.0, -10.0},
+                           value_type{2.0, -4.0}}),
+                        (r_mixed<value_type, TypeParam>()) * 1e1);
+}
+
+
+TYPED_TEST(Ir, SolvesTriangularSystemUsingAdvancedApplyMixedComplex)
+{
+    using Scalar = gko::matrix::Dense<
+        gko::next_precision<typename TestFixture::value_type>>;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto solver = this->ir_factory->generate(this->mtx);
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    auto b = gko::initialize<Mtx>(
+        {value_type{3.9, -7.8}, value_type{9.0, -18.0}, value_type{2.2, -4.4}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}},
+        this->exec);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.5, -3.0}, value_type{5.0, -10.0},
+                           value_type{2.0, -4.0}}),
+                        r<value_type>::value * 1e1);
+}
+
+
 TYPED_TEST(Ir, SolvesMultipleStencilSystemsUsingAdvancedApply)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/reference/test/solver/lower_trs_kernels.cpp b/reference/test/solver/lower_trs_kernels.cpp
index 4b7ff6b94a7..cf42d923b22 100644
--- a/reference/test/solver/lower_trs_kernels.cpp
+++ b/reference/test/solver/lower_trs_kernels.cpp
@@ -122,6 +122,67 @@ TYPED_TEST(LowerTrs, SolvesTriangularSystem)
 }
 
 
+TYPED_TEST(LowerTrs, SolvesTriangularSystemMixed)
+{
+    using other_value_type = typename TestFixture::value_type;
+    using value_type = gko::next_precision<other_value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    std::shared_ptr<Mtx> b = gko::initialize<Mtx>({1.0, 2.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+    auto solver = this->lower_trs_factory->generate(this->mtx);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, -1.0, 2.0}),
+                        (r_mixed<value_type, other_value_type>()));
+}
+
+
+TYPED_TEST(LowerTrs, SolvesTriangularSystemComplex)
+{
+    using Scalar = typename TestFixture::Mtx;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    std::shared_ptr<Mtx> b = gko::initialize<Mtx>(
+        {value_type{1.0, -2.0}, value_type{2.0, -4.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+    auto solver = this->lower_trs_factory->generate(this->mtx);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{-1.0, 2.0},
+                           value_type{2.0, -4.0}}),
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(LowerTrs, SolvesTriangularSystemMixedComplex)
+{
+    using other_value_type = typename TestFixture::value_type;
+    using Scalar = gko::matrix::Dense<gko::next_precision<other_value_type>>;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    std::shared_ptr<Mtx> b = gko::initialize<Mtx>(
+        {value_type{1.0, -2.0}, value_type{2.0, -4.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+    auto solver = this->lower_trs_factory->generate(this->mtx);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{-1.0, 2.0},
+                           value_type{2.0, -4.0}}),
+                        (r_mixed<value_type, other_value_type>()));
+}
+
+
 TYPED_TEST(LowerTrs, SolvesMultipleTriangularSystems)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -170,6 +231,73 @@ TYPED_TEST(LowerTrs, SolvesTriangularSystemUsingAdvancedApply)
 }
 
 
+TYPED_TEST(LowerTrs, SolvesTriangularSystemUsingAdvancedApplyMixed)
+{
+    using other_value_type = typename TestFixture::value_type;
+    using value_type = gko::next_precision<other_value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    std::shared_ptr<Mtx> b = gko::initialize<Mtx>({1.0, 2.0, 1.0}, this->exec);
+    auto x = gko::initialize<Mtx>({1.0, -1.0, 1.0}, this->exec);
+    auto solver = this->lower_trs_factory->generate(this->mtx);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, -1.0, 3.0}),
+                        (r_mixed<value_type, other_value_type>()));
+}
+
+
+TYPED_TEST(LowerTrs, SolvesTriangularSystemUsingAdvancedApplyComplex)
+{
+    using Scalar = typename TestFixture::Mtx;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    std::shared_ptr<Mtx> b = gko::initialize<Mtx>(
+        {value_type{1.0, -2.0}, value_type{2.0, -4.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{1.0, -2.0}, value_type{-1.0, 2.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto solver = this->lower_trs_factory->generate(this->mtx);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{-1.0, 2.0},
+                           value_type{3.0, -6.0}}),
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(LowerTrs, SolvesTriangularSystemUsingAdvancedApplyMixedComplex)
+{
+    using other_value_type = typename TestFixture::value_type;
+    using Scalar = gko::matrix::Dense<gko::next_precision<other_value_type>>;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    std::shared_ptr<Mtx> b = gko::initialize<Mtx>(
+        {value_type{1.0, -2.0}, value_type{2.0, -4.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{1.0, -2.0}, value_type{-1.0, 2.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto solver = this->lower_trs_factory->generate(this->mtx);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{1.0, -2.0}, value_type{-1.0, 2.0},
+                           value_type{3.0, -6.0}}),
+                        (r_mixed<value_type, other_value_type>()));
+}
+
+
 TYPED_TEST(LowerTrs, SolvesMultipleTriangularSystemsUsingAdvancedApply)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/reference/test/solver/upper_trs_kernels.cpp b/reference/test/solver/upper_trs_kernels.cpp
index 1fdab317316..29139ab5d2e 100644
--- a/reference/test/solver/upper_trs_kernels.cpp
+++ b/reference/test/solver/upper_trs_kernels.cpp
@@ -120,6 +120,67 @@ TYPED_TEST(UpperTrs, SolvesTriangularSystem)
 }
 
 
+TYPED_TEST(UpperTrs, SolvesTriangularSystemMixed)
+{
+    using other_value_type = typename TestFixture::value_type;
+    using value_type = gko::next_precision<other_value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    std::shared_ptr<Mtx> b = gko::initialize<Mtx>({4.0, 2.0, 3.0}, this->exec);
+    auto x = gko::initialize<Mtx>({0.0, 0.0, 0.0}, this->exec);
+    auto solver = this->upper_trs_factory->generate(this->mtx);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({13.0, -4.0, 3.0}),
+                        (r_mixed<value_type, other_value_type>()));
+}
+
+
+TYPED_TEST(UpperTrs, SolvesTriangularSystemComplex)
+{
+    using Scalar = typename TestFixture::Mtx;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    std::shared_ptr<Mtx> b = gko::initialize<Mtx>(
+        {value_type{4.0, -8.0}, value_type{2.0, -4.0}, value_type{3.0, -6.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+    auto solver = this->upper_trs_factory->generate(this->mtx);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{13.0, -26.0}, value_type{-4.0, 8.0},
+                           value_type{3.0, -6.0}}),
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(UpperTrs, SolvesTriangularSystemMixedComplex)
+{
+    using other_value_type = typename TestFixture::value_type;
+    using Scalar = gko::matrix::Dense<gko::next_precision<other_value_type>>;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    std::shared_ptr<Mtx> b = gko::initialize<Mtx>(
+        {value_type{4.0, -8.0}, value_type{2.0, -4.0}, value_type{3.0, -6.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}},
+        this->exec);
+    auto solver = this->upper_trs_factory->generate(this->mtx);
+
+    solver->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{13.0, -26.0}, value_type{-4.0, 8.0},
+                           value_type{3.0, -6.0}}),
+                        (r_mixed<value_type, other_value_type>()));
+}
+
+
 TYPED_TEST(UpperTrs, SolvesMultipleTriangularSystems)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -169,6 +230,73 @@ TYPED_TEST(UpperTrs, SolvesTriangularSystemUsingAdvancedApply)
 }
 
 
+TYPED_TEST(UpperTrs, SolvesTriangularSystemUsingAdvancedApplyMixed)
+{
+    using other_value_type = typename TestFixture::value_type;
+    using value_type = gko::next_precision<other_value_type>;
+    using Mtx = gko::matrix::Dense<value_type>;
+    auto alpha = gko::initialize<Mtx>({2.0}, this->exec);
+    auto beta = gko::initialize<Mtx>({-1.0}, this->exec);
+    std::shared_ptr<Mtx> b = gko::initialize<Mtx>({4.0, 2.0, 3.0}, this->exec);
+    auto x = gko::initialize<Mtx>({1.0, -1.0, 1.0}, this->exec);
+    auto solver = this->upper_trs_factory->generate(this->mtx);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({25.0, -7.0, 5.0}),
+                        (r_mixed<value_type, other_value_type>()));
+}
+
+
+TYPED_TEST(UpperTrs, SolvesTriangularSystemUsingAdvancedApplyComplex)
+{
+    using Scalar = typename TestFixture::Mtx;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    std::shared_ptr<Mtx> b = gko::initialize<Mtx>(
+        {value_type{4.0, -8.0}, value_type{2.0, -4.0}, value_type{3.0, -6.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{1.0, -2.0}, value_type{-1.0, 2.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto solver = this->upper_trs_factory->generate(this->mtx);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{25.0, -50.0}, value_type{-7.0, 14.0},
+                           value_type{5.0, -10.0}}),
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(UpperTrs, SolvesTriangularSystemUsingAdvancedApplyMixedComplex)
+{
+    using other_value_type = typename TestFixture::value_type;
+    using Scalar = gko::matrix::Dense<gko::next_precision<other_value_type>>;
+    using Mtx = gko::to_complex<typename TestFixture::Mtx>;
+    using value_type = typename Mtx::value_type;
+    auto alpha = gko::initialize<Scalar>({2.0}, this->exec);
+    auto beta = gko::initialize<Scalar>({-1.0}, this->exec);
+    std::shared_ptr<Mtx> b = gko::initialize<Mtx>(
+        {value_type{4.0, -8.0}, value_type{2.0, -4.0}, value_type{3.0, -6.0}},
+        this->exec);
+    auto x = gko::initialize<Mtx>(
+        {value_type{1.0, -2.0}, value_type{-1.0, 2.0}, value_type{1.0, -2.0}},
+        this->exec);
+    auto solver = this->upper_trs_factory->generate(this->mtx);
+
+    solver->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x,
+                        l({value_type{25.0, -50.0}, value_type{-7.0, 14.0},
+                           value_type{5.0, -10.0}}),
+                        (r_mixed<value_type, other_value_type>()));
+}
+
+
 TYPED_TEST(UpperTrs, SolvesMultipleTriangularSystemsUsingAdvancedApply)
 {
     using Mtx = typename TestFixture::Mtx;

From 7b3de163f23bba8fb62ac701ef79b06c319c45d3 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 20 Mar 2021 10:15:45 +0100
Subject: [PATCH 0589/1008] fix cbgmres for complex types
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* add missing conjugation to CBGMRES
* work around complex accessor issues

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 common/solver/cb_gmres_kernels.hpp.inc |  8 ++++----
 omp/solver/cb_gmres_kernels.cpp        | 10 ++++++----
 reference/solver/cb_gmres_kernels.cpp  |  9 ++++++---
 3 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/common/solver/cb_gmres_kernels.hpp.inc b/common/solver/cb_gmres_kernels.hpp.inc
index 524be799374..41110c05780 100644
--- a/common/solver/cb_gmres_kernels.hpp.inc
+++ b/common/solver/cb_gmres_kernels.hpp.inc
@@ -326,8 +326,8 @@ __global__ __launch_bounds__(dot_dim *dot_dim) void multidot_kernel(
     if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
         for (size_type i = start_row; i < end_row; i += blockDim.y) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
-            local_res += next_krylov_basis[next_krylov_idx] *
-                         krylov_bases(k, i, col_idx);
+            ValueType other_basis = krylov_bases(k, i, col_idx);
+            local_res += next_krylov_basis[next_krylov_idx] * conj(other_basis);
         }
     }
     // Transpose local_res, so each warp contains a local_res from the same
@@ -380,8 +380,8 @@ __global__ __launch_bounds__(block_size) void singledot_kernel(
     if (!stop_status[col_idx].has_stopped()) {
         for (size_type i = start_row; i < end_row; i += block_size) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
-            local_res += next_krylov_basis[next_krylov_idx] *
-                         krylov_bases(k, i, col_idx);
+            ValueType other_basis = krylov_bases(k, i, col_idx);
+            local_res += next_krylov_basis[next_krylov_idx] * conj(other_basis);
         }
     }
     // Transpose local_res, so each warp contains a local_res from the same
diff --git a/omp/solver/cb_gmres_kernels.cpp b/omp/solver/cb_gmres_kernels.cpp
index 46a982e44bb..09c0c2265ba 100644
--- a/omp/solver/cb_gmres_kernels.cpp
+++ b/omp/solver/cb_gmres_kernels.cpp
@@ -93,8 +93,8 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
         for (size_type k = 0; k < iter + 1; ++k) {
             ValueType hessenberg_iter_entry = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
-                hessenberg_iter_entry +=
-                    next_krylov_basis->at(j, i) * krylov_bases(k, j, i);
+                hessenberg_iter_entry += next_krylov_basis->at(j, i) *
+                                         conj(ValueType{krylov_bases(k, j, i)});
             }
             hessenberg_iter->at(k, i) = hessenberg_iter_entry;
         }
@@ -136,7 +136,8 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     hessenberg_iter_entry +=
-                        next_krylov_basis->at(j, i) * krylov_bases(k, j, i);
+                        next_krylov_basis->at(j, i) *
+                        conj(ValueType{krylov_bases(k, j, i)});
                 }
                 buffer_iter->at(k, i) = hessenberg_iter_entry;
             }
@@ -148,7 +149,8 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
-                        buffer_iter->at(k, i) * krylov_bases(k, j, i);
+                        buffer_iter->at(k, i) *
+                        conj(ValueType{krylov_bases(k, j, i)});
                 }
             }
             // for i in 1:iter
diff --git a/reference/solver/cb_gmres_kernels.cpp b/reference/solver/cb_gmres_kernels.cpp
index 811926a67ab..6b232571b07 100644
--- a/reference/solver/cb_gmres_kernels.cpp
+++ b/reference/solver/cb_gmres_kernels.cpp
@@ -89,7 +89,8 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
             hessenberg_iter->at(k, i) = zero<ValueType>();
             for (size_type j = 0; j < next_krylov_basis->get_size()[0]; ++j) {
                 hessenberg_iter->at(k, i) +=
-                    next_krylov_basis->at(j, i) * krylov_bases(k, j, i);
+                    next_krylov_basis->at(j, i) *
+                    conj(ValueType{krylov_bases(k, j, i)});
             }
         }
         // for i in 1:iter
@@ -128,7 +129,8 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     buffer_iter->at(k, i) +=
-                        next_krylov_basis->at(j, i) * krylov_bases(k, j, i);
+                        next_krylov_basis->at(j, i) *
+                        conj(ValueType{krylov_bases(k, j, i)});
                 }
             }
             // for i in 1:iter
@@ -138,7 +140,8 @@ void finish_arnoldi_CGS(matrix::Dense<ValueType> *next_krylov_basis,
                 for (size_type j = 0; j < next_krylov_basis->get_size()[0];
                      ++j) {
                     next_krylov_basis->at(j, i) -=
-                        buffer_iter->at(k, i) * krylov_bases(k, j, i);
+                        buffer_iter->at(k, i) *
+                        conj(ValueType{krylov_bases(k, j, i)});
                 }
                 hessenberg_iter->at(k, i) += buffer_iter->at(k, i);
             }

From caa4605313ed3a4d9f36548e33b577d9377592da Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 30 Mar 2021 14:26:34 +0200
Subject: [PATCH 0590/1008] use simple apply_impl in solver advanced apply

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 core/matrix/diagonal.cpp  | 2 +-
 core/solver/bicg.cpp      | 2 +-
 core/solver/bicgstab.cpp  | 2 +-
 core/solver/cb_gmres.cpp  | 2 +-
 core/solver/cg.cpp        | 2 +-
 core/solver/cgs.cpp       | 2 +-
 core/solver/fcg.cpp       | 2 +-
 core/solver/gmres.cpp     | 2 +-
 core/solver/idr.cpp       | 2 +-
 core/solver/ir.cpp        | 2 +-
 core/solver/lower_trs.cpp | 2 +-
 core/solver/upper_trs.cpp | 2 +-
 12 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/core/matrix/diagonal.cpp b/core/matrix/diagonal.cpp
index 18a4785fb13..dab0b3fa35a 100644
--- a/core/matrix/diagonal.cpp
+++ b/core/matrix/diagonal.cpp
@@ -120,7 +120,7 @@ void Diagonal<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
             auto x_clone = dense_x->clone();
-            this->apply(dense_b, x_clone.get());
+            this->apply_impl(dense_b, x_clone.get());
             dense_x->scale(dense_beta);
             dense_x->add_scaled(dense_alpha, x_clone.get());
         },
diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp
index 642ffb30364..29309955b03 100644
--- a/core/solver/bicg.cpp
+++ b/core/solver/bicg.cpp
@@ -249,7 +249,7 @@ void Bicg<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
             auto x_clone = dense_x->clone();
-            this->apply(dense_b, x_clone.get());
+            this->apply_dense_impl(dense_b, x_clone.get());
             dense_x->scale(dense_beta);
             dense_x->add_scaled(dense_alpha, x_clone.get());
         },
diff --git a/core/solver/bicgstab.cpp b/core/solver/bicgstab.cpp
index 8dd89b24607..3996f165b5c 100644
--- a/core/solver/bicgstab.cpp
+++ b/core/solver/bicgstab.cpp
@@ -228,7 +228,7 @@ void Bicgstab<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
             auto x_clone = dense_x->clone();
-            this->apply(dense_b, x_clone.get());
+            this->apply_dense_impl(dense_b, x_clone.get());
             dense_x->scale(dense_beta);
             dense_x->add_scaled(dense_alpha, x_clone.get());
         },
diff --git a/core/solver/cb_gmres.cpp b/core/solver/cb_gmres.cpp
index fb2acca9067..8478ea49b5e 100644
--- a/core/solver/cb_gmres.cpp
+++ b/core/solver/cb_gmres.cpp
@@ -510,7 +510,7 @@ void CbGmres<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
             auto x_clone = dense_x->clone();
-            this->apply(dense_b, x_clone.get());
+            this->apply_dense_impl(dense_b, x_clone.get());
             dense_x->scale(dense_beta);
             dense_x->add_scaled(dense_alpha, x_clone.get());
         },
diff --git a/core/solver/cg.cpp b/core/solver/cg.cpp
index 2a4e6c1d8b7..a675b4c51fa 100644
--- a/core/solver/cg.cpp
+++ b/core/solver/cg.cpp
@@ -188,7 +188,7 @@ void Cg<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
             auto x_clone = dense_x->clone();
-            this->apply(dense_b, x_clone.get());
+            this->apply_dense_impl(dense_b, x_clone.get());
             dense_x->scale(dense_beta);
             dense_x->add_scaled(dense_alpha, x_clone.get());
         },
diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index e798d593b58..30ccf643b1f 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -209,7 +209,7 @@ void Cgs<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
             auto x_clone = dense_x->clone();
-            this->apply(dense_b, x_clone.get());
+            this->apply_dense_impl(dense_b, x_clone.get());
             dense_x->scale(dense_beta);
             dense_x->add_scaled(dense_alpha, x_clone.get());
         },
diff --git a/core/solver/fcg.cpp b/core/solver/fcg.cpp
index 0676206b998..c9abc9e0e71 100644
--- a/core/solver/fcg.cpp
+++ b/core/solver/fcg.cpp
@@ -193,7 +193,7 @@ void Fcg<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
             auto x_clone = dense_x->clone();
-            this->apply(dense_b, x_clone.get());
+            this->apply_dense_impl(dense_b, x_clone.get());
             dense_x->scale(dense_beta);
             dense_x->add_scaled(dense_alpha, x_clone.get());
         },
diff --git a/core/solver/gmres.cpp b/core/solver/gmres.cpp
index 9476f339ebe..ac0c484a34b 100644
--- a/core/solver/gmres.cpp
+++ b/core/solver/gmres.cpp
@@ -330,7 +330,7 @@ void Gmres<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
             auto x_clone = dense_x->clone();
-            this->apply(dense_b, x_clone.get());
+            this->apply_dense_impl(dense_b, x_clone.get());
             dense_x->scale(dense_beta);
             dense_x->add_scaled(dense_alpha, x_clone.get());
         },
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index 649ca7ddf0f..388988ddcef 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -297,7 +297,7 @@ void Idr<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
             auto x_clone = dense_x->clone();
-            this->apply(dense_b, x_clone.get());
+            this->apply_impl(dense_b, x_clone.get());
             dense_x->scale(dense_beta);
             dense_x->add_scaled(dense_alpha, x_clone.get());
         },
diff --git a/core/solver/ir.cpp b/core/solver/ir.cpp
index a249c660a54..5d7aa749b87 100644
--- a/core/solver/ir.cpp
+++ b/core/solver/ir.cpp
@@ -167,7 +167,7 @@ void Ir<ValueType>::apply_impl(const LinOp *alpha, const LinOp *b,
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
             auto x_clone = dense_x->clone();
-            this->apply(dense_b, x_clone.get());
+            this->apply_dense_impl(dense_b, x_clone.get());
             dense_x->scale(dense_beta);
             dense_x->add_scaled(dense_alpha, x_clone.get());
         },
diff --git a/core/solver/lower_trs.cpp b/core/solver/lower_trs.cpp
index 3fe87356625..9c9f8306a92 100644
--- a/core/solver/lower_trs.cpp
+++ b/core/solver/lower_trs.cpp
@@ -144,7 +144,7 @@ void LowerTrs<ValueType, IndexType>::apply_impl(const LinOp *alpha,
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
             auto x_clone = dense_x->clone();
-            this->apply(dense_b, x_clone.get());
+            this->apply_impl(dense_b, x_clone.get());
             dense_x->scale(dense_beta);
             dense_x->add_scaled(dense_alpha, x_clone.get());
         },
diff --git a/core/solver/upper_trs.cpp b/core/solver/upper_trs.cpp
index bd1c5a9d896..d4529fefd69 100644
--- a/core/solver/upper_trs.cpp
+++ b/core/solver/upper_trs.cpp
@@ -144,7 +144,7 @@ void UpperTrs<ValueType, IndexType>::apply_impl(const LinOp *alpha,
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
             auto x_clone = dense_x->clone();
-            this->apply(dense_b, x_clone.get());
+            this->apply_impl(dense_b, x_clone.get());
             dense_x->scale(dense_beta);
             dense_x->add_scaled(dense_alpha, x_clone.get());
         },

From 9ec369c7d72c51a5ac7559dcfeeb9ec3fd01e481 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 30 Mar 2021 22:26:43 +0200
Subject: [PATCH 0591/1008] review updates

* fix missing documentation
* fix mixed precision tests
* add tests for device temporary conversion
* add tests for make_temporary_conversion behavior

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
---
 core/test/base/utils.cpp                      | 13 +++
 cuda/test/matrix/dense_kernels.cpp            | 71 ++++++++++++++++
 hip/test/matrix/dense_kernels.hip.cpp         | 71 ++++++++++++++++
 .../ginkgo/core/base/temporary_conversion.hpp |  2 +-
 omp/test/matrix/dense_kernels.cpp             | 81 ++++++++++++++++++-
 reference/test/base/combination.cpp           |  4 +-
 reference/test/matrix/dense_kernels.cpp       | 45 +++++++++++
 reference/test/matrix/sellp_kernels.cpp       |  6 +-
 .../test/matrix/sparsity_csr_kernels.cpp      |  8 +-
 9 files changed, 288 insertions(+), 13 deletions(-)

diff --git a/core/test/base/utils.cpp b/core/test/base/utils.cpp
index 5e6c1a1a956..b66a1260a3a 100644
--- a/core/test/base/utils.cpp
+++ b/core/test/base/utils.cpp
@@ -422,6 +422,19 @@ TEST_F(TemporaryClone, CopiesBackAfterLeavingScope)
 }
 
 
+TEST_F(TemporaryClone, DoesntCopyBackConstAfterLeavingScope)
+{
+    {
+        auto clone = make_temporary_clone(
+            omp, static_cast<const DummyObject *>(gko::lend(obj)));
+        obj->data = 7;
+    }
+
+    ASSERT_EQ(obj->get_executor(), ref);
+    ASSERT_EQ(obj->data, 7);
+}
+
+
 TEST_F(TemporaryClone, AvoidsCopyOnSameExecutor)
 {
     auto clone = make_temporary_clone(ref, gko::lend(obj));
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index 4b906b2e4f8..36b4836ed1f 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -61,9 +61,12 @@ class Dense : public ::testing::Test {
     using itype = int;
     using vtype = double;
     using Mtx = gko::matrix::Dense<vtype>;
+    using MixedMtx = gko::matrix::Dense<gko::next_precision<vtype>>;
     using NormVector = gko::matrix::Dense<gko::remove_complex<vtype>>;
     using Arr = gko::Array<itype>;
     using ComplexMtx = gko::matrix::Dense<std::complex<vtype>>;
+    using MixedComplexMtx =
+        gko::matrix::Dense<gko::next_precision<std::complex<vtype>>>;
 
     Dense() : rand_engine(15) {}
 
@@ -154,6 +157,14 @@ class Dense : public ::testing::Test {
             std::unique_ptr<Arr>(new Arr{ref, tmp3.begin(), tmp3.end()});
     }
 
+    template <typename ConvertedType, typename InputType>
+    std::unique_ptr<ConvertedType> convert(InputType &&input)
+    {
+        auto result = ConvertedType::create(input->get_executor());
+        input->convert_to(result.get());
+        return result;
+    }
+
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::CudaExecutor> cuda;
 
@@ -347,6 +358,17 @@ TEST_F(Dense, SimpleApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, SimpleApplyMixedIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    x->apply(convert<MixedMtx>(y).get(), convert<MixedMtx>(expected).get());
+    dx->apply(convert<MixedMtx>(dy).get(), convert<MixedMtx>(dresult).get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-7);
+}
+
+
 TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -358,6 +380,19 @@ TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, AdvancedApplyMixedIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    x->apply(convert<MixedMtx>(alpha).get(), convert<MixedMtx>(y).get(),
+             convert<MixedMtx>(beta).get(), convert<MixedMtx>(expected).get());
+    dx->apply(convert<MixedMtx>(dalpha).get(), convert<MixedMtx>(dy).get(),
+              convert<MixedMtx>(dbeta).get(), convert<MixedMtx>(dresult).get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-7);
+}
+
+
 TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -375,6 +410,23 @@ TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ApplyToMixedComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<MixedComplexMtx>(25, 1);
+    auto dcomplex_b = MixedComplexMtx::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<MixedComplexMtx>(65, 1);
+    auto dcomplex_x = MixedComplexMtx::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(complex_b.get(), complex_x.get());
+    dx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-7);
+}
+
+
 TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -392,6 +444,25 @@ TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, AdvancedApplyToMixedComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<MixedComplexMtx>(25, 1);
+    auto dcomplex_b = MixedComplexMtx::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<MixedComplexMtx>(65, 1);
+    auto dcomplex_x = MixedComplexMtx::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(convert<MixedMtx>(alpha).get(), complex_b.get(),
+             convert<MixedMtx>(beta).get(), complex_x.get());
+    dx->apply(convert<MixedMtx>(dalpha).get(), dcomplex_b.get(),
+              convert<MixedMtx>(dbeta).get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-7);
+}
+
+
 TEST_F(Dense, IsTransposable)
 {
     set_up_apply_data();
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index c311571415f..56d43fd829e 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -61,9 +61,12 @@ class Dense : public ::testing::Test {
     using itype = int;
     using vtype = double;
     using Mtx = gko::matrix::Dense<vtype>;
+    using MixedMtx = gko::matrix::Dense<gko::next_precision<vtype>>;
     using NormVector = gko::matrix::Dense<gko::remove_complex<vtype>>;
     using Arr = gko::Array<itype>;
     using ComplexMtx = gko::matrix::Dense<std::complex<vtype>>;
+    using MixedComplexMtx =
+        gko::matrix::Dense<gko::next_precision<std::complex<vtype>>>;
 
     Dense() : rand_engine(15) {}
 
@@ -151,6 +154,14 @@ class Dense : public ::testing::Test {
             std::unique_ptr<Arr>(new Arr{ref, tmp3.begin(), tmp3.end()});
     }
 
+    template <typename ConvertedType, typename InputType>
+    std::unique_ptr<ConvertedType> convert(InputType &&input)
+    {
+        auto result = ConvertedType::create(input->get_executor());
+        input->convert_to(result.get());
+        return result;
+    }
+
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::HipExecutor> hip;
 
@@ -342,6 +353,17 @@ TEST_F(Dense, SimpleApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, SimpleApplyMixedIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    x->apply(convert<MixedMtx>(y).get(), convert<MixedMtx>(expected).get());
+    dx->apply(convert<MixedMtx>(dy).get(), convert<MixedMtx>(dresult).get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-7);
+}
+
+
 TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -353,6 +375,19 @@ TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, AdvancedApplyMixedIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    x->apply(convert<MixedMtx>(alpha).get(), convert<MixedMtx>(y).get(),
+             convert<MixedMtx>(beta).get(), convert<MixedMtx>(expected).get());
+    dx->apply(convert<MixedMtx>(dalpha).get(), convert<MixedMtx>(dy).get(),
+              convert<MixedMtx>(dbeta).get(), convert<MixedMtx>(dresult).get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-7);
+}
+
+
 TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -370,6 +405,23 @@ TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ApplyToMixedComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<MixedComplexMtx>(25, 1);
+    auto dcomplex_b = MixedComplexMtx::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<MixedComplexMtx>(65, 1);
+    auto dcomplex_x = MixedComplexMtx::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(complex_b.get(), complex_x.get());
+    dx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-7);
+}
+
+
 TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -387,6 +439,25 @@ TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, AdvancedApplyToMixedComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<MixedComplexMtx>(25, 1);
+    auto dcomplex_b = MixedComplexMtx::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<MixedComplexMtx>(65, 1);
+    auto dcomplex_x = MixedComplexMtx::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(convert<MixedMtx>(alpha).get(), complex_b.get(),
+             convert<MixedMtx>(beta).get(), complex_x.get());
+    dx->apply(convert<MixedMtx>(dalpha).get(), dcomplex_b.get(),
+              convert<MixedMtx>(dbeta).get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-7);
+}
+
+
 TEST_F(Dense, IsTransposable)
 {
     set_up_apply_data();
diff --git a/include/ginkgo/core/base/temporary_conversion.hpp b/include/ginkgo/core/base/temporary_conversion.hpp
index 946d8378b2e..f651855e345 100644
--- a/include/ginkgo/core/base/temporary_conversion.hpp
+++ b/include/ginkgo/core/base/temporary_conversion.hpp
@@ -112,7 +112,7 @@ class convert_back_deleter<const CopyType, const OrigType> {
  *
  * Helper type that attempts to statically find the dynamic type of a given
  * LinOp from a list of ConversionCandidates and, on the first match, converts
- * it to TargetType with an appropriate .
+ * it to TargetType with an appropriate convert_back_deleter.
  *
  * @tparam ConversionCandidates  list of potential dynamic types of the input
  *                               object to be checked.
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index e6a1ad38e02..fc1e8e91063 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -63,10 +63,15 @@ namespace {
 
 class Dense : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Dense<>;
-    using NormVector = gko::matrix::Dense<gko::remove_complex<Mtx::value_type>>;
-    using Arr = gko::Array<int>;
-    using ComplexMtx = gko::matrix::Dense<std::complex<double>>;
+    using itype = int;
+    using vtype = double;
+    using Mtx = gko::matrix::Dense<vtype>;
+    using MixedMtx = gko::matrix::Dense<gko::next_precision<vtype>>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<vtype>>;
+    using Arr = gko::Array<itype>;
+    using ComplexMtx = gko::matrix::Dense<std::complex<vtype>>;
+    using MixedComplexMtx =
+        gko::matrix::Dense<gko::next_precision<std::complex<vtype>>>;
 
     Dense() : rand_engine(15) {}
 
@@ -166,6 +171,14 @@ class Dense : public ::testing::Test {
             std::unique_ptr<Arr>(new Arr{ref, tmp3.begin(), tmp3.end()});
     }
 
+    template <typename ConvertedType, typename InputType>
+    std::unique_ptr<ConvertedType> convert(InputType &&input)
+    {
+        auto result = ConvertedType::create(input->get_executor());
+        input->convert_to(result.get());
+        return result;
+    }
+
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::OmpExecutor> omp;
 
@@ -359,6 +372,17 @@ TEST_F(Dense, SimpleApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, SimpleApplyMixedIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    x->apply(convert<MixedMtx>(y).get(), convert<MixedMtx>(expected).get());
+    dx->apply(convert<MixedMtx>(dy).get(), convert<MixedMtx>(dresult).get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-7);
+}
+
+
 TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -370,6 +394,19 @@ TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, AdvancedApplyMixedIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    x->apply(convert<MixedMtx>(alpha).get(), convert<MixedMtx>(y).get(),
+             convert<MixedMtx>(beta).get(), convert<MixedMtx>(expected).get());
+    dx->apply(convert<MixedMtx>(dalpha).get(), convert<MixedMtx>(dy).get(),
+              convert<MixedMtx>(dbeta).get(), convert<MixedMtx>(dresult).get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-7);
+}
+
+
 TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -387,6 +424,23 @@ TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ApplyToMixedComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<MixedComplexMtx>(25, 1);
+    auto dcomplex_b = MixedComplexMtx::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<MixedComplexMtx>(40, 1);
+    auto dcomplex_x = MixedComplexMtx::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(complex_b.get(), complex_x.get());
+    dx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-7);
+}
+
+
 TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -404,6 +458,25 @@ TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, AdvancedApplyToMixedComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<MixedComplexMtx>(25, 1);
+    auto dcomplex_b = MixedComplexMtx::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<MixedComplexMtx>(40, 1);
+    auto dcomplex_x = MixedComplexMtx::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(convert<MixedMtx>(alpha).get(), complex_b.get(),
+             convert<MixedMtx>(beta).get(), complex_x.get());
+    dx->apply(convert<MixedMtx>(dalpha).get(), dcomplex_b.get(),
+              convert<MixedMtx>(dbeta).get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-7);
+}
+
+
 TEST_F(Dense, ConvertToCooIsEquivalentToRef)
 {
     auto rmtx = gen_mtx<Mtx>(532, 231);
diff --git a/reference/test/base/combination.cpp b/reference/test/base/combination.cpp
index 5f15fcdd701..542501fa246 100644
--- a/reference/test/base/combination.cpp
+++ b/reference/test/base/combination.cpp
@@ -96,7 +96,7 @@ TYPED_TEST(Combination, AppliesToMixedVector)
               [ 5 4 ]
     */
     using value_type = gko::next_precision<TypeParam>;
-    using Mtx = typename TestFixture::Mtx;
+    using Mtx = gko::matrix::Dense<value_type>;
     auto cmb = gko::Combination<TypeParam>::create(
         this->coefficients[0], this->operators[0], this->coefficients[1],
         this->operators[1]);
@@ -182,7 +182,7 @@ TYPED_TEST(Combination, AppliesLinearCombinationToMixedVector)
               [ 5 4 ]
     */
     using value_type = gko::next_precision<TypeParam>;
-    using Mtx = typename TestFixture::Mtx;
+    using Mtx = gko::matrix::Dense<value_type>;
     auto cmb = gko::Combination<TypeParam>::create(
         this->coefficients[0], this->operators[0], this->coefficients[1],
         this->operators[1]);
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 8dcfece1f04..f385a838b05 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/precision_dispatch.hpp>
 #include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/diagonal.hpp>
@@ -2685,6 +2686,50 @@ TYPED_TEST(Dense, GetImagWithGivenResultFailsForWrongDimensions)
 }
 
 
+TYPED_TEST(Dense, MakeTemporaryConversionDoesntConvertOnMatch)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto alpha = gko::initialize<Mtx>({8.0}, this->exec);
+
+    ASSERT_EQ(gko::make_temporary_conversion<T>(alpha.get()).get(),
+              alpha.get());
+}
+
+
+TYPED_TEST(Dense, MakeTemporaryConversionConvertsBack)
+{
+    using MixedMtx = typename TestFixture::MixedMtx;
+    using T = typename TestFixture::value_type;
+    using MixedT = typename MixedMtx::value_type;
+    auto alpha = gko::initialize<MixedMtx>({8.0}, this->exec);
+
+    {
+        auto conversion = gko::make_temporary_conversion<T>(alpha.get());
+        conversion->at(0, 0) = T{7.0};
+    }
+
+    ASSERT_EQ(alpha->at(0, 0), MixedT{7.0});
+}
+
+
+TYPED_TEST(Dense, MakeTemporaryConversionConstDoesntConvertBack)
+{
+    using MixedMtx = typename TestFixture::MixedMtx;
+    using T = typename TestFixture::value_type;
+    using MixedT = typename MixedMtx::value_type;
+    auto alpha = gko::initialize<MixedMtx>({8.0}, this->exec);
+
+    {
+        auto conversion = gko::make_temporary_conversion<T>(
+            static_cast<const MixedMtx *>(alpha.get()));
+        alpha->at(0, 0) = MixedT{7.0};
+    }
+
+    ASSERT_EQ(alpha->at(0, 0), MixedT{7.0});
+}
+
+
 template <typename T>
 class DenseComplex : public ::testing::Test {
 protected:
diff --git a/reference/test/matrix/sellp_kernels.cpp b/reference/test/matrix/sellp_kernels.cpp
index 9ee2eebd38e..1a8ec3e9e07 100644
--- a/reference/test/matrix/sellp_kernels.cpp
+++ b/reference/test/matrix/sellp_kernels.cpp
@@ -98,7 +98,8 @@ TYPED_TEST(Sellp, AppliesToDenseVector)
 
 TYPED_TEST(Sellp, AppliesToMixedDenseVector)
 {
-    using Vec = typename TestFixture::Vec;
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Vec = gko::matrix::Dense<value_type>;
     auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
     auto y = Vec::create(this->exec, gko::dim<2>{2, 1});
 
@@ -146,7 +147,8 @@ TYPED_TEST(Sellp, AppliesLinearCombinationToDenseVector)
 
 TYPED_TEST(Sellp, AppliesLinearCombinationToMixedDenseVector)
 {
-    using Vec = typename TestFixture::Vec;
+    using value_type = gko::next_precision<typename TestFixture::value_type>;
+    using Vec = gko::matrix::Dense<value_type>;
     auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
     auto beta = gko::initialize<Vec>({2.0}, this->exec);
     auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
diff --git a/reference/test/matrix/sparsity_csr_kernels.cpp b/reference/test/matrix/sparsity_csr_kernels.cpp
index 95f7123e3cc..f0c3239271f 100644
--- a/reference/test/matrix/sparsity_csr_kernels.cpp
+++ b/reference/test/matrix/sparsity_csr_kernels.cpp
@@ -176,8 +176,8 @@ TYPED_TEST(SparsityCsr, AppliesToDenseVector)
 
 TYPED_TEST(SparsityCsr, AppliesToMixedDenseVector)
 {
-    using Vec = typename TestFixture::Vec;
-    using T = typename TestFixture::value_type;
+    using T = gko::next_precision<typename TestFixture::value_type>;
+    using Vec = gko::matrix::Dense<T>;
     auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
     auto y = Vec::create(this->exec, gko::dim<2>{2, 1});
 
@@ -223,8 +223,8 @@ TYPED_TEST(SparsityCsr, AppliesLinearCombinationToDenseVector)
 
 TYPED_TEST(SparsityCsr, AppliesLinearCombinationToMixedDenseVector)
 {
-    using Vec = typename TestFixture::Vec;
-    using T = typename TestFixture::value_type;
+    using T = gko::next_precision<typename TestFixture::value_type>;
+    using Vec = gko::matrix::Dense<T>;
     auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
     auto beta = gko::initialize<Vec>({2.0}, this->exec);
     auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);

From 7f6a9a22fd5ea0a6e5ce689530f5e865ab4a4fae Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Mon, 29 Mar 2021 00:18:57 +0200
Subject: [PATCH 0592/1008] Add has_converged check for ConvergenceLogger

---
 core/log/convergence.cpp      | 10 ++++++++--
 core/test/log/convergence.cpp |  1 +
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/core/log/convergence.cpp b/core/log/convergence.cpp
index 09db4d30523..7a9366229b3 100644
--- a/core/log/convergence.cpp
+++ b/core/log/convergence.cpp
@@ -46,16 +46,22 @@ namespace log {
 template <typename ValueType>
 void Convergence<ValueType>::on_criterion_check_completed(
     const stop::Criterion *criterion, const size_type &num_iterations,
-    const LinOp *residual, const LinOp *residual_norm, const LinOp *solution,
+    const LinOp *residual, const LinOp *residual_norm,
+    const LinOp *implicit_sq_resnorm, const LinOp *solution,
     const uint8 &stopping_id, const bool &set_finalized,
-    const Array<stopping_status> *status, const bool &oneChanged,
+    const Array<stopping_status> *status, const bool &one_changed,
     const bool &converged) const
 {
     if (converged) {
+        this->convergence_status_ = true;
         this->num_iterations_ = num_iterations;
         if (residual != nullptr) {
             this->residual_.reset(residual->clone().release());
         }
+        if (implicit_sq_resnorm != nullptr) {
+            this->implicit_sq_resnorm_.reset(
+                implicit_sq_resnorm->clone().release());
+        }
         if (residual_norm != nullptr) {
             this->residual_norm_.reset(residual_norm->clone().release());
         } else if (residual != nullptr) {
diff --git a/core/test/log/convergence.cpp b/core/test/log/convergence.cpp
index a77b9ad0db0..e53de5a681b 100644
--- a/core/test/log/convergence.cpp
+++ b/core/test/log/convergence.cpp
@@ -56,6 +56,7 @@ TYPED_TEST(Convergence, CanGetData)
     auto logger = gko::log::Convergence<TypeParam>::create(
         exec, gko::log::Logger::iteration_complete_mask);
 
+    ASSERT_EQ(logger->has_converged(), false);
     ASSERT_EQ(logger->get_num_iterations(), 0);
     ASSERT_EQ(logger->get_residual(), nullptr);
     ASSERT_EQ(logger->get_residual_norm(), nullptr);

From f2e0b260b2659440cada948b5d0af5a299ee5062 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Mon, 29 Mar 2021 00:19:16 +0200
Subject: [PATCH 0593/1008] Add implicit_sq_resnorm check for ConvLogger

---
 core/log/convergence.cpp                | 14 +++++++++++
 include/ginkgo/core/log/convergence.hpp | 28 ++++++++++++++++++++++
 include/ginkgo/core/log/logger.hpp      | 12 ++++++++++
 reference/test/log/convergence.cpp      | 31 +++++++++++++++++++++++++
 4 files changed, 85 insertions(+)

diff --git a/core/log/convergence.cpp b/core/log/convergence.cpp
index 7a9366229b3..f850fd59880 100644
--- a/core/log/convergence.cpp
+++ b/core/log/convergence.cpp
@@ -76,6 +76,20 @@ void Convergence<ValueType>::on_criterion_check_completed(
 }
 
 
+template <typename ValueType>
+void Convergence<ValueType>::on_criterion_check_completed(
+    const stop::Criterion *criterion, const size_type &num_iterations,
+    const LinOp *residual, const LinOp *residual_norm, const LinOp *solution,
+    const uint8 &stopping_id, const bool &set_finalized,
+    const Array<stopping_status> *status, const bool &one_changed,
+    const bool &converged) const
+{
+    this->on_criterion_check_completed(
+        criterion, num_iterations, residual, residual_norm, nullptr, solution,
+        stopping_id, set_finalized, status, one_changed, converged);
+}
+
+
 #define GKO_DECLARE_CONVERGENCE(_type) class Convergence<_type>
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CONVERGENCE);
 
diff --git a/include/ginkgo/core/log/convergence.hpp b/include/ginkgo/core/log/convergence.hpp
index 8addf0d2747..29a2ed728a1 100644
--- a/include/ginkgo/core/log/convergence.hpp
+++ b/include/ginkgo/core/log/convergence.hpp
@@ -72,6 +72,15 @@ class Convergence : public Logger {
         const bool &set_finalized, const Array<stopping_status> *status,
         const bool &one_changed, const bool &all_converged) const override;
 
+    void on_criterion_check_completed(
+        const stop::Criterion *criterion, const size_type &num_iterations,
+        const LinOp *residual, const LinOp *residual_norm,
+        const LinOp *implicit_sq_resnorm, const LinOp *solution,
+        const uint8 &stopping_id, const bool &set_finalized,
+        const Array<stopping_status> *status, const bool &one_changed,
+        const bool &all_converged) const override;
+
+
     /**
      * Creates a convergence logger. This dynamically allocates the memory,
      * constructs the object and returns an std::unique_ptr to this object.
@@ -94,6 +103,13 @@ class Convergence : public Logger {
             new Convergence(exec, enabled_events));
     }
 
+    /**
+     * Returns true if the solver has converged.
+     *
+     * @return the bool flag for convergence status
+     */
+    bool has_converged() const noexcept { return convergence_status_; }
+
     /**
      * Returns the number of iterations
      *
@@ -121,6 +137,16 @@ class Convergence : public Logger {
         return residual_norm_.get();
     }
 
+    /**
+     * Returns the implicit squared residual norm
+     *
+     * @return the implicit squared residual norm
+     */
+    const LinOp *get_implicit_sq_resnorm() const noexcept
+    {
+        return implicit_sq_resnorm_.get();
+    }
+
 protected:
     /**
      * Creates a Convergence logger.
@@ -136,9 +162,11 @@ class Convergence : public Logger {
     {}
 
 private:
+    mutable bool convergence_status_{false};
     mutable size_type num_iterations_{};
     mutable std::unique_ptr<LinOp> residual_{};
     mutable std::unique_ptr<LinOp> residual_norm_{};
+    mutable std::unique_ptr<LinOp> implicit_sq_resnorm_{};
 };
 
 
diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp
index 114381aaab9..4a80d13171e 100644
--- a/include/ginkgo/core/log/logger.hpp
+++ b/include/ginkgo/core/log/logger.hpp
@@ -388,6 +388,18 @@ public:                                                              \
         const uint8 &stopping_id, const bool &set_finalized,
         const Array<stopping_status> *status, const bool &one_changed,
         const bool &all_converged)
+protected:
+    virtual void on_criterion_check_completed(
+        const stop::Criterion *criterion, const size_type &it, const LinOp *r,
+        const LinOp *tau, const LinOp *implicit_sq_tau, const LinOp *x,
+        const uint8 &stopping_id, const bool &set_finalized,
+        const Array<stopping_status> *status, const bool &one_changed,
+        const bool &all_converged) const
+    {
+        this->on_criterion_check_completed(criterion, it, r, x, tau, x,
+                                           stopping_id, set_finalized, status,
+                                           one_changed, all_converged);
+    }
 
     /**
      * Register the `iteration_complete` event which logs every completed
diff --git a/reference/test/log/convergence.cpp b/reference/test/log/convergence.cpp
index 1e10f370996..cf10a472890 100644
--- a/reference/test/log/convergence.cpp
+++ b/reference/test/log/convergence.cpp
@@ -73,6 +73,37 @@ TYPED_TEST(Convergence, CatchesCriterionCheckCompleted)
         RelativeStoppingId, true, &stop_status, true, true);
 
     ASSERT_EQ(logger->get_num_iterations(), 1);
+    ASSERT_EQ(logger->has_converged(), true);
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(logger->get_residual()),
+                        l({1.0, 2.0, 2.0}), 0.0);
+    GKO_ASSERT_MTX_NEAR(gko::as<NormVector>(logger->get_residual_norm()),
+                        l({3.0}), 0.0);
+}
+
+
+TYPED_TEST(Convergence, CatchesCriterionCheckCompletedWithImplicitNorm)
+{
+    auto exec = gko::ReferenceExecutor::create();
+    auto logger = gko::log::Convergence<TypeParam>::create(
+        exec, gko::log::Logger::criterion_check_completed_mask);
+    auto criterion =
+        gko::stop::Iteration::build().with_max_iters(3u).on(exec)->generate(
+            nullptr, nullptr, nullptr);
+    constexpr gko::uint8 RelativeStoppingId{42};
+    gko::Array<gko::stopping_status> stop_status(exec, 1);
+    using Mtx = gko::matrix::Dense<TypeParam>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<TypeParam>>;
+    auto residual = gko::initialize<Mtx>({1.0, 2.0, 2.0}, exec);
+    auto implicit_sq_resnorm = gko::initialize<Mtx>({4.0}, exec);
+
+    logger->template on<gko::log::Logger::criterion_check_completed>(
+        criterion.get(), 1, residual.get(), nullptr, implicit_sq_resnorm.get(),
+        nullptr, RelativeStoppingId, true, &stop_status, true, true);
+
+    ASSERT_EQ(logger->get_num_iterations(), 1);
+    ASSERT_EQ(logger->has_converged(), true);
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(logger->get_implicit_sq_resnorm()),
+                        l({4.0}), 0.0);
     GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(logger->get_residual()),
                         l({1.0, 2.0, 2.0}), 0.0);
     GKO_ASSERT_MTX_NEAR(gko::as<NormVector>(logger->get_residual_norm()),

From 2ef52b351976f2d99c6fb99e3ab59b4b5a8e31c8 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Mon, 29 Mar 2021 14:59:47 +0200
Subject: [PATCH 0594/1008] Update doc.

---
 include/ginkgo/core/log/convergence.hpp |  1 -
 include/ginkgo/core/log/logger.hpp      | 37 ++++++++++++++++++++++++-
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/include/ginkgo/core/log/convergence.hpp b/include/ginkgo/core/log/convergence.hpp
index 29a2ed728a1..e6019b85494 100644
--- a/include/ginkgo/core/log/convergence.hpp
+++ b/include/ginkgo/core/log/convergence.hpp
@@ -80,7 +80,6 @@ class Convergence : public Logger {
         const Array<stopping_status> *status, const bool &one_changed,
         const bool &all_converged) const override;
 
-
     /**
      * Creates a convergence logger. This dynamically allocates the memory,
      * constructs the object and returns an std::unique_ptr to this object.
diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp
index 4a80d13171e..22a809bab75 100644
--- a/include/ginkgo/core/log/logger.hpp
+++ b/include/ginkgo/core/log/logger.hpp
@@ -381,6 +381,10 @@ public:                                                              \
      * @param status  the stopping status of the right hand sides
      * @param one_changed  whether at least one right hand side converged or not
      * @param all_converged  whether all right hand sides
+     *
+     * @note The on_criterion_check_completed function that this macro declares
+     * is deprecated. Please use the one with the additional implicit_tau_sq
+     * parameter as below.
      */
     GKO_LOGGER_REGISTER_EVENT(
         20, criterion_check_completed, const stop::Criterion *criterion,
@@ -389,9 +393,26 @@ public:                                                              \
         const Array<stopping_status> *status, const bool &one_changed,
         const bool &all_converged)
 protected:
+    /**
+     * stop::Criterion's check completed event. Parameters are the Criterion,
+     * the stoppingId, the finalized boolean, the stopping status, plus the
+     * output one_changed boolean and output all_converged boolean.
+     *
+     * @param criterion  the criterion used
+     * @param it  the current iteration count
+     * @param r  the residual
+     * @param tau  the residual norm
+     * @param implicit_tau_sq  the implicit residual norm squared
+     * @param x  the solution
+     * @param stopping_id  the id of the stopping criterion
+     * @param set_finalized  whether this finalizes the iteration
+     * @param status  the stopping status of the right hand sides
+     * @param one_changed  whether at least one right hand side converged or not
+     * @param all_converged  whether all right hand sides
+     */
     virtual void on_criterion_check_completed(
         const stop::Criterion *criterion, const size_type &it, const LinOp *r,
-        const LinOp *tau, const LinOp *implicit_sq_tau, const LinOp *x,
+        const LinOp *tau, const LinOp *implicit_tau_sq, const LinOp *x,
         const uint8 &stopping_id, const bool &set_finalized,
         const Array<stopping_status> *status, const bool &one_changed,
         const bool &all_converged) const
@@ -409,12 +430,26 @@ public:                                                              \
      * @param r  the residual
      * @param x  the solution vector (optional)
      * @param tau  the residual norm (optional)
+     *
+     * @note The on_iteration_complete function that this macro declares is
+     * deprecated. Please use the one with the additional implicit_tau_sq
+     * parameter as below.
      */
     GKO_LOGGER_REGISTER_EVENT(21, iteration_complete, const LinOp *solver,
                               const size_type &it, const LinOp *r,
                               const LinOp *x = nullptr,
                               const LinOp *tau = nullptr)
 protected:
+    /**
+     * Register the `iteration_complete` event which logs every completed
+     * iterations.
+     *
+     * @param it  the current iteration count
+     * @param r  the residual
+     * @param x  the solution vector (optional)
+     * @param tau  the residual norm (optional)
+     * @param implicit_tau_sq  the implicit residual norm squared (optional)
+     */
     virtual void on_iteration_complete(const LinOp *solver, const size_type &it,
                                        const LinOp *r, const LinOp *x,
                                        const LinOp *tau,

From 8dd9b22848e09b7b273cab42ab25866c99301899 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Mon, 29 Mar 2021 23:46:18 +0200
Subject: [PATCH 0595/1008] Use new crit_check_comp in stop crit logger call

---
 include/ginkgo/core/stop/criterion.hpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/include/ginkgo/core/stop/criterion.hpp b/include/ginkgo/core/stop/criterion.hpp
index 20f1ce8d1f5..5a3740a5c92 100644
--- a/include/ginkgo/core/stop/criterion.hpp
+++ b/include/ginkgo/core/stop/criterion.hpp
@@ -162,8 +162,9 @@ class Criterion : public EnableAbstractPolymorphicObject<Criterion> {
             stopping_id, set_finalized, stop_status, one_changed, updater);
         this->template log<log::Logger::criterion_check_completed>(
             this, updater.num_iterations_, updater.residual_,
-            updater.residual_norm_, updater.solution_, stopping_id,
-            set_finalized, stop_status, *one_changed, all_converged);
+            updater.residual_norm_, updater.implicit_sq_residual_norm_,
+            updater.solution_, stopping_id, set_finalized, stop_status,
+            *one_changed, all_converged);
         return all_converged;
     }
 

From 149dea94311884c49da97082bc3b07820b6df4f1 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Mon, 29 Mar 2021 23:46:59 +0200
Subject: [PATCH 0596/1008] Update example with convergence logger

---
 .../adaptiveprecision-blockjacobi.cpp                         | 4 ++++
 examples/adaptiveprecision-blockjacobi/doc/results.dox        | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp b/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
index 2d1cb9dd4c5..b3e672e6c1f 100644
--- a/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
+++ b/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp
@@ -133,6 +133,7 @@ int main(int argc, char *argv[])
                                      .on(exec))
             .on(exec);
     // Create solver
+    solver_gen->add_logger(logger);
     auto solver = solver_gen->generate(A);
 
 
@@ -148,11 +149,14 @@ int main(int argc, char *argv[])
     auto res = gko::initialize<real_vec>({0.0}, exec);
     A->apply(lend(one), lend(x), lend(neg_one), lend(b));
     b->compute_norm2(lend(res));
+    auto impl_res = gko::as<real_vec>(logger->get_implicit_sq_resnorm());
 
     std::cout << "Initial residual norm sqrt(r^T r):\n";
     write(std::cout, lend(initres));
     std::cout << "Final residual norm sqrt(r^T r):\n";
     write(std::cout, lend(res));
+    std::cout << "Implicit residual norm squared (r^2):\n";
+    write(std::cout, lend(impl_res));
 
     // Print solver statistics
     std::cout << "CG iteration count:     " << logger->get_num_iterations()
diff --git a/examples/adaptiveprecision-blockjacobi/doc/results.dox b/examples/adaptiveprecision-blockjacobi/doc/results.dox
index d3db070f8fa..b15a46994ca 100644
--- a/examples/adaptiveprecision-blockjacobi/doc/results.dox
+++ b/examples/adaptiveprecision-blockjacobi/doc/results.dox
@@ -11,6 +11,10 @@ Final residual norm sqrt(r^T r):
 %%MatrixMarket matrix array real general
 1 1
 5.69384e-06
+Implicit residual norm squared (r^2):
+%%MatrixMarket matrix array real general
+1 1
+1.27043e-15
 CG iteration count:     5
 CG execution time [ms]: 0.080041
 

From 5a5d69f08101a4da3e50d4513aca02f35079f057 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 30 Mar 2021 23:56:19 +0200
Subject: [PATCH 0597/1008] Update ConvLogger to check for actual convergence.

+ Differentiate between stopping and converging.
+ has_converged returns true only when all the ids have
  converged and false if any of them stopped.
---
 core/log/convergence.cpp                | 16 +++++--
 include/ginkgo/core/log/convergence.hpp |  8 ++--
 reference/test/log/convergence.cpp      | 62 ++++++++++++++++++++++++-
 3 files changed, 77 insertions(+), 9 deletions(-)

diff --git a/core/log/convergence.cpp b/core/log/convergence.cpp
index f850fd59880..7db17ebb531 100644
--- a/core/log/convergence.cpp
+++ b/core/log/convergence.cpp
@@ -50,10 +50,18 @@ void Convergence<ValueType>::on_criterion_check_completed(
     const LinOp *implicit_sq_resnorm, const LinOp *solution,
     const uint8 &stopping_id, const bool &set_finalized,
     const Array<stopping_status> *status, const bool &one_changed,
-    const bool &converged) const
+    const bool &stopped) const
 {
-    if (converged) {
+    if (stopped) {
+        Array<stopping_status> tmp(status->get_executor()->get_master(),
+                                   *status);
         this->convergence_status_ = true;
+        for (int i = 0; i < status->get_num_elems(); i++) {
+            if (!tmp.get_data()[i].has_converged()) {
+                this->convergence_status_ = false;
+                break;
+            }
+        }
         this->num_iterations_ = num_iterations;
         if (residual != nullptr) {
             this->residual_.reset(residual->clone().release());
@@ -82,11 +90,11 @@ void Convergence<ValueType>::on_criterion_check_completed(
     const LinOp *residual, const LinOp *residual_norm, const LinOp *solution,
     const uint8 &stopping_id, const bool &set_finalized,
     const Array<stopping_status> *status, const bool &one_changed,
-    const bool &converged) const
+    const bool &stopped) const
 {
     this->on_criterion_check_completed(
         criterion, num_iterations, residual, residual_norm, nullptr, solution,
-        stopping_id, set_finalized, status, one_changed, converged);
+        stopping_id, set_finalized, status, one_changed, stopped);
 }
 
 
diff --git a/include/ginkgo/core/log/convergence.hpp b/include/ginkgo/core/log/convergence.hpp
index e6019b85494..81dd9d31145 100644
--- a/include/ginkgo/core/log/convergence.hpp
+++ b/include/ginkgo/core/log/convergence.hpp
@@ -54,7 +54,7 @@ namespace log {
 /**
  * Convergence is a Logger which logs data strictly from the
  * `criterion_check_completed` event. The purpose of this logger is to give a
- * simple access to standard data generated by the solver once it has converged
+ * simple access to standard data generated by the solver once it has stopped
  * with minimal overhead.
  *
  * This logger also computes the residual norm from the residual when the
@@ -70,7 +70,7 @@ class Convergence : public Logger {
         const LinOp *residual, const LinOp *residual_norm,
         const LinOp *solution, const uint8 &stopping_id,
         const bool &set_finalized, const Array<stopping_status> *status,
-        const bool &one_changed, const bool &all_converged) const override;
+        const bool &one_changed, const bool &all_stopped) const override;
 
     void on_criterion_check_completed(
         const stop::Criterion *criterion, const size_type &num_iterations,
@@ -78,7 +78,7 @@ class Convergence : public Logger {
         const LinOp *implicit_sq_resnorm, const LinOp *solution,
         const uint8 &stopping_id, const bool &set_finalized,
         const Array<stopping_status> *status, const bool &one_changed,
-        const bool &all_converged) const override;
+        const bool &all_stopped) const override;
 
     /**
      * Creates a convergence logger. This dynamically allocates the memory,
@@ -103,7 +103,7 @@ class Convergence : public Logger {
     }
 
     /**
-     * Returns true if the solver has converged.
+     * Returns true if the solver has stopped.
      *
      * @return the bool flag for convergence status
      */
diff --git a/reference/test/log/convergence.cpp b/reference/test/log/convergence.cpp
index cf10a472890..68de2e6f4f5 100644
--- a/reference/test/log/convergence.cpp
+++ b/reference/test/log/convergence.cpp
@@ -64,6 +64,7 @@ TYPED_TEST(Convergence, CatchesCriterionCheckCompleted)
             nullptr, nullptr, nullptr);
     constexpr gko::uint8 RelativeStoppingId{42};
     gko::Array<gko::stopping_status> stop_status(exec, 1);
+    stop_status.get_data()[0].reset();
     using Mtx = gko::matrix::Dense<TypeParam>;
     using NormVector = gko::matrix::Dense<gko::remove_complex<TypeParam>>;
     auto residual = gko::initialize<Mtx>({1.0, 2.0, 2.0}, exec);
@@ -73,6 +74,35 @@ TYPED_TEST(Convergence, CatchesCriterionCheckCompleted)
         RelativeStoppingId, true, &stop_status, true, true);
 
     ASSERT_EQ(logger->get_num_iterations(), 1);
+    ASSERT_EQ(logger->has_converged(), false);
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(logger->get_residual()),
+                        l({1.0, 2.0, 2.0}), 0.0);
+    GKO_ASSERT_MTX_NEAR(gko::as<NormVector>(logger->get_residual_norm()),
+                        l({3.0}), 0.0);
+}
+
+
+TYPED_TEST(Convergence, CatchesCriterionCheckCompletedWithConvCheck)
+{
+    auto exec = gko::ReferenceExecutor::create();
+    auto logger = gko::log::Convergence<TypeParam>::create(
+        exec, gko::log::Logger::criterion_check_completed_mask);
+    auto criterion =
+        gko::stop::Iteration::build().with_max_iters(3u).on(exec)->generate(
+            nullptr, nullptr, nullptr);
+    constexpr gko::uint8 RelativeStoppingId{42};
+    gko::Array<gko::stopping_status> stop_status(exec, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[0].converge(0);
+    using Mtx = gko::matrix::Dense<TypeParam>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<TypeParam>>;
+    auto residual = gko::initialize<Mtx>({1.0, 2.0, 2.0}, exec);
+
+    logger->template on<gko::log::Logger::criterion_check_completed>(
+        criterion.get(), 3, residual.get(), nullptr, nullptr,
+        RelativeStoppingId, true, &stop_status, true, true);
+
+    ASSERT_EQ(logger->get_num_iterations(), 3);
     ASSERT_EQ(logger->has_converged(), true);
     GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(logger->get_residual()),
                         l({1.0, 2.0, 2.0}), 0.0);
@@ -81,6 +111,35 @@ TYPED_TEST(Convergence, CatchesCriterionCheckCompleted)
 }
 
 
+TYPED_TEST(Convergence, CatchesCriterionCheckCompletedWithStopCheck)
+{
+    auto exec = gko::ReferenceExecutor::create();
+    auto logger = gko::log::Convergence<TypeParam>::create(
+        exec, gko::log::Logger::criterion_check_completed_mask);
+    auto criterion =
+        gko::stop::Iteration::build().with_max_iters(3u).on(exec)->generate(
+            nullptr, nullptr, nullptr);
+    constexpr gko::uint8 RelativeStoppingId{42};
+    gko::Array<gko::stopping_status> stop_status(exec, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[0].stop(0);
+    using Mtx = gko::matrix::Dense<TypeParam>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<TypeParam>>;
+    auto residual = gko::initialize<Mtx>({1.0, 2.0, 2.0}, exec);
+
+    logger->template on<gko::log::Logger::criterion_check_completed>(
+        criterion.get(), 3, residual.get(), nullptr, nullptr,
+        RelativeStoppingId, true, &stop_status, true, true);
+
+    ASSERT_EQ(logger->get_num_iterations(), 3);
+    ASSERT_EQ(logger->has_converged(), false);
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(logger->get_residual()),
+                        l({1.0, 2.0, 2.0}), 0.0);
+    GKO_ASSERT_MTX_NEAR(gko::as<NormVector>(logger->get_residual_norm()),
+                        l({3.0}), 0.0);
+}
+
+
 TYPED_TEST(Convergence, CatchesCriterionCheckCompletedWithImplicitNorm)
 {
     auto exec = gko::ReferenceExecutor::create();
@@ -91,6 +150,7 @@ TYPED_TEST(Convergence, CatchesCriterionCheckCompletedWithImplicitNorm)
             nullptr, nullptr, nullptr);
     constexpr gko::uint8 RelativeStoppingId{42};
     gko::Array<gko::stopping_status> stop_status(exec, 1);
+    stop_status.get_data()[0].reset();
     using Mtx = gko::matrix::Dense<TypeParam>;
     using NormVector = gko::matrix::Dense<gko::remove_complex<TypeParam>>;
     auto residual = gko::initialize<Mtx>({1.0, 2.0, 2.0}, exec);
@@ -101,7 +161,7 @@ TYPED_TEST(Convergence, CatchesCriterionCheckCompletedWithImplicitNorm)
         nullptr, RelativeStoppingId, true, &stop_status, true, true);
 
     ASSERT_EQ(logger->get_num_iterations(), 1);
-    ASSERT_EQ(logger->has_converged(), true);
+    ASSERT_EQ(logger->has_converged(), false);
     GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(logger->get_implicit_sq_resnorm()),
                         l({4.0}), 0.0);
     GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(logger->get_residual()),

From 9514d78244d80be9e246acefea25c1930bcd7d77 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 31 Mar 2021 08:09:04 +0200
Subject: [PATCH 0598/1008] Allow resetting of convergence_status

---
 include/ginkgo/core/log/convergence.hpp |  7 ++++++-
 reference/test/log/convergence.cpp      | 28 +++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/include/ginkgo/core/log/convergence.hpp b/include/ginkgo/core/log/convergence.hpp
index 81dd9d31145..b3ceb5781f9 100644
--- a/include/ginkgo/core/log/convergence.hpp
+++ b/include/ginkgo/core/log/convergence.hpp
@@ -103,12 +103,17 @@ class Convergence : public Logger {
     }
 
     /**
-     * Returns true if the solver has stopped.
+     * Returns true if the solver has converged.
      *
      * @return the bool flag for convergence status
      */
     bool has_converged() const noexcept { return convergence_status_; }
 
+    /**
+     * Resets the convergence status to false.
+     */
+    void reset_convergence_status() { this->convergence_status_ = false; }
+
     /**
      * Returns the number of iterations
      *
diff --git a/reference/test/log/convergence.cpp b/reference/test/log/convergence.cpp
index 68de2e6f4f5..e438bf948d3 100644
--- a/reference/test/log/convergence.cpp
+++ b/reference/test/log/convergence.cpp
@@ -140,6 +140,34 @@ TYPED_TEST(Convergence, CatchesCriterionCheckCompletedWithStopCheck)
 }
 
 
+TYPED_TEST(Convergence, CanResetConvergenceStatus)
+{
+    auto exec = gko::ReferenceExecutor::create();
+    auto logger = gko::log::Convergence<TypeParam>::create(
+        exec, gko::log::Logger::criterion_check_completed_mask);
+    auto criterion =
+        gko::stop::Iteration::build().with_max_iters(3u).on(exec)->generate(
+            nullptr, nullptr, nullptr);
+    constexpr gko::uint8 RelativeStoppingId{42};
+    gko::Array<gko::stopping_status> stop_status(exec, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[0].converge(0);
+    using Mtx = gko::matrix::Dense<TypeParam>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<TypeParam>>;
+    auto residual = gko::initialize<Mtx>({1.0, 2.0, 2.0}, exec);
+
+    logger->template on<gko::log::Logger::criterion_check_completed>(
+        criterion.get(), 3, residual.get(), nullptr, nullptr,
+        RelativeStoppingId, true, &stop_status, true, true);
+    ASSERT_EQ(logger->get_num_iterations(), 3);
+    ASSERT_EQ(logger->has_converged(), true);
+
+    logger->reset_convergence_status();
+
+    ASSERT_EQ(logger->has_converged(), false);
+}
+
+
 TYPED_TEST(Convergence, CatchesCriterionCheckCompletedWithImplicitNorm)
 {
     auto exec = gko::ReferenceExecutor::create();

From 26b14b2331671c8f5864d1a0daaf532826cc3aa5 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Fri, 26 Mar 2021 17:39:51 +0100
Subject: [PATCH 0599/1008] Support HIP 4.1.x and breaking HIP_PLATFORM change

---
 CMakeLists.txt                 |  8 ++++++--
 INSTALL.md                     | 24 ++++++++++++------------
 benchmark/CMakeLists.txt       |  6 ++++--
 cmake/build_type_helpers.cmake |  4 ++--
 cmake/create_test.cmake        |  2 +-
 core/CMakeLists.txt            |  2 +-
 hip/CMakeLists.txt             | 25 ++++++++++++++-----------
 hip/test/base/CMakeLists.txt   |  2 +-
 third_party/CMakeLists.txt     |  3 ++-
 9 files changed, 43 insertions(+), 33 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 564b5c71931..d2ba3066483 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -181,6 +181,8 @@ if(MSVC OR WIN32 OR CYGWIN OR APPLE)
     set(GINKGO_HAVE_HWLOC 0)
 endif()
 
+# We keep using NVCC/HCC for consistency with previous releases even if AMD
+# updated everything to use NVIDIA/AMD in ROCM 4.1
 set(GINKGO_HIP_PLATFORM_NVCC 0)
 set(GINKGO_HIP_PLATFORM_HCC 0)
 
@@ -196,10 +198,12 @@ if(GINKGO_BUILD_HIP)
             "Set and export the environment variable HIP_PLATFORM.")
     endif()
     message(STATUS "HIP platform set to ${GINKGO_HIP_PLATFORM}")
+    set(HIP_PLATFORM_AMD_REGEX "hcc|amd")
+    set(HIP_PLATFORM_NVIDIA_REGEX "nvcc|nvidia")
 
-    if (GINKGO_HIP_PLATFORM STREQUAL "hcc")
+    if (GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_AMD_REGEX}")
         set(GINKGO_HIP_PLATFORM_HCC 1)
-    elseif (GINKGO_HIP_PLATFORM STREQUAL "nvcc")
+    elseif (GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}")
         set(GINKGO_HIP_PLATFORM_NVCC 1)
     endif()
 endif()
diff --git a/INSTALL.md b/INSTALL.md
index 110006a2c91..c415e85080f 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -220,26 +220,26 @@ imposed by the `HIP` tool suite. The variables are the following:
 
 #### HIP platform detection of AMD and NVIDIA
 By default, Ginkgo uses the output of `/opt/rocm/hip/bin/hipconfig --platform`
-to select the backend. The accepted values are either `hcc` (AMD) or `nvcc`
-(NVIDIA). When on an AMD or NVIDIA system, this should output the correct
-platform by default. When on a system without GPUs, this should output `hcc` by
-default. To change this value, export the environment variable `HIP_PLATFORM`
-like so:
+to select the backend. The accepted values are either `hcc` (`amd` with ROCM >=
+4.1) or `nvcc` (`nvidia` with ROCM >= 4.1). When on an AMD or NVIDIA system,
+this should output the correct platform by default. When on a system without
+GPUs, this should output `hcc` by default. To change this value, export the
+environment variable `HIP_PLATFORM` like so:
 ```bash
-export HIP_PLATFORM=nvcc
+export HIP_PLATFORM=nvcc # or nvidia for ROCM >= 4.1
 ```
 
-When using `HIP_PLATFORM=hcc`, note that two `HIP` compilers can be set, the old
-`hcc` or since ROCm 3.5, `clang`. Ginkgo is only compatible with the `clang`
-based installations. Although this setting should be automatically done, it is
-also possible to manually set the `HIP` compiler to `clang`:
+When using `HIP_PLATFORM=hcc` (or `amd`), note that two `HIP` compilers can be
+set: the old `hcc`, or since ROCm 3.5, `clang`. Ginkgo is only compatible with
+the `clang` based installations. Although this setting should be automatically
+done, it is also possible to manually set the `HIP` compiler to `clang`:
 ```
 export HIP_COMPILER=clang
 ```
 
 #### Setting platform specific compilation flags
-Platform specific compilation flags can be given through the following
-CMake variables:
+Platform specific compilation flags can be given through the following CMake
+variables:
 + `-DGINKGO_HIP_COMPILER_FLAGS=`: compilation flags given to all platforms.
 + `-DGINKGO_HIP_NVCC_COMPILER_FLAGS=`: compilation flags given to NVIDIA platforms.
 + `-DGINKGO_HIP_CLANG_COMPILER_FLAGS=`: compilation flags given to AMD clang compiler.
diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
index af09ee108af..a8356d2af52 100644
--- a/benchmark/CMakeLists.txt
+++ b/benchmark/CMakeLists.txt
@@ -4,11 +4,13 @@ if (NOT CMAKE_BUILD_TYPE STREQUAL "Release")
         "will be affected")
 endif()
 
-if (GINKGO_BUILD_CUDA AND GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc")
+if (GINKGO_BUILD_CUDA AND GINKGO_BUILD_HIP AND
+    GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_AMD_REGEX}")
     message(FATAL_ERROR "Building the benchmarks for both HIP AMD and CUDA "
         "at the same time is currently not supported. "
         "Disable the benchmark build using `-DGINKGO_BUILD_BENCHMARKS=OFF` "
-        "or use `export HIP_PLATFORM=nvcc` in your build environment instead.")
+        "or use `export HIP_PLATFORM=nvcc` (ROCM <=4.0) or "
+        "`export HIP_PLATFORM=nvidia` (ROCM >= 4.1) in your build environment instead.")
 endif()
 
 function(ginkgo_benchmark_add_tuning_maybe name)
diff --git a/cmake/build_type_helpers.cmake b/cmake/build_type_helpers.cmake
index f3366b031e0..46ebf9b3b24 100644
--- a/cmake/build_type_helpers.cmake
+++ b/cmake/build_type_helpers.cmake
@@ -91,10 +91,10 @@ foreach(_LANG IN LISTS ENABLED_LANGUAGES ITEMS "HIP")
                 message(STATUS "Skipping ${_LANG}, not supported by build_type.cmake script")
             endif()
             set(${PROJECT_NAME}_${_LANG}_${_TYPE}_SUPPORTED FALSE)
-		        continue()
+            continue()
         endif()
         if(${PROJECT_NAME}_${_LANG}_${_TYPE}_SUPPORTED)
-            if(_LANG STREQUAL "HIP" AND GINKGO_HIP_PLATFORM STREQUAL "nvcc")
+            if(_LANG STREQUAL "HIP" AND GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}")
                 set(CMAKE_${_LANG}_FLAGS_${_TYPE}
                     ${${PROJECT_NAME}_NVCC_${_TYPE}_COMPILER_FLAGS}
                     CACHE STRING "Flags used by the ${_LANG} compiler during ${_TYPE} builds." FORCE
diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index 8621765c965..34e9d0ee32a 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -112,7 +112,7 @@ function(ginkgo_create_hip_test test_name)
     # NOTE: With how HIP works, passing the flags `HIPCC_OPTIONS` etc. here
     # creates a redefinition of all flags. This creates some issues with `nvcc`,
     # but `clang` seems fine with the redefinitions.
-    if (GINKGO_HIP_PLATFORM STREQUAL "nvcc")
+    if (GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}")
         hip_add_executable(${TEST_TARGET_NAME} ${test_name}.hip.cpp
             # If `FindHIP.cmake`, namely `HIP_PARSE_HIPCC_OPTIONS` macro and
             # call gets fixed, uncomment this.
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 7c67744d69a..700b713b129 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -75,7 +75,7 @@ endif()
 
 # Since we have a public dependency on HIP, this dependency appears
 # here as well
-if (GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "hcc")
+if (GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_AMD_REGEX}")
     list(APPEND GKO_RPATH_ADDITIONS "${HIP_PATH}/lib")
 endif()
 
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 2b9ed806938..fc305d898dd 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -1,4 +1,5 @@
-if (GINKGO_HIP_PLATFORM MATCHES "nvcc" AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 9.2)
+if (GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}"
+    AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 9.2)
     message(FATAL_ERROR "Ginkgo HIP backend requires CUDA >= 9.2.")
 endif()
 
@@ -95,19 +96,20 @@ execute_process(
             )
 set(GINKGO_HIP_VERSION ${GINKGO_HIP_VERSION} PARENT_SCOPE)
 
-if (GINKGO_HIP_PLATFORM MATCHES "nvcc") # ensure ENV{CUDA_PATH} is set by the user
+if (GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}")
+    # ensure ENV{CUDA_PATH} is set by the user
     if (NOT DEFINED ENV{CUDA_PATH})
         find_path(GINKGO_HIP_DEFAULT_CUDA_PATH "cuda.h" PATH /usr/local/cuda/include NO_DEFAULT_PATH)
         if (NOT GINKGO_HIP_DEFAULT_CUDA_PATH)
-            message(FATAL_ERROR "HIP nvcc backend was requested but CUDA could not be located. "
-                "Set and export the environment variable CUDA_PATH.")
+            message(FATAL_ERROR "HIP nvidia backend was requested but CUDA could not be "
+                "located. Set and export the environment variable CUDA_PATH.")
          endif()
      endif()
 endif()
 
 ## Setup all CMAKE variables to find HIP and its dependencies
 list(APPEND CMAKE_MODULE_PATH "${HIP_PATH}/cmake")
-if (GINKGO_HIP_PLATFORM STREQUAL "hcc")
+if (GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_AMD_REGEX}")
     list(APPEND CMAKE_PREFIX_PATH "${HIP_PATH}/lib/cmake")
 endif()
 list(APPEND CMAKE_PREFIX_PATH
@@ -192,7 +194,7 @@ set(GINKGO_HIP_SOURCES
     stop/residual_norm_kernels.hip.cpp)
 
 set(GINKGO_HIP_NVCC_ARCH "")
-if (GINKGO_HIP_PLATFORM MATCHES "nvcc")
+if (GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}")
     if (NOT CMAKE_CUDA_HOST_COMPILER AND NOT GINKGO_CUDA_DEFAULT_HOST_COMPILER)
         set(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}" CACHE STRING "" FORCE)
     elseif(GINKGO_CUDA_DEFAULT_HOST_COMPILER)
@@ -205,8 +207,9 @@ if (GINKGO_HIP_PLATFORM MATCHES "nvcc")
     # Remove false positive CUDA warnings when calling one<T>() and zero<T>()
     list(APPEND GINKGO_HIP_NVCC_ADDITIONAL_FLAGS "--expt-relaxed-constexpr")
 
-    if (GINKGO_HIP_PLATFROM MATCHES "nvcc" AND CMAKE_CUDA_COMPILER_VERSION
-            MATCHES "9.2" AND CMAKE_CUDA_HOST_COMPILER MATCHES ".*clang.*" )
+    if (GINKGO_HIP_PLATFROM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}"
+            AND CMAKE_CUDA_COMPILER_VERSION MATCHES "9.2"
+            AND CMAKE_CUDA_HOST_COMPILER MATCHES ".*clang.*" )
         ginkgo_extract_clang_version(${CMAKE_CUDA_HOST_COMPILER} GINKGO_CUDA_HOST_CLANG_VERSION)
 
         if (GINKGO_CUDA_HOST_CLANG_VERSION MATCHES "5\.0.*")
@@ -255,7 +258,7 @@ target_link_libraries(ginkgo_hip PRIVATE roc::hipblas roc::hipsparse hip::hipran
 
 target_compile_options(ginkgo_hip PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${GINKGO_COMPILER_FLAGS}>)
 
-if(GINKGO_HIP_PLATFORM MATCHES "hcc")
+if(GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_AMD_REGEX}")
     find_package(hip REQUIRED)
     # To save a bit of pain, we directly link against the `library` instead of
     # linking against the target.
@@ -266,14 +269,14 @@ if(GINKGO_HIP_PLATFORM MATCHES "hcc")
         get_target_property(HIP_LIBAMDHIP64_LIBRARIES hip::amdhip64 IMPORTED_LOCATION_RELEASE)
     endif()
     target_link_libraries(ginkgo_hip PUBLIC ${HIP_LIBAMDHIP64_LIBRARIES})
-elseif(GINKGO_HIP_PLATFORM MATCHES "nvcc")
+elseif(GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}")
     find_package(CUDA 9.2 REQUIRED)
     target_link_libraries(ginkgo_hip PUBLIC ${CUDA_LIBRARIES})
 endif()
 
 # Try to find everything in /opt/rocm/lib first.
 set(GKO_HIP_RPATH "${ROCM_PATH}/lib" )
-if (GINKGO_HIP_PLATFORM MATCHES "nvcc")
+if (GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}")
     list(GET CUDA_LIBRARIES 0 CUDA_FIRST_LIB)
     get_filename_component(GKO_CUDA_LIBDIR "${CUDA_FIRST_LIB}" DIRECTORY)
     list(APPEND GKO_HIP_RPATH "${GKO_CUDA_LIBDIR}")
diff --git a/hip/test/base/CMakeLists.txt b/hip/test/base/CMakeLists.txt
index 7938ae4870c..fe0205794ee 100644
--- a/hip/test/base/CMakeLists.txt
+++ b/hip/test/base/CMakeLists.txt
@@ -5,7 +5,7 @@ endif()
 ginkgo_create_hip_test(lin_op)
 ginkgo_create_hip_test(math)
 # Only hcc needs the libraries. nvcc only requires the headers.
-if (GINKGO_HIP_PLATFORM MATCHES "hcc")
+if (GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_AMD_REGEX}")
     ginkgo_create_hip_test(exception_helpers roc::hipblas roc::hipsparse hip::hiprand roc::rocrand)
 else()
     ginkgo_create_hip_test(exception_helpers)
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index 507d12b3c18..b42d5eaf20e 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -1,4 +1,5 @@
-if(GINKGO_BUILD_CUDA OR (GINKGO_BUILD_HIP AND GINKGO_HIP_PLATFORM STREQUAL "nvcc"))
+if(GINKGO_BUILD_CUDA OR (GINKGO_BUILD_HIP AND
+        GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}"))
     enable_language(CUDA)
     if (GINKGO_USE_EXTERNAL_CAS)
         include(CudaArchitectureSelector RESULT_VARIABLE GINKGO_CAS_FILE)

From 774f4c99a11b9ef8f90a4084bd980c71467383ff Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 6 Apr 2021 10:56:59 +0200
Subject: [PATCH 0600/1008] Bump the current version to 1.4.0

This will be the next master release.
---
 CMakeLists.txt          | 2 +-
 examples/build-setup.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d2ba3066483..f3e52aae9c3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -11,7 +11,7 @@ if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
     cmake_policy(SET CMP0104 OLD)
 endif()
 
-project(Ginkgo LANGUAGES C CXX VERSION 1.3.0 DESCRIPTION "A numerical linear algebra library targeting many-core architectures")
+project(Ginkgo LANGUAGES C CXX VERSION 1.4.0 DESCRIPTION "A numerical linear algebra library targeting many-core architectures")
 set(Ginkgo_VERSION_TAG "master")
 set(PROJECT_VERSION_TAG ${Ginkgo_VERSION_TAG})
 
diff --git a/examples/build-setup.sh b/examples/build-setup.sh
index 0fa7639a508..45c827e07d3 100644
--- a/examples/build-setup.sh
+++ b/examples/build-setup.sh
@@ -4,7 +4,7 @@
 LIBRARY_DIRS="core core/device_hooks reference omp cuda hip"
 LIBRARY_NAMES="ginkgo ginkgo_reference ginkgo_omp ginkgo_cuda ginkgo_hip ginkgo_dpcpp"
 SUFFIXES=".so .dylib .dll d.so d.dylib d.dll"
-VERSION="1.3.0"
+VERSION="1.4.0"
 for prefix in ${LIBRARY_DIRS}; do
     for name in ${LIBRARY_NAMES}; do
         for suffix in ${SUFFIXES}; do

From 3bf9128bd4a1f4d4a13ece6d7fd0366389cfe41a Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 6 Apr 2021 17:54:13 +0200
Subject: [PATCH 0601/1008] log first residual norm in CGS

---
 core/solver/cgs.cpp | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index 30ccf643b1f..db90730564b 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -162,6 +162,19 @@ void Cgs<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
      */
     while (true) {
         r->compute_dot(r_tld.get(), rho.get());
+
+        ++iter;
+        this->template log<log::Logger::iteration_complete>(
+            this, iter, r.get(), dense_x, nullptr, rho.get());
+        if (stop_criterion->update()
+                .num_iterations(iter)
+                .residual(r.get())
+                .implicit_sq_residual_norm(rho.get())
+                .solution(dense_x)
+                .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
+            break;
+        }
+
         // beta = rho / rho_prev
         // u = r + beta * q
         // p = u + beta * ( q + beta * p )
@@ -185,18 +198,6 @@ void Cgs<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
         exec->run(cgs::make_step_3(t.get(), u_hat.get(), r.get(), dense_x,
                                    alpha.get(), &stop_status));
 
-        ++iter;
-        this->template log<log::Logger::iteration_complete>(
-            this, iter, r.get(), dense_x, nullptr, rho.get());
-        if (stop_criterion->update()
-                .num_iterations(iter)
-                .residual(r.get())
-                .implicit_sq_residual_norm(rho.get())
-                .solution(dense_x)
-                .check(RelativeStoppingId, true, &stop_status, &one_changed)) {
-            break;
-        }
-
         swap(rho_prev, rho);
     }
 }

From f593d986281543ab15ecb295a4b6a42d9ac47f3a Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 7 Apr 2021 09:28:59 +0200
Subject: [PATCH 0602/1008] make iteration counter in CGS 0-based again

---
 core/solver/cgs.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index db90730564b..d79c9c2518d 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -149,7 +149,7 @@ void Cgs<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
         r.get());
     r_tld->copy_from(r.get());
 
-    int iter = 0;
+    int iter = -1;
     /* Memory movement summary:
      * 28n * values + 2 * matrix/preconditioner storage
      * 2x SpMV:                4n * values + 2 * storage

From 6fed006c8589b28ccf9f3226230f2c51197e1eed Mon Sep 17 00:00:00 2001
From: Aditya Kashi <aditya.kashi@kit.edu>
Date: Wed, 31 Mar 2021 00:14:30 +0200
Subject: [PATCH 0603/1008] Fixed issue with corner case of reading empty fbcsr
 matrix

The issue was flagged by SonarCloud - we would be accessing invalid
memory in case the input was empty. This is now fixed.
Also added a test for the empty input case.
---
 core/matrix/fbcsr.cpp      |  5 +++++
 core/test/matrix/fbcsr.cpp | 18 ++++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp
index 5019a047923..9e4a264b3e6 100644
--- a/core/matrix/fbcsr.cpp
+++ b/core/matrix/fbcsr.cpp
@@ -336,6 +336,11 @@ void Fbcsr<ValueType, IndexType>::read(const mat_data &data)
                              blocks.size() * bs * bs, bs);
 
     tmp->row_ptrs_.get_data()[0] = 0;
+    if (data.nonzeros.size() == 0) {
+        tmp->move_to(this);
+        return;
+    }
+
     index_type cur_brow = 0;
     index_type cur_bnz = 0;
     index_type cur_bcol = blocks.begin()->first.block_column;
diff --git a/core/test/matrix/fbcsr.cpp b/core/test/matrix/fbcsr.cpp
index 6b59ce1d30d..80890e146e5 100644
--- a/core/test/matrix/fbcsr.cpp
+++ b/core/test/matrix/fbcsr.cpp
@@ -460,6 +460,24 @@ TYPED_TEST(Fbcsr, CanBeReadFromMatrixData)
 }
 
 
+TYPED_TEST(Fbcsr, CanBeReadFromEmptyMatrixData)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using MtxData = typename TestFixture::MtxData;
+    auto m = Mtx::create(this->exec);
+    m->set_block_size(this->fbsample.bs);
+    MtxData mdata;
+    mdata.size = gko::dim<2>{0, 0};
+
+    m->read(mdata);
+
+    ASSERT_EQ(m->get_size(), (gko::dim<2>{0, 0}));
+    ASSERT_EQ(m->get_const_row_ptrs()[0], 0);
+    ASSERT_EQ(m->get_const_col_idxs(), nullptr);
+    ASSERT_EQ(m->get_const_values(), nullptr);
+}
+
+
 TYPED_TEST(Fbcsr, GeneratesCorrectMatrixData)
 {
     using value_type = typename TestFixture::value_type;

From e047953aef6ff9680e54ccdd01bf34009ef66a6e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 8 Apr 2021 15:58:02 +0200
Subject: [PATCH 0604/1008] update git-cmake-format and version requirement

---
 third_party/git-cmake-format/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/third_party/git-cmake-format/CMakeLists.txt b/third_party/git-cmake-format/CMakeLists.txt
index ad709c14152..87620c1338c 100644
--- a/third_party/git-cmake-format/CMakeLists.txt
+++ b/third_party/git-cmake-format/CMakeLists.txt
@@ -1,7 +1,7 @@
 message(STATUS "Configuring and building git-cmake-format")
 ginkgo_load_git_package(git-cmake-format
     "https://github.com/ginkgo-project/git-cmake-format.git"
-    "29c23665d624e1cae1308bec651706fdaa8fe38b"
-    "-DGCF_CLANGFORMAT_MINIMAL_VERSION=5.0.0")
+    "be9554a5d71030dcc23f1eb355c41ab9d876e776"
+    "-DGCF_CLANGFORMAT_MINIMAL_VERSION=8.0.0")
 add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/src
     ${CMAKE_CURRENT_BINARY_DIR}/build EXCLUDE_FROM_ALL)

From 67ec18e975aa56b9351180bcb6fddc8ba17d2be7 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 8 Apr 2021 15:58:42 +0200
Subject: [PATCH 0605/1008] fix issues with deleted files in format step

---
 .github/bot-pr-base.sh        | 2 +-
 .github/bot-pr-format-base.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/bot-pr-base.sh b/.github/bot-pr-base.sh
index b261bf74324..67f42e91bd0 100644
--- a/.github/bot-pr-base.sh
+++ b/.github/bot-pr-base.sh
@@ -66,7 +66,7 @@ bot_get_all_changed_files() {
   while true; do
     # this api allows 100 items per page
     # github action uses `bash -e`. The last empty page will leads jq error, use `|| :` to ignore the error.
-    local pr_page_files=$(api_get "$pr_url/files?&per_page=100&page=${page}" | jq -er '.[] | .filename' || :)
+    local pr_page_files=$(api_get "$pr_url/files?&per_page=100&page=${page}" | jq -er '.[] | select(.status != "removed") | .filename' || :)
     if [ "${pr_page_files}" = "" ]; then
       break
     fi
diff --git a/.github/bot-pr-format-base.sh b/.github/bot-pr-format-base.sh
index c4171aac779..6821306322c 100644
--- a/.github/bot-pr-format-base.sh
+++ b/.github/bot-pr-format-base.sh
@@ -34,7 +34,7 @@ cp /tmp/format_header.sh dev_tools/scripts/
 cp /tmp/update_ginkgo_header.sh dev_tools/scripts/
 
 # format files
-CLANG_FORMAT=clang-format-8
+CLANG_FORMAT=clang-format-9
 dev_tools/scripts/add_license.sh
 dev_tools/scripts/update_ginkgo_header.sh
 for f in $(echo "$TO_FORMAT" | grep -E $FORMAT_HEADER_REGEX); do dev_tools/scripts/format_header.sh "$f"; done

From 9ed54110a0af7fc5b1fa33fb1a67bea065cde0e8 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 8 Apr 2021 19:16:07 +0200
Subject: [PATCH 0606/1008] update clang-format version in README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ef045569249..36f06d4f4d3 100644
--- a/README.md
+++ b/README.md
@@ -55,7 +55,7 @@ The Ginkgo CUDA module has the following __additional__ requirements:
 In addition, if you want to contribute code to Ginkgo, you will also need the
 following:
 
-*   _clang-format 5.0.0+_ (ships as part of _clang_)
+*   _clang-format 8.0.0+_ (ships as part of _clang_)
 *   _clang-tidy_ (optional, when setting the flag `-DGINKGO_WITH_CLANG_TIDY=ON`)
 *   _iwyu_ (Include What You Use, optional, when setting the flag `-DGINKGO_WITH_IWYU=ON`)
 

From 032f22f8075e7a775f14b8d2dfe65222cba4d153 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Fri, 9 Apr 2021 11:32:00 +0200
Subject: [PATCH 0607/1008] Add CCACHE variables to the information.

---
 .gitlab-ci.yml       |  2 ++
 cmake/get_info.cmake | 29 +++++++++++++++++++++--------
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 87151c84c78..9e3a954ffdb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -34,6 +34,8 @@ stages:
   - export NUM_CORES=${CI_PARALLELISM}
   - export OMP_NUM_THREADS=${NUM_CORES}
   - export CUDA_VISIBLE_DEVICES=0
+  - export CCACHE_DIR=${CCACHE_DIR}
+  - export CCACHE_MAXSIZE=${CCACHE_MAXSIZE}
 
 .before_script_git_template: &git_before_script
   - eval $(ssh-agent -s)
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index d60ef65e6f1..be150e0c3f4 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -77,10 +77,10 @@ function(ginkgo_print_env_variable log_type var_name)
     string(SUBSTRING
         "
 --        ${var_name}:                                                          " 0 55 upd_string)
-    if(NOT ENV{${var_name}})
-        set(str_value "<empty>")
-    else()
+    if(DEFINED ENV{${var_name}})
         set(str_value "$ENV{${var_name}}")
+    else()
+        set(str_value "<empty>")
     endif()
     string(APPEND upd_string "${str_value}")
     FILE(APPEND ${log_type} "${upd_string}")
@@ -100,7 +100,7 @@ IF("${GINKGO_GIT_SHORTREV}" STREQUAL "")
     ginkgo_print_generic_header(${detailed_log} "${to_print}")
     ginkgo_print_generic_header(${minimal_log} "${to_print}")
 ELSE()
-    set(to_print "Summary of Configuration for (Ginkgo version ${Ginkgo_VERSION} with tag ${Ginkgo_VERSION_TAG}, shortrev ${GINKGO_GIT_SHORTREV})"
+    set(to_print "Summary of Configuration for Ginkgo (version ${Ginkgo_VERSION} with tag ${Ginkgo_VERSION_TAG}, shortrev ${GINKGO_GIT_SHORTREV})"
         )
     ginkgo_print_generic_header(${detailed_log} "${to_print}")
     ginkgo_print_generic_header(${minimal_log} "${to_print}")
@@ -133,10 +133,6 @@ foreach(log_type ${log_types})
         "GINKGO_BUILD_TESTS;GINKGO_BUILD_EXAMPLES;GINKGO_EXTLIB_EXAMPLE;GINKGO_BUILD_BENCHMARKS;GINKGO_BENCHMARK_ENABLE_TUNING")
     ginkgo_print_module_footer(${${log_type}} "  Documentation:")
     ginkgo_print_foreach_variable(${${log_type}} "GINKGO_BUILD_DOC;GINKGO_VERBOSE_LEVEL")
-    ginkgo_print_module_footer(${${log_type}} "  Developer helpers:")
-    ginkgo_print_foreach_variable(${${log_type}}
-        "GINKGO_DEVEL_TOOLS;GINKGO_WITH_CLANG_TIDY;GINKGO_WITH_IWYU"
-        "GINKGO_CHECK_CIRCULAR_DEPS;GINKGO_CHECK_PATH")
     ginkgo_print_module_footer(${${log_type}} "")
 endforeach()
 
@@ -168,6 +164,23 @@ IF(GINKGO_BUILD_DPCPP)
     include(dpcpp/get_info.cmake)
 ENDIF()
 
+ginkgo_print_generic_header(${minimal_log} "  Developer Tools:")
+ginkgo_print_generic_header(${detailed_log} "  Developer Tools:")
+ginkgo_print_foreach_variable(${minimal_log}
+        "GINKGO_DEVEL_TOOLS;GINKGO_WITH_CLANG_TIDY;GINKGO_WITH_IWYU"
+        "GINKGO_CHECK_CIRCULAR_DEPS;GINKGO_CHECK_PATH;GINKGO_WITH_CCACHE")
+ginkgo_print_foreach_variable(${detailed_log}
+        "GINKGO_DEVEL_TOOLS;GINKGO_WITH_CLANG_TIDY;GINKGO_WITH_IWYU"
+        "GINKGO_CHECK_CIRCULAR_DEPS;GINKGO_CHECK_PATH;GINKGO_WITH_CCACHE")
+ginkgo_print_module_footer(${detailed_log} "  CCACHE:")
+ginkgo_print_variable(${detailed_log} "CCACHE_PROGRAM")
+ginkgo_print_env_variable(${detailed_log} "CCACHE_DIR")
+ginkgo_print_env_variable(${detailed_log} "CCACHE_MAXSIZE")
+ginkgo_print_module_footer(${detailed_log} "  PATH of other tools:")
+ginkgo_print_variable(${detailed_log} "GINKGO_CLANG_TIDY_PATH")
+ginkgo_print_variable(${detailed_log} "GINKGO_IWYU_PATH")
+ginkgo_print_module_footer(${detailed_log} "")
+
 ginkgo_print_generic_header(${minimal_log} "  Components:")
 ginkgo_print_generic_header(${detailed_log} "  Components:")
 if(PAPI_sde_FOUND)

From a07a35b72a29094c2433f0539022b194a207f010 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Thu, 11 Mar 2021 18:15:53 +0100
Subject: [PATCH 0608/1008] Fix wrong executor documentation.

---
 include/ginkgo/core/base/executor.hpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index ecd8dbc0cad..e69cf883e13 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -317,9 +317,9 @@ private:                                                                     \
  * int main() {
  *     // create executors
  *     auto omp = OmpExecutor::create();
- *     auto cuda = CudaExecutor::create(omp, 0);
- *     auto hip = HipExecutor::create(omp, 0);
- *     auto dpcpp = DpcppExecutor::create(omp, 0);
+ *     auto cuda = CudaExecutor::create(0, omp);
+ *     auto hip = HipExecutor::create(0, omp);
+ *     auto dpcpp = DpcppExecutor::create(0, omp);
  *     auto ref = ReferenceExecutor::create();
  *
  *     // create the operation
@@ -342,7 +342,7 @@ private:                                                                     \
             ::gko::syn::as_list<::gko::syn::range<0, sizeof...(Args)>>;       \
                                                                               \
     public:                                                                   \
-        explicit _name##_operation(Args &&... args)                           \
+        explicit _name##_operation(Args &&...args)                            \
             : data(std::forward<Args>(args)...)                               \
         {}                                                                    \
                                                                               \
@@ -368,7 +368,7 @@ private:                                                                     \
     };                                                                        \
                                                                               \
     template <typename... Args>                                               \
-    static _name##_operation<Args...> make_##_name(Args &&... args)           \
+    static _name##_operation<Args...> make_##_name(Args &&...args)            \
     {                                                                         \
         return _name##_operation<Args...>(std::forward<Args>(args)...);       \
     }                                                                         \

From 55aad4f803e96f9d6e7d26dec26db4161f37acc6 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Thu, 11 Mar 2021 18:16:44 +0100
Subject: [PATCH 0609/1008] Make test_install be one file only.

---
 CMakeLists.txt                    |   2 +
 test_install/CMakeLists.txt       |   9 +-
 test_install/test_install.cpp     | 142 ++++++-----
 test_install/test_install.hip.cpp | 386 -----------------------------
 test_install/test_install_cuda.cu | 388 ------------------------------
 5 files changed, 85 insertions(+), 842 deletions(-)
 delete mode 100644 test_install/test_install.hip.cpp
 delete mode 100644 test_install/test_install_cuda.cu

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f3e52aae9c3..044ff9081a1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -351,6 +351,8 @@ else()
         set(GINKGO_TEST_INSTALL_HIP_COMMAND "${Ginkgo_BINARY_DIR}/test_install/test_install_hip")
     endif()
 endif()
+
+file(MAKE_DIRECTORY ${Ginkgo_BINARY_DIR}/test_install)
 add_custom_target(test_install
     COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} -H${Ginkgo_SOURCE_DIR}/test_install
     -B${Ginkgo_BINARY_DIR}/test_install
diff --git a/test_install/CMakeLists.txt b/test_install/CMakeLists.txt
index 6396b6ba88b..9b8a258dddf 100644
--- a/test_install/CMakeLists.txt
+++ b/test_install/CMakeLists.txt
@@ -33,25 +33,28 @@ if(GINKGO_BUILD_CUDA)
             ginkgo_switch_to_windows_static("CUDA")
         endif()
     endif()
-    add_executable(test_install_cuda test_install_cuda.cu)
+    set_source_file_properties(test_install.cpp PROPERTIES LANGUAGE CUDA)
+    add_executable(test_install_cuda test_install.cpp)
+    target_compile_definitions(test_install_cuda PRIVATE HAS_CUDA=1)
     target_link_libraries(test_install_cuda PRIVATE Ginkgo::ginkgo)
 endif()
 
 if(GINKGO_BUILD_HIP)
     find_package(HIP REQUIRED)
 
-    set_source_files_properties(test_install.hip.cpp PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
+    set_source_files_properties(test_install.cpp PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
     # The library was compiled with `-fPIC` when in shared library mode
     if (GINKGO_BUILD_SHARED_LIBS)
         set (GINKGO_PIC_OPTION "-fPIC")
         set (GINKGO_CUDA_PIC_OPTION "-Xcompiler '-fPIC'")
     endif()
-    hip_add_executable(test_install_hip test_install.hip.cpp
+    hip_add_executable(test_install_hip test_install.cpp
         HIPCC_OPTIONS "-std=c++14"
         CLANG_OPTIONS "${GINKGO_PIC_OPTION}"
         NVCC_OPTIONS "${GINKGO_CUDA_PIC_OPTION}")
 
     target_link_libraries(test_install_hip PRIVATE Ginkgo::ginkgo)
+    target_compile_definitions(test_install_hip PRIVATE HAS_HIP=1)
 
     # If we always link with CXX there is no RPATH issue
     set_target_properties(test_install_hip PROPERTIES LINKER_LANGUAGE CXX)
diff --git a/test_install/test_install.cpp b/test_install/test_install.cpp
index f55fef3fd0a..b5bf7864495 100644
--- a/test_install/test_install.cpp
+++ b/test_install/test_install.cpp
@@ -48,7 +48,13 @@ class PolymorphicObjectTest : public gko::PolymorphicObject {};
 
 int main(int, char **)
 {
-    auto refExec = gko::ReferenceExecutor::create();
+#if defined(HAS_CUDA)
+    auto exec = gko::CudaExecutor::create(0, gko::ReferenceExecutor::create());
+#elif defined(HAS_HIP)
+    auto exec = gko::HipExecutor::create(0, gko::ReferenceExecutor::create());
+#else
+    auto exec = gko::ReferenceExecutor::create();
+#endif
     // core/base/abstract_factory.hpp
     {
         using type1 = int;
@@ -177,12 +183,12 @@ int main(int, char **)
 
     // core/factorization/par_ilu.hpp
     {
-        auto test = gko::factorization::ParIlu<>::build().on(refExec);
+        auto test = gko::factorization::ParIlu<>::build().on(exec);
     }
 
     // core/log/convergence.hpp
     {
-        auto test = gko::log::Convergence<>::create(refExec);
+        auto test = gko::log::Convergence<>::create(exec);
     }
 
     // core/log/record.hpp
@@ -192,69 +198,69 @@ int main(int, char **)
 
     // core/log/stream.hpp
     {
-        auto test = gko::log::Stream<>::create(refExec);
+        auto test = gko::log::Stream<>::create(exec);
     }
 
 #if GKO_HAVE_PAPI_SDE
     // core/log/papi.hpp
     {
-        auto test = gko::log::Papi<>::create(refExec);
+        auto test = gko::log::Papi<>::create(exec);
     }
 #endif  // GKO_HAVE_PAPI_SDE
 
     // core/matrix/coo.hpp
     {
         using Mtx = gko::matrix::Coo<>;
-        auto test = Mtx::create(refExec, gko::dim<2>{2, 2}, 2);
+        auto test = Mtx::create(exec, gko::dim<2>{2, 2}, 2);
     }
 
     // core/matrix/csr.hpp
     {
         using Mtx = gko::matrix::Csr<>;
-        auto test = Mtx::create(refExec, gko::dim<2>{2, 2}, 2,
+        auto test = Mtx::create(exec, gko::dim<2>{2, 2}, 2,
                                 std::make_shared<Mtx::load_balance>(2));
     }
 
     // core/matrix/dense.hpp
     {
         using Mtx = gko::matrix::Dense<>;
-        auto test = Mtx::create(refExec, gko::dim<2>{2, 2});
+        auto test = Mtx::create(exec, gko::dim<2>{2, 2});
     }
 
     // core/matrix/ell.hpp
     {
         using Mtx = gko::matrix::Ell<>;
-        auto test = Mtx::create(refExec, gko::dim<2>{2, 2}, 2);
+        auto test = Mtx::create(exec, gko::dim<2>{2, 2}, 2);
     }
 
     // core/matrix/hybrid.hpp
     {
         using Mtx = gko::matrix::Hybrid<>;
-        auto test = Mtx::create(refExec, gko::dim<2>{2, 2}, 2, 2, 1);
+        auto test = Mtx::create(exec, gko::dim<2>{2, 2}, 2, 2, 1);
     }
 
     // core/matrix/identity.hpp
     {
         using Mtx = gko::matrix::Identity<>;
-        auto test = Mtx::create(refExec);
+        auto test = Mtx::create(exec);
     }
 
     // core/matrix/permutation.hpp
     {
         using Mtx = gko::matrix::Permutation<>;
-        auto test = Mtx::create(refExec, gko::dim<2>{2, 2});
+        auto test = Mtx::create(exec, gko::dim<2>{2, 2});
     }
 
     // core/matrix/sellp.hpp
     {
         using Mtx = gko::matrix::Sellp<>;
-        auto test = Mtx::create(refExec, gko::dim<2>{2, 2}, 2);
+        auto test = Mtx::create(exec, gko::dim<2>{2, 2}, 2);
     }
 
     // core/matrix/sparsity_csr.hpp
     {
         using Mtx = gko::matrix::SparsityCsr<>;
-        auto test = Mtx::create(refExec, gko::dim<2>{2, 2});
+        auto test = Mtx::create(exec, gko::dim<2>{2, 2});
     }
 
     // core/multigrid/amgx_pgm.hpp
@@ -264,130 +270,130 @@ int main(int, char **)
 
     // core/preconditioner/ilu.hpp
     {
-        auto test = gko::preconditioner::Ilu<>::build().on(refExec);
+        auto test = gko::preconditioner::Ilu<>::build().on(exec);
     }
 
     // core/preconditioner/isai.hpp
     {
-        auto test_l = gko::preconditioner::LowerIsai<>::build().on(refExec);
-        auto test_u = gko::preconditioner::UpperIsai<>::build().on(refExec);
+        auto test_l = gko::preconditioner::LowerIsai<>::build().on(exec);
+        auto test_u = gko::preconditioner::UpperIsai<>::build().on(exec);
     }
 
     // core/preconditioner/jacobi.hpp
     {
         using Bj = gko::preconditioner::Jacobi<>;
-        auto test = Bj::build().with_max_block_size(1u).on(refExec);
+        auto test = Bj::build().with_max_block_size(1u).on(exec);
     }
 
     // core/solver/bicgstab.hpp
     {
         using Solver = gko::solver::Bicgstab<>;
-        auto test = Solver::build()
-                        .with_criteria(
-                            gko::stop::Iteration::build().with_max_iters(1u).on(
-                                refExec))
-                        .on(refExec);
+        auto test =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(1u).on(exec))
+                .on(exec);
     }
 
     // core/solver/cb_gmres.hpp
     {
         using Solver = gko::solver::CbGmres<>;
-        auto test = Solver::build()
-                        .with_criteria(
-                            gko::stop::Iteration::build().with_max_iters(1u).on(
-                                refExec))
-                        .on(refExec);
+        auto test =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(1u).on(exec))
+                .on(exec);
     }
 
     // core/solver/cg.hpp
     {
         using Solver = gko::solver::Cg<>;
-        auto test = Solver::build()
-                        .with_criteria(
-                            gko::stop::Iteration::build().with_max_iters(1u).on(
-                                refExec))
-                        .on(refExec);
+        auto test =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(1u).on(exec))
+                .on(exec);
     }
 
     // core/solver/cgs.hpp
     {
         using Solver = gko::solver::Cgs<>;
-        auto test = Solver::build()
-                        .with_criteria(
-                            gko::stop::Iteration::build().with_max_iters(1u).on(
-                                refExec))
-                        .on(refExec);
+        auto test =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(1u).on(exec))
+                .on(exec);
     }
 
     // core/solver/fcg.hpp
     {
         using Solver = gko::solver::Fcg<>;
-        auto test = Solver::build()
-                        .with_criteria(
-                            gko::stop::Iteration::build().with_max_iters(1u).on(
-                                refExec))
-                        .on(refExec);
+        auto test =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(1u).on(exec))
+                .on(exec);
     }
 
     // core/solver/gmres.hpp
     {
         using Solver = gko::solver::Gmres<>;
-        auto test = Solver::build()
-                        .with_criteria(
-                            gko::stop::Iteration::build().with_max_iters(1u).on(
-                                refExec))
-                        .on(refExec);
+        auto test =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(1u).on(exec))
+                .on(exec);
     }
 
     // core/solver/ir.hpp
     {
         using Solver = gko::solver::Ir<>;
-        auto test = Solver::build()
-                        .with_criteria(
-                            gko::stop::Iteration::build().with_max_iters(1u).on(
-                                refExec))
-                        .on(refExec);
+        auto test =
+            Solver::build()
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(1u).on(exec))
+                .on(exec);
     }
 
     // core/solver/lower_trs.hpp
     {
         using Solver = gko::solver::LowerTrs<>;
-        auto test = Solver::build().on(refExec);
+        auto test = Solver::build().on(exec);
     }
 
     // core/stop/
     {
         // iteration.hpp
         auto iteration =
-            gko::stop::Iteration::build().with_max_iters(1u).on(refExec);
+            gko::stop::Iteration::build().with_max_iters(1u).on(exec);
 
         // time.hpp
         auto time = gko::stop::Time::build()
                         .with_time_limit(std::chrono::milliseconds(10))
-                        .on(refExec);
+                        .on(exec);
 
         // residual_norm.hpp
         auto main_res = gko::stop::ResidualNorm<>::build()
                             .with_reduction_factor(1e-10)
                             .with_baseline(gko::stop::mode::absolute)
-                            .on(refExec);
+                            .on(exec);
 
         auto implicit_res = gko::stop::ImplicitResidualNorm<>::build()
                                 .with_reduction_factor(1e-10)
                                 .with_baseline(gko::stop::mode::absolute)
-                                .on(refExec);
+                                .on(exec);
 
         auto res_red = gko::stop::ResidualNormReduction<>::build()
                            .with_reduction_factor(1e-10)
-                           .on(refExec);
+                           .on(exec);
 
         auto rel_res =
             gko::stop::RelativeResidualNorm<>::build().with_tolerance(1e-10).on(
-                refExec);
+                exec);
 
         auto abs_res =
             gko::stop::AbsoluteResidualNorm<>::build().with_tolerance(1e-10).on(
-                refExec);
+                exec);
 
         // stopping_status.hpp
         auto stop_status = gko::stopping_status{};
@@ -396,10 +402,16 @@ int main(int, char **)
         auto combined =
             gko::stop::Combined::build()
                 .with_criteria(std::move(time), std::move(iteration))
-                .on(refExec);
-    }
-
-    std::cout << "test_install: the Ginkgo installation was correctly detected "
+                .on(exec);
+    }
+#if defined(HAS_CUDA)
+    auto extra_info = "(CUDA)";
+#elif defined(HAS_HIP)
+    auto extra_info = "(HIP)";
+#else
+    auto extra_info = "";
+#endif
+    std::cout << "test_install"<< extra_info << ": the Ginkgo installation was correctly detected "
                  "and is complete."
               << std::endl;
 
diff --git a/test_install/test_install.hip.cpp b/test_install/test_install.hip.cpp
deleted file mode 100644
index 65708b0cffa..00000000000
--- a/test_install/test_install.hip.cpp
+++ /dev/null
@@ -1,386 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/ginkgo.hpp>
-
-
-#include <hip/hip_runtime.h>
-
-
-#include <chrono>
-#include <iostream>
-#include <map>
-#include <string>
-#include <type_traits>
-#include <utility>
-#include <vector>
-
-
-// core/base/polymorphic_object.hpp
-class PolymorphicObjectTest : public gko::PolymorphicObject {};
-
-
-int main(int, char **)
-{
-    auto refExec = gko::ReferenceExecutor::create();
-    auto hipExec = gko::HipExecutor::create(0, refExec);
-    // core/base/abstract_factory.hpp
-    {
-        using type1 = int;
-        using type2 = double;
-        static_assert(
-            std::is_same<
-                gko::AbstractFactory<type1, type2>::abstract_product_type,
-                type1>::value,
-            "abstract_factory.hpp not included properly!");
-    }
-
-    // core/base/array.hpp
-    {
-        using type1 = int;
-        using ArrayType = gko::Array<type1>;
-        ArrayType{};
-    }
-
-    // core/base/combination.hpp
-    {
-        using type1 = int;
-        static_assert(
-            std::is_same<gko::Combination<type1>::value_type, type1>::value,
-            "combination.hpp not included properly!");
-    }
-
-    // core/base/composition.hpp
-    {
-        using type1 = int;
-        static_assert(
-            std::is_same<gko::Composition<type1>::value_type, type1>::value,
-            "composition.hpp not included properly");
-    }
-
-    // core/base/dim.hpp
-    {
-        using type1 = int;
-        gko::dim<3, type1>{4, 4, 4};
-    }
-
-    // core/base/exception.hpp
-    {
-        gko::Error(std::string("file"), 12,
-                   std::string("Test for an error class."));
-    }
-
-    // core/base/exception_helpers.hpp
-    {
-        auto test = gko::dim<2>{3};
-        GKO_ASSERT_IS_SQUARE_MATRIX(test);
-    }
-
-    // core/base/executor.hpp
-    {
-        gko::ReferenceExecutor::create();
-    }
-
-    // core/base/math.hpp
-    {
-        using testType = double;
-        static_assert(gko::is_complex<testType>() == false,
-                      "math.hpp not included properly!");
-    }
-
-    // core/base/matrix_data.hpp
-    {
-        gko::matrix_data<>{};
-    }
-
-    // core/base/mtx_io.hpp
-    {
-        static_assert(gko::layout_type::array != gko::layout_type::coordinate,
-                      "mtx_io.hpp not included properly!");
-    }
-
-    // core/base/name_demangling.hpp
-    {
-        auto testVar = 3.0;
-        gko::name_demangling::get_static_type(testVar);
-    }
-
-    // core/base/polymorphic_object.hpp
-    {
-        gko::PolymorphicObject *test;
-        (void)test;  // silence unused variable warning
-    }
-
-    // core/base/range.hpp
-    {
-        gko::span{12};
-    }
-
-    // core/base/range_accessors.hpp
-    {
-        auto testVar = 12;
-        gko::range<gko::accessor::row_major<decltype(testVar), 2>>(&testVar, 1u,
-                                                                   1u, 1u);
-    }
-
-    // core/base/perturbation.hpp
-    {
-        using type1 = int;
-        static_assert(
-            std::is_same<gko::Perturbation<type1>::value_type, type1>::value,
-            "perturbation.hpp not included properly");
-    }
-
-    // core/base/std_extensions.hpp
-    {
-        static_assert(std::is_same<gko::xstd::void_t<double>, void>::value,
-                      "std_extensions.hpp not included properly!");
-    }
-
-    // core/base/types.hpp
-    {
-        static_assert(gko::size_type{12} == 12,
-                      "types.hpp not included properly");
-    }
-
-    // core/base/utils.hpp
-    {
-        gko::null_deleter<double>{};
-    }
-
-    // core/base/version.hpp
-    {
-        gko::version_info::get().header_version;
-    }
-
-    // core/factorization/par_ilu.hpp
-    {
-        gko::factorization::ParIlu<>::build().on(hipExec);
-    }
-
-    // core/log/convergence.hpp
-    {
-        gko::log::Convergence<>::create(hipExec);
-    }
-
-    // core/log/record.hpp
-    {
-        gko::log::executor_data{};
-    }
-
-    // core/log/stream.hpp
-    {
-        gko::log::Stream<>::create(hipExec);
-    }
-
-#if GKO_HAVE_PAPI_SDE
-    // core/log/papi.hpp
-    {
-        gko::log::Papi<>::create(hipExec);
-    }
-#endif  // GKO_HAVE_PAPI_SDE
-
-    // core/matrix/coo.hpp
-    {
-        using Mtx = gko::matrix::Coo<>;
-        Mtx::create(hipExec, gko::dim<2>{2, 2}, 2);
-    }
-
-    // core/matrix/csr.hpp
-    {
-        using Mtx = gko::matrix::Csr<>;
-        Mtx::create(hipExec, gko::dim<2>{2, 2}, 2,
-                    std::make_shared<Mtx::load_balance>(2));
-    }
-
-    // core/matrix/dense.hpp
-    {
-        using Mtx = gko::matrix::Dense<>;
-        Mtx::create(hipExec, gko::dim<2>{2, 2});
-    }
-
-    // core/matrix/ell.hpp
-    {
-        using Mtx = gko::matrix::Ell<>;
-        Mtx::create(hipExec, gko::dim<2>{2, 2}, 2);
-    }
-
-    // core/matrix/hybrid.hpp
-    {
-        using Mtx = gko::matrix::Hybrid<>;
-        Mtx::create(hipExec, gko::dim<2>{2, 2}, 2, 2, 1);
-    }
-
-    // core/matrix/identity.hpp
-    {
-        using Mtx = gko::matrix::Identity<>;
-        Mtx::create(hipExec);
-    }
-
-    // core/matrix/permutation.hpp
-    {
-        using Mtx = gko::matrix::Permutation<>;
-        Mtx::create(hipExec, gko::dim<2>{2, 2});
-    }
-
-    // core/matrix/sellp.hpp
-    {
-        using Mtx = gko::matrix::Sellp<>;
-        Mtx::create(hipExec, gko::dim<2>{2, 2}, 2);
-    }
-
-    // core/matrix/sparsity_csr.hpp
-    {
-        using Mtx = gko::matrix::SparsityCsr<>;
-        Mtx::create(hipExec, gko::dim<2>{2, 2});
-    }
-
-    // core/preconditioner/ilu.hpp
-    {
-        gko::preconditioner::Ilu<>::build().on(hipExec);
-    }
-
-    // core/preconditioner/jacobi.hpp
-    {
-        using Bj = gko::preconditioner::Jacobi<>;
-        Bj::build().with_max_block_size(1u).on(hipExec);
-    }
-
-    // core/solver/bicgstab.hpp
-    {
-        using Solver = gko::solver::Bicgstab<>;
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(hipExec))
-            .on(hipExec);
-    }
-
-    // core/solver/cg.hpp
-    {
-        using Solver = gko::solver::Cg<>;
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(hipExec))
-            .on(hipExec);
-    }
-
-    // core/solver/cgs.hpp
-    {
-        using Solver = gko::solver::Cgs<>;
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(hipExec))
-            .on(hipExec);
-    }
-
-    // core/solver/fcg.hpp
-    {
-        using Solver = gko::solver::Fcg<>;
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(hipExec))
-            .on(hipExec);
-    }
-
-    // core/solver/gmres.hpp
-    {
-        using Solver = gko::solver::Gmres<>;
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(hipExec))
-            .on(hipExec);
-    }
-
-    // core/solver/ir.hpp
-    {
-        using Solver = gko::solver::Ir<>;
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(hipExec))
-            .on(hipExec);
-    }
-
-    // core/solver/lower_trs.hpp
-    {
-        using Solver = gko::solver::LowerTrs<>;
-        Solver::build().on(hipExec);
-    }
-
-    // core/stop/
-    {
-        // iteration.hpp
-        auto iteration =
-            gko::stop::Iteration::build().with_max_iters(1u).on(hipExec);
-
-        // time.hpp
-        auto time = gko::stop::Time::build()
-                        .with_time_limit(std::chrono::milliseconds(10))
-                        .on(hipExec);
-
-        // residual_norm.hpp
-        gko::stop::ResidualNormReduction<>::build()
-            .with_reduction_factor(1e-10)
-            .on(hipExec);
-
-        gko::stop::ResidualNorm<>::build()
-            .with_reduction_factor(1e-10)
-            .with_baseline(gko::stop::mode::absolute)
-            .on(hipExec);
-
-        gko::stop::ImplicitResidualNorm<>::build()
-            .with_reduction_factor(1e-10)
-            .with_baseline(gko::stop::mode::absolute)
-            .on(hipExec);
-
-        gko::stop::RelativeResidualNorm<>::build().with_tolerance(1e-10).on(
-            hipExec);
-
-        gko::stop::AbsoluteResidualNorm<>::build().with_tolerance(1e-10).on(
-            hipExec);
-
-        // stopping_status.hpp
-        gko::stopping_status{};
-
-        // combined.hpp
-        auto combined =
-            gko::stop::Combined::build()
-                .with_criteria(std::move(time), std::move(iteration))
-                .on(hipExec);
-    }
-
-    std::cout
-        << "test_install_hip: the Ginkgo installation was correctly detected "
-           "and is complete."
-        << std::endl;
-
-    return 0;
-}
diff --git a/test_install/test_install_cuda.cu b/test_install/test_install_cuda.cu
deleted file mode 100644
index 1c404236110..00000000000
--- a/test_install/test_install_cuda.cu
+++ /dev/null
@@ -1,388 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/ginkgo.hpp>
-
-
-#include <chrono>
-#include <iostream>
-#include <map>
-#include <string>
-#include <type_traits>
-#include <utility>
-#include <vector>
-
-
-// core/base/polymorphic_object.hpp
-class PolymorphicObjectTest : public gko::PolymorphicObject {};
-
-
-int main(int, char **)
-{
-    auto refExec = gko::ReferenceExecutor::create();
-    auto cudaExec = gko::CudaExecutor::create(0, refExec);
-    // core/base/abstract_factory.hpp
-    {
-        using type1 = int;
-        using type2 = double;
-        static_assert(
-            std::is_same<
-                gko::AbstractFactory<type1, type2>::abstract_product_type,
-                type1>::value,
-            "abstract_factory.hpp not included properly!");
-    }
-
-    // core/base/array.hpp
-    {
-        using type1 = int;
-        using ArrayType = gko::Array<type1>;
-        ArrayType{};
-    }
-
-    // core/base/combination.hpp
-    {
-        using type1 = int;
-        static_assert(
-            std::is_same<gko::Combination<type1>::value_type, type1>::value,
-            "combination.hpp not included properly!");
-    }
-
-    // core/base/composition.hpp
-    {
-        using type1 = int;
-        static_assert(
-            std::is_same<gko::Composition<type1>::value_type, type1>::value,
-            "composition.hpp not included properly");
-    }
-
-    // core/base/dim.hpp
-    {
-        using type1 = int;
-        gko::dim<3, type1>{4, 4, 4};
-    }
-
-    // core/base/exception.hpp
-    {
-        gko::Error(std::string("file"), 12,
-                   std::string("Test for an error class."));
-    }
-
-    // core/base/exception_helpers.hpp
-    {
-        auto test = gko::dim<2>{3};
-        GKO_ASSERT_IS_SQUARE_MATRIX(test);
-    }
-
-    // core/base/executor.hpp
-    {
-        gko::ReferenceExecutor::create();
-    }
-
-    // core/base/math.hpp
-    {
-        using testType = double;
-        static_assert(gko::is_complex<testType>() == false,
-                      "math.hpp not included properly!");
-    }
-
-    // core/base/matrix_data.hpp
-    {
-        gko::matrix_data<>{};
-    }
-
-    // core/base/mtx_io.hpp
-    {
-        static_assert(gko::layout_type::array != gko::layout_type::coordinate,
-                      "mtx_io.hpp not included properly!");
-    }
-
-    // core/base/name_demangling.hpp
-    {
-        auto testVar = 3.0;
-        gko::name_demangling::get_static_type(testVar);
-    }
-
-    // core/base/polymorphic_object.hpp
-    {
-        gko::PolymorphicObject *test;
-        (void)test;  // silence unused variable warning
-    }
-
-    // core/base/range.hpp
-    {
-        gko::span{12};
-    }
-
-    // core/base/range_accessors.hpp
-    {
-        auto testVar = 12;
-        gko::range<gko::accessor::row_major<decltype(testVar), 2>>(&testVar, 1u,
-                                                                   1u, 1u);
-    }
-
-    // core/base/perturbation.hpp
-    {
-        using type1 = int;
-        static_assert(
-            std::is_same<gko::Perturbation<type1>::value_type, type1>::value,
-            "perturbation.hpp not included properly");
-    }
-
-    // core/base/std_extensions.hpp
-    {
-        static_assert(std::is_same<gko::xstd::void_t<double>, void>::value,
-                      "std_extensions.hpp not included properly!");
-    }
-
-    // core/base/types.hpp
-    {
-        static_assert(gko::size_type{12} == 12,
-                      "types.hpp not included properly");
-    }
-
-    // core/base/utils.hpp
-    {
-        gko::null_deleter<double>{};
-    }
-
-    // core/base/version.hpp
-    {
-        gko::version_info::get().header_version;
-    }
-
-    // core/factorization/par_ilu.hpp
-    {
-        gko::factorization::ParIlu<>::build().on(cudaExec);
-    }
-
-    // core/log/convergence.hpp
-    {
-        gko::log::Convergence<>::create(cudaExec);
-    }
-
-    // core/log/record.hpp
-    {
-        gko::log::executor_data{};
-    }
-
-    // core/log/stream.hpp
-    {
-        gko::log::Stream<>::create(cudaExec);
-    }
-
-#if GKO_HAVE_PAPI_SDE
-    // core/log/papi.hpp
-    {
-        gko::log::Papi<>::create(cudaExec);
-    }
-#endif  // GKO_HAVE_PAPI_SDE
-
-    // core/matrix/coo.hpp
-    {
-        using Mtx = gko::matrix::Coo<>;
-        Mtx::create(cudaExec, gko::dim<2>{2, 2}, 2);
-    }
-
-    // core/matrix/csr.hpp
-    {
-        using Mtx = gko::matrix::Csr<>;
-        Mtx::create(cudaExec, gko::dim<2>{2, 2}, 2,
-                    std::make_shared<Mtx::load_balance>(2));
-    }
-
-    // core/matrix/dense.hpp
-    {
-        using Mtx = gko::matrix::Dense<>;
-        Mtx::create(cudaExec, gko::dim<2>{2, 2});
-    }
-
-    // core/matrix/ell.hpp
-    {
-        using Mtx = gko::matrix::Ell<>;
-        Mtx::create(cudaExec, gko::dim<2>{2, 2}, 2);
-    }
-
-    // core/matrix/hybrid.hpp
-    {
-        using Mtx = gko::matrix::Hybrid<>;
-        Mtx::create(cudaExec, gko::dim<2>{2, 2}, 2, 2, 1);
-    }
-
-    // core/matrix/identity.hpp
-    {
-        using Mtx = gko::matrix::Identity<>;
-        Mtx::create(cudaExec);
-    }
-
-    // core/matrix/permutation.hpp
-    {
-        using Mtx = gko::matrix::Permutation<>;
-        Mtx::create(cudaExec, gko::dim<2>{2, 2});
-    }
-
-    // core/matrix/sellp.hpp
-    {
-        using Mtx = gko::matrix::Sellp<>;
-        Mtx::create(cudaExec, gko::dim<2>{2, 2}, 2);
-    }
-
-    // core/matrix/sparsity_csr.hpp
-    {
-        using Mtx = gko::matrix::SparsityCsr<>;
-        Mtx::create(cudaExec, gko::dim<2>{2, 2});
-    }
-
-    // core/multigrid/amgx_pgm.hpp
-    {
-        gko::multigrid::AmgxPgm<>::build().on(cudaExec);
-    }
-
-    // core/preconditioner/ilu.hpp
-    {
-        gko::preconditioner::Ilu<>::build().on(cudaExec);
-    }
-
-    // core/preconditioner/jacobi.hpp
-    {
-        using Bj = gko::preconditioner::Jacobi<>;
-        Bj::build().with_max_block_size(1u).on(cudaExec);
-    }
-
-    // core/solver/bicgstab.hpp
-    {
-        using Solver = gko::solver::Bicgstab<>;
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(cudaExec))
-            .on(cudaExec);
-    }
-
-    // core/solver/cg.hpp
-    {
-        using Solver = gko::solver::Cg<>;
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(cudaExec))
-            .on(cudaExec);
-    }
-
-    // core/solver/cgs.hpp
-    {
-        using Solver = gko::solver::Cgs<>;
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(cudaExec))
-            .on(cudaExec);
-    }
-
-    // core/solver/fcg.hpp
-    {
-        using Solver = gko::solver::Fcg<>;
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(cudaExec))
-            .on(cudaExec);
-    }
-
-    // core/solver/gmres.hpp
-    {
-        using Solver = gko::solver::Gmres<>;
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(cudaExec))
-            .on(cudaExec);
-    }
-
-    // core/solver/ir.hpp
-    {
-        using Solver = gko::solver::Ir<>;
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(1u).on(cudaExec))
-            .on(cudaExec);
-    }
-
-    // core/solver/lower_trs.hpp
-    {
-        using Solver = gko::solver::LowerTrs<>;
-        Solver::build().on(cudaExec);
-    }
-
-    // core/stop/
-    {
-        // iteration.hpp
-        auto iteration =
-            gko::stop::Iteration::build().with_max_iters(1u).on(cudaExec);
-
-        // time.hpp
-        auto time = gko::stop::Time::build()
-                        .with_time_limit(std::chrono::milliseconds(10))
-                        .on(cudaExec);
-
-        // residual_norm.hpp
-        gko::stop::ResidualNormReduction<>::build()
-            .with_reduction_factor(1e-10)
-            .on(cudaExec);
-
-        gko::stop::ResidualNorm<>::build()
-            .with_reduction_factor(1e-10)
-            .with_baseline(gko::stop::mode::absolute)
-            .on(cudaExec);
-
-        gko::stop::ImplicitResidualNorm<>::build()
-            .with_reduction_factor(1e-10)
-            .with_baseline(gko::stop::mode::absolute)
-            .on(cudaExec);
-
-        gko::stop::RelativeResidualNorm<>::build().with_tolerance(1e-10).on(
-            cudaExec);
-
-        gko::stop::AbsoluteResidualNorm<>::build().with_tolerance(1e-10).on(
-            cudaExec);
-
-        // stopping_status.hpp
-        gko::stopping_status{};
-
-        // combined.hpp
-        auto combined =
-            gko::stop::Combined::build()
-                .with_criteria(std::move(time), std::move(iteration))
-                .on(cudaExec);
-    }
-
-    std::cout
-        << "test_install_cuda: the Ginkgo installation was correctly detected "
-           "and is complete."
-        << std::endl;
-
-    return 0;
-}

From 0cc284d69df34f779e1252f4e53661cf0fa2cbf5 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Fri, 12 Mar 2021 08:40:04 +0100
Subject: [PATCH 0610/1008] Make test_install actually run some tests.

+ Check most SpMV.
+ Check most Solvers.
---
 CMakeLists.txt                |   1 +
 test_install/CMakeLists.txt   |  12 ++-
 test_install/data/A.mtx       | 114 +++++++++++++++++++++++++
 test_install/data/b.mtx       |  21 +++++
 test_install/data/x0.mtx      |  21 +++++
 test_install/test_install.cpp | 151 +++++++++++++++++++++++++---------
 6 files changed, 280 insertions(+), 40 deletions(-)
 create mode 100644 test_install/data/A.mtx
 create mode 100644 test_install/data/b.mtx
 create mode 100644 test_install/data/x0.mtx

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 044ff9081a1..e173fdd153d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -359,6 +359,7 @@ add_custom_target(test_install
     -DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX}/${GINKGO_INSTALL_CONFIG_DIR}
     -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
     -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
+    WORKING_DIRECTORY ${Ginkgo_BINARY_DIR}/test_install
     # `--config cfg` is ignored by single-configuration generator.
     # `$<CONFIG>` is always be the same as `CMAKE_BUILD_TYPE` in
     # single-configuration generator.
diff --git a/test_install/CMakeLists.txt b/test_install/CMakeLists.txt
index 9b8a258dddf..e5751d72df9 100644
--- a/test_install/CMakeLists.txt
+++ b/test_install/CMakeLists.txt
@@ -7,6 +7,11 @@ find_package(Ginkgo REQUIRED
     # Alternatively, use `cmake -DCMAKE_PREFIX_PATH=<ginkgo_install_dir>` to specify the install directory
     )
 
+# Copy the data to the build directory
+configure_file(data/A.mtx data/A.mtx COPYONLY)
+configure_file(data/b.mtx data/b.mtx COPYONLY)
+configure_file(data/x0.mtx data/x0.mtx COPYONLY)
+
 if(MSVC)
     if(GINKGO_BUILD_SHARED_LIBS)
         ginkgo_switch_to_windows_dynamic("CXX")
@@ -20,8 +25,13 @@ endif()
 include(CheckLanguage)
 check_language(CUDA)
 
+set(HAS_REFERENCE 0)
+if (GINKGO_BUILD_REFERENCE)
+    set(HAS_REFERENCE 1)
+endif()
 add_executable(test_install test_install.cpp)
 target_compile_features(test_install PUBLIC cxx_std_14)
+target_compile_definitions(test_install PRIVATE HAS_REFERENCE=${HAS_REFERENCE})
 target_link_libraries(test_install PRIVATE Ginkgo::ginkgo)
 
 if(GINKGO_BUILD_CUDA)
@@ -33,7 +43,7 @@ if(GINKGO_BUILD_CUDA)
             ginkgo_switch_to_windows_static("CUDA")
         endif()
     endif()
-    set_source_file_properties(test_install.cpp PROPERTIES LANGUAGE CUDA)
+    set_source_files_properties(test_install.cpp PROPERTIES LANGUAGE CUDA)
     add_executable(test_install_cuda test_install.cpp)
     target_compile_definitions(test_install_cuda PRIVATE HAS_CUDA=1)
     target_link_libraries(test_install_cuda PRIVATE Ginkgo::ginkgo)
diff --git a/test_install/data/A.mtx b/test_install/data/A.mtx
new file mode 100644
index 00000000000..c67437da567
--- /dev/null
+++ b/test_install/data/A.mtx
@@ -0,0 +1,114 @@
+%%MatrixMarket matrix coordinate integer symmetric
+%-------------------------------------------------------------------------------
+% UF Sparse Matrix Collection, Tim Davis
+% http://www.cise.ufl.edu/research/sparse/matrices/JGD_Trefethen/Trefethen_20b
+% name: JGD_Trefethen/Trefethen_20b
+% [Diagonal matrices with primes, Nick Trefethen, Oxford Univ.]
+% id: 2203
+% date: 2008
+% author: N. Trefethen
+% ed: J.-G. Dumas
+% fields: name title A id date author ed kind notes
+% kind: combinatorial problem
+%-------------------------------------------------------------------------------
+% notes:
+% Diagonal matrices with primes, Nick Trefethen, Oxford Univ.          
+% From Jean-Guillaume Dumas' Sparse Integer Matrix Collection,         
+% http://ljk.imag.fr/membres/Jean-Guillaume.Dumas/simc.html            
+%                                                                      
+% Problem 7 of the Hundred-dollar, Hundred-digit Challenge Problems,   
+% SIAM News, vol 35, no. 1.                                            
+%                                                                      
+% 7. Let A be the 20,000 x 20,000 matrix whose entries are zero        
+% everywhere except for the primes 2, 3, 5, 7, . . . , 224737 along the
+% main diagonal and the number 1 in all the positions A(i,j) with      
+% |i-j| = 1,2,4,8, . . . ,16384.  What is the (1,1) entry of inv(A)?   
+%                                                                      
+% http://www.siam.org/news/news.php?id=388                             
+%                                                                      
+% Filename in JGD collection: Trefethen/trefethen_20__19_minor.sms     
+%-------------------------------------------------------------------------------
+19 19 83
+1 1 3
+2 1 1
+3 1 1
+5 1 1
+9 1 1
+17 1 1
+2 2 5
+3 2 1
+4 2 1
+6 2 1
+10 2 1
+18 2 1
+3 3 7
+4 3 1
+5 3 1
+7 3 1
+11 3 1
+19 3 1
+4 4 11
+5 4 1
+6 4 1
+8 4 1
+12 4 1
+5 5 13
+6 5 1
+7 5 1
+9 5 1
+13 5 1
+6 6 17
+7 6 1
+8 6 1
+10 6 1
+14 6 1
+7 7 19
+8 7 1
+9 7 1
+11 7 1
+15 7 1
+8 8 23
+9 8 1
+10 8 1
+12 8 1
+16 8 1
+9 9 29
+10 9 1
+11 9 1
+13 9 1
+17 9 1
+10 10 31
+11 10 1
+12 10 1
+14 10 1
+18 10 1
+11 11 37
+12 11 1
+13 11 1
+15 11 1
+19 11 1
+12 12 41
+13 12 1
+14 12 1
+16 12 1
+13 13 43
+14 13 1
+15 13 1
+17 13 1
+14 14 47
+15 14 1
+16 14 1
+18 14 1
+15 15 53
+16 15 1
+17 15 1
+19 15 1
+16 16 59
+17 16 1
+18 16 1
+17 17 61
+18 17 1
+19 17 1
+18 18 67
+19 18 1
+19 19 71
diff --git a/test_install/data/b.mtx b/test_install/data/b.mtx
new file mode 100644
index 00000000000..05d92ecc6f7
--- /dev/null
+++ b/test_install/data/b.mtx
@@ -0,0 +1,21 @@
+%%MatrixMarket matrix array real general
+19 1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
diff --git a/test_install/data/x0.mtx b/test_install/data/x0.mtx
new file mode 100644
index 00000000000..91d470cdbcd
--- /dev/null
+++ b/test_install/data/x0.mtx
@@ -0,0 +1,21 @@
+%%MatrixMarket matrix array real general
+19 1
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
diff --git a/test_install/test_install.cpp b/test_install/test_install.cpp
index b5bf7864495..3e15e295abd 100644
--- a/test_install/test_install.cpp
+++ b/test_install/test_install.cpp
@@ -34,6 +34,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <chrono>
+#include <cmath>
+#include <fstream>
 #include <iostream>
 #include <map>
 #include <string>
@@ -42,6 +44,94 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <vector>
 
 
+void assert_similar_matrices(const gko::matrix::Dense<> *m1,
+                             const gko::matrix::Dense<> *m2, double prec)
+{
+    assert(m1->get_size()[0] == m2->get_size()[0]);
+    assert(m1->get_size()[1] == m2->get_size()[1]);
+    for (gko::size_type i = 0; i < m1->get_size()[0]; ++i) {
+        for (gko::size_type j = 0; j < m2->get_size()[1]; ++j) {
+            assert(std::abs(m1->at(i, j) - m2->at(i, j)) < prec);
+        }
+    }
+}
+
+
+template <typename Mtx>
+void check_spmv(std::shared_ptr<gko::Executor> exec,
+                gko::matrix_data<double> &A_raw, gko::matrix::Dense<> *b,
+                gko::matrix::Dense<> *x)
+{
+    auto test = Mtx::create(exec);
+#if HAS_REFERENCE
+    auto x_clone = gko::clone(x);
+    test->read(A_raw);
+    test->apply(b, gko::lend(x_clone));
+
+#if defined(HAS_HIP) || defined(HAS_CUDA)
+    auto test_ref = Mtx::create(exec->get_master());
+    auto x_ref = gko::clone(exec->get_master(), x);
+    test_ref->read(A_raw);
+    test_ref->apply(b, gko::lend(x_ref));
+
+    auto x_clone_ref = gko::clone(exec->get_master(), gko::lend(x_clone));
+    assert_similar_matrices(gko::lend(x_clone_ref), gko::lend(x_ref), 1e-14);
+#endif  // defined(HAS_HIP) || defined(HAS_CUDA)
+#endif  // HAS_REFERENCE
+}
+
+
+template <typename Solver>
+void check_solver(std::shared_ptr<gko::Executor> exec,
+                  gko::matrix_data<double> &A_raw, gko::matrix::Dense<> *b,
+                  gko::matrix::Dense<> *x)
+{
+    using Mtx = gko::matrix::Csr<>;
+    auto A =
+        gko::share(Mtx::create(exec, std::make_shared<Mtx::load_balance>()));
+#if HAS_REFERENCE
+    A->read(A_raw);
+#endif  // HAS_REFERENCE
+
+    auto num_iters = 20u;
+    double reduction_factor = 1e-7;
+    auto solver_gen =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(num_iters).on(
+                    exec),
+                gko::stop::ResidualNorm<>::build()
+                    .with_reduction_factor(reduction_factor)
+                    .on(exec))
+            .on(exec);
+#if HAS_REFERENCE
+    auto x_clone = gko::clone(x);
+    solver_gen->generate(A)->apply(b, gko::lend(x_clone));
+
+#if defined(HAS_HIP) || defined(HAS_CUDA)
+    auto A_ref =
+        gko::share(Mtx::create(exec, std::make_shared<Mtx::load_balance>()));
+    A_ref->read(A_raw);
+    auto refExec = exec->get_master();
+    auto solver_gen_ref =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(num_iters).on(
+                    exec),
+                gko::stop::ResidualNorm<>::build()
+                    .with_reduction_factor(reduction_factor)
+                    .on(refExec))
+            .on(refExec);
+    auto x_ref = gko::clone(exec->get_master(), x);
+    solver_gen->generate(A_ref)->apply(b, gko::lend(x_ref));
+
+    auto x_clone_ref = gko::clone(exec->get_master(), gko::lend(x_clone));
+    assert_similar_matrices(gko::lend(x_clone_ref), gko::lend(x_ref), 1e-12);
+#endif  // defined(HAS_HIP) || defined(HAS_CUDA)
+#endif  // HAS_REFERENCE
+}
+
+
 // core/base/polymorphic_object.hpp
 class PolymorphicObjectTest : public gko::PolymorphicObject {};
 
@@ -55,6 +145,13 @@ int main(int, char **)
 #else
     auto exec = gko::ReferenceExecutor::create();
 #endif
+
+    using vec = gko::matrix::Dense<>;
+    auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
+    auto x = gko::read<vec>(std::ifstream("data/x0.mtx"), exec);
+    std::ifstream A_file("data/A.mtx");
+    auto A_raw = gko::read_raw<double>(A_file);
+
     // core/base/abstract_factory.hpp
     {
         using type1 = int;
@@ -211,32 +308,31 @@ int main(int, char **)
     // core/matrix/coo.hpp
     {
         using Mtx = gko::matrix::Coo<>;
-        auto test = Mtx::create(exec, gko::dim<2>{2, 2}, 2);
+        check_spmv<Mtx>(exec, A_raw, gko::lend(b), gko::lend(x));
     }
 
     // core/matrix/csr.hpp
     {
         using Mtx = gko::matrix::Csr<>;
-        auto test = Mtx::create(exec, gko::dim<2>{2, 2}, 2,
-                                std::make_shared<Mtx::load_balance>(2));
+        auto test = Mtx::create(exec, std::make_shared<Mtx::load_balance>());
     }
 
     // core/matrix/dense.hpp
     {
         using Mtx = gko::matrix::Dense<>;
-        auto test = Mtx::create(exec, gko::dim<2>{2, 2});
+        check_spmv<Mtx>(exec, A_raw, gko::lend(b), gko::lend(x));
     }
 
     // core/matrix/ell.hpp
     {
         using Mtx = gko::matrix::Ell<>;
-        auto test = Mtx::create(exec, gko::dim<2>{2, 2}, 2);
+        check_spmv<Mtx>(exec, A_raw, gko::lend(b), gko::lend(x));
     }
 
     // core/matrix/hybrid.hpp
     {
         using Mtx = gko::matrix::Hybrid<>;
-        auto test = Mtx::create(exec, gko::dim<2>{2, 2}, 2, 2, 1);
+        check_spmv<Mtx>(exec, A_raw, gko::lend(b), gko::lend(x));
     }
 
     // core/matrix/identity.hpp
@@ -254,7 +350,7 @@ int main(int, char **)
     // core/matrix/sellp.hpp
     {
         using Mtx = gko::matrix::Sellp<>;
-        auto test = Mtx::create(exec, gko::dim<2>{2, 2}, 2);
+        check_spmv<Mtx>(exec, A_raw, gko::lend(b), gko::lend(x));
     }
 
     // core/matrix/sparsity_csr.hpp
@@ -265,7 +361,7 @@ int main(int, char **)
 
     // core/multigrid/amgx_pgm.hpp
     {
-        auto test = gko::multigrid::AmgxPgm<>::build().on(refExec);
+        auto test = gko::multigrid::AmgxPgm<>::build().on(exec);
     }
 
     // core/preconditioner/ilu.hpp
@@ -288,61 +384,37 @@ int main(int, char **)
     // core/solver/bicgstab.hpp
     {
         using Solver = gko::solver::Bicgstab<>;
-        auto test =
-            Solver::build()
-                .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(1u).on(exec))
-                .on(exec);
+        check_solver<Solver>(exec, A_raw, gko::lend(b), gko::lend(x));
     }
 
     // core/solver/cb_gmres.hpp
     {
         using Solver = gko::solver::CbGmres<>;
-        auto test =
-            Solver::build()
-                .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(1u).on(exec))
-                .on(exec);
+        check_solver<Solver>(exec, A_raw, gko::lend(b), gko::lend(x));
     }
 
     // core/solver/cg.hpp
     {
         using Solver = gko::solver::Cg<>;
-        auto test =
-            Solver::build()
-                .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(1u).on(exec))
-                .on(exec);
+        check_solver<Solver>(exec, A_raw, gko::lend(b), gko::lend(x));
     }
 
     // core/solver/cgs.hpp
     {
         using Solver = gko::solver::Cgs<>;
-        auto test =
-            Solver::build()
-                .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(1u).on(exec))
-                .on(exec);
+        check_solver<Solver>(exec, A_raw, gko::lend(b), gko::lend(x));
     }
 
     // core/solver/fcg.hpp
     {
         using Solver = gko::solver::Fcg<>;
-        auto test =
-            Solver::build()
-                .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(1u).on(exec))
-                .on(exec);
+        check_solver<Solver>(exec, A_raw, gko::lend(b), gko::lend(x));
     }
 
     // core/solver/gmres.hpp
     {
         using Solver = gko::solver::Gmres<>;
-        auto test =
-            Solver::build()
-                .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(1u).on(exec))
-                .on(exec);
+        check_solver<Solver>(exec, A_raw, gko::lend(b), gko::lend(x));
     }
 
     // core/solver/ir.hpp
@@ -411,7 +483,8 @@ int main(int, char **)
 #else
     auto extra_info = "";
 #endif
-    std::cout << "test_install"<< extra_info << ": the Ginkgo installation was correctly detected "
+    std::cout << "test_install" << extra_info
+              << ": the Ginkgo installation was correctly detected "
                  "and is complete."
               << std::endl;
 

From 061899a141cf23c01282d665be98c767f9adadeb Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Thu, 25 Mar 2021 20:06:30 +0100
Subject: [PATCH 0611/1008] Add possibility to install the smoke tests.

Caveat: the test installs need to be ran from the binary directory.
---
 .gitlab-ci.yml              |  3 +++
 test_install/CMakeLists.txt | 25 ++++++++++++++++++++++++-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9e3a954ffdb..3b2d1f50ffe 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -108,6 +108,9 @@ stages:
         (( $(ctest -N | tail -1 | sed 's/Total Tests: //') != 0 )) || exit 1
     - ctest -V
     - ninja test_install
+    - pushd test_install
+    - ninja install
+    - popd
     - |
       if [ "${RUN_EXAMPLES}" == "ON" ]; then
          export EX_ARG="reference"
diff --git a/test_install/CMakeLists.txt b/test_install/CMakeLists.txt
index e5751d72df9..e8b2936859d 100644
--- a/test_install/CMakeLists.txt
+++ b/test_install/CMakeLists.txt
@@ -45,6 +45,8 @@ if(GINKGO_BUILD_CUDA)
     endif()
     set_source_files_properties(test_install.cpp PROPERTIES LANGUAGE CUDA)
     add_executable(test_install_cuda test_install.cpp)
+    target_compile_options(test_install_cuda
+        PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_ARCH_FLAGS}>")
     target_compile_definitions(test_install_cuda PRIVATE HAS_CUDA=1)
     target_link_libraries(test_install_cuda PRIVATE Ginkgo::ginkgo)
 endif()
@@ -58,10 +60,13 @@ if(GINKGO_BUILD_HIP)
         set (GINKGO_PIC_OPTION "-fPIC")
         set (GINKGO_CUDA_PIC_OPTION "-Xcompiler '-fPIC'")
     endif()
+    if (CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
+        set(TESTINSTALL_CUDA_ARCH_FLAGS "${GINKGO_CUDA_ARCH_FLAGS}")
+    endif()
     hip_add_executable(test_install_hip test_install.cpp
         HIPCC_OPTIONS "-std=c++14"
         CLANG_OPTIONS "${GINKGO_PIC_OPTION}"
-        NVCC_OPTIONS "${GINKGO_CUDA_PIC_OPTION}")
+        NVCC_OPTIONS "${GINKGO_CUDA_PIC_OPTION}" "${TESTINSTALL_CUDA_ARCH_FLAGS}")
 
     target_link_libraries(test_install_hip PRIVATE Ginkgo::ginkgo)
     target_compile_definitions(test_install_hip PRIVATE HAS_HIP=1)
@@ -72,3 +77,21 @@ if(GINKGO_BUILD_HIP)
     # Instead of using CXX, it's possible to instead use
     # target_link_libraries(test_install_hip PRIVATE ${GINKGO_INSTALL_RPATH_FOR_HIP})
 endif()
+
+# Installation step
+set(TESTINSTALL_INSTALL_DIR "${GINKGO_INSTALL_PREFIX}/smoke_tests")
+set(TESTINSTALL_INSTALL_DATADIR "${TESTINSTALL_INSTALL_DIR}/data")
+set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON)
+
+set_target_properties(test_install PROPERTIES INSTALL_RPATH "${GINKGO_INSTALL_LIBRARY_DIR}")
+install(TARGETS test_install RUNTIME DESTINATION ${TESTINSTALL_INSTALL_DIR})
+if(GINKGO_BUILD_CUDA)
+    set_target_properties(test_install_cuda PROPERTIES INSTALL_RPATH "${GINKGO_INSTALL_LIBRARY_DIR}")
+    install(TARGETS test_install_cuda RUNTIME DESTINATION ${TESTINSTALL_INSTALL_DIR})
+endif()
+if(GINKGO_BUILD_HIP)
+    set_target_properties(test_install_hip PROPERTIES INSTALL_RPATH "${GINKGO_INSTALL_LIBRARY_DIR}")
+    install(TARGETS test_install_hip RUNTIME DESTINATION ${TESTINSTALL_INSTALL_DIR})
+endif()
+install(DIRECTORY "${TestInstall_BINARY_DIR}/data/"
+    DESTINATION "${TESTINSTALL_INSTALL_DATADIR}")

From 6a0b2b2db68a3dfb8e3fd06438c903be7307b5bf Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Fri, 26 Mar 2021 15:10:30 +0100
Subject: [PATCH 0612/1008] Review updates.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

+ Ensure proper use of `HAS_REFERENCE`.
+ Move to `exec_ref` in all cases
+ Improve code readability and add a few documentation.
+ Improve robustness of the CMakeLists.txt.
+ Improve quality of `test_install.cpp` code.

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 test_exportbuild/CMakeLists.txt |  2 ++
 test_install/CMakeLists.txt     |  6 ++--
 test_install/test_install.cpp   | 55 +++++++++++++++++++++------------
 3 files changed, 41 insertions(+), 22 deletions(-)

diff --git a/test_exportbuild/CMakeLists.txt b/test_exportbuild/CMakeLists.txt
index fd8ec8afc48..79b538f437c 100644
--- a/test_exportbuild/CMakeLists.txt
+++ b/test_exportbuild/CMakeLists.txt
@@ -13,6 +13,8 @@ if(MSVC)
     endif()
 endif()
 
+# Here, we use test install without any data. We instantiate the
+# interface only.
 add_executable(test_exportbuild ../test_install/test_install.cpp)
 target_compile_features(test_exportbuild PUBLIC cxx_std_14)
 target_link_libraries(test_exportbuild PRIVATE Ginkgo::ginkgo)
diff --git a/test_install/CMakeLists.txt b/test_install/CMakeLists.txt
index e8b2936859d..4c5fc9eea58 100644
--- a/test_install/CMakeLists.txt
+++ b/test_install/CMakeLists.txt
@@ -43,11 +43,12 @@ if(GINKGO_BUILD_CUDA)
             ginkgo_switch_to_windows_static("CUDA")
         endif()
     endif()
-    set_source_files_properties(test_install.cpp PROPERTIES LANGUAGE CUDA)
-    add_executable(test_install_cuda test_install.cpp)
+    configure_file(test_install.cpp test_install.cu COPYONLY)
+    add_executable(test_install_cuda ${CMAKE_CURRENT_BINARY_DIR}/test_install.cu)
     target_compile_options(test_install_cuda
         PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_ARCH_FLAGS}>")
     target_compile_definitions(test_install_cuda PRIVATE HAS_CUDA=1)
+    target_compile_definitions(test_install_cuda PRIVATE HAS_REFERENCE=${HAS_REFERENCE})
     target_link_libraries(test_install_cuda PRIVATE Ginkgo::ginkgo)
 endif()
 
@@ -70,6 +71,7 @@ if(GINKGO_BUILD_HIP)
 
     target_link_libraries(test_install_hip PRIVATE Ginkgo::ginkgo)
     target_compile_definitions(test_install_hip PRIVATE HAS_HIP=1)
+    target_compile_definitions(test_install_hip PRIVATE HAS_REFERENCE=${HAS_REFERENCE})
 
     # If we always link with CXX there is no RPATH issue
     set_target_properties(test_install_hip PROPERTIES LINKER_LANGUAGE CXX)
diff --git a/test_install/test_install.cpp b/test_install/test_install.cpp
index 3e15e295abd..fb9f479c06a 100644
--- a/test_install/test_install.cpp
+++ b/test_install/test_install.cpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <fstream>
 #include <iostream>
 #include <map>
+#include <memory>
 #include <string>
 #include <type_traits>
 #include <utility>
@@ -59,22 +60,27 @@ void assert_similar_matrices(const gko::matrix::Dense<> *m1,
 
 template <typename Mtx>
 void check_spmv(std::shared_ptr<gko::Executor> exec,
-                gko::matrix_data<double> &A_raw, gko::matrix::Dense<> *b,
-                gko::matrix::Dense<> *x)
+                const gko::matrix_data<double> &A_raw,
+                const gko::matrix::Dense<> *b, gko::matrix::Dense<> *x)
 {
     auto test = Mtx::create(exec);
 #if HAS_REFERENCE
     auto x_clone = gko::clone(x);
     test->read(A_raw);
     test->apply(b, gko::lend(x_clone));
+    // x_clone has the device result if using HIP or CUDA, otherwise it is
+    // reference only
 
 #if defined(HAS_HIP) || defined(HAS_CUDA)
-    auto test_ref = Mtx::create(exec->get_master());
-    auto x_ref = gko::clone(exec->get_master(), x);
+    // If we are on a device, we need to run a reference test to compare against
+    auto exec_ref = exec->get_master();
+    auto test_ref = Mtx::create(exec_ref);
+    auto x_ref = gko::clone(exec_ref, x);
     test_ref->read(A_raw);
     test_ref->apply(b, gko::lend(x_ref));
 
-    auto x_clone_ref = gko::clone(exec->get_master(), gko::lend(x_clone));
+    // Actually check that `x_clone` is similar to `x_ref`
+    auto x_clone_ref = gko::clone(exec_ref, gko::lend(x_clone));
     assert_similar_matrices(gko::lend(x_clone_ref), gko::lend(x_ref), 1e-14);
 #endif  // defined(HAS_HIP) || defined(HAS_CUDA)
 #endif  // HAS_REFERENCE
@@ -83,15 +89,12 @@ void check_spmv(std::shared_ptr<gko::Executor> exec,
 
 template <typename Solver>
 void check_solver(std::shared_ptr<gko::Executor> exec,
-                  gko::matrix_data<double> &A_raw, gko::matrix::Dense<> *b,
-                  gko::matrix::Dense<> *x)
+                  const gko::matrix_data<double> &A_raw,
+                  const gko::matrix::Dense<> *b, gko::matrix::Dense<> *x)
 {
     using Mtx = gko::matrix::Csr<>;
     auto A =
         gko::share(Mtx::create(exec, std::make_shared<Mtx::load_balance>()));
-#if HAS_REFERENCE
-    A->read(A_raw);
-#endif  // HAS_REFERENCE
 
     auto num_iters = 20u;
     double reduction_factor = 1e-7;
@@ -105,27 +108,32 @@ void check_solver(std::shared_ptr<gko::Executor> exec,
                     .on(exec))
             .on(exec);
 #if HAS_REFERENCE
+    A->read(A_raw);
     auto x_clone = gko::clone(x);
     solver_gen->generate(A)->apply(b, gko::lend(x_clone));
+    // x_clone has the device result if using HIP or CUDA, otherwise it is
+    // reference only
 
 #if defined(HAS_HIP) || defined(HAS_CUDA)
-    auto A_ref =
-        gko::share(Mtx::create(exec, std::make_shared<Mtx::load_balance>()));
+    // If we are on a device, we need to run a reference test to compare against
+    auto exec_ref = exec->get_master();
+    auto A_ref = gko::share(
+        Mtx::create(exec_ref, std::make_shared<Mtx::load_balance>()));
     A_ref->read(A_raw);
-    auto refExec = exec->get_master();
     auto solver_gen_ref =
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(num_iters).on(
-                    exec),
+                    exec_ref),
                 gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(reduction_factor)
-                    .on(refExec))
-            .on(refExec);
-    auto x_ref = gko::clone(exec->get_master(), x);
+                    .on(exec_ref))
+            .on(exec_ref);
+    auto x_ref = gko::clone(exec_ref, x);
     solver_gen->generate(A_ref)->apply(b, gko::lend(x_ref));
 
-    auto x_clone_ref = gko::clone(exec->get_master(), gko::lend(x_clone));
+    // Actually check that `x_clone` is similar to `x_ref`
+    auto x_clone_ref = gko::clone(exec_ref, gko::lend(x_clone));
     assert_similar_matrices(gko::lend(x_clone_ref), gko::lend(x_ref), 1e-12);
 #endif  // defined(HAS_HIP) || defined(HAS_CUDA)
 #endif  // HAS_REFERENCE
@@ -136,7 +144,7 @@ void check_solver(std::shared_ptr<gko::Executor> exec,
 class PolymorphicObjectTest : public gko::PolymorphicObject {};
 
 
-int main(int, char **)
+int main()
 {
 #if defined(HAS_CUDA)
     auto exec = gko::CudaExecutor::create(0, gko::ReferenceExecutor::create());
@@ -147,10 +155,17 @@ int main(int, char **)
 #endif
 
     using vec = gko::matrix::Dense<>;
+#if HAS_REFERENCE
     auto b = gko::read<vec>(std::ifstream("data/b.mtx"), exec);
     auto x = gko::read<vec>(std::ifstream("data/x0.mtx"), exec);
     std::ifstream A_file("data/A.mtx");
     auto A_raw = gko::read_raw<double>(A_file);
+#else
+    // Instantiate dummy data, they will be unused
+    auto b = vec::create(exec);
+    auto x = vec::create(exec);
+    gko::matrix_data<double> A_raw{};
+#endif
 
     // core/base/abstract_factory.hpp
     {
@@ -481,7 +496,7 @@ int main(int, char **)
 #elif defined(HAS_HIP)
     auto extra_info = "(HIP)";
 #else
-    auto extra_info = "";
+    auto extra_info = "(REFERENCE)";
 #endif
     std::cout << "test_install" << extra_info
               << ": the Ginkgo installation was correctly detected "

From 1458174010671b0466241dfd95d60b2f0539f6ff Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Sun, 11 Apr 2021 09:27:05 +0000
Subject: [PATCH 0613/1008] Format files

Co-authored-by: Terry Cojean <tcojean@users.noreply.github.com>
---
 include/ginkgo/core/base/executor.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index e69cf883e13..baeea6b7d63 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -342,7 +342,7 @@ private:                                                                     \
             ::gko::syn::as_list<::gko::syn::range<0, sizeof...(Args)>>;       \
                                                                               \
     public:                                                                   \
-        explicit _name##_operation(Args &&...args)                            \
+        explicit _name##_operation(Args &&... args)                           \
             : data(std::forward<Args>(args)...)                               \
         {}                                                                    \
                                                                               \
@@ -368,7 +368,7 @@ private:                                                                     \
     };                                                                        \
                                                                               \
     template <typename... Args>                                               \
-    static _name##_operation<Args...> make_##_name(Args &&...args)            \
+    static _name##_operation<Args...> make_##_name(Args &&... args)           \
     {                                                                         \
         return _name##_operation<Args...>(std::forward<Args>(args)...);       \
     }                                                                         \

From 789cf80a762956bd8d6f792893cc6c8db8dd545b Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sun, 11 Apr 2021 15:09:06 +0200
Subject: [PATCH 0614/1008] optionally reduce test input sizes for a few tests

---
 .gitlab-ci.yml                                | 16 +++++++++++
 CMakeLists.txt                                |  1 +
 cmake/create_test.cmake                       | 28 +++++++++++++++----
 cuda/test/factorization/par_ilut_kernels.cpp  |  4 +++
 cuda/test/matrix/csr_kernels.cpp              | 17 +++++++----
 cuda/test/matrix/diagonal_kernels.cpp         |  9 +++++-
 cuda/test/solver/cb_gmres_kernels.cpp         |  4 +++
 cuda/test/solver/gmres_kernels.cpp            |  4 +++
 .../factorization/par_ilut_kernels.hip.cpp    |  4 +++
 hip/test/matrix/csr_kernels.hip.cpp           | 17 +++++++----
 hip/test/matrix/diagonal_kernels.hip.cpp      |  9 +++++-
 hip/test/solver/cb_gmres_kernels.cpp          |  4 +++
 hip/test/solver/gmres_kernels.cpp             |  4 +++
 omp/test/factorization/par_ict_kernels.cpp    |  4 +++
 omp/test/factorization/par_ilut_kernels.cpp   |  4 +++
 omp/test/matrix/csr_kernels.cpp               | 17 +++++++----
 omp/test/matrix/diagonal_kernels.cpp          |  9 +++++-
 omp/test/solver/cb_gmres_kernels.cpp          |  4 +++
 omp/test/solver/gmres_kernels.cpp             |  4 +++
 19 files changed, 140 insertions(+), 23 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3b2d1f50ffe..3036303d464 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,6 +23,7 @@ stages:
   BUILD_CUDA: "OFF"
   BUILD_HIP: "OFF"
   BUILD_HWLOC: "ON"
+  FAST_TESTS: "OFF"
   RUN_EXAMPLES: "OFF"
   CONFIG_LOG: "ON"
   CXX_FLAGS: ""
@@ -68,6 +69,7 @@ stages:
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
         -DGINKGO_BUILD_HWLOC=${BUILD_HWLOC}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
+        -DGINKGO_FAST_TESTS=${FAST_TESTS}
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
         -DGINKGO_EXPORT_BUILD_DIR=${EXPORT_BUILD_DIR}
@@ -100,6 +102,7 @@ stages:
         -DGINKGO_BUILD_HIP=${BUILD_HIP}
         -DGINKGO_BUILD_HWLOC=${BUILD_HWLOC}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
+        -DGINKGO_FAST_TESTS=${FAST_TESTS}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_EXPORT_BUILD_DIR=${EXPORT_BUILD_DIR}
@@ -172,6 +175,7 @@ build/cuda90/gcc/all/debug/shared:
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_TYPE: "Debug"
+    FAST_TESTS: "ON"
     RUN_EXAMPLES: "ON"
     CUDA_ARCH: 35
   only:
@@ -211,6 +215,7 @@ build/cuda91/gcc/all/debug/static:
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_TYPE: "Debug"
+    FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
   only:
@@ -295,6 +300,7 @@ build/cuda100/gcc/all/debug/shared:
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     BUILD_TYPE: "Debug"
+    FAST_TESTS: "ON"
     CUDA_ARCH: 35
   only:
     variables:
@@ -354,6 +360,7 @@ build/cuda101/gcc/all/debug/shared:
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     BUILD_TYPE: "Debug"
+    FAST_TESTS: "ON"
     CUDA_ARCH: 35
   only:
     variables:
@@ -415,6 +422,7 @@ build/clang-cuda101/clang/cuda/debug/static:
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_TYPE: "Debug"
+    FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
   only:
@@ -437,6 +445,7 @@ build/cuda102/gcc/all/debug/shared:
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     BUILD_TYPE: "Debug"
+    FAST_TESTS: "ON"
     BUILD_HWLOC: "OFF"
     CUDA_ARCH: 35
   only:
@@ -479,6 +488,7 @@ build/cuda102/intel/cuda/debug/static:
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_TYPE: "Debug"
+    FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
   only:
@@ -498,6 +508,7 @@ build/cuda110/gcc/cuda/debug/shared:
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_TYPE: "Debug"
+    FAST_TESTS: "ON"
     CUDA_ARCH: 35
   only:
     variables:
@@ -537,6 +548,7 @@ build/cuda110/intel/cuda/debug/static:
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_TYPE: "Debug"
+    FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
   only:
@@ -557,6 +569,7 @@ build/amd/gcc/hip/debug/shared:
     BUILD_HIP: "ON"
     RUN_EXAMPLES: "ON"
     BUILD_TYPE: "Debug"
+    FAST_TESTS: "ON"
   only:
     variables:
       - $RUN_CI_TAG
@@ -592,6 +605,7 @@ build/nocuda/gcc/core/debug/static:
     <<: *default_variables
     BUILD_REFERENCE: "OFF"
     BUILD_TYPE: "Debug"
+    FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     BUILD_HWLOC: "OFF"
   only:
@@ -626,6 +640,7 @@ build/nocuda/intel/core/debug/shared:
     CXX_COMPILER: "icpc"
     BUILD_REFERENCE: "OFF"
     BUILD_TYPE: "Debug"
+    FAST_TESTS: "ON"
   only:
     variables:
       - $RUN_CI_TAG
@@ -657,6 +672,7 @@ build/nocuda/clang/omp/debug/static:
     CXX_COMPILER: "clang++"
     BUILD_OMP: "ON"
     BUILD_TYPE: "Debug"
+    FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
   only:
     variables:
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e173fdd153d..c6d8cb67c44 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,6 +32,7 @@ option(GINKGO_BUILD_DPCPP
 option(GINKGO_BUILD_CUDA "Compile kernels for NVIDIA GPUs" ${GINKGO_HAS_CUDA})
 option(GINKGO_BUILD_HIP "Compile kernels for AMD or NVIDIA GPUs" ${GINKGO_HAS_HIP})
 option(GINKGO_BUILD_DOC "Generate documentation" OFF)
+option(GINKGO_FAST_TESTS "Reduces the input size for a few tests known to be time-intensive" OFF)
 option(GINKGO_SKIP_DEPENDENCY_UPDATE
     "Do not update dependencies each time the project is rebuilt" ON)
 option(GINKGO_EXPORT_BUILD_DIR
diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index 34e9d0ee32a..fcec9d689e6 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -10,6 +10,9 @@ function(ginkgo_create_test test_name)
         )
     set_target_properties(${TEST_TARGET_NAME} PROPERTIES
         OUTPUT_NAME ${test_name})
+    if (GINKGO_FAST_TESTS)
+        target_compile_definitions(${TEST_TARGET_NAME} PRIVATE GINKGO_FAST_TESTS)
+    endif()
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
@@ -30,6 +33,9 @@ function(ginkgo_create_dpcpp_test test_name)
         )
     set_target_properties(${TEST_TARGET_NAME} PROPERTIES
         OUTPUT_NAME ${test_name})
+    if (GINKGO_FAST_TESTS)
+        target_compile_definitions(${TEST_TARGET_NAME} PRIVATE GINKGO_FAST_TESTS)
+    endif()
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
@@ -51,6 +57,9 @@ function(ginkgo_create_thread_test test_name)
         )
     set_target_properties(${TEST_TARGET_NAME} PROPERTIES
         OUTPUT_NAME ${test_name})
+    if (GINKGO_FAST_TESTS)
+        target_compile_definitions(${TEST_TARGET_NAME} PRIVATE GINKGO_FAST_TESTS)
+    endif()
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
@@ -72,6 +81,9 @@ function(ginkgo_create_test_cpp_cuda_header test_name)
         )
     set_target_properties(${TEST_TARGET_NAME} PROPERTIES
         OUTPUT_NAME ${test_name})
+    if (GINKGO_FAST_TESTS)
+        target_compile_definitions(${TEST_TARGET_NAME} PRIVATE GINKGO_FAST_TESTS)
+    endif()
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
@@ -93,7 +105,9 @@ function(ginkgo_create_cuda_test test_name)
         PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_ARCH_FLAGS}>")
     set_target_properties(${TEST_TARGET_NAME} PROPERTIES
         OUTPUT_NAME ${test_name})
-
+    if (GINKGO_FAST_TESTS)
+        target_compile_definitions(${TEST_TARGET_NAME} PRIVATE GINKGO_FAST_TESTS)
+    endif()
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
@@ -108,6 +122,10 @@ function(ginkgo_create_hip_test test_name)
     string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
 
     set_source_files_properties(${test_name}.hip.cpp PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
+    set(GINKGO_TEST_HIP_DEFINES)
+    if (GINKGO_FAST_TESTS)
+        set(GINKGO_TEST_HIP_DEFINES -DGINKGO_FAST_TESTS)
+    endif()
 
     # NOTE: With how HIP works, passing the flags `HIPCC_OPTIONS` etc. here
     # creates a redefinition of all flags. This creates some issues with `nvcc`,
@@ -116,13 +134,13 @@ function(ginkgo_create_hip_test test_name)
         hip_add_executable(${TEST_TARGET_NAME} ${test_name}.hip.cpp
             # If `FindHIP.cmake`, namely `HIP_PARSE_HIPCC_OPTIONS` macro and
             # call gets fixed, uncomment this.
-            # HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS}
-            # NVCC_OPTIONS  ${GINKGO_HIP_NVCC_OPTIONS}
-            # CLANG_OPTIONS ${GINKGO_HIP_CLANG_OPTIONS}
+            HIPCC_OPTIONS ${GINKGO_TEST_HIP_DEFINES} # ${GINKGO_HIPCC_OPTIONS}
+            # NVCC_OPTIONS  ${GINKGO_TEST_HIP_DEFINES} ${GINKGO_HIP_NVCC_OPTIONS}
+            # CLANG_OPTIONS ${GINKGO_TEST_HIP_DEFINES} ${GINKGO_HIP_CLANG_OPTIONS}
             )
     else() # hcc/clang
         hip_add_executable(${TEST_TARGET_NAME} ${test_name}.hip.cpp
-            HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS}
+            HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS} ${GINKGO_TEST_HIP_DEFINES}
             NVCC_OPTIONS  ${GINKGO_HIP_NVCC_OPTIONS}
             CLANG_OPTIONS ${GINKGO_HIP_CLANG_OPTIONS}
             )
diff --git a/cuda/test/factorization/par_ilut_kernels.cpp b/cuda/test/factorization/par_ilut_kernels.cpp
index 3004c2cbfd5..b59c57d0be8 100644
--- a/cuda/test/factorization/par_ilut_kernels.cpp
+++ b/cuda/test/factorization/par_ilut_kernels.cpp
@@ -71,7 +71,11 @@ class ParIlut : public ::testing::Test {
     using ComplexCsr = gko::matrix::Csr<std::complex<value_type>, index_type>;
 
     ParIlut()
+#ifdef GINKGO_FAST_TESTS
+        : mtx_size(152, 231),
+#else
         : mtx_size(532, 423),
+#endif
           rand_engine(1337),
           ref(gko::ReferenceExecutor::create()),
           cuda(gko::CudaExecutor::create(0, ref))
diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index 2b34a817f6d..a523612270a 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -67,7 +67,14 @@ class Csr : public ::testing::Test {
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
     using ComplexMtx = gko::matrix::Csr<std::complex<double>>;
 
-    Csr() : mtx_size(532, 231), rand_engine(42) {}
+    Csr()
+#ifdef GINKGO_FAST_TESTS
+        : mtx_size(152, 231),
+#else
+        : mtx_size(532, 231),
+#endif
+          rand_engine(42)
+    {}
 
     void SetUp()
     {
@@ -464,10 +471,10 @@ TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
 TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>());
-    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto complex_b = gen_mtx<ComplexVec>(this->mtx_size[1], 3, 1);
     auto dcomplex_b = ComplexVec::create(cuda);
     dcomplex_b->copy_from(complex_b.get());
-    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto complex_x = gen_mtx<ComplexVec>(this->mtx_size[0], 3, 1);
     auto dcomplex_x = ComplexVec::create(cuda);
     dcomplex_x->copy_from(complex_x.get());
 
@@ -481,10 +488,10 @@ TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
 TEST_F(Csr, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>());
-    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto complex_b = gen_mtx<ComplexVec>(this->mtx_size[1], 3, 1);
     auto dcomplex_b = ComplexVec::create(cuda);
     dcomplex_b->copy_from(complex_b.get());
-    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto complex_x = gen_mtx<ComplexVec>(this->mtx_size[0], 3, 1);
     auto dcomplex_x = ComplexVec::create(cuda);
     dcomplex_x->copy_from(complex_x.get());
 
diff --git a/cuda/test/matrix/diagonal_kernels.cpp b/cuda/test/matrix/diagonal_kernels.cpp
index c9a6580076b..d0016b0a079 100644
--- a/cuda/test/matrix/diagonal_kernels.cpp
+++ b/cuda/test/matrix/diagonal_kernels.cpp
@@ -62,7 +62,14 @@ class Diagonal : public ::testing::Test {
     using Dense = gko::matrix::Dense<ValueType>;
     using ComplexDiag = gko::matrix::Diagonal<ComplexValueType>;
 
-    Diagonal() : mtx_size(532, 231), rand_engine(42) {}
+    Diagonal()
+#ifdef GINKGO_FAST_TESTS
+        : mtx_size(152, 231),
+#else
+        : mtx_size(532, 231),
+#endif
+          rand_engine(42)
+    {}
 
     void SetUp()
     {
diff --git a/cuda/test/solver/cb_gmres_kernels.cpp b/cuda/test/solver/cb_gmres_kernels.cpp
index e4c6164e258..cb4b96f65c9 100644
--- a/cuda/test/solver/cb_gmres_kernels.cpp
+++ b/cuda/test/solver/cb_gmres_kernels.cpp
@@ -119,7 +119,11 @@ class CbGmres : public ::testing::Test {
 
     void initialize_data()
     {
+#ifdef GINKGO_FAST_TESTS
+        int m = 123;
+#else
         int m = 597;
+#endif
         int n = 43;
         x = gen_mtx(m, n);
         y = gen_mtx(default_krylov_dim_mixed, n);
diff --git a/cuda/test/solver/gmres_kernels.cpp b/cuda/test/solver/gmres_kernels.cpp
index 103b3f9edfb..7571bf848e5 100644
--- a/cuda/test/solver/gmres_kernels.cpp
+++ b/cuda/test/solver/gmres_kernels.cpp
@@ -92,7 +92,11 @@ class Gmres : public ::testing::Test {
 
     void initialize_data(int nrhs = 43)
     {
+#ifdef GINKGO_FAST_TESTS
+        int m = 123;
+#else
         int m = 597;
+#endif
         x = gen_mtx(m, nrhs);
         y = gen_mtx(gko::solver::default_krylov_dim, nrhs);
         before_preconditioner = Mtx::create_with_config_of(x.get());
diff --git a/hip/test/factorization/par_ilut_kernels.hip.cpp b/hip/test/factorization/par_ilut_kernels.hip.cpp
index 9fe1b758f2f..1a362c3717e 100644
--- a/hip/test/factorization/par_ilut_kernels.hip.cpp
+++ b/hip/test/factorization/par_ilut_kernels.hip.cpp
@@ -72,7 +72,11 @@ class ParIlut : public ::testing::Test {
     using ComplexCsr = gko::matrix::Csr<std::complex<value_type>, index_type>;
 
     ParIlut()
+#ifdef GINKGO_FAST_TESTS
+        : mtx_size(152, 231),
+#else
         : mtx_size(500, 700),
+#endif
           rand_engine(1337),
           ref(gko::ReferenceExecutor::create()),
           hip(gko::HipExecutor::create(0, ref))
diff --git a/hip/test/matrix/csr_kernels.hip.cpp b/hip/test/matrix/csr_kernels.hip.cpp
index 4f67cd74421..7a4e7a28ade 100644
--- a/hip/test/matrix/csr_kernels.hip.cpp
+++ b/hip/test/matrix/csr_kernels.hip.cpp
@@ -67,7 +67,14 @@ class Csr : public ::testing::Test {
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
     using ComplexMtx = gko::matrix::Csr<std::complex<double>>;
 
-    Csr() : mtx_size(532, 231), rand_engine(42) {}
+    Csr()
+#ifdef GINKGO_FAST_TESTS
+        : mtx_size(152, 231),
+#else
+        : mtx_size(532, 231),
+#endif
+          rand_engine(42)
+    {}
 
     void SetUp()
     {
@@ -458,10 +465,10 @@ TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
 TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
-    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto complex_b = gen_mtx<ComplexVec>(this->mtx_size[1], 3, 1);
     auto dcomplex_b = ComplexVec::create(hip);
     dcomplex_b->copy_from(complex_b.get());
-    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto complex_x = gen_mtx<ComplexVec>(this->mtx_size[0], 3, 1);
     auto dcomplex_x = ComplexVec::create(hip);
     dcomplex_x->copy_from(complex_x.get());
 
@@ -475,10 +482,10 @@ TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
 TEST_F(Csr, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>(hip));
-    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto complex_b = gen_mtx<ComplexVec>(this->mtx_size[1], 3, 1);
     auto dcomplex_b = ComplexVec::create(hip);
     dcomplex_b->copy_from(complex_b.get());
-    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto complex_x = gen_mtx<ComplexVec>(this->mtx_size[0], 3, 1);
     auto dcomplex_x = ComplexVec::create(hip);
     dcomplex_x->copy_from(complex_x.get());
 
diff --git a/hip/test/matrix/diagonal_kernels.hip.cpp b/hip/test/matrix/diagonal_kernels.hip.cpp
index 3b62ad8316e..2c97e7e4b73 100644
--- a/hip/test/matrix/diagonal_kernels.hip.cpp
+++ b/hip/test/matrix/diagonal_kernels.hip.cpp
@@ -62,7 +62,14 @@ class Diagonal : public ::testing::Test {
     using Dense = gko::matrix::Dense<ValueType>;
     using ComplexDiag = gko::matrix::Diagonal<ComplexValueType>;
 
-    Diagonal() : mtx_size(532, 231), rand_engine(42) {}
+    Diagonal()
+#ifdef GINKGO_FAST_TESTS
+        : mtx_size(152, 231),
+#else
+        : mtx_size(532, 231),
+#endif
+          rand_engine(42)
+    {}
 
     void SetUp()
     {
diff --git a/hip/test/solver/cb_gmres_kernels.cpp b/hip/test/solver/cb_gmres_kernels.cpp
index 872b23a1556..19572f73c26 100644
--- a/hip/test/solver/cb_gmres_kernels.cpp
+++ b/hip/test/solver/cb_gmres_kernels.cpp
@@ -119,7 +119,11 @@ class CbGmres : public ::testing::Test {
 
     void initialize_data()
     {
+#ifdef GINKGO_FAST_TESTS
+        int m = 123;
+#else
         int m = 597;
+#endif
         int n = 43;
         x = gen_mtx(m, n);
         y = gen_mtx(default_krylov_dim_mixed, n);
diff --git a/hip/test/solver/gmres_kernels.cpp b/hip/test/solver/gmres_kernels.cpp
index 04c2097bea5..42a9668e325 100644
--- a/hip/test/solver/gmres_kernels.cpp
+++ b/hip/test/solver/gmres_kernels.cpp
@@ -93,7 +93,11 @@ class Gmres : public ::testing::Test {
 
     void initialize_data(int nrhs = 43)
     {
+#ifdef GINKGO_FAST_TESTS
+        int m = 123;
+#else
         int m = 597;
+#endif
         x = gen_mtx(m, nrhs);
         y = gen_mtx(gko::solver::default_krylov_dim, nrhs);
         before_preconditioner = Mtx::create_with_config_of(x.get());
diff --git a/omp/test/factorization/par_ict_kernels.cpp b/omp/test/factorization/par_ict_kernels.cpp
index 43e13fee7c9..2ded4cc3ef4 100644
--- a/omp/test/factorization/par_ict_kernels.cpp
+++ b/omp/test/factorization/par_ict_kernels.cpp
@@ -71,7 +71,11 @@ class ParIct : public ::testing::Test {
     using Csr = gko::matrix::Csr<value_type, index_type>;
 
     ParIct()
+#ifdef GINKGO_FAST_TESTS
+        : mtx_size(152, 152),
+#else
         : mtx_size(532, 532),
+#endif
           rand_engine(567321),
           ref(gko::ReferenceExecutor::create()),
           omp(gko::OmpExecutor::create())
diff --git a/omp/test/factorization/par_ilut_kernels.cpp b/omp/test/factorization/par_ilut_kernels.cpp
index eaea0c2d41e..97ff5974877 100644
--- a/omp/test/factorization/par_ilut_kernels.cpp
+++ b/omp/test/factorization/par_ilut_kernels.cpp
@@ -72,7 +72,11 @@ class ParIlut : public ::testing::Test {
     using Csr = gko::matrix::Csr<value_type, index_type>;
 
     ParIlut()
+#ifdef GINKGO_FAST_TESTS
+        : mtx_size(152, 231),
+#else
         : mtx_size(532, 423),
+#endif
           rand_engine(1337),
           ref(gko::ReferenceExecutor::create()),
           omp(gko::OmpExecutor::create())
diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index a27f6638a04..6215b336586 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -67,7 +67,14 @@ class Csr : public ::testing::Test {
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
     using ComplexMtx = gko::matrix::Csr<std::complex<double>>;
 
-    Csr() : mtx_size(532, 231), rand_engine(42) {}
+    Csr()
+#ifdef GINKGO_FAST_TESTS
+        : mtx_size(152, 185),
+#else
+        : mtx_size(532, 231),
+#endif
+          rand_engine(42)
+    {}
 
     void SetUp()
     {
@@ -285,10 +292,10 @@ TEST_F(Csr, AdvancedApplyToDenseMatrixIsEquivalentToRef)
 TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(3);
-    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto complex_b = gen_mtx<ComplexVec>(this->mtx_size[1], 3, 1);
     auto dcomplex_b = ComplexVec::create(omp);
     dcomplex_b->copy_from(complex_b.get());
-    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto complex_x = gen_mtx<ComplexVec>(this->mtx_size[0], 3, 1);
     auto dcomplex_x = ComplexVec::create(omp);
     dcomplex_x->copy_from(complex_x.get());
 
@@ -302,10 +309,10 @@ TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
 TEST_F(Csr, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(3);
-    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto complex_b = gen_mtx<ComplexVec>(this->mtx_size[1], 3, 1);
     auto dcomplex_b = ComplexVec::create(omp);
     dcomplex_b->copy_from(complex_b.get());
-    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto complex_x = gen_mtx<ComplexVec>(this->mtx_size[0], 3, 1);
     auto dcomplex_x = ComplexVec::create(omp);
     dcomplex_x->copy_from(complex_x.get());
 
diff --git a/omp/test/matrix/diagonal_kernels.cpp b/omp/test/matrix/diagonal_kernels.cpp
index 16de6c4d7f5..b024f18ba0a 100644
--- a/omp/test/matrix/diagonal_kernels.cpp
+++ b/omp/test/matrix/diagonal_kernels.cpp
@@ -62,7 +62,14 @@ class Diagonal : public ::testing::Test {
     using Dense = gko::matrix::Dense<ValueType>;
     using ComplexDiag = gko::matrix::Diagonal<ComplexValueType>;
 
-    Diagonal() : mtx_size(532, 231), rand_engine(42) {}
+    Diagonal()
+#ifdef GINKGO_FAST_TESTS
+        : mtx_size(152, 185),
+#else
+        : mtx_size(532, 231),
+#endif
+          rand_engine(42)
+    {}
 
     void SetUp()
     {
diff --git a/omp/test/solver/cb_gmres_kernels.cpp b/omp/test/solver/cb_gmres_kernels.cpp
index 27f891eb9fe..7aa7fe20540 100644
--- a/omp/test/solver/cb_gmres_kernels.cpp
+++ b/omp/test/solver/cb_gmres_kernels.cpp
@@ -121,7 +121,11 @@ class CbGmres : public ::testing::Test {
 
     void initialize_data()
     {
+#ifdef GINKGO_FAST_TESTS
+        int m = 123;
+#else
         int m = 597;
+#endif
         int n = 43;
         x = gen_mtx(m, n);
         y = gen_mtx(default_krylov_dim_mixed, n);
diff --git a/omp/test/solver/gmres_kernels.cpp b/omp/test/solver/gmres_kernels.cpp
index 31692fb0628..58b369e748e 100644
--- a/omp/test/solver/gmres_kernels.cpp
+++ b/omp/test/solver/gmres_kernels.cpp
@@ -125,7 +125,11 @@ class Gmres : public ::testing::Test {
 
     void initialize_data()
     {
+#ifdef GINKGO_FAST_TESTS
+        int m = 123;
+#else
         int m = 597;
+#endif
         int n = 43;
         x = gen_mtx(m, n);
         y = gen_mtx(gko::solver::default_krylov_dim, n);

From 3cffc4c34440d781303ee709c3701e5e70f39b7b Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 12 Apr 2021 10:34:44 +0200
Subject: [PATCH 0615/1008] randomly load-balance between CUDA devices in CI

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3036303d464..0e49972117f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -34,7 +34,7 @@ stages:
 .before_script_template: &default_before_script
   - export NUM_CORES=${CI_PARALLELISM}
   - export OMP_NUM_THREADS=${NUM_CORES}
-  - export CUDA_VISIBLE_DEVICES=0
+  - export CUDA_VISIBLE_DEVICES=$((RANDOM % 2))
   - export CCACHE_DIR=${CCACHE_DIR}
   - export CCACHE_MAXSIZE=${CCACHE_MAXSIZE}
 

From 33fdff173c8cc17c2286188a27571cdaaedfe82e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 12 Apr 2021 11:14:05 +0200
Subject: [PATCH 0616/1008] add quick_test target for tests

---
 CMakeLists.txt |  1 +
 TESTING.md     | 11 +++++++++++
 2 files changed, 12 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c6d8cb67c44..ddcfaa55b99 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -143,6 +143,7 @@ if(GINKGO_BUILD_TESTS)
 
     enable_testing()
     include(CTest)
+    add_custom_target(quick_test "${CMAKE_CTEST_COMMAND}" -R 'core|reference')
 endif()
 
 if(GINKGO_WITH_CLANG_TIDY)
diff --git a/TESTING.md b/TESTING.md
index 482f8721792..c8e7ec38c89 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -27,6 +27,17 @@ run the following from the build folder:
 
 where `path/to/test` is the path returned by `make test`.
 
+#### Using make quick_test
+After compiling Ginkgo, use the following command inside the build folder to run
+a small subset of tests that should execute quickly:
+
+```sh
+make quick_test
+```
+
+These tests do not use GPU features except for a few device property queries, so
+they may still fail if Ginkgo was compiled with GPU support, but no such GPU is
+available. The output is equivalent to `make test`.
 
 #### Using CTest 
 The tests can also be ran through CTest from the command line, for example when

From 4b0faf8061cbe7bd442ad69a8908e278f073b03e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 12 Apr 2021 15:47:31 +0200
Subject: [PATCH 0617/1008] add GINKGO_FAST_TESTS to summary and documentation

---
 INSTALL.md           | 2 ++
 cmake/get_info.cmake | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/INSTALL.md b/INSTALL.md
index c415e85080f..a477b2f6b9a 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -20,6 +20,8 @@ Ginkgo adds the following additional switches to control what is being built:
     default is `OFF`.
 *   `-DGINKGO_BUILD_TESTS={ON, OFF}` builds Ginkgo's tests
     (will download googletest), default is `ON`.
+*   `-DGINKGO_FAST_TESTS={ON, OFF}` reduces the input sizes for a few slow tests
+    to speed them up, default is `OFF`.
 *   `-DGINKGO_BUILD_BENCHMARKS={ON, OFF}` builds Ginkgo's benchmarks
     (will download gflags and rapidjson), default is `ON`.
 *   `-DGINKGO_BUILD_EXAMPLES={ON, OFF}` builds Ginkgo's examples, default is `ON`
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index be150e0c3f4..835bdf0c45b 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -130,7 +130,7 @@ foreach(log_type ${log_types})
         "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP")
     ginkgo_print_module_footer(${${log_type}} "  Tests, benchmarks and examples:")
     ginkgo_print_foreach_variable(${${log_type}}
-        "GINKGO_BUILD_TESTS;GINKGO_BUILD_EXAMPLES;GINKGO_EXTLIB_EXAMPLE;GINKGO_BUILD_BENCHMARKS;GINKGO_BENCHMARK_ENABLE_TUNING")
+        "GINKGO_BUILD_TESTS;GINKGO_FAST_TESTS;GINKGO_BUILD_EXAMPLES;GINKGO_EXTLIB_EXAMPLE;GINKGO_BUILD_BENCHMARKS;GINKGO_BENCHMARK_ENABLE_TUNING")
     ginkgo_print_module_footer(${${log_type}} "  Documentation:")
     ginkgo_print_foreach_variable(${${log_type}} "GINKGO_BUILD_DOC;GINKGO_VERBOSE_LEVEL")
     ginkgo_print_module_footer(${${log_type}} "")

From 68c813fc5ce9af6925828d597b76fefe4d70151c Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 14 Apr 2021 10:50:17 +0200
Subject: [PATCH 0618/1008] fix infinite recursion with criterion check logger

Also fix a warning in record.hpp
---
 core/log/record.cpp                | 16 ++++++++++++++-
 core/test/log/record.cpp           | 32 +++++++++++++++++++++++++++++-
 include/ginkgo/core/log/logger.hpp |  2 +-
 include/ginkgo/core/log/record.hpp |  8 ++++++++
 4 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/core/log/record.cpp b/core/log/record.cpp
index 32be42931c9..ba249edf3e6 100644
--- a/core/log/record.cpp
+++ b/core/log/record.cpp
@@ -244,7 +244,8 @@ void Record::on_criterion_check_started(
 
 void Record::on_criterion_check_completed(
     const stop::Criterion *criterion, const size_type &num_iterations,
-    const LinOp *residual, const LinOp *residual_norm, const LinOp *solution,
+    const LinOp *residual, const LinOp *residual_norm,
+    const LinOp *implicit_residual_norm_sq, const LinOp *solution,
     const uint8 &stopping_id, const bool &set_finalized,
     const Array<stopping_status> *status, const bool &oneChanged,
     const bool &converged) const
@@ -257,6 +258,19 @@ void Record::on_criterion_check_completed(
 }
 
 
+void Record::on_criterion_check_completed(
+    const stop::Criterion *criterion, const size_type &num_iterations,
+    const LinOp *residual, const LinOp *residual_norm, const LinOp *solution,
+    const uint8 &stopping_id, const bool &set_finalized,
+    const Array<stopping_status> *status, const bool &oneChanged,
+    const bool &converged) const
+{
+    this->on_criterion_check_completed(
+        criterion, num_iterations, residual, residual_norm, nullptr, solution,
+        stopping_id, set_finalized, status, oneChanged, converged);
+}
+
+
 void Record::on_iteration_complete(const LinOp *solver,
                                    const size_type &num_iterations,
                                    const LinOp *residual, const LinOp *solution,
diff --git a/core/test/log/record.cpp b/core/test/log/record.cpp
index ffde5bb0e23..a9a0947d199 100644
--- a/core/test/log/record.cpp
+++ b/core/test/log/record.cpp
@@ -464,7 +464,7 @@ TEST(Record, CatchesCriterionCheckStarted)
 }
 
 
-TEST(Record, CatchesCriterionCheckCompleted)
+TEST(Record, CatchesCriterionCheckCompletedOld)
 {
     auto exec = gko::ReferenceExecutor::create();
     auto logger = gko::log::Record::create(
@@ -494,6 +494,36 @@ TEST(Record, CatchesCriterionCheckCompleted)
 }
 
 
+TEST(Record, CatchesCriterionCheckCompleted)
+{
+    auto exec = gko::ReferenceExecutor::create();
+    auto logger = gko::log::Record::create(
+        exec, gko::log::Logger::criterion_check_completed_mask);
+    auto criterion =
+        gko::stop::Iteration::build().with_max_iters(3u).on(exec)->generate(
+            nullptr, nullptr, nullptr);
+    constexpr gko::uint8 RelativeStoppingId{42};
+    gko::Array<gko::stopping_status> stop_status(exec, 1);
+
+    logger->on<gko::log::Logger::criterion_check_completed>(
+        criterion.get(), 1, nullptr, nullptr, nullptr, nullptr,
+        RelativeStoppingId, true, &stop_status, true, true);
+
+    stop_status.get_data()->reset();
+    stop_status.get_data()->stop(RelativeStoppingId);
+    auto &data = logger->get().criterion_check_completed.back();
+    ASSERT_NE(data->criterion, nullptr);
+    ASSERT_EQ(data->stopping_id, RelativeStoppingId);
+    ASSERT_EQ(data->set_finalized, true);
+    ASSERT_EQ(data->status->get_const_data()->has_stopped(), true);
+    ASSERT_EQ(data->status->get_const_data()->get_id(),
+              stop_status.get_const_data()->get_id());
+    ASSERT_EQ(data->status->get_const_data()->is_finalized(), true);
+    ASSERT_EQ(data->oneChanged, true);
+    ASSERT_EQ(data->converged, true);
+}
+
+
 TEST(Record, CatchesIterations)
 {
     using Dense = gko::matrix::Dense<>;
diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp
index 22a809bab75..049d964fa51 100644
--- a/include/ginkgo/core/log/logger.hpp
+++ b/include/ginkgo/core/log/logger.hpp
@@ -417,7 +417,7 @@ public:                                                              \
         const Array<stopping_status> *status, const bool &one_changed,
         const bool &all_converged) const
     {
-        this->on_criterion_check_completed(criterion, it, r, x, tau, x,
+        this->on_criterion_check_completed(criterion, it, r, tau, x,
                                            stopping_id, set_finalized, status,
                                            one_changed, all_converged);
     }
diff --git a/include/ginkgo/core/log/record.hpp b/include/ginkgo/core/log/record.hpp
index eb20ae0a40a..71b31e9e794 100644
--- a/include/ginkgo/core/log/record.hpp
+++ b/include/ginkgo/core/log/record.hpp
@@ -363,6 +363,14 @@ class Record : public Logger {
                                     const uint8 &stopping_id,
                                     const bool &set_finalized) const override;
 
+    void on_criterion_check_completed(
+        const stop::Criterion *criterion, const size_type &num_iterations,
+        const LinOp *residual, const LinOp *residual_norm,
+        const LinOp *implicit_residual_norm_sq, const LinOp *solution,
+        const uint8 &stopping_id, const bool &set_finalized,
+        const Array<stopping_status> *status, const bool &one_changed,
+        const bool &all_converged) const override;
+
     void on_criterion_check_completed(
         const stop::Criterion *criterion, const size_type &num_iterations,
         const LinOp *residual, const LinOp *residual_norm,

From 587ffca9b5f39d6bf6c64e4c61cbdbb6daa83299 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 1 Apr 2021 23:53:36 +0800
Subject: [PATCH 0619/1008] fix memory leak and use ref not omp in device test

---
 .gitlab-ci.yml                                | 38 +++++++++++++++++++
 cuda/base/executor.cpp                        | 12 +++---
 cuda/test/factorization/par_ic_kernels.cpp    |  2 +-
 cuda/test/reorder/rcm_kernels.cpp             |  3 +-
 hip/base/executor.hip.cpp                     | 12 +++---
 hip/test/factorization/par_ic_kernels.hip.cpp |  2 +-
 6 files changed, 54 insertions(+), 15 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0e49972117f..242de328838 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -288,6 +288,26 @@ build/cuda92/intel/cuda/release/static:
     - cuda
     - gpu
 
+# Build CUDA NVIDIA without omp
+build/cuda92/intel/cuda_wo_omp/release/shared:
+  <<: *default_build_with_test
+  image: localhost:5000/gko-cuda92-gnu7-llvm50-intel2017
+  variables:
+    <<: *default_variables
+    C_COMPILER: "icc"
+    CXX_COMPILER: "icpc"
+    BUILD_CUDA: "ON"
+    BUILD_HWLOC: "OFF"
+    BUILD_TYPE: "Release"
+    CUDA_ARCH: 35
+  only:
+    variables:
+      - $RUN_CI_TAG
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
 # cuda 10.0 and friends
 # Make sure that our jobs run when using self-installed
 # third-party HWLOC.
@@ -597,6 +617,24 @@ build/amd/clang/hip/release/static:
     - amd
     - gpu
 
+# Build HIP AMD without omp
+build/amd/clang/hip_wo_omp/release/shared:
+  <<: *default_build_with_test
+  image: localhost:5000/gko-amd-gnu8-llvm7
+  variables:
+    <<: *default_variables
+    C_COMPILER: "clang"
+    CXX_COMPILER: "clang++"
+    BUILD_HIP: "ON"
+    BUILD_TYPE: "Release"
+  only:
+    variables:
+      - $RUN_CI_TAG
+  tags:
+    - private_ci
+    - amd
+    - gpu
+
 # no cuda but latest gcc and clang
 build/nocuda/gcc/core/debug/static:
   <<: *default_build_with_test
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 7f41fe87f3d..0cbaa82b3b9 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -61,9 +61,9 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
     return std::shared_ptr<CudaExecutor>(
         new CudaExecutor(device_id, std::move(master), device_reset),
         [device_id](CudaExecutor *exec) {
+            auto device_reset = exec->get_device_reset();
             delete exec;
-            if (!CudaExecutor::get_num_execs(device_id) &&
-                exec->get_device_reset()) {
+            if (!CudaExecutor::get_num_execs(device_id) && device_reset) {
                 cuda::device_guard g(device_id);
                 cudaDeviceReset();
             }
@@ -76,9 +76,9 @@ void CudaExecutor::populate_exec_info(const MachineTopology *mach_topo)
     if (this->get_device_id() < this->get_num_devices() &&
         this->get_device_id() >= 0) {
         cuda::device_guard g(this->get_device_id());
-        GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetPCIBusId(
-            const_cast<char *>(this->get_exec_info().pci_bus_id.data()), 13,
-            this->get_device_id()));
+        GKO_ASSERT_NO_CUDA_ERRORS(
+            cudaDeviceGetPCIBusId(&(this->get_exec_info().pci_bus_id.front()),
+                                  13, this->get_device_id()));
 
         auto cuda_hwloc_obj =
             mach_topo->get_pci_device(this->get_exec_info().pci_bus_id);
@@ -230,7 +230,7 @@ void CudaExecutor::set_gpu_property()
         GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
             &max_threads_per_block, cudaDevAttrMaxThreadsPerBlock,
             this->get_device_id()));
-        std::vector<int> max_threads_per_block_dim{3, 0};
+        std::vector<int> max_threads_per_block_dim(3, 0);
         GKO_ASSERT_NO_CUDA_ERRORS(cudaDeviceGetAttribute(
             &max_threads_per_block_dim[0], cudaDevAttrMaxBlockDimX,
             this->get_device_id()));
diff --git a/cuda/test/factorization/par_ic_kernels.cpp b/cuda/test/factorization/par_ic_kernels.cpp
index ac454eb6b80..e957efbadf4 100644
--- a/cuda/test/factorization/par_ic_kernels.cpp
+++ b/cuda/test/factorization/par_ic_kernels.cpp
@@ -71,7 +71,7 @@ class ParIc : public ::testing::Test {
         : mtx_size(624, 624),
           rand_engine(43456),
           ref(gko::ReferenceExecutor::create()),
-          cuda(gko::CudaExecutor::create(0, gko::OmpExecutor::create()))
+          cuda(gko::CudaExecutor::create(0, gko::ReferenceExecutor::create()))
     {
         mtx_l = gko::test::generate_random_lower_triangular_matrix<Csr>(
             mtx_size[0], mtx_size[0], false,
diff --git a/cuda/test/reorder/rcm_kernels.cpp b/cuda/test/reorder/rcm_kernels.cpp
index 0a066918b95..afb3336134b 100644
--- a/cuda/test/reorder/rcm_kernels.cpp
+++ b/cuda/test/reorder/rcm_kernels.cpp
@@ -52,7 +52,8 @@ class Rcm : public ::testing::Test {
 
 
     Rcm()
-        : exec(gko::CudaExecutor::create(0, gko::OmpExecutor::create(), true)),
+        : exec(gko::CudaExecutor::create(0, gko::ReferenceExecutor::create(),
+                                         true)),
           // clang-format off
           p_mtx(gko::initialize<CsrMtx>({{1.0, 2.0, 0.0, -1.3, 2.1},
                                          {2.0, 5.0, 1.5, 0.0, 0.0},
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 706270aaa47..49013544d98 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -61,9 +61,9 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
     return std::shared_ptr<HipExecutor>(
         new HipExecutor(device_id, std::move(master), device_reset),
         [device_id](HipExecutor *exec) {
+            auto device_reset = exec->get_device_reset();
             delete exec;
-            if (!HipExecutor::get_num_execs(device_id) &&
-                exec->get_device_reset()) {
+            if (!HipExecutor::get_num_execs(device_id) && device_reset) {
                 hip::device_guard g(device_id);
                 hipDeviceReset();
             }
@@ -76,9 +76,9 @@ void HipExecutor::populate_exec_info(const MachineTopology *mach_topo)
     if (this->get_device_id() < this->get_num_devices() &&
         this->get_device_id() >= 0) {
         hip::device_guard g(this->get_device_id());
-        GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetPCIBusId(
-            const_cast<char *>(this->get_exec_info().pci_bus_id.data()), 13,
-            this->get_device_id()));
+        GKO_ASSERT_NO_HIP_ERRORS(
+            hipDeviceGetPCIBusId(&(this->get_exec_info().pci_bus_id.front()),
+                                 13, this->get_device_id()));
 
         auto hip_hwloc_obj =
             mach_topo->get_pci_device(this->get_exec_info().pci_bus_id);
@@ -232,7 +232,7 @@ void HipExecutor::set_gpu_property()
             this->get_device_id()));
         this->get_exec_info().max_workitem_sizes.push_back(
             max_threads_per_block);
-        std::vector<int> max_threads_per_block_dim{3, 0};
+        std::vector<int> max_threads_per_block_dim(3, 0);
         GKO_ASSERT_NO_HIP_ERRORS(hipDeviceGetAttribute(
             &max_threads_per_block_dim[0], hipDeviceAttributeMaxBlockDimX,
             this->get_device_id()));
diff --git a/hip/test/factorization/par_ic_kernels.hip.cpp b/hip/test/factorization/par_ic_kernels.hip.cpp
index 230ade961e5..e58b4da93e4 100644
--- a/hip/test/factorization/par_ic_kernels.hip.cpp
+++ b/hip/test/factorization/par_ic_kernels.hip.cpp
@@ -71,7 +71,7 @@ class ParIc : public ::testing::Test {
         : mtx_size(585, 585),
           rand_engine(10667),
           ref(gko::ReferenceExecutor::create()),
-          hip(gko::HipExecutor::create(0, gko::OmpExecutor::create()))
+          hip(gko::HipExecutor::create(0, gko::ReferenceExecutor::create()))
     {
         mtx_l = gko::test::generate_random_lower_triangular_matrix<Csr>(
             mtx_size[0], mtx_size[0], false,

From 07845905b890455bf92fa7e28f6ebed55bbaa6d4 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 6 Apr 2021 15:46:53 +0800
Subject: [PATCH 0620/1008] fix install's description, relax 1 isai test check

---
 INSTALL.md                                   | 14 +++++++-------
 cuda/test/preconditioner/isai_kernels.cpp    |  2 +-
 hip/test/preconditioner/isai_kernels.hip.cpp |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index a477b2f6b9a..77bc55cef3f 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -99,7 +99,7 @@ Ginkgo adds the following additional switches to control what is being built:
     list of architectures. Supported values are:
 
     *   `Auto`
-    *   `Kepler`, `Maxwell`, `Pascal`, `Volta`, `Ampere`
+    *   `Kepler`, `Maxwell`, `Pascal`, `Volta`, `Turing` `Ampere`
     *   `CODE`, `CODE(COMPUTE)`, `(COMPUTE)`
 
     `Auto` will automatically detect the present CUDA-enabled GPU architectures
@@ -137,7 +137,7 @@ Depending on the configuration settings, some manual work might be required:
 * Build Ginkgo as shared library:
   Add `PROJECT_BINARY_DIR/GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH` into the environment variable `PATH`.
   `GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH` is `windows_shared_library` by default. More Details are available in the [Installation page](./INSTALL.md).
-  * cmd: `set PATH="<PROJECT_BINARY_DIR/GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH>;%PATH%"`
+  * cmd: `set PATH=<PROJECT_BINARY_DIR/GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH>;%PATH%`
   * powershell: `$env:PATH="<PROJECT_BINARY_DIR/GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH>;$env:PATH"`
 
   CMake will give the following error message if the path is not correct.
@@ -147,16 +147,16 @@ Depending on the configuration settings, some manual work might be required:
   where `<path>` is the needed `<PROJECT_BINARY_DIR/GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH>`.
 * Build Ginkgo with Debug mode:
   Some Debug build specific issues can appear depending on the machine and environment. The known issues are the following:
-  1. `bigobj` issue: encountering  `too many sections` needs the compilation flags `\bigobj` or `-Wa,-mbig-obj`
+  1. `bigobj` issue: encountering  `too many sections` needs the compilation flags `/bigobj` or `-Wa,-mbig-obj`
   2. `ld` issue: encountering  `ld: error: export ordinal too large` needs the compilation flag `-O1`
 
   The following are the details for different environments:
   * _Microsoft Visual Studio_:
     1. `bigobj` issue
-      * `cmake -DCMAKE_CXX_FLAGS=\bigobj <other parameters> <source_folder>` which might overwrite the default settings.
-      * add `\bigobj` into the environment variable `CXXFLAGS` (only available in the first cmake configuration)
-        * cmd: `set CXXFLAGS=\bigobj`
-        * powershell: `$env:CXXFLAGS=\bigobj`
+      * `cmake -DCMAKE_CXX_FLAGS=/bigobj <other parameters> <source_folder>` which might overwrite the default settings.
+      * add `/bigobj` into the environment variable `CXXFLAGS` (only available in the first cmake configuration)
+        * cmd: `set CXXFLAGS=/bigobj`
+        * powershell: `$env:CXXFLAGS=/bigobj`
     2. `ld` issue (_Microsoft Visual Studio_ does not have this issue)
   * _Cygwin_:
     1. `bigobj` issue
diff --git a/cuda/test/preconditioner/isai_kernels.cpp b/cuda/test/preconditioner/isai_kernels.cpp
index f099e3f503b..9385c0f109d 100644
--- a/cuda/test/preconditioner/isai_kernels.cpp
+++ b/cuda/test/preconditioner/isai_kernels.cpp
@@ -244,7 +244,7 @@ TEST_F(Isai, CudaIsaiGenerateSpdinverseShortIsEquivalentToRef)
         true);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
-    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 10 * r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 15 * r<value_type>::value);
     GKO_ASSERT_ARRAY_EQ(a1, da1);
     GKO_ASSERT_ARRAY_EQ(a2, da2);
     ASSERT_EQ(a1.get_const_data()[num_rows], 0);
diff --git a/hip/test/preconditioner/isai_kernels.hip.cpp b/hip/test/preconditioner/isai_kernels.hip.cpp
index 18470c3d77c..bfcb226bbc1 100644
--- a/hip/test/preconditioner/isai_kernels.hip.cpp
+++ b/hip/test/preconditioner/isai_kernels.hip.cpp
@@ -225,7 +225,7 @@ TEST_F(Isai, HipIsaiGenerateSpdinverseShortIsEquivalentToRef)
         true);
 
     GKO_ASSERT_MTX_EQ_SPARSITY(inverse, d_inverse);
-    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 10 * r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(inverse, d_inverse, 15 * r<value_type>::value);
     GKO_ASSERT_ARRAY_EQ(a1, da1);
     GKO_ASSERT_ARRAY_EQ(a2, da2);
     ASSERT_EQ(a1.get_const_data()[num_rows], 0);

From 5d2e6d3cbfa1cdb08d6a6cc027004cfbec75ad80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 8 Apr 2021 10:42:59 +0200
Subject: [PATCH 0621/1008] Fix runtime errors of MSVC 2019

Although MSVC describe the constructor of mutex as constexpr, MSVC comments the constexpr keyword in their header in particular.
---
 cuda/test/base/array.cu                  | 2 +-
 cuda/test/matrix/diagonal_kernels.cpp    | 8 ++++----
 cuda/test/solver/cb_gmres_kernels.cpp    | 2 +-
 cuda/test/utils.hpp                      | 5 +++++
 hip/test/matrix/diagonal_kernels.hip.cpp | 8 ++++----
 hip/test/solver/cb_gmres_kernels.cpp     | 2 +-
 hip/test/utils.hip.hpp                   | 5 +++++
 7 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/cuda/test/base/array.cu b/cuda/test/base/array.cu
index f35ac061972..4d12cff3988 100644
--- a/cuda/test/base/array.cu
+++ b/cuda/test/base/array.cu
@@ -39,7 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
-#include "core/test/utils.hpp"
+#include "cuda/test/utils.hpp"
 
 
 template <typename T>
diff --git a/cuda/test/matrix/diagonal_kernels.cpp b/cuda/test/matrix/diagonal_kernels.cpp
index d0016b0a079..d3c6f8c5973 100644
--- a/cuda/test/matrix/diagonal_kernels.cpp
+++ b/cuda/test/matrix/diagonal_kernels.cpp
@@ -125,10 +125,10 @@ class Diagonal : public ::testing::Test {
         diag = gen_diag(mtx_size[0]);
         ddiag = Diag::create(cuda);
         ddiag->copy_from(diag.get());
-        dense1 = gen_mtx<Dense>(mtx_size[0], mtx_size[1], mtx_size[0]);
-        dense2 = gen_mtx<Dense>(mtx_size[1], mtx_size[0], mtx_size[1]);
-        denseexpected1 = gen_mtx<Dense>(mtx_size[0], mtx_size[1], mtx_size[0]);
-        denseexpected2 = gen_mtx<Dense>(mtx_size[1], mtx_size[0], mtx_size[1]);
+        dense1 = gen_mtx<Dense>(mtx_size[0], mtx_size[1], mtx_size[1]);
+        dense2 = gen_mtx<Dense>(mtx_size[1], mtx_size[0], mtx_size[0]);
+        denseexpected1 = gen_mtx<Dense>(mtx_size[0], mtx_size[1], mtx_size[1]);
+        denseexpected2 = gen_mtx<Dense>(mtx_size[1], mtx_size[0], mtx_size[0]);
         ddense1 = Dense::create(cuda);
         ddense1->copy_from(dense1.get());
         ddense2 = Dense::create(cuda);
diff --git a/cuda/test/solver/cb_gmres_kernels.cpp b/cuda/test/solver/cb_gmres_kernels.cpp
index cb4b96f65c9..ce28556ef24 100644
--- a/cuda/test/solver/cb_gmres_kernels.cpp
+++ b/cuda/test/solver/cb_gmres_kernels.cpp
@@ -49,7 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/solver/cb_gmres_accessor.hpp"
 #include "core/solver/cb_gmres_kernels.hpp"
-#include "core/test/utils.hpp"
+#include "cuda/test/utils.hpp"
 
 
 namespace {
diff --git a/cuda/test/utils.hpp b/cuda/test/utils.hpp
index 4b6e0e0a667..7667c7beb9f 100644
--- a/cuda/test/utils.hpp
+++ b/cuda/test/utils.hpp
@@ -43,9 +43,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
+// Visual Studio does not define the constructor of std::mutex as constexpr,
+// causing it to not be initialized when creating this executor (which uses
+// the mutex)
+#if !defined(_MSC_VER)
 // prevent device reset after each test
 auto no_reset_exec =
     gko::CudaExecutor::create(0, gko::ReferenceExecutor::create(), true);
+#endif
 
 
 }  // namespace
diff --git a/hip/test/matrix/diagonal_kernels.hip.cpp b/hip/test/matrix/diagonal_kernels.hip.cpp
index 2c97e7e4b73..606e39db7e6 100644
--- a/hip/test/matrix/diagonal_kernels.hip.cpp
+++ b/hip/test/matrix/diagonal_kernels.hip.cpp
@@ -125,10 +125,10 @@ class Diagonal : public ::testing::Test {
         diag = gen_diag(mtx_size[0]);
         ddiag = Diag::create(hip);
         ddiag->copy_from(diag.get());
-        dense1 = gen_mtx<Dense>(mtx_size[0], mtx_size[1], mtx_size[0]);
-        dense2 = gen_mtx<Dense>(mtx_size[1], mtx_size[0], mtx_size[1]);
-        denseexpected1 = gen_mtx<Dense>(mtx_size[0], mtx_size[1], mtx_size[0]);
-        denseexpected2 = gen_mtx<Dense>(mtx_size[1], mtx_size[0], mtx_size[1]);
+        dense1 = gen_mtx<Dense>(mtx_size[0], mtx_size[1], mtx_size[1]);
+        dense2 = gen_mtx<Dense>(mtx_size[1], mtx_size[0], mtx_size[0]);
+        denseexpected1 = gen_mtx<Dense>(mtx_size[0], mtx_size[1], mtx_size[1]);
+        denseexpected2 = gen_mtx<Dense>(mtx_size[1], mtx_size[0], mtx_size[0]);
         ddense1 = Dense::create(hip);
         ddense1->copy_from(dense1.get());
         ddense2 = Dense::create(hip);
diff --git a/hip/test/solver/cb_gmres_kernels.cpp b/hip/test/solver/cb_gmres_kernels.cpp
index 19572f73c26..b5114129935 100644
--- a/hip/test/solver/cb_gmres_kernels.cpp
+++ b/hip/test/solver/cb_gmres_kernels.cpp
@@ -49,7 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/solver/cb_gmres_accessor.hpp"
 #include "core/solver/cb_gmres_kernels.hpp"
-#include "core/test/utils.hpp"
+#include "hip/test/utils.hip.hpp"
 
 
 namespace {
diff --git a/hip/test/utils.hip.hpp b/hip/test/utils.hip.hpp
index dda2c068524..03d4f2ba6c7 100644
--- a/hip/test/utils.hip.hpp
+++ b/hip/test/utils.hip.hpp
@@ -43,9 +43,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
+// Visual Studio does not define the constructor of std::mutex as constexpr,
+// causing it to not be initialized when creating this executor (which uses
+// the mutex)
+#if !defined(_MSC_VER)
 // prevent device reset after each test
 auto no_reset_exec =
     gko::HipExecutor::create(0, gko::ReferenceExecutor::create(), true);
+#endif
 
 
 }  // namespace

From 734347868568db03dc242ba60d486680e084ca38 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 15 Apr 2021 17:59:34 +0800
Subject: [PATCH 0622/1008] =?UTF-8?q?descr=C2=A0switch=C2=A0static/shared,?=
 =?UTF-8?q?=C2=A0hwloc=C2=A0off=C2=A0on=C2=A0win/mac,=20mv=20third=5Fparty?=
 =?UTF-8?q?=20build,=20restore=20MSVC=20static=20build=20-=20move=20the=20?=
 =?UTF-8?q?third=20party=20build=20after=20flag=20configuration=20-=C2=A0A?=
 =?UTF-8?q?dd=C2=A0description=C2=A0of=C2=A0possible=20issue=20when=20swit?=
 =?UTF-8?q?iching=C2=A0static/shared=C2=A0with=C2=A0MSVC=C2=A0in=C2=A0INST?=
 =?UTF-8?q?ALL.md.=20It=20should=20be=20fixed,=20putting=20it=20for=20refe?=
 =?UTF-8?q?rence=20-=C2=A0HWLOC=C2=A0is=C2=A0default=C2=A0OFF=C2=A0on?=
 =?UTF-8?q?=C2=A0win/mac=C2=A0and=C2=A0force=C2=A0set=C2=A0off=C2=A0when?=
 =?UTF-8?q?=C2=A0turning=C2=A0on?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 .github/workflows/windows-build.yml |  2 +-
 CMakeLists.txt                      | 46 ++++++++++++++++-------------
 INSTALL.md                          | 10 +++++--
 3 files changed, 34 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml
index ccd0d978eeb..3eae206f6a9 100644
--- a/.github/workflows/windows-build.yml
+++ b/.github/workflows/windows-build.yml
@@ -51,7 +51,7 @@ jobs:
         $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
         mkdir build
         cd build
-        cmake  -DCMAKE_CXX_FLAGS=/bigobj -DCMAKE_CXX_FLAGS_DEBUG="/MDd /Zi /Ob1 /Od /RTC1" -DGINKGO_BUILD_CUDA=OFF -DGINKGO_BUILD_OMP=OFF ..
+        cmake  -DCMAKE_CXX_FLAGS=/bigobj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_CXX_FLAGS_DEBUG="/MDd /Zi /Ob1 /Od /RTC1" -DGINKGO_BUILD_CUDA=OFF -DGINKGO_BUILD_OMP=OFF ..
         cmake --build . -j4 --config ${{ matrix.config.build_type }}
         ctest . -C ${{ matrix.config.build_type }} --output-on-failure
     - name: install
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ddcfaa55b99..80b5a3a092b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -73,7 +73,11 @@ set(GINKGO_HIP_AMDGPU "" CACHE STRING
     "The amdgpu_target(s) variable passed to hipcc. The default is none (auto).")
 option(GINKGO_JACOBI_FULL_OPTIMIZATIONS "Use all the optimizations for the CUDA Jacobi algorithm" OFF)
 option(BUILD_SHARED_LIBS "Build shared (.so, .dylib, .dll) libraries" ON)
-option(GINKGO_BUILD_HWLOC "Build Ginkgo with HWLOC. Default is ON. If a system HWLOC is not found, then we try to build it ourselves. Switch this OFF to disable HWLOC." ON)
+if(MSVC OR WIN32 OR CYGWIN OR APPLE)
+    option(GINKGO_BUILD_HWLOC "Build Ginkgo with HWLOC. Default is OFF. Ginkgo does not support HWLOC on Windows/MacOS" OFF)
+else()
+    option(GINKGO_BUILD_HWLOC "Build Ginkgo with HWLOC. Default is ON. If a system HWLOC is not found, then we try to build it ourselves. Switch this OFF to disable HWLOC." ON)
+endif()
 option(GINKGO_INSTALL_RPATH "Set the RPATH when installing its libraries." ON)
 option(GINKGO_INSTALL_RPATH_ORIGIN "Add $ORIGIN (Linux) or @loader_path (MacOS) to the installation RPATH." ON)
 option(GINKGO_INSTALL_RPATH_DEPENDENCIES "Add dependencies to the installation RPATH." OFF)
@@ -169,6 +173,11 @@ if(PAPI_sde_FOUND)
     set(GINKGO_HAVE_PAPI_SDE 1)
 endif()
 
+# Switch off HWLOC for Windows and MacOS
+if(GINKGO_BUILD_HWLOC AND (MSVC OR WIN32 OR CYGWIN OR APPLE))
+    set(GINKGO_BUILD_HWLOC OFF CACHE BOOL "Build Ginkgo with HWLOC. Default is OFF. Ginkgo does not support HWLOC on Windows/MacOS" FORCE)
+    message(WARNING "Ginkgo does not support HWLOC on Windows/MacOS, switch GINKGO_BUILD_HWLOC to OFF")
+endif()
 if(GINKGO_BUILD_HWLOC)
     # By default always use external HWLOC
     set(GINKGO_USE_EXTERNAL_HWLOC 1)
@@ -178,10 +187,6 @@ else()
     set(GINKGO_HAVE_HWLOC 0)
     message(STATUS "HWLOC is being forcibly switched off")
 endif()
-# Switch off HWLOC for Windows and MacOS
-if(MSVC OR WIN32 OR CYGWIN OR APPLE)
-    set(GINKGO_HAVE_HWLOC 0)
-endif()
 
 # We keep using NVCC/HCC for consistency with previous releases even if AMD
 # updated everything to use NVIDIA/AMD in ROCM 4.1
@@ -211,6 +216,21 @@ if(GINKGO_BUILD_HIP)
 endif()
 
 
+if(MSVC)
+    # This is modified from
+    # https://gitlab.kitware.com/cmake/community/wikis/FAQ#dynamic-replace
+    include(cmake/windows_helpers.cmake)
+    if(BUILD_SHARED_LIBS)
+        ginkgo_switch_to_windows_dynamic("CXX")
+        ginkgo_switch_to_windows_dynamic("C")
+        set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE)
+    else()
+        ginkgo_switch_to_windows_static("CXX")
+        ginkgo_switch_to_windows_static("C")
+        set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS FALSE)
+    endif()
+endif()
+
 # Try to find the third party packages before using our subdirectories
 include(cmake/package_helpers.cmake)
 ginkgo_find_package(GTest "GTest::GTest;GTest::Main" FALSE 1.8.1)
@@ -222,26 +242,12 @@ if(GINKGO_HAVE_HWLOC)
         set(GINKGO_USE_EXTERNAL_HWLOC 0)
     endif()
 endif()
+# third_party needs to be after flag modification.
 add_subdirectory(third_party)    # Third-party tools and libraries
 
 # Load CMake helpers
 include(cmake/build_helpers.cmake)
 include(cmake/install_helpers.cmake)
-include(cmake/windows_helpers.cmake)
-
-# This is modified from
-# https://gitlab.kitware.com/cmake/community/wikis/FAQ#dynamic-replace
-if(MSVC)
-    if(BUILD_SHARED_LIBS)
-        ginkgo_switch_to_windows_dynamic("CXX")
-        ginkgo_switch_to_windows_dynamic("C")
-        set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE)
-    else()
-        ginkgo_switch_to_windows_static("CXX")
-        ginkgo_switch_to_windows_static("C")
-        set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS FALSE)
-    endif()
-endif()
 
 configure_file(${Ginkgo_SOURCE_DIR}/include/ginkgo/config.hpp.in
     ${Ginkgo_BINARY_DIR}/include/ginkgo/config.hpp @ONLY)
diff --git a/INSTALL.md b/INSTALL.md
index 77bc55cef3f..d85bcc48329 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -44,7 +44,8 @@ Ginkgo adds the following additional switches to control what is being built:
 *   `-DGINKGO_HIP_AMDGPU="gpuarch1;gpuarch2"` the amdgpu_target(s) variable
     passed to hipcc for the `hcc` HIP backend. The default is none (auto).
 *   `-DGINKGO_BUILD_HWLOC={ON, OFF}` builds Ginkgo with HWLOC. If system HWLOC
-    is not found, Ginkgo will try to build it. Default is `ON`.
+    is not found, Ginkgo will try to build it. Default is `ON` on Linux. Ginkgo
+    does not support HWLOC on Windows/MacOS, so the default is `OFF` on Windows/MacOS.
 *   `-DGINKGO_BUILD_DOC={ON, OFF}` creates an HTML version of Ginkgo's documentation
     from inline comments in the code. The default is `OFF`.
 *   `-DGINKGO_DOC_GENERATE_EXAMPLES={ON, OFF}` generates the documentation of examples
@@ -99,7 +100,7 @@ Ginkgo adds the following additional switches to control what is being built:
     list of architectures. Supported values are:
 
     *   `Auto`
-    *   `Kepler`, `Maxwell`, `Pascal`, `Volta`, `Turing` `Ampere`
+    *   `Kepler`, `Maxwell`, `Pascal`, `Volta`, `Turing`, `Ampere`
     *   `CODE`, `CODE(COMPUTE)`, `(COMPUTE)`
 
     `Auto` will automatically detect the present CUDA-enabled GPU architectures
@@ -175,7 +176,10 @@ Depending on the configuration settings, some manual work might be required:
     2. `ld` issue (If building Ginkgo as static library, this is not needed)
       * `cmake -DGINKGO_COMPILER_FLAGS="-Wpedantic -O1" <other parameters> <source_folder>` (`GINKGO_COMPILER_FLAGS` is `-Wpedantic` by default)
       * add `-O1` in the environement variable `CXX_FLAGS` or `CMAKE_CXX_FLAGS`
-* Build Ginkgo in _MinGW_:
+* Possible issue when switching static/shared of Ginkgo with MSVC in the same build directory:\
+  If an issue occurs from mixing MD/MT runtime library when enabling `GINKGO_BUILD_BENCHMARKS`, it means the third-party flags are not updated correctly.
+  To update the third party flags, turn off `GINKGO_SKIP_DEPENDENCY_UPDATE` (`-DGINKGO_SKIP_DEPENDENCY_UPDATE=OFF`).
+* Build Ginkgo in _MinGW_:\
   If encountering the issue `cc1plus.exe: out of memory allocating 65536 bytes`, please follow the workaround in
   [reference](https://www.intel.com/content/www/us/en/programmable/support/support-resources/knowledge-base/embedded/2016/cc1plus-exe--out-of-memory-allocating-65536-bytes.html),
   or trying to compile ginkgo again might work.

From 11429b3e8e4c3c18c293e00bd0fedccd61503a19 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 7 Apr 2021 17:21:16 +0200
Subject: [PATCH 0623/1008] add status update jobs

---
 .gitlab-ci.yml | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 242de328838..715db863913 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,4 +1,5 @@
 stages:
+  - init-status
   - sync
   - build
   - code_quality
@@ -8,6 +9,7 @@ stages:
   - benchmark-cuda
   - benchmark-omp
   - benchmark-reference
+  - on-success
   - on-failure
 
 # Templates with reasonable defaults for builds and tests
@@ -141,6 +143,70 @@ stages:
   except:
       - schedules
 
+status_pending:
+  stage: init-status
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  variables:
+    STATUS_CONTEXT: "quick"
+  script: |
+    curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${BOT_STATUS_TOKEN}" \
+    https://api.github.com/repos/ginkgo-project/ginkgo/statuses/${CI_COMMIT_SHA} \
+    -d "{\"state\":\"pending\",\"context\":\"ci/gitlab/${STATUS_CONTEXT}\",\"target_url\":\"${CI_PIPELINE_URL}\"}"
+  only:
+    variables:
+      - $RUN_CI_TAG
+  except:
+    refs:
+      - develop
+      - master
+      - tags    
+  tags:
+    - private_ci
+    - cpu
+
+status_success:
+  stage: on-success
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  variables:
+    STATUS_CONTEXT: "quick"
+  script: |
+    curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${BOT_STATUS_TOKEN}" \
+    https://api.github.com/repos/ginkgo-project/ginkgo/statuses/${CI_COMMIT_SHA} \
+    -d "{\"state\":\"success\",\"context\":\"ci/gitlab/${STATUS_CONTEXT}\",\"target_url\":\"${CI_PIPELINE_URL}\"}"
+  only:
+    variables:
+      - $RUN_CI_TAG
+  except:
+    refs:
+      - develop
+      - master
+      - tags
+  tags:
+    - private_ci
+    - cpu
+
+status_failure:
+  stage: on-failure
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  variables:
+    STATUS_CONTEXT: "quick"
+  script: |
+    curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${BOT_STATUS_TOKEN}" \
+    https://api.github.com/repos/ginkgo-project/ginkgo/statuses/${CI_COMMIT_SHA} \
+    -d "{\"state\":\"failure\",\"context\":\"ci/gitlab/${STATUS_CONTEXT}\",\"target_url\":\"${CI_PIPELINE_URL}\"}"
+  when: on_failure
+  only:
+    variables:
+      - $RUN_CI_TAG
+  except:
+    refs:
+      - develop
+      - master
+      - tags
+  tags:
+    - private_ci
+    - cpu
+
 sync:
   stage: sync
   image: localhost:5000/gko-nocuda-gnu9-llvm8

From fe3deff154e55a393b0a251d95f8df8e2e686daf Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Sat, 10 Apr 2021 21:19:47 +0800
Subject: [PATCH 0624/1008] add trigger-pipeline and move condition

---
 .gitlab-ci.yml        | 317 ++++++++++++++++++------------------------
 .gitlab-condition.yml |  32 +++++
 2 files changed, 166 insertions(+), 183 deletions(-)
 create mode 100644 .gitlab-condition.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 715db863913..8a5984c1683 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,7 @@
 stages:
   - init-status
   - sync
+  - trigger_pipeline
   - build
   - code_quality
   - deploy
@@ -9,8 +10,11 @@ stages:
   - benchmark-cuda
   - benchmark-omp
   - benchmark-reference
-  - on-success
   - on-failure
+  - finalize-status
+
+include:
+  - local: '.gitlab-condition.yml'
 
 # Templates with reasonable defaults for builds and tests
 .variables_template: &default_variables
@@ -152,20 +156,15 @@ status_pending:
     curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${BOT_STATUS_TOKEN}" \
     https://api.github.com/repos/ginkgo-project/ginkgo/statuses/${CI_COMMIT_SHA} \
     -d "{\"state\":\"pending\",\"context\":\"ci/gitlab/${STATUS_CONTEXT}\",\"target_url\":\"${CI_PIPELINE_URL}\"}"
-  only:
-    variables:
-      - $RUN_CI_TAG
-  except:
-    refs:
-      - develop
-      - master
-      - tags    
+  extends:
+    - .pr_condition
+  dependencies: []
   tags:
     - private_ci
     - cpu
 
 status_success:
-  stage: on-success
+  stage: finalize-status
   image: localhost:5000/gko-nocuda-gnu9-llvm8
   variables:
     STATUS_CONTEXT: "quick"
@@ -173,20 +172,15 @@ status_success:
     curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${BOT_STATUS_TOKEN}" \
     https://api.github.com/repos/ginkgo-project/ginkgo/statuses/${CI_COMMIT_SHA} \
     -d "{\"state\":\"success\",\"context\":\"ci/gitlab/${STATUS_CONTEXT}\",\"target_url\":\"${CI_PIPELINE_URL}\"}"
-  only:
-    variables:
-      - $RUN_CI_TAG
-  except:
-    refs:
-      - develop
-      - master
-      - tags
+  extends:
+    - .pr_condition
+  dependencies: []
   tags:
     - private_ci
     - cpu
 
 status_failure:
-  stage: on-failure
+  stage: finalize-status
   image: localhost:5000/gko-nocuda-gnu9-llvm8
   variables:
     STATUS_CONTEXT: "quick"
@@ -195,14 +189,9 @@ status_failure:
     https://api.github.com/repos/ginkgo-project/ginkgo/statuses/${CI_COMMIT_SHA} \
     -d "{\"state\":\"failure\",\"context\":\"ci/gitlab/${STATUS_CONTEXT}\",\"target_url\":\"${CI_PIPELINE_URL}\"}"
   when: on_failure
-  only:
-    variables:
-      - $RUN_CI_TAG
-  except:
-    refs:
-      - develop
-      - master
-      - tags
+  extends:
+    - .pr_condition
+  dependencies: []
   tags:
     - private_ci
     - cpu
@@ -230,6 +219,40 @@ sync:
     - private_ci
     - cpu
 
+trigger_pipeline:
+  stage: trigger_pipeline
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  variables:
+    STATUS_CONTEXT: "quick"
+  script:
+    - PR_ID=$(curl "https://api.github.com/search/issues?q=sha:${CI_COMMIT_SHA}"
+      | jq '.items[0].number')
+    - |
+      if [[ "${PR_ID}" != "null" ]]; then
+        echo "Find the corresponding Pull Request - ${PR_ID}"
+        echo "Check whether the PR contains ST:ready-to-merge or ST:run-full-test label"
+        ENABLE_FULL_PIPELINE=$(curl -X GET -s -H "Accept: application/vnd.github.v3+json" \
+          "https://api.github.com/repos/ginkgo-project/ginkgo/issues/${PR_ID}" | jq -r \
+          'any( [.labels | .[] | .name ] | .[] ; . == "ST:ready-to-merge" or . == "ST:run-full-test")')
+        if [[ "${ENABLE_FULL_PIPELINE}" == "true" ]]; then
+          echo "trigger full pipeline"
+          curl -X POST -F token=${CI_JOB_TOKEN} -F "ref=${CI_COMMIT_REF_NAME}" -F "variables[STATUS_CONTEXT]=full" \
+            https://gitlab.com/api/v4/projects/6431537/trigger/pipeline
+        else
+          echo "does not contain required label"
+        fi
+      else
+        echo "Can not find the corresponding Pull Request"
+      fi
+  extends:
+    - .pr_condition
+  only:
+    variables:
+      - $RUN_CI_TAG && $STATUS_CONTEXT == "quick"
+  dependencies: []
+  tags:
+    - private_ci
+    - cpu
 
 # Build jobs
 # Job with example runs.
@@ -244,9 +267,8 @@ build/cuda90/gcc/all/debug/shared:
     FAST_TESTS: "ON"
     RUN_EXAMPLES: "ON"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .quick_test_condition
   tags:
     - private_ci
     - cuda
@@ -264,9 +286,8 @@ build/cuda90/clang/all/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cuda
@@ -284,9 +305,8 @@ build/cuda91/gcc/all/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cuda
@@ -303,9 +323,8 @@ build/cuda91/clang/all/release/shared:
     BUILD_CUDA: "ON"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cuda
@@ -323,9 +342,8 @@ build/cuda92/gcc/all/release/debug:
     BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cuda
@@ -346,9 +364,8 @@ build/cuda92/intel/cuda/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .quick_test_condition
   tags:
     - private_ci
     - cuda
@@ -388,9 +405,8 @@ build/cuda100/gcc/all/debug/shared:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .quick_test_condition
   tags:
     - private_ci
     - cuda
@@ -409,9 +425,8 @@ build/cuda100/clang/all/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cuda
@@ -428,9 +443,8 @@ build/cuda100/intel/cuda/release/shared:
     BUILD_CUDA: "ON"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cuda
@@ -448,9 +462,8 @@ build/cuda101/gcc/all/debug/shared:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cuda
@@ -469,9 +482,8 @@ build/cuda101/clang/all/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cuda
@@ -489,9 +501,8 @@ build/clang-cuda101/gcc/all/release/shared:
     BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .quick_test_condition
   tags:
     - private_ci
     - cuda
@@ -511,9 +522,8 @@ build/clang-cuda101/clang/cuda/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cuda
@@ -534,9 +544,8 @@ build/cuda102/gcc/all/debug/shared:
     FAST_TESTS: "ON"
     BUILD_HWLOC: "OFF"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cuda
@@ -556,9 +565,8 @@ build/cuda102/clang/all/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cuda
@@ -577,9 +585,8 @@ build/cuda102/intel/cuda/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cuda
@@ -596,9 +603,8 @@ build/cuda110/gcc/cuda/debug/shared:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cuda
@@ -616,9 +622,8 @@ build/cuda110/clang/cuda/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cuda
@@ -637,9 +642,8 @@ build/cuda110/intel/cuda/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .quick_test_condition
   tags:
     - private_ci
     - cuda
@@ -656,9 +660,8 @@ build/amd/gcc/hip/debug/shared:
     RUN_EXAMPLES: "ON"
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .quick_test_condition
   tags:
     - private_ci
     - amd
@@ -675,9 +678,8 @@ build/amd/clang/hip/release/static:
     BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .quick_test_condition
   tags:
     - private_ci
     - amd
@@ -712,9 +714,8 @@ build/nocuda/gcc/core/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     BUILD_HWLOC: "OFF"
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cpu
@@ -728,9 +729,8 @@ build/nocuda/clang/core/release/shared:
     CXX_COMPILER: "clang++"
     BUILD_REFERENCE: "OFF"
     BUILD_TYPE: "Release"
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cpu
@@ -745,9 +745,8 @@ build/nocuda/intel/core/debug/shared:
     BUILD_REFERENCE: "OFF"
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .quick_test_condition
   tags:
     - private_ci
     - cuda
@@ -760,9 +759,8 @@ build/nocuda/gcc/omp/release/shared:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_TYPE: "Release"
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cpu
@@ -778,9 +776,8 @@ build/nocuda/clang/omp/debug/static:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   tags:
     - private_ci
     - cpu
@@ -795,9 +792,8 @@ build/nocuda/intel/omp/release/static:
     BUILD_OMP: "ON"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .quick_test_condition
   tags:
     - private_ci
     - cuda
@@ -814,9 +810,8 @@ build/dpcpp/cpu/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     SYCL_DEVICE_TYPE: "CPU"
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .quick_test_condition
   tags:
     - private_ci
     - cuda
@@ -833,9 +828,8 @@ warnings:
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     CXX_FLAGS: "-Werror=pedantic -pedantic-errors"
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   dependencies: []
   allow_failure: yes
   tags:
@@ -854,9 +848,8 @@ no-circular-deps:
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     EXTRA_CMAKE_FLAGS: '-DGINKGO_CHECK_CIRCULAR_DEPS=on'
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   dependencies: []
   allow_failure: no
   tags:
@@ -873,9 +866,8 @@ subdir-build:
     <<: *default_variables
     BUILD_OMP: "ON"
     CI_PROJECT_PATH_SUFFIX: "/test_subdir"
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   dependencies: []
   allow_failure: no
   tags:
@@ -891,9 +883,8 @@ export-build:
     <<: *default_variables
     BUILD_OMP: "ON"
     EXPORT_BUILD_DIR: "ON"
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   dependencies: []
   allow_failure: no
   tags:
@@ -911,9 +902,8 @@ clang-tidy:
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     EXTRA_CMAKE_FLAGS: '-DGINKGO_WITH_CLANG_TIDY=ON'
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   dependencies: []
   allow_failure: yes
   tags:
@@ -931,9 +921,8 @@ iwyu:
     BUILD_CUDA: "ON"
     BUILD_CUDA: "HIP"
     EXTRA_CMAKE_FLAGS: '-DGINKGO_WITH_IWYU=ON'
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .full_test_condition
   dependencies: []
   allow_failure: yes
   tags:
@@ -974,9 +963,8 @@ sonarqube_cov_:
       - develop
       - master
       - tags
-  only:
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .quick_test_condition
   tags:
     - private_ci
     - cuda
@@ -997,13 +985,8 @@ sonarqube_cov:
       -Dsonar.branch.name=${CI_COMMIT_REF_NAME}
     - bash <(curl -s https://codecov.io/bash) -f "\!*examples*" -f "\!*third_party*" -f "\!*c\\+\\+*" -f "\!*benchmark*"
   dependencies: []
-  only:
-    refs:
-      - develop
-      - master
-      - tags
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .deploy_condition
   tags:
     - private_ci
     - cuda
@@ -1044,15 +1027,8 @@ gh-pages:
     - git diff --quiet HEAD ||
       (git commit -m "Update documentation from ${CURRENT_SHA}" && git push)
   dependencies: []
-  only:
-    refs:
-      - develop
-      - master
-      - tags
-    variables:
-      - $RUN_CI_TAG
-  except:
-      - schedules
+  extends:
+    - .deploy_condition
   tags:
     - private_ci
     - cpu
@@ -1070,13 +1046,8 @@ threadsanitizer:
         -DCTEST_MEMORYCHECK_SANITIZER_OPTIONS=ignore_noninstrumented_modules=1
         --timeout 6000
   dependencies: []
-  only:
-    refs:
-      - master
-      - develop
-      - tags
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .deploy_condition
   tags:
     - private_ci
     - cuda
@@ -1090,13 +1061,8 @@ leaksanitizer:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=LSAN
       -DCTEST_MEMORYCHECK_TYPE=LeakSanitizer
   dependencies: []
-  only:
-    refs:
-      - master
-      - develop
-      - tags
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .deploy_condition
   tags:
     - private_ci
     - cuda
@@ -1110,13 +1076,8 @@ addresssanitizer:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=ASAN
       -DCTEST_MEMORYCHECK_TYPE=AddressSanitizer
   dependencies: []
-  only:
-    refs:
-      - master
-      - develop
-      - tags
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .deploy_condition
   tags:
     - private_ci
     - cuda
@@ -1132,13 +1093,8 @@ undefinedsanitizer:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=UBSAN
       -DCTEST_MEMORYCHECK_TYPE=UndefinedBehaviorSanitizer
   dependencies: []
-  only:
-    refs:
-      - master
-      - develop
-      - tags
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .deploy_condition
   tags:
     - private_ci
     - cuda
@@ -1152,13 +1108,8 @@ cudamemcheck:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=RelWithDebInfo
       -DCTEST_MEMORYCHECK_TYPE=CudaMemcheck
   dependencies: []
-  only:
-    refs:
-      - master
-      - develop
-      - tags
-    variables:
-      - $RUN_CI_TAG
+  extends:
+    - .deploy_condition
   tags:
     - private_ci
     - cuda
diff --git a/.gitlab-condition.yml b/.gitlab-condition.yml
new file mode 100644
index 00000000000..b9162544bca
--- /dev/null
+++ b/.gitlab-condition.yml
@@ -0,0 +1,32 @@
+.pr_condition:
+  only:
+    variables:
+      - $RUN_CI_TAG
+  except:
+    refs:
+      - develop
+      - master
+      - tags
+
+.full_test_condition:
+  only:
+      variables:
+        - $RUN_CI_TAG && $STATUS_CONTEXT == "full"
+        - $RUN_CI_TAG && ($CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "develop")
+        - $RUN_CI_TAG && $CI_COMMIT_TAG
+
+.quick_test_condition:
+  only:
+    variables:
+      - $RUN_CI_TAG && $STATUS_CONTEXT == null
+
+.deploy_condition:
+  only:
+    refs:
+      - develop
+      - master
+      - tags
+    variables:
+      - $RUN_CI_TAG
+  except:
+      - schedules

From 8cacdf9b91bfe16ccfa9127f79833e667219b697 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 14 Apr 2021 15:47:03 +0800
Subject: [PATCH 0625/1008] extract image/tag, and move components to .gitlab

- only export CUDA_VISIBLE_DEVICES when define CUDA_ARCH
- add exit_code such that pipeline retry also activates the finalize-status

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 .gitlab-ci.yml                                | 513 ++++++------------
 .../condition.yml                             |   4 +
 .gitlab/image.yml                             |  82 +++
 3 files changed, 248 insertions(+), 351 deletions(-)
 rename .gitlab-condition.yml => .gitlab/condition.yml (88%)
 create mode 100644 .gitlab/image.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8a5984c1683..cc5836e1809 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,7 +14,8 @@ stages:
   - finalize-status
 
 include:
-  - local: '.gitlab-condition.yml'
+  - local: '.gitlab/condition.yml'
+  - local: '.gitlab/image.yml'
 
 # Templates with reasonable defaults for builds and tests
 .variables_template: &default_variables
@@ -40,7 +41,6 @@ include:
 .before_script_template: &default_before_script
   - export NUM_CORES=${CI_PARALLELISM}
   - export OMP_NUM_THREADS=${NUM_CORES}
-  - export CUDA_VISIBLE_DEVICES=$((RANDOM % 2))
   - export CCACHE_DIR=${CCACHE_DIR}
   - export CCACHE_MAXSIZE=${CCACHE_MAXSIZE}
 
@@ -62,6 +62,7 @@ include:
     - if [ -n "${CUDA_ARCH}" ]; then
       CUDA_ARCH_STR=-DGINKGO_CUDA_ARCHITECTURES=${CUDA_ARCH};
       CUDA_HOST_STR=-DCMAKE_CUDA_HOST_COMPILER=$(which ${CXX_COMPILER});
+      export CUDA_VISIBLE_DEVICES=$((RANDOM % 2));
       fi
     - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then export SYCL_DEVICE_TYPE; fi
     - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
@@ -95,6 +96,7 @@ include:
     - if [ -n "${CUDA_ARCH}" ]; then
       CUDA_ARCH_STR=-DGINKGO_CUDA_ARCHITECTURES=${CUDA_ARCH};
       CUDA_HOST_STR=-DCMAKE_CUDA_HOST_COMPILER=$(which ${CXX_COMPILER});
+      export CUDA_VISIBLE_DEVICES=$((RANDOM % 2));
       fi
     - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then export SYCL_DEVICE_TYPE; fi
     - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
@@ -149,56 +151,54 @@ include:
 
 status_pending:
   stage: init-status
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .pr_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     STATUS_CONTEXT: "quick"
   script: |
     curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${BOT_STATUS_TOKEN}" \
     https://api.github.com/repos/ginkgo-project/ginkgo/statuses/${CI_COMMIT_SHA} \
     -d "{\"state\":\"pending\",\"context\":\"ci/gitlab/${STATUS_CONTEXT}\",\"target_url\":\"${CI_PIPELINE_URL}\"}"
-  extends:
-    - .pr_condition
-  dependencies: []
-  tags:
-    - private_ci
-    - cpu
 
 status_success:
   stage: finalize-status
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .pr_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     STATUS_CONTEXT: "quick"
+  # we always exit with the code 3 such that it will process when retrying
   script: |
     curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${BOT_STATUS_TOKEN}" \
     https://api.github.com/repos/ginkgo-project/ginkgo/statuses/${CI_COMMIT_SHA} \
     -d "{\"state\":\"success\",\"context\":\"ci/gitlab/${STATUS_CONTEXT}\",\"target_url\":\"${CI_PIPELINE_URL}\"}"
-  extends:
-    - .pr_condition
-  dependencies: []
-  tags:
-    - private_ci
-    - cpu
+    exit 3
+  allow_failure:
+    exit_codes: 3
 
 status_failure:
   stage: finalize-status
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .pr_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     STATUS_CONTEXT: "quick"
+  # we always exit with the code 3 such that it will process when retrying
   script: |
     curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${BOT_STATUS_TOKEN}" \
     https://api.github.com/repos/ginkgo-project/ginkgo/statuses/${CI_COMMIT_SHA} \
     -d "{\"state\":\"failure\",\"context\":\"ci/gitlab/${STATUS_CONTEXT}\",\"target_url\":\"${CI_PIPELINE_URL}\"}"
+    exit 3
   when: on_failure
-  extends:
-    - .pr_condition
-  dependencies: []
-  tags:
-    - private_ci
-    - cpu
+  allow_failure:
+    exit_codes: 3
+
 
 sync:
   stage: sync
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     GIT_STRATEGY: none
     PRIVATE_REPO: git@gitlab.com:ginkgo-project/ginkgo.git
@@ -215,13 +215,12 @@ sync:
     - develop
   except:
     - schedules
-  tags:
-    - private_ci
-    - cpu
 
 trigger_pipeline:
   stage: trigger_pipeline
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .pr_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     STATUS_CONTEXT: "quick"
   script:
@@ -231,7 +230,7 @@ trigger_pipeline:
       if [[ "${PR_ID}" != "null" ]]; then
         echo "Find the corresponding Pull Request - ${PR_ID}"
         echo "Check whether the PR contains ST:ready-to-merge or ST:run-full-test label"
-        ENABLE_FULL_PIPELINE=$(curl -X GET -s -H "Accept: application/vnd.github.v3+json" \
+        ENABLE_FULL_PIPELINE=$(curl -X GET -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${BOT_STATUS_TOKEN}" \
           "https://api.github.com/repos/ginkgo-project/ginkgo/issues/${PR_ID}" | jq -r \
           'any( [.labels | .[] | .name ] | .[] ; . == "ST:ready-to-merge" or . == "ST:run-full-test")')
         if [[ "${ENABLE_FULL_PIPELINE}" == "true" ]]; then
@@ -244,21 +243,18 @@ trigger_pipeline:
       else
         echo "Can not find the corresponding Pull Request"
       fi
-  extends:
-    - .pr_condition
+  # Override variables condition
   only:
     variables:
       - $RUN_CI_TAG && $STATUS_CONTEXT == "quick"
-  dependencies: []
-  tags:
-    - private_ci
-    - cpu
 
 # Build jobs
 # Job with example runs.
 build/cuda90/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda90-gnu5-llvm39
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda90-gnu5-llvm39
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -267,16 +263,12 @@ build/cuda90/gcc/all/debug/shared:
     FAST_TESTS: "ON"
     RUN_EXAMPLES: "ON"
     CUDA_ARCH: 35
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda90/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda90-gnu5-llvm39
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda90-gnu5-llvm39
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -286,17 +278,13 @@ build/cuda90/clang/all/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # cuda 9.1 and friends
 build/cuda91/gcc/all/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda91-gnu6-llvm40
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda91-gnu6-llvm40
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -305,16 +293,12 @@ build/cuda91/gcc/all/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda91/clang/all/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda91-gnu6-llvm40
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda91-gnu6-llvm40
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -323,18 +307,14 @@ build/cuda91/clang/all/release/shared:
     BUILD_CUDA: "ON"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 
 # cuda 9.2 and friends
-build/cuda92/gcc/all/release/debug:
+build/cuda92/gcc/all/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda92-gnu7-llvm50-intel2017
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -342,18 +322,14 @@ build/cuda92/gcc/all/release/debug:
     BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # Make sure that our jobs run when HWLOC is
 # forcibly switched off
 build/cuda92/intel/cuda/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda92-gnu7-llvm50-intel2017
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -364,12 +340,6 @@ build/cuda92/intel/cuda/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # Build CUDA NVIDIA without omp
 build/cuda92/intel/cuda_wo_omp/release/shared:
@@ -396,7 +366,9 @@ build/cuda92/intel/cuda_wo_omp/release/shared:
 # third-party HWLOC.
 build/cuda100/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -405,16 +377,12 @@ build/cuda100/gcc/all/debug/shared:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     CUDA_ARCH: 35
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda100/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -425,16 +393,12 @@ build/cuda100/clang/all/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda100/intel/cuda/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda100-gnu7-llvm60-intel2018
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -443,17 +407,13 @@ build/cuda100/intel/cuda/release/shared:
     BUILD_CUDA: "ON"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # cuda 10.1 and friends
 build/cuda101/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -462,16 +422,12 @@ build/cuda101/gcc/all/debug/shared:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda101/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -482,17 +438,13 @@ build/cuda101/clang/all/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # clang-cuda with cuda 10.1 and friends
 build/clang-cuda101/gcc/all/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda101-gnu8-llvm10-intel2019
   variables:
     <<: *default_variables
     CUDA_COMPILER: "clang++"
@@ -501,16 +453,12 @@ build/clang-cuda101/gcc/all/release/shared:
     BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/clang-cuda101/clang/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda101-gnu8-llvm10-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -522,19 +470,15 @@ build/clang-cuda101/clang/cuda/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # cuda 10.2 and friends
 
 # works when there is no hwloc and tpl hwloc is also switched off.
 build/cuda102/gcc/all/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -544,17 +488,13 @@ build/cuda102/gcc/all/debug/shared:
     FAST_TESTS: "ON"
     BUILD_HWLOC: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # Use TPL hwloc when no system hwloc is available
 build/cuda102/clang/all/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -565,16 +505,12 @@ build/cuda102/clang/all/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda102/intel/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda102-gnu8-llvm8-intel2019
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -585,17 +521,13 @@ build/cuda102/intel/cuda/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # cuda 11.0 and friends
 build/cuda110/gcc/cuda/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -603,16 +535,12 @@ build/cuda110/gcc/cuda/debug/shared:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda110/clang/cuda/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -622,16 +550,12 @@ build/cuda110/clang/cuda/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 build/cuda110/intel/cuda/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda110-gnu9-llvm9-intel2020
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -642,17 +566,13 @@ build/cuda110/intel/cuda/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # HIP AMD
 build/amd/gcc/hip/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-amd-gnu8-llvm7
+  extends:
+    - .quick_test_condition
+    - .use_gko-amd-gnu8-llvm7
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
@@ -660,16 +580,12 @@ build/amd/gcc/hip/debug/shared:
     RUN_EXAMPLES: "ON"
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - amd
-    - gpu
 
 build/amd/clang/hip/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-amd-gnu8-llvm7
+  extends:
+    - .quick_test_condition
+    - .use_gko-amd-gnu8-llvm7
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -678,12 +594,6 @@ build/amd/clang/hip/release/static:
     BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - amd
-    - gpu
 
 # Build HIP AMD without omp
 build/amd/clang/hip_wo_omp/release/shared:
@@ -706,7 +616,9 @@ build/amd/clang/hip_wo_omp/release/shared:
 # no cuda but latest gcc and clang
 build/nocuda/gcc/core/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .quick_test_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_REFERENCE: "OFF"
@@ -714,30 +626,24 @@ build/nocuda/gcc/core/debug/static:
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
     BUILD_HWLOC: "OFF"
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cpu
 
 build/nocuda/clang/core/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .quick_test_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
     CXX_COMPILER: "clang++"
     BUILD_REFERENCE: "OFF"
     BUILD_TYPE: "Release"
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cpu
 
 build/nocuda/intel/core/debug/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .quick_test_condition
+    - .use_gko-nocuda-gnu9-llvm8-intel
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -745,29 +651,22 @@ build/nocuda/intel/core/debug/shared:
     BUILD_REFERENCE: "OFF"
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - cpu
 
 build/nocuda/gcc/omp/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .quick_test_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_TYPE: "Release"
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cpu
 
 build/nocuda/clang/omp/debug/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .full_test_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
@@ -776,15 +675,12 @@ build/nocuda/clang/omp/debug/static:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
-  extends:
-    - .full_test_condition
-  tags:
-    - private_ci
-    - cpu
 
 build/nocuda/intel/omp/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .quick_test_condition
+    - .use_gko-nocuda-gnu9-llvm8-intel
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
@@ -792,16 +688,12 @@ build/nocuda/intel/omp/release/static:
     BUILD_OMP: "ON"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - cpu
 
 build/dpcpp/cpu/release/static:
   <<: *default_build_with_test
-  image: localhost:5000/gko-oneapi
+  extends:
+    - .quick_test_condition
+    - .use_gko-oneapi
   variables:
     <<: *default_variables
     C_COMPILER: "gcc"
@@ -810,131 +702,99 @@ build/dpcpp/cpu/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     SYCL_DEVICE_TYPE: "CPU"
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - cpu
 
 # Job with important warnings as error
 warnings:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     CXX_FLAGS: "-Werror=pedantic -pedantic-errors"
-  extends:
-    - .full_test_condition
-  dependencies: []
   allow_failure: yes
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # Ensure kernel modules do not depend on core
 no-circular-deps:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     EXTRA_CMAKE_FLAGS: '-DGINKGO_CHECK_CIRCULAR_DEPS=on'
-  extends:
-    - .full_test_condition
-  dependencies: []
   allow_failure: no
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # Ensure Ginkgo builds from a subdirectory
 subdir-build:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .full_test_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
     CI_PROJECT_PATH_SUFFIX: "/test_subdir"
-  extends:
-    - .full_test_condition
-  dependencies: []
   allow_failure: no
-  tags:
-    - private_ci
-    - cpu
 
 # Ensure Ginkgo can be used when exporting the build directory
 export-build:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .full_test_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
     EXPORT_BUILD_DIR: "ON"
-  extends:
-    - .full_test_condition
-  dependencies: []
   allow_failure: no
-  tags:
-    - private_ci
-    - cpu
 
 # Run clang-tidy and iwyu
 clang-tidy:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     EXTRA_CMAKE_FLAGS: '-DGINKGO_WITH_CLANG_TIDY=ON'
-  extends:
-    - .full_test_condition
-  dependencies: []
   allow_failure: yes
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 iwyu:
   <<: *default_build
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   variables:
     <<: *default_variables
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_CUDA: "HIP"
     EXTRA_CMAKE_FLAGS: '-DGINKGO_WITH_IWYU=ON'
-  extends:
-    - .full_test_condition
-  dependencies: []
   allow_failure: yes
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # Code analysis, coverage and reporting tool
 # For short living branches or PRs, try to detect an open PR
 sonarqube_cov_:
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   before_script: *default_before_script
   script:
     - PR_ID=$(curl "https://api.github.com/search/issues?q=sha:${CI_COMMIT_SHA}"
@@ -957,24 +817,19 @@ sonarqube_cov_:
       -Dsonar.cfamily.gcov.reportsPath=build/Testing/CoverageInfo
       ${sonar_branching}
     - bash <(curl -s https://codecov.io/bash) -f "\!*examples*" -f "\!*third_party*" -f "\!*c\\+\\+*" -f "\!*benchmark*"
-  dependencies: []
   except:
     refs:
       - develop
       - master
       - tags
-  extends:
-    - .quick_test_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # For long living branches, do not detect the PR. A PR would always be detected
 # (the one that was merged).
 sonarqube_cov:
   stage: code_quality
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  extends:
+    - .deploy_condition
+    - .use_gko-cuda101-gnu8-llvm7-intel2019
   before_script: *default_before_script
   script:
     - ctest -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=COVERAGE
@@ -984,19 +839,14 @@ sonarqube_cov:
       -Dsonar.cfamily.gcov.reportsPath=build/Testing/CoverageInfo
       -Dsonar.branch.name=${CI_COMMIT_REF_NAME}
     - bash <(curl -s https://codecov.io/bash) -f "\!*examples*" -f "\!*third_party*" -f "\!*c\\+\\+*" -f "\!*benchmark*"
-  dependencies: []
-  extends:
-    - .deploy_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 
 # Deploy documentation to github-pages
 gh-pages:
   stage: deploy
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .deploy_condition
+    - .use_gko-nocuda-gnu9-llvm8
   variables:
     <<: *default_variables
     PUBLIC_REPO: git@github.com:ginkgo-project/ginkgo.git
@@ -1026,17 +876,13 @@ gh-pages:
     - git add -A
     - git diff --quiet HEAD ||
       (git commit -m "Update documentation from ${CURRENT_SHA}" && git push)
-  dependencies: []
-  extends:
-    - .deploy_condition
-  tags:
-    - private_ci
-    - cpu
 
 
 threadsanitizer:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  extends:
+    - .deploy_condition
+    - .use_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - LD_PRELOAD=/usr/local/lib/libomp.so
@@ -1045,75 +891,48 @@ threadsanitizer:
         -DCTEST_MEMORYCHECK_TYPE=ThreadSanitizer
         -DCTEST_MEMORYCHECK_SANITIZER_OPTIONS=ignore_noninstrumented_modules=1
         --timeout 6000
-  dependencies: []
-  extends:
-    - .deploy_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 leaksanitizer:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  extends:
+    - .deploy_condition
+    - .use_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=LSAN
       -DCTEST_MEMORYCHECK_TYPE=LeakSanitizer
-  dependencies: []
-  extends:
-    - .deploy_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 addresssanitizer:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  extends:
+    - .deploy_condition
+    - .use_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=ASAN
       -DCTEST_MEMORYCHECK_TYPE=AddressSanitizer
-  dependencies: []
-  extends:
-    - .deploy_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 undefinedsanitizer:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  extends:
+    - .deploy_condition
+    - .use_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     # the Gold linker is required because of a linker flag issues given by UBsan
     # in the Ubuntu setup we are using.
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=UBSAN
       -DCTEST_MEMORYCHECK_TYPE=UndefinedBehaviorSanitizer
-  dependencies: []
-  extends:
-    - .deploy_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 cudamemcheck:
   stage: QoS_tools
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  extends:
+    - .deploy_condition
+    - .use_gko-cuda101-gnu8-llvm10-intel2019
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=RelWithDebInfo
       -DCTEST_MEMORYCHECK_TYPE=CudaMemcheck
-  dependencies: []
-  extends:
-    - .deploy_condition
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # Benchmark build
 .benchmark_before_script_template: &default_benchmark_before_script
@@ -1136,7 +955,8 @@ cudamemcheck:
 
 fineci-benchmark-build:
   stage: benchmark-build
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .use_gko-nocuda-gnu9-llvm8-intel
   variables:
     <<: *default_variables
     BENCHMARK_SERVER: FINECI
@@ -1175,10 +995,6 @@ fineci-benchmark-build:
     - schedules
 #    - develop
 #    - master
-  tags:
-    - private_ci
-    - cpu
-    - cuda
 
 
 # Benchmark runs
@@ -1217,7 +1033,8 @@ fineci-benchmark-build:
 
 fineci-benchmark-cuda:
   stage: benchmark-cuda
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .use_gko-nocuda-gnu9-llvm8-intel
   variables:
     <<: *default_variables
     BENCHMARK_SERVER: FINECI
@@ -1226,10 +1043,6 @@ fineci-benchmark-cuda:
     BENCHMARK_REPO: git@github.com:ginkgo-project/ginkgo-data.git
     SYSTEM_NAME: K20Xm
   <<: *default_benchmark
-  tags:
-    - private_ci
-    - cpu
-    - cuda
 
 # fineci-benchmark-omp:
 #   stage: benchmark-omp
@@ -1255,7 +1068,8 @@ fineci-benchmark-cuda:
 
 new-issue-on-failure:
   stage: on-failure
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  extends:
+    - .use_gko-nocuda-gnu9-llvm8
   script: curl --request POST "https://gitlab.com/api/v4/projects/${PROJECT_ID}/issues?private_token=${BOT_ACCESS_TOKEN}&title=Error%20in%20${CI_PROJECT_NAME}%20with%20pipeline%20${CI_PIPELINE_ID}%20for%20commit%20${CI_COMMIT_SHA}&labels&description=${CI_PIPELINE_URL}"
   when: on_failure
   only:
@@ -1263,6 +1077,3 @@ new-issue-on-failure:
       - develop
       - master
   dependencies: []
-  tags:
-    - private_ci
-    - cpu
diff --git a/.gitlab-condition.yml b/.gitlab/condition.yml
similarity index 88%
rename from .gitlab-condition.yml
rename to .gitlab/condition.yml
index b9162544bca..fffb88738a5 100644
--- a/.gitlab-condition.yml
+++ b/.gitlab/condition.yml
@@ -7,6 +7,7 @@
       - develop
       - master
       - tags
+  dependencies: []
 
 .full_test_condition:
   only:
@@ -14,11 +15,13 @@
         - $RUN_CI_TAG && $STATUS_CONTEXT == "full"
         - $RUN_CI_TAG && ($CI_COMMIT_BRANCH == "master" || $CI_COMMIT_BRANCH == "develop")
         - $RUN_CI_TAG && $CI_COMMIT_TAG
+  dependencies: []
 
 .quick_test_condition:
   only:
     variables:
       - $RUN_CI_TAG && $STATUS_CONTEXT == null
+  dependencies: []
 
 .deploy_condition:
   only:
@@ -30,3 +33,4 @@
       - $RUN_CI_TAG
   except:
       - schedules
+  dependencies: []
diff --git a/.gitlab/image.yml b/.gitlab/image.yml
new file mode 100644
index 00000000000..3fca4782061
--- /dev/null
+++ b/.gitlab/image.yml
@@ -0,0 +1,82 @@
+.use_gko-nocuda-gnu9-llvm8:
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  tags:
+    - private_ci
+    - cpu
+
+.use_gko-nocuda-gnu9-llvm8-intel:
+  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  tags:
+    - private_ci
+    - cpu
+    - cuda
+
+.use_gko-cuda90-gnu5-llvm39:
+  image: localhost:5000/gko-cuda90-gnu5-llvm39
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-cuda91-gnu6-llvm40:
+  image: localhost:5000/gko-cuda91-gnu6-llvm40
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-cuda92-gnu7-llvm50-intel2017:
+  image: localhost:5000/gko-cuda92-gnu7-llvm50-intel2017
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-cuda100-gnu7-llvm60-intel2018:
+  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-cuda101-gnu8-llvm7-intel2019:
+  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-cuda101-gnu8-llvm10-intel2019:
+  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-cuda102-gnu8-llvm8-intel2019:
+  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-cuda110-gnu9-llvm9-intel2020:
+  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
+  tags:
+    - private_ci
+    - cuda
+    - gpu
+
+.use_gko-amd-gnu8-llvm7:
+  image: localhost:5000/gko-amd-gnu8-llvm7
+  tags:
+    - private_ci
+    - amd
+    - gpu
+
+.use_gko-oneapi:
+  image: localhost:5000/gko-oneapi
+  tags:
+    - private_ci
+    - cuda
+    - cpu

From 6efd652dc497efe0b0e3c6dc25cbdc15ad2c1e56 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 16 Apr 2021 23:56:52 +0800
Subject: [PATCH 0626/1008] update wo_omp job, fix test_install_hip host

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 .gitlab-ci.yml              | 37 ++++++++++++++-----------------------
 test_install/CMakeLists.txt |  5 ++++-
 2 files changed, 18 insertions(+), 24 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cc5836e1809..dcd47e63880 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -224,12 +224,12 @@ trigger_pipeline:
   variables:
     STATUS_CONTEXT: "quick"
   script:
-    - PR_ID=$(curl "https://api.github.com/search/issues?q=sha:${CI_COMMIT_SHA}"
+    - PR_ID=$(curl -s "https://api.github.com/search/issues?q=sha:${CI_COMMIT_SHA}"
       | jq '.items[0].number')
     - |
       if [[ "${PR_ID}" != "null" ]]; then
-        echo "Find the corresponding Pull Request - ${PR_ID}"
-        echo "Check whether the PR contains ST:ready-to-merge or ST:run-full-test label"
+        echo "Finding the corresponding Pull Request - ${PR_ID}"
+        echo "Checking whether the PR contains ST:ready-to-merge or ST:run-full-test labels"
         ENABLE_FULL_PIPELINE=$(curl -X GET -s -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${BOT_STATUS_TOKEN}" \
           "https://api.github.com/repos/ginkgo-project/ginkgo/issues/${PR_ID}" | jq -r \
           'any( [.labels | .[] | .name ] | .[] ; . == "ST:ready-to-merge" or . == "ST:run-full-test")')
@@ -238,7 +238,7 @@ trigger_pipeline:
           curl -X POST -F token=${CI_JOB_TOKEN} -F "ref=${CI_COMMIT_REF_NAME}" -F "variables[STATUS_CONTEXT]=full" \
             https://gitlab.com/api/v4/projects/6431537/trigger/pipeline
         else
-          echo "does not contain required label"
+          echo "does not contain required labels"
         fi
       else
         echo "Can not find the corresponding Pull Request"
@@ -328,7 +328,7 @@ build/cuda92/gcc/all/release/shared:
 build/cuda92/intel/cuda/release/static:
   <<: *default_build_with_test
   extends:
-    - .quick_test_condition
+    - .full_test_condition
     - .use_gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
@@ -344,22 +344,18 @@ build/cuda92/intel/cuda/release/static:
 # Build CUDA NVIDIA without omp
 build/cuda92/intel/cuda_wo_omp/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-cuda92-gnu7-llvm50-intel2017
+  extends:
+    - .quick_test_condition
+    - .use_gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
     C_COMPILER: "icc"
     CXX_COMPILER: "icpc"
     BUILD_CUDA: "ON"
+    BUILD_HIP: "ON"
     BUILD_HWLOC: "OFF"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
-  only:
-    variables:
-      - $RUN_CI_TAG
-  tags:
-    - private_ci
-    - cuda
-    - gpu
 
 # cuda 10.0 and friends
 # Make sure that our jobs run when using self-installed
@@ -598,20 +594,15 @@ build/amd/clang/hip/release/static:
 # Build HIP AMD without omp
 build/amd/clang/hip_wo_omp/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-amd-gnu8-llvm7
+  extends:
+    - .full_test_condition
+    - .use_gko-amd-gnu8-llvm7
   variables:
     <<: *default_variables
     C_COMPILER: "clang"
     CXX_COMPILER: "clang++"
     BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
-  only:
-    variables:
-      - $RUN_CI_TAG
-  tags:
-    - private_ci
-    - amd
-    - gpu
 
 # no cuda but latest gcc and clang
 build/nocuda/gcc/core/debug/static:
@@ -797,10 +788,10 @@ sonarqube_cov_:
     - .use_gko-cuda101-gnu8-llvm7-intel2019
   before_script: *default_before_script
   script:
-    - PR_ID=$(curl "https://api.github.com/search/issues?q=sha:${CI_COMMIT_SHA}"
+    - PR_ID=$(curl -s "https://api.github.com/search/issues?q=sha:${CI_COMMIT_SHA}"
       | jq '.items[0].number')
     - if [[ "${PR_ID}" != "null" ]]; then
-        target_branch=$(curl
+        target_branch=$(curl -s
           "https://api.github.com/repos/ginkgo-project/ginkgo/pulls/${PR_ID}" | jq
           '.base.ref' | sed 's/"//g');
         sonar_branching="-Dsonar.pullrequest.branch=${CI_COMMIT_REF_NAME}
diff --git a/test_install/CMakeLists.txt b/test_install/CMakeLists.txt
index 4c5fc9eea58..a629f000008 100644
--- a/test_install/CMakeLists.txt
+++ b/test_install/CMakeLists.txt
@@ -64,10 +64,13 @@ if(GINKGO_BUILD_HIP)
     if (CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
         set(TESTINSTALL_CUDA_ARCH_FLAGS "${GINKGO_CUDA_ARCH_FLAGS}")
     endif()
+    if (CMAKE_CUDA_HOST_COMPILER)
+        set(TESTINSTALL_CUDA_HOST_COMPILER "-ccbin=${CMAKE_CUDA_HOST_COMPILER}")
+    endif()
     hip_add_executable(test_install_hip test_install.cpp
         HIPCC_OPTIONS "-std=c++14"
         CLANG_OPTIONS "${GINKGO_PIC_OPTION}"
-        NVCC_OPTIONS "${GINKGO_CUDA_PIC_OPTION}" "${TESTINSTALL_CUDA_ARCH_FLAGS}")
+        NVCC_OPTIONS "${GINKGO_CUDA_PIC_OPTION}" "${TESTINSTALL_CUDA_ARCH_FLAGS}" "${TESTINSTALL_CUDA_HOST_COMPILER}")
 
     target_link_libraries(test_install_hip PRIVATE Ginkgo::ginkgo)
     target_compile_definitions(test_install_hip PRIVATE HAS_HIP=1)

From 4181c893bed1a707634de2cad068631d23893363 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 21 Apr 2021 11:35:00 +0200
Subject: [PATCH 0627/1008] Add Debug CI setup to OSX-build.

---
 .github/workflows/osx.yml | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml
index de5c9d36c7d..6827c5fe476 100644
--- a/.github/workflows/osx.yml
+++ b/.github/workflows/osx.yml
@@ -1,6 +1,13 @@
 name: OSX-build
 
-on: [push]
+on:
+  push:
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
 
 jobs:
   osx-clang-omp:
@@ -15,13 +22,20 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+
     - name: setup
       run: |
         brew install libomp
+
     - name: info
       run: |
         g++ -v
         cmake --version
+
+    - name: Debug over SSH (tmate)
+      uses: mxschmitt/action-tmate@v3.5
+      if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
+
     - name: configure
       run: |
         mkdir build
@@ -29,6 +43,7 @@ jobs:
         cmake .. -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }}
         make -j8
         ctest -j10 --output-on-failure
+
     - name: install
       run: |
         cd build

From f1271f6232f357110fcacc41f137e609e8aaadc0 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 21 Apr 2021 14:10:22 +0200
Subject: [PATCH 0628/1008] Move Windows builds to separate files, add debug

+ Adds ssh debugging capability with tmate for Windows builds.
---
 .github/workflows/windows-build.yml     | 126 ------------------------
 .github/workflows/windows-cygwin.yml    |  51 ++++++++++
 .github/workflows/windows-mingw.yml     |  49 +++++++++
 .github/workflows/windows-msvc-cuda.yml |  48 +++++++++
 .github/workflows/windows-msvc-ref.yml  |  45 +++++++++
 5 files changed, 193 insertions(+), 126 deletions(-)
 delete mode 100644 .github/workflows/windows-build.yml
 create mode 100644 .github/workflows/windows-cygwin.yml
 create mode 100644 .github/workflows/windows-mingw.yml
 create mode 100644 .github/workflows/windows-msvc-cuda.yml
 create mode 100644 .github/workflows/windows-msvc-ref.yml

diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml
deleted file mode 100644
index 3eae206f6a9..00000000000
--- a/.github/workflows/windows-build.yml
+++ /dev/null
@@ -1,126 +0,0 @@
-name: Windows-build
-
-on: [push]
-
-jobs:
-  windows_cuda:
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-        - {version: "10.2.89.20191206", name: "cuda102/release/shared"}
-        - {version: "latest", name: "cuda-latest/release/shared"}
-    name: msvc/${{ matrix.config.name }} (only compile)
-    runs-on: [windows-latest]
-    steps:
-    - uses: actions/checkout@v2
-    - name: setup (versioned)
-      if: matrix.config.version != 'latest'
-      run: |
-        choco install cuda --version=${{ matrix.config.version }} -y
-    - name: setup (latest)
-      if: matrix.config.version == 'latest'
-      run: |
-        choco install cuda -y
-    - name: configure
-      run: |
-        $env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.."
-        Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
-        refreshenv
-        $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
-        mkdir build
-        cd build
-        cmake -DCMAKE_CXX_FLAGS=/bigobj -DGINKGO_BUILD_CUDA=ON -DGINKGO_BUILD_OMP=OFF -DGINKGO_CUDA_ARCHITECTURES=60 ..
-        cmake --build . -j4 --config Release
-
-  windows_ref:
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-        - {shared: "ON", build_type: "Debug", name: "reference/debug/shared"}
-        - {shared: "OFF", build_type: "Release", name: "reference/release/static"}
-        # Debug static needs too much storage
-        # - {shared: "OFF", build_type: "Debug", name: "reference/debug/static"}
-    name: msvc/${{ matrix.config.name }}
-    runs-on: [windows-latest]
-    steps:
-    - uses: actions/checkout@v2
-    - name: configure
-      run: |
-        $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
-        mkdir build
-        cd build
-        cmake  -DCMAKE_CXX_FLAGS=/bigobj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_CXX_FLAGS_DEBUG="/MDd /Zi /Ob1 /Od /RTC1" -DGINKGO_BUILD_CUDA=OFF -DGINKGO_BUILD_OMP=OFF ..
-        cmake --build . -j4 --config ${{ matrix.config.build_type }}
-        ctest . -C ${{ matrix.config.build_type }} --output-on-failure
-    - name: install
-      run: |
-        $env:PATH="$env:PATH;C:\Program Files (x86)\Ginkgo\lib"
-        cd build
-        cmake --install . --config ${{ matrix.config.build_type }}
-        cmake --build . --target test_install --config ${{ matrix.config.build_type }}
-  windows_mingw:
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-        - {shared: "OFF", build_type: "Release", name: "omp/release/static", cflags: ""}
-    name: mingw/${{ matrix.config.name }}
-    runs-on: [windows-latest]
-    steps:
-    - uses: actions/checkout@v2
-    - name: configure
-    # Use cmd to remove the path easily
-      run: |
-        bcdedit /set IncreaseUserVa 3072
-        editbin /LARGEADDRESSAWARE "C:\Program Files\Git\mingw64\bin\cc1plus.exe"
-        path %PATH:C:\Program Files\Git\bin;=%
-        path %PATH:C:\Program Files\Git\usr\bin;=%;%GITHUB_WORKSPACE%\build\windows_shared_library
-        mkdir build
-        cd build
-        cmake -G "MinGW Makefiles" -DCMAKE_CXX_FLAGS=-Wa,-mbig-obj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_COMPILER_FLAGS=${{ matrix.config.cflags }} ..
-        cmake --build . -j4
-        ctest . --output-on-failure
-      shell: cmd
-    - name: install
-      run: |
-        set PATH=%PATH:C:\Program Files\Git\bin;=%
-        set PATH=%PATH:C:\Program Files\Git\usr\bin;=%;C:\Program Files (x86)\Ginkgo\lib
-        cd build
-        cmake --install .
-        cmake --build . --target test_install
-      shell: cmd
-
-  windows_cygwin:
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-        - {shared: "OFF", build_type: "Release", name: "omp/release/static", cflags: ""}
-    name: cygwin/${{ matrix.config.name }}
-    runs-on: [windows-latest]
-    steps:
-    - run: git config --global core.autocrlf input
-    - uses: actions/checkout@v2
-    - name: setup
-      run: |
-        choco install cygwin -y
-        choco install cyg-get -y
-        cyg-get cmake make gcc-g++ git
-    - name: configure
-      run: |
-        path C:\tools\cygwin\bin;%GITHUB_WORKSPACE%\build\windows_shared_library
-        mkdir build
-        cd build
-        bash -c "cmake -DCMAKE_CXX_FLAGS=-Wa,-mbig-obj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_COMPILER_FLAGS=${{ matrix.config.cflags }} .."
-        bash -c "make -j4"
-        bash -c "ctest . --output-on-failure"
-      shell: cmd
-    - name: install
-      run: |
-        path C:\tools\cygwin\bin
-        cd build
-        bash -c "make install"
-        bash -c "export PATH=/usr/local/lib:$PATH && make test_install"
-      shell: cmd
diff --git a/.github/workflows/windows-cygwin.yml b/.github/workflows/windows-cygwin.yml
new file mode 100644
index 00000000000..afbb49ac9e6
--- /dev/null
+++ b/.github/workflows/windows-cygwin.yml
@@ -0,0 +1,51 @@
+name: Windows-CygWin
+
+on:
+  push:
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
+
+jobs:
+  windows_cygwin:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+        - {shared: "OFF", build_type: "Release", name: "omp/release/static", cflags: ""}
+    name: cygwin/${{ matrix.config.name }}
+    runs-on: [windows-latest]
+    steps:
+    - run: git config --global core.autocrlf input
+    - uses: actions/checkout@v2
+
+    - name: setup
+      run: |
+        choco install cygwin -y
+        choco install cyg-get -y
+        cyg-get cmake make gcc-g++ git
+
+    - name: Debug over SSH (tmate)
+      uses: mxschmitt/action-tmate@v3.5
+      if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
+
+    - name: configure
+      run: |
+        path C:\tools\cygwin\bin;%GITHUB_WORKSPACE%\build\windows_shared_library
+        mkdir build
+        cd build
+        bash -c "cmake -DCMAKE_CXX_FLAGS=-Wa,-mbig-obj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_COMPILER_FLAGS=${{ matrix.config.cflags }} .."
+        bash -c "make -j4"
+        bash -c "ctest . --output-on-failure"
+      shell: cmd
+
+    - name: install
+      run: |
+        path C:\tools\cygwin\bin
+        cd build
+        bash -c "make install"
+        bash -c "export PATH=/usr/local/lib:$PATH && make test_install"
+      shell: cmd
diff --git a/.github/workflows/windows-mingw.yml b/.github/workflows/windows-mingw.yml
new file mode 100644
index 00000000000..8381c41e9e8
--- /dev/null
+++ b/.github/workflows/windows-mingw.yml
@@ -0,0 +1,49 @@
+name: Windows-MinGW
+
+on:
+  push:
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
+
+jobs:
+  windows_mingw:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+        - {shared: "OFF", build_type: "Release", name: "omp/release/static", cflags: ""}
+    name: mingw/${{ matrix.config.name }}
+    runs-on: [windows-latest]
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Debug over SSH (tmate)
+      uses: mxschmitt/action-tmate@v3.5
+      if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
+
+    - name: configure
+    # Use cmd to remove the path easily
+      run: |
+        bcdedit /set IncreaseUserVa 3072
+        editbin /LARGEADDRESSAWARE "C:\Program Files\Git\mingw64\bin\cc1plus.exe"
+        path %PATH:C:\Program Files\Git\bin;=%
+        path %PATH:C:\Program Files\Git\usr\bin;=%;%GITHUB_WORKSPACE%\build\windows_shared_library
+        mkdir build
+        cd build
+        cmake -G "MinGW Makefiles" -DCMAKE_CXX_FLAGS=-Wa,-mbig-obj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_COMPILER_FLAGS=${{ matrix.config.cflags }} ..
+        cmake --build . -j4
+        ctest . --output-on-failure
+      shell: cmd
+
+    - name: install
+      run: |
+        set PATH=%PATH:C:\Program Files\Git\bin;=%
+        set PATH=%PATH:C:\Program Files\Git\usr\bin;=%;C:\Program Files (x86)\Ginkgo\lib
+        cd build
+        cmake --install .
+        cmake --build . --target test_install
+      shell: cmd
diff --git a/.github/workflows/windows-msvc-cuda.yml b/.github/workflows/windows-msvc-cuda.yml
new file mode 100644
index 00000000000..e8857d4e8e3
--- /dev/null
+++ b/.github/workflows/windows-msvc-cuda.yml
@@ -0,0 +1,48 @@
+name: Windows-MSVC-CUDA (compile-only)
+
+on:
+  push:
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
+
+jobs:
+  windows_cuda:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+        - {version: "10.2.89.20191206", name: "cuda102/release/shared"}
+        - {version: "latest", name: "cuda-latest/release/shared"}
+    name: msvc/${{ matrix.config.name }} (only compile)
+    runs-on: [windows-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: setup (versioned)
+      if: matrix.config.version != 'latest'
+      run: |
+        choco install cuda --version=${{ matrix.config.version }} -y
+
+    - name: setup (latest)
+      if: matrix.config.version == 'latest'
+      run: |
+        choco install cuda -y
+
+    - name: Debug over SSH (tmate)
+      uses: mxschmitt/action-tmate@v3.5
+      if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
+
+    - name: configure
+      run: |
+        $env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.."
+        Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
+        refreshenv
+        $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
+        mkdir build
+        cd build
+        cmake -DCMAKE_CXX_FLAGS=/bigobj -DGINKGO_BUILD_CUDA=ON -DGINKGO_BUILD_OMP=OFF -DGINKGO_CUDA_ARCHITECTURES=60 ..
+        cmake --build . -j4 --config Release
diff --git a/.github/workflows/windows-msvc-ref.yml b/.github/workflows/windows-msvc-ref.yml
new file mode 100644
index 00000000000..be6fd58e55c
--- /dev/null
+++ b/.github/workflows/windows-msvc-ref.yml
@@ -0,0 +1,45 @@
+name: Windows-MSVC-Reference
+
+on:
+  push:
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
+
+jobs:
+  windows_ref:
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+        - {shared: "ON", build_type: "Debug", name: "reference/debug/shared"}
+        - {shared: "OFF", build_type: "Release", name: "reference/release/static"}
+        # Debug static needs too much storage
+        # - {shared: "OFF", build_type: "Debug", name: "reference/debug/static"}
+    name: msvc/${{ matrix.config.name }}
+    runs-on: [windows-latest]
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Debug over SSH (tmate)
+      uses: mxschmitt/action-tmate@v3.5
+      if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
+
+    - name: configure
+      run: |
+        $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
+        mkdir build
+        cd build
+        cmake  -DCMAKE_CXX_FLAGS=/bigobj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_CXX_FLAGS_DEBUG="/MDd /Zi /Ob1 /Od /RTC1" -DGINKGO_BUILD_CUDA=OFF -DGINKGO_BUILD_OMP=OFF ..
+        cmake --build . -j4 --config ${{ matrix.config.build_type }}
+        ctest . -C ${{ matrix.config.build_type }} --output-on-failure
+
+    - name: install
+      run: |
+        $env:PATH="$env:PATH;C:\Program Files (x86)\Ginkgo\lib"
+        cd build
+        cmake --install . --config ${{ matrix.config.build_type }}
+        cmake --build . --target test_install --config ${{ matrix.config.build_type }}

From 7c9d2d530d084f8609d0aebc096147d0236e197e Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 22 Apr 2021 09:40:04 +0200
Subject: [PATCH 0629/1008] Fix typo

Co-authored-by: Yuhsiang M. Tsai <19565938+yhmtsai@users.noreply.github.com>
---
 .github/workflows/windows-cygwin.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/windows-cygwin.yml b/.github/workflows/windows-cygwin.yml
index afbb49ac9e6..bd03328ac53 100644
--- a/.github/workflows/windows-cygwin.yml
+++ b/.github/workflows/windows-cygwin.yml
@@ -1,4 +1,4 @@
-name: Windows-CygWin
+name: Windows-Cygwin
 
 on:
   push:

From 7914074c8643f8a1d5597be5e046fe46c03600c8 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 22 Apr 2021 13:25:41 +0200
Subject: [PATCH 0630/1008] Restrict windows ssh-debugging to collaborators

---
 .github/workflows/windows-cygwin.yml    | 2 ++
 .github/workflows/windows-mingw.yml     | 2 ++
 .github/workflows/windows-msvc-cuda.yml | 2 ++
 .github/workflows/windows-msvc-ref.yml  | 2 ++
 4 files changed, 8 insertions(+)

diff --git a/.github/workflows/windows-cygwin.yml b/.github/workflows/windows-cygwin.yml
index bd03328ac53..dbbba5f1e1c 100644
--- a/.github/workflows/windows-cygwin.yml
+++ b/.github/workflows/windows-cygwin.yml
@@ -31,6 +31,8 @@ jobs:
     - name: Debug over SSH (tmate)
       uses: mxschmitt/action-tmate@v3.5
       if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
+      with:
+        limit-access-to-actor: true
 
     - name: configure
       run: |
diff --git a/.github/workflows/windows-mingw.yml b/.github/workflows/windows-mingw.yml
index 8381c41e9e8..7cfa250757b 100644
--- a/.github/workflows/windows-mingw.yml
+++ b/.github/workflows/windows-mingw.yml
@@ -24,6 +24,8 @@ jobs:
     - name: Debug over SSH (tmate)
       uses: mxschmitt/action-tmate@v3.5
       if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
+      with:
+        limit-access-to-actor: true
 
     - name: configure
     # Use cmd to remove the path easily
diff --git a/.github/workflows/windows-msvc-cuda.yml b/.github/workflows/windows-msvc-cuda.yml
index e8857d4e8e3..bc4557e96bf 100644
--- a/.github/workflows/windows-msvc-cuda.yml
+++ b/.github/workflows/windows-msvc-cuda.yml
@@ -35,6 +35,8 @@ jobs:
     - name: Debug over SSH (tmate)
       uses: mxschmitt/action-tmate@v3.5
       if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
+      with:
+        limit-access-to-actor: true
 
     - name: configure
       run: |
diff --git a/.github/workflows/windows-msvc-ref.yml b/.github/workflows/windows-msvc-ref.yml
index be6fd58e55c..13a29f6cc51 100644
--- a/.github/workflows/windows-msvc-ref.yml
+++ b/.github/workflows/windows-msvc-ref.yml
@@ -27,6 +27,8 @@ jobs:
     - name: Debug over SSH (tmate)
       uses: mxschmitt/action-tmate@v3.5
       if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
+      with:
+        limit-access-to-actor: true
 
     - name: configure
       run: |

From d2456188dfd3bc18b61f85312c515609cd56d249 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 14 Apr 2021 17:27:25 +0200
Subject: [PATCH 0631/1008] Various CMake fixes.

---
 CMakeLists.txt                |  1 +
 cmake/GinkgoConfig.cmake.in   |  3 ++
 cmake/Modules/FindNUMA.cmake  | 59 +++++++++++++++++++++++++++++++++++
 cmake/install_helpers.cmake   |  9 ++++--
 cuda/test/base/CMakeLists.txt |  3 +-
 hip/CMakeLists.txt            |  6 +++-
 hip/test/base/CMakeLists.txt  |  3 +-
 test_install/CMakeLists.txt   | 11 ++++---
 8 files changed, 85 insertions(+), 10 deletions(-)
 create mode 100644 cmake/Modules/FindNUMA.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 80b5a3a092b..671805b7b8d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -364,6 +364,7 @@ file(MAKE_DIRECTORY ${Ginkgo_BINARY_DIR}/test_install)
 add_custom_target(test_install
     COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} -H${Ginkgo_SOURCE_DIR}/test_install
     -B${Ginkgo_BINARY_DIR}/test_install
+    -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
     -DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX}/${GINKGO_INSTALL_CONFIG_DIR}
     -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
     -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 460b626ded3..fbb901bc48d 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -108,6 +108,9 @@ set(GINKGO_INSTALL_PKGCONFIG_DIR "${GINKGO_INSTALL_PREFIX}/@GINKGO_INSTALL_PKGCO
 set(GINKGO_INSTALL_CONFIG_DIR "${GINKGO_INSTALL_PREFIX}/@GINKGO_INSTALL_CONFIG_DIR@")
 set(GINKGO_INSTALL_MODULE_DIR "${GINKGO_INSTALL_PREFIX}/@GINKGO_INSTALL_MODULE_DIR@")
 set(GINKGO_INSTALL_RPATH_FOR_HIP "-Wl,-rpath,${GINKGO_INSTALL_LIBRARY_DIR}")
+set(GINKGO_INSTALL_RPATH ${GINKGO_INSTALL_RPATH})
+set(GINKGO_INSTALL_RPATH_USE_ORIGIN ${GINKGO_INSTALL_RPATH_USE_ORIGIN})
+set(GINKGO_INSTALL_RPATH_DEPENDENCIES ${GINKGO_INSTALL_RPATH_DEPENDENCIES})
 
 # Forward Ginkgo's MODULE PATH and the PREFIX PATH for HIP and more
 list(APPEND CMAKE_MODULE_PATH "@CMAKE_MODULE_PATH@" "${GINKGO_INSTALL_MODULE_DIR}")
diff --git a/cmake/Modules/FindNUMA.cmake b/cmake/Modules/FindNUMA.cmake
new file mode 100644
index 00000000000..89774584dcf
--- /dev/null
+++ b/cmake/Modules/FindNUMA.cmake
@@ -0,0 +1,59 @@
+#.rst:
+# FindNUMA
+# -------
+#
+# Find the NUMA library, usually provided by `numactl`.
+#
+# Imported targets
+# ^^^^^^^^^^^^^^^^
+#
+# This module defines the following :prop_tgt:`IMPORTED` target:
+#
+# ``NUMA::NUMA``
+#   The NUMA library, if found.
+#
+# Result variables
+# ^^^^^^^^^^^^^^^^
+#
+# This module will set the following variables in your project:
+#
+# ``NUMA_INCLUDE_DIRS``
+#   where to find numa.h
+#
+# ``NUMA_LIBRARIES``
+#   the libraries to link against in order to use the NUMA library.
+#
+# ``NUMA_FOUND``
+#   If false, do not try to use the NUMA library.
+
+
+find_path(NUMA_ROOT_DIR NAMES include/numa.h)
+
+find_path(NUMA_INCLUDE_DIR NAMES numa.h HINTS ${NUMA_ROOT_DIR})
+mark_as_advanced(NUMA_INCLUDE_DIR)
+
+
+if(NOT NUMA_LIBRARY)
+    find_library(NUMA_LIBRARY NAMES numa HINTS ${NUMA_ROOT_DIR})
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(NUMA REQUIRED_VARS NUMA_LIBRARY NUMA_INCLUDE_DIR)
+
+if(NUMA_FOUND)
+    set(NUMA_LIBRARIES ${NUMA_LIBRARY})
+    set(NUMA_INCLUDE_DIRS ${NUMA_INCLUDE_DIR})
+    unset(NUMA_LIBRARY)
+    unset(NUMA_INCLUDE_DIR)
+
+    if(NOT TARGET NUMA::NUMA)
+        add_library(NUMA::NUMA UNKNOWN IMPORTED)
+        set_target_properties(NUMA::NUMA PROPERTIES
+            INTERFACE_INCLUDE_DIRECTORIES "${NUMA_INCLUDE_DIRS}")
+        if(EXISTS "${NUMA_LIBRARIES}")
+            set_target_properties(NUMA::NUMA PROPERTIES
+                IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+                IMPORTED_LOCATION "${NUMA_LIBRARIES}")
+        endif()
+    endif()
+endif()
diff --git a/cmake/install_helpers.cmake b/cmake/install_helpers.cmake
index 2ee22f39f92..ad49d46b268 100644
--- a/cmake/install_helpers.cmake
+++ b/cmake/install_helpers.cmake
@@ -84,9 +84,6 @@ function(ginkgo_install)
         install(FILES "${Ginkgo_SOURCE_DIR}/third_party/papi_sde/papi_sde_interface.h"
             DESTINATION "${GINKGO_INSTALL_INCLUDE_DIR}/third_party/papi_sde"
             )
-        install(FILES "${Ginkgo_SOURCE_DIR}/cmake/Modules/FindPAPI.cmake"
-            DESTINATION "${GINKGO_INSTALL_MODULE_DIR}/"
-            )
     endif()
 
     if  (GINKGO_HAVE_HWLOC AND NOT GINKGO_USE_EXTERNAL_HWLOC)
@@ -104,6 +101,12 @@ function(ginkgo_install)
             )
     endif()
 
+    # Install CMake modules
+    install(DIRECTORY "${Ginkgo_SOURCE_DIR}/cmake/Modules"
+        DESTINATION "${GINKGO_INSTALL_MODULE_DIR}"
+        FILES_MATCHING PATTERN "*.cmake"
+        )
+
     # export targets
     export(EXPORT Ginkgo
         NAMESPACE Ginkgo::
diff --git a/cuda/test/base/CMakeLists.txt b/cuda/test/base/CMakeLists.txt
index b92138a52dc..4759ff90e14 100644
--- a/cuda/test/base/CMakeLists.txt
+++ b/cuda/test/base/CMakeLists.txt
@@ -1,7 +1,8 @@
 ginkgo_create_cuda_test(array)
 ginkgo_create_cuda_test(cuda_executor)
 if(GINKGO_HAVE_HWLOC)
-    ginkgo_create_cuda_test(cuda_executor_topology numa)
+    find_package(NUMA REQUIRED)
+    ginkgo_create_cuda_test(cuda_executor_topology NUMA::NUMA)
 endif()
 ginkgo_create_cuda_test(exception_helpers)
 ginkgo_create_cuda_test(lin_op)
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index fc305d898dd..608a9aeff18 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -265,8 +265,12 @@ if(GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_AMD_REGEX}")
     get_target_property(HIP_LIBAMDHIP64_CONFIGURATIONS hip::amdhip64 IMPORTED_CONFIGURATIONS)
     if (CMAKE_BUILD_TYPE MATCHES "[Dd][Ee][Bb][Uu][Gg]" AND HIP_LIBAMDHIP64_CONFIGURATIONS MATCHES "[Dd][Ee][Bb][Uu][Gg]")
         get_target_property(HIP_LIBAMDHIP64_LIBRARIES hip::amdhip64 IMPORTED_LOCATION_DEBUG)
-    else()
+    elseif (CMAKE_BUILD_TYPE MATCHES "[Rr][Ee][Ll][Ee][Aa][Ss][Ee]" AND HIP_LIBAMDHIP64_CONFIGURATIONS MATCHES "[Rr][Ee][Ll][Ee][Aa][Ss][Ee]")
         get_target_property(HIP_LIBAMDHIP64_LIBRARIES hip::amdhip64 IMPORTED_LOCATION_RELEASE)
+    else ()
+        # Just get what is available
+        list(GET HIP_LIBAMDHIP64_CONFIGURATIONS 0 HIP_LIBAMDHIP64_FIRST_CONFIG)
+        get_target_property(HIP_LIBAMDHIP64_LIBRARIES hip::amdhip64 IMPORTED_LOCATION_${HIP_LIBAMDHIP64_FIRST_CONFIG})
     endif()
     target_link_libraries(ginkgo_hip PUBLIC ${HIP_LIBAMDHIP64_LIBRARIES})
 elseif(GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}")
diff --git a/hip/test/base/CMakeLists.txt b/hip/test/base/CMakeLists.txt
index fe0205794ee..62527c1764b 100644
--- a/hip/test/base/CMakeLists.txt
+++ b/hip/test/base/CMakeLists.txt
@@ -1,6 +1,7 @@
 ginkgo_create_hip_test(hip_executor)
 if(GINKGO_HAVE_HWLOC)
-    ginkgo_create_hip_test(hip_executor_topology numa)
+    find_package(NUMA REQUIRED)
+    ginkgo_create_hip_test(hip_executor_topology NUMA::NUMA)
 endif()
 ginkgo_create_hip_test(lin_op)
 ginkgo_create_hip_test(math)
diff --git a/test_install/CMakeLists.txt b/test_install/CMakeLists.txt
index a629f000008..cb4b08de706 100644
--- a/test_install/CMakeLists.txt
+++ b/test_install/CMakeLists.txt
@@ -7,6 +7,12 @@ find_package(Ginkgo REQUIRED
     # Alternatively, use `cmake -DCMAKE_PREFIX_PATH=<ginkgo_install_dir>` to specify the install directory
     )
 
+
+# Manage RPATH. This needs to be before the targets.
+set(CMAKE_SKIP_BUILD_RPATH OFF)
+set(CMAKE_SKIP_INSTALL_RPATH OFF)
+set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON)
+
 # Copy the data to the build directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
@@ -86,16 +92,13 @@ endif()
 # Installation step
 set(TESTINSTALL_INSTALL_DIR "${GINKGO_INSTALL_PREFIX}/smoke_tests")
 set(TESTINSTALL_INSTALL_DATADIR "${TESTINSTALL_INSTALL_DIR}/data")
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON)
 
-set_target_properties(test_install PROPERTIES INSTALL_RPATH "${GINKGO_INSTALL_LIBRARY_DIR}")
+# Install the binaries and data
 install(TARGETS test_install RUNTIME DESTINATION ${TESTINSTALL_INSTALL_DIR})
 if(GINKGO_BUILD_CUDA)
-    set_target_properties(test_install_cuda PROPERTIES INSTALL_RPATH "${GINKGO_INSTALL_LIBRARY_DIR}")
     install(TARGETS test_install_cuda RUNTIME DESTINATION ${TESTINSTALL_INSTALL_DIR})
 endif()
 if(GINKGO_BUILD_HIP)
-    set_target_properties(test_install_hip PROPERTIES INSTALL_RPATH "${GINKGO_INSTALL_LIBRARY_DIR}")
     install(TARGETS test_install_hip RUNTIME DESTINATION ${TESTINSTALL_INSTALL_DIR})
 endif()
 install(DIRECTORY "${TestInstall_BINARY_DIR}/data/"

From eecd230879aeb69a20d52db58e81718cbc4d1047 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Fri, 16 Apr 2021 18:50:46 +0200
Subject: [PATCH 0632/1008] Also fix the CMAKE_CUDA_HOST_COMPILER setup.

---
 cmake/GinkgoConfig.cmake.in | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index fbb901bc48d..70ccacccf97 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -70,6 +70,8 @@ set(GINKGO_HIP_HCC_COMPILER_FLAGS @GINKGO_HIP_HCC_COMPILER_FLAGS@)
 set(GINKGO_HIP_NVCC_COMPILER_FLAGS @GINKGO_HIP_NVCC_COMPILER_FLAGS@)
 set(GINKGO_HIP_CLANG_COMPILER_FLAGS @GINKGO_HIP_CLANG_COMPILER_FLAGS@)
 set(GINKGO_HIP_PLATFORM @GINKGO_HIP_PLATFORM@)
+set(GINKGO_HIP_PLATFORM_AMD_REGEX @HIP_PLATFORM_AMD_REGEX@)
+set(GINKGO_HIP_PLATFORM_NVIDIA_REGEX @HIP_PLATFORM_NVIDIA_REGEX@)
 set(GINKGO_HIP_AMDGPU @GINKGO_HIP_AMDGPU@)
 set(GINKGO_HIP_VERSION @GINKGO_HIP_VERSION@)
 set(GINKGO_AMD_ARCH_FLAGS @GINKGO_AMD_ARCH_FLAGS@)
@@ -150,10 +152,14 @@ endif()
 # NOTE: we do not export benchmarks, examples, tests or devel tools
 #     so `third_party` libraries are currently unneeded.
 
-# propagate CUDA_HOST_COMPILER if Ginkgo was built with CUDA
-if (GINKGO_BUILD_CUDA AND GINKGO_CUDA_HOST_COMPILER AND NOT CMAKE_CUDA_HOST_COMPILER)
-    message(STATUS "Ginkgo: Setting CUDA host compiler to ${GINKGO_CXX_COMPILER}")
-    set(CMAKE_CUDA_HOST_COMPILER "${GINKGO_CXX_COMPILER}" CACHE STRING "" FORCE)
+# propagate CUDA_HOST_COMPILER if Ginkgo needed
+if (GINKGO_BUILD_CUDA OR (GINKGO_BUILD_HIP
+    AND GINKGO_HIP_PLATFORM MATCHES "${GINKGO_HIP_PLATFORM_NVIDIA_REGEX}"))
+    if (GINKGO_CUDA_HOST_COMPILER AND NOT CMAKE_CUDA_HOST_COMPILER
+        AND EXISTS "${GINKGO_CUDA_HOST_COMPILER}")
+        message(STATUS "Ginkgo: Setting CUDA host compiler to ${GINKGO_CUDA_HOST_COMPILER}")
+        set(CMAKE_CUDA_HOST_COMPILER "${GINKGO_CUDA_HOST_COMPILER}" CACHE STRING "" FORCE)
+    endif()
 endif()
 
 if(GINKGO_HAVE_PAPI_SDE)

From cc4db0c270343dc9ade3a69543a043ca426ff8ee Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Mon, 19 Apr 2021 11:28:15 +0200
Subject: [PATCH 0633/1008] Review update.

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 cmake/GinkgoConfig.cmake.in   |  2 +-
 cmake/Modules/FindHWLOC.cmake |  2 +-
 hip/CMakeLists.txt            | 17 ++++++++---------
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 70ccacccf97..1d7bfc38bc5 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -152,7 +152,7 @@ endif()
 # NOTE: we do not export benchmarks, examples, tests or devel tools
 #     so `third_party` libraries are currently unneeded.
 
-# propagate CUDA_HOST_COMPILER if Ginkgo needed
+# propagate CUDA_HOST_COMPILER if needed
 if (GINKGO_BUILD_CUDA OR (GINKGO_BUILD_HIP
     AND GINKGO_HIP_PLATFORM MATCHES "${GINKGO_HIP_PLATFORM_NVIDIA_REGEX}"))
     if (GINKGO_CUDA_HOST_COMPILER AND NOT CMAKE_CUDA_HOST_COMPILER
diff --git a/cmake/Modules/FindHWLOC.cmake b/cmake/Modules/FindHWLOC.cmake
index 7f8bb2ba36d..161ee406e00 100644
--- a/cmake/Modules/FindHWLOC.cmake
+++ b/cmake/Modules/FindHWLOC.cmake
@@ -120,7 +120,7 @@ if(HWLOC_INCLUDE_DIRS AND HWLOC_LIBRARIES)
     endif()
 endif(HWLOC_INCLUDE_DIRS AND HWLOC_LIBRARIES)
 
-if(HWLOC_FOUND)
+if(HWLOC_FOUND AND NOT TARGET hwloc)
     add_library(hwloc SHARED IMPORTED GLOBAL)
     set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION ${HWLOC_LIBRARIES})
     set_target_properties(hwloc PROPERTIES INTERFACE_LINK_LIBRARIES ${HWLOC_LIBRARIES})
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 608a9aeff18..cc4d66534fe 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -262,15 +262,14 @@ if(GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_AMD_REGEX}")
     find_package(hip REQUIRED)
     # To save a bit of pain, we directly link against the `library` instead of
     # linking against the target.
-    get_target_property(HIP_LIBAMDHIP64_CONFIGURATIONS hip::amdhip64 IMPORTED_CONFIGURATIONS)
-    if (CMAKE_BUILD_TYPE MATCHES "[Dd][Ee][Bb][Uu][Gg]" AND HIP_LIBAMDHIP64_CONFIGURATIONS MATCHES "[Dd][Ee][Bb][Uu][Gg]")
-        get_target_property(HIP_LIBAMDHIP64_LIBRARIES hip::amdhip64 IMPORTED_LOCATION_DEBUG)
-    elseif (CMAKE_BUILD_TYPE MATCHES "[Rr][Ee][Ll][Ee][Aa][Ss][Ee]" AND HIP_LIBAMDHIP64_CONFIGURATIONS MATCHES "[Rr][Ee][Ll][Ee][Aa][Ss][Ee]")
-        get_target_property(HIP_LIBAMDHIP64_LIBRARIES hip::amdhip64 IMPORTED_LOCATION_RELEASE)
-    else ()
-        # Just get what is available
-        list(GET HIP_LIBAMDHIP64_CONFIGURATIONS 0 HIP_LIBAMDHIP64_FIRST_CONFIG)
-        get_target_property(HIP_LIBAMDHIP64_LIBRARIES hip::amdhip64 IMPORTED_LOCATION_${HIP_LIBAMDHIP64_FIRST_CONFIG})
+    if (CMAKE_BUILD_TYPE)
+        # Check if our configuration is available first
+        string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPER_BUILD_TYPE)
+        get_target_property(HIP_LIBAMDHIP64_LIBRARIES hip::amdhip64 IMPORTED_LOCATION_${UPPER_BUILD_TYPE})
+    endif()
+    if (NOT HIP_LIBAMDHIP64_LIBRARIES)
+        # Fall back to anything
+        get_target_property(HIP_LIBAMDHIP64_LIBRARIES hip::amdhip64 IMPORTED_LOCATION)
     endif()
     target_link_libraries(ginkgo_hip PUBLIC ${HIP_LIBAMDHIP64_LIBRARIES})
 elseif(GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}")

From e93372d210ecdb07109288222391983ec2216cb2 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Mon, 26 Apr 2021 13:57:41 +0200
Subject: [PATCH 0634/1008] Run the CUDA jobs on the CI controllers.

---
 .gitlab-ci.yml    | 42 ++++++++++++++++----------------
 .gitlab/image.yml | 61 ++++++++++++++++++++++++-----------------------
 2 files changed, 52 insertions(+), 51 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dcd47e63880..b3d57c7ff07 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -80,7 +80,7 @@ include:
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
         -DGINKGO_EXPORT_BUILD_DIR=${EXPORT_BUILD_DIR}
-    - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT}
+    - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT} install
     - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then unset SYCL_DEVICE_TYPE; fi
     - if [ "${EXPORT_BUILD_DIR}" == "ON" ]; then ninja test_exportbuild; fi
   dependencies: []
@@ -251,7 +251,7 @@ trigger_pipeline:
 # Build jobs
 # Job with example runs.
 build/cuda90/gcc/all/debug/shared:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .quick_test_condition
     - .use_gko-cuda90-gnu5-llvm39
@@ -265,7 +265,7 @@ build/cuda90/gcc/all/debug/shared:
     CUDA_ARCH: 35
 
 build/cuda90/clang/all/release/static:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda90-gnu5-llvm39
@@ -281,7 +281,7 @@ build/cuda90/clang/all/release/static:
 
 # cuda 9.1 and friends
 build/cuda91/gcc/all/debug/static:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda91-gnu6-llvm40
@@ -295,7 +295,7 @@ build/cuda91/gcc/all/debug/static:
     CUDA_ARCH: 35
 
 build/cuda91/clang/all/release/shared:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda91-gnu6-llvm40
@@ -311,7 +311,7 @@ build/cuda91/clang/all/release/shared:
 
 # cuda 9.2 and friends
 build/cuda92/gcc/all/release/shared:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda92-gnu7-llvm50-intel2017
@@ -326,7 +326,7 @@ build/cuda92/gcc/all/release/shared:
 # Make sure that our jobs run when HWLOC is
 # forcibly switched off
 build/cuda92/intel/cuda/release/static:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda92-gnu7-llvm50-intel2017
@@ -343,7 +343,7 @@ build/cuda92/intel/cuda/release/static:
 
 # Build CUDA NVIDIA without omp
 build/cuda92/intel/cuda_wo_omp/release/shared:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .quick_test_condition
     - .use_gko-cuda92-gnu7-llvm50-intel2017
@@ -361,7 +361,7 @@ build/cuda92/intel/cuda_wo_omp/release/shared:
 # Make sure that our jobs run when using self-installed
 # third-party HWLOC.
 build/cuda100/gcc/all/debug/shared:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .quick_test_condition
     - .use_gko-cuda100-gnu7-llvm60-intel2018
@@ -375,7 +375,7 @@ build/cuda100/gcc/all/debug/shared:
     CUDA_ARCH: 35
 
 build/cuda100/clang/all/release/static:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda100-gnu7-llvm60-intel2018
@@ -391,7 +391,7 @@ build/cuda100/clang/all/release/static:
     CUDA_ARCH: 35
 
 build/cuda100/intel/cuda/release/shared:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda100-gnu7-llvm60-intel2018
@@ -406,7 +406,7 @@ build/cuda100/intel/cuda/release/shared:
 
 # cuda 10.1 and friends
 build/cuda101/gcc/all/debug/shared:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda101-gnu8-llvm7-intel2019
@@ -420,7 +420,7 @@ build/cuda101/gcc/all/debug/shared:
     CUDA_ARCH: 35
 
 build/cuda101/clang/all/release/static:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda101-gnu8-llvm7-intel2019
@@ -437,7 +437,7 @@ build/cuda101/clang/all/release/static:
 
 # clang-cuda with cuda 10.1 and friends
 build/clang-cuda101/gcc/all/release/shared:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .quick_test_condition
     - .use_gko-cuda101-gnu8-llvm10-intel2019
@@ -451,7 +451,7 @@ build/clang-cuda101/gcc/all/release/shared:
     CUDA_ARCH: 35
 
 build/clang-cuda101/clang/cuda/debug/static:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda101-gnu8-llvm10-intel2019
@@ -471,7 +471,7 @@ build/clang-cuda101/clang/cuda/debug/static:
 
 # works when there is no hwloc and tpl hwloc is also switched off.
 build/cuda102/gcc/all/debug/shared:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda102-gnu8-llvm8-intel2019
@@ -487,7 +487,7 @@ build/cuda102/gcc/all/debug/shared:
 
 # Use TPL hwloc when no system hwloc is available
 build/cuda102/clang/all/release/static:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda102-gnu8-llvm8-intel2019
@@ -503,7 +503,7 @@ build/cuda102/clang/all/release/static:
     CUDA_ARCH: 35
 
 build/cuda102/intel/cuda/debug/static:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda102-gnu8-llvm8-intel2019
@@ -520,7 +520,7 @@ build/cuda102/intel/cuda/debug/static:
 
 # cuda 11.0 and friends
 build/cuda110/gcc/cuda/debug/shared:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda110-gnu9-llvm9-intel2020
@@ -533,7 +533,7 @@ build/cuda110/gcc/cuda/debug/shared:
     CUDA_ARCH: 35
 
 build/cuda110/clang/cuda/release/static:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .full_test_condition
     - .use_gko-cuda110-gnu9-llvm9-intel2020
@@ -548,7 +548,7 @@ build/cuda110/clang/cuda/release/static:
     CUDA_ARCH: 35
 
 build/cuda110/intel/cuda/debug/static:
-  <<: *default_build_with_test
+  <<: *default_build
   extends:
     - .quick_test_condition
     - .use_gko-cuda110-gnu9-llvm9-intel2020
diff --git a/.gitlab/image.yml b/.gitlab/image.yml
index 3fca4782061..801a8a4e836 100644
--- a/.gitlab/image.yml
+++ b/.gitlab/image.yml
@@ -1,82 +1,83 @@
 .use_gko-nocuda-gnu9-llvm8:
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  image: ginkgohub/cpu:gnu9-llvm8
   tags:
     - private_ci
     - cpu
+    - amd
 
 .use_gko-nocuda-gnu9-llvm8-intel:
-  image: localhost:5000/gko-nocuda-gnu9-llvm8
+  image: ginkgohub/cpu:gnu9-llvm8-intel2020
   tags:
     - private_ci
     - cpu
-    - cuda
+    - controller
 
 .use_gko-cuda90-gnu5-llvm39:
-  image: localhost:5000/gko-cuda90-gnu5-llvm39
+  image: ginkgohub/cuda:90-gnu5-llvm39
   tags:
     - private_ci
-    - cuda
-    - gpu
+    - controller
+    - cpu
 
 .use_gko-cuda91-gnu6-llvm40:
-  image: localhost:5000/gko-cuda91-gnu6-llvm40
+  image: ginkgohub/cuda:91-gnu6-llvm40
   tags:
     - private_ci
-    - cuda
-    - gpu
+    - controller
+    - cpu
 
 .use_gko-cuda92-gnu7-llvm50-intel2017:
-  image: localhost:5000/gko-cuda92-gnu7-llvm50-intel2017
+  image: ginkgohub/cuda:92-gnu7-llvm50-intel2017
   tags:
     - private_ci
-    - cuda
-    - gpu
+    - controller
+    - cpu
 
 .use_gko-cuda100-gnu7-llvm60-intel2018:
-  image: localhost:5000/gko-cuda100-gnu7-llvm60-intel2018
+  image: ginkgohub/cuda:100-gnu7-llvm60-intel2018
   tags:
     - private_ci
-    - cuda
-    - gpu
+    - controller
+    - cpu
 
 .use_gko-cuda101-gnu8-llvm7-intel2019:
-  image: localhost:5000/gko-cuda101-gnu8-llvm7-intel2019
+  image: ginkgohub/cuda:101-gnu8-llvm7-intel2019
   tags:
     - private_ci
-    - cuda
-    - gpu
+    - controller
+    - cpu
 
 .use_gko-cuda101-gnu8-llvm10-intel2019:
-  image: localhost:5000/gko-cuda101-gnu8-llvm10-intel2019
+  image: ginkgohub/cuda:101-gnu8-llvm10-intel2019
   tags:
     - private_ci
-    - cuda
-    - gpu
+    - controller
+    - cpu
 
 .use_gko-cuda102-gnu8-llvm8-intel2019:
-  image: localhost:5000/gko-cuda102-gnu8-llvm8-intel2019
+  image: ginkgohub/cuda:102-gnu8-llvm8-intel2019
   tags:
     - private_ci
-    - cuda
-    - gpu
+    - controller
+    - cpu
 
 .use_gko-cuda110-gnu9-llvm9-intel2020:
-  image: localhost:5000/gko-cuda110-gnu9-llvm9-intel2020
+  image: ginkgohub/cuda:110-gnu9-llvm9-intel2020
   tags:
     - private_ci
-    - cuda
-    - gpu
+    - controller
+    - cpu
 
 .use_gko-amd-gnu8-llvm7:
-  image: localhost:5000/gko-amd-gnu8-llvm7
+  image: ginkgohub/rocm:gnu8-llvm7
   tags:
     - private_ci
     - amd
     - gpu
 
 .use_gko-oneapi:
-  image: localhost:5000/gko-oneapi
+  image: ginkgohub/oneapi:latest
   tags:
     - private_ci
-    - cuda
+    - bwuni
     - cpu

From 9dcce4a77219f8b26de8a54f7805da563d194b64 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 27 Apr 2021 09:45:16 +0200
Subject: [PATCH 0635/1008] Remove DPC++ job. It'll come back in another PR.

---
 .gitlab-ci.yml | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b3d57c7ff07..6da6284c3a9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -680,20 +680,6 @@ build/nocuda/intel/omp/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
 
-build/dpcpp/cpu/release/static:
-  <<: *default_build_with_test
-  extends:
-    - .quick_test_condition
-    - .use_gko-oneapi
-  variables:
-    <<: *default_variables
-    C_COMPILER: "gcc"
-    CXX_COMPILER: "dpcpp"
-    BUILD_DPCPP: "ON"
-    BUILD_TYPE: "Release"
-    BUILD_SHARED_LIBS: "OFF"
-    SYCL_DEVICE_TYPE: "CPU"
-
 # Job with important warnings as error
 warnings:
   <<: *default_build

From 4ddb3e25bd626e1d2122da190795448655135bfd Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 27 Apr 2021 14:27:17 +0200
Subject: [PATCH 0636/1008] Disable device hwloc.

---
 cmake/DownloadNonCMakeCMakeLists.txt.in | 2 +-
 third_party/hwloc/CMakeLists.txt        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/DownloadNonCMakeCMakeLists.txt.in b/cmake/DownloadNonCMakeCMakeLists.txt.in
index 7f994a8fcf4..c2d848e8d49 100644
--- a/cmake/DownloadNonCMakeCMakeLists.txt.in
+++ b/cmake/DownloadNonCMakeCMakeLists.txt.in
@@ -8,7 +8,7 @@ ExternalProject_Add(${package_name}
     DOWNLOAD_NO_PROGRESS TRUE
     SOURCE_DIR        "${CMAKE_CURRENT_BINARY_DIR}/src"
     BINARY_DIR        "${CMAKE_CURRENT_BINARY_DIR}/build"
-    CONFIGURE_COMMAND "${config_command}"
+    CONFIGURE_COMMAND "${config_command}" "${ARGN}"
     INSTALL_COMMAND ""
     UPDATE_DISCONNECTED ${GINKGO_SKIP_DEPENDENCY_UPDATE}
     )
diff --git a/third_party/hwloc/CMakeLists.txt b/third_party/hwloc/CMakeLists.txt
index be1e057c91c..b2c8b0ee08e 100644
--- a/third_party/hwloc/CMakeLists.txt
+++ b/third_party/hwloc/CMakeLists.txt
@@ -15,7 +15,7 @@ else()
     message(STATUS "Downloading HWLOC tars and building")
     ginkgo_load_and_configure_package(hwloc_external "https://download.open-mpi.org/release/hwloc/v2.3/hwloc-2.3.0.tar.gz"
         "SHA1=69d8e2f088d365cf5e86d04f049c6b1e09e2b9c3"
-        "${TPL_HWLOC_PATH}/src/configure"
+        "${TPL_HWLOC_PATH}/src/configure" "--disable-nvml" "--disable-cuda" "--disable-rsmi"
         )
 
 endif()

From 150d05b46041634cc37b69e9832ef46a0cc4ce4d Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 27 Apr 2021 15:31:15 +0200
Subject: [PATCH 0637/1008] Update tags.

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
---
 .gitlab/image.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab/image.yml b/.gitlab/image.yml
index 801a8a4e836..b97db8f6f90 100644
--- a/.gitlab/image.yml
+++ b/.gitlab/image.yml
@@ -3,7 +3,7 @@
   tags:
     - private_ci
     - cpu
-    - amd
+    - amdci
 
 .use_gko-nocuda-gnu9-llvm8-intel:
   image: ginkgohub/cpu:gnu9-llvm8-intel2020
@@ -72,7 +72,7 @@
   image: ginkgohub/rocm:gnu8-llvm7
   tags:
     - private_ci
-    - amd
+    - amdci
     - gpu
 
 .use_gko-oneapi:

From cb573439387fba56a4b8aa741a8685cb346179f8 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sun, 22 Nov 2020 23:02:23 +0100
Subject: [PATCH 0638/1008] dense reduction + conj_dot

---
 common/matrix/dense_kernels.hpp.inc      | 19 ++++++++-
 core/device_hooks/common_kernels.inc.cpp |  5 +++
 core/matrix/dense.cpp                    | 27 +++++++++---
 core/matrix/dense_kernels.hpp            |  8 ++++
 cuda/matrix/dense_kernels.cu             | 45 +++++++++++++++++---
 cuda/test/matrix/dense_kernels.cpp       | 22 ++++++++++
 hip/matrix/dense_kernels.hip.cpp         | 42 +++++++++++++++++-
 hip/test/matrix/dense_kernels.hip.cpp    | 22 ++++++++++
 include/ginkgo/core/matrix/dense.hpp     | 26 +++++++++++-
 omp/matrix/dense_kernels.cpp             | 23 +++++++++-
 omp/test/matrix/dense_kernels.cpp        | 22 ++++++++++
 reference/matrix/dense_kernels.cpp       | 21 ++++++++-
 reference/test/matrix/dense_kernels.cpp  | 54 +++++++++++++++++++++++-
 13 files changed, 314 insertions(+), 22 deletions(-)

diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index 5bafe38b4d5..fd516d7f1ec 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -164,6 +164,21 @@ __global__ __launch_bounds__(block_size) void compute_partial_dot(
     size_type num_rows, const ValueType *__restrict__ x, size_type stride_x,
     const ValueType *__restrict__ y, size_type stride_y,
     ValueType *__restrict__ work)
+{
+    compute_partial_reduce<block_size>(
+        num_rows, work,
+        [x, stride_x, y, stride_y](size_type i) {
+            return x[i * stride_x] * y[i * stride_y];
+        },
+        [](const ValueType &x, const ValueType &y) { return x + y; });
+}
+
+
+template <size_type block_size, typename ValueType>
+__global__ __launch_bounds__(block_size) void compute_partial_conj_dot(
+    size_type num_rows, const ValueType *__restrict__ x, size_type stride_x,
+    const ValueType *__restrict__ y, size_type stride_y,
+    ValueType *__restrict__ work)
 {
     compute_partial_reduce<block_size>(
         num_rows, work,
@@ -175,7 +190,7 @@ __global__ __launch_bounds__(block_size) void compute_partial_dot(
 
 
 template <size_type block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void finalize_dot_computation(
+__global__ __launch_bounds__(block_size) void finalize_sum_reduce_computation(
     size_type size, const ValueType *work, ValueType *result)
 {
     finalize_reduce_computation<block_size>(
@@ -199,7 +214,7 @@ __global__ __launch_bounds__(block_size) void compute_partial_norm2(
 
 
 template <size_type block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void finalize_norm2_computation(
+__global__ __launch_bounds__(block_size) void finalize_sqrt_reduce_computation(
     size_type size, const ValueType *work, ValueType *result)
 {
     finalize_reduce_computation<block_size>(
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 81d74a33f88..cbf1e2d9737 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -157,6 +157,11 @@ GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL);
 
+template <typename ValueType>
+GKO_DECLARE_DENSE_COMPUTE_CONJ_DOT_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_CONJ_DOT_KERNEL);
+
 template <typename ValueType>
 GKO_DECLARE_DENSE_COMPUTE_NORM2_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 1fe202fade8..02edd22ce78 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -68,6 +68,7 @@ GKO_REGISTER_OPERATION(scale, dense::scale);
 GKO_REGISTER_OPERATION(add_scaled, dense::add_scaled);
 GKO_REGISTER_OPERATION(add_scaled_diag, dense::add_scaled_diag);
 GKO_REGISTER_OPERATION(compute_dot, dense::compute_dot);
+GKO_REGISTER_OPERATION(compute_conj_dot, dense::compute_conj_dot);
 GKO_REGISTER_OPERATION(compute_norm2, dense::compute_norm2);
 GKO_REGISTER_OPERATION(count_nonzeros, dense::count_nonzeros);
 GKO_REGISTER_OPERATION(calculate_max_nnz_per_row,
@@ -295,9 +296,23 @@ void Dense<ValueType>::compute_dot_impl(const LinOp *b, LinOp *result) const
     GKO_ASSERT_EQUAL_DIMENSIONS(this, b);
     GKO_ASSERT_EQUAL_DIMENSIONS(result, dim<2>(1, this->get_size()[1]));
     auto exec = this->get_executor();
-    exec->run(dense::make_compute_dot(
-        this, make_temporary_conversion<ValueType>(b).get(),
-        make_temporary_conversion<ValueType>(result).get()));
+    auto dense_b = make_temporary_conversion<ValueType>(b);
+    auto dense_res = make_temporary_conversion<ValueType>(result);
+    exec->run(dense::make_compute_dot(this, dense_b.get(), dense_res.get()));
+}
+
+
+template <typename ValueType>
+void Dense<ValueType>::compute_conj_dot_impl(const LinOp *b,
+                                             LinOp *result) const
+{
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, b);
+    GKO_ASSERT_EQUAL_DIMENSIONS(result, dim<2>(1, this->get_size()[1]));
+    auto exec = this->get_executor();
+    auto dense_b = make_temporary_conversion<ValueType>(b);
+    auto dense_res = make_temporary_conversion<ValueType>(result);
+    exec->run(
+        dense::make_compute_conj_dot(this, dense_b.get(), dense_res.get()));
 }
 
 
@@ -306,9 +321,9 @@ void Dense<ValueType>::compute_norm2_impl(LinOp *result) const
 {
     GKO_ASSERT_EQUAL_DIMENSIONS(result, dim<2>(1, this->get_size()[1]));
     auto exec = this->get_executor();
-    exec->run(dense::make_compute_norm2(
-        this,
-        make_temporary_conversion<remove_complex<ValueType>>(result).get()));
+    auto dense_res =
+        make_temporary_conversion<remove_complex<ValueType>>(result);
+    exec->run(dense::make_compute_norm2(this, dense_res.get()));
 }
 
 
diff --git a/core/matrix/dense_kernels.hpp b/core/matrix/dense_kernels.hpp
index df3abaef209..1cbe7f1001a 100644
--- a/core/matrix/dense_kernels.hpp
+++ b/core/matrix/dense_kernels.hpp
@@ -82,6 +82,12 @@ namespace kernels {
                      const matrix::Dense<_type> *y,               \
                      matrix::Dense<_type> *result)
 
+#define GKO_DECLARE_DENSE_COMPUTE_CONJ_DOT_KERNEL(_type)               \
+    void compute_conj_dot(std::shared_ptr<const DefaultExecutor> exec, \
+                          const matrix::Dense<_type> *x,               \
+                          const matrix::Dense<_type> *y,               \
+                          matrix::Dense<_type> *result)
+
 #define GKO_DECLARE_DENSE_COMPUTE_NORM2_KERNEL(_type)               \
     void compute_norm2(std::shared_ptr<const DefaultExecutor> exec, \
                        const matrix::Dense<_type> *x,               \
@@ -230,6 +236,8 @@ namespace kernels {
     template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL(ValueType);                        \
     template <typename ValueType>                                           \
+    GKO_DECLARE_DENSE_COMPUTE_CONJ_DOT_KERNEL(ValueType);                   \
+    template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_COMPUTE_NORM2_KERNEL(ValueType);                      \
     template <typename ValueType, typename IndexType>                       \
     GKO_DECLARE_DENSE_CONVERT_TO_COO_KERNEL(ValueType, IndexType);          \
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index 915c38b053a..581653ca0b7 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -233,9 +233,10 @@ void compute_dot(std::shared_ptr<const CudaExecutor> exec,
                 x->get_size()[0], as_cuda_type(x->get_const_values() + col),
                 x->get_stride(), as_cuda_type(y->get_const_values() + col),
                 y->get_stride(), as_cuda_type(work.get_data()));
-            kernel::finalize_dot_computation<block_size><<<1, block_dim>>>(
-                grid_dim.x, as_cuda_type(work.get_const_data()),
-                as_cuda_type(result->get_values() + col));
+            kernel::finalize_sum_reduce_computation<block_size>
+                <<<1, block_dim>>>(grid_dim.x,
+                                   as_cuda_type(work.get_const_data()),
+                                   as_cuda_type(result->get_values() + col));
         }
     }
 }
@@ -243,6 +244,37 @@ void compute_dot(std::shared_ptr<const CudaExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL);
 
 
+template <typename ValueType>
+void compute_conj_dot(std::shared_ptr<const CudaExecutor> exec,
+                      const matrix::Dense<ValueType> *x,
+                      const matrix::Dense<ValueType> *y,
+                      matrix::Dense<ValueType> *result)
+{
+    // TODO: these are tuning parameters obtained experimentally, once
+    // we decide how to handle this uniformly, they should be modified
+    // appropriately
+    constexpr auto work_per_thread = 32;
+    constexpr auto block_size = 1024;
+
+    constexpr auto work_per_block = work_per_thread * block_size;
+    const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
+    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
+    Array<ValueType> work(exec, grid_dim.x);
+    // TODO: write a kernel which does this more efficiently
+    for (size_type col = 0; col < x->get_size()[1]; ++col) {
+        kernel::compute_partial_conj_dot<block_size><<<grid_dim, block_dim>>>(
+            x->get_size()[0], as_cuda_type(x->get_const_values() + col),
+            x->get_stride(), as_cuda_type(y->get_const_values() + col),
+            y->get_stride(), as_cuda_type(work.get_data()));
+        kernel::finalize_sum_reduce_computation<block_size>
+            <<<1, block_dim>>>(grid_dim.x, as_cuda_type(work.get_const_data()),
+                               as_cuda_type(result->get_values() + col));
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_CONJ_DOT_KERNEL);
+
+
 template <typename ValueType>
 void compute_norm2(std::shared_ptr<const CudaExecutor> exec,
                    const matrix::Dense<ValueType> *x,
@@ -272,9 +304,10 @@ void compute_norm2(std::shared_ptr<const CudaExecutor> exec,
             kernel::compute_partial_norm2<block_size><<<grid_dim, block_dim>>>(
                 x->get_size()[0], as_cuda_type(x->get_const_values() + col),
                 x->get_stride(), as_cuda_type(work.get_data()));
-            kernel::finalize_norm2_computation<block_size><<<1, block_dim>>>(
-                grid_dim.x, as_cuda_type(work.get_const_data()),
-                as_cuda_type(result->get_values() + col));
+            kernel::finalize_sqrt_reduce_computation<block_size>
+                <<<1, block_dim>>>(grid_dim.x,
+                                   as_cuda_type(work.get_const_data()),
+                                   as_cuda_type(result->get_values() + col));
         }
     }
 }
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index 36b4836ed1f..6997233b8c6 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -333,6 +333,28 @@ TEST_F(Dense, MultipleVectorCudaComputeDotIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, SingleVectorCudaComputeConjDotIsEquivalentToRef)
+{
+    set_up_vector_data(1);
+
+    x->compute_conj_dot(y.get(), expected.get());
+    dx->compute_conj_dot(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Dense, MultipleVectorCudaComputeConjDotIsEquivalentToRef)
+{
+    set_up_vector_data(20);
+
+    x->compute_conj_dot(y.get(), expected.get());
+    dx->compute_conj_dot(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Dense, CudaComputeNorm2IsEquivalentToRef)
 {
     set_up_vector_data(20);
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index 628772a1c12..7a02ab6367b 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -243,7 +243,8 @@ void compute_dot(std::shared_ptr<const HipExecutor> exec,
                 as_hip_type(y->get_const_values() + col), y->get_stride(),
                 as_hip_type(work.get_data()));
             hipLaunchKernelGGL(
-                HIP_KERNEL_NAME(kernel::finalize_dot_computation<block_size>),
+                HIP_KERNEL_NAME(
+                    kernel::finalize_sum_reduce_computation<block_size>),
                 dim3(1), dim3(block_dim), 0, 0, grid_dim.x,
                 as_hip_type(work.get_const_data()),
                 as_hip_type(result->get_values() + col));
@@ -254,6 +255,42 @@ void compute_dot(std::shared_ptr<const HipExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL);
 
 
+template <typename ValueType>
+void compute_conj_dot(std::shared_ptr<const HipExecutor> exec,
+                      const matrix::Dense<ValueType> *x,
+                      const matrix::Dense<ValueType> *y,
+                      matrix::Dense<ValueType> *result)
+{
+    // TODO: these are tuning parameters obtained experimentally, once
+    // we decide how to handle this uniformly, they should be modified
+    // appropriately
+    constexpr auto work_per_thread = 32;
+    constexpr auto block_size = 1024;
+
+    constexpr auto work_per_block = work_per_thread * block_size;
+    const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
+    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
+    Array<ValueType> work(exec, grid_dim.x);
+    // TODO: write a kernel which does this more efficiently
+    for (size_type col = 0; col < x->get_size()[1]; ++col) {
+        hipLaunchKernelGGL(
+            HIP_KERNEL_NAME(kernel::compute_partial_conj_dot<block_size>),
+            dim3(grid_dim), dim3(block_dim), 0, 0, x->get_size()[0],
+            as_hip_type(x->get_const_values() + col), x->get_stride(),
+            as_hip_type(y->get_const_values() + col), y->get_stride(),
+            as_hip_type(work.get_data()));
+        hipLaunchKernelGGL(
+            HIP_KERNEL_NAME(
+                kernel::finalize_sum_reduce_computation<block_size>),
+            dim3(1), dim3(block_dim), 0, 0, grid_dim.x,
+            as_hip_type(work.get_const_data()),
+            as_hip_type(result->get_values() + col));
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_CONJ_DOT_KERNEL);
+
+
 template <typename ValueType>
 void compute_norm2(std::shared_ptr<const HipExecutor> exec,
                    const matrix::Dense<ValueType> *x,
@@ -286,7 +323,8 @@ void compute_norm2(std::shared_ptr<const HipExecutor> exec,
                 as_hip_type(x->get_const_values() + col), x->get_stride(),
                 as_hip_type(work.get_data()));
             hipLaunchKernelGGL(
-                HIP_KERNEL_NAME(kernel::finalize_norm2_computation<block_size>),
+                HIP_KERNEL_NAME(
+                    kernel::finalize_sqrt_reduce_computation<block_size>),
                 dim3(1), dim3(block_dim), 0, 0, grid_dim.x,
                 as_hip_type(work.get_const_data()),
                 as_hip_type(result->get_values() + col));
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index 56d43fd829e..18721bc88c0 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -328,6 +328,28 @@ TEST_F(Dense, MultipleVectorHipComputeDotIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, SingleVectorHipComputeConjDotIsEquivalentToRef)
+{
+    set_up_vector_data(1);
+
+    x->compute_conj_dot(y.get(), expected.get());
+    dx->compute_conj_dot(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Dense, MultipleVectorHipComputeConjDotIsEquivalentToRef)
+{
+    set_up_vector_data(20);
+
+    x->compute_conj_dot(y.get(), expected.get());
+    dx->compute_conj_dot(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Dense, HipComputeNorm2IsEquivalentToRef)
 {
     set_up_vector_data(20);
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 01b41b5bff5..8af136ec8b0 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -503,8 +503,7 @@ class Dense
     }
 
     /**
-     * Computes the column-wise dot product of this matrix and `b`. The
-     * conjugate of this is taken.
+     * Computes the column-wise dot product of this matrix and `b`.
      *
      * @param b  a Dense matrix of same dimension as this
      * @param result  a Dense row vector, used to store the dot product
@@ -518,6 +517,21 @@ class Dense
                                make_temporary_clone(exec, result).get());
     }
 
+    /**
+     * Computes the column-wise dot product of this matrix and `conj(b)`.
+     *
+     * @param b  a Dense matrix of same dimension as this
+     * @param result  a Dense row vector, used to store the dot product
+     *                (the number of column in the vector must match the number
+     *                of columns of this)
+     */
+    void compute_conj_dot(const LinOp *b, LinOp *result) const
+    {
+        auto exec = this->get_executor();
+        this->compute_conj_dot_impl(make_temporary_clone(exec, b).get(),
+                                    make_temporary_clone(exec, result).get());
+    }
+
     /**
      * Computes the Euclidian (L^2) norm of this matrix.
      *
@@ -711,6 +725,14 @@ class Dense
      */
     virtual void compute_dot_impl(const LinOp *b, LinOp *result) const;
 
+    /**
+     * @copydoc compute_conj_dot(const LinOp *, LinOp *) const
+     *
+     * @note  Other implementations of dense should override this function
+     *        instead of compute_conj_dot(const LinOp *b, LinOp *result).
+     */
+    virtual void compute_conj_dot_impl(const LinOp *b, LinOp *result) const;
+
     /**
      * @copydoc compute_norm2(LinOp *) const
      *
diff --git a/omp/matrix/dense_kernels.cpp b/omp/matrix/dense_kernels.cpp
index 3830b96615a..92b6f9d8389 100644
--- a/omp/matrix/dense_kernels.cpp
+++ b/omp/matrix/dense_kernels.cpp
@@ -220,7 +220,7 @@ void compute_dot(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for
     for (size_type j = 0; j < x->get_size()[1]; ++j) {
         for (size_type i = 0; i < x->get_size()[0]; ++i) {
-            result->at(0, j) += conj(x->at(i, j)) * y->at(i, j);
+            result->at(0, j) += x->at(i, j) * y->at(i, j);
         }
     }
 }
@@ -228,6 +228,27 @@ void compute_dot(std::shared_ptr<const OmpExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL);
 
 
+template <typename ValueType>
+void compute_conj_dot(std::shared_ptr<const OmpExecutor> exec,
+                      const matrix::Dense<ValueType> *x,
+                      const matrix::Dense<ValueType> *y,
+                      matrix::Dense<ValueType> *result)
+{
+#pragma omp parallel for
+    for (size_type j = 0; j < x->get_size()[1]; ++j) {
+        result->at(0, j) = zero<ValueType>();
+    }
+#pragma omp parallel for
+    for (size_type j = 0; j < x->get_size()[1]; ++j) {
+        for (size_type i = 0; i < x->get_size()[0]; ++i) {
+            result->at(0, j) += x->at(i, j) * conj(y->at(i, j));
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_CONJ_DOT_KERNEL);
+
+
 template <typename ValueType>
 void compute_norm2(std::shared_ptr<const OmpExecutor> exec,
                    const matrix::Dense<ValueType> *x,
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index fc1e8e91063..b057c6b6aae 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -347,6 +347,28 @@ TEST_F(Dense, MultipleVectorOmpComputeDotIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, SingleVectorOmpComputeConjDotIsEquivalentToRef)
+{
+    set_up_vector_data(1);
+
+    x->compute_conj_dot(y.get(), expected.get());
+    dx->compute_conj_dot(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Dense, MultipleVectorOmpComputeConjDotIsEquivalentToRef)
+{
+    set_up_vector_data(20);
+
+    x->compute_conj_dot(y.get(), expected.get());
+    dx->compute_conj_dot(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
 TEST_F(Dense, ComputesNorm2IsEquivalentToRef)
 {
     set_up_vector_data(20);
diff --git a/reference/matrix/dense_kernels.cpp b/reference/matrix/dense_kernels.cpp
index 5e47b0376fa..17100775d6a 100644
--- a/reference/matrix/dense_kernels.cpp
+++ b/reference/matrix/dense_kernels.cpp
@@ -201,7 +201,7 @@ void compute_dot(std::shared_ptr<const ReferenceExecutor> exec,
     }
     for (size_type i = 0; i < x->get_size()[0]; ++i) {
         for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            result->at(0, j) += conj(x->at(i, j)) * y->at(i, j);
+            result->at(0, j) += x->at(i, j) * y->at(i, j);
         }
     }
 }
@@ -209,6 +209,25 @@ void compute_dot(std::shared_ptr<const ReferenceExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL);
 
 
+template <typename ValueType>
+void compute_conj_dot(std::shared_ptr<const ReferenceExecutor> exec,
+                      const matrix::Dense<ValueType> *x,
+                      const matrix::Dense<ValueType> *y,
+                      matrix::Dense<ValueType> *result)
+{
+    for (size_type j = 0; j < x->get_size()[1]; ++j) {
+        result->at(0, j) = zero<ValueType>();
+    }
+    for (size_type i = 0; i < x->get_size()[0]; ++i) {
+        for (size_type j = 0; j < x->get_size()[1]; ++j) {
+            result->at(0, j) += x->at(i, j) * conj(y->at(i, j));
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_CONJ_DOT_KERNEL);
+
+
 template <typename ValueType>
 void compute_norm2(std::shared_ptr<const ReferenceExecutor> exec,
                    const matrix::Dense<ValueType> *x,
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index f385a838b05..46ea7704030 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -434,6 +434,36 @@ TYPED_TEST(Dense, ComputesDotMixed)
 }
 
 
+TYPED_TEST(Dense, ComputesConjDot)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto result = Mtx::create(this->exec, gko::dim<2>{1, 3});
+
+    this->mtx1->compute_conj_dot(this->mtx3.get(), result.get());
+
+    EXPECT_EQ(result->at(0, 0), T{1.75});
+    EXPECT_EQ(result->at(0, 1), T{7.75});
+    ASSERT_EQ(result->at(0, 2), T{17.75});
+}
+
+
+TYPED_TEST(Dense, ComputesConjDotMixed)
+{
+    using MixedMtx = typename TestFixture::MixedMtx;
+    using MixedT = typename MixedMtx::value_type;
+    auto mmtx3 = MixedMtx::create(this->exec);
+    this->mtx3->convert_to(mmtx3.get());
+    auto result = MixedMtx::create(this->exec, gko::dim<2>{1, 3});
+
+    this->mtx1->compute_conj_dot(this->mtx3.get(), result.get());
+
+    EXPECT_EQ(result->at(0, 0), MixedT{1.75});
+    EXPECT_EQ(result->at(0, 1), MixedT{7.75});
+    ASSERT_EQ(result->at(0, 2), MixedT{17.75});
+}
+
+
 TYPED_TEST(Dense, ComputesNorm2)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -469,7 +499,7 @@ TYPED_TEST(Dense, ComputesNorm2Mixed)
 }
 
 
-TYPED_TEST(Dense, ComputDotFailsOnWrongInputSize)
+TYPED_TEST(Dense, ComputeDotFailsOnWrongInputSize)
 {
     using Mtx = typename TestFixture::Mtx;
     auto result = Mtx::create(this->exec, gko::dim<2>{1, 3});
@@ -479,7 +509,7 @@ TYPED_TEST(Dense, ComputDotFailsOnWrongInputSize)
 }
 
 
-TYPED_TEST(Dense, ComputDotFailsOnWrongResultSize)
+TYPED_TEST(Dense, ComputeDotFailsOnWrongResultSize)
 {
     using Mtx = typename TestFixture::Mtx;
     auto result = Mtx::create(this->exec, gko::dim<2>{1, 2});
@@ -489,6 +519,26 @@ TYPED_TEST(Dense, ComputDotFailsOnWrongResultSize)
 }
 
 
+TYPED_TEST(Dense, ComputeConjDotFailsOnWrongInputSize)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto result = Mtx::create(this->exec, gko::dim<2>{1, 3});
+
+    ASSERT_THROW(this->mtx1->compute_conj_dot(this->mtx2.get(), result.get()),
+                 gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(Dense, ComputeConjDotFailsOnWrongResultSize)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto result = Mtx::create(this->exec, gko::dim<2>{1, 2});
+
+    ASSERT_THROW(this->mtx1->compute_conj_dot(this->mtx3.get(), result.get()),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Dense, ConvertsToPrecision)
 {
     using Dense = typename TestFixture::Mtx;

From c7f7faf9ff5c1105bd660b23b98b48b6562ea5ca Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 28 Apr 2021 00:22:43 +0200
Subject: [PATCH 0639/1008] use conj_dot explicitly in solvers

---
 core/solver/bicg.cpp     | 4 ++--
 core/solver/bicgstab.cpp | 8 ++++----
 core/solver/cg.cpp       | 4 ++--
 core/solver/cgs.cpp      | 4 ++--
 core/solver/fcg.cpp      | 6 +++---
 core/solver/idr.cpp      | 4 ++--
 6 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp
index 29309955b03..f79832dedc0 100644
--- a/core/solver/bicg.cpp
+++ b/core/solver/bicg.cpp
@@ -208,7 +208,7 @@ void Bicg<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
     while (true) {
         get_preconditioner()->apply(r.get(), z.get());
         conj_trans_preconditioner->apply(r2.get(), z2.get());
-        z->compute_dot(r2.get(), rho.get());
+        z->compute_conj_dot(r2.get(), rho.get());
 
         ++iter;
         this->template log<log::Logger::iteration_complete>(
@@ -229,7 +229,7 @@ void Bicg<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
                                     rho.get(), prev_rho.get(), &stop_status));
         system_matrix_->apply(p.get(), q.get());
         conj_trans_A->apply(p2.get(), q2.get());
-        p2->compute_dot(q.get(), beta.get());
+        p2->compute_conj_dot(q.get(), beta.get());
         // tmp = rho / beta
         // x = x + tmp * p
         // r = r - tmp * q
diff --git a/core/solver/bicgstab.cpp b/core/solver/bicgstab.cpp
index 3996f165b5c..7c8c852fa11 100644
--- a/core/solver/bicgstab.cpp
+++ b/core/solver/bicgstab.cpp
@@ -166,7 +166,7 @@ void Bicgstab<ValueType>::apply_dense_impl(
         ++iter;
         this->template log<log::Logger::iteration_complete>(
             this, iter, r.get(), dense_x, nullptr, rho.get());
-        rr->compute_dot(r.get(), rho.get());
+        rr->compute_conj_dot(r.get(), rho.get());
 
         if (stop_criterion->update()
                 .num_iterations(iter)
@@ -185,7 +185,7 @@ void Bicgstab<ValueType>::apply_dense_impl(
 
         get_preconditioner()->apply(p.get(), y.get());
         system_matrix_->apply(y.get(), v.get());
-        rr->compute_dot(v.get(), beta.get());
+        rr->compute_conj_dot(v.get(), beta.get());
         // alpha = rho / beta
         // s = r - alpha * v
         exec->run(bicgstab::make_step_2(r.get(), s.get(), v.get(), rho.get(),
@@ -208,8 +208,8 @@ void Bicgstab<ValueType>::apply_dense_impl(
 
         get_preconditioner()->apply(s.get(), z.get());
         system_matrix_->apply(z.get(), t.get());
-        s->compute_dot(t.get(), gamma.get());
-        t->compute_dot(t.get(), beta.get());
+        s->compute_conj_dot(t.get(), gamma.get());
+        t->compute_conj_dot(t.get(), beta.get());
         // omega = gamma / beta
         // x = x + alpha * y + omega * z
         // r = s - omega * t
diff --git a/core/solver/cg.cpp b/core/solver/cg.cpp
index a675b4c51fa..4dc624584e8 100644
--- a/core/solver/cg.cpp
+++ b/core/solver/cg.cpp
@@ -151,7 +151,7 @@ void Cg<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
      */
     while (true) {
         get_preconditioner()->apply(r.get(), z.get());
-        r->compute_dot(z.get(), rho.get());
+        r->compute_conj_dot(z.get(), rho.get());
 
         ++iter;
         this->template log<log::Logger::iteration_complete>(
@@ -170,7 +170,7 @@ void Cg<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
         exec->run(cg::make_step_1(p.get(), z.get(), rho.get(), prev_rho.get(),
                                   &stop_status));
         system_matrix_->apply(p.get(), q.get());
-        p->compute_dot(q.get(), beta.get());
+        p->compute_conj_dot(q.get(), beta.get());
         // tmp = rho / beta
         // x = x + tmp * p
         // r = r - tmp * q
diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index d79c9c2518d..3d8419cb73c 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -161,7 +161,7 @@ void Cgs<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
      * 1x norm2 residual        n
      */
     while (true) {
-        r->compute_dot(r_tld.get(), rho.get());
+        r->compute_conj_dot(r_tld.get(), rho.get());
 
         ++iter;
         this->template log<log::Logger::iteration_complete>(
@@ -183,7 +183,7 @@ void Cgs<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
                                    &stop_status));
         get_preconditioner()->apply(p.get(), t.get());
         system_matrix_->apply(t.get(), v_hat.get());
-        r_tld->compute_dot(v_hat.get(), gamma.get());
+        r_tld->compute_conj_dot(v_hat.get(), gamma.get());
         // alpha = rho / gamma
         // q = u - alpha * v_hat
         // t = u + q
diff --git a/core/solver/fcg.cpp b/core/solver/fcg.cpp
index c9abc9e0e71..ee1eff61434 100644
--- a/core/solver/fcg.cpp
+++ b/core/solver/fcg.cpp
@@ -153,8 +153,8 @@ void Fcg<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
      */
     while (true) {
         get_preconditioner()->apply(r.get(), z.get());
-        r->compute_dot(z.get(), rho.get());
-        t->compute_dot(z.get(), rho_t.get());
+        r->compute_conj_dot(z.get(), rho.get());
+        t->compute_conj_dot(z.get(), rho_t.get());
 
         ++iter;
         this->template log<log::Logger::iteration_complete>(
@@ -173,7 +173,7 @@ void Fcg<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
         exec->run(fcg::make_step_1(p.get(), z.get(), rho_t.get(),
                                    prev_rho.get(), &stop_status));
         system_matrix_->apply(p.get(), q.get());
-        p->compute_dot(q.get(), beta.get());
+        p->compute_conj_dot(q.get(), beta.get());
         // tmp = rho / beta
         // [prev_r = r] in registers
         // x = x + tmp * p
diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp
index 388988ddcef..411ab892dd9 100644
--- a/core/solver/idr.cpp
+++ b/core/solver/idr.cpp
@@ -246,8 +246,8 @@ void Idr<ValueType>::iterate(const matrix::Dense<SubspaceType> *dense_b,
         get_preconditioner()->apply(residual.get(), helper.get());
         system_matrix_->apply(helper.get(), t.get());
 
-        t->compute_dot(residual.get(), omega.get());
-        t->compute_dot(t.get(), tht.get());
+        t->compute_conj_dot(residual.get(), omega.get());
+        t->compute_conj_dot(t.get(), tht.get());
         residual->compute_norm2(residual_norm.get());
 
         // omega = (t^H * residual) / (t^H * t)

From 0a5286a9a4262cd41aa7b0931863e2610288d4dc Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 28 Apr 2021 09:51:30 +0200
Subject: [PATCH 0640/1008] swap parameters for conj_dot CPU kernels

---
 omp/matrix/dense_kernels.cpp       | 2 +-
 reference/matrix/dense_kernels.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/omp/matrix/dense_kernels.cpp b/omp/matrix/dense_kernels.cpp
index 92b6f9d8389..65938de584f 100644
--- a/omp/matrix/dense_kernels.cpp
+++ b/omp/matrix/dense_kernels.cpp
@@ -241,7 +241,7 @@ void compute_conj_dot(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for
     for (size_type j = 0; j < x->get_size()[1]; ++j) {
         for (size_type i = 0; i < x->get_size()[0]; ++i) {
-            result->at(0, j) += x->at(i, j) * conj(y->at(i, j));
+            result->at(0, j) += conj(x->at(i, j)) * y->at(i, j);
         }
     }
 }
diff --git a/reference/matrix/dense_kernels.cpp b/reference/matrix/dense_kernels.cpp
index 17100775d6a..fa55253a4ac 100644
--- a/reference/matrix/dense_kernels.cpp
+++ b/reference/matrix/dense_kernels.cpp
@@ -220,7 +220,7 @@ void compute_conj_dot(std::shared_ptr<const ReferenceExecutor> exec,
     }
     for (size_type i = 0; i < x->get_size()[0]; ++i) {
         for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            result->at(0, j) += x->at(i, j) * conj(y->at(i, j));
+            result->at(0, j) += conj(x->at(i, j)) * y->at(i, j);
         }
     }
 }

From 81e2af14da57e3ea5c1f513ace52b9096a2fc9a9 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 28 Apr 2021 14:31:48 +0200
Subject: [PATCH 0641/1008] add tests for complex dot products

---
 cuda/test/matrix/dense_kernels.cpp      | 38 +++++++++++++++++++++++++
 hip/test/matrix/dense_kernels.hip.cpp   | 38 +++++++++++++++++++++++++
 omp/test/matrix/dense_kernels.cpp       | 38 +++++++++++++++++++++++++
 reference/test/matrix/dense_kernels.cpp | 34 ++++++++++++++++++++++
 4 files changed, 148 insertions(+)

diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index 6997233b8c6..61cc527f71b 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -485,6 +485,44 @@ TEST_F(Dense, AdvancedApplyToMixedComplexIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ComputeDotComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_b = ComplexMtx::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_x = ComplexMtx::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+    auto result = ComplexMtx::create(ref, gko::dim<2>{1, 2});
+    auto dresult = ComplexMtx::create(cuda, gko::dim<2>{1, 2});
+
+    complex_b->compute_dot(complex_x.get(), result.get());
+    dcomplex_b->compute_dot(dcomplex_x.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(result, dresult, 1e-14);
+}
+
+
+TEST_F(Dense, ComputeConjDotComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_b = ComplexMtx::create(cuda);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_x = ComplexMtx::create(cuda);
+    dcomplex_x->copy_from(complex_x.get());
+    auto result = ComplexMtx::create(ref, gko::dim<2>{1, 2});
+    auto dresult = ComplexMtx::create(cuda, gko::dim<2>{1, 2});
+
+    complex_b->compute_conj_dot(complex_x.get(), result.get());
+    dcomplex_b->compute_conj_dot(dcomplex_x.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(result, dresult, 1e-14);
+}
+
+
 TEST_F(Dense, IsTransposable)
 {
     set_up_apply_data();
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index 18721bc88c0..3140717c348 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -480,6 +480,44 @@ TEST_F(Dense, AdvancedApplyToMixedComplexIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ComputeDotComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_b = ComplexMtx::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_x = ComplexMtx::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+    auto result = ComplexMtx::create(ref, gko::dim<2>{1, 2});
+    auto dresult = ComplexMtx::create(hip, gko::dim<2>{1, 2});
+
+    complex_b->compute_dot(complex_x.get(), result.get());
+    dcomplex_b->compute_dot(dcomplex_x.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(result, dresult, 1e-14);
+}
+
+
+TEST_F(Dense, ComputeConjDotComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_b = ComplexMtx::create(hip);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_x = ComplexMtx::create(hip);
+    dcomplex_x->copy_from(complex_x.get());
+    auto result = ComplexMtx::create(ref, gko::dim<2>{1, 2});
+    auto dresult = ComplexMtx::create(hip, gko::dim<2>{1, 2});
+
+    complex_b->compute_conj_dot(complex_x.get(), result.get());
+    dcomplex_b->compute_conj_dot(dcomplex_x.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(result, dresult, 1e-14);
+}
+
+
 TEST_F(Dense, IsTransposable)
 {
     set_up_apply_data();
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index b057c6b6aae..a3942d2d82b 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -499,6 +499,44 @@ TEST_F(Dense, AdvancedApplyToMixedComplexIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ComputeDotComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_b = ComplexMtx::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_x = ComplexMtx::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+    auto result = ComplexMtx::create(ref, gko::dim<2>{1, 2});
+    auto dresult = ComplexMtx::create(omp, gko::dim<2>{1, 2});
+
+    complex_b->compute_dot(complex_x.get(), result.get());
+    dcomplex_b->compute_dot(dcomplex_x.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(result, dresult, 1e-14);
+}
+
+
+TEST_F(Dense, ComputeConjDotComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_b = ComplexMtx::create(omp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_x = ComplexMtx::create(omp);
+    dcomplex_x->copy_from(complex_x.get());
+    auto result = ComplexMtx::create(ref, gko::dim<2>{1, 2});
+    auto dresult = ComplexMtx::create(omp, gko::dim<2>{1, 2});
+
+    complex_b->compute_conj_dot(complex_x.get(), result.get());
+    dcomplex_b->compute_conj_dot(dcomplex_x.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(result, dresult, 1e-14);
+}
+
+
 TEST_F(Dense, ConvertToCooIsEquivalentToRef)
 {
     auto rmtx = gen_mtx<Mtx>(532, 231);
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 46ea7704030..a42968bfa52 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -2971,4 +2971,38 @@ TYPED_TEST(DenseComplex, GetImagWithGivenResult)
 }
 
 
+TYPED_TEST(DenseComplex, Dot)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = gko::ReferenceExecutor::create();
+    auto a =
+        gko::initialize<Mtx>({T{1.0, 0.0}, T{3.0, 4.0}, T{1.0, 2.0}}, exec);
+    auto b =
+        gko::initialize<Mtx>({T{1.0, -2.0}, T{5.0, 0.0}, T{0.0, -3.0}}, exec);
+    auto result = gko::initialize<Mtx>({T{0.0, 0.0}}, exec);
+
+    a->compute_dot(b.get(), result.get());
+
+    GKO_ASSERT_MTX_NEAR(result, l({T{22.0, 15.0}}), 0.0);
+}
+
+
+TYPED_TEST(DenseComplex, ConjDot)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = gko::ReferenceExecutor::create();
+    auto a =
+        gko::initialize<Mtx>({T{1.0, 0.0}, T{3.0, 4.0}, T{1.0, 2.0}}, exec);
+    auto b =
+        gko::initialize<Mtx>({T{1.0, -2.0}, T{5.0, 0.0}, T{0.0, -3.0}}, exec);
+    auto result = gko::initialize<Mtx>({T{0.0, 0.0}}, exec);
+
+    a->compute_conj_dot(b.get(), result.get());
+
+    GKO_ASSERT_MTX_NEAR(result, l({T{10.0, -25.0}}), 0.0);
+}
+
+
 }  // namespace

From 1c2dd14001d747b813255243eab87051ea47b18e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 28 Apr 2021 14:32:40 +0200
Subject: [PATCH 0642/1008] fix GPU conjugate dot product

---
 common/matrix/dense_kernels.hpp.inc |  2 +-
 cuda/base/cublas_bindings.hpp       | 27 ++++++++++++-
 cuda/matrix/dense_kernels.cu        | 51 +++++++++++++++----------
 hip/base/hipblas_bindings.hip.hpp   | 28 +++++++++++++-
 hip/matrix/dense_kernels.hip.cpp    | 59 +++++++++++++++++------------
 5 files changed, 119 insertions(+), 48 deletions(-)

diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index fd516d7f1ec..f1ab299698f 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -183,7 +183,7 @@ __global__ __launch_bounds__(block_size) void compute_partial_conj_dot(
     compute_partial_reduce<block_size>(
         num_rows, work,
         [x, stride_x, y, stride_y](size_type i) {
-            return x[i * stride_x] * conj(y[i * stride_y]);
+            return conj(x[i * stride_x]) * y[i * stride_y];
         },
         [](const ValueType &x, const ValueType &y) { return x + y; });
 }
diff --git a/cuda/base/cublas_bindings.hpp b/cuda/base/cublas_bindings.hpp
index 1850c5238d7..6dfb383ca39 100644
--- a/cuda/base/cublas_bindings.hpp
+++ b/cuda/base/cublas_bindings.hpp
@@ -206,14 +206,37 @@ GKO_BIND_CUBLAS_AXPY(ValueType, detail::not_implemented);
 
 GKO_BIND_CUBLAS_DOT(float, cublasSdot);
 GKO_BIND_CUBLAS_DOT(double, cublasDdot);
-GKO_BIND_CUBLAS_DOT(std::complex<float>, cublasCdotc);
-GKO_BIND_CUBLAS_DOT(std::complex<double>, cublasZdotc);
+GKO_BIND_CUBLAS_DOT(std::complex<float>, cublasCdotu);
+GKO_BIND_CUBLAS_DOT(std::complex<double>, cublasZdotu);
 template <typename ValueType>
 GKO_BIND_CUBLAS_DOT(ValueType, detail::not_implemented);
 
 #undef GKO_BIND_CUBLAS_DOT
 
 
+#define GKO_BIND_CUBLAS_CONJ_DOT(ValueType, CublasName)                       \
+    inline void conj_dot(cublasHandle_t handle, int n, const ValueType *x,    \
+                         int incx, const ValueType *y, int incy,              \
+                         ValueType *result)                                   \
+    {                                                                         \
+        GKO_ASSERT_NO_CUBLAS_ERRORS(CublasName(handle, n, as_culibs_type(x),  \
+                                               incx, as_culibs_type(y), incy, \
+                                               as_culibs_type(result)));      \
+    }                                                                         \
+    static_assert(true,                                                       \
+                  "This assert is used to counter the false positive extra "  \
+                  "semi-colon warnings")
+
+GKO_BIND_CUBLAS_CONJ_DOT(float, cublasSdot);
+GKO_BIND_CUBLAS_CONJ_DOT(double, cublasDdot);
+GKO_BIND_CUBLAS_CONJ_DOT(std::complex<float>, cublasCdotc);
+GKO_BIND_CUBLAS_CONJ_DOT(std::complex<double>, cublasZdotc);
+template <typename ValueType>
+GKO_BIND_CUBLAS_CONJ_DOT(ValueType, detail::not_implemented);
+
+#undef GKO_BIND_CUBLAS_CONJ_DOT
+
+
 #define GKO_BIND_CUBLAS_NORM2(ValueType, CublasName)                           \
     inline void norm2(cublasHandle_t handle, int n, const ValueType *x,        \
                       int incx, remove_complex<ValueType> *result)             \
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index 581653ca0b7..be02377a2b9 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -250,25 +250,38 @@ void compute_conj_dot(std::shared_ptr<const CudaExecutor> exec,
                       const matrix::Dense<ValueType> *y,
                       matrix::Dense<ValueType> *result)
 {
-    // TODO: these are tuning parameters obtained experimentally, once
-    // we decide how to handle this uniformly, they should be modified
-    // appropriately
-    constexpr auto work_per_thread = 32;
-    constexpr auto block_size = 1024;
-
-    constexpr auto work_per_block = work_per_thread * block_size;
-    const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
-    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
-    Array<ValueType> work(exec, grid_dim.x);
-    // TODO: write a kernel which does this more efficiently
-    for (size_type col = 0; col < x->get_size()[1]; ++col) {
-        kernel::compute_partial_conj_dot<block_size><<<grid_dim, block_dim>>>(
-            x->get_size()[0], as_cuda_type(x->get_const_values() + col),
-            x->get_stride(), as_cuda_type(y->get_const_values() + col),
-            y->get_stride(), as_cuda_type(work.get_data()));
-        kernel::finalize_sum_reduce_computation<block_size>
-            <<<1, block_dim>>>(grid_dim.x, as_cuda_type(work.get_const_data()),
-                               as_cuda_type(result->get_values() + col));
+    if (cublas::is_supported<ValueType>::value) {
+        // TODO: write a custom kernel which does this more efficiently
+        for (size_type col = 0; col < x->get_size()[1]; ++col) {
+            cublas::conj_dot(exec->get_cublas_handle(), x->get_size()[0],
+                             x->get_const_values() + col, x->get_stride(),
+                             y->get_const_values() + col, y->get_stride(),
+                             result->get_values() + col);
+        }
+    } else {
+        // TODO: these are tuning parameters obtained experimentally, once
+        // we decide how to handle this uniformly, they should be modified
+        // appropriately
+        constexpr auto work_per_thread = 32;
+        constexpr auto block_size = 1024;
+
+        constexpr auto work_per_block = work_per_thread * block_size;
+        const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
+        const dim3 block_dim{config::warp_size, 1,
+                             block_size / config::warp_size};
+        Array<ValueType> work(exec, grid_dim.x);
+        // TODO: write a kernel which does this more efficiently
+        for (size_type col = 0; col < x->get_size()[1]; ++col) {
+            kernel::compute_partial_conj_dot<block_size>
+                <<<grid_dim, block_dim>>>(
+                    x->get_size()[0], as_cuda_type(x->get_const_values() + col),
+                    x->get_stride(), as_cuda_type(y->get_const_values() + col),
+                    y->get_stride(), as_cuda_type(work.get_data()));
+            kernel::finalize_sum_reduce_computation<block_size>
+                <<<1, block_dim>>>(grid_dim.x,
+                                   as_cuda_type(work.get_const_data()),
+                                   as_cuda_type(result->get_values() + col));
+        }
     }
 }
 
diff --git a/hip/base/hipblas_bindings.hip.hpp b/hip/base/hipblas_bindings.hip.hpp
index 019206f8ea9..3889df69ff1 100644
--- a/hip/base/hipblas_bindings.hip.hpp
+++ b/hip/base/hipblas_bindings.hip.hpp
@@ -209,8 +209,8 @@ GKO_BIND_HIPBLAS_AXPY(ValueType, detail::not_implemented);
 
 GKO_BIND_HIPBLAS_DOT(float, hipblasSdot);
 GKO_BIND_HIPBLAS_DOT(double, hipblasDdot);
-GKO_BIND_HIPBLAS_DOT(std::complex<float>, hipblasCdotc);
-GKO_BIND_HIPBLAS_DOT(std::complex<double>, hipblasZdotc);
+GKO_BIND_HIPBLAS_DOT(std::complex<float>, hipblasCdotu);
+GKO_BIND_HIPBLAS_DOT(std::complex<double>, hipblasZdotu);
 
 template <typename ValueType>
 GKO_BIND_HIPBLAS_DOT(ValueType, detail::not_implemented);
@@ -218,6 +218,30 @@ GKO_BIND_HIPBLAS_DOT(ValueType, detail::not_implemented);
 #undef GKO_BIND_HIPBLAS_DOT
 
 
+#define GKO_BIND_HIPBLAS_CONJ_DOT(ValueType, HipblasName)                    \
+    inline void conj_dot(hipblasHandle_t handle, int n, const ValueType *x,  \
+                         int incx, const ValueType *y, int incy,             \
+                         ValueType *result)                                  \
+    {                                                                        \
+        GKO_ASSERT_NO_HIPBLAS_ERRORS(                                        \
+            HipblasName(handle, n, as_hipblas_type(x), incx,                 \
+                        as_hipblas_type(y), incy, as_hipblas_type(result))); \
+    }                                                                        \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
+                  "semi-colon warnings")
+
+GKO_BIND_HIPBLAS_CONJ_DOT(float, hipblasSdot);
+GKO_BIND_HIPBLAS_CONJ_DOT(double, hipblasDdot);
+GKO_BIND_HIPBLAS_CONJ_DOT(std::complex<float>, hipblasCdotc);
+GKO_BIND_HIPBLAS_CONJ_DOT(std::complex<double>, hipblasZdotc);
+
+template <typename ValueType>
+GKO_BIND_HIPBLAS_CONJ_DOT(ValueType, detail::not_implemented);
+
+#undef GKO_BIND_HIPBLAS_CONJ_DOT
+
+
 #define GKO_BIND_HIPBLAS_NORM2(ValueType, HipblasName)                       \
     inline void norm2(hipblasHandle_t handle, int n, const ValueType *x,     \
                       int incx, remove_complex<ValueType> *result)           \
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index 7a02ab6367b..13ddd959dcf 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -261,30 +261,41 @@ void compute_conj_dot(std::shared_ptr<const HipExecutor> exec,
                       const matrix::Dense<ValueType> *y,
                       matrix::Dense<ValueType> *result)
 {
-    // TODO: these are tuning parameters obtained experimentally, once
-    // we decide how to handle this uniformly, they should be modified
-    // appropriately
-    constexpr auto work_per_thread = 32;
-    constexpr auto block_size = 1024;
-
-    constexpr auto work_per_block = work_per_thread * block_size;
-    const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
-    const dim3 block_dim{config::warp_size, 1, block_size / config::warp_size};
-    Array<ValueType> work(exec, grid_dim.x);
-    // TODO: write a kernel which does this more efficiently
-    for (size_type col = 0; col < x->get_size()[1]; ++col) {
-        hipLaunchKernelGGL(
-            HIP_KERNEL_NAME(kernel::compute_partial_conj_dot<block_size>),
-            dim3(grid_dim), dim3(block_dim), 0, 0, x->get_size()[0],
-            as_hip_type(x->get_const_values() + col), x->get_stride(),
-            as_hip_type(y->get_const_values() + col), y->get_stride(),
-            as_hip_type(work.get_data()));
-        hipLaunchKernelGGL(
-            HIP_KERNEL_NAME(
-                kernel::finalize_sum_reduce_computation<block_size>),
-            dim3(1), dim3(block_dim), 0, 0, grid_dim.x,
-            as_hip_type(work.get_const_data()),
-            as_hip_type(result->get_values() + col));
+    if (hipblas::is_supported<ValueType>::value) {
+        // TODO: write a custom kernel which does this more efficiently
+        for (size_type col = 0; col < x->get_size()[1]; ++col) {
+            hipblas::conj_dot(exec->get_hipblas_handle(), x->get_size()[0],
+                              x->get_const_values() + col, x->get_stride(),
+                              y->get_const_values() + col, y->get_stride(),
+                              result->get_values() + col);
+        }
+    } else {
+        // TODO: these are tuning parameters obtained experimentally, once
+        // we decide how to handle this uniformly, they should be modified
+        // appropriately
+        constexpr auto work_per_thread = 32;
+        constexpr auto block_size = 1024;
+
+        constexpr auto work_per_block = work_per_thread * block_size;
+        const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
+        const dim3 block_dim{config::warp_size, 1,
+                             block_size / config::warp_size};
+        Array<ValueType> work(exec, grid_dim.x);
+        // TODO: write a kernel which does this more efficiently
+        for (size_type col = 0; col < x->get_size()[1]; ++col) {
+            hipLaunchKernelGGL(
+                HIP_KERNEL_NAME(kernel::compute_partial_conj_dot<block_size>),
+                dim3(grid_dim), dim3(block_dim), 0, 0, x->get_size()[0],
+                as_hip_type(x->get_const_values() + col), x->get_stride(),
+                as_hip_type(y->get_const_values() + col), y->get_stride(),
+                as_hip_type(work.get_data()));
+            hipLaunchKernelGGL(
+                HIP_KERNEL_NAME(
+                    kernel::finalize_sum_reduce_computation<block_size>),
+                dim3(1), dim3(block_dim), 0, 0, grid_dim.x,
+                as_hip_type(work.get_const_data()),
+                as_hip_type(result->get_values() + col));
+        }
     }
 }
 

From 0c7713b0f30abb8e24b0d15b2e707b1c2c65bdb3 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 28 Apr 2021 15:05:54 +0200
Subject: [PATCH 0643/1008] update documentation for conj_dot

Co-authored-by: Pratik Nayak <pratik.nayak@kit.edu>
---
 include/ginkgo/core/matrix/dense.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 8af136ec8b0..83ada5e4dc1 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -518,7 +518,7 @@ class Dense
     }
 
     /**
-     * Computes the column-wise dot product of this matrix and `conj(b)`.
+     * Computes the column-wise dot product of `conj(this matrix)` and `b`.
      *
      * @param b  a Dense matrix of same dimension as this
      * @param result  a Dense row vector, used to store the dot product

From dec7625979891d98c300266fa999554dfe6b8a9d Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 27 Apr 2021 16:38:34 +0200
Subject: [PATCH 0644/1008] add logger access and clear functions

---
 include/ginkgo/core/log/logger.hpp | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp
index 049d964fa51..81a2fe9365c 100644
--- a/include/ginkgo/core/log/logger.hpp
+++ b/include/ginkgo/core/log/logger.hpp
@@ -594,6 +594,19 @@ class EnableLogging : public Loggable {
         }
     }
 
+    /**
+     * Returns the vector containing all loggers registered at this object.
+     *
+     * @return the vector containing all registered loggers.
+     */
+    const std::vector<std::shared_ptr<const Logger>> &get_loggers() const
+    {
+        return loggers_;
+    }
+
+    /** Remove all loggers registered at this object. */
+    void clear_loggers() { loggers_.clear(); }
+
 protected:
     template <size_type Event, typename... Params>
     void log(Params &&... params) const

From 53104f9c1bcd58c10b1acd47ff7dfd766f636b92 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 27 Apr 2021 17:27:30 +0200
Subject: [PATCH 0645/1008] test logger access

---
 core/test/log/logger.cpp | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/core/test/log/logger.cpp b/core/test/log/logger.cpp
index 483cc47093d..d191752b5c0 100644
--- a/core/test/log/logger.cpp
+++ b/core/test/log/logger.cpp
@@ -86,6 +86,40 @@ TEST(DummyLogged, CanAddMultipleLoggers)
 }
 
 
+TEST(DummyLogged, CanAccessLoggers)
+{
+    auto exec = gko::ReferenceExecutor::create();
+    DummyLoggedClass c;
+
+    auto logger1 = gko::share(
+        gko::log::Record::create(exec, gko::log::Logger::all_events_mask));
+    auto logger2 = gko::share(gko::log::Stream<>::create(
+        exec, gko::log::Logger::all_events_mask, std::cout));
+
+    c.add_logger(logger1);
+    c.add_logger(logger2);
+
+    ASSERT_EQ(c.get_loggers()[0], logger1);
+    ASSERT_EQ(c.get_loggers()[1], logger2);
+    ASSERT_EQ(c.get_num_loggers(), 2);
+}
+
+
+TEST(DummyLogged, CanClearLoggers)
+{
+    auto exec = gko::ReferenceExecutor::create();
+    DummyLoggedClass c;
+    c.add_logger(
+        gko::log::Record::create(exec, gko::log::Logger::all_events_mask));
+    c.add_logger(gko::log::Stream<>::create(
+        exec, gko::log::Logger::all_events_mask, std::cout));
+
+    c.clear_loggers();
+
+    ASSERT_EQ(c.get_num_loggers(), 0);
+}
+
+
 TEST(DummyLogged, CanRemoveLogger)
 {
     auto exec = gko::ReferenceExecutor::create();

From 440d3f9d2be08701ddaecdc0ad7a449a797e0e89 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 30 Mar 2021 17:19:39 +0200
Subject: [PATCH 0646/1008] Make DPC++ tests work with the new system.

---
 .gitlab-ci.yml                                | 40 +++++++-
 .gitlab/image.yml                             | 16 ++-
 CMakeLists.txt                                | 12 ++-
 benchmark/CMakeLists.txt                      |  1 +
 cmake/create_test.cmake                       |  9 ++
 core/test/log/logger.cpp                      |  8 +-
 core/test/multigrid/amgx_pgm.cpp              |  2 +-
 core/test/solver/cb_gmres.cpp                 |  2 +-
 core/test/solver/idr.cpp                      |  2 +-
 core/test/utils.hpp                           | 63 +++++++++---
 dpcpp/CMakeLists.txt                          |  6 +-
 dpcpp/components/absolute_array.dp.cpp        |  4 +-
 dpcpp/test/CMakeLists.txt                     |  1 +
 dpcpp/test/base/executor.dp.cpp               | 99 ++++++-------------
 dpcpp/test/components/absolute_array.cpp      |  5 +
 .../test/components/precision_conversion.cpp  |  2 +
 include/ginkgo/core/base/math.hpp             | 43 +++-----
 include/ginkgo/core/base/types.hpp            | 48 +++++++++
 reference/test/solver/cb_gmres_kernels.cpp    |  2 +-
 reference/test/solver/idr_kernels.cpp         |  2 +-
 20 files changed, 236 insertions(+), 131 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6da6284c3a9..e1387eca1b9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -31,6 +31,7 @@ include:
   BUILD_HIP: "OFF"
   BUILD_HWLOC: "ON"
   FAST_TESTS: "OFF"
+  DPCPP_SINGLE_MODE: "OFF"
   RUN_EXAMPLES: "OFF"
   CONFIG_LOG: "ON"
   CXX_FLAGS: ""
@@ -64,7 +65,6 @@ include:
       CUDA_HOST_STR=-DCMAKE_CUDA_HOST_COMPILER=$(which ${CXX_COMPILER});
       export CUDA_VISIBLE_DEVICES=$((RANDOM % 2));
       fi
-    - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then export SYCL_DEVICE_TYPE; fi
     - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
         -GNinja
         -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${CXX_COMPILER}
@@ -79,9 +79,9 @@ include:
         -DGINKGO_FAST_TESTS=${FAST_TESTS}
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
+        -DGINKGO_DPCPP_SINGLE_MODE=${DPCPP_SINGLE_MODE}
         -DGINKGO_EXPORT_BUILD_DIR=${EXPORT_BUILD_DIR}
     - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT} install
-    - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then unset SYCL_DEVICE_TYPE; fi
     - if [ "${EXPORT_BUILD_DIR}" == "ON" ]; then ninja test_exportbuild; fi
   dependencies: []
   except:
@@ -98,7 +98,7 @@ include:
       CUDA_HOST_STR=-DCMAKE_CUDA_HOST_COMPILER=$(which ${CXX_COMPILER});
       export CUDA_VISIBLE_DEVICES=$((RANDOM % 2));
       fi
-    - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then export SYCL_DEVICE_TYPE; fi
+    - if [ -n "${SYCL_DEVICE_FILTER}" ]; then export SYCL_DEVICE_FILTER; fi
     - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
         -GNinja
         -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${CXX_COMPILER}
@@ -112,6 +112,7 @@ include:
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_FAST_TESTS=${FAST_TESTS}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
+        -DGINKGO_DPCPP_SINGLE_MODE=${DPCPP_SINGLE_MODE}
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_EXPORT_BUILD_DIR=${EXPORT_BUILD_DIR}
     - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT} install
@@ -143,7 +144,7 @@ include:
             ninja validate_all_examples
         fi
       fi
-    - if [ ! -z ${SYCL_DEVICE_TYPE+x} ]; then unset SYCL_DEVICE_TYPE; fi
+    - if [ -n "${SYCL_DEVICE_FILTER}" ]; then unset SYCL_DEVICE_FILTER; fi
     - if [ "${EXPORT_BUILD_DIR}" == "ON" ]; then ninja test_exportbuild; fi
   dependencies: []
   except:
@@ -680,6 +681,37 @@ build/nocuda/intel/omp/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
 
+
+build/dpcpp/igpu/release/static:
+  <<: *default_build_with_test
+  extends:
+    - .full_test_condition
+    - .use_gko-oneapi-igpu
+  variables:
+    <<: *default_variables
+    C_COMPILER: "gcc"
+    CXX_COMPILER: "dpcpp"
+    BUILD_DPCPP: "ON"
+    BUILD_TYPE: "Release"
+    BUILD_SHARED_LIBS: "OFF"
+    DPCPP_SINGLE_MODE: "ON"
+    SYCL_DEVICE_FILTER: "GPU"
+
+build/dpcpp/dgpu/debug/shared:
+  <<: *default_build_with_test
+  image: localhost:5000/gko-oneapi
+  extends:
+    - .quick_test_condition
+    - .use_gko-oneapi-dgpu
+  variables:
+    <<: *default_variables
+    C_COMPILER: "gcc"
+    CXX_COMPILER: "dpcpp"
+    BUILD_DPCPP: "ON"
+    BUILD_TYPE: "Debug"
+    DPCPP_SINGLE_MODE: "ON"
+    SYCL_DEVICE_FILTER: "Level_Zero:GPU"
+
 # Job with important warnings as error
 warnings:
   <<: *default_build
diff --git a/.gitlab/image.yml b/.gitlab/image.yml
index b97db8f6f90..9559563e064 100644
--- a/.gitlab/image.yml
+++ b/.gitlab/image.yml
@@ -75,9 +75,21 @@
     - amdci
     - gpu
 
-.use_gko-oneapi:
+.use_gko-oneapi-cpu:
   image: ginkgohub/oneapi:latest
   tags:
     - private_ci
-    - bwuni
+    - intelci-igpu
     - cpu
+
+.use_gko-oneapi-igpu:
+  image: localhost:5000/gko-oneapi
+  tags:
+    - private_ci
+    - intel-igpu
+
+.use_gko-oneapi-dgpu:
+  image: localhost:5000/gko-oneapi
+  tags:
+    - private_ci
+    - intel-dgpu
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 671805b7b8d..d716522db4c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,9 +1,7 @@
-cmake_minimum_required(VERSION 3.9)
+cmake_minimum_required(VERSION 3.13)
 
 # Use *_ROOT environment variables for find_package calls
-if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.12)
-    cmake_policy(SET CMP0074 NEW)
-endif()
+cmake_policy(SET CMP0074 NEW)
 
 # Let CAS handle the CUDA architecture flags (for now)
 # Windows still gives CMP0104 warning if putting it in cuda.
@@ -54,6 +52,11 @@ set(GINKGO_VERBOSE_LEVEL "1" CACHE STRING
 if(MSVC)
     set(GINKGO_COMPILER_FLAGS "" CACHE STRING
         "Set the required CXX compiler flags, mainly used for warnings. Current default is ``")
+elseif(GINKGO_BUILD_DPCPP OR CMAKE_CXX_COMPILER MATCHES "dpcpp")
+    # For now always use `-ffp-model=precise` with DPC++. This can be removed when
+    # the floating point issues are fixed.
+    set(GINKGO_COMPILER_FLAGS "-Wpedantic;-ffp-model=precise" CACHE STRING
+        "Set the required CXX compiler flags, mainly used for warnings. Current default is `-Wpedantic;-fp-model strict`")
 else()
     set(GINKGO_COMPILER_FLAGS "-Wpedantic" CACHE STRING
         "Set the required CXX compiler flags, mainly used for warnings. Current default is `-Wpedantic`")
@@ -78,6 +81,7 @@ if(MSVC OR WIN32 OR CYGWIN OR APPLE)
 else()
     option(GINKGO_BUILD_HWLOC "Build Ginkgo with HWLOC. Default is ON. If a system HWLOC is not found, then we try to build it ourselves. Switch this OFF to disable HWLOC." ON)
 endif()
+option(GINKGO_DPCPP_SINGLE_MODE "Do not compile double kernels for the DPC++ backend." OFF)
 option(GINKGO_INSTALL_RPATH "Set the RPATH when installing its libraries." ON)
 option(GINKGO_INSTALL_RPATH_ORIGIN "Add $ORIGIN (Linux) or @loader_path (MacOS) to the installation RPATH." ON)
 option(GINKGO_INSTALL_RPATH_DEPENDENCIES "Add dependencies to the installation RPATH." OFF)
diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
index a8356d2af52..ad719467c2b 100644
--- a/benchmark/CMakeLists.txt
+++ b/benchmark/CMakeLists.txt
@@ -65,6 +65,7 @@ function(ginkgo_add_single_benchmark_executable name use_lib_linops macro_def)
     add_executable("${name}" ${ARGN})
     target_link_libraries("${name}" ginkgo gflags rapidjson)
     target_compile_definitions("${name}" PRIVATE "${macro_def}")
+    target_compile_options("${name}" PRIVATE ${GINKGO_COMPILER_FLAGS})
     ginkgo_benchmark_add_tuning_maybe("${name}")
     if("${use_lib_linops}")
         if (GINKGO_BUILD_CUDA)
diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index fcec9d689e6..d7a420c2066 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -4,6 +4,7 @@ function(ginkgo_create_test test_name)
     string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
     add_executable(${TEST_TARGET_NAME} ${test_name}.cpp)
     target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_14)
+    target_compile_options("${TEST_TARGET_NAME}" PRIVATE ${GINKGO_COMPILER_FLAGS})
     target_include_directories("${TEST_TARGET_NAME}"
         PRIVATE
         "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
@@ -13,6 +14,9 @@ function(ginkgo_create_test test_name)
     if (GINKGO_FAST_TESTS)
         target_compile_definitions(${TEST_TARGET_NAME} PRIVATE GINKGO_FAST_TESTS)
     endif()
+    if (GINKGO_COMPILING_DPCPP_TEST AND GINKGO_DPCPP_SINGLE_MODE)
+        target_compile_definitions("${TEST_TARGET_NAME}" PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
+    endif()
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
@@ -27,6 +31,9 @@ function(ginkgo_create_dpcpp_test test_name)
     add_executable(${TEST_TARGET_NAME} ${test_name}.dp.cpp)
     target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_17)
     target_compile_options("${TEST_TARGET_NAME}" PRIVATE "${GINKGO_DPCPP_FLAGS}")
+    if (GINKGO_DPCPP_SINGLE_MODE)
+        target_compile_definitions("${TEST_TARGET_NAME}" PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
+    endif()
     target_include_directories("${TEST_TARGET_NAME}"
         PRIVATE
         "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
@@ -51,6 +58,7 @@ function(ginkgo_create_thread_test test_name)
     string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
     add_executable(${TEST_TARGET_NAME} ${test_name}.cpp)
     target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_14)
+    target_compile_options("${TEST_TARGET_NAME}" PRIVATE ${GINKGO_COMPILER_FLAGS})
     target_include_directories("${TEST_TARGET_NAME}"
         PRIVATE
         "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
@@ -74,6 +82,7 @@ function(ginkgo_create_test_cpp_cuda_header test_name)
     string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
     add_executable(${TEST_TARGET_NAME} ${test_name}.cpp)
     target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_14)
+    target_compile_options("${TEST_TARGET_NAME}" PRIVATE ${GINKGO_COMPILER_FLAGS})
     target_include_directories("${TEST_TARGET_NAME}"
         PRIVATE
         "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
diff --git a/core/test/log/logger.cpp b/core/test/log/logger.cpp
index d191752b5c0..a831f833b8f 100644
--- a/core/test/log/logger.cpp
+++ b/core/test/log/logger.cpp
@@ -39,7 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
-#include <ginkgo/core/log/record.hpp>
+#include <ginkgo/core/log/convergence.hpp>
 #include <ginkgo/core/log/stream.hpp>
 
 
@@ -66,7 +66,7 @@ TEST(DummyLogged, CanAddLogger)
     DummyLoggedClass c;
 
     c.add_logger(
-        gko::log::Record::create(exec, gko::log::Logger::all_events_mask));
+        gko::log::Convergence<>::create(exec, gko::log::Logger::all_events_mask));
 
     ASSERT_EQ(c.get_num_loggers(), 1);
 }
@@ -78,7 +78,7 @@ TEST(DummyLogged, CanAddMultipleLoggers)
     DummyLoggedClass c;
 
     c.add_logger(
-        gko::log::Record::create(exec, gko::log::Logger::all_events_mask));
+        gko::log::Convergence<>::create(exec, gko::log::Logger::all_events_mask));
     c.add_logger(gko::log::Stream<>::create(
         exec, gko::log::Logger::all_events_mask, std::cout));
 
@@ -125,7 +125,7 @@ TEST(DummyLogged, CanRemoveLogger)
     auto exec = gko::ReferenceExecutor::create();
     DummyLoggedClass c;
     auto r = gko::share(
-        gko::log::Record::create(exec, gko::log::Logger::all_events_mask));
+        gko::log::Convergence<>::create(exec, gko::log::Logger::all_events_mask));
     c.add_logger(r);
     c.add_logger(gko::log::Stream<>::create(
         exec, gko::log::Logger::all_events_mask, std::cout));
diff --git a/core/test/multigrid/amgx_pgm.cpp b/core/test/multigrid/amgx_pgm.cpp
index 8648f38e3ba..b1b434b9fe2 100644
--- a/core/test/multigrid/amgx_pgm.cpp
+++ b/core/test/multigrid/amgx_pgm.cpp
@@ -72,7 +72,7 @@ class AmgxPgmFactory : public ::testing::Test {
     std::unique_ptr<typename MgLevel::Factory> amgxpgm_factory;
 };
 
-TYPED_TEST_CASE(AmgxPgmFactory, gko::test::ValueIndexTypes);
+TYPED_TEST_SUITE(AmgxPgmFactory, gko::test::ValueIndexTypes);
 
 
 TYPED_TEST(AmgxPgmFactory, FactoryKnowsItsExecutor)
diff --git a/core/test/solver/cb_gmres.cpp b/core/test/solver/cb_gmres.cpp
index 7b41667fc28..722decbdb7a 100644
--- a/core/test/solver/cb_gmres.cpp
+++ b/core/test/solver/cb_gmres.cpp
@@ -143,7 +143,7 @@ using TestTypes =
                      std::tuple<std::complex<double>, st_r2>,
                      std::tuple<std::complex<float>, st_keep>>;
 
-TYPED_TEST_CASE(CbGmres, TestTypes);
+TYPED_TEST_SUITE(CbGmres, TestTypes);
 
 
 TYPED_TEST(CbGmres, CbGmresFactoryKnowsItsExecutor)
diff --git a/core/test/solver/idr.cpp b/core/test/solver/idr.cpp
index af0080de435..62dfc1c7e03 100644
--- a/core/test/solver/idr.cpp
+++ b/core/test/solver/idr.cpp
@@ -89,7 +89,7 @@ class Idr : public ::testing::Test {
     }
 };
 
-TYPED_TEST_CASE(Idr, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Idr, gko::test::ValueTypes);
 
 
 TYPED_TEST(Idr, IdrFactoryKnowsItsExecutor)
diff --git a/core/test/utils.hpp b/core/test/utils.hpp
index fde94fcc652..6cc2ad102c5 100644
--- a/core/test/utils.hpp
+++ b/core/test/utils.hpp
@@ -57,41 +57,80 @@ namespace test {
 
 
 using ValueTypes =
+#if GINKGO_DPCPP_SINGLE_MODE
+    ::testing::Types<float, std::complex<float>>;
+#else
     ::testing::Types<float, double, std::complex<float>, std::complex<double>>;
+#endif
 
 
 using ComplexValueTypes =
+#if GINKGO_DPCPP_SINGLE_MODE
+    ::testing::Types<std::complex<float>>;
+#else
     ::testing::Types<std::complex<float>, std::complex<double>>;
+#endif
 
 
 using IndexTypes = ::testing::Types<gko::int32, gko::int64>;
 
 
 using ValueAndIndexTypes =
+#if GINKGO_DPCPP_SINGLE_MODE
+    ::testing::Types<float, std::complex<float>, gko::int32, gko::int64,
+                     gko::size_type>;
+#else
     ::testing::Types<float, double, std::complex<float>, std::complex<double>,
                      gko::int32, gko::int64, gko::size_type>;
+#endif
+
 
 using RealValueAndIndexTypes =
+#if GINKGO_DPCPP_SINGLE_MODE
+    ::testing::Types<float, gko::int32, gko::int64, gko::size_type>;
+#else
     ::testing::Types<float, double, gko::int32, gko::int64, gko::size_type>;
-
-using ValueIndexTypes = ::testing::Types<
-    std::tuple<float, gko::int32>, std::tuple<double, gko::int32>,
-    std::tuple<std::complex<float>, gko::int32>,
-    std::tuple<std::complex<double>, gko::int32>, std::tuple<float, gko::int64>,
-    std::tuple<double, gko::int64>, std::tuple<std::complex<float>, gko::int64>,
-    std::tuple<std::complex<double>, gko::int64>>;
-
-
-using RealValueIndexTypes = ::testing::Types<
-    std::tuple<float, gko::int32>, std::tuple<double, gko::int32>,
-    std::tuple<float, gko::int64>, std::tuple<double, gko::int64>>;
+#endif
+
+
+using ValueIndexTypes =
+#if GINKGO_DPCPP_SINGLE_MODE
+    ::testing::Types<std::tuple<float, gko::int32>,
+                     std::tuple<std::complex<float>, gko::int32>,
+                     std::tuple<float, gko::int64>,
+                     std::tuple<std::complex<float>, gko::int64>>;
+#else
+    ::testing::Types<
+        std::tuple<float, gko::int32>, std::tuple<double, gko::int32>,
+        std::tuple<std::complex<float>, gko::int32>,
+        std::tuple<std::complex<double>, gko::int32>,
+        std::tuple<float, gko::int64>, std::tuple<double, gko::int64>,
+        std::tuple<std::complex<float>, gko::int64>,
+        std::tuple<std::complex<double>, gko::int64>>;
+#endif
+
+
+using RealValueIndexTypes =
+#if GINKGO_DPCPP_SINGLE_MODE
+    ::testing::Types<std::tuple<float, gko::int32>,
+                     std::tuple<float, gko::int64>>;
+#else
+    ::testing::Types<
+        std::tuple<float, gko::int32>, std::tuple<double, gko::int32>,
+        std::tuple<float, gko::int64>, std::tuple<double, gko::int64>>;
+#endif
 
 
 using ComplexValueIndexTypes =
+#if GINKGO_DPCPP_SINGLE_MODE
+    ::testing::Types<std::tuple<std::complex<float>, gko::int32>,
+                     std::tuple<std::complex<float>, gko::int64>>;
+#else
     ::testing::Types<std::tuple<std::complex<float>, gko::int32>,
                      std::tuple<std::complex<double>, gko::int32>,
                      std::tuple<std::complex<float>, gko::int64>,
                      std::tuple<std::complex<double>, gko::int64>>;
+#endif
 
 
 template <typename Precision, typename OutputType>
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index c52c77e109b..3e009730865 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -51,11 +51,15 @@ target_sources(ginkgo_dpcpp
 
 ginkgo_compile_features(ginkgo_dpcpp)
 
-set(GINKGO_DPCPP_FLAGS ${GINKGO_COMPILER_FLAGS} -fsycl -fsycl-device-lib=libc)
+set(GINKGO_DPCPP_FLAGS ${GINKGO_COMPILER_FLAGS} -fsycl)
 set(GINKGO_DPCPP_FLAGS ${GINKGO_DPCPP_FLAGS} PARENT_SCOPE)
 target_compile_options(ginkgo_dpcpp PRIVATE "${GINKGO_DPCPP_FLAGS}")
 target_compile_features(ginkgo_dpcpp PRIVATE cxx_std_17)
+target_link_options(ginkgo_dpcpp PRIVATE -fsycl-device-lib=all)
 target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
+if (GINKGO_DPCPP_SINGLE_MODE)
+    target_compile_definitions(ginkgo_dpcpp PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
+endif()
 
 ginkgo_default_includes(ginkgo_dpcpp)
 ginkgo_install_library(ginkgo_dpcpp)
diff --git a/dpcpp/components/absolute_array.dp.cpp b/dpcpp/components/absolute_array.dp.cpp
index 6562ac8d2c8..42c5216f8a7 100644
--- a/dpcpp/components/absolute_array.dp.cpp
+++ b/dpcpp/components/absolute_array.dp.cpp
@@ -52,7 +52,7 @@ void inplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec,
     exec->get_queue()->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl::range<1>{n}, [=](sycl::id<1> idx_id) {
             const auto idx = idx_id[0];
-            data[idx] = dpcpp::abs(data[idx]);
+            data[idx] = abs(data[idx]);
         });
     });
 }
@@ -68,7 +68,7 @@ void outplace_absolute_array(std::shared_ptr<const DefaultExecutor> exec,
     exec->get_queue()->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl::range<1>{n}, [=](sycl::id<1> idx_id) {
             const auto idx = idx_id[0];
-            out[idx] = dpcpp::abs(in[idx]);
+            out[idx] = abs(in[idx]);
         });
     });
 }
diff --git a/dpcpp/test/CMakeLists.txt b/dpcpp/test/CMakeLists.txt
index 8e34766a1c5..daa5367f268 100644
--- a/dpcpp/test/CMakeLists.txt
+++ b/dpcpp/test/CMakeLists.txt
@@ -1,4 +1,5 @@
 include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
+set(GINKGO_COMPILING_DPCPP_TEST ON)
 
 add_subdirectory(base)
 add_subdirectory(components)
diff --git a/dpcpp/test/base/executor.dp.cpp b/dpcpp/test/base/executor.dp.cpp
index eb29ea6546c..2bb57e25933 100644
--- a/dpcpp/test/base/executor.dp.cpp
+++ b/dpcpp/test/base/executor.dp.cpp
@@ -60,31 +60,47 @@ class DpcppExecutor : public ::testing::Test {
 
     void SetUp()
     {
-        ASSERT_GT(gko::DpcppExecutor::get_num_devices("cpu"), 0);
-        dpcpp = gko::DpcppExecutor::create(0, omp, "cpu");
         if (gko::DpcppExecutor::get_num_devices("gpu") > 0) {
-            dpcpp2 = gko::DpcppExecutor::create(0, omp, "gpu");
+            dpcpp = gko::DpcppExecutor::create(0, omp, "gpu");
+            if (gko::DpcppExecutor::get_num_devices("gpu") > 1) {
+                dpcpp2 = gko::DpcppExecutor::create(1, omp, "gpu");
+            }
+        }
+        else if (gko::DpcppExecutor::get_num_devices("cpu") > 0) {
+            dpcpp = gko::DpcppExecutor::create(0, omp, "cpu");
+            if (gko::DpcppExecutor::get_num_devices("cpu") > 1)
+            {
+                dpcpp2 = gko::DpcppExecutor::create(1, omp, "cpu");
+            }
+        }
+        else {
+            GKO_NOT_IMPLEMENTED;
         }
     }
 
     void TearDown()
     {
-        if (dpcpp != nullptr) {
-            // ensure that previous calls finished and didn't throw an error
-            ASSERT_NO_THROW(dpcpp->synchronize());
+        // ensure that previous calls finished and didn't throw an error
+        ASSERT_NO_THROW(dpcpp->synchronize());
+        if (dpcpp2 != nullptr)
+        {
+            ASSERT_NO_THROW(dpcpp2->synchronize());
         }
     }
 
-    std::shared_ptr<gko::Executor> omp;
-    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
-    std::shared_ptr<const gko::DpcppExecutor> dpcpp2;
+    std::shared_ptr<gko::Executor> omp{};
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp{};
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp2{};
 };
 
 
 TEST_F(DpcppExecutor, CanInstantiateTwoExecutorsOnOneDevice)
 {
     auto dpcpp = gko::DpcppExecutor::create(0, omp);
-    auto dpcpp2 = gko::DpcppExecutor::create(0, omp);
+    if (dpcpp2 != nullptr)
+    {
+        auto dpcpp2 = gko::DpcppExecutor::create(0, omp);
+    }
 
     // We want automatic deinitialization to not create any error
 }
@@ -143,7 +159,7 @@ TEST_F(DpcppExecutor, KnowsNumberOfDevicesOfTypeAccelerator)
 }
 
 
-TEST_F(DpcppExecutor, AllocatesAndFreesMemoryOnCPU)
+TEST_F(DpcppExecutor, AllocatesAndFreesMemory)
 {
     int *ptr = nullptr;
 
@@ -152,18 +168,6 @@ TEST_F(DpcppExecutor, AllocatesAndFreesMemoryOnCPU)
 }
 
 
-TEST_F(DpcppExecutor, AllocatesAndFreesMemoryOnGPU)
-{
-    if (!dpcpp2) {
-        GTEST_SKIP() << "No DPC++ compatible GPU.";
-    }
-    int *ptr = nullptr;
-
-    ASSERT_NO_THROW(ptr = dpcpp2->alloc<int>(2));
-    ASSERT_NO_THROW(dpcpp2->free(ptr));
-}
-
-
 TEST_F(DpcppExecutor, FailsWhenOverallocating)
 {
     const gko::size_type num_elems = 1ll << 50;  // 4PB of integers
@@ -208,30 +212,6 @@ TEST_F(DpcppExecutor, CopiesDataToCPU)
     dpcpp->free(copy);
 }
 
-
-TEST_F(DpcppExecutor, CopiesDataToGPU)
-{
-    if (!dpcpp2) {
-        GTEST_SKIP() << "No DPC++ compatible GPU.";
-    }
-    int orig[] = {3, 8};
-    auto *copy = dpcpp2->alloc<int>(2);
-    gko::Array<bool> is_set(omp, 1);
-
-    dpcpp2->copy_from(omp.get(), 2, orig, copy);
-
-    is_set.set_executor(dpcpp2);
-    ASSERT_NO_THROW(dpcpp2->get_queue()->submit([&](sycl::handler &cgh) {
-        auto *is_set_ptr = is_set.get_data();
-        cgh.single_task([=]() { check_data(copy, is_set_ptr); });
-    }));
-    is_set.set_executor(omp);
-    ASSERT_EQ(*is_set.get_data(), true);
-    ASSERT_NO_THROW(dpcpp2->synchronize());
-    dpcpp2->free(copy);
-}
-
-
 void init_data(int *data)
 {
     data[0] = 3;
@@ -254,30 +234,13 @@ TEST_F(DpcppExecutor, CopiesDataFromCPU)
 }
 
 
-TEST_F(DpcppExecutor, CopiesDataFromGPU)
-{
-    if (!dpcpp2) {
-        GTEST_SKIP() << "No DPC++ compatible GPU.";
-    }
-    int copy[2];
-    auto orig = dpcpp2->alloc<int>(2);
-    dpcpp2->get_queue()->submit([&](sycl::handler &cgh) {
-        cgh.single_task([=]() { init_data(orig); });
-    });
-
-    omp->copy_from(dpcpp2.get(), 2, orig, copy);
-
-    EXPECT_EQ(3, copy[0]);
-    ASSERT_EQ(8, copy[1]);
-    dpcpp2->free(orig);
-}
-
-
 TEST_F(DpcppExecutor, CopiesDataFromDpcppToDpcpp)
 {
-    if (!dpcpp2) {
-        GTEST_SKIP() << "No DPC++ compatible GPU.";
+    if (dpcpp2 == nullptr)
+    {
+        GTEST_SKIP();
     }
+
     int copy[2];
     gko::Array<bool> is_set(omp, 1);
     auto orig = dpcpp->alloc<int>(2);
diff --git a/dpcpp/test/components/absolute_array.cpp b/dpcpp/test/components/absolute_array.cpp
index 71156c15fe9..47eeab25980 100644
--- a/dpcpp/test/components/absolute_array.cpp
+++ b/dpcpp/test/components/absolute_array.cpp
@@ -52,8 +52,13 @@ namespace {
 
 class AbsoluteArray : public ::testing::Test {
 protected:
+#if GINKGO_DPCPP_SINGLE_MODE
+    using value_type = float;
+    using complex_type = std::complex<float>;
+#else
     using value_type = double;
     using complex_type = std::complex<double>;
+#endif  // GINKGO_DPCPP_SINGLE_MODE
     AbsoluteArray()
         : ref(gko::ReferenceExecutor::create()),
           exec(gko::DpcppExecutor::create(0, ref)),
diff --git a/dpcpp/test/components/precision_conversion.cpp b/dpcpp/test/components/precision_conversion.cpp
index dc8e331d8fa..e9584e3b2db 100644
--- a/dpcpp/test/components/precision_conversion.cpp
+++ b/dpcpp/test/components/precision_conversion.cpp
@@ -97,6 +97,7 @@ class PrecisionConversion : public ::testing::Test {
 };
 
 
+#ifndef GINKGO_DPCPP_SINGLE_MODE
 TEST_F(PrecisionConversion, ConvertsReal)
 {
     gko::Array<double> dtmp;
@@ -168,6 +169,7 @@ TEST_F(PrecisionConversion, ConvertsComplexFromRef)
 
     GKO_ASSERT_ARRAY_EQ(dcvals, dout);
 }
+#endif  // GINKGO_DPCPP_SINGLE_MODE
 
 
 }  // namespace
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 91058522b3b..721de3a2066 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -98,6 +98,20 @@ using std::sqrt;
 }  // namespace kernels
 
 
+namespace dpcpp {
+namespace reference {
+
+
+using std::abs;
+
+
+using std::sqrt;
+
+
+}  // namespace reference
+}  // namespace kernels
+
+
 namespace test {
 
 
@@ -913,13 +927,7 @@ GKO_INLINE GKO_ATTRIBUTES constexpr xstd::enable_if_t<is_complex_s<T>::value,
                                                       remove_complex<T>>
 abs(const T &x)
 {
-#ifdef CL_SYCL_LANGUAGE_VERSION
-    // FIXME: This implementation is due to a DPC++ issue:
-    // plain `sqrt` call evaluates to `std::sqrt` which fails on GPUs
-    return cl::sycl::sqrt(squared_norm(x));
-#else
     return sqrt(squared_norm(x));
-#endif
 }
 
 
@@ -977,11 +985,7 @@ GKO_INLINE GKO_ATTRIBUTES std::enable_if_t<!is_complex_s<T>::value, bool>
 is_finite(const T &value)
 {
     constexpr T infinity{detail::infinity_impl<T>::value};
-#ifdef CL_SYCL_LANGUAGE_VERSION
-    return ::gko::abs(value) < infinity;
-#else
     return abs(value) < infinity;
-#endif
 }
 
 
@@ -1004,25 +1008,6 @@ is_finite(const T &value)
 }
 
 
-namespace kernels {
-namespace dpcpp {
-
-
-// For now this seems to be useless. Somehow, DPC++ doesn't use this
-// declaration and anyway always replace calls to `abs` by `std::abs`. To
-// reference this declaration, use `dpcpp::abs`.
-using ::gko::abs;
-
-
-#ifdef CL_SYCL_LANGUAGE_VERSION
-using cl::sycl::sqrt;
-#endif
-
-
-}  // namespace dpcpp
-}  // namespace kernels
-
-
 }  // namespace gko
 
 
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index d42325f2118..2c77aaef826 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -424,9 +424,16 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
  *                Should take one argument, which is replaced by the
  *                value type.
  */
+#if GINKGO_DPCPP_SINGLE_MODE
+#define GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(_macro) \
+    template _macro(float);                                     \
+    template <>                                                 \
+    _macro(double) GKO_NOT_IMPLEMENTED
+#else
 #define GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(_macro) \
     template _macro(float);                                     \
     template _macro(double)
+#endif
 
 
 /**
@@ -437,10 +444,18 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
  *                Should take one argument, which is replaced by the
  *                value type.
  */
+#if GINKGO_DPCPP_SINGLE_MODE
+#define GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(_macro)          \
+    GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(_macro); \
+    template _macro(std::complex<float>);                    \
+    template <>                                              \
+    _macro(std::complex<double>) GKO_NOT_IMPLEMENTED
+#else
 #define GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(_macro)          \
     GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(_macro); \
     template _macro(std::complex<float>);                    \
     template _macro(std::complex<double>)
+#endif
 
 
 /**
@@ -465,11 +480,21 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
  *                Should take two arguments, which are replaced by the
  *                value and index types.
  */
+#if GINKGO_DPCPP_SINGLE_MODE
+#define GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(_macro) \
+    template _macro(float, int32);                                        \
+    template <>                                                           \
+    _macro(double, int32) GKO_NOT_IMPLEMENTED;                            \
+    template _macro(float, int64);                                        \
+    template <>                                                           \
+    _macro(double, int64) GKO_NOT_IMPLEMENTED
+#else
 #define GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(_macro) \
     template _macro(float, int32);                                        \
     template _macro(double, int32);                                       \
     template _macro(float, int64);                                        \
     template _macro(double, int64)
+#endif
 
 
 /**
@@ -480,12 +505,23 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
  *                Should take two arguments, which are replaced by the
  *                value and index types.
  */
+#if GINKGO_DPCPP_SINGLE_MODE
+#define GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(_macro)          \
+    GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(_macro); \
+    template _macro(std::complex<float>, int32);                       \
+    template <>                                                        \
+    _macro(std::complex<double>, int32) GKO_NOT_IMPLEMENTED;           \
+    template _macro(std::complex<float>, int64);                       \
+    template <>                                                        \
+    _macro(std::complex<double>, int64) GKO_NOT_IMPLEMENTED
+#else
 #define GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(_macro)          \
     GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(_macro); \
     template _macro(std::complex<float>, int32);                       \
     template _macro(std::complex<double>, int32);                      \
     template _macro(std::complex<float>, int64);                       \
     template _macro(std::complex<double>, int64)
+#endif
 
 
 /**
@@ -497,11 +533,23 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
  *                Should take two arguments `src` and `dst`, which
  *                are replaced by the source and destination value type.
  */
+#if GINKGO_DPCPP_SINGLE_MODE
+#define GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(_macro)                  \
+    template <>                                                            \
+    _macro(float, double) GKO_NOT_IMPLEMENTED;                             \
+    template <>                                                            \
+    _macro(double, float) GKO_NOT_IMPLEMENTED;                             \
+    template <>                                                            \
+    _macro(std::complex<float>, std::complex<double>) GKO_NOT_IMPLEMENTED; \
+    template <>                                                            \
+    _macro(std::complex<double>, std::complex<float>) GKO_NOT_IMPLEMENTED
+#else
 #define GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(_macro)       \
     template _macro(float, double);                             \
     template _macro(double, float);                             \
     template _macro(std::complex<float>, std::complex<double>); \
     template _macro(std::complex<double>, std::complex<float>)
+#endif
 
 
 /**
diff --git a/reference/test/solver/cb_gmres_kernels.cpp b/reference/test/solver/cb_gmres_kernels.cpp
index b9ac6cfbc53..76f5d55e8b9 100644
--- a/reference/test/solver/cb_gmres_kernels.cpp
+++ b/reference/test/solver/cb_gmres_kernels.cpp
@@ -182,7 +182,7 @@ using TestTypes =
                      std::tuple<std::complex<double>, st_r2>,
                      std::tuple<std::complex<float>, st_keep>>;
 
-TYPED_TEST_CASE(CbGmres, TestTypes);
+TYPED_TEST_SUITE(CbGmres, TestTypes);
 
 
 TYPED_TEST(CbGmres, SolvesStencilSystem)
diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp
index c90882406d2..59c76b8c5c6 100644
--- a/reference/test/solver/idr_kernels.cpp
+++ b/reference/test/solver/idr_kernels.cpp
@@ -95,7 +95,7 @@ class Idr : public ::testing::Test {
     std::unique_ptr<typename Solver::Factory> idr_factory_precision;
 };
 
-TYPED_TEST_CASE(Idr, gko::test::ValueTypes);
+TYPED_TEST_SUITE(Idr, gko::test::ValueTypes);
 
 
 TYPED_TEST(Idr, SolvesDenseSystem)

From cbb866f0f94d698100c91944f48f09e82b1802f9 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Fri, 9 Apr 2021 09:37:31 +0200
Subject: [PATCH 0647/1008] Review update.

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
---
 .gitlab-ci.yml                    | 15 +++++++++++++++
 .gitlab/image.yml                 |  4 ++--
 CMakeLists.txt                    |  2 +-
 dpcpp/CMakeLists.txt              |  1 +
 dpcpp/get_info.cmake              |  1 +
 include/ginkgo/core/base/math.hpp |  4 ++--
 6 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e1387eca1b9..c48467ec0e1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -682,6 +682,20 @@ build/nocuda/intel/omp/release/static:
     BUILD_SHARED_LIBS: "OFF"
 
 
+build/dpcpp/cpu/release/static:
+  <<: *default_build_with_test
+  extends:
+    - .quick_test_condition
+    - .use_gko-oneapi-cpu
+  variables:
+    <<: *default_variables
+    C_COMPILER: "gcc"
+    CXX_COMPILER: "dpcpp"
+    BUILD_DPCPP: "ON"
+    BUILD_TYPE: "Release"
+    BUILD_SHARED_LIBS: "ON"
+    SYCL_DEVICE_FILTER: "CPU"
+
 build/dpcpp/igpu/release/static:
   <<: *default_build_with_test
   extends:
@@ -712,6 +726,7 @@ build/dpcpp/dgpu/debug/shared:
     DPCPP_SINGLE_MODE: "ON"
     SYCL_DEVICE_FILTER: "Level_Zero:GPU"
 
+
 # Job with important warnings as error
 warnings:
   <<: *default_build
diff --git a/.gitlab/image.yml b/.gitlab/image.yml
index 9559563e064..9779553522e 100644
--- a/.gitlab/image.yml
+++ b/.gitlab/image.yml
@@ -83,13 +83,13 @@
     - cpu
 
 .use_gko-oneapi-igpu:
-  image: localhost:5000/gko-oneapi
+  image: ginkgohub/oneapi:latest
   tags:
     - private_ci
     - intel-igpu
 
 .use_gko-oneapi-dgpu:
-  image: localhost:5000/gko-oneapi
+  image: ginkgohub/oneapi:latest
   tags:
     - private_ci
     - intel-dgpu
diff --git a/CMakeLists.txt b/CMakeLists.txt
index d716522db4c..9314d27664e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -56,7 +56,7 @@ elseif(GINKGO_BUILD_DPCPP OR CMAKE_CXX_COMPILER MATCHES "dpcpp")
     # For now always use `-ffp-model=precise` with DPC++. This can be removed when
     # the floating point issues are fixed.
     set(GINKGO_COMPILER_FLAGS "-Wpedantic;-ffp-model=precise" CACHE STRING
-        "Set the required CXX compiler flags, mainly used for warnings. Current default is `-Wpedantic;-fp-model strict`")
+        "Set the required CXX compiler flags, mainly used for warnings. Current default is `-Wpedantic;-ffp-model=precise`")
 else()
     set(GINKGO_COMPILER_FLAGS "-Wpedantic" CACHE STRING
         "Set the required CXX compiler flags, mainly used for warnings. Current default is `-Wpedantic`")
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 3e009730865..f9cc2183a49 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -56,6 +56,7 @@ set(GINKGO_DPCPP_FLAGS ${GINKGO_DPCPP_FLAGS} PARENT_SCOPE)
 target_compile_options(ginkgo_dpcpp PRIVATE "${GINKGO_DPCPP_FLAGS}")
 target_compile_features(ginkgo_dpcpp PRIVATE cxx_std_17)
 target_link_options(ginkgo_dpcpp PRIVATE -fsycl-device-lib=all)
+target_link_options(ginkgo_dpcpp PRIVATE -fsycl-device-code-split=per_kernel)
 target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
 if (GINKGO_DPCPP_SINGLE_MODE)
     target_compile_definitions(ginkgo_dpcpp PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
diff --git a/dpcpp/get_info.cmake b/dpcpp/get_info.cmake
index 7cbfa34f7d0..36918a3a8c6 100644
--- a/dpcpp/get_info.cmake
+++ b/dpcpp/get_info.cmake
@@ -1,6 +1,7 @@
 ginkgo_print_module_header(${detailed_log} "DPCPP")
 ginkgo_print_module_footer(${detailed_log} "DPCPP variables:")
 ginkgo_print_variable(${detailed_log} "GINKGO_DPCPP_FLAGS")
+ginkgo_print_variable(${detailed_log} "GINKGO_DPCPP_SINGLE_MODE")
 ginkgo_print_module_footer(${detailed_log} "DPCPP environment variables:")
 ginkgo_print_env_variable(${detailed_log} "SYCL_DEVICE_TYPE")
 ginkgo_print_env_variable(${detailed_log} "SYCL_BE")
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 721de3a2066..762eef63049 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -98,8 +98,8 @@ using std::sqrt;
 }  // namespace kernels
 
 
+namespace kernels {
 namespace dpcpp {
-namespace reference {
 
 
 using std::abs;
@@ -108,7 +108,7 @@ using std::abs;
 using std::sqrt;
 
 
-}  // namespace reference
+}  // namespace dpcpp
 }  // namespace kernels
 
 

From e6ae134db5ac093eb36eec738e27f986075d4308 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 28 Apr 2021 14:44:47 +0200
Subject: [PATCH 0648/1008] Use CMake property LOCATION for the HIP fallback.

---
 hip/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index cc4d66534fe..e0474c0b89d 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -269,7 +269,7 @@ if(GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_AMD_REGEX}")
     endif()
     if (NOT HIP_LIBAMDHIP64_LIBRARIES)
         # Fall back to anything
-        get_target_property(HIP_LIBAMDHIP64_LIBRARIES hip::amdhip64 IMPORTED_LOCATION)
+        get_target_property(HIP_LIBAMDHIP64_LIBRARIES hip::amdhip64 LOCATION)
     endif()
     target_link_libraries(ginkgo_hip PUBLIC ${HIP_LIBAMDHIP64_LIBRARIES})
 elseif(GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}")

From 75d13929d9b35b13bd88d4bcd82b2e5e428ef65f Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 28 Apr 2021 18:27:47 +0200
Subject: [PATCH 0649/1008] Fix side CMake issues.

+ Make HIP on CUDA platform not do the CMAKE_CUDA_COMPILER_VERSION
  check.
+ Fix an important bug affecting the CMAKE_REQUIRED_LIBRARIES management
  of build_type_helpers.cmake.

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 cmake/build_type_helpers.cmake | 3 +--
 hip/CMakeLists.txt             | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/cmake/build_type_helpers.cmake b/cmake/build_type_helpers.cmake
index 46ebf9b3b24..a39de34cf82 100644
--- a/cmake/build_type_helpers.cmake
+++ b/cmake/build_type_helpers.cmake
@@ -88,10 +88,9 @@ foreach(_LANG IN LISTS ENABLED_LANGUAGES ITEMS "HIP")
                 ${PROJECT_NAME}_${_LANG}_${_TYPE}_SUPPORTED)
         else()
             if(DEFINED ${PROJECT_NAME}_${_LANG}_${_TYPE}_SUPPORTED)
-                message(STATUS "Skipping ${_LANG}, not supported by build_type.cmake script")
+                message(STATUS "Skipping ${_LANG}, not supported by build_type_helpers.cmake script")
             endif()
             set(${PROJECT_NAME}_${_LANG}_${_TYPE}_SUPPORTED FALSE)
-            continue()
         endif()
         if(${PROJECT_NAME}_${_LANG}_${_TYPE}_SUPPORTED)
             if(_LANG STREQUAL "HIP" AND GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}")
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index e0474c0b89d..86edd13513e 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -1,5 +1,5 @@
 if (GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}"
-    AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 9.2)
+    AND GINKGO_BUILD_CUDA AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 9.2)
     message(FATAL_ERROR "Ginkgo HIP backend requires CUDA >= 9.2.")
 endif()
 

From 3b4072cc9a926120dac6218ffcc79329971a8bde Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Fri, 30 Apr 2021 13:01:29 +0200
Subject: [PATCH 0650/1008] Fix bugs with logger and conj_dot.

---
 core/test/log/logger.cpp           |  1 +
 dpcpp/matrix/dense_kernels.dp.cpp  |  9 +++++++++
 include/ginkgo/core/log/logger.hpp | 26 ++++++++++++++++----------
 3 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/core/test/log/logger.cpp b/core/test/log/logger.cpp
index a831f833b8f..eef70fba1f8 100644
--- a/core/test/log/logger.cpp
+++ b/core/test/log/logger.cpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/log/convergence.hpp>
+#include <ginkgo/core/log/record.hpp>
 #include <ginkgo/core/log/stream.hpp>
 
 
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 88c86b6849b..42f4a960861 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -126,6 +126,15 @@ void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL);
 
 
+template <typename ValueType>
+void compute_conj_dot(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Dense<ValueType> *x,
+                      const matrix::Dense<ValueType> *y,
+                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_CONJ_DOT_KERNEL);
+
+
 template <typename ValueType>
 void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::Dense<ValueType> *x,
diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp
index 81a2fe9365c..c26d83b48bb 100644
--- a/include/ginkgo/core/log/logger.hpp
+++ b/include/ginkgo/core/log/logger.hpp
@@ -130,7 +130,7 @@ protected:                                                           \
 public:                                                              \
     template <size_type Event, typename... Params>                   \
     std::enable_if_t<Event == _id && (_id < event_count_max)> on(    \
-        Params &&... params) const                                   \
+        Params &&...params) const                                    \
     {                                                                \
         if (enabled_events_ & (mask_type{1} << _id)) {               \
             this->on_##_event_name(std::forward<Params>(params)...); \
@@ -557,6 +557,17 @@ class Loggable {
      *       equal.
      */
     virtual void remove_logger(const Logger *logger) = 0;
+
+    /**
+     * Returns the vector containing all loggers registered at this object.
+     *
+     * @return the vector containing all registered loggers.
+     */
+    virtual const std::vector<std::shared_ptr<const Logger>> &get_loggers()
+        const = 0;
+
+    /** Remove all loggers registered at this object. */
+    virtual void clear_loggers() = 0;
 };
 
 
@@ -594,22 +605,17 @@ class EnableLogging : public Loggable {
         }
     }
 
-    /**
-     * Returns the vector containing all loggers registered at this object.
-     *
-     * @return the vector containing all registered loggers.
-     */
-    const std::vector<std::shared_ptr<const Logger>> &get_loggers() const
+    const std::vector<std::shared_ptr<const Logger>> &get_loggers()
+        const override
     {
         return loggers_;
     }
 
-    /** Remove all loggers registered at this object. */
-    void clear_loggers() { loggers_.clear(); }
+    void clear_loggers() override { loggers_.clear(); }
 
 protected:
     template <size_type Event, typename... Params>
-    void log(Params &&... params) const
+    void log(Params &&...params) const
     {
         for (auto &logger : loggers_) {
             logger->template on<Event>(std::forward<Params>(params)...);

From e01e5a37abd20f7190244c23a66a808024e397c4 Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Mon, 12 Apr 2021 15:58:51 +0000
Subject: [PATCH 0651/1008] Format files

Co-authored-by: Terry Cojean <tcojean@users.noreply.github.com>
---
 core/test/log/logger.cpp           | 12 ++++++------
 dpcpp/test/base/executor.dp.cpp    | 18 ++++++------------
 include/ginkgo/core/log/logger.hpp |  4 ++--
 3 files changed, 14 insertions(+), 20 deletions(-)

diff --git a/core/test/log/logger.cpp b/core/test/log/logger.cpp
index eef70fba1f8..d9b1cd11f4e 100644
--- a/core/test/log/logger.cpp
+++ b/core/test/log/logger.cpp
@@ -66,8 +66,8 @@ TEST(DummyLogged, CanAddLogger)
     auto exec = gko::ReferenceExecutor::create();
     DummyLoggedClass c;
 
-    c.add_logger(
-        gko::log::Convergence<>::create(exec, gko::log::Logger::all_events_mask));
+    c.add_logger(gko::log::Convergence<>::create(
+        exec, gko::log::Logger::all_events_mask));
 
     ASSERT_EQ(c.get_num_loggers(), 1);
 }
@@ -78,8 +78,8 @@ TEST(DummyLogged, CanAddMultipleLoggers)
     auto exec = gko::ReferenceExecutor::create();
     DummyLoggedClass c;
 
-    c.add_logger(
-        gko::log::Convergence<>::create(exec, gko::log::Logger::all_events_mask));
+    c.add_logger(gko::log::Convergence<>::create(
+        exec, gko::log::Logger::all_events_mask));
     c.add_logger(gko::log::Stream<>::create(
         exec, gko::log::Logger::all_events_mask, std::cout));
 
@@ -125,8 +125,8 @@ TEST(DummyLogged, CanRemoveLogger)
 {
     auto exec = gko::ReferenceExecutor::create();
     DummyLoggedClass c;
-    auto r = gko::share(
-        gko::log::Convergence<>::create(exec, gko::log::Logger::all_events_mask));
+    auto r = gko::share(gko::log::Convergence<>::create(
+        exec, gko::log::Logger::all_events_mask));
     c.add_logger(r);
     c.add_logger(gko::log::Stream<>::create(
         exec, gko::log::Logger::all_events_mask, std::cout));
diff --git a/dpcpp/test/base/executor.dp.cpp b/dpcpp/test/base/executor.dp.cpp
index 2bb57e25933..39d9e1e3313 100644
--- a/dpcpp/test/base/executor.dp.cpp
+++ b/dpcpp/test/base/executor.dp.cpp
@@ -65,15 +65,12 @@ class DpcppExecutor : public ::testing::Test {
             if (gko::DpcppExecutor::get_num_devices("gpu") > 1) {
                 dpcpp2 = gko::DpcppExecutor::create(1, omp, "gpu");
             }
-        }
-        else if (gko::DpcppExecutor::get_num_devices("cpu") > 0) {
+        } else if (gko::DpcppExecutor::get_num_devices("cpu") > 0) {
             dpcpp = gko::DpcppExecutor::create(0, omp, "cpu");
-            if (gko::DpcppExecutor::get_num_devices("cpu") > 1)
-            {
+            if (gko::DpcppExecutor::get_num_devices("cpu") > 1) {
                 dpcpp2 = gko::DpcppExecutor::create(1, omp, "cpu");
             }
-        }
-        else {
+        } else {
             GKO_NOT_IMPLEMENTED;
         }
     }
@@ -82,8 +79,7 @@ class DpcppExecutor : public ::testing::Test {
     {
         // ensure that previous calls finished and didn't throw an error
         ASSERT_NO_THROW(dpcpp->synchronize());
-        if (dpcpp2 != nullptr)
-        {
+        if (dpcpp2 != nullptr) {
             ASSERT_NO_THROW(dpcpp2->synchronize());
         }
     }
@@ -97,8 +93,7 @@ class DpcppExecutor : public ::testing::Test {
 TEST_F(DpcppExecutor, CanInstantiateTwoExecutorsOnOneDevice)
 {
     auto dpcpp = gko::DpcppExecutor::create(0, omp);
-    if (dpcpp2 != nullptr)
-    {
+    if (dpcpp2 != nullptr) {
         auto dpcpp2 = gko::DpcppExecutor::create(0, omp);
     }
 
@@ -236,8 +231,7 @@ TEST_F(DpcppExecutor, CopiesDataFromCPU)
 
 TEST_F(DpcppExecutor, CopiesDataFromDpcppToDpcpp)
 {
-    if (dpcpp2 == nullptr)
-    {
+    if (dpcpp2 == nullptr) {
         GTEST_SKIP();
     }
 
diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp
index c26d83b48bb..9ae4298aab0 100644
--- a/include/ginkgo/core/log/logger.hpp
+++ b/include/ginkgo/core/log/logger.hpp
@@ -130,7 +130,7 @@ protected:                                                           \
 public:                                                              \
     template <size_type Event, typename... Params>                   \
     std::enable_if_t<Event == _id && (_id < event_count_max)> on(    \
-        Params &&...params) const                                    \
+        Params &&... params) const                                   \
     {                                                                \
         if (enabled_events_ & (mask_type{1} << _id)) {               \
             this->on_##_event_name(std::forward<Params>(params)...); \
@@ -615,7 +615,7 @@ class EnableLogging : public Loggable {
 
 protected:
     template <size_type Event, typename... Params>
-    void log(Params &&...params) const
+    void log(Params &&... params) const
     {
         for (auto &logger : loggers_) {
             logger->template on<Event>(std::forward<Params>(params)...);

From 9c8623788b7fe64366e7be7b3604841acfa13ed9 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 9 Apr 2021 18:41:43 +0200
Subject: [PATCH 0652/1008] fix linking issues with OpenMP and pthread

This can sometimes affect std::shared_ptr refcount atomics
---
 cmake/GinkgoConfig.cmake.in |  4 ++--
 omp/CMakeLists.txt          | 11 +++++++++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 1d7bfc38bc5..8e5de759102 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -170,8 +170,8 @@ if(GINKGO_HAVE_HWLOC)
     find_package(HWLOC REQUIRED)
 endif()
 
-# HIP depends on Threads::Threads in some circumstances, but doesn't find it
-if (GINKGO_BUILD_HIP)
+# HIP and OpenMP depend on Threads::Threads in some circumstances, but don't find it
+if (GINKGO_BUILD_HIP OR GINKGO_BUILD_OMP)
     find_package(Threads REQUIRED)
 endif()
 
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index acfe33d3ef6..87cbc30cd53 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -44,6 +44,17 @@ target_sources(ginkgo_omp
 
 ginkgo_compile_features(ginkgo_omp)
 
+# The dependency to OpenMP's potential pthread dependency needs to be public
+# because well, it's a long story:
+# std::shared_ptr decides whether we need to increment/decrement the reference
+# counters atomically by calling the pthread function __gthread_active_p, i.e.
+# checking whether there are multiple active threads. Linking against OpenMP
+# also links against pthread on Linux. Due to issues with static initialization
+# order for pthread_key_create, it may happen that ginkgo_omp assumes that a
+# piece of code is being executed sequentially, causing race conditions in the
+# reference counters.
+find_package(Threads REQUIRED)
+target_link_libraries(ginkgo_omp PUBLIC Threads::Threads)
 target_link_libraries(ginkgo_omp PRIVATE "${OpenMP_CXX_LIBRARIES}")
 target_include_directories(ginkgo_omp PRIVATE "${OpenMP_CXX_INCLUDE_DIRS}")
 # We first separate the arguments, otherwise, the target_compile_options adds it as a string

From 873f1ba10240023cb7455f92c4faa45be0ada152 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 10 Apr 2021 02:28:49 +0200
Subject: [PATCH 0653/1008] extract allocations from Jacobi OpenMP loops

---
 omp/preconditioner/jacobi_kernels.cpp | 83 ++++++++++++++++-----------
 1 file changed, 48 insertions(+), 35 deletions(-)

diff --git a/omp/preconditioner/jacobi_kernels.cpp b/omp/preconditioner/jacobi_kernels.cpp
index df97082319d..1b0c0c6c1a5 100644
--- a/omp/preconditioner/jacobi_kernels.cpp
+++ b/omp/preconditioner/jacobi_kernels.cpp
@@ -326,28 +326,27 @@ inline bool invert_block(IndexType block_size, IndexType *perm,
 
 
 template <typename ReducedType, typename ValueType, typename IndexType>
-inline bool validate_precision_reduction_feasibility(
-    std::shared_ptr<const OmpExecutor> exec, IndexType block_size,
-    const ValueType *block, size_type stride)
+inline bool validate_precision_reduction_feasibility(IndexType block_size,
+                                                     const ValueType *block,
+                                                     size_type stride,
+                                                     ValueType *tmp_buffer,
+                                                     IndexType *tmp_perm)
 {
     using gko::detail::float_traits;
-    vector<ValueType> tmp(block_size * block_size, {}, exec);
-    vector<IndexType> perm(block_size, {}, exec);
-    std::iota(begin(perm), end(perm), IndexType{0});
+    std::iota(tmp_perm, tmp_perm + block_size, IndexType{0});
     for (IndexType i = 0; i < block_size; ++i) {
         for (IndexType j = 0; j < block_size; ++j) {
-            tmp[i * block_size + j] = static_cast<ValueType>(
+            tmp_buffer[i * block_size + j] = static_cast<ValueType>(
                 static_cast<ReducedType>(block[i * stride + j]));
         }
     }
     auto cond =
-        compute_inf_norm(block_size, block_size, tmp.data(), block_size);
-    auto succeeded =
-        invert_block(block_size, perm.data(), tmp.data(), block_size);
+        compute_inf_norm(block_size, block_size, tmp_buffer, block_size);
+    auto succeeded = invert_block(block_size, tmp_perm, tmp_buffer, block_size);
     if (!succeeded) {
         return false;
     }
-    cond *= compute_inf_norm(block_size, block_size, tmp.data(), block_size);
+    cond *= compute_inf_norm(block_size, block_size, tmp_buffer, block_size);
     return cond >= 1.0 &&
            cond * float_traits<remove_complex<ValueType>>::eps < 1e-3;
 }
@@ -371,52 +370,65 @@ void generate(std::shared_ptr<const OmpExecutor> exec,
     const auto prec = block_precisions.get_data();
     const auto group_size = storage_scheme.get_group_size();
     const auto cond = conditioning.get_data();
+    const auto num_threads = omp_get_max_threads();
+    // group_size blocks for the actual blocks, 1 for temporary storage
+    const auto parallel_blocks = (group_size + 1) * num_threads;
+    Array<ValueType> blocks_storage{
+        exec, static_cast<size_type>(parallel_blocks * max_block_size *
+                                     max_block_size)};
+    Array<IndexType> perm_storage{
+        exec, static_cast<size_type>(parallel_blocks * max_block_size)};
+    Array<uint32> pr_descriptor_storage(exec, parallel_blocks);
 #pragma omp parallel for
     for (size_type g = 0; g < num_blocks; g += group_size) {
-        vector<Array<ValueType>> block(group_size, {}, exec);
-        vector<Array<IndexType>> perm(group_size, {}, exec);
-        vector<uint32> pr_descriptors(group_size, uint32{} - 1, exec);
+        const auto thread_id = omp_get_thread_num();
+        const auto thread_offset = thread_id * (group_size + 1);
+        auto local_blocks_tmp =
+            blocks_storage.get_data() +
+            (thread_offset * max_block_size * max_block_size);
+        auto local_blocks = local_blocks_tmp + max_block_size * max_block_size;
+        auto local_perms_tmp =
+            perm_storage.get_data() + (thread_offset * max_block_size);
+        auto local_perms = local_perms_tmp + max_block_size;
+        auto pr_descriptors = pr_descriptor_storage.get_data() + thread_offset;
+        std::fill_n(pr_descriptors, group_size, uint32{} - 1);
         // extract group of blocks, invert them, figure out storage precision
         for (size_type b = 0; b < group_size; ++b) {
             if (b + g >= num_blocks) {
                 break;
             }
             const auto block_size = ptrs[g + b + 1] - ptrs[g + b];
-            block[b] = Array<ValueType>(exec, block_size * block_size);
-            perm[b] = Array<IndexType>(exec, block_size);
-            std::iota(perm[b].get_data(), perm[b].get_data() + block_size,
-                      IndexType(0));
-            extract_block(system_matrix, block_size, ptrs[g + b],
-                          block[b].get_data(), block_size);
+            auto block = &local_blocks[b * max_block_size * max_block_size];
+            auto perm = &local_perms[b * max_block_size];
+            std::iota(perm, perm + block_size, IndexType{0});
+            extract_block(system_matrix, block_size, ptrs[g + b], block,
+                          block_size);
             if (cond) {
                 cond[g + b] =
-                    compute_inf_norm(block_size, block_size,
-                                     block[b].get_const_data(), block_size);
+                    compute_inf_norm(block_size, block_size, block, block_size);
             }
-            invert_block(block_size, perm[b].get_data(), block[b].get_data(),
-                         block_size);
+            invert_block(block_size, perm, block, block_size);
             if (cond) {
                 cond[g + b] *=
-                    compute_inf_norm(block_size, block_size,
-                                     block[b].get_const_data(), block_size);
+                    compute_inf_norm(block_size, block_size, block, block_size);
             }
             const auto local_prec = prec ? prec[g + b] : precision_reduction();
             if (local_prec == precision_reduction::autodetect() && cond) {
                 using preconditioner::detail::get_supported_storage_reductions;
                 pr_descriptors[b] = get_supported_storage_reductions<ValueType>(
                     accuracy, cond[g + b],
-                    [&exec, &block_size, &block, &b] {
+                    [&] {
                         using target = reduce_precision<ValueType>;
                         return validate_precision_reduction_feasibility<target>(
-                            exec, block_size, block[b].get_const_data(),
-                            block_size);
+                            block_size, block, block_size, local_blocks_tmp,
+                            local_perms_tmp);
                     },
-                    [&exec, &block_size, &block, &b] {
+                    [&] {
                         using target =
                             reduce_precision<reduce_precision<ValueType>>;
                         return validate_precision_reduction_feasibility<target>(
-                            exec, block_size, block[b].get_const_data(),
-                            block_size);
+                            block_size, block, block_size, local_blocks_tmp,
+                            local_perms_tmp);
                     });
             } else {
                 pr_descriptors[b] = preconditioner::detail::
@@ -426,7 +438,7 @@ void generate(std::shared_ptr<const OmpExecutor> exec,
 
         // make sure everyone in the group uses the same precision
         const auto p = preconditioner::detail::get_optimal_storage_reduction(
-            std::accumulate(begin(pr_descriptors), end(pr_descriptors),
+            std::accumulate(pr_descriptors, pr_descriptors + group_size,
                             uint32{} - 1,
                             [](uint32 x, uint32 y) { return x & y; }));
 
@@ -439,11 +451,12 @@ void generate(std::shared_ptr<const OmpExecutor> exec,
                 prec[g + b] = p;
             }
             const auto block_size = ptrs[g + b + 1] - ptrs[g + b];
+            auto block = &local_blocks[b * max_block_size * max_block_size];
+            auto perm = &local_perms[b * max_block_size];
             GKO_PRECONDITIONER_JACOBI_RESOLVE_PRECISION(
                 ValueType, p,
                 permute_and_transpose_block(
-                    block_size, perm[b].get_data(), block[b].get_data(),
-                    block_size,
+                    block_size, perm, block, block_size,
                     reinterpret_cast<resolved_precision *>(
                         blocks.get_data() +
                         storage_scheme.get_group_offset(g + b)) +

From f40790aec215f184304b9054cb123249ad7a91b9 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 10 Apr 2021 02:32:10 +0200
Subject: [PATCH 0654/1008] extract allocations from CSR OpenMP

---
 omp/matrix/csr_kernels.cpp | 305 ++++++++++++++++++++++++++-----------
 1 file changed, 216 insertions(+), 89 deletions(-)

diff --git a/omp/matrix/csr_kernels.cpp b/omp/matrix/csr_kernels.cpp
index 44e8f18a74e..e48a7262a8d 100644
--- a/omp/matrix/csr_kernels.cpp
+++ b/omp/matrix/csr_kernels.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
+#include <limits>
 #include <numeric>
 #include <utility>
 
@@ -51,6 +52,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/base/allocator.hpp"
 #include "core/base/iterator_factory.hpp"
+#include "core/base/utils.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/csr_builder.hpp"
 #include "omp/components/csr_spgeam.hpp"
@@ -130,81 +132,133 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_ADVANCED_SPMV_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void spgemm_insert_row(unordered_set<IndexType> &cols,
-                       const matrix::Csr<ValueType, IndexType> *c,
-                       size_type row)
-{
-    auto row_ptrs = c->get_const_row_ptrs();
-    auto col_idxs = c->get_const_col_idxs();
-    cols.insert(col_idxs + row_ptrs[row], col_idxs + row_ptrs[row + 1]);
-}
+namespace {
 
 
 template <typename ValueType, typename IndexType>
-void spgemm_insert_row2(unordered_set<IndexType> &cols,
-                        const matrix::Csr<ValueType, IndexType> *a,
-                        const matrix::Csr<ValueType, IndexType> *b,
-                        size_type row)
-{
-    auto a_row_ptrs = a->get_const_row_ptrs();
-    auto a_col_idxs = a->get_const_col_idxs();
-    auto b_row_ptrs = b->get_const_row_ptrs();
-    auto b_col_idxs = b->get_const_col_idxs();
-    for (size_type a_nz = a_row_ptrs[row];
-         a_nz < size_type(a_row_ptrs[row + 1]); ++a_nz) {
-        auto a_col = a_col_idxs[a_nz];
-        auto b_row = a_col;
-        cols.insert(b_col_idxs + b_row_ptrs[b_row],
-                    b_col_idxs + b_row_ptrs[b_row + 1]);
-    }
-}
+struct col_heap_element {
+    using value_type = ValueType;
+    using index_type = IndexType;
+    using matrix_type = matrix::Csr<ValueType, IndexType>;
+
+    IndexType idx;
+    IndexType end;
+    IndexType col;
+
+    ValueType val() const { return zero<ValueType>(); }
+
+    col_heap_element(IndexType idx, IndexType end, IndexType col, ValueType)
+        : idx{idx}, end{end}, col{col}
+    {}
+};
 
 
 template <typename ValueType, typename IndexType>
-void spgemm_accumulate_row(map<IndexType, ValueType> &cols,
-                           const matrix::Csr<ValueType, IndexType> *c,
-                           ValueType scale, size_type row)
+struct val_heap_element {
+    using value_type = ValueType;
+    using index_type = IndexType;
+    using matrix_type = matrix::Csr<ValueType, IndexType>;
+
+    IndexType idx;
+    IndexType end;
+    IndexType col;
+    ValueType val_;
+
+    ValueType val() const { return val_; }
+
+    val_heap_element(IndexType idx, IndexType end, IndexType col, ValueType val)
+        : idx{idx}, end{end}, col{col}, val_{val}
+    {}
+};
+
+
+template <typename HeapElement>
+void sift_down(HeapElement *heap, typename HeapElement::index_type idx,
+               typename HeapElement::index_type size)
 {
-    auto row_ptrs = c->get_const_row_ptrs();
-    auto col_idxs = c->get_const_col_idxs();
-    auto vals = c->get_const_values();
-    for (size_type c_nz = row_ptrs[row]; c_nz < size_type(row_ptrs[row + 1]);
-         ++c_nz) {
-        auto c_col = col_idxs[c_nz];
-        auto c_val = vals[c_nz];
-        cols[c_col] += scale * c_val;
+    auto curcol = heap[idx].col;
+    while (idx * 2 + 1 < size) {
+        auto lchild = idx * 2 + 1;
+        auto rchild = min(lchild + 1, size - 1);
+        auto lcol = heap[lchild].col;
+        auto rcol = heap[rchild].col;
+        auto mincol = min(lcol, rcol);
+        if (mincol >= curcol) {
+            break;
+        }
+        auto minchild = lcol == mincol ? lchild : rchild;
+        std::swap(heap[minchild], heap[idx]);
+        idx = minchild;
     }
 }
 
 
-template <typename ValueType, typename IndexType>
-void spgemm_accumulate_row2(map<IndexType, ValueType> &cols,
-                            const matrix::Csr<ValueType, IndexType> *a,
-                            const matrix::Csr<ValueType, IndexType> *b,
-                            ValueType scale, size_type row)
+template <typename HeapElement, typename InitCallback, typename StepCallback,
+          typename ColCallback>
+auto spgemm_multiway_merge(size_type row,
+                           const typename HeapElement::matrix_type *a,
+                           const typename HeapElement::matrix_type *b,
+                           HeapElement *heap, InitCallback init_cb,
+                           StepCallback step_cb, ColCallback col_cb)
+    -> decltype(init_cb(0))
 {
     auto a_row_ptrs = a->get_const_row_ptrs();
-    auto a_col_idxs = a->get_const_col_idxs();
+    auto a_cols = a->get_const_col_idxs();
     auto a_vals = a->get_const_values();
     auto b_row_ptrs = b->get_const_row_ptrs();
-    auto b_col_idxs = b->get_const_col_idxs();
+    auto b_cols = b->get_const_col_idxs();
     auto b_vals = b->get_const_values();
-    for (size_type a_nz = a_row_ptrs[row];
-         a_nz < size_type(a_row_ptrs[row + 1]); ++a_nz) {
-        auto a_col = a_col_idxs[a_nz];
-        auto a_val = a_vals[a_nz];
-        auto b_row = a_col;
-        for (size_type b_nz = b_row_ptrs[b_row];
-             b_nz < size_type(b_row_ptrs[b_row + 1]); ++b_nz) {
-            auto b_col = b_col_idxs[b_nz];
-            auto b_val = b_vals[b_nz];
-            cols[b_col] += scale * a_val * b_val;
+    auto a_begin = a_row_ptrs[row];
+    auto a_end = a_row_ptrs[row + 1];
+
+    using index_type = typename HeapElement::index_type;
+    constexpr auto sentinel = std::numeric_limits<index_type>::max();
+
+    auto state = init_cb(row);
+
+    // initialize the heap
+    for (auto a_nz = a_begin; a_nz < a_end; ++a_nz) {
+        auto b_row = a_cols[a_nz];
+        auto b_begin = b_row_ptrs[b_row];
+        auto b_end = b_row_ptrs[b_row + 1];
+        heap[a_nz] = {b_begin, b_end,
+                      checked_load(b_cols, b_begin, b_end, sentinel),
+                      a_vals[a_nz]};
+    }
+
+    if (a_begin != a_end) {
+        // make heap:
+        auto a_size = a_end - a_begin;
+        for (auto i = (a_size - 2) / 2; i >= 0; --i) {
+            sift_down(heap + a_begin, i, a_size);
+        }
+        auto &top = heap[a_begin];
+        auto &bot = heap[a_end - 1];
+        auto col = top.col;
+
+        while (top.col != sentinel) {
+            step_cb(b_vals[top.idx] * top.val(), top.col, state);
+            // move to the next element
+            top.idx++;
+            top.col = checked_load(b_cols, top.idx, top.end, sentinel);
+            // restore heap property
+            // pop_heap swaps top and bot, we need to prevent that
+            // so that we do a simple sift_down instead
+            sift_down(heap + a_begin, index_type{}, a_size);
+            if (top.col != col) {
+                col_cb(col, state);
+            }
+            col = top.col;
         }
     }
+
+    return state;
 }
 
 
+}  // namespace
+
+
 template <typename ValueType, typename IndexType>
 void spgemm(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Csr<ValueType, IndexType> *a,
@@ -212,18 +266,29 @@ void spgemm(std::shared_ptr<const OmpExecutor> exec,
             matrix::Csr<ValueType, IndexType> *c)
 {
     auto num_rows = a->get_size()[0];
-
-    // first sweep: count nnz for each row
     auto c_row_ptrs = c->get_row_ptrs();
 
-    unordered_set<IndexType> local_col_idxs(exec);
-#pragma omp parallel for firstprivate(local_col_idxs)
+    Array<col_heap_element<ValueType, IndexType>> col_heap_array(
+        exec, a->get_num_stored_elements());
+
+    auto col_heap = col_heap_array.get_data();
+
+    // first sweep: count nnz for each row
+#pragma omp parallel for
     for (size_type a_row = 0; a_row < num_rows; ++a_row) {
-        local_col_idxs.clear();
-        spgemm_insert_row2(local_col_idxs, a, b, a_row);
-        c_row_ptrs[a_row] = local_col_idxs.size();
+        c_row_ptrs[a_row] = spgemm_multiway_merge(
+            a_row, a, b, col_heap, [](size_type) { return IndexType{}; },
+            [](ValueType, IndexType, IndexType &) {},
+            [](IndexType, IndexType &nnz) { nnz++; });
     }
 
+    col_heap_array.clear();
+
+    Array<val_heap_element<ValueType, IndexType>> heap_array(
+        exec, a->get_num_stored_elements());
+
+    auto heap = heap_array.get_data();
+
     // build row pointers
     components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
 
@@ -237,18 +302,21 @@ void spgemm(std::shared_ptr<const OmpExecutor> exec,
     auto c_col_idxs = c_col_idxs_array.get_data();
     auto c_vals = c_vals_array.get_data();
 
-    map<IndexType, ValueType> local_row_nzs(exec);
-#pragma omp parallel for firstprivate(local_row_nzs)
+#pragma omp parallel for
     for (size_type a_row = 0; a_row < num_rows; ++a_row) {
-        local_row_nzs.clear();
-        spgemm_accumulate_row2(local_row_nzs, a, b, one<ValueType>(), a_row);
-        // store result
-        auto c_nz = c_row_ptrs[a_row];
-        for (auto pair : local_row_nzs) {
-            c_col_idxs[c_nz] = pair.first;
-            c_vals[c_nz] = pair.second;
-            ++c_nz;
-        }
+        spgemm_multiway_merge(
+            a_row, a, b, heap,
+            [&](size_type row) {
+                return std::make_pair(zero<ValueType>(), c_row_ptrs[row]);
+            },
+            [](ValueType val, IndexType,
+               std::pair<ValueType, IndexType> &state) { state.first += val; },
+            [&](IndexType col, std::pair<ValueType, IndexType> &state) {
+                c_col_idxs[state.second] = col;
+                c_vals[state.second] = state.first;
+                state.first = zero<ValueType>();
+                state.second++;
+            });
     }
 }
 
@@ -267,17 +335,41 @@ void advanced_spgemm(std::shared_ptr<const OmpExecutor> exec,
     auto num_rows = a->get_size()[0];
     auto valpha = alpha->at(0, 0);
     auto vbeta = beta->at(0, 0);
+    constexpr auto sentinel = std::numeric_limits<IndexType>::max();
 
     // first sweep: count nnz for each row
     auto c_row_ptrs = c->get_row_ptrs();
+    auto d_row_ptrs = d->get_const_row_ptrs();
+    auto d_cols = d->get_const_col_idxs();
+    auto d_vals = d->get_const_values();
+
+    Array<val_heap_element<ValueType, IndexType>> heap_array(
+        exec, a->get_num_stored_elements());
 
-    unordered_set<IndexType> local_col_idxs(exec);
-#pragma omp parallel for firstprivate(local_col_idxs)
+    auto heap = heap_array.get_data();
+    auto col_heap =
+        reinterpret_cast<col_heap_element<ValueType, IndexType> *>(heap);
+
+    // first sweep: count nnz for each row
+#pragma omp parallel for
     for (size_type a_row = 0; a_row < num_rows; ++a_row) {
-        local_col_idxs.clear();
-        spgemm_insert_row(local_col_idxs, d, a_row);
-        spgemm_insert_row2(local_col_idxs, a, b, a_row);
-        c_row_ptrs[a_row] = local_col_idxs.size();
+        auto d_nz = d_row_ptrs[a_row];
+        auto d_end = d_row_ptrs[a_row + 1];
+        auto d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+        c_row_ptrs[a_row] = spgemm_multiway_merge(
+            a_row, a, b, col_heap, [](size_type row) { return IndexType{}; },
+            [](ValueType, IndexType, IndexType &) {},
+            [&](IndexType col, IndexType &nnz) {
+                // skip smaller elements from d
+                while (d_col <= col) {
+                    d_nz++;
+                    nnz += d_col != col;
+                    d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+                }
+                nnz++;
+            });
+        // handle the remaining columns from d
+        c_row_ptrs[a_row] += d_end - d_nz;
     }
 
     // build row pointers
@@ -293,18 +385,53 @@ void advanced_spgemm(std::shared_ptr<const OmpExecutor> exec,
     auto c_col_idxs = c_col_idxs_array.get_data();
     auto c_vals = c_vals_array.get_data();
 
-    map<IndexType, ValueType> local_row_nzs(exec);
-#pragma omp parallel for firstprivate(local_row_nzs)
+#pragma omp parallel for
     for (size_type a_row = 0; a_row < num_rows; ++a_row) {
-        local_row_nzs.clear();
-        spgemm_accumulate_row(local_row_nzs, d, vbeta, a_row);
-        spgemm_accumulate_row2(local_row_nzs, a, b, valpha, a_row);
-        // store result
-        auto c_nz = c_row_ptrs[a_row];
-        for (auto pair : local_row_nzs) {
-            c_col_idxs[c_nz] = pair.first;
-            c_vals[c_nz] = pair.second;
-            ++c_nz;
+        auto d_nz = d_row_ptrs[a_row];
+        auto d_end = d_row_ptrs[a_row + 1];
+        auto d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+        auto d_val = checked_load(d_vals, d_nz, d_end, zero<ValueType>());
+        auto c_nz =
+            spgemm_multiway_merge(
+                a_row, a, b, heap,
+                [&](size_type row) {
+                    return std::make_pair(zero<ValueType>(), c_row_ptrs[row]);
+                },
+                [](ValueType val, IndexType,
+                   std::pair<ValueType, IndexType> &state) {
+                    state.first += val;
+                },
+                [&](IndexType col, std::pair<ValueType, IndexType> &state) {
+                    // handle smaller elements from d
+                    ValueType part_d_val{};
+                    while (d_col <= col) {
+                        if (d_col == col) {
+                            part_d_val = d_val;
+                        } else {
+                            c_col_idxs[state.second] = d_col;
+                            c_vals[state.second] = vbeta * d_val;
+                            state.second++;
+                        }
+                        d_nz++;
+                        d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+                        d_val = checked_load(d_vals, d_nz, d_end,
+                                             zero<ValueType>());
+                    }
+                    c_col_idxs[state.second] = col;
+                    c_vals[state.second] =
+                        vbeta * part_d_val + valpha * state.first;
+                    state.first = zero<ValueType>();
+                    state.second++;
+                })
+                .second;
+        // handle remaining elements from d
+        while (d_col < sentinel) {
+            c_col_idxs[c_nz] = d_col;
+            c_vals[c_nz] = vbeta * d_val;
+            c_nz++;
+            d_nz++;
+            d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+            d_val = checked_load(d_vals, d_nz, d_end, zero<ValueType>());
         }
     }
 }

From 07bdd288782301641a4f0dd524ed2e3a7ecdbfc7 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 19 Apr 2021 11:53:56 +0200
Subject: [PATCH 0655/1008] add SpGEMM documentation

---
 omp/matrix/csr_kernels.cpp | 62 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/omp/matrix/csr_kernels.cpp b/omp/matrix/csr_kernels.cpp
index e48a7262a8d..ebc67823c2a 100644
--- a/omp/matrix/csr_kernels.cpp
+++ b/omp/matrix/csr_kernels.cpp
@@ -135,6 +135,15 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 namespace {
 
 
+/**
+ * @internal
+ *
+ * Entry in a heap storing a column index and associated non-zero index
+ * (and row end) from a matrix.
+ *
+ * @tparam ValueType  The value type for matrices.
+ * @tparam IndexType  The index type for matrices.
+ */
 template <typename ValueType, typename IndexType>
 struct col_heap_element {
     using value_type = ValueType;
@@ -153,6 +162,15 @@ struct col_heap_element {
 };
 
 
+/**
+ * @internal
+ *
+ * Entry in a heap storing a entry (value and column index) and associated
+ * non-zero index (and row end) from a matrix.
+ *
+ * @tparam ValueType  The value type for matrices.
+ * @tparam IndexType  The index type for matrices.
+ */
 template <typename ValueType, typename IndexType>
 struct val_heap_element {
     using value_type = ValueType;
@@ -172,6 +190,20 @@ struct val_heap_element {
 };
 
 
+/**
+ * @internal
+ *
+ * Restores the binary heap condition downwards from a given index.
+ *
+ * The heap condition is: col(child) >= col(parent)
+ *
+ * @param heap  a pointer to the array containing the heap elements.
+ * @param idx  the index of the starting heap node that potentially
+ *             violates the heap condition.
+ * @param size  the number of elements in the heap.
+ * @tparam HeapElement  the element type in the heap. See col_heap_element and
+ *                      val_heap_element
+ */
 template <typename HeapElement>
 void sift_down(HeapElement *heap, typename HeapElement::index_type idx,
                typename HeapElement::index_type size)
@@ -193,6 +225,36 @@ void sift_down(HeapElement *heap, typename HeapElement::index_type idx,
 }
 
 
+/**
+ * @internal
+ *
+ * Generic SpGEMM implementation for a single output row of A * B using binary
+ * heap-based multiway merging.
+ *
+ * @param row  The row for which to compute the SpGEMM
+ * @param a  The input matrix A
+ * @param b  The input matrix B (its column indices must be sorted within each
+ *           row!)
+ * @param heap  The heap to use for this implementation. It must have as many
+ *              entries as the input row has non-zeros.
+ * @param init_cb  function to initialize the state for a single row. Its return
+ *                 value will be updated by subsequent calls of other callbacks,
+ *                 and then returned by this function. Its signature must be
+ *                 compatible with `return_type state = init_cb(row)`.
+ * @param step_cb  function that will be called for each accumulation from an
+ *                 entry of B into the output state. Its signature must be
+ *                 compatible with `step_cb(value, column, state)`.
+ * @param col_cb  function that will be called once for each output column after
+ *                all accumulations into it are completed. Its signature must be
+ *                compatible with `col_cb(column, state)`.
+ * @return the value initializaed by init_cb and updated by step_cb and col_cb
+ *
+ * @tparam HeapElement  the heap element type. See col_heap_element and
+ *                      val_heap_element
+ * @tparam InitCallback  functor type for init_cb
+ * @tparam StepCallback  functor type for step_cb
+ * @tparam ColCallback  functor type for col_cb
+ */
 template <typename HeapElement, typename InitCallback, typename StepCallback,
           typename ColCallback>
 auto spgemm_multiway_merge(size_type row,

From cbaf984debc44371f20b5b14abd53e5243cdcf5f Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 27 Apr 2021 18:31:40 +0200
Subject: [PATCH 0656/1008] add more OpenMP CSR SpGEMM tests

---
 omp/test/matrix/csr_kernels.cpp | 65 ++++++++++++++++++++++++++++++++-
 1 file changed, 63 insertions(+), 2 deletions(-)

diff --git a/omp/test/matrix/csr_kernels.cpp b/omp/test/matrix/csr_kernels.cpp
index 6215b336586..cd7a90e8005 100644
--- a/omp/test/matrix/csr_kernels.cpp
+++ b/omp/test/matrix/csr_kernels.cpp
@@ -91,14 +91,21 @@ class Csr : public ::testing::Test {
 
     template <typename MtxType>
     std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
-                                     int min_nnz_row)
+                                     int min_nnz_row, int max_nnz_row)
     {
         return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols,
-            std::uniform_int_distribution<>(min_nnz_row, num_cols),
+            std::uniform_int_distribution<>(min_nnz_row, max_nnz_row),
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
     }
 
+    template <typename MtxType>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row)
+    {
+        return gen_mtx<MtxType>(num_rows, num_cols, min_nnz_row, num_cols);
+    }
+
     void set_up_apply_data(int num_vectors = 1)
     {
         mtx = Mtx::create(ref);
@@ -257,6 +264,60 @@ TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, SimpleApplyToSparseCsrMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto mtx2 =
+        gen_mtx<Mtx>(mtx->get_size()[1], square_mtx->get_size()[1], 0, 10);
+    auto dmtx2 = Mtx::create(omp, mtx2->get_size());
+    dmtx2->copy_from(mtx2.get());
+
+    mtx->apply(mtx2.get(), square_mtx.get());
+    dmtx->apply(dmtx2.get(), square_dmtx.get());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
+    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
+}
+
+
+TEST_F(Csr, SimpleApplySparseToSparseCsrMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto mtx1 = gen_mtx<Mtx>(mtx->get_size()[0], mtx->get_size()[1], 0, 10);
+    auto mtx2 =
+        gen_mtx<Mtx>(mtx->get_size()[1], square_mtx->get_size()[1], 0, 10);
+    auto dmtx1 = Mtx::create(omp, mtx1->get_size());
+    auto dmtx2 = Mtx::create(omp, mtx2->get_size());
+    dmtx1->copy_from(mtx1.get());
+    dmtx2->copy_from(mtx2.get());
+
+    mtx1->apply(mtx2.get(), square_mtx.get());
+    dmtx1->apply(dmtx2.get(), square_dmtx.get());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
+    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
+}
+
+
+TEST_F(Csr, SimpleApplyToEmptyCsrMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto mtx2 =
+        gen_mtx<Mtx>(mtx->get_size()[1], square_mtx->get_size()[1], 0, 0);
+    auto dmtx2 = Mtx::create(omp, mtx2->get_size());
+    dmtx2->copy_from(mtx2.get());
+
+    mtx->apply(mtx2.get(), square_mtx.get());
+    dmtx->apply(dmtx2.get(), square_dmtx.get());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
+    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
+}
+
+
 TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
 {
     set_up_apply_data();

From 40044373e566092947681c11700b2b13096881a6 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 4 May 2021 10:08:28 +0200
Subject: [PATCH 0657/1008] remove hip-nvcc from clang-cuda builds

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c48467ec0e1..bbe85b1fbe8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -437,7 +437,7 @@ build/cuda101/clang/all/release/static:
     CUDA_ARCH: 35
 
 # clang-cuda with cuda 10.1 and friends
-build/clang-cuda101/gcc/all/release/shared:
+build/clang-cuda101/gcc/cuda/release/shared:
   <<: *default_build
   extends:
     - .quick_test_condition
@@ -447,7 +447,7 @@ build/clang-cuda101/gcc/all/release/shared:
     CUDA_COMPILER: "clang++"
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
-    BUILD_HIP: "ON"
+    BUILD_HIP: "OFF"
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
 

From d94a5f4ce07f9f7827e60535cff04f8d8e7fb853 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 4 May 2021 10:25:33 +0200
Subject: [PATCH 0658/1008] review update
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
Co-authored-by: Pratik Nayak <pratik.nayak@kit.edu>
---
 omp/matrix/csr_kernels.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/omp/matrix/csr_kernels.cpp b/omp/matrix/csr_kernels.cpp
index ebc67823c2a..4b56fe27048 100644
--- a/omp/matrix/csr_kernels.cpp
+++ b/omp/matrix/csr_kernels.cpp
@@ -165,7 +165,7 @@ struct col_heap_element {
 /**
  * @internal
  *
- * Entry in a heap storing a entry (value and column index) and associated
+ * Entry in a heap storing an entry (value and column index) and associated
  * non-zero index (and row end) from a matrix.
  *
  * @tparam ValueType  The value type for matrices.
@@ -247,7 +247,9 @@ void sift_down(HeapElement *heap, typename HeapElement::index_type idx,
  * @param col_cb  function that will be called once for each output column after
  *                all accumulations into it are completed. Its signature must be
  *                compatible with `col_cb(column, state)`.
- * @return the value initializaed by init_cb and updated by step_cb and col_cb
+ * @return the value initialized by init_cb and updated by step_cb and col_cb
+ * @note If the columns of B are not sorted, the output may have duplicate
+ *       column entries.
  *
  * @tparam HeapElement  the heap element type. See col_heap_element and
  *                      val_heap_element

From d6824fdfa167c157d3a69514b66eba68827aa492 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 8 Jul 2020 11:16:25 +0200
Subject: [PATCH 0659/1008] implement cuda amgx_pgm size2 expect for generate
 and related test

---
 common/multigrid/amgx_pgm_kernels.hpp.inc | 279 ++++++++++++++++++
 cuda/multigrid/amgx_pgm_kernels.cu        |  83 +++++-
 cuda/test/CMakeLists.txt                  |   1 +
 cuda/test/multigrid/CMakeLists.txt        |   1 +
 cuda/test/multigrid/amgx_pgm_kernels.cpp  | 341 ++++++++++++++++++++++
 5 files changed, 693 insertions(+), 12 deletions(-)
 create mode 100644 common/multigrid/amgx_pgm_kernels.hpp.inc
 create mode 100644 cuda/test/multigrid/CMakeLists.txt
 create mode 100644 cuda/test/multigrid/amgx_pgm_kernels.cpp

diff --git a/common/multigrid/amgx_pgm_kernels.hpp.inc b/common/multigrid/amgx_pgm_kernels.hpp.inc
new file mode 100644
index 00000000000..6f460ad98c2
--- /dev/null
+++ b/common/multigrid/amgx_pgm_kernels.hpp.inc
@@ -0,0 +1,279 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+namespace kernel {
+
+
+template <typename ValueType, typename IndexType>
+__global__ __launch_bounds__(default_block_size) void restrict_apply_kernel(
+    const IndexType *__restrict__ agg, const size_type num_rows,
+    const size_type num_rhs, const ValueType *__restrict__ b,
+    const size_type b_stride, ValueType *__restrict__ x,
+    const size_type x_stride)
+{
+    auto tidx = thread::get_thread_id_flat();
+    auto row = tidx / num_rhs;
+    if (row >= num_rows) {
+        return;
+    }
+    auto col = tidx % num_rhs;
+    auto ind = agg[row];
+    atomic_add(x + ind * x_stride + col, b[row * b_stride + col]);
+}
+
+
+template <typename ValueType, typename IndexType>
+__global__ __launch_bounds__(default_block_size) void prolong_applyadd_kernel(
+    const IndexType *__restrict__ agg, const size_type num_rows,
+    const size_type num_rhs, const ValueType *__restrict__ b,
+    const size_type b_stride, ValueType *__restrict__ x,
+    const size_type x_stride)
+{
+    auto tidx = thread::get_thread_id_flat();
+    auto row = tidx / num_rhs;
+    if (row >= num_rows) {
+        return;
+    }
+    auto col = tidx % num_rhs;
+    auto ind = agg[row];
+    x[row * x_stride + col] += b[ind * b_stride + col];
+}
+
+
+template <typename IndexType>
+__global__ __launch_bounds__(default_block_size) void replace_kernel(
+    size_type size, const IndexType *__restrict__ source,
+    IndexType *__restrict__ result)
+{
+    auto tidx = thread::get_thread_id_flat();
+    if (tidx >= size) {
+        return;
+    }
+
+    result[tidx] = source[tidx] == -1;
+}
+
+
+template <typename IndexType>
+__global__ __launch_bounds__(default_block_size) void match_edge_kernel(
+    size_type num, const IndexType *__restrict__ strongest_neighbor_vals,
+    IndexType *__restrict__ agg_vals)
+{
+    auto tidx = thread::get_thread_id_flat();
+    if (tidx >= num) {
+        return;
+    }
+    if (agg_vals[tidx] != -1) {
+        return;
+    }
+    auto neighbor = strongest_neighbor_vals[tidx];
+    if (neighbor != -1 && strongest_neighbor_vals[neighbor] == tidx) {
+        agg_vals[tidx] = tidx;
+        agg_vals[neighbor] = tidx;
+        // Use the smaller index as agg point
+    }
+}
+
+
+template <typename IndexType>
+__global__ __launch_bounds__(default_block_size) void activate_kernel(
+    size_type num, const IndexType *__restrict__ agg,
+    IndexType *__restrict__ active_agg)
+{
+    auto tidx = thread::get_thread_id_flat();
+    if (tidx >= num) {
+        return;
+    }
+    active_agg[tidx] = agg[tidx] == -1;
+}
+
+
+template <typename IndexType>
+__global__ __launch_bounds__(default_block_size) void fill_agg_kernel(
+    size_type num, const IndexType *__restrict__ index,
+    IndexType *__restrict__ result)
+{
+    auto tidx = thread::get_thread_id_flat();
+    if (tidx >= num) {
+        return;
+    }
+    result[index[tidx]] = 1;
+}
+
+
+template <typename IndexType>
+__global__ __launch_bounds__(default_block_size) void renumber_kernel(
+    size_type num, const IndexType *__restrict__ map,
+    IndexType *__restrict__ result)
+{
+    auto tidx = thread::get_thread_id_flat();
+    if (tidx >= num) {
+        return;
+    }
+    result[tidx] = map[result[tidx]];
+}
+
+
+template <typename ValueType, typename IndexType>
+__global__
+    __launch_bounds__(default_block_size) void find_strongest_neighbor_kernel(
+        const size_type num, const IndexType *__restrict__ row_ptrs,
+        const IndexType *__restrict__ col_idxs,
+        const ValueType *__restrict__ weight_vals,
+        const ValueType *__restrict__ diag, const size_type diag_stride,
+        IndexType *__restrict__ agg, IndexType *__restrict__ strongest_neighbor)
+{
+    auto row = thread::get_thread_id_flat();
+    if (row >= num) {
+        return;
+    }
+
+    auto max_weight_unagg = zero<ValueType>();
+    auto max_weight_agg = zero<ValueType>();
+    IndexType strongest_unagg = -1;
+    IndexType strongest_agg = -1;
+    if (agg[row] != -1) {
+        return;
+    }
+    for (auto idx = row_ptrs[row]; idx < row_ptrs[row + 1]; idx++) {
+        auto col = col_idxs[idx];
+        if (col == row) {
+            continue;
+        }
+        auto weight = weight_vals[idx] / max(abs(diag[row * diag_stride]),
+                                             abs(diag[col * diag_stride]));
+        if (agg[col] == -1 &&
+            (weight > max_weight_unagg ||
+             (weight == max_weight_unagg && col > strongest_unagg))) {
+            max_weight_unagg = weight;
+            strongest_unagg = col;
+        } else if (agg[col] != -1 &&
+                   (weight > max_weight_agg ||
+                    (weight == max_weight_agg && col > strongest_agg))) {
+            max_weight_agg = weight;
+            strongest_agg = col;
+        }
+    }
+
+    if (strongest_unagg == -1 && strongest_agg != -1) {
+        // all neighbor is agg, connect to the strongest agg
+        // Also, no others will use this item as their strongest_neighbor
+        // because they are already aggregated. Thus, it is determinstic
+        // behavior
+        agg[row] = agg[strongest_agg];
+    } else if (strongest_unagg != -1) {
+        // set the strongest neighbor in the unagg group
+        strongest_neighbor[row] = strongest_unagg;
+    } else {
+        // no neighbor
+        strongest_neighbor[row] = row;
+    }
+}
+
+
+template <typename ValueType, typename IndexType>
+__global__
+    __launch_bounds__(default_block_size) void assign_to_exist_agg_kernel(
+        const size_type num, const IndexType *__restrict__ row_ptrs,
+        const IndexType *__restrict__ col_idxs,
+        const ValueType *__restrict__ weight_vals,
+        const ValueType *__restrict__ diag, const size_type diag_stride,
+        const IndexType *__restrict__ agg_const_val,
+        IndexType *__restrict__ agg_val)
+{
+    auto row = thread::get_thread_id_flat();
+    if (row >= num) {
+        return;
+    }
+    if (agg_val[row] != -1) {
+        return;
+    }
+    ValueType max_weight_agg = zero<ValueType>();
+    IndexType strongest_agg = -1;
+    for (auto idx = row_ptrs[row]; idx < row_ptrs[row + 1]; idx++) {
+        auto col = col_idxs[idx];
+        if (col == row) {
+            continue;
+        }
+        auto weight = weight_vals[idx] / max(abs(diag[row * diag_stride]),
+                                             abs(diag[col * diag_stride]));
+        if (agg_const_val[col] != -1 &&
+            (weight > max_weight_agg ||
+             (weight == max_weight_agg && col > strongest_agg))) {
+            max_weight_agg = weight;
+            strongest_agg = col;
+        }
+    }
+    if (strongest_agg != -1) {
+        agg_val[row] = agg_const_val[strongest_agg];
+    } else {
+        agg_val[row] = row;
+    }
+}
+
+
+template <typename ValueType, typename IndexType>
+__global__ __launch_bounds__(default_block_size) void merge_col_kernel(
+    const size_type num, const IndexType *__restrict__ row_ptrs,
+    const IndexType *__restrict__ source_col,
+    const ValueType *__restrict__ source_val,
+    IndexType *__restrict__ result_col, ValueType *__restrict__ result_val)
+{
+    auto row = thread::get_thread_id_flat();
+    if (row >= num) {
+        return;
+    }
+    const auto start = row_ptrs[row];
+    const auto end = row_ptrs[row + 1];
+    auto result_ind = start;
+    auto col = source_col[result_ind];
+    auto temp = source_val[result_ind];
+    for (size_type idx = start + 1; idx < end; idx++) {
+        auto temp_col = source_col[idx];
+        if (temp_col != col) {
+            result_col[result_ind] = col;
+            result_val[result_ind] = temp;
+            result_ind++;
+            col = temp_col;
+            temp = zero<ValueType>();
+        }
+        temp += source_val[idx];
+    }
+    // If start != end, need to process the final column
+    if (start != end) {
+        result_col[result_ind] = col;
+        result_val[result_ind] = temp;
+    }
+}
+
+
+}  // namespace kernel
\ No newline at end of file
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index 9153f120d93..abe32153f4e 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -45,10 +45,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/multigrid/amgx_pgm.hpp>
 
 
+#include "core/components/fill_array.hpp"
+#include "core/components/prefix_sum.hpp"
 #include "cuda/base/cusparse_bindings.hpp"
 #include "cuda/base/math.hpp"
 #include "cuda/base/types.hpp"
-#include "cuda/solver/common_trs_kernels.cuh"
+#include "cuda/components/atomic.cuh"
+#include "cuda/components/reduction.cuh"
+#include "cuda/components/thread_ids.cuh"
 
 
 namespace gko {
@@ -62,25 +66,58 @@ namespace cuda {
 namespace amgx_pgm {
 
 
+constexpr int default_block_size = 512;
+
+
+#include "common/multigrid/amgx_pgm_kernels.hpp.inc"
+
+
 template <typename IndexType>
 void match_edge(std::shared_ptr<const CudaExecutor> exec,
                 const Array<IndexType> &strongest_neighbor,
-                Array<IndexType> &agg) GKO_NOT_IMPLEMENTED;
+                Array<IndexType> &agg)
+{
+    const auto num = agg.get_num_elems();
+    const dim3 grid(ceildiv(num, default_block_size));
+    kernel::match_edge_kernel<<<grid, default_block_size>>>(
+        num, strongest_neighbor.get_const_data(), agg.get_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL);
 
 
 template <typename IndexType>
 void count_unagg(std::shared_ptr<const CudaExecutor> exec,
-                 const Array<IndexType> &agg,
-                 IndexType *num_unagg) GKO_NOT_IMPLEMENTED;
+                 const Array<IndexType> &agg, IndexType *num_unagg)
+{
+    Array<IndexType> active_agg(exec, agg.get_num_elems());
+    const dim3 grid(ceildiv(active_agg.get_num_elems(), default_block_size));
+    kernel::activate_kernel<<<grid, default_block_size>>>(
+        active_agg.get_num_elems(), agg.get_const_data(),
+        active_agg.get_data());
+    *num_unagg = reduce_add_array(exec, active_agg.get_num_elems(),
+                                  active_agg.get_const_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL);
 
 
 template <typename IndexType>
 void renumber(std::shared_ptr<const CudaExecutor> exec, Array<IndexType> &agg,
-              IndexType *num_agg) GKO_NOT_IMPLEMENTED;
+              IndexType *num_agg)
+{
+    const auto num = agg.get_num_elems();
+    Array<IndexType> agg_map(exec, num + 1);
+    components::fill_array(exec, agg_map.get_data(), agg_map.get_num_elems(),
+                           zero<IndexType>());
+    const dim3 grid(ceildiv(num, default_block_size));
+    kernel::fill_agg_kernel<<<grid, default_block_size>>>(
+        num, agg.get_const_data(), agg_map.get_data());
+    components::prefix_sum(exec, agg_map.get_data(), agg_map.get_num_elems());
+    kernel::renumber_kernel<<<grid, default_block_size>>>(
+        num, agg_map.get_const_data(), agg.get_data());
+    *num_agg = exec->copy_val_to_host(agg_map.get_const_data() + num);
+}
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
 
@@ -90,18 +127,40 @@ void find_strongest_neighbor(
     std::shared_ptr<const CudaExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *weight_mtx,
     const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
-    Array<IndexType> &strongest_neighbor) GKO_NOT_IMPLEMENTED;
+    Array<IndexType> &strongest_neighbor)
+{
+    const auto num = agg.get_num_elems();
+    const dim3 grid(ceildiv(num, default_block_size));
+    kernel::find_strongest_neighbor_kernel<<<grid, default_block_size>>>(
+        num, weight_mtx->get_const_row_ptrs(), weight_mtx->get_const_col_idxs(),
+        weight_mtx->get_const_values(), diag->get_const_values(),
+        diag->get_stride(), agg.get_data(), strongest_neighbor.get_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
 
-
 template <typename ValueType, typename IndexType>
-void assign_to_exist_agg(
-    std::shared_ptr<const CudaExecutor> exec,
-    const matrix::Csr<ValueType, IndexType> *weight_mtx,
-    const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
-    Array<IndexType> &intermediate_agg) GKO_NOT_IMPLEMENTED;
+void assign_to_exist_agg(std::shared_ptr<const CudaExecutor> exec,
+                         const matrix::Csr<ValueType, IndexType> *weight_mtx,
+                         const matrix::Diagonal<ValueType> *diag,
+                         Array<IndexType> &agg,
+                         Array<IndexType> &intermediate_agg)
+{
+    auto agg_val = (intermediate_agg.get_num_elems() > 0)
+                       ? intermediate_agg.get_data()
+                       : agg.get_data();
+    const auto num = agg.get_num_elems();
+    const dim3 grid(ceildiv(num, default_block_size));
+    kernel::assign_to_exist_agg_kernel<<<grid, default_block_size>>>(
+        num, weight_mtx->get_const_row_ptrs(), weight_mtx->get_const_col_idxs(),
+        weight_mtx->get_const_values(), diag->get_const_values(),
+        diag->get_stride(), agg.get_const_data(), agg_val);
+    if (intermediate_agg.get_num_elems() > 0) {
+        // Copy the intermediate_agg to agg
+        agg = intermediate_agg;
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
diff --git a/cuda/test/CMakeLists.txt b/cuda/test/CMakeLists.txt
index 83d7b39f35e..fb66aaf270f 100644
--- a/cuda/test/CMakeLists.txt
+++ b/cuda/test/CMakeLists.txt
@@ -4,6 +4,7 @@ add_subdirectory(base)
 add_subdirectory(components)
 add_subdirectory(factorization)
 add_subdirectory(matrix)
+add_subdirectory(multigrid)
 add_subdirectory(preconditioner)
 add_subdirectory(reorder)
 add_subdirectory(solver)
diff --git a/cuda/test/multigrid/CMakeLists.txt b/cuda/test/multigrid/CMakeLists.txt
new file mode 100644
index 00000000000..8fe8bbeba48
--- /dev/null
+++ b/cuda/test/multigrid/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_test(amgx_pgm_kernels)
diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
new file mode 100644
index 00000000000..d1c899d0f54
--- /dev/null
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -0,0 +1,341 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+
+
+#include <fstream>
+#include <iostream>
+#include <random>
+#include <string>
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
+
+
+#include "core/multigrid/amgx_pgm_kernels.hpp"
+#include "core/test/utils/matrix_generator.hpp"
+#include "cuda/test/utils.hpp"
+
+
+namespace {
+
+
+template <typename Array, typename ValueDistribution, typename Engine>
+Array generate_random_array(gko::size_type num, ValueDistribution &&value_dist,
+                            Engine &&engine,
+                            std::shared_ptr<const gko::Executor> exec)
+{
+    using value_type = typename Array::value_type;
+    Array array_host(exec->get_master(), num);
+    auto val = array_host.get_data();
+    for (int i = 0; i < num; i++) {
+        val[i] =
+            gko::test::detail::get_rand_value<value_type>(value_dist, engine);
+    }
+    Array array(exec);
+    array = array_host;
+    return array;
+}
+
+
+class AmgxPgm : public ::testing::Test {
+protected:
+    using value_type = gko::default_precision;
+    using index_type = gko::int32;
+    using Mtx = gko::matrix::Dense<>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    AmgxPgm() : rand_engine(30) {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        cuda = gko::CudaExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (cuda != nullptr) {
+            ASSERT_NO_THROW(cuda->synchronize());
+        }
+    }
+
+    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<Mtx>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(num_cols, num_cols),
+            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    gko::Array<index_type> gen_array(gko::size_type num, index_type min_val,
+                                     index_type max_val)
+    {
+        return generate_random_array<gko::Array<index_type>>(
+            num, std::uniform_int_distribution<>(min_val, max_val), rand_engine,
+            ref);
+    }
+
+    void initialize_data()
+    {
+        int m = 597;
+        int n = 300;
+        int nrhs = 3;
+
+        agg = gen_array(m, 0, n - 1);
+        unfinished_agg = gen_array(m, -1, n - 1);
+        strongest_neighbor = gen_array(m, 0, n - 1);
+        coarse_vector = gen_mtx(n, nrhs);
+        fine_vector = gen_mtx(m, nrhs);
+        auto weight = gen_mtx(m, m);
+        make_weight(weight.get());
+        weight_csr = Csr::create(ref);
+        weight->convert_to(weight_csr.get());
+        weight_diag = weight_csr->extract_diagonal();
+
+        d_agg.set_executor(cuda);
+        d_unfinished_agg.set_executor(cuda);
+        d_strongest_neighbor.set_executor(cuda);
+        d_coarse_vector = Mtx::create(cuda);
+        d_fine_vector = Mtx::create(cuda);
+        d_weight_csr = Csr::create(cuda);
+        d_weight_diag = Mtx::create(cuda);
+        d_agg = agg;
+        d_unfinished_agg = unfinished_agg;
+        d_strongest_neighbor = strongest_neighbor;
+        d_coarse_vector->copy_from(coarse_vector.get());
+        d_fine_vector->copy_from(fine_vector.get());
+        d_weight_csr->copy_from(weight_csr.get());
+        d_weight_diag->copy_from(weight_diag.get());
+    }
+
+    void make_symetric(Mtx *mtx)
+    {
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
+                mtx->at(i, j) = mtx->at(j, i);
+            }
+        }
+    }
+
+    // only for real value
+    void make_absoulte(Mtx *mtx)
+    {
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            for (int j = 0; j < mtx->get_size()[1]; ++j) {
+                mtx->at(i, j) = abs(mtx->at(i, j));
+            }
+        }
+    }
+
+    void make_diag_dominant(Mtx *mtx)
+    {
+        using std::abs;
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            auto sum = gko::zero<Mtx::value_type>();
+            for (int j = 0; j < mtx->get_size()[1]; ++j) {
+                sum += abs(mtx->at(i, j));
+            }
+            mtx->at(i, i) = sum;
+        }
+    }
+
+    void make_spd(Mtx *mtx)
+    {
+        make_symetric(mtx);
+        make_diag_dominant(mtx);
+    }
+
+    void make_weight(Mtx *mtx)
+    {
+        make_symetric(mtx);
+        make_absoulte(mtx);
+        make_diag_dominant(mtx);
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::CudaExecutor> cuda;
+
+    std::ranlux48 rand_engine;
+
+    gko::Array<index_type> agg;
+    gko::Array<index_type> unfinished_agg;
+    gko::Array<index_type> strongest_neighbor;
+
+    gko::Array<index_type> d_agg;
+    gko::Array<index_type> d_unfinished_agg;
+    gko::Array<index_type> d_strongest_neighbor;
+
+    std::unique_ptr<Mtx> coarse_vector;
+    std::unique_ptr<Mtx> fine_vector;
+    std::unique_ptr<Mtx> weight_diag;
+    std::unique_ptr<Csr> weight_csr;
+
+    std::unique_ptr<Mtx> d_coarse_vector;
+    std::unique_ptr<Mtx> d_fine_vector;
+    std::unique_ptr<Mtx> d_weight_diag;
+    std::unique_ptr<Csr> d_weight_csr;
+};
+
+
+TEST_F(AmgxPgm, RestrictApplyIsEquivalentToRef)
+{
+    initialize_data();
+    // fine->coarse
+    auto x = Mtx::create_with_config_of(gko::lend(coarse_vector));
+    auto d_x = Mtx::create_with_config_of(gko::lend(d_coarse_vector));
+
+    gko::kernels::reference::amgx_pgm::restrict_apply(
+        ref, agg, fine_vector.get(), x.get());
+    gko::kernels::cuda::amgx_pgm::restrict_apply(
+        cuda, d_agg, d_fine_vector.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+}
+
+
+TEST_F(AmgxPgm, ProlongApplyaddIsEquivalentToRef)
+{
+    initialize_data();
+    // coarse->fine
+    auto x = fine_vector->clone();
+    auto d_x = d_fine_vector->clone();
+
+    gko::kernels::reference::amgx_pgm::prolong_applyadd(
+        ref, agg, coarse_vector.get(), x.get());
+    gko::kernels::cuda::amgx_pgm::prolong_applyadd(
+        cuda, d_agg, d_coarse_vector.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+}
+
+
+TEST_F(AmgxPgm, MatchEdgeIsEquivalentToRef)
+{
+    initialize_data();
+    auto x = unfinished_agg;
+    auto d_x = d_unfinished_agg;
+
+    gko::kernels::reference::amgx_pgm::match_edge(ref, strongest_neighbor, x);
+    gko::kernels::cuda::amgx_pgm::match_edge(cuda, d_strongest_neighbor, d_x);
+
+    GKO_ASSERT_ARRAY_EQ(d_x, x);
+}
+
+
+TEST_F(AmgxPgm, CountUnaggIsEquivalentToRef)
+{
+    initialize_data();
+    gko::size_type num_unagg;
+    gko::size_type d_num_unagg;
+
+    gko::kernels::reference::amgx_pgm::count_unagg(ref, agg, &num_unagg);
+    gko::kernels::cuda::amgx_pgm::count_unagg(cuda, d_agg, &d_num_unagg);
+
+    ASSERT_EQ(d_num_unagg, num_unagg);
+}
+
+
+TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
+{
+    initialize_data();
+    auto x = unfinished_agg;
+    auto d_x = d_unfinished_agg;
+    gko::size_type num_agg;
+    gko::size_type d_num_agg;
+
+    gko::kernels::reference::amgx_pgm::renumber(ref, agg, &num_agg);
+    gko::kernels::cuda::amgx_pgm::renumber(cuda, d_agg, &d_num_agg);
+
+    ASSERT_EQ(d_num_agg, num_agg);
+    GKO_ASSERT_ARRAY_EQ(d_agg, agg);
+    ASSERT_LE(num_agg, 300);
+}
+
+
+TEST_F(AmgxPgm, FindStrongestNeighborIsEquivalentToRef)
+{
+    initialize_data();
+    auto snb = strongest_neighbor;
+    auto d_snb = d_strongest_neighbor;
+
+    gko::kernels::reference::amgx_pgm::find_strongest_neighbor(
+        ref, weight_csr.get(), weight_diag.get(), agg, snb);
+    gko::kernels::cuda::amgx_pgm::find_strongest_neighbor(
+        cuda, d_weight_csr.get(), d_weight_diag.get(), d_agg, d_snb);
+
+    GKO_ASSERT_ARRAY_EQ(d_snb, snb);
+}
+
+
+TEST_F(AmgxPgm, AssignToExistAggIsEquivalentToRef)
+{
+    initialize_data();
+    auto x = unfinished_agg;
+    auto d_x = d_unfinished_agg;
+    auto intermediate_agg = x;
+    auto d_intermediate_agg = d_x;
+
+    gko::kernels::reference::amgx_pgm::assign_to_exist_agg(
+        ref, weight_csr.get(), weight_diag.get(), x, intermediate_agg);
+    gko::kernels::cuda::amgx_pgm::assign_to_exist_agg(
+        cuda, d_weight_csr.get(), d_weight_diag.get(), d_x, d_intermediate_agg);
+
+    GKO_ASSERT_ARRAY_EQ(d_x, x);
+}
+
+
+TEST_F(AmgxPgm, AssignToExistAggUnderteminsticIsEquivalentToRef)
+{
+    initialize_data();
+    auto d_x = d_unfinished_agg;
+    auto d_intermediate_agg = gko::Array<index_type>(cuda, 0);
+    gko::size_type d_num_unagg;
+
+    gko::kernels::cuda::amgx_pgm::assign_to_exist_agg(
+        cuda, d_weight_csr.get(), d_weight_diag.get(), d_x, d_intermediate_agg);
+    gko::kernels::cuda::amgx_pgm::count_unagg(cuda, d_agg, &d_num_unagg);
+
+    // only test whether all elements are aggregated.
+    GKO_ASSERT_EQ(d_num_unagg, 0);
+}
+
+
+}  // namespace

From a8920720716ed97812d244c0203533b7f72e9e93 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 16 Jul 2020 15:10:16 +0200
Subject: [PATCH 0660/1008] cuda amgx_pgm size2 generation and related test

---
 common/multigrid/amgx_pgm_kernels.hpp.inc     | 108 ++++++++++++++----
 core/multigrid/amgx_pgm.cpp                   |   5 +-
 core/multigrid/amgx_pgm_kernels.hpp           |   3 +-
 cuda/multigrid/amgx_pgm_kernels.cu            |  53 ++++++++-
 cuda/test/multigrid/amgx_pgm_kernels.cpp      |  25 +++-
 hip/multigrid/amgx_pgm_kernels.hip.cpp        |   3 +-
 omp/multigrid/amgx_pgm_kernels.cpp            |   3 +-
 reference/multigrid/amgx_pgm_kernels.cpp      |   3 +-
 reference/test/multigrid/amgx_pgm_kernels.cpp |   3 +-
 9 files changed, 175 insertions(+), 31 deletions(-)

diff --git a/common/multigrid/amgx_pgm_kernels.hpp.inc b/common/multigrid/amgx_pgm_kernels.hpp.inc
index 6f460ad98c2..5bed8696b10 100644
--- a/common/multigrid/amgx_pgm_kernels.hpp.inc
+++ b/common/multigrid/amgx_pgm_kernels.hpp.inc
@@ -241,37 +241,103 @@ __global__
 }
 
 
+template <typename IndexType>
+__global__ __launch_bounds__(default_block_size) void get_source_row_map_kernel(
+    const size_type source_nrows, const IndexType *__restrict__ agg_val,
+    const IndexType *__restrict__ source_row_ptrs,
+    IndexType *__restrict__ result_row_ptrs, IndexType *__restrict__ row_map)
+{
+    auto row = thread::get_thread_id_flat();
+    if (row >= source_nrows) {
+        return;
+    }
+    const auto num_elems = source_row_ptrs[row + 1] - source_row_ptrs[row];
+    const auto result_idx = agg_val[row];
+    // atomic_add returns the old value, so it can be the starting point.
+    row_map[row] = atomic_add(result_row_ptrs + result_idx, num_elems);
+}
+
+template <typename ValueType, typename IndexType>
+__global__ __launch_bounds__(default_block_size) void move_row_kernel(
+    const size_type source_nrows, const IndexType *__restrict__ agg_val,
+    const IndexType *__restrict__ row_map,
+    const IndexType *__restrict__ source_row_ptrs,
+    const IndexType *__restrict__ source_col_idxs,
+    const ValueType *__restrict__ source_values,
+    const IndexType *__restrict__ result_row_ptrs,
+    IndexType *__restrict__ result_col_idxs,
+    ValueType *__restrict__ result_values)
+{
+    auto row = thread::get_thread_id_flat();
+    if (row >= source_nrows) {
+        return;
+    }
+    auto result_i = result_row_ptrs[agg_val[row]] + row_map[row];
+    for (auto i = source_row_ptrs[row]; i < source_row_ptrs[row + 1];
+         i++, result_i++) {
+        result_col_idxs[result_i] = agg_val[source_col_idxs[i]];
+        result_values[result_i] = source_values[i];
+    }
+}
+
+
 template <typename ValueType, typename IndexType>
 __global__ __launch_bounds__(default_block_size) void merge_col_kernel(
-    const size_type num, const IndexType *__restrict__ row_ptrs,
-    const IndexType *__restrict__ source_col,
-    const ValueType *__restrict__ source_val,
-    IndexType *__restrict__ result_col, ValueType *__restrict__ result_val)
+    const size_type nrows, const IndexType *__restrict__ temp_row_ptrs,
+    IndexType *__restrict__ temp_col_idxs, ValueType *__restrict__ temp_values,
+    IndexType *__restrict__ coarse_row_ptrs)
 {
     auto row = thread::get_thread_id_flat();
-    if (row >= num) {
+    if (row >= nrows) {
         return;
     }
-    const auto start = row_ptrs[row];
-    const auto end = row_ptrs[row + 1];
-    auto result_ind = start;
-    auto col = source_col[result_ind];
-    auto temp = source_val[result_ind];
-    for (size_type idx = start + 1; idx < end; idx++) {
-        auto temp_col = source_col[idx];
-        if (temp_col != col) {
-            result_col[result_ind] = col;
-            result_val[result_ind] = temp;
-            result_ind++;
-            col = temp_col;
-            temp = zero<ValueType>();
+
+    IndexType num_elems = zero<IndexType>();
+    const auto start = temp_row_ptrs[row];
+    const auto end = temp_row_ptrs[row + 1];
+    IndexType col = temp_col_idxs[start];
+    ValueType value = temp_values[start];
+    for (auto i = start + 1; i < end; i++) {
+        const auto current_col = temp_col_idxs[i];
+        if (current_col != col) {
+            // apply to the original data. It is sorted, so the writing position
+            // is before read position
+            temp_col_idxs[start + num_elems] = col;
+            temp_values[start + num_elems] = value;
+            value = zero<ValueType>();
+            col = current_col;
+            num_elems++;
         }
-        temp += source_val[idx];
+        value += temp_values[i];
     }
     // If start != end, need to process the final column
     if (start != end) {
-        result_col[result_ind] = col;
-        result_val[result_ind] = temp;
+        temp_col_idxs[start + num_elems] = col;
+        temp_values[start + num_elems] = value;
+        num_elems++;
+    }
+    coarse_row_ptrs[row] = num_elems;
+}
+
+
+template <typename ValueType, typename IndexType>
+__global__ __launch_bounds__(default_block_size) void copy_to_coarse_kernel(
+    const size_type nrows, const IndexType *__restrict__ temp_row_ptrs,
+    const IndexType *__restrict__ temp_col_idxs,
+    const ValueType *__restrict__ temp_values,
+    const IndexType *__restrict__ coarse_row_ptrs,
+    IndexType *__restrict__ coarse_col_idxs,
+    ValueType *__restrict__ coarse_values)
+{
+    auto row = thread::get_thread_id_flat();
+    if (row >= nrows) {
+        return;
+    }
+    auto temp_i = temp_row_ptrs[row];
+    for (auto i = coarse_row_ptrs[row]; i < coarse_row_ptrs[row + 1];
+         i++, temp_i++) {
+        coarse_col_idxs[i] = temp_col_idxs[temp_i];
+        coarse_values[i] = temp_values[temp_i];
     }
 }
 
diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index 09edd76d101..1803c4252d2 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -79,7 +79,10 @@ std::unique_ptr<LinOp> amgx_pgm_generate(
 {
     auto coarse = matrix::Csr<ValueType, IndexType>::create(
         exec, dim<2>{num_agg, num_agg}, 0, source->get_strategy());
-    exec->run(amgx_pgm::make_amgx_pgm_generate(source, agg, coarse.get()));
+    auto temp = matrix::Csr<ValueType, IndexType>::create(
+        exec, dim<2>{num_agg, num_agg}, source->get_num_stored_elements());
+    exec->run(amgx_pgm::make_amgx_pgm_generate(source, agg, coarse.get(),
+                                               temp.get()));
     return std::move(coarse);
 }
 
diff --git a/core/multigrid/amgx_pgm_kernels.hpp b/core/multigrid/amgx_pgm_kernels.hpp
index 9109c59e610..e0006be63a7 100644
--- a/core/multigrid/amgx_pgm_kernels.hpp
+++ b/core/multigrid/amgx_pgm_kernels.hpp
@@ -82,7 +82,8 @@ namespace amgx_pgm {
     void amgx_pgm_generate(std::shared_ptr<const DefaultExecutor> exec,     \
                            const matrix::Csr<ValueType, IndexType> *source, \
                            const Array<IndexType> &agg,                     \
-                           matrix::Csr<ValueType, IndexType> *coarse)
+                           matrix::Csr<ValueType, IndexType> *coarse,       \
+                           matrix::Csr<ValueType, IndexType> *temp)
 
 #define GKO_DECLARE_ALL_AS_TEMPLATES                                    \
     template <typename IndexType>                                       \
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index abe32153f4e..d84766557cb 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -47,6 +47,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/components/fill_array.hpp"
 #include "core/components/prefix_sum.hpp"
+#include "core/matrix/csr_builder.hpp"
+#include "core/matrix/csr_kernels.hpp"
 #include "cuda/base/cusparse_bindings.hpp"
 #include "cuda/base/math.hpp"
 #include "cuda/base/types.hpp"
@@ -170,8 +172,55 @@ template <typename ValueType, typename IndexType>
 void amgx_pgm_generate(std::shared_ptr<const CudaExecutor> exec,
                        const matrix::Csr<ValueType, IndexType> *source,
                        const Array<IndexType> &agg,
-                       matrix::Csr<ValueType, IndexType> *coarse)
-    GKO_NOT_IMPLEMENTED;
+                       matrix::Csr<ValueType, IndexType> *coarse,
+                       matrix::Csr<ValueType, IndexType> *temp)
+{
+    const auto source_nrows = source->get_size()[0];
+    const auto source_nnz = source->get_num_stored_elements();
+    const auto coarse_nrows = coarse->get_size()[0];
+    Array<IndexType> row_map(exec, source_nrows);
+    // fill coarse row pointer as zero
+    components::fill_array(exec, temp->get_row_ptrs(), coarse_nrows + 1,
+                           zero<IndexType>());
+    // compute each source row should be moved and also change column index
+    dim3 grid(ceildiv(source_nrows, default_block_size));
+    // agg source_row (for row size) coarse row source map
+    kernel::get_source_row_map_kernel<<<grid, default_block_size>>>(
+        source_nrows, agg.get_const_data(), source->get_const_row_ptrs(),
+        temp->get_row_ptrs(), row_map.get_data());
+    // prefix sum of temp_row_ptrs
+    components::prefix_sum(exec, temp->get_row_ptrs(), coarse_nrows + 1);
+    // copy source -> to coarse and change column index
+    kernel::move_row_kernel<<<grid, default_block_size>>>(
+        source_nrows, agg.get_const_data(), row_map.get_const_data(),
+        source->get_const_row_ptrs(), source->get_const_col_idxs(),
+        as_cuda_type(source->get_const_values()), temp->get_const_row_ptrs(),
+        temp->get_col_idxs(), as_cuda_type(temp->get_values()));
+    // sort csr
+    csr::sort_by_column_index(exec, temp);
+    // summation of the elements with same position
+    grid = ceildiv(coarse_nrows, default_block_size);
+    kernel::merge_col_kernel<<<grid, default_block_size>>>(
+        coarse_nrows, temp->get_const_row_ptrs(), temp->get_col_idxs(),
+        as_cuda_type(temp->get_values()), coarse->get_row_ptrs());
+    // build the coarse matrix
+    components::prefix_sum(exec, coarse->get_row_ptrs(), coarse_nrows + 1);
+    // prefix sum of coarse->get_row_ptrs
+    const auto coarse_nnz =
+        exec->copy_val_to_host(coarse->get_row_ptrs() + coarse_nrows);
+    // reallocate size of column and values
+    matrix::CsrBuilder<ValueType, IndexType> coarse_builder{coarse};
+    auto &coarse_col_idxs_array = coarse_builder.get_col_idx_array();
+    auto &coarse_vals_array = coarse_builder.get_value_array();
+    coarse_col_idxs_array.resize_and_reset(coarse_nnz);
+    coarse_vals_array.resize_and_reset(coarse_nnz);
+    // copy the result
+    kernel::copy_to_coarse_kernel<<<grid, default_block_size>>>(
+        coarse_nrows, temp->get_const_row_ptrs(), temp->get_const_col_idxs(),
+        as_cuda_type(temp->get_const_values()), coarse->get_const_row_ptrs(),
+        coarse_col_idxs_array.get_data(),
+        as_cuda_type(coarse_vals_array.get_data()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
 
diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
index d1c899d0f54..746d9cf7c40 100644
--- a/cuda/test/multigrid/amgx_pgm_kernels.cpp
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -34,10 +34,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <fstream>
-#include <iostream>
 #include <random>
 #include <string>
 
+
 #include <gtest/gtest.h>
 
 
@@ -116,7 +116,7 @@ class AmgxPgm : public ::testing::Test {
     void initialize_data()
     {
         int m = 597;
-        int n = 300;
+        n = 300;
         int nrhs = 3;
 
         agg = gen_array(m, 0, n - 1);
@@ -212,6 +212,8 @@ class AmgxPgm : public ::testing::Test {
     std::unique_ptr<Mtx> d_fine_vector;
     std::unique_ptr<Mtx> d_weight_diag;
     std::unique_ptr<Csr> d_weight_csr;
+
+    int n;
 };
 
 
@@ -338,4 +340,23 @@ TEST_F(AmgxPgm, AssignToExistAggUnderteminsticIsEquivalentToRef)
 }
 
 
+TEST_F(AmgxPgm, GenerateMtxIsEquivalentToRef)
+{
+    initialize_data();
+    auto csr_coarse = Csr::create(ref, gko::dim<2>{n, n}, 0);
+    auto d_csr_coarse = Csr::create(cuda, gko::dim<2>{n, n}, 0);
+    auto csr_temp = Csr::create(ref, gko::dim<2>{n, n},
+                                weight_csr->get_num_stored_elements());
+    auto d_csr_temp = Csr::create(cuda, gko::dim<2>{n, n},
+                                  d_weight_csr->get_num_stored_elements());
+
+    gko::kernels::cuda::amgx_pgm::amgx_pgm_generate(
+        cuda, d_weight_csr.get(), d_agg, d_csr_coarse.get(), d_csr_temp.get());
+    gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
+        ref, weight_csr.get(), agg, csr_coarse.get(), csr_temp.get());
+
+    GKO_ASSERT_MTX_NEAR(d_csr_coarse, csr_coarse, 1e-14);
+}
+
+
 }  // namespace
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index d401bb72b72..88c2db995f1 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -111,7 +111,8 @@ template <typename ValueType, typename IndexType>
 void amgx_pgm_generate(std::shared_ptr<const HipExecutor> exec,
                        const matrix::Csr<ValueType, IndexType> *source,
                        const Array<IndexType> &agg,
-                       matrix::Csr<ValueType, IndexType> *coarse)
+                       matrix::Csr<ValueType, IndexType> *coarse,
+                       matrix::Csr<ValueType, IndexType> *temp)
     GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index d56b3ff9dab..a0db864a250 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -108,7 +108,8 @@ template <typename ValueType, typename IndexType>
 void amgx_pgm_generate(std::shared_ptr<const OmpExecutor> exec,
                        const matrix::Csr<ValueType, IndexType> *source,
                        const Array<IndexType> &agg,
-                       matrix::Csr<ValueType, IndexType> *coarse)
+                       matrix::Csr<ValueType, IndexType> *coarse,
+                       matrix::Csr<ValueType, IndexType> *temp)
     GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index 1ae7ee860b0..64788db9805 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -233,7 +233,8 @@ template <typename ValueType, typename IndexType>
 void amgx_pgm_generate(std::shared_ptr<const ReferenceExecutor> exec,
                        const matrix::Csr<ValueType, IndexType> *source,
                        const Array<IndexType> &agg,
-                       matrix::Csr<ValueType, IndexType> *coarse)
+                       matrix::Csr<ValueType, IndexType> *coarse,
+                       matrix::Csr<ValueType, IndexType> *temp)
 {
     // agg[i] -> I, agg[j] -> J
     const auto coarse_nrows = coarse->get_size()[0];
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index 63f8baf2e62..847e859529e 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -525,9 +525,10 @@ TYPED_TEST(AmgxPgm, GenerateMtx)
                        {2, 1, -2},
                        {2, 2, 5}}});
     auto csr_coarse = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
+    auto empty = gko::matrix::Csr<value_type, index_type>::create(this->exec);
 
     gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
-        this->exec, this->mtx.get(), agg, csr_coarse.get());
+        this->exec, this->mtx.get(), agg, csr_coarse.get(), empty.get());
 
     GKO_ASSERT_MTX_NEAR(csr_coarse, coarse_ans, r<value_type>::value);
 }

From 0459cac43e903370844f4865d1711a9ff60d0156 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 17 Jul 2020 12:22:26 +0200
Subject: [PATCH 0661/1008] fix windows issue

---
 cuda/test/multigrid/amgx_pgm_kernels.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
index 746d9cf7c40..6c2c34885a1 100644
--- a/cuda/test/multigrid/amgx_pgm_kernels.cpp
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -213,7 +213,7 @@ class AmgxPgm : public ::testing::Test {
     std::unique_ptr<Mtx> d_weight_diag;
     std::unique_ptr<Csr> d_weight_csr;
 
-    int n;
+    gko::size_type n;
 };
 
 

From 1a23119b4c7bdb4378cf8d6b8448ba1c47c58f0c Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 21 Jul 2020 17:41:34 +0200
Subject: [PATCH 0662/1008] omp implementation

---
 omp/multigrid/amgx_pgm_kernels.cpp      | 194 ++++++++++++-
 omp/test/CMakeLists.txt                 |   1 +
 omp/test/multigrid/CMakeLists.txt       |   1 +
 omp/test/multigrid/amgx_pgm_kernels.cpp | 353 ++++++++++++++++++++++++
 4 files changed, 538 insertions(+), 11 deletions(-)
 create mode 100644 omp/test/multigrid/CMakeLists.txt
 create mode 100644 omp/test/multigrid/amgx_pgm_kernels.cpp

diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index a0db864a250..80f1b2f88ce 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -48,6 +48,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/multigrid/amgx_pgm.hpp>
 
 
+#include "core/base/allocator.hpp"
+#include "core/components/fill_array.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/csr_builder.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace omp {
@@ -62,22 +68,61 @@ namespace amgx_pgm {
 template <typename IndexType>
 void match_edge(std::shared_ptr<const OmpExecutor> exec,
                 const Array<IndexType> &strongest_neighbor,
-                Array<IndexType> &agg) GKO_NOT_IMPLEMENTED;
+                Array<IndexType> &agg)
+{
+    auto agg_vals = agg.get_data();
+    auto strongest_neighbor_vals = strongest_neighbor.get_const_data();
+#pragma omp parallel for
+    for (size_type i = 0; i < agg.get_num_elems(); i++) {
+        if (agg_vals[i] == -1) {
+            auto neighbor = strongest_neighbor_vals[i];
+            if (neighbor != -1 && strongest_neighbor_vals[neighbor] == i) {
+                agg_vals[i] = i;
+                agg_vals[neighbor] = i;
+                // Use the smaller index as agg point
+            }
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL);
 
 
 template <typename IndexType>
 void count_unagg(std::shared_ptr<const OmpExecutor> exec,
-                 const Array<IndexType> &agg,
-                 IndexType *num_unagg) GKO_NOT_IMPLEMENTED;
+                 const Array<IndexType> &agg, IndexType *num_unagg)
+{
+    IndexType unagg = 0;
+#pragma omp parallel for reduction(+ : unagg)
+    for (size_type i = 0; i < agg.get_num_elems(); i++) {
+        unagg += (agg.get_const_data()[i] == -1);
+    }
+    *num_unagg = unagg;
+}
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL);
 
 
 template <typename IndexType>
 void renumber(std::shared_ptr<const OmpExecutor> exec, Array<IndexType> &agg,
-              IndexType *num_agg) GKO_NOT_IMPLEMENTED;
+              IndexType *num_agg)
+{
+    const auto num = agg.get_num_elems();
+    Array<IndexType> agg_map(exec, num + 1);
+    auto agg_vals = agg.get_data();
+    auto agg_map_vals = agg_map.get_data();
+    components::fill_array(exec, agg_map_vals, num + 1, zero<IndexType>());
+#pragma omp parallel for
+    for (size_type i = 0; i < num; i++) {
+        agg_map_vals[agg_vals[i]] = 1;
+    }
+    components::prefix_sum(exec, agg_map_vals, num + 1);
+#pragma omp parallel for
+    for (size_type i = 0; i < num; i++) {
+        agg_vals[i] = agg_map_vals[agg_vals[i]];
+    }
+    *num_agg = agg_map_vals[num];
+}
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
 
@@ -87,18 +132,104 @@ void find_strongest_neighbor(
     std::shared_ptr<const OmpExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *weight_mtx,
     const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
-    Array<IndexType> &strongest_neighbor) GKO_NOT_IMPLEMENTED;
+    Array<IndexType> &strongest_neighbor)
+{
+    const auto row_ptrs = weight_mtx->get_const_row_ptrs();
+    const auto col_idxs = weight_mtx->get_const_col_idxs();
+    const auto vals = weight_mtx->get_const_values();
+#pragma omp parallel for
+    for (size_type row = 0; row < agg.get_num_elems(); row++) {
+        auto max_weight_unagg = zero<ValueType>();
+        auto max_weight_agg = zero<ValueType>();
+        IndexType strongest_unagg = -1;
+        IndexType strongest_agg = -1;
+        if (agg.get_const_data()[row] == -1) {
+            for (auto idx = row_ptrs[row]; idx < row_ptrs[row + 1]; idx++) {
+                auto col = col_idxs[idx];
+                if (col == row) {
+                    continue;
+                }
+                auto weight = vals[idx] /
+                              max(abs(diag->at(row, 0)), abs(diag->at(col, 0)));
+                if (agg.get_const_data()[col] == -1 &&
+                    (weight > max_weight_unagg ||
+                     (weight == max_weight_unagg && col > strongest_unagg))) {
+                    max_weight_unagg = weight;
+                    strongest_unagg = col;
+                } else if (agg.get_const_data()[col] != -1 &&
+                           (weight > max_weight_agg ||
+                            (weight == max_weight_agg &&
+                             col > strongest_agg))) {
+                    max_weight_agg = weight;
+                    strongest_agg = col;
+                }
+            }
+
+            if (strongest_unagg == -1 && strongest_agg != -1) {
+                // all neighbor is agg, connect to the strongest agg
+                agg.get_data()[row] = agg.get_data()[strongest_agg];
+            } else if (strongest_unagg != -1) {
+                // set the strongest neighbor in the unagg group
+                strongest_neighbor.get_data()[row] = strongest_unagg;
+            } else {
+                // no neighbor
+                strongest_neighbor.get_data()[row] = row;
+            }
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
 
 
 template <typename ValueType, typename IndexType>
-void assign_to_exist_agg(
-    std::shared_ptr<const OmpExecutor> exec,
-    const matrix::Csr<ValueType, IndexType> *weight_mtx,
-    const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
-    Array<IndexType> &intermediate_agg) GKO_NOT_IMPLEMENTED;
+void assign_to_exist_agg(std::shared_ptr<const OmpExecutor> exec,
+                         const matrix::Csr<ValueType, IndexType> *weight_mtx,
+                         const matrix::Diagonal<ValueType> *diag,
+                         Array<IndexType> &agg,
+                         Array<IndexType> &intermediate_agg)
+{
+    const auto row_ptrs = weight_mtx->get_const_row_ptrs();
+    const auto col_idxs = weight_mtx->get_const_col_idxs();
+    const auto vals = weight_mtx->get_const_values();
+    auto max_weight_agg = zero<ValueType>();
+    const auto agg_const_val = agg.get_const_data();
+    auto agg_val = (intermediate_agg.get_num_elems() > 0)
+                       ? intermediate_agg.get_data()
+                       : agg.get_data();
+#pragma omp parallel for
+    for (IndexType row = 0; row < agg.get_num_elems(); row++) {
+        if (agg_const_val[row] != -1) {
+            continue;
+        }
+        IndexType strongest_agg = -1;
+        for (auto idx = row_ptrs[row]; idx < row_ptrs[row + 1]; idx++) {
+            auto col = col_idxs[idx];
+            if (col == row) {
+                continue;
+            }
+            auto weight =
+                vals[idx] / max(abs(diag->at(row, 0)), abs(diag->at(col, 0)));
+            if (agg_const_val[col] != -1 &&
+                (weight > max_weight_agg ||
+                 (weight == max_weight_agg && col > strongest_agg))) {
+                max_weight_agg = weight;
+                strongest_agg = col;
+            }
+        }
+        if (strongest_agg != -1) {
+            agg_val[row] = agg_const_val[strongest_agg];
+        } else {
+            agg_val[row] = row;
+        }
+    }
+
+    if (intermediate_agg.get_num_elems() > 0) {
+        // Copy the intermediate_agg to agg
+        agg = intermediate_agg;
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
@@ -110,7 +241,48 @@ void amgx_pgm_generate(std::shared_ptr<const OmpExecutor> exec,
                        const Array<IndexType> &agg,
                        matrix::Csr<ValueType, IndexType> *coarse,
                        matrix::Csr<ValueType, IndexType> *temp)
-    GKO_NOT_IMPLEMENTED;
+{
+    // agg[i] -> I, agg[j] -> J
+    const auto coarse_nrows = coarse->get_size()[0];
+    const auto source_nrows = source->get_size()[0];
+    const auto source_row_ptrs = source->get_const_row_ptrs();
+    const auto source_col_idxs = source->get_const_col_idxs();
+    const auto source_vals = source->get_const_values();
+    vector<map<IndexType, ValueType>> row_list(
+        source_nrows, map<IndexType, ValueType>{exec}, exec);
+    for (size_type i = 0; i < source_nrows; i++) {
+        IndexType row_idx = agg.get_const_data()[i];
+        for (auto j = source_row_ptrs[i]; j < source_row_ptrs[i + 1]; j++) {
+            const auto col = agg.get_const_data()[source_col_idxs[j]];
+            const auto val = source_vals[j];
+            row_list[row_idx][col] += val;
+        }
+    }
+    auto coarse_row_ptrs = coarse->get_row_ptrs();
+#pragma omp parallel for
+    for (size_type i = 0; i < coarse_nrows; i++) {
+        coarse_row_ptrs[i] = row_list[i].size();
+    }
+    components::prefix_sum(exec, coarse_row_ptrs, coarse_nrows + 1);
+
+    auto nnz = coarse_row_ptrs[coarse_nrows];
+    matrix::CsrBuilder<ValueType, IndexType> coarse_builder{coarse};
+    auto &coarse_col_idxs_array = coarse_builder.get_col_idx_array();
+    auto &coarse_vals_array = coarse_builder.get_value_array();
+    coarse_col_idxs_array.resize_and_reset(nnz);
+    coarse_vals_array.resize_and_reset(nnz);
+    auto coarse_col_idxs = coarse_col_idxs_array.get_data();
+    auto coarse_vals = coarse_vals_array.get_data();
+#pragma omp parallel for
+    for (size_type i = 0; i < coarse_nrows; i++) {
+        auto ind = coarse_row_ptrs[i];
+        for (auto pair : row_list[i]) {
+            coarse_col_idxs[ind] = pair.first;
+            coarse_vals[ind] = pair.second;
+            ind++;
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
 
diff --git a/omp/test/CMakeLists.txt b/omp/test/CMakeLists.txt
index 2f506e51109..3759a39bc77 100644
--- a/omp/test/CMakeLists.txt
+++ b/omp/test/CMakeLists.txt
@@ -3,6 +3,7 @@ include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
 add_subdirectory(components)
 add_subdirectory(factorization)
 add_subdirectory(matrix)
+add_subdirectory(multigrid)
 add_subdirectory(preconditioner)
 add_subdirectory(reorder)
 add_subdirectory(solver)
diff --git a/omp/test/multigrid/CMakeLists.txt b/omp/test/multigrid/CMakeLists.txt
new file mode 100644
index 00000000000..8fe8bbeba48
--- /dev/null
+++ b/omp/test/multigrid/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_test(amgx_pgm_kernels)
diff --git a/omp/test/multigrid/amgx_pgm_kernels.cpp b/omp/test/multigrid/amgx_pgm_kernels.cpp
new file mode 100644
index 00000000000..cc822135a5d
--- /dev/null
+++ b/omp/test/multigrid/amgx_pgm_kernels.cpp
@@ -0,0 +1,353 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+
+
+#include <fstream>
+#include <random>
+#include <string>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
+
+
+#include "core/multigrid/amgx_pgm_kernels.hpp"
+#include "core/test/utils.hpp"
+#include "core/test/utils/matrix_generator.hpp"
+
+namespace {
+
+
+template <typename Array, typename ValueDistribution, typename Engine>
+Array generate_random_array(gko::size_type num, ValueDistribution &&value_dist,
+                            Engine &&engine,
+                            std::shared_ptr<const gko::Executor> exec)
+{
+    using value_type = typename Array::value_type;
+    Array array_host(exec->get_master(), num);
+    auto val = array_host.get_data();
+    for (int i = 0; i < num; i++) {
+        val[i] =
+            gko::test::detail::get_rand_value<value_type>(value_dist, engine);
+    }
+    Array array(exec);
+    array = array_host;
+    return array;
+}
+
+
+class AmgxPgm : public ::testing::Test {
+protected:
+    using value_type = gko::default_precision;
+    using index_type = gko::int32;
+    using Mtx = gko::matrix::Dense<>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    AmgxPgm() : rand_engine(30) {}
+
+    void SetUp()
+    {
+        ref = gko::ReferenceExecutor::create();
+        omp = gko::OmpExecutor::create();
+    }
+
+    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<Mtx>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(num_cols, num_cols),
+            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    gko::Array<index_type> gen_array(gko::size_type num, index_type min_val,
+                                     index_type max_val)
+    {
+        return generate_random_array<gko::Array<index_type>>(
+            num, std::uniform_int_distribution<>(min_val, max_val), rand_engine,
+            ref);
+    }
+
+    void initialize_data()
+    {
+        int m = 597;
+        n = 300;
+        int nrhs = 3;
+
+        agg = gen_array(m, 0, n - 1);
+        unfinished_agg = gen_array(m, -1, n - 1);
+        strongest_neighbor = gen_array(m, 0, n - 1);
+        coarse_vector = gen_mtx(n, nrhs);
+        fine_vector = gen_mtx(m, nrhs);
+        auto weight = gen_mtx(m, m);
+        make_weight(weight.get());
+        weight_csr = Csr::create(ref);
+        weight->convert_to(weight_csr.get());
+        weight_diag = weight_csr->extract_diagonal();
+
+        d_agg.set_executor(omp);
+        d_unfinished_agg.set_executor(omp);
+        d_strongest_neighbor.set_executor(omp);
+        d_coarse_vector = Mtx::create(omp);
+        d_fine_vector = Mtx::create(omp);
+        d_weight_csr = Csr::create(omp);
+        d_weight_diag = Mtx::create(omp);
+        d_agg = agg;
+        d_unfinished_agg = unfinished_agg;
+        d_strongest_neighbor = strongest_neighbor;
+        d_coarse_vector->copy_from(coarse_vector.get());
+        d_fine_vector->copy_from(fine_vector.get());
+        d_weight_csr->copy_from(weight_csr.get());
+        d_weight_diag->copy_from(weight_diag.get());
+    }
+
+    void make_symetric(Mtx *mtx)
+    {
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
+                mtx->at(i, j) = mtx->at(j, i);
+            }
+        }
+    }
+
+    // only for real value
+    void make_absoulte(Mtx *mtx)
+    {
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            for (int j = 0; j < mtx->get_size()[1]; ++j) {
+                mtx->at(i, j) = abs(mtx->at(i, j));
+            }
+        }
+    }
+
+    void make_diag_dominant(Mtx *mtx)
+    {
+        using std::abs;
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            auto sum = gko::zero<Mtx::value_type>();
+            for (int j = 0; j < mtx->get_size()[1]; ++j) {
+                sum += abs(mtx->at(i, j));
+            }
+            mtx->at(i, i) = sum;
+        }
+    }
+
+    void make_spd(Mtx *mtx)
+    {
+        make_symetric(mtx);
+        make_diag_dominant(mtx);
+    }
+
+    void make_weight(Mtx *mtx)
+    {
+        make_symetric(mtx);
+        make_absoulte(mtx);
+        make_diag_dominant(mtx);
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::OmpExecutor> omp;
+
+    std::ranlux48 rand_engine;
+
+    gko::Array<index_type> agg;
+    gko::Array<index_type> unfinished_agg;
+    gko::Array<index_type> strongest_neighbor;
+
+    gko::Array<index_type> d_agg;
+    gko::Array<index_type> d_unfinished_agg;
+    gko::Array<index_type> d_strongest_neighbor;
+
+    std::unique_ptr<Mtx> coarse_vector;
+    std::unique_ptr<Mtx> fine_vector;
+    std::unique_ptr<Mtx> weight_diag;
+    std::unique_ptr<Csr> weight_csr;
+
+    std::unique_ptr<Mtx> d_coarse_vector;
+    std::unique_ptr<Mtx> d_fine_vector;
+    std::unique_ptr<Mtx> d_weight_diag;
+    std::unique_ptr<Csr> d_weight_csr;
+
+    gko::size_type n;
+};
+
+
+TEST_F(AmgxPgm, RestrictApplyIsEquivalentToRef)
+{
+    initialize_data();
+    // fine->coarse
+    auto x = Mtx::create_with_config_of(gko::lend(coarse_vector));
+    auto d_x = Mtx::create_with_config_of(gko::lend(d_coarse_vector));
+
+    gko::kernels::reference::amgx_pgm::restrict_apply(
+        ref, agg, fine_vector.get(), x.get());
+    gko::kernels::omp::amgx_pgm::restrict_apply(omp, d_agg, d_fine_vector.get(),
+                                                d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+}
+
+
+TEST_F(AmgxPgm, ProlongApplyaddIsEquivalentToRef)
+{
+    initialize_data();
+    // coarse->fine
+    auto x = fine_vector->clone();
+    auto d_x = d_fine_vector->clone();
+
+    gko::kernels::reference::amgx_pgm::prolong_applyadd(
+        ref, agg, coarse_vector.get(), x.get());
+    gko::kernels::omp::amgx_pgm::prolong_applyadd(
+        omp, d_agg, d_coarse_vector.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+}
+
+
+TEST_F(AmgxPgm, MatchEdgeIsEquivalentToRef)
+{
+    initialize_data();
+    auto x = unfinished_agg;
+    auto d_x = d_unfinished_agg;
+
+    gko::kernels::reference::amgx_pgm::match_edge(ref, strongest_neighbor, x);
+    gko::kernels::omp::amgx_pgm::match_edge(omp, d_strongest_neighbor, d_x);
+
+    GKO_ASSERT_ARRAY_EQ(d_x, x);
+}
+
+
+TEST_F(AmgxPgm, CountUnaggIsEquivalentToRef)
+{
+    initialize_data();
+    gko::size_type num_unagg;
+    gko::size_type d_num_unagg;
+
+    gko::kernels::reference::amgx_pgm::count_unagg(ref, agg, &num_unagg);
+    gko::kernels::omp::amgx_pgm::count_unagg(omp, d_agg, &d_num_unagg);
+
+    ASSERT_EQ(d_num_unagg, num_unagg);
+}
+
+
+TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
+{
+    initialize_data();
+    auto x = unfinished_agg;
+    auto d_x = d_unfinished_agg;
+    gko::size_type num_agg;
+    gko::size_type d_num_agg;
+
+    gko::kernels::reference::amgx_pgm::renumber(ref, agg, &num_agg);
+    gko::kernels::omp::amgx_pgm::renumber(omp, d_agg, &d_num_agg);
+
+    ASSERT_EQ(d_num_agg, num_agg);
+    GKO_ASSERT_ARRAY_EQ(d_agg, agg);
+    ASSERT_LE(num_agg, 300);
+}
+
+
+TEST_F(AmgxPgm, FindStrongestNeighborIsEquivalentToRef)
+{
+    initialize_data();
+    auto snb = strongest_neighbor;
+    auto d_snb = d_strongest_neighbor;
+
+    gko::kernels::reference::amgx_pgm::find_strongest_neighbor(
+        ref, weight_csr.get(), weight_diag.get(), agg, snb);
+    gko::kernels::omp::amgx_pgm::find_strongest_neighbor(
+        omp, d_weight_csr.get(), d_weight_diag.get(), d_agg, d_snb);
+
+    GKO_ASSERT_ARRAY_EQ(d_snb, snb);
+}
+
+
+TEST_F(AmgxPgm, AssignToExistAggIsEquivalentToRef)
+{
+    initialize_data();
+    auto x = unfinished_agg;
+    auto d_x = d_unfinished_agg;
+    auto intermediate_agg = x;
+    auto d_intermediate_agg = d_x;
+
+    gko::kernels::reference::amgx_pgm::assign_to_exist_agg(
+        ref, weight_csr.get(), weight_diag.get(), x, intermediate_agg);
+    gko::kernels::omp::amgx_pgm::assign_to_exist_agg(
+        omp, d_weight_csr.get(), d_weight_diag.get(), d_x, d_intermediate_agg);
+
+    GKO_ASSERT_ARRAY_EQ(d_x, x);
+}
+
+
+TEST_F(AmgxPgm, AssignToExistAggUnderteminsticIsEquivalentToRef)
+{
+    initialize_data();
+    auto d_x = d_unfinished_agg;
+    auto d_intermediate_agg = gko::Array<index_type>(omp, 0);
+    gko::size_type d_num_unagg;
+
+    gko::kernels::omp::amgx_pgm::assign_to_exist_agg(
+        omp, d_weight_csr.get(), d_weight_diag.get(), d_x, d_intermediate_agg);
+    gko::kernels::omp::amgx_pgm::count_unagg(omp, d_agg, &d_num_unagg);
+
+    // only test whether all elements are aggregated.
+    GKO_ASSERT_EQ(d_num_unagg, 0);
+}
+
+
+TEST_F(AmgxPgm, GenerateMtxIsEquivalentToRef)
+{
+    initialize_data();
+    auto csr_coarse = Csr::create(ref, gko::dim<2>{n, n}, 0);
+    auto d_csr_coarse = Csr::create(omp, gko::dim<2>{n, n}, 0);
+    auto csr_temp = Csr::create(ref, gko::dim<2>{n, n},
+                                weight_csr->get_num_stored_elements());
+    auto d_csr_temp = Csr::create(omp, gko::dim<2>{n, n},
+                                  d_weight_csr->get_num_stored_elements());
+
+    gko::kernels::omp::amgx_pgm::amgx_pgm_generate(
+        omp, d_weight_csr.get(), d_agg, d_csr_coarse.get(), d_csr_temp.get());
+    gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
+        ref, weight_csr.get(), agg, csr_coarse.get(), csr_temp.get());
+
+    GKO_ASSERT_MTX_NEAR(d_csr_coarse, csr_coarse, 1e-14);
+}
+
+
+}  // namespace

From 3f2172674538bdc203487f8911afee2e754c2c27 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 21 Jul 2020 17:54:54 +0200
Subject: [PATCH 0663/1008] hip implementation

---
 hip/multigrid/amgx_pgm_kernels.hip.cpp      | 144 +++++++-
 hip/test/CMakeLists.txt                     |   1 +
 hip/test/multigrid/CMakeLists.txt           |   1 +
 hip/test/multigrid/amgx_pgm_kernels.hip.cpp | 362 ++++++++++++++++++++
 4 files changed, 496 insertions(+), 12 deletions(-)
 create mode 100644 hip/test/multigrid/CMakeLists.txt
 create mode 100644 hip/test/multigrid/amgx_pgm_kernels.hip.cpp

diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index 88c2db995f1..e3e90a06b74 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -45,10 +45,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/multigrid/amgx_pgm.hpp>
 
 
+#include "core/components/fill_array.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/csr_builder.hpp"
+#include "core/matrix/csr_kernels.hpp"
 #include "hip/base/hipsparse_bindings.hip.hpp"
 #include "hip/base/math.hip.hpp"
 #include "hip/base/types.hip.hpp"
-#include "hip/solver/common_trs_kernels.hip.hpp"
+#include "hip/components/atomic.hip.hpp"
+#include "hip/components/reduction.hip.hpp"
+#include "hip/components/thread_ids.hip.hpp"
 
 
 namespace gko {
@@ -62,25 +68,62 @@ namespace hip {
 namespace amgx_pgm {
 
 
+constexpr int default_block_size = 512;
+
+
+#include "common/multigrid/amgx_pgm_kernels.hpp.inc"
+
+
 template <typename IndexType>
 void match_edge(std::shared_ptr<const HipExecutor> exec,
                 const Array<IndexType> &strongest_neighbor,
-                Array<IndexType> &agg) GKO_NOT_IMPLEMENTED;
+                Array<IndexType> &agg)
+{
+    const auto num = agg.get_num_elems();
+    const dim3 grid(ceildiv(num, default_block_size));
+    hipLaunchKernelGGL(kernel::match_edge_kernel, dim3(grid),
+                       dim3(default_block_size), 0, 0, num,
+                       strongest_neighbor.get_const_data(), agg.get_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL);
 
 
 template <typename IndexType>
 void count_unagg(std::shared_ptr<const HipExecutor> exec,
-                 const Array<IndexType> &agg,
-                 IndexType *num_unagg) GKO_NOT_IMPLEMENTED;
+                 const Array<IndexType> &agg, IndexType *num_unagg)
+{
+    Array<IndexType> active_agg(exec, agg.get_num_elems());
+    const dim3 grid(ceildiv(active_agg.get_num_elems(), default_block_size));
+    hipLaunchKernelGGL(kernel::activate_kernel, dim3(grid),
+                       dim3(default_block_size), 0, 0,
+                       active_agg.get_num_elems(), agg.get_const_data(),
+                       active_agg.get_data());
+    *num_unagg = reduce_add_array(exec, active_agg.get_num_elems(),
+                                  active_agg.get_const_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_COUNT_UNAGG_KERNEL);
 
 
 template <typename IndexType>
 void renumber(std::shared_ptr<const HipExecutor> exec, Array<IndexType> &agg,
-              IndexType *num_agg) GKO_NOT_IMPLEMENTED;
+              IndexType *num_agg)
+{
+    const auto num = agg.get_num_elems();
+    Array<IndexType> agg_map(exec, num + 1);
+    components::fill_array(exec, agg_map.get_data(), agg_map.get_num_elems(),
+                           zero<IndexType>());
+    const dim3 grid(ceildiv(num, default_block_size));
+    hipLaunchKernelGGL(kernel::fill_agg_kernel, dim3(grid),
+                       dim3(default_block_size), 0, 0, num,
+                       agg.get_const_data(), agg_map.get_data());
+    components::prefix_sum(exec, agg_map.get_data(), agg_map.get_num_elems());
+    hipLaunchKernelGGL(kernel::renumber_kernel, dim3(grid),
+                       dim3(default_block_size), 0, 0, num,
+                       agg_map.get_const_data(), agg.get_data());
+    *num_agg = exec->copy_val_to_host(agg_map.get_const_data() + num);
+}
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_RENUMBER_KERNEL);
 
@@ -90,18 +133,45 @@ void find_strongest_neighbor(
     std::shared_ptr<const HipExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *weight_mtx,
     const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
-    Array<IndexType> &strongest_neighbor) GKO_NOT_IMPLEMENTED;
+    Array<IndexType> &strongest_neighbor)
+{
+    const auto num = agg.get_num_elems();
+    const dim3 grid(ceildiv(num, default_block_size));
+    hipLaunchKernelGGL(
+        kernel::find_strongest_neighbor_kernel, dim3(grid),
+        dim3(default_block_size), 0, 0, num, weight_mtx->get_const_row_ptrs(),
+        weight_mtx->get_const_col_idxs(), weight_mtx->get_const_values(),
+        diag->get_const_values(), diag->get_stride(), agg.get_data(),
+        strongest_neighbor.get_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR);
 
 
 template <typename ValueType, typename IndexType>
-void assign_to_exist_agg(
-    std::shared_ptr<const HipExecutor> exec,
-    const matrix::Csr<ValueType, IndexType> *weight_mtx,
-    const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg,
-    Array<IndexType> &intermediate_agg) GKO_NOT_IMPLEMENTED;
+void assign_to_exist_agg(std::shared_ptr<const HipExecutor> exec,
+                         const matrix::Csr<ValueType, IndexType> *weight_mtx,
+                         const matrix::Diagonal<ValueType> *diag,
+                         Array<IndexType> &agg,
+                         Array<IndexType> &intermediate_agg)
+{
+    auto agg_val = (intermediate_agg.get_num_elems() > 0)
+                       ? intermediate_agg.get_data()
+                       : agg.get_data();
+    const auto num = agg.get_num_elems();
+    const dim3 grid(ceildiv(num, default_block_size));
+    hipLaunchKernelGGL(kernel::assign_to_exist_agg_kernel, dim3(grid),
+                       dim3(default_block_size), 0, 0, num,
+                       weight_mtx->get_const_row_ptrs(),
+                       weight_mtx->get_const_col_idxs(),
+                       weight_mtx->get_const_values(), diag->get_const_values(),
+                       diag->get_stride(), agg.get_const_data(), agg_val);
+    if (intermediate_agg.get_num_elems() > 0) {
+        // Copy the intermediate_agg to agg
+        agg = intermediate_agg;
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
@@ -113,7 +183,57 @@ void amgx_pgm_generate(std::shared_ptr<const HipExecutor> exec,
                        const Array<IndexType> &agg,
                        matrix::Csr<ValueType, IndexType> *coarse,
                        matrix::Csr<ValueType, IndexType> *temp)
-    GKO_NOT_IMPLEMENTED;
+{
+    const auto source_nrows = source->get_size()[0];
+    const auto source_nnz = source->get_num_stored_elements();
+    const auto coarse_nrows = coarse->get_size()[0];
+    Array<IndexType> row_map(exec, source_nrows);
+    // fill coarse row pointer as zero
+    components::fill_array(exec, temp->get_row_ptrs(), coarse_nrows + 1,
+                           zero<IndexType>());
+    // compute each source row should be moved and also change column index
+    dim3 grid(ceildiv(source_nrows, default_block_size));
+    // agg source_row (for row size) coarse row source map
+    hipLaunchKernelGGL(kernel::get_source_row_map_kernel, dim3(grid),
+                       dim3(default_block_size), 0, 0, source_nrows,
+                       agg.get_const_data(), source->get_const_row_ptrs(),
+                       temp->get_row_ptrs(), row_map.get_data());
+    // prefix sum of temp_row_ptrs
+    components::prefix_sum(exec, temp->get_row_ptrs(), coarse_nrows + 1);
+    // copy source -> to coarse and change column index
+    hipLaunchKernelGGL(
+        kernel::move_row_kernel, dim3(grid), dim3(default_block_size), 0, 0,
+        source_nrows, agg.get_const_data(), row_map.get_const_data(),
+        source->get_const_row_ptrs(), source->get_const_col_idxs(),
+        as_hip_type(source->get_const_values()), temp->get_const_row_ptrs(),
+        temp->get_col_idxs(), as_hip_type(temp->get_values()));
+    // sort csr
+    csr::sort_by_column_index(exec, temp);
+    // summation of the elements with same position
+    grid = ceildiv(coarse_nrows, default_block_size);
+    hipLaunchKernelGGL(kernel::merge_col_kernel, dim3(grid),
+                       dim3(default_block_size), 0, 0, coarse_nrows,
+                       temp->get_const_row_ptrs(), temp->get_col_idxs(),
+                       as_hip_type(temp->get_values()), coarse->get_row_ptrs());
+    // build the coarse matrix
+    components::prefix_sum(exec, coarse->get_row_ptrs(), coarse_nrows + 1);
+    // prefix sum of coarse->get_row_ptrs
+    const auto coarse_nnz =
+        exec->copy_val_to_host(coarse->get_row_ptrs() + coarse_nrows);
+    // reallocate size of column and values
+    matrix::CsrBuilder<ValueType, IndexType> coarse_builder{coarse};
+    auto &coarse_col_idxs_array = coarse_builder.get_col_idx_array();
+    auto &coarse_vals_array = coarse_builder.get_value_array();
+    coarse_col_idxs_array.resize_and_reset(coarse_nnz);
+    coarse_vals_array.resize_and_reset(coarse_nnz);
+    // copy the result
+    hipLaunchKernelGGL(
+        kernel::copy_to_coarse_kernel, dim3(grid), dim3(default_block_size), 0,
+        0, coarse_nrows, temp->get_const_row_ptrs(), temp->get_const_col_idxs(),
+        as_hip_type(temp->get_const_values()), coarse->get_const_row_ptrs(),
+        coarse_col_idxs_array.get_data(),
+        as_hip_type(coarse_vals_array.get_data()));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
 
diff --git a/hip/test/CMakeLists.txt b/hip/test/CMakeLists.txt
index 9c8e39ca4ab..a5f126893bf 100644
--- a/hip/test/CMakeLists.txt
+++ b/hip/test/CMakeLists.txt
@@ -4,6 +4,7 @@ add_subdirectory(base)
 add_subdirectory(components)
 add_subdirectory(factorization)
 add_subdirectory(matrix)
+add_subdirectory(multigrid)
 add_subdirectory(solver)
 add_subdirectory(preconditioner)
 add_subdirectory(stop)
diff --git a/hip/test/multigrid/CMakeLists.txt b/hip/test/multigrid/CMakeLists.txt
new file mode 100644
index 00000000000..1c8534e5337
--- /dev/null
+++ b/hip/test/multigrid/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_hip_test(amgx_pgm_kernels)
diff --git a/hip/test/multigrid/amgx_pgm_kernels.hip.cpp b/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
new file mode 100644
index 00000000000..10befbe9315
--- /dev/null
+++ b/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
@@ -0,0 +1,362 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/multigrid/amgx_pgm.hpp>
+
+
+#include <fstream>
+#include <random>
+#include <string>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
+
+
+#include "core/multigrid/amgx_pgm_kernels.hpp"
+#include "core/test/utils/matrix_generator.hpp"
+#include "hip/test/utils.hip.hpp"
+
+
+namespace {
+
+
+template <typename Array, typename ValueDistribution, typename Engine>
+Array generate_random_array(gko::size_type num, ValueDistribution &&value_dist,
+                            Engine &&engine,
+                            std::shared_ptr<const gko::Executor> exec)
+{
+    using value_type = typename Array::value_type;
+    Array array_host(exec->get_master(), num);
+    auto val = array_host.get_data();
+    for (int i = 0; i < num; i++) {
+        val[i] =
+            gko::test::detail::get_rand_value<value_type>(value_dist, engine);
+    }
+    Array array(exec);
+    array = array_host;
+    return array;
+}
+
+
+class AmgxPgm : public ::testing::Test {
+protected:
+    using value_type = gko::default_precision;
+    using index_type = gko::int32;
+    using Mtx = gko::matrix::Dense<>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    AmgxPgm() : rand_engine(30) {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::HipExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        hip = gko::HipExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (hip != nullptr) {
+            ASSERT_NO_THROW(hip->synchronize());
+        }
+    }
+
+    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<Mtx>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(num_cols, num_cols),
+            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    gko::Array<index_type> gen_array(gko::size_type num, index_type min_val,
+                                     index_type max_val)
+    {
+        return generate_random_array<gko::Array<index_type>>(
+            num, std::uniform_int_distribution<>(min_val, max_val), rand_engine,
+            ref);
+    }
+
+    void initialize_data()
+    {
+        int m = 597;
+        n = 300;
+        int nrhs = 3;
+
+        agg = gen_array(m, 0, n - 1);
+        unfinished_agg = gen_array(m, -1, n - 1);
+        strongest_neighbor = gen_array(m, 0, n - 1);
+        coarse_vector = gen_mtx(n, nrhs);
+        fine_vector = gen_mtx(m, nrhs);
+        auto weight = gen_mtx(m, m);
+        make_weight(weight.get());
+        weight_csr = Csr::create(ref);
+        weight->convert_to(weight_csr.get());
+        weight_diag = weight_csr->extract_diagonal();
+
+        d_agg.set_executor(hip);
+        d_unfinished_agg.set_executor(hip);
+        d_strongest_neighbor.set_executor(hip);
+        d_coarse_vector = Mtx::create(hip);
+        d_fine_vector = Mtx::create(hip);
+        d_weight_csr = Csr::create(hip);
+        d_weight_diag = Mtx::create(hip);
+        d_agg = agg;
+        d_unfinished_agg = unfinished_agg;
+        d_strongest_neighbor = strongest_neighbor;
+        d_coarse_vector->copy_from(coarse_vector.get());
+        d_fine_vector->copy_from(fine_vector.get());
+        d_weight_csr->copy_from(weight_csr.get());
+        d_weight_diag->copy_from(weight_diag.get());
+    }
+
+    void make_symetric(Mtx *mtx)
+    {
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
+                mtx->at(i, j) = mtx->at(j, i);
+            }
+        }
+    }
+
+    // only for real value
+    void make_absoulte(Mtx *mtx)
+    {
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            for (int j = 0; j < mtx->get_size()[1]; ++j) {
+                mtx->at(i, j) = abs(mtx->at(i, j));
+            }
+        }
+    }
+
+    void make_diag_dominant(Mtx *mtx)
+    {
+        using std::abs;
+        for (int i = 0; i < mtx->get_size()[0]; ++i) {
+            auto sum = gko::zero<Mtx::value_type>();
+            for (int j = 0; j < mtx->get_size()[1]; ++j) {
+                sum += abs(mtx->at(i, j));
+            }
+            mtx->at(i, i) = sum;
+        }
+    }
+
+    void make_spd(Mtx *mtx)
+    {
+        make_symetric(mtx);
+        make_diag_dominant(mtx);
+    }
+
+    void make_weight(Mtx *mtx)
+    {
+        make_symetric(mtx);
+        make_absoulte(mtx);
+        make_diag_dominant(mtx);
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::HipExecutor> hip;
+
+    std::ranlux48 rand_engine;
+
+    gko::Array<index_type> agg;
+    gko::Array<index_type> unfinished_agg;
+    gko::Array<index_type> strongest_neighbor;
+
+    gko::Array<index_type> d_agg;
+    gko::Array<index_type> d_unfinished_agg;
+    gko::Array<index_type> d_strongest_neighbor;
+
+    std::unique_ptr<Mtx> coarse_vector;
+    std::unique_ptr<Mtx> fine_vector;
+    std::unique_ptr<Mtx> weight_diag;
+    std::unique_ptr<Csr> weight_csr;
+
+    std::unique_ptr<Mtx> d_coarse_vector;
+    std::unique_ptr<Mtx> d_fine_vector;
+    std::unique_ptr<Mtx> d_weight_diag;
+    std::unique_ptr<Csr> d_weight_csr;
+
+    gko::size_type n;
+};
+
+
+TEST_F(AmgxPgm, RestrictApplyIsEquivalentToRef)
+{
+    initialize_data();
+    // fine->coarse
+    auto x = Mtx::create_with_config_of(gko::lend(coarse_vector));
+    auto d_x = Mtx::create_with_config_of(gko::lend(d_coarse_vector));
+
+    gko::kernels::reference::amgx_pgm::restrict_apply(
+        ref, agg, fine_vector.get(), x.get());
+    gko::kernels::hip::amgx_pgm::restrict_apply(hip, d_agg, d_fine_vector.get(),
+                                                d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+}
+
+
+TEST_F(AmgxPgm, ProlongApplyaddIsEquivalentToRef)
+{
+    initialize_data();
+    // coarse->fine
+    auto x = fine_vector->clone();
+    auto d_x = d_fine_vector->clone();
+
+    gko::kernels::reference::amgx_pgm::prolong_applyadd(
+        ref, agg, coarse_vector.get(), x.get());
+    gko::kernels::hip::amgx_pgm::prolong_applyadd(
+        hip, d_agg, d_coarse_vector.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+}
+
+
+TEST_F(AmgxPgm, MatchEdgeIsEquivalentToRef)
+{
+    initialize_data();
+    auto x = unfinished_agg;
+    auto d_x = d_unfinished_agg;
+
+    gko::kernels::reference::amgx_pgm::match_edge(ref, strongest_neighbor, x);
+    gko::kernels::hip::amgx_pgm::match_edge(hip, d_strongest_neighbor, d_x);
+
+    GKO_ASSERT_ARRAY_EQ(d_x, x);
+}
+
+
+TEST_F(AmgxPgm, CountUnaggIsEquivalentToRef)
+{
+    initialize_data();
+    gko::size_type num_unagg;
+    gko::size_type d_num_unagg;
+
+    gko::kernels::reference::amgx_pgm::count_unagg(ref, agg, &num_unagg);
+    gko::kernels::hip::amgx_pgm::count_unagg(hip, d_agg, &d_num_unagg);
+
+    ASSERT_EQ(d_num_unagg, num_unagg);
+}
+
+
+TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
+{
+    initialize_data();
+    auto x = unfinished_agg;
+    auto d_x = d_unfinished_agg;
+    gko::size_type num_agg;
+    gko::size_type d_num_agg;
+
+    gko::kernels::reference::amgx_pgm::renumber(ref, agg, &num_agg);
+    gko::kernels::hip::amgx_pgm::renumber(hip, d_agg, &d_num_agg);
+
+    ASSERT_EQ(d_num_agg, num_agg);
+    GKO_ASSERT_ARRAY_EQ(d_agg, agg);
+    ASSERT_LE(num_agg, 300);
+}
+
+
+TEST_F(AmgxPgm, FindStrongestNeighborIsEquivalentToRef)
+{
+    initialize_data();
+    auto snb = strongest_neighbor;
+    auto d_snb = d_strongest_neighbor;
+
+    gko::kernels::reference::amgx_pgm::find_strongest_neighbor(
+        ref, weight_csr.get(), weight_diag.get(), agg, snb);
+    gko::kernels::hip::amgx_pgm::find_strongest_neighbor(
+        hip, d_weight_csr.get(), d_weight_diag.get(), d_agg, d_snb);
+
+    GKO_ASSERT_ARRAY_EQ(d_snb, snb);
+}
+
+
+TEST_F(AmgxPgm, AssignToExistAggIsEquivalentToRef)
+{
+    initialize_data();
+    auto x = unfinished_agg;
+    auto d_x = d_unfinished_agg;
+    auto intermediate_agg = x;
+    auto d_intermediate_agg = d_x;
+
+    gko::kernels::reference::amgx_pgm::assign_to_exist_agg(
+        ref, weight_csr.get(), weight_diag.get(), x, intermediate_agg);
+    gko::kernels::hip::amgx_pgm::assign_to_exist_agg(
+        hip, d_weight_csr.get(), d_weight_diag.get(), d_x, d_intermediate_agg);
+
+    GKO_ASSERT_ARRAY_EQ(d_x, x);
+}
+
+
+TEST_F(AmgxPgm, AssignToExistAggUnderteminsticIsEquivalentToRef)
+{
+    initialize_data();
+    auto d_x = d_unfinished_agg;
+    auto d_intermediate_agg = gko::Array<index_type>(hip, 0);
+    gko::size_type d_num_unagg;
+
+    gko::kernels::hip::amgx_pgm::assign_to_exist_agg(
+        hip, d_weight_csr.get(), d_weight_diag.get(), d_x, d_intermediate_agg);
+    gko::kernels::hip::amgx_pgm::count_unagg(hip, d_agg, &d_num_unagg);
+
+    // only test whether all elements are aggregated.
+    GKO_ASSERT_EQ(d_num_unagg, 0);
+}
+
+
+TEST_F(AmgxPgm, GenerateMtxIsEquivalentToRef)
+{
+    initialize_data();
+    auto csr_coarse = Csr::create(ref, gko::dim<2>{n, n}, 0);
+    auto d_csr_coarse = Csr::create(hip, gko::dim<2>{n, n}, 0);
+    auto csr_temp = Csr::create(ref, gko::dim<2>{n, n},
+                                weight_csr->get_num_stored_elements());
+    auto d_csr_temp = Csr::create(hip, gko::dim<2>{n, n},
+                                  d_weight_csr->get_num_stored_elements());
+
+    gko::kernels::hip::amgx_pgm::amgx_pgm_generate(
+        hip, d_weight_csr.get(), d_agg, d_csr_coarse.get(), d_csr_temp.get());
+    gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
+        ref, weight_csr.get(), agg, csr_coarse.get(), csr_temp.get());
+
+    GKO_ASSERT_MTX_NEAR(d_csr_coarse, csr_coarse, 1e-14);
+}
+
+
+}  // namespace

From aa46d5ed18ff2833f27773b3e6693f889b2d7f16 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 12 Aug 2020 11:10:15 +0200
Subject: [PATCH 0664/1008] fix amg_pgm_kernel determinstic problem

---
 common/multigrid/amgx_pgm_kernels.hpp.inc | 9 +++++----
 omp/multigrid/amgx_pgm_kernels.cpp        | 9 +++++----
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/common/multigrid/amgx_pgm_kernels.hpp.inc b/common/multigrid/amgx_pgm_kernels.hpp.inc
index 5bed8696b10..851a7023409 100644
--- a/common/multigrid/amgx_pgm_kernels.hpp.inc
+++ b/common/multigrid/amgx_pgm_kernels.hpp.inc
@@ -95,11 +95,12 @@ __global__ __launch_bounds__(default_block_size) void match_edge_kernel(
     if (agg_vals[tidx] != -1) {
         return;
     }
-    auto neighbor = strongest_neighbor_vals[tidx];
+    size_type neighbor = strongest_neighbor_vals[tidx];
     if (neighbor != -1 && strongest_neighbor_vals[neighbor] == tidx) {
-        agg_vals[tidx] = tidx;
-        agg_vals[neighbor] = tidx;
         // Use the smaller index as agg point
+        auto group = min(tidx, neighbor);
+        agg_vals[tidx] = group;
+        agg_vals[neighbor] = group;
     }
 }
 
@@ -342,4 +343,4 @@ __global__ __launch_bounds__(default_block_size) void copy_to_coarse_kernel(
 }
 
 
-}  // namespace kernel
\ No newline at end of file
+}  // namespace kernel
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index 80f1b2f88ce..540a7bdddcf 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -75,11 +75,12 @@ void match_edge(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for
     for (size_type i = 0; i < agg.get_num_elems(); i++) {
         if (agg_vals[i] == -1) {
-            auto neighbor = strongest_neighbor_vals[i];
+            size_type neighbor = strongest_neighbor_vals[i];
             if (neighbor != -1 && strongest_neighbor_vals[neighbor] == i) {
-                agg_vals[i] = i;
-                agg_vals[neighbor] = i;
                 // Use the smaller index as agg point
+                auto group = min(i, neighbor);
+                agg_vals[i] = group;
+                agg_vals[neighbor] = group;
             }
         }
     }
@@ -193,7 +194,6 @@ void assign_to_exist_agg(std::shared_ptr<const OmpExecutor> exec,
     const auto row_ptrs = weight_mtx->get_const_row_ptrs();
     const auto col_idxs = weight_mtx->get_const_col_idxs();
     const auto vals = weight_mtx->get_const_values();
-    auto max_weight_agg = zero<ValueType>();
     const auto agg_const_val = agg.get_const_data();
     auto agg_val = (intermediate_agg.get_num_elems() > 0)
                        ? intermediate_agg.get_data()
@@ -203,6 +203,7 @@ void assign_to_exist_agg(std::shared_ptr<const OmpExecutor> exec,
         if (agg_const_val[row] != -1) {
             continue;
         }
+        auto max_weight_agg = zero<ValueType>();
         IndexType strongest_agg = -1;
         for (auto idx = row_ptrs[row]; idx < row_ptrs[row + 1]; idx++) {
             auto col = col_idxs[idx];

From 642726b402e342b1f0aae84dc488f6166a71ffa7 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 31 Aug 2020 22:05:42 +0200
Subject: [PATCH 0665/1008] update diag related function to Diagonal format

---
 common/multigrid/amgx_pgm_kernels.hpp.inc   | 12 +++++------
 cuda/multigrid/amgx_pgm_kernels.cu          |  4 ++--
 cuda/test/multigrid/amgx_pgm_kernels.cpp    |  8 ++++---
 hip/multigrid/amgx_pgm_kernels.hip.cpp      | 23 ++++++++++-----------
 hip/test/multigrid/amgx_pgm_kernels.hip.cpp |  8 ++++---
 omp/multigrid/amgx_pgm_kernels.cpp          |  8 ++++---
 omp/test/multigrid/amgx_pgm_kernels.cpp     |  8 ++++---
 7 files changed, 38 insertions(+), 33 deletions(-)

diff --git a/common/multigrid/amgx_pgm_kernels.hpp.inc b/common/multigrid/amgx_pgm_kernels.hpp.inc
index 851a7023409..d8bda57a2e0 100644
--- a/common/multigrid/amgx_pgm_kernels.hpp.inc
+++ b/common/multigrid/amgx_pgm_kernels.hpp.inc
@@ -150,8 +150,8 @@ __global__
         const size_type num, const IndexType *__restrict__ row_ptrs,
         const IndexType *__restrict__ col_idxs,
         const ValueType *__restrict__ weight_vals,
-        const ValueType *__restrict__ diag, const size_type diag_stride,
-        IndexType *__restrict__ agg, IndexType *__restrict__ strongest_neighbor)
+        const ValueType *__restrict__ diag, IndexType *__restrict__ agg,
+        IndexType *__restrict__ strongest_neighbor)
 {
     auto row = thread::get_thread_id_flat();
     if (row >= num) {
@@ -170,8 +170,7 @@ __global__
         if (col == row) {
             continue;
         }
-        auto weight = weight_vals[idx] / max(abs(diag[row * diag_stride]),
-                                             abs(diag[col * diag_stride]));
+        auto weight = weight_vals[idx] / max(abs(diag[row]), abs(diag[col]));
         if (agg[col] == -1 &&
             (weight > max_weight_unagg ||
              (weight == max_weight_unagg && col > strongest_unagg))) {
@@ -207,7 +206,7 @@ __global__
         const size_type num, const IndexType *__restrict__ row_ptrs,
         const IndexType *__restrict__ col_idxs,
         const ValueType *__restrict__ weight_vals,
-        const ValueType *__restrict__ diag, const size_type diag_stride,
+        const ValueType *__restrict__ diag,
         const IndexType *__restrict__ agg_const_val,
         IndexType *__restrict__ agg_val)
 {
@@ -225,8 +224,7 @@ __global__
         if (col == row) {
             continue;
         }
-        auto weight = weight_vals[idx] / max(abs(diag[row * diag_stride]),
-                                             abs(diag[col * diag_stride]));
+        auto weight = weight_vals[idx] / max(abs(diag[row]), abs(diag[col]));
         if (agg_const_val[col] != -1 &&
             (weight > max_weight_agg ||
              (weight == max_weight_agg && col > strongest_agg))) {
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index d84766557cb..b8700466365 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -136,7 +136,7 @@ void find_strongest_neighbor(
     kernel::find_strongest_neighbor_kernel<<<grid, default_block_size>>>(
         num, weight_mtx->get_const_row_ptrs(), weight_mtx->get_const_col_idxs(),
         weight_mtx->get_const_values(), diag->get_const_values(),
-        diag->get_stride(), agg.get_data(), strongest_neighbor.get_data());
+        agg.get_data(), strongest_neighbor.get_data());
 }
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
@@ -157,7 +157,7 @@ void assign_to_exist_agg(std::shared_ptr<const CudaExecutor> exec,
     kernel::assign_to_exist_agg_kernel<<<grid, default_block_size>>>(
         num, weight_mtx->get_const_row_ptrs(), weight_mtx->get_const_col_idxs(),
         weight_mtx->get_const_values(), diag->get_const_values(),
-        diag->get_stride(), agg.get_const_data(), agg_val);
+        agg.get_const_data(), agg_val);
     if (intermediate_agg.get_num_elems() > 0) {
         // Copy the intermediate_agg to agg
         agg = intermediate_agg;
diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
index 6c2c34885a1..fa4e582085d 100644
--- a/cuda/test/multigrid/amgx_pgm_kernels.cpp
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm.hpp>
@@ -81,6 +82,7 @@ class AmgxPgm : public ::testing::Test {
     using index_type = gko::int32;
     using Mtx = gko::matrix::Dense<>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
+    using Diag = gko::matrix::Diagonal<value_type>;
     AmgxPgm() : rand_engine(30) {}
 
     void SetUp()
@@ -136,7 +138,7 @@ class AmgxPgm : public ::testing::Test {
         d_coarse_vector = Mtx::create(cuda);
         d_fine_vector = Mtx::create(cuda);
         d_weight_csr = Csr::create(cuda);
-        d_weight_diag = Mtx::create(cuda);
+        d_weight_diag = Diag::create(cuda);
         d_agg = agg;
         d_unfinished_agg = unfinished_agg;
         d_strongest_neighbor = strongest_neighbor;
@@ -205,12 +207,12 @@ class AmgxPgm : public ::testing::Test {
 
     std::unique_ptr<Mtx> coarse_vector;
     std::unique_ptr<Mtx> fine_vector;
-    std::unique_ptr<Mtx> weight_diag;
+    std::unique_ptr<Diag> weight_diag;
     std::unique_ptr<Csr> weight_csr;
 
     std::unique_ptr<Mtx> d_coarse_vector;
     std::unique_ptr<Mtx> d_fine_vector;
-    std::unique_ptr<Mtx> d_weight_diag;
+    std::unique_ptr<Diag> d_weight_diag;
     std::unique_ptr<Csr> d_weight_csr;
 
     gko::size_type n;
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index e3e90a06b74..41fc38854a8 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -137,12 +137,12 @@ void find_strongest_neighbor(
 {
     const auto num = agg.get_num_elems();
     const dim3 grid(ceildiv(num, default_block_size));
-    hipLaunchKernelGGL(
-        kernel::find_strongest_neighbor_kernel, dim3(grid),
-        dim3(default_block_size), 0, 0, num, weight_mtx->get_const_row_ptrs(),
-        weight_mtx->get_const_col_idxs(), weight_mtx->get_const_values(),
-        diag->get_const_values(), diag->get_stride(), agg.get_data(),
-        strongest_neighbor.get_data());
+    hipLaunchKernelGGL(kernel::find_strongest_neighbor_kernel, dim3(grid),
+                       dim3(default_block_size), 0, 0, num,
+                       weight_mtx->get_const_row_ptrs(),
+                       weight_mtx->get_const_col_idxs(),
+                       weight_mtx->get_const_values(), diag->get_const_values(),
+                       agg.get_data(), strongest_neighbor.get_data());
 }
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
@@ -161,12 +161,11 @@ void assign_to_exist_agg(std::shared_ptr<const HipExecutor> exec,
                        : agg.get_data();
     const auto num = agg.get_num_elems();
     const dim3 grid(ceildiv(num, default_block_size));
-    hipLaunchKernelGGL(kernel::assign_to_exist_agg_kernel, dim3(grid),
-                       dim3(default_block_size), 0, 0, num,
-                       weight_mtx->get_const_row_ptrs(),
-                       weight_mtx->get_const_col_idxs(),
-                       weight_mtx->get_const_values(), diag->get_const_values(),
-                       diag->get_stride(), agg.get_const_data(), agg_val);
+    hipLaunchKernelGGL(
+        kernel::assign_to_exist_agg_kernel, dim3(grid),
+        dim3(default_block_size), 0, 0, num, weight_mtx->get_const_row_ptrs(),
+        weight_mtx->get_const_col_idxs(), weight_mtx->get_const_values(),
+        diag->get_const_values(), agg.get_const_data(), agg_val);
     if (intermediate_agg.get_num_elems() > 0) {
         // Copy the intermediate_agg to agg
         agg = intermediate_agg;
diff --git a/hip/test/multigrid/amgx_pgm_kernels.hip.cpp b/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
index 10befbe9315..9575e854207 100644
--- a/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm.hpp>
@@ -81,6 +82,7 @@ class AmgxPgm : public ::testing::Test {
     using index_type = gko::int32;
     using Mtx = gko::matrix::Dense<>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
+    using Diag = gko::matrix::Diagonal<value_type>;
     AmgxPgm() : rand_engine(30) {}
 
     void SetUp()
@@ -136,7 +138,7 @@ class AmgxPgm : public ::testing::Test {
         d_coarse_vector = Mtx::create(hip);
         d_fine_vector = Mtx::create(hip);
         d_weight_csr = Csr::create(hip);
-        d_weight_diag = Mtx::create(hip);
+        d_weight_diag = Diag::create(hip);
         d_agg = agg;
         d_unfinished_agg = unfinished_agg;
         d_strongest_neighbor = strongest_neighbor;
@@ -205,12 +207,12 @@ class AmgxPgm : public ::testing::Test {
 
     std::unique_ptr<Mtx> coarse_vector;
     std::unique_ptr<Mtx> fine_vector;
-    std::unique_ptr<Mtx> weight_diag;
+    std::unique_ptr<Diag> weight_diag;
     std::unique_ptr<Csr> weight_csr;
 
     std::unique_ptr<Mtx> d_coarse_vector;
     std::unique_ptr<Mtx> d_fine_vector;
-    std::unique_ptr<Mtx> d_weight_diag;
+    std::unique_ptr<Diag> d_weight_diag;
     std::unique_ptr<Csr> d_weight_csr;
 
     gko::size_type n;
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index 540a7bdddcf..9c4efefebe6 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -138,6 +138,7 @@ void find_strongest_neighbor(
     const auto row_ptrs = weight_mtx->get_const_row_ptrs();
     const auto col_idxs = weight_mtx->get_const_col_idxs();
     const auto vals = weight_mtx->get_const_values();
+    const auto diag_vals = diag->get_const_values();
 #pragma omp parallel for
     for (size_type row = 0; row < agg.get_num_elems(); row++) {
         auto max_weight_unagg = zero<ValueType>();
@@ -150,8 +151,8 @@ void find_strongest_neighbor(
                 if (col == row) {
                     continue;
                 }
-                auto weight = vals[idx] /
-                              max(abs(diag->at(row, 0)), abs(diag->at(col, 0)));
+                auto weight =
+                    vals[idx] / max(abs(diag_vals[col]), abs(diag_vals[col]));
                 if (agg.get_const_data()[col] == -1 &&
                     (weight > max_weight_unagg ||
                      (weight == max_weight_unagg && col > strongest_unagg))) {
@@ -198,6 +199,7 @@ void assign_to_exist_agg(std::shared_ptr<const OmpExecutor> exec,
     auto agg_val = (intermediate_agg.get_num_elems() > 0)
                        ? intermediate_agg.get_data()
                        : agg.get_data();
+    const auto diag_vals = diag->get_const_values();
 #pragma omp parallel for
     for (IndexType row = 0; row < agg.get_num_elems(); row++) {
         if (agg_const_val[row] != -1) {
@@ -211,7 +213,7 @@ void assign_to_exist_agg(std::shared_ptr<const OmpExecutor> exec,
                 continue;
             }
             auto weight =
-                vals[idx] / max(abs(diag->at(row, 0)), abs(diag->at(col, 0)));
+                vals[idx] / max(abs(diag_vals[row]), abs(diag_vals[col]));
             if (agg_const_val[col] != -1 &&
                 (weight > max_weight_agg ||
                  (weight == max_weight_agg && col > strongest_agg))) {
diff --git a/omp/test/multigrid/amgx_pgm_kernels.cpp b/omp/test/multigrid/amgx_pgm_kernels.cpp
index cc822135a5d..09e1cf918a3 100644
--- a/omp/test/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/test/multigrid/amgx_pgm_kernels.cpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm.hpp>
@@ -80,6 +81,7 @@ class AmgxPgm : public ::testing::Test {
     using index_type = gko::int32;
     using Mtx = gko::matrix::Dense<>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
+    using Diag = gko::matrix::Diagonal<value_type>;
     AmgxPgm() : rand_engine(30) {}
 
     void SetUp()
@@ -127,7 +129,7 @@ class AmgxPgm : public ::testing::Test {
         d_coarse_vector = Mtx::create(omp);
         d_fine_vector = Mtx::create(omp);
         d_weight_csr = Csr::create(omp);
-        d_weight_diag = Mtx::create(omp);
+        d_weight_diag = Diag::create(omp);
         d_agg = agg;
         d_unfinished_agg = unfinished_agg;
         d_strongest_neighbor = strongest_neighbor;
@@ -196,12 +198,12 @@ class AmgxPgm : public ::testing::Test {
 
     std::unique_ptr<Mtx> coarse_vector;
     std::unique_ptr<Mtx> fine_vector;
-    std::unique_ptr<Mtx> weight_diag;
+    std::unique_ptr<Diag> weight_diag;
     std::unique_ptr<Csr> weight_csr;
 
     std::unique_ptr<Mtx> d_coarse_vector;
     std::unique_ptr<Mtx> d_fine_vector;
-    std::unique_ptr<Mtx> d_weight_diag;
+    std::unique_ptr<Diag> d_weight_diag;
     std::unique_ptr<Csr> d_weight_csr;
 
     gko::size_type n;

From e014369ebb2a7999ff2d42a02edb2373a155f625 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 27 Jan 2021 16:37:50 +0800
Subject: [PATCH 0666/1008] fix the type of the tests

---
 cuda/test/multigrid/amgx_pgm_kernels.cpp    | 10 +++++-----
 hip/test/multigrid/amgx_pgm_kernels.hip.cpp | 10 +++++-----
 omp/test/multigrid/amgx_pgm_kernels.cpp     | 10 +++++-----
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
index fa4e582085d..a9852bd3118 100644
--- a/cuda/test/multigrid/amgx_pgm_kernels.cpp
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -267,8 +267,8 @@ TEST_F(AmgxPgm, MatchEdgeIsEquivalentToRef)
 TEST_F(AmgxPgm, CountUnaggIsEquivalentToRef)
 {
     initialize_data();
-    gko::size_type num_unagg;
-    gko::size_type d_num_unagg;
+    index_type num_unagg;
+    index_type d_num_unagg;
 
     gko::kernels::reference::amgx_pgm::count_unagg(ref, agg, &num_unagg);
     gko::kernels::cuda::amgx_pgm::count_unagg(cuda, d_agg, &d_num_unagg);
@@ -282,8 +282,8 @@ TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
     initialize_data();
     auto x = unfinished_agg;
     auto d_x = d_unfinished_agg;
-    gko::size_type num_agg;
-    gko::size_type d_num_agg;
+    index_type num_agg;
+    index_type d_num_agg;
 
     gko::kernels::reference::amgx_pgm::renumber(ref, agg, &num_agg);
     gko::kernels::cuda::amgx_pgm::renumber(cuda, d_agg, &d_num_agg);
@@ -331,7 +331,7 @@ TEST_F(AmgxPgm, AssignToExistAggUnderteminsticIsEquivalentToRef)
     initialize_data();
     auto d_x = d_unfinished_agg;
     auto d_intermediate_agg = gko::Array<index_type>(cuda, 0);
-    gko::size_type d_num_unagg;
+    index_type d_num_unagg;
 
     gko::kernels::cuda::amgx_pgm::assign_to_exist_agg(
         cuda, d_weight_csr.get(), d_weight_diag.get(), d_x, d_intermediate_agg);
diff --git a/hip/test/multigrid/amgx_pgm_kernels.hip.cpp b/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
index 9575e854207..0d059b631b2 100644
--- a/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
@@ -267,8 +267,8 @@ TEST_F(AmgxPgm, MatchEdgeIsEquivalentToRef)
 TEST_F(AmgxPgm, CountUnaggIsEquivalentToRef)
 {
     initialize_data();
-    gko::size_type num_unagg;
-    gko::size_type d_num_unagg;
+    index_type num_unagg;
+    index_type d_num_unagg;
 
     gko::kernels::reference::amgx_pgm::count_unagg(ref, agg, &num_unagg);
     gko::kernels::hip::amgx_pgm::count_unagg(hip, d_agg, &d_num_unagg);
@@ -282,8 +282,8 @@ TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
     initialize_data();
     auto x = unfinished_agg;
     auto d_x = d_unfinished_agg;
-    gko::size_type num_agg;
-    gko::size_type d_num_agg;
+    index_type num_agg;
+    index_type d_num_agg;
 
     gko::kernels::reference::amgx_pgm::renumber(ref, agg, &num_agg);
     gko::kernels::hip::amgx_pgm::renumber(hip, d_agg, &d_num_agg);
@@ -331,7 +331,7 @@ TEST_F(AmgxPgm, AssignToExistAggUnderteminsticIsEquivalentToRef)
     initialize_data();
     auto d_x = d_unfinished_agg;
     auto d_intermediate_agg = gko::Array<index_type>(hip, 0);
-    gko::size_type d_num_unagg;
+    index_type d_num_unagg;
 
     gko::kernels::hip::amgx_pgm::assign_to_exist_agg(
         hip, d_weight_csr.get(), d_weight_diag.get(), d_x, d_intermediate_agg);
diff --git a/omp/test/multigrid/amgx_pgm_kernels.cpp b/omp/test/multigrid/amgx_pgm_kernels.cpp
index 09e1cf918a3..3b00ea89054 100644
--- a/omp/test/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/test/multigrid/amgx_pgm_kernels.cpp
@@ -258,8 +258,8 @@ TEST_F(AmgxPgm, MatchEdgeIsEquivalentToRef)
 TEST_F(AmgxPgm, CountUnaggIsEquivalentToRef)
 {
     initialize_data();
-    gko::size_type num_unagg;
-    gko::size_type d_num_unagg;
+    index_type num_unagg;
+    index_type d_num_unagg;
 
     gko::kernels::reference::amgx_pgm::count_unagg(ref, agg, &num_unagg);
     gko::kernels::omp::amgx_pgm::count_unagg(omp, d_agg, &d_num_unagg);
@@ -273,8 +273,8 @@ TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
     initialize_data();
     auto x = unfinished_agg;
     auto d_x = d_unfinished_agg;
-    gko::size_type num_agg;
-    gko::size_type d_num_agg;
+    index_type num_agg;
+    index_type d_num_agg;
 
     gko::kernels::reference::amgx_pgm::renumber(ref, agg, &num_agg);
     gko::kernels::omp::amgx_pgm::renumber(omp, d_agg, &d_num_agg);
@@ -322,7 +322,7 @@ TEST_F(AmgxPgm, AssignToExistAggUnderteminsticIsEquivalentToRef)
     initialize_data();
     auto d_x = d_unfinished_agg;
     auto d_intermediate_agg = gko::Array<index_type>(omp, 0);
-    gko::size_type d_num_unagg;
+    index_type d_num_unagg;
 
     gko::kernels::omp::amgx_pgm::assign_to_exist_agg(
         omp, d_weight_csr.get(), d_weight_diag.get(), d_x, d_intermediate_agg);

From 7947017c0ff009866caad39d164674e6b1023822 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 27 Jan 2021 16:39:49 +0800
Subject: [PATCH 0667/1008] format

---
 cuda/test/multigrid/amgx_pgm_kernels.cpp    | 2 +-
 hip/test/multigrid/amgx_pgm_kernels.hip.cpp | 2 +-
 omp/test/multigrid/amgx_pgm_kernels.cpp     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
index a9852bd3118..2d855dc328b 100644
--- a/cuda/test/multigrid/amgx_pgm_kernels.cpp
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/hip/test/multigrid/amgx_pgm_kernels.hip.cpp b/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
index 0d059b631b2..6015b2f5522 100644
--- a/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/omp/test/multigrid/amgx_pgm_kernels.cpp b/omp/test/multigrid/amgx_pgm_kernels.cpp
index 3b00ea89054..22fea84bcaa 100644
--- a/omp/test/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/test/multigrid/amgx_pgm_kernels.cpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

From c13cf5e86d1639c45f82fc0dcbca6c47c7e21979 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 10 Mar 2021 23:53:40 +0800
Subject: [PATCH 0668/1008] delete the tests using unused function

---
 common/multigrid/amgx_pgm_kernels.hpp.inc   | 38 +--------------------
 cuda/test/multigrid/amgx_pgm_kernels.cpp    | 32 -----------------
 hip/test/multigrid/amgx_pgm_kernels.hip.cpp | 32 -----------------
 omp/test/multigrid/amgx_pgm_kernels.cpp     | 32 -----------------
 4 files changed, 1 insertion(+), 133 deletions(-)

diff --git a/common/multigrid/amgx_pgm_kernels.hpp.inc b/common/multigrid/amgx_pgm_kernels.hpp.inc
index d8bda57a2e0..8565aaa6227 100644
--- a/common/multigrid/amgx_pgm_kernels.hpp.inc
+++ b/common/multigrid/amgx_pgm_kernels.hpp.inc
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -33,42 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace kernel {
 
 
-template <typename ValueType, typename IndexType>
-__global__ __launch_bounds__(default_block_size) void restrict_apply_kernel(
-    const IndexType *__restrict__ agg, const size_type num_rows,
-    const size_type num_rhs, const ValueType *__restrict__ b,
-    const size_type b_stride, ValueType *__restrict__ x,
-    const size_type x_stride)
-{
-    auto tidx = thread::get_thread_id_flat();
-    auto row = tidx / num_rhs;
-    if (row >= num_rows) {
-        return;
-    }
-    auto col = tidx % num_rhs;
-    auto ind = agg[row];
-    atomic_add(x + ind * x_stride + col, b[row * b_stride + col]);
-}
-
-
-template <typename ValueType, typename IndexType>
-__global__ __launch_bounds__(default_block_size) void prolong_applyadd_kernel(
-    const IndexType *__restrict__ agg, const size_type num_rows,
-    const size_type num_rhs, const ValueType *__restrict__ b,
-    const size_type b_stride, ValueType *__restrict__ x,
-    const size_type x_stride)
-{
-    auto tidx = thread::get_thread_id_flat();
-    auto row = tidx / num_rhs;
-    if (row >= num_rows) {
-        return;
-    }
-    auto col = tidx % num_rhs;
-    auto ind = agg[row];
-    x[row * x_stride + col] += b[ind * b_stride + col];
-}
-
-
 template <typename IndexType>
 __global__ __launch_bounds__(default_block_size) void replace_kernel(
     size_type size, const IndexType *__restrict__ source,
diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
index 2d855dc328b..6a3071e5ce0 100644
--- a/cuda/test/multigrid/amgx_pgm_kernels.cpp
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -219,38 +219,6 @@ class AmgxPgm : public ::testing::Test {
 };
 
 
-TEST_F(AmgxPgm, RestrictApplyIsEquivalentToRef)
-{
-    initialize_data();
-    // fine->coarse
-    auto x = Mtx::create_with_config_of(gko::lend(coarse_vector));
-    auto d_x = Mtx::create_with_config_of(gko::lend(d_coarse_vector));
-
-    gko::kernels::reference::amgx_pgm::restrict_apply(
-        ref, agg, fine_vector.get(), x.get());
-    gko::kernels::cuda::amgx_pgm::restrict_apply(
-        cuda, d_agg, d_fine_vector.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-TEST_F(AmgxPgm, ProlongApplyaddIsEquivalentToRef)
-{
-    initialize_data();
-    // coarse->fine
-    auto x = fine_vector->clone();
-    auto d_x = d_fine_vector->clone();
-
-    gko::kernels::reference::amgx_pgm::prolong_applyadd(
-        ref, agg, coarse_vector.get(), x.get());
-    gko::kernels::cuda::amgx_pgm::prolong_applyadd(
-        cuda, d_agg, d_coarse_vector.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
 TEST_F(AmgxPgm, MatchEdgeIsEquivalentToRef)
 {
     initialize_data();
diff --git a/hip/test/multigrid/amgx_pgm_kernels.hip.cpp b/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
index 6015b2f5522..ad1e8f040e7 100644
--- a/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
@@ -219,38 +219,6 @@ class AmgxPgm : public ::testing::Test {
 };
 
 
-TEST_F(AmgxPgm, RestrictApplyIsEquivalentToRef)
-{
-    initialize_data();
-    // fine->coarse
-    auto x = Mtx::create_with_config_of(gko::lend(coarse_vector));
-    auto d_x = Mtx::create_with_config_of(gko::lend(d_coarse_vector));
-
-    gko::kernels::reference::amgx_pgm::restrict_apply(
-        ref, agg, fine_vector.get(), x.get());
-    gko::kernels::hip::amgx_pgm::restrict_apply(hip, d_agg, d_fine_vector.get(),
-                                                d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-TEST_F(AmgxPgm, ProlongApplyaddIsEquivalentToRef)
-{
-    initialize_data();
-    // coarse->fine
-    auto x = fine_vector->clone();
-    auto d_x = d_fine_vector->clone();
-
-    gko::kernels::reference::amgx_pgm::prolong_applyadd(
-        ref, agg, coarse_vector.get(), x.get());
-    gko::kernels::hip::amgx_pgm::prolong_applyadd(
-        hip, d_agg, d_coarse_vector.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
 TEST_F(AmgxPgm, MatchEdgeIsEquivalentToRef)
 {
     initialize_data();
diff --git a/omp/test/multigrid/amgx_pgm_kernels.cpp b/omp/test/multigrid/amgx_pgm_kernels.cpp
index 22fea84bcaa..9a27ef3f4fc 100644
--- a/omp/test/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/test/multigrid/amgx_pgm_kernels.cpp
@@ -210,38 +210,6 @@ class AmgxPgm : public ::testing::Test {
 };
 
 
-TEST_F(AmgxPgm, RestrictApplyIsEquivalentToRef)
-{
-    initialize_data();
-    // fine->coarse
-    auto x = Mtx::create_with_config_of(gko::lend(coarse_vector));
-    auto d_x = Mtx::create_with_config_of(gko::lend(d_coarse_vector));
-
-    gko::kernels::reference::amgx_pgm::restrict_apply(
-        ref, agg, fine_vector.get(), x.get());
-    gko::kernels::omp::amgx_pgm::restrict_apply(omp, d_agg, d_fine_vector.get(),
-                                                d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-TEST_F(AmgxPgm, ProlongApplyaddIsEquivalentToRef)
-{
-    initialize_data();
-    // coarse->fine
-    auto x = fine_vector->clone();
-    auto d_x = d_fine_vector->clone();
-
-    gko::kernels::reference::amgx_pgm::prolong_applyadd(
-        ref, agg, coarse_vector.get(), x.get());
-    gko::kernels::omp::amgx_pgm::prolong_applyadd(
-        omp, d_agg, d_coarse_vector.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
 TEST_F(AmgxPgm, MatchEdgeIsEquivalentToRef)
 {
     initialize_data();

From 932ecf1c36c237aba201ac398a6562739e42ceb5 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 15 Mar 2021 15:24:23 +0800
Subject: [PATCH 0669/1008] change deprecated header, fix wrong type, add test

---
 common/multigrid/amgx_pgm_kernels.hpp.inc     | 45 ++++++++++++++++---
 cuda/multigrid/amgx_pgm_kernels.cu            | 19 +++++---
 cuda/test/multigrid/amgx_pgm_kernels.cpp      | 30 +++++++++++++
 dpcpp/multigrid/amgx_pgm_kernels.dp.cpp       | 10 ++---
 hip/multigrid/amgx_pgm_kernels.hip.cpp        | 24 ++++++----
 hip/test/multigrid/CMakeLists.txt             |  2 +-
 ...m_kernels.hip.cpp => amgx_pgm_kernels.cpp} | 30 +++++++++++++
 reference/test/multigrid/amgx_pgm_kernels.cpp |  2 +-
 8 files changed, 134 insertions(+), 28 deletions(-)
 rename hip/test/multigrid/{amgx_pgm_kernels.hip.cpp => amgx_pgm_kernels.cpp} (88%)

diff --git a/common/multigrid/amgx_pgm_kernels.hpp.inc b/common/multigrid/amgx_pgm_kernels.hpp.inc
index 8565aaa6227..eda2854763e 100644
--- a/common/multigrid/amgx_pgm_kernels.hpp.inc
+++ b/common/multigrid/amgx_pgm_kernels.hpp.inc
@@ -52,14 +52,14 @@ __global__ __launch_bounds__(default_block_size) void match_edge_kernel(
     size_type num, const IndexType *__restrict__ strongest_neighbor_vals,
     IndexType *__restrict__ agg_vals)
 {
-    auto tidx = thread::get_thread_id_flat();
+    auto tidx = thread::get_thread_id_flat<IndexType>();
     if (tidx >= num) {
         return;
     }
     if (agg_vals[tidx] != -1) {
         return;
     }
-    size_type neighbor = strongest_neighbor_vals[tidx];
+    auto neighbor = strongest_neighbor_vals[tidx];
     if (neighbor != -1 && strongest_neighbor_vals[neighbor] == tidx) {
         // Use the smaller index as agg point
         auto group = min(tidx, neighbor);
@@ -175,10 +175,7 @@ __global__
         IndexType *__restrict__ agg_val)
 {
     auto row = thread::get_thread_id_flat();
-    if (row >= num) {
-        return;
-    }
-    if (agg_val[row] != -1) {
+    if (row >= num || agg_val[row] != -1) {
         return;
     }
     ValueType max_weight_agg = zero<ValueType>();
@@ -203,6 +200,42 @@ __global__
     }
 }
 
+// This is the undeterminstic implementation which is the same implementation of
+// the previous one but agg_val == agg_const_val.
+template <typename ValueType, typename IndexType>
+__global__
+    __launch_bounds__(default_block_size) void assign_to_exist_agg_kernel(
+        const size_type num, const IndexType *__restrict__ row_ptrs,
+        const IndexType *__restrict__ col_idxs,
+        const ValueType *__restrict__ weight_vals,
+        const ValueType *__restrict__ diag, IndexType *__restrict__ agg_val)
+{
+    auto row = thread::get_thread_id_flat();
+    if (row >= num || agg_val[row] != -1) {
+        return;
+    }
+    ValueType max_weight_agg = zero<ValueType>();
+    IndexType strongest_agg = -1;
+    for (auto idx = row_ptrs[row]; idx < row_ptrs[row + 1]; idx++) {
+        auto col = col_idxs[idx];
+        if (col == row) {
+            continue;
+        }
+        auto weight = weight_vals[idx] / max(abs(diag[row]), abs(diag[col]));
+        if (agg_val[col] != -1 &&
+            (weight > max_weight_agg ||
+             (weight == max_weight_agg && col > strongest_agg))) {
+            max_weight_agg = weight;
+            strongest_agg = col;
+        }
+    }
+    if (strongest_agg != -1) {
+        agg_val[row] = agg_val[strongest_agg];
+    } else {
+        agg_val[row] = row;
+    }
+}
+
 
 template <typename IndexType>
 __global__ __launch_bounds__(default_block_size) void get_source_row_map_kernel(
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index b8700466365..1560d7bf7f7 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -149,18 +149,23 @@ void assign_to_exist_agg(std::shared_ptr<const CudaExecutor> exec,
                          Array<IndexType> &agg,
                          Array<IndexType> &intermediate_agg)
 {
-    auto agg_val = (intermediate_agg.get_num_elems() > 0)
-                       ? intermediate_agg.get_data()
-                       : agg.get_data();
     const auto num = agg.get_num_elems();
     const dim3 grid(ceildiv(num, default_block_size));
-    kernel::assign_to_exist_agg_kernel<<<grid, default_block_size>>>(
-        num, weight_mtx->get_const_row_ptrs(), weight_mtx->get_const_col_idxs(),
-        weight_mtx->get_const_values(), diag->get_const_values(),
-        agg.get_const_data(), agg_val);
     if (intermediate_agg.get_num_elems() > 0) {
+        // determinstic kernel
+        kernel::assign_to_exist_agg_kernel<<<grid, default_block_size>>>(
+            num, weight_mtx->get_const_row_ptrs(),
+            weight_mtx->get_const_col_idxs(), weight_mtx->get_const_values(),
+            diag->get_const_values(), agg.get_const_data(),
+            intermediate_agg.get_data());
         // Copy the intermediate_agg to agg
         agg = intermediate_agg;
+    } else {
+        // undeterminstic kernel
+        kernel::assign_to_exist_agg_kernel<<<grid, default_block_size>>>(
+            num, weight_mtx->get_const_row_ptrs(),
+            weight_mtx->get_const_col_idxs(), weight_mtx->get_const_values(),
+            diag->get_const_values(), agg.get_data());
     }
 }
 
diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
index 6a3071e5ce0..10d2b97a8bd 100644
--- a/cuda/test/multigrid/amgx_pgm_kernels.cpp
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -131,6 +131,10 @@ class AmgxPgm : public ::testing::Test {
         weight_csr = Csr::create(ref);
         weight->convert_to(weight_csr.get());
         weight_diag = weight_csr->extract_diagonal();
+        auto system_dense = gen_mtx(m, m);
+        make_spd(system_dense.get());
+        system_mtx = Csr::create(ref);
+        system_dense->convert_to(system_mtx.get());
 
         d_agg.set_executor(cuda);
         d_unfinished_agg.set_executor(cuda);
@@ -139,6 +143,7 @@ class AmgxPgm : public ::testing::Test {
         d_fine_vector = Mtx::create(cuda);
         d_weight_csr = Csr::create(cuda);
         d_weight_diag = Diag::create(cuda);
+        d_system_mtx = Csr::create(cuda);
         d_agg = agg;
         d_unfinished_agg = unfinished_agg;
         d_strongest_neighbor = strongest_neighbor;
@@ -146,6 +151,7 @@ class AmgxPgm : public ::testing::Test {
         d_fine_vector->copy_from(fine_vector.get());
         d_weight_csr->copy_from(weight_csr.get());
         d_weight_diag->copy_from(weight_diag.get());
+        d_system_mtx->copy_from(system_mtx.get());
     }
 
     void make_symetric(Mtx *mtx)
@@ -209,11 +215,13 @@ class AmgxPgm : public ::testing::Test {
     std::unique_ptr<Mtx> fine_vector;
     std::unique_ptr<Diag> weight_diag;
     std::unique_ptr<Csr> weight_csr;
+    std::shared_ptr<Csr> system_mtx;
 
     std::unique_ptr<Mtx> d_coarse_vector;
     std::unique_ptr<Mtx> d_fine_vector;
     std::unique_ptr<Diag> d_weight_diag;
     std::unique_ptr<Csr> d_weight_csr;
+    std::shared_ptr<Csr> d_system_mtx;
 
     gko::size_type n;
 };
@@ -329,4 +337,26 @@ TEST_F(AmgxPgm, GenerateMtxIsEquivalentToRef)
 }
 
 
+TEST_F(AmgxPgm, GenerateMgLevelIsEquivalentToRef)
+{
+    initialize_data();
+    auto mg_level_factory = gko::multigrid::AmgxPgm<double, int>::build()
+                                .with_deterministic(true)
+                                .on(ref);
+    auto d_mg_level_factory = gko::multigrid::AmgxPgm<double, int>::build()
+                                  .with_deterministic(true)
+                                  .on(cuda);
+
+    auto mg_level = mg_level_factory->generate(system_mtx);
+    auto d_mg_level = d_mg_level_factory->generate(d_system_mtx);
+
+    GKO_ASSERT_MTX_NEAR(gko::as<Csr>(d_mg_level->get_restrict_op()),
+                        gko::as<Csr>(mg_level->get_restrict_op()), 1e-14);
+    GKO_ASSERT_MTX_NEAR(gko::as<Csr>(d_mg_level->get_coarse_op()),
+                        gko::as<Csr>(mg_level->get_coarse_op()), 1e-14);
+    GKO_ASSERT_MTX_NEAR(gko::as<Csr>(d_mg_level->get_prolong_op()),
+                        gko::as<Csr>(mg_level->get_prolong_op()), 1e-14);
+}
+
+
 }  // namespace
diff --git a/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp b/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
index 933fc6fda7d..32817e5e834 100644
--- a/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
+++ b/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
@@ -101,11 +101,11 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void amgx_pgm_generate(std::shared_ptr<const DpcppExecutor> exec,
-                       const matrix::Csr<ValueType, IndexType> *source,
-                       const Array<IndexType> &agg,
-                       matrix::Csr<ValueType, IndexType> *coarse)
-    GKO_NOT_IMPLEMENTED;
+void amgx_pgm_generate(
+    std::shared_ptr<const DpcppExecutor> exec,
+    const matrix::Csr<ValueType, IndexType> *source,
+    const Array<IndexType> &agg, matrix::Csr<ValueType, IndexType> *coarse,
+    matrix::Csr<ValueType, IndexType> *temp) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
 
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index 41fc38854a8..2199f53e5c3 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -156,19 +156,27 @@ void assign_to_exist_agg(std::shared_ptr<const HipExecutor> exec,
                          Array<IndexType> &agg,
                          Array<IndexType> &intermediate_agg)
 {
-    auto agg_val = (intermediate_agg.get_num_elems() > 0)
-                       ? intermediate_agg.get_data()
-                       : agg.get_data();
     const auto num = agg.get_num_elems();
     const dim3 grid(ceildiv(num, default_block_size));
-    hipLaunchKernelGGL(
-        kernel::assign_to_exist_agg_kernel, dim3(grid),
-        dim3(default_block_size), 0, 0, num, weight_mtx->get_const_row_ptrs(),
-        weight_mtx->get_const_col_idxs(), weight_mtx->get_const_values(),
-        diag->get_const_values(), agg.get_const_data(), agg_val);
+
     if (intermediate_agg.get_num_elems() > 0) {
+        // determinstic kernel
+        hipLaunchKernelGGL(
+            kernel::assign_to_exist_agg_kernel, dim3(grid),
+            dim3(default_block_size), 0, 0, num,
+            weight_mtx->get_const_row_ptrs(), weight_mtx->get_const_col_idxs(),
+            weight_mtx->get_const_values(), diag->get_const_values(),
+            agg.get_const_data(), intermediate_agg.get_data());
         // Copy the intermediate_agg to agg
         agg = intermediate_agg;
+    } else {
+        // undeterminstic kernel
+        hipLaunchKernelGGL(kernel::assign_to_exist_agg_kernel, dim3(grid),
+                           dim3(default_block_size), 0, 0, num,
+                           weight_mtx->get_const_row_ptrs(),
+                           weight_mtx->get_const_col_idxs(),
+                           weight_mtx->get_const_values(),
+                           diag->get_const_values(), agg.get_data());
     }
 }
 
diff --git a/hip/test/multigrid/CMakeLists.txt b/hip/test/multigrid/CMakeLists.txt
index 1c8534e5337..481c2cc1bf2 100644
--- a/hip/test/multigrid/CMakeLists.txt
+++ b/hip/test/multigrid/CMakeLists.txt
@@ -1 +1 @@
-ginkgo_create_hip_test(amgx_pgm_kernels)
+ginkgo_create_hip_test_special_linkage(amgx_pgm_kernels)
diff --git a/hip/test/multigrid/amgx_pgm_kernels.hip.cpp b/hip/test/multigrid/amgx_pgm_kernels.cpp
similarity index 88%
rename from hip/test/multigrid/amgx_pgm_kernels.hip.cpp
rename to hip/test/multigrid/amgx_pgm_kernels.cpp
index ad1e8f040e7..71acf6ed06a 100644
--- a/hip/test/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/test/multigrid/amgx_pgm_kernels.cpp
@@ -131,6 +131,10 @@ class AmgxPgm : public ::testing::Test {
         weight_csr = Csr::create(ref);
         weight->convert_to(weight_csr.get());
         weight_diag = weight_csr->extract_diagonal();
+        auto system_dense = gen_mtx(m, m);
+        make_spd(system_dense.get());
+        system_mtx = Csr::create(ref);
+        system_dense->convert_to(system_mtx.get());
 
         d_agg.set_executor(hip);
         d_unfinished_agg.set_executor(hip);
@@ -139,6 +143,7 @@ class AmgxPgm : public ::testing::Test {
         d_fine_vector = Mtx::create(hip);
         d_weight_csr = Csr::create(hip);
         d_weight_diag = Diag::create(hip);
+        d_system_mtx = Csr::create(hip);
         d_agg = agg;
         d_unfinished_agg = unfinished_agg;
         d_strongest_neighbor = strongest_neighbor;
@@ -146,6 +151,7 @@ class AmgxPgm : public ::testing::Test {
         d_fine_vector->copy_from(fine_vector.get());
         d_weight_csr->copy_from(weight_csr.get());
         d_weight_diag->copy_from(weight_diag.get());
+        d_system_mtx->copy_from(system_mtx.get());
     }
 
     void make_symetric(Mtx *mtx)
@@ -209,11 +215,13 @@ class AmgxPgm : public ::testing::Test {
     std::unique_ptr<Mtx> fine_vector;
     std::unique_ptr<Diag> weight_diag;
     std::unique_ptr<Csr> weight_csr;
+    std::shared_ptr<Csr> system_mtx;
 
     std::unique_ptr<Mtx> d_coarse_vector;
     std::unique_ptr<Mtx> d_fine_vector;
     std::unique_ptr<Diag> d_weight_diag;
     std::unique_ptr<Csr> d_weight_csr;
+    std::shared_ptr<Csr> d_system_mtx;
 
     gko::size_type n;
 };
@@ -329,4 +337,26 @@ TEST_F(AmgxPgm, GenerateMtxIsEquivalentToRef)
 }
 
 
+TEST_F(AmgxPgm, GenerateMgLevelIsEquivalentToRef)
+{
+    initialize_data();
+    auto mg_level_factory = gko::multigrid::AmgxPgm<double, int>::build()
+                                .with_deterministic(true)
+                                .on(ref);
+    auto d_mg_level_factory = gko::multigrid::AmgxPgm<double, int>::build()
+                                  .with_deterministic(true)
+                                  .on(hip);
+
+    auto mg_level = mg_level_factory->generate(system_mtx);
+    auto d_mg_level = d_mg_level_factory->generate(d_system_mtx);
+
+    GKO_ASSERT_MTX_NEAR(gko::as<Csr>(d_mg_level->get_restrict_op()),
+                        gko::as<Csr>(mg_level->get_restrict_op()), 1e-14);
+    GKO_ASSERT_MTX_NEAR(gko::as<Csr>(d_mg_level->get_coarse_op()),
+                        gko::as<Csr>(mg_level->get_coarse_op()), 1e-14);
+    GKO_ASSERT_MTX_NEAR(gko::as<Csr>(d_mg_level->get_prolong_op()),
+                        gko::as<Csr>(mg_level->get_prolong_op()), 1e-14);
+}
+
+
 }  // namespace
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index 847e859529e..09918310823 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -47,7 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/diagonal.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm_reduction.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
 #include <ginkgo/core/stop/time.hpp>
 
 

From c15ad30b2d9277f1b111e7044a354f1b32920e25 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 26 Mar 2021 03:05:41 +0800
Subject: [PATCH 0670/1008] review update, fix omp issue, collect dup. helpers

---
 common/multigrid/amgx_pgm_kernels.hpp.inc | 22 +-----
 core/test/utils.hpp                       |  3 +
 core/test/utils/array_generator.hpp       | 69 +++++++++++++++++
 core/test/utils/matrix_generator.hpp      | 23 +-----
 core/test/utils/matrix_utils.hpp          | 84 ++++++++++++++++++++
 core/test/utils/value_generator.hpp       | 69 +++++++++++++++++
 cuda/test/multigrid/amgx_pgm_kernels.cpp  | 67 ++--------------
 cuda/test/solver/bicg_kernels.cpp         | 29 +------
 cuda/test/solver/bicgstab_kernels.cpp     | 14 +---
 cuda/test/solver/cg_kernels.cpp           | 29 +------
 cuda/test/solver/cgs_kernels.cpp          | 14 +---
 cuda/test/solver/fcg_kernels.cpp          | 29 +------
 cuda/test/solver/idr_kernels.cpp          | 12 ---
 hip/test/multigrid/CMakeLists.txt         |  2 +-
 hip/test/multigrid/amgx_pgm_kernels.cpp   | 66 ++--------------
 hip/test/solver/bicg_kernels.cpp          | 29 +------
 hip/test/solver/bicgstab_kernels.cpp      | 14 +---
 hip/test/solver/cg_kernels.cpp            | 29 +------
 hip/test/solver/cgs_kernels.cpp           | 14 +---
 hip/test/solver/fcg_kernels.cpp           | 29 +------
 hip/test/solver/idr_kernels.cpp           | 12 ---
 omp/multigrid/amgx_pgm_kernels.cpp        | 13 ++--
 omp/test/multigrid/amgx_pgm_kernels.cpp   | 94 +++++++++--------------
 omp/test/solver/bicg_kernels.cpp          | 29 +------
 omp/test/solver/bicgstab_kernels.cpp      | 14 +---
 omp/test/solver/cg_kernels.cpp            | 29 +------
 omp/test/solver/cgs_kernels.cpp           | 14 +---
 omp/test/solver/fcg_kernels.cpp           | 29 +------
 omp/test/solver/gmres_kernels.cpp         | 12 ---
 omp/test/solver/idr_kernels.cpp           | 12 ---
 reference/multigrid/amgx_pgm_kernels.cpp  |  6 +-
 31 files changed, 306 insertions(+), 605 deletions(-)
 create mode 100644 core/test/utils/array_generator.hpp
 create mode 100644 core/test/utils/matrix_utils.hpp
 create mode 100644 core/test/utils/value_generator.hpp

diff --git a/common/multigrid/amgx_pgm_kernels.hpp.inc b/common/multigrid/amgx_pgm_kernels.hpp.inc
index eda2854763e..9c2d434f508 100644
--- a/common/multigrid/amgx_pgm_kernels.hpp.inc
+++ b/common/multigrid/amgx_pgm_kernels.hpp.inc
@@ -33,20 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace kernel {
 
 
-template <typename IndexType>
-__global__ __launch_bounds__(default_block_size) void replace_kernel(
-    size_type size, const IndexType *__restrict__ source,
-    IndexType *__restrict__ result)
-{
-    auto tidx = thread::get_thread_id_flat();
-    if (tidx >= size) {
-        return;
-    }
-
-    result[tidx] = source[tidx] == -1;
-}
-
-
 template <typename IndexType>
 __global__ __launch_bounds__(default_block_size) void match_edge_kernel(
     size_type num, const IndexType *__restrict__ strongest_neighbor_vals,
@@ -60,11 +46,11 @@ __global__ __launch_bounds__(default_block_size) void match_edge_kernel(
         return;
     }
     auto neighbor = strongest_neighbor_vals[tidx];
-    if (neighbor != -1 && strongest_neighbor_vals[neighbor] == tidx) {
+    if (neighbor != -1 && strongest_neighbor_vals[neighbor] == tidx &&
+        tidx < neighbor) {
         // Use the smaller index as agg point
-        auto group = min(tidx, neighbor);
-        agg_vals[tidx] = group;
-        agg_vals[neighbor] = group;
+        agg_vals[tidx] = tidx;
+        agg_vals[neighbor] = tidx;
     }
 }
 
diff --git a/core/test/utils.hpp b/core/test/utils.hpp
index 6cc2ad102c5..05bf216c8bb 100644
--- a/core/test/utils.hpp
+++ b/core/test/utils.hpp
@@ -48,8 +48,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/base/extended_float.hpp"
+#include "core/test/utils/array_generator.hpp"
 #include "core/test/utils/assertions.hpp"
 #include "core/test/utils/matrix_generator.hpp"
+#include "core/test/utils/matrix_utils.hpp"
+#include "core/test/utils/value_generator.hpp"
 
 
 namespace gko {
diff --git a/core/test/utils/array_generator.hpp b/core/test/utils/array_generator.hpp
new file mode 100644
index 00000000000..1ad52c18e26
--- /dev/null
+++ b/core/test/utils/array_generator.hpp
@@ -0,0 +1,69 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_TEST_UTILS_ARRAY_GENERATOR_HPP_
+#define GKO_CORE_TEST_UTILS_ARRAY_GENERATOR_HPP_
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/utils.hpp>
+
+
+#include "core/test/utils/value_generator.hpp"
+
+
+namespace gko {
+namespace test {
+
+
+template <typename ValueType, typename ValueDistribution, typename Engine>
+Array<ValueType> generate_random_array(size_type num,
+                                       ValueDistribution &&value_dist,
+                                       Engine &&engine,
+                                       std::shared_ptr<const Executor> exec)
+{
+    Array<ValueType> array(exec->get_master(), num);
+    auto val = array.get_data();
+    for (int i = 0; i < num; i++) {
+        val[i] = detail::get_rand_value<ValueType>(value_dist, engine);
+    }
+    array.set_executor(exec);
+    return array;
+}
+
+
+}  // namespace test
+}  // namespace gko
+
+
+#endif  // GKO_CORE_TEST_UTILS_ARRAY_GENERATOR_HPP_
diff --git a/core/test/utils/matrix_generator.hpp b/core/test/utils/matrix_generator.hpp
index c3f9b2fb50d..04f98126c41 100644
--- a/core/test/utils/matrix_generator.hpp
+++ b/core/test/utils/matrix_generator.hpp
@@ -46,28 +46,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
-namespace gko {
-namespace test {
-namespace detail {
-
-
-template <typename ValueType, typename Distribution, typename Generator>
-typename std::enable_if<!is_complex_s<ValueType>::value, ValueType>::type
-get_rand_value(Distribution &&dist, Generator &&gen)
-{
-    return dist(gen);
-}
+#include "core/test/utils/value_generator.hpp"
 
 
-template <typename ValueType, typename Distribution, typename Generator>
-typename std::enable_if<is_complex_s<ValueType>::value, ValueType>::type
-get_rand_value(Distribution &&dist, Generator &&gen)
-{
-    return ValueType(dist(gen), dist(gen));
-}
-
-
-}  // namespace detail
+namespace gko {
+namespace test {
 
 
 /**
diff --git a/core/test/utils/matrix_utils.hpp b/core/test/utils/matrix_utils.hpp
new file mode 100644
index 00000000000..c8b269d493f
--- /dev/null
+++ b/core/test/utils/matrix_utils.hpp
@@ -0,0 +1,84 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_TEST_UTILS_MATRIX_UTILS_HPP_
+#define GKO_CORE_TEST_UTILS_MATRIX_UTILS_HPP_
+
+
+#include <ginkgo/core/matrix/dense.hpp>
+
+#include "core/test/utils/value_generator.hpp"
+
+
+namespace gko {
+namespace test {
+
+
+template <typename ValueType>
+void make_symmetric(matrix::Dense<ValueType> *mtx)
+{
+    assert(mtx->get_executor() == mtx->get_executor()->get_master());
+    for (size_type i = 0; i < mtx->get_size()[0]; ++i) {
+        for (size_type j = i + 1; j < mtx->get_size()[1]; ++j) {
+            mtx->at(i, j) = mtx->at(j, i);
+        }
+    }
+}
+
+
+template <typename ValueType>
+void make_diag_dominant(matrix::Dense<ValueType> *mtx)
+{
+    assert(mtx->get_executor() == mtx->get_executor()->get_master());
+    using std::abs;
+    for (int i = 0; i < mtx->get_size()[0]; ++i) {
+        auto sum = gko::zero<ValueType>();
+        for (int j = 0; j < mtx->get_size()[1]; ++j) {
+            sum += abs(mtx->at(i, j));
+        }
+        mtx->at(i, i) = sum;
+    }
+}
+
+
+template <typename ValueType>
+void make_spd(matrix::Dense<ValueType> *mtx)
+{
+    make_symmetric(mtx);
+    make_diag_dominant(mtx);
+}
+
+
+}  // namespace test
+}  // namespace gko
+
+#endif  // GKO_CORE_TEST_UTILS_MATRIX_UTILS_HPP_
diff --git a/core/test/utils/value_generator.hpp b/core/test/utils/value_generator.hpp
new file mode 100644
index 00000000000..8b82ea63d0a
--- /dev/null
+++ b/core/test/utils/value_generator.hpp
@@ -0,0 +1,69 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_TEST_UTILS_VALUE_GENERATOR_HPP_
+#define GKO_CORE_TEST_UTILS_VALUE_GENERATOR_HPP_
+
+
+#include <random>
+#include <type_traits>
+
+
+#include <ginkgo/core/base/math.hpp>
+
+
+namespace gko {
+namespace test {
+namespace detail {
+
+
+template <typename ValueType, typename Distribution, typename Generator>
+typename std::enable_if<!is_complex_s<ValueType>::value, ValueType>::type
+get_rand_value(Distribution &&dist, Generator &&gen)
+{
+    return dist(gen);
+}
+
+
+template <typename ValueType, typename Distribution, typename Generator>
+typename std::enable_if<is_complex_s<ValueType>::value, ValueType>::type
+get_rand_value(Distribution &&dist, Generator &&gen)
+{
+    return ValueType(dist(gen), dist(gen));
+}
+
+
+}  // namespace detail
+}  // namespace test
+}  // namespace gko
+
+#endif  // GKO_CORE_TEST_UTILS_VALUE_GENERATOR_HPP_
diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
index 10d2b97a8bd..2814f9177c0 100644
--- a/cuda/test/multigrid/amgx_pgm_kernels.cpp
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -58,24 +58,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
-template <typename Array, typename ValueDistribution, typename Engine>
-Array generate_random_array(gko::size_type num, ValueDistribution &&value_dist,
-                            Engine &&engine,
-                            std::shared_ptr<const gko::Executor> exec)
-{
-    using value_type = typename Array::value_type;
-    Array array_host(exec->get_master(), num);
-    auto val = array_host.get_data();
-    for (int i = 0; i < num; i++) {
-        val[i] =
-            gko::test::detail::get_rand_value<value_type>(value_dist, engine);
-    }
-    Array array(exec);
-    array = array_host;
-    return array;
-}
-
-
 class AmgxPgm : public ::testing::Test {
 protected:
     using value_type = gko::default_precision;
@@ -83,6 +65,7 @@ class AmgxPgm : public ::testing::Test {
     using Mtx = gko::matrix::Dense<>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
     using Diag = gko::matrix::Diagonal<value_type>;
+
     AmgxPgm() : rand_engine(30) {}
 
     void SetUp()
@@ -110,7 +93,7 @@ class AmgxPgm : public ::testing::Test {
     gko::Array<index_type> gen_array(gko::size_type num, index_type min_val,
                                      index_type max_val)
     {
-        return generate_random_array<gko::Array<index_type>>(
+        return gko::test::generate_random_array<index_type>(
             num, std::uniform_int_distribution<>(min_val, max_val), rand_engine,
             ref);
     }
@@ -132,7 +115,7 @@ class AmgxPgm : public ::testing::Test {
         weight->convert_to(weight_csr.get());
         weight_diag = weight_csr->extract_diagonal();
         auto system_dense = gen_mtx(m, m);
-        make_spd(system_dense.get());
+        gko::test::make_spd(system_dense.get());
         system_mtx = Csr::create(ref);
         system_dense->convert_to(system_mtx.get());
 
@@ -154,48 +137,12 @@ class AmgxPgm : public ::testing::Test {
         d_system_mtx->copy_from(system_mtx.get());
     }
 
-    void make_symetric(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = mtx->at(j, i);
-            }
-        }
-    }
-
-    // only for real value
-    void make_absoulte(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = abs(mtx->at(i, j));
-            }
-        }
-    }
-
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
-    void make_spd(Mtx *mtx)
-    {
-        make_symetric(mtx);
-        make_diag_dominant(mtx);
-    }
-
     void make_weight(Mtx *mtx)
     {
-        make_symetric(mtx);
-        make_absoulte(mtx);
-        make_diag_dominant(mtx);
+        gko::test::make_symmetric(mtx);
+        // only works for realvalue cases
+        mtx->compute_absolute_inplace();
+        gko::test::make_diag_dominant(mtx);
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
diff --git a/cuda/test/solver/bicg_kernels.cpp b/cuda/test/solver/bicg_kernels.cpp
index a825ae5aa9b..d82cc0a28a0 100644
--- a/cuda/test/solver/bicg_kernels.cpp
+++ b/cuda/test/solver/bicg_kernels.cpp
@@ -152,33 +152,6 @@ class Bicg : public ::testing::Test {
         *d_stop_status = *stop_status;
     }
 
-    void make_symetric(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = mtx->at(j, i);
-            }
-        }
-    }
-
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
-    void make_spd(Mtx *mtx)
-    {
-        make_symetric(mtx);
-        make_diag_dominant(mtx);
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::CudaExecutor> cuda;
 
@@ -285,7 +258,7 @@ TEST_F(Bicg, CudaBicgStep2IsEquivalentToRef)
 TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    make_spd(mtx.get());
+    gko::test::make_spd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(cuda);
diff --git a/cuda/test/solver/bicgstab_kernels.cpp b/cuda/test/solver/bicgstab_kernels.cpp
index 7808226870c..b1e0ca85d1b 100644
--- a/cuda/test/solver/bicgstab_kernels.cpp
+++ b/cuda/test/solver/bicgstab_kernels.cpp
@@ -69,7 +69,7 @@ class Bicgstab : public ::testing::Test {
         cuda = gko::CudaExecutor::create(0, ref);
 
         mtx = gen_mtx(123, 123);
-        make_diag_dominant(mtx.get());
+        gko::test::make_diag_dominant(mtx.get());
         d_mtx = Mtx::create(cuda);
         d_mtx->copy_from(mtx.get());
 
@@ -171,18 +171,6 @@ class Bicgstab : public ::testing::Test {
             *stop_status;  // copy_from is not a public member function of Array
     }
 
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::CudaExecutor> cuda;
 
diff --git a/cuda/test/solver/cg_kernels.cpp b/cuda/test/solver/cg_kernels.cpp
index d8a98a83805..8333fc7d0a8 100644
--- a/cuda/test/solver/cg_kernels.cpp
+++ b/cuda/test/solver/cg_kernels.cpp
@@ -123,33 +123,6 @@ class Cg : public ::testing::Test {
         *d_stop_status = *stop_status;
     }
 
-    void make_symetric(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = mtx->at(j, i);
-            }
-        }
-    }
-
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
-    void make_spd(Mtx *mtx)
-    {
-        make_symetric(mtx);
-        make_diag_dominant(mtx);
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::CudaExecutor> cuda;
 
@@ -234,7 +207,7 @@ TEST_F(Cg, CudaCgStep2IsEquivalentToRef)
 TEST_F(Cg, ApplyIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    make_spd(mtx.get());
+    gko::test::make_spd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(cuda);
diff --git a/cuda/test/solver/cgs_kernels.cpp b/cuda/test/solver/cgs_kernels.cpp
index 543f580cf53..f8cb89d55bb 100644
--- a/cuda/test/solver/cgs_kernels.cpp
+++ b/cuda/test/solver/cgs_kernels.cpp
@@ -68,7 +68,7 @@ class Cgs : public ::testing::Test {
         cuda = gko::CudaExecutor::create(0, ref);
 
         mtx = gen_mtx(123, 123);
-        make_diag_dominant(mtx.get());
+        gko::test::make_diag_dominant(mtx.get());
         d_mtx = Mtx::create(cuda);
         d_mtx->copy_from(mtx.get());
         cuda_cgs_factory =
@@ -166,18 +166,6 @@ class Cgs : public ::testing::Test {
         *d_stop_status = *stop_status;
     }
 
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::CudaExecutor> cuda;
 
diff --git a/cuda/test/solver/fcg_kernels.cpp b/cuda/test/solver/fcg_kernels.cpp
index 85b0b3b7017..b2832945666 100644
--- a/cuda/test/solver/fcg_kernels.cpp
+++ b/cuda/test/solver/fcg_kernels.cpp
@@ -131,33 +131,6 @@ class Fcg : public ::testing::Test {
         *d_stop_status = *stop_status;
     }
 
-    void make_symetric(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = mtx->at(j, i);
-            }
-        }
-    }
-
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
-    void make_spd(Mtx *mtx)
-    {
-        make_symetric(mtx);
-        make_diag_dominant(mtx);
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::CudaExecutor> cuda;
 
@@ -247,7 +220,7 @@ TEST_F(Fcg, CudaFcgStep2IsEquivalentToRef)
 TEST_F(Fcg, ApplyIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    make_spd(mtx.get());
+    gko::test::make_spd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(cuda);
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index 04ea559b6cc..9aa925187a4 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -160,18 +160,6 @@ class Idr : public ::testing::Test {
             *stop_status;  // copy_from is not a public member function of Array
     }
 
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::CudaExecutor> cuda;
 
diff --git a/hip/test/multigrid/CMakeLists.txt b/hip/test/multigrid/CMakeLists.txt
index 481c2cc1bf2..8fe8bbeba48 100644
--- a/hip/test/multigrid/CMakeLists.txt
+++ b/hip/test/multigrid/CMakeLists.txt
@@ -1 +1 @@
-ginkgo_create_hip_test_special_linkage(amgx_pgm_kernels)
+ginkgo_create_test(amgx_pgm_kernels)
diff --git a/hip/test/multigrid/amgx_pgm_kernels.cpp b/hip/test/multigrid/amgx_pgm_kernels.cpp
index 71acf6ed06a..39546df6cff 100644
--- a/hip/test/multigrid/amgx_pgm_kernels.cpp
+++ b/hip/test/multigrid/amgx_pgm_kernels.cpp
@@ -58,24 +58,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
-template <typename Array, typename ValueDistribution, typename Engine>
-Array generate_random_array(gko::size_type num, ValueDistribution &&value_dist,
-                            Engine &&engine,
-                            std::shared_ptr<const gko::Executor> exec)
-{
-    using value_type = typename Array::value_type;
-    Array array_host(exec->get_master(), num);
-    auto val = array_host.get_data();
-    for (int i = 0; i < num; i++) {
-        val[i] =
-            gko::test::detail::get_rand_value<value_type>(value_dist, engine);
-    }
-    Array array(exec);
-    array = array_host;
-    return array;
-}
-
-
 class AmgxPgm : public ::testing::Test {
 protected:
     using value_type = gko::default_precision;
@@ -110,7 +92,7 @@ class AmgxPgm : public ::testing::Test {
     gko::Array<index_type> gen_array(gko::size_type num, index_type min_val,
                                      index_type max_val)
     {
-        return generate_random_array<gko::Array<index_type>>(
+        return gko::test::generate_random_array<index_type>(
             num, std::uniform_int_distribution<>(min_val, max_val), rand_engine,
             ref);
     }
@@ -132,7 +114,7 @@ class AmgxPgm : public ::testing::Test {
         weight->convert_to(weight_csr.get());
         weight_diag = weight_csr->extract_diagonal();
         auto system_dense = gen_mtx(m, m);
-        make_spd(system_dense.get());
+        gko::test::make_spd(system_dense.get());
         system_mtx = Csr::create(ref);
         system_dense->convert_to(system_mtx.get());
 
@@ -154,48 +136,12 @@ class AmgxPgm : public ::testing::Test {
         d_system_mtx->copy_from(system_mtx.get());
     }
 
-    void make_symetric(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = mtx->at(j, i);
-            }
-        }
-    }
-
-    // only for real value
-    void make_absoulte(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = abs(mtx->at(i, j));
-            }
-        }
-    }
-
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
-    void make_spd(Mtx *mtx)
-    {
-        make_symetric(mtx);
-        make_diag_dominant(mtx);
-    }
-
     void make_weight(Mtx *mtx)
     {
-        make_symetric(mtx);
-        make_absoulte(mtx);
-        make_diag_dominant(mtx);
+        gko::test::make_symmetric(mtx);
+        // only works for realvalue cases
+        mtx->compute_absolute_inplace();
+        gko::test::make_diag_dominant(mtx);
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
diff --git a/hip/test/solver/bicg_kernels.cpp b/hip/test/solver/bicg_kernels.cpp
index 52073945050..c29bd74374c 100644
--- a/hip/test/solver/bicg_kernels.cpp
+++ b/hip/test/solver/bicg_kernels.cpp
@@ -152,33 +152,6 @@ class Bicg : public ::testing::Test {
         *d_stop_status = *stop_status;
     }
 
-    void make_symetric(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = mtx->at(j, i);
-            }
-        }
-    }
-
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
-    void make_spd(Mtx *mtx)
-    {
-        make_symetric(mtx);
-        make_diag_dominant(mtx);
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::HipExecutor> hip;
 
@@ -285,7 +258,7 @@ TEST_F(Bicg, HipBicgStep2IsEquivalentToRef)
 TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    make_spd(mtx.get());
+    gko::test::make_spd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(hip);
diff --git a/hip/test/solver/bicgstab_kernels.cpp b/hip/test/solver/bicgstab_kernels.cpp
index bcf4191a426..5a995656306 100644
--- a/hip/test/solver/bicgstab_kernels.cpp
+++ b/hip/test/solver/bicgstab_kernels.cpp
@@ -69,7 +69,7 @@ class Bicgstab : public ::testing::Test {
         hip = gko::HipExecutor::create(0, ref);
 
         mtx = gen_mtx(123, 123);
-        make_diag_dominant(mtx.get());
+        gko::test::make_diag_dominant(mtx.get());
         d_mtx = Mtx::create(hip);
         d_mtx->copy_from(mtx.get());
 
@@ -171,18 +171,6 @@ class Bicgstab : public ::testing::Test {
             *stop_status;  // copy_from is not a public member function of Array
     }
 
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::HipExecutor> hip;
 
diff --git a/hip/test/solver/cg_kernels.cpp b/hip/test/solver/cg_kernels.cpp
index a46d698cc22..cd4000171c0 100644
--- a/hip/test/solver/cg_kernels.cpp
+++ b/hip/test/solver/cg_kernels.cpp
@@ -123,33 +123,6 @@ class Cg : public ::testing::Test {
         *d_stop_status = *stop_status;
     }
 
-    void make_symetric(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = mtx->at(j, i);
-            }
-        }
-    }
-
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
-    void make_spd(Mtx *mtx)
-    {
-        make_symetric(mtx);
-        make_diag_dominant(mtx);
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::HipExecutor> hip;
 
@@ -234,7 +207,7 @@ TEST_F(Cg, HipCgStep2IsEquivalentToRef)
 TEST_F(Cg, ApplyIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    make_spd(mtx.get());
+    gko::test::make_spd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(hip);
diff --git a/hip/test/solver/cgs_kernels.cpp b/hip/test/solver/cgs_kernels.cpp
index 861f9498780..2277d4417ba 100644
--- a/hip/test/solver/cgs_kernels.cpp
+++ b/hip/test/solver/cgs_kernels.cpp
@@ -68,7 +68,7 @@ class Cgs : public ::testing::Test {
         hip = gko::HipExecutor::create(0, ref);
 
         mtx = gen_mtx(123, 123);
-        make_diag_dominant(mtx.get());
+        gko::test::make_diag_dominant(mtx.get());
         d_mtx = Mtx::create(hip);
         d_mtx->copy_from(mtx.get());
         hip_cgs_factory =
@@ -166,18 +166,6 @@ class Cgs : public ::testing::Test {
         *d_stop_status = *stop_status;
     }
 
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::HipExecutor> hip;
 
diff --git a/hip/test/solver/fcg_kernels.cpp b/hip/test/solver/fcg_kernels.cpp
index 663c811227c..ca86775ee94 100644
--- a/hip/test/solver/fcg_kernels.cpp
+++ b/hip/test/solver/fcg_kernels.cpp
@@ -131,33 +131,6 @@ class Fcg : public ::testing::Test {
         *d_stop_status = *stop_status;
     }
 
-    void make_symetric(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = mtx->at(j, i);
-            }
-        }
-    }
-
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
-    void make_spd(Mtx *mtx)
-    {
-        make_symetric(mtx);
-        make_diag_dominant(mtx);
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::HipExecutor> hip;
 
@@ -247,7 +220,7 @@ TEST_F(Fcg, HipFcgStep2IsEquivalentToRef)
 TEST_F(Fcg, ApplyIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    make_spd(mtx.get());
+    gko::test::make_spd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(hip);
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index c59ed90c9a7..de7f5a14125 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -160,18 +160,6 @@ class Idr : public ::testing::Test {
             *stop_status;  // copy_from is not a public member function of Array
     }
 
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::HipExecutor> hip;
 
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index 9c4efefebe6..3ce1b087fd8 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -75,12 +75,12 @@ void match_edge(std::shared_ptr<const OmpExecutor> exec,
 #pragma omp parallel for
     for (size_type i = 0; i < agg.get_num_elems(); i++) {
         if (agg_vals[i] == -1) {
-            size_type neighbor = strongest_neighbor_vals[i];
-            if (neighbor != -1 && strongest_neighbor_vals[neighbor] == i) {
+            auto neighbor = strongest_neighbor_vals[i];
+            if (neighbor != -1 && strongest_neighbor_vals[neighbor] == i &&
+                i < neighbor) {
                 // Use the smaller index as agg point
-                auto group = min(i, neighbor);
-                agg_vals[i] = group;
-                agg_vals[neighbor] = group;
+                agg_vals[i] = i;
+                agg_vals[neighbor] = i;
             }
         }
     }
@@ -152,7 +152,7 @@ void find_strongest_neighbor(
                     continue;
                 }
                 auto weight =
-                    vals[idx] / max(abs(diag_vals[col]), abs(diag_vals[col]));
+                    vals[idx] / max(abs(diag_vals[row]), abs(diag_vals[col]));
                 if (agg.get_const_data()[col] == -1 &&
                     (weight > max_weight_unagg ||
                      (weight == max_weight_unagg && col > strongest_unagg))) {
@@ -200,7 +200,6 @@ void assign_to_exist_agg(std::shared_ptr<const OmpExecutor> exec,
                        ? intermediate_agg.get_data()
                        : agg.get_data();
     const auto diag_vals = diag->get_const_values();
-#pragma omp parallel for
     for (IndexType row = 0; row < agg.get_num_elems(); row++) {
         if (agg_const_val[row] != -1) {
             continue;
diff --git a/omp/test/multigrid/amgx_pgm_kernels.cpp b/omp/test/multigrid/amgx_pgm_kernels.cpp
index 9a27ef3f4fc..e0b063de0bd 100644
--- a/omp/test/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/test/multigrid/amgx_pgm_kernels.cpp
@@ -57,24 +57,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
-template <typename Array, typename ValueDistribution, typename Engine>
-Array generate_random_array(gko::size_type num, ValueDistribution &&value_dist,
-                            Engine &&engine,
-                            std::shared_ptr<const gko::Executor> exec)
-{
-    using value_type = typename Array::value_type;
-    Array array_host(exec->get_master(), num);
-    auto val = array_host.get_data();
-    for (int i = 0; i < num; i++) {
-        val[i] =
-            gko::test::detail::get_rand_value<value_type>(value_dist, engine);
-    }
-    Array array(exec);
-    array = array_host;
-    return array;
-}
-
-
 class AmgxPgm : public ::testing::Test {
 protected:
     using value_type = gko::default_precision;
@@ -101,7 +83,7 @@ class AmgxPgm : public ::testing::Test {
     gko::Array<index_type> gen_array(gko::size_type num, index_type min_val,
                                      index_type max_val)
     {
-        return generate_random_array<gko::Array<index_type>>(
+        return gko::test::generate_random_array<index_type>(
             num, std::uniform_int_distribution<>(min_val, max_val), rand_engine,
             ref);
     }
@@ -122,6 +104,10 @@ class AmgxPgm : public ::testing::Test {
         weight_csr = Csr::create(ref);
         weight->convert_to(weight_csr.get());
         weight_diag = weight_csr->extract_diagonal();
+        auto system_dense = gen_mtx(m, m);
+        gko::test::make_spd(system_dense.get());
+        system_mtx = Csr::create(ref);
+        system_dense->convert_to(system_mtx.get());
 
         d_agg.set_executor(omp);
         d_unfinished_agg.set_executor(omp);
@@ -130,6 +116,7 @@ class AmgxPgm : public ::testing::Test {
         d_fine_vector = Mtx::create(omp);
         d_weight_csr = Csr::create(omp);
         d_weight_diag = Diag::create(omp);
+        d_system_mtx = Csr::create(omp);
         d_agg = agg;
         d_unfinished_agg = unfinished_agg;
         d_strongest_neighbor = strongest_neighbor;
@@ -137,50 +124,15 @@ class AmgxPgm : public ::testing::Test {
         d_fine_vector->copy_from(fine_vector.get());
         d_weight_csr->copy_from(weight_csr.get());
         d_weight_diag->copy_from(weight_diag.get());
-    }
-
-    void make_symetric(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = mtx->at(j, i);
-            }
-        }
-    }
-
-    // only for real value
-    void make_absoulte(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = abs(mtx->at(i, j));
-            }
-        }
-    }
-
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
-    void make_spd(Mtx *mtx)
-    {
-        make_symetric(mtx);
-        make_diag_dominant(mtx);
+        d_system_mtx->copy_from(system_mtx.get());
     }
 
     void make_weight(Mtx *mtx)
     {
-        make_symetric(mtx);
-        make_absoulte(mtx);
-        make_diag_dominant(mtx);
+        gko::test::make_symmetric(mtx);
+        // it is only works for realvalue case.
+        mtx->compute_absolute_inplace();
+        gko::test::make_diag_dominant(mtx);
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -200,11 +152,13 @@ class AmgxPgm : public ::testing::Test {
     std::unique_ptr<Mtx> fine_vector;
     std::unique_ptr<Diag> weight_diag;
     std::unique_ptr<Csr> weight_csr;
+    std::shared_ptr<Csr> system_mtx;
 
     std::unique_ptr<Mtx> d_coarse_vector;
     std::unique_ptr<Mtx> d_fine_vector;
     std::unique_ptr<Diag> d_weight_diag;
     std::unique_ptr<Csr> d_weight_csr;
+    std::shared_ptr<Csr> d_system_mtx;
 
     gko::size_type n;
 };
@@ -320,4 +274,26 @@ TEST_F(AmgxPgm, GenerateMtxIsEquivalentToRef)
 }
 
 
+TEST_F(AmgxPgm, GenerateMgLevelIsEquivalentToRef)
+{
+    initialize_data();
+    auto mg_level_factory = gko::multigrid::AmgxPgm<double, int>::build()
+                                .with_deterministic(true)
+                                .on(ref);
+    auto d_mg_level_factory = gko::multigrid::AmgxPgm<double, int>::build()
+                                  .with_deterministic(true)
+                                  .on(omp);
+
+    auto mg_level = mg_level_factory->generate(system_mtx);
+    auto d_mg_level = d_mg_level_factory->generate(d_system_mtx);
+
+    GKO_ASSERT_MTX_NEAR(gko::as<Csr>(d_mg_level->get_restrict_op()),
+                        gko::as<Csr>(mg_level->get_restrict_op()), 1e-14);
+    GKO_ASSERT_MTX_NEAR(gko::as<Csr>(d_mg_level->get_coarse_op()),
+                        gko::as<Csr>(mg_level->get_coarse_op()), 1e-14);
+    GKO_ASSERT_MTX_NEAR(gko::as<Csr>(d_mg_level->get_prolong_op()),
+                        gko::as<Csr>(mg_level->get_prolong_op()), 1e-14);
+}
+
+
 }  // namespace
diff --git a/omp/test/solver/bicg_kernels.cpp b/omp/test/solver/bicg_kernels.cpp
index fc51d1847ae..2018fa5f788 100644
--- a/omp/test/solver/bicg_kernels.cpp
+++ b/omp/test/solver/bicg_kernels.cpp
@@ -134,33 +134,6 @@ class Bicg : public ::testing::Test {
         *d_stop_status = *stop_status;
     }
 
-    void make_symetric(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = mtx->at(j, i);
-            }
-        }
-    }
-
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
-    void make_spd(Mtx *mtx)
-    {
-        make_symetric(mtx);
-        make_diag_dominant(mtx);
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::OmpExecutor> omp;
 
@@ -265,7 +238,7 @@ TEST_F(Bicg, OmpBicgStep2IsEquivalentToRef)
 TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    make_spd(mtx.get());
+    gko::test::make_spd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(omp);
diff --git a/omp/test/solver/bicgstab_kernels.cpp b/omp/test/solver/bicgstab_kernels.cpp
index 74085b6e7a7..9e51d841e64 100644
--- a/omp/test/solver/bicgstab_kernels.cpp
+++ b/omp/test/solver/bicgstab_kernels.cpp
@@ -68,7 +68,7 @@ class Bicgstab : public ::testing::Test {
         omp = gko::OmpExecutor::create();
 
         mtx = gen_mtx(123, 123);
-        make_diag_dominant(mtx.get());
+        gko::test::make_diag_dominant(mtx.get());
         d_mtx = Mtx::create(omp);
         d_mtx->copy_from(mtx.get());
         omp_bicgstab_factory =
@@ -170,18 +170,6 @@ class Bicgstab : public ::testing::Test {
             *stop_status;  // copy_from is not a public member function of Array
     }
 
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::OmpExecutor> omp;
 
diff --git a/omp/test/solver/cg_kernels.cpp b/omp/test/solver/cg_kernels.cpp
index e71fad019b5..1e86956602a 100644
--- a/omp/test/solver/cg_kernels.cpp
+++ b/omp/test/solver/cg_kernels.cpp
@@ -122,33 +122,6 @@ class Cg : public ::testing::Test {
         *d_stop_status = *stop_status;
     }
 
-    void make_symetric(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = mtx->at(j, i);
-            }
-        }
-    }
-
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
-    void make_spd(Mtx *mtx)
-    {
-        make_symetric(mtx);
-        make_diag_dominant(mtx);
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::OmpExecutor> omp;
 
@@ -233,7 +206,7 @@ TEST_F(Cg, OmpCgStep2IsEquivalentToRef)
 TEST_F(Cg, ApplyIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    make_spd(mtx.get());
+    gko::test::make_spd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(omp);
diff --git a/omp/test/solver/cgs_kernels.cpp b/omp/test/solver/cgs_kernels.cpp
index f05d0604919..d473f830bdc 100644
--- a/omp/test/solver/cgs_kernels.cpp
+++ b/omp/test/solver/cgs_kernels.cpp
@@ -67,7 +67,7 @@ class Cgs : public ::testing::Test {
         omp = gko::OmpExecutor::create();
 
         mtx = gen_mtx(123, 123);
-        make_diag_dominant(mtx.get());
+        gko::test::make_diag_dominant(mtx.get());
         d_mtx = Mtx::create(omp);
         d_mtx->copy_from(mtx.get());
         omp_cgs_factory =
@@ -165,18 +165,6 @@ class Cgs : public ::testing::Test {
         *d_stop_status = *stop_status;
     }
 
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::OmpExecutor> omp;
 
diff --git a/omp/test/solver/fcg_kernels.cpp b/omp/test/solver/fcg_kernels.cpp
index bcf6c57d288..632a2c80808 100644
--- a/omp/test/solver/fcg_kernels.cpp
+++ b/omp/test/solver/fcg_kernels.cpp
@@ -130,33 +130,6 @@ class Fcg : public ::testing::Test {
         *d_stop_status = *stop_status;
     }
 
-    void make_symetric(Mtx *mtx)
-    {
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            for (int j = i + 1; j < mtx->get_size()[1]; ++j) {
-                mtx->at(i, j) = mtx->at(j, i);
-            }
-        }
-    }
-
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
-    void make_spd(Mtx *mtx)
-    {
-        make_symetric(mtx);
-        make_diag_dominant(mtx);
-    }
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::OmpExecutor> omp;
 
@@ -246,7 +219,7 @@ TEST_F(Fcg, OmpFcgStep2IsEquivalentToRef)
 TEST_F(Fcg, ApplyIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    make_spd(mtx.get());
+    gko::test::make_spd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(omp);
diff --git a/omp/test/solver/gmres_kernels.cpp b/omp/test/solver/gmres_kernels.cpp
index 58b369e748e..66ebe091ce8 100644
--- a/omp/test/solver/gmres_kernels.cpp
+++ b/omp/test/solver/gmres_kernels.cpp
@@ -111,18 +111,6 @@ class Gmres : public ::testing::Test {
             std::normal_distribution<ValueType>(-1.0, 1.0), rand_engine, ref);
     }
 
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum;
-        }
-    }
-
     void initialize_data()
     {
 #ifdef GINKGO_FAST_TESTS
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index f26bf4dcf4b..794d68cf3f2 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -166,18 +166,6 @@ class Idr : public ::testing::Test {
             *stop_status;  // copy_from is not a public member function of Array
     }
 
-    void make_diag_dominant(Mtx *mtx)
-    {
-        using std::abs;
-        for (int i = 0; i < mtx->get_size()[0]; ++i) {
-            auto sum = gko::zero<Mtx::value_type>();
-            for (int j = 0; j < mtx->get_size()[1]; ++j) {
-                sum += abs(mtx->at(i, j));
-            }
-            mtx->at(i, i) = sum / 4;
-        }
-    }
-
     std::shared_ptr<const gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::OmpExecutor> omp;
 
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index 64788db9805..d5b1ddbebb1 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -71,10 +71,12 @@ void match_edge(std::shared_ptr<const ReferenceExecutor> exec,
     for (size_type i = 0; i < agg.get_num_elems(); i++) {
         if (agg_vals[i] == -1) {
             auto neighbor = strongest_neighbor_vals[i];
-            if (neighbor != -1 && strongest_neighbor_vals[neighbor] == i) {
+            // i < neighbor always holds when neighbor is not -1
+            if (neighbor != -1 && strongest_neighbor_vals[neighbor] == i &&
+                i < neighbor) {
+                // Use the smaller index as agg point
                 agg_vals[i] = i;
                 agg_vals[neighbor] = i;
-                // Use the smaller index as agg point
             }
         }
     }

From 64fbdc79338e5e87a619a4ed06ea2844e998facf Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 30 Mar 2021 23:50:17 +0800
Subject: [PATCH 0671/1008] collect mtx/array utils and add doc and typed_test
 - symmetric/hermitian/diag_dominant/hpd - array generator - add type test and
 documents for all in core/test/utils

Remind to myself: class member is initialized by declaring order
---
 core/test/utils.hpp                       |   1 -
 core/test/utils/CMakeLists.txt            |   3 +
 core/test/utils/array_generator.hpp       |  14 ++
 core/test/utils/array_generator_test.cpp  | 125 +++++++++++++
 core/test/utils/matrix_generator.hpp      |  13 +-
 core/test/utils/matrix_generator_test.cpp | 207 +++++++++++++++-------
 core/test/utils/matrix_utils.hpp          | 100 +++++++++--
 core/test/utils/matrix_utils_test.cpp     | 201 +++++++++++++++++++++
 core/test/utils/unsort_matrix_test.cpp    |  53 +++---
 core/test/utils/value_generator.hpp       |  30 +++-
 core/test/utils/value_generator_test.cpp  | 118 ++++++++++++
 cuda/test/multigrid/amgx_pgm_kernels.cpp  |   2 +-
 cuda/test/solver/bicg_kernels.cpp         |   2 +-
 cuda/test/solver/cg_kernels.cpp           |   2 +-
 cuda/test/solver/fcg_kernels.cpp          |   2 +-
 hip/test/multigrid/amgx_pgm_kernels.cpp   |   2 +-
 hip/test/solver/bicg_kernels.cpp          |   2 +-
 hip/test/solver/cg_kernels.cpp            |   2 +-
 hip/test/solver/fcg_kernels.cpp           |   2 +-
 omp/multigrid/amgx_pgm_kernels.cpp        |   1 +
 omp/test/multigrid/amgx_pgm_kernels.cpp   |   2 +-
 omp/test/solver/bicg_kernels.cpp          |   2 +-
 omp/test/solver/cg_kernels.cpp            |   2 +-
 omp/test/solver/fcg_kernels.cpp           |   2 +-
 24 files changed, 766 insertions(+), 124 deletions(-)
 create mode 100644 core/test/utils/array_generator_test.cpp
 create mode 100644 core/test/utils/matrix_utils_test.cpp
 create mode 100644 core/test/utils/value_generator_test.cpp

diff --git a/core/test/utils.hpp b/core/test/utils.hpp
index 05bf216c8bb..f493b375831 100644
--- a/core/test/utils.hpp
+++ b/core/test/utils.hpp
@@ -66,7 +66,6 @@ using ValueTypes =
     ::testing::Types<float, double, std::complex<float>, std::complex<double>>;
 #endif
 
-
 using ComplexValueTypes =
 #if GINKGO_DPCPP_SINGLE_MODE
     ::testing::Types<std::complex<float>>;
diff --git a/core/test/utils/CMakeLists.txt b/core/test/utils/CMakeLists.txt
index 84e3d46958d..9b3e0e5e349 100644
--- a/core/test/utils/CMakeLists.txt
+++ b/core/test/utils/CMakeLists.txt
@@ -1,3 +1,6 @@
+ginkgo_create_test(array_generator_test)
 ginkgo_create_test(assertions_test)
 ginkgo_create_test(matrix_generator_test)
+ginkgo_create_test(matrix_utils_test)
 ginkgo_create_test(unsort_matrix_test)
+ginkgo_create_test(value_generator_test)
diff --git a/core/test/utils/array_generator.hpp b/core/test/utils/array_generator.hpp
index 1ad52c18e26..8dbdeacc9a7 100644
--- a/core/test/utils/array_generator.hpp
+++ b/core/test/utils/array_generator.hpp
@@ -46,6 +46,20 @@ namespace gko {
 namespace test {
 
 
+/**
+ * Generate a random array
+ *
+ * @tparam ValueType  valuetype of the array to generate
+ * @tparam ValueDistribution  type of value distribution
+ * @tparam Engine  type of random engine
+ *
+ * @param num  the number of elements of array
+ * @param value_dist  distribution of array values
+ * @param engine  a random engine
+ * @param exec  executor where the array should be allocated
+ *
+ * @return Array<ValueType>
+ */
 template <typename ValueType, typename ValueDistribution, typename Engine>
 Array<ValueType> generate_random_array(size_type num,
                                        ValueDistribution &&value_dist,
diff --git a/core/test/utils/array_generator_test.cpp b/core/test/utils/array_generator_test.cpp
new file mode 100644
index 00000000000..ba8c6651be9
--- /dev/null
+++ b/core/test/utils/array_generator_test.cpp
@@ -0,0 +1,125 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/test/utils/array_generator.hpp"
+
+
+#include <cmath>
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include "core/test/utils.hpp"
+
+namespace {
+
+
+template <typename T>
+class ArrayGenerator : public ::testing::Test {
+protected:
+    using value_type = T;
+
+    ArrayGenerator() : exec(gko::ReferenceExecutor::create())
+    {
+        array = gko::test::generate_random_array<T>(
+            500, std::normal_distribution<gko::remove_complex<T>>(20.0, 5.0),
+            std::ranlux48(42), exec);
+    }
+
+    std::shared_ptr<const gko::Executor> exec;
+    gko::Array<T> array;
+
+    template <typename InputIterator, typename ValueType, typename Closure>
+    ValueType get_nth_moment(int n, ValueType c, InputIterator sample_start,
+                             InputIterator sample_end, Closure closure_op)
+    {
+        using std::pow;
+        ValueType res = 0;
+        ValueType num_elems = 0;
+        while (sample_start != sample_end) {
+            auto tmp = *(sample_start++);
+            res += pow(closure_op(tmp) - c, n);
+            num_elems += 1;
+        }
+        return res / num_elems;
+    }
+
+    template <typename ValueType, typename InputIterator, typename Closure>
+    void check_average_and_deviation(
+        InputIterator sample_start, InputIterator sample_end,
+        gko::remove_complex<ValueType> average_ans,
+        gko::remove_complex<ValueType> deviation_ans, Closure closure_op)
+    {
+        auto average =
+            this->get_nth_moment(1, gko::zero<gko::remove_complex<ValueType>>(),
+                                 sample_start, sample_end, closure_op);
+        auto deviation = sqrt(this->get_nth_moment(2, average, sample_start,
+                                                   sample_end, closure_op));
+
+        // check that average & deviation is within 10% of the required amount
+        ASSERT_NEAR(average, average_ans, average_ans * 0.1);
+        ASSERT_NEAR(deviation, deviation_ans, deviation_ans * 0.1);
+    }
+};
+
+TYPED_TEST_SUITE(ArrayGenerator, gko::test::ValueTypes);
+
+
+TYPED_TEST(ArrayGenerator, OutputHasCorrectSize)
+{
+    ASSERT_EQ(this->array.get_num_elems(), 500);
+}
+
+
+TYPED_TEST(ArrayGenerator, OutputHasCorrectAverageAndDeviation)
+{
+    using std::sqrt;
+    using T = typename TestFixture::value_type;
+
+    // check the real part
+    this->template check_average_and_deviation<T>(
+        this->array.get_const_data(),
+        this->array.get_const_data() + this->array.get_num_elems(), 20.0, 5.0,
+        [](T &val) { return gko::real(val); });
+    // check the imag part when the type is complex
+    if (!std::is_same<T, gko::remove_complex<T>>::value) {
+        this->template check_average_and_deviation<T>(
+            this->array.get_const_data(),
+            this->array.get_const_data() + this->array.get_num_elems(), 20.0,
+            5.0, [](T &val) { return gko::imag(val); });
+    }
+}
+
+
+}  // namespace
diff --git a/core/test/utils/matrix_generator.hpp b/core/test/utils/matrix_generator.hpp
index 04f98126c41..9a101f4b043 100644
--- a/core/test/utils/matrix_generator.hpp
+++ b/core/test/utils/matrix_generator.hpp
@@ -70,6 +70,8 @@ namespace test {
  * @param engine  a random engine
  * @param exec  executor where the matrix should be allocated
  * @param args  additional arguments for the matrix constructor
+ *
+ * @return the unique pointer of MatrixType
  */
 template <typename MatrixType = matrix::Dense<>, typename NonzeroDistribution,
           typename ValueDistribution, typename Engine, typename... MatrixArgs>
@@ -128,6 +130,8 @@ std::unique_ptr<MatrixType> generate_random_matrix(
  * @param engine  a random engine
  * @param exec  executor where the matrix should be allocated
  * @param args  additional arguments for the matrix constructor
+ *
+ * @return the unique pointer of MatrixType
  */
 template <typename MatrixType = matrix::Dense<>, typename NonzeroDistribution,
           typename Engine, typename... MatrixArgs>
@@ -189,6 +193,8 @@ std::unique_ptr<MatrixType> generate_random_sparsity_matrix(
  * @param engine  a random engine
  * @param exec  executor where the matrix should be allocated
  * @param args  additional arguments for the matrix constructor
+ *
+ * @return the unique pointer of MatrixType
  */
 template <typename MatrixType = matrix::Dense<>, typename NonzeroDistribution,
           typename ValueDistribution, typename Engine, typename... MatrixArgs>
@@ -274,6 +280,8 @@ std::unique_ptr<MatrixType> generate_random_triangular_matrix(
  * @param engine  a random engine
  * @param exec  executor where the matrix should be allocated
  * @param args  additional arguments for the matrix constructor
+ *
+ * @return the unique pointer of MatrixType
  */
 template <typename MatrixType = matrix::Dense<>, typename NonzeroDistribution,
           typename ValueDistribution, typename Engine, typename... MatrixArgs>
@@ -308,6 +316,8 @@ std::unique_ptr<MatrixType> generate_random_lower_triangular_matrix(
  * @param engine  a random engine
  * @param exec  executor where the matrix should be allocated
  * @param args  additional arguments for the matrix constructor
+ *
+ * @return the unique pointer of MatrixType
  */
 template <typename MatrixType = matrix::Dense<>, typename NonzeroDistribution,
           typename ValueDistribution, typename Engine, typename... MatrixArgs>
@@ -341,6 +351,8 @@ std::unique_ptr<MatrixType> generate_random_upper_triangular_matrix(
  * @param engine  a random engine
  * @param exec  executor where the matrix should be allocated
  * @param args  additional arguments for the matrix constructor
+ *
+ * @return the unique pointer of MatrixType
  */
 template <typename MatrixType = matrix::Dense<>, typename ValueDistribution,
           typename Engine, typename... MatrixArgs>
@@ -353,7 +365,6 @@ std::unique_ptr<MatrixType> generate_random_band_matrix(
     using index_type = typename MatrixType::index_type;
 
     matrix_data<value_type, index_type> data{gko::dim<2>{size, size}, {}};
-
     for (size_type row = 0; row < size; ++row) {
         for (size_type col = row < lower_bandwidth ? 0 : row - lower_bandwidth;
              col <= std::min(row + upper_bandwidth, size - 1); col++) {
diff --git a/core/test/utils/matrix_generator_test.cpp b/core/test/utils/matrix_generator_test.cpp
index 1a85604a788..687560a202b 100644
--- a/core/test/utils/matrix_generator_test.cpp
+++ b/core/test/utils/matrix_generator_test.cpp
@@ -40,129 +40,204 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
+#include "core/test/utils.hpp"
+
+
 namespace {
 
 
+template <typename T>
 class MatrixGenerator : public ::testing::Test {
 protected:
+    using value_type = T;
+    using real_type = gko::remove_complex<T>;
+    using mtx_type = gko::matrix::Dense<T>;
+
     MatrixGenerator()
         : exec(gko::ReferenceExecutor::create()),
-          mtx(gko::test::generate_random_matrix(
-              500, 100, std::normal_distribution<double>(50, 5),
-              std::normal_distribution<double>(20.0, 5.0), std::ranlux48(42),
+          mtx(gko::test::generate_random_matrix<mtx_type>(
+              500, 100, std::normal_distribution<real_type>(50, 5),
+              std::normal_distribution<real_type>(20.0, 5.0), std::ranlux48(42),
               exec)),
-          l_mtx(gko::test::generate_random_lower_triangular_matrix(
-              4, 3, true, std::normal_distribution<double>(50, 5),
-              std::normal_distribution<double>(20.0, 5.0), std::ranlux48(42),
+          l_mtx(gko::test::generate_random_lower_triangular_matrix<mtx_type>(
+              4, 3, true, std::normal_distribution<real_type>(50, 5),
+              std::normal_distribution<real_type>(20.0, 5.0), std::ranlux48(42),
               exec)),
-          u_mtx(gko::test::generate_random_upper_triangular_matrix(
-              3, 4, true, std::normal_distribution<double>(50, 5),
-              std::normal_distribution<double>(20.0, 5.0), std::ranlux48(42),
+          u_mtx(gko::test::generate_random_upper_triangular_matrix<mtx_type>(
+              3, 4, true, std::normal_distribution<real_type>(50, 5),
+              std::normal_distribution<real_type>(20.0, 5.0), std::ranlux48(42),
+              exec)),
+          lower_bandwidth(2),
+          upper_bandwidth(3),
+          band_mtx(gko::test::generate_random_band_matrix<mtx_type>(
+              100, lower_bandwidth, upper_bandwidth,
+              std::normal_distribution<real_type>(20.0, 5.0), std::ranlux48(42),
               exec)),
           nnz_per_row_sample(500, 0),
-          values_sample(0)
+          values_sample(0),
+          band_values_sample(0)
     {
         // collect samples of nnz/row and values from the matrix
         for (int row = 0; row < mtx->get_size()[0]; ++row) {
             for (int col = 0; col < mtx->get_size()[1]; ++col) {
                 auto val = mtx->at(row, col);
-                if (val != 0.0) {
+                if (val != gko::zero<T>()) {
                     ++nnz_per_row_sample[row];
                     values_sample.push_back(val);
                 }
             }
         }
+
+        // collect samples of values from the band matrix
+        for (int row = 0; row < band_mtx->get_size()[0]; ++row) {
+            for (int col = 0; col < band_mtx->get_size()[1]; ++col) {
+                auto val = band_mtx->at(row, col);
+                if ((col - row <= upper_bandwidth) &&
+                    (row - col <= lower_bandwidth)) {
+                    band_values_sample.push_back(val);
+                }
+            }
+        }
     }
 
     std::shared_ptr<const gko::Executor> exec;
-    std::unique_ptr<gko::matrix::Dense<>> mtx;
-    std::unique_ptr<gko::matrix::Dense<>> l_mtx;
-    std::unique_ptr<gko::matrix::Dense<>> u_mtx;
+    int lower_bandwidth;
+    int upper_bandwidth;
+    std::unique_ptr<mtx_type> mtx;
+    std::unique_ptr<mtx_type> l_mtx;
+    std::unique_ptr<mtx_type> u_mtx;
+    std::unique_ptr<mtx_type> band_mtx;
     std::vector<int> nnz_per_row_sample;
-    std::vector<double> values_sample;
+    std::vector<T> values_sample;
+    std::vector<T> band_values_sample;
+
 
-    template <typename InputIterator, typename ValueType>
+    template <typename InputIterator, typename ValueType, typename Closure>
     ValueType get_nth_moment(int n, ValueType c, InputIterator sample_start,
-                             InputIterator sample_end)
+                             InputIterator sample_end, Closure closure_op)
     {
         using std::pow;
         ValueType res = 0;
         ValueType num_elems = 0;
         while (sample_start != sample_end) {
             auto tmp = *(sample_start++);
-            res += pow(tmp - c, n);
+            res += pow(closure_op(tmp) - c, n);
             num_elems += 1;
         }
         return res / num_elems;
     }
+
+    template <typename ValueType, typename InputIterator, typename Closure>
+    void check_average_and_deviation(
+        InputIterator sample_start, InputIterator sample_end,
+        gko::remove_complex<ValueType> average_ans,
+        gko::remove_complex<ValueType> deviation_ans, Closure closure_op)
+    {
+        auto average =
+            this->get_nth_moment(1, gko::zero<gko::remove_complex<ValueType>>(),
+                                 sample_start, sample_end, closure_op);
+        auto deviation = sqrt(this->get_nth_moment(2, average, sample_start,
+                                                   sample_end, closure_op));
+
+        // check that average & deviation is within 10% of the required amount
+        ASSERT_NEAR(average, average_ans, average_ans * 0.1);
+        ASSERT_NEAR(deviation, deviation_ans, deviation_ans * 0.1);
+    }
 };
 
+TYPED_TEST_SUITE(MatrixGenerator, gko::test::ValueTypes);
+
 
-TEST_F(MatrixGenerator, OutputHasCorrectSize)
+TYPED_TEST(MatrixGenerator, OutputHasCorrectSize)
 {
-    ASSERT_EQ(mtx->get_size(), gko::dim<2>(500, 100));
+    ASSERT_EQ(this->mtx->get_size(), gko::dim<2>(500, 100));
 }
 
 
-TEST_F(MatrixGenerator, OutputHasCorrectNonzeroAverageAndDeviation)
+TYPED_TEST(MatrixGenerator, OutputHasCorrectNonzeroAverageAndDeviation)
 {
-    using std::sqrt;
-    auto average = get_nth_moment(1, 0.0, begin(nnz_per_row_sample),
-                                  end(nnz_per_row_sample));
-    auto deviation = sqrt(get_nth_moment(2, average, begin(nnz_per_row_sample),
-                                         end(nnz_per_row_sample)));
-
-    // check that average & deviation is within 10% of the required amount
-    ASSERT_NEAR(average, 50.0, 5);
-    ASSERT_NEAR(deviation, 5.0, 0.5);
+    using T = typename TestFixture::value_type;
+    // the nonzeros only needs to check the real part
+    this->template check_average_and_deviation<T>(
+        begin(this->nnz_per_row_sample), end(this->nnz_per_row_sample), 50.0,
+        5.0, [](T val) { return gko::real(val); });
 }
 
 
-TEST_F(MatrixGenerator, OutputHasCorrectValuesAverageAndDeviation)
+TYPED_TEST(MatrixGenerator, OutputHasCorrectValuesAverageAndDeviation)
 {
-    using std::sqrt;
-    auto average =
-        get_nth_moment(1, 0.0, begin(values_sample), end(values_sample));
-    auto deviation = sqrt(
-        get_nth_moment(2, average, begin(values_sample), end(values_sample)));
-
-    // check that average and deviation is within 10% of the required amount
-    ASSERT_NEAR(average, 20.0, 2.0);
-    ASSERT_NEAR(deviation, 5.0, 0.5);
+    using T = typename TestFixture::value_type;
+    // check the real part
+    this->template check_average_and_deviation<T>(
+        begin(this->values_sample), end(this->values_sample), 20.0, 5.0,
+        [](T &val) { return gko::real(val); });
+    // check the imag part when the type is complex
+    if (!std::is_same<T, gko::remove_complex<T>>::value) {
+        this->template check_average_and_deviation<T>(
+            begin(this->values_sample), end(this->values_sample), 20.0, 5.0,
+            [](T &val) { return gko::imag(val); });
+    }
 }
 
 
-TEST_F(MatrixGenerator, CanGenerateLowerTriangularMatrixWithDiagonalOnes)
+TYPED_TEST(MatrixGenerator, CanGenerateLowerTriangularMatrixWithDiagonalOnes)
 {
-    ASSERT_EQ(l_mtx->at(0, 0), 1.0);
-    ASSERT_EQ(l_mtx->at(0, 1), 0.0);
-    ASSERT_EQ(l_mtx->at(0, 2), 0.0);
-    ASSERT_NE(l_mtx->at(1, 0), 0.0);
-    ASSERT_EQ(l_mtx->at(1, 1), 1.0);
-    ASSERT_EQ(l_mtx->at(1, 2), 0.0);
-    ASSERT_NE(l_mtx->at(2, 0), 0.0);
-    ASSERT_NE(l_mtx->at(2, 1), 0.0);
-    ASSERT_EQ(l_mtx->at(2, 2), 1.0);
-    ASSERT_NE(l_mtx->at(3, 0), 0.0);
-    ASSERT_NE(l_mtx->at(3, 1), 0.0);
-    ASSERT_NE(l_mtx->at(3, 2), 0.0);
+    using T = typename TestFixture::value_type;
+    ASSERT_EQ(this->l_mtx->at(0, 0), T{1.0});
+    ASSERT_EQ(this->l_mtx->at(0, 1), T{0.0});
+    ASSERT_EQ(this->l_mtx->at(0, 2), T{0.0});
+    ASSERT_NE(this->l_mtx->at(1, 0), T{0.0});
+    ASSERT_EQ(this->l_mtx->at(1, 1), T{1.0});
+    ASSERT_EQ(this->l_mtx->at(1, 2), T{0.0});
+    ASSERT_NE(this->l_mtx->at(2, 0), T{0.0});
+    ASSERT_NE(this->l_mtx->at(2, 1), T{0.0});
+    ASSERT_EQ(this->l_mtx->at(2, 2), T{1.0});
+    ASSERT_NE(this->l_mtx->at(3, 0), T{0.0});
+    ASSERT_NE(this->l_mtx->at(3, 1), T{0.0});
+    ASSERT_NE(this->l_mtx->at(3, 2), T{0.0});
 }
 
 
-TEST_F(MatrixGenerator, CanGenerateUpperTriangularMatrixWithDiagonalOnes)
+TYPED_TEST(MatrixGenerator, CanGenerateUpperTriangularMatrixWithDiagonalOnes)
 {
-    ASSERT_EQ(u_mtx->at(0, 0), 1.0);
-    ASSERT_NE(u_mtx->at(0, 1), 0.0);
-    ASSERT_NE(u_mtx->at(0, 2), 0.0);
-    ASSERT_NE(u_mtx->at(0, 3), 0.0);
-    ASSERT_EQ(u_mtx->at(1, 0), 0.0);
-    ASSERT_EQ(u_mtx->at(1, 1), 1.0);
-    ASSERT_NE(u_mtx->at(1, 2), 0.0);
-    ASSERT_NE(u_mtx->at(1, 3), 0.0);
-    ASSERT_EQ(u_mtx->at(2, 0), 0.0);
-    ASSERT_EQ(u_mtx->at(2, 1), 0.0);
-    ASSERT_EQ(u_mtx->at(2, 2), 1.0);
-    ASSERT_NE(u_mtx->at(2, 3), 0.0);
+    using T = typename TestFixture::value_type;
+    ASSERT_EQ(this->u_mtx->at(0, 0), T{1.0});
+    ASSERT_NE(this->u_mtx->at(0, 1), T{0.0});
+    ASSERT_NE(this->u_mtx->at(0, 2), T{0.0});
+    ASSERT_NE(this->u_mtx->at(0, 3), T{0.0});
+    ASSERT_EQ(this->u_mtx->at(1, 0), T{0.0});
+    ASSERT_EQ(this->u_mtx->at(1, 1), T{1.0});
+    ASSERT_NE(this->u_mtx->at(1, 2), T{0.0});
+    ASSERT_NE(this->u_mtx->at(1, 3), T{0.0});
+    ASSERT_EQ(this->u_mtx->at(2, 0), T{0.0});
+    ASSERT_EQ(this->u_mtx->at(2, 1), T{0.0});
+    ASSERT_EQ(this->u_mtx->at(2, 2), T{1.0});
+    ASSERT_NE(this->u_mtx->at(2, 3), T{0.0});
+}
+
+
+TYPED_TEST(MatrixGenerator, CanGenerateBandMatrix)
+{
+    using T = typename TestFixture::value_type;
+    // the elements out of band are zero
+    for (int row = 0; row < this->band_mtx->get_size()[0]; row++) {
+        for (int col = 0; col < this->band_mtx->get_size()[1]; col++) {
+            if ((col - row > this->upper_bandwidth) ||
+                (row - col > this->lower_bandwidth)) {
+                ASSERT_EQ(this->band_mtx->at(row, col), T{0.0});
+            }
+        }
+    }
+    // check the real part of elements in band
+    this->template check_average_and_deviation<T>(
+        begin(this->band_values_sample), end(this->band_values_sample), 20.0,
+        5.0, [](T &val) { return gko::real(val); });
+    // check the imag part when the type is complex
+    if (!std::is_same<T, gko::remove_complex<T>>::value) {
+        this->template check_average_and_deviation<T>(
+            begin(this->band_values_sample), end(this->band_values_sample),
+            20.0, 5.0, [](T &val) { return gko::imag(val); });
+    }
 }
 
 
diff --git a/core/test/utils/matrix_utils.hpp b/core/test/utils/matrix_utils.hpp
index c8b269d493f..90aa3ea1d6e 100644
--- a/core/test/utils/matrix_utils.hpp
+++ b/core/test/utils/matrix_utils.hpp
@@ -34,8 +34,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_TEST_UTILS_MATRIX_UTILS_HPP_
 
 
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/temporary_clone.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
+
 #include "core/test/utils/value_generator.hpp"
 
 
@@ -43,38 +46,109 @@ namespace gko {
 namespace test {
 
 
+/**
+ * Make a symmetric matrix
+ *
+ * @tparam ValueType  valuetype of Dense matrix to process
+ *
+ * @param mtx  the dense matrix
+ */
 template <typename ValueType>
 void make_symmetric(matrix::Dense<ValueType> *mtx)
 {
-    assert(mtx->get_executor() == mtx->get_executor()->get_master());
-    for (size_type i = 0; i < mtx->get_size()[0]; ++i) {
-        for (size_type j = i + 1; j < mtx->get_size()[1]; ++j) {
-            mtx->at(i, j) = mtx->at(j, i);
+    GKO_ASSERT_IS_SQUARE_MATRIX(mtx);
+    auto mtx_host =
+        make_temporary_clone(mtx->get_executor()->get_master(), mtx);
+
+    for (size_type i = 0; i < mtx_host->get_size()[0]; ++i) {
+        for (size_type j = i + 1; j < mtx_host->get_size()[1]; ++j) {
+            mtx_host->at(i, j) = mtx_host->at(j, i);
         }
     }
 }
 
 
+/**
+ * Make a hermitian matrix
+ *
+ * @tparam ValueType  valuetype of Dense matrix to process
+ *
+ * @param mtx  the dense matrix
+ */
 template <typename ValueType>
-void make_diag_dominant(matrix::Dense<ValueType> *mtx)
+void make_hermitian(matrix::Dense<ValueType> *mtx)
 {
-    assert(mtx->get_executor() == mtx->get_executor()->get_master());
+    GKO_ASSERT_IS_SQUARE_MATRIX(mtx);
+    auto mtx_host =
+        make_temporary_clone(mtx->get_executor()->get_master(), mtx);
+
+    for (size_type i = 0; i < mtx_host->get_size()[0]; ++i) {
+        for (size_type j = i + 1; j < mtx_host->get_size()[1]; ++j) {
+            mtx_host->at(i, j) = conj(mtx_host->at(j, i));
+        }
+        mtx_host->at(i, i) = gko::real(mtx_host->at(i, i));
+    }
+}
+
+
+/**
+ * Make a (strictly) diagonal dominant matrix. It will set the diag value from
+ * the summation among the absoulue value of the row's elements. When ratio is
+ * larger than 1, the result will be strictly diagonal dominant matrix except
+ * for the empty row. When ratio is 1, the result will be diagonal dominant
+ * matirx.
+ *
+ * @tparam ValueType  valuetype of Dense matrix to process
+ *
+ * @param mtx  the dense matrix
+ * @param ratio  the scale to set the diagonal value. default is 1 and it must
+ *               be larger than or equal to 1.
+ */
+template <typename ValueType>
+void make_diag_dominant(matrix::Dense<ValueType> *mtx,
+                        remove_complex<ValueType> ratio = 1.0)
+{
+    // To keep the diag dominant, the ratio should be larger than or equal to 1
+    GKO_ASSERT_EQ(ratio >= 1.0, true);
+    auto mtx_host =
+        make_temporary_clone(mtx->get_executor()->get_master(), mtx);
+
     using std::abs;
-    for (int i = 0; i < mtx->get_size()[0]; ++i) {
+    for (size_type i = 0; i < mtx_host->get_size()[0]; ++i) {
         auto sum = gko::zero<ValueType>();
-        for (int j = 0; j < mtx->get_size()[1]; ++j) {
-            sum += abs(mtx->at(i, j));
+        for (size_type j = 0; j < mtx_host->get_size()[1]; ++j) {
+            sum += abs(mtx_host->at(i, j));
         }
-        mtx->at(i, i) = sum;
+        mtx_host->at(i, i) = sum * ratio;
     }
 }
 
 
+/**
+ * Make a Hermitian postive definite matrix.
+ *
+ * @tparam ValueType  valuetype of Dense matrix to process
+ *
+ * @param mtx  the dense matrix
+ * @param ratio  the ratio for make_diag_dominant. default is 1.001 and it must
+ *               be larger than 1.
+ */
 template <typename ValueType>
-void make_spd(matrix::Dense<ValueType> *mtx)
+void make_hpd(matrix::Dense<ValueType> *mtx,
+              remove_complex<ValueType> ratio = 1.001)
 {
-    make_symmetric(mtx);
-    make_diag_dominant(mtx);
+    GKO_ASSERT_IS_SQUARE_MATRIX(mtx);
+    // To get strictly diagonally dominant matrix, the ratio should be larger
+    // than 1.
+    GKO_ASSERT_EQ(ratio > 1.0, true);
+
+    auto mtx_host =
+        make_temporary_clone(mtx->get_executor()->get_master(), mtx);
+    make_hermitian(mtx_host.get());
+    // Construct strictly diagonally dominant matrix to ensure positive
+    // definite. In complex case, the diagonal is set as absolute value and is
+    // larger than 0, so it still gives positive definite.
+    make_diag_dominant(mtx_host.get(), ratio);
 }
 
 
diff --git a/core/test/utils/matrix_utils_test.cpp b/core/test/utils/matrix_utils_test.cpp
new file mode 100644
index 00000000000..dea1b4a55d6
--- /dev/null
+++ b/core/test/utils/matrix_utils_test.cpp
@@ -0,0 +1,201 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/test/utils/matrix_utils.hpp"
+
+
+#include <cmath>
+#include <random>
+#include <type_traits>
+
+
+#include <gtest/gtest.h>
+
+
+#include "core/test/utils.hpp"
+#include "core/test/utils/matrix_generator.hpp"
+
+
+namespace {
+
+
+template <typename T>
+class MatrixUtils : public ::testing::Test {
+protected:
+    using value_type = T;
+    using real_type = gko::remove_complex<T>;
+    using mtx_type = gko::matrix::Dense<T>;
+
+    MatrixUtils()
+        : exec(gko::ReferenceExecutor::create()),
+          mtx(gko::test::generate_random_matrix<mtx_type>(
+              500, 500, std::normal_distribution<real_type>(50, 5),
+              std::normal_distribution<real_type>(20.0, 5.0), std::ranlux48(42),
+              exec)),
+          unsquare_mtx(mtx_type::create(exec, gko::dim<2>(500, 100)))
+    {}
+
+    std::shared_ptr<const gko::Executor> exec;
+    std::unique_ptr<mtx_type> mtx;
+    std::unique_ptr<mtx_type> unsquare_mtx;
+};
+
+TYPED_TEST_SUITE(MatrixUtils, gko::test::ValueTypes);
+
+
+TYPED_TEST(MatrixUtils, MakeSymmetricThrowsError)
+{
+    ASSERT_THROW(gko::test::make_symmetric(gko::lend(this->unsquare_mtx)),
+                 gko::DimensionMismatch);
+}
+
+TYPED_TEST(MatrixUtils, MakeHermitianThrowsError)
+{
+    ASSERT_THROW(gko::test::make_hermitian(gko::lend(this->unsquare_mtx)),
+                 gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(MatrixUtils, MakeDiagDominantThrowsError)
+{
+    ASSERT_THROW(gko::test::make_diag_dominant(gko::lend(this->mtx), 0.9),
+                 gko::ValueMismatch);
+}
+
+
+TYPED_TEST(MatrixUtils, MakeHpdMatrixThrowsError)
+{
+    ASSERT_THROW(gko::test::make_hpd(gko::lend(this->mtx), 1.0),
+                 gko::ValueMismatch);
+}
+
+
+TYPED_TEST(MatrixUtils, MakeSymmetricCorrectly)
+{
+    gko::test::make_symmetric(gko::lend(this->mtx));
+
+    for (gko::size_type i = 0; i < this->mtx->get_size()[0]; i++) {
+        for (gko::size_type j = 0; j <= i; j++) {
+            ASSERT_EQ(this->mtx->at(i, j), this->mtx->at(j, i));
+        }
+    }
+}
+
+
+TYPED_TEST(MatrixUtils, MakeHermitianCorrectly)
+{
+    gko::test::make_hermitian(gko::lend(this->mtx));
+
+    for (gko::size_type i = 0; i < this->mtx->get_size()[0]; i++) {
+        for (gko::size_type j = 0; j <= i; j++) {
+            ASSERT_EQ(this->mtx->at(i, j), gko::conj(this->mtx->at(j, i)));
+        }
+    }
+}
+
+
+TYPED_TEST(MatrixUtils, MakeDiagDominantCorrectly)
+{
+    using T = typename TestFixture::value_type;
+    // make_diag_dominant also consider diag value.
+    // To check the ratio easily, set the diag zeros
+    for (gko::size_type i = 0; i < this->mtx->get_size()[0]; i++) {
+        this->mtx->at(i, i) = 0;
+    }
+
+    gko::test::make_diag_dominant(gko::lend(this->mtx));
+
+    for (gko::size_type i = 0; i < this->mtx->get_size()[0]; i++) {
+        gko::remove_complex<T> off_diag_abs = 0;
+        for (gko::size_type j = 0; j < this->mtx->get_size()[1]; j++) {
+            if (j != i) {
+                off_diag_abs += std::abs(this->mtx->at(i, j));
+            }
+        }
+        ASSERT_NEAR(gko::real(this->mtx->at(i, i)), off_diag_abs, r<T>::value);
+    }
+}
+
+
+TYPED_TEST(MatrixUtils, MakeDiagDominantWithRatioCorrectly)
+{
+    using T = typename TestFixture::value_type;
+    gko::remove_complex<T> ratio = 1.001;
+    // make_diag_dominant also consider diag value.
+    // To check the ratio easily, set the diag zeros
+    for (gko::size_type i = 0; i < this->mtx->get_size()[0]; i++) {
+        this->mtx->at(i, i) = 0;
+    }
+
+    gko::test::make_diag_dominant(gko::lend(this->mtx), ratio);
+
+    for (gko::size_type i = 0; i < this->mtx->get_size()[0]; i++) {
+        gko::remove_complex<T> off_diag_abs = 0;
+        for (gko::size_type j = 0; j < this->mtx->get_size()[1]; j++) {
+            if (j != i) {
+                off_diag_abs += std::abs(this->mtx->at(i, j));
+            }
+        }
+        ASSERT_NEAR(gko::real(this->mtx->at(i, i)), off_diag_abs * ratio,
+                    r<T>::value);
+    }
+}
+
+
+TYPED_TEST(MatrixUtils, MakeHpdMatrixCorrectly)
+{
+    using T = typename TestFixture::value_type;
+    auto cpy_mtx = this->mtx->clone();
+
+    gko::test::make_hpd(gko::lend(this->mtx));
+    gko::test::make_hermitian(gko::lend(cpy_mtx));
+    gko::test::make_diag_dominant(gko::lend(cpy_mtx), 1.001);
+
+    GKO_ASSERT_MTX_NEAR(this->mtx, cpy_mtx, r<T>::value);
+}
+
+
+TYPED_TEST(MatrixUtils, MakeHpdMatrixWithRatioCorrectly)
+{
+    using T = typename TestFixture::value_type;
+    gko::remove_complex<T> ratio = 1.00001;
+    auto cpy_mtx = this->mtx->clone();
+
+    gko::test::make_hpd(gko::lend(this->mtx), ratio);
+    gko::test::make_hermitian(gko::lend(cpy_mtx));
+    gko::test::make_diag_dominant(gko::lend(cpy_mtx), ratio);
+
+    GKO_ASSERT_MTX_NEAR(this->mtx, cpy_mtx, r<T>::value);
+}
+
+
+}  // namespace
diff --git a/core/test/utils/unsort_matrix_test.cpp b/core/test/utils/unsort_matrix_test.cpp
index 0a242c7dc2c..90fbbc2ccf2 100644
--- a/core/test/utils/unsort_matrix_test.cpp
+++ b/core/test/utils/unsort_matrix_test.cpp
@@ -55,18 +55,19 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
+template <typename ValueIndexType>
 class UnsortMatrix : public ::testing::Test {
 protected:
-    using value_type = double;
-    using index_type = gko::int32;
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
     using Csr = gko::matrix::Csr<value_type, index_type>;
     using Coo = gko::matrix::Coo<value_type, index_type>;
     using Dense = gko::matrix::Dense<value_type>;
     UnsortMatrix()
         : exec(gko::ReferenceExecutor::create()),
           rand_engine(42),
-          /*
-           */
           csr_empty(Csr::create(exec, gko::dim<2>(0, 0))),
           coo_empty(Coo::create(exec, gko::dim<2>(0, 0)))
     {}
@@ -148,54 +149,58 @@ class UnsortMatrix : public ::testing::Test {
     std::unique_ptr<Coo> coo_empty;
 };
 
+TYPED_TEST_SUITE(UnsortMatrix, gko::test::ValueIndexTypes);
 
-TEST_F(UnsortMatrix, CsrWorks)
+
+TYPED_TEST(UnsortMatrix, CsrWorks)
 {
-    auto csr = get_sorted_csr();
-    const auto ref_mtx = get_sorted_csr();
-    bool was_sorted = is_csr_matrix_sorted(gko::lend(csr));
+    auto csr = this->get_sorted_csr();
+    const auto ref_mtx = this->get_sorted_csr();
+    bool was_sorted = this->is_csr_matrix_sorted(gko::lend(csr));
 
-    gko::test::unsort_matrix(gko::lend(csr), rand_engine);
+    gko::test::unsort_matrix(gko::lend(csr), this->rand_engine);
 
-    ASSERT_FALSE(is_csr_matrix_sorted(gko::lend(csr)));
+    ASSERT_FALSE(this->is_csr_matrix_sorted(gko::lend(csr)));
     ASSERT_TRUE(was_sorted);
     GKO_ASSERT_MTX_NEAR(csr, ref_mtx, 0.);
 }
 
 
-TEST_F(UnsortMatrix, CsrWorksWithEmpty)
+TYPED_TEST(UnsortMatrix, CsrWorksWithEmpty)
 {
-    const bool was_sorted = is_csr_matrix_sorted(gko::lend(csr_empty));
+    const bool was_sorted =
+        this->is_csr_matrix_sorted(gko::lend(this->csr_empty));
 
-    gko::test::unsort_matrix(gko::lend(csr_empty), rand_engine);
+    gko::test::unsort_matrix(gko::lend(this->csr_empty), this->rand_engine);
 
     ASSERT_TRUE(was_sorted);
-    ASSERT_EQ(csr_empty->get_num_stored_elements(), 0);
+    ASSERT_EQ(this->csr_empty->get_num_stored_elements(), 0);
 }
 
 
-TEST_F(UnsortMatrix, CooWorks)
+TYPED_TEST(UnsortMatrix, CooWorks)
 {
-    auto coo = get_sorted_coo();
-    const auto ref_mtx = get_sorted_coo();
-    const bool was_sorted = is_coo_matrix_sorted(gko::lend(coo));
+    auto coo = this->get_sorted_coo();
+    const auto ref_mtx = this->get_sorted_coo();
+    const bool was_sorted = this->is_coo_matrix_sorted(gko::lend(coo));
 
-    gko::test::unsort_matrix(gko::lend(coo), rand_engine);
+    gko::test::unsort_matrix(gko::lend(coo), this->rand_engine);
 
-    ASSERT_FALSE(is_coo_matrix_sorted(gko::lend(coo)));
+    ASSERT_FALSE(this->is_coo_matrix_sorted(gko::lend(coo)));
     ASSERT_TRUE(was_sorted);
     GKO_ASSERT_MTX_NEAR(coo, ref_mtx, 0.);
 }
 
 
-TEST_F(UnsortMatrix, CooWorksWithEmpty)
+TYPED_TEST(UnsortMatrix, CooWorksWithEmpty)
 {
-    const bool was_sorted = is_coo_matrix_sorted(gko::lend(coo_empty));
+    const bool was_sorted =
+        this->is_coo_matrix_sorted(gko::lend(this->coo_empty));
 
-    gko::test::unsort_matrix(gko::lend(coo_empty), rand_engine);
+    gko::test::unsort_matrix(gko::lend(this->coo_empty), this->rand_engine);
 
     ASSERT_TRUE(was_sorted);
-    ASSERT_EQ(coo_empty->get_num_stored_elements(), 0);
+    ASSERT_EQ(this->coo_empty->get_num_stored_elements(), 0);
 }
 
 
diff --git a/core/test/utils/value_generator.hpp b/core/test/utils/value_generator.hpp
index 8b82ea63d0a..8791bf6ce01 100644
--- a/core/test/utils/value_generator.hpp
+++ b/core/test/utils/value_generator.hpp
@@ -46,19 +46,35 @@ namespace test {
 namespace detail {
 
 
-template <typename ValueType, typename Distribution, typename Generator>
+/**
+ * Generate a random value.
+ *
+ * @tparam ValueType  valuetype of the value
+ * @tparam ValueDistribution  type of value distribution
+ * @tparam Engine  type of random engine
+ *
+ * @param value_dist  distribution of array values
+ * @param engine  a random engine
+ *
+ * @return ValueType
+ */
+template <typename ValueType, typename ValueDistribution, typename Engine>
 typename std::enable_if<!is_complex_s<ValueType>::value, ValueType>::type
-get_rand_value(Distribution &&dist, Generator &&gen)
+get_rand_value(ValueDistribution &&value_dist, Engine &&gen)
 {
-    return dist(gen);
+    return value_dist(gen);
 }
 
-
-template <typename ValueType, typename Distribution, typename Generator>
+/**
+ * Specialization for complex types.
+ *
+ * @copydoc get_rand_value
+ */
+template <typename ValueType, typename ValueDistribution, typename Engine>
 typename std::enable_if<is_complex_s<ValueType>::value, ValueType>::type
-get_rand_value(Distribution &&dist, Generator &&gen)
+get_rand_value(ValueDistribution &&value_dist, Engine &&gen)
 {
-    return ValueType(dist(gen), dist(gen));
+    return ValueType(value_dist(gen), value_dist(gen));
 }
 
 
diff --git a/core/test/utils/value_generator_test.cpp b/core/test/utils/value_generator_test.cpp
new file mode 100644
index 00000000000..58f033404a9
--- /dev/null
+++ b/core/test/utils/value_generator_test.cpp
@@ -0,0 +1,118 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/test/utils/value_generator.hpp"
+
+
+#include <cmath>
+#include <random>
+#include <type_traits>
+
+
+#include <gtest/gtest.h>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+template <typename T>
+class ValueGenerator : public ::testing::Test {
+protected:
+    using value_type = T;
+
+    ValueGenerator() {}
+
+    template <typename InputIterator, typename ValueType, typename Closure>
+    ValueType get_nth_moment(int n, ValueType c, InputIterator sample_start,
+                             InputIterator sample_end, Closure closure_op)
+    {
+        using std::pow;
+        ValueType res = 0;
+        ValueType num_elems = 0;
+        while (sample_start != sample_end) {
+            auto tmp = *(sample_start++);
+            res += pow(closure_op(tmp) - c, n);
+            num_elems += 1;
+        }
+        return res / num_elems;
+    }
+
+    template <typename ValueType, typename InputIterator, typename Closure>
+    void check_average_and_deviation(
+        InputIterator sample_start, InputIterator sample_end,
+        gko::remove_complex<ValueType> average_ans,
+        gko::remove_complex<ValueType> deviation_ans, Closure closure_op)
+    {
+        auto average =
+            this->get_nth_moment(1, gko::zero<gko::remove_complex<ValueType>>(),
+                                 sample_start, sample_end, closure_op);
+        auto deviation = sqrt(this->get_nth_moment(2, average, sample_start,
+                                                   sample_end, closure_op));
+
+        // check that average & deviation is within 10% of the required amount
+        ASSERT_NEAR(average, average_ans, average_ans * 0.1);
+        ASSERT_NEAR(deviation, deviation_ans, deviation_ans * 0.1);
+    }
+};
+
+TYPED_TEST_SUITE(ValueGenerator, gko::test::ValueTypes);
+
+
+TYPED_TEST(ValueGenerator, OutputHasCorrectAverageAndDeviation)
+{
+    using T = typename TestFixture::value_type;
+    int num = 500;
+    std::vector<T> values(num);
+    auto dist = std::normal_distribution<double>(20.0, 5.0);
+    auto engine = std::ranlux48(42);
+
+    for (int i = 0; i < num; i++) {
+        values.at(i) = gko::test::detail::get_rand_value<T>(dist, engine);
+    }
+
+    // check the real part
+    this->template check_average_and_deviation<T>(
+        begin(values), end(values), 20.0, 5.0,
+        [](T &val) { return gko::real(val); });
+    // check the imag part when the type is complex
+    if (!std::is_same<T, gko::remove_complex<T>>::value) {
+        this->template check_average_and_deviation<T>(
+            begin(values), end(values), 20.0, 5.0,
+            [](T &val) { return gko::imag(val); });
+    }
+}
+
+
+}  // namespace
diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
index 2814f9177c0..524f4a09f3d 100644
--- a/cuda/test/multigrid/amgx_pgm_kernels.cpp
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -115,7 +115,7 @@ class AmgxPgm : public ::testing::Test {
         weight->convert_to(weight_csr.get());
         weight_diag = weight_csr->extract_diagonal();
         auto system_dense = gen_mtx(m, m);
-        gko::test::make_spd(system_dense.get());
+        gko::test::make_hpd(system_dense.get());
         system_mtx = Csr::create(ref);
         system_dense->convert_to(system_mtx.get());
 
diff --git a/cuda/test/solver/bicg_kernels.cpp b/cuda/test/solver/bicg_kernels.cpp
index d82cc0a28a0..fb62f702ccf 100644
--- a/cuda/test/solver/bicg_kernels.cpp
+++ b/cuda/test/solver/bicg_kernels.cpp
@@ -258,7 +258,7 @@ TEST_F(Bicg, CudaBicgStep2IsEquivalentToRef)
 TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    gko::test::make_spd(mtx.get());
+    gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(cuda);
diff --git a/cuda/test/solver/cg_kernels.cpp b/cuda/test/solver/cg_kernels.cpp
index 8333fc7d0a8..e6443543efb 100644
--- a/cuda/test/solver/cg_kernels.cpp
+++ b/cuda/test/solver/cg_kernels.cpp
@@ -207,7 +207,7 @@ TEST_F(Cg, CudaCgStep2IsEquivalentToRef)
 TEST_F(Cg, ApplyIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    gko::test::make_spd(mtx.get());
+    gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(cuda);
diff --git a/cuda/test/solver/fcg_kernels.cpp b/cuda/test/solver/fcg_kernels.cpp
index b2832945666..c18444ee850 100644
--- a/cuda/test/solver/fcg_kernels.cpp
+++ b/cuda/test/solver/fcg_kernels.cpp
@@ -220,7 +220,7 @@ TEST_F(Fcg, CudaFcgStep2IsEquivalentToRef)
 TEST_F(Fcg, ApplyIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    gko::test::make_spd(mtx.get());
+    gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(cuda);
diff --git a/hip/test/multigrid/amgx_pgm_kernels.cpp b/hip/test/multigrid/amgx_pgm_kernels.cpp
index 39546df6cff..ce63db8a8dd 100644
--- a/hip/test/multigrid/amgx_pgm_kernels.cpp
+++ b/hip/test/multigrid/amgx_pgm_kernels.cpp
@@ -114,7 +114,7 @@ class AmgxPgm : public ::testing::Test {
         weight->convert_to(weight_csr.get());
         weight_diag = weight_csr->extract_diagonal();
         auto system_dense = gen_mtx(m, m);
-        gko::test::make_spd(system_dense.get());
+        gko::test::make_hpd(system_dense.get());
         system_mtx = Csr::create(ref);
         system_dense->convert_to(system_mtx.get());
 
diff --git a/hip/test/solver/bicg_kernels.cpp b/hip/test/solver/bicg_kernels.cpp
index c29bd74374c..d8510e46145 100644
--- a/hip/test/solver/bicg_kernels.cpp
+++ b/hip/test/solver/bicg_kernels.cpp
@@ -258,7 +258,7 @@ TEST_F(Bicg, HipBicgStep2IsEquivalentToRef)
 TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    gko::test::make_spd(mtx.get());
+    gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(hip);
diff --git a/hip/test/solver/cg_kernels.cpp b/hip/test/solver/cg_kernels.cpp
index cd4000171c0..a93c20a3287 100644
--- a/hip/test/solver/cg_kernels.cpp
+++ b/hip/test/solver/cg_kernels.cpp
@@ -207,7 +207,7 @@ TEST_F(Cg, HipCgStep2IsEquivalentToRef)
 TEST_F(Cg, ApplyIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    gko::test::make_spd(mtx.get());
+    gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(hip);
diff --git a/hip/test/solver/fcg_kernels.cpp b/hip/test/solver/fcg_kernels.cpp
index ca86775ee94..7a97115a998 100644
--- a/hip/test/solver/fcg_kernels.cpp
+++ b/hip/test/solver/fcg_kernels.cpp
@@ -220,7 +220,7 @@ TEST_F(Fcg, HipFcgStep2IsEquivalentToRef)
 TEST_F(Fcg, ApplyIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    gko::test::make_spd(mtx.get());
+    gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(hip);
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index 3ce1b087fd8..5729d01785f 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -200,6 +200,7 @@ void assign_to_exist_agg(std::shared_ptr<const OmpExecutor> exec,
                        ? intermediate_agg.get_data()
                        : agg.get_data();
     const auto diag_vals = diag->get_const_values();
+#pragma omp parallel for
     for (IndexType row = 0; row < agg.get_num_elems(); row++) {
         if (agg_const_val[row] != -1) {
             continue;
diff --git a/omp/test/multigrid/amgx_pgm_kernels.cpp b/omp/test/multigrid/amgx_pgm_kernels.cpp
index e0b063de0bd..a452c9783ce 100644
--- a/omp/test/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/test/multigrid/amgx_pgm_kernels.cpp
@@ -105,7 +105,7 @@ class AmgxPgm : public ::testing::Test {
         weight->convert_to(weight_csr.get());
         weight_diag = weight_csr->extract_diagonal();
         auto system_dense = gen_mtx(m, m);
-        gko::test::make_spd(system_dense.get());
+        gko::test::make_hpd(system_dense.get());
         system_mtx = Csr::create(ref);
         system_dense->convert_to(system_mtx.get());
 
diff --git a/omp/test/solver/bicg_kernels.cpp b/omp/test/solver/bicg_kernels.cpp
index 2018fa5f788..5049ab9ec10 100644
--- a/omp/test/solver/bicg_kernels.cpp
+++ b/omp/test/solver/bicg_kernels.cpp
@@ -238,7 +238,7 @@ TEST_F(Bicg, OmpBicgStep2IsEquivalentToRef)
 TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    gko::test::make_spd(mtx.get());
+    gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(omp);
diff --git a/omp/test/solver/cg_kernels.cpp b/omp/test/solver/cg_kernels.cpp
index 1e86956602a..0a5014aad64 100644
--- a/omp/test/solver/cg_kernels.cpp
+++ b/omp/test/solver/cg_kernels.cpp
@@ -206,7 +206,7 @@ TEST_F(Cg, OmpCgStep2IsEquivalentToRef)
 TEST_F(Cg, ApplyIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    gko::test::make_spd(mtx.get());
+    gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(omp);
diff --git a/omp/test/solver/fcg_kernels.cpp b/omp/test/solver/fcg_kernels.cpp
index 632a2c80808..85fd4cfebfd 100644
--- a/omp/test/solver/fcg_kernels.cpp
+++ b/omp/test/solver/fcg_kernels.cpp
@@ -219,7 +219,7 @@ TEST_F(Fcg, OmpFcgStep2IsEquivalentToRef)
 TEST_F(Fcg, ApplyIsEquivalentToRef)
 {
     auto mtx = gen_mtx(50, 50);
-    gko::test::make_spd(mtx.get());
+    gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
     auto d_mtx = Mtx::create(omp);

From 96281b2be41c5e4fde421d5bb2fb7af597ec0c5b Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 9 Apr 2021 02:34:18 +0800
Subject: [PATCH 0672/1008] Review update

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 common/multigrid/amgx_pgm_kernels.hpp.inc | 21 +++++++++++----------
 core/test/utils/matrix_utils.hpp          |  2 +-
 cuda/multigrid/amgx_pgm_kernels.cu        |  1 +
 cuda/test/multigrid/amgx_pgm_kernels.cpp  |  4 ++--
 hip/multigrid/amgx_pgm_kernels.hip.cpp    |  1 +
 hip/test/multigrid/amgx_pgm_kernels.cpp   |  4 ++--
 omp/multigrid/amgx_pgm_kernels.cpp        | 14 +++++++-------
 omp/test/multigrid/amgx_pgm_kernels.cpp   |  4 ++--
 reference/multigrid/amgx_pgm_kernels.cpp  | 14 +++++++-------
 9 files changed, 34 insertions(+), 31 deletions(-)

diff --git a/common/multigrid/amgx_pgm_kernels.hpp.inc b/common/multigrid/amgx_pgm_kernels.hpp.inc
index 9c2d434f508..623a8945a44 100644
--- a/common/multigrid/amgx_pgm_kernels.hpp.inc
+++ b/common/multigrid/amgx_pgm_kernels.hpp.inc
@@ -122,13 +122,13 @@ __global__
         }
         auto weight = weight_vals[idx] / max(abs(diag[row]), abs(diag[col]));
         if (agg[col] == -1 &&
-            (weight > max_weight_unagg ||
-             (weight == max_weight_unagg && col > strongest_unagg))) {
+            thrust::tie(weight, col) >
+                thrust::tie(max_weight_unagg, strongest_unagg)) {
             max_weight_unagg = weight;
             strongest_unagg = col;
         } else if (agg[col] != -1 &&
-                   (weight > max_weight_agg ||
-                    (weight == max_weight_agg && col > strongest_agg))) {
+                   thrust::tie(weight, col) >
+                       thrust::tie(max_weight_agg, strongest_agg)) {
             max_weight_agg = weight;
             strongest_agg = col;
         }
@@ -173,8 +173,8 @@ __global__
         }
         auto weight = weight_vals[idx] / max(abs(diag[row]), abs(diag[col]));
         if (agg_const_val[col] != -1 &&
-            (weight > max_weight_agg ||
-             (weight == max_weight_agg && col > strongest_agg))) {
+            thrust::tie(weight, col) >
+                thrust::tie(max_weight_agg, strongest_agg)) {
             max_weight_agg = weight;
             strongest_agg = col;
         }
@@ -209,8 +209,8 @@ __global__
         }
         auto weight = weight_vals[idx] / max(abs(diag[row]), abs(diag[col]));
         if (agg_val[col] != -1 &&
-            (weight > max_weight_agg ||
-             (weight == max_weight_agg && col > strongest_agg))) {
+            thrust::tie(weight, col) >
+                thrust::tie(max_weight_agg, strongest_agg)) {
             max_weight_agg = weight;
             strongest_agg = col;
         }
@@ -239,6 +239,7 @@ __global__ __launch_bounds__(default_block_size) void get_source_row_map_kernel(
     row_map[row] = atomic_add(result_row_ptrs + result_idx, num_elems);
 }
 
+
 template <typename ValueType, typename IndexType>
 __global__ __launch_bounds__(default_block_size) void move_row_kernel(
     const size_type source_nrows, const IndexType *__restrict__ agg_val,
@@ -316,10 +317,10 @@ __global__ __launch_bounds__(default_block_size) void copy_to_coarse_kernel(
         return;
     }
     auto temp_i = temp_row_ptrs[row];
-    for (auto i = coarse_row_ptrs[row]; i < coarse_row_ptrs[row + 1];
-         i++, temp_i++) {
+    for (auto i = coarse_row_ptrs[row]; i < coarse_row_ptrs[row + 1]; i++) {
         coarse_col_idxs[i] = temp_col_idxs[temp_i];
         coarse_values[i] = temp_values[temp_i];
+        temp_i++;
     }
 }
 
diff --git a/core/test/utils/matrix_utils.hpp b/core/test/utils/matrix_utils.hpp
index 90aa3ea1d6e..dc6586f07b7 100644
--- a/core/test/utils/matrix_utils.hpp
+++ b/core/test/utils/matrix_utils.hpp
@@ -96,7 +96,7 @@ void make_hermitian(matrix::Dense<ValueType> *mtx)
  * the summation among the absoulue value of the row's elements. When ratio is
  * larger than 1, the result will be strictly diagonal dominant matrix except
  * for the empty row. When ratio is 1, the result will be diagonal dominant
- * matirx.
+ * matrix.
  *
  * @tparam ValueType  valuetype of Dense matrix to process
  *
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index 1560d7bf7f7..834f8d17f31 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <cuda.h>
 #include <cusparse.h>
+#include <thrust/tuple.h>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
index 524f4a09f3d..6de292a8a7f 100644
--- a/cuda/test/multigrid/amgx_pgm_kernels.cpp
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -140,7 +140,7 @@ class AmgxPgm : public ::testing::Test {
     void make_weight(Mtx *mtx)
     {
         gko::test::make_symmetric(mtx);
-        // only works for realvalue cases
+        // only works for real value cases.
         mtx->compute_absolute_inplace();
         gko::test::make_diag_dominant(mtx);
     }
@@ -213,7 +213,7 @@ TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
 
     ASSERT_EQ(d_num_agg, num_agg);
     GKO_ASSERT_ARRAY_EQ(d_agg, agg);
-    ASSERT_LE(num_agg, 300);
+    ASSERT_LE(num_agg, n);
 }
 
 
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index 2199f53e5c3..57498bca7df 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <hip/hip_runtime.h>
 #include <hipsparse.h>
+#include <thrust/tuple.h>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
diff --git a/hip/test/multigrid/amgx_pgm_kernels.cpp b/hip/test/multigrid/amgx_pgm_kernels.cpp
index ce63db8a8dd..dd1ae6032c0 100644
--- a/hip/test/multigrid/amgx_pgm_kernels.cpp
+++ b/hip/test/multigrid/amgx_pgm_kernels.cpp
@@ -139,7 +139,7 @@ class AmgxPgm : public ::testing::Test {
     void make_weight(Mtx *mtx)
     {
         gko::test::make_symmetric(mtx);
-        // only works for realvalue cases
+        // only works for real value cases
         mtx->compute_absolute_inplace();
         gko::test::make_diag_dominant(mtx);
     }
@@ -212,7 +212,7 @@ TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
 
     ASSERT_EQ(d_num_agg, num_agg);
     GKO_ASSERT_ARRAY_EQ(d_agg, agg);
-    ASSERT_LE(num_agg, 300);
+    ASSERT_LE(num_agg, n);
 }
 
 
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index 5729d01785f..408478242dc 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <memory>
+#include <tuple>
 
 
 #include <omp.h>
@@ -154,14 +155,13 @@ void find_strongest_neighbor(
                 auto weight =
                     vals[idx] / max(abs(diag_vals[row]), abs(diag_vals[col]));
                 if (agg.get_const_data()[col] == -1 &&
-                    (weight > max_weight_unagg ||
-                     (weight == max_weight_unagg && col > strongest_unagg))) {
+                    std::tie(weight, col) >
+                        std::tie(max_weight_unagg, strongest_unagg)) {
                     max_weight_unagg = weight;
                     strongest_unagg = col;
                 } else if (agg.get_const_data()[col] != -1 &&
-                           (weight > max_weight_agg ||
-                            (weight == max_weight_agg &&
-                             col > strongest_agg))) {
+                           std::tie(weight, col) >
+                               std::tie(max_weight_agg, strongest_agg)) {
                     max_weight_agg = weight;
                     strongest_agg = col;
                 }
@@ -215,8 +215,8 @@ void assign_to_exist_agg(std::shared_ptr<const OmpExecutor> exec,
             auto weight =
                 vals[idx] / max(abs(diag_vals[row]), abs(diag_vals[col]));
             if (agg_const_val[col] != -1 &&
-                (weight > max_weight_agg ||
-                 (weight == max_weight_agg && col > strongest_agg))) {
+                std::tie(weight, col) >
+                    std::tie(max_weight_agg, strongest_agg)) {
                 max_weight_agg = weight;
                 strongest_agg = col;
             }
diff --git a/omp/test/multigrid/amgx_pgm_kernels.cpp b/omp/test/multigrid/amgx_pgm_kernels.cpp
index a452c9783ce..bd20b3749b1 100644
--- a/omp/test/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/test/multigrid/amgx_pgm_kernels.cpp
@@ -130,7 +130,7 @@ class AmgxPgm : public ::testing::Test {
     void make_weight(Mtx *mtx)
     {
         gko::test::make_symmetric(mtx);
-        // it is only works for realvalue case.
+        // it is only works for real value case.
         mtx->compute_absolute_inplace();
         gko::test::make_diag_dominant(mtx);
     }
@@ -203,7 +203,7 @@ TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
 
     ASSERT_EQ(d_num_agg, num_agg);
     GKO_ASSERT_ARRAY_EQ(d_agg, agg);
-    ASSERT_LE(num_agg, 300);
+    ASSERT_LE(num_agg, n);
 }
 
 
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index d5b1ddbebb1..16002259fcf 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <memory>
+#include <tuple>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
@@ -148,14 +149,13 @@ void find_strongest_neighbor(
                 auto weight =
                     vals[idx] / max(abs(diag_vals[row]), abs(diag_vals[col]));
                 if (agg.get_const_data()[col] == -1 &&
-                    (weight > max_weight_unagg ||
-                     (weight == max_weight_unagg && col > strongest_unagg))) {
+                    std::tie(weight, col) >
+                        std::tie(max_weight_unagg, strongest_unagg)) {
                     max_weight_unagg = weight;
                     strongest_unagg = col;
                 } else if (agg.get_const_data()[col] != -1 &&
-                           (weight > max_weight_agg ||
-                            (weight == max_weight_agg &&
-                             col > strongest_agg))) {
+                           std::tie(weight, col) >
+                               std::tie(max_weight_agg, strongest_agg)) {
                     max_weight_agg = weight;
                     strongest_agg = col;
                 }
@@ -208,8 +208,8 @@ void assign_to_exist_agg(std::shared_ptr<const ReferenceExecutor> exec,
             auto weight =
                 vals[idx] / max(abs(diag_vals[row]), abs(diag_vals[col]));
             if (agg_const_val[col] != -1 &&
-                (weight > max_weight_agg ||
-                 (weight == max_weight_agg && col > strongest_agg))) {
+                std::tie(weight, col) >
+                    std::tie(max_weight_agg, strongest_agg)) {
                 max_weight_agg = weight;
                 strongest_agg = col;
             }

From 4ce5c29954dc8cd2f6dfcce20a9319cda7be715c Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 27 Apr 2021 02:46:32 +0800
Subject: [PATCH 0673/1008] improve renumber and pass rstr/prlg into generate

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 common/multigrid/amgx_pgm_kernels.hpp.inc     |  4 +-
 core/multigrid/amgx_pgm.cpp                   | 23 +++----
 core/multigrid/amgx_pgm_kernels.hpp           | 14 ++--
 cuda/multigrid/amgx_pgm_kernels.cu            | 10 +--
 cuda/test/multigrid/amgx_pgm_kernels.cpp      | 64 +++++++++++++++----
 dpcpp/multigrid/amgx_pgm_kernels.dp.cpp       | 12 ++--
 hip/multigrid/amgx_pgm_kernels.hip.cpp        | 10 +--
 hip/test/multigrid/amgx_pgm_kernels.cpp       | 64 +++++++++++++++----
 omp/multigrid/amgx_pgm_kernels.cpp            | 13 ++--
 omp/test/multigrid/amgx_pgm_kernels.cpp       | 64 +++++++++++++++----
 reference/multigrid/amgx_pgm_kernels.cpp      |  8 ++-
 reference/test/multigrid/amgx_pgm_kernels.cpp | 12 ++--
 12 files changed, 218 insertions(+), 80 deletions(-)

diff --git a/common/multigrid/amgx_pgm_kernels.hpp.inc b/common/multigrid/amgx_pgm_kernels.hpp.inc
index 623a8945a44..3ec6bf81a7b 100644
--- a/common/multigrid/amgx_pgm_kernels.hpp.inc
+++ b/common/multigrid/amgx_pgm_kernels.hpp.inc
@@ -77,7 +77,9 @@ __global__ __launch_bounds__(default_block_size) void fill_agg_kernel(
     if (tidx >= num) {
         return;
     }
-    result[index[tidx]] = 1;
+    // agg_vals[i] == i always holds in the aggregated group whose identifier is
+    // i because we use the index of element as the aggregated group identifier.
+    result[tidx] = (index[tidx] == tidx);
 }
 
 
diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index 1803c4252d2..d9fac590691 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -74,15 +74,17 @@ namespace {
 template <typename ValueType, typename IndexType>
 std::unique_ptr<LinOp> amgx_pgm_generate(
     std::shared_ptr<const Executor> exec,
-    const matrix::Csr<ValueType, IndexType> *source, const size_type num_agg,
-    const Array<IndexType> &agg)
+    const matrix::Csr<ValueType, IndexType> *source,
+    const matrix::Csr<ValueType, IndexType> *prolong_op,
+    const matrix::Csr<ValueType, IndexType> *restrict_op)
 {
+    auto num_agg = prolong_op->get_size()[1];
     auto coarse = matrix::Csr<ValueType, IndexType>::create(
         exec, dim<2>{num_agg, num_agg}, 0, source->get_strategy());
     auto temp = matrix::Csr<ValueType, IndexType>::create(
         exec, dim<2>{num_agg, num_agg}, source->get_num_stored_elements());
-    exec->run(amgx_pgm::make_amgx_pgm_generate(source, agg, coarse.get(),
-                                               temp.get()));
+    exec->run(amgx_pgm::make_amgx_pgm_generate(source, prolong_op, restrict_op,
+                                               coarse.get(), temp.get()));
     return std::move(coarse);
 }
 
@@ -158,13 +160,8 @@ void AmgxPgm<ValueType, IndexType>::generate()
     // Renumber the index
     exec->run(amgx_pgm::make_renumber(agg_, &num_agg));
 
-    // Construct the coarse matrix
-    auto coarse_matrix =
-        share(amgx_pgm_generate(exec, amgxpgm_op, num_agg, agg_));
-    // this->set_multigrid_level(system_matrix_, coarse_matrix);
-    auto coarse_dim = coarse_matrix->get_size()[0];
+    auto coarse_dim = num_agg;
     auto fine_dim = system_matrix_->get_size()[0];
-
     // TODO: prolong_op can be done with lightway format
     auto prolong_op = share(
         matrix_type::create(exec, gko::dim<2>{fine_dim, coarse_dim}, fine_dim));
@@ -175,7 +172,11 @@ void AmgxPgm<ValueType, IndexType>::generate()
     exec->run(amgx_pgm::make_fill_array(prolong_op->get_values(), fine_dim,
                                         one<ValueType>()));
     // TODO: implement the restrict_op from aggregation.
-    auto restrict_op = share(prolong_op->transpose());
+    auto restrict_op = gko::as<matrix_type>(share(prolong_op->transpose()));
+
+    // Construct the coarse matrix
+    auto coarse_matrix = share(amgx_pgm_generate(
+        exec, amgxpgm_op, prolong_op.get(), restrict_op.get()));
     this->set_multigrid_level(prolong_op, coarse_matrix, restrict_op);
 }
 
diff --git a/core/multigrid/amgx_pgm_kernels.hpp b/core/multigrid/amgx_pgm_kernels.hpp
index e0006be63a7..5bbb48b9594 100644
--- a/core/multigrid/amgx_pgm_kernels.hpp
+++ b/core/multigrid/amgx_pgm_kernels.hpp
@@ -78,12 +78,14 @@ namespace amgx_pgm {
         const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg, \
         Array<IndexType> &intermediate_agg)
 
-#define GKO_DECLARE_AMGX_PGM_GENERATE(ValueType, IndexType)                 \
-    void amgx_pgm_generate(std::shared_ptr<const DefaultExecutor> exec,     \
-                           const matrix::Csr<ValueType, IndexType> *source, \
-                           const Array<IndexType> &agg,                     \
-                           matrix::Csr<ValueType, IndexType> *coarse,       \
-                           matrix::Csr<ValueType, IndexType> *temp)
+#define GKO_DECLARE_AMGX_PGM_GENERATE(ValueType, IndexType)   \
+    void amgx_pgm_generate(                                   \
+        std::shared_ptr<const DefaultExecutor> exec,          \
+        const matrix::Csr<ValueType, IndexType> *source,      \
+        const matrix::Csr<ValueType, IndexType> *prolong_op,  \
+        const matrix::Csr<ValueType, IndexType> *restrict_op, \
+        matrix::Csr<ValueType, IndexType> *coarse,            \
+        matrix::Csr<ValueType, IndexType> *temp)
 
 #define GKO_DECLARE_ALL_AS_TEMPLATES                                    \
     template <typename IndexType>                                       \
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index 834f8d17f31..03f657573a7 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -111,8 +111,6 @@ void renumber(std::shared_ptr<const CudaExecutor> exec, Array<IndexType> &agg,
 {
     const auto num = agg.get_num_elems();
     Array<IndexType> agg_map(exec, num + 1);
-    components::fill_array(exec, agg_map.get_data(), agg_map.get_num_elems(),
-                           zero<IndexType>());
     const dim3 grid(ceildiv(num, default_block_size));
     kernel::fill_agg_kernel<<<grid, default_block_size>>>(
         num, agg.get_const_data(), agg_map.get_data());
@@ -177,10 +175,12 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void amgx_pgm_generate(std::shared_ptr<const CudaExecutor> exec,
                        const matrix::Csr<ValueType, IndexType> *source,
-                       const Array<IndexType> &agg,
+                       const matrix::Csr<ValueType, IndexType> *prolong_op,
+                       const matrix::Csr<ValueType, IndexType> *restrict_op,
                        matrix::Csr<ValueType, IndexType> *coarse,
                        matrix::Csr<ValueType, IndexType> *temp)
 {
+    const auto agg_const_val = prolong_op->get_const_col_idxs();
     const auto source_nrows = source->get_size()[0];
     const auto source_nnz = source->get_num_stored_elements();
     const auto coarse_nrows = coarse->get_size()[0];
@@ -192,13 +192,13 @@ void amgx_pgm_generate(std::shared_ptr<const CudaExecutor> exec,
     dim3 grid(ceildiv(source_nrows, default_block_size));
     // agg source_row (for row size) coarse row source map
     kernel::get_source_row_map_kernel<<<grid, default_block_size>>>(
-        source_nrows, agg.get_const_data(), source->get_const_row_ptrs(),
+        source_nrows, agg_const_val, source->get_const_row_ptrs(),
         temp->get_row_ptrs(), row_map.get_data());
     // prefix sum of temp_row_ptrs
     components::prefix_sum(exec, temp->get_row_ptrs(), coarse_nrows + 1);
     // copy source -> to coarse and change column index
     kernel::move_row_kernel<<<grid, default_block_size>>>(
-        source_nrows, agg.get_const_data(), row_map.get_const_data(),
+        source_nrows, agg_const_val, row_map.get_const_data(),
         source->get_const_row_ptrs(), source->get_const_col_idxs(),
         as_cuda_type(source->get_const_values()), temp->get_const_row_ptrs(),
         temp->get_col_idxs(), as_cuda_type(temp->get_values()));
diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
index 6de292a8a7f..b92934ce089 100644
--- a/cuda/test/multigrid/amgx_pgm_kernels.cpp
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -98,13 +98,35 @@ class AmgxPgm : public ::testing::Test {
             ref);
     }
 
+    gko::Array<index_type> gen_agg_array(gko::size_type num,
+                                         gko::size_type num_agg)
+    {
+        auto agg_array = gen_array(num, 0, num_agg - 1);
+        auto agg_array_val = agg_array.get_data();
+        std::vector<index_type> select_agg(num);
+        std::iota(select_agg.begin(), select_agg.end(), 0);
+        // use the first num_agg item as the aggregated index.
+        std::shuffle(select_agg.begin(), select_agg.end(), rand_engine);
+        // the value of agg_array is the i-th of aggregate group
+        for (gko::size_type i = 0; i < num; i++) {
+            agg_array_val[i] = select_agg[agg_array_val[i]];
+        }
+        // the aggregated group must contain the identifier-th element
+        // agg_val[i] == i holds in the aggregated group whose identifier is i
+        for (gko::size_type i = 0; i < num_agg; i++) {
+            auto agg_idx = select_agg[i];
+            agg_array_val[agg_idx] = agg_idx;
+        }
+        return agg_array;
+    }
+
     void initialize_data()
     {
-        int m = 597;
+        m = 597;
         n = 300;
         int nrhs = 3;
 
-        agg = gen_array(m, 0, n - 1);
+        agg = gen_agg_array(m, n);
         unfinished_agg = gen_array(m, -1, n - 1);
         strongest_neighbor = gen_array(m, 0, n - 1);
         coarse_vector = gen_mtx(n, nrhs);
@@ -171,6 +193,7 @@ class AmgxPgm : public ::testing::Test {
     std::shared_ptr<Csr> d_system_mtx;
 
     gko::size_type n;
+    gko::size_type m;
 };
 
 
@@ -193,8 +216,10 @@ TEST_F(AmgxPgm, CountUnaggIsEquivalentToRef)
     index_type num_unagg;
     index_type d_num_unagg;
 
-    gko::kernels::reference::amgx_pgm::count_unagg(ref, agg, &num_unagg);
-    gko::kernels::cuda::amgx_pgm::count_unagg(cuda, d_agg, &d_num_unagg);
+    gko::kernels::reference::amgx_pgm::count_unagg(ref, unfinished_agg,
+                                                   &num_unagg);
+    gko::kernels::cuda::amgx_pgm::count_unagg(cuda, d_unfinished_agg,
+                                              &d_num_unagg);
 
     ASSERT_EQ(d_num_unagg, num_unagg);
 }
@@ -203,8 +228,6 @@ TEST_F(AmgxPgm, CountUnaggIsEquivalentToRef)
 TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
 {
     initialize_data();
-    auto x = unfinished_agg;
-    auto d_x = d_unfinished_agg;
     index_type num_agg;
     index_type d_num_agg;
 
@@ -213,7 +236,7 @@ TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
 
     ASSERT_EQ(d_num_agg, num_agg);
     GKO_ASSERT_ARRAY_EQ(d_agg, agg);
-    ASSERT_LE(num_agg, n);
+    ASSERT_EQ(num_agg, n);
 }
 
 
@@ -271,15 +294,34 @@ TEST_F(AmgxPgm, GenerateMtxIsEquivalentToRef)
     auto csr_coarse = Csr::create(ref, gko::dim<2>{n, n}, 0);
     auto d_csr_coarse = Csr::create(cuda, gko::dim<2>{n, n}, 0);
     auto csr_temp = Csr::create(ref, gko::dim<2>{n, n},
-                                weight_csr->get_num_stored_elements());
+                                system_mtx->get_num_stored_elements());
     auto d_csr_temp = Csr::create(cuda, gko::dim<2>{n, n},
-                                  d_weight_csr->get_num_stored_elements());
+                                  d_system_mtx->get_num_stored_elements());
+    index_type num_agg;
+    // renumber again
+    gko::kernels::reference::amgx_pgm::renumber(ref, agg, &num_agg);
+    auto prolong_op = Csr::create(ref, gko::dim<2>{m, n}, m);
+    for (int i = 0; i < m; i++) {
+        prolong_op->get_col_idxs()[i] = agg.get_const_data()[i];
+    }
+    std::iota(prolong_op->get_row_ptrs(), prolong_op->get_row_ptrs() + m + 1,
+              0);
+    std::fill_n(prolong_op->get_values(), m, gko::one<value_type>());
+    auto restrict_op = gko::as<Csr>(prolong_op->transpose());
+    auto d_prolong_op = Csr::create(cuda);
+    auto d_restrict_op = Csr::create(cuda);
+    d_prolong_op->copy_from(prolong_op.get());
+    d_restrict_op->copy_from(restrict_op.get());
 
     gko::kernels::cuda::amgx_pgm::amgx_pgm_generate(
-        cuda, d_weight_csr.get(), d_agg, d_csr_coarse.get(), d_csr_temp.get());
+        cuda, d_system_mtx.get(), d_prolong_op.get(), d_restrict_op.get(),
+        d_csr_coarse.get(), d_csr_temp.get());
     gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
-        ref, weight_csr.get(), agg, csr_coarse.get(), csr_temp.get());
+        ref, system_mtx.get(), prolong_op.get(), restrict_op.get(),
+        csr_coarse.get(), csr_temp.get());
 
+    // it should be checked already in renumber
+    GKO_ASSERT_EQ(num_agg, n);
     GKO_ASSERT_MTX_NEAR(d_csr_coarse, csr_coarse, 1e-14);
 }
 
diff --git a/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp b/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
index 32817e5e834..283127ce6d5 100644
--- a/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
+++ b/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
@@ -101,11 +101,13 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void amgx_pgm_generate(
-    std::shared_ptr<const DpcppExecutor> exec,
-    const matrix::Csr<ValueType, IndexType> *source,
-    const Array<IndexType> &agg, matrix::Csr<ValueType, IndexType> *coarse,
-    matrix::Csr<ValueType, IndexType> *temp) GKO_NOT_IMPLEMENTED;
+void amgx_pgm_generate(std::shared_ptr<const DpcppExecutor> exec,
+                       const matrix::Csr<ValueType, IndexType> *source,
+                       const matrix::Csr<ValueType, IndexType> *prolong_op,
+                       const matrix::Csr<ValueType, IndexType> *restrict_op,
+                       matrix::Csr<ValueType, IndexType> *coarse,
+                       matrix::Csr<ValueType, IndexType> *temp)
+    GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
 
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index 57498bca7df..e2ed35d42e9 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -113,8 +113,6 @@ void renumber(std::shared_ptr<const HipExecutor> exec, Array<IndexType> &agg,
 {
     const auto num = agg.get_num_elems();
     Array<IndexType> agg_map(exec, num + 1);
-    components::fill_array(exec, agg_map.get_data(), agg_map.get_num_elems(),
-                           zero<IndexType>());
     const dim3 grid(ceildiv(num, default_block_size));
     hipLaunchKernelGGL(kernel::fill_agg_kernel, dim3(grid),
                        dim3(default_block_size), 0, 0, num,
@@ -188,10 +186,12 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void amgx_pgm_generate(std::shared_ptr<const HipExecutor> exec,
                        const matrix::Csr<ValueType, IndexType> *source,
-                       const Array<IndexType> &agg,
+                       const matrix::Csr<ValueType, IndexType> *prolong_op,
+                       const matrix::Csr<ValueType, IndexType> *restrict_op,
                        matrix::Csr<ValueType, IndexType> *coarse,
                        matrix::Csr<ValueType, IndexType> *temp)
 {
+    const auto agg_const_val = prolong_op->get_const_col_idxs();
     const auto source_nrows = source->get_size()[0];
     const auto source_nnz = source->get_num_stored_elements();
     const auto coarse_nrows = coarse->get_size()[0];
@@ -204,14 +204,14 @@ void amgx_pgm_generate(std::shared_ptr<const HipExecutor> exec,
     // agg source_row (for row size) coarse row source map
     hipLaunchKernelGGL(kernel::get_source_row_map_kernel, dim3(grid),
                        dim3(default_block_size), 0, 0, source_nrows,
-                       agg.get_const_data(), source->get_const_row_ptrs(),
+                       agg_const_val, source->get_const_row_ptrs(),
                        temp->get_row_ptrs(), row_map.get_data());
     // prefix sum of temp_row_ptrs
     components::prefix_sum(exec, temp->get_row_ptrs(), coarse_nrows + 1);
     // copy source -> to coarse and change column index
     hipLaunchKernelGGL(
         kernel::move_row_kernel, dim3(grid), dim3(default_block_size), 0, 0,
-        source_nrows, agg.get_const_data(), row_map.get_const_data(),
+        source_nrows, agg_const_val, row_map.get_const_data(),
         source->get_const_row_ptrs(), source->get_const_col_idxs(),
         as_hip_type(source->get_const_values()), temp->get_const_row_ptrs(),
         temp->get_col_idxs(), as_hip_type(temp->get_values()));
diff --git a/hip/test/multigrid/amgx_pgm_kernels.cpp b/hip/test/multigrid/amgx_pgm_kernels.cpp
index dd1ae6032c0..73909e522aa 100644
--- a/hip/test/multigrid/amgx_pgm_kernels.cpp
+++ b/hip/test/multigrid/amgx_pgm_kernels.cpp
@@ -97,13 +97,35 @@ class AmgxPgm : public ::testing::Test {
             ref);
     }
 
+    gko::Array<index_type> gen_agg_array(gko::size_type num,
+                                         gko::size_type num_agg)
+    {
+        auto agg_array = gen_array(num, 0, num_agg - 1);
+        auto agg_array_val = agg_array.get_data();
+        std::vector<index_type> select_agg(num);
+        std::iota(select_agg.begin(), select_agg.end(), 0);
+        // use the first num_agg item as the aggregated index.
+        std::shuffle(select_agg.begin(), select_agg.end(), rand_engine);
+        // the value of agg_array is the i-th of aggregate group
+        for (gko::size_type i = 0; i < num; i++) {
+            agg_array_val[i] = select_agg[agg_array_val[i]];
+        }
+        // the aggregated group must contain the identifier-th element
+        // agg_val[i] == i holds in the aggregated group whose identifier is i
+        for (gko::size_type i = 0; i < num_agg; i++) {
+            auto agg_idx = select_agg[i];
+            agg_array_val[agg_idx] = agg_idx;
+        }
+        return agg_array;
+    }
+
     void initialize_data()
     {
-        int m = 597;
+        m = 597;
         n = 300;
         int nrhs = 3;
 
-        agg = gen_array(m, 0, n - 1);
+        agg = gen_agg_array(m, n);
         unfinished_agg = gen_array(m, -1, n - 1);
         strongest_neighbor = gen_array(m, 0, n - 1);
         coarse_vector = gen_mtx(n, nrhs);
@@ -170,6 +192,7 @@ class AmgxPgm : public ::testing::Test {
     std::shared_ptr<Csr> d_system_mtx;
 
     gko::size_type n;
+    gko::size_type m;
 };
 
 
@@ -192,8 +215,10 @@ TEST_F(AmgxPgm, CountUnaggIsEquivalentToRef)
     index_type num_unagg;
     index_type d_num_unagg;
 
-    gko::kernels::reference::amgx_pgm::count_unagg(ref, agg, &num_unagg);
-    gko::kernels::hip::amgx_pgm::count_unagg(hip, d_agg, &d_num_unagg);
+    gko::kernels::reference::amgx_pgm::count_unagg(ref, unfinished_agg,
+                                                   &num_unagg);
+    gko::kernels::hip::amgx_pgm::count_unagg(hip, d_unfinished_agg,
+                                             &d_num_unagg);
 
     ASSERT_EQ(d_num_unagg, num_unagg);
 }
@@ -202,8 +227,6 @@ TEST_F(AmgxPgm, CountUnaggIsEquivalentToRef)
 TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
 {
     initialize_data();
-    auto x = unfinished_agg;
-    auto d_x = d_unfinished_agg;
     index_type num_agg;
     index_type d_num_agg;
 
@@ -212,7 +235,7 @@ TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
 
     ASSERT_EQ(d_num_agg, num_agg);
     GKO_ASSERT_ARRAY_EQ(d_agg, agg);
-    ASSERT_LE(num_agg, n);
+    ASSERT_EQ(num_agg, n);
 }
 
 
@@ -270,15 +293,34 @@ TEST_F(AmgxPgm, GenerateMtxIsEquivalentToRef)
     auto csr_coarse = Csr::create(ref, gko::dim<2>{n, n}, 0);
     auto d_csr_coarse = Csr::create(hip, gko::dim<2>{n, n}, 0);
     auto csr_temp = Csr::create(ref, gko::dim<2>{n, n},
-                                weight_csr->get_num_stored_elements());
+                                system_mtx->get_num_stored_elements());
     auto d_csr_temp = Csr::create(hip, gko::dim<2>{n, n},
-                                  d_weight_csr->get_num_stored_elements());
+                                  d_system_mtx->get_num_stored_elements());
+    index_type num_agg;
+    // renumber again
+    gko::kernels::reference::amgx_pgm::renumber(ref, agg, &num_agg);
+    auto prolong_op = Csr::create(ref, gko::dim<2>{m, n}, m);
+    for (int i = 0; i < m; i++) {
+        prolong_op->get_col_idxs()[i] = agg.get_const_data()[i];
+    }
+    std::iota(prolong_op->get_row_ptrs(), prolong_op->get_row_ptrs() + m + 1,
+              0);
+    std::fill_n(prolong_op->get_values(), m, gko::one<value_type>());
+    auto restrict_op = gko::as<Csr>(prolong_op->transpose());
+    auto d_prolong_op = Csr::create(hip);
+    auto d_restrict_op = Csr::create(hip);
+    d_prolong_op->copy_from(prolong_op.get());
+    d_restrict_op->copy_from(restrict_op.get());
 
     gko::kernels::hip::amgx_pgm::amgx_pgm_generate(
-        hip, d_weight_csr.get(), d_agg, d_csr_coarse.get(), d_csr_temp.get());
+        hip, d_system_mtx.get(), d_prolong_op.get(), d_restrict_op.get(),
+        d_csr_coarse.get(), d_csr_temp.get());
     gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
-        ref, weight_csr.get(), agg, csr_coarse.get(), csr_temp.get());
+        ref, system_mtx.get(), prolong_op.get(), restrict_op.get(),
+        csr_coarse.get(), csr_temp.get());
 
+    // it should be checked already in renumber
+    GKO_ASSERT_EQ(num_agg, n);
     GKO_ASSERT_MTX_NEAR(d_csr_coarse, csr_coarse, 1e-14);
 }
 
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index 408478242dc..dc6329d982c 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -113,10 +113,11 @@ void renumber(std::shared_ptr<const OmpExecutor> exec, Array<IndexType> &agg,
     Array<IndexType> agg_map(exec, num + 1);
     auto agg_vals = agg.get_data();
     auto agg_map_vals = agg_map.get_data();
-    components::fill_array(exec, agg_map_vals, num + 1, zero<IndexType>());
+    // agg_vals[i] == i always holds in the aggregated group whose identifier is
+    // i because we use the index of element as the aggregated group identifier.
 #pragma omp parallel for
     for (size_type i = 0; i < num; i++) {
-        agg_map_vals[agg_vals[i]] = 1;
+        agg_map_vals[i] = (agg_vals[i] == i);
     }
     components::prefix_sum(exec, agg_map_vals, num + 1);
 #pragma omp parallel for
@@ -241,11 +242,13 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void amgx_pgm_generate(std::shared_ptr<const OmpExecutor> exec,
                        const matrix::Csr<ValueType, IndexType> *source,
-                       const Array<IndexType> &agg,
+                       const matrix::Csr<ValueType, IndexType> *prolong_op,
+                       const matrix::Csr<ValueType, IndexType> *restrict_op,
                        matrix::Csr<ValueType, IndexType> *coarse,
                        matrix::Csr<ValueType, IndexType> *temp)
 {
     // agg[i] -> I, agg[j] -> J
+    const auto agg_const_val = prolong_op->get_const_col_idxs();
     const auto coarse_nrows = coarse->get_size()[0];
     const auto source_nrows = source->get_size()[0];
     const auto source_row_ptrs = source->get_const_row_ptrs();
@@ -254,9 +257,9 @@ void amgx_pgm_generate(std::shared_ptr<const OmpExecutor> exec,
     vector<map<IndexType, ValueType>> row_list(
         source_nrows, map<IndexType, ValueType>{exec}, exec);
     for (size_type i = 0; i < source_nrows; i++) {
-        IndexType row_idx = agg.get_const_data()[i];
+        IndexType row_idx = agg_const_val[i];
         for (auto j = source_row_ptrs[i]; j < source_row_ptrs[i + 1]; j++) {
-            const auto col = agg.get_const_data()[source_col_idxs[j]];
+            const auto col = agg_const_val[source_col_idxs[j]];
             const auto val = source_vals[j];
             row_list[row_idx][col] += val;
         }
diff --git a/omp/test/multigrid/amgx_pgm_kernels.cpp b/omp/test/multigrid/amgx_pgm_kernels.cpp
index bd20b3749b1..91dc2c2fce8 100644
--- a/omp/test/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/test/multigrid/amgx_pgm_kernels.cpp
@@ -88,13 +88,35 @@ class AmgxPgm : public ::testing::Test {
             ref);
     }
 
+    gko::Array<index_type> gen_agg_array(gko::size_type num,
+                                         gko::size_type num_agg)
+    {
+        auto agg_array = gen_array(num, 0, num_agg - 1);
+        auto agg_array_val = agg_array.get_data();
+        std::vector<index_type> select_agg(num);
+        std::iota(select_agg.begin(), select_agg.end(), 0);
+        // use the first num_agg item as the aggregated index.
+        std::shuffle(select_agg.begin(), select_agg.end(), rand_engine);
+        // the value of agg_array is the i-th of aggregate group
+        for (gko::size_type i = 0; i < num; i++) {
+            agg_array_val[i] = select_agg[agg_array_val[i]];
+        }
+        // the aggregated group must contain the identifier-th element
+        // agg_val[i] == i holds in the aggregated group whose identifier is i
+        for (gko::size_type i = 0; i < num_agg; i++) {
+            auto agg_idx = select_agg[i];
+            agg_array_val[agg_idx] = agg_idx;
+        }
+        return agg_array;
+    }
+
     void initialize_data()
     {
-        int m = 597;
+        m = 597;
         n = 300;
         int nrhs = 3;
 
-        agg = gen_array(m, 0, n - 1);
+        agg = gen_agg_array(m, n);
         unfinished_agg = gen_array(m, -1, n - 1);
         strongest_neighbor = gen_array(m, 0, n - 1);
         coarse_vector = gen_mtx(n, nrhs);
@@ -161,6 +183,7 @@ class AmgxPgm : public ::testing::Test {
     std::shared_ptr<Csr> d_system_mtx;
 
     gko::size_type n;
+    gko::size_type m;
 };
 
 
@@ -183,8 +206,10 @@ TEST_F(AmgxPgm, CountUnaggIsEquivalentToRef)
     index_type num_unagg;
     index_type d_num_unagg;
 
-    gko::kernels::reference::amgx_pgm::count_unagg(ref, agg, &num_unagg);
-    gko::kernels::omp::amgx_pgm::count_unagg(omp, d_agg, &d_num_unagg);
+    gko::kernels::reference::amgx_pgm::count_unagg(ref, unfinished_agg,
+                                                   &num_unagg);
+    gko::kernels::omp::amgx_pgm::count_unagg(omp, d_unfinished_agg,
+                                             &d_num_unagg);
 
     ASSERT_EQ(d_num_unagg, num_unagg);
 }
@@ -193,8 +218,6 @@ TEST_F(AmgxPgm, CountUnaggIsEquivalentToRef)
 TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
 {
     initialize_data();
-    auto x = unfinished_agg;
-    auto d_x = d_unfinished_agg;
     index_type num_agg;
     index_type d_num_agg;
 
@@ -203,7 +226,7 @@ TEST_F(AmgxPgm, RenumberIsEquivalentToRef)
 
     ASSERT_EQ(d_num_agg, num_agg);
     GKO_ASSERT_ARRAY_EQ(d_agg, agg);
-    ASSERT_LE(num_agg, n);
+    ASSERT_EQ(num_agg, n);
 }
 
 
@@ -261,15 +284,34 @@ TEST_F(AmgxPgm, GenerateMtxIsEquivalentToRef)
     auto csr_coarse = Csr::create(ref, gko::dim<2>{n, n}, 0);
     auto d_csr_coarse = Csr::create(omp, gko::dim<2>{n, n}, 0);
     auto csr_temp = Csr::create(ref, gko::dim<2>{n, n},
-                                weight_csr->get_num_stored_elements());
+                                system_mtx->get_num_stored_elements());
     auto d_csr_temp = Csr::create(omp, gko::dim<2>{n, n},
-                                  d_weight_csr->get_num_stored_elements());
+                                  d_system_mtx->get_num_stored_elements());
+    index_type num_agg;
+    // renumber again
+    gko::kernels::reference::amgx_pgm::renumber(ref, agg, &num_agg);
+    auto prolong_op = Csr::create(ref, gko::dim<2>{m, n}, m);
+    for (int i = 0; i < m; i++) {
+        prolong_op->get_col_idxs()[i] = agg.get_const_data()[i];
+    }
+    std::iota(prolong_op->get_row_ptrs(), prolong_op->get_row_ptrs() + m + 1,
+              0);
+    std::fill_n(prolong_op->get_values(), m, gko::one<value_type>());
+    auto restrict_op = gko::as<Csr>(prolong_op->transpose());
+    auto d_prolong_op = Csr::create(omp);
+    auto d_restrict_op = Csr::create(omp);
+    d_prolong_op->copy_from(prolong_op.get());
+    d_restrict_op->copy_from(restrict_op.get());
 
     gko::kernels::omp::amgx_pgm::amgx_pgm_generate(
-        omp, d_weight_csr.get(), d_agg, d_csr_coarse.get(), d_csr_temp.get());
+        omp, d_system_mtx.get(), d_prolong_op.get(), d_restrict_op.get(),
+        d_csr_coarse.get(), d_csr_temp.get());
     gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
-        ref, weight_csr.get(), agg, csr_coarse.get(), csr_temp.get());
+        ref, system_mtx.get(), prolong_op.get(), restrict_op.get(),
+        csr_coarse.get(), csr_temp.get());
 
+    // it should be checked already in renumber
+    GKO_ASSERT_EQ(num_agg, n);
     GKO_ASSERT_MTX_NEAR(d_csr_coarse, csr_coarse, 1e-14);
 }
 
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index 16002259fcf..7d03fdff9f0 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -234,11 +234,13 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void amgx_pgm_generate(std::shared_ptr<const ReferenceExecutor> exec,
                        const matrix::Csr<ValueType, IndexType> *source,
-                       const Array<IndexType> &agg,
+                       const matrix::Csr<ValueType, IndexType> *prolong_op,
+                       const matrix::Csr<ValueType, IndexType> *restrict_op,
                        matrix::Csr<ValueType, IndexType> *coarse,
                        matrix::Csr<ValueType, IndexType> *temp)
 {
     // agg[i] -> I, agg[j] -> J
+    const auto agg_const_val = prolong_op->get_const_col_idxs();
     const auto coarse_nrows = coarse->get_size()[0];
     const auto source_nrows = source->get_size()[0];
     const auto source_row_ptrs = source->get_const_row_ptrs();
@@ -247,9 +249,9 @@ void amgx_pgm_generate(std::shared_ptr<const ReferenceExecutor> exec,
     gko::vector<gko::map<IndexType, ValueType>> row_list(
         source_nrows, gko::map<IndexType, ValueType>{exec}, exec);
     for (size_type i = 0; i < source_nrows; i++) {
-        IndexType row_idx = agg.get_const_data()[i];
+        IndexType row_idx = agg_const_val[i];
         for (auto j = source_row_ptrs[i]; j < source_row_ptrs[i + 1]; j++) {
-            const auto col = agg.get_const_data()[source_col_idxs[j]];
+            const auto col = agg_const_val[source_col_idxs[j]];
             const auto val = source_vals[j];
             row_list[row_idx][col] += val;
         }
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index 09918310823..2cff6c8537b 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -508,11 +508,10 @@ TYPED_TEST(AmgxPgm, GenerateMtx)
     gko::Array<index_type> agg(this->exec, 5);
     auto agg_vals = agg.get_data();
     // 0 - 2, 1 - 3, 4
-    agg_vals[0] = 0;
-    agg_vals[1] = 1;
-    agg_vals[2] = 0;
-    agg_vals[3] = 1;
-    agg_vals[4] = 2;
+    auto prolong_op = mtx_type::create(this->exec, gko::dim<2>{5, 3}, 0);
+    prolong_op->read(
+        {{5, 3}, {{0, 0, 1}, {1, 1, 1}, {2, 0, 1}, {3, 1, 1}, {4, 2, 1}}});
+    auto restrict_op = gko::as<mtx_type>(prolong_op->transpose());
     auto coarse_ans = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
     coarse_ans->read({{3, 3},
                       {{0, 0, 4},
@@ -528,7 +527,8 @@ TYPED_TEST(AmgxPgm, GenerateMtx)
     auto empty = gko::matrix::Csr<value_type, index_type>::create(this->exec);
 
     gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
-        this->exec, this->mtx.get(), agg, csr_coarse.get(), empty.get());
+        this->exec, this->mtx.get(), prolong_op.get(), restrict_op.get(),
+        csr_coarse.get(), empty.get());
 
     GKO_ASSERT_MTX_NEAR(csr_coarse, coarse_ans, r<value_type>::value);
 }

From 86983e3fd2a4088cb3d842c61cc31fca5279be0e Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 30 Apr 2021 20:56:15 +0800
Subject: [PATCH 0674/1008] use two csr multiplication to generate coarse

need to improve it with less memory footprint
---
 common/multigrid/amgx_pgm_kernels.hpp.inc     | 102 ------------------
 core/device_hooks/common_kernels.inc.cpp      |   5 -
 core/multigrid/amgx_pgm.cpp                   |  36 ++-----
 core/multigrid/amgx_pgm_kernels.hpp           |  13 +--
 cuda/multigrid/amgx_pgm_kernels.cu            |  59 ----------
 cuda/test/multigrid/amgx_pgm_kernels.cpp      |  38 -------
 dpcpp/multigrid/amgx_pgm_kernels.dp.cpp       |  12 ---
 hip/multigrid/amgx_pgm_kernels.hip.cpp        |  63 -----------
 hip/test/multigrid/amgx_pgm_kernels.cpp       |  38 -------
 omp/multigrid/amgx_pgm_kernels.cpp            |  54 ----------
 omp/test/multigrid/amgx_pgm_kernels.cpp       |  38 -------
 reference/multigrid/amgx_pgm_kernels.cpp      |  53 ---------
 reference/test/multigrid/amgx_pgm_kernels.cpp |  43 +++-----
 13 files changed, 24 insertions(+), 530 deletions(-)

diff --git a/common/multigrid/amgx_pgm_kernels.hpp.inc b/common/multigrid/amgx_pgm_kernels.hpp.inc
index 3ec6bf81a7b..540478ba427 100644
--- a/common/multigrid/amgx_pgm_kernels.hpp.inc
+++ b/common/multigrid/amgx_pgm_kernels.hpp.inc
@@ -225,106 +225,4 @@ __global__
 }
 
 
-template <typename IndexType>
-__global__ __launch_bounds__(default_block_size) void get_source_row_map_kernel(
-    const size_type source_nrows, const IndexType *__restrict__ agg_val,
-    const IndexType *__restrict__ source_row_ptrs,
-    IndexType *__restrict__ result_row_ptrs, IndexType *__restrict__ row_map)
-{
-    auto row = thread::get_thread_id_flat();
-    if (row >= source_nrows) {
-        return;
-    }
-    const auto num_elems = source_row_ptrs[row + 1] - source_row_ptrs[row];
-    const auto result_idx = agg_val[row];
-    // atomic_add returns the old value, so it can be the starting point.
-    row_map[row] = atomic_add(result_row_ptrs + result_idx, num_elems);
-}
-
-
-template <typename ValueType, typename IndexType>
-__global__ __launch_bounds__(default_block_size) void move_row_kernel(
-    const size_type source_nrows, const IndexType *__restrict__ agg_val,
-    const IndexType *__restrict__ row_map,
-    const IndexType *__restrict__ source_row_ptrs,
-    const IndexType *__restrict__ source_col_idxs,
-    const ValueType *__restrict__ source_values,
-    const IndexType *__restrict__ result_row_ptrs,
-    IndexType *__restrict__ result_col_idxs,
-    ValueType *__restrict__ result_values)
-{
-    auto row = thread::get_thread_id_flat();
-    if (row >= source_nrows) {
-        return;
-    }
-    auto result_i = result_row_ptrs[agg_val[row]] + row_map[row];
-    for (auto i = source_row_ptrs[row]; i < source_row_ptrs[row + 1];
-         i++, result_i++) {
-        result_col_idxs[result_i] = agg_val[source_col_idxs[i]];
-        result_values[result_i] = source_values[i];
-    }
-}
-
-
-template <typename ValueType, typename IndexType>
-__global__ __launch_bounds__(default_block_size) void merge_col_kernel(
-    const size_type nrows, const IndexType *__restrict__ temp_row_ptrs,
-    IndexType *__restrict__ temp_col_idxs, ValueType *__restrict__ temp_values,
-    IndexType *__restrict__ coarse_row_ptrs)
-{
-    auto row = thread::get_thread_id_flat();
-    if (row >= nrows) {
-        return;
-    }
-
-    IndexType num_elems = zero<IndexType>();
-    const auto start = temp_row_ptrs[row];
-    const auto end = temp_row_ptrs[row + 1];
-    IndexType col = temp_col_idxs[start];
-    ValueType value = temp_values[start];
-    for (auto i = start + 1; i < end; i++) {
-        const auto current_col = temp_col_idxs[i];
-        if (current_col != col) {
-            // apply to the original data. It is sorted, so the writing position
-            // is before read position
-            temp_col_idxs[start + num_elems] = col;
-            temp_values[start + num_elems] = value;
-            value = zero<ValueType>();
-            col = current_col;
-            num_elems++;
-        }
-        value += temp_values[i];
-    }
-    // If start != end, need to process the final column
-    if (start != end) {
-        temp_col_idxs[start + num_elems] = col;
-        temp_values[start + num_elems] = value;
-        num_elems++;
-    }
-    coarse_row_ptrs[row] = num_elems;
-}
-
-
-template <typename ValueType, typename IndexType>
-__global__ __launch_bounds__(default_block_size) void copy_to_coarse_kernel(
-    const size_type nrows, const IndexType *__restrict__ temp_row_ptrs,
-    const IndexType *__restrict__ temp_col_idxs,
-    const ValueType *__restrict__ temp_values,
-    const IndexType *__restrict__ coarse_row_ptrs,
-    IndexType *__restrict__ coarse_col_idxs,
-    ValueType *__restrict__ coarse_values)
-{
-    auto row = thread::get_thread_id_flat();
-    if (row >= nrows) {
-        return;
-    }
-    auto temp_i = temp_row_ptrs[row];
-    for (auto i = coarse_row_ptrs[row]; i < coarse_row_ptrs[row + 1]; i++) {
-        coarse_col_idxs[i] = temp_col_idxs[temp_i];
-        coarse_values[i] = temp_values[temp_i];
-        temp_i++;
-    }
-}
-
-
 }  // namespace kernel
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index cbf1e2d9737..877051c3e04 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -1328,11 +1328,6 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
 
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_AMGX_PGM_GENERATE(ValueType, IndexType)
-GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
-
 
 }  // namespace amgx_pgm
 
diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index d9fac590691..83421932016 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -60,7 +60,6 @@ GKO_REGISTER_OPERATION(renumber, amgx_pgm::renumber);
 GKO_REGISTER_OPERATION(find_strongest_neighbor,
                        amgx_pgm::find_strongest_neighbor);
 GKO_REGISTER_OPERATION(assign_to_exist_agg, amgx_pgm::assign_to_exist_agg);
-GKO_REGISTER_OPERATION(amgx_pgm_generate, amgx_pgm::amgx_pgm_generate);
 GKO_REGISTER_OPERATION(fill_array, components::fill_array);
 GKO_REGISTER_OPERATION(fill_seq_array, components::fill_seq_array);
 
@@ -68,30 +67,6 @@ GKO_REGISTER_OPERATION(fill_seq_array, components::fill_seq_array);
 }  // namespace amgx_pgm
 
 
-namespace {
-
-
-template <typename ValueType, typename IndexType>
-std::unique_ptr<LinOp> amgx_pgm_generate(
-    std::shared_ptr<const Executor> exec,
-    const matrix::Csr<ValueType, IndexType> *source,
-    const matrix::Csr<ValueType, IndexType> *prolong_op,
-    const matrix::Csr<ValueType, IndexType> *restrict_op)
-{
-    auto num_agg = prolong_op->get_size()[1];
-    auto coarse = matrix::Csr<ValueType, IndexType>::create(
-        exec, dim<2>{num_agg, num_agg}, 0, source->get_strategy());
-    auto temp = matrix::Csr<ValueType, IndexType>::create(
-        exec, dim<2>{num_agg, num_agg}, source->get_num_stored_elements());
-    exec->run(amgx_pgm::make_amgx_pgm_generate(source, prolong_op, restrict_op,
-                                               coarse.get(), temp.get()));
-    return std::move(coarse);
-}
-
-
-}  // namespace
-
-
 template <typename ValueType, typename IndexType>
 void AmgxPgm<ValueType, IndexType>::generate()
 {
@@ -160,7 +135,7 @@ void AmgxPgm<ValueType, IndexType>::generate()
     // Renumber the index
     exec->run(amgx_pgm::make_renumber(agg_, &num_agg));
 
-    auto coarse_dim = num_agg;
+    gko::dim<2>::dimension_type coarse_dim = num_agg;
     auto fine_dim = system_matrix_->get_size()[0];
     // TODO: prolong_op can be done with lightway format
     auto prolong_op = share(
@@ -175,8 +150,13 @@ void AmgxPgm<ValueType, IndexType>::generate()
     auto restrict_op = gko::as<matrix_type>(share(prolong_op->transpose()));
 
     // Construct the coarse matrix
-    auto coarse_matrix = share(amgx_pgm_generate(
-        exec, amgxpgm_op, prolong_op.get(), restrict_op.get()));
+    // TODO: use less memory footprint to improve it
+    auto coarse_matrix =
+        share(matrix_type::create(exec, gko::dim<2>{coarse_dim, coarse_dim}));
+    auto tmp = matrix_type::create(exec, gko::dim<2>{coarse_dim, fine_dim});
+    restrict_op->apply(amgxpgm_op, tmp.get());
+    tmp->apply(prolong_op.get(), coarse_matrix.get());
+
     this->set_multigrid_level(prolong_op, coarse_matrix, restrict_op);
 }
 
diff --git a/core/multigrid/amgx_pgm_kernels.hpp b/core/multigrid/amgx_pgm_kernels.hpp
index 5bbb48b9594..793780ae505 100644
--- a/core/multigrid/amgx_pgm_kernels.hpp
+++ b/core/multigrid/amgx_pgm_kernels.hpp
@@ -78,15 +78,6 @@ namespace amgx_pgm {
         const matrix::Diagonal<ValueType> *diag, Array<IndexType> &agg, \
         Array<IndexType> &intermediate_agg)
 
-#define GKO_DECLARE_AMGX_PGM_GENERATE(ValueType, IndexType)   \
-    void amgx_pgm_generate(                                   \
-        std::shared_ptr<const DefaultExecutor> exec,          \
-        const matrix::Csr<ValueType, IndexType> *source,      \
-        const matrix::Csr<ValueType, IndexType> *prolong_op,  \
-        const matrix::Csr<ValueType, IndexType> *restrict_op, \
-        matrix::Csr<ValueType, IndexType> *coarse,            \
-        matrix::Csr<ValueType, IndexType> *temp)
-
 #define GKO_DECLARE_ALL_AS_TEMPLATES                                    \
     template <typename IndexType>                                       \
     GKO_DECLARE_AMGX_PGM_MATCH_EDGE_KERNEL(IndexType);                  \
@@ -97,9 +88,7 @@ namespace amgx_pgm {
     template <typename ValueType, typename IndexType>                   \
     GKO_DECLARE_AMGX_PGM_FIND_STRONGEST_NEIGHBOR(ValueType, IndexType); \
     template <typename ValueType, typename IndexType>                   \
-    GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG(ValueType, IndexType);     \
-    template <typename ValueType, typename IndexType>                   \
-    GKO_DECLARE_AMGX_PGM_GENERATE(ValueType, IndexType)
+    GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG(ValueType, IndexType)
 
 
 }  // namespace amgx_pgm
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index 03f657573a7..15d9940b22e 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -172,65 +172,6 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
 
 
-template <typename ValueType, typename IndexType>
-void amgx_pgm_generate(std::shared_ptr<const CudaExecutor> exec,
-                       const matrix::Csr<ValueType, IndexType> *source,
-                       const matrix::Csr<ValueType, IndexType> *prolong_op,
-                       const matrix::Csr<ValueType, IndexType> *restrict_op,
-                       matrix::Csr<ValueType, IndexType> *coarse,
-                       matrix::Csr<ValueType, IndexType> *temp)
-{
-    const auto agg_const_val = prolong_op->get_const_col_idxs();
-    const auto source_nrows = source->get_size()[0];
-    const auto source_nnz = source->get_num_stored_elements();
-    const auto coarse_nrows = coarse->get_size()[0];
-    Array<IndexType> row_map(exec, source_nrows);
-    // fill coarse row pointer as zero
-    components::fill_array(exec, temp->get_row_ptrs(), coarse_nrows + 1,
-                           zero<IndexType>());
-    // compute each source row should be moved and also change column index
-    dim3 grid(ceildiv(source_nrows, default_block_size));
-    // agg source_row (for row size) coarse row source map
-    kernel::get_source_row_map_kernel<<<grid, default_block_size>>>(
-        source_nrows, agg_const_val, source->get_const_row_ptrs(),
-        temp->get_row_ptrs(), row_map.get_data());
-    // prefix sum of temp_row_ptrs
-    components::prefix_sum(exec, temp->get_row_ptrs(), coarse_nrows + 1);
-    // copy source -> to coarse and change column index
-    kernel::move_row_kernel<<<grid, default_block_size>>>(
-        source_nrows, agg_const_val, row_map.get_const_data(),
-        source->get_const_row_ptrs(), source->get_const_col_idxs(),
-        as_cuda_type(source->get_const_values()), temp->get_const_row_ptrs(),
-        temp->get_col_idxs(), as_cuda_type(temp->get_values()));
-    // sort csr
-    csr::sort_by_column_index(exec, temp);
-    // summation of the elements with same position
-    grid = ceildiv(coarse_nrows, default_block_size);
-    kernel::merge_col_kernel<<<grid, default_block_size>>>(
-        coarse_nrows, temp->get_const_row_ptrs(), temp->get_col_idxs(),
-        as_cuda_type(temp->get_values()), coarse->get_row_ptrs());
-    // build the coarse matrix
-    components::prefix_sum(exec, coarse->get_row_ptrs(), coarse_nrows + 1);
-    // prefix sum of coarse->get_row_ptrs
-    const auto coarse_nnz =
-        exec->copy_val_to_host(coarse->get_row_ptrs() + coarse_nrows);
-    // reallocate size of column and values
-    matrix::CsrBuilder<ValueType, IndexType> coarse_builder{coarse};
-    auto &coarse_col_idxs_array = coarse_builder.get_col_idx_array();
-    auto &coarse_vals_array = coarse_builder.get_value_array();
-    coarse_col_idxs_array.resize_and_reset(coarse_nnz);
-    coarse_vals_array.resize_and_reset(coarse_nnz);
-    // copy the result
-    kernel::copy_to_coarse_kernel<<<grid, default_block_size>>>(
-        coarse_nrows, temp->get_const_row_ptrs(), temp->get_const_col_idxs(),
-        as_cuda_type(temp->get_const_values()), coarse->get_const_row_ptrs(),
-        coarse_col_idxs_array.get_data(),
-        as_cuda_type(coarse_vals_array.get_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
-
-
 }  // namespace amgx_pgm
 }  // namespace cuda
 }  // namespace kernels
diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
index b92934ce089..07672359615 100644
--- a/cuda/test/multigrid/amgx_pgm_kernels.cpp
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -288,44 +288,6 @@ TEST_F(AmgxPgm, AssignToExistAggUnderteminsticIsEquivalentToRef)
 }
 
 
-TEST_F(AmgxPgm, GenerateMtxIsEquivalentToRef)
-{
-    initialize_data();
-    auto csr_coarse = Csr::create(ref, gko::dim<2>{n, n}, 0);
-    auto d_csr_coarse = Csr::create(cuda, gko::dim<2>{n, n}, 0);
-    auto csr_temp = Csr::create(ref, gko::dim<2>{n, n},
-                                system_mtx->get_num_stored_elements());
-    auto d_csr_temp = Csr::create(cuda, gko::dim<2>{n, n},
-                                  d_system_mtx->get_num_stored_elements());
-    index_type num_agg;
-    // renumber again
-    gko::kernels::reference::amgx_pgm::renumber(ref, agg, &num_agg);
-    auto prolong_op = Csr::create(ref, gko::dim<2>{m, n}, m);
-    for (int i = 0; i < m; i++) {
-        prolong_op->get_col_idxs()[i] = agg.get_const_data()[i];
-    }
-    std::iota(prolong_op->get_row_ptrs(), prolong_op->get_row_ptrs() + m + 1,
-              0);
-    std::fill_n(prolong_op->get_values(), m, gko::one<value_type>());
-    auto restrict_op = gko::as<Csr>(prolong_op->transpose());
-    auto d_prolong_op = Csr::create(cuda);
-    auto d_restrict_op = Csr::create(cuda);
-    d_prolong_op->copy_from(prolong_op.get());
-    d_restrict_op->copy_from(restrict_op.get());
-
-    gko::kernels::cuda::amgx_pgm::amgx_pgm_generate(
-        cuda, d_system_mtx.get(), d_prolong_op.get(), d_restrict_op.get(),
-        d_csr_coarse.get(), d_csr_temp.get());
-    gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
-        ref, system_mtx.get(), prolong_op.get(), restrict_op.get(),
-        csr_coarse.get(), csr_temp.get());
-
-    // it should be checked already in renumber
-    GKO_ASSERT_EQ(num_agg, n);
-    GKO_ASSERT_MTX_NEAR(d_csr_coarse, csr_coarse, 1e-14);
-}
-
-
 TEST_F(AmgxPgm, GenerateMgLevelIsEquivalentToRef)
 {
     initialize_data();
diff --git a/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp b/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
index 283127ce6d5..14baa306bca 100644
--- a/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
+++ b/dpcpp/multigrid/amgx_pgm_kernels.dp.cpp
@@ -100,18 +100,6 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
 
 
-template <typename ValueType, typename IndexType>
-void amgx_pgm_generate(std::shared_ptr<const DpcppExecutor> exec,
-                       const matrix::Csr<ValueType, IndexType> *source,
-                       const matrix::Csr<ValueType, IndexType> *prolong_op,
-                       const matrix::Csr<ValueType, IndexType> *restrict_op,
-                       matrix::Csr<ValueType, IndexType> *coarse,
-                       matrix::Csr<ValueType, IndexType> *temp)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
-
-
 }  // namespace amgx_pgm
 }  // namespace dpcpp
 }  // namespace kernels
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index e2ed35d42e9..d7f4685b785 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -183,69 +183,6 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
 
 
-template <typename ValueType, typename IndexType>
-void amgx_pgm_generate(std::shared_ptr<const HipExecutor> exec,
-                       const matrix::Csr<ValueType, IndexType> *source,
-                       const matrix::Csr<ValueType, IndexType> *prolong_op,
-                       const matrix::Csr<ValueType, IndexType> *restrict_op,
-                       matrix::Csr<ValueType, IndexType> *coarse,
-                       matrix::Csr<ValueType, IndexType> *temp)
-{
-    const auto agg_const_val = prolong_op->get_const_col_idxs();
-    const auto source_nrows = source->get_size()[0];
-    const auto source_nnz = source->get_num_stored_elements();
-    const auto coarse_nrows = coarse->get_size()[0];
-    Array<IndexType> row_map(exec, source_nrows);
-    // fill coarse row pointer as zero
-    components::fill_array(exec, temp->get_row_ptrs(), coarse_nrows + 1,
-                           zero<IndexType>());
-    // compute each source row should be moved and also change column index
-    dim3 grid(ceildiv(source_nrows, default_block_size));
-    // agg source_row (for row size) coarse row source map
-    hipLaunchKernelGGL(kernel::get_source_row_map_kernel, dim3(grid),
-                       dim3(default_block_size), 0, 0, source_nrows,
-                       agg_const_val, source->get_const_row_ptrs(),
-                       temp->get_row_ptrs(), row_map.get_data());
-    // prefix sum of temp_row_ptrs
-    components::prefix_sum(exec, temp->get_row_ptrs(), coarse_nrows + 1);
-    // copy source -> to coarse and change column index
-    hipLaunchKernelGGL(
-        kernel::move_row_kernel, dim3(grid), dim3(default_block_size), 0, 0,
-        source_nrows, agg_const_val, row_map.get_const_data(),
-        source->get_const_row_ptrs(), source->get_const_col_idxs(),
-        as_hip_type(source->get_const_values()), temp->get_const_row_ptrs(),
-        temp->get_col_idxs(), as_hip_type(temp->get_values()));
-    // sort csr
-    csr::sort_by_column_index(exec, temp);
-    // summation of the elements with same position
-    grid = ceildiv(coarse_nrows, default_block_size);
-    hipLaunchKernelGGL(kernel::merge_col_kernel, dim3(grid),
-                       dim3(default_block_size), 0, 0, coarse_nrows,
-                       temp->get_const_row_ptrs(), temp->get_col_idxs(),
-                       as_hip_type(temp->get_values()), coarse->get_row_ptrs());
-    // build the coarse matrix
-    components::prefix_sum(exec, coarse->get_row_ptrs(), coarse_nrows + 1);
-    // prefix sum of coarse->get_row_ptrs
-    const auto coarse_nnz =
-        exec->copy_val_to_host(coarse->get_row_ptrs() + coarse_nrows);
-    // reallocate size of column and values
-    matrix::CsrBuilder<ValueType, IndexType> coarse_builder{coarse};
-    auto &coarse_col_idxs_array = coarse_builder.get_col_idx_array();
-    auto &coarse_vals_array = coarse_builder.get_value_array();
-    coarse_col_idxs_array.resize_and_reset(coarse_nnz);
-    coarse_vals_array.resize_and_reset(coarse_nnz);
-    // copy the result
-    hipLaunchKernelGGL(
-        kernel::copy_to_coarse_kernel, dim3(grid), dim3(default_block_size), 0,
-        0, coarse_nrows, temp->get_const_row_ptrs(), temp->get_const_col_idxs(),
-        as_hip_type(temp->get_const_values()), coarse->get_const_row_ptrs(),
-        coarse_col_idxs_array.get_data(),
-        as_hip_type(coarse_vals_array.get_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
-
-
 }  // namespace amgx_pgm
 }  // namespace hip
 }  // namespace kernels
diff --git a/hip/test/multigrid/amgx_pgm_kernels.cpp b/hip/test/multigrid/amgx_pgm_kernels.cpp
index 73909e522aa..879eae1876e 100644
--- a/hip/test/multigrid/amgx_pgm_kernels.cpp
+++ b/hip/test/multigrid/amgx_pgm_kernels.cpp
@@ -287,44 +287,6 @@ TEST_F(AmgxPgm, AssignToExistAggUnderteminsticIsEquivalentToRef)
 }
 
 
-TEST_F(AmgxPgm, GenerateMtxIsEquivalentToRef)
-{
-    initialize_data();
-    auto csr_coarse = Csr::create(ref, gko::dim<2>{n, n}, 0);
-    auto d_csr_coarse = Csr::create(hip, gko::dim<2>{n, n}, 0);
-    auto csr_temp = Csr::create(ref, gko::dim<2>{n, n},
-                                system_mtx->get_num_stored_elements());
-    auto d_csr_temp = Csr::create(hip, gko::dim<2>{n, n},
-                                  d_system_mtx->get_num_stored_elements());
-    index_type num_agg;
-    // renumber again
-    gko::kernels::reference::amgx_pgm::renumber(ref, agg, &num_agg);
-    auto prolong_op = Csr::create(ref, gko::dim<2>{m, n}, m);
-    for (int i = 0; i < m; i++) {
-        prolong_op->get_col_idxs()[i] = agg.get_const_data()[i];
-    }
-    std::iota(prolong_op->get_row_ptrs(), prolong_op->get_row_ptrs() + m + 1,
-              0);
-    std::fill_n(prolong_op->get_values(), m, gko::one<value_type>());
-    auto restrict_op = gko::as<Csr>(prolong_op->transpose());
-    auto d_prolong_op = Csr::create(hip);
-    auto d_restrict_op = Csr::create(hip);
-    d_prolong_op->copy_from(prolong_op.get());
-    d_restrict_op->copy_from(restrict_op.get());
-
-    gko::kernels::hip::amgx_pgm::amgx_pgm_generate(
-        hip, d_system_mtx.get(), d_prolong_op.get(), d_restrict_op.get(),
-        d_csr_coarse.get(), d_csr_temp.get());
-    gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
-        ref, system_mtx.get(), prolong_op.get(), restrict_op.get(),
-        csr_coarse.get(), csr_temp.get());
-
-    // it should be checked already in renumber
-    GKO_ASSERT_EQ(num_agg, n);
-    GKO_ASSERT_MTX_NEAR(d_csr_coarse, csr_coarse, 1e-14);
-}
-
-
 TEST_F(AmgxPgm, GenerateMgLevelIsEquivalentToRef)
 {
     initialize_data();
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index dc6329d982c..07e84630d85 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -239,60 +239,6 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
 
 
-template <typename ValueType, typename IndexType>
-void amgx_pgm_generate(std::shared_ptr<const OmpExecutor> exec,
-                       const matrix::Csr<ValueType, IndexType> *source,
-                       const matrix::Csr<ValueType, IndexType> *prolong_op,
-                       const matrix::Csr<ValueType, IndexType> *restrict_op,
-                       matrix::Csr<ValueType, IndexType> *coarse,
-                       matrix::Csr<ValueType, IndexType> *temp)
-{
-    // agg[i] -> I, agg[j] -> J
-    const auto agg_const_val = prolong_op->get_const_col_idxs();
-    const auto coarse_nrows = coarse->get_size()[0];
-    const auto source_nrows = source->get_size()[0];
-    const auto source_row_ptrs = source->get_const_row_ptrs();
-    const auto source_col_idxs = source->get_const_col_idxs();
-    const auto source_vals = source->get_const_values();
-    vector<map<IndexType, ValueType>> row_list(
-        source_nrows, map<IndexType, ValueType>{exec}, exec);
-    for (size_type i = 0; i < source_nrows; i++) {
-        IndexType row_idx = agg_const_val[i];
-        for (auto j = source_row_ptrs[i]; j < source_row_ptrs[i + 1]; j++) {
-            const auto col = agg_const_val[source_col_idxs[j]];
-            const auto val = source_vals[j];
-            row_list[row_idx][col] += val;
-        }
-    }
-    auto coarse_row_ptrs = coarse->get_row_ptrs();
-#pragma omp parallel for
-    for (size_type i = 0; i < coarse_nrows; i++) {
-        coarse_row_ptrs[i] = row_list[i].size();
-    }
-    components::prefix_sum(exec, coarse_row_ptrs, coarse_nrows + 1);
-
-    auto nnz = coarse_row_ptrs[coarse_nrows];
-    matrix::CsrBuilder<ValueType, IndexType> coarse_builder{coarse};
-    auto &coarse_col_idxs_array = coarse_builder.get_col_idx_array();
-    auto &coarse_vals_array = coarse_builder.get_value_array();
-    coarse_col_idxs_array.resize_and_reset(nnz);
-    coarse_vals_array.resize_and_reset(nnz);
-    auto coarse_col_idxs = coarse_col_idxs_array.get_data();
-    auto coarse_vals = coarse_vals_array.get_data();
-#pragma omp parallel for
-    for (size_type i = 0; i < coarse_nrows; i++) {
-        auto ind = coarse_row_ptrs[i];
-        for (auto pair : row_list[i]) {
-            coarse_col_idxs[ind] = pair.first;
-            coarse_vals[ind] = pair.second;
-            ind++;
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
-
-
 }  // namespace amgx_pgm
 }  // namespace omp
 }  // namespace kernels
diff --git a/omp/test/multigrid/amgx_pgm_kernels.cpp b/omp/test/multigrid/amgx_pgm_kernels.cpp
index 91dc2c2fce8..59b40657703 100644
--- a/omp/test/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/test/multigrid/amgx_pgm_kernels.cpp
@@ -278,44 +278,6 @@ TEST_F(AmgxPgm, AssignToExistAggUnderteminsticIsEquivalentToRef)
 }
 
 
-TEST_F(AmgxPgm, GenerateMtxIsEquivalentToRef)
-{
-    initialize_data();
-    auto csr_coarse = Csr::create(ref, gko::dim<2>{n, n}, 0);
-    auto d_csr_coarse = Csr::create(omp, gko::dim<2>{n, n}, 0);
-    auto csr_temp = Csr::create(ref, gko::dim<2>{n, n},
-                                system_mtx->get_num_stored_elements());
-    auto d_csr_temp = Csr::create(omp, gko::dim<2>{n, n},
-                                  d_system_mtx->get_num_stored_elements());
-    index_type num_agg;
-    // renumber again
-    gko::kernels::reference::amgx_pgm::renumber(ref, agg, &num_agg);
-    auto prolong_op = Csr::create(ref, gko::dim<2>{m, n}, m);
-    for (int i = 0; i < m; i++) {
-        prolong_op->get_col_idxs()[i] = agg.get_const_data()[i];
-    }
-    std::iota(prolong_op->get_row_ptrs(), prolong_op->get_row_ptrs() + m + 1,
-              0);
-    std::fill_n(prolong_op->get_values(), m, gko::one<value_type>());
-    auto restrict_op = gko::as<Csr>(prolong_op->transpose());
-    auto d_prolong_op = Csr::create(omp);
-    auto d_restrict_op = Csr::create(omp);
-    d_prolong_op->copy_from(prolong_op.get());
-    d_restrict_op->copy_from(restrict_op.get());
-
-    gko::kernels::omp::amgx_pgm::amgx_pgm_generate(
-        omp, d_system_mtx.get(), d_prolong_op.get(), d_restrict_op.get(),
-        d_csr_coarse.get(), d_csr_temp.get());
-    gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
-        ref, system_mtx.get(), prolong_op.get(), restrict_op.get(),
-        csr_coarse.get(), csr_temp.get());
-
-    // it should be checked already in renumber
-    GKO_ASSERT_EQ(num_agg, n);
-    GKO_ASSERT_MTX_NEAR(d_csr_coarse, csr_coarse, 1e-14);
-}
-
-
 TEST_F(AmgxPgm, GenerateMgLevelIsEquivalentToRef)
 {
     initialize_data();
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index 7d03fdff9f0..5ad1ff8d108 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -231,59 +231,6 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_AMGX_PGM_ASSIGN_TO_EXIST_AGG);
 
 
-template <typename ValueType, typename IndexType>
-void amgx_pgm_generate(std::shared_ptr<const ReferenceExecutor> exec,
-                       const matrix::Csr<ValueType, IndexType> *source,
-                       const matrix::Csr<ValueType, IndexType> *prolong_op,
-                       const matrix::Csr<ValueType, IndexType> *restrict_op,
-                       matrix::Csr<ValueType, IndexType> *coarse,
-                       matrix::Csr<ValueType, IndexType> *temp)
-{
-    // agg[i] -> I, agg[j] -> J
-    const auto agg_const_val = prolong_op->get_const_col_idxs();
-    const auto coarse_nrows = coarse->get_size()[0];
-    const auto source_nrows = source->get_size()[0];
-    const auto source_row_ptrs = source->get_const_row_ptrs();
-    const auto source_col_idxs = source->get_const_col_idxs();
-    const auto source_vals = source->get_const_values();
-    gko::vector<gko::map<IndexType, ValueType>> row_list(
-        source_nrows, gko::map<IndexType, ValueType>{exec}, exec);
-    for (size_type i = 0; i < source_nrows; i++) {
-        IndexType row_idx = agg_const_val[i];
-        for (auto j = source_row_ptrs[i]; j < source_row_ptrs[i + 1]; j++) {
-            const auto col = agg_const_val[source_col_idxs[j]];
-            const auto val = source_vals[j];
-            row_list[row_idx][col] += val;
-        }
-    }
-    auto coarse_row_ptrs = coarse->get_row_ptrs();
-    for (size_type i = 0; i < coarse_nrows; i++) {
-        coarse_row_ptrs[i] = row_list[i].size();
-    }
-    components::prefix_sum(exec, coarse_row_ptrs, coarse_nrows + 1);
-
-    auto nnz = coarse_row_ptrs[coarse_nrows];
-    matrix::CsrBuilder<ValueType, IndexType> coarse_builder{coarse};
-    auto &coarse_col_idxs_array = coarse_builder.get_col_idx_array();
-    auto &coarse_vals_array = coarse_builder.get_value_array();
-    coarse_col_idxs_array.resize_and_reset(nnz);
-    coarse_vals_array.resize_and_reset(nnz);
-    auto coarse_col_idxs = coarse_col_idxs_array.get_data();
-    auto coarse_vals = coarse_vals_array.get_data();
-
-    for (size_type i = 0; i < coarse_nrows; i++) {
-        auto ind = coarse_row_ptrs[i];
-        for (auto pair : row_list[i]) {
-            coarse_col_idxs[ind] = pair.first;
-            coarse_vals[ind] = pair.second;
-            ind++;
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_AMGX_PGM_GENERATE);
-
-
 }  // namespace amgx_pgm
 }  // namespace reference
 }  // namespace kernels
diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index 2cff6c8537b..c0270312d6c 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -500,37 +500,24 @@ TYPED_TEST(AmgxPgm, AssignToExistAgg)
 }
 
 
-TYPED_TEST(AmgxPgm, GenerateMtx)
+TYPED_TEST(AmgxPgm, GenerateMgLevel)
 {
-    using index_type = typename TestFixture::index_type;
     using value_type = typename TestFixture::value_type;
-    using mtx_type = typename TestFixture::Mtx;
-    gko::Array<index_type> agg(this->exec, 5);
-    auto agg_vals = agg.get_data();
-    // 0 - 2, 1 - 3, 4
-    auto prolong_op = mtx_type::create(this->exec, gko::dim<2>{5, 3}, 0);
+    using Mtx = typename TestFixture::Mtx;
+    auto prolong_op = gko::share(Mtx::create(this->exec, gko::dim<2>{5, 2}, 0));
+    // 0-2-4, 1-3
     prolong_op->read(
-        {{5, 3}, {{0, 0, 1}, {1, 1, 1}, {2, 0, 1}, {3, 1, 1}, {4, 2, 1}}});
-    auto restrict_op = gko::as<mtx_type>(prolong_op->transpose());
-    auto coarse_ans = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
-    coarse_ans->read({{3, 3},
-                      {{0, 0, 4},
-                       {0, 1, -3},
-                       {0, 2, -1},
-                       {1, 0, -3},
-                       {1, 1, 5},
-                       {1, 2, -1},
-                       {2, 0, -2},
-                       {2, 1, -2},
-                       {2, 2, 5}}});
-    auto csr_coarse = mtx_type::create(this->exec, gko::dim<2>{3, 3}, 0);
-    auto empty = gko::matrix::Csr<value_type, index_type>::create(this->exec);
-
-    gko::kernels::reference::amgx_pgm::amgx_pgm_generate(
-        this->exec, this->mtx.get(), prolong_op.get(), restrict_op.get(),
-        csr_coarse.get(), empty.get());
-
-    GKO_ASSERT_MTX_NEAR(csr_coarse, coarse_ans, r<value_type>::value);
+        {{5, 2}, {{0, 0, 1}, {1, 1, 1}, {2, 0, 1}, {3, 1, 1}, {4, 0, 1}}});
+    auto restrict_op = gko::share(gko::as<Mtx>(prolong_op->transpose()));
+
+    auto coarse_fine = this->amgxpgm_factory->generate(this->mtx);
+
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(coarse_fine->get_restrict_op()),
+                        restrict_op, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(coarse_fine->get_coarse_op()),
+                        this->coarse, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(gko::as<Mtx>(coarse_fine->get_prolong_op()), prolong_op,
+                        r<value_type>::value);
 }
 
 

From 744cfc18acddd34f2ebecc42a73d139e956a3082 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 6 May 2021 10:27:49 +0200
Subject: [PATCH 0675/1008] make array constructor explicit

---
 include/ginkgo/core/base/array.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/ginkgo/core/base/array.hpp b/include/ginkgo/core/base/array.hpp
index b7f31529da9..c9f4aaa57c4 100644
--- a/include/ginkgo/core/base/array.hpp
+++ b/include/ginkgo/core/base/array.hpp
@@ -122,7 +122,7 @@ class Array {
      *
      * @param exec  the Executor where the array data is allocated
      */
-    Array(std::shared_ptr<const Executor> exec) noexcept
+    explicit Array(std::shared_ptr<const Executor> exec) noexcept
         : num_elems_(0),
           data_(nullptr, default_deleter{exec}),
           exec_(std::move(exec))

From a80f1647c7d2b214591c73a89367677b3461cbb7 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 17 Dec 2020 02:58:25 +0100
Subject: [PATCH 0676/1008] add ELL mixed-precision support

---
 benchmark/utils/formats.hpp              | 30 ++++++++++--
 common/matrix/ell_kernels.hpp.inc        | 62 ++++++++++++++----------
 core/device_hooks/common_kernels.inc.cpp | 17 ++++---
 core/matrix/ell_kernels.hpp              | 39 +++++++++------
 cuda/matrix/ell_kernels.cu               | 42 +++++++++-------
 include/ginkgo/core/base/types.hpp       | 51 +++++++++++++++++++
 omp/matrix/ell_kernels.cpp               | 35 +++++++------
 reference/matrix/ell_kernels.cpp         | 34 +++++++------
 8 files changed, 212 insertions(+), 98 deletions(-)

diff --git a/benchmark/utils/formats.hpp b/benchmark/utils/formats.hpp
index d25be4657ec..6b344aec06d 100644
--- a/benchmark/utils/formats.hpp
+++ b/benchmark/utils/formats.hpp
@@ -59,7 +59,8 @@ namespace formats {
 
 
 std::string available_format =
-    "coo, csr, ell, sellp, hybrid, hybrid0, hybrid25, hybrid33, hybrid40, "
+    "coo, csr, ell, fell, sellp, hybrid, hybrid0, hybrid25, hybrid33, "
+    "hybrid40, "
     "hybrid60, hybrid80, hybridlimit0, hybridlimit25, hybridlimit33, "
     "hybridminstorage"
 #ifdef HAS_CUDA
@@ -90,6 +91,9 @@ std::string format_description =
     "csrm: Ginkgo's CSR implementation with merge_path strategy.\n"
     "ell: Ellpack format according to Bell and Garland: Efficient Sparse "
     "Matrix-Vector Multiplication on CUDA.\n"
+    "fell: float Ellpack format according to Bell and Garland: Efficient "
+    "Sparse "
+    "Matrix-Vector Multiplication on CUDA.\n"
     "sellp: Sliced Ellpack uses a default block size of 32.\n"
     "hybrid: Hybrid uses ell and coo to represent the matrix.\n"
     "hybrid0, hybrid25, hybrid33, hybrid40, hybrid60, hybrid80: Hybrid uses "
@@ -204,6 +208,23 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
         {"csrc", READ_MATRIX(csr, std::make_shared<csr::classical>())},
         {"coo", read_matrix_from_data<gko::matrix::Coo<etype>>},
         {"ell", read_matrix_from_data<gko::matrix::Ell<etype>>},
+        {"fell",
+         [](std::shared_ptr<const gko::Executor> exec,
+            const gko::matrix_data<> &data) {
+             gko::matrix_data<float> conv_data;
+             conv_data.size = data.size;
+             conv_data.nonzeros.resize(data.nonzeros.size());
+             auto it = conv_data.nonzeros.begin();
+             for (auto &el : data.nonzeros) {
+                 it->row = el.row;
+                 it->column = el.column;
+                 it->value = el.value;
+                 ++it;
+             }
+             auto mat = gko::matrix::Ell<float>::create(std::move(exec));
+             mat->read(conv_data);
+             return mat;
+         }},
 #ifdef HAS_CUDA
 #if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
         {"cusp_csr", read_matrix_from_data<cusp_csr>},
@@ -212,8 +233,8 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
         {"cusp_hybrid", read_matrix_from_data<cusp_hybrid>},
         {"cusp_coo", read_matrix_from_data<cusp_coo>},
         {"cusp_ell", read_matrix_from_data<cusp_ell>},
-#else // CUDA_VERSION >= 11000
-        // cusp_csr, cusp_coo use the generic ones from CUDA 11
+#else  // CUDA_VERSION >= 11000
+       // cusp_csr, cusp_coo use the generic ones from CUDA 11
         {"cusp_csr", read_matrix_from_data<cusp_gcsr>},
         {"cusp_coo", read_matrix_from_data<cusp_gcoo>},
 #endif
@@ -260,7 +281,8 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
         {"hybridminstorage",
          READ_MATRIX(hybrid,
                      std::make_shared<hybrid::minimal_storage_limit>())},
-        {"sellp", read_matrix_from_data<gko::matrix::Sellp<etype>>}};
+        {"sellp", read_matrix_from_data<gko::matrix::Sellp<etype>>}
+};
 // clang-format on
 
 
diff --git a/common/matrix/ell_kernels.hpp.inc b/common/matrix/ell_kernels.hpp.inc
index 00eb2d6871d..2f155142efd 100644
--- a/common/matrix/ell_kernels.hpp.inc
+++ b/common/matrix/ell_kernels.hpp.inc
@@ -34,22 +34,25 @@ namespace kernel {
 namespace {
 
 
-template <int num_thread_per_worker, bool atomic, typename ValueType,
+template <int num_thread_per_worker, bool atomic, typename InputValueType,
+          typename MatrixValueType, typename OutputValueType,
           typename IndexType, typename Closure>
 __device__ void spmv_kernel(
     const size_type num_rows, const int num_worker_per_row,
-    const ValueType *__restrict__ val, const IndexType *__restrict__ col,
+    const MatrixValueType *__restrict__ val, const IndexType *__restrict__ col,
     const size_type stride, const size_type num_stored_elements_per_row,
-    const ValueType *__restrict__ b, const size_type b_stride,
-    ValueType *__restrict__ c, const size_type c_stride, Closure op)
+    const InputValueType *__restrict__ b, const size_type b_stride,
+    OutputValueType *__restrict__ c, const size_type c_stride, Closure op)
 {
     const auto tidx = thread::get_thread_id_flat();
     const auto column_id = blockIdx.y;
+    using compute_type =
+        decltype(InputValueType{} + MatrixValueType{} + OutputValueType{});
     if (num_thread_per_worker == 1) {
         // Specialize the num_thread_per_worker = 1. It doesn't need the shared
         // memory, __syncthreads, and atomic_add
         if (tidx < num_rows) {
-            ValueType temp = zero<ValueType>();
+            auto temp = zero<compute_type>();
             for (size_type idx = 0; idx < num_stored_elements_per_row; idx++) {
                 const auto ind = tidx + idx * stride;
                 const auto col_idx = col[ind];
@@ -68,14 +71,14 @@ __device__ void spmv_kernel(
             const auto x = tidx % num_rows;
             const auto worker_id = tidx / num_rows;
             const auto step_size = num_worker_per_row * num_thread_per_worker;
-            __shared__ UninitializedArray<ValueType, default_block_size /
-                                                         num_thread_per_worker>
+            __shared__ UninitializedArray<
+                compute_type, default_block_size / num_thread_per_worker>
                 storage;
             if (idx_in_worker == 0) {
                 storage[threadIdx.x] = 0;
             }
             __syncthreads();
-            ValueType temp = zero<ValueType>();
+            auto temp = zero<compute_type>();
             for (size_type idx =
                      worker_id * num_thread_per_worker + idx_in_worker;
                  idx < num_stored_elements_per_row; idx += step_size) {
@@ -102,35 +105,41 @@ __device__ void spmv_kernel(
 }
 
 
-template <int num_thread_per_worker, bool atomic = false, typename ValueType,
-          typename IndexType>
+template <int num_thread_per_worker, bool atomic = false,
+          typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
 __global__ __launch_bounds__(default_block_size) void spmv(
     const size_type num_rows, const int num_worker_per_row,
-    const ValueType *__restrict__ val, const IndexType *__restrict__ col,
+    const MatrixValueType *__restrict__ val, const IndexType *__restrict__ col,
     const size_type stride, const size_type num_stored_elements_per_row,
-    const ValueType *__restrict__ b, const size_type b_stride,
-    ValueType *__restrict__ c, const size_type c_stride)
+    const InputValueType *__restrict__ b, const size_type b_stride,
+    OutputValueType *__restrict__ c, const size_type c_stride)
 {
+    using compute_type =
+        decltype(InputValueType{} + MatrixValueType{} + OutputValueType{});
     spmv_kernel<num_thread_per_worker, atomic>(
         num_rows, num_worker_per_row, val, col, stride,
         num_stored_elements_per_row, b, b_stride, c, c_stride,
-        [](const ValueType &x, const ValueType &y) { return x; });
+        [](const compute_type &x, const OutputValueType &y) { return x; });
 }
 
 
-template <int num_thread_per_worker, bool atomic = false, typename ValueType,
-          typename IndexType>
+template <int num_thread_per_worker, bool atomic = false,
+          typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
 __global__ __launch_bounds__(default_block_size) void spmv(
     const size_type num_rows, const int num_worker_per_row,
-    const ValueType *__restrict__ alpha, const ValueType *__restrict__ val,
-    const IndexType *__restrict__ col, const size_type stride,
-    const size_type num_stored_elements_per_row,
-    const ValueType *__restrict__ b, const size_type b_stride,
-    const ValueType *__restrict__ beta, ValueType *__restrict__ c,
+    const MatrixValueType *__restrict__ alpha,
+    const MatrixValueType *__restrict__ val, const IndexType *__restrict__ col,
+    const size_type stride, const size_type num_stored_elements_per_row,
+    const InputValueType *__restrict__ b, const size_type b_stride,
+    const OutputValueType *__restrict__ beta, OutputValueType *__restrict__ c,
     const size_type c_stride)
 {
-    const ValueType alpha_val = alpha[0];
-    const ValueType beta_val = beta[0];
+    using compute_type =
+        decltype(InputValueType{} + MatrixValueType{} + OutputValueType{});
+    const compute_type alpha_val = alpha[0];
+    const compute_type beta_val = beta[0];
     // Because the atomic operation changes the values of c during computation,
     // it can not do the right alpha * a * b + beta * c operation.
     // Thus, the cuda kernel only computes alpha * a * b when it uses atomic
@@ -139,15 +148,16 @@ __global__ __launch_bounds__(default_block_size) void spmv(
         spmv_kernel<num_thread_per_worker, atomic>(
             num_rows, num_worker_per_row, val, col, stride,
             num_stored_elements_per_row, b, b_stride, c, c_stride,
-            [&alpha_val](const ValueType &x, const ValueType &y) {
+            [&alpha_val](const compute_type &x, const OutputValueType &y) {
                 return alpha_val * x;
             });
     } else {
         spmv_kernel<num_thread_per_worker, atomic>(
             num_rows, num_worker_per_row, val, col, stride,
             num_stored_elements_per_row, b, b_stride, c, c_stride,
-            [&alpha_val, &beta_val](const ValueType &x, const ValueType &y) {
-                return alpha_val * x + beta_val * y;
+            [&alpha_val, &beta_val](const compute_type &x,
+                                    const OutputValueType &y) {
+                return alpha_val * x + beta_val * compute_type{y};
             });
     }
 }
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 877051c3e04..42121074df4 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -931,15 +931,20 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 namespace ell {
 
 
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_ELL_SPMV_KERNEL(ValueType, IndexType)
+template <typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
+GKO_DECLARE_ELL_SPMV_KERNEL(InputValueType, MatrixValueType, OutputValueType,
+                            IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ELL_SPMV_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ELL_SPMV_KERNEL);
 
-template <typename ValueType, typename IndexType>
-GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL(ValueType, IndexType)
+template <typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
+GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL(InputValueType, MatrixValueType,
+                                     OutputValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL);
 
 template <typename ValueType, typename IndexType>
diff --git a/core/matrix/ell_kernels.hpp b/core/matrix/ell_kernels.hpp
index 7a1ec261d15..7cbe54b3720 100644
--- a/core/matrix/ell_kernels.hpp
+++ b/core/matrix/ell_kernels.hpp
@@ -46,18 +46,21 @@ namespace gko {
 namespace kernels {
 
 
-#define GKO_DECLARE_ELL_SPMV_KERNEL(ValueType, IndexType)  \
-    void spmv(std::shared_ptr<const DefaultExecutor> exec, \
-              const matrix::Ell<ValueType, IndexType> *a,  \
-              const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
-
-#define GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL(ValueType, IndexType)  \
-    void advanced_spmv(std::shared_ptr<const DefaultExecutor> exec, \
-                       const matrix::Dense<ValueType> *alpha,       \
-                       const matrix::Ell<ValueType, IndexType> *a,  \
-                       const matrix::Dense<ValueType> *b,           \
-                       const matrix::Dense<ValueType> *beta,        \
-                       matrix::Dense<ValueType> *c)
+#define GKO_DECLARE_ELL_SPMV_KERNEL(InputValueType, MatrixValueType, \
+                                    OutputValueType, IndexType)      \
+    void spmv(std::shared_ptr<const DefaultExecutor> exec,           \
+              const matrix::Ell<MatrixValueType, IndexType> *a,      \
+              const matrix::Dense<InputValueType> *b,                \
+              matrix::Dense<OutputValueType> *c)
+
+#define GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL(InputValueType, MatrixValueType, \
+                                             OutputValueType, IndexType)      \
+    void advanced_spmv(std::shared_ptr<const DefaultExecutor> exec,           \
+                       const matrix::Dense<MatrixValueType> *alpha,           \
+                       const matrix::Ell<MatrixValueType, IndexType> *a,      \
+                       const matrix::Dense<InputValueType> *b,                \
+                       const matrix::Dense<OutputValueType> *beta,            \
+                       matrix::Dense<OutputValueType> *c)
 
 #define GKO_DECLARE_ELL_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType)      \
     void convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,     \
@@ -87,10 +90,14 @@ namespace kernels {
                           matrix::Diagonal<ValueType> *diag)
 
 #define GKO_DECLARE_ALL_AS_TEMPLATES                                         \
-    template <typename ValueType, typename IndexType>                        \
-    GKO_DECLARE_ELL_SPMV_KERNEL(ValueType, IndexType);                       \
-    template <typename ValueType, typename IndexType>                        \
-    GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL(ValueType, IndexType);              \
+    template <typename InputValueType, typename MatrixValueType,             \
+              typename OutputValueType, typename IndexType>                  \
+    GKO_DECLARE_ELL_SPMV_KERNEL(InputValueType, MatrixValueType,             \
+                                OutputValueType, IndexType);                 \
+    template <typename InputValueType, typename MatrixValueType,             \
+              typename OutputValueType, typename IndexType>                  \
+    GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL(InputValueType, MatrixValueType,    \
+                                         OutputValueType, IndexType);        \
     template <typename ValueType, typename IndexType>                        \
     GKO_DECLARE_ELL_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType);           \
     template <typename ValueType, typename IndexType>                        \
diff --git a/cuda/matrix/ell_kernels.cu b/cuda/matrix/ell_kernels.cu
index 9bc47f71e63..df223dfd7cf 100644
--- a/cuda/matrix/ell_kernels.cu
+++ b/cuda/matrix/ell_kernels.cu
@@ -108,13 +108,14 @@ using compiled_kernels = syn::value_list<int, 0, 1, 2, 4, 8, 16, 32>;
 namespace {
 
 
-template <int info, typename ValueType, typename IndexType>
+template <int info, typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
 void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
-                   const matrix::Ell<ValueType, IndexType> *a,
-                   const matrix::Dense<ValueType> *b,
-                   matrix::Dense<ValueType> *c,
-                   const matrix::Dense<ValueType> *alpha = nullptr,
-                   const matrix::Dense<ValueType> *beta = nullptr)
+                   const matrix::Ell<MatrixValueType, IndexType> *a,
+                   const matrix::Dense<InputValueType> *b,
+                   matrix::Dense<OutputValueType> *c,
+                   const matrix::Dense<MatrixValueType> *alpha = nullptr,
+                   const matrix::Dense<OutputValueType> *beta = nullptr)
 {
     const auto nrows = a->get_size()[0];
     constexpr int num_thread_per_worker =
@@ -194,10 +195,12 @@ std::array<int, 3> compute_thread_worker_and_atomicity(
 }  // namespace
 
 
-template <typename ValueType, typename IndexType>
+template <typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
 void spmv(std::shared_ptr<const CudaExecutor> exec,
-          const matrix::Ell<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+          const matrix::Ell<MatrixValueType, IndexType> *a,
+          const matrix::Dense<InputValueType> *b,
+          matrix::Dense<OutputValueType> *c)
 {
     const auto data = compute_thread_worker_and_atomicity(exec, a);
     const int num_thread_per_worker = std::get<0>(data);
@@ -212,7 +215,8 @@ void spmv(std::shared_ptr<const CudaExecutor> exec,
     const int info = (!atomic) * num_thread_per_worker;
     if (atomic) {
         components::fill_array(exec, c->get_values(),
-                               c->get_num_stored_elements(), zero<ValueType>());
+                               c->get_num_stored_elements(),
+                               zero<OutputValueType>());
     }
     select_abstract_spmv(
         compiled_kernels(),
@@ -221,16 +225,18 @@ void spmv(std::shared_ptr<const CudaExecutor> exec,
         c);
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ELL_SPMV_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ELL_SPMV_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
+template <typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
 void advanced_spmv(std::shared_ptr<const CudaExecutor> exec,
-                   const matrix::Dense<ValueType> *alpha,
-                   const matrix::Ell<ValueType, IndexType> *a,
-                   const matrix::Dense<ValueType> *b,
-                   const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c)
+                   const matrix::Dense<MatrixValueType> *alpha,
+                   const matrix::Ell<MatrixValueType, IndexType> *a,
+                   const matrix::Dense<InputValueType> *b,
+                   const matrix::Dense<OutputValueType> *beta,
+                   matrix::Dense<OutputValueType> *c)
 {
     const auto data = compute_thread_worker_and_atomicity(exec, a);
     const int num_thread_per_worker = std::get<0>(data);
@@ -253,7 +259,7 @@ void advanced_spmv(std::shared_ptr<const CudaExecutor> exec,
         alpha, beta);
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL);
 
 
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 2c77aaef826..2f6da1e9e42 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -524,6 +524,57 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
 #endif
 
 
+#define GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(_macro) \
+    template _macro(float, float, float, int32);                    \
+    template _macro(float, float, double, int32);                   \
+    template _macro(float, double, float, int32);                   \
+    template _macro(float, double, double, int32);                  \
+    template _macro(double, float, float, int32);                   \
+    template _macro(double, float, double, int32);                  \
+    template _macro(double, double, float, int32);                  \
+    template _macro(double, double, double, int32);                 \
+    template _macro(std::complex<float>, std::complex<float>,       \
+                    std::complex<float>, int32);                    \
+    template _macro(std::complex<float>, std::complex<float>,       \
+                    std::complex<double>, int32);                   \
+    template _macro(std::complex<float>, std::complex<double>,      \
+                    std::complex<float>, int32);                    \
+    template _macro(std::complex<float>, std::complex<double>,      \
+                    std::complex<double>, int32);                   \
+    template _macro(std::complex<double>, std::complex<float>,      \
+                    std::complex<float>, int32);                    \
+    template _macro(std::complex<double>, std::complex<float>,      \
+                    std::complex<double>, int32);                   \
+    template _macro(std::complex<double>, std::complex<double>,     \
+                    std::complex<float>, int32);                    \
+    template _macro(std::complex<double>, std::complex<double>,     \
+                    std::complex<double>, int32);                   \
+    template _macro(float, float, float, int64);                    \
+    template _macro(float, float, double, int64);                   \
+    template _macro(float, double, float, int64);                   \
+    template _macro(float, double, double, int64);                  \
+    template _macro(double, float, float, int64);                   \
+    template _macro(double, float, double, int64);                  \
+    template _macro(double, double, float, int64);                  \
+    template _macro(double, double, double, int64);                 \
+    template _macro(std::complex<float>, std::complex<float>,       \
+                    std::complex<float>, int64);                    \
+    template _macro(std::complex<float>, std::complex<float>,       \
+                    std::complex<double>, int64);                   \
+    template _macro(std::complex<float>, std::complex<double>,      \
+                    std::complex<float>, int64);                    \
+    template _macro(std::complex<float>, std::complex<double>,      \
+                    std::complex<double>, int64);                   \
+    template _macro(std::complex<double>, std::complex<float>,      \
+                    std::complex<float>, int64);                    \
+    template _macro(std::complex<double>, std::complex<float>,      \
+                    std::complex<double>, int64);                   \
+    template _macro(std::complex<double>, std::complex<double>,     \
+                    std::complex<float>, int64);                    \
+    template _macro(std::complex<double>, std::complex<double>,     \
+                    std::complex<double>, int64)
+
+
 /**
  * Instantiates a template for each value type conversion pair compiled by
  * Ginkgo.
diff --git a/omp/matrix/ell_kernels.cpp b/omp/matrix/ell_kernels.cpp
index a590357a1c2..621e1cfe940 100644
--- a/omp/matrix/ell_kernels.cpp
+++ b/omp/matrix/ell_kernels.cpp
@@ -56,38 +56,44 @@ namespace omp {
 namespace ell {
 
 
-template <typename ValueType, typename IndexType>
+template <typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
 void spmv(std::shared_ptr<const OmpExecutor> exec,
-          const matrix::Ell<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+          const matrix::Ell<MatrixValueType, IndexType> *a,
+          const matrix::Dense<InputValueType> *b,
+          matrix::Dense<OutputValueType> *c)
 {
     auto num_stored_elements_per_row = a->get_num_stored_elements_per_row();
 
 #pragma omp parallel for
     for (size_type row = 0; row < a->get_size()[0]; row++) {
         for (size_type j = 0; j < c->get_size()[1]; j++) {
-            c->at(row, j) = zero<ValueType>();
+            c->at(row, j) = zero<OutputValueType>();
         }
         for (size_type i = 0; i < num_stored_elements_per_row; i++) {
             auto val = a->val_at(row, i);
             auto col = a->col_at(row, i);
+
             for (size_type j = 0; j < c->get_size()[1]; j++) {
-                c->at(row, j) += val * b->at(col, j);
+                c->at(row, j) +=
+                    OutputValueType(val * MatrixValueType(b->at(col, j)));
             }
         }
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ELL_SPMV_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ELL_SPMV_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
+template <typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
 void advanced_spmv(std::shared_ptr<const OmpExecutor> exec,
-                   const matrix::Dense<ValueType> *alpha,
-                   const matrix::Ell<ValueType, IndexType> *a,
-                   const matrix::Dense<ValueType> *b,
-                   const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c)
+                   const matrix::Dense<MatrixValueType> *alpha,
+                   const matrix::Ell<MatrixValueType, IndexType> *a,
+                   const matrix::Dense<InputValueType> *b,
+                   const matrix::Dense<OutputValueType> *beta,
+                   matrix::Dense<OutputValueType> *c)
 {
     auto num_stored_elements_per_row = a->get_num_stored_elements_per_row();
     auto alpha_val = alpha->at(0, 0);
@@ -102,13 +108,14 @@ void advanced_spmv(std::shared_ptr<const OmpExecutor> exec,
             auto val = a->val_at(row, i);
             auto col = a->col_at(row, i);
             for (size_type j = 0; j < c->get_size()[1]; j++) {
-                c->at(row, j) += alpha_val * val * b->at(col, j);
+                c->at(row, j) += OutputValueType(
+                    alpha_val * val * MatrixValueType(b->at(col, j)));
             }
         }
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL);
 
 
diff --git a/reference/matrix/ell_kernels.cpp b/reference/matrix/ell_kernels.cpp
index fdbaff5dfc5..fee0dd5f7fc 100644
--- a/reference/matrix/ell_kernels.cpp
+++ b/reference/matrix/ell_kernels.cpp
@@ -50,37 +50,42 @@ namespace reference {
 namespace ell {
 
 
-template <typename ValueType, typename IndexType>
+template <typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
 void spmv(std::shared_ptr<const ReferenceExecutor> exec,
-          const matrix::Ell<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+          const matrix::Ell<MatrixValueType, IndexType> *a,
+          const matrix::Dense<InputValueType> *b,
+          matrix::Dense<OutputValueType> *c)
 {
     auto num_stored_elements_per_row = a->get_num_stored_elements_per_row();
 
     for (size_type row = 0; row < a->get_size()[0]; row++) {
         for (size_type j = 0; j < c->get_size()[1]; j++) {
-            c->at(row, j) = zero<ValueType>();
+            c->at(row, j) = zero<OutputValueType>();
         }
         for (size_type i = 0; i < num_stored_elements_per_row; i++) {
             auto val = a->val_at(row, i);
             auto col = a->col_at(row, i);
             for (size_type j = 0; j < c->get_size()[1]; j++) {
-                c->at(row, j) += val * b->at(col, j);
+                c->at(row, j) +=
+                    OutputValueType(val * MatrixValueType(b->at(col, j)));
             }
         }
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ELL_SPMV_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ELL_SPMV_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
+template <typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
 void advanced_spmv(std::shared_ptr<const ReferenceExecutor> exec,
-                   const matrix::Dense<ValueType> *alpha,
-                   const matrix::Ell<ValueType, IndexType> *a,
-                   const matrix::Dense<ValueType> *b,
-                   const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c)
+                   const matrix::Dense<MatrixValueType> *alpha,
+                   const matrix::Ell<MatrixValueType, IndexType> *a,
+                   const matrix::Dense<InputValueType> *b,
+                   const matrix::Dense<OutputValueType> *beta,
+                   matrix::Dense<OutputValueType> *c)
 {
     auto num_stored_elements_per_row = a->get_num_stored_elements_per_row();
     auto alpha_val = alpha->at(0, 0);
@@ -94,13 +99,14 @@ void advanced_spmv(std::shared_ptr<const ReferenceExecutor> exec,
             auto val = a->val_at(row, i);
             auto col = a->col_at(row, i);
             for (size_type j = 0; j < c->get_size()[1]; j++) {
-                c->at(row, j) += alpha_val * val * b->at(col, j);
+                c->at(row, j) += OutputValueType(
+                    alpha_val * val * MatrixValueType(b->at(col, j)));
             }
         }
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL);
 
 

From dedc96d7953e29ef523580fce242eb812a6c3d59 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Fri, 12 Mar 2021 16:40:50 +0100
Subject: [PATCH 0677/1008] fix benchmark formats, make ell spmv use accessor

---
 benchmark/utils/formats.hpp       |  8 +--
 common/matrix/ell_kernels.hpp.inc | 65 ++++++++++------------
 cuda/matrix/ell_kernels.cu        | 49 +++++++++++++----
 hip/matrix/ell_kernels.hip.cpp    | 89 ++++++++++++++++++++++---------
 omp/matrix/ell_kernels.cpp        | 48 +++++++++++++----
 reference/matrix/ell_kernels.cpp  | 49 +++++++++++++----
 6 files changed, 210 insertions(+), 98 deletions(-)

diff --git a/benchmark/utils/formats.hpp b/benchmark/utils/formats.hpp
index 6b344aec06d..ed20f91119e 100644
--- a/benchmark/utils/formats.hpp
+++ b/benchmark/utils/formats.hpp
@@ -208,10 +208,10 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
         {"csrc", READ_MATRIX(csr, std::make_shared<csr::classical>())},
         {"coo", read_matrix_from_data<gko::matrix::Coo<etype>>},
         {"ell", read_matrix_from_data<gko::matrix::Ell<etype>>},
-        {"fell",
+        {"ell_mixed",
          [](std::shared_ptr<const gko::Executor> exec,
-            const gko::matrix_data<> &data) {
-             gko::matrix_data<float> conv_data;
+            const gko::matrix_data<etype> &data) {
+             gko::matrix_data<gko::next_precision<etype>> conv_data;
              conv_data.size = data.size;
              conv_data.nonzeros.resize(data.nonzeros.size());
              auto it = conv_data.nonzeros.begin();
@@ -221,7 +221,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
                  it->value = el.value;
                  ++it;
              }
-             auto mat = gko::matrix::Ell<float>::create(std::move(exec));
+             auto mat = gko::matrix::Ell<gko::next_precision<etype>>::create(std::move(exec));
              mat->read(conv_data);
              return mat;
          }},
diff --git a/common/matrix/ell_kernels.hpp.inc b/common/matrix/ell_kernels.hpp.inc
index 2f155142efd..79ed4ed979b 100644
--- a/common/matrix/ell_kernels.hpp.inc
+++ b/common/matrix/ell_kernels.hpp.inc
@@ -34,32 +34,30 @@ namespace kernel {
 namespace {
 
 
-template <int num_thread_per_worker, bool atomic, typename InputValueType,
-          typename MatrixValueType, typename OutputValueType,
-          typename IndexType, typename Closure>
+template <int num_thread_per_worker, bool atomic, typename b_accessor,
+          typename a_accessor, typename OutputValueType, typename IndexType,
+          typename Closure>
 __device__ void spmv_kernel(
     const size_type num_rows, const int num_worker_per_row,
-    const MatrixValueType *__restrict__ val, const IndexType *__restrict__ col,
+    acc::range<a_accessor> val, const IndexType *__restrict__ col,
     const size_type stride, const size_type num_stored_elements_per_row,
-    const InputValueType *__restrict__ b, const size_type b_stride,
+    acc::range<b_accessor> b, const size_type b_stride,
     OutputValueType *__restrict__ c, const size_type c_stride, Closure op)
 {
     const auto tidx = thread::get_thread_id_flat();
-    const auto column_id = blockIdx.y;
-    using compute_type =
-        decltype(InputValueType{} + MatrixValueType{} + OutputValueType{});
+    const decltype(tidx) column_id = blockIdx.y;
     if (num_thread_per_worker == 1) {
         // Specialize the num_thread_per_worker = 1. It doesn't need the shared
         // memory, __syncthreads, and atomic_add
         if (tidx < num_rows) {
-            auto temp = zero<compute_type>();
+            auto temp = zero<OutputValueType>();
             for (size_type idx = 0; idx < num_stored_elements_per_row; idx++) {
                 const auto ind = tidx + idx * stride;
                 const auto col_idx = col[ind];
                 if (col_idx < idx) {
                     break;
                 } else {
-                    temp += val[ind] * b[col_idx * b_stride + column_id];
+                    temp += val(ind) * b(col_idx, column_id);
                 }
             }
             const auto c_ind = tidx * c_stride + column_id;
@@ -72,13 +70,13 @@ __device__ void spmv_kernel(
             const auto worker_id = tidx / num_rows;
             const auto step_size = num_worker_per_row * num_thread_per_worker;
             __shared__ UninitializedArray<
-                compute_type, default_block_size / num_thread_per_worker>
+                OutputValueType, default_block_size / num_thread_per_worker>
                 storage;
             if (idx_in_worker == 0) {
                 storage[threadIdx.x] = 0;
             }
             __syncthreads();
-            auto temp = zero<compute_type>();
+            auto temp = zero<OutputValueType>();
             for (size_type idx =
                      worker_id * num_thread_per_worker + idx_in_worker;
                  idx < num_stored_elements_per_row; idx += step_size) {
@@ -87,7 +85,7 @@ __device__ void spmv_kernel(
                 if (col_idx < idx) {
                     break;
                 } else {
-                    temp += val[ind] * b[col_idx * b_stride + column_id];
+                    temp += val(ind) * b(col_idx, column_id);
                 }
             }
             atomic_add(&storage[threadIdx.x], temp);
@@ -105,41 +103,34 @@ __device__ void spmv_kernel(
 }
 
 
-template <int num_thread_per_worker, bool atomic = false,
-          typename InputValueType, typename MatrixValueType,
-          typename OutputValueType, typename IndexType>
+template <int num_thread_per_worker, bool atomic = false, typename b_accessor,
+          typename a_accessor, typename OutputValueType, typename IndexType>
 __global__ __launch_bounds__(default_block_size) void spmv(
     const size_type num_rows, const int num_worker_per_row,
-    const MatrixValueType *__restrict__ val, const IndexType *__restrict__ col,
+    acc::range<a_accessor> val, const IndexType *__restrict__ col,
     const size_type stride, const size_type num_stored_elements_per_row,
-    const InputValueType *__restrict__ b, const size_type b_stride,
+    acc::range<b_accessor> b, const size_type b_stride,
     OutputValueType *__restrict__ c, const size_type c_stride)
 {
-    using compute_type =
-        decltype(InputValueType{} + MatrixValueType{} + OutputValueType{});
     spmv_kernel<num_thread_per_worker, atomic>(
         num_rows, num_worker_per_row, val, col, stride,
         num_stored_elements_per_row, b, b_stride, c, c_stride,
-        [](const compute_type &x, const OutputValueType &y) { return x; });
+        [](const OutputValueType &x, const OutputValueType &y) { return x; });
 }
 
 
-template <int num_thread_per_worker, bool atomic = false,
-          typename InputValueType, typename MatrixValueType,
-          typename OutputValueType, typename IndexType>
+template <int num_thread_per_worker, bool atomic = false, typename b_accessor,
+          typename a_accessor, typename OutputValueType, typename IndexType>
 __global__ __launch_bounds__(default_block_size) void spmv(
     const size_type num_rows, const int num_worker_per_row,
-    const MatrixValueType *__restrict__ alpha,
-    const MatrixValueType *__restrict__ val, const IndexType *__restrict__ col,
-    const size_type stride, const size_type num_stored_elements_per_row,
-    const InputValueType *__restrict__ b, const size_type b_stride,
-    const OutputValueType *__restrict__ beta, OutputValueType *__restrict__ c,
-    const size_type c_stride)
+    acc::range<a_accessor> alpha, acc::range<a_accessor> val,
+    const IndexType *__restrict__ col, const size_type stride,
+    const size_type num_stored_elements_per_row, acc::range<b_accessor> b,
+    const size_type b_stride, const OutputValueType *__restrict__ beta,
+    OutputValueType *__restrict__ c, const size_type c_stride)
 {
-    using compute_type =
-        decltype(InputValueType{} + MatrixValueType{} + OutputValueType{});
-    const compute_type alpha_val = alpha[0];
-    const compute_type beta_val = beta[0];
+    const OutputValueType alpha_val = alpha(0);
+    const OutputValueType beta_val = beta[0];
     // Because the atomic operation changes the values of c during computation,
     // it can not do the right alpha * a * b + beta * c operation.
     // Thus, the cuda kernel only computes alpha * a * b when it uses atomic
@@ -148,16 +139,16 @@ __global__ __launch_bounds__(default_block_size) void spmv(
         spmv_kernel<num_thread_per_worker, atomic>(
             num_rows, num_worker_per_row, val, col, stride,
             num_stored_elements_per_row, b, b_stride, c, c_stride,
-            [&alpha_val](const compute_type &x, const OutputValueType &y) {
+            [&alpha_val](const OutputValueType &x, const OutputValueType &y) {
                 return alpha_val * x;
             });
     } else {
         spmv_kernel<num_thread_per_worker, atomic>(
             num_rows, num_worker_per_row, val, col, stride,
             num_stored_elements_per_row, b, b_stride, c, c_stride,
-            [&alpha_val, &beta_val](const compute_type &x,
+            [&alpha_val, &beta_val](const OutputValueType &x,
                                     const OutputValueType &y) {
-                return alpha_val * x + beta_val * compute_type{y};
+                return alpha_val * x + beta_val * y;
             });
     }
 }
diff --git a/cuda/matrix/ell_kernels.cu b/cuda/matrix/ell_kernels.cu
index df223dfd7cf..33db65c4c1c 100644
--- a/cuda/matrix/ell_kernels.cu
+++ b/cuda/matrix/ell_kernels.cu
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "accessor/reduced_row_major.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/dense_kernels.hpp"
@@ -107,6 +108,17 @@ using compiled_kernels = syn::value_list<int, 0, 1, 2, 4, 8, 16, 32>;
 
 namespace {
 
+template <int dim, typename Type1, typename Type2>
+GKO_INLINE auto as_cuda_accessor(
+    const acc::range<acc::reduced_row_major<dim, Type1, Type2>> &acc)
+{
+    return acc::range<
+        acc::reduced_row_major<dim, cuda_type<Type1>, cuda_type<Type2>>>(
+        acc.get_accessor().get_size(),
+        as_cuda_type(acc.get_accessor().get_stored_data()),
+        acc.get_accessor().get_stride());
+}
+
 
 template <int info, typename InputValueType, typename MatrixValueType,
           typename OutputValueType, typename IndexType>
@@ -117,7 +129,16 @@ void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
                    const matrix::Dense<MatrixValueType> *alpha = nullptr,
                    const matrix::Dense<OutputValueType> *beta = nullptr)
 {
+    using a_accessor =
+        gko::acc::reduced_row_major<1, OutputValueType, const MatrixValueType>;
+    using b_accessor =
+        gko::acc::reduced_row_major<2, OutputValueType, const InputValueType>;
+
     const auto nrows = a->get_size()[0];
+    const auto stride = a->get_stride();
+    const auto num_stored_elements_per_row =
+        a->get_num_stored_elements_per_row();
+
     constexpr int num_thread_per_worker =
         (info == 0) ? max_thread_per_worker : info;
     constexpr bool atomic = (info == 0);
@@ -125,23 +146,31 @@ void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
                           num_thread_per_worker, 1);
     const dim3 grid_size(ceildiv(nrows * num_worker_per_row, block_size.x),
                          b->get_size()[1], 1);
+
+    const auto a_vals = gko::acc::range<a_accessor>(
+        std::array<long unsigned int, 1>{
+            {num_stored_elements_per_row * stride}},
+        a->get_const_values());
+    const auto b_vals = gko::acc::range<b_accessor>(
+        std::array<long unsigned int, 2>{{nrows, b->get_stride()}},
+        b->get_const_values());
+
     if (alpha == nullptr && beta == nullptr) {
         kernel::spmv<num_thread_per_worker, atomic>
             <<<grid_size, block_size, 0, 0>>>(
-                nrows, num_worker_per_row, as_cuda_type(a->get_const_values()),
-                a->get_const_col_idxs(), a->get_stride(),
-                a->get_num_stored_elements_per_row(),
-                as_cuda_type(b->get_const_values()), b->get_stride(),
+                nrows, num_worker_per_row, as_cuda_accessor(a_vals),
+                a->get_const_col_idxs(), stride, num_stored_elements_per_row,
+                as_cuda_accessor(b_vals), b->get_stride(),
                 as_cuda_type(c->get_values()), c->get_stride());
     } else if (alpha != nullptr && beta != nullptr) {
+        const auto alpha_val = gko::acc::range<a_accessor>(
+            std::array<long unsigned int, 1>{1}, alpha->get_const_values());
         kernel::spmv<num_thread_per_worker, atomic>
             <<<grid_size, block_size, 0, 0>>>(
-                nrows, num_worker_per_row,
-                as_cuda_type(alpha->get_const_values()),
-                as_cuda_type(a->get_const_values()), a->get_const_col_idxs(),
-                a->get_stride(), a->get_num_stored_elements_per_row(),
-                as_cuda_type(b->get_const_values()), b->get_stride(),
-                as_cuda_type(beta->get_const_values()),
+                nrows, num_worker_per_row, as_cuda_accessor(alpha_val),
+                as_cuda_accessor(a_vals), a->get_const_col_idxs(), stride,
+                num_stored_elements_per_row, as_cuda_accessor(b_vals),
+                b->get_stride(), as_cuda_type(beta->get_const_values()),
                 as_cuda_type(c->get_values()), c->get_stride());
     } else {
         GKO_KERNEL_NOT_FOUND;
diff --git a/hip/matrix/ell_kernels.hip.cpp b/hip/matrix/ell_kernels.hip.cpp
index 2e7a4eeeb50..8bcadd02370 100644
--- a/hip/matrix/ell_kernels.hip.cpp
+++ b/hip/matrix/ell_kernels.hip.cpp
@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "accessor/reduced_row_major.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/dense_kernels.hpp"
@@ -111,15 +112,37 @@ using compiled_kernels = syn::value_list<int, 0, 1, 2, 4, 8, 16, 32>;
 namespace {
 
 
-template <int info, typename ValueType, typename IndexType>
+template <int dim, typename Type1, typename Type2>
+GKO_INLINE auto as_hip_accessor(
+    const acc::range<acc::reduced_row_major<dim, Type1, Type2>> &acc)
+{
+    return acc::range<
+        acc::reduced_row_major<dim, hip_type<Type1>, hip_type<Type2>>>(
+        acc.get_accessor().get_size(),
+        as_hip_type(acc.get_accessor().get_stored_data()),
+        acc.get_accessor().get_stride());
+}
+
+
+template <int info, typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
 void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
-                   const matrix::Ell<ValueType, IndexType> *a,
-                   const matrix::Dense<ValueType> *b,
-                   matrix::Dense<ValueType> *c,
-                   const matrix::Dense<ValueType> *alpha = nullptr,
-                   const matrix::Dense<ValueType> *beta = nullptr)
+                   const matrix::Ell<MatrixValueType, IndexType> *a,
+                   const matrix::Dense<InputValueType> *b,
+                   matrix::Dense<OutputValueType> *c,
+                   const matrix::Dense<MatrixValueType> *alpha = nullptr,
+                   const matrix::Dense<OutputValueType> *beta = nullptr)
 {
+    using a_accessor =
+        gko::acc::reduced_row_major<1, OutputValueType, const MatrixValueType>;
+    using b_accessor =
+        gko::acc::reduced_row_major<2, OutputValueType, const InputValueType>;
+
     const auto nrows = a->get_size()[0];
+    const auto stride = a->get_stride();
+    const auto num_stored_elements_per_row =
+        a->get_num_stored_elements_per_row();
+
     constexpr int num_thread_per_worker =
         (info == 0) ? max_thread_per_worker : info;
     constexpr bool atomic = (info == 0);
@@ -127,22 +150,31 @@ void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
                           num_thread_per_worker, 1);
     const dim3 grid_size(ceildiv(nrows * num_worker_per_row, block_size.x),
                          b->get_size()[1], 1);
+
+    const auto a_vals = gko::acc::range<a_accessor>(
+        std::array<long unsigned int, 1>{
+            {num_stored_elements_per_row * stride}},
+        a->get_const_values());
+    const auto b_vals = gko::acc::range<b_accessor>(
+        std::array<long unsigned int, 2>{{nrows, b->get_stride()}},
+        b->get_const_values());
+
     if (alpha == nullptr && beta == nullptr) {
         hipLaunchKernelGGL(
             HIP_KERNEL_NAME(kernel::spmv<num_thread_per_worker, atomic>),
             dim3(grid_size), dim3(block_size), 0, 0, nrows, num_worker_per_row,
-            as_hip_type(a->get_const_values()), a->get_const_col_idxs(),
-            a->get_stride(), a->get_num_stored_elements_per_row(),
-            as_hip_type(b->get_const_values()), b->get_stride(),
-            as_hip_type(c->get_values()), c->get_stride());
+            as_hip_accessor(a_vals), a->get_const_col_idxs(), stride,
+            num_stored_elements_per_row, as_hip_accessor(b_vals),
+            b->get_stride(), as_hip_type(c->get_values()), c->get_stride());
     } else if (alpha != nullptr && beta != nullptr) {
+        const auto alpha_val = gko::acc::range<a_accessor>(
+            std::array<long unsigned int, 1>{1}, alpha->get_const_values());
         hipLaunchKernelGGL(
             HIP_KERNEL_NAME(kernel::spmv<num_thread_per_worker, atomic>),
             dim3(grid_size), dim3(block_size), 0, 0, nrows, num_worker_per_row,
-            as_hip_type(alpha->get_const_values()),
-            as_hip_type(a->get_const_values()), a->get_const_col_idxs(),
-            a->get_stride(), a->get_num_stored_elements_per_row(),
-            as_hip_type(b->get_const_values()), b->get_stride(),
+            as_hip_accessor(alpha_val), as_hip_accessor(a_vals),
+            a->get_const_col_idxs(), stride, num_stored_elements_per_row,
+            as_hip_accessor(b_vals), b->get_stride(),
             as_hip_type(beta->get_const_values()), as_hip_type(c->get_values()),
             c->get_stride());
     } else {
@@ -197,10 +229,12 @@ std::array<int, 3> compute_thread_worker_and_atomicity(
 }  // namespace
 
 
-template <typename ValueType, typename IndexType>
+template <typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
 void spmv(std::shared_ptr<const HipExecutor> exec,
-          const matrix::Ell<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+          const matrix::Ell<MatrixValueType, IndexType> *a,
+          const matrix::Dense<InputValueType> *b,
+          matrix::Dense<OutputValueType> *c)
 {
     const auto data = compute_thread_worker_and_atomicity(exec, a);
     const int num_thread_per_worker = std::get<0>(data);
@@ -215,7 +249,8 @@ void spmv(std::shared_ptr<const HipExecutor> exec,
     const int info = (!atomic) * num_thread_per_worker;
     if (atomic) {
         components::fill_array(exec, c->get_values(),
-                               c->get_num_stored_elements(), zero<ValueType>());
+                               c->get_num_stored_elements(),
+                               zero<OutputValueType>());
     }
     select_abstract_spmv(
         compiled_kernels(),
@@ -224,16 +259,18 @@ void spmv(std::shared_ptr<const HipExecutor> exec,
         c);
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ELL_SPMV_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ELL_SPMV_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
+template <typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
 void advanced_spmv(std::shared_ptr<const HipExecutor> exec,
-                   const matrix::Dense<ValueType> *alpha,
-                   const matrix::Ell<ValueType, IndexType> *a,
-                   const matrix::Dense<ValueType> *b,
-                   const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c)
+                   const matrix::Dense<MatrixValueType> *alpha,
+                   const matrix::Ell<MatrixValueType, IndexType> *a,
+                   const matrix::Dense<InputValueType> *b,
+                   const matrix::Dense<OutputValueType> *beta,
+                   matrix::Dense<OutputValueType> *c)
 {
     const auto data = compute_thread_worker_and_atomicity(exec, a);
     const int num_thread_per_worker = std::get<0>(data);
@@ -256,7 +293,7 @@ void advanced_spmv(std::shared_ptr<const HipExecutor> exec,
         alpha, beta);
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL);
 
 
diff --git a/omp/matrix/ell_kernels.cpp b/omp/matrix/ell_kernels.cpp
index 621e1cfe940..76e7eea80ed 100644
--- a/omp/matrix/ell_kernels.cpp
+++ b/omp/matrix/ell_kernels.cpp
@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "accessor/reduced_row_major.hpp"
 #include "omp/components/format_conversion.hpp"
 
 
@@ -63,7 +64,21 @@ void spmv(std::shared_ptr<const OmpExecutor> exec,
           const matrix::Dense<InputValueType> *b,
           matrix::Dense<OutputValueType> *c)
 {
-    auto num_stored_elements_per_row = a->get_num_stored_elements_per_row();
+    using a_accessor =
+        gko::acc::reduced_row_major<1, OutputValueType, const MatrixValueType>;
+    using b_accessor =
+        gko::acc::reduced_row_major<2, OutputValueType, const InputValueType>;
+
+    const auto num_stored_elements_per_row =
+        a->get_num_stored_elements_per_row();
+    const auto stride = a->get_stride();
+    const auto num_rows = b->get_size()[0];
+    const auto a_vals = gko::acc::range<a_accessor>(
+        std::array<long unsigned int, 1>{num_stored_elements_per_row * stride},
+        a->get_const_values());
+    const auto b_vals = gko::acc::range<b_accessor>(
+        std::array<long unsigned int, 2>{num_rows, b->get_stride()},
+        b->get_const_values());
 
 #pragma omp parallel for
     for (size_type row = 0; row < a->get_size()[0]; row++) {
@@ -71,12 +86,10 @@ void spmv(std::shared_ptr<const OmpExecutor> exec,
             c->at(row, j) = zero<OutputValueType>();
         }
         for (size_type i = 0; i < num_stored_elements_per_row; i++) {
-            auto val = a->val_at(row, i);
+            auto val = a_vals(row + i * stride);
             auto col = a->col_at(row, i);
-
             for (size_type j = 0; j < c->get_size()[1]; j++) {
-                c->at(row, j) +=
-                    OutputValueType(val * MatrixValueType(b->at(col, j)));
+                c->at(row, j) += val * b_vals(col, j);
             }
         }
     }
@@ -95,9 +108,23 @@ void advanced_spmv(std::shared_ptr<const OmpExecutor> exec,
                    const matrix::Dense<OutputValueType> *beta,
                    matrix::Dense<OutputValueType> *c)
 {
-    auto num_stored_elements_per_row = a->get_num_stored_elements_per_row();
-    auto alpha_val = alpha->at(0, 0);
-    auto beta_val = beta->at(0, 0);
+    using a_accessor =
+        gko::acc::reduced_row_major<1, OutputValueType, const MatrixValueType>;
+    using b_accessor =
+        gko::acc::reduced_row_major<2, OutputValueType, const InputValueType>;
+
+    const auto num_stored_elements_per_row =
+        a->get_num_stored_elements_per_row();
+    const auto stride = a->get_stride();
+    const auto num_rows = b->get_size()[0];
+    const auto a_vals = gko::acc::range<a_accessor>(
+        std::array<long unsigned int, 1>{num_stored_elements_per_row * stride},
+        a->get_const_values());
+    const auto b_vals = gko::acc::range<b_accessor>(
+        std::array<long unsigned int, 2>{num_rows, b->get_stride()},
+        b->get_const_values());
+    const auto alpha_val = OutputValueType(alpha->at(0, 0));
+    const auto beta_val = beta->at(0, 0);
 
 #pragma omp parallel for
     for (size_type row = 0; row < a->get_size()[0]; row++) {
@@ -105,11 +132,10 @@ void advanced_spmv(std::shared_ptr<const OmpExecutor> exec,
             c->at(row, j) *= beta_val;
         }
         for (size_type i = 0; i < num_stored_elements_per_row; i++) {
-            auto val = a->val_at(row, i);
+            auto val = a_vals(row + i * stride);
             auto col = a->col_at(row, i);
             for (size_type j = 0; j < c->get_size()[1]; j++) {
-                c->at(row, j) += OutputValueType(
-                    alpha_val * val * MatrixValueType(b->at(col, j)));
+                c->at(row, j) += alpha_val * val * b_vals(col, j);
             }
         }
     }
diff --git a/reference/matrix/ell_kernels.cpp b/reference/matrix/ell_kernels.cpp
index fee0dd5f7fc..ba6d687d18a 100644
--- a/reference/matrix/ell_kernels.cpp
+++ b/reference/matrix/ell_kernels.cpp
@@ -39,6 +39,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "accessor/reduced_row_major.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace reference {
@@ -57,18 +60,31 @@ void spmv(std::shared_ptr<const ReferenceExecutor> exec,
           const matrix::Dense<InputValueType> *b,
           matrix::Dense<OutputValueType> *c)
 {
-    auto num_stored_elements_per_row = a->get_num_stored_elements_per_row();
+    using a_accessor =
+        gko::acc::reduced_row_major<1, OutputValueType, const MatrixValueType>;
+    using b_accessor =
+        gko::acc::reduced_row_major<2, OutputValueType, const InputValueType>;
+
+    const auto num_stored_elements_per_row =
+        a->get_num_stored_elements_per_row();
+    const auto stride = a->get_stride();
+    const auto num_rows = b->get_size()[0];
+    const auto a_vals = gko::acc::range<a_accessor>(
+        std::array<long unsigned int, 1>{num_stored_elements_per_row * stride},
+        a->get_const_values());
+    const auto b_vals = gko::acc::range<b_accessor>(
+        std::array<long unsigned int, 2>{num_rows, b->get_stride()},
+        b->get_const_values());
 
     for (size_type row = 0; row < a->get_size()[0]; row++) {
         for (size_type j = 0; j < c->get_size()[1]; j++) {
             c->at(row, j) = zero<OutputValueType>();
         }
         for (size_type i = 0; i < num_stored_elements_per_row; i++) {
-            auto val = a->val_at(row, i);
+            auto val = a_vals(row + i * stride);
             auto col = a->col_at(row, i);
             for (size_type j = 0; j < c->get_size()[1]; j++) {
-                c->at(row, j) +=
-                    OutputValueType(val * MatrixValueType(b->at(col, j)));
+                c->at(row, j) += val * b_vals(col, j);
             }
         }
     }
@@ -87,20 +103,33 @@ void advanced_spmv(std::shared_ptr<const ReferenceExecutor> exec,
                    const matrix::Dense<OutputValueType> *beta,
                    matrix::Dense<OutputValueType> *c)
 {
-    auto num_stored_elements_per_row = a->get_num_stored_elements_per_row();
-    auto alpha_val = alpha->at(0, 0);
-    auto beta_val = beta->at(0, 0);
+    using a_accessor =
+        gko::acc::reduced_row_major<1, OutputValueType, const MatrixValueType>;
+    using b_accessor =
+        gko::acc::reduced_row_major<2, OutputValueType, const InputValueType>;
+
+    const auto num_stored_elements_per_row =
+        a->get_num_stored_elements_per_row();
+    const auto stride = a->get_stride();
+    const auto num_rows = b->get_size()[0];
+    const auto a_vals = gko::acc::range<a_accessor>(
+        std::array<long unsigned int, 1>{num_stored_elements_per_row * stride},
+        a->get_const_values());
+    const auto b_vals = gko::acc::range<b_accessor>(
+        std::array<long unsigned int, 2>{num_rows, b->get_stride()},
+        b->get_const_values());
+    const auto alpha_val = OutputValueType(alpha->at(0, 0));
+    const auto beta_val = beta->at(0, 0);
 
     for (size_type row = 0; row < a->get_size()[0]; row++) {
         for (size_type j = 0; j < c->get_size()[1]; j++) {
             c->at(row, j) *= beta_val;
         }
         for (size_type i = 0; i < num_stored_elements_per_row; i++) {
-            auto val = a->val_at(row, i);
+            auto val = a_vals(row + i * stride);
             auto col = a->col_at(row, i);
             for (size_type j = 0; j < c->get_size()[1]; j++) {
-                c->at(row, j) += OutputValueType(
-                    alpha_val * val * MatrixValueType(b->at(col, j)));
+                c->at(row, j) += alpha_val * val * b_vals(col, j);
             }
         }
     }

From ffbb5124e4d9984d707be122374fcb8eb43f4d6e Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Fri, 12 Mar 2021 17:55:32 +0100
Subject: [PATCH 0678/1008] Add tests for mixed precision ELL

---
 cuda/test/matrix/ell_kernels.cpp      | 222 +++++++++++++++++++-
 hip/test/matrix/ell_kernels.hip.cpp   | 221 ++++++++++++++++++++
 omp/test/matrix/ell_kernels.cpp       | 289 +++++++++++++++++++++++++-
 reference/test/matrix/ell_kernels.cpp | 254 +++++++++++++++++++++-
 4 files changed, 974 insertions(+), 12 deletions(-)

diff --git a/cuda/test/matrix/ell_kernels.cpp b/cuda/test/matrix/ell_kernels.cpp
index dff573a284f..e24f7083728 100644
--- a/cuda/test/matrix/ell_kernels.cpp
+++ b/cuda/test/matrix/ell_kernels.cpp
@@ -58,6 +58,7 @@ class Ell : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Ell<>;
     using Vec = gko::matrix::Dense<>;
+    using Vec2 = gko::matrix::Dense<float>;
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Ell() : rand_engine(42) {}
@@ -92,22 +93,35 @@ class Ell : public ::testing::Test {
                           stride);
         mtx->copy_from(gen_mtx(num_rows, num_cols));
         expected = gen_mtx(num_rows, num_vectors);
+        expected2 = Vec2::create(ref);
+        expected2->copy_from(expected.get());
         y = gen_mtx(num_cols, num_vectors);
+        y2 = Vec2::create(ref);
+        y2->copy_from(y.get());
         alpha = gko::initialize<Vec>({2.0}, ref);
+        alpha2 = gko::initialize<Vec2>({2.0}, ref);
         beta = gko::initialize<Vec>({-1.0}, ref);
+        beta2 = gko::initialize<Vec2>({-1.0}, ref);
         dmtx = Mtx::create(cuda);
         dmtx->copy_from(mtx.get());
         dresult = Vec::create(cuda);
         dresult->copy_from(expected.get());
+        dresult2 = Vec2::create(cuda);
+        dresult2->copy_from(expected2.get());
         dy = Vec::create(cuda);
         dy->copy_from(y.get());
+        dy2 = Vec2::create(cuda);
+        dy2->copy_from(y2.get());
         dalpha = Vec::create(cuda);
         dalpha->copy_from(alpha.get());
+        dalpha2 = Vec2::create(cuda);
+        dalpha2->copy_from(alpha2.get());
         dbeta = Vec::create(cuda);
         dbeta->copy_from(beta.get());
+        dbeta2 = Vec2::create(cuda);
+        dbeta2->copy_from(beta2.get());
     }
 
-
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::CudaExecutor> cuda;
 
@@ -115,15 +129,23 @@ class Ell : public ::testing::Test {
 
     std::unique_ptr<Mtx> mtx;
     std::unique_ptr<Vec> expected;
+    std::unique_ptr<Vec2> expected2;
     std::unique_ptr<Vec> y;
+    std::unique_ptr<Vec2> y2;
     std::unique_ptr<Vec> alpha;
+    std::unique_ptr<Vec2> alpha2;
     std::unique_ptr<Vec> beta;
+    std::unique_ptr<Vec2> beta2;
 
     std::unique_ptr<Mtx> dmtx;
     std::unique_ptr<Vec> dresult;
+    std::unique_ptr<Vec2> dresult2;
     std::unique_ptr<Vec> dy;
+    std::unique_ptr<Vec2> dy2;
     std::unique_ptr<Vec> dalpha;
+    std::unique_ptr<Vec2> dalpha2;
     std::unique_ptr<Vec> dbeta;
+    std::unique_ptr<Vec2> dbeta2;
 };
 
 
@@ -138,6 +160,39 @@ TEST_F(Ell, SimpleApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef1)
+{
+    set_up_apply_data();
+
+    mtx->apply(y2.get(), expected2.get());
+    dmtx->apply(dy2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef2)
+{
+    set_up_apply_data();
+
+    mtx->apply(y2.get(), expected.get());
+    dmtx->apply(dy2.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef3)
+{
+    set_up_apply_data();
+
+    mtx->apply(y.get(), expected2.get());
+    dmtx->apply(dy.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, AdvancedApplyIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -149,6 +204,39 @@ TEST_F(Ell, AdvancedApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef1)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef2)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef3)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, SimpleApplyWithStrideIsEquivalentToRef)
 {
     set_up_apply_data(532, 231, 1, 300, 600);
@@ -160,6 +248,39 @@ TEST_F(Ell, SimpleApplyWithStrideIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef1)
+{
+    set_up_apply_data(532, 231, 1, 300, 600);
+
+    mtx->apply(y2.get(), expected2.get());
+    dmtx->apply(dy2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef2)
+{
+    set_up_apply_data(532, 231, 1, 300, 600);
+
+    mtx->apply(y2.get(), expected.get());
+    dmtx->apply(dy2.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef3)
+{
+    set_up_apply_data(532, 231, 1, 300, 600);
+
+    mtx->apply(y.get(), expected2.get());
+    dmtx->apply(dy.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, AdvancedApplyWithStrideIsEquivalentToRef)
 {
     set_up_apply_data(532, 231, 1, 300, 600);
@@ -170,6 +291,39 @@ TEST_F(Ell, AdvancedApplyWithStrideIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef1)
+{
+    set_up_apply_data(532, 231, 1, 300, 600);
+
+    mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef2)
+{
+    set_up_apply_data(532, 231, 1, 300, 600);
+
+    mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef3)
+{
+    set_up_apply_data(532, 231, 1, 300, 600);
+
+    mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, SimpleApplyWithStrideToDenseMatrixIsEquivalentToRef)
 {
     set_up_apply_data(532, 231, 3, 300, 600);
@@ -181,6 +335,39 @@ TEST_F(Ell, SimpleApplyWithStrideToDenseMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef1)
+{
+    set_up_apply_data(532, 231, 3, 300, 600);
+
+    mtx->apply(y2.get(), expected2.get());
+    dmtx->apply(dy2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef2)
+{
+    set_up_apply_data(532, 231, 3, 300, 600);
+
+    mtx->apply(y2.get(), expected.get());
+    dmtx->apply(dy2.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef3)
+{
+    set_up_apply_data(532, 231, 3, 300, 600);
+
+    mtx->apply(y.get(), expected2.get());
+    dmtx->apply(dy.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, AdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef)
 {
     set_up_apply_data(532, 231, 3, 300, 600);
@@ -192,6 +379,39 @@ TEST_F(Ell, AdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef1)
+{
+    set_up_apply_data(532, 231, 3, 300, 600);
+
+    mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef2)
+{
+    set_up_apply_data(532, 231, 3, 300, 600);
+
+    mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef3)
+{
+    set_up_apply_data(532, 231, 3, 300, 600);
+
+    mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, SimpleApplyByAtomicIsEquivalentToRef)
 {
     set_up_apply_data(10, 10000);
diff --git a/hip/test/matrix/ell_kernels.hip.cpp b/hip/test/matrix/ell_kernels.hip.cpp
index e9a875eb544..77253125abe 100644
--- a/hip/test/matrix/ell_kernels.hip.cpp
+++ b/hip/test/matrix/ell_kernels.hip.cpp
@@ -58,6 +58,7 @@ class Ell : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Ell<>;
     using Vec = gko::matrix::Dense<>;
+    using Vec2 = gko::matrix::Dense<float>;
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Ell() : rand_engine(42) {}
@@ -92,19 +93,33 @@ class Ell : public ::testing::Test {
                           stride);
         mtx->copy_from(gen_mtx(num_rows, num_cols));
         expected = gen_mtx(num_rows, num_vectors);
+        expected2 = Vec2::create(ref);
+        expected2->copy_from(expected.get());
         y = gen_mtx(num_cols, num_vectors);
+        y2 = Vec2::create(ref);
+        y2->copy_from(y.get());
         alpha = gko::initialize<Vec>({2.0}, ref);
+        alpha2 = gko::initialize<Vec2>({2.0}, ref);
         beta = gko::initialize<Vec>({-1.0}, ref);
+        beta2 = gko::initialize<Vec2>({-1.0}, ref);
         dmtx = Mtx::create(hip);
         dmtx->copy_from(mtx.get());
         dresult = Vec::create(hip);
         dresult->copy_from(expected.get());
+        dresult2 = Vec2::create(hip);
+        dresult2->copy_from(expected2.get());
         dy = Vec::create(hip);
         dy->copy_from(y.get());
+        dy2 = Vec2::create(hip);
+        dy2->copy_from(y2.get());
         dalpha = Vec::create(hip);
         dalpha->copy_from(alpha.get());
+        dalpha2 = Vec2::create(hip);
+        dalpha2->copy_from(alpha2.get());
         dbeta = Vec::create(hip);
         dbeta->copy_from(beta.get());
+        dbeta2 = Vec2::create(hip);
+        dbeta2->copy_from(beta2.get());
     }
 
 
@@ -115,15 +130,23 @@ class Ell : public ::testing::Test {
 
     std::unique_ptr<Mtx> mtx;
     std::unique_ptr<Vec> expected;
+    std::unique_ptr<Vec2> expected2;
     std::unique_ptr<Vec> y;
+    std::unique_ptr<Vec2> y2;
     std::unique_ptr<Vec> alpha;
+    std::unique_ptr<Vec2> alpha2;
     std::unique_ptr<Vec> beta;
+    std::unique_ptr<Vec2> beta2;
 
     std::unique_ptr<Mtx> dmtx;
     std::unique_ptr<Vec> dresult;
+    std::unique_ptr<Vec2> dresult2;
     std::unique_ptr<Vec> dy;
+    std::unique_ptr<Vec2> dy2;
     std::unique_ptr<Vec> dalpha;
+    std::unique_ptr<Vec2> dalpha2;
     std::unique_ptr<Vec> dbeta;
+    std::unique_ptr<Vec2> dbeta2;
 };
 
 
@@ -138,6 +161,39 @@ TEST_F(Ell, SimpleApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef1)
+{
+    set_up_apply_data();
+
+    mtx->apply(y2.get(), expected2.get());
+    dmtx->apply(dy2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef2)
+{
+    set_up_apply_data();
+
+    mtx->apply(y2.get(), expected.get());
+    dmtx->apply(dy2.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef3)
+{
+    set_up_apply_data();
+
+    mtx->apply(y.get(), expected2.get());
+    dmtx->apply(dy.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, AdvancedApplyIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -149,6 +205,39 @@ TEST_F(Ell, AdvancedApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef1)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef2)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef3)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, SimpleApplyWithStrideIsEquivalentToRef)
 {
     set_up_apply_data(532, 231, 1, 300, 600);
@@ -160,6 +249,39 @@ TEST_F(Ell, SimpleApplyWithStrideIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef1)
+{
+    set_up_apply_data(532, 231, 1, 300, 600);
+
+    mtx->apply(y2.get(), expected2.get());
+    dmtx->apply(dy2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef2)
+{
+    set_up_apply_data(532, 231, 1, 300, 600);
+
+    mtx->apply(y2.get(), expected.get());
+    dmtx->apply(dy2.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef3)
+{
+    set_up_apply_data(532, 231, 1, 300, 600);
+
+    mtx->apply(y.get(), expected2.get());
+    dmtx->apply(dy.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, AdvancedApplyWithStrideIsEquivalentToRef)
 {
     set_up_apply_data(532, 231, 1, 300, 600);
@@ -170,6 +292,39 @@ TEST_F(Ell, AdvancedApplyWithStrideIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef1)
+{
+    set_up_apply_data(532, 231, 1, 300, 600);
+
+    mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef2)
+{
+    set_up_apply_data(532, 231, 1, 300, 600);
+
+    mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef3)
+{
+    set_up_apply_data(532, 231, 1, 300, 600);
+
+    mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, SimpleApplyWithStrideToDenseMatrixIsEquivalentToRef)
 {
     set_up_apply_data(532, 231, 3, 300, 600);
@@ -181,6 +336,39 @@ TEST_F(Ell, SimpleApplyWithStrideToDenseMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef1)
+{
+    set_up_apply_data(532, 231, 3, 300, 600);
+
+    mtx->apply(y2.get(), expected2.get());
+    dmtx->apply(dy2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef2)
+{
+    set_up_apply_data(532, 231, 3, 300, 600);
+
+    mtx->apply(y2.get(), expected.get());
+    dmtx->apply(dy2.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef3)
+{
+    set_up_apply_data(532, 231, 3, 300, 600);
+
+    mtx->apply(y.get(), expected2.get());
+    dmtx->apply(dy.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, AdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef)
 {
     set_up_apply_data(532, 231, 3, 300, 600);
@@ -192,6 +380,39 @@ TEST_F(Ell, AdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef1)
+{
+    set_up_apply_data(532, 231, 3, 300, 600);
+
+    mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef2)
+{
+    set_up_apply_data(532, 231, 3, 300, 600);
+
+    mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef3)
+{
+    set_up_apply_data(532, 231, 3, 300, 600);
+
+    mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, SimpleApplyByAtomicIsEquivalentToRef)
 {
     set_up_apply_data(10, 10000);
diff --git a/omp/test/matrix/ell_kernels.cpp b/omp/test/matrix/ell_kernels.cpp
index 281292e9684..01f1a31121d 100644
--- a/omp/test/matrix/ell_kernels.cpp
+++ b/omp/test/matrix/ell_kernels.cpp
@@ -57,6 +57,7 @@ class Ell : public ::testing::Test {
 protected:
     using Mtx = gko::matrix::Ell<>;
     using Vec = gko::matrix::Dense<>;
+    using Vec2 = gko::matrix::Dense<float>;
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
     Ell() : rand_engine(42) {}
@@ -90,19 +91,33 @@ class Ell : public ::testing::Test {
         mtx = Mtx::create(ref, gko::dim<2>{}, max_nonzeros_per_row, stride);
         mtx->copy_from(gen_mtx(532, 231, 1));
         expected = gen_mtx(532, num_vectors, 1);
+        expected2 = Vec2::create(ref);
+        expected2->copy_from(expected.get());
         y = gen_mtx(231, num_vectors, 1);
+        y2 = Vec2::create(ref);
+        y2->copy_from(y.get());
         alpha = gko::initialize<Vec>({2.0}, ref);
+        alpha2 = gko::initialize<Vec2>({2.0}, ref);
         beta = gko::initialize<Vec>({-1.0}, ref);
+        beta2 = gko::initialize<Vec2>({-1.0}, ref);
         dmtx = Mtx::create(omp);
         dmtx->copy_from(mtx.get());
         dresult = Vec::create(omp);
         dresult->copy_from(expected.get());
+        dresult2 = Vec2::create(omp);
+        dresult2->copy_from(expected2.get());
         dy = Vec::create(omp);
         dy->copy_from(y.get());
+        dy2 = Vec2::create(omp);
+        dy2->copy_from(y2.get());
         dalpha = Vec::create(omp);
         dalpha->copy_from(alpha.get());
+        dalpha2 = Vec2::create(omp);
+        dalpha2->copy_from(alpha2.get());
         dbeta = Vec::create(omp);
         dbeta->copy_from(beta.get());
+        dbeta2 = Vec2::create(omp);
+        dbeta2->copy_from(beta2.get());
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -112,15 +127,23 @@ class Ell : public ::testing::Test {
 
     std::unique_ptr<Mtx> mtx;
     std::unique_ptr<Vec> expected;
+    std::unique_ptr<Vec2> expected2;
     std::unique_ptr<Vec> y;
+    std::unique_ptr<Vec2> y2;
     std::unique_ptr<Vec> alpha;
+    std::unique_ptr<Vec2> alpha2;
     std::unique_ptr<Vec> beta;
+    std::unique_ptr<Vec2> beta2;
 
     std::unique_ptr<Mtx> dmtx;
     std::unique_ptr<Vec> dresult;
+    std::unique_ptr<Vec2> dresult2;
     std::unique_ptr<Vec> dy;
+    std::unique_ptr<Vec2> dy2;
     std::unique_ptr<Vec> dalpha;
+    std::unique_ptr<Vec2> dalpha2;
     std::unique_ptr<Vec> dbeta;
+    std::unique_ptr<Vec2> dbeta2;
 };
 
 
@@ -135,6 +158,39 @@ TEST_F(Ell, SimpleApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef1)
+{
+    set_up_apply_data();
+
+    mtx->apply(y2.get(), expected2.get());
+    dmtx->apply(dy2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef2)
+{
+    set_up_apply_data();
+
+    mtx->apply(y2.get(), expected.get());
+    dmtx->apply(dy2.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef3)
+{
+    set_up_apply_data();
+
+    mtx->apply(y.get(), expected2.get());
+    dmtx->apply(dy.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, AdvancedApplyIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -146,6 +202,39 @@ TEST_F(Ell, AdvancedApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef1)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef2)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef3)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, SimpleApplyWithPaddingIsEquivalentToRef)
 {
     set_up_apply_data(300, 600);
@@ -157,6 +246,39 @@ TEST_F(Ell, SimpleApplyWithPaddingIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedSimpleApplyWithPaddingIsEquivalentToRef1)
+{
+    set_up_apply_data(300, 600);
+
+    mtx->apply(y2.get(), expected2.get());
+    dmtx->apply(dy2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithPaddingIsEquivalentToRef2)
+{
+    set_up_apply_data(300, 600);
+
+    mtx->apply(y2.get(), expected.get());
+    dmtx->apply(dy2.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithPaddingIsEquivalentToRef3)
+{
+    set_up_apply_data(300, 600);
+
+    mtx->apply(y.get(), expected2.get());
+    dmtx->apply(dy.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, AdvancedApplyWithPaddingIsEquivalentToRef)
 {
     set_up_apply_data(300, 600);
@@ -167,6 +289,39 @@ TEST_F(Ell, AdvancedApplyWithPaddingIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedAdvancedApplyWithPaddingIsEquivalentToRef1)
+{
+    set_up_apply_data(300, 600);
+
+    mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithPaddingIsEquivalentToRef2)
+{
+    set_up_apply_data(300, 600);
+
+    mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithPaddingIsEquivalentToRef3)
+{
+    set_up_apply_data(300, 600);
+
+    mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, SimpleApplyToDenseMatrixIsEquivalentToRef)
 {
     set_up_apply_data(0, 0, 3);
@@ -178,6 +333,39 @@ TEST_F(Ell, SimpleApplyToDenseMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedSimpleApplyToDenseMatrixIsEquivalentToRef1)
+{
+    set_up_apply_data(0, 0, 3);
+
+    mtx->apply(y2.get(), expected2.get());
+    dmtx->apply(dy2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyToDenseMatrixIsEquivalentToRef2)
+{
+    set_up_apply_data(0, 0, 3);
+
+    mtx->apply(y2.get(), expected.get());
+    dmtx->apply(dy2.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyToDenseMatrixIsEquivalentToRef3)
+{
+    set_up_apply_data(0, 0, 3);
+
+    mtx->apply(y.get(), expected2.get());
+    dmtx->apply(dy.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, AdvancedApplyToDenseMatrixIsEquivalentToRef)
 {
     set_up_apply_data(0, 0, 3);
@@ -189,7 +377,40 @@ TEST_F(Ell, AdvancedApplyToDenseMatrixIsEquivalentToRef)
 }
 
 
-TEST_F(Ell, SimpleApplyWithPaddinToDenseMatrixIsEquivalentToRef)
+TEST_F(Ell, MixedAdvancedApplyToDenseMatrixIsEquivalentToRef1)
+{
+    set_up_apply_data(0, 0, 3);
+
+    mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyToDenseMatrixIsEquivalentToRef2)
+{
+    set_up_apply_data(0, 0, 3);
+
+    mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyToDenseMatrixIsEquivalentToRef3)
+{
+    set_up_apply_data(0, 0, 3);
+
+    mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, SimpleApplyWithPaddingToDenseMatrixIsEquivalentToRef)
 {
     set_up_apply_data(300, 600, 3);
 
@@ -200,6 +421,39 @@ TEST_F(Ell, SimpleApplyWithPaddinToDenseMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedSimpleApplyWithPaddingToDenseMatrixIsEquivalentToRef1)
+{
+    set_up_apply_data(300, 600, 3);
+
+    mtx->apply(y2.get(), expected2.get());
+    dmtx->apply(dy2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithPaddingToDenseMatrixIsEquivalentToRef2)
+{
+    set_up_apply_data(300, 600, 3);
+
+    mtx->apply(y2.get(), expected.get());
+    dmtx->apply(dy2.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithPaddingToDenseMatrixIsEquivalentToRef3)
+{
+    set_up_apply_data(300, 600, 3);
+
+    mtx->apply(y.get(), expected2.get());
+    dmtx->apply(dy.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, AdvancedApplyWithPaddingToDenseMatrixIsEquivalentToRef)
 {
     set_up_apply_data(300, 600, 3);
@@ -210,6 +464,39 @@ TEST_F(Ell, AdvancedApplyWithPaddingToDenseMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Ell, MixedAdvancedApplyWithPaddingToDenseMatrixIsEquivalentToRef1)
+{
+    set_up_apply_data(300, 600, 3);
+
+    mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithPaddingToDenseMatrixIsEquivalentToRef2)
+{
+    set_up_apply_data(300, 600, 3);
+
+    mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithPaddingToDenseMatrixIsEquivalentToRef3)
+{
+    set_up_apply_data(300, 600, 3);
+
+    mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+}
+
+
 TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data(300, 600);
diff --git a/reference/test/matrix/ell_kernels.cpp b/reference/test/matrix/ell_kernels.cpp
index 77436ad1287..c3326737e4f 100644
--- a/reference/test/matrix/ell_kernels.cpp
+++ b/reference/test/matrix/ell_kernels.cpp
@@ -120,11 +120,46 @@ TYPED_TEST(Ell, AppliesToDenseVector)
 }
 
 
-TYPED_TEST(Ell, AppliesToMixedDenseVector)
+TYPED_TEST(Ell, MixedAppliesToDenseVector1)
 {
-    using MixedVec = typename TestFixture::MixedVec;
-    auto x = gko::initialize<MixedVec>({2.0, 1.0, 4.0}, this->exec);
-    auto y = MixedVec::create(this->exec, gko::dim<2>{2, 1});
+    // Both vectors have the same value type which differs from the matrix
+    using T = typename TestFixture::value_type;
+    using next_T = gko::next_precision<T>;
+    using Vec = typename gko::matrix::Dense<next_T>;
+    auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = Vec::create(this->exec, gko::dim<2>{2, 1});
+
+    this->mtx1->apply(x.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({13.0, 5.0}), 0.0);
+}
+
+
+TYPED_TEST(Ell, MixedAppliesToDenseVector2)
+{
+    // Input vector has same value type as matrix
+    using T = typename TestFixture::value_type;
+    using next_T = gko::next_precision<T>;
+    using Vec1 = typename TestFixture::Vec;
+    using Vec2 = gko::matrix::Dense<next_T>;
+    auto x = gko::initialize<Vec1>({2.0, 1.0, 4.0}, this->exec);
+    auto y = Vec2::create(this->exec, gko::dim<2>{2, 1});
+
+    this->mtx1->apply(x.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({13.0, 5.0}), 0.0);
+}
+
+
+TYPED_TEST(Ell, MixedAppliesToDenseVector3)
+{
+    // Output vector has same value type as matrix
+    using T = typename TestFixture::value_type;
+    using next_T = gko::next_precision<T>;
+    using Vec1 = typename TestFixture::Vec;
+    using Vec2 = gko::matrix::Dense<gko::next_precision<T>>;
+    auto x = gko::initialize<Vec2>({2.0, 1.0, 4.0}, this->exec);
+    auto y = Vec1::create(this->exec, gko::dim<2>{2, 1});
 
     this->mtx1->apply(x.get(), y.get());
 
@@ -154,6 +189,80 @@ TYPED_TEST(Ell, AppliesToDenseMatrix)
 }
 
 
+TYPED_TEST(Ell, MixedAppliesToDenseMatrix1)
+{
+    // Both vectors have the same value type which differs from the matrix
+    using T = typename TestFixture::value_type;
+    using next_T = gko::next_precision<T>;
+    using Vec = gko::matrix::Dense<next_T>;
+    // clang-format off
+    auto x = gko::initialize<Vec>(
+        {I<next_T>{2.0, 3.0},
+         I<next_T>{1.0, -1.5},
+         I<next_T>{4.0, 2.5}}, this->exec);
+    // clang-format on
+    auto y = Vec::create(this->exec, gko::dim<2>{2});
+
+    this->mtx1->apply(x.get(), y.get());
+
+    // clang-format off
+    GKO_ASSERT_MTX_NEAR(y,
+                        l({{13.0,  3.5},
+                           { 5.0, -7.5}}), 0.0);
+    // clang-format on
+}
+
+
+TYPED_TEST(Ell, MixedAppliesToDenseMatrix2)
+{
+    // Input vector has same value type as matrix
+    using T = typename TestFixture::value_type;
+    using next_T = gko::next_precision<T>;
+    using Vec1 = typename TestFixture::Vec;
+    using Vec2 = gko::matrix::Dense<next_T>;
+    // clang-format off
+    auto x = gko::initialize<Vec1>(
+        {I<T>{2.0, 3.0},
+         I<T>{1.0, -1.5},
+         I<T>{4.0, 2.5}}, this->exec);
+    // clang-format on
+    auto y = Vec2::create(this->exec, gko::dim<2>{2});
+
+    this->mtx1->apply(x.get(), y.get());
+
+    // clang-format off
+    GKO_ASSERT_MTX_NEAR(y,
+                        l({{13.0,  3.5},
+                           { 5.0, -7.5}}), 0.0);
+    // clang-format on
+}
+
+
+TYPED_TEST(Ell, MixedAppliesToDenseMatrix3)
+{
+    // Output vector has same value type as matrix
+    using T = typename TestFixture::value_type;
+    using next_T = gko::next_precision<T>;
+    using Vec1 = typename TestFixture::Vec;
+    using Vec2 = gko::matrix::Dense<next_T>;
+    // clang-format off
+    auto x = gko::initialize<Vec2>(
+        {I<next_T>{2.0, 3.0},
+         I<next_T>{1.0, -1.5},
+         I<next_T>{4.0, 2.5}}, this->exec);
+    // clang-format on
+    auto y = Vec1::create(this->exec, gko::dim<2>{2});
+
+    this->mtx1->apply(x.get(), y.get());
+
+    // clang-format off
+    GKO_ASSERT_MTX_NEAR(y,
+                        l({{13.0,  3.5},
+                           { 5.0, -7.5}}), 0.0);
+    // clang-format on
+}
+
+
 TYPED_TEST(Ell, AppliesLinearCombinationToDenseVector)
 {
     using Vec = typename TestFixture::Vec;
@@ -168,13 +277,52 @@ TYPED_TEST(Ell, AppliesLinearCombinationToDenseVector)
 }
 
 
-TYPED_TEST(Ell, AppliesLinearCombinationToMixedDenseVector)
+TYPED_TEST(Ell, MixedAppliesLinearCombinationToDenseVector1)
+{
+    // Both vectors have the same value type which differs from the matrix
+    using T = typename TestFixture::value_type;
+    using next_T = gko::next_precision<T>;
+    using Vec = gko::matrix::Dense<next_T>;
+    auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
+    auto beta = gko::initialize<Vec>({2.0}, this->exec);
+    auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = gko::initialize<Vec>({1.0, 2.0}, this->exec);
+
+    this->mtx1->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({-11.0, -1.0}), 0.0);
+}
+
+
+TYPED_TEST(Ell, MixedAppliesLinearCombinationToDenseVector2)
 {
-    using MixedVec = typename TestFixture::MixedVec;
-    auto alpha = gko::initialize<MixedVec>({-1.0}, this->exec);
-    auto beta = gko::initialize<MixedVec>({2.0}, this->exec);
-    auto x = gko::initialize<MixedVec>({2.0, 1.0, 4.0}, this->exec);
-    auto y = gko::initialize<MixedVec>({1.0, 2.0}, this->exec);
+    // Input vector has same value type as matrix
+    using T = typename TestFixture::value_type;
+    using next_T = gko::next_precision<T>;
+    using Vec1 = typename TestFixture::Vec;
+    using Vec2 = gko::matrix::Dense<next_T>;
+    auto alpha = gko::initialize<Vec1>({-1.0}, this->exec);
+    auto beta = gko::initialize<Vec2>({2.0}, this->exec);
+    auto x = gko::initialize<Vec1>({2.0, 1.0, 4.0}, this->exec);
+    auto y = gko::initialize<Vec2>({1.0, 2.0}, this->exec);
+
+    this->mtx1->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({-11.0, -1.0}), 0.0);
+}
+
+
+TYPED_TEST(Ell, MixedAppliesLinearCombinationToDenseVector3)
+{
+    // Output vector has same value type as matrix
+    using T = typename TestFixture::value_type;
+    using next_T = gko::next_precision<T>;
+    using Vec1 = typename TestFixture::Vec;
+    using Vec2 = gko::matrix::Dense<next_T>;
+    auto alpha = gko::initialize<Vec2>({-1.0}, this->exec);
+    auto beta = gko::initialize<Vec1>({2.0}, this->exec);
+    auto x = gko::initialize<Vec2>({2.0, 1.0, 4.0}, this->exec);
+    auto y = gko::initialize<Vec1>({1.0, 2.0}, this->exec);
 
     this->mtx1->apply(alpha.get(), x.get(), beta.get(), y.get());
 
@@ -208,6 +356,92 @@ TYPED_TEST(Ell, AppliesLinearCombinationToDenseMatrix)
 }
 
 
+TYPED_TEST(Ell, MixedAppliesLinearCombinationToDenseMatrix1)
+{
+    // Both vectors have the same value type which differs from the matrix
+    using T = typename TestFixture::value_type;
+    using next_T = gko::next_precision<T>;
+    using Vec = gko::matrix::Dense<next_T>;
+    auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
+    auto beta = gko::initialize<Vec>({2.0}, this->exec);
+    // clang-format off
+    auto x = gko::initialize<Vec>(
+        {I<next_T>{2.0, 3.0},
+         I<next_T>{1.0, -1.5},
+         I<next_T>{4.0, 2.5}}, this->exec);
+    auto y = gko::initialize<Vec>(
+        {I<next_T>{1.0, 0.5},
+         I<next_T>{2.0, -1.5}}, this->exec);
+    // clang-format on
+
+    this->mtx1->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    // clang-format off
+    GKO_ASSERT_MTX_NEAR(y,
+                        l({{-11.0, -2.5},
+                           { -1.0,  4.5}}), 0.0);
+    // clang-format on
+}
+
+
+TYPED_TEST(Ell, MixedAppliesLinearCombinationToDenseMatrix2)
+{
+    // Input vector has same value type as matrix
+    using T = typename TestFixture::value_type;
+    using next_T = gko::next_precision<T>;
+    using Vec1 = typename TestFixture::Vec;
+    using Vec2 = gko::matrix::Dense<next_T>;
+    auto alpha = gko::initialize<Vec1>({-1.0}, this->exec);
+    auto beta = gko::initialize<Vec2>({2.0}, this->exec);
+    // clang-format off
+    auto x = gko::initialize<Vec1>(
+        {I<T>{2.0, 3.0},
+         I<T>{1.0, -1.5},
+         I<T>{4.0, 2.5}}, this->exec);
+    auto y = gko::initialize<Vec2>(
+        {I<next_T>{1.0, 0.5},
+         I<next_T>{2.0, -1.5}}, this->exec);
+    // clang-format on
+
+    this->mtx1->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    // clang-format off
+    GKO_ASSERT_MTX_NEAR(y,
+                        l({{-11.0, -2.5},
+                           { -1.0,  4.5}}), 0.0);
+    // clang-format on
+}
+
+
+TYPED_TEST(Ell, MixedAppliesLinearCombinationToDenseMatrix3)
+{
+    // Output vector has same value type as matrix
+    using T = typename TestFixture::value_type;
+    using next_T = gko::next_precision<T>;
+    using Vec1 = typename TestFixture::Vec;
+    using Vec2 = gko::matrix::Dense<next_T>;
+    auto alpha = gko::initialize<Vec2>({-1.0}, this->exec);
+    auto beta = gko::initialize<Vec1>({2.0}, this->exec);
+    // clang-format off
+    auto x = gko::initialize<Vec2>(
+        {I<next_T>{2.0, 3.0},
+         I<next_T>{1.0, -1.5},
+         I<next_T>{4.0, 2.5}}, this->exec);
+    auto y = gko::initialize<Vec1>(
+        {I<T>{1.0, 0.5},
+         I<T>{2.0, -1.5}}, this->exec);
+    // clang-format on
+
+    this->mtx1->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    // clang-format off
+    GKO_ASSERT_MTX_NEAR(y,
+                        l({{-11.0, -2.5},
+                           { -1.0,  4.5}}), 0.0);
+    // clang-format on
+}
+
+
 TYPED_TEST(Ell, ApplyFailsOnWrongInnerDimension)
 {
     using Vec = typename TestFixture::Vec;

From 0868576872d6321e5fb95fe20bb1a77c37fdf672 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Mon, 15 Mar 2021 12:27:24 +0100
Subject: [PATCH 0679/1008] adjust tolerances for lp value types

---
 cuda/test/matrix/ell_kernels.cpp    | 24 ++++++++++++------------
 hip/test/matrix/ell_kernels.hip.cpp | 24 ++++++++++++------------
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/cuda/test/matrix/ell_kernels.cpp b/cuda/test/matrix/ell_kernels.cpp
index e24f7083728..0df427f139e 100644
--- a/cuda/test/matrix/ell_kernels.cpp
+++ b/cuda/test/matrix/ell_kernels.cpp
@@ -167,7 +167,7 @@ TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef1)
     mtx->apply(y2.get(), expected2.get());
     dmtx->apply(dy2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -189,7 +189,7 @@ TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef3)
     mtx->apply(y.get(), expected2.get());
     dmtx->apply(dy.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -211,7 +211,7 @@ TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef1)
     mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -233,7 +233,7 @@ TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef3)
     mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -255,7 +255,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef1)
     mtx->apply(y2.get(), expected2.get());
     dmtx->apply(dy2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -277,7 +277,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef3)
     mtx->apply(y.get(), expected2.get());
     dmtx->apply(dy.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -298,7 +298,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef1)
     mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -320,7 +320,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef3)
     mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -342,7 +342,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef1)
     mtx->apply(y2.get(), expected2.get());
     dmtx->apply(dy2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -364,7 +364,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef3)
     mtx->apply(y.get(), expected2.get());
     dmtx->apply(dy.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -386,7 +386,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef1)
     mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -408,7 +408,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef3)
     mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
diff --git a/hip/test/matrix/ell_kernels.hip.cpp b/hip/test/matrix/ell_kernels.hip.cpp
index 77253125abe..9d1e1440544 100644
--- a/hip/test/matrix/ell_kernels.hip.cpp
+++ b/hip/test/matrix/ell_kernels.hip.cpp
@@ -168,7 +168,7 @@ TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef1)
     mtx->apply(y2.get(), expected2.get());
     dmtx->apply(dy2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -190,7 +190,7 @@ TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef3)
     mtx->apply(y.get(), expected2.get());
     dmtx->apply(dy.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -212,7 +212,7 @@ TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef1)
     mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -234,7 +234,7 @@ TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef3)
     mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -256,7 +256,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef1)
     mtx->apply(y2.get(), expected2.get());
     dmtx->apply(dy2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -278,7 +278,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef3)
     mtx->apply(y.get(), expected2.get());
     dmtx->apply(dy.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -299,7 +299,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef1)
     mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -321,7 +321,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef3)
     mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -343,7 +343,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef1)
     mtx->apply(y2.get(), expected2.get());
     dmtx->apply(dy2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -365,7 +365,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef3)
     mtx->apply(y.get(), expected2.get());
     dmtx->apply(dy.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -387,7 +387,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef1)
     mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 
@@ -409,7 +409,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef3)
     mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
 }
 
 

From d7ce47b0e48ab88575601fdb85b28d67c19263f1 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Mon, 15 Mar 2021 13:43:42 +0100
Subject: [PATCH 0680/1008] use size_type for accessor dimensions

---
 cuda/matrix/ell_kernels.cu       | 7 +++----
 hip/matrix/ell_kernels.hip.cpp   | 7 +++----
 omp/matrix/ell_kernels.cpp       | 8 ++++----
 reference/matrix/ell_kernels.cpp | 8 ++++----
 4 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/cuda/matrix/ell_kernels.cu b/cuda/matrix/ell_kernels.cu
index 33db65c4c1c..2dc669ba5a7 100644
--- a/cuda/matrix/ell_kernels.cu
+++ b/cuda/matrix/ell_kernels.cu
@@ -148,11 +148,10 @@ void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
                          b->get_size()[1], 1);
 
     const auto a_vals = gko::acc::range<a_accessor>(
-        std::array<long unsigned int, 1>{
-            {num_stored_elements_per_row * stride}},
+        std::array<size_type, 1>{{num_stored_elements_per_row * stride}},
         a->get_const_values());
     const auto b_vals = gko::acc::range<b_accessor>(
-        std::array<long unsigned int, 2>{{nrows, b->get_stride()}},
+        std::array<size_type, 2>{{nrows, b->get_stride()}},
         b->get_const_values());
 
     if (alpha == nullptr && beta == nullptr) {
@@ -164,7 +163,7 @@ void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
                 as_cuda_type(c->get_values()), c->get_stride());
     } else if (alpha != nullptr && beta != nullptr) {
         const auto alpha_val = gko::acc::range<a_accessor>(
-            std::array<long unsigned int, 1>{1}, alpha->get_const_values());
+            std::array<size_type, 1>{1}, alpha->get_const_values());
         kernel::spmv<num_thread_per_worker, atomic>
             <<<grid_size, block_size, 0, 0>>>(
                 nrows, num_worker_per_row, as_cuda_accessor(alpha_val),
diff --git a/hip/matrix/ell_kernels.hip.cpp b/hip/matrix/ell_kernels.hip.cpp
index 8bcadd02370..24966a64113 100644
--- a/hip/matrix/ell_kernels.hip.cpp
+++ b/hip/matrix/ell_kernels.hip.cpp
@@ -152,11 +152,10 @@ void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
                          b->get_size()[1], 1);
 
     const auto a_vals = gko::acc::range<a_accessor>(
-        std::array<long unsigned int, 1>{
-            {num_stored_elements_per_row * stride}},
+        std::array<size_type, 1>{{num_stored_elements_per_row * stride}},
         a->get_const_values());
     const auto b_vals = gko::acc::range<b_accessor>(
-        std::array<long unsigned int, 2>{{nrows, b->get_stride()}},
+        std::array<size_type, 2>{{nrows, b->get_stride()}},
         b->get_const_values());
 
     if (alpha == nullptr && beta == nullptr) {
@@ -168,7 +167,7 @@ void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
             b->get_stride(), as_hip_type(c->get_values()), c->get_stride());
     } else if (alpha != nullptr && beta != nullptr) {
         const auto alpha_val = gko::acc::range<a_accessor>(
-            std::array<long unsigned int, 1>{1}, alpha->get_const_values());
+            std::array<size_type, 1>{1}, alpha->get_const_values());
         hipLaunchKernelGGL(
             HIP_KERNEL_NAME(kernel::spmv<num_thread_per_worker, atomic>),
             dim3(grid_size), dim3(block_size), 0, 0, nrows, num_worker_per_row,
diff --git a/omp/matrix/ell_kernels.cpp b/omp/matrix/ell_kernels.cpp
index 76e7eea80ed..040176053d6 100644
--- a/omp/matrix/ell_kernels.cpp
+++ b/omp/matrix/ell_kernels.cpp
@@ -74,10 +74,10 @@ void spmv(std::shared_ptr<const OmpExecutor> exec,
     const auto stride = a->get_stride();
     const auto num_rows = b->get_size()[0];
     const auto a_vals = gko::acc::range<a_accessor>(
-        std::array<long unsigned int, 1>{num_stored_elements_per_row * stride},
+        std::array<size_type, 1>{num_stored_elements_per_row * stride},
         a->get_const_values());
     const auto b_vals = gko::acc::range<b_accessor>(
-        std::array<long unsigned int, 2>{num_rows, b->get_stride()},
+        std::array<size_type, 2>{num_rows, b->get_stride()},
         b->get_const_values());
 
 #pragma omp parallel for
@@ -118,10 +118,10 @@ void advanced_spmv(std::shared_ptr<const OmpExecutor> exec,
     const auto stride = a->get_stride();
     const auto num_rows = b->get_size()[0];
     const auto a_vals = gko::acc::range<a_accessor>(
-        std::array<long unsigned int, 1>{num_stored_elements_per_row * stride},
+        std::array<size_type, 1>{num_stored_elements_per_row * stride},
         a->get_const_values());
     const auto b_vals = gko::acc::range<b_accessor>(
-        std::array<long unsigned int, 2>{num_rows, b->get_stride()},
+        std::array<size_type, 2>{num_rows, b->get_stride()},
         b->get_const_values());
     const auto alpha_val = OutputValueType(alpha->at(0, 0));
     const auto beta_val = beta->at(0, 0);
diff --git a/reference/matrix/ell_kernels.cpp b/reference/matrix/ell_kernels.cpp
index ba6d687d18a..8b0f541799a 100644
--- a/reference/matrix/ell_kernels.cpp
+++ b/reference/matrix/ell_kernels.cpp
@@ -70,10 +70,10 @@ void spmv(std::shared_ptr<const ReferenceExecutor> exec,
     const auto stride = a->get_stride();
     const auto num_rows = b->get_size()[0];
     const auto a_vals = gko::acc::range<a_accessor>(
-        std::array<long unsigned int, 1>{num_stored_elements_per_row * stride},
+        std::array<size_type, 1>{num_stored_elements_per_row * stride},
         a->get_const_values());
     const auto b_vals = gko::acc::range<b_accessor>(
-        std::array<long unsigned int, 2>{num_rows, b->get_stride()},
+        std::array<size_type, 2>{num_rows, b->get_stride()},
         b->get_const_values());
 
     for (size_type row = 0; row < a->get_size()[0]; row++) {
@@ -113,10 +113,10 @@ void advanced_spmv(std::shared_ptr<const ReferenceExecutor> exec,
     const auto stride = a->get_stride();
     const auto num_rows = b->get_size()[0];
     const auto a_vals = gko::acc::range<a_accessor>(
-        std::array<long unsigned int, 1>{num_stored_elements_per_row * stride},
+        std::array<size_type, 1>{num_stored_elements_per_row * stride},
         a->get_const_values());
     const auto b_vals = gko::acc::range<b_accessor>(
-        std::array<long unsigned int, 2>{num_rows, b->get_stride()},
+        std::array<size_type, 2>{num_rows, b->get_stride()},
         b->get_const_values());
     const auto alpha_val = OutputValueType(alpha->at(0, 0));
     const auto beta_val = beta->at(0, 0);

From aa6b34fe3e7fce5c4322ffbf4413eb3fe6932e12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Wed, 17 Mar 2021 11:29:28 +0100
Subject: [PATCH 0681/1008] Fix HIP and CUDA ELL SpMV Debug error

---
 cuda/matrix/ell_kernels.cu     | 4 ++--
 hip/matrix/ell_kernels.hip.cpp | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/cuda/matrix/ell_kernels.cu b/cuda/matrix/ell_kernels.cu
index 2dc669ba5a7..57617269ed0 100644
--- a/cuda/matrix/ell_kernels.cu
+++ b/cuda/matrix/ell_kernels.cu
@@ -151,8 +151,8 @@ void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
         std::array<size_type, 1>{{num_stored_elements_per_row * stride}},
         a->get_const_values());
     const auto b_vals = gko::acc::range<b_accessor>(
-        std::array<size_type, 2>{{nrows, b->get_stride()}},
-        b->get_const_values());
+        std::array<size_type, 2>{{b->get_size()[0], b->get_size()[1]}},
+        b->get_const_values(), std::array<size_type, 1>{{b->get_stride()}});
 
     if (alpha == nullptr && beta == nullptr) {
         kernel::spmv<num_thread_per_worker, atomic>
diff --git a/hip/matrix/ell_kernels.hip.cpp b/hip/matrix/ell_kernels.hip.cpp
index 24966a64113..f1972635385 100644
--- a/hip/matrix/ell_kernels.hip.cpp
+++ b/hip/matrix/ell_kernels.hip.cpp
@@ -155,8 +155,8 @@ void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
         std::array<size_type, 1>{{num_stored_elements_per_row * stride}},
         a->get_const_values());
     const auto b_vals = gko::acc::range<b_accessor>(
-        std::array<size_type, 2>{{nrows, b->get_stride()}},
-        b->get_const_values());
+        std::array<size_type, 2>{{b->get_size()[0], b->get_size()[1]}},
+        b->get_const_values(), std::array<size_type, 1>{{b->get_stride()}});
 
     if (alpha == nullptr && beta == nullptr) {
         hipLaunchKernelGGL(

From 6ddc1e6676e5b3dc69b7c8468a6f4427d53796b4 Mon Sep 17 00:00:00 2001
From: Fritz Goebel <goebel.fritz@googlemail.com>
Date: Thu, 18 Mar 2021 11:07:22 +0100
Subject: [PATCH 0682/1008] Code Review

---
 common/matrix/ell_kernels.hpp.inc | 18 +++++++++---------
 cuda/matrix/ell_kernels.cu        |  6 +++---
 dpcpp/matrix/ell_kernels.dp.cpp   | 27 +++++++++++++++------------
 hip/matrix/ell_kernels.hip.cpp    |  7 +++----
 omp/matrix/ell_kernels.cpp        |  8 ++++----
 reference/matrix/ell_kernels.cpp  |  8 ++++----
 6 files changed, 38 insertions(+), 36 deletions(-)

diff --git a/common/matrix/ell_kernels.hpp.inc b/common/matrix/ell_kernels.hpp.inc
index 79ed4ed979b..2323d512258 100644
--- a/common/matrix/ell_kernels.hpp.inc
+++ b/common/matrix/ell_kernels.hpp.inc
@@ -41,8 +41,8 @@ __device__ void spmv_kernel(
     const size_type num_rows, const int num_worker_per_row,
     acc::range<a_accessor> val, const IndexType *__restrict__ col,
     const size_type stride, const size_type num_stored_elements_per_row,
-    acc::range<b_accessor> b, const size_type b_stride,
-    OutputValueType *__restrict__ c, const size_type c_stride, Closure op)
+    acc::range<b_accessor> b, OutputValueType *__restrict__ c,
+    const size_type c_stride, Closure op)
 {
     const auto tidx = thread::get_thread_id_flat();
     const decltype(tidx) column_id = blockIdx.y;
@@ -109,12 +109,12 @@ __global__ __launch_bounds__(default_block_size) void spmv(
     const size_type num_rows, const int num_worker_per_row,
     acc::range<a_accessor> val, const IndexType *__restrict__ col,
     const size_type stride, const size_type num_stored_elements_per_row,
-    acc::range<b_accessor> b, const size_type b_stride,
-    OutputValueType *__restrict__ c, const size_type c_stride)
+    acc::range<b_accessor> b, OutputValueType *__restrict__ c,
+    const size_type c_stride)
 {
     spmv_kernel<num_thread_per_worker, atomic>(
         num_rows, num_worker_per_row, val, col, stride,
-        num_stored_elements_per_row, b, b_stride, c, c_stride,
+        num_stored_elements_per_row, b, c, c_stride,
         [](const OutputValueType &x, const OutputValueType &y) { return x; });
 }
 
@@ -126,8 +126,8 @@ __global__ __launch_bounds__(default_block_size) void spmv(
     acc::range<a_accessor> alpha, acc::range<a_accessor> val,
     const IndexType *__restrict__ col, const size_type stride,
     const size_type num_stored_elements_per_row, acc::range<b_accessor> b,
-    const size_type b_stride, const OutputValueType *__restrict__ beta,
-    OutputValueType *__restrict__ c, const size_type c_stride)
+    const OutputValueType *__restrict__ beta, OutputValueType *__restrict__ c,
+    const size_type c_stride)
 {
     const OutputValueType alpha_val = alpha(0);
     const OutputValueType beta_val = beta[0];
@@ -138,14 +138,14 @@ __global__ __launch_bounds__(default_block_size) void spmv(
     if (atomic) {
         spmv_kernel<num_thread_per_worker, atomic>(
             num_rows, num_worker_per_row, val, col, stride,
-            num_stored_elements_per_row, b, b_stride, c, c_stride,
+            num_stored_elements_per_row, b, c, c_stride,
             [&alpha_val](const OutputValueType &x, const OutputValueType &y) {
                 return alpha_val * x;
             });
     } else {
         spmv_kernel<num_thread_per_worker, atomic>(
             num_rows, num_worker_per_row, val, col, stride,
-            num_stored_elements_per_row, b, b_stride, c, c_stride,
+            num_stored_elements_per_row, b, c, c_stride,
             [&alpha_val, &beta_val](const OutputValueType &x,
                                     const OutputValueType &y) {
                 return alpha_val * x + beta_val * y;
diff --git a/cuda/matrix/ell_kernels.cu b/cuda/matrix/ell_kernels.cu
index 57617269ed0..8808a502ebb 100644
--- a/cuda/matrix/ell_kernels.cu
+++ b/cuda/matrix/ell_kernels.cu
@@ -159,8 +159,8 @@ void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
             <<<grid_size, block_size, 0, 0>>>(
                 nrows, num_worker_per_row, as_cuda_accessor(a_vals),
                 a->get_const_col_idxs(), stride, num_stored_elements_per_row,
-                as_cuda_accessor(b_vals), b->get_stride(),
-                as_cuda_type(c->get_values()), c->get_stride());
+                as_cuda_accessor(b_vals), as_cuda_type(c->get_values()),
+                c->get_stride());
     } else if (alpha != nullptr && beta != nullptr) {
         const auto alpha_val = gko::acc::range<a_accessor>(
             std::array<size_type, 1>{1}, alpha->get_const_values());
@@ -169,7 +169,7 @@ void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
                 nrows, num_worker_per_row, as_cuda_accessor(alpha_val),
                 as_cuda_accessor(a_vals), a->get_const_col_idxs(), stride,
                 num_stored_elements_per_row, as_cuda_accessor(b_vals),
-                b->get_stride(), as_cuda_type(beta->get_const_values()),
+                as_cuda_type(beta->get_const_values()),
                 as_cuda_type(c->get_values()), c->get_stride());
     } else {
         GKO_KERNEL_NOT_FOUND;
diff --git a/dpcpp/matrix/ell_kernels.dp.cpp b/dpcpp/matrix/ell_kernels.dp.cpp
index 01a40b5d9b5..b4dfeafeb54 100644
--- a/dpcpp/matrix/ell_kernels.dp.cpp
+++ b/dpcpp/matrix/ell_kernels.dp.cpp
@@ -56,24 +56,27 @@ namespace dpcpp {
 namespace ell {
 
 
-template <typename ValueType, typename IndexType>
+template <typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
 void spmv(std::shared_ptr<const DpcppExecutor> exec,
-          const matrix::Ell<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b,
-          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+          const matrix::Ell<MatrixValueType, IndexType> *a,
+          const matrix::Dense<InputValueType> *b,
+          matrix::Dense<OutputValueType> *c) GKO_NOT_IMPLEMENTED;
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_ELL_SPMV_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_ELL_SPMV_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
+template <typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
 void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
-                   const matrix::Dense<ValueType> *alpha,
-                   const matrix::Ell<ValueType, IndexType> *a,
-                   const matrix::Dense<ValueType> *b,
-                   const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+                   const matrix::Dense<MatrixValueType> *alpha,
+                   const matrix::Ell<MatrixValueType, IndexType> *a,
+                   const matrix::Dense<InputValueType> *b,
+                   const matrix::Dense<OutputValueType> *beta,
+                   matrix::Dense<OutputValueType> *c) GKO_NOT_IMPLEMENTED;
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL);
 
 
diff --git a/hip/matrix/ell_kernels.hip.cpp b/hip/matrix/ell_kernels.hip.cpp
index f1972635385..fc10fc7d0e1 100644
--- a/hip/matrix/ell_kernels.hip.cpp
+++ b/hip/matrix/ell_kernels.hip.cpp
@@ -164,7 +164,7 @@ void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
             dim3(grid_size), dim3(block_size), 0, 0, nrows, num_worker_per_row,
             as_hip_accessor(a_vals), a->get_const_col_idxs(), stride,
             num_stored_elements_per_row, as_hip_accessor(b_vals),
-            b->get_stride(), as_hip_type(c->get_values()), c->get_stride());
+            as_hip_type(c->get_values()), c->get_stride());
     } else if (alpha != nullptr && beta != nullptr) {
         const auto alpha_val = gko::acc::range<a_accessor>(
             std::array<size_type, 1>{1}, alpha->get_const_values());
@@ -173,9 +173,8 @@ void abstract_spmv(syn::value_list<int, info>, int num_worker_per_row,
             dim3(grid_size), dim3(block_size), 0, 0, nrows, num_worker_per_row,
             as_hip_accessor(alpha_val), as_hip_accessor(a_vals),
             a->get_const_col_idxs(), stride, num_stored_elements_per_row,
-            as_hip_accessor(b_vals), b->get_stride(),
-            as_hip_type(beta->get_const_values()), as_hip_type(c->get_values()),
-            c->get_stride());
+            as_hip_accessor(b_vals), as_hip_type(beta->get_const_values()),
+            as_hip_type(c->get_values()), c->get_stride());
     } else {
         GKO_KERNEL_NOT_FOUND;
     }
diff --git a/omp/matrix/ell_kernels.cpp b/omp/matrix/ell_kernels.cpp
index 040176053d6..db7fe3cbc0d 100644
--- a/omp/matrix/ell_kernels.cpp
+++ b/omp/matrix/ell_kernels.cpp
@@ -77,8 +77,8 @@ void spmv(std::shared_ptr<const OmpExecutor> exec,
         std::array<size_type, 1>{num_stored_elements_per_row * stride},
         a->get_const_values());
     const auto b_vals = gko::acc::range<b_accessor>(
-        std::array<size_type, 2>{num_rows, b->get_stride()},
-        b->get_const_values());
+        std::array<size_type, 2>{{b->get_size()[0], b->get_size()[1]}},
+        b->get_const_values(), std::array<size_type, 1>{{b->get_stride()}});
 
 #pragma omp parallel for
     for (size_type row = 0; row < a->get_size()[0]; row++) {
@@ -121,8 +121,8 @@ void advanced_spmv(std::shared_ptr<const OmpExecutor> exec,
         std::array<size_type, 1>{num_stored_elements_per_row * stride},
         a->get_const_values());
     const auto b_vals = gko::acc::range<b_accessor>(
-        std::array<size_type, 2>{num_rows, b->get_stride()},
-        b->get_const_values());
+        std::array<size_type, 2>{{b->get_size()[0], b->get_size()[1]}},
+        b->get_const_values(), std::array<size_type, 1>{{b->get_stride()}});
     const auto alpha_val = OutputValueType(alpha->at(0, 0));
     const auto beta_val = beta->at(0, 0);
 
diff --git a/reference/matrix/ell_kernels.cpp b/reference/matrix/ell_kernels.cpp
index 8b0f541799a..58df834bfba 100644
--- a/reference/matrix/ell_kernels.cpp
+++ b/reference/matrix/ell_kernels.cpp
@@ -73,8 +73,8 @@ void spmv(std::shared_ptr<const ReferenceExecutor> exec,
         std::array<size_type, 1>{num_stored_elements_per_row * stride},
         a->get_const_values());
     const auto b_vals = gko::acc::range<b_accessor>(
-        std::array<size_type, 2>{num_rows, b->get_stride()},
-        b->get_const_values());
+        std::array<size_type, 2>{{b->get_size()[0], b->get_size()[1]}},
+        b->get_const_values(), std::array<size_type, 1>{{b->get_stride()}});
 
     for (size_type row = 0; row < a->get_size()[0]; row++) {
         for (size_type j = 0; j < c->get_size()[1]; j++) {
@@ -116,8 +116,8 @@ void advanced_spmv(std::shared_ptr<const ReferenceExecutor> exec,
         std::array<size_type, 1>{num_stored_elements_per_row * stride},
         a->get_const_values());
     const auto b_vals = gko::acc::range<b_accessor>(
-        std::array<size_type, 2>{num_rows, b->get_stride()},
-        b->get_const_values());
+        std::array<size_type, 2>{{b->get_size()[0], b->get_size()[1]}},
+        b->get_const_values(), std::array<size_type, 1>{{b->get_stride()}});
     const auto alpha_val = OutputValueType(alpha->at(0, 0));
     const auto beta_val = beta->at(0, 0);
 

From 78d4024fcec46d1fe957b04401050bb6723d62b3 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 15 Apr 2021 13:44:43 +0200
Subject: [PATCH 0683/1008] add GINKGO_MIXED_PRECISION flag and dispatch

---
 .gitlab-ci.yml                                |  41 +++++++
 CMakeLists.txt                                |   1 +
 INSTALL.md                                    |   4 +
 cmake/get_info.cmake                          |   2 +-
 core/base/mixed_precision_types.hpp           | 108 ++++++++++++++++++
 core/device_hooks/common_kernels.inc.cpp      |   1 +
 core/matrix/ell.cpp                           |  13 ++-
 cuda/matrix/ell_kernels.cu                    |   1 +
 dpcpp/matrix/ell_kernels.dp.cpp               |   1 +
 hip/matrix/ell_kernels.hip.cpp                |   1 +
 include/ginkgo/config.hpp.in                  |   4 +
 .../ginkgo/core/base/precision_dispatch.hpp   |  94 +++++++++++++++
 include/ginkgo/core/base/types.hpp            |  51 ---------
 omp/matrix/ell_kernels.cpp                    |   1 +
 reference/matrix/ell_kernels.cpp              |   1 +
 15 files changed, 267 insertions(+), 57 deletions(-)
 create mode 100644 core/base/mixed_precision_types.hpp

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bbe85b1fbe8..11e66475301 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,6 +32,7 @@ include:
   BUILD_HWLOC: "ON"
   FAST_TESTS: "OFF"
   DPCPP_SINGLE_MODE: "OFF"
+  MIXED_PRECISION: "ON"
   RUN_EXAMPLES: "OFF"
   CONFIG_LOG: "ON"
   CXX_FLAGS: ""
@@ -77,6 +78,7 @@ include:
         -DGINKGO_BUILD_HWLOC=${BUILD_HWLOC}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_FAST_TESTS=${FAST_TESTS}
+        -DGINKGO_MIXED_PRECISION=${MIXED_PRECISION}
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
         -DGINKGO_DPCPP_SINGLE_MODE=${DPCPP_SINGLE_MODE}
@@ -111,6 +113,7 @@ include:
         -DGINKGO_BUILD_HWLOC=${BUILD_HWLOC}
         -DGINKGO_BUILD_TESTS=ON -DGINKGO_BUILD_EXAMPLES=ON
         -DGINKGO_FAST_TESTS=${FAST_TESTS}
+        -DGINKGO_MIXED_PRECISION=${MIXED_PRECISION}
         -DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
         -DGINKGO_DPCPP_SINGLE_MODE=${DPCPP_SINGLE_MODE}
         -DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
@@ -681,6 +684,44 @@ build/nocuda/intel/omp/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
 
+build/nocuda-nomixed/gcc/omp/release/shared:
+  <<: *default_build_with_test
+  extends:
+    - .quick_test_condition
+    - .use_gko-nocuda-gnu9-llvm8
+  variables:
+    <<: *default_variables
+    BUILD_OMP: "ON"
+    BUILD_TYPE: "Release"
+    MIXED_PRECISION: "OFF"
+
+build/nocuda-nomixed/clang/omp/debug/static:
+  <<: *default_build_with_test
+  extends:
+    - .full_test_condition
+    - .use_gko-nocuda-gnu9-llvm8
+  variables:
+    <<: *default_variables
+    C_COMPILER: "clang"
+    CXX_COMPILER: "clang++"
+    BUILD_OMP: "ON"
+    BUILD_TYPE: "Debug"
+    BUILD_SHARED_LIBS: "OFF"
+    MIXED_PRECISION: "OFF"
+
+build/nocuda-nomixed/intel/omp/release/static:
+  <<: *default_build_with_test
+  extends:
+    - .full_test_condition
+    - .use_gko-nocuda-gnu9-llvm8-intel
+  variables:
+    <<: *default_variables
+    C_COMPILER: "icc"
+    CXX_COMPILER: "icpc"
+    BUILD_OMP: "ON"
+    BUILD_TYPE: "Release"
+    BUILD_SHARED_LIBS: "OFF"
+    MIXED_PRECISION: "OFF"
 
 build/dpcpp/cpu/release/static:
   <<: *default_build_with_test
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9314d27664e..b021eabefa8 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -31,6 +31,7 @@ option(GINKGO_BUILD_CUDA "Compile kernels for NVIDIA GPUs" ${GINKGO_HAS_CUDA})
 option(GINKGO_BUILD_HIP "Compile kernels for AMD or NVIDIA GPUs" ${GINKGO_HAS_HIP})
 option(GINKGO_BUILD_DOC "Generate documentation" OFF)
 option(GINKGO_FAST_TESTS "Reduces the input size for a few tests known to be time-intensive" OFF)
+option(GINKGO_MIXED_PRECISION "Add true mixed-precision kernel (otherwise they will be conversion-based)" OFF)
 option(GINKGO_SKIP_DEPENDENCY_UPDATE
     "Do not update dependencies each time the project is rebuilt" ON)
 option(GINKGO_EXPORT_BUILD_DIR
diff --git a/INSTALL.md b/INSTALL.md
index d85bcc48329..4fcc60d9ae4 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -18,6 +18,10 @@ Ginkgo adds the following additional switches to control what is being built:
 *   `-DGINKGO_DEVEL_TOOLS={ON, OFF}` sets up the build system for development
     (requires clang-format, will also download git-cmake-format),
     default is `OFF`.
+*   `-DGINKGO_MIXED_PRECISION={ON, OFF}` compiles true mixed-precision kernels
+    instead of converting data on the fly, default is `OFF`.
+    Enabling this flag increases the library size, but improves performance of
+    mixed-precision kernels.
 *   `-DGINKGO_BUILD_TESTS={ON, OFF}` builds Ginkgo's tests
     (will download googletest), default is `ON`.
 *   `-DGINKGO_FAST_TESTS={ON, OFF}` reduces the input sizes for a few slow tests
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index 835bdf0c45b..508bb6e0191 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -127,7 +127,7 @@ foreach(log_type ${log_types})
     ginkgo_print_module_footer(${${log_type}} "User configuration:")
     ginkgo_print_module_footer(${${log_type}} "  Enabled modules:")
     ginkgo_print_foreach_variable(${${log_type}}
-        "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP")
+        "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP;GINKGO_MIXED_PRECISION")
     ginkgo_print_module_footer(${${log_type}} "  Tests, benchmarks and examples:")
     ginkgo_print_foreach_variable(${${log_type}}
         "GINKGO_BUILD_TESTS;GINKGO_FAST_TESTS;GINKGO_BUILD_EXAMPLES;GINKGO_EXTLIB_EXAMPLE;GINKGO_BUILD_BENCHMARKS;GINKGO_BENCHMARK_ENABLE_TUNING")
diff --git a/core/base/mixed_precision_types.hpp b/core/base/mixed_precision_types.hpp
new file mode 100644
index 00000000000..079fcb8a075
--- /dev/null
+++ b/core/base/mixed_precision_types.hpp
@@ -0,0 +1,108 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_BASE_MIXED_PRECISION_TYPES_HPP_
+#define GKO_CORE_BASE_MIXED_PRECISION_TYPES_HPP_
+
+
+#include <ginkgo/config.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+#ifdef GINKGO_MIXED_PRECISION
+#define GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(_macro) \
+    template _macro(float, float, float, int32);                    \
+    template _macro(float, float, double, int32);                   \
+    template _macro(float, double, float, int32);                   \
+    template _macro(float, double, double, int32);                  \
+    template _macro(double, float, float, int32);                   \
+    template _macro(double, float, double, int32);                  \
+    template _macro(double, double, float, int32);                  \
+    template _macro(double, double, double, int32);                 \
+    template _macro(std::complex<float>, std::complex<float>,       \
+                    std::complex<float>, int32);                    \
+    template _macro(std::complex<float>, std::complex<float>,       \
+                    std::complex<double>, int32);                   \
+    template _macro(std::complex<float>, std::complex<double>,      \
+                    std::complex<float>, int32);                    \
+    template _macro(std::complex<float>, std::complex<double>,      \
+                    std::complex<double>, int32);                   \
+    template _macro(std::complex<double>, std::complex<float>,      \
+                    std::complex<float>, int32);                    \
+    template _macro(std::complex<double>, std::complex<float>,      \
+                    std::complex<double>, int32);                   \
+    template _macro(std::complex<double>, std::complex<double>,     \
+                    std::complex<float>, int32);                    \
+    template _macro(std::complex<double>, std::complex<double>,     \
+                    std::complex<double>, int32);                   \
+    template _macro(float, float, float, int64);                    \
+    template _macro(float, float, double, int64);                   \
+    template _macro(float, double, float, int64);                   \
+    template _macro(float, double, double, int64);                  \
+    template _macro(double, float, float, int64);                   \
+    template _macro(double, float, double, int64);                  \
+    template _macro(double, double, float, int64);                  \
+    template _macro(double, double, double, int64);                 \
+    template _macro(std::complex<float>, std::complex<float>,       \
+                    std::complex<float>, int64);                    \
+    template _macro(std::complex<float>, std::complex<float>,       \
+                    std::complex<double>, int64);                   \
+    template _macro(std::complex<float>, std::complex<double>,      \
+                    std::complex<float>, int64);                    \
+    template _macro(std::complex<float>, std::complex<double>,      \
+                    std::complex<double>, int64);                   \
+    template _macro(std::complex<double>, std::complex<float>,      \
+                    std::complex<float>, int64);                    \
+    template _macro(std::complex<double>, std::complex<float>,      \
+                    std::complex<double>, int64);                   \
+    template _macro(std::complex<double>, std::complex<double>,     \
+                    std::complex<float>, int64);                    \
+    template _macro(std::complex<double>, std::complex<double>,     \
+                    std::complex<double>, int64)
+#else
+#define GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(_macro) \
+    template _macro(float, float, float, int32);                    \
+    template _macro(double, double, double, int32);                 \
+    template _macro(std::complex<float>, std::complex<float>,       \
+                    std::complex<float>, int32);                    \
+    template _macro(std::complex<double>, std::complex<double>,     \
+                    std::complex<double>, int32);                   \
+    template _macro(float, float, float, int64);                    \
+    template _macro(double, double, double, int64);                 \
+    template _macro(std::complex<float>, std::complex<float>,       \
+                    std::complex<float>, int64);                    \
+    template _macro(std::complex<double>, std::complex<double>,     \
+                    std::complex<double>, int64)
+#endif
+
+
+#endif  // GKO_CORE_BASE_MIXED_PRECISION_TYPES_HPP_
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 42121074df4..810a3fab4e4 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 
 
+#include "core/base/mixed_precision_types.hpp"
 #include "core/components/absolute_array.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/components/precision_conversion.hpp"
diff --git a/core/matrix/ell.cpp b/core/matrix/ell.cpp
index 0dc9a950f75..fcb198106cd 100644
--- a/core/matrix/ell.cpp
+++ b/core/matrix/ell.cpp
@@ -102,7 +102,7 @@ size_type calculate_max_nnz_per_row(
 template <typename ValueType, typename IndexType>
 void Ell<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
 {
-    precision_dispatch_real_complex<ValueType>(
+    mixed_precision_dispatch_real_complex<ValueType>(
         [this](auto dense_b, auto dense_x) {
             this->get_executor()->run(ell::make_spmv(this, dense_b, dense_x));
         },
@@ -114,12 +114,15 @@ template <typename ValueType, typename IndexType>
 void Ell<ValueType, IndexType>::apply_impl(const LinOp *alpha, const LinOp *b,
                                            const LinOp *beta, LinOp *x) const
 {
-    precision_dispatch_real_complex<ValueType>(
-        [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
+    mixed_precision_dispatch_real_complex<ValueType>(
+        [this, alpha, beta](auto dense_b, auto dense_x) {
+            auto dense_alpha = make_temporary_conversion<ValueType>(alpha);
+            auto dense_beta = make_temporary_conversion<
+                typename std::decay_t<decltype(*dense_x)>::value_type>(beta);
             this->get_executor()->run(ell::make_advanced_spmv(
-                dense_alpha, this, dense_b, dense_beta, dense_x));
+                dense_alpha.get(), this, dense_b, dense_beta.get(), dense_x));
         },
-        alpha, b, beta, x);
+        b, x);
 }
 
 
diff --git a/cuda/matrix/ell_kernels.cu b/cuda/matrix/ell_kernels.cu
index 8808a502ebb..953ce4be8d7 100644
--- a/cuda/matrix/ell_kernels.cu
+++ b/cuda/matrix/ell_kernels.cu
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "accessor/reduced_row_major.hpp"
+#include "core/base/mixed_precision_types.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/dense_kernels.hpp"
diff --git a/dpcpp/matrix/ell_kernels.dp.cpp b/dpcpp/matrix/ell_kernels.dp.cpp
index b4dfeafeb54..3c1d93eb323 100644
--- a/dpcpp/matrix/ell_kernels.dp.cpp
+++ b/dpcpp/matrix/ell_kernels.dp.cpp
@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "core/base/mixed_precision_types.hpp"
 #include "dpcpp/components/format_conversion.dp.hpp"
 
 
diff --git a/hip/matrix/ell_kernels.hip.cpp b/hip/matrix/ell_kernels.hip.cpp
index fc10fc7d0e1..b1ad7bf4662 100644
--- a/hip/matrix/ell_kernels.hip.cpp
+++ b/hip/matrix/ell_kernels.hip.cpp
@@ -47,6 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "accessor/reduced_row_major.hpp"
+#include "core/base/mixed_precision_types.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/dense_kernels.hpp"
diff --git a/include/ginkgo/config.hpp.in b/include/ginkgo/config.hpp.in
index edef91e6d9e..1c6a31ea481 100644
--- a/include/ginkgo/config.hpp.in
+++ b/include/ginkgo/config.hpp.in
@@ -63,6 +63,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #cmakedefine GINKGO_BENCHMARK_ENABLE_TUNING
 
 
+/* Should we compile mixed-precision kernels for Ginkgo? */
+#cmakedefine GINKGO_MIXED_PRECISION
+
+
 /* What is HIP compiled for, hcc or nvcc? */
 // clang-format off
 #define GINKGO_HIP_PLATFORM_HCC @GINKGO_HIP_PLATFORM_HCC@
diff --git a/include/ginkgo/core/base/precision_dispatch.hpp b/include/ginkgo/core/base/precision_dispatch.hpp
index 5588d566b41..17901e207c1 100644
--- a/include/ginkgo/core/base/precision_dispatch.hpp
+++ b/include/ginkgo/core/base/precision_dispatch.hpp
@@ -226,6 +226,100 @@ void precision_dispatch_real_complex(Function fn, const LinOp *alpha,
 }
 
 
+/**
+ * Calls the given function with each given argument LinOp
+ * converted into matrix::Dense<ValueType> as parameters.
+ *
+ * If GINKGO_MIXED_PRECISION is defined, this means that the function will be
+ * called with its dynamic type with its static type, so the (templated/generic)
+ * function will be instantiated with all pairs of Dense<ValueType> and
+ * Dense<next_precision<ValueType>> parameter types, and the appropriate
+ * overload will be called based on the dynamic type of the parameter.
+ *
+ * If GINKGO_MIXED_PRECISION is not defined, it will behave exactly like
+ * precision_dispatch.
+ *
+ *
+ * @param fn  the given function. It will be passed one (potentially const)
+ *            matrix::Dense<...>* parameter per parameter in the parameter
+ *            pack `linops`.
+ * @param linops  the given arguments to be converted and passed on to fn.
+ *
+ * @tparam ValueType  the value type to use for the parameters of `fn`.
+ * @tparam Function  the function pointer, lambda or other functor type to call
+ *                   with the converted arguments.
+ * @tparam Args  the argument type list.
+ */
+template <typename ValueType, typename Function>
+void mixed_precision_dispatch(Function fn, const LinOp *in, LinOp *out)
+{
+#ifdef GINKGO_MIXED_PRECISION
+    using fst_type = matrix::Dense<ValueType>;
+    using snd_type = matrix::Dense<next_precision<ValueType>>;
+    if (auto dense_in = dynamic_cast<const fst_type *>(in)) {
+        if (auto dense_out = dynamic_cast<fst_type *>(out)) {
+            fn(dense_in, dense_out);
+        } else if (auto dense_out = dynamic_cast<snd_type *>(out)) {
+            fn(dense_in, dense_out);
+        } else {
+            GKO_NOT_SUPPORTED(out);
+        }
+    } else if (auto dense_in = dynamic_cast<const snd_type *>(in)) {
+        if (auto dense_out = dynamic_cast<fst_type *>(out)) {
+            fn(dense_in, dense_out);
+        } else if (auto dense_out = dynamic_cast<snd_type *>(out)) {
+            fn(dense_in, dense_out);
+        } else {
+            GKO_NOT_SUPPORTED(out);
+        }
+    } else {
+        GKO_NOT_SUPPORTED(in);
+    }
+#else
+    precision_dispatch<ValueType>(fn, in, out);
+#endif
+}
+
+
+/**
+ * Calls the given function with the given LinOps cast to their dynamic type
+ * matrix::Dense<ValueType>* as parameters.
+ * If ValueType is real and both `in` and `out` are complex, uses
+ * matrix::Dense::get_real_view() to convert them into real matrices after
+ * precision conversion.
+ *
+ * @see mixed_precision_dispatch()
+ */
+template <typename ValueType, typename Function>
+void mixed_precision_dispatch_real_complex(Function fn, const LinOp *in,
+                                           LinOp *out)
+{
+#ifdef GINKGO_MIXED_PRECISION
+    // do we need to convert complex Dense to real Dense?
+    auto complex_to_real =
+        !(is_complex<ValueType>() ||
+          dynamic_cast<const ConvertibleTo<matrix::Dense<>> *>(in));
+    if (complex_to_real) {
+        mixed_precision_dispatch<to_complex<ValueType>>(
+            [&fn](auto dense_in, auto dense_out) {
+                using Dense = matrix::Dense<ValueType>;
+                // These dynamic_casts are only needed to make the code compile
+                // If ValueType is complex, this branch will never be taken
+                // If ValueType is real, the cast is a no-op
+                fn(dynamic_cast<const Dense *>(
+                       dense_in->create_real_view().get()),
+                   dynamic_cast<Dense *>(dense_out->create_real_view().get()));
+            },
+            in, out);
+    } else {
+        mixed_precision_dispatch<ValueType>(fn, in, out);
+    }
+#else
+    precision_dispatch_real_complex<ValueType>(fn, in, out);
+#endif
+}
+
+
 }  // namespace gko
 
 
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 2f6da1e9e42..2c77aaef826 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -524,57 +524,6 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
 #endif
 
 
-#define GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(_macro) \
-    template _macro(float, float, float, int32);                    \
-    template _macro(float, float, double, int32);                   \
-    template _macro(float, double, float, int32);                   \
-    template _macro(float, double, double, int32);                  \
-    template _macro(double, float, float, int32);                   \
-    template _macro(double, float, double, int32);                  \
-    template _macro(double, double, float, int32);                  \
-    template _macro(double, double, double, int32);                 \
-    template _macro(std::complex<float>, std::complex<float>,       \
-                    std::complex<float>, int32);                    \
-    template _macro(std::complex<float>, std::complex<float>,       \
-                    std::complex<double>, int32);                   \
-    template _macro(std::complex<float>, std::complex<double>,      \
-                    std::complex<float>, int32);                    \
-    template _macro(std::complex<float>, std::complex<double>,      \
-                    std::complex<double>, int32);                   \
-    template _macro(std::complex<double>, std::complex<float>,      \
-                    std::complex<float>, int32);                    \
-    template _macro(std::complex<double>, std::complex<float>,      \
-                    std::complex<double>, int32);                   \
-    template _macro(std::complex<double>, std::complex<double>,     \
-                    std::complex<float>, int32);                    \
-    template _macro(std::complex<double>, std::complex<double>,     \
-                    std::complex<double>, int32);                   \
-    template _macro(float, float, float, int64);                    \
-    template _macro(float, float, double, int64);                   \
-    template _macro(float, double, float, int64);                   \
-    template _macro(float, double, double, int64);                  \
-    template _macro(double, float, float, int64);                   \
-    template _macro(double, float, double, int64);                  \
-    template _macro(double, double, float, int64);                  \
-    template _macro(double, double, double, int64);                 \
-    template _macro(std::complex<float>, std::complex<float>,       \
-                    std::complex<float>, int64);                    \
-    template _macro(std::complex<float>, std::complex<float>,       \
-                    std::complex<double>, int64);                   \
-    template _macro(std::complex<float>, std::complex<double>,      \
-                    std::complex<float>, int64);                    \
-    template _macro(std::complex<float>, std::complex<double>,      \
-                    std::complex<double>, int64);                   \
-    template _macro(std::complex<double>, std::complex<float>,      \
-                    std::complex<float>, int64);                    \
-    template _macro(std::complex<double>, std::complex<float>,      \
-                    std::complex<double>, int64);                   \
-    template _macro(std::complex<double>, std::complex<double>,     \
-                    std::complex<float>, int64);                    \
-    template _macro(std::complex<double>, std::complex<double>,     \
-                    std::complex<double>, int64)
-
-
 /**
  * Instantiates a template for each value type conversion pair compiled by
  * Ginkgo.
diff --git a/omp/matrix/ell_kernels.cpp b/omp/matrix/ell_kernels.cpp
index db7fe3cbc0d..58ef6c3a72e 100644
--- a/omp/matrix/ell_kernels.cpp
+++ b/omp/matrix/ell_kernels.cpp
@@ -43,6 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "accessor/reduced_row_major.hpp"
+#include "core/base/mixed_precision_types.hpp"
 #include "omp/components/format_conversion.hpp"
 
 
diff --git a/reference/matrix/ell_kernels.cpp b/reference/matrix/ell_kernels.cpp
index 58df834bfba..25de03cf709 100644
--- a/reference/matrix/ell_kernels.cpp
+++ b/reference/matrix/ell_kernels.cpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "accessor/reduced_row_major.hpp"
+#include "core/base/mixed_precision_types.hpp"
 
 
 namespace gko {

From c76b884a001ca336896b678b69735ef77563600f Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 15 Apr 2021 16:42:53 +0200
Subject: [PATCH 0684/1008] fix mixed_precision_dispatch no-op casts

---
 include/ginkgo/core/base/precision_dispatch.hpp | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/include/ginkgo/core/base/precision_dispatch.hpp b/include/ginkgo/core/base/precision_dispatch.hpp
index 17901e207c1..4c2624863e5 100644
--- a/include/ginkgo/core/base/precision_dispatch.hpp
+++ b/include/ginkgo/core/base/precision_dispatch.hpp
@@ -302,13 +302,24 @@ void mixed_precision_dispatch_real_complex(Function fn, const LinOp *in,
     if (complex_to_real) {
         mixed_precision_dispatch<to_complex<ValueType>>(
             [&fn](auto dense_in, auto dense_out) {
-                using Dense = matrix::Dense<ValueType>;
+                using DenseIn = typename std::decay_t<decltype(*dense_in)>;
+                using DenseOut = typename std::decay_t<decltype(*dense_out)>;
+                using DenseDummy = matrix::Dense<ValueType>;
+                using RealIn = remove_complex<typename DenseIn::value_type>;
+                using RealOut = remove_complex<typename DenseOut::value_type>;
+                using CastIn =
+                    std::conditional_t<is_complex<ValueType>(), DenseDummy,
+                                       matrix::Dense<RealIn>>;
+                using CastOut =
+                    std::conditional_t<is_complex<ValueType>(), DenseDummy,
+                                       matrix::Dense<RealOut>>;
                 // These dynamic_casts are only needed to make the code compile
                 // If ValueType is complex, this branch will never be taken
                 // If ValueType is real, the cast is a no-op
-                fn(dynamic_cast<const Dense *>(
+                fn(dynamic_cast<const CastIn *>(
                        dense_in->create_real_view().get()),
-                   dynamic_cast<Dense *>(dense_out->create_real_view().get()));
+                   dynamic_cast<CastOut *>(
+                       dense_out->create_real_view().get()));
             },
             in, out);
     } else {

From a4d78df01c7a099ee3560513af43b0af883413b4 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 17 Apr 2021 19:08:10 +0200
Subject: [PATCH 0685/1008] review updates
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* formatting and documentation
* improve CMake configuration output
* remove unused variables
* extract test dimensions from ell_kernels tests
* use SFINAE instead of casting in precision_dispatch.hpp

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
Co-authored-by: Pratik Nayak <pratik.nayak@kit.edu>
---
 CMakeLists.txt                                |  2 +-
 benchmark/utils/formats.hpp                   |  5 +-
 cmake/get_info.cmake                          |  5 +-
 cuda/test/matrix/ell_kernels.cpp              | 47 +++++++-------
 hip/test/matrix/ell_kernels.hip.cpp           | 47 +++++++-------
 .../ginkgo/core/base/precision_dispatch.hpp   | 63 +++++++++----------
 omp/matrix/ell_kernels.cpp                    |  2 -
 reference/matrix/ell_kernels.cpp              |  2 -
 8 files changed, 90 insertions(+), 83 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b021eabefa8..18da015f8c6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -31,7 +31,7 @@ option(GINKGO_BUILD_CUDA "Compile kernels for NVIDIA GPUs" ${GINKGO_HAS_CUDA})
 option(GINKGO_BUILD_HIP "Compile kernels for AMD or NVIDIA GPUs" ${GINKGO_HAS_HIP})
 option(GINKGO_BUILD_DOC "Generate documentation" OFF)
 option(GINKGO_FAST_TESTS "Reduces the input size for a few tests known to be time-intensive" OFF)
-option(GINKGO_MIXED_PRECISION "Add true mixed-precision kernel (otherwise they will be conversion-based)" OFF)
+option(GINKGO_MIXED_PRECISION "Instantiate true mixed-precision kernels (otherwise they will be conversion-based using implicit temporary storage)" OFF)
 option(GINKGO_SKIP_DEPENDENCY_UPDATE
     "Do not update dependencies each time the project is rebuilt" ON)
 option(GINKGO_EXPORT_BUILD_DIR
diff --git a/benchmark/utils/formats.hpp b/benchmark/utils/formats.hpp
index ed20f91119e..8c1aabbcafb 100644
--- a/benchmark/utils/formats.hpp
+++ b/benchmark/utils/formats.hpp
@@ -91,9 +91,8 @@ std::string format_description =
     "csrm: Ginkgo's CSR implementation with merge_path strategy.\n"
     "ell: Ellpack format according to Bell and Garland: Efficient Sparse "
     "Matrix-Vector Multiplication on CUDA.\n"
-    "fell: float Ellpack format according to Bell and Garland: Efficient "
-    "Sparse "
-    "Matrix-Vector Multiplication on CUDA.\n"
+    "fell: Mixed Precision Ellpack format according to Bell and Garland: "
+    "Efficient Sparse Matrix-Vector Multiplication on CUDA.\n"
     "sellp: Sliced Ellpack uses a default block size of 32.\n"
     "hybrid: Hybrid uses ell and coo to represent the matrix.\n"
     "hybrid0, hybrid25, hybrid33, hybrid40, hybrid60, hybrid80: Hybrid uses "
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index 508bb6e0191..bae1e864a02 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -127,7 +127,10 @@ foreach(log_type ${log_types})
     ginkgo_print_module_footer(${${log_type}} "User configuration:")
     ginkgo_print_module_footer(${${log_type}} "  Enabled modules:")
     ginkgo_print_foreach_variable(${${log_type}}
-        "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP;GINKGO_MIXED_PRECISION")
+        "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP")
+    ginkgo_print_module_footer(${${log_type}} "  Enabled features:")
+    ginkgo_print_foreach_variable(${${log_type}}
+        "GINKGO_MIXED_PRECISION")
     ginkgo_print_module_footer(${${log_type}} "  Tests, benchmarks and examples:")
     ginkgo_print_foreach_variable(${${log_type}}
         "GINKGO_BUILD_TESTS;GINKGO_FAST_TESTS;GINKGO_BUILD_EXAMPLES;GINKGO_EXTLIB_EXAMPLE;GINKGO_BUILD_BENCHMARKS;GINKGO_BENCHMARK_ENABLE_TUNING")
diff --git a/cuda/test/matrix/ell_kernels.cpp b/cuda/test/matrix/ell_kernels.cpp
index 0df427f139e..2df1c397f4c 100644
--- a/cuda/test/matrix/ell_kernels.cpp
+++ b/cuda/test/matrix/ell_kernels.cpp
@@ -61,7 +61,9 @@ class Ell : public ::testing::Test {
     using Vec2 = gko::matrix::Dense<float>;
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
-    Ell() : rand_engine(42) {}
+    Ell()
+        : rand_engine(42), size{532, 231}, num_els_rowwise{300}, ell_stride{600}
+    {}
 
     void SetUp()
     {
@@ -126,6 +128,9 @@ class Ell : public ::testing::Test {
     std::shared_ptr<const gko::CudaExecutor> cuda;
 
     std::ranlux48 rand_engine;
+    gko::dim<2> size;
+    gko::size_type num_els_rowwise;
+    gko::size_type ell_stride;
 
     std::unique_ptr<Mtx> mtx;
     std::unique_ptr<Vec> expected;
@@ -239,7 +244,7 @@ TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef3)
 
 TEST_F(Ell, SimpleApplyWithStrideIsEquivalentToRef)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -250,7 +255,7 @@ TEST_F(Ell, SimpleApplyWithStrideIsEquivalentToRef)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef1)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(y2.get(), expected2.get());
     dmtx->apply(dy2.get(), dresult2.get());
@@ -261,7 +266,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef1)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef2)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(y2.get(), expected.get());
     dmtx->apply(dy2.get(), dresult.get());
@@ -272,7 +277,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef2)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef3)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(y.get(), expected2.get());
     dmtx->apply(dy.get(), dresult2.get());
@@ -283,7 +288,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef3)
 
 TEST_F(Ell, AdvancedApplyWithStrideIsEquivalentToRef)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
@@ -293,7 +298,7 @@ TEST_F(Ell, AdvancedApplyWithStrideIsEquivalentToRef)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef1)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
@@ -304,7 +309,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef1)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef2)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
     dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
@@ -315,7 +320,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef2)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef3)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
@@ -326,7 +331,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef3)
 
 TEST_F(Ell, SimpleApplyWithStrideToDenseMatrixIsEquivalentToRef)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -337,7 +342,7 @@ TEST_F(Ell, SimpleApplyWithStrideToDenseMatrixIsEquivalentToRef)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef1)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(y2.get(), expected2.get());
     dmtx->apply(dy2.get(), dresult2.get());
@@ -348,7 +353,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef1)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef2)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(y2.get(), expected.get());
     dmtx->apply(dy2.get(), dresult.get());
@@ -359,7 +364,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef2)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef3)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(y.get(), expected2.get());
     dmtx->apply(dy.get(), dresult2.get());
@@ -370,7 +375,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef3)
 
 TEST_F(Ell, AdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
@@ -381,7 +386,7 @@ TEST_F(Ell, AdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef1)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
@@ -392,7 +397,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef1)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef2)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
     dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
@@ -403,7 +408,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef2)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef3)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
@@ -503,10 +508,10 @@ TEST_F(Ell, AdvancedApplyOnSmallMatrixIsEquivalentToRef)
 TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
-    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto complex_b = gen_mtx<ComplexVec>(size[1], 3);
     auto dcomplex_b = ComplexVec::create(cuda);
     dcomplex_b->copy_from(complex_b.get());
-    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto complex_x = gen_mtx<ComplexVec>(size[0], 3);
     auto dcomplex_x = ComplexVec::create(cuda);
     dcomplex_x->copy_from(complex_x.get());
 
@@ -520,10 +525,10 @@ TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
 TEST_F(Ell, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
-    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto complex_b = gen_mtx<ComplexVec>(size[1], 3);
     auto dcomplex_b = ComplexVec::create(cuda);
     dcomplex_b->copy_from(complex_b.get());
-    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto complex_x = gen_mtx<ComplexVec>(size[0], 3);
     auto dcomplex_x = ComplexVec::create(cuda);
     dcomplex_x->copy_from(complex_x.get());
 
diff --git a/hip/test/matrix/ell_kernels.hip.cpp b/hip/test/matrix/ell_kernels.hip.cpp
index 9d1e1440544..13c81fdafcf 100644
--- a/hip/test/matrix/ell_kernels.hip.cpp
+++ b/hip/test/matrix/ell_kernels.hip.cpp
@@ -61,7 +61,9 @@ class Ell : public ::testing::Test {
     using Vec2 = gko::matrix::Dense<float>;
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
 
-    Ell() : rand_engine(42) {}
+    Ell()
+        : rand_engine(42), size{532, 231}, num_els_rowwise{300}, ell_stride{600}
+    {}
 
     void SetUp()
     {
@@ -127,6 +129,9 @@ class Ell : public ::testing::Test {
     std::shared_ptr<const gko::HipExecutor> hip;
 
     std::ranlux48 rand_engine;
+    gko::dim<2> size;
+    gko::size_type num_els_rowwise;
+    gko::size_type ell_stride;
 
     std::unique_ptr<Mtx> mtx;
     std::unique_ptr<Vec> expected;
@@ -240,7 +245,7 @@ TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef3)
 
 TEST_F(Ell, SimpleApplyWithStrideIsEquivalentToRef)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -251,7 +256,7 @@ TEST_F(Ell, SimpleApplyWithStrideIsEquivalentToRef)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef1)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(y2.get(), expected2.get());
     dmtx->apply(dy2.get(), dresult2.get());
@@ -262,7 +267,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef1)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef2)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(y2.get(), expected.get());
     dmtx->apply(dy2.get(), dresult.get());
@@ -273,7 +278,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef2)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef3)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(y.get(), expected2.get());
     dmtx->apply(dy.get(), dresult2.get());
@@ -284,7 +289,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef3)
 
 TEST_F(Ell, AdvancedApplyWithStrideIsEquivalentToRef)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
@@ -294,7 +299,7 @@ TEST_F(Ell, AdvancedApplyWithStrideIsEquivalentToRef)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef1)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
@@ -305,7 +310,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef1)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef2)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
     dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
@@ -316,7 +321,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef2)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef3)
 {
-    set_up_apply_data(532, 231, 1, 300, 600);
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
@@ -327,7 +332,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef3)
 
 TEST_F(Ell, SimpleApplyWithStrideToDenseMatrixIsEquivalentToRef)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -338,7 +343,7 @@ TEST_F(Ell, SimpleApplyWithStrideToDenseMatrixIsEquivalentToRef)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef1)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(y2.get(), expected2.get());
     dmtx->apply(dy2.get(), dresult2.get());
@@ -349,7 +354,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef1)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef2)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(y2.get(), expected.get());
     dmtx->apply(dy2.get(), dresult.get());
@@ -360,7 +365,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef2)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef3)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(y.get(), expected2.get());
     dmtx->apply(dy.get(), dresult2.get());
@@ -371,7 +376,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef3)
 
 TEST_F(Ell, AdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
@@ -382,7 +387,7 @@ TEST_F(Ell, AdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef1)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
@@ -393,7 +398,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef1)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef2)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
     dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
@@ -404,7 +409,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef2)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef3)
 {
-    set_up_apply_data(532, 231, 3, 300, 600);
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
@@ -504,10 +509,10 @@ TEST_F(Ell, AdvancedApplyOnSmallMatrixIsEquivalentToRef)
 TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
-    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto complex_b = gen_mtx<ComplexVec>(size[1], 3);
     auto dcomplex_b = ComplexVec::create(hip);
     dcomplex_b->copy_from(complex_b.get());
-    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto complex_x = gen_mtx<ComplexVec>(size[0], 3);
     auto dcomplex_x = ComplexVec::create(hip);
     dcomplex_x->copy_from(complex_x.get());
 
@@ -521,10 +526,10 @@ TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
 TEST_F(Ell, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_data();
-    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto complex_b = gen_mtx<ComplexVec>(size[1], 3);
     auto dcomplex_b = ComplexVec::create(hip);
     dcomplex_b->copy_from(complex_b.get());
-    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto complex_x = gen_mtx<ComplexVec>(size[0], 3);
     auto dcomplex_x = ComplexVec::create(hip);
     dcomplex_x->copy_from(complex_x.get());
 
diff --git a/include/ginkgo/core/base/precision_dispatch.hpp b/include/ginkgo/core/base/precision_dispatch.hpp
index 4c2624863e5..376f126f5c5 100644
--- a/include/ginkgo/core/base/precision_dispatch.hpp
+++ b/include/ginkgo/core/base/precision_dispatch.hpp
@@ -231,7 +231,7 @@ void precision_dispatch_real_complex(Function fn, const LinOp *alpha,
  * converted into matrix::Dense<ValueType> as parameters.
  *
  * If GINKGO_MIXED_PRECISION is defined, this means that the function will be
- * called with its dynamic type with its static type, so the (templated/generic)
+ * called with its dynamic type as a static type, so the (templated/generic)
  * function will be instantiated with all pairs of Dense<ValueType> and
  * Dense<next_precision<ValueType>> parameter types, and the appropriate
  * overload will be called based on the dynamic type of the parameter.
@@ -239,16 +239,21 @@ void precision_dispatch_real_complex(Function fn, const LinOp *alpha,
  * If GINKGO_MIXED_PRECISION is not defined, it will behave exactly like
  * precision_dispatch.
  *
+ * @param fn  the given function. It will be called with one const and one
+ *            non-const matrix::Dense<...> parameter based on the dynamic type
+ *            of the inputs (GINKGO_MIXED_PRECISION) or of type
+ *            matrix::Dense<ValueType> (no GINKGO_MIXED_PRECISION).
+ * @param in  The first parameter to be cast (GINKGO_MIXED_PRECISION) or
+ *            converted (no GINKGO_MIXED_PRECISION) and used to call `fn`.
+ * @param out  The second parameter to be cast (GINKGO_MIXED_PRECISION) or
+ *             converted (no GINKGO_MIXED_PRECISION) and used to call `fn`.
  *
- * @param fn  the given function. It will be passed one (potentially const)
- *            matrix::Dense<...>* parameter per parameter in the parameter
- *            pack `linops`.
- * @param linops  the given arguments to be converted and passed on to fn.
- *
- * @tparam ValueType  the value type to use for the parameters of `fn`.
+ * @tparam ValueType  the value type to use for the parameters of `fn` (no
+ *                    GINKGO_MIXED_PRECISION). With GINKGO_MIXED_PRECISION
+ *                    enabled, it only matters whether this type is complex or
+ *                    real.
  * @tparam Function  the function pointer, lambda or other functor type to call
  *                   with the converted arguments.
- * @tparam Args  the argument type list.
  */
 template <typename ValueType, typename Function>
 void mixed_precision_dispatch(Function fn, const LinOp *in, LinOp *out)
@@ -290,36 +295,30 @@ void mixed_precision_dispatch(Function fn, const LinOp *in, LinOp *out)
  *
  * @see mixed_precision_dispatch()
  */
-template <typename ValueType, typename Function>
+template <typename ValueType, typename Function,
+          std::enable_if_t<is_complex<ValueType>()> * = nullptr>
 void mixed_precision_dispatch_real_complex(Function fn, const LinOp *in,
                                            LinOp *out)
 {
 #ifdef GINKGO_MIXED_PRECISION
-    // do we need to convert complex Dense to real Dense?
-    auto complex_to_real =
-        !(is_complex<ValueType>() ||
-          dynamic_cast<const ConvertibleTo<matrix::Dense<>> *>(in));
-    if (complex_to_real) {
+    mixed_precision_dispatch<ValueType>(fn, in, out);
+#else
+    precision_dispatch<ValueType>(fn, in, out);
+#endif
+}
+
+
+template <typename ValueType, typename Function,
+          std::enable_if_t<!is_complex<ValueType>()> * = nullptr>
+void mixed_precision_dispatch_real_complex(Function fn, const LinOp *in,
+                                           LinOp *out)
+{
+#ifdef GINKGO_MIXED_PRECISION
+    if (!dynamic_cast<const ConvertibleTo<matrix::Dense<>> *>(in)) {
         mixed_precision_dispatch<to_complex<ValueType>>(
             [&fn](auto dense_in, auto dense_out) {
-                using DenseIn = typename std::decay_t<decltype(*dense_in)>;
-                using DenseOut = typename std::decay_t<decltype(*dense_out)>;
-                using DenseDummy = matrix::Dense<ValueType>;
-                using RealIn = remove_complex<typename DenseIn::value_type>;
-                using RealOut = remove_complex<typename DenseOut::value_type>;
-                using CastIn =
-                    std::conditional_t<is_complex<ValueType>(), DenseDummy,
-                                       matrix::Dense<RealIn>>;
-                using CastOut =
-                    std::conditional_t<is_complex<ValueType>(), DenseDummy,
-                                       matrix::Dense<RealOut>>;
-                // These dynamic_casts are only needed to make the code compile
-                // If ValueType is complex, this branch will never be taken
-                // If ValueType is real, the cast is a no-op
-                fn(dynamic_cast<const CastIn *>(
-                       dense_in->create_real_view().get()),
-                   dynamic_cast<CastOut *>(
-                       dense_out->create_real_view().get()));
+                fn(dense_in->create_real_view().get(),
+                   dense_out->create_real_view().get());
             },
             in, out);
     } else {
diff --git a/omp/matrix/ell_kernels.cpp b/omp/matrix/ell_kernels.cpp
index 58ef6c3a72e..96f4d5347d2 100644
--- a/omp/matrix/ell_kernels.cpp
+++ b/omp/matrix/ell_kernels.cpp
@@ -73,7 +73,6 @@ void spmv(std::shared_ptr<const OmpExecutor> exec,
     const auto num_stored_elements_per_row =
         a->get_num_stored_elements_per_row();
     const auto stride = a->get_stride();
-    const auto num_rows = b->get_size()[0];
     const auto a_vals = gko::acc::range<a_accessor>(
         std::array<size_type, 1>{num_stored_elements_per_row * stride},
         a->get_const_values());
@@ -117,7 +116,6 @@ void advanced_spmv(std::shared_ptr<const OmpExecutor> exec,
     const auto num_stored_elements_per_row =
         a->get_num_stored_elements_per_row();
     const auto stride = a->get_stride();
-    const auto num_rows = b->get_size()[0];
     const auto a_vals = gko::acc::range<a_accessor>(
         std::array<size_type, 1>{num_stored_elements_per_row * stride},
         a->get_const_values());
diff --git a/reference/matrix/ell_kernels.cpp b/reference/matrix/ell_kernels.cpp
index 25de03cf709..24f166e52db 100644
--- a/reference/matrix/ell_kernels.cpp
+++ b/reference/matrix/ell_kernels.cpp
@@ -69,7 +69,6 @@ void spmv(std::shared_ptr<const ReferenceExecutor> exec,
     const auto num_stored_elements_per_row =
         a->get_num_stored_elements_per_row();
     const auto stride = a->get_stride();
-    const auto num_rows = b->get_size()[0];
     const auto a_vals = gko::acc::range<a_accessor>(
         std::array<size_type, 1>{num_stored_elements_per_row * stride},
         a->get_const_values());
@@ -112,7 +111,6 @@ void advanced_spmv(std::shared_ptr<const ReferenceExecutor> exec,
     const auto num_stored_elements_per_row =
         a->get_num_stored_elements_per_row();
     const auto stride = a->get_stride();
-    const auto num_rows = b->get_size()[0];
     const auto a_vals = gko::acc::range<a_accessor>(
         std::array<size_type, 1>{num_stored_elements_per_row * stride},
         a->get_const_values());

From 5b2afc13a8d18af98f6f673051a629a12ebd2276 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 19 Apr 2021 15:43:36 +0200
Subject: [PATCH 0686/1008] rename mixed precision ELL benchmark format to
 ell-mixed

Co-authored-by: Yuhsiang M. Tsai <19565938+yhmtsai@users.noreply.github.com>
---
 benchmark/utils/formats.hpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/benchmark/utils/formats.hpp b/benchmark/utils/formats.hpp
index 8c1aabbcafb..64de4e10f90 100644
--- a/benchmark/utils/formats.hpp
+++ b/benchmark/utils/formats.hpp
@@ -59,7 +59,7 @@ namespace formats {
 
 
 std::string available_format =
-    "coo, csr, ell, fell, sellp, hybrid, hybrid0, hybrid25, hybrid33, "
+    "coo, csr, ell, ell-mixed, sellp, hybrid, hybrid0, hybrid25, hybrid33, "
     "hybrid40, "
     "hybrid60, hybrid80, hybridlimit0, hybridlimit25, hybridlimit33, "
     "hybridminstorage"
@@ -91,7 +91,7 @@ std::string format_description =
     "csrm: Ginkgo's CSR implementation with merge_path strategy.\n"
     "ell: Ellpack format according to Bell and Garland: Efficient Sparse "
     "Matrix-Vector Multiplication on CUDA.\n"
-    "fell: Mixed Precision Ellpack format according to Bell and Garland: "
+    "ell-mixed: Mixed Precision Ellpack format according to Bell and Garland: "
     "Efficient Sparse Matrix-Vector Multiplication on CUDA.\n"
     "sellp: Sliced Ellpack uses a default block size of 32.\n"
     "hybrid: Hybrid uses ell and coo to represent the matrix.\n"
@@ -207,7 +207,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
         {"csrc", READ_MATRIX(csr, std::make_shared<csr::classical>())},
         {"coo", read_matrix_from_data<gko::matrix::Coo<etype>>},
         {"ell", read_matrix_from_data<gko::matrix::Ell<etype>>},
-        {"ell_mixed",
+        {"ell-mixed",
          [](std::shared_ptr<const gko::Executor> exec,
             const gko::matrix_data<etype> &data) {
              gko::matrix_data<gko::next_precision<etype>> conv_data;

From 5492686ae1ae6f3f82bff17579475432f477a67f Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 19 Apr 2021 15:47:43 +0200
Subject: [PATCH 0687/1008] add GINKGO_MIXED_PRECISION to Windows CI

---
 .github/workflows/windows-msvc-cuda.yml | 6 +++---
 .github/workflows/windows-msvc-ref.yml  | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/windows-msvc-cuda.yml b/.github/workflows/windows-msvc-cuda.yml
index bc4557e96bf..28468950af8 100644
--- a/.github/workflows/windows-msvc-cuda.yml
+++ b/.github/workflows/windows-msvc-cuda.yml
@@ -15,8 +15,8 @@ jobs:
       fail-fast: false
       matrix:
         config:
-        - {version: "10.2.89.20191206", name: "cuda102/release/shared"}
-        - {version: "latest", name: "cuda-latest/release/shared"}
+        - {version: "10.2.89.20191206", name: "cuda102/release/shared", "mixed": "ON"}
+        - {version: "latest", name: "cuda-latest/release/shared", "mixed": "OFF"}
     name: msvc/${{ matrix.config.name }} (only compile)
     runs-on: [windows-latest]
 
@@ -46,5 +46,5 @@ jobs:
         $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
         mkdir build
         cd build
-        cmake -DCMAKE_CXX_FLAGS=/bigobj -DGINKGO_BUILD_CUDA=ON -DGINKGO_BUILD_OMP=OFF -DGINKGO_CUDA_ARCHITECTURES=60 ..
+        cmake -DCMAKE_CXX_FLAGS=/bigobj -DGINKGO_BUILD_CUDA=ON -DGINKGO_BUILD_OMP=OFF -DGINKGO_MIXED_PRECISION=${{ matrix.config.mixed }} -DGINKGO_CUDA_ARCHITECTURES=60 ..
         cmake --build . -j4 --config Release
diff --git a/.github/workflows/windows-msvc-ref.yml b/.github/workflows/windows-msvc-ref.yml
index 13a29f6cc51..833f3692676 100644
--- a/.github/workflows/windows-msvc-ref.yml
+++ b/.github/workflows/windows-msvc-ref.yml
@@ -15,8 +15,8 @@ jobs:
       fail-fast: false
       matrix:
         config:
-        - {shared: "ON", build_type: "Debug", name: "reference/debug/shared"}
-        - {shared: "OFF", build_type: "Release", name: "reference/release/static"}
+        - {shared: "ON", build_type: "Debug", name: "reference/debug/shared", "mixed": "ON"}
+        - {shared: "OFF", build_type: "Release", name: "reference/release/static", "mixed": "OFF"}
         # Debug static needs too much storage
         # - {shared: "OFF", build_type: "Debug", name: "reference/debug/static"}
     name: msvc/${{ matrix.config.name }}
@@ -35,7 +35,7 @@ jobs:
         $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
         mkdir build
         cd build
-        cmake  -DCMAKE_CXX_FLAGS=/bigobj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_CXX_FLAGS_DEBUG="/MDd /Zi /Ob1 /Od /RTC1" -DGINKGO_BUILD_CUDA=OFF -DGINKGO_BUILD_OMP=OFF ..
+        cmake  -DCMAKE_CXX_FLAGS=/bigobj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_CXX_FLAGS_DEBUG="/MDd /Zi /Ob1 /Od /RTC1" -DGINKGO_BUILD_CUDA=OFF -DGINKGO_BUILD_OMP=OFF -DGINKGO_MIXED_PRECISION=${{ matrix.config.mixed }} ..
         cmake --build . -j4 --config ${{ matrix.config.build_type }}
         ctest . -C ${{ matrix.config.build_type }} --output-on-failure
 

From 7b6917451fcb06758f9b139baacc82e38010b770 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 19 Apr 2021 15:50:03 +0200
Subject: [PATCH 0688/1008] add GINKGO_MIXED_PRECISION to OSX CI

---
 .github/workflows/osx.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml
index 6827c5fe476..79caf2169a4 100644
--- a/.github/workflows/osx.yml
+++ b/.github/workflows/osx.yml
@@ -15,8 +15,8 @@ jobs:
       fail-fast: false
       matrix:
         config:
-        - {shared: "ON", build_type: "Debug", name: "omp/debug/shared"}
-        - {shared: "OFF", build_type: "Release", name: "omp/release/static"}
+        - {shared: "ON", build_type: "Debug", name: "omp/debug/shared", "mixed": "OFF"}
+        - {shared: "OFF", build_type: "Release", name: "omp/release/static", "mixed": "ON"}
     name: ${{ matrix.config.name }}
     runs-on: [macos-latest]
 
@@ -40,7 +40,7 @@ jobs:
       run: |
         mkdir build
         cd build
-        cmake .. -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }}
+        cmake .. -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_MIXED_PRECISION=${{ matrix.config.mixed }}
         make -j8
         ctest -j10 --output-on-failure
 

From 54e96698290666c93242c6c1625072594de7793e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 4 May 2021 11:51:11 +0200
Subject: [PATCH 0689/1008] simplify mixed-precision instantiation macros

---
 core/base/mixed_precision_types.hpp | 99 +++++++++++------------------
 1 file changed, 37 insertions(+), 62 deletions(-)

diff --git a/core/base/mixed_precision_types.hpp b/core/base/mixed_precision_types.hpp
index 079fcb8a075..d725eff82bf 100644
--- a/core/base/mixed_precision_types.hpp
+++ b/core/base/mixed_precision_types.hpp
@@ -39,70 +39,45 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #ifdef GINKGO_MIXED_PRECISION
-#define GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(_macro) \
-    template _macro(float, float, float, int32);                    \
-    template _macro(float, float, double, int32);                   \
-    template _macro(float, double, float, int32);                   \
-    template _macro(float, double, double, int32);                  \
-    template _macro(double, float, float, int32);                   \
-    template _macro(double, float, double, int32);                  \
-    template _macro(double, double, float, int32);                  \
-    template _macro(double, double, double, int32);                 \
-    template _macro(std::complex<float>, std::complex<float>,       \
-                    std::complex<float>, int32);                    \
-    template _macro(std::complex<float>, std::complex<float>,       \
-                    std::complex<double>, int32);                   \
-    template _macro(std::complex<float>, std::complex<double>,      \
-                    std::complex<float>, int32);                    \
-    template _macro(std::complex<float>, std::complex<double>,      \
-                    std::complex<double>, int32);                   \
-    template _macro(std::complex<double>, std::complex<float>,      \
-                    std::complex<float>, int32);                    \
-    template _macro(std::complex<double>, std::complex<float>,      \
-                    std::complex<double>, int32);                   \
-    template _macro(std::complex<double>, std::complex<double>,     \
-                    std::complex<float>, int32);                    \
-    template _macro(std::complex<double>, std::complex<double>,     \
-                    std::complex<double>, int32);                   \
-    template _macro(float, float, float, int64);                    \
-    template _macro(float, float, double, int64);                   \
-    template _macro(float, double, float, int64);                   \
-    template _macro(float, double, double, int64);                  \
-    template _macro(double, float, float, int64);                   \
-    template _macro(double, float, double, int64);                  \
-    template _macro(double, double, float, int64);                  \
-    template _macro(double, double, double, int64);                 \
-    template _macro(std::complex<float>, std::complex<float>,       \
-                    std::complex<float>, int64);                    \
-    template _macro(std::complex<float>, std::complex<float>,       \
-                    std::complex<double>, int64);                   \
-    template _macro(std::complex<float>, std::complex<double>,      \
-                    std::complex<float>, int64);                    \
-    template _macro(std::complex<float>, std::complex<double>,      \
-                    std::complex<double>, int64);                   \
-    template _macro(std::complex<double>, std::complex<float>,      \
-                    std::complex<float>, int64);                    \
-    template _macro(std::complex<double>, std::complex<float>,      \
-                    std::complex<double>, int64);                   \
-    template _macro(std::complex<double>, std::complex<double>,     \
-                    std::complex<float>, int64);                    \
-    template _macro(std::complex<double>, std::complex<double>,     \
-                    std::complex<double>, int64)
+#define GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_TYPE(_macro, ...)  \
+    template _macro(float, float, float, __VA_ARGS__);          \
+    template _macro(float, float, double, __VA_ARGS__);         \
+    template _macro(float, double, float, __VA_ARGS__);         \
+    template _macro(float, double, double, __VA_ARGS__);        \
+    template _macro(double, float, float, __VA_ARGS__);         \
+    template _macro(double, float, double, __VA_ARGS__);        \
+    template _macro(double, double, float, __VA_ARGS__);        \
+    template _macro(double, double, double, __VA_ARGS__);       \
+    template _macro(std::complex<float>, std::complex<float>,   \
+                    std::complex<float>, __VA_ARGS__);          \
+    template _macro(std::complex<float>, std::complex<float>,   \
+                    std::complex<double>, __VA_ARGS__);         \
+    template _macro(std::complex<float>, std::complex<double>,  \
+                    std::complex<float>, __VA_ARGS__);          \
+    template _macro(std::complex<float>, std::complex<double>,  \
+                    std::complex<double>, __VA_ARGS__);         \
+    template _macro(std::complex<double>, std::complex<float>,  \
+                    std::complex<float>, __VA_ARGS__);          \
+    template _macro(std::complex<double>, std::complex<float>,  \
+                    std::complex<double>, __VA_ARGS__);         \
+    template _macro(std::complex<double>, std::complex<double>, \
+                    std::complex<float>, __VA_ARGS__);          \
+    template _macro(std::complex<double>, std::complex<double>, \
+                    std::complex<double>, __VA_ARGS__)
 #else
-#define GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(_macro) \
-    template _macro(float, float, float, int32);                    \
-    template _macro(double, double, double, int32);                 \
-    template _macro(std::complex<float>, std::complex<float>,       \
-                    std::complex<float>, int32);                    \
-    template _macro(std::complex<double>, std::complex<double>,     \
-                    std::complex<double>, int32);                   \
-    template _macro(float, float, float, int64);                    \
-    template _macro(double, double, double, int64);                 \
-    template _macro(std::complex<float>, std::complex<float>,       \
-                    std::complex<float>, int64);                    \
-    template _macro(std::complex<double>, std::complex<double>,     \
-                    std::complex<double>, int64)
+#define GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_TYPE(_macro, ...)  \
+    template _macro(float, float, float, __VA_ARGS__);          \
+    template _macro(double, double, double, __VA_ARGS__);       \
+    template _macro(std::complex<float>, std::complex<float>,   \
+                    std::complex<float>, __VA_ARGS__);          \
+    template _macro(std::complex<double>, std::complex<double>, \
+                    std::complex<double>, __VA_ARGS__)
 #endif
 
 
+#define GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(_macro) \
+    GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_TYPE(_macro, int32);       \
+    GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_TYPE(_macro, int64)
+
+
 #endif  // GKO_CORE_BASE_MIXED_PRECISION_TYPES_HPP_

From a81c3d4a250ca191090c27e7629f5b35fd854193 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Gr=C3=BCtzmacher?= <thomas.gruetzmacher@kit.edu>
Date: Thu, 6 May 2021 17:21:47 +0200
Subject: [PATCH 0690/1008] Add set_krylov_dim to CB-GMRES

---
 core/test/solver/cb_gmres.cpp           | 35 ++++++++++++++++++++-----
 include/ginkgo/core/solver/cb_gmres.hpp | 19 +++++++++-----
 include/ginkgo/core/solver/gmres.hpp    | 12 ++++-----
 3 files changed, 48 insertions(+), 18 deletions(-)

diff --git a/core/test/solver/cb_gmres.cpp b/core/test/solver/cb_gmres.cpp
index 722decbdb7a..8cf060190f6 100644
--- a/core/test/solver/cb_gmres.cpp
+++ b/core/test/solver/cb_gmres.cpp
@@ -155,6 +155,7 @@ TYPED_TEST(CbGmres, CbGmresFactoryKnowsItsExecutor)
 TYPED_TEST(CbGmres, CbGmresFactoryCreatesCorrectSolver)
 {
     using Solver = typename TestFixture::Solver;
+
     ASSERT_EQ(this->solver->get_size(), gko::dim<2>(3, 3));
     auto cb_gmres_solver = static_cast<Solver *>(this->solver.get());
     ASSERT_NE(cb_gmres_solver->get_system_matrix(), nullptr);
@@ -265,18 +266,40 @@ TYPED_TEST(CbGmres, CanSetKrylovDim)
     using value_type = typename TestFixture::value_type;
     using nc_value_type = typename TestFixture::nc_value_type;
     using Solver = typename TestFixture::Solver;
+    const gko::size_type new_krylov_dim{4u};
+
     auto cb_gmres_factory =
         Solver::build()
-            .with_krylov_dim(4u)
+            .with_krylov_dim(new_krylov_dim)
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(4u).on(this->exec),
-                gko::stop::ResidualNormReduction<value_type>::build()
-                    .with_reduction_factor(nc_value_type{1e-6})
-                    .on(this->exec))
+                gko::stop::Iteration::build().with_max_iters(4u).on(this->exec))
             .on(this->exec);
     auto solver = cb_gmres_factory->generate(this->mtx);
-    auto krylov_dim = solver->get_krylov_dim();
 
+    ASSERT_EQ(solver->get_krylov_dim(), new_krylov_dim);
+    // Also test the default storage_recision
+    ASSERT_EQ(solver->get_storage_precision(),
+              gko::solver::cb_gmres::storage_precision::reduce1);
+}
+
+
+TYPED_TEST(CbGmres, CanUseSetKrylovDim)
+{
+    using value_type = typename TestFixture::value_type;
+    using nc_value_type = typename TestFixture::nc_value_type;
+    using Solver = typename TestFixture::Solver;
+    const gko::size_type new_krylov_dim{40u};
+    auto cb_gmres_factory =
+        Solver::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(4u).on(this->exec))
+            .on(this->exec);
+    auto solver = cb_gmres_factory->generate(this->mtx);
+
+    solver->set_krylov_dim(new_krylov_dim);
+
+    ASSERT_EQ(solver->get_krylov_dim(), new_krylov_dim);
+    // Also test the default storage_recision
     ASSERT_EQ(solver->get_storage_precision(),
               gko::solver::cb_gmres::storage_precision::reduce1);
 }
diff --git a/include/ginkgo/core/solver/cb_gmres.hpp b/include/ginkgo/core/solver/cb_gmres.hpp
index ebc85b7e323..4ee9fe18711 100644
--- a/include/ginkgo/core/solver/cb_gmres.hpp
+++ b/include/ginkgo/core/solver/cb_gmres.hpp
@@ -107,10 +107,10 @@ enum class storage_precision {
  * are merged into 2 separate steps. Classical Gram-Schmidt with
  * reorthogonalization is used.
  *
- * The krylov basis can be stored in reduced precision (compressed) to reduce
- * memory accesses, while all computations (including krylov basis operations)
+ * The Krylov basis can be stored in reduced precision (compressed) to reduce
+ * memory accesses, while all computations (including Krylov basis operations)
  * are performed in the same arithmetic precision ValueType. By default, the
- * krylov basis are stored in one precision lower than ValueType.
+ * Krylov basis are stored in one precision lower than ValueType.
  *
  * @tparam ValueType  the arithmetic precision and the precision of matrix
  *                    elements
@@ -138,12 +138,19 @@ class CbGmres : public EnableLinOp<CbGmres<ValueType>>,
     }
 
     /**
-     * Returns the krylov dimension.
+     * Returns the Krylov dimension.
      *
-     * @return the krylov dimension
+     * @return the Krylov dimension
      */
     size_type get_krylov_dim() const { return krylov_dim_; }
 
+    /**
+     * Sets the Krylov dimension
+     *
+     * @param other  the new Krylov dimension
+     */
+    void set_krylov_dim(size_type other) { krylov_dim_ = other; }
+
     /**
      * Returns the storage precision used internally.
      *
@@ -182,7 +189,7 @@ class CbGmres : public EnableLinOp<CbGmres<ValueType>>,
             generated_preconditioner, nullptr);
 
         /**
-         * krylov dimension factory.
+         * Krylov dimension factory.
          */
         size_type GKO_FACTORY_PARAMETER_SCALAR(krylov_dim, 100u);
     };
diff --git a/include/ginkgo/core/solver/gmres.hpp b/include/ginkgo/core/solver/gmres.hpp
index 0f643ece69a..a24e42392d3 100644
--- a/include/ginkgo/core/solver/gmres.hpp
+++ b/include/ginkgo/core/solver/gmres.hpp
@@ -102,18 +102,18 @@ class Gmres : public EnableLinOp<Gmres<ValueType>>,
     bool apply_uses_initial_guess() const override { return true; }
 
     /**
-     * Gets the krylov dimension of the solver
+     * Gets the Krylov dimension of the solver
      *
-     * @return the krylov dimension
+     * @return the Krylov dimension
      */
     size_type get_krylov_dim() const { return krylov_dim_; }
 
     /**
-     * Sets the krylov dimension
+     * Sets the Krylov dimension
      *
-     * @param other  the new krylov dimension
+     * @param other  the new Krylov dimension
      */
-    void set_krylov_dim(const size_type &other) { krylov_dim_ = other; }
+    void set_krylov_dim(size_type other) { krylov_dim_ = other; }
 
     /**
      * Gets the stopping criterion factory of the solver.
@@ -159,7 +159,7 @@ class Gmres : public EnableLinOp<Gmres<ValueType>>,
             generated_preconditioner, nullptr);
 
         /**
-         * krylov dimension factory.
+         * Krylov dimension factory.
          */
         size_type GKO_FACTORY_PARAMETER_SCALAR(krylov_dim, 0u);
     };

From a70304b3fd12e8312cc7f7fc56f4cdaed45ed159 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 4 May 2021 23:38:03 +0200
Subject: [PATCH 0691/1008] Allow using device memory in debug mode

---
 core/device_hooks/cuda_hooks.cpp      |  7 +++--
 core/device_hooks/hip_hooks.cpp       |  7 +++--
 cuda/base/executor.cpp                | 17 ++++++-----
 cuda/test/base/cuda_executor.cu       | 31 ++++++++++++++++++-
 hip/base/executor.hip.cpp             | 17 ++++++-----
 hip/test/base/hip_executor.hip.cpp    | 41 ++++++++++++++++++++++++-
 include/ginkgo/core/base/executor.hpp | 43 ++++++++++++++++++++++-----
 7 files changed, 133 insertions(+), 30 deletions(-)

diff --git a/core/device_hooks/cuda_hooks.cpp b/core/device_hooks/cuda_hooks.cpp
index 0d06e888b7f..f6004a965a5 100644
--- a/core/device_hooks/cuda_hooks.cpp
+++ b/core/device_hooks/cuda_hooks.cpp
@@ -55,10 +55,11 @@ version version_info::get_cuda_version() noexcept
 
 
 std::shared_ptr<CudaExecutor> CudaExecutor::create(
-    int device_id, std::shared_ptr<Executor> master, bool device_reset)
+    int device_id, std::shared_ptr<Executor> master, bool device_reset,
+    bool use_unified_mem)
 {
-    return std::shared_ptr<CudaExecutor>(
-        new CudaExecutor(device_id, std::move(master), device_reset));
+    return std::shared_ptr<CudaExecutor>(new CudaExecutor(
+        device_id, std::move(master), device_reset, use_unified_mem));
 }
 
 
diff --git a/core/device_hooks/hip_hooks.cpp b/core/device_hooks/hip_hooks.cpp
index 6a5d79b6f73..57c08e1b43f 100644
--- a/core/device_hooks/hip_hooks.cpp
+++ b/core/device_hooks/hip_hooks.cpp
@@ -52,10 +52,11 @@ version version_info::get_hip_version() noexcept
 
 
 std::shared_ptr<HipExecutor> HipExecutor::create(
-    int device_id, std::shared_ptr<Executor> master, bool device_reset)
+    int device_id, std::shared_ptr<Executor> master, bool device_reset,
+    bool use_unified_mem)
 {
-    return std::shared_ptr<HipExecutor>(
-        new HipExecutor(device_id, std::move(master), device_reset));
+    return std::shared_ptr<HipExecutor>(new HipExecutor(
+        device_id, std::move(master), device_reset, use_unified_mem));
 }
 
 
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 0cbaa82b3b9..65065bae451 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -56,10 +56,12 @@ namespace gko {
 
 
 std::shared_ptr<CudaExecutor> CudaExecutor::create(
-    int device_id, std::shared_ptr<Executor> master, bool device_reset)
+    int device_id, std::shared_ptr<Executor> master, bool device_reset,
+    bool use_unified_mem)
 {
     return std::shared_ptr<CudaExecutor>(
-        new CudaExecutor(device_id, std::move(master), device_reset),
+        new CudaExecutor(device_id, std::move(master), device_reset,
+                         use_unified_mem),
         [device_id](CudaExecutor *exec) {
             auto device_reset = exec->get_device_reset();
             delete exec;
@@ -124,11 +126,12 @@ void *CudaExecutor::raw_alloc(size_type num_bytes) const
 {
     void *dev_ptr = nullptr;
     cuda::device_guard g(this->get_device_id());
-#ifdef NDEBUG
-    auto error_code = cudaMalloc(&dev_ptr, num_bytes);
-#else
-    auto error_code = cudaMallocManaged(&dev_ptr, num_bytes);
-#endif
+    int error_code = 0;
+    if (this->use_unified_memory_) {
+        error_code = cudaMallocManaged(&dev_ptr, num_bytes);
+    } else {
+        error_code = cudaMalloc(&dev_ptr, num_bytes);
+    }
     if (error_code != cudaErrorMemoryAllocation) {
         GKO_ASSERT_NO_CUDA_ERRORS(error_code);
     }
diff --git a/cuda/test/base/cuda_executor.cu b/cuda/test/base/cuda_executor.cu
index 902514ee6c8..0c01090fa0e 100644
--- a/cuda/test/base/cuda_executor.cu
+++ b/cuda/test/base/cuda_executor.cu
@@ -86,7 +86,10 @@ public:
 class CudaExecutor : public ::testing::Test {
 protected:
     CudaExecutor()
-        : omp(gko::OmpExecutor::create()), cuda(nullptr), cuda2(nullptr)
+        : omp(gko::OmpExecutor::create()),
+          cuda(nullptr),
+          cuda2(nullptr),
+          cuda3(nullptr)
     {}
 
     void SetUp()
@@ -95,6 +98,7 @@ protected:
         cuda = gko::CudaExecutor::create(0, omp);
         cuda2 = gko::CudaExecutor::create(
             gko::CudaExecutor::get_num_devices() - 1, omp);
+        cuda3 = gko::CudaExecutor::create(0, omp, false, true);
     }
 
     void TearDown()
@@ -108,6 +112,7 @@ protected:
     std::shared_ptr<gko::Executor> omp;
     std::shared_ptr<gko::CudaExecutor> cuda;
     std::shared_ptr<gko::CudaExecutor> cuda2;
+    std::shared_ptr<gko::CudaExecutor> cuda3;
 };
 
 
@@ -163,6 +168,7 @@ __global__ void check_data(int *data)
     }
 }
 
+
 TEST_F(CudaExecutor, CopiesDataToCuda)
 {
     int orig[] = {3, 8};
@@ -176,6 +182,29 @@ TEST_F(CudaExecutor, CopiesDataToCuda)
 }
 
 
+__global__ void check_data2(int *data)
+{
+    if (data[0] != 4 || data[1] != 8) {
+        asm("trap;");
+    }
+}
+
+
+TEST_F(CudaExecutor, CanAllocateOnUnifiedMemory)
+{
+    int orig[] = {3, 8};
+    auto *copy = cuda3->alloc<int>(2);
+
+    cuda3->copy_from(omp.get(), 2, orig, copy);
+
+    check_data<<<1, 1>>>(copy);
+    ASSERT_NO_THROW(cuda3->synchronize());
+    copy[0] = 4;
+    check_data2<<<1, 1>>>(copy);
+    cuda3->free(copy);
+}
+
+
 __global__ void init_data(int *data)
 {
     data[0] = 3;
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 49013544d98..bfad12fd237 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -56,10 +56,12 @@ namespace gko {
 
 
 std::shared_ptr<HipExecutor> HipExecutor::create(
-    int device_id, std::shared_ptr<Executor> master, bool device_reset)
+    int device_id, std::shared_ptr<Executor> master, bool device_reset,
+    bool use_unified_mem)
 {
     return std::shared_ptr<HipExecutor>(
-        new HipExecutor(device_id, std::move(master), device_reset),
+        new HipExecutor(device_id, std::move(master), device_reset,
+                        use_unified_mem),
         [device_id](HipExecutor *exec) {
             auto device_reset = exec->get_device_reset();
             delete exec;
@@ -124,11 +126,12 @@ void *HipExecutor::raw_alloc(size_type num_bytes) const
 {
     void *dev_ptr = nullptr;
     hip::device_guard g(this->get_device_id());
-#if defined(NDEBUG) || (GINKGO_HIP_PLATFORM_HCC == 1)
-    auto error_code = hipMalloc(&dev_ptr, num_bytes);
-#else
-    auto error_code = hipMallocManaged(&dev_ptr, num_bytes);
-#endif
+    int error_code = 0;
+    if (this->use_unified_memory_) {
+        error_code = hipMallocManaged(&dev_ptr, num_bytes);
+    } else {
+        error_code = hipMalloc(&dev_ptr, num_bytes);
+    }
     if (error_code != hipErrorMemoryAllocation) {
         GKO_ASSERT_NO_HIP_ERRORS(error_code);
     }
diff --git a/hip/test/base/hip_executor.hip.cpp b/hip/test/base/hip_executor.hip.cpp
index dba2003450e..a3ef1644dd0 100644
--- a/hip/test/base/hip_executor.hip.cpp
+++ b/hip/test/base/hip_executor.hip.cpp
@@ -91,7 +91,11 @@ class ExampleOperation : public gko::Operation {
 
 class HipExecutor : public ::testing::Test {
 protected:
-    HipExecutor() : omp(gko::OmpExecutor::create()), hip(nullptr), hip2(nullptr)
+    HipExecutor()
+        : omp(gko::OmpExecutor::create()),
+          hip(nullptr),
+          hip2(nullptr),
+          hip3(nullptr)
     {}
 
     void SetUp()
@@ -100,6 +104,7 @@ class HipExecutor : public ::testing::Test {
         hip = gko::HipExecutor::create(0, omp);
         hip2 = gko::HipExecutor::create(gko::HipExecutor::get_num_devices() - 1,
                                         omp);
+        hip = gko::HipExecutor::create(0, omp, false, true);
     }
 
     void TearDown()
@@ -113,6 +118,7 @@ class HipExecutor : public ::testing::Test {
     std::shared_ptr<gko::Executor> omp;
     std::shared_ptr<gko::HipExecutor> hip;
     std::shared_ptr<gko::HipExecutor> hip2;
+    std::shared_ptr<gko::HipExecutor> hip3;
 };
 
 
@@ -185,6 +191,39 @@ TEST_F(HipExecutor, CopiesDataToHip)
 }
 
 
+__global__ void check_data2(int *data)
+{
+    if (data[0] != 4 || data[1] != 8) {
+#if GINKGO_HIP_PLATFORM_HCC
+        asm("s_trap 0x02;");
+#else  // GINKGO_HIP_PLATFORM_NVCC
+        asm("trap;");
+#endif
+    }
+}
+
+
+#if GINKGO_HIP_PLATFORM_NVCC
+
+
+TEST_F(HipExecutor, CanAllocateOnUnifiedMemory)
+{
+    int orig[] = {3, 8};
+    auto *copy = hip->alloc<int>(2);
+
+    hip3->copy_from(omp.get(), 2, orig, copy);
+
+    check_data<<<1, 1>>>(copy);
+    ASSERT_NO_THROW(hip3->synchronize());
+    copy[0] = 4;
+    check_data2<<<1, 1>>>(copy);
+    hip3->free(copy);
+}
+
+
+#endif
+
+
 __global__ void init_data(int *data)
 {
     data[0] = 3;
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index baeea6b7d63..0779c5611eb 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -50,6 +50,23 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/synthesizer/containers.hpp>
 
 
+#if NDEBUG
+
+#define DEFAULT_USE_CUDA_UNIFIED_MEMORY true
+
+#if (GINKGO_HIP_PLATFORM_HCC == 1)
+#define DEFAULT_USE_HIP_UNIFIED_MEMORY true
+#else
+#define DEFAULT_USE_HIP_UNIFIED_MEMORY false
+#endif
+
+#else
+
+#define DEFAULT_USE_CUDA_UNIFIED_MEMORY false
+#define DEFAULT_USE_HIP_UNIFIED_MEMORY false
+
+#endif
+
 inline namespace cl {
 namespace sycl {
 
@@ -1239,7 +1256,8 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
      */
     static std::shared_ptr<CudaExecutor> create(
         int device_id, std::shared_ptr<Executor> master,
-        bool device_reset = false);
+        bool device_reset = false,
+        bool use_unified_memory = DEFAULT_USE_CUDA_UNIFIED_MEMORY);
 
     ~CudaExecutor() { decrease_num_execs(this->get_device_id()); }
 
@@ -1353,8 +1371,11 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     void init_handles();
 
     CudaExecutor(int device_id, std::shared_ptr<Executor> master,
-                 bool device_reset = false)
-        : EnableDeviceReset{device_reset}, master_(master)
+                 bool device_reset = false,
+                 bool use_unified_memory = DEFAULT_USE_CUDA_UNIFIED_MEMORY)
+        : EnableDeviceReset{device_reset},
+          use_unified_memory_{use_unified_memory},
+          master_(master)
     {
         this->get_exec_info().device_id = device_id;
         this->get_exec_info().num_computing_units = 0;
@@ -1416,6 +1437,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     static constexpr int max_devices = 64;
     static unsigned num_execs[max_devices];
     static std::mutex mutex[max_devices];
+    bool use_unified_memory_;
 };
 
 
@@ -1445,9 +1467,10 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
      * @param master  an executor on the host that is used to invoke the device
      *                kernels
      */
-    static std::shared_ptr<HipExecutor> create(int device_id,
-                                               std::shared_ptr<Executor> master,
-                                               bool device_reset = false);
+    static std::shared_ptr<HipExecutor> create(
+        int device_id, std::shared_ptr<Executor> master,
+        bool device_reset = false,
+        bool use_unified_memory = DEFAULT_USE_HIP_UNIFIED_MEMORY);
 
     ~HipExecutor() { decrease_num_execs(this->get_device_id()); }
 
@@ -1561,8 +1584,11 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     void init_handles();
 
     HipExecutor(int device_id, std::shared_ptr<Executor> master,
-                bool device_reset = false)
-        : EnableDeviceReset{device_reset}, master_(master)
+                bool device_reset = false,
+                bool use_unified_memory = DEFAULT_USE_HIP_UNIFIED_MEMORY)
+        : EnableDeviceReset{device_reset},
+          use_unified_memory_{use_unified_memory},
+          master_(master)
     {
         this->get_exec_info().device_id = device_id;
         this->get_exec_info().num_computing_units = 0;
@@ -1624,6 +1650,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     static constexpr int max_devices = 64;
     static int num_execs[max_devices];
     static std::mutex mutex[max_devices];
+    bool use_unified_memory_;
 };
 
 

From 04ea72398f2d517db8bb6d964185c61dfac307d6 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 5 May 2021 13:04:15 +0200
Subject: [PATCH 0692/1008] Update to enum, add different modes

---
 core/device_hooks/cuda_hooks.cpp      |  4 +-
 core/device_hooks/hip_hooks.cpp       |  4 +-
 cuda/base/executor.cpp                | 15 ++++---
 cuda/test/base/cuda_executor.cu       |  3 +-
 hip/base/executor.hip.cpp             | 15 ++++---
 hip/test/base/hip_executor.hip.cpp    |  5 ++-
 include/ginkgo/core/base/executor.hpp | 61 ++++++++++++++++++++-------
 7 files changed, 73 insertions(+), 34 deletions(-)

diff --git a/core/device_hooks/cuda_hooks.cpp b/core/device_hooks/cuda_hooks.cpp
index f6004a965a5..a58ef92e5ab 100644
--- a/core/device_hooks/cuda_hooks.cpp
+++ b/core/device_hooks/cuda_hooks.cpp
@@ -56,10 +56,10 @@ version version_info::get_cuda_version() noexcept
 
 std::shared_ptr<CudaExecutor> CudaExecutor::create(
     int device_id, std::shared_ptr<Executor> master, bool device_reset,
-    bool use_unified_mem)
+    allocation_mode alloc_mode)
 {
     return std::shared_ptr<CudaExecutor>(new CudaExecutor(
-        device_id, std::move(master), device_reset, use_unified_mem));
+        device_id, std::move(master), device_reset, alloc_mode));
 }
 
 
diff --git a/core/device_hooks/hip_hooks.cpp b/core/device_hooks/hip_hooks.cpp
index 57c08e1b43f..232a86b803b 100644
--- a/core/device_hooks/hip_hooks.cpp
+++ b/core/device_hooks/hip_hooks.cpp
@@ -53,10 +53,10 @@ version version_info::get_hip_version() noexcept
 
 std::shared_ptr<HipExecutor> HipExecutor::create(
     int device_id, std::shared_ptr<Executor> master, bool device_reset,
-    bool use_unified_mem)
+    allocation_mode alloc_mode)
 {
     return std::shared_ptr<HipExecutor>(new HipExecutor(
-        device_id, std::move(master), device_reset, use_unified_mem));
+        device_id, std::move(master), device_reset, alloc_mode));
 }
 
 
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 65065bae451..0c7bbef8f10 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -57,11 +57,11 @@ namespace gko {
 
 std::shared_ptr<CudaExecutor> CudaExecutor::create(
     int device_id, std::shared_ptr<Executor> master, bool device_reset,
-    bool use_unified_mem)
+    allocation_mode alloc_mode)
 {
     return std::shared_ptr<CudaExecutor>(
         new CudaExecutor(device_id, std::move(master), device_reset,
-                         use_unified_mem),
+                         alloc_mode),
         [device_id](CudaExecutor *exec) {
             auto device_reset = exec->get_device_reset();
             delete exec;
@@ -127,10 +127,15 @@ void *CudaExecutor::raw_alloc(size_type num_bytes) const
     void *dev_ptr = nullptr;
     cuda::device_guard g(this->get_device_id());
     int error_code = 0;
-    if (this->use_unified_memory_) {
-        error_code = cudaMallocManaged(&dev_ptr, num_bytes);
-    } else {
+    if (this->alloc_mode_ == allocation_mode::unified_host) {
+        error_code = cudaMallocManaged(&dev_ptr, num_bytes, cudaMemAttachHost);
+    } else if (this->alloc_mode_ == allocation_mode::unified_global) {
+        error_code =
+            cudaMallocManaged(&dev_ptr, num_bytes, cudaMemAttachGlobal);
+    } else if (this->alloc_mode_ == allocation_mode::device) {
         error_code = cudaMalloc(&dev_ptr, num_bytes);
+    } else {
+        GKO_NOT_SUPPORTED(this->alloc_mode_);
     }
     if (error_code != cudaErrorMemoryAllocation) {
         GKO_ASSERT_NO_CUDA_ERRORS(error_code);
diff --git a/cuda/test/base/cuda_executor.cu b/cuda/test/base/cuda_executor.cu
index 0c01090fa0e..a12ec9e3203 100644
--- a/cuda/test/base/cuda_executor.cu
+++ b/cuda/test/base/cuda_executor.cu
@@ -98,7 +98,8 @@ protected:
         cuda = gko::CudaExecutor::create(0, omp);
         cuda2 = gko::CudaExecutor::create(
             gko::CudaExecutor::get_num_devices() - 1, omp);
-        cuda3 = gko::CudaExecutor::create(0, omp, false, true);
+        cuda3 = gko::CudaExecutor::create(0, omp, false,
+                                          gko::allocation_mode::unified_host);
     }
 
     void TearDown()
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index bfad12fd237..32b77f462c7 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -57,11 +57,10 @@ namespace gko {
 
 std::shared_ptr<HipExecutor> HipExecutor::create(
     int device_id, std::shared_ptr<Executor> master, bool device_reset,
-    bool use_unified_mem)
+    allocation_mode alloc_mode)
 {
     return std::shared_ptr<HipExecutor>(
-        new HipExecutor(device_id, std::move(master), device_reset,
-                        use_unified_mem),
+        new HipExecutor(device_id, std::move(master), device_reset, alloc_mode),
         [device_id](HipExecutor *exec) {
             auto device_reset = exec->get_device_reset();
             delete exec;
@@ -127,10 +126,14 @@ void *HipExecutor::raw_alloc(size_type num_bytes) const
     void *dev_ptr = nullptr;
     hip::device_guard g(this->get_device_id());
     int error_code = 0;
-    if (this->use_unified_memory_) {
-        error_code = hipMallocManaged(&dev_ptr, num_bytes);
-    } else {
+    if (this->alloc_mode_ == allocation_mode::unified_host) {
+        error_code = hipMallocManaged(&dev_ptr, num_bytes, hipMemAttachHost);
+    } else if (this->alloc_mode_ == allocation_mode::unified_global) {
+        error_code = hipMallocManaged(&dev_ptr, num_bytes, hipMemAttachGlobal);
+    } else if (this->alloc_mode_ == allocation_mode::device) {
         error_code = hipMalloc(&dev_ptr, num_bytes);
+    } else {
+        GKO_NOT_SUPPORTED(this->alloc_mode_);
     }
     if (error_code != hipErrorMemoryAllocation) {
         GKO_ASSERT_NO_HIP_ERRORS(error_code);
diff --git a/hip/test/base/hip_executor.hip.cpp b/hip/test/base/hip_executor.hip.cpp
index a3ef1644dd0..387300526a7 100644
--- a/hip/test/base/hip_executor.hip.cpp
+++ b/hip/test/base/hip_executor.hip.cpp
@@ -104,7 +104,8 @@ class HipExecutor : public ::testing::Test {
         hip = gko::HipExecutor::create(0, omp);
         hip2 = gko::HipExecutor::create(gko::HipExecutor::get_num_devices() - 1,
                                         omp);
-        hip = gko::HipExecutor::create(0, omp, false, true);
+        hip3 = gko::HipExecutor::create(0, omp, false,
+                                        gko::allocation_mode::unified_global);
     }
 
     void TearDown()
@@ -209,7 +210,7 @@ __global__ void check_data2(int *data)
 TEST_F(HipExecutor, CanAllocateOnUnifiedMemory)
 {
     int orig[] = {3, 8};
-    auto *copy = hip->alloc<int>(2);
+    auto *copy = hip3->alloc<int>(2);
 
     hip3->copy_from(omp.get(), 2, orig, copy);
 
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 0779c5611eb..9f961e4d6b1 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -50,20 +50,49 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/synthesizer/containers.hpp>
 
 
-#if NDEBUG
+namespace gko {
 
-#define DEFAULT_USE_CUDA_UNIFIED_MEMORY true
 
-#if (GINKGO_HIP_PLATFORM_HCC == 1)
-#define DEFAULT_USE_HIP_UNIFIED_MEMORY true
+/**
+ * Specify the mode of allocation for CUDA/HIP GPUs.
+ *
+ * `device` allocates memory on the device and Unified Memory model is not used.
+ *
+ * `unified_global` allocates memory on the device, but is accessible by the
+ * host through the Unified memory model.
+ *
+ * `unified_host` allocates memory on the
+ * host and it is not available on devices which do not have concurrent acesses
+ * switched on, but this access can be explictly switched on, when necessary.
+ */
+enum class allocation_mode { device, unified_global, unified_host };
+
+
+}  // namespace gko
+
+
+#ifdef NDEBUG
+
+// When in release, always prefer device allocations
+#define GKO_DEFAULT_CUDA_ALLOC_MODE allocation_mode::device
+
+#define GKO_DEFAULT_HIP_ALLOC_MODE allocation_mode::device
+
 #else
-#define DEFAULT_USE_HIP_UNIFIED_MEMORY false
-#endif
+
+// When in debug, always prefer UM allocations.
+#define GKO_DEFAULT_CUDA_ALLOC_MODE allocation_mode::unified_global
+
+#if (GINKGO_HIP_PLATFORM_HCC == 1)
+
+// HIP on AMD GPUs does not support UM, so always prefer device allocations.
+#define GKO_DEFAULT_HIP_ALLOC_MODE allocation_mode::device
 
 #else
 
-#define DEFAULT_USE_CUDA_UNIFIED_MEMORY false
-#define DEFAULT_USE_HIP_UNIFIED_MEMORY false
+#define GKO_DEFAULT_HIP_ALLOC_MODE allocation_mode::unified_global
+
+#endif
 
 #endif
 
@@ -1257,7 +1286,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     static std::shared_ptr<CudaExecutor> create(
         int device_id, std::shared_ptr<Executor> master,
         bool device_reset = false,
-        bool use_unified_memory = DEFAULT_USE_CUDA_UNIFIED_MEMORY);
+        allocation_mode alloc_mode = GKO_DEFAULT_CUDA_ALLOC_MODE);
 
     ~CudaExecutor() { decrease_num_execs(this->get_device_id()); }
 
@@ -1372,9 +1401,9 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
     CudaExecutor(int device_id, std::shared_ptr<Executor> master,
                  bool device_reset = false,
-                 bool use_unified_memory = DEFAULT_USE_CUDA_UNIFIED_MEMORY)
+                 allocation_mode alloc_mode = GKO_DEFAULT_CUDA_ALLOC_MODE)
         : EnableDeviceReset{device_reset},
-          use_unified_memory_{use_unified_memory},
+          alloc_mode_{alloc_mode},
           master_(master)
     {
         this->get_exec_info().device_id = device_id;
@@ -1437,7 +1466,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     static constexpr int max_devices = 64;
     static unsigned num_execs[max_devices];
     static std::mutex mutex[max_devices];
-    bool use_unified_memory_;
+    allocation_mode alloc_mode_;
 };
 
 
@@ -1470,7 +1499,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     static std::shared_ptr<HipExecutor> create(
         int device_id, std::shared_ptr<Executor> master,
         bool device_reset = false,
-        bool use_unified_memory = DEFAULT_USE_HIP_UNIFIED_MEMORY);
+        allocation_mode alloc_mode = GKO_DEFAULT_HIP_ALLOC_MODE);
 
     ~HipExecutor() { decrease_num_execs(this->get_device_id()); }
 
@@ -1585,9 +1614,9 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
     HipExecutor(int device_id, std::shared_ptr<Executor> master,
                 bool device_reset = false,
-                bool use_unified_memory = DEFAULT_USE_HIP_UNIFIED_MEMORY)
+                allocation_mode alloc_mode = GKO_DEFAULT_HIP_ALLOC_MODE)
         : EnableDeviceReset{device_reset},
-          use_unified_memory_{use_unified_memory},
+          alloc_mode_(alloc_mode),
           master_(master)
     {
         this->get_exec_info().device_id = device_id;
@@ -1650,7 +1679,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     static constexpr int max_devices = 64;
     static int num_execs[max_devices];
     static std::mutex mutex[max_devices];
-    bool use_unified_memory_;
+    allocation_mode alloc_mode_;
 };
 
 

From 8c6b0800d367f701c455a4e93c45c16d18f7b7b0 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Fri, 7 May 2021 11:59:24 +0200
Subject: [PATCH 0693/1008] Review update.

+ Add Doc
+ Use constexpr

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
Co-authored-by: Yu-Hsiang Tsai <yhmtsai@gmail.com>
---
 cuda/test/base/cuda_executor.cu       |  2 +-
 hip/base/executor.hip.cpp             | 10 ++++---
 include/ginkgo/core/base/executor.hpp | 39 +++++++++++++++++----------
 3 files changed, 32 insertions(+), 19 deletions(-)

diff --git a/cuda/test/base/cuda_executor.cu b/cuda/test/base/cuda_executor.cu
index a12ec9e3203..c53f9497f62 100644
--- a/cuda/test/base/cuda_executor.cu
+++ b/cuda/test/base/cuda_executor.cu
@@ -99,7 +99,7 @@ protected:
         cuda2 = gko::CudaExecutor::create(
             gko::CudaExecutor::get_num_devices() - 1, omp);
         cuda3 = gko::CudaExecutor::create(0, omp, false,
-                                          gko::allocation_mode::unified_host);
+                                          gko::allocation_mode::unified_global);
     }
 
     void TearDown()
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 32b77f462c7..1e43fe96549 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -126,12 +126,14 @@ void *HipExecutor::raw_alloc(size_type num_bytes) const
     void *dev_ptr = nullptr;
     hip::device_guard g(this->get_device_id());
     int error_code = 0;
-    if (this->alloc_mode_ == allocation_mode::unified_host) {
-        error_code = hipMallocManaged(&dev_ptr, num_bytes, hipMemAttachHost);
+    if (this->alloc_mode_ == allocation_mode::device) {
+        error_code = hipMalloc(&dev_ptr, num_bytes);
+#if !(GKO_HIP_PLATFORM_HCC == 1)
     } else if (this->alloc_mode_ == allocation_mode::unified_global) {
         error_code = hipMallocManaged(&dev_ptr, num_bytes, hipMemAttachGlobal);
-    } else if (this->alloc_mode_ == allocation_mode::device) {
-        error_code = hipMalloc(&dev_ptr, num_bytes);
+    } else if (this->alloc_mode_ == allocation_mode::unified_host) {
+        error_code = hipMallocManaged(&dev_ptr, num_bytes, hipMemAttachHost);
+#endif
     } else {
         GKO_NOT_SUPPORTED(this->alloc_mode_);
     }
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 9f961e4d6b1..d9ca2e3af32 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -68,34 +68,37 @@ namespace gko {
 enum class allocation_mode { device, unified_global, unified_host };
 
 
-}  // namespace gko
-
-
 #ifdef NDEBUG
 
-// When in release, always prefer device allocations
-#define GKO_DEFAULT_CUDA_ALLOC_MODE allocation_mode::device
+// When in release, prefer device allocations
+constexpr allocation_mode default_cuda_alloc_mode = allocation_mode::device;
 
-#define GKO_DEFAULT_HIP_ALLOC_MODE allocation_mode::device
+constexpr allocation_mode default_hip_alloc_mode = allocation_mode::device;
 
 #else
 
-// When in debug, always prefer UM allocations.
-#define GKO_DEFAULT_CUDA_ALLOC_MODE allocation_mode::unified_global
+// When in debug, always UM allocations.
+constexpr allocation_mode default_cuda_alloc_mode =
+    allocation_mode::unified_global;
 
 #if (GINKGO_HIP_PLATFORM_HCC == 1)
 
 // HIP on AMD GPUs does not support UM, so always prefer device allocations.
-#define GKO_DEFAULT_HIP_ALLOC_MODE allocation_mode::device
+constexpr allocation_mode default_hip_alloc_mode = allocation_mode::device;
 
 #else
 
-#define GKO_DEFAULT_HIP_ALLOC_MODE allocation_mode::unified_global
+// HIP on NVIDIA GPUs supports UM, so prefer UM allocations.
+constexpr allocation_mode default_hip_alloc_mode =
+    allocation_mode::unified_global;
 
 #endif
 
 #endif
 
+
+}  // namespace gko
+
 inline namespace cl {
 namespace sycl {
 
@@ -1282,11 +1285,15 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
      * @param device_id  the CUDA device id of this device
      * @param master  an executor on the host that is used to invoke the device
      * kernels
+     * @param device_reset  whether to reset the device after the object exits
+     *                      the scope.
+     * @param alloc_mode  the allocation mode that the executor should operate
+     *                    on. See @allocation_mode for more details
      */
     static std::shared_ptr<CudaExecutor> create(
         int device_id, std::shared_ptr<Executor> master,
         bool device_reset = false,
-        allocation_mode alloc_mode = GKO_DEFAULT_CUDA_ALLOC_MODE);
+        allocation_mode alloc_mode = default_cuda_alloc_mode);
 
     ~CudaExecutor() { decrease_num_execs(this->get_device_id()); }
 
@@ -1401,7 +1408,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
     CudaExecutor(int device_id, std::shared_ptr<Executor> master,
                  bool device_reset = false,
-                 allocation_mode alloc_mode = GKO_DEFAULT_CUDA_ALLOC_MODE)
+                 allocation_mode alloc_mode = default_cuda_alloc_mode)
         : EnableDeviceReset{device_reset},
           alloc_mode_{alloc_mode},
           master_(master)
@@ -1495,11 +1502,15 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
      * @param device_id  the HIP device id of this device
      * @param master  an executor on the host that is used to invoke the device
      *                kernels
+     * @param device_reset  whether to reset the device after the object exits
+     *                      the scope.
+     * @param alloc_mode  the allocation mode that the executor should operate
+     *                    on. See @allocation_mode for more details
      */
     static std::shared_ptr<HipExecutor> create(
         int device_id, std::shared_ptr<Executor> master,
         bool device_reset = false,
-        allocation_mode alloc_mode = GKO_DEFAULT_HIP_ALLOC_MODE);
+        allocation_mode alloc_mode = default_hip_alloc_mode);
 
     ~HipExecutor() { decrease_num_execs(this->get_device_id()); }
 
@@ -1614,7 +1625,7 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
     HipExecutor(int device_id, std::shared_ptr<Executor> master,
                 bool device_reset = false,
-                allocation_mode alloc_mode = GKO_DEFAULT_HIP_ALLOC_MODE)
+                allocation_mode alloc_mode = default_hip_alloc_mode)
         : EnableDeviceReset{device_reset},
           alloc_mode_(alloc_mode),
           master_(master)

From 921b72f0fd393188af131356f4939a32ae77abd7 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 10 Mar 2021 21:56:14 +0100
Subject: [PATCH 0694/1008] make example builds stand-alone capable

---
 examples/CMakeLists.txt                       | 16 ++++++++--
 .../CMakeLists.txt                            | 15 +++++++---
 examples/cb-gmres/CMakeLists.txt              | 16 +++++++---
 examples/custom-logger/CMakeLists.txt         | 16 +++++++---
 examples/custom-matrix-format/CMakeLists.txt  | 30 +++++++++++--------
 .../custom-stopping-criterion/CMakeLists.txt  | 20 +++++++++----
 examples/ginkgo-overhead/CMakeLists.txt       | 14 ++++++---
 examples/ginkgo-ranges/CMakeLists.txt         | 14 ++++++---
 examples/heat-equation/CMakeLists.txt         | 23 +++++++-------
 .../ilu-preconditioned-solver/CMakeLists.txt  | 16 +++++++---
 examples/inverse-iteration/CMakeLists.txt     | 16 +++++++---
 .../CMakeLists.txt                            | 16 +++++++---
 examples/iterative-refinement/CMakeLists.txt  | 16 +++++++---
 examples/minimal-cuda-solver/CMakeLists.txt   | 16 +++++++---
 examples/mixed-precision-ir/CMakeLists.txt    | 16 +++++++---
 .../nine-pt-stencil-solver/CMakeLists.txt     | 14 ++++++---
 examples/papi-logging/CMakeLists.txt          | 22 +++++++++-----
 examples/par-ilu-convergence/CMakeLists.txt   | 16 +++++++---
 examples/performance-debugging/CMakeLists.txt | 16 +++++++---
 examples/poisson-solver/CMakeLists.txt        | 14 ++++++---
 examples/preconditioned-solver/CMakeLists.txt | 15 +++++++---
 examples/preconditioner-export/CMakeLists.txt | 16 +++++++---
 examples/simple-solver-logging/CMakeLists.txt | 16 +++++++---
 examples/simple-solver/CMakeLists.txt         | 16 +++++++---
 .../three-pt-stencil-solver/CMakeLists.txt    | 14 ++++++---
 25 files changed, 299 insertions(+), 120 deletions(-)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 43b6f027990..559dea1b5c1 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -19,21 +19,31 @@ set(EXAMPLES_EXEC_LIST
 
 set(EXAMPLES_LIST
     ${EXAMPLES_EXEC_LIST}
-    custom-matrix-format
     custom-stopping-criterion
     ginkgo-overhead
-    heat-equation
     minimal-cuda-solver
-    papi-logging
     par-ilu-convergence
     performance-debugging
     preconditioner-export
     simple-solver-logging)
 
+if(GINKGO_BUILD_CUDA AND GINKGO_BUILD_OMP)
+    list(APPEND EXAMPLES_LIST custom-matrix-format)
+endif()
+
 if(GINKGO_BUILD_EXTLIB_EXAMPLE)
     list(APPEND EXAMPLES_LIST external-lib-interfacing)
 endif()
 
+find_package(OpenCV)
+if(OpenCV_FOUND)
+    list(APPEND EXAMPLES_LIST heat-equation)
+endif()
+
+if(GINKGO_HAVE_PAPI_SDE)
+    list(APPEND EXAMPLES_LIST papi-logging)
+endif()
+
 foreach(example ${EXAMPLES_LIST})
     add_subdirectory(${example})
 endforeach()
diff --git a/examples/adaptiveprecision-blockjacobi/CMakeLists.txt b/examples/adaptiveprecision-blockjacobi/CMakeLists.txt
index 5073564c8d1..b3d83725d0f 100644
--- a/examples/adaptiveprecision-blockjacobi/CMakeLists.txt
+++ b/examples/adaptiveprecision-blockjacobi/CMakeLists.txt
@@ -1,5 +1,12 @@
-set(target_name "adaptiveprecision-blockjacobi")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(adaptiveprecision-blockjacobi)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(adaptiveprecision-blockjacobi adaptiveprecision-blockjacobi.cpp)
+target_link_libraries(adaptiveprecision-blockjacobi Ginkgo::ginkgo)
+
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/cb-gmres/CMakeLists.txt b/examples/cb-gmres/CMakeLists.txt
index e3265883919..0234e4128ac 100644
--- a/examples/cb-gmres/CMakeLists.txt
+++ b/examples/cb-gmres/CMakeLists.txt
@@ -1,5 +1,13 @@
-set(target_name "cb-gmres")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(cb-gmres)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(cb-gmres cb-gmres.cpp)
+target_link_libraries(cb-gmres Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
 configure_file("${Ginkgo_SOURCE_DIR}/matrices/test/ani1.mtx" data/A.mtx COPYONLY)
diff --git a/examples/custom-logger/CMakeLists.txt b/examples/custom-logger/CMakeLists.txt
index d7978bed813..cf6fb7f4521 100644
--- a/examples/custom-logger/CMakeLists.txt
+++ b/examples/custom-logger/CMakeLists.txt
@@ -1,7 +1,15 @@
-set(target_name "custom-logger")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(custom-logger)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(custom-logger custom-logger.cpp)
+target_link_libraries(custom-logger Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/custom-matrix-format/CMakeLists.txt b/examples/custom-matrix-format/CMakeLists.txt
index 52e52c1bca4..c008a86f7b0 100644
--- a/examples/custom-matrix-format/CMakeLists.txt
+++ b/examples/custom-matrix-format/CMakeLists.txt
@@ -1,14 +1,18 @@
-if (GINKGO_BUILD_CUDA AND GINKGO_BUILD_OMP)
-    enable_language(CUDA)
-    set(CMAKE_CUDA_STANDARD 14)
-    set(CMAKE_CUDA_STANDARD_REQUIRED ON)
-    set(target_name "custom-matrix-format")
-    add_executable(${target_name}
-        ${target_name}.cpp
-        stencil_kernel.cu)
-    target_link_libraries(${target_name} Ginkgo::ginkgo)
-    target_include_directories(${target_name} PRIVATE
-        ${PROJECT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
-    # workaround for clang-cuda/g++ interaction
-    set_target_properties(${target_name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+cmake_minimum_required(VERSION 3.9)
+project(custom-matrix-format CXX CUDA)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
 endif()
+
+set(CMAKE_CUDA_STANDARD 14)
+set(CMAKE_CUDA_STANDARD_REQUIRED ON)
+
+add_executable(custom-matrix-format
+    custom-matrix-format.cpp
+        stencil_kernel.cu)
+target_link_libraries(custom-matrix-format Ginkgo::ginkgo)
+
+# workaround for clang-cuda/g++ interaction
+set_target_properties(custom-matrix-format PROPERTIES POSITION_INDEPENDENT_CODE ON)
diff --git a/examples/custom-stopping-criterion/CMakeLists.txt b/examples/custom-stopping-criterion/CMakeLists.txt
index 0a2196b814b..7d1db25a116 100644
--- a/examples/custom-stopping-criterion/CMakeLists.txt
+++ b/examples/custom-stopping-criterion/CMakeLists.txt
@@ -1,12 +1,20 @@
-set(target_name "custom-stopping-criterion")
+cmake_minimum_required(VERSION 3.9)
+project(custom-stopping-criterion)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
-add_executable(${target_name}
-	${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo
+
+add_executable(custom-stopping-criterion
+	custom-stopping-criterion.cpp)
+
+target_link_libraries(custom-stopping-criterion Ginkgo::ginkgo
 	Threads::Threads)
-target_include_directories(${target_name}
-	PRIVATE ${PROJECT_SOURCE_DIR})
+
+# Copy the data files to the execution directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/ginkgo-overhead/CMakeLists.txt b/examples/ginkgo-overhead/CMakeLists.txt
index 6f42ba09eb1..16328502185 100644
--- a/examples/ginkgo-overhead/CMakeLists.txt
+++ b/examples/ginkgo-overhead/CMakeLists.txt
@@ -1,4 +1,10 @@
-set(target_name "ginkgo-overhead")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(ginkgo-overhead)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(ginkgo-overhead ginkgo-overhead.cpp)
+target_link_libraries(ginkgo-overhead Ginkgo::ginkgo)
diff --git a/examples/ginkgo-ranges/CMakeLists.txt b/examples/ginkgo-ranges/CMakeLists.txt
index fc5ffa05f9c..943d8c2aef2 100644
--- a/examples/ginkgo-ranges/CMakeLists.txt
+++ b/examples/ginkgo-ranges/CMakeLists.txt
@@ -1,4 +1,10 @@
-set(target_name "ginkgo-ranges")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(ginkgo-ranges)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+add_executable(ginkgo-ranges ginkgo-ranges.cpp)
+target_link_libraries(ginkgo-ranges Ginkgo::ginkgo)
+target_include_directories(ginkgo-ranges PRIVATE ${PROJECT_SOURCE_DIR})
diff --git a/examples/heat-equation/CMakeLists.txt b/examples/heat-equation/CMakeLists.txt
index 75ba544a218..9de0f9ea83a 100644
--- a/examples/heat-equation/CMakeLists.txt
+++ b/examples/heat-equation/CMakeLists.txt
@@ -1,12 +1,15 @@
-set(target_name "heat-equation")
-find_package(OpenCV)
+cmake_minimum_required(VERSION 3.9)
+project(heat-equation)
 
-if (OpenCV_FOUND)
-    add_executable(${target_name} ${target_name}.cpp)
-    target_link_libraries(${target_name} Ginkgo::ginkgo ${OpenCV_LIBS})
-    target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
-    configure_file(data/initial.mtx data/initial.mtx COPYONLY)
-    configure_file(data/source.mtx data/source.mtx COPYONLY)
-else()
-    message("OpenCV not found, skipping heat-equation example")
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
 endif()
+find_package(OpenCV REQUIRED)
+
+add_executable(heat-equation heat-equation.cpp)
+target_link_libraries(heat-equation Ginkgo::ginkgo ${OpenCV_LIBS})
+
+# Copy the data files to the execution directory
+configure_file(data/initial.mtx data/initial.mtx COPYONLY)
+configure_file(data/source.mtx data/source.mtx COPYONLY)
diff --git a/examples/ilu-preconditioned-solver/CMakeLists.txt b/examples/ilu-preconditioned-solver/CMakeLists.txt
index f33f050c356..6e971588215 100644
--- a/examples/ilu-preconditioned-solver/CMakeLists.txt
+++ b/examples/ilu-preconditioned-solver/CMakeLists.txt
@@ -1,7 +1,15 @@
-set(target_name "ilu-preconditioned-solver")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(ilu-preconditioned-solver)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(ilu-preconditioned-solver ilu-preconditioned-solver.cpp)
+target_link_libraries(ilu-preconditioned-solver Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/inverse-iteration/CMakeLists.txt b/examples/inverse-iteration/CMakeLists.txt
index b7f9789eb65..03aa016b340 100644
--- a/examples/inverse-iteration/CMakeLists.txt
+++ b/examples/inverse-iteration/CMakeLists.txt
@@ -1,5 +1,13 @@
-set(target_name "inverse-iteration")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(inverse-iteration)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(inverse-iteration inverse-iteration.cpp)
+target_link_libraries(inverse-iteration Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/ir-ilu-preconditioned-solver/CMakeLists.txt b/examples/ir-ilu-preconditioned-solver/CMakeLists.txt
index 6b4c267d6a3..76a11d405f1 100644
--- a/examples/ir-ilu-preconditioned-solver/CMakeLists.txt
+++ b/examples/ir-ilu-preconditioned-solver/CMakeLists.txt
@@ -1,5 +1,13 @@
-set(target_name "ir-ilu-preconditioned-solver")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(ir-ilu-preconditioned-solver)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(ir-ilu-preconditioned-solver ir-ilu-preconditioned-solver.cpp)
+target_link_libraries(ir-ilu-preconditioned-solver Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/iterative-refinement/CMakeLists.txt b/examples/iterative-refinement/CMakeLists.txt
index 01d5600cdea..47c3942ae43 100644
--- a/examples/iterative-refinement/CMakeLists.txt
+++ b/examples/iterative-refinement/CMakeLists.txt
@@ -1,5 +1,13 @@
-set(target_name "iterative-refinement")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(iterative-refinement)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(iterative-refinement iterative-refinement.cpp)
+target_link_libraries(iterative-refinement Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/minimal-cuda-solver/CMakeLists.txt b/examples/minimal-cuda-solver/CMakeLists.txt
index 4aa13017904..ffc9274e16b 100644
--- a/examples/minimal-cuda-solver/CMakeLists.txt
+++ b/examples/minimal-cuda-solver/CMakeLists.txt
@@ -1,7 +1,15 @@
-set(target_name "minimal-cuda-solver")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(minimal-cuda-solver)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(minimal-cuda-solver minimal-cuda-solver.cpp)
+target_link_libraries(minimal-cuda-solver Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/mixed-precision-ir/CMakeLists.txt b/examples/mixed-precision-ir/CMakeLists.txt
index 109810c8657..8c23e367a82 100644
--- a/examples/mixed-precision-ir/CMakeLists.txt
+++ b/examples/mixed-precision-ir/CMakeLists.txt
@@ -1,5 +1,13 @@
-set(target_name "mixed-precision-ir")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(mixed-precision-ir)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(mixed-precision-ir mixed-precision-ir.cpp)
+target_link_libraries(mixed-precision-ir Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/nine-pt-stencil-solver/CMakeLists.txt b/examples/nine-pt-stencil-solver/CMakeLists.txt
index bed21d0f93d..510f0e85f07 100644
--- a/examples/nine-pt-stencil-solver/CMakeLists.txt
+++ b/examples/nine-pt-stencil-solver/CMakeLists.txt
@@ -1,4 +1,10 @@
-set(target_name "nine-pt-stencil-solver")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(nine-pt-stencil-solver)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(nine-pt-stencil-solver nine-pt-stencil-solver.cpp)
+target_link_libraries(nine-pt-stencil-solver Ginkgo::ginkgo)
diff --git a/examples/papi-logging/CMakeLists.txt b/examples/papi-logging/CMakeLists.txt
index a5f079025ba..e3836ecbe3b 100644
--- a/examples/papi-logging/CMakeLists.txt
+++ b/examples/papi-logging/CMakeLists.txt
@@ -1,9 +1,15 @@
-if (GINKGO_HAVE_PAPI_SDE)
-    set(target_name "papi-logging")
-    add_executable(${target_name} ${target_name}.cpp)
-    target_link_libraries(${target_name} ginkgo PAPI::PAPI)
-    target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
-    configure_file(data/A.mtx data/A.mtx COPYONLY)
-    configure_file(data/b.mtx data/b.mtx COPYONLY)
-    configure_file(data/x0.mtx data/x0.mtx COPYONLY)
+cmake_minimum_required(VERSION 3.9)
+project(papi-logging)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
 endif()
+
+add_executable(papi-logging papi-logging.cpp)
+target_link_libraries(papi-logging ginkgo PAPI::PAPI)
+
+# Copy the data files to the execution directory
+configure_file(data/A.mtx data/A.mtx COPYONLY)
+configure_file(data/b.mtx data/b.mtx COPYONLY)
+configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/par-ilu-convergence/CMakeLists.txt b/examples/par-ilu-convergence/CMakeLists.txt
index a81f0520a9a..ca2d88db12c 100644
--- a/examples/par-ilu-convergence/CMakeLists.txt
+++ b/examples/par-ilu-convergence/CMakeLists.txt
@@ -1,5 +1,13 @@
-set(target_name "par-ilu-convergence")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(par-ilu-convergence)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(par-ilu-convergence par-ilu-convergence.cpp)
+target_link_libraries(par-ilu-convergence Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/performance-debugging/CMakeLists.txt b/examples/performance-debugging/CMakeLists.txt
index 6ca05939515..6161b858d22 100644
--- a/examples/performance-debugging/CMakeLists.txt
+++ b/examples/performance-debugging/CMakeLists.txt
@@ -1,5 +1,13 @@
-set(target_name "performance-debugging")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(performance-debugging)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(performance-debugging performance-debugging.cpp)
+target_link_libraries(performance-debugging Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/poisson-solver/CMakeLists.txt b/examples/poisson-solver/CMakeLists.txt
index 5fc1e2ad83f..6c20c0fe087 100644
--- a/examples/poisson-solver/CMakeLists.txt
+++ b/examples/poisson-solver/CMakeLists.txt
@@ -1,4 +1,10 @@
-set(target_name "poisson-solver")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(poisson-solver)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(poisson-solver poisson-solver.cpp)
+target_link_libraries(poisson-solver Ginkgo::ginkgo)
diff --git a/examples/preconditioned-solver/CMakeLists.txt b/examples/preconditioned-solver/CMakeLists.txt
index a2c7898fd49..002f4ae7218 100644
--- a/examples/preconditioned-solver/CMakeLists.txt
+++ b/examples/preconditioned-solver/CMakeLists.txt
@@ -1,7 +1,14 @@
-set(target_name "preconditioned-solver")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(preconditioned-solver)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+add_executable(preconditioned-solver preconditioned-solver.cpp)
+target_link_libraries(preconditioned-solver Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/preconditioner-export/CMakeLists.txt b/examples/preconditioner-export/CMakeLists.txt
index 04134b532af..81969576dcb 100644
--- a/examples/preconditioner-export/CMakeLists.txt
+++ b/examples/preconditioner-export/CMakeLists.txt
@@ -1,5 +1,13 @@
-set(target_name "preconditioner-export")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(preconditioner-export)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(preconditioner-export preconditioner-export.cpp)
+target_link_libraries(preconditioner-export Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/simple-solver-logging/CMakeLists.txt b/examples/simple-solver-logging/CMakeLists.txt
index 545a1d9977c..1927690945e 100644
--- a/examples/simple-solver-logging/CMakeLists.txt
+++ b/examples/simple-solver-logging/CMakeLists.txt
@@ -1,7 +1,15 @@
-set(target_name "simple-solver-logging")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(simple-solver-logging)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(simple-solver-logging simple-solver-logging.cpp)
+target_link_libraries(simple-solver-logging Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/simple-solver/CMakeLists.txt b/examples/simple-solver/CMakeLists.txt
index f30cb01ffca..5603fea0377 100644
--- a/examples/simple-solver/CMakeLists.txt
+++ b/examples/simple-solver/CMakeLists.txt
@@ -1,7 +1,15 @@
-set(target_name "simple-solver")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(simple-solver)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(simple-solver simple-solver.cpp)
+target_link_libraries(simple-solver Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
 configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
diff --git a/examples/three-pt-stencil-solver/CMakeLists.txt b/examples/three-pt-stencil-solver/CMakeLists.txt
index c12b2ee7ff2..34842e283af 100644
--- a/examples/three-pt-stencil-solver/CMakeLists.txt
+++ b/examples/three-pt-stencil-solver/CMakeLists.txt
@@ -1,4 +1,10 @@
-set(target_name "three-pt-stencil-solver")
-add_executable(${target_name} ${target_name}.cpp)
-target_link_libraries(${target_name} Ginkgo::ginkgo)
-target_include_directories(${target_name} PRIVATE ${PROJECT_SOURCE_DIR})
+cmake_minimum_required(VERSION 3.9)
+project(three-pt-stencil-solver)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.3.0 REQUIRED)
+endif()
+
+add_executable(three-pt-stencil-solver three-pt-stencil-solver.cpp)
+target_link_libraries(three-pt-stencil-solver Ginkgo::ginkgo)

From f9e6c8e1a1ebefb5104c801d7bed7edd26e1a854 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 10 Mar 2021 21:57:17 +0100
Subject: [PATCH 0695/1008] add OpenMP version requirement

---
 cmake/autodetect_executors.cmake | 2 +-
 omp/CMakeLists.txt               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake/autodetect_executors.cmake b/cmake/autodetect_executors.cmake
index c432a92d593..71e2456edbc 100644
--- a/cmake/autodetect_executors.cmake
+++ b/cmake/autodetect_executors.cmake
@@ -2,7 +2,7 @@ set(GINKGO_HAS_OMP OFF)
 set(GINKGO_HAS_CUDA OFF)
 set(GINKGO_HAS_DPCPP OFF)
 set(GINKGO_HAS_HIP OFF)
-find_package(OpenMP)
+find_package(OpenMP 3.0)
 include(CheckLanguage)
 check_language(CUDA)
 try_compile(GKO_CAN_COMPILE_DPCPP ${PROJECT_BINARY_DIR}/dpcpp
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index 87cbc30cd53..625cb2bc870 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -1,4 +1,4 @@
-find_package(OpenMP REQUIRED)
+find_package(OpenMP 3.0 REQUIRED)
 
 add_library(ginkgo_omp $<TARGET_OBJECTS:ginkgo_omp_device> "")
 target_sources(ginkgo_omp

From 01960e576cac1d38a8a9b0278cfc5c2e76eeb01a Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 10 Mar 2021 21:57:51 +0100
Subject: [PATCH 0696/1008] add /bigobj flags on Windows by default

---
 .github/workflows/windows-cygwin.yml    | 2 +-
 .github/workflows/windows-mingw.yml     | 2 +-
 .github/workflows/windows-msvc-cuda.yml | 2 +-
 .github/workflows/windows-msvc-ref.yml  | 2 +-
 CMakeLists.txt                          | 7 +++++++
 5 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/windows-cygwin.yml b/.github/workflows/windows-cygwin.yml
index dbbba5f1e1c..ac890430d17 100644
--- a/.github/workflows/windows-cygwin.yml
+++ b/.github/workflows/windows-cygwin.yml
@@ -39,7 +39,7 @@ jobs:
         path C:\tools\cygwin\bin;%GITHUB_WORKSPACE%\build\windows_shared_library
         mkdir build
         cd build
-        bash -c "cmake -DCMAKE_CXX_FLAGS=-Wa,-mbig-obj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_COMPILER_FLAGS=${{ matrix.config.cflags }} .."
+        bash -c "cmake -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_COMPILER_FLAGS=${{ matrix.config.cflags }} .."
         bash -c "make -j4"
         bash -c "ctest . --output-on-failure"
       shell: cmd
diff --git a/.github/workflows/windows-mingw.yml b/.github/workflows/windows-mingw.yml
index 7cfa250757b..a7e1b9b8597 100644
--- a/.github/workflows/windows-mingw.yml
+++ b/.github/workflows/windows-mingw.yml
@@ -36,7 +36,7 @@ jobs:
         path %PATH:C:\Program Files\Git\usr\bin;=%;%GITHUB_WORKSPACE%\build\windows_shared_library
         mkdir build
         cd build
-        cmake -G "MinGW Makefiles" -DCMAKE_CXX_FLAGS=-Wa,-mbig-obj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_COMPILER_FLAGS=${{ matrix.config.cflags }} ..
+        cmake -G "MinGW Makefiles" -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_COMPILER_FLAGS=${{ matrix.config.cflags }} ..
         cmake --build . -j4
         ctest . --output-on-failure
       shell: cmd
diff --git a/.github/workflows/windows-msvc-cuda.yml b/.github/workflows/windows-msvc-cuda.yml
index 28468950af8..616409b9e3e 100644
--- a/.github/workflows/windows-msvc-cuda.yml
+++ b/.github/workflows/windows-msvc-cuda.yml
@@ -46,5 +46,5 @@ jobs:
         $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
         mkdir build
         cd build
-        cmake -DCMAKE_CXX_FLAGS=/bigobj -DGINKGO_BUILD_CUDA=ON -DGINKGO_BUILD_OMP=OFF -DGINKGO_MIXED_PRECISION=${{ matrix.config.mixed }} -DGINKGO_CUDA_ARCHITECTURES=60 ..
+        cmake -DGINKGO_BUILD_CUDA=ON -DGINKGO_BUILD_OMP=OFF -DGINKGO_MIXED_PRECISION=${{ matrix.config.mixed }} -DGINKGO_CUDA_ARCHITECTURES=60 ..
         cmake --build . -j4 --config Release
diff --git a/.github/workflows/windows-msvc-ref.yml b/.github/workflows/windows-msvc-ref.yml
index 833f3692676..9a62ad3ca80 100644
--- a/.github/workflows/windows-msvc-ref.yml
+++ b/.github/workflows/windows-msvc-ref.yml
@@ -35,7 +35,7 @@ jobs:
         $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
         mkdir build
         cd build
-        cmake  -DCMAKE_CXX_FLAGS=/bigobj -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_CXX_FLAGS_DEBUG="/MDd /Zi /Ob1 /Od /RTC1" -DGINKGO_BUILD_CUDA=OFF -DGINKGO_BUILD_OMP=OFF -DGINKGO_MIXED_PRECISION=${{ matrix.config.mixed }} ..
+        cmake  -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_CXX_FLAGS_DEBUG="/MDd /Zi /Ob1 /Od /RTC1" -DGINKGO_BUILD_CUDA=OFF -DGINKGO_BUILD_OMP=OFF -DGINKGO_MIXED_PRECISION=${{ matrix.config.mixed }} ..
         cmake --build . -j4 --config ${{ matrix.config.build_type }}
         ctest . -C ${{ matrix.config.build_type }} --output-on-failure
 
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 18da015f8c6..63677a6c5c3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -18,6 +18,13 @@ include(cmake/hip_path.cmake)
 include(cmake/autodetect_executors.cmake)
 include(cmake/build_type_helpers.cmake)
 
+if (MSVC)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj")
+endif()
+if (MINGW OR CYGWIN)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wa,-mbig-obj")
+endif()
+
 # Ginkgo configuration options
 option(GINKGO_DEVEL_TOOLS "Add development tools to the build system" OFF)
 option(GINKGO_BUILD_TESTS "Generate build files for unit tests" ON)

From 2d5a42bb44f10b1577af8b927409cc2f9cbab8c6 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 10 Mar 2021 22:11:24 +0100
Subject: [PATCH 0697/1008] don't modify CRT linking flags on windows

---
 CMakeLists.txt                   | 24 ++++++++----------------
 cmake/GinkgoConfig.cmake.in      |  4 ----
 cmake/install_helpers.cmake      | 11 -----------
 cmake/windows_helpers.cmake      | 22 ----------------------
 cuda/CMakeLists.txt              |  7 -------
 test_exportbuild/CMakeLists.txt  | 10 ----------
 test_install/CMakeLists.txt      | 17 -----------------
 third_party/gtest/CMakeLists.txt |  2 +-
 8 files changed, 9 insertions(+), 88 deletions(-)
 delete mode 100644 cmake/windows_helpers.cmake

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 63677a6c5c3..a11873fb9d5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -227,22 +227,6 @@ if(GINKGO_BUILD_HIP)
     endif()
 endif()
 
-
-if(MSVC)
-    # This is modified from
-    # https://gitlab.kitware.com/cmake/community/wikis/FAQ#dynamic-replace
-    include(cmake/windows_helpers.cmake)
-    if(BUILD_SHARED_LIBS)
-        ginkgo_switch_to_windows_dynamic("CXX")
-        ginkgo_switch_to_windows_dynamic("C")
-        set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE)
-    else()
-        ginkgo_switch_to_windows_static("CXX")
-        ginkgo_switch_to_windows_static("C")
-        set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS FALSE)
-    endif()
-endif()
-
 # Try to find the third party packages before using our subdirectories
 include(cmake/package_helpers.cmake)
 ginkgo_find_package(GTest "GTest::GTest;GTest::Main" FALSE 1.8.1)
@@ -261,6 +245,14 @@ add_subdirectory(third_party)    # Third-party tools and libraries
 include(cmake/build_helpers.cmake)
 include(cmake/install_helpers.cmake)
 
+if(MSVC)
+    if(BUILD_SHARED_LIBS)
+        set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE)
+    else()
+        set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS FALSE)
+    endif()
+endif()
+
 configure_file(${Ginkgo_SOURCE_DIR}/include/ginkgo/config.hpp.in
     ${Ginkgo_BINARY_DIR}/include/ginkgo/config.hpp @ONLY)
 
diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 8e5de759102..2387d4b8848 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -145,10 +145,6 @@ set(GINKGO_OPENMP_LIBRARIES @OpenMP_CXX_LIBRARIES@)
 
 set(GINKGO_OPENMP_FLAGS "@OpenMP_CXX_FLAGS@")
 
-if (WIN32 OR CYGWIN)
-    include(${CMAKE_CURRENT_LIST_DIR}/windows_helpers.cmake)
-endif()
-
 # NOTE: we do not export benchmarks, examples, tests or devel tools
 #     so `third_party` libraries are currently unneeded.
 
diff --git a/cmake/install_helpers.cmake b/cmake/install_helpers.cmake
index ad49d46b268..077c17c5ebc 100644
--- a/cmake/install_helpers.cmake
+++ b/cmake/install_helpers.cmake
@@ -124,22 +124,11 @@ function(ginkgo_install)
         "${Ginkgo_BINARY_DIR}/GinkgoConfig.cmake"
         INSTALL_DESTINATION "${GINKGO_INSTALL_CONFIG_DIR}"
         )
-    set(HELPERS "windows_helpers.cmake")
-    foreach (helper ${HELPERS})
-        configure_file(${Ginkgo_SOURCE_DIR}/cmake/${helper}
-            ${Ginkgo_BINARY_DIR}/${helper} COPYONLY)
-    endforeach()
     install(FILES
         "${Ginkgo_BINARY_DIR}/GinkgoConfig.cmake"
         "${Ginkgo_BINARY_DIR}/GinkgoConfigVersion.cmake"
         DESTINATION "${GINKGO_INSTALL_CONFIG_DIR}"
         )
-    if (WIN32 OR CYGWIN)
-        install(FILES
-            "${Ginkgo_SOURCE_DIR}/cmake/windows_helpers.cmake"
-            DESTINATION "${GINKGO_INSTALL_CONFIG_DIR}"
-            )
-    endif()
     install(EXPORT Ginkgo
         NAMESPACE Ginkgo::
         FILE GinkgoTargets.cmake
diff --git a/cmake/windows_helpers.cmake b/cmake/windows_helpers.cmake
deleted file mode 100644
index 5f517a555ad..00000000000
--- a/cmake/windows_helpers.cmake
+++ /dev/null
@@ -1,22 +0,0 @@
-function(ginkgo_switch_windows_link lang from to)
-    foreach(flag_var
-        "CMAKE_${lang}_FLAGS" "CMAKE_${lang}_FLAGS_DEBUG" "CMAKE_${lang}_FLAGS_RELEASE"
-        "CMAKE_${lang}_FLAGS_MINSIZEREL" "CMAKE_${lang}_FLAGS_RELWITHDEBINFO"
-        )
-        if(${flag_var} MATCHES "/${from}")
-            string(REGEX REPLACE "/${from}" "/${to}" ${flag_var} "${${flag_var}}")
-        endif(${flag_var} MATCHES "/${from}")
-        if(${flag_var} MATCHES "-${from}")
-            string(REGEX REPLACE "-${from}" "-${to}" ${flag_var} "${${flag_var}}")
-        endif(${flag_var} MATCHES "-${from}")
-        set(${flag_var} "${${flag_var}}" CACHE STRING "" FORCE)
-    endforeach()
-endfunction()
-
-macro(ginkgo_switch_to_windows_static lang)
-    ginkgo_switch_windows_link(${lang} "MD" "MT")
-endmacro()
-
-macro(ginkgo_switch_to_windows_dynamic lang)
-    ginkgo_switch_windows_link(${lang} "MT" "MD")
-endmacro()
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 842f4b75259..8ba183d1635 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -16,13 +16,6 @@ if(MSVC)
     if("${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES}" STREQUAL "")
         set(CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES "${CMAKE_CUDA_ROOT_DIR}/lib/x64")
     endif()
-
-    # This is modified from https://gitlab.kitware.com/cmake/community/wikis/FAQ#dynamic-replace
-    if(BUILD_SHARED_LIBS)
-        ginkgo_switch_to_windows_dynamic("CUDA")
-    else()
-        ginkgo_switch_to_windows_static("CUDA")
-    endif()
 endif()
 
 include(CudaArchitectureSelector)
diff --git a/test_exportbuild/CMakeLists.txt b/test_exportbuild/CMakeLists.txt
index 79b538f437c..52a8d3851cd 100644
--- a/test_exportbuild/CMakeLists.txt
+++ b/test_exportbuild/CMakeLists.txt
@@ -3,16 +3,6 @@ project(GinkgoExportBuildTest LANGUAGES CXX)
 
 find_package(Ginkgo REQUIRED)
 
-if(MSVC)
-    if(GINKGO_BUILD_SHARED_LIBS)
-        ginkgo_switch_to_windows_dynamic("CXX")
-        ginkgo_switch_to_windows_dynamic("C")
-    else()
-        ginkgo_switch_to_windows_static("CXX")
-        ginkgo_switch_to_windows_static("C")
-    endif()
-endif()
-
 # Here, we use test install without any data. We instantiate the
 # interface only.
 add_executable(test_exportbuild ../test_install/test_install.cpp)
diff --git a/test_install/CMakeLists.txt b/test_install/CMakeLists.txt
index cb4b08de706..69797e0b70c 100644
--- a/test_install/CMakeLists.txt
+++ b/test_install/CMakeLists.txt
@@ -18,16 +18,6 @@ configure_file(data/A.mtx data/A.mtx COPYONLY)
 configure_file(data/b.mtx data/b.mtx COPYONLY)
 configure_file(data/x0.mtx data/x0.mtx COPYONLY)
 
-if(MSVC)
-    if(GINKGO_BUILD_SHARED_LIBS)
-        ginkgo_switch_to_windows_dynamic("CXX")
-        ginkgo_switch_to_windows_dynamic("C")
-    else()
-        ginkgo_switch_to_windows_static("CXX")
-        ginkgo_switch_to_windows_static("C")
-    endif()
-endif()
-
 include(CheckLanguage)
 check_language(CUDA)
 
@@ -42,13 +32,6 @@ target_link_libraries(test_install PRIVATE Ginkgo::ginkgo)
 
 if(GINKGO_BUILD_CUDA)
     enable_language(CUDA)
-    if(MSVC)
-        if(GINKGO_BUILD_SHARED_LIBS)
-            ginkgo_switch_to_windows_dynamic("CUDA")
-        else()
-            ginkgo_switch_to_windows_static("CUDA")
-        endif()
-    endif()
     configure_file(test_install.cpp test_install.cu COPYONLY)
     add_executable(test_install_cuda ${CMAKE_CURRENT_BINARY_DIR}/test_install.cu)
     target_compile_options(test_install_cuda
diff --git a/third_party/gtest/CMakeLists.txt b/third_party/gtest/CMakeLists.txt
index 31da9c5eda3..691fd2fa5f6 100644
--- a/third_party/gtest/CMakeLists.txt
+++ b/third_party/gtest/CMakeLists.txt
@@ -5,7 +5,7 @@ if(MSVC)
         "https://github.com/google/googletest.git"
         "6a7ed316a5cdc07b6d26362c90770787513822d4"
         # Work around the linking errors when compiling gtest with CUDA
-        "-Dgtest_disable_pthreads=ON" "-Dgtest_force_shared_crt=${BUILD_SHARED_LIBS}")
+        "-Dgtest_disable_pthreads=ON" "-Dgtest_force_shared_crt=ON")
 else()
     ginkgo_load_git_package(gtest_external
         "https://github.com/google/googletest.git"

From bc8c3a68dbbb0dce20a9793c51b260e8f6a3ae42 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 11 Mar 2021 00:03:53 +0100
Subject: [PATCH 0698/1008] store DLLs in runtime instead of library path

---
 .github/workflows/windows-cygwin.yml   | 2 +-
 .github/workflows/windows-mingw.yml    | 2 +-
 .github/workflows/windows-msvc-ref.yml | 2 +-
 cmake/install_helpers.cmake            | 3 ++-
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/windows-cygwin.yml b/.github/workflows/windows-cygwin.yml
index ac890430d17..2cb1291efb7 100644
--- a/.github/workflows/windows-cygwin.yml
+++ b/.github/workflows/windows-cygwin.yml
@@ -49,5 +49,5 @@ jobs:
         path C:\tools\cygwin\bin
         cd build
         bash -c "make install"
-        bash -c "export PATH=/usr/local/lib:$PATH && make test_install"
+        bash -c "export PATH=/usr/local/bin:$PATH && make test_install"
       shell: cmd
diff --git a/.github/workflows/windows-mingw.yml b/.github/workflows/windows-mingw.yml
index a7e1b9b8597..5b79da0b3be 100644
--- a/.github/workflows/windows-mingw.yml
+++ b/.github/workflows/windows-mingw.yml
@@ -44,7 +44,7 @@ jobs:
     - name: install
       run: |
         set PATH=%PATH:C:\Program Files\Git\bin;=%
-        set PATH=%PATH:C:\Program Files\Git\usr\bin;=%;C:\Program Files (x86)\Ginkgo\lib
+        set PATH=%PATH:C:\Program Files\Git\usr\bin;=%;C:\Program Files (x86)\Ginkgo\bin
         cd build
         cmake --install .
         cmake --build . --target test_install
diff --git a/.github/workflows/windows-msvc-ref.yml b/.github/workflows/windows-msvc-ref.yml
index 9a62ad3ca80..edf79726f9f 100644
--- a/.github/workflows/windows-msvc-ref.yml
+++ b/.github/workflows/windows-msvc-ref.yml
@@ -41,7 +41,7 @@ jobs:
 
     - name: install
       run: |
-        $env:PATH="$env:PATH;C:\Program Files (x86)\Ginkgo\lib"
+        $env:PATH="$env:PATH;C:\Program Files (x86)\Ginkgo\bin"
         cd build
         cmake --install . --config ${{ matrix.config.build_type }}
         cmake --build . --target test_install --config ${{ matrix.config.build_type }}
diff --git a/cmake/install_helpers.cmake b/cmake/install_helpers.cmake
index 077c17c5ebc..3991796a147 100644
--- a/cmake/install_helpers.cmake
+++ b/cmake/install_helpers.cmake
@@ -4,6 +4,7 @@ include(GNUInstallDirs)
 
 set(GINKGO_INSTALL_INCLUDE_DIR "${CMAKE_INSTALL_INCLUDEDIR}")
 set(GINKGO_INSTALL_LIBRARY_DIR "${CMAKE_INSTALL_LIBDIR}")
+set(GINKGO_INSTALL_RUNTIME_DIR "${CMAKE_INSTALL_BINDIR}")
 set(GINKGO_INSTALL_PKGCONFIG_DIR "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 set(GINKGO_INSTALL_CONFIG_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Ginkgo")
 set(GINKGO_INSTALL_MODULE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/Ginkgo/Modules")
@@ -56,7 +57,7 @@ function(ginkgo_install_library name)
             EXPORT Ginkgo
             LIBRARY DESTINATION "${GINKGO_INSTALL_LIBRARY_DIR}"
             ARCHIVE DESTINATION "${GINKGO_INSTALL_LIBRARY_DIR}"
-            RUNTIME DESTINATION "${GINKGO_INSTALL_LIBRARY_DIR}"
+            RUNTIME DESTINATION "${GINKGO_INSTALL_RUNTIME_DIR}"
             )
     else ()
         # install .so and .a files

From ff7c5b5f9b0ee80ba5115d833417fa08640caebf Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 11 Mar 2021 00:49:45 +0100
Subject: [PATCH 0699/1008] store output libraries in a single path

---
 .github/workflows/windows-cygwin.yml    |  2 +-
 .github/workflows/windows-mingw.yml     |  2 +-
 .github/workflows/windows-msvc-cuda.yml |  1 -
 .github/workflows/windows-msvc-ref.yml  |  1 -
 CMakeLists.txt                          | 15 +------
 INSTALL.md                              | 45 ++-----------------
 README.md                               |  6 +--
 cmake/build_helpers.cmake               | 59 +++----------------------
 cmake/create_test.cmake                 | 25 ++++++++---
 cmake/get_info.cmake                    |  4 +-
 10 files changed, 34 insertions(+), 126 deletions(-)

diff --git a/.github/workflows/windows-cygwin.yml b/.github/workflows/windows-cygwin.yml
index 2cb1291efb7..317e3608253 100644
--- a/.github/workflows/windows-cygwin.yml
+++ b/.github/workflows/windows-cygwin.yml
@@ -36,7 +36,7 @@ jobs:
 
     - name: configure
       run: |
-        path C:\tools\cygwin\bin;%GITHUB_WORKSPACE%\build\windows_shared_library
+        path C:\tools\cygwin\bin
         mkdir build
         cd build
         bash -c "cmake -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_COMPILER_FLAGS=${{ matrix.config.cflags }} .."
diff --git a/.github/workflows/windows-mingw.yml b/.github/workflows/windows-mingw.yml
index 5b79da0b3be..beeeb34d840 100644
--- a/.github/workflows/windows-mingw.yml
+++ b/.github/workflows/windows-mingw.yml
@@ -33,7 +33,7 @@ jobs:
         bcdedit /set IncreaseUserVa 3072
         editbin /LARGEADDRESSAWARE "C:\Program Files\Git\mingw64\bin\cc1plus.exe"
         path %PATH:C:\Program Files\Git\bin;=%
-        path %PATH:C:\Program Files\Git\usr\bin;=%;%GITHUB_WORKSPACE%\build\windows_shared_library
+        path %PATH:C:\Program Files\Git\usr\bin;=%
         mkdir build
         cd build
         cmake -G "MinGW Makefiles" -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_COMPILER_FLAGS=${{ matrix.config.cflags }} ..
diff --git a/.github/workflows/windows-msvc-cuda.yml b/.github/workflows/windows-msvc-cuda.yml
index 616409b9e3e..fba49eb6adb 100644
--- a/.github/workflows/windows-msvc-cuda.yml
+++ b/.github/workflows/windows-msvc-cuda.yml
@@ -43,7 +43,6 @@ jobs:
         $env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.."
         Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
         refreshenv
-        $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
         mkdir build
         cd build
         cmake -DGINKGO_BUILD_CUDA=ON -DGINKGO_BUILD_OMP=OFF -DGINKGO_MIXED_PRECISION=${{ matrix.config.mixed }} -DGINKGO_CUDA_ARCHITECTURES=60 ..
diff --git a/.github/workflows/windows-msvc-ref.yml b/.github/workflows/windows-msvc-ref.yml
index edf79726f9f..c2df41390c1 100644
--- a/.github/workflows/windows-msvc-ref.yml
+++ b/.github/workflows/windows-msvc-ref.yml
@@ -32,7 +32,6 @@ jobs:
 
     - name: configure
       run: |
-        $env:PATH="$env:PATH;$pwd\build\windows_shared_library"
         mkdir build
         cd build
         cmake  -DBUILD_SHARED_LIBS=${{ matrix.config.shared }} -DCMAKE_CXX_FLAGS_DEBUG="/MDd /Zi /Ob1 /Od /RTC1" -DGINKGO_BUILD_CUDA=OFF -DGINKGO_BUILD_OMP=OFF -DGINKGO_MIXED_PRECISION=${{ matrix.config.mixed }} ..
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a11873fb9d5..7b440d2666e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -25,6 +25,8 @@ if (MINGW OR CYGWIN)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wa,-mbig-obj")
 endif()
 
+set(GINKGO_LIBRARY_PATH "${PROJECT_BINARY_DIR}/lib")
+
 # Ginkgo configuration options
 option(GINKGO_DEVEL_TOOLS "Add development tools to the build system" OFF)
 option(GINKGO_BUILD_TESTS "Generate build files for unit tests" ON)
@@ -114,19 +116,6 @@ if(GINKGO_BENCHMARK_ENABLE_TUNING)
     set(GINKGO_BUILD_EXAMPLES OFF)
 endif()
 
-if(BUILD_SHARED_LIBS AND (WIN32 OR CYGWIN) AND (GINKGO_BUILD_TESTS OR GINKGO_BUILD_EXAMPLES OR GINKGO_BUILD_BENCHMARKS))
-    # Change shared libraries output only if this build has executable program
-    # with shared libraries.
-    set(GINKGO_CHANGED_SHARED_LIBRARY TRUE)
-    option(GINKGO_CHECK_PATH "Tell Ginkgo to check if the environment variable PATH is available for this build." ON)
-    set(GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH "windows_shared_library" CACHE STRING
-        "Set Ginkgo's shared library relative path in windows. Current default is `windows_shared_library`. \
-        This absolute path ${PROJECT_BINARY_DIR}/GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH must be in the environment variable PATH.")
-    set(GINKGO_WINDOWS_SHARED_LIBRARY_PATH ${PROJECT_BINARY_DIR}/${GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH})
-else()
-    set(GINKGO_CHANGED_SHARED_LIBRARY FALSE)
-endif()
-
 if(GINKGO_BUILD_TESTS AND (GINKGO_BUILD_CUDA OR GINKGO_BUILD_OMP OR GINKGO_BUILD_HIP OR GINKGO_BUILD_DPCPP))
     message(STATUS "GINKGO_BUILD_TESTS is ON, enabling GINKGO_BUILD_REFERENCE")
     set(GINKGO_BUILD_REFERENCE ON CACHE BOOL "Compile reference CPU kernels" FORCE)
diff --git a/INSTALL.md b/INSTALL.md
index 4fcc60d9ae4..0f558d155e3 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -116,14 +116,6 @@ Ginkgo adds the following additional switches to control what is being built:
     this option see the
     [`ARCHITECTURES` specification list](https://github.com/ginkgo-project/CudaArchitectureSelector/blob/master/CudaArchitectureSelector.cmake#L58)
     section in the documentation of the CudaArchitectureSelector CMake module.
-* `-DGINKGO_WINDOWS_SHARED_LIBRARY_RELPATH=<path>` where <path> is a relative
-    path built with `PROJECT_BINARY_DIR`. Users must add the absolute path
-    (`PROJECT_BINARY_DIR`/`GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH`) into the
-    environment variable PATH when building shared libraries and executable
-    program, default is `windows_shared_library`.
-* `-DGINKGO_CHECK_PATH={ON, OFF}` checks if the environment variable PATH is valid.
-    It is checked only when building shared libraries and executable program,
-    default is `ON`.
 
 For example, to build everything (in debug mode), use:
 
@@ -139,50 +131,21 @@ generators. Other CMake generators are untested.
 
 ### Building Ginkgo in Windows
 Depending on the configuration settings, some manual work might be required:
-* Build Ginkgo as shared library:
-  Add `PROJECT_BINARY_DIR/GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH` into the environment variable `PATH`.
-  `GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH` is `windows_shared_library` by default. More Details are available in the [Installation page](./INSTALL.md).
-  * cmd: `set PATH=<PROJECT_BINARY_DIR/GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH>;%PATH%`
-  * powershell: `$env:PATH="<PROJECT_BINARY_DIR/GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH>;$env:PATH"`
-
-  CMake will give the following error message if the path is not correct.
-  ```
-  Did not find this build in the environment variable PATH. Please add <path> into the environment variable PATH.
-  ```
-  where `<path>` is the needed `<PROJECT_BINARY_DIR/GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH>`.
 * Build Ginkgo with Debug mode:
   Some Debug build specific issues can appear depending on the machine and environment. The known issues are the following:
-  1. `bigobj` issue: encountering  `too many sections` needs the compilation flags `/bigobj` or `-Wa,-mbig-obj`
-  2. `ld` issue: encountering  `ld: error: export ordinal too large` needs the compilation flag `-O1`
+  1. `ld` issue: encountering  `ld: error: export ordinal too large` needs the compilation flag `-O1`
 
   The following are the details for different environments:
   * _Microsoft Visual Studio_:
-    1. `bigobj` issue
-      * `cmake -DCMAKE_CXX_FLAGS=/bigobj <other parameters> <source_folder>` which might overwrite the default settings.
-      * add `/bigobj` into the environment variable `CXXFLAGS` (only available in the first cmake configuration)
-        * cmd: `set CXXFLAGS=/bigobj`
-        * powershell: `$env:CXXFLAGS=/bigobj`
-    2. `ld` issue (_Microsoft Visual Studio_ does not have this issue)
+    1. `ld` issue (_Microsoft Visual Studio_ does not have this issue)
   * _Cygwin_:
-    1. `bigobj` issue
-      * add `-Wa,-mbig-obj -O1` into the environment variable `CXXFLAGS` (only available in the first cmake configuration)
-        * `export CXXFLAGS="-Wa,-mbig-obj -O1"`
-      * `cmake -DCMAKE_CXX_FLAGS=-Wa,-mbig-obj <other parameters> <source_folder>`, which might overwrite the default settings.
-    2. `ld` issue (If building Ginkgo as static library, this is not needed)
+    1. `ld` issue (If building Ginkgo as static library, this is not needed)
       * `cmake -DGINKGO_COMPILER_FLAGS="-Wpedantic -O1" <other parameters> <source_folder>` (`GINKGO_COMPILER_FLAGS` is `-Wpedantic` by default)
       * add `-O1` in the environement variable `CXX_FLAGS` or `CMAKE_CXX_FLAGS`
   * _MinGW_:
-    1. `bigobj` issue
-      * add `-Wa,-mbig-obj -O1` into the environment variable `CXXFLAGS` (only available in the first cmake configuration)
-        * cmd: `set CXXFLAGS="-Wa,-mbig-obj"`
-        * powershell: `$env:CXXFLAGS="-Wa,-mbig-obj"`
-      * `cmake -DCMAKE_CXX_FLAGS=-Wa,-mbig-obj <other parameters> <source_folder>`, which might overwrite the default settings.
-    2. `ld` issue (If building Ginkgo as static library, this is not needed)
+    1. `ld` issue (If building Ginkgo as static library, this is not needed)
       * `cmake -DGINKGO_COMPILER_FLAGS="-Wpedantic -O1" <other parameters> <source_folder>` (`GINKGO_COMPILER_FLAGS` is `-Wpedantic` by default)
       * add `-O1` in the environement variable `CXX_FLAGS` or `CMAKE_CXX_FLAGS`
-* Possible issue when switching static/shared of Ginkgo with MSVC in the same build directory:\
-  If an issue occurs from mixing MD/MT runtime library when enabling `GINKGO_BUILD_BENCHMARKS`, it means the third-party flags are not updated correctly.
-  To update the third party flags, turn off `GINKGO_SKIP_DEPENDENCY_UPDATE` (`-DGINKGO_SKIP_DEPENDENCY_UPDATE=OFF`).
 * Build Ginkgo in _MinGW_:\
   If encountering the issue `cc1plus.exe: out of memory allocating 65536 bytes`, please follow the workaround in
   [reference](https://www.intel.com/content/www/us/en/programmable/support/support-resources/knowledge-base/embedded/2016/cc1plus-exe--out-of-memory-allocating-65536-bytes.html),
diff --git a/README.md b/README.md
index 36f06d4f4d3..9f3c93ba00c 100644
--- a/README.md
+++ b/README.md
@@ -89,13 +89,9 @@ The Ginkgo OMP module has the following __additional__ requirements:
 *  _MinGW_ or _Cygwin_
 
 Depending on the configuration settings, some manual work might be required. More details are availble in [windows section in INSTALL.md](INSTALL.md#building-ginkgo-in-windows):
-* Build Ginkgo as shared library:
-  Add `PROJECT_BINARY_DIR/GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH` into the environment variable `PATH`.
-  `GINKGO_WINDOWS_SHARED_LIBRARY_RELPATH` is `windows_shared_library` by default.
 * Build Ginkgo with Debug mode:
   Some Debug build specific issues can appear depending on the machine and environment. The known issues are the following:
-  1. `bigobj` issue: encountering  `too many sections` needs the compilation flags `\bigobj` or `-Wa,-mbig-obj`
-  2. `ld` issue: encountering  `ld: error: export ordinal too large` needs the compilation flag `-O1`
+  1. `ld` issue: encountering  `ld: error: export ordinal too large` needs the compilation flag `-O1`
 * Build Ginkgo in _MinGW_:
   If encountering the issue `cc1plus.exe: out of memory allocating 65536 bytes`, please follow the workaround in
   [reference](https://www.intel.com/content/www/us/en/programmable/support/support-resources/knowledge-base/embedded/2016/cc1plus-exe--out-of-memory-allocating-65536-bytes.html),
diff --git a/cmake/build_helpers.cmake b/cmake/build_helpers.cmake
index 34689ecefc5..456398a0d1c 100644
--- a/cmake/build_helpers.cmake
+++ b/cmake/build_helpers.cmake
@@ -26,26 +26,11 @@ function(ginkgo_compile_features name)
     # Set an appropriate SONAME
     set_property(TARGET "${name}" PROPERTY
         SOVERSION "${Ginkgo_VERSION}")
-    if(GINKGO_CHANGED_SHARED_LIBRARY)
-        # Put all shared libraries and corresponding imported libraries into the specified path
-        set_property(TARGET "${name}" PROPERTY
-            RUNTIME_OUTPUT_DIRECTORY "${GINKGO_WINDOWS_SHARED_LIBRARY_PATH}")
-        set_property(TARGET "${name}" PROPERTY
-            ARCHIVE_OUTPUT_DIRECTORY "${GINKGO_WINDOWS_SHARED_LIBRARY_PATH}")
-        if(MSVC)
-            # MSVC would create subfolder according to build_type. Ginkgo forces the output be the same whatever build_type is.
-            foreach(CONFIG ${CMAKE_CONFIGURATION_TYPES})
-                string(TOUPPER ${CONFIG} CONFIG )
-                set_property(TARGET "${name}" PROPERTY
-                    RUNTIME_OUTPUT_DIRECTORY_${CONFIG} "${GINKGO_WINDOWS_SHARED_LIBRARY_PATH}")
-                set_property(TARGET "${name}" PROPERTY
-                    ARCHIVE_OUTPUT_DIRECTORY_${CONFIG} "${GINKGO_WINDOWS_SHARED_LIBRARY_PATH}")
-            endforeach()
-        endif()
-        if(GINKGO_CHECK_PATH)
-            ginkgo_check_shared_library("${CMAKE_SHARED_LIBRARY_PREFIX}${name}${CMAKE_SHARED_LIBRARY_SUFFIX}")
-        endif()
-    endif()
+    # Put all shared libraries and corresponding imported libraries into the specified path
+    set_property(TARGET "${name}" PROPERTY
+        RUNTIME_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}")
+    set_property(TARGET "${name}" PROPERTY
+        ARCHIVE_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}")
 
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries("${name}" PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
@@ -104,40 +89,6 @@ function(ginkgo_check_headers target)
     endif()
 endfunction()
 
-function(ginkgo_check_shared_library name)
-    # Cygwin uses : not ; to split path
-    if(CYGWIN)
-        string(REPLACE ":" ";" ENV_PATH "$ENV{PATH}")
-    else()
-        set(ENV_PATH "$ENV{PATH}")
-    endif()
-    set(PATH_LIST ${ENV_PATH})
-    set(PASSED_TEST FALSE)
-    foreach(ITEM IN LISTS PATH_LIST)
-        string(REPLACE "\\" "/" ITEM "${ITEM}")
-        if("${ITEM}" STREQUAL "${GINKGO_WINDOWS_SHARED_LIBRARY_PATH}")
-            set(PASSED_TEST TRUE)
-            break()
-        else()
-            # If any path before this build, the path must not contain the ginkgo shared library
-            find_file(EXISTING_DLL "${name}" PATHS "${ITEM}" NO_DEFAULT_PATH)
-            if(NOT "${EXISTING_DLL}" STREQUAL "EXISTING_DLL-NOTFOUND")
-                # clean the EXISTING_DLL before termination
-                unset(EXISTING_DLL CACHE)
-                message(FATAL_ERROR "Detect ${name} in ${ITEM} eariler than this build. "
-                    "Please add ${GINKGO_WINDOWS_SHARED_LIBRARY_PATH} before other ginkgo path.")
-            endif()
-            # do not keep this variable in cache
-            unset(EXISTING_DLL CACHE)
-        endif()
-    endforeach()
-    if(NOT PASSED_TEST)
-        # Did not find this build in the environment variable PATH
-        message(FATAL_ERROR "Did not find this build in the environment variable PATH. "
-            "Please add ${GINKGO_WINDOWS_SHARED_LIBRARY_PATH} into the environment variable PATH.")
-    endif()
-endfunction()
-
 macro(ginkgo_modify_flags name)
     # add escape before "
     # the result var is ${name}_MODIFY
diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index d7a420c2066..c5285b0c10a 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -21,7 +21,9 @@ function(ginkgo_create_test test_name)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
     target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
-    add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
+    add_test(NAME ${REL_BINARY_DIR}/${test_name}
+        COMMAND ${TEST_TARGET_NAME}
+        WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
 endfunction(ginkgo_create_test)
 
 function(ginkgo_create_dpcpp_test test_name)
@@ -47,7 +49,9 @@ function(ginkgo_create_dpcpp_test test_name)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
     target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
-    add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
+    add_test(NAME ${REL_BINARY_DIR}/${test_name}
+        COMMAND ${TEST_TARGET_NAME}
+        WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
 endfunction(ginkgo_create_dpcpp_test)
 
 function(ginkgo_create_thread_test test_name)
@@ -73,7 +77,9 @@ function(ginkgo_create_thread_test test_name)
     endif()
     target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest
         Threads::Threads ${ARGN})
-    add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
+        add_test(NAME ${REL_BINARY_DIR}/${test_name}
+            COMMAND ${TEST_TARGET_NAME}
+            WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
 endfunction(ginkgo_create_thread_test)
 
 function(ginkgo_create_test_cpp_cuda_header test_name)
@@ -97,7 +103,9 @@ function(ginkgo_create_test_cpp_cuda_header test_name)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
     target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
-    add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
+    add_test(NAME ${REL_BINARY_DIR}/${test_name}
+        COMMAND ${TEST_TARGET_NAME}
+        WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
 endfunction(ginkgo_create_test_cpp_cuda_header)
 
 function(ginkgo_create_cuda_test test_name)
@@ -121,10 +129,11 @@ function(ginkgo_create_cuda_test test_name)
         target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
     target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
-    add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
+    add_test(NAME ${REL_BINARY_DIR}/${test_name}
+        COMMAND ${TEST_TARGET_NAME}
+        WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
 endfunction(ginkgo_create_cuda_test)
 
-
 function(ginkgo_create_hip_test test_name)
     file(RELATIVE_PATH REL_BINARY_DIR
         ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
@@ -180,5 +189,7 @@ function(ginkgo_create_hip_test test_name)
     endif()
 
     target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
-    add_test(NAME ${REL_BINARY_DIR}/${test_name} COMMAND ${TEST_TARGET_NAME})
+    add_test(NAME ${REL_BINARY_DIR}/${test_name}
+        COMMAND ${TEST_TARGET_NAME}
+        WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
 endfunction(ginkgo_create_hip_test)
diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
index bae1e864a02..99c387d30d3 100644
--- a/cmake/get_info.cmake
+++ b/cmake/get_info.cmake
@@ -171,10 +171,10 @@ ginkgo_print_generic_header(${minimal_log} "  Developer Tools:")
 ginkgo_print_generic_header(${detailed_log} "  Developer Tools:")
 ginkgo_print_foreach_variable(${minimal_log}
         "GINKGO_DEVEL_TOOLS;GINKGO_WITH_CLANG_TIDY;GINKGO_WITH_IWYU"
-        "GINKGO_CHECK_CIRCULAR_DEPS;GINKGO_CHECK_PATH;GINKGO_WITH_CCACHE")
+        "GINKGO_CHECK_CIRCULAR_DEPS;GINKGO_WITH_CCACHE")
 ginkgo_print_foreach_variable(${detailed_log}
         "GINKGO_DEVEL_TOOLS;GINKGO_WITH_CLANG_TIDY;GINKGO_WITH_IWYU"
-        "GINKGO_CHECK_CIRCULAR_DEPS;GINKGO_CHECK_PATH;GINKGO_WITH_CCACHE")
+        "GINKGO_CHECK_CIRCULAR_DEPS;GINKGO_WITH_CCACHE")
 ginkgo_print_module_footer(${detailed_log} "  CCACHE:")
 ginkgo_print_variable(${detailed_log} "CCACHE_PROGRAM")
 ginkgo_print_env_variable(${detailed_log} "CCACHE_DIR")

From 5900044bd919ff2c0adce16ec59196509330aea8 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 13 Apr 2021 11:59:21 +0200
Subject: [PATCH 0700/1008] update examples to use Ginkgo 1.4.0

---
 examples/adaptiveprecision-blockjacobi/CMakeLists.txt | 2 +-
 examples/cb-gmres/CMakeLists.txt                      | 2 +-
 examples/custom-logger/CMakeLists.txt                 | 2 +-
 examples/custom-matrix-format/CMakeLists.txt          | 2 +-
 examples/custom-stopping-criterion/CMakeLists.txt     | 2 +-
 examples/ginkgo-overhead/CMakeLists.txt               | 2 +-
 examples/ginkgo-ranges/CMakeLists.txt                 | 2 +-
 examples/heat-equation/CMakeLists.txt                 | 2 +-
 examples/ilu-preconditioned-solver/CMakeLists.txt     | 2 +-
 examples/inverse-iteration/CMakeLists.txt             | 2 +-
 examples/ir-ilu-preconditioned-solver/CMakeLists.txt  | 2 +-
 examples/iterative-refinement/CMakeLists.txt          | 2 +-
 examples/minimal-cuda-solver/CMakeLists.txt           | 2 +-
 examples/mixed-precision-ir/CMakeLists.txt            | 2 +-
 examples/nine-pt-stencil-solver/CMakeLists.txt        | 2 +-
 examples/papi-logging/CMakeLists.txt                  | 2 +-
 examples/par-ilu-convergence/CMakeLists.txt           | 2 +-
 examples/performance-debugging/CMakeLists.txt         | 2 +-
 examples/poisson-solver/CMakeLists.txt                | 2 +-
 examples/preconditioned-solver/CMakeLists.txt         | 2 +-
 examples/preconditioner-export/CMakeLists.txt         | 2 +-
 examples/simple-solver-logging/CMakeLists.txt         | 2 +-
 examples/simple-solver/CMakeLists.txt                 | 2 +-
 examples/three-pt-stencil-solver/CMakeLists.txt       | 2 +-
 24 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/examples/adaptiveprecision-blockjacobi/CMakeLists.txt b/examples/adaptiveprecision-blockjacobi/CMakeLists.txt
index b3d83725d0f..c3cb3183d22 100644
--- a/examples/adaptiveprecision-blockjacobi/CMakeLists.txt
+++ b/examples/adaptiveprecision-blockjacobi/CMakeLists.txt
@@ -3,7 +3,7 @@ project(adaptiveprecision-blockjacobi)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(adaptiveprecision-blockjacobi adaptiveprecision-blockjacobi.cpp)
diff --git a/examples/cb-gmres/CMakeLists.txt b/examples/cb-gmres/CMakeLists.txt
index 0234e4128ac..dc9b4befdf5 100644
--- a/examples/cb-gmres/CMakeLists.txt
+++ b/examples/cb-gmres/CMakeLists.txt
@@ -3,7 +3,7 @@ project(cb-gmres)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(cb-gmres cb-gmres.cpp)
diff --git a/examples/custom-logger/CMakeLists.txt b/examples/custom-logger/CMakeLists.txt
index cf6fb7f4521..1bc89d54e63 100644
--- a/examples/custom-logger/CMakeLists.txt
+++ b/examples/custom-logger/CMakeLists.txt
@@ -3,7 +3,7 @@ project(custom-logger)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(custom-logger custom-logger.cpp)
diff --git a/examples/custom-matrix-format/CMakeLists.txt b/examples/custom-matrix-format/CMakeLists.txt
index c008a86f7b0..c6de4475eb9 100644
--- a/examples/custom-matrix-format/CMakeLists.txt
+++ b/examples/custom-matrix-format/CMakeLists.txt
@@ -3,7 +3,7 @@ project(custom-matrix-format CXX CUDA)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 set(CMAKE_CUDA_STANDARD 14)
diff --git a/examples/custom-stopping-criterion/CMakeLists.txt b/examples/custom-stopping-criterion/CMakeLists.txt
index 7d1db25a116..7009e575bc5 100644
--- a/examples/custom-stopping-criterion/CMakeLists.txt
+++ b/examples/custom-stopping-criterion/CMakeLists.txt
@@ -3,7 +3,7 @@ project(custom-stopping-criterion)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
diff --git a/examples/ginkgo-overhead/CMakeLists.txt b/examples/ginkgo-overhead/CMakeLists.txt
index 16328502185..82512ea0567 100644
--- a/examples/ginkgo-overhead/CMakeLists.txt
+++ b/examples/ginkgo-overhead/CMakeLists.txt
@@ -3,7 +3,7 @@ project(ginkgo-overhead)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(ginkgo-overhead ginkgo-overhead.cpp)
diff --git a/examples/ginkgo-ranges/CMakeLists.txt b/examples/ginkgo-ranges/CMakeLists.txt
index 943d8c2aef2..ebefcf103a3 100644
--- a/examples/ginkgo-ranges/CMakeLists.txt
+++ b/examples/ginkgo-ranges/CMakeLists.txt
@@ -3,7 +3,7 @@ project(ginkgo-ranges)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 add_executable(ginkgo-ranges ginkgo-ranges.cpp)
 target_link_libraries(ginkgo-ranges Ginkgo::ginkgo)
diff --git a/examples/heat-equation/CMakeLists.txt b/examples/heat-equation/CMakeLists.txt
index 9de0f9ea83a..c13a69e8fb7 100644
--- a/examples/heat-equation/CMakeLists.txt
+++ b/examples/heat-equation/CMakeLists.txt
@@ -3,7 +3,7 @@ project(heat-equation)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 find_package(OpenCV REQUIRED)
 
diff --git a/examples/ilu-preconditioned-solver/CMakeLists.txt b/examples/ilu-preconditioned-solver/CMakeLists.txt
index 6e971588215..eb35d1f4254 100644
--- a/examples/ilu-preconditioned-solver/CMakeLists.txt
+++ b/examples/ilu-preconditioned-solver/CMakeLists.txt
@@ -3,7 +3,7 @@ project(ilu-preconditioned-solver)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(ilu-preconditioned-solver ilu-preconditioned-solver.cpp)
diff --git a/examples/inverse-iteration/CMakeLists.txt b/examples/inverse-iteration/CMakeLists.txt
index 03aa016b340..a9ec2edfcca 100644
--- a/examples/inverse-iteration/CMakeLists.txt
+++ b/examples/inverse-iteration/CMakeLists.txt
@@ -3,7 +3,7 @@ project(inverse-iteration)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(inverse-iteration inverse-iteration.cpp)
diff --git a/examples/ir-ilu-preconditioned-solver/CMakeLists.txt b/examples/ir-ilu-preconditioned-solver/CMakeLists.txt
index 76a11d405f1..00d7af61a04 100644
--- a/examples/ir-ilu-preconditioned-solver/CMakeLists.txt
+++ b/examples/ir-ilu-preconditioned-solver/CMakeLists.txt
@@ -3,7 +3,7 @@ project(ir-ilu-preconditioned-solver)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(ir-ilu-preconditioned-solver ir-ilu-preconditioned-solver.cpp)
diff --git a/examples/iterative-refinement/CMakeLists.txt b/examples/iterative-refinement/CMakeLists.txt
index 47c3942ae43..112f70634cf 100644
--- a/examples/iterative-refinement/CMakeLists.txt
+++ b/examples/iterative-refinement/CMakeLists.txt
@@ -3,7 +3,7 @@ project(iterative-refinement)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(iterative-refinement iterative-refinement.cpp)
diff --git a/examples/minimal-cuda-solver/CMakeLists.txt b/examples/minimal-cuda-solver/CMakeLists.txt
index ffc9274e16b..cd04ddb68de 100644
--- a/examples/minimal-cuda-solver/CMakeLists.txt
+++ b/examples/minimal-cuda-solver/CMakeLists.txt
@@ -3,7 +3,7 @@ project(minimal-cuda-solver)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(minimal-cuda-solver minimal-cuda-solver.cpp)
diff --git a/examples/mixed-precision-ir/CMakeLists.txt b/examples/mixed-precision-ir/CMakeLists.txt
index 8c23e367a82..242b525d2af 100644
--- a/examples/mixed-precision-ir/CMakeLists.txt
+++ b/examples/mixed-precision-ir/CMakeLists.txt
@@ -3,7 +3,7 @@ project(mixed-precision-ir)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(mixed-precision-ir mixed-precision-ir.cpp)
diff --git a/examples/nine-pt-stencil-solver/CMakeLists.txt b/examples/nine-pt-stencil-solver/CMakeLists.txt
index 510f0e85f07..b9a3fc8421e 100644
--- a/examples/nine-pt-stencil-solver/CMakeLists.txt
+++ b/examples/nine-pt-stencil-solver/CMakeLists.txt
@@ -3,7 +3,7 @@ project(nine-pt-stencil-solver)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(nine-pt-stencil-solver nine-pt-stencil-solver.cpp)
diff --git a/examples/papi-logging/CMakeLists.txt b/examples/papi-logging/CMakeLists.txt
index e3836ecbe3b..a8f0f8a260c 100644
--- a/examples/papi-logging/CMakeLists.txt
+++ b/examples/papi-logging/CMakeLists.txt
@@ -3,7 +3,7 @@ project(papi-logging)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(papi-logging papi-logging.cpp)
diff --git a/examples/par-ilu-convergence/CMakeLists.txt b/examples/par-ilu-convergence/CMakeLists.txt
index ca2d88db12c..b5e0c8492e5 100644
--- a/examples/par-ilu-convergence/CMakeLists.txt
+++ b/examples/par-ilu-convergence/CMakeLists.txt
@@ -3,7 +3,7 @@ project(par-ilu-convergence)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(par-ilu-convergence par-ilu-convergence.cpp)
diff --git a/examples/performance-debugging/CMakeLists.txt b/examples/performance-debugging/CMakeLists.txt
index 6161b858d22..97cf82cb0ff 100644
--- a/examples/performance-debugging/CMakeLists.txt
+++ b/examples/performance-debugging/CMakeLists.txt
@@ -3,7 +3,7 @@ project(performance-debugging)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(performance-debugging performance-debugging.cpp)
diff --git a/examples/poisson-solver/CMakeLists.txt b/examples/poisson-solver/CMakeLists.txt
index 6c20c0fe087..fcbc831a6e5 100644
--- a/examples/poisson-solver/CMakeLists.txt
+++ b/examples/poisson-solver/CMakeLists.txt
@@ -3,7 +3,7 @@ project(poisson-solver)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(poisson-solver poisson-solver.cpp)
diff --git a/examples/preconditioned-solver/CMakeLists.txt b/examples/preconditioned-solver/CMakeLists.txt
index 002f4ae7218..bd2fcd3cb60 100644
--- a/examples/preconditioned-solver/CMakeLists.txt
+++ b/examples/preconditioned-solver/CMakeLists.txt
@@ -3,7 +3,7 @@ project(preconditioned-solver)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 add_executable(preconditioned-solver preconditioned-solver.cpp)
 target_link_libraries(preconditioned-solver Ginkgo::ginkgo)
diff --git a/examples/preconditioner-export/CMakeLists.txt b/examples/preconditioner-export/CMakeLists.txt
index 81969576dcb..8195bf8a969 100644
--- a/examples/preconditioner-export/CMakeLists.txt
+++ b/examples/preconditioner-export/CMakeLists.txt
@@ -3,7 +3,7 @@ project(preconditioner-export)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(preconditioner-export preconditioner-export.cpp)
diff --git a/examples/simple-solver-logging/CMakeLists.txt b/examples/simple-solver-logging/CMakeLists.txt
index 1927690945e..1d5e75912e0 100644
--- a/examples/simple-solver-logging/CMakeLists.txt
+++ b/examples/simple-solver-logging/CMakeLists.txt
@@ -3,7 +3,7 @@ project(simple-solver-logging)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(simple-solver-logging simple-solver-logging.cpp)
diff --git a/examples/simple-solver/CMakeLists.txt b/examples/simple-solver/CMakeLists.txt
index 5603fea0377..ed067ec02e1 100644
--- a/examples/simple-solver/CMakeLists.txt
+++ b/examples/simple-solver/CMakeLists.txt
@@ -3,7 +3,7 @@ project(simple-solver)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(simple-solver simple-solver.cpp)
diff --git a/examples/three-pt-stencil-solver/CMakeLists.txt b/examples/three-pt-stencil-solver/CMakeLists.txt
index 34842e283af..661f064b36a 100644
--- a/examples/three-pt-stencil-solver/CMakeLists.txt
+++ b/examples/three-pt-stencil-solver/CMakeLists.txt
@@ -3,7 +3,7 @@ project(three-pt-stencil-solver)
 
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
-    find_package(Ginkgo 1.3.0 REQUIRED)
+    find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
 add_executable(three-pt-stencil-solver three-pt-stencil-solver.cpp)

From b7b5b592a21305a1397407e11c17cde6bcb330e4 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 13 Apr 2021 12:02:31 +0200
Subject: [PATCH 0701/1008] review updates

Co-authored-by: Yuhsiang M. Tsai <yhmtsai@gmail.com>
---
 examples/custom-matrix-format/CMakeLists.txt | 4 +---
 examples/ginkgo-ranges/CMakeLists.txt        | 1 -
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/examples/custom-matrix-format/CMakeLists.txt b/examples/custom-matrix-format/CMakeLists.txt
index c6de4475eb9..89b5f7f737d 100644
--- a/examples/custom-matrix-format/CMakeLists.txt
+++ b/examples/custom-matrix-format/CMakeLists.txt
@@ -9,9 +9,7 @@ endif()
 set(CMAKE_CUDA_STANDARD 14)
 set(CMAKE_CUDA_STANDARD_REQUIRED ON)
 
-add_executable(custom-matrix-format
-    custom-matrix-format.cpp
-        stencil_kernel.cu)
+add_executable(custom-matrix-format custom-matrix-format.cpp stencil_kernel.cu)
 target_link_libraries(custom-matrix-format Ginkgo::ginkgo)
 
 # workaround for clang-cuda/g++ interaction
diff --git a/examples/ginkgo-ranges/CMakeLists.txt b/examples/ginkgo-ranges/CMakeLists.txt
index ebefcf103a3..c3cd06954c9 100644
--- a/examples/ginkgo-ranges/CMakeLists.txt
+++ b/examples/ginkgo-ranges/CMakeLists.txt
@@ -7,4 +7,3 @@ if (NOT GINKGO_BUILD_EXAMPLES)
 endif()
 add_executable(ginkgo-ranges ginkgo-ranges.cpp)
 target_link_libraries(ginkgo-ranges Ginkgo::ginkgo)
-target_include_directories(ginkgo-ranges PRIVATE ${PROJECT_SOURCE_DIR})

From aacf23c0a31e0ad0ff365ee45f294258d7de262a Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 13 Apr 2021 15:55:49 +0200
Subject: [PATCH 0702/1008] store .so files in lib/ as well

---
 cmake/build_helpers.cmake | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cmake/build_helpers.cmake b/cmake/build_helpers.cmake
index 456398a0d1c..8ef56577f16 100644
--- a/cmake/build_helpers.cmake
+++ b/cmake/build_helpers.cmake
@@ -31,6 +31,8 @@ function(ginkgo_compile_features name)
         RUNTIME_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}")
     set_property(TARGET "${name}" PROPERTY
         ARCHIVE_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}")
+    set_property(TARGET "${name}" PROPERTY
+        LIBRARY_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}")
 
     if (GINKGO_CHECK_CIRCULAR_DEPS)
         target_link_libraries("${name}" PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")

From ede5386ae50cf784f964f7003dab667fed72b4d4 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 13 Apr 2021 15:56:20 +0200
Subject: [PATCH 0703/1008] fix example build.sh

---
 examples/build-setup.sh                | 24 +++++++++++-------------
 examples/cb-gmres/build.sh             |  3 +++
 examples/custom-matrix-format/build.sh |  9 ++-------
 3 files changed, 16 insertions(+), 20 deletions(-)

diff --git a/examples/build-setup.sh b/examples/build-setup.sh
index 45c827e07d3..b71f97f627a 100644
--- a/examples/build-setup.sh
+++ b/examples/build-setup.sh
@@ -2,27 +2,25 @@
 
 # copy libraries
 LIBRARY_DIRS="core core/device_hooks reference omp cuda hip"
-LIBRARY_NAMES="ginkgo ginkgo_reference ginkgo_omp ginkgo_cuda ginkgo_hip ginkgo_dpcpp"
+LIBRARY_NAMES="ginkgo ginkgo_reference ginkgo_omp ginkgo_cuda ginkgo_hip ginkgo_dpcpp ginkgo_device"
 SUFFIXES=".so .dylib .dll d.so d.dylib d.dll"
 VERSION="1.4.0"
-for prefix in ${LIBRARY_DIRS}; do
-    for name in ${LIBRARY_NAMES}; do
-        for suffix in ${SUFFIXES}; do
-            cp ${BUILD_DIR}/${prefix}/lib${name}${suffix}.${VERSION} \
-                ${THIS_DIR} 2>/dev/null
-            if [ -e "${THIS_DIR}/lib${name}${suffix}.${VERSION}" ]
-            then
-                ln -s ${THIS_DIR}/lib${name}${suffix}.${VERSION} ${THIS_DIR}/lib${name}${suffix} 2>/dev/null
-            fi
-        done
+for name in ${LIBRARY_NAMES}; do
+    for suffix in ${SUFFIXES}; do
+        cp ${BUILD_DIR}/lib/lib${name}${suffix}.${VERSION} \
+            ${THIS_DIR} 2>/dev/null
+        if [ -e "${THIS_DIR}/lib${name}${suffix}.${VERSION}" ]
+        then
+            ln -s ${THIS_DIR}/lib${name}${suffix}.${VERSION} ${THIS_DIR}/lib${name}${suffix} 2>/dev/null
+        fi
     done
 done
 
 # figure out correct compiler flags
 if ls ${THIS_DIR} | grep -F "libginkgo." >/dev/null; then
-    LINK_FLAGS="-lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference -lginkgo_hip -lginkgo_dpcpp -Wl,-rpath,${THIS_DIR}"
+    LINK_FLAGS="-lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference -lginkgo_hip -lginkgo_dpcpp -lginkgo_device -Wl,-rpath,${THIS_DIR}"
 else
-    LINK_FLAGS="-lginkgod -lginkgo_ompd -lginkgo_cudad -lginkgo_referenced -lginkgo_hipd -lginkgo_dpcppd -Wl,-rpath,${THIS_DIR}"
+    LINK_FLAGS="-lginkgod -lginkgo_ompd -lginkgo_cudad -lginkgo_referenced -lginkgo_hipd -lginkgo_dpcppd -lginkgo_deviced -Wl,-rpath,${THIS_DIR}"
 fi
 if [ -z "${CXX}" ]; then
     CXX="c++"
diff --git a/examples/cb-gmres/build.sh b/examples/cb-gmres/build.sh
index 9cbd24ca500..da7e54829b6 100755
--- a/examples/cb-gmres/build.sh
+++ b/examples/cb-gmres/build.sh
@@ -10,6 +10,9 @@ THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" &>/dev/null && pwd )
 
 source ${THIS_DIR}/../build-setup.sh
 
+mkdir -p data
+cp ${THIS_DIR}/../../matrices/test/ani1.mtx data/A.mtx
+
 # build
 ${CXX} -std=c++14 -o ${THIS_DIR}/cb-gmres ${THIS_DIR}/cb-gmres.cpp \
        -I${THIS_DIR}/../../include -I${BUILD_DIR}/include \
diff --git a/examples/custom-matrix-format/build.sh b/examples/custom-matrix-format/build.sh
index 1821aa72bd5..10c2d188666 100755
--- a/examples/custom-matrix-format/build.sh
+++ b/examples/custom-matrix-format/build.sh
@@ -12,13 +12,8 @@ source ${THIS_DIR}/../build-setup.sh
 
 CXX="nvcc"
 
-# figure out correct compiler flags
-if ls ${THIS_DIR} | grep -F "libginkgo." >/dev/null; then
-    LINK_FLAGS="-lginkgo -lginkgo_omp -lginkgo_cuda -lginkgo_reference -lginkgo_dpcpp -lginkgo_hip -Xlinker -rpath -Xlinker ${THIS_DIR}"
-else
-    LINK_FLAGS="-lginkgod -lginkgo_ompd -lginkgo_cudad -lginkgo_referenced -lginkgo_dpcppd -lginkgo_hipd -Xlinker -rpath -Xlinker ${THIS_DIR}"
-fi
-
+# adjust to nvcc style link flags
+LINK_FLAGS="${LINK_FLAGS/-Wl,-rpath,/-Xlinker -rpath -Xlinker }"
 
 # build
 ${CXX} -std=c++14 -o ${THIS_DIR}/custom-matrix-format \

From 1e6831811712e6b0ccd5fba2488076735dd35be6 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 7 May 2021 16:14:13 +0200
Subject: [PATCH 0704/1008] review updates

Co-authored-by: Author: Terry Cojean <terry.cojean@kit.edu>
---
 CMakeLists.txt                               |  2 --
 INSTALL.md                                   | 17 +++--------------
 README.md                                    | 14 ++++----------
 cmake/build_helpers.cmake                    |  2 ++
 examples/build-setup.sh                      |  1 -
 examples/custom-matrix-format/CMakeLists.txt |  8 +++++++-
 examples/papi-logging/CMakeLists.txt         |  4 ++++
 7 files changed, 20 insertions(+), 28 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7b440d2666e..7c4eaf29caa 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -25,8 +25,6 @@ if (MINGW OR CYGWIN)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wa,-mbig-obj")
 endif()
 
-set(GINKGO_LIBRARY_PATH "${PROJECT_BINARY_DIR}/lib")
-
 # Ginkgo configuration options
 option(GINKGO_DEVEL_TOOLS "Add development tools to the build system" OFF)
 option(GINKGO_BUILD_TESTS "Generate build files for unit tests" ON)
diff --git a/INSTALL.md b/INSTALL.md
index 0f558d155e3..dad87874b26 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -132,20 +132,9 @@ generators. Other CMake generators are untested.
 ### Building Ginkgo in Windows
 Depending on the configuration settings, some manual work might be required:
 * Build Ginkgo with Debug mode:
-  Some Debug build specific issues can appear depending on the machine and environment. The known issues are the following:
-  1. `ld` issue: encountering  `ld: error: export ordinal too large` needs the compilation flag `-O1`
-
-  The following are the details for different environments:
-  * _Microsoft Visual Studio_:
-    1. `ld` issue (_Microsoft Visual Studio_ does not have this issue)
-  * _Cygwin_:
-    1. `ld` issue (If building Ginkgo as static library, this is not needed)
-      * `cmake -DGINKGO_COMPILER_FLAGS="-Wpedantic -O1" <other parameters> <source_folder>` (`GINKGO_COMPILER_FLAGS` is `-Wpedantic` by default)
-      * add `-O1` in the environement variable `CXX_FLAGS` or `CMAKE_CXX_FLAGS`
-  * _MinGW_:
-    1. `ld` issue (If building Ginkgo as static library, this is not needed)
-      * `cmake -DGINKGO_COMPILER_FLAGS="-Wpedantic -O1" <other parameters> <source_folder>` (`GINKGO_COMPILER_FLAGS` is `-Wpedantic` by default)
-      * add `-O1` in the environement variable `CXX_FLAGS` or `CMAKE_CXX_FLAGS`
+  Some Debug build specific issues can appear depending on the machine and environment:
+  When you encounter the error message `ld: error: export ordinal too large`, add the compilation flag `-O1`
+  by adding `-DCMAKE_CXX_FLAGS=-O1` to the CMake invocation.
 * Build Ginkgo in _MinGW_:\
   If encountering the issue `cc1plus.exe: out of memory allocating 65536 bytes`, please follow the workaround in
   [reference](https://www.intel.com/content/www/us/en/programmable/support/support-resources/knowledge-base/embedded/2016/cc1plus-exe--out-of-memory-allocating-65536-bytes.html),
diff --git a/README.md b/README.md
index 9f3c93ba00c..271d211427b 100644
--- a/README.md
+++ b/README.md
@@ -88,16 +88,10 @@ The Ginkgo CUDA module has the following __additional__ requirements:
 The Ginkgo OMP module has the following __additional__ requirements:
 *  _MinGW_ or _Cygwin_
 
-Depending on the configuration settings, some manual work might be required. More details are availble in [windows section in INSTALL.md](INSTALL.md#building-ginkgo-in-windows):
-* Build Ginkgo with Debug mode:
-  Some Debug build specific issues can appear depending on the machine and environment. The known issues are the following:
-  1. `ld` issue: encountering  `ld: error: export ordinal too large` needs the compilation flag `-O1`
-* Build Ginkgo in _MinGW_:
-  If encountering the issue `cc1plus.exe: out of memory allocating 65536 bytes`, please follow the workaround in
-  [reference](https://www.intel.com/content/www/us/en/programmable/support/support-resources/knowledge-base/embedded/2016/cc1plus-exe--out-of-memory-allocating-65536-bytes.html),
-  or compile ginkgo again might work.
-
-__NOTE:__ _Microsoft Visual Studio_ only supports OpenMP 2.0, so it can not compile the ginkgo OMP module.
+In these environments, two problems can be encountered, the solution for which is described in the
+[windows section in INSTALL.md](INSTALL.md#building-ginkgo-in-windows):
+* `ld: error: export ordinal too large` needs the compilation flag `-O1`
+* `cc1plus.exe: out of memory allocating 65536 bytes` requires a modification of the environment
 
 __NOTE:__ Some restrictions will also apply on the version of C and C++ standard
 libraries installed on the system. This needs further investigation.
diff --git a/cmake/build_helpers.cmake b/cmake/build_helpers.cmake
index 8ef56577f16..3a264660733 100644
--- a/cmake/build_helpers.cmake
+++ b/cmake/build_helpers.cmake
@@ -1,3 +1,5 @@
+set(GINKGO_LIBRARY_PATH "${PROJECT_BINARY_DIR}/lib")
+
 function(ginkgo_default_includes name)
     # set include path depending on used interface
     target_include_directories("${name}"
diff --git a/examples/build-setup.sh b/examples/build-setup.sh
index b71f97f627a..d5698c8f7e7 100644
--- a/examples/build-setup.sh
+++ b/examples/build-setup.sh
@@ -1,7 +1,6 @@
 #!/bin/bash
 
 # copy libraries
-LIBRARY_DIRS="core core/device_hooks reference omp cuda hip"
 LIBRARY_NAMES="ginkgo ginkgo_reference ginkgo_omp ginkgo_cuda ginkgo_hip ginkgo_dpcpp ginkgo_device"
 SUFFIXES=".so .dylib .dll d.so d.dylib d.dll"
 VERSION="1.4.0"
diff --git a/examples/custom-matrix-format/CMakeLists.txt b/examples/custom-matrix-format/CMakeLists.txt
index 89b5f7f737d..c0821a2cf35 100644
--- a/examples/custom-matrix-format/CMakeLists.txt
+++ b/examples/custom-matrix-format/CMakeLists.txt
@@ -4,13 +4,19 @@ project(custom-matrix-format CXX CUDA)
 # We only need to find Ginkgo if we build this example stand-alone
 if (NOT GINKGO_BUILD_EXAMPLES)
     find_package(Ginkgo 1.4.0 REQUIRED)
+    find_package(OpenMP 3.0 REQUIRED)
+endif()
+
+if(NOT (GINKGO_BUILD_CUDA AND GINKGO_BUILD_OMP))
+    message(FATAL_ERROR
+        "This example needs Ginkgo built with CUDA and OpenMP support")
 endif()
 
 set(CMAKE_CUDA_STANDARD 14)
 set(CMAKE_CUDA_STANDARD_REQUIRED ON)
 
 add_executable(custom-matrix-format custom-matrix-format.cpp stencil_kernel.cu)
-target_link_libraries(custom-matrix-format Ginkgo::ginkgo)
+target_link_libraries(custom-matrix-format Ginkgo::ginkgo OpenMP::OpenMP_CXX)
 
 # workaround for clang-cuda/g++ interaction
 set_target_properties(custom-matrix-format PROPERTIES POSITION_INDEPENDENT_CODE ON)
diff --git a/examples/papi-logging/CMakeLists.txt b/examples/papi-logging/CMakeLists.txt
index a8f0f8a260c..d7db0ef89fa 100644
--- a/examples/papi-logging/CMakeLists.txt
+++ b/examples/papi-logging/CMakeLists.txt
@@ -6,6 +6,10 @@ if (NOT GINKGO_BUILD_EXAMPLES)
     find_package(Ginkgo 1.4.0 REQUIRED)
 endif()
 
+if (NOT GINKGO_HAVE_PAPI_SDE)
+    message(FATAL_ERROR "This example needs Ginkgo built with PAPI support")
+endif()
+
 add_executable(papi-logging papi-logging.cpp)
 target_link_libraries(papi-logging ginkgo PAPI::PAPI)
 

From 1e79f29f35889afb967e71e2678467aea1bb8e57 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 8 May 2021 23:12:31 +0200
Subject: [PATCH 0705/1008] parallelize COO OpenMP SpMV

---
 omp/components/atomic.hpp  |  73 ++++++++++++++++++++++++++
 omp/matrix/coo_kernels.cpp | 104 ++++++++++++++++++++++++++++++-------
 2 files changed, 157 insertions(+), 20 deletions(-)
 create mode 100644 omp/components/atomic.hpp

diff --git a/omp/components/atomic.hpp b/omp/components/atomic.hpp
new file mode 100644
index 00000000000..07dd0972ed6
--- /dev/null
+++ b/omp/components/atomic.hpp
@@ -0,0 +1,73 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_OMP_COMPONENTS_ATOMIC_HPP_
+#define GKO_OMP_COMPONENTS_ATOMIC_HPP_
+
+
+#include <type_traits>
+
+
+#include <ginkgo/core/base/types.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+
+
+template <typename ValueType,
+          std::enable_if_t<!is_complex<ValueType>()> * = nullptr>
+void atomic_add(ValueType &out, ValueType val)
+{
+#pragma omp atomic
+    out += val;
+}
+
+template <typename ValueType,
+          std::enable_if_t<is_complex<ValueType>()> * = nullptr>
+void atomic_add(ValueType &out, ValueType val)
+{
+    auto values = reinterpret_cast<gko::remove_complex<ValueType> *>(&out);
+#pragma omp atomic
+    values[0] += real(val);
+#pragma omp atomic
+    values[1] += imag(val);
+}
+
+
+}  // namespace omp
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_OMP_COMPONENTS_ATOMIC_HPP_
diff --git a/omp/matrix/coo_kernels.cpp b/omp/matrix/coo_kernels.cpp
index 676051af076..07f9e0b71dd 100644
--- a/omp/matrix/coo_kernels.cpp
+++ b/omp/matrix/coo_kernels.cpp
@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "omp/components/atomic.hpp"
 #include "omp/components/format_conversion.hpp"
 
 
@@ -103,15 +104,46 @@ void spmv2(std::shared_ptr<const OmpExecutor> exec,
            const matrix::Coo<ValueType, IndexType> *a,
            const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
 {
-    auto coo_val = a->get_const_values();
-    auto coo_col = a->get_const_col_idxs();
-    auto coo_row = a->get_const_row_idxs();
-    auto num_cols = b->get_size()[1];
-
-#pragma omp parallel for
-    for (size_type j = 0; j < num_cols; j++) {
-        for (size_type i = 0; i < a->get_num_stored_elements(); i++) {
-            c->at(coo_row[i], j) += coo_val[i] * b->at(coo_col[i], j);
+    const auto coo_val = a->get_const_values();
+    const auto coo_col = a->get_const_col_idxs();
+    const auto coo_row = a->get_const_row_idxs();
+    const auto num_rhs = b->get_size()[1];
+    const auto sentinel_row = a->get_size()[0] + 1;
+    const auto nnz = a->get_num_stored_elements();
+    const auto num_threads = omp_get_max_threads();
+    const auto work_per_thread =
+        static_cast<size_type>(ceildiv(nnz, num_threads));
+
+#pragma omp parallel num_threads(num_threads)
+    {
+        const auto thread_id = static_cast<size_type>(omp_get_thread_num());
+        const auto begin = work_per_thread * thread_id;
+        const auto end = std::min(begin + work_per_thread, nnz);
+        if (begin < end) {
+            const auto first = begin > 0 ? coo_row[begin - 1] : sentinel_row;
+            const auto last = end < nnz ? coo_row[end] : sentinel_row;
+            auto nz = begin;
+            for (; nz < end && coo_row[nz] == first; nz++) {
+                const auto row = first;
+                const auto col = coo_col[nz];
+                for (size_type rhs = 0; rhs < num_rhs; rhs++) {
+                    atomic_add(c->at(row, rhs), coo_val[nz] * b->at(col, rhs));
+                }
+            }
+            for (; nz < end && coo_row[nz] != last; nz++) {
+                const auto row = coo_row[nz];
+                const auto col = coo_col[nz];
+                for (size_type rhs = 0; rhs < num_rhs; rhs++) {
+                    c->at(row, rhs) += coo_val[nz] * b->at(col, rhs);
+                }
+            }
+            for (; nz < end; nz++) {
+                const auto row = last;
+                const auto col = coo_col[nz];
+                for (size_type rhs = 0; rhs < num_rhs; rhs++) {
+                    atomic_add(c->at(row, rhs), coo_val[nz] * b->at(col, rhs));
+                }
+            }
         }
     }
 }
@@ -126,17 +158,49 @@ void advanced_spmv2(std::shared_ptr<const OmpExecutor> exec,
                     const matrix::Dense<ValueType> *b,
                     matrix::Dense<ValueType> *c)
 {
-    auto coo_val = a->get_const_values();
-    auto coo_col = a->get_const_col_idxs();
-    auto coo_row = a->get_const_row_idxs();
-    auto alpha_val = alpha->at(0, 0);
-    auto num_cols = b->get_size()[1];
-
-#pragma omp parallel for
-    for (size_type j = 0; j < num_cols; j++) {
-        for (size_type i = 0; i < a->get_num_stored_elements(); i++) {
-            c->at(coo_row[i], j) +=
-                alpha_val * coo_val[i] * b->at(coo_col[i], j);
+    const auto coo_val = a->get_const_values();
+    const auto coo_col = a->get_const_col_idxs();
+    const auto coo_row = a->get_const_row_idxs();
+    const auto num_rhs = b->get_size()[1];
+    const auto sentinel_row = a->get_size()[0] + 1;
+    const auto nnz = a->get_num_stored_elements();
+    const auto num_threads = omp_get_max_threads();
+    const auto work_per_thread =
+        static_cast<size_type>(ceildiv(nnz, num_threads));
+    const auto scale = alpha->at(0, 0);
+
+#pragma omp parallel num_threads(num_threads)
+    {
+        const auto thread_id = static_cast<size_type>(omp_get_thread_num());
+        const auto begin = work_per_thread * thread_id;
+        const auto end = std::min(begin + work_per_thread, nnz);
+        if (begin < end) {
+            const auto first = begin > 0 ? coo_row[begin - 1] : sentinel_row;
+            const auto last = end < nnz ? coo_row[end] : sentinel_row;
+            auto nz = begin;
+            for (; nz < end && coo_row[nz] == first; nz++) {
+                const auto row = first;
+                const auto col = coo_col[nz];
+                for (size_type rhs = 0; rhs < num_rhs; rhs++) {
+                    atomic_add(c->at(row, rhs),
+                               scale * coo_val[nz] * b->at(col, rhs));
+                }
+            }
+            for (; nz < end && coo_row[nz] != last; nz++) {
+                const auto row = coo_row[nz];
+                const auto col = coo_col[nz];
+                for (size_type rhs = 0; rhs < num_rhs; rhs++) {
+                    c->at(row, rhs) += scale * coo_val[nz] * b->at(col, rhs);
+                }
+            }
+            for (; nz < end; nz++) {
+                const auto row = last;
+                const auto col = coo_col[nz];
+                for (size_type rhs = 0; rhs < num_rhs; rhs++) {
+                    atomic_add(c->at(row, rhs),
+                               scale * coo_val[nz] * b->at(col, rhs));
+                }
+            }
         }
     }
 }

From f1c3b31aa0a91c147ab416ca0bf91eae50d424fb Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 19 May 2021 11:28:03 +0200
Subject: [PATCH 0706/1008] fix compilation issues with is_complex

---
 omp/components/atomic.hpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/omp/components/atomic.hpp b/omp/components/atomic.hpp
index 07dd0972ed6..c754f28fb20 100644
--- a/omp/components/atomic.hpp
+++ b/omp/components/atomic.hpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <type_traits>
 
 
+#include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
 
 

From e0d96e51d3c1e532f1cd55958efec60ba5d3bb21 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 19 May 2021 12:03:31 +0200
Subject: [PATCH 0707/1008] move work assignment inside parallel region

---
 omp/matrix/coo_kernels.cpp | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/omp/matrix/coo_kernels.cpp b/omp/matrix/coo_kernels.cpp
index 07f9e0b71dd..511fc3975a8 100644
--- a/omp/matrix/coo_kernels.cpp
+++ b/omp/matrix/coo_kernels.cpp
@@ -110,12 +110,12 @@ void spmv2(std::shared_ptr<const OmpExecutor> exec,
     const auto num_rhs = b->get_size()[1];
     const auto sentinel_row = a->get_size()[0] + 1;
     const auto nnz = a->get_num_stored_elements();
-    const auto num_threads = omp_get_max_threads();
-    const auto work_per_thread =
-        static_cast<size_type>(ceildiv(nnz, num_threads));
 
-#pragma omp parallel num_threads(num_threads)
+#pragma omp parallel
     {
+        const auto num_threads = omp_get_num_threads();
+        const auto work_per_thread =
+            static_cast<size_type>(ceildiv(nnz, num_threads));
         const auto thread_id = static_cast<size_type>(omp_get_thread_num());
         const auto begin = work_per_thread * thread_id;
         const auto end = std::min(begin + work_per_thread, nnz);
@@ -164,13 +164,13 @@ void advanced_spmv2(std::shared_ptr<const OmpExecutor> exec,
     const auto num_rhs = b->get_size()[1];
     const auto sentinel_row = a->get_size()[0] + 1;
     const auto nnz = a->get_num_stored_elements();
-    const auto num_threads = omp_get_max_threads();
-    const auto work_per_thread =
-        static_cast<size_type>(ceildiv(nnz, num_threads));
     const auto scale = alpha->at(0, 0);
 
-#pragma omp parallel num_threads(num_threads)
+#pragma omp parallel
     {
+        const auto num_threads = omp_get_num_threads();
+        const auto work_per_thread =
+            static_cast<size_type>(ceildiv(nnz, num_threads));
         const auto thread_id = static_cast<size_type>(omp_get_thread_num());
         const auto begin = work_per_thread * thread_id;
         const auto end = std::min(begin + work_per_thread, nnz);

From b6fb0d2526b10eb71c9ecdd8be555e079a43c255 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 19 May 2021 12:08:18 +0200
Subject: [PATCH 0708/1008] add comment for complex casting and C++ standard

---
 omp/components/atomic.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/omp/components/atomic.hpp b/omp/components/atomic.hpp
index c754f28fb20..5488355ac7f 100644
--- a/omp/components/atomic.hpp
+++ b/omp/components/atomic.hpp
@@ -58,6 +58,8 @@ template <typename ValueType,
           std::enable_if_t<is_complex<ValueType>()> * = nullptr>
 void atomic_add(ValueType &out, ValueType val)
 {
+    // The C++ standard explicitly allows casting complex<double>* to double*
+    // [complex.numbers.general]
     auto values = reinterpret_cast<gko::remove_complex<ValueType> *>(&out);
 #pragma omp atomic
     values[0] += real(val);

From e197ffaade0143603f837aaec00b89e5248d0b26 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 20 May 2021 16:09:28 +0200
Subject: [PATCH 0709/1008] use diag size on extract_diagonal

---
 cuda/matrix/dense_kernels.cu     | 2 +-
 hip/matrix/dense_kernels.hip.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index be02377a2b9..b455d16f63b 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -785,7 +785,7 @@ void extract_diagonal(std::shared_ptr<const CudaExecutor> exec,
 {
     const dim3 grid_dim = ceildiv(diag->get_size()[0], default_block_size);
     kernel::extract_diagonal<<<grid_dim, default_block_size>>>(
-        orig->get_size()[0], as_cuda_type(orig->get_const_values()),
+        diag->get_size()[0], as_cuda_type(orig->get_const_values()),
         orig->get_stride(), as_cuda_type(diag->get_values()));
 }
 
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index 13ddd959dcf..9b7a2f60a6a 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -824,7 +824,7 @@ void extract_diagonal(std::shared_ptr<const HipExecutor> exec,
 {
     const dim3 grid_dim = ceildiv(diag->get_size()[0], default_block_size);
     hipLaunchKernelGGL(kernel::extract_diagonal, dim3(grid_dim),
-                       dim3(default_block_size), 0, 0, orig->get_size()[0],
+                       dim3(default_block_size), 0, 0, diag->get_size()[0],
                        as_hip_type(orig->get_const_values()),
                        orig->get_stride(), as_hip_type(diag->get_values()));
 }

From 1db038587ad379a7c26f456b629bd2900455e39f Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 21 May 2021 15:58:29 +0200
Subject: [PATCH 0710/1008] add short-and-fat test for extract_diag

---
 cuda/test/matrix/dense_kernels.cpp      | 13 ++++++++++++-
 hip/test/matrix/dense_kernels.hip.cpp   | 13 ++++++++++++-
 omp/test/matrix/dense_kernels.cpp       | 13 ++++++++++++-
 reference/test/matrix/dense_kernels.cpp | 25 +++++++++++++++++++++++--
 4 files changed, 59 insertions(+), 5 deletions(-)

diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index 61cc527f71b..e0986cff2de 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -829,7 +829,7 @@ TEST_F(Dense, IsInverseColPermutable)
 }
 
 
-TEST_F(Dense, ExtractDiagonalIsEquivalentToRef)
+TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
 {
     set_up_apply_data();
 
@@ -840,6 +840,17 @@ TEST_F(Dense, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto diag = y->extract_diagonal();
+    auto ddiag = dy->extract_diagonal();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
 TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index 3140717c348..72e69c7cc40 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -812,7 +812,7 @@ TEST_F(Dense, IsInverseColPermutable)
 }
 
 
-TEST_F(Dense, ExtractDiagonalIsEquivalentToRef)
+TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
 {
     set_up_apply_data();
 
@@ -823,6 +823,17 @@ TEST_F(Dense, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto diag = y->extract_diagonal();
+    auto ddiag = dy->extract_diagonal();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
 TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index a3942d2d82b..b59c43ed4b0 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -950,7 +950,7 @@ TEST_F(Dense, IsInverseColPermutable)
 }
 
 
-TEST_F(Dense, ExtractDiagonalIsEquivalentToRef)
+TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
 {
     set_up_apply_data();
 
@@ -961,6 +961,17 @@ TEST_F(Dense, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto diag = y->extract_diagonal();
+    auto ddiag = dy->extract_diagonal();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
 TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index a42968bfa52..4b6a6e73b26 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -83,7 +83,9 @@ class Dense : public ::testing::Test {
           mtx5(gko::initialize<Mtx>(
               {{1.0, -1.0, -0.5}, {-2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}, exec)),
           mtx6(gko::initialize<Mtx>({{1.0, 2.0, 0.0}, {0.0, 1.5, 0.0}}, exec)),
-          mtx7(gko::initialize<Mtx>({{1.0, 2.0, 3.0}, {0.0, 1.5, 0.0}}, exec))
+          mtx7(gko::initialize<Mtx>({{1.0, 2.0, 3.0}, {0.0, 1.5, 0.0}}, exec)),
+          mtx8(gko::initialize<Mtx>(
+              {I<T>({1.0, -1.0}), I<T>({-2.0, 2.0}), I<T>({-3.0, 3.0})}, exec))
     {}
 
     std::shared_ptr<const gko::Executor> exec;
@@ -94,6 +96,7 @@ class Dense : public ::testing::Test {
     std::unique_ptr<Mtx> mtx5;
     std::unique_ptr<Mtx> mtx6;
     std::unique_ptr<Mtx> mtx7;
+    std::unique_ptr<Mtx> mtx8;
 
     std::ranlux48 rand_engine;
 
@@ -2383,7 +2386,7 @@ TYPED_TEST(Dense, ExtractsDiagonalFromSquareMatrix)
 }
 
 
-TYPED_TEST(Dense, ExtractsDiagonalFromNonSquareMatrix)
+TYPED_TEST(Dense, ExtractsDiagonalFromTallSkinnyMatrix)
 {
     using T = typename TestFixture::value_type;
 
@@ -2400,6 +2403,24 @@ TYPED_TEST(Dense, ExtractsDiagonalFromNonSquareMatrix)
 }
 
 
+TYPED_TEST(Dense, ExtractsDiagonalFromShortFatMatrix)
+{
+    using T = typename TestFixture::value_type;
+
+    // clang-format off
+    // { 1.0, -1.0},
+    // {-2.0,  2.0},
+    // {-3.0,  3.0}
+    // clang-format on
+    auto diag = this->mtx8->extract_diagonal();
+
+    ASSERT_EQ(diag->get_size()[0], 2);
+    ASSERT_EQ(diag->get_size()[1], 2);
+    ASSERT_EQ(diag->get_values()[0], T{1.});
+    ASSERT_EQ(diag->get_values()[1], T{2.});
+}
+
+
 TYPED_TEST(Dense, InplaceAbsolute)
 {
     using T = typename TestFixture::value_type;

From 3cc593a0b3939c88fd97eda4e77092702294287e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 22 May 2021 22:25:07 +0200
Subject: [PATCH 0711/1008] add BLAS benchmark

---
 benchmark/CMakeLists.txt      |   1 +
 benchmark/blas/CMakeLists.txt |   1 +
 benchmark/blas/blas.cpp       | 540 ++++++++++++++++++++++++++++++++++
 3 files changed, 542 insertions(+)
 create mode 100644 benchmark/blas/CMakeLists.txt
 create mode 100644 benchmark/blas/blas.cpp

diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
index ad719467c2b..da1c80b31e1 100644
--- a/benchmark/CMakeLists.txt
+++ b/benchmark/CMakeLists.txt
@@ -97,6 +97,7 @@ function(ginkgo_add_typed_benchmark_executables name use_lib_linops)
 endfunction(ginkgo_add_typed_benchmark_executables)
 
 
+add_subdirectory(blas)
 add_subdirectory(conversions)
 add_subdirectory(matrix_generator)
 add_subdirectory(matrix_statistics)
diff --git a/benchmark/blas/CMakeLists.txt b/benchmark/blas/CMakeLists.txt
new file mode 100644
index 00000000000..c3e40e80bc2
--- /dev/null
+++ b/benchmark/blas/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_add_typed_benchmark_executables(blas "NO" blas.cpp)
diff --git a/benchmark/blas/blas.cpp b/benchmark/blas/blas.cpp
new file mode 100644
index 00000000000..1f9a7325a2f
--- /dev/null
+++ b/benchmark/blas/blas.cpp
@@ -0,0 +1,540 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/ginkgo.hpp>
+
+
+#include <algorithm>
+#include <chrono>
+#include <cstdlib>
+#include <exception>
+#include <iomanip>
+#include <iostream>
+#include <typeinfo>
+
+
+#include "benchmark/utils/general.hpp"
+#include "benchmark/utils/loggers.hpp"
+#include "benchmark/utils/timer.hpp"
+#include "benchmark/utils/types.hpp"
+
+
+// Command-line arguments
+DEFINE_string(
+    operations, "copy,axpy,scal",
+    "A comma-separated list of BLAS operations to benchmark.\nCandidates are"
+    "   copy (y = x),\n"
+    "   axpy (y = y + a * x),\n"
+    "   multiaxpy (like axpy, but a has one entry per column),\n"
+    "   scal (y = a * y),\n"
+    "   multiscal (like scal, but a has one entry per column),\n"
+    "   dot (a = x' * y),"
+    "   norm (a = sqrt(x' * x)),\n"
+    "   mm (C = A * B),\n"
+    "   gemm (C = a * A * B + b * C)\n"
+    "where A has dimensions n x k, B has dimensions k x m,\n"
+    "C has dimensions n x m and x and y have dimensions n x r");
+
+
+class BenchmarkOperation {
+public:
+    virtual ~BenchmarkOperation() = default;
+
+    virtual void write_error(rapidjson::Value &blas_case,
+                             rapidjson::MemoryPoolAllocator<> &allocator)
+    {}
+    virtual gko::size_type get_flops() const = 0;
+    virtual gko::size_type get_memory() const = 0;
+    virtual void prepare(){};
+    virtual void run() = 0;
+};
+
+
+class CopyOperation : public BenchmarkOperation {
+public:
+    CopyOperation(std::shared_ptr<const gko::Executor> exec,
+                  gko::size_type rows, gko::size_type cols,
+                  gko::size_type istride, gko::size_type ostride)
+    {
+        in_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{rows, cols},
+                                                istride);
+        out_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{rows, cols},
+                                                 ostride);
+        in_->fill(1);
+    }
+
+    gko::size_type get_flops() const override
+    {
+        return in_->get_size()[0] * in_->get_size()[1];
+    }
+
+    gko::size_type get_memory() const override
+    {
+        return in_->get_size()[0] * in_->get_size()[1] * sizeof(etype) * 2;
+    }
+
+    void run() override { in_->convert_to(lend(out_)); }
+
+private:
+    std::unique_ptr<gko::matrix::Dense<etype>> in_;
+    std::unique_ptr<gko::matrix::Dense<etype>> out_;
+};
+
+
+class AxpyOperation : public BenchmarkOperation {
+public:
+    AxpyOperation(std::shared_ptr<const gko::Executor> exec,
+                  gko::size_type rows, gko::size_type cols,
+                  gko::size_type stride_in, gko::size_type stride_out,
+                  bool multi)
+    {
+        alpha_ = gko::matrix::Dense<etype>::create(
+            exec, gko::dim<2>{1, multi ? cols : 1});
+        x_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{rows, cols},
+                                               stride_in);
+        y_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{rows, cols},
+                                               stride_out);
+        alpha_->fill(1);
+        x_->fill(1);
+    }
+
+    gko::size_type get_flops() const override
+    {
+        return y_->get_size()[0] * y_->get_size()[1] * 2;
+    }
+
+    gko::size_type get_memory() const override
+    {
+        return y_->get_size()[0] * y_->get_size()[1] * sizeof(etype) * 3;
+    }
+
+    void prepare() override { y_->fill(1); }
+
+    void run() override { y_->add_scaled(lend(alpha_), lend(x_)); }
+
+private:
+    std::unique_ptr<gko::matrix::Dense<etype>> alpha_;
+    std::unique_ptr<gko::matrix::Dense<etype>> x_;
+    std::unique_ptr<gko::matrix::Dense<etype>> y_;
+};
+
+
+class ScalOperation : public BenchmarkOperation {
+public:
+    ScalOperation(std::shared_ptr<const gko::Executor> exec,
+                  gko::size_type rows, gko::size_type cols,
+                  gko::size_type stride, bool multi)
+    {
+        alpha_ = gko::matrix::Dense<etype>::create(
+            exec, gko::dim<2>{1, multi ? cols : 1});
+        y_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{rows, cols},
+                                               stride);
+        alpha_->fill(1);
+    }
+
+    gko::size_type get_flops() const override
+    {
+        return y_->get_size()[0] * y_->get_size()[1];
+    }
+
+    gko::size_type get_memory() const override
+    {
+        return y_->get_size()[0] * y_->get_size()[1] * sizeof(etype) * 2;
+    }
+
+    void prepare() override { y_->fill(1); }
+
+    void run() override { y_->scale(lend(alpha_)); }
+
+private:
+    std::unique_ptr<gko::matrix::Dense<etype>> alpha_;
+    std::unique_ptr<gko::matrix::Dense<etype>> y_;
+};
+
+
+class DotOperation : public BenchmarkOperation {
+public:
+    DotOperation(std::shared_ptr<const gko::Executor> exec, gko::size_type rows,
+                 gko::size_type cols, gko::size_type stride_x,
+                 gko::size_type stride_y)
+    {
+        alpha_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{1, cols});
+        x_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{rows, cols},
+                                               stride_x);
+        y_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{rows, cols},
+                                               stride_y);
+        x_->fill(1);
+        y_->fill(1);
+    }
+
+    gko::size_type get_flops() const override
+    {
+        return y_->get_size()[0] * y_->get_size()[1] * 2;
+    }
+
+    gko::size_type get_memory() const override
+    {
+        return y_->get_size()[0] * y_->get_size()[1] * sizeof(etype) * 2;
+    }
+
+    void run() override { x_->compute_dot(lend(y_), lend(alpha_)); }
+
+private:
+    std::unique_ptr<gko::matrix::Dense<etype>> alpha_;
+    std::unique_ptr<gko::matrix::Dense<etype>> x_;
+    std::unique_ptr<gko::matrix::Dense<etype>> y_;
+};
+
+
+class NormOperation : public BenchmarkOperation {
+public:
+    NormOperation(std::shared_ptr<const gko::Executor> exec,
+                  gko::size_type rows, gko::size_type cols,
+                  gko::size_type stride)
+    {
+        alpha_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{1, cols});
+        y_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{rows, cols},
+                                               stride);
+        y_->fill(1);
+    }
+
+    gko::size_type get_flops() const override
+    {
+        return y_->get_size()[0] * y_->get_size()[1] * 2;
+    }
+
+    gko::size_type get_memory() const override
+    {
+        return y_->get_size()[0] * y_->get_size()[1] * sizeof(etype);
+    }
+
+    void run() override { y_->compute_norm2(lend(alpha_)); }
+
+private:
+    std::unique_ptr<gko::matrix::Dense<etype>> alpha_;
+    std::unique_ptr<gko::matrix::Dense<etype>> y_;
+};
+
+
+class ApplyOperation : public BenchmarkOperation {
+public:
+    ApplyOperation(std::shared_ptr<const gko::Executor> exec, gko::size_type n,
+                   gko::size_type k, gko::size_type m, gko::size_type stride_A,
+                   gko::size_type stride_B, gko::size_type stride_C)
+    {
+        A_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{n, k},
+                                               stride_A);
+        B_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{k, m},
+                                               stride_B);
+        C_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{n, m},
+                                               stride_C);
+        A_->fill(1);
+        B_->fill(1);
+    }
+
+    gko::size_type get_flops() const override
+    {
+        return A_->get_size()[0] * A_->get_size()[1] * B_->get_size()[1] * 2;
+    }
+
+    gko::size_type get_memory() const override
+    {
+        return (A_->get_size()[0] * A_->get_size()[1] +
+                B_->get_size()[0] * B_->get_size()[1] +
+                C_->get_size()[0] * C_->get_size()[1]) *
+               sizeof(etype);
+    }
+
+    void run() override { B_->apply(lend(A_), lend(C_)); }
+
+private:
+    std::unique_ptr<gko::matrix::Dense<etype>> A_;
+    std::unique_ptr<gko::matrix::Dense<etype>> B_;
+    std::unique_ptr<gko::matrix::Dense<etype>> C_;
+};
+
+
+class AdvancedApplyOperation : public BenchmarkOperation {
+public:
+    AdvancedApplyOperation(std::shared_ptr<const gko::Executor> exec,
+                           gko::size_type n, gko::size_type k, gko::size_type m,
+                           gko::size_type stride_A, gko::size_type stride_B,
+                           gko::size_type stride_C)
+    {
+        A_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{n, k},
+                                               stride_A);
+        B_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{k, m},
+                                               stride_B);
+        C_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{n, m},
+                                               stride_C);
+        alpha_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{1, 1});
+        beta_ = gko::matrix::Dense<etype>::create(exec, gko::dim<2>{1, 1});
+        A_->fill(1);
+        B_->fill(1);
+        alpha_->fill(1);
+        beta_->fill(1);
+    }
+
+    gko::size_type get_flops() const override
+    {
+        return A_->get_size()[0] * A_->get_size()[1] * B_->get_size()[1] * 2 +
+               C_->get_size()[0] * C_->get_size()[1] * 3;
+    }
+
+    gko::size_type get_memory() const override
+    {
+        return (A_->get_size()[0] * A_->get_size()[1] +
+                B_->get_size()[0] * B_->get_size()[1] +
+                C_->get_size()[0] * C_->get_size()[1] * 2) *
+               sizeof(etype);
+    }
+
+    void run() override
+    {
+        B_->apply(lend(alpha_), lend(A_), lend(beta_), lend(C_));
+    }
+
+private:
+    std::unique_ptr<gko::matrix::Dense<etype>> alpha_;
+    std::unique_ptr<gko::matrix::Dense<etype>> beta_;
+    std::unique_ptr<gko::matrix::Dense<etype>> A_;
+    std::unique_ptr<gko::matrix::Dense<etype>> B_;
+    std::unique_ptr<gko::matrix::Dense<etype>> C_;
+};
+
+
+struct dimensions {
+    gko::size_type n;
+    gko::size_type k;
+    gko::size_type m;
+    gko::size_type r;
+    gko::size_type stride_x;
+    gko::size_type stride_y;
+    gko::size_type stride_A;
+    gko::size_type stride_B;
+    gko::size_type stride_C;
+};
+
+
+gko::size_type get_optional(rapidjson::Value &obj, const char *name,
+                            gko::size_type default_value)
+{
+    if (obj.HasMember(name)) {
+        return obj[name].GetUint64();
+    } else {
+        return default_value;
+    }
+}
+
+
+dimensions parse_dims(rapidjson::Value &test_case)
+{
+    dimensions result;
+    result.n = test_case["n"].GetInt64();
+    result.k = get_optional(test_case, "k", result.n);
+    result.m = get_optional(test_case, "m", result.n);
+    result.r = get_optional(test_case, "r", 1);
+    if (test_case.HasMember("stride")) {
+        result.stride_x = test_case["stride"].GetInt64();
+        result.stride_y = result.stride_x;
+    } else {
+        result.stride_x = get_optional(test_case, "stride_x", result.r);
+        result.stride_y = get_optional(test_case, "stride_y", result.r);
+    }
+    result.stride_A = get_optional(test_case, "stride_A", result.k);
+    result.stride_B = get_optional(test_case, "stride_B", result.m);
+    result.stride_C = get_optional(test_case, "stride_C", result.m);
+    return result;
+}
+
+
+std::map<std::string, std::function<std::unique_ptr<BenchmarkOperation>(
+                          std::shared_ptr<const gko::Executor>, dimensions)>>
+    operation_map{
+        {"copy",
+         [](std::shared_ptr<const gko::Executor> exec, dimensions dims) {
+             return std::make_unique<CopyOperation>(
+                 exec, dims.n, dims.r, dims.stride_x, dims.stride_y);
+         }},
+        {"axpy",
+         [](std::shared_ptr<const gko::Executor> exec, dimensions dims) {
+             return std::make_unique<AxpyOperation>(
+                 exec, dims.n, dims.r, dims.stride_x, dims.stride_y, false);
+         }},
+        {"multiaxpy",
+         [](std::shared_ptr<const gko::Executor> exec, dimensions dims) {
+             return std::make_unique<AxpyOperation>(
+                 exec, dims.n, dims.r, dims.stride_x, dims.stride_y, true);
+         }},
+        {"scal",
+         [](std::shared_ptr<const gko::Executor> exec, dimensions dims) {
+             return std::make_unique<ScalOperation>(exec, dims.n, dims.r,
+                                                    dims.stride_y, false);
+         }},
+        {"multiscal",
+         [](std::shared_ptr<const gko::Executor> exec, dimensions dims) {
+             return std::make_unique<ScalOperation>(exec, dims.n, dims.r,
+                                                    dims.stride_y, true);
+         }},
+        {"dot",
+         [](std::shared_ptr<const gko::Executor> exec, dimensions dims) {
+             return std::make_unique<DotOperation>(
+                 exec, dims.n, dims.r, dims.stride_x, dims.stride_y);
+         }},
+        {"norm",
+         [](std::shared_ptr<const gko::Executor> exec, dimensions dims) {
+             return std::make_unique<NormOperation>(exec, dims.n, dims.r,
+                                                    dims.stride_y);
+         }},
+        {"mm",
+         [](std::shared_ptr<const gko::Executor> exec, dimensions dims) {
+             return std::make_unique<ApplyOperation>(
+                 exec, dims.n, dims.k, dims.m, dims.stride_A, dims.stride_B,
+                 dims.stride_C);
+         }},
+        {"gemm",
+         [](std::shared_ptr<const gko::Executor> exec, dimensions dims) {
+             return std::make_unique<AdvancedApplyOperation>(
+                 exec, dims.n, dims.k, dims.m, dims.stride_A, dims.stride_B,
+                 dims.stride_C);
+         }}};
+
+
+void apply_blas(const char *operation_name, std::shared_ptr<gko::Executor> exec,
+                rapidjson::Value &test_case,
+                rapidjson::MemoryPoolAllocator<> &allocator)
+{
+    try {
+        auto &blas_case = test_case["blas"];
+        add_or_set_member(blas_case, operation_name,
+                          rapidjson::Value(rapidjson::kObjectType), allocator);
+
+        auto op = operation_map[operation_name](exec, parse_dims(test_case));
+
+        // warm run
+        for (unsigned int i = 0; i < FLAGS_warmup; i++) {
+            exec->synchronize();
+            op->run();
+            exec->synchronize();
+        }
+
+        // timed run
+        auto timer = get_timer(exec, FLAGS_gpu_timer);
+        for (unsigned int i = 0; i < FLAGS_repetitions; i++) {
+            exec->synchronize();
+            timer->tic();
+            op->run();
+            timer->toc();
+        }
+        auto runtime = timer->compute_average_time();
+        auto flops = static_cast<double>(op->get_flops());
+        auto mem = static_cast<double>(op->get_memory());
+        add_or_set_member(blas_case[operation_name], "time", runtime,
+                          allocator);
+        add_or_set_member(blas_case[operation_name], "flops", flops / runtime,
+                          allocator);
+        add_or_set_member(blas_case[operation_name], "bandwidth", mem / runtime,
+                          allocator);
+
+        // compute and write benchmark data
+        add_or_set_member(blas_case[operation_name], "completed", true,
+                          allocator);
+    } catch (const std::exception &e) {
+        add_or_set_member(test_case["blas"][operation_name], "completed", false,
+                          allocator);
+        std::cerr << "Error when processing test case " << test_case << "\n"
+                  << "what(): " << e.what() << std::endl;
+    }
+}
+
+
+int main(int argc, char *argv[])
+{
+    std::string header =
+        "A benchmark for measuring performance of Ginkgo's BLAS-like "
+        "operations.\n";
+    std::string format = std::string() + "  [\n    { \"n\": 100 },\n" +
+                         "    { \"n\": 200, \"m\": 200, \"k\": 200 }\n" +
+                         "  ]\n\n";
+    initialize_argument_parsing(&argc, &argv, header, format);
+
+    std::string extra_information = "The operations are " + FLAGS_operations;
+    print_general_information(extra_information);
+
+    auto exec = executor_factory.at(FLAGS_executor)();
+    auto engine = get_engine();
+    auto operations = split(FLAGS_operations, ',');
+
+    rapidjson::IStreamWrapper jcin(std::cin);
+    rapidjson::Document test_cases;
+    test_cases.ParseStream(jcin);
+    if (!test_cases.IsArray()) {
+        std::cerr
+            << "Input has to be a JSON array of benchmark configurations:\n"
+            << format;
+        std::exit(1);
+    }
+
+    auto &allocator = test_cases.GetAllocator();
+
+    for (auto &test_case : test_cases.GetArray()) {
+        try {
+            // set up benchmark
+            if (!test_case.HasMember("blas")) {
+                test_case.AddMember("blas",
+                                    rapidjson::Value(rapidjson::kObjectType),
+                                    allocator);
+            }
+            auto &blas_case = test_case["blas"];
+            if (!FLAGS_overwrite &&
+                all_of(begin(operations), end(operations),
+                       [&blas_case](const std::string &s) {
+                           return blas_case.HasMember(s.c_str());
+                       })) {
+                continue;
+            }
+            std::clog << "Running test case: " << test_case << std::endl;
+
+            for (const auto &operation_name : operations) {
+                apply_blas(operation_name.c_str(), exec, test_case, allocator);
+                std::clog << "Current state:" << std::endl
+                          << test_cases << std::endl;
+                backup_results(test_cases);
+            }
+        } catch (const std::exception &e) {
+            std::cerr << "Error setting up benchmark, what(): " << e.what()
+                      << std::endl;
+        }
+    }
+
+    std::cout << test_cases << std::endl;
+}

From 8cf71ed6c7a86f62e0eebc02971b8d7c88c64726 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sun, 23 May 2021 21:24:05 +0200
Subject: [PATCH 0712/1008] fix windows range build issues

---
 include/ginkgo/core/base/range.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/ginkgo/core/base/range.hpp b/include/ginkgo/core/base/range.hpp
index 2fe87a364a7..e0ea8bcf284 100644
--- a/include/ginkgo/core/base/range.hpp
+++ b/include/ginkgo/core/base/range.hpp
@@ -342,7 +342,7 @@ class range {
         const -> decltype(std::declval<accessor>()(
             std::forward<DimensionTypes>(dimensions)...))
     {
-        static_assert(sizeof...(dimensions) <= dimensionality,
+        static_assert(sizeof...(DimensionTypes) <= dimensionality,
                       "Too many dimensions in range call");
         return accessor_(std::forward<DimensionTypes>(dimensions)...);
     }

From 3e52389023899952cbe46dea1a196337a09466aa Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 25 May 2021 11:57:28 +0200
Subject: [PATCH 0713/1008] review updates

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
---
 benchmark/blas/blas.cpp | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/benchmark/blas/blas.cpp b/benchmark/blas/blas.cpp
index 1f9a7325a2f..1934aac853a 100644
--- a/benchmark/blas/blas.cpp
+++ b/benchmark/blas/blas.cpp
@@ -69,9 +69,6 @@ class BenchmarkOperation {
 public:
     virtual ~BenchmarkOperation() = default;
 
-    virtual void write_error(rapidjson::Value &blas_case,
-                             rapidjson::MemoryPoolAllocator<> &allocator)
-    {}
     virtual gko::size_type get_flops() const = 0;
     virtual gko::size_type get_memory() const = 0;
     virtual void prepare(){};
@@ -442,6 +439,7 @@ void apply_blas(const char *operation_name, std::shared_ptr<gko::Executor> exec,
 
         // warm run
         for (unsigned int i = 0; i < FLAGS_warmup; i++) {
+            op->prepare();
             exec->synchronize();
             op->run();
             exec->synchronize();
@@ -450,6 +448,7 @@ void apply_blas(const char *operation_name, std::shared_ptr<gko::Executor> exec,
         // timed run
         auto timer = get_timer(exec, FLAGS_gpu_timer);
         for (unsigned int i = 0; i < FLAGS_repetitions; i++) {
+            op->prepare();
             exec->synchronize();
             timer->tic();
             op->run();
@@ -481,7 +480,17 @@ int main(int argc, char *argv[])
 {
     std::string header =
         "A benchmark for measuring performance of Ginkgo's BLAS-like "
-        "operations.\n";
+        "operations.\nParameters for a benchmark case are:\n"
+        "    n: number of rows for vectors and gemm output (required)\n"
+        "    r: number of columns for vectors (optional, default 1)\n"
+        "    m: number of columns for gemm output (optional, default n)\n"
+        "    k: inner dimension of the gemm (optional, default n)\n"
+        "    stride: storage stride for both vectors (optional, default r)\n"
+        "    stride_x: stride for input vector x (optional, default r)\n"
+        "    stride_y: stride for in/out vector y (optional, default r)\n"
+        "    stride_A: stride for A matrix in gemm (optional, default k)\n"
+        "    stride_B: stride for B matrix in gemm (optional, default m)\n"
+        "    stride_C: stride for C matrix in gemm (optional, default m)\n";
     std::string format = std::string() + "  [\n    { \"n\": 100 },\n" +
                          "    { \"n\": 200, \"m\": 200, \"k\": 200 }\n" +
                          "  ]\n\n";

From b5df464e731b8f25540dc8c9aa62d5a31960dc77 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 3 May 2021 03:00:50 +0800
Subject: [PATCH 0714/1008] initial cooperative group work

---
 dev_tools/scripts/format_header.sh            |  16 +-
 dpcpp/base/config.hpp                         |  75 +++
 dpcpp/base/dim3.dp.hpp                        |  61 +++
 dpcpp/base/dpct.hpp                           |  50 ++
 dpcpp/components/cooperative_groups.dp.hpp    | 512 ++++++++++++++++++
 .../cooperative_groups_kernels.dp.cpp         | 198 +++++++
 dpcpp/test/utils.hpp                          |  54 ++
 7 files changed, 959 insertions(+), 7 deletions(-)
 create mode 100644 dpcpp/base/config.hpp
 create mode 100644 dpcpp/base/dim3.dp.hpp
 create mode 100644 dpcpp/base/dpct.hpp
 create mode 100644 dpcpp/components/cooperative_groups.dp.hpp
 create mode 100644 dpcpp/test/components/cooperative_groups_kernels.dp.cpp
 create mode 100644 dpcpp/test/utils.hpp

diff --git a/dev_tools/scripts/format_header.sh b/dev_tools/scripts/format_header.sh
index 1b2b6f76966..09cb00cd939 100755
--- a/dev_tools/scripts/format_header.sh
+++ b/dev_tools/scripts/format_header.sh
@@ -50,7 +50,7 @@ remove_regroup () {
 #   - CoreSuffix: "core_suffix_regex"           (default "")
 #   - PathPrefix: "path_prefix_regex"           (default "")
 #   - PathIgnore: "path_ignore_number"          (default "0")
-#   - RemoveTest: "false/true"                  (default "test")
+#   - RemoveTest: "false/true"                  (default "false")
 #   - FixInclude: "the specific main header"    (default "")
 # Only "file_regex" without any setting is fine, and it means find the same name with header suffix
 # For example, /path/to/file.cpp will change to /path/to/file.hpp
@@ -121,6 +121,7 @@ GINKGO_LICENSE_BEACON="******************************<GINKGO LICENSE>***********
 
 CONTENT="content.cpp" # Store the residual part (start from namespace)
 BEFORE="before.cpp" # Store the main header and the #ifdef/#define of header file
+BEGIN="begin.cpp" # Store the header before license
 HAS_HIP_RUNTIME="false"
 DURING_LICENSE="false"
 INCLUDE_REGEX="^#include.*"
@@ -154,12 +155,13 @@ CONSIDER_REGEX="${START_BLOCK_REX}|${END_BLOCK_REX}|${COMMENT_REGEX}|${INCLUDE_R
 
 # This part capture the main header and give the possible fail arrangement information
 while IFS='' read -r line || [ -n "$line" ]; do
-    if [ "${line}" = '#include "hip/hip_runtime.h"' ] && [ "${SKIP}" = "true" ]; then
-        HAS_HIP_RUNTIME="true"
-    elif [ "${line}" = "/*${GINKGO_LICENSE_BEACON}" ] || [ "${DURING_LICENSE}" = "true" ]; then
+    # if [ "${line}" = '#include "hip/hip_runtime.h"' ] && [ "${SKIP}" = "true" ]; then
+    #     HAS_HIP_RUNTIME="true"
+    if [ "${line}" = "/*${GINKGO_LICENSE_BEACON}" ] || [ "${DURING_LICENSE}" = "true" ]; then
         DURING_LICENSE="true"
         if [ "${line}" = "${GINKGO_LICENSE_BEACON}*/" ]; then
             DURING_LICENSE="false"
+            SKIP="true"
         fi
     elif [ "${SKIP}" = "true" ] && ([ "$line" = "${FORCE_TOP_ON}" ] || [ "${DURING_FORCE_TOP}" = "true" ]); then
         DURING_FORCE_TOP="true"
@@ -266,9 +268,9 @@ fi
 # Arrange the remain files and give
 if [ -f "${CONTENT}" ]; then
     add_regroup
-    if [ "${HAS_HIP_RUNTIME}" = "true" ]; then
-        echo "#include <hip/hip_runtime.h>" > temp
-    fi
+    # if [ "${HAS_HIP_RUNTIME}" = "true" ]; then
+    #     echo "#include <hip/hip_runtime.h>" > temp
+    # fi
     head -n -${KEEP_LINES} ${CONTENT} >> temp
     if [ ! -z "${IFNDEF}" ] && [ ! -z "${DEFINE}" ]; then
         # Ignore the last line #endif
diff --git a/dpcpp/base/config.hpp b/dpcpp/base/config.hpp
new file mode 100644
index 00000000000..71639405d8d
--- /dev/null
+++ b/dpcpp/base/config.hpp
@@ -0,0 +1,75 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_BASE_CONFIG_HPP_
+#define GKO_DPCPP_BASE_CONFIG_HPP_
+
+
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+struct config {
+    /**
+     * The type containing a bitmask over all lanes of a warp.
+     */
+    using lane_mask_type = uint64;
+
+    /**
+     * The number of threads within a DPCPP warp.
+     */
+    static constexpr uint32 warp_size = 32;
+
+    /**
+     * The bitmask of the entire warp.
+     */
+    static constexpr auto full_lane_mask = ~zero<lane_mask_type>();
+
+    /**
+     * The minimal amount of warps that need to be scheduled for each block
+     * to maximize GPU occupancy.
+     */
+    static constexpr uint32 min_warps_per_block = 4;
+};
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_DPCPP_BASE_CONFIG_HPP_
diff --git a/dpcpp/base/dim3.dp.hpp b/dpcpp/base/dim3.dp.hpp
new file mode 100644
index 00000000000..267f0f35b73
--- /dev/null
+++ b/dpcpp/base/dim3.dp.hpp
@@ -0,0 +1,61 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef DPCPP_BASE_DIM3_DP_HPP_
+#define DPCPP_BASE_DIM3_DP_HPP_
+
+
+#include <CL/sycl.hpp>
+
+
+struct dim3 {
+    unsigned int x;
+    unsigned int y;
+    unsigned int z;
+
+    dim3(unsigned int xval, unsigned int yval = 1, unsigned int zval = 1)
+        : x(xval), y(yval), z(zval)
+    {}
+
+    sycl::range<3> reverse() { return sycl::range<3>(z, y, x); }
+};
+
+
+sycl::nd_range<3> sycl_nd_range(dim3 grid, dim3 block)
+{
+    auto local_range = block.reverse();
+    auto global_range = grid.reverse() * local_range;
+    return sycl::nd_range<3>(global_range, local_range)
+}
+
+
+#endif  // DPCPP_BASE_DIM3_DP_HPP_
diff --git a/dpcpp/base/dpct.hpp b/dpcpp/base/dpct.hpp
new file mode 100644
index 00000000000..a75800be5ee
--- /dev/null
+++ b/dpcpp/base/dpct.hpp
@@ -0,0 +1,50 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef DPCPP_BASE_DPCT_HPP_
+#define DPCPP_BASE_DPCT_HPP_
+
+
+#include <CL/sycl.hpp>
+
+
+// This is partial extraction from dpct/dpct.hpp of Intel
+#if defined(_MSC_VER)
+#define __dpct_align__(n) __declspec(align(n))
+#define __dpct_inline__ __forceinline
+#else
+#define __dpct_align__(n) __attribute__((aligned(n)))
+#define __dpct_inline__ __inline__ __attribute__((always_inline))
+#endif
+
+
+#endif  // DPCPP_BASE_DPCT_HPP_
diff --git a/dpcpp/components/cooperative_groups.dp.hpp b/dpcpp/components/cooperative_groups.dp.hpp
new file mode 100644
index 00000000000..ed503e463bb
--- /dev/null
+++ b/dpcpp/components/cooperative_groups.dp.hpp
@@ -0,0 +1,512 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_COMPONENTS_COOPERATIVE_GROUPS_DP_HPP_
+#define GKO_DPCPP_COMPONENTS_COOPERATIVE_GROUPS_DP_HPP_
+
+
+#include <type_traits>
+
+
+#include "dpcpp/base/config.hpp"
+#include "dpcpp/base/dpct.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+/**
+ * Ginkgo uses cooperative groups to handle communication among the threads.
+ *
+ * However, DPCPP's implementation of cooperative groups is still quite limited
+ * in functionality, and some parts are not supported on all hardware
+ * interesting for Ginkgo. For this reason, Ginkgo exposes only a part of the
+ * original functionality, and possibly extends it if it is required. Thus,
+ * developers should include and use this header and the gko::group namespace
+ * instead of the standard cooperative_groups.h header. The interface exposed
+ * by Ginkgo's implementation is equivalent to the standard interface, with some
+ * useful extensions.
+ *
+ * A cooperative group (both from standard DPCPP and from Ginkgo) is not a
+ * specific type, but a concept. That is, any type satisfying the interface
+ * imposed by the cooperative groups API is considered a cooperative
+ * group (a.k.a. "duck typing"). To maximize the generality of components that
+ * need cooperative groups, instead of creating the group manually, consider
+ * requesting one as an input parameter. Make sure its type is a template
+ * parameter to maximize the set of groups for which your algorithm can be
+ * invoked. To maximize the amount of contexts in which your algorithm can be
+ * called and avoid hidden requirements, do not depend on a specific setup of
+ * kernel launch parameters (i.e. grid dimensions and block dimensions).
+ * Instead, use the thread_rank() method of the group to distinguish between
+ * distinct threads of a group.
+ *
+ * The original DPCPP implementation does not provide ways to verify if a
+ * certain type represents a cooperative group. Ginkgo's implementation provides
+ * metafunctions which do that. Additionally, not all cooperative groups have
+ * equivalent functionality, so Ginkgo splits the cooperative group concept into
+ * three sub-concepts which describe what functionality is available. Here is a
+ * list of concepts and their interfaces:
+ *
+ * ```c++
+ * concept Group {
+ *   unsigned size() const;
+ *   unsigned thread_rank() const;
+ * };
+ *
+ * concept SynchronizableGroup : Group {
+ *   void sync();
+ * };
+ *
+ * concept CommunicatorGroup : SynchronizableGroup {
+ *   template <typename T>
+ *   T shfl(T var, int srcLane);
+ *   T shfl_up(T var, unsigned delta);
+ *   T shfl_down(T var, unsigned delta);
+ *   T shfl_xor(T var, int laneMask);
+ *   int all(int predicate);
+ *   int any(int predicate);
+ *   unsigned ballot(int predicate);
+ * };
+ * ```
+ *
+ * To check if a group T satisfies one of the concepts, one can use the
+ * metafunctions is_group<T>::value, is_synchronizable_group<T>::value and
+ * is_communicator_group<T>::value.
+ *
+ * @note Please note that the current implementation of cooperative groups
+ *       contains only a subset of functionalities provided by those APIs. If
+ *       you need more functionality, please add the appropriate implementations
+ *       to existing cooperative groups, or create new groups if the existing
+ *       groups do not cover your use-case. For an example, see the
+ *       enable_extended_shuffle mixin, which adds extended shuffles support
+ *       to built-in DPCPP cooperative groups.
+ */
+namespace group {
+
+
+// metafunctions
+namespace detail {
+
+
+template <typename T>
+struct is_group_impl : std::false_type {};
+
+
+template <typename T>
+struct is_synchronizable_group_impl : std::false_type {};
+
+
+template <typename T>
+struct is_communicator_group_impl : std::true_type {};
+
+
+}  // namespace detail
+
+
+/**
+ * Check if T is a Group.
+ */
+template <typename T>
+using is_group = detail::is_group_impl<std::decay_t<T>>;
+
+
+/**
+ * Check if T is a SynchronizableGroup.
+ */
+template <typename T>
+using is_synchronizable_group =
+    detail::is_synchronizable_group_impl<std::decay_t<T>>;
+
+
+/**
+ * Check if T is a CommunicatorGroup.
+ */
+template <typename T>
+using is_communicator_group =
+    detail::is_communicator_group_impl<std::decay_t<T>>;
+
+
+// types
+namespace detail {
+
+
+/**
+ * This is a limited implementation of the DPCPP thread_block_tile.
+ */
+template <unsigned Size>
+class thread_block_tile : public cl::sycl::ONEAPI::sub_group {
+    using sub_group = cl::sycl::ONEAPI::sub_group;
+    using id_type = sub_group::id_type;
+    using mask_type = config::lane_mask_type;
+
+public:
+    // note: intel calls nd_item.get_sub_group(), but it still call
+    // intel::sub_group() to create the sub_group.
+    template <typename Group>
+    explicit thread_block_tile(const Group &parent_group)
+        : data_{Size, 0}, sub_group()
+    {
+#ifndef NDEBUG
+        assert(this->get_local_range().get(0) == Size);
+#endif
+        data_.rank = this->get_local_id();
+    }
+
+
+    __dpct_inline__ unsigned thread_rank() const noexcept { return data_.rank; }
+
+    __dpct_inline__ unsigned size() const noexcept { return Size; }
+
+    __dpct_inline__ void sync() const noexcept {}
+
+#define GKO_BIND_SHFL(ShflOpName, ShflOp)                                      \
+    template <typename ValueType, typename SelectorType>                       \
+    __dpct_inline__ ValueType ShflOpName(ValueType var, SelectorType selector) \
+        const noexcept                                                         \
+    {                                                                          \
+        return this->ShflOp(var, selector);                                    \
+    }                                                                          \
+    static_assert(true,                                                        \
+                  "This assert is used to counter the false positive extra "   \
+                  "semi-colon warnings")
+
+    GKO_BIND_SHFL(shfl, shuffle);
+    GKO_BIND_SHFL(shfl_xor, shuffle_xor);
+
+    // the shfl_up of out-of-range value gives undetermined behavior, we
+    // manually set it as the original value such that give the same result as
+    // cuda/hip.
+    template <typename ValueType, typename SelectorType>
+    __dpct_inline__ ValueType shfl_up(ValueType var,
+                                      SelectorType selector) const noexcept
+    {
+        const auto result = this->shuffle_up(var, selector);
+        return (data_.rank < selector) ? var : result;
+    }
+
+    // the shfl_down of out-of-range value gives undetermined behavior, we
+    // manually set it as the original value such that give the same result as
+    // cuda/hip.
+    template <typename ValueType, typename SelectorType>
+    __dpct_inline__ ValueType shfl_down(ValueType var,
+                                        SelectorType selector) const noexcept
+    {
+        const auto result = this->shuffle_down(var, selector);
+        return (data_.rank + selector >= Size) ? var : result;
+    }
+
+    /**
+     * Returns a bitmask containing the value of the given predicate
+     * for all threads in the group.
+     * This means that the ith bit is equal to the predicate of the
+     * thread with thread_rank() == i in the group.
+     * Note that the whole group needs to execute the same operation.
+     */
+    __dpct_inline__ mask_type ballot(int predicate) const noexcept
+    {
+        // todo: change it when OneAPI update the mask related api
+        return cl::sycl::ONEAPI::reduce(
+            *this, (predicate != 0) ? mask_type(1) << data_.rank : mask_type(0),
+            cl::sycl::ONEAPI::plus<mask_type>());
+    }
+
+    /**
+     * Returns true iff the predicate is true for at least one threads in the
+     * group. Note that the whole group needs to execute the same operation.
+     */
+    __dpct_inline__ bool any(int predicate) const noexcept
+    {
+        return cl::sycl::ONEAPI::any_of(*this, (predicate != 0));
+    }
+
+    /**
+     * Returns true iff the predicate is true for all threads in the group.
+     * Note that the whole group needs to execute the same operation.
+     */
+    __dpct_inline__ bool all(int predicate) const noexcept
+    {
+        return cl::sycl::ONEAPI::all_of(*this, (predicate != 0));
+    }
+
+
+private:
+    struct alignas(8) {
+        unsigned size;
+        unsigned rank;
+    } data_;
+};
+
+
+// specialization for 1
+template <>
+class thread_block_tile<1> {
+    using mask_type = config::lane_mask_type;
+    static constexpr unsigned Size = 1;
+
+public:
+    template <typename Group>
+    explicit thread_block_tile(const Group &parent_group) : data_{Size, 0}
+    {}
+
+
+    __dpct_inline__ unsigned thread_rank() const noexcept { return data_.rank; }
+
+    __dpct_inline__ unsigned size() const noexcept { return Size; }
+
+    __dpct_inline__ void sync() const noexcept {}
+
+
+#define GKO_DISABLE_SHFL(ShflOpName)                                           \
+    template <typename ValueType, typename SelectorType>                       \
+    __dpct_inline__ ValueType ShflOpName(ValueType var, SelectorType selector) \
+        const noexcept                                                         \
+    {                                                                          \
+        return var;                                                            \
+    }                                                                          \
+    static_assert(true,                                                        \
+                  "This assert is used to counter the false positive extra "   \
+                  "semi-colon warnings")
+
+    GKO_DISABLE_SHFL(shfl);
+    GKO_DISABLE_SHFL(shfl_up);
+    GKO_DISABLE_SHFL(shfl_down);
+    GKO_DISABLE_SHFL(shfl_xor);
+
+    /**
+     * Returns a bitmask containing the value of the given predicate
+     * for all threads in the group.
+     * This means that the ith bit is equal to the predicate of the
+     * thread with thread_rank() == i in the group.
+     * Note that the whole group needs to execute the same operation.
+     */
+    __dpct_inline__ mask_type ballot(int predicate) const noexcept
+    {
+        return (predicate != 0) ? mask_type(1) << data_.rank : mask_type(0);
+    }
+
+    /**
+     * Returns true iff the predicate is true for at least one threads in the
+     * group. Note that the whole group needs to execute the same operation.
+     */
+    __dpct_inline__ bool any(int predicate) const noexcept
+    {
+        return (predicate != 0);
+    }
+
+    /**
+     * Returns true iff the predicate is true for all threads in the group.
+     * Note that the whole group needs to execute the same operation.
+     */
+    __dpct_inline__ bool all(int predicate) const noexcept
+    {
+        return (predicate != 0);
+    }
+
+
+private:
+    struct alignas(8) {
+        unsigned size;
+        unsigned rank;
+    } data_;
+};
+
+
+}  // namespace detail
+
+
+using detail::thread_block_tile;
+
+
+// Only support tile_partition with 8, 16, 32.
+template <unsigned Size, typename Group>
+    __dpct_inline__ std::enable_if_t<Size> 1 && Size <= 64 &&
+    (Size & (Size - 1)),
+    detail::thread_block_tile<Size>> tiled_partition
+        [[intel::reqd_sub_group_size(Size)]] (const Group &group)
+{
+    return detail::thread_block_tile<Size>(group);
+}
+
+
+template <unsigned Size, typename Group>
+__dpct_inline__ std::enable_if_t<Size == 1, detail::thread_block_tile<Size>>
+tiled_partition(const Group &group)
+{
+    return detail::thread_block_tile<Size>(group);
+}
+
+
+namespace detail {
+
+
+template <unsigned Size>
+struct is_group_impl<thread_block_tile<Size>> : std::true_type {};
+
+
+template <unsigned Size>
+struct is_synchronizable_group_impl<thread_block_tile<Size>> : std::true_type {
+};
+
+
+template <unsigned Size>
+struct is_communicator_group_impl<thread_block_tile<Size>> : std::true_type {};
+
+
+}  // namespace detail
+
+
+class thread_block {
+    friend __dpct_inline__ thread_block this_thread_block(sycl::nd_item<3> &);
+
+public:
+    __dpct_inline__ unsigned thread_rank() const noexcept { return data_.rank; }
+
+    __dpct_inline__ unsigned size() const noexcept { return data_.size; }
+
+    __dpct_inline__ void sync() const noexcept { group_.barrier(); }
+
+private:
+    __dpct_inline__ thread_block(sycl::nd_item<3> &group)
+        : group_{group},
+          data_{static_cast<unsigned>(group.get_local_range().size()),
+                static_cast<unsigned>(group.get_local_linear_id())}
+    {}
+    struct alignas(8) {
+        unsigned size;
+        unsigned rank;
+    } data_;
+
+    sycl::nd_item<3> &group_;
+};
+
+
+__dpct_inline__ thread_block this_thread_block(sycl::nd_item<3> &group)
+{
+    return thread_block(group);
+}
+
+
+namespace detail {
+
+
+template <>
+struct is_group_impl<thread_block> : std::true_type {};
+
+
+template <>
+struct is_synchronizable_group_impl<thread_block> : std::true_type {};
+
+
+}  // namespace detail
+
+
+/**
+ * This is a limited implementation of the DPCPP grid_group that works even on
+ * devices that do not support device-wide synchronization and without special
+ * kernel launch syntax.
+ *
+ * Note that this implementation does not support large grids, since it uses 32
+ * bits to represent sizes and ranks, while at least 73 bits (63 bit grid + 10
+ * bit block) would have to be used to represent the full space of thread ranks.
+ */
+class grid_group {
+    friend __dpct_inline__ grid_group this_grid(sycl::nd_item<3> &);
+
+public:
+    __dpct_inline__ unsigned size() const noexcept { return data_.size; }
+
+    __dpct_inline__ unsigned thread_rank() const noexcept { return data_.rank; }
+
+private:
+    __dpct_inline__ grid_group(sycl::nd_item<3> &group)
+        : data_{static_cast<unsigned>(group.get_global_range().size()),
+                static_cast<unsigned>(group.get_global_linear_id())}
+    {}
+
+    struct alignas(8) {
+        unsigned size;
+        unsigned rank;
+    } data_;
+};
+
+// Not using this, as grid_group is not universally supported.
+// grid_group this_grid()
+// using cooperative_groups::this_grid;
+// Instead, use our limited implementation:
+__dpct_inline__ grid_group this_grid(sycl::nd_item<3> &group)
+{
+    return grid_group(group);
+}
+
+
+}  // namespace group
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+// Enable group can directly use group function
+__SYCL_INLINE_NAMESPACE(cl)
+{
+    namespace sycl {
+    namespace detail {
+
+
+    template <unsigned Size>
+    struct is_sub_group<
+        ::gko::kernels::dpcpp::group::detail::thread_block_tile<Size>>
+        : std::true_type {};
+
+
+    namespace spirv {
+
+
+    template <typename Group>
+    struct group_scope;
+
+    template <unsigned Size>
+    struct group_scope<
+        ::gko::kernels::dpcpp::group::detail::thread_block_tile<Size>> {
+        static constexpr __spv::Scope::Flag value =
+            __spv::Scope::Flag::Subgroup;
+    };
+
+
+    }  // namespace spirv
+    }  // namespace detail
+    }  // namespace sycl
+}  // __SYCL_INLINE_NAMESPACE(cl)
+
+
+#endif  // GKO_DPCPP_COMPONENTS_COOPERATIVE_GROUPS_DP_HPP_
diff --git a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
new file mode 100644
index 00000000000..93b21295086
--- /dev/null
+++ b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
@@ -0,0 +1,198 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "dpcpp/components/cooperative_groups.dp.hpp"
+
+
+#include <memory>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include "dpcpp/base/config.hpp"
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/test/utils.hpp"
+
+
+namespace {
+
+
+using namespace gko::kernels::dpcpp;
+
+
+class CooperativeGroups : public ::testing::Test {
+protected:
+    CooperativeGroups()
+        : ref(gko::ReferenceExecutor::create()),
+          dpcpp(gko::DpcppExecutor::create(0, ref)),
+          result(ref, 1),
+          dresult(dpcpp)
+    {
+        *result.get_data() = true;
+        dresult = result;
+    }
+
+    template <typename Kernel>
+    void test(Kernel kernel)
+    {
+        // functioname kernel
+        kernel(1, config::warp_size, 0, dpcpp->get_queue(), dresult.get_data());
+        result = dresult;
+        auto success = *result.get_const_data();
+
+        ASSERT_TRUE(success);
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<gko::DpcppExecutor> dpcpp;
+    gko::Array<bool> result;
+    gko::Array<bool> dresult;
+};
+
+
+void test_assert(bool *success, bool partial)
+{
+    if (!partial) {
+        *success = false;
+    }
+}
+
+void cg_shuffle(bool *s, sycl::nd_item<3> item_ct1)
+{
+    auto group = group::tiled_partition<config::warp_size>(
+        group::this_thread_block(item_ct1));
+    auto i = int(group.thread_rank());
+    test_assert(s, group.shfl_up(i, 1) == sycl::max(0, (int)(i - 1)));
+    test_assert(s, group.shfl_down(i, 1) ==
+                       sycl::min((unsigned int)(i + 1),
+                                 (unsigned int)(config::warp_size - 1)));
+    test_assert(s, group.shfl(i, 0) == 0);
+}
+
+void cg_shuffle_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                     sycl::queue *stream, bool *s)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block),
+            [=](sycl::nd_item<3> item_ct1) { cg_shuffle(s, item_ct1); });
+    });
+}
+
+TEST_F(CooperativeGroups, Shuffle) { test(cg_shuffle_host); }
+
+
+void cg_all(bool *s, sycl::nd_item<3> item_ct1)
+{
+    auto group = group::tiled_partition<config::warp_size>(
+        group::this_thread_block(item_ct1));
+    test_assert(s, group.all(true));
+    test_assert(s, !group.all(false));
+    test_assert(s, !group.all(item_ct1.get_local_id(2) < 13));
+}
+
+void cg_all_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                 sycl::queue *stream, bool *s)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(
+            sycl::nd_range<3>(global_range, local_range),
+            [=](sycl::nd_item<3> item_ct1) { cg_all(s, item_ct1); });
+    });
+}
+
+TEST_F(CooperativeGroups, All) { test(cg_all_host); }
+
+
+void cg_any(bool *s, sycl::nd_item<3> item_ct1)
+{
+    auto group = group::tiled_partition<config::warp_size>(
+        group::this_thread_block(item_ct1));
+    test_assert(s, group.any(true));
+    test_assert(s, group.any(item_ct1.get_local_id(2) == 0));
+    test_assert(s, !group.any(false));
+}
+
+void cg_any_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                 sycl::queue *stream, bool *s)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(
+            sycl::nd_range<3>(global_range, local_range),
+            [=](sycl::nd_item<3> item_ct1) { cg_any(s, item_ct1); });
+    });
+}
+
+TEST_F(CooperativeGroups, Any) { test(cg_any_host); }
+
+
+void cg_ballot(bool *s, sycl::nd_item<3> item_ct1)
+{
+    auto group = group::tiled_partition<config::warp_size>(
+        group::this_thread_block(item_ct1));
+    auto active = ~(~config::lane_mask_type{} << config::warp_size);
+    test_assert(s, group.ballot(false) == 0);
+    test_assert(s, group.ballot(true) == (~config::lane_mask_type{} & active));
+    test_assert(s, group.ballot(item_ct1.get_local_id(2) < 4) == 0xf);
+}
+
+void cg_ballot_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                    sycl::queue *stream, bool *s)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(
+            sycl::nd_range<3>(global_range, local_range),
+            [=](sycl::nd_item<3> item_ct1) { cg_ballot(s, item_ct1); });
+    });
+}
+
+TEST_F(CooperativeGroups, Ballot) { test(cg_ballot_host); }
+
+
+}  // namespace
diff --git a/dpcpp/test/utils.hpp b/dpcpp/test/utils.hpp
new file mode 100644
index 00000000000..15dcc6103d4
--- /dev/null
+++ b/dpcpp/test/utils.hpp
@@ -0,0 +1,54 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_TEST_UTILS_HPP_
+#define GKO_DPCPP_TEST_UTILS_HPP_
+
+
+#include "core/test/utils.hpp"
+
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+namespace {
+
+
+// prevent device reset after each test
+auto no_reset_exec =
+    gko::DpcppExecutor::create(0, gko::ReferenceExecutor::create());
+
+
+}  // namespace
+
+
+#endif  // GKO_DPCPP_TEST_UTILS_HPP_

From 6eb0d453bfe04d73e176c5843599b42981a2b6f6 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 3 May 2021 17:57:01 +0800
Subject: [PATCH 0715/1008] manually add config selection

---
 .../cooperative_groups_kernels.dp.cpp         | 33 +++++++++++++++----
 include/ginkgo/core/base/executor.hpp         | 14 ++++++++
 include/ginkgo/core/base/types.hpp            | 22 +++++++++++++
 3 files changed, 63 insertions(+), 6 deletions(-)

diff --git a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
index 93b21295086..550e9fae44c 100644
--- a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
+++ b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
+#include "core/synthesizer/implementation_selection.hpp"
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
 #include "dpcpp/test/utils.hpp"
@@ -94,29 +95,49 @@ void test_assert(bool *success, bool partial)
     }
 }
 
+template <Config config>
 void cg_shuffle(bool *s, sycl::nd_item<3> item_ct1)
 {
-    auto group = group::tiled_partition<config::warp_size>(
+    auto group = group::tiled_partition<get_warp_size(config)>(
         group::this_thread_block(item_ct1));
     auto i = int(group.thread_rank());
     test_assert(s, group.shfl_up(i, 1) == sycl::max(0, (int)(i - 1)));
     test_assert(s, group.shfl_down(i, 1) ==
                        sycl::min((unsigned int)(i + 1),
-                                 (unsigned int)(config::warp_size - 1)));
+                                 (unsigned int)(get_warp_size(config) - 1)));
     test_assert(s, group.shfl(i, 0) == 0);
 }
 
+template <Config config>
 void cg_shuffle_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                      sycl::queue *stream, bool *s)
 {
     stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block),
-            [=](sycl::nd_item<3> item_ct1) { cg_shuffle(s, item_ct1); });
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             cg_shuffle<config>(s, item_ct1);
+                         });
     });
 }
 
-TEST_F(CooperativeGroups, Shuffle) { test(cg_shuffle_host); }
+GKO_ENABLE_IMPLEMENTATION_SELECTION(cg_shuffle_config, cg_shuffle_host)
+
+void cg_shuffle_config(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                       sycl::queue *stream,
+                       std::shared_ptr<const DpcppExecutor> exec, bool *s)
+{
+    auto exec_info = exec->get_exec_info();
+    constexpr auto default_config_list =
+        ::gko::syn::value_list<Config, config_set(32, 32)>();
+    cg_shuffle_config()(
+        config_list,
+        [&exec_info](Config config) { return exec_info.validate(config); },
+        ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
+        ::gko::syn::type_list<>(), grid, block, dynamic_shared_memory, stream,
+        s);
+}
+
+TEST_F(CooperativeGroups, Shuffle) { test(cg_shuffle_config); }
 
 
 void cg_all(bool *s, sycl::nd_item<3> item_ct1)
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index d9ca2e3af32..f1fe183f7dc 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -811,6 +811,20 @@ class Executor : public log::EnableLogging<Executor> {
          *       multicore processor, or a single thread in a SMT processor"
          */
         std::vector<int> closest_pu_ids{};
+
+        /**
+         * The validate function for Config
+         */
+        bool validate(Config config)
+        {
+            bool allowed = false;
+            auto blocksize = get_block_size(config);
+            auto warpsize = get_warp_size(config);
+            for (auto &i : subgroup_sizes) {
+                allowed |= (i == warpsize);
+            }
+            return allowed && (blocksize <= max_workgroup_size);
+        }
     };
 
     /**
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 2c77aaef826..36a3ff847d8 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -193,6 +193,28 @@ using default_precision = double;
 constexpr size_type byte_size = CHAR_BIT;
 
 
+/**
+ * Config type
+ */
+using Config = int;
+
+constexpr int config_scaler = 1000;
+
+constexpr Config config_set(int block_size, int warp_size)
+{
+    return block_size * config_scaler + warp_size;
+}
+
+constexpr int get_warp_size(Config config_set)
+{
+    return config_set % config_scaler;
+}
+
+constexpr int get_block_size(Config config_set)
+{
+    return config_set / config_scaler;
+}
+
 /**
  * Evaluates if all template arguments Args fulfill std::is_integral. If that is
  * the case, this class inherits from `std::true_type`, otherwise, it inherits

From ee7190baa11747f9a3a24d229a7ba274013071f1 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 4 May 2021 00:08:56 +0800
Subject: [PATCH 0716/1008] try config selection in cooperative group test

---
 cmake/create_test.cmake                       |  1 +
 core/synthesizer/implementation_selection.hpp | 33 +++++++++
 dpcpp/base/dim3.dp.hpp                        |  2 +-
 dpcpp/components/cooperative_groups.dp.hpp    |  9 +--
 dpcpp/test/components/CMakeLists.txt          |  1 +
 .../cooperative_groups_kernels.dp.cpp         | 70 +++++++++++++------
 include/ginkgo/core/base/executor.hpp         |  4 ++
 7 files changed, 95 insertions(+), 25 deletions(-)

diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index c5285b0c10a..8bca662455a 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -33,6 +33,7 @@ function(ginkgo_create_dpcpp_test test_name)
     add_executable(${TEST_TARGET_NAME} ${test_name}.dp.cpp)
     target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_17)
     target_compile_options("${TEST_TARGET_NAME}" PRIVATE "${GINKGO_DPCPP_FLAGS}")
+    target_link_options("${TEST_TARGET_NAME}" PRIVATE -fsycl-device-code-split=per_kernel)
     if (GINKGO_DPCPP_SINGLE_MODE)
         target_compile_definitions("${TEST_TARGET_NAME}" PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
     endif()
diff --git a/core/synthesizer/implementation_selection.hpp b/core/synthesizer/implementation_selection.hpp
index b8be20f5043..8c6333e8e9f 100644
--- a/core/synthesizer/implementation_selection.hpp
+++ b/core/synthesizer/implementation_selection.hpp
@@ -70,6 +70,39 @@ namespace syn {
         }                                                                    \
     }
 
+#define GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(_name, _callable)           \
+    template <typename Predicate, bool... BoolArgs, int... IntArgs,            \
+              gko::size_type... SizeTArgs, typename... TArgs,                  \
+              typename... InferredArgs>                                        \
+    inline void _name(::gko::syn::value_list<gko::Config>, Predicate,          \
+                      ::gko::syn::value_list<bool, BoolArgs...>,               \
+                      ::gko::syn::value_list<int, IntArgs...>,                 \
+                      ::gko::syn::value_list<gko::size_type, SizeTArgs...>,    \
+                      ::gko::syn::type_list<TArgs...>, InferredArgs...)        \
+        GKO_KERNEL_NOT_FOUND;                                                  \
+                                                                               \
+    template <gko::Config K, gko::Config... Rest, typename Predicate,          \
+              bool... BoolArgs, int... IntArgs, gko::size_type... SizeTArgs,   \
+              typename... TArgs, typename... InferredArgs>                     \
+    inline void _name(                                                         \
+        ::gko::syn::value_list<gko::Config, K, Rest...>,                       \
+        Predicate is_eligible,                                                 \
+        ::gko::syn::value_list<bool, BoolArgs...> bool_args,                   \
+        ::gko::syn::value_list<int, IntArgs...> int_args,                      \
+        ::gko::syn::value_list<gko::size_type, SizeTArgs...> size_args,        \
+        ::gko::syn::type_list<TArgs...> type_args, InferredArgs... args)       \
+    {                                                                          \
+        if (is_eligible(K)) {                                                  \
+            std::cout << "call " << K << std::endl;                            \
+            _callable<BoolArgs..., IntArgs..., SizeTArgs..., TArgs..., K>(     \
+                std::forward<InferredArgs>(args)...);                          \
+        } else {                                                               \
+            _name(::gko::syn::value_list<gko::Config, Rest...>(), is_eligible, \
+                  bool_args, int_args, size_args, type_args,                   \
+                  std::forward<InferredArgs>(args)...);                        \
+        }                                                                      \
+    }
+
 
 }  // namespace syn
 }  // namespace gko
diff --git a/dpcpp/base/dim3.dp.hpp b/dpcpp/base/dim3.dp.hpp
index 267f0f35b73..2bac206058f 100644
--- a/dpcpp/base/dim3.dp.hpp
+++ b/dpcpp/base/dim3.dp.hpp
@@ -54,7 +54,7 @@ sycl::nd_range<3> sycl_nd_range(dim3 grid, dim3 block)
 {
     auto local_range = block.reverse();
     auto global_range = grid.reverse() * local_range;
-    return sycl::nd_range<3>(global_range, local_range)
+    return sycl::nd_range<3>(global_range, local_range);
 }
 
 
diff --git a/dpcpp/components/cooperative_groups.dp.hpp b/dpcpp/components/cooperative_groups.dp.hpp
index ed503e463bb..64ce81bfd29 100644
--- a/dpcpp/components/cooperative_groups.dp.hpp
+++ b/dpcpp/components/cooperative_groups.dp.hpp
@@ -350,10 +350,11 @@ using detail::thread_block_tile;
 
 // Only support tile_partition with 8, 16, 32.
 template <unsigned Size, typename Group>
-    __dpct_inline__ std::enable_if_t<Size> 1 && Size <= 64 &&
-    (Size & (Size - 1)),
-    detail::thread_block_tile<Size>> tiled_partition
-        [[intel::reqd_sub_group_size(Size)]] (const Group &group)
+__dpct_inline__
+    std::enable_if_t<(Size > 1) && Size <= 64 && !(Size & (Size - 1)),
+                     detail::thread_block_tile<Size>>
+        tiled_partition
+    [[intel::reqd_sub_group_size(Size)]] (const Group &group)
 {
     return detail::thread_block_tile<Size>(group);
 }
diff --git a/dpcpp/test/components/CMakeLists.txt b/dpcpp/test/components/CMakeLists.txt
index f6ca175d105..87a034a64df 100644
--- a/dpcpp/test/components/CMakeLists.txt
+++ b/dpcpp/test/components/CMakeLists.txt
@@ -1,3 +1,4 @@
 ginkgo_create_test(absolute_array)
+ginkgo_create_dpcpp_test(cooperative_groups_kernels)
 ginkgo_create_test(fill_array)
 ginkgo_create_test(precision_conversion)
diff --git a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
index 550e9fae44c..b759a228977 100644
--- a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
+++ b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
@@ -40,7 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <gtest/gtest.h>
-
+#include <iostream>
 
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/executor.hpp>
@@ -81,6 +81,19 @@ class CooperativeGroups : public ::testing::Test {
         ASSERT_TRUE(success);
     }
 
+    template <typename Kernel>
+    void test_all_subgroup(Kernel kernel)
+    {
+        auto exec_info = dpcpp->get_const_exec_info();
+        for (auto &i : exec_info.subgroup_sizes) {
+            kernel(1, i, 0, dpcpp->get_queue(), dpcpp, dresult.get_data());
+            result = dresult;
+            auto success = *result.get_const_data();
+            ASSERT_TRUE(success);
+            std::cout << i << " success" << std::endl;
+        }
+    }
+
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<gko::DpcppExecutor> dpcpp;
     gko::Array<bool> result;
@@ -95,20 +108,24 @@ void test_assert(bool *success, bool partial)
     }
 }
 
-template <Config config>
-void cg_shuffle(bool *s, sycl::nd_item<3> item_ct1)
+// kernel implementation
+template <gko::Config config>
+[[intel::reqd_work_group_size(1, 1, gko::get_warp_size(config))]] void
+cg_shuffle(bool *s, sycl::nd_item<3> item_ct1)
 {
-    auto group = group::tiled_partition<get_warp_size(config)>(
+    auto group = group::tiled_partition<gko::get_warp_size(config)>(
         group::this_thread_block(item_ct1));
     auto i = int(group.thread_rank());
     test_assert(s, group.shfl_up(i, 1) == sycl::max(0, (int)(i - 1)));
-    test_assert(s, group.shfl_down(i, 1) ==
-                       sycl::min((unsigned int)(i + 1),
-                                 (unsigned int)(get_warp_size(config) - 1)));
+    test_assert(s,
+                group.shfl_down(i, 1) ==
+                    sycl::min((unsigned int)(i + 1),
+                              (unsigned int)(gko::get_warp_size(config) - 1)));
     test_assert(s, group.shfl(i, 0) == 0);
 }
 
-template <Config config>
+// group all kernel things together
+template <gko::Config config>
 void cg_shuffle_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                      sycl::queue *stream, bool *s)
 {
@@ -120,24 +137,37 @@ void cg_shuffle_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
     });
 }
 
-GKO_ENABLE_IMPLEMENTATION_SELECTION(cg_shuffle_config, cg_shuffle_host)
+// config selection
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_shuffle_config, cg_shuffle_host)
 
-void cg_shuffle_config(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                       sycl::queue *stream,
-                       std::shared_ptr<const DpcppExecutor> exec, bool *s)
+// the call
+void cg_shuffle_config_call(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                            sycl::queue *stream,
+                            std::shared_ptr<const gko::DpcppExecutor> exec,
+                            bool *s)
 {
-    auto exec_info = exec->get_exec_info();
+    auto exec_info = exec->get_const_exec_info();
     constexpr auto default_config_list =
-        ::gko::syn::value_list<Config, config_set(32, 32)>();
-    cg_shuffle_config()(
-        config_list,
-        [&exec_info](Config config) { return exec_info.validate(config); },
+        ::gko::syn::value_list<gko::Config, gko::config_set(32, 32),
+                               gko::config_set(16, 16), gko::config_set(8, 8),
+                               gko::config_set(4, 4)>();
+    std::cout << "block.x " << block.x << std::endl;
+    cg_shuffle_config(
+        default_config_list,
+        // validate
+        [&exec_info, &block](gko::Config config) {
+            return exec_info.validate(config) &&
+                   (gko::get_warp_size(config) == block.x);
+        },
         ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
-        ::gko::syn::type_list<>(), grid, block, dynamic_shared_memory, stream,
-        s);
+        ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
+        grid, block, dynamic_shared_memory, stream, s);
 }
 
-TEST_F(CooperativeGroups, Shuffle) { test(cg_shuffle_config); }
+TEST_F(CooperativeGroups, Shuffle)
+{
+    test_all_subgroup(cg_shuffle_config_call);
+}
 
 
 void cg_all(bool *s, sycl::nd_item<3> item_ct1)
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index f1fe183f7dc..e9c761a3bdb 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -827,6 +827,10 @@ class Executor : public log::EnableLogging<Executor> {
         }
     };
 
+public:
+    const exec_info get_const_exec_info() const { return this->exec_info_; }
+
+protected:
     /**
      * Gets the exec info struct
      *

From 4052731d0776f413537feae8e66cb57757fa091b Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 17 May 2021 15:26:17 +0200
Subject: [PATCH 0717/1008] Use general ConfigSet

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 core/synthesizer/implementation_selection.hpp | 61 +++++++++--------
 .../cooperative_groups_kernels.dp.cpp         | 30 ++++-----
 include/ginkgo/core/base/executor.hpp         |  4 +-
 include/ginkgo/core/base/types.hpp            | 66 ++++++++++++++++---
 4 files changed, 104 insertions(+), 57 deletions(-)

diff --git a/core/synthesizer/implementation_selection.hpp b/core/synthesizer/implementation_selection.hpp
index 8c6333e8e9f..998a0dbffc1 100644
--- a/core/synthesizer/implementation_selection.hpp
+++ b/core/synthesizer/implementation_selection.hpp
@@ -70,37 +70,36 @@ namespace syn {
         }                                                                    \
     }
 
-#define GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(_name, _callable)           \
-    template <typename Predicate, bool... BoolArgs, int... IntArgs,            \
-              gko::size_type... SizeTArgs, typename... TArgs,                  \
-              typename... InferredArgs>                                        \
-    inline void _name(::gko::syn::value_list<gko::Config>, Predicate,          \
-                      ::gko::syn::value_list<bool, BoolArgs...>,               \
-                      ::gko::syn::value_list<int, IntArgs...>,                 \
-                      ::gko::syn::value_list<gko::size_type, SizeTArgs...>,    \
-                      ::gko::syn::type_list<TArgs...>, InferredArgs...)        \
-        GKO_KERNEL_NOT_FOUND;                                                  \
-                                                                               \
-    template <gko::Config K, gko::Config... Rest, typename Predicate,          \
-              bool... BoolArgs, int... IntArgs, gko::size_type... SizeTArgs,   \
-              typename... TArgs, typename... InferredArgs>                     \
-    inline void _name(                                                         \
-        ::gko::syn::value_list<gko::Config, K, Rest...>,                       \
-        Predicate is_eligible,                                                 \
-        ::gko::syn::value_list<bool, BoolArgs...> bool_args,                   \
-        ::gko::syn::value_list<int, IntArgs...> int_args,                      \
-        ::gko::syn::value_list<gko::size_type, SizeTArgs...> size_args,        \
-        ::gko::syn::type_list<TArgs...> type_args, InferredArgs... args)       \
-    {                                                                          \
-        if (is_eligible(K)) {                                                  \
-            std::cout << "call " << K << std::endl;                            \
-            _callable<BoolArgs..., IntArgs..., SizeTArgs..., TArgs..., K>(     \
-                std::forward<InferredArgs>(args)...);                          \
-        } else {                                                               \
-            _name(::gko::syn::value_list<gko::Config, Rest...>(), is_eligible, \
-                  bool_args, int_args, size_args, type_args,                   \
-                  std::forward<InferredArgs>(args)...);                        \
-        }                                                                      \
+#define GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(_name, _callable)          \
+    template <typename Predicate, bool... BoolArgs, int... IntArgs,           \
+              gko::size_type... SizeTArgs, typename... TArgs,                 \
+              typename... InferredArgs>                                       \
+    inline void _name(::gko::syn::value_list<int>, Predicate,                 \
+                      ::gko::syn::value_list<bool, BoolArgs...>,              \
+                      ::gko::syn::value_list<int, IntArgs...>,                \
+                      ::gko::syn::value_list<gko::size_type, SizeTArgs...>,   \
+                      ::gko::syn::type_list<TArgs...>, InferredArgs...)       \
+        GKO_KERNEL_NOT_FOUND;                                                 \
+                                                                              \
+    template <int K, int... Rest, typename Predicate, bool... BoolArgs,       \
+              int... IntArgs, gko::size_type... SizeTArgs, typename... TArgs, \
+              typename... InferredArgs>                                       \
+    inline void _name(                                                        \
+        ::gko::syn::value_list<int, K, Rest...>, Predicate is_eligible,       \
+        ::gko::syn::value_list<bool, BoolArgs...> bool_args,                  \
+        ::gko::syn::value_list<int, IntArgs...> int_args,                     \
+        ::gko::syn::value_list<gko::size_type, SizeTArgs...> size_args,       \
+        ::gko::syn::type_list<TArgs...> type_args, InferredArgs... args)      \
+    {                                                                         \
+        if (is_eligible(K)) {                                                 \
+            std::cout << "call " << K << std::endl;                           \
+            _callable<BoolArgs..., IntArgs..., SizeTArgs..., TArgs..., K>(    \
+                std::forward<InferredArgs>(args)...);                         \
+        } else {                                                              \
+            _name(::gko::syn::value_list<int, Rest...>(), is_eligible,        \
+                  bool_args, int_args, size_args, type_args,                  \
+                  std::forward<InferredArgs>(args)...);                       \
+        }                                                                     \
     }
 
 
diff --git a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
index b759a228977..93e028cfbcd 100644
--- a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
+++ b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
@@ -56,6 +56,7 @@ namespace {
 
 
 using namespace gko::kernels::dpcpp;
+using KCfg = gko::ConfigSet<12, 7>;
 
 
 class CooperativeGroups : public ::testing::Test {
@@ -109,23 +110,22 @@ void test_assert(bool *success, bool partial)
 }
 
 // kernel implementation
-template <gko::Config config>
-[[intel::reqd_work_group_size(1, 1, gko::get_warp_size(config))]] void
-cg_shuffle(bool *s, sycl::nd_item<3> item_ct1)
+template <int config>
+[[intel::reqd_work_group_size(1, 1, KCfg::decode<0>(config))]] void cg_shuffle(
+    bool *s, sycl::nd_item<3> item_ct1)
 {
-    auto group = group::tiled_partition<gko::get_warp_size(config)>(
+    auto group = group::tiled_partition<KCfg::decode<0>(config)>(
         group::this_thread_block(item_ct1));
     auto i = int(group.thread_rank());
     test_assert(s, group.shfl_up(i, 1) == sycl::max(0, (int)(i - 1)));
-    test_assert(s,
-                group.shfl_down(i, 1) ==
-                    sycl::min((unsigned int)(i + 1),
-                              (unsigned int)(gko::get_warp_size(config) - 1)));
+    test_assert(s, group.shfl_down(i, 1) ==
+                       sycl::min((unsigned int)(i + 1),
+                                 (unsigned int)(KCfg::decode<0>(config) - 1)));
     test_assert(s, group.shfl(i, 0) == 0);
 }
 
 // group all kernel things together
-template <gko::Config config>
+template <int config>
 void cg_shuffle_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                      sycl::queue *stream, bool *s)
 {
@@ -148,16 +148,16 @@ void cg_shuffle_config_call(dim3 grid, dim3 block, size_t dynamic_shared_memory,
 {
     auto exec_info = exec->get_const_exec_info();
     constexpr auto default_config_list =
-        ::gko::syn::value_list<gko::Config, gko::config_set(32, 32),
-                               gko::config_set(16, 16), gko::config_set(8, 8),
-                               gko::config_set(4, 4)>();
+        ::gko::syn::value_list<int, KCfg::encode(32, 32), KCfg::encode(16, 16),
+                               KCfg::encode(8, 8), KCfg::encode(4, 4)>();
     std::cout << "block.x " << block.x << std::endl;
     cg_shuffle_config(
         default_config_list,
         // validate
-        [&exec_info, &block](gko::Config config) {
-            return exec_info.validate(config) &&
-                   (gko::get_warp_size(config) == block.x);
+        [&exec_info, &block](int config) {
+            return exec_info.validate(KCfg::decode<0>(config),
+                                      KCfg::decode<1>(config)) &&
+                   (KCfg::decode<1>(config) == block.x);
         },
         ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
         ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index e9c761a3bdb..b1044cf4056 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -815,11 +815,9 @@ class Executor : public log::EnableLogging<Executor> {
         /**
          * The validate function for Config
          */
-        bool validate(Config config)
+        bool validate(int blocksize, int warpsize)
         {
             bool allowed = false;
-            auto blocksize = get_block_size(config);
-            auto warpsize = get_warp_size(config);
             for (auto &i : subgroup_sizes) {
                 allowed |= (i == warpsize);
             }
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 36a3ff847d8..a166a8ac1ec 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_PUBLIC_CORE_BASE_TYPES_HPP_
 
 
+#include <array>
 #include <cassert>
 #include <climits>
 #include <complex>
@@ -196,25 +197,74 @@ constexpr size_type byte_size = CHAR_BIT;
 /**
  * Config type
  */
-using Config = int;
+namespace helper {
 
-constexpr int config_scaler = 1000;
 
-constexpr Config config_set(int block_size, int warp_size)
+template <size_type current_size>
+constexpr int mask()
 {
-    return block_size * config_scaler + warp_size;
+    return (1 << (current_size - 1)) | mask<current_size - 1>();
 }
 
-constexpr int get_warp_size(Config config_set)
+template <>
+constexpr int mask<0>()
 {
-    return config_set % config_scaler;
+    return 0;
 }
 
-constexpr int get_block_size(Config config_set)
+template <int num_groups, int current_shift>
+constexpr std::enable_if_t<(num_groups == current_shift + 1), int> shift(
+    const std::array<char, num_groups> &bits)
 {
-    return config_set / config_scaler;
+    return 0;
 }
 
+template <int num_groups, int current_shift>
+constexpr std::enable_if_t<(num_groups > current_shift + 1), int> shift(
+    const std::array<char, num_groups> &bits)
+{
+    return bits[current_shift + 1] +
+           shift<num_groups, (current_shift + 1)>(bits);
+}
+
+
+}  // namespace helper
+
+
+template <int... num_bits>
+class ConfigSet {
+public:
+    static constexpr size_type num_groups = sizeof...(num_bits);
+    static constexpr std::array<char, num_groups> bits{num_bits...};
+
+    template <int position>
+    static constexpr int decode(int encoded)
+    {
+        static_assert(position < num_groups,
+                      "This position is over the bounds.");
+        constexpr int shift = helper::shift<num_groups, position>(bits);
+        constexpr int mask = helper::mask<bits[position]>();
+        return (encoded >> shift) & mask;
+    }
+
+    template <size_type current_iter>
+    static constexpr std::enable_if_t<(current_iter == num_groups), int>
+    encode()
+    {
+        return 0;
+    }
+
+    template <size_type current_iter = 0, typename First, typename... Rest>
+    static constexpr std::enable_if_t<(current_iter < num_groups), int> encode(
+        First first, Rest &&... rest)
+    {
+        constexpr int shift = helper::shift<num_groups, current_iter>(bits);
+        return (first << shift) |
+               encode<current_iter + 1>(std::forward<Rest>(rest)...);
+    }
+};
+
+
 /**
  * Evaluates if all template arguments Args fulfill std::is_integral. If that is
  * the case, this class inherits from `std::true_type`, otherwise, it inherits

From 5049570b3b10d3db40ac7574f14356a93c3fb1a7 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 17 May 2021 22:36:20 +0200
Subject: [PATCH 0718/1008] review update and all coop test use ConfigSet

---
 dpcpp/components/cooperative_groups.dp.hpp    |  18 +-
 .../cooperative_groups_kernels.dp.cpp         | 158 ++++++++++++------
 dpcpp/test/utils.hpp                          |  54 ------
 3 files changed, 120 insertions(+), 110 deletions(-)
 delete mode 100644 dpcpp/test/utils.hpp

diff --git a/dpcpp/components/cooperative_groups.dp.hpp b/dpcpp/components/cooperative_groups.dp.hpp
index 64ce81bfd29..0c2a6e141cc 100644
--- a/dpcpp/components/cooperative_groups.dp.hpp
+++ b/dpcpp/components/cooperative_groups.dp.hpp
@@ -165,8 +165,8 @@ namespace detail {
  * This is a limited implementation of the DPCPP thread_block_tile.
  */
 template <unsigned Size>
-class thread_block_tile : public cl::sycl::ONEAPI::sub_group {
-    using sub_group = cl::sycl::ONEAPI::sub_group;
+class thread_block_tile : public sycl::ONEAPI::sub_group {
+    using sub_group = sycl::ONEAPI::sub_group;
     using id_type = sub_group::id_type;
     using mask_type = config::lane_mask_type;
 
@@ -188,7 +188,7 @@ class thread_block_tile : public cl::sycl::ONEAPI::sub_group {
 
     __dpct_inline__ unsigned size() const noexcept { return Size; }
 
-    __dpct_inline__ void sync() const noexcept {}
+    __dpct_inline__ void sync() const noexcept { this->barrier(); }
 
 #define GKO_BIND_SHFL(ShflOpName, ShflOp)                                      \
     template <typename ValueType, typename SelectorType>                       \
@@ -204,7 +204,7 @@ class thread_block_tile : public cl::sycl::ONEAPI::sub_group {
     GKO_BIND_SHFL(shfl, shuffle);
     GKO_BIND_SHFL(shfl_xor, shuffle_xor);
 
-    // the shfl_up of out-of-range value gives undetermined behavior, we
+    // the shfl_up of out-of-range value gives undefined behavior, we
     // manually set it as the original value such that give the same result as
     // cuda/hip.
     template <typename ValueType, typename SelectorType>
@@ -215,7 +215,7 @@ class thread_block_tile : public cl::sycl::ONEAPI::sub_group {
         return (data_.rank < selector) ? var : result;
     }
 
-    // the shfl_down of out-of-range value gives undetermined behavior, we
+    // the shfl_down of out-of-range value gives undefined behavior, we
     // manually set it as the original value such that give the same result as
     // cuda/hip.
     template <typename ValueType, typename SelectorType>
@@ -236,9 +236,9 @@ class thread_block_tile : public cl::sycl::ONEAPI::sub_group {
     __dpct_inline__ mask_type ballot(int predicate) const noexcept
     {
         // todo: change it when OneAPI update the mask related api
-        return cl::sycl::ONEAPI::reduce(
+        return sycl::ONEAPI::reduce(
             *this, (predicate != 0) ? mask_type(1) << data_.rank : mask_type(0),
-            cl::sycl::ONEAPI::plus<mask_type>());
+            sycl::ONEAPI::plus<mask_type>());
     }
 
     /**
@@ -247,7 +247,7 @@ class thread_block_tile : public cl::sycl::ONEAPI::sub_group {
      */
     __dpct_inline__ bool any(int predicate) const noexcept
     {
-        return cl::sycl::ONEAPI::any_of(*this, (predicate != 0));
+        return sycl::ONEAPI::any_of(*this, (predicate != 0));
     }
 
     /**
@@ -256,7 +256,7 @@ class thread_block_tile : public cl::sycl::ONEAPI::sub_group {
      */
     __dpct_inline__ bool all(int predicate) const noexcept
     {
-        return cl::sycl::ONEAPI::all_of(*this, (predicate != 0));
+        return sycl::ONEAPI::all_of(*this, (predicate != 0));
     }
 
 
diff --git a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
index 93e028cfbcd..e0b3931a8a8 100644
--- a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
+++ b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 
 
+#include <iostream>
 #include <memory>
 
 
@@ -40,7 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <gtest/gtest.h>
-#include <iostream>
+
 
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/executor.hpp>
@@ -49,7 +50,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/synthesizer/implementation_selection.hpp"
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
-#include "dpcpp/test/utils.hpp"
 
 
 namespace {
@@ -57,7 +57,10 @@ namespace {
 
 using namespace gko::kernels::dpcpp;
 using KCfg = gko::ConfigSet<12, 7>;
-
+constexpr auto default_config_list =
+    ::gko::syn::value_list<int, KCfg::encode(64, 64), KCfg::encode(32, 32),
+                           KCfg::encode(16, 16), KCfg::encode(8, 8),
+                           KCfg::encode(4, 4)>();
 
 class CooperativeGroups : public ::testing::Test {
 protected:
@@ -71,17 +74,6 @@ class CooperativeGroups : public ::testing::Test {
         dresult = result;
     }
 
-    template <typename Kernel>
-    void test(Kernel kernel)
-    {
-        // functioname kernel
-        kernel(1, config::warp_size, 0, dpcpp->get_queue(), dresult.get_data());
-        result = dresult;
-        auto success = *result.get_const_data();
-
-        ASSERT_TRUE(success);
-    }
-
     template <typename Kernel>
     void test_all_subgroup(Kernel kernel)
     {
@@ -114,13 +106,13 @@ template <int config>
 [[intel::reqd_work_group_size(1, 1, KCfg::decode<0>(config))]] void cg_shuffle(
     bool *s, sycl::nd_item<3> item_ct1)
 {
-    auto group = group::tiled_partition<KCfg::decode<0>(config)>(
+    auto group = group::tiled_partition<KCfg::decode<1>(config)>(
         group::this_thread_block(item_ct1));
     auto i = int(group.thread_rank());
     test_assert(s, group.shfl_up(i, 1) == sycl::max(0, (int)(i - 1)));
     test_assert(s, group.shfl_down(i, 1) ==
                        sycl::min((unsigned int)(i + 1),
-                                 (unsigned int)(KCfg::decode<0>(config) - 1)));
+                                 (unsigned int)(KCfg::decode<1>(config) - 1)));
     test_assert(s, group.shfl(i, 0) == 0);
 }
 
@@ -147,17 +139,13 @@ void cg_shuffle_config_call(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                             bool *s)
 {
     auto exec_info = exec->get_const_exec_info();
-    constexpr auto default_config_list =
-        ::gko::syn::value_list<int, KCfg::encode(32, 32), KCfg::encode(16, 16),
-                               KCfg::encode(8, 8), KCfg::encode(4, 4)>();
-    std::cout << "block.x " << block.x << std::endl;
     cg_shuffle_config(
         default_config_list,
         // validate
         [&exec_info, &block](int config) {
             return exec_info.validate(KCfg::decode<0>(config),
                                       KCfg::decode<1>(config)) &&
-                   (KCfg::decode<1>(config) == block.x);
+                   (KCfg::decode<0>(config) == block.x);
         },
         ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
         ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
@@ -170,80 +158,156 @@ TEST_F(CooperativeGroups, Shuffle)
 }
 
 
-void cg_all(bool *s, sycl::nd_item<3> item_ct1)
+template <int config>
+[[intel::reqd_work_group_size(1, 1, KCfg::decode<0>(config))]] void cg_all(
+    bool *s, sycl::nd_item<3> item_ct1)
 {
-    auto group = group::tiled_partition<config::warp_size>(
+    auto group = group::tiled_partition<KCfg::decode<1>(config)>(
         group::this_thread_block(item_ct1));
     test_assert(s, group.all(true));
     test_assert(s, !group.all(false));
-    test_assert(s, !group.all(item_ct1.get_local_id(2) < 13));
+    test_assert(s, group.all(item_ct1.get_local_id(2) < 13) ==
+                       KCfg::decode<1>(config) < 13);
 }
 
+template <int config>
 void cg_all_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                  sycl::queue *stream, bool *s)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
         cgh.parallel_for(
-            sycl::nd_range<3>(global_range, local_range),
-            [=](sycl::nd_item<3> item_ct1) { cg_all(s, item_ct1); });
+            sycl_nd_range(grid, block),
+            [=](sycl::nd_item<3> item_ct1) { cg_all<config>(s, item_ct1); });
     });
 }
 
-TEST_F(CooperativeGroups, All) { test(cg_all_host); }
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_all_config, cg_all_host)
+
+void cg_all_config_call(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                        sycl::queue *stream,
+                        std::shared_ptr<const gko::DpcppExecutor> exec, bool *s)
+{
+    auto exec_info = exec->get_const_exec_info();
+    cg_all_config(
+        default_config_list,
+        // validate
+        [&exec_info, &block](int config) {
+            return exec_info.validate(KCfg::decode<0>(config),
+                                      KCfg::decode<1>(config)) &&
+                   (KCfg::decode<0>(config) == block.x);
+        },
+        ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
+        ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
+        grid, block, dynamic_shared_memory, stream, s);
+}
+
+TEST_F(CooperativeGroups, All) { test_all_subgroup(cg_all_config_call); }
 
 
-void cg_any(bool *s, sycl::nd_item<3> item_ct1)
+template <int config>
+[[intel::reqd_work_group_size(1, 1, KCfg::decode<0>(config))]] void cg_any(
+    bool *s, sycl::nd_item<3> item_ct1)
 {
-    auto group = group::tiled_partition<config::warp_size>(
+    auto group = group::tiled_partition<KCfg::decode<1>(config)>(
         group::this_thread_block(item_ct1));
     test_assert(s, group.any(true));
     test_assert(s, group.any(item_ct1.get_local_id(2) == 0));
     test_assert(s, !group.any(false));
 }
 
+template <int config>
 void cg_any_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                  sycl::queue *stream, bool *s)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
         cgh.parallel_for(
-            sycl::nd_range<3>(global_range, local_range),
-            [=](sycl::nd_item<3> item_ct1) { cg_any(s, item_ct1); });
+            sycl_nd_range(grid, block),
+            [=](sycl::nd_item<3> item_ct1) { cg_any<config>(s, item_ct1); });
     });
 }
 
-TEST_F(CooperativeGroups, Any) { test(cg_any_host); }
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_any_config, cg_any_host)
+
+void cg_any_config_call(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                        sycl::queue *stream,
+                        std::shared_ptr<const gko::DpcppExecutor> exec, bool *s)
+{
+    auto exec_info = exec->get_const_exec_info();
+    cg_any_config(
+        default_config_list,
+        // validate
+        [&exec_info, &block](int config) {
+            return exec_info.validate(KCfg::decode<0>(config),
+                                      KCfg::decode<1>(config)) &&
+                   (KCfg::decode<0>(config) == block.x);
+        },
+        ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
+        ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
+        grid, block, dynamic_shared_memory, stream, s);
+}
+
+TEST_F(CooperativeGroups, Any) { test_all_subgroup(cg_any_config_call); }
 
 
-void cg_ballot(bool *s, sycl::nd_item<3> item_ct1)
+template <int Size>
+config::lane_mask_type active_mask()
 {
-    auto group = group::tiled_partition<config::warp_size>(
+    return (config::lane_mask_type{1} << Size) - 1;
+}
+
+template <>
+config::lane_mask_type active_mask<64>()
+{
+    return ~config::lane_mask_type{};
+}
+
+
+template <int config>
+[[intel::reqd_work_group_size(1, 1, KCfg::decode<0>(config))]] void cg_ballot(
+    bool *s, sycl::nd_item<3> item_ct1)
+{
+    constexpr auto subgroup_size = KCfg::decode<1>(config);
+    auto group = group::tiled_partition<subgroup_size>(
         group::this_thread_block(item_ct1));
-    auto active = ~(~config::lane_mask_type{} << config::warp_size);
+    auto active = active_mask<subgroup_size>();
     test_assert(s, group.ballot(false) == 0);
     test_assert(s, group.ballot(true) == (~config::lane_mask_type{} & active));
     test_assert(s, group.ballot(item_ct1.get_local_id(2) < 4) == 0xf);
 }
 
+template <int config>
 void cg_ballot_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                     sycl::queue *stream, bool *s)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
         cgh.parallel_for(
-            sycl::nd_range<3>(global_range, local_range),
-            [=](sycl::nd_item<3> item_ct1) { cg_ballot(s, item_ct1); });
+            sycl_nd_range(grid, block),
+            [=](sycl::nd_item<3> item_ct1) { cg_ballot<config>(s, item_ct1); });
     });
 }
 
-TEST_F(CooperativeGroups, Ballot) { test(cg_ballot_host); }
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_ballot_config, cg_ballot_host)
+
+void cg_ballot_config_call(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                           sycl::queue *stream,
+                           std::shared_ptr<const gko::DpcppExecutor> exec,
+                           bool *s)
+{
+    auto exec_info = exec->get_const_exec_info();
+    cg_ballot_config(
+        default_config_list,
+        // validate
+        [&exec_info, &block](int config) {
+            return exec_info.validate(KCfg::decode<0>(config),
+                                      KCfg::decode<1>(config)) &&
+                   (KCfg::decode<0>(config) == block.x);
+        },
+        ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
+        ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
+        grid, block, dynamic_shared_memory, stream, s);
+}
+
+TEST_F(CooperativeGroups, Ballot) { test_all_subgroup(cg_ballot_config_call); }
 
 
 }  // namespace
diff --git a/dpcpp/test/utils.hpp b/dpcpp/test/utils.hpp
deleted file mode 100644
index 15dcc6103d4..00000000000
--- a/dpcpp/test/utils.hpp
+++ /dev/null
@@ -1,54 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GKO_DPCPP_TEST_UTILS_HPP_
-#define GKO_DPCPP_TEST_UTILS_HPP_
-
-
-#include "core/test/utils.hpp"
-
-
-#include <ginkgo/core/base/executor.hpp>
-
-
-namespace {
-
-
-// prevent device reset after each test
-auto no_reset_exec =
-    gko::DpcppExecutor::create(0, gko::ReferenceExecutor::create());
-
-
-}  // namespace
-
-
-#endif  // GKO_DPCPP_TEST_UTILS_HPP_

From b18d731f1c3632ae4907b9aa4a1dfeed4db905ba Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 18 May 2021 12:09:49 +0200
Subject: [PATCH 0719/1008] add default_helper and add validate_function

---
 dpcpp/base/config.hpp                         |   5 -
 dpcpp/base/helper.hpp                         |  81 +++++++++++++
 .../cooperative_groups_kernels.dp.cpp         | 109 ++++--------------
 include/ginkgo/core/base/executor.hpp         |  13 +++
 4 files changed, 114 insertions(+), 94 deletions(-)
 create mode 100644 dpcpp/base/helper.hpp

diff --git a/dpcpp/base/config.hpp b/dpcpp/base/config.hpp
index 71639405d8d..78fe25978a7 100644
--- a/dpcpp/base/config.hpp
+++ b/dpcpp/base/config.hpp
@@ -49,11 +49,6 @@ struct config {
      */
     using lane_mask_type = uint64;
 
-    /**
-     * The number of threads within a DPCPP warp.
-     */
-    static constexpr uint32 warp_size = 32;
-
     /**
      * The bitmask of the entire warp.
      */
diff --git a/dpcpp/base/helper.hpp b/dpcpp/base/helper.hpp
new file mode 100644
index 00000000000..93564d6b8e4
--- /dev/null
+++ b/dpcpp/base/helper.hpp
@@ -0,0 +1,81 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef DPCPP_BASE_DPCT_HPP_
+#define DPCPP_BASE_DPCT_HPP_
+
+#include <utility>
+
+
+#include <CL/sycl.hpp>
+
+
+#include "dpcpp/base/dim3.dp.hpp"
+
+
+#define GKO_ENABLE_DEFAULT_HOST_CONFIG(name_, kernel_)                     \
+    template <int config, typename... InferredArgs>                        \
+    inline void name_(dim3 grid, dim3 block, size_t dynamic_shared_memory, \
+                      sycl::queue *stream, InferredArgs... args)           \
+    {                                                                      \
+        stream->submit([&](sycl::handler &cgh) {                           \
+            cgh.parallel_for(sycl_nd_range(grid, block),                   \
+                             [=](sycl::nd_item<3> item_ct1) {              \
+                                 kernel_<config>(args..., item_ct1);       \
+                             });                                           \
+        });                                                                \
+    }
+
+#define GKO_ENABLE_DEFAULT_CONFIG_CALL(name_, callable_, cfg_, list_)      \
+    template <typename... InferredArgs>                                    \
+    void name_(dim3 grid, dim3 block, size_t dynamic_shared_memory,        \
+               sycl::queue *stream,                                        \
+               std::shared_ptr<const gko::DpcppExecutor> exec,             \
+               InferredArgs... args)                                       \
+    {                                                                      \
+        auto exec_info = exec->get_const_exec_info();                      \
+        callable_(                                                         \
+            list_,                                                         \
+            [&exec_info, &block](int config) {                             \
+                return exec_info.validate(cfg_::decode<0>(config),         \
+                                          cfg_::decode<1>(config)) &&      \
+                       (cfg_::decode<0>(config) == block.x);               \
+            },                                                             \
+            ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(), \
+            ::gko::syn::value_list<gko::size_type>(),                      \
+            ::gko::syn::type_list<>(), grid, block, dynamic_shared_memory, \
+            stream, std::forward<InferredArgs>(args)...);                  \
+    }
+
+#define __WG_BOUND__(x) [[intel::reqd_work_group_size(1, 1, x)]]
+
+#endif  // DPCPP_BASE_DPCT_HPP_
diff --git a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
index e0b3931a8a8..46818aac9f5 100644
--- a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
+++ b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
@@ -50,6 +50,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/synthesizer/implementation_selection.hpp"
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/base/helper.hpp"
 
 
 namespace {
@@ -170,43 +171,20 @@ template <int config>
                        KCfg::decode<1>(config) < 13);
 }
 
-template <int config>
-void cg_all_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                 sycl::queue *stream, bool *s)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block),
-            [=](sycl::nd_item<3> item_ct1) { cg_all<config>(s, item_ct1); });
-    });
-}
-
+GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_all_host, cg_all)
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_all_config, cg_all_host)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_all_config_call, cg_all_config, KCfg,
+                               default_config_list)
 
-void cg_all_config_call(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                        sycl::queue *stream,
-                        std::shared_ptr<const gko::DpcppExecutor> exec, bool *s)
+TEST_F(CooperativeGroups, All)
 {
-    auto exec_info = exec->get_const_exec_info();
-    cg_all_config(
-        default_config_list,
-        // validate
-        [&exec_info, &block](int config) {
-            return exec_info.validate(KCfg::decode<0>(config),
-                                      KCfg::decode<1>(config)) &&
-                   (KCfg::decode<0>(config) == block.x);
-        },
-        ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
-        ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
-        grid, block, dynamic_shared_memory, stream, s);
+    test_all_subgroup(cg_all_config_call<bool *>);
 }
 
-TEST_F(CooperativeGroups, All) { test_all_subgroup(cg_all_config_call); }
-
 
 template <int config>
-[[intel::reqd_work_group_size(1, 1, KCfg::decode<0>(config))]] void cg_any(
-    bool *s, sycl::nd_item<3> item_ct1)
+__WG_BOUND__(KCfg::decode<0>(config))
+void cg_any(bool *s, sycl::nd_item<3> item_ct1)
 {
     auto group = group::tiled_partition<KCfg::decode<1>(config)>(
         group::this_thread_block(item_ct1));
@@ -215,39 +193,16 @@ template <int config>
     test_assert(s, !group.any(false));
 }
 
-template <int config>
-void cg_any_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                 sycl::queue *stream, bool *s)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block),
-            [=](sycl::nd_item<3> item_ct1) { cg_any<config>(s, item_ct1); });
-    });
-}
-
+GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_any_host, cg_any)
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_any_config, cg_any_host)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_any_config_call, cg_any_config, KCfg,
+                               default_config_list)
 
-void cg_any_config_call(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                        sycl::queue *stream,
-                        std::shared_ptr<const gko::DpcppExecutor> exec, bool *s)
+TEST_F(CooperativeGroups, Any)
 {
-    auto exec_info = exec->get_const_exec_info();
-    cg_any_config(
-        default_config_list,
-        // validate
-        [&exec_info, &block](int config) {
-            return exec_info.validate(KCfg::decode<0>(config),
-                                      KCfg::decode<1>(config)) &&
-                   (KCfg::decode<0>(config) == block.x);
-        },
-        ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
-        ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
-        grid, block, dynamic_shared_memory, stream, s);
+    test_all_subgroup(cg_any_config_call<bool *>);
 }
 
-TEST_F(CooperativeGroups, Any) { test_all_subgroup(cg_any_config_call); }
-
 
 template <int Size>
 config::lane_mask_type active_mask()
@@ -263,8 +218,8 @@ config::lane_mask_type active_mask<64>()
 
 
 template <int config>
-[[intel::reqd_work_group_size(1, 1, KCfg::decode<0>(config))]] void cg_ballot(
-    bool *s, sycl::nd_item<3> item_ct1)
+__WG_BOUND__(KCfg::decode<0>(config))
+void cg_ballot(bool *s, sycl::nd_item<3> item_ct1)
 {
     constexpr auto subgroup_size = KCfg::decode<1>(config);
     auto group = group::tiled_partition<subgroup_size>(
@@ -275,39 +230,15 @@ template <int config>
     test_assert(s, group.ballot(item_ct1.get_local_id(2) < 4) == 0xf);
 }
 
-template <int config>
-void cg_ballot_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                    sycl::queue *stream, bool *s)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block),
-            [=](sycl::nd_item<3> item_ct1) { cg_ballot<config>(s, item_ct1); });
-    });
-}
-
+GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_ballot_host, cg_ballot)
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_ballot_config, cg_ballot_host)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_ballot_config_call, cg_ballot_config, KCfg,
+                               default_config_list)
 
-void cg_ballot_config_call(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                           sycl::queue *stream,
-                           std::shared_ptr<const gko::DpcppExecutor> exec,
-                           bool *s)
+TEST_F(CooperativeGroups, Ballot)
 {
-    auto exec_info = exec->get_const_exec_info();
-    cg_ballot_config(
-        default_config_list,
-        // validate
-        [&exec_info, &block](int config) {
-            return exec_info.validate(KCfg::decode<0>(config),
-                                      KCfg::decode<1>(config)) &&
-                   (KCfg::decode<0>(config) == block.x);
-        },
-        ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
-        ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
-        grid, block, dynamic_shared_memory, stream, s);
+    test_all_subgroup(cg_ballot_config_call<bool *>);
 }
 
-TEST_F(CooperativeGroups, Ballot) { test_all_subgroup(cg_ballot_config_call); }
-
 
 }  // namespace
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index b1044cf4056..fb1cd1ed232 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -823,6 +823,19 @@ class Executor : public log::EnableLogging<Executor> {
             }
             return allowed && (blocksize <= max_workgroup_size);
         }
+
+        auto validate_function() const
+        {
+            return [subgroup_sizes = subgroup_sizes,
+                    max_workgroup_size = max_workgroup_size](int blocksize,
+                                                             int warpsize) {
+                bool allowed = false;
+                for (auto &i : subgroup_sizes) {
+                    allowed |= (i == warpsize);
+                }
+                return allowed && (blocksize <= max_workgroup_size);
+            };
+        }
     };
 
 public:

From bce849b9602af5a8792b23ad855d86a8fd2deb1c Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 18 May 2021 17:49:36 +0200
Subject: [PATCH 0720/1008] implement full WG_BOUND

---
 dpcpp/base/dim3.dp.hpp |  6 +++---
 dpcpp/base/dpct.hpp    |  6 +++---
 dpcpp/base/helper.hpp  | 34 +++++++++++++++++++++++++++++-----
 3 files changed, 35 insertions(+), 11 deletions(-)

diff --git a/dpcpp/base/dim3.dp.hpp b/dpcpp/base/dim3.dp.hpp
index 2bac206058f..edd86a388bd 100644
--- a/dpcpp/base/dim3.dp.hpp
+++ b/dpcpp/base/dim3.dp.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef DPCPP_BASE_DIM3_DP_HPP_
-#define DPCPP_BASE_DIM3_DP_HPP_
+#ifndef GKO_DPCPP_BASE_DIM3_DP_HPP_
+#define GKO_DPCPP_BASE_DIM3_DP_HPP_
 
 
 #include <CL/sycl.hpp>
@@ -58,4 +58,4 @@ sycl::nd_range<3> sycl_nd_range(dim3 grid, dim3 block)
 }
 
 
-#endif  // DPCPP_BASE_DIM3_DP_HPP_
+#endif  // GKO_DPCPP_BASE_DIM3_DP_HPP_
diff --git a/dpcpp/base/dpct.hpp b/dpcpp/base/dpct.hpp
index a75800be5ee..f29132be4d4 100644
--- a/dpcpp/base/dpct.hpp
+++ b/dpcpp/base/dpct.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef DPCPP_BASE_DPCT_HPP_
-#define DPCPP_BASE_DPCT_HPP_
+#ifndef GKO_DPCPP_BASE_DPCT_HPP_
+#define GKO_DPCPP_BASE_DPCT_HPP_
 
 
 #include <CL/sycl.hpp>
@@ -47,4 +47,4 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 
 
-#endif  // DPCPP_BASE_DPCT_HPP_
+#endif  // GKO_DPCPP_BASE_DPCT_HPP_
diff --git a/dpcpp/base/helper.hpp b/dpcpp/base/helper.hpp
index 93564d6b8e4..ed3adf2c228 100644
--- a/dpcpp/base/helper.hpp
+++ b/dpcpp/base/helper.hpp
@@ -30,8 +30,9 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef DPCPP_BASE_DPCT_HPP_
-#define DPCPP_BASE_DPCT_HPP_
+#ifndef GKO_DPCPP_BASE_HELPER_HPP_
+#define GKO_DPCPP_BASE_HELPER_HPP_
+
 
 #include <utility>
 
@@ -42,6 +43,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "dpcpp/base/dim3.dp.hpp"
 
 
+/**
+ * GKO_ENABLE_DEFAULT_HOST_CONFIG gives a default host implementation for those
+ * kernels which require config but do not need explicit template parameter and
+ * share memory
+ */
 #define GKO_ENABLE_DEFAULT_HOST_CONFIG(name_, kernel_)                     \
     template <int config, typename... InferredArgs>                        \
     inline void name_(dim3 grid, dim3 block, size_t dynamic_shared_memory, \
@@ -55,6 +61,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
         });                                                                \
     }
 
+/**
+ * GKO_ENABLE_DEFAULT_CONFIG_CALL gives a default config selection call
+ * implementation for those kernels which require config selection but do not
+ * need explicit template parameter
+ */
 #define GKO_ENABLE_DEFAULT_CONFIG_CALL(name_, callable_, cfg_, list_)      \
     template <typename... InferredArgs>                                    \
     void name_(dim3 grid, dim3 block, size_t dynamic_shared_memory,        \
@@ -76,6 +87,19 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
             stream, std::forward<InferredArgs>(args)...);                  \
     }
 
-#define __WG_BOUND__(x) [[intel::reqd_work_group_size(1, 1, x)]]
-
-#endif  // DPCPP_BASE_DPCT_HPP_
+// __WG_BOUND__ gives the cuda-like launch bound in cuda ordering
+#define __WG_BOUND_1D__(x) [[intel::reqd_work_group_size(1, 1, x)]]
+#define __WG_BOUND_2D__(x, y) [[intel::reqd_work_group_size(1, y, x)]]
+#define __WG_BOUND_3D__(x, y, z) [[intel::reqd_work_group_size(z, y, x)]]
+#define WG_BOUND_OVERLOAD(_1, _2, _3, NAME, ...) NAME
+#define __WG_BOUND__(...)                                            \
+    WG_BOUND_OVERLOAD(__VA_ARGS__, __WG_BOUND_3D__, __WG_BOUND_2D__, \
+                      __WG_BOUND_1D__, UNUSED)                       \
+    (__VA_ARGS__)
+
+// __WG_CONFIG_BOUND__ use ConfigSet to unpack the config
+#define __WG_CONFIG_BOUND__(CFG, cfg)                         \
+    __WG_BOUND_3D__(CFG::decode<0>(cfg), CFG::decode<1>(cfg), \
+                    CFG::decode<2>(cfg))
+
+#endif  // GKO_DPCPP_BASE_HELPER_HPP_

From 8918c7ce5e101a3ec86695755688cf965a91ac4e Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 19 May 2021 00:00:47 +0200
Subject: [PATCH 0721/1008] improve the test, add description and dim3 test

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/synthesizer/implementation_selection.hpp |   1 -
 dev_tools/scripts/config                      |   2 +
 dpcpp/base/dim3.dp.hpp                        |  37 +++++-
 dpcpp/components/cooperative_groups.dp.hpp    |   2 +-
 dpcpp/test/base/CMakeLists.txt                |   1 +
 dpcpp/test/base/dim3.dp.cpp                   | 118 +++++++++++++++++
 .../cooperative_groups_kernels.dp.cpp         | 122 ++++++++++--------
 include/ginkgo/core/base/types.hpp            |  91 +++++++++++--
 8 files changed, 304 insertions(+), 70 deletions(-)
 create mode 100644 dpcpp/test/base/dim3.dp.cpp

diff --git a/core/synthesizer/implementation_selection.hpp b/core/synthesizer/implementation_selection.hpp
index 998a0dbffc1..b6e9400ad52 100644
--- a/core/synthesizer/implementation_selection.hpp
+++ b/core/synthesizer/implementation_selection.hpp
@@ -92,7 +92,6 @@ namespace syn {
         ::gko::syn::type_list<TArgs...> type_args, InferredArgs... args)      \
     {                                                                         \
         if (is_eligible(K)) {                                                 \
-            std::cout << "call " << K << std::endl;                           \
             _callable<BoolArgs..., IntArgs..., SizeTArgs..., TArgs..., K>(    \
                 std::forward<InferredArgs>(args)...);                         \
         } else {                                                              \
diff --git a/dev_tools/scripts/config b/dev_tools/scripts/config
index 00cb06893d8..10ea75302e6 100644
--- a/dev_tools/scripts/config
+++ b/dev_tools/scripts/config
@@ -32,6 +32,8 @@
     - RemoveTest: "true"
 - "_builder\.hpp"
     - CoreSuffix: "_builder"
+- "dpcpp/test/base/dim3\.dp\.cpp"
+    - FixInclude: "dpcpp/base/dim3.dp.hpp"
 - "components.*_kernels(\.hip|\.dp)?\.(cu|cpp|hpp|cuh)"
     - CoreSuffix: "_kernels"
     - RemoveTest: "true"
diff --git a/dpcpp/base/dim3.dp.hpp b/dpcpp/base/dim3.dp.hpp
index edd86a388bd..238d39f4f52 100644
--- a/dpcpp/base/dim3.dp.hpp
+++ b/dpcpp/base/dim3.dp.hpp
@@ -37,20 +37,50 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <CL/sycl.hpp>
 
 
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+/**
+ * dim3 is a cuda-like dim3 for sycl-range, which provides the same ordering as
+ * cuda and gets the sycl-range in reverse ordering.
+ */
 struct dim3 {
     unsigned int x;
     unsigned int y;
     unsigned int z;
 
+    /**
+     * Creates a dim3 with x, y, z
+     *
+     * @param xval  x dim val
+     * @param yval  y dim val and default is 1
+     * @param zval  z dim val and default is 1
+     */
     dim3(unsigned int xval, unsigned int yval = 1, unsigned int zval = 1)
         : x(xval), y(yval), z(zval)
     {}
 
+    /**
+     * reverse returns the range for sycl with correct ordering (reverse of
+     * cuda)
+     *
+     * @return sycl::range<3>
+     */
     sycl::range<3> reverse() { return sycl::range<3>(z, y, x); }
 };
 
 
-sycl::nd_range<3> sycl_nd_range(dim3 grid, dim3 block)
+/**
+ * sycl_nd_range will generate the proper sycl::nd_range<3> from grid, block
+ *
+ * @param grid  the dim3 for grid
+ * @param block  the dim3 for block
+ *
+ * @return sycl::nd_range<3>
+ */
+inline sycl::nd_range<3> sycl_nd_range(dim3 grid, dim3 block)
 {
     auto local_range = block.reverse();
     auto global_range = grid.reverse() * local_range;
@@ -58,4 +88,9 @@ sycl::nd_range<3> sycl_nd_range(dim3 grid, dim3 block)
 }
 
 
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
 #endif  // GKO_DPCPP_BASE_DIM3_DP_HPP_
diff --git a/dpcpp/components/cooperative_groups.dp.hpp b/dpcpp/components/cooperative_groups.dp.hpp
index 0c2a6e141cc..1cf3c6955e2 100644
--- a/dpcpp/components/cooperative_groups.dp.hpp
+++ b/dpcpp/components/cooperative_groups.dp.hpp
@@ -312,7 +312,7 @@ class thread_block_tile<1> {
      */
     __dpct_inline__ mask_type ballot(int predicate) const noexcept
     {
-        return (predicate != 0) ? mask_type(1) << data_.rank : mask_type(0);
+        return (predicate != 0) ? mask_type(1) : mask_type(0);
     }
 
     /**
diff --git a/dpcpp/test/base/CMakeLists.txt b/dpcpp/test/base/CMakeLists.txt
index adb2f2505b5..c31c0f1527e 100644
--- a/dpcpp/test/base/CMakeLists.txt
+++ b/dpcpp/test/base/CMakeLists.txt
@@ -1 +1,2 @@
 ginkgo_create_dpcpp_test(executor)
+ginkgo_create_dpcpp_test(dim3)
diff --git a/dpcpp/test/base/dim3.dp.cpp b/dpcpp/test/base/dim3.dp.cpp
new file mode 100644
index 00000000000..8b6a08fa996
--- /dev/null
+++ b/dpcpp/test/base/dim3.dp.cpp
@@ -0,0 +1,118 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "dpcpp/base/dim3.dp.hpp"
+
+
+#include <CL/sycl.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+namespace {
+
+
+using namespace gko::kernels::dpcpp;
+
+
+class DpcppDim3 : public ::testing::Test {
+protected:
+    DpcppDim3() {}
+
+    void SetUp() {}
+
+    void TearDown() {}
+};
+
+
+TEST_F(DpcppDim3, CanGenerate1DRange)
+{
+    dim3 block(3);
+    auto sycl_block = block.reverse();
+
+    ASSERT_EQ(block.x, 3);
+    ASSERT_EQ(block.y, 1);
+    ASSERT_EQ(block.z, 1);
+    ASSERT_EQ(sycl_block.get(0), 1);
+    ASSERT_EQ(sycl_block.get(1), 1);
+    ASSERT_EQ(sycl_block.get(2), 3);
+}
+
+
+TEST_F(DpcppDim3, CanGenerate2DRange)
+{
+    dim3 block(3, 5);
+    auto sycl_block = block.reverse();
+
+    ASSERT_EQ(block.x, 3);
+    ASSERT_EQ(block.y, 5);
+    ASSERT_EQ(block.z, 1);
+    ASSERT_EQ(sycl_block.get(0), 1);
+    ASSERT_EQ(sycl_block.get(1), 5);
+    ASSERT_EQ(sycl_block.get(2), 3);
+}
+
+
+TEST_F(DpcppDim3, CanGenerate3DRange)
+{
+    dim3 block(3, 5, 7);
+    auto sycl_block = block.reverse();
+
+    ASSERT_EQ(block.x, 3);
+    ASSERT_EQ(block.y, 5);
+    ASSERT_EQ(block.z, 7);
+    ASSERT_EQ(sycl_block.get(0), 7);
+    ASSERT_EQ(sycl_block.get(1), 5);
+    ASSERT_EQ(sycl_block.get(2), 3);
+}
+
+
+TEST_F(DpcppDim3, CanGenerateNDRange)
+{
+    dim3 block(3, 5, 7);
+    dim3 grid(17, 13, 11);
+
+    auto ndrange = sycl_nd_range(grid, block);
+    auto global_size = ndrange.get_global_range();
+    auto local_size = ndrange.get_local_range();
+
+    ASSERT_EQ(local_size.get(0), 7);
+    ASSERT_EQ(local_size.get(1), 5);
+    ASSERT_EQ(local_size.get(2), 3);
+    ASSERT_EQ(global_size.get(0), 7 * 11);
+    ASSERT_EQ(global_size.get(1), 5 * 13);
+    ASSERT_EQ(global_size.get(2), 3 * 17);
+}
+
+
+}  // namespace
diff --git a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
index 46818aac9f5..a5c34149381 100644
--- a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
+++ b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
@@ -48,6 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/synthesizer/implementation_selection.hpp"
+#include "core/test/utils/assertions.hpp"
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
 #include "dpcpp/base/helper.hpp"
@@ -63,31 +64,46 @@ constexpr auto default_config_list =
                            KCfg::encode(16, 16), KCfg::encode(8, 8),
                            KCfg::encode(4, 4)>();
 
-class CooperativeGroups : public ::testing::Test {
+
+class CooperativeGroups : public testing::TestWithParam<int> {
 protected:
     CooperativeGroups()
         : ref(gko::ReferenceExecutor::create()),
           dpcpp(gko::DpcppExecutor::create(0, ref)),
-          result(ref, 1),
+          max_test_case(3),
+          max_num(max_test_case * 64),
+          result(ref, max_num),
           dresult(dpcpp)
     {
-        *result.get_data() = true;
+        for (int i = 0; i < max_num; i++) {
+            result.get_data()[i] = false;
+        }
         dresult = result;
     }
 
     template <typename Kernel>
     void test_all_subgroup(Kernel kernel)
     {
+        auto subgroup_size = GetParam();
         auto exec_info = dpcpp->get_const_exec_info();
-        for (auto &i : exec_info.subgroup_sizes) {
-            kernel(1, i, 0, dpcpp->get_queue(), dpcpp, dresult.get_data());
-            result = dresult;
-            auto success = *result.get_const_data();
-            ASSERT_TRUE(success);
-            std::cout << i << " success" << std::endl;
+        if (exec_info.validate(subgroup_size, subgroup_size)) {
+            for (int i = 0; i < max_test_case * subgroup_size; i++) {
+                result.get_data()[i] = true;
+            }
+
+            kernel(1, subgroup_size, 0, dpcpp->get_queue(), dpcpp,
+                   dresult.get_data());
+
+            // each subgreoup size segment for one test
+            GKO_ASSERT_ARRAY_EQ(result, dresult);
+        } else {
+            GTEST_SKIP() << "This device does not contain this subgroup size "
+                         << subgroup_size;
         }
     }
 
+    int max_test_case;
+    int max_num;
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<gko::DpcppExecutor> dpcpp;
     gko::Array<bool> result;
@@ -104,17 +120,19 @@ void test_assert(bool *success, bool partial)
 
 // kernel implementation
 template <int config>
-[[intel::reqd_work_group_size(1, 1, KCfg::decode<0>(config))]] void cg_shuffle(
-    bool *s, sycl::nd_item<3> item_ct1)
+__WG_BOUND__(KCfg::decode<0>(config))
+void cg_shuffle(bool *s, sycl::nd_item<3> item_ct1)
 {
-    auto group = group::tiled_partition<KCfg::decode<1>(config)>(
-        group::this_thread_block(item_ct1));
+    constexpr auto sg_size = KCfg::decode<1>(config);
+    auto group =
+        group::tiled_partition<sg_size>(group::this_thread_block(item_ct1));
     auto i = int(group.thread_rank());
-    test_assert(s, group.shfl_up(i, 1) == sycl::max(0, (int)(i - 1)));
-    test_assert(s, group.shfl_down(i, 1) ==
-                       sycl::min((unsigned int)(i + 1),
-                                 (unsigned int)(KCfg::decode<1>(config) - 1)));
-    test_assert(s, group.shfl(i, 0) == 0);
+
+    s[i] = group.shfl_up(i, 1) == sycl::max(0, (int)(i - 1));
+    s[i + sg_size] =
+        group.shfl_down(i, 1) ==
+        sycl::min((unsigned int)(i + 1), (unsigned int)(sg_size - 1));
+    s[i + sg_size * 2] = group.shfl(i, 0) == 0;
 }
 
 // group all kernel things together
@@ -153,22 +171,25 @@ void cg_shuffle_config_call(dim3 grid, dim3 block, size_t dynamic_shared_memory,
         grid, block, dynamic_shared_memory, stream, s);
 }
 
-TEST_F(CooperativeGroups, Shuffle)
+TEST_P(CooperativeGroups, Shuffle)
 {
     test_all_subgroup(cg_shuffle_config_call);
 }
 
 
 template <int config>
-[[intel::reqd_work_group_size(1, 1, KCfg::decode<0>(config))]] void cg_all(
-    bool *s, sycl::nd_item<3> item_ct1)
+__WG_BOUND__(KCfg::decode<0>(config))
+void cg_all(bool *s, sycl::nd_item<3> item_ct1)
 {
-    auto group = group::tiled_partition<KCfg::decode<1>(config)>(
-        group::this_thread_block(item_ct1));
-    test_assert(s, group.all(true));
-    test_assert(s, !group.all(false));
-    test_assert(s, group.all(item_ct1.get_local_id(2) < 13) ==
-                       KCfg::decode<1>(config) < 13);
+    constexpr auto sg_size = KCfg::decode<1>(config);
+    auto group =
+        group::tiled_partition<sg_size>(group::this_thread_block(item_ct1));
+    auto i = int(group.thread_rank());
+
+    s[i] = group.all(true);
+    s[i + sg_size] = !group.all(false);
+    s[i + sg_size * 2] =
+        group.all(item_ct1.get_local_id(2) < 13) == sg_size < 13;
 }
 
 GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_all_host, cg_all)
@@ -176,7 +197,7 @@ GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_all_config, cg_all_host)
 GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_all_config_call, cg_all_config, KCfg,
                                default_config_list)
 
-TEST_F(CooperativeGroups, All)
+TEST_P(CooperativeGroups, All)
 {
     test_all_subgroup(cg_all_config_call<bool *>);
 }
@@ -186,11 +207,14 @@ template <int config>
 __WG_BOUND__(KCfg::decode<0>(config))
 void cg_any(bool *s, sycl::nd_item<3> item_ct1)
 {
+    constexpr auto sg_size = KCfg::decode<1>(config);
     auto group = group::tiled_partition<KCfg::decode<1>(config)>(
         group::this_thread_block(item_ct1));
-    test_assert(s, group.any(true));
-    test_assert(s, group.any(item_ct1.get_local_id(2) == 0));
-    test_assert(s, !group.any(false));
+    auto i = int(group.thread_rank());
+
+    s[i] = group.any(true);
+    s[i + sg_size] = group.any(item_ct1.get_local_id(2) == 0);
+    s[i + sg_size * 2] = !group.any(false);
 }
 
 GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_any_host, cg_any)
@@ -198,36 +222,25 @@ GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_any_config, cg_any_host)
 GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_any_config_call, cg_any_config, KCfg,
                                default_config_list)
 
-TEST_F(CooperativeGroups, Any)
+TEST_P(CooperativeGroups, Any)
 {
     test_all_subgroup(cg_any_config_call<bool *>);
 }
 
 
-template <int Size>
-config::lane_mask_type active_mask()
-{
-    return (config::lane_mask_type{1} << Size) - 1;
-}
-
-template <>
-config::lane_mask_type active_mask<64>()
-{
-    return ~config::lane_mask_type{};
-}
-
-
 template <int config>
 __WG_BOUND__(KCfg::decode<0>(config))
 void cg_ballot(bool *s, sycl::nd_item<3> item_ct1)
 {
-    constexpr auto subgroup_size = KCfg::decode<1>(config);
-    auto group = group::tiled_partition<subgroup_size>(
-        group::this_thread_block(item_ct1));
-    auto active = active_mask<subgroup_size>();
-    test_assert(s, group.ballot(false) == 0);
-    test_assert(s, group.ballot(true) == (~config::lane_mask_type{} & active));
-    test_assert(s, group.ballot(item_ct1.get_local_id(2) < 4) == 0xf);
+    constexpr auto sg_size = KCfg::decode<1>(config);
+    auto group =
+        group::tiled_partition<sg_size>(group::this_thread_block(item_ct1));
+    auto active = gko::detail::mask<sg_size, config::lane_mask_type>();
+    auto i = int(group.thread_rank());
+
+    s[i] = group.ballot(false) == 0;
+    s[i + sg_size] = group.ballot(true) == (~config::lane_mask_type{} & active);
+    s[i + sg_size * 2] = group.ballot(item_ct1.get_local_id(2) < 4) == 0xf;
 }
 
 GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_ballot_host, cg_ballot)
@@ -235,10 +248,13 @@ GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_ballot_config, cg_ballot_host)
 GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_ballot_config_call, cg_ballot_config, KCfg,
                                default_config_list)
 
-TEST_F(CooperativeGroups, Ballot)
+TEST_P(CooperativeGroups, Ballot)
 {
     test_all_subgroup(cg_ballot_config_call<bool *>);
 }
 
+INSTANTIATE_TEST_SUITE_P(DifferentSubgroup, CooperativeGroups,
+                         testing::Values(4, 8, 16, 32, 64),
+                         testing::PrintToStringParamName());
 
 }  // namespace
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index a166a8ac1ec..f942b80cc89 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -194,24 +194,45 @@ using default_precision = double;
 constexpr size_type byte_size = CHAR_BIT;
 
 
-/**
- * Config type
- */
-namespace helper {
+namespace detail {
 
 
-template <size_type current_size>
-constexpr int mask()
+/**
+ * mask gives the integer with Size activated bits in the end
+ *
+ * @tparam Size  the number of activated bits
+ * @tparam ValueType  the type of mask, which uses unsigned int as default
+ *
+ * @return the ValueType with Size activated bits in the end
+ */
+template <int Size, typename ValueType = unsigned int>
+constexpr std::enable_if_t<(Size < sizeof(ValueType) * 8), ValueType> mask()
 {
-    return (1 << (current_size - 1)) | mask<current_size - 1>();
+    return (ValueType{1} << Size) - 1;
 }
 
-template <>
-constexpr int mask<0>()
+/**
+ * @copydoc mask()
+ *
+ * @note this is special case for the Size = the number of bits of ValueType
+ */
+template <int Size, typename ValueType = unsigned int>
+constexpr std::enable_if_t<Size == sizeof(ValueType) * 8, ValueType> mask()
 {
-    return 0;
+    return ~ValueType{};
 }
 
+
+/**
+ * shift calculates the number of bits for shifting
+ *
+ * @tparam num_groups  the number of elements in array
+ * @tparam current_shift  the current position of shifting
+ *
+ * @return the number of shifting bits
+ *
+ * @note this is the last case of nested template
+ */
 template <int num_groups, int current_shift>
 constexpr std::enable_if_t<(num_groups == current_shift + 1), int> shift(
     const std::array<char, num_groups> &bits)
@@ -219,6 +240,11 @@ constexpr std::enable_if_t<(num_groups == current_shift + 1), int> shift(
     return 0;
 }
 
+/**
+ * @copydoc shift(const std::array<char, num_groups>)
+ *
+ * @note this is the usual case of nested template
+ */
 template <int num_groups, int current_shift>
 constexpr std::enable_if_t<(num_groups > current_shift + 1), int> shift(
     const std::array<char, num_groups> &bits)
@@ -228,25 +254,51 @@ constexpr std::enable_if_t<(num_groups > current_shift + 1), int> shift(
 }
 
 
-}  // namespace helper
+}  // namespace detail
 
 
+/**
+ * ConfigSet is a way to embed several information into one integer by given
+ * certain bits. The usage will be the following
+ * Set the method with bits Cfg = ConfigSet<nb_1, nb_2, ..., nb_k>
+ * Encode the given infomation encoded = Cfg::encode(i_1, i_2, ..., i_k)
+ * Decode the specific position information i_t = Cfg::decode<t>(encoded)
+ * The encoded result will use 32 bits to record
+ * rrrrr1..12....2...k..k, which 1/2/k means the bits store the information for
+ * 1/2/k position and r is for rest of unused bits.
+ *
+ * @tparam num_bits...  the number of bits for each position.
+ */
 template <int... num_bits>
 class ConfigSet {
 public:
     static constexpr size_type num_groups = sizeof...(num_bits);
     static constexpr std::array<char, num_groups> bits{num_bits...};
 
+    /**
+     * Decodes the `position` information from encoded
+     *
+     * @tparam position  the position of desired information
+     *
+     * @param encoded  the encoded integer
+     *
+     * @return the decoded information at position
+     */
     template <int position>
     static constexpr int decode(int encoded)
     {
         static_assert(position < num_groups,
                       "This position is over the bounds.");
-        constexpr int shift = helper::shift<num_groups, position>(bits);
-        constexpr int mask = helper::mask<bits[position]>();
+        constexpr int shift = detail::shift<num_groups, position>(bits);
+        constexpr auto mask = detail::mask<bits[position]>();
         return (encoded >> shift) & mask;
     }
 
+    /**
+     * Encodes the information with given bit set to encoded integer.
+     *
+     * @note the last case of nested template.
+     */
     template <size_type current_iter>
     static constexpr std::enable_if_t<(current_iter == num_groups), int>
     encode()
@@ -254,11 +306,22 @@ class ConfigSet {
         return 0;
     }
 
+    /**
+     * Encodes the information with given bit set to encoded integer.
+     *
+     * @tparam current_iter  the encoded place
+     * @tparam First  the current_iter type
+     * @tparam Rest...  the rest information
+     *
+     * @param informations... the information will be encoded
+     *
+     * @return the encoded integer
+     */
     template <size_type current_iter = 0, typename First, typename... Rest>
     static constexpr std::enable_if_t<(current_iter < num_groups), int> encode(
         First first, Rest &&... rest)
     {
-        constexpr int shift = helper::shift<num_groups, current_iter>(bits);
+        constexpr int shift = detail::shift<num_groups, current_iter>(bits);
         return (first << shift) |
                encode<current_iter + 1>(std::forward<Rest>(rest)...);
     }

From 3a11fd17a53ebd867a6c5974f8dd9e292ad33ac8 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 19 May 2021 20:43:20 +0200
Subject: [PATCH 0722/1008] add the test for ConfigSet and review update

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 core/test/base/types.cpp                      | 110 ++++++++++++++++++
 dev_tools/scripts/format_header.sh            |   5 -
 dpcpp/base/dim3.dp.hpp                        |   8 +-
 dpcpp/base/helper.hpp                         |  26 +++--
 dpcpp/components/cooperative_groups.dp.hpp    |   2 +-
 dpcpp/test/base/dim3.dp.cpp                   |  24 ++--
 .../cooperative_groups_kernels.dp.cpp         |  15 +--
 include/ginkgo/core/base/types.hpp            |  18 +--
 8 files changed, 154 insertions(+), 54 deletions(-)

diff --git a/core/test/base/types.cpp b/core/test/base/types.cpp
index 5446d2865b1..71c0101954f 100644
--- a/core/test/base/types.cpp
+++ b/core/test/base/types.cpp
@@ -33,6 +33,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
+#include <array>
+#include <cstdint>
+#include <type_traits>
+
+
 #include <gtest/gtest.h>
 
 
@@ -100,4 +105,109 @@ TEST(PrecisionReduction, ComputesCommonEncoding)
 }
 
 
+TEST(ConfigSet, MaskCorrectly)
+{
+    constexpr auto mask3_u = gko::detail::mask<3>();
+    constexpr auto fullmask_u = gko::detail::mask<32>();
+    constexpr auto mask3_u64 = gko::detail::mask<3, std::uint64_t>();
+    constexpr auto fullmask_u64 = gko::detail::mask<64, std::uint64_t>();
+
+    ASSERT_EQ(mask3_u, 7u);
+    ASSERT_EQ(fullmask_u, 0xffffffffu);
+    ASSERT_EQ((std::is_same<decltype(mask3_u), const unsigned int>::value),
+              true);
+    ASSERT_EQ((std::is_same<decltype(fullmask_u), const unsigned int>::value),
+              true);
+    ASSERT_EQ(mask3_u64, 7ull);
+    ASSERT_EQ(fullmask_u64, 0xffffffffffffffffull);
+    ASSERT_EQ((std::is_same<decltype(mask3_u64), const std::uint64_t>::value),
+              true);
+    ASSERT_EQ(
+        (std::is_same<decltype(fullmask_u64), const std::uint64_t>::value),
+        true);
+}
+
+
+TEST(ConfigSet, ShiftCorrectly)
+{
+    constexpr std::array<char, 3> bits{3, 5, 7};
+
+    constexpr auto shift0 = gko::detail::shift<3, 0>(bits);
+    constexpr auto shift1 = gko::detail::shift<3, 1>(bits);
+    constexpr auto shift2 = gko::detail::shift<3, 2>(bits);
+
+    ASSERT_EQ(shift0, 12);
+    ASSERT_EQ(shift1, 7);
+    ASSERT_EQ(shift2, 0);
+}
+
+
+TEST(ConfigSet, ConfigSet1Correctly)
+{
+    using Cfg = gko::ConfigSet<3>;
+
+    constexpr auto encoded = Cfg::encode(2);
+    constexpr auto decoded = Cfg::decode<0>(encoded);
+
+    ASSERT_EQ(encoded, 2);
+    ASSERT_EQ(decoded, 2);
+}
+
+
+TEST(ConfigSet, ConfigSet1FullCorrectly)
+{
+    using Cfg = gko::ConfigSet<32>;
+
+    constexpr auto encoded = Cfg::encode(0xffffffff);
+    constexpr auto decoded = Cfg::decode<0>(encoded);
+
+    ASSERT_EQ(encoded, 0xffffffff);
+    ASSERT_EQ(decoded, 0xffffffff);
+}
+
+
+TEST(ConfigSet, ConfigSet2FullCorrectly)
+{
+    using Cfg = gko::ConfigSet<1, 31>;
+
+    constexpr auto encoded = Cfg::encode(1, 33);
+
+    ASSERT_EQ(encoded, (1u << 31) + 33);
+}
+
+
+TEST(ConfigSet, ConfigSetSomeCorrectly)
+{
+    using Cfg = gko::ConfigSet<3, 5, 7>;
+
+    constexpr auto encoded = Cfg::encode(2, 11, 13);
+    constexpr auto decoded_0 = Cfg::decode<0>(encoded);
+    constexpr auto decoded_1 = Cfg::decode<1>(encoded);
+    constexpr auto decoded_2 = Cfg::decode<2>(encoded);
+
+    ASSERT_EQ(encoded, (2 << 12) + (11 << 7) + 13);
+    ASSERT_EQ(decoded_0, 2);
+    ASSERT_EQ(decoded_1, 11);
+    ASSERT_EQ(decoded_2, 13);
+}
+
+
+TEST(ConfigSet, ConfigSetSomeFullCorrectly)
+{
+    using Cfg = gko::ConfigSet<2, 6, 7, 17>;
+
+    constexpr auto encoded = Cfg::encode(2, 11, 13, 19);
+    constexpr auto decoded_0 = Cfg::decode<0>(encoded);
+    constexpr auto decoded_1 = Cfg::decode<1>(encoded);
+    constexpr auto decoded_2 = Cfg::decode<2>(encoded);
+    constexpr auto decoded_3 = Cfg::decode<3>(encoded);
+
+    ASSERT_EQ(encoded, (2 << 30) + (11 << 24) + (13 << 17) + 19);
+    ASSERT_EQ(decoded_0, 2);
+    ASSERT_EQ(decoded_1, 11);
+    ASSERT_EQ(decoded_2, 13);
+    ASSERT_EQ(decoded_3, 19);
+}
+
+
 }  // namespace
diff --git a/dev_tools/scripts/format_header.sh b/dev_tools/scripts/format_header.sh
index 09cb00cd939..cbe1b2e8ab9 100755
--- a/dev_tools/scripts/format_header.sh
+++ b/dev_tools/scripts/format_header.sh
@@ -155,8 +155,6 @@ CONSIDER_REGEX="${START_BLOCK_REX}|${END_BLOCK_REX}|${COMMENT_REGEX}|${INCLUDE_R
 
 # This part capture the main header and give the possible fail arrangement information
 while IFS='' read -r line || [ -n "$line" ]; do
-    # if [ "${line}" = '#include "hip/hip_runtime.h"' ] && [ "${SKIP}" = "true" ]; then
-    #     HAS_HIP_RUNTIME="true"
     if [ "${line}" = "/*${GINKGO_LICENSE_BEACON}" ] || [ "${DURING_LICENSE}" = "true" ]; then
         DURING_LICENSE="true"
         if [ "${line}" = "${GINKGO_LICENSE_BEACON}*/" ]; then
@@ -268,9 +266,6 @@ fi
 # Arrange the remain files and give
 if [ -f "${CONTENT}" ]; then
     add_regroup
-    # if [ "${HAS_HIP_RUNTIME}" = "true" ]; then
-    #     echo "#include <hip/hip_runtime.h>" > temp
-    # fi
     head -n -${KEEP_LINES} ${CONTENT} >> temp
     if [ ! -z "${IFNDEF}" ] && [ ! -z "${DEFINE}" ]; then
         # Ignore the last line #endif
diff --git a/dpcpp/base/dim3.dp.hpp b/dpcpp/base/dim3.dp.hpp
index 238d39f4f52..7017cb7b306 100644
--- a/dpcpp/base/dim3.dp.hpp
+++ b/dpcpp/base/dim3.dp.hpp
@@ -63,12 +63,12 @@ struct dim3 {
     {}
 
     /**
-     * reverse returns the range for sycl with correct ordering (reverse of
+     * get_range returns the range for sycl with correct ordering (reverse of
      * cuda)
      *
      * @return sycl::range<3>
      */
-    sycl::range<3> reverse() { return sycl::range<3>(z, y, x); }
+    sycl::range<3> get_range() { return sycl::range<3>(z, y, x); }
 };
 
 
@@ -82,8 +82,8 @@ struct dim3 {
  */
 inline sycl::nd_range<3> sycl_nd_range(dim3 grid, dim3 block)
 {
-    auto local_range = block.reverse();
-    auto global_range = grid.reverse() * local_range;
+    auto local_range = block.get_range();
+    auto global_range = grid.get_range() * local_range;
     return sycl::nd_range<3>(global_range, local_range);
 }
 
diff --git a/dpcpp/base/helper.hpp b/dpcpp/base/helper.hpp
index ed3adf2c228..8137ee841d6 100644
--- a/dpcpp/base/helper.hpp
+++ b/dpcpp/base/helper.hpp
@@ -45,18 +45,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 /**
  * GKO_ENABLE_DEFAULT_HOST_CONFIG gives a default host implementation for those
- * kernels which require config but do not need explicit template parameter and
- * share memory
+ * kernels which require encoded config but do not need explicit template
+ * parameter and share memory
+ *
+ * @param name_  the name of the host function with config
+ * @param kernel_  the kernel name
  */
 #define GKO_ENABLE_DEFAULT_HOST_CONFIG(name_, kernel_)                     \
-    template <int config, typename... InferredArgs>                        \
+    template <int encoded, typename... InferredArgs>                       \
     inline void name_(dim3 grid, dim3 block, size_t dynamic_shared_memory, \
-                      sycl::queue *stream, InferredArgs... args)           \
+                      sycl::queue *queue, InferredArgs... args)            \
     {                                                                      \
-        stream->submit([&](sycl::handler &cgh) {                           \
+        queue->submit([&](sycl::handler &cgh) {                            \
             cgh.parallel_for(sycl_nd_range(grid, block),                   \
                              [=](sycl::nd_item<3> item_ct1) {              \
-                                 kernel_<config>(args..., item_ct1);       \
+                                 kernel_<encoded>(args..., item_ct1);      \
                              });                                           \
         });                                                                \
     }
@@ -65,11 +68,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  * GKO_ENABLE_DEFAULT_CONFIG_CALL gives a default config selection call
  * implementation for those kernels which require config selection but do not
  * need explicit template parameter
+ *
+ * @param name_  the name of the calling function
+ * @param callable_  the host function with selection
+ * @param cfg_  the ConfigSet for encode/decode method
+ * @param list_  the list for encoded config selection, whose value should be
+ *               available to decode<0> for blocksize and decode<1> for
+ *               subgroup_size by cfg_
  */
 #define GKO_ENABLE_DEFAULT_CONFIG_CALL(name_, callable_, cfg_, list_)      \
     template <typename... InferredArgs>                                    \
     void name_(dim3 grid, dim3 block, size_t dynamic_shared_memory,        \
-               sycl::queue *stream,                                        \
+               sycl::queue *queue,                                         \
                std::shared_ptr<const gko::DpcppExecutor> exec,             \
                InferredArgs... args)                                       \
     {                                                                      \
@@ -84,7 +94,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
             ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(), \
             ::gko::syn::value_list<gko::size_type>(),                      \
             ::gko::syn::type_list<>(), grid, block, dynamic_shared_memory, \
-            stream, std::forward<InferredArgs>(args)...);                  \
+            queue, std::forward<InferredArgs>(args)...);                   \
     }
 
 // __WG_BOUND__ gives the cuda-like launch bound in cuda ordering
diff --git a/dpcpp/components/cooperative_groups.dp.hpp b/dpcpp/components/cooperative_groups.dp.hpp
index 1cf3c6955e2..78f9d63d698 100644
--- a/dpcpp/components/cooperative_groups.dp.hpp
+++ b/dpcpp/components/cooperative_groups.dp.hpp
@@ -348,7 +348,7 @@ class thread_block_tile<1> {
 using detail::thread_block_tile;
 
 
-// Only support tile_partition with 8, 16, 32.
+// Only support tile_partition with 2, 4, 8, 16, 32, 64.
 template <unsigned Size, typename Group>
 __dpct_inline__
     std::enable_if_t<(Size > 1) && Size <= 64 && !(Size & (Size - 1)),
diff --git a/dpcpp/test/base/dim3.dp.cpp b/dpcpp/test/base/dim3.dp.cpp
index 8b6a08fa996..8d5485749b8 100644
--- a/dpcpp/test/base/dim3.dp.cpp
+++ b/dpcpp/test/base/dim3.dp.cpp
@@ -45,20 +45,10 @@ namespace {
 using namespace gko::kernels::dpcpp;
 
 
-class DpcppDim3 : public ::testing::Test {
-protected:
-    DpcppDim3() {}
-
-    void SetUp() {}
-
-    void TearDown() {}
-};
-
-
-TEST_F(DpcppDim3, CanGenerate1DRange)
+TEST(DpcppDim3, CanGenerate1DRange)
 {
     dim3 block(3);
-    auto sycl_block = block.reverse();
+    auto sycl_block = block.get_range();
 
     ASSERT_EQ(block.x, 3);
     ASSERT_EQ(block.y, 1);
@@ -69,10 +59,10 @@ TEST_F(DpcppDim3, CanGenerate1DRange)
 }
 
 
-TEST_F(DpcppDim3, CanGenerate2DRange)
+TEST(DpcppDim3, CanGenerate2DRange)
 {
     dim3 block(3, 5);
-    auto sycl_block = block.reverse();
+    auto sycl_block = block.get_range();
 
     ASSERT_EQ(block.x, 3);
     ASSERT_EQ(block.y, 5);
@@ -83,10 +73,10 @@ TEST_F(DpcppDim3, CanGenerate2DRange)
 }
 
 
-TEST_F(DpcppDim3, CanGenerate3DRange)
+TEST(DpcppDim3, CanGenerate3DRange)
 {
     dim3 block(3, 5, 7);
-    auto sycl_block = block.reverse();
+    auto sycl_block = block.get_range();
 
     ASSERT_EQ(block.x, 3);
     ASSERT_EQ(block.y, 5);
@@ -97,7 +87,7 @@ TEST_F(DpcppDim3, CanGenerate3DRange)
 }
 
 
-TEST_F(DpcppDim3, CanGenerateNDRange)
+TEST(DpcppDim3, CanGenerateNDRange)
 {
     dim3 block(3, 5, 7);
     dim3 grid(17, 13, 11);
diff --git a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
index a5c34149381..5cf9667ed2a 100644
--- a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
+++ b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
@@ -70,8 +70,8 @@ class CooperativeGroups : public testing::TestWithParam<int> {
     CooperativeGroups()
         : ref(gko::ReferenceExecutor::create()),
           dpcpp(gko::DpcppExecutor::create(0, ref)),
-          max_test_case(3),
-          max_num(max_test_case * 64),
+          test_case(3),
+          max_num(test_case * 64),
           result(ref, max_num),
           dresult(dpcpp)
     {
@@ -87,7 +87,7 @@ class CooperativeGroups : public testing::TestWithParam<int> {
         auto subgroup_size = GetParam();
         auto exec_info = dpcpp->get_const_exec_info();
         if (exec_info.validate(subgroup_size, subgroup_size)) {
-            for (int i = 0; i < max_test_case * subgroup_size; i++) {
+            for (int i = 0; i < test_case * subgroup_size; i++) {
                 result.get_data()[i] = true;
             }
 
@@ -102,7 +102,7 @@ class CooperativeGroups : public testing::TestWithParam<int> {
         }
     }
 
-    int max_test_case;
+    int test_case;
     int max_num;
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<gko::DpcppExecutor> dpcpp;
@@ -111,13 +111,6 @@ class CooperativeGroups : public testing::TestWithParam<int> {
 };
 
 
-void test_assert(bool *success, bool partial)
-{
-    if (!partial) {
-        *success = false;
-    }
-}
-
 // kernel implementation
 template <int config>
 __WG_BOUND__(KCfg::decode<0>(config))
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index f942b80cc89..38b8a4a9256 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -268,6 +268,8 @@ constexpr std::enable_if_t<(num_groups > current_shift + 1), int> shift(
  * 1/2/k position and r is for rest of unused bits.
  *
  * @tparam num_bits...  the number of bits for each position.
+ *
+ * @note the num_bit is required at least $log_2(maxval) + 1$
  */
 template <int... num_bits>
 class ConfigSet {
@@ -285,7 +287,7 @@ class ConfigSet {
      * @return the decoded information at position
      */
     template <int position>
-    static constexpr int decode(int encoded)
+    static constexpr unsigned decode(unsigned encoded)
     {
         static_assert(position < num_groups,
                       "This position is over the bounds.");
@@ -300,7 +302,7 @@ class ConfigSet {
      * @note the last case of nested template.
      */
     template <size_type current_iter>
-    static constexpr std::enable_if_t<(current_iter == num_groups), int>
+    static constexpr std::enable_if_t<(current_iter == num_groups), unsigned>
     encode()
     {
         return 0;
@@ -310,16 +312,16 @@ class ConfigSet {
      * Encodes the information with given bit set to encoded integer.
      *
      * @tparam current_iter  the encoded place
-     * @tparam First  the current_iter type
-     * @tparam Rest...  the rest information
+     * @tparam Rest...  the rest type
      *
-     * @param informations... the information will be encoded
+     * @param first  the current encoded information
+     * @param rest...  the rest of others information waits for encoding
      *
      * @return the encoded integer
      */
-    template <size_type current_iter = 0, typename First, typename... Rest>
-    static constexpr std::enable_if_t<(current_iter < num_groups), int> encode(
-        First first, Rest &&... rest)
+    template <size_type current_iter = 0, typename... Rest>
+    static constexpr std::enable_if_t<(current_iter < num_groups), unsigned>
+    encode(unsigned first, Rest &&... rest)
     {
         constexpr int shift = detail::shift<num_groups, current_iter>(bits);
         return (first << shift) |

From c0dde738cf542681c1707217be6befe74e40dbb9 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 24 May 2021 00:28:11 +0200
Subject: [PATCH 0723/1008] directly use queue not exec_info

---
 dpcpp/base/helper.hpp                         | 38 +++++++++---
 .../cooperative_groups_kernels.dp.cpp         | 60 +++++++------------
 include/ginkgo/core/base/executor.hpp         | 28 ---------
 3 files changed, 54 insertions(+), 72 deletions(-)

diff --git a/dpcpp/base/helper.hpp b/dpcpp/base/helper.hpp
index 8137ee841d6..55ab9450b36 100644
--- a/dpcpp/base/helper.hpp
+++ b/dpcpp/base/helper.hpp
@@ -79,16 +79,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_ENABLE_DEFAULT_CONFIG_CALL(name_, callable_, cfg_, list_)      \
     template <typename... InferredArgs>                                    \
     void name_(dim3 grid, dim3 block, size_t dynamic_shared_memory,        \
-               sycl::queue *queue,                                         \
-               std::shared_ptr<const gko::DpcppExecutor> exec,             \
-               InferredArgs... args)                                       \
+               sycl::queue *queue, InferredArgs... args)                   \
     {                                                                      \
-        auto exec_info = exec->get_const_exec_info();                      \
         callable_(                                                         \
             list_,                                                         \
-            [&exec_info, &block](int config) {                             \
-                return exec_info.validate(cfg_::decode<0>(config),         \
-                                          cfg_::decode<1>(config)) &&      \
+            [&block, &queue](unsigned int config) {                        \
+                return gko::kernels::dpcpp::validate(                      \
+                           queue, cfg_::decode<0>(config),                 \
+                           cfg_::decode<1>(config)) &&                     \
                        (cfg_::decode<0>(config) == block.x);               \
             },                                                             \
             ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(), \
@@ -112,4 +110,30 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     __WG_BOUND_3D__(CFG::decode<0>(cfg), CFG::decode<1>(cfg), \
                     CFG::decode<2>(cfg))
 
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+bool validate(sycl::queue *queue, unsigned int workgroup_size,
+              unsigned int subgroup_size)
+{
+    auto device = queue->get_device();
+    auto subgroup_size_list =
+        device.get_info<cl::sycl::info::device::sub_group_sizes>();
+    auto max_workgroup_size =
+        device.get_info<sycl::info::device::max_work_group_size>();
+    bool allowed = false;
+    for (auto &i : subgroup_size_list) {
+        allowed |= (i == subgroup_size);
+    }
+    return allowed && (workgroup_size <= max_workgroup_size);
+}
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
 #endif  // GKO_DPCPP_BASE_HELPER_HPP_
diff --git a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
index 5cf9667ed2a..997b15aecb8 100644
--- a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
+++ b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
@@ -85,14 +85,14 @@ class CooperativeGroups : public testing::TestWithParam<int> {
     void test_all_subgroup(Kernel kernel)
     {
         auto subgroup_size = GetParam();
-        auto exec_info = dpcpp->get_const_exec_info();
-        if (exec_info.validate(subgroup_size, subgroup_size)) {
+        auto queue = dpcpp->get_queue();
+        if (gko::kernels::dpcpp::validate(queue, subgroup_size,
+                                          subgroup_size)) {
             for (int i = 0; i < test_case * subgroup_size; i++) {
                 result.get_data()[i] = true;
             }
 
-            kernel(1, subgroup_size, 0, dpcpp->get_queue(), dpcpp,
-                   dresult.get_data());
+            kernel(1, subgroup_size, 0, dpcpp->get_queue(), dresult.get_data());
 
             // each subgreoup size segment for one test
             GKO_ASSERT_ARRAY_EQ(result, dresult);
@@ -131,9 +131,9 @@ void cg_shuffle(bool *s, sycl::nd_item<3> item_ct1)
 // group all kernel things together
 template <int config>
 void cg_shuffle_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                     sycl::queue *stream, bool *s)
+                     sycl::queue *queue, bool *s)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              cg_shuffle<config>(s, item_ct1);
@@ -146,22 +146,19 @@ GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_shuffle_config, cg_shuffle_host)
 
 // the call
 void cg_shuffle_config_call(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                            sycl::queue *stream,
-                            std::shared_ptr<const gko::DpcppExecutor> exec,
-                            bool *s)
+                            sycl::queue *queue, bool *s)
 {
-    auto exec_info = exec->get_const_exec_info();
     cg_shuffle_config(
         default_config_list,
         // validate
-        [&exec_info, &block](int config) {
-            return exec_info.validate(KCfg::decode<0>(config),
-                                      KCfg::decode<1>(config)) &&
+        [&block, &queue](unsigned int config) {
+            return gko::kernels::dpcpp::validate(queue, KCfg::decode<0>(config),
+                                                 KCfg::decode<1>(config)) &&
                    (KCfg::decode<0>(config) == block.x);
         },
         ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
         ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
-        grid, block, dynamic_shared_memory, stream, s);
+        grid, block, dynamic_shared_memory, queue, s);
 }
 
 TEST_P(CooperativeGroups, Shuffle)
@@ -185,15 +182,11 @@ void cg_all(bool *s, sycl::nd_item<3> item_ct1)
         group.all(item_ct1.get_local_id(2) < 13) == sg_size < 13;
 }
 
-GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_all_host, cg_all)
-GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_all_config, cg_all_host)
-GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_all_config_call, cg_all_config, KCfg,
-                               default_config_list)
+GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_all, cg_all)
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_all, cg_all)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_all_call, cg_all, KCfg, default_config_list)
 
-TEST_P(CooperativeGroups, All)
-{
-    test_all_subgroup(cg_all_config_call<bool *>);
-}
+TEST_P(CooperativeGroups, All) { test_all_subgroup(cg_all_call<bool *>); }
 
 
 template <int config>
@@ -210,15 +203,11 @@ void cg_any(bool *s, sycl::nd_item<3> item_ct1)
     s[i + sg_size * 2] = !group.any(false);
 }
 
-GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_any_host, cg_any)
-GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_any_config, cg_any_host)
-GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_any_config_call, cg_any_config, KCfg,
-                               default_config_list)
+GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_any, cg_any)
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_any, cg_any)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_any_call, cg_any, KCfg, default_config_list)
 
-TEST_P(CooperativeGroups, Any)
-{
-    test_all_subgroup(cg_any_config_call<bool *>);
-}
+TEST_P(CooperativeGroups, Any) { test_all_subgroup(cg_any_call<bool *>); }
 
 
 template <int config>
@@ -236,15 +225,12 @@ void cg_ballot(bool *s, sycl::nd_item<3> item_ct1)
     s[i + sg_size * 2] = group.ballot(item_ct1.get_local_id(2) < 4) == 0xf;
 }
 
-GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_ballot_host, cg_ballot)
-GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_ballot_config, cg_ballot_host)
-GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_ballot_config_call, cg_ballot_config, KCfg,
+GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_ballot, cg_ballot)
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_ballot, cg_ballot)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_ballot_call, cg_ballot, KCfg,
                                default_config_list)
 
-TEST_P(CooperativeGroups, Ballot)
-{
-    test_all_subgroup(cg_ballot_config_call<bool *>);
-}
+TEST_P(CooperativeGroups, Ballot) { test_all_subgroup(cg_ballot_call<bool *>); }
 
 INSTANTIATE_TEST_SUITE_P(DifferentSubgroup, CooperativeGroups,
                          testing::Values(4, 8, 16, 32, 64),
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index fb1cd1ed232..a118cc60c29 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -811,36 +811,8 @@ class Executor : public log::EnableLogging<Executor> {
          *       multicore processor, or a single thread in a SMT processor"
          */
         std::vector<int> closest_pu_ids{};
-
-        /**
-         * The validate function for Config
-         */
-        bool validate(int blocksize, int warpsize)
-        {
-            bool allowed = false;
-            for (auto &i : subgroup_sizes) {
-                allowed |= (i == warpsize);
-            }
-            return allowed && (blocksize <= max_workgroup_size);
-        }
-
-        auto validate_function() const
-        {
-            return [subgroup_sizes = subgroup_sizes,
-                    max_workgroup_size = max_workgroup_size](int blocksize,
-                                                             int warpsize) {
-                bool allowed = false;
-                for (auto &i : subgroup_sizes) {
-                    allowed |= (i == warpsize);
-                }
-                return allowed && (blocksize <= max_workgroup_size);
-            };
-        }
     };
 
-public:
-    const exec_info get_const_exec_info() const { return this->exec_info_; }
-
 protected:
     /**
      * Gets the exec info struct

From 237c3bdd78331d237a70ccf6e4f4e9fba3e262c7 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 24 May 2021 11:56:03 +0200
Subject: [PATCH 0724/1008] using ::gko::ConfigSetType

---
 core/synthesizer/implementation_selection.hpp | 60 ++++++++++---------
 dpcpp/base/helper.hpp                         |  7 ++-
 .../cooperative_groups_kernels.dp.cpp         | 19 +++---
 include/ginkgo/core/base/executor.hpp         |  1 -
 include/ginkgo/core/base/types.hpp            | 13 ++--
 5 files changed, 56 insertions(+), 44 deletions(-)

diff --git a/core/synthesizer/implementation_selection.hpp b/core/synthesizer/implementation_selection.hpp
index b6e9400ad52..734024d749c 100644
--- a/core/synthesizer/implementation_selection.hpp
+++ b/core/synthesizer/implementation_selection.hpp
@@ -70,35 +70,37 @@ namespace syn {
         }                                                                    \
     }
 
-#define GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(_name, _callable)          \
-    template <typename Predicate, bool... BoolArgs, int... IntArgs,           \
-              gko::size_type... SizeTArgs, typename... TArgs,                 \
-              typename... InferredArgs>                                       \
-    inline void _name(::gko::syn::value_list<int>, Predicate,                 \
-                      ::gko::syn::value_list<bool, BoolArgs...>,              \
-                      ::gko::syn::value_list<int, IntArgs...>,                \
-                      ::gko::syn::value_list<gko::size_type, SizeTArgs...>,   \
-                      ::gko::syn::type_list<TArgs...>, InferredArgs...)       \
-        GKO_KERNEL_NOT_FOUND;                                                 \
-                                                                              \
-    template <int K, int... Rest, typename Predicate, bool... BoolArgs,       \
-              int... IntArgs, gko::size_type... SizeTArgs, typename... TArgs, \
-              typename... InferredArgs>                                       \
-    inline void _name(                                                        \
-        ::gko::syn::value_list<int, K, Rest...>, Predicate is_eligible,       \
-        ::gko::syn::value_list<bool, BoolArgs...> bool_args,                  \
-        ::gko::syn::value_list<int, IntArgs...> int_args,                     \
-        ::gko::syn::value_list<gko::size_type, SizeTArgs...> size_args,       \
-        ::gko::syn::type_list<TArgs...> type_args, InferredArgs... args)      \
-    {                                                                         \
-        if (is_eligible(K)) {                                                 \
-            _callable<BoolArgs..., IntArgs..., SizeTArgs..., TArgs..., K>(    \
-                std::forward<InferredArgs>(args)...);                         \
-        } else {                                                              \
-            _name(::gko::syn::value_list<int, Rest...>(), is_eligible,        \
-                  bool_args, int_args, size_args, type_args,                  \
-                  std::forward<InferredArgs>(args)...);                       \
-        }                                                                     \
+#define GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(_name, _callable)           \
+    template <typename Predicate, bool... BoolArgs, int... IntArgs,            \
+              gko::size_type... SizeTArgs, typename... TArgs,                  \
+              typename... InferredArgs>                                        \
+    inline void _name(::gko::syn::value_list<::gko::ConfigSetType>, Predicate, \
+                      ::gko::syn::value_list<bool, BoolArgs...>,               \
+                      ::gko::syn::value_list<int, IntArgs...>,                 \
+                      ::gko::syn::value_list<gko::size_type, SizeTArgs...>,    \
+                      ::gko::syn::type_list<TArgs...>, InferredArgs...)        \
+        GKO_KERNEL_NOT_FOUND;                                                  \
+                                                                               \
+    template <::gko::ConfigSetType K, ::gko::ConfigSetType... Rest,            \
+              typename Predicate, bool... BoolArgs, int... IntArgs,            \
+              gko::size_type... SizeTArgs, typename... TArgs,                  \
+              typename... InferredArgs>                                        \
+    inline void _name(                                                         \
+        ::gko::syn::value_list<::gko::ConfigSetType, K, Rest...>,              \
+        Predicate is_eligible,                                                 \
+        ::gko::syn::value_list<bool, BoolArgs...> bool_args,                   \
+        ::gko::syn::value_list<int, IntArgs...> int_args,                      \
+        ::gko::syn::value_list<gko::size_type, SizeTArgs...> size_args,        \
+        ::gko::syn::type_list<TArgs...> type_args, InferredArgs... args)       \
+    {                                                                          \
+        if (is_eligible(K)) {                                                  \
+            _callable<BoolArgs..., IntArgs..., SizeTArgs..., TArgs..., K>(     \
+                std::forward<InferredArgs>(args)...);                          \
+        } else {                                                               \
+            _name(::gko::syn::value_list<::gko::ConfigSetType, Rest...>(),     \
+                  is_eligible, bool_args, int_args, size_args, type_args,      \
+                  std::forward<InferredArgs>(args)...);                        \
+        }                                                                      \
     }
 
 
diff --git a/dpcpp/base/helper.hpp b/dpcpp/base/helper.hpp
index 55ab9450b36..476cfcce1b2 100644
--- a/dpcpp/base/helper.hpp
+++ b/dpcpp/base/helper.hpp
@@ -40,6 +40,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <CL/sycl.hpp>
 
 
+#include <ginkgo/core/base/types.hpp>
+
+
 #include "dpcpp/base/dim3.dp.hpp"
 
 
@@ -52,7 +55,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  * @param kernel_  the kernel name
  */
 #define GKO_ENABLE_DEFAULT_HOST_CONFIG(name_, kernel_)                     \
-    template <int encoded, typename... InferredArgs>                       \
+    template <::gko::ConfigSetType encoded, typename... InferredArgs>      \
     inline void name_(dim3 grid, dim3 block, size_t dynamic_shared_memory, \
                       sycl::queue *queue, InferredArgs... args)            \
     {                                                                      \
@@ -83,7 +86,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     {                                                                      \
         callable_(                                                         \
             list_,                                                         \
-            [&block, &queue](unsigned int config) {                        \
+            [&block, &queue](::gko::ConfigSetType config) {                \
                 return gko::kernels::dpcpp::validate(                      \
                            queue, cfg_::decode<0>(config),                 \
                            cfg_::decode<1>(config)) &&                     \
diff --git a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
index 997b15aecb8..3114e6564fc 100644
--- a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
+++ b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/types.hpp>
 
 
 #include "core/synthesizer/implementation_selection.hpp"
@@ -60,12 +61,12 @@ namespace {
 using namespace gko::kernels::dpcpp;
 using KCfg = gko::ConfigSet<12, 7>;
 constexpr auto default_config_list =
-    ::gko::syn::value_list<int, KCfg::encode(64, 64), KCfg::encode(32, 32),
-                           KCfg::encode(16, 16), KCfg::encode(8, 8),
-                           KCfg::encode(4, 4)>();
+    ::gko::syn::value_list<::gko::ConfigSetType, KCfg::encode(64, 64),
+                           KCfg::encode(32, 32), KCfg::encode(16, 16),
+                           KCfg::encode(8, 8), KCfg::encode(4, 4)>();
 
 
-class CooperativeGroups : public testing::TestWithParam<int> {
+class CooperativeGroups : public testing::TestWithParam<unsigned int> {
 protected:
     CooperativeGroups()
         : ref(gko::ReferenceExecutor::create()),
@@ -112,7 +113,7 @@ class CooperativeGroups : public testing::TestWithParam<int> {
 
 
 // kernel implementation
-template <int config>
+template <::gko::ConfigSetType config>
 __WG_BOUND__(KCfg::decode<0>(config))
 void cg_shuffle(bool *s, sycl::nd_item<3> item_ct1)
 {
@@ -167,7 +168,7 @@ TEST_P(CooperativeGroups, Shuffle)
 }
 
 
-template <int config>
+template <::gko::ConfigSetType config>
 __WG_BOUND__(KCfg::decode<0>(config))
 void cg_all(bool *s, sycl::nd_item<3> item_ct1)
 {
@@ -189,7 +190,7 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_all_call, cg_all, KCfg, default_config_list)
 TEST_P(CooperativeGroups, All) { test_all_subgroup(cg_all_call<bool *>); }
 
 
-template <int config>
+template <::gko::ConfigSetType config>
 __WG_BOUND__(KCfg::decode<0>(config))
 void cg_any(bool *s, sycl::nd_item<3> item_ct1)
 {
@@ -210,7 +211,7 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_any_call, cg_any, KCfg, default_config_list)
 TEST_P(CooperativeGroups, Any) { test_all_subgroup(cg_any_call<bool *>); }
 
 
-template <int config>
+template <::gko::ConfigSetType config>
 __WG_BOUND__(KCfg::decode<0>(config))
 void cg_ballot(bool *s, sycl::nd_item<3> item_ct1)
 {
@@ -232,8 +233,10 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_ballot_call, cg_ballot, KCfg,
 
 TEST_P(CooperativeGroups, Ballot) { test_all_subgroup(cg_ballot_call<bool *>); }
 
+
 INSTANTIATE_TEST_SUITE_P(DifferentSubgroup, CooperativeGroups,
                          testing::Values(4, 8, 16, 32, 64),
                          testing::PrintToStringParamName());
 
+
 }  // namespace
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index a118cc60c29..d9ca2e3af32 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -813,7 +813,6 @@ class Executor : public log::EnableLogging<Executor> {
         std::vector<int> closest_pu_ids{};
     };
 
-protected:
     /**
      * Gets the exec info struct
      *
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 38b8a4a9256..9c2c021f0da 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -257,6 +257,9 @@ constexpr std::enable_if_t<(num_groups > current_shift + 1), int> shift(
 }  // namespace detail
 
 
+// a shortcut type for ConfigSet
+using ConfigSetType = unsigned int;
+
 /**
  * ConfigSet is a way to embed several information into one integer by given
  * certain bits. The usage will be the following
@@ -287,7 +290,7 @@ class ConfigSet {
      * @return the decoded information at position
      */
     template <int position>
-    static constexpr unsigned decode(unsigned encoded)
+    static constexpr ConfigSetType decode(ConfigSetType encoded)
     {
         static_assert(position < num_groups,
                       "This position is over the bounds.");
@@ -302,7 +305,8 @@ class ConfigSet {
      * @note the last case of nested template.
      */
     template <size_type current_iter>
-    static constexpr std::enable_if_t<(current_iter == num_groups), unsigned>
+    static constexpr std::enable_if_t<(current_iter == num_groups),
+                                      ConfigSetType>
     encode()
     {
         return 0;
@@ -320,8 +324,9 @@ class ConfigSet {
      * @return the encoded integer
      */
     template <size_type current_iter = 0, typename... Rest>
-    static constexpr std::enable_if_t<(current_iter < num_groups), unsigned>
-    encode(unsigned first, Rest &&... rest)
+    static constexpr std::enable_if_t<(current_iter < num_groups),
+                                      ConfigSetType>
+    encode(ConfigSetType first, Rest &&... rest)
     {
         constexpr int shift = detail::shift<num_groups, current_iter>(bits);
         return (first << shift) |

From f71277d3c41e231905cdb33f3da1951f6987b7af Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 25 May 2021 22:58:55 +0200
Subject: [PATCH 0725/1008] pass desired_cfg in arg to ensure cfg is desired.

---
 dpcpp/base/helper.hpp                         | 14 ++++++-------
 .../cooperative_groups_kernels.dp.cpp         | 20 +++++++++----------
 2 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/dpcpp/base/helper.hpp b/dpcpp/base/helper.hpp
index 476cfcce1b2..e6039757a37 100644
--- a/dpcpp/base/helper.hpp
+++ b/dpcpp/base/helper.hpp
@@ -79,18 +79,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *               available to decode<0> for blocksize and decode<1> for
  *               subgroup_size by cfg_
  */
-#define GKO_ENABLE_DEFAULT_CONFIG_CALL(name_, callable_, cfg_, list_)      \
+#define GKO_ENABLE_DEFAULT_CONFIG_CALL(name_, callable_, list_)            \
     template <typename... InferredArgs>                                    \
-    void name_(dim3 grid, dim3 block, size_t dynamic_shared_memory,        \
-               sycl::queue *queue, InferredArgs... args)                   \
+    void name_(::gko::ConfigSetType desired_cfg, dim3 grid, dim3 block,    \
+               size_t dynamic_shared_memory, sycl::queue *queue,           \
+               InferredArgs... args)                                       \
     {                                                                      \
         callable_(                                                         \
             list_,                                                         \
-            [&block, &queue](::gko::ConfigSetType config) {                \
-                return gko::kernels::dpcpp::validate(                      \
-                           queue, cfg_::decode<0>(config),                 \
-                           cfg_::decode<1>(config)) &&                     \
-                       (cfg_::decode<0>(config) == block.x);               \
+            [&desired_cfg](::gko::ConfigSetType cfg) {                     \
+                return cfg == desired_cfg;                                 \
             },                                                             \
             ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(), \
             ::gko::syn::value_list<gko::size_type>(),                      \
diff --git a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
index 3114e6564fc..d1526b3c125 100644
--- a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
+++ b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
@@ -89,11 +89,13 @@ class CooperativeGroups : public testing::TestWithParam<unsigned int> {
         auto queue = dpcpp->get_queue();
         if (gko::kernels::dpcpp::validate(queue, subgroup_size,
                                           subgroup_size)) {
+            const auto cfg = KCfg::encode(subgroup_size, subgroup_size);
             for (int i = 0; i < test_case * subgroup_size; i++) {
                 result.get_data()[i] = true;
             }
 
-            kernel(1, subgroup_size, 0, dpcpp->get_queue(), dresult.get_data());
+            kernel(cfg, 1, subgroup_size, 0, dpcpp->get_queue(),
+                   dresult.get_data());
 
             // each subgreoup size segment for one test
             GKO_ASSERT_ARRAY_EQ(result, dresult);
@@ -146,17 +148,14 @@ void cg_shuffle_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_shuffle_config, cg_shuffle_host)
 
 // the call
-void cg_shuffle_config_call(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+void cg_shuffle_config_call(::gko::ConfigSetType desired_cfg, dim3 grid,
+                            dim3 block, size_t dynamic_shared_memory,
                             sycl::queue *queue, bool *s)
 {
     cg_shuffle_config(
         default_config_list,
         // validate
-        [&block, &queue](unsigned int config) {
-            return gko::kernels::dpcpp::validate(queue, KCfg::decode<0>(config),
-                                                 KCfg::decode<1>(config)) &&
-                   (KCfg::decode<0>(config) == block.x);
-        },
+        [&desired_cfg](::gko::ConfigSetType cfg) { return cfg == desired_cfg; },
         ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
         ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
         grid, block, dynamic_shared_memory, queue, s);
@@ -185,7 +184,7 @@ void cg_all(bool *s, sycl::nd_item<3> item_ct1)
 
 GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_all, cg_all)
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_all, cg_all)
-GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_all_call, cg_all, KCfg, default_config_list)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_all_call, cg_all, default_config_list)
 
 TEST_P(CooperativeGroups, All) { test_all_subgroup(cg_all_call<bool *>); }
 
@@ -206,7 +205,7 @@ void cg_any(bool *s, sycl::nd_item<3> item_ct1)
 
 GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_any, cg_any)
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_any, cg_any)
-GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_any_call, cg_any, KCfg, default_config_list)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_any_call, cg_any, default_config_list)
 
 TEST_P(CooperativeGroups, Any) { test_all_subgroup(cg_any_call<bool *>); }
 
@@ -228,8 +227,7 @@ void cg_ballot(bool *s, sycl::nd_item<3> item_ct1)
 
 GKO_ENABLE_DEFAULT_HOST_CONFIG(cg_ballot, cg_ballot)
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_ballot, cg_ballot)
-GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_ballot_call, cg_ballot, KCfg,
-                               default_config_list)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_ballot_call, cg_ballot, default_config_list)
 
 TEST_P(CooperativeGroups, Ballot) { test_all_subgroup(cg_ballot_call<bool *>); }
 

From 4ea56437d789918361ff6cb6474082b1dc5357e0 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 28 May 2021 01:34:58 +0200
Subject: [PATCH 0726/1008] switch shift order, add ConfigSet check, review
 update

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/test/base/types.cpp                      | 36 ++++++----
 .../cooperative_groups_kernels.dp.cpp         |  2 +-
 include/ginkgo/core/base/types.hpp            | 70 ++++++++++++++-----
 3 files changed, 76 insertions(+), 32 deletions(-)

diff --git a/core/test/base/types.cpp b/core/test/base/types.cpp
index 71c0101954f..68cfd76eeb9 100644
--- a/core/test/base/types.cpp
+++ b/core/test/base/types.cpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <array>
 #include <cstdint>
+#include <stdexcept>
 #include <type_traits>
 
 
@@ -114,27 +115,26 @@ TEST(ConfigSet, MaskCorrectly)
 
     ASSERT_EQ(mask3_u, 7u);
     ASSERT_EQ(fullmask_u, 0xffffffffu);
-    ASSERT_EQ((std::is_same<decltype(mask3_u), const unsigned int>::value),
-              true);
-    ASSERT_EQ((std::is_same<decltype(fullmask_u), const unsigned int>::value),
-              true);
+    ASSERT_TRUE((std::is_same<decltype(mask3_u), const unsigned int>::value));
+    ASSERT_TRUE(
+        (std::is_same<decltype(fullmask_u), const unsigned int>::value));
     ASSERT_EQ(mask3_u64, 7ull);
     ASSERT_EQ(fullmask_u64, 0xffffffffffffffffull);
-    ASSERT_EQ((std::is_same<decltype(mask3_u64), const std::uint64_t>::value),
-              true);
-    ASSERT_EQ(
-        (std::is_same<decltype(fullmask_u64), const std::uint64_t>::value),
-        true);
+    ASSERT_TRUE(
+        (std::is_same<decltype(mask3_u64), const std::uint64_t>::value));
+    ASSERT_TRUE(
+        (std::is_same<decltype(fullmask_u64), const std::uint64_t>::value));
 }
 
 
 TEST(ConfigSet, ShiftCorrectly)
 {
-    constexpr std::array<char, 3> bits{3, 5, 7};
+    constexpr std::array<unsigned char, 3> bits{3, 5, 7};
 
-    constexpr auto shift0 = gko::detail::shift<3, 0>(bits);
-    constexpr auto shift1 = gko::detail::shift<3, 1>(bits);
-    constexpr auto shift2 = gko::detail::shift<3, 2>(bits);
+
+    constexpr auto shift0 = gko::detail::shift<0, 3>(bits);
+    constexpr auto shift1 = gko::detail::shift<1, 3>(bits);
+    constexpr auto shift2 = gko::detail::shift<2, 3>(bits);
 
     ASSERT_EQ(shift0, 12);
     ASSERT_EQ(shift1, 7);
@@ -210,4 +210,14 @@ TEST(ConfigSet, ConfigSetSomeFullCorrectly)
 }
 
 
+TEST(ConfigSet, ThrowOutOfBoundWhenExceedRepresentation)
+{
+    using Cfg = gko::ConfigSet<3, 2, 1>;
+
+    ASSERT_THROW(auto a = Cfg::encode(0, 0, 2), std::out_of_range);
+    ASSERT_THROW(auto a = Cfg::encode(0, 4, 0), std::out_of_range);
+    ASSERT_THROW(auto a = Cfg::encode(8, 0, 0), std::out_of_range);
+}
+
+
 }  // namespace
diff --git a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
index d1526b3c125..f581ef9f19a 100644
--- a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
+++ b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
@@ -59,7 +59,7 @@ namespace {
 
 
 using namespace gko::kernels::dpcpp;
-using KCfg = gko::ConfigSet<12, 7>;
+using KCfg = gko::ConfigSet<11, 7>;
 constexpr auto default_config_list =
     ::gko::syn::value_list<::gko::ConfigSetType, KCfg::encode(64, 64),
                            KCfg::encode(32, 32), KCfg::encode(16, 16),
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 9c2c021f0da..08c98fb2e95 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -41,6 +41,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <cstddef>
 #include <cstdint>
 #include <limits>
+#include <stdexcept>
+#include <string>
 #include <type_traits>
 
 
@@ -226,16 +228,16 @@ constexpr std::enable_if_t<Size == sizeof(ValueType) * 8, ValueType> mask()
 /**
  * shift calculates the number of bits for shifting
  *
- * @tparam num_groups  the number of elements in array
  * @tparam current_shift  the current position of shifting
+ * @tparam num_groups  the number of elements in array
  *
  * @return the number of shifting bits
  *
  * @note this is the last case of nested template
  */
-template <int num_groups, int current_shift>
+template <int current_shift, int num_groups>
 constexpr std::enable_if_t<(num_groups == current_shift + 1), int> shift(
-    const std::array<char, num_groups> &bits)
+    const std::array<unsigned char, num_groups> &bits)
 {
     return 0;
 }
@@ -245,12 +247,12 @@ constexpr std::enable_if_t<(num_groups == current_shift + 1), int> shift(
  *
  * @note this is the usual case of nested template
  */
-template <int num_groups, int current_shift>
+template <int current_shift, int num_groups>
 constexpr std::enable_if_t<(num_groups > current_shift + 1), int> shift(
-    const std::array<char, num_groups> &bits)
+    const std::array<unsigned char, num_groups> &bits)
 {
     return bits[current_shift + 1] +
-           shift<num_groups, (current_shift + 1)>(bits);
+           shift<(current_shift + 1), num_groups>(bits);
 }
 
 
@@ -262,23 +264,42 @@ using ConfigSetType = unsigned int;
 
 /**
  * ConfigSet is a way to embed several information into one integer by given
- * certain bits. The usage will be the following
- * Set the method with bits Cfg = ConfigSet<nb_1, nb_2, ..., nb_k>
- * Encode the given infomation encoded = Cfg::encode(i_1, i_2, ..., i_k)
- * Decode the specific position information i_t = Cfg::decode<t>(encoded)
+ * certain bits.
+ *
+ * The usage will be the following
+ * Set the method with bits Cfg = ConfigSet<b_0, b_1, ..., b_k>
+ * Encode the given infomation encoded = Cfg::encode(x_0, x_1, ..., x_k)
+ * Decode the specific position information x_t = Cfg::decode<t>(encoded)
  * The encoded result will use 32 bits to record
- * rrrrr1..12....2...k..k, which 1/2/k means the bits store the information for
- * 1/2/k position and r is for rest of unused bits.
+ * rrrrr0..01....1...k..k, which 1/2/.../k means the bits store the information
+ * for 1/2/.../k position and r is for rest of unused bits.
+ *
+ * Denote B_t = sum_(t+1)^(k) b_i and F(X) = Cfg::encode(x_0, ..., x_k)
+ * We can write F(X) = sum_0^k (x_i << B_i)
+ * for all i, we have 0 <= x_i < 2^(b_i)
+ * x_i, B_i are non-negative, so the F(X) = 0 <=> X = {0}, x_i = 0 for all i
+ * Assume F(X) = F(Y), then
+ * 0 = |F(X) - F(Y)| = |F(X-Y)| = F(|X - Y|)
+ * |x_i - y_i| is still in the same range 0 <= |x_i - y_i| < 2^(b_i)
+ * Thus, F(|X - Y|) = 0 -> |X - Y| = {0}, x_i - y_i = 0 -> X = Y
+ * F is one-to-one function if 0 <= x_i < 2^(b_i) for all i
+ * For any encoded result R, we can use the following to get the decoded series.
+ * for i = k to 0
+ *   x_i = R % b_i
+ *   R = R / bi
+ * Thus, any R in the range [0, 2^(B_0)) we have a series X such that F(X) = R
+ * F is onto function.
+ * Thus, F is bijection
  *
  * @tparam num_bits...  the number of bits for each position.
  *
- * @note the num_bit is required at least $log_2(maxval) + 1$
+ * @note the num_bit is required at least $ceil(log_2(maxval) + 1)$
  */
-template <int... num_bits>
+template <unsigned char... num_bits>
 class ConfigSet {
 public:
     static constexpr size_type num_groups = sizeof...(num_bits);
-    static constexpr std::array<char, num_groups> bits{num_bits...};
+    static constexpr std::array<unsigned char, num_groups> bits{num_bits...};
 
     /**
      * Decodes the `position` information from encoded
@@ -294,7 +315,7 @@ class ConfigSet {
     {
         static_assert(position < num_groups,
                       "This position is over the bounds.");
-        constexpr int shift = detail::shift<num_groups, position>(bits);
+        constexpr int shift = detail::shift<position, num_groups>(bits);
         constexpr auto mask = detail::mask<bits[position]>();
         return (encoded >> shift) & mask;
     }
@@ -319,7 +340,7 @@ class ConfigSet {
      * @tparam Rest...  the rest type
      *
      * @param first  the current encoded information
-     * @param rest...  the rest of others information waits for encoding
+     * @param rest...  the rest of other information waiting for encoding
      *
      * @return the encoded integer
      */
@@ -328,7 +349,20 @@ class ConfigSet {
                                       ConfigSetType>
     encode(ConfigSetType first, Rest &&... rest)
     {
-        constexpr int shift = detail::shift<num_groups, current_iter>(bits);
+        constexpr auto bound = detail::mask<bits[current_iter]>();
+        if (first > bound) {
+            throw std::out_of_range(
+                std::to_string(first) + " at " + std::to_string(current_iter) +
+                " postion is out of range of " +
+                std::to_string(detail::mask<bits[current_iter]>()) +
+                " representation");
+        }
+        constexpr int shift = detail::shift<current_iter, num_groups>(bits);
+        if (current_iter == 0) {
+            static_assert(
+                bits[current_iter] + shift <= sizeof(ConfigSetType) * 8,
+                "the total bits usage is larger than ConfigSetType bits");
+        }
         return (first << shift) |
                encode<current_iter + 1>(std::forward<Rest>(rest)...);
     }

From 3d65dfbd4d9818764275588d320c72630417693e Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 28 May 2021 21:21:57 +0200
Subject: [PATCH 0727/1008] use std::uint32_t, move ConfigSet to core, and del
 throw in constexpr delete throw in constexpr because it fails in gcc <= 5.x

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/base/types.hpp                           | 209 ++++++++++++++++++
 core/synthesizer/implementation_selection.hpp |  61 +++--
 core/test/base/types.cpp                      |  17 +-
 dpcpp/base/helper.hpp                         |  34 ++-
 .../cooperative_groups_kernels.dp.cpp         |  19 +-
 include/ginkgo/core/base/types.hpp            | 173 ---------------
 6 files changed, 270 insertions(+), 243 deletions(-)
 create mode 100644 core/base/types.hpp

diff --git a/core/base/types.hpp b/core/base/types.hpp
new file mode 100644
index 00000000000..a820ddfd91e
--- /dev/null
+++ b/core/base/types.hpp
@@ -0,0 +1,209 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_BASE_TYPES_HPP_
+#define GKO_CORE_BASE_TYPES_HPP_
+
+
+#include <array>
+#include <cstdint>
+#include <type_traits>
+
+
+namespace gko {
+namespace detail {
+
+
+/**
+ * mask gives the integer with Size activated bits in the end
+ *
+ * @tparam Size  the number of activated bits
+ * @tparam ValueType  the type of mask, which uses std::uint32_t as default
+ *
+ * @return the ValueType with Size activated bits in the end
+ */
+template <int Size, typename ValueType = std::uint32_t>
+constexpr std::enable_if_t<(Size < sizeof(ValueType) * 8), ValueType> mask()
+{
+    return (ValueType{1} << Size) - 1;
+}
+
+/**
+ * @copydoc mask()
+ *
+ * @note this is special case for the Size = the number of bits of ValueType
+ */
+template <int Size, typename ValueType = std::uint32_t>
+constexpr std::enable_if_t<Size == sizeof(ValueType) * 8, ValueType> mask()
+{
+    return ~ValueType{};
+}
+
+
+/**
+ * shift calculates the number of bits for shifting
+ *
+ * @tparam current_shift  the current position of shifting
+ * @tparam num_groups  the number of elements in array
+ *
+ * @return the number of shifting bits
+ *
+ * @note this is the last case of nested template
+ */
+template <int current_shift, int num_groups>
+constexpr std::enable_if_t<(num_groups == current_shift + 1), int> shift(
+    const std::array<unsigned char, num_groups> &bits)
+{
+    return 0;
+}
+
+/**
+ * @copydoc shift(const std::array<char, num_groups>)
+ *
+ * @note this is the usual case of nested template
+ */
+template <int current_shift, int num_groups>
+constexpr std::enable_if_t<(num_groups > current_shift + 1), int> shift(
+    const std::array<unsigned char, num_groups> &bits)
+{
+    return bits[current_shift + 1] +
+           shift<(current_shift + 1), num_groups>(bits);
+}
+
+
+}  // namespace detail
+
+
+/**
+ * ConfigSet is a way to embed several information into one integer by given
+ * certain bits.
+ *
+ * The usage will be the following
+ * Set the method with bits Cfg = ConfigSet<b_0, b_1, ..., b_k>
+ * Encode the given infomation encoded = Cfg::encode(x_0, x_1, ..., x_k)
+ * Decode the specific position information x_t = Cfg::decode<t>(encoded)
+ * The encoded result will use 32 bits to record
+ * rrrrr0..01....1...k..k, which 1/2/.../k means the bits store the information
+ * for 1/2/.../k position and r is for rest of unused bits.
+ *
+ * Denote $B_t = \sum_{i = t+1}^k b_i$ and $F(X) = Cfg::encode(x_0, ..., x_k)$.
+ * Have $F(X) = \sum_{i = 0}^k (x_i << B_i) = \sum_{i = 0}^k (x_i * 2^{B_i})$.
+ * For all i, we have $0 <= x_i < 2^{b_i}$.
+ * $x_i$, $2^{B_i}$ are non-negative, so
+ * $F(X) = 0$ <=> $X = \{0\}$, $x_i = 0$ for all i.
+ * Assume $F(X) = F(Y)$, then
+ * $0 = |F(X) - F(Y)| = |F(X-Y)| = F(|X - Y|)$.
+ * $|x_i - y_i|$ is still in the same range $0 <= |x_i - y_i| < 2^{b_i}$.
+ * Thus, $F(|X - Y|) = 0$ -> $|X - Y| = \{0\}$, $x_i - y_i = 0$ -> $X = Y$.
+ * F is one-to-one function if $0 <= x_i < 2^{b_i}$ for all i.
+ * For any encoded result R, we can use the following to get the decoded series.
+ * for i = k to 0;
+ *   $x_i = R % b_i$;
+ *   $R = R / bi$;
+ * endfor;
+ * For any R in the range $[0, 2^{B_0})$, we have X such that $F(X) = R$.
+ * F is onto function.
+ * Thus, F is bijection.
+ *
+ * @tparam num_bits...  the number of bits for each position.
+ *
+ * @note the num_bit is required at least $ceil(log_2(maxval) + 1)$
+ */
+template <unsigned char... num_bits>
+class ConfigSet {
+public:
+    static constexpr unsigned num_groups = sizeof...(num_bits);
+    static constexpr std::array<unsigned char, num_groups> bits{num_bits...};
+
+    /**
+     * Decodes the `position` information from encoded
+     *
+     * @tparam position  the position of desired information
+     *
+     * @param encoded  the encoded integer
+     *
+     * @return the decoded information at position
+     */
+    template <int position>
+    static constexpr std::uint32_t decode(std::uint32_t encoded)
+    {
+        static_assert(position < num_groups,
+                      "This position is over the bounds.");
+        constexpr int shift = detail::shift<position, num_groups>(bits);
+        constexpr auto mask = detail::mask<bits[position]>();
+        return (encoded >> shift) & mask;
+    }
+
+    /**
+     * Encodes the information with given bit set to encoded integer.
+     *
+     * @note the last case of nested template.
+     */
+    template <unsigned current_iter>
+    static constexpr std::enable_if_t<(current_iter == num_groups),
+                                      std::uint32_t>
+    encode()
+    {
+        return 0;
+    }
+
+    /**
+     * Encodes the information with given bit set to encoded integer.
+     *
+     * @tparam current_iter  the encoded place
+     * @tparam Rest...  the rest type
+     *
+     * @param first  the current encoded information
+     * @param rest...  the rest of other information waiting for encoding
+     *
+     * @return the encoded integer
+     */
+    template <unsigned current_iter = 0, typename... Rest>
+    static constexpr std::enable_if_t<(current_iter < num_groups),
+                                      std::uint32_t>
+    encode(std::uint32_t first, Rest &&... rest)
+    {
+        constexpr int shift = detail::shift<current_iter, num_groups>(bits);
+        if (current_iter == 0) {
+            static_assert(
+                bits[current_iter] + shift <= sizeof(std::uint32_t) * 8,
+                "the total bits usage is larger than std::uint32_t bits");
+        }
+        return (first << shift) |
+               encode<current_iter + 1>(std::forward<Rest>(rest)...);
+    }
+};
+
+
+}  // namespace gko
+
+#endif  // GKO_CORE_BASE_TYPES_HPP_
diff --git a/core/synthesizer/implementation_selection.hpp b/core/synthesizer/implementation_selection.hpp
index 734024d749c..2965ceb34a2 100644
--- a/core/synthesizer/implementation_selection.hpp
+++ b/core/synthesizer/implementation_selection.hpp
@@ -70,37 +70,36 @@ namespace syn {
         }                                                                    \
     }
 
-#define GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(_name, _callable)           \
-    template <typename Predicate, bool... BoolArgs, int... IntArgs,            \
-              gko::size_type... SizeTArgs, typename... TArgs,                  \
-              typename... InferredArgs>                                        \
-    inline void _name(::gko::syn::value_list<::gko::ConfigSetType>, Predicate, \
-                      ::gko::syn::value_list<bool, BoolArgs...>,               \
-                      ::gko::syn::value_list<int, IntArgs...>,                 \
-                      ::gko::syn::value_list<gko::size_type, SizeTArgs...>,    \
-                      ::gko::syn::type_list<TArgs...>, InferredArgs...)        \
-        GKO_KERNEL_NOT_FOUND;                                                  \
-                                                                               \
-    template <::gko::ConfigSetType K, ::gko::ConfigSetType... Rest,            \
-              typename Predicate, bool... BoolArgs, int... IntArgs,            \
-              gko::size_type... SizeTArgs, typename... TArgs,                  \
-              typename... InferredArgs>                                        \
-    inline void _name(                                                         \
-        ::gko::syn::value_list<::gko::ConfigSetType, K, Rest...>,              \
-        Predicate is_eligible,                                                 \
-        ::gko::syn::value_list<bool, BoolArgs...> bool_args,                   \
-        ::gko::syn::value_list<int, IntArgs...> int_args,                      \
-        ::gko::syn::value_list<gko::size_type, SizeTArgs...> size_args,        \
-        ::gko::syn::type_list<TArgs...> type_args, InferredArgs... args)       \
-    {                                                                          \
-        if (is_eligible(K)) {                                                  \
-            _callable<BoolArgs..., IntArgs..., SizeTArgs..., TArgs..., K>(     \
-                std::forward<InferredArgs>(args)...);                          \
-        } else {                                                               \
-            _name(::gko::syn::value_list<::gko::ConfigSetType, Rest...>(),     \
-                  is_eligible, bool_args, int_args, size_args, type_args,      \
-                  std::forward<InferredArgs>(args)...);                        \
-        }                                                                      \
+#define GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(_name, _callable)         \
+    template <typename Predicate, bool... BoolArgs, int... IntArgs,          \
+              gko::size_type... SizeTArgs, typename... TArgs,                \
+              typename... InferredArgs>                                      \
+    inline void _name(::gko::syn::value_list<std::uint32_t>, Predicate,      \
+                      ::gko::syn::value_list<bool, BoolArgs...>,             \
+                      ::gko::syn::value_list<int, IntArgs...>,               \
+                      ::gko::syn::value_list<gko::size_type, SizeTArgs...>,  \
+                      ::gko::syn::type_list<TArgs...>, InferredArgs...)      \
+        GKO_KERNEL_NOT_FOUND;                                                \
+                                                                             \
+    template <std::uint32_t K, std::uint32_t... Rest, typename Predicate,    \
+              bool... BoolArgs, int... IntArgs, gko::size_type... SizeTArgs, \
+              typename... TArgs, typename... InferredArgs>                   \
+    inline void _name(                                                       \
+        ::gko::syn::value_list<std::uint32_t, K, Rest...>,                   \
+        Predicate is_eligible,                                               \
+        ::gko::syn::value_list<bool, BoolArgs...> bool_args,                 \
+        ::gko::syn::value_list<int, IntArgs...> int_args,                    \
+        ::gko::syn::value_list<gko::size_type, SizeTArgs...> size_args,      \
+        ::gko::syn::type_list<TArgs...> type_args, InferredArgs... args)     \
+    {                                                                        \
+        if (is_eligible(K)) {                                                \
+            _callable<BoolArgs..., IntArgs..., SizeTArgs..., TArgs..., K>(   \
+                std::forward<InferredArgs>(args)...);                        \
+        } else {                                                             \
+            _name(::gko::syn::value_list<std::uint32_t, Rest...>(),          \
+                  is_eligible, bool_args, int_args, size_args, type_args,    \
+                  std::forward<InferredArgs>(args)...);                      \
+        }                                                                    \
     }
 
 
diff --git a/core/test/base/types.cpp b/core/test/base/types.cpp
index 68cfd76eeb9..7f8a0924025 100644
--- a/core/test/base/types.cpp
+++ b/core/test/base/types.cpp
@@ -42,6 +42,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
+#include "core/base/types.hpp"
+
+
 namespace {
 
 
@@ -115,9 +118,9 @@ TEST(ConfigSet, MaskCorrectly)
 
     ASSERT_EQ(mask3_u, 7u);
     ASSERT_EQ(fullmask_u, 0xffffffffu);
-    ASSERT_TRUE((std::is_same<decltype(mask3_u), const unsigned int>::value));
+    ASSERT_TRUE((std::is_same<decltype(mask3_u), const std::uint32_t>::value));
     ASSERT_TRUE(
-        (std::is_same<decltype(fullmask_u), const unsigned int>::value));
+        (std::is_same<decltype(fullmask_u), const std::uint32_t>::value));
     ASSERT_EQ(mask3_u64, 7ull);
     ASSERT_EQ(fullmask_u64, 0xffffffffffffffffull);
     ASSERT_TRUE(
@@ -210,14 +213,4 @@ TEST(ConfigSet, ConfigSetSomeFullCorrectly)
 }
 
 
-TEST(ConfigSet, ThrowOutOfBoundWhenExceedRepresentation)
-{
-    using Cfg = gko::ConfigSet<3, 2, 1>;
-
-    ASSERT_THROW(auto a = Cfg::encode(0, 0, 2), std::out_of_range);
-    ASSERT_THROW(auto a = Cfg::encode(0, 4, 0), std::out_of_range);
-    ASSERT_THROW(auto a = Cfg::encode(8, 0, 0), std::out_of_range);
-}
-
-
 }  // namespace
diff --git a/dpcpp/base/helper.hpp b/dpcpp/base/helper.hpp
index e6039757a37..c888eb9d99d 100644
--- a/dpcpp/base/helper.hpp
+++ b/dpcpp/base/helper.hpp
@@ -55,7 +55,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  * @param kernel_  the kernel name
  */
 #define GKO_ENABLE_DEFAULT_HOST_CONFIG(name_, kernel_)                     \
-    template <::gko::ConfigSetType encoded, typename... InferredArgs>      \
+    template <std::uint32_t encoded, typename... InferredArgs>             \
     inline void name_(dim3 grid, dim3 block, size_t dynamic_shared_memory, \
                       sycl::queue *queue, InferredArgs... args)            \
     {                                                                      \
@@ -79,21 +79,19 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *               available to decode<0> for blocksize and decode<1> for
  *               subgroup_size by cfg_
  */
-#define GKO_ENABLE_DEFAULT_CONFIG_CALL(name_, callable_, list_)            \
-    template <typename... InferredArgs>                                    \
-    void name_(::gko::ConfigSetType desired_cfg, dim3 grid, dim3 block,    \
-               size_t dynamic_shared_memory, sycl::queue *queue,           \
-               InferredArgs... args)                                       \
-    {                                                                      \
-        callable_(                                                         \
-            list_,                                                         \
-            [&desired_cfg](::gko::ConfigSetType cfg) {                     \
-                return cfg == desired_cfg;                                 \
-            },                                                             \
-            ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(), \
-            ::gko::syn::value_list<gko::size_type>(),                      \
-            ::gko::syn::type_list<>(), grid, block, dynamic_shared_memory, \
-            queue, std::forward<InferredArgs>(args)...);                   \
+#define GKO_ENABLE_DEFAULT_CONFIG_CALL(name_, callable_, list_)               \
+    template <typename... InferredArgs>                                       \
+    void name_(std::uint32_t desired_cfg, dim3 grid, dim3 block,              \
+               size_t dynamic_shared_memory, sycl::queue *queue,              \
+               InferredArgs... args)                                          \
+    {                                                                         \
+        callable_(                                                            \
+            list_,                                                            \
+            [&desired_cfg](std::uint32_t cfg) { return cfg == desired_cfg; }, \
+            ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),    \
+            ::gko::syn::value_list<gko::size_type>(),                         \
+            ::gko::syn::type_list<>(), grid, block, dynamic_shared_memory,    \
+            queue, std::forward<InferredArgs>(args)...);                      \
     }
 
 // __WG_BOUND__ gives the cuda-like launch bound in cuda ordering
@@ -116,8 +114,8 @@ namespace kernels {
 namespace dpcpp {
 
 
-bool validate(sycl::queue *queue, unsigned int workgroup_size,
-              unsigned int subgroup_size)
+bool validate(sycl::queue *queue, unsigned workgroup_size,
+              unsigned subgroup_size)
 {
     auto device = queue->get_device();
     auto subgroup_size_list =
diff --git a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
index f581ef9f19a..9b46e3ba0f3 100644
--- a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
+++ b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
@@ -48,6 +48,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
+#include "core/base/types.hpp"
 #include "core/synthesizer/implementation_selection.hpp"
 #include "core/test/utils/assertions.hpp"
 #include "dpcpp/base/config.hpp"
@@ -61,7 +62,7 @@ namespace {
 using namespace gko::kernels::dpcpp;
 using KCfg = gko::ConfigSet<11, 7>;
 constexpr auto default_config_list =
-    ::gko::syn::value_list<::gko::ConfigSetType, KCfg::encode(64, 64),
+    ::gko::syn::value_list<std::uint32_t, KCfg::encode(64, 64),
                            KCfg::encode(32, 32), KCfg::encode(16, 16),
                            KCfg::encode(8, 8), KCfg::encode(4, 4)>();
 
@@ -115,7 +116,7 @@ class CooperativeGroups : public testing::TestWithParam<unsigned int> {
 
 
 // kernel implementation
-template <::gko::ConfigSetType config>
+template <std::uint32_t config>
 __WG_BOUND__(KCfg::decode<0>(config))
 void cg_shuffle(bool *s, sycl::nd_item<3> item_ct1)
 {
@@ -148,14 +149,14 @@ void cg_shuffle_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_shuffle_config, cg_shuffle_host)
 
 // the call
-void cg_shuffle_config_call(::gko::ConfigSetType desired_cfg, dim3 grid,
-                            dim3 block, size_t dynamic_shared_memory,
-                            sycl::queue *queue, bool *s)
+void cg_shuffle_config_call(std::uint32_t desired_cfg, dim3 grid, dim3 block,
+                            size_t dynamic_shared_memory, sycl::queue *queue,
+                            bool *s)
 {
     cg_shuffle_config(
         default_config_list,
         // validate
-        [&desired_cfg](::gko::ConfigSetType cfg) { return cfg == desired_cfg; },
+        [&desired_cfg](std::uint32_t cfg) { return cfg == desired_cfg; },
         ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
         ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
         grid, block, dynamic_shared_memory, queue, s);
@@ -167,7 +168,7 @@ TEST_P(CooperativeGroups, Shuffle)
 }
 
 
-template <::gko::ConfigSetType config>
+template <std::uint32_t config>
 __WG_BOUND__(KCfg::decode<0>(config))
 void cg_all(bool *s, sycl::nd_item<3> item_ct1)
 {
@@ -189,7 +190,7 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_all_call, cg_all, default_config_list)
 TEST_P(CooperativeGroups, All) { test_all_subgroup(cg_all_call<bool *>); }
 
 
-template <::gko::ConfigSetType config>
+template <std::uint32_t config>
 __WG_BOUND__(KCfg::decode<0>(config))
 void cg_any(bool *s, sycl::nd_item<3> item_ct1)
 {
@@ -210,7 +211,7 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(cg_any_call, cg_any, default_config_list)
 TEST_P(CooperativeGroups, Any) { test_all_subgroup(cg_any_call<bool *>); }
 
 
-template <::gko::ConfigSetType config>
+template <std::uint32_t config>
 __WG_BOUND__(KCfg::decode<0>(config))
 void cg_ballot(bool *s, sycl::nd_item<3> item_ct1)
 {
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 08c98fb2e95..a15d8e24f2f 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -196,179 +196,6 @@ using default_precision = double;
 constexpr size_type byte_size = CHAR_BIT;
 
 
-namespace detail {
-
-
-/**
- * mask gives the integer with Size activated bits in the end
- *
- * @tparam Size  the number of activated bits
- * @tparam ValueType  the type of mask, which uses unsigned int as default
- *
- * @return the ValueType with Size activated bits in the end
- */
-template <int Size, typename ValueType = unsigned int>
-constexpr std::enable_if_t<(Size < sizeof(ValueType) * 8), ValueType> mask()
-{
-    return (ValueType{1} << Size) - 1;
-}
-
-/**
- * @copydoc mask()
- *
- * @note this is special case for the Size = the number of bits of ValueType
- */
-template <int Size, typename ValueType = unsigned int>
-constexpr std::enable_if_t<Size == sizeof(ValueType) * 8, ValueType> mask()
-{
-    return ~ValueType{};
-}
-
-
-/**
- * shift calculates the number of bits for shifting
- *
- * @tparam current_shift  the current position of shifting
- * @tparam num_groups  the number of elements in array
- *
- * @return the number of shifting bits
- *
- * @note this is the last case of nested template
- */
-template <int current_shift, int num_groups>
-constexpr std::enable_if_t<(num_groups == current_shift + 1), int> shift(
-    const std::array<unsigned char, num_groups> &bits)
-{
-    return 0;
-}
-
-/**
- * @copydoc shift(const std::array<char, num_groups>)
- *
- * @note this is the usual case of nested template
- */
-template <int current_shift, int num_groups>
-constexpr std::enable_if_t<(num_groups > current_shift + 1), int> shift(
-    const std::array<unsigned char, num_groups> &bits)
-{
-    return bits[current_shift + 1] +
-           shift<(current_shift + 1), num_groups>(bits);
-}
-
-
-}  // namespace detail
-
-
-// a shortcut type for ConfigSet
-using ConfigSetType = unsigned int;
-
-/**
- * ConfigSet is a way to embed several information into one integer by given
- * certain bits.
- *
- * The usage will be the following
- * Set the method with bits Cfg = ConfigSet<b_0, b_1, ..., b_k>
- * Encode the given infomation encoded = Cfg::encode(x_0, x_1, ..., x_k)
- * Decode the specific position information x_t = Cfg::decode<t>(encoded)
- * The encoded result will use 32 bits to record
- * rrrrr0..01....1...k..k, which 1/2/.../k means the bits store the information
- * for 1/2/.../k position and r is for rest of unused bits.
- *
- * Denote B_t = sum_(t+1)^(k) b_i and F(X) = Cfg::encode(x_0, ..., x_k)
- * We can write F(X) = sum_0^k (x_i << B_i)
- * for all i, we have 0 <= x_i < 2^(b_i)
- * x_i, B_i are non-negative, so the F(X) = 0 <=> X = {0}, x_i = 0 for all i
- * Assume F(X) = F(Y), then
- * 0 = |F(X) - F(Y)| = |F(X-Y)| = F(|X - Y|)
- * |x_i - y_i| is still in the same range 0 <= |x_i - y_i| < 2^(b_i)
- * Thus, F(|X - Y|) = 0 -> |X - Y| = {0}, x_i - y_i = 0 -> X = Y
- * F is one-to-one function if 0 <= x_i < 2^(b_i) for all i
- * For any encoded result R, we can use the following to get the decoded series.
- * for i = k to 0
- *   x_i = R % b_i
- *   R = R / bi
- * Thus, any R in the range [0, 2^(B_0)) we have a series X such that F(X) = R
- * F is onto function.
- * Thus, F is bijection
- *
- * @tparam num_bits...  the number of bits for each position.
- *
- * @note the num_bit is required at least $ceil(log_2(maxval) + 1)$
- */
-template <unsigned char... num_bits>
-class ConfigSet {
-public:
-    static constexpr size_type num_groups = sizeof...(num_bits);
-    static constexpr std::array<unsigned char, num_groups> bits{num_bits...};
-
-    /**
-     * Decodes the `position` information from encoded
-     *
-     * @tparam position  the position of desired information
-     *
-     * @param encoded  the encoded integer
-     *
-     * @return the decoded information at position
-     */
-    template <int position>
-    static constexpr ConfigSetType decode(ConfigSetType encoded)
-    {
-        static_assert(position < num_groups,
-                      "This position is over the bounds.");
-        constexpr int shift = detail::shift<position, num_groups>(bits);
-        constexpr auto mask = detail::mask<bits[position]>();
-        return (encoded >> shift) & mask;
-    }
-
-    /**
-     * Encodes the information with given bit set to encoded integer.
-     *
-     * @note the last case of nested template.
-     */
-    template <size_type current_iter>
-    static constexpr std::enable_if_t<(current_iter == num_groups),
-                                      ConfigSetType>
-    encode()
-    {
-        return 0;
-    }
-
-    /**
-     * Encodes the information with given bit set to encoded integer.
-     *
-     * @tparam current_iter  the encoded place
-     * @tparam Rest...  the rest type
-     *
-     * @param first  the current encoded information
-     * @param rest...  the rest of other information waiting for encoding
-     *
-     * @return the encoded integer
-     */
-    template <size_type current_iter = 0, typename... Rest>
-    static constexpr std::enable_if_t<(current_iter < num_groups),
-                                      ConfigSetType>
-    encode(ConfigSetType first, Rest &&... rest)
-    {
-        constexpr auto bound = detail::mask<bits[current_iter]>();
-        if (first > bound) {
-            throw std::out_of_range(
-                std::to_string(first) + " at " + std::to_string(current_iter) +
-                " postion is out of range of " +
-                std::to_string(detail::mask<bits[current_iter]>()) +
-                " representation");
-        }
-        constexpr int shift = detail::shift<current_iter, num_groups>(bits);
-        if (current_iter == 0) {
-            static_assert(
-                bits[current_iter] + shift <= sizeof(ConfigSetType) * 8,
-                "the total bits usage is larger than ConfigSetType bits");
-        }
-        return (first << shift) |
-               encode<current_iter + 1>(std::forward<Rest>(rest)...);
-    }
-};
-
-
 /**
  * Evaluates if all template arguments Args fulfill std::is_integral. If that is
  * the case, this class inherits from `std::true_type`, otherwise, it inherits

From 226d0668f1646af4b634dacd4e7f3f7998fa0fff Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 22 May 2021 22:24:40 +0200
Subject: [PATCH 0728/1008] avoid modifying stride on Dense convert_to

---
 common/matrix/dense_kernels.hpp.inc      | 16 +++++++++++++
 core/device_hooks/common_kernels.inc.cpp |  5 ++++
 core/matrix/dense.cpp                    | 29 ++++++++++++++++++++++++
 core/matrix/dense_kernels.hpp            |  6 +++++
 cuda/matrix/dense_kernels.cu             | 16 +++++++++++++
 cuda/test/matrix/dense_kernels.cpp       | 19 +++++++++++++++-
 dpcpp/matrix/dense_kernels.dp.cpp        |  8 +++++++
 hip/matrix/dense_kernels.hip.cpp         | 17 ++++++++++++++
 hip/test/matrix/dense_kernels.hip.cpp    | 17 ++++++++++++++
 include/ginkgo/core/matrix/dense.hpp     |  4 ++++
 omp/matrix/dense_kernels.cpp             | 16 +++++++++++++
 omp/test/matrix/dense_kernels.cpp        | 17 ++++++++++++++
 reference/matrix/dense_kernels.cpp       | 15 ++++++++++++
 reference/test/matrix/dense_kernels.cpp  | 19 ++++++++++++++++
 14 files changed, 203 insertions(+), 1 deletion(-)

diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index f1ab299698f..f7c621e5b9c 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -33,6 +33,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace kernel {
 
 
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void strided_copy(
+    size_type num_rows, size_type num_cols, size_type in_stride,
+    size_type out_stride, const ValueType *__restrict__ input,
+    ValueType *__restrict__ output)
+{
+    const auto global_id = thread::get_thread_id_flat();
+    const auto row_id = global_id / num_cols;
+    const auto col_id = global_id % num_cols;
+    if (row_id < num_rows) {
+        output[row_id * out_stride + col_id] =
+            input[row_id * in_stride + col_id];
+    }
+}
+
+
 template <typename ValueType>
 __global__ __launch_bounds__(default_block_size) void strided_fill(
     size_type num_rows, size_type num_cols, size_type stride,
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 810a3fab4e4..c5509037ce2 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -133,6 +133,11 @@ GKO_DECLARE_DENSE_APPLY_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
+template <typename ValueType>
+GKO_DECLARE_DENSE_COPY_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COPY_KERNEL);
+
 template <typename ValueType>
 GKO_DECLARE_DENSE_FILL_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 02edd22ce78..7222205b57a 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -63,6 +63,7 @@ namespace dense {
 
 GKO_REGISTER_OPERATION(simple_apply, dense::simple_apply);
 GKO_REGISTER_OPERATION(apply, dense::apply);
+GKO_REGISTER_OPERATION(copy, dense::copy);
 GKO_REGISTER_OPERATION(fill, dense::fill);
 GKO_REGISTER_OPERATION(scale, dense::scale);
 GKO_REGISTER_OPERATION(add_scaled, dense::add_scaled);
@@ -327,6 +328,34 @@ void Dense<ValueType>::compute_norm2_impl(LinOp *result) const
 }
 
 
+template <typename ValueType>
+void Dense<ValueType>::convert_to(Dense<ValueType> *result) const
+{
+    if (result->get_size() && result->get_size() == this->get_size()) {
+        // it's important we clone the const *this instead of result, since
+        // otherwise the copy_back_deleter of the temporary_clone calls
+        // convert_to again with the same dimensions, causing an infinite
+        // recursion. For the same reason we need to check that the result size
+        // is not zero, since otherwise it could come from the initial
+        // convert_to call in make_temporary_clone.
+        auto exec = result->get_executor();
+        exec->run(
+            dense::make_copy(make_temporary_clone(exec, this).get(), result));
+    } else {
+        result->values_ = this->values_;
+        result->stride_ = this->stride_;
+        result->set_size(this->get_size());
+    }
+}
+
+
+template <typename ValueType>
+void Dense<ValueType>::move_to(Dense<ValueType> *result)
+{
+    this->convert_to(result);
+}
+
+
 template <typename ValueType>
 void Dense<ValueType>::convert_to(
     Dense<next_precision<ValueType>> *result) const
diff --git a/core/matrix/dense_kernels.hpp b/core/matrix/dense_kernels.hpp
index 1cbe7f1001a..05128443122 100644
--- a/core/matrix/dense_kernels.hpp
+++ b/core/matrix/dense_kernels.hpp
@@ -57,6 +57,10 @@ namespace kernels {
                const matrix::Dense<_type> *a, const matrix::Dense<_type> *b, \
                const matrix::Dense<_type> *beta, matrix::Dense<_type> *c)
 
+#define GKO_DECLARE_DENSE_COPY_KERNEL(_type)               \
+    void copy(std::shared_ptr<const DefaultExecutor> exec, \
+              const matrix::Dense<_type> *input, matrix::Dense<_type> *output)
+
 #define GKO_DECLARE_DENSE_FILL_KERNEL(_type)               \
     void fill(std::shared_ptr<const DefaultExecutor> exec, \
               matrix::Dense<_type> *mat, _type value)
@@ -226,6 +230,8 @@ namespace kernels {
     template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_APPLY_KERNEL(ValueType);                              \
     template <typename ValueType>                                           \
+    GKO_DECLARE_DENSE_COPY_KERNEL(ValueType);                               \
+    template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_FILL_KERNEL(ValueType);                               \
     template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_SCALE_KERNEL(ValueType);                              \
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index b455d16f63b..000640c1a32 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -117,6 +117,22 @@ void apply(std::shared_ptr<const CudaExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
+template <typename ValueType>
+void copy(std::shared_ptr<const DefaultExecutor> exec,
+          const matrix::Dense<ValueType> *input,
+          matrix::Dense<ValueType> *output)
+{
+    const auto num_blocks = ceildiv(input->get_size()[0] * input->get_size()[1],
+                                    default_block_size);
+    kernel::strided_copy<<<num_blocks, default_block_size>>>(
+        input->get_size()[0], input->get_size()[1], input->get_stride(),
+        output->get_stride(), as_cuda_type(input->get_const_values()),
+        as_cuda_type(output->get_values()));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COPY_KERNEL);
+
+
 template <typename ValueType>
 void fill(std::shared_ptr<const DefaultExecutor> exec,
           matrix::Dense<ValueType> *mat, ValueType value)
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index e0986cff2de..7ff0efb4a3b 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -190,6 +190,23 @@ class Dense : public ::testing::Test {
 };
 
 
+TEST_F(Dense, CudaCopyRespectsStride)
+{
+    set_up_vector_data(3);
+    auto result = Mtx::create(cuda, dx->get_size(), dx->get_size()[1] + 1);
+    double val = 123456789.0;
+    auto original_data = result->get_values();
+    auto padding_ptr = original_data + dx->get_size()[1];
+    cuda->copy_from(ref.get(), 1, &val, padding_ptr);
+
+    dx->convert_to(result.get());
+
+    GKO_ASSERT_MTX_NEAR(result, dx, 0);
+    ASSERT_EQ(cuda->copy_val_to_host(padding_ptr), val);
+    ASSERT_EQ(result->get_values(), original_data);
+}
+
+
 TEST_F(Dense, CudaFillIsEquivalentToRef)
 {
     set_up_vector_data(3);
@@ -199,7 +216,7 @@ TEST_F(Dense, CudaFillIsEquivalentToRef)
     dx->fill(42);
     result->copy_from(dx.get());
 
-    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(result, x, 0);
 }
 
 
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 42f4a960861..4c4dfcb188a 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -84,6 +84,14 @@ void apply(std::shared_ptr<const DpcppExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
+template <typename ValueType>
+void copy(std::shared_ptr<const DefaultExecutor> exec,
+          const matrix::Dense<ValueType> *input,
+          matrix::Dense<ValueType> *output) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COPY_KERNEL);
+
+
 template <typename ValueType>
 void fill(std::shared_ptr<const DefaultExecutor> exec,
           matrix::Dense<ValueType> *mat, ValueType value) GKO_NOT_IMPLEMENTED;
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index 9b7a2f60a6a..f27c3354b9c 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -120,6 +120,23 @@ void apply(std::shared_ptr<const HipExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
+template <typename ValueType>
+void copy(std::shared_ptr<const DefaultExecutor> exec,
+          const matrix::Dense<ValueType> *input,
+          matrix::Dense<ValueType> *output)
+{
+    const auto num_blocks = ceildiv(input->get_size()[0] * input->get_size()[1],
+                                    default_block_size);
+    hipLaunchKernelGGL(kernel::strided_copy, num_blocks, default_block_size, 0,
+                       0, input->get_size()[0], input->get_size()[1],
+                       input->get_stride(), output->get_stride(),
+                       as_hip_type(input->get_const_values()),
+                       as_hip_type(output->get_values()));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COPY_KERNEL);
+
+
 template <typename ValueType>
 void fill(std::shared_ptr<const DefaultExecutor> exec,
           matrix::Dense<ValueType> *mat, ValueType value)
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index 72e69c7cc40..82a18634af4 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -185,6 +185,23 @@ class Dense : public ::testing::Test {
 };
 
 
+TEST_F(Dense, HipCopyRespectsStride)
+{
+    set_up_vector_data(3);
+    auto result = Mtx::create(hip, dx->get_size(), dx->get_size()[1] + 1);
+    double val = 123456789.0;
+    auto original_data = result->get_values();
+    auto padding_ptr = original_data + dx->get_size()[1];
+    hip->copy_from(ref.get(), 1, &val, padding_ptr);
+
+    dx->convert_to(result.get());
+
+    GKO_ASSERT_MTX_NEAR(result, dx, 0);
+    ASSERT_EQ(hip->copy_val_to_host(padding_ptr), val);
+    ASSERT_EQ(result->get_values(), original_data);
+}
+
+
 TEST_F(Dense, HipFillIsEquivalentToRef)
 {
     set_up_vector_data(3);
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 83ada5e4dc1..17495617b74 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -198,6 +198,10 @@ class Dense
 
     friend class Dense<next_precision<ValueType>>;
 
+    void convert_to(Dense<ValueType> *result) const override;
+
+    void move_to(Dense<ValueType> *result) override;
+
     void convert_to(Dense<next_precision<ValueType>> *result) const override;
 
     void move_to(Dense<next_precision<ValueType>> *result) override;
diff --git a/omp/matrix/dense_kernels.cpp b/omp/matrix/dense_kernels.cpp
index 65938de584f..aed8644d815 100644
--- a/omp/matrix/dense_kernels.cpp
+++ b/omp/matrix/dense_kernels.cpp
@@ -127,6 +127,22 @@ void apply(std::shared_ptr<const OmpExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
+template <typename ValueType>
+void copy(std::shared_ptr<const DefaultExecutor> exec,
+          const matrix::Dense<ValueType> *input,
+          matrix::Dense<ValueType> *output)
+{
+#pragma omp parallel for
+    for (size_type row = 0; row < input->get_size()[0]; ++row) {
+        for (size_type col = 0; col < input->get_size()[1]; ++col) {
+            output->at(row, col) = input->at(row, col);
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COPY_KERNEL);
+
+
 template <typename ValueType>
 void fill(std::shared_ptr<const DefaultExecutor> exec,
           matrix::Dense<ValueType> *mat, ValueType value)
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index b59c43ed4b0..49d553fbfc3 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -204,6 +204,23 @@ class Dense : public ::testing::Test {
 };
 
 
+TEST_F(Dense, OmpCopyRespectsStride)
+{
+    set_up_vector_data(3);
+    auto result = Mtx::create(omp, dx->get_size(), dx->get_size()[1] + 1);
+    double val = 123456789.0;
+    auto original_data = result->get_values();
+    auto padding_ptr = original_data + dx->get_size()[1];
+    omp->copy_from(ref.get(), 1, &val, padding_ptr);
+
+    dx->convert_to(result.get());
+
+    GKO_ASSERT_MTX_NEAR(result, dx, 0);
+    ASSERT_EQ(omp->copy_val_to_host(padding_ptr), val);
+    ASSERT_EQ(result->get_values(), original_data);
+}
+
+
 TEST_F(Dense, OmpFillIsEquivalentToRef)
 {
     set_up_vector_data(3);
diff --git a/reference/matrix/dense_kernels.cpp b/reference/matrix/dense_kernels.cpp
index fa55253a4ac..2d68ffdfaaf 100644
--- a/reference/matrix/dense_kernels.cpp
+++ b/reference/matrix/dense_kernels.cpp
@@ -116,6 +116,21 @@ void apply(std::shared_ptr<const ReferenceExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
+template <typename ValueType>
+void copy(std::shared_ptr<const DefaultExecutor> exec,
+          const matrix::Dense<ValueType> *input,
+          matrix::Dense<ValueType> *output)
+{
+    for (size_type row = 0; row < input->get_size()[0]; ++row) {
+        for (size_type col = 0; col < input->get_size()[1]; ++col) {
+            output->at(row, col) = input->at(row, col);
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COPY_KERNEL);
+
+
 template <typename ValueType>
 void fill(std::shared_ptr<const DefaultExecutor> exec,
           matrix::Dense<ValueType> *mat, ValueType value)
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 4b6a6e73b26..ff31dd543e9 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -115,6 +115,25 @@ class Dense : public ::testing::Test {
 TYPED_TEST_SUITE(Dense, gko::test::ValueTypes);
 
 
+TYPED_TEST(Dense, CopyRespectsStride)
+{
+    using value_type = typename TestFixture::value_type;
+    auto m =
+        gko::initialize<gko::matrix::Dense<TypeParam>>({1.0, 2.0}, this->exec);
+    auto m2 =
+        gko::matrix::Dense<TypeParam>::create(this->exec, gko::dim<2>{2, 1}, 2);
+    auto original_data = m2->get_values();
+    original_data[1] = TypeParam{3.0};
+
+    m->convert_to(m2.get());
+
+    EXPECT_EQ(m2->at(0, 0), value_type{1.0});
+    EXPECT_EQ(m2->at(1, 0), value_type{2.0});
+    EXPECT_EQ(m2->get_values(), original_data);
+    EXPECT_EQ(original_data[1], TypeParam{3.0});
+}
+
+
 TYPED_TEST(Dense, CanBeFilledWithValue)
 {
     using value_type = typename TestFixture::value_type;

From 28ae97bcc31dadb685a40813b60db5b1ef8df407 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 27 May 2021 21:34:32 +0200
Subject: [PATCH 0729/1008] add stride handling to Dense precision conversion

---
 common/matrix/dense_kernels.hpp.inc      |  8 ++++----
 core/device_hooks/common_kernels.inc.cpp |  7 ++++---
 core/matrix/dense.cpp                    | 14 +++++++++++---
 core/matrix/dense_kernels.hpp            |  9 +++++----
 cuda/matrix/dense_kernels.cu             |  9 +++++----
 dpcpp/matrix/dense_kernels.dp.cpp        |  9 +++++----
 hip/matrix/dense_kernels.hip.cpp         |  9 +++++----
 include/ginkgo/core/base/types.hpp       | 16 ++++++++++++++++
 omp/matrix/dense_kernels.cpp             | 12 +++++++-----
 reference/matrix/dense_kernels.cpp       | 12 +++++++-----
 10 files changed, 69 insertions(+), 36 deletions(-)

diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index f7c621e5b9c..8bbba33fd6d 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -33,18 +33,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace kernel {
 
 
-template <typename ValueType>
+template <typename InValueType, typename OutValueType>
 __global__ __launch_bounds__(default_block_size) void strided_copy(
     size_type num_rows, size_type num_cols, size_type in_stride,
-    size_type out_stride, const ValueType *__restrict__ input,
-    ValueType *__restrict__ output)
+    size_type out_stride, const InValueType *__restrict__ input,
+    OutValueType *__restrict__ output)
 {
     const auto global_id = thread::get_thread_id_flat();
     const auto row_id = global_id / num_cols;
     const auto col_id = global_id % num_cols;
     if (row_id < num_rows) {
         output[row_id * out_stride + col_id] =
-            input[row_id * in_stride + col_id];
+            static_cast<OutValueType>(input[row_id * in_stride + col_id]);
     }
 }
 
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index c5509037ce2..c4244c8317c 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -133,10 +133,11 @@ GKO_DECLARE_DENSE_APPLY_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
-template <typename ValueType>
-GKO_DECLARE_DENSE_COPY_KERNEL(ValueType)
+template <typename InValueType, typename OutValueType>
+GKO_DECLARE_DENSE_COPY_KERNEL(InValueType, OutValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COPY_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(
+    GKO_DECLARE_DENSE_COPY_KERNEL);
 
 template <typename ValueType>
 GKO_DECLARE_DENSE_FILL_KERNEL(ValueType)
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 7222205b57a..b06ae5085d5 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -360,9 +360,17 @@ template <typename ValueType>
 void Dense<ValueType>::convert_to(
     Dense<next_precision<ValueType>> *result) const
 {
-    result->values_ = this->values_;
-    result->stride_ = this->stride_;
-    result->set_size(this->get_size());
+    if (result->get_size() && result->get_size() == this->get_size()) {
+        // here we don't need to use the target executor, since the copy back
+        // invokes the non-converting convert_to(Dense<ValueType>*).
+        auto exec = this->get_executor();
+        exec->run(
+            dense::make_copy(this, make_temporary_clone(exec, result).get()));
+    } else {
+        result->values_ = this->values_;
+        result->stride_ = this->stride_;
+        result->set_size(this->get_size());
+    }
 }
 
 
diff --git a/core/matrix/dense_kernels.hpp b/core/matrix/dense_kernels.hpp
index 05128443122..6b1e57f0a27 100644
--- a/core/matrix/dense_kernels.hpp
+++ b/core/matrix/dense_kernels.hpp
@@ -57,9 +57,10 @@ namespace kernels {
                const matrix::Dense<_type> *a, const matrix::Dense<_type> *b, \
                const matrix::Dense<_type> *beta, matrix::Dense<_type> *c)
 
-#define GKO_DECLARE_DENSE_COPY_KERNEL(_type)               \
+#define GKO_DECLARE_DENSE_COPY_KERNEL(_intype, _outtype)   \
     void copy(std::shared_ptr<const DefaultExecutor> exec, \
-              const matrix::Dense<_type> *input, matrix::Dense<_type> *output)
+              const matrix::Dense<_intype> *input,         \
+              matrix::Dense<_outtype> *output)
 
 #define GKO_DECLARE_DENSE_FILL_KERNEL(_type)               \
     void fill(std::shared_ptr<const DefaultExecutor> exec, \
@@ -229,8 +230,8 @@ namespace kernels {
     GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL(ValueType);                       \
     template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_APPLY_KERNEL(ValueType);                              \
-    template <typename ValueType>                                           \
-    GKO_DECLARE_DENSE_COPY_KERNEL(ValueType);                               \
+    template <typename InValueType, typename OutValueType>                  \
+    GKO_DECLARE_DENSE_COPY_KERNEL(InValueType, OutValueType);               \
     template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_FILL_KERNEL(ValueType);                               \
     template <typename ValueType>                                           \
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index 000640c1a32..52caf38ddc0 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -117,10 +117,10 @@ void apply(std::shared_ptr<const CudaExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
-template <typename ValueType>
+template <typename InValueType, typename OutValueType>
 void copy(std::shared_ptr<const DefaultExecutor> exec,
-          const matrix::Dense<ValueType> *input,
-          matrix::Dense<ValueType> *output)
+          const matrix::Dense<InValueType> *input,
+          matrix::Dense<OutValueType> *output)
 {
     const auto num_blocks = ceildiv(input->get_size()[0] * input->get_size()[1],
                                     default_block_size);
@@ -130,7 +130,8 @@ void copy(std::shared_ptr<const DefaultExecutor> exec,
         as_cuda_type(output->get_values()));
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COPY_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(
+    GKO_DECLARE_DENSE_COPY_KERNEL);
 
 
 template <typename ValueType>
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 4c4dfcb188a..48dcefbce00 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -84,12 +84,13 @@ void apply(std::shared_ptr<const DpcppExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
-template <typename ValueType>
+template <typename InValueType, typename OutValueType>
 void copy(std::shared_ptr<const DefaultExecutor> exec,
-          const matrix::Dense<ValueType> *input,
-          matrix::Dense<ValueType> *output) GKO_NOT_IMPLEMENTED;
+          const matrix::Dense<InValueType> *input,
+          matrix::Dense<OutValueType> *output) GKO_NOT_IMPLEMENTED;
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COPY_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(
+    GKO_DECLARE_DENSE_COPY_KERNEL);
 
 
 template <typename ValueType>
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index f27c3354b9c..257b2b5961d 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -120,10 +120,10 @@ void apply(std::shared_ptr<const HipExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
-template <typename ValueType>
+template <typename InValueType, typename OutValueType>
 void copy(std::shared_ptr<const DefaultExecutor> exec,
-          const matrix::Dense<ValueType> *input,
-          matrix::Dense<ValueType> *output)
+          const matrix::Dense<InValueType> *input,
+          matrix::Dense<OutValueType> *output)
 {
     const auto num_blocks = ceildiv(input->get_size()[0] * input->get_size()[1],
                                     default_block_size);
@@ -134,7 +134,8 @@ void copy(std::shared_ptr<const DefaultExecutor> exec,
                        as_hip_type(output->get_values()));
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COPY_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(
+    GKO_DECLARE_DENSE_COPY_KERNEL);
 
 
 template <typename ValueType>
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index a15d8e24f2f..0288316dee8 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -546,12 +546,28 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
     _macro(std::complex<float>, std::complex<double>) GKO_NOT_IMPLEMENTED; \
     template <>                                                            \
     _macro(std::complex<double>, std::complex<float>) GKO_NOT_IMPLEMENTED
+#define GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(_macro)         \
+    GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(_macro);                    \
+    template <>                                                           \
+    _macro(float, float) GKO_NOT_IMPLEMENTED;                             \
+    template <>                                                           \
+    _macro(double, double) GKO_NOT_IMPLEMENTED;                           \
+    template <>                                                           \
+    _macro(std::complex<float>, std::complex<float>) GKO_NOT_IMPLEMENTED; \
+    template <>                                                           \
+    _macro(std::complex<double>, std::complex<double>) GKO_NOT_IMPLEMENTED
 #else
 #define GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(_macro)       \
     template _macro(float, double);                             \
     template _macro(double, float);                             \
     template _macro(std::complex<float>, std::complex<double>); \
     template _macro(std::complex<double>, std::complex<float>)
+#define GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(_macro) \
+    GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(_macro);            \
+    template _macro(float, float);                                \
+    template _macro(double, double);                              \
+    template _macro(std::complex<float>, std::complex<float>);    \
+    template _macro(std::complex<double>, std::complex<double>)
 #endif
 
 
diff --git a/omp/matrix/dense_kernels.cpp b/omp/matrix/dense_kernels.cpp
index aed8644d815..0534f1deb41 100644
--- a/omp/matrix/dense_kernels.cpp
+++ b/omp/matrix/dense_kernels.cpp
@@ -127,20 +127,22 @@ void apply(std::shared_ptr<const OmpExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
-template <typename ValueType>
+template <typename InValueType, typename OutValueType>
 void copy(std::shared_ptr<const DefaultExecutor> exec,
-          const matrix::Dense<ValueType> *input,
-          matrix::Dense<ValueType> *output)
+          const matrix::Dense<InValueType> *input,
+          matrix::Dense<OutValueType> *output)
 {
 #pragma omp parallel for
     for (size_type row = 0; row < input->get_size()[0]; ++row) {
         for (size_type col = 0; col < input->get_size()[1]; ++col) {
-            output->at(row, col) = input->at(row, col);
+            output->at(row, col) =
+                static_cast<OutValueType>(input->at(row, col));
         }
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COPY_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(
+    GKO_DECLARE_DENSE_COPY_KERNEL);
 
 
 template <typename ValueType>
diff --git a/reference/matrix/dense_kernels.cpp b/reference/matrix/dense_kernels.cpp
index 2d68ffdfaaf..49721aab418 100644
--- a/reference/matrix/dense_kernels.cpp
+++ b/reference/matrix/dense_kernels.cpp
@@ -116,19 +116,21 @@ void apply(std::shared_ptr<const ReferenceExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
-template <typename ValueType>
+template <typename InValueType, typename OutValueType>
 void copy(std::shared_ptr<const DefaultExecutor> exec,
-          const matrix::Dense<ValueType> *input,
-          matrix::Dense<ValueType> *output)
+          const matrix::Dense<InValueType> *input,
+          matrix::Dense<OutValueType> *output)
 {
     for (size_type row = 0; row < input->get_size()[0]; ++row) {
         for (size_type col = 0; col < input->get_size()[1]; ++col) {
-            output->at(row, col) = input->at(row, col);
+            output->at(row, col) =
+                static_cast<OutValueType>(input->at(row, col));
         }
     }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COPY_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(
+    GKO_DECLARE_DENSE_COPY_KERNEL);
 
 
 template <typename ValueType>

From 8674356b70bb69cd89439ecefeb7321ea60a9d2f Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 27 May 2021 21:36:22 +0200
Subject: [PATCH 0730/1008] add out-parameter versions to Dense functions

---
 core/matrix/dense.cpp                | 363 +++++++++++++++++----------
 include/ginkgo/core/matrix/dense.hpp | 158 +++++++++++-
 2 files changed, 387 insertions(+), 134 deletions(-)

diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index b06ae5085d5..f8d8421a9dd 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -674,123 +674,188 @@ void Dense<ValueType>::write(mat_data32 &data) const
 template <typename ValueType>
 std::unique_ptr<LinOp> Dense<ValueType>::transpose() const
 {
-    auto exec = this->get_executor();
-    auto trans_cpy = Dense::create(exec, gko::transpose(this->get_size()));
+    auto result =
+        Dense::create(this->get_executor(), gko::transpose(this->get_size()));
+    this->transpose(result.get());
+    return result;
+}
 
-    exec->run(dense::make_transpose(this, trans_cpy.get()));
 
-    return std::move(trans_cpy);
+template <typename ValueType>
+std::unique_ptr<LinOp> Dense<ValueType>::conj_transpose() const
+{
+    auto result =
+        Dense::create(this->get_executor(), gko::transpose(this->get_size()));
+    this->conj_transpose(result.get());
+    return result;
 }
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Dense<ValueType>::conj_transpose() const
+void Dense<ValueType>::transpose(Dense<ValueType> *output) const
 {
+    GKO_ASSERT_EQUAL_DIMENSIONS(output, gko::transpose(this->get_size()));
     auto exec = this->get_executor();
-    auto trans_cpy = Dense::create(exec, gko::transpose(this->get_size()));
+    exec->run(
+        dense::make_transpose(this, make_temporary_clone(exec, output).get()));
+}
 
-    exec->run(dense::make_conj_transpose(this, trans_cpy.get()));
-    return std::move(trans_cpy);
+
+template <typename ValueType>
+void Dense<ValueType>::conj_transpose(Dense<ValueType> *output) const
+{
+    GKO_ASSERT_EQUAL_DIMENSIONS(output, gko::transpose(this->get_size()));
+    auto exec = this->get_executor();
+    exec->run(dense::make_conj_transpose(
+        this, make_temporary_clone(exec, output).get()));
 }
 
 
 template <typename ValueType>
 std::unique_ptr<LinOp> Dense<ValueType>::permute(
     const Array<int32> *permutation_indices) const
+{
+    auto result = Dense::create(this->get_executor(), this->get_size());
+    this->permute(permutation_indices, result.get());
+    return result;
+}
+
+
+template <typename ValueType>
+std::unique_ptr<LinOp> Dense<ValueType>::permute(
+    const Array<int64> *permutation_indices) const
+{
+    auto result = Dense::create(this->get_executor(), this->get_size());
+    this->permute(permutation_indices, result.get());
+    return result;
+}
+
+
+template <typename ValueType>
+void Dense<ValueType>::permute(const Array<int32> *permutation_indices,
+                               Dense<ValueType> *output) const
 {
     GKO_ASSERT_IS_SQUARE_MATRIX(this);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
     GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
     auto exec = this->get_executor();
-    auto permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_symm_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        permute_cpy.get()));
-
-    return std::move(permute_cpy);
+        make_temporary_clone(exec, output).get()));
 }
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Dense<ValueType>::permute(
-    const Array<int64> *permutation_indices) const
+void Dense<ValueType>::permute(const Array<int64> *permutation_indices,
+                               Dense<ValueType> *output) const
 {
     GKO_ASSERT_IS_SQUARE_MATRIX(this);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
     GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
     auto exec = this->get_executor();
-    auto permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_symm_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        permute_cpy.get()));
-
-    return std::move(permute_cpy);
+        make_temporary_clone(exec, output).get()));
 }
 
 
 template <typename ValueType>
 std::unique_ptr<LinOp> Dense<ValueType>::inverse_permute(
     const Array<int32> *permutation_indices) const
+{
+    auto result = Dense::create(this->get_executor(), this->get_size());
+    this->inverse_permute(permutation_indices, result.get());
+    return result;
+}
+
+
+template <typename ValueType>
+std::unique_ptr<LinOp> Dense<ValueType>::inverse_permute(
+    const Array<int64> *permutation_indices) const
+{
+    auto result = Dense::create(this->get_executor(), this->get_size());
+    this->inverse_permute(permutation_indices, result.get());
+    return result;
+}
+
+
+template <typename ValueType>
+void Dense<ValueType>::inverse_permute(const Array<int32> *permutation_indices,
+                                       Dense<ValueType> *output) const
 {
     GKO_ASSERT_IS_SQUARE_MATRIX(this);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
     GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
     auto exec = this->get_executor();
-    auto permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_inv_symm_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        permute_cpy.get()));
-
-    return std::move(permute_cpy);
+        make_temporary_clone(exec, output).get()));
 }
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Dense<ValueType>::inverse_permute(
-    const Array<int64> *permutation_indices) const
+void Dense<ValueType>::inverse_permute(const Array<int64> *permutation_indices,
+                                       Dense<ValueType> *output) const
 {
     GKO_ASSERT_IS_SQUARE_MATRIX(this);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
     GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
     auto exec = this->get_executor();
-    auto permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_inv_symm_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        permute_cpy.get()));
-
-    return std::move(permute_cpy);
+        make_temporary_clone(exec, output).get()));
 }
 
 
 template <typename ValueType>
 std::unique_ptr<LinOp> Dense<ValueType>::row_permute(
     const Array<int32> *permutation_indices) const
+{
+    auto result = Dense::create(this->get_executor(), this->get_size());
+    this->row_permute(permutation_indices, result.get());
+    return result;
+}
+
+
+template <typename ValueType>
+std::unique_ptr<LinOp> Dense<ValueType>::row_permute(
+    const Array<int64> *permutation_indices) const
+{
+    auto result = Dense::create(this->get_executor(), this->get_size());
+    this->row_permute(permutation_indices, result.get());
+    return result;
+}
+
+
+template <typename ValueType>
+void Dense<ValueType>::row_permute(const Array<int32> *permutation_indices,
+                                   Dense<ValueType> *output) const
 {
     GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
     auto exec = this->get_executor();
-    auto permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_row_gather(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        permute_cpy.get()));
-
-    return std::move(permute_cpy);
+        make_temporary_clone(exec, output).get()));
 }
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Dense<ValueType>::row_permute(
-    const Array<int64> *permutation_indices) const
+void Dense<ValueType>::row_permute(const Array<int64> *permutation_indices,
+                                   Dense<ValueType> *output) const
 {
     GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
     auto exec = this->get_executor();
-    auto permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_row_gather(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        permute_cpy.get()));
-
-    return std::move(permute_cpy);
+        make_temporary_clone(exec, output).get()));
 }
 
 
@@ -800,12 +865,9 @@ std::unique_ptr<Dense<ValueType>> Dense<ValueType>::row_gather(
 {
     auto exec = this->get_executor();
     dim<2> out_dim{row_indices->get_num_elems(), this->get_size()[1]};
-    auto row_gathered = Dense::create(exec, out_dim);
-
-    exec->run(
-        dense::make_row_gather(make_temporary_clone(exec, row_indices).get(),
-                               this, row_gathered.get()));
-    return row_gathered;
+    auto result = Dense::create(exec, out_dim);
+    this->row_gather(row_indices, result.get());
+    return result;
 }
 
 
@@ -815,12 +877,9 @@ std::unique_ptr<Dense<ValueType>> Dense<ValueType>::row_gather(
 {
     auto exec = this->get_executor();
     dim<2> out_dim{row_indices->get_num_elems(), this->get_size()[1]};
-    auto row_gathered = Dense::create(exec, out_dim);
-
-    exec->run(
-        dense::make_row_gather(make_temporary_clone(exec, row_indices).get(),
-                               this, row_gathered.get()));
-    return row_gathered;
+    auto result = Dense::create(exec, out_dim);
+    this->row_gather(row_indices, result.get());
+    return result;
 }
 
 
@@ -854,109 +913,167 @@ void Dense<ValueType>::row_gather(const Array<int64> *row_indices,
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Dense<ValueType>::column_permute(
-    const Array<int32> *permutation_indices) const
+void Dense<ValueType>::column_permute(const Array<int32> *permutation_indices,
+                                      Dense<ValueType> *output) const
 {
     GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
     auto exec = this->get_executor();
-    auto permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_column_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        permute_cpy.get()));
-
-    return std::move(permute_cpy);
+        make_temporary_clone(exec, output).get()));
 }
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Dense<ValueType>::column_permute(
-    const Array<int64> *permutation_indices) const
+void Dense<ValueType>::column_permute(const Array<int64> *permutation_indices,
+                                      Dense<ValueType> *output) const
 {
     GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
     auto exec = this->get_executor();
-    auto permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_column_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        permute_cpy.get()));
-
-    return std::move(permute_cpy);
+        make_temporary_clone(exec, output).get()));
 }
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Dense<ValueType>::inverse_row_permute(
+std::unique_ptr<LinOp> Dense<ValueType>::column_permute(
     const Array<int32> *permutation_indices) const
+{
+    auto result = Dense::create(this->get_executor(), this->get_size());
+    this->column_permute(permutation_indices, result.get());
+    return result;
+}
+
+
+template <typename ValueType>
+std::unique_ptr<LinOp> Dense<ValueType>::column_permute(
+    const Array<int64> *permutation_indices) const
+{
+    auto result = Dense::create(this->get_executor(), this->get_size());
+    this->column_permute(permutation_indices, result.get());
+    return result;
+}
+
+
+template <typename ValueType>
+void Dense<ValueType>::inverse_row_permute(
+    const Array<int32> *permutation_indices, Dense<ValueType> *output) const
 {
     GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
     auto exec = this->get_executor();
-    auto inverse_permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_inverse_row_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        inverse_permute_cpy.get()));
-
-    return std::move(inverse_permute_cpy);
+        make_temporary_clone(exec, output).get()));
 }
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Dense<ValueType>::inverse_row_permute(
-    const Array<int64> *permutation_indices) const
+void Dense<ValueType>::inverse_row_permute(
+    const Array<int64> *permutation_indices, Dense<ValueType> *output) const
 {
     GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
     auto exec = this->get_executor();
-    auto inverse_permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_inverse_row_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        inverse_permute_cpy.get()));
-
-    return std::move(inverse_permute_cpy);
+        make_temporary_clone(exec, output).get()));
 }
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Dense<ValueType>::inverse_column_permute(
+std::unique_ptr<LinOp> Dense<ValueType>::inverse_row_permute(
     const Array<int32> *permutation_indices) const
+{
+    auto result = Dense::create(this->get_executor(), this->get_size());
+    this->inverse_row_permute(permutation_indices, result.get());
+    return result;
+}
+
+
+template <typename ValueType>
+std::unique_ptr<LinOp> Dense<ValueType>::inverse_row_permute(
+    const Array<int64> *permutation_indices) const
+{
+    auto result = Dense::create(this->get_executor(), this->get_size());
+    this->inverse_row_permute(permutation_indices, result.get());
+    return result;
+}
+
+
+template <typename ValueType>
+void Dense<ValueType>::inverse_column_permute(
+    const Array<int32> *permutation_indices, Dense<ValueType> *output) const
 {
     GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
     auto exec = this->get_executor();
-    auto inverse_permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_inverse_column_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        inverse_permute_cpy.get()));
-
-    return std::move(inverse_permute_cpy);
+        make_temporary_clone(exec, output).get()));
 }
 
 
 template <typename ValueType>
-std::unique_ptr<LinOp> Dense<ValueType>::inverse_column_permute(
-    const Array<int64> *permutation_indices) const
+void Dense<ValueType>::inverse_column_permute(
+    const Array<int64> *permutation_indices, Dense<ValueType> *output) const
 {
     GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
     auto exec = this->get_executor();
-    auto inverse_permute_cpy = Dense::create(exec, this->get_size());
 
     exec->run(dense::make_inverse_column_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        inverse_permute_cpy.get()));
+        make_temporary_clone(exec, output).get()));
+}
+
+
+template <typename ValueType>
+std::unique_ptr<LinOp> Dense<ValueType>::inverse_column_permute(
+    const Array<int32> *permutation_indices) const
+{
+    auto result = Dense::create(this->get_executor(), this->get_size());
+    this->inverse_column_permute(permutation_indices, result.get());
+    return result;
+}
+
 
-    return std::move(inverse_permute_cpy);
+template <typename ValueType>
+std::unique_ptr<LinOp> Dense<ValueType>::inverse_column_permute(
+    const Array<int64> *permutation_indices) const
+{
+    auto result = Dense::create(this->get_executor(), this->get_size());
+    this->inverse_column_permute(permutation_indices, result.get());
+    return result;
 }
 
 
 template <typename ValueType>
-std::unique_ptr<Diagonal<ValueType>> Dense<ValueType>::extract_diagonal() const
+void Dense<ValueType>::extract_diagonal(Diagonal<ValueType> *output) const
 {
     auto exec = this->get_executor();
+    const auto diag_size = std::min(this->get_size()[0], this->get_size()[1]);
+    GKO_ASSERT_EQ(output->get_size()[0], diag_size);
+
+    exec->run(dense::make_extract_diagonal(
+        this, make_temporary_clone(exec, output).get()));
+}
 
+
+template <typename ValueType>
+std::unique_ptr<Diagonal<ValueType>> Dense<ValueType>::extract_diagonal() const
+{
     const auto diag_size = std::min(this->get_size()[0], this->get_size()[1]);
-    auto diag = Diagonal<ValueType>::create(exec, diag_size);
-    exec->run(dense::make_extract_diagonal(this, lend(diag)));
+    auto diag = Diagonal<ValueType>::create(this->get_executor(), diag_size);
+    this->extract_diagonal(diag.get());
     return diag;
 }
 
@@ -964,9 +1081,7 @@ std::unique_ptr<Diagonal<ValueType>> Dense<ValueType>::extract_diagonal() const
 template <typename ValueType>
 void Dense<ValueType>::compute_absolute_inplace()
 {
-    auto exec = this->get_executor();
-
-    exec->run(dense::make_inplace_absolute_dense(this));
+    this->get_executor()->run(dense::make_inplace_absolute_dense(this));
 }
 
 
@@ -974,37 +1089,41 @@ template <typename ValueType>
 std::unique_ptr<typename Dense<ValueType>::absolute_type>
 Dense<ValueType>::compute_absolute() const
 {
-    auto exec = this->get_executor();
-
     // do not inherit the stride
-    auto abs_dense = absolute_type::create(exec, this->get_size());
-
-    exec->run(dense::make_outplace_absolute_dense(this, abs_dense.get()));
-
-    return abs_dense;
+    auto result = absolute_type::create(this->get_executor(), this->get_size());
+    this->compute_absolute(result.get());
+    return result;
 }
 
 
 template <typename ValueType>
-std::unique_ptr<typename Dense<ValueType>::complex_type>
-Dense<ValueType>::make_complex() const
+void Dense<ValueType>::compute_absolute(
+    Dense<ValueType>::absolute_type *output) const
 {
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
     auto exec = this->get_executor();
 
-    auto complex_dense = complex_type::create(exec, this->get_size());
+    exec->run(dense::make_outplace_absolute_dense(
+        this, make_temporary_clone(exec, output).get()));
+}
 
-    exec->run(dense::make_make_complex(this, complex_dense.get()));
 
-    return complex_dense;
+template <typename ValueType>
+std::unique_ptr<typename Dense<ValueType>::complex_type>
+Dense<ValueType>::make_complex() const
+{
+    auto result = complex_type::create(this->get_executor(), this->get_size());
+    this->make_complex(result.get());
+    return result;
 }
 
 
 template <typename ValueType>
-void Dense<ValueType>::make_complex(Dense<to_complex<ValueType>> *result) const
+void Dense<ValueType>::make_complex(
+    typename Dense<ValueType>::complex_type *result) const
 {
-    auto exec = this->get_executor();
-
     GKO_ASSERT_EQUAL_DIMENSIONS(this, result);
+    auto exec = this->get_executor();
 
     exec->run(dense::make_make_complex(
         this, make_temporary_clone(exec, result).get()));
@@ -1012,25 +1131,21 @@ void Dense<ValueType>::make_complex(Dense<to_complex<ValueType>> *result) const
 
 
 template <typename ValueType>
-std::unique_ptr<typename Dense<ValueType>::absolute_type>
+std::unique_ptr<typename Dense<ValueType>::real_type>
 Dense<ValueType>::get_real() const
 {
-    auto exec = this->get_executor();
-
-    auto real_dense = absolute_type::create(exec, this->get_size());
-
-    exec->run(dense::make_get_real(this, real_dense.get()));
-
-    return real_dense;
+    auto result = real_type::create(this->get_executor(), this->get_size());
+    this->get_real(result.get());
+    return result;
 }
 
 
 template <typename ValueType>
-void Dense<ValueType>::get_real(Dense<remove_complex<ValueType>> *result) const
+void Dense<ValueType>::get_real(
+    typename Dense<ValueType>::real_type *result) const
 {
-    auto exec = this->get_executor();
-
     GKO_ASSERT_EQUAL_DIMENSIONS(this, result);
+    auto exec = this->get_executor();
 
     exec->run(
         dense::make_get_real(this, make_temporary_clone(exec, result).get()));
@@ -1038,25 +1153,21 @@ void Dense<ValueType>::get_real(Dense<remove_complex<ValueType>> *result) const
 
 
 template <typename ValueType>
-std::unique_ptr<typename Dense<ValueType>::absolute_type>
+std::unique_ptr<typename Dense<ValueType>::real_type>
 Dense<ValueType>::get_imag() const
 {
-    auto exec = this->get_executor();
-
-    auto imag_dense = absolute_type::create(exec, this->get_size());
-
-    exec->run(dense::make_get_imag(this, imag_dense.get()));
-
-    return imag_dense;
+    auto result = real_type::create(this->get_executor(), this->get_size());
+    this->get_imag(result.get());
+    return result;
 }
 
 
 template <typename ValueType>
-void Dense<ValueType>::get_imag(Dense<remove_complex<ValueType>> *result) const
+void Dense<ValueType>::get_imag(
+    typename Dense<ValueType>::real_type *result) const
 {
-    auto exec = this->get_executor();
-
     GKO_ASSERT_EQUAL_DIMENSIONS(this, result);
+    auto exec = this->get_executor();
 
     exec->run(
         dense::make_get_imag(this, make_temporary_clone(exec, result).get()));
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 17495617b74..9a6e81b94ca 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -143,6 +143,7 @@ class Dense
     using mat_data = gko::matrix_data<ValueType, int64>;
     using mat_data32 = gko::matrix_data<ValueType, int32>;
     using absolute_type = remove_complex<Dense>;
+    using real_type = absolute_type;
     using complex_type = to_complex<Dense>;
 
     using row_major_range = gko::range<gko::accessor::row_major<ValueType, 2>>;
@@ -266,18 +267,76 @@ class Dense
 
     std::unique_ptr<LinOp> conj_transpose() const override;
 
+    /**
+     * Writes the transposed matrix into the given output matrix.
+     *
+     * @param output  The output matrix. It must have the dimensions
+     *                `gko::transpose(this->get_size())`
+     */
+    void transpose(Dense *output) const;
+
+    /**
+     * Writes the conjugate-transposed matrix into the given output matrix.
+     *
+     * @param output  The output matrix. It must have the dimensions
+     *                `gko::transpose(this->get_size())`
+     */
+    void conj_transpose(Dense *output) const;
+
+    /**
+     * Fill the dense matrix with a given value.
+     *
+     * @param value  the value to be filled
+     */
+    void fill(const ValueType value);
+
     std::unique_ptr<LinOp> permute(
         const Array<int32> *permutation_indices) const override;
 
     std::unique_ptr<LinOp> permute(
         const Array<int64> *permutation_indices) const override;
 
+    /**
+     * Writes the symetrically permuted matrix into the given output matrix.
+     *
+     * @param permutation_indices  The array containing permutation indices.
+     *                             It must have `this->get_size()[0]` elements.
+     * @param output  The output matrix. It must have the dimensions
+     *                `this->get_size()`
+     * @see Dense::permute(const Array<int32>*)
+     */
+    void permute(const Array<int32> *permutation_indices, Dense *output) const;
+
+    /**
+     * @copydoc Dense::permute(const Array<int32>*, Dense*)
+     */
+    void permute(const Array<int64> *permutation_indices, Dense *output) const;
+
     std::unique_ptr<LinOp> inverse_permute(
         const Array<int32> *permutation_indices) const override;
 
     std::unique_ptr<LinOp> inverse_permute(
         const Array<int64> *permutation_indices) const override;
 
+    /**
+     * Writes the inverse symetrically permuted matrix into the given output
+     * matrix.
+     *
+     * @param permutation_indices  The array containing permutation indices.
+     *                             It must have `this->get_size()[0]` elements.
+     * @param output  The output matrix. It must have the dimensions
+     *                `this->get_size()`
+     * @see Dense::inverse_permute(const Array<int32>*)
+     */
+    void inverse_permute(const Array<int32> *permutation_indices,
+                         Dense *output) const;
+
+    /**
+     * @copydoc Dense::inverse_permute(const Array<int32>*, Dense*)
+     */
+    void inverse_permute(const Array<int64> *permutation_indices,
+                         Dense *output) const;
+
     std::unique_ptr<LinOp> row_permute(
         const Array<int32> *permutation_indices) const override;
 
@@ -285,11 +344,22 @@ class Dense
         const Array<int64> *permutation_indices) const override;
 
     /**
-     * Fill the dense matrix with a given value.
+     * Writes the row-permuted matrix into the given output matrix.
      *
-     * @param value  the value to be filled
+     * @param permutation_indices  The array containing permutation indices.
+     *                             It must have `this->get_size()[0]` elements.
+     * @param output  The output matrix. It must have the dimensions
+     *                `this->get_size()`
+     * @see Dense::row_permute(const Array<int32>*)
      */
-    void fill(const ValueType value);
+    void row_permute(const Array<int32> *permutation_indices,
+                     Dense *output) const;
+
+    /**
+     * @copydoc Dense::row_permute(const Array<int32>*, Dense*)
+     */
+    void row_permute(const Array<int64> *permutation_indices,
+                     Dense *output) const;
 
     /**
      * Create a Dense matrix consisting of the given rows from this matrix.
@@ -334,22 +404,94 @@ class Dense
     std::unique_ptr<LinOp> column_permute(
         const Array<int64> *permutation_indices) const override;
 
+    /**
+     * Writes the column-permuted matrix into the given output matrix.
+     *
+     * @param permutation_indices  The array containing permutation indices.
+     *                             It must have `this->get_size()[1]` elements.
+     * @param output  The output matrix. It must have the dimensions
+     *                `this->get_size()`
+     * @see Dense::column_permute(const Array<int32>*)
+     */
+    void column_permute(const Array<int32> *permutation_indices,
+                        Dense *output) const;
+
+    /**
+     * @copydoc Dense::column_permute(const Array<int32>*, Dense*)
+     */
+    void column_permute(const Array<int64> *permutation_indices,
+                        Dense *output) const;
+
     std::unique_ptr<LinOp> inverse_row_permute(
         const Array<int32> *permutation_indices) const override;
 
     std::unique_ptr<LinOp> inverse_row_permute(
         const Array<int64> *permutation_indices) const override;
 
+    /**
+     * Writes the inverse row-permuted matrix into the given output matrix.
+     *
+     * @param permutation_indices  The array containing permutation indices.
+     *                             It must have `this->get_size()[0]` elements.
+     * @param output  The output matrix. It must have the dimensions
+     *                `this->get_size()`
+     * @see Dense::inverse_row_permute(const Array<int32>*)
+     */
+    void inverse_row_permute(const Array<int32> *permutation_indices,
+                             Dense *output) const;
+
+    /**
+     * @copydoc Dense::inverse_row_permute(const Array<int32>*, Dense*)
+     */
+    void inverse_row_permute(const Array<int64> *permutation_indices,
+                             Dense *output) const;
+
     std::unique_ptr<LinOp> inverse_column_permute(
         const Array<int32> *permutation_indices) const override;
 
     std::unique_ptr<LinOp> inverse_column_permute(
         const Array<int64> *permutation_indices) const override;
 
+    /**
+     * Writes the inverse column-permuted matrix into the given output matrix.
+     *
+     * @param permutation_indices  The array containing permutation indices.
+     *                             It must have `this->get_size()[1]` elements.
+     * @param output  The output matrix. It must have the dimensions
+     *                `this->get_size()`
+     * @see Dense::inverse_column_permute(const Array<int32>*)
+     */
+    void inverse_column_permute(const Array<int32> *permutation_indices,
+                                Dense *output) const;
+
+    /**
+     * @copydoc Dense::inverse_column_permute(const Array<int32>*, Dense*)
+     */
+    void inverse_column_permute(const Array<int64> *permutation_indices,
+                                Dense *output) const;
+
     std::unique_ptr<Diagonal<ValueType>> extract_diagonal() const override;
 
+    /**
+     * Writes the diagonal of this matrix into an existing diagonal matrix.
+     *
+     * @param output  The output matrix. Its size must match the size of this
+     *                matrix's diagonal.
+     * @see Dense::extract_diagonal()
+     */
+    void extract_diagonal(Diagonal<ValueType> *output) const;
+
     std::unique_ptr<absolute_type> compute_absolute() const override;
 
+    /**
+     * Writes the absolute values of this matrix into an existing matrix.
+     *
+     * @param output  The output matrix. Its size must match the size of this
+     *                matrix.
+     * @see Dense::compute_absolute()
+     */
+    void compute_absolute(absolute_type *output) const;
+
     void compute_absolute_inplace() override;
 
     /**
@@ -363,30 +505,30 @@ class Dense
      * If the original matrix was real, the imaginary part of the result will
      * be zero.
      */
-    void make_complex(Dense<to_complex<ValueType>> *result) const;
+    void make_complex(complex_type *result) const;
 
     /**
      * Creates a new real matrix and extracts the real part of the original
      * matrix into that.
      */
-    std::unique_ptr<absolute_type> get_real() const;
+    std::unique_ptr<real_type> get_real() const;
 
     /**
      * Extracts the real part of the original matrix into a given real matrix.
      */
-    void get_real(Dense<remove_complex<ValueType>> *result) const;
+    void get_real(real_type *result) const;
 
     /**
      * Creates a new real matrix and extracts the imaginary part of the
      * original matrix into that.
      */
-    std::unique_ptr<absolute_type> get_imag() const;
+    std::unique_ptr<real_type> get_imag() const;
 
     /**
      * Extracts the imaginary part of the original matrix into a given real
      * matrix.
      */
-    void get_imag(Dense<remove_complex<ValueType>> *result) const;
+    void get_imag(real_type *result) const;
 
     /**
      * Returns a pointer to the array of values of the matrix.

From 3950cf67a78981e122a82bf678168e283285ad6c Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 27 May 2021 21:59:55 +0200
Subject: [PATCH 0731/1008] improve Dense reference tests

---
 reference/test/matrix/dense_kernels.cpp | 390 ++++++++++--------------
 1 file changed, 161 insertions(+), 229 deletions(-)

diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index ff31dd543e9..e476f217906 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -1705,7 +1705,7 @@ TYPED_TEST(Dense, ConvertsToAndFromSellpWithMoreThanOneSlice)
     x->convert_to(sellp_mtx.get());
     sellp_mtx->convert_to(dense_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), x.get(), r<TypeParam>::value);
+    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), x.get(), 0.0);
 }
 
 
@@ -1928,24 +1928,22 @@ TYPED_TEST(Dense, MovesEmptyToSellp)
 TYPED_TEST(Dense, SquareMatrixIsTransposable)
 {
     using Mtx = typename TestFixture::Mtx;
-    auto trans = this->mtx5->transpose();
-    auto trans_as_dense = static_cast<Mtx *>(trans.get());
+    using T = typename TestFixture::value_type;
+    auto trans = gko::as<Mtx>(this->mtx5->transpose());
 
     GKO_ASSERT_MTX_NEAR(
-        trans_as_dense,
-        l({{1.0, -2.0, 2.1}, {-1.0, 2.0, 3.4}, {-0.5, 4.5, 1.2}}),
-        r<TypeParam>::value);
+        trans, l<T>({{1.0, -2.0, 2.1}, {-1.0, 2.0, 3.4}, {-0.5, 4.5, 1.2}}),
+        0.0);
 }
 
 
 TYPED_TEST(Dense, NonSquareMatrixIsTransposable)
 {
     using Mtx = typename TestFixture::Mtx;
-    auto trans = this->mtx4->transpose();
-    auto trans_as_dense = static_cast<Mtx *>(trans.get());
+    using T = typename TestFixture::value_type;
+    auto trans = gko::as<Mtx>(this->mtx4->transpose());
 
-    GKO_ASSERT_MTX_NEAR(trans_as_dense, l({{1.0, 0.0}, {3.0, 5.0}, {2.0, 0.0}}),
-                        r<TypeParam>::value);
+    GKO_ASSERT_MTX_NEAR(trans, l<T>({{1.0, 0.0}, {3.0, 5.0}, {2.0, 0.0}}), 0.0);
 }
 
 
@@ -1957,16 +1955,14 @@ TYPED_TEST(Dense, SquareMatrixCanGatherRows)
     // {2.1, 3.4, 1.2}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx5->get_executor();
     gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
 
     auto row_gathered = this->mtx5->row_gather(&permute_idxs);
 
-    // clang-format off
     GKO_ASSERT_MTX_NEAR(row_gathered,
-                        l({{-2.0, 2.0, 4.5},
-                           {1.0, -1.0, -0.5}}), r<TypeParam>::value);
-    // clang-format on
+                        l<T>({{-2.0, 2.0, 4.5}, {1.0, -1.0, -0.5}}), 0.0);
 }
 
 
@@ -1978,17 +1974,15 @@ TYPED_TEST(Dense, SquareMatrixCanGatherRowsIntoDense)
     // {2.1, 3.4, 1.2}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx5->get_executor();
     gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
     auto row_gathered = Mtx::create(exec, gko::dim<2>{2, 3});
 
     this->mtx5->row_gather(&permute_idxs, row_gathered.get());
 
-    // clang-format off
     GKO_ASSERT_MTX_NEAR(row_gathered,
-                        l({{-2.0, 2.0, 4.5},
-                           {1.0, -1.0, -0.5}}), r<TypeParam>::value);
-    // clang-format on
+                        l<T>({{-2.0, 2.0, 4.5}, {1.0, -1.0, -0.5}}), 0.0);
 }
 
 
@@ -2000,16 +1994,14 @@ TYPED_TEST(Dense, SquareMatrixCanGatherRows64)
     // {2.1, 3.4, 1.2}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx5->get_executor();
     gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
 
     auto row_gathered = this->mtx5->row_gather(&permute_idxs);
 
-    // clang-format off
     GKO_ASSERT_MTX_NEAR(row_gathered,
-                        l({{-2.0, 2.0, 4.5},
-                           {1.0, -1.0, -0.5}}), r<TypeParam>::value);
-    // clang-format on
+                        l<T>({{-2.0, 2.0, 4.5}, {1.0, -1.0, -0.5}}), 0.0);
 }
 
 
@@ -2021,17 +2013,15 @@ TYPED_TEST(Dense, SquareMatrixCanGatherRowsIntoDense64)
     // {2.1, 3.4, 1.2}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx5->get_executor();
     gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
     auto row_gathered = Mtx::create(exec, gko::dim<2>{2, 3});
 
     this->mtx5->row_gather(&permute_idxs, row_gathered.get());
 
-    // clang-format off
     GKO_ASSERT_MTX_NEAR(row_gathered,
-                        l({{-2.0, 2.0, 4.5},
-                           {1.0, -1.0, -0.5}}), r<TypeParam>::value);
-    // clang-format on
+                        l<T>({{-2.0, 2.0, 4.5}, {1.0, -1.0, -0.5}}), 0.0);
 }
 
 
@@ -2051,7 +2041,7 @@ TYPED_TEST(Dense, SquareMatrixIsPermutable)
                          ->column_permute(&permute_idxs));
     auto permuted = gko::as<Mtx>(this->mtx5->permute(&permute_idxs));
 
-    GKO_ASSERT_MTX_NEAR(ref_permuted, ref_permuted, r<TypeParam>::value);
+    GKO_ASSERT_MTX_NEAR(permuted, ref_permuted, 0.0);
 }
 
 
@@ -2071,7 +2061,7 @@ TYPED_TEST(Dense, SquareMatrixIsInversePermutable)
             ->inverse_column_permute(&permute_idxs));
     auto permuted = gko::as<Mtx>(this->mtx5->inverse_permute(&permute_idxs));
 
-    GKO_ASSERT_MTX_NEAR(ref_permuted, ref_permuted, r<TypeParam>::value);
+    GKO_ASSERT_MTX_NEAR(permuted, ref_permuted, 0.0);
 }
 
 
@@ -2091,7 +2081,7 @@ TYPED_TEST(Dense, SquareMatrixIsPermutable64)
                          ->column_permute(&permute_idxs));
     auto permuted = gko::as<Mtx>(this->mtx5->permute(&permute_idxs));
 
-    GKO_ASSERT_MTX_NEAR(ref_permuted, ref_permuted, r<TypeParam>::value);
+    GKO_ASSERT_MTX_NEAR(permuted, ref_permuted, 0.0);
 }
 
 
@@ -2111,7 +2101,7 @@ TYPED_TEST(Dense, SquareMatrixIsInversePermutable64)
             ->inverse_column_permute(&permute_idxs));
     auto permuted = gko::as<Mtx>(this->mtx5->inverse_permute(&permute_idxs));
 
-    GKO_ASSERT_MTX_NEAR(ref_permuted, ref_permuted, r<TypeParam>::value);
+    GKO_ASSERT_MTX_NEAR(permuted, ref_permuted, 0.0);
 }
 
 
@@ -2123,18 +2113,15 @@ TYPED_TEST(Dense, SquareMatrixIsRowPermutable)
     // {2.1, 3.4, 1.2}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx5->get_executor();
     gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
 
-    auto row_permute = this->mtx5->row_permute(&permute_idxs);
-    auto row_permute_dense = static_cast<Mtx *>(row_permute.get());
+    auto row_permute = gko::as<Mtx>(this->mtx5->row_permute(&permute_idxs));
 
-    // clang-format off
-    GKO_ASSERT_MTX_NEAR(row_permute_dense,
-                        l({{-2.0, 2.0, 4.5},
-                           {2.1, 3.4, 1.2},
-                           {1.0, -1.0, -0.5}}), r<TypeParam>::value);
-    // clang-format on
+    GKO_ASSERT_MTX_NEAR(
+        row_permute,
+        l<T>({{-2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}, {1.0, -1.0, -0.5}}), 0.0);
 }
 
 
@@ -2145,17 +2132,14 @@ TYPED_TEST(Dense, NonSquareMatrixIsRowPermutable)
     // {0.0, 5.0, 0.0}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
 
-    auto row_permute = this->mtx4->row_permute(&permute_idxs);
-    auto row_permute_dense = static_cast<Mtx *>(row_permute.get());
+    auto row_permute = gko::as<Mtx>(this->mtx4->row_permute(&permute_idxs));
 
-    // clang-format off
-    GKO_ASSERT_MTX_NEAR(row_permute_dense,
-                        l({{0.0, 5.0, 0.0},
-                           {1.0, 3.0, 2.0}}), r<TypeParam>::value);
-    // clang-format on
+    GKO_ASSERT_MTX_NEAR(row_permute, l<T>({{0.0, 5.0, 0.0}, {1.0, 3.0, 2.0}}),
+                        0.0);
 }
 
 
@@ -2167,16 +2151,15 @@ TYPED_TEST(Dense, SquareMatrixIsColPermutable)
     // {2.1, 3.4, 1.2}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx5->get_executor();
     gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
 
-    auto c_permute = this->mtx5->column_permute(&permute_idxs);
-    auto c_permute_dense = static_cast<Mtx *>(c_permute.get());
+    auto c_permute = gko::as<Mtx>(this->mtx5->column_permute(&permute_idxs));
 
-    // clang-format off
-    GKO_ASSERT_MTX_NEAR(c_permute_dense,
-                        l({{-1.0, -0.5, 1.0}, {2.0, 4.5, -2.0}, {3.4, 1.2, 2.1}}), r<TypeParam>::value);
-    // clang-format on
+    GKO_ASSERT_MTX_NEAR(
+        c_permute, l<T>({{-1.0, -0.5, 1.0}, {2.0, 4.5, -2.0}, {3.4, 1.2, 2.1}}),
+        0.0);
 }
 
 
@@ -2187,18 +2170,14 @@ TYPED_TEST(Dense, NonSquareMatrixIsColPermutable)
     // {0.0, 5.0, 0.0}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
 
-    auto c_permute = this->mtx4->column_permute(&permute_idxs);
-    auto c_permute_dense = static_cast<Mtx *>(c_permute.get());
+    auto c_permute = gko::as<Mtx>(this->mtx4->column_permute(&permute_idxs));
 
-    // clang-format off
-    GKO_ASSERT_MTX_NEAR(c_permute_dense,
-                        l({{3.0, 2.0, 1.0},
-                           {5.0, 0.0, 0.0}}),
-                        r<TypeParam>::value);
-    // clang-format on
+    GKO_ASSERT_MTX_NEAR(c_permute, l<T>({{3.0, 2.0, 1.0}, {5.0, 0.0, 0.0}}),
+                        0.0);
 }
 
 
@@ -2210,20 +2189,16 @@ TYPED_TEST(Dense, SquareMatrixIsInverseRowPermutable)
     // {2.1, 3.4, 1.2}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx5->get_executor();
     gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 2, 0}};
 
     auto inverse_row_permute =
-        this->mtx5->inverse_row_permute(&inverse_permute_idxs);
-    auto inverse_row_permute_dense =
-        static_cast<Mtx *>(inverse_row_permute.get());
+        gko::as<Mtx>(this->mtx5->inverse_row_permute(&inverse_permute_idxs));
 
-    // clang-format off
-    GKO_ASSERT_MTX_NEAR(inverse_row_permute_dense,
-                        l({{2.1, 3.4, 1.2},
-                           {1.0, -1.0, -0.5},
-                           {-2.0, 2.0, 4.5}}), r<TypeParam>::value);
-    // clang-format on
+    GKO_ASSERT_MTX_NEAR(
+        inverse_row_permute,
+        l<T>({{2.1, 3.4, 1.2}, {1.0, -1.0, -0.5}, {-2.0, 2.0, 4.5}}), 0.0);
 }
 
 
@@ -2234,19 +2209,15 @@ TYPED_TEST(Dense, NonSquareMatrixIsInverseRowPermutable)
     // {0.0, 5.0, 0.0}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 0}};
 
     auto inverse_row_permute =
-        this->mtx4->inverse_row_permute(&inverse_permute_idxs);
-    auto inverse_row_permute_dense =
-        static_cast<Mtx *>(inverse_row_permute.get());
+        gko::as<Mtx>(this->mtx4->inverse_row_permute(&inverse_permute_idxs));
 
-    // clang-format off
-    GKO_ASSERT_MTX_NEAR(inverse_row_permute_dense,
-                        l({{0.0, 5.0, 0.0},
-                           {1.0, 3.0, 2.0}}), r<TypeParam>::value);
-    // clang-format on
+    GKO_ASSERT_MTX_NEAR(inverse_row_permute,
+                        l<T>({{0.0, 5.0, 0.0}, {1.0, 3.0, 2.0}}), 0.0);
 }
 
 
@@ -2258,19 +2229,16 @@ TYPED_TEST(Dense, SquareMatrixIsInverseColPermutable)
     // {2.1, 3.4, 1.2}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx5->get_executor();
     gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 2, 0}};
 
     auto inverse_c_permute =
-        this->mtx5->inverse_column_permute(&inverse_permute_idxs);
-    auto inverse_c_permute_dense = static_cast<Mtx *>(inverse_c_permute.get());
+        gko::as<Mtx>(this->mtx5->inverse_column_permute(&inverse_permute_idxs));
 
-    // clang-format off
-    GKO_ASSERT_MTX_NEAR(inverse_c_permute_dense,
-                        l({{-0.5, 1.0, -1.0},
-                           {4.5, -2.0, 2.0},
-                           {1.2, 2.1, 3.4}}), r<TypeParam>::value);
-    // clang-format on
+    GKO_ASSERT_MTX_NEAR(
+        inverse_c_permute,
+        l<T>({{-0.5, 1.0, -1.0}, {4.5, -2.0, 2.0}, {1.2, 2.1, 3.4}}), 0.0);
 }
 
 
@@ -2281,19 +2249,15 @@ TYPED_TEST(Dense, NonSquareMatrixIsInverseColPermutable)
     // {0.0, 5.0, 0.0}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 2, 0}};
 
     auto inverse_c_permute =
-        this->mtx4->inverse_column_permute(&inverse_permute_idxs);
-    auto inverse_c_permute_dense = static_cast<Mtx *>(inverse_c_permute.get());
+        gko::as<Mtx>(this->mtx4->inverse_column_permute(&inverse_permute_idxs));
 
-    // clang-format off
-    GKO_ASSERT_MTX_NEAR(inverse_c_permute_dense,
-                        l({{2.0, 1.0, 3.0},
-                           {0.0, 0.0, 5.0}}),
-                        r<TypeParam>::value);
-    // clang-format on
+    GKO_ASSERT_MTX_NEAR(inverse_c_permute,
+                        l<T>({{2.0, 1.0, 3.0}, {0.0, 0.0, 5.0}}), 0.0);
 }
 
 
@@ -2304,17 +2268,14 @@ TYPED_TEST(Dense, NonSquareMatrixIsRowPermutable64)
     // {0.0, 5.0, 0.0}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
 
-    auto row_permute = this->mtx4->row_permute(&permute_idxs);
-    auto row_permute_dense = static_cast<Mtx *>(row_permute.get());
+    auto row_permute = gko::as<Mtx>(this->mtx4->row_permute(&permute_idxs));
 
-    // clang-format off
-    GKO_ASSERT_MTX_NEAR(row_permute_dense,
-                        l({{0.0, 5.0, 0.0},
-                           {1.0, 3.0, 2.0}}), r<TypeParam>::value);
-    // clang-format on
+    GKO_ASSERT_MTX_NEAR(row_permute, l<T>({{0.0, 5.0, 0.0}, {1.0, 3.0, 2.0}}),
+                        0.0);
 }
 
 
@@ -2325,18 +2286,14 @@ TYPED_TEST(Dense, NonSquareMatrixIsColPermutable64)
     // {0.0, 5.0, 0.0}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
 
-    auto c_permute = this->mtx4->column_permute(&permute_idxs);
-    auto c_permute_dense = static_cast<Mtx *>(c_permute.get());
+    auto c_permute = gko::as<Mtx>(this->mtx4->column_permute(&permute_idxs));
 
-    // clang-format off
-    GKO_ASSERT_MTX_NEAR(c_permute_dense,
-                        l({{3.0, 2.0, 1.0},
-                           {5.0, 0.0, 0.0}}),
-                        r<TypeParam>::value);
-    // clang-format on
+    GKO_ASSERT_MTX_NEAR(c_permute, l<T>({{3.0, 2.0, 1.0}, {5.0, 0.0, 0.0}}),
+                        0.0);
 }
 
 
@@ -2347,19 +2304,15 @@ TYPED_TEST(Dense, NonSquareMatrixIsInverseRowPermutable64)
     // {0.0, 5.0, 0.0}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int64> inverse_permute_idxs{exec, {1, 0}};
 
     auto inverse_row_permute =
-        this->mtx4->inverse_row_permute(&inverse_permute_idxs);
-    auto inverse_row_permute_dense =
-        static_cast<Mtx *>(inverse_row_permute.get());
+        gko::as<Mtx>(this->mtx4->inverse_row_permute(&inverse_permute_idxs));
 
-    // clang-format off
-    GKO_ASSERT_MTX_NEAR(inverse_row_permute_dense,
-                        l({{0.0, 5.0, 0.0},
-                           {1.0, 3.0, 2.0}}), r<TypeParam>::value);
-    // clang-format on
+    GKO_ASSERT_MTX_NEAR(inverse_row_permute,
+                        l<T>({{0.0, 5.0, 0.0}, {1.0, 3.0, 2.0}}), 0.0);
 }
 
 
@@ -2370,19 +2323,15 @@ TYPED_TEST(Dense, NonSquareMatrixIsInverseColPermutable64)
     // {0.0, 5.0, 0.0}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
     auto exec = this->mtx4->get_executor();
     gko::Array<gko::int64> inverse_permute_idxs{exec, {1, 2, 0}};
 
     auto inverse_c_permute =
-        this->mtx4->inverse_column_permute(&inverse_permute_idxs);
-    auto inverse_c_permute_dense = static_cast<Mtx *>(inverse_c_permute.get());
+        gko::as<Mtx>(this->mtx4->inverse_column_permute(&inverse_permute_idxs));
 
-    // clang-format off
-    GKO_ASSERT_MTX_NEAR(inverse_c_permute_dense,
-                        l({{2.0, 1.0, 3.0},
-                           {0.0, 0.0, 5.0}}),
-                        r<TypeParam>::value);
-    // clang-format on
+    GKO_ASSERT_MTX_NEAR(inverse_c_permute,
+                        l<T>({{2.0, 1.0, 3.0}, {0.0, 0.0, 5.0}}), 0.0);
 }
 
 
@@ -2451,9 +2400,9 @@ TYPED_TEST(Dense, InplaceAbsolute)
 
     this->mtx5->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(this->mtx5,
-                        l({{1.0, 1.0, 0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
-                        r<TypeParam>::value);
+    GKO_ASSERT_MTX_NEAR(
+        this->mtx5, l<T>({{1.0, 1.0, 0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
+        0.0);
 }
 
 
@@ -2470,9 +2419,9 @@ TYPED_TEST(Dense, InplaceAbsoluteSubMatrix)
 
     mtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(this->mtx5,
-                        l({{1.0, 1.0, -0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
-                        r<TypeParam>::value);
+    GKO_ASSERT_MTX_NEAR(
+        this->mtx5, l<T>({{1.0, 1.0, -0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
+        0.0);
 }
 
 
@@ -2487,9 +2436,9 @@ TYPED_TEST(Dense, OutplaceAbsolute)
 
     auto abs_mtx = this->mtx5->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx,
-                        l({{1.0, 1.0, 0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
-                        r<TypeParam>::value);
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l<T>({{1.0, 1.0, 0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
+        0.0);
 }
 
 
@@ -2506,8 +2455,7 @@ TYPED_TEST(Dense, OutplaceAbsoluteSubMatrix)
 
     auto abs_mtx = mtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx, l({{1.0, 1.0}, {2.0, 2.0}}),
-                        r<TypeParam>::value);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, l<T>({{1.0, 1.0}, {2.0, 2.0}}), 0);
     GKO_ASSERT_EQ(abs_mtx->get_stride(), 2);
 }
 
@@ -2519,13 +2467,12 @@ TYPED_TEST(Dense, AppliesToComplex)
     using Vec = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
-    // clang-format off
-    auto b = gko::initialize<Vec>(
-        {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
-         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
-         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
-    auto x = Vec::create(exec, gko::dim<2>{2,2});
-    // clang-format on
+    auto b =
+        gko::initialize<Vec>({{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
+                              {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
+                              {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}},
+                             exec);
+    auto x = Vec::create(exec, gko::dim<2>{2, 2});
 
     this->mtx1->apply(b.get(), x.get());
 
@@ -2545,13 +2492,12 @@ TYPED_TEST(Dense, AppliesToMixedComplex)
     using Vec = gko::matrix::Dense<mixed_complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
-    // clang-format off
     auto b = gko::initialize<Vec>(
         {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
          {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}},
-         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec);
-    auto x = Vec::create(exec, gko::dim<2>{2,2});
-    // clang-format on
+         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}},
+        exec);
+    auto x = Vec::create(exec, gko::dim<2>{2, 2});
 
     this->mtx1->apply(b.get(), x.get());
 
@@ -2571,17 +2517,17 @@ TYPED_TEST(Dense, AdvancedAppliesToComplex)
     using DenseComplex = gko::matrix::Dense<complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
-    // clang-format off
     auto b = gko::initialize<DenseComplex>(
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
          {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}},
-         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec);
+         {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}},
+        exec);
     auto x = gko::initialize<DenseComplex>(
         {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}},
-         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec);
+         {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}},
+        exec);
     auto alpha = gko::initialize<Dense>({-1.0}, this->exec);
     auto beta = gko::initialize<Dense>({2.0}, this->exec);
-    // clang-format on
 
     this->mtx1->apply(alpha.get(), b.get(), beta.get(), x.get());
 
@@ -2602,17 +2548,17 @@ TYPED_TEST(Dense, AdvancedAppliesToMixedComplex)
     using MixedDenseComplex = gko::matrix::Dense<mixed_complex_type>;
     auto exec = gko::ReferenceExecutor::create();
 
-    // clang-format off
     auto b = gko::initialize<MixedDenseComplex>(
         {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
          {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}},
-         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec);
+         {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}},
+        exec);
     auto x = gko::initialize<MixedDenseComplex>(
         {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}},
-         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}}, exec);
+         {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}},
+        exec);
     auto alpha = gko::initialize<MixedDense>({-1.0}, this->exec);
     auto beta = gko::initialize<MixedDense>({2.0}, this->exec);
-    // clang-format on
 
     this->mtx1->apply(alpha.get(), b.get(), beta.get(), x.get());
 
@@ -2636,11 +2582,11 @@ TYPED_TEST(Dense, MakeComplex)
 
     auto complex_mtx = this->mtx5->make_complex();
 
-    GKO_ASSERT_MTX_NEAR(complex_mtx, this->mtx5, 0);
+    GKO_ASSERT_MTX_NEAR(complex_mtx, this->mtx5, 0.0);
 }
 
 
-TYPED_TEST(Dense, MakeComplexWithGivenResult)
+TYPED_TEST(Dense, MakeComplexIntoDense)
 {
     using T = typename TestFixture::value_type;
     using ComplexMtx = typename TestFixture::ComplexMtx;
@@ -2654,11 +2600,11 @@ TYPED_TEST(Dense, MakeComplexWithGivenResult)
     auto complex_mtx = ComplexMtx::create(exec, this->mtx5->get_size());
     this->mtx5->make_complex(complex_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(complex_mtx, this->mtx5, 0);
+    GKO_ASSERT_MTX_NEAR(complex_mtx, this->mtx5, 0.0);
 }
 
 
-TYPED_TEST(Dense, MakeComplexWithGivenResultFailsForWrongDimensions)
+TYPED_TEST(Dense, MakeComplexIntoDenseFailsForWrongDimensions)
 {
     using T = typename TestFixture::value_type;
     using ComplexMtx = typename TestFixture::ComplexMtx;
@@ -2687,11 +2633,11 @@ TYPED_TEST(Dense, GetReal)
 
     auto real_mtx = this->mtx5->get_real();
 
-    GKO_ASSERT_MTX_NEAR(real_mtx, this->mtx5, 0);
+    GKO_ASSERT_MTX_NEAR(real_mtx, this->mtx5, 0.0);
 }
 
 
-TYPED_TEST(Dense, GetRealWithGivenResult)
+TYPED_TEST(Dense, GetRealIntoDense)
 {
     using T = typename TestFixture::value_type;
     using RealMtx = typename TestFixture::RealMtx;
@@ -2705,11 +2651,11 @@ TYPED_TEST(Dense, GetRealWithGivenResult)
     auto real_mtx = RealMtx::create(exec, this->mtx5->get_size());
     this->mtx5->get_real(real_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(real_mtx, this->mtx5, 0);
+    GKO_ASSERT_MTX_NEAR(real_mtx, this->mtx5, 0.0);
 }
 
 
-TYPED_TEST(Dense, GetRealWithGivenResultFailsForWrongDimensions)
+TYPED_TEST(Dense, GetRealIntoDenseFailsForWrongDimensions)
 {
     using T = typename TestFixture::value_type;
     using RealMtx = typename TestFixture::RealMtx;
@@ -2737,11 +2683,12 @@ TYPED_TEST(Dense, GetImag)
     auto imag_mtx = this->mtx5->get_imag();
 
     GKO_ASSERT_MTX_NEAR(
-        imag_mtx, l({{0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}}), 0);
+        imag_mtx, l<T>({{0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}}),
+        0.0);
 }
 
 
-TYPED_TEST(Dense, GetImagWithGivenResult)
+TYPED_TEST(Dense, GetImagIntoDense)
 {
     using T = typename TestFixture::value_type;
     using RealMtx = typename TestFixture::RealMtx;
@@ -2756,11 +2703,12 @@ TYPED_TEST(Dense, GetImagWithGivenResult)
     this->mtx5->get_imag(imag_mtx.get());
 
     GKO_ASSERT_MTX_NEAR(
-        imag_mtx, l({{0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}}), 0);
+        imag_mtx, l<T>({{0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}, {0.0, 0.0, 0.0}}),
+        0.0);
 }
 
 
-TYPED_TEST(Dense, GetImagWithGivenResultFailsForWrongDimensions)
+TYPED_TEST(Dense, GetImagIntoDenseFailsForWrongDimensions)
 {
     using T = typename TestFixture::value_type;
     using RealMtx = typename TestFixture::RealMtx;
@@ -2842,12 +2790,11 @@ TYPED_TEST(DenseComplex, NonSquareMatrixIsConjugateTransposable)
                                        {T{1.0, 0.0}, T{0.0, 1.0}}},
                                       exec);
 
-    auto trans = mtx->conj_transpose();
-    auto trans_as_dense = static_cast<Dense *>(trans.get());
+    auto trans = gko::as<Dense>(mtx->conj_transpose());
 
-    GKO_ASSERT_MTX_NEAR(trans_as_dense,
-                        l({{T{1.0, -2.0}, T{-2.0, -1.5}, T{1.0, 0.0}},
-                           {T{-1.0, -2.1}, T{4.5, 0.0}, T{0.0, -1.0}}}),
+    GKO_ASSERT_MTX_NEAR(trans,
+                        l<T>({{T{1.0, -2.0}, T{-2.0, -1.5}, T{1.0, 0.0}},
+                              {T{-1.0, -2.1}, T{4.5, 0.0}, T{0.0, -1.0}}}),
                         0.0);
 }
 
@@ -2857,17 +2804,15 @@ TYPED_TEST(DenseComplex, InplaceAbsolute)
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     auto exec = gko::ReferenceExecutor::create();
-    // clang-format off
-    auto mtx = gko::initialize<Mtx>(
-        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
-         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
-         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
-    // clang-format on
+    auto mtx = gko::initialize<Mtx>({{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+                                     {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+                                     {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}},
+                                    exec);
 
     mtx->compute_absolute_inplace();
 
     GKO_ASSERT_MTX_NEAR(
-        mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+        mtx, l<T>({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
 }
 
 
@@ -2876,17 +2821,16 @@ TYPED_TEST(DenseComplex, OutplaceAbsolute)
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     auto exec = gko::ReferenceExecutor::create();
-    // clang-format off
-    auto mtx = gko::initialize<Mtx>(
-        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
-         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
-         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
-    // clang-format on
+    auto mtx = gko::initialize<Mtx>({{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+                                     {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+                                     {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}},
+                                    exec);
 
     auto abs_mtx = mtx->compute_absolute();
 
     GKO_ASSERT_MTX_NEAR(
-        abs_mtx, l({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}), 0.0);
+        abs_mtx, l<T>({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}),
+        0.0);
 }
 
 
@@ -2895,12 +2839,10 @@ TYPED_TEST(DenseComplex, MakeComplex)
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     auto exec = gko::ReferenceExecutor::create();
-    // clang-format off
-    auto mtx = gko::initialize<Mtx>(
-        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
-         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
-         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
-    // clang-format on
+    auto mtx = gko::initialize<Mtx>({{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+                                     {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+                                     {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}},
+                                    exec);
 
     auto complex_mtx = mtx->make_complex();
 
@@ -2908,17 +2850,15 @@ TYPED_TEST(DenseComplex, MakeComplex)
 }
 
 
-TYPED_TEST(DenseComplex, MakeComplexWithGivenResult)
+TYPED_TEST(DenseComplex, MakeComplexIntoDense)
 {
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     auto exec = gko::ReferenceExecutor::create();
-    // clang-format off
-    auto mtx = gko::initialize<Mtx>(
-        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
-         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
-         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
-    // clang-format on
+    auto mtx = gko::initialize<Mtx>({{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+                                     {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+                                     {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}},
+                                    exec);
 
     auto complex_mtx = Mtx::create(exec, mtx->get_size());
     mtx->make_complex(complex_mtx.get());
@@ -2932,39 +2872,35 @@ TYPED_TEST(DenseComplex, GetReal)
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     auto exec = gko::ReferenceExecutor::create();
-    // clang-format off
-    auto mtx = gko::initialize<Mtx>(
-        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
-         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
-         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
-    // clang-format on
+    auto mtx = gko::initialize<Mtx>({{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+                                     {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+                                     {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}},
+                                    exec);
 
     auto real_mtx = mtx->get_real();
 
     GKO_ASSERT_MTX_NEAR(
-        real_mtx, l({{1.0, 3.0, 0.0}, {-4.0, -1.0, 0.0}, {0.0, 0.0, 2.0}}),
+        real_mtx, l<T>({{1.0, 3.0, 0.0}, {-4.0, -1.0, 0.0}, {0.0, 0.0, 2.0}}),
         0.0);
 }
 
 
-TYPED_TEST(DenseComplex, GetRealWithGivenResult)
+TYPED_TEST(DenseComplex, GetRealIntoDense)
 {
     using Mtx = typename TestFixture::Mtx;
     using RealMtx = typename TestFixture::RealMtx;
     using T = typename TestFixture::value_type;
     auto exec = gko::ReferenceExecutor::create();
-    // clang-format off
-    auto mtx = gko::initialize<Mtx>(
-        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
-         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
-         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
-    // clang-format on
+    auto mtx = gko::initialize<Mtx>({{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+                                     {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+                                     {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}},
+                                    exec);
 
     auto real_mtx = RealMtx::create(exec, mtx->get_size());
     mtx->get_real(real_mtx.get());
 
     GKO_ASSERT_MTX_NEAR(
-        real_mtx, l({{1.0, 3.0, 0.0}, {-4.0, -1.0, 0.0}, {0.0, 0.0, 2.0}}),
+        real_mtx, l<T>({{1.0, 3.0, 0.0}, {-4.0, -1.0, 0.0}, {0.0, 0.0, 2.0}}),
         0.0);
 }
 
@@ -2974,39 +2910,35 @@ TYPED_TEST(DenseComplex, GetImag)
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     auto exec = gko::ReferenceExecutor::create();
-    // clang-format off
-    auto mtx = gko::initialize<Mtx>(
-        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
-         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
-         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
-    // clang-format on
+    auto mtx = gko::initialize<Mtx>({{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+                                     {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+                                     {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}},
+                                    exec);
 
     auto imag_mtx = mtx->get_imag();
 
     GKO_ASSERT_MTX_NEAR(
-        imag_mtx, l({{0.0, 4.0, 2.0}, {-3.0, 0.0, 0.0}, {0.0, -1.5, 0.0}}),
+        imag_mtx, l<T>({{0.0, 4.0, 2.0}, {-3.0, 0.0, 0.0}, {0.0, -1.5, 0.0}}),
         0.0);
 }
 
 
-TYPED_TEST(DenseComplex, GetImagWithGivenResult)
+TYPED_TEST(DenseComplex, GetImagIntoDense)
 {
     using Mtx = typename TestFixture::Mtx;
     using RealMtx = typename TestFixture::RealMtx;
     using T = typename TestFixture::value_type;
     auto exec = gko::ReferenceExecutor::create();
-    // clang-format off
-    auto mtx = gko::initialize<Mtx>(
-        {{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
-         {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
-         {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}}, exec);
-    // clang-format on
+    auto mtx = gko::initialize<Mtx>({{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+                                     {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+                                     {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}},
+                                    exec);
 
     auto imag_mtx = RealMtx::create(exec, mtx->get_size());
     mtx->get_imag(imag_mtx.get());
 
     GKO_ASSERT_MTX_NEAR(
-        imag_mtx, l({{0.0, 4.0, 2.0}, {-3.0, 0.0, 0.0}, {0.0, -1.5, 0.0}}),
+        imag_mtx, l<T>({{0.0, 4.0, 2.0}, {-3.0, 0.0, 0.0}, {0.0, -1.5, 0.0}}),
         0.0);
 }
 
@@ -3024,7 +2956,7 @@ TYPED_TEST(DenseComplex, Dot)
 
     a->compute_dot(b.get(), result.get());
 
-    GKO_ASSERT_MTX_NEAR(result, l({T{22.0, 15.0}}), 0.0);
+    GKO_ASSERT_MTX_NEAR(result, l<T>({T{22.0, 15.0}}), 0.0);
 }
 
 
@@ -3041,7 +2973,7 @@ TYPED_TEST(DenseComplex, ConjDot)
 
     a->compute_conj_dot(b.get(), result.get());
 
-    GKO_ASSERT_MTX_NEAR(result, l({T{10.0, -25.0}}), 0.0);
+    GKO_ASSERT_MTX_NEAR(result, l<T>({T{10.0, -25.0}}), 0.0);
 }
 
 

From b037bfe4b23e6db6ab983a3c6a7944f7129a57c4 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 28 May 2021 00:00:35 +0200
Subject: [PATCH 0732/1008] add Reference tests for Dense output parameters

---
 reference/test/matrix/dense_kernels.cpp | 1428 +++++++++++++++++++++--
 1 file changed, 1329 insertions(+), 99 deletions(-)

diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index e476f217906..d0b1e724246 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -1937,6 +1937,42 @@ TYPED_TEST(Dense, SquareMatrixIsTransposable)
 }
 
 
+TYPED_TEST(Dense, SquareMatrixIsTransposableIntoDense)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto trans = Mtx::create(this->exec, this->mtx5->get_size());
+
+    this->mtx5->transpose(trans.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        trans, l<T>({{1.0, -2.0, 2.1}, {-1.0, 2.0, 3.4}, {-0.5, 4.5, 1.2}}),
+        0.0);
+}
+
+
+TYPED_TEST(Dense, SquareSubmatrixIsTransposableIntoDense)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto trans = Mtx::create(this->exec, gko::dim<2>{2, 2}, 4);
+
+    this->mtx5->create_submatrix({0, 2}, {0, 2})->transpose(trans.get());
+
+    GKO_ASSERT_MTX_NEAR(trans, l<T>({{1.0, -2.0}, {-1.0, 2.0}}), 0.0);
+    ASSERT_EQ(trans->get_stride(), 4);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsTransposableIntoDenseFailsForWrongDimensions)
+{
+    using Mtx = typename TestFixture::Mtx;
+
+    ASSERT_THROW(this->mtx5->transpose(Mtx::create(this->exec).get()),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Dense, NonSquareMatrixIsTransposable)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -1947,6 +1983,41 @@ TYPED_TEST(Dense, NonSquareMatrixIsTransposable)
 }
 
 
+TYPED_TEST(Dense, NonSquareMatrixIsTransposableIntoDense)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto trans =
+        Mtx::create(this->exec, gko::transpose(this->mtx4->get_size()));
+
+    this->mtx4->transpose(trans.get());
+
+    GKO_ASSERT_MTX_NEAR(trans, l<T>({{1.0, 0.0}, {3.0, 5.0}, {2.0, 0.0}}), 0.0);
+}
+
+
+TYPED_TEST(Dense, NonSquareSubmatrixIsTransposableIntoDense)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto trans = Mtx::create(this->exec, gko::dim<2>{2, 1}, 5);
+
+    this->mtx4->create_submatrix({0, 1}, {0, 2})->transpose(trans.get());
+
+    GKO_ASSERT_MTX_NEAR(trans, l<T>({{1.0}, {3.0}}), 0.0);
+    ASSERT_EQ(trans->get_stride(), 5);
+}
+
+
+TYPED_TEST(Dense, NonSquareMatrixIsTransposableIntoDenseFailsForWrongDimensions)
+{
+    using Mtx = typename TestFixture::Mtx;
+
+    ASSERT_THROW(this->mtx4->transpose(Mtx::create(this->exec).get()),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Dense, SquareMatrixCanGatherRows)
 {
     // clang-format off
@@ -1986,6 +2057,40 @@ TYPED_TEST(Dense, SquareMatrixCanGatherRowsIntoDense)
 }
 
 
+TYPED_TEST(Dense, SquareSubmatrixCanGatherRowsIntoDense)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
+    auto row_gathered = Mtx::create(exec, gko::dim<2>{2, 2}, 4);
+
+    this->mtx5->create_submatrix({0, 2}, {1, 3})
+        ->row_gather(&permute_idxs, row_gathered.get());
+
+    GKO_ASSERT_MTX_NEAR(row_gathered, l<T>({{2.0, 4.5}, {-1.0, -0.5}}), 0.0);
+    ASSERT_EQ(row_gathered->get_stride(), 4);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixGatherRowsIntoDenseFailsForWrongDimensions)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
+
+    ASSERT_THROW(this->mtx5->row_gather(&permute_idxs, Mtx::create(exec).get()),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Dense, SquareMatrixCanGatherRows64)
 {
     // clang-format off
@@ -2025,6 +2130,40 @@ TYPED_TEST(Dense, SquareMatrixCanGatherRowsIntoDense64)
 }
 
 
+TYPED_TEST(Dense, SquareSubmatrixCanGatherRowsIntoDense64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
+    auto row_gathered = Mtx::create(exec, gko::dim<2>{2, 2}, 4);
+
+    this->mtx5->create_submatrix({0, 2}, {1, 3})
+        ->row_gather(&permute_idxs, row_gathered.get());
+
+    GKO_ASSERT_MTX_NEAR(row_gathered, l<T>({{2.0, 4.5}, {-1.0, -0.5}}), 0.0);
+    ASSERT_EQ(row_gathered->get_stride(), 4);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixGatherRowsIntoDenseFailsForWrongDimensions64)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
+
+    ASSERT_THROW(this->mtx5->row_gather(&permute_idxs, Mtx::create(exec).get()),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Dense, SquareMatrixIsPermutable)
 {
     // clang-format off
@@ -2045,6 +2184,83 @@ TYPED_TEST(Dense, SquareMatrixIsPermutable)
 }
 
 
+TYPED_TEST(Dense, SquareMatrixIsPermutableIntoDense)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+    auto permuted = Mtx::create(exec, this->mtx5->get_size());
+
+    auto ref_permuted =
+        gko::as<Mtx>(gko::as<Mtx>(this->mtx5->row_permute(&permute_idxs))
+                         ->column_permute(&permute_idxs));
+    this->mtx5->permute(&permute_idxs, permuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, ref_permuted, 0.0);
+}
+
+
+TYPED_TEST(Dense, SquareSubmatrixIsPermutableIntoDense)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
+    auto permuted = Mtx::create(exec, gko::dim<2>{2, 2}, 4);
+    auto mtx = this->mtx5->create_submatrix({0, 2}, {1, 3});
+
+    auto ref_permuted =
+        gko::as<Mtx>(gko::as<Mtx>(mtx->row_permute(&permute_idxs))
+                         ->column_permute(&permute_idxs));
+    mtx->permute(&permute_idxs, permuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, ref_permuted, 0.0);
+    ASSERT_EQ(permuted->get_stride(), 4);
+}
+
+
+TYPED_TEST(Dense, NonSquareMatrixPermuteIntoDenseFails)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx4->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+
+    ASSERT_THROW(this->mtx4->permute(&permute_idxs, this->mtx4->clone().get()),
+                 gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixPermuteIntoDenseFailsForWrongPermutationSize)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2}};
+
+    ASSERT_THROW(this->mtx5->permute(&permute_idxs, this->mtx5->clone().get()),
+                 gko::ValueMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixPermuteIntoDenseFailsForWrongDimensions)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+
+    ASSERT_THROW(this->mtx5->permute(&permute_idxs, Mtx::create(exec).get()),
+                 gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Dense, SquareMatrixIsInversePermutable)
 {
     // clang-format off
@@ -2065,6 +2281,87 @@ TYPED_TEST(Dense, SquareMatrixIsInversePermutable)
 }
 
 
+TYPED_TEST(Dense, SquareMatrixIsInversePermutableIntoDense)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+    auto permuted = Mtx::create(exec, this->mtx5->get_size());
+
+    auto ref_permuted = gko::as<Mtx>(
+        gko::as<Mtx>(this->mtx5->inverse_row_permute(&permute_idxs))
+            ->inverse_column_permute(&permute_idxs));
+    this->mtx5->inverse_permute(&permute_idxs, permuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, ref_permuted, 0.0);
+}
+
+
+TYPED_TEST(Dense, SquareSubmatrixIsInversePermutableIntoDense)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
+    auto permuted = Mtx::create(exec, gko::dim<2>{2, 2}, 4);
+    auto mtx = this->mtx5->create_submatrix({0, 2}, {1, 3});
+
+    auto ref_permuted =
+        gko::as<Mtx>(gko::as<Mtx>(mtx->inverse_row_permute(&permute_idxs))
+                         ->inverse_column_permute(&permute_idxs));
+    mtx->inverse_permute(&permute_idxs, permuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, ref_permuted, 0.0);
+    ASSERT_EQ(permuted->get_stride(), 4);
+}
+
+
+TYPED_TEST(Dense, NonSquareMatrixInversePermuteIntoDenseFails)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx4->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+
+    ASSERT_THROW(
+        this->mtx4->inverse_permute(&permute_idxs, this->mtx4->clone().get()),
+        gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(Dense,
+           SquareMatrixInversePermuteIntoDenseFailsForWrongPermutationSize)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {0, 1}};
+
+    ASSERT_THROW(
+        this->mtx5->inverse_permute(&permute_idxs, this->mtx5->clone().get()),
+        gko::ValueMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixInversePermuteIntoDenseFailsForWrongDimensions)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+
+    ASSERT_THROW(
+        this->mtx5->inverse_permute(&permute_idxs, Mtx::create(exec).get()),
+        gko::DimensionMismatch);
+}
+
+
 TYPED_TEST(Dense, SquareMatrixIsPermutable64)
 {
     // clang-format off
@@ -2085,7 +2382,28 @@ TYPED_TEST(Dense, SquareMatrixIsPermutable64)
 }
 
 
-TYPED_TEST(Dense, SquareMatrixIsInversePermutable64)
+TYPED_TEST(Dense, SquareMatrixIsPermutableIntoDense64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+    auto permuted = Mtx::create(exec, this->mtx5->get_size());
+
+    auto ref_permuted =
+        gko::as<Mtx>(gko::as<Mtx>(this->mtx5->row_permute(&permute_idxs))
+                         ->column_permute(&permute_idxs));
+    this->mtx5->permute(&permute_idxs, permuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, ref_permuted, 0.0);
+}
+
+
+TYPED_TEST(Dense, SquareSubmatrixIsPermutableIntoDense64)
 {
     // clang-format off
     // {1.0, -1.0, -0.5},
@@ -2094,18 +2412,799 @@ TYPED_TEST(Dense, SquareMatrixIsInversePermutable64)
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
     auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
+    auto permuted = Mtx::create(exec, gko::dim<2>{2, 2}, 4);
+    auto mtx = this->mtx5->create_submatrix({0, 2}, {1, 3});
+
+    auto ref_permuted =
+        gko::as<Mtx>(gko::as<Mtx>(mtx->row_permute(&permute_idxs))
+                         ->column_permute(&permute_idxs));
+    mtx->permute(&permute_idxs, permuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, ref_permuted, 0.0);
+    ASSERT_EQ(permuted->get_stride(), 4);
+}
+
+
+TYPED_TEST(Dense, NonSquareMatrixPermuteIntoDenseFails64)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx4->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+
+    ASSERT_THROW(this->mtx4->permute(&permute_idxs, this->mtx4->clone().get()),
+                 gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixPermuteIntoDenseFailsForWrongPermutationSize64)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2}};
+
+    ASSERT_THROW(this->mtx5->permute(&permute_idxs, this->mtx5->clone().get()),
+                 gko::ValueMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixPermuteIntoDenseFailsForWrongDimensions64)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+
+    ASSERT_THROW(this->mtx5->permute(&permute_idxs, Mtx::create(exec).get()),
+                 gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsInversePermutable64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+
+    auto ref_permuted = gko::as<Mtx>(
+        gko::as<Mtx>(this->mtx5->inverse_row_permute(&permute_idxs))
+            ->inverse_column_permute(&permute_idxs));
+    auto permuted = gko::as<Mtx>(this->mtx5->inverse_permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(permuted, ref_permuted, 0.0);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsInversePermutableIntoDense64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+    auto permuted = Mtx::create(exec, this->mtx5->get_size());
+
+    auto ref_permuted = gko::as<Mtx>(
+        gko::as<Mtx>(this->mtx5->inverse_row_permute(&permute_idxs))
+            ->inverse_column_permute(&permute_idxs));
+    this->mtx5->inverse_permute(&permute_idxs, permuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, ref_permuted, 0.0);
+}
+
+
+TYPED_TEST(Dense, SquareSubmatrixIsInversePermutableIntoDense64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
+    auto permuted = Mtx::create(exec, gko::dim<2>{2, 2}, 4);
+    auto mtx = this->mtx5->create_submatrix({0, 2}, {1, 3});
+
+    auto ref_permuted =
+        gko::as<Mtx>(gko::as<Mtx>(mtx->inverse_row_permute(&permute_idxs))
+                         ->inverse_column_permute(&permute_idxs));
+    mtx->inverse_permute(&permute_idxs, permuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, ref_permuted, 0.0);
+    ASSERT_EQ(permuted->get_stride(), 4);
+}
+
+
+TYPED_TEST(Dense, NonSquareMatrixInversePermuteIntoDenseFails64)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx4->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+
+    ASSERT_THROW(
+        this->mtx4->inverse_permute(&permute_idxs, this->mtx4->clone().get()),
+        gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(Dense,
+           SquareMatrixInversePermuteIntoDenseFailsForWrongPermutationSize64)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2}};
+
+    ASSERT_THROW(
+        this->mtx5->inverse_permute(&permute_idxs, this->mtx5->clone().get()),
+        gko::ValueMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixInversePermuteIntoDenseFailsForWrongDimensions64)
+{
+    using Mtx = typename TestFixture::Mtx;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+
+    ASSERT_THROW(
+        this->mtx5->inverse_permute(&permute_idxs, Mtx::create(exec).get()),
+        gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsRowPermutable)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+
+    auto row_permute = gko::as<Mtx>(this->mtx5->row_permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(
+        row_permute,
+        l<T>({{-2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}, {1.0, -1.0, -0.5}}), 0.0);
+}
+
+
+TYPED_TEST(Dense, NonSquareMatrixIsRowPermutable)
+{
+    // clang-format off
+    // {1.0, 3.0, 2.0},
+    // {0.0, 5.0, 0.0}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx4->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
+
+    auto row_permute = gko::as<Mtx>(this->mtx4->row_permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(row_permute, l<T>({{0.0, 5.0, 0.0}, {1.0, 3.0, 2.0}}),
+                        0.0);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsRowPermutableIntoDense)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+    auto row_permute = Mtx::create(exec, this->mtx5->get_size());
+
+    this->mtx5->row_permute(&permute_idxs, row_permute.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        row_permute,
+        l<T>({{-2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}, {1.0, -1.0, -0.5}}), 0.0);
+}
+
+
+TYPED_TEST(Dense, SquareSubmatrixIsRowPermutableIntoDense)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
+    auto row_permute = Mtx::create(exec, gko::dim<2>{2, 2}, 4);
+
+    this->mtx5->create_submatrix({0, 2}, {0, 2})
+        ->row_permute(&permute_idxs, row_permute.get());
+
+    GKO_ASSERT_MTX_NEAR(row_permute, l<T>({{-2.0, 2.0}, {1.0, -1.0}}), 0.0);
+    ASSERT_EQ(row_permute->get_stride(), 4);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixRowPermuteIntoDenseFailsForWrongPermutationSize)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2}};
+    auto row_permute = Mtx::create(exec, this->mtx5->get_size());
+
+    ASSERT_THROW(this->mtx5->row_permute(&permute_idxs, row_permute.get()),
+                 gko::ValueMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixRowPermuteIntoDenseFailsForWrongDimensions)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+
+    ASSERT_THROW(
+        this->mtx5->row_permute(&permute_idxs, Mtx::create(exec).get()),
+        gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsColPermutable)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+
+    auto c_permute = gko::as<Mtx>(this->mtx5->column_permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(
+        c_permute, l<T>({{-1.0, -0.5, 1.0}, {2.0, 4.5, -2.0}, {3.4, 1.2, 2.1}}),
+        0.0);
+}
+
+
+TYPED_TEST(Dense, NonSquareMatrixIsColPermutable)
+{
+    // clang-format off
+    // {1.0, 3.0, 2.0},
+    // {0.0, 5.0, 0.0}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx4->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+
+    auto c_permute = gko::as<Mtx>(this->mtx4->column_permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(c_permute, l<T>({{3.0, 2.0, 1.0}, {5.0, 0.0, 0.0}}),
+                        0.0);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsColPermutableIntoDense)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+    auto c_permute = Mtx::create(exec, this->mtx5->get_size());
+
+    this->mtx5->column_permute(&permute_idxs, c_permute.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        c_permute, l<T>({{-1.0, -0.5, 1.0}, {2.0, 4.5, -2.0}, {3.4, 1.2, 2.1}}),
+        0.0);
+}
+
+
+TYPED_TEST(Dense, SquareSubmatrixIsColPermutableIntoDense)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
+    auto c_permute = Mtx::create(exec, gko::dim<2>{2, 2}, 4);
+
+    this->mtx5->create_submatrix({0, 2}, {0, 2})
+        ->column_permute(&permute_idxs, c_permute.get());
+
+    GKO_ASSERT_MTX_NEAR(c_permute, l<T>({{-1.0, 1.0}, {2.0, -2.0}}), 0.0);
+    ASSERT_EQ(c_permute->get_stride(), 4);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixColPermuteIntoDenseFailsForWrongPermutationSize)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2}};
+    auto row_permute = Mtx::create(exec, this->mtx5->get_size());
+
+    ASSERT_THROW(this->mtx5->column_permute(&permute_idxs, row_permute.get()),
+                 gko::ValueMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixColPermuteIntoDenseFailsForWrongDimensions)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+
+    ASSERT_THROW(
+        this->mtx5->column_permute(&permute_idxs, Mtx::create(exec).get()),
+        gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsInverseRowPermutable)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 2, 0}};
+
+    auto inverse_row_permute =
+        gko::as<Mtx>(this->mtx5->inverse_row_permute(&inverse_permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(
+        inverse_row_permute,
+        l<T>({{2.1, 3.4, 1.2}, {1.0, -1.0, -0.5}, {-2.0, 2.0, 4.5}}), 0.0);
+}
+
+
+TYPED_TEST(Dense, NonSquareMatrixIsInverseRowPermutable)
+{
+    // clang-format off
+    // {1.0, 3.0, 2.0},
+    // {0.0, 5.0, 0.0}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx4->get_executor();
+    gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 0}};
+
+    auto inverse_row_permute =
+        gko::as<Mtx>(this->mtx4->inverse_row_permute(&inverse_permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(inverse_row_permute,
+                        l<T>({{0.0, 5.0, 0.0}, {1.0, 3.0, 2.0}}), 0.0);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsInverseRowPermutableIntoDense)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+    auto row_permute = Mtx::create(exec, this->mtx5->get_size());
+
+    this->mtx5->inverse_row_permute(&permute_idxs, row_permute.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        row_permute,
+        l<T>({{2.1, 3.4, 1.2}, {1.0, -1.0, -0.5}, {-2.0, 2.0, 4.5}}), 0.0);
+}
+
+
+TYPED_TEST(Dense, SquareSubmatrixIsInverseRowPermutableIntoDense)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
+    auto row_permute = Mtx::create(exec, gko::dim<2>{2, 2}, 4);
+
+    this->mtx5->create_submatrix({0, 2}, {0, 2})
+        ->inverse_row_permute(&permute_idxs, row_permute.get());
+
+    GKO_ASSERT_MTX_NEAR(row_permute, l<T>({{-2.0, 2.0}, {1.0, -1.0}}), 0.0);
+    ASSERT_EQ(row_permute->get_stride(), 4);
+}
+
+
+TYPED_TEST(Dense,
+           SquareMatrixInverseRowPermuteIntoDenseFailsForWrongPermutationSize)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2}};
+    auto row_permute = Mtx::create(exec, this->mtx5->get_size());
+
+    ASSERT_THROW(
+        this->mtx5->inverse_row_permute(&permute_idxs, row_permute.get()),
+        gko::ValueMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixInverseRowPermuteIntoDenseFailsForWrongDimensions)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+
+    ASSERT_THROW(
+        this->mtx5->inverse_row_permute(&permute_idxs, Mtx::create(exec).get()),
+        gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsInverseColPermutable)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 2, 0}};
+
+    auto inverse_c_permute =
+        gko::as<Mtx>(this->mtx5->inverse_column_permute(&inverse_permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(
+        inverse_c_permute,
+        l<T>({{-0.5, 1.0, -1.0}, {4.5, -2.0, 2.0}, {1.2, 2.1, 3.4}}), 0.0);
+}
+
+
+TYPED_TEST(Dense, NonSquareMatrixIsInverseColPermutable)
+{
+    // clang-format off
+    // {1.0, 3.0, 2.0},
+    // {0.0, 5.0, 0.0}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx4->get_executor();
+    gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 2, 0}};
+
+    auto inverse_c_permute =
+        gko::as<Mtx>(this->mtx4->inverse_column_permute(&inverse_permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(inverse_c_permute,
+                        l<T>({{2.0, 1.0, 3.0}, {0.0, 0.0, 5.0}}), 0.0);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsInverseColPermutableIntoDense)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+    auto c_permute = Mtx::create(exec, this->mtx5->get_size());
+
+    this->mtx5->inverse_column_permute(&permute_idxs, c_permute.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        c_permute, l<T>({{-0.5, 1.0, -1.0}, {4.5, -2.0, 2.0}, {1.2, 2.1, 3.4}}),
+        0.0);
+}
+
+
+TYPED_TEST(Dense, SquareSubmatrixIsInverseColPermutableIntoDense)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
+    auto c_permute = Mtx::create(exec, gko::dim<2>{2, 2}, 4);
+
+    this->mtx5->create_submatrix({0, 2}, {0, 2})
+        ->column_permute(&permute_idxs, c_permute.get());
+
+    GKO_ASSERT_MTX_NEAR(c_permute, l<T>({{-1.0, 1.0}, {2.0, -2.0}}), 0.0);
+    ASSERT_EQ(c_permute->get_stride(), 4);
+}
+
+
+TYPED_TEST(Dense,
+           SquareMatrixInverseColPermuteIntoDenseFailsForWrongPermutationSize)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2}};
+    auto row_permute = Mtx::create(exec, this->mtx5->get_size());
+
+    ASSERT_THROW(
+        this->mtx5->inverse_column_permute(&permute_idxs, row_permute.get()),
+        gko::ValueMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixInverseColPermuteIntoDenseFailsForWrongDimensions)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+
+    ASSERT_THROW(this->mtx5->inverse_column_permute(&permute_idxs,
+                                                    Mtx::create(exec).get()),
+                 gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsRowPermutable64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+
+    auto row_permute = gko::as<Mtx>(this->mtx5->row_permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(
+        row_permute,
+        l<T>({{-2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}, {1.0, -1.0, -0.5}}), 0.0);
+}
+
+
+TYPED_TEST(Dense, NonSquareMatrixIsRowPermutable64)
+{
+    // clang-format off
+    // {1.0, 3.0, 2.0},
+    // {0.0, 5.0, 0.0}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx4->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
+
+    auto row_permute = gko::as<Mtx>(this->mtx4->row_permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(row_permute, l<T>({{0.0, 5.0, 0.0}, {1.0, 3.0, 2.0}}),
+                        0.0);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsRowPermutableIntoDense64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+    auto row_permute = Mtx::create(exec, this->mtx5->get_size());
+
+    this->mtx5->row_permute(&permute_idxs, row_permute.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        row_permute,
+        l<T>({{-2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}, {1.0, -1.0, -0.5}}), 0.0);
+}
+
+
+TYPED_TEST(Dense, SquareSubmatrixIsRowPermutableIntoDense64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
+    auto row_permute = Mtx::create(exec, gko::dim<2>{2, 2}, 4);
+
+    this->mtx5->create_submatrix({0, 2}, {0, 2})
+        ->row_permute(&permute_idxs, row_permute.get());
+
+    GKO_ASSERT_MTX_NEAR(row_permute, l<T>({{-2.0, 2.0}, {1.0, -1.0}}), 0.0);
+    ASSERT_EQ(row_permute->get_stride(), 4);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixRowPermuteIntoDenseFailsForWrongPermutationSize64)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2}};
+    auto row_permute = Mtx::create(exec, this->mtx5->get_size());
+
+    ASSERT_THROW(this->mtx5->row_permute(&permute_idxs, row_permute.get()),
+                 gko::ValueMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixRowPermuteIntoDenseFailsForWrongDimensions64)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+
+    ASSERT_THROW(
+        this->mtx5->row_permute(&permute_idxs, Mtx::create(exec).get()),
+        gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsColPermutable64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+
+    auto c_permute = gko::as<Mtx>(this->mtx5->column_permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(
+        c_permute, l<T>({{-1.0, -0.5, 1.0}, {2.0, 4.5, -2.0}, {3.4, 1.2, 2.1}}),
+        0.0);
+}
+
+
+TYPED_TEST(Dense, NonSquareMatrixIsColPermutable64)
+{
+    // clang-format off
+    // {1.0, 3.0, 2.0},
+    // {0.0, 5.0, 0.0}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx4->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+
+    auto c_permute = gko::as<Mtx>(this->mtx4->column_permute(&permute_idxs));
+
+    GKO_ASSERT_MTX_NEAR(c_permute, l<T>({{3.0, 2.0, 1.0}, {5.0, 0.0, 0.0}}),
+                        0.0);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixIsColPermutableIntoDense64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+    auto c_permute = Mtx::create(exec, this->mtx5->get_size());
+
+    this->mtx5->column_permute(&permute_idxs, c_permute.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        c_permute, l<T>({{-1.0, -0.5, 1.0}, {2.0, 4.5, -2.0}, {3.4, 1.2, 2.1}}),
+        0.0);
+}
+
+
+TYPED_TEST(Dense, SquareSubmatrixIsColPermutableIntoDense64)
+{
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
+    auto c_permute = Mtx::create(exec, gko::dim<2>{2, 2}, 4);
+
+    this->mtx5->create_submatrix({0, 2}, {0, 2})
+        ->column_permute(&permute_idxs, c_permute.get());
+
+    GKO_ASSERT_MTX_NEAR(c_permute, l<T>({{-1.0, 1.0}, {2.0, -2.0}}), 0.0);
+    ASSERT_EQ(c_permute->get_stride(), 4);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixColPermuteIntoDenseFailsForWrongPermutationSize64)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2}};
+    auto row_permute = Mtx::create(exec, this->mtx5->get_size());
+
+    ASSERT_THROW(this->mtx5->column_permute(&permute_idxs, row_permute.get()),
+                 gko::ValueMismatch);
+}
+
+
+TYPED_TEST(Dense, SquareMatrixColPermuteIntoDenseFailsForWrongDimensions64)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = this->mtx5->get_executor();
     gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
 
-    auto ref_permuted = gko::as<Mtx>(
-        gko::as<Mtx>(this->mtx5->inverse_row_permute(&permute_idxs))
-            ->inverse_column_permute(&permute_idxs));
-    auto permuted = gko::as<Mtx>(this->mtx5->inverse_permute(&permute_idxs));
-
-    GKO_ASSERT_MTX_NEAR(permuted, ref_permuted, 0.0);
+    ASSERT_THROW(
+        this->mtx5->column_permute(&permute_idxs, Mtx::create(exec).get()),
+        gko::DimensionMismatch);
 }
 
 
-TYPED_TEST(Dense, SquareMatrixIsRowPermutable)
+TYPED_TEST(Dense, SquareMatrixIsInverseRowPermutable64)
 {
     // clang-format off
     // {1.0, -1.0, -0.5},
@@ -2115,17 +3214,18 @@ TYPED_TEST(Dense, SquareMatrixIsRowPermutable)
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     auto exec = this->mtx5->get_executor();
-    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+    gko::Array<gko::int64> inverse_permute_idxs{exec, {1, 2, 0}};
 
-    auto row_permute = gko::as<Mtx>(this->mtx5->row_permute(&permute_idxs));
+    auto inverse_row_permute =
+        gko::as<Mtx>(this->mtx5->inverse_row_permute(&inverse_permute_idxs));
 
     GKO_ASSERT_MTX_NEAR(
-        row_permute,
-        l<T>({{-2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}, {1.0, -1.0, -0.5}}), 0.0);
+        inverse_row_permute,
+        l<T>({{2.1, 3.4, 1.2}, {1.0, -1.0, -0.5}, {-2.0, 2.0, 4.5}}), 0.0);
 }
 
 
-TYPED_TEST(Dense, NonSquareMatrixIsRowPermutable)
+TYPED_TEST(Dense, NonSquareMatrixIsInverseRowPermutable64)
 {
     // clang-format off
     // {1.0, 3.0, 2.0},
@@ -2134,16 +3234,17 @@ TYPED_TEST(Dense, NonSquareMatrixIsRowPermutable)
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     auto exec = this->mtx4->get_executor();
-    gko::Array<gko::int32> permute_idxs{exec, {1, 0}};
+    gko::Array<gko::int64> inverse_permute_idxs{exec, {1, 0}};
 
-    auto row_permute = gko::as<Mtx>(this->mtx4->row_permute(&permute_idxs));
+    auto inverse_row_permute =
+        gko::as<Mtx>(this->mtx4->inverse_row_permute(&inverse_permute_idxs));
 
-    GKO_ASSERT_MTX_NEAR(row_permute, l<T>({{0.0, 5.0, 0.0}, {1.0, 3.0, 2.0}}),
-                        0.0);
+    GKO_ASSERT_MTX_NEAR(inverse_row_permute,
+                        l<T>({{0.0, 5.0, 0.0}, {1.0, 3.0, 2.0}}), 0.0);
 }
 
 
-TYPED_TEST(Dense, SquareMatrixIsColPermutable)
+TYPED_TEST(Dense, SquareMatrixIsInverseRowPermutableIntoDense64)
 {
     // clang-format off
     // {1.0, -1.0, -0.5},
@@ -2153,75 +3254,68 @@ TYPED_TEST(Dense, SquareMatrixIsColPermutable)
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     auto exec = this->mtx5->get_executor();
-    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+    auto row_permute = Mtx::create(exec, this->mtx5->get_size());
 
-    auto c_permute = gko::as<Mtx>(this->mtx5->column_permute(&permute_idxs));
+    this->mtx5->inverse_row_permute(&permute_idxs, row_permute.get());
 
     GKO_ASSERT_MTX_NEAR(
-        c_permute, l<T>({{-1.0, -0.5, 1.0}, {2.0, 4.5, -2.0}, {3.4, 1.2, 2.1}}),
-        0.0);
+        row_permute,
+        l<T>({{2.1, 3.4, 1.2}, {1.0, -1.0, -0.5}, {-2.0, 2.0, 4.5}}), 0.0);
 }
 
 
-TYPED_TEST(Dense, NonSquareMatrixIsColPermutable)
+TYPED_TEST(Dense, SquareSubmatrixIsInverseRowPermutableIntoDense64)
 {
     // clang-format off
-    // {1.0, 3.0, 2.0},
-    // {0.0, 5.0, 0.0}
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
-    auto exec = this->mtx4->get_executor();
-    gko::Array<gko::int32> permute_idxs{exec, {1, 2, 0}};
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
+    auto row_permute = Mtx::create(exec, gko::dim<2>{2, 2}, 4);
 
-    auto c_permute = gko::as<Mtx>(this->mtx4->column_permute(&permute_idxs));
+    this->mtx5->create_submatrix({0, 2}, {0, 2})
+        ->inverse_row_permute(&permute_idxs, row_permute.get());
 
-    GKO_ASSERT_MTX_NEAR(c_permute, l<T>({{3.0, 2.0, 1.0}, {5.0, 0.0, 0.0}}),
-                        0.0);
+    GKO_ASSERT_MTX_NEAR(row_permute, l<T>({{-2.0, 2.0}, {1.0, -1.0}}), 0.0);
+    ASSERT_EQ(row_permute->get_stride(), 4);
 }
 
 
-TYPED_TEST(Dense, SquareMatrixIsInverseRowPermutable)
+TYPED_TEST(Dense,
+           SquareMatrixInverseRowPermuteIntoDenseFailsForWrongPermutationSize64)
 {
-    // clang-format off
-    // {1.0, -1.0, -0.5},
-    // {-2.0, 2.0, 4.5},
-    // {2.1, 3.4, 1.2}
-    // clang-format on
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     auto exec = this->mtx5->get_executor();
-    gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 2, 0}};
-
-    auto inverse_row_permute =
-        gko::as<Mtx>(this->mtx5->inverse_row_permute(&inverse_permute_idxs));
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2}};
+    auto row_permute = Mtx::create(exec, this->mtx5->get_size());
 
-    GKO_ASSERT_MTX_NEAR(
-        inverse_row_permute,
-        l<T>({{2.1, 3.4, 1.2}, {1.0, -1.0, -0.5}, {-2.0, 2.0, 4.5}}), 0.0);
+    ASSERT_THROW(
+        this->mtx5->inverse_row_permute(&permute_idxs, row_permute.get()),
+        gko::ValueMismatch);
 }
 
 
-TYPED_TEST(Dense, NonSquareMatrixIsInverseRowPermutable)
+TYPED_TEST(Dense,
+           SquareMatrixInverseRowPermuteIntoDenseFailsForWrongDimensions64)
 {
-    // clang-format off
-    // {1.0, 3.0, 2.0},
-    // {0.0, 5.0, 0.0}
-    // clang-format on
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
-    auto exec = this->mtx4->get_executor();
-    gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 0}};
-
-    auto inverse_row_permute =
-        gko::as<Mtx>(this->mtx4->inverse_row_permute(&inverse_permute_idxs));
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
 
-    GKO_ASSERT_MTX_NEAR(inverse_row_permute,
-                        l<T>({{0.0, 5.0, 0.0}, {1.0, 3.0, 2.0}}), 0.0);
+    ASSERT_THROW(
+        this->mtx5->inverse_row_permute(&permute_idxs, Mtx::create(exec).get()),
+        gko::DimensionMismatch);
 }
 
 
-TYPED_TEST(Dense, SquareMatrixIsInverseColPermutable)
+TYPED_TEST(Dense, SquareMatrixIsInverseColPermutable64)
 {
     // clang-format off
     // {1.0, -1.0, -0.5},
@@ -2231,7 +3325,7 @@ TYPED_TEST(Dense, SquareMatrixIsInverseColPermutable)
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     auto exec = this->mtx5->get_executor();
-    gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 2, 0}};
+    gko::Array<gko::int64> inverse_permute_idxs{exec, {1, 2, 0}};
 
     auto inverse_c_permute =
         gko::as<Mtx>(this->mtx5->inverse_column_permute(&inverse_permute_idxs));
@@ -2242,7 +3336,7 @@ TYPED_TEST(Dense, SquareMatrixIsInverseColPermutable)
 }
 
 
-TYPED_TEST(Dense, NonSquareMatrixIsInverseColPermutable)
+TYPED_TEST(Dense, NonSquareMatrixIsInverseColPermutable64)
 {
     // clang-format off
     // {1.0, 3.0, 2.0},
@@ -2251,7 +3345,7 @@ TYPED_TEST(Dense, NonSquareMatrixIsInverseColPermutable)
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
     auto exec = this->mtx4->get_executor();
-    gko::Array<gko::int32> inverse_permute_idxs{exec, {1, 2, 0}};
+    gko::Array<gko::int64> inverse_permute_idxs{exec, {1, 2, 0}};
 
     auto inverse_c_permute =
         gko::as<Mtx>(this->mtx4->inverse_column_permute(&inverse_permute_idxs));
@@ -2261,77 +3355,74 @@ TYPED_TEST(Dense, NonSquareMatrixIsInverseColPermutable)
 }
 
 
-TYPED_TEST(Dense, NonSquareMatrixIsRowPermutable64)
+TYPED_TEST(Dense, SquareMatrixIsInverseColPermutableIntoDense64)
 {
     // clang-format off
-    // {1.0, 3.0, 2.0},
-    // {0.0, 5.0, 0.0}
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
-    auto exec = this->mtx4->get_executor();
-    gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+    auto c_permute = Mtx::create(exec, this->mtx5->get_size());
 
-    auto row_permute = gko::as<Mtx>(this->mtx4->row_permute(&permute_idxs));
+    this->mtx5->inverse_column_permute(&permute_idxs, c_permute.get());
 
-    GKO_ASSERT_MTX_NEAR(row_permute, l<T>({{0.0, 5.0, 0.0}, {1.0, 3.0, 2.0}}),
-                        0.0);
+    GKO_ASSERT_MTX_NEAR(
+        c_permute, l<T>({{-0.5, 1.0, -1.0}, {4.5, -2.0, 2.0}, {1.2, 2.1, 3.4}}),
+        0.0);
 }
 
 
-TYPED_TEST(Dense, NonSquareMatrixIsColPermutable64)
+TYPED_TEST(Dense, SquareSubmatrixIsInverseColPermutableIntoDense64)
 {
     // clang-format off
-    // {1.0, 3.0, 2.0},
-    // {0.0, 5.0, 0.0}
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
     // clang-format on
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
-    auto exec = this->mtx4->get_executor();
-    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 0}};
+    auto c_permute = Mtx::create(exec, gko::dim<2>{2, 2}, 4);
 
-    auto c_permute = gko::as<Mtx>(this->mtx4->column_permute(&permute_idxs));
+    this->mtx5->create_submatrix({0, 2}, {0, 2})
+        ->column_permute(&permute_idxs, c_permute.get());
 
-    GKO_ASSERT_MTX_NEAR(c_permute, l<T>({{3.0, 2.0, 1.0}, {5.0, 0.0, 0.0}}),
-                        0.0);
+    GKO_ASSERT_MTX_NEAR(c_permute, l<T>({{-1.0, 1.0}, {2.0, -2.0}}), 0.0);
+    ASSERT_EQ(c_permute->get_stride(), 4);
 }
 
 
-TYPED_TEST(Dense, NonSquareMatrixIsInverseRowPermutable64)
+TYPED_TEST(Dense,
+           SquareMatrixInverseColPermuteIntoDenseFailsForWrongPermutationSize64)
 {
-    // clang-format off
-    // {1.0, 3.0, 2.0},
-    // {0.0, 5.0, 0.0}
-    // clang-format on
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
-    auto exec = this->mtx4->get_executor();
-    gko::Array<gko::int64> inverse_permute_idxs{exec, {1, 0}};
-
-    auto inverse_row_permute =
-        gko::as<Mtx>(this->mtx4->inverse_row_permute(&inverse_permute_idxs));
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2}};
+    auto row_permute = Mtx::create(exec, this->mtx5->get_size());
 
-    GKO_ASSERT_MTX_NEAR(inverse_row_permute,
-                        l<T>({{0.0, 5.0, 0.0}, {1.0, 3.0, 2.0}}), 0.0);
+    ASSERT_THROW(
+        this->mtx5->inverse_column_permute(&permute_idxs, row_permute.get()),
+        gko::ValueMismatch);
 }
 
 
-TYPED_TEST(Dense, NonSquareMatrixIsInverseColPermutable64)
+TYPED_TEST(Dense,
+           SquareMatrixInverseColPermuteIntoDenseFailsForWrongDimensions64)
 {
-    // clang-format off
-    // {1.0, 3.0, 2.0},
-    // {0.0, 5.0, 0.0}
-    // clang-format on
     using Mtx = typename TestFixture::Mtx;
     using T = typename TestFixture::value_type;
-    auto exec = this->mtx4->get_executor();
-    gko::Array<gko::int64> inverse_permute_idxs{exec, {1, 2, 0}};
-
-    auto inverse_c_permute =
-        gko::as<Mtx>(this->mtx4->inverse_column_permute(&inverse_permute_idxs));
+    auto exec = this->mtx5->get_executor();
+    gko::Array<gko::int64> permute_idxs{exec, {1, 2, 0}};
 
-    GKO_ASSERT_MTX_NEAR(inverse_c_permute,
-                        l<T>({{2.0, 1.0, 3.0}, {0.0, 0.0, 5.0}}), 0.0);
+    ASSERT_THROW(this->mtx5->inverse_column_permute(&permute_idxs,
+                                                    Mtx::create(exec).get()),
+                 gko::DimensionMismatch);
 }
 
 
@@ -2389,6 +3480,66 @@ TYPED_TEST(Dense, ExtractsDiagonalFromShortFatMatrix)
 }
 
 
+TYPED_TEST(Dense, ExtractsDiagonalFromSquareMatrixIntoDiagonal)
+{
+    using T = typename TestFixture::value_type;
+
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    auto diag = gko::matrix::Diagonal<T>::create(this->exec, 3);
+
+    this->mtx5->extract_diagonal(diag.get());
+
+    ASSERT_EQ(diag->get_size()[0], 3);
+    ASSERT_EQ(diag->get_size()[1], 3);
+    ASSERT_EQ(diag->get_values()[0], T{1.});
+    ASSERT_EQ(diag->get_values()[1], T{2.});
+    ASSERT_EQ(diag->get_values()[2], T{1.2});
+}
+
+
+TYPED_TEST(Dense, ExtractsDiagonalFromTallSkinnyMatrixIntoDiagonal)
+{
+    using T = typename TestFixture::value_type;
+
+    // clang-format off
+    // {1.0, 3.0, 2.0},
+    // {0.0, 5.0, 0.0}
+    // clang-format on
+    auto diag = gko::matrix::Diagonal<T>::create(this->exec, 2);
+
+    this->mtx4->extract_diagonal(diag.get());
+
+    ASSERT_EQ(diag->get_size()[0], 2);
+    ASSERT_EQ(diag->get_size()[1], 2);
+    ASSERT_EQ(diag->get_values()[0], T{1.});
+    ASSERT_EQ(diag->get_values()[1], T{5.});
+}
+
+
+TYPED_TEST(Dense, ExtractsDiagonalFromShortFatMatrixIntoDiagonal)
+{
+    using T = typename TestFixture::value_type;
+
+    // clang-format off
+    // { 1.0, -1.0},
+    // {-2.0,  2.0},
+    // {-3.0,  3.0}
+    // clang-format on
+    auto diag = gko::matrix::Diagonal<T>::create(this->exec, 2);
+
+    this->mtx8->extract_diagonal(diag.get());
+
+    ASSERT_EQ(diag->get_size()[0], 2);
+    ASSERT_EQ(diag->get_size()[1], 2);
+    ASSERT_EQ(diag->get_values()[0], T{1.});
+    ASSERT_EQ(diag->get_values()[1], T{2.});
+}
+
+
 TYPED_TEST(Dense, InplaceAbsolute)
 {
     using T = typename TestFixture::value_type;
@@ -2442,6 +3593,26 @@ TYPED_TEST(Dense, OutplaceAbsolute)
 }
 
 
+TYPED_TEST(Dense, OutplaceAbsoluteIntoDense)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    auto abs_mtx =
+        gko::remove_complex<Mtx>::create(this->exec, this->mtx5->get_size());
+
+    this->mtx5->compute_absolute(abs_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l<T>({{1.0, 1.0, 0.5}, {2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}),
+        0.0);
+}
+
+
 TYPED_TEST(Dense, OutplaceAbsoluteSubMatrix)
 {
     using T = typename TestFixture::value_type;
@@ -2460,6 +3631,26 @@ TYPED_TEST(Dense, OutplaceAbsoluteSubMatrix)
 }
 
 
+TYPED_TEST(Dense, OutplaceSubmatrixAbsoluteIntoDense)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    // clang-format off
+    // {1.0, -1.0, -0.5},
+    // {-2.0, 2.0, 4.5},
+    // {2.1, 3.4, 1.2}
+    // clang-format on
+    auto mtx = this->mtx5->create_submatrix(gko::span{0, 2}, gko::span{0, 2});
+    auto abs_mtx =
+        gko::remove_complex<Mtx>::create(this->exec, gko::dim<2>{2, 2}, 4);
+
+    mtx->compute_absolute(abs_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx, l<T>({{1.0, 1.0}, {2.0, 2.0}}), 0);
+    GKO_ASSERT_EQ(abs_mtx->get_stride(), 4);
+}
+
+
 TYPED_TEST(Dense, AppliesToComplex)
 {
     using value_type = typename TestFixture::value_type;
@@ -2799,6 +3990,26 @@ TYPED_TEST(DenseComplex, NonSquareMatrixIsConjugateTransposable)
 }
 
 
+TYPED_TEST(DenseComplex, NonSquareMatrixIsConjugateTransposableIntoDense)
+{
+    using Dense = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = gko::ReferenceExecutor::create();
+    auto mtx = gko::initialize<Dense>({{T{1.0, 2.0}, T{-1.0, 2.1}},
+                                       {T{-2.0, 1.5}, T{4.5, 0.0}},
+                                       {T{1.0, 0.0}, T{0.0, 1.0}}},
+                                      exec);
+    auto trans = Dense::create(exec, gko::transpose(mtx->get_size()));
+
+    mtx->conj_transpose(trans.get());
+
+    GKO_ASSERT_MTX_NEAR(trans,
+                        l<T>({{T{1.0, -2.0}, T{-2.0, -1.5}, T{1.0, 0.0}},
+                              {T{-1.0, -2.1}, T{4.5, 0.0}, T{0.0, -1.0}}}),
+                        0.0);
+}
+
+
 TYPED_TEST(DenseComplex, InplaceAbsolute)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -2834,6 +4045,25 @@ TYPED_TEST(DenseComplex, OutplaceAbsolute)
 }
 
 
+TYPED_TEST(DenseComplex, OutplaceAbsoluteIntoDense)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto exec = gko::ReferenceExecutor::create();
+    auto mtx = gko::initialize<Mtx>({{T{1.0, 0.0}, T{3.0, 4.0}, T{0.0, 2.0}},
+                                     {T{-4.0, -3.0}, T{-1.0, 0}, T{0.0, 0.0}},
+                                     {T{0.0, 0.0}, T{0.0, -1.5}, T{2.0, 0.0}}},
+                                    exec);
+    auto abs_mtx = gko::remove_complex<Mtx>::create(exec, mtx->get_size());
+
+    mtx->compute_absolute(abs_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        abs_mtx, l<T>({{1.0, 5.0, 2.0}, {5.0, 1.0, 0.0}, {0.0, 1.5, 2.0}}),
+        0.0);
+}
+
+
 TYPED_TEST(DenseComplex, MakeComplex)
 {
     using Mtx = typename TestFixture::Mtx;

From 065473719a4efa6fe023693634137a1a5f36916f Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 28 May 2021 11:42:10 +0200
Subject: [PATCH 0733/1008] add device tests for Dense output functions

---
 cuda/test/matrix/dense_kernels.cpp    | 239 ++++++++++++++++++++++----
 hip/test/matrix/dense_kernels.hip.cpp | 195 ++++++++++++++++++---
 omp/test/matrix/dense_kernels.cpp     | 222 ++++++++++++++++++++++--
 3 files changed, 588 insertions(+), 68 deletions(-)

diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index 7ff0efb4a3b..8e509017def 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -65,6 +65,7 @@ class Dense : public ::testing::Test {
     using NormVector = gko::matrix::Dense<gko::remove_complex<vtype>>;
     using Arr = gko::Array<itype>;
     using ComplexMtx = gko::matrix::Dense<std::complex<vtype>>;
+    using Diagonal = gko::matrix::Diagonal<vtype>;
     using MixedComplexMtx =
         gko::matrix::Dense<gko::next_precision<std::complex<vtype>>>;
 
@@ -540,30 +541,6 @@ TEST_F(Dense, ComputeConjDotComplexIsEquivalentToRef)
 }
 
 
-TEST_F(Dense, IsTransposable)
-{
-    set_up_apply_data();
-
-    auto trans = x->transpose();
-    auto dtrans = dx->transpose();
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(dtrans.get()),
-                        static_cast<Mtx *>(trans.get()), 0);
-}
-
-
-TEST_F(Dense, IsConjugateTransposable)
-{
-    set_up_apply_data();
-
-    auto trans = c_x->conj_transpose();
-    auto dtrans = dc_x->conj_transpose();
-
-    GKO_ASSERT_MTX_NEAR(static_cast<ComplexMtx *>(dtrans.get()),
-                        static_cast<ComplexMtx *>(trans.get()), 0);
-}
-
-
 TEST_F(Dense, ConvertToCooIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -747,6 +724,70 @@ TEST_F(Dense, CalculateTotalColsIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, IsTransposable)
+{
+    set_up_apply_data();
+
+    auto trans = x->transpose();
+    auto dtrans = dx->transpose();
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(dtrans.get()),
+                        static_cast<Mtx *>(trans.get()), 0);
+}
+
+
+TEST_F(Dense, IsTransposableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto row_span = gko::span{0, x->get_size()[0] - 2};
+    auto col_span = gko::span{0, x->get_size()[1] - 2};
+    auto sub_x = x->create_submatrix(row_span, col_span);
+    auto sub_dx = dx->create_submatrix(row_span, col_span);
+    // create the target matrices on another executor to
+    // force temporary clone
+    auto trans = Mtx::create(ref, gko::transpose(sub_x->get_size()));
+    auto dtrans = Mtx::create(ref, gko::transpose(sub_x->get_size()),
+                              sub_x->get_size()[0] + 4);
+
+    sub_x->transpose(trans.get());
+    sub_dx->transpose(dtrans.get());
+
+    GKO_ASSERT_MTX_NEAR(dtrans, trans, 0);
+}
+
+
+TEST_F(Dense, IsConjugateTransposable)
+{
+    set_up_apply_data();
+
+    auto trans = c_x->conj_transpose();
+    auto dtrans = dc_x->conj_transpose();
+
+    GKO_ASSERT_MTX_NEAR(static_cast<ComplexMtx *>(dtrans.get()),
+                        static_cast<ComplexMtx *>(trans.get()), 0);
+}
+
+
+TEST_F(Dense, IsConjugateTransposableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto row_span = gko::span{0, c_x->get_size()[0] - 2};
+    auto col_span = gko::span{0, c_x->get_size()[1] - 2};
+    auto sub_x = c_x->create_submatrix(row_span, col_span);
+    auto sub_dx = dc_x->create_submatrix(row_span, col_span);
+    // create the target matrices on another executor to
+    // force temporary clone
+    auto trans = ComplexMtx::create(ref, gko::transpose(sub_x->get_size()));
+    auto dtrans = ComplexMtx::create(ref, gko::transpose(sub_x->get_size()),
+                                     sub_x->get_size()[0] + 4);
+
+    sub_x->conj_transpose(trans.get());
+    sub_dx->conj_transpose(dtrans.get());
+
+    GKO_ASSERT_MTX_NEAR(dtrans, trans, 0);
+}
+
+
 TEST_F(Dense, CanGatherRows)
 {
     set_up_apply_data();
@@ -786,6 +827,21 @@ TEST_F(Dense, IsPermutable)
 }
 
 
+TEST_F(Dense, IsPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, square->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted =
+        Mtx::create(ref, square->get_size(), square->get_size()[1] + 2);
+
+    square->permute(rpermute_idxs.get(), permuted.get());
+    dsquare->permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsInversePermutable)
 {
     set_up_apply_data();
@@ -798,6 +854,21 @@ TEST_F(Dense, IsInversePermutable)
 }
 
 
+TEST_F(Dense, IsInversePermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, square->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted =
+        Mtx::create(ref, square->get_size(), square->get_size()[1] + 2);
+
+    square->inverse_permute(rpermute_idxs.get(), permuted.get());
+    dsquare->inverse_permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsRowPermutable)
 {
     set_up_apply_data();
@@ -810,6 +881,20 @@ TEST_F(Dense, IsRowPermutable)
 }
 
 
+TEST_F(Dense, IsRowPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->row_permute(rpermute_idxs.get(), permuted.get());
+    dx->row_permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsColPermutable)
 {
     set_up_apply_data();
@@ -822,6 +907,20 @@ TEST_F(Dense, IsColPermutable)
 }
 
 
+TEST_F(Dense, IsColPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->column_permute(cpermute_idxs.get(), permuted.get());
+    dx->column_permute(cpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsInverseRowPermutable)
 {
     set_up_apply_data();
@@ -834,6 +933,20 @@ TEST_F(Dense, IsInverseRowPermutable)
 }
 
 
+TEST_F(Dense, IsInverseRowPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->inverse_row_permute(rpermute_idxs.get(), permuted.get());
+    dx->inverse_row_permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsInverseColPermutable)
 {
     set_up_apply_data();
@@ -846,6 +959,20 @@ TEST_F(Dense, IsInverseColPermutable)
 }
 
 
+TEST_F(Dense, IsInverseColPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->inverse_column_permute(cpermute_idxs.get(), permuted.get());
+    dx->inverse_column_permute(cpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -857,6 +984,20 @@ TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ExtractDiagonalOnTallSkinnyIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto diag = Diagonal::create(ref, x->get_size()[1]);
+    // test make_temporary_clone
+    auto ddiag = Diagonal::create(ref, x->get_size()[1]);
+
+    x->extract_diagonal(diag.get());
+    dx->extract_diagonal(ddiag.get());
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
 TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -868,6 +1009,20 @@ TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ExtractDiagonalOnShortFatIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto diag = Diagonal::create(ref, y->get_size()[0]);
+    // test make_temporary_clone
+    auto ddiag = Diagonal::create(ref, y->get_size()[0]);
+
+    y->extract_diagonal(diag.get());
+    dy->extract_diagonal(ddiag.get());
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
 TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -890,6 +1045,20 @@ TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, OutplaceAbsoluteMatrixIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto abs_x = NormVector::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dabs_x = NormVector::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->compute_absolute(abs_x.get());
+    dx->compute_absolute(dabs_x.get());
+
+    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, 1e-14);
+}
+
+
 TEST_F(Dense, MakeComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -901,13 +1070,15 @@ TEST_F(Dense, MakeComplexIsEquivalentToRef)
 }
 
 
-TEST_F(Dense, MakeComplexWithGivenResultIsEquivalentToRef)
+TEST_F(Dense, MakeComplexIntoDenseCrossExecutor)
 {
     set_up_apply_data();
-
     auto complex_x = ComplexMtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dcomplex_x =
+        ComplexMtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
     x->make_complex(complex_x.get());
-    auto dcomplex_x = ComplexMtx::create(cuda, x->get_size());
     dx->make_complex(dcomplex_x.get());
 
     GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
@@ -925,13 +1096,14 @@ TEST_F(Dense, GetRealIsEquivalentToRef)
 }
 
 
-TEST_F(Dense, GetRealWithGivenResultIsEquivalentToRef)
+TEST_F(Dense, GetRealIntoDenseCrossExecutor)
 {
     set_up_apply_data();
-
     auto real_x = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dreal_x = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
     x->get_real(real_x.get());
-    auto dreal_x = Mtx::create(cuda, dx->get_size());
     dx->get_real(dreal_x.get());
 
     GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
@@ -949,13 +1121,14 @@ TEST_F(Dense, GetImagIsEquivalentToRef)
 }
 
 
-TEST_F(Dense, GetImagWithGivenResultIsEquivalentToRef)
+TEST_F(Dense, GetImagIntoDenseCrossExecutor)
 {
     set_up_apply_data();
-
     auto imag_x = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dimag_x = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
     x->get_imag(imag_x.get());
-    auto dimag_x = Mtx::create(cuda, dx->get_size());
     dx->get_imag(dimag_x.get());
 
     GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index 82a18634af4..0fce6afe9a3 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -65,6 +65,7 @@ class Dense : public ::testing::Test {
     using NormVector = gko::matrix::Dense<gko::remove_complex<vtype>>;
     using Arr = gko::Array<itype>;
     using ComplexMtx = gko::matrix::Dense<std::complex<vtype>>;
+    using Diagonal = gko::matrix::Diagonal<vtype>;
     using MixedComplexMtx =
         gko::matrix::Dense<gko::next_precision<std::complex<vtype>>>;
 
@@ -535,18 +536,6 @@ TEST_F(Dense, ComputeConjDotComplexIsEquivalentToRef)
 }
 
 
-TEST_F(Dense, IsTransposable)
-{
-    set_up_apply_data();
-
-    auto trans = x->transpose();
-    auto dtrans = dx->transpose();
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(dtrans.get()),
-                        static_cast<Mtx *>(trans.get()), 0);
-}
-
-
 TEST_F(Dense, ConvertToCooIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -730,6 +719,38 @@ TEST_F(Dense, CalculateTotalColsIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, IsTransposable)
+{
+    set_up_apply_data();
+
+    auto trans = x->transpose();
+    auto dtrans = dx->transpose();
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(dtrans.get()),
+                        static_cast<Mtx *>(trans.get()), 0);
+}
+
+
+TEST_F(Dense, IsTransposableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto row_span = gko::span{0, x->get_size()[0] - 2};
+    auto col_span = gko::span{0, x->get_size()[1] - 2};
+    auto sub_x = x->create_submatrix(row_span, col_span);
+    auto sub_dx = dx->create_submatrix(row_span, col_span);
+    // create the target matrices on another executor to
+    // force temporary clone
+    auto trans = Mtx::create(ref, gko::transpose(sub_x->get_size()));
+    auto dtrans = Mtx::create(ref, gko::transpose(sub_x->get_size()),
+                              sub_x->get_size()[0] + 4);
+
+    sub_x->transpose(trans.get());
+    sub_dx->transpose(dtrans.get());
+
+    GKO_ASSERT_MTX_NEAR(dtrans, trans, 0);
+}
+
+
 TEST_F(Dense, CanGatherRows)
 {
     set_up_apply_data();
@@ -769,6 +790,21 @@ TEST_F(Dense, IsPermutable)
 }
 
 
+TEST_F(Dense, IsPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, square->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted =
+        Mtx::create(ref, square->get_size(), square->get_size()[1] + 2);
+
+    square->permute(rpermute_idxs.get(), permuted.get());
+    dsquare->permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsInversePermutable)
 {
     set_up_apply_data();
@@ -781,6 +817,21 @@ TEST_F(Dense, IsInversePermutable)
 }
 
 
+TEST_F(Dense, IsInversePermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, square->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted =
+        Mtx::create(ref, square->get_size(), square->get_size()[1] + 2);
+
+    square->inverse_permute(rpermute_idxs.get(), permuted.get());
+    dsquare->inverse_permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsRowPermutable)
 {
     set_up_apply_data();
@@ -793,6 +844,20 @@ TEST_F(Dense, IsRowPermutable)
 }
 
 
+TEST_F(Dense, IsRowPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->row_permute(rpermute_idxs.get(), permuted.get());
+    dx->row_permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsColPermutable)
 {
     set_up_apply_data();
@@ -805,6 +870,20 @@ TEST_F(Dense, IsColPermutable)
 }
 
 
+TEST_F(Dense, IsColPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->column_permute(cpermute_idxs.get(), permuted.get());
+    dx->column_permute(cpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsInverseRowPermutable)
 {
     set_up_apply_data();
@@ -817,6 +896,20 @@ TEST_F(Dense, IsInverseRowPermutable)
 }
 
 
+TEST_F(Dense, IsInverseRowPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->inverse_row_permute(rpermute_idxs.get(), permuted.get());
+    dx->inverse_row_permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsInverseColPermutable)
 {
     set_up_apply_data();
@@ -829,6 +922,20 @@ TEST_F(Dense, IsInverseColPermutable)
 }
 
 
+TEST_F(Dense, IsInverseColPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->inverse_column_permute(cpermute_idxs.get(), permuted.get());
+    dx->inverse_column_permute(cpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -840,6 +947,20 @@ TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ExtractDiagonalOnTallSkinnyIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto diag = Diagonal::create(ref, x->get_size()[1]);
+    // test make_temporary_clone
+    auto ddiag = Diagonal::create(ref, x->get_size()[1]);
+
+    x->extract_diagonal(diag.get());
+    dx->extract_diagonal(ddiag.get());
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
 TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -851,6 +972,20 @@ TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ExtractDiagonalOnShortFatIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto diag = Diagonal::create(ref, y->get_size()[0]);
+    // test make_temporary_clone
+    auto ddiag = Diagonal::create(ref, y->get_size()[0]);
+
+    y->extract_diagonal(diag.get());
+    dy->extract_diagonal(ddiag.get());
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
 TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -873,6 +1008,20 @@ TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, OutplaceAbsoluteMatrixIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto abs_x = NormVector::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dabs_x = NormVector::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->compute_absolute(abs_x.get());
+    dx->compute_absolute(dabs_x.get());
+
+    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, 1e-14);
+}
+
+
 TEST_F(Dense, MakeComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -884,13 +1033,15 @@ TEST_F(Dense, MakeComplexIsEquivalentToRef)
 }
 
 
-TEST_F(Dense, MakeComplexWithGivenResultIsEquivalentToRef)
+TEST_F(Dense, MakeComplexIntoDenseCrossExecutor)
 {
     set_up_apply_data();
-
     auto complex_x = ComplexMtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dcomplex_x =
+        ComplexMtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
     x->make_complex(complex_x.get());
-    auto dcomplex_x = ComplexMtx::create(hip, x->get_size());
     dx->make_complex(dcomplex_x.get());
 
     GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
@@ -908,13 +1059,14 @@ TEST_F(Dense, GetRealIsEquivalentToRef)
 }
 
 
-TEST_F(Dense, GetRealWithGivenResultIsEquivalentToRef)
+TEST_F(Dense, GetRealIntoDenseCrossExecutor)
 {
     set_up_apply_data();
-
     auto real_x = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dreal_x = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
     x->get_real(real_x.get());
-    auto dreal_x = Mtx::create(hip, dx->get_size());
     dx->get_real(dreal_x.get());
 
     GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
@@ -932,13 +1084,14 @@ TEST_F(Dense, GetImagIsEquivalentToRef)
 }
 
 
-TEST_F(Dense, GetImagWithGivenResultIsEquivalentToRef)
+TEST_F(Dense, GetImagIntoDenseCrossExecutor)
 {
     set_up_apply_data();
-
     auto imag_x = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dimag_x = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
     x->get_imag(imag_x.get());
-    auto dimag_x = Mtx::create(hip, dx->get_size());
     dx->get_imag(dimag_x.get());
 
     GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index 49d553fbfc3..29a24fa37b7 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -70,6 +70,7 @@ class Dense : public ::testing::Test {
     using NormVector = gko::matrix::Dense<gko::remove_complex<vtype>>;
     using Arr = gko::Array<itype>;
     using ComplexMtx = gko::matrix::Dense<std::complex<vtype>>;
+    using Diagonal = gko::matrix::Diagonal<vtype>;
     using MixedComplexMtx =
         gko::matrix::Dense<gko::next_precision<std::complex<vtype>>>;
 
@@ -810,6 +811,23 @@ TEST_F(Dense, ConvertsEmptyToSellp)
 }
 
 
+TEST_F(Dense, CalculateNNZPerRowIsEquivalentToRef)
+{
+    set_up_apply_data();
+    gko::Array<gko::size_type> nnz_per_row(ref);
+    nnz_per_row.resize_and_reset(x->get_size()[0]);
+    gko::Array<gko::size_type> dnnz_per_row(omp);
+    dnnz_per_row.resize_and_reset(dx->get_size()[0]);
+
+    gko::kernels::reference::dense::calculate_nonzeros_per_row(ref, x.get(),
+                                                               &nnz_per_row);
+    gko::kernels::omp::dense::calculate_nonzeros_per_row(omp, dx.get(),
+                                                         &dnnz_per_row);
+
+    GKO_ASSERT_ARRAY_EQ(nnz_per_row, dnnz_per_row);
+}
+
+
 TEST_F(Dense, CalculateMaxNNZPerRowIsEquivalentToRef)
 {
     std::size_t ref_max_nnz_per_row = 0;
@@ -856,6 +874,26 @@ TEST_F(Dense, IsTransposable)
 }
 
 
+TEST_F(Dense, IsTransposableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto row_span = gko::span{0, x->get_size()[0] - 2};
+    auto col_span = gko::span{0, x->get_size()[1] - 2};
+    auto sub_x = x->create_submatrix(row_span, col_span);
+    auto sub_dx = dx->create_submatrix(row_span, col_span);
+    // create the target matrices on another executor to
+    // force temporary clone
+    auto trans = Mtx::create(ref, gko::transpose(sub_x->get_size()));
+    auto dtrans = Mtx::create(ref, gko::transpose(sub_x->get_size()),
+                              sub_x->get_size()[0] + 4);
+
+    sub_x->transpose(trans.get());
+    sub_dx->transpose(dtrans.get());
+
+    GKO_ASSERT_MTX_NEAR(dtrans, trans, 0);
+}
+
+
 TEST_F(Dense, IsConjugateTransposable)
 {
     set_up_apply_data();
@@ -868,6 +906,26 @@ TEST_F(Dense, IsConjugateTransposable)
 }
 
 
+TEST_F(Dense, IsConjugateTransposableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto row_span = gko::span{0, c_x->get_size()[0] - 2};
+    auto col_span = gko::span{0, c_x->get_size()[1] - 2};
+    auto sub_x = c_x->create_submatrix(row_span, col_span);
+    auto sub_dx = dc_x->create_submatrix(row_span, col_span);
+    // create the target matrices on another executor to
+    // force temporary clone
+    auto trans = ComplexMtx::create(ref, gko::transpose(sub_x->get_size()));
+    auto dtrans = ComplexMtx::create(ref, gko::transpose(sub_x->get_size()),
+                                     sub_x->get_size()[0] + 4);
+
+    sub_x->conj_transpose(trans.get());
+    sub_dx->conj_transpose(dtrans.get());
+
+    GKO_ASSERT_MTX_NEAR(dtrans, trans, 0);
+}
+
+
 TEST_F(Dense, CanGatherRows)
 {
     set_up_apply_data();
@@ -879,17 +937,21 @@ TEST_F(Dense, CanGatherRows)
 }
 
 
-TEST_F(Dense, CanGatherRowsIntoDense)
+TEST_F(Dense, CanGatherRowsIntoDenseCrossExecutor)
 {
     set_up_apply_data();
+    auto row_span = gko::span{0, x->get_size()[0]};
+    auto col_span = gko::span{0, x->get_size()[1] - 2};
+    auto sub_x = x->create_submatrix(row_span, col_span);
+    auto sub_dx = dx->create_submatrix(row_span, col_span);
     auto gather_size =
-        gko::dim<2>{rgather_idxs->get_num_elems(), x->get_size()[1]};
+        gko::dim<2>{rgather_idxs->get_num_elems(), sub_x->get_size()[1]};
     auto r_gather = Mtx::create(ref, gather_size);
     // test make_temporary_clone and non-default stride
-    auto dr_gather = Mtx::create(ref, gather_size, x->get_size()[1] + 2);
+    auto dr_gather = Mtx::create(ref, gather_size, sub_x->get_size()[1] + 2);
 
-    x->row_gather(rgather_idxs.get(), r_gather.get());
-    dx->row_gather(rgather_idxs.get(), dr_gather.get());
+    sub_x->row_gather(rgather_idxs.get(), r_gather.get());
+    sub_dx->row_gather(rgather_idxs.get(), dr_gather.get());
 
     GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
 }
@@ -907,6 +969,21 @@ TEST_F(Dense, IsPermutable)
 }
 
 
+TEST_F(Dense, IsPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, square->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted =
+        Mtx::create(ref, square->get_size(), square->get_size()[1] + 2);
+
+    square->permute(rpermute_idxs.get(), permuted.get());
+    dsquare->permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsInversePermutable)
 {
     set_up_apply_data();
@@ -919,6 +996,21 @@ TEST_F(Dense, IsInversePermutable)
 }
 
 
+TEST_F(Dense, IsInversePermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, square->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted =
+        Mtx::create(ref, square->get_size(), square->get_size()[1] + 2);
+
+    square->inverse_permute(rpermute_idxs.get(), permuted.get());
+    dsquare->inverse_permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsRowPermutable)
 {
     set_up_apply_data();
@@ -931,6 +1023,20 @@ TEST_F(Dense, IsRowPermutable)
 }
 
 
+TEST_F(Dense, IsRowPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->row_permute(rpermute_idxs.get(), permuted.get());
+    dx->row_permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsColPermutable)
 {
     set_up_apply_data();
@@ -943,6 +1049,20 @@ TEST_F(Dense, IsColPermutable)
 }
 
 
+TEST_F(Dense, IsColPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->column_permute(cpermute_idxs.get(), permuted.get());
+    dx->column_permute(cpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsInverseRowPermutable)
 {
     set_up_apply_data();
@@ -955,6 +1075,20 @@ TEST_F(Dense, IsInverseRowPermutable)
 }
 
 
+TEST_F(Dense, IsInverseRowPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->inverse_row_permute(rpermute_idxs.get(), permuted.get());
+    dx->inverse_row_permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, IsInverseColPermutable)
 {
     set_up_apply_data();
@@ -967,6 +1101,20 @@ TEST_F(Dense, IsInverseColPermutable)
 }
 
 
+TEST_F(Dense, IsInverseColPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->inverse_column_permute(cpermute_idxs.get(), permuted.get());
+    dx->inverse_column_permute(cpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
 TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -978,6 +1126,20 @@ TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ExtractDiagonalOnTallSkinnyIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto diag = Diagonal::create(ref, x->get_size()[1]);
+    // test make_temporary_clone
+    auto ddiag = Diagonal::create(ref, x->get_size()[1]);
+
+    x->extract_diagonal(diag.get());
+    dx->extract_diagonal(ddiag.get());
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
 TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -989,6 +1151,20 @@ TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ExtractDiagonalOnShortFatIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto diag = Diagonal::create(ref, y->get_size()[0]);
+    // test make_temporary_clone
+    auto ddiag = Diagonal::create(ref, y->get_size()[0]);
+
+    y->extract_diagonal(diag.get());
+    dy->extract_diagonal(ddiag.get());
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
 TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -1011,6 +1187,20 @@ TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, OutplaceAbsoluteMatrixIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto abs_x = NormVector::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dabs_x = NormVector::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->compute_absolute(abs_x.get());
+    dx->compute_absolute(dabs_x.get());
+
+    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, 1e-14);
+}
+
+
 TEST_F(Dense, MakeComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -1022,13 +1212,15 @@ TEST_F(Dense, MakeComplexIsEquivalentToRef)
 }
 
 
-TEST_F(Dense, MakeComplexWithGivenResultIsEquivalentToRef)
+TEST_F(Dense, MakeComplexIntoDenseCrossExecutor)
 {
     set_up_apply_data();
-
     auto complex_x = ComplexMtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dcomplex_x =
+        ComplexMtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
     x->make_complex(complex_x.get());
-    auto dcomplex_x = ComplexMtx::create(omp, x->get_size());
     dx->make_complex(dcomplex_x.get());
 
     GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
@@ -1046,13 +1238,14 @@ TEST_F(Dense, GetRealIsEquivalentToRef)
 }
 
 
-TEST_F(Dense, GetRealWithGivenResultIsEquivalentToRef)
+TEST_F(Dense, GetRealIntoDenseCrossExecutor)
 {
     set_up_apply_data();
-
     auto real_x = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dreal_x = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
     x->get_real(real_x.get());
-    auto dreal_x = Mtx::create(omp, dx->get_size());
     dx->get_real(dreal_x.get());
 
     GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
@@ -1070,13 +1263,14 @@ TEST_F(Dense, GetImagIsEquivalentToRef)
 }
 
 
-TEST_F(Dense, GetImagWithGivenResultIsEquivalentToRef)
+TEST_F(Dense, GetImagIntoDenseCrossExecutor)
 {
     set_up_apply_data();
-
     auto imag_x = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dimag_x = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
     x->get_imag(imag_x.get());
-    auto dimag_x = Mtx::create(omp, dx->get_size());
     dx->get_imag(dimag_x.get());
 
     GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);

From 320d5c1553ea8e9a0dc5d5fd554dfc46f97fb136 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 28 May 2021 17:13:43 +0200
Subject: [PATCH 0734/1008] remove Identity kernel calls from core tests

---
 core/test/matrix/identity.cpp      | 30 ------------------------------
 reference/test/matrix/identity.cpp | 16 ++++++++++++++++
 2 files changed, 16 insertions(+), 30 deletions(-)

diff --git a/core/test/matrix/identity.cpp b/core/test/matrix/identity.cpp
index f2d9ad58084..cffe69d0bcc 100644
--- a/core/test/matrix/identity.cpp
+++ b/core/test/matrix/identity.cpp
@@ -96,36 +96,6 @@ TYPED_TEST(Identity, FailsConstructionWithRectangularSize)
 }
 
 
-TYPED_TEST(Identity, AppliesToVector)
-{
-    using Id = typename TestFixture::Id;
-    using Vec = typename TestFixture::Vec;
-    auto identity = Id::create(this->exec, 3);
-    auto x = Vec::create(this->exec, gko::dim<2>{3, 1});
-    auto b = gko::initialize<Vec>({2.0, 1.0, 5.0}, this->exec);
-
-    identity->apply(b.get(), x.get());
-
-    GKO_ASSERT_MTX_NEAR(x, l({2.0, 1.0, 5.0}), 0.0);
-}
-
-
-TYPED_TEST(Identity, AppliesToMultipleVectors)
-{
-    using Id = typename TestFixture::Id;
-    using Vec = typename TestFixture::Vec;
-    using T = typename TestFixture::value_type;
-    auto identity = Id::create(this->exec, 3);
-    auto x = Vec::create(this->exec, gko::dim<2>{3, 2}, 3);
-    auto b = gko::initialize<Vec>(
-        3, {I<T>{2.0, 3.0}, I<T>{1.0, 2.0}, I<T>{5.0, -1.0}}, this->exec);
-
-    identity->apply(b.get(), x.get());
-
-    GKO_ASSERT_MTX_NEAR(x, l({{2.0, 3.0}, {1.0, 2.0}, {5.0, -1.0}}), 0.0);
-}
-
-
 template <typename T>
 class IdentityFactory : public ::testing::Test {
 protected:
diff --git a/reference/test/matrix/identity.cpp b/reference/test/matrix/identity.cpp
index 04e032d4b33..9e59e88c0ce 100644
--- a/reference/test/matrix/identity.cpp
+++ b/reference/test/matrix/identity.cpp
@@ -78,6 +78,22 @@ TYPED_TEST(Identity, AppliesToVector)
 }
 
 
+TYPED_TEST(Identity, AppliesToMultipleVectors)
+{
+    using Id = typename TestFixture::Id;
+    using Vec = typename TestFixture::Vec;
+    using T = typename TestFixture::value_type;
+    auto identity = Id::create(this->exec, 3);
+    auto x = Vec::create(this->exec, gko::dim<2>{3, 2}, 3);
+    auto b = gko::initialize<Vec>(
+        3, {I<T>{2.0, 3.0}, I<T>{1.0, 2.0}, I<T>{5.0, -1.0}}, this->exec);
+
+    identity->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({{2.0, 3.0}, {1.0, 2.0}, {5.0, -1.0}}), 0.0);
+}
+
+
 TYPED_TEST(Identity, AppliesToMixedVector)
 {
     using Id = typename TestFixture::Id;

From a9c45bb6d0d3c2cb040a28db2e75c8e858cf647a Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 28 May 2021 17:20:20 +0200
Subject: [PATCH 0735/1008] fix icc compilation issues

---
 reference/test/matrix/dense_kernels.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index d0b1e724246..32b9406b968 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -2004,7 +2004,7 @@ TYPED_TEST(Dense, NonSquareSubmatrixIsTransposableIntoDense)
 
     this->mtx4->create_submatrix({0, 1}, {0, 2})->transpose(trans.get());
 
-    GKO_ASSERT_MTX_NEAR(trans, l<T>({{1.0}, {3.0}}), 0.0);
+    GKO_ASSERT_MTX_NEAR(trans, l({1.0, 3.0}), 0.0);
     ASSERT_EQ(trans->get_stride(), 5);
 }
 
@@ -4186,7 +4186,7 @@ TYPED_TEST(DenseComplex, Dot)
 
     a->compute_dot(b.get(), result.get());
 
-    GKO_ASSERT_MTX_NEAR(result, l<T>({T{22.0, 15.0}}), 0.0);
+    GKO_ASSERT_MTX_NEAR(result, l({T{22.0, 15.0}}), 0.0);
 }
 
 
@@ -4203,7 +4203,7 @@ TYPED_TEST(DenseComplex, ConjDot)
 
     a->compute_conj_dot(b.get(), result.get());
 
-    GKO_ASSERT_MTX_NEAR(result, l<T>({T{10.0, -25.0}}), 0.0);
+    GKO_ASSERT_MTX_NEAR(result, l({T{10.0, -25.0}}), 0.0);
 }
 
 

From 557e7a378eedd4961cc756c9334e5caa0eb7265a Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 28 May 2021 22:06:18 +0200
Subject: [PATCH 0736/1008] remove support for Intel 2017

---
 .gitlab-ci.yml | 53 ++++++++++++++++++--------------------------------
 README.md      |  4 ++--
 2 files changed, 21 insertions(+), 36 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 11e66475301..8a4f31dce4c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -327,40 +327,6 @@ build/cuda92/gcc/all/release/shared:
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
 
-# Make sure that our jobs run when HWLOC is
-# forcibly switched off
-build/cuda92/intel/cuda/release/static:
-  <<: *default_build
-  extends:
-    - .full_test_condition
-    - .use_gko-cuda92-gnu7-llvm50-intel2017
-  variables:
-    <<: *default_variables
-    C_COMPILER: "icc"
-    CXX_COMPILER: "icpc"
-    BUILD_OMP: "ON"
-    BUILD_CUDA: "ON"
-    BUILD_HWLOC: "OFF"
-    BUILD_TYPE: "Release"
-    BUILD_SHARED_LIBS: "OFF"
-    CUDA_ARCH: 35
-
-# Build CUDA NVIDIA without omp
-build/cuda92/intel/cuda_wo_omp/release/shared:
-  <<: *default_build
-  extends:
-    - .quick_test_condition
-    - .use_gko-cuda92-gnu7-llvm50-intel2017
-  variables:
-    <<: *default_variables
-    C_COMPILER: "icc"
-    CXX_COMPILER: "icpc"
-    BUILD_CUDA: "ON"
-    BUILD_HIP: "ON"
-    BUILD_HWLOC: "OFF"
-    BUILD_TYPE: "Release"
-    CUDA_ARCH: 35
-
 # cuda 10.0 and friends
 # Make sure that our jobs run when using self-installed
 # third-party HWLOC.
@@ -378,6 +344,8 @@ build/cuda100/gcc/all/debug/shared:
     FAST_TESTS: "ON"
     CUDA_ARCH: 35
 
+# Make sure that our jobs run when HWLOC is
+# forcibly switched off
 build/cuda100/clang/all/release/static:
   <<: *default_build
   extends:
@@ -390,6 +358,7 @@ build/cuda100/clang/all/release/static:
     BUILD_OMP: "ON"
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
+    BUILD_HWLOC: "OFF"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     CUDA_ARCH: 35
@@ -408,6 +377,22 @@ build/cuda100/intel/cuda/release/shared:
     BUILD_TYPE: "Release"
     CUDA_ARCH: 35
 
+# Build CUDA NVIDIA without omp
+build/cuda100/intel/cuda_wo_omp/release/shared:
+  <<: *default_build
+  extends:
+    - .full_test_condition
+    - .use_gko-cuda100-gnu7-llvm60-intel2018
+  variables:
+    <<: *default_variables
+    C_COMPILER: "icc"
+    CXX_COMPILER: "icpc"
+    BUILD_CUDA: "ON"
+    BUILD_HIP: "ON"
+    BUILD_HWLOC: "OFF"
+    BUILD_TYPE: "Release"
+    CUDA_ARCH: 35
+
 # cuda 10.1 and friends
 build/cuda101/gcc/all/debug/shared:
   <<: *default_build
diff --git a/README.md b/README.md
index 271d211427b..5f66f2fe6d6 100644
--- a/README.md
+++ b/README.md
@@ -40,8 +40,8 @@ For Ginkgo core library:
 *   C++14 compliant compiler, one of:
     *   _gcc 5.3+, 6.3+, 7.3+, all versions after 8.1+_
     *   _clang 3.9+_
-    *   _Intel compiler 2017+_
-    *   _Apple LLVM 8.0+_ (__TODO__: verify)
+    *   _Intel compiler 2018+_
+    *   _Apple LLVM 8.0+_
 
 The Ginkgo CUDA module has the following __additional__ requirements:
 

From cd6204ee29c585cdbd7fafd8fe1b676b074a3b77 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 1 Jun 2021 23:16:23 +0200
Subject: [PATCH 0737/1008] review update

Co-authored-by: Gregor Olenik <gregor.olenik@web.de>
Co-authored-by: Pratik Nayak <pratik.nayak@kit.edu>
---
 cuda/test/matrix/dense_kernels.cpp      |  4 ++-
 hip/test/matrix/dense_kernels.hip.cpp   |  4 ++-
 include/ginkgo/core/base/types.hpp      | 33 ++++++++++++++++++-------
 include/ginkgo/core/matrix/dense.hpp    |  4 +--
 omp/test/matrix/dense_kernels.cpp       |  4 ++-
 reference/test/matrix/dense_kernels.cpp |  1 +
 6 files changed, 36 insertions(+), 14 deletions(-)

diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index 8e509017def..23d394ad546 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -194,7 +194,8 @@ class Dense : public ::testing::Test {
 TEST_F(Dense, CudaCopyRespectsStride)
 {
     set_up_vector_data(3);
-    auto result = Mtx::create(cuda, dx->get_size(), dx->get_size()[1] + 1);
+    auto stride = dx->get_size()[1] + 1;
+    auto result = Mtx::create(cuda, dx->get_size(), stride);
     double val = 123456789.0;
     auto original_data = result->get_values();
     auto padding_ptr = original_data + dx->get_size()[1];
@@ -203,6 +204,7 @@ TEST_F(Dense, CudaCopyRespectsStride)
     dx->convert_to(result.get());
 
     GKO_ASSERT_MTX_NEAR(result, dx, 0);
+    ASSERT_EQ(result->get_stride(), stride);
     ASSERT_EQ(cuda->copy_val_to_host(padding_ptr), val);
     ASSERT_EQ(result->get_values(), original_data);
 }
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index 0fce6afe9a3..18c5a6dc0b5 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -189,7 +189,8 @@ class Dense : public ::testing::Test {
 TEST_F(Dense, HipCopyRespectsStride)
 {
     set_up_vector_data(3);
-    auto result = Mtx::create(hip, dx->get_size(), dx->get_size()[1] + 1);
+    auto stride = dx->get_size()[1] + 1;
+    auto result = Mtx::create(hip, dx->get_size(), stride);
     double val = 123456789.0;
     auto original_data = result->get_values();
     auto padding_ptr = original_data + dx->get_size()[1];
@@ -198,6 +199,7 @@ TEST_F(Dense, HipCopyRespectsStride)
     dx->convert_to(result.get());
 
     GKO_ASSERT_MTX_NEAR(result, dx, 0);
+    ASSERT_EQ(result->get_stride(), stride);
     ASSERT_EQ(hip->copy_val_to_host(padding_ptr), val);
     ASSERT_EQ(result->get_values(), original_data);
 }
diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index 0288316dee8..a2d922c9a9a 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -527,15 +527,6 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
 #endif
 
 
-/**
- * Instantiates a template for each value type conversion pair compiled by
- * Ginkgo.
- *
- * @param _macro  A macro which expands the template instantiation
- *                (not including the leading `template` specifier).
- *                Should take two arguments `src` and `dst`, which
- *                are replaced by the source and destination value type.
- */
 #if GINKGO_DPCPP_SINGLE_MODE
 #define GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(_macro)                  \
     template <>                                                            \
@@ -546,6 +537,8 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
     _macro(std::complex<float>, std::complex<double>) GKO_NOT_IMPLEMENTED; \
     template <>                                                            \
     _macro(std::complex<double>, std::complex<float>) GKO_NOT_IMPLEMENTED
+
+
 #define GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(_macro)         \
     GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(_macro);                    \
     template <>                                                           \
@@ -557,11 +550,33 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
     template <>                                                           \
     _macro(std::complex<double>, std::complex<double>) GKO_NOT_IMPLEMENTED
 #else
+
+
+/**
+ * Instantiates a template for each value type conversion pair compiled by
+ * Ginkgo.
+ *
+ * @param _macro  A macro which expands the template instantiation
+ *                (not including the leading `template` specifier).
+ *                Should take two arguments `src` and `dst`, which
+ *                are replaced by the source and destination value type.
+ */
 #define GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(_macro)       \
     template _macro(float, double);                             \
     template _macro(double, float);                             \
     template _macro(std::complex<float>, std::complex<double>); \
     template _macro(std::complex<double>, std::complex<float>)
+
+
+/**
+ * Instantiates a template for each value type conversion or copy pair compiled
+ * by Ginkgo.
+ *
+ * @param _macro  A macro which expands the template instantiation
+ *                (not including the leading `template` specifier).
+ *                Should take two arguments `src` and `dst`, which
+ *                are replaced by the source and destination value type.
+ */
 #define GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(_macro) \
     GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(_macro);            \
     template _macro(float, float);                                \
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 9a6e81b94ca..cde4a8d91cf 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -297,7 +297,7 @@ class Dense
         const Array<int64> *permutation_indices) const override;
 
     /**
-     * Writes the symetrically permuted matrix into the given output matrix.
+     * Writes the symmetrically permuted matrix into the given output matrix.
      *
      * @param permutation_indices  The array containing permutation indices.
      *                             It must have `this->get_size()[0]` elements.
@@ -319,7 +319,7 @@ class Dense
         const Array<int64> *permutation_indices) const override;
 
     /**
-     * Writes the inverse symetrically permuted matrix into the given output
+     * Writes the inverse symmetrically permuted matrix into the given output
      * matrix.
      *
      * @param permutation_indices  The array containing permutation indices.
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index 29a24fa37b7..8ade4b939af 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -208,7 +208,8 @@ class Dense : public ::testing::Test {
 TEST_F(Dense, OmpCopyRespectsStride)
 {
     set_up_vector_data(3);
-    auto result = Mtx::create(omp, dx->get_size(), dx->get_size()[1] + 1);
+    auto stride = dx->get_size()[1] + 1;
+    auto result = Mtx::create(omp, dx->get_size(), stride);
     double val = 123456789.0;
     auto original_data = result->get_values();
     auto padding_ptr = original_data + dx->get_size()[1];
@@ -217,6 +218,7 @@ TEST_F(Dense, OmpCopyRespectsStride)
     dx->convert_to(result.get());
 
     GKO_ASSERT_MTX_NEAR(result, dx, 0);
+    ASSERT_EQ(result->get_stride(), stride);
     ASSERT_EQ(omp->copy_val_to_host(padding_ptr), val);
     ASSERT_EQ(result->get_values(), original_data);
 }
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 32b9406b968..36abd6a02c2 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -128,6 +128,7 @@ TYPED_TEST(Dense, CopyRespectsStride)
     m->convert_to(m2.get());
 
     EXPECT_EQ(m2->at(0, 0), value_type{1.0});
+    EXPECT_EQ(m2->get_stride(), 2);
     EXPECT_EQ(m2->at(1, 0), value_type{2.0});
     EXPECT_EQ(m2->get_values(), original_data);
     EXPECT_EQ(original_data[1], TypeParam{3.0});

From f581c7874b17fb8d0c084b2d5ec8ac73a2494aad Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 2 Jun 2021 00:45:24 +0200
Subject: [PATCH 0738/1008] improve dense convert_to

use a temporary clone of the array
instead of working around recursion in convert_to
---
 core/matrix/dense.cpp | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index f8d8421a9dd..5bf1cf538bc 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -331,16 +331,19 @@ void Dense<ValueType>::compute_norm2_impl(LinOp *result) const
 template <typename ValueType>
 void Dense<ValueType>::convert_to(Dense<ValueType> *result) const
 {
-    if (result->get_size() && result->get_size() == this->get_size()) {
-        // it's important we clone the const *this instead of result, since
-        // otherwise the copy_back_deleter of the temporary_clone calls
-        // convert_to again with the same dimensions, causing an infinite
-        // recursion. For the same reason we need to check that the result size
-        // is not zero, since otherwise it could come from the initial
-        // convert_to call in make_temporary_clone.
-        auto exec = result->get_executor();
-        exec->run(
-            dense::make_copy(make_temporary_clone(exec, this).get(), result));
+    if (result->get_size() == this->get_size()) {
+        // we need to create a executor-local clone of the target data, that
+        // will be copied back later.
+        auto exec = this->get_executor();
+        auto result_array = make_temporary_clone(exec, &result->values_);
+        // create a (value, not pointer to avoid allocation overhead) view
+        // matrix on the array to avoid special-casing cross-executor copies
+        auto tmp_result =
+            Dense{exec, result->get_size(),
+                  Array<ValueType>::view(exec, result_array->get_num_elems(),
+                                         result_array->get_data()),
+                  result->get_stride()};
+        exec->run(dense::make_copy(this, &tmp_result));
     } else {
         result->values_ = this->values_;
         result->stride_ = this->stride_;
@@ -360,9 +363,7 @@ template <typename ValueType>
 void Dense<ValueType>::convert_to(
     Dense<next_precision<ValueType>> *result) const
 {
-    if (result->get_size() && result->get_size() == this->get_size()) {
-        // here we don't need to use the target executor, since the copy back
-        // invokes the non-converting convert_to(Dense<ValueType>*).
+    if (result->get_size() == this->get_size()) {
         auto exec = this->get_executor();
         exec->run(
             dense::make_copy(this, make_temporary_clone(exec, result).get()));

From eba2a3f33536f4515528243a677a2f2421003278 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 2 Jun 2021 00:45:52 +0200
Subject: [PATCH 0739/1008] add make_temporary_output_clone

This provides an alternative to make_temporary_clone
that doesn't initialize the content for output-parameters
---
 core/matrix/dense.cpp                        | 54 ++++++++++----------
 core/test/base/array.cpp                     | 29 +++++++++++
 core/test/base/utils.cpp                     | 26 ++++++++++
 include/ginkgo/core/base/array.hpp           | 11 ++--
 include/ginkgo/core/base/temporary_clone.hpp | 38 +++++++++++---
 include/ginkgo/core/matrix/dense.hpp         | 37 +++++++++++---
 reference/test/matrix/dense_kernels.cpp      | 22 ++++++++
 7 files changed, 175 insertions(+), 42 deletions(-)

diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 5bf1cf538bc..bbee89b1dc1 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -335,7 +335,7 @@ void Dense<ValueType>::convert_to(Dense<ValueType> *result) const
         // we need to create a executor-local clone of the target data, that
         // will be copied back later.
         auto exec = this->get_executor();
-        auto result_array = make_temporary_clone(exec, &result->values_);
+        auto result_array = make_temporary_output_clone(exec, &result->values_);
         // create a (value, not pointer to avoid allocation overhead) view
         // matrix on the array to avoid special-casing cross-executor copies
         auto tmp_result =
@@ -365,8 +365,8 @@ void Dense<ValueType>::convert_to(
 {
     if (result->get_size() == this->get_size()) {
         auto exec = this->get_executor();
-        exec->run(
-            dense::make_copy(this, make_temporary_clone(exec, result).get()));
+        exec->run(dense::make_copy(
+            this, make_temporary_output_clone(exec, result).get()));
     } else {
         result->values_ = this->values_;
         result->stride_ = this->stride_;
@@ -697,8 +697,8 @@ void Dense<ValueType>::transpose(Dense<ValueType> *output) const
 {
     GKO_ASSERT_EQUAL_DIMENSIONS(output, gko::transpose(this->get_size()));
     auto exec = this->get_executor();
-    exec->run(
-        dense::make_transpose(this, make_temporary_clone(exec, output).get()));
+    exec->run(dense::make_transpose(
+        this, make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -708,7 +708,7 @@ void Dense<ValueType>::conj_transpose(Dense<ValueType> *output) const
     GKO_ASSERT_EQUAL_DIMENSIONS(output, gko::transpose(this->get_size()));
     auto exec = this->get_executor();
     exec->run(dense::make_conj_transpose(
-        this, make_temporary_clone(exec, output).get()));
+        this, make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -743,7 +743,7 @@ void Dense<ValueType>::permute(const Array<int32> *permutation_indices,
 
     exec->run(dense::make_symm_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_clone(exec, output).get()));
+        make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -758,7 +758,7 @@ void Dense<ValueType>::permute(const Array<int64> *permutation_indices,
 
     exec->run(dense::make_symm_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_clone(exec, output).get()));
+        make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -793,7 +793,7 @@ void Dense<ValueType>::inverse_permute(const Array<int32> *permutation_indices,
 
     exec->run(dense::make_inv_symm_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_clone(exec, output).get()));
+        make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -808,7 +808,7 @@ void Dense<ValueType>::inverse_permute(const Array<int64> *permutation_indices,
 
     exec->run(dense::make_inv_symm_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_clone(exec, output).get()));
+        make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -842,7 +842,7 @@ void Dense<ValueType>::row_permute(const Array<int32> *permutation_indices,
 
     exec->run(dense::make_row_gather(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_clone(exec, output).get()));
+        make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -856,7 +856,7 @@ void Dense<ValueType>::row_permute(const Array<int64> *permutation_indices,
 
     exec->run(dense::make_row_gather(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_clone(exec, output).get()));
+        make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -894,7 +894,7 @@ void Dense<ValueType>::row_gather(const Array<int32> *row_indices,
 
     exec->run(dense::make_row_gather(
         make_temporary_clone(exec, row_indices).get(), this,
-        make_temporary_clone(exec, row_gathered).get()));
+        make_temporary_output_clone(exec, row_gathered).get()));
 }
 
 
@@ -909,7 +909,7 @@ void Dense<ValueType>::row_gather(const Array<int64> *row_indices,
 
     this->get_executor()->run(dense::make_row_gather(
         make_temporary_clone(exec, row_indices).get(), this,
-        make_temporary_clone(exec, row_gathered).get()));
+        make_temporary_output_clone(exec, row_gathered).get()));
 }
 
 
@@ -923,7 +923,7 @@ void Dense<ValueType>::column_permute(const Array<int32> *permutation_indices,
 
     exec->run(dense::make_column_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_clone(exec, output).get()));
+        make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -937,7 +937,7 @@ void Dense<ValueType>::column_permute(const Array<int64> *permutation_indices,
 
     exec->run(dense::make_column_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_clone(exec, output).get()));
+        make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -971,7 +971,7 @@ void Dense<ValueType>::inverse_row_permute(
 
     exec->run(dense::make_inverse_row_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_clone(exec, output).get()));
+        make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -985,7 +985,7 @@ void Dense<ValueType>::inverse_row_permute(
 
     exec->run(dense::make_inverse_row_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_clone(exec, output).get()));
+        make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -1019,7 +1019,7 @@ void Dense<ValueType>::inverse_column_permute(
 
     exec->run(dense::make_inverse_column_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_clone(exec, output).get()));
+        make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -1033,7 +1033,7 @@ void Dense<ValueType>::inverse_column_permute(
 
     exec->run(dense::make_inverse_column_permute(
         make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_clone(exec, output).get()));
+        make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -1065,7 +1065,7 @@ void Dense<ValueType>::extract_diagonal(Diagonal<ValueType> *output) const
     GKO_ASSERT_EQ(output->get_size()[0], diag_size);
 
     exec->run(dense::make_extract_diagonal(
-        this, make_temporary_clone(exec, output).get()));
+        this, make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -1105,7 +1105,7 @@ void Dense<ValueType>::compute_absolute(
     auto exec = this->get_executor();
 
     exec->run(dense::make_outplace_absolute_dense(
-        this, make_temporary_clone(exec, output).get()));
+        this, make_temporary_output_clone(exec, output).get()));
 }
 
 
@@ -1127,7 +1127,7 @@ void Dense<ValueType>::make_complex(
     auto exec = this->get_executor();
 
     exec->run(dense::make_make_complex(
-        this, make_temporary_clone(exec, result).get()));
+        this, make_temporary_output_clone(exec, result).get()));
 }
 
 
@@ -1148,8 +1148,8 @@ void Dense<ValueType>::get_real(
     GKO_ASSERT_EQUAL_DIMENSIONS(this, result);
     auto exec = this->get_executor();
 
-    exec->run(
-        dense::make_get_real(this, make_temporary_clone(exec, result).get()));
+    exec->run(dense::make_get_real(
+        this, make_temporary_output_clone(exec, result).get()));
 }
 
 
@@ -1170,8 +1170,8 @@ void Dense<ValueType>::get_imag(
     GKO_ASSERT_EQUAL_DIMENSIONS(this, result);
     auto exec = this->get_executor();
 
-    exec->run(
-        dense::make_get_imag(this, make_temporary_clone(exec, result).get()));
+    exec->run(dense::make_get_imag(
+        this, make_temporary_output_clone(exec, result).get()));
 }
 
 
diff --git a/core/test/base/array.cpp b/core/test/base/array.cpp
index 91c5e8867b9..708bfcf5dad 100644
--- a/core/test/base/array.cpp
+++ b/core/test/base/array.cpp
@@ -272,6 +272,14 @@ TYPED_TEST(Array, CanCreateTemporaryCloneOnSameExecutor)
 }
 
 
+TYPED_TEST(Array, CanCreateTemporaryOutputCloneOnSameExecutor)
+{
+    auto tmp_clone = make_temporary_output_clone(this->exec, &this->x);
+
+    ASSERT_EQ(tmp_clone.get(), &this->x);
+}
+
+
 // For tests between different memory, check cuda/test/base/array.cu
 TYPED_TEST(Array, DoesNotCreateATemporaryCloneBetweenSameMemory)
 {
@@ -299,6 +307,27 @@ TYPED_TEST(Array, DoesNotCopyBackTemporaryCloneBetweenSameMemory)
 }
 
 
+TYPED_TEST(Array, CanCreateTemporaryOutputCloneOnDifferentExecutors)
+{
+    auto other = gko::OmpExecutor::create();
+
+    {
+        auto tmp_clone = make_temporary_output_clone(other, &this->x);
+        tmp_clone->get_data()[0] = 4;
+        tmp_clone->get_data()[1] = 5;
+
+        // there is no reliable way to check the memory is uninitialized
+        ASSERT_EQ(tmp_clone->get_num_elems(), this->x.get_num_elems());
+        ASSERT_EQ(tmp_clone->get_executor(), other);
+        ASSERT_EQ(this->x.get_executor(), this->exec);
+        ASSERT_EQ(this->x.get_data()[0], TypeParam{5});
+        ASSERT_EQ(this->x.get_data()[1], TypeParam{2});
+    }
+    ASSERT_EQ(this->x.get_data()[0], TypeParam{4});
+    ASSERT_EQ(this->x.get_data()[1], TypeParam{5});
+}
+
+
 TYPED_TEST(Array, CanBeCleared)
 {
     this->x.clear();
diff --git a/core/test/base/utils.cpp b/core/test/base/utils.cpp
index b66a1260a3a..6d8985e1dff 100644
--- a/core/test/base/utils.cpp
+++ b/core/test/base/utils.cpp
@@ -410,13 +410,39 @@ TEST_F(TemporaryClone, DoesNotCopyToSameMemory)
 }
 
 
+TEST_F(TemporaryClone, OutputDoesNotCopyToSameMemory)
+{
+    auto other = gko::ReferenceExecutor::create();
+    auto clone = make_temporary_output_clone(other, gko::lend(obj));
+
+    ASSERT_NE(clone.get()->get_executor(), other);
+    ASSERT_EQ(obj->get_executor(), ref);
+}
+
+
 TEST_F(TemporaryClone, CopiesBackAfterLeavingScope)
 {
+    obj->data = 4;
     {
         auto clone = make_temporary_clone(omp, gko::lend(obj));
         clone.get()->data = 7;
+
+        ASSERT_EQ(obj->data, 4);
     }
+    ASSERT_EQ(obj->get_executor(), ref);
+    ASSERT_EQ(obj->data, 7);
+}
 
+
+TEST_F(TemporaryClone, OutputCopiesBackAfterLeavingScope)
+{
+    obj->data = 4;
+    {
+        auto clone = make_temporary_output_clone(omp, gko::lend(obj));
+        clone.get()->data = 7;
+
+        ASSERT_EQ(obj->data, 4);
+    }
     ASSERT_EQ(obj->get_executor(), ref);
     ASSERT_EQ(obj->data, 7);
 }
diff --git a/include/ginkgo/core/base/array.hpp b/include/ginkgo/core/base/array.hpp
index c9f4aaa57c4..c4aa2323417 100644
--- a/include/ginkgo/core/base/array.hpp
+++ b/include/ginkgo/core/base/array.hpp
@@ -576,16 +576,21 @@ namespace detail {
 template <typename T>
 struct temporary_clone_helper<Array<T>> {
     static std::unique_ptr<Array<T>> create(
-        std::shared_ptr<const Executor> exec, Array<T> *ptr)
+        std::shared_ptr<const Executor> exec, Array<T> *ptr, bool copy_data)
     {
-        return std::make_unique<Array<T>>(std::move(exec), *ptr);
+        if (copy_data) {
+            return std::make_unique<Array<T>>(std::move(exec), *ptr);
+        } else {
+            return std::make_unique<Array<T>>(std::move(exec),
+                                              ptr->get_num_elems());
+        }
     }
 };
 
 template <typename T>
 struct temporary_clone_helper<const Array<T>> {
     static std::unique_ptr<const Array<T>> create(
-        std::shared_ptr<const Executor> exec, const Array<T> *ptr)
+        std::shared_ptr<const Executor> exec, const Array<T> *ptr, bool)
     {
         return std::make_unique<const Array<T>>(std::move(exec), *ptr);
     }
diff --git a/include/ginkgo/core/base/temporary_clone.hpp b/include/ginkgo/core/base/temporary_clone.hpp
index c3309d8736a..8f1f059a647 100644
--- a/include/ginkgo/core/base/temporary_clone.hpp
+++ b/include/ginkgo/core/base/temporary_clone.hpp
@@ -108,7 +108,7 @@ class copy_back_deleter<const T> {
 template <typename T>
 struct temporary_clone_helper {
     static std::unique_ptr<T> create(std::shared_ptr<const Executor> exec,
-                                     T *ptr)
+                                     T *ptr, bool)
     {
         return gko::clone(std::move(exec), ptr);
     }
@@ -137,8 +137,11 @@ class temporary_clone {
      *
      * @param exec  the executor where the clone will be created
      * @param ptr  a pointer to the object of which the clone will be created
+     * @param copy_data  should the data be copied to the executor, or should
+     *                   only the result be copied back afterwards?
      */
-    explicit temporary_clone(std::shared_ptr<const Executor> exec, pointer ptr)
+    explicit temporary_clone(std::shared_ptr<const Executor> exec, pointer ptr,
+                             bool copy_data = true)
     {
         if (ptr->get_executor()->memory_accessible(exec)) {
             // just use the object we already have
@@ -146,10 +149,10 @@ class temporary_clone {
         } else {
             // clone the object to the new executor and make sure it's copied
             // back before we delete it
-            handle_ = handle_type(
-                temporary_clone_helper<T>::create(std::move(exec), ptr)
-                    .release(),
-                copy_back_deleter<T>(ptr));
+            handle_ = handle_type(temporary_clone_helper<T>::create(
+                                      std::move(exec), ptr, copy_data)
+                                      .release(),
+                                  copy_back_deleter<T>(ptr));
         }
     }
 
@@ -196,6 +199,29 @@ detail::temporary_clone<T> make_temporary_clone(
 }
 
 
+/**
+ * Creates a uninitialized temporary_clone that will be copied back to the input
+ * afterwards. It can be used for output parameters to avoid an unnecessary copy
+ * in make_temporary_clone.
+ *
+ * This is a helper function which avoids the need to explicitly specify the
+ * type of the object, as would be the case if using the constructor of
+ * temporary_clone.
+ *
+ * @param exec  the executor where the uninitialized clone will be created
+ * @param ptr  a pointer to the object of which the clone will be created
+ */
+template <typename T>
+detail::temporary_clone<T> make_temporary_output_clone(
+    std::shared_ptr<const Executor> exec, T *ptr)
+{
+    static_assert(
+        !std::is_const<T>::value,
+        "make_temporary_output_clone should only be used on non-const objects");
+    return detail::temporary_clone<T>(std::move(exec), ptr);
+}
+
+
 }  // namespace gko
 
 
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index cde4a8d91cf..25ed80c7b85 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -660,7 +660,7 @@ class Dense
     {
         auto exec = this->get_executor();
         this->compute_dot_impl(make_temporary_clone(exec, b).get(),
-                               make_temporary_clone(exec, result).get());
+                               make_temporary_output_clone(exec, result).get());
     }
 
     /**
@@ -674,8 +674,9 @@ class Dense
     void compute_conj_dot(const LinOp *b, LinOp *result) const
     {
         auto exec = this->get_executor();
-        this->compute_conj_dot_impl(make_temporary_clone(exec, b).get(),
-                                    make_temporary_clone(exec, result).get());
+        this->compute_conj_dot_impl(
+            make_temporary_clone(exec, b).get(),
+            make_temporary_output_clone(exec, result).get());
     }
 
     /**
@@ -688,7 +689,8 @@ class Dense
     void compute_norm2(LinOp *result) const
     {
         auto exec = this->get_executor();
-        this->compute_norm2_impl(make_temporary_clone(exec, result).get());
+        this->compute_norm2_impl(
+            make_temporary_output_clone(exec, result).get());
     }
 
     /**
@@ -818,8 +820,10 @@ class Dense
           values_{exec, std::forward<ValuesArray>(values)},
           stride_{stride}
     {
-        GKO_ENSURE_IN_BOUNDS((size[0] - 1) * stride + size[1] - 1,
-                             values_.get_num_elems());
+        if (size[0] > 0 && size[1] > 0) {
+            GKO_ENSURE_IN_BOUNDS((size[0] - 1) * stride + size[1] - 1,
+                                 values_.get_num_elems());
+        }
     }
 
     /**
@@ -938,6 +942,27 @@ class Dense
 }  // namespace matrix
 
 
+namespace detail {
+
+
+template <typename ValueType>
+struct temporary_clone_helper<matrix::Dense<ValueType>> {
+    static std::unique_ptr<matrix::Dense<ValueType>> create(
+        std::shared_ptr<const Executor> exec, matrix::Dense<ValueType> *ptr,
+        bool copy_data)
+    {
+        if (copy_data) {
+            return gko::clone(std::move(exec), ptr);
+        } else {
+            return matrix::Dense<ValueType>::create(exec, ptr->get_size());
+        }
+    }
+};
+
+
+}  // namespace detail
+
+
 /**
  * Creates and initializes a column-vector.
  *
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 36abd6a02c2..4da0248b196 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -135,6 +135,28 @@ TYPED_TEST(Dense, CopyRespectsStride)
 }
 
 
+TYPED_TEST(Dense, TemporaryOutputCloneWorks)
+{
+    using value_type = typename TestFixture::value_type;
+    auto other = gko::OmpExecutor::create();
+    auto m =
+        gko::initialize<gko::matrix::Dense<TypeParam>>({1.0, 2.0}, this->exec);
+
+    {
+        auto clone = gko::make_temporary_output_clone(other, m.get());
+        clone->at(0) = 4.0;
+        clone->at(1) = 5.0;
+
+        ASSERT_EQ(m->at(0), value_type{1.0});
+        ASSERT_EQ(m->at(1), value_type{2.0});
+        ASSERT_EQ(clone->get_size(), m->get_size());
+        ASSERT_EQ(clone->get_executor(), other);
+    }
+    ASSERT_EQ(m->at(0), value_type{4.0});
+    ASSERT_EQ(m->at(1), value_type{5.0});
+}
+
+
 TYPED_TEST(Dense, CanBeFilledWithValue)
 {
     using value_type = typename TestFixture::value_type;

From 488511c69aecd3e9cfac1f6a68d2a56653b875a5 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 2 Jun 2021 11:33:26 +0200
Subject: [PATCH 0740/1008] fix ref dense temporary clone test without omp

---
 reference/test/matrix/dense_kernels.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 4da0248b196..47703ff8349 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -139,18 +139,17 @@ TYPED_TEST(Dense, TemporaryOutputCloneWorks)
 {
     using value_type = typename TestFixture::value_type;
     auto other = gko::OmpExecutor::create();
-    auto m =
-        gko::initialize<gko::matrix::Dense<TypeParam>>({1.0, 2.0}, this->exec);
+    auto m = gko::initialize<gko::matrix::Dense<TypeParam>>({1.0, 2.0}, other);
 
     {
-        auto clone = gko::make_temporary_output_clone(other, m.get());
+        auto clone = gko::make_temporary_output_clone(this->exec, m.get());
         clone->at(0) = 4.0;
         clone->at(1) = 5.0;
 
         ASSERT_EQ(m->at(0), value_type{1.0});
         ASSERT_EQ(m->at(1), value_type{2.0});
         ASSERT_EQ(clone->get_size(), m->get_size());
-        ASSERT_EQ(clone->get_executor(), other);
+        ASSERT_EQ(clone->get_executor(), this->exec);
     }
     ASSERT_EQ(m->at(0), value_type{4.0});
     ASSERT_EQ(m->at(1), value_type{5.0});

From 297d269a29161699afc718dce81f52ec12c61516 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 2 Jun 2021 12:36:32 +0200
Subject: [PATCH 0741/1008] actually avoid copies in temporary_output_clone

---
 include/ginkgo/core/base/temporary_clone.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/ginkgo/core/base/temporary_clone.hpp b/include/ginkgo/core/base/temporary_clone.hpp
index 8f1f059a647..589a0ca9114 100644
--- a/include/ginkgo/core/base/temporary_clone.hpp
+++ b/include/ginkgo/core/base/temporary_clone.hpp
@@ -218,7 +218,7 @@ detail::temporary_clone<T> make_temporary_output_clone(
     static_assert(
         !std::is_const<T>::value,
         "make_temporary_output_clone should only be used on non-const objects");
-    return detail::temporary_clone<T>(std::move(exec), ptr);
+    return detail::temporary_clone<T>(std::move(exec), ptr, false);
 }
 
 

From 3961f3ac4b757fed068a3fc396147103947b2978 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 2 Jun 2021 13:35:39 +0200
Subject: [PATCH 0742/1008] unify permutation core implementations

---
 core/matrix/dense.cpp                | 264 ++++++++++++++-------------
 include/ginkgo/core/matrix/dense.hpp |  27 +++
 2 files changed, 168 insertions(+), 123 deletions(-)

diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index bbee89b1dc1..51fbd4260bd 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -712,6 +712,113 @@ void Dense<ValueType>::conj_transpose(Dense<ValueType> *output) const
 }
 
 
+template <typename ValueType>
+template <typename IndexType>
+void Dense<ValueType>::permute_impl(const Array<IndexType> *permutation_indices,
+                                    Dense<ValueType> *output) const
+{
+    GKO_ASSERT_IS_SQUARE_MATRIX(this);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+    auto exec = this->get_executor();
+
+    exec->run(dense::make_symm_permute(
+        make_temporary_clone(exec, permutation_indices).get(), this,
+        make_temporary_output_clone(exec, output).get()));
+}
+
+
+template <typename ValueType>
+template <typename IndexType>
+void Dense<ValueType>::inverse_permute_impl(
+    const Array<IndexType> *permutation_indices, Dense<ValueType> *output) const
+{
+    GKO_ASSERT_IS_SQUARE_MATRIX(this);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+    auto exec = this->get_executor();
+
+    exec->run(dense::make_inv_symm_permute(
+        make_temporary_clone(exec, permutation_indices).get(), this,
+        make_temporary_output_clone(exec, output).get()));
+}
+
+
+template <typename ValueType>
+template <typename IndexType>
+void Dense<ValueType>::row_permute_impl(
+    const Array<IndexType> *permutation_indices, Dense<ValueType> *output) const
+{
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
+    auto exec = this->get_executor();
+
+    exec->run(dense::make_row_gather(
+        make_temporary_clone(exec, permutation_indices).get(), this,
+        make_temporary_output_clone(exec, output).get()));
+}
+
+
+template <typename ValueType>
+template <typename IndexType>
+void Dense<ValueType>::row_gather_impl(const Array<IndexType> *row_indices,
+                                       Dense<ValueType> *row_gathered) const
+{
+    auto exec = this->get_executor();
+    dim<2> expected_dim{row_indices->get_num_elems(), this->get_size()[1]};
+    GKO_ASSERT_EQUAL_DIMENSIONS(expected_dim, row_gathered);
+
+    exec->run(dense::make_row_gather(
+        make_temporary_clone(exec, row_indices).get(), this,
+        make_temporary_output_clone(exec, row_gathered).get()));
+}
+
+
+template <typename ValueType>
+template <typename IndexType>
+void Dense<ValueType>::column_permute_impl(
+    const Array<IndexType> *permutation_indices, Dense<ValueType> *output) const
+{
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
+    auto exec = this->get_executor();
+
+    exec->run(dense::make_column_permute(
+        make_temporary_clone(exec, permutation_indices).get(), this,
+        make_temporary_output_clone(exec, output).get()));
+}
+
+
+template <typename ValueType>
+template <typename IndexType>
+void Dense<ValueType>::inverse_row_permute_impl(
+    const Array<IndexType> *permutation_indices, Dense<ValueType> *output) const
+{
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
+    auto exec = this->get_executor();
+
+    exec->run(dense::make_inverse_row_permute(
+        make_temporary_clone(exec, permutation_indices).get(), this,
+        make_temporary_output_clone(exec, output).get()));
+}
+
+
+template <typename ValueType>
+template <typename IndexType>
+void Dense<ValueType>::inverse_column_permute_impl(
+    const Array<IndexType> *permutation_indices, Dense<ValueType> *output) const
+{
+    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
+    auto exec = this->get_executor();
+
+    exec->run(dense::make_inverse_column_permute(
+        make_temporary_clone(exec, permutation_indices).get(), this,
+        make_temporary_output_clone(exec, output).get()));
+}
+
+
 template <typename ValueType>
 std::unique_ptr<LinOp> Dense<ValueType>::permute(
     const Array<int32> *permutation_indices) const
@@ -736,14 +843,7 @@ template <typename ValueType>
 void Dense<ValueType>::permute(const Array<int32> *permutation_indices,
                                Dense<ValueType> *output) const
 {
-    GKO_ASSERT_IS_SQUARE_MATRIX(this);
-    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
-    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
-    auto exec = this->get_executor();
-
-    exec->run(dense::make_symm_permute(
-        make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_output_clone(exec, output).get()));
+    this->permute_impl(permutation_indices, output);
 }
 
 
@@ -751,14 +851,7 @@ template <typename ValueType>
 void Dense<ValueType>::permute(const Array<int64> *permutation_indices,
                                Dense<ValueType> *output) const
 {
-    GKO_ASSERT_IS_SQUARE_MATRIX(this);
-    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
-    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
-    auto exec = this->get_executor();
-
-    exec->run(dense::make_symm_permute(
-        make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_output_clone(exec, output).get()));
+    this->permute_impl(permutation_indices, output);
 }
 
 
@@ -786,14 +879,7 @@ template <typename ValueType>
 void Dense<ValueType>::inverse_permute(const Array<int32> *permutation_indices,
                                        Dense<ValueType> *output) const
 {
-    GKO_ASSERT_IS_SQUARE_MATRIX(this);
-    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
-    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
-    auto exec = this->get_executor();
-
-    exec->run(dense::make_inv_symm_permute(
-        make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_output_clone(exec, output).get()));
+    this->inverse_permute_impl(permutation_indices, output);
 }
 
 
@@ -801,14 +887,7 @@ template <typename ValueType>
 void Dense<ValueType>::inverse_permute(const Array<int64> *permutation_indices,
                                        Dense<ValueType> *output) const
 {
-    GKO_ASSERT_IS_SQUARE_MATRIX(this);
-    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
-    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
-    auto exec = this->get_executor();
-
-    exec->run(dense::make_inv_symm_permute(
-        make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_output_clone(exec, output).get()));
+    this->inverse_permute_impl(permutation_indices, output);
 }
 
 
@@ -836,13 +915,7 @@ template <typename ValueType>
 void Dense<ValueType>::row_permute(const Array<int32> *permutation_indices,
                                    Dense<ValueType> *output) const
 {
-    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
-    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
-    auto exec = this->get_executor();
-
-    exec->run(dense::make_row_gather(
-        make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_output_clone(exec, output).get()));
+    this->row_permute_impl(permutation_indices, output);
 }
 
 
@@ -850,13 +923,7 @@ template <typename ValueType>
 void Dense<ValueType>::row_permute(const Array<int64> *permutation_indices,
                                    Dense<ValueType> *output) const
 {
-    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
-    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
-    auto exec = this->get_executor();
-
-    exec->run(dense::make_row_gather(
-        make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_output_clone(exec, output).get()));
+    this->row_permute_impl(permutation_indices, output);
 }
 
 
@@ -888,13 +955,7 @@ template <typename ValueType>
 void Dense<ValueType>::row_gather(const Array<int32> *row_indices,
                                   Dense<ValueType> *row_gathered) const
 {
-    auto exec = this->get_executor();
-    dim<2> expected_dim{row_indices->get_num_elems(), this->get_size()[1]};
-    GKO_ASSERT_EQUAL_DIMENSIONS(expected_dim, row_gathered);
-
-    exec->run(dense::make_row_gather(
-        make_temporary_clone(exec, row_indices).get(), this,
-        make_temporary_output_clone(exec, row_gathered).get()));
+    this->row_gather_impl(row_indices, row_gathered);
 }
 
 
@@ -902,42 +963,7 @@ template <typename ValueType>
 void Dense<ValueType>::row_gather(const Array<int64> *row_indices,
                                   Dense<ValueType> *row_gathered) const
 {
-    dim<2> expected_dim{row_indices->get_num_elems(), this->get_size()[1]};
-    GKO_ASSERT_EQUAL_DIMENSIONS(expected_dim, row_gathered);
-
-    auto exec = this->get_executor();
-
-    this->get_executor()->run(dense::make_row_gather(
-        make_temporary_clone(exec, row_indices).get(), this,
-        make_temporary_output_clone(exec, row_gathered).get()));
-}
-
-
-template <typename ValueType>
-void Dense<ValueType>::column_permute(const Array<int32> *permutation_indices,
-                                      Dense<ValueType> *output) const
-{
-    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
-    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
-    auto exec = this->get_executor();
-
-    exec->run(dense::make_column_permute(
-        make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_output_clone(exec, output).get()));
-}
-
-
-template <typename ValueType>
-void Dense<ValueType>::column_permute(const Array<int64> *permutation_indices,
-                                      Dense<ValueType> *output) const
-{
-    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
-    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
-    auto exec = this->get_executor();
-
-    exec->run(dense::make_column_permute(
-        make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_output_clone(exec, output).get()));
+    this->row_gather_impl(row_indices, row_gathered);
 }
 
 
@@ -962,30 +988,18 @@ std::unique_ptr<LinOp> Dense<ValueType>::column_permute(
 
 
 template <typename ValueType>
-void Dense<ValueType>::inverse_row_permute(
-    const Array<int32> *permutation_indices, Dense<ValueType> *output) const
+void Dense<ValueType>::column_permute(const Array<int32> *permutation_indices,
+                                      Dense<ValueType> *output) const
 {
-    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
-    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
-    auto exec = this->get_executor();
-
-    exec->run(dense::make_inverse_row_permute(
-        make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_output_clone(exec, output).get()));
+    this->column_permute_impl(permutation_indices, output);
 }
 
 
 template <typename ValueType>
-void Dense<ValueType>::inverse_row_permute(
-    const Array<int64> *permutation_indices, Dense<ValueType> *output) const
+void Dense<ValueType>::column_permute(const Array<int64> *permutation_indices,
+                                      Dense<ValueType> *output) const
 {
-    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[0]);
-    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
-    auto exec = this->get_executor();
-
-    exec->run(dense::make_inverse_row_permute(
-        make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_output_clone(exec, output).get()));
+    this->column_permute_impl(permutation_indices, output);
 }
 
 
@@ -1010,30 +1024,18 @@ std::unique_ptr<LinOp> Dense<ValueType>::inverse_row_permute(
 
 
 template <typename ValueType>
-void Dense<ValueType>::inverse_column_permute(
+void Dense<ValueType>::inverse_row_permute(
     const Array<int32> *permutation_indices, Dense<ValueType> *output) const
 {
-    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
-    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
-    auto exec = this->get_executor();
-
-    exec->run(dense::make_inverse_column_permute(
-        make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_output_clone(exec, output).get()));
+    this->inverse_row_permute_impl(permutation_indices, output);
 }
 
 
 template <typename ValueType>
-void Dense<ValueType>::inverse_column_permute(
+void Dense<ValueType>::inverse_row_permute(
     const Array<int64> *permutation_indices, Dense<ValueType> *output) const
 {
-    GKO_ASSERT_EQ(permutation_indices->get_num_elems(), this->get_size()[1]);
-    GKO_ASSERT_EQUAL_DIMENSIONS(this, output);
-    auto exec = this->get_executor();
-
-    exec->run(dense::make_inverse_column_permute(
-        make_temporary_clone(exec, permutation_indices).get(), this,
-        make_temporary_output_clone(exec, output).get()));
+    this->inverse_row_permute_impl(permutation_indices, output);
 }
 
 
@@ -1057,6 +1059,22 @@ std::unique_ptr<LinOp> Dense<ValueType>::inverse_column_permute(
 }
 
 
+template <typename ValueType>
+void Dense<ValueType>::inverse_column_permute(
+    const Array<int32> *permutation_indices, Dense<ValueType> *output) const
+{
+    this->inverse_column_permute_impl(permutation_indices, output);
+}
+
+
+template <typename ValueType>
+void Dense<ValueType>::inverse_column_permute(
+    const Array<int64> *permutation_indices, Dense<ValueType> *output) const
+{
+    this->inverse_column_permute_impl(permutation_indices, output);
+}
+
+
 template <typename ValueType>
 void Dense<ValueType>::extract_diagonal(Diagonal<ValueType> *output) const
 {
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 25ed80c7b85..331be1aa28e 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -933,6 +933,33 @@ class Dense
                                idx % this->get_size()[1]);
     }
 
+    template <typename IndexType>
+    void permute_impl(const Array<IndexType> *permutation, Dense *output) const;
+
+    template <typename IndexType>
+    void inverse_permute_impl(const Array<IndexType> *permutation,
+                              Dense *output) const;
+
+    template <typename IndexType>
+    void row_permute_impl(const Array<IndexType> *permutation,
+                          Dense *output) const;
+
+    template <typename IndexType>
+    void inverse_row_permute_impl(const Array<IndexType> *permutation,
+                                  Dense *output) const;
+
+    template <typename IndexType>
+    void row_gather_impl(const Array<IndexType> *row_indices,
+                         Dense *output) const;
+
+    template <typename IndexType>
+    void column_permute_impl(const Array<IndexType> *permutation,
+                             Dense *output) const;
+
+    template <typename IndexType>
+    void inverse_column_permute_impl(const Array<IndexType> *permutation,
+                                     Dense *output) const;
+
 private:
     Array<value_type> values_;
     size_type stride_;

From 859c1700247179b01f5cc109717cd3f974315092 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 2 Jun 2021 15:41:43 +0200
Subject: [PATCH 0743/1008] avoid copies on empty matrices

This fixes test failures in core tests with Reference disabled
---
 core/matrix/dense.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 51fbd4260bd..09f19e9b360 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -331,7 +331,7 @@ void Dense<ValueType>::compute_norm2_impl(LinOp *result) const
 template <typename ValueType>
 void Dense<ValueType>::convert_to(Dense<ValueType> *result) const
 {
-    if (result->get_size() == this->get_size()) {
+    if (this->get_size() && result->get_size() == this->get_size()) {
         // we need to create a executor-local clone of the target data, that
         // will be copied back later.
         auto exec = this->get_executor();

From d82efcb818d573f23f5e44843b025c470a87a709 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 3 Jun 2021 13:17:54 +0200
Subject: [PATCH 0744/1008] fix single-precision Dense copy on dpcpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 include/ginkgo/core/base/types.hpp | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/include/ginkgo/core/base/types.hpp b/include/ginkgo/core/base/types.hpp
index a2d922c9a9a..dc14b8ea83b 100644
--- a/include/ginkgo/core/base/types.hpp
+++ b/include/ginkgo/core/base/types.hpp
@@ -539,15 +539,13 @@ GKO_ATTRIBUTES constexpr bool operator!=(precision_reduction x,
     _macro(std::complex<double>, std::complex<float>) GKO_NOT_IMPLEMENTED
 
 
-#define GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(_macro)         \
-    GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(_macro);                    \
-    template <>                                                           \
-    _macro(float, float) GKO_NOT_IMPLEMENTED;                             \
-    template <>                                                           \
-    _macro(double, double) GKO_NOT_IMPLEMENTED;                           \
-    template <>                                                           \
-    _macro(std::complex<float>, std::complex<float>) GKO_NOT_IMPLEMENTED; \
-    template <>                                                           \
+#define GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(_macro) \
+    GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(_macro);            \
+    template _macro(float, float);                                \
+    template <>                                                   \
+    _macro(double, double) GKO_NOT_IMPLEMENTED;                   \
+    template _macro(std::complex<float>, std::complex<float>);    \
+    template <>                                                   \
     _macro(std::complex<double>, std::complex<double>) GKO_NOT_IMPLEMENTED
 #else
 

From 1ddbfcdffbf5dbcc5f4b45cca39e833c954e92e7 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 2 Jun 2021 17:13:58 +0200
Subject: [PATCH 0745/1008] Run the CUDA tests with a few configurations

---
 .gitlab-ci.yml    | 14 ++++++--------
 .gitlab/image.yml |  3 +--
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8a4f31dce4c..65559c5ca07 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -64,7 +64,6 @@ include:
     - if [ -n "${CUDA_ARCH}" ]; then
       CUDA_ARCH_STR=-DGINKGO_CUDA_ARCHITECTURES=${CUDA_ARCH};
       CUDA_HOST_STR=-DCMAKE_CUDA_HOST_COMPILER=$(which ${CXX_COMPILER});
-      export CUDA_VISIBLE_DEVICES=$((RANDOM % 2));
       fi
     - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
         -GNinja
@@ -98,7 +97,6 @@ include:
     - if [ -n "${CUDA_ARCH}" ]; then
       CUDA_ARCH_STR=-DGINKGO_CUDA_ARCHITECTURES=${CUDA_ARCH};
       CUDA_HOST_STR=-DCMAKE_CUDA_HOST_COMPILER=$(which ${CXX_COMPILER});
-      export CUDA_VISIBLE_DEVICES=$((RANDOM % 2));
       fi
     - if [ -n "${SYCL_DEVICE_FILTER}" ]; then export SYCL_DEVICE_FILTER; fi
     - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
@@ -509,7 +507,7 @@ build/cuda102/intel/cuda/debug/static:
 
 # cuda 11.0 and friends
 build/cuda110/gcc/cuda/debug/shared:
-  <<: *default_build
+  <<: *default_build_with_test
   extends:
     - .full_test_condition
     - .use_gko-cuda110-gnu9-llvm9-intel2020
@@ -519,10 +517,10 @@ build/cuda110/gcc/cuda/debug/shared:
     BUILD_CUDA: "ON"
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
-    CUDA_ARCH: 35
+    CUDA_ARCH: 61
 
 build/cuda110/clang/cuda/release/static:
-  <<: *default_build
+  <<: *default_build_with_test
   extends:
     - .full_test_condition
     - .use_gko-cuda110-gnu9-llvm9-intel2020
@@ -534,10 +532,10 @@ build/cuda110/clang/cuda/release/static:
     BUILD_CUDA: "ON"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
-    CUDA_ARCH: 35
+    CUDA_ARCH: 61
 
 build/cuda110/intel/cuda/debug/static:
-  <<: *default_build
+  <<: *default_build_with_test
   extends:
     - .quick_test_condition
     - .use_gko-cuda110-gnu9-llvm9-intel2020
@@ -550,7 +548,7 @@ build/cuda110/intel/cuda/debug/static:
     BUILD_TYPE: "Debug"
     FAST_TESTS: "ON"
     BUILD_SHARED_LIBS: "OFF"
-    CUDA_ARCH: 35
+    CUDA_ARCH: 61
 
 # HIP AMD
 build/amd/gcc/hip/debug/shared:
diff --git a/.gitlab/image.yml b/.gitlab/image.yml
index 9779553522e..5ea3889d8df 100644
--- a/.gitlab/image.yml
+++ b/.gitlab/image.yml
@@ -65,8 +65,7 @@
   image: ginkgohub/cuda:110-gnu9-llvm9-intel2020
   tags:
     - private_ci
-    - controller
-    - cpu
+    - nvidia-gpu
 
 .use_gko-amd-gnu8-llvm7:
   image: ginkgohub/rocm:gnu8-llvm7

From 76011ff44a29b94f140c40fcf24a66067b1cc010 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <19565938+yhmtsai@users.noreply.github.com>
Date: Fri, 4 Jun 2021 17:48:15 +0200
Subject: [PATCH 0746/1008] add `-T version=14.25`

---
 .github/workflows/windows-msvc-cuda.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/windows-msvc-cuda.yml b/.github/workflows/windows-msvc-cuda.yml
index fba49eb6adb..3eb26f008e5 100644
--- a/.github/workflows/windows-msvc-cuda.yml
+++ b/.github/workflows/windows-msvc-cuda.yml
@@ -45,5 +45,5 @@ jobs:
         refreshenv
         mkdir build
         cd build
-        cmake -DGINKGO_BUILD_CUDA=ON -DGINKGO_BUILD_OMP=OFF -DGINKGO_MIXED_PRECISION=${{ matrix.config.mixed }} -DGINKGO_CUDA_ARCHITECTURES=60 ..
+        cmake -T version=14.25 -DGINKGO_BUILD_CUDA=ON -DGINKGO_BUILD_OMP=OFF -DGINKGO_MIXED_PRECISION=${{ matrix.config.mixed }} -DGINKGO_CUDA_ARCHITECTURES=60 ..
         cmake --build . -j4 --config Release

From aabf986c13c25f0fd8e7b9c9c983f748f4cdd657 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Sat, 5 Jun 2021 00:17:57 +0200
Subject: [PATCH 0747/1008] pass toolset to ensure the same toolset

---
 CMakeLists.txt              |  8 ++++++--
 cmake/package_helpers.cmake | 12 ++++++++++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7c4eaf29caa..92375a92d9b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -352,8 +352,12 @@ else()
 endif()
 
 file(MAKE_DIRECTORY ${Ginkgo_BINARY_DIR}/test_install)
+set(TOOLSET "")
+if (NOT "${CMAKE_GENERATOR_TOOLSET}" STREQUAL "")
+    set(TOOLSET "-T ${CMAKE_GENERATOR_TOOLSET}")
+endif()
 add_custom_target(test_install
-    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} -H${Ginkgo_SOURCE_DIR}/test_install
+    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${TOOLSET} -H${Ginkgo_SOURCE_DIR}/test_install
     -B${Ginkgo_BINARY_DIR}/test_install
     -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
     -DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX}/${GINKGO_INSTALL_CONFIG_DIR}
@@ -370,7 +374,7 @@ add_custom_target(test_install
     COMMENT "Running a test on the installed binaries. This requires running `(sudo) make install` first.")
 
 add_custom_target(test_exportbuild
-    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} -H${Ginkgo_SOURCE_DIR}/test_exportbuild
+    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${TOOLSET} -H${Ginkgo_SOURCE_DIR}/test_exportbuild
     -B${Ginkgo_BINARY_DIR}/test_exportbuild
     -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
     -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
diff --git a/cmake/package_helpers.cmake b/cmake/package_helpers.cmake
index 78385f1d1b7..e2678f90522 100644
--- a/cmake/package_helpers.cmake
+++ b/cmake/package_helpers.cmake
@@ -18,7 +18,11 @@ function(ginkgo_load_git_package package_name package_url package_tag)
     endif()
     configure_file(${PACKAGE_DOWNLOADER_SCRIPT}
         download/CMakeLists.txt)
-    execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
+    set(TOOLSET "")
+    if (NOT "${CMAKE_GENERATOR_TOOLSET}" STREQUAL "")
+        set(TOOLSET "-T ${CMAKE_GENERATOR_TOOLSET}")
+    endif()
+    execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" "${TOOLSET}" .
         RESULT_VARIABLE result
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/download)
     if(result)
@@ -67,7 +71,11 @@ function(ginkgo_load_and_configure_package package_name package_url package_hash
     endif()
     configure_file(${NON_CMAKE_PACKAGE_DOWNLOADER_SCRIPT}
         download/CMakeLists.txt)
-    execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
+    set(TOOLSET "")
+    if (NOT "${CMAKE_GENERATOR_TOOLSET}" STREQUAL "")
+        set(TOOLSET "-T ${CMAKE_GENERATOR_TOOLSET}")
+    endif()
+    execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" "${TOOLSET}" .
         RESULT_VARIABLE result
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/download)
     if(result)

From 18b05abb2be26a9880da7b336bda6650366f9103 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Sat, 5 Jun 2021 01:04:26 +0200
Subject: [PATCH 0748/1008] try eliminate space

---
 CMakeLists.txt              | 2 +-
 cmake/package_helpers.cmake | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 92375a92d9b..314439b49ab 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -354,7 +354,7 @@ endif()
 file(MAKE_DIRECTORY ${Ginkgo_BINARY_DIR}/test_install)
 set(TOOLSET "")
 if (NOT "${CMAKE_GENERATOR_TOOLSET}" STREQUAL "")
-    set(TOOLSET "-T ${CMAKE_GENERATOR_TOOLSET}")
+    set(TOOLSET "-T${CMAKE_GENERATOR_TOOLSET}")
 endif()
 add_custom_target(test_install
     COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${TOOLSET} -H${Ginkgo_SOURCE_DIR}/test_install
diff --git a/cmake/package_helpers.cmake b/cmake/package_helpers.cmake
index e2678f90522..afa0b4033fc 100644
--- a/cmake/package_helpers.cmake
+++ b/cmake/package_helpers.cmake
@@ -20,7 +20,7 @@ function(ginkgo_load_git_package package_name package_url package_tag)
         download/CMakeLists.txt)
     set(TOOLSET "")
     if (NOT "${CMAKE_GENERATOR_TOOLSET}" STREQUAL "")
-        set(TOOLSET "-T ${CMAKE_GENERATOR_TOOLSET}")
+        set(TOOLSET "-T${CMAKE_GENERATOR_TOOLSET}")
     endif()
     execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" "${TOOLSET}" .
         RESULT_VARIABLE result
@@ -73,7 +73,7 @@ function(ginkgo_load_and_configure_package package_name package_url package_hash
         download/CMakeLists.txt)
     set(TOOLSET "")
     if (NOT "${CMAKE_GENERATOR_TOOLSET}" STREQUAL "")
-        set(TOOLSET "-T ${CMAKE_GENERATOR_TOOLSET}")
+        set(TOOLSET "-T${CMAKE_GENERATOR_TOOLSET}")
     endif()
     execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" "${TOOLSET}" .
         RESULT_VARIABLE result

From 78435a935556324b37119ce4f70ec21b8cf7188b Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@posteo.de>
Date: Fri, 4 Jun 2021 16:40:09 +0200
Subject: [PATCH 0749/1008] clarifies the documentation for dense norm
 computation

---
 include/ginkgo/core/matrix/dense.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 331be1aa28e..eb4243ade18 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -680,7 +680,7 @@ class Dense
     }
 
     /**
-     * Computes the Euclidian (L^2) norm of this matrix.
+     * Computes the column-wise Euclidian (L^2) norm of this matrix.
      *
      * @param result  a Dense row vector, used to store the norm
      *                (the number of columns in the vector must match the number

From 56d1bde44f7913b1549a6c4f9c443fdc88c3df7d Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Fri, 4 Jun 2021 17:07:13 +0200
Subject: [PATCH 0750/1008] adds Marcel Koch to contributors.txt

---
 contributors.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contributors.txt b/contributors.txt
index a7a558499e2..c3637859fd2 100644
--- a/contributors.txt
+++ b/contributors.txt
@@ -15,6 +15,7 @@ Heroux Mike <maherou@sandia.gov> Sandia National Laboratories
 Hoemmen Mark <mhoemme@sandia.gov> Sandia National Laboratories
 Holeksa Claudius <mail@keldu.de> Karlsruhe Institute of Technology
 Kashi Aditya <aditya.kashi@kit.edu> Karlsruhe Institute of Technology
+Koch Marcel <marcel.koch@kit.edu> Karlsruhe Institute of Technology
 Maier Matthias <matthias@43-1.org> Texas A&M University
 Nayak Pratik <pratik.nayak@kit.edu> Karlsruhe Institute of Technology
 Olenik Gregor <go@hpsim.de> HPSim

From c185d7a07e5a4c9926a50d49282beb81abf14d1c Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 4 Jun 2021 14:49:09 +0200
Subject: [PATCH 0751/1008] fix no-circular-deps header compilation for CUDA

---
 cmake/build_helpers.cmake | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/cmake/build_helpers.cmake b/cmake/build_helpers.cmake
index 3a264660733..ac1574939af 100644
--- a/cmake/build_helpers.cmake
+++ b/cmake/build_helpers.cmake
@@ -55,6 +55,14 @@ function(ginkgo_check_headers target)
     list(FILTER HIP_HEADERS EXCLUDE REGEX "^test.*")
 
     set(SOURCES "")
+    # if we have any CUDA files in there, compile everything as CUDA
+    if (CUDA_HEADERS)
+        set(CUDA_HEADERS ${CUDA_HEADERS} ${CXX_HEADERS})
+        set(CXX_HEADERS "")
+        if (HIP_HEADERS) 
+            message(FATAL_ERROR "Mixing CUDA and HIP files in header check")
+        endif()
+    endif()
     foreach(HEADER ${CUDA_HEADERS})
         set(HEADER_SOURCEFILE "${CMAKE_CURRENT_BINARY_DIR}/${HEADER}.cu")
         file(WRITE "${HEADER_SOURCEFILE}" "#include \"${HEADER}\"")

From 882f48876df5ea6ccc1fa1d35b206b1c94778d53 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 31 May 2021 15:05:34 +0200
Subject: [PATCH 0752/1008] add mixed-spmv example

---
 examples/CMakeLists.txt             |   1 +
 examples/mixed-spmv/CMakeLists.txt  |  13 ++
 examples/mixed-spmv/build.sh        |  16 ++
 examples/mixed-spmv/data/A.mtx      | 122 +++++++++++++
 examples/mixed-spmv/doc/builds-on   |   1 +
 examples/mixed-spmv/doc/intro.dox   |  19 ++
 examples/mixed-spmv/doc/kind        |   1 +
 examples/mixed-spmv/doc/results.dox |  16 ++
 examples/mixed-spmv/doc/short-intro |   1 +
 examples/mixed-spmv/doc/tooltip     |   1 +
 examples/mixed-spmv/mixed-spmv.cpp  | 263 ++++++++++++++++++++++++++++
 11 files changed, 454 insertions(+)
 create mode 100644 examples/mixed-spmv/CMakeLists.txt
 create mode 100755 examples/mixed-spmv/build.sh
 create mode 100644 examples/mixed-spmv/data/A.mtx
 create mode 100644 examples/mixed-spmv/doc/builds-on
 create mode 100644 examples/mixed-spmv/doc/intro.dox
 create mode 100644 examples/mixed-spmv/doc/kind
 create mode 100644 examples/mixed-spmv/doc/results.dox
 create mode 100644 examples/mixed-spmv/doc/short-intro
 create mode 100644 examples/mixed-spmv/doc/tooltip
 create mode 100644 examples/mixed-spmv/mixed-spmv.cpp

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 559dea1b5c1..81f71466cc3 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -22,6 +22,7 @@ set(EXAMPLES_LIST
     custom-stopping-criterion
     ginkgo-overhead
     minimal-cuda-solver
+    mixed-spmv
     par-ilu-convergence
     performance-debugging
     preconditioner-export
diff --git a/examples/mixed-spmv/CMakeLists.txt b/examples/mixed-spmv/CMakeLists.txt
new file mode 100644
index 00000000000..044b3a38bb8
--- /dev/null
+++ b/examples/mixed-spmv/CMakeLists.txt
@@ -0,0 +1,13 @@
+cmake_minimum_required(VERSION 3.9)
+project(mixed-spmv)
+
+# We only need to find Ginkgo if we build this example stand-alone
+if (NOT GINKGO_BUILD_EXAMPLES)
+    find_package(Ginkgo 1.4.0 REQUIRED)
+endif()
+
+add_executable(mixed-spmv mixed-spmv.cpp)
+target_link_libraries(mixed-spmv Ginkgo::ginkgo)
+
+# Copy the data files to the execution directory
+configure_file(data/A.mtx data/A.mtx COPYONLY)
diff --git a/examples/mixed-spmv/build.sh b/examples/mixed-spmv/build.sh
new file mode 100755
index 00000000000..3137d5656c1
--- /dev/null
+++ b/examples/mixed-spmv/build.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# set up script
+if [ $# -ne 1 ]; then
+    echo -e "Usage: $0 GINKGO_BUILD_DIRECTORY"
+    exit 1
+fi
+BUILD_DIR=$1
+THIS_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" &>/dev/null && pwd )
+
+source ${THIS_DIR}/../build-setup.sh
+
+# build
+${CXX} -std=c++14 -o ${THIS_DIR}/mixed-spmv ${THIS_DIR}/mixed-spmv.cpp \
+       -I${THIS_DIR}/../../include -I${BUILD_DIR}/include \
+       -L${THIS_DIR} ${LINK_FLAGS}
diff --git a/examples/mixed-spmv/data/A.mtx b/examples/mixed-spmv/data/A.mtx
new file mode 100644
index 00000000000..ec6363a2fd2
--- /dev/null
+++ b/examples/mixed-spmv/data/A.mtx
@@ -0,0 +1,122 @@
+%%MatrixMarket matrix coordinate pattern symmetric
+%-------------------------------------------------------------------------------
+% UF Sparse Matrix Collection, Tim Davis
+% http://www.cise.ufl.edu/research/sparse/matrices/HB/bcspwr02
+% name: HB/bcspwr02
+% [SYMMETRIC STRUCTURE OF A SMALL TEST POWER SYSTEM]
+% id: 14
+% date: 1981
+% author: B. Dembart, J. Lewis
+% ed: I. Duff, R. Grimes, J. Lewis
+% fields: title A name id date author ed kind
+% kind: power network problem
+%-------------------------------------------------------------------------------
+49 49 108
+1 1
+2 1
+2 2
+3 2
+4 2
+3 3
+30 3
+37 3
+4 4
+5 5
+28 5
+32 5
+6 6
+15 6
+33 6
+7 7
+17 7
+20 7
+34 7
+8 8
+17 8
+37 8
+9 9
+14 9
+10 10
+14 10
+11 11
+14 11
+12 12
+15 12
+13 13
+16 13
+14 14
+15 14
+41 14
+15 15
+16 15
+36 15
+16 16
+22 16
+17 17
+18 18
+19 18
+30 18
+37 18
+19 19
+21 19
+20 20
+28 20
+31 20
+21 21
+45 21
+22 22
+23 22
+38 22
+45 22
+23 23
+24 23
+25 23
+26 23
+38 23
+49 23
+24 24
+25 25
+26 26
+27 27
+28 27
+29 27
+28 28
+32 28
+35 28
+29 29
+30 30
+31 31
+32 32
+33 32
+34 32
+37 32
+33 33
+42 33
+34 34
+35 35
+36 35
+36 36
+37 37
+38 37
+38 38
+39 39
+40 39
+49 39
+40 40
+41 41
+42 41
+43 41
+42 42
+44 42
+43 43
+44 44
+45 45
+46 45
+49 45
+46 46
+47 46
+48 46
+49 46
+47 47
+48 48
+49 49
diff --git a/examples/mixed-spmv/doc/builds-on b/examples/mixed-spmv/doc/builds-on
new file mode 100644
index 00000000000..8b137891791
--- /dev/null
+++ b/examples/mixed-spmv/doc/builds-on
@@ -0,0 +1 @@
+
diff --git a/examples/mixed-spmv/doc/intro.dox b/examples/mixed-spmv/doc/intro.dox
new file mode 100644
index 00000000000..1e2d4744bb7
--- /dev/null
+++ b/examples/mixed-spmv/doc/intro.dox
@@ -0,0 +1,19 @@
+<a name="Intro"></a>
+<h1>Introduction</h1>
+This mixed spmv example should give the usage of Ginkgo mixed precision. This example is meant for you to understand
+how Ginkgo works with different precision of data. We encourage you to play with the code,
+change the parameters and see what is best suited for your purposes.
+
+<h3> About the example </h3>
+Each example has the following sections:
+<ol>
+    <li> <b>Introduction:</b>This gives an overview of the example and mentions
+    any interesting aspects in the example that might help the reader.
+    <li> <b>The commented program:</b> This section is intended for you to
+    understand the details of the example so that you can play with it and understand
+    Ginkgo and its features better.
+    <li> <b>Results:</b> This section shows the results of the code when run. Though the
+    results may not be completely the same, you can expect the behaviour to be similar.
+    <li> <b>The plain program:</b> This is the complete code without any comments to have
+    an complete overview of the code.
+</ol>
diff --git a/examples/mixed-spmv/doc/kind b/examples/mixed-spmv/doc/kind
new file mode 100644
index 00000000000..15a13db4511
--- /dev/null
+++ b/examples/mixed-spmv/doc/kind
@@ -0,0 +1 @@
+basic
diff --git a/examples/mixed-spmv/doc/results.dox b/examples/mixed-spmv/doc/results.dox
new file mode 100644
index 00000000000..3750bb12b95
--- /dev/null
+++ b/examples/mixed-spmv/doc/results.dox
@@ -0,0 +1,16 @@
+<h1>Results</h1>
+The following is the expected result (omp):
+
+@code{.cpp}
+
+High Precision time(s): 1.2272200000e-05
+High Precision result norm: 1.1581546305e+01
+Low Precision time(s): 1.2360400000e-05
+Low Precision relative error: 4.4554843634e-08
+Hp * Lp -> Hp time(s): 1.2302300000e-05
+Hp * Lp -> Hp relative error: 1.4490698720e-08
+Lp * Lp -> Hp time(s): 1.2614800000e-05
+Lp * Lp -> Hp relative error: 1.4490698720e-08
+@endcode
+
+<h3> Comments about programming and debugging </h3>
diff --git a/examples/mixed-spmv/doc/short-intro b/examples/mixed-spmv/doc/short-intro
new file mode 100644
index 00000000000..daf3b4d36bd
--- /dev/null
+++ b/examples/mixed-spmv/doc/short-intro
@@ -0,0 +1 @@
+The mixed spmv example.
diff --git a/examples/mixed-spmv/doc/tooltip b/examples/mixed-spmv/doc/tooltip
new file mode 100644
index 00000000000..7db892e5a27
--- /dev/null
+++ b/examples/mixed-spmv/doc/tooltip
@@ -0,0 +1 @@
+Mixed precision apply. Read a matrix from a file and generate right hand side randomly.
diff --git a/examples/mixed-spmv/mixed-spmv.cpp b/examples/mixed-spmv/mixed-spmv.cpp
new file mode 100644
index 00000000000..4ce1cbe8c35
--- /dev/null
+++ b/examples/mixed-spmv/mixed-spmv.cpp
@@ -0,0 +1,263 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+// @sect3{Include files}
+
+// This is the main ginkgo header file.
+#include <ginkgo/ginkgo.hpp>
+
+// Add the fstream header to read from data from files.
+#include <fstream>
+// Add the C++ iostream header to output information to the console.
+#include <iostream>
+// Add the STL map header for the executor selection.
+#include <map>
+// Add the string manipulation header to handle strings.
+#include <string>
+// Add the timing header for timing.
+#include <chrono>
+// Add the random header to generate random vectors.
+#include <random>
+
+namespace {
+
+
+/**
+ * Generate a random value.
+ *
+ * @tparam ValueType  valuetype of the value
+ * @tparam ValueDistribution  type of value distribution
+ * @tparam Engine  type of random engine
+ *
+ * @param value_dist  distribution of array values
+ * @param engine  a random engine
+ *
+ * @return ValueType
+ */
+template <typename ValueType, typename ValueDistribution, typename Engine>
+typename std::enable_if<!gko::is_complex_s<ValueType>::value, ValueType>::type
+get_rand_value(ValueDistribution &&value_dist, Engine &&gen)
+{
+    return value_dist(gen);
+}
+
+/**
+ * Specialization for complex types.
+ *
+ * @copydoc get_rand_value
+ */
+template <typename ValueType, typename ValueDistribution, typename Engine>
+typename std::enable_if<gko::is_complex_s<ValueType>::value, ValueType>::type
+get_rand_value(ValueDistribution &&value_dist, Engine &&gen)
+{
+    return ValueType(value_dist(gen), value_dist(gen));
+}
+
+/**
+ * timing the apply operation A->apply(b, x). It will runs 2 warmup and get
+ * average time among 10 times.
+ *
+ * @return seconds
+ */
+double timing(std::shared_ptr<const gko::Executor> exec,
+              std::shared_ptr<gko::LinOp> A, std::shared_ptr<gko::LinOp> b,
+              std::shared_ptr<gko::LinOp> x)
+{
+    int warmup = 2;
+    int rep = 10;
+    for (int i = 0; i < warmup; i++) {
+        A->apply(lend(b), lend(x));
+    }
+    exec->synchronize();
+    auto start = std::chrono::steady_clock::now();
+    for (int i = 0; i < rep; i++) {
+        A->apply(lend(b), lend(x));
+    }
+    exec->synchronize();
+    auto stop = std::chrono::steady_clock::now();
+    std::chrono::duration<double> duration_time = stop - start;
+    return duration_time.count() / rep;
+}
+
+
+}  // namespace
+
+
+int main(int argc, char *argv[])
+{
+    // Use some shortcuts. In Ginkgo, vectors are seen as a gko::matrix::Dense
+    // with one column/one row. The advantage of this concept is that using
+    // multiple vectors is a now a natural extension of adding columns/rows are
+    // necessary.
+    using HighPrecision = double;
+    using RealValueType = gko::remove_complex<HighPrecision>;
+    using LowPrecision = float;
+    using IndexType = int;
+    using hp_vec = gko::matrix::Dense<HighPrecision>;
+    using lp_vec = gko::matrix::Dense<LowPrecision>;
+    using real_vec = gko::matrix::Dense<RealValueType>;
+    // The gko::matrix::Ell class is used here, but any other matrix class such
+    // as gko::matrix::Coo, gko::matrix::Hybrid, gko::matrix::Csr or
+    // gko::matrix::Sellp could also be used.
+    // Note. the behavior will depends GINKGO_MIXED_PRECISION flags and the
+    // actual implementation from different matrices.
+    using hp_mtx = gko::matrix::Ell<HighPrecision, IndexType>;
+    using lp_mtx = gko::matrix::Ell<LowPrecision, IndexType>;
+
+    // Print the ginkgo version information.
+    std::cout << gko::version_info::get() << std::endl;
+
+    if (argc == 2 && (std::string(argv[1]) == "--help")) {
+        std::cerr << "Usage: " << argv[0] << " [executor] " << std::endl;
+        std::exit(-1);
+    }
+
+    // @sect3{Where do you want to run your operation?}
+    // The gko::Executor class is one of the cornerstones of Ginkgo. Currently,
+    // we have support for
+    // an gko::OmpExecutor, which uses OpenMP multi-threading in most of its
+    // kernels, a gko::ReferenceExecutor, a single threaded specialization of
+    // the OpenMP executor and a gko::CudaExecutor which runs the code on a
+    // NVIDIA GPU if available.
+    // @note With the help of C++, you see that you only ever need to change the
+    // executor and all the other functions/ routines within Ginkgo should
+    // automatically work and run on the executor with any other changes.
+    const auto executor_string = argc >= 2 ? argv[1] : "reference";
+    std::map<std::string, std::function<std::shared_ptr<gko::Executor>()>>
+        exec_map{
+            {"omp", [] { return gko::OmpExecutor::create(); }},
+            {"cuda",
+             [] {
+                 return gko::CudaExecutor::create(0, gko::OmpExecutor::create(),
+                                                  true);
+             }},
+            {"hip",
+             [] {
+                 return gko::HipExecutor::create(0, gko::OmpExecutor::create(),
+                                                 true);
+             }},
+            {"dpcpp",
+             [] {
+                 return gko::DpcppExecutor::create(0,
+                                                   gko::OmpExecutor::create());
+             }},
+            {"reference", [] { return gko::ReferenceExecutor::create(); }}};
+
+    // executor where Ginkgo will perform the computation
+    const auto exec = exec_map.at(executor_string)();  // throws if not valid
+
+    // @sect3{Preparing your data and transfer to the proper device.}
+    // Read the matrix using the @ref read function and set the right hand side
+    // randomly.
+    // @note Ginkgo uses C++ smart pointers to automatically manage memory. To
+    // this end, we use our own object ownership transfer functions that under
+    // the hood call the required smart pointer functions to manage object
+    // ownership. The gko::share , gko::give and gko::lend are the functions
+    // that you would need to use.
+
+    // read the matrix into HighPrecision and LowPrecision.
+    auto hp_A = share(gko::read<hp_mtx>(std::ifstream("data/A.mtx"), exec));
+    auto lp_A = share(gko::read<lp_mtx>(std::ifstream("data/A.mtx"), exec));
+    auto host_b =
+        hp_vec::create(exec->get_master(), gko::dim<2>{hp_A->get_size()[1], 1});
+    std::ranlux48 rand_engine(30);
+    auto dist = std::uniform_real_distribution<RealValueType>(0.0, 1.0);
+    for (int i = 0; i < host_b->get_size()[0]; i++) {
+        host_b->get_values()[i] =
+            get_rand_value<HighPrecision>(dist, rand_engine);
+    }
+    auto hp_b = share(hp_vec::create(exec));
+    auto lp_b = share(lp_vec::create(exec));
+    hp_b->copy_from(lend(host_b));
+    lp_b->copy_from(lend(hp_b));
+    auto hp_x = share(hp_vec::create(
+        exec, gko::dim<2>{hp_A->get_size()[0], hp_b->get_size()[1]}));
+    auto lp_x = share(lp_vec::create(
+        exec, gko::dim<2>{lp_A->get_size()[0], lp_b->get_size()[1]}));
+    auto hplp_x = share(hp_x->clone());
+    auto lplp_x = share(hp_x->clone());
+
+    // @sect3{Measure the time of applu}
+    // We measure the time among different combination of apply operation.
+
+    // Hp * Hp -> Hp
+    auto hp_sec = timing(exec, hp_A, hp_b, hp_x);
+    // Lp * Lp -> Lp
+    auto lp_sec = timing(exec, lp_A, lp_b, lp_x);
+    // Hp * Lp -> Hp
+    auto hplp_sec = timing(exec, hp_A, lp_b, hplp_x);
+    // Lp * Lp -> Lp
+    auto lplp_sec = timing(exec, lp_A, lp_b, lplp_x);
+
+
+    // To measure error of result.
+    // neg_one are objects that represent the numbers which allow for a
+    // uniform interface when computing on any device. To compute the residual,
+    // all you need to do is call the add_scaled method, which in this case is
+    // an axpy and equivalent to the LAPACK axpy routine. Finally, you compute
+    // the euclidean 2-norm with the compute_norm2 function.
+    auto neg_one = gko::initialize<hp_vec>({-1.0}, exec);
+    auto hp_x_norm = gko::initialize<real_vec>({0.0}, exec->get_master());
+    auto lp_diff_norm = gko::initialize<real_vec>({0.0}, exec->get_master());
+    auto hplp_diff_norm = gko::initialize<real_vec>({0.0}, exec->get_master());
+    auto lplp_diff_norm = gko::initialize<real_vec>({0.0}, exec->get_master());
+    auto lp_diff = hp_x->clone();
+    auto hplp_diff = hp_x->clone();
+    auto lplp_diff = hp_x->clone();
+
+    hp_x->compute_norm2(lend(hp_x_norm));
+    lp_diff->add_scaled(lend(neg_one), lend(lp_x));
+    lp_diff->compute_norm2(lend(lp_diff_norm));
+    hplp_diff->add_scaled(lend(neg_one), lend(hplp_x));
+    hplp_diff->compute_norm2(lend(hplp_diff_norm));
+    lplp_diff->add_scaled(lend(neg_one), lend(lplp_x));
+    lplp_diff->compute_norm2(lend(lplp_diff_norm));
+    exec->synchronize();
+
+    std::cout.precision(10);
+    std::cout << std::scientific;
+    std::cout << "High Precision time(s): " << hp_sec << std::endl;
+    std::cout << "High Precision result norm: " << hp_x_norm->get_values()[0]
+              << std::endl;
+    std::cout << "Low Precision time(s): " << lp_sec << std::endl;
+    std::cout << "Low Precision relative error: "
+              << lp_diff_norm->get_values()[0] / hp_x_norm->get_values()[0]
+              << "\n";
+    std::cout << "Hp * Lp -> Hp time(s): " << hplp_sec << std::endl;
+    std::cout << "Hp * Lp -> Hp relative error: "
+              << hplp_diff_norm->get_values()[0] / hp_x_norm->get_values()[0]
+              << "\n";
+    std::cout << "Lp * Lp -> Hp time(s): " << lplp_sec << std::endl;
+    std::cout << "Lp * Lp -> Hp relative error: "
+              << lplp_diff_norm->get_values()[0] / hp_x_norm->get_values()[0]
+              << "\n";
+}

From 6aa1682ca8d299b899c623658606f620586f95da Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 1 Jun 2021 16:39:43 +0200
Subject: [PATCH 0753/1008] clone x in timing, add mixed-precision ex kind

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 doc/scripts/examples.pl                       |  6 +++--
 .../adaptiveprecision-blockjacobi/doc/kind    |  2 +-
 examples/mixed-precision-ir/doc/kind          |  2 +-
 examples/mixed-spmv/doc/kind                  |  2 +-
 examples/mixed-spmv/mixed-spmv.cpp            | 22 +++++++++++++------
 5 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/doc/scripts/examples.pl b/doc/scripts/examples.pl
index e9f273c1931..87632d2e721 100644
--- a/doc/scripts/examples.pl
+++ b/doc/scripts/examples.pl
@@ -34,6 +34,7 @@
     "logging"         => ',height=.25,width=.25,fillcolor="gold"',
     "stopping-criteria"         => ',height=.25,width=.25,fillcolor="deepskyblue"',
     "preconditioners" => ',height=.25,width=.25,fillcolor="crimson"',
+    "mixed-precision" => ',height=.25,width=.25,fillcolor="aquamarine"',
     "unfinished"     => ',height=.25,width=.25,style="dashed"',
     );
 
@@ -194,6 +195,7 @@
     "logging"         => 'Logging in Ginkgo',
     "stopping-criteria"         => 'Stopping criteria',
     "preconditioners" => 'Preconditioners',
+    "mixed-precision" => 'Mixed Precision',
     "unfinished"     => 'Unfinished codes',
     );
 
@@ -211,12 +213,12 @@
 }
 # now add connections to make sure they appear nicely next to each other
 # in the legend
-print "  basic -- techniques -- logging -- stopping_criteria -- preconditioners -- unfinished;\n";
+print "  basic -- techniques -- logging -- stopping_criteria -- preconditioners -- mixed_precision -- unfinished;\n";
 
 # we need to tell 'dot' that all of these are at the same
 # rank to ensure they appear next to (as opposed to atop)
 # each other
-print "  {rank=same; basic, techniques, logging, stopping_criteria, preconditioners, unfinished}";
+print "  {rank=same; basic, techniques, logging, stopping_criteria, preconditioners, mixed_precision, unfinished}";
 
 # end the graph
 print "}\n";
diff --git a/examples/adaptiveprecision-blockjacobi/doc/kind b/examples/adaptiveprecision-blockjacobi/doc/kind
index 53a96d5771f..082f7497da3 100644
--- a/examples/adaptiveprecision-blockjacobi/doc/kind
+++ b/examples/adaptiveprecision-blockjacobi/doc/kind
@@ -1 +1 @@
-preconditioners
+mixed-precision
diff --git a/examples/mixed-precision-ir/doc/kind b/examples/mixed-precision-ir/doc/kind
index c1d9154931a..082f7497da3 100644
--- a/examples/mixed-precision-ir/doc/kind
+++ b/examples/mixed-precision-ir/doc/kind
@@ -1 +1 @@
-techniques
+mixed-precision
diff --git a/examples/mixed-spmv/doc/kind b/examples/mixed-spmv/doc/kind
index 15a13db4511..082f7497da3 100644
--- a/examples/mixed-spmv/doc/kind
+++ b/examples/mixed-spmv/doc/kind
@@ -1 +1 @@
-basic
+mixed-precision
diff --git a/examples/mixed-spmv/mixed-spmv.cpp b/examples/mixed-spmv/mixed-spmv.cpp
index 4ce1cbe8c35..deb64d983a3 100644
--- a/examples/mixed-spmv/mixed-spmv.cpp
+++ b/examples/mixed-spmv/mixed-spmv.cpp
@@ -97,15 +97,23 @@ double timing(std::shared_ptr<const gko::Executor> exec,
     for (int i = 0; i < warmup; i++) {
         A->apply(lend(b), lend(x));
     }
-    exec->synchronize();
-    auto start = std::chrono::steady_clock::now();
+    double total_sec = 0;
     for (int i = 0; i < rep; i++) {
-        A->apply(lend(b), lend(x));
+        auto xx = x->clone();
+        exec->synchronize();
+        auto start = std::chrono::steady_clock::now();
+        A->apply(lend(b), lend(xx));
+        exec->synchronize();
+        auto stop = std::chrono::steady_clock::now();
+        std::chrono::duration<double> duration_time = stop - start;
+        total_sec += duration_time.count();
+        if (i + 1 == rep) {
+            // copy the result back to x
+            x->copy_from(lend(xx));
+        }
     }
-    exec->synchronize();
-    auto stop = std::chrono::steady_clock::now();
-    std::chrono::duration<double> duration_time = stop - start;
-    return duration_time.count() / rep;
+
+    return total_sec / rep;
 }
 
 

From 11dd67d798d9c6f5ea03a3314e6868d902d43189 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 4 Jun 2021 20:57:46 +0200
Subject: [PATCH 0754/1008] update doc, add new op, use another mtx

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 examples/mixed-spmv/data/A.mtx      | 184 ++++++++++------------------
 examples/mixed-spmv/doc/results.dox |  18 +--
 examples/mixed-spmv/mixed-spmv.cpp  |  56 ++++++---
 3 files changed, 111 insertions(+), 147 deletions(-)

diff --git a/examples/mixed-spmv/data/A.mtx b/examples/mixed-spmv/data/A.mtx
index ec6363a2fd2..affdafd027d 100644
--- a/examples/mixed-spmv/data/A.mtx
+++ b/examples/mixed-spmv/data/A.mtx
@@ -1,122 +1,68 @@
-%%MatrixMarket matrix coordinate pattern symmetric
+%%MatrixMarket matrix coordinate real symmetric
 %-------------------------------------------------------------------------------
 % UF Sparse Matrix Collection, Tim Davis
-% http://www.cise.ufl.edu/research/sparse/matrices/HB/bcspwr02
-% name: HB/bcspwr02
-% [SYMMETRIC STRUCTURE OF A SMALL TEST POWER SYSTEM]
-% id: 14
-% date: 1981
-% author: B. Dembart, J. Lewis
-% ed: I. Duff, R. Grimes, J. Lewis
-% fields: title A name id date author ed kind
-% kind: power network problem
+% http://www.cise.ufl.edu/research/sparse/matrices/Oberwolfach/LF10
+% name: Oberwolfach/LF10
+% [Oberwolfach: linear 1D beam]
+% id: 1438
+% date: 2004
+% author: J. Lienemann, A. Greiner, J. Korvink
+% ed: E. Rudnyi
+% fields: name title A id notes aux date author ed kind
+% aux: M E B C
+% kind: model reduction problem
 %-------------------------------------------------------------------------------
-49 49 108
-1 1
-2 1
-2 2
-3 2
-4 2
-3 3
-30 3
-37 3
-4 4
-5 5
-28 5
-32 5
-6 6
-15 6
-33 6
-7 7
-17 7
-20 7
-34 7
-8 8
-17 8
-37 8
-9 9
-14 9
-10 10
-14 10
-11 11
-14 11
-12 12
-15 12
-13 13
-16 13
-14 14
-15 14
-41 14
-15 15
-16 15
-36 15
-16 16
-22 16
-17 17
-18 18
-19 18
-30 18
-37 18
-19 19
-21 19
-20 20
-28 20
-31 20
-21 21
-45 21
-22 22
-23 22
-38 22
-45 22
-23 23
-24 23
-25 23
-26 23
-38 23
-49 23
-24 24
-25 25
-26 26
-27 27
-28 27
-29 27
-28 28
-32 28
-35 28
-29 29
-30 30
-31 31
-32 32
-33 32
-34 32
-37 32
-33 33
-42 33
-34 34
-35 35
-36 35
-36 36
-37 37
-38 37
-38 38
-39 39
-40 39
-49 39
-40 40
-41 41
-42 41
-43 41
-42 42
-44 42
-43 43
-44 44
-45 45
-46 45
-49 45
-46 46
-47 46
-48 46
-49 46
-47 47
-48 48
-49 49
+% notes:
+% Primary matrix in this model reduction problem is the Oberwolfach K matrix
+%-------------------------------------------------------------------------------
+18 18 50
+1 1 3.5344800000000003
+2 1 -477.1548
+3 1 1.7672400000000001
+2 2 171775.728
+4 2 -85887.864
+5 2 477.1548
+3 3 7.068960000000001
+4 3 -477.1548
+5 3 1.7672400000000001
+4 4 171775.728
+6 4 -85887.864
+7 4 477.1548
+5 5 7.068960000000001
+6 5 -477.1548
+7 5 1.7672400000000001
+6 6 171775.728
+8 6 -85887.864
+9 6 477.1548
+7 7 7.068960000000001
+8 7 -477.1548
+9 7 1.7672400000000001
+8 8 171775.728
+10 8 -85887.864
+11 8 477.1548
+9 9 7.068960000000001
+10 9 -477.1548
+11 9 1.7672400000000001
+10 10 171775.728
+12 10 -85887.864
+13 10 477.1548
+11 11 7.068960000000001
+12 11 -477.1548
+13 11 1.7672400000000001
+12 12 171775.728
+14 12 -85887.864
+15 12 477.1548
+13 13 7.068960000000001
+14 13 -477.1548
+15 13 1.7672400000000001
+14 14 171775.728
+16 14 -85887.864
+17 14 477.1548
+15 15 7.068960000000001
+16 15 -477.1548
+17 15 1.7672400000000001
+16 16 171775.728
+18 16 477.1548
+17 17 7.068960000000001
+18 17 1.7672400000000001
+18 18 3.5344800000000003
diff --git a/examples/mixed-spmv/doc/results.dox b/examples/mixed-spmv/doc/results.dox
index 3750bb12b95..60baf566562 100644
--- a/examples/mixed-spmv/doc/results.dox
+++ b/examples/mixed-spmv/doc/results.dox
@@ -3,14 +3,16 @@ The following is the expected result (omp):
 
 @code{.cpp}
 
-High Precision time(s): 1.2272200000e-05
-High Precision result norm: 1.1581546305e+01
-Low Precision time(s): 1.2360400000e-05
-Low Precision relative error: 4.4554843634e-08
-Hp * Lp -> Hp time(s): 1.2302300000e-05
-Hp * Lp -> Hp relative error: 1.4490698720e-08
-Lp * Lp -> Hp time(s): 1.2614800000e-05
-Lp * Lp -> Hp relative error: 1.4490698720e-08
+High Precision time(s): 9.8980000000e-07
+High Precision result norm: 2.5547848401e+05
+Low Precision time(s): 9.8890000000e-07
+Low Precision relative error: 5.5253439244e-08
+Hp * Lp -> Hp time(s): 1.3829000000e-06
+Hp * Lp -> Hp relative error: 1.6328092846e-08
+Lp * Lp -> Hp time(s): 1.3761000000e-06
+Lp * Lp -> Hp relative error: 2.5540873856e-08
+Lp * Hp -> Hp time(s): 1.3761000000e-06
+Lp * Hp -> Hp relative error: 3.7166469483e-08
 @endcode
 
 <h3> Comments about programming and debugging </h3>
diff --git a/examples/mixed-spmv/mixed-spmv.cpp b/examples/mixed-spmv/mixed-spmv.cpp
index deb64d983a3..7a6e6a8b542 100644
--- a/examples/mixed-spmv/mixed-spmv.cpp
+++ b/examples/mixed-spmv/mixed-spmv.cpp
@@ -89,7 +89,8 @@ get_rand_value(ValueDistribution &&value_dist, Engine &&gen)
  * @return seconds
  */
 double timing(std::shared_ptr<const gko::Executor> exec,
-              std::shared_ptr<gko::LinOp> A, std::shared_ptr<gko::LinOp> b,
+              std::shared_ptr<const gko::LinOp> A,
+              std::shared_ptr<const gko::LinOp> b,
               std::shared_ptr<gko::LinOp> x)
 {
     int warmup = 2;
@@ -99,12 +100,16 @@ double timing(std::shared_ptr<const gko::Executor> exec,
     }
     double total_sec = 0;
     for (int i = 0; i < rep; i++) {
+        // always clone the x in each apply
         auto xx = x->clone();
+        // synchronize to make sure data is already on device
         exec->synchronize();
         auto start = std::chrono::steady_clock::now();
         A->apply(lend(b), lend(xx));
+        // synchronize to make sure the operation is done
         exec->synchronize();
         auto stop = std::chrono::steady_clock::now();
+        // get the duration in seconds
         std::chrono::duration<double> duration_time = stop - start;
         total_sec += duration_time.count();
         if (i + 1 == rep) {
@@ -195,26 +200,31 @@ int main(int argc, char *argv[])
     // read the matrix into HighPrecision and LowPrecision.
     auto hp_A = share(gko::read<hp_mtx>(std::ifstream("data/A.mtx"), exec));
     auto lp_A = share(gko::read<lp_mtx>(std::ifstream("data/A.mtx"), exec));
-    auto host_b =
-        hp_vec::create(exec->get_master(), gko::dim<2>{hp_A->get_size()[1], 1});
-    std::ranlux48 rand_engine(30);
+    // Set the shortcut for each dimension
+    auto A_dim = hp_A->get_size();
+    auto b_dim = gko::dim<2>{A_dim[1], 1};
+    auto x_dim = gko::dim<2>{A_dim[0], b_dim[1]};
+    auto host_b = hp_vec::create(exec->get_master(), b_dim);
+    // fill the b vector with some random data
+    std::ranlux48 rand_engine(32);
     auto dist = std::uniform_real_distribution<RealValueType>(0.0, 1.0);
     for (int i = 0; i < host_b->get_size()[0]; i++) {
-        host_b->get_values()[i] =
-            get_rand_value<HighPrecision>(dist, rand_engine);
+        host_b->at(i, 0) = get_rand_value<HighPrecision>(dist, rand_engine);
     }
+    // copy the data from host to device
     auto hp_b = share(hp_vec::create(exec));
     auto lp_b = share(lp_vec::create(exec));
     hp_b->copy_from(lend(host_b));
     lp_b->copy_from(lend(hp_b));
-    auto hp_x = share(hp_vec::create(
-        exec, gko::dim<2>{hp_A->get_size()[0], hp_b->get_size()[1]}));
-    auto lp_x = share(lp_vec::create(
-        exec, gko::dim<2>{lp_A->get_size()[0], lp_b->get_size()[1]}));
+
+    // create several result x vector in different precision
+    auto hp_x = share(hp_vec::create(exec, x_dim));
+    auto lp_x = share(lp_vec::create(exec, x_dim));
     auto hplp_x = share(hp_x->clone());
     auto lplp_x = share(hp_x->clone());
+    auto lphp_x = share(hp_x->clone());
 
-    // @sect3{Measure the time of applu}
+    // @sect3{Measure the time of apply}
     // We measure the time among different combination of apply operation.
 
     // Hp * Hp -> Hp
@@ -223,12 +233,14 @@ int main(int argc, char *argv[])
     auto lp_sec = timing(exec, lp_A, lp_b, lp_x);
     // Hp * Lp -> Hp
     auto hplp_sec = timing(exec, hp_A, lp_b, hplp_x);
-    // Lp * Lp -> Lp
+    // Lp * Lp -> Hp
     auto lplp_sec = timing(exec, lp_A, lp_b, lplp_x);
+    // Lp * Hp -> Hp
+    auto lphp_sec = timing(exec, lp_A, hp_b, lphp_x);
 
 
     // To measure error of result.
-    // neg_one are objects that represent the numbers which allow for a
+    // neg_one is an object that represent the number -1.0 which allows for a
     // uniform interface when computing on any device. To compute the residual,
     // all you need to do is call the add_scaled method, which in this case is
     // an axpy and equivalent to the LAPACK axpy routine. Finally, you compute
@@ -238,9 +250,11 @@ int main(int argc, char *argv[])
     auto lp_diff_norm = gko::initialize<real_vec>({0.0}, exec->get_master());
     auto hplp_diff_norm = gko::initialize<real_vec>({0.0}, exec->get_master());
     auto lplp_diff_norm = gko::initialize<real_vec>({0.0}, exec->get_master());
+    auto lphp_diff_norm = gko::initialize<real_vec>({0.0}, exec->get_master());
     auto lp_diff = hp_x->clone();
     auto hplp_diff = hp_x->clone();
     auto lplp_diff = hp_x->clone();
+    auto lphp_diff = hp_x->clone();
 
     hp_x->compute_norm2(lend(hp_x_norm));
     lp_diff->add_scaled(lend(neg_one), lend(lp_x));
@@ -249,23 +263,25 @@ int main(int argc, char *argv[])
     hplp_diff->compute_norm2(lend(hplp_diff_norm));
     lplp_diff->add_scaled(lend(neg_one), lend(lplp_x));
     lplp_diff->compute_norm2(lend(lplp_diff_norm));
+    lphp_diff->add_scaled(lend(neg_one), lend(lphp_x));
+    lphp_diff->compute_norm2(lend(lphp_diff_norm));
     exec->synchronize();
 
     std::cout.precision(10);
     std::cout << std::scientific;
     std::cout << "High Precision time(s): " << hp_sec << std::endl;
-    std::cout << "High Precision result norm: " << hp_x_norm->get_values()[0]
+    std::cout << "High Precision result norm: " << hp_x_norm->at(0)
               << std::endl;
     std::cout << "Low Precision time(s): " << lp_sec << std::endl;
     std::cout << "Low Precision relative error: "
-              << lp_diff_norm->get_values()[0] / hp_x_norm->get_values()[0]
-              << "\n";
+              << lp_diff_norm->at(0) / hp_x_norm->at(0) << "\n";
     std::cout << "Hp * Lp -> Hp time(s): " << hplp_sec << std::endl;
     std::cout << "Hp * Lp -> Hp relative error: "
-              << hplp_diff_norm->get_values()[0] / hp_x_norm->get_values()[0]
-              << "\n";
+              << hplp_diff_norm->at(0) / hp_x_norm->at(0) << "\n";
     std::cout << "Lp * Lp -> Hp time(s): " << lplp_sec << std::endl;
     std::cout << "Lp * Lp -> Hp relative error: "
-              << lplp_diff_norm->get_values()[0] / hp_x_norm->get_values()[0]
-              << "\n";
+              << lplp_diff_norm->at(0) / hp_x_norm->at(0) << "\n";
+    std::cout << "Lp * Hp -> Hp time(s): " << lplp_sec << std::endl;
+    std::cout << "Lp * Hp -> Hp relative error: "
+              << lphp_diff_norm->at(0) / hp_x_norm->at(0) << "\n";
 }

From 031d9a7ef375a0cafe06e3d16434112f850dcd94 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sun, 6 Jun 2021 14:54:35 +0200
Subject: [PATCH 0755/1008] silence CMake CUDA_ARCHITECTURES warnings

---
 cmake/create_test.cmake                      | 4 ++++
 cuda/CMakeLists.txt                          | 4 ++++
 examples/custom-matrix-format/CMakeLists.txt | 8 ++++++++
 test_install/CMakeLists.txt                  | 4 ++++
 4 files changed, 20 insertions(+)

diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index 8bca662455a..84d80214440 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -121,6 +121,10 @@ function(ginkgo_create_cuda_test test_name)
         )
     target_compile_options(${TEST_TARGET_NAME}
         PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_ARCH_FLAGS}>")
+    # we handle CUDA architecture flags for now, disable CMake handling
+    if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
+        set_target_properties(${TEST_TARGET_NAME} PROPERTIES CUDA_ARCHITECTURES OFF)
+    endif()
     set_target_properties(${TEST_TARGET_NAME} PROPERTIES
         OUTPUT_NAME ${test_name})
     if (GINKGO_FAST_TESTS)
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 8ba183d1635..c63a35c89e1 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -157,6 +157,10 @@ target_link_libraries(ginkgo_cuda PRIVATE ${CUDA_RUNTIME_LIBS} ${CUBLAS} ${CUSPA
 target_link_libraries(ginkgo_cuda PUBLIC ginkgo_device)
 target_compile_options(ginkgo_cuda
         PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_ARCH_FLAGS}>")
+# we handle CUDA architecture flags for now, disable CMake handling
+if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
+    set_target_properties(ginkgo_cuda PROPERTIES CUDA_ARCHITECTURES OFF)
+endif()
 list(GET CUDA_RUNTIME_LIBS 0 CUDA_FIRST_LIB)
 get_filename_component(GKO_CUDA_LIBDIR "${CUDA_FIRST_LIB}" DIRECTORY)
 
diff --git a/examples/custom-matrix-format/CMakeLists.txt b/examples/custom-matrix-format/CMakeLists.txt
index c0821a2cf35..b8e98cfe3d3 100644
--- a/examples/custom-matrix-format/CMakeLists.txt
+++ b/examples/custom-matrix-format/CMakeLists.txt
@@ -18,5 +18,13 @@ set(CMAKE_CUDA_STANDARD_REQUIRED ON)
 add_executable(custom-matrix-format custom-matrix-format.cpp stencil_kernel.cu)
 target_link_libraries(custom-matrix-format Ginkgo::ginkgo OpenMP::OpenMP_CXX)
 
+# inherit CUDA architecture flags from Ginkgo
+target_compile_options(custom-matrix-format
+    PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_ARCH_FLAGS}>")
+# we handle CUDA architecture flags for now, disable CMake handling
+if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
+    set_target_properties(custom-matrix-format PROPERTIES CUDA_ARCHITECTURES OFF)
+endif()
+
 # workaround for clang-cuda/g++ interaction
 set_target_properties(custom-matrix-format PROPERTIES POSITION_INDEPENDENT_CODE ON)
diff --git a/test_install/CMakeLists.txt b/test_install/CMakeLists.txt
index 69797e0b70c..467d09c8186 100644
--- a/test_install/CMakeLists.txt
+++ b/test_install/CMakeLists.txt
@@ -36,6 +36,10 @@ if(GINKGO_BUILD_CUDA)
     add_executable(test_install_cuda ${CMAKE_CURRENT_BINARY_DIR}/test_install.cu)
     target_compile_options(test_install_cuda
         PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_ARCH_FLAGS}>")
+    # we handle CUDA architecture flags for now, disable CMake handling
+    if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
+        set_target_properties(test_install_cuda PROPERTIES CUDA_ARCHITECTURES OFF)
+    endif()
     target_compile_definitions(test_install_cuda PRIVATE HAS_CUDA=1)
     target_compile_definitions(test_install_cuda PRIVATE HAS_REFERENCE=${HAS_REFERENCE})
     target_link_libraries(test_install_cuda PRIVATE Ginkgo::ginkgo)

From 19a7b62e28fe66bdc189c89bed15e676cf557d6d Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sun, 6 Jun 2021 14:56:35 +0200
Subject: [PATCH 0756/1008] use FetchContent instead of ExternalProject

---
 CMakeLists.txt                                |   7 +-
 cmake/DownloadCMakeLists.txt.in               |  25 ----
 cmake/package_helpers.cmake                   | 109 ------------------
 third_party/CMakeLists.txt                    |  14 +--
 .../CudaArchitectureSelector/CMakeLists.txt   |  18 ++-
 third_party/gflags/CMakeLists.txt             |  49 +++-----
 third_party/git-cmake-format/CMakeLists.txt   |  19 +--
 third_party/gtest/CMakeLists.txt              |  40 +++----
 third_party/rapidjson/CMakeLists.txt          |  29 +++--
 9 files changed, 84 insertions(+), 226 deletions(-)
 delete mode 100644 cmake/DownloadCMakeLists.txt.in

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 314439b49ab..411f5963a1e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -216,16 +216,15 @@ endif()
 
 # Try to find the third party packages before using our subdirectories
 include(cmake/package_helpers.cmake)
-ginkgo_find_package(GTest "GTest::GTest;GTest::Main" FALSE 1.8.1)
-ginkgo_find_package(gflags gflags FALSE 2.2.2)
-ginkgo_find_package(RapidJSON rapidjson TRUE 1.1.0)
+find_package(GTest 1.10.0)
+find_package(gflags QUIET 2.2.2)
+find_package(RapidJSON QUIET 1.1.0)
 if(GINKGO_HAVE_HWLOC)
     ginkgo_find_package(HWLOC hwloc FALSE 2.1)
     if (NOT HWLOC_FOUND)
         set(GINKGO_USE_EXTERNAL_HWLOC 0)
     endif()
 endif()
-# third_party needs to be after flag modification.
 add_subdirectory(third_party)    # Third-party tools and libraries
 
 # Load CMake helpers
diff --git a/cmake/DownloadCMakeLists.txt.in b/cmake/DownloadCMakeLists.txt.in
deleted file mode 100644
index 2eb96a85dea..00000000000
--- a/cmake/DownloadCMakeLists.txt.in
+++ /dev/null
@@ -1,25 +0,0 @@
-cmake_minimum_required(VERSION 3.9)
-project(${package_name})
-
-include(ExternalProject)
-ExternalProject_Add(${package_name}
-    GIT_REPOSITORY    "${package_url}"
-    GIT_TAG           "${package_tag}"
-    SOURCE_DIR        "${CMAKE_CURRENT_BINARY_DIR}/src"
-    BINARY_DIR        "${CMAKE_CURRENT_BINARY_DIR}/build"
-    CMAKE_ARGS "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}"
-               "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}"
-               "-DCMAKE_BUILD_TYPE=${GINKGO_THIRD_PARTY_BUILD_TYPE}"
-               # These are only useful if you're cross-compiling.
-               # They, however, will not hurt regardless.
-               "-DCMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME}"
-               "-DCMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}"
-               "-DCMAKE_AR=${CMAKE_AR}"
-               "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
-               "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
-               "-DCMAKE_FIND_ROOT_PATH=${CMAKE_FIND_ROOT_PATH}"
-               "${ARGN}"
-    INSTALL_COMMAND   ""
-    TEST_COMMAND      ""
-    UPDATE_DISCONNECTED ${GINKGO_SKIP_DEPENDENCY_UPDATE}
-)
diff --git a/cmake/package_helpers.cmake b/cmake/package_helpers.cmake
index afa0b4033fc..fdb676a10ea 100644
--- a/cmake/package_helpers.cmake
+++ b/cmake/package_helpers.cmake
@@ -5,58 +5,6 @@ set(NON_CMAKE_PACKAGE_DOWNLOADER_SCRIPT
     "${CMAKE_CURRENT_LIST_DIR}/DownloadNonCMakeCMakeLists.txt.in")
 
 
-#   Load a package from its repository
-#
-#   \param package_name  Name of the package
-#   \param package_url   Url of the package
-#   \param package_tag   Tag or version of the package to be downloaded.
-#
-function(ginkgo_load_git_package package_name package_url package_tag)
-    set(GINKGO_THIRD_PARTY_BUILD_TYPE "Debug")
-    if (CMAKE_BUILD_TYPE MATCHES "[Rr][Ee][Ll][Ee][Aa][Ss][Ee]")
-        set(GINKGO_THIRD_PARTY_BUILD_TYPE "Release")
-    endif()
-    configure_file(${PACKAGE_DOWNLOADER_SCRIPT}
-        download/CMakeLists.txt)
-    set(TOOLSET "")
-    if (NOT "${CMAKE_GENERATOR_TOOLSET}" STREQUAL "")
-        set(TOOLSET "-T${CMAKE_GENERATOR_TOOLSET}")
-    endif()
-    execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" "${TOOLSET}" .
-        RESULT_VARIABLE result
-        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/download)
-    if(result)
-        message(FATAL_ERROR
-            "CMake step for ${package_name}/download failed: ${result}")
-    endif()
-    if(MSVC)
-        # MSVC decides the build_type in build step not cmake step, so Ginkgo builds Debug and Release type.
-        execute_process(COMMAND ${CMAKE_COMMAND} --build . --config Debug
-            RESULT_VARIABLE result
-            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/download)
-        if(result)
-            message(FATAL_ERROR
-                "Build Debug step for ${package_name}/download failed: ${result}")
-        endif()
-        execute_process(COMMAND ${CMAKE_COMMAND} --build . --config Release
-            RESULT_VARIABLE result
-            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/download)
-        if(result)
-            message(FATAL_ERROR
-                "Build Release step for ${package_name}/download failed: ${result}")
-        endif()
-    else()
-        execute_process(COMMAND ${CMAKE_COMMAND} --build .
-            RESULT_VARIABLE result
-            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/download)
-        if(result)
-            message(FATAL_ERROR
-                "Build step for ${package_name}/download failed: ${result}")
-        endif()
-    endif()
-endfunction()
-
-
 #   Load a package from the url provided and run configure (Non-CMake projects)
 #
 #   \param package_name     Name of the package
@@ -128,63 +76,6 @@ macro(ginkgo_add_tpl_target new_target external_name includedir libdir header_on
 endmacro(ginkgo_add_tpl_target)
 
 
-#   Add external target to external project.
-#   Create a new target and declare it as `IMPORTED` for libraries or `INTERFACE`
-#       for header only projects.
-#
-#   \param new_target       New target for the external project
-#   \param external_name    Name of the external project
-#   \param includedir       Path to include directory
-#   \param libdir           Path to library directory
-#   \param build_type       Build type {STATIC, SHARED}
-#   \param debug_postfix    The debug postfix to use when building in debug mode
-#   \param external         Name of the external target
-#   \param header_only      Boolean indicating if this should be a header only target
-#
-macro(ginkgo_add_external_target new_target external_name includedir libdir build_type debug_postfix external header_only)
-    # Declare include directories and library files
-    set(${external_name}_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/${libdir})
-    set(${external_name}_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/${includedir}")
-    if(MSVC)
-        # Ginkgo only builds Debug and Release, so set the path without CMAKE_CFG_INTDIR.
-        set(${external_name}_LIBRARY_RELEASE "${${external_name}_BINARY_DIR}/Release/${CMAKE_${build_type}_LIBRARY_PREFIX}${external_name}${CMAKE_${build_type}_LIBRARY_SUFFIX}")
-        set(${external_name}_LIBRARY_DEBUG "${${external_name}_BINARY_DIR}/Debug/${CMAKE_${build_type}_LIBRARY_PREFIX}${external_name}${debug_postfix}${CMAKE_${build_type}_LIBRARY_SUFFIX}")
-    else()
-        set(${external_name}_LIBRARY_RELEASE "${${external_name}_BINARY_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_${build_type}_LIBRARY_PREFIX}${external_name}${CMAKE_${build_type}_LIBRARY_SUFFIX}")
-        set(${external_name}_LIBRARY_DEBUG "${${external_name}_BINARY_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_${build_type}_LIBRARY_PREFIX}${external_name}${debug_postfix}${CMAKE_${build_type}_LIBRARY_SUFFIX}")
-    endif()
-    # Create an IMPORTED external library available in the GLOBAL scope
-    if (${header_only})
-        add_library(${new_target} INTERFACE)
-    else()
-        add_library(${new_target} ${build_type} IMPORTED GLOBAL)
-    endif()
-
-    # Set a dependency to the external target (ExternalProject fetcher and builder)
-    add_dependencies(${new_target} ${external})
-
-    # Set the target's properties, namely library file and include directory
-    if (NOT ${header_only})
-        set_target_properties(${new_target} PROPERTIES IMPORTED_LOCATION_RELEASE ${${external_name}_LIBRARY_RELEASE})
-        set_target_properties(${new_target} PROPERTIES IMPORTED_LOCATION_DEBUG ${${external_name}_LIBRARY_DEBUG})
-        # Since we do not really manage other build types, let's globally use the DEBUG symbols
-        if(MSVC)
-            # Only Debug build uses MDd or MTd, and others use MD or MT.
-            # MSVC would like to use same runtime library, so we use Debug third-party in Debug and Release third-party in others.
-            set_target_properties(${new_target} PROPERTIES IMPORTED_LOCATION
-                ${${external_name}_LIBRARY_RELEASE})
-        else()
-            if (NOT CMAKE_BUILD_TYPE MATCHES "[Rr][Ee][Ll][Ee][Aa][Ss][Ee]"
-                    AND NOT CMAKE_BUILD_TYPE MATCHES "[Dd][Ee][Bb][Uu][Gg]")
-                set_target_properties(${new_target} PROPERTIES IMPORTED_LOCATION
-                    ${${external_name}_LIBRARY_DEBUG})
-            endif()
-        endif()
-    endif()
-    set_target_properties(${new_target} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${${external_name}_INCLUDE_DIR})
-endmacro(ginkgo_add_external_target)
-
-
 #   Ginkgo specific add_subdirectory helper macro.
 #   If the package was not found or if requested by the user, use the
 #       internal version of the package.
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index b42d5eaf20e..e0ce87a30e7 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -11,7 +11,7 @@ endif()
 
 # For each external package, check if they are already available, otherwise use ours
 if(GINKGO_BUILD_TESTS)
-    ginkgo_add_subdirectory(GTest gtest)
+    add_subdirectory(gtest)
 endif()
 
 if(GINKGO_HAVE_HWLOC AND NOT GINKGO_USE_EXTERNAL_HWLOC)
@@ -26,16 +26,8 @@ else()
 endif()
 
 if(GINKGO_BUILD_BENCHMARKS)
-    ginkgo_add_subdirectory(gflags gflags)
-    ginkgo_add_subdirectory(RapidJSON rapidjson)
-    if(WIN32)
-        target_link_libraries(gflags INTERFACE shlwapi.lib)
-    endif()
-    # RapidJSON does not provide a modern CMake target so far. Let's fix that!
-    if(RapidJSON_FOUND)
-        add_library(rapidjson INTERFACE)
-        target_include_directories(rapidjson INTERFACE ${RapidJSON_INCLUDE_DIRS})
-    endif()
+    add_subdirectory(gflags)
+    add_subdirectory(rapidjson)
 endif()
 
 set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" PARENT_SCOPE)
diff --git a/third_party/CudaArchitectureSelector/CMakeLists.txt b/third_party/CudaArchitectureSelector/CMakeLists.txt
index 5530b7c691a..d0c32f8f2f6 100644
--- a/third_party/CudaArchitectureSelector/CMakeLists.txt
+++ b/third_party/CudaArchitectureSelector/CMakeLists.txt
@@ -1,7 +1,13 @@
-message(STATUS "Configuring and building CAS")
-ginkgo_load_git_package(CudaArchitectureSelector
-    "https://github.com/ginkgo-project/CudaArchitectureSelector.git"
-    "97022db486e25ede6d7cbf683f4ac8d48694222a")
-add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/src
-    ${CMAKE_CURRENT_BINARY_DIR}/build EXCLUDE_FROM_ALL)
+message(STATUS "Fetching external CAS")
+include(FetchContent)
+FetchContent_Declare(
+    cuda_architecture_selector
+    GIT_REPOSITORY https://github.com/ginkgo-project/CudaArchitectureSelector.git
+    GIT_TAG        97022db486e25ede6d7cbf683f4ac8d48694222a
+)
+FetchContent_GetProperties(cuda_architecture_selector)
+if(NOT cuda_architecture_selector_POPULATED)
+    FetchContent_Populate(cuda_architecture_selector)
+endif()
+add_subdirectory("${cuda_architecture_selector_SOURCE_DIR}" "${cuda_architecture_selector_BINARY_DIR}" EXCLUDE_FROM_ALL)
 set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" PARENT_SCOPE)
diff --git a/third_party/gflags/CMakeLists.txt b/third_party/gflags/CMakeLists.txt
index 44618b89228..bd27968567f 100644
--- a/third_party/gflags/CMakeLists.txt
+++ b/third_party/gflags/CMakeLists.txt
@@ -1,36 +1,17 @@
-message(STATUS "Configuring and building gflags")
-if(MSVC)
-    # cmake links dynamic runtime libraries by default in Visual Studio
-    # use the ginkgo's flags to use the same runtime libraries as ginkgo
-    ginkgo_load_git_package(gflags_external
-        "https://github.com/gflags/gflags.git"
-        "f7388c6655e699f777a5a74a3c9880b9cfaabe59"
-        "-DGFLAGS_BUILD_TESTING=OFF" "-DGFLAGS_BUILD_gflags_LIB=OFF"
-        "-DGFLAGS_BUILD_gflags_nothreads_LIB=ON" "-DGFLAGS_BUILD_STATIC_LIBS=ON"
-        "-DGFLAGS_BUILD_PACKAGING=OFF" "-DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}"
-        "-DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}" "-DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}"
-        "-DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}")
-else()
-    # There is a weird issue with Intel 19 and c++17 causing a linking error.
-    # Use c++14 instead.
-    set(INTEL19_STD_FIX "")
-    if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" AND CMAKE_CXX_COMPILER_VERSION MATCHES "19")
-        set(INTEL19_STD_FIX "-DCMAKE_CXX_FLAGS=-std=c++14")
+if(NOT gflags_FOUND)
+    message(STATUS "Fetching external GFlags")
+    include(FetchContent)
+    FetchContent_Declare(
+        gflags
+        GIT_REPOSITORY https://github.com/gflags/gflags.git
+        GIT_TAG        f7388c6655e699f777a5a74a3c9880b9cfaabe59
+    )
+    set(gtest_disable_pthreads ON)
+    FetchContent_GetProperties(gflags)
+    if(NOT gflags_POPULATED)
+        FetchContent_Populate(gflags)
     endif()
-    ginkgo_load_git_package(gflags_external
-        "https://github.com/gflags/gflags.git"
-        "f7388c6655e699f777a5a74a3c9880b9cfaabe59"
-        "-DGFLAGS_BUILD_TESTING=OFF" "-DGFLAGS_BUILD_gflags_LIB=OFF"
-        "-DGFLAGS_BUILD_gflags_nothreads_LIB=ON" "-DGFLAGS_BUILD_STATIC_LIBS=ON"
-        "-DGFLAGS_BUILD_PACKAGING=OFF"
-        "${INTEL19_STD_FIX}"
-        )
-endif()
-if(WIN32)
-    # gflags uses gflags_nothreads_static not gflags_nothreads in Windows.
-    ginkgo_add_external_target(gflags gflags_nothreads_static build/include build/lib
-        STATIC "_debug" gflags_external FALSE)
-else()
-    ginkgo_add_external_target(gflags gflags_nothreads build/include build/lib
-        STATIC "_debug" gflags_external FALSE)
+    set(GFLAGS_BUILD_TESTING OFF)
+    set(GFLAGS_BUILD_PACKAGING OFF)
+    add_subdirectory(${gflags_SOURCE_DIR} ${gflags_BINARY_DIR} EXCLUDE_FROM_ALL)
 endif()
diff --git a/third_party/git-cmake-format/CMakeLists.txt b/third_party/git-cmake-format/CMakeLists.txt
index 87620c1338c..e8c0f7b63fc 100644
--- a/third_party/git-cmake-format/CMakeLists.txt
+++ b/third_party/git-cmake-format/CMakeLists.txt
@@ -1,7 +1,12 @@
-message(STATUS "Configuring and building git-cmake-format")
-ginkgo_load_git_package(git-cmake-format
-    "https://github.com/ginkgo-project/git-cmake-format.git"
-    "be9554a5d71030dcc23f1eb355c41ab9d876e776"
-    "-DGCF_CLANGFORMAT_MINIMAL_VERSION=8.0.0")
-add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/src
-    ${CMAKE_CURRENT_BINARY_DIR}/build EXCLUDE_FROM_ALL)
+include(FetchContent)
+FetchContent_Declare(
+    git_cmake_format
+    GIT_REPOSITORY https://github.com/ginkgo-project/git-cmake-format.git
+    GIT_TAG        be9554a5d71030dcc23f1eb355c41ab9d876e776
+)
+FetchContent_GetProperties(git_cmake_format)
+if(NOT git_cmake_format_POPULATED)
+    message(STATUS "Fetching git-cmake-format")
+    FetchContent_Populate(git_cmake_format)
+endif()
+add_subdirectory(${git_cmake_format_SOURCE_DIR} ${git_cmake_format_BINARY_DIR} EXCLUDE_FROM_ALL)
diff --git a/third_party/gtest/CMakeLists.txt b/third_party/gtest/CMakeLists.txt
index 691fd2fa5f6..f38768ee11a 100644
--- a/third_party/gtest/CMakeLists.txt
+++ b/third_party/gtest/CMakeLists.txt
@@ -1,21 +1,21 @@
-message(STATUS "Configuring and building gtest")
-if(MSVC)
-    # Force using shared runtime library when MSVC builds shared libraries
-    ginkgo_load_git_package(gtest_external
-        "https://github.com/google/googletest.git"
-        "6a7ed316a5cdc07b6d26362c90770787513822d4"
-        # Work around the linking errors when compiling gtest with CUDA
-        "-Dgtest_disable_pthreads=ON" "-Dgtest_force_shared_crt=ON")
-else()
-    ginkgo_load_git_package(gtest_external
-        "https://github.com/google/googletest.git"
-        "6a7ed316a5cdc07b6d26362c90770787513822d4"
-        # Work around the linking errors when compiling gtest with CUDA
-        "-Dgtest_disable_pthreads=ON"
-        "-DCMAKE_CXX_FLAGS=-fPIC")
+if(NOT GTest_FOUND)
+    message(STATUS "Fetching external GTest")
+    include(FetchContent)
+    FetchContent_Declare(
+        googletest
+        GIT_REPOSITORY https://github.com/google/googletest.git
+        GIT_TAG        6a7ed316a5cdc07b6d26362c90770787513822d4
+    )
+    set(gtest_disable_pthreads ON)
+    if(MSVC)
+        set(gtest_force_shared_crt ON)
+    endif()
+    FetchContent_GetProperties(googletest)
+    if(NOT googletest_POPULATED)
+        FetchContent_Populate(googletest)
+    endif()
+    add_subdirectory(${googletest_SOURCE_DIR} ${googletest_BINARY_DIR} EXCLUDE_FROM_ALL)
+    # by default, the outdated targets are not being exported
+    add_library(GTest::Main ALIAS gtest_main)
+    add_library(GTest::GTest ALIAS gtest)
 endif()
-
-ginkgo_add_external_target(GTest::GTest gtest src/googletest/include build/lib
-    STATIC "d" gtest_external FALSE)
-ginkgo_add_external_target(GTest::Main gtest_main src/googletest/include build/lib
-    STATIC "d" gtest_external FALSE)
diff --git a/third_party/rapidjson/CMakeLists.txt b/third_party/rapidjson/CMakeLists.txt
index 58118263a18..1914aa69f0e 100644
--- a/third_party/rapidjson/CMakeLists.txt
+++ b/third_party/rapidjson/CMakeLists.txt
@@ -1,10 +1,19 @@
-message(STATUS "Configuring and building rapidjson")
-ginkgo_load_git_package(rapidjson_external
-    "https://github.com/Tencent/rapidjson.git"
-    "88bd956d66d348f478bceebfdadb8e26c6844695"
-    "-DRAPIDJSON_BUILD_DOC=OFF" "-DRAPIDJSON_BUILD_EXAMPLES=OFF"
-    "-DRAPIDJSON_BUILD_TESTS=OFF" "-DRAPIDJSON_BUILD_THIRDPARTY_GTEST=OFF"
-    "-DRAPIDJSON_BUILD_CXX11=ON")
-# Remove directory with questionable licensing
-file(REMOVE_RECURSE ${CMAKE_CURRENT_BINARY_DIR}/src/bin/jsonchecker)
-ginkgo_add_external_target(rapidjson rapidjson src/include "" STATIC "" rapidjson_external TRUE)
+if(NOT RapidJSON_FOUND)
+    message(STATUS "Fetching external RapidJSON")
+    include(FetchContent)
+    FetchContent_Declare(
+        rapidjson
+        GIT_REPOSITORY https://github.com/Tencent/rapidjson.git
+        GIT_TAG        88bd956d66d348f478bceebfdadb8e26c6844695
+    )
+    FetchContent_GetProperties(rapidjson)
+    if(NOT rapidjson_POPULATED)
+        FetchContent_Populate(rapidjson)
+    endif()
+    set(RapidJSON_INCLUDE_DIR "${rapidjson_SOURCE_DIR}/include")
+endif()
+# some installed versions of RapidJSON already export a target
+if (NOT TARGET rapidjson)
+    add_library(rapidjson INTERFACE)
+    set_target_properties(rapidjson PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${RapidJSON_INCLUDE_DIR}")
+endif()

From 05bd22b72151ca3a42c054d4546afee9ea6b39c5 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 7 Jun 2021 13:40:20 +0200
Subject: [PATCH 0757/1008] set GTest location to lib/

---
 CMakeLists.txt                   | 8 ++++----
 third_party/gtest/CMakeLists.txt | 5 +++++
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 411f5963a1e..c10bc79c397 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -18,6 +18,10 @@ include(cmake/hip_path.cmake)
 include(cmake/autodetect_executors.cmake)
 include(cmake/build_type_helpers.cmake)
 
+# Load other CMake helpers
+include(cmake/build_helpers.cmake)
+include(cmake/install_helpers.cmake)
+
 if (MSVC)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj")
 endif()
@@ -227,10 +231,6 @@ if(GINKGO_HAVE_HWLOC)
 endif()
 add_subdirectory(third_party)    # Third-party tools and libraries
 
-# Load CMake helpers
-include(cmake/build_helpers.cmake)
-include(cmake/install_helpers.cmake)
-
 if(MSVC)
     if(BUILD_SHARED_LIBS)
         set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE)
diff --git a/third_party/gtest/CMakeLists.txt b/third_party/gtest/CMakeLists.txt
index f38768ee11a..29f8a700e51 100644
--- a/third_party/gtest/CMakeLists.txt
+++ b/third_party/gtest/CMakeLists.txt
@@ -15,6 +15,11 @@ if(NOT GTest_FOUND)
         FetchContent_Populate(googletest)
     endif()
     add_subdirectory(${googletest_SOURCE_DIR} ${googletest_BINARY_DIR} EXCLUDE_FROM_ALL)
+    # make sure the tests DLLs are placed in the working path for CTest
+    set_target_properties(gtest gtest_main PROPERTIES
+        RUNTIME_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}"
+        ARCHIVE_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}"
+        LIBRARY_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}")
     # by default, the outdated targets are not being exported
     add_library(GTest::Main ALIAS gtest_main)
     add_library(GTest::GTest ALIAS gtest)

From bd44238b130aff5afb68e9c0a1f512c80725a824 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 7 Jun 2021 14:01:57 +0200
Subject: [PATCH 0758/1008] set accessor test working directory

---
 core/test/accessor/CMakeLists.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/test/accessor/CMakeLists.txt b/core/test/accessor/CMakeLists.txt
index 9fb216ad00c..34745096ac2 100644
--- a/core/test/accessor/CMakeLists.txt
+++ b/core/test/accessor/CMakeLists.txt
@@ -14,7 +14,9 @@ function(create_accessor_test test_name)
     set_target_properties("${TEST_TARGET_NAME}" PROPERTIES
         OUTPUT_NAME "${test_name}")
     target_link_libraries("${TEST_TARGET_NAME}" PRIVATE GTest::Main GTest::GTest ${ARGN})
-    add_test(NAME "${REL_BINARY_DIR}/${test_name}" COMMAND "${TEST_TARGET_NAME}")
+    add_test(NAME "${REL_BINARY_DIR}/${test_name}"
+        COMMAND "${TEST_TARGET_NAME}"
+        WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
 endfunction(create_accessor_test)
 
 create_accessor_test(index_span)

From 359e0f0fd335b7e811ad41f89579ac2b323ca774 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 7 Jun 2021 22:48:11 +0200
Subject: [PATCH 0759/1008] fix third-party parameter passing

---
 third_party/gflags/CMakeLists.txt | 3 ++-
 third_party/gtest/CMakeLists.txt  | 5 +++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/third_party/gflags/CMakeLists.txt b/third_party/gflags/CMakeLists.txt
index bd27968567f..b14a33ce8a8 100644
--- a/third_party/gflags/CMakeLists.txt
+++ b/third_party/gflags/CMakeLists.txt
@@ -6,7 +6,8 @@ if(NOT gflags_FOUND)
         GIT_REPOSITORY https://github.com/gflags/gflags.git
         GIT_TAG        f7388c6655e699f777a5a74a3c9880b9cfaabe59
     )
-    set(gtest_disable_pthreads ON)
+    # need to set the variables in CACHE due to CMP0077
+    set(gtest_disable_pthreads ON CACHE INTERNAL "")
     FetchContent_GetProperties(gflags)
     if(NOT gflags_POPULATED)
         FetchContent_Populate(gflags)
diff --git a/third_party/gtest/CMakeLists.txt b/third_party/gtest/CMakeLists.txt
index 29f8a700e51..49d0fc6e40a 100644
--- a/third_party/gtest/CMakeLists.txt
+++ b/third_party/gtest/CMakeLists.txt
@@ -6,9 +6,10 @@ if(NOT GTest_FOUND)
         GIT_REPOSITORY https://github.com/google/googletest.git
         GIT_TAG        6a7ed316a5cdc07b6d26362c90770787513822d4
     )
-    set(gtest_disable_pthreads ON)
+    # need to set the variables in CACHE due to CMP0077
+    set(gtest_disable_pthreads ON CACHE INTERNAL "")
     if(MSVC)
-        set(gtest_force_shared_crt ON)
+        set(gtest_force_shared_crt ON CACHE INTERNAL "")
     endif()
     FetchContent_GetProperties(googletest)
     if(NOT googletest_POPULATED)

From 80ac26b440567d9234accd664ed5b2ead4731b6a Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 8 Jun 2021 15:54:29 +0200
Subject: [PATCH 0760/1008] vendor CAS module

---
 .../CudaArchitectureSelector/CMakeLists.txt   |  13 +-
 .../CudaArchitectureSelector.cmake            | 406 ++++++++++++++++++
 2 files changed, 407 insertions(+), 12 deletions(-)
 create mode 100644 third_party/CudaArchitectureSelector/CudaArchitectureSelector.cmake

diff --git a/third_party/CudaArchitectureSelector/CMakeLists.txt b/third_party/CudaArchitectureSelector/CMakeLists.txt
index d0c32f8f2f6..2c803679107 100644
--- a/third_party/CudaArchitectureSelector/CMakeLists.txt
+++ b/third_party/CudaArchitectureSelector/CMakeLists.txt
@@ -1,13 +1,2 @@
-message(STATUS "Fetching external CAS")
-include(FetchContent)
-FetchContent_Declare(
-    cuda_architecture_selector
-    GIT_REPOSITORY https://github.com/ginkgo-project/CudaArchitectureSelector.git
-    GIT_TAG        97022db486e25ede6d7cbf683f4ac8d48694222a
-)
-FetchContent_GetProperties(cuda_architecture_selector)
-if(NOT cuda_architecture_selector_POPULATED)
-    FetchContent_Populate(cuda_architecture_selector)
-endif()
-add_subdirectory("${cuda_architecture_selector_SOURCE_DIR}" "${cuda_architecture_selector_BINARY_DIR}" EXCLUDE_FROM_ALL)
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}")
 set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" PARENT_SCOPE)
diff --git a/third_party/CudaArchitectureSelector/CudaArchitectureSelector.cmake b/third_party/CudaArchitectureSelector/CudaArchitectureSelector.cmake
new file mode 100644
index 00000000000..aefda72e9b8
--- /dev/null
+++ b/third_party/CudaArchitectureSelector/CudaArchitectureSelector.cmake
@@ -0,0 +1,406 @@
+################################ License #######################################
+# Copyright (c) 2017-2021, the Ginkgo authors
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+# 
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+################################ License #######################################
+
+#.rst:
+# CudaArchitectureSelector
+# ------------------------
+#
+# Sets the target architectures for which to compile CUDA code.
+#
+# Functions
+# ^^^^^^^^^
+#
+# CudaArchitectureSelector exposes the following function:
+#
+# ::
+# 
+#   cas_target_cuda_architectures(
+#    <target>                   # target for which to set the architectures
+#    [ARCHITECTURES <spec>...]  # list of architecture specifications
+#    [UNSUPPORTED <arch>...]    # list of architectures not supported by the
+#                               # target
+#   )
+#
+# The command adds the appropriate list of additional compiler flags so that the
+# CUDA sources of the target are compiled for all architectures described in the
+# specification. Optionally, the list of flags can be filtered by specifying
+# which architectures are not supported by the target.
+#
+# ::
+# 
+#   cas_variable_cuda_architectures(
+#    <variable>                 # variable for storing architectures compiler 
+#                               # flag
+#    [ARCHITECTURES <spec>...]  # list of architecture specifications
+#    [UNSUPPORTED <arch>...]    # list of architectures not supported
+#   )
+#
+# The command has the same result as ``cas_target_cuda_architectures``. It does 
+# not add the compiler flags to the target, but stores the compiler flags in 
+# the variable (string).
+#
+# 
+# ``ARCHITECTURES`` specification list
+# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+#
+# The specification list passed as the ``ARCHITECTURES`` parameter can contain
+# the following types of entries (entries of distinct types can be combined in
+# the same list):
+#
+# Automatic detection:
+#   Specified by the string ``Auto``. Flags for CUBIN code generation will be
+#   added for all GPU architectures detected on the system. When nothing is
+#   detected, the behavior is the same as ``All``.
+#
+# All available architectures:
+#   Specified by the string ``All``. This option will add flags for CUBIN code
+#   generation for all GPU architectures supported by the compiler.
+#
+# GPU generation name:
+#   Has to be one of the strings ``Tesla``, ``Fermi``, ``Kepler``, ``Maxwell``,
+#   ``Pascal``, ``Volta``, ``Turing``, ``Ampere``. Specifying one of the strings
+#   will add flags for the generation of CUBIN code for all architectures
+#   belonging to that GPU generation (except the ones listed in the
+#   ``UNSUPPORTED`` list).
+#
+# Virtual and physical architecture specification:
+#   A string of the form ``XX(YY)``, where ``XX`` is the identifier of the
+#   physical architecture (e.g. ``XX=32`` represent the physical architecture
+#   ``sm_32``) and ``YY`` is the identifier of the virtual architecture (e.g.
+#   ``YY=52`` represents the virtual architecture ``compute_52``).
+#   Flags necessary to generate CUBIN code for the specified combination of the
+#   physical and virtual architecture will be added to the compiler flags.
+#
+# Only physical architecture specification
+#   A string of the form ``XX``. Functionally exactly equivalent to ``XX(XX)``.
+#
+# Only virtual architecture specification
+#   A string of the form ``(YY)``, where ``YY`` is the identifier of the virtual
+#   architecture. Flags necessary to generate the PTX code (which can be used by
+#   the jitter to compile for the specific device at runtime) for the specified
+#   architecture will be added to the compiler flags.
+#
+# clang-cuda makes no distinction between virtual and physical architecture,
+# and the physical architecture takes precedence over the virtual architecture.
+#
+# ``UNSUPPORTED`` architectures list
+# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+#
+# This list has entries of the form ``XX``, which represent the architecture
+# (both physical and virtual) identifiers. All code generation for both the
+# physical (CUBIN) and virtual (PTX) architectures matching one of the
+# identifiers in this list will be removed from the list specified by the
+# ``ARCHITECTURES`` list. A warning will be printed for each removed entry.
+# The list also supports aggregates ``All``, ``Auto`` and GPU generation names
+# wich have the same meaning as in the ``ARCHITECTURES'' specification list.
+
+
+if(NOT DEFINED CMAKE_CUDA_COMPILER)
+    message(FATAL_ERROR "CUDA language support is not enabled")
+endif()
+
+
+set(cas_spec_regex "^([0-9]+)?(\\(([0-9]+)\\))?\$")
+
+
+# returns a list of GPU architectures supported by the compiler
+function(cas_get_supported_architectures output)
+    if(DEFINED CAS_SUPPORTED_ARCHITECTURES)
+        set(${output} ${CAS_SUPPORTED_ARCHITECTURES} PARENT_SCOPE)
+        return()
+    endif()
+    set(CAS_NVCC_EXEC "${CMAKE_CUDA_COMPILER}")
+    # for clang-cuda determine underlying nvcc location
+    if(CMAKE_CUDA_COMPILER_ID STREQUAL "Clang")
+        execute_process(
+            COMMAND "${CMAKE_CUDA_COMPILER}" -v
+            RESULT_VARIABLE status
+            ERROR_VARIABLE clang_info_content
+            OUTPUT_QUIET)
+        if(NOT (status EQUAL 0))
+            message(FATAL_ERROR "Unable to execute clang-cuda")
+        endif()
+        if (clang_info_content MATCHES "Found CUDA installation: (.*), version ")
+            set(CAS_NVCC_EXEC "${CMAKE_MATCH_1}/bin/nvcc")
+        else()
+            message(FATAL_ERROR "Unable to determine CUDA installation path from clang-cuda")
+        endif()
+    endif()
+    execute_process(
+        COMMAND "${CAS_NVCC_EXEC}" --help
+        RESULT_VARIABLE status
+        OUTPUT_VARIABLE help_content
+        ERROR_QUIET)
+    if(NOT (status EQUAL 0))
+        message(FATAL_ERROR "Unable to determine supported GPU architectures")
+    endif()
+    string(REGEX MATCHALL "sm_[0-9]+" extracted_info ${help_content})
+    list(SORT extracted_info)
+    list(REMOVE_DUPLICATES extracted_info)
+    foreach(item IN LISTS extracted_info)
+        set(detector_name "${PROJECT_BINARY_DIR}/CMakeFiles/cas_supported.cu")
+        file(WRITE "${detector_name}"
+            "int main() {"
+            "  return 0;"
+            "}")
+        if(CMAKE_CUDA_COMPILER_ID STREQUAL "Clang")
+            set(CMAKE_CUDA_FLAGS "--cuda-gpu-arch=${item}")
+        else()
+            set(CMAKE_CUDA_FLAGS "--gpu-architecture=${item}")
+        endif()
+        try_compile(status "${PROJECT_BINARY_DIR}"
+            SOURCES "${detector_name}")
+        if (status)
+            string(REGEX REPLACE "sm_([0-9]+)" "\\1" temp ${item})
+            list(APPEND supported ${temp})
+        endif()
+        file(REMOVE "${detector_name}")
+        unset(CMAKE_CUDA_FLAGS)
+    endforeach()
+    if(NOT DEFINED supported)
+        message(FATAL_ERROR "Unable to determine supported GPU architectures")
+    endif()
+    set(CAS_SUPPORTED_ARCHITECTURES ${supported} CACHE INTERNAL
+        "GPU architectures supported by the compiler")
+    mark_as_advanced(FORCE CAS_SUPPORTED_ARCHITECTURES)
+    message(STATUS 
+        "The CUDA compiler supports the following architectures: ${supported}")
+    set(${output} ${CAS_SUPPORTED_ARCHITECTURES} PARENT_SCOPE)
+endfunction()
+
+
+# returns a list of GPU architectures present on the system
+function(cas_get_onboard_architectures output)
+    # Optional argument: disable warning. This is useful when calling this in a
+    # function which already prints a warning.
+    set(ENABLE_WARNING ON)
+    if (ARGV1 EQUAL 0)
+        set(ENABLE_WARNING OFF)
+    endif()
+    if(DEFINED CAS_ONBOARD_ARCHITECTURES)
+        set(${output} "${CAS_ONBOARD_ARCHITECTURES}" PARENT_SCOPE)
+        return()
+    endif()
+    set(detector_name "${PROJECT_BINARY_DIR}/CMakeFiles/cas_detector.cu")
+    file(WRITE ${detector_name}
+        "#include <iostream>\n"
+        "int main() {"
+        "  int n = 0;"
+        "  cudaError_t r = cudaGetDeviceCount(&n);"
+        "  if (r == cudaErrorNoDevice) return 0;"
+        "  if (r != cudaSuccess) return 1;"
+        "  char *sep = \"\";"
+        "  for (int i = 0; i < n; ++i) {"
+        "    cudaDeviceProp p;"
+        "    if (cudaGetDeviceProperties(&p, i) == cudaSuccess) {"
+        "      std::cout << sep << p.major << p.minor;"
+        "      sep = \";\";"
+        "    }"
+        "  }"
+        "  return 0;"
+        "}")
+    try_run(status unused
+        "${PROJECT_BINARY_DIR}/CMakeFiles" "${detector_name}"
+        RUN_OUTPUT_VARIABLE detected)
+    if(status EQUAL 0)
+        list(SORT detected)
+        list(REMOVE_DUPLICATES detected)
+        set(CAS_ONBOARD_ARCHITECTURES ${detected} CACHE INTERNAL
+            "List of detected GPU architectures")
+        mark_as_advanced(FORCE CAS_ONBOARD_ARCHITECTURES)
+        message(STATUS
+            "Detected GPU devices of the following architectures: ${detected}")
+    elseif(ENABLE_WARNING)
+        message(WARNING
+            "GPU detection failed -- something seems to be wrong "
+            "with the CUDA installation")
+    endif()
+    set(${output} "${CAS_ONBOARD_ARCHITECTURES}" PARENT_SCOPE)
+endfunction()
+
+
+# returns the list of GPU architectures associated to a specific GPU generation
+function(cas_get_architectures_by_name name output)
+    # add new name to compute capability bindings to this list as new GPU
+    # generations get released
+    set(  tesla_version 1)
+    set(  fermi_version 2)
+    set( kepler_version 3)
+    set(maxwell_version 5)
+    set( pascal_version 6)
+    set(  volta_version "7(0|2)")
+    set( turing_version 75)
+    set( ampere_version 8)
+    string(TOLOWER ${name} lower_name)
+    if(NOT DEFINED ${lower_name}_version)
+        message(FATAL_ERROR "${name} is not a valid GPU generation name")
+    endif()
+    cas_get_supported_architectures(architecture_list)
+    list(FILTER architecture_list INCLUDE REGEX "^${${lower_name}_version}[0-9]?")
+    set(${output} "${architecture_list}" PARENT_SCOPE)
+endfunction()
+
+
+# checks if the specified architecture specification is supported by the
+# selected CUDA compiler
+function(cas_is_supported_architecture_spec arch output)
+    if (NOT (arch STREQUAL "") AND (arch MATCHES "${cas_spec_regex}"))
+        set(code "${CMAKE_MATCH_1}")
+        set(arch "${CMAKE_MATCH_3}")
+        cas_get_supported_architectures(supported)
+        list(FIND supported "${arch}" arch_supported)
+        list(FIND supported "${code}" code_supported)
+        if ((arch STREQUAL "" OR NOT (arch_supported EQUAL "-1")) AND 
+            (code STREQUAL "" OR NOT (code_supported EQUAL "-1")))
+            set(${output} TRUE PARENT_SCOPE)
+        else()
+            set(${output} FALSE PARENT_SCOPE)
+        endif()
+    else()
+        message(FATAL_ERROR
+            "${arch} is not a valid GPU architecture specification")
+    endif()
+endfunction()
+
+
+# Adds or removes entries from a flag list using the ARCHITECTURES or
+# UNSUPPORTED lists.
+function(cas_update_flag_list flags_name mode)
+    set(flags "${${flags_name}}")
+    if(mode STREQUAL "ARCHITECTURES")
+        foreach(spec IN LISTS ARGN)
+            cas_is_supported_architecture_spec(${spec} is_arch_supported)
+            if(NOT is_arch_supported)
+                message(FATAL_ERROR
+                    "${spec} is not an architecture specification supported by "
+                    "the selected CUDA compiler")
+            endif()
+            string(REGEX MATCH "${cas_spec_regex}" unused "${spec}")
+            set(code "${CMAKE_MATCH_1}")
+            set(arch "${CMAKE_MATCH_3}")
+            if(CMAKE_CUDA_COMPILER_ID STREQUAL "Clang")
+                if(code STREQUAL "")
+                    set(new_flag "sm_${arch}")
+                else()
+                    set(new_flag "sm_${code}")
+                endif()
+                list(APPEND flags "--cuda-gpu-arch=${new_flag}")
+            else()
+                if(arch STREQUAL "")
+                    # virtual architecture not specified, use the one corresponding
+                    # to the real architecture
+                    set(arch "${code}")
+                endif()
+                if(code STREQUAL "")
+                    # real architecture not specified, only generate PTX for the
+                    # virtual architecture
+                    set(new_flag "arch=compute_${arch},code=compute_${arch}")
+                else()
+                    # real architecture specified, generate CUBIN object for that
+                    # architecture
+                    set(new_flag "arch=compute_${arch},code=sm_${code}")
+                endif()
+                list(APPEND flags "--generate-code=${new_flag}")
+            endif()
+        endforeach()
+    elseif(mode STREQUAL "UNSUPPORTED")
+        foreach(arch IN LISTS ARGN)
+            cas_get_supported_architectures(supported)
+            if(NOT (supported MATCHES "${arch}"))
+                continue()
+            endif()
+            foreach(flag IN ITEMS ${flags})
+                if(flag MATCHES ".*${arch}.*")
+                    message(WARNING
+                        "Removing flag for unsupported architecture ${arch}: "
+                        "${flag}")
+                    list(REMOVE_ITEM flags "${flag}")
+                endif()
+            endforeach()
+        endforeach()
+    else()
+        message(FATAL_ERROR 
+            "Unknown mode ${mode} passed to cas_update_flag_list")
+    endif()
+    set(${flags_name} "${flags}" PARENT_SCOPE)
+endfunction()
+
+
+# Returns a list of flags that should be passed to the CUDA compiler to build
+# for the passed architecture configurations.
+function(cas_get_compiler_flags output)
+    set(flags "")
+    foreach(arg IN LISTS ARGN)
+        if(arg STREQUAL "ARCHITECTURES" OR arg STREQUAL "UNSUPPORTED")
+            set(stage ${arg})
+            continue()
+        endif()
+        if(NOT DEFINED stage)
+            message(FATAL_ERROR
+                "cas_get_compiler_flags given unknown argument ${arg}")
+        endif()
+        if(arg STREQUAL "Auto")
+            cas_get_onboard_architectures(detected 0)
+            if(detected STREQUAL "")
+                message(WARNING
+                    "No GPUs detected on the system -- adding All flags "
+                    "to the list of architectures")
+                cas_get_supported_architectures(supported)
+                cas_update_flag_list(flags ${stage} ${supported})
+            endif()
+            cas_update_flag_list(flags ${stage} ${detected})
+        elseif(arg STREQUAL "All")
+            cas_get_supported_architectures(supported)
+            cas_update_flag_list(flags ${stage} ${supported})
+        elseif(NOT (arg STREQUAL "") AND (arg MATCHES "${cas_spec_regex}"))
+            cas_update_flag_list(flags ${stage} ${arg})
+        elseif(NOT (arg STREQUAL ""))
+            cas_get_architectures_by_name("${arg}" gen)
+            cas_update_flag_list(flags ${stage} ${gen})
+        endif()
+    endforeach()
+    list(REMOVE_DUPLICATES flags)
+    list(SORT flags)
+    set(${output} "${flags}" PARENT_SCOPE)
+endfunction()
+
+
+# Sets the CUDA architectures that a target should be compiled for.
+function(cas_target_cuda_architectures target)
+    cas_get_compiler_flags(flags ${ARGN})
+    target_compile_options(${target}
+        PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${flags}>")
+endfunction()
+
+# It only outputs the compiler flags.
+function(cas_variable_cuda_architectures variable)
+    cas_get_compiler_flags(flags ${ARGN})
+    set(${variable} "${flags}" PARENT_SCOPE)
+endfunction()

From 6c4b65feb2e0df721a2b8f8c2247b3546bdfb5cf Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 8 Jun 2021 15:54:42 +0200
Subject: [PATCH 0761/1008] fix find_package parameter order

---
 CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c10bc79c397..6c084242a61 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -221,8 +221,8 @@ endif()
 # Try to find the third party packages before using our subdirectories
 include(cmake/package_helpers.cmake)
 find_package(GTest 1.10.0)
-find_package(gflags QUIET 2.2.2)
-find_package(RapidJSON QUIET 1.1.0)
+find_package(gflags 2.2.2 QUIET)
+find_package(RapidJSON 1.1.0 QUIET)
 if(GINKGO_HAVE_HWLOC)
     ginkgo_find_package(HWLOC hwloc FALSE 2.1)
     if (NOT HWLOC_FOUND)

From 9e0c3c3fa07f71209fb77ae24238ee58fa38c5ff Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 8 Jun 2021 20:05:37 +0200
Subject: [PATCH 0762/1008] review updates

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 third_party/CMakeLists.txt                  | 10 +++--
 third_party/gflags/CMakeLists.txt           | 29 ++++++-------
 third_party/git-cmake-format/CMakeLists.txt |  4 +-
 third_party/gtest/CMakeLists.txt            | 48 ++++++++++-----------
 third_party/rapidjson/CMakeLists.txt        | 31 ++++++-------
 5 files changed, 58 insertions(+), 64 deletions(-)

diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index e0ce87a30e7..6ff5af5e351 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -10,7 +10,7 @@ if(GINKGO_BUILD_CUDA OR (GINKGO_BUILD_HIP AND
 endif()
 
 # For each external package, check if they are already available, otherwise use ours
-if(GINKGO_BUILD_TESTS)
+if(GINKGO_BUILD_TESTS AND (NOT GTest_FOUND))
     add_subdirectory(gtest)
 endif()
 
@@ -26,8 +26,12 @@ else()
 endif()
 
 if(GINKGO_BUILD_BENCHMARKS)
-    add_subdirectory(gflags)
-    add_subdirectory(rapidjson)
+    if (NOT gflags_FOUND)
+        add_subdirectory(gflags)
+    endif()
+    if (NOT RapidJSON_FOUND)
+        add_subdirectory(rapidjson)
+    endif()
 endif()
 
 set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}" PARENT_SCOPE)
diff --git a/third_party/gflags/CMakeLists.txt b/third_party/gflags/CMakeLists.txt
index b14a33ce8a8..7eea739001f 100644
--- a/third_party/gflags/CMakeLists.txt
+++ b/third_party/gflags/CMakeLists.txt
@@ -1,18 +1,15 @@
-if(NOT gflags_FOUND)
-    message(STATUS "Fetching external GFlags")
-    include(FetchContent)
-    FetchContent_Declare(
-        gflags
-        GIT_REPOSITORY https://github.com/gflags/gflags.git
-        GIT_TAG        f7388c6655e699f777a5a74a3c9880b9cfaabe59
-    )
-    # need to set the variables in CACHE due to CMP0077
-    set(gtest_disable_pthreads ON CACHE INTERNAL "")
-    FetchContent_GetProperties(gflags)
-    if(NOT gflags_POPULATED)
-        FetchContent_Populate(gflags)
-    endif()
-    set(GFLAGS_BUILD_TESTING OFF)
-    set(GFLAGS_BUILD_PACKAGING OFF)
+message(STATUS "Fetching external GFlags")
+include(FetchContent)
+FetchContent_Declare(
+    gflags
+    GIT_REPOSITORY https://github.com/gflags/gflags.git
+    GIT_TAG        f7388c6655e699f777a5a74a3c9880b9cfaabe59
+)
+# need to set the variables in CACHE due to CMP0077
+set(GFLAGS_BUILD_TESTING OFF CACHE INTERNAL "")
+set(GFLAGS_BUILD_PACKAGING OFF CACHE INTERNAL "")
+FetchContent_GetProperties(gflags)
+if(NOT gflags_POPULATED)
+    FetchContent_Populate(gflags)
     add_subdirectory(${gflags_SOURCE_DIR} ${gflags_BINARY_DIR} EXCLUDE_FROM_ALL)
 endif()
diff --git a/third_party/git-cmake-format/CMakeLists.txt b/third_party/git-cmake-format/CMakeLists.txt
index e8c0f7b63fc..fe1a44ad15a 100644
--- a/third_party/git-cmake-format/CMakeLists.txt
+++ b/third_party/git-cmake-format/CMakeLists.txt
@@ -1,3 +1,4 @@
+message(STATUS "Fetching git-cmake-format")
 include(FetchContent)
 FetchContent_Declare(
     git_cmake_format
@@ -6,7 +7,6 @@ FetchContent_Declare(
 )
 FetchContent_GetProperties(git_cmake_format)
 if(NOT git_cmake_format_POPULATED)
-    message(STATUS "Fetching git-cmake-format")
     FetchContent_Populate(git_cmake_format)
+    add_subdirectory(${git_cmake_format_SOURCE_DIR} ${git_cmake_format_BINARY_DIR} EXCLUDE_FROM_ALL)
 endif()
-add_subdirectory(${git_cmake_format_SOURCE_DIR} ${git_cmake_format_BINARY_DIR} EXCLUDE_FROM_ALL)
diff --git a/third_party/gtest/CMakeLists.txt b/third_party/gtest/CMakeLists.txt
index 49d0fc6e40a..25f340ce122 100644
--- a/third_party/gtest/CMakeLists.txt
+++ b/third_party/gtest/CMakeLists.txt
@@ -1,27 +1,25 @@
-if(NOT GTest_FOUND)
-    message(STATUS "Fetching external GTest")
-    include(FetchContent)
-    FetchContent_Declare(
-        googletest
-        GIT_REPOSITORY https://github.com/google/googletest.git
-        GIT_TAG        6a7ed316a5cdc07b6d26362c90770787513822d4
-    )
-    # need to set the variables in CACHE due to CMP0077
-    set(gtest_disable_pthreads ON CACHE INTERNAL "")
-    if(MSVC)
-        set(gtest_force_shared_crt ON CACHE INTERNAL "")
-    endif()
-    FetchContent_GetProperties(googletest)
-    if(NOT googletest_POPULATED)
-        FetchContent_Populate(googletest)
-    endif()
+message(STATUS "Fetching external GTest")
+include(FetchContent)
+FetchContent_Declare(
+    googletest
+    GIT_REPOSITORY https://github.com/google/googletest.git
+    GIT_TAG        6a7ed316a5cdc07b6d26362c90770787513822d4
+)
+# need to set the variables in CACHE due to CMP0077
+set(gtest_disable_pthreads ON CACHE INTERNAL "")
+if(MSVC)
+    set(gtest_force_shared_crt ON CACHE INTERNAL "")
+endif()
+FetchContent_GetProperties(googletest)
+if(NOT googletest_POPULATED)
+    FetchContent_Populate(googletest)
     add_subdirectory(${googletest_SOURCE_DIR} ${googletest_BINARY_DIR} EXCLUDE_FROM_ALL)
-    # make sure the tests DLLs are placed in the working path for CTest
-    set_target_properties(gtest gtest_main PROPERTIES
-        RUNTIME_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}"
-        ARCHIVE_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}"
-        LIBRARY_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}")
-    # by default, the outdated targets are not being exported
-    add_library(GTest::Main ALIAS gtest_main)
-    add_library(GTest::GTest ALIAS gtest)
 endif()
+# make sure the tests DLLs are placed in the working path for CTest
+set_target_properties(gtest gtest_main PROPERTIES
+    RUNTIME_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}"
+    ARCHIVE_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}"
+    LIBRARY_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}")
+# by default, the outdated targets are not being exported
+add_library(GTest::Main ALIAS gtest_main)
+add_library(GTest::GTest ALIAS gtest)
\ No newline at end of file
diff --git a/third_party/rapidjson/CMakeLists.txt b/third_party/rapidjson/CMakeLists.txt
index 1914aa69f0e..c272a8b41a0 100644
--- a/third_party/rapidjson/CMakeLists.txt
+++ b/third_party/rapidjson/CMakeLists.txt
@@ -1,19 +1,14 @@
-if(NOT RapidJSON_FOUND)
-    message(STATUS "Fetching external RapidJSON")
-    include(FetchContent)
-    FetchContent_Declare(
-        rapidjson
-        GIT_REPOSITORY https://github.com/Tencent/rapidjson.git
-        GIT_TAG        88bd956d66d348f478bceebfdadb8e26c6844695
-    )
-    FetchContent_GetProperties(rapidjson)
-    if(NOT rapidjson_POPULATED)
-        FetchContent_Populate(rapidjson)
-    endif()
-    set(RapidJSON_INCLUDE_DIR "${rapidjson_SOURCE_DIR}/include")
-endif()
-# some installed versions of RapidJSON already export a target
-if (NOT TARGET rapidjson)
-    add_library(rapidjson INTERFACE)
-    set_target_properties(rapidjson PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${RapidJSON_INCLUDE_DIR}")
+message(STATUS "Fetching external RapidJSON")
+include(FetchContent)
+FetchContent_Declare(
+    rapidjson
+    GIT_REPOSITORY https://github.com/Tencent/rapidjson.git
+    GIT_TAG        88bd956d66d348f478bceebfdadb8e26c6844695
+)
+FetchContent_GetProperties(rapidjson)
+if(NOT rapidjson_POPULATED)
+    FetchContent_Populate(rapidjson)
 endif()
+set(RapidJSON_INCLUDE_DIR "${rapidjson_SOURCE_DIR}/include")
+add_library(rapidjson INTERFACE)
+set_target_properties(rapidjson PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${RapidJSON_INCLUDE_DIR}")
\ No newline at end of file

From afd28c6cef4c6afd628e15780db54cda894e46b4 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 10 Jun 2021 12:10:59 +0200
Subject: [PATCH 0763/1008] simplify hwloc integration

* remove MSVC support completely
* only find packages if they are needed
---
 CMakeLists.txt                   | 20 ++++----
 INSTALL.md                       | 30 +++--------
 cmake/GinkgoConfig.cmake.in      | 17 -------
 cmake/install_helpers.cmake      |  4 +-
 cmake/package_helpers.cmake      | 87 --------------------------------
 third_party/CMakeLists.txt       |  4 +-
 third_party/hwloc/CMakeLists.txt | 51 ++++++-------------
 7 files changed, 34 insertions(+), 179 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6c084242a61..99985f16101 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -182,11 +182,8 @@ if(GINKGO_BUILD_HWLOC AND (MSVC OR WIN32 OR CYGWIN OR APPLE))
     message(WARNING "Ginkgo does not support HWLOC on Windows/MacOS, switch GINKGO_BUILD_HWLOC to OFF")
 endif()
 if(GINKGO_BUILD_HWLOC)
-    # By default always use external HWLOC
-    set(GINKGO_USE_EXTERNAL_HWLOC 1)
     set(GINKGO_HAVE_HWLOC 1)
 else()
-    set(GINKGO_USE_EXTERNAL_HWLOC 0)
     set(GINKGO_HAVE_HWLOC 0)
     message(STATUS "HWLOC is being forcibly switched off")
 endif()
@@ -220,14 +217,15 @@ endif()
 
 # Try to find the third party packages before using our subdirectories
 include(cmake/package_helpers.cmake)
-find_package(GTest 1.10.0)
-find_package(gflags 2.2.2 QUIET)
-find_package(RapidJSON 1.1.0 QUIET)
-if(GINKGO_HAVE_HWLOC)
-    ginkgo_find_package(HWLOC hwloc FALSE 2.1)
-    if (NOT HWLOC_FOUND)
-        set(GINKGO_USE_EXTERNAL_HWLOC 0)
-    endif()
+if(GINKGO_BUILD_TESTS)
+    find_package(GTest 1.10.0) # No need for QUIET as CMake ships FindGTest
+endif()
+if(GINKGO_BUILD_BENCHMARKS)
+    find_package(gflags 2.2.2 QUIET)
+    find_package(RapidJSON 1.1.0 QUIET)
+endif()
+if(GINKGO_BUILD_HWLOC)
+    find_package(HWLOC 2.1) # No need for QUIET as we ship FindHWLOC
 endif()
 add_subdirectory(third_party)    # Third-party tools and libraries
 
diff --git a/INSTALL.md b/INSTALL.md
index dad87874b26..377014512ff 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -212,9 +212,6 @@ variables:
 Ginkgo relies on third party packages in different cases. These third party
 packages can be turned off by disabling the relevant options.
 
-+ GINKGO_BUILD_CUDA=ON:
-  [CudaArchitectureSelector](https://github.com/ginkgo-project/CudaArchitectureSelector)
-  (CAS) is a CMake helper to manage CUDA architecture settings;
 + GINKGO_BUILD_TESTS=ON: Our tests are implemented with [Google
   Test](https://github.com/google/googletest);
 + GINKGO_BUILD_BENCHMARKS=ON: For argument management we use
@@ -227,14 +224,10 @@ packages can be turned off by disabling the relevant options.
   [hwloc](https://www.open-mpi.org/projects/hwloc) to detect and control cores
   and devices.
 
-Usually, Ginkgo uses the internal version of each package. For each of the
-packages `GTEST`, `GFLAGS`, `RAPIDJSON` and `CAS`, it is possible to force
-Ginkgo to try to use an external version of a package. For this, Ginkgo provides
-two ways to find packages. To rely on the CMake `find_package` command, use the
-CMake option `-DGINKGO_USE_EXTERNAL_<package>=ON`. `HWLOC` works the opposite
-way, Ginkgo always looks for the system's `hwloc` first. In addition, when
-installing Ginkgo, `hwloc` will be installed as well into the Ginkgo directory
-if there were no system `hwloc` detected.
+Ginkgo attempts to use pre-installed versions of these package if they match
+version requirements using `find_package`. Otherwise, the configuration step
+will download the files for each of the packages `GTest`, `gflags`,
+`RapidJSON` and `hwloc` and build them internally.
 
 Note that, if the external packages were not installed to the default location,
 the CMake option `-DCMAKE_PREFIX_PATH=<path-list>` needs to be set to the
@@ -243,19 +236,8 @@ more Information, see the [CMake documentation for
 CMAKE_PREFIX_PATH](https://cmake.org/cmake/help/v3.9/variable/CMAKE_PREFIX_PATH.html)
 for details.
 
-To manually configure the paths, Ginkgo relies on the [standard xSDK Installation
-policies](https://xsdk.info/policies/) for all packages except `CAS` (as it is
-neither a library nor a header, it cannot be expressed through the `TPL`
-format):
-+ `-DTPL_ENABLE_<package>=ON`
-+ `-DTPL_<package>_LIBRARIES=/path/to/libraries.{so|a}`
-+ `-DTPL_<package>_INCLUDE_DIRS=/path/to/header/directory`
-
-When applicable (e.g. for `GTest` libraries), a `;` separated list can be given
-to the `TPL_<package>_{LIBRARIES|INCLUDE_DIRS}` variables.
-
-Note that for convenience, the options `GINKGO_INSTALL_RPATH[_.*]` can be used
-to make bind the Ginkgo shared libraries to the path of its dependencies.
+For convenience, the options `GINKGO_INSTALL_RPATH[_.*]` can be used
+to bind the installed Ginkgo shared libraries to the path of its dependencies.
 
 ### Installing Ginkgo
 
diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 2387d4b8848..61da405cf6f 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -84,23 +84,6 @@ set(GINKGO_HAVE_PAPI_SDE @GINKGO_HAVE_PAPI_SDE@)
 set(GINKGO_HAVE_HWLOC @GINKGO_HAVE_HWLOC@)
 set(GINKGO_BUILD_HWLOC @GINKGO_BUILD_HWLOC@)
 
-# Ginkgo external package variables
-set(GINKGO_USE_EXTERNAL_CAS "@GINKGO_USE_EXTERNAL_CAS@")
-set(GINKGO_USE_EXTERNAL_GTEST "@GINKGO_USE_EXTERNAL_GTEST@")
-set(GINKGO_USE_EXTERNAL_GFLAGS "@GINKGO_USE_EXTERNAL_GFLAGS@")
-set(GINKGO_USE_EXTERNAL_RAPIDJSON "@GINKGO_USE_EXTERNAL_RAPIDJSON@")
-set(GINKGO_USE_EXTERNAL_HWLOC "@GINKGO_USE_EXTERNAL_HWLOC@")
-
-set(TPL_ENABLE_GTEST "@TPL_ENABLE_GTEST@")
-set(TPL_GTEST_LIBRARIES "@TPL_GTEST_LIBRARIES@")
-set(TPL_GTEST_INCLUDE_DIRS "@TPL_GTEST_INCLUDE_DIRS@")
-set(TPL_ENABLE_GFLAGS "@TPL_ENABLE_GFLAGS@")
-set(TPL_GFLAGS_LIBRARIES "@TPL_GFLAGS_LIBRARIES@")
-set(TPL_GFLAGS_INCLUDE_DIRS "@TPL_GFLAGS_INCLUDE_DIRS@")
-set(TPL_ENABLE_RAPIDJSON "@TPL_ENABLE_RAPIDJSON@")
-set(TPL_RAPIDJSON_LIBRARIES "@TPL_RAPIDJSON_LIBRARIES@")
-set(TPL_RAPIDJSON_INCLUDE_DIRS "@TPL_RAPIDJSON_INCLUDE_DIRS@")
-
 # Ginkgo installation configuration
 set(GINKGO_CONFIG_FILE_PATH "${CMAKE_CURRENT_LIST_DIR}")
 string(REPLACE "@GINKGO_INSTALL_CONFIG_DIR@" "" GINKGO_INSTALL_PREFIX "${GINKGO_CONFIG_FILE_PATH}")
diff --git a/cmake/install_helpers.cmake b/cmake/install_helpers.cmake
index 3991796a147..af5bf5b18b8 100644
--- a/cmake/install_helpers.cmake
+++ b/cmake/install_helpers.cmake
@@ -33,7 +33,7 @@ function(ginkgo_add_install_rpath name)
     endif()
     if (GINKGO_INSTALL_RPATH_DEPENDENCIES)
         set(RPATH_DEPENDENCIES "${ARGN}")
-        if  (GINKGO_HAVE_HWLOC AND GINKGO_USE_EXTERNAL_HWLOC)
+        if(GINKGO_HAVE_HWLOC AND HWLOC_FOUND)
             get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY)
             list(APPEND RPATH_DEPENDENCIES "${HWLOC_LIBRARIES}")
         endif()
@@ -87,7 +87,7 @@ function(ginkgo_install)
             )
     endif()
 
-    if  (GINKGO_HAVE_HWLOC AND NOT GINKGO_USE_EXTERNAL_HWLOC)
+    if  (GINKGO_HAVE_HWLOC AND NOT HWLOC_FOUND)
         get_filename_component(HWLOC_LIB_PATH ${HWLOC_LIBRARIES} DIRECTORY)
         file(GLOB HWLOC_LIBS "${HWLOC_LIB_PATH}/libhwloc*")
         install(FILES ${HWLOC_LIBS}
diff --git a/cmake/package_helpers.cmake b/cmake/package_helpers.cmake
index fdb676a10ea..e1d196ad553 100644
--- a/cmake/package_helpers.cmake
+++ b/cmake/package_helpers.cmake
@@ -1,6 +1,3 @@
-set(PACKAGE_DOWNLOADER_SCRIPT
-    "${CMAKE_CURRENT_LIST_DIR}/DownloadCMakeLists.txt.in")
-
 set(NON_CMAKE_PACKAGE_DOWNLOADER_SCRIPT
     "${CMAKE_CURRENT_LIST_DIR}/DownloadNonCMakeCMakeLists.txt.in")
 
@@ -42,90 +39,6 @@ function(ginkgo_load_and_configure_package package_name package_url package_hash
 endfunction()
 
 
-#   Add external target to external project.
-#   Create a new target and declare it as `IMPORTED` for libraries or `INTERFACE`
-#       for header only projects.
-#
-#   \param new_target       New target for the external project
-#   \param external_name    Name of the external project
-#   \param includedir       Path to include directory
-#   \param libdir           Path to library directory
-#   \param header_only      Boolean indicating if this should be a header only target
-#
-macro(ginkgo_add_tpl_target new_target external_name includedir libdir header_only)
-    # Declare include directories and library files
-    set(${external_name}_INCLUDE_DIR "${includedir}")
-    set(${external_name}_LIBRARY "${libdir}")
-
-    # Create an IMPORTED external library available in the GLOBAL scope
-    if (${header_only})
-        add_library(${new_target} INTERFACE)
-    else()
-        add_library(${new_target} UNKNOWN IMPORTED GLOBAL)
-    endif()
-
-    # Set the target's properties, namely library file and include directory
-    if (NOT ${header_only})
-        foreach (lib in LISTS ${${external_name}_LIBRARY})
-            set_target_properties(${new_target} PROPERTIES IMPORTED_LOCATION ${lib})
-        endforeach()
-    endif()
-    foreach (inc in LISTS ${${external_name}_INCLUDE_DIR})
-        set_target_properties(${new_target} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${inc})
-    endforeach()
-endmacro(ginkgo_add_tpl_target)
-
-
-#   Ginkgo specific add_subdirectory helper macro.
-#   If the package was not found or if requested by the user, use the
-#       internal version of the package.
-#
-#   \param package_name     Name of package to be found
-#   \param dir_name         Name of the subdirectory for the package
-#
-macro(ginkgo_add_subdirectory package_name dir_name)
-    if (NOT ${package_name}_FOUND)
-        add_subdirectory(${dir_name})
-    endif()
-endmacro(ginkgo_add_subdirectory)
-
-
-#   Ginkgo specific find_package helper macro. Use this macro for third
-#       party libraries.
-#   If the user does not specify otherwise, try to find the package.
-#
-#   \param package_name     Name of package to be found
-#   \param target_list      For TPL packages, declare a new target for each library
-#   \param header_only      For TPL packages, declare the tpl package as header only
-#   \param ARGN             Extra specifications for the package finder
-#
-macro(ginkgo_find_package package_name target_list header_only)
-    string(TOUPPER ${package_name} _UPACKAGE_NAME)
-    if (GINKGO_USE_EXTERNAL_${_UPACKAGE_NAME} OR TPL_ENABLE_${_UPACKAGE_NAME})
-        if (TPL_${_UPACKAGE_NAME}_LIBRARIES AND TPL_${_UPACKAGE_NAME}_INCLUDE_DIRS)
-            set(${package_name}_FOUND "${TPL_${_UPACKAGE_NAME}_LIBRARIES};${TPL_${_UPACKAGE_NAME}_INCLUDE_DIRS}")
-            set(_target_list ${target_list}) # CMake weirdness: target_list is not a list anymore
-            # Count the number of elements in the list. Substract by one to iterate from 0 to the end.
-            list(LENGTH _target_list _GKO_len1)
-            math(EXPR _GKO_len2 "${_GKO_len1} - 1")
-            foreach(val RANGE ${_GKO_len2})
-                list(GET _target_list ${val} target) # access element number "val" in _target_list
-                list(GET TPL_${_UPACKAGE_NAME}_LIBRARIES ${val} lib)
-                ginkgo_add_tpl_target("${target}" "${_UPACKAGE_NAME}" "${TPL_${_UPACKAGE_NAME}_INCLUDE_DIRS}"
-                    "${lib}" ${header_only})
-            endforeach()
-        else()
-            find_package(${package_name} ${ARGN})
-            if (${package_name}_FOUND)
-                message(STATUS "Using external version of package ${package_name}. In case of problems, consider setting -DGINKGO_USE_EXTERNAL_${_UPACKAGE_NAME}=OFF.")
-            else()
-                message(STATUS "Ginkgo could not find ${package_name}. The internal version will be used. Consider setting `-DCMAKE_PREFIX_PATH` if the package was not system-installed.")
-            endif()
-        endif()
-    endif()
-endmacro(ginkgo_find_package)
-
-
 #   Download a file and verify the download
 #
 #   \param url          The url of file to be downloaded
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
index 6ff5af5e351..7220e48e18b 100644
--- a/third_party/CMakeLists.txt
+++ b/third_party/CMakeLists.txt
@@ -14,8 +14,8 @@ if(GINKGO_BUILD_TESTS AND (NOT GTest_FOUND))
     add_subdirectory(gtest)
 endif()
 
-if(GINKGO_HAVE_HWLOC AND NOT GINKGO_USE_EXTERNAL_HWLOC)
-    ginkgo_add_subdirectory(HWLOC hwloc)
+if(GINKGO_BUILD_HWLOC AND (NOT HWLOC_FOUND))
+    add_subdirectory(hwloc)
 endif()
 
 if(GINKGO_DEVEL_TOOLS)
diff --git a/third_party/hwloc/CMakeLists.txt b/third_party/hwloc/CMakeLists.txt
index b2c8b0ee08e..c4d82d5e654 100644
--- a/third_party/hwloc/CMakeLists.txt
+++ b/third_party/hwloc/CMakeLists.txt
@@ -1,42 +1,21 @@
 message(STATUS "Configuring and building HWLOC")
 set(TPL_HWLOC_PATH "${PROJECT_BINARY_DIR}/third_party/hwloc")
-if(MSVC OR WIN32)
-    message(STATUS "Downloading HWLOC binaries for win32")
-    ginkgo_download_file("https://download.open-mpi.org/release/hwloc/v2.3/hwloc-win32-build-2.3.0.zip"
-        "${TPL_HWLOC_PATH}/hwloc.zip"
-        "SHA1"
-        "1d53dc8db809dd19d8281556f5f3785105aba97f"
-        )
-    file(ARCHIVE_EXTRACT INPUT "${TPL_HWLOC_PATH}/hwloc.zip"
-        DESTINATION "${TPL_HWLOC_PATH}/hwloc"
-        VERBOSE
-        )
-else()
-    message(STATUS "Downloading HWLOC tars and building")
-    ginkgo_load_and_configure_package(hwloc_external "https://download.open-mpi.org/release/hwloc/v2.3/hwloc-2.3.0.tar.gz"
-        "SHA1=69d8e2f088d365cf5e86d04f049c6b1e09e2b9c3"
-        "${TPL_HWLOC_PATH}/src/configure" "--disable-nvml" "--disable-cuda" "--disable-rsmi"
-        )
+ginkgo_load_and_configure_package(hwloc_external "https://download.open-mpi.org/release/hwloc/v2.3/hwloc-2.3.0.tar.gz"
+    "SHA1=69d8e2f088d365cf5e86d04f049c6b1e09e2b9c3"
+    "${TPL_HWLOC_PATH}/src/configure" "--disable-nvml" "--disable-cuda" "--disable-rsmi"
+    )
 
-endif()
-
-if(MSVC OR WIN32)
-    add_library(hwloc STATIC IMPORTED GLOBAL)
-    set(HWLOC_LIBRARIES "${TPL_HWLOC_PATH}/hwloc/hwloc-win32-build-2.3.0/lib/libhwloc.a" CACHE FILEPATH "The path to HWLOC library libhwloc.so" FORCE)
-    set(HWLOC_INCLUDE_DIRS "${TPL_HWLOC_PATH}/hwloc/hwloc-win32-build-2.3.0/include" CACHE PATH "The directory containing the hwloc header, hwloc.h" FORCE)
-else()
-    add_library(hwloc SHARED IMPORTED GLOBAL)
-    add_dependencies(hwloc hwloc_external )
-    file(MAKE_DIRECTORY ${TPL_HWLOC_PATH}/lib/)
-    file(GLOB HWLOC_LIBS "${TPL_HWLOC_PATH}/build/hwloc/.libs/libhwloc.so*")
-    configure_file("${TPL_HWLOC_PATH}/build/include/hwloc/autogen/config.h" "${TPL_HWLOC_PATH}/src/include/hwloc/autogen/config.h" COPYONLY)
-    foreach(lib ${HWLOC_LIBS})
-        get_filename_component(lib_name ${lib} NAME)
-        configure_file("${lib}" "${TPL_HWLOC_PATH}/lib/${lib_name}" COPYONLY)
-    endforeach()
-    set(HWLOC_LIBRARIES "${TPL_HWLOC_PATH}/lib/libhwloc.so" CACHE FILEPATH "The path to HWLOC library libhwloc.so" FORCE)
-    set(HWLOC_INCLUDE_DIRS "${TPL_HWLOC_PATH}/src/include" CACHE PATH "The directory containing the hwloc header, hwloc.h" FORCE)
-endif()
+add_library(hwloc SHARED IMPORTED GLOBAL)
+add_dependencies(hwloc hwloc_external )
+file(MAKE_DIRECTORY ${TPL_HWLOC_PATH}/lib/)
+file(GLOB HWLOC_LIBS "${TPL_HWLOC_PATH}/build/hwloc/.libs/libhwloc.so*")
+configure_file("${TPL_HWLOC_PATH}/build/include/hwloc/autogen/config.h" "${TPL_HWLOC_PATH}/src/include/hwloc/autogen/config.h" COPYONLY)
+foreach(lib ${HWLOC_LIBS})
+    get_filename_component(lib_name ${lib} NAME)
+    configure_file("${lib}" "${TPL_HWLOC_PATH}/lib/${lib_name}" COPYONLY)
+endforeach()
+set(HWLOC_LIBRARIES "${TPL_HWLOC_PATH}/lib/libhwloc.so" CACHE FILEPATH "The path to HWLOC library libhwloc.so" FORCE)
+set(HWLOC_INCLUDE_DIRS "${TPL_HWLOC_PATH}/src/include" CACHE PATH "The directory containing the hwloc header, hwloc.h" FORCE)
 set_target_properties(hwloc PROPERTIES IMPORTED_LOCATION ${HWLOC_LIBRARIES})
 set_target_properties(hwloc PROPERTIES INTERFACE_LINK_LIBRARIES ${HWLOC_LIBRARIES})
 set_target_properties(hwloc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${HWLOC_INCLUDE_DIRS}")

From c2ff5e1b3b74faa34a4902a75e02934a74c44f28 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 16 Jun 2021 11:34:13 +0200
Subject: [PATCH 0764/1008] quiet OpenCV find_package

This avoids printing a lengthy and maybe surprising error message:

CMake Warning at examples/CMakeLists.txt:39 (find_package):
  By not providing "FindOpenCV.cmake" in CMAKE_MODULE_PATH this project has
  asked CMake to find a package configuration file provided by "OpenCV", but
  CMake did not find one.

  Could not find a package configuration file provided by "OpenCV" with any
  of the following names:

    OpenCVConfig.cmake
    opencv-config.cmake

  Add the installation prefix of "OpenCV" to CMAKE_PREFIX_PATH or set
  "OpenCV_DIR" to a directory containing one of the above files.  If "OpenCV"
  provides a separate development package or SDK, be sure it has been
  installed.
---
 examples/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 81f71466cc3..f895f86005b 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -36,7 +36,7 @@ if(GINKGO_BUILD_EXTLIB_EXAMPLE)
     list(APPEND EXAMPLES_LIST external-lib-interfacing)
 endif()
 
-find_package(OpenCV)
+find_package(OpenCV QUIET)
 if(OpenCV_FOUND)
     list(APPEND EXAMPLES_LIST heat-equation)
 endif()

From d4929a31b3628503218b0312726dbb6bcf594bab Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 16 Jun 2021 12:10:31 +0200
Subject: [PATCH 0765/1008] add message if no OpenCV was found

---
 examples/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index f895f86005b..61f1d58f50f 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -39,6 +39,8 @@ endif()
 find_package(OpenCV QUIET)
 if(OpenCV_FOUND)
     list(APPEND EXAMPLES_LIST heat-equation)
+else()
+    message(STATUS "No OpenCV found, disabling heat-equation example")
 endif()
 
 if(GINKGO_HAVE_PAPI_SDE)

From 8e82035036ab00417be5ac37bea3bd7be0b59788 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 1 Jun 2021 22:56:37 +0200
Subject: [PATCH 0766/1008] add reference tests for simple solver kernels

---
 core/solver/cgs.cpp                        |   2 +-
 reference/test/solver/bicg_kernels.cpp     | 176 +++++++++++-
 reference/test/solver/bicgstab_kernels.cpp | 299 ++++++++++++++++++++-
 reference/test/solver/cg_kernels.cpp       | 142 +++++++++-
 reference/test/solver/cgs_kernels.cpp      | 208 +++++++++++++-
 reference/test/solver/fcg_kernels.cpp      | 155 ++++++++++-
 reference/test/solver/ir_kernels.cpp       |  19 +-
 7 files changed, 989 insertions(+), 12 deletions(-)

diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp
index 3d8419cb73c..c33a3b7b9c1 100644
--- a/core/solver/cgs.cpp
+++ b/core/solver/cgs.cpp
@@ -139,7 +139,7 @@ void Cgs<ValueType>::apply_dense_impl(const matrix::Dense<ValueType> *dense_b,
     // r = dense_b
     // r_tld = r
     // rho = 0.0
-    // rho_prev = 1.0
+    // rho_prev = alpha = beta = gamma = 1.0
     // p = q = u = u_hat = v_hat = t = 0
 
     system_matrix_->apply(neg_one_op.get(), dense_x, one_op.get(), r.get());
diff --git a/reference/test/solver/bicg_kernels.cpp b/reference/test/solver/bicg_kernels.cpp
index bee4ce18ece..aac5a26bd6d 100644
--- a/reference/test/solver/bicg_kernels.cpp
+++ b/reference/test/solver/bicg_kernels.cpp
@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/time.hpp>
 
 
+#include "core/solver/bicg_kernels.hpp"
 #include "core/test/utils.hpp"
 
 
@@ -102,12 +103,55 @@ class Bicg : public ::testing::Test {
               {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec))
 
 
-    {}
+    {
+        auto small_size = gko::dim<2>{2, 2};
+        auto small_scalar_size = gko::dim<2>{1, small_size[1]};
+        small_b = Mtx::create(exec, small_size, small_size[1] + 1);
+        small_x = Mtx::create(exec, small_size, small_size[1] + 2);
+        small_one = Mtx::create(exec, small_size);
+        small_zero = Mtx::create(exec, small_size);
+        small_prev_rho = Mtx::create(exec, small_scalar_size);
+        small_rho = Mtx::create(exec, small_scalar_size);
+        small_beta = Mtx::create(exec, small_scalar_size);
+        small_zero->fill(0);
+        small_one->fill(1);
+        small_r = small_zero->clone();
+        small_z = small_zero->clone();
+        small_p = small_zero->clone();
+        small_q = small_zero->clone();
+        small_r2 = small_zero->clone();
+        small_z2 = small_zero->clone();
+        small_p2 = small_zero->clone();
+        small_q2 = small_zero->clone();
+        small_stop = gko::Array<gko::stopping_status>(exec, small_size[1]);
+        stopped.stop(1);
+        non_stopped.reset();
+        std::fill_n(small_stop.get_data(), small_stop.get_num_elems(),
+                    non_stopped);
+    }
 
-    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<const gko::ReferenceExecutor> exec;
     std::shared_ptr<Mtx> mtx;
     std::shared_ptr<Mtx> mtx_big;
     std::shared_ptr<Mtx> mtx_non_symmetric;
+    std::unique_ptr<Mtx> small_one;
+    std::unique_ptr<Mtx> small_zero;
+    std::unique_ptr<Mtx> small_prev_rho;
+    std::unique_ptr<Mtx> small_beta;
+    std::unique_ptr<Mtx> small_rho;
+    std::unique_ptr<Mtx> small_x;
+    std::unique_ptr<Mtx> small_b;
+    std::unique_ptr<Mtx> small_r;
+    std::unique_ptr<Mtx> small_z;
+    std::unique_ptr<Mtx> small_p;
+    std::unique_ptr<Mtx> small_q;
+    std::unique_ptr<Mtx> small_r2;
+    std::unique_ptr<Mtx> small_z2;
+    std::unique_ptr<Mtx> small_p2;
+    std::unique_ptr<Mtx> small_q2;
+    gko::Array<gko::stopping_status> small_stop;
+    gko::stopping_status stopped;
+    gko::stopping_status non_stopped;
     std::unique_ptr<typename Solver::Factory> bicg_factory;
     std::unique_ptr<typename Solver::Factory> bicg_factory_big;
     std::unique_ptr<typename Solver::Factory> bicg_factory_big2;
@@ -117,6 +161,134 @@ class Bicg : public ::testing::Test {
 TYPED_TEST_SUITE(Bicg, gko::test::ValueTypes);
 
 
+TYPED_TEST(Bicg, KernelInitialize)
+{
+    this->small_b->fill(2);
+    this->small_r->fill(0);
+    this->small_z->fill(1);
+    this->small_p->fill(1);
+    this->small_q->fill(1);
+    this->small_r2->fill(0);
+    this->small_z2->fill(1);
+    this->small_p2->fill(1);
+    this->small_q2->fill(1);
+    this->small_prev_rho->fill(0);
+    this->small_rho->fill(1);
+    std::fill_n(this->small_stop.get_data(), this->small_stop.get_num_elems(),
+                this->stopped);
+
+    gko::kernels::reference::bicg::initialize(
+        this->exec, this->small_b.get(), this->small_r.get(),
+        this->small_z.get(), this->small_p.get(), this->small_q.get(),
+        this->small_prev_rho.get(), this->small_rho.get(), this->small_r2.get(),
+        this->small_z2.get(), this->small_p2.get(), this->small_q2.get(),
+        &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_r, this->small_b, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_z, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_p, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_q, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_r2, this->small_b, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_z2, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_p2, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_q2, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_rho, l({{0.0, 0.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_prev_rho, l({{1.0, 1.0}}), 0);
+    ASSERT_EQ(this->small_stop.get_data()[0], this->non_stopped);
+    ASSERT_EQ(this->small_stop.get_data()[1], this->non_stopped);
+}
+
+
+TYPED_TEST(Bicg, KernelStep1)
+{
+    this->small_p->fill(3);
+    this->small_z->fill(-2);
+    this->small_p2->fill(3);
+    this->small_z2->fill(-2);
+    this->small_rho->at(0) = 2;
+    this->small_rho->at(1) = 3;
+    this->small_prev_rho->at(0) = 8;
+    this->small_prev_rho->at(1) = 3;
+    this->small_stop.get_data()[1] = this->stopped;
+
+    gko::kernels::reference::bicg::step_1(
+        this->exec, this->small_p.get(), this->small_z.get(),
+        this->small_p2.get(), this->small_z2.get(), this->small_rho.get(),
+        this->small_prev_rho.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_p, l({{-1.25, 3.0}, {-1.25, 3.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_p2, l({{-1.25, 3.0}, {-1.25, 3.0}}), 0);
+}
+
+
+TYPED_TEST(Bicg, KernelStep1DivByZero)
+{
+    this->small_p->fill(3);
+    this->small_z->fill(-2);
+    this->small_p2->fill(3);
+    this->small_z2->fill(-2);
+    this->small_rho->fill(1);
+    this->small_prev_rho->fill(0);
+
+    gko::kernels::reference::bicg::step_1(
+        this->exec, this->small_p.get(), this->small_z.get(),
+        this->small_p2.get(), this->small_z2.get(), this->small_rho.get(),
+        this->small_prev_rho.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_p, l({{-2.0, -2.0}, {-2.0, -2.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_p2, l({{-2.0, -2.0}, {-2.0, -2.0}}), 0);
+}
+
+
+TYPED_TEST(Bicg, KernelStep2)
+{
+    this->small_x->fill(-2);
+    this->small_p->fill(3);
+    this->small_r->fill(4);
+    this->small_q->fill(-5);
+    this->small_r2->fill(4);
+    this->small_q2->fill(-5);
+    this->small_rho->at(0) = 2;
+    this->small_rho->at(1) = 3;
+    this->small_beta->at(0) = 8;
+    this->small_beta->at(1) = 3;
+    this->small_stop.get_data()[1] = this->stopped;
+
+    gko::kernels::reference::bicg::step_2(
+        this->exec, this->small_x.get(), this->small_r.get(),
+        this->small_r2.get(), this->small_p.get(), this->small_q.get(),
+        this->small_q2.get(), this->small_beta.get(), this->small_rho.get(),
+        &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_x, l({{-1.25, -2.0}, {-1.25, -2.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_r, l({{5.25, 4.0}, {5.25, 4.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_r2, l({{5.25, 4.0}, {5.25, 4.0}}), 0);
+}
+
+
+TYPED_TEST(Bicg, KernelStep2DivByZero)
+{
+    this->small_x->fill(-2);
+    this->small_p->fill(3);
+    this->small_r->fill(4);
+    this->small_q->fill(-5);
+    this->small_r2->fill(4);
+    this->small_q2->fill(-5);
+    this->small_rho->fill(1);
+    this->small_beta->fill(0);
+
+    gko::kernels::reference::bicg::step_2(
+        this->exec, this->small_x.get(), this->small_r.get(),
+        this->small_r2.get(), this->small_p.get(), this->small_q.get(),
+        this->small_q2.get(), this->small_beta.get(), this->small_rho.get(),
+        &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_x, l({{-2.0, -2.0}, {-2.0, -2.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_r, l({{4.0, 4.0}, {4.0, 4.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_r2, l({{4.0, 4.0}, {4.0, 4.0}}), 0);
+}
+
+
 TYPED_TEST(Bicg, SolvesStencilSystem)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/reference/test/solver/bicgstab_kernels.cpp b/reference/test/solver/bicgstab_kernels.cpp
index d88a527fd47..cbbf60b955c 100644
--- a/reference/test/solver/bicgstab_kernels.cpp
+++ b/reference/test/solver/bicgstab_kernels.cpp
@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/time.hpp>
 
 
+#include "core/solver/bicgstab_kernels.hpp"
 #include "core/test/utils.hpp"
 
 
@@ -96,10 +97,61 @@ class Bicgstab : public ::testing::Test {
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec))
-    {}
+    {
+        auto small_size = gko::dim<2>{2, 2};
+        auto small_scalar_size = gko::dim<2>{1, small_size[1]};
+        small_b = Mtx::create(exec, small_size, small_size[1] + 1);
+        small_x = Mtx::create(exec, small_size, small_size[1] + 2);
+        small_one = Mtx::create(exec, small_size);
+        small_zero = Mtx::create(exec, small_size);
+        small_prev_rho = Mtx::create(exec, small_scalar_size);
+        small_rho = Mtx::create(exec, small_scalar_size);
+        small_alpha = Mtx::create(exec, small_scalar_size);
+        small_beta = Mtx::create(exec, small_scalar_size);
+        small_gamma = Mtx::create(exec, small_scalar_size);
+        small_omega = Mtx::create(exec, small_scalar_size);
+        small_zero->fill(0);
+        small_one->fill(1);
+        small_r = small_zero->clone();
+        small_rr = small_zero->clone();
+        small_v = small_zero->clone();
+        small_s = small_zero->clone();
+        small_t = small_zero->clone();
+        small_z = small_zero->clone();
+        small_y = small_zero->clone();
+        small_p = small_zero->clone();
+        small_stop = gko::Array<gko::stopping_status>(exec, small_size[1]);
+        stopped.stop(1, false);
+        finalized.stop(1, true);
+        non_stopped.reset();
+        std::fill_n(small_stop.get_data(), small_stop.get_num_elems(),
+                    non_stopped);
+    }
 
-    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<const gko::ReferenceExecutor> exec;
     std::shared_ptr<Mtx> mtx;
+    std::unique_ptr<Mtx> small_one;
+    std::unique_ptr<Mtx> small_zero;
+    std::unique_ptr<Mtx> small_prev_rho;
+    std::unique_ptr<Mtx> small_rho;
+    std::unique_ptr<Mtx> small_alpha;
+    std::unique_ptr<Mtx> small_beta;
+    std::unique_ptr<Mtx> small_gamma;
+    std::unique_ptr<Mtx> small_omega;
+    std::unique_ptr<Mtx> small_x;
+    std::unique_ptr<Mtx> small_b;
+    std::unique_ptr<Mtx> small_r;
+    std::unique_ptr<Mtx> small_rr;
+    std::unique_ptr<Mtx> small_v;
+    std::unique_ptr<Mtx> small_s;
+    std::unique_ptr<Mtx> small_t;
+    std::unique_ptr<Mtx> small_z;
+    std::unique_ptr<Mtx> small_y;
+    std::unique_ptr<Mtx> small_p;
+    gko::Array<gko::stopping_status> small_stop;
+    gko::stopping_status stopped;
+    gko::stopping_status finalized;
+    gko::stopping_status non_stopped;
     std::unique_ptr<typename Solver::Factory> bicgstab_factory;
     std::unique_ptr<typename Solver::Factory> bicgstab_factory2;
     std::unique_ptr<typename Solver::Factory> bicgstab_factory_precision;
@@ -108,6 +160,249 @@ class Bicgstab : public ::testing::Test {
 TYPED_TEST_SUITE(Bicgstab, gko::test::ValueTypes);
 
 
+TYPED_TEST(Bicgstab, KernelInitialize)
+{
+    this->small_b->fill(2);
+    this->small_r->fill(0);
+    this->small_rr->fill(1);
+    this->small_v->fill(1);
+    this->small_s->fill(1);
+    this->small_t->fill(1);
+    this->small_z->fill(1);
+    this->small_y->fill(1);
+    this->small_p->fill(1);
+    this->small_prev_rho->fill(0);
+    this->small_rho->fill(0);
+    this->small_alpha->fill(0);
+    this->small_beta->fill(0);
+    this->small_gamma->fill(0);
+    this->small_omega->fill(0);
+    std::fill_n(this->small_stop.get_data(), this->small_stop.get_num_elems(),
+                this->stopped);
+
+    gko::kernels::reference::bicgstab::initialize(
+        this->exec, this->small_b.get(), this->small_r.get(),
+        this->small_rr.get(), this->small_y.get(), this->small_s.get(),
+        this->small_t.get(), this->small_z.get(), this->small_v.get(),
+        this->small_p.get(), this->small_prev_rho.get(), this->small_rho.get(),
+        this->small_alpha.get(), this->small_beta.get(),
+        this->small_gamma.get(), this->small_omega.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_r, this->small_b, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_rr, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_v, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_s, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_t, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_z, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_y, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_p, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_rho, l({{1.0, 1.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_prev_rho, l({{1.0, 1.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_alpha, l({{1.0, 1.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_beta, l({{1.0, 1.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_gamma, l({{1.0, 1.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_omega, l({{1.0, 1.0}}), 0);
+    ASSERT_EQ(this->small_stop.get_data()[0], this->non_stopped);
+    ASSERT_EQ(this->small_stop.get_data()[1], this->non_stopped);
+}
+
+
+TYPED_TEST(Bicgstab, KernelStep1)
+{
+    this->small_p->fill(3);
+    this->small_r->fill(-2);
+    this->small_v->fill(1);
+    this->small_rho->at(0) = 2;
+    this->small_rho->at(1) = 3;
+    this->small_prev_rho->at(0) = 7;
+    this->small_prev_rho->at(1) = 6;
+    this->small_alpha->at(0) = 7;
+    this->small_alpha->at(1) = 5;
+    this->small_omega->at(0) = 8;
+    this->small_omega->at(1) = 3;
+    this->small_stop.get_data()[1] = this->stopped;
+
+    gko::kernels::reference::bicgstab::step_1(
+        this->exec, this->small_r.get(), this->small_p.get(),
+        this->small_v.get(), this->small_rho.get(), this->small_prev_rho.get(),
+        this->small_alpha.get(), this->small_omega.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_p, l({{-3.25, 3.0}, {-3.25, 3.0}}), 0);
+}
+
+
+TYPED_TEST(Bicgstab, KernelStep1DivRhoZero)
+{
+    this->small_p->fill(3);
+    this->small_r->fill(-2);
+    this->small_v->fill(1);
+    this->small_rho->fill(2);
+    this->small_prev_rho->fill(0);
+    this->small_alpha->fill(1);
+    this->small_omega->fill(1);
+
+    gko::kernels::reference::bicgstab::step_1(
+        this->exec, this->small_r.get(), this->small_p.get(),
+        this->small_v.get(), this->small_rho.get(), this->small_prev_rho.get(),
+        this->small_alpha.get(), this->small_omega.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_p, l({{-2.0, -2.0}, {-2.0, -2.0}}), 0);
+}
+
+
+TYPED_TEST(Bicgstab, KernelStep1DivOmegaZero)
+{
+    this->small_p->fill(3);
+    this->small_r->fill(-2);
+    this->small_v->fill(1);
+    this->small_rho->fill(2);
+    this->small_prev_rho->fill(1);
+    this->small_alpha->fill(1);
+    this->small_omega->fill(0);
+
+    gko::kernels::reference::bicgstab::step_1(
+        this->exec, this->small_r.get(), this->small_p.get(),
+        this->small_v.get(), this->small_rho.get(), this->small_prev_rho.get(),
+        this->small_alpha.get(), this->small_omega.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_p, l({{-2.0, -2.0}, {-2.0, -2.0}}), 0);
+}
+
+
+TYPED_TEST(Bicgstab, KernelStep1DivBothZero)
+{
+    this->small_p->fill(3);
+    this->small_r->fill(-2);
+    this->small_v->fill(1);
+    this->small_rho->fill(2);
+    this->small_prev_rho->fill(0);
+    this->small_alpha->fill(1);
+    this->small_omega->fill(0);
+
+    gko::kernels::reference::bicgstab::step_1(
+        this->exec, this->small_r.get(), this->small_p.get(),
+        this->small_v.get(), this->small_rho.get(), this->small_prev_rho.get(),
+        this->small_alpha.get(), this->small_omega.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_p, l({{-2.0, -2.0}, {-2.0, -2.0}}), 0);
+}
+
+
+TYPED_TEST(Bicgstab, KernelStep2)
+{
+    this->small_s->fill(5);
+    this->small_r->fill(-2);
+    this->small_v->fill(1);
+    this->small_alpha->fill(0);
+    this->small_rho->at(0) = 2;
+    this->small_rho->at(1) = 3;
+    this->small_beta->at(0) = 8;
+    this->small_beta->at(1) = 3;
+    this->small_stop.get_data()[1] = this->stopped;
+
+    gko::kernels::reference::bicgstab::step_2(
+        this->exec, this->small_r.get(), this->small_s.get(),
+        this->small_v.get(), this->small_rho.get(), this->small_alpha.get(),
+        this->small_beta.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_s, l({{-2.25, 5.0}, {-2.25, 5.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_alpha, l({{0.25, 0.0}}), 0);
+}
+
+
+TYPED_TEST(Bicgstab, KernelStep2DivZero)
+{
+    this->small_s->fill(5);
+    this->small_r->fill(-2);
+    this->small_v->fill(1);
+    this->small_alpha->fill(4);
+    this->small_rho->fill(1);
+    this->small_beta->fill(0);
+
+    gko::kernels::reference::bicgstab::step_2(
+        this->exec, this->small_r.get(), this->small_s.get(),
+        this->small_v.get(), this->small_rho.get(), this->small_alpha.get(),
+        this->small_beta.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_s, l({{-2.0, -2.0}, {-2.0, -2.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_alpha, l({{0.0, 0.0}}), 0);
+}
+
+
+TYPED_TEST(Bicgstab, KernelStep3)
+{
+    this->small_x->fill(5);
+    this->small_r->fill(-2);
+    this->small_s->fill(1);
+    this->small_y->fill(4);
+    this->small_z->fill(-6);
+    this->small_t->fill(7);
+    this->small_omega->fill(10);
+    this->small_beta->at(0) = 2;
+    this->small_beta->at(1) = 3;
+    this->small_gamma->at(0) = 8;
+    this->small_gamma->at(1) = 3;
+    this->small_alpha->at(0) = 1;
+    this->small_alpha->at(1) = -2;
+    this->small_stop.get_data()[1] = this->stopped;
+
+    gko::kernels::reference::bicgstab::step_3(
+        this->exec, this->small_x.get(), this->small_r.get(),
+        this->small_s.get(), this->small_t.get(), this->small_y.get(),
+        this->small_z.get(), this->small_alpha.get(), this->small_beta.get(),
+        this->small_gamma.get(), this->small_omega.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_x, l({{-15.0, 5.0}, {-15.0, 5.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_r, l({{-27.0, -2.0}, {-27.0, -2.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_omega, l({{4.0, 10.0}}), 0);
+}
+
+
+TYPED_TEST(Bicgstab, KernelStep3DivZero)
+{
+    this->small_x->fill(5);
+    this->small_r->fill(-2);
+    this->small_s->fill(1);
+    this->small_y->fill(4);
+    this->small_z->fill(-6);
+    this->small_t->fill(7);
+    this->small_omega->fill(10);
+    this->small_beta->fill(0);
+    this->small_gamma->at(0) = 8;
+    this->small_gamma->at(1) = 3;
+    this->small_alpha->at(0) = 1;
+    this->small_alpha->at(1) = -2;
+
+    gko::kernels::reference::bicgstab::step_3(
+        this->exec, this->small_x.get(), this->small_r.get(),
+        this->small_s.get(), this->small_t.get(), this->small_y.get(),
+        this->small_z.get(), this->small_alpha.get(), this->small_beta.get(),
+        this->small_gamma.get(), this->small_omega.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_x, l({{9.0, -3.0}, {9.0, -3.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_omega, l({{0.0, 0.0}}), 0);
+}
+
+
+TYPED_TEST(Bicgstab, KernelFinalize)
+{
+    this->small_x->fill(5);
+    this->small_y->fill(4);
+    this->small_alpha->at(0) = 1;
+    this->small_alpha->at(1) = -2;
+    this->small_stop.get_data()[0] = this->stopped;
+    this->small_stop.get_data()[1] = this->finalized;
+
+    gko::kernels::reference::bicgstab::finalize(
+        this->exec, this->small_x.get(), this->small_y.get(),
+        this->small_alpha.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_x, l({{9.0, 5.0}, {9.0, 5.0}}), 0);
+    ASSERT_EQ(this->small_stop.get_data()[0], this->finalized);
+    ASSERT_EQ(this->small_stop.get_data()[1], this->finalized);
+}
+
+
 TYPED_TEST(Bicgstab, SolvesDenseSystem)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/reference/test/solver/cg_kernels.cpp b/reference/test/solver/cg_kernels.cpp
index 6c443cfa6a9..bb2f9aa1ea6 100644
--- a/reference/test/solver/cg_kernels.cpp
+++ b/reference/test/solver/cg_kernels.cpp
@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/time.hpp>
 
 
+#include "core/solver/cg_kernels.hpp"
 #include "core/test/utils.hpp"
 
 
@@ -99,11 +100,46 @@ class Cg : public ::testing::Test {
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec))
-    {}
+    {
+        auto small_size = gko::dim<2>{2, 2};
+        auto small_scalar_size = gko::dim<2>{1, small_size[1]};
+        small_b = Mtx::create(exec, small_size, small_size[1] + 1);
+        small_x = Mtx::create(exec, small_size, small_size[1] + 2);
+        small_one = Mtx::create(exec, small_size);
+        small_zero = Mtx::create(exec, small_size);
+        small_prev_rho = Mtx::create(exec, small_scalar_size);
+        small_rho = Mtx::create(exec, small_scalar_size);
+        small_beta = Mtx::create(exec, small_scalar_size);
+        small_zero->fill(0);
+        small_one->fill(1);
+        small_r = small_zero->clone();
+        small_z = small_zero->clone();
+        small_p = small_zero->clone();
+        small_q = small_zero->clone();
+        small_stop = gko::Array<gko::stopping_status>(exec, small_size[1]);
+        stopped.stop(1);
+        non_stopped.reset();
+        std::fill_n(small_stop.get_data(), small_stop.get_num_elems(),
+                    non_stopped);
+    }
 
-    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<const gko::ReferenceExecutor> exec;
     std::shared_ptr<Mtx> mtx;
     std::shared_ptr<Mtx> mtx_big;
+    std::unique_ptr<Mtx> small_one;
+    std::unique_ptr<Mtx> small_zero;
+    std::unique_ptr<Mtx> small_prev_rho;
+    std::unique_ptr<Mtx> small_beta;
+    std::unique_ptr<Mtx> small_rho;
+    std::unique_ptr<Mtx> small_x;
+    std::unique_ptr<Mtx> small_b;
+    std::unique_ptr<Mtx> small_r;
+    std::unique_ptr<Mtx> small_z;
+    std::unique_ptr<Mtx> small_p;
+    std::unique_ptr<Mtx> small_q;
+    gko::Array<gko::stopping_status> small_stop;
+    gko::stopping_status stopped;
+    gko::stopping_status non_stopped;
     std::unique_ptr<typename Solver::Factory> cg_factory;
     std::unique_ptr<typename Solver::Factory> cg_factory_big;
     std::unique_ptr<typename Solver::Factory> cg_factory_big2;
@@ -112,6 +148,108 @@ class Cg : public ::testing::Test {
 TYPED_TEST_SUITE(Cg, gko::test::ValueTypes);
 
 
+TYPED_TEST(Cg, KernelInitialize)
+{
+    this->small_b->fill(2);
+    this->small_r->fill(0);
+    this->small_z->fill(1);
+    this->small_p->fill(1);
+    this->small_q->fill(1);
+    this->small_prev_rho->fill(0);
+    this->small_rho->fill(1);
+    std::fill_n(this->small_stop.get_data(), this->small_stop.get_num_elems(),
+                this->stopped);
+
+    gko::kernels::reference::cg::initialize(
+        this->exec, this->small_b.get(), this->small_r.get(),
+        this->small_z.get(), this->small_p.get(), this->small_q.get(),
+        this->small_prev_rho.get(), this->small_rho.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_r, this->small_b, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_z, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_p, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_q, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_rho, l({{0.0, 0.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_prev_rho, l({{1.0, 1.0}}), 0);
+    ASSERT_EQ(this->small_stop.get_data()[0], this->non_stopped);
+    ASSERT_EQ(this->small_stop.get_data()[1], this->non_stopped);
+}
+
+
+TYPED_TEST(Cg, KernelStep1)
+{
+    this->small_p->fill(3);
+    this->small_z->fill(-2);
+    this->small_rho->at(0) = 2;
+    this->small_rho->at(1) = 3;
+    this->small_prev_rho->at(0) = 8;
+    this->small_prev_rho->at(1) = 3;
+    this->small_stop.get_data()[1] = this->stopped;
+
+    gko::kernels::reference::cg::step_1(
+        this->exec, this->small_p.get(), this->small_z.get(),
+        this->small_rho.get(), this->small_prev_rho.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_p, l({{-1.25, 3.0}, {-1.25, 3.0}}), 0);
+}
+
+
+TYPED_TEST(Cg, KernelStep1DivByZero)
+{
+    this->small_p->fill(3);
+    this->small_z->fill(-2);
+    this->small_rho->fill(1);
+    this->small_prev_rho->fill(0);
+
+    gko::kernels::reference::cg::step_1(
+        this->exec, this->small_p.get(), this->small_z.get(),
+        this->small_rho.get(), this->small_prev_rho.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_p, l({{-2.0, -2.0}, {-2.0, -2.0}}), 0);
+}
+
+
+TYPED_TEST(Cg, KernelStep2)
+{
+    this->small_x->fill(-2);
+    this->small_p->fill(3);
+    this->small_r->fill(4);
+    this->small_q->fill(-5);
+    this->small_rho->at(0) = 2;
+    this->small_rho->at(1) = 3;
+    this->small_beta->at(0) = 8;
+    this->small_beta->at(1) = 3;
+    this->small_stop.get_data()[1] = this->stopped;
+
+    gko::kernels::reference::cg::step_2(
+        this->exec, this->small_x.get(), this->small_r.get(),
+        this->small_p.get(), this->small_q.get(), this->small_beta.get(),
+        this->small_rho.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_x, l({{-1.25, -2.0}, {-1.25, -2.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_r, l({{5.25, 4.0}, {5.25, 4.0}}), 0);
+}
+
+
+TYPED_TEST(Cg, KernelStep2DivByZero)
+{
+    this->small_x->fill(-2);
+    this->small_p->fill(3);
+    this->small_r->fill(4);
+    this->small_q->fill(-5);
+    this->small_rho->fill(1);
+    this->small_beta->fill(0);
+
+    gko::kernels::reference::cg::step_2(
+        this->exec, this->small_x.get(), this->small_r.get(),
+        this->small_p.get(), this->small_q.get(), this->small_beta.get(),
+        this->small_rho.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_x, l({{-2.0, -2.0}, {-2.0, -2.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_r, l({{4.0, 4.0}, {4.0, 4.0}}), 0);
+}
+
+
 TYPED_TEST(Cg, SolvesStencilSystem)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/reference/test/solver/cgs_kernels.cpp b/reference/test/solver/cgs_kernels.cpp
index 6916c0a8a2d..ad1c502c12f 100644
--- a/reference/test/solver/cgs_kernels.cpp
+++ b/reference/test/solver/cgs_kernels.cpp
@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/time.hpp>
 
 
+#include "core/solver/cgs_kernels.hpp"
 #include "core/test/utils.hpp"
 
 
@@ -97,11 +98,61 @@ class Cgs : public ::testing::Test {
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec))
-    {}
+    {
+        auto small_size = gko::dim<2>{2, 2};
+        auto small_scalar_size = gko::dim<2>{1, small_size[1]};
+        small_b = Mtx::create(exec, small_size, small_size[1] + 1);
+        small_x = Mtx::create(exec, small_size, small_size[1] + 2);
+        small_one = Mtx::create(exec, small_size);
+        small_zero = Mtx::create(exec, small_size);
+        small_prev_rho = Mtx::create(exec, small_scalar_size);
+        small_rho = Mtx::create(exec, small_scalar_size);
+        small_alpha = Mtx::create(exec, small_scalar_size);
+        small_beta = Mtx::create(exec, small_scalar_size);
+        small_gamma = Mtx::create(exec, small_scalar_size);
+        small_zero->fill(0);
+        small_one->fill(1);
+        small_r = small_zero->clone();
+        small_r = small_zero->clone();
+        small_r_tld = small_zero->clone();
+        small_p = small_zero->clone();
+        small_q = small_zero->clone();
+        small_u = small_zero->clone();
+        small_u_hat = small_zero->clone();
+        small_v = small_zero->clone();
+        small_v_hat = small_zero->clone();
+        small_t = small_zero->clone();
+        small_stop = gko::Array<gko::stopping_status>(exec, small_size[1]);
+        stopped.stop(1);
+        non_stopped.reset();
+        std::fill_n(small_stop.get_data(), small_stop.get_num_elems(),
+                    non_stopped);
+    }
 
-    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<const gko::ReferenceExecutor> exec;
     std::shared_ptr<Mtx> mtx;
     std::shared_ptr<Mtx> mtx_big;
+    std::unique_ptr<Mtx> small_one;
+    std::unique_ptr<Mtx> small_zero;
+    std::unique_ptr<Mtx> small_prev_rho;
+    std::unique_ptr<Mtx> small_rho;
+    std::unique_ptr<Mtx> small_alpha;
+    std::unique_ptr<Mtx> small_beta;
+    std::unique_ptr<Mtx> small_gamma;
+    std::unique_ptr<Mtx> small_x;
+    std::unique_ptr<Mtx> small_b;
+    std::unique_ptr<Mtx> small_r;
+    std::unique_ptr<Mtx> small_r_tld;
+    std::unique_ptr<Mtx> small_p;
+    std::unique_ptr<Mtx> small_q;
+    std::unique_ptr<Mtx> small_u;
+    std::unique_ptr<Mtx> small_u_hat;
+    std::unique_ptr<Mtx> small_v;
+    std::unique_ptr<Mtx> small_v_hat;
+    std::unique_ptr<Mtx> small_t;
+    gko::Array<gko::stopping_status> small_stop;
+    gko::stopping_status stopped;
+    gko::stopping_status non_stopped;
     std::unique_ptr<typename Solver::Factory> cgs_factory;
     std::unique_ptr<typename Solver::Factory> cgs_factory_big;
     std::unique_ptr<typename Solver::Factory> cgs_factory_big2;
@@ -110,6 +161,159 @@ class Cgs : public ::testing::Test {
 TYPED_TEST_SUITE(Cgs, gko::test::ValueTypes);
 
 
+TYPED_TEST(Cgs, KernelInitialize)
+{
+    this->small_b->fill(2);
+    this->small_r->fill(0);
+    this->small_r_tld->fill(0);
+    this->small_p->fill(1);
+    this->small_q->fill(1);
+    this->small_u->fill(1);
+    this->small_u_hat->fill(1);
+    this->small_v_hat->fill(1);
+    this->small_t->fill(1);
+    this->small_prev_rho->fill(0);
+    this->small_rho->fill(1);
+    std::fill_n(this->small_stop.get_data(), this->small_stop.get_num_elems(),
+                this->stopped);
+
+    gko::kernels::reference::cgs::initialize(
+        this->exec, this->small_b.get(), this->small_r.get(),
+        this->small_r_tld.get(), this->small_p.get(), this->small_q.get(),
+        this->small_u.get(), this->small_u_hat.get(), this->small_v_hat.get(),
+        this->small_t.get(), this->small_alpha.get(), this->small_beta.get(),
+        this->small_gamma.get(), this->small_prev_rho.get(),
+        this->small_rho.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_r, this->small_b, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_r_tld, this->small_b, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_p, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_q, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_u, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_u_hat, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_v_hat, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_t, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_rho, l({{0.0, 0.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_prev_rho, l({{1.0, 1.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_alpha, l({{1.0, 1.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_beta, l({{1.0, 1.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_gamma, l({{1.0, 1.0}}), 0);
+    ASSERT_EQ(this->small_stop.get_data()[0], this->non_stopped);
+    ASSERT_EQ(this->small_stop.get_data()[1], this->non_stopped);
+}
+
+
+TYPED_TEST(Cgs, KernelStep1)
+{
+    this->small_r->fill(1);
+    this->small_p->fill(-2);
+    this->small_q->fill(3);
+    this->small_u->fill(-4);
+    this->small_beta->fill(2);
+    this->small_prev_rho->at(0) = 2;
+    this->small_prev_rho->at(1) = 3;
+    this->small_rho->at(0) = -4;
+    this->small_rho->at(1) = 4;
+    this->small_stop.get_data()[1] = this->stopped;
+
+    gko::kernels::reference::cgs::step_1(
+        this->exec, this->small_r.get(), this->small_u.get(),
+        this->small_p.get(), this->small_q.get(), this->small_beta.get(),
+        this->small_rho.get(), this->small_prev_rho.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_u, l({{-5.0, -4.0}, {-5.0, -4.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_p, l({{-19.0, -2.0}, {-19.0, -2.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_beta, l({{-2.0, 2.0}}), 0);
+}
+
+
+TYPED_TEST(Cgs, KernelStep1DivZero)
+{
+    this->small_r->fill(1);
+    this->small_p->fill(-2);
+    this->small_q->fill(3);
+    this->small_u->fill(-4);
+    this->small_beta->fill(2);
+    this->small_prev_rho->fill(0);
+    this->small_rho->fill(3);
+
+    gko::kernels::reference::cgs::step_1(
+        this->exec, this->small_r.get(), this->small_u.get(),
+        this->small_p.get(), this->small_q.get(), this->small_beta.get(),
+        this->small_rho.get(), this->small_prev_rho.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_u, l({{7.0, 7.0}, {7.0, 7.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_p, l({{5.0, 5.0}, {5.0, 5.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_beta, l({{2.0, 2.0}}), 0);
+}
+
+
+TYPED_TEST(Cgs, KernelStep2)
+{
+    this->small_q->fill(1);
+    this->small_u->fill(-2);
+    this->small_v_hat->fill(3);
+    this->small_t->fill(-4);
+    this->small_alpha->fill(2);
+    this->small_gamma->at(0) = 2;
+    this->small_gamma->at(1) = 3;
+    this->small_rho->at(0) = -4;
+    this->small_rho->at(1) = 4;
+    this->small_stop.get_data()[1] = this->stopped;
+
+    gko::kernels::reference::cgs::step_2(
+        this->exec, this->small_u.get(), this->small_v_hat.get(),
+        this->small_q.get(), this->small_t.get(), this->small_alpha.get(),
+        this->small_rho.get(), this->small_gamma.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_q, l({{4.0, 1.0}, {4.0, 1.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_t, l({{2.0, -4.0}, {2.0, -4.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_alpha, l({{-2.0, 2.0}}), 0);
+}
+
+
+TYPED_TEST(Cgs, KernelStep2DivZero)
+{
+    this->small_q->fill(1);
+    this->small_u->fill(-2);
+    this->small_v_hat->fill(3);
+    this->small_t->fill(-4);
+    this->small_alpha->fill(2);
+    this->small_gamma->fill(0);
+    this->small_rho->fill(-3);
+
+    gko::kernels::reference::cgs::step_2(
+        this->exec, this->small_u.get(), this->small_v_hat.get(),
+        this->small_q.get(), this->small_t.get(), this->small_alpha.get(),
+        this->small_rho.get(), this->small_gamma.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_q, l({{-8.0, -8.0}, {-8.0, -8.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_t, l({{-10.0, -10.0}, {-10.0, -10.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_alpha, l({{2.0, 2.0}}), 0);
+}
+
+
+TYPED_TEST(Cgs, KernelStep3)
+{
+    this->small_r->fill(1);
+    this->small_t->fill(-2);
+    this->small_x->fill(3);
+    this->small_u_hat->fill(-4);
+    this->small_beta->fill(2);
+    this->small_alpha->at(0) = 2;
+    this->small_alpha->at(1) = 3;
+    this->small_stop.get_data()[1] = this->stopped;
+
+    gko::kernels::reference::cgs::step_3(
+        this->exec, this->small_t.get(), this->small_u_hat.get(),
+        this->small_r.get(), this->small_x.get(), this->small_alpha.get(),
+        &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_r, l({{5.0, 1.0}, {5.0, 1.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_x, l({{-5.0, 3.0}, {-5.0, 3.0}}), 0);
+}
+
+
 TYPED_TEST(Cgs, SolvesDenseSystem)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/reference/test/solver/fcg_kernels.cpp b/reference/test/solver/fcg_kernels.cpp
index e374211f91b..7422935a8db 100644
--- a/reference/test/solver/fcg_kernels.cpp
+++ b/reference/test/solver/fcg_kernels.cpp
@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/time.hpp>
 
 
+#include "core/solver/fcg_kernels.hpp"
 #include "core/test/utils.hpp"
 
 
@@ -99,11 +100,50 @@ class Fcg : public ::testing::Test {
                           .with_reduction_factor(r<value_type>::value)
                           .on(exec))
                   .on(exec))
-    {}
+    {
+        auto small_size = gko::dim<2>{2, 2};
+        auto small_scalar_size = gko::dim<2>{1, small_size[1]};
+        small_b = Mtx::create(exec, small_size, small_size[1] + 1);
+        small_x = Mtx::create(exec, small_size, small_size[1] + 2);
+        small_one = Mtx::create(exec, small_size);
+        small_zero = Mtx::create(exec, small_size);
+        small_prev_rho = Mtx::create(exec, small_scalar_size);
+        small_rho = Mtx::create(exec, small_scalar_size);
+        small_rho_t = Mtx::create(exec, small_scalar_size);
+        small_beta = Mtx::create(exec, small_scalar_size);
+        small_zero->fill(0);
+        small_one->fill(1);
+        small_r = small_zero->clone();
+        small_t = small_zero->clone();
+        small_z = small_zero->clone();
+        small_p = small_zero->clone();
+        small_q = small_zero->clone();
+        small_stop = gko::Array<gko::stopping_status>(exec, small_size[1]);
+        stopped.stop(1);
+        non_stopped.reset();
+        std::fill_n(small_stop.get_data(), small_stop.get_num_elems(),
+                    non_stopped);
+    }
 
-    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<const gko::ReferenceExecutor> exec;
     std::shared_ptr<Mtx> mtx;
     std::shared_ptr<Mtx> mtx_big;
+    std::unique_ptr<Mtx> small_one;
+    std::unique_ptr<Mtx> small_zero;
+    std::unique_ptr<Mtx> small_prev_rho;
+    std::unique_ptr<Mtx> small_beta;
+    std::unique_ptr<Mtx> small_rho;
+    std::unique_ptr<Mtx> small_rho_t;
+    std::unique_ptr<Mtx> small_x;
+    std::unique_ptr<Mtx> small_b;
+    std::unique_ptr<Mtx> small_r;
+    std::unique_ptr<Mtx> small_t;
+    std::unique_ptr<Mtx> small_z;
+    std::unique_ptr<Mtx> small_p;
+    std::unique_ptr<Mtx> small_q;
+    gko::Array<gko::stopping_status> small_stop;
+    gko::stopping_status stopped;
+    gko::stopping_status non_stopped;
     std::unique_ptr<typename Solver::Factory> fcg_factory;
     std::unique_ptr<typename Solver::Factory> fcg_factory_big;
     std::unique_ptr<typename Solver::Factory> fcg_factory_big2;
@@ -112,6 +152,117 @@ class Fcg : public ::testing::Test {
 TYPED_TEST_SUITE(Fcg, gko::test::ValueTypes);
 
 
+TYPED_TEST(Fcg, KernelInitialize)
+{
+    this->small_b->fill(2);
+    this->small_r->fill(0);
+    this->small_z->fill(1);
+    this->small_p->fill(1);
+    this->small_q->fill(1);
+    this->small_t->fill(1);
+    this->small_prev_rho->fill(0);
+    this->small_rho->fill(1);
+    this->small_rho_t->fill(0);
+    std::fill_n(this->small_stop.get_data(), this->small_stop.get_num_elems(),
+                this->stopped);
+
+    gko::kernels::reference::fcg::initialize(
+        this->exec, this->small_b.get(), this->small_r.get(),
+        this->small_z.get(), this->small_p.get(), this->small_q.get(),
+        this->small_t.get(), this->small_prev_rho.get(), this->small_rho.get(),
+        this->small_rho_t.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_r, this->small_b, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_t, this->small_b, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_z, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_p, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_q, this->small_zero, 0);
+    GKO_ASSERT_MTX_NEAR(this->small_rho, l({{0.0, 0.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_rho_t, l({{1.0, 1.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_prev_rho, l({{1.0, 1.0}}), 0);
+    ASSERT_EQ(this->small_stop.get_data()[0], this->non_stopped);
+    ASSERT_EQ(this->small_stop.get_data()[1], this->non_stopped);
+}
+
+
+TYPED_TEST(Fcg, KernelStep1)
+{
+    this->small_p->fill(3);
+    this->small_z->fill(-2);
+    this->small_rho_t->at(0) = 2;
+    this->small_rho_t->at(1) = 3;
+    this->small_prev_rho->at(0) = 8;
+    this->small_prev_rho->at(1) = 3;
+    this->small_stop.get_data()[1] = this->stopped;
+
+    gko::kernels::reference::fcg::step_1(
+        this->exec, this->small_p.get(), this->small_z.get(),
+        this->small_rho_t.get(), this->small_prev_rho.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_p, l({{-1.25, 3.0}, {-1.25, 3.0}}), 0);
+}
+
+
+TYPED_TEST(Fcg, KernelStep1DivByZero)
+{
+    this->small_p->fill(3);
+    this->small_z->fill(-2);
+    this->small_rho_t->fill(1);
+    this->small_prev_rho->fill(0);
+
+    gko::kernels::reference::fcg::step_1(
+        this->exec, this->small_p.get(), this->small_z.get(),
+        this->small_rho_t.get(), this->small_prev_rho.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_p, l({{-2.0, -2.0}, {-2.0, -2.0}}), 0);
+}
+
+
+TYPED_TEST(Fcg, KernelStep2)
+{
+    this->small_x->fill(-2);
+    this->small_p->fill(3);
+    this->small_r->fill(4);
+    this->small_q->fill(-5);
+    this->small_t->fill(8);
+    this->small_rho->at(0) = 2;
+    this->small_rho->at(1) = 3;
+    this->small_beta->at(0) = 8;
+    this->small_beta->at(1) = 3;
+    this->small_stop.get_data()[1] = this->stopped;
+
+    gko::kernels::reference::fcg::step_2(
+        this->exec, this->small_x.get(), this->small_r.get(),
+        this->small_t.get(), this->small_p.get(), this->small_q.get(),
+        this->small_beta.get(), this->small_rho.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_x, l({{-1.25, -2.0}, {-1.25, -2.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_r, l({{5.25, 4.0}, {5.25, 4.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_t, l({{1.25, 8.0}, {1.25, 8.0}}), 0);
+}
+
+
+TYPED_TEST(Fcg, KernelStep2DivByZero)
+{
+    this->small_x->fill(-2);
+    this->small_p->fill(3);
+    this->small_r->fill(4);
+    this->small_q->fill(-5);
+    this->small_t->fill(8);
+    this->small_rho->fill(1);
+    this->small_beta->fill(0);
+
+    gko::kernels::reference::fcg::step_2(
+        this->exec, this->small_x.get(), this->small_r.get(),
+        this->small_t.get(), this->small_p.get(), this->small_q.get(),
+        this->small_beta.get(), this->small_rho.get(), &this->small_stop);
+
+    GKO_ASSERT_MTX_NEAR(this->small_x, l({{-2.0, -2.0}, {-2.0, -2.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_r, l({{4.0, 4.0}, {4.0, 4.0}}), 0);
+    GKO_ASSERT_MTX_NEAR(this->small_t, l({{8.0, 8.0}, {8.0, 8.0}}), 0);
+}
+
+
 TYPED_TEST(Fcg, SolvesStencilSystem)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/reference/test/solver/ir_kernels.cpp b/reference/test/solver/ir_kernels.cpp
index c4007bc5fe6..590c8ee31d5 100644
--- a/reference/test/solver/ir_kernels.cpp
+++ b/reference/test/solver/ir_kernels.cpp
@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/residual_norm.hpp>
 
 
+#include "core/solver/ir_kernels.hpp"
 #include "core/test/utils.hpp"
 
 
@@ -75,7 +76,7 @@ class Ir : public ::testing::Test {
                   .on(exec))
     {}
 
-    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<const gko::ReferenceExecutor> exec;
     std::shared_ptr<Mtx> mtx;
     std::unique_ptr<typename Solver::Factory> ir_factory;
 };
@@ -83,6 +84,22 @@ class Ir : public ::testing::Test {
 TYPED_TEST_SUITE(Ir, gko::test::ValueTypes);
 
 
+TYPED_TEST(Ir, KernelInitialize)
+{
+    gko::stopping_status stopped;
+    gko::stopping_status non_stopped;
+    auto stop = gko::Array<gko::stopping_status>(this->exec, 2);
+    stopped.stop(1);
+    non_stopped.reset();
+    std::fill_n(stop.get_data(), stop.get_num_elems(), non_stopped);
+
+    gko::kernels::reference::ir::initialize(this->exec, &stop);
+
+    ASSERT_EQ(stop.get_data()[0], non_stopped);
+    ASSERT_EQ(stop.get_data()[1], non_stopped);
+}
+
+
 TYPED_TEST(Ir, SolvesTriangularSystem)
 {
     using Mtx = typename TestFixture::Mtx;

From 2c37f28c2867c47572a4dc623dcf64baff0cf65f Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 11 Jun 2021 09:08:43 +0200
Subject: [PATCH 0767/1008] fix uninitialized data in tests

---
 reference/test/solver/bicg_kernels.cpp     | 2 ++
 reference/test/solver/bicgstab_kernels.cpp | 3 +++
 reference/test/solver/cg_kernels.cpp       | 2 ++
 reference/test/solver/cgs_kernels.cpp      | 2 ++
 reference/test/solver/fcg_kernels.cpp      | 2 ++
 reference/test/solver/ir_kernels.cpp       | 4 ++--
 6 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/reference/test/solver/bicg_kernels.cpp b/reference/test/solver/bicg_kernels.cpp
index aac5a26bd6d..1a5b98d1778 100644
--- a/reference/test/solver/bicg_kernels.cpp
+++ b/reference/test/solver/bicg_kernels.cpp
@@ -62,6 +62,8 @@ class Bicg : public ::testing::Test {
         : exec(gko::ReferenceExecutor::create()),
           mtx(gko::initialize<Mtx>(
               {{2, -1.0, 0.0}, {-1.0, 2, -1.0}, {0.0, -1.0, 2}}, exec)),
+          stopped{},
+          non_stopped{},
           bicg_factory(
               Solver::build()
                   .with_criteria(
diff --git a/reference/test/solver/bicgstab_kernels.cpp b/reference/test/solver/bicgstab_kernels.cpp
index cbbf60b955c..85b3a591378 100644
--- a/reference/test/solver/bicgstab_kernels.cpp
+++ b/reference/test/solver/bicgstab_kernels.cpp
@@ -63,6 +63,9 @@ class Bicgstab : public ::testing::Test {
         : exec(gko::ReferenceExecutor::create()),
           mtx(gko::initialize<Mtx>(
               {{1.0, -3.0, 0.0}, {-4.0, 1.0, -3.0}, {2.0, -1.0, 2.0}}, exec)),
+          stopped{},
+          finalized{},
+          non_stopped{},
           bicgstab_factory(
               Solver::build()
                   .with_criteria(
diff --git a/reference/test/solver/cg_kernels.cpp b/reference/test/solver/cg_kernels.cpp
index bb2f9aa1ea6..d5b296afe61 100644
--- a/reference/test/solver/cg_kernels.cpp
+++ b/reference/test/solver/cg_kernels.cpp
@@ -62,6 +62,8 @@ class Cg : public ::testing::Test {
         : exec(gko::ReferenceExecutor::create()),
           mtx(gko::initialize<Mtx>(
               {{2, -1.0, 0.0}, {-1.0, 2, -1.0}, {0.0, -1.0, 2}}, exec)),
+          stopped{},
+          non_stopped{},
           cg_factory(
               Solver::build()
                   .with_criteria(
diff --git a/reference/test/solver/cgs_kernels.cpp b/reference/test/solver/cgs_kernels.cpp
index ad1c502c12f..74c80c8bfe7 100644
--- a/reference/test/solver/cgs_kernels.cpp
+++ b/reference/test/solver/cgs_kernels.cpp
@@ -63,6 +63,8 @@ class Cgs : public ::testing::Test {
         : exec(gko::ReferenceExecutor::create()),
           mtx(gko::initialize<Mtx>(
               {{1.0, -3.0, 0.0}, {-4.0, 1.0, -3.0}, {2.0, -1.0, 2.0}}, exec)),
+          stopped{},
+          non_stopped{},
           cgs_factory(
               Solver::build()
                   .with_criteria(
diff --git a/reference/test/solver/fcg_kernels.cpp b/reference/test/solver/fcg_kernels.cpp
index 7422935a8db..5be80feb2ec 100644
--- a/reference/test/solver/fcg_kernels.cpp
+++ b/reference/test/solver/fcg_kernels.cpp
@@ -63,6 +63,8 @@ class Fcg : public ::testing::Test {
         : exec(gko::ReferenceExecutor::create()),
           mtx(gko::initialize<Mtx>(
               {{2, -1.0, 0.0}, {-1.0, 2, -1.0}, {0.0, -1.0, 2}}, exec)),
+          stopped{},
+          non_stopped{},
           fcg_factory(
               Solver::build()
                   .with_criteria(
diff --git a/reference/test/solver/ir_kernels.cpp b/reference/test/solver/ir_kernels.cpp
index 590c8ee31d5..43a3acbb6d7 100644
--- a/reference/test/solver/ir_kernels.cpp
+++ b/reference/test/solver/ir_kernels.cpp
@@ -86,8 +86,8 @@ TYPED_TEST_SUITE(Ir, gko::test::ValueTypes);
 
 TYPED_TEST(Ir, KernelInitialize)
 {
-    gko::stopping_status stopped;
-    gko::stopping_status non_stopped;
+    gko::stopping_status stopped{};
+    gko::stopping_status non_stopped{};
     auto stop = gko::Array<gko::stopping_status>(this->exec, 2);
     stopped.stop(1);
     non_stopped.reset();

From e89e4515e4316d6ef04412048939f1cd208388ea Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 16 Jun 2021 15:43:28 +0200
Subject: [PATCH 0768/1008] fail on unsupported build type

---
 cmake/build_type_helpers.cmake | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cmake/build_type_helpers.cmake b/cmake/build_type_helpers.cmake
index a39de34cf82..09fc4a7dc9b 100644
--- a/cmake/build_type_helpers.cmake
+++ b/cmake/build_type_helpers.cmake
@@ -115,6 +115,11 @@ foreach(_LANG IN LISTS ENABLED_LANGUAGES ITEMS "HIP")
     endforeach()
 endforeach()
 
+if (CMAKE_BUILD_TYPE
+    AND (CMAKE_BUILD_TYPE IN_LIST ${PROJECT_NAME}_CUSTOM_BUILD_TYPES)
+    AND (NOT ${PROJECT_NAME}_${CMAKE_BUILD_TYPE}_SUPPORTED))
+    message(FATAL_ERROR "Custom CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE} not supported by the compiler")
+endif()
 
 foreach(_TYPE IN LISTS ${PROJECT_NAME}_CUSTOM_BUILD_TYPES)
     cmake_dependent_option(${PROJECT_NAME}_${_TYPE}_IN_CONFIGURATION_TYPES

From 37f7e8169e5837668a36c4bbb40ca2e345fe0b5b Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 16 Jun 2021 15:43:51 +0200
Subject: [PATCH 0769/1008] fix Idr UBSAN warning

---
 include/ginkgo/core/solver/idr.hpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp
index a24ea0db177..45551eea415 100644
--- a/include/ginkgo/core/solver/idr.hpp
+++ b/include/ginkgo/core/solver/idr.hpp
@@ -287,10 +287,10 @@ class Idr : public EnableLinOp<Idr<ValueType>>,
 private:
     std::shared_ptr<const LinOp> system_matrix_{};
     std::shared_ptr<const stop::CriterionFactory> stop_criterion_factory_{};
-    size_type subspace_dim_;
-    remove_complex<ValueType> kappa_;
-    bool deterministic_;
-    bool complex_subspace_;
+    size_type subspace_dim_{};
+    remove_complex<ValueType> kappa_{};
+    bool deterministic_{};
+    bool complex_subspace_{};
 };
 
 

From 5214ad15efeafd49426366351bb88f09612a7bdb Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 16 Jun 2021 15:44:15 +0200
Subject: [PATCH 0770/1008] fix ISAI out-of-bounds read

---
 common/preconditioner/isai_kernels.hpp.inc | 3 +++
 omp/preconditioner/isai_kernels.cpp        | 3 +++
 reference/preconditioner/isai_kernels.cpp  | 3 +++
 3 files changed, 9 insertions(+)

diff --git a/common/preconditioner/isai_kernels.hpp.inc b/common/preconditioner/isai_kernels.hpp.inc
index d2ee667e7a4..27fc0c35b32 100644
--- a/common/preconditioner/isai_kernels.hpp.inc
+++ b/common/preconditioner/isai_kernels.hpp.inc
@@ -381,6 +381,9 @@ __global__ __launch_bounds__(default_block_size) void scale_excess_solution(
     const IndexType offset = excess_block_ptrs[e_start];
     const IndexType block_begin = excess_block_ptrs[row] - offset;
     const IndexType block_end = excess_block_ptrs[row + 1] - offset;
+    if (block_end == block_begin) {
+        return;
+    }
     const auto diag = excess_solution[block_end - 1];
     const ValueType scal = one<ValueType>() / sqrt(diag);
 
diff --git a/omp/preconditioner/isai_kernels.cpp b/omp/preconditioner/isai_kernels.cpp
index 7cac20afab1..b7866770b1e 100644
--- a/omp/preconditioner/isai_kernels.cpp
+++ b/omp/preconditioner/isai_kernels.cpp
@@ -424,6 +424,9 @@ void scale_excess_solution(std::shared_ptr<const DefaultExecutor>,
     for (size_type row = e_start; row < e_end; ++row) {
         IndexType block_start = excess_block_ptrs[row] - offset;
         IndexType block_end = excess_block_ptrs[row + 1] - offset;
+        if (block_end == block_start) {
+            continue;
+        }
         const ValueType diag = excess_values[block_end - 1];
         const ValueType scal = one<ValueType>() / sqrt(diag);
         for (size_type i = block_start; i < block_end; i++) {
diff --git a/reference/preconditioner/isai_kernels.cpp b/reference/preconditioner/isai_kernels.cpp
index 5e0dccbea09..2b91b54cd07 100644
--- a/reference/preconditioner/isai_kernels.cpp
+++ b/reference/preconditioner/isai_kernels.cpp
@@ -410,6 +410,9 @@ void scale_excess_solution(std::shared_ptr<const DefaultExecutor>,
     for (size_type row = e_start; row < e_end; ++row) {
         block_start = excess_block_ptrs[row] - offset;
         block_end = excess_block_ptrs[row + 1] - offset;
+        if (block_end == block_start) {
+            continue;
+        }
         const ValueType scal =
             one<ValueType>() / sqrt(excess_values[block_end - 1]);
         for (size_type i = block_start; i < block_end; i++) {

From 85fcff8ca7b94fbcf69734fe031fcbeefe5243e2 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 16 Jun 2021 15:45:16 +0200
Subject: [PATCH 0771/1008] run cuda-memcheck on amdci

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 65559c5ca07..780ca79fd46 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -976,7 +976,7 @@ cudamemcheck:
   stage: QoS_tools
   extends:
     - .deploy_condition
-    - .use_gko-cuda101-gnu8-llvm10-intel2019
+    - .use_gko-cuda110-gnu9-llvm9-intel2020
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=RelWithDebInfo

From 059076fdfa4c881e35dacc81a8b5846dca4270ca Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 17 Jun 2021 15:56:33 +0200
Subject: [PATCH 0772/1008] run cuda-memcheck with CUDA 10.1 on amdci

HIP doesn't support CUDA 11.*
---
 .gitlab-ci.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 780ca79fd46..4299f00a5bb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -976,7 +976,10 @@ cudamemcheck:
   stage: QoS_tools
   extends:
     - .deploy_condition
-    - .use_gko-cuda110-gnu9-llvm9-intel2020
+  image: ginkgohub/cuda:101-gnu8-llvm10-intel2019
+  tags:
+    - private_ci
+    - nvidia-gpu
   before_script: *default_before_script
   script:
     - ctest -V -S cmake/CTestScript.cmake -DCTEST_BUILD_CONFIGURATION=RelWithDebInfo

From b14d6c0f014e415fcce17867ae3ebc3acd90835b Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 22 Jun 2021 12:14:10 +0200
Subject: [PATCH 0773/1008] add matrix dimensions to matrix_statistics

---
 benchmark/matrix_statistics/matrix_statistics.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/benchmark/matrix_statistics/matrix_statistics.cpp b/benchmark/matrix_statistics/matrix_statistics.cpp
index 4501f67a489..d8a3ce8fc1b 100644
--- a/benchmark/matrix_statistics/matrix_statistics.cpp
+++ b/benchmark/matrix_statistics/matrix_statistics.cpp
@@ -150,6 +150,10 @@ void extract_matrix_statistics(gko::matrix_data<etype, gko::int64> &data,
         ++col_dist[v.column];
     }
 
+    add_or_set_member(problem, "rows", data.size[0], allocator);
+    add_or_set_member(problem, "columns", data.size[1], allocator);
+    add_or_set_member(problem, "nonzeros", data.nonzeros.size(), allocator);
+
     std::sort(begin(row_dist), end(row_dist));
     add_or_set_member(problem, "row_distribution",
                       rapidjson::Value(rapidjson::kObjectType), allocator);

From 4f1273a2632c90871e701b579c6341190ba44dc2 Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Fri, 11 Jun 2021 11:19:50 +0200
Subject: [PATCH 0774/1008] changes repetition flag type to string

---
 benchmark/blas/blas.cpp                     |  3 ++-
 benchmark/conversions/conversions.cpp       |  3 ++-
 benchmark/preconditioner/preconditioner.cpp | 18 +++++++++---------
 benchmark/solver/solver.cpp                 |  5 +++--
 benchmark/spmv/spmv.cpp                     |  5 +++--
 benchmark/utils/general.hpp                 |  9 ++++++++-
 6 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/benchmark/blas/blas.cpp b/benchmark/blas/blas.cpp
index 1934aac853a..6bcff9c15bf 100644
--- a/benchmark/blas/blas.cpp
+++ b/benchmark/blas/blas.cpp
@@ -447,7 +447,8 @@ void apply_blas(const char *operation_name, std::shared_ptr<gko::Executor> exec,
 
         // timed run
         auto timer = get_timer(exec, FLAGS_gpu_timer);
-        for (unsigned int i = 0; i < FLAGS_repetitions; i++) {
+        const auto repetitions = get_repetitions();
+        for (unsigned int i = 0; i < repetitions; i++) {
             op->prepare();
             exec->synchronize();
             timer->tic();
diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index 9c36ad24537..116280c5270 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -80,8 +80,9 @@ void convert_matrix(const gko::LinOp *matrix_from, const char *format_to,
             matrix_to->clear();
         }
         auto timer = get_timer(exec, FLAGS_gpu_timer);
+        const auto repetitions = get_repetitions();
         // timed run
-        for (unsigned int i = 0; i < FLAGS_repetitions; i++) {
+        for (unsigned int i = 0; i < repetitions; i++) {
             exec->synchronize();
             timer->tic();
             matrix_to->copy_from(matrix_from);
diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index f86011632d5..685edebdca8 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -159,6 +159,7 @@ void run_preconditioner(const char *precond_name,
                               allocator);
         }
 
+        const auto repetitions = get_repetitions();
         {
             // fast run, gets total time
             auto x_clone = clone(x);
@@ -174,28 +175,27 @@ void run_preconditioner(const char *precond_name,
             exec->synchronize();
             generate_timer->tic();
             std::unique_ptr<gko::LinOp> precond_op;
-            for (auto i = 0u; i < FLAGS_repetitions; ++i) {
+            for (auto i = 0u; i < repetitions; ++i) {
                 precond_op = precond->generate(system_matrix);
             }
             generate_timer->toc();
 
             // the timer is out of the loops to reduce calling synchronize
             // overhead, so the timer does not know the number of repetitions.
-            auto generate_time =
-                generate_timer->get_total_time() / FLAGS_repetitions;
+            auto generate_time = generate_timer->get_total_time() / repetitions;
             add_or_set_member(this_precond_data["generate"], "time",
                               generate_time, allocator);
 
             exec->synchronize();
             apply_timer->tic();
-            for (auto i = 0u; i < FLAGS_repetitions; ++i) {
+            for (auto i = 0u; i < repetitions; ++i) {
                 precond_op->apply(lend(b), lend(x_clone));
             }
             apply_timer->toc();
 
             // the timer is out of the loops to reduce calling synchronize
             // overhead, so the timer does not know the number of repetitions.
-            auto apply_time = apply_timer->get_total_time() / FLAGS_repetitions;
+            auto apply_time = apply_timer->get_total_time() / repetitions;
             add_or_set_member(this_precond_data["apply"], "time", apply_time,
                               allocator);
         }
@@ -209,24 +209,24 @@ void run_preconditioner(const char *precond_name,
                 std::make_shared<OperationLogger>(exec, FLAGS_nested_names);
             exec->add_logger(gen_logger);
             std::unique_ptr<gko::LinOp> precond_op;
-            for (auto i = 0u; i < FLAGS_repetitions; ++i) {
+            for (auto i = 0u; i < repetitions; ++i) {
                 precond_op = precond->generate(system_matrix);
             }
             exec->remove_logger(gko::lend(gen_logger));
 
             gen_logger->write_data(this_precond_data["generate"]["components"],
-                                   allocator, FLAGS_repetitions);
+                                   allocator, repetitions);
 
             auto apply_logger =
                 std::make_shared<OperationLogger>(exec, FLAGS_nested_names);
             exec->add_logger(apply_logger);
-            for (auto i = 0u; i < FLAGS_repetitions; ++i) {
+            for (auto i = 0u; i < repetitions; ++i) {
                 precond_op->apply(lend(b), lend(x_clone));
             }
             exec->remove_logger(gko::lend(apply_logger));
 
             apply_logger->write_data(this_precond_data["apply"]["components"],
-                                     allocator, FLAGS_repetitions);
+                                     allocator, repetitions);
         }
 
         add_or_set_member(this_precond_data, "completed", true, allocator);
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index e90bc738f4c..fb608d9d760 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -473,7 +473,8 @@ void solve_system(const std::string &solver_name,
         // timed run
         auto generate_timer = get_timer(exec, FLAGS_gpu_timer);
         auto apply_timer = get_timer(exec, FLAGS_gpu_timer);
-        for (unsigned int i = 0; i < FLAGS_repetitions; i++) {
+        const auto repetitions = get_repetitions();
+        for (unsigned int i = 0; i < repetitions; i++) {
             auto x_clone = clone(x);
 
             exec->synchronize();
@@ -488,7 +489,7 @@ void solve_system(const std::string &solver_name,
             solver->apply(lend(b), lend(x_clone));
             apply_timer->toc();
 
-            if (b->get_size()[1] == 1 && i == FLAGS_repetitions - 1 &&
+            if (b->get_size()[1] == 1 && i == repetitions - 1 &&
                 !FLAGS_overhead) {
                 auto residual = compute_residual_norm(lend(system_matrix),
                                                       lend(b), lend(x_clone));
diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp
index c455bbfa78f..ceb6196d484 100644
--- a/benchmark/spmv/spmv.cpp
+++ b/benchmark/spmv/spmv.cpp
@@ -99,6 +99,7 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
             exec->synchronize();
         }
 
+        const auto repetitions = get_repetitions();
         // tuning run
 #ifdef GINKGO_BENCHMARK_ENABLE_TUNING
         auto &format_case = spmv_case[format_name];
@@ -123,7 +124,7 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
             // variable is used.
             gko::_tuned_value = val;
             auto tuning_timer = get_timer(exec, FLAGS_gpu_timer);
-            for (unsigned int i = 0; i < FLAGS_repetitions; i++) {
+            for (unsigned int i = 0; i < repetitions; i++) {
                 auto x_clone = clone(x);
                 exec->synchronize();
                 tuning_timer->tic();
@@ -141,7 +142,7 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
 
         // timed run
         auto timer = get_timer(exec, FLAGS_gpu_timer);
-        for (unsigned int i = 0; i < FLAGS_repetitions; i++) {
+        for (unsigned int i = 0; i < repetitions; i++) {
             auto x_clone = clone(x);
             exec->synchronize();
             timer->tic();
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index d03875d87b1..0e38fbb5723 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -87,7 +87,7 @@ DEFINE_uint32(seed, 42, "Seed used for the random number generator");
 
 DEFINE_uint32(warmup, 2, "Warm-up repetitions");
 
-DEFINE_uint32(repetitions, 10,
+DEFINE_string(repetitions, "10",
               "Number of runs used to obtain an averaged result.");
 
 
@@ -434,4 +434,11 @@ gko::remove_complex<ValueType> compute_max_relative_norm2(
 }
 
 
+// parses the repetitions flag
+unsigned int get_repetitions()
+{
+    const std::string reps_str = FLAGS_repetitions;
+    return static_cast<unsigned int>(std::stoi(reps_str));
+}
+
 #endif  // GKO_BENCHMARK_UTILS_GENERAL_HPP_

From 43c3329abf2dfeceff2705bb1f596c81dec3ce6e Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Fri, 11 Jun 2021 11:26:14 +0200
Subject: [PATCH 0775/1008] adds repetition number to benchmark json output

---
 benchmark/blas/blas.cpp               | 2 ++
 benchmark/conversions/conversions.cpp | 2 ++
 benchmark/solver/solver.cpp           | 1 +
 benchmark/spmv/spmv.cpp               | 2 ++
 4 files changed, 7 insertions(+)

diff --git a/benchmark/blas/blas.cpp b/benchmark/blas/blas.cpp
index 6bcff9c15bf..839add915e6 100644
--- a/benchmark/blas/blas.cpp
+++ b/benchmark/blas/blas.cpp
@@ -464,6 +464,8 @@ void apply_blas(const char *operation_name, std::shared_ptr<gko::Executor> exec,
                           allocator);
         add_or_set_member(blas_case[operation_name], "bandwidth", mem / runtime,
                           allocator);
+        add_or_set_member(blas_case[operation_name], "repetitions", repetitions,
+                          allocator);
 
         // compute and write benchmark data
         add_or_set_member(blas_case[operation_name], "completed", true,
diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index 116280c5270..46eeb6e3d4c 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -91,6 +91,8 @@ void convert_matrix(const gko::LinOp *matrix_from, const char *format_to,
         }
         add_or_set_member(conversion_case[conversion_name], "time",
                           timer->compute_average_time(), allocator);
+        add_or_set_member(conversion_case[conversion_name], "repetitions",
+                          repetitions, allocator);
 
         // compute and write benchmark data
         add_or_set_member(conversion_case[conversion_name], "completed", true,
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index fb608d9d760..1000685c860 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -501,6 +501,7 @@ void solve_system(const std::string &solver_name,
                           generate_timer->compute_average_time(), allocator);
         add_or_set_member(solver_json["apply"], "time",
                           apply_timer->compute_average_time(), allocator);
+        add_or_set_member(solver_json, "repetitions", repetitions, allocator);
 
         // compute and write benchmark data
         add_or_set_member(solver_json, "completed", true, allocator);
diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp
index ceb6196d484..30650e6ca73 100644
--- a/benchmark/spmv/spmv.cpp
+++ b/benchmark/spmv/spmv.cpp
@@ -151,6 +151,8 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
         }
         add_or_set_member(spmv_case[format_name], "time",
                           timer->compute_average_time(), allocator);
+        add_or_set_member(spmv_case[format_name], "repetitions", repetitions,
+                          allocator);
 
         // compute and write benchmark data
         add_or_set_member(spmv_case[format_name], "completed", true, allocator);

From 8c018971504358b00655baec87a996c763a9dd88 Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Fri, 11 Jun 2021 14:13:44 +0200
Subject: [PATCH 0776/1008] adds include guard in benchmark/utils/timer.hpp

---
 benchmark/utils/timer.hpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/benchmark/utils/timer.hpp b/benchmark/utils/timer.hpp
index 60e45367d5a..9dbba1f17bb 100644
--- a/benchmark/utils/timer.hpp
+++ b/benchmark/utils/timer.hpp
@@ -30,6 +30,10 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#ifndef GKO_BENCHMARK_UTILS_TIMER_HPP_
+#define GKO_BENCHMARK_UTILS_TIMER_HPP_
+
+
 #include <ginkgo/ginkgo.hpp>
 
 
@@ -385,3 +389,5 @@ std::shared_ptr<Timer> get_timer(std::shared_ptr<const gko::Executor> exec,
     // No cuda/hip executor available or no gpu_timer used
     return std::make_shared<CpuTimer>(exec);
 }
+
+#endif  // GKO_BENCHMARK_UTILS_TIMER_HPP_

From 232529c81f6a6ced7fff7c7ebc298e2674572069 Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Fri, 18 Jun 2021 16:12:50 +0200
Subject: [PATCH 0777/1008] checks benchmark completion in each iteration

This reworks the previous adaptiv benchmarking.
Now, the number of iteration is determined on-the-fly instead of before-hand.
Also new cmd-line flags have been added to allow for greater control over the adaptive benchmarks.
The usage is similar to google benchmark.

Currently, it is not possible to use this approach for the preconditioner benchmark, as these do not update the runtime in each iteration.
---
 benchmark/blas/blas.cpp                     |  18 ++-
 benchmark/conversions/conversions.cpp       |  12 +-
 benchmark/preconditioner/preconditioner.cpp |   9 +-
 benchmark/solver/solver.cpp                 |  17 +-
 benchmark/spmv/spmv.cpp                     |  22 +--
 benchmark/utils/general.hpp                 | 165 ++++++++++++++++++--
 6 files changed, 203 insertions(+), 40 deletions(-)

diff --git a/benchmark/blas/blas.cpp b/benchmark/blas/blas.cpp
index 839add915e6..ee0bc8a5879 100644
--- a/benchmark/blas/blas.cpp
+++ b/benchmark/blas/blas.cpp
@@ -437,8 +437,11 @@ void apply_blas(const char *operation_name, std::shared_ptr<gko::Executor> exec,
 
         auto op = operation_map[operation_name](exec, parse_dims(test_case));
 
+        auto timer = get_timer(exec, FLAGS_gpu_timer);
+        IterationControl ic(timer);
+
         // warm run
-        for (unsigned int i = 0; i < FLAGS_warmup; i++) {
+        for (auto _ : ic.warmup_run()) {
             op->prepare();
             exec->synchronize();
             op->run();
@@ -446,18 +449,17 @@ void apply_blas(const char *operation_name, std::shared_ptr<gko::Executor> exec,
         }
 
         // timed run
-        auto timer = get_timer(exec, FLAGS_gpu_timer);
-        const auto repetitions = get_repetitions();
-        for (unsigned int i = 0; i < repetitions; i++) {
-            op->prepare();
+        op->prepare();
+        for (auto _ : ic.run()) {
             exec->synchronize();
             timer->tic();
             op->run();
             timer->toc();
         }
-        auto runtime = timer->compute_average_time();
-        auto flops = static_cast<double>(op->get_flops());
-        auto mem = static_cast<double>(op->get_memory());
+        const auto runtime = timer->compute_average_time();
+        const auto flops = static_cast<double>(op->get_flops());
+        const auto mem = static_cast<double>(op->get_memory());
+        const auto repetitions = timer->get_num_repetitions();
         add_or_set_member(blas_case[operation_name], "time", runtime,
                           allocator);
         add_or_set_member(blas_case[operation_name], "flops", flops / runtime,
diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index 46eeb6e3d4c..46870c3981d 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -72,17 +72,19 @@ void convert_matrix(const gko::LinOp *matrix_from, const char *format_to,
         gko::matrix_data<etype> data{gko::dim<2>{1, 1}, 1};
         auto matrix_to =
             share(formats::matrix_factory.at(format_to)(exec, data));
+
+        auto timer = get_timer(exec, FLAGS_gpu_timer);
+        IterationControl ic{timer};
+
         // warm run
-        for (unsigned int i = 0; i < FLAGS_warmup; i++) {
+        for (auto _ : ic.warmup_run()) {
             exec->synchronize();
             matrix_to->copy_from(matrix_from);
             exec->synchronize();
             matrix_to->clear();
         }
-        auto timer = get_timer(exec, FLAGS_gpu_timer);
-        const auto repetitions = get_repetitions();
         // timed run
-        for (unsigned int i = 0; i < repetitions; i++) {
+        for (auto _ : ic.run()) {
             exec->synchronize();
             timer->tic();
             matrix_to->copy_from(matrix_from);
@@ -92,7 +94,7 @@ void convert_matrix(const gko::LinOp *matrix_from, const char *format_to,
         add_or_set_member(conversion_case[conversion_name], "time",
                           timer->compute_average_time(), allocator);
         add_or_set_member(conversion_case[conversion_name], "repetitions",
-                          repetitions, allocator);
+                          timer->get_num_repetitions(), allocator);
 
         // compute and write benchmark data
         add_or_set_member(conversion_case[conversion_name], "completed", true,
diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index 685edebdca8..9298d3c457f 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -159,7 +159,8 @@ void run_preconditioner(const char *precond_name,
                               allocator);
         }
 
-        const auto repetitions = get_repetitions();
+        const auto repetitions =
+            static_cast<unsigned int>(std::stoi(FLAGS_repetitions));
         {
             // fast run, gets total time
             auto x_clone = clone(x);
@@ -255,6 +256,12 @@ int main(int argc, char *argv[])
 
     std::string extra_information =
         "Running with preconditioners: " + FLAGS_preconditioners + "\n";
+    if (FLAGS_repetitions == "auto") {
+        FLAGS_repetitions = "10";
+        extra_information +=
+            "Warning: 'repetitions = auto' is not supported.\n"
+            "          Using the fallback value of 10 repetitions.\n";
+    }
     print_general_information(extra_information);
 
     auto exec = get_executor();
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 1000685c860..7b338230c6a 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -399,9 +399,12 @@ void solve_system(const std::string &solver_name,
                               allocator);
         }
 
+        auto apply_timer = get_timer(exec, FLAGS_gpu_timer);
+        IterationControl ic{apply_timer};
+
         // warm run
         auto it_logger = std::make_shared<IterationLogger>(exec);
-        for (unsigned int i = 0; i < FLAGS_warmup; i++) {
+        for (auto _ : ic.warmup_run()) {
             auto x_clone = clone(x);
             auto precond = precond_factory.at(precond_name)(exec);
             auto solver = generate_solver(exec, give(precond), solver_name)
@@ -472,9 +475,7 @@ void solve_system(const std::string &solver_name,
 
         // timed run
         auto generate_timer = get_timer(exec, FLAGS_gpu_timer);
-        auto apply_timer = get_timer(exec, FLAGS_gpu_timer);
-        const auto repetitions = get_repetitions();
-        for (unsigned int i = 0; i < repetitions; i++) {
+        for (auto status : ic.run()) {
             auto x_clone = clone(x);
 
             exec->synchronize();
@@ -489,7 +490,7 @@ void solve_system(const std::string &solver_name,
             solver->apply(lend(b), lend(x_clone));
             apply_timer->toc();
 
-            if (b->get_size()[1] == 1 && i == repetitions - 1 &&
+            if (b->get_size()[1] == 1 && status.is_last_iteration() &&
                 !FLAGS_overhead) {
                 auto residual = compute_residual_norm(lend(system_matrix),
                                                       lend(b), lend(x_clone));
@@ -501,7 +502,8 @@ void solve_system(const std::string &solver_name,
                           generate_timer->compute_average_time(), allocator);
         add_or_set_member(solver_json["apply"], "time",
                           apply_timer->compute_average_time(), allocator);
-        add_or_set_member(solver_json, "repetitions", repetitions, allocator);
+        add_or_set_member(solver_json, "repetitions",
+                          apply_timer->get_num_repetitions(), allocator);
 
         // compute and write benchmark data
         add_or_set_member(solver_json, "completed", true, allocator);
@@ -517,7 +519,8 @@ void solve_system(const std::string &solver_name,
 int main(int argc, char *argv[])
 {
     // Set the default repetitions = 1.
-    FLAGS_repetitions = 1;
+    FLAGS_repetitions = "1";
+    FLAGS_min_repetitions = 1;
     std::string header =
         "A benchmark for measuring performance of Ginkgo's solvers.\n";
     std::string format =
diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp
index 30650e6ca73..49b95e1d1fc 100644
--- a/benchmark/spmv/spmv.cpp
+++ b/benchmark/spmv/spmv.cpp
@@ -91,16 +91,20 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
             add_or_set_member(spmv_case[format_name], "max_relative_norm2",
                               max_relative_norm2, allocator);
         }
+
+        auto timer = get_timer(exec, FLAGS_gpu_timer);
+        IterationControl ic{timer};
         // warm run
-        for (unsigned int i = 0; i < FLAGS_warmup; i++) {
+        for (auto _ : ic.warmup_run()) {
             auto x_clone = clone(x);
             exec->synchronize();
             system_matrix->apply(lend(b), lend(x_clone));
             exec->synchronize();
         }
 
-        const auto repetitions = get_repetitions();
-        // tuning run
+        auto x_clone = clone(x);  // TODO: assure that cloning once is enough
+                                  // for guessing repetitions
+                                  // tuning run
 #ifdef GINKGO_BENCHMARK_ENABLE_TUNING
         auto &format_case = spmv_case[format_name];
         if (!format_case.HasMember("tuning")) {
@@ -124,7 +128,8 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
             // variable is used.
             gko::_tuned_value = val;
             auto tuning_timer = get_timer(exec, FLAGS_gpu_timer);
-            for (unsigned int i = 0; i < repetitions; i++) {
+            IterationControl ic_tuning{tuning_timer};
+            for (auto _ : ic_tuning.run()) {
                 auto x_clone = clone(x);
                 exec->synchronize();
                 tuning_timer->tic();
@@ -141,9 +146,8 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
 #endif  // GINKGO_BENCHMARK_ENABLE_TUNING
 
         // timed run
-        auto timer = get_timer(exec, FLAGS_gpu_timer);
-        for (unsigned int i = 0; i < repetitions; i++) {
-            auto x_clone = clone(x);
+        for (auto _ : ic.run()) {
+            x_clone = clone(x);
             exec->synchronize();
             timer->tic();
             system_matrix->apply(lend(b), lend(x_clone));
@@ -151,8 +155,8 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
         }
         add_or_set_member(spmv_case[format_name], "time",
                           timer->compute_average_time(), allocator);
-        add_or_set_member(spmv_case[format_name], "repetitions", repetitions,
-                          allocator);
+        add_or_set_member(spmv_case[format_name], "repetitions",
+                          timer->get_num_repetitions(), allocator);
 
         // compute and write benchmark data
         add_or_set_member(spmv_case[format_name], "completed", true, allocator);
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index 0e38fbb5723..621e4cec3f3 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -57,6 +57,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <rapidjson/ostreamwrapper.h>
 #include <rapidjson/prettywriter.h>
 
+#include "benchmark/utils/timer.hpp"  // perhaps fwd decl + new cpp file?
+
 
 // Global command-line arguments
 DEFINE_string(executor, "reference",
@@ -88,7 +90,23 @@ DEFINE_uint32(seed, 42, "Seed used for the random number generator");
 DEFINE_uint32(warmup, 2, "Warm-up repetitions");
 
 DEFINE_string(repetitions, "10",
-              "Number of runs used to obtain an averaged result.");
+              "The number of runs used to obtain an averaged result, if 'auto' "
+              "is used the number is adaptively chosen."
+              " In that case, the benchmark runs at least 'min_repetitions'"
+              " times until either 'max_repetitions' is reached or the total "
+              "runtime is larger than 'min_runtime'");
+
+DEFINE_double(min_runtime, 0.05,
+              "If 'repetitions = auto' is used, the minimal runtime of"
+              " a single benchmark.");
+
+DEFINE_uint32(min_repetitions, 10,
+              "If 'repetitions = auto' is used, the minimal number of"
+              " repetitions for a single benchmark.");
+
+DEFINE_uint32(max_repetitions, std::numeric_limits<unsigned int>::max(),
+              "If 'repetitions = auto' is used, the maximal number of"
+              " repetitions for a single benchmark.");
 
 
 /**
@@ -134,9 +152,17 @@ void print_general_information(std::string &extra)
     std::clog << gko::version_info::get() << std::endl
               << "Running on " << FLAGS_executor << "(" << FLAGS_device_id
               << ")" << std::endl
-              << "Running with " << FLAGS_warmup << " warm iterations and "
-              << FLAGS_repetitions << " running iterations" << std::endl
-              << "The random seed for right hand sides is " << FLAGS_seed
+              << "Running with " << FLAGS_warmup << " warm iterations and ";
+    if (FLAGS_repetitions == "auto") {
+        std::clog << "adaptively determined repetititions with "
+                  << FLAGS_min_repetitions
+                  << " <= rep <= " << FLAGS_max_repetitions
+                  << " and a minimal runtime of " << FLAGS_min_runtime << "s"
+                  << std::endl;
+    } else {
+        std::clog << FLAGS_repetitions << " running iterations" << std::endl;
+    }
+    std::clog << "The random seed for right hand sides is " << FLAGS_seed
               << std::endl
               << extra;
 }
@@ -434,11 +460,130 @@ gko::remove_complex<ValueType> compute_max_relative_norm2(
 }
 
 
-// parses the repetitions flag
-unsigned int get_repetitions()
-{
-    const std::string reps_str = FLAGS_repetitions;
-    return static_cast<unsigned int>(std::stoi(reps_str));
-}
+/**
+ * A class for controlling the number warmup and timed iterations.
+ *
+ * The behavior is determined by the following flags
+ * - 'repetitions' switch between fixed and adaptive number of iterations
+ * - 'warmup' warmup iterations, applies in fixed and adaptive case
+ * - 'min_repetitions' minimal number of repetitions (adaptive case)
+ * - 'max_repetitions' maximal number of repetitions (adaptive case)
+ * - 'min_runtime' minimal total runtime (adaptive case)
+ *
+ * Usage:
+ * `IterationControl` exposes two member functions:
+ * - `warmup_run()`: uses `warmup` flag
+ * - `run()`: uses all other flags
+ * Both methods return an object that is to be used in a range-based for loop:
+ * ```
+ * auto timer = get_timer(...);
+ * IterationControl ic(timer);
+ * for(auto stat: ic.[warmup_run|run]()){
+ *   timer->start();  // has to be the same timer as passed to ic
+ *   // execute benchmark
+ *   timer->stop();
+ * }
+ * ```
+ * At the beginning of both methods, the timer is reset.
+ *
+ */
+class IterationControl {
+    using IndexType = unsigned int;  //!< to be compatible with GFLAGS type
+
+    class run_control;
+
+public:
+    // criteria to stop adaptive benchmark run
+    struct status {
+        std::shared_ptr<Timer> timer = nullptr;
+
+        IndexType min_it = 0;
+        IndexType max_it = 0;
+        double max_runtime = 0.;
+
+        IndexType cur_it = 0;
+
+        // returns true (i.e. run is complete) if:
+        // - the minimum number of iteration is reached
+        // - and either:
+        //   - the maximum number of repetitions is reached
+        //   - the total runtime is above the threshold
+        bool is_finished()
+        {
+            return cur_it >= min_it &&
+                   (cur_it >= max_it || timer->get_total_time() >= max_runtime);
+        }
+        bool is_last_iteration()
+        {
+            return status{timer, min_it, max_it, max_runtime, cur_it + 1}
+                .is_finished();
+        }
+    };
+
+    explicit IterationControl(std::shared_ptr<Timer> timer) : timer_(timer)
+    {
+        status_warmup_ = {timer, FLAGS_warmup, FLAGS_warmup, 0., 0};
+        if (FLAGS_repetitions == "auto") {
+            status_run_ = {timer, FLAGS_min_repetitions, FLAGS_max_repetitions,
+                           FLAGS_min_runtime};
+        } else {
+            const auto reps =
+                static_cast<unsigned int>(std::stoi(FLAGS_repetitions));
+            status_run_ = {timer, reps, reps, 0., 0};
+        }
+    }
+
+    IterationControl() = default;
+    IterationControl(const IterationControl &) = default;
+    IterationControl(IterationControl &&) = default;
+
+    run_control warmup_run()
+    {
+        timer_->clear();
+        status_warmup_.cur_it = 0;
+        return run_control{status_warmup_};
+    }
+
+    run_control run()
+    {
+        timer_->clear();
+        status_run_.cur_it = 0;
+        return run_control{status_run_};
+    }
+
+private:
+    // class to be used in range-based for
+    struct run_control {
+        struct iterator {
+            iterator operator++()
+            {
+                cur_info.cur_it++;
+                return *this;
+            }
+
+            status operator*() const { return cur_info; }
+
+            bool operator!=(const iterator &)
+            {
+                return !cur_info.is_finished();
+            }
+
+            status cur_info;
+        };
+
+        iterator begin() const { return iterator{info}; }
+
+        // not used, could potentially used in c++17 (with different type)
+        iterator end() const { return iterator{}; }
+
+        status info;
+    };
+
+    std::shared_ptr<Timer> timer_;
+
+    status status_warmup_;
+    status status_run_;
+};
+
 
 #endif  // GKO_BENCHMARK_UTILS_GENERAL_HPP_

From d78c174a3027735083e6e9b1bed07643ab60778c Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Mon, 21 Jun 2021 11:26:16 +0200
Subject: [PATCH 0778/1008] enables adaptive benchmarking for preconditioner

this changes the preconditioner benchmark to time each preconditioner apply/generate individually, unifying the timing approach across all benchmarks.
---
 benchmark/preconditioner/preconditioner.cpp | 61 ++++++++++-----------
 1 file changed, 29 insertions(+), 32 deletions(-)

diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index 9298d3c457f..dfafc086f28 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -159,46 +159,47 @@ void run_preconditioner(const char *precond_name,
                               allocator);
         }
 
-        const auto repetitions =
-            static_cast<unsigned int>(std::stoi(FLAGS_repetitions));
+        auto generate_timer = get_timer(exec, FLAGS_gpu_timer);
+        auto apply_timer = get_timer(exec, FLAGS_gpu_timer);
+
         {
             // fast run, gets total time
             auto x_clone = clone(x);
 
             auto precond = precond_factory.at(precond_name)(exec);
 
-            for (auto i = 0u; i < FLAGS_warmup; ++i) {
+            IterationControl ic_gen{generate_timer};
+            IterationControl ic_apply{apply_timer};
+
+            for (auto _ : ic_apply.warmup_run()) {
                 precond->generate(system_matrix)->apply(lend(b), lend(x_clone));
             }
-            auto generate_timer = get_timer(exec, FLAGS_gpu_timer);
-            auto apply_timer = get_timer(exec, FLAGS_gpu_timer);
 
-            exec->synchronize();
-            generate_timer->tic();
             std::unique_ptr<gko::LinOp> precond_op;
-            for (auto i = 0u; i < repetitions; ++i) {
+            for (auto _ : ic_gen.run()) {
+                exec->synchronize();
+                generate_timer->tic();
                 precond_op = precond->generate(system_matrix);
+                generate_timer->toc();
             }
-            generate_timer->toc();
 
-            // the timer is out of the loops to reduce calling synchronize
-            // overhead, so the timer does not know the number of repetitions.
-            auto generate_time = generate_timer->get_total_time() / repetitions;
             add_or_set_member(this_precond_data["generate"], "time",
-                              generate_time, allocator);
+                              generate_timer->compute_average_time(),
+                              allocator);
+            add_or_set_member(this_precond_data["generate"], "repetitions",
+                              generate_timer->get_num_repetitions(), allocator);
 
-            exec->synchronize();
-            apply_timer->tic();
-            for (auto i = 0u; i < repetitions; ++i) {
+            for (auto _ : ic_apply.run()) {
+                exec->synchronize();
+                apply_timer->tic();
                 precond_op->apply(lend(b), lend(x_clone));
+                apply_timer->toc();
             }
-            apply_timer->toc();
 
-            // the timer is out of the loops to reduce calling synchronize
-            // overhead, so the timer does not know the number of repetitions.
-            auto apply_time = apply_timer->get_total_time() / repetitions;
-            add_or_set_member(this_precond_data["apply"], "time", apply_time,
-                              allocator);
+            add_or_set_member(this_precond_data["apply"], "time",
+                              apply_timer->compute_average_time(), allocator);
+            add_or_set_member(this_precond_data["apply"], "repetitions",
+                              apply_timer->get_num_repetitions(), allocator);
         }
 
         if (FLAGS_detailed) {
@@ -210,24 +211,26 @@ void run_preconditioner(const char *precond_name,
                 std::make_shared<OperationLogger>(exec, FLAGS_nested_names);
             exec->add_logger(gen_logger);
             std::unique_ptr<gko::LinOp> precond_op;
-            for (auto i = 0u; i < repetitions; ++i) {
+            for (auto i = 0u; i < generate_timer->get_num_repetitions(); ++i) {
                 precond_op = precond->generate(system_matrix);
             }
             exec->remove_logger(gko::lend(gen_logger));
 
             gen_logger->write_data(this_precond_data["generate"]["components"],
-                                   allocator, repetitions);
+                                   allocator,
+                                   generate_timer->get_num_repetitions());
 
             auto apply_logger =
                 std::make_shared<OperationLogger>(exec, FLAGS_nested_names);
             exec->add_logger(apply_logger);
-            for (auto i = 0u; i < repetitions; ++i) {
+            for (auto i = 0u; i < apply_timer->get_num_repetitions(); ++i) {
                 precond_op->apply(lend(b), lend(x_clone));
             }
             exec->remove_logger(gko::lend(apply_logger));
 
             apply_logger->write_data(this_precond_data["apply"]["components"],
-                                     allocator, repetitions);
+                                     allocator,
+                                     apply_timer->get_num_repetitions());
         }
 
         add_or_set_member(this_precond_data, "completed", true, allocator);
@@ -256,12 +259,6 @@ int main(int argc, char *argv[])
 
     std::string extra_information =
         "Running with preconditioners: " + FLAGS_preconditioners + "\n";
-    if (FLAGS_repetitions == "auto") {
-        FLAGS_repetitions = "10";
-        extra_information +=
-            "Warning: 'repetitions = auto' is not supported.\n"
-            "          Using the fallback value of 10 repetitions.\n";
-    }
     print_general_information(extra_information);
 
     auto exec = get_executor();

From 70207e8e10c059bf80c74da033bd1f58e4863fe7 Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Tue, 22 Jun 2021 12:01:36 +0200
Subject: [PATCH 0779/1008] Review update and `status` visability change - adds
 more documentation - minor formatting - made `status` private s.t. it is not
 part of the   public interface of `IterationControl`

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 benchmark/utils/general.hpp | 116 ++++++++++++++++++++++++++----------
 1 file changed, 84 insertions(+), 32 deletions(-)

diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index 621e4cec3f3..b5ecaeaedad 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -57,7 +57,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <rapidjson/ostreamwrapper.h>
 #include <rapidjson/prettywriter.h>
 
-#include "benchmark/utils/timer.hpp"  // perhaps fwd decl + new cpp file?
+
+
+#include "benchmark/utils/timer.hpp"
 
 
 // Global command-line arguments
@@ -478,14 +480,18 @@ gko::remove_complex<ValueType> compute_max_relative_norm2(
  * ```
  * auto timer = get_timer(...);
  * IterationControl ic(timer);
- * for(auto stat: ic.[warmup_run|run]()){
+ * for(auto status: ic.[warmup_run|run]()){
  *   timer->start();  // has to be the same timer as passed to ic
  *   // execute benchmark
  *   timer->stop();
  * }
  * ```
  * At the beginning of both methods, the timer is reset.
- *
+ * The `status` object exposes the member
+ * - `cur_it`, containing the current iteration number,
+ * and the methods
+ * - `is_finished`, checks if the benchmark is finished,
+ * - `is_last_iteration`, checks if the current iteration is the last one.
  */
 class IterationControl {
     using IndexType = unsigned int;  //!< to be compatible with GFLAGS type
@@ -493,33 +499,14 @@ class IterationControl {
     class run_control;
 
 public:
-    // criteria to stop adaptive benchmark run
-    struct status {
-        std::shared_ptr<Timer> timer = nullptr;
-
-        IndexType min_it = 0;
-        IndexType max_it = 0;
-        double max_runtime = 0.;
-
-        IndexType cur_it = 0;
-
-        // returns true (i.e. run is complete) if:
-        // - the minimum number of iteration is reached
-        // - and either:
-        //   - the maximum number of repetitions is reached
-        //   - the total runtime is above the threshold
-        bool is_finished()
-        {
-            return cur_it >= min_it &&
-                   (cur_it >= max_it || timer->get_total_time() >= max_runtime);
-        }
-        bool is_last_iteration()
-        {
-            return status{timer, min_it, max_it, max_runtime, cur_it + 1}
-                .is_finished();
-        }
-    };
-
+    /**
+     * Creates an `IterationControl` object.
+     *
+     * Uses the commandline flags to setup the stopping criteria for the
+     * warmup and timed run.
+     *
+     * @param timer the same timer that is to be used for the timings
+     */
     explicit IterationControl(std::shared_ptr<Timer> timer) : timer_(timer)
     {
         status_warmup_ = {timer, FLAGS_warmup, FLAGS_warmup, 0., 0};
@@ -537,6 +524,11 @@ class IterationControl {
     IterationControl(const IterationControl &) = default;
     IterationControl(IterationControl &&) = default;
 
+    /**
+     * Creates iterable `run_control` object for the warmup run.
+     *
+     * This run uses always a fixed number of iterations.
+     */
     run_control warmup_run()
     {
         timer_->clear();
@@ -544,6 +536,11 @@ class IterationControl {
         return run_control{status_warmup_};
     }
 
+    /**
+     * Creates iterable `run_control` object for the timed run.
+     *
+     * This run may be adaptive, depending on the commandline flags.
+     */
     run_control run()
     {
         timer_->clear();
@@ -552,7 +549,54 @@ class IterationControl {
     }
 
 private:
-    // class to be used in range-based for
+    /**
+     * Stores stopping criteria of the adaptive benchmark run as well as the
+     * current iteration number.
+     */
+    struct status {
+        std::shared_ptr<Timer> timer = nullptr;
+
+        IndexType min_it = 0;
+        IndexType max_it = 0;
+        double max_runtime = 0.;
+
+        IndexType cur_it = 0;
+
+        /**
+         * checks if the adaptive run is complete
+         *
+         * the adaptive run is complete if:
+         * - the minimum number of iteration is reached
+         * - and either:
+         *   - the maximum number of repetitions is reached
+         *   - the total runtime is above the threshold
+         *
+         * @return completeness state of the adaptive run
+         */
+        bool is_finished()
+        {
+            return cur_it >= min_it &&
+                   (cur_it >= max_it || timer->get_total_time() >= max_runtime);
+        }
+
+        /**
+         * checks if the benchmark will stop after the current iteration
+         *
+         * @note Should only be called after the timing of the current iteration
+         * is complete.
+         */
+        bool is_last_iteration()
+        {
+            return status{timer, min_it, max_it, max_runtime, cur_it + 1}
+                .is_finished();
+        }
+    };
+
+    /**
+     * Iterable class managing the benchmark iteration.
+     *
+     * Has to be used in a range-based for loop.
+     */
     struct run_control {
         struct iterator {
             iterator operator++()
@@ -563,6 +607,14 @@ class IterationControl {
 
             status operator*() const { return cur_info; }
 
+            /**
+             * checks if the benchmar is finished
+             *
+             * uses only the information from the `status` object, i.e.
+             * the right hand side is ignored.
+             *
+             * @return true if benchmark is not finished, else false
+             */
             bool operator!=(const iterator &)
             {
                 return !cur_info.is_finished();
@@ -573,7 +625,7 @@ class IterationControl {
 
         iterator begin() const { return iterator{info}; }
 
-        // not used, could potentially used in c++17 (with different type)
+        // not used, could potentially used in c++17 as a sentinel
         iterator end() const { return iterator{}; }
 
         status info;

From 3639dc93f95172dc577ab55c09b6850c0c9d4429 Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Wed, 23 Jun 2021 15:07:23 +0200
Subject: [PATCH 0780/1008] increases the interval between two timings

Now the `run_control` object also controls taking the timings. The user does not need to issue the timings by hand anymore. This allows to use increasingly larger intervals between two timings, unitl the benchmark run is finished.

Drawback: everything within the `ic.run()` loop gets timed, parts that should be exempt need to be moved outside of the loop.
---
 benchmark/blas/blas.cpp                     |  7 +-
 benchmark/conversions/conversions.cpp       |  4 --
 benchmark/preconditioner/preconditioner.cpp | 31 +++------
 benchmark/solver/solver.cpp                 | 51 +++++++-------
 benchmark/spmv/spmv.cpp                     | 21 ++----
 benchmark/utils/general.hpp                 | 74 ++++++++++++++-------
 6 files changed, 92 insertions(+), 96 deletions(-)

diff --git a/benchmark/blas/blas.cpp b/benchmark/blas/blas.cpp
index ee0bc8a5879..66c71c21ae8 100644
--- a/benchmark/blas/blas.cpp
+++ b/benchmark/blas/blas.cpp
@@ -451,15 +451,12 @@ void apply_blas(const char *operation_name, std::shared_ptr<gko::Executor> exec,
         // timed run
         op->prepare();
         for (auto _ : ic.run()) {
-            exec->synchronize();
-            timer->tic();
             op->run();
-            timer->toc();
         }
-        const auto runtime = timer->compute_average_time();
+        const auto runtime = ic.compute_average_time();
         const auto flops = static_cast<double>(op->get_flops());
         const auto mem = static_cast<double>(op->get_memory());
-        const auto repetitions = timer->get_num_repetitions();
+        const auto repetitions = ic.get_num_repetitions();
         add_or_set_member(blas_case[operation_name], "time", runtime,
                           allocator);
         add_or_set_member(blas_case[operation_name], "flops", flops / runtime,
diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index 46870c3981d..fd9481189b0 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -85,11 +85,7 @@ void convert_matrix(const gko::LinOp *matrix_from, const char *format_to,
         }
         // timed run
         for (auto _ : ic.run()) {
-            exec->synchronize();
-            timer->tic();
             matrix_to->copy_from(matrix_from);
-            timer->toc();
-            matrix_to->clear();
         }
         add_or_set_member(conversion_case[conversion_name], "time",
                           timer->compute_average_time(), allocator);
diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index dfafc086f28..695c1e78314 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -159,8 +159,8 @@ void run_preconditioner(const char *precond_name,
                               allocator);
         }
 
-        auto generate_timer = get_timer(exec, FLAGS_gpu_timer);
-        auto apply_timer = get_timer(exec, FLAGS_gpu_timer);
+        IterationControl ic_gen{get_timer(exec, FLAGS_gpu_timer)};
+        IterationControl ic_apply{get_timer(exec, FLAGS_gpu_timer)};
 
         {
             // fast run, gets total time
@@ -168,8 +168,6 @@ void run_preconditioner(const char *precond_name,
 
             auto precond = precond_factory.at(precond_name)(exec);
 
-            IterationControl ic_gen{generate_timer};
-            IterationControl ic_apply{apply_timer};
 
             for (auto _ : ic_apply.warmup_run()) {
                 precond->generate(system_matrix)->apply(lend(b), lend(x_clone));
@@ -177,29 +175,22 @@ void run_preconditioner(const char *precond_name,
 
             std::unique_ptr<gko::LinOp> precond_op;
             for (auto _ : ic_gen.run()) {
-                exec->synchronize();
-                generate_timer->tic();
                 precond_op = precond->generate(system_matrix);
-                generate_timer->toc();
             }
 
             add_or_set_member(this_precond_data["generate"], "time",
-                              generate_timer->compute_average_time(),
-                              allocator);
+                              ic_gen.compute_average_time(), allocator);
             add_or_set_member(this_precond_data["generate"], "repetitions",
-                              generate_timer->get_num_repetitions(), allocator);
+                              ic_gen.get_num_repetitions(), allocator);
 
             for (auto _ : ic_apply.run()) {
-                exec->synchronize();
-                apply_timer->tic();
                 precond_op->apply(lend(b), lend(x_clone));
-                apply_timer->toc();
             }
 
             add_or_set_member(this_precond_data["apply"], "time",
-                              apply_timer->compute_average_time(), allocator);
+                              ic_apply.compute_average_time(), allocator);
             add_or_set_member(this_precond_data["apply"], "repetitions",
-                              apply_timer->get_num_repetitions(), allocator);
+                              ic_apply.get_num_repetitions(), allocator);
         }
 
         if (FLAGS_detailed) {
@@ -211,26 +202,24 @@ void run_preconditioner(const char *precond_name,
                 std::make_shared<OperationLogger>(exec, FLAGS_nested_names);
             exec->add_logger(gen_logger);
             std::unique_ptr<gko::LinOp> precond_op;
-            for (auto i = 0u; i < generate_timer->get_num_repetitions(); ++i) {
+            for (auto i = 0u; i < ic_gen.get_num_repetitions(); ++i) {
                 precond_op = precond->generate(system_matrix);
             }
             exec->remove_logger(gko::lend(gen_logger));
 
             gen_logger->write_data(this_precond_data["generate"]["components"],
-                                   allocator,
-                                   generate_timer->get_num_repetitions());
+                                   allocator, ic_gen.get_num_repetitions());
 
             auto apply_logger =
                 std::make_shared<OperationLogger>(exec, FLAGS_nested_names);
             exec->add_logger(apply_logger);
-            for (auto i = 0u; i < apply_timer->get_num_repetitions(); ++i) {
+            for (auto i = 0u; i < ic_apply.get_num_repetitions(); ++i) {
                 precond_op->apply(lend(b), lend(x_clone));
             }
             exec->remove_logger(gko::lend(apply_logger));
 
             apply_logger->write_data(this_precond_data["apply"]["components"],
-                                     allocator,
-                                     apply_timer->get_num_repetitions());
+                                     allocator, ic_apply.get_num_repetitions());
         }
 
         add_or_set_member(this_precond_data, "completed", true, allocator);
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 7b338230c6a..6ea5c996fcf 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -400,11 +400,12 @@ void solve_system(const std::string &solver_name,
         }
 
         auto apply_timer = get_timer(exec, FLAGS_gpu_timer);
-        IterationControl ic{apply_timer};
+        IterationControl ic_gen{get_timer(exec, FLAGS_gpu_timer)};
+        IterationControl ic_apply{apply_timer};
 
         // warm run
         auto it_logger = std::make_shared<IterationLogger>(exec);
-        for (auto _ : ic.warmup_run()) {
+        for (auto _ : ic_apply.warmup_run()) {
             auto x_clone = clone(x);
             auto precond = precond_factory.at(precond_name)(exec);
             auto solver = generate_solver(exec, give(precond), solver_name)
@@ -474,36 +475,34 @@ void solve_system(const std::string &solver_name,
         }
 
         // timed run
-        auto generate_timer = get_timer(exec, FLAGS_gpu_timer);
-        for (auto status : ic.run()) {
-            auto x_clone = clone(x);
-
-            exec->synchronize();
-            generate_timer->tic();
+        for (auto _ : ic_gen.run()) {
             auto precond = precond_factory.at(precond_name)(exec);
             auto solver = generate_solver(exec, give(precond), solver_name)
                               ->generate(system_matrix);
-            generate_timer->toc();
-
-            exec->synchronize();
-            apply_timer->tic();
-            solver->apply(lend(b), lend(x_clone));
-            apply_timer->toc();
-
-            if (b->get_size()[1] == 1 && status.is_last_iteration() &&
-                !FLAGS_overhead) {
-                auto residual = compute_residual_norm(lend(system_matrix),
-                                                      lend(b), lend(x_clone));
-                add_or_set_member(solver_json, "residual_norm", residual,
-                                  allocator);
-            }
         }
         add_or_set_member(solver_json["generate"], "time",
-                          generate_timer->compute_average_time(), allocator);
+                          ic_gen.compute_average_time(), allocator);
+        add_or_set_member(solver_json["generate"], "repetitions",
+                          ic_gen.get_num_repetitions(), allocator);
+
+        auto precond = precond_factory.at(precond_name)(exec);
+        auto solver = generate_solver(exec, give(precond), solver_name)
+                          ->generate(system_matrix);
+        auto x_clone = clone(x);
+        for (auto _ : ic_apply.run()) {
+            x_clone = clone(x);
+            solver->apply(lend(b), lend(x_clone));
+        }
+        if (b->get_size()[1] == 1 && !FLAGS_overhead) {
+            auto residual = compute_residual_norm(lend(system_matrix), lend(b),
+                                                  lend(x_clone));
+            add_or_set_member(solver_json, "residual_norm", residual,
+                              allocator);
+        }
         add_or_set_member(solver_json["apply"], "time",
-                          apply_timer->compute_average_time(), allocator);
-        add_or_set_member(solver_json, "repetitions",
-                          apply_timer->get_num_repetitions(), allocator);
+                          ic_apply.compute_average_time(), allocator);
+        add_or_set_member(solver_json["apply"], "repetitions",
+                          ic_apply.get_num_repetitions(), allocator);
 
         // compute and write benchmark data
         add_or_set_member(solver_json, "completed", true, allocator);
diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp
index 49b95e1d1fc..2bf07e8ad45 100644
--- a/benchmark/spmv/spmv.cpp
+++ b/benchmark/spmv/spmv.cpp
@@ -92,8 +92,7 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
                               max_relative_norm2, allocator);
         }
 
-        auto timer = get_timer(exec, FLAGS_gpu_timer);
-        IterationControl ic{timer};
+        IterationControl ic{get_timer(exec, FLAGS_gpu_timer)};
         // warm run
         for (auto _ : ic.warmup_run()) {
             auto x_clone = clone(x);
@@ -102,9 +101,7 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
             exec->synchronize();
         }
 
-        auto x_clone = clone(x);  // TODO: assure that cloning once is enough
-                                  // for guessing repetitions
-                                  // tuning run
+        // tuning run
 #ifdef GINKGO_BENCHMARK_ENABLE_TUNING
         auto &format_case = spmv_case[format_name];
         if (!format_case.HasMember("tuning")) {
@@ -129,12 +126,9 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
             gko::_tuned_value = val;
             auto tuning_timer = get_timer(exec, FLAGS_gpu_timer);
             IterationControl ic_tuning{tuning_timer};
+            auto x_clone = clone(x);
             for (auto _ : ic_tuning.run()) {
-                auto x_clone = clone(x);
-                exec->synchronize();
-                tuning_timer->tic();
                 system_matrix->apply(lend(b), lend(x_clone));
-                tuning_timer->toc();
             }
             tuning_case["time"].PushBack(tuning_timer->compute_average_time(),
                                          allocator);
@@ -146,17 +140,14 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
 #endif  // GINKGO_BENCHMARK_ENABLE_TUNING
 
         // timed run
+        auto x_clone = clone(x);
         for (auto _ : ic.run()) {
-            x_clone = clone(x);
-            exec->synchronize();
-            timer->tic();
             system_matrix->apply(lend(b), lend(x_clone));
-            timer->toc();
         }
         add_or_set_member(spmv_case[format_name], "time",
-                          timer->compute_average_time(), allocator);
+                          ic.compute_average_time(), allocator);
         add_or_set_member(spmv_case[format_name], "repetitions",
-                          timer->get_num_repetitions(), allocator);
+                          ic.get_num_repetitions(), allocator);
 
         // compute and write benchmark data
         add_or_set_member(spmv_case[format_name], "completed", true, allocator);
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index b5ecaeaedad..22c58f430d5 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -58,7 +58,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <rapidjson/prettywriter.h>
 
 
-
 #include "benchmark/utils/timer.hpp"
 
 
@@ -481,9 +480,7 @@ gko::remove_complex<ValueType> compute_max_relative_norm2(
  * auto timer = get_timer(...);
  * IterationControl ic(timer);
  * for(auto status: ic.[warmup_run|run]()){
- *   timer->start();  // has to be the same timer as passed to ic
  *   // execute benchmark
- *   timer->stop();
  * }
  * ```
  * At the beginning of both methods, the timer is reset.
@@ -491,7 +488,6 @@ gko::remove_complex<ValueType> compute_max_relative_norm2(
  * - `cur_it`, containing the current iteration number,
  * and the methods
  * - `is_finished`, checks if the benchmark is finished,
- * - `is_last_iteration`, checks if the current iteration is the last one.
  */
 class IterationControl {
     using IndexType = unsigned int;  //!< to be compatible with GFLAGS type
@@ -533,7 +529,7 @@ class IterationControl {
     {
         timer_->clear();
         status_warmup_.cur_it = 0;
-        return run_control{status_warmup_};
+        return run_control{&status_warmup_};
     }
 
     /**
@@ -545,9 +541,17 @@ class IterationControl {
     {
         timer_->clear();
         status_run_.cur_it = 0;
-        return run_control{status_run_};
+        return run_control{&status_run_};
+    }
+
+
+    double compute_average_time() const
+    {
+        return status_run_.timer->get_total_time() / get_num_repetitions();
     }
 
+    IndexType get_num_repetitions() const { return status_run_.cur_it; }
+
 private:
     /**
      * Stores stopping criteria of the adaptive benchmark run as well as the
@@ -578,18 +582,6 @@ class IterationControl {
             return cur_it >= min_it &&
                    (cur_it >= max_it || timer->get_total_time() >= max_runtime);
         }
-
-        /**
-         * checks if the benchmark will stop after the current iteration
-         *
-         * @note Should only be called after the timing of the current iteration
-         * is complete.
-         */
-        bool is_last_iteration()
-        {
-            return status{timer, min_it, max_it, max_runtime, cur_it + 1}
-                .is_finished();
-        }
     };
 
     /**
@@ -599,28 +591,60 @@ class IterationControl {
      */
     struct run_control {
         struct iterator {
+            /**
+             * Increases the current iteration count and finishes timing if
+             * necessary.
+             *
+             * As `++it` is the last step of a for-loop, the timer is stopped,
+             * if enough iterations have passed since the last timing.
+             * The interval between two timings is steadily increased to
+             * reduce the timing overhead.
+             */
             iterator operator++()
             {
-                cur_info.cur_it++;
+                cur_info->cur_it++;
+                if (cur_info->cur_it >= next_timing && !stopped) {
+                    cur_info->timer->toc();
+                    stopped = true;
+                    next_timing =
+                        static_cast<IndexType>(std::ceil(next_timing * 1.5));
+                }
                 return *this;
             }
 
-            status operator*() const { return cur_info; }
+            status operator*() const { return *cur_info; }
 
             /**
-             * checks if the benchmar is finished
+             * Checks if the benchmark is finished and handles timing, if
+             * necessary.
+             *
+             * As `begin != end` is the first step in a for-loop, the timer is
+             * started, if it was previously stopped.
+             * Additionally, if the benchmark is complete and the timer is still
+             * running it is stopped. (This may occur if the maximal number of
+             * repetitions is surpassed)
              *
-             * uses only the information from the `status` object, i.e.
+             * Uses only the information from the `status` object, i.e.
              * the right hand side is ignored.
              *
              * @return true if benchmark is not finished, else false
              */
             bool operator!=(const iterator &)
             {
-                return !cur_info.is_finished();
+                const bool is_finished = cur_info->is_finished();
+                if (!is_finished && stopped) {
+                    stopped = false;
+                    cur_info->timer->tic();
+                } else if (is_finished && !stopped) {
+                    cur_info->timer->toc();
+                    stopped = true;
+                }
+                return !is_finished;
             }
 
-            status cur_info;
+            status *cur_info;
+            IndexType next_timing = 1;  //!< next iteration to stop timing
+            bool stopped = true;
         };
 
         iterator begin() const { return iterator{info}; }
@@ -628,7 +652,7 @@ class IterationControl {
         // not used, could potentially used in c++17 as a sentinel
         iterator end() const { return iterator{}; }
 
-        status info;
+        status *info;
     };
 
     std::shared_ptr<Timer> timer_;

From 29be0c778fb83c4956c3a1c48a2e24a0613f4696 Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Wed, 23 Jun 2021 16:10:40 +0200
Subject: [PATCH 0781/1008] synchronizes all timers on `tic/toc`

---
 benchmark/utils/timer.hpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/benchmark/utils/timer.hpp b/benchmark/utils/timer.hpp
index 9dbba1f17bb..c61cb546a55 100644
--- a/benchmark/utils/timer.hpp
+++ b/benchmark/utils/timer.hpp
@@ -264,6 +264,7 @@ class CudaTimer : public Timer {
 protected:
     void tic_impl() override
     {
+        exec_->synchronize();
         gko::cuda::device_guard g{id_};
         // Currently, gko::CudaExecutor always use default stream.
         GKO_ASSERT_NO_CUDA_ERRORS(cudaEventRecord(start_));
@@ -271,6 +272,7 @@ class CudaTimer : public Timer {
 
     double toc_impl() override
     {
+        exec_->synchronize();
         gko::cuda::device_guard g{id_};
         // Currently, gko::CudaExecutor always use default stream.
         GKO_ASSERT_NO_CUDA_ERRORS(cudaEventRecord(stop_));
@@ -330,6 +332,7 @@ class HipTimer : public Timer {
 protected:
     void tic_impl() override
     {
+        exec_->synchronize();
         gko::hip::device_guard g{id_};
         // Currently, gko::HipExecutor always use default stream.
         GKO_ASSERT_NO_HIP_ERRORS(hipEventRecord(start_));
@@ -337,6 +340,7 @@ class HipTimer : public Timer {
 
     double toc_impl() override
     {
+        exec_->synchronize();
         gko::hip::device_guard g{id_};
         // Currently, gko::HipExecutor always use default stream.
         GKO_ASSERT_NO_HIP_ERRORS(hipEventRecord(stop_));

From d3be987f175e3748da93327a7200125c92a19524 Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Wed, 23 Jun 2021 17:14:05 +0200
Subject: [PATCH 0782/1008] adds switch deactivating timings in `run_control`

Internally this uses a thin wrapper class for the `timer` object, which just skips the `tic/toc` calls, if the `run_control` object does not manage the timings. In that case, the timings have to be issued outside as before.
---
 benchmark/solver/solver.cpp | 36 ++++++++++++-------------
 benchmark/utils/general.hpp | 53 ++++++++++++++++++++++++++++---------
 2 files changed, 58 insertions(+), 31 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 6ea5c996fcf..55fc2f78c41 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -400,12 +400,11 @@ void solve_system(const std::string &solver_name,
         }
 
         auto apply_timer = get_timer(exec, FLAGS_gpu_timer);
-        IterationControl ic_gen{get_timer(exec, FLAGS_gpu_timer)};
-        IterationControl ic_apply{apply_timer};
+        IterationControl ic{apply_timer};
 
         // warm run
         auto it_logger = std::make_shared<IterationLogger>(exec);
-        for (auto _ : ic_apply.warmup_run()) {
+        for (auto _ : ic.warmup_run()) {
             auto x_clone = clone(x);
             auto precond = precond_factory.at(precond_name)(exec);
             auto solver = generate_solver(exec, give(precond), solver_name)
@@ -475,23 +474,22 @@ void solve_system(const std::string &solver_name,
         }
 
         // timed run
-        for (auto _ : ic_gen.run()) {
+        auto generate_timer = get_timer(exec, FLAGS_gpu_timer);
+        auto x_clone = clone(x);
+        for (auto status : ic.run(false)) {
+            x_clone = clone(x);
+
+            exec->synchronize();
+            generate_timer->tic();
             auto precond = precond_factory.at(precond_name)(exec);
             auto solver = generate_solver(exec, give(precond), solver_name)
                               ->generate(system_matrix);
-        }
-        add_or_set_member(solver_json["generate"], "time",
-                          ic_gen.compute_average_time(), allocator);
-        add_or_set_member(solver_json["generate"], "repetitions",
-                          ic_gen.get_num_repetitions(), allocator);
+            generate_timer->toc();
 
-        auto precond = precond_factory.at(precond_name)(exec);
-        auto solver = generate_solver(exec, give(precond), solver_name)
-                          ->generate(system_matrix);
-        auto x_clone = clone(x);
-        for (auto _ : ic_apply.run()) {
-            x_clone = clone(x);
+            exec->synchronize();
+            apply_timer->tic();
             solver->apply(lend(b), lend(x_clone));
+            apply_timer->toc();
         }
         if (b->get_size()[1] == 1 && !FLAGS_overhead) {
             auto residual = compute_residual_norm(lend(system_matrix), lend(b),
@@ -499,10 +497,12 @@ void solve_system(const std::string &solver_name,
             add_or_set_member(solver_json, "residual_norm", residual,
                               allocator);
         }
+        add_or_set_member(solver_json["generate"], "time",
+                          generate_timer->compute_average_time(), allocator);
         add_or_set_member(solver_json["apply"], "time",
-                          ic_apply.compute_average_time(), allocator);
-        add_or_set_member(solver_json["apply"], "repetitions",
-                          ic_apply.get_num_repetitions(), allocator);
+                          apply_timer->compute_average_time(), allocator);
+        add_or_set_member(solver_json, "repetitions",
+                          apply_timer->get_num_repetitions(), allocator);
 
         // compute and write benchmark data
         add_or_set_member(solver_json, "completed", true, allocator);
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index 22c58f430d5..1afd4c1f91a 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -479,8 +479,10 @@ gko::remove_complex<ValueType> compute_max_relative_norm2(
  * ```
  * auto timer = get_timer(...);
  * IterationControl ic(timer);
- * for(auto status: ic.[warmup_run|run]()){
+ * for(auto status: ic.[warmup_run|run](manage_timings [default is true])){
+ *   if(! manage_timings) timer->tic();
  *   // execute benchmark
+ *   if(! manage_timings) timer->toc();
  * }
  * ```
  * At the beginning of both methods, the timer is reset.
@@ -503,16 +505,17 @@ class IterationControl {
      *
      * @param timer the same timer that is to be used for the timings
      */
-    explicit IterationControl(std::shared_ptr<Timer> timer) : timer_(timer)
+    explicit IterationControl(const std::shared_ptr<Timer> &timer)
     {
-        status_warmup_ = {timer, FLAGS_warmup, FLAGS_warmup, 0., 0};
+        status_warmup_ = {TimerManager{timer, false}, FLAGS_warmup,
+                          FLAGS_warmup, 0., 0};
         if (FLAGS_repetitions == "auto") {
-            status_run_ = {timer, FLAGS_min_repetitions, FLAGS_max_repetitions,
-                           FLAGS_min_runtime};
+            status_run_ = {TimerManager{timer, true}, FLAGS_min_repetitions,
+                           FLAGS_max_repetitions, FLAGS_min_runtime};
         } else {
             const auto reps =
                 static_cast<unsigned int>(std::stoi(FLAGS_repetitions));
-            status_run_ = {timer, reps, reps, 0., 0};
+            status_run_ = {TimerManager{timer, true}, reps, reps, 0., 0};
         }
     }
 
@@ -527,8 +530,8 @@ class IterationControl {
      */
     run_control warmup_run()
     {
-        timer_->clear();
         status_warmup_.cur_it = 0;
+        status_warmup_.timer->clear();
         return run_control{&status_warmup_};
     }
 
@@ -536,11 +539,15 @@ class IterationControl {
      * Creates iterable `run_control` object for the timed run.
      *
      * This run may be adaptive, depending on the commandline flags.
+     *
+     * @param manage_timings If true, the timer calls (`tic/toc`) are handled
+     * by the `run_control` object, otherwise they need to be executed outside
      */
-    run_control run()
+    run_control run(bool manage_timings = true)
     {
-        timer_->clear();
         status_run_.cur_it = 0;
+        status_run_.timer->clear();
+        status_run_.timer->manage_timings = manage_timings;
         return run_control{&status_run_};
     }
 
@@ -553,12 +560,34 @@ class IterationControl {
     IndexType get_num_repetitions() const { return status_run_.cur_it; }
 
 private:
+    struct TimerManager {
+        std::shared_ptr<Timer> timer;
+        bool manage_timings = false;
+
+        void tic()
+        {
+            if (manage_timings) timer->tic();
+        }
+        void toc()
+        {
+            if (manage_timings) timer->toc();
+        }
+
+        void clear() { timer->clear(); }
+
+        double get_total_time() const { return timer->get_total_time(); }
+
+        // emulate shared_ptr behavior
+        const TimerManager *operator->() const { return this; }
+        TimerManager *operator->() { return this; }
+    };
+
     /**
      * Stores stopping criteria of the adaptive benchmark run as well as the
      * current iteration number.
      */
     struct status {
-        std::shared_ptr<Timer> timer = nullptr;
+        TimerManager timer{};
 
         IndexType min_it = 0;
         IndexType max_it = 0;
@@ -577,7 +606,7 @@ class IterationControl {
          *
          * @return completeness state of the adaptive run
          */
-        bool is_finished()
+        bool is_finished() const
         {
             return cur_it >= min_it &&
                    (cur_it >= max_it || timer->get_total_time() >= max_runtime);
@@ -655,8 +684,6 @@ class IterationControl {
         status *info;
     };
 
-    std::shared_ptr<Timer> timer_;
-
     status status_warmup_;
     status status_run_;
 };

From edbec844bae5488802ba34ee9e2a8b99ba117dac Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Wed, 23 Jun 2021 17:38:05 +0200
Subject: [PATCH 0783/1008] removes sync from `toc` call

---
 benchmark/utils/timer.hpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/benchmark/utils/timer.hpp b/benchmark/utils/timer.hpp
index c61cb546a55..5e09e738b19 100644
--- a/benchmark/utils/timer.hpp
+++ b/benchmark/utils/timer.hpp
@@ -272,7 +272,6 @@ class CudaTimer : public Timer {
 
     double toc_impl() override
     {
-        exec_->synchronize();
         gko::cuda::device_guard g{id_};
         // Currently, gko::CudaExecutor always use default stream.
         GKO_ASSERT_NO_CUDA_ERRORS(cudaEventRecord(stop_));
@@ -340,7 +339,6 @@ class HipTimer : public Timer {
 
     double toc_impl() override
     {
-        exec_->synchronize();
         gko::hip::device_guard g{id_};
         // Currently, gko::HipExecutor always use default stream.
         GKO_ASSERT_NO_HIP_ERRORS(hipEventRecord(stop_));

From 2163aa116ae11c4541bd3d0b384af5ed05dbe8cb Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Thu, 24 Jun 2021 17:30:54 +0200
Subject: [PATCH 0784/1008] Review updates

- clarify documentation
- add accessor to underlying timer
- formatting
- adds flag to choose repetitions growth factor

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 benchmark/solver/solver.cpp |  4 +-
 benchmark/utils/general.hpp | 94 +++++++++++++++++++++++--------------
 2 files changed, 60 insertions(+), 38 deletions(-)

diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index 55fc2f78c41..d709cf442c5 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -399,8 +399,7 @@ void solve_system(const std::string &solver_name,
                               allocator);
         }
 
-        auto apply_timer = get_timer(exec, FLAGS_gpu_timer);
-        IterationControl ic{apply_timer};
+        IterationControl ic{get_timer(exec, FLAGS_gpu_timer)};
 
         // warm run
         auto it_logger = std::make_shared<IterationLogger>(exec);
@@ -475,6 +474,7 @@ void solve_system(const std::string &solver_name,
 
         // timed run
         auto generate_timer = get_timer(exec, FLAGS_gpu_timer);
+        auto apply_timer = ic.get_timer();
         auto x_clone = clone(x);
         for (auto status : ic.run(false)) {
             x_clone = clone(x);
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index 1afd4c1f91a..84fb5926af4 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -98,8 +98,8 @@ DEFINE_string(repetitions, "10",
               "runtime is larger than 'min_runtime'");
 
 DEFINE_double(min_runtime, 0.05,
-              "If 'repetitions = auto' is used, the minimal runtime of"
-              " a single benchmark.");
+              "If 'repetitions = auto' is used, the minimal runtime (seconds) "
+              "of a single benchmark.");
 
 DEFINE_uint32(min_repetitions, 10,
               "If 'repetitions = auto' is used, the minimal number of"
@@ -109,6 +109,10 @@ DEFINE_uint32(max_repetitions, std::numeric_limits<unsigned int>::max(),
               "If 'repetitions = auto' is used, the maximal number of"
               " repetitions for a single benchmark.");
 
+DEFINE_double(repetition_growth_factor, 1.5,
+              "If 'repetitions = auto' is used, the factor with which the"
+              " repetitions between two timings increase.");
+
 
 /**
  * Parses arguments through gflags and initialize a documentation string.
@@ -470,20 +474,32 @@ gko::remove_complex<ValueType> compute_max_relative_norm2(
  * - 'min_repetitions' minimal number of repetitions (adaptive case)
  * - 'max_repetitions' maximal number of repetitions (adaptive case)
  * - 'min_runtime' minimal total runtime (adaptive case)
+ * - 'repetition_growth_factor' controls the increase between two successive
+ *   timings
  *
  * Usage:
- * `IterationControl` exposes two member functions:
- * - `warmup_run()`: uses `warmup` flag
- * - `run()`: uses all other flags
- * Both methods return an object that is to be used in a range-based for loop:
+ * `IterationControl` exposes the member functions:
+ * - `warmup_run()`: controls run defined by `warmup` flag
+ * - `run(bool)`: controls run defined by all other flags
+ * - `get_timer()`: access to underlying timer
+ * The first two methods return an object that is to be used in a range-based
+ * for loop:
  * ```
- * auto timer = get_timer(...);
- * IterationControl ic(timer);
- * for(auto status: ic.[warmup_run|run](manage_timings [default is true])){
- *   if(! manage_timings) timer->tic();
+ * IterationControl ic(get_timer(...));
+ *
+ * // warmup run always uses fixed number of iteration and does not issue
+ * // timings
+ * for(auto status: ic.warmup_run()){
  *   // execute benchmark
- *   if(! manage_timings) timer->toc();
  * }
+ * // run may use adaptive number of iterations (depending on cmd line flag)
+ * // and issues timing (unless manage_timings is false)
+ * for(auto status: ic.run(manage_timings [default is true])){
+ *   if(! manage_timings) ic.get_timer->tic();
+ *   // execute benchmark
+ *   if(! manage_timings) ic.get_timer->toc();
+ * }
+ *
  * ```
  * At the beginning of both methods, the timer is reset.
  * The `status` object exposes the member
@@ -503,7 +519,7 @@ class IterationControl {
      * Uses the commandline flags to setup the stopping criteria for the
      * warmup and timed run.
      *
-     * @param timer the same timer that is to be used for the timings
+     * @param timer  the timer that is to be used for the timings
      */
     explicit IterationControl(const std::shared_ptr<Timer> &timer)
     {
@@ -531,7 +547,7 @@ class IterationControl {
     run_control warmup_run()
     {
         status_warmup_.cur_it = 0;
-        status_warmup_.timer->clear();
+        status_warmup_.managed_timer.clear();
         return run_control{&status_warmup_};
     }
 
@@ -546,15 +562,20 @@ class IterationControl {
     run_control run(bool manage_timings = true)
     {
         status_run_.cur_it = 0;
-        status_run_.timer->clear();
-        status_run_.timer->manage_timings = manage_timings;
+        status_run_.managed_timer.clear();
+        status_run_.managed_timer.manage_timings = manage_timings;
         return run_control{&status_run_};
     }
 
+    std::shared_ptr<Timer> get_timer() const
+    {
+        return status_run_.managed_timer.timer;
+    }
 
     double compute_average_time() const
     {
-        return status_run_.timer->get_total_time() / get_num_repetitions();
+        return status_run_.managed_timer.get_total_time() /
+               get_num_repetitions();
     }
 
     IndexType get_num_repetitions() const { return status_run_.cur_it; }
@@ -566,20 +587,20 @@ class IterationControl {
 
         void tic()
         {
-            if (manage_timings) timer->tic();
+            if (manage_timings) {
+                timer->tic();
+            }
         }
         void toc()
         {
-            if (manage_timings) timer->toc();
+            if (manage_timings) {
+                timer->toc();
+            }
         }
 
         void clear() { timer->clear(); }
 
         double get_total_time() const { return timer->get_total_time(); }
-
-        // emulate shared_ptr behavior
-        const TimerManager *operator->() const { return this; }
-        TimerManager *operator->() { return this; }
     };
 
     /**
@@ -587,7 +608,7 @@ class IterationControl {
      * current iteration number.
      */
     struct status {
-        TimerManager timer{};
+        TimerManager managed_timer{};
 
         IndexType min_it = 0;
         IndexType max_it = 0;
@@ -609,7 +630,8 @@ class IterationControl {
         bool is_finished() const
         {
             return cur_it >= min_it &&
-                   (cur_it >= max_it || timer->get_total_time() >= max_runtime);
+                   (cur_it >= max_it ||
+                    managed_timer.get_total_time() >= max_runtime);
         }
     };
 
@@ -624,8 +646,8 @@ class IterationControl {
              * Increases the current iteration count and finishes timing if
              * necessary.
              *
-             * As `++it` is the last step of a for-loop, the timer is stopped,
-             * if enough iterations have passed since the last timing.
+             * As `++it` is the last step of a for-loop, the managed_timer is
+             * stopped, if enough iterations have passed since the last timing.
              * The interval between two timings is steadily increased to
              * reduce the timing overhead.
              */
@@ -633,10 +655,10 @@ class IterationControl {
             {
                 cur_info->cur_it++;
                 if (cur_info->cur_it >= next_timing && !stopped) {
-                    cur_info->timer->toc();
+                    cur_info->managed_timer.toc();
                     stopped = true;
-                    next_timing =
-                        static_cast<IndexType>(std::ceil(next_timing * 1.5));
+                    next_timing = static_cast<IndexType>(std::ceil(
+                        next_timing * FLAGS_repetition_growth_factor));
                 }
                 return *this;
             }
@@ -647,11 +669,11 @@ class IterationControl {
              * Checks if the benchmark is finished and handles timing, if
              * necessary.
              *
-             * As `begin != end` is the first step in a for-loop, the timer is
-             * started, if it was previously stopped.
-             * Additionally, if the benchmark is complete and the timer is still
-             * running it is stopped. (This may occur if the maximal number of
-             * repetitions is surpassed)
+             * As `begin != end` is the first step in a for-loop, the
+             * managed_timer is started, if it was previously stopped.
+             * Additionally, if the benchmark is complete and the managed_timer
+             * is still running it is stopped. (This may occur if the maximal
+             * number of repetitions is surpassed)
              *
              * Uses only the information from the `status` object, i.e.
              * the right hand side is ignored.
@@ -663,9 +685,9 @@ class IterationControl {
                 const bool is_finished = cur_info->is_finished();
                 if (!is_finished && stopped) {
                     stopped = false;
-                    cur_info->timer->tic();
+                    cur_info->managed_timer.tic();
                 } else if (is_finished && !stopped) {
-                    cur_info->timer->toc();
+                    cur_info->managed_timer.toc();
                     stopped = true;
                 }
                 return !is_finished;

From ffeb4468dd6f13bf1e5c3695d44f1f5010709e23 Mon Sep 17 00:00:00 2001
From: Fabian Castelli <fabian.castelli@kit.edu>
Date: Mon, 28 Jun 2021 14:43:29 +0200
Subject: [PATCH 0785/1008] Add whitespace

---
 core/base/version.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/base/version.cpp b/core/base/version.cpp
index 37db8e1dab8..628410d8f98 100644
--- a/core/base/version.cpp
+++ b/core/base/version.cpp
@@ -64,7 +64,7 @@ std::ostream &operator<<(std::ostream &os, const version_info &ver_info)
     print_version(os, ver_info.omp_version);
     os << "\n    the CUDA      module is  ";
     print_version(os, ver_info.cuda_version);
-    os << "\n    the HIP      module is  ";
+    os << "\n    the HIP       module is  ";
     print_version(os, ver_info.hip_version);
     return os;
 }

From f82502fc2ca644fb83364b0d5a02d8f1225295a2 Mon Sep 17 00:00:00 2001
From: Fabian Castelli <fabian.castelli@kit.edu>
Date: Mon, 28 Jun 2021 18:09:28 +0200
Subject: [PATCH 0786/1008] Write out dpcpp version number

---
 core/base/version.cpp                | 2 ++
 include/ginkgo/core/base/version.hpp | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/base/version.cpp b/core/base/version.cpp
index 628410d8f98..1cc7a8c849b 100644
--- a/core/base/version.cpp
+++ b/core/base/version.cpp
@@ -66,6 +66,8 @@ std::ostream &operator<<(std::ostream &os, const version_info &ver_info)
     print_version(os, ver_info.cuda_version);
     os << "\n    the HIP       module is  ";
     print_version(os, ver_info.hip_version);
+    os << "\n    the DPCPP     module is  ";
+    print_version(os, ver_info.dpcpp_version);
     return os;
 }
 
diff --git a/include/ginkgo/core/base/version.hpp b/include/ginkgo/core/base/version.hpp
index b5a712ed1d1..bc9a7a478d8 100644
--- a/include/ginkgo/core/base/version.hpp
+++ b/include/ginkgo/core/base/version.hpp
@@ -154,7 +154,7 @@ inline std::ostream &operator<<(std::ostream &os, const version &ver)
  *     earlier version may have this implemented or fixed in a later version).
  *
  * This structure provides versions of different parts of Ginkgo: the headers,
- * the core and the kernel modules (reference, OpenMP, CUDA, HIP).
+ * the core and the kernel modules (reference, OpenMP, CUDA, HIP, DPCPP).
  * To obtain an instance of version_info filled with information about the
  * current version of Ginkgo, call the version_info::get() static method.
  */

From bac95e9124ee500cd09c95150136f51b52cf7441 Mon Sep 17 00:00:00 2001
From: Fabian Castelli <fabian.castelli@kit.edu>
Date: Tue, 29 Jun 2021 15:54:24 +0200
Subject: [PATCH 0787/1008] Add FC to contributors.txt

---
 contributors.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contributors.txt b/contributors.txt
index c3637859fd2..047fe37d792 100644
--- a/contributors.txt
+++ b/contributors.txt
@@ -4,6 +4,7 @@ license, as specified in the repository's LICENSE file.
 Aliaga José I. <aliaga@uji.es> Universitat Jaume I
 Anzt Hartwig <hartwig.anzt@kit.edu> Karlsruhe Institute of Technology, The University of Tennessee Knoxville
 Boman Erik <egboman@sandia.gov> Sandia National Laboratories
+Castelli Fabian <fabian.castelli@kit.edu> Karlsruhe Institute of Technology
 Chen Yenchen <yanjen224@gmail.com> National Taiwan University
 Cojean Terry <terry.cojean@kit.edu> Karlsruhe Institute of Technology
 Drzaic Jelena <jelena.drzaic1@gmail.com> University of Zagreb

From c769fd28b8345492170d4e6d81eda5f570e3673e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 29 Jun 2021 18:58:05 +0200
Subject: [PATCH 0788/1008] update example output validation

Since we added the DPCPP version information, we need to skip one
additional line
---
 examples/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 61f1d58f50f..57ee5e6e4b7 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -59,7 +59,7 @@ if(GINKGO_RUN_EXAMPLES)
             COMMAND chmod +x ${example_path}/target-wrapper.sh && ${example_path}/target-wrapper.sh > ${example_path}/${example}.out
             WORKING_DIRECTORY ${example_path})
         file(WRITE ${example_path}/diff-command "#!/bin/bash
-diff <(sed -n '7,$p' ${example_path}/${example}.out | sed -E 's/([^a-z,\":\\s\\)\\(\\{\\}_]+)//g') <(sed -n '6,$p' ${CMAKE_SOURCE_DIR}/examples/${example}/doc/results.dox | head -n -4 | sed -E 's/([^a-z,\":\\s\\)\\(\\{\\}_]+)//g')")
+diff <(sed -n '8,$p' ${example_path}/${example}.out | sed -E 's/([^a-z,\":\\s\\)\\(\\{\\}_]+)//g') <(sed -n '6,$p' ${CMAKE_SOURCE_DIR}/examples/${example}/doc/results.dox | head -n -4 | sed -E 's/([^a-z,\":\\s\\)\\(\\{\\}_]+)//g')")
         add_custom_target("validate-${example}"
             COMMAND chmod +x ${example_path}/diff-command && ${example_path}/diff-command
             WORKING_DIRECTORY ${example_path})

From 2691eab0c4e471069531660cf432997359de6aec Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 30 Jun 2021 16:36:22 +0200
Subject: [PATCH 0789/1008] fix Coo overwriting padding

---
 omp/matrix/coo_kernels.cpp            | 17 ++++++++------
 omp/test/matrix/coo_kernels.cpp       | 32 +++++++++++++++++++++++++
 reference/matrix/coo_kernels.cpp      | 12 ++++++----
 reference/test/matrix/coo_kernels.cpp | 34 +++++++++++++++++++++++++++
 4 files changed, 84 insertions(+), 11 deletions(-)

diff --git a/omp/matrix/coo_kernels.cpp b/omp/matrix/coo_kernels.cpp
index 511fc3975a8..cf5a28f78ab 100644
--- a/omp/matrix/coo_kernels.cpp
+++ b/omp/matrix/coo_kernels.cpp
@@ -67,11 +67,12 @@ void spmv(std::shared_ptr<const OmpExecutor> exec,
           const matrix::Coo<ValueType, IndexType> *a,
           const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
 {
-#pragma omp parallel for
-    for (size_type i = 0; i < c->get_num_stored_elements(); i++) {
-        c->at(i) = zero<ValueType>();
+#pragma omp parallel for collapse(2)
+    for (size_type i = 0; i < c->get_size()[0]; i++) {
+        for (size_type j = 0; j < c->get_size()[1]; j++) {
+            c->at(i, j) = zero<ValueType>();
+        }
     }
-
     spmv2(exec, a, b, c);
 }
 
@@ -87,9 +88,11 @@ void advanced_spmv(std::shared_ptr<const OmpExecutor> exec,
                    matrix::Dense<ValueType> *c)
 {
     auto beta_val = beta->at(0, 0);
-#pragma omp parallel for
-    for (size_type i = 0; i < c->get_num_stored_elements(); i++) {
-        c->at(i) *= beta_val;
+#pragma omp parallel for collapse(2)
+    for (size_type i = 0; i < c->get_size()[0]; i++) {
+        for (size_type j = 0; j < c->get_size()[1]; j++) {
+            c->at(i, j) *= beta_val;
+        }
     }
 
     advanced_spmv2(exec, alpha, a, b, c);
diff --git a/omp/test/matrix/coo_kernels.cpp b/omp/test/matrix/coo_kernels.cpp
index 8f90fe7336c..f40422f5bae 100644
--- a/omp/test/matrix/coo_kernels.cpp
+++ b/omp/test/matrix/coo_kernels.cpp
@@ -157,6 +157,22 @@ TEST_F(Coo, SimpleApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Coo, SimpleApplyDoesntOverwritePadding)
+{
+    set_up_apply_data();
+    auto dresult_padded =
+        Vec::create(omp, dresult->get_size(), dresult->get_stride() + 1);
+    dresult_padded->copy_from(dresult.get());
+    dresult_padded->get_values()[1] = 1234.0;
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    ASSERT_EQ(dresult_padded->get_values()[1], 1234.0);
+}
+
+
 TEST_F(Coo, AdvancedApplyIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -168,6 +184,22 @@ TEST_F(Coo, AdvancedApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Coo, AdvancedApplyDoesntOverwritePadding)
+{
+    set_up_apply_data();
+    auto dresult_padded =
+        Vec::create(omp, dresult->get_size(), dresult->get_stride() + 1);
+    dresult_padded->copy_from(dresult.get());
+    dresult_padded->get_values()[1] = 1234.0;
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    ASSERT_EQ(dresult_padded->get_values()[1], 1234.0);
+}
+
+
 TEST_F(Coo, SimpleApplyAddIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/reference/matrix/coo_kernels.cpp b/reference/matrix/coo_kernels.cpp
index 77a5de52913..4fa6743dd20 100644
--- a/reference/matrix/coo_kernels.cpp
+++ b/reference/matrix/coo_kernels.cpp
@@ -63,8 +63,10 @@ void spmv(std::shared_ptr<const ReferenceExecutor> exec,
           const matrix::Coo<ValueType, IndexType> *a,
           const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
 {
-    for (size_type i = 0; i < c->get_num_stored_elements(); i++) {
-        c->at(i) = zero<ValueType>();
+    for (size_type i = 0; i < c->get_size()[0]; i++) {
+        for (size_type j = 0; j < c->get_size()[1]; j++) {
+            c->at(i, j) = zero<ValueType>();
+        }
     }
     spmv2(exec, a, b, c);
 }
@@ -81,8 +83,10 @@ void advanced_spmv(std::shared_ptr<const ReferenceExecutor> exec,
                    matrix::Dense<ValueType> *c)
 {
     auto beta_val = beta->at(0, 0);
-    for (size_type i = 0; i < c->get_num_stored_elements(); i++) {
-        c->at(i) *= beta_val;
+    for (size_type i = 0; i < c->get_size()[0]; i++) {
+        for (size_type j = 0; j < c->get_size()[1]; j++) {
+            c->at(i, j) *= beta_val;
+        }
     }
     advanced_spmv2(exec, alpha, a, b, c);
 }
diff --git a/reference/test/matrix/coo_kernels.cpp b/reference/test/matrix/coo_kernels.cpp
index 1993db54843..b975214dcbf 100644
--- a/reference/test/matrix/coo_kernels.cpp
+++ b/reference/test/matrix/coo_kernels.cpp
@@ -350,6 +350,21 @@ TYPED_TEST(Coo, AppliesToDenseVector)
 }
 
 
+TYPED_TEST(Coo, ApplyToStridedVectorKeepsPadding)
+{
+    using Vec = typename TestFixture::Vec;
+    using T = typename TestFixture::value_type;
+    auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = Vec::create(this->exec, gko::dim<2>{2, 1}, 2);
+    y->get_values()[1] = 1234;
+
+    this->mtx->apply(x.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({13.0, 5.0}), 0.0);
+    ASSERT_EQ(y->get_values()[1], T{1234});
+}
+
+
 TYPED_TEST(Coo, AppliesToMixedDenseVector)
 {
     using MixedVec = typename TestFixture::MixedVec;
@@ -432,6 +447,25 @@ TYPED_TEST(Coo, AppliesLinearCombinationToDenseVector)
 }
 
 
+TYPED_TEST(Coo, ApplyLinearCombinationToStridedVectorKeepsPadding)
+{
+    using Vec = typename TestFixture::Vec;
+    using T = typename TestFixture::value_type;
+    auto alpha = gko::initialize<Vec>({-1.0}, this->exec);
+    auto beta = gko::initialize<Vec>({2.0}, this->exec);
+    auto x = gko::initialize<Vec>({2.0, 1.0, 4.0}, this->exec);
+    auto y = Vec::create(this->exec, gko::dim<2>{2, 1}, 2);
+    y->get_values()[1] = 1234;
+    y->at(0, 0) = 1.0;
+    y->at(1, 0) = 2.0;
+
+    this->mtx->apply(alpha.get(), x.get(), beta.get(), y.get());
+
+    GKO_ASSERT_MTX_NEAR(y, l({-11.0, -1.0}), 0.0);
+    ASSERT_EQ(y->get_values()[1], T{1234});
+}
+
+
 TYPED_TEST(Coo, AppliesLinearCombinationToMixedDenseVector)
 {
     using MixedVec = typename TestFixture::MixedVec;

From c9258ddbd13434d0cf583e1d894cd6113f49bcdd Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 30 Jun 2021 20:58:06 +0200
Subject: [PATCH 0790/1008] review updates

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
---
 cuda/test/matrix/coo_kernels.cpp    | 36 +++++++++++++++++++++++++++++
 hip/test/matrix/coo_kernels.hip.cpp | 36 +++++++++++++++++++++++++++++
 omp/matrix/coo_kernels.cpp          | 17 +++-----------
 reference/matrix/coo_kernels.cpp    | 14 +++--------
 4 files changed, 78 insertions(+), 25 deletions(-)

diff --git a/cuda/test/matrix/coo_kernels.cpp b/cuda/test/matrix/coo_kernels.cpp
index 57593d27f9f..89c16183996 100644
--- a/cuda/test/matrix/coo_kernels.cpp
+++ b/cuda/test/matrix/coo_kernels.cpp
@@ -141,6 +141,24 @@ TEST_F(Coo, SimpleApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Coo, SimpleApplyDoesntOverwritePadding)
+{
+    set_up_apply_data();
+    auto dresult_padded =
+        Vec::create(cuda, dresult->get_size(), dresult->get_stride() + 1);
+    dresult_padded->copy_from(dresult.get());
+    double padding_val{1234.0};
+    cuda->copy_from(cuda->get_master().get(), 1, &padding_val,
+                    dresult_padded->get_values() + 1);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    ASSERT_EQ(cuda->copy_val_to_host(dresult_padded->get_values() + 1), 1234.0);
+}
+
+
 TEST_F(Coo, SimpleApplyIsEquivalentToRefUnsorted)
 {
     set_up_apply_data();
@@ -164,6 +182,24 @@ TEST_F(Coo, AdvancedApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Coo, AdvancedApplyDoesntOverwritePadding)
+{
+    set_up_apply_data();
+    auto dresult_padded =
+        Vec::create(cuda, dresult->get_size(), dresult->get_stride() + 1);
+    dresult_padded->copy_from(dresult.get());
+    double padding_val{1234.0};
+    cuda->copy_from(cuda->get_master().get(), 1, &padding_val,
+                    dresult_padded->get_values() + 1);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    ASSERT_EQ(cuda->copy_val_to_host(dresult_padded->get_values() + 1), 1234.0);
+}
+
+
 TEST_F(Coo, SimpleApplyAddIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/hip/test/matrix/coo_kernels.hip.cpp b/hip/test/matrix/coo_kernels.hip.cpp
index f40fc9436e4..ed7f82ae1e6 100644
--- a/hip/test/matrix/coo_kernels.hip.cpp
+++ b/hip/test/matrix/coo_kernels.hip.cpp
@@ -142,6 +142,24 @@ TEST_F(Coo, SimpleApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Coo, SimpleApplyDoesntOverwritePadding)
+{
+    set_up_apply_data();
+    auto dresult_padded =
+        Vec::create(hip, dresult->get_size(), dresult->get_stride() + 1);
+    dresult_padded->copy_from(dresult.get());
+    double padding_val{1234.0};
+    hip->copy_from(hip->get_master().get(), 1, &padding_val,
+                   dresult_padded->get_values() + 1);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    ASSERT_EQ(hip->copy_val_to_host(dresult_padded->get_values() + 1), 1234.0);
+}
+
+
 TEST_F(Coo, SimpleApplyIsEquivalentToRefUnsorted)
 {
     set_up_apply_data();
@@ -165,6 +183,24 @@ TEST_F(Coo, AdvancedApplyIsEquivalentToRef)
 }
 
 
+TEST_F(Coo, AdvancedApplyDoesntOverwritePadding)
+{
+    set_up_apply_data();
+    auto dresult_padded =
+        Vec::create(hip, dresult->get_size(), dresult->get_stride() + 1);
+    dresult_padded->copy_from(dresult.get());
+    double padding_val{1234.0};
+    hip->copy_from(hip->get_master().get(), 1, &padding_val,
+                   dresult_padded->get_values() + 1);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    ASSERT_EQ(hip->copy_val_to_host(dresult_padded->get_values() + 1), 1234.0);
+}
+
+
 TEST_F(Coo, SimpleApplyAddIsEquivalentToRef)
 {
     set_up_apply_data();
diff --git a/omp/matrix/coo_kernels.cpp b/omp/matrix/coo_kernels.cpp
index cf5a28f78ab..71f5283104c 100644
--- a/omp/matrix/coo_kernels.cpp
+++ b/omp/matrix/coo_kernels.cpp
@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "core/matrix/dense_kernels.hpp"
 #include "omp/components/atomic.hpp"
 #include "omp/components/format_conversion.hpp"
 
@@ -67,12 +68,7 @@ void spmv(std::shared_ptr<const OmpExecutor> exec,
           const matrix::Coo<ValueType, IndexType> *a,
           const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
 {
-#pragma omp parallel for collapse(2)
-    for (size_type i = 0; i < c->get_size()[0]; i++) {
-        for (size_type j = 0; j < c->get_size()[1]; j++) {
-            c->at(i, j) = zero<ValueType>();
-        }
-    }
+    dense::fill(exec, c, zero<ValueType>());
     spmv2(exec, a, b, c);
 }
 
@@ -87,14 +83,7 @@ void advanced_spmv(std::shared_ptr<const OmpExecutor> exec,
                    const matrix::Dense<ValueType> *beta,
                    matrix::Dense<ValueType> *c)
 {
-    auto beta_val = beta->at(0, 0);
-#pragma omp parallel for collapse(2)
-    for (size_type i = 0; i < c->get_size()[0]; i++) {
-        for (size_type j = 0; j < c->get_size()[1]; j++) {
-            c->at(i, j) *= beta_val;
-        }
-    }
-
+    dense::scale(exec, beta, c);
     advanced_spmv2(exec, alpha, a, b, c);
 }
 
diff --git a/reference/matrix/coo_kernels.cpp b/reference/matrix/coo_kernels.cpp
index 4fa6743dd20..b695d266fe3 100644
--- a/reference/matrix/coo_kernels.cpp
+++ b/reference/matrix/coo_kernels.cpp
@@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "core/matrix/dense_kernels.hpp"
 #include "reference/components/format_conversion.hpp"
 
 
@@ -63,11 +64,7 @@ void spmv(std::shared_ptr<const ReferenceExecutor> exec,
           const matrix::Coo<ValueType, IndexType> *a,
           const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
 {
-    for (size_type i = 0; i < c->get_size()[0]; i++) {
-        for (size_type j = 0; j < c->get_size()[1]; j++) {
-            c->at(i, j) = zero<ValueType>();
-        }
-    }
+    dense::fill(exec, c, zero<ValueType>());
     spmv2(exec, a, b, c);
 }
 
@@ -82,12 +79,7 @@ void advanced_spmv(std::shared_ptr<const ReferenceExecutor> exec,
                    const matrix::Dense<ValueType> *beta,
                    matrix::Dense<ValueType> *c)
 {
-    auto beta_val = beta->at(0, 0);
-    for (size_type i = 0; i < c->get_size()[0]; i++) {
-        for (size_type j = 0; j < c->get_size()[1]; j++) {
-            c->at(i, j) *= beta_val;
-        }
-    }
+    dense::scale(exec, beta, c);
     advanced_spmv2(exec, alpha, a, b, c);
 }
 

From a90d53c83011a475f0ef1b030ed452d52be7b43b Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 1 Jul 2021 11:30:22 +0200
Subject: [PATCH 0791/1008] review updates

Actually use the padded vector in coo_kernels tests,
and fix the resulting issues

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
---
 cuda/matrix/coo_kernels.cu          | 5 +----
 cuda/test/matrix/coo_kernels.cpp    | 8 ++++----
 hip/matrix/coo_kernels.hip.cpp      | 5 +----
 hip/test/matrix/coo_kernels.hip.cpp | 8 ++++----
 omp/test/matrix/coo_kernels.cpp     | 8 ++++----
 5 files changed, 14 insertions(+), 20 deletions(-)

diff --git a/cuda/matrix/coo_kernels.cu b/cuda/matrix/coo_kernels.cu
index c9c1328038b..7ad48c76b3f 100644
--- a/cuda/matrix/coo_kernels.cu
+++ b/cuda/matrix/coo_kernels.cu
@@ -40,7 +40,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
-#include "core/components/fill_array.hpp"
 #include "core/matrix/dense_kernels.hpp"
 #include "cuda/base/config.hpp"
 #include "cuda/base/cusparse_bindings.hpp"
@@ -82,9 +81,7 @@ void spmv(std::shared_ptr<const CudaExecutor> exec,
           const matrix::Coo<ValueType, IndexType> *a,
           const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
 {
-    components::fill_array(exec, c->get_values(), c->get_num_stored_elements(),
-                           zero<ValueType>());
-
+    dense::fill(exec, c, zero<ValueType>());
     spmv2(exec, a, b, c);
 }
 
diff --git a/cuda/test/matrix/coo_kernels.cpp b/cuda/test/matrix/coo_kernels.cpp
index 89c16183996..12b8b815d98 100644
--- a/cuda/test/matrix/coo_kernels.cpp
+++ b/cuda/test/matrix/coo_kernels.cpp
@@ -152,9 +152,9 @@ TEST_F(Coo, SimpleApplyDoesntOverwritePadding)
                     dresult_padded->get_values() + 1);
 
     mtx->apply(y.get(), expected.get());
-    dmtx->apply(dy.get(), dresult.get());
+    dmtx->apply(dy.get(), dresult_padded.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult_padded, expected, 1e-14);
     ASSERT_EQ(cuda->copy_val_to_host(dresult_padded->get_values() + 1), 1234.0);
 }
 
@@ -193,9 +193,9 @@ TEST_F(Coo, AdvancedApplyDoesntOverwritePadding)
                     dresult_padded->get_values() + 1);
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult_padded.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult_padded, expected, 1e-14);
     ASSERT_EQ(cuda->copy_val_to_host(dresult_padded->get_values() + 1), 1234.0);
 }
 
diff --git a/hip/matrix/coo_kernels.hip.cpp b/hip/matrix/coo_kernels.hip.cpp
index 8f68bfa0867..c3c1a654f66 100644
--- a/hip/matrix/coo_kernels.hip.cpp
+++ b/hip/matrix/coo_kernels.hip.cpp
@@ -43,7 +43,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
-#include "core/components/fill_array.hpp"
 #include "core/matrix/dense_kernels.hpp"
 #include "hip/base/config.hip.hpp"
 #include "hip/base/hipsparse_bindings.hip.hpp"
@@ -85,9 +84,7 @@ void spmv(std::shared_ptr<const HipExecutor> exec,
           const matrix::Coo<ValueType, IndexType> *a,
           const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
 {
-    components::fill_array(exec, c->get_values(), c->get_num_stored_elements(),
-                           zero<ValueType>());
-
+    dense::fill(exec, c, zero<ValueType>());
     spmv2(exec, a, b, c);
 }
 
diff --git a/hip/test/matrix/coo_kernels.hip.cpp b/hip/test/matrix/coo_kernels.hip.cpp
index ed7f82ae1e6..aeaa9a50206 100644
--- a/hip/test/matrix/coo_kernels.hip.cpp
+++ b/hip/test/matrix/coo_kernels.hip.cpp
@@ -153,9 +153,9 @@ TEST_F(Coo, SimpleApplyDoesntOverwritePadding)
                    dresult_padded->get_values() + 1);
 
     mtx->apply(y.get(), expected.get());
-    dmtx->apply(dy.get(), dresult.get());
+    dmtx->apply(dy.get(), dresult_padded.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult_padded, expected, 1e-14);
     ASSERT_EQ(hip->copy_val_to_host(dresult_padded->get_values() + 1), 1234.0);
 }
 
@@ -194,9 +194,9 @@ TEST_F(Coo, AdvancedApplyDoesntOverwritePadding)
                    dresult_padded->get_values() + 1);
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult_padded.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult_padded, expected, 1e-14);
     ASSERT_EQ(hip->copy_val_to_host(dresult_padded->get_values() + 1), 1234.0);
 }
 
diff --git a/omp/test/matrix/coo_kernels.cpp b/omp/test/matrix/coo_kernels.cpp
index f40422f5bae..31f86355f8d 100644
--- a/omp/test/matrix/coo_kernels.cpp
+++ b/omp/test/matrix/coo_kernels.cpp
@@ -166,9 +166,9 @@ TEST_F(Coo, SimpleApplyDoesntOverwritePadding)
     dresult_padded->get_values()[1] = 1234.0;
 
     mtx->apply(y.get(), expected.get());
-    dmtx->apply(dy.get(), dresult.get());
+    dmtx->apply(dy.get(), dresult_padded.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult_padded, expected, 1e-14);
     ASSERT_EQ(dresult_padded->get_values()[1], 1234.0);
 }
 
@@ -193,9 +193,9 @@ TEST_F(Coo, AdvancedApplyDoesntOverwritePadding)
     dresult_padded->get_values()[1] = 1234.0;
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
-    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult_padded.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult_padded, expected, 1e-14);
     ASSERT_EQ(dresult_padded->get_values()[1], 1234.0);
 }
 

From 1d7a373a7652ee0932fcfda4e80545d3a1782998 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 29 Jun 2021 14:56:06 +0200
Subject: [PATCH 0792/1008] Add scalar jacobi framework

---
 core/device_hooks/common_kernels.inc.cpp      | 11 +++
 core/preconditioner/jacobi.cpp                | 86 ++++++++++++-------
 include/ginkgo/core/preconditioner/jacobi.hpp |  9 +-
 3 files changed, 71 insertions(+), 35 deletions(-)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index c4244c8317c..4f09888e88f 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -1081,6 +1081,17 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_SIMPLE_APPLY_KERNEL);
 
+template <typename ValueType>
+GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
+
+template <typename ValueType>
+GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL);
+
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_JACOBI_TRANSPOSE_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index 5f135b6c010..78f3ffb586e 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -57,7 +57,9 @@ namespace jacobi {
 
 
 GKO_REGISTER_OPERATION(simple_apply, jacobi::simple_apply);
+GKO_REGISTER_OPERATION(simple_scalar_apply, jacobi::simple_scalar_apply);
 GKO_REGISTER_OPERATION(apply, jacobi::apply);
+GKO_REGISTER_OPERATION(scalar_apply, jacobi::scalar_apply);
 GKO_REGISTER_OPERATION(find_blocks, jacobi::find_blocks);
 GKO_REGISTER_OPERATION(generate, jacobi::generate);
 GKO_REGISTER_OPERATION(transpose_jacobi, jacobi::transpose_jacobi);
@@ -74,10 +76,15 @@ void Jacobi<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
 {
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_b, auto dense_x) {
-            this->get_executor()->run(jacobi::make_simple_apply(
-                num_blocks_, parameters_.max_block_size, storage_scheme_,
-                parameters_.storage_optimization.block_wise,
-                parameters_.block_pointers, blocks_, dense_b, dense_x));
+            if (parameters_.max_block_size == 1) {
+                this->get_executor()->run(jacobi::make_simple_scalar_apply(
+                    this->diag_.get(), dense_b, dense_x));
+            } else {
+                this->get_executor()->run(jacobi::make_simple_apply(
+                    num_blocks_, parameters_.max_block_size, storage_scheme_,
+                    parameters_.storage_optimization.block_wise,
+                    parameters_.block_pointers, blocks_, dense_b, dense_x));
+            }
         },
         b, x);
 }
@@ -90,11 +97,17 @@ void Jacobi<ValueType, IndexType>::apply_impl(const LinOp *alpha,
 {
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
-            this->get_executor()->run(jacobi::make_apply(
-                num_blocks_, parameters_.max_block_size, storage_scheme_,
-                parameters_.storage_optimization.block_wise,
-                parameters_.block_pointers, blocks_, dense_alpha, dense_b,
-                dense_beta, dense_x));
+            if (parameters_.max_block_size == 1) {
+                this->get_executor()->run(
+                    jacobi::make_scalar_apply(this->diag_.get(), dense_alpha,
+                                              dense_b, dense_beta, dense_x));
+            } else {
+                this->get_executor()->run(jacobi::make_apply(
+                    num_blocks_, parameters_.max_block_size, storage_scheme_,
+                    parameters_.storage_optimization.block_wise,
+                    parameters_.block_pointers, blocks_, dense_alpha, dense_b,
+                    dense_beta, dense_x));
+            }
         },
         alpha, b, beta, x);
 }
@@ -218,33 +231,40 @@ void Jacobi<ValueType, IndexType>::generate(const LinOp *system_matrix,
     GKO_ASSERT_IS_SQUARE_MATRIX(system_matrix);
     using csr_type = matrix::Csr<ValueType, IndexType>;
     const auto exec = this->get_executor();
-    auto csr_mtx =
-        convert_to_with_sorting<csr_type>(exec, system_matrix, skip_sorting);
-
-    if (parameters_.block_pointers.get_data() == nullptr) {
-        this->detect_blocks(csr_mtx.get());
-    }
+    if (parameters_.max_block_size == 1) {
+        this->diag_ = as<DiagonalExtractable<ValueType>>(system_matrix)
+                          ->extract_diagonal();
+    } else {
+        auto csr_mtx = convert_to_with_sorting<csr_type>(exec, system_matrix,
+                                                         skip_sorting);
+
+        if (parameters_.block_pointers.get_data() == nullptr) {
+            this->detect_blocks(csr_mtx.get());
+        }
 
-    const auto all_block_opt = parameters_.storage_optimization.of_all_blocks;
-    auto &precisions = parameters_.storage_optimization.block_wise;
-    // if adaptive version is used, make sure that the precision array is of
-    // the correct size by replicating it multiple times if needed
-    if (parameters_.storage_optimization.is_block_wise ||
-        all_block_opt != precision_reduction(0, 0)) {
-        if (!parameters_.storage_optimization.is_block_wise) {
-            precisions = gko::Array<precision_reduction>(exec, {all_block_opt});
+        const auto all_block_opt =
+            parameters_.storage_optimization.of_all_blocks;
+        auto &precisions = parameters_.storage_optimization.block_wise;
+        // if adaptive version is used, make sure that the precision array is of
+        // the correct size by replicating it multiple times if needed
+        if (parameters_.storage_optimization.is_block_wise ||
+            all_block_opt != precision_reduction(0, 0)) {
+            if (!parameters_.storage_optimization.is_block_wise) {
+                precisions =
+                    gko::Array<precision_reduction>(exec, {all_block_opt});
+            }
+            Array<precision_reduction> tmp(
+                exec, parameters_.block_pointers.get_num_elems() - 1);
+            exec->run(jacobi::make_initialize_precisions(precisions, tmp));
+            precisions = std::move(tmp);
+            conditioning_.resize_and_reset(num_blocks_);
         }
-        Array<precision_reduction> tmp(
-            exec, parameters_.block_pointers.get_num_elems() - 1);
-        exec->run(jacobi::make_initialize_precisions(precisions, tmp));
-        precisions = std::move(tmp);
-        conditioning_.resize_and_reset(num_blocks_);
-    }
 
-    exec->run(jacobi::make_generate(
-        csr_mtx.get(), num_blocks_, parameters_.max_block_size,
-        parameters_.accuracy, storage_scheme_, conditioning_, precisions,
-        parameters_.block_pointers, blocks_));
+        exec->run(jacobi::make_generate(
+            csr_mtx.get(), num_blocks_, parameters_.max_block_size,
+            parameters_.accuracy, storage_scheme_, conditioning_, precisions,
+            parameters_.block_pointers, blocks_));
+    }
 }
 
 
diff --git a/include/ginkgo/core/preconditioner/jacobi.hpp b/include/ginkgo/core/preconditioner/jacobi.hpp
index 85924ac3772..5d40d2ad169 100644
--- a/include/ginkgo/core/preconditioner/jacobi.hpp
+++ b/include/ginkgo/core/preconditioner/jacobi.hpp
@@ -36,7 +36,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/lin_op.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
 
 
 namespace gko {
@@ -500,7 +502,8 @@ class Jacobi : public EnableLinOp<Jacobi<ValueType, IndexType>>,
         : EnableLinOp<Jacobi>(exec),
           num_blocks_{},
           blocks_(exec),
-          conditioning_(exec)
+          conditioning_(exec),
+          diag_(matrix::Diagonal<value_type>::create(exec))
     {
         parameters_.block_pointers.set_executor(exec);
         parameters_.storage_optimization.block_wise.set_executor(exec);
@@ -524,7 +527,8 @@ class Jacobi : public EnableLinOp<Jacobi<ValueType, IndexType>>,
           blocks_(factory->get_executor(),
                   storage_scheme_.compute_storage_space(
                       parameters_.block_pointers.get_num_elems() - 1)),
-          conditioning_(factory->get_executor())
+          conditioning_(factory->get_executor()),
+          diag_(matrix::Diagonal<value_type>::create(factory->get_executor()))
     {
         parameters_.block_pointers.set_executor(this->get_executor());
         parameters_.storage_optimization.block_wise.set_executor(
@@ -604,6 +608,7 @@ class Jacobi : public EnableLinOp<Jacobi<ValueType, IndexType>>,
     size_type num_blocks_;
     Array<value_type> blocks_;
     Array<remove_complex<value_type>> conditioning_;
+    std::shared_ptr<matrix::Diagonal<value_type>> diag_;
 };
 
 

From a1c4f8fc3ef825b081b013248fdc050b61019b96 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 29 Jun 2021 14:56:40 +0200
Subject: [PATCH 0793/1008] Add scalar jacobi reference and omp kernels

---
 core/preconditioner/jacobi_kernels.hpp        | 18 ++++
 dpcpp/preconditioner/jacobi_kernels.dp.cpp    | 21 +++++
 omp/preconditioner/jacobi_kernels.cpp         | 39 ++++++++
 reference/preconditioner/jacobi_kernels.cpp   | 51 +++++++++--
 .../test/preconditioner/jacobi_kernels.cpp    | 88 +++++++++++++++++++
 5 files changed, 210 insertions(+), 7 deletions(-)

diff --git a/core/preconditioner/jacobi_kernels.hpp b/core/preconditioner/jacobi_kernels.hpp
index e5738e9f51f..32c5c0d377c 100644
--- a/core/preconditioner/jacobi_kernels.hpp
+++ b/core/preconditioner/jacobi_kernels.hpp
@@ -74,6 +74,12 @@ namespace kernels {
         const matrix::Dense<ValueType> *b,                                     \
         const matrix::Dense<ValueType> *beta, matrix::Dense<ValueType> *x)
 
+#define GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL(ValueType)          \
+    void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec, \
+                             const matrix::Diagonal<ValueType> *diag,     \
+                             const matrix::Dense<ValueType> *b,           \
+                             matrix::Dense<ValueType> *x)
+
 #define GKO_DECLARE_JACOBI_SIMPLE_APPLY_KERNEL(ValueType, IndexType)       \
     void simple_apply(                                                     \
         std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks, \
@@ -85,6 +91,14 @@ namespace kernels {
         const Array<ValueType> &blocks, const matrix::Dense<ValueType> *b, \
         matrix::Dense<ValueType> *x)
 
+#define GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL(ValueType)          \
+    void scalar_apply(std::shared_ptr<const DefaultExecutor> exec, \
+                      const matrix::Diagonal<ValueType> *diag,     \
+                      const matrix::Dense<ValueType> *alpha,       \
+                      const matrix::Dense<ValueType> *b,           \
+                      const matrix::Dense<ValueType> *beta,        \
+                      matrix::Dense<ValueType> *x)
+
 #define GKO_DECLARE_JACOBI_TRANSPOSE_KERNEL(ValueType, IndexType)          \
     void transpose_jacobi(                                                 \
         std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks, \
@@ -127,8 +141,12 @@ namespace kernels {
     GKO_DECLARE_JACOBI_FIND_BLOCKS_KERNEL(ValueType, IndexType);      \
     template <typename ValueType, typename IndexType>                 \
     GKO_DECLARE_JACOBI_GENERATE_KERNEL(ValueType, IndexType);         \
+    template <typename ValueType>                                     \
+    GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL(ValueType);                \
     template <typename ValueType, typename IndexType>                 \
     GKO_DECLARE_JACOBI_APPLY_KERNEL(ValueType, IndexType);            \
+    template <typename ValueType>                                     \
+    GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL(ValueType);         \
     template <typename ValueType, typename IndexType>                 \
     GKO_DECLARE_JACOBI_SIMPLE_APPLY_KERNEL(ValueType, IndexType);     \
     template <typename ValueType, typename IndexType>                 \
diff --git a/dpcpp/preconditioner/jacobi_kernels.dp.cpp b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
index 623e2c121ec..bad11a76952 100644
--- a/dpcpp/preconditioner/jacobi_kernels.dp.cpp
+++ b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
@@ -280,6 +280,27 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL);
 
 
+template <typename ValueType>
+void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
+                  const matrix::Diagonal<ValueType> *diag,
+                  const matrix::Dense<ValueType> *alpha,
+                  const matrix::Dense<ValueType> *b,
+                  const matrix::Dense<ValueType> *beta,
+                  matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
+
+
+template <typename ValueType>
+void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
+                         const matrix::Diagonal<ValueType> *diag,
+                         const matrix::Dense<ValueType> *b,
+                         matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL);
+
+
 }  // namespace jacobi
 }  // namespace dpcpp
 }  // namespace kernels
diff --git a/omp/preconditioner/jacobi_kernels.cpp b/omp/preconditioner/jacobi_kernels.cpp
index 1b0c0c6c1a5..163b90ab5b1 100644
--- a/omp/preconditioner/jacobi_kernels.cpp
+++ b/omp/preconditioner/jacobi_kernels.cpp
@@ -696,6 +696,45 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL);
 
 
+template <typename ValueType>
+void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
+                  const matrix::Diagonal<ValueType> *diag,
+                  const matrix::Dense<ValueType> *alpha,
+                  const matrix::Dense<ValueType> *b,
+                  const matrix::Dense<ValueType> *beta,
+                  matrix::Dense<ValueType> *x)
+{
+#pragma omp parallel for
+    for (size_type i = 0; i < x->get_size()[0]; ++i) {
+        for (size_type j = 0; j < x->get_size()[1]; ++j) {
+            x->at(i, j) =
+                beta->at(0) * x->at(i, j) +
+                alpha->at(0) * b->at(i, j) / diag->get_const_values()[i];
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
+
+
+template <typename ValueType>
+void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
+                         const matrix::Diagonal<ValueType> *diag,
+                         const matrix::Dense<ValueType> *b,
+                         matrix::Dense<ValueType> *x)
+{
+#pragma omp parallel for
+    for (size_type i = 0; i < x->get_size()[0]; ++i) {
+        for (size_type j = 0; j < x->get_size()[1]; ++j) {
+            x->at(i, j) = b->at(i, j) / diag->get_const_values()[i];
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL);
+
+
 }  // namespace jacobi
 }  // namespace omp
 }  // namespace kernels
diff --git a/reference/preconditioner/jacobi_kernels.cpp b/reference/preconditioner/jacobi_kernels.cpp
index b96af16dca2..11fa2cfc9e6 100644
--- a/reference/preconditioner/jacobi_kernels.cpp
+++ b/reference/preconditioner/jacobi_kernels.cpp
@@ -136,7 +136,7 @@ inline size_type agglomerate_supervariables(uint32 max_block_size,
 
 
 template <typename ValueType, typename IndexType>
-void find_blocks(std::shared_ptr<const ReferenceExecutor> exec,
+void find_blocks(std::shared_ptr<const DefaultExecutor> exec,
                  const matrix::Csr<ValueType, IndexType> *system_matrix,
                  uint32 max_block_size, size_type &num_blocks,
                  Array<IndexType> &block_pointers)
@@ -310,7 +310,7 @@ inline bool invert_block(IndexType block_size, IndexType *perm,
 
 template <typename ReducedType, typename ValueType, typename IndexType>
 inline bool validate_precision_reduction_feasibility(
-    std::shared_ptr<const ReferenceExecutor> exec, IndexType block_size,
+    std::shared_ptr<const DefaultExecutor> exec, IndexType block_size,
     const ValueType *block, size_type stride)
 {
     using gko::detail::float_traits;
@@ -340,7 +340,7 @@ inline bool validate_precision_reduction_feasibility(
 
 
 template <typename ValueType, typename IndexType>
-void generate(std::shared_ptr<const ReferenceExecutor> exec,
+void generate(std::shared_ptr<const DefaultExecutor> exec,
               const matrix::Csr<ValueType, IndexType> *system_matrix,
               size_type num_blocks, uint32 max_block_size,
               remove_complex<ValueType> accuracy,
@@ -480,7 +480,7 @@ inline void apply_block(size_type block_size, size_type num_rhs,
 }  // namespace
 
 
-void initialize_precisions(std::shared_ptr<const ReferenceExecutor> exec,
+void initialize_precisions(std::shared_ptr<const DefaultExecutor> exec,
                            const Array<precision_reduction> &source,
                            Array<precision_reduction> &precisions)
 {
@@ -492,7 +492,7 @@ void initialize_precisions(std::shared_ptr<const ReferenceExecutor> exec,
 
 
 template <typename ValueType, typename IndexType>
-void apply(std::shared_ptr<const ReferenceExecutor> exec, size_type num_blocks,
+void apply(std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks,
            uint32 max_block_size,
            const preconditioner::block_interleaved_storage_scheme<IndexType>
                &storage_scheme,
@@ -528,7 +528,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_JACOBI_APPLY_KERNEL);
 
 template <typename ValueType, typename IndexType>
 void simple_apply(
-    std::shared_ptr<const ReferenceExecutor> exec, size_type num_blocks,
+    std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks,
     uint32 max_block_size,
     const preconditioner::block_interleaved_storage_scheme<IndexType>
         &storage_scheme,
@@ -560,6 +560,43 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_SIMPLE_APPLY_KERNEL);
 
 
+template <typename ValueType>
+void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
+                  const matrix::Diagonal<ValueType> *diag,
+                  const matrix::Dense<ValueType> *alpha,
+                  const matrix::Dense<ValueType> *b,
+                  const matrix::Dense<ValueType> *beta,
+                  matrix::Dense<ValueType> *x)
+{
+    for (size_type i = 0; i < x->get_size()[0]; ++i) {
+        for (size_type j = 0; j < x->get_size()[1]; ++j) {
+            x->at(i, j) =
+                beta->at(0) * x->at(i, j) +
+                alpha->at(0) * b->at(i, j) / diag->get_const_values()[i];
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
+
+
+template <typename ValueType>
+void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
+                         const matrix::Diagonal<ValueType> *diag,
+                         const matrix::Dense<ValueType> *b,
+                         matrix::Dense<ValueType> *x)
+{
+    for (size_type i = 0; i < x->get_size()[0]; ++i) {
+        for (size_type j = 0; j < x->get_size()[1]; ++j) {
+            x->at(i, j) = b->at(i, j) / diag->get_const_values()[i];
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void transpose_jacobi(
     std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks,
@@ -634,7 +671,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void convert_to_dense(
-    std::shared_ptr<const ReferenceExecutor> exec, size_type num_blocks,
+    std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks,
     const Array<precision_reduction> &block_precisions,
     const Array<IndexType> &block_pointers, const Array<ValueType> &blocks,
     const preconditioner::block_interleaved_storage_scheme<IndexType>
diff --git a/reference/test/preconditioner/jacobi_kernels.cpp b/reference/test/preconditioner/jacobi_kernels.cpp
index f9cda1ad532..55192bf36c6 100644
--- a/reference/test/preconditioner/jacobi_kernels.cpp
+++ b/reference/test/preconditioner/jacobi_kernels.cpp
@@ -75,6 +75,7 @@ class Jacobi : public ::testing::Test {
         block_pointers.get_data()[2] = 5;
         block_precisions.get_data()[0] = gko::precision_reduction(0, 1);
         block_precisions.get_data()[1] = gko::precision_reduction(0, 0);
+        scalar_j_factory = Bj::build().with_max_block_size(1u).on(exec);
         bj_factory = Bj::build()
                          .with_max_block_size(3u)
                          .with_block_pointers(block_pointers)
@@ -111,6 +112,7 @@ class Jacobi : public ::testing::Test {
 
     std::shared_ptr<const gko::Executor> exec;
     std::unique_ptr<typename Bj::Factory> bj_factory;
+    std::unique_ptr<typename Bj::Factory> scalar_j_factory;
     std::unique_ptr<typename Bj::Factory> adaptive_bj_factory;
     gko::Array<index_type> block_pointers;
     gko::Array<gko::precision_reduction> block_precisions;
@@ -656,6 +658,21 @@ TYPED_TEST(Jacobi, AppliesToVector)
 }
 
 
+TYPED_TEST(Jacobi, ScalarJacobiAppliesToVector)
+{
+    using Vec = typename TestFixture::Vec;
+    using value_type = typename TestFixture::value_type;
+    auto x = gko::initialize<Vec>({1.0, -1.0, 2.0, -2.0, 3.0}, this->exec);
+    auto b = gko::initialize<Vec>({4.0, -1.0, -2.0, 4.0, -1.0}, this->exec);
+    auto sj = this->scalar_j_factory->generate(this->mtx);
+
+    sj->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, -0.25, -0.5, 1.0, -0.25}),
+                        r<value_type>::value);
+}
+
+
 TYPED_TEST(Jacobi, AppliesToMixedVector)
 {
     using value_type = gko::next_precision<typename TestFixture::value_type>;
@@ -783,6 +800,32 @@ TYPED_TEST(Jacobi, AppliesToMultipleVectors)
 }
 
 
+TYPED_TEST(Jacobi, ScalarJacobiAppliesToMultipleVectors)
+{
+    using Vec = typename TestFixture::Vec;
+    using value_type = typename TestFixture::value_type;
+    using T = value_type;
+    auto x =
+        gko::initialize<Vec>(3,
+                             {I<T>{1.0, 0.5}, I<T>{-1.0, -0.5}, I<T>{2.0, 1.0},
+                              I<T>{-2.0, -1.0}, I<T>{3.0, 1.5}},
+                             this->exec);
+    auto b =
+        gko::initialize<Vec>(3,
+                             {I<T>{4.0, -2.0}, I<T>{-1.0, 4.0}, I<T>{-2.0, 0.0},
+                              I<T>{4.0, -2.0}, I<T>{-1.0, 4.0}},
+                             this->exec);
+    auto sj = this->scalar_j_factory->generate(this->mtx);
+
+    sj->apply(b.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x,
+        l({{1.0, -0.5}, {-0.25, 1.0}, {-0.5, 0.0}, {1.0, -0.5}, {-0.25, 1.0}}),
+        r<value_type>::value);
+}
+
+
 TYPED_TEST(Jacobi, AppliesToMultipleVectorsWithAdaptivePrecision)
 {
     using Vec = typename TestFixture::Vec;
@@ -859,6 +902,23 @@ TYPED_TEST(Jacobi, AppliesLinearCombinationToVector)
 }
 
 
+TYPED_TEST(Jacobi, ScalarJacobiAppliesLinearCombinationToVector)
+{
+    using Vec = typename TestFixture::Vec;
+    using value_type = typename TestFixture::value_type;
+    auto x = gko::initialize<Vec>({1.0, -1.0, 2.0, -2.0, 3.0}, this->exec);
+    auto b = gko::initialize<Vec>({4.0, -1.0, -2.0, 4.0, -1.0}, this->exec);
+    auto alpha = gko::initialize<Vec>({2.0}, this->exec);
+    auto beta = gko::initialize<Vec>({-1.0}, this->exec);
+    auto sj = this->scalar_j_factory->generate(this->mtx);
+
+    sj->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(x, l({1.0, 0.5, -3.0, 4.0, -3.5}),
+                        r<value_type>::value);
+}
+
+
 TYPED_TEST(Jacobi, AppliesLinearCombinationToMixedVector)
 {
     using value_type = gko::next_precision<typename TestFixture::value_type>;
@@ -973,6 +1033,34 @@ TYPED_TEST(Jacobi, AppliesLinearCombinationToMultipleVectors)
 }
 
 
+TYPED_TEST(Jacobi, ScalarAppliesLinearCombinationToMultipleVectors)
+{
+    using Vec = typename TestFixture::Vec;
+    using value_type = typename TestFixture::value_type;
+    using T = value_type;
+    auto half_tol = std::sqrt(r<value_type>::value);
+    auto x =
+        gko::initialize<Vec>(3,
+                             {I<T>{1.0, 0.5}, I<T>{-1.0, -0.5}, I<T>{2.0, 1.0},
+                              I<T>{-2.0, -1.0}, I<T>{3.0, 1.5}},
+                             this->exec);
+    auto b =
+        gko::initialize<Vec>(3,
+                             {I<T>{4.0, -2.0}, I<T>{-1.0, 4.0}, I<T>{-2.0, 0.0},
+                              I<T>{4.0, -2.0}, I<T>{-1.0, 4.0}},
+                             this->exec);
+    auto alpha = gko::initialize<Vec>({2.0}, this->exec);
+    auto beta = gko::initialize<Vec>({-1.0}, this->exec);
+    auto sj = this->scalar_j_factory->generate(this->mtx);
+
+    sj->apply(alpha.get(), b.get(), beta.get(), x.get());
+
+    GKO_ASSERT_MTX_NEAR(
+        x, l({{1.0, -1.5}, {0.5, 2.5}, {-3.0, -1.0}, {4.0, 0.0}, {-3.5, 0.5}}),
+        r<value_type>::value);
+}
+
+
 TYPED_TEST(Jacobi,
            AppliesLinearCombinationToMultipleVectorsWithAdaptivePrecision)
 {

From 0b5c713bbbf3a0f3e356ddab17034370fbd79bec Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 29 Jun 2021 14:57:04 +0200
Subject: [PATCH 0794/1008] Add cuda and hip kernels

---
 common/preconditioner/jacobi_kernels.hpp.inc | 43 +++++++++++++
 cuda/preconditioner/jacobi_kernels.cu        | 67 ++++++++++++++++++--
 hip/preconditioner/jacobi_kernels.hip.cpp    | 57 +++++++++++++++++
 3 files changed, 162 insertions(+), 5 deletions(-)

diff --git a/common/preconditioner/jacobi_kernels.hpp.inc b/common/preconditioner/jacobi_kernels.hpp.inc
index c3fa889b210..d4d2c69e326 100644
--- a/common/preconditioner/jacobi_kernels.hpp.inc
+++ b/common/preconditioner/jacobi_kernels.hpp.inc
@@ -213,3 +213,46 @@ __launch_bounds__(warps_per_block *config::warp_size) adaptive_transpose_jacobi(
             });
     }
 }
+
+
+namespace kernel {
+
+
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void scalar_apply(
+    size_type num_rows, size_type num_cols, const ValueType *__restrict__ diag,
+    const ValueType *__restrict__ alpha, size_type source_stride,
+    const ValueType *__restrict__ source_values,
+    const ValueType *__restrict__ beta, size_type result_stride,
+    ValueType *__restrict__ result_values)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    const auto row = tidx / num_cols;
+    const auto col = tidx % num_cols;
+
+    if (row < num_rows) {
+        result_values[row * result_stride + col] =
+            result_values[row * result_stride + col] * beta[0] +
+            alpha[0] * source_values[row * source_stride + col] / diag[row];
+    }
+}
+
+
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void simple_scalar_apply(
+    size_type num_rows, size_type num_cols, const ValueType *__restrict__ diag,
+    size_type source_stride, const ValueType *__restrict__ source_values,
+    size_type result_stride, ValueType *__restrict__ result_values)
+{
+    const auto tidx = thread::get_thread_id_flat();
+    const auto row = tidx / num_cols;
+    const auto col = tidx % num_cols;
+
+    if (row < num_rows) {
+        result_values[row * result_stride + col] =
+            source_values[row * source_stride + col] / diag[row];
+    }
+}
+
+
+}  // namespace kernel
diff --git a/cuda/preconditioner/jacobi_kernels.cu b/cuda/preconditioner/jacobi_kernels.cu
index 889fd1b0db6..fc8e64b00d8 100644
--- a/cuda/preconditioner/jacobi_kernels.cu
+++ b/cuda/preconditioner/jacobi_kernels.cu
@@ -65,12 +65,14 @@ constexpr int default_num_warps = 32;
 // current GPUs have at most 84 SMs)
 constexpr int default_grid_size = 32 * 32 * 128;
 
+constexpr int default_block_size = 512;
+
 
 #include "common/preconditioner/jacobi_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
-size_type find_natural_blocks(std::shared_ptr<const CudaExecutor> exec,
+size_type find_natural_blocks(std::shared_ptr<const DefaultExecutor> exec,
                               const matrix::Csr<ValueType, IndexType> *mtx,
                               int32 max_block_size,
                               IndexType *__restrict__ block_ptrs)
@@ -95,7 +97,7 @@ size_type find_natural_blocks(std::shared_ptr<const CudaExecutor> exec,
 
 template <typename IndexType>
 inline size_type agglomerate_supervariables(
-    std::shared_ptr<const CudaExecutor> exec, int32 max_block_size,
+    std::shared_ptr<const DefaultExecutor> exec, int32 max_block_size,
     size_type num_natural_blocks, IndexType *block_ptrs)
 {
     Array<size_type> nums(exec, 1);
@@ -111,7 +113,7 @@ inline size_type agglomerate_supervariables(
 }  // namespace
 
 
-void initialize_precisions(std::shared_ptr<const CudaExecutor> exec,
+void initialize_precisions(std::shared_ptr<const DefaultExecutor> exec,
                            const Array<precision_reduction> &source,
                            Array<precision_reduction> &precisions)
 {
@@ -126,7 +128,7 @@ void initialize_precisions(std::shared_ptr<const CudaExecutor> exec,
 
 
 template <typename ValueType, typename IndexType>
-void find_blocks(std::shared_ptr<const CudaExecutor> exec,
+void find_blocks(std::shared_ptr<const DefaultExecutor> exec,
                  const matrix::Csr<ValueType, IndexType> *system_matrix,
                  uint32 max_block_size, size_type &num_blocks,
                  Array<IndexType> &block_pointers)
@@ -230,7 +232,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType, typename IndexType>
 void convert_to_dense(
-    std::shared_ptr<const CudaExecutor> exec, size_type num_blocks,
+    std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks,
     const Array<precision_reduction> &block_precisions,
     const Array<IndexType> &block_pointers, const Array<ValueType> &blocks,
     const preconditioner::block_interleaved_storage_scheme<IndexType>
@@ -241,6 +243,61 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL);
 
 
+template <typename ValueType>
+void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
+                  const matrix::Diagonal<ValueType> *diag,
+                  const matrix::Dense<ValueType> *alpha,
+                  const matrix::Dense<ValueType> *b,
+                  const matrix::Dense<ValueType> *beta,
+                  matrix::Dense<ValueType> *x)
+{
+    const auto b_size = b->get_size();
+    const auto num_rows = b_size[0];
+    const auto num_cols = b_size[1];
+    const auto b_stride = b->get_stride();
+    const auto x_stride = x->get_stride();
+    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
+
+    const auto b_values = b->get_const_values();
+    const auto diag_values = diag->get_const_values();
+    auto x_values = x->get_values();
+
+    kernel::scalar_apply<<<grid_dim, default_block_size>>>(
+        num_rows, num_cols, as_cuda_type(diag_values),
+        as_cuda_type(alpha->get_const_values()), b_stride,
+        as_cuda_type(b_values), as_cuda_type(beta->get_const_values()),
+        x_stride, as_cuda_type(x_values));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
+
+
+template <typename ValueType>
+void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
+                         const matrix::Diagonal<ValueType> *diag,
+                         const matrix::Dense<ValueType> *b,
+                         matrix::Dense<ValueType> *x)
+{
+    const auto b_size = b->get_size();
+    const auto num_rows = b_size[0];
+    const auto num_cols = b_size[1];
+    const auto b_stride = b->get_stride();
+    const auto x_stride = x->get_stride();
+    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
+
+    const auto b_values = b->get_const_values();
+    const auto diag_values = diag->get_const_values();
+    auto x_values = x->get_values();
+
+    kernel::simple_scalar_apply<<<grid_dim, default_block_size>>>(
+        num_rows, num_cols, as_cuda_type(diag_values), b_stride,
+        as_cuda_type(b_values), x_stride, as_cuda_type(x_values));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL);
+
+
 }  // namespace jacobi
 }  // namespace cuda
 }  // namespace kernels
diff --git a/hip/preconditioner/jacobi_kernels.hip.cpp b/hip/preconditioner/jacobi_kernels.hip.cpp
index fc020837a22..4cd7dc01040 100644
--- a/hip/preconditioner/jacobi_kernels.hip.cpp
+++ b/hip/preconditioner/jacobi_kernels.hip.cpp
@@ -72,6 +72,7 @@ constexpr int default_num_warps = 32;
 // current GPUs have at most 84 SMs)
 constexpr int default_grid_size = 32 * 32 * 128;
 
+constexpr int default_block_size = 512;
 
 #include "common/preconditioner/jacobi_kernels.hpp.inc"
 
@@ -256,6 +257,62 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL);
 
 
+template <typename ValueType>
+void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
+                  const matrix::Diagonal<ValueType> *diag,
+                  const matrix::Dense<ValueType> *alpha,
+                  const matrix::Dense<ValueType> *b,
+                  const matrix::Dense<ValueType> *beta,
+                  matrix::Dense<ValueType> *x)
+{
+    const auto b_size = b->get_size();
+    const auto num_rows = b_size[0];
+    const auto num_cols = b_size[1];
+    const auto b_stride = b->get_stride();
+    const auto x_stride = x->get_stride();
+    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
+
+    const auto b_values = b->get_const_values();
+    const auto diag_values = diag->get_const_values();
+    auto x_values = x->get_values();
+
+    hipLaunchKernelGGL(
+        kernel::scalar_apply, dim3(grid_dim), dim3(default_block_size), 0, 0,
+        num_rows, num_cols, as_hip_type(diag_values),
+        as_hip_type(alpha->get_const_values()), b_stride, as_hip_type(b_values),
+        as_hip_type(beta->get_const_values()), x_stride, as_hip_type(x_values));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
+
+
+template <typename ValueType>
+void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
+                         const matrix::Diagonal<ValueType> *diag,
+                         const matrix::Dense<ValueType> *b,
+                         matrix::Dense<ValueType> *x)
+{
+    const auto b_size = b->get_size();
+    const auto num_rows = b_size[0];
+    const auto num_cols = b_size[1];
+    const auto b_stride = b->get_stride();
+    const auto x_stride = x->get_stride();
+    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
+
+    const auto b_values = b->get_const_values();
+    const auto diag_values = diag->get_const_values();
+    auto x_values = x->get_values();
+
+    hipLaunchKernelGGL(kernel::simple_scalar_apply, dim3(grid_dim),
+                       dim3(default_block_size), 0, 0, num_rows, num_cols,
+                       as_hip_type(diag_values), b_stride,
+                       as_hip_type(b_values), x_stride, as_hip_type(x_values));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL);
+
+
 }  // namespace jacobi
 }  // namespace hip
 }  // namespace kernels

From ce8efdf649abd722f5849219c9151d74cffd9284 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 30 Jun 2021 16:44:10 +0200
Subject: [PATCH 0795/1008] Add omp, cuda and hip tests

---
 cuda/test/preconditioner/jacobi_kernels.cpp | 69 +++++++++++++++++++++
 hip/test/preconditioner/jacobi_kernels.cpp  | 69 +++++++++++++++++++++
 omp/test/preconditioner/jacobi_kernels.cpp  | 69 +++++++++++++++++++++
 3 files changed, 207 insertions(+)

diff --git a/cuda/test/preconditioner/jacobi_kernels.cpp b/cuda/test/preconditioner/jacobi_kernels.cpp
index aafd6546996..808e25047fd 100644
--- a/cuda/test/preconditioner/jacobi_kernels.cpp
+++ b/cuda/test/preconditioner/jacobi_kernels.cpp
@@ -412,6 +412,37 @@ TEST_F(Jacobi, CudaApplyEquivalentToRef)
 }
 
 
+TEST_F(Jacobi, CudaScalarApplyEquivalentToRef)
+{
+    gko::size_type dim = 313;
+    std::ranlux48 engine(42);
+    auto dense_smtx = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, dim, std::uniform_int_distribution<>(1, dim),
+        std::normal_distribution<>(1.0, 2.0), engine, ref));
+    gko::test::make_diag_dominant(dense_smtx.get());
+    auto smtx = gko::share(Mtx::create(ref));
+    smtx->copy_from(dense_smtx.get());
+    auto sb = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, 3, std::uniform_int_distribution<>(1, 1),
+        std::normal_distribution<>(0.0, 1.0), engine, ref));
+    auto sx = Vec::create(ref, sb->get_size());
+
+    auto d_smtx = gko::share(Mtx::create(cuda));
+    auto d_sb = gko::share(Vec::create(cuda));
+    auto d_sx = gko::share(Vec::create(cuda, sb->get_size()));
+    d_smtx->copy_from(smtx.get());
+    d_sb->copy_from(sb.get());
+
+    auto sj = Bj::build().with_max_block_size(1u).on(ref)->generate(smtx);
+    auto d_sj = Bj::build().with_max_block_size(1u).on(cuda)->generate(d_smtx);
+
+    sj->apply(sb.get(), sx.get());
+    d_sj->apply(d_sb.get(), d_sx.get());
+
+    GKO_ASSERT_MTX_NEAR(sx.get(), d_sx.get(), 1e-12);
+}
+
+
 TEST_F(Jacobi, CudaLinearCombinationApplyEquivalentToRef)
 {
     initialize_data({0, 11, 24, 33, 45, 55, 67, 70, 80, 92, 100}, {}, {}, 13,
@@ -430,6 +461,44 @@ TEST_F(Jacobi, CudaLinearCombinationApplyEquivalentToRef)
 }
 
 
+TEST_F(Jacobi, CudaScalarLinearCombinationApplyEquivalentToRef)
+{
+    gko::size_type dim = 313;
+    std::ranlux48 engine(42);
+    auto dense_smtx = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, dim, std::uniform_int_distribution<>(1, dim),
+        std::normal_distribution<>(1.0, 2.0), engine, ref));
+    gko::test::make_diag_dominant(dense_smtx.get());
+    auto smtx = gko::share(Mtx::create(ref));
+    smtx->copy_from(dense_smtx.get());
+    auto sb = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, 3, std::uniform_int_distribution<>(1, 1),
+        std::normal_distribution<>(0.0, 1.0), engine, ref));
+    auto sx = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, 3, std::uniform_int_distribution<>(1, 1),
+        std::normal_distribution<>(0.0, 1.0), engine, ref));
+
+    auto d_smtx = gko::share(Mtx::create(cuda));
+    auto d_sb = gko::share(Vec::create(cuda));
+    auto d_sx = gko::share(Vec::create(cuda));
+    d_smtx->copy_from(smtx.get());
+    d_sb->copy_from(sb.get());
+    d_sx->copy_from(sx.get());
+    auto alpha = gko::initialize<Vec>({2.0}, ref);
+    auto d_alpha = gko::initialize<Vec>({2.0}, cuda);
+    auto beta = gko::initialize<Vec>({-1.0}, ref);
+    auto d_beta = gko::initialize<Vec>({-1.0}, cuda);
+
+    auto sj = Bj::build().with_max_block_size(1u).on(ref)->generate(smtx);
+    auto d_sj = Bj::build().with_max_block_size(1u).on(cuda)->generate(d_smtx);
+
+    sj->apply(alpha.get(), sb.get(), beta.get(), sx.get());
+    d_sj->apply(d_alpha.get(), d_sb.get(), d_beta.get(), d_sx.get());
+
+    GKO_ASSERT_MTX_NEAR(sx.get(), d_sx.get(), 1e-12);
+}
+
+
 TEST_F(Jacobi, CudaApplyToMultipleVectorsEquivalentToRef)
 {
     initialize_data({0, 11, 24, 33, 45, 55, 67, 70, 80, 92, 100}, {}, {}, 13,
diff --git a/hip/test/preconditioner/jacobi_kernels.cpp b/hip/test/preconditioner/jacobi_kernels.cpp
index 3931f666f12..582abc51cde 100644
--- a/hip/test/preconditioner/jacobi_kernels.cpp
+++ b/hip/test/preconditioner/jacobi_kernels.cpp
@@ -443,6 +443,37 @@ TEST_F(Jacobi, HipApplyEquivalentToRef)
 }
 
 
+TEST_F(Jacobi, HipScalarApplyEquivalentToRef)
+{
+    gko::size_type dim = 313;
+    std::ranlux48 engine(42);
+    auto dense_smtx = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, dim, std::uniform_int_distribution<>(1, dim),
+        std::normal_distribution<>(1.0, 2.0), engine, ref));
+    gko::test::make_diag_dominant(dense_smtx.get());
+    auto smtx = gko::share(Mtx::create(ref));
+    smtx->copy_from(dense_smtx.get());
+    auto sb = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, 3, std::uniform_int_distribution<>(1, 1),
+        std::normal_distribution<>(0.0, 1.0), engine, ref));
+    auto sx = Vec::create(ref, sb->get_size());
+
+    auto d_smtx = gko::share(Mtx::create(hip));
+    auto d_sb = gko::share(Vec::create(hip));
+    auto d_sx = gko::share(Vec::create(hip, sb->get_size()));
+    d_smtx->copy_from(smtx.get());
+    d_sb->copy_from(sb.get());
+
+    auto sj = Bj::build().with_max_block_size(1u).on(ref)->generate(smtx);
+    auto d_sj = Bj::build().with_max_block_size(1u).on(hip)->generate(d_smtx);
+
+    sj->apply(sb.get(), sx.get());
+    d_sj->apply(d_sb.get(), d_sx.get());
+
+    GKO_ASSERT_MTX_NEAR(sx.get(), d_sx.get(), 1e-12);
+}
+
+
 TEST_F(Jacobi, HipLinearCombinationApplyEquivalentToRef)
 {
     initialize_data({0, 11, 24, 33, 45, 55, 67, 70, 80, 92, 100}, {}, {}, 13,
@@ -461,6 +492,44 @@ TEST_F(Jacobi, HipLinearCombinationApplyEquivalentToRef)
 }
 
 
+TEST_F(Jacobi, HipScalarLinearCombinationApplyEquivalentToRef)
+{
+    gko::size_type dim = 313;
+    std::ranlux48 engine(42);
+    auto dense_smtx = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, dim, std::uniform_int_distribution<>(1, dim),
+        std::normal_distribution<>(1.0, 2.0), engine, ref));
+    gko::test::make_diag_dominant(dense_smtx.get());
+    auto smtx = gko::share(Mtx::create(ref));
+    smtx->copy_from(dense_smtx.get());
+    auto sb = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, 3, std::uniform_int_distribution<>(1, 1),
+        std::normal_distribution<>(0.0, 1.0), engine, ref));
+    auto sx = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, 3, std::uniform_int_distribution<>(1, 1),
+        std::normal_distribution<>(0.0, 1.0), engine, ref));
+
+    auto d_smtx = gko::share(Mtx::create(hip));
+    auto d_sb = gko::share(Vec::create(hip));
+    auto d_sx = gko::share(Vec::create(hip));
+    d_smtx->copy_from(smtx.get());
+    d_sb->copy_from(sb.get());
+    d_sx->copy_from(sx.get());
+    auto alpha = gko::initialize<Vec>({2.0}, ref);
+    auto d_alpha = gko::initialize<Vec>({2.0}, hip);
+    auto beta = gko::initialize<Vec>({-1.0}, ref);
+    auto d_beta = gko::initialize<Vec>({-1.0}, hip);
+
+    auto sj = Bj::build().with_max_block_size(1u).on(ref)->generate(smtx);
+    auto d_sj = Bj::build().with_max_block_size(1u).on(hip)->generate(d_smtx);
+
+    sj->apply(alpha.get(), sb.get(), beta.get(), sx.get());
+    d_sj->apply(d_alpha.get(), d_sb.get(), d_beta.get(), d_sx.get());
+
+    GKO_ASSERT_MTX_NEAR(sx.get(), d_sx.get(), 1e-12);
+}
+
+
 TEST_F(Jacobi, HipApplyToMultipleVectorsEquivalentToRef)
 {
     initialize_data({0, 11, 24, 33, 45, 55, 67, 70, 80, 92, 100}, {}, {}, 13,
diff --git a/omp/test/preconditioner/jacobi_kernels.cpp b/omp/test/preconditioner/jacobi_kernels.cpp
index 78663913f71..9e8cacacb0c 100644
--- a/omp/test/preconditioner/jacobi_kernels.cpp
+++ b/omp/test/preconditioner/jacobi_kernels.cpp
@@ -397,6 +397,37 @@ TEST_F(Jacobi, OmpApplyEquivalentToRefWithDifferentBlockSize)
 }
 
 
+TEST_F(Jacobi, OmpScalarApplyEquivalentToRef)
+{
+    gko::size_type dim = 313;
+    std::ranlux48 engine(42);
+    auto dense_smtx = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, dim, std::uniform_int_distribution<>(1, dim),
+        std::normal_distribution<>(1.0, 2.0), engine, ref));
+    gko::test::make_diag_dominant(dense_smtx.get());
+    auto smtx = gko::share(Mtx::create(ref));
+    smtx->copy_from(dense_smtx.get());
+    auto sb = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, 3, std::uniform_int_distribution<>(1, 1),
+        std::normal_distribution<>(0.0, 1.0), engine, ref));
+    auto sx = Vec::create(ref, sb->get_size());
+
+    auto d_smtx = gko::share(Mtx::create(omp));
+    auto d_sb = gko::share(Vec::create(omp));
+    auto d_sx = gko::share(Vec::create(omp, sb->get_size()));
+    d_smtx->copy_from(smtx.get());
+    d_sb->copy_from(sb.get());
+
+    auto sj = Bj::build().with_max_block_size(1u).on(ref)->generate(smtx);
+    auto d_sj = Bj::build().with_max_block_size(1u).on(omp)->generate(d_smtx);
+
+    sj->apply(sb.get(), sx.get());
+    d_sj->apply(d_sb.get(), d_sx.get());
+
+    GKO_ASSERT_MTX_NEAR(sx.get(), d_sx.get(), 1e-12);
+}
+
+
 TEST_F(Jacobi, OmpApplyEquivalentToRef)
 {
     initialize_data({0, 11, 24, 33, 45, 55, 67, 70, 80, 92, 100}, {}, {}, 13,
@@ -429,6 +460,44 @@ TEST_F(Jacobi, OmpLinearCombinationApplyEquivalentToRef)
 }
 
 
+TEST_F(Jacobi, OmpScalarLinearCombinationApplyEquivalentToRef)
+{
+    gko::size_type dim = 5;
+    std::ranlux48 engine(42);
+    auto dense_smtx = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, dim, std::uniform_int_distribution<>(1, dim),
+        std::normal_distribution<>(1.0, 2.0), engine, ref));
+    gko::test::make_diag_dominant(dense_smtx.get());
+    auto smtx = gko::share(Mtx::create(ref));
+    smtx->copy_from(dense_smtx.get());
+    auto sb = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, 3, std::uniform_int_distribution<>(1, 1),
+        std::normal_distribution<>(0.0, 1.0), engine, ref));
+    auto sx = gko::share(gko::test::generate_random_matrix<Vec>(
+        dim, 3, std::uniform_int_distribution<>(1, 1),
+        std::normal_distribution<>(0.0, 1.0), engine, ref));
+
+    auto d_smtx = gko::share(Mtx::create(omp));
+    auto d_sb = gko::share(Vec::create(omp));
+    auto d_sx = gko::share(Vec::create(omp));
+    d_smtx->copy_from(smtx.get());
+    d_sb->copy_from(sb.get());
+    d_sx->copy_from(sx.get());
+    auto alpha = gko::initialize<Vec>({2.0}, ref);
+    auto d_alpha = gko::initialize<Vec>({2.0}, omp);
+    auto beta = gko::initialize<Vec>({-1.0}, ref);
+    auto d_beta = gko::initialize<Vec>({-1.0}, omp);
+
+    auto sj = Bj::build().with_max_block_size(1u).on(ref)->generate(smtx);
+    auto d_sj = Bj::build().with_max_block_size(1u).on(omp)->generate(d_smtx);
+
+    sj->apply(alpha.get(), sb.get(), beta.get(), sx.get());
+    d_sj->apply(d_alpha.get(), d_sb.get(), d_beta.get(), d_sx.get());
+
+    GKO_ASSERT_MTX_NEAR(sx.get(), d_sx.get(), 1e-12);
+}
+
+
 TEST_F(Jacobi, OmpApplyToMultipleVectorsEquivalentToRef)
 {
     initialize_data({0, 11, 24, 33, 45, 55, 67, 70, 80, 92, 100}, {}, {}, 13,

From b9b95ccdb5fa0f5684ad0acd5f1afbd286215f95 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 1 Jul 2021 13:01:13 +0200
Subject: [PATCH 0796/1008] Review update: s/Diagonal matrix/blocks array

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 common/preconditioner/jacobi_kernels.hpp.inc  |  8 ++++++--
 core/preconditioner/jacobi.cpp                | 14 ++++++++------
 core/preconditioner/jacobi_kernels.hpp        | 15 +++++++--------
 cuda/preconditioner/jacobi_kernels.cu         |  8 ++++----
 cuda/test/preconditioner/jacobi_kernels.cpp   |  6 ++++--
 dpcpp/preconditioner/jacobi_kernels.dp.cpp    |  4 ++--
 hip/preconditioner/jacobi_kernels.hip.cpp     |  8 ++++----
 hip/test/preconditioner/jacobi_kernels.cpp    |  6 ++++--
 include/ginkgo/core/preconditioner/jacobi.hpp |  7 ++-----
 omp/preconditioner/jacobi_kernels.cpp         | 17 +++++++++++------
 omp/test/preconditioner/jacobi_kernels.cpp    |  6 ++++--
 reference/preconditioner/jacobi_kernels.cpp   | 17 +++++++++++------
 12 files changed, 67 insertions(+), 49 deletions(-)

diff --git a/common/preconditioner/jacobi_kernels.hpp.inc b/common/preconditioner/jacobi_kernels.hpp.inc
index d4d2c69e326..61678280d56 100644
--- a/common/preconditioner/jacobi_kernels.hpp.inc
+++ b/common/preconditioner/jacobi_kernels.hpp.inc
@@ -231,9 +231,11 @@ __global__ __launch_bounds__(default_block_size) void scalar_apply(
     const auto col = tidx % num_cols;
 
     if (row < num_rows) {
+        auto diag_val =
+            diag[row] == zero<ValueType>() ? one<ValueType>() : diag[row];
         result_values[row * result_stride + col] =
             result_values[row * result_stride + col] * beta[0] +
-            alpha[0] * source_values[row * source_stride + col] / diag[row];
+            alpha[0] * source_values[row * source_stride + col] / diag_val;
     }
 }
 
@@ -249,8 +251,10 @@ __global__ __launch_bounds__(default_block_size) void simple_scalar_apply(
     const auto col = tidx % num_cols;
 
     if (row < num_rows) {
+        auto diag_val =
+            diag[row] == zero<ValueType>() ? one<ValueType>() : diag[row];
         result_values[row * result_stride + col] =
-            source_values[row * source_stride + col] / diag[row];
+            source_values[row * source_stride + col] / diag_val;
     }
 }
 
diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index 78f3ffb586e..8270265c2ec 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -78,7 +78,7 @@ void Jacobi<ValueType, IndexType>::apply_impl(const LinOp *b, LinOp *x) const
         [this](auto dense_b, auto dense_x) {
             if (parameters_.max_block_size == 1) {
                 this->get_executor()->run(jacobi::make_simple_scalar_apply(
-                    this->diag_.get(), dense_b, dense_x));
+                    this->blocks_, dense_b, dense_x));
             } else {
                 this->get_executor()->run(jacobi::make_simple_apply(
                     num_blocks_, parameters_.max_block_size, storage_scheme_,
@@ -98,9 +98,8 @@ void Jacobi<ValueType, IndexType>::apply_impl(const LinOp *alpha,
     precision_dispatch_real_complex<ValueType>(
         [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) {
             if (parameters_.max_block_size == 1) {
-                this->get_executor()->run(
-                    jacobi::make_scalar_apply(this->diag_.get(), dense_alpha,
-                                              dense_b, dense_beta, dense_x));
+                this->get_executor()->run(jacobi::make_scalar_apply(
+                    this->blocks_, dense_alpha, dense_b, dense_beta, dense_x));
             } else {
                 this->get_executor()->run(jacobi::make_apply(
                     num_blocks_, parameters_.max_block_size, storage_scheme_,
@@ -232,8 +231,11 @@ void Jacobi<ValueType, IndexType>::generate(const LinOp *system_matrix,
     using csr_type = matrix::Csr<ValueType, IndexType>;
     const auto exec = this->get_executor();
     if (parameters_.max_block_size == 1) {
-        this->diag_ = as<DiagonalExtractable<ValueType>>(system_matrix)
-                          ->extract_diagonal();
+        auto diag = as<DiagonalExtractable<ValueType>>(system_matrix)
+                        ->extract_diagonal();
+        auto temp = gko::Array<ValueType>::view(
+            exec, system_matrix->get_size()[0], diag->get_values());
+        this->blocks_ = temp;
     } else {
         auto csr_mtx = convert_to_with_sorting<csr_type>(exec, system_matrix,
                                                          skip_sorting);
diff --git a/core/preconditioner/jacobi_kernels.hpp b/core/preconditioner/jacobi_kernels.hpp
index 32c5c0d377c..22cfa512bb8 100644
--- a/core/preconditioner/jacobi_kernels.hpp
+++ b/core/preconditioner/jacobi_kernels.hpp
@@ -76,7 +76,7 @@ namespace kernels {
 
 #define GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL(ValueType)          \
     void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec, \
-                             const matrix::Diagonal<ValueType> *diag,     \
+                             const Array<ValueType> &diag,                \
                              const matrix::Dense<ValueType> *b,           \
                              matrix::Dense<ValueType> *x)
 
@@ -91,13 +91,12 @@ namespace kernels {
         const Array<ValueType> &blocks, const matrix::Dense<ValueType> *b, \
         matrix::Dense<ValueType> *x)
 
-#define GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL(ValueType)          \
-    void scalar_apply(std::shared_ptr<const DefaultExecutor> exec, \
-                      const matrix::Diagonal<ValueType> *diag,     \
-                      const matrix::Dense<ValueType> *alpha,       \
-                      const matrix::Dense<ValueType> *b,           \
-                      const matrix::Dense<ValueType> *beta,        \
-                      matrix::Dense<ValueType> *x)
+#define GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL(ValueType)                    \
+    void scalar_apply(                                                       \
+        std::shared_ptr<const DefaultExecutor> exec,                         \
+        const Array<ValueType> &diag, const matrix::Dense<ValueType> *alpha, \
+        const matrix::Dense<ValueType> *b,                                   \
+        const matrix::Dense<ValueType> *beta, matrix::Dense<ValueType> *x)
 
 #define GKO_DECLARE_JACOBI_TRANSPOSE_KERNEL(ValueType, IndexType)          \
     void transpose_jacobi(                                                 \
diff --git a/cuda/preconditioner/jacobi_kernels.cu b/cuda/preconditioner/jacobi_kernels.cu
index fc8e64b00d8..bb9b918f5d9 100644
--- a/cuda/preconditioner/jacobi_kernels.cu
+++ b/cuda/preconditioner/jacobi_kernels.cu
@@ -245,7 +245,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType>
 void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                  const matrix::Diagonal<ValueType> *diag,
+                  const Array<ValueType> &diag,
                   const matrix::Dense<ValueType> *alpha,
                   const matrix::Dense<ValueType> *b,
                   const matrix::Dense<ValueType> *beta,
@@ -259,7 +259,7 @@ void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
     const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
 
     const auto b_values = b->get_const_values();
-    const auto diag_values = diag->get_const_values();
+    const auto diag_values = diag.get_const_data();
     auto x_values = x->get_values();
 
     kernel::scalar_apply<<<grid_dim, default_block_size>>>(
@@ -274,7 +274,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
 
 template <typename ValueType>
 void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                         const matrix::Diagonal<ValueType> *diag,
+                         const Array<ValueType> &diag,
                          const matrix::Dense<ValueType> *b,
                          matrix::Dense<ValueType> *x)
 {
@@ -286,7 +286,7 @@ void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
     const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
 
     const auto b_values = b->get_const_values();
-    const auto diag_values = diag->get_const_values();
+    const auto diag_values = diag.get_const_data();
     auto x_values = x->get_values();
 
     kernel::simple_scalar_apply<<<grid_dim, default_block_size>>>(
diff --git a/cuda/test/preconditioner/jacobi_kernels.cpp b/cuda/test/preconditioner/jacobi_kernels.cpp
index 808e25047fd..87e0cbb2ab9 100644
--- a/cuda/test/preconditioner/jacobi_kernels.cpp
+++ b/cuda/test/preconditioner/jacobi_kernels.cpp
@@ -473,10 +473,12 @@ TEST_F(Jacobi, CudaScalarLinearCombinationApplyEquivalentToRef)
     smtx->copy_from(dense_smtx.get());
     auto sb = gko::share(gko::test::generate_random_matrix<Vec>(
         dim, 3, std::uniform_int_distribution<>(1, 1),
-        std::normal_distribution<>(0.0, 1.0), engine, ref));
+        std::normal_distribution<>(0.0, 1.0), engine, ref, gko::dim<2>(dim, 3),
+        4));
     auto sx = gko::share(gko::test::generate_random_matrix<Vec>(
         dim, 3, std::uniform_int_distribution<>(1, 1),
-        std::normal_distribution<>(0.0, 1.0), engine, ref));
+        std::normal_distribution<>(0.0, 1.0), engine, ref, gko::dim<2>(dim, 3),
+        4));
 
     auto d_smtx = gko::share(Mtx::create(cuda));
     auto d_sb = gko::share(Vec::create(cuda));
diff --git a/dpcpp/preconditioner/jacobi_kernels.dp.cpp b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
index bad11a76952..3b7f1efb90f 100644
--- a/dpcpp/preconditioner/jacobi_kernels.dp.cpp
+++ b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
@@ -282,7 +282,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType>
 void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                  const matrix::Diagonal<ValueType> *diag,
+                  const Array<ValueType> &diag,
                   const matrix::Dense<ValueType> *alpha,
                   const matrix::Dense<ValueType> *b,
                   const matrix::Dense<ValueType> *beta,
@@ -293,7 +293,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
 
 template <typename ValueType>
 void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                         const matrix::Diagonal<ValueType> *diag,
+                         const Array<ValueType> &diag,
                          const matrix::Dense<ValueType> *b,
                          matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
 
diff --git a/hip/preconditioner/jacobi_kernels.hip.cpp b/hip/preconditioner/jacobi_kernels.hip.cpp
index 4cd7dc01040..57d513774b5 100644
--- a/hip/preconditioner/jacobi_kernels.hip.cpp
+++ b/hip/preconditioner/jacobi_kernels.hip.cpp
@@ -259,7 +259,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType>
 void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                  const matrix::Diagonal<ValueType> *diag,
+                  const Array<ValueType> &diag,
                   const matrix::Dense<ValueType> *alpha,
                   const matrix::Dense<ValueType> *b,
                   const matrix::Dense<ValueType> *beta,
@@ -273,7 +273,7 @@ void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
     const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
 
     const auto b_values = b->get_const_values();
-    const auto diag_values = diag->get_const_values();
+    const auto diag_values = diag.get_const_data();
     auto x_values = x->get_values();
 
     hipLaunchKernelGGL(
@@ -288,7 +288,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
 
 template <typename ValueType>
 void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                         const matrix::Diagonal<ValueType> *diag,
+                         const Array<ValueType> &diag,
                          const matrix::Dense<ValueType> *b,
                          matrix::Dense<ValueType> *x)
 {
@@ -300,7 +300,7 @@ void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
     const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
 
     const auto b_values = b->get_const_values();
-    const auto diag_values = diag->get_const_values();
+    const auto diag_values = diag.get_const_data();
     auto x_values = x->get_values();
 
     hipLaunchKernelGGL(kernel::simple_scalar_apply, dim3(grid_dim),
diff --git a/hip/test/preconditioner/jacobi_kernels.cpp b/hip/test/preconditioner/jacobi_kernels.cpp
index 582abc51cde..4ef05dc7fff 100644
--- a/hip/test/preconditioner/jacobi_kernels.cpp
+++ b/hip/test/preconditioner/jacobi_kernels.cpp
@@ -504,10 +504,12 @@ TEST_F(Jacobi, HipScalarLinearCombinationApplyEquivalentToRef)
     smtx->copy_from(dense_smtx.get());
     auto sb = gko::share(gko::test::generate_random_matrix<Vec>(
         dim, 3, std::uniform_int_distribution<>(1, 1),
-        std::normal_distribution<>(0.0, 1.0), engine, ref));
+        std::normal_distribution<>(0.0, 1.0), engine, ref, gko::dim<2>(dim, 3),
+        4));
     auto sx = gko::share(gko::test::generate_random_matrix<Vec>(
         dim, 3, std::uniform_int_distribution<>(1, 1),
-        std::normal_distribution<>(0.0, 1.0), engine, ref));
+        std::normal_distribution<>(0.0, 1.0), engine, ref, gko::dim<2>(dim, 3),
+        4));
 
     auto d_smtx = gko::share(Mtx::create(hip));
     auto d_sb = gko::share(Vec::create(hip));
diff --git a/include/ginkgo/core/preconditioner/jacobi.hpp b/include/ginkgo/core/preconditioner/jacobi.hpp
index 5d40d2ad169..dd05296abde 100644
--- a/include/ginkgo/core/preconditioner/jacobi.hpp
+++ b/include/ginkgo/core/preconditioner/jacobi.hpp
@@ -502,8 +502,7 @@ class Jacobi : public EnableLinOp<Jacobi<ValueType, IndexType>>,
         : EnableLinOp<Jacobi>(exec),
           num_blocks_{},
           blocks_(exec),
-          conditioning_(exec),
-          diag_(matrix::Diagonal<value_type>::create(exec))
+          conditioning_(exec)
     {
         parameters_.block_pointers.set_executor(exec);
         parameters_.storage_optimization.block_wise.set_executor(exec);
@@ -527,8 +526,7 @@ class Jacobi : public EnableLinOp<Jacobi<ValueType, IndexType>>,
           blocks_(factory->get_executor(),
                   storage_scheme_.compute_storage_space(
                       parameters_.block_pointers.get_num_elems() - 1)),
-          conditioning_(factory->get_executor()),
-          diag_(matrix::Diagonal<value_type>::create(factory->get_executor()))
+          conditioning_(factory->get_executor())
     {
         parameters_.block_pointers.set_executor(this->get_executor());
         parameters_.storage_optimization.block_wise.set_executor(
@@ -608,7 +606,6 @@ class Jacobi : public EnableLinOp<Jacobi<ValueType, IndexType>>,
     size_type num_blocks_;
     Array<value_type> blocks_;
     Array<remove_complex<value_type>> conditioning_;
-    std::shared_ptr<matrix::Diagonal<value_type>> diag_;
 };
 
 
diff --git a/omp/preconditioner/jacobi_kernels.cpp b/omp/preconditioner/jacobi_kernels.cpp
index 163b90ab5b1..605ba433578 100644
--- a/omp/preconditioner/jacobi_kernels.cpp
+++ b/omp/preconditioner/jacobi_kernels.cpp
@@ -698,7 +698,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType>
 void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                  const matrix::Diagonal<ValueType> *diag,
+                  const Array<ValueType> &diag,
                   const matrix::Dense<ValueType> *alpha,
                   const matrix::Dense<ValueType> *b,
                   const matrix::Dense<ValueType> *beta,
@@ -707,9 +707,11 @@ void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
 #pragma omp parallel for
     for (size_type i = 0; i < x->get_size()[0]; ++i) {
         for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            x->at(i, j) =
-                beta->at(0) * x->at(i, j) +
-                alpha->at(0) * b->at(i, j) / diag->get_const_values()[i];
+            auto diag_val = diag.get_const_data()[i] == zero<ValueType>()
+                                ? one<ValueType>()
+                                : diag.get_const_data()[i];
+            x->at(i, j) = beta->at(0) * x->at(i, j) +
+                          alpha->at(0) * b->at(i, j) / diag_val;
         }
     }
 }
@@ -719,14 +721,17 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
 
 template <typename ValueType>
 void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                         const matrix::Diagonal<ValueType> *diag,
+                         const Array<ValueType> &diag,
                          const matrix::Dense<ValueType> *b,
                          matrix::Dense<ValueType> *x)
 {
 #pragma omp parallel for
     for (size_type i = 0; i < x->get_size()[0]; ++i) {
         for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            x->at(i, j) = b->at(i, j) / diag->get_const_values()[i];
+            auto diag_val = diag.get_const_data()[i] == zero<ValueType>()
+                                ? one<ValueType>()
+                                : diag.get_const_data()[i];
+            x->at(i, j) = b->at(i, j) / diag_val;
         }
     }
 }
diff --git a/omp/test/preconditioner/jacobi_kernels.cpp b/omp/test/preconditioner/jacobi_kernels.cpp
index 9e8cacacb0c..095729a5e9a 100644
--- a/omp/test/preconditioner/jacobi_kernels.cpp
+++ b/omp/test/preconditioner/jacobi_kernels.cpp
@@ -472,10 +472,12 @@ TEST_F(Jacobi, OmpScalarLinearCombinationApplyEquivalentToRef)
     smtx->copy_from(dense_smtx.get());
     auto sb = gko::share(gko::test::generate_random_matrix<Vec>(
         dim, 3, std::uniform_int_distribution<>(1, 1),
-        std::normal_distribution<>(0.0, 1.0), engine, ref));
+        std::normal_distribution<>(0.0, 1.0), engine, ref, gko::dim<2>(dim, 3),
+        4));
     auto sx = gko::share(gko::test::generate_random_matrix<Vec>(
         dim, 3, std::uniform_int_distribution<>(1, 1),
-        std::normal_distribution<>(0.0, 1.0), engine, ref));
+        std::normal_distribution<>(0.0, 1.0), engine, ref, gko::dim<2>(dim, 3),
+        4));
 
     auto d_smtx = gko::share(Mtx::create(omp));
     auto d_sb = gko::share(Vec::create(omp));
diff --git a/reference/preconditioner/jacobi_kernels.cpp b/reference/preconditioner/jacobi_kernels.cpp
index 11fa2cfc9e6..a176b01e4bf 100644
--- a/reference/preconditioner/jacobi_kernels.cpp
+++ b/reference/preconditioner/jacobi_kernels.cpp
@@ -562,7 +562,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType>
 void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                  const matrix::Diagonal<ValueType> *diag,
+                  const Array<ValueType> &diag,
                   const matrix::Dense<ValueType> *alpha,
                   const matrix::Dense<ValueType> *b,
                   const matrix::Dense<ValueType> *beta,
@@ -570,9 +570,11 @@ void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
 {
     for (size_type i = 0; i < x->get_size()[0]; ++i) {
         for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            x->at(i, j) =
-                beta->at(0) * x->at(i, j) +
-                alpha->at(0) * b->at(i, j) / diag->get_const_values()[i];
+            auto diag_val = diag.get_const_data()[i] == zero<ValueType>()
+                                ? one<ValueType>()
+                                : diag.get_const_data()[i];
+            x->at(i, j) = beta->at(0) * x->at(i, j) +
+                          alpha->at(0) * b->at(i, j) / diag_val;
         }
     }
 }
@@ -582,13 +584,16 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
 
 template <typename ValueType>
 void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                         const matrix::Diagonal<ValueType> *diag,
+                         const Array<ValueType> &diag,
                          const matrix::Dense<ValueType> *b,
                          matrix::Dense<ValueType> *x)
 {
     for (size_type i = 0; i < x->get_size()[0]; ++i) {
         for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            x->at(i, j) = b->at(i, j) / diag->get_const_values()[i];
+            auto diag_val = diag.get_const_data()[i] == zero<ValueType>()
+                                ? one<ValueType>()
+                                : diag.get_const_data()[i];
+            x->at(i, j) = b->at(i, j) / diag_val;
         }
     }
 }

From 8adf357d78d12c36c1bf6dcd4d1f02f61c661a32 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 2 Jul 2021 21:36:15 +0200
Subject: [PATCH 0797/1008] avoid benchmarking ELL format with heavy imbalance

---
 benchmark/conversions/conversions.cpp |  3 +--
 benchmark/run_all_benchmarks.sh       | 24 +++++++++++++++++++++---
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index fd9481189b0..a8165983377 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -156,8 +156,7 @@ int main(int argc, char *argv[])
             try {
                 auto matrix_from =
                     share(formats::matrix_factory.at(format_from)(exec, data));
-                for (const auto &format : formats::matrix_factory) {
-                    const auto format_to = std::get<0>(format);
+                for (const auto &format_to : formats) {
                     if (format_from == format_to) {
                         continue;
                     }
diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index bd481169057..1a0b8a76ce6 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -35,6 +35,11 @@ if [ ! "${FORMATS}" ]; then
     FORMATS="csr,coo,ell,hybrid,sellp"
 fi
 
+if [ ! "${ELL_IMBALANCE_LIMIT}" ]; then
+    echo "ELL_IMBALANCE_LIMIT    environment variable not set - assuming 100" 1>&2
+    ELL_IMBALANCE_LIMIT=100
+fi
+
 if [ ! "${SOLVERS}" ]; then
     SOLVERS="bicgstab,cg,cgs,fcg,gmres,cb_gmres_reduce1,idr"
     echo "SOLVERS    environment variable not set - assuming \"${SOLVERS}\"" 1>&2
@@ -67,7 +72,7 @@ fi
 
 if [ ! "${SOLVERS_JACOBI_MAX_BS}" ]; then
     SOLVERS_JACOBI_MAX_BS="32"
-    "SOLVERS_JACOBI_MAX_BS environment variable not set - assuming \"${SOLVERS_JACOBI_MAX_BS}\"" 1>&2
+    echo "SOLVERS_JACOBI_MAX_BS environment variable not set - assuming \"${SOLVERS_JACOBI_MAX_BS}\"" 1>&2
 fi
 
 if [ ! "${BENCHMARK_PRECISION}" ]; then
@@ -191,6 +196,17 @@ compute_matrix_statistics() {
 }
 
 
+remove_ell_worstcase() {
+    local IMBALANCE=$(jq '.[0].problem.row_distribution | .max / (.median + 1) | floor' $1)
+    # if the imbalance is too large, remove ELL formats from the list.
+    if [[ "${IMBALANCE}" -gt "${ELL_IMBALANCE_LIMIT}" ]]; then
+        echo -n $FORMATS | tr ',' '\n' | grep -vE '^ell' | tr '\n' ',' | sed 's/,$//'
+    else
+        echo -n $FORMATS
+    fi
+}
+
+
 # Runs the conversion benchmarks for all matrix formats by using file $1 as the
 # input, and updating it with the results. Backups are created after each
 # benchmark run, to prevent data loss in case of a crash. Once the benchmarking
@@ -199,8 +215,9 @@ compute_matrix_statistics() {
 run_conversion_benchmarks() {
     [ "${DRY_RUN}" == "true" ] && return
     cp "$1" "$1.imd" # make sure we're not loosing the original input
+    local LOCAL_FORMATS=$(remove_ell_worstcase "$1")
     ./conversions/conversions${BENCH_SUFFIX} --backup="$1.bkp" --double_buffer="$1.bkp2" \
-                --executor="${EXECUTOR}" --formats="${FORMATS}" \
+                --executor="${EXECUTOR}" --formats="${LOCAL_FORMATS}" \
                 --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
                 <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
@@ -214,9 +231,10 @@ run_conversion_benchmarks() {
 # taken as the final result.
 run_spmv_benchmarks() {
     [ "${DRY_RUN}" == "true" ] && return
+    local LOCAL_FORMATS=$(remove_ell_worstcase "$1")
     cp "$1" "$1.imd" # make sure we're not loosing the original input
     ./spmv/spmv${BENCH_SUFFIX} --backup="$1.bkp" --double_buffer="$1.bkp2" \
-                --executor="${EXECUTOR}" --formats="${FORMATS}" \
+                --executor="${EXECUTOR}" --formats="${LOCAL_FORMATS}" \
                 --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
                 <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"

From 53cc179164fea5feba03120931a36301727eb10e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 3 Jul 2021 20:22:47 +0200
Subject: [PATCH 0798/1008] use better ELL benchmark exclusion criterion

This actually measures the storage overhead
---
 benchmark/run_all_benchmarks.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index 1a0b8a76ce6..eb94b6c7189 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -197,7 +197,7 @@ compute_matrix_statistics() {
 
 
 remove_ell_worstcase() {
-    local IMBALANCE=$(jq '.[0].problem.row_distribution | .max / (.median + 1) | floor' $1)
+    local IMBALANCE=$(jq '.[0].problem.row_distribution | .max / (.mean + 1) | floor' $1)
     # if the imbalance is too large, remove ELL formats from the list.
     if [[ "${IMBALANCE}" -gt "${ELL_IMBALANCE_LIMIT}" ]]; then
         echo -n $FORMATS | tr ',' '\n' | grep -vE '^ell' | tr '\n' ',' | sed 's/,$//'

From 17cb315971d0fd39dfdebff081e81ea4604b3ac6 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sun, 4 Jul 2021 11:40:10 +0200
Subject: [PATCH 0799/1008] fix OpenMP Coo/Hybrid -> Csr conversion

empty COO matrices caused issues due to overflowing unsigned subtraction
---
 omp/components/format_conversion.hpp | 22 +++++++++++++++-------
 omp/matrix/coo_kernels.cpp           |  7 +++----
 omp/matrix/hybrid_kernels.cpp        |  3 +--
 omp/test/matrix/hybrid_kernels.cpp   | 27 ++++++++++++++++++++++++++-
 4 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/omp/components/format_conversion.hpp b/omp/components/format_conversion.hpp
index 706c686ac83..6424f333621 100644
--- a/omp/components/format_conversion.hpp
+++ b/omp/components/format_conversion.hpp
@@ -84,16 +84,24 @@ inline void convert_unsorted_idxs_to_ptrs(const IndexType *idxs,
 template <typename IndexType>
 inline void convert_sorted_idxs_to_ptrs(const IndexType *idxs,
                                         size_type num_nonzeros, IndexType *ptrs,
-                                        size_type length)
+                                        size_type num_rows)
 {
     ptrs[0] = 0;
-    ptrs[length - 1] = num_nonzeros;
 
-#pragma omp parallel for schedule( \
-    static, ceildiv(num_nonzeros, omp_get_max_threads()))
-    for (size_type i = 0; i < num_nonzeros - 1; i++) {
-        for (size_type j = idxs[i] + 1; j <= idxs[i + 1]; j++) {
-            ptrs[j] = i + 1;
+    if (num_nonzeros == 0) {
+#pragma omp parallel for
+        for (size_type row = 0; row < num_rows; row++) {
+            ptrs[row + 1] = 0;
+        }
+    } else {
+#pragma omp parallel for
+        for (size_type i = 0; i < num_nonzeros; i++) {
+            const auto end_row = i < num_nonzeros - 1
+                                     ? idxs[i + 1]
+                                     : static_cast<IndexType>(num_rows);
+            for (size_type row = idxs[i]; row < end_row; row++) {
+                ptrs[row + 1] = i + 1;
+            }
         }
     }
 }
diff --git a/omp/matrix/coo_kernels.cpp b/omp/matrix/coo_kernels.cpp
index 71f5283104c..1f9055dc129 100644
--- a/omp/matrix/coo_kernels.cpp
+++ b/omp/matrix/coo_kernels.cpp
@@ -204,9 +204,9 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename IndexType>
 void convert_row_idxs_to_ptrs(std::shared_ptr<const OmpExecutor> exec,
                               const IndexType *idxs, size_type num_nonzeros,
-                              IndexType *ptrs, size_type length)
+                              IndexType *ptrs, size_type num_rows)
 {
-    convert_sorted_idxs_to_ptrs(idxs, num_nonzeros, ptrs, length);
+    convert_sorted_idxs_to_ptrs(idxs, num_nonzeros, ptrs, num_rows);
 }
 
 
@@ -222,8 +222,7 @@ void convert_to_csr(std::shared_ptr<const OmpExecutor> exec,
 
     const auto source_row_idxs = source->get_const_row_idxs();
 
-    convert_row_idxs_to_ptrs(exec, source_row_idxs, nnz, row_ptrs,
-                             num_rows + 1);
+    convert_row_idxs_to_ptrs(exec, source_row_idxs, nnz, row_ptrs, num_rows);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
diff --git a/omp/matrix/hybrid_kernels.cpp b/omp/matrix/hybrid_kernels.cpp
index 204da570121..135703c6708 100644
--- a/omp/matrix/hybrid_kernels.cpp
+++ b/omp/matrix/hybrid_kernels.cpp
@@ -114,8 +114,7 @@ void convert_to_csr(std::shared_ptr<const OmpExecutor> exec,
     const auto num_rows = source->get_size()[0];
     auto coo_row_ptrs = Array<IndexType>(exec, num_rows + 1);
     auto coo_row_ptrs_val = coo_row_ptrs.get_data();
-    convert_sorted_idxs_to_ptrs(coo_row, coo_nnz, coo_row_ptrs_val,
-                                num_rows + 1);
+    convert_sorted_idxs_to_ptrs(coo_row, coo_nnz, coo_row_ptrs_val, num_rows);
 
     // Compute the row offset of Coo without zeros
     auto coo_offset = Array<IndexType>(exec, num_rows);
diff --git a/omp/test/matrix/hybrid_kernels.cpp b/omp/test/matrix/hybrid_kernels.cpp
index 39f56d8442c..ad8950d8d24 100644
--- a/omp/test/matrix/hybrid_kernels.cpp
+++ b/omp/test/matrix/hybrid_kernels.cpp
@@ -77,10 +77,17 @@ class Hybrid : public ::testing::Test {
     template <typename MtxType = Vec>
     std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
                                      int min_nnz_row)
+    {
+        return gen_mtx<MtxType>(num_rows, num_cols, min_nnz_row, num_cols);
+    }
+
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row, int max_nnz_row)
     {
         return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols,
-            std::uniform_int_distribution<>(min_nnz_row, num_cols),
+            std::uniform_int_distribution<>(min_nnz_row, max_nnz_row),
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
     }
 
@@ -230,6 +237,24 @@ TEST_F(Hybrid, CountNonzerosIsEquivalentToRef)
 }
 
 
+TEST_F(Hybrid, ConvertEmptyCooToCsrIsEquivalentToRef)
+{
+    auto balanced_mtx =
+        Mtx::create(ref, std::make_shared<Mtx::column_limit>(4));
+    balanced_mtx->copy_from(gen_mtx(400, 200, 4, 4).get());
+    auto dbalanced_mtx =
+        Mtx::create(omp, std::make_shared<Mtx::column_limit>(4));
+    dbalanced_mtx->copy_from(balanced_mtx.get());
+    auto csr_mtx = gko::matrix::Csr<>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<>::create(omp);
+
+    balanced_mtx->convert_to(csr_mtx.get());
+    dbalanced_mtx->convert_to(dcsr_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), 1e-14);
+}
+
+
 TEST_F(Hybrid, ConvertToCsrIsEquivalentToRef)
 {
     set_up_apply_data(1, std::make_shared<Mtx::column_limit>(2));

From fd7a3c2a41082fbe45764dada6513b29b27eec02 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sun, 4 Jul 2021 12:28:08 +0200
Subject: [PATCH 0800/1008] fix conversion benchmark repetition count

---
 benchmark/conversions/conversions.cpp | 4 ++--
 benchmark/spmv/spmv.cpp               | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index a8165983377..9e48f6ca327 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -88,9 +88,9 @@ void convert_matrix(const gko::LinOp *matrix_from, const char *format_to,
             matrix_to->copy_from(matrix_from);
         }
         add_or_set_member(conversion_case[conversion_name], "time",
-                          timer->compute_average_time(), allocator);
+                          ic.compute_average_time(), allocator);
         add_or_set_member(conversion_case[conversion_name], "repetitions",
-                          timer->get_num_repetitions(), allocator);
+                          ic.get_num_repetitions(), allocator);
 
         // compute and write benchmark data
         add_or_set_member(conversion_case[conversion_name], "completed", true,
diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp
index 2bf07e8ad45..4617bb4d7e6 100644
--- a/benchmark/spmv/spmv.cpp
+++ b/benchmark/spmv/spmv.cpp
@@ -130,7 +130,7 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
             for (auto _ : ic_tuning.run()) {
                 system_matrix->apply(lend(b), lend(x_clone));
             }
-            tuning_case["time"].PushBack(tuning_timer->compute_average_time(),
+            tuning_case["time"].PushBack(ic_tuning.compute_average_time(),
                                          allocator);
             tuning_case["values"].PushBack(val, allocator);
         }

From 451e063f98d0d384350fbbf541ab507048e73eda Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sun, 4 Jul 2021 12:35:15 +0200
Subject: [PATCH 0801/1008] add repetitions flag to benchmark script

---
 benchmark/run_all_benchmarks.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index eb94b6c7189..4a377462af7 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -16,6 +16,11 @@ if [ ! "${EXECUTOR}" ]; then
     echo "EXECUTOR    environment variable not set - assuming \"${EXECUTOR}\"" 1>&2
 fi
 
+if [ ! "${REPETITIONS}" ]; then
+    REPETITIONS=10
+    echo "REPETITIONS    environment variable not set - assuming ${REPETITIONS}" 1>&2
+fi
+
 if [ ! "${SEGMENTS}" ]; then
     echo "SEGMENTS    environment variable not set - running entire suite" 1>&2
     SEGMENTS=1
@@ -219,6 +224,7 @@ run_conversion_benchmarks() {
     ./conversions/conversions${BENCH_SUFFIX} --backup="$1.bkp" --double_buffer="$1.bkp2" \
                 --executor="${EXECUTOR}" --formats="${LOCAL_FORMATS}" \
                 --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
+                --repetitions="${REPETITIONS}" \
                 <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }
@@ -236,6 +242,7 @@ run_spmv_benchmarks() {
     ./spmv/spmv${BENCH_SUFFIX} --backup="$1.bkp" --double_buffer="$1.bkp2" \
                 --executor="${EXECUTOR}" --formats="${LOCAL_FORMATS}" \
                 --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
+                --repetitions="${REPETITIONS}" \
                 <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }
@@ -257,6 +264,7 @@ run_solver_benchmarks() {
                     --gpu_timer=${GPU_TIMER} \
                     --jacobi_max_block_size=${SOLVERS_JACOBI_MAX_BS} --device_id="${DEVICE_ID}" \
                     --gmres_restart="${SOLVERS_GMRES_RESTART}" \
+                    --repetitions="${REPETITIONS}" \
                     <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }
@@ -283,6 +291,7 @@ run_preconditioner_benchmarks() {
                 --jacobi_max_block_size="${bsize}" \
                 --jacobi_storage="${prec}" \
                 --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
+                --repetitions="${REPETITIONS}" \
                 <"$1.imd" 2>&1 >"$1"
             keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
         done

From dc9626ec4060f83ecc5f190244d2679ba53d609b Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sun, 4 Jul 2021 20:04:45 +0200
Subject: [PATCH 0802/1008] fix OpenMP Hybrid->Csr conversion

for real this time
---
 omp/components/format_conversion.hpp | 16 ++++---
 omp/matrix/hybrid_kernels.cpp        | 66 ++++++++--------------------
 omp/test/matrix/hybrid_kernels.cpp   | 26 +++++++++++
 3 files changed, 55 insertions(+), 53 deletions(-)

diff --git a/omp/components/format_conversion.hpp b/omp/components/format_conversion.hpp
index 6424f333621..d634541c6f3 100644
--- a/omp/components/format_conversion.hpp
+++ b/omp/components/format_conversion.hpp
@@ -40,6 +40,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
+#include "core/components/prefix_sum.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace omp {
@@ -94,13 +97,14 @@ inline void convert_sorted_idxs_to_ptrs(const IndexType *idxs,
             ptrs[row + 1] = 0;
         }
     } else {
+        // add virtual sentinel values 0 and num_rows to handle empty first and
+        // last rows
 #pragma omp parallel for
-        for (size_type i = 0; i < num_nonzeros; i++) {
-            const auto end_row = i < num_nonzeros - 1
-                                     ? idxs[i + 1]
-                                     : static_cast<IndexType>(num_rows);
-            for (size_type row = idxs[i]; row < end_row; row++) {
-                ptrs[row + 1] = i + 1;
+        for (size_type i = 0; i <= num_nonzeros; i++) {
+            auto begin_row = i == 0 ? size_type{} : idxs[i - 1];
+            auto end_row = i == num_nonzeros ? num_rows : idxs[i];
+            for (auto row = begin_row; row < end_row; row++) {
+                ptrs[row + 1] = i;
             }
         }
     }
diff --git a/omp/matrix/hybrid_kernels.cpp b/omp/matrix/hybrid_kernels.cpp
index 135703c6708..f97bee4a640 100644
--- a/omp/matrix/hybrid_kernels.cpp
+++ b/omp/matrix/hybrid_kernels.cpp
@@ -112,57 +112,30 @@ void convert_to_csr(std::shared_ptr<const OmpExecutor> exec,
     const auto coo_row = source->get_const_coo_row_idxs();
     const auto coo_nnz = source->get_coo_num_stored_elements();
     const auto num_rows = source->get_size()[0];
-    auto coo_row_ptrs = Array<IndexType>(exec, num_rows + 1);
-    auto coo_row_ptrs_val = coo_row_ptrs.get_data();
-    convert_sorted_idxs_to_ptrs(coo_row, coo_nnz, coo_row_ptrs_val, num_rows);
-
-    // Compute the row offset of Coo without zeros
-    auto coo_offset = Array<IndexType>(exec, num_rows);
-    auto coo_offset_val = coo_offset.get_data();
-    for (size_type i = 0; i < num_rows; i++) {
-        IndexType nonzeros = 0;
-#pragma omp parallel for reduction(+ : nonzeros)
-        for (size_type j = coo_row_ptrs_val[i]; j < coo_row_ptrs_val[i + 1];
-             j++) {
-            nonzeros += coo_val[j] != zero<ValueType>();
-        }
-        coo_offset_val[i] = nonzeros;
-    }
-
-    // Compute row pointer of Csr
-    csr_row_ptrs[0] = 0;
-#pragma omp parallel for
-    for (size_type i = 0; i < num_rows; i++) {
-        csr_row_ptrs[i + 1] = coo_offset_val[i];
-    }
-
-    for (size_type col = 0; col < max_nnz_per_row; col++) {
-#pragma omp parallel for
-        for (size_type row = 0; row < num_rows; row++) {
-            csr_row_ptrs[row + 1] +=
-                (ell->val_at(row, col) != zero<ValueType>());
-        }
-    }
+    auto coo_row_ptrs_array = Array<IndexType>(exec, num_rows + 1);
+    auto coo_row_ptrs = coo_row_ptrs_array.get_data();
+    convert_sorted_idxs_to_ptrs(coo_row, coo_nnz, coo_row_ptrs, num_rows);
 
-    auto workspace = Array<IndexType>(exec, num_rows + 1);
-    auto workspace_val = workspace.get_data();
-    for (size_type i = 1; i < num_rows + 1; i <<= 1) {
+    // Compute the row sizes of Coo without zeros
 #pragma omp parallel for
-        for (size_type j = i; j < num_rows + 1; j++) {
-            workspace_val[j] = csr_row_ptrs[j] + csr_row_ptrs[j - i];
+    for (size_type row = 0; row < num_rows; row++) {
+        IndexType nonzeros{};
+        for (auto j = coo_row_ptrs[row]; j < coo_row_ptrs[row + 1]; j++) {
+            nonzeros += coo_val[j] != zero<ValueType>();
         }
-#pragma omp parallel for
-        for (size_type j = i; j < num_rows + 1; j++) {
-            csr_row_ptrs[j] = workspace_val[j];
+        for (size_type col = 0; col < max_nnz_per_row; col++) {
+            nonzeros += (ell->val_at(row, col) != zero<ValueType>());
         }
+        csr_row_ptrs[row] = nonzeros;
     }
 
-// Fill in Csr
+    components::prefix_sum(exec, csr_row_ptrs, num_rows + 1);
+
+    // Fill in Csr
 #pragma omp parallel for
     for (IndexType row = 0; row < num_rows; row++) {
         // Ell part
-        size_type csr_idx = csr_row_ptrs[row];
-        size_type coo_idx = coo_offset_val[row];
+        auto csr_idx = csr_row_ptrs[row];
         for (IndexType col = 0; col < max_nnz_per_row; col++) {
             const auto val = ell->val_at(row, col);
             if (val != zero<ValueType>()) {
@@ -172,11 +145,10 @@ void convert_to_csr(std::shared_ptr<const OmpExecutor> exec,
             }
         }
         // Coo part
-        for (auto coo_idx = coo_row_ptrs_val[row];
-             coo_idx < coo_row_ptrs_val[row + 1]; coo_idx++) {
-            if (coo_val[coo_idx] != zero<ValueType>()) {
-                csr_val[csr_idx] = coo_val[coo_idx];
-                csr_col_idxs[csr_idx] = coo_col[coo_idx];
+        for (auto j = coo_row_ptrs[row]; j < coo_row_ptrs[row + 1]; j++) {
+            if (coo_val[j] != zero<ValueType>()) {
+                csr_val[csr_idx] = coo_val[j];
+                csr_col_idxs[csr_idx] = coo_col[j];
                 csr_idx++;
             }
         }
diff --git a/omp/test/matrix/hybrid_kernels.cpp b/omp/test/matrix/hybrid_kernels.cpp
index ad8950d8d24..2ebe179d15b 100644
--- a/omp/test/matrix/hybrid_kernels.cpp
+++ b/omp/test/matrix/hybrid_kernels.cpp
@@ -55,6 +55,7 @@ namespace {
 
 class Hybrid : public ::testing::Test {
 protected:
+    using value_type = double;
     using Mtx = gko::matrix::Hybrid<>;
     using Vec = gko::matrix::Dense<>;
     using ComplexVec = gko::matrix::Dense<std::complex<double>>;
@@ -255,6 +256,31 @@ TEST_F(Hybrid, ConvertEmptyCooToCsrIsEquivalentToRef)
 }
 
 
+TEST_F(Hybrid, ConvertWithEmptyFirstAndLastRowToCsrIsEquivalentToRef)
+{
+    // create a dense matrix for easier manipulation
+    auto dense_mtx = gen_mtx(400, 200, 0, 4);
+    // set first and last row to zero
+    for (gko::size_type col = 0; col < dense_mtx->get_size()[1]; col++) {
+        dense_mtx->at(0, col) = gko::zero<value_type>();
+        dense_mtx->at(dense_mtx->get_size()[0] - 1, col) =
+            gko::zero<value_type>();
+    }
+    // now convert them to hybrid matrices
+    auto balanced_mtx = Mtx::create(ref);
+    balanced_mtx->copy_from(dense_mtx.get());
+    auto dbalanced_mtx = Mtx::create(omp);
+    dbalanced_mtx->copy_from(balanced_mtx.get());
+    auto csr_mtx = gko::matrix::Csr<>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<>::create(omp);
+
+    balanced_mtx->convert_to(csr_mtx.get());
+    dbalanced_mtx->convert_to(dcsr_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), 1e-14);
+}
+
+
 TEST_F(Hybrid, ConvertToCsrIsEquivalentToRef)
 {
     set_up_apply_data(1, std::make_shared<Mtx::column_limit>(2));

From d1d014ae86c6aff2e41ce002bd99b2532c53a643 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 5 Jul 2021 12:45:14 +0200
Subject: [PATCH 0803/1008] fix benchmarks with BUILD_CUDA on non-GPU systems

The strategies created temporary GPU executors,
which needs to be avoided, as they throw an exception
---
 benchmark/utils/formats.hpp | 42 ++++++++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)

diff --git a/benchmark/utils/formats.hpp b/benchmark/utils/formats.hpp
index 64de4e10f90..597344a2a2b 100644
--- a/benchmark/utils/formats.hpp
+++ b/benchmark/utils/formats.hpp
@@ -181,6 +181,27 @@ std::unique_ptr<MatrixType> read_matrix_from_data(
     return mat;
 }
 
+
+/**
+ * Creates a CSR strategy of the given type for the given executor if possible,
+ * falls back to csr::classical for executors without support for this strategy.
+ *
+ * @tparam Strategy  one of csr::automatical or csr::load_balance
+ */
+template <typename Strategy>
+std::shared_ptr<csr::strategy_type> create_gpu_strategy(
+    std::shared_ptr<const gko::Executor> exec)
+{
+    if (auto cuda = dynamic_cast<const gko::CudaExecutor *>(exec.get())) {
+        return std::make_shared<Strategy>(cuda->shared_from_this());
+    } else if (auto hip = dynamic_cast<const gko::HipExecutor *>(exec.get())) {
+        return std::make_shared<Strategy>(hip->shared_from_this());
+    } else {
+        return std::make_shared<csr::classical>();
+    }
+}
+
+
 /**
  * Creates a Ginkgo matrix from the intermediate data representation format
  * gko::matrix_data with support for variable arguments.
@@ -201,8 +222,22 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
                                 std::shared_ptr<const gko::Executor>,
                                 const gko::matrix_data<etype> &)>>
     matrix_factory{
-        {"csr", READ_MATRIX(csr, std::make_shared<csr::automatical>())},
-        {"csri", READ_MATRIX(csr, std::make_shared<csr::load_balance>())},
+        {"csr",
+         [](std::shared_ptr<const gko::Executor> exec,
+            const gko::matrix_data<etype> &data) -> std::unique_ptr<csr> {
+            auto mat =
+                csr::create(exec, create_gpu_strategy<csr::automatical>(exec));
+            mat->read(data);
+            return mat;
+         }},
+        {"csri",
+         [](std::shared_ptr<const gko::Executor> exec,
+            const gko::matrix_data<etype> &data) -> std::unique_ptr<csr> {
+             auto mat = csr::create(
+                 exec, create_gpu_strategy<csr::load_balance>(exec));
+             mat->read(data);
+             return mat;
+         }},
         {"csrm", READ_MATRIX(csr, std::make_shared<csr::merge_path>())},
         {"csrc", READ_MATRIX(csr, std::make_shared<csr::classical>())},
         {"coo", read_matrix_from_data<gko::matrix::Coo<etype>>},
@@ -220,7 +255,8 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
                  it->value = el.value;
                  ++it;
              }
-             auto mat = gko::matrix::Ell<gko::next_precision<etype>>::create(std::move(exec));
+             auto mat = gko::matrix::Ell<gko::next_precision<etype>>::create(
+                 std::move(exec));
              mat->read(conv_data);
              return mat;
          }},

From 7aaedc1e1fbd44b1e1702a8960f532482e7c9196 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 5 Jul 2021 14:05:43 +0200
Subject: [PATCH 0804/1008] check ELL imbalance limit in benchmarks directly

---
 benchmark/run_all_benchmarks.sh | 19 ++++-------------
 benchmark/utils/formats.hpp     | 38 ++++++++++++++++++++++++++++++++-
 2 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index 4a377462af7..0220496399d 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -201,17 +201,6 @@ compute_matrix_statistics() {
 }
 
 
-remove_ell_worstcase() {
-    local IMBALANCE=$(jq '.[0].problem.row_distribution | .max / (.mean + 1) | floor' $1)
-    # if the imbalance is too large, remove ELL formats from the list.
-    if [[ "${IMBALANCE}" -gt "${ELL_IMBALANCE_LIMIT}" ]]; then
-        echo -n $FORMATS | tr ',' '\n' | grep -vE '^ell' | tr '\n' ',' | sed 's/,$//'
-    else
-        echo -n $FORMATS
-    fi
-}
-
-
 # Runs the conversion benchmarks for all matrix formats by using file $1 as the
 # input, and updating it with the results. Backups are created after each
 # benchmark run, to prevent data loss in case of a crash. Once the benchmarking
@@ -220,11 +209,11 @@ remove_ell_worstcase() {
 run_conversion_benchmarks() {
     [ "${DRY_RUN}" == "true" ] && return
     cp "$1" "$1.imd" # make sure we're not loosing the original input
-    local LOCAL_FORMATS=$(remove_ell_worstcase "$1")
     ./conversions/conversions${BENCH_SUFFIX} --backup="$1.bkp" --double_buffer="$1.bkp2" \
-                --executor="${EXECUTOR}" --formats="${LOCAL_FORMATS}" \
+                --executor="${EXECUTOR}" --formats="${FORMATS}" \
                 --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
                 --repetitions="${REPETITIONS}" \
+                --ell_imbalance_limit="${ELL_IMBALANCE_LIMIT}" \
                 <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }
@@ -237,12 +226,12 @@ run_conversion_benchmarks() {
 # taken as the final result.
 run_spmv_benchmarks() {
     [ "${DRY_RUN}" == "true" ] && return
-    local LOCAL_FORMATS=$(remove_ell_worstcase "$1")
     cp "$1" "$1.imd" # make sure we're not loosing the original input
     ./spmv/spmv${BENCH_SUFFIX} --backup="$1.bkp" --double_buffer="$1.bkp2" \
-                --executor="${EXECUTOR}" --formats="${LOCAL_FORMATS}" \
+                --executor="${EXECUTOR}" --formats="${FORMATS}" \
                 --device_id="${DEVICE_ID}" --gpu_timer=${GPU_TIMER} \
                 --repetitions="${REPETITIONS}" \
+                --ell_imbalance_limit="${ELL_IMBALANCE_LIMIT}" \
                 <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }
diff --git a/benchmark/utils/formats.hpp b/benchmark/utils/formats.hpp
index 597344a2a2b..208b5612139 100644
--- a/benchmark/utils/formats.hpp
+++ b/benchmark/utils/formats.hpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/ginkgo.hpp>
 
 
+#include <algorithm>
 #include <map>
 #include <string>
 
@@ -152,6 +153,10 @@ std::string format_command =
 // the formats command-line argument
 DEFINE_string(formats, "coo", formats::format_command.c_str());
 
+DEFINE_uint64(
+    ell_imbalance_limit, 100,
+    "Maximal storage overhead above which ELL benchmarks will be skipped");
+
 
 namespace formats {
 
@@ -202,6 +207,30 @@ std::shared_ptr<csr::strategy_type> create_gpu_strategy(
 }
 
 
+/**
+ * Checks whether the given matrix data exceeds the ELL imbalance limit set by
+ * the --ell_imbalance_limit flag
+ * @throws gko::Error if the imbalance limit is exceeded
+ */
+void check_ell_admissibility(const gko::matrix_data<etype> &data)
+{
+    if (data.size[0] == 0) {
+        return;
+    }
+    std::vector<gko::size_type> row_lengths(data.size[0]);
+    for (auto nz : data.nonzeros) {
+        row_lengths[nz.row]++;
+    }
+    auto max_len = *std::max_element(row_lengths.begin(), row_lengths.end());
+    auto avg_len =
+        std::max<gko::size_type>(data.nonzeros.size() / data.size[0], 1);
+    if (max_len / avg_len > FLAGS_ell_imbalance_limit) {
+        throw gko::Error(__FILE__, __LINE__,
+                         "Matrix exceeds ELL imbalance limit");
+    }
+}
+
+
 /**
  * Creates a Ginkgo matrix from the intermediate data representation format
  * gko::matrix_data with support for variable arguments.
@@ -241,10 +270,17 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
         {"csrm", READ_MATRIX(csr, std::make_shared<csr::merge_path>())},
         {"csrc", READ_MATRIX(csr, std::make_shared<csr::classical>())},
         {"coo", read_matrix_from_data<gko::matrix::Coo<etype>>},
-        {"ell", read_matrix_from_data<gko::matrix::Ell<etype>>},
+        {"ell", [](std::shared_ptr<const gko::Executor> exec,
+            const gko::matrix_data<etype> &data) {
+             check_ell_admissibility(data);
+             auto mat = gko::matrix::Ell<etype>::create(exec);
+             mat->read(data);
+             return mat;
+         }},
         {"ell-mixed",
          [](std::shared_ptr<const gko::Executor> exec,
             const gko::matrix_data<etype> &data) {
+             check_ell_admissibility(data);
              gko::matrix_data<gko::next_precision<etype>> conv_data;
              conv_data.size = data.size;
              conv_data.nonzeros.resize(data.nonzeros.size());

From 9778e127e979e96681477234e2ee4767a242adf2 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Mon, 5 Jul 2021 14:16:34 +0200
Subject: [PATCH 0805/1008] store benchmarking exceptions in JSON

---
 benchmark/blas/blas.cpp                     | 6 ++++++
 benchmark/conversions/conversions.cpp       | 6 ++++++
 benchmark/preconditioner/preconditioner.cpp | 6 ++++++
 benchmark/solver/solver.cpp                 | 6 ++++++
 benchmark/spmv/spmv.cpp                     | 6 ++++++
 benchmark/utils/general.hpp                 | 4 ++++
 6 files changed, 34 insertions(+)

diff --git a/benchmark/blas/blas.cpp b/benchmark/blas/blas.cpp
index 66c71c21ae8..8b97a42fdd3 100644
--- a/benchmark/blas/blas.cpp
+++ b/benchmark/blas/blas.cpp
@@ -472,6 +472,12 @@ void apply_blas(const char *operation_name, std::shared_ptr<gko::Executor> exec,
     } catch (const std::exception &e) {
         add_or_set_member(test_case["blas"][operation_name], "completed", false,
                           allocator);
+        if (FLAGS_keep_errors) {
+            rapidjson::Value msg_value;
+            msg_value.SetString(e.what(), allocator);
+            add_or_set_member(test_case["blas"][operation_name], "error",
+                              msg_value, allocator);
+        }
         std::cerr << "Error when processing test case " << test_case << "\n"
                   << "what(): " << e.what() << std::endl;
     }
diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index 9e48f6ca327..18313bf26db 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -98,6 +98,12 @@ void convert_matrix(const gko::LinOp *matrix_from, const char *format_to,
     } catch (const std::exception &e) {
         add_or_set_member(test_case["conversions"][conversion_name],
                           "completed", false, allocator);
+        if (FLAGS_keep_errors) {
+            rapidjson::Value msg_value;
+            msg_value.SetString(e.what(), allocator);
+            add_or_set_member(test_case["conversions"][conversion_name],
+                              "error", msg_value, allocator);
+        }
         std::cerr << "Error when processing test case " << test_case << "\n"
                   << "what(): " << e.what() << std::endl;
     }
diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index 695c1e78314..002faeeddff 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -229,6 +229,12 @@ void run_preconditioner(const char *precond_name,
                           rapidjson::Value(rapidjson::kObjectType), allocator);
         add_or_set_member(test_case["preconditioner"][encoded_name.c_str()],
                           "completed", false, allocator);
+        if (FLAGS_keep_errors) {
+            rapidjson::Value msg_value;
+            msg_value.SetString(e.what(), allocator);
+            add_or_set_member(test_case["preconditioner"][encoded_name.c_str()],
+                              "error", msg_value, allocator);
+        }
         std::cerr << "Error when processing test case " << test_case << "\n"
                   << "what(): " << e.what() << std::endl;
     }
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index d709cf442c5..e4df2948cb9 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -509,6 +509,12 @@ void solve_system(const std::string &solver_name,
     } catch (const std::exception &e) {
         add_or_set_member(test_case["solver"][precond_solver_name], "completed",
                           false, allocator);
+        if (FLAGS_keep_errors) {
+            rapidjson::Value msg_value;
+            msg_value.SetString(e.what(), allocator);
+            add_or_set_member(test_case["solver"][precond_solver_name], "error",
+                              msg_value, allocator);
+        }
         std::cerr << "Error when processing test case " << test_case << "\n"
                   << "what(): " << e.what() << std::endl;
     }
diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp
index 4617bb4d7e6..6d21ddd88e6 100644
--- a/benchmark/spmv/spmv.cpp
+++ b/benchmark/spmv/spmv.cpp
@@ -154,6 +154,12 @@ void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
     } catch (const std::exception &e) {
         add_or_set_member(test_case["spmv"][format_name], "completed", false,
                           allocator);
+        if (FLAGS_keep_errors) {
+            rapidjson::Value msg_value;
+            msg_value.SetString(e.what(), allocator);
+            add_or_set_member(test_case["spmv"][format_name], "error",
+                              msg_value, allocator);
+        }
         std::cerr << "Error when processing test case " << test_case << "\n"
                   << "what(): " << e.what() << std::endl;
     }
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index 84fb5926af4..dca22e7fe27 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -84,6 +84,10 @@ DEFINE_string(double_buffer, "",
 DEFINE_bool(detailed, true,
             "If set, performs several runs to obtain more detailed results");
 
+DEFINE_bool(keep_errors, false,
+            "If set, writes exception messages during the execution into the "
+            "JSON output");
+
 DEFINE_bool(nested_names, false, "If set, separately logs nested operations");
 
 DEFINE_uint32(seed, 42, "Seed used for the random number generator");

From d1a6a6815f6e81aa8f80d279e7820b6a2c569a8e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 6 Jul 2021 14:59:14 +0200
Subject: [PATCH 0806/1008] review updates

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
---
 benchmark/run_all_benchmarks.sh |  7 ++++++-
 benchmark/utils/formats.hpp     | 12 ++++++------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/benchmark/run_all_benchmarks.sh b/benchmark/run_all_benchmarks.sh
index 0220496399d..db7fa5bb5ca 100644
--- a/benchmark/run_all_benchmarks.sh
+++ b/benchmark/run_all_benchmarks.sh
@@ -21,6 +21,11 @@ if [ ! "${REPETITIONS}" ]; then
     echo "REPETITIONS    environment variable not set - assuming ${REPETITIONS}" 1>&2
 fi
 
+if [ ! "${SOLVER_REPETITIONS}" ]; then
+    SOLVER_REPETITIONS=1
+    echo "SOLVER_REPETITIONS    environment variable not set - assuming ${SOLVER_REPETITIONS}" 1>&2
+fi
+
 if [ ! "${SEGMENTS}" ]; then
     echo "SEGMENTS    environment variable not set - running entire suite" 1>&2
     SEGMENTS=1
@@ -253,7 +258,7 @@ run_solver_benchmarks() {
                     --gpu_timer=${GPU_TIMER} \
                     --jacobi_max_block_size=${SOLVERS_JACOBI_MAX_BS} --device_id="${DEVICE_ID}" \
                     --gmres_restart="${SOLVERS_GMRES_RESTART}" \
-                    --repetitions="${REPETITIONS}" \
+                    --repetitions="${SOLVER_REPETITIONS}" \
                     <"$1.imd" 2>&1 >"$1"
     keep_latest "$1" "$1.bkp" "$1.bkp2" "$1.imd"
 }
diff --git a/benchmark/utils/formats.hpp b/benchmark/utils/formats.hpp
index 208b5612139..bbce4f25cf2 100644
--- a/benchmark/utils/formats.hpp
+++ b/benchmark/utils/formats.hpp
@@ -153,9 +153,9 @@ std::string format_command =
 // the formats command-line argument
 DEFINE_string(formats, "coo", formats::format_command.c_str());
 
-DEFINE_uint64(
-    ell_imbalance_limit, 100,
-    "Maximal storage overhead above which ELL benchmarks will be skipped");
+DEFINE_int64(ell_imbalance_limit, 100,
+             "Maximal storage overhead above which ELL benchmarks will be "
+             "skipped. Negative values mean no limit.");
 
 
 namespace formats {
@@ -210,11 +210,12 @@ std::shared_ptr<csr::strategy_type> create_gpu_strategy(
 /**
  * Checks whether the given matrix data exceeds the ELL imbalance limit set by
  * the --ell_imbalance_limit flag
+ *
  * @throws gko::Error if the imbalance limit is exceeded
  */
 void check_ell_admissibility(const gko::matrix_data<etype> &data)
 {
-    if (data.size[0] == 0) {
+    if (data.size[0] == 0 || FLAGS_ell_imbalance_limit < 0) {
         return;
     }
     std::vector<gko::size_type> row_lengths(data.size[0]);
@@ -222,8 +223,7 @@ void check_ell_admissibility(const gko::matrix_data<etype> &data)
         row_lengths[nz.row]++;
     }
     auto max_len = *std::max_element(row_lengths.begin(), row_lengths.end());
-    auto avg_len =
-        std::max<gko::size_type>(data.nonzeros.size() / data.size[0], 1);
+    auto avg_len = data.nonzeros.size() / std::max<double>(data.size[0], 1);
     if (max_len / avg_len > FLAGS_ell_imbalance_limit) {
         throw gko::Error(__FILE__, __LINE__,
                          "Matrix exceeds ELL imbalance limit");

From 7c84f28d4c3c89434e8b7c6d365f1602ff6135cc Mon Sep 17 00:00:00 2001
From: Fabian Castelli <50630942+gfcas@users.noreply.github.com>
Date: Wed, 7 Jul 2021 15:28:22 +0200
Subject: [PATCH 0807/1008] Merge Add missing examples in list

- Add missing examples in list
- update example topics
- add mixed-precision group in advantage

Co-authored-by: Fabian Castelli <fabian.castelli@kit.edu>
Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Yuhsiang M. Tsai <yhmtsai@gmail.com>

Related PR: https://github.com/ginkgo-project/ginkgo/pull/819
---
 doc/examples/examples.hpp.in | 96 ++++++++++++++++++++++++++++++++----
 1 file changed, 87 insertions(+), 9 deletions(-)

diff --git a/doc/examples/examples.hpp.in b/doc/examples/examples.hpp.in
index b511b71e48f..adc7287a337 100644
--- a/doc/examples/examples.hpp.in
+++ b/doc/examples/examples.hpp.in
@@ -187,6 +187,60 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *            factorization.
  *       </td></tr>
  *
+ *   <tr valign="top">
+ *       <td>@ref mixed_spmv</td>
+ *       <td> Shows the Ginkgo mixed precision spmv functionality.
+ *       </td></tr>
+ *
+ *   <tr valign="top">
+ *       <td>@ref mixed_precision_ir</td>
+ *       <td> Manual implementation of a Mixed Precision Iterative Refinement
+ *            (MPIR) solver.
+ *       </td></tr>
+ *
+ *   <tr valign="top">
+ *       <td>@ref adaptiveprecision_blockjacobi</td>
+ *       <td> Shows how to use the adaptive precision block-Jacobi
+ *            preconditioner.
+ *       </td></tr>
+ *
+ *   <tr valign="top">
+ *       <td>@ref cb_gmres</td>
+ *       <td> Using the Ginkgo CB-GMRES solver (Compressed Basis GMRES).
+ *       </td></tr>
+ *
+ *   <tr valign="top">
+ *       <td>@ref heat_equation</td>
+ *       <td> Solving a 2D heat equation and showing matrix assembly, vector
+ *            initalization and solver setup in a more complex setting with
+ *            output visualization.
+ *       </td></tr>
+ *
+ *   <tr valign="top">
+ *       <td>@ref iterative_refinement</td>
+ *       <td> Using a low accuracy CG solver as an inner solver to an iterative
+ *            refinement (IR) method which solves a linear system.
+ *       </td></tr>
+ *
+ *   <tr valign="top">
+ *       <td>@ref ir_ilu_preconditioned_solver</td>
+ *       <td> Combining iterative refinement with the adaptive precision
+ *            block-Jacobi preconditioner to approximate triangular systems
+ *            occurring in ILU preconditioning.
+ *       </td></tr>
+ *
+ *   <tr valign="top">
+ *       <td>@ref par_ilu_convergence</td>
+ *       <td> Convergence analysis at the examples of parallel incomplete
+ *            factorization solver.
+ *       </td></tr>
+ *
+ *   <tr valign="top">
+ *       <td>@ref preconditioner_export</td>
+ *       <td> Explicit generation and storage of preconditioners for given
+ *            matrices.
+ *       </td></tr>
+ *
  * </table>
  *
  * <a name="topic"></a>
@@ -196,16 +250,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  * <table align="center" width="90%">
  *
  *   <tr valign="top">
- *     <td> Solving a simple linear system with choice of executors.
+ *     <td> Solving a simple linear system with choice of executors
  *     </td>
  *     <td>@ref simple_solver
  *     </td>
  *   </tr>
  *
  *   <tr valign="top">
- *     <td> Debug the performance of a solver using loggers.
+ *     <td> Debug the performance of a solver or preconditioner
  *     </td>
  *     <td>@ref performance_debugging
+ *         @ref preconditioner_export
  *     </td>
  *   </tr>
  *
@@ -220,7 +275,20 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *     <td width="400px"> Using preconditioners
  *     </td>
  *     <td>@ref preconditioned_solver,
- *         @ref ilu_preconditioned_solver
+ *         @ref ilu_preconditioned_solver,
+ *         @ref ir_ilu_preconditioned_solver,
+ *         @ref adaptiveprecision_blockjacobi,
+ *         @ref par_ilu_convergence,
+ *         @ref preconditioner_export
+ *     </td>
+ *   </tr>
+ *
+ *   <tr valign="top">
+ *     <td> Iterative refinement
+ *     </td>
+ *     <td>@ref iterative_refinement,
+ *         @ref mixed_precision_ir,
+ *         @ref ir_ilu_preconditioned_solver
  *     </td>
  *   </tr>
  *
@@ -235,7 +303,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *   </tr>
  *
  *   <tr valign="top">
- *     <td width="400px"> Reading in a matrix and right hand side from a file.
+ *     <td width="400px"> Reading in a matrix and right hand side from a file
  *     </td>
  *     <td>@ref simple_solver,
  *         @ref minimal_cuda_solver,
@@ -255,7 +323,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  * <table align="center" width="90%">
  *
  *   <tr valign="top">
- *     <td> Using Ginkgo with external libraries.
+ *     <td> Using Ginkgo with external libraries
  *     </td>
  *     <td>@ref external_lib_interfacing
  *     </td>
@@ -278,33 +346,43 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *   </tr>
  *
  *   <tr valign="top">
- *     <td> Using Ginkgo to construct more complex linear algebra routines.
+ *     <td> Using Ginkgo to construct more complex linear algebra routines
  *     </td>
  *     <td> @ref inverse_iteration
  *     </td>
  *   </tr>
  *
  *   <tr valign="top">
- *     <td> Logging within Ginkgo.
+ *     <td> Logging within Ginkgo
  *     </td>
  *     <td> @ref simple_solver_logging,
  *          @ref papi_logging,
+ *          @ref performance_debugging
  *          @ref custom_logger
  *     </td>
  *   </tr>
  *
  *   <tr valign="top">
- *     <td> Constructing your own stopping criterion.
+ *     <td> Constructing your own stopping criterion
  *     </td>
  *     <td> @ref custom_stopping_criterion
  *     </td>
  *   </tr>
  *
  *   <tr valign="top">
- *     <td> Using ranges in Ginkgo.
+ *     <td> Using ranges in Ginkgo
  *     </td>
  *     <td> @ref ginkgo_ranges
  *     </td>
  *   </tr>
+ *
+ *   <tr valign="top">
+ *     <td> Mixed precision
+ *     </td>
+ *     <td>@ref mixed_spmv,
+ *         @ref mixed_precision_ir,
+ *         @ref adaptiveprecision_blockjacobi
+ *     </td>
+ *   </tr>
  * </table>
  */

From f5c987cb3877552b31d366cd6f203cc37855cc62 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 8 Jul 2021 18:27:41 +0200
Subject: [PATCH 0808/1008] fix blas benchmark gemm

---
 benchmark/blas/blas.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/benchmark/blas/blas.cpp b/benchmark/blas/blas.cpp
index 8b97a42fdd3..bf52992a4fa 100644
--- a/benchmark/blas/blas.cpp
+++ b/benchmark/blas/blas.cpp
@@ -271,7 +271,7 @@ class ApplyOperation : public BenchmarkOperation {
                sizeof(etype);
     }
 
-    void run() override { B_->apply(lend(A_), lend(C_)); }
+    void run() override { A_->apply(lend(B_), lend(C_)); }
 
 private:
     std::unique_ptr<gko::matrix::Dense<etype>> A_;
@@ -317,7 +317,7 @@ class AdvancedApplyOperation : public BenchmarkOperation {
 
     void run() override
     {
-        B_->apply(lend(alpha_), lend(A_), lend(beta_), lend(C_));
+        A_->apply(lend(alpha_), lend(B_), lend(beta_), lend(C_));
     }
 
 private:

From a7a63973328b8fe21ecb4b4408d8615ba1ae569b Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 24 Mar 2021 11:47:15 +0100
Subject: [PATCH 0809/1008] add simple kernel support

---
 include/ginkgo/core/base/executor.hpp       |  19 +-
 include/ginkgo/core/synthesizer/kernels.hpp | 155 +++++++++++
 include/ginkgo/ginkgo.hpp                   |   1 +
 omp/base/kernel_launch.hpp                  | 265 +++++++++++++++++++
 omp/solver/cg_kernels.cpp                   |  25 +-
 reference/base/kernel_launch.hpp            | 268 ++++++++++++++++++++
 reference/solver/cg_kernels.cpp             |  25 +-
 7 files changed, 732 insertions(+), 26 deletions(-)
 create mode 100644 include/ginkgo/core/synthesizer/kernels.hpp
 create mode 100644 omp/base/kernel_launch.hpp
 create mode 100644 reference/base/kernel_launch.hpp

diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index d9ca2e3af32..6a19ddf8644 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -391,7 +391,7 @@ private:                                                                     \
             ::gko::syn::as_list<::gko::syn::range<0, sizeof...(Args)>>;       \
                                                                               \
     public:                                                                   \
-        explicit _name##_operation(Args &&... args)                           \
+        explicit _name##_operation(Args &&...args)                            \
             : data(std::forward<Args>(args)...)                               \
         {}                                                                    \
                                                                               \
@@ -417,7 +417,7 @@ private:                                                                     \
     };                                                                        \
                                                                               \
     template <typename... Args>                                               \
-    static _name##_operation<Args...> make_##_name(Args &&... args)           \
+    static _name##_operation<Args...> make_##_name(Args &&...args)            \
     {                                                                         \
         return _name##_operation<Args...>(std::forward<Args>(args)...);       \
     }                                                                         \
@@ -1174,6 +1174,9 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
         return this->get_exec_info().num_pu_per_cu;
     }
 
+    template <typename KernelDescription>
+    void run_kernel(KernelDescription kernel) const;
+
 protected:
     OmpExecutor()
     {
@@ -1229,6 +1232,9 @@ class ReferenceExecutor : public OmpExecutor {
         this->template log<log::Logger::operation_completed>(this, &op);
     }
 
+    template <typename KernelDescription>
+    void run_kernel(KernelDescription kernel) const;
+
 protected:
     ReferenceExecutor()
     {
@@ -1305,6 +1311,9 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
     void run(const Operation &op) const override;
 
+    template <typename KernelDescription>
+    void run_kernel(KernelDescription kernel) const;
+
     /**
      * Get the CUDA device id of the device associated to this executor.
      */
@@ -1522,6 +1531,9 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
     void run(const Operation &op) const override;
 
+    template <typename KernelDescription>
+    void run_kernel(KernelDescription kernel) const;
+
     /**
      * Get the HIP device id of the device associated to this executor.
      */
@@ -1733,6 +1745,9 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
 
     void run(const Operation &op) const override;
 
+    template <typename KernelDescription>
+    void run_kernel(KernelDescription kernel) const;
+
     /**
      * Get the DPCPP device id of the device associated to this executor.
      *
diff --git a/include/ginkgo/core/synthesizer/kernels.hpp b/include/ginkgo/core/synthesizer/kernels.hpp
new file mode 100644
index 00000000000..a05ce69bbc2
--- /dev/null
+++ b/include/ginkgo/core/synthesizer/kernels.hpp
@@ -0,0 +1,155 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <tuple>
+
+
+#include <ginkgo/core/base/range.hpp>
+#include <ginkgo/core/base/range_accessors.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+namespace gko {
+namespace syn {
+namespace detail {
+
+
+template <typename KernelFunctor, typename... Args>
+struct kernel_2d_impl {
+    KernelFunctor fn;
+    dim<2> size;
+    std::tuple<Args...> args;
+};
+
+
+template <typename T>
+struct pointwise_wrapper {
+    T *obj;
+};
+
+
+template <typename T>
+struct rowwise_wrapper {
+    T *obj;
+};
+
+
+template <typename T>
+struct colwise_wrapper {
+    T *obj;
+};
+
+
+template <typename T>
+struct scalar_wrapper {
+    T *obj;
+};
+
+
+struct row_wrapper {};
+
+
+struct col_wrapper {};
+
+
+template <typename T>
+struct tuple_wrap_helper {
+    static std::tuple<T> wrap(T obj) { return std::make_tuple(obj); }
+};
+
+template <typename... Ts>
+struct tuple_wrap_helper<std::tuple<Ts...>> {
+    static std::tuple<Ts...> wrap(std::tuple<Ts...> obj) { return obj; }
+};
+
+
+}  // namespace detail
+
+
+inline std::tuple<detail::row_wrapper> row() { return {}; }
+
+
+inline std::tuple<detail::col_wrapper> col() { return {}; }
+
+
+inline std::tuple<detail::row_wrapper, detail::col_wrapper> row_col()
+{
+    return {};
+}
+
+
+template <typename... Ts>
+std::tuple<detail::pointwise_wrapper<Ts>...> pointwise(Ts *... args)
+{
+    return std::make_tuple(detail::pointwise_wrapper<Ts>{args}...);
+}
+
+
+template <typename... Ts>
+std::tuple<detail::rowwise_wrapper<Ts>...> rowwise(Ts *... args)
+{
+    return std::make_tuple(detail::rowwise_wrapper<Ts>{args}...);
+}
+
+
+template <typename... Ts>
+std::tuple<detail::colwise_wrapper<Ts>...> colwise(Ts *... args)
+{
+    return std::make_tuple(detail::colwise_wrapper<Ts>{args}...);
+}
+
+
+template <typename... Ts>
+std::tuple<detail::scalar_wrapper<Ts>...> scalar(Ts *... args)
+{
+    return std::make_tuple(detail::scalar_wrapper<Ts>{args}...);
+}
+
+
+template <typename KernelFunctor, typename... Args>
+detail::kernel_2d_impl<KernelFunctor, Args...> kernel_2d_unpacked(
+    KernelFunctor fn, dim<2> size, std::tuple<Args...> args)
+{
+    return {fn, size, args};
+}
+
+
+template <typename KernelFunctor, typename... Args>
+decltype(auto) kernel_2d(KernelFunctor fn, dim<2> size, Args... args)
+{
+    return kernel_2d_unpacked(
+        fn, size,
+        std::tuple_cat(detail::tuple_wrap_helper<Args>::wrap(args)...));
+}
+
+
+}  // namespace syn
+}  // namespace gko
\ No newline at end of file
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index a757a5a6072..22d0f27f5f4 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -122,6 +122,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/time.hpp>
 
 #include <ginkgo/core/synthesizer/containers.hpp>
+#include <ginkgo/core/synthesizer/kernels.hpp>
 
 
 #endif  // GKO_GINKGO_HPP_
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
new file mode 100644
index 00000000000..929d770c0de
--- /dev/null
+++ b/omp/base/kernel_launch.hpp
@@ -0,0 +1,265 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <utility>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/range.hpp>
+#include <ginkgo/core/base/range_accessors.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+namespace gko {
+namespace syn {
+namespace detail {
+
+
+template <typename ValueType>
+struct matrix_accessor {
+    ValueType *data;
+    size_type stride;
+
+    ValueType &operator()(size_type row, size_type col)
+    {
+        return data[row * stride + col];
+    }
+};
+
+
+template <typename Tuple, typename Functor, std::size_t... Is>
+decltype(auto) map_tuple_impl(Tuple &&tuple, Functor fn,
+                              std::index_sequence<Is...>)
+{
+    return std::make_tuple(fn(std::get<Is>(tuple))...);
+}
+
+
+template <typename Tuple, typename Functor, std::size_t... Is>
+decltype(auto) invoke_tuple_impl(Tuple &&tuple, Functor fn,
+                                 std::index_sequence<Is...>)
+{
+    return fn(std::get<Is>(tuple)...);
+}
+
+
+}  // namespace detail
+
+
+template <typename... Ts, typename Functor>
+decltype(auto) map_tuple(const std::tuple<Ts...> &tuple, Functor fn)
+{
+    return detail::map_tuple_impl(tuple, fn,
+                                  std::index_sequence_for<Ts &...>{});
+}
+
+
+template <typename... Ts, typename Functor>
+decltype(auto) invoke_tuple(const std::tuple<Ts...> &tuple, Functor fn)
+{
+    return detail::invoke_tuple_impl(tuple, fn,
+                                     std::index_sequence_for<Ts...>{});
+}
+
+
+// 2D accessors for matrices
+template <typename T>
+T &map_omp_2d(detail::rowwise_wrapper<matrix::Dense<T>> &obj, size_type row,
+              size_type col)
+{
+    return obj.obj->at(row, 0);
+}
+
+
+template <typename T>
+const T &map_omp_2d(detail::rowwise_wrapper<const matrix::Dense<T>> &obj,
+                    size_type row, size_type col)
+{
+    return obj.obj->get_const_values()[row * obj.obj->get_stride()];
+}
+
+
+template <typename T>
+T &map_omp_2d(detail::colwise_wrapper<matrix::Dense<T>> &obj, size_type row,
+              size_type col)
+{
+    return obj.obj->at(0, col);
+}
+
+
+template <typename T>
+const T &map_omp_2d(detail::colwise_wrapper<const matrix::Dense<T>> &obj,
+                    size_type row, size_type col)
+{
+    return obj.obj->get_const_values()[col];
+}
+
+
+template <typename T>
+T &map_omp_2d(detail::pointwise_wrapper<matrix::Dense<T>> &obj, size_type row,
+              size_type col)
+{
+    return obj.obj->at(row, col);
+}
+
+
+template <typename T>
+const T &map_omp_2d(detail::pointwise_wrapper<const matrix::Dense<T>> &obj,
+                    size_type row, size_type col)
+{
+    return obj.obj->get_const_values()[col + obj.obj->get_stride() * row];
+}
+
+
+template <typename T>
+detail::matrix_accessor<T> map_omp_2d(matrix::Dense<T> *obj, size_type row,
+                                      size_type col)
+{
+    return detail::matrix_accessor<T>{obj->get_values(), obj->get_stride()};
+}
+
+
+template <typename T>
+detail::matrix_accessor<const T> map_omp_2d(const matrix::Dense<T> *obj,
+                                            size_type row, size_type col)
+{
+    return detail::matrix_accessor<const T>{obj->get_const_values(),
+                                            obj->get_stride()};
+}
+
+
+// 2D accessors for arrays
+template <typename T>
+T &map_omp_2d(detail::rowwise_wrapper<Array<T>> &obj, size_type row,
+              size_type col)
+{
+    return obj.obj->get_data()[row];
+}
+
+
+template <typename T>
+const T &map_omp_2d(detail::rowwise_wrapper<const Array<T>> &obj, size_type row,
+                    size_type col)
+{
+    return obj.obj->get_const_data()[row];
+}
+
+
+template <typename T>
+T &map_omp_2d(detail::colwise_wrapper<Array<T>> &obj, size_type row,
+              size_type col)
+{
+    return obj.obj->get_data()[col];
+}
+
+
+template <typename T>
+const T &map_omp_2d(detail::colwise_wrapper<const Array<T>> &obj, size_type row,
+                    size_type col)
+{
+    return obj.obj->get_const_data()[col];
+}
+
+
+template <typename T>
+stopping_status *map_omp_2d(Array<T> *obj, size_type, size_type)
+{
+    return obj->get_data();
+}
+
+
+template <typename T>
+const stopping_status *map_omp_2d(const Array<T> *obj, size_type, size_type)
+{
+    return obj->get_const_data();
+}
+
+
+// 2D accessors for raw pointers
+template <typename T>
+T &map_omp_2d(detail::rowwise_wrapper<T> &obj, size_type row, size_type col)
+{
+    return obj[row];
+}
+
+
+template <typename T>
+T &map_omp_2d(detail::colwise_wrapper<T> &obj, size_type row, size_type col)
+{
+    return obj[col];
+}
+
+
+template <typename T>
+T *map_omp_2d(T *obj, size_type, size_type)
+{
+    return obj;
+}
+
+
+size_type map_omp_2d(detail::row_wrapper, size_type row, size_type col)
+{
+    return row;
+}
+
+
+size_type map_omp_2d(detail::col_wrapper, size_type row, size_type col)
+{
+    return col;
+}
+
+
+template <typename KernelFunctor, typename... Args>
+void run_omp_kernel(detail::kernel_2d_impl<KernelFunctor, Args...> kernel)
+{
+#pragma omp parallel for collapse(2)
+    for (size_type row = 0; row < kernel.size[0]; row++) {
+        for (size_type col = 0; col < kernel.size[1]; col++) {
+            invoke_tuple(kernel.args, [&](auto... args) {
+                kernel.fn(map_omp_2d(args, row, col)...);
+            });
+        }
+    }
+}
+
+
+}  // namespace syn
+
+
+template <typename KernelDescription>
+void OmpExecutor::run_kernel(KernelDescription kernel) const
+{
+    syn::run_omp_kernel(kernel);
+}
+
+
+}  // namespace gko
\ No newline at end of file
diff --git a/omp/solver/cg_kernels.cpp b/omp/solver/cg_kernels.cpp
index b012c2cb260..d8e87fc74c5 100644
--- a/omp/solver/cg_kernels.cpp
+++ b/omp/solver/cg_kernels.cpp
@@ -40,6 +40,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/synthesizer/kernels.hpp>
+
+
+#include "omp/base/kernel_launch.hpp"
 
 
 namespace gko {
@@ -114,19 +118,16 @@ void step_2(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Dense<ValueType> *rho,
             const Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type i = 0; i < x->get_size()[0]; ++i) {
-        for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            if (stop_status->get_const_data()[j].has_stopped()) {
-                continue;
-            }
-            if (beta->at(j) != zero<ValueType>()) {
-                auto tmp = rho->at(j) / beta->at(j);
-                x->at(i, j) += tmp * p->at(i, j);
-                r->at(i, j) -= tmp * q->at(i, j);
+    exec->run_kernel(syn::kernel_2d(
+        [](auto &x, auto &r, auto p, auto q, auto beta, auto rho, auto stop) {
+            if (!stop.has_stopped() && beta != zero(beta)) {
+                auto tmp = rho / beta;
+                x += tmp * p;
+                r -= tmp * q;
             }
-        }
-    }
+        },
+        x->get_size(), syn::pointwise(x, r, p, q),
+        syn::colwise(beta, rho, stop_status)));
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
diff --git a/reference/base/kernel_launch.hpp b/reference/base/kernel_launch.hpp
new file mode 100644
index 00000000000..8c16b28b488
--- /dev/null
+++ b/reference/base/kernel_launch.hpp
@@ -0,0 +1,268 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <utility>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/range.hpp>
+#include <ginkgo/core/base/range_accessors.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+namespace gko {
+namespace syn {
+namespace detail {
+
+
+template <typename ValueType>
+struct matrix_accessor {
+    ValueType *data;
+    size_type stride;
+
+    ValueType &operator()(size_type row, size_type col)
+    {
+        return data[row * stride + col];
+    }
+};
+
+
+template <typename Tuple, typename Functor, std::size_t... Is>
+decltype(auto) map_tuple_impl(Tuple &&tuple, Functor fn,
+                              std::index_sequence<Is...>)
+{
+    return std::make_tuple(fn(std::get<Is>(tuple))...);
+}
+
+
+template <typename Tuple, typename Functor, std::size_t... Is>
+decltype(auto) invoke_tuple_impl(Tuple &&tuple, Functor fn,
+                                 std::index_sequence<Is...>)
+{
+    return fn(std::get<Is>(tuple)...);
+}
+
+
+}  // namespace detail
+
+
+template <typename... Ts, typename Functor>
+decltype(auto) map_tuple(const std::tuple<Ts...> &tuple, Functor fn)
+{
+    return detail::map_tuple_impl(tuple, fn,
+                                  std::index_sequence_for<Ts &...>{});
+}
+
+
+template <typename... Ts, typename Functor>
+decltype(auto) invoke_tuple(const std::tuple<Ts...> &tuple, Functor fn)
+{
+    return detail::invoke_tuple_impl(tuple, fn,
+                                     std::index_sequence_for<Ts...>{});
+}
+
+
+// 2D accessors for matrices
+template <typename T>
+T &map_reference_2d(detail::rowwise_wrapper<matrix::Dense<T>> &obj,
+                    size_type row, size_type col)
+{
+    return obj.obj->at(row, 0);
+}
+
+
+template <typename T>
+const T &map_reference_2d(detail::rowwise_wrapper<const matrix::Dense<T>> &obj,
+                          size_type row, size_type col)
+{
+    return obj.obj->get_const_values()[row * obj.obj->get_stride()];
+}
+
+
+template <typename T>
+T &map_reference_2d(detail::colwise_wrapper<matrix::Dense<T>> &obj,
+                    size_type row, size_type col)
+{
+    return obj.obj->at(0, col);
+}
+
+
+template <typename T>
+const T &map_reference_2d(detail::colwise_wrapper<const matrix::Dense<T>> &obj,
+                          size_type row, size_type col)
+{
+    return obj.obj->get_const_values()[col];
+}
+
+
+template <typename T>
+T &map_reference_2d(detail::pointwise_wrapper<matrix::Dense<T>> &obj,
+                    size_type row, size_type col)
+{
+    return obj.obj->at(row, col);
+}
+
+
+template <typename T>
+const T &map_reference_2d(
+    detail::pointwise_wrapper<const matrix::Dense<T>> &obj, size_type row,
+    size_type col)
+{
+    return obj.obj->get_const_values()[col + obj.obj->get_stride() * row];
+}
+
+
+template <typename T>
+detail::matrix_accessor<T> map_reference_2d(matrix::Dense<T> *obj,
+                                            size_type row, size_type col)
+{
+    return detail::matrix_accessor<T>{obj->get_values(), obj->get_stride()};
+}
+
+
+template <typename T>
+detail::matrix_accessor<const T> map_reference_2d(const matrix::Dense<T> *obj,
+                                                  size_type row, size_type col)
+{
+    return detail::matrix_accessor<const T>{obj->get_const_values(),
+                                            obj->get_stride()};
+}
+
+
+// 2D accessors for arrays
+template <typename T>
+T &map_reference_2d(detail::rowwise_wrapper<Array<T>> &obj, size_type row,
+                    size_type col)
+{
+    return obj.obj->get_data()[row];
+}
+
+
+template <typename T>
+const T &map_reference_2d(detail::rowwise_wrapper<const Array<T>> &obj,
+                          size_type row, size_type col)
+{
+    return obj.obj->get_const_data()[row];
+}
+
+
+template <typename T>
+T &map_reference_2d(detail::colwise_wrapper<Array<T>> &obj, size_type row,
+                    size_type col)
+{
+    return obj.obj->get_data()[col];
+}
+
+
+template <typename T>
+const T &map_reference_2d(detail::colwise_wrapper<const Array<T>> &obj,
+                          size_type row, size_type col)
+{
+    return obj.obj->get_const_data()[col];
+}
+
+
+template <typename T>
+stopping_status *map_reference_2d(Array<T> *obj, size_type, size_type)
+{
+    return obj->get_data();
+}
+
+
+template <typename T>
+const stopping_status *map_reference_2d(const Array<T> *obj, size_type,
+                                        size_type)
+{
+    return obj->get_const_data();
+}
+
+
+// 2D accessors for raw pointers
+template <typename T>
+T &map_reference_2d(detail::rowwise_wrapper<T> &obj, size_type row,
+                    size_type col)
+{
+    return obj[row];
+}
+
+
+template <typename T>
+T &map_reference_2d(detail::colwise_wrapper<T> &obj, size_type row,
+                    size_type col)
+{
+    return obj[col];
+}
+
+
+template <typename T>
+T *map_reference_2d(T *obj, size_type, size_type)
+{
+    return obj;
+}
+
+
+size_type map_reference_2d(detail::row_wrapper, size_type row, size_type col)
+{
+    return row;
+}
+
+
+size_type map_reference_2d(detail::col_wrapper, size_type row, size_type col)
+{
+    return col;
+}
+
+
+template <typename KernelFunctor, typename... Args>
+void run_reference_kernel(detail::kernel_2d_impl<KernelFunctor, Args...> kernel)
+{
+    for (size_type row = 0; row < kernel.size[0]; row++) {
+        for (size_type col = 0; col < kernel.size[1]; col++) {
+            invoke_tuple(kernel.args, [&](auto... args) {
+                kernel.fn(map_reference_2d(args, row, col)...);
+            });
+        }
+    }
+}
+
+
+}  // namespace syn
+
+
+template <typename KernelDescription>
+void ReferenceExecutor::run_kernel(KernelDescription kernel) const
+{
+    syn::run_reference_kernel(kernel);
+}
+
+
+}  // namespace gko
\ No newline at end of file
diff --git a/reference/solver/cg_kernels.cpp b/reference/solver/cg_kernels.cpp
index 4202d41a2e3..137f8e30e53 100644
--- a/reference/solver/cg_kernels.cpp
+++ b/reference/solver/cg_kernels.cpp
@@ -39,6 +39,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
+#include "reference/base/kernel_launch.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace reference {
@@ -100,7 +103,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const ReferenceExecutor> exec,
+void step_2(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
             const matrix::Dense<ValueType> *p,
             const matrix::Dense<ValueType> *q,
@@ -108,18 +111,16 @@ void step_2(std::shared_ptr<const ReferenceExecutor> exec,
             const matrix::Dense<ValueType> *rho,
             const Array<stopping_status> *stop_status)
 {
-    for (size_type i = 0; i < x->get_size()[0]; ++i) {
-        for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            if (stop_status->get_const_data()[j].has_stopped()) {
-                continue;
+    exec->run_kernel(
+        [](auto &x, auto &r, auto p, auto q, auto beta, auto rho, auto stop) {
+            if (!stop[col].has_stopped() &&
+                beta(0, col) != zero(beta(0, col))) {
+                auto tmp = rho(0, col) / beta(0, col);
+                x(row, col) += tmp * p(row, col);
+                r(row, col) -= tmp * q(row, col);
             }
-            if (beta->at(j) != zero<ValueType>()) {
-                auto tmp = rho->at(j) / beta->at(j);
-                x->at(i, j) += tmp * p->at(i, j);
-                r->at(i, j) -= tmp * q->at(i, j);
-            }
-        }
-    }
+        },
+        x->get_size(), x, r, p, q, beta, rho, stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);

From 213c58d345677a383243cf1255f18b119d2ecced Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 2 Apr 2021 17:31:25 +0200
Subject: [PATCH 0810/1008] Revert "add simple kernel support"

This reverts commit da6ed9dfec4aa328010e1dde204320ba9bd4c50f.
---
 include/ginkgo/core/base/executor.hpp       |  19 +-
 include/ginkgo/core/synthesizer/kernels.hpp | 155 -----------
 include/ginkgo/ginkgo.hpp                   |   1 -
 omp/base/kernel_launch.hpp                  | 265 -------------------
 omp/solver/cg_kernels.cpp                   |  25 +-
 reference/base/kernel_launch.hpp            | 268 --------------------
 reference/solver/cg_kernels.cpp             |  25 +-
 7 files changed, 26 insertions(+), 732 deletions(-)
 delete mode 100644 include/ginkgo/core/synthesizer/kernels.hpp
 delete mode 100644 omp/base/kernel_launch.hpp
 delete mode 100644 reference/base/kernel_launch.hpp

diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 6a19ddf8644..d9ca2e3af32 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -391,7 +391,7 @@ private:                                                                     \
             ::gko::syn::as_list<::gko::syn::range<0, sizeof...(Args)>>;       \
                                                                               \
     public:                                                                   \
-        explicit _name##_operation(Args &&...args)                            \
+        explicit _name##_operation(Args &&... args)                           \
             : data(std::forward<Args>(args)...)                               \
         {}                                                                    \
                                                                               \
@@ -417,7 +417,7 @@ private:                                                                     \
     };                                                                        \
                                                                               \
     template <typename... Args>                                               \
-    static _name##_operation<Args...> make_##_name(Args &&...args)            \
+    static _name##_operation<Args...> make_##_name(Args &&... args)           \
     {                                                                         \
         return _name##_operation<Args...>(std::forward<Args>(args)...);       \
     }                                                                         \
@@ -1174,9 +1174,6 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
         return this->get_exec_info().num_pu_per_cu;
     }
 
-    template <typename KernelDescription>
-    void run_kernel(KernelDescription kernel) const;
-
 protected:
     OmpExecutor()
     {
@@ -1232,9 +1229,6 @@ class ReferenceExecutor : public OmpExecutor {
         this->template log<log::Logger::operation_completed>(this, &op);
     }
 
-    template <typename KernelDescription>
-    void run_kernel(KernelDescription kernel) const;
-
 protected:
     ReferenceExecutor()
     {
@@ -1311,9 +1305,6 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
     void run(const Operation &op) const override;
 
-    template <typename KernelDescription>
-    void run_kernel(KernelDescription kernel) const;
-
     /**
      * Get the CUDA device id of the device associated to this executor.
      */
@@ -1531,9 +1522,6 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
     void run(const Operation &op) const override;
 
-    template <typename KernelDescription>
-    void run_kernel(KernelDescription kernel) const;
-
     /**
      * Get the HIP device id of the device associated to this executor.
      */
@@ -1745,9 +1733,6 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
 
     void run(const Operation &op) const override;
 
-    template <typename KernelDescription>
-    void run_kernel(KernelDescription kernel) const;
-
     /**
      * Get the DPCPP device id of the device associated to this executor.
      *
diff --git a/include/ginkgo/core/synthesizer/kernels.hpp b/include/ginkgo/core/synthesizer/kernels.hpp
deleted file mode 100644
index a05ce69bbc2..00000000000
--- a/include/ginkgo/core/synthesizer/kernels.hpp
+++ /dev/null
@@ -1,155 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <tuple>
-
-
-#include <ginkgo/core/base/range.hpp>
-#include <ginkgo/core/base/range_accessors.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-
-namespace gko {
-namespace syn {
-namespace detail {
-
-
-template <typename KernelFunctor, typename... Args>
-struct kernel_2d_impl {
-    KernelFunctor fn;
-    dim<2> size;
-    std::tuple<Args...> args;
-};
-
-
-template <typename T>
-struct pointwise_wrapper {
-    T *obj;
-};
-
-
-template <typename T>
-struct rowwise_wrapper {
-    T *obj;
-};
-
-
-template <typename T>
-struct colwise_wrapper {
-    T *obj;
-};
-
-
-template <typename T>
-struct scalar_wrapper {
-    T *obj;
-};
-
-
-struct row_wrapper {};
-
-
-struct col_wrapper {};
-
-
-template <typename T>
-struct tuple_wrap_helper {
-    static std::tuple<T> wrap(T obj) { return std::make_tuple(obj); }
-};
-
-template <typename... Ts>
-struct tuple_wrap_helper<std::tuple<Ts...>> {
-    static std::tuple<Ts...> wrap(std::tuple<Ts...> obj) { return obj; }
-};
-
-
-}  // namespace detail
-
-
-inline std::tuple<detail::row_wrapper> row() { return {}; }
-
-
-inline std::tuple<detail::col_wrapper> col() { return {}; }
-
-
-inline std::tuple<detail::row_wrapper, detail::col_wrapper> row_col()
-{
-    return {};
-}
-
-
-template <typename... Ts>
-std::tuple<detail::pointwise_wrapper<Ts>...> pointwise(Ts *... args)
-{
-    return std::make_tuple(detail::pointwise_wrapper<Ts>{args}...);
-}
-
-
-template <typename... Ts>
-std::tuple<detail::rowwise_wrapper<Ts>...> rowwise(Ts *... args)
-{
-    return std::make_tuple(detail::rowwise_wrapper<Ts>{args}...);
-}
-
-
-template <typename... Ts>
-std::tuple<detail::colwise_wrapper<Ts>...> colwise(Ts *... args)
-{
-    return std::make_tuple(detail::colwise_wrapper<Ts>{args}...);
-}
-
-
-template <typename... Ts>
-std::tuple<detail::scalar_wrapper<Ts>...> scalar(Ts *... args)
-{
-    return std::make_tuple(detail::scalar_wrapper<Ts>{args}...);
-}
-
-
-template <typename KernelFunctor, typename... Args>
-detail::kernel_2d_impl<KernelFunctor, Args...> kernel_2d_unpacked(
-    KernelFunctor fn, dim<2> size, std::tuple<Args...> args)
-{
-    return {fn, size, args};
-}
-
-
-template <typename KernelFunctor, typename... Args>
-decltype(auto) kernel_2d(KernelFunctor fn, dim<2> size, Args... args)
-{
-    return kernel_2d_unpacked(
-        fn, size,
-        std::tuple_cat(detail::tuple_wrap_helper<Args>::wrap(args)...));
-}
-
-
-}  // namespace syn
-}  // namespace gko
\ No newline at end of file
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index 22d0f27f5f4..a757a5a6072 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -122,7 +122,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/time.hpp>
 
 #include <ginkgo/core/synthesizer/containers.hpp>
-#include <ginkgo/core/synthesizer/kernels.hpp>
 
 
 #endif  // GKO_GINKGO_HPP_
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
deleted file mode 100644
index 929d770c0de..00000000000
--- a/omp/base/kernel_launch.hpp
+++ /dev/null
@@ -1,265 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <utility>
-
-
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/base/range.hpp>
-#include <ginkgo/core/base/range_accessors.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-
-
-namespace gko {
-namespace syn {
-namespace detail {
-
-
-template <typename ValueType>
-struct matrix_accessor {
-    ValueType *data;
-    size_type stride;
-
-    ValueType &operator()(size_type row, size_type col)
-    {
-        return data[row * stride + col];
-    }
-};
-
-
-template <typename Tuple, typename Functor, std::size_t... Is>
-decltype(auto) map_tuple_impl(Tuple &&tuple, Functor fn,
-                              std::index_sequence<Is...>)
-{
-    return std::make_tuple(fn(std::get<Is>(tuple))...);
-}
-
-
-template <typename Tuple, typename Functor, std::size_t... Is>
-decltype(auto) invoke_tuple_impl(Tuple &&tuple, Functor fn,
-                                 std::index_sequence<Is...>)
-{
-    return fn(std::get<Is>(tuple)...);
-}
-
-
-}  // namespace detail
-
-
-template <typename... Ts, typename Functor>
-decltype(auto) map_tuple(const std::tuple<Ts...> &tuple, Functor fn)
-{
-    return detail::map_tuple_impl(tuple, fn,
-                                  std::index_sequence_for<Ts &...>{});
-}
-
-
-template <typename... Ts, typename Functor>
-decltype(auto) invoke_tuple(const std::tuple<Ts...> &tuple, Functor fn)
-{
-    return detail::invoke_tuple_impl(tuple, fn,
-                                     std::index_sequence_for<Ts...>{});
-}
-
-
-// 2D accessors for matrices
-template <typename T>
-T &map_omp_2d(detail::rowwise_wrapper<matrix::Dense<T>> &obj, size_type row,
-              size_type col)
-{
-    return obj.obj->at(row, 0);
-}
-
-
-template <typename T>
-const T &map_omp_2d(detail::rowwise_wrapper<const matrix::Dense<T>> &obj,
-                    size_type row, size_type col)
-{
-    return obj.obj->get_const_values()[row * obj.obj->get_stride()];
-}
-
-
-template <typename T>
-T &map_omp_2d(detail::colwise_wrapper<matrix::Dense<T>> &obj, size_type row,
-              size_type col)
-{
-    return obj.obj->at(0, col);
-}
-
-
-template <typename T>
-const T &map_omp_2d(detail::colwise_wrapper<const matrix::Dense<T>> &obj,
-                    size_type row, size_type col)
-{
-    return obj.obj->get_const_values()[col];
-}
-
-
-template <typename T>
-T &map_omp_2d(detail::pointwise_wrapper<matrix::Dense<T>> &obj, size_type row,
-              size_type col)
-{
-    return obj.obj->at(row, col);
-}
-
-
-template <typename T>
-const T &map_omp_2d(detail::pointwise_wrapper<const matrix::Dense<T>> &obj,
-                    size_type row, size_type col)
-{
-    return obj.obj->get_const_values()[col + obj.obj->get_stride() * row];
-}
-
-
-template <typename T>
-detail::matrix_accessor<T> map_omp_2d(matrix::Dense<T> *obj, size_type row,
-                                      size_type col)
-{
-    return detail::matrix_accessor<T>{obj->get_values(), obj->get_stride()};
-}
-
-
-template <typename T>
-detail::matrix_accessor<const T> map_omp_2d(const matrix::Dense<T> *obj,
-                                            size_type row, size_type col)
-{
-    return detail::matrix_accessor<const T>{obj->get_const_values(),
-                                            obj->get_stride()};
-}
-
-
-// 2D accessors for arrays
-template <typename T>
-T &map_omp_2d(detail::rowwise_wrapper<Array<T>> &obj, size_type row,
-              size_type col)
-{
-    return obj.obj->get_data()[row];
-}
-
-
-template <typename T>
-const T &map_omp_2d(detail::rowwise_wrapper<const Array<T>> &obj, size_type row,
-                    size_type col)
-{
-    return obj.obj->get_const_data()[row];
-}
-
-
-template <typename T>
-T &map_omp_2d(detail::colwise_wrapper<Array<T>> &obj, size_type row,
-              size_type col)
-{
-    return obj.obj->get_data()[col];
-}
-
-
-template <typename T>
-const T &map_omp_2d(detail::colwise_wrapper<const Array<T>> &obj, size_type row,
-                    size_type col)
-{
-    return obj.obj->get_const_data()[col];
-}
-
-
-template <typename T>
-stopping_status *map_omp_2d(Array<T> *obj, size_type, size_type)
-{
-    return obj->get_data();
-}
-
-
-template <typename T>
-const stopping_status *map_omp_2d(const Array<T> *obj, size_type, size_type)
-{
-    return obj->get_const_data();
-}
-
-
-// 2D accessors for raw pointers
-template <typename T>
-T &map_omp_2d(detail::rowwise_wrapper<T> &obj, size_type row, size_type col)
-{
-    return obj[row];
-}
-
-
-template <typename T>
-T &map_omp_2d(detail::colwise_wrapper<T> &obj, size_type row, size_type col)
-{
-    return obj[col];
-}
-
-
-template <typename T>
-T *map_omp_2d(T *obj, size_type, size_type)
-{
-    return obj;
-}
-
-
-size_type map_omp_2d(detail::row_wrapper, size_type row, size_type col)
-{
-    return row;
-}
-
-
-size_type map_omp_2d(detail::col_wrapper, size_type row, size_type col)
-{
-    return col;
-}
-
-
-template <typename KernelFunctor, typename... Args>
-void run_omp_kernel(detail::kernel_2d_impl<KernelFunctor, Args...> kernel)
-{
-#pragma omp parallel for collapse(2)
-    for (size_type row = 0; row < kernel.size[0]; row++) {
-        for (size_type col = 0; col < kernel.size[1]; col++) {
-            invoke_tuple(kernel.args, [&](auto... args) {
-                kernel.fn(map_omp_2d(args, row, col)...);
-            });
-        }
-    }
-}
-
-
-}  // namespace syn
-
-
-template <typename KernelDescription>
-void OmpExecutor::run_kernel(KernelDescription kernel) const
-{
-    syn::run_omp_kernel(kernel);
-}
-
-
-}  // namespace gko
\ No newline at end of file
diff --git a/omp/solver/cg_kernels.cpp b/omp/solver/cg_kernels.cpp
index d8e87fc74c5..b012c2cb260 100644
--- a/omp/solver/cg_kernels.cpp
+++ b/omp/solver/cg_kernels.cpp
@@ -40,10 +40,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/synthesizer/kernels.hpp>
-
-
-#include "omp/base/kernel_launch.hpp"
 
 
 namespace gko {
@@ -118,16 +114,19 @@ void step_2(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Dense<ValueType> *rho,
             const Array<stopping_status> *stop_status)
 {
-    exec->run_kernel(syn::kernel_2d(
-        [](auto &x, auto &r, auto p, auto q, auto beta, auto rho, auto stop) {
-            if (!stop.has_stopped() && beta != zero(beta)) {
-                auto tmp = rho / beta;
-                x += tmp * p;
-                r -= tmp * q;
+#pragma omp parallel for
+    for (size_type i = 0; i < x->get_size()[0]; ++i) {
+        for (size_type j = 0; j < x->get_size()[1]; ++j) {
+            if (stop_status->get_const_data()[j].has_stopped()) {
+                continue;
+            }
+            if (beta->at(j) != zero<ValueType>()) {
+                auto tmp = rho->at(j) / beta->at(j);
+                x->at(i, j) += tmp * p->at(i, j);
+                r->at(i, j) -= tmp * q->at(i, j);
             }
-        },
-        x->get_size(), syn::pointwise(x, r, p, q),
-        syn::colwise(beta, rho, stop_status)));
+        }
+    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
diff --git a/reference/base/kernel_launch.hpp b/reference/base/kernel_launch.hpp
deleted file mode 100644
index 8c16b28b488..00000000000
--- a/reference/base/kernel_launch.hpp
+++ /dev/null
@@ -1,268 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <utility>
-
-
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/base/range.hpp>
-#include <ginkgo/core/base/range_accessors.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-
-
-namespace gko {
-namespace syn {
-namespace detail {
-
-
-template <typename ValueType>
-struct matrix_accessor {
-    ValueType *data;
-    size_type stride;
-
-    ValueType &operator()(size_type row, size_type col)
-    {
-        return data[row * stride + col];
-    }
-};
-
-
-template <typename Tuple, typename Functor, std::size_t... Is>
-decltype(auto) map_tuple_impl(Tuple &&tuple, Functor fn,
-                              std::index_sequence<Is...>)
-{
-    return std::make_tuple(fn(std::get<Is>(tuple))...);
-}
-
-
-template <typename Tuple, typename Functor, std::size_t... Is>
-decltype(auto) invoke_tuple_impl(Tuple &&tuple, Functor fn,
-                                 std::index_sequence<Is...>)
-{
-    return fn(std::get<Is>(tuple)...);
-}
-
-
-}  // namespace detail
-
-
-template <typename... Ts, typename Functor>
-decltype(auto) map_tuple(const std::tuple<Ts...> &tuple, Functor fn)
-{
-    return detail::map_tuple_impl(tuple, fn,
-                                  std::index_sequence_for<Ts &...>{});
-}
-
-
-template <typename... Ts, typename Functor>
-decltype(auto) invoke_tuple(const std::tuple<Ts...> &tuple, Functor fn)
-{
-    return detail::invoke_tuple_impl(tuple, fn,
-                                     std::index_sequence_for<Ts...>{});
-}
-
-
-// 2D accessors for matrices
-template <typename T>
-T &map_reference_2d(detail::rowwise_wrapper<matrix::Dense<T>> &obj,
-                    size_type row, size_type col)
-{
-    return obj.obj->at(row, 0);
-}
-
-
-template <typename T>
-const T &map_reference_2d(detail::rowwise_wrapper<const matrix::Dense<T>> &obj,
-                          size_type row, size_type col)
-{
-    return obj.obj->get_const_values()[row * obj.obj->get_stride()];
-}
-
-
-template <typename T>
-T &map_reference_2d(detail::colwise_wrapper<matrix::Dense<T>> &obj,
-                    size_type row, size_type col)
-{
-    return obj.obj->at(0, col);
-}
-
-
-template <typename T>
-const T &map_reference_2d(detail::colwise_wrapper<const matrix::Dense<T>> &obj,
-                          size_type row, size_type col)
-{
-    return obj.obj->get_const_values()[col];
-}
-
-
-template <typename T>
-T &map_reference_2d(detail::pointwise_wrapper<matrix::Dense<T>> &obj,
-                    size_type row, size_type col)
-{
-    return obj.obj->at(row, col);
-}
-
-
-template <typename T>
-const T &map_reference_2d(
-    detail::pointwise_wrapper<const matrix::Dense<T>> &obj, size_type row,
-    size_type col)
-{
-    return obj.obj->get_const_values()[col + obj.obj->get_stride() * row];
-}
-
-
-template <typename T>
-detail::matrix_accessor<T> map_reference_2d(matrix::Dense<T> *obj,
-                                            size_type row, size_type col)
-{
-    return detail::matrix_accessor<T>{obj->get_values(), obj->get_stride()};
-}
-
-
-template <typename T>
-detail::matrix_accessor<const T> map_reference_2d(const matrix::Dense<T> *obj,
-                                                  size_type row, size_type col)
-{
-    return detail::matrix_accessor<const T>{obj->get_const_values(),
-                                            obj->get_stride()};
-}
-
-
-// 2D accessors for arrays
-template <typename T>
-T &map_reference_2d(detail::rowwise_wrapper<Array<T>> &obj, size_type row,
-                    size_type col)
-{
-    return obj.obj->get_data()[row];
-}
-
-
-template <typename T>
-const T &map_reference_2d(detail::rowwise_wrapper<const Array<T>> &obj,
-                          size_type row, size_type col)
-{
-    return obj.obj->get_const_data()[row];
-}
-
-
-template <typename T>
-T &map_reference_2d(detail::colwise_wrapper<Array<T>> &obj, size_type row,
-                    size_type col)
-{
-    return obj.obj->get_data()[col];
-}
-
-
-template <typename T>
-const T &map_reference_2d(detail::colwise_wrapper<const Array<T>> &obj,
-                          size_type row, size_type col)
-{
-    return obj.obj->get_const_data()[col];
-}
-
-
-template <typename T>
-stopping_status *map_reference_2d(Array<T> *obj, size_type, size_type)
-{
-    return obj->get_data();
-}
-
-
-template <typename T>
-const stopping_status *map_reference_2d(const Array<T> *obj, size_type,
-                                        size_type)
-{
-    return obj->get_const_data();
-}
-
-
-// 2D accessors for raw pointers
-template <typename T>
-T &map_reference_2d(detail::rowwise_wrapper<T> &obj, size_type row,
-                    size_type col)
-{
-    return obj[row];
-}
-
-
-template <typename T>
-T &map_reference_2d(detail::colwise_wrapper<T> &obj, size_type row,
-                    size_type col)
-{
-    return obj[col];
-}
-
-
-template <typename T>
-T *map_reference_2d(T *obj, size_type, size_type)
-{
-    return obj;
-}
-
-
-size_type map_reference_2d(detail::row_wrapper, size_type row, size_type col)
-{
-    return row;
-}
-
-
-size_type map_reference_2d(detail::col_wrapper, size_type row, size_type col)
-{
-    return col;
-}
-
-
-template <typename KernelFunctor, typename... Args>
-void run_reference_kernel(detail::kernel_2d_impl<KernelFunctor, Args...> kernel)
-{
-    for (size_type row = 0; row < kernel.size[0]; row++) {
-        for (size_type col = 0; col < kernel.size[1]; col++) {
-            invoke_tuple(kernel.args, [&](auto... args) {
-                kernel.fn(map_reference_2d(args, row, col)...);
-            });
-        }
-    }
-}
-
-
-}  // namespace syn
-
-
-template <typename KernelDescription>
-void ReferenceExecutor::run_kernel(KernelDescription kernel) const
-{
-    syn::run_reference_kernel(kernel);
-}
-
-
-}  // namespace gko
\ No newline at end of file
diff --git a/reference/solver/cg_kernels.cpp b/reference/solver/cg_kernels.cpp
index 137f8e30e53..4202d41a2e3 100644
--- a/reference/solver/cg_kernels.cpp
+++ b/reference/solver/cg_kernels.cpp
@@ -39,9 +39,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
-#include "reference/base/kernel_launch.hpp"
-
-
 namespace gko {
 namespace kernels {
 namespace reference {
@@ -103,7 +100,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const DefaultExecutor> exec,
+void step_2(std::shared_ptr<const ReferenceExecutor> exec,
             matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
             const matrix::Dense<ValueType> *p,
             const matrix::Dense<ValueType> *q,
@@ -111,16 +108,18 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *rho,
             const Array<stopping_status> *stop_status)
 {
-    exec->run_kernel(
-        [](auto &x, auto &r, auto p, auto q, auto beta, auto rho, auto stop) {
-            if (!stop[col].has_stopped() &&
-                beta(0, col) != zero(beta(0, col))) {
-                auto tmp = rho(0, col) / beta(0, col);
-                x(row, col) += tmp * p(row, col);
-                r(row, col) -= tmp * q(row, col);
+    for (size_type i = 0; i < x->get_size()[0]; ++i) {
+        for (size_type j = 0; j < x->get_size()[1]; ++j) {
+            if (stop_status->get_const_data()[j].has_stopped()) {
+                continue;
             }
-        },
-        x->get_size(), x, r, p, q, beta, rho, stop_status);
+            if (beta->at(j) != zero<ValueType>()) {
+                auto tmp = rho->at(j) / beta->at(j);
+                x->at(i, j) += tmp * p->at(i, j);
+                r->at(i, j) -= tmp * q->at(i, j);
+            }
+        }
+    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);

From 236d8c669b601aa069ee9f63385c1eca81709040 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 2 Apr 2021 19:37:28 +0200
Subject: [PATCH 0811/1008] add direct kernel launch wrappers

---
 cuda/base/kernel_launch.hpp                  | 182 ++++++++++++++++++
 cuda/components/precision_conversion.cu      |  15 +-
 cuda/solver/cg_kernels.cu                    |  77 ++++----
 dpcpp/base/kernel_launch.dp.hpp              | 176 ++++++++++++++++++
 dpcpp/components/precision_conversion.dp.cpp |  11 +-
 dpcpp/solver/cg_kernels.dp.cpp               |  54 +++++-
 hip/base/kernel_launch.hip.hpp               | 186 +++++++++++++++++++
 hip/components/precision_conversion.hip.cpp  |  16 +-
 hip/solver/cg_kernels.hip.cpp                |  84 ++++-----
 include/ginkgo/core/base/executor.hpp        |  32 ++++
 omp/base/kernel_launch.hpp                   | 149 +++++++++++++++
 omp/components/precision_conversion.cpp      |  10 +-
 omp/solver/cg_kernels.cpp                    |  77 ++++----
 13 files changed, 891 insertions(+), 178 deletions(-)
 create mode 100644 cuda/base/kernel_launch.hpp
 create mode 100644 dpcpp/base/kernel_launch.dp.hpp
 create mode 100644 hip/base/kernel_launch.hip.hpp
 create mode 100644 omp/base/kernel_launch.hpp

diff --git a/cuda/base/kernel_launch.hpp b/cuda/base/kernel_launch.hpp
new file mode 100644
index 00000000000..54f00dca084
--- /dev/null
+++ b/cuda/base/kernel_launch.hpp
@@ -0,0 +1,182 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CUDA_BASE_KERNEL_LAUNCH_HPP_
+#define GKO_CUDA_BASE_KERNEL_LAUNCH_HPP_
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "cuda/base/types.hpp"
+#include "cuda/components/thread_ids.cuh"
+
+
+#ifdef GKO_KERNEL
+#error "Only one kernel_launch.hpp file can be included at a time."
+#else
+#define GKO_KERNEL __device__
+#endif
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+
+
+constexpr auto default_block_size = 512;
+
+
+template <typename ValueType>
+struct matrix_accessor {
+    ValueType *data;
+    size_type stride;
+
+    __device__ ValueType &operator()(size_type row, size_type col)
+    {
+        return data[row * stride + col];
+    }
+
+    __device__ ValueType &operator[](size_type idx) { return data[idx]; }
+};
+
+
+template <typename KernelFunction, typename... KernelArgs>
+__global__ __launch_bounds__(default_block_size) void generic_kernel_1d(
+    size_type size, KernelFunction fn, KernelArgs... args)
+{
+    auto tidx = thread::get_thread_id_flat();
+    if (tidx >= size) {
+        return;
+    }
+    fn(tidx, args...);
+}
+
+
+template <typename KernelFunction, typename... KernelArgs>
+__global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
+    size_type rows, size_type cols, KernelFunction fn, KernelArgs... args)
+{
+    auto tidx = thread::get_thread_id_flat();
+    auto col = tidx % cols;
+    auto row = tidx / cols;
+    if (row >= rows) {
+        return;
+    }
+    fn(row, col, args...);
+}
+
+
+template <typename ValueType>
+matrix_accessor<cuda_type<ValueType>> map_to_device(
+    matrix::Dense<ValueType> *mtx)
+{
+    return {as_cuda_type(mtx->get_values()), mtx->get_stride()};
+}
+
+
+template <typename ValueType>
+matrix_accessor<const cuda_type<ValueType>> map_to_device(
+    const matrix::Dense<ValueType> *mtx)
+{
+    return {as_cuda_type(mtx->get_const_values()), mtx->get_stride()};
+}
+
+
+template <typename ValueType>
+typename std::enable_if<std::is_arithmetic<ValueType>::value,
+                        cuda_type<ValueType>>::type *
+map_to_device(ValueType *data)
+{
+    return as_cuda_type(data);
+}
+
+
+template <typename ValueType>
+cuda_type<std::complex<ValueType>> *map_to_device(std::complex<ValueType> *data)
+{
+    return as_cuda_type(data);
+}
+
+
+template <typename ValueType>
+const cuda_type<std::complex<ValueType>> *map_to_device(
+    const std::complex<ValueType> *data)
+{
+    return as_cuda_type(data);
+}
+
+
+template <typename ValueType>
+cuda_type<ValueType> *map_to_device(Array<ValueType> &mtx)
+{
+    return as_cuda_type(mtx.get_data());
+}
+
+
+template <typename ValueType>
+const cuda_type<ValueType> *map_to_device(const Array<ValueType> &mtx)
+{
+    return as_cuda_type(mtx.get_const_data());
+}
+
+
+}  // namespace cuda
+}  // namespace kernels
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void CudaExecutor::run_kernel(KernelFunction fn, size_type size,
+                              KernelArgs &&... args) const
+{
+    constexpr auto block_size = kernels::cuda::default_block_size;
+    auto num_blocks = ceildiv(size, block_size);
+    kernels::cuda::generic_kernel_1d<<<num_blocks, block_size>>>(
+        size, fn, kernels::cuda::map_to_device(args)...);
+}
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void CudaExecutor::run_kernel(KernelFunction fn, dim<2> size,
+                              KernelArgs &&... args) const
+{
+    constexpr auto block_size = kernels::cuda::default_block_size;
+    auto num_blocks = ceildiv(size[0] * size[1], block_size);
+    kernels::cuda::generic_kernel_2d<<<num_blocks, block_size>>>(
+        size[0], size[1], fn, kernels::cuda::map_to_device(args)...);
+}
+
+
+}  // namespace gko
+
+#endif  // GKO_CUDA_BASE_KERNEL_LAUNCH_HPP_
diff --git a/cuda/components/precision_conversion.cu b/cuda/components/precision_conversion.cu
index 335158ff439..20dd87a9b2c 100644
--- a/cuda/components/precision_conversion.cu
+++ b/cuda/components/precision_conversion.cu
@@ -33,8 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/precision_conversion.hpp"
 
 
-#include "cuda/base/types.hpp"
-#include "cuda/components/thread_ids.cuh"
+#include "cuda/base/kernel_launch.hpp"
 
 
 namespace gko {
@@ -43,19 +42,13 @@ namespace cuda {
 namespace components {
 
 
-constexpr int default_block_size = 512;
-
-
-#include "common/components/precision_conversion.hpp.inc"
-
-
 template <typename SourceType, typename TargetType>
 void convert_precision(std::shared_ptr<const DefaultExecutor> exec,
                        size_type size, const SourceType *in, TargetType *out)
 {
-    auto num_blocks = ceildiv(size, default_block_size);
-    convert_precision<<<num_blocks, default_block_size>>>(
-        size, as_cuda_type(in), as_cuda_type(out));
+    exec->run_kernel(
+        [] GKO_KERNEL(auto idx, auto in, auto out) { out[idx] = in[idx]; },
+        size, in, out);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_CONVERT_PRECISION_KERNEL);
diff --git a/cuda/solver/cg_kernels.cu b/cuda/solver/cg_kernels.cu
index 0c7df9e06d8..bba3e8e2845 100644
--- a/cuda/solver/cg_kernels.cu
+++ b/cuda/solver/cg_kernels.cu
@@ -33,13 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/cg_kernels.hpp"
 
 
-#include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "cuda/base/math.hpp"
-#include "cuda/base/types.hpp"
-#include "cuda/components/thread_ids.cuh"
+#include "cuda/base/kernel_launch.hpp"
 
 
 namespace gko {
@@ -53,12 +50,6 @@ namespace cuda {
 namespace cg {
 
 
-constexpr int default_block_size = 512;
-
-
-#include "common/solver/cg_kernels.hpp.inc"
-
-
 template <typename ValueType>
 void initialize(std::shared_ptr<const CudaExecutor> exec,
                 const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
@@ -67,16 +58,20 @@ void initialize(std::shared_ptr<const CudaExecutor> exec,
                 matrix::Dense<ValueType> *rho,
                 Array<stopping_status> *stop_status)
 {
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(b->get_size()[0] * b->get_stride(), block_size.x), 1, 1);
-
-    initialize_kernel<<<grid_size, block_size, 0, 0>>>(
-        b->get_size()[0], b->get_size()[1], b->get_stride(),
-        as_cuda_type(b->get_const_values()), as_cuda_type(r->get_values()),
-        as_cuda_type(z->get_values()), as_cuda_type(p->get_values()),
-        as_cuda_type(q->get_values()), as_cuda_type(prev_rho->get_values()),
-        as_cuda_type(rho->get_values()), as_cuda_type(stop_status->get_data()));
+    exec->run_kernel(
+        [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto z, auto p,
+                      auto q, auto prev_rho, auto rho, auto stop) {
+            if (row == 0) {
+                rho[col] = zero(rho[col]);
+                prev_rho[col] = one(prev_rho[col]);
+                stop[col].reset();
+            }
+            r(row, col) = b(row, col);
+            z(row, col) = zero(z(row, col));
+            p(row, col) = zero(p(row, col));
+            q(row, col) = zero(q(row, col));
+        },
+        p->get_size(), b, r, z, p, q, prev_rho, rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
@@ -89,16 +84,16 @@ void step_1(std::shared_ptr<const CudaExecutor> exec,
             const matrix::Dense<ValueType> *prev_rho,
             const Array<stopping_status> *stop_status)
 {
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(p->get_size()[0] * p->get_stride(), block_size.x), 1, 1);
-
-    step_1_kernel<<<grid_size, block_size, 0, 0>>>(
-        p->get_size()[0], p->get_size()[1], p->get_stride(),
-        as_cuda_type(p->get_values()), as_cuda_type(z->get_const_values()),
-        as_cuda_type(rho->get_const_values()),
-        as_cuda_type(prev_rho->get_const_values()),
-        as_cuda_type(stop_status->get_const_data()));
+    exec->run_kernel(
+        [] GKO_KERNEL(auto row, auto col, auto p, auto z, auto rho,
+                      auto prev_rho, auto stop) {
+            if (!stop[col].has_stopped() &&
+                prev_rho[col] != zero(prev_rho[col])) {
+                auto tmp = rho[col] / prev_rho[col];
+                p(row, col) = z(row, col) + tmp * p(row, col);
+            }
+        },
+        p->get_size(), p, z, rho, prev_rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
@@ -113,18 +108,16 @@ void step_2(std::shared_ptr<const CudaExecutor> exec,
             const matrix::Dense<ValueType> *rho,
             const Array<stopping_status> *stop_status)
 {
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(p->get_size()[0] * p->get_stride(), block_size.x), 1, 1);
-
-    step_2_kernel<<<grid_size, block_size, 0, 0>>>(
-        p->get_size()[0], p->get_size()[1], p->get_stride(), x->get_stride(),
-        as_cuda_type(x->get_values()), as_cuda_type(r->get_values()),
-        as_cuda_type(p->get_const_values()),
-        as_cuda_type(q->get_const_values()),
-        as_cuda_type(beta->get_const_values()),
-        as_cuda_type(rho->get_const_values()),
-        as_cuda_type(stop_status->get_const_data()));
+    exec->run_kernel(
+        [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto p, auto q,
+                      auto beta, auto rho, auto stop) {
+            if (!stop[col].has_stopped() && beta[col] != zero(beta[col])) {
+                auto tmp = rho[col] / beta[col];
+                x(row, col) += tmp * p(row, col);
+                r(row, col) -= tmp * q(row, col);
+            }
+        },
+        x->get_size(), x, r, p, q, beta, rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
new file mode 100644
index 00000000000..c957fe6104a
--- /dev/null
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -0,0 +1,176 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_BASE_KERNEL_LAUNCH_DP_HPP_
+#define GKO_DPCPP_BASE_KERNEL_LAUNCH_DP_HPP_
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#ifdef GKO_KERNEL
+#error "Only one kernel_launch.hpp file can be included at a time."
+#else
+#define GKO_KERNEL
+#endif
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+constexpr auto default_block_size = 512;
+
+
+template <typename ValueType>
+struct matrix_accessor {
+    ValueType *data;
+    size_type stride;
+
+    ValueType &operator()(size_type row, size_type col)
+    {
+        return data[row * stride + col];
+    }
+
+    ValueType &operator[](size_type idx) { return data[idx]; }
+};
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void generic_kernel_1d(sycl::handler &cgh, size_type size, KernelFunction fn,
+                       KernelArgs... args)
+{
+    cgh.parallel_for(sycl::range<1>{size}, [=](sycl::id<1> idx_id) {
+        fn(static_cast<size_type>(idx_id[0]), args...);
+    });
+}
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void generic_kernel_2d(sycl::handler &cgh, size_type rows, size_type cols,
+                       KernelFunction fn, KernelArgs... args)
+{
+    cgh.parallel_for(sycl::range<1>{rows * cols}, [=](sycl::id<1> idx_id) {
+        auto idx = static_cast<size_type>(idx_id[0]);
+        auto row = idx / cols;
+        auto col = idx % cols;
+        fn(row, col, args...);
+    });
+}
+
+
+template <typename ValueType>
+matrix_accessor<ValueType> map_to_device(matrix::Dense<ValueType> *mtx)
+{
+    return {mtx->get_values(), mtx->get_stride()};
+}
+
+
+template <typename ValueType>
+matrix_accessor<const ValueType> map_to_device(
+    const matrix::Dense<ValueType> *mtx)
+{
+    return {mtx->get_const_values(), mtx->get_stride()};
+}
+
+
+template <typename ValueType>
+typename std::enable_if<std::is_arithmetic<ValueType>::value, ValueType>::type *
+map_to_device(ValueType *data)
+{
+    return data;
+}
+
+
+template <typename ValueType>
+std::complex<ValueType> *map_to_device(std::complex<ValueType> *data)
+{
+    return data;
+}
+
+
+template <typename ValueType>
+const std::complex<ValueType> *map_to_device(
+    const std::complex<ValueType> *data)
+{
+    return data;
+}
+
+
+template <typename ValueType>
+ValueType *map_to_device(Array<ValueType> &mtx)
+{
+    return mtx.get_data();
+}
+
+
+template <typename ValueType>
+const ValueType *map_to_device(const Array<ValueType> &mtx)
+{
+    return mtx.get_const_data();
+}
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void DpcppExecutor::run_kernel(KernelFunction fn, size_type size,
+                               KernelArgs &&... args) const
+{
+    this->get_queue()->submit([&](sycl::handler &cgh) {
+        kernels::dpcpp::generic_kernel_1d(
+            cgh, size, fn, kernels::dpcpp::map_to_device(args)...);
+    });
+}
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void DpcppExecutor::run_kernel(KernelFunction fn, dim<2> size,
+                               KernelArgs &&... args) const
+{
+    this->get_queue()->submit([&](sycl::handler &cgh) {
+        kernels::dpcpp::generic_kernel_2d(
+            cgh, size[0], size[1], fn, kernels::dpcpp::map_to_device(args)...);
+    });
+}
+
+
+}  // namespace gko
+
+#endif  // GKO_DPCPP_BASE_KERNEL_LAUNCH_DP_HPP_
diff --git a/dpcpp/components/precision_conversion.dp.cpp b/dpcpp/components/precision_conversion.dp.cpp
index 27c8098bd4b..e70316f6f83 100644
--- a/dpcpp/components/precision_conversion.dp.cpp
+++ b/dpcpp/components/precision_conversion.dp.cpp
@@ -33,7 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/precision_conversion.hpp"
 
 
-#include <CL/sycl.hpp>
+#include "dpcpp/base/kernel_launch.dp.hpp"
 
 
 namespace gko {
@@ -46,12 +46,9 @@ template <typename SourceType, typename TargetType>
 void convert_precision(std::shared_ptr<const DefaultExecutor> exec,
                        size_type size, const SourceType *in, TargetType *out)
 {
-    exec->get_queue()->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl::range<1>{size}, [=](sycl::id<1> idx_id) {
-            const auto idx = idx_id[0];
-            out[idx] = in[idx];
-        });
-    });
+    exec->run_kernel(
+        [] GKO_KERNEL(auto idx, auto in, auto out) { out[idx] = in[idx]; },
+        size, in, out);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_CONVERT_PRECISION_KERNEL);
diff --git a/dpcpp/solver/cg_kernels.dp.cpp b/dpcpp/solver/cg_kernels.dp.cpp
index d7f85000be1..00a815668b0 100644
--- a/dpcpp/solver/cg_kernels.dp.cpp
+++ b/dpcpp/solver/cg_kernels.dp.cpp
@@ -33,13 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/cg_kernels.hpp"
 
 
-#include <CL/sycl.hpp>
-
-
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
+#include "dpcpp/base/kernel_launch.dp.hpp"
 
 
 namespace gko {
@@ -59,7 +53,23 @@ void initialize(std::shared_ptr<const DpcppExecutor> exec,
                 matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
                 matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
                 matrix::Dense<ValueType> *rho,
-                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+                Array<stopping_status> *stop_status)
+{
+    exec->run_kernel(
+        [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto z, auto p,
+                      auto q, auto prev_rho, auto rho, auto stop) {
+            if (row == 0) {
+                rho[col] = zero(rho[col]);
+                prev_rho[col] = one(prev_rho[col]);
+                stop[col].reset();
+            }
+            r(row, col) = b(row, col);
+            z(row, col) = zero(z(row, col));
+            p(row, col) = zero(p(row, col));
+            q(row, col) = zero(q(row, col));
+        },
+        p->get_size(), b, r, z, p, q, prev_rho, rho, *stop_status);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
 
@@ -69,7 +79,19 @@ void step_1(std::shared_ptr<const DpcppExecutor> exec,
             matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
             const matrix::Dense<ValueType> *rho,
             const matrix::Dense<ValueType> *prev_rho,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+            const Array<stopping_status> *stop_status)
+{
+    exec->run_kernel(
+        [] GKO_KERNEL(auto row, auto col, auto p, auto z, auto rho,
+                      auto prev_rho, auto stop) {
+            if (!stop[col].has_stopped() &&
+                prev_rho[col] != zero(prev_rho[col])) {
+                auto tmp = rho[col] / prev_rho[col];
+                p(row, col) = z(row, col) + tmp * p(row, col);
+            }
+        },
+        p->get_size(), p, z, rho, prev_rho, *stop_status);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
 
@@ -81,7 +103,19 @@ void step_2(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Dense<ValueType> *q,
             const matrix::Dense<ValueType> *beta,
             const matrix::Dense<ValueType> *rho,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+            const Array<stopping_status> *stop_status)
+{
+    exec->run_kernel(
+        [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto p, auto q,
+                      auto beta, auto rho, auto stop) {
+            if (!stop[col].has_stopped() && beta[col] != zero(beta[col])) {
+                auto tmp = rho[col] / beta[col];
+                x(row, col) += tmp * p(row, col);
+                r(row, col) -= tmp * q(row, col);
+            }
+        },
+        x->get_size(), x, r, p, q, beta, rho, *stop_status);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
 
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
new file mode 100644
index 00000000000..0bac79c221f
--- /dev/null
+++ b/hip/base/kernel_launch.hip.hpp
@@ -0,0 +1,186 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_HIP_BASE_KERNEL_LAUNCH_HIP_HPP_
+#define GKO_HIP_BASE_KERNEL_LAUNCH_HIP_HPP_
+
+
+#include <hip/hip_runtime.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "hip/base/types.hip.hpp"
+#include "hip/components/thread_ids.hip.hpp"
+
+
+#ifdef GKO_KERNEL
+#error "Only one kernel_launch.hpp file can be included at a time."
+#else
+#define GKO_KERNEL __device__
+#endif
+
+
+namespace gko {
+namespace kernels {
+namespace hip {
+
+
+constexpr auto default_block_size = 512;
+
+
+template <typename ValueType>
+struct matrix_accessor {
+    ValueType *data;
+    size_type stride;
+
+    __device__ ValueType &operator()(size_type row, size_type col)
+    {
+        return data[row * stride + col];
+    }
+
+    __device__ ValueType &operator[](size_type idx) { return data[idx]; }
+};
+
+
+template <typename KernelFunction, typename... KernelArgs>
+__global__ __launch_bounds__(default_block_size) void generic_kernel_1d(
+    size_type size, KernelFunction fn, KernelArgs... args)
+{
+    auto tidx = thread::get_thread_id_flat();
+    if (tidx >= size) {
+        return;
+    }
+    fn(tidx, args...);
+}
+
+
+template <typename KernelFunction, typename... KernelArgs>
+__global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
+    size_type rows, size_type cols, KernelFunction fn, KernelArgs... args)
+{
+    auto tidx = thread::get_thread_id_flat();
+    auto col = tidx % cols;
+    auto row = tidx / cols;
+    if (row >= rows) {
+        return;
+    }
+    fn(row, col, args...);
+}
+
+
+template <typename ValueType>
+matrix_accessor<hip_type<ValueType>> map_to_device(
+    matrix::Dense<ValueType> *mtx)
+{
+    return {as_hip_type(mtx->get_values()), mtx->get_stride()};
+}
+
+
+template <typename ValueType>
+matrix_accessor<const hip_type<ValueType>> map_to_device(
+    const matrix::Dense<ValueType> *mtx)
+{
+    return {as_hip_type(mtx->get_const_values()), mtx->get_stride()};
+}
+
+
+template <typename ValueType>
+typename std::enable_if<std::is_arithmetic<ValueType>::value,
+                        hip_type<ValueType>>::type *
+map_to_device(ValueType *data)
+{
+    return as_hip_type(data);
+}
+
+
+template <typename ValueType>
+hip_type<std::complex<ValueType>> *map_to_device(std::complex<ValueType> *data)
+{
+    return as_hip_type(data);
+}
+
+
+template <typename ValueType>
+const hip_type<std::complex<ValueType>> *map_to_device(
+    const std::complex<ValueType> *data)
+{
+    return as_hip_type(data);
+}
+
+
+template <typename ValueType>
+hip_type<ValueType> *map_to_device(Array<ValueType> &mtx)
+{
+    return as_hip_type(mtx.get_data());
+}
+
+
+template <typename ValueType>
+const hip_type<ValueType> *map_to_device(const Array<ValueType> &mtx)
+{
+    return as_hip_type(mtx.get_const_data());
+}
+
+
+}  // namespace hip
+}  // namespace kernels
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void HipExecutor::run_kernel(KernelFunction fn, size_type size,
+                             KernelArgs &&... args) const
+{
+    constexpr auto block_size = kernels::hip::default_block_size;
+    auto num_blocks = ceildiv(size, block_size);
+    hipLaunchKernelGGL(kernels::hip::generic_kernel_1d, num_blocks, block_size,
+                       0, 0, size, fn, kernels::hip::map_to_device(args)...);
+}
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void HipExecutor::run_kernel(KernelFunction fn, dim<2> size,
+                             KernelArgs &&... args) const
+{
+    constexpr auto block_size = kernels::hip::default_block_size;
+    auto num_blocks = ceildiv(size[0] * size[1], block_size);
+    hipLaunchKernelGGL(kernels::hip::generic_kernel_2d, num_blocks, block_size,
+                       0, 0, size[0], size[1], fn,
+                       kernels::hip::map_to_device(args)...);
+}
+
+
+}  // namespace gko
+
+#endif  // GKO_HIP_BASE_KERNEL_LAUNCH_HIP_HPP_
diff --git a/hip/components/precision_conversion.hip.cpp b/hip/components/precision_conversion.hip.cpp
index bef1727e901..3176dd7174c 100644
--- a/hip/components/precision_conversion.hip.cpp
+++ b/hip/components/precision_conversion.hip.cpp
@@ -33,8 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/precision_conversion.hpp"
 
 
-#include "hip/base/types.hip.hpp"
-#include "hip/components/thread_ids.hip.hpp"
+#include "hip/base/kernel_launch.hip.hpp"
 
 
 namespace gko {
@@ -43,20 +42,13 @@ namespace hip {
 namespace components {
 
 
-constexpr int default_block_size = 512;
-
-
-#include "common/components/precision_conversion.hpp.inc"
-
-
 template <typename SourceType, typename TargetType>
 void convert_precision(std::shared_ptr<const DefaultExecutor> exec,
                        size_type size, const SourceType *in, TargetType *out)
 {
-    auto num_blocks = ceildiv(size, default_block_size);
-    hipLaunchKernelGGL(HIP_KERNEL_NAME(convert_precision), num_blocks,
-                       default_block_size, 0, 0, size, as_hip_type(in),
-                       as_hip_type(out));
+    exec->run_kernel(
+        [] GKO_KERNEL(auto idx, auto in, auto out) { out[idx] = in[idx]; },
+        size, in, out);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_CONVERT_PRECISION_KERNEL);
diff --git a/hip/solver/cg_kernels.hip.cpp b/hip/solver/cg_kernels.hip.cpp
index 72b3de4261b..d54ad1ff309 100644
--- a/hip/solver/cg_kernels.hip.cpp
+++ b/hip/solver/cg_kernels.hip.cpp
@@ -33,16 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/cg_kernels.hpp"
 
 
-#include <hip/hip_runtime.h>
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "hip/base/math.hip.hpp"
-#include "hip/base/types.hip.hpp"
-#include "hip/components/thread_ids.hip.hpp"
+#include "hip/base/kernel_launch.hip.hpp"
 
 
 namespace gko {
@@ -54,14 +48,6 @@ namespace hip {
  * @ingroup cg
  */
 namespace cg {
-
-
-constexpr int default_block_size = 512;
-
-
-#include "common/solver/cg_kernels.hpp.inc"
-
-
 template <typename ValueType>
 void initialize(std::shared_ptr<const HipExecutor> exec,
                 const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
@@ -70,17 +56,20 @@ void initialize(std::shared_ptr<const HipExecutor> exec,
                 matrix::Dense<ValueType> *rho,
                 Array<stopping_status> *stop_status)
 {
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(b->get_size()[0] * b->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(
-        initialize_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-        b->get_size()[0], b->get_size()[1], b->get_stride(),
-        as_hip_type(b->get_const_values()), as_hip_type(r->get_values()),
-        as_hip_type(z->get_values()), as_hip_type(p->get_values()),
-        as_hip_type(q->get_values()), as_hip_type(prev_rho->get_values()),
-        as_hip_type(rho->get_values()), as_hip_type(stop_status->get_data()));
+    exec->run_kernel(
+        [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto z, auto p,
+                      auto q, auto prev_rho, auto rho, auto stop) {
+            if (row == 0) {
+                rho[col] = zero(rho[col]);
+                prev_rho[col] = one(prev_rho[col]);
+                stop[col].reset();
+            }
+            r(row, col) = b(row, col);
+            z(row, col) = zero(z(row, col));
+            p(row, col) = zero(p(row, col));
+            q(row, col) = zero(q(row, col));
+        },
+        p->get_size(), b, r, z, p, q, prev_rho, rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
@@ -93,17 +82,16 @@ void step_1(std::shared_ptr<const HipExecutor> exec,
             const matrix::Dense<ValueType> *prev_rho,
             const Array<stopping_status> *stop_status)
 {
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(p->get_size()[0] * p->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(step_1_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-                       p->get_size()[0], p->get_size()[1], p->get_stride(),
-                       as_hip_type(p->get_values()),
-                       as_hip_type(z->get_const_values()),
-                       as_hip_type(rho->get_const_values()),
-                       as_hip_type(prev_rho->get_const_values()),
-                       as_hip_type(stop_status->get_const_data()));
+    exec->run_kernel(
+        [] GKO_KERNEL(auto row, auto col, auto p, auto z, auto rho,
+                      auto prev_rho, auto stop) {
+            if (!stop[col].has_stopped() &&
+                prev_rho[col] != zero(prev_rho[col])) {
+                auto tmp = rho[col] / prev_rho[col];
+                p(row, col) = z(row, col) + tmp * p(row, col);
+            }
+        },
+        p->get_size(), p, z, rho, prev_rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
@@ -118,18 +106,16 @@ void step_2(std::shared_ptr<const HipExecutor> exec,
             const matrix::Dense<ValueType> *rho,
             const Array<stopping_status> *stop_status)
 {
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(p->get_size()[0] * p->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(
-        step_2_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-        p->get_size()[0], p->get_size()[1], p->get_stride(), x->get_stride(),
-        as_hip_type(x->get_values()), as_hip_type(r->get_values()),
-        as_hip_type(p->get_const_values()), as_hip_type(q->get_const_values()),
-        as_hip_type(beta->get_const_values()),
-        as_hip_type(rho->get_const_values()),
-        as_hip_type(stop_status->get_const_data()));
+    exec->run_kernel(
+        [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto p, auto q,
+                      auto beta, auto rho, auto stop) {
+            if (!stop[col].has_stopped() && beta[col] != zero(beta[col])) {
+                auto tmp = rho[col] / beta[col];
+                x(row, col) += tmp * p(row, col);
+                r(row, col) -= tmp * q(row, col);
+            }
+        },
+        x->get_size(), x, r, p, q, beta, rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index d9ca2e3af32..d2d1f506bc8 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -1174,6 +1174,14 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
         return this->get_exec_info().num_pu_per_cu;
     }
 
+    template <typename KernelFunction, typename... KernelArgs>
+    void run_kernel(KernelFunction fn, size_type size,
+                    KernelArgs &&... args) const;
+
+    template <typename KernelFunction, typename... KernelArgs>
+    void run_kernel(KernelFunction fn, dim<2> size,
+                    KernelArgs &&... args) const;
+
 protected:
     OmpExecutor()
     {
@@ -1305,6 +1313,14 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
     void run(const Operation &op) const override;
 
+    template <typename KernelFunction, typename... KernelArgs>
+    void run_kernel(KernelFunction fn, size_type size,
+                    KernelArgs &&... args) const;
+
+    template <typename KernelFunction, typename... KernelArgs>
+    void run_kernel(KernelFunction fn, dim<2> size,
+                    KernelArgs &&... args) const;
+
     /**
      * Get the CUDA device id of the device associated to this executor.
      */
@@ -1522,6 +1538,14 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
     void run(const Operation &op) const override;
 
+    template <typename KernelFunction, typename... KernelArgs>
+    void run_kernel(KernelFunction fn, size_type size,
+                    KernelArgs &&... args) const;
+
+    template <typename KernelFunction, typename... KernelArgs>
+    void run_kernel(KernelFunction fn, dim<2> size,
+                    KernelArgs &&... args) const;
+
     /**
      * Get the HIP device id of the device associated to this executor.
      */
@@ -1733,6 +1757,14 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
 
     void run(const Operation &op) const override;
 
+    template <typename KernelFunction, typename... KernelArgs>
+    void run_kernel(KernelFunction fn, size_type size,
+                    KernelArgs &&... args) const;
+
+    template <typename KernelFunction, typename... KernelArgs>
+    void run_kernel(KernelFunction fn, dim<2> size,
+                    KernelArgs &&... args) const;
+
     /**
      * Get the DPCPP device id of the device associated to this executor.
      *
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
new file mode 100644
index 00000000000..18f04bad836
--- /dev/null
+++ b/omp/base/kernel_launch.hpp
@@ -0,0 +1,149 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_OMP_BASE_KERNEL_LAUNCH_HPP_
+#define GKO_OMP_BASE_KERNEL_LAUNCH_HPP_
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#ifdef GKO_KERNEL
+#error "Only one kernel_launch.hpp file can be included at a time."
+#else
+#define GKO_KERNEL
+#endif
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+
+
+template <typename ValueType>
+struct matrix_accessor {
+    ValueType *data;
+    size_type stride;
+
+    ValueType &operator()(size_type row, size_type col)
+    {
+        return data[row * stride + col];
+    }
+
+    ValueType &operator[](size_type idx) { return data[idx]; }
+};
+
+
+template <typename ValueType>
+matrix_accessor<ValueType> map_to_device(matrix::Dense<ValueType> *mtx)
+{
+    return {mtx->get_values(), mtx->get_stride()};
+}
+
+
+template <typename ValueType>
+matrix_accessor<const ValueType> map_to_device(
+    const matrix::Dense<ValueType> *mtx)
+{
+    return {mtx->get_const_values(), mtx->get_stride()};
+}
+
+
+template <typename ValueType>
+typename std::enable_if<std::is_arithmetic<ValueType>::value, ValueType>::type *
+map_to_device(ValueType *data)
+{
+    return data;
+}
+
+
+template <typename ValueType>
+std::complex<ValueType> *map_to_device(std::complex<ValueType> *data)
+{
+    return data;
+}
+
+
+template <typename ValueType>
+const std::complex<ValueType> *map_to_device(
+    const std::complex<ValueType> *data)
+{
+    return data;
+}
+
+
+template <typename ValueType>
+ValueType *map_to_device(Array<ValueType> &mtx)
+{
+    return mtx.get_data();
+}
+
+
+template <typename ValueType>
+const ValueType *map_to_device(const Array<ValueType> &mtx)
+{
+    return mtx.get_const_data();
+}
+
+
+}  // namespace omp
+}  // namespace kernels
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void OmpExecutor::run_kernel(KernelFunction fn, size_type size,
+                             KernelArgs &&... args) const
+{
+#pragma omp parallel for
+    for (size_type i = 0; i < size; i++) {
+        fn(i, kernels::omp::map_to_device(args)...);
+    }
+}
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void OmpExecutor::run_kernel(KernelFunction fn, dim<2> size,
+                             KernelArgs &&... args) const
+{
+#pragma omp parallel for
+    for (size_type i = 0; i < size[0] * size[1]; i++) {
+        auto row = i / size[1];
+        auto col = i % size[1];
+        fn(row, col, kernels::omp::map_to_device(args)...);
+    }
+}
+
+
+}  // namespace gko
+
+#endif  // GKO_OMP_BASE_KERNEL_LAUNCH_HPP_
diff --git a/omp/components/precision_conversion.cpp b/omp/components/precision_conversion.cpp
index 75f55e39b46..47160d4a857 100644
--- a/omp/components/precision_conversion.cpp
+++ b/omp/components/precision_conversion.cpp
@@ -33,6 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/precision_conversion.hpp"
 
 
+#include "omp/base/kernel_launch.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace omp {
@@ -43,10 +46,9 @@ template <typename SourceType, typename TargetType>
 void convert_precision(std::shared_ptr<const DefaultExecutor> exec,
                        size_type size, const SourceType *in, TargetType *out)
 {
-#pragma omp parallel for
-    for (size_type i = 0; i < size; ++i) {
-        out[i] = in[i];
-    }
+    exec->run_kernel(
+        [] GKO_KERNEL(auto idx, auto in, auto out) { out[idx] = in[idx]; },
+        size, in, out);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_CONVERT_PRECISION_KERNEL);
diff --git a/omp/solver/cg_kernels.cpp b/omp/solver/cg_kernels.cpp
index b012c2cb260..f65b7849d26 100644
--- a/omp/solver/cg_kernels.cpp
+++ b/omp/solver/cg_kernels.cpp
@@ -33,13 +33,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/cg_kernels.hpp"
 
 
-#include <omp.h>
+#include <ginkgo/core/base/math.hpp>
 
 
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
+#include "omp/base/kernel_launch.hpp"
 
 
 namespace gko {
@@ -61,19 +58,20 @@ void initialize(std::shared_ptr<const OmpExecutor> exec,
                 matrix::Dense<ValueType> *rho,
                 Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type j = 0; j < b->get_size()[1]; ++j) {
-        rho->at(j) = zero<ValueType>();
-        prev_rho->at(j) = one<ValueType>();
-        stop_status->get_data()[j].reset();
-    }
-#pragma omp parallel for
-    for (size_type i = 0; i < b->get_size()[0]; ++i) {
-        for (size_type j = 0; j < b->get_size()[1]; ++j) {
-            r->at(i, j) = b->at(i, j);
-            z->at(i, j) = p->at(i, j) = q->at(i, j) = zero<ValueType>();
-        }
-    }
+    exec->run_kernel(
+        [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto z, auto p,
+                      auto q, auto prev_rho, auto rho, auto stop) {
+            if (row == 0) {
+                rho[col] = zero(rho[col]);
+                prev_rho[col] = one(prev_rho[col]);
+                stop[col].reset();
+            }
+            r(row, col) = b(row, col);
+            z(row, col) = zero(z(row, col));
+            p(row, col) = zero(p(row, col));
+            q(row, col) = zero(q(row, col));
+        },
+        p->get_size(), b, r, z, p, q, prev_rho, rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
@@ -86,20 +84,16 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Dense<ValueType> *prev_rho,
             const Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type i = 0; i < p->get_size()[0]; ++i) {
-        for (size_type j = 0; j < p->get_size()[1]; ++j) {
-            if (stop_status->get_const_data()[j].has_stopped()) {
-                continue;
+    exec->run_kernel(
+        [] GKO_KERNEL(auto row, auto col, auto p, auto z, auto rho,
+                      auto prev_rho, auto stop) {
+            if (!stop[col].has_stopped() &&
+                prev_rho[col] != zero(prev_rho[col])) {
+                auto tmp = rho[col] / prev_rho[col];
+                p(row, col) = z(row, col) + tmp * p(row, col);
             }
-            if (prev_rho->at(j) == zero<ValueType>()) {
-                p->at(i, j) = z->at(i, j);
-            } else {
-                auto tmp = rho->at(j) / prev_rho->at(j);
-                p->at(i, j) = z->at(i, j) + tmp * p->at(i, j);
-            }
-        }
-    }
+        },
+        p->get_size(), p, z, rho, prev_rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
@@ -114,19 +108,16 @@ void step_2(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Dense<ValueType> *rho,
             const Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type i = 0; i < x->get_size()[0]; ++i) {
-        for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            if (stop_status->get_const_data()[j].has_stopped()) {
-                continue;
-            }
-            if (beta->at(j) != zero<ValueType>()) {
-                auto tmp = rho->at(j) / beta->at(j);
-                x->at(i, j) += tmp * p->at(i, j);
-                r->at(i, j) -= tmp * q->at(i, j);
+    exec->run_kernel(
+        [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto p, auto q,
+                      auto beta, auto rho, auto stop) {
+            if (!stop[col].has_stopped() && beta[col] != zero(beta[col])) {
+                auto tmp = rho[col] / beta[col];
+                x(row, col) += tmp * p(row, col);
+                r(row, col) -= tmp * q(row, col);
             }
-        }
-    }
+        },
+        x->get_size(), x, r, p, q, beta, rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);

From b3be8b2302c6981a3e462c5a5a232b8d28c19dba Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 2 Apr 2021 21:00:15 +0200
Subject: [PATCH 0812/1008] set correct nvcc flags

---
 cuda/CMakeLists.txt | 2 +-
 hip/CMakeLists.txt  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index c63a35c89e1..92b381ddd44 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -121,7 +121,7 @@ if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
     # and allows the usage of std::array for nvidia GPUs
     target_compile_options(ginkgo_cuda
         PRIVATE
-            $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>)
+            $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr --expt-extended-lambda>)
 endif()
 
 if (NOT CMAKE_CUDA_HOST_COMPILER AND NOT GINKGO_CUDA_DEFAULT_HOST_COMPILER)
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 86edd13513e..0715e80095b 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -205,7 +205,7 @@ if (GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}")
     endif()
 
     # Remove false positive CUDA warnings when calling one<T>() and zero<T>()
-    list(APPEND GINKGO_HIP_NVCC_ADDITIONAL_FLAGS "--expt-relaxed-constexpr")
+    list(APPEND GINKGO_HIP_NVCC_ADDITIONAL_FLAGS --expt-relaxed-constexpr --expt-extended-lambda)
 
     if (GINKGO_HIP_PLATFROM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}"
             AND CMAKE_CUDA_COMPILER_VERSION MATCHES "9.2"

From 0433b8e0a17bf129c93ff2a294d5d609762d1bf4 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 3 Apr 2021 00:40:02 +0200
Subject: [PATCH 0813/1008] work around icpc internal compiler error

---
 omp/base/kernel_launch.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index 18f04bad836..7cd10c2c30f 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -126,7 +126,7 @@ void OmpExecutor::run_kernel(KernelFunction fn, size_type size,
 {
 #pragma omp parallel for
     for (size_type i = 0; i < size; i++) {
-        fn(i, kernels::omp::map_to_device(args)...);
+        [&]() { fn(i, kernels::omp::map_to_device(args)...); }();
     }
 }
 
@@ -139,7 +139,7 @@ void OmpExecutor::run_kernel(KernelFunction fn, dim<2> size,
     for (size_type i = 0; i < size[0] * size[1]; i++) {
         auto row = i / size[1];
         auto col = i % size[1];
-        fn(row, col, kernels::omp::map_to_device(args)...);
+        [&]() { fn(row, col, kernels::omp::map_to_device(args)...); }();
     }
 }
 

From 46a2d2bf8e7322b3978f750d8ef71526b7373c63 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 3 Apr 2021 00:42:41 +0200
Subject: [PATCH 0814/1008] workaround for MSVC extended-lambda issue

---
 cuda/CMakeLists.txt | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 92b381ddd44..4347a83022d 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -121,7 +121,16 @@ if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
     # and allows the usage of std::array for nvidia GPUs
     target_compile_options(ginkgo_cuda
         PRIVATE
-            $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr --expt-extended-lambda>)
+            $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr>)
+    if(MSVC)
+        target_compile_options(ginkgo_cuda
+            PRIVATE
+                $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda>)
+    else()
+        target_compile_options(ginkgo_cuda
+            PRIVATE
+                $<$<COMPILE_LANGUAGE:CUDA>:--expt-extended-lambda>)
+    endif()
 endif()
 
 if (NOT CMAKE_CUDA_HOST_COMPILER AND NOT GINKGO_CUDA_DEFAULT_HOST_COMPILER)

From 75e09ef987c5a72acb29cc41db0e3f9601025e37 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 21 May 2021 12:33:42 +0200
Subject: [PATCH 0815/1008] unify cg and precision_conversion kernel files

---
 core/base/simple_kernels.hpp                  | 64 +++++++++++++++++++
 cuda/CMakeLists.txt                           |  1 +
 .../{kernel_launch.hpp => kernel_launch.cuh}  |  0
 cuda/components/precision_conversion.cu       |  8 +--
 cuda/solver/cg_kernels.cu                     | 14 ++--
 dpcpp/CMakeLists.txt                          |  1 +
 dpcpp/components/precision_conversion.dp.cpp  |  8 +--
 dpcpp/solver/cg_kernels.dp.cpp                | 13 ++--
 hip/CMakeLists.txt                            |  2 +-
 hip/components/precision_conversion.hip.cpp   |  8 +--
 hip/solver/cg_kernels.hip.cpp                 | 16 ++---
 omp/CMakeLists.txt                            |  1 +
 omp/components/precision_conversion.cpp       |  8 +--
 omp/solver/cg_kernels.cpp                     | 14 ++--
 14 files changed, 107 insertions(+), 51 deletions(-)
 create mode 100644 core/base/simple_kernels.hpp
 rename cuda/base/{kernel_launch.hpp => kernel_launch.cuh} (100%)

diff --git a/core/base/simple_kernels.hpp b/core/base/simple_kernels.hpp
new file mode 100644
index 00000000000..de90bbb6233
--- /dev/null
+++ b/core/base/simple_kernels.hpp
@@ -0,0 +1,64 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_BASE_SIMPLE_KERNELS_HPP_
+#define GKO_CORE_BASE_SIMPLE_KERNELS_HPP_
+
+
+#if defined(GKO_COMPILING_CUDA)
+
+#include "cuda/base/kernel_launch.cuh"
+#define GKO_DEVICE_NAMESPACE cuda
+
+#elif defined(GKO_COMPILING_HIP)
+
+#include "hip/base/kernel_launch.hip.hpp"
+#define GKO_DEVICE_NAMESPACE hip
+
+#elif defined(GKO_COMPILING_DPCPP)
+
+#include "dpcpp/base/kernel_launch.dp.hpp"
+#define GKO_DEVICE_NAMESPACE dpcpp
+
+#elif defined(GKO_COMPILING_OMP)
+
+#include "omp/base/kernel_launch.hpp"
+#define GKO_DEVICE_NAMESPACE omp
+
+#else
+
+#error "This file should only be used inside Ginkgo device compilation"
+
+#endif
+
+
+#endif  // GKO_CORE_BASE_SIMPLE_KERNELS_HPP_
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 4347a83022d..6ded5507f80 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -160,6 +160,7 @@ endif()
 target_compile_options(ginkgo_cuda PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_COMPILER_FLAGS}>)
 target_compile_options(ginkgo_cuda PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${GINKGO_COMPILER_FLAGS}>)
 ginkgo_compile_features(ginkgo_cuda)
+target_compile_definitions(ginkgo_cuda PRIVATE GKO_COMPILING_CUDA)
 target_include_directories(ginkgo_cuda
     SYSTEM PRIVATE ${CUDA_INCLUDE_DIRS})
 target_link_libraries(ginkgo_cuda PRIVATE ${CUDA_RUNTIME_LIBS} ${CUBLAS} ${CUSPARSE} ${CURAND})
diff --git a/cuda/base/kernel_launch.hpp b/cuda/base/kernel_launch.cuh
similarity index 100%
rename from cuda/base/kernel_launch.hpp
rename to cuda/base/kernel_launch.cuh
diff --git a/cuda/components/precision_conversion.cu b/cuda/components/precision_conversion.cu
index 20dd87a9b2c..74deb9f717d 100644
--- a/cuda/components/precision_conversion.cu
+++ b/cuda/components/precision_conversion.cu
@@ -30,15 +30,13 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include "core/base/simple_kernels.hpp"
 #include "core/components/precision_conversion.hpp"
 
 
-#include "cuda/base/kernel_launch.hpp"
-
-
 namespace gko {
 namespace kernels {
-namespace cuda {
+namespace GKO_DEVICE_NAMESPACE {
 namespace components {
 
 
@@ -55,6 +53,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_CONVERT_PRECISION_KERNEL);
 
 
 }  // namespace components
-}  // namespace cuda
+}  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
 }  // namespace gko
diff --git a/cuda/solver/cg_kernels.cu b/cuda/solver/cg_kernels.cu
index bba3e8e2845..97ee0194648 100644
--- a/cuda/solver/cg_kernels.cu
+++ b/cuda/solver/cg_kernels.cu
@@ -30,18 +30,16 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include "core/base/simple_kernels.hpp"
 #include "core/solver/cg_kernels.hpp"
 
 
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "cuda/base/kernel_launch.hpp"
-
-
 namespace gko {
 namespace kernels {
-namespace cuda {
+namespace GKO_DEVICE_NAMESPACE {
 /**
  * @brief The CG solver namespace.
  *
@@ -51,7 +49,7 @@ namespace cg {
 
 
 template <typename ValueType>
-void initialize(std::shared_ptr<const CudaExecutor> exec,
+void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
                 matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
                 matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
@@ -78,7 +76,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const CudaExecutor> exec,
+void step_1(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
             const matrix::Dense<ValueType> *rho,
             const matrix::Dense<ValueType> *prev_rho,
@@ -100,7 +98,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const CudaExecutor> exec,
+void step_2(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
             const matrix::Dense<ValueType> *p,
             const matrix::Dense<ValueType> *q,
@@ -124,6 +122,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
 
 
 }  // namespace cg
-}  // namespace cuda
+}  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
 }  // namespace gko
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index f9cc2183a49..1515ba741c2 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -50,6 +50,7 @@ target_sources(ginkgo_dpcpp
     stop/residual_norm_kernels.dp.cpp)
 
 ginkgo_compile_features(ginkgo_dpcpp)
+target_compile_definitions(ginkgo_dpcpp PRIVATE GKO_COMPILING_DPCPP)
 
 set(GINKGO_DPCPP_FLAGS ${GINKGO_COMPILER_FLAGS} -fsycl)
 set(GINKGO_DPCPP_FLAGS ${GINKGO_DPCPP_FLAGS} PARENT_SCOPE)
diff --git a/dpcpp/components/precision_conversion.dp.cpp b/dpcpp/components/precision_conversion.dp.cpp
index e70316f6f83..74deb9f717d 100644
--- a/dpcpp/components/precision_conversion.dp.cpp
+++ b/dpcpp/components/precision_conversion.dp.cpp
@@ -30,15 +30,13 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include "core/base/simple_kernels.hpp"
 #include "core/components/precision_conversion.hpp"
 
 
-#include "dpcpp/base/kernel_launch.dp.hpp"
-
-
 namespace gko {
 namespace kernels {
-namespace dpcpp {
+namespace GKO_DEVICE_NAMESPACE {
 namespace components {
 
 
@@ -55,6 +53,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_CONVERT_PRECISION_KERNEL);
 
 
 }  // namespace components
-}  // namespace dpcpp
+}  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
 }  // namespace gko
diff --git a/dpcpp/solver/cg_kernels.dp.cpp b/dpcpp/solver/cg_kernels.dp.cpp
index 00a815668b0..97ee0194648 100644
--- a/dpcpp/solver/cg_kernels.dp.cpp
+++ b/dpcpp/solver/cg_kernels.dp.cpp
@@ -30,15 +30,16 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include "core/base/simple_kernels.hpp"
 #include "core/solver/cg_kernels.hpp"
 
 
-#include "dpcpp/base/kernel_launch.dp.hpp"
+#include <ginkgo/core/base/math.hpp>
 
 
 namespace gko {
 namespace kernels {
-namespace dpcpp {
+namespace GKO_DEVICE_NAMESPACE {
 /**
  * @brief The CG solver namespace.
  *
@@ -48,7 +49,7 @@ namespace cg {
 
 
 template <typename ValueType>
-void initialize(std::shared_ptr<const DpcppExecutor> exec,
+void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
                 matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
                 matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
@@ -75,7 +76,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const DpcppExecutor> exec,
+void step_1(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
             const matrix::Dense<ValueType> *rho,
             const matrix::Dense<ValueType> *prev_rho,
@@ -97,7 +98,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const DpcppExecutor> exec,
+void step_2(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
             const matrix::Dense<ValueType> *p,
             const matrix::Dense<ValueType> *q,
@@ -121,6 +122,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
 
 
 }  // namespace cg
-}  // namespace dpcpp
+}  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
 }  // namespace gko
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 0715e80095b..0c64d20ce51 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -232,7 +232,7 @@ if(GINKGO_HIP_AMDGPU)
     endforeach()
 endif()
 
-set(GINKGO_HIPCC_OPTIONS ${GINKGO_HIP_COMPILER_FLAGS} "-std=c++14")
+set(GINKGO_HIPCC_OPTIONS ${GINKGO_HIP_COMPILER_FLAGS} "-std=c++14 -DGKO_COMPILING_HIP")
 set(GINKGO_HIP_NVCC_OPTIONS ${GINKGO_HIP_NVCC_COMPILER_FLAGS} ${GINKGO_HIP_NVCC_ARCH} ${GINKGO_HIP_NVCC_ADDITIONAL_FLAGS})
 set(GINKGO_HIP_CLANG_OPTIONS ${GINKGO_HIP_CLANG_COMPILER_FLAGS} ${GINKGO_AMD_ARCH_FLAGS})
 
diff --git a/hip/components/precision_conversion.hip.cpp b/hip/components/precision_conversion.hip.cpp
index 3176dd7174c..74deb9f717d 100644
--- a/hip/components/precision_conversion.hip.cpp
+++ b/hip/components/precision_conversion.hip.cpp
@@ -30,15 +30,13 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include "core/base/simple_kernels.hpp"
 #include "core/components/precision_conversion.hpp"
 
 
-#include "hip/base/kernel_launch.hip.hpp"
-
-
 namespace gko {
 namespace kernels {
-namespace hip {
+namespace GKO_DEVICE_NAMESPACE {
 namespace components {
 
 
@@ -55,6 +53,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_CONVERT_PRECISION_KERNEL);
 
 
 }  // namespace components
-}  // namespace hip
+}  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
 }  // namespace gko
diff --git a/hip/solver/cg_kernels.hip.cpp b/hip/solver/cg_kernels.hip.cpp
index d54ad1ff309..97ee0194648 100644
--- a/hip/solver/cg_kernels.hip.cpp
+++ b/hip/solver/cg_kernels.hip.cpp
@@ -30,26 +30,26 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include "core/base/simple_kernels.hpp"
 #include "core/solver/cg_kernels.hpp"
 
 
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "hip/base/kernel_launch.hip.hpp"
-
-
 namespace gko {
 namespace kernels {
-namespace hip {
+namespace GKO_DEVICE_NAMESPACE {
 /**
  * @brief The CG solver namespace.
  *
  * @ingroup cg
  */
 namespace cg {
+
+
 template <typename ValueType>
-void initialize(std::shared_ptr<const HipExecutor> exec,
+void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
                 matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
                 matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
@@ -76,7 +76,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const HipExecutor> exec,
+void step_1(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
             const matrix::Dense<ValueType> *rho,
             const matrix::Dense<ValueType> *prev_rho,
@@ -98,7 +98,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const HipExecutor> exec,
+void step_2(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
             const matrix::Dense<ValueType> *p,
             const matrix::Dense<ValueType> *q,
@@ -122,6 +122,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
 
 
 }  // namespace cg
-}  // namespace hip
+}  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
 }  // namespace gko
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index 625cb2bc870..d7a8795df4a 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -43,6 +43,7 @@ target_sources(ginkgo_omp
     stop/residual_norm_kernels.cpp)
 
 ginkgo_compile_features(ginkgo_omp)
+target_compile_definitions(ginkgo_omp PRIVATE GKO_COMPILING_OMP)
 
 # The dependency to OpenMP's potential pthread dependency needs to be public
 # because well, it's a long story:
diff --git a/omp/components/precision_conversion.cpp b/omp/components/precision_conversion.cpp
index 47160d4a857..ac55b456099 100644
--- a/omp/components/precision_conversion.cpp
+++ b/omp/components/precision_conversion.cpp
@@ -31,14 +31,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #include "core/components/precision_conversion.hpp"
-
-
-#include "omp/base/kernel_launch.hpp"
+#include "core/base/simple_kernels.hpp"
 
 
 namespace gko {
 namespace kernels {
-namespace omp {
+namespace GKO_DEVICE_NAMESPACE {
 namespace components {
 
 
@@ -55,6 +53,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_CONVERT_PRECISION_KERNEL);
 
 
 }  // namespace components
-}  // namespace omp
+}  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
 }  // namespace gko
diff --git a/omp/solver/cg_kernels.cpp b/omp/solver/cg_kernels.cpp
index f65b7849d26..42114c3672b 100644
--- a/omp/solver/cg_kernels.cpp
+++ b/omp/solver/cg_kernels.cpp
@@ -31,17 +31,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #include "core/solver/cg_kernels.hpp"
+#include "core/base/simple_kernels.hpp"
 
 
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "omp/base/kernel_launch.hpp"
-
-
 namespace gko {
 namespace kernels {
-namespace omp {
+namespace GKO_DEVICE_NAMESPACE {
 /**
  * @brief The CG solver namespace.
  *
@@ -51,7 +49,7 @@ namespace cg {
 
 
 template <typename ValueType>
-void initialize(std::shared_ptr<const OmpExecutor> exec,
+void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
                 matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
                 matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
@@ -78,7 +76,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const OmpExecutor> exec,
+void step_1(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
             const matrix::Dense<ValueType> *rho,
             const matrix::Dense<ValueType> *prev_rho,
@@ -100,7 +98,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const OmpExecutor> exec,
+void step_2(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
             const matrix::Dense<ValueType> *p,
             const matrix::Dense<ValueType> *q,
@@ -124,6 +122,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
 
 
 }  // namespace cg
-}  // namespace omp
+}  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
 }  // namespace gko

From ab62da17afd9fe35c5536dda75cb1ce0be7bcaba Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 21 May 2021 12:39:55 +0200
Subject: [PATCH 0816/1008] review updates and formatting

Co-authored-by: Yuhsiang M. Tsai <yhmtsai@gmail.com>
---
 cuda/base/kernel_launch.cuh                  | 12 +++---------
 cuda/components/precision_conversion.cu      |  4 +++-
 cuda/solver/cg_kernels.cu                    |  4 +++-
 dpcpp/base/kernel_launch.dp.hpp              | 10 +---------
 dpcpp/components/precision_conversion.dp.cpp |  4 +++-
 dpcpp/solver/cg_kernels.dp.cpp               |  4 +++-
 hip/base/kernel_launch.hip.hpp               |  6 ------
 hip/components/precision_conversion.hip.cpp  |  4 +++-
 hip/solver/cg_kernels.hip.cpp                |  4 +++-
 omp/base/kernel_launch.hpp                   |  6 ------
 omp/components/precision_conversion.cpp      |  2 ++
 omp/solver/cg_kernels.cpp                    |  4 +++-
 12 files changed, 27 insertions(+), 37 deletions(-)

diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index 54f00dca084..9ed3d0350fd 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CUDA_BASE_KERNEL_LAUNCH_HPP_
-#define GKO_CUDA_BASE_KERNEL_LAUNCH_HPP_
+#ifndef GKO_CUDA_BASE_KERNEL_LAUNCH_CUH_
+#define GKO_CUDA_BASE_KERNEL_LAUNCH_CUH_
 
 
 #include <ginkgo/core/base/executor.hpp>
@@ -104,7 +104,6 @@ matrix_accessor<cuda_type<ValueType>> map_to_device(
     return {as_cuda_type(mtx->get_values()), mtx->get_stride()};
 }
 
-
 template <typename ValueType>
 matrix_accessor<const cuda_type<ValueType>> map_to_device(
     const matrix::Dense<ValueType> *mtx)
@@ -112,7 +111,6 @@ matrix_accessor<const cuda_type<ValueType>> map_to_device(
     return {as_cuda_type(mtx->get_const_values()), mtx->get_stride()};
 }
 
-
 template <typename ValueType>
 typename std::enable_if<std::is_arithmetic<ValueType>::value,
                         cuda_type<ValueType>>::type *
@@ -121,14 +119,12 @@ map_to_device(ValueType *data)
     return as_cuda_type(data);
 }
 
-
 template <typename ValueType>
 cuda_type<std::complex<ValueType>> *map_to_device(std::complex<ValueType> *data)
 {
     return as_cuda_type(data);
 }
 
-
 template <typename ValueType>
 const cuda_type<std::complex<ValueType>> *map_to_device(
     const std::complex<ValueType> *data)
@@ -136,14 +132,12 @@ const cuda_type<std::complex<ValueType>> *map_to_device(
     return as_cuda_type(data);
 }
 
-
 template <typename ValueType>
 cuda_type<ValueType> *map_to_device(Array<ValueType> &mtx)
 {
     return as_cuda_type(mtx.get_data());
 }
 
-
 template <typename ValueType>
 const cuda_type<ValueType> *map_to_device(const Array<ValueType> &mtx)
 {
@@ -179,4 +173,4 @@ void CudaExecutor::run_kernel(KernelFunction fn, dim<2> size,
 
 }  // namespace gko
 
-#endif  // GKO_CUDA_BASE_KERNEL_LAUNCH_HPP_
+#endif  // GKO_CUDA_BASE_KERNEL_LAUNCH_CUH_
diff --git a/cuda/components/precision_conversion.cu b/cuda/components/precision_conversion.cu
index 74deb9f717d..a4975b1f68d 100644
--- a/cuda/components/precision_conversion.cu
+++ b/cuda/components/precision_conversion.cu
@@ -30,10 +30,12 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/base/simple_kernels.hpp"
 #include "core/components/precision_conversion.hpp"
 
 
+#include "core/base/simple_kernels.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace GKO_DEVICE_NAMESPACE {
diff --git a/cuda/solver/cg_kernels.cu b/cuda/solver/cg_kernels.cu
index 97ee0194648..079e77fee9a 100644
--- a/cuda/solver/cg_kernels.cu
+++ b/cuda/solver/cg_kernels.cu
@@ -30,13 +30,15 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/base/simple_kernels.hpp"
 #include "core/solver/cg_kernels.hpp"
 
 
 #include <ginkgo/core/base/math.hpp>
 
 
+#include "core/base/simple_kernels.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace GKO_DEVICE_NAMESPACE {
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
index c957fe6104a..0f23ef872fe 100644
--- a/dpcpp/base/kernel_launch.dp.hpp
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -53,9 +53,6 @@ namespace kernels {
 namespace dpcpp {
 
 
-constexpr auto default_block_size = 512;
-
-
 template <typename ValueType>
 struct matrix_accessor {
     ValueType *data;
@@ -99,7 +96,6 @@ matrix_accessor<ValueType> map_to_device(matrix::Dense<ValueType> *mtx)
     return {mtx->get_values(), mtx->get_stride()};
 }
 
-
 template <typename ValueType>
 matrix_accessor<const ValueType> map_to_device(
     const matrix::Dense<ValueType> *mtx)
@@ -107,7 +103,6 @@ matrix_accessor<const ValueType> map_to_device(
     return {mtx->get_const_values(), mtx->get_stride()};
 }
 
-
 template <typename ValueType>
 typename std::enable_if<std::is_arithmetic<ValueType>::value, ValueType>::type *
 map_to_device(ValueType *data)
@@ -115,14 +110,12 @@ map_to_device(ValueType *data)
     return data;
 }
 
-
 template <typename ValueType>
 std::complex<ValueType> *map_to_device(std::complex<ValueType> *data)
 {
     return data;
 }
 
-
 template <typename ValueType>
 const std::complex<ValueType> *map_to_device(
     const std::complex<ValueType> *data)
@@ -130,14 +123,12 @@ const std::complex<ValueType> *map_to_device(
     return data;
 }
 
-
 template <typename ValueType>
 ValueType *map_to_device(Array<ValueType> &mtx)
 {
     return mtx.get_data();
 }
 
-
 template <typename ValueType>
 const ValueType *map_to_device(const Array<ValueType> &mtx)
 {
@@ -173,4 +164,5 @@ void DpcppExecutor::run_kernel(KernelFunction fn, dim<2> size,
 
 }  // namespace gko
 
+
 #endif  // GKO_DPCPP_BASE_KERNEL_LAUNCH_DP_HPP_
diff --git a/dpcpp/components/precision_conversion.dp.cpp b/dpcpp/components/precision_conversion.dp.cpp
index 74deb9f717d..a4975b1f68d 100644
--- a/dpcpp/components/precision_conversion.dp.cpp
+++ b/dpcpp/components/precision_conversion.dp.cpp
@@ -30,10 +30,12 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/base/simple_kernels.hpp"
 #include "core/components/precision_conversion.hpp"
 
 
+#include "core/base/simple_kernels.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace GKO_DEVICE_NAMESPACE {
diff --git a/dpcpp/solver/cg_kernels.dp.cpp b/dpcpp/solver/cg_kernels.dp.cpp
index 97ee0194648..079e77fee9a 100644
--- a/dpcpp/solver/cg_kernels.dp.cpp
+++ b/dpcpp/solver/cg_kernels.dp.cpp
@@ -30,13 +30,15 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/base/simple_kernels.hpp"
 #include "core/solver/cg_kernels.hpp"
 
 
 #include <ginkgo/core/base/math.hpp>
 
 
+#include "core/base/simple_kernels.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace GKO_DEVICE_NAMESPACE {
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index 0bac79c221f..00e1b29da99 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -107,7 +107,6 @@ matrix_accessor<hip_type<ValueType>> map_to_device(
     return {as_hip_type(mtx->get_values()), mtx->get_stride()};
 }
 
-
 template <typename ValueType>
 matrix_accessor<const hip_type<ValueType>> map_to_device(
     const matrix::Dense<ValueType> *mtx)
@@ -115,7 +114,6 @@ matrix_accessor<const hip_type<ValueType>> map_to_device(
     return {as_hip_type(mtx->get_const_values()), mtx->get_stride()};
 }
 
-
 template <typename ValueType>
 typename std::enable_if<std::is_arithmetic<ValueType>::value,
                         hip_type<ValueType>>::type *
@@ -124,14 +122,12 @@ map_to_device(ValueType *data)
     return as_hip_type(data);
 }
 
-
 template <typename ValueType>
 hip_type<std::complex<ValueType>> *map_to_device(std::complex<ValueType> *data)
 {
     return as_hip_type(data);
 }
 
-
 template <typename ValueType>
 const hip_type<std::complex<ValueType>> *map_to_device(
     const std::complex<ValueType> *data)
@@ -139,14 +135,12 @@ const hip_type<std::complex<ValueType>> *map_to_device(
     return as_hip_type(data);
 }
 
-
 template <typename ValueType>
 hip_type<ValueType> *map_to_device(Array<ValueType> &mtx)
 {
     return as_hip_type(mtx.get_data());
 }
 
-
 template <typename ValueType>
 const hip_type<ValueType> *map_to_device(const Array<ValueType> &mtx)
 {
diff --git a/hip/components/precision_conversion.hip.cpp b/hip/components/precision_conversion.hip.cpp
index 74deb9f717d..a4975b1f68d 100644
--- a/hip/components/precision_conversion.hip.cpp
+++ b/hip/components/precision_conversion.hip.cpp
@@ -30,10 +30,12 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/base/simple_kernels.hpp"
 #include "core/components/precision_conversion.hpp"
 
 
+#include "core/base/simple_kernels.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace GKO_DEVICE_NAMESPACE {
diff --git a/hip/solver/cg_kernels.hip.cpp b/hip/solver/cg_kernels.hip.cpp
index 97ee0194648..079e77fee9a 100644
--- a/hip/solver/cg_kernels.hip.cpp
+++ b/hip/solver/cg_kernels.hip.cpp
@@ -30,13 +30,15 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/base/simple_kernels.hpp"
 #include "core/solver/cg_kernels.hpp"
 
 
 #include <ginkgo/core/base/math.hpp>
 
 
+#include "core/base/simple_kernels.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace GKO_DEVICE_NAMESPACE {
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index 7cd10c2c30f..a54ffa75588 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -70,7 +70,6 @@ matrix_accessor<ValueType> map_to_device(matrix::Dense<ValueType> *mtx)
     return {mtx->get_values(), mtx->get_stride()};
 }
 
-
 template <typename ValueType>
 matrix_accessor<const ValueType> map_to_device(
     const matrix::Dense<ValueType> *mtx)
@@ -78,7 +77,6 @@ matrix_accessor<const ValueType> map_to_device(
     return {mtx->get_const_values(), mtx->get_stride()};
 }
 
-
 template <typename ValueType>
 typename std::enable_if<std::is_arithmetic<ValueType>::value, ValueType>::type *
 map_to_device(ValueType *data)
@@ -86,14 +84,12 @@ map_to_device(ValueType *data)
     return data;
 }
 
-
 template <typename ValueType>
 std::complex<ValueType> *map_to_device(std::complex<ValueType> *data)
 {
     return data;
 }
 
-
 template <typename ValueType>
 const std::complex<ValueType> *map_to_device(
     const std::complex<ValueType> *data)
@@ -101,14 +97,12 @@ const std::complex<ValueType> *map_to_device(
     return data;
 }
 
-
 template <typename ValueType>
 ValueType *map_to_device(Array<ValueType> &mtx)
 {
     return mtx.get_data();
 }
 
-
 template <typename ValueType>
 const ValueType *map_to_device(const Array<ValueType> &mtx)
 {
diff --git a/omp/components/precision_conversion.cpp b/omp/components/precision_conversion.cpp
index ac55b456099..a4975b1f68d 100644
--- a/omp/components/precision_conversion.cpp
+++ b/omp/components/precision_conversion.cpp
@@ -31,6 +31,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #include "core/components/precision_conversion.hpp"
+
+
 #include "core/base/simple_kernels.hpp"
 
 
diff --git a/omp/solver/cg_kernels.cpp b/omp/solver/cg_kernels.cpp
index 42114c3672b..079e77fee9a 100644
--- a/omp/solver/cg_kernels.cpp
+++ b/omp/solver/cg_kernels.cpp
@@ -31,12 +31,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #include "core/solver/cg_kernels.hpp"
-#include "core/base/simple_kernels.hpp"
 
 
 #include <ginkgo/core/base/math.hpp>
 
 
+#include "core/base/simple_kernels.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace GKO_DEVICE_NAMESPACE {

From a6c85982d0462da8f0f4d1c4ffaa33cda527782d Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 21 May 2021 13:04:04 +0200
Subject: [PATCH 0817/1008] unify kernel files

---
 .../components/precision_conversion.cpp       |   0
 .../components/precision_conversion.hpp.inc   |  41 ------
 {omp => common}/solver/cg_kernels.cpp         |   0
 common/solver/cg_kernels.hpp.inc              |  98 -------------
 cuda/CMakeLists.txt                           |  14 +-
 cuda/components/precision_conversion.cu       |  60 --------
 cuda/solver/cg_kernels.cu                     | 129 ------------------
 dpcpp/CMakeLists.txt                          |   7 +-
 dpcpp/components/precision_conversion.dp.cpp  |  60 --------
 dpcpp/solver/cg_kernels.dp.cpp                | 129 ------------------
 hip/CMakeLists.txt                            |   7 +-
 hip/components/precision_conversion.hip.cpp   |  60 --------
 hip/solver/cg_kernels.hip.cpp                 | 129 ------------------
 omp/CMakeLists.txt                            |   7 +-
 14 files changed, 23 insertions(+), 718 deletions(-)
 rename {omp => common}/components/precision_conversion.cpp (100%)
 delete mode 100644 common/components/precision_conversion.hpp.inc
 rename {omp => common}/solver/cg_kernels.cpp (100%)
 delete mode 100644 common/solver/cg_kernels.hpp.inc
 delete mode 100644 cuda/components/precision_conversion.cu
 delete mode 100644 cuda/solver/cg_kernels.cu
 delete mode 100644 dpcpp/components/precision_conversion.dp.cpp
 delete mode 100644 dpcpp/solver/cg_kernels.dp.cpp
 delete mode 100644 hip/components/precision_conversion.hip.cpp
 delete mode 100644 hip/solver/cg_kernels.hip.cpp

diff --git a/omp/components/precision_conversion.cpp b/common/components/precision_conversion.cpp
similarity index 100%
rename from omp/components/precision_conversion.cpp
rename to common/components/precision_conversion.cpp
diff --git a/common/components/precision_conversion.hpp.inc b/common/components/precision_conversion.hpp.inc
deleted file mode 100644
index af59f35d058..00000000000
--- a/common/components/precision_conversion.hpp.inc
+++ /dev/null
@@ -1,41 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-template <typename SourceType, typename TargetType>
-__global__ void convert_precision(size_type size, const SourceType *in,
-                                  TargetType *out)
-{
-    auto tnum = thread::get_thread_num_flat();
-    for (auto i = thread::get_thread_id_flat(); i < size; i += tnum) {
-        out[i] = in[i];
-    }
-}
\ No newline at end of file
diff --git a/omp/solver/cg_kernels.cpp b/common/solver/cg_kernels.cpp
similarity index 100%
rename from omp/solver/cg_kernels.cpp
rename to common/solver/cg_kernels.cpp
diff --git a/common/solver/cg_kernels.hpp.inc b/common/solver/cg_kernels.hpp.inc
deleted file mode 100644
index 7a87fc9a9ba..00000000000
--- a/common/solver/cg_kernels.hpp.inc
+++ /dev/null
@@ -1,98 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void initialize_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    const ValueType *__restrict__ b, ValueType *__restrict__ r,
-    ValueType *__restrict__ z, ValueType *__restrict__ p,
-    ValueType *__restrict__ q, ValueType *__restrict__ prev_rho,
-    ValueType *__restrict__ rho, stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-
-    if (tidx < num_cols) {
-        rho[tidx] = zero<ValueType>();
-        prev_rho[tidx] = one<ValueType>();
-        stop_status[tidx].reset();
-    }
-
-    if (tidx < num_rows * stride) {
-        r[tidx] = b[tidx];
-        z[tidx] = zero<ValueType>();
-        p[tidx] = zero<ValueType>();
-        q[tidx] = zero<ValueType>();
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void step_1_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    ValueType *__restrict__ p, const ValueType *__restrict__ z,
-    const ValueType *__restrict__ rho, const ValueType *__restrict__ prev_rho,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto col = tidx % stride;
-    if (col >= num_cols || tidx >= num_rows * stride ||
-        stop_status[col].has_stopped()) {
-        return;
-    }
-    const auto tmp = rho[col] / prev_rho[col];
-    p[tidx] =
-        prev_rho[col] == zero<ValueType>() ? z[tidx] : z[tidx] + tmp * p[tidx];
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void step_2_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    size_type x_stride, ValueType *__restrict__ x, ValueType *__restrict__ r,
-    const ValueType *__restrict__ p, const ValueType *__restrict__ q,
-    const ValueType *__restrict__ beta, const ValueType *__restrict__ rho,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto row = tidx / stride;
-    const auto col = tidx % stride;
-
-    if (col >= num_cols || tidx >= num_rows * num_cols ||
-        stop_status[col].has_stopped()) {
-        return;
-    }
-    if (beta[col] != zero<ValueType>()) {
-        const auto tmp = rho[col] / beta[col];
-        x[row * x_stride + col] += tmp * p[tidx];
-        r[tidx] -= tmp * q[tidx];
-    }
-}
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 6ded5507f80..77fdc8cca17 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -65,6 +65,10 @@ find_library(CURAND curand
     HINT ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
 
 add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_cuda_device> "")
+set(GKO_CUDA_COMMON_SOURCES    
+    ../common/components/precision_conversion.cpp
+    ../common/solver/cg_kernels.cpp
+    )
 target_sources(ginkgo_cuda
     PRIVATE
     base/exception.cpp
@@ -72,7 +76,6 @@ target_sources(ginkgo_cuda
     base/version.cpp
     components/absolute_array.cu
     components/fill_array.cu
-    components/precision_conversion.cu
     components/prefix_sum.cu
     factorization/factorization_kernels.cu
     factorization/ic_kernels.cu
@@ -104,7 +107,6 @@ target_sources(ginkgo_cuda
     reorder/rcm_kernels.cu
     solver/bicg_kernels.cu
     solver/bicgstab_kernels.cu
-    solver/cg_kernels.cu
     solver/cgs_kernels.cu
     solver/fcg_kernels.cu
     solver/gmres_kernels.cu
@@ -114,7 +116,13 @@ target_sources(ginkgo_cuda
     solver/lower_trs_kernels.cu
     solver/upper_trs_kernels.cu
     stop/criterion_kernels.cu
-    stop/residual_norm_kernels.cu)
+    stop/residual_norm_kernels.cu
+    ${GKO_CUDA_COMMON_SOURCES}
+    )
+# override the default language mapping for the common files, set them to CUDA
+foreach(source_file IN LISTS GKO_CUDA_COMMON_SOURCES)
+    set_source_files_properties(${source_file} PROPERTIES LANGUAGE CUDA)
+endforeach(source_file)
 
 if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
     # remove false positive CUDA warnings when calling one<T>() and zero<T>()
diff --git a/cuda/components/precision_conversion.cu b/cuda/components/precision_conversion.cu
deleted file mode 100644
index a4975b1f68d..00000000000
--- a/cuda/components/precision_conversion.cu
+++ /dev/null
@@ -1,60 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/components/precision_conversion.hpp"
-
-
-#include "core/base/simple_kernels.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace GKO_DEVICE_NAMESPACE {
-namespace components {
-
-
-template <typename SourceType, typename TargetType>
-void convert_precision(std::shared_ptr<const DefaultExecutor> exec,
-                       size_type size, const SourceType *in, TargetType *out)
-{
-    exec->run_kernel(
-        [] GKO_KERNEL(auto idx, auto in, auto out) { out[idx] = in[idx]; },
-        size, in, out);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_CONVERT_PRECISION_KERNEL);
-
-
-}  // namespace components
-}  // namespace GKO_DEVICE_NAMESPACE
-}  // namespace kernels
-}  // namespace gko
diff --git a/cuda/solver/cg_kernels.cu b/cuda/solver/cg_kernels.cu
deleted file mode 100644
index 079e77fee9a..00000000000
--- a/cuda/solver/cg_kernels.cu
+++ /dev/null
@@ -1,129 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/cg_kernels.hpp"
-
-
-#include <ginkgo/core/base/math.hpp>
-
-
-#include "core/base/simple_kernels.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace GKO_DEVICE_NAMESPACE {
-/**
- * @brief The CG solver namespace.
- *
- * @ingroup cg
- */
-namespace cg {
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const DefaultExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
-                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
-                matrix::Dense<ValueType> *rho,
-                Array<stopping_status> *stop_status)
-{
-    exec->run_kernel(
-        [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto z, auto p,
-                      auto q, auto prev_rho, auto rho, auto stop) {
-            if (row == 0) {
-                rho[col] = zero(rho[col]);
-                prev_rho[col] = one(prev_rho[col]);
-                stop[col].reset();
-            }
-            r(row, col) = b(row, col);
-            z(row, col) = zero(z(row, col));
-            p(row, col) = zero(p(row, col));
-            q(row, col) = zero(q(row, col));
-        },
-        p->get_size(), b, r, z, p, q, prev_rho, rho, *stop_status);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const DefaultExecutor> exec,
-            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
-            const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *prev_rho,
-            const Array<stopping_status> *stop_status)
-{
-    exec->run_kernel(
-        [] GKO_KERNEL(auto row, auto col, auto p, auto z, auto rho,
-                      auto prev_rho, auto stop) {
-            if (!stop[col].has_stopped() &&
-                prev_rho[col] != zero(prev_rho[col])) {
-                auto tmp = rho[col] / prev_rho[col];
-                p(row, col) = z(row, col) + tmp * p(row, col);
-            }
-        },
-        p->get_size(), p, z, rho, prev_rho, *stop_status);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const DefaultExecutor> exec,
-            matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
-            const matrix::Dense<ValueType> *p,
-            const matrix::Dense<ValueType> *q,
-            const matrix::Dense<ValueType> *beta,
-            const matrix::Dense<ValueType> *rho,
-            const Array<stopping_status> *stop_status)
-{
-    exec->run_kernel(
-        [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto p, auto q,
-                      auto beta, auto rho, auto stop) {
-            if (!stop[col].has_stopped() && beta[col] != zero(beta[col])) {
-                auto tmp = rho[col] / beta[col];
-                x(row, col) += tmp * p(row, col);
-                r(row, col) -= tmp * q(row, col);
-            }
-        },
-        x->get_size(), x, r, p, q, beta, rho, *stop_status);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
-
-
-}  // namespace cg
-}  // namespace GKO_DEVICE_NAMESPACE
-}  // namespace kernels
-}  // namespace gko
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 1515ba741c2..f03fc1b0743 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -13,7 +13,6 @@ target_sources(ginkgo_dpcpp
     base/executor.dp.cpp
     components/absolute_array.dp.cpp
     components/fill_array.dp.cpp
-    components/precision_conversion.dp.cpp
     components/prefix_sum.dp.cpp
     factorization/ic_kernels.dp.cpp
     factorization/ilu_kernels.dp.cpp
@@ -37,7 +36,6 @@ target_sources(ginkgo_dpcpp
     reorder/rcm_kernels.dp.cpp
     solver/bicg_kernels.dp.cpp
     solver/bicgstab_kernels.dp.cpp
-    solver/cg_kernels.dp.cpp
     solver/cgs_kernels.dp.cpp
     solver/fcg_kernels.dp.cpp
     solver/gmres_kernels.dp.cpp
@@ -47,7 +45,10 @@ target_sources(ginkgo_dpcpp
     solver/lower_trs_kernels.dp.cpp
     solver/upper_trs_kernels.dp.cpp
     stop/criterion_kernels.dp.cpp
-    stop/residual_norm_kernels.dp.cpp)
+    stop/residual_norm_kernels.dp.cpp
+    ../common/components/precision_conversion.cpp
+    ../common/solver/cg_kernels.cpp
+    )
 
 ginkgo_compile_features(ginkgo_dpcpp)
 target_compile_definitions(ginkgo_dpcpp PRIVATE GKO_COMPILING_DPCPP)
diff --git a/dpcpp/components/precision_conversion.dp.cpp b/dpcpp/components/precision_conversion.dp.cpp
deleted file mode 100644
index a4975b1f68d..00000000000
--- a/dpcpp/components/precision_conversion.dp.cpp
+++ /dev/null
@@ -1,60 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/components/precision_conversion.hpp"
-
-
-#include "core/base/simple_kernels.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace GKO_DEVICE_NAMESPACE {
-namespace components {
-
-
-template <typename SourceType, typename TargetType>
-void convert_precision(std::shared_ptr<const DefaultExecutor> exec,
-                       size_type size, const SourceType *in, TargetType *out)
-{
-    exec->run_kernel(
-        [] GKO_KERNEL(auto idx, auto in, auto out) { out[idx] = in[idx]; },
-        size, in, out);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_CONVERT_PRECISION_KERNEL);
-
-
-}  // namespace components
-}  // namespace GKO_DEVICE_NAMESPACE
-}  // namespace kernels
-}  // namespace gko
diff --git a/dpcpp/solver/cg_kernels.dp.cpp b/dpcpp/solver/cg_kernels.dp.cpp
deleted file mode 100644
index 079e77fee9a..00000000000
--- a/dpcpp/solver/cg_kernels.dp.cpp
+++ /dev/null
@@ -1,129 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/cg_kernels.hpp"
-
-
-#include <ginkgo/core/base/math.hpp>
-
-
-#include "core/base/simple_kernels.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace GKO_DEVICE_NAMESPACE {
-/**
- * @brief The CG solver namespace.
- *
- * @ingroup cg
- */
-namespace cg {
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const DefaultExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
-                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
-                matrix::Dense<ValueType> *rho,
-                Array<stopping_status> *stop_status)
-{
-    exec->run_kernel(
-        [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto z, auto p,
-                      auto q, auto prev_rho, auto rho, auto stop) {
-            if (row == 0) {
-                rho[col] = zero(rho[col]);
-                prev_rho[col] = one(prev_rho[col]);
-                stop[col].reset();
-            }
-            r(row, col) = b(row, col);
-            z(row, col) = zero(z(row, col));
-            p(row, col) = zero(p(row, col));
-            q(row, col) = zero(q(row, col));
-        },
-        p->get_size(), b, r, z, p, q, prev_rho, rho, *stop_status);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const DefaultExecutor> exec,
-            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
-            const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *prev_rho,
-            const Array<stopping_status> *stop_status)
-{
-    exec->run_kernel(
-        [] GKO_KERNEL(auto row, auto col, auto p, auto z, auto rho,
-                      auto prev_rho, auto stop) {
-            if (!stop[col].has_stopped() &&
-                prev_rho[col] != zero(prev_rho[col])) {
-                auto tmp = rho[col] / prev_rho[col];
-                p(row, col) = z(row, col) + tmp * p(row, col);
-            }
-        },
-        p->get_size(), p, z, rho, prev_rho, *stop_status);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const DefaultExecutor> exec,
-            matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
-            const matrix::Dense<ValueType> *p,
-            const matrix::Dense<ValueType> *q,
-            const matrix::Dense<ValueType> *beta,
-            const matrix::Dense<ValueType> *rho,
-            const Array<stopping_status> *stop_status)
-{
-    exec->run_kernel(
-        [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto p, auto q,
-                      auto beta, auto rho, auto stop) {
-            if (!stop[col].has_stopped() && beta[col] != zero(beta[col])) {
-                auto tmp = rho[col] / beta[col];
-                x(row, col) += tmp * p(row, col);
-                r(row, col) -= tmp * q(row, col);
-            }
-        },
-        x->get_size(), x, r, p, q, beta, rho, *stop_status);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
-
-
-}  // namespace cg
-}  // namespace GKO_DEVICE_NAMESPACE
-}  // namespace kernels
-}  // namespace gko
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 0c64d20ce51..9277a3e2ad1 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -149,7 +149,6 @@ set(GINKGO_HIP_SOURCES
     base/version.hip.cpp
     components/absolute_array.hip.cpp
     components/fill_array.hip.cpp
-    components/precision_conversion.hip.cpp
     components/prefix_sum.hip.cpp
     factorization/factorization_kernels.hip.cpp
     factorization/ic_kernels.hip.cpp
@@ -181,7 +180,6 @@ set(GINKGO_HIP_SOURCES
     reorder/rcm_kernels.hip.cpp
     solver/bicg_kernels.hip.cpp
     solver/bicgstab_kernels.hip.cpp
-    solver/cg_kernels.hip.cpp
     solver/cgs_kernels.hip.cpp
     solver/fcg_kernels.hip.cpp
     solver/gmres_kernels.hip.cpp
@@ -191,7 +189,10 @@ set(GINKGO_HIP_SOURCES
     solver/lower_trs_kernels.hip.cpp
     solver/upper_trs_kernels.hip.cpp
     stop/criterion_kernels.hip.cpp
-    stop/residual_norm_kernels.hip.cpp)
+    stop/residual_norm_kernels.hip.cpp
+    ../common/components/precision_conversion.cpp
+    ../common/solver/cg_kernels.cpp
+    )
 
 set(GINKGO_HIP_NVCC_ARCH "")
 if (GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}")
diff --git a/hip/components/precision_conversion.hip.cpp b/hip/components/precision_conversion.hip.cpp
deleted file mode 100644
index a4975b1f68d..00000000000
--- a/hip/components/precision_conversion.hip.cpp
+++ /dev/null
@@ -1,60 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/components/precision_conversion.hpp"
-
-
-#include "core/base/simple_kernels.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace GKO_DEVICE_NAMESPACE {
-namespace components {
-
-
-template <typename SourceType, typename TargetType>
-void convert_precision(std::shared_ptr<const DefaultExecutor> exec,
-                       size_type size, const SourceType *in, TargetType *out)
-{
-    exec->run_kernel(
-        [] GKO_KERNEL(auto idx, auto in, auto out) { out[idx] = in[idx]; },
-        size, in, out);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_CONVERT_PRECISION_KERNEL);
-
-
-}  // namespace components
-}  // namespace GKO_DEVICE_NAMESPACE
-}  // namespace kernels
-}  // namespace gko
diff --git a/hip/solver/cg_kernels.hip.cpp b/hip/solver/cg_kernels.hip.cpp
deleted file mode 100644
index 079e77fee9a..00000000000
--- a/hip/solver/cg_kernels.hip.cpp
+++ /dev/null
@@ -1,129 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/cg_kernels.hpp"
-
-
-#include <ginkgo/core/base/math.hpp>
-
-
-#include "core/base/simple_kernels.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace GKO_DEVICE_NAMESPACE {
-/**
- * @brief The CG solver namespace.
- *
- * @ingroup cg
- */
-namespace cg {
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const DefaultExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
-                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
-                matrix::Dense<ValueType> *rho,
-                Array<stopping_status> *stop_status)
-{
-    exec->run_kernel(
-        [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto z, auto p,
-                      auto q, auto prev_rho, auto rho, auto stop) {
-            if (row == 0) {
-                rho[col] = zero(rho[col]);
-                prev_rho[col] = one(prev_rho[col]);
-                stop[col].reset();
-            }
-            r(row, col) = b(row, col);
-            z(row, col) = zero(z(row, col));
-            p(row, col) = zero(p(row, col));
-            q(row, col) = zero(q(row, col));
-        },
-        p->get_size(), b, r, z, p, q, prev_rho, rho, *stop_status);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const DefaultExecutor> exec,
-            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
-            const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *prev_rho,
-            const Array<stopping_status> *stop_status)
-{
-    exec->run_kernel(
-        [] GKO_KERNEL(auto row, auto col, auto p, auto z, auto rho,
-                      auto prev_rho, auto stop) {
-            if (!stop[col].has_stopped() &&
-                prev_rho[col] != zero(prev_rho[col])) {
-                auto tmp = rho[col] / prev_rho[col];
-                p(row, col) = z(row, col) + tmp * p(row, col);
-            }
-        },
-        p->get_size(), p, z, rho, prev_rho, *stop_status);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const DefaultExecutor> exec,
-            matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
-            const matrix::Dense<ValueType> *p,
-            const matrix::Dense<ValueType> *q,
-            const matrix::Dense<ValueType> *beta,
-            const matrix::Dense<ValueType> *rho,
-            const Array<stopping_status> *stop_status)
-{
-    exec->run_kernel(
-        [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto p, auto q,
-                      auto beta, auto rho, auto stop) {
-            if (!stop[col].has_stopped() && beta[col] != zero(beta[col])) {
-                auto tmp = rho[col] / beta[col];
-                x(row, col) += tmp * p(row, col);
-                r(row, col) -= tmp * q(row, col);
-            }
-        },
-        x->get_size(), x, r, p, q, beta, rho, *stop_status);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
-
-
-}  // namespace cg
-}  // namespace GKO_DEVICE_NAMESPACE
-}  // namespace kernels
-}  // namespace gko
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index d7a8795df4a..9dd7ccbe26a 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -6,7 +6,6 @@ target_sources(ginkgo_omp
     base/version.cpp
     components/absolute_array.cpp
     components/fill_array.cpp
-    components/precision_conversion.cpp
     components/prefix_sum.cpp
     factorization/factorization_kernels.cpp
     factorization/ic_kernels.cpp
@@ -30,7 +29,6 @@ target_sources(ginkgo_omp
     reorder/rcm_kernels.cpp
     solver/bicg_kernels.cpp
     solver/bicgstab_kernels.cpp
-    solver/cg_kernels.cpp
     solver/cgs_kernels.cpp
     solver/fcg_kernels.cpp
     solver/gmres_kernels.cpp
@@ -40,7 +38,10 @@ target_sources(ginkgo_omp
     solver/lower_trs_kernels.cpp
     solver/upper_trs_kernels.cpp
     stop/criterion_kernels.cpp
-    stop/residual_norm_kernels.cpp)
+    stop/residual_norm_kernels.cpp
+    ../common/components/precision_conversion.cpp
+    ../common/solver/cg_kernels.cpp
+    )
 
 ginkgo_compile_features(ginkgo_omp)
 target_compile_definitions(ginkgo_omp PRIVATE GKO_COMPILING_OMP)

From 43023896a9d66dc636cd46184f3c7d4c2b9e26d1 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 21 May 2021 16:05:45 +0200
Subject: [PATCH 0818/1008] make simple kernel launchers free functions

---
 common/components/precision_conversion.cpp |  3 +-
 common/solver/cg_kernels.cpp               |  9 ++++--
 cuda/base/kernel_launch.cuh                | 11 +++++---
 dpcpp/base/kernel_launch.dp.hpp            | 12 ++++----
 hip/base/kernel_launch.hip.hpp             | 11 +++++---
 include/ginkgo/core/base/executor.hpp      | 32 ----------------------
 omp/base/kernel_launch.hpp                 |  8 +++---
 7 files changed, 32 insertions(+), 54 deletions(-)

diff --git a/common/components/precision_conversion.cpp b/common/components/precision_conversion.cpp
index a4975b1f68d..9982ea71c10 100644
--- a/common/components/precision_conversion.cpp
+++ b/common/components/precision_conversion.cpp
@@ -46,7 +46,8 @@ template <typename SourceType, typename TargetType>
 void convert_precision(std::shared_ptr<const DefaultExecutor> exec,
                        size_type size, const SourceType *in, TargetType *out)
 {
-    exec->run_kernel(
+    run_kernel(
+        exec,
         [] GKO_KERNEL(auto idx, auto in, auto out) { out[idx] = in[idx]; },
         size, in, out);
 }
diff --git a/common/solver/cg_kernels.cpp b/common/solver/cg_kernels.cpp
index 079e77fee9a..82bdba4f4a5 100644
--- a/common/solver/cg_kernels.cpp
+++ b/common/solver/cg_kernels.cpp
@@ -58,7 +58,8 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 matrix::Dense<ValueType> *rho,
                 Array<stopping_status> *stop_status)
 {
-    exec->run_kernel(
+    run_kernel(
+        exec,
         [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto z, auto p,
                       auto q, auto prev_rho, auto rho, auto stop) {
             if (row == 0) {
@@ -84,7 +85,8 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *prev_rho,
             const Array<stopping_status> *stop_status)
 {
-    exec->run_kernel(
+    run_kernel(
+        exec,
         [] GKO_KERNEL(auto row, auto col, auto p, auto z, auto rho,
                       auto prev_rho, auto stop) {
             if (!stop[col].has_stopped() &&
@@ -108,7 +110,8 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *rho,
             const Array<stopping_status> *stop_status)
 {
-    exec->run_kernel(
+    run_kernel(
+        exec,
         [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto p, auto q,
                       auto beta, auto rho, auto stop) {
             if (!stop[col].has_stopped() && beta[col] != zero(beta[col])) {
diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index 9ed3d0350fd..375891b3a3c 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "cuda/base/device_guard.hpp"
 #include "cuda/base/types.hpp"
 #include "cuda/components/thread_ids.cuh"
 
@@ -150,9 +151,10 @@ const cuda_type<ValueType> *map_to_device(const Array<ValueType> &mtx)
 
 
 template <typename KernelFunction, typename... KernelArgs>
-void CudaExecutor::run_kernel(KernelFunction fn, size_type size,
-                              KernelArgs &&... args) const
+void run_kernel(std::shared_ptr<const CudaExecutor> exec, KernelFunction fn,
+                size_type size, KernelArgs &&... args)
 {
+    cuda::device_guard guard{exec->get_device_id()};
     constexpr auto block_size = kernels::cuda::default_block_size;
     auto num_blocks = ceildiv(size, block_size);
     kernels::cuda::generic_kernel_1d<<<num_blocks, block_size>>>(
@@ -161,9 +163,10 @@ void CudaExecutor::run_kernel(KernelFunction fn, size_type size,
 
 
 template <typename KernelFunction, typename... KernelArgs>
-void CudaExecutor::run_kernel(KernelFunction fn, dim<2> size,
-                              KernelArgs &&... args) const
+void run_kernel(std::shared_ptr<const CudaExecutor> exec, KernelFunction fn,
+                dim<2> size, KernelArgs &&... args)
 {
+    cuda::device_guard guard{exec->get_device_id()};
     constexpr auto block_size = kernels::cuda::default_block_size;
     auto num_blocks = ceildiv(size[0] * size[1], block_size);
     kernels::cuda::generic_kernel_2d<<<num_blocks, block_size>>>(
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
index 0f23ef872fe..ea71123e794 100644
--- a/dpcpp/base/kernel_launch.dp.hpp
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -141,10 +141,10 @@ const ValueType *map_to_device(const Array<ValueType> &mtx)
 
 
 template <typename KernelFunction, typename... KernelArgs>
-void DpcppExecutor::run_kernel(KernelFunction fn, size_type size,
-                               KernelArgs &&... args) const
+void run_kernel(std::shared_ptr<const DpcppExecutor> exec, KernelFunction fn,
+                size_type size, KernelArgs &&... args)
 {
-    this->get_queue()->submit([&](sycl::handler &cgh) {
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
         kernels::dpcpp::generic_kernel_1d(
             cgh, size, fn, kernels::dpcpp::map_to_device(args)...);
     });
@@ -152,10 +152,10 @@ void DpcppExecutor::run_kernel(KernelFunction fn, size_type size,
 
 
 template <typename KernelFunction, typename... KernelArgs>
-void DpcppExecutor::run_kernel(KernelFunction fn, dim<2> size,
-                               KernelArgs &&... args) const
+void run_kernel(std::shared_ptr<const DpcppExecutor> exec, KernelFunction fn,
+                dim<2> size, KernelArgs &&... args)
 {
-    this->get_queue()->submit([&](sycl::handler &cgh) {
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
         kernels::dpcpp::generic_kernel_2d(
             cgh, size[0], size[1], fn, kernels::dpcpp::map_to_device(args)...);
     });
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index 00e1b29da99..0c090fa98fe 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "hip/base/device_guard.hip.hpp"
 #include "hip/base/types.hip.hpp"
 #include "hip/components/thread_ids.hip.hpp"
 
@@ -153,9 +154,10 @@ const hip_type<ValueType> *map_to_device(const Array<ValueType> &mtx)
 
 
 template <typename KernelFunction, typename... KernelArgs>
-void HipExecutor::run_kernel(KernelFunction fn, size_type size,
-                             KernelArgs &&... args) const
+void run_kernel(std::shared_ptr<const HipExecutor> exec, KernelFunction fn,
+                size_type size, KernelArgs &&... args)
 {
+    hip::device_guard guard{exec->get_device_id()};
     constexpr auto block_size = kernels::hip::default_block_size;
     auto num_blocks = ceildiv(size, block_size);
     hipLaunchKernelGGL(kernels::hip::generic_kernel_1d, num_blocks, block_size,
@@ -164,9 +166,10 @@ void HipExecutor::run_kernel(KernelFunction fn, size_type size,
 
 
 template <typename KernelFunction, typename... KernelArgs>
-void HipExecutor::run_kernel(KernelFunction fn, dim<2> size,
-                             KernelArgs &&... args) const
+void run_kernel(std::shared_ptr<const HipExecutor> exec, KernelFunction fn,
+                dim<2> size, KernelArgs &&... args)
 {
+    hip::device_guard guard{exec->get_device_id()};
     constexpr auto block_size = kernels::hip::default_block_size;
     auto num_blocks = ceildiv(size[0] * size[1], block_size);
     hipLaunchKernelGGL(kernels::hip::generic_kernel_2d, num_blocks, block_size,
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index d2d1f506bc8..d9ca2e3af32 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -1174,14 +1174,6 @@ class OmpExecutor : public detail::ExecutorBase<OmpExecutor>,
         return this->get_exec_info().num_pu_per_cu;
     }
 
-    template <typename KernelFunction, typename... KernelArgs>
-    void run_kernel(KernelFunction fn, size_type size,
-                    KernelArgs &&... args) const;
-
-    template <typename KernelFunction, typename... KernelArgs>
-    void run_kernel(KernelFunction fn, dim<2> size,
-                    KernelArgs &&... args) const;
-
 protected:
     OmpExecutor()
     {
@@ -1313,14 +1305,6 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
     void run(const Operation &op) const override;
 
-    template <typename KernelFunction, typename... KernelArgs>
-    void run_kernel(KernelFunction fn, size_type size,
-                    KernelArgs &&... args) const;
-
-    template <typename KernelFunction, typename... KernelArgs>
-    void run_kernel(KernelFunction fn, dim<2> size,
-                    KernelArgs &&... args) const;
-
     /**
      * Get the CUDA device id of the device associated to this executor.
      */
@@ -1538,14 +1522,6 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
     void run(const Operation &op) const override;
 
-    template <typename KernelFunction, typename... KernelArgs>
-    void run_kernel(KernelFunction fn, size_type size,
-                    KernelArgs &&... args) const;
-
-    template <typename KernelFunction, typename... KernelArgs>
-    void run_kernel(KernelFunction fn, dim<2> size,
-                    KernelArgs &&... args) const;
-
     /**
      * Get the HIP device id of the device associated to this executor.
      */
@@ -1757,14 +1733,6 @@ class DpcppExecutor : public detail::ExecutorBase<DpcppExecutor>,
 
     void run(const Operation &op) const override;
 
-    template <typename KernelFunction, typename... KernelArgs>
-    void run_kernel(KernelFunction fn, size_type size,
-                    KernelArgs &&... args) const;
-
-    template <typename KernelFunction, typename... KernelArgs>
-    void run_kernel(KernelFunction fn, dim<2> size,
-                    KernelArgs &&... args) const;
-
     /**
      * Get the DPCPP device id of the device associated to this executor.
      *
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index a54ffa75588..2dbf8c5196a 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -115,8 +115,8 @@ const ValueType *map_to_device(const Array<ValueType> &mtx)
 
 
 template <typename KernelFunction, typename... KernelArgs>
-void OmpExecutor::run_kernel(KernelFunction fn, size_type size,
-                             KernelArgs &&... args) const
+void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
+                size_type size, KernelArgs &&... args)
 {
 #pragma omp parallel for
     for (size_type i = 0; i < size; i++) {
@@ -126,8 +126,8 @@ void OmpExecutor::run_kernel(KernelFunction fn, size_type size,
 
 
 template <typename KernelFunction, typename... KernelArgs>
-void OmpExecutor::run_kernel(KernelFunction fn, dim<2> size,
-                             KernelArgs &&... args) const
+void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
+                dim<2> size, KernelArgs &&... args)
 {
 #pragma omp parallel for
     for (size_type i = 0; i < size[0] * size[1]; i++) {

From 187e7eba006c30c99b6c60c023b5e0176a304c02 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 21 May 2021 20:56:02 +0200
Subject: [PATCH 0819/1008] make device mapping more flexible

---
 cuda/base/kernel_launch.cuh     | 70 +++++++++++++++++----------------
 dpcpp/base/kernel_launch.dp.hpp | 68 +++++++++++++++++---------------
 hip/base/kernel_launch.hip.hpp  | 70 +++++++++++++++++----------------
 omp/base/kernel_launch.hpp      | 68 +++++++++++++++++---------------
 4 files changed, 144 insertions(+), 132 deletions(-)

diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index 375891b3a3c..807442cce70 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -98,51 +98,53 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
 }
 
 
-template <typename ValueType>
-matrix_accessor<cuda_type<ValueType>> map_to_device(
-    matrix::Dense<ValueType> *mtx)
-{
-    return {as_cuda_type(mtx->get_values()), mtx->get_stride()};
-}
+template <typename T>
+struct device_map_impl {
+    using type = std::decay_t<cuda_type<T>>;
+    static type map_to_device(T in) { return as_cuda_type(in); }
+};
 
 template <typename ValueType>
-matrix_accessor<const cuda_type<ValueType>> map_to_device(
-    const matrix::Dense<ValueType> *mtx)
-{
-    return {as_cuda_type(mtx->get_const_values()), mtx->get_stride()};
-}
+struct device_map_impl<matrix::Dense<ValueType> *&> {
+    using type = matrix_accessor<cuda_type<ValueType>>;
+    static type map_to_device(matrix::Dense<ValueType> *mtx)
+    {
+        return {as_cuda_type(mtx->get_values()), mtx->get_stride()};
+    }
+};
 
 template <typename ValueType>
-typename std::enable_if<std::is_arithmetic<ValueType>::value,
-                        cuda_type<ValueType>>::type *
-map_to_device(ValueType *data)
-{
-    return as_cuda_type(data);
-}
+struct device_map_impl<const matrix::Dense<ValueType> *&> {
+    using type = matrix_accessor<const cuda_type<ValueType>>;
+    static type map_to_device(const matrix::Dense<ValueType> *mtx)
+    {
+        return {as_cuda_type(mtx->get_const_values()), mtx->get_stride()};
+    }
+};
 
 template <typename ValueType>
-cuda_type<std::complex<ValueType>> *map_to_device(std::complex<ValueType> *data)
-{
-    return as_cuda_type(data);
-}
+struct device_map_impl<Array<ValueType> &> {
+    using type = cuda_type<ValueType> *;
+    static type map_to_device(Array<ValueType> &array)
+    {
+        return as_cuda_type(array.get_data());
+    }
+};
 
 template <typename ValueType>
-const cuda_type<std::complex<ValueType>> *map_to_device(
-    const std::complex<ValueType> *data)
-{
-    return as_cuda_type(data);
-}
+struct device_map_impl<const Array<ValueType> &> {
+    using type = const cuda_type<ValueType> *;
+    static type map_to_device(const Array<ValueType> &array)
+    {
+        return as_cuda_type(array.get_const_data());
+    }
+};
 
-template <typename ValueType>
-cuda_type<ValueType> *map_to_device(Array<ValueType> &mtx)
-{
-    return as_cuda_type(mtx.get_data());
-}
 
-template <typename ValueType>
-const cuda_type<ValueType> *map_to_device(const Array<ValueType> &mtx)
+template <typename T>
+typename device_map_impl<T>::type map_to_device(T &&param)
 {
-    return as_cuda_type(mtx.get_const_data());
+    return device_map_impl<T>::map_to_device(param);
 }
 
 
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
index ea71123e794..3f724a6e2db 100644
--- a/dpcpp/base/kernel_launch.dp.hpp
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -90,49 +90,53 @@ void generic_kernel_2d(sycl::handler &cgh, size_type rows, size_type cols,
 }
 
 
-template <typename ValueType>
-matrix_accessor<ValueType> map_to_device(matrix::Dense<ValueType> *mtx)
-{
-    return {mtx->get_values(), mtx->get_stride()};
-}
+template <typename T>
+struct device_map_impl {
+    using type = std::decay_t<T>;
+    static type map_to_device(T in) { return in; }
+};
 
 template <typename ValueType>
-matrix_accessor<const ValueType> map_to_device(
-    const matrix::Dense<ValueType> *mtx)
-{
-    return {mtx->get_const_values(), mtx->get_stride()};
-}
+struct device_map_impl<matrix::Dense<ValueType> *&> {
+    using type = matrix_accessor<ValueType>;
+    static type map_to_device(matrix::Dense<ValueType> *mtx)
+    {
+        return {mtx->get_values(), mtx->get_stride()};
+    }
+};
 
 template <typename ValueType>
-typename std::enable_if<std::is_arithmetic<ValueType>::value, ValueType>::type *
-map_to_device(ValueType *data)
-{
-    return data;
-}
+struct device_map_impl<const matrix::Dense<ValueType> *&> {
+    using type = matrix_accessor<const ValueType>;
+    static type map_to_device(const matrix::Dense<ValueType> *mtx)
+    {
+        return {mtx->get_const_values(), mtx->get_stride()};
+    }
+};
 
 template <typename ValueType>
-std::complex<ValueType> *map_to_device(std::complex<ValueType> *data)
-{
-    return data;
-}
+struct device_map_impl<Array<ValueType> &> {
+    using type = ValueType *;
+    static type map_to_device(Array<ValueType> &array)
+    {
+        return array.get_data();
+    }
+};
 
 template <typename ValueType>
-const std::complex<ValueType> *map_to_device(
-    const std::complex<ValueType> *data)
-{
-    return data;
-}
+struct device_map_impl<const Array<ValueType> &> {
+    using type = const ValueType *;
+    static type map_to_device(const Array<ValueType> &array)
+    {
+        return array.get_const_data();
+    }
+};
 
-template <typename ValueType>
-ValueType *map_to_device(Array<ValueType> &mtx)
-{
-    return mtx.get_data();
-}
 
-template <typename ValueType>
-const ValueType *map_to_device(const Array<ValueType> &mtx)
+template <typename T>
+typename device_map_impl<T>::type map_to_device(T &&param)
 {
-    return mtx.get_const_data();
+    return device_map_impl<T>::map_to_device(param);
 }
 
 
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index 0c090fa98fe..48cf044130e 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -101,51 +101,53 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
 }
 
 
-template <typename ValueType>
-matrix_accessor<hip_type<ValueType>> map_to_device(
-    matrix::Dense<ValueType> *mtx)
-{
-    return {as_hip_type(mtx->get_values()), mtx->get_stride()};
-}
+template <typename T>
+struct device_map_impl {
+    using type = std::decay_t<hip_type<T>>;
+    static type map_to_device(T in) { return as_hip_type(in); }
+};
 
 template <typename ValueType>
-matrix_accessor<const hip_type<ValueType>> map_to_device(
-    const matrix::Dense<ValueType> *mtx)
-{
-    return {as_hip_type(mtx->get_const_values()), mtx->get_stride()};
-}
+struct device_map_impl<matrix::Dense<ValueType> *&> {
+    using type = matrix_accessor<hip_type<ValueType>>;
+    static type map_to_device(matrix::Dense<ValueType> *mtx)
+    {
+        return {as_hip_type(mtx->get_values()), mtx->get_stride()};
+    }
+};
 
 template <typename ValueType>
-typename std::enable_if<std::is_arithmetic<ValueType>::value,
-                        hip_type<ValueType>>::type *
-map_to_device(ValueType *data)
-{
-    return as_hip_type(data);
-}
+struct device_map_impl<const matrix::Dense<ValueType> *&> {
+    using type = matrix_accessor<const hip_type<ValueType>>;
+    static type map_to_device(const matrix::Dense<ValueType> *mtx)
+    {
+        return {as_hip_type(mtx->get_const_values()), mtx->get_stride()};
+    }
+};
 
 template <typename ValueType>
-hip_type<std::complex<ValueType>> *map_to_device(std::complex<ValueType> *data)
-{
-    return as_hip_type(data);
-}
+struct device_map_impl<Array<ValueType> &> {
+    using type = hip_type<ValueType> *;
+    static type map_to_device(Array<ValueType> &array)
+    {
+        return as_hip_type(array.get_data());
+    }
+};
 
 template <typename ValueType>
-const hip_type<std::complex<ValueType>> *map_to_device(
-    const std::complex<ValueType> *data)
-{
-    return as_hip_type(data);
-}
+struct device_map_impl<const Array<ValueType> &> {
+    using type = const hip_type<ValueType> *;
+    static type map_to_device(const Array<ValueType> &array)
+    {
+        return as_hip_type(array.get_const_data());
+    }
+};
 
-template <typename ValueType>
-hip_type<ValueType> *map_to_device(Array<ValueType> &mtx)
-{
-    return as_hip_type(mtx.get_data());
-}
 
-template <typename ValueType>
-const hip_type<ValueType> *map_to_device(const Array<ValueType> &mtx)
+template <typename T>
+typename device_map_impl<T>::type map_to_device(T &&param)
 {
-    return as_hip_type(mtx.get_const_data());
+    return device_map_impl<T>::map_to_device(param);
 }
 
 
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index 2dbf8c5196a..22fe67742b7 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -64,49 +64,53 @@ struct matrix_accessor {
 };
 
 
-template <typename ValueType>
-matrix_accessor<ValueType> map_to_device(matrix::Dense<ValueType> *mtx)
-{
-    return {mtx->get_values(), mtx->get_stride()};
-}
+template <typename T>
+struct device_map_impl {
+    using type = std::decay_t<T>;
+    static type map_to_device(T in) { return in; }
+};
 
 template <typename ValueType>
-matrix_accessor<const ValueType> map_to_device(
-    const matrix::Dense<ValueType> *mtx)
-{
-    return {mtx->get_const_values(), mtx->get_stride()};
-}
+struct device_map_impl<matrix::Dense<ValueType> *&> {
+    using type = matrix_accessor<ValueType>;
+    static type map_to_device(matrix::Dense<ValueType> *mtx)
+    {
+        return {mtx->get_values(), mtx->get_stride()};
+    }
+};
 
 template <typename ValueType>
-typename std::enable_if<std::is_arithmetic<ValueType>::value, ValueType>::type *
-map_to_device(ValueType *data)
-{
-    return data;
-}
+struct device_map_impl<const matrix::Dense<ValueType> *&> {
+    using type = matrix_accessor<const ValueType>;
+    static type map_to_device(const matrix::Dense<ValueType> *mtx)
+    {
+        return {mtx->get_const_values(), mtx->get_stride()};
+    }
+};
 
 template <typename ValueType>
-std::complex<ValueType> *map_to_device(std::complex<ValueType> *data)
-{
-    return data;
-}
+struct device_map_impl<Array<ValueType> &> {
+    using type = ValueType *;
+    static type map_to_device(Array<ValueType> &array)
+    {
+        return array.get_data();
+    }
+};
 
 template <typename ValueType>
-const std::complex<ValueType> *map_to_device(
-    const std::complex<ValueType> *data)
-{
-    return data;
-}
+struct device_map_impl<const Array<ValueType> &> {
+    using type = const ValueType *;
+    static type map_to_device(const Array<ValueType> &array)
+    {
+        return array.get_const_data();
+    }
+};
 
-template <typename ValueType>
-ValueType *map_to_device(Array<ValueType> &mtx)
-{
-    return mtx.get_data();
-}
 
-template <typename ValueType>
-const ValueType *map_to_device(const Array<ValueType> &mtx)
+template <typename T>
+typename device_map_impl<T>::type map_to_device(T &&param)
 {
-    return mtx.get_const_data();
+    return device_map_impl<T>::map_to_device(param);
 }
 
 

From 52ebda7331bb8f3bc43ba6d9cfeb145a36f7f535 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 21 May 2021 20:56:29 +0200
Subject: [PATCH 0820/1008] move dense operations to simple interface

---
 common/matrix/dense_kernels.cpp     | 308 ++++++++++++++++++++++++++++
 common/matrix/dense_kernels.hpp.inc | 234 ---------------------
 cuda/CMakeLists.txt                 |   1 +
 cuda/matrix/dense_kernels.cu        | 262 -----------------------
 dpcpp/CMakeLists.txt                |   1 +
 dpcpp/matrix/dense_kernels.dp.cpp   | 140 -------------
 hip/CMakeLists.txt                  |   1 +
 hip/matrix/dense_kernels.hip.cpp    | 281 -------------------------
 omp/CMakeLists.txt                  |   1 +
 omp/matrix/dense_kernels.cpp        | 269 ------------------------
 10 files changed, 312 insertions(+), 1186 deletions(-)
 create mode 100644 common/matrix/dense_kernels.cpp

diff --git a/common/matrix/dense_kernels.cpp b/common/matrix/dense_kernels.cpp
new file mode 100644
index 00000000000..a8da0c3a8f1
--- /dev/null
+++ b/common/matrix/dense_kernels.cpp
@@ -0,0 +1,308 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/dense_kernels.hpp"
+
+
+#include <ginkgo/core/base/math.hpp>
+
+
+#include "core/base/simple_kernels.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace GKO_DEVICE_NAMESPACE {
+/**
+ * @brief The Dense matrix format namespace.
+ *
+ * @ingroup dense
+ */
+namespace dense {
+
+
+template <typename InValueType, typename OutValueType>
+void copy(std::shared_ptr<const DefaultExecutor> exec,
+          const matrix::Dense<InValueType> *input,
+          matrix::Dense<OutValueType> *output)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto input, auto output) {
+            output(row, col) = input(row, col);
+        },
+        input->get_size(), input, output);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(
+    GKO_DECLARE_DENSE_COPY_KERNEL);
+
+
+template <typename ValueType>
+void fill(std::shared_ptr<const DefaultExecutor> exec,
+          matrix::Dense<ValueType> *mat, ValueType value)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto mat, auto value) {
+            mat(row, col) = value;
+        },
+        mat->get_size(), mat, value);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_FILL_KERNEL);
+
+
+template <typename ValueType>
+void add_scaled_diag(std::shared_ptr<const DefaultExecutor> exec,
+                     const matrix::Dense<ValueType> *alpha,
+                     const matrix::Diagonal<ValueType> *x,
+                     matrix::Dense<ValueType> *y)
+{
+    const auto diag_values = x->get_const_values();
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto alpha, auto diag, auto y) {
+            y(i, i) += alpha(0, 0) * diag[i];
+        },
+        x->get_size()[0], alpha, x->get_const_values(), y);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void symm_permute(std::shared_ptr<const DefaultExecutor> exec,
+                  const Array<IndexType> *permutation_indices,
+                  const matrix::Dense<ValueType> *orig,
+                  matrix::Dense<ValueType> *permuted)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto orig, auto perm, auto permuted) {
+            permuted(row, col) = orig(perm[row], perm[col]);
+        },
+        orig->get_size(), orig, *permutation_indices, permuted);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inv_symm_permute(std::shared_ptr<const DefaultExecutor> exec,
+                      const Array<IndexType> *permutation_indices,
+                      const matrix::Dense<ValueType> *orig,
+                      matrix::Dense<ValueType> *permuted)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto orig, auto perm, auto permuted) {
+            permuted(perm[row], perm[col]) = orig(row, col);
+        },
+        orig->get_size(), orig, *permutation_indices, permuted);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void row_gather(std::shared_ptr<const DefaultExecutor> exec,
+                const Array<IndexType> *row_indices,
+                const matrix::Dense<ValueType> *orig,
+                matrix::Dense<ValueType> *row_gathered)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto orig, auto rows, auto gathered) {
+            gathered(row, col) = orig(rows[row], col);
+        },
+        dim<2>{row_indices->get_num_elems(), orig->get_size()[1]}, orig,
+        *row_indices, row_gathered);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_ROW_GATHER_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void column_permute(std::shared_ptr<const DefaultExecutor> exec,
+                    const Array<IndexType> *permutation_indices,
+                    const matrix::Dense<ValueType> *orig,
+                    matrix::Dense<ValueType> *column_permuted)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto orig, auto perm, auto permuted) {
+            permuted(row, col) = orig(row, perm[col]);
+        },
+        orig->get_size(), orig, *permutation_indices, column_permuted);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_row_permute(std::shared_ptr<const DefaultExecutor> exec,
+                         const Array<IndexType> *permutation_indices,
+                         const matrix::Dense<ValueType> *orig,
+                         matrix::Dense<ValueType> *row_permuted)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto orig, auto perm, auto permuted) {
+            permuted(perm[row], col) = orig(row, col);
+        },
+        orig->get_size(), orig, *permutation_indices, row_permuted);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_column_permute(std::shared_ptr<const DefaultExecutor> exec,
+                            const Array<IndexType> *permutation_indices,
+                            const matrix::Dense<ValueType> *orig,
+                            matrix::Dense<ValueType> *column_permuted)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto orig, auto perm, auto permuted) {
+            permuted(row, perm[col]) = orig(row, col);
+        },
+        orig->get_size(), orig, *permutation_indices, column_permuted);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL);
+
+
+template <typename ValueType>
+void extract_diagonal(std::shared_ptr<const DefaultExecutor> exec,
+                      const matrix::Dense<ValueType> *orig,
+                      matrix::Diagonal<ValueType> *diag)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto orig, auto diag) { diag[i] = orig(i, i); },
+        diag->get_size()[0], orig, diag->get_values());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL);
+
+
+template <typename ValueType>
+void inplace_absolute_dense(std::shared_ptr<const DefaultExecutor> exec,
+                            matrix::Dense<ValueType> *source)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto source) {
+            source(row, col) = abs(source(row, col));
+        },
+        source->get_size(), source);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL);
+
+
+template <typename ValueType>
+void outplace_absolute_dense(std::shared_ptr<const DefaultExecutor> exec,
+                             const matrix::Dense<ValueType> *source,
+                             matrix::Dense<remove_complex<ValueType>> *result)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto source, auto result) {
+            result(row, col) = abs(source(row, col));
+        },
+        source->get_size(), source, result);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
+
+
+template <typename ValueType>
+void make_complex(std::shared_ptr<const DefaultExecutor> exec,
+                  const matrix::Dense<ValueType> *source,
+                  matrix::Dense<to_complex<ValueType>> *result)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto source, auto result) {
+            result(row, col) = source(row, col);
+        },
+        source->get_size(), source, result);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MAKE_COMPLEX_KERNEL);
+
+
+template <typename ValueType>
+void get_real(std::shared_ptr<const DefaultExecutor> exec,
+              const matrix::Dense<ValueType> *source,
+              matrix::Dense<remove_complex<ValueType>> *result)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto source, auto result) {
+            result(row, col) = real(source(row, col));
+        },
+        source->get_size(), source, result);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_REAL_KERNEL);
+
+
+template <typename ValueType>
+void get_imag(std::shared_ptr<const DefaultExecutor> exec,
+              const matrix::Dense<ValueType> *source,
+              matrix::Dense<remove_complex<ValueType>> *result)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto source, auto result) {
+            result(row, col) = imag(source(row, col));
+        },
+        source->get_size(), source, result);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_IMAG_KERNEL);
+
+
+}  // namespace dense
+}  // namespace GKO_DEVICE_NAMESPACE
+}  // namespace kernels
+}  // namespace gko
diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index 8bbba33fd6d..4f30fd4c109 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -33,36 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace kernel {
 
 
-template <typename InValueType, typename OutValueType>
-__global__ __launch_bounds__(default_block_size) void strided_copy(
-    size_type num_rows, size_type num_cols, size_type in_stride,
-    size_type out_stride, const InValueType *__restrict__ input,
-    OutValueType *__restrict__ output)
-{
-    const auto global_id = thread::get_thread_id_flat();
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        output[row_id * out_stride + col_id] =
-            static_cast<OutValueType>(input[row_id * in_stride + col_id]);
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void strided_fill(
-    size_type num_rows, size_type num_cols, size_type stride,
-    ValueType *__restrict__ mat, ValueType value)
-{
-    const auto global_id = thread::get_thread_id_flat();
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        mat[row_id * stride + col_id] = value;
-    }
-}
-
-
 template <size_type block_size, typename ValueType>
 __global__ __launch_bounds__(block_size) void scale(
     size_type num_rows, size_type num_cols, size_type num_alpha_cols,
@@ -103,22 +73,6 @@ __global__ __launch_bounds__(block_size) void add_scaled(
 }
 
 
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void add_scaled_diag(
-    size_type size, const ValueType *__restrict__ alpha,
-    const ValueType *__restrict__ diag, ValueType *__restrict__ y,
-    size_type stride_y)
-{
-    const auto tidx = thread::get_thread_id_flat();
-
-    if (tidx >= size) {
-        return;
-    }
-
-    y[tidx * stride_y + tidx] += alpha[0] * diag[tidx];
-}
-
-
 template <size_type block_size, typename OutType, typename CallableGetValue,
           typename CallableReduce>
 __device__ void compute_partial_reduce(size_type num_rows,
@@ -466,192 +420,4 @@ __global__ __launch_bounds__(default_block_size) void reduce_total_cols(
 }
 
 
-template <typename IndexType, typename ValueType>
-__global__ __launch_bounds__(default_block_size) void symm_permute(
-    size_type num_rows, size_type num_cols,
-    const IndexType *__restrict__ perm_idxs, const ValueType *__restrict__ orig,
-    size_type stride_orig, ValueType *__restrict__ result,
-    size_type stride_result)
-{
-    const auto global_id = thread::get_thread_id_flat();
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        result[row_id * stride_result + col_id] =
-            orig[perm_idxs[row_id] * stride_orig + perm_idxs[col_id]];
-    }
-}
-
-
-template <typename IndexType, typename ValueType>
-__global__ __launch_bounds__(default_block_size) void inv_symm_permute(
-    size_type num_rows, size_type num_cols,
-    const IndexType *__restrict__ perm_idxs, const ValueType *__restrict__ orig,
-    size_type stride_orig, ValueType *__restrict__ result,
-    size_type stride_result)
-{
-    const auto global_id = thread::get_thread_id_flat();
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        result[perm_idxs[row_id] * stride_result + perm_idxs[col_id]] =
-            orig[row_id * stride_orig + col_id];
-    }
-}
-
-
-template <typename IndexType, typename ValueType>
-__global__ __launch_bounds__(default_block_size) void row_gather(
-    size_type num_rows, size_type num_cols,
-    const IndexType *__restrict__ perm_idxs, const ValueType *__restrict__ orig,
-    size_type stride_orig, ValueType *__restrict__ result,
-    size_type stride_result)
-{
-    const auto global_id = thread::get_thread_id_flat();
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        result[row_id * stride_result + col_id] =
-            orig[perm_idxs[row_id] * stride_orig + col_id];
-    }
-}
-
-
-template <typename IndexType, typename ValueType>
-__global__ __launch_bounds__(default_block_size) void column_permute(
-    size_type num_rows, size_type num_cols,
-    const IndexType *__restrict__ perm_idxs, const ValueType *__restrict__ orig,
-    size_type stride_orig, ValueType *__restrict__ result,
-    size_type stride_result)
-{
-    const auto global_id = thread::get_thread_id_flat();
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        result[row_id * stride_result + col_id] =
-            orig[row_id * stride_orig + perm_idxs[col_id]];
-    }
-}
-
-
-template <typename IndexType, typename ValueType>
-__global__ __launch_bounds__(default_block_size) void inverse_row_permute(
-    size_type num_rows, size_type num_cols,
-    const IndexType *__restrict__ perm_idxs, const ValueType *__restrict__ orig,
-    size_type stride_orig, ValueType *__restrict__ result,
-    size_type stride_result)
-{
-    const auto global_id = thread::get_thread_id_flat();
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        result[perm_idxs[row_id] * stride_result + col_id] =
-            orig[row_id * stride_orig + col_id];
-    }
-}
-
-
-template <typename IndexType, typename ValueType>
-__global__ __launch_bounds__(default_block_size) void inverse_column_permute(
-    size_type num_rows, size_type num_cols,
-    const IndexType *__restrict__ perm_idxs, const ValueType *__restrict__ orig,
-    size_type stride_orig, ValueType *__restrict__ result,
-    size_type stride_result)
-{
-    const auto global_id = thread::get_thread_id_flat();
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        result[row_id * stride_result + perm_idxs[col_id]] =
-            orig[row_id * stride_orig + col_id];
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void extract_diagonal(
-    size_type problem_size, const ValueType *__restrict__ orig,
-    size_type stride_orig, ValueType *__restrict__ diag)
-{
-    const auto tidx = thread::get_thread_id_flat();
-
-    if (tidx < problem_size) {
-        diag[tidx] = orig[tidx * stride_orig + tidx];
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void inplace_absolute_dense(
-    size_type num_rows, size_type num_cols, ValueType *__restrict__ data,
-    size_type stride)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    auto row = tidx / num_cols;
-    auto col = tidx % num_cols;
-    if (row < num_rows) {
-        data[row * stride + col] = abs(data[row * stride + col]);
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void outplace_absolute_dense(
-    size_type num_rows, size_type num_cols, const ValueType *__restrict__ in,
-    size_type stride_in, remove_complex<ValueType> *__restrict__ out,
-    size_type stride_out)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    auto row = tidx / num_cols;
-    auto col = tidx % num_cols;
-    if (row < num_rows) {
-        out[row * stride_out + col] = abs(in[row * stride_in + col]);
-    }
-}
-
-
-template <typename ValueType, typename ComplexType>
-__global__ __launch_bounds__(default_block_size) void make_complex(
-    size_type num_rows, size_type num_cols, const ValueType *__restrict__ in,
-    size_type stride_in, ComplexType *__restrict__ out, size_type stride_out)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    auto row = tidx / num_cols;
-    auto col = tidx % num_cols;
-    if (row < num_rows) {
-        out[row * stride_out + col] = in[row * stride_in + col];
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void get_real(
-    size_type num_rows, size_type num_cols, const ValueType *__restrict__ in,
-    size_type stride_in, remove_complex<ValueType> *__restrict__ out,
-    size_type stride_out)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    auto row = tidx / num_cols;
-    auto col = tidx % num_cols;
-    if (row < num_rows) {
-        out[row * stride_out + col] = real(in[row * stride_in + col]);
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void get_imag(
-    size_type num_rows, size_type num_cols, const ValueType *__restrict__ in,
-    size_type stride_in, remove_complex<ValueType> *__restrict__ out,
-    size_type stride_out)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    auto row = tidx / num_cols;
-    auto col = tidx % num_cols;
-    if (row < num_rows) {
-        out[row * stride_out + col] = imag(in[row * stride_in + col]);
-    }
-}
-
-
 }  // namespace kernel
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 77fdc8cca17..86fcb315904 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -67,6 +67,7 @@ find_library(CURAND curand
 add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_cuda_device> "")
 set(GKO_CUDA_COMMON_SOURCES    
     ../common/components/precision_conversion.cpp
+    ../common/matrix/dense_kernels.cpp
     ../common/solver/cg_kernels.cpp
     )
 target_sources(ginkgo_cuda
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index 52caf38ddc0..4f06232866f 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -117,37 +117,6 @@ void apply(std::shared_ptr<const CudaExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
-template <typename InValueType, typename OutValueType>
-void copy(std::shared_ptr<const DefaultExecutor> exec,
-          const matrix::Dense<InValueType> *input,
-          matrix::Dense<OutValueType> *output)
-{
-    const auto num_blocks = ceildiv(input->get_size()[0] * input->get_size()[1],
-                                    default_block_size);
-    kernel::strided_copy<<<num_blocks, default_block_size>>>(
-        input->get_size()[0], input->get_size()[1], input->get_stride(),
-        output->get_stride(), as_cuda_type(input->get_const_values()),
-        as_cuda_type(output->get_values()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(
-    GKO_DECLARE_DENSE_COPY_KERNEL);
-
-
-template <typename ValueType>
-void fill(std::shared_ptr<const DefaultExecutor> exec,
-          matrix::Dense<ValueType> *mat, ValueType value)
-{
-    const auto num_blocks =
-        ceildiv(mat->get_size()[0] * mat->get_size()[1], default_block_size);
-    kernel::strided_fill<<<num_blocks, default_block_size>>>(
-        mat->get_size()[0], mat->get_size()[1], mat->get_stride(),
-        as_cuda_type(mat->get_values()), as_cuda_type(value));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_FILL_KERNEL);
-
-
 template <typename ValueType>
 void scale(std::shared_ptr<const CudaExecutor> exec,
            const matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *x)
@@ -200,24 +169,6 @@ void add_scaled(std::shared_ptr<const CudaExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_KERNEL);
 
 
-template <typename ValueType>
-void add_scaled_diag(std::shared_ptr<const CudaExecutor> exec,
-                     const matrix::Dense<ValueType> *alpha,
-                     const matrix::Diagonal<ValueType> *x,
-                     matrix::Dense<ValueType> *y)
-{
-    const auto size = y->get_size()[0];
-    const auto grid_dim = ceildiv(size, default_block_size);
-
-    kernel::add_scaled_diag<<<grid_dim, default_block_size>>>(
-        size, as_cuda_type(alpha->get_const_values()),
-        as_cuda_type(x->get_const_values()), as_cuda_type(y->get_values()),
-        y->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL);
-
-
 template <typename ValueType>
 void compute_dot(std::shared_ptr<const CudaExecutor> exec,
                  const matrix::Dense<ValueType> *x,
@@ -679,219 +630,6 @@ void conj_transpose(std::shared_ptr<const CudaExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void symm_permute(std::shared_ptr<const CudaExecutor> exec,
-                  const Array<IndexType> *permutation_indices,
-                  const matrix::Dense<ValueType> *orig,
-                  matrix::Dense<ValueType> *permuted)
-{
-    const auto num_blocks =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], default_block_size);
-    kernel::symm_permute<<<num_blocks, default_block_size>>>(
-        orig->get_size()[0], orig->get_size()[1],
-        permutation_indices->get_const_data(),
-        as_cuda_type(orig->get_const_values()), orig->get_stride(),
-        as_cuda_type(permuted->get_values()), permuted->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inv_symm_permute(std::shared_ptr<const CudaExecutor> exec,
-                      const Array<IndexType> *permutation_indices,
-                      const matrix::Dense<ValueType> *orig,
-                      matrix::Dense<ValueType> *permuted)
-{
-    const auto num_blocks =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], default_block_size);
-    kernel::inv_symm_permute<<<num_blocks, default_block_size>>>(
-        orig->get_size()[0], orig->get_size()[1],
-        permutation_indices->get_const_data(),
-        as_cuda_type(orig->get_const_values()), orig->get_stride(),
-        as_cuda_type(permuted->get_values()), permuted->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void row_gather(std::shared_ptr<const CudaExecutor> exec,
-                const Array<IndexType> *row_indices,
-                const matrix::Dense<ValueType> *orig,
-                matrix::Dense<ValueType> *row_gathered)
-{
-    auto out_num_rows = row_indices->get_num_elems();
-    const auto num_blocks =
-        ceildiv(out_num_rows * orig->get_size()[1], default_block_size);
-    kernel::row_gather<<<num_blocks, default_block_size>>>(
-        out_num_rows, orig->get_size()[1], row_indices->get_const_data(),
-        as_cuda_type(orig->get_const_values()), orig->get_stride(),
-        as_cuda_type(row_gathered->get_values()), row_gathered->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_ROW_GATHER_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void column_permute(std::shared_ptr<const CudaExecutor> exec,
-                    const Array<IndexType> *permutation_indices,
-                    const matrix::Dense<ValueType> *orig,
-                    matrix::Dense<ValueType> *column_permuted)
-{
-    const auto num_blocks =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], default_block_size);
-    kernel::column_permute<<<num_blocks, default_block_size>>>(
-        orig->get_size()[0], orig->get_size()[1],
-        permutation_indices->get_const_data(),
-        as_cuda_type(orig->get_const_values()), orig->get_stride(),
-        as_cuda_type(column_permuted->get_values()),
-        column_permuted->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_row_permute(std::shared_ptr<const CudaExecutor> exec,
-                         const Array<IndexType> *permutation_indices,
-                         const matrix::Dense<ValueType> *orig,
-                         matrix::Dense<ValueType> *row_permuted)
-{
-    const auto num_blocks =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], default_block_size);
-    kernel::inverse_row_permute<<<num_blocks, default_block_size>>>(
-        orig->get_size()[0], orig->get_size()[1],
-        permutation_indices->get_const_data(),
-        as_cuda_type(orig->get_const_values()), orig->get_stride(),
-        as_cuda_type(row_permuted->get_values()), row_permuted->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(std::shared_ptr<const CudaExecutor> exec,
-                            const Array<IndexType> *permutation_indices,
-                            const matrix::Dense<ValueType> *orig,
-                            matrix::Dense<ValueType> *column_permuted)
-{
-    const auto num_blocks =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], default_block_size);
-    kernel::inverse_column_permute<<<num_blocks, default_block_size>>>(
-        orig->get_size()[0], orig->get_size()[1],
-        permutation_indices->get_const_data(),
-        as_cuda_type(orig->get_const_values()), orig->get_stride(),
-        as_cuda_type(column_permuted->get_values()),
-        column_permuted->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL);
-
-
-template <typename ValueType>
-void extract_diagonal(std::shared_ptr<const CudaExecutor> exec,
-                      const matrix::Dense<ValueType> *orig,
-                      matrix::Diagonal<ValueType> *diag)
-{
-    const dim3 grid_dim = ceildiv(diag->get_size()[0], default_block_size);
-    kernel::extract_diagonal<<<grid_dim, default_block_size>>>(
-        diag->get_size()[0], as_cuda_type(orig->get_const_values()),
-        orig->get_stride(), as_cuda_type(diag->get_values()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL);
-
-
-template <typename ValueType>
-void inplace_absolute_dense(std::shared_ptr<const CudaExecutor> exec,
-                            matrix::Dense<ValueType> *source)
-{
-    auto dim = source->get_size();
-    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
-
-    kernel::inplace_absolute_dense<<<grid_dim, default_block_size>>>(
-        dim[0], dim[1], as_cuda_type(source->get_values()),
-        source->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL);
-
-
-template <typename ValueType>
-void outplace_absolute_dense(std::shared_ptr<const CudaExecutor> exec,
-                             const matrix::Dense<ValueType> *source,
-                             matrix::Dense<remove_complex<ValueType>> *result)
-{
-    auto dim = source->get_size();
-    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
-
-    kernel::outplace_absolute_dense<<<grid_dim, default_block_size>>>(
-        dim[0], dim[1], as_cuda_type(source->get_const_values()),
-        source->get_stride(), as_cuda_type(result->get_values()),
-        result->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
-
-
-template <typename ValueType>
-void make_complex(std::shared_ptr<const CudaExecutor> exec,
-                  const matrix::Dense<ValueType> *source,
-                  matrix::Dense<to_complex<ValueType>> *result)
-{
-    auto dim = source->get_size();
-    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
-
-    kernel::make_complex<<<grid_dim, default_block_size>>>(
-        dim[0], dim[1], as_cuda_type(source->get_const_values()),
-        source->get_stride(), as_cuda_type(result->get_values()),
-        result->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MAKE_COMPLEX_KERNEL);
-
-
-template <typename ValueType>
-void get_real(std::shared_ptr<const CudaExecutor> exec,
-              const matrix::Dense<ValueType> *source,
-              matrix::Dense<remove_complex<ValueType>> *result)
-{
-    auto dim = source->get_size();
-    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
-
-    kernel::get_real<<<grid_dim, default_block_size>>>(
-        dim[0], dim[1], as_cuda_type(source->get_const_values()),
-        source->get_stride(), as_cuda_type(result->get_values()),
-        result->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_REAL_KERNEL);
-
-
-template <typename ValueType>
-void get_imag(std::shared_ptr<const CudaExecutor> exec,
-              const matrix::Dense<ValueType> *source,
-              matrix::Dense<remove_complex<ValueType>> *result)
-{
-    auto dim = source->get_size();
-    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
-
-    kernel::get_imag<<<grid_dim, default_block_size>>>(
-        dim[0], dim[1], as_cuda_type(source->get_const_values()),
-        source->get_stride(), as_cuda_type(result->get_values()),
-        result->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_IMAG_KERNEL);
-
-
 }  // namespace dense
 }  // namespace cuda
 }  // namespace kernels
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index f03fc1b0743..853ccf5e215 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -47,6 +47,7 @@ target_sources(ginkgo_dpcpp
     stop/criterion_kernels.dp.cpp
     stop/residual_norm_kernels.dp.cpp
     ../common/components/precision_conversion.cpp
+    ../common/matrix/dense_kernels.cpp
     ../common/solver/cg_kernels.cpp
     )
 
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 48dcefbce00..2ea3c8db043 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -84,22 +84,6 @@ void apply(std::shared_ptr<const DpcppExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
-template <typename InValueType, typename OutValueType>
-void copy(std::shared_ptr<const DefaultExecutor> exec,
-          const matrix::Dense<InValueType> *input,
-          matrix::Dense<OutValueType> *output) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(
-    GKO_DECLARE_DENSE_COPY_KERNEL);
-
-
-template <typename ValueType>
-void fill(std::shared_ptr<const DefaultExecutor> exec,
-          matrix::Dense<ValueType> *mat, ValueType value) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_FILL_KERNEL);
-
-
 template <typename ValueType>
 void scale(std::shared_ptr<const DpcppExecutor> exec,
            const matrix::Dense<ValueType> *alpha,
@@ -117,15 +101,6 @@ void add_scaled(std::shared_ptr<const DpcppExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_KERNEL);
 
 
-template <typename ValueType>
-void add_scaled_diag(std::shared_ptr<const DpcppExecutor> exec,
-                     const matrix::Dense<ValueType> *alpha,
-                     const matrix::Diagonal<ValueType> *x,
-                     matrix::Dense<ValueType> *y) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL);
-
-
 template <typename ValueType>
 void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
                  const matrix::Dense<ValueType> *x,
@@ -265,121 +240,6 @@ void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void symm_permute(std::shared_ptr<const DpcppExecutor> exec,
-                  const Array<IndexType> *permutation_indices,
-                  const matrix::Dense<ValueType> *orig,
-                  matrix::Dense<ValueType> *permuted) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inv_symm_permute(std::shared_ptr<const DpcppExecutor> exec,
-                      const Array<IndexType> *permutation_indices,
-                      const matrix::Dense<ValueType> *orig,
-                      matrix::Dense<ValueType> *permuted) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void row_gather(std::shared_ptr<const DpcppExecutor> exec,
-                const Array<IndexType> *gather_indices,
-                const matrix::Dense<ValueType> *orig,
-                matrix::Dense<ValueType> *row_gathered) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_ROW_GATHER_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void column_permute(std::shared_ptr<const DpcppExecutor> exec,
-                    const Array<IndexType> *permutation_indices,
-                    const matrix::Dense<ValueType> *orig,
-                    matrix::Dense<ValueType> *column_permuted)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_row_permute(std::shared_ptr<const DpcppExecutor> exec,
-                         const Array<IndexType> *permutation_indices,
-                         const matrix::Dense<ValueType> *orig,
-                         matrix::Dense<ValueType> *row_permuted)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(std::shared_ptr<const DpcppExecutor> exec,
-                            const Array<IndexType> *permutation_indices,
-                            const matrix::Dense<ValueType> *orig,
-                            matrix::Dense<ValueType> *column_permuted)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL);
-
-
-template <typename ValueType>
-void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
-                      const matrix::Dense<ValueType> *orig,
-                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL);
-
-
-template <typename ValueType>
-void inplace_absolute_dense(std::shared_ptr<const DpcppExecutor> exec,
-                            matrix::Dense<ValueType> *source)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL);
-
-
-template <typename ValueType>
-void outplace_absolute_dense(std::shared_ptr<const DpcppExecutor> exec,
-                             const matrix::Dense<ValueType> *source,
-                             matrix::Dense<remove_complex<ValueType>> *result)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
-
-
-template <typename ValueType>
-void make_complex(std::shared_ptr<const DpcppExecutor> exec,
-                  const matrix::Dense<ValueType> *source,
-                  matrix::Dense<to_complex<ValueType>> *result)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MAKE_COMPLEX_KERNEL);
-
-
-template <typename ValueType>
-void get_real(std::shared_ptr<const DpcppExecutor> exec,
-              const matrix::Dense<ValueType> *source,
-              matrix::Dense<remove_complex<ValueType>> *result)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_REAL_KERNEL);
-
-
-template <typename ValueType>
-void get_imag(std::shared_ptr<const DpcppExecutor> exec,
-              const matrix::Dense<ValueType> *source,
-              matrix::Dense<remove_complex<ValueType>> *result)
-    GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_IMAG_KERNEL);
-
-
 }  // namespace dense
 }  // namespace dpcpp
 }  // namespace kernels
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 9277a3e2ad1..1da3755f75c 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -191,6 +191,7 @@ set(GINKGO_HIP_SOURCES
     stop/criterion_kernels.hip.cpp
     stop/residual_norm_kernels.hip.cpp
     ../common/components/precision_conversion.cpp
+    ../common/matrix/dense_kernels.cpp
     ../common/solver/cg_kernels.cpp
     )
 
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index 257b2b5961d..fbf35ca520d 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -120,40 +120,6 @@ void apply(std::shared_ptr<const HipExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
-template <typename InValueType, typename OutValueType>
-void copy(std::shared_ptr<const DefaultExecutor> exec,
-          const matrix::Dense<InValueType> *input,
-          matrix::Dense<OutValueType> *output)
-{
-    const auto num_blocks = ceildiv(input->get_size()[0] * input->get_size()[1],
-                                    default_block_size);
-    hipLaunchKernelGGL(kernel::strided_copy, num_blocks, default_block_size, 0,
-                       0, input->get_size()[0], input->get_size()[1],
-                       input->get_stride(), output->get_stride(),
-                       as_hip_type(input->get_const_values()),
-                       as_hip_type(output->get_values()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(
-    GKO_DECLARE_DENSE_COPY_KERNEL);
-
-
-template <typename ValueType>
-void fill(std::shared_ptr<const DefaultExecutor> exec,
-          matrix::Dense<ValueType> *mat, ValueType value)
-{
-    constexpr auto block_size = default_block_size;
-    const auto num_blocks =
-        ceildiv(mat->get_size()[0] * mat->get_size()[1], block_size);
-    hipLaunchKernelGGL(kernel::strided_fill, num_blocks, block_size, 0, 0,
-                       mat->get_size()[0], mat->get_size()[1],
-                       mat->get_stride(), as_hip_type(mat->get_values()),
-                       as_hip_type(value));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_FILL_KERNEL);
-
-
 template <typename ValueType>
 void scale(std::shared_ptr<const HipExecutor> exec,
            const matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *x)
@@ -208,24 +174,6 @@ void add_scaled(std::shared_ptr<const HipExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_KERNEL);
 
 
-template <typename ValueType>
-void add_scaled_diag(std::shared_ptr<const HipExecutor> exec,
-                     const matrix::Dense<ValueType> *alpha,
-                     const matrix::Diagonal<ValueType> *x,
-                     matrix::Dense<ValueType> *y)
-{
-    const auto size = y->get_size()[0];
-    const auto grid_dim = ceildiv(size, default_block_size);
-
-    hipLaunchKernelGGL(kernel::add_scaled_diag, grid_dim, default_block_size, 0,
-                       0, size, as_hip_type(alpha->get_const_values()),
-                       as_hip_type(x->get_const_values()),
-                       as_hip_type(y->get_values()), y->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL);
-
-
 template <typename ValueType>
 void compute_dot(std::shared_ptr<const HipExecutor> exec,
                  const matrix::Dense<ValueType> *x,
@@ -708,235 +656,6 @@ void conj_transpose(std::shared_ptr<const HipExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void symm_permute(std::shared_ptr<const HipExecutor> exec,
-                  const Array<IndexType> *permutation_indices,
-                  const matrix::Dense<ValueType> *orig,
-                  matrix::Dense<ValueType> *permuted)
-{
-    constexpr auto block_size = default_block_size;
-    const auto num_blocks =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
-    hipLaunchKernelGGL(
-        kernel::symm_permute, num_blocks, block_size, 0, 0, orig->get_size()[0],
-        orig->get_size()[1], permutation_indices->get_const_data(),
-        as_hip_type(orig->get_const_values()), orig->get_stride(),
-        as_hip_type(permuted->get_values()), permuted->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inv_symm_permute(std::shared_ptr<const HipExecutor> exec,
-                      const Array<IndexType> *permutation_indices,
-                      const matrix::Dense<ValueType> *orig,
-                      matrix::Dense<ValueType> *permuted)
-{
-    constexpr auto block_size = default_block_size;
-    const auto num_blocks =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
-    hipLaunchKernelGGL(kernel::inv_symm_permute, num_blocks, block_size, 0, 0,
-                       orig->get_size()[0], orig->get_size()[1],
-                       permutation_indices->get_const_data(),
-                       as_hip_type(orig->get_const_values()),
-                       orig->get_stride(), as_hip_type(permuted->get_values()),
-                       permuted->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void row_gather(std::shared_ptr<const HipExecutor> exec,
-                const Array<IndexType> *row_indices,
-                const matrix::Dense<ValueType> *orig,
-                matrix::Dense<ValueType> *row_gathered)
-{
-    constexpr auto block_size = default_block_size;
-    auto out_num_rows = row_indices->get_num_elems();
-    const auto num_blocks =
-        ceildiv(out_num_rows * orig->get_size()[1], block_size);
-    hipLaunchKernelGGL(
-        kernel::row_gather, num_blocks, block_size, 0, 0, out_num_rows,
-        orig->get_size()[1], row_indices->get_const_data(),
-        as_hip_type(orig->get_const_values()), orig->get_stride(),
-        as_hip_type(row_gathered->get_values()), row_gathered->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_ROW_GATHER_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void column_permute(std::shared_ptr<const HipExecutor> exec,
-                    const Array<IndexType> *permutation_indices,
-                    const matrix::Dense<ValueType> *orig,
-                    matrix::Dense<ValueType> *column_permuted)
-{
-    constexpr auto block_size = default_block_size;
-    const auto num_blocks =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
-    hipLaunchKernelGGL(kernel::column_permute, num_blocks, block_size, 0, 0,
-                       orig->get_size()[0], orig->get_size()[1],
-                       permutation_indices->get_const_data(),
-                       as_hip_type(orig->get_const_values()),
-                       orig->get_stride(),
-                       as_hip_type(column_permuted->get_values()),
-                       column_permuted->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_row_permute(std::shared_ptr<const HipExecutor> exec,
-                         const Array<IndexType> *permutation_indices,
-                         const matrix::Dense<ValueType> *orig,
-                         matrix::Dense<ValueType> *row_permuted)
-{
-    constexpr auto block_size = default_block_size;
-    const auto num_blocks =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
-    hipLaunchKernelGGL(
-        kernel::inverse_row_permute, num_blocks, block_size, 0, 0,
-        orig->get_size()[0], orig->get_size()[1],
-        permutation_indices->get_const_data(),
-        as_hip_type(orig->get_const_values()), orig->get_stride(),
-        as_hip_type(row_permuted->get_values()), row_permuted->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(std::shared_ptr<const HipExecutor> exec,
-                            const Array<IndexType> *permutation_indices,
-                            const matrix::Dense<ValueType> *orig,
-                            matrix::Dense<ValueType> *column_permuted)
-{
-    constexpr auto block_size = default_block_size;
-    const auto num_blocks =
-        ceildiv(orig->get_size()[0] * orig->get_size()[1], block_size);
-    hipLaunchKernelGGL(kernel::inverse_column_permute, num_blocks, block_size,
-                       0, 0, orig->get_size()[0], orig->get_size()[1],
-                       permutation_indices->get_const_data(),
-                       as_hip_type(orig->get_const_values()),
-                       orig->get_stride(),
-                       as_hip_type(column_permuted->get_values()),
-                       column_permuted->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL);
-
-
-template <typename ValueType>
-void extract_diagonal(std::shared_ptr<const HipExecutor> exec,
-                      const matrix::Dense<ValueType> *orig,
-                      matrix::Diagonal<ValueType> *diag)
-{
-    const dim3 grid_dim = ceildiv(diag->get_size()[0], default_block_size);
-    hipLaunchKernelGGL(kernel::extract_diagonal, dim3(grid_dim),
-                       dim3(default_block_size), 0, 0, diag->get_size()[0],
-                       as_hip_type(orig->get_const_values()),
-                       orig->get_stride(), as_hip_type(diag->get_values()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL);
-
-
-template <typename ValueType>
-void inplace_absolute_dense(std::shared_ptr<const HipExecutor> exec,
-                            matrix::Dense<ValueType> *source)
-{
-    auto dim = source->get_size();
-    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
-
-    hipLaunchKernelGGL(kernel::inplace_absolute_dense, dim3(grid_dim),
-                       dim3(default_block_size), 0, 0, dim[0], dim[1],
-                       as_hip_type(source->get_values()), source->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL);
-
-
-template <typename ValueType>
-void outplace_absolute_dense(std::shared_ptr<const HipExecutor> exec,
-                             const matrix::Dense<ValueType> *source,
-                             matrix::Dense<remove_complex<ValueType>> *result)
-{
-    auto dim = source->get_size();
-    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
-
-    hipLaunchKernelGGL(kernel::outplace_absolute_dense, dim3(grid_dim),
-                       dim3(default_block_size), 0, 0, dim[0], dim[1],
-                       as_hip_type(source->get_const_values()),
-                       source->get_stride(), as_hip_type(result->get_values()),
-                       result->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
-
-
-template <typename ValueType>
-void make_complex(std::shared_ptr<const HipExecutor> exec,
-                  const matrix::Dense<ValueType> *source,
-                  matrix::Dense<to_complex<ValueType>> *result)
-{
-    auto dim = source->get_size();
-    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
-
-    hipLaunchKernelGGL(kernel::make_complex, dim3(grid_dim),
-                       dim3(default_block_size), 0, 0, dim[0], dim[1],
-                       as_hip_type(source->get_const_values()),
-                       source->get_stride(), as_hip_type(result->get_values()),
-                       result->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MAKE_COMPLEX_KERNEL);
-
-
-template <typename ValueType>
-void get_real(std::shared_ptr<const HipExecutor> exec,
-              const matrix::Dense<ValueType> *source,
-              matrix::Dense<remove_complex<ValueType>> *result)
-{
-    auto dim = source->get_size();
-    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
-
-    hipLaunchKernelGGL(kernel::get_real, dim3(grid_dim),
-                       dim3(default_block_size), 0, 0, dim[0], dim[1],
-                       as_hip_type(source->get_const_values()),
-                       source->get_stride(), as_hip_type(result->get_values()),
-                       result->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_REAL_KERNEL);
-
-
-template <typename ValueType>
-void get_imag(std::shared_ptr<const HipExecutor> exec,
-              const matrix::Dense<ValueType> *source,
-              matrix::Dense<remove_complex<ValueType>> *result)
-{
-    auto dim = source->get_size();
-    const dim3 grid_dim = ceildiv(dim[0] * dim[1], default_block_size);
-
-    hipLaunchKernelGGL(kernel::get_imag, dim3(grid_dim),
-                       dim3(default_block_size), 0, 0, dim[0], dim[1],
-                       as_hip_type(source->get_const_values()),
-                       source->get_stride(), as_hip_type(result->get_values()),
-                       result->get_stride());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_IMAG_KERNEL);
-
-
 }  // namespace dense
 }  // namespace hip
 }  // namespace kernels
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index 9dd7ccbe26a..e2167146335 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -40,6 +40,7 @@ target_sources(ginkgo_omp
     stop/criterion_kernels.cpp
     stop/residual_norm_kernels.cpp
     ../common/components/precision_conversion.cpp
+    ../common/matrix/dense_kernels.cpp
     ../common/solver/cg_kernels.cpp
     )
 
diff --git a/omp/matrix/dense_kernels.cpp b/omp/matrix/dense_kernels.cpp
index 0534f1deb41..7590d91e38e 100644
--- a/omp/matrix/dense_kernels.cpp
+++ b/omp/matrix/dense_kernels.cpp
@@ -127,39 +127,6 @@ void apply(std::shared_ptr<const OmpExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
-template <typename InValueType, typename OutValueType>
-void copy(std::shared_ptr<const DefaultExecutor> exec,
-          const matrix::Dense<InValueType> *input,
-          matrix::Dense<OutValueType> *output)
-{
-#pragma omp parallel for
-    for (size_type row = 0; row < input->get_size()[0]; ++row) {
-        for (size_type col = 0; col < input->get_size()[1]; ++col) {
-            output->at(row, col) =
-                static_cast<OutValueType>(input->at(row, col));
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION_OR_COPY(
-    GKO_DECLARE_DENSE_COPY_KERNEL);
-
-
-template <typename ValueType>
-void fill(std::shared_ptr<const DefaultExecutor> exec,
-          matrix::Dense<ValueType> *mat, ValueType value)
-{
-#pragma omp parallel for
-    for (size_type row = 0; row < mat->get_size()[0]; ++row) {
-        for (size_type col = 0; col < mat->get_size()[1]; ++col) {
-            mat->at(row, col) = value;
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_FILL_KERNEL);
-
-
 template <typename ValueType>
 void scale(std::shared_ptr<const OmpExecutor> exec,
            const matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *x)
@@ -209,22 +176,6 @@ void add_scaled(std::shared_ptr<const OmpExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_KERNEL);
 
 
-template <typename ValueType>
-void add_scaled_diag(std::shared_ptr<const OmpExecutor> exec,
-                     const matrix::Dense<ValueType> *alpha,
-                     const matrix::Diagonal<ValueType> *x,
-                     matrix::Dense<ValueType> *y)
-{
-    const auto diag_values = x->get_const_values();
-#pragma omp parallel for
-    for (size_type i = 0; i < x->get_size()[0]; i++) {
-        y->at(i, i) += alpha->at(0, 0) * diag_values[i];
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL);
-
-
 template <typename ValueType>
 void compute_dot(std::shared_ptr<const OmpExecutor> exec,
                  const matrix::Dense<ValueType> *x,
@@ -733,226 +684,6 @@ void conj_transpose(std::shared_ptr<const OmpExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void symm_permute(std::shared_ptr<const OmpExecutor> exec,
-                  const Array<IndexType> *permutation_indices,
-                  const matrix::Dense<ValueType> *orig,
-                  matrix::Dense<ValueType> *permuted)
-{
-    auto perm = permutation_indices->get_const_data();
-    auto size = orig->get_size()[0];
-#pragma omp parallel for
-    for (size_type i = 0; i < size; ++i) {
-        for (size_type j = 0; j < size; ++j) {
-            permuted->at(i, j) = orig->at(perm[i], perm[j]);
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_SYMM_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inv_symm_permute(std::shared_ptr<const OmpExecutor> exec,
-                      const Array<IndexType> *permutation_indices,
-                      const matrix::Dense<ValueType> *orig,
-                      matrix::Dense<ValueType> *permuted)
-{
-    auto perm = permutation_indices->get_const_data();
-    auto size = orig->get_size()[0];
-#pragma omp parallel for
-    for (size_type i = 0; i < size; ++i) {
-        for (size_type j = 0; j < size; ++j) {
-            permuted->at(perm[i], perm[j]) = orig->at(i, j);
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_INV_SYMM_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void row_gather(std::shared_ptr<const OmpExecutor> exec,
-                const Array<IndexType> *row_indices,
-                const matrix::Dense<ValueType> *orig,
-                matrix::Dense<ValueType> *row_gathered)
-{
-    auto rows = row_indices->get_const_data();
-#pragma omp parallel for
-    for (size_type i = 0; i < row_indices->get_num_elems(); ++i) {
-        for (size_type j = 0; j < orig->get_size()[1]; ++j) {
-            row_gathered->at(i, j) = orig->at(rows[i], j);
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_ROW_GATHER_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void column_permute(std::shared_ptr<const OmpExecutor> exec,
-                    const Array<IndexType> *permutation_indices,
-                    const matrix::Dense<ValueType> *orig,
-                    matrix::Dense<ValueType> *column_permuted)
-{
-    auto perm = permutation_indices->get_const_data();
-#pragma omp parallel for
-    for (size_type i = 0; i < orig->get_size()[0]; ++i) {
-        for (size_type j = 0; j < orig->get_size()[1]; ++j) {
-            column_permuted->at(i, j) = orig->at(i, perm[j]);
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_COLUMN_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_row_permute(std::shared_ptr<const OmpExecutor> exec,
-                         const Array<IndexType> *permutation_indices,
-                         const matrix::Dense<ValueType> *orig,
-                         matrix::Dense<ValueType> *row_permuted)
-{
-    auto perm = permutation_indices->get_const_data();
-#pragma omp parallel for
-    for (size_type i = 0; i < orig->get_size()[0]; ++i) {
-        for (size_type j = 0; j < orig->get_size()[1]; ++j) {
-            row_permuted->at(perm[i], j) = orig->at(i, j);
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_INV_ROW_PERMUTE_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(std::shared_ptr<const OmpExecutor> exec,
-                            const Array<IndexType> *permutation_indices,
-                            const matrix::Dense<ValueType> *orig,
-                            matrix::Dense<ValueType> *column_permuted)
-{
-    auto perm = permutation_indices->get_const_data();
-#pragma omp parallel for
-    for (size_type i = 0; i < orig->get_size()[0]; ++i) {
-        for (size_type j = 0; j < orig->get_size()[1]; ++j) {
-            column_permuted->at(i, perm[j]) = orig->at(i, j);
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DENSE_INV_COLUMN_PERMUTE_KERNEL);
-
-
-template <typename ValueType>
-void extract_diagonal(std::shared_ptr<const OmpExecutor> exec,
-                      const matrix::Dense<ValueType> *orig,
-                      matrix::Diagonal<ValueType> *diag)
-{
-    auto diag_values = diag->get_values();
-#pragma omp parallel for
-    for (size_type i = 0; i < diag->get_size()[0]; ++i) {
-        diag_values[i] = orig->at(i, i);
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL);
-
-
-template <typename ValueType>
-void inplace_absolute_dense(std::shared_ptr<const OmpExecutor> exec,
-                            matrix::Dense<ValueType> *source)
-{
-    auto dim = source->get_size();
-
-#pragma omp parallel for
-    for (size_type row = 0; row < dim[0]; row++) {
-        for (size_type col = 0; col < dim[1]; col++) {
-            source->at(row, col) = abs(source->at(row, col));
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL);
-
-
-template <typename ValueType>
-void outplace_absolute_dense(std::shared_ptr<const OmpExecutor> exec,
-                             const matrix::Dense<ValueType> *source,
-                             matrix::Dense<remove_complex<ValueType>> *result)
-{
-    auto dim = source->get_size();
-
-#pragma omp parallel for
-    for (size_type row = 0; row < dim[0]; row++) {
-        for (size_type col = 0; col < dim[1]; col++) {
-            result->at(row, col) = abs(source->at(row, col));
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL);
-
-
-template <typename ValueType>
-void make_complex(std::shared_ptr<const OmpExecutor> exec,
-                  const matrix::Dense<ValueType> *source,
-                  matrix::Dense<to_complex<ValueType>> *result)
-{
-    auto dim = source->get_size();
-
-#pragma omp parallel for
-    for (size_type row = 0; row < dim[0]; row++) {
-        for (size_type col = 0; col < dim[1]; col++) {
-            result->at(row, col) = to_complex<ValueType>{source->at(row, col)};
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MAKE_COMPLEX_KERNEL);
-
-
-template <typename ValueType>
-void get_real(std::shared_ptr<const OmpExecutor> exec,
-              const matrix::Dense<ValueType> *source,
-              matrix::Dense<remove_complex<ValueType>> *result)
-{
-    auto dim = source->get_size();
-
-#pragma omp parallel for
-    for (size_type row = 0; row < dim[0]; row++) {
-        for (size_type col = 0; col < dim[1]; col++) {
-            result->at(row, col) = real(source->at(row, col));
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_REAL_KERNEL);
-
-
-template <typename ValueType>
-void get_imag(std::shared_ptr<const OmpExecutor> exec,
-              const matrix::Dense<ValueType> *source,
-              matrix::Dense<remove_complex<ValueType>> *result)
-{
-    auto dim = source->get_size();
-
-#pragma omp parallel for
-    for (size_type row = 0; row < dim[0]; row++) {
-        for (size_type col = 0; col < dim[1]; col++) {
-            result->at(row, col) = imag(source->at(row, col));
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_IMAG_KERNEL);
-
-
 }  // namespace dense
 }  // namespace omp
 }  // namespace kernels

From 96ec6d72121224b2a7b5463ae8a7269e437f5a21 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 21 May 2021 23:10:15 +0200
Subject: [PATCH 0821/1008] add simple Krylov kernels

---
 {omp => common}/solver/bicg_kernels.cpp     | 102 +++++-----
 common/solver/bicg_kernels.hpp.inc          | 111 -----------
 {omp => common}/solver/bicgstab_kernels.cpp | 178 +++++++----------
 common/solver/bicgstab_kernels.hpp.inc      | 168 ----------------
 common/solver/cg_kernels.cpp                |  13 +-
 {omp => common}/solver/cgs_kernels.cpp      | 138 ++++++-------
 common/solver/cgs_kernels.hpp.inc           | 137 -------------
 {omp => common}/solver/fcg_kernels.cpp      |  92 ++++-----
 common/solver/fcg_kernels.hpp.inc           | 104 ----------
 {omp => common}/solver/ir_kernels.cpp       |  15 +-
 common/solver/ir_kernels.hpp.inc            |  41 ----
 cuda/CMakeLists.txt                         |  10 +-
 cuda/solver/bicg_kernels.cu                 | 144 --------------
 cuda/solver/bicgstab_kernels.cu             | 201 -------------------
 cuda/solver/cgs_kernels.cu                  | 169 ----------------
 cuda/solver/fcg_kernels.cu                  | 139 -------------
 cuda/solver/ir_kernels.cu                   |  74 -------
 dpcpp/CMakeLists.txt                        |  10 +-
 dpcpp/solver/bicg_kernels.dp.cpp            |  96 ---------
 dpcpp/solver/bicgstab_kernels.dp.cpp        | 123 ------------
 dpcpp/solver/cgs_kernels.dp.cpp             | 107 ----------
 dpcpp/solver/fcg_kernels.dp.cpp             |  92 ---------
 dpcpp/solver/ir_kernels.dp.cpp              |  57 ------
 hip/CMakeLists.txt                          |  10 +-
 hip/solver/bicg_kernels.hip.cpp             | 149 --------------
 hip/solver/bicgstab_kernels.hip.cpp         | 205 --------------------
 hip/solver/cgs_kernels.hip.cpp              | 176 -----------------
 hip/solver/fcg_kernels.hip.cpp              | 144 --------------
 hip/solver/ir_kernels.hip.cpp               |  78 --------
 include/ginkgo/core/base/math.hpp           |  18 ++
 omp/CMakeLists.txt                          |  10 +-
 31 files changed, 276 insertions(+), 2835 deletions(-)
 rename {omp => common}/solver/bicg_kernels.cpp (62%)
 delete mode 100644 common/solver/bicg_kernels.hpp.inc
 rename {omp => common}/solver/bicgstab_kernels.cpp (53%)
 delete mode 100644 common/solver/bicgstab_kernels.hpp.inc
 rename {omp => common}/solver/cgs_kernels.cpp (58%)
 delete mode 100644 common/solver/cgs_kernels.hpp.inc
 rename {omp => common}/solver/fcg_kernels.cpp (63%)
 delete mode 100644 common/solver/fcg_kernels.hpp.inc
 rename {omp => common}/solver/ir_kernels.cpp (86%)
 delete mode 100644 common/solver/ir_kernels.hpp.inc
 delete mode 100644 cuda/solver/bicg_kernels.cu
 delete mode 100644 cuda/solver/bicgstab_kernels.cu
 delete mode 100644 cuda/solver/cgs_kernels.cu
 delete mode 100644 cuda/solver/fcg_kernels.cu
 delete mode 100644 cuda/solver/ir_kernels.cu
 delete mode 100644 dpcpp/solver/bicg_kernels.dp.cpp
 delete mode 100644 dpcpp/solver/bicgstab_kernels.dp.cpp
 delete mode 100644 dpcpp/solver/cgs_kernels.dp.cpp
 delete mode 100644 dpcpp/solver/fcg_kernels.dp.cpp
 delete mode 100644 dpcpp/solver/ir_kernels.dp.cpp
 delete mode 100644 hip/solver/bicg_kernels.hip.cpp
 delete mode 100644 hip/solver/bicgstab_kernels.hip.cpp
 delete mode 100644 hip/solver/cgs_kernels.hip.cpp
 delete mode 100644 hip/solver/fcg_kernels.hip.cpp
 delete mode 100644 hip/solver/ir_kernels.hip.cpp

diff --git a/omp/solver/bicg_kernels.cpp b/common/solver/bicg_kernels.cpp
similarity index 62%
rename from omp/solver/bicg_kernels.cpp
rename to common/solver/bicg_kernels.cpp
index 9e98a000699..9a7abb73a6e 100644
--- a/omp/solver/bicg_kernels.cpp
+++ b/common/solver/bicg_kernels.cpp
@@ -33,18 +33,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/bicg_kernels.hpp"
 
 
-#include <omp.h>
+#include <ginkgo/core/base/math.hpp>
 
 
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
+#include "core/base/simple_kernels.hpp"
 
 
 namespace gko {
 namespace kernels {
-namespace omp {
+namespace GKO_DEVICE_NAMESPACE {
 /**
  * @brief The BICG solver namespace.
  *
@@ -54,7 +51,7 @@ namespace bicg {
 
 
 template <typename ValueType>
-void initialize(std::shared_ptr<const OmpExecutor> exec,
+void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
                 matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
                 matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
@@ -63,57 +60,58 @@ void initialize(std::shared_ptr<const OmpExecutor> exec,
                 matrix::Dense<ValueType> *q2,
                 Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type j = 0; j < b->get_size()[1]; ++j) {
-        rho->at(j) = zero<ValueType>();
-        prev_rho->at(j) = one<ValueType>();
-        stop_status->get_data()[j].reset();
-    }
-#pragma omp parallel for
-    for (size_type i = 0; i < b->get_size()[0]; ++i) {
-        for (size_type j = 0; j < b->get_size()[1]; ++j) {
-            r->at(i, j) = b->at(i, j);
-            r2->at(i, j) = b->at(i, j);
-            z->at(i, j) = p->at(i, j) = q->at(i, j) = zero<ValueType>();
-            z2->at(i, j) = p2->at(i, j) = q2->at(i, j) = zero<ValueType>();
-        }
-    }
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto z, auto p,
+                      auto q, auto prev_rho, auto rho, auto r2, auto z2,
+                      auto p2, auto q2, auto stop) {
+            if (row == 0) {
+                rho[col] = zero(rho[col]);
+                prev_rho[col] = one(prev_rho[col]);
+                stop[col].reset();
+            }
+            r(row, col) = b(row, col);
+            r2(row, col) = b(row, col);
+            z(row, col) = zero(z(row, col));
+            p(row, col) = zero(p(row, col));
+            q(row, col) = zero(q(row, col));
+            z2(row, col) = zero(z2(row, col));
+            p2(row, col) = zero(p2(row, col));
+            q2(row, col) = zero(q2(row, col));
+        },
+        p->get_size(), b, r, z, p, q, prev_rho, rho, r2, z2, p2, q2,
+        *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_INITIALIZE_KERNEL);
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const OmpExecutor> exec,
+void step_1(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
             matrix::Dense<ValueType> *p2, const matrix::Dense<ValueType> *z2,
             const matrix::Dense<ValueType> *rho,
             const matrix::Dense<ValueType> *prev_rho,
             const Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type i = 0; i < p->get_size()[0]; ++i) {
-        for (size_type j = 0; j < p->get_size()[1]; ++j) {
-            if (stop_status->get_const_data()[j].has_stopped()) {
-                continue;
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto p, auto z, auto p2, auto z2,
+                      auto rho, auto prev_rho, auto stop) {
+            if (!stop[col].has_stopped()) {
+                auto tmp = safe_divide(rho[col], prev_rho[col]);
+                p(row, col) = z(row, col) + tmp * p(row, col);
+                p2(row, col) = z2(row, col) + tmp * p2(row, col);
             }
-            if (prev_rho->at(j) == zero<ValueType>()) {
-                p->at(i, j) = z->at(i, j);
-                p2->at(i, j) = z2->at(i, j);
-            } else {
-                auto tmp = rho->at(j) / prev_rho->at(j);
-                p->at(i, j) = z->at(i, j) + tmp * p->at(i, j);
-                p2->at(i, j) = z2->at(i, j) + tmp * p2->at(i, j);
-            }
-        }
-    }
+        },
+        p->get_size(), p, z, p2, z2, rho, prev_rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const OmpExecutor> exec,
+void step_2(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
             matrix::Dense<ValueType> *r2, const matrix::Dense<ValueType> *p,
             const matrix::Dense<ValueType> *q,
@@ -122,26 +120,24 @@ void step_2(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Dense<ValueType> *rho,
             const Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type i = 0; i < x->get_size()[0]; ++i) {
-        for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            if (stop_status->get_const_data()[j].has_stopped()) {
-                continue;
-            }
-            if (beta->at(j) != zero<ValueType>()) {
-                auto tmp = rho->at(j) / beta->at(j);
-                x->at(i, j) += tmp * p->at(i, j);
-                r->at(i, j) -= tmp * q->at(i, j);
-                r2->at(i, j) -= tmp * q2->at(i, j);
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto r2, auto p,
+                      auto q, auto q2, auto beta, auto rho, auto stop) {
+            if (!stop[col].has_stopped()) {
+                auto tmp = safe_divide(rho[col], beta[col]);
+                x(row, col) += tmp * p(row, col);
+                r(row, col) -= tmp * q(row, col);
+                r2(row, col) -= tmp * q2(row, col);
             }
-        }
-    }
+        },
+        p->get_size(), x, r, r2, p, q, q2, beta, rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_2_KERNEL);
 
 
 }  // namespace bicg
-}  // namespace omp
+}  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
 }  // namespace gko
diff --git a/common/solver/bicg_kernels.hpp.inc b/common/solver/bicg_kernels.hpp.inc
deleted file mode 100644
index 3402fa219b5..00000000000
--- a/common/solver/bicg_kernels.hpp.inc
+++ /dev/null
@@ -1,111 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void initialize_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    const ValueType *__restrict__ b, ValueType *__restrict__ r,
-    ValueType *__restrict__ z, ValueType *__restrict__ p,
-    ValueType *__restrict__ q, ValueType *__restrict__ r2,
-    ValueType *__restrict__ z2, ValueType *__restrict__ p2,
-    ValueType *__restrict__ q2, ValueType *__restrict__ prev_rho,
-    ValueType *__restrict__ rho, stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-
-    if (tidx < num_cols) {
-        rho[tidx] = zero<ValueType>();
-        prev_rho[tidx] = one<ValueType>();
-        stop_status[tidx].reset();
-    }
-
-    if (tidx < num_rows * stride) {
-        r[tidx] = b[tidx];
-        z[tidx] = zero<ValueType>();
-        p[tidx] = zero<ValueType>();
-        q[tidx] = zero<ValueType>();
-        r2[tidx] = b[tidx];
-        z2[tidx] = zero<ValueType>();
-        p2[tidx] = zero<ValueType>();
-        q2[tidx] = zero<ValueType>();
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void step_1_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    ValueType *__restrict__ p, const ValueType *__restrict__ z,
-    ValueType *__restrict__ p2, const ValueType *__restrict__ z2,
-    const ValueType *__restrict__ rho, const ValueType *__restrict__ prev_rho,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto col = tidx % stride;
-    if (col >= num_cols || tidx >= num_rows * stride ||
-        stop_status[col].has_stopped()) {
-        return;
-    }
-    const auto tmp = rho[col] / prev_rho[col];
-
-    p[tidx] =
-        prev_rho[col] == zero<ValueType>() ? z[tidx] : z[tidx] + tmp * p[tidx];
-
-    p2[tidx] = prev_rho[col] == zero<ValueType>() ? z2[tidx]
-                                                  : z2[tidx] + tmp * p2[tidx];
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void step_2_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    size_type x_stride, ValueType *__restrict__ x, ValueType *__restrict__ r,
-    ValueType *__restrict__ r2, const ValueType *__restrict__ p,
-    const ValueType *__restrict__ q, const ValueType *__restrict__ q2,
-    const ValueType *__restrict__ beta, const ValueType *__restrict__ rho,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto row = tidx / stride;
-    const auto col = tidx % stride;
-
-    if (col >= num_cols || tidx >= num_rows * num_cols ||
-        stop_status[col].has_stopped()) {
-        return;
-    }
-    if (beta[col] != zero<ValueType>()) {
-        const auto tmp = rho[col] / beta[col];
-        x[row * x_stride + col] += tmp * p[tidx];
-        r[tidx] -= tmp * q[tidx];
-        r2[tidx] -= tmp * q2[tidx];
-    }
-}
diff --git a/omp/solver/bicgstab_kernels.cpp b/common/solver/bicgstab_kernels.cpp
similarity index 53%
rename from omp/solver/bicgstab_kernels.cpp
rename to common/solver/bicgstab_kernels.cpp
index 228f7c50e27..c669431d1fc 100644
--- a/omp/solver/bicgstab_kernels.cpp
+++ b/common/solver/bicgstab_kernels.cpp
@@ -33,20 +33,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/bicgstab_kernels.hpp"
 
 
-#include <algorithm>
-
-
-#include <omp.h>
+#include <ginkgo/core/base/math.hpp>
 
 
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
+#include "core/base/simple_kernels.hpp"
 
 
 namespace gko {
 namespace kernels {
-namespace omp {
+namespace GKO_DEVICE_NAMESPACE {
 /**
  * @brief The BICGSTAB solver namespace.
  *
@@ -56,7 +51,7 @@ namespace bicgstab {
 
 
 template <typename ValueType>
-void initialize(std::shared_ptr<const OmpExecutor> exec,
+void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
                 matrix::Dense<ValueType> *rr, matrix::Dense<ValueType> *y,
                 matrix::Dense<ValueType> *s, matrix::Dense<ValueType> *t,
@@ -67,36 +62,30 @@ void initialize(std::shared_ptr<const OmpExecutor> exec,
                 matrix::Dense<ValueType> *omega,
                 Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type j = 0; j < b->get_size()[1]; ++j) {
-        rho->at(j) = one<ValueType>();
-        prev_rho->at(j) = one<ValueType>();
-        alpha->at(j) = one<ValueType>();
-        beta->at(j) = one<ValueType>();
-        gamma->at(j) = one<ValueType>();
-        omega->at(j) = one<ValueType>();
-        stop_status->get_data()[j].reset();
-    }
-#pragma omp parallel for
-    for (size_type i = 0; i < b->get_size()[0]; ++i) {
-        for (size_type j = 0; j < b->get_size()[1]; ++j) {
-            r->at(i, j) = b->at(i, j);
-            rr->at(i, j) = zero<ValueType>();
-            z->at(i, j) = zero<ValueType>();
-            v->at(i, j) = zero<ValueType>();
-            s->at(i, j) = zero<ValueType>();
-            t->at(i, j) = zero<ValueType>();
-            y->at(i, j) = zero<ValueType>();
-            p->at(i, j) = zero<ValueType>();
-        }
-    }
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto rr, auto y,
+                      auto s, auto t, auto z, auto v, auto p, auto prev_rho,
+                      auto rho, auto alpha, auto beta, auto gamma, auto omega,
+                      auto stop) {
+            if (row == 0) {
+                rho[col] = prev_rho[col] = alpha[col] = beta[col] = gamma[col] =
+                    omega[col] = one(rho[col]);
+                stop[col].reset();
+            }
+            r(row, col) = b(row, col);
+            rr(row, col) = z(row, col) = v(row, col) = s(row, col) =
+                t(row, col) = y(row, col) = p(row, col) = zero(rr(row, col));
+        },
+        p->get_size(), b, r, rr, y, s, t, z, v, p, prev_rho, rho, alpha, beta,
+        gamma, omega, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_INITIALIZE_KERNEL);
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const OmpExecutor> exec,
+void step_1(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *p,
             const matrix::Dense<ValueType> *v,
             const matrix::Dense<ValueType> *rho,
@@ -105,29 +94,25 @@ void step_1(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Dense<ValueType> *omega,
             const Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type i = 0; i < p->get_size()[0]; ++i) {
-        for (size_type j = 0; j < p->get_size()[1]; ++j) {
-            if (stop_status->get_const_data()[j].has_stopped()) {
-                continue;
-            }
-            if (prev_rho->at(j) * omega->at(j) != zero<ValueType>()) {
-                const auto tmp =
-                    rho->at(j) / prev_rho->at(j) * alpha->at(j) / omega->at(j);
-                p->at(i, j) = r->at(i, j) +
-                              tmp * (p->at(i, j) - omega->at(j) * v->at(i, j));
-            } else {
-                p->at(i, j) = r->at(i, j);
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto r, auto p, auto v, auto rho,
+                      auto prev_rho, auto alpha, auto omega, auto stop) {
+            if (!stop[col].has_stopped()) {
+                auto tmp = safe_divide(rho[col], prev_rho[col]) *
+                           safe_divide(alpha[col], omega[col]);
+                p(row, col) = r(row, col) +
+                              tmp * (p(row, col) - omega[col] * v(row, col));
             }
-        }
-    }
+        },
+        p->get_size(), r, p, v, rho, prev_rho, alpha, omega, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const OmpExecutor> exec,
+void step_2(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *s,
             const matrix::Dense<ValueType> *v,
             const matrix::Dense<ValueType> *rho,
@@ -135,30 +120,19 @@ void step_2(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Dense<ValueType> *beta,
             const Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type j = 0; j < s->get_size()[1]; ++j) {
-        if (stop_status->get_const_data()[j].has_stopped()) {
-            continue;
-        }
-        if (beta->at(j) != zero<ValueType>()) {
-            alpha->at(j) = rho->at(j) / beta->at(j);
-        } else {
-            alpha->at(j) = zero<ValueType>();
-        }
-    }
-#pragma omp parallel for
-    for (size_type i = 0; i < s->get_size()[0]; ++i) {
-        for (size_type j = 0; j < s->get_size()[1]; ++j) {
-            if (stop_status->get_const_data()[j].has_stopped()) {
-                continue;
-            }
-            if (beta->at(j) != zero<ValueType>()) {
-                s->at(i, j) = r->at(i, j) - alpha->at(j) * v->at(i, j);
-            } else {
-                s->at(i, j) = r->at(i, j);
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto r, auto s, auto v, auto rho,
+                      auto alpha, auto beta, auto stop) {
+            if (!stop[col].has_stopped()) {
+                auto tmp = safe_divide(rho[col], beta[col]);
+                if (row == 0) {
+                    alpha[col] = tmp;
+                }
+                s(row, col) = r(row, col) - tmp * v(row, col);
             }
-        }
-    }
+        },
+        r->get_size(), r, s, v, rho, alpha, beta, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_2_KERNEL);
@@ -166,62 +140,56 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_2_KERNEL);
 
 template <typename ValueType>
 void step_3(
-    std::shared_ptr<const OmpExecutor> exec, matrix::Dense<ValueType> *x,
+    std::shared_ptr<const DefaultExecutor> exec, matrix::Dense<ValueType> *x,
     matrix::Dense<ValueType> *r, const matrix::Dense<ValueType> *s,
     const matrix::Dense<ValueType> *t, const matrix::Dense<ValueType> *y,
     const matrix::Dense<ValueType> *z, const matrix::Dense<ValueType> *alpha,
     const matrix::Dense<ValueType> *beta, const matrix::Dense<ValueType> *gamma,
     matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type j = 0; j < x->get_size()[1]; ++j) {
-        if (stop_status->get_const_data()[j].has_stopped()) {
-            continue;
-        }
-        if (beta->at(j) != zero<ValueType>()) {
-            omega->at(j) = gamma->at(j) / beta->at(j);
-        } else {
-            omega->at(j) = zero<ValueType>();
-        }
-    }
-#pragma omp parallel for
-    for (size_type i = 0; i < x->get_size()[0]; ++i) {
-        for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            if (stop_status->get_const_data()[j].has_stopped()) {
-                continue;
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto s, auto t,
+                      auto y, auto z, auto alpha, auto beta, auto gamma,
+                      auto omega, auto stop) {
+            if (!stop[col].has_stopped()) {
+                auto tmp = safe_divide(gamma[col], beta[col]);
+                if (row == 0) {
+                    omega[col] = tmp;
+                }
+                x(row, col) += alpha[col] * y(row, col) + tmp * z(row, col);
+                r(row, col) = s(row, col) - tmp * t(row, col);
             }
-            x->at(i, j) +=
-                alpha->at(j) * y->at(i, j) + omega->at(j) * z->at(i, j);
-            r->at(i, j) = s->at(i, j) - omega->at(j) * t->at(i, j);
-        }
-    }
+        },
+        r->get_size(), x, r, s, t, y, z, alpha, beta, gamma, omega,
+        *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_3_KERNEL);
 
 
 template <typename ValueType>
-void finalize(std::shared_ptr<const OmpExecutor> exec,
+void finalize(std::shared_ptr<const DefaultExecutor> exec,
               matrix::Dense<ValueType> *x, const matrix::Dense<ValueType> *y,
               const matrix::Dense<ValueType> *alpha,
               Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type j = 0; j < x->get_size()[1]; ++j) {
-        if (stop_status->get_const_data()[j].has_stopped() &&
-            !stop_status->get_const_data()[j].is_finalized()) {
-            for (size_type i = 0; i < x->get_size()[0]; ++i) {
-                x->at(i, j) += alpha->at(j) * y->at(i, j);
-                stop_status->get_data()[j].finalize();
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto x, auto y, auto alpha,
+                      auto stop) {
+            if (stop[col].has_stopped() && !stop[col].is_finalized()) {
+                x(row, col) += alpha[col] * y(row, col);
+                stop[col].finalize();
             }
-        }
-    }
+        },
+        x->get_size(), x, y, alpha, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_FINALIZE_KERNEL);
 
 
 }  // namespace bicgstab
-}  // namespace omp
+}  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
 }  // namespace gko
diff --git a/common/solver/bicgstab_kernels.hpp.inc b/common/solver/bicgstab_kernels.hpp.inc
deleted file mode 100644
index dce6c3651cd..00000000000
--- a/common/solver/bicgstab_kernels.hpp.inc
+++ /dev/null
@@ -1,168 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void initialize_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    const ValueType *__restrict__ b, ValueType *__restrict__ r,
-    ValueType *__restrict__ rr, ValueType *__restrict__ y,
-    ValueType *__restrict__ s, ValueType *__restrict__ t,
-    ValueType *__restrict__ z, ValueType *__restrict__ v,
-    ValueType *__restrict__ p, ValueType *__restrict__ prev_rho,
-    ValueType *__restrict__ rho, ValueType *__restrict__ alpha,
-    ValueType *__restrict__ beta, ValueType *__restrict__ gamma,
-    ValueType *__restrict__ omega, stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-
-    if (tidx < num_cols) {
-        prev_rho[tidx] = one<ValueType>();
-        rho[tidx] = one<ValueType>();
-        alpha[tidx] = one<ValueType>();
-        beta[tidx] = one<ValueType>();
-        gamma[tidx] = one<ValueType>();
-        omega[tidx] = one<ValueType>();
-        stop_status[tidx].reset();
-    }
-
-    if (tidx < num_rows * stride) {
-        r[tidx] = b[tidx];
-        rr[tidx] = zero<ValueType>();
-        y[tidx] = zero<ValueType>();
-        s[tidx] = zero<ValueType>();
-        t[tidx] = zero<ValueType>();
-        z[tidx] = zero<ValueType>();
-        v[tidx] = zero<ValueType>();
-        p[tidx] = zero<ValueType>();
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void step_1_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    const ValueType *__restrict__ r, ValueType *__restrict__ p,
-    const ValueType *__restrict__ v, const ValueType *__restrict__ rho,
-    const ValueType *__restrict__ prev_rho, const ValueType *__restrict__ alpha,
-    const ValueType *__restrict__ omega,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto col = tidx % stride;
-    if (col >= num_cols || tidx >= num_rows * stride ||
-        stop_status[col].has_stopped()) {
-        return;
-    }
-    auto res = r[tidx];
-    if (prev_rho[col] * omega[col] != zero<ValueType>()) {
-        const auto tmp = (rho[col] / prev_rho[col]) * (alpha[col] / omega[col]);
-        res += tmp * (p[tidx] - omega[col] * v[tidx]);
-    }
-    p[tidx] = res;
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void step_2_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    const ValueType *__restrict__ r, ValueType *__restrict__ s,
-    const ValueType *__restrict__ v, const ValueType *__restrict__ rho,
-    ValueType *__restrict__ alpha, const ValueType *__restrict__ beta,
-    const stopping_status *__restrict__ stop_status)
-{
-    const size_type tidx = thread::get_thread_id_flat();
-    const size_type col = tidx % stride;
-    if (col >= num_cols || tidx >= num_rows * stride ||
-        stop_status[col].has_stopped()) {
-        return;
-    }
-    auto t_alpha = zero<ValueType>();
-    auto t_s = r[tidx];
-    if (beta[col] != zero<ValueType>()) {
-        t_alpha = rho[col] / beta[col];
-        t_s -= t_alpha * v[tidx];
-    }
-    alpha[col] = t_alpha;
-    s[tidx] = t_s;
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void step_3_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    size_type x_stride, ValueType *__restrict__ x, ValueType *__restrict__ r,
-    const ValueType *__restrict__ s, const ValueType *__restrict__ t,
-    const ValueType *__restrict__ y, const ValueType *__restrict__ z,
-    const ValueType *__restrict__ alpha, const ValueType *__restrict__ beta,
-    const ValueType *__restrict__ gamma, ValueType *__restrict__ omega,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto row = tidx / stride;
-    const auto col = tidx % stride;
-    if (col >= num_cols || tidx >= num_rows * stride ||
-        stop_status[col].has_stopped()) {
-        return;
-    }
-    const auto x_pos = row * x_stride + col;
-    auto t_omega = zero<ValueType>();
-    auto t_x = x[x_pos] + alpha[col] * y[tidx];
-    auto t_r = s[tidx];
-    if (beta[col] != zero<ValueType>()) {
-        t_omega = gamma[col] / beta[col];
-        t_x += t_omega * z[tidx];
-        t_r -= t_omega * t[tidx];
-    }
-    omega[col] = t_omega;
-    x[x_pos] = t_x;
-    r[tidx] = t_r;
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void finalize_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    size_type x_stride, ValueType *__restrict__ x,
-    const ValueType *__restrict__ y, const ValueType *__restrict__ alpha,
-    stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto row = tidx / stride;
-    const auto col = tidx % stride;
-    if (col >= num_cols || tidx >= num_rows * stride ||
-        stop_status[col].is_finalized() || !stop_status[col].has_stopped()) {
-        return;
-    }
-    const auto x_pos = row * x_stride + col;
-    x[x_pos] = x[x_pos] + alpha[col] * y[tidx];
-    stop_status[col].finalize();
-}
diff --git a/common/solver/cg_kernels.cpp b/common/solver/cg_kernels.cpp
index 82bdba4f4a5..7943a5e46ae 100644
--- a/common/solver/cg_kernels.cpp
+++ b/common/solver/cg_kernels.cpp
@@ -68,9 +68,7 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 stop[col].reset();
             }
             r(row, col) = b(row, col);
-            z(row, col) = zero(z(row, col));
-            p(row, col) = zero(p(row, col));
-            q(row, col) = zero(q(row, col));
+            z(row, col) = p(row, col) = q(row, col) = zero(z(row, col));
         },
         p->get_size(), b, r, z, p, q, prev_rho, rho, *stop_status);
 }
@@ -89,9 +87,8 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
         exec,
         [] GKO_KERNEL(auto row, auto col, auto p, auto z, auto rho,
                       auto prev_rho, auto stop) {
-            if (!stop[col].has_stopped() &&
-                prev_rho[col] != zero(prev_rho[col])) {
-                auto tmp = rho[col] / prev_rho[col];
+            if (!stop[col].has_stopped()) {
+                auto tmp = safe_divide(rho[col], prev_rho[col]);
                 p(row, col) = z(row, col) + tmp * p(row, col);
             }
         },
@@ -114,8 +111,8 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
         exec,
         [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto p, auto q,
                       auto beta, auto rho, auto stop) {
-            if (!stop[col].has_stopped() && beta[col] != zero(beta[col])) {
-                auto tmp = rho[col] / beta[col];
+            if (!stop[col].has_stopped()) {
+                auto tmp = safe_divide(rho[col], beta[col]);
                 x(row, col) += tmp * p(row, col);
                 r(row, col) -= tmp * q(row, col);
             }
diff --git a/omp/solver/cgs_kernels.cpp b/common/solver/cgs_kernels.cpp
similarity index 58%
rename from omp/solver/cgs_kernels.cpp
rename to common/solver/cgs_kernels.cpp
index 34323bc7216..427228eb57a 100644
--- a/omp/solver/cgs_kernels.cpp
+++ b/common/solver/cgs_kernels.cpp
@@ -33,18 +33,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/cgs_kernels.hpp"
 
 
-#include <omp.h>
+#include <ginkgo/core/base/math.hpp>
 
 
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
+#include "core/base/simple_kernels.hpp"
 
 
 namespace gko {
 namespace kernels {
-namespace omp {
+namespace GKO_DEVICE_NAMESPACE {
 /**
  * @brief The CGS solver namespace.
  *
@@ -54,7 +51,7 @@ namespace cgs {
 
 
 template <typename ValueType>
-void initialize(std::shared_ptr<const OmpExecutor> exec,
+void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
                 matrix::Dense<ValueType> *r_tld, matrix::Dense<ValueType> *p,
                 matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *u,
@@ -66,65 +63,59 @@ void initialize(std::shared_ptr<const OmpExecutor> exec,
                 matrix::Dense<ValueType> *rho,
                 Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type j = 0; j < b->get_size()[1]; ++j) {
-        rho->at(j) = zero<ValueType>();
-        prev_rho->at(j) = one<ValueType>();
-        alpha->at(j) = one<ValueType>();
-        beta->at(j) = one<ValueType>();
-        gamma->at(j) = one<ValueType>();
-        stop_status->get_data()[j].reset();
-    }
-#pragma omp parallel for
-    for (size_type i = 0; i < b->get_size()[0]; ++i) {
-        for (size_type j = 0; j < b->get_size()[1]; ++j) {
-            r->at(i, j) = b->at(i, j);
-            r_tld->at(i, j) = b->at(i, j);
-            u->at(i, j) = u_hat->at(i, j) = p->at(i, j) = q->at(i, j) =
-                v_hat->at(i, j) = t->at(i, j) = zero<ValueType>();
-        }
-    }
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto r_tld, auto p,
+                      auto q, auto u, auto u_hat, auto v_hat, auto t,
+                      auto alpha, auto beta, auto gamma, auto prev_rho,
+                      auto rho, auto stop) {
+            if (row == 0) {
+                rho[col] = zero(rho[col]);
+                prev_rho[col] = alpha[col] = beta[col] = gamma[col] =
+                    one(prev_rho[col]);
+                stop[col].reset();
+            }
+            r(row, col) = r_tld(row, col) = b(row, col);
+            u(row, col) = u_hat(row, col) = p(row, col) = q(row, col) =
+                v_hat(row, col) = t(row, col) = zero(u(row, col));
+        },
+        p->get_size(), b, r, r_tld, p, q, u, u_hat, v_hat, t, alpha, beta,
+        gamma, prev_rho, rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_INITIALIZE_KERNEL);
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const OmpExecutor> exec,
+void step_1(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *u,
             matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *q,
             matrix::Dense<ValueType> *beta, const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *rho_prev,
+            const matrix::Dense<ValueType> *prev_rho,
             const Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type j = 0; j < p->get_size()[1]; ++j) {
-        if (stop_status->get_const_data()[j].has_stopped()) {
-            continue;
-        }
-        if (rho_prev->at(j) != zero<ValueType>()) {
-            beta->at(j) = rho->at(j) / rho_prev->at(j);
-        }
-    }
-#pragma omp parallel for
-    for (size_type i = 0; i < p->get_size()[0]; ++i) {
-        for (size_type j = 0; j < p->get_size()[1]; ++j) {
-            if (stop_status->get_const_data()[j].has_stopped()) {
-                continue;
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto r, auto u, auto p, auto q,
+                      auto beta, auto rho, auto prev_rho, auto stop) {
+            if (!stop[col].has_stopped()) {
+                auto tmp = safe_divide(rho[col], prev_rho[col]);
+                if (row == 0) {
+                    beta[col] = tmp;
+                }
+                u(row, col) = r(row, col) + tmp * q(row, col);
+                p(row, col) =
+                    u(row, col) + tmp * (q(row, col) + tmp * p(row, col));
             }
-            u->at(i, j) = r->at(i, j) + beta->at(j) * q->at(i, j);
-            p->at(i, j) =
-                u->at(i, j) +
-                beta->at(j) * (q->at(i, j) + beta->at(j) * p->at(i, j));
-        }
-    }
+        },
+        p->get_size(), r, u, p, q, beta, rho, prev_rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const OmpExecutor> exec,
+void step_2(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *u,
             const matrix::Dense<ValueType> *v_hat, matrix::Dense<ValueType> *q,
             matrix::Dense<ValueType> *t, matrix::Dense<ValueType> *alpha,
@@ -132,25 +123,20 @@ void step_2(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Dense<ValueType> *gamma,
             const Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type j = 0; j < u->get_size()[1]; ++j) {
-        if (stop_status->get_const_data()[j].has_stopped()) {
-            continue;
-        }
-        if (gamma->at(j) != zero<ValueType>()) {
-            alpha->at(j) = rho->at(j) / gamma->at(j);
-        }
-    }
-#pragma omp parallel for
-    for (size_type i = 0; i < u->get_size()[0]; ++i) {
-        for (size_type j = 0; j < u->get_size()[1]; ++j) {
-            if (stop_status->get_const_data()[j].has_stopped()) {
-                continue;
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto u, auto v_hat, auto q, auto t,
+                      auto alpha, auto rho, auto gamma, auto stop) {
+            if (!stop[col].has_stopped()) {
+                auto tmp = safe_divide(rho[col], gamma[col]);
+                if (row == 0) {
+                    alpha[col] = tmp;
+                }
+                q(row, col) = u(row, col) - tmp * v_hat(row, col);
+                t(row, col) = u(row, col) + q(row, col);
             }
-            q->at(i, j) = u->at(i, j) - alpha->at(j) * v_hat->at(i, j);
-            t->at(i, j) = u->at(i, j) + q->at(i, j);
-        }
-    }
+        },
+        u->get_size(), u, v_hat, q, t, alpha, rho, gamma, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_2_KERNEL);
@@ -162,22 +148,22 @@ void step_3(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *x, const matrix::Dense<ValueType> *alpha,
             const Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type i = 0; i < x->get_size()[0]; ++i) {
-        for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            if (stop_status->get_const_data()[j].has_stopped()) {
-                continue;
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto t, auto u_hat, auto r, auto x,
+                      auto alpha, auto stop) {
+            if (!stop[col].has_stopped()) {
+                x(row, col) += alpha[col] * u_hat(row, col);
+                r(row, col) -= alpha[col] * t(row, col);
             }
-            x->at(i, j) += alpha->at(j) * u_hat->at(i, j);
-            r->at(i, j) -= alpha->at(j) * t->at(i, j);
-        }
-    }
+        },
+        t->get_size(), t, u_hat, r, x, alpha, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_3_KERNEL);
 
 
 }  // namespace cgs
-}  // namespace omp
+}  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
 }  // namespace gko
diff --git a/common/solver/cgs_kernels.hpp.inc b/common/solver/cgs_kernels.hpp.inc
deleted file mode 100644
index 0aba6af73f3..00000000000
--- a/common/solver/cgs_kernels.hpp.inc
+++ /dev/null
@@ -1,137 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void initialize_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    const ValueType *__restrict__ b, ValueType *__restrict__ r,
-    ValueType *__restrict__ r_tld, ValueType *__restrict__ p,
-    ValueType *__restrict__ q, ValueType *__restrict__ u,
-    ValueType *__restrict__ u_hat, ValueType *__restrict__ v_hat,
-    ValueType *__restrict__ t, ValueType *__restrict__ alpha,
-    ValueType *__restrict__ beta, ValueType *__restrict__ gamma,
-    ValueType *__restrict__ rho_prev, ValueType *__restrict__ rho,
-    stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-
-    if (tidx < num_cols) {
-        rho[tidx] = zero<ValueType>();
-        alpha[tidx] = one<ValueType>();
-        beta[tidx] = one<ValueType>();
-        gamma[tidx] = one<ValueType>();
-        rho_prev[tidx] = one<ValueType>();
-        stop_status[tidx].reset();
-    }
-
-    if (tidx < num_rows * stride) {
-        r[tidx] = b[tidx];
-        r_tld[tidx] = b[tidx];
-        u[tidx] = zero<ValueType>();
-        p[tidx] = zero<ValueType>();
-        q[tidx] = zero<ValueType>();
-        u_hat[tidx] = zero<ValueType>();
-        v_hat[tidx] = zero<ValueType>();
-        t[tidx] = zero<ValueType>();
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void step_1_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    const ValueType *__restrict__ r, ValueType *__restrict__ u,
-    ValueType *__restrict__ p, const ValueType *__restrict__ q,
-    ValueType *__restrict__ beta, const ValueType *__restrict__ rho,
-    const ValueType *__restrict__ rho_prev,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto col = tidx % stride;
-
-    if (col >= num_cols || tidx >= num_rows * stride ||
-        stop_status[col].has_stopped()) {
-        return;
-    }
-    if (rho_prev[col] != zero<ValueType>()) {
-        beta[col] = rho[col] / rho_prev[col];
-        u[tidx] = r[tidx] + beta[col] * q[tidx];
-        p[tidx] = u[tidx] + beta[col] * (q[tidx] + beta[col] * p[tidx]);
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void step_2_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    const ValueType *__restrict__ u, const ValueType *__restrict__ v_hat,
-    ValueType *__restrict__ q, ValueType *__restrict__ t,
-    ValueType *__restrict__ alpha, const ValueType *__restrict__ rho,
-    const ValueType *__restrict__ gamma,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto col = tidx % stride;
-
-    if (col >= num_cols || tidx >= num_rows * stride ||
-        stop_status[col].has_stopped()) {
-        return;
-    }
-    if (gamma[col] != zero<ValueType>()) {
-        alpha[col] = rho[col] / gamma[col];
-        q[tidx] = u[tidx] - alpha[col] * v_hat[tidx];
-        t[tidx] = u[tidx] + q[tidx];
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void step_3_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    size_type x_stride, const ValueType *__restrict__ t,
-    const ValueType *__restrict__ v_hat, ValueType *__restrict__ r,
-    ValueType *__restrict__ x, const ValueType *__restrict__ alpha,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto row = tidx / stride;
-    const auto col = tidx % stride;
-    if (col >= num_cols || tidx >= num_rows * stride ||
-        stop_status[col].has_stopped()) {
-        return;
-    }
-    const auto x_pos = row * x_stride + col;
-    auto t_x = x[x_pos] + alpha[col] * v_hat[tidx];
-    auto t_r = r[tidx] - alpha[col] * t[tidx];
-    x[x_pos] = t_x;
-    r[tidx] = t_r;
-}
\ No newline at end of file
diff --git a/omp/solver/fcg_kernels.cpp b/common/solver/fcg_kernels.cpp
similarity index 63%
rename from omp/solver/fcg_kernels.cpp
rename to common/solver/fcg_kernels.cpp
index 3bba54d5b0f..da6acec75f9 100644
--- a/omp/solver/fcg_kernels.cpp
+++ b/common/solver/fcg_kernels.cpp
@@ -33,17 +33,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/fcg_kernels.hpp"
 
 
-#include <omp.h>
+#include <ginkgo/core/base/math.hpp>
 
 
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
+#include "core/base/simple_kernels.hpp"
 
 
 namespace gko {
 namespace kernels {
-namespace omp {
+namespace GKO_DEVICE_NAMESPACE {
 /**
  * @brief The FCG solver namespace.
  *
@@ -53,7 +51,7 @@ namespace fcg {
 
 
 template <typename ValueType>
-void initialize(std::shared_ptr<const OmpExecutor> exec,
+void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
                 matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
                 matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *t,
@@ -61,53 +59,49 @@ void initialize(std::shared_ptr<const OmpExecutor> exec,
                 matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *rho_t,
                 Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type j = 0; j < b->get_size()[1]; ++j) {
-        rho->at(j) = zero<ValueType>();
-        prev_rho->at(j) = one<ValueType>();
-        rho_t->at(j) = one<ValueType>();
-        stop_status->get_data()[j].reset();
-    }
-#pragma omp parallel for
-    for (size_type i = 0; i < b->get_size()[0]; ++i) {
-        for (size_type j = 0; j < b->get_size()[1]; ++j) {
-            t->at(i, j) = r->at(i, j) = b->at(i, j);
-            z->at(i, j) = p->at(i, j) = q->at(i, j) = zero<ValueType>();
-        }
-    }
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto z, auto p,
+                      auto q, auto t, auto prev_rho, auto rho, auto rho_t,
+                      auto stop) {
+            if (row == 0) {
+                rho[col] = zero(rho[col]);
+                prev_rho[col] = rho_t[col] = one(prev_rho[col]);
+                stop[col].reset();
+            }
+            t(row, col) = r(row, col) = b(row, col);
+            z(row, col) = p(row, col) = q(row, col) = zero(z(row, col));
+        },
+        p->get_size(), b, r, z, p, q, t, prev_rho, rho, rho_t, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_INITIALIZE_KERNEL);
 
 
 template <typename ValueType>
-void step_1(std::shared_ptr<const OmpExecutor> exec,
+void step_1(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
             const matrix::Dense<ValueType> *rho_t,
             const matrix::Dense<ValueType> *prev_rho,
             const Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type i = 0; i < p->get_size()[0]; ++i) {
-        for (size_type j = 0; j < p->get_size()[1]; ++j) {
-            if (stop_status->get_const_data()[j].has_stopped()) {
-                continue;
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto p, auto z, auto rho_t,
+                      auto prev_rho, auto stop) {
+            if (!stop[col].has_stopped()) {
+                auto tmp = safe_divide(rho_t[col], prev_rho[col]);
+                p(row, col) = z(row, col) + tmp * p(row, col);
             }
-            if (prev_rho->at(j) == zero<ValueType>()) {
-                p->at(i, j) = z->at(i, j);
-            } else {
-                auto tmp = rho_t->at(j) / prev_rho->at(j);
-                p->at(i, j) = z->at(i, j) + tmp * p->at(i, j);
-            }
-        }
-    }
+        },
+        p->get_size(), p, z, rho_t, prev_rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_1_KERNEL);
 
 
 template <typename ValueType>
-void step_2(std::shared_ptr<const OmpExecutor> exec,
+void step_2(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
             matrix::Dense<ValueType> *t, const matrix::Dense<ValueType> *p,
             const matrix::Dense<ValueType> *q,
@@ -115,27 +109,25 @@ void step_2(std::shared_ptr<const OmpExecutor> exec,
             const matrix::Dense<ValueType> *rho,
             const Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type i = 0; i < x->get_size()[0]; ++i) {
-        for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            if (stop_status->get_const_data()[j].has_stopped()) {
-                continue;
-            }
-            if (beta->at(j) != zero<ValueType>()) {
-                auto tmp = rho->at(j) / beta->at(j);
-                auto prev_r = r->at(i, j);
-                x->at(i, j) += tmp * p->at(i, j);
-                r->at(i, j) -= tmp * q->at(i, j);
-                t->at(i, j) = r->at(i, j) - prev_r;
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto t, auto p,
+                      auto q, auto beta, auto rho, auto stop) {
+            if (!stop[col].has_stopped()) {
+                auto tmp = safe_divide(rho[col], beta[col]);
+                auto prev_r = r(row, col);
+                x(row, col) += tmp * p(row, col);
+                r(row, col) -= tmp * q(row, col);
+                t(row, col) = r(row, col) - prev_r;
             }
-        }
-    }
+        },
+        x->get_size(), x, r, t, p, q, beta, rho, *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_2_KERNEL);
 
 
 }  // namespace fcg
-}  // namespace omp
+}  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
 }  // namespace gko
diff --git a/common/solver/fcg_kernels.hpp.inc b/common/solver/fcg_kernels.hpp.inc
deleted file mode 100644
index 312b0bad50d..00000000000
--- a/common/solver/fcg_kernels.hpp.inc
+++ /dev/null
@@ -1,104 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void initialize_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    const ValueType *__restrict__ b, ValueType *__restrict__ r,
-    ValueType *__restrict__ z, ValueType *__restrict__ p,
-    ValueType *__restrict__ q, ValueType *__restrict__ t,
-    ValueType *__restrict__ prev_rho, ValueType *__restrict__ rho,
-    ValueType *__restrict__ rho_t, stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-
-    if (tidx < num_cols) {
-        rho[tidx] = zero<ValueType>();
-        prev_rho[tidx] = one<ValueType>();
-        rho_t[tidx] = one<ValueType>();
-        stop_status[tidx].reset();
-    }
-
-    if (tidx < num_rows * stride) {
-        r[tidx] = b[tidx];
-        z[tidx] = zero<ValueType>();
-        p[tidx] = zero<ValueType>();
-        q[tidx] = zero<ValueType>();
-        t[tidx] = b[tidx];
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void step_1_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    ValueType *__restrict__ p, const ValueType *__restrict__ z,
-    const ValueType *__restrict__ rho, const ValueType *__restrict__ prev_rho,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto col = tidx % stride;
-    if (col >= num_cols || tidx >= num_rows * stride ||
-        stop_status[col].has_stopped()) {
-        return;
-    }
-    const auto tmp = rho[col] / prev_rho[col];
-    p[tidx] =
-        prev_rho[col] == zero<ValueType>() ? z[tidx] : z[tidx] + tmp * p[tidx];
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void step_2_kernel(
-    size_type num_rows, size_type num_cols, size_type stride,
-    size_type x_stride, ValueType *__restrict__ x, ValueType *__restrict__ r,
-    ValueType *__restrict__ t, const ValueType *__restrict__ p,
-    const ValueType *__restrict__ q, const ValueType *__restrict__ beta,
-    const ValueType *__restrict__ rho,
-    const stopping_status *__restrict__ stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto row = tidx / stride;
-    const auto col = tidx % stride;
-
-    if (col >= num_cols || tidx >= num_rows * num_cols ||
-        stop_status[col].has_stopped()) {
-        return;
-    }
-    if (beta[col] != zero<ValueType>()) {
-        const auto tmp = rho[col] / beta[col];
-        const auto prev_r = r[tidx];
-        x[row * x_stride + col] += tmp * p[tidx];
-        r[tidx] -= tmp * q[tidx];
-        t[tidx] = r[tidx] - prev_r;
-    }
-}
\ No newline at end of file
diff --git a/omp/solver/ir_kernels.cpp b/common/solver/ir_kernels.cpp
similarity index 86%
rename from omp/solver/ir_kernels.cpp
rename to common/solver/ir_kernels.cpp
index 4b9c8b21a61..6fe9e5721d2 100644
--- a/omp/solver/ir_kernels.cpp
+++ b/common/solver/ir_kernels.cpp
@@ -33,12 +33,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/ir_kernels.hpp"
 
 
-#include <omp.h>
+#include "core/base/simple_kernels.hpp"
 
 
 namespace gko {
 namespace kernels {
-namespace omp {
+namespace GKO_DEVICE_NAMESPACE {
 /**
  * @brief The IR solver namespace.
  *
@@ -47,17 +47,16 @@ namespace omp {
 namespace ir {
 
 
-void initialize(std::shared_ptr<const OmpExecutor> exec,
+void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 Array<stopping_status> *stop_status)
 {
-#pragma omp parallel for
-    for (size_type j = 0; j < stop_status->get_num_elems(); ++j) {
-        stop_status->get_data()[j].reset();
-    }
+    run_kernel(
+        exec, [] GKO_KERNEL(auto i, auto stop) { stop[i].reset(); },
+        stop_status->get_num_elems(), *stop_status);
 }
 
 
 }  // namespace ir
-}  // namespace omp
+}  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
 }  // namespace gko
diff --git a/common/solver/ir_kernels.hpp.inc b/common/solver/ir_kernels.hpp.inc
deleted file mode 100644
index 39a8d04f7fc..00000000000
--- a/common/solver/ir_kernels.hpp.inc
+++ /dev/null
@@ -1,41 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-__global__ __launch_bounds__(default_block_size) void initialize_kernel(
-    size_type num_cols, stopping_status *stop_status)
-{
-    const auto tidx = thread::get_thread_id_flat();
-
-    if (tidx < num_cols) {
-        stop_status[tidx].reset();
-    }
-}
\ No newline at end of file
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 86fcb315904..99301a6afe3 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -68,7 +68,12 @@ add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_cuda_device> "")
 set(GKO_CUDA_COMMON_SOURCES    
     ../common/components/precision_conversion.cpp
     ../common/matrix/dense_kernels.cpp
+    ../common/solver/bicg_kernels.cpp
+    ../common/solver/bicgstab_kernels.cpp
     ../common/solver/cg_kernels.cpp
+    ../common/solver/cgs_kernels.cpp
+    ../common/solver/fcg_kernels.cpp
+    ../common/solver/ir_kernels.cpp
     )
 target_sources(ginkgo_cuda
     PRIVATE
@@ -106,14 +111,9 @@ target_sources(ginkgo_cuda
     preconditioner/jacobi_kernels.cu
     preconditioner/jacobi_simple_apply_kernel.cu
     reorder/rcm_kernels.cu
-    solver/bicg_kernels.cu
-    solver/bicgstab_kernels.cu
-    solver/cgs_kernels.cu
-    solver/fcg_kernels.cu
     solver/gmres_kernels.cu
     solver/cb_gmres_kernels.cu
     solver/idr_kernels.cu
-    solver/ir_kernels.cu
     solver/lower_trs_kernels.cu
     solver/upper_trs_kernels.cu
     stop/criterion_kernels.cu
diff --git a/cuda/solver/bicg_kernels.cu b/cuda/solver/bicg_kernels.cu
deleted file mode 100644
index 414d637f01c..00000000000
--- a/cuda/solver/bicg_kernels.cu
+++ /dev/null
@@ -1,144 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/bicg_kernels.hpp"
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-
-
-#include "cuda/base/math.hpp"
-#include "cuda/base/types.hpp"
-#include "cuda/components/thread_ids.cuh"
-
-
-namespace gko {
-namespace kernels {
-namespace cuda {
-/**
- * @brief The BICG solver namespace.
- *
- * @ingroup bicg
- */
-namespace bicg {
-
-
-constexpr int default_block_size = 512;
-
-
-#include "common/solver/bicg_kernels.hpp.inc"
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const CudaExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
-                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
-                matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *r2,
-                matrix::Dense<ValueType> *z2, matrix::Dense<ValueType> *p2,
-                matrix::Dense<ValueType> *q2,
-                Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(b->get_size()[0] * b->get_stride(), block_size.x), 1, 1);
-
-    initialize_kernel<<<grid_size, block_size, 0, 0>>>(
-        b->get_size()[0], b->get_size()[1], b->get_stride(),
-        as_cuda_type(b->get_const_values()), as_cuda_type(r->get_values()),
-        as_cuda_type(z->get_values()), as_cuda_type(p->get_values()),
-        as_cuda_type(q->get_values()), as_cuda_type(r2->get_values()),
-        as_cuda_type(z2->get_values()), as_cuda_type(p2->get_values()),
-        as_cuda_type(q2->get_values()), as_cuda_type(prev_rho->get_values()),
-        as_cuda_type(rho->get_values()), as_cuda_type(stop_status->get_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const CudaExecutor> exec,
-            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
-            matrix::Dense<ValueType> *p2, const matrix::Dense<ValueType> *z2,
-            const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *prev_rho,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(p->get_size()[0] * p->get_stride(), block_size.x), 1, 1);
-
-    step_1_kernel<<<grid_size, block_size, 0, 0>>>(
-        p->get_size()[0], p->get_size()[1], p->get_stride(),
-        as_cuda_type(p->get_values()), as_cuda_type(z->get_const_values()),
-        as_cuda_type(p2->get_values()), as_cuda_type(z2->get_const_values()),
-        as_cuda_type(rho->get_const_values()),
-        as_cuda_type(prev_rho->get_const_values()),
-        as_cuda_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const CudaExecutor> exec,
-            matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
-            matrix::Dense<ValueType> *r2, const matrix::Dense<ValueType> *p,
-            const matrix::Dense<ValueType> *q,
-            const matrix::Dense<ValueType> *q2,
-            const matrix::Dense<ValueType> *beta,
-            const matrix::Dense<ValueType> *rho,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(p->get_size()[0] * p->get_stride(), block_size.x), 1, 1);
-
-    step_2_kernel<<<grid_size, block_size, 0, 0>>>(
-        p->get_size()[0], p->get_size()[1], p->get_stride(), x->get_stride(),
-        as_cuda_type(x->get_values()), as_cuda_type(r->get_values()),
-        as_cuda_type(r2->get_values()), as_cuda_type(p->get_const_values()),
-        as_cuda_type(q->get_const_values()),
-        as_cuda_type(q2->get_const_values()),
-        as_cuda_type(beta->get_const_values()),
-        as_cuda_type(rho->get_const_values()),
-        as_cuda_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_2_KERNEL);
-
-
-}  // namespace bicg
-}  // namespace cuda
-}  // namespace kernels
-}  // namespace gko
diff --git a/cuda/solver/bicgstab_kernels.cu b/cuda/solver/bicgstab_kernels.cu
deleted file mode 100644
index 959ddc15283..00000000000
--- a/cuda/solver/bicgstab_kernels.cu
+++ /dev/null
@@ -1,201 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/bicgstab_kernels.hpp"
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-
-
-#include "cuda/base/math.hpp"
-#include "cuda/base/types.hpp"
-#include "cuda/components/thread_ids.cuh"
-
-
-namespace gko {
-namespace kernels {
-namespace cuda {
-/**
- * @brief The BICGSTAB solver namespace.
- *
- * @ingroup bicgstab
- */
-namespace bicgstab {
-
-
-constexpr int default_block_size = 512;
-
-
-#include "common/solver/bicgstab_kernels.hpp.inc"
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const CudaExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *rr, matrix::Dense<ValueType> *y,
-                matrix::Dense<ValueType> *s, matrix::Dense<ValueType> *t,
-                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *v,
-                matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *prev_rho,
-                matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *alpha,
-                matrix::Dense<ValueType> *beta, matrix::Dense<ValueType> *gamma,
-                matrix::Dense<ValueType> *omega,
-                Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(b->get_size()[0] * b->get_stride(), block_size.x), 1, 1);
-
-    initialize_kernel<<<grid_size, block_size, 0, 0>>>(
-        b->get_size()[0], b->get_size()[1], b->get_stride(),
-        as_cuda_type(b->get_const_values()), as_cuda_type(r->get_values()),
-        as_cuda_type(rr->get_values()), as_cuda_type(y->get_values()),
-        as_cuda_type(s->get_values()), as_cuda_type(t->get_values()),
-        as_cuda_type(z->get_values()), as_cuda_type(v->get_values()),
-        as_cuda_type(p->get_values()), as_cuda_type(prev_rho->get_values()),
-        as_cuda_type(rho->get_values()), as_cuda_type(alpha->get_values()),
-        as_cuda_type(beta->get_values()), as_cuda_type(gamma->get_values()),
-        as_cuda_type(omega->get_values()),
-        as_cuda_type(stop_status->get_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const CudaExecutor> exec,
-            const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *p,
-            const matrix::Dense<ValueType> *v,
-            const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *prev_rho,
-            const matrix::Dense<ValueType> *alpha,
-            const matrix::Dense<ValueType> *omega,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-
-    step_1_kernel<<<grid_size, block_size, 0, 0>>>(
-        r->get_size()[0], r->get_size()[1], r->get_stride(),
-        as_cuda_type(r->get_const_values()), as_cuda_type(p->get_values()),
-        as_cuda_type(v->get_const_values()),
-        as_cuda_type(rho->get_const_values()),
-        as_cuda_type(prev_rho->get_const_values()),
-        as_cuda_type(alpha->get_const_values()),
-        as_cuda_type(omega->get_const_values()),
-        as_cuda_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const CudaExecutor> exec,
-            const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *s,
-            const matrix::Dense<ValueType> *v,
-            const matrix::Dense<ValueType> *rho,
-            matrix::Dense<ValueType> *alpha,
-            const matrix::Dense<ValueType> *beta,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-
-    step_2_kernel<<<grid_size, block_size, 0, 0>>>(
-        r->get_size()[0], r->get_size()[1], r->get_stride(),
-        as_cuda_type(r->get_const_values()), as_cuda_type(s->get_values()),
-        as_cuda_type(v->get_const_values()),
-        as_cuda_type(rho->get_const_values()),
-        as_cuda_type(alpha->get_values()),
-        as_cuda_type(beta->get_const_values()),
-        as_cuda_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_2_KERNEL);
-
-
-template <typename ValueType>
-void step_3(
-    std::shared_ptr<const CudaExecutor> exec, matrix::Dense<ValueType> *x,
-    matrix::Dense<ValueType> *r, const matrix::Dense<ValueType> *s,
-    const matrix::Dense<ValueType> *t, const matrix::Dense<ValueType> *y,
-    const matrix::Dense<ValueType> *z, const matrix::Dense<ValueType> *alpha,
-    const matrix::Dense<ValueType> *beta, const matrix::Dense<ValueType> *gamma,
-    matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-
-    step_3_kernel<<<grid_size, block_size, 0, 0>>>(
-        r->get_size()[0], r->get_size()[1], r->get_stride(), x->get_stride(),
-        as_cuda_type(x->get_values()), as_cuda_type(r->get_values()),
-        as_cuda_type(s->get_const_values()),
-        as_cuda_type(t->get_const_values()),
-        as_cuda_type(y->get_const_values()),
-        as_cuda_type(z->get_const_values()),
-        as_cuda_type(alpha->get_const_values()),
-        as_cuda_type(beta->get_const_values()),
-        as_cuda_type(gamma->get_const_values()),
-        as_cuda_type(omega->get_values()),
-        as_cuda_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_3_KERNEL);
-
-
-template <typename ValueType>
-void finalize(std::shared_ptr<const CudaExecutor> exec,
-              matrix::Dense<ValueType> *x, const matrix::Dense<ValueType> *y,
-              const matrix::Dense<ValueType> *alpha,
-              Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(y->get_size()[0] * y->get_stride(), block_size.x), 1, 1);
-
-    finalize_kernel<<<grid_size, block_size, 0, 0>>>(
-        y->get_size()[0], y->get_size()[1], y->get_stride(), x->get_stride(),
-        as_cuda_type(x->get_values()), as_cuda_type(y->get_const_values()),
-        as_cuda_type(alpha->get_const_values()),
-        as_cuda_type(stop_status->get_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_FINALIZE_KERNEL);
-
-
-}  // namespace bicgstab
-}  // namespace cuda
-}  // namespace kernels
-}  // namespace gko
diff --git a/cuda/solver/cgs_kernels.cu b/cuda/solver/cgs_kernels.cu
deleted file mode 100644
index ef80b20b1e3..00000000000
--- a/cuda/solver/cgs_kernels.cu
+++ /dev/null
@@ -1,169 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/cgs_kernels.hpp"
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-
-
-#include "cuda/base/math.hpp"
-#include "cuda/base/types.hpp"
-#include "cuda/components/thread_ids.cuh"
-
-
-namespace gko {
-namespace kernels {
-namespace cuda {
-/**
- * @brief The CGS solver namespace.
- *
- * @ingroup cgs
- */
-namespace cgs {
-
-
-constexpr int default_block_size = 512;
-
-
-#include "common/solver/cgs_kernels.hpp.inc"
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const CudaExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *r_tld, matrix::Dense<ValueType> *p,
-                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *u,
-                matrix::Dense<ValueType> *u_hat,
-                matrix::Dense<ValueType> *v_hat, matrix::Dense<ValueType> *t,
-                matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *beta,
-                matrix::Dense<ValueType> *gamma,
-                matrix::Dense<ValueType> *rho_prev,
-                matrix::Dense<ValueType> *rho,
-                Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(b->get_size()[0] * b->get_stride(), block_size.x), 1, 1);
-
-    initialize_kernel<<<grid_size, block_size, 0, 0>>>(
-        b->get_size()[0], b->get_size()[1], b->get_stride(),
-        as_cuda_type(b->get_const_values()), as_cuda_type(r->get_values()),
-        as_cuda_type(r_tld->get_values()), as_cuda_type(p->get_values()),
-        as_cuda_type(q->get_values()), as_cuda_type(u->get_values()),
-        as_cuda_type(u_hat->get_values()), as_cuda_type(v_hat->get_values()),
-        as_cuda_type(t->get_values()), as_cuda_type(alpha->get_values()),
-        as_cuda_type(beta->get_values()), as_cuda_type(gamma->get_values()),
-        as_cuda_type(rho_prev->get_values()), as_cuda_type(rho->get_values()),
-        as_cuda_type(stop_status->get_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const CudaExecutor> exec,
-            const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *u,
-            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *q,
-            matrix::Dense<ValueType> *beta, const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *rho_prev,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(p->get_size()[0] * p->get_stride(), block_size.x), 1, 1);
-
-    step_1_kernel<<<grid_size, block_size, 0, 0>>>(
-        p->get_size()[0], p->get_size()[1], p->get_stride(),
-        as_cuda_type(r->get_const_values()), as_cuda_type(u->get_values()),
-        as_cuda_type(p->get_values()), as_cuda_type(q->get_const_values()),
-        as_cuda_type(beta->get_values()), as_cuda_type(rho->get_const_values()),
-        as_cuda_type(rho_prev->get_const_values()),
-        as_cuda_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const CudaExecutor> exec,
-            const matrix::Dense<ValueType> *u,
-            const matrix::Dense<ValueType> *v_hat, matrix::Dense<ValueType> *q,
-            matrix::Dense<ValueType> *t, matrix::Dense<ValueType> *alpha,
-            const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *gamma,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(u->get_size()[0] * u->get_stride(), block_size.x), 1, 1);
-
-    step_2_kernel<<<grid_size, block_size, 0, 0>>>(
-        u->get_size()[0], u->get_size()[1], u->get_stride(),
-        as_cuda_type(u->get_const_values()),
-        as_cuda_type(v_hat->get_const_values()), as_cuda_type(q->get_values()),
-        as_cuda_type(t->get_values()), as_cuda_type(alpha->get_values()),
-        as_cuda_type(rho->get_const_values()),
-        as_cuda_type(gamma->get_const_values()),
-        as_cuda_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_2_KERNEL);
-
-
-template <typename ValueType>
-void step_3(std::shared_ptr<const CudaExecutor> exec,
-            const matrix::Dense<ValueType> *t,
-            const matrix::Dense<ValueType> *u_hat, matrix::Dense<ValueType> *r,
-            matrix::Dense<ValueType> *x, const matrix::Dense<ValueType> *alpha,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(t->get_size()[0] * t->get_stride(), block_size.x), 1, 1);
-
-    step_3_kernel<<<grid_size, block_size, 0, 0>>>(
-        t->get_size()[0], t->get_size()[1], t->get_stride(), x->get_stride(),
-        as_cuda_type(t->get_const_values()),
-        as_cuda_type(u_hat->get_const_values()), as_cuda_type(r->get_values()),
-        as_cuda_type(x->get_values()), as_cuda_type(alpha->get_const_values()),
-        as_cuda_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_3_KERNEL);
-
-
-}  // namespace cgs
-}  // namespace cuda
-}  // namespace kernels
-}  // namespace gko
diff --git a/cuda/solver/fcg_kernels.cu b/cuda/solver/fcg_kernels.cu
deleted file mode 100644
index 214fd59bddb..00000000000
--- a/cuda/solver/fcg_kernels.cu
+++ /dev/null
@@ -1,139 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/fcg_kernels.hpp"
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-
-
-#include "cuda/base/math.hpp"
-#include "cuda/base/types.hpp"
-#include "cuda/components/thread_ids.cuh"
-
-
-namespace gko {
-namespace kernels {
-namespace cuda {
-/**
- * @brief The FCG solver namespace.
- *
- * @ingroup fcg
- */
-namespace fcg {
-
-
-constexpr int default_block_size = 512;
-
-
-#include "common/solver/fcg_kernels.hpp.inc"
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const CudaExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
-                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *t,
-                matrix::Dense<ValueType> *prev_rho,
-                matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *rho_t,
-                Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(b->get_size()[0] * b->get_stride(), block_size.x), 1, 1);
-
-    initialize_kernel<<<grid_size, block_size, 0, 0>>>(
-        b->get_size()[0], b->get_size()[1], b->get_stride(),
-        as_cuda_type(b->get_const_values()), as_cuda_type(r->get_values()),
-        as_cuda_type(z->get_values()), as_cuda_type(p->get_values()),
-        as_cuda_type(q->get_values()), as_cuda_type(t->get_values()),
-        as_cuda_type(prev_rho->get_values()), as_cuda_type(rho->get_values()),
-        as_cuda_type(rho_t->get_values()),
-        as_cuda_type(stop_status->get_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const CudaExecutor> exec,
-            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
-            const matrix::Dense<ValueType> *rho_t,
-            const matrix::Dense<ValueType> *prev_rho,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(p->get_size()[0] * p->get_stride(), block_size.x), 1, 1);
-
-    step_1_kernel<<<grid_size, block_size, 0, 0>>>(
-        p->get_size()[0], p->get_size()[1], p->get_stride(),
-        as_cuda_type(p->get_values()), as_cuda_type(z->get_const_values()),
-        as_cuda_type(rho_t->get_const_values()),
-        as_cuda_type(prev_rho->get_const_values()),
-        as_cuda_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const CudaExecutor> exec,
-            matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
-            matrix::Dense<ValueType> *t, const matrix::Dense<ValueType> *p,
-            const matrix::Dense<ValueType> *q,
-            const matrix::Dense<ValueType> *beta,
-            const matrix::Dense<ValueType> *rho,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(p->get_size()[0] * p->get_stride(), block_size.x), 1, 1);
-
-    step_2_kernel<<<grid_size, block_size, 0, 0>>>(
-        p->get_size()[0], p->get_size()[1], p->get_stride(), x->get_stride(),
-        as_cuda_type(x->get_values()), as_cuda_type(r->get_values()),
-        as_cuda_type(t->get_values()), as_cuda_type(p->get_const_values()),
-        as_cuda_type(q->get_const_values()),
-        as_cuda_type(beta->get_const_values()),
-        as_cuda_type(rho->get_const_values()),
-        as_cuda_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_2_KERNEL);
-
-
-}  // namespace fcg
-}  // namespace cuda
-}  // namespace kernels
-}  // namespace gko
diff --git a/cuda/solver/ir_kernels.cu b/cuda/solver/ir_kernels.cu
deleted file mode 100644
index 02d6c5bf561..00000000000
--- a/cuda/solver/ir_kernels.cu
+++ /dev/null
@@ -1,74 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/ir_kernels.hpp"
-
-
-#include <ginkgo/core/base/math.hpp>
-
-
-#include "cuda/components/thread_ids.cuh"
-
-
-namespace gko {
-namespace kernels {
-namespace cuda {
-/**
- * @brief The IR solver namespace.
- *
- * @ingroup ir
- */
-namespace ir {
-
-
-constexpr int default_block_size = 512;
-
-
-#include "common/solver/ir_kernels.hpp.inc"
-
-
-void initialize(std::shared_ptr<const CudaExecutor> exec,
-                Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(ceildiv(stop_status->get_num_elems(), block_size.x), 1,
-                         1);
-
-    initialize_kernel<<<grid_size, block_size, 0, 0>>>(
-        stop_status->get_num_elems(), stop_status->get_data());
-}
-
-
-}  // namespace ir
-}  // namespace cuda
-}  // namespace kernels
-}  // namespace gko
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 853ccf5e215..e90777c4ad3 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -34,21 +34,21 @@ target_sources(ginkgo_dpcpp
     preconditioner/isai_kernels.dp.cpp
     preconditioner/jacobi_kernels.dp.cpp
     reorder/rcm_kernels.dp.cpp
-    solver/bicg_kernels.dp.cpp
-    solver/bicgstab_kernels.dp.cpp
-    solver/cgs_kernels.dp.cpp
-    solver/fcg_kernels.dp.cpp
     solver/gmres_kernels.dp.cpp
     solver/cb_gmres_kernels.dp.cpp
     solver/idr_kernels.dp.cpp
-    solver/ir_kernels.dp.cpp
     solver/lower_trs_kernels.dp.cpp
     solver/upper_trs_kernels.dp.cpp
     stop/criterion_kernels.dp.cpp
     stop/residual_norm_kernels.dp.cpp
     ../common/components/precision_conversion.cpp
     ../common/matrix/dense_kernels.cpp
+    ../common/solver/bicg_kernels.cpp
+    ../common/solver/bicgstab_kernels.cpp
     ../common/solver/cg_kernels.cpp
+    ../common/solver/cgs_kernels.cpp
+    ../common/solver/fcg_kernels.cpp
+    ../common/solver/ir_kernels.cpp
     )
 
 ginkgo_compile_features(ginkgo_dpcpp)
diff --git a/dpcpp/solver/bicg_kernels.dp.cpp b/dpcpp/solver/bicg_kernels.dp.cpp
deleted file mode 100644
index db558863934..00000000000
--- a/dpcpp/solver/bicg_kernels.dp.cpp
+++ /dev/null
@@ -1,96 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/bicg_kernels.hpp"
-
-
-#include <CL/sycl.hpp>
-
-
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
-
-
-namespace gko {
-namespace kernels {
-namespace dpcpp {
-/**
- * @brief The BICG solver namespace.
- *
- * @ingroup bicg
- */
-namespace bicg {
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const DpcppExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
-                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
-                matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *r2,
-                matrix::Dense<ValueType> *z2, matrix::Dense<ValueType> *p2,
-                matrix::Dense<ValueType> *q2,
-                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const DpcppExecutor> exec,
-            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
-            matrix::Dense<ValueType> *p2, const matrix::Dense<ValueType> *z2,
-            const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *prev_rho,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const DpcppExecutor> exec,
-            matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
-            matrix::Dense<ValueType> *r2, const matrix::Dense<ValueType> *p,
-            const matrix::Dense<ValueType> *q,
-            const matrix::Dense<ValueType> *q2,
-            const matrix::Dense<ValueType> *beta,
-            const matrix::Dense<ValueType> *rho,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_2_KERNEL);
-
-
-}  // namespace bicg
-}  // namespace dpcpp
-}  // namespace kernels
-}  // namespace gko
diff --git a/dpcpp/solver/bicgstab_kernels.dp.cpp b/dpcpp/solver/bicgstab_kernels.dp.cpp
deleted file mode 100644
index 70ecd523193..00000000000
--- a/dpcpp/solver/bicgstab_kernels.dp.cpp
+++ /dev/null
@@ -1,123 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/bicgstab_kernels.hpp"
-
-
-#include <algorithm>
-
-
-#include <CL/sycl.hpp>
-
-
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-
-
-namespace gko {
-namespace kernels {
-namespace dpcpp {
-/**
- * @brief The BICGSTAB solver namespace.
- *
- * @ingroup bicgstab
- */
-namespace bicgstab {
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const DpcppExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *rr, matrix::Dense<ValueType> *y,
-                matrix::Dense<ValueType> *s, matrix::Dense<ValueType> *t,
-                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *v,
-                matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *prev_rho,
-                matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *alpha,
-                matrix::Dense<ValueType> *beta, matrix::Dense<ValueType> *gamma,
-                matrix::Dense<ValueType> *omega,
-                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const DpcppExecutor> exec,
-            const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *p,
-            const matrix::Dense<ValueType> *v,
-            const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *prev_rho,
-            const matrix::Dense<ValueType> *alpha,
-            const matrix::Dense<ValueType> *omega,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const DpcppExecutor> exec,
-            const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *s,
-            const matrix::Dense<ValueType> *v,
-            const matrix::Dense<ValueType> *rho,
-            matrix::Dense<ValueType> *alpha,
-            const matrix::Dense<ValueType> *beta,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_2_KERNEL);
-
-
-template <typename ValueType>
-void step_3(
-    std::shared_ptr<const DpcppExecutor> exec, matrix::Dense<ValueType> *x,
-    matrix::Dense<ValueType> *r, const matrix::Dense<ValueType> *s,
-    const matrix::Dense<ValueType> *t, const matrix::Dense<ValueType> *y,
-    const matrix::Dense<ValueType> *z, const matrix::Dense<ValueType> *alpha,
-    const matrix::Dense<ValueType> *beta, const matrix::Dense<ValueType> *gamma,
-    matrix::Dense<ValueType> *omega,
-    const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_3_KERNEL);
-
-
-template <typename ValueType>
-void finalize(std::shared_ptr<const DpcppExecutor> exec,
-              matrix::Dense<ValueType> *x, const matrix::Dense<ValueType> *y,
-              const matrix::Dense<ValueType> *alpha,
-              Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_FINALIZE_KERNEL);
-
-
-}  // namespace bicgstab
-}  // namespace dpcpp
-}  // namespace kernels
-}  // namespace gko
diff --git a/dpcpp/solver/cgs_kernels.dp.cpp b/dpcpp/solver/cgs_kernels.dp.cpp
deleted file mode 100644
index ade102c1ac2..00000000000
--- a/dpcpp/solver/cgs_kernels.dp.cpp
+++ /dev/null
@@ -1,107 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/cgs_kernels.hpp"
-
-
-#include <CL/sycl.hpp>
-
-
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
-
-
-namespace gko {
-namespace kernels {
-namespace dpcpp {
-/**
- * @brief The CGS solver namespace.
- *
- * @ingroup cgs
- */
-namespace cgs {
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const DpcppExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *r_tld, matrix::Dense<ValueType> *p,
-                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *u,
-                matrix::Dense<ValueType> *u_hat,
-                matrix::Dense<ValueType> *v_hat, matrix::Dense<ValueType> *t,
-                matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *beta,
-                matrix::Dense<ValueType> *gamma,
-                matrix::Dense<ValueType> *prev_rho,
-                matrix::Dense<ValueType> *rho,
-                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const DpcppExecutor> exec,
-            const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *u,
-            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *q,
-            matrix::Dense<ValueType> *beta, const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *rho_prev,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const DpcppExecutor> exec,
-            const matrix::Dense<ValueType> *u,
-            const matrix::Dense<ValueType> *v_hat, matrix::Dense<ValueType> *q,
-            matrix::Dense<ValueType> *t, matrix::Dense<ValueType> *alpha,
-            const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *gamma,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_2_KERNEL);
-
-template <typename ValueType>
-void step_3(std::shared_ptr<const DefaultExecutor> exec,
-            const matrix::Dense<ValueType> *t,
-            const matrix::Dense<ValueType> *u_hat, matrix::Dense<ValueType> *r,
-            matrix::Dense<ValueType> *x, const matrix::Dense<ValueType> *alpha,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_3_KERNEL);
-
-
-}  // namespace cgs
-}  // namespace dpcpp
-}  // namespace kernels
-}  // namespace gko
diff --git a/dpcpp/solver/fcg_kernels.dp.cpp b/dpcpp/solver/fcg_kernels.dp.cpp
deleted file mode 100644
index 752beaa1884..00000000000
--- a/dpcpp/solver/fcg_kernels.dp.cpp
+++ /dev/null
@@ -1,92 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/fcg_kernels.hpp"
-
-
-#include <CL/sycl.hpp>
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
-
-
-namespace gko {
-namespace kernels {
-namespace dpcpp {
-/**
- * @brief The FCG solver namespace.
- *
- * @ingroup fcg
- */
-namespace fcg {
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const DpcppExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
-                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *t,
-                matrix::Dense<ValueType> *prev_rho,
-                matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *rho_t,
-                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const DpcppExecutor> exec,
-            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
-            const matrix::Dense<ValueType> *rho_t,
-            const matrix::Dense<ValueType> *prev_rho,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const DpcppExecutor> exec,
-            matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
-            matrix::Dense<ValueType> *t, const matrix::Dense<ValueType> *p,
-            const matrix::Dense<ValueType> *q,
-            const matrix::Dense<ValueType> *beta,
-            const matrix::Dense<ValueType> *rho,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_2_KERNEL);
-
-
-}  // namespace fcg
-}  // namespace dpcpp
-}  // namespace kernels
-}  // namespace gko
diff --git a/dpcpp/solver/ir_kernels.dp.cpp b/dpcpp/solver/ir_kernels.dp.cpp
deleted file mode 100644
index a60d9b5273d..00000000000
--- a/dpcpp/solver/ir_kernels.dp.cpp
+++ /dev/null
@@ -1,57 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/ir_kernels.hpp"
-
-
-#include <CL/sycl.hpp>
-
-
-namespace gko {
-namespace kernels {
-namespace dpcpp {
-/**
- * @brief The IR solver namespace.
- *
- * @ingroup ir
- */
-namespace ir {
-
-
-void initialize(std::shared_ptr<const DpcppExecutor> exec,
-                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
-
-
-}  // namespace ir
-}  // namespace dpcpp
-}  // namespace kernels
-}  // namespace gko
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 1da3755f75c..e6f6199bc48 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -178,21 +178,21 @@ set(GINKGO_HIP_SOURCES
     preconditioner/jacobi_kernels.hip.cpp
     preconditioner/jacobi_simple_apply_kernel.hip.cpp
     reorder/rcm_kernels.hip.cpp
-    solver/bicg_kernels.hip.cpp
-    solver/bicgstab_kernels.hip.cpp
-    solver/cgs_kernels.hip.cpp
-    solver/fcg_kernels.hip.cpp
     solver/gmres_kernels.hip.cpp
     solver/cb_gmres_kernels.hip.cpp
     solver/idr_kernels.hip.cpp
-    solver/ir_kernels.hip.cpp
     solver/lower_trs_kernels.hip.cpp
     solver/upper_trs_kernels.hip.cpp
     stop/criterion_kernels.hip.cpp
     stop/residual_norm_kernels.hip.cpp
     ../common/components/precision_conversion.cpp
     ../common/matrix/dense_kernels.cpp
+    ../common/solver/bicg_kernels.cpp
+    ../common/solver/bicgstab_kernels.cpp
     ../common/solver/cg_kernels.cpp
+    ../common/solver/cgs_kernels.cpp
+    ../common/solver/fcg_kernels.cpp
+    ../common/solver/ir_kernels.cpp
     )
 
 set(GINKGO_HIP_NVCC_ARCH "")
diff --git a/hip/solver/bicg_kernels.hip.cpp b/hip/solver/bicg_kernels.hip.cpp
deleted file mode 100644
index 62bfd05fb14..00000000000
--- a/hip/solver/bicg_kernels.hip.cpp
+++ /dev/null
@@ -1,149 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/bicg_kernels.hpp"
-
-
-#include <hip/hip_runtime.h>
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-
-
-#include "hip/base/math.hip.hpp"
-#include "hip/base/types.hip.hpp"
-#include "hip/components/thread_ids.hip.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace hip {
-/**
- * @brief The BICG solver namespace.
- *
- * @ingroup bicg
- */
-namespace bicg {
-
-
-constexpr int default_block_size = 512;
-
-
-#include "common/solver/bicg_kernels.hpp.inc"
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const HipExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
-                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *prev_rho,
-                matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *r2,
-                matrix::Dense<ValueType> *z2, matrix::Dense<ValueType> *p2,
-                matrix::Dense<ValueType> *q2,
-                Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(b->get_size()[0] * b->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(
-        initialize_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-        b->get_size()[0], b->get_size()[1], b->get_stride(),
-        as_hip_type(b->get_const_values()), as_hip_type(r->get_values()),
-        as_hip_type(z->get_values()), as_hip_type(p->get_values()),
-        as_hip_type(q->get_values()), as_hip_type(r2->get_values()),
-        as_hip_type(z2->get_values()), as_hip_type(p2->get_values()),
-        as_hip_type(q2->get_values()), as_hip_type(prev_rho->get_values()),
-        as_hip_type(rho->get_values()), as_hip_type(stop_status->get_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const HipExecutor> exec,
-            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
-            matrix::Dense<ValueType> *p2, const matrix::Dense<ValueType> *z2,
-            const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *prev_rho,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(p->get_size()[0] * p->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(
-        step_1_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-        p->get_size()[0], p->get_size()[1], p->get_stride(),
-        as_hip_type(p->get_values()), as_hip_type(z->get_const_values()),
-        as_hip_type(p2->get_values()), as_hip_type(z2->get_const_values()),
-        as_hip_type(rho->get_const_values()),
-        as_hip_type(prev_rho->get_const_values()),
-        as_hip_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const HipExecutor> exec,
-            matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
-            matrix::Dense<ValueType> *r2, const matrix::Dense<ValueType> *p,
-            const matrix::Dense<ValueType> *q,
-            const matrix::Dense<ValueType> *q2,
-            const matrix::Dense<ValueType> *beta,
-            const matrix::Dense<ValueType> *rho,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(p->get_size()[0] * p->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(
-        step_2_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-        p->get_size()[0], p->get_size()[1], p->get_stride(), x->get_stride(),
-        as_hip_type(x->get_values()), as_hip_type(r->get_values()),
-        as_hip_type(r2->get_values()), as_hip_type(p->get_const_values()),
-        as_hip_type(q->get_const_values()), as_hip_type(q2->get_const_values()),
-        as_hip_type(beta->get_const_values()),
-        as_hip_type(rho->get_const_values()),
-        as_hip_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_2_KERNEL);
-
-
-}  // namespace bicg
-}  // namespace hip
-}  // namespace kernels
-}  // namespace gko
diff --git a/hip/solver/bicgstab_kernels.hip.cpp b/hip/solver/bicgstab_kernels.hip.cpp
deleted file mode 100644
index 8b6a43ac70f..00000000000
--- a/hip/solver/bicgstab_kernels.hip.cpp
+++ /dev/null
@@ -1,205 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/bicgstab_kernels.hpp"
-
-
-#include <hip/hip_runtime.h>
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-
-
-#include "hip/base/math.hip.hpp"
-#include "hip/base/types.hip.hpp"
-#include "hip/components/thread_ids.hip.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace hip {
-/**
- * @brief The BICGSTAB solver namespace.
- *
- * @ingroup bicgstab
- */
-namespace bicgstab {
-
-
-constexpr int default_block_size = 512;
-
-
-#include "common/solver/bicgstab_kernels.hpp.inc"
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const HipExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *rr, matrix::Dense<ValueType> *y,
-                matrix::Dense<ValueType> *s, matrix::Dense<ValueType> *t,
-                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *v,
-                matrix::Dense<ValueType> *p, matrix::Dense<ValueType> *prev_rho,
-                matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *alpha,
-                matrix::Dense<ValueType> *beta, matrix::Dense<ValueType> *gamma,
-                matrix::Dense<ValueType> *omega,
-                Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(b->get_size()[0] * b->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(
-        initialize_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-        b->get_size()[0], b->get_size()[1], b->get_stride(),
-        as_hip_type(b->get_const_values()), as_hip_type(r->get_values()),
-        as_hip_type(rr->get_values()), as_hip_type(y->get_values()),
-        as_hip_type(s->get_values()), as_hip_type(t->get_values()),
-        as_hip_type(z->get_values()), as_hip_type(v->get_values()),
-        as_hip_type(p->get_values()), as_hip_type(prev_rho->get_values()),
-        as_hip_type(rho->get_values()), as_hip_type(alpha->get_values()),
-        as_hip_type(beta->get_values()), as_hip_type(gamma->get_values()),
-        as_hip_type(omega->get_values()), as_hip_type(stop_status->get_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const HipExecutor> exec,
-            const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *p,
-            const matrix::Dense<ValueType> *v,
-            const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *prev_rho,
-            const matrix::Dense<ValueType> *alpha,
-            const matrix::Dense<ValueType> *omega,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(step_1_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-                       r->get_size()[0], r->get_size()[1], r->get_stride(),
-                       as_hip_type(r->get_const_values()),
-                       as_hip_type(p->get_values()),
-                       as_hip_type(v->get_const_values()),
-                       as_hip_type(rho->get_const_values()),
-                       as_hip_type(prev_rho->get_const_values()),
-                       as_hip_type(alpha->get_const_values()),
-                       as_hip_type(omega->get_const_values()),
-                       as_hip_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const HipExecutor> exec,
-            const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *s,
-            const matrix::Dense<ValueType> *v,
-            const matrix::Dense<ValueType> *rho,
-            matrix::Dense<ValueType> *alpha,
-            const matrix::Dense<ValueType> *beta,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(
-        step_2_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-        r->get_size()[0], r->get_size()[1], r->get_stride(),
-        as_hip_type(r->get_const_values()), as_hip_type(s->get_values()),
-        as_hip_type(v->get_const_values()),
-        as_hip_type(rho->get_const_values()), as_hip_type(alpha->get_values()),
-        as_hip_type(beta->get_const_values()),
-        as_hip_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_2_KERNEL);
-
-
-template <typename ValueType>
-void step_3(
-    std::shared_ptr<const HipExecutor> exec, matrix::Dense<ValueType> *x,
-    matrix::Dense<ValueType> *r, const matrix::Dense<ValueType> *s,
-    const matrix::Dense<ValueType> *t, const matrix::Dense<ValueType> *y,
-    const matrix::Dense<ValueType> *z, const matrix::Dense<ValueType> *alpha,
-    const matrix::Dense<ValueType> *beta, const matrix::Dense<ValueType> *gamma,
-    matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(r->get_size()[0] * r->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(
-        step_3_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-        r->get_size()[0], r->get_size()[1], r->get_stride(), x->get_stride(),
-        as_hip_type(x->get_values()), as_hip_type(r->get_values()),
-        as_hip_type(s->get_const_values()), as_hip_type(t->get_const_values()),
-        as_hip_type(y->get_const_values()), as_hip_type(z->get_const_values()),
-        as_hip_type(alpha->get_const_values()),
-        as_hip_type(beta->get_const_values()),
-        as_hip_type(gamma->get_const_values()),
-        as_hip_type(omega->get_values()),
-        as_hip_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_3_KERNEL);
-
-
-template <typename ValueType>
-void finalize(std::shared_ptr<const HipExecutor> exec,
-              matrix::Dense<ValueType> *x, const matrix::Dense<ValueType> *y,
-              const matrix::Dense<ValueType> *alpha,
-              Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(y->get_size()[0] * y->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(finalize_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-                       y->get_size()[0], y->get_size()[1], y->get_stride(),
-                       x->get_stride(), as_hip_type(x->get_values()),
-                       as_hip_type(y->get_const_values()),
-                       as_hip_type(alpha->get_const_values()),
-                       as_hip_type(stop_status->get_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_FINALIZE_KERNEL);
-
-
-}  // namespace bicgstab
-}  // namespace hip
-}  // namespace kernels
-}  // namespace gko
diff --git a/hip/solver/cgs_kernels.hip.cpp b/hip/solver/cgs_kernels.hip.cpp
deleted file mode 100644
index 8c5314648cc..00000000000
--- a/hip/solver/cgs_kernels.hip.cpp
+++ /dev/null
@@ -1,176 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/cgs_kernels.hpp"
-
-
-#include <hip/hip_runtime.h>
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-
-
-#include "hip/base/math.hip.hpp"
-#include "hip/base/types.hip.hpp"
-#include "hip/components/thread_ids.hip.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace hip {
-/**
- * @brief The CGS solver namespace.
- *
- * @ingroup cgs
- */
-namespace cgs {
-
-
-constexpr int default_block_size = 512;
-
-
-#include "common/solver/cgs_kernels.hpp.inc"
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const HipExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *r_tld, matrix::Dense<ValueType> *p,
-                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *u,
-                matrix::Dense<ValueType> *u_hat,
-                matrix::Dense<ValueType> *v_hat, matrix::Dense<ValueType> *t,
-                matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *beta,
-                matrix::Dense<ValueType> *gamma,
-                matrix::Dense<ValueType> *rho_prev,
-                matrix::Dense<ValueType> *rho,
-                Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(b->get_size()[0] * b->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(
-        initialize_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-        b->get_size()[0], b->get_size()[1], b->get_stride(),
-        as_hip_type(b->get_const_values()), as_hip_type(r->get_values()),
-        as_hip_type(r_tld->get_values()), as_hip_type(p->get_values()),
-        as_hip_type(q->get_values()), as_hip_type(u->get_values()),
-        as_hip_type(u_hat->get_values()), as_hip_type(v_hat->get_values()),
-        as_hip_type(t->get_values()), as_hip_type(alpha->get_values()),
-        as_hip_type(beta->get_values()), as_hip_type(gamma->get_values()),
-        as_hip_type(rho_prev->get_values()), as_hip_type(rho->get_values()),
-        as_hip_type(stop_status->get_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const HipExecutor> exec,
-            const matrix::Dense<ValueType> *r, matrix::Dense<ValueType> *u,
-            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *q,
-            matrix::Dense<ValueType> *beta, const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *rho_prev,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(p->get_size()[0] * p->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(
-        step_1_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-        p->get_size()[0], p->get_size()[1], p->get_stride(),
-        as_hip_type(r->get_const_values()), as_hip_type(u->get_values()),
-        as_hip_type(p->get_values()), as_hip_type(q->get_const_values()),
-        as_hip_type(beta->get_values()), as_hip_type(rho->get_const_values()),
-        as_hip_type(rho_prev->get_const_values()),
-        as_hip_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const HipExecutor> exec,
-            const matrix::Dense<ValueType> *u,
-            const matrix::Dense<ValueType> *v_hat, matrix::Dense<ValueType> *q,
-            matrix::Dense<ValueType> *t, matrix::Dense<ValueType> *alpha,
-            const matrix::Dense<ValueType> *rho,
-            const matrix::Dense<ValueType> *gamma,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(u->get_size()[0] * u->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(
-        step_2_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-        u->get_size()[0], u->get_size()[1], u->get_stride(),
-        as_hip_type(u->get_const_values()),
-        as_hip_type(v_hat->get_const_values()), as_hip_type(q->get_values()),
-        as_hip_type(t->get_values()), as_hip_type(alpha->get_values()),
-        as_hip_type(rho->get_const_values()),
-        as_hip_type(gamma->get_const_values()),
-        as_hip_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_2_KERNEL);
-
-
-template <typename ValueType>
-void step_3(std::shared_ptr<const HipExecutor> exec,
-            const matrix::Dense<ValueType> *t,
-            const matrix::Dense<ValueType> *u_hat, matrix::Dense<ValueType> *r,
-            matrix::Dense<ValueType> *x, const matrix::Dense<ValueType> *alpha,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(t->get_size()[0] * t->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(
-        step_3_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-        t->get_size()[0], t->get_size()[1], t->get_stride(), x->get_stride(),
-        as_hip_type(t->get_const_values()),
-        as_hip_type(u_hat->get_const_values()), as_hip_type(r->get_values()),
-        as_hip_type(x->get_values()), as_hip_type(alpha->get_const_values()),
-        as_hip_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_3_KERNEL);
-
-
-}  // namespace cgs
-}  // namespace hip
-}  // namespace kernels
-}  // namespace gko
diff --git a/hip/solver/fcg_kernels.hip.cpp b/hip/solver/fcg_kernels.hip.cpp
deleted file mode 100644
index c41bdeafc5b..00000000000
--- a/hip/solver/fcg_kernels.hip.cpp
+++ /dev/null
@@ -1,144 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/fcg_kernels.hpp"
-
-
-#include <hip/hip_runtime.h>
-
-
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-
-
-#include "hip/base/math.hip.hpp"
-#include "hip/base/types.hip.hpp"
-#include "hip/components/thread_ids.hip.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace hip {
-/**
- * @brief The FCG solver namespace.
- *
- * @ingroup fcg
- */
-namespace fcg {
-
-
-constexpr int default_block_size = 512;
-
-
-#include "common/solver/fcg_kernels.hpp.inc"
-
-
-template <typename ValueType>
-void initialize(std::shared_ptr<const HipExecutor> exec,
-                const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *r,
-                matrix::Dense<ValueType> *z, matrix::Dense<ValueType> *p,
-                matrix::Dense<ValueType> *q, matrix::Dense<ValueType> *t,
-                matrix::Dense<ValueType> *prev_rho,
-                matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *rho_t,
-                Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(b->get_size()[0] * b->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(
-        initialize_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-        b->get_size()[0], b->get_size()[1], b->get_stride(),
-        as_hip_type(b->get_const_values()), as_hip_type(r->get_values()),
-        as_hip_type(z->get_values()), as_hip_type(p->get_values()),
-        as_hip_type(q->get_values()), as_hip_type(t->get_values()),
-        as_hip_type(prev_rho->get_values()), as_hip_type(rho->get_values()),
-        as_hip_type(rho_t->get_values()), as_hip_type(stop_status->get_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_INITIALIZE_KERNEL);
-
-
-template <typename ValueType>
-void step_1(std::shared_ptr<const HipExecutor> exec,
-            matrix::Dense<ValueType> *p, const matrix::Dense<ValueType> *z,
-            const matrix::Dense<ValueType> *rho_t,
-            const matrix::Dense<ValueType> *prev_rho,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(p->get_size()[0] * p->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(step_1_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-                       p->get_size()[0], p->get_size()[1], p->get_stride(),
-                       as_hip_type(p->get_values()),
-                       as_hip_type(z->get_const_values()),
-                       as_hip_type(rho_t->get_const_values()),
-                       as_hip_type(prev_rho->get_const_values()),
-                       as_hip_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_1_KERNEL);
-
-
-template <typename ValueType>
-void step_2(std::shared_ptr<const HipExecutor> exec,
-            matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *r,
-            matrix::Dense<ValueType> *t, const matrix::Dense<ValueType> *p,
-            const matrix::Dense<ValueType> *q,
-            const matrix::Dense<ValueType> *beta,
-            const matrix::Dense<ValueType> *rho,
-            const Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(
-        ceildiv(p->get_size()[0] * p->get_stride(), block_size.x), 1, 1);
-
-    hipLaunchKernelGGL(
-        step_2_kernel, dim3(grid_size), dim3(block_size), 0, 0,
-        p->get_size()[0], p->get_size()[1], p->get_stride(), x->get_stride(),
-        as_hip_type(x->get_values()), as_hip_type(r->get_values()),
-        as_hip_type(t->get_values()), as_hip_type(p->get_const_values()),
-        as_hip_type(q->get_const_values()),
-        as_hip_type(beta->get_const_values()),
-        as_hip_type(rho->get_const_values()),
-        as_hip_type(stop_status->get_const_data()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_2_KERNEL);
-
-
-}  // namespace fcg
-}  // namespace hip
-}  // namespace kernels
-}  // namespace gko
diff --git a/hip/solver/ir_kernels.hip.cpp b/hip/solver/ir_kernels.hip.cpp
deleted file mode 100644
index 32c3fc6df02..00000000000
--- a/hip/solver/ir_kernels.hip.cpp
+++ /dev/null
@@ -1,78 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include "core/solver/ir_kernels.hpp"
-
-
-#include <hip/hip_runtime.h>
-
-
-#include <ginkgo/core/base/math.hpp>
-
-
-#include "hip/components/thread_ids.hip.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace hip {
-/**
- * @brief The IR solver namespace.
- *
- * @ingroup ir
- */
-namespace ir {
-
-
-constexpr int default_block_size = 512;
-
-
-#include "common/solver/ir_kernels.hpp.inc"
-
-
-void initialize(std::shared_ptr<const HipExecutor> exec,
-                Array<stopping_status> *stop_status)
-{
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(ceildiv(stop_status->get_num_elems(), block_size.x), 1,
-                         1);
-
-    hipLaunchKernelGGL(initialize_kernel, dim3(grid_size), dim3(block_size), 0,
-                       0, stop_status->get_num_elems(),
-                       stop_status->get_data());
-}
-
-
-}  // namespace ir
-}  // namespace hip
-}  // namespace kernels
-}  // namespace gko
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 762eef63049..21720933057 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -1008,6 +1008,24 @@ is_finite(const T &value)
 }
 
 
+/**
+ * Computes the quotient of the given parameters, guarding against division by
+ * zero.
+ *
+ * @tparam T  value type of the parameters
+ *
+ * @param a  the first parameter
+ * @param b  the second parameter
+ *
+ * @return the value of `a / b` if b is non-zero, zero otherwise.
+ */
+template <typename T>
+GKO_INLINE GKO_ATTRIBUTES T safe_divide(T a, T b)
+{
+    return b == zero<T>() ? zero<T>() : a / b;
+}
+
+
 }  // namespace gko
 
 
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index e2167146335..097ceab9695 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -27,21 +27,21 @@ target_sources(ginkgo_omp
     preconditioner/isai_kernels.cpp
     preconditioner/jacobi_kernels.cpp
     reorder/rcm_kernels.cpp
-    solver/bicg_kernels.cpp
-    solver/bicgstab_kernels.cpp
-    solver/cgs_kernels.cpp
-    solver/fcg_kernels.cpp
     solver/gmres_kernels.cpp
     solver/cb_gmres_kernels.cpp
     solver/idr_kernels.cpp
-    solver/ir_kernels.cpp
     solver/lower_trs_kernels.cpp
     solver/upper_trs_kernels.cpp
     stop/criterion_kernels.cpp
     stop/residual_norm_kernels.cpp
     ../common/components/precision_conversion.cpp
     ../common/matrix/dense_kernels.cpp
+    ../common/solver/bicg_kernels.cpp
+    ../common/solver/bicgstab_kernels.cpp
     ../common/solver/cg_kernels.cpp
+    ../common/solver/cgs_kernels.cpp
+    ../common/solver/fcg_kernels.cpp
+    ../common/solver/ir_kernels.cpp
     )
 
 ginkgo_compile_features(ginkgo_omp)

From 2523f03a0839fc1ec95fb43804ab8c5e6813d117 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 22 May 2021 14:15:45 +0200
Subject: [PATCH 0822/1008] eliminate unnecessary stride parameters

---
 common/solver/bicg_kernels.cpp     |  11 ++-
 common/solver/bicgstab_kernels.cpp |  17 ++--
 common/solver/cg_kernels.cpp       |   9 +-
 common/solver/cgs_kernels.cpp      |  15 ++--
 cuda/base/kernel_launch.cuh        | 128 +++++++++++++++++++++++------
 dpcpp/base/kernel_launch.dp.hpp    | 117 ++++++++++++++++++++------
 hip/base/kernel_launch.hip.hpp     | 128 +++++++++++++++++++++++------
 omp/base/kernel_launch.hpp         |  97 +++++++++++++++++++++-
 8 files changed, 427 insertions(+), 95 deletions(-)

diff --git a/common/solver/bicg_kernels.cpp b/common/solver/bicg_kernels.cpp
index 9a7abb73a6e..96aa807a12d 100644
--- a/common/solver/bicg_kernels.cpp
+++ b/common/solver/bicg_kernels.cpp
@@ -79,8 +79,9 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             p2(row, col) = zero(p2(row, col));
             q2(row, col) = zero(q2(row, col));
         },
-        p->get_size(), b, r, z, p, q, prev_rho, rho, r2, z2, p2, q2,
-        *stop_status);
+        p->get_size(), b, compact(r), compact(z), compact(p), compact(q),
+        vector(prev_rho), vector(rho), compact(r2), compact(z2), compact(p2),
+        compact(q2), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_INITIALIZE_KERNEL);
@@ -104,7 +105,8 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
                 p2(row, col) = z2(row, col) + tmp * p2(row, col);
             }
         },
-        p->get_size(), p, z, p2, z2, rho, prev_rho, *stop_status);
+        p->get_size(), compact(p), compact(z), compact(p2), compact(z2),
+        vector(rho), vector(prev_rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_1_KERNEL);
@@ -131,7 +133,8 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
                 r2(row, col) -= tmp * q2(row, col);
             }
         },
-        p->get_size(), x, r, r2, p, q, q2, beta, rho, *stop_status);
+        p->get_size(), x, compact(r), compact(r2), compact(p), compact(q),
+        compact(q2), vector(beta), vector(rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_2_KERNEL);
diff --git a/common/solver/bicgstab_kernels.cpp b/common/solver/bicgstab_kernels.cpp
index c669431d1fc..03a8e107245 100644
--- a/common/solver/bicgstab_kernels.cpp
+++ b/common/solver/bicgstab_kernels.cpp
@@ -77,8 +77,10 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             rr(row, col) = z(row, col) = v(row, col) = s(row, col) =
                 t(row, col) = y(row, col) = p(row, col) = zero(rr(row, col));
         },
-        p->get_size(), b, r, rr, y, s, t, z, v, p, prev_rho, rho, alpha, beta,
-        gamma, omega, *stop_status);
+        p->get_size(), b, compact(r), compact(rr), compact(y), compact(s),
+        compact(t), compact(z), compact(v), compact(p), vector(prev_rho),
+        vector(rho), vector(alpha), vector(beta), vector(gamma), vector(omega),
+        *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_INITIALIZE_KERNEL);
@@ -105,7 +107,8 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
                               tmp * (p(row, col) - omega[col] * v(row, col));
             }
         },
-        p->get_size(), r, p, v, rho, prev_rho, alpha, omega, *stop_status);
+        p->get_size(), compact(r), compact(p), compact(v), vector(rho),
+        vector(prev_rho), vector(alpha), vector(omega), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_1_KERNEL);
@@ -132,7 +135,8 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
                 s(row, col) = r(row, col) - tmp * v(row, col);
             }
         },
-        r->get_size(), r, s, v, rho, alpha, beta, *stop_status);
+        r->get_size(), compact(r), compact(s), compact(v), vector(rho),
+        vector(alpha), vector(beta), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_2_KERNEL);
@@ -161,7 +165,8 @@ void step_3(
                 r(row, col) = s(row, col) - tmp * t(row, col);
             }
         },
-        r->get_size(), x, r, s, t, y, z, alpha, beta, gamma, omega,
+        r->get_size(), x, compact(r), compact(s), compact(t), compact(y),
+        compact(z), vector(alpha), vector(beta), vector(gamma), vector(omega),
         *stop_status);
 }
 
@@ -183,7 +188,7 @@ void finalize(std::shared_ptr<const DefaultExecutor> exec,
                 stop[col].finalize();
             }
         },
-        x->get_size(), x, y, alpha, *stop_status);
+        x->get_size(), x, compact(y), vector(alpha), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_FINALIZE_KERNEL);
diff --git a/common/solver/cg_kernels.cpp b/common/solver/cg_kernels.cpp
index 7943a5e46ae..54c0a8a9328 100644
--- a/common/solver/cg_kernels.cpp
+++ b/common/solver/cg_kernels.cpp
@@ -70,7 +70,8 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             r(row, col) = b(row, col);
             z(row, col) = p(row, col) = q(row, col) = zero(z(row, col));
         },
-        p->get_size(), b, r, z, p, q, prev_rho, rho, *stop_status);
+        p->get_size(), b, compact(r), compact(z), compact(p), compact(q),
+        vector(prev_rho), vector(rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
@@ -92,7 +93,8 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
                 p(row, col) = z(row, col) + tmp * p(row, col);
             }
         },
-        p->get_size(), p, z, rho, prev_rho, *stop_status);
+        p->get_size(), compact(p), compact(z), vector(rho), vector(prev_rho),
+        *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
@@ -117,7 +119,8 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
                 r(row, col) -= tmp * q(row, col);
             }
         },
-        x->get_size(), x, r, p, q, beta, rho, *stop_status);
+        x->get_size(), x, compact(r), compact(p), compact(q), vector(beta),
+        vector(rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
diff --git a/common/solver/cgs_kernels.cpp b/common/solver/cgs_kernels.cpp
index 427228eb57a..83c5d8893b1 100644
--- a/common/solver/cgs_kernels.cpp
+++ b/common/solver/cgs_kernels.cpp
@@ -79,8 +79,10 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             u(row, col) = u_hat(row, col) = p(row, col) = q(row, col) =
                 v_hat(row, col) = t(row, col) = zero(u(row, col));
         },
-        p->get_size(), b, r, r_tld, p, q, u, u_hat, v_hat, t, alpha, beta,
-        gamma, prev_rho, rho, *stop_status);
+        p->get_size(), b, compact(r), compact(r_tld), compact(p), compact(q),
+        compact(u), compact(u_hat), compact(v_hat), compact(t), vector(alpha),
+        vector(beta), vector(gamma), vector(prev_rho), vector(rho),
+        *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_INITIALIZE_KERNEL);
@@ -108,7 +110,8 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
                     u(row, col) + tmp * (q(row, col) + tmp * p(row, col));
             }
         },
-        p->get_size(), r, u, p, q, beta, rho, prev_rho, *stop_status);
+        p->get_size(), compact(r), compact(u), compact(p), compact(q),
+        vector(beta), vector(rho), vector(prev_rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_1_KERNEL);
@@ -136,7 +139,8 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
                 t(row, col) = u(row, col) + q(row, col);
             }
         },
-        u->get_size(), u, v_hat, q, t, alpha, rho, gamma, *stop_status);
+        u->get_size(), compact(u), compact(v_hat), compact(q), compact(t),
+        vector(alpha), vector(rho), vector(gamma), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_2_KERNEL);
@@ -157,7 +161,8 @@ void step_3(std::shared_ptr<const DefaultExecutor> exec,
                 r(row, col) -= alpha[col] * t(row, col);
             }
         },
-        t->get_size(), t, u_hat, r, x, alpha, *stop_status);
+        t->get_size(), compact(t), compact(u_hat), compact(r), x, vector(alpha),
+        *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_3_KERNEL);
diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index 807442cce70..977e6f25234 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -72,32 +72,6 @@ struct matrix_accessor {
 };
 
 
-template <typename KernelFunction, typename... KernelArgs>
-__global__ __launch_bounds__(default_block_size) void generic_kernel_1d(
-    size_type size, KernelFunction fn, KernelArgs... args)
-{
-    auto tidx = thread::get_thread_id_flat();
-    if (tidx >= size) {
-        return;
-    }
-    fn(tidx, args...);
-}
-
-
-template <typename KernelFunction, typename... KernelArgs>
-__global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
-    size_type rows, size_type cols, KernelFunction fn, KernelArgs... args)
-{
-    auto tidx = thread::get_thread_id_flat();
-    auto col = tidx % cols;
-    auto row = tidx / cols;
-    if (row >= rows) {
-        return;
-    }
-    fn(row, col, args...);
-}
-
-
 template <typename T>
 struct device_map_impl {
     using type = std::decay_t<cuda_type<T>>;
@@ -148,6 +122,108 @@ typename device_map_impl<T>::type map_to_device(T &&param)
 }
 
 
+template <typename ValueType>
+struct compact_dense_wrapper {
+    ValueType *data;
+};
+
+
+template <typename ValueType>
+compact_dense_wrapper<cuda_type<ValueType>> compact(
+    matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {as_cuda_type(mtx->get_values())};
+}
+
+
+template <typename ValueType>
+compact_dense_wrapper<const cuda_type<ValueType>> compact(
+    const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {as_cuda_type(mtx->get_const_values())};
+}
+
+
+template <typename ValueType>
+cuda_type<ValueType> *vector(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return as_cuda_type(mtx->get_values());
+}
+
+
+template <typename ValueType>
+const cuda_type<ValueType> *vector(const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return as_cuda_type(mtx->get_const_values());
+}
+
+
+template <typename T>
+struct device_unpack_1d_impl {
+    using type = T;
+    static __device__ __forceinline__ type unpack(T param, size_type, size_type)
+    {
+        return param;
+    }
+};
+
+
+template <typename T>
+struct device_unpack_2d_impl {
+    using type = T;
+    static __device__ __forceinline__ type unpack(T param, size_type, size_type,
+                                                  size_type, size_type)
+    {
+        return param;
+    }
+};
+
+template <typename ValueType>
+struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
+    using type = matrix_accessor<ValueType>;
+    static __device__ __forceinline__ type
+    unpack(compact_dense_wrapper<ValueType> param, size_type, size_type,
+           size_type, size_type num_cols)
+    {
+        return {param.data, num_cols};
+    }
+};
+
+
+template <typename KernelFunction, typename... KernelArgs>
+__global__ __launch_bounds__(default_block_size) void generic_kernel_1d(
+    size_type size, KernelFunction fn, KernelArgs... args)
+{
+    auto tidx = thread::get_thread_id_flat();
+    if (tidx >= size) {
+        return;
+    }
+    fn(tidx, device_unpack_1d_impl<KernelArgs>::unpack(args, tidx, size)...);
+}
+
+
+template <typename KernelFunction, typename... KernelArgs>
+__global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
+    size_type rows, size_type cols, KernelFunction fn, KernelArgs... args)
+{
+    auto tidx = thread::get_thread_id_flat();
+    auto col = tidx % cols;
+    auto row = tidx / cols;
+    if (row >= rows) {
+        return;
+    }
+    fn(row, col,
+       device_unpack_2d_impl<KernelArgs>::unpack(args, row, col, rows,
+                                                 cols)...);
+}
+
+
 }  // namespace cuda
 }  // namespace kernels
 
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
index 3f724a6e2db..fb9d9e93908 100644
--- a/dpcpp/base/kernel_launch.dp.hpp
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -67,29 +67,6 @@ struct matrix_accessor {
 };
 
 
-template <typename KernelFunction, typename... KernelArgs>
-void generic_kernel_1d(sycl::handler &cgh, size_type size, KernelFunction fn,
-                       KernelArgs... args)
-{
-    cgh.parallel_for(sycl::range<1>{size}, [=](sycl::id<1> idx_id) {
-        fn(static_cast<size_type>(idx_id[0]), args...);
-    });
-}
-
-
-template <typename KernelFunction, typename... KernelArgs>
-void generic_kernel_2d(sycl::handler &cgh, size_type rows, size_type cols,
-                       KernelFunction fn, KernelArgs... args)
-{
-    cgh.parallel_for(sycl::range<1>{rows * cols}, [=](sycl::id<1> idx_id) {
-        auto idx = static_cast<size_type>(idx_id[0]);
-        auto row = idx / cols;
-        auto col = idx % cols;
-        fn(row, col, args...);
-    });
-}
-
-
 template <typename T>
 struct device_map_impl {
     using type = std::decay_t<T>;
@@ -140,6 +117,100 @@ typename device_map_impl<T>::type map_to_device(T &&param)
 }
 
 
+template <typename ValueType>
+struct compact_dense_wrapper {
+    ValueType *data;
+};
+
+
+template <typename ValueType>
+compact_dense_wrapper<ValueType> compact(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {mtx->get_values()};
+}
+
+
+template <typename ValueType>
+compact_dense_wrapper<const ValueType> compact(
+    const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {mtx->get_const_values()};
+}
+
+
+template <typename ValueType>
+ValueType *vector(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return mtx->get_values();
+}
+
+
+template <typename ValueType>
+const ValueType *vector(const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return mtx->get_const_values();
+}
+
+
+template <typename T>
+struct device_unpack_1d_impl {
+    using type = T;
+    static type unpack(T param, size_type, size_type) { return param; }
+};
+
+
+template <typename T>
+struct device_unpack_2d_impl {
+    using type = T;
+    static type unpack(T param, size_type, size_type, size_type, size_type)
+    {
+        return param;
+    }
+};
+
+template <typename ValueType>
+struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
+    using type = matrix_accessor<ValueType>;
+    static type unpack(compact_dense_wrapper<ValueType> param, size_type,
+                       size_type, size_type, size_type num_cols)
+    {
+        return {param.data, num_cols};
+    }
+};
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void generic_kernel_1d(sycl::handler &cgh, size_type size, KernelFunction fn,
+                       KernelArgs... args)
+{
+    cgh.parallel_for(sycl::range<1>{size}, [=](sycl::id<1> idx_id) {
+        auto idx = static_cast<size_type>(idx_id[0]);
+        fn(idx, device_unpack_1d_impl<KernelArgs>::unpack(args, idx, size)...);
+    });
+}
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void generic_kernel_2d(sycl::handler &cgh, size_type rows, size_type cols,
+                       KernelFunction fn, KernelArgs... args)
+{
+    cgh.parallel_for(sycl::range<1>{rows * cols}, [=](sycl::id<1> idx_id) {
+        auto idx = static_cast<size_type>(idx_id[0]);
+        auto row = idx / cols;
+        auto col = idx % cols;
+        fn(row, col,
+           device_unpack_2d_impl<KernelArgs>::unpack(args, row, col, rows,
+                                                     cols)...);
+    });
+}
+
+
 }  // namespace dpcpp
 }  // namespace kernels
 
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index 48cf044130e..1a2e94c8d26 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -75,32 +75,6 @@ struct matrix_accessor {
 };
 
 
-template <typename KernelFunction, typename... KernelArgs>
-__global__ __launch_bounds__(default_block_size) void generic_kernel_1d(
-    size_type size, KernelFunction fn, KernelArgs... args)
-{
-    auto tidx = thread::get_thread_id_flat();
-    if (tidx >= size) {
-        return;
-    }
-    fn(tidx, args...);
-}
-
-
-template <typename KernelFunction, typename... KernelArgs>
-__global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
-    size_type rows, size_type cols, KernelFunction fn, KernelArgs... args)
-{
-    auto tidx = thread::get_thread_id_flat();
-    auto col = tidx % cols;
-    auto row = tidx / cols;
-    if (row >= rows) {
-        return;
-    }
-    fn(row, col, args...);
-}
-
-
 template <typename T>
 struct device_map_impl {
     using type = std::decay_t<hip_type<T>>;
@@ -151,6 +125,108 @@ typename device_map_impl<T>::type map_to_device(T &&param)
 }
 
 
+template <typename ValueType>
+struct compact_dense_wrapper {
+    ValueType *data;
+};
+
+
+template <typename ValueType>
+compact_dense_wrapper<hip_type<ValueType>> compact(
+    matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {as_hip_type(mtx->get_values())};
+}
+
+
+template <typename ValueType>
+compact_dense_wrapper<const hip_type<ValueType>> compact(
+    const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {as_hip_type(mtx->get_const_values())};
+}
+
+
+template <typename ValueType>
+hip_type<ValueType> *vector(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return as_hip_type(mtx->get_values());
+}
+
+
+template <typename ValueType>
+const hip_type<ValueType> *vector(const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return as_hip_type(mtx->get_const_values());
+}
+
+
+template <typename T>
+struct device_unpack_1d_impl {
+    using type = T;
+    static __device__ __forceinline__ type unpack(T param, size_type, size_type)
+    {
+        return param;
+    }
+};
+
+
+template <typename T>
+struct device_unpack_2d_impl {
+    using type = T;
+    static __device__ __forceinline__ type unpack(T param, size_type, size_type,
+                                                  size_type, size_type)
+    {
+        return param;
+    }
+};
+
+template <typename ValueType>
+struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
+    using type = matrix_accessor<ValueType>;
+    static __device__ __forceinline__ type
+    unpack(compact_dense_wrapper<ValueType> param, size_type, size_type,
+           size_type, size_type num_cols)
+    {
+        return {param.data, num_cols};
+    }
+};
+
+
+template <typename KernelFunction, typename... KernelArgs>
+__global__ __launch_bounds__(default_block_size) void generic_kernel_1d(
+    size_type size, KernelFunction fn, KernelArgs... args)
+{
+    auto tidx = thread::get_thread_id_flat();
+    if (tidx >= size) {
+        return;
+    }
+    fn(tidx, device_unpack_1d_impl<KernelArgs>::unpack(args, tidx, size)...);
+}
+
+
+template <typename KernelFunction, typename... KernelArgs>
+__global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
+    size_type rows, size_type cols, KernelFunction fn, KernelArgs... args)
+{
+    auto tidx = thread::get_thread_id_flat();
+    auto col = tidx % cols;
+    auto row = tidx / cols;
+    if (row >= rows) {
+        return;
+    }
+    fn(row, col,
+       device_unpack_2d_impl<KernelArgs>::unpack(args, row, col, rows,
+                                                 cols)...);
+}
+
+
 }  // namespace hip
 }  // namespace kernels
 
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index 22fe67742b7..dc5568fbae5 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -114,6 +114,93 @@ typename device_map_impl<T>::type map_to_device(T &&param)
 }
 
 
+template <typename ValueType>
+struct compact_dense_wrapper {
+    ValueType *data;
+};
+
+
+template <typename ValueType>
+compact_dense_wrapper<ValueType> compact(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {mtx->get_values()};
+}
+
+
+template <typename ValueType>
+compact_dense_wrapper<const ValueType> compact(
+    const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {mtx->get_const_values()};
+}
+
+
+template <typename ValueType>
+ValueType *vector(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return mtx->get_values();
+}
+
+
+template <typename ValueType>
+const ValueType *vector(const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return mtx->get_const_values();
+}
+
+
+template <typename T>
+struct device_unpack_1d_impl {
+    using type = T;
+    static type unpack(T param, size_type, size_type) { return param; }
+};
+
+
+template <typename T>
+struct device_unpack_2d_impl {
+    using type = T;
+    static type unpack(T param, size_type, size_type, size_type, size_type)
+    {
+        return param;
+    }
+};
+
+template <typename ValueType>
+struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
+    using type = matrix_accessor<ValueType>;
+    static type unpack(compact_dense_wrapper<ValueType> param, size_type,
+                       size_type, size_type, size_type num_cols)
+    {
+        return {param.data, num_cols};
+    }
+};
+
+
+template <typename T>
+typename device_unpack_1d_impl<typename device_map_impl<T>::type>::type
+map_unpack_to_device(T &&param, size_type i, size_type size)
+{
+    return device_unpack_1d_impl<typename device_map_impl<T>::type>::unpack(
+        device_map_impl<T>::map_to_device(param), i, size);
+}
+
+
+template <typename T>
+typename device_unpack_2d_impl<typename device_map_impl<T>::type>::type
+map_unpack_to_device(T &&param, size_type row, size_type col,
+                     size_type num_rows, size_type num_cols)
+{
+    return device_unpack_2d_impl<typename device_map_impl<T>::type>::unpack(
+        device_map_impl<T>::map_to_device(param), row, col, num_rows, num_cols);
+}
+
+
 }  // namespace omp
 }  // namespace kernels
 
@@ -124,7 +211,9 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
 {
 #pragma omp parallel for
     for (size_type i = 0; i < size; i++) {
-        [&]() { fn(i, kernels::omp::map_to_device(args)...); }();
+        [&]() {
+            fn(i, kernels::omp::map_unpack_to_device(args, i, size)...);
+        }();
     }
 }
 
@@ -137,7 +226,11 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
     for (size_type i = 0; i < size[0] * size[1]; i++) {
         auto row = i / size[1];
         auto col = i % size[1];
-        [&]() { fn(row, col, kernels::omp::map_to_device(args)...); }();
+        [&]() {
+            fn(row, col,
+               kernels::omp::map_unpack_to_device(args, row, col, size[0],
+                                                  size[1])...);
+        }();
     }
 }
 

From b2208cb6df7a60dab9e833a73ecc997f6c3b3636 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 22 May 2021 14:16:09 +0200
Subject: [PATCH 0823/1008] replace BLAS axpy and scal by simple kernels

---
 common/matrix/dense_kernels.cpp     | 53 ++++++++++++++++++++++++++--
 common/matrix/dense_kernels.hpp.inc | 40 ---------------------
 cuda/matrix/dense_kernels.cu        | 52 ---------------------------
 dpcpp/matrix/dense_kernels.dp.cpp   | 17 ---------
 hip/matrix/dense_kernels.hip.cpp    | 54 -----------------------------
 omp/matrix/dense_kernels.cpp        | 49 --------------------------
 6 files changed, 51 insertions(+), 214 deletions(-)

diff --git a/common/matrix/dense_kernels.cpp b/common/matrix/dense_kernels.cpp
index a8da0c3a8f1..dfc92a150f8 100644
--- a/common/matrix/dense_kernels.cpp
+++ b/common/matrix/dense_kernels.cpp
@@ -82,6 +82,55 @@ void fill(std::shared_ptr<const DefaultExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_FILL_KERNEL);
 
 
+template <typename ValueType>
+void scale(std::shared_ptr<const DefaultExecutor> exec,
+           const matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *x)
+{
+    if (alpha->get_size()[1] > 1) {
+        run_kernel(
+            exec,
+            [] GKO_KERNEL(auto row, auto col, auto alpha, auto x) {
+                x(row, col) *= alpha[col];
+            },
+            x->get_size(), vector(alpha), x);
+    } else {
+        run_kernel(
+            exec,
+            [] GKO_KERNEL(auto row, auto col, auto alpha, auto x) {
+                x(row, col) *= alpha[0];
+            },
+            x->get_size(), vector(alpha), x);
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SCALE_KERNEL);
+
+
+template <typename ValueType>
+void add_scaled(std::shared_ptr<const DefaultExecutor> exec,
+                const matrix::Dense<ValueType> *alpha,
+                const matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *y)
+{
+    if (alpha->get_size()[1] > 1) {
+        run_kernel(
+            exec,
+            [] GKO_KERNEL(auto row, auto col, auto alpha, auto x, auto y) {
+                y(row, col) += alpha[col] * x(row, col);
+            },
+            x->get_size(), vector(alpha), x, y);
+    } else {
+        run_kernel(
+            exec,
+            [] GKO_KERNEL(auto row, auto col, auto alpha, auto x, auto y) {
+                y(row, col) += alpha[0] * x(row, col);
+            },
+            x->get_size(), vector(alpha), x, y);
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_KERNEL);
+
+
 template <typename ValueType>
 void add_scaled_diag(std::shared_ptr<const DefaultExecutor> exec,
                      const matrix::Dense<ValueType> *alpha,
@@ -92,9 +141,9 @@ void add_scaled_diag(std::shared_ptr<const DefaultExecutor> exec,
     run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto alpha, auto diag, auto y) {
-            y(i, i) += alpha(0, 0) * diag[i];
+            y(i, i) += alpha[0] * diag[i];
         },
-        x->get_size()[0], alpha, x->get_const_values(), y);
+        x->get_size()[0], vector(alpha), x->get_const_values(), y);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL);
diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index 4f30fd4c109..d46b202a8ff 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -33,46 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace kernel {
 
 
-template <size_type block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void scale(
-    size_type num_rows, size_type num_cols, size_type num_alpha_cols,
-    const ValueType *__restrict__ alpha, ValueType *__restrict__ x,
-    size_type stride_x)
-{
-    constexpr auto warps_per_block = block_size / config::warp_size;
-    const auto global_id =
-        thread::get_thread_id<config::warp_size, warps_per_block>();
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    const auto alpha_id = num_alpha_cols == 1 ? 0 : col_id;
-    if (row_id < num_rows) {
-        x[row_id * stride_x + col_id] =
-            alpha[alpha_id] == zero<ValueType>()
-                ? zero<ValueType>()
-                : x[row_id * stride_x + col_id] * alpha[alpha_id];
-    }
-}
-
-
-template <size_type block_size, typename ValueType>
-__global__ __launch_bounds__(block_size) void add_scaled(
-    size_type num_rows, size_type num_cols, size_type num_alpha_cols,
-    const ValueType *__restrict__ alpha, const ValueType *__restrict__ x,
-    size_type stride_x, ValueType *__restrict__ y, size_type stride_y)
-{
-    constexpr auto warps_per_block = block_size / config::warp_size;
-    const auto global_id =
-        thread::get_thread_id<config::warp_size, warps_per_block>();
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    const auto alpha_id = num_alpha_cols == 1 ? 0 : col_id;
-    if (row_id < num_rows && alpha[alpha_id] != zero<ValueType>()) {
-        y[row_id * stride_y + col_id] +=
-            x[row_id * stride_x + col_id] * alpha[alpha_id];
-    }
-}
-
-
 template <size_type block_size, typename OutType, typename CallableGetValue,
           typename CallableReduce>
 __device__ void compute_partial_reduce(size_type num_rows,
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index 4f06232866f..1f6e5a2ab16 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -117,58 +117,6 @@ void apply(std::shared_ptr<const CudaExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
-template <typename ValueType>
-void scale(std::shared_ptr<const CudaExecutor> exec,
-           const matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *x)
-{
-    if (cublas::is_supported<ValueType>::value && x->get_size()[1] == 1) {
-        cublas::scal(exec->get_cublas_handle(), x->get_size()[0],
-                     alpha->get_const_values(), x->get_values(),
-                     x->get_stride());
-    } else {
-        // TODO: tune this parameter
-        constexpr auto block_size = default_block_size;
-        const dim3 grid_dim =
-            ceildiv(x->get_size()[0] * x->get_size()[1], block_size);
-        const dim3 block_dim{config::warp_size, 1,
-                             block_size / config::warp_size};
-        kernel::scale<block_size><<<grid_dim, block_dim>>>(
-            x->get_size()[0], x->get_size()[1], alpha->get_size()[1],
-            as_cuda_type(alpha->get_const_values()),
-            as_cuda_type(x->get_values()), x->get_stride());
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SCALE_KERNEL);
-
-
-template <typename ValueType>
-void add_scaled(std::shared_ptr<const CudaExecutor> exec,
-                const matrix::Dense<ValueType> *alpha,
-                const matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *y)
-{
-    if (cublas::is_supported<ValueType>::value && x->get_size()[1] == 1) {
-        cublas::axpy(exec->get_cublas_handle(), x->get_size()[0],
-                     alpha->get_const_values(), x->get_const_values(),
-                     x->get_stride(), y->get_values(), y->get_stride());
-    } else {
-        // TODO: tune this parameter
-        constexpr auto block_size = default_block_size;
-        const dim3 grid_dim =
-            ceildiv(x->get_size()[0] * x->get_size()[1], block_size);
-        const dim3 block_dim{config::warp_size, 1,
-                             block_size / config::warp_size};
-        kernel::add_scaled<block_size><<<grid_dim, block_dim>>>(
-            x->get_size()[0], x->get_size()[1], alpha->get_size()[1],
-            as_cuda_type(alpha->get_const_values()),
-            as_cuda_type(x->get_const_values()), x->get_stride(),
-            as_cuda_type(y->get_values()), y->get_stride());
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_KERNEL);
-
-
 template <typename ValueType>
 void compute_dot(std::shared_ptr<const CudaExecutor> exec,
                  const matrix::Dense<ValueType> *x,
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 2ea3c8db043..531cfd52259 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -84,23 +84,6 @@ void apply(std::shared_ptr<const DpcppExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
-template <typename ValueType>
-void scale(std::shared_ptr<const DpcppExecutor> exec,
-           const matrix::Dense<ValueType> *alpha,
-           matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SCALE_KERNEL);
-
-
-template <typename ValueType>
-void add_scaled(std::shared_ptr<const DpcppExecutor> exec,
-                const matrix::Dense<ValueType> *alpha,
-                const matrix::Dense<ValueType> *x,
-                matrix::Dense<ValueType> *y) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_KERNEL);
-
-
 template <typename ValueType>
 void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
                  const matrix::Dense<ValueType> *x,
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index fbf35ca520d..4aebe4b9376 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -120,60 +120,6 @@ void apply(std::shared_ptr<const HipExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
-template <typename ValueType>
-void scale(std::shared_ptr<const HipExecutor> exec,
-           const matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *x)
-{
-    if (hipblas::is_supported<ValueType>::value && x->get_size()[1] == 1) {
-        hipblas::scal(exec->get_hipblas_handle(), x->get_size()[0],
-                      alpha->get_const_values(), x->get_values(),
-                      x->get_stride());
-    } else {
-        // TODO: tune this parameter
-        constexpr auto block_size = default_block_size;
-        const dim3 grid_dim =
-            ceildiv(x->get_size()[0] * x->get_size()[1], block_size);
-        const dim3 block_dim{config::warp_size, 1,
-                             block_size / config::warp_size};
-        hipLaunchKernelGGL(
-            HIP_KERNEL_NAME(kernel::scale<block_size>), dim3(grid_dim),
-            dim3(block_dim), 0, 0, x->get_size()[0], x->get_size()[1],
-            alpha->get_size()[1], as_hip_type(alpha->get_const_values()),
-            as_hip_type(x->get_values()), x->get_stride());
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SCALE_KERNEL);
-
-
-template <typename ValueType>
-void add_scaled(std::shared_ptr<const HipExecutor> exec,
-                const matrix::Dense<ValueType> *alpha,
-                const matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *y)
-{
-    if (hipblas::is_supported<ValueType>::value && x->get_size()[1] == 1) {
-        hipblas::axpy(exec->get_hipblas_handle(), x->get_size()[0],
-                      alpha->get_const_values(), x->get_const_values(),
-                      x->get_stride(), y->get_values(), y->get_stride());
-    } else {
-        // TODO: tune this parameter
-        constexpr auto block_size = default_block_size;
-        const dim3 grid_dim =
-            ceildiv(x->get_size()[0] * x->get_size()[1], block_size);
-        const dim3 block_dim{config::warp_size, 1,
-                             block_size / config::warp_size};
-        hipLaunchKernelGGL(
-            HIP_KERNEL_NAME(kernel::add_scaled<block_size>), dim3(grid_dim),
-            dim3(block_dim), 0, 0, x->get_size()[0], x->get_size()[1],
-            alpha->get_size()[1], as_hip_type(alpha->get_const_values()),
-            as_hip_type(x->get_const_values()), x->get_stride(),
-            as_hip_type(y->get_values()), y->get_stride());
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_KERNEL);
-
-
 template <typename ValueType>
 void compute_dot(std::shared_ptr<const HipExecutor> exec,
                  const matrix::Dense<ValueType> *x,
diff --git a/omp/matrix/dense_kernels.cpp b/omp/matrix/dense_kernels.cpp
index 7590d91e38e..fcbb5c58abd 100644
--- a/omp/matrix/dense_kernels.cpp
+++ b/omp/matrix/dense_kernels.cpp
@@ -127,55 +127,6 @@ void apply(std::shared_ptr<const OmpExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
-template <typename ValueType>
-void scale(std::shared_ptr<const OmpExecutor> exec,
-           const matrix::Dense<ValueType> *alpha, matrix::Dense<ValueType> *x)
-{
-    if (alpha->get_size()[1] == 1) {
-#pragma omp parallel for
-        for (size_type i = 0; i < x->get_size()[0]; ++i) {
-            for (size_type j = 0; j < x->get_size()[1]; ++j) {
-                x->at(i, j) *= alpha->at(0, 0);
-            }
-        }
-    } else {
-#pragma omp parallel for
-        for (size_type i = 0; i < x->get_size()[0]; ++i) {
-            for (size_type j = 0; j < x->get_size()[1]; ++j) {
-                x->at(i, j) *= alpha->at(0, j);
-            }
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SCALE_KERNEL);
-
-
-template <typename ValueType>
-void add_scaled(std::shared_ptr<const OmpExecutor> exec,
-                const matrix::Dense<ValueType> *alpha,
-                const matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *y)
-{
-    if (alpha->get_size()[1] == 1) {
-#pragma omp parallel for
-        for (size_type i = 0; i < x->get_size()[0]; ++i) {
-            for (size_type j = 0; j < x->get_size()[1]; ++j) {
-                y->at(i, j) += alpha->at(0, 0) * x->at(i, j);
-            }
-        }
-    } else {
-#pragma omp parallel for
-        for (size_type i = 0; i < x->get_size()[0]; ++i) {
-            for (size_type j = 0; j < x->get_size()[1]; ++j) {
-                y->at(i, j) += alpha->at(0, j) * x->at(i, j);
-            }
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_KERNEL);
-
-
 template <typename ValueType>
 void compute_dot(std::shared_ptr<const OmpExecutor> exec,
                  const matrix::Dense<ValueType> *x,

From 1087b33aaf0a55db08804399c6a6567b4ae59358 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 1 Jun 2021 11:38:56 +0200
Subject: [PATCH 0824/1008] don't preserve stride on create_with_same_config

---
 core/test/matrix/dense.cpp           | 12 ++++++++++++
 include/ginkgo/core/matrix/dense.hpp |  8 ++++----
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/core/test/matrix/dense.cpp b/core/test/matrix/dense.cpp
index f352648fadd..3a809bae7b1 100644
--- a/core/test/matrix/dense.cpp
+++ b/core/test/matrix/dense.cpp
@@ -138,6 +138,18 @@ TYPED_TEST(Dense, CanBeConstructedFromExistingData)
 }
 
 
+TYPED_TEST(Dense, CreateWithSameConfigDoesntKeepStride)
+{
+    auto m =
+        gko::matrix::Dense<TypeParam>::create(this->exec, gko::dim<2>{2, 3}, 4);
+    auto m2 = gko::matrix::Dense<TypeParam>::create_with_config_of(m.get());
+
+    ASSERT_EQ(m2->get_size(), gko::dim<2>(2, 3));
+    EXPECT_EQ(m2->get_stride(), 3);
+    ASSERT_EQ(m2->get_num_stored_elements(), 6);
+}
+
+
 TYPED_TEST(Dense, KnowsItsSizeAndValues)
 {
     this->assert_equal_to_original_mtx(this->mtx.get());
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index eb4243ade18..41fcb32cd4f 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -827,14 +827,14 @@ class Dense
     }
 
     /**
-     * Creates a Dense matrix with the same configuration as the callers matrix.
+     * Creates a Dense matrix with the same size as the callers matrix.
+     * Note that the stride is not preserved!
      *
-     * @returns a Dense matrix with the same configuration as the caller.
+     * @returns a Dense matrix with the same size as the caller.
      */
     virtual std::unique_ptr<Dense> create_with_same_config() const
     {
-        return Dense::create(this->get_executor(), this->get_size(),
-                             this->get_stride());
+        return Dense::create(this->get_executor(), this->get_size());
     }
 
     /**

From 2c18d134bdaea52f3eae81f305b4888a872c4ad9 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 1 Jun 2021 12:31:49 +0200
Subject: [PATCH 0825/1008] review updates

* make behavior equivalent to reference
* rename device_map_impl to to_device_type_impl
* add comments

Co-authored-by: Yuhsiang Tsai <yhmtsai@gmail.com>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 common/solver/bicg_kernels.cpp     |  4 ++--
 common/solver/bicgstab_kernels.cpp |  6 +++---
 common/solver/cg_kernels.cpp       |  2 +-
 common/solver/cgs_kernels.cpp      | 14 ++++++++------
 common/solver/fcg_kernels.cpp      | 13 ++++++++-----
 cuda/base/kernel_launch.cuh        | 15 +++++++--------
 dpcpp/base/kernel_launch.dp.hpp    | 15 +++++++--------
 hip/base/kernel_launch.hip.hpp     | 15 +++++++--------
 include/ginkgo/core/base/math.hpp  |  4 ++--
 omp/base/kernel_launch.hpp         | 28 ++++++++++++++--------------
 10 files changed, 59 insertions(+), 57 deletions(-)

diff --git a/common/solver/bicg_kernels.cpp b/common/solver/bicg_kernels.cpp
index 96aa807a12d..992b7cdb26b 100644
--- a/common/solver/bicg_kernels.cpp
+++ b/common/solver/bicg_kernels.cpp
@@ -79,7 +79,7 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             p2(row, col) = zero(p2(row, col));
             q2(row, col) = zero(q2(row, col));
         },
-        p->get_size(), b, compact(r), compact(z), compact(p), compact(q),
+        b->get_size(), b, compact(r), compact(z), compact(p), compact(q),
         vector(prev_rho), vector(rho), compact(r2), compact(z2), compact(p2),
         compact(q2), *stop_status);
 }
@@ -133,7 +133,7 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
                 r2(row, col) -= tmp * q2(row, col);
             }
         },
-        p->get_size(), x, compact(r), compact(r2), compact(p), compact(q),
+        x->get_size(), x, compact(r), compact(r2), compact(p), compact(q),
         compact(q2), vector(beta), vector(rho), *stop_status);
 }
 
diff --git a/common/solver/bicgstab_kernels.cpp b/common/solver/bicgstab_kernels.cpp
index 03a8e107245..803089c3a0c 100644
--- a/common/solver/bicgstab_kernels.cpp
+++ b/common/solver/bicgstab_kernels.cpp
@@ -77,7 +77,7 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             rr(row, col) = z(row, col) = v(row, col) = s(row, col) =
                 t(row, col) = y(row, col) = p(row, col) = zero(rr(row, col));
         },
-        p->get_size(), b, compact(r), compact(rr), compact(y), compact(s),
+        b->get_size(), b, compact(r), compact(rr), compact(y), compact(s),
         compact(t), compact(z), compact(v), compact(p), vector(prev_rho),
         vector(rho), vector(alpha), vector(beta), vector(gamma), vector(omega),
         *stop_status);
@@ -107,7 +107,7 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
                               tmp * (p(row, col) - omega[col] * v(row, col));
             }
         },
-        p->get_size(), compact(r), compact(p), compact(v), vector(rho),
+        r->get_size(), compact(r), compact(p), compact(v), vector(rho),
         vector(prev_rho), vector(alpha), vector(omega), *stop_status);
 }
 
@@ -165,7 +165,7 @@ void step_3(
                 r(row, col) = s(row, col) - tmp * t(row, col);
             }
         },
-        r->get_size(), x, compact(r), compact(s), compact(t), compact(y),
+        x->get_size(), x, compact(r), compact(s), compact(t), compact(y),
         compact(z), vector(alpha), vector(beta), vector(gamma), vector(omega),
         *stop_status);
 }
diff --git a/common/solver/cg_kernels.cpp b/common/solver/cg_kernels.cpp
index 54c0a8a9328..82ed384df31 100644
--- a/common/solver/cg_kernels.cpp
+++ b/common/solver/cg_kernels.cpp
@@ -70,7 +70,7 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             r(row, col) = b(row, col);
             z(row, col) = p(row, col) = q(row, col) = zero(z(row, col));
         },
-        p->get_size(), b, compact(r), compact(z), compact(p), compact(q),
+        b->get_size(), b, compact(r), compact(z), compact(p), compact(q),
         vector(prev_rho), vector(rho), *stop_status);
 }
 
diff --git a/common/solver/cgs_kernels.cpp b/common/solver/cgs_kernels.cpp
index 83c5d8893b1..df96ab4976c 100644
--- a/common/solver/cgs_kernels.cpp
+++ b/common/solver/cgs_kernels.cpp
@@ -79,7 +79,7 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             u(row, col) = u_hat(row, col) = p(row, col) = q(row, col) =
                 v_hat(row, col) = t(row, col) = zero(u(row, col));
         },
-        p->get_size(), b, compact(r), compact(r_tld), compact(p), compact(q),
+        b->get_size(), b, compact(r), compact(r_tld), compact(p), compact(q),
         compact(u), compact(u_hat), compact(v_hat), compact(t), vector(alpha),
         vector(beta), vector(gamma), vector(prev_rho), vector(rho),
         *stop_status);
@@ -101,8 +101,9 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
         [] GKO_KERNEL(auto row, auto col, auto r, auto u, auto p, auto q,
                       auto beta, auto rho, auto prev_rho, auto stop) {
             if (!stop[col].has_stopped()) {
-                auto tmp = safe_divide(rho[col], prev_rho[col]);
-                if (row == 0) {
+                auto prev_rho_zero = prev_rho[col] == zero(prev_rho[col]);
+                auto tmp = prev_rho_zero ? beta[col] : rho[col] / prev_rho[col];
+                if (row == 0 && !prev_rho_zero) {
                     beta[col] = tmp;
                 }
                 u(row, col) = r(row, col) + tmp * q(row, col);
@@ -110,7 +111,7 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
                     u(row, col) + tmp * (q(row, col) + tmp * p(row, col));
             }
         },
-        p->get_size(), compact(r), compact(u), compact(p), compact(q),
+        r->get_size(), compact(r), compact(u), compact(p), compact(q),
         vector(beta), vector(rho), vector(prev_rho), *stop_status);
 }
 
@@ -131,8 +132,9 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
         [] GKO_KERNEL(auto row, auto col, auto u, auto v_hat, auto q, auto t,
                       auto alpha, auto rho, auto gamma, auto stop) {
             if (!stop[col].has_stopped()) {
-                auto tmp = safe_divide(rho[col], gamma[col]);
-                if (row == 0) {
+                auto gamma_is_zero = gamma[col] == zero(gamma[col]);
+                auto tmp = gamma_is_zero ? alpha[col] : rho[col] / gamma[col];
+                if (row == 0 && !gamma_is_zero) {
                     alpha[col] = tmp;
                 }
                 q(row, col) = u(row, col) - tmp * v_hat(row, col);
diff --git a/common/solver/fcg_kernels.cpp b/common/solver/fcg_kernels.cpp
index da6acec75f9..1a4c92ac3bf 100644
--- a/common/solver/fcg_kernels.cpp
+++ b/common/solver/fcg_kernels.cpp
@@ -72,7 +72,8 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             t(row, col) = r(row, col) = b(row, col);
             z(row, col) = p(row, col) = q(row, col) = zero(z(row, col));
         },
-        p->get_size(), b, r, z, p, q, t, prev_rho, rho, rho_t, *stop_status);
+        b->get_size(), b, compact(r), compact(z), compact(p), compact(q),
+        compact(t), vector(prev_rho), vector(rho), vector(rho_t), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_INITIALIZE_KERNEL);
@@ -94,7 +95,8 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
                 p(row, col) = z(row, col) + tmp * p(row, col);
             }
         },
-        p->get_size(), p, z, rho_t, prev_rho, *stop_status);
+        p->get_size(), compact(p), compact(z), vector(rho_t), vector(prev_rho),
+        *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_1_KERNEL);
@@ -113,15 +115,16 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
         exec,
         [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto t, auto p,
                       auto q, auto beta, auto rho, auto stop) {
-            if (!stop[col].has_stopped()) {
-                auto tmp = safe_divide(rho[col], beta[col]);
+            if (!stop[col].has_stopped() && beta[col] != zero(beta[col])) {
+                auto tmp = rho[col] / beta[col];
                 auto prev_r = r(row, col);
                 x(row, col) += tmp * p(row, col);
                 r(row, col) -= tmp * q(row, col);
                 t(row, col) = r(row, col) - prev_r;
             }
         },
-        x->get_size(), x, r, t, p, q, beta, rho, *stop_status);
+        x->get_size(), x, compact(r), compact(t), compact(p), compact(q),
+        vector(beta), vector(rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_2_KERNEL);
diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index 977e6f25234..2caec536e53 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -73,13 +73,13 @@ struct matrix_accessor {
 
 
 template <typename T>
-struct device_map_impl {
+struct to_device_type_impl {
     using type = std::decay_t<cuda_type<T>>;
     static type map_to_device(T in) { return as_cuda_type(in); }
 };
 
 template <typename ValueType>
-struct device_map_impl<matrix::Dense<ValueType> *&> {
+struct to_device_type_impl<matrix::Dense<ValueType> *&> {
     using type = matrix_accessor<cuda_type<ValueType>>;
     static type map_to_device(matrix::Dense<ValueType> *mtx)
     {
@@ -88,7 +88,7 @@ struct device_map_impl<matrix::Dense<ValueType> *&> {
 };
 
 template <typename ValueType>
-struct device_map_impl<const matrix::Dense<ValueType> *&> {
+struct to_device_type_impl<const matrix::Dense<ValueType> *&> {
     using type = matrix_accessor<const cuda_type<ValueType>>;
     static type map_to_device(const matrix::Dense<ValueType> *mtx)
     {
@@ -97,7 +97,7 @@ struct device_map_impl<const matrix::Dense<ValueType> *&> {
 };
 
 template <typename ValueType>
-struct device_map_impl<Array<ValueType> &> {
+struct to_device_type_impl<Array<ValueType> &> {
     using type = cuda_type<ValueType> *;
     static type map_to_device(Array<ValueType> &array)
     {
@@ -106,7 +106,7 @@ struct device_map_impl<Array<ValueType> &> {
 };
 
 template <typename ValueType>
-struct device_map_impl<const Array<ValueType> &> {
+struct to_device_type_impl<const Array<ValueType> &> {
     using type = const cuda_type<ValueType> *;
     static type map_to_device(const Array<ValueType> &array)
     {
@@ -116,9 +116,9 @@ struct device_map_impl<const Array<ValueType> &> {
 
 
 template <typename T>
-typename device_map_impl<T>::type map_to_device(T &&param)
+typename to_device_type_impl<T>::type map_to_device(T &&param)
 {
-    return device_map_impl<T>::map_to_device(param);
+    return to_device_type_impl<T>::map_to_device(param);
 }
 
 
@@ -239,7 +239,6 @@ void run_kernel(std::shared_ptr<const CudaExecutor> exec, KernelFunction fn,
         size, fn, kernels::cuda::map_to_device(args)...);
 }
 
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const CudaExecutor> exec, KernelFunction fn,
                 dim<2> size, KernelArgs &&... args)
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
index fb9d9e93908..6387431c931 100644
--- a/dpcpp/base/kernel_launch.dp.hpp
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -68,13 +68,13 @@ struct matrix_accessor {
 
 
 template <typename T>
-struct device_map_impl {
+struct to_device_type_impl {
     using type = std::decay_t<T>;
     static type map_to_device(T in) { return in; }
 };
 
 template <typename ValueType>
-struct device_map_impl<matrix::Dense<ValueType> *&> {
+struct to_device_type_impl<matrix::Dense<ValueType> *&> {
     using type = matrix_accessor<ValueType>;
     static type map_to_device(matrix::Dense<ValueType> *mtx)
     {
@@ -83,7 +83,7 @@ struct device_map_impl<matrix::Dense<ValueType> *&> {
 };
 
 template <typename ValueType>
-struct device_map_impl<const matrix::Dense<ValueType> *&> {
+struct to_device_type_impl<const matrix::Dense<ValueType> *&> {
     using type = matrix_accessor<const ValueType>;
     static type map_to_device(const matrix::Dense<ValueType> *mtx)
     {
@@ -92,7 +92,7 @@ struct device_map_impl<const matrix::Dense<ValueType> *&> {
 };
 
 template <typename ValueType>
-struct device_map_impl<Array<ValueType> &> {
+struct to_device_type_impl<Array<ValueType> &> {
     using type = ValueType *;
     static type map_to_device(Array<ValueType> &array)
     {
@@ -101,7 +101,7 @@ struct device_map_impl<Array<ValueType> &> {
 };
 
 template <typename ValueType>
-struct device_map_impl<const Array<ValueType> &> {
+struct to_device_type_impl<const Array<ValueType> &> {
     using type = const ValueType *;
     static type map_to_device(const Array<ValueType> &array)
     {
@@ -111,9 +111,9 @@ struct device_map_impl<const Array<ValueType> &> {
 
 
 template <typename T>
-typename device_map_impl<T>::type map_to_device(T &&param)
+typename to_device_type_impl<T>::type map_to_device(T &&param)
 {
-    return device_map_impl<T>::map_to_device(param);
+    return to_device_type_impl<T>::map_to_device(param);
 }
 
 
@@ -225,7 +225,6 @@ void run_kernel(std::shared_ptr<const DpcppExecutor> exec, KernelFunction fn,
     });
 }
 
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const DpcppExecutor> exec, KernelFunction fn,
                 dim<2> size, KernelArgs &&... args)
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index 1a2e94c8d26..350fb301dbd 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -76,13 +76,13 @@ struct matrix_accessor {
 
 
 template <typename T>
-struct device_map_impl {
+struct to_device_type_impl {
     using type = std::decay_t<hip_type<T>>;
     static type map_to_device(T in) { return as_hip_type(in); }
 };
 
 template <typename ValueType>
-struct device_map_impl<matrix::Dense<ValueType> *&> {
+struct to_device_type_impl<matrix::Dense<ValueType> *&> {
     using type = matrix_accessor<hip_type<ValueType>>;
     static type map_to_device(matrix::Dense<ValueType> *mtx)
     {
@@ -91,7 +91,7 @@ struct device_map_impl<matrix::Dense<ValueType> *&> {
 };
 
 template <typename ValueType>
-struct device_map_impl<const matrix::Dense<ValueType> *&> {
+struct to_device_type_impl<const matrix::Dense<ValueType> *&> {
     using type = matrix_accessor<const hip_type<ValueType>>;
     static type map_to_device(const matrix::Dense<ValueType> *mtx)
     {
@@ -100,7 +100,7 @@ struct device_map_impl<const matrix::Dense<ValueType> *&> {
 };
 
 template <typename ValueType>
-struct device_map_impl<Array<ValueType> &> {
+struct to_device_type_impl<Array<ValueType> &> {
     using type = hip_type<ValueType> *;
     static type map_to_device(Array<ValueType> &array)
     {
@@ -109,7 +109,7 @@ struct device_map_impl<Array<ValueType> &> {
 };
 
 template <typename ValueType>
-struct device_map_impl<const Array<ValueType> &> {
+struct to_device_type_impl<const Array<ValueType> &> {
     using type = const hip_type<ValueType> *;
     static type map_to_device(const Array<ValueType> &array)
     {
@@ -119,9 +119,9 @@ struct device_map_impl<const Array<ValueType> &> {
 
 
 template <typename T>
-typename device_map_impl<T>::type map_to_device(T &&param)
+typename to_device_type_impl<T>::type map_to_device(T &&param)
 {
-    return device_map_impl<T>::map_to_device(param);
+    return to_device_type_impl<T>::map_to_device(param);
 }
 
 
@@ -242,7 +242,6 @@ void run_kernel(std::shared_ptr<const HipExecutor> exec, KernelFunction fn,
                        0, 0, size, fn, kernels::hip::map_to_device(args)...);
 }
 
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const HipExecutor> exec, KernelFunction fn,
                 dim<2> size, KernelArgs &&... args)
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 21720933057..2cd1784b28e 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -1014,8 +1014,8 @@ is_finite(const T &value)
  *
  * @tparam T  value type of the parameters
  *
- * @param a  the first parameter
- * @param b  the second parameter
+ * @param a  the dividend
+ * @param b  the divisor
  *
  * @return the value of `a / b` if b is non-zero, zero otherwise.
  */
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index dc5568fbae5..546d53adfd4 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -65,13 +65,13 @@ struct matrix_accessor {
 
 
 template <typename T>
-struct device_map_impl {
+struct to_device_type_impl {
     using type = std::decay_t<T>;
     static type map_to_device(T in) { return in; }
 };
 
 template <typename ValueType>
-struct device_map_impl<matrix::Dense<ValueType> *&> {
+struct to_device_type_impl<matrix::Dense<ValueType> *&> {
     using type = matrix_accessor<ValueType>;
     static type map_to_device(matrix::Dense<ValueType> *mtx)
     {
@@ -80,7 +80,7 @@ struct device_map_impl<matrix::Dense<ValueType> *&> {
 };
 
 template <typename ValueType>
-struct device_map_impl<const matrix::Dense<ValueType> *&> {
+struct to_device_type_impl<const matrix::Dense<ValueType> *&> {
     using type = matrix_accessor<const ValueType>;
     static type map_to_device(const matrix::Dense<ValueType> *mtx)
     {
@@ -89,7 +89,7 @@ struct device_map_impl<const matrix::Dense<ValueType> *&> {
 };
 
 template <typename ValueType>
-struct device_map_impl<Array<ValueType> &> {
+struct to_device_type_impl<Array<ValueType> &> {
     using type = ValueType *;
     static type map_to_device(Array<ValueType> &array)
     {
@@ -98,7 +98,7 @@ struct device_map_impl<Array<ValueType> &> {
 };
 
 template <typename ValueType>
-struct device_map_impl<const Array<ValueType> &> {
+struct to_device_type_impl<const Array<ValueType> &> {
     using type = const ValueType *;
     static type map_to_device(const Array<ValueType> &array)
     {
@@ -108,9 +108,9 @@ struct device_map_impl<const Array<ValueType> &> {
 
 
 template <typename T>
-typename device_map_impl<T>::type map_to_device(T &&param)
+typename to_device_type_impl<T>::type map_to_device(T &&param)
 {
-    return device_map_impl<T>::map_to_device(param);
+    return to_device_type_impl<T>::map_to_device(param);
 }
 
 
@@ -183,21 +183,22 @@ struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
 
 
 template <typename T>
-typename device_unpack_1d_impl<typename device_map_impl<T>::type>::type
+typename device_unpack_1d_impl<typename to_device_type_impl<T>::type>::type
 map_unpack_to_device(T &&param, size_type i, size_type size)
 {
-    return device_unpack_1d_impl<typename device_map_impl<T>::type>::unpack(
-        device_map_impl<T>::map_to_device(param), i, size);
+    return device_unpack_1d_impl<typename to_device_type_impl<T>::type>::unpack(
+        to_device_type_impl<T>::map_to_device(param), i, size);
 }
 
 
 template <typename T>
-typename device_unpack_2d_impl<typename device_map_impl<T>::type>::type
+typename device_unpack_2d_impl<typename to_device_type_impl<T>::type>::type
 map_unpack_to_device(T &&param, size_type row, size_type col,
                      size_type num_rows, size_type num_cols)
 {
-    return device_unpack_2d_impl<typename device_map_impl<T>::type>::unpack(
-        device_map_impl<T>::map_to_device(param), row, col, num_rows, num_cols);
+    return device_unpack_2d_impl<typename to_device_type_impl<T>::type>::unpack(
+        to_device_type_impl<T>::map_to_device(param), row, col, num_rows,
+        num_cols);
 }
 
 
@@ -217,7 +218,6 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
     }
 }
 
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
                 dim<2> size, KernelArgs &&... args)

From 004691ef8b39900ff5e37ff363a1c362c57f345a Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 8 Jun 2021 23:57:25 +0200
Subject: [PATCH 0826/1008] unify simple solver kernel tests

---
 CMakeLists.txt                                |  34 +-
 cmake/create_test.cmake                       | 219 +++++------
 cuda/test/solver/CMakeLists.txt               |   6 -
 cuda/test/solver/bicgstab_kernels.cpp         | 345 ------------------
 cuda/test/solver/cg_kernels.cpp               | 245 -------------
 cuda/test/solver/cgs_kernels.cpp              | 337 -----------------
 cuda/test/solver/fcg_kernels.cpp              | 258 -------------
 cuda/test/solver/ir_kernels.cpp               | 259 -------------
 hip/test/solver/CMakeLists.txt                |   6 -
 hip/test/solver/bicg_kernels.cpp              | 330 -----------------
 hip/test/solver/bicgstab_kernels.cpp          | 345 ------------------
 hip/test/solver/cg_kernels.cpp                | 245 -------------
 hip/test/solver/cgs_kernels.cpp               | 337 -----------------
 hip/test/solver/fcg_kernels.cpp               | 258 -------------
 hip/test/solver/ir_kernels.cpp                | 259 -------------
 omp/test/solver/CMakeLists.txt                |   6 -
 omp/test/solver/bicg_kernels.cpp              | 313 ----------------
 test/CMakeLists.txt                           |   1 +
 .../build_export}/CMakeLists.txt              |   2 +-
 .../build_install}/CMakeLists.txt             |   0
 .../build_install}/data/A.mtx                 |   0
 .../build_install}/data/b.mtx                 |   0
 .../build_install}/data/x0.mtx                |   0
 .../build_install}/test_install.cpp           |   0
 .../build_subdir}/CMakeLists.txt              |   4 +-
 test/solver/CMakeLists.txt                    |   6 +
 {cuda/test => test}/solver/bicg_kernels.cpp   | 137 ++++---
 .../test => test}/solver/bicgstab_kernels.cpp | 101 ++---
 {omp/test => test}/solver/cg_kernels.cpp      |  64 ++--
 {omp/test => test}/solver/cgs_kernels.cpp     | 103 +++---
 {omp/test => test}/solver/fcg_kernels.cpp     |  66 ++--
 {omp/test => test}/solver/ir_kernels.cpp      |  57 +--
 test/utils/executor.hpp                       |  78 ++++
 33 files changed, 479 insertions(+), 3942 deletions(-)
 delete mode 100644 cuda/test/solver/bicgstab_kernels.cpp
 delete mode 100644 cuda/test/solver/cg_kernels.cpp
 delete mode 100644 cuda/test/solver/cgs_kernels.cpp
 delete mode 100644 cuda/test/solver/fcg_kernels.cpp
 delete mode 100644 cuda/test/solver/ir_kernels.cpp
 delete mode 100644 hip/test/solver/bicg_kernels.cpp
 delete mode 100644 hip/test/solver/bicgstab_kernels.cpp
 delete mode 100644 hip/test/solver/cg_kernels.cpp
 delete mode 100644 hip/test/solver/cgs_kernels.cpp
 delete mode 100644 hip/test/solver/fcg_kernels.cpp
 delete mode 100644 hip/test/solver/ir_kernels.cpp
 delete mode 100644 omp/test/solver/bicg_kernels.cpp
 create mode 100644 test/CMakeLists.txt
 rename {test_exportbuild => test/build_export}/CMakeLists.txt (82%)
 rename {test_install => test/build_install}/CMakeLists.txt (100%)
 rename {test_install => test/build_install}/data/A.mtx (100%)
 rename {test_install => test/build_install}/data/b.mtx (100%)
 rename {test_install => test/build_install}/data/x0.mtx (100%)
 rename {test_install => test/build_install}/test_install.cpp (100%)
 rename {test_subdir => test/build_subdir}/CMakeLists.txt (60%)
 create mode 100644 test/solver/CMakeLists.txt
 rename {cuda/test => test}/solver/bicg_kernels.cpp (74%)
 rename {omp/test => test}/solver/bicgstab_kernels.cpp (80%)
 rename {omp/test => test}/solver/cg_kernels.cpp (81%)
 rename {omp/test => test}/solver/cgs_kernels.cpp (79%)
 rename {omp/test => test}/solver/fcg_kernels.cpp (83%)
 rename {omp/test => test}/solver/ir_kernels.cpp (89%)
 create mode 100644 test/utils/executor.hpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 99985f16101..a9b11e2375c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -261,6 +261,9 @@ if (GINKGO_BUILD_OMP)
 endif()
 add_subdirectory(core)           # Core Ginkgo types and top-level functions
 add_subdirectory(include)        # Public API self-contained check
+if (GINKGO_BUILD_TESTS)
+    add_subdirectory(test)       # Tests running on all executors
+endif()
 
 # Non core directories and targets
 if(GINKGO_BUILD_EXAMPLES)
@@ -331,21 +334,20 @@ ginkgo_modify_flags(CMAKE_CUDA_FLAGS_RELEASE)
 ginkgo_install()
 
 if(MSVC)
-    # Set path/command with $<CONFIG>
-    set(GINKGO_TEST_INSTALL_COMMAND "${Ginkgo_BINARY_DIR}/test_install/$<CONFIG>/test_install")
-    set(GINKGO_TEST_EXPORTBUILD_COMMAND "${Ginkgo_BINARY_DIR}/test_exportbuild/$<CONFIG>/test_exportbuild")
-    if(GINKGO_BUILD_CUDA)
-        set(GINKGO_TEST_INSTALL_CUDA_COMMAND "${Ginkgo_BINARY_DIR}/test_install/$<CONFIG>/test_install_cuda")
-    endif()
+    set(GINKGO_TEST_INSTALL_DIR "${Ginkgo_BINARY_DIR}/test/build_install/$<CONFIG>/")
+    set(GINKGO_TEST_EXPORTBUILD_DIR "${Ginkgo_BINARY_DIR}/test/build_export/$<CONFIG>/")
 else()
-    set(GINKGO_TEST_INSTALL_COMMAND "${Ginkgo_BINARY_DIR}/test_install/test_install")
-    set(GINKGO_TEST_EXPORTBUILD_COMMAND "${Ginkgo_BINARY_DIR}/test_exportbuild/test_exportbuild")
-    if(GINKGO_BUILD_CUDA)
-        set(GINKGO_TEST_INSTALL_CUDA_COMMAND "${Ginkgo_BINARY_DIR}/test_install/test_install_cuda")
-    endif()
-    if(GINKGO_BUILD_HIP)
-        set(GINKGO_TEST_INSTALL_HIP_COMMAND "${Ginkgo_BINARY_DIR}/test_install/test_install_hip")
-    endif()
+    set(GINKGO_TEST_INSTALL_DIR "${Ginkgo_BINARY_DIR}/test/build_install/")
+    set(GINKGO_TEST_EXPORTBUILD_DIR "${Ginkgo_BINARY_DIR}/test/build_export/")
+endif()
+
+set(GINKGO_TEST_INSTALL_COMMAND "${GINKGO_TEST_INSTALL_DIR}/test_install")
+set(GINKGO_TEST_EXPORTBUILD_COMMAND "${GINKGO_TEST_EXPORTBUILD_DIR}/test_exportbuild")
+if(GINKGO_BUILD_CUDA)
+    set(GINKGO_TEST_INSTALL_CUDA_COMMAND "${GINKGO_TEST_EXPORTBUILD_DIR}/test_install_cuda")
+endif()
+if(GINKGO_BUILD_HIP)
+    set(GINKGO_TEST_INSTALL_HIP_COMMAND "${GINKGO_TEST_INSTALL_DIR}/test_install_hip")
 endif()
 
 file(MAKE_DIRECTORY ${Ginkgo_BINARY_DIR}/test_install)
@@ -354,7 +356,7 @@ if (NOT "${CMAKE_GENERATOR_TOOLSET}" STREQUAL "")
     set(TOOLSET "-T${CMAKE_GENERATOR_TOOLSET}")
 endif()
 add_custom_target(test_install
-    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${TOOLSET} -H${Ginkgo_SOURCE_DIR}/test_install
+    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${TOOLSET} -H${GINKGO_TEST_INSTALL_DIR}
     -B${Ginkgo_BINARY_DIR}/test_install
     -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
     -DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX}/${GINKGO_INSTALL_CONFIG_DIR}
@@ -371,7 +373,7 @@ add_custom_target(test_install
     COMMENT "Running a test on the installed binaries. This requires running `(sudo) make install` first.")
 
 add_custom_target(test_exportbuild
-    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${TOOLSET} -H${Ginkgo_SOURCE_DIR}/test_exportbuild
+    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${TOOLSET} -H${GINKGO_TEST_EXPORTBUILD_DIR}
     -B${Ginkgo_BINARY_DIR}/test_exportbuild
     -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
     -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index 84d80214440..cf5728a79d2 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -1,149 +1,86 @@
-function(ginkgo_create_test test_name)
+function(ginkgo_build_test_name test_name target_name)
     file(RELATIVE_PATH REL_BINARY_DIR
         ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
     string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
-    add_executable(${TEST_TARGET_NAME} ${test_name}.cpp)
-    target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_14)
-    target_compile_options("${TEST_TARGET_NAME}" PRIVATE ${GINKGO_COMPILER_FLAGS})
-    target_include_directories("${TEST_TARGET_NAME}"
-        PRIVATE
-        "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
-        )
-    set_target_properties(${TEST_TARGET_NAME} PROPERTIES
+    set(${target_name} ${TEST_TARGET_NAME} PARENT_SCOPE)
+endfunction()
+
+function(ginkgo_set_test_target_properties test_name test_target_name)
+    file(RELATIVE_PATH REL_BINARY_DIR
+        ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+    set_target_properties(${test_target_name} PROPERTIES
         OUTPUT_NAME ${test_name})
     if (GINKGO_FAST_TESTS)
-        target_compile_definitions(${TEST_TARGET_NAME} PRIVATE GINKGO_FAST_TESTS)
+        target_compile_definitions(${test_target_name} PRIVATE GINKGO_FAST_TESTS)
     endif()
     if (GINKGO_COMPILING_DPCPP_TEST AND GINKGO_DPCPP_SINGLE_MODE)
-        target_compile_definitions("${TEST_TARGET_NAME}" PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
+        target_compile_definitions(${test_target_name} PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
     endif()
     if (GINKGO_CHECK_CIRCULAR_DEPS)
-        target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
+        target_link_libraries(${test_target_name} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
     endif()
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
     add_test(NAME ${REL_BINARY_DIR}/${test_name}
-        COMMAND ${TEST_TARGET_NAME}
+        COMMAND ${test_target_name}
         WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
+    target_include_directories(${test_target_name} PRIVATE ${Ginkgo_BINARY_DIR})
+    target_link_libraries(${test_target_name} PRIVATE ginkgo GTest::Main GTest::GTest)
+endfunction()
+
+function(ginkgo_create_test test_name)
+    ginkgo_build_test_name(${test_name} test_target_name)
+    add_executable(${test_target_name} ${test_name}.cpp)
+    target_compile_features(${test_target_name} PUBLIC cxx_std_14)
+    target_compile_options(${test_target_name} PRIVATE ${GINKGO_COMPILER_FLAGS})
+    target_link_libraries(${test_target_name} PRIVATE ${ARGN})
+    ginkgo_set_test_target_properties(${test_name} ${test_target_name})
 endfunction(ginkgo_create_test)
 
 function(ginkgo_create_dpcpp_test test_name)
-    file(RELATIVE_PATH REL_BINARY_DIR
-        ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
-    string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
-    add_executable(${TEST_TARGET_NAME} ${test_name}.dp.cpp)
-    target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_17)
-    target_compile_options("${TEST_TARGET_NAME}" PRIVATE "${GINKGO_DPCPP_FLAGS}")
-    target_link_options("${TEST_TARGET_NAME}" PRIVATE -fsycl-device-code-split=per_kernel)
-    if (GINKGO_DPCPP_SINGLE_MODE)
-        target_compile_definitions("${TEST_TARGET_NAME}" PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
-    endif()
-    target_include_directories("${TEST_TARGET_NAME}"
-        PRIVATE
-        "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
-        )
-    set_target_properties(${TEST_TARGET_NAME} PROPERTIES
-        OUTPUT_NAME ${test_name})
-    if (GINKGO_FAST_TESTS)
-        target_compile_definitions(${TEST_TARGET_NAME} PRIVATE GINKGO_FAST_TESTS)
-    endif()
-    if (GINKGO_CHECK_CIRCULAR_DEPS)
-        target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
-    endif()
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
-    add_test(NAME ${REL_BINARY_DIR}/${test_name}
-        COMMAND ${TEST_TARGET_NAME}
-        WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
+    ginkgo_build_test_name(${test_name} TEST_TARGET_NAME)
+    add_executable(${test_target_name} ${test_name}.dp.cpp)
+    target_compile_features(${test_target_name} PUBLIC cxx_std_17)
+    target_compile_options(${test_target_name} PRIVATE "${GINKGO_DPCPP_FLAGS}")
+    target_link_options(${test_target_name} PRIVATE -fsycl-device-code-split=per_kernel)
+    ginkgo_set_test_target_properties(${test_name} ${test_target_name})
 endfunction(ginkgo_create_dpcpp_test)
 
 function(ginkgo_create_thread_test test_name)
     set(THREADS_PREFER_PTHREAD_FLAG ON)
     find_package(Threads REQUIRED)
-    file(RELATIVE_PATH REL_BINARY_DIR
-        ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
-    string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
-    add_executable(${TEST_TARGET_NAME} ${test_name}.cpp)
-    target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_14)
-    target_compile_options("${TEST_TARGET_NAME}" PRIVATE ${GINKGO_COMPILER_FLAGS})
-    target_include_directories("${TEST_TARGET_NAME}"
-        PRIVATE
-        "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
-        )
-    set_target_properties(${TEST_TARGET_NAME} PROPERTIES
-        OUTPUT_NAME ${test_name})
-    if (GINKGO_FAST_TESTS)
-        target_compile_definitions(${TEST_TARGET_NAME} PRIVATE GINKGO_FAST_TESTS)
-    endif()
-    if (GINKGO_CHECK_CIRCULAR_DEPS)
-        target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
-    endif()
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest
-        Threads::Threads ${ARGN})
-        add_test(NAME ${REL_BINARY_DIR}/${test_name}
-            COMMAND ${TEST_TARGET_NAME}
-            WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
+    ginkgo_build_test_name(${test_name} test_target_name)
+    add_executable(${test_target_name} ${test_name}.cpp)
+    target_compile_features(${test_target_name} PUBLIC cxx_std_14)
+    target_compile_options(${test_target_name} PRIVATE ${GINKGO_COMPILER_FLAGS})
+    target_link_libraries(${test_target_name} PRIVATE Threads::Threads ${ARGN})
+    ginkgo_set_test_target_properties(${test_name} ${test_target_name})
 endfunction(ginkgo_create_thread_test)
 
 function(ginkgo_create_test_cpp_cuda_header test_name)
-    file(RELATIVE_PATH REL_BINARY_DIR
-        ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
-    string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
-    add_executable(${TEST_TARGET_NAME} ${test_name}.cpp)
-    target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_14)
-    target_compile_options("${TEST_TARGET_NAME}" PRIVATE ${GINKGO_COMPILER_FLAGS})
-    target_include_directories("${TEST_TARGET_NAME}"
-        PRIVATE
-        "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
-        "${CUDA_INCLUDE_DIRS}"
-        )
-    set_target_properties(${TEST_TARGET_NAME} PROPERTIES
-        OUTPUT_NAME ${test_name})
-    if (GINKGO_FAST_TESTS)
-        target_compile_definitions(${TEST_TARGET_NAME} PRIVATE GINKGO_FAST_TESTS)
-    endif()
-    if (GINKGO_CHECK_CIRCULAR_DEPS)
-        target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
-    endif()
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
-    add_test(NAME ${REL_BINARY_DIR}/${test_name}
-        COMMAND ${TEST_TARGET_NAME}
-        WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
+    ginkgo_build_test_name(${test_name} test_target_name)
+    add_executable(${test_target_name} ${test_name}.cpp)
+    target_compile_features(${test_target_name} PUBLIC cxx_std_14)
+    target_compile_options(${test_target_name} PRIVATE ${GINKGO_COMPILER_FLAGS})
+    target_include_directories(${test_target_name} PRIVATE "${CUDA_INCLUDE_DIRS}")
+    target_link_libraries(${test_target_name} PRIVATE ${ARGN})
+    ginkgo_set_test_target_properties(${test_name} ${test_target_name})
 endfunction(ginkgo_create_test_cpp_cuda_header)
 
 function(ginkgo_create_cuda_test test_name)
-    file(RELATIVE_PATH REL_BINARY_DIR
-        ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
-    string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
-    add_executable(${TEST_TARGET_NAME} ${test_name}.cu)
-    target_compile_features("${TEST_TARGET_NAME}" PUBLIC cxx_std_14)
-    target_include_directories("${TEST_TARGET_NAME}"
-        PRIVATE
-        "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
-        )
-    target_compile_options(${TEST_TARGET_NAME}
+    ginkgo_build_test_name(${test_name} test_target_name)
+    add_executable(${test_target_name} ${test_name}.cu)
+    target_compile_features(${test_target_name} PUBLIC cxx_std_14)
+    target_compile_options(${test_target_name}
         PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_ARCH_FLAGS}>")
+    target_link_libraries(${test_target_name} PRIVATE ${ARGN})
     # we handle CUDA architecture flags for now, disable CMake handling
     if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
-        set_target_properties(${TEST_TARGET_NAME} PROPERTIES CUDA_ARCHITECTURES OFF)
+        set_target_properties(${test_target_name} PROPERTIES CUDA_ARCHITECTURES OFF)
     endif()
-    set_target_properties(${TEST_TARGET_NAME} PROPERTIES
-        OUTPUT_NAME ${test_name})
-    if (GINKGO_FAST_TESTS)
-        target_compile_definitions(${TEST_TARGET_NAME} PRIVATE GINKGO_FAST_TESTS)
-    endif()
-    if (GINKGO_CHECK_CIRCULAR_DEPS)
-        target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
-    endif()
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
-    add_test(NAME ${REL_BINARY_DIR}/${test_name}
-        COMMAND ${TEST_TARGET_NAME}
-        WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
+    ginkgo_set_test_target_properties(${test_name} ${test_target_name})
 endfunction(ginkgo_create_cuda_test)
 
 function(ginkgo_create_hip_test test_name)
-    file(RELATIVE_PATH REL_BINARY_DIR
-        ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
-    string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
-
+ginkgo_build_test_name(${test_name} test_target_name)
     set_source_files_properties(${test_name}.hip.cpp PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
     set(GINKGO_TEST_HIP_DEFINES)
     if (GINKGO_FAST_TESTS)
@@ -154,7 +91,7 @@ function(ginkgo_create_hip_test test_name)
     # creates a redefinition of all flags. This creates some issues with `nvcc`,
     # but `clang` seems fine with the redefinitions.
     if (GINKGO_HIP_PLATFORM MATCHES "${HIP_PLATFORM_NVIDIA_REGEX}")
-        hip_add_executable(${TEST_TARGET_NAME} ${test_name}.hip.cpp
+        hip_add_executable(${test_target_name} ${test_name}.hip.cpp
             # If `FindHIP.cmake`, namely `HIP_PARSE_HIPCC_OPTIONS` macro and
             # call gets fixed, uncomment this.
             HIPCC_OPTIONS ${GINKGO_TEST_HIP_DEFINES} # ${GINKGO_HIPCC_OPTIONS}
@@ -162,7 +99,7 @@ function(ginkgo_create_hip_test test_name)
             # CLANG_OPTIONS ${GINKGO_TEST_HIP_DEFINES} ${GINKGO_HIP_CLANG_OPTIONS}
             )
     else() # hcc/clang
-        hip_add_executable(${TEST_TARGET_NAME} ${test_name}.hip.cpp
+        hip_add_executable(${test_target_name} ${test_name}.hip.cpp
             HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS} ${GINKGO_TEST_HIP_DEFINES}
             NVCC_OPTIONS  ${GINKGO_HIP_NVCC_OPTIONS}
             CLANG_OPTIONS ${GINKGO_HIP_CLANG_OPTIONS}
@@ -170,11 +107,10 @@ function(ginkgo_create_hip_test test_name)
     endif()
 
     # Let's use a normal compiler for linking
-    set_target_properties(${TEST_TARGET_NAME} PROPERTIES LINKER_LANGUAGE CXX)
+    set_target_properties(${test_target_name} PROPERTIES LINKER_LANGUAGE CXX)
 
-    target_include_directories("${TEST_TARGET_NAME}"
+    target_include_directories(${test_target_name}
         PRIVATE
-        "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
         # Only `math` requires it so far, but it's much easier
         # to put these this way.
         ${GINKGO_HIP_THRUST_PATH}
@@ -184,17 +120,38 @@ function(ginkgo_create_hip_test test_name)
         ${hiprand_INCLUDE_DIRS}
         ${HIPSPARSE_INCLUDE_DIRS}
         )
-    set_target_properties(${TEST_TARGET_NAME} PROPERTIES
-        OUTPUT_NAME ${test_name})
-
-    if(BUILD_SHARED_LIBS)
-        if (GINKGO_CHECK_CIRCULAR_DEPS)
-            target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
-        endif()
-    endif()
-
-    target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
-    add_test(NAME ${REL_BINARY_DIR}/${test_name}
-        COMMAND ${TEST_TARGET_NAME}
-        WORKING_DIRECTORY "$<TARGET_FILE_DIR:ginkgo>")
+    target_link_libraries(${test_target_name} PRIVATE ${ARGN})
+    ginkgo_set_test_target_properties(${test_name} ${test_target_name})
 endfunction(ginkgo_create_hip_test)
+
+function(ginkgo_create_common_test test_name)
+    set(executors)
+    if(GINKGO_BUILD_OMP)
+        list(APPEND executors omp)
+    endif()
+    if(GINKGO_BUILD_HIP)
+        list(APPEND executors hip)
+    endif()
+    if(GINKGO_BUILD_CUDA)
+        list(APPEND executors cuda)
+    endif()
+    if(GINKGO_BUILD_DPCPP)
+        list(APPEND executors dpcpp)
+    endif()
+    foreach(exec ${executors})
+        ginkgo_build_test_name(${test_name} test_target_name)
+        # build executor typename out of shorthand
+        string(SUBSTRING ${exec} 0 1 exec_initial)
+        string(SUBSTRING ${exec} 1 -1 exec_tail)
+        string(TOUPPER ${exec_initial} exec_initial)
+        set(exec_type ${exec_initial}${exec_tail}Executor)
+        # set up actual test
+        set(test_target_name ${test_target_name}_${exec})
+        add_executable(${test_target_name} ${test_name}.cpp)
+        target_compile_features(${test_target_name} PUBLIC cxx_std_14)
+        target_compile_options(${test_target_name} PRIVATE ${GINKGO_COMPILER_FLAGS})
+        target_compile_definitions(${test_target_name} PRIVATE EXEC_TYPE=${exec_type} EXEC_NAMESPACE=${exec})
+        target_link_libraries(${test_target_name} PRIVATE ${ARGN})
+        ginkgo_set_test_target_properties(${test_name}_${exec} ${test_target_name})
+    endforeach()
+endfunction(ginkgo_create_common_test)
\ No newline at end of file
diff --git a/cuda/test/solver/CMakeLists.txt b/cuda/test/solver/CMakeLists.txt
index 623c3076ed8..ae4c6304c92 100644
--- a/cuda/test/solver/CMakeLists.txt
+++ b/cuda/test/solver/CMakeLists.txt
@@ -1,11 +1,5 @@
-ginkgo_create_test(bicg_kernels)
-ginkgo_create_test(bicgstab_kernels)
-ginkgo_create_test(cg_kernels)
-ginkgo_create_test(cgs_kernels)
-ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
 ginkgo_create_test(cb_gmres_kernels)
 ginkgo_create_test(idr_kernels)
-ginkgo_create_test(ir_kernels)
 ginkgo_create_test_cpp_cuda_header(lower_trs_kernels)
 ginkgo_create_test_cpp_cuda_header(upper_trs_kernels)
diff --git a/cuda/test/solver/bicgstab_kernels.cpp b/cuda/test/solver/bicgstab_kernels.cpp
deleted file mode 100644
index b1e0ca85d1b..00000000000
--- a/cuda/test/solver/bicgstab_kernels.cpp
+++ /dev/null
@@ -1,345 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/solver/bicgstab.hpp>
-
-
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/stop/combined.hpp>
-#include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm.hpp>
-
-
-#include "core/solver/bicgstab_kernels.hpp"
-#include "cuda/test/utils.hpp"
-
-
-namespace {
-
-
-class Bicgstab : public ::testing::Test {
-protected:
-    using Mtx = gko::matrix::Dense<>;
-    using Solver = gko::solver::Bicgstab<>;
-
-    Bicgstab() : rand_engine(30) {}
-
-    void SetUp()
-    {
-        ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
-        ref = gko::ReferenceExecutor::create();
-        cuda = gko::CudaExecutor::create(0, ref);
-
-        mtx = gen_mtx(123, 123);
-        gko::test::make_diag_dominant(mtx.get());
-        d_mtx = Mtx::create(cuda);
-        d_mtx->copy_from(mtx.get());
-
-        cuda_bicgstab_factory =
-            Solver::build()
-                .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(cuda),
-                    gko::stop::ResidualNorm<>::build()
-                        .with_reduction_factor(1e-15)
-                        .on(cuda))
-                .on(cuda);
-        ref_bicgstab_factory =
-            Solver::build()
-                .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNorm<>::build()
-                        .with_reduction_factor(1e-15)
-                        .on(ref))
-                .on(ref);
-    }
-
-    void TearDown()
-    {
-        if (cuda != nullptr) {
-            ASSERT_NO_THROW(cuda->synchronize());
-        }
-    }
-
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
-    {
-        return gko::test::generate_random_matrix<Mtx>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
-    }
-
-    void initialize_data()
-    {
-        int m = 597;
-        int n = 17;
-        x = gen_mtx(m, n);
-        b = gen_mtx(m, n);
-        r = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        rr = gen_mtx(m, n);
-        s = gen_mtx(m, n);
-        t = gen_mtx(m, n);
-        y = gen_mtx(m, n);
-        v = gen_mtx(m, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        alpha = gen_mtx(1, n);
-        beta = gen_mtx(1, n);
-        gamma = gen_mtx(1, n);
-        omega = gen_mtx(1, n);
-        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(ref, n));
-        for (size_t i = 0; i < n; ++i) {
-            stop_status->get_data()[i].reset();
-        }
-
-        d_x = Mtx::create(cuda);
-        d_b = Mtx::create(cuda);
-        d_r = Mtx::create(cuda);
-        d_z = Mtx::create(cuda);
-        d_p = Mtx::create(cuda);
-        d_t = Mtx::create(cuda);
-        d_s = Mtx::create(cuda);
-        d_y = Mtx::create(cuda);
-        d_v = Mtx::create(cuda);
-        d_rr = Mtx::create(cuda);
-        d_prev_rho = Mtx::create(cuda);
-        d_rho = Mtx::create(cuda);
-        d_alpha = Mtx::create(cuda);
-        d_beta = Mtx::create(cuda);
-        d_gamma = Mtx::create(cuda);
-        d_omega = Mtx::create(cuda);
-        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(cuda));
-
-        d_x->copy_from(x.get());
-        d_b->copy_from(b.get());
-        d_r->copy_from(r.get());
-        d_z->copy_from(z.get());
-        d_p->copy_from(p.get());
-        d_v->copy_from(v.get());
-        d_y->copy_from(y.get());
-        d_t->copy_from(t.get());
-        d_s->copy_from(s.get());
-        d_rr->copy_from(rr.get());
-        d_prev_rho->copy_from(prev_rho.get());
-        d_rho->copy_from(rho.get());
-        d_alpha->copy_from(alpha.get());
-        d_beta->copy_from(beta.get());
-        d_gamma->copy_from(gamma.get());
-        d_omega->copy_from(omega.get());
-        *d_stop_status =
-            *stop_status;  // copy_from is not a public member function of Array
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::CudaExecutor> cuda;
-
-    std::ranlux48 rand_engine;
-
-    std::shared_ptr<Mtx> mtx;
-    std::shared_ptr<Mtx> d_mtx;
-    std::unique_ptr<Solver::Factory> cuda_bicgstab_factory;
-    std::unique_ptr<Solver::Factory> ref_bicgstab_factory;
-
-    std::unique_ptr<Mtx> x;
-    std::unique_ptr<Mtx> b;
-    std::unique_ptr<Mtx> r;
-    std::unique_ptr<Mtx> z;
-    std::unique_ptr<Mtx> p;
-    std::unique_ptr<Mtx> rr;
-    std::unique_ptr<Mtx> s;
-    std::unique_ptr<Mtx> t;
-    std::unique_ptr<Mtx> y;
-    std::unique_ptr<Mtx> v;
-    std::unique_ptr<Mtx> prev_rho;
-    std::unique_ptr<Mtx> rho;
-    std::unique_ptr<Mtx> alpha;
-    std::unique_ptr<Mtx> beta;
-    std::unique_ptr<Mtx> gamma;
-    std::unique_ptr<Mtx> omega;
-    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
-
-    std::unique_ptr<Mtx> d_x;
-    std::unique_ptr<Mtx> d_b;
-    std::unique_ptr<Mtx> d_r;
-    std::unique_ptr<Mtx> d_z;
-    std::unique_ptr<Mtx> d_p;
-    std::unique_ptr<Mtx> d_t;
-    std::unique_ptr<Mtx> d_s;
-    std::unique_ptr<Mtx> d_y;
-    std::unique_ptr<Mtx> d_v;
-    std::unique_ptr<Mtx> d_rr;
-    std::unique_ptr<Mtx> d_prev_rho;
-    std::unique_ptr<Mtx> d_rho;
-    std::unique_ptr<Mtx> d_alpha;
-    std::unique_ptr<Mtx> d_beta;
-    std::unique_ptr<Mtx> d_gamma;
-    std::unique_ptr<Mtx> d_omega;
-    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
-};
-
-
-TEST_F(Bicgstab, CudaBicgstabInitializeIsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::bicgstab::initialize(
-        ref, b.get(), r.get(), rr.get(), y.get(), s.get(), t.get(), z.get(),
-        v.get(), p.get(), prev_rho.get(), rho.get(), alpha.get(), beta.get(),
-        gamma.get(), omega.get(), stop_status.get());
-    gko::kernels::cuda::bicgstab::initialize(
-        cuda, d_b.get(), d_r.get(), d_rr.get(), d_y.get(), d_s.get(), d_t.get(),
-        d_z.get(), d_v.get(), d_p.get(), d_prev_rho.get(), d_rho.get(),
-        d_alpha.get(), d_beta.get(), d_gamma.get(), d_omega.get(),
-        d_stop_status.get());
-
-    GKO_EXPECT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_y, y, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_t, t, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_s, s, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_rr, rr, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_v, v, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_rho, rho, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_alpha, alpha, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_beta, beta, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_gamma, gamma, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_omega, omega, 1e-14);
-    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
-}
-
-
-TEST_F(Bicgstab, CudaBicgstabStep1IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::bicgstab::step_1(
-        ref, r.get(), p.get(), v.get(), rho.get(), prev_rho.get(), alpha.get(),
-        omega.get(), stop_status.get());
-    gko::kernels::cuda::bicgstab::step_1(
-        cuda, d_r.get(), d_p.get(), d_v.get(), d_rho.get(), d_prev_rho.get(),
-        d_alpha.get(), d_omega.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-}
-
-
-TEST_F(Bicgstab, CudaBicgstabStep2IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::bicgstab::step_2(ref, r.get(), s.get(), v.get(),
-                                              rho.get(), alpha.get(),
-                                              beta.get(), stop_status.get());
-    gko::kernels::cuda::bicgstab::step_2(cuda, d_r.get(), d_s.get(), d_v.get(),
-                                         d_rho.get(), d_alpha.get(),
-                                         d_beta.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_s, s, 1e-14);
-}
-
-
-TEST_F(Bicgstab, CudaBicgstabStep3IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::bicgstab::step_3(
-        ref, x.get(), r.get(), s.get(), t.get(), y.get(), z.get(), alpha.get(),
-        beta.get(), gamma.get(), omega.get(), stop_status.get());
-    gko::kernels::cuda::bicgstab::step_3(
-        cuda, d_x.get(), d_r.get(), d_s.get(), d_t.get(), d_y.get(), d_z.get(),
-        d_alpha.get(), d_beta.get(), d_gamma.get(), d_omega.get(),
-        d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_omega, omega, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-}
-
-
-TEST_F(Bicgstab, CudaBicgstabApplyOneRHSIsEquivalentToRef)
-{
-    int m = 123;
-    int n = 1;
-    auto ref_solver = ref_bicgstab_factory->generate(mtx);
-    auto cuda_solver = cuda_bicgstab_factory->generate(d_mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(cuda);
-    auto d_x = Mtx::create(cuda);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
-
-    ref_solver->apply(b.get(), x.get());
-    cuda_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-}
-
-
-TEST_F(Bicgstab, CudaBicgstabApplyMultipleRHSIsEquivalentToRef)
-{
-    int m = 123;
-    int n = 16;
-    auto cuda_solver = cuda_bicgstab_factory->generate(d_mtx);
-    auto ref_solver = ref_bicgstab_factory->generate(mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(cuda);
-    auto d_x = Mtx::create(cuda);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
-
-    ref_solver->apply(b.get(), x.get());
-    cuda_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-}
-
-
-}  // namespace
diff --git a/cuda/test/solver/cg_kernels.cpp b/cuda/test/solver/cg_kernels.cpp
deleted file mode 100644
index e6443543efb..00000000000
--- a/cuda/test/solver/cg_kernels.cpp
+++ /dev/null
@@ -1,245 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/solver/cg.hpp>
-
-
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/stop/combined.hpp>
-#include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm.hpp>
-
-
-#include "core/solver/cg_kernels.hpp"
-#include "cuda/test/utils.hpp"
-
-
-namespace {
-
-
-class Cg : public ::testing::Test {
-protected:
-    using Mtx = gko::matrix::Dense<>;
-    Cg() : rand_engine(30) {}
-
-    void SetUp()
-    {
-        ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
-        ref = gko::ReferenceExecutor::create();
-        cuda = gko::CudaExecutor::create(0, ref);
-    }
-
-    void TearDown()
-    {
-        if (cuda != nullptr) {
-            ASSERT_NO_THROW(cuda->synchronize());
-        }
-    }
-
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
-    {
-        return gko::test::generate_random_matrix<Mtx>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
-    }
-
-    void initialize_data()
-    {
-        int m = 597;
-        int n = 43;
-        b = gen_mtx(m, n);
-        r = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        q = gen_mtx(m, n);
-        x = gen_mtx(m, n);
-        beta = gen_mtx(1, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(ref, n));
-        for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
-            stop_status->get_data()[i].reset();
-        }
-
-        d_b = Mtx::create(cuda);
-        d_b->copy_from(b.get());
-        d_r = Mtx::create(cuda);
-        d_r->copy_from(r.get());
-        d_z = Mtx::create(cuda);
-        d_z->copy_from(z.get());
-        d_p = Mtx::create(cuda);
-        d_p->copy_from(p.get());
-        d_q = Mtx::create(cuda);
-        d_q->copy_from(q.get());
-        d_x = Mtx::create(cuda);
-        d_x->copy_from(x.get());
-        d_beta = Mtx::create(cuda);
-        d_beta->copy_from(beta.get());
-        d_prev_rho = Mtx::create(cuda);
-        d_prev_rho->copy_from(prev_rho.get());
-        d_rho = Mtx::create(cuda);
-        d_rho->copy_from(rho.get());
-        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(cuda, n));
-        *d_stop_status = *stop_status;
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::CudaExecutor> cuda;
-
-    std::ranlux48 rand_engine;
-
-    std::unique_ptr<Mtx> b;
-    std::unique_ptr<Mtx> r;
-    std::unique_ptr<Mtx> z;
-    std::unique_ptr<Mtx> p;
-    std::unique_ptr<Mtx> q;
-    std::unique_ptr<Mtx> x;
-    std::unique_ptr<Mtx> beta;
-    std::unique_ptr<Mtx> prev_rho;
-    std::unique_ptr<Mtx> rho;
-    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
-
-    std::unique_ptr<Mtx> d_b;
-    std::unique_ptr<Mtx> d_r;
-    std::unique_ptr<Mtx> d_z;
-    std::unique_ptr<Mtx> d_p;
-    std::unique_ptr<Mtx> d_q;
-    std::unique_ptr<Mtx> d_x;
-    std::unique_ptr<Mtx> d_beta;
-    std::unique_ptr<Mtx> d_prev_rho;
-    std::unique_ptr<Mtx> d_rho;
-    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
-};
-
-
-TEST_F(Cg, CudaCgInitializeIsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::cg::initialize(ref, b.get(), r.get(), z.get(),
-                                            p.get(), q.get(), prev_rho.get(),
-                                            rho.get(), stop_status.get());
-    gko::kernels::cuda::cg::initialize(cuda, d_b.get(), d_r.get(), d_z.get(),
-                                       d_p.get(), d_q.get(), d_prev_rho.get(),
-                                       d_rho.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho, rho, 1e-14);
-    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
-}
-
-
-TEST_F(Cg, CudaCgStep1IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::cg::step_1(ref, p.get(), z.get(), rho.get(),
-                                        prev_rho.get(), stop_status.get());
-    gko::kernels::cuda::cg::step_1(cuda, d_p.get(), d_z.get(), d_rho.get(),
-                                   d_prev_rho.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-}
-
-
-TEST_F(Cg, CudaCgStep2IsEquivalentToRef)
-{
-    initialize_data();
-    gko::kernels::reference::cg::step_2(ref, x.get(), r.get(), p.get(), q.get(),
-                                        beta.get(), rho.get(),
-                                        stop_status.get());
-    gko::kernels::cuda::cg::step_2(cuda, d_x.get(), d_r.get(), d_p.get(),
-                                   d_q.get(), d_beta.get(), d_rho.get(),
-                                   d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-}
-
-
-TEST_F(Cg, ApplyIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    gko::test::make_hpd(mtx.get());
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = Mtx::create(cuda);
-    d_mtx->copy_from(mtx.get());
-    auto d_x = Mtx::create(cuda);
-    d_x->copy_from(x.get());
-    auto d_b = Mtx::create(cuda);
-    d_b->copy_from(b.get());
-    auto cg_factory =
-        gko::solver::Cg<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(ref))
-            .on(ref);
-    auto d_cg_factory =
-        gko::solver::Cg<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(cuda),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(cuda))
-            .on(cuda);
-    auto solver = cg_factory->generate(std::move(mtx));
-    auto d_solver = d_cg_factory->generate(std::move(d_mtx));
-
-    solver->apply(b.get(), x.get());
-    d_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-}  // namespace
diff --git a/cuda/test/solver/cgs_kernels.cpp b/cuda/test/solver/cgs_kernels.cpp
deleted file mode 100644
index f8cb89d55bb..00000000000
--- a/cuda/test/solver/cgs_kernels.cpp
+++ /dev/null
@@ -1,337 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/solver/cgs.hpp>
-
-
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/stop/combined.hpp>
-#include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm.hpp>
-
-
-#include "core/solver/cgs_kernels.hpp"
-#include "cuda/test/utils.hpp"
-
-
-namespace {
-
-
-class Cgs : public ::testing::Test {
-protected:
-    using Mtx = gko::matrix::Dense<>;
-    using Solver = gko::solver::Cgs<>;
-
-    Cgs() : rand_engine(30) {}
-
-    void SetUp()
-    {
-        ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
-        ref = gko::ReferenceExecutor::create();
-        cuda = gko::CudaExecutor::create(0, ref);
-
-        mtx = gen_mtx(123, 123);
-        gko::test::make_diag_dominant(mtx.get());
-        d_mtx = Mtx::create(cuda);
-        d_mtx->copy_from(mtx.get());
-        cuda_cgs_factory =
-            Solver::build()
-                .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(cuda),
-                    gko::stop::ResidualNorm<>::build()
-                        .with_reduction_factor(1e-15)
-                        .on(cuda))
-                .on(cuda);
-        ref_cgs_factory =
-            Solver::build()
-                .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNorm<>::build()
-                        .with_reduction_factor(1e-15)
-                        .on(ref))
-                .on(ref);
-    }
-
-    void TearDown()
-    {
-        if (cuda != nullptr) {
-            ASSERT_NO_THROW(cuda->synchronize());
-        }
-    }
-
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
-    {
-        return gko::test::generate_random_matrix<Mtx>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
-    }
-
-    void initialize_data()
-    {
-        int m = 597;
-        int n = 43;
-        b = gen_mtx(m, n);
-        r = gen_mtx(m, n);
-        r_tld = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        q = gen_mtx(m, n);
-        u = gen_mtx(m, n);
-        u_hat = gen_mtx(m, n);
-        v_hat = gen_mtx(m, n);
-        t = gen_mtx(m, n);
-        x = gen_mtx(m, n);
-        alpha = gen_mtx(1, n);
-        beta = gen_mtx(1, n);
-        gamma = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        rho_prev = gen_mtx(1, n);
-        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(ref, n));
-        for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
-            stop_status->get_data()[i].reset();
-        }
-
-        d_b = Mtx::create(cuda);
-        d_b->copy_from(b.get());
-        d_r = Mtx::create(cuda);
-        d_r->copy_from(r.get());
-        d_r_tld = Mtx::create(cuda);
-        d_r_tld->copy_from(r_tld.get());
-        d_p = Mtx::create(cuda);
-        d_p->copy_from(p.get());
-        d_q = Mtx::create(cuda);
-        d_q->copy_from(q.get());
-        d_u = Mtx::create(cuda);
-        d_u->copy_from(u.get());
-        d_u_hat = Mtx::create(cuda);
-        d_u_hat->copy_from(u_hat.get());
-        d_v_hat = Mtx::create(cuda);
-        d_v_hat->copy_from(v_hat.get());
-        d_t = Mtx::create(cuda);
-        d_t->copy_from(t.get());
-        d_x = Mtx::create(cuda);
-        d_x->copy_from(x.get());
-        d_alpha = Mtx::create(cuda);
-        d_alpha->copy_from(alpha.get());
-        d_beta = Mtx::create(cuda);
-        d_beta->copy_from(beta.get());
-        d_gamma = Mtx::create(cuda);
-        d_gamma->copy_from(gamma.get());
-        d_rho_prev = Mtx::create(cuda);
-        d_rho_prev->copy_from(rho_prev.get());
-        d_rho = Mtx::create(cuda);
-        d_rho->copy_from(rho.get());
-        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(cuda, n));
-        // because there is no public function copy_from, use overloaded =
-        // operator
-        *d_stop_status = *stop_status;
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::CudaExecutor> cuda;
-
-    std::ranlux48 rand_engine;
-
-    std::shared_ptr<Mtx> mtx;
-    std::shared_ptr<Mtx> d_mtx;
-    std::unique_ptr<Solver::Factory> cuda_cgs_factory;
-    std::unique_ptr<Solver::Factory> ref_cgs_factory;
-
-    std::unique_ptr<Mtx> b;
-    std::unique_ptr<Mtx> r;
-    std::unique_ptr<Mtx> r_tld;
-    std::unique_ptr<Mtx> t;
-    std::unique_ptr<Mtx> p;
-    std::unique_ptr<Mtx> q;
-    std::unique_ptr<Mtx> u;
-    std::unique_ptr<Mtx> u_hat;
-    std::unique_ptr<Mtx> v_hat;
-    std::unique_ptr<Mtx> x;
-    std::unique_ptr<Mtx> alpha;
-    std::unique_ptr<Mtx> beta;
-    std::unique_ptr<Mtx> gamma;
-    std::unique_ptr<Mtx> rho;
-    std::unique_ptr<Mtx> rho_prev;
-    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
-
-    std::unique_ptr<Mtx> d_b;
-    std::unique_ptr<Mtx> d_r;
-    std::unique_ptr<Mtx> d_r_tld;
-    std::unique_ptr<Mtx> d_t;
-    std::unique_ptr<Mtx> d_p;
-    std::unique_ptr<Mtx> d_q;
-    std::unique_ptr<Mtx> d_u;
-    std::unique_ptr<Mtx> d_u_hat;
-    std::unique_ptr<Mtx> d_v_hat;
-    std::unique_ptr<Mtx> d_x;
-    std::unique_ptr<Mtx> d_alpha;
-    std::unique_ptr<Mtx> d_beta;
-    std::unique_ptr<Mtx> d_gamma;
-    std::unique_ptr<Mtx> d_rho;
-    std::unique_ptr<Mtx> d_rho_prev;
-    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
-};
-
-
-TEST_F(Cgs, CudaCgsInitializeIsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::cgs::initialize(
-        ref, b.get(), r.get(), r_tld.get(), p.get(), q.get(), u.get(),
-        u_hat.get(), v_hat.get(), t.get(), alpha.get(), beta.get(), gamma.get(),
-        rho_prev.get(), rho.get(), stop_status.get());
-    gko::kernels::cuda::cgs::initialize(
-        cuda, d_b.get(), d_r.get(), d_r_tld.get(), d_p.get(), d_q.get(),
-        d_u.get(), d_u_hat.get(), d_v_hat.get(), d_t.get(), d_alpha.get(),
-        d_beta.get(), d_gamma.get(), d_rho_prev.get(), d_rho.get(),
-        d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r_tld, r_tld, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_u, u, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_t, t, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_u_hat, u_hat, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_v_hat, v_hat, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho_prev, rho_prev, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho, rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_beta, beta, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_gamma, gamma, 1e-14);
-    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
-}
-
-
-TEST_F(Cgs, CudaCgsStep1IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::cgs::step_1(ref, r.get(), u.get(), p.get(),
-                                         q.get(), beta.get(), rho.get(),
-                                         rho_prev.get(), stop_status.get());
-    gko::kernels::cuda::cgs::step_1(cuda, d_r.get(), d_u.get(), d_p.get(),
-                                    d_q.get(), d_beta.get(), d_rho.get(),
-                                    d_rho_prev.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_beta, beta, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_u, u, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-}
-
-
-TEST_F(Cgs, CudaCgsStep2IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::cgs::step_2(ref, u.get(), v_hat.get(), q.get(),
-                                         t.get(), alpha.get(), rho.get(),
-                                         gamma.get(), stop_status.get());
-    gko::kernels::cuda::cgs::step_2(cuda, d_u.get(), d_v_hat.get(), d_q.get(),
-                                    d_t.get(), d_alpha.get(), d_rho.get(),
-                                    d_gamma.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_t, t, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-}
-
-
-TEST_F(Cgs, CudaCgsStep3IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::cgs::step_3(ref, t.get(), u_hat.get(), r.get(),
-                                         x.get(), alpha.get(),
-                                         stop_status.get());
-    gko::kernels::cuda::cgs::step_3(cuda, d_t.get(), d_u_hat.get(), d_r.get(),
-                                    d_x.get(), d_alpha.get(),
-                                    d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-}
-
-
-TEST_F(Cgs, CudaCgsApplyOneRHSIsEquivalentToRef)
-{
-    int m = 123;
-    int n = 1;
-    auto ref_solver = ref_cgs_factory->generate(mtx);
-    auto cuda_solver = cuda_cgs_factory->generate(d_mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(cuda);
-    auto d_x = Mtx::create(cuda);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
-
-    ref_solver->apply(b.get(), x.get());
-    cuda_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-}
-
-
-TEST_F(Cgs, CudaCgsApplyMultipleRHSIsEquivalentToRef)
-{
-    int m = 123;
-    int n = 16;
-    auto cuda_solver = cuda_cgs_factory->generate(d_mtx);
-    auto ref_solver = ref_cgs_factory->generate(mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(cuda);
-    auto d_x = Mtx::create(cuda);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
-
-    ref_solver->apply(b.get(), x.get());
-    cuda_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-}
-
-}  // namespace
diff --git a/cuda/test/solver/fcg_kernels.cpp b/cuda/test/solver/fcg_kernels.cpp
deleted file mode 100644
index c18444ee850..00000000000
--- a/cuda/test/solver/fcg_kernels.cpp
+++ /dev/null
@@ -1,258 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/solver/fcg.hpp>
-
-
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/stop/combined.hpp>
-#include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm.hpp>
-
-
-#include "core/solver/fcg_kernels.hpp"
-#include "cuda/test/utils.hpp"
-
-
-namespace {
-
-
-class Fcg : public ::testing::Test {
-protected:
-    using Mtx = gko::matrix::Dense<>;
-    using Solver = gko::solver::Fcg<>;
-
-    Fcg() : rand_engine(30) {}
-
-    void SetUp()
-    {
-        ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
-        ref = gko::ReferenceExecutor::create();
-        cuda = gko::CudaExecutor::create(0, ref);
-    }
-
-    void TearDown()
-    {
-        if (cuda != nullptr) {
-            ASSERT_NO_THROW(cuda->synchronize());
-        }
-    }
-
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
-    {
-        return gko::test::generate_random_matrix<Mtx>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
-    }
-
-    void initialize_data()
-    {
-        int m = 597;
-        int n = 43;
-        b = gen_mtx(m, n);
-        r = gen_mtx(m, n);
-        t = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        q = gen_mtx(m, n);
-        x = gen_mtx(m, n);
-        beta = gen_mtx(1, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        rho_t = gen_mtx(1, n);
-        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(ref, n));
-        for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
-            stop_status->get_data()[i].reset();
-        }
-
-        d_b = Mtx::create(cuda);
-        d_b->copy_from(b.get());
-        d_r = Mtx::create(cuda);
-        d_r->copy_from(r.get());
-        d_t = Mtx::create(cuda);
-        d_t->copy_from(t.get());
-        d_z = Mtx::create(cuda);
-        d_z->copy_from(z.get());
-        d_p = Mtx::create(cuda);
-        d_p->copy_from(p.get());
-        d_q = Mtx::create(cuda);
-        d_q->copy_from(q.get());
-        d_x = Mtx::create(cuda);
-        d_x->copy_from(x.get());
-        d_beta = Mtx::create(cuda);
-        d_beta->copy_from(beta.get());
-        d_prev_rho = Mtx::create(cuda);
-        d_prev_rho->copy_from(prev_rho.get());
-        d_rho_t = Mtx::create(cuda);
-        d_rho_t->copy_from(rho_t.get());
-        d_rho = Mtx::create(cuda);
-        d_rho->copy_from(rho.get());
-        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(cuda, n));
-        *d_stop_status = *stop_status;
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::CudaExecutor> cuda;
-
-    std::ranlux48 rand_engine;
-
-    std::unique_ptr<Mtx> b;
-    std::unique_ptr<Mtx> r;
-    std::unique_ptr<Mtx> t;
-    std::unique_ptr<Mtx> z;
-    std::unique_ptr<Mtx> p;
-    std::unique_ptr<Mtx> q;
-    std::unique_ptr<Mtx> x;
-    std::unique_ptr<Mtx> beta;
-    std::unique_ptr<Mtx> prev_rho;
-    std::unique_ptr<Mtx> rho;
-    std::unique_ptr<Mtx> rho_t;
-    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
-
-    std::unique_ptr<Mtx> d_b;
-    std::unique_ptr<Mtx> d_r;
-    std::unique_ptr<Mtx> d_t;
-    std::unique_ptr<Mtx> d_z;
-    std::unique_ptr<Mtx> d_p;
-    std::unique_ptr<Mtx> d_q;
-    std::unique_ptr<Mtx> d_x;
-    std::unique_ptr<Mtx> d_beta;
-    std::unique_ptr<Mtx> d_prev_rho;
-    std::unique_ptr<Mtx> d_rho;
-    std::unique_ptr<Mtx> d_rho_t;
-    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
-};
-
-
-TEST_F(Fcg, CudaFcgInitializeIsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::fcg::initialize(
-        ref, b.get(), r.get(), z.get(), p.get(), q.get(), t.get(),
-        prev_rho.get(), rho.get(), rho_t.get(), stop_status.get());
-    gko::kernels::cuda::fcg::initialize(
-        cuda, d_b.get(), d_r.get(), d_z.get(), d_p.get(), d_q.get(), d_t.get(),
-        d_prev_rho.get(), d_rho.get(), d_rho_t.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_t, t, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho, rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho_t, rho_t, 1e-14);
-    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
-}
-
-
-TEST_F(Fcg, CudaFcgStep1IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::fcg::step_1(ref, p.get(), z.get(), rho_t.get(),
-                                         prev_rho.get(), stop_status.get());
-    gko::kernels::cuda::fcg::step_1(cuda, d_p.get(), d_z.get(), d_rho_t.get(),
-                                    d_prev_rho.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-}
-
-
-TEST_F(Fcg, CudaFcgStep2IsEquivalentToRef)
-{
-    initialize_data();
-    gko::kernels::reference::fcg::step_2(ref, x.get(), r.get(), t.get(),
-                                         p.get(), q.get(), beta.get(),
-                                         rho.get(), stop_status.get());
-    gko::kernels::cuda::fcg::step_2(cuda, d_x.get(), d_r.get(), d_t.get(),
-                                    d_p.get(), d_q.get(), d_beta.get(),
-                                    d_rho.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_t, t, 1e-14);
-}
-
-
-TEST_F(Fcg, ApplyIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    gko::test::make_hpd(mtx.get());
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = Mtx::create(cuda);
-    d_mtx->copy_from(mtx.get());
-    auto d_x = Mtx::create(cuda);
-    d_x->copy_from(x.get());
-    auto d_b = Mtx::create(cuda);
-    d_b->copy_from(b.get());
-    auto fcg_factory =
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(ref))
-            .on(ref);
-    auto d_fcg_factory =
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(cuda),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(cuda))
-            .on(cuda);
-    auto solver = fcg_factory->generate(std::move(mtx));
-    auto d_solver = d_fcg_factory->generate(std::move(d_mtx));
-
-    solver->apply(b.get(), x.get());
-    d_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-}  // namespace
diff --git a/cuda/test/solver/ir_kernels.cpp b/cuda/test/solver/ir_kernels.cpp
deleted file mode 100644
index 88e89456128..00000000000
--- a/cuda/test/solver/ir_kernels.cpp
+++ /dev/null
@@ -1,259 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/solver/ir.hpp>
-
-
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/solver/gmres.hpp>
-#include <ginkgo/core/stop/combined.hpp>
-#include <ginkgo/core/stop/iteration.hpp>
-
-
-#include "core/solver/ir_kernels.hpp"
-#include "cuda/test/utils.hpp"
-
-
-namespace {
-
-
-class Ir : public ::testing::Test {
-protected:
-    using Mtx = gko::matrix::Dense<>;
-    Ir() : rand_engine(30) {}
-
-    void SetUp()
-    {
-        ref = gko::ReferenceExecutor::create();
-        cuda = gko::CudaExecutor::create(0, ref);
-    }
-
-    void TearDown()
-    {
-        if (cuda != nullptr) {
-            ASSERT_NO_THROW(cuda->synchronize());
-        }
-    }
-
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
-    {
-        return gko::test::generate_random_matrix<Mtx>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::CudaExecutor> cuda;
-
-    std::ranlux48 rand_engine;
-};
-
-
-TEST_F(Ir, InitializeIsEquivalentToRef)
-{
-    auto stop_status = gko::Array<gko::stopping_status>(ref, 43);
-    for (size_t i = 0; i < stop_status.get_num_elems(); ++i) {
-        stop_status.get_data()[i].reset();
-    }
-    auto d_stop_status = gko::Array<gko::stopping_status>(cuda, stop_status);
-
-    gko::kernels::reference::ir::initialize(ref, &stop_status);
-    gko::kernels::cuda::ir::initialize(cuda, &d_stop_status);
-
-    auto tmp = gko::Array<gko::stopping_status>(ref, d_stop_status);
-    for (int i = 0; i < stop_status.get_num_elems(); ++i) {
-        ASSERT_EQ(stop_status.get_const_data()[i], tmp.get_const_data()[i]);
-    }
-}
-
-
-TEST_F(Ir, ApplyIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = clone(cuda, mtx);
-    auto d_x = clone(cuda, x);
-    auto d_b = clone(cuda, b);
-    // Forget about accuracy - Richardson is not going to converge for a random
-    // matrix, just check that a couple of iterations gives the same result on
-    // both executors
-    auto ir_factory =
-        gko::solver::Ir<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(ref))
-            .on(ref);
-    auto d_ir_factory =
-        gko::solver::Ir<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(cuda))
-            .on(cuda);
-    auto solver = ir_factory->generate(std::move(mtx));
-    auto d_solver = d_ir_factory->generate(std::move(d_mtx));
-
-    solver->apply(lend(b), lend(x));
-    d_solver->apply(lend(d_b), lend(d_x));
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = clone(cuda, mtx);
-    auto d_x = clone(cuda, x);
-    auto d_b = clone(cuda, b);
-
-    auto ir_factory =
-        gko::solver::Ir<>::build()
-            .with_solver(
-                gko::solver::Gmres<>::build()
-                    .with_criteria(
-                        gko::stop::Iteration::build().with_max_iters(1u).on(
-                            ref))
-                    .on(ref))
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(ref))
-            .on(ref);
-    auto d_ir_factory =
-        gko::solver::Ir<>::build()
-            .with_solver(
-                gko::solver::Gmres<>::build()
-                    .with_criteria(
-                        gko::stop::Iteration::build().with_max_iters(1u).on(
-                            cuda))
-                    .on(cuda))
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(cuda))
-            .on(cuda);
-    auto solver = ir_factory->generate(std::move(mtx));
-    auto d_solver = d_ir_factory->generate(std::move(d_mtx));
-
-    solver->apply(lend(b), lend(x));
-    d_solver->apply(lend(d_b), lend(d_x));
-
-    // Note: 1e-12 instead of 1e-14, as the difference in the inner gmres
-    // iteration gets amplified by the difference in IR.
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
-}
-
-
-TEST_F(Ir, RichardsonApplyIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = clone(cuda, mtx);
-    auto d_x = clone(cuda, x);
-    auto d_b = clone(cuda, b);
-    // Forget about accuracy - Richardson is not going to converge for a random
-    // matrix, just check that a couple of iterations gives the same result on
-    // both executors
-    auto ir_factory =
-        gko::solver::Ir<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(ref))
-            .with_relaxation_factor(0.9)
-            .on(ref);
-    auto d_ir_factory =
-        gko::solver::Ir<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(cuda))
-            .with_relaxation_factor(0.9)
-            .on(cuda);
-    auto solver = ir_factory->generate(std::move(mtx));
-    auto d_solver = d_ir_factory->generate(std::move(d_mtx));
-
-    solver->apply(lend(b), lend(x));
-    d_solver->apply(lend(d_b), lend(d_x));
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-TEST_F(Ir, RichardsonApplyWithIterativeInnerSolverIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = clone(cuda, mtx);
-    auto d_x = clone(cuda, x);
-    auto d_b = clone(cuda, b);
-    auto ir_factory =
-        gko::solver::Ir<>::build()
-            .with_solver(
-                gko::solver::Gmres<>::build()
-                    .with_criteria(
-                        gko::stop::Iteration::build().with_max_iters(1u).on(
-                            ref))
-                    .on(ref))
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(ref))
-            .with_relaxation_factor(0.9)
-            .on(ref);
-    auto d_ir_factory =
-        gko::solver::Ir<>::build()
-            .with_solver(
-                gko::solver::Gmres<>::build()
-                    .with_criteria(
-                        gko::stop::Iteration::build().with_max_iters(1u).on(
-                            cuda))
-                    .on(cuda))
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(cuda))
-            .with_relaxation_factor(0.9)
-            .on(cuda);
-    auto solver = ir_factory->generate(std::move(mtx));
-    auto d_solver = d_ir_factory->generate(std::move(d_mtx));
-
-    solver->apply(lend(b), lend(x));
-    d_solver->apply(lend(d_b), lend(d_x));
-
-    // Note: 1e-12 instead of 1e-14, as the difference in the inner gmres
-    // iteration gets amplified by the difference in IR.
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
-}
-
-
-}  // namespace
diff --git a/hip/test/solver/CMakeLists.txt b/hip/test/solver/CMakeLists.txt
index a69af95bd22..29454e2b5e7 100644
--- a/hip/test/solver/CMakeLists.txt
+++ b/hip/test/solver/CMakeLists.txt
@@ -1,11 +1,5 @@
-ginkgo_create_test(bicg_kernels)
-ginkgo_create_test(bicgstab_kernels)
-ginkgo_create_test(cg_kernels)
-ginkgo_create_test(cgs_kernels)
-ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
 ginkgo_create_test(cb_gmres_kernels)
 ginkgo_create_test(idr_kernels)
-ginkgo_create_test(ir_kernels)
 ginkgo_create_test(lower_trs_kernels)
 ginkgo_create_test(upper_trs_kernels)
diff --git a/hip/test/solver/bicg_kernels.cpp b/hip/test/solver/bicg_kernels.cpp
deleted file mode 100644
index d8510e46145..00000000000
--- a/hip/test/solver/bicg_kernels.cpp
+++ /dev/null
@@ -1,330 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/solver/bicg.hpp>
-
-
-#include <fstream>
-#include <random>
-#include <string>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/stop/combined.hpp>
-#include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm.hpp>
-
-
-#include "core/solver/bicg_kernels.hpp"
-#include "hip/test/utils.hip.hpp"
-#include "matrices/config.hpp"
-
-
-namespace {
-
-
-class Bicg : public ::testing::Test {
-protected:
-    using value_type = gko::default_precision;
-    using index_type = gko::int32;
-    using Mtx = gko::matrix::Dense<>;
-    using Csr = gko::matrix::Csr<value_type, index_type>;
-    Bicg() : rand_engine(30) {}
-
-    void SetUp()
-    {
-        ASSERT_GT(gko::HipExecutor::get_num_devices(), 0);
-        ref = gko::ReferenceExecutor::create();
-        hip = gko::HipExecutor::create(0, ref);
-
-        std::string file_name(gko::matrices::location_ani1_mtx);
-        auto input_file = std::ifstream(file_name, std::ios::in);
-        if (!input_file) {
-            FAIL() << "Could not find the file \"" << file_name
-                   << "\", which is required for this test.\n";
-        }
-        csr_ref = gko::read<Csr>(input_file, ref);
-        auto csr_hip_temp = Csr::create(hip);
-        csr_hip_temp->copy_from(gko::lend(csr_ref));
-        csr_hip = gko::give(csr_hip_temp);
-    }
-
-    void TearDown()
-    {
-        if (hip != nullptr) {
-            ASSERT_NO_THROW(hip->synchronize());
-        }
-    }
-
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
-    {
-        return gko::test::generate_random_matrix<Mtx>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
-    }
-
-    void initialize_data()
-    {
-        int m = 597;
-        int n = 43;
-        b = gen_mtx(m, n);
-        r = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        q = gen_mtx(m, n);
-        r2 = gen_mtx(m, n);
-        z2 = gen_mtx(m, n);
-        p2 = gen_mtx(m, n);
-        q2 = gen_mtx(m, n);
-        x = gen_mtx(m, n);
-        beta = gen_mtx(1, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(ref, n));
-        for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
-            stop_status->get_data()[i].reset();
-        }
-
-        d_b = Mtx::create(hip);
-        d_b->copy_from(b.get());
-        d_r = Mtx::create(hip);
-        d_r->copy_from(r.get());
-        d_z = Mtx::create(hip);
-        d_z->copy_from(z.get());
-        d_p = Mtx::create(hip);
-        d_p->copy_from(p.get());
-        d_q = Mtx::create(hip);
-        d_q->copy_from(q.get());
-        d_r2 = Mtx::create(hip);
-        d_r2->copy_from(r2.get());
-        d_z2 = Mtx::create(hip);
-        d_z2->copy_from(z2.get());
-        d_p2 = Mtx::create(hip);
-        d_p2->copy_from(p2.get());
-        d_q2 = Mtx::create(hip);
-        d_q2->copy_from(q2.get());
-        d_x = Mtx::create(hip);
-        d_x->copy_from(x.get());
-        d_beta = Mtx::create(hip);
-        d_beta->copy_from(beta.get());
-        d_prev_rho = Mtx::create(hip);
-        d_prev_rho->copy_from(prev_rho.get());
-        d_rho = Mtx::create(hip);
-        d_rho->copy_from(rho.get());
-        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(hip, n));
-        *d_stop_status = *stop_status;
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::HipExecutor> hip;
-
-    std::ranlux48 rand_engine;
-
-    std::unique_ptr<Mtx> b;
-    std::unique_ptr<Mtx> r;
-    std::unique_ptr<Mtx> z;
-    std::unique_ptr<Mtx> p;
-    std::unique_ptr<Mtx> q;
-    std::unique_ptr<Mtx> r2;
-    std::unique_ptr<Mtx> z2;
-    std::unique_ptr<Mtx> p2;
-    std::unique_ptr<Mtx> q2;
-    std::unique_ptr<Mtx> x;
-    std::unique_ptr<Mtx> beta;
-    std::unique_ptr<Mtx> prev_rho;
-    std::unique_ptr<Mtx> rho;
-    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
-
-    std::unique_ptr<Mtx> d_b;
-    std::unique_ptr<Mtx> d_r;
-    std::unique_ptr<Mtx> d_z;
-    std::unique_ptr<Mtx> d_p;
-    std::unique_ptr<Mtx> d_q;
-    std::unique_ptr<Mtx> d_r2;
-    std::unique_ptr<Mtx> d_z2;
-    std::unique_ptr<Mtx> d_p2;
-    std::unique_ptr<Mtx> d_q2;
-    std::unique_ptr<Mtx> d_x;
-    std::unique_ptr<Mtx> d_beta;
-    std::unique_ptr<Mtx> d_prev_rho;
-    std::unique_ptr<Mtx> d_rho;
-    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
-    std::shared_ptr<const Csr> csr_ref;
-    std::shared_ptr<const Csr> csr_hip;
-};
-
-
-TEST_F(Bicg, HipBicgInitializeIsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::bicg::initialize(
-        ref, b.get(), r.get(), z.get(), p.get(), q.get(), prev_rho.get(),
-        rho.get(), r2.get(), z2.get(), p2.get(), q2.get(), stop_status.get());
-    gko::kernels::hip::bicg::initialize(
-        hip, d_b.get(), d_r.get(), d_z.get(), d_p.get(), d_q.get(),
-        d_prev_rho.get(), d_rho.get(), d_r2.get(), d_z2.get(), d_p2.get(),
-        d_q2.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r2, r2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z2, z2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p2, p2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q2, q2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho, rho, 1e-14);
-    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
-}
-
-
-TEST_F(Bicg, HipBicgStep1IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::bicg::step_1(ref, p.get(), z.get(), p2.get(),
-                                          z2.get(), rho.get(), prev_rho.get(),
-                                          stop_status.get());
-    gko::kernels::hip::bicg::step_1(hip, d_p.get(), d_z.get(), d_p2.get(),
-                                    d_z2.get(), d_rho.get(), d_prev_rho.get(),
-                                    d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p2, p2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z2, z2, 1e-14);
-}
-
-
-TEST_F(Bicg, HipBicgStep2IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::bicg::step_2(
-        ref, x.get(), r.get(), r2.get(), p.get(), q.get(), q2.get(), beta.get(),
-        rho.get(), stop_status.get());
-    gko::kernels::hip::bicg::step_2(
-        hip, d_x.get(), d_r.get(), d_r2.get(), d_p.get(), d_q.get(), d_q2.get(),
-        d_beta.get(), d_rho.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r2, r2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q2, q2, 1e-14);
-}
-
-
-TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    gko::test::make_hpd(mtx.get());
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = Mtx::create(hip);
-    d_mtx->copy_from(mtx.get());
-    auto d_x = Mtx::create(hip);
-    d_x->copy_from(x.get());
-    auto d_b = Mtx::create(hip);
-    d_b->copy_from(b.get());
-    auto bicg_factory =
-        gko::solver::Bicg<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(ref))
-            .on(ref);
-    auto d_bicg_factory =
-        gko::solver::Bicg<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(hip),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(hip))
-            .on(hip);
-    auto solver = bicg_factory->generate(std::move(mtx));
-    auto d_solver = d_bicg_factory->generate(std::move(d_mtx));
-
-    solver->apply(b.get(), x.get());
-    d_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
-{
-    auto x = gen_mtx(36, 1);
-    auto b = gen_mtx(36, 1);
-    auto d_x = Mtx::create(hip);
-    d_x->copy_from(x.get());
-    auto d_b = Mtx::create(hip);
-    d_b->copy_from(b.get());
-    auto bicg_factory =
-        gko::solver::Bicg<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(ref))
-            .on(ref);
-    auto d_bicg_factory =
-        gko::solver::Bicg<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(hip),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(hip))
-            .on(hip);
-    auto solver = bicg_factory->generate(std::move(csr_ref));
-    auto d_solver = d_bicg_factory->generate(std::move(csr_hip));
-
-    solver->apply(b.get(), x.get());
-    d_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-}  // namespace
diff --git a/hip/test/solver/bicgstab_kernels.cpp b/hip/test/solver/bicgstab_kernels.cpp
deleted file mode 100644
index 5a995656306..00000000000
--- a/hip/test/solver/bicgstab_kernels.cpp
+++ /dev/null
@@ -1,345 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/solver/bicgstab.hpp>
-
-
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/stop/combined.hpp>
-#include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm.hpp>
-
-
-#include "core/solver/bicgstab_kernels.hpp"
-#include "hip/test/utils.hip.hpp"
-
-
-namespace {
-
-
-class Bicgstab : public ::testing::Test {
-protected:
-    using Mtx = gko::matrix::Dense<>;
-    using Solver = gko::solver::Bicgstab<>;
-
-    Bicgstab() : rand_engine(30) {}
-
-    void SetUp()
-    {
-        ASSERT_GT(gko::HipExecutor::get_num_devices(), 0);
-        ref = gko::ReferenceExecutor::create();
-        hip = gko::HipExecutor::create(0, ref);
-
-        mtx = gen_mtx(123, 123);
-        gko::test::make_diag_dominant(mtx.get());
-        d_mtx = Mtx::create(hip);
-        d_mtx->copy_from(mtx.get());
-
-        hip_bicgstab_factory =
-            Solver::build()
-                .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(hip),
-                    gko::stop::ResidualNorm<>::build()
-                        .with_reduction_factor(1e-15)
-                        .on(hip))
-                .on(hip);
-        ref_bicgstab_factory =
-            Solver::build()
-                .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNorm<>::build()
-                        .with_reduction_factor(1e-15)
-                        .on(ref))
-                .on(ref);
-    }
-
-    void TearDown()
-    {
-        if (hip != nullptr) {
-            ASSERT_NO_THROW(hip->synchronize());
-        }
-    }
-
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
-    {
-        return gko::test::generate_random_matrix<Mtx>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
-    }
-
-    void initialize_data()
-    {
-        int m = 597;
-        int n = 17;
-        x = gen_mtx(m, n);
-        b = gen_mtx(m, n);
-        r = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        rr = gen_mtx(m, n);
-        s = gen_mtx(m, n);
-        t = gen_mtx(m, n);
-        y = gen_mtx(m, n);
-        v = gen_mtx(m, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        alpha = gen_mtx(1, n);
-        beta = gen_mtx(1, n);
-        gamma = gen_mtx(1, n);
-        omega = gen_mtx(1, n);
-        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(ref, n));
-        for (size_t i = 0; i < n; ++i) {
-            stop_status->get_data()[i].reset();
-        }
-
-        d_x = Mtx::create(hip);
-        d_b = Mtx::create(hip);
-        d_r = Mtx::create(hip);
-        d_z = Mtx::create(hip);
-        d_p = Mtx::create(hip);
-        d_t = Mtx::create(hip);
-        d_s = Mtx::create(hip);
-        d_y = Mtx::create(hip);
-        d_v = Mtx::create(hip);
-        d_rr = Mtx::create(hip);
-        d_prev_rho = Mtx::create(hip);
-        d_rho = Mtx::create(hip);
-        d_alpha = Mtx::create(hip);
-        d_beta = Mtx::create(hip);
-        d_gamma = Mtx::create(hip);
-        d_omega = Mtx::create(hip);
-        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(hip));
-
-        d_x->copy_from(x.get());
-        d_b->copy_from(b.get());
-        d_r->copy_from(r.get());
-        d_z->copy_from(z.get());
-        d_p->copy_from(p.get());
-        d_v->copy_from(v.get());
-        d_y->copy_from(y.get());
-        d_t->copy_from(t.get());
-        d_s->copy_from(s.get());
-        d_rr->copy_from(rr.get());
-        d_prev_rho->copy_from(prev_rho.get());
-        d_rho->copy_from(rho.get());
-        d_alpha->copy_from(alpha.get());
-        d_beta->copy_from(beta.get());
-        d_gamma->copy_from(gamma.get());
-        d_omega->copy_from(omega.get());
-        *d_stop_status =
-            *stop_status;  // copy_from is not a public member function of Array
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::HipExecutor> hip;
-
-    std::ranlux48 rand_engine;
-
-    std::shared_ptr<Mtx> mtx;
-    std::shared_ptr<Mtx> d_mtx;
-    std::unique_ptr<Solver::Factory> hip_bicgstab_factory;
-    std::unique_ptr<Solver::Factory> ref_bicgstab_factory;
-
-    std::unique_ptr<Mtx> x;
-    std::unique_ptr<Mtx> b;
-    std::unique_ptr<Mtx> r;
-    std::unique_ptr<Mtx> z;
-    std::unique_ptr<Mtx> p;
-    std::unique_ptr<Mtx> rr;
-    std::unique_ptr<Mtx> s;
-    std::unique_ptr<Mtx> t;
-    std::unique_ptr<Mtx> y;
-    std::unique_ptr<Mtx> v;
-    std::unique_ptr<Mtx> prev_rho;
-    std::unique_ptr<Mtx> rho;
-    std::unique_ptr<Mtx> alpha;
-    std::unique_ptr<Mtx> beta;
-    std::unique_ptr<Mtx> gamma;
-    std::unique_ptr<Mtx> omega;
-    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
-
-    std::unique_ptr<Mtx> d_x;
-    std::unique_ptr<Mtx> d_b;
-    std::unique_ptr<Mtx> d_r;
-    std::unique_ptr<Mtx> d_z;
-    std::unique_ptr<Mtx> d_p;
-    std::unique_ptr<Mtx> d_t;
-    std::unique_ptr<Mtx> d_s;
-    std::unique_ptr<Mtx> d_y;
-    std::unique_ptr<Mtx> d_v;
-    std::unique_ptr<Mtx> d_rr;
-    std::unique_ptr<Mtx> d_prev_rho;
-    std::unique_ptr<Mtx> d_rho;
-    std::unique_ptr<Mtx> d_alpha;
-    std::unique_ptr<Mtx> d_beta;
-    std::unique_ptr<Mtx> d_gamma;
-    std::unique_ptr<Mtx> d_omega;
-    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
-};
-
-
-TEST_F(Bicgstab, HipBicgstabInitializeIsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::bicgstab::initialize(
-        ref, b.get(), r.get(), rr.get(), y.get(), s.get(), t.get(), z.get(),
-        v.get(), p.get(), prev_rho.get(), rho.get(), alpha.get(), beta.get(),
-        gamma.get(), omega.get(), stop_status.get());
-    gko::kernels::hip::bicgstab::initialize(
-        hip, d_b.get(), d_r.get(), d_rr.get(), d_y.get(), d_s.get(), d_t.get(),
-        d_z.get(), d_v.get(), d_p.get(), d_prev_rho.get(), d_rho.get(),
-        d_alpha.get(), d_beta.get(), d_gamma.get(), d_omega.get(),
-        d_stop_status.get());
-
-    GKO_EXPECT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_y, y, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_t, t, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_s, s, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_rr, rr, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_v, v, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_rho, rho, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_alpha, alpha, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_beta, beta, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_gamma, gamma, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_omega, omega, 1e-14);
-    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
-}
-
-
-TEST_F(Bicgstab, HipBicgstabStep1IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::bicgstab::step_1(
-        ref, r.get(), p.get(), v.get(), rho.get(), prev_rho.get(), alpha.get(),
-        omega.get(), stop_status.get());
-    gko::kernels::hip::bicgstab::step_1(
-        hip, d_r.get(), d_p.get(), d_v.get(), d_rho.get(), d_prev_rho.get(),
-        d_alpha.get(), d_omega.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-}
-
-
-TEST_F(Bicgstab, HipBicgstabStep2IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::bicgstab::step_2(ref, r.get(), s.get(), v.get(),
-                                              rho.get(), alpha.get(),
-                                              beta.get(), stop_status.get());
-    gko::kernels::hip::bicgstab::step_2(hip, d_r.get(), d_s.get(), d_v.get(),
-                                        d_rho.get(), d_alpha.get(),
-                                        d_beta.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_s, s, 1e-14);
-}
-
-
-TEST_F(Bicgstab, HipBicgstabStep3IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::bicgstab::step_3(
-        ref, x.get(), r.get(), s.get(), t.get(), y.get(), z.get(), alpha.get(),
-        beta.get(), gamma.get(), omega.get(), stop_status.get());
-    gko::kernels::hip::bicgstab::step_3(
-        hip, d_x.get(), d_r.get(), d_s.get(), d_t.get(), d_y.get(), d_z.get(),
-        d_alpha.get(), d_beta.get(), d_gamma.get(), d_omega.get(),
-        d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_omega, omega, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-}
-
-
-TEST_F(Bicgstab, HipBicgstabApplyOneRHSIsEquivalentToRef)
-{
-    int m = 123;
-    int n = 1;
-    auto ref_solver = ref_bicgstab_factory->generate(mtx);
-    auto hip_solver = hip_bicgstab_factory->generate(d_mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(hip);
-    auto d_x = Mtx::create(hip);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
-
-    ref_solver->apply(b.get(), x.get());
-    hip_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-}
-
-
-TEST_F(Bicgstab, HipBicgstabApplyMultipleRHSIsEquivalentToRef)
-{
-    int m = 123;
-    int n = 16;
-    auto hip_solver = hip_bicgstab_factory->generate(d_mtx);
-    auto ref_solver = ref_bicgstab_factory->generate(mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(hip);
-    auto d_x = Mtx::create(hip);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
-
-    ref_solver->apply(b.get(), x.get());
-    hip_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-}
-
-
-}  // namespace
diff --git a/hip/test/solver/cg_kernels.cpp b/hip/test/solver/cg_kernels.cpp
deleted file mode 100644
index a93c20a3287..00000000000
--- a/hip/test/solver/cg_kernels.cpp
+++ /dev/null
@@ -1,245 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/solver/cg.hpp>
-
-
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/stop/combined.hpp>
-#include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm.hpp>
-
-
-#include "core/solver/cg_kernels.hpp"
-#include "hip/test/utils.hip.hpp"
-
-
-namespace {
-
-
-class Cg : public ::testing::Test {
-protected:
-    using Mtx = gko::matrix::Dense<>;
-    Cg() : rand_engine(30) {}
-
-    void SetUp()
-    {
-        ASSERT_GT(gko::HipExecutor::get_num_devices(), 0);
-        ref = gko::ReferenceExecutor::create();
-        hip = gko::HipExecutor::create(0, ref);
-    }
-
-    void TearDown()
-    {
-        if (hip != nullptr) {
-            ASSERT_NO_THROW(hip->synchronize());
-        }
-    }
-
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
-    {
-        return gko::test::generate_random_matrix<Mtx>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
-    }
-
-    void initialize_data()
-    {
-        int m = 597;
-        int n = 43;
-        b = gen_mtx(m, n);
-        r = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        q = gen_mtx(m, n);
-        x = gen_mtx(m, n);
-        beta = gen_mtx(1, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(ref, n));
-        for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
-            stop_status->get_data()[i].reset();
-        }
-
-        d_b = Mtx::create(hip);
-        d_b->copy_from(b.get());
-        d_r = Mtx::create(hip);
-        d_r->copy_from(r.get());
-        d_z = Mtx::create(hip);
-        d_z->copy_from(z.get());
-        d_p = Mtx::create(hip);
-        d_p->copy_from(p.get());
-        d_q = Mtx::create(hip);
-        d_q->copy_from(q.get());
-        d_x = Mtx::create(hip);
-        d_x->copy_from(x.get());
-        d_beta = Mtx::create(hip);
-        d_beta->copy_from(beta.get());
-        d_prev_rho = Mtx::create(hip);
-        d_prev_rho->copy_from(prev_rho.get());
-        d_rho = Mtx::create(hip);
-        d_rho->copy_from(rho.get());
-        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(hip, n));
-        *d_stop_status = *stop_status;
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::HipExecutor> hip;
-
-    std::ranlux48 rand_engine;
-
-    std::unique_ptr<Mtx> b;
-    std::unique_ptr<Mtx> r;
-    std::unique_ptr<Mtx> z;
-    std::unique_ptr<Mtx> p;
-    std::unique_ptr<Mtx> q;
-    std::unique_ptr<Mtx> x;
-    std::unique_ptr<Mtx> beta;
-    std::unique_ptr<Mtx> prev_rho;
-    std::unique_ptr<Mtx> rho;
-    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
-
-    std::unique_ptr<Mtx> d_b;
-    std::unique_ptr<Mtx> d_r;
-    std::unique_ptr<Mtx> d_z;
-    std::unique_ptr<Mtx> d_p;
-    std::unique_ptr<Mtx> d_q;
-    std::unique_ptr<Mtx> d_x;
-    std::unique_ptr<Mtx> d_beta;
-    std::unique_ptr<Mtx> d_prev_rho;
-    std::unique_ptr<Mtx> d_rho;
-    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
-};
-
-
-TEST_F(Cg, HipCgInitializeIsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::cg::initialize(ref, b.get(), r.get(), z.get(),
-                                            p.get(), q.get(), prev_rho.get(),
-                                            rho.get(), stop_status.get());
-    gko::kernels::hip::cg::initialize(hip, d_b.get(), d_r.get(), d_z.get(),
-                                      d_p.get(), d_q.get(), d_prev_rho.get(),
-                                      d_rho.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho, rho, 1e-14);
-    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
-}
-
-
-TEST_F(Cg, HipCgStep1IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::cg::step_1(ref, p.get(), z.get(), rho.get(),
-                                        prev_rho.get(), stop_status.get());
-    gko::kernels::hip::cg::step_1(hip, d_p.get(), d_z.get(), d_rho.get(),
-                                  d_prev_rho.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-}
-
-
-TEST_F(Cg, HipCgStep2IsEquivalentToRef)
-{
-    initialize_data();
-    gko::kernels::reference::cg::step_2(ref, x.get(), r.get(), p.get(), q.get(),
-                                        beta.get(), rho.get(),
-                                        stop_status.get());
-    gko::kernels::hip::cg::step_2(hip, d_x.get(), d_r.get(), d_p.get(),
-                                  d_q.get(), d_beta.get(), d_rho.get(),
-                                  d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-}
-
-
-TEST_F(Cg, ApplyIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    gko::test::make_hpd(mtx.get());
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = Mtx::create(hip);
-    d_mtx->copy_from(mtx.get());
-    auto d_x = Mtx::create(hip);
-    d_x->copy_from(x.get());
-    auto d_b = Mtx::create(hip);
-    d_b->copy_from(b.get());
-    auto cg_factory =
-        gko::solver::Cg<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(ref))
-            .on(ref);
-    auto d_cg_factory =
-        gko::solver::Cg<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(hip),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(hip))
-            .on(hip);
-    auto solver = cg_factory->generate(std::move(mtx));
-    auto d_solver = d_cg_factory->generate(std::move(d_mtx));
-
-    solver->apply(b.get(), x.get());
-    d_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-}  // namespace
diff --git a/hip/test/solver/cgs_kernels.cpp b/hip/test/solver/cgs_kernels.cpp
deleted file mode 100644
index 2277d4417ba..00000000000
--- a/hip/test/solver/cgs_kernels.cpp
+++ /dev/null
@@ -1,337 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/solver/cgs.hpp>
-
-
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/stop/combined.hpp>
-#include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm.hpp>
-
-
-#include "core/solver/cgs_kernels.hpp"
-#include "hip/test/utils.hip.hpp"
-
-
-namespace {
-
-
-class Cgs : public ::testing::Test {
-protected:
-    using Mtx = gko::matrix::Dense<>;
-    using Solver = gko::solver::Cgs<>;
-
-    Cgs() : rand_engine(30) {}
-
-    void SetUp()
-    {
-        ASSERT_GT(gko::HipExecutor::get_num_devices(), 0);
-        ref = gko::ReferenceExecutor::create();
-        hip = gko::HipExecutor::create(0, ref);
-
-        mtx = gen_mtx(123, 123);
-        gko::test::make_diag_dominant(mtx.get());
-        d_mtx = Mtx::create(hip);
-        d_mtx->copy_from(mtx.get());
-        hip_cgs_factory =
-            Solver::build()
-                .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(hip),
-                    gko::stop::ResidualNorm<>::build()
-                        .with_reduction_factor(1e-15)
-                        .on(hip))
-                .on(hip);
-        ref_cgs_factory =
-            Solver::build()
-                .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNorm<>::build()
-                        .with_reduction_factor(1e-15)
-                        .on(ref))
-                .on(ref);
-    }
-
-    void TearDown()
-    {
-        if (hip != nullptr) {
-            ASSERT_NO_THROW(hip->synchronize());
-        }
-    }
-
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
-    {
-        return gko::test::generate_random_matrix<Mtx>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
-    }
-
-    void initialize_data()
-    {
-        int m = 597;
-        int n = 43;
-        b = gen_mtx(m, n);
-        r = gen_mtx(m, n);
-        r_tld = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        q = gen_mtx(m, n);
-        u = gen_mtx(m, n);
-        u_hat = gen_mtx(m, n);
-        v_hat = gen_mtx(m, n);
-        t = gen_mtx(m, n);
-        x = gen_mtx(m, n);
-        alpha = gen_mtx(1, n);
-        beta = gen_mtx(1, n);
-        gamma = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        rho_prev = gen_mtx(1, n);
-        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(ref, n));
-        for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
-            stop_status->get_data()[i].reset();
-        }
-
-        d_b = Mtx::create(hip);
-        d_b->copy_from(b.get());
-        d_r = Mtx::create(hip);
-        d_r->copy_from(r.get());
-        d_r_tld = Mtx::create(hip);
-        d_r_tld->copy_from(r_tld.get());
-        d_p = Mtx::create(hip);
-        d_p->copy_from(p.get());
-        d_q = Mtx::create(hip);
-        d_q->copy_from(q.get());
-        d_u = Mtx::create(hip);
-        d_u->copy_from(u.get());
-        d_u_hat = Mtx::create(hip);
-        d_u_hat->copy_from(u_hat.get());
-        d_v_hat = Mtx::create(hip);
-        d_v_hat->copy_from(v_hat.get());
-        d_t = Mtx::create(hip);
-        d_t->copy_from(t.get());
-        d_x = Mtx::create(hip);
-        d_x->copy_from(x.get());
-        d_alpha = Mtx::create(hip);
-        d_alpha->copy_from(alpha.get());
-        d_beta = Mtx::create(hip);
-        d_beta->copy_from(beta.get());
-        d_gamma = Mtx::create(hip);
-        d_gamma->copy_from(gamma.get());
-        d_rho_prev = Mtx::create(hip);
-        d_rho_prev->copy_from(rho_prev.get());
-        d_rho = Mtx::create(hip);
-        d_rho->copy_from(rho.get());
-        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(hip, n));
-        // because there is no public function copy_from, use overloaded =
-        // operator
-        *d_stop_status = *stop_status;
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::HipExecutor> hip;
-
-    std::ranlux48 rand_engine;
-
-    std::shared_ptr<Mtx> mtx;
-    std::shared_ptr<Mtx> d_mtx;
-    std::unique_ptr<Solver::Factory> hip_cgs_factory;
-    std::unique_ptr<Solver::Factory> ref_cgs_factory;
-
-    std::unique_ptr<Mtx> b;
-    std::unique_ptr<Mtx> r;
-    std::unique_ptr<Mtx> r_tld;
-    std::unique_ptr<Mtx> t;
-    std::unique_ptr<Mtx> p;
-    std::unique_ptr<Mtx> q;
-    std::unique_ptr<Mtx> u;
-    std::unique_ptr<Mtx> u_hat;
-    std::unique_ptr<Mtx> v_hat;
-    std::unique_ptr<Mtx> x;
-    std::unique_ptr<Mtx> alpha;
-    std::unique_ptr<Mtx> beta;
-    std::unique_ptr<Mtx> gamma;
-    std::unique_ptr<Mtx> rho;
-    std::unique_ptr<Mtx> rho_prev;
-    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
-
-    std::unique_ptr<Mtx> d_b;
-    std::unique_ptr<Mtx> d_r;
-    std::unique_ptr<Mtx> d_r_tld;
-    std::unique_ptr<Mtx> d_t;
-    std::unique_ptr<Mtx> d_p;
-    std::unique_ptr<Mtx> d_q;
-    std::unique_ptr<Mtx> d_u;
-    std::unique_ptr<Mtx> d_u_hat;
-    std::unique_ptr<Mtx> d_v_hat;
-    std::unique_ptr<Mtx> d_x;
-    std::unique_ptr<Mtx> d_alpha;
-    std::unique_ptr<Mtx> d_beta;
-    std::unique_ptr<Mtx> d_gamma;
-    std::unique_ptr<Mtx> d_rho;
-    std::unique_ptr<Mtx> d_rho_prev;
-    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
-};
-
-
-TEST_F(Cgs, HipCgsInitializeIsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::cgs::initialize(
-        ref, b.get(), r.get(), r_tld.get(), p.get(), q.get(), u.get(),
-        u_hat.get(), v_hat.get(), t.get(), alpha.get(), beta.get(), gamma.get(),
-        rho_prev.get(), rho.get(), stop_status.get());
-    gko::kernels::hip::cgs::initialize(
-        hip, d_b.get(), d_r.get(), d_r_tld.get(), d_p.get(), d_q.get(),
-        d_u.get(), d_u_hat.get(), d_v_hat.get(), d_t.get(), d_alpha.get(),
-        d_beta.get(), d_gamma.get(), d_rho_prev.get(), d_rho.get(),
-        d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r_tld, r_tld, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_u, u, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_t, t, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_u_hat, u_hat, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_v_hat, v_hat, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho_prev, rho_prev, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho, rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_beta, beta, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_gamma, gamma, 1e-14);
-    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
-}
-
-
-TEST_F(Cgs, HipCgsStep1IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::cgs::step_1(ref, r.get(), u.get(), p.get(),
-                                         q.get(), beta.get(), rho.get(),
-                                         rho_prev.get(), stop_status.get());
-    gko::kernels::hip::cgs::step_1(hip, d_r.get(), d_u.get(), d_p.get(),
-                                   d_q.get(), d_beta.get(), d_rho.get(),
-                                   d_rho_prev.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_beta, beta, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_u, u, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-}
-
-
-TEST_F(Cgs, HipCgsStep2IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::cgs::step_2(ref, u.get(), v_hat.get(), q.get(),
-                                         t.get(), alpha.get(), rho.get(),
-                                         gamma.get(), stop_status.get());
-    gko::kernels::hip::cgs::step_2(hip, d_u.get(), d_v_hat.get(), d_q.get(),
-                                   d_t.get(), d_alpha.get(), d_rho.get(),
-                                   d_gamma.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_t, t, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-}
-
-
-TEST_F(Cgs, HipCgsStep3IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::cgs::step_3(ref, t.get(), u_hat.get(), r.get(),
-                                         x.get(), alpha.get(),
-                                         stop_status.get());
-    gko::kernels::hip::cgs::step_3(hip, d_t.get(), d_u_hat.get(), d_r.get(),
-                                   d_x.get(), d_alpha.get(),
-                                   d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-}
-
-
-TEST_F(Cgs, HipCgsApplyOneRHSIsEquivalentToRef)
-{
-    int m = 123;
-    int n = 1;
-    auto ref_solver = ref_cgs_factory->generate(mtx);
-    auto hip_solver = hip_cgs_factory->generate(d_mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(hip);
-    auto d_x = Mtx::create(hip);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
-
-    ref_solver->apply(b.get(), x.get());
-    hip_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-}
-
-
-TEST_F(Cgs, HipCgsApplyMultipleRHSIsEquivalentToRef)
-{
-    int m = 123;
-    int n = 16;
-    auto hip_solver = hip_cgs_factory->generate(d_mtx);
-    auto ref_solver = ref_cgs_factory->generate(mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(hip);
-    auto d_x = Mtx::create(hip);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
-
-    ref_solver->apply(b.get(), x.get());
-    hip_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
-}
-
-}  // namespace
diff --git a/hip/test/solver/fcg_kernels.cpp b/hip/test/solver/fcg_kernels.cpp
deleted file mode 100644
index 7a97115a998..00000000000
--- a/hip/test/solver/fcg_kernels.cpp
+++ /dev/null
@@ -1,258 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/solver/fcg.hpp>
-
-
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/stop/combined.hpp>
-#include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm.hpp>
-
-
-#include "core/solver/fcg_kernels.hpp"
-#include "hip/test/utils.hip.hpp"
-
-
-namespace {
-
-
-class Fcg : public ::testing::Test {
-protected:
-    using Mtx = gko::matrix::Dense<>;
-    using Solver = gko::solver::Fcg<>;
-
-    Fcg() : rand_engine(30) {}
-
-    void SetUp()
-    {
-        ASSERT_GT(gko::HipExecutor::get_num_devices(), 0);
-        ref = gko::ReferenceExecutor::create();
-        hip = gko::HipExecutor::create(0, ref);
-    }
-
-    void TearDown()
-    {
-        if (hip != nullptr) {
-            ASSERT_NO_THROW(hip->synchronize());
-        }
-    }
-
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
-    {
-        return gko::test::generate_random_matrix<Mtx>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
-    }
-
-    void initialize_data()
-    {
-        int m = 597;
-        int n = 43;
-        b = gen_mtx(m, n);
-        r = gen_mtx(m, n);
-        t = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        q = gen_mtx(m, n);
-        x = gen_mtx(m, n);
-        beta = gen_mtx(1, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        rho_t = gen_mtx(1, n);
-        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(ref, n));
-        for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
-            stop_status->get_data()[i].reset();
-        }
-
-        d_b = Mtx::create(hip);
-        d_b->copy_from(b.get());
-        d_r = Mtx::create(hip);
-        d_r->copy_from(r.get());
-        d_t = Mtx::create(hip);
-        d_t->copy_from(t.get());
-        d_z = Mtx::create(hip);
-        d_z->copy_from(z.get());
-        d_p = Mtx::create(hip);
-        d_p->copy_from(p.get());
-        d_q = Mtx::create(hip);
-        d_q->copy_from(q.get());
-        d_x = Mtx::create(hip);
-        d_x->copy_from(x.get());
-        d_beta = Mtx::create(hip);
-        d_beta->copy_from(beta.get());
-        d_prev_rho = Mtx::create(hip);
-        d_prev_rho->copy_from(prev_rho.get());
-        d_rho_t = Mtx::create(hip);
-        d_rho_t->copy_from(rho_t.get());
-        d_rho = Mtx::create(hip);
-        d_rho->copy_from(rho.get());
-        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(hip, n));
-        *d_stop_status = *stop_status;
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::HipExecutor> hip;
-
-    std::ranlux48 rand_engine;
-
-    std::unique_ptr<Mtx> b;
-    std::unique_ptr<Mtx> r;
-    std::unique_ptr<Mtx> t;
-    std::unique_ptr<Mtx> z;
-    std::unique_ptr<Mtx> p;
-    std::unique_ptr<Mtx> q;
-    std::unique_ptr<Mtx> x;
-    std::unique_ptr<Mtx> beta;
-    std::unique_ptr<Mtx> prev_rho;
-    std::unique_ptr<Mtx> rho;
-    std::unique_ptr<Mtx> rho_t;
-    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
-
-    std::unique_ptr<Mtx> d_b;
-    std::unique_ptr<Mtx> d_r;
-    std::unique_ptr<Mtx> d_t;
-    std::unique_ptr<Mtx> d_z;
-    std::unique_ptr<Mtx> d_p;
-    std::unique_ptr<Mtx> d_q;
-    std::unique_ptr<Mtx> d_x;
-    std::unique_ptr<Mtx> d_beta;
-    std::unique_ptr<Mtx> d_prev_rho;
-    std::unique_ptr<Mtx> d_rho;
-    std::unique_ptr<Mtx> d_rho_t;
-    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
-};
-
-
-TEST_F(Fcg, HipFcgInitializeIsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::fcg::initialize(
-        ref, b.get(), r.get(), z.get(), p.get(), q.get(), t.get(),
-        prev_rho.get(), rho.get(), rho_t.get(), stop_status.get());
-    gko::kernels::hip::fcg::initialize(
-        hip, d_b.get(), d_r.get(), d_z.get(), d_p.get(), d_q.get(), d_t.get(),
-        d_prev_rho.get(), d_rho.get(), d_rho_t.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_t, t, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho, rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho_t, rho_t, 1e-14);
-    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
-}
-
-
-TEST_F(Fcg, HipFcgStep1IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::fcg::step_1(ref, p.get(), z.get(), rho_t.get(),
-                                         prev_rho.get(), stop_status.get());
-    gko::kernels::hip::fcg::step_1(hip, d_p.get(), d_z.get(), d_rho_t.get(),
-                                   d_prev_rho.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-}
-
-
-TEST_F(Fcg, HipFcgStep2IsEquivalentToRef)
-{
-    initialize_data();
-    gko::kernels::reference::fcg::step_2(ref, x.get(), r.get(), t.get(),
-                                         p.get(), q.get(), beta.get(),
-                                         rho.get(), stop_status.get());
-    gko::kernels::hip::fcg::step_2(hip, d_x.get(), d_r.get(), d_t.get(),
-                                   d_p.get(), d_q.get(), d_beta.get(),
-                                   d_rho.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_t, t, 1e-14);
-}
-
-
-TEST_F(Fcg, ApplyIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    gko::test::make_hpd(mtx.get());
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = Mtx::create(hip);
-    d_mtx->copy_from(mtx.get());
-    auto d_x = Mtx::create(hip);
-    d_x->copy_from(x.get());
-    auto d_b = Mtx::create(hip);
-    d_b->copy_from(b.get());
-    auto fcg_factory =
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(ref))
-            .on(ref);
-    auto d_fcg_factory =
-        Solver::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(hip),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(hip))
-            .on(hip);
-    auto solver = fcg_factory->generate(std::move(mtx));
-    auto d_solver = d_fcg_factory->generate(std::move(d_mtx));
-
-    solver->apply(b.get(), x.get());
-    d_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-}  // namespace
diff --git a/hip/test/solver/ir_kernels.cpp b/hip/test/solver/ir_kernels.cpp
deleted file mode 100644
index 48dd88b89f8..00000000000
--- a/hip/test/solver/ir_kernels.cpp
+++ /dev/null
@@ -1,259 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/solver/ir.hpp>
-
-
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/solver/gmres.hpp>
-#include <ginkgo/core/stop/combined.hpp>
-#include <ginkgo/core/stop/iteration.hpp>
-
-
-#include "core/solver/ir_kernels.hpp"
-#include "hip/test/utils.hip.hpp"
-
-
-namespace {
-
-
-class Ir : public ::testing::Test {
-protected:
-    using Mtx = gko::matrix::Dense<>;
-    Ir() : rand_engine(30) {}
-
-    void SetUp()
-    {
-        ref = gko::ReferenceExecutor::create();
-        hip = gko::HipExecutor::create(0, ref);
-    }
-
-    void TearDown()
-    {
-        if (hip != nullptr) {
-            ASSERT_NO_THROW(hip->synchronize());
-        }
-    }
-
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
-    {
-        return gko::test::generate_random_matrix<Mtx>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::HipExecutor> hip;
-
-    std::ranlux48 rand_engine;
-};
-
-
-TEST_F(Ir, InitializeIsEquivalentToRef)
-{
-    auto stop_status = gko::Array<gko::stopping_status>(ref, 43);
-    for (size_t i = 0; i < stop_status.get_num_elems(); ++i) {
-        stop_status.get_data()[i].reset();
-    }
-    auto d_stop_status = gko::Array<gko::stopping_status>(hip, stop_status);
-
-    gko::kernels::reference::ir::initialize(ref, &stop_status);
-    gko::kernels::hip::ir::initialize(hip, &d_stop_status);
-
-    auto tmp = gko::Array<gko::stopping_status>(ref, d_stop_status);
-    for (int i = 0; i < stop_status.get_num_elems(); ++i) {
-        ASSERT_EQ(stop_status.get_const_data()[i], tmp.get_const_data()[i]);
-    }
-}
-
-
-TEST_F(Ir, ApplyIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = clone(hip, mtx);
-    auto d_x = clone(hip, x);
-    auto d_b = clone(hip, b);
-    // Forget about accuracy - Richardson is not going to converge for a random
-    // matrix, just check that a couple of iterations gives the same result on
-    // both executors
-    auto ir_factory =
-        gko::solver::Ir<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(ref))
-            .on(ref);
-    auto d_ir_factory =
-        gko::solver::Ir<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(hip))
-            .on(hip);
-    auto solver = ir_factory->generate(std::move(mtx));
-    auto d_solver = d_ir_factory->generate(std::move(d_mtx));
-
-    solver->apply(lend(b), lend(x));
-    d_solver->apply(lend(d_b), lend(d_x));
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = clone(hip, mtx);
-    auto d_x = clone(hip, x);
-    auto d_b = clone(hip, b);
-
-    auto ir_factory =
-        gko::solver::Ir<>::build()
-            .with_solver(
-                gko::solver::Gmres<>::build()
-                    .with_criteria(
-                        gko::stop::Iteration::build().with_max_iters(1u).on(
-                            ref))
-                    .on(ref))
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(ref))
-            .on(ref);
-    auto d_ir_factory =
-        gko::solver::Ir<>::build()
-            .with_solver(
-                gko::solver::Gmres<>::build()
-                    .with_criteria(
-                        gko::stop::Iteration::build().with_max_iters(1u).on(
-                            hip))
-                    .on(hip))
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(hip))
-            .on(hip);
-    auto solver = ir_factory->generate(std::move(mtx));
-    auto d_solver = d_ir_factory->generate(std::move(d_mtx));
-
-    solver->apply(lend(b), lend(x));
-    d_solver->apply(lend(d_b), lend(d_x));
-
-    // Note: 1e-12 instead of 1e-14, as the difference in the inner gmres
-    // iteration gets amplified by the difference in IR.
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
-}
-
-
-TEST_F(Ir, RichardsonApplyIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = clone(hip, mtx);
-    auto d_x = clone(hip, x);
-    auto d_b = clone(hip, b);
-    // Forget about accuracy - Richardson is not going to converge for a random
-    // matrix, just check that a couple of iterations gives the same result on
-    // both executors
-    auto ir_factory =
-        gko::solver::Ir<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(ref))
-            .with_relaxation_factor(0.9)
-            .on(ref);
-    auto d_ir_factory =
-        gko::solver::Ir<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(hip))
-            .with_relaxation_factor(0.9)
-            .on(hip);
-    auto solver = ir_factory->generate(std::move(mtx));
-    auto d_solver = d_ir_factory->generate(std::move(d_mtx));
-
-    solver->apply(lend(b), lend(x));
-    d_solver->apply(lend(d_b), lend(d_x));
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-TEST_F(Ir, RichardsonApplyWithIterativeInnerSolverIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = clone(hip, mtx);
-    auto d_x = clone(hip, x);
-    auto d_b = clone(hip, b);
-    auto ir_factory =
-        gko::solver::Ir<>::build()
-            .with_solver(
-                gko::solver::Gmres<>::build()
-                    .with_criteria(
-                        gko::stop::Iteration::build().with_max_iters(1u).on(
-                            ref))
-                    .on(ref))
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(ref))
-            .with_relaxation_factor(0.9)
-            .on(ref);
-    auto d_ir_factory =
-        gko::solver::Ir<>::build()
-            .with_solver(
-                gko::solver::Gmres<>::build()
-                    .with_criteria(
-                        gko::stop::Iteration::build().with_max_iters(1u).on(
-                            hip))
-                    .on(hip))
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(hip))
-            .with_relaxation_factor(0.9)
-            .on(hip);
-    auto solver = ir_factory->generate(std::move(mtx));
-    auto d_solver = d_ir_factory->generate(std::move(d_mtx));
-
-    solver->apply(lend(b), lend(x));
-    d_solver->apply(lend(d_b), lend(d_x));
-
-    // Note: 1e-12 instead of 1e-14, as the difference in the inner gmres
-    // iteration gets amplified by the difference in IR.
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
-}
-
-
-}  // namespace
diff --git a/omp/test/solver/CMakeLists.txt b/omp/test/solver/CMakeLists.txt
index a69af95bd22..29454e2b5e7 100644
--- a/omp/test/solver/CMakeLists.txt
+++ b/omp/test/solver/CMakeLists.txt
@@ -1,11 +1,5 @@
-ginkgo_create_test(bicg_kernels)
-ginkgo_create_test(bicgstab_kernels)
-ginkgo_create_test(cg_kernels)
-ginkgo_create_test(cgs_kernels)
-ginkgo_create_test(fcg_kernels)
 ginkgo_create_test(gmres_kernels)
 ginkgo_create_test(cb_gmres_kernels)
 ginkgo_create_test(idr_kernels)
-ginkgo_create_test(ir_kernels)
 ginkgo_create_test(lower_trs_kernels)
 ginkgo_create_test(upper_trs_kernels)
diff --git a/omp/test/solver/bicg_kernels.cpp b/omp/test/solver/bicg_kernels.cpp
deleted file mode 100644
index 5049ab9ec10..00000000000
--- a/omp/test/solver/bicg_kernels.cpp
+++ /dev/null
@@ -1,313 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <ginkgo/core/solver/bicg.hpp>
-
-
-#include <random>
-
-
-#include <gtest/gtest.h>
-
-
-#include <ginkgo/core/base/exception.hpp>
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/stop/combined.hpp>
-#include <ginkgo/core/stop/iteration.hpp>
-#include <ginkgo/core/stop/residual_norm.hpp>
-
-
-#include "core/solver/bicg_kernels.hpp"
-#include "core/test/utils.hpp"
-
-
-namespace {
-
-
-class Bicg : public ::testing::Test {
-protected:
-    using Mtx = gko::matrix::Dense<>;
-    Bicg() : rand_engine(30) {}
-
-    void SetUp()
-    {
-        ref = gko::ReferenceExecutor::create();
-        omp = gko::OmpExecutor::create();
-    }
-
-    void TearDown()
-    {
-        if (omp != nullptr) {
-            ASSERT_NO_THROW(omp->synchronize());
-        }
-    }
-
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
-    {
-        return gko::test::generate_random_matrix<Mtx>(
-            num_rows, num_cols,
-            std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
-    }
-
-    void initialize_data()
-    {
-        int m = 597;
-        int n = 43;
-        b = gen_mtx(m, n);
-        r = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        q = gen_mtx(m, n);
-        r2 = gen_mtx(m, n);
-        z2 = gen_mtx(m, n);
-        p2 = gen_mtx(m, n);
-        q2 = gen_mtx(m, n);
-        x = gen_mtx(m, n);
-        beta = gen_mtx(1, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(ref, n));
-        for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
-            stop_status->get_data()[i].reset();
-        }
-
-        d_b = Mtx::create(omp);
-        d_b->copy_from(b.get());
-        d_r = Mtx::create(omp);
-        d_r->copy_from(r.get());
-        d_z = Mtx::create(omp);
-        d_z->copy_from(z.get());
-        d_p = Mtx::create(omp);
-        d_p->copy_from(p.get());
-        d_q = Mtx::create(omp);
-        d_q->copy_from(q.get());
-        d_r2 = Mtx::create(omp);
-        d_r2->copy_from(r2.get());
-        d_z2 = Mtx::create(omp);
-        d_z2->copy_from(z2.get());
-        d_p2 = Mtx::create(omp);
-        d_p2->copy_from(p2.get());
-        d_q2 = Mtx::create(omp);
-        d_q2->copy_from(q2.get());
-        d_x = Mtx::create(omp);
-        d_x->copy_from(x.get());
-        d_beta = Mtx::create(omp);
-        d_beta->copy_from(beta.get());
-        d_prev_rho = Mtx::create(omp);
-        d_prev_rho->copy_from(prev_rho.get());
-        d_rho = Mtx::create(omp);
-        d_rho->copy_from(rho.get());
-        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(omp, n));
-        *d_stop_status = *stop_status;
-    }
-
-    std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::OmpExecutor> omp;
-
-    std::ranlux48 rand_engine;
-
-    std::unique_ptr<Mtx> b;
-    std::unique_ptr<Mtx> r;
-    std::unique_ptr<Mtx> z;
-    std::unique_ptr<Mtx> p;
-    std::unique_ptr<Mtx> q;
-    std::unique_ptr<Mtx> r2;
-    std::unique_ptr<Mtx> z2;
-    std::unique_ptr<Mtx> p2;
-    std::unique_ptr<Mtx> q2;
-    std::unique_ptr<Mtx> x;
-    std::unique_ptr<Mtx> beta;
-    std::unique_ptr<Mtx> prev_rho;
-    std::unique_ptr<Mtx> rho;
-    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
-
-    std::unique_ptr<Mtx> d_b;
-    std::unique_ptr<Mtx> d_r;
-    std::unique_ptr<Mtx> d_z;
-    std::unique_ptr<Mtx> d_p;
-    std::unique_ptr<Mtx> d_q;
-    std::unique_ptr<Mtx> d_r2;
-    std::unique_ptr<Mtx> d_z2;
-    std::unique_ptr<Mtx> d_p2;
-    std::unique_ptr<Mtx> d_q2;
-    std::unique_ptr<Mtx> d_x;
-    std::unique_ptr<Mtx> d_beta;
-    std::unique_ptr<Mtx> d_prev_rho;
-    std::unique_ptr<Mtx> d_rho;
-    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
-};
-
-
-TEST_F(Bicg, OmpBicgInitializeIsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::bicg::initialize(
-        ref, b.get(), r.get(), z.get(), p.get(), q.get(), prev_rho.get(),
-        rho.get(), r2.get(), z2.get(), p2.get(), q2.get(), stop_status.get());
-    gko::kernels::omp::bicg::initialize(
-        omp, d_b.get(), d_r.get(), d_z.get(), d_p.get(), d_q.get(),
-        d_prev_rho.get(), d_rho.get(), d_r2.get(), d_z2.get(), d_p2.get(),
-        d_q2.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r2, r2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z2, z2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p2, p2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q2, q2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho, rho, 1e-14);
-    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
-}
-
-
-TEST_F(Bicg, OmpBicgStep1IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::bicg::step_1(ref, p.get(), z.get(), p2.get(),
-                                          z2.get(), rho.get(), prev_rho.get(),
-                                          stop_status.get());
-    gko::kernels::omp::bicg::step_1(omp, d_p.get(), d_z.get(), d_p2.get(),
-                                    d_z2.get(), d_rho.get(), d_prev_rho.get(),
-                                    d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p2, p2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z2, z2, 1e-14);
-}
-
-
-TEST_F(Bicg, OmpBicgStep2IsEquivalentToRef)
-{
-    initialize_data();
-
-    gko::kernels::reference::bicg::step_2(
-        ref, x.get(), r.get(), r2.get(), p.get(), q.get(), q2.get(), beta.get(),
-        rho.get(), stop_status.get());
-    gko::kernels::omp::bicg::step_2(
-        omp, d_x.get(), d_r.get(), d_r2.get(), d_p.get(), d_q.get(), d_q2.get(),
-        d_beta.get(), d_rho.get(), d_stop_status.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r2, r2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q2, q2, 1e-14);
-}
-
-
-TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    gko::test::make_hpd(mtx.get());
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = Mtx::create(omp);
-    d_mtx->copy_from(mtx.get());
-    auto d_x = Mtx::create(omp);
-    d_x->copy_from(x.get());
-    auto d_b = Mtx::create(omp);
-    d_b->copy_from(b.get());
-    auto bicg_factory =
-        gko::solver::Bicg<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(ref))
-            .on(ref);
-    auto d_bicg_factory =
-        gko::solver::Bicg<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(omp),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(omp))
-            .on(omp);
-    auto solver = bicg_factory->generate(std::move(mtx));
-    auto d_solver = d_bicg_factory->generate(std::move(d_mtx));
-
-    solver->apply(b.get(), x.get());
-    d_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-TEST_F(Bicg, ApplyWithRandomMatrixIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = Mtx::create(omp);
-    d_mtx->copy_from(mtx.get());
-    auto d_x = Mtx::create(omp);
-    d_x->copy_from(x.get());
-    auto d_b = Mtx::create(omp);
-    d_b->copy_from(b.get());
-    auto bicg_factory =
-        gko::solver::Bicg<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(ref))
-            .on(ref);
-    auto d_bicg_factory =
-        gko::solver::Bicg<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(omp),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(omp))
-            .on(omp);
-    auto solver = bicg_factory->generate(std::move(mtx));
-    auto d_solver = d_bicg_factory->generate(std::move(d_mtx));
-
-    solver->apply(b.get(), x.get());
-    d_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
-}  // namespace
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
new file mode 100644
index 00000000000..55efa399aaf
--- /dev/null
+++ b/test/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(solver)
\ No newline at end of file
diff --git a/test_exportbuild/CMakeLists.txt b/test/build_export/CMakeLists.txt
similarity index 82%
rename from test_exportbuild/CMakeLists.txt
rename to test/build_export/CMakeLists.txt
index 52a8d3851cd..8588b63a471 100644
--- a/test_exportbuild/CMakeLists.txt
+++ b/test/build_export/CMakeLists.txt
@@ -5,6 +5,6 @@ find_package(Ginkgo REQUIRED)
 
 # Here, we use test install without any data. We instantiate the
 # interface only.
-add_executable(test_exportbuild ../test_install/test_install.cpp)
+add_executable(test_exportbuild ../build_install/test_install.cpp)
 target_compile_features(test_exportbuild PUBLIC cxx_std_14)
 target_link_libraries(test_exportbuild PRIVATE Ginkgo::ginkgo)
diff --git a/test_install/CMakeLists.txt b/test/build_install/CMakeLists.txt
similarity index 100%
rename from test_install/CMakeLists.txt
rename to test/build_install/CMakeLists.txt
diff --git a/test_install/data/A.mtx b/test/build_install/data/A.mtx
similarity index 100%
rename from test_install/data/A.mtx
rename to test/build_install/data/A.mtx
diff --git a/test_install/data/b.mtx b/test/build_install/data/b.mtx
similarity index 100%
rename from test_install/data/b.mtx
rename to test/build_install/data/b.mtx
diff --git a/test_install/data/x0.mtx b/test/build_install/data/x0.mtx
similarity index 100%
rename from test_install/data/x0.mtx
rename to test/build_install/data/x0.mtx
diff --git a/test_install/test_install.cpp b/test/build_install/test_install.cpp
similarity index 100%
rename from test_install/test_install.cpp
rename to test/build_install/test_install.cpp
diff --git a/test_subdir/CMakeLists.txt b/test/build_subdir/CMakeLists.txt
similarity index 60%
rename from test_subdir/CMakeLists.txt
rename to test/build_subdir/CMakeLists.txt
index 8596cc9a3d6..811cc61f430 100644
--- a/test_subdir/CMakeLists.txt
+++ b/test/build_subdir/CMakeLists.txt
@@ -1,9 +1,9 @@
 cmake_minimum_required(VERSION 3.14)
 project(GinkgoSubdirTest LANGUAGES CXX)
-file(CREATE_LINK "${CMAKE_CURRENT_SOURCE_DIR}/.." "${CMAKE_CURRENT_BINARY_DIR}/ginkgo" SYMBOLIC)
+file(CREATE_LINK "${CMAKE_CURRENT_SOURCE_DIR}/../.." "${CMAKE_CURRENT_BINARY_DIR}/ginkgo" SYMBOLIC)
 
 add_subdirectory("${CMAKE_CURRENT_BINARY_DIR}/ginkgo")
 
-add_executable(test_subdir ../test_install/test_install.cpp)
+add_executable(test_subdir ../build_install/test_install.cpp)
 target_compile_features(test_subdir PUBLIC cxx_std_14)
 target_link_libraries(test_subdir PRIVATE Ginkgo::ginkgo)
diff --git a/test/solver/CMakeLists.txt b/test/solver/CMakeLists.txt
new file mode 100644
index 00000000000..7764b7b9cf3
--- /dev/null
+++ b/test/solver/CMakeLists.txt
@@ -0,0 +1,6 @@
+ginkgo_create_common_test(bicg_kernels)
+ginkgo_create_common_test(bicgstab_kernels)
+ginkgo_create_common_test(cg_kernels)
+ginkgo_create_common_test(cgs_kernels)
+ginkgo_create_common_test(fcg_kernels)
+ginkgo_create_common_test(ir_kernels)
\ No newline at end of file
diff --git a/cuda/test/solver/bicg_kernels.cpp b/test/solver/bicg_kernels.cpp
similarity index 74%
rename from cuda/test/solver/bicg_kernels.cpp
rename to test/solver/bicg_kernels.cpp
index fb62f702ccf..8c2e21b9dc1 100644
--- a/cuda/test/solver/bicg_kernels.cpp
+++ b/test/solver/bicg_kernels.cpp
@@ -33,9 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/solver/bicg.hpp>
 
 
-#include <fstream>
 #include <random>
-#include <string>
 
 
 #include <gtest/gtest.h>
@@ -50,8 +48,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/solver/bicg_kernels.hpp"
-#include "cuda/test/utils.hpp"
+#include "core/test/utils.hpp"
 #include "matrices/config.hpp"
+#include "test/utils/executor.hpp"
 
 
 namespace {
@@ -67,9 +66,8 @@ class Bicg : public ::testing::Test {
 
     void SetUp()
     {
-        ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
         ref = gko::ReferenceExecutor::create();
-        cuda = gko::CudaExecutor::create(0, ref);
+        init_executor(ref, exec);
 
         std::string file_name(gko::matrices::location_ani1_mtx);
         auto input_file = std::ifstream(file_name, std::ios::in);
@@ -77,16 +75,16 @@ class Bicg : public ::testing::Test {
             FAIL() << "Could not find the file \"" << file_name
                    << "\", which is required for this test.\n";
         }
-        csr_ref = gko::read<Csr>(input_file, ref);
-        auto csr_cuda_temp = Csr::create(cuda);
-        csr_cuda_temp->copy_from(gko::lend(csr_ref));
-        csr_cuda = gko::give(csr_cuda_temp);
+        csr = gko::read<Csr>(input_file, ref);
+        auto csr_hip_temp = Csr::create(exec);
+        csr_hip_temp->copy_from(gko::lend(csr));
+        d_csr = gko::give(csr_hip_temp);
     }
 
     void TearDown()
     {
-        if (cuda != nullptr) {
-            ASSERT_NO_THROW(cuda->synchronize());
+        if (exec != nullptr) {
+            ASSERT_NO_THROW(exec->synchronize());
         }
     }
 
@@ -121,39 +119,39 @@ class Bicg : public ::testing::Test {
             stop_status->get_data()[i].reset();
         }
 
-        d_b = Mtx::create(cuda);
+        d_b = Mtx::create(exec);
         d_b->copy_from(b.get());
-        d_r = Mtx::create(cuda);
+        d_r = Mtx::create(exec);
         d_r->copy_from(r.get());
-        d_z = Mtx::create(cuda);
+        d_z = Mtx::create(exec);
         d_z->copy_from(z.get());
-        d_p = Mtx::create(cuda);
+        d_p = Mtx::create(exec);
         d_p->copy_from(p.get());
-        d_q = Mtx::create(cuda);
+        d_q = Mtx::create(exec);
         d_q->copy_from(q.get());
-        d_r2 = Mtx::create(cuda);
+        d_r2 = Mtx::create(exec);
         d_r2->copy_from(r2.get());
-        d_z2 = Mtx::create(cuda);
+        d_z2 = Mtx::create(exec);
         d_z2->copy_from(z2.get());
-        d_p2 = Mtx::create(cuda);
+        d_p2 = Mtx::create(exec);
         d_p2->copy_from(p2.get());
-        d_q2 = Mtx::create(cuda);
+        d_q2 = Mtx::create(exec);
         d_q2->copy_from(q2.get());
-        d_x = Mtx::create(cuda);
+        d_x = Mtx::create(exec);
         d_x->copy_from(x.get());
-        d_beta = Mtx::create(cuda);
+        d_beta = Mtx::create(exec);
         d_beta->copy_from(beta.get());
-        d_prev_rho = Mtx::create(cuda);
+        d_prev_rho = Mtx::create(exec);
         d_prev_rho->copy_from(prev_rho.get());
-        d_rho = Mtx::create(cuda);
+        d_rho = Mtx::create(exec);
         d_rho->copy_from(rho.get());
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(cuda, n));
+            new gko::Array<gko::stopping_status>(exec, n));
         *d_stop_status = *stop_status;
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::CudaExecutor> cuda;
+    std::shared_ptr<gko::EXEC_TYPE> exec;
 
     std::ranlux48 rand_engine;
 
@@ -170,6 +168,7 @@ class Bicg : public ::testing::Test {
     std::unique_ptr<Mtx> beta;
     std::unique_ptr<Mtx> prev_rho;
     std::unique_ptr<Mtx> rho;
+    std::shared_ptr<Csr> csr;
     std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
 
     std::unique_ptr<Mtx> d_b;
@@ -185,21 +184,20 @@ class Bicg : public ::testing::Test {
     std::unique_ptr<Mtx> d_beta;
     std::unique_ptr<Mtx> d_prev_rho;
     std::unique_ptr<Mtx> d_rho;
+    std::shared_ptr<Csr> d_csr;
     std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
-    std::shared_ptr<const Csr> csr_ref;
-    std::shared_ptr<const Csr> csr_cuda;
 };
 
 
-TEST_F(Bicg, CudaBicgInitializeIsEquivalentToRef)
+TEST_F(Bicg, BicgInitializeIsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::bicg::initialize(
         ref, b.get(), r.get(), z.get(), p.get(), q.get(), prev_rho.get(),
         rho.get(), r2.get(), z2.get(), p2.get(), q2.get(), stop_status.get());
-    gko::kernels::cuda::bicg::initialize(
-        cuda, d_b.get(), d_r.get(), d_z.get(), d_p.get(), d_q.get(),
+    gko::kernels::EXEC_NAMESPACE::bicg::initialize(
+        exec, d_b.get(), d_r.get(), d_z.get(), d_p.get(), d_q.get(),
         d_prev_rho.get(), d_rho.get(), d_r2.get(), d_z2.get(), d_p2.get(),
         d_q2.get(), d_stop_status.get());
 
@@ -217,16 +215,16 @@ TEST_F(Bicg, CudaBicgInitializeIsEquivalentToRef)
 }
 
 
-TEST_F(Bicg, CudaBicgStep1IsEquivalentToRef)
+TEST_F(Bicg, BicgStep1IsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::bicg::step_1(ref, p.get(), z.get(), p2.get(),
                                           z2.get(), rho.get(), prev_rho.get(),
                                           stop_status.get());
-    gko::kernels::cuda::bicg::step_1(cuda, d_p.get(), d_z.get(), d_p2.get(),
-                                     d_z2.get(), d_rho.get(), d_prev_rho.get(),
-                                     d_stop_status.get());
+    gko::kernels::EXEC_NAMESPACE::bicg::step_1(
+        exec, d_p.get(), d_z.get(), d_p2.get(), d_z2.get(), d_rho.get(),
+        d_prev_rho.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
@@ -235,15 +233,15 @@ TEST_F(Bicg, CudaBicgStep1IsEquivalentToRef)
 }
 
 
-TEST_F(Bicg, CudaBicgStep2IsEquivalentToRef)
+TEST_F(Bicg, BicgStep2IsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::bicg::step_2(
         ref, x.get(), r.get(), r2.get(), p.get(), q.get(), q2.get(), beta.get(),
         rho.get(), stop_status.get());
-    gko::kernels::cuda::bicg::step_2(
-        cuda, d_x.get(), d_r.get(), d_r2.get(), d_p.get(), d_q.get(),
+    gko::kernels::EXEC_NAMESPACE::bicg::step_2(
+        exec, d_x.get(), d_r.get(), d_r2.get(), d_p.get(), d_q.get(),
         d_q2.get(), d_beta.get(), d_rho.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
@@ -261,11 +259,11 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
     gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
-    auto d_mtx = Mtx::create(cuda);
+    auto d_mtx = Mtx::create(exec);
     d_mtx->copy_from(mtx.get());
-    auto d_x = Mtx::create(cuda);
+    auto d_x = Mtx::create(exec);
     d_x->copy_from(x.get());
-    auto d_b = Mtx::create(cuda);
+    auto d_b = Mtx::create(exec);
     d_b->copy_from(b.get());
     auto bicg_factory =
         gko::solver::Bicg<>::build()
@@ -278,11 +276,11 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
     auto d_bicg_factory =
         gko::solver::Bicg<>::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(cuda),
+                gko::stop::Iteration::build().with_max_iters(50u).on(exec),
                 gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
-                    .on(cuda))
-            .on(cuda);
+                    .on(exec))
+            .on(exec);
     auto solver = bicg_factory->generate(std::move(mtx));
     auto d_solver = d_bicg_factory->generate(std::move(d_mtx));
 
@@ -297,9 +295,9 @@ TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
 {
     auto x = gen_mtx(36, 1);
     auto b = gen_mtx(36, 1);
-    auto d_x = Mtx::create(cuda);
+    auto d_x = Mtx::create(exec);
     d_x->copy_from(x.get());
-    auto d_b = Mtx::create(cuda);
+    auto d_b = Mtx::create(exec);
     d_b->copy_from(b.get());
     auto bicg_factory =
         gko::solver::Bicg<>::build()
@@ -312,13 +310,50 @@ TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
     auto d_bicg_factory =
         gko::solver::Bicg<>::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(cuda),
+                gko::stop::Iteration::build().with_max_iters(50u).on(exec),
                 gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
-                    .on(cuda))
-            .on(cuda);
-    auto solver = bicg_factory->generate(std::move(csr_ref));
-    auto d_solver = d_bicg_factory->generate(std::move(csr_cuda));
+                    .on(exec))
+            .on(exec);
+    auto solver = bicg_factory->generate(std::move(csr));
+    auto d_solver = d_bicg_factory->generate(std::move(d_csr));
+
+    solver->apply(b.get(), x.get());
+    d_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+}
+
+
+TEST_F(Bicg, ApplyWithRandomMatrixIsEquivalentToRef)
+{
+    auto mtx = gen_mtx(50, 50);
+    auto x = gen_mtx(50, 3);
+    auto b = gen_mtx(50, 3);
+    auto d_mtx = Mtx::create(exec);
+    d_mtx->copy_from(mtx.get());
+    auto d_x = Mtx::create(exec);
+    d_x->copy_from(x.get());
+    auto d_b = Mtx::create(exec);
+    d_b->copy_from(b.get());
+    auto bicg_factory =
+        gko::solver::Bicg<>::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(50u).on(ref),
+                gko::stop::ResidualNorm<>::build()
+                    .with_reduction_factor(1e-14)
+                    .on(ref))
+            .on(ref);
+    auto d_bicg_factory =
+        gko::solver::Bicg<>::build()
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(50u).on(exec),
+                gko::stop::ResidualNorm<>::build()
+                    .with_reduction_factor(1e-14)
+                    .on(exec))
+            .on(exec);
+    auto solver = bicg_factory->generate(std::move(mtx));
+    auto d_solver = d_bicg_factory->generate(std::move(d_mtx));
 
     solver->apply(b.get(), x.get());
     d_solver->apply(d_b.get(), d_x.get());
diff --git a/omp/test/solver/bicgstab_kernels.cpp b/test/solver/bicgstab_kernels.cpp
similarity index 80%
rename from omp/test/solver/bicgstab_kernels.cpp
rename to test/solver/bicgstab_kernels.cpp
index 9e51d841e64..307c7e4ac0e 100644
--- a/omp/test/solver/bicgstab_kernels.cpp
+++ b/test/solver/bicgstab_kernels.cpp
@@ -50,6 +50,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/solver/bicgstab_kernels.hpp"
 #include "core/test/utils.hpp"
+#include "test/utils/executor.hpp"
 
 
 namespace {
@@ -65,20 +66,20 @@ class Bicgstab : public ::testing::Test {
     void SetUp()
     {
         ref = gko::ReferenceExecutor::create();
-        omp = gko::OmpExecutor::create();
+        init_executor(ref, exec);
 
         mtx = gen_mtx(123, 123);
         gko::test::make_diag_dominant(mtx.get());
-        d_mtx = Mtx::create(omp);
+        d_mtx = Mtx::create(exec);
         d_mtx->copy_from(mtx.get());
-        omp_bicgstab_factory =
+        exec_bicgstab_factory =
             Solver::build()
                 .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(omp),
+                    gko::stop::Iteration::build().with_max_iters(246u).on(exec),
                     gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
-                        .on(omp))
-                .on(omp);
+                        .on(exec))
+                .on(exec);
 
         ref_bicgstab_factory =
             Solver::build()
@@ -92,8 +93,8 @@ class Bicgstab : public ::testing::Test {
 
     void TearDown()
     {
-        if (omp != nullptr) {
-            ASSERT_NO_THROW(omp->synchronize());
+        if (exec != nullptr) {
+            ASSERT_NO_THROW(exec->synchronize());
         }
     }
 
@@ -131,24 +132,24 @@ class Bicgstab : public ::testing::Test {
             stop_status->get_data()[i].reset();
         }
 
-        d_x = Mtx::create(omp);
-        d_b = Mtx::create(omp);
-        d_r = Mtx::create(omp);
-        d_z = Mtx::create(omp);
-        d_p = Mtx::create(omp);
-        d_t = Mtx::create(omp);
-        d_s = Mtx::create(omp);
-        d_y = Mtx::create(omp);
-        d_v = Mtx::create(omp);
-        d_rr = Mtx::create(omp);
-        d_prev_rho = Mtx::create(omp);
-        d_rho = Mtx::create(omp);
-        d_alpha = Mtx::create(omp);
-        d_beta = Mtx::create(omp);
-        d_gamma = Mtx::create(omp);
-        d_omega = Mtx::create(omp);
+        d_x = Mtx::create(exec);
+        d_b = Mtx::create(exec);
+        d_r = Mtx::create(exec);
+        d_z = Mtx::create(exec);
+        d_p = Mtx::create(exec);
+        d_t = Mtx::create(exec);
+        d_s = Mtx::create(exec);
+        d_y = Mtx::create(exec);
+        d_v = Mtx::create(exec);
+        d_rr = Mtx::create(exec);
+        d_prev_rho = Mtx::create(exec);
+        d_rho = Mtx::create(exec);
+        d_alpha = Mtx::create(exec);
+        d_beta = Mtx::create(exec);
+        d_gamma = Mtx::create(exec);
+        d_omega = Mtx::create(exec);
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(omp));
+            new gko::Array<gko::stopping_status>(exec));
 
         d_x->copy_from(x.get());
         d_b->copy_from(b.get());
@@ -171,13 +172,13 @@ class Bicgstab : public ::testing::Test {
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::OmpExecutor> omp;
+    std::shared_ptr<gko::EXEC_TYPE> exec;
 
     std::ranlux48 rand_engine;
 
     std::shared_ptr<Mtx> mtx;
     std::shared_ptr<Mtx> d_mtx;
-    std::unique_ptr<Solver::Factory> omp_bicgstab_factory;
+    std::unique_ptr<Solver::Factory> exec_bicgstab_factory;
     std::unique_ptr<Solver::Factory> ref_bicgstab_factory;
 
     std::unique_ptr<Mtx> x;
@@ -218,7 +219,7 @@ class Bicgstab : public ::testing::Test {
 };
 
 
-TEST_F(Bicgstab, OmpBicgstabInitializeIsEquivalentToRef)
+TEST_F(Bicgstab, BicgstabInitializeIsEquivalentToRef)
 {
     initialize_data();
 
@@ -226,8 +227,8 @@ TEST_F(Bicgstab, OmpBicgstabInitializeIsEquivalentToRef)
         ref, b.get(), r.get(), rr.get(), y.get(), s.get(), t.get(), z.get(),
         v.get(), p.get(), prev_rho.get(), rho.get(), alpha.get(), beta.get(),
         gamma.get(), omega.get(), stop_status.get());
-    gko::kernels::omp::bicgstab::initialize(
-        omp, d_b.get(), d_r.get(), d_rr.get(), d_y.get(), d_s.get(), d_t.get(),
+    gko::kernels::EXEC_NAMESPACE::bicgstab::initialize(
+        exec, d_b.get(), d_r.get(), d_rr.get(), d_y.get(), d_s.get(), d_t.get(),
         d_z.get(), d_v.get(), d_p.get(), d_prev_rho.get(), d_rho.get(),
         d_alpha.get(), d_beta.get(), d_gamma.get(), d_omega.get(),
         d_stop_status.get());
@@ -250,46 +251,46 @@ TEST_F(Bicgstab, OmpBicgstabInitializeIsEquivalentToRef)
 }
 
 
-TEST_F(Bicgstab, OmpBicgstabStep1IsEquivalentToRef)
+TEST_F(Bicgstab, BicgstabStep1IsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::bicgstab::step_1(
         ref, r.get(), p.get(), v.get(), rho.get(), prev_rho.get(), alpha.get(),
         omega.get(), stop_status.get());
-    gko::kernels::omp::bicgstab::step_1(
-        omp, d_r.get(), d_p.get(), d_v.get(), d_rho.get(), d_prev_rho.get(),
+    gko::kernels::EXEC_NAMESPACE::bicgstab::step_1(
+        exec, d_r.get(), d_p.get(), d_v.get(), d_rho.get(), d_prev_rho.get(),
         d_alpha.get(), d_omega.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
 }
 
 
-TEST_F(Bicgstab, OmpBicgstabStep2IsEquivalentToRef)
+TEST_F(Bicgstab, BicgstabStep2IsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::bicgstab::step_2(ref, r.get(), s.get(), v.get(),
                                               rho.get(), alpha.get(),
                                               beta.get(), stop_status.get());
-    gko::kernels::omp::bicgstab::step_2(omp, d_r.get(), d_s.get(), d_v.get(),
-                                        d_rho.get(), d_alpha.get(),
-                                        d_beta.get(), d_stop_status.get());
+    gko::kernels::EXEC_NAMESPACE::bicgstab::step_2(
+        exec, d_r.get(), d_s.get(), d_v.get(), d_rho.get(), d_alpha.get(),
+        d_beta.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_s, s, 1e-14);
 }
 
 
-TEST_F(Bicgstab, OmpBicgstabStep3IsEquivalentToRef)
+TEST_F(Bicgstab, BicgstabStep3IsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::bicgstab::step_3(
         ref, x.get(), r.get(), s.get(), t.get(), y.get(), z.get(), alpha.get(),
         beta.get(), gamma.get(), omega.get(), stop_status.get());
-    gko::kernels::omp::bicgstab::step_3(
-        omp, d_x.get(), d_r.get(), d_s.get(), d_t.get(), d_y.get(), d_z.get(),
+    gko::kernels::EXEC_NAMESPACE::bicgstab::step_3(
+        exec, d_x.get(), d_r.get(), d_s.get(), d_t.get(), d_y.get(), d_z.get(),
         d_alpha.get(), d_beta.get(), d_gamma.get(), d_omega.get(),
         d_stop_status.get());
 
@@ -299,42 +300,42 @@ TEST_F(Bicgstab, OmpBicgstabStep3IsEquivalentToRef)
 }
 
 
-TEST_F(Bicgstab, OmpBicgstabApplyOneRHSIsEquivalentToRef)
+TEST_F(Bicgstab, BicgstabApplyOneRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 1;
     auto ref_solver = ref_bicgstab_factory->generate(mtx);
-    auto omp_solver = omp_bicgstab_factory->generate(d_mtx);
+    auto exec_solver = exec_bicgstab_factory->generate(d_mtx);
     auto b = gen_mtx(m, n);
     auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(omp);
-    auto d_x = Mtx::create(omp);
+    auto d_b = Mtx::create(exec);
+    auto d_x = Mtx::create(exec);
     d_b->copy_from(b.get());
     d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
-    omp_solver->apply(d_b.get(), d_x.get());
+    exec_solver->apply(d_b.get(), d_x.get());
 
     GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
 }
 
 
-TEST_F(Bicgstab, OmpBicgstabApplyMultipleRHSIsEquivalentToRef)
+TEST_F(Bicgstab, BicgstabApplyMultipleRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 16;
-    auto omp_solver = omp_bicgstab_factory->generate(d_mtx);
+    auto exec_solver = exec_bicgstab_factory->generate(d_mtx);
     auto ref_solver = ref_bicgstab_factory->generate(mtx);
     auto b = gen_mtx(m, n);
     auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(omp);
-    auto d_x = Mtx::create(omp);
+    auto d_b = Mtx::create(exec);
+    auto d_x = Mtx::create(exec);
     d_b->copy_from(b.get());
     d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
-    omp_solver->apply(d_b.get(), d_x.get());
+    exec_solver->apply(d_b.get(), d_x.get());
 
     GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
diff --git a/omp/test/solver/cg_kernels.cpp b/test/solver/cg_kernels.cpp
similarity index 81%
rename from omp/test/solver/cg_kernels.cpp
rename to test/solver/cg_kernels.cpp
index 0a5014aad64..129cc54631d 100644
--- a/omp/test/solver/cg_kernels.cpp
+++ b/test/solver/cg_kernels.cpp
@@ -49,6 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/solver/cg_kernels.hpp"
 #include "core/test/utils.hpp"
+#include "test/utils/executor.hpp"
 
 
 namespace {
@@ -62,13 +63,13 @@ class Cg : public ::testing::Test {
     void SetUp()
     {
         ref = gko::ReferenceExecutor::create();
-        omp = gko::OmpExecutor::create();
+        init_executor(ref, exec);
     }
 
     void TearDown()
     {
-        if (omp != nullptr) {
-            ASSERT_NO_THROW(omp->synchronize());
+        if (exec != nullptr) {
+            ASSERT_NO_THROW(exec->synchronize());
         }
     }
 
@@ -99,31 +100,31 @@ class Cg : public ::testing::Test {
             stop_status->get_data()[i].reset();
         }
 
-        d_b = Mtx::create(omp);
+        d_b = Mtx::create(exec);
         d_b->copy_from(b.get());
-        d_r = Mtx::create(omp);
+        d_r = Mtx::create(exec);
         d_r->copy_from(r.get());
-        d_z = Mtx::create(omp);
+        d_z = Mtx::create(exec);
         d_z->copy_from(z.get());
-        d_p = Mtx::create(omp);
+        d_p = Mtx::create(exec);
         d_p->copy_from(p.get());
-        d_q = Mtx::create(omp);
+        d_q = Mtx::create(exec);
         d_q->copy_from(q.get());
-        d_x = Mtx::create(omp);
+        d_x = Mtx::create(exec);
         d_x->copy_from(x.get());
-        d_beta = Mtx::create(omp);
+        d_beta = Mtx::create(exec);
         d_beta->copy_from(beta.get());
-        d_prev_rho = Mtx::create(omp);
+        d_prev_rho = Mtx::create(exec);
         d_prev_rho->copy_from(prev_rho.get());
-        d_rho = Mtx::create(omp);
+        d_rho = Mtx::create(exec);
         d_rho->copy_from(rho.get());
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(omp, n));
+            new gko::Array<gko::stopping_status>(exec, n));
         *d_stop_status = *stop_status;
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::OmpExecutor> omp;
+    std::shared_ptr<gko::EXEC_TYPE> exec;
 
     std::ranlux48 rand_engine;
 
@@ -151,16 +152,16 @@ class Cg : public ::testing::Test {
 };
 
 
-TEST_F(Cg, OmpCgInitializeIsEquivalentToRef)
+TEST_F(Cg, CgInitializeIsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::cg::initialize(ref, b.get(), r.get(), z.get(),
                                             p.get(), q.get(), prev_rho.get(),
                                             rho.get(), stop_status.get());
-    gko::kernels::omp::cg::initialize(omp, d_b.get(), d_r.get(), d_z.get(),
-                                      d_p.get(), d_q.get(), d_prev_rho.get(),
-                                      d_rho.get(), d_stop_status.get());
+    gko::kernels::EXEC_NAMESPACE::cg::initialize(
+        exec, d_b.get(), d_r.get(), d_z.get(), d_p.get(), d_q.get(),
+        d_prev_rho.get(), d_rho.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
@@ -172,29 +173,30 @@ TEST_F(Cg, OmpCgInitializeIsEquivalentToRef)
 }
 
 
-TEST_F(Cg, OmpCgStep1IsEquivalentToRef)
+TEST_F(Cg, CgStep1IsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::cg::step_1(ref, p.get(), z.get(), rho.get(),
                                         prev_rho.get(), stop_status.get());
-    gko::kernels::omp::cg::step_1(omp, d_p.get(), d_z.get(), d_rho.get(),
-                                  d_prev_rho.get(), d_stop_status.get());
+    gko::kernels::EXEC_NAMESPACE::cg::step_1(exec, d_p.get(), d_z.get(),
+                                             d_rho.get(), d_prev_rho.get(),
+                                             d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
 }
 
 
-TEST_F(Cg, OmpCgStep2IsEquivalentToRef)
+TEST_F(Cg, CgStep2IsEquivalentToRef)
 {
     initialize_data();
     gko::kernels::reference::cg::step_2(ref, x.get(), r.get(), p.get(), q.get(),
                                         beta.get(), rho.get(),
                                         stop_status.get());
-    gko::kernels::omp::cg::step_2(omp, d_x.get(), d_r.get(), d_p.get(),
-                                  d_q.get(), d_beta.get(), d_rho.get(),
-                                  d_stop_status.get());
+    gko::kernels::EXEC_NAMESPACE::cg::step_2(exec, d_x.get(), d_r.get(),
+                                             d_p.get(), d_q.get(), d_beta.get(),
+                                             d_rho.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
@@ -209,11 +211,11 @@ TEST_F(Cg, ApplyIsEquivalentToRef)
     gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
-    auto d_mtx = Mtx::create(omp);
+    auto d_mtx = Mtx::create(exec);
     d_mtx->copy_from(mtx.get());
-    auto d_x = Mtx::create(omp);
+    auto d_x = Mtx::create(exec);
     d_x->copy_from(x.get());
-    auto d_b = Mtx::create(omp);
+    auto d_b = Mtx::create(exec);
     d_b->copy_from(b.get());
     auto cg_factory =
         gko::solver::Cg<>::build()
@@ -226,11 +228,11 @@ TEST_F(Cg, ApplyIsEquivalentToRef)
     auto d_cg_factory =
         gko::solver::Cg<>::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(omp),
+                gko::stop::Iteration::build().with_max_iters(50u).on(exec),
                 gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
-                    .on(omp))
-            .on(omp);
+                    .on(exec))
+            .on(exec);
     auto solver = cg_factory->generate(std::move(mtx));
     auto d_solver = d_cg_factory->generate(std::move(d_mtx));
 
diff --git a/omp/test/solver/cgs_kernels.cpp b/test/solver/cgs_kernels.cpp
similarity index 79%
rename from omp/test/solver/cgs_kernels.cpp
rename to test/solver/cgs_kernels.cpp
index d473f830bdc..6b6b812c1e1 100644
--- a/omp/test/solver/cgs_kernels.cpp
+++ b/test/solver/cgs_kernels.cpp
@@ -49,6 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/solver/cgs_kernels.hpp"
 #include "core/test/utils.hpp"
+#include "test/utils/executor.hpp"
 
 
 namespace {
@@ -64,20 +65,20 @@ class Cgs : public ::testing::Test {
     void SetUp()
     {
         ref = gko::ReferenceExecutor::create();
-        omp = gko::OmpExecutor::create();
+        init_executor(ref, exec);
 
         mtx = gen_mtx(123, 123);
         gko::test::make_diag_dominant(mtx.get());
-        d_mtx = Mtx::create(omp);
+        d_mtx = Mtx::create(exec);
         d_mtx->copy_from(mtx.get());
-        omp_cgs_factory =
+        exec_cgs_factory =
             Solver::build()
                 .with_criteria(
-                    gko::stop::Iteration::build().with_max_iters(246u).on(omp),
+                    gko::stop::Iteration::build().with_max_iters(246u).on(exec),
                     gko::stop::ResidualNorm<>::build()
                         .with_reduction_factor(1e-15)
-                        .on(omp))
-                .on(omp);
+                        .on(exec))
+                .on(exec);
         ref_cgs_factory =
             Solver::build()
                 .with_criteria(
@@ -90,8 +91,8 @@ class Cgs : public ::testing::Test {
 
     void TearDown()
     {
-        if (omp != nullptr) {
-            ASSERT_NO_THROW(omp->synchronize());
+        if (exec != nullptr) {
+            ASSERT_NO_THROW(exec->synchronize());
         }
     }
 
@@ -128,51 +129,51 @@ class Cgs : public ::testing::Test {
             stop_status->get_data()[i].reset();
         }
 
-        d_b = Mtx::create(omp);
+        d_b = Mtx::create(exec);
         d_b->copy_from(b.get());
-        d_r = Mtx::create(omp);
+        d_r = Mtx::create(exec);
         d_r->copy_from(r.get());
-        d_r_tld = Mtx::create(omp);
+        d_r_tld = Mtx::create(exec);
         d_r_tld->copy_from(r_tld.get());
-        d_p = Mtx::create(omp);
+        d_p = Mtx::create(exec);
         d_p->copy_from(p.get());
-        d_q = Mtx::create(omp);
+        d_q = Mtx::create(exec);
         d_q->copy_from(q.get());
-        d_u = Mtx::create(omp);
+        d_u = Mtx::create(exec);
         d_u->copy_from(u.get());
-        d_u_hat = Mtx::create(omp);
+        d_u_hat = Mtx::create(exec);
         d_u_hat->copy_from(u_hat.get());
-        d_v_hat = Mtx::create(omp);
+        d_v_hat = Mtx::create(exec);
         d_v_hat->copy_from(v_hat.get());
-        d_t = Mtx::create(omp);
+        d_t = Mtx::create(exec);
         d_t->copy_from(t.get());
-        d_x = Mtx::create(omp);
+        d_x = Mtx::create(exec);
         d_x->copy_from(x.get());
-        d_alpha = Mtx::create(omp);
+        d_alpha = Mtx::create(exec);
         d_alpha->copy_from(alpha.get());
-        d_beta = Mtx::create(omp);
+        d_beta = Mtx::create(exec);
         d_beta->copy_from(beta.get());
-        d_gamma = Mtx::create(omp);
+        d_gamma = Mtx::create(exec);
         d_gamma->copy_from(gamma.get());
-        d_rho_prev = Mtx::create(omp);
+        d_rho_prev = Mtx::create(exec);
         d_rho_prev->copy_from(rho_prev.get());
-        d_rho = Mtx::create(omp);
+        d_rho = Mtx::create(exec);
         d_rho->copy_from(rho.get());
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(omp, n));
+            new gko::Array<gko::stopping_status>(exec, n));
         // because there is no public function copy_from, use overloaded =
         // operator
         *d_stop_status = *stop_status;
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::OmpExecutor> omp;
+    std::shared_ptr<gko::EXEC_TYPE> exec;
 
     std::ranlux48 rand_engine;
 
     std::shared_ptr<Mtx> mtx;
     std::shared_ptr<Mtx> d_mtx;
-    std::unique_ptr<Solver::Factory> omp_cgs_factory;
+    std::unique_ptr<Solver::Factory> exec_cgs_factory;
     std::unique_ptr<Solver::Factory> ref_cgs_factory;
 
     std::unique_ptr<Mtx> b;
@@ -211,7 +212,7 @@ class Cgs : public ::testing::Test {
 };
 
 
-TEST_F(Cgs, OmpCgsInitializeIsEquivalentToRef)
+TEST_F(Cgs, CgsInitializeIsEquivalentToRef)
 {
     initialize_data();
 
@@ -219,8 +220,8 @@ TEST_F(Cgs, OmpCgsInitializeIsEquivalentToRef)
         ref, b.get(), r.get(), r_tld.get(), p.get(), q.get(), u.get(),
         u_hat.get(), v_hat.get(), t.get(), alpha.get(), beta.get(), gamma.get(),
         rho_prev.get(), rho.get(), stop_status.get());
-    gko::kernels::omp::cgs::initialize(
-        omp, d_b.get(), d_r.get(), d_r_tld.get(), d_p.get(), d_q.get(),
+    gko::kernels::EXEC_NAMESPACE::cgs::initialize(
+        exec, d_b.get(), d_r.get(), d_r_tld.get(), d_p.get(), d_q.get(),
         d_u.get(), d_u_hat.get(), d_v_hat.get(), d_t.get(), d_alpha.get(),
         d_beta.get(), d_gamma.get(), d_rho_prev.get(), d_rho.get(),
         d_stop_status.get());
@@ -242,16 +243,16 @@ TEST_F(Cgs, OmpCgsInitializeIsEquivalentToRef)
 }
 
 
-TEST_F(Cgs, OmpCgsStep1IsEquivalentToRef)
+TEST_F(Cgs, CgsStep1IsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::cgs::step_1(ref, r.get(), u.get(), p.get(),
                                          q.get(), beta.get(), rho.get(),
                                          rho_prev.get(), stop_status.get());
-    gko::kernels::omp::cgs::step_1(omp, d_r.get(), d_u.get(), d_p.get(),
-                                   d_q.get(), d_beta.get(), d_rho.get(),
-                                   d_rho_prev.get(), d_stop_status.get());
+    gko::kernels::EXEC_NAMESPACE::cgs::step_1(
+        exec, d_r.get(), d_u.get(), d_p.get(), d_q.get(), d_beta.get(),
+        d_rho.get(), d_rho_prev.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(d_beta, beta, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_u, u, 1e-14);
@@ -259,16 +260,16 @@ TEST_F(Cgs, OmpCgsStep1IsEquivalentToRef)
 }
 
 
-TEST_F(Cgs, OmpCgsStep2IsEquivalentToRef)
+TEST_F(Cgs, CgsStep2IsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::cgs::step_2(ref, u.get(), v_hat.get(), q.get(),
                                          t.get(), alpha.get(), rho.get(),
                                          gamma.get(), stop_status.get());
-    gko::kernels::omp::cgs::step_2(omp, d_u.get(), d_v_hat.get(), d_q.get(),
-                                   d_t.get(), d_alpha.get(), d_rho.get(),
-                                   d_gamma.get(), d_stop_status.get());
+    gko::kernels::EXEC_NAMESPACE::cgs::step_2(
+        exec, d_u.get(), d_v_hat.get(), d_q.get(), d_t.get(), d_alpha.get(),
+        d_rho.get(), d_gamma.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_t, t, 1e-14);
@@ -276,58 +277,58 @@ TEST_F(Cgs, OmpCgsStep2IsEquivalentToRef)
 }
 
 
-TEST_F(Cgs, OmpCgsStep3IsEquivalentToRef)
+TEST_F(Cgs, CgsStep3IsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::cgs::step_3(ref, t.get(), u_hat.get(), r.get(),
                                          x.get(), alpha.get(),
                                          stop_status.get());
-    gko::kernels::omp::cgs::step_3(omp, d_t.get(), d_u_hat.get(), d_r.get(),
-                                   d_x.get(), d_alpha.get(),
-                                   d_stop_status.get());
+    gko::kernels::EXEC_NAMESPACE::cgs::step_3(
+        exec, d_t.get(), d_u_hat.get(), d_r.get(), d_x.get(), d_alpha.get(),
+        d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
 }
 
 
-TEST_F(Cgs, OmpCgsApplyOneRHSIsEquivalentToRef)
+TEST_F(Cgs, CgsApplyOneRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 1;
     auto ref_solver = ref_cgs_factory->generate(mtx);
-    auto omp_solver = omp_cgs_factory->generate(d_mtx);
+    auto exec_solver = exec_cgs_factory->generate(d_mtx);
     auto b = gen_mtx(m, n);
     auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(omp);
-    auto d_x = Mtx::create(omp);
+    auto d_b = Mtx::create(exec);
+    auto d_x = Mtx::create(exec);
     d_b->copy_from(b.get());
     d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
-    omp_solver->apply(d_b.get(), d_x.get());
+    exec_solver->apply(d_b.get(), d_x.get());
 
     GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
 }
 
 
-TEST_F(Cgs, OmpCgsApplyMultipleRHSIsEquivalentToRef)
+TEST_F(Cgs, CgsApplyMultipleRHSIsEquivalentToRef)
 {
     int m = 123;
     int n = 16;
-    auto omp_solver = omp_cgs_factory->generate(d_mtx);
+    auto exec_solver = exec_cgs_factory->generate(d_mtx);
     auto ref_solver = ref_cgs_factory->generate(mtx);
     auto b = gen_mtx(m, n);
     auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(omp);
-    auto d_x = Mtx::create(omp);
+    auto d_b = Mtx::create(exec);
+    auto d_x = Mtx::create(exec);
     d_b->copy_from(b.get());
     d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
-    omp_solver->apply(d_b.get(), d_x.get());
+    exec_solver->apply(d_b.get(), d_x.get());
 
     GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
diff --git a/omp/test/solver/fcg_kernels.cpp b/test/solver/fcg_kernels.cpp
similarity index 83%
rename from omp/test/solver/fcg_kernels.cpp
rename to test/solver/fcg_kernels.cpp
index 85fd4cfebfd..fd7a6280628 100644
--- a/omp/test/solver/fcg_kernels.cpp
+++ b/test/solver/fcg_kernels.cpp
@@ -49,6 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/solver/fcg_kernels.hpp"
 #include "core/test/utils.hpp"
+#include "test/utils/executor.hpp"
 
 
 namespace {
@@ -64,13 +65,13 @@ class Fcg : public ::testing::Test {
     void SetUp()
     {
         ref = gko::ReferenceExecutor::create();
-        omp = gko::OmpExecutor::create();
+        init_executor(ref, exec);
     }
 
     void TearDown()
     {
-        if (omp != nullptr) {
-            ASSERT_NO_THROW(omp->synchronize());
+        if (exec != nullptr) {
+            ASSERT_NO_THROW(exec->synchronize());
         }
     }
 
@@ -103,35 +104,35 @@ class Fcg : public ::testing::Test {
             stop_status->get_data()[i].reset();
         }
 
-        d_b = Mtx::create(omp);
+        d_b = Mtx::create(exec);
         d_b->copy_from(b.get());
-        d_r = Mtx::create(omp);
+        d_r = Mtx::create(exec);
         d_r->copy_from(r.get());
-        d_t = Mtx::create(omp);
+        d_t = Mtx::create(exec);
         d_t->copy_from(t.get());
-        d_z = Mtx::create(omp);
+        d_z = Mtx::create(exec);
         d_z->copy_from(z.get());
-        d_p = Mtx::create(omp);
+        d_p = Mtx::create(exec);
         d_p->copy_from(p.get());
-        d_q = Mtx::create(omp);
+        d_q = Mtx::create(exec);
         d_q->copy_from(q.get());
-        d_x = Mtx::create(omp);
+        d_x = Mtx::create(exec);
         d_x->copy_from(x.get());
-        d_beta = Mtx::create(omp);
+        d_beta = Mtx::create(exec);
         d_beta->copy_from(beta.get());
-        d_prev_rho = Mtx::create(omp);
+        d_prev_rho = Mtx::create(exec);
         d_prev_rho->copy_from(prev_rho.get());
-        d_rho_t = Mtx::create(omp);
+        d_rho_t = Mtx::create(exec);
         d_rho_t->copy_from(rho_t.get());
-        d_rho = Mtx::create(omp);
+        d_rho = Mtx::create(exec);
         d_rho->copy_from(rho.get());
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
-            new gko::Array<gko::stopping_status>(omp, n));
+            new gko::Array<gko::stopping_status>(exec, n));
         *d_stop_status = *stop_status;
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::OmpExecutor> omp;
+    std::shared_ptr<gko::EXEC_TYPE> exec;
 
     std::ranlux48 rand_engine;
 
@@ -163,15 +164,15 @@ class Fcg : public ::testing::Test {
 };
 
 
-TEST_F(Fcg, OmpFcgInitializeIsEquivalentToRef)
+TEST_F(Fcg, FcgInitializeIsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::fcg::initialize(
         ref, b.get(), r.get(), z.get(), p.get(), q.get(), t.get(),
         prev_rho.get(), rho.get(), rho_t.get(), stop_status.get());
-    gko::kernels::omp::fcg::initialize(
-        omp, d_b.get(), d_r.get(), d_z.get(), d_p.get(), d_q.get(), d_t.get(),
+    gko::kernels::EXEC_NAMESPACE::fcg::initialize(
+        exec, d_b.get(), d_r.get(), d_z.get(), d_p.get(), d_q.get(), d_t.get(),
         d_prev_rho.get(), d_rho.get(), d_rho_t.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
@@ -186,29 +187,30 @@ TEST_F(Fcg, OmpFcgInitializeIsEquivalentToRef)
 }
 
 
-TEST_F(Fcg, OmpFcgStep1IsEquivalentToRef)
+TEST_F(Fcg, FcgStep1IsEquivalentToRef)
 {
     initialize_data();
 
     gko::kernels::reference::fcg::step_1(ref, p.get(), z.get(), rho_t.get(),
                                          prev_rho.get(), stop_status.get());
-    gko::kernels::omp::fcg::step_1(omp, d_p.get(), d_z.get(), d_rho_t.get(),
-                                   d_prev_rho.get(), d_stop_status.get());
+    gko::kernels::EXEC_NAMESPACE::fcg::step_1(exec, d_p.get(), d_z.get(),
+                                              d_rho_t.get(), d_prev_rho.get(),
+                                              d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
 }
 
 
-TEST_F(Fcg, OmpFcgStep2IsEquivalentToRef)
+TEST_F(Fcg, FcgStep2IsEquivalentToRef)
 {
     initialize_data();
     gko::kernels::reference::fcg::step_2(ref, x.get(), r.get(), t.get(),
                                          p.get(), q.get(), beta.get(),
                                          rho.get(), stop_status.get());
-    gko::kernels::omp::fcg::step_2(omp, d_x.get(), d_r.get(), d_t.get(),
-                                   d_p.get(), d_q.get(), d_beta.get(),
-                                   d_rho.get(), d_stop_status.get());
+    gko::kernels::EXEC_NAMESPACE::fcg::step_2(
+        exec, d_x.get(), d_r.get(), d_t.get(), d_p.get(), d_q.get(),
+        d_beta.get(), d_rho.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
     GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
@@ -222,11 +224,11 @@ TEST_F(Fcg, ApplyIsEquivalentToRef)
     gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
-    auto d_mtx = Mtx::create(omp);
+    auto d_mtx = Mtx::create(exec);
     d_mtx->copy_from(mtx.get());
-    auto d_x = Mtx::create(omp);
+    auto d_x = Mtx::create(exec);
     d_x->copy_from(x.get());
-    auto d_b = Mtx::create(omp);
+    auto d_b = Mtx::create(exec);
     d_b->copy_from(b.get());
     auto fcg_factory =
         Solver::build()
@@ -239,11 +241,11 @@ TEST_F(Fcg, ApplyIsEquivalentToRef)
     auto d_fcg_factory =
         Solver::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(omp),
+                gko::stop::Iteration::build().with_max_iters(50u).on(exec),
                 gko::stop::ResidualNorm<>::build()
                     .with_reduction_factor(1e-14)
-                    .on(omp))
-            .on(omp);
+                    .on(exec))
+            .on(exec);
     auto solver = fcg_factory->generate(std::move(mtx));
     auto d_solver = d_fcg_factory->generate(std::move(d_mtx));
 
diff --git a/omp/test/solver/ir_kernels.cpp b/test/solver/ir_kernels.cpp
similarity index 89%
rename from omp/test/solver/ir_kernels.cpp
rename to test/solver/ir_kernels.cpp
index 86312d491d6..4c5c35544ee 100644
--- a/omp/test/solver/ir_kernels.cpp
+++ b/test/solver/ir_kernels.cpp
@@ -49,6 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/solver/ir_kernels.hpp"
 #include "core/test/utils.hpp"
+#include "test/utils/executor.hpp"
 
 
 namespace {
@@ -62,7 +63,7 @@ class Ir : public ::testing::Test {
     void SetUp()
     {
         ref = gko::ReferenceExecutor::create();
-        omp = gko::OmpExecutor::create();
+        init_executor(ref, exec);
     }
 
     std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
@@ -74,7 +75,7 @@ class Ir : public ::testing::Test {
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
-    std::shared_ptr<const gko::OmpExecutor> omp;
+    std::shared_ptr<gko::EXEC_TYPE> exec;
 
     std::ranlux48 rand_engine;
 };
@@ -86,10 +87,10 @@ TEST_F(Ir, InitializeIsEquivalentToRef)
     for (size_t i = 0; i < stop_status.get_num_elems(); ++i) {
         stop_status.get_data()[i].reset();
     }
-    auto d_stop_status = gko::Array<gko::stopping_status>(omp, stop_status);
+    auto d_stop_status = gko::Array<gko::stopping_status>(exec, stop_status);
 
     gko::kernels::reference::ir::initialize(ref, &stop_status);
-    gko::kernels::omp::ir::initialize(omp, &d_stop_status);
+    gko::kernels::EXEC_NAMESPACE::ir::initialize(exec, &d_stop_status);
 
     auto tmp = gko::Array<gko::stopping_status>(ref, d_stop_status);
     for (int i = 0; i < stop_status.get_num_elems(); ++i) {
@@ -103,9 +104,9 @@ TEST_F(Ir, ApplyIsEquivalentToRef)
     auto mtx = gen_mtx(50, 50);
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
-    auto d_mtx = clone(omp, mtx);
-    auto d_x = clone(omp, x);
-    auto d_b = clone(omp, b);
+    auto d_mtx = clone(exec, mtx);
+    auto d_x = clone(exec, x);
+    auto d_b = clone(exec, b);
     // Forget about accuracy - Richardson is not going to converge for a random
     // matrix, just check that a couple of iterations gives the same result on
     // both executors
@@ -117,8 +118,8 @@ TEST_F(Ir, ApplyIsEquivalentToRef)
     auto d_ir_factory =
         gko::solver::Ir<>::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(omp))
-            .on(omp);
+                gko::stop::Iteration::build().with_max_iters(2u).on(exec))
+            .on(exec);
     auto solver = ir_factory->generate(std::move(mtx));
     auto d_solver = d_ir_factory->generate(std::move(d_mtx));
 
@@ -134,9 +135,9 @@ TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef)
     auto mtx = gen_mtx(50, 50);
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
-    auto d_mtx = clone(omp, mtx);
-    auto d_x = clone(omp, x);
-    auto d_b = clone(omp, b);
+    auto d_mtx = clone(exec, mtx);
+    auto d_x = clone(exec, x);
+    auto d_b = clone(exec, b);
 
     auto ir_factory =
         gko::solver::Ir<>::build()
@@ -155,11 +156,11 @@ TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef)
                 gko::solver::Gmres<>::build()
                     .with_criteria(
                         gko::stop::Iteration::build().with_max_iters(1u).on(
-                            omp))
-                    .on(omp))
+                            exec))
+                    .on(exec))
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(omp))
-            .on(omp);
+                gko::stop::Iteration::build().with_max_iters(2u).on(exec))
+            .on(exec);
     auto solver = ir_factory->generate(std::move(mtx));
     auto d_solver = d_ir_factory->generate(std::move(d_mtx));
 
@@ -177,9 +178,9 @@ TEST_F(Ir, RichardsonApplyIsEquivalentToRef)
     auto mtx = gen_mtx(50, 50);
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
-    auto d_mtx = clone(omp, mtx);
-    auto d_x = clone(omp, x);
-    auto d_b = clone(omp, b);
+    auto d_mtx = clone(exec, mtx);
+    auto d_x = clone(exec, x);
+    auto d_b = clone(exec, b);
     // Forget about accuracy - Richardson is not going to converge for a random
     // matrix, just check that a couple of iterations gives the same result on
     // both executors
@@ -192,9 +193,9 @@ TEST_F(Ir, RichardsonApplyIsEquivalentToRef)
     auto d_ir_factory =
         gko::solver::Ir<>::build()
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(omp))
+                gko::stop::Iteration::build().with_max_iters(2u).on(exec))
             .with_relaxation_factor(0.9)
-            .on(omp);
+            .on(exec);
     auto solver = ir_factory->generate(std::move(mtx));
     auto d_solver = d_ir_factory->generate(std::move(d_mtx));
 
@@ -210,9 +211,9 @@ TEST_F(Ir, RichardsonApplyWithIterativeInnerSolverIsEquivalentToRef)
     auto mtx = gen_mtx(50, 50);
     auto x = gen_mtx(50, 3);
     auto b = gen_mtx(50, 3);
-    auto d_mtx = clone(omp, mtx);
-    auto d_x = clone(omp, x);
-    auto d_b = clone(omp, b);
+    auto d_mtx = clone(exec, mtx);
+    auto d_x = clone(exec, x);
+    auto d_b = clone(exec, b);
     auto ir_factory =
         gko::solver::Ir<>::build()
             .with_solver(
@@ -231,12 +232,12 @@ TEST_F(Ir, RichardsonApplyWithIterativeInnerSolverIsEquivalentToRef)
                 gko::solver::Gmres<>::build()
                     .with_criteria(
                         gko::stop::Iteration::build().with_max_iters(1u).on(
-                            omp))
-                    .on(omp))
+                            exec))
+                    .on(exec))
             .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(2u).on(omp))
+                gko::stop::Iteration::build().with_max_iters(2u).on(exec))
             .with_relaxation_factor(0.9)
-            .on(omp);
+            .on(exec);
     auto solver = ir_factory->generate(std::move(mtx));
     auto d_solver = d_ir_factory->generate(std::move(d_mtx));
 
diff --git a/test/utils/executor.hpp b/test/utils/executor.hpp
new file mode 100644
index 00000000000..c30add581cf
--- /dev/null
+++ b/test/utils/executor.hpp
@@ -0,0 +1,78 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_TEST_UTILS_EXECUTOR_HPP_
+#define GKO_TEST_UTILS_EXECUTOR_HPP_
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+void init_executor(std::shared_ptr<gko::ReferenceExecutor> ref,
+                   std::shared_ptr<gko::OmpExecutor> &exec)
+{
+    exec = gko::OmpExecutor::create();
+}
+
+
+void init_executor(std::shared_ptr<gko::ReferenceExecutor> ref,
+                   std::shared_ptr<gko::CudaExecutor> &exec)
+{
+    ASSERT_GT(gko::CudaExecutor::get_num_devices(), 0);
+    exec = gko::CudaExecutor::create(0, ref);
+}
+
+
+void init_executor(std::shared_ptr<gko::ReferenceExecutor> ref,
+                   std::shared_ptr<gko::HipExecutor> &exec)
+{
+    ASSERT_GT(gko::HipExecutor::get_num_devices(), 0);
+    exec = gko::HipExecutor::create(0, ref);
+}
+
+
+void init_executor(std::shared_ptr<gko::ReferenceExecutor> ref,
+                   std::shared_ptr<gko::DpcppExecutor> &exec)
+{
+    if (gko::DpcppExecutor::get_num_devices("gpu") > 0) {
+        exec = gko::DpcppExecutor::create(0, ref, "gpu");
+    } else if (gko::DpcppExecutor::get_num_devices("cpu") > 0) {
+        exec = gko::DpcppExecutor::create(0, ref, "cpu");
+    } else {
+        FAIL() << "No suitable DPC++ devices";
+    }
+}
+
+
+#endif  // GKO_TEST_UTILS_EXECUTOR_HPP_

From 1f1a078c0ea819e967c29272964f48564b5beb6b Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 9 Jun 2021 00:40:51 +0200
Subject: [PATCH 0827/1008] add tests for stride handling and zero division

---
 test/solver/bicg_kernels.cpp     | 20 +++++++++++++++-----
 test/solver/bicgstab_kernels.cpp | 23 ++++++++++++++++++-----
 test/solver/cg_kernels.cpp       | 20 +++++++++++++++-----
 test/solver/cgs_kernels.cpp      | 20 +++++++++++++++-----
 test/solver/fcg_kernels.cpp      | 20 +++++++++++++++-----
 5 files changed, 78 insertions(+), 25 deletions(-)

diff --git a/test/solver/bicg_kernels.cpp b/test/solver/bicg_kernels.cpp
index 8c2e21b9dc1..48f8b4aed69 100644
--- a/test/solver/bicg_kernels.cpp
+++ b/test/solver/bicg_kernels.cpp
@@ -88,7 +88,8 @@ class Bicg : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    std::unique_ptr<Mtx> gen_mtx(gko::size_type num_rows,
+                                 gko::size_type num_cols)
     {
         return gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
@@ -98,9 +99,10 @@ class Bicg : public ::testing::Test {
 
     void initialize_data()
     {
-        int m = 597;
-        int n = 43;
-        b = gen_mtx(m, n);
+        gko::size_type m = 597;
+        gko::size_type n = 43;
+        b_full = gen_mtx(m, n + 2);
+        b = b_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
         r = gen_mtx(m, n);
         z = gen_mtx(m, n);
         p = gen_mtx(m, n);
@@ -109,15 +111,21 @@ class Bicg : public ::testing::Test {
         z2 = gen_mtx(m, n);
         p2 = gen_mtx(m, n);
         q2 = gen_mtx(m, n);
-        x = gen_mtx(m, n);
+        x_full = gen_mtx(m, n + 3);
+        x = x_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
         beta = gen_mtx(1, n);
         prev_rho = gen_mtx(1, n);
         rho = gen_mtx(1, n);
+        // check correct handling for zero values
+        beta->at(2) = 0.0;
+        prev_rho->at(2) = 0.0;
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(ref, n));
         for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
             stop_status->get_data()[i].reset();
         }
+        // check correct handling for stopped columns
+        stop_status->get_data()[1].stop(1);
 
         d_b = Mtx::create(exec);
         d_b->copy_from(b.get());
@@ -155,6 +163,7 @@ class Bicg : public ::testing::Test {
 
     std::ranlux48 rand_engine;
 
+    std::unique_ptr<Mtx> b_full;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
     std::unique_ptr<Mtx> z;
@@ -164,6 +173,7 @@ class Bicg : public ::testing::Test {
     std::unique_ptr<Mtx> z2;
     std::unique_ptr<Mtx> p2;
     std::unique_ptr<Mtx> q2;
+    std::unique_ptr<Mtx> x_full;
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> beta;
     std::unique_ptr<Mtx> prev_rho;
diff --git a/test/solver/bicgstab_kernels.cpp b/test/solver/bicgstab_kernels.cpp
index 307c7e4ac0e..3eef55e5799 100644
--- a/test/solver/bicgstab_kernels.cpp
+++ b/test/solver/bicgstab_kernels.cpp
@@ -98,7 +98,8 @@ class Bicgstab : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    std::unique_ptr<Mtx> gen_mtx(gko::size_type num_rows,
+                                 gko::size_type num_cols)
     {
         return gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
@@ -108,10 +109,12 @@ class Bicgstab : public ::testing::Test {
 
     void initialize_data()
     {
-        int m = 597;
-        int n = 17;
-        x = gen_mtx(m, n);
-        b = gen_mtx(m, n);
+        gko::size_type m = 597;
+        gko::size_type n = 17;
+        x_full = gen_mtx(m, n + 3);
+        b_full = gen_mtx(m, n + 2);
+        x = x_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
+        b = b_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
         r = gen_mtx(m, n);
         z = gen_mtx(m, n);
         p = gen_mtx(m, n);
@@ -126,11 +129,19 @@ class Bicgstab : public ::testing::Test {
         beta = gen_mtx(1, n);
         gamma = gen_mtx(1, n);
         omega = gen_mtx(1, n);
+        // check correct handling for zero values
+        prev_rho->at(2) = 0.0;
+        prev_rho->at(4) = 0.0;
+        beta->at(2) = 0.0;
+        omega->at(2) = 0.0;
+        omega->at(3) = 0.0;
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(ref, n));
         for (size_t i = 0; i < n; ++i) {
             stop_status->get_data()[i].reset();
         }
+        // check correct handling for stopped columns
+        stop_status->get_data()[1].stop(1);
 
         d_x = Mtx::create(exec);
         d_b = Mtx::create(exec);
@@ -181,6 +192,8 @@ class Bicgstab : public ::testing::Test {
     std::unique_ptr<Solver::Factory> exec_bicgstab_factory;
     std::unique_ptr<Solver::Factory> ref_bicgstab_factory;
 
+    std::unique_ptr<Mtx> x_full;
+    std::unique_ptr<Mtx> b_full;
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
diff --git a/test/solver/cg_kernels.cpp b/test/solver/cg_kernels.cpp
index 129cc54631d..8c6f1e1c2d5 100644
--- a/test/solver/cg_kernels.cpp
+++ b/test/solver/cg_kernels.cpp
@@ -73,7 +73,8 @@ class Cg : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    std::unique_ptr<Mtx> gen_mtx(gko::size_type num_rows,
+                                 gko::size_type num_cols)
     {
         return gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
@@ -83,22 +84,29 @@ class Cg : public ::testing::Test {
 
     void initialize_data()
     {
-        int m = 597;
-        int n = 43;
-        b = gen_mtx(m, n);
+        gko::size_type m = 597;
+        gko::size_type n = 43;
+        b_full = gen_mtx(m, n + 2);
+        b = b_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
         r = gen_mtx(m, n);
         z = gen_mtx(m, n);
         p = gen_mtx(m, n);
         q = gen_mtx(m, n);
-        x = gen_mtx(m, n);
+        x_full = gen_mtx(m, n + 3);
+        x = x_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
         beta = gen_mtx(1, n);
         prev_rho = gen_mtx(1, n);
         rho = gen_mtx(1, n);
+        // check correct handling for zero values
+        beta->at(2) = 0.0;
+        prev_rho->at(2) = 0.0;
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(ref, n));
         for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
             stop_status->get_data()[i].reset();
         }
+        // check correct handling for stopped columns
+        stop_status->get_data()[1].stop(1);
 
         d_b = Mtx::create(exec);
         d_b->copy_from(b.get());
@@ -128,11 +136,13 @@ class Cg : public ::testing::Test {
 
     std::ranlux48 rand_engine;
 
+    std::unique_ptr<Mtx> b_full;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
     std::unique_ptr<Mtx> z;
     std::unique_ptr<Mtx> p;
     std::unique_ptr<Mtx> q;
+    std::unique_ptr<Mtx> x_full;
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> beta;
     std::unique_ptr<Mtx> prev_rho;
diff --git a/test/solver/cgs_kernels.cpp b/test/solver/cgs_kernels.cpp
index 6b6b812c1e1..078c2efa1fa 100644
--- a/test/solver/cgs_kernels.cpp
+++ b/test/solver/cgs_kernels.cpp
@@ -96,7 +96,8 @@ class Cgs : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    std::unique_ptr<Mtx> gen_mtx(gko::size_type num_rows,
+                                 gko::size_type num_cols)
     {
         return gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
@@ -106,9 +107,10 @@ class Cgs : public ::testing::Test {
 
     void initialize_data()
     {
-        int m = 597;
-        int n = 43;
-        b = gen_mtx(m, n);
+        gko::size_type m = 597;
+        gko::size_type n = 43;
+        b_full = gen_mtx(m, n + 2);
+        b = b_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
         r = gen_mtx(m, n);
         r_tld = gen_mtx(m, n);
         p = gen_mtx(m, n);
@@ -117,17 +119,23 @@ class Cgs : public ::testing::Test {
         u_hat = gen_mtx(m, n);
         v_hat = gen_mtx(m, n);
         t = gen_mtx(m, n);
-        x = gen_mtx(m, n);
+        x_full = gen_mtx(m, n + 3);
+        x = x_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
         alpha = gen_mtx(1, n);
         beta = gen_mtx(1, n);
         gamma = gen_mtx(1, n);
         rho = gen_mtx(1, n);
         rho_prev = gen_mtx(1, n);
+        // check correct handling for zero values
+        gamma->at(2) = 0.0;
+        rho_prev->at(2) = 0.0;
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(ref, n));
         for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
             stop_status->get_data()[i].reset();
         }
+        // check correct handling for stopped columns
+        stop_status->get_data()[1].stop(1);
 
         d_b = Mtx::create(exec);
         d_b->copy_from(b.get());
@@ -176,6 +184,7 @@ class Cgs : public ::testing::Test {
     std::unique_ptr<Solver::Factory> exec_cgs_factory;
     std::unique_ptr<Solver::Factory> ref_cgs_factory;
 
+    std::unique_ptr<Mtx> b_full;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
     std::unique_ptr<Mtx> r_tld;
@@ -185,6 +194,7 @@ class Cgs : public ::testing::Test {
     std::unique_ptr<Mtx> u;
     std::unique_ptr<Mtx> u_hat;
     std::unique_ptr<Mtx> v_hat;
+    std::unique_ptr<Mtx> x_full;
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> alpha;
     std::unique_ptr<Mtx> beta;
diff --git a/test/solver/fcg_kernels.cpp b/test/solver/fcg_kernels.cpp
index fd7a6280628..3a25bc45dd0 100644
--- a/test/solver/fcg_kernels.cpp
+++ b/test/solver/fcg_kernels.cpp
@@ -75,7 +75,8 @@ class Fcg : public ::testing::Test {
         }
     }
 
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    std::unique_ptr<Mtx> gen_mtx(gko::size_type num_rows,
+                                 gko::size_type num_cols)
     {
         return gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
@@ -85,24 +86,31 @@ class Fcg : public ::testing::Test {
 
     void initialize_data()
     {
-        int m = 597;
-        int n = 43;
-        b = gen_mtx(m, n);
+        gko::size_type m = 597;
+        gko::size_type n = 43;
+        b_full = gen_mtx(m, n + 2);
+        b = b_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
         r = gen_mtx(m, n);
         t = gen_mtx(m, n);
         z = gen_mtx(m, n);
         p = gen_mtx(m, n);
         q = gen_mtx(m, n);
-        x = gen_mtx(m, n);
+        x_full = gen_mtx(m, n + 3);
+        x = x_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
         beta = gen_mtx(1, n);
         prev_rho = gen_mtx(1, n);
         rho = gen_mtx(1, n);
         rho_t = gen_mtx(1, n);
+        // check correct handling for zero values
+        beta->at(2) = 0.0;
+        prev_rho->at(2) = 0.0;
         stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(ref, n));
         for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
             stop_status->get_data()[i].reset();
         }
+        // check correct handling for stopped columns
+        stop_status->get_data()[1].stop(1);
 
         d_b = Mtx::create(exec);
         d_b->copy_from(b.get());
@@ -136,12 +144,14 @@ class Fcg : public ::testing::Test {
 
     std::ranlux48 rand_engine;
 
+    std::unique_ptr<Mtx> b_full;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
     std::unique_ptr<Mtx> t;
     std::unique_ptr<Mtx> z;
     std::unique_ptr<Mtx> p;
     std::unique_ptr<Mtx> q;
+    std::unique_ptr<Mtx> x_full;
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> beta;
     std::unique_ptr<Mtx> prev_rho;

From 0a4d062356deb16e2d3d7484fdd0f9952f546198 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 9 Jun 2021 00:57:25 +0200
Subject: [PATCH 0828/1008] review updates

* remove 1d device_unpack
* use 2D indices in OpenMP and SYCL

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
---
 cuda/base/kernel_launch.cuh     | 12 +----------
 dpcpp/base/kernel_launch.dp.hpp | 16 ++++----------
 hip/base/kernel_launch.hip.hpp  | 12 +----------
 omp/base/kernel_launch.hpp      | 38 +++++++++------------------------
 4 files changed, 16 insertions(+), 62 deletions(-)

diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index 2caec536e53..0f8b43430a1 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -164,16 +164,6 @@ const cuda_type<ValueType> *vector(const matrix::Dense<ValueType> *mtx)
 }
 
 
-template <typename T>
-struct device_unpack_1d_impl {
-    using type = T;
-    static __device__ __forceinline__ type unpack(T param, size_type, size_type)
-    {
-        return param;
-    }
-};
-
-
 template <typename T>
 struct device_unpack_2d_impl {
     using type = T;
@@ -204,7 +194,7 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_1d(
     if (tidx >= size) {
         return;
     }
-    fn(tidx, device_unpack_1d_impl<KernelArgs>::unpack(args, tidx, size)...);
+    fn(tidx, args...);
 }
 
 
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
index 6387431c931..807e6df3c5c 100644
--- a/dpcpp/base/kernel_launch.dp.hpp
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -158,13 +158,6 @@ const ValueType *vector(const matrix::Dense<ValueType> *mtx)
 }
 
 
-template <typename T>
-struct device_unpack_1d_impl {
-    using type = T;
-    static type unpack(T param, size_type, size_type) { return param; }
-};
-
-
 template <typename T>
 struct device_unpack_2d_impl {
     using type = T;
@@ -191,7 +184,7 @@ void generic_kernel_1d(sycl::handler &cgh, size_type size, KernelFunction fn,
 {
     cgh.parallel_for(sycl::range<1>{size}, [=](sycl::id<1> idx_id) {
         auto idx = static_cast<size_type>(idx_id[0]);
-        fn(idx, device_unpack_1d_impl<KernelArgs>::unpack(args, idx, size)...);
+        fn(idx, args...);
     });
 }
 
@@ -200,10 +193,9 @@ template <typename KernelFunction, typename... KernelArgs>
 void generic_kernel_2d(sycl::handler &cgh, size_type rows, size_type cols,
                        KernelFunction fn, KernelArgs... args)
 {
-    cgh.parallel_for(sycl::range<1>{rows * cols}, [=](sycl::id<1> idx_id) {
-        auto idx = static_cast<size_type>(idx_id[0]);
-        auto row = idx / cols;
-        auto col = idx % cols;
+    cgh.parallel_for(sycl::range<2>{rows, cols}, [=](sycl::id<2> idx) {
+        auto row = static_cast<size_type>(idx[0]);
+        auto col = static_cast<size_type>(idx[1]);
         fn(row, col,
            device_unpack_2d_impl<KernelArgs>::unpack(args, row, col, rows,
                                                      cols)...);
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index 350fb301dbd..c9916d0d90b 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -167,16 +167,6 @@ const hip_type<ValueType> *vector(const matrix::Dense<ValueType> *mtx)
 }
 
 
-template <typename T>
-struct device_unpack_1d_impl {
-    using type = T;
-    static __device__ __forceinline__ type unpack(T param, size_type, size_type)
-    {
-        return param;
-    }
-};
-
-
 template <typename T>
 struct device_unpack_2d_impl {
     using type = T;
@@ -207,7 +197,7 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_1d(
     if (tidx >= size) {
         return;
     }
-    fn(tidx, device_unpack_1d_impl<KernelArgs>::unpack(args, tidx, size)...);
+    fn(tidx, args...);
 }
 
 
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index 546d53adfd4..456e1596e6b 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -155,13 +155,6 @@ const ValueType *vector(const matrix::Dense<ValueType> *mtx)
 }
 
 
-template <typename T>
-struct device_unpack_1d_impl {
-    using type = T;
-    static type unpack(T param, size_type, size_type) { return param; }
-};
-
-
 template <typename T>
 struct device_unpack_2d_impl {
     using type = T;
@@ -182,15 +175,6 @@ struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
 };
 
 
-template <typename T>
-typename device_unpack_1d_impl<typename to_device_type_impl<T>::type>::type
-map_unpack_to_device(T &&param, size_type i, size_type size)
-{
-    return device_unpack_1d_impl<typename to_device_type_impl<T>::type>::unpack(
-        to_device_type_impl<T>::map_to_device(param), i, size);
-}
-
-
 template <typename T>
 typename device_unpack_2d_impl<typename to_device_type_impl<T>::type>::type
 map_unpack_to_device(T &&param, size_type row, size_type col,
@@ -212,9 +196,7 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
 {
 #pragma omp parallel for
     for (size_type i = 0; i < size; i++) {
-        [&]() {
-            fn(i, kernels::omp::map_unpack_to_device(args, i, size)...);
-        }();
+        [&]() { fn(i, kernels::omp::map_to_device(args)...); }();
     }
 }
 
@@ -222,15 +204,15 @@ template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
                 dim<2> size, KernelArgs &&... args)
 {
-#pragma omp parallel for
-    for (size_type i = 0; i < size[0] * size[1]; i++) {
-        auto row = i / size[1];
-        auto col = i % size[1];
-        [&]() {
-            fn(row, col,
-               kernels::omp::map_unpack_to_device(args, row, col, size[0],
-                                                  size[1])...);
-        }();
+#pragma omp parallel for collapse(2)
+    for (size_type row = 0; row < size[0]; row++) {
+        for (size_type col = 0; col < size[1]; col++) {
+            [&]() {
+                fn(row, col,
+                   kernels::omp::map_unpack_to_device(args, row, col, size[0],
+                                                      size[1])...);
+            }();
+        }
     }
 }
 

From 67957bbf3679c0d77190f9cc68bc591dea5712ab Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 9 Jun 2021 01:00:55 +0200
Subject: [PATCH 0829/1008] use consistent zero assignment in common kernels

---
 common/solver/bicg_kernels.cpp | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/common/solver/bicg_kernels.cpp b/common/solver/bicg_kernels.cpp
index 992b7cdb26b..e84ae5d9ec2 100644
--- a/common/solver/bicg_kernels.cpp
+++ b/common/solver/bicg_kernels.cpp
@@ -72,12 +72,8 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             }
             r(row, col) = b(row, col);
             r2(row, col) = b(row, col);
-            z(row, col) = zero(z(row, col));
-            p(row, col) = zero(p(row, col));
-            q(row, col) = zero(q(row, col));
-            z2(row, col) = zero(z2(row, col));
-            p2(row, col) = zero(p2(row, col));
-            q2(row, col) = zero(q2(row, col));
+            z(row, col) = p(row, col) = q(row, col) = z2(row, col) =
+                p2(row, col) = q2(row, col) = zero(z(row, col));
         },
         b->get_size(), b, compact(r), compact(z), compact(p), compact(q),
         vector(prev_rho), vector(rho), compact(r2), compact(z2), compact(p2),

From 2f85a55700f8688aeb5b8f66136b76a2e16a2c20 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 9 Jun 2021 01:19:43 +0200
Subject: [PATCH 0830/1008] remove unstable bicg test

---
 test/solver/bicg_kernels.cpp | 37 ------------------------------------
 1 file changed, 37 deletions(-)

diff --git a/test/solver/bicg_kernels.cpp b/test/solver/bicg_kernels.cpp
index 48f8b4aed69..e984ff7de6a 100644
--- a/test/solver/bicg_kernels.cpp
+++ b/test/solver/bicg_kernels.cpp
@@ -335,41 +335,4 @@ TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
 }
 
 
-TEST_F(Bicg, ApplyWithRandomMatrixIsEquivalentToRef)
-{
-    auto mtx = gen_mtx(50, 50);
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
-    auto d_mtx = Mtx::create(exec);
-    d_mtx->copy_from(mtx.get());
-    auto d_x = Mtx::create(exec);
-    d_x->copy_from(x.get());
-    auto d_b = Mtx::create(exec);
-    d_b->copy_from(b.get());
-    auto bicg_factory =
-        gko::solver::Bicg<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(ref))
-            .on(ref);
-    auto d_bicg_factory =
-        gko::solver::Bicg<>::build()
-            .with_criteria(
-                gko::stop::Iteration::build().with_max_iters(50u).on(exec),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
-                    .on(exec))
-            .on(exec);
-    auto solver = bicg_factory->generate(std::move(mtx));
-    auto d_solver = d_bicg_factory->generate(std::move(d_mtx));
-
-    solver->apply(b.get(), x.get());
-    d_solver->apply(d_b.get(), d_x.get());
-
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-}
-
-
 }  // namespace

From 3f8a3d822f81dbe9baf8786489d2f83a3b0df41e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 9 Jun 2021 03:33:55 +0200
Subject: [PATCH 0831/1008] add kernel launch tests

---
 cmake/create_test.cmake              |  12 +-
 cuda/base/kernel_launch.cuh          |  73 ++++----
 cuda/test/base/CMakeLists.txt        |   1 +
 cuda/test/base/kernel_launch.cu      | 269 +++++++++++++++++++++++++++
 dpcpp/base/kernel_launch.dp.hpp      |  71 +++----
 dpcpp/test/base/CMakeLists.txt       |   1 +
 dpcpp/test/base/kernel_launch.dp.cpp | 242 ++++++++++++++++++++++++
 hip/base/kernel_launch.hip.hpp       |  73 ++++----
 hip/test/base/CMakeLists.txt         |   1 +
 hip/test/base/kernel_launch.hip.cpp  | 269 +++++++++++++++++++++++++++
 omp/base/kernel_launch.hpp           |  71 +++----
 omp/test/CMakeLists.txt              |   1 +
 omp/test/base/CMakeLists.txt         |   3 +
 omp/test/base/kernel_launch.cpp      | 235 +++++++++++++++++++++++
 test/CMakeLists.txt                  |   2 +
 15 files changed, 1181 insertions(+), 143 deletions(-)
 create mode 100644 cuda/test/base/kernel_launch.cu
 create mode 100644 dpcpp/test/base/kernel_launch.dp.cpp
 create mode 100644 hip/test/base/kernel_launch.hip.cpp
 create mode 100644 omp/test/base/CMakeLists.txt
 create mode 100644 omp/test/base/kernel_launch.cpp

diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index cf5728a79d2..98fd1377909 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -36,7 +36,7 @@ function(ginkgo_create_test test_name)
 endfunction(ginkgo_create_test)
 
 function(ginkgo_create_dpcpp_test test_name)
-    ginkgo_build_test_name(${test_name} TEST_TARGET_NAME)
+    ginkgo_build_test_name(${test_name} test_target_name)
     add_executable(${test_target_name} ${test_name}.dp.cpp)
     target_compile_features(${test_target_name} PUBLIC cxx_std_17)
     target_compile_options(${test_target_name} PRIVATE "${GINKGO_DPCPP_FLAGS}")
@@ -71,6 +71,15 @@ function(ginkgo_create_cuda_test test_name)
     target_compile_features(${test_target_name} PUBLIC cxx_std_14)
     target_compile_options(${test_target_name}
         PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:${GINKGO_CUDA_ARCH_FLAGS}>")
+    if(MSVC)
+        target_compile_options(${test_target_name}
+            PRIVATE
+                $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda --expt-relaxed-constexpr>)
+    else()
+        target_compile_options(${test_target_name}
+            PRIVATE
+                $<$<COMPILE_LANGUAGE:CUDA>:--expt-extended-lambda --expt-relaxed-constexpr>)
+    endif()
     target_link_libraries(${test_target_name} PRIVATE ${ARGN})
     # we handle CUDA architecture flags for now, disable CMake handling
     if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
@@ -97,6 +106,7 @@ ginkgo_build_test_name(${test_name} test_target_name)
             HIPCC_OPTIONS ${GINKGO_TEST_HIP_DEFINES} # ${GINKGO_HIPCC_OPTIONS}
             # NVCC_OPTIONS  ${GINKGO_TEST_HIP_DEFINES} ${GINKGO_HIP_NVCC_OPTIONS}
             # CLANG_OPTIONS ${GINKGO_TEST_HIP_DEFINES} ${GINKGO_HIP_CLANG_OPTIONS}
+            --expt-relaxed-constexpr --expt-extended-lambda
             )
     else() # hcc/clang
         hip_add_executable(${test_target_name} ${test_name}.hip.cpp
diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index 0f8b43430a1..cfe444c55c3 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -128,42 +128,6 @@ struct compact_dense_wrapper {
 };
 
 
-template <typename ValueType>
-compact_dense_wrapper<cuda_type<ValueType>> compact(
-    matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {as_cuda_type(mtx->get_values())};
-}
-
-
-template <typename ValueType>
-compact_dense_wrapper<const cuda_type<ValueType>> compact(
-    const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {as_cuda_type(mtx->get_const_values())};
-}
-
-
-template <typename ValueType>
-cuda_type<ValueType> *vector(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return as_cuda_type(mtx->get_values());
-}
-
-
-template <typename ValueType>
-const cuda_type<ValueType> *vector(const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return as_cuda_type(mtx->get_const_values());
-}
-
-
 template <typename T>
 struct device_unpack_2d_impl {
     using type = T;
@@ -218,6 +182,43 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
 }  // namespace kernels
 
 
+template <typename ValueType>
+kernels::cuda::compact_dense_wrapper<kernels::cuda::cuda_type<ValueType>>
+compact(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {kernels::cuda::as_cuda_type(mtx->get_values())};
+}
+
+
+template <typename ValueType>
+kernels::cuda::compact_dense_wrapper<const kernels::cuda::cuda_type<ValueType>>
+compact(const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {kernels::cuda::as_cuda_type(mtx->get_const_values())};
+}
+
+
+template <typename ValueType>
+kernels::cuda::cuda_type<ValueType> *vector(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return kernels::cuda::as_cuda_type(mtx->get_values());
+}
+
+
+template <typename ValueType>
+const kernels::cuda::cuda_type<ValueType> *vector(
+    const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return kernels::cuda::as_cuda_type(mtx->get_const_values());
+}
+
+
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const CudaExecutor> exec, KernelFunction fn,
                 size_type size, KernelArgs &&... args)
diff --git a/cuda/test/base/CMakeLists.txt b/cuda/test/base/CMakeLists.txt
index 4759ff90e14..63318636a2b 100644
--- a/cuda/test/base/CMakeLists.txt
+++ b/cuda/test/base/CMakeLists.txt
@@ -5,5 +5,6 @@ if(GINKGO_HAVE_HWLOC)
     ginkgo_create_cuda_test(cuda_executor_topology NUMA::NUMA)
 endif()
 ginkgo_create_cuda_test(exception_helpers)
+ginkgo_create_cuda_test(kernel_launch)
 ginkgo_create_cuda_test(lin_op)
 ginkgo_create_cuda_test(math)
diff --git a/cuda/test/base/kernel_launch.cu b/cuda/test/base/kernel_launch.cu
new file mode 100644
index 00000000000..cc4113b34ef
--- /dev/null
+++ b/cuda/test/base/kernel_launch.cu
@@ -0,0 +1,269 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "cuda/base/kernel_launch.cuh"
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+using gko::dim;
+using gko::size_type;
+using std::is_same;
+
+
+class KernelLaunch : public ::testing::Test {
+protected:
+    KernelLaunch()
+        : exec(gko::CudaExecutor::create(0, gko::ReferenceExecutor::create())),
+          zero_array(exec->get_master(), 16),
+          iota_array(exec->get_master(), 16),
+          iota_transp_array(exec->get_master(), 16),
+          iota_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
+          zero_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
+          zero_dense_padded(
+              gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 5)),
+          vec_dense(gko::matrix::Dense<>::create(exec, dim<2>{1, 4}))
+    {
+        auto ref_iota_dense = gko::matrix::Dense<>::create(exec, dim<2>{4, 4});
+        for (int i = 0; i < 16; i++) {
+            zero_array.get_data()[i] = 0;
+            iota_array.get_data()[i] = i;
+            iota_transp_array.get_data()[i] = (i % 4 * 4) + i / 4;
+            ref_iota_dense->at(i / 4, i % 4) = i;
+        }
+        zero_dense->fill(0.0);
+        zero_dense_padded->fill(0.0);
+        iota_dense->copy_from(ref_iota_dense.get());
+        zero_array.set_executor(exec);
+        iota_array.set_executor(exec);
+        iota_transp_array.set_executor(exec);
+    }
+
+    std::shared_ptr<gko::CudaExecutor> exec;
+    gko::Array<int> zero_array;
+    gko::Array<int> iota_array;
+    gko::Array<int> iota_transp_array;
+    std::unique_ptr<gko::matrix::Dense<>> iota_dense;
+    std::unique_ptr<gko::matrix::Dense<>> zero_dense;
+    std::unique_ptr<gko::matrix::Dense<>> zero_dense_padded;
+    std::unique_ptr<gko::matrix::Dense<>> vec_dense;
+};
+
+
+// nvcc doesn't like device lambdas declared in complex classes, move it out
+void run1d(std::shared_ptr<gko::CudaExecutor> exec, size_type dim, int *data)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto d) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            d[i] = i;
+        },
+        dim, data);
+}
+
+TEST_F(KernelLaunch, Runs1D)
+{
+    run1d(exec, zero_array.get_num_elems(), zero_array.get_data());
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_array);
+}
+
+
+void run1d(std::shared_ptr<gko::CudaExecutor> exec, gko::Array<int> &data)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto d, auto d_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            static_assert(is_same<decltype(d_ptr), const int *>::value, "type");
+            if (d == d_ptr) {
+                d[i] = i;
+            } else {
+                d[i] = 0;
+            }
+        },
+        data.get_num_elems(), data, data.get_const_data());
+}
+
+TEST_F(KernelLaunch, Runs1DArray)
+{
+    run1d(exec, zero_array);
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_array);
+}
+
+
+void run1d(std::shared_ptr<gko::CudaExecutor> exec, gko::matrix::Dense<> *m)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto d, auto d2, auto d_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d(0, 0)), double &>::value, "type");
+            static_assert(is_same<decltype(d2(0, 0)), const double &>::value,
+                          "type");
+            static_assert(is_same<decltype(d_ptr), const double *>::value,
+                          "type");
+            bool pointers_correct = d.data == d_ptr && d2.data == d_ptr;
+            bool strides_correct = d.stride == 5 && d2.stride == 5;
+            bool accessors_2d_correct =
+                &d(0, 0) == d_ptr && &d(1, 0) == d_ptr + d.stride &&
+                &d2(0, 0) == d_ptr && &d2(1, 0) == d_ptr + d.stride;
+            bool accessors_1d_correct = &d[0] == d_ptr && &d2[0] == d_ptr;
+            if (pointers_correct && strides_correct && accessors_2d_correct &&
+                accessors_1d_correct) {
+                d(i / 4, i % 4) = i;
+            } else {
+                d(i / 4, i % 4) = 0;
+            }
+        },
+        16, m, static_cast<const gko::matrix::Dense<> *>(m),
+        m->get_const_values());
+}
+
+TEST_F(KernelLaunch, Runs1DDense)
+{
+    run1d(exec, zero_dense_padded.get());
+
+    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+}
+
+
+void run2d(std::shared_ptr<gko::CudaExecutor> exec, int *data)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto j, auto d) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(j), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            d[i + 4 * j] = 4 * i + j;
+        },
+        dim<2>{4, 4}, data);
+}
+
+TEST_F(KernelLaunch, Runs2D)
+{
+    run2d(exec, zero_array.get_data());
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_transp_array);
+}
+
+
+void run2d(std::shared_ptr<gko::CudaExecutor> exec, gko::Array<int> &data)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto j, auto d, auto d_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(j), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            static_assert(is_same<decltype(d_ptr), const int *>::value, "type");
+            if (d == d_ptr) {
+                d[i + 4 * j] = 4 * i + j;
+            } else {
+                d[i + 4 * j] = 0;
+            }
+        },
+        dim<2>{4, 4}, data, data.get_const_data());
+}
+
+TEST_F(KernelLaunch, Runs2DArray)
+{
+    run2d(exec, zero_array);
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_transp_array);
+}
+
+
+void run2d(std::shared_ptr<gko::CudaExecutor> exec, gko::matrix::Dense<> *m1,
+           gko::matrix::Dense<> *m2, gko::matrix::Dense<> *m3)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto j, auto d, auto d2, auto d_ptr, auto d3,
+                      auto d4, auto d2_ptr, auto d3_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d(0, 0)), double &>::value, "type");
+            static_assert(is_same<decltype(d2(0, 0)), const double &>::value,
+                          "type");
+            static_assert(is_same<decltype(d_ptr), const double *>::value,
+                          "type");
+            static_assert(is_same<decltype(d3(0, 0)), double &>::value, "type");
+            static_assert(is_same<decltype(d4), double *>::value, "type");
+            static_assert(is_same<decltype(d2_ptr), double *>::value, "type");
+            static_assert(is_same<decltype(d3_ptr), double *>::value, "type");
+            bool pointers_correct = d.data == d_ptr && d2.data == d_ptr &&
+                                    d3.data == d2_ptr && d4 == d3_ptr;
+            bool strides_correct =
+                d.stride == 5 && d2.stride == 5 && d3.stride == 4;
+            bool accessors_2d_correct =
+                &d(0, 0) == d_ptr && &d(1, 0) == d_ptr + d.stride &&
+                &d2(0, 0) == d_ptr && &d2(1, 0) == d_ptr + d2.stride &&
+                &d3(0, 0) == d2_ptr && &d3(1, 0) == d2_ptr + d3.stride;
+            bool accessors_1d_correct =
+                &d[0] == d_ptr && &d2[0] == d_ptr && &d3[0] == d2_ptr;
+            if (pointers_correct && strides_correct && accessors_2d_correct &&
+                accessors_1d_correct) {
+                d(i, j) = 4 * i + j;
+            } else {
+                d(i, j) = 0;
+            }
+        },
+        dim<2>{4, 4}, m1, static_cast<const gko::matrix::Dense<> *>(m1),
+        m1->get_const_values(), gko::compact(m2), gko::vector(m3),
+        m2->get_values(), m3->get_values());
+}
+
+TEST_F(KernelLaunch, Runs2DDense)
+{
+    run2d(exec, zero_dense_padded.get(), zero_dense.get(), vec_dense.get());
+
+    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+}
+
+
+}  // namespace
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
index 807e6df3c5c..4a3db0dd45e 100644
--- a/dpcpp/base/kernel_launch.dp.hpp
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -123,41 +123,6 @@ struct compact_dense_wrapper {
 };
 
 
-template <typename ValueType>
-compact_dense_wrapper<ValueType> compact(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {mtx->get_values()};
-}
-
-
-template <typename ValueType>
-compact_dense_wrapper<const ValueType> compact(
-    const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {mtx->get_const_values()};
-}
-
-
-template <typename ValueType>
-ValueType *vector(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return mtx->get_values();
-}
-
-
-template <typename ValueType>
-const ValueType *vector(const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return mtx->get_const_values();
-}
-
-
 template <typename T>
 struct device_unpack_2d_impl {
     using type = T;
@@ -207,6 +172,42 @@ void generic_kernel_2d(sycl::handler &cgh, size_type rows, size_type cols,
 }  // namespace kernels
 
 
+template <typename ValueType>
+kernels::dpcpp::compact_dense_wrapper<ValueType> compact(
+    matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {mtx->get_values()};
+}
+
+
+template <typename ValueType>
+kernels::dpcpp::compact_dense_wrapper<const ValueType> compact(
+    const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {mtx->get_const_values()};
+}
+
+
+template <typename ValueType>
+ValueType *vector(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return mtx->get_values();
+}
+
+
+template <typename ValueType>
+const ValueType *vector(const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return mtx->get_const_values();
+}
+
+
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const DpcppExecutor> exec, KernelFunction fn,
                 size_type size, KernelArgs &&... args)
diff --git a/dpcpp/test/base/CMakeLists.txt b/dpcpp/test/base/CMakeLists.txt
index c31c0f1527e..8ed2d944eea 100644
--- a/dpcpp/test/base/CMakeLists.txt
+++ b/dpcpp/test/base/CMakeLists.txt
@@ -1,2 +1,3 @@
 ginkgo_create_dpcpp_test(executor)
 ginkgo_create_dpcpp_test(dim3)
+ginkgo_create_dpcpp_test(kernel_launch)
diff --git a/dpcpp/test/base/kernel_launch.dp.cpp b/dpcpp/test/base/kernel_launch.dp.cpp
new file mode 100644
index 00000000000..62cb89c2103
--- /dev/null
+++ b/dpcpp/test/base/kernel_launch.dp.cpp
@@ -0,0 +1,242 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "dpcpp/base/kernel_launch.dp.hpp"
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+using gko::dim;
+using gko::size_type;
+using std::is_same;
+
+
+class KernelLaunch : public ::testing::Test {
+protected:
+    KernelLaunch()
+        : exec(gko::DpcppExecutor::create(
+              0, gko::ReferenceExecutor::create(),
+              gko::DpcppExecutor::get_num_devices("gpu") > 0 ? "gpu" : "cpu")),
+          zero_array(exec->get_master(), 16),
+          iota_array(exec->get_master(), 16),
+          iota_transp_array(exec->get_master(), 16),
+          iota_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
+          zero_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
+          zero_dense_padded(
+              gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 5)),
+          vec_dense(gko::matrix::Dense<>::create(exec, dim<2>{1, 4}))
+    {
+        auto ref_iota_dense = gko::matrix::Dense<>::create(exec, dim<2>{4, 4});
+        for (int i = 0; i < 16; i++) {
+            zero_array.get_data()[i] = 0;
+            iota_array.get_data()[i] = i;
+            iota_transp_array.get_data()[i] = (i % 4 * 4) + i / 4;
+            ref_iota_dense->at(i / 4, i % 4) = i;
+        }
+        zero_dense->fill(0.0);
+        zero_dense_padded->fill(0.0);
+        iota_dense->copy_from(ref_iota_dense.get());
+        zero_array.set_executor(exec);
+        iota_array.set_executor(exec);
+        iota_transp_array.set_executor(exec);
+    }
+
+    std::shared_ptr<gko::DpcppExecutor> exec;
+    gko::Array<int> zero_array;
+    gko::Array<int> iota_array;
+    gko::Array<int> iota_transp_array;
+    std::unique_ptr<gko::matrix::Dense<>> iota_dense;
+    std::unique_ptr<gko::matrix::Dense<>> zero_dense;
+    std::unique_ptr<gko::matrix::Dense<>> zero_dense_padded;
+    std::unique_ptr<gko::matrix::Dense<>> vec_dense;
+};
+
+
+TEST_F(KernelLaunch, Runs1D)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto d) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            d[i] = i;
+        },
+        zero_array.get_num_elems(), zero_array.get_data());
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_array);
+}
+
+
+TEST_F(KernelLaunch, Runs1DArray)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto d, auto d_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            static_assert(is_same<decltype(d_ptr), const int *>::value, "type");
+            if (d == d_ptr) {
+                d[i] = i;
+            } else {
+                d[i] = 0;
+            }
+        },
+        zero_array.get_num_elems(), zero_array, zero_array.get_const_data());
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_array);
+}
+
+
+TEST_F(KernelLaunch, Runs1DDense)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto d, auto d2, auto d_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d(0, 0)), double &>::value, "type");
+            static_assert(is_same<decltype(d2(0, 0)), const double &>::value,
+                          "type");
+            static_assert(is_same<decltype(d_ptr), const double *>::value,
+                          "type");
+            bool pointers_correct = d.data == d_ptr && d2.data == d_ptr;
+            bool strides_correct = d.stride == 5 && d2.stride == 5;
+            bool accessors_2d_correct =
+                &d(0, 0) == d_ptr && &d(1, 0) == d_ptr + d.stride &&
+                &d2(0, 0) == d_ptr && &d2(1, 0) == d_ptr + d.stride;
+            bool accessors_1d_correct = &d[0] == d_ptr && &d2[0] == d_ptr;
+            if (pointers_correct && strides_correct && accessors_2d_correct &&
+                accessors_1d_correct) {
+                d(i / 4, i % 4) = i;
+            } else {
+                d(i / 4, i % 4) = 0;
+            }
+        },
+        16, zero_dense_padded.get(),
+        static_cast<const gko::matrix::Dense<> *>(zero_dense_padded.get()),
+        zero_dense_padded->get_const_values());
+
+    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+}
+
+
+TEST_F(KernelLaunch, Runs2D)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto j, auto d) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(j), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            d[i + 4 * j] = 4 * i + j;
+        },
+        dim<2>{4, 4}, zero_array.get_data());
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_transp_array);
+}
+
+
+TEST_F(KernelLaunch, Runs2DArray)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto j, auto d, auto d_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(j), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            static_assert(is_same<decltype(d_ptr), const int *>::value, "type");
+            if (d == d_ptr) {
+                d[i + 4 * j] = 4 * i + j;
+            } else {
+                d[i + 4 * j] = 0;
+            }
+        },
+        dim<2>{4, 4}, zero_array, zero_array.get_const_data());
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_transp_array);
+}
+
+
+TEST_F(KernelLaunch, Runs2DDense)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto j, auto d, auto d2, auto d_ptr, auto d3,
+                      auto d4, auto d2_ptr, auto d3_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d(0, 0)), double &>::value, "type");
+            static_assert(is_same<decltype(d2(0, 0)), const double &>::value,
+                          "type");
+            static_assert(is_same<decltype(d_ptr), const double *>::value,
+                          "type");
+            static_assert(is_same<decltype(d3(0, 0)), double &>::value, "type");
+            static_assert(is_same<decltype(d4), double *>::value, "type");
+            static_assert(is_same<decltype(d2_ptr), double *>::value, "type");
+            static_assert(is_same<decltype(d3_ptr), double *>::value, "type");
+            bool pointers_correct = d.data == d_ptr && d2.data == d_ptr &&
+                                    d3.data == d2_ptr && d4 == d3_ptr;
+            bool strides_correct =
+                d.stride == 5 && d2.stride == 5 && d3.stride == 4;
+            bool accessors_2d_correct =
+                &d(0, 0) == d_ptr && &d(1, 0) == d_ptr + d.stride &&
+                &d2(0, 0) == d_ptr && &d2(1, 0) == d_ptr + d2.stride &&
+                &d3(0, 0) == d2_ptr && &d3(1, 0) == d2_ptr + d3.stride;
+            bool accessors_1d_correct =
+                &d[0] == d_ptr && &d2[0] == d_ptr && &d3[0] == d2_ptr;
+            if (pointers_correct && strides_correct && accessors_2d_correct &&
+                accessors_1d_correct) {
+                d(i, j) = 4 * i + j;
+            } else {
+                d(i, j) = 0;
+            }
+        },
+        dim<2>{4, 4}, zero_dense_padded.get(),
+        static_cast<const gko::matrix::Dense<> *>(zero_dense_padded.get()),
+        zero_dense_padded->get_const_values(), gko::compact(zero_dense.get()),
+        gko::vector(vec_dense.get()), zero_dense->get_values(),
+        vec_dense->get_values());
+
+    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+}
+
+
+}  // namespace
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index c9916d0d90b..f4f07e6f1ad 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -131,42 +131,6 @@ struct compact_dense_wrapper {
 };
 
 
-template <typename ValueType>
-compact_dense_wrapper<hip_type<ValueType>> compact(
-    matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {as_hip_type(mtx->get_values())};
-}
-
-
-template <typename ValueType>
-compact_dense_wrapper<const hip_type<ValueType>> compact(
-    const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {as_hip_type(mtx->get_const_values())};
-}
-
-
-template <typename ValueType>
-hip_type<ValueType> *vector(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return as_hip_type(mtx->get_values());
-}
-
-
-template <typename ValueType>
-const hip_type<ValueType> *vector(const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return as_hip_type(mtx->get_const_values());
-}
-
-
 template <typename T>
 struct device_unpack_2d_impl {
     using type = T;
@@ -221,6 +185,43 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
 }  // namespace kernels
 
 
+template <typename ValueType>
+kernels::hip::compact_dense_wrapper<kernels::hip::hip_type<ValueType>> compact(
+    matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {kernels::hip::as_hip_type(mtx->get_values())};
+}
+
+
+template <typename ValueType>
+kernels::hip::compact_dense_wrapper<const kernels::hip::hip_type<ValueType>>
+compact(const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {kernels::hip::as_hip_type(mtx->get_const_values())};
+}
+
+
+template <typename ValueType>
+kernels::hip::hip_type<ValueType> *vector(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return kernels::hip::as_hip_type(mtx->get_values());
+}
+
+
+template <typename ValueType>
+const kernels::hip::hip_type<ValueType> *vector(
+    const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return kernels::hip::as_hip_type(mtx->get_const_values());
+}
+
+
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const HipExecutor> exec, KernelFunction fn,
                 size_type size, KernelArgs &&... args)
diff --git a/hip/test/base/CMakeLists.txt b/hip/test/base/CMakeLists.txt
index 62527c1764b..9659cf465aa 100644
--- a/hip/test/base/CMakeLists.txt
+++ b/hip/test/base/CMakeLists.txt
@@ -3,6 +3,7 @@ if(GINKGO_HAVE_HWLOC)
     find_package(NUMA REQUIRED)
     ginkgo_create_hip_test(hip_executor_topology NUMA::NUMA)
 endif()
+ginkgo_create_hip_test(kernel_launch)
 ginkgo_create_hip_test(lin_op)
 ginkgo_create_hip_test(math)
 # Only hcc needs the libraries. nvcc only requires the headers.
diff --git a/hip/test/base/kernel_launch.hip.cpp b/hip/test/base/kernel_launch.hip.cpp
new file mode 100644
index 00000000000..e88805cb4e2
--- /dev/null
+++ b/hip/test/base/kernel_launch.hip.cpp
@@ -0,0 +1,269 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "hip/base/kernel_launch.hip.hpp"
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+using gko::dim;
+using gko::size_type;
+using std::is_same;
+
+
+class KernelLaunch : public ::testing::Test {
+protected:
+    KernelLaunch()
+        : exec(gko::HipExecutor::create(0, gko::ReferenceExecutor::create())),
+          zero_array(exec->get_master(), 16),
+          iota_array(exec->get_master(), 16),
+          iota_transp_array(exec->get_master(), 16),
+          iota_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
+          zero_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
+          zero_dense_padded(
+              gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 5)),
+          vec_dense(gko::matrix::Dense<>::create(exec, dim<2>{1, 4}))
+    {
+        auto ref_iota_dense = gko::matrix::Dense<>::create(exec, dim<2>{4, 4});
+        for (int i = 0; i < 16; i++) {
+            zero_array.get_data()[i] = 0;
+            iota_array.get_data()[i] = i;
+            iota_transp_array.get_data()[i] = (i % 4 * 4) + i / 4;
+            ref_iota_dense->at(i / 4, i % 4) = i;
+        }
+        zero_dense->fill(0.0);
+        zero_dense_padded->fill(0.0);
+        iota_dense->copy_from(ref_iota_dense.get());
+        zero_array.set_executor(exec);
+        iota_array.set_executor(exec);
+        iota_transp_array.set_executor(exec);
+    }
+
+    std::shared_ptr<gko::HipExecutor> exec;
+    gko::Array<int> zero_array;
+    gko::Array<int> iota_array;
+    gko::Array<int> iota_transp_array;
+    std::unique_ptr<gko::matrix::Dense<>> iota_dense;
+    std::unique_ptr<gko::matrix::Dense<>> zero_dense;
+    std::unique_ptr<gko::matrix::Dense<>> zero_dense_padded;
+    std::unique_ptr<gko::matrix::Dense<>> vec_dense;
+};
+
+
+// nvcc doesn't like device lambdas declared in complex classes, move it out
+void run1d(std::shared_ptr<gko::HipExecutor> exec, size_type dim, int *data)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto d) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            d[i] = i;
+        },
+        dim, data);
+}
+
+TEST_F(KernelLaunch, Runs1D)
+{
+    run1d(exec, zero_array.get_num_elems(), zero_array.get_data());
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_array);
+}
+
+
+void run1d(std::shared_ptr<gko::HipExecutor> exec, gko::Array<int> &data)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto d, auto d_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            static_assert(is_same<decltype(d_ptr), const int *>::value, "type");
+            if (d == d_ptr) {
+                d[i] = i;
+            } else {
+                d[i] = 0;
+            }
+        },
+        data.get_num_elems(), data, data.get_const_data());
+}
+
+TEST_F(KernelLaunch, Runs1DArray)
+{
+    run1d(exec, zero_array);
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_array);
+}
+
+
+void run1d(std::shared_ptr<gko::HipExecutor> exec, gko::matrix::Dense<> *m)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto d, auto d2, auto d_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d(0, 0)), double &>::value, "type");
+            static_assert(is_same<decltype(d2(0, 0)), const double &>::value,
+                          "type");
+            static_assert(is_same<decltype(d_ptr), const double *>::value,
+                          "type");
+            bool pointers_correct = d.data == d_ptr && d2.data == d_ptr;
+            bool strides_correct = d.stride == 5 && d2.stride == 5;
+            bool accessors_2d_correct =
+                &d(0, 0) == d_ptr && &d(1, 0) == d_ptr + d.stride &&
+                &d2(0, 0) == d_ptr && &d2(1, 0) == d_ptr + d.stride;
+            bool accessors_1d_correct = &d[0] == d_ptr && &d2[0] == d_ptr;
+            if (pointers_correct && strides_correct && accessors_2d_correct &&
+                accessors_1d_correct) {
+                d(i / 4, i % 4) = i;
+            } else {
+                d(i / 4, i % 4) = 0;
+            }
+        },
+        16, m, static_cast<const gko::matrix::Dense<> *>(m),
+        m->get_const_values());
+}
+
+TEST_F(KernelLaunch, Runs1DDense)
+{
+    run1d(exec, zero_dense_padded.get());
+
+    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+}
+
+
+void run2d(std::shared_ptr<gko::HipExecutor> exec, int *data)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto j, auto d) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(j), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            d[i + 4 * j] = 4 * i + j;
+        },
+        dim<2>{4, 4}, data);
+}
+
+TEST_F(KernelLaunch, Runs2D)
+{
+    run2d(exec, zero_array.get_data());
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_transp_array);
+}
+
+
+void run2d(std::shared_ptr<gko::HipExecutor> exec, gko::Array<int> &data)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto j, auto d, auto d_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(j), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            static_assert(is_same<decltype(d_ptr), const int *>::value, "type");
+            if (d == d_ptr) {
+                d[i + 4 * j] = 4 * i + j;
+            } else {
+                d[i + 4 * j] = 0;
+            }
+        },
+        dim<2>{4, 4}, data, data.get_const_data());
+}
+
+TEST_F(KernelLaunch, Runs2DArray)
+{
+    run2d(exec, zero_array);
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_transp_array);
+}
+
+
+void run2d(std::shared_ptr<gko::HipExecutor> exec, gko::matrix::Dense<> *m1,
+           gko::matrix::Dense<> *m2, gko::matrix::Dense<> *m3)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto j, auto d, auto d2, auto d_ptr, auto d3,
+                      auto d4, auto d2_ptr, auto d3_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d(0, 0)), double &>::value, "type");
+            static_assert(is_same<decltype(d2(0, 0)), const double &>::value,
+                          "type");
+            static_assert(is_same<decltype(d_ptr), const double *>::value,
+                          "type");
+            static_assert(is_same<decltype(d3(0, 0)), double &>::value, "type");
+            static_assert(is_same<decltype(d4), double *>::value, "type");
+            static_assert(is_same<decltype(d2_ptr), double *>::value, "type");
+            static_assert(is_same<decltype(d3_ptr), double *>::value, "type");
+            bool pointers_correct = d.data == d_ptr && d2.data == d_ptr &&
+                                    d3.data == d2_ptr && d4 == d3_ptr;
+            bool strides_correct =
+                d.stride == 5 && d2.stride == 5 && d3.stride == 4;
+            bool accessors_2d_correct =
+                &d(0, 0) == d_ptr && &d(1, 0) == d_ptr + d.stride &&
+                &d2(0, 0) == d_ptr && &d2(1, 0) == d_ptr + d2.stride &&
+                &d3(0, 0) == d2_ptr && &d3(1, 0) == d2_ptr + d3.stride;
+            bool accessors_1d_correct =
+                &d[0] == d_ptr && &d2[0] == d_ptr && &d3[0] == d2_ptr;
+            if (pointers_correct && strides_correct && accessors_2d_correct &&
+                accessors_1d_correct) {
+                d(i, j) = 4 * i + j;
+            } else {
+                d(i, j) = 0;
+            }
+        },
+        dim<2>{4, 4}, m1, static_cast<const gko::matrix::Dense<> *>(m1),
+        m1->get_const_values(), gko::compact(m2), gko::vector(m3),
+        m2->get_values(), m3->get_values());
+}
+
+TEST_F(KernelLaunch, Runs2DDense)
+{
+    run2d(exec, zero_dense_padded.get(), zero_dense.get(), vec_dense.get());
+
+    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+}
+
+
+}  // namespace
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index 456e1596e6b..5beffd12669 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -120,41 +120,6 @@ struct compact_dense_wrapper {
 };
 
 
-template <typename ValueType>
-compact_dense_wrapper<ValueType> compact(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {mtx->get_values()};
-}
-
-
-template <typename ValueType>
-compact_dense_wrapper<const ValueType> compact(
-    const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {mtx->get_const_values()};
-}
-
-
-template <typename ValueType>
-ValueType *vector(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return mtx->get_values();
-}
-
-
-template <typename ValueType>
-const ValueType *vector(const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return mtx->get_const_values();
-}
-
-
 template <typename T>
 struct device_unpack_2d_impl {
     using type = T;
@@ -190,6 +155,42 @@ map_unpack_to_device(T &&param, size_type row, size_type col,
 }  // namespace kernels
 
 
+template <typename ValueType>
+kernels::omp::compact_dense_wrapper<ValueType> compact(
+    matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {mtx->get_values()};
+}
+
+
+template <typename ValueType>
+kernels::omp::compact_dense_wrapper<const ValueType> compact(
+    const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
+    return {mtx->get_const_values()};
+}
+
+
+template <typename ValueType>
+ValueType *vector(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return mtx->get_values();
+}
+
+
+template <typename ValueType>
+const ValueType *vector(const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return mtx->get_const_values();
+}
+
+
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
                 size_type size, KernelArgs &&... args)
diff --git a/omp/test/CMakeLists.txt b/omp/test/CMakeLists.txt
index 3759a39bc77..35590c60df1 100644
--- a/omp/test/CMakeLists.txt
+++ b/omp/test/CMakeLists.txt
@@ -1,5 +1,6 @@
 include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
 
+add_subdirectory(base)
 add_subdirectory(components)
 add_subdirectory(factorization)
 add_subdirectory(matrix)
diff --git a/omp/test/base/CMakeLists.txt b/omp/test/base/CMakeLists.txt
new file mode 100644
index 00000000000..475e27ff3df
--- /dev/null
+++ b/omp/test/base/CMakeLists.txt
@@ -0,0 +1,3 @@
+ginkgo_create_test(kernel_launch)
+find_package(OpenMP 3.0 REQUIRED)
+target_link_libraries(omp_test_base_kernel_launch PRIVATE OpenMP::OpenMP_CXX)
\ No newline at end of file
diff --git a/omp/test/base/kernel_launch.cpp b/omp/test/base/kernel_launch.cpp
new file mode 100644
index 00000000000..5a82b90bcf2
--- /dev/null
+++ b/omp/test/base/kernel_launch.cpp
@@ -0,0 +1,235 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "omp/base/kernel_launch.hpp"
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+using gko::dim;
+using gko::size_type;
+using std::is_same;
+
+
+class KernelLaunch : public ::testing::Test {
+protected:
+    KernelLaunch()
+        : exec(gko::OmpExecutor::create()),
+          zero_array(exec, 16),
+          iota_array(exec, 16),
+          iota_transp_array(exec, 16),
+          iota_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
+          zero_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
+          zero_dense_padded(
+              gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 5)),
+          vec_dense(gko::matrix::Dense<>::create(exec, dim<2>{1, 4}))
+    {
+        for (int i = 0; i < 16; i++) {
+            zero_array.get_data()[i] = 0;
+            iota_array.get_data()[i] = i;
+            iota_transp_array.get_data()[i] = (i % 4 * 4) + i / 4;
+            iota_dense->at(i / 4, i % 4) = i;
+            zero_dense->at(i / 4, i % 4) = 0.0;
+            zero_dense_padded->at(i / 4, i % 4) = 0.0;
+        }
+    }
+
+    std::shared_ptr<gko::OmpExecutor> exec;
+    gko::Array<int> zero_array;
+    gko::Array<int> iota_array;
+    gko::Array<int> iota_transp_array;
+    std::unique_ptr<gko::matrix::Dense<>> iota_dense;
+    std::unique_ptr<gko::matrix::Dense<>> zero_dense;
+    std::unique_ptr<gko::matrix::Dense<>> zero_dense_padded;
+    std::unique_ptr<gko::matrix::Dense<>> vec_dense;
+};
+
+
+TEST_F(KernelLaunch, Runs1D)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto d) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            d[i] = i;
+        },
+        zero_array.get_num_elems(), zero_array.get_data());
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_array);
+}
+
+
+TEST_F(KernelLaunch, Runs1DArray)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto d, auto d_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            static_assert(is_same<decltype(d_ptr), const int *>::value, "type");
+            if (d == d_ptr) {
+                d[i] = i;
+            } else {
+                d[i] = 0;
+            }
+        },
+        zero_array.get_num_elems(), zero_array, zero_array.get_const_data());
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_array);
+}
+
+
+TEST_F(KernelLaunch, Runs1DDense)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto d, auto d2, auto d_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d(0, 0)), double &>::value, "type");
+            static_assert(is_same<decltype(d2(0, 0)), const double &>::value,
+                          "type");
+            static_assert(is_same<decltype(d_ptr), const double *>::value,
+                          "type");
+            bool pointers_correct = d.data == d_ptr && d2.data == d_ptr;
+            bool strides_correct = d.stride == 5 && d2.stride == 5;
+            bool accessors_2d_correct =
+                &d(0, 0) == d_ptr && &d(1, 0) == d_ptr + d.stride &&
+                &d2(0, 0) == d_ptr && &d2(1, 0) == d_ptr + d.stride;
+            bool accessors_1d_correct = &d[0] == d_ptr && &d2[0] == d_ptr;
+            if (pointers_correct && strides_correct && accessors_2d_correct &&
+                accessors_1d_correct) {
+                d(i / 4, i % 4) = i;
+            } else {
+                d(i / 4, i % 4) = 0;
+            }
+        },
+        16, zero_dense_padded.get(),
+        static_cast<const gko::matrix::Dense<> *>(zero_dense_padded.get()),
+        zero_dense_padded->get_const_values());
+
+    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+}
+
+
+TEST_F(KernelLaunch, Runs2D)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto j, auto d) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(j), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            d[i + 4 * j] = 4 * i + j;
+        },
+        dim<2>{4, 4}, zero_array.get_data());
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_transp_array);
+}
+
+
+TEST_F(KernelLaunch, Runs2DArray)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto j, auto d, auto d_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(j), size_type>::value, "index");
+            static_assert(is_same<decltype(d), int *>::value, "type");
+            static_assert(is_same<decltype(d_ptr), const int *>::value, "type");
+            if (d == d_ptr) {
+                d[i + 4 * j] = 4 * i + j;
+            } else {
+                d[i + 4 * j] = 0;
+            }
+        },
+        dim<2>{4, 4}, zero_array, zero_array.get_const_data());
+
+    GKO_ASSERT_ARRAY_EQ(zero_array, iota_transp_array);
+}
+
+
+TEST_F(KernelLaunch, Runs2DDense)
+{
+    gko::run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto j, auto d, auto d2, auto d_ptr, auto d3,
+                      auto d4, auto d2_ptr, auto d3_ptr) {
+            static_assert(is_same<decltype(i), size_type>::value, "index");
+            static_assert(is_same<decltype(d(0, 0)), double &>::value, "type");
+            static_assert(is_same<decltype(d2(0, 0)), const double &>::value,
+                          "type");
+            static_assert(is_same<decltype(d_ptr), const double *>::value,
+                          "type");
+            static_assert(is_same<decltype(d3(0, 0)), double &>::value, "type");
+            static_assert(is_same<decltype(d4), double *>::value, "type");
+            static_assert(is_same<decltype(d2_ptr), double *>::value, "type");
+            static_assert(is_same<decltype(d3_ptr), double *>::value, "type");
+            bool pointers_correct = d.data == d_ptr && d2.data == d_ptr &&
+                                    d3.data == d2_ptr && d4 == d3_ptr;
+            bool strides_correct =
+                d.stride == 5 && d2.stride == 5 && d3.stride == 4;
+            bool accessors_2d_correct =
+                &d(0, 0) == d_ptr && &d(1, 0) == d_ptr + d.stride &&
+                &d2(0, 0) == d_ptr && &d2(1, 0) == d_ptr + d2.stride &&
+                &d3(0, 0) == d2_ptr && &d3(1, 0) == d2_ptr + d3.stride;
+            bool accessors_1d_correct =
+                &d[0] == d_ptr && &d2[0] == d_ptr && &d3[0] == d2_ptr;
+            if (pointers_correct && strides_correct && accessors_2d_correct &&
+                accessors_1d_correct) {
+                d(i, j) = 4 * i + j;
+            } else {
+                d(i, j) = 0;
+            }
+        },
+        dim<2>{4, 4}, zero_dense_padded.get(),
+        static_cast<const gko::matrix::Dense<> *>(zero_dense_padded.get()),
+        zero_dense_padded->get_const_values(), gko::compact(zero_dense.get()),
+        gko::vector(vec_dense.get()), zero_dense->get_values(),
+        vec_dense->get_values());
+
+    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+}
+
+
+}  // namespace
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 55efa399aaf..8a32b18862d 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -1 +1,3 @@
+include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
+
 add_subdirectory(solver)
\ No newline at end of file

From f03ace10b2693e1af32dd0ebac9f73c3f3697a53 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 11 Jun 2021 11:36:15 +0200
Subject: [PATCH 0832/1008] simplify device_unpack_2d_impl parameters

---
 cuda/base/kernel_launch.cuh     | 10 +++-------
 dpcpp/base/kernel_launch.dp.hpp | 13 ++++---------
 hip/base/kernel_launch.hip.hpp  | 10 +++-------
 omp/base/kernel_launch.hpp      | 18 ++++++------------
 4 files changed, 16 insertions(+), 35 deletions(-)

diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index cfe444c55c3..ea62c049ccf 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -131,8 +131,7 @@ struct compact_dense_wrapper {
 template <typename T>
 struct device_unpack_2d_impl {
     using type = T;
-    static __device__ __forceinline__ type unpack(T param, size_type, size_type,
-                                                  size_type, size_type)
+    static __device__ __forceinline__ type unpack(T param, size_type)
     {
         return param;
     }
@@ -142,8 +141,7 @@ template <typename ValueType>
 struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
     using type = matrix_accessor<ValueType>;
     static __device__ __forceinline__ type
-    unpack(compact_dense_wrapper<ValueType> param, size_type, size_type,
-           size_type, size_type num_cols)
+    unpack(compact_dense_wrapper<ValueType> param, size_type num_cols)
     {
         return {param.data, num_cols};
     }
@@ -172,9 +170,7 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
     if (row >= rows) {
         return;
     }
-    fn(row, col,
-       device_unpack_2d_impl<KernelArgs>::unpack(args, row, col, rows,
-                                                 cols)...);
+    fn(row, col, device_unpack_2d_impl<KernelArgs>::unpack(args, cols)...);
 }
 
 
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
index 4a3db0dd45e..03b8f659d8c 100644
--- a/dpcpp/base/kernel_launch.dp.hpp
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -126,17 +126,14 @@ struct compact_dense_wrapper {
 template <typename T>
 struct device_unpack_2d_impl {
     using type = T;
-    static type unpack(T param, size_type, size_type, size_type, size_type)
-    {
-        return param;
-    }
+    static type unpack(T param, size_type) { return param; }
 };
 
 template <typename ValueType>
 struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
     using type = matrix_accessor<ValueType>;
-    static type unpack(compact_dense_wrapper<ValueType> param, size_type,
-                       size_type, size_type, size_type num_cols)
+    static type unpack(compact_dense_wrapper<ValueType> param,
+                       size_type num_cols)
     {
         return {param.data, num_cols};
     }
@@ -161,9 +158,7 @@ void generic_kernel_2d(sycl::handler &cgh, size_type rows, size_type cols,
     cgh.parallel_for(sycl::range<2>{rows, cols}, [=](sycl::id<2> idx) {
         auto row = static_cast<size_type>(idx[0]);
         auto col = static_cast<size_type>(idx[1]);
-        fn(row, col,
-           device_unpack_2d_impl<KernelArgs>::unpack(args, row, col, rows,
-                                                     cols)...);
+        fn(row, col, device_unpack_2d_impl<KernelArgs>::unpack(args, cols)...);
     });
 }
 
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index f4f07e6f1ad..0bf70a515f4 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -134,8 +134,7 @@ struct compact_dense_wrapper {
 template <typename T>
 struct device_unpack_2d_impl {
     using type = T;
-    static __device__ __forceinline__ type unpack(T param, size_type, size_type,
-                                                  size_type, size_type)
+    static __device__ __forceinline__ type unpack(T param, size_type)
     {
         return param;
     }
@@ -145,8 +144,7 @@ template <typename ValueType>
 struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
     using type = matrix_accessor<ValueType>;
     static __device__ __forceinline__ type
-    unpack(compact_dense_wrapper<ValueType> param, size_type, size_type,
-           size_type, size_type num_cols)
+    unpack(compact_dense_wrapper<ValueType> param, size_type num_cols)
     {
         return {param.data, num_cols};
     }
@@ -175,9 +173,7 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
     if (row >= rows) {
         return;
     }
-    fn(row, col,
-       device_unpack_2d_impl<KernelArgs>::unpack(args, row, col, rows,
-                                                 cols)...);
+    fn(row, col, device_unpack_2d_impl<KernelArgs>::unpack(args, cols)...);
 }
 
 
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index 5beffd12669..d93759cc7cb 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -123,17 +123,14 @@ struct compact_dense_wrapper {
 template <typename T>
 struct device_unpack_2d_impl {
     using type = T;
-    static type unpack(T param, size_type, size_type, size_type, size_type)
-    {
-        return param;
-    }
+    static type unpack(T param, size_type) { return param; }
 };
 
 template <typename ValueType>
 struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
     using type = matrix_accessor<ValueType>;
-    static type unpack(compact_dense_wrapper<ValueType> param, size_type,
-                       size_type, size_type, size_type num_cols)
+    static type unpack(compact_dense_wrapper<ValueType> param,
+                       size_type num_cols)
     {
         return {param.data, num_cols};
     }
@@ -142,12 +139,10 @@ struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
 
 template <typename T>
 typename device_unpack_2d_impl<typename to_device_type_impl<T>::type>::type
-map_unpack_to_device(T &&param, size_type row, size_type col,
-                     size_type num_rows, size_type num_cols)
+map_unpack_to_device(T &&param, size_type num_cols)
 {
     return device_unpack_2d_impl<typename to_device_type_impl<T>::type>::unpack(
-        to_device_type_impl<T>::map_to_device(param), row, col, num_rows,
-        num_cols);
+        to_device_type_impl<T>::map_to_device(param), num_cols);
 }
 
 
@@ -210,8 +205,7 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
         for (size_type col = 0; col < size[1]; col++) {
             [&]() {
                 fn(row, col,
-                   kernels::omp::map_unpack_to_device(args, row, col, size[0],
-                                                      size[1])...);
+                   kernels::omp::map_unpack_to_device(args, size[1])...);
             }();
         }
     }

From 6cdfba8c3848b31f616005bc9db7516764a0d4ac Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 11 Jun 2021 13:00:09 +0200
Subject: [PATCH 0833/1008] adapt test_install paths

---
 .gitlab-ci.yml |  2 +-
 CMakeLists.txt | 66 ++++++++++++++++++++++++++++++--------------------
 2 files changed, 41 insertions(+), 27 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4299f00a5bb..4c7267072b4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -121,7 +121,7 @@ include:
         (( $(ctest -N | tail -1 | sed 's/Total Tests: //') != 0 )) || exit 1
     - ctest -V
     - ninja test_install
-    - pushd test_install
+    - pushd test/build_install
     - ninja install
     - popd
     - |
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a9b11e2375c..c9691c27325 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -333,55 +333,69 @@ ginkgo_modify_flags(CMAKE_CUDA_FLAGS_DEBUG)
 ginkgo_modify_flags(CMAKE_CUDA_FLAGS_RELEASE)
 ginkgo_install()
 
+set(GINKGO_TEST_INSTALL_SRC_DIR "${Ginkgo_SOURCE_DIR}/test/build_install/")
+set(GINKGO_TEST_INSTALL_BIN_DIR "${Ginkgo_BINARY_DIR}/test/build_install/")
+set(GINKGO_TEST_EXPORTBUILD_SRC_DIR "${Ginkgo_SOURCE_DIR}/test/build_export/")
+set(GINKGO_TEST_EXPORTBUILD_BIN_DIR "${Ginkgo_BINARY_DIR}/test/build_export/")
 if(MSVC)
-    set(GINKGO_TEST_INSTALL_DIR "${Ginkgo_BINARY_DIR}/test/build_install/$<CONFIG>/")
-    set(GINKGO_TEST_EXPORTBUILD_DIR "${Ginkgo_BINARY_DIR}/test/build_export/$<CONFIG>/")
+    set(GINKGO_TEST_INSTALL_CMD ${GINKGO_TEST_INSTALL_BIN_DIR}/$<CONFIG>/test_install)
+    set(GINKGO_TEST_EXPORTBUILD_CMD ${GINKGO_TEST_EXPORTBUILD_BIN_DIR}/$<CONFIG>/test_exportbuild)
+    if(GINKGO_BUILD_CUDA)
+        set(GINKGO_TEST_INSTALL_CUDA_CMD ${GINKGO_TEST_INSTALL_BIN_DIR}/$<CONFIG>/test_install_cuda)
+    endif()
 else()
-    set(GINKGO_TEST_INSTALL_DIR "${Ginkgo_BINARY_DIR}/test/build_install/")
-    set(GINKGO_TEST_EXPORTBUILD_DIR "${Ginkgo_BINARY_DIR}/test/build_export/")
-endif()
-
-set(GINKGO_TEST_INSTALL_COMMAND "${GINKGO_TEST_INSTALL_DIR}/test_install")
-set(GINKGO_TEST_EXPORTBUILD_COMMAND "${GINKGO_TEST_EXPORTBUILD_DIR}/test_exportbuild")
-if(GINKGO_BUILD_CUDA)
-    set(GINKGO_TEST_INSTALL_CUDA_COMMAND "${GINKGO_TEST_EXPORTBUILD_DIR}/test_install_cuda")
+    set(GINKGO_TEST_INSTALL_CMD ${GINKGO_TEST_INSTALL_BIN_DIR}/test_install)
+    set(GINKGO_TEST_EXPORTBUILD_CMD ${GINKGO_TEST_EXPORTBUILD_BIN_DIR}/test_exportbuild)
+    if(GINKGO_BUILD_CUDA)
+        set(GINKGO_TEST_INSTALL_CUDA_CMD ${GINKGO_TEST_INSTALL_BIN_DIR}/test_install_cuda)
+    endif()
 endif()
 if(GINKGO_BUILD_HIP)
-    set(GINKGO_TEST_INSTALL_HIP_COMMAND "${GINKGO_TEST_INSTALL_DIR}/test_install_hip")
+    set(GINKGO_TEST_INSTALL_HIP_CMD ${GINKGO_TEST_INSTALL_BIN_DIR}/test_install_hip)
 endif()
 
-file(MAKE_DIRECTORY ${Ginkgo_BINARY_DIR}/test_install)
+file(MAKE_DIRECTORY "${GINKGO_TEST_INSTALL_BIN_DIR}")
+file(MAKE_DIRECTORY "${GINKGO_TEST_EXPORTBUILD_BIN_DIR}")
 set(TOOLSET "")
 if (NOT "${CMAKE_GENERATOR_TOOLSET}" STREQUAL "")
     set(TOOLSET "-T${CMAKE_GENERATOR_TOOLSET}")
 endif()
 add_custom_target(test_install
-    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${TOOLSET} -H${GINKGO_TEST_INSTALL_DIR}
-    -B${Ginkgo_BINARY_DIR}/test_install
+    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${TOOLSET}
+    -H${GINKGO_TEST_INSTALL_SRC_DIR}
+    -B${GINKGO_TEST_INSTALL_BIN_DIR}
     -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
     -DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX}/${GINKGO_INSTALL_CONFIG_DIR}
-    -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+    -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+    -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
     -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
-    WORKING_DIRECTORY ${Ginkgo_BINARY_DIR}/test_install
     # `--config cfg` is ignored by single-configuration generator.
     # `$<CONFIG>` is always be the same as `CMAKE_BUILD_TYPE` in
     # single-configuration generator.
-    COMMAND ${CMAKE_COMMAND} --build ${Ginkgo_BINARY_DIR}/test_install --config $<CONFIG>
-    COMMAND ${GINKGO_TEST_INSTALL_COMMAND}
-    COMMAND ${GINKGO_TEST_INSTALL_CUDA_COMMAND}
-    COMMAND ${GINKGO_TEST_INSTALL_HIP_COMMAND}
-    COMMENT "Running a test on the installed binaries. This requires running `(sudo) make install` first.")
+    COMMAND ${CMAKE_COMMAND}
+    --build ${GINKGO_TEST_INSTALL_BIN_DIR}
+    --config $<CONFIG>
+    COMMAND ${GINKGO_TEST_INSTALL_CMD}
+    COMMAND ${GINKGO_TEST_INSTALL_CUDA_CMD}
+    COMMAND ${GINKGO_TEST_INSTALL_HIP_CMD}
+    WORKING_DIRECTORY ${GINKGO_TEST_INSTALL_BIN_DIR}
+    COMMENT "Running a test on the installed binaries. "
+    "This requires running `(sudo) make install` first.")
 
 add_custom_target(test_exportbuild
-    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${TOOLSET} -H${GINKGO_TEST_EXPORTBUILD_DIR}
-    -B${Ginkgo_BINARY_DIR}/test_exportbuild
-    -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+    COMMAND ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} ${TOOLSET}
+    -H${GINKGO_TEST_EXPORTBUILD_SRC_DIR}
+    -B${GINKGO_TEST_EXPORTBUILD_BIN_DIR}
+    -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+    -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
     -DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}
     # `--config cfg` is ignored by single-configuration generator.
     # `$<CONFIG>` is always be the same as `CMAKE_BUILD_TYPE` in
     # single-configuration generator.
-    COMMAND ${CMAKE_COMMAND} --build ${Ginkgo_BINARY_DIR}/test_exportbuild --config $<CONFIG>
-    COMMAND ${GINKGO_TEST_EXPORTBUILD_COMMAND}
+    COMMAND ${CMAKE_COMMAND}
+    --build ${GINKGO_TEST_EXPORTBUILD_BIN_DIR}
+    --config $<CONFIG>
+    COMMAND ${GINKGO_TEST_EXPORTBUILD_CMD}
     COMMENT "Running a test on Ginkgo's exported build directory. "
     "This requires compiling Ginkgo with `-DGINKGO_EXPORT_BUILD_DIR=ON` first.")
 

From c1e73477cc1a5dd1826a9d9ade2a0bfcdb33be80 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 11 Jun 2021 13:27:57 +0200
Subject: [PATCH 0834/1008] fix kernel launch tests

---
 cuda/test/base/kernel_launch.cu      | 6 ++++--
 dpcpp/test/base/kernel_launch.dp.cpp | 3 ++-
 hip/test/base/kernel_launch.hip.cpp  | 3 ++-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/cuda/test/base/kernel_launch.cu b/cuda/test/base/kernel_launch.cu
index cc4113b34ef..801e93d6485 100644
--- a/cuda/test/base/kernel_launch.cu
+++ b/cuda/test/base/kernel_launch.cu
@@ -54,7 +54,8 @@ using std::is_same;
 class KernelLaunch : public ::testing::Test {
 protected:
     KernelLaunch()
-        : exec(gko::CudaExecutor::create(0, gko::ReferenceExecutor::create())),
+        : exec(gko::CudaExecutor::create(0, gko::ReferenceExecutor::create(),
+                                         false, gko::allocation_mode::device)),
           zero_array(exec->get_master(), 16),
           iota_array(exec->get_master(), 16),
           iota_transp_array(exec->get_master(), 16),
@@ -64,7 +65,8 @@ protected:
               gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 5)),
           vec_dense(gko::matrix::Dense<>::create(exec, dim<2>{1, 4}))
     {
-        auto ref_iota_dense = gko::matrix::Dense<>::create(exec, dim<2>{4, 4});
+        auto ref_iota_dense =
+            gko::matrix::Dense<>::create(exec->get_master(), dim<2>{4, 4});
         for (int i = 0; i < 16; i++) {
             zero_array.get_data()[i] = 0;
             iota_array.get_data()[i] = i;
diff --git a/dpcpp/test/base/kernel_launch.dp.cpp b/dpcpp/test/base/kernel_launch.dp.cpp
index 62cb89c2103..aa6c03c33dd 100644
--- a/dpcpp/test/base/kernel_launch.dp.cpp
+++ b/dpcpp/test/base/kernel_launch.dp.cpp
@@ -66,7 +66,8 @@ class KernelLaunch : public ::testing::Test {
               gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 5)),
           vec_dense(gko::matrix::Dense<>::create(exec, dim<2>{1, 4}))
     {
-        auto ref_iota_dense = gko::matrix::Dense<>::create(exec, dim<2>{4, 4});
+        auto ref_iota_dense =
+            gko::matrix::Dense<>::create(exec->get_master(), dim<2>{4, 4});
         for (int i = 0; i < 16; i++) {
             zero_array.get_data()[i] = 0;
             iota_array.get_data()[i] = i;
diff --git a/hip/test/base/kernel_launch.hip.cpp b/hip/test/base/kernel_launch.hip.cpp
index e88805cb4e2..d35fd92460e 100644
--- a/hip/test/base/kernel_launch.hip.cpp
+++ b/hip/test/base/kernel_launch.hip.cpp
@@ -64,7 +64,8 @@ class KernelLaunch : public ::testing::Test {
               gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 5)),
           vec_dense(gko::matrix::Dense<>::create(exec, dim<2>{1, 4}))
     {
-        auto ref_iota_dense = gko::matrix::Dense<>::create(exec, dim<2>{4, 4});
+        auto ref_iota_dense =
+            gko::matrix::Dense<>::create(exec->get_master(), dim<2>{4, 4});
         for (int i = 0; i < 16; i++) {
             zero_array.get_data()[i] = 0;
             iota_array.get_data()[i] = i;

From 7a26fd70cc2ae5f93b2683e4cbf26c09ace55c4a Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 11 Jun 2021 14:09:21 +0200
Subject: [PATCH 0835/1008] add support for DPCPP_SINGLE_MODE to tests

---
 cmake/create_test.cmake              |  4 ++
 dpcpp/test/base/kernel_launch.dp.cpp | 56 ++++++++++++--------
 test/solver/bicg_kernels.cpp         | 79 +++++++++++++++-------------
 test/solver/bicgstab_kernels.cpp     | 68 +++++++++++++-----------
 test/solver/cg_kernels.cpp           | 48 +++++++++--------
 test/solver/cgs_kernels.cpp          | 69 +++++++++++++-----------
 test/solver/fcg_kernels.cpp          | 47 +++++++++--------
 test/solver/ir_kernels.cpp           | 60 ++++++++++++---------
 8 files changed, 240 insertions(+), 191 deletions(-)

diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index 98fd1377909..e2d04e1893c 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -162,6 +162,10 @@ function(ginkgo_create_common_test test_name)
         target_compile_options(${test_target_name} PRIVATE ${GINKGO_COMPILER_FLAGS})
         target_compile_definitions(${test_target_name} PRIVATE EXEC_TYPE=${exec_type} EXEC_NAMESPACE=${exec})
         target_link_libraries(${test_target_name} PRIVATE ${ARGN})
+        # use float for DPC++ if necessary
+        if((exec STREQUAL "dpcpp") AND GINKGO_DPCPP_SINGLE_MODE)
+            target_compile_definitions(${test_target_name} PRIVATE GINKGO_COMMON_SINGLE_MODE=1)
+        endif() 
         ginkgo_set_test_target_properties(${test_name}_${exec} ${test_target_name})
     endforeach()
 endfunction(ginkgo_create_common_test)
\ No newline at end of file
diff --git a/dpcpp/test/base/kernel_launch.dp.cpp b/dpcpp/test/base/kernel_launch.dp.cpp
index aa6c03c33dd..ab03967e872 100644
--- a/dpcpp/test/base/kernel_launch.dp.cpp
+++ b/dpcpp/test/base/kernel_launch.dp.cpp
@@ -53,6 +53,13 @@ using std::is_same;
 
 class KernelLaunch : public ::testing::Test {
 protected:
+#if GINKGO_DPCPP_SINGLE_MODE
+    using value_type = float;
+#else
+    using value_type = double;
+#endif
+    using Mtx = gko::matrix::Dense<value_type>;
+
     KernelLaunch()
         : exec(gko::DpcppExecutor::create(
               0, gko::ReferenceExecutor::create(),
@@ -60,14 +67,12 @@ class KernelLaunch : public ::testing::Test {
           zero_array(exec->get_master(), 16),
           iota_array(exec->get_master(), 16),
           iota_transp_array(exec->get_master(), 16),
-          iota_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
-          zero_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
-          zero_dense_padded(
-              gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 5)),
-          vec_dense(gko::matrix::Dense<>::create(exec, dim<2>{1, 4}))
+          iota_dense(Mtx::create(exec, dim<2>{4, 4})),
+          zero_dense(Mtx::create(exec, dim<2>{4, 4})),
+          zero_dense_padded(Mtx::create(exec, dim<2>{4, 4}, 5)),
+          vec_dense(Mtx::create(exec, dim<2>{1, 4}))
     {
-        auto ref_iota_dense =
-            gko::matrix::Dense<>::create(exec->get_master(), dim<2>{4, 4});
+        auto ref_iota_dense = Mtx::create(exec->get_master(), dim<2>{4, 4});
         for (int i = 0; i < 16; i++) {
             zero_array.get_data()[i] = 0;
             iota_array.get_data()[i] = i;
@@ -86,10 +91,10 @@ class KernelLaunch : public ::testing::Test {
     gko::Array<int> zero_array;
     gko::Array<int> iota_array;
     gko::Array<int> iota_transp_array;
-    std::unique_ptr<gko::matrix::Dense<>> iota_dense;
-    std::unique_ptr<gko::matrix::Dense<>> zero_dense;
-    std::unique_ptr<gko::matrix::Dense<>> zero_dense_padded;
-    std::unique_ptr<gko::matrix::Dense<>> vec_dense;
+    std::unique_ptr<Mtx> iota_dense;
+    std::unique_ptr<Mtx> zero_dense;
+    std::unique_ptr<Mtx> zero_dense_padded;
+    std::unique_ptr<Mtx> vec_dense;
 };
 
 
@@ -134,10 +139,11 @@ TEST_F(KernelLaunch, Runs1DDense)
         exec,
         [] GKO_KERNEL(auto i, auto d, auto d2, auto d_ptr) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
-            static_assert(is_same<decltype(d(0, 0)), double &>::value, "type");
-            static_assert(is_same<decltype(d2(0, 0)), const double &>::value,
+            static_assert(is_same<decltype(d(0, 0)), value_type &>::value,
                           "type");
-            static_assert(is_same<decltype(d_ptr), const double *>::value,
+            static_assert(
+                is_same<decltype(d2(0, 0)), const value_type &>::value, "type");
+            static_assert(is_same<decltype(d_ptr), const value_type *>::value,
                           "type");
             bool pointers_correct = d.data == d_ptr && d2.data == d_ptr;
             bool strides_correct = d.stride == 5 && d2.stride == 5;
@@ -153,7 +159,7 @@ TEST_F(KernelLaunch, Runs1DDense)
             }
         },
         16, zero_dense_padded.get(),
-        static_cast<const gko::matrix::Dense<> *>(zero_dense_padded.get()),
+        static_cast<const Mtx *>(zero_dense_padded.get()),
         zero_dense_padded->get_const_values());
 
     GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
@@ -204,15 +210,19 @@ TEST_F(KernelLaunch, Runs2DDense)
         [] GKO_KERNEL(auto i, auto j, auto d, auto d2, auto d_ptr, auto d3,
                       auto d4, auto d2_ptr, auto d3_ptr) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
-            static_assert(is_same<decltype(d(0, 0)), double &>::value, "type");
-            static_assert(is_same<decltype(d2(0, 0)), const double &>::value,
+            static_assert(is_same<decltype(d(0, 0)), value_type &>::value,
+                          "type");
+            static_assert(
+                is_same<decltype(d2(0, 0)), const value_type &>::value, "type");
+            static_assert(is_same<decltype(d_ptr), const value_type *>::value,
+                          "type");
+            static_assert(is_same<decltype(d3(0, 0)), value_type &>::value,
+                          "type");
+            static_assert(is_same<decltype(d4), value_type *>::value, "type");
+            static_assert(is_same<decltype(d2_ptr), value_type *>::value,
                           "type");
-            static_assert(is_same<decltype(d_ptr), const double *>::value,
+            static_assert(is_same<decltype(d3_ptr), value_type *>::value,
                           "type");
-            static_assert(is_same<decltype(d3(0, 0)), double &>::value, "type");
-            static_assert(is_same<decltype(d4), double *>::value, "type");
-            static_assert(is_same<decltype(d2_ptr), double *>::value, "type");
-            static_assert(is_same<decltype(d3_ptr), double *>::value, "type");
             bool pointers_correct = d.data == d_ptr && d2.data == d_ptr &&
                                     d3.data == d2_ptr && d4 == d3_ptr;
             bool strides_correct =
@@ -231,7 +241,7 @@ TEST_F(KernelLaunch, Runs2DDense)
             }
         },
         dim<2>{4, 4}, zero_dense_padded.get(),
-        static_cast<const gko::matrix::Dense<> *>(zero_dense_padded.get()),
+        static_cast<const Mtx *>(zero_dense_padded.get()),
         zero_dense_padded->get_const_values(), gko::compact(zero_dense.get()),
         gko::vector(vec_dense.get()), zero_dense->get_values(),
         vec_dense->get_values());
diff --git a/test/solver/bicg_kernels.cpp b/test/solver/bicg_kernels.cpp
index e984ff7de6a..f4b57867510 100644
--- a/test/solver/bicg_kernels.cpp
+++ b/test/solver/bicg_kernels.cpp
@@ -58,10 +58,15 @@ namespace {
 
 class Bicg : public ::testing::Test {
 protected:
-    using value_type = gko::default_precision;
+#if GINKGO_COMMON_SINGLE_MODE
+    using value_type = float;
+#else
+    using value_type = double;
+#endif
+    using Mtx = gko::matrix::Dense<value_type>;
     using index_type = gko::int32;
-    using Mtx = gko::matrix::Dense<>;
     using Csr = gko::matrix::Csr<value_type, index_type>;
+
     Bicg() : rand_engine(30) {}
 
     void SetUp()
@@ -94,7 +99,7 @@ class Bicg : public ::testing::Test {
         return gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+            std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
     }
 
     void initialize_data()
@@ -211,16 +216,16 @@ TEST_F(Bicg, BicgInitializeIsEquivalentToRef)
         d_prev_rho.get(), d_rho.get(), d_r2.get(), d_z2.get(), d_p2.get(),
         d_q2.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r2, r2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z2, z2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p2, p2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q2, q2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho, rho, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_r, r, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_z, z, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_p, p, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_q, q, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_r2, r2, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_z2, z2, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_p2, p2, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_q2, q2, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_prev_rho, prev_rho, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_rho, rho, ::r<value_type>::value);
     GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
 }
 
@@ -236,10 +241,10 @@ TEST_F(Bicg, BicgStep1IsEquivalentToRef)
         exec, d_p.get(), d_z.get(), d_p2.get(), d_z2.get(), d_rho.get(),
         d_prev_rho.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p2, p2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z2, z2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_p, p, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_z, z, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_p2, p2, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_z2, z2, ::r<value_type>::value);
 }
 
 
@@ -254,12 +259,12 @@ TEST_F(Bicg, BicgStep2IsEquivalentToRef)
         exec, d_x.get(), d_r.get(), d_r2.get(), d_p.get(), d_q.get(),
         d_q2.get(), d_beta.get(), d_rho.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r2, r2, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q2, q2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_r, r, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_r2, r2, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_p, p, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_q, q, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_q2, q2, ::r<value_type>::value);
 }
 
 
@@ -276,19 +281,19 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
     auto d_b = Mtx::create(exec);
     d_b->copy_from(b.get());
     auto bicg_factory =
-        gko::solver::Bicg<>::build()
+        gko::solver::Bicg<value_type>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
+                gko::stop::ResidualNorm<value_type>::build()
+                    .with_reduction_factor(::r<value_type>::value)
                     .on(ref))
             .on(ref);
     auto d_bicg_factory =
-        gko::solver::Bicg<>::build()
+        gko::solver::Bicg<value_type>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(exec),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
+                gko::stop::ResidualNorm<value_type>::build()
+                    .with_reduction_factor(::r<value_type>::value)
                     .on(exec))
             .on(exec);
     auto solver = bicg_factory->generate(std::move(mtx));
@@ -297,7 +302,7 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
     solver->apply(b.get(), x.get());
     d_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 10);
 }
 
 
@@ -310,19 +315,19 @@ TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
     auto d_b = Mtx::create(exec);
     d_b->copy_from(b.get());
     auto bicg_factory =
-        gko::solver::Bicg<>::build()
+        gko::solver::Bicg<value_type>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
+                gko::stop::ResidualNorm<value_type>::build()
+                    .with_reduction_factor(::r<value_type>::value)
                     .on(ref))
             .on(ref);
     auto d_bicg_factory =
-        gko::solver::Bicg<>::build()
+        gko::solver::Bicg<value_type>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(exec),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
+                gko::stop::ResidualNorm<value_type>::build()
+                    .with_reduction_factor(::r<value_type>::value)
                     .on(exec))
             .on(exec);
     auto solver = bicg_factory->generate(std::move(csr));
@@ -331,7 +336,7 @@ TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
     solver->apply(b.get(), x.get());
     d_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value);
 }
 
 
diff --git a/test/solver/bicgstab_kernels.cpp b/test/solver/bicgstab_kernels.cpp
index 3eef55e5799..7ffcbfd06b7 100644
--- a/test/solver/bicgstab_kernels.cpp
+++ b/test/solver/bicgstab_kernels.cpp
@@ -58,8 +58,14 @@ namespace {
 
 class Bicgstab : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Dense<>;
-    using Solver = gko::solver::Bicgstab<>;
+#if GINKGO_COMMON_SINGLE_MODE
+    using value_type = float;
+#else
+    using value_type = double;
+#endif
+    using index_type = gko::int32;
+    using Mtx = gko::matrix::Dense<value_type>;
+    using Solver = gko::solver::Bicgstab<value_type>;
 
     Bicgstab() : rand_engine(30) {}
 
@@ -76,8 +82,8 @@ class Bicgstab : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(exec),
-                    gko::stop::ResidualNorm<>::build()
-                        .with_reduction_factor(1e-15)
+                    gko::stop::ResidualNorm<value_type>::build()
+                        .with_reduction_factor(::r<value_type>::value)
                         .on(exec))
                 .on(exec);
 
@@ -85,8 +91,8 @@ class Bicgstab : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNorm<>::build()
-                        .with_reduction_factor(1e-15)
+                    gko::stop::ResidualNorm<value_type>::build()
+                        .with_reduction_factor(::r<value_type>::value)
                         .on(ref))
                 .on(ref);
     }
@@ -104,7 +110,7 @@ class Bicgstab : public ::testing::Test {
         return gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
+            std::normal_distribution<value_type>(0.0, 1.0), rand_engine, ref);
     }
 
     void initialize_data()
@@ -246,20 +252,20 @@ TEST_F(Bicgstab, BicgstabInitializeIsEquivalentToRef)
         d_alpha.get(), d_beta.get(), d_gamma.get(), d_omega.get(),
         d_stop_status.get());
 
-    GKO_EXPECT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_y, y, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_t, t, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_s, s, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_rr, rr, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_v, v, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_rho, rho, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_alpha, alpha, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_beta, beta, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_gamma, gamma, 1e-14);
-    GKO_EXPECT_MTX_NEAR(d_omega, omega, 1e-14);
+    GKO_EXPECT_MTX_NEAR(d_r, r, ::r<value_type>::value);
+    GKO_EXPECT_MTX_NEAR(d_z, z, ::r<value_type>::value);
+    GKO_EXPECT_MTX_NEAR(d_p, p, ::r<value_type>::value);
+    GKO_EXPECT_MTX_NEAR(d_y, y, ::r<value_type>::value);
+    GKO_EXPECT_MTX_NEAR(d_t, t, ::r<value_type>::value);
+    GKO_EXPECT_MTX_NEAR(d_s, s, ::r<value_type>::value);
+    GKO_EXPECT_MTX_NEAR(d_rr, rr, ::r<value_type>::value);
+    GKO_EXPECT_MTX_NEAR(d_v, v, ::r<value_type>::value);
+    GKO_EXPECT_MTX_NEAR(d_prev_rho, prev_rho, ::r<value_type>::value);
+    GKO_EXPECT_MTX_NEAR(d_rho, rho, ::r<value_type>::value);
+    GKO_EXPECT_MTX_NEAR(d_alpha, alpha, ::r<value_type>::value);
+    GKO_EXPECT_MTX_NEAR(d_beta, beta, ::r<value_type>::value);
+    GKO_EXPECT_MTX_NEAR(d_gamma, gamma, ::r<value_type>::value);
+    GKO_EXPECT_MTX_NEAR(d_omega, omega, ::r<value_type>::value);
     GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
 }
 
@@ -275,7 +281,7 @@ TEST_F(Bicgstab, BicgstabStep1IsEquivalentToRef)
         exec, d_r.get(), d_p.get(), d_v.get(), d_rho.get(), d_prev_rho.get(),
         d_alpha.get(), d_omega.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_p, p, ::r<value_type>::value);
 }
 
 
@@ -290,8 +296,8 @@ TEST_F(Bicgstab, BicgstabStep2IsEquivalentToRef)
         exec, d_r.get(), d_s.get(), d_v.get(), d_rho.get(), d_alpha.get(),
         d_beta.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_s, s, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_s, s, ::r<value_type>::value);
 }
 
 
@@ -307,9 +313,9 @@ TEST_F(Bicgstab, BicgstabStep3IsEquivalentToRef)
         d_alpha.get(), d_beta.get(), d_gamma.get(), d_omega.get(),
         d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_omega, omega, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_omega, omega, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_r, r, ::r<value_type>::value);
 }
 
 
@@ -329,8 +335,8 @@ TEST_F(Bicgstab, BicgstabApplyOneRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     exec_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 10);
 }
 
 
@@ -350,8 +356,8 @@ TEST_F(Bicgstab, BicgstabApplyMultipleRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     exec_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 10);
 }
 
 
diff --git a/test/solver/cg_kernels.cpp b/test/solver/cg_kernels.cpp
index 8c6f1e1c2d5..5370ba5b308 100644
--- a/test/solver/cg_kernels.cpp
+++ b/test/solver/cg_kernels.cpp
@@ -57,7 +57,13 @@ namespace {
 
 class Cg : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Dense<>;
+#if GINKGO_COMMON_SINGLE_MODE
+    using value_type = float;
+#else
+    using value_type = double;
+#endif
+    using Mtx = gko::matrix::Dense<value_type>;
+
     Cg() : rand_engine(30) {}
 
     void SetUp()
@@ -79,7 +85,7 @@ class Cg : public ::testing::Test {
         return gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+            std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
     }
 
     void initialize_data()
@@ -173,12 +179,12 @@ TEST_F(Cg, CgInitializeIsEquivalentToRef)
         exec, d_b.get(), d_r.get(), d_z.get(), d_p.get(), d_q.get(),
         d_prev_rho.get(), d_rho.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho, rho, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_r, r, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_z, z, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_p, p, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_q, q, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_prev_rho, prev_rho, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_rho, rho, ::r<value_type>::value);
     GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
 }
 
@@ -193,8 +199,8 @@ TEST_F(Cg, CgStep1IsEquivalentToRef)
                                              d_rho.get(), d_prev_rho.get(),
                                              d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_p, p, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_z, z, ::r<value_type>::value);
 }
 
 
@@ -208,10 +214,10 @@ TEST_F(Cg, CgStep2IsEquivalentToRef)
                                              d_p.get(), d_q.get(), d_beta.get(),
                                              d_rho.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_r, r, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_p, p, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_q, q, ::r<value_type>::value);
 }
 
 
@@ -228,19 +234,19 @@ TEST_F(Cg, ApplyIsEquivalentToRef)
     auto d_b = Mtx::create(exec);
     d_b->copy_from(b.get());
     auto cg_factory =
-        gko::solver::Cg<>::build()
+        gko::solver::Cg<value_type>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
+                gko::stop::ResidualNorm<value_type>::build()
+                    .with_reduction_factor(::r<value_type>::value)
                     .on(ref))
             .on(ref);
     auto d_cg_factory =
-        gko::solver::Cg<>::build()
+        gko::solver::Cg<value_type>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(exec),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
+                gko::stop::ResidualNorm<value_type>::build()
+                    .with_reduction_factor(::r<value_type>::value)
                     .on(exec))
             .on(exec);
     auto solver = cg_factory->generate(std::move(mtx));
@@ -249,7 +255,7 @@ TEST_F(Cg, ApplyIsEquivalentToRef)
     solver->apply(b.get(), x.get());
     d_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 10);
 }
 
 
diff --git a/test/solver/cgs_kernels.cpp b/test/solver/cgs_kernels.cpp
index 078c2efa1fa..8001e3000d4 100644
--- a/test/solver/cgs_kernels.cpp
+++ b/test/solver/cgs_kernels.cpp
@@ -57,8 +57,13 @@ namespace {
 
 class Cgs : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Dense<>;
-    using Solver = gko::solver::Cgs<>;
+#if GINKGO_COMMON_SINGLE_MODE
+    using value_type = float;
+#else
+    using value_type = double;
+#endif
+    using Mtx = gko::matrix::Dense<value_type>;
+    using Solver = gko::solver::Cgs<value_type>;
 
     Cgs() : rand_engine(30) {}
 
@@ -75,16 +80,16 @@ class Cgs : public ::testing::Test {
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(exec),
-                    gko::stop::ResidualNorm<>::build()
-                        .with_reduction_factor(1e-15)
+                    gko::stop::ResidualNorm<value_type>::build()
+                        .with_reduction_factor(::r<value_type>::value)
                         .on(exec))
                 .on(exec);
         ref_cgs_factory =
             Solver::build()
                 .with_criteria(
                     gko::stop::Iteration::build().with_max_iters(246u).on(ref),
-                    gko::stop::ResidualNorm<>::build()
-                        .with_reduction_factor(1e-15)
+                    gko::stop::ResidualNorm<value_type>::build()
+                        .with_reduction_factor(::r<value_type>::value)
                         .on(ref))
                 .on(ref);
     }
@@ -102,7 +107,7 @@ class Cgs : public ::testing::Test {
         return gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
+            std::normal_distribution<value_type>(0.0, 1.0), rand_engine, ref);
     }
 
     void initialize_data()
@@ -236,19 +241,19 @@ TEST_F(Cgs, CgsInitializeIsEquivalentToRef)
         d_beta.get(), d_gamma.get(), d_rho_prev.get(), d_rho.get(),
         d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r_tld, r_tld, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_u, u, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_t, t, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_u_hat, u_hat, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_v_hat, v_hat, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho_prev, rho_prev, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho, rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_beta, beta, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_gamma, gamma, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_r, r, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_r_tld, r_tld, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_p, p, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_q, q, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_u, u, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_t, t, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_u_hat, u_hat, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_v_hat, v_hat, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_rho_prev, rho_prev, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_rho, rho, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_beta, beta, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_gamma, gamma, ::r<value_type>::value);
     GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
 }
 
@@ -264,9 +269,9 @@ TEST_F(Cgs, CgsStep1IsEquivalentToRef)
         exec, d_r.get(), d_u.get(), d_p.get(), d_q.get(), d_beta.get(),
         d_rho.get(), d_rho_prev.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_beta, beta, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_u, u, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_beta, beta, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_u, u, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_p, p, ::r<value_type>::value);
 }
 
 
@@ -281,9 +286,9 @@ TEST_F(Cgs, CgsStep2IsEquivalentToRef)
         exec, d_u.get(), d_v_hat.get(), d_q.get(), d_t.get(), d_alpha.get(),
         d_rho.get(), d_gamma.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_t, t, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_alpha, alpha, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_t, t, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_q, q, ::r<value_type>::value);
 }
 
 
@@ -298,8 +303,8 @@ TEST_F(Cgs, CgsStep3IsEquivalentToRef)
         exec, d_t.get(), d_u_hat.get(), d_r.get(), d_x.get(), d_alpha.get(),
         d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_r, r, ::r<value_type>::value);
 }
 
 
@@ -319,8 +324,8 @@ TEST_F(Cgs, CgsApplyOneRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     exec_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 10);
 }
 
 
@@ -340,8 +345,8 @@ TEST_F(Cgs, CgsApplyMultipleRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     exec_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 10);
 }
 
 }  // namespace
diff --git a/test/solver/fcg_kernels.cpp b/test/solver/fcg_kernels.cpp
index 3a25bc45dd0..7a97ba6f8f5 100644
--- a/test/solver/fcg_kernels.cpp
+++ b/test/solver/fcg_kernels.cpp
@@ -57,8 +57,13 @@ namespace {
 
 class Fcg : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Dense<>;
-    using Solver = gko::solver::Fcg<>;
+#if GINKGO_COMMON_SINGLE_MODE
+    using value_type = float;
+#else
+    using value_type = double;
+#endif
+    using Mtx = gko::matrix::Dense<value_type>;
+    using Solver = gko::solver::Fcg<value_type>;
 
     Fcg() : rand_engine(30) {}
 
@@ -81,7 +86,7 @@ class Fcg : public ::testing::Test {
         return gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
+            std::normal_distribution<value_type>(0.0, 1.0), rand_engine, ref);
     }
 
     void initialize_data()
@@ -185,14 +190,14 @@ TEST_F(Fcg, FcgInitializeIsEquivalentToRef)
         exec, d_b.get(), d_r.get(), d_z.get(), d_p.get(), d_q.get(), d_t.get(),
         d_prev_rho.get(), d_rho.get(), d_rho_t.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_t, t, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_q, q, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_prev_rho, prev_rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho, rho, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_rho_t, rho_t, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_r, r, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_t, t, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_z, z, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_p, p, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_q, q, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_prev_rho, prev_rho, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_rho, rho, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_rho_t, rho_t, ::r<value_type>::value);
     GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
 }
 
@@ -207,8 +212,8 @@ TEST_F(Fcg, FcgStep1IsEquivalentToRef)
                                               d_rho_t.get(), d_prev_rho.get(),
                                               d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_p, p, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_z, z, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_p, p, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_z, z, ::r<value_type>::value);
 }
 
 
@@ -222,9 +227,9 @@ TEST_F(Fcg, FcgStep2IsEquivalentToRef)
         exec, d_x.get(), d_r.get(), d_t.get(), d_p.get(), d_q.get(),
         d_beta.get(), d_rho.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_r, r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_t, t, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_r, r, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_t, t, ::r<value_type>::value);
 }
 
 
@@ -244,16 +249,16 @@ TEST_F(Fcg, ApplyIsEquivalentToRef)
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(ref),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
+                gko::stop::ResidualNorm<value_type>::build()
+                    .with_reduction_factor(::r<value_type>::value)
                     .on(ref))
             .on(ref);
     auto d_fcg_factory =
         Solver::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(50u).on(exec),
-                gko::stop::ResidualNorm<>::build()
-                    .with_reduction_factor(1e-14)
+                gko::stop::ResidualNorm<value_type>::build()
+                    .with_reduction_factor(::r<value_type>::value)
                     .on(exec))
             .on(exec);
     auto solver = fcg_factory->generate(std::move(mtx));
@@ -262,7 +267,7 @@ TEST_F(Fcg, ApplyIsEquivalentToRef)
     solver->apply(b.get(), x.get());
     d_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 10);
 }
 
 
diff --git a/test/solver/ir_kernels.cpp b/test/solver/ir_kernels.cpp
index 4c5c35544ee..1e7375db717 100644
--- a/test/solver/ir_kernels.cpp
+++ b/test/solver/ir_kernels.cpp
@@ -57,7 +57,13 @@ namespace {
 
 class Ir : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Dense<>;
+#if GINKGO_COMMON_SINGLE_MODE
+    using value_type = float;
+#else
+    using value_type = double;
+#endif
+    using Mtx = gko::matrix::Dense<value_type>;
+
     Ir() : rand_engine(30) {}
 
     void SetUp()
@@ -71,7 +77,7 @@ class Ir : public ::testing::Test {
         return gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+            std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -111,12 +117,12 @@ TEST_F(Ir, ApplyIsEquivalentToRef)
     // matrix, just check that a couple of iterations gives the same result on
     // both executors
     auto ir_factory =
-        gko::solver::Ir<>::build()
+        gko::solver::Ir<value_type>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(2u).on(ref))
             .on(ref);
     auto d_ir_factory =
-        gko::solver::Ir<>::build()
+        gko::solver::Ir<value_type>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(2u).on(exec))
             .on(exec);
@@ -126,7 +132,7 @@ TEST_F(Ir, ApplyIsEquivalentToRef)
     solver->apply(lend(b), lend(x));
     d_solver->apply(lend(d_b), lend(d_x));
 
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, r<value_type>::value);
 }
 
 
@@ -140,9 +146,9 @@ TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef)
     auto d_b = clone(exec, b);
 
     auto ir_factory =
-        gko::solver::Ir<>::build()
+        gko::solver::Ir<value_type>::build()
             .with_solver(
-                gko::solver::Gmres<>::build()
+                gko::solver::Gmres<value_type>::build()
                     .with_criteria(
                         gko::stop::Iteration::build().with_max_iters(1u).on(
                             ref))
@@ -151,9 +157,9 @@ TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef)
                 gko::stop::Iteration::build().with_max_iters(2u).on(ref))
             .on(ref);
     auto d_ir_factory =
-        gko::solver::Ir<>::build()
+        gko::solver::Ir<value_type>::build()
             .with_solver(
-                gko::solver::Gmres<>::build()
+                gko::solver::Gmres<value_type>::build()
                     .with_criteria(
                         gko::stop::Iteration::build().with_max_iters(1u).on(
                             exec))
@@ -167,9 +173,10 @@ TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef)
     solver->apply(lend(b), lend(x));
     d_solver->apply(lend(d_b), lend(d_x));
 
-    // Note: 1e-12 instead of 1e-14, as the difference in the inner gmres
-    // iteration gets amplified by the difference in IR.
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
+    // Note:r<value_type>::value * 1e2 instead ofr<value_type>::value, as
+    // the difference in the inner gmres iteration gets amplified by the
+    // difference in IR.
+    GKO_ASSERT_MTX_NEAR(d_x, x, r<value_type>::value * 1e2);
 }
 
 
@@ -185,16 +192,16 @@ TEST_F(Ir, RichardsonApplyIsEquivalentToRef)
     // matrix, just check that a couple of iterations gives the same result on
     // both executors
     auto ir_factory =
-        gko::solver::Ir<>::build()
+        gko::solver::Ir<value_type>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(2u).on(ref))
-            .with_relaxation_factor(0.9)
+            .with_relaxation_factor(value_type{0.9})
             .on(ref);
     auto d_ir_factory =
-        gko::solver::Ir<>::build()
+        gko::solver::Ir<value_type>::build()
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(2u).on(exec))
-            .with_relaxation_factor(0.9)
+            .with_relaxation_factor(value_type{0.9})
             .on(exec);
     auto solver = ir_factory->generate(std::move(mtx));
     auto d_solver = d_ir_factory->generate(std::move(d_mtx));
@@ -202,7 +209,7 @@ TEST_F(Ir, RichardsonApplyIsEquivalentToRef)
     solver->apply(lend(b), lend(x));
     d_solver->apply(lend(d_b), lend(d_x));
 
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, r<value_type>::value);
 }
 
 
@@ -215,28 +222,28 @@ TEST_F(Ir, RichardsonApplyWithIterativeInnerSolverIsEquivalentToRef)
     auto d_x = clone(exec, x);
     auto d_b = clone(exec, b);
     auto ir_factory =
-        gko::solver::Ir<>::build()
+        gko::solver::Ir<value_type>::build()
             .with_solver(
-                gko::solver::Gmres<>::build()
+                gko::solver::Gmres<value_type>::build()
                     .with_criteria(
                         gko::stop::Iteration::build().with_max_iters(1u).on(
                             ref))
                     .on(ref))
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(2u).on(ref))
-            .with_relaxation_factor(0.9)
+            .with_relaxation_factor(value_type{0.9})
             .on(ref);
     auto d_ir_factory =
-        gko::solver::Ir<>::build()
+        gko::solver::Ir<value_type>::build()
             .with_solver(
-                gko::solver::Gmres<>::build()
+                gko::solver::Gmres<value_type>::build()
                     .with_criteria(
                         gko::stop::Iteration::build().with_max_iters(1u).on(
                             exec))
                     .on(exec))
             .with_criteria(
                 gko::stop::Iteration::build().with_max_iters(2u).on(exec))
-            .with_relaxation_factor(0.9)
+            .with_relaxation_factor(value_type{0.9})
             .on(exec);
     auto solver = ir_factory->generate(std::move(mtx));
     auto d_solver = d_ir_factory->generate(std::move(d_mtx));
@@ -244,9 +251,10 @@ TEST_F(Ir, RichardsonApplyWithIterativeInnerSolverIsEquivalentToRef)
     solver->apply(lend(b), lend(x));
     d_solver->apply(lend(d_b), lend(d_x));
 
-    // Note: 1e-12 instead of 1e-14, as the difference in the inner gmres
-    // iteration gets amplified by the difference in IR.
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
+    // Note:r<value_type>::value * 1e2 instead ofr<value_type>::value, as
+    // the difference in the inner gmres iteration gets amplified by the
+    // difference in IR.
+    GKO_ASSERT_MTX_NEAR(d_x, x, r<value_type>::value * 1e2);
 }
 
 

From fd858a34165fec8947552e8b8c85376e481ec8a1 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 11 Jun 2021 14:57:39 +0200
Subject: [PATCH 0836/1008] fix clang-cuda tests

---
 cmake/create_test.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index e2d04e1893c..ebf70232dd3 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -75,7 +75,7 @@ function(ginkgo_create_cuda_test test_name)
         target_compile_options(${test_target_name}
             PRIVATE
                 $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda --expt-relaxed-constexpr>)
-    else()
+    elseif(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA")
         target_compile_options(${test_target_name}
             PRIVATE
                 $<$<COMPILE_LANGUAGE:CUDA>:--expt-extended-lambda --expt-relaxed-constexpr>)

From 286966b07c53900912f49dd3590a01516403de14 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 11 Jun 2021 16:30:09 +0200
Subject: [PATCH 0837/1008] fix OpenMP collapse compilation issue

---
 omp/base/kernel_launch.hpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index d93759cc7cb..c1d46680e3f 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -200,9 +200,11 @@ template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
                 dim<2> size, KernelArgs &&... args)
 {
+    const auto rows = size[0];
+    const auto cols = size[1];
 #pragma omp parallel for collapse(2)
-    for (size_type row = 0; row < size[0]; row++) {
-        for (size_type col = 0; col < size[1]; col++) {
+    for (size_type row = 0; row < rows; row++) {
+        for (size_type col = 0; col < cols; col++) {
             [&]() {
                 fn(row, col,
                    kernels::omp::map_unpack_to_device(args, size[1])...);

From 47361c51cab2b675faa91e37a438e3ef8e4b91be Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 11 Jun 2021 16:33:07 +0200
Subject: [PATCH 0838/1008] relax solver test precision

---
 test/solver/bicg_kernels.cpp     | 4 ++--
 test/solver/bicgstab_kernels.cpp | 8 ++++----
 test/solver/cg_kernels.cpp       | 2 +-
 test/solver/cgs_kernels.cpp      | 8 ++++----
 test/solver/fcg_kernels.cpp      | 2 +-
 test/solver/ir_kernels.cpp       | 8 ++++----
 6 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/test/solver/bicg_kernels.cpp b/test/solver/bicg_kernels.cpp
index f4b57867510..4d087981d60 100644
--- a/test/solver/bicg_kernels.cpp
+++ b/test/solver/bicg_kernels.cpp
@@ -302,7 +302,7 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
     solver->apply(b.get(), x.get());
     d_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 100);
 }
 
 
@@ -336,7 +336,7 @@ TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
     solver->apply(b.get(), x.get());
     d_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 100);
 }
 
 
diff --git a/test/solver/bicgstab_kernels.cpp b/test/solver/bicgstab_kernels.cpp
index 7ffcbfd06b7..5a1403b0e35 100644
--- a/test/solver/bicgstab_kernels.cpp
+++ b/test/solver/bicgstab_kernels.cpp
@@ -335,8 +335,8 @@ TEST_F(Bicgstab, BicgstabApplyOneRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     exec_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 10);
-    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 100);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 100);
 }
 
 
@@ -356,8 +356,8 @@ TEST_F(Bicgstab, BicgstabApplyMultipleRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     exec_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 10);
-    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 100);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 100);
 }
 
 
diff --git a/test/solver/cg_kernels.cpp b/test/solver/cg_kernels.cpp
index 5370ba5b308..ac182fb2090 100644
--- a/test/solver/cg_kernels.cpp
+++ b/test/solver/cg_kernels.cpp
@@ -255,7 +255,7 @@ TEST_F(Cg, ApplyIsEquivalentToRef)
     solver->apply(b.get(), x.get());
     d_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 100);
 }
 
 
diff --git a/test/solver/cgs_kernels.cpp b/test/solver/cgs_kernels.cpp
index 8001e3000d4..b25ced0351d 100644
--- a/test/solver/cgs_kernels.cpp
+++ b/test/solver/cgs_kernels.cpp
@@ -324,8 +324,8 @@ TEST_F(Cgs, CgsApplyOneRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     exec_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 10);
-    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 100);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 100);
 }
 
 
@@ -345,8 +345,8 @@ TEST_F(Cgs, CgsApplyMultipleRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     exec_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 10);
-    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 100);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 100);
 }
 
 }  // namespace
diff --git a/test/solver/fcg_kernels.cpp b/test/solver/fcg_kernels.cpp
index 7a97ba6f8f5..e65fab5bd6e 100644
--- a/test/solver/fcg_kernels.cpp
+++ b/test/solver/fcg_kernels.cpp
@@ -267,7 +267,7 @@ TEST_F(Fcg, ApplyIsEquivalentToRef)
     solver->apply(b.get(), x.get());
     d_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 100);
 }
 
 
diff --git a/test/solver/ir_kernels.cpp b/test/solver/ir_kernels.cpp
index 1e7375db717..6ce9d98f7a2 100644
--- a/test/solver/ir_kernels.cpp
+++ b/test/solver/ir_kernels.cpp
@@ -173,10 +173,10 @@ TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef)
     solver->apply(lend(b), lend(x));
     d_solver->apply(lend(d_b), lend(d_x));
 
-    // Note:r<value_type>::value * 1e2 instead ofr<value_type>::value, as
+    // Note: r<value_type>::value * 1e2 instead of r<value_type>::value, as
     // the difference in the inner gmres iteration gets amplified by the
     // difference in IR.
-    GKO_ASSERT_MTX_NEAR(d_x, x, r<value_type>::value * 1e2);
+    GKO_ASSERT_MTX_NEAR(d_x, x, r<value_type>::value * 100);
 }
 
 
@@ -251,10 +251,10 @@ TEST_F(Ir, RichardsonApplyWithIterativeInnerSolverIsEquivalentToRef)
     solver->apply(lend(b), lend(x));
     d_solver->apply(lend(d_b), lend(d_x));
 
-    // Note:r<value_type>::value * 1e2 instead ofr<value_type>::value, as
+    // Note: r<value_type>::value * 1e2 instead of r<value_type>::value, as
     // the difference in the inner gmres iteration gets amplified by the
     // difference in IR.
-    GKO_ASSERT_MTX_NEAR(d_x, x, r<value_type>::value * 1e2);
+    GKO_ASSERT_MTX_NEAR(d_x, x, r<value_type>::value * 100);
 }
 
 

From acac9e0d0e0e670ab6b4c5e30b4d52869a9a4f9b Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 20 May 2021 23:43:32 +0200
Subject: [PATCH 0839/1008] use SYCL_DEVICE_TYPE/FILTER to make sure using
 gpu/cpu data cannot be passed between different backends on discrete gpu.

---
 .gitlab-ci.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4c7267072b4..0b4f96f4371 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -98,6 +98,7 @@ include:
       CUDA_ARCH_STR=-DGINKGO_CUDA_ARCHITECTURES=${CUDA_ARCH};
       CUDA_HOST_STR=-DCMAKE_CUDA_HOST_COMPILER=$(which ${CXX_COMPILER});
       fi
+    - if [ -n "${SYCL_DEVICE_TYPE}" ]; then export SYCL_DEVICE_TYPE; fi
     - if [ -n "${SYCL_DEVICE_FILTER}" ]; then export SYCL_DEVICE_FILTER; fi
     - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
         -GNinja
@@ -145,6 +146,7 @@ include:
             ninja validate_all_examples
         fi
       fi
+    - if [ -n "${SYCL_DEVICE_TYPE}" ]; then unset SYCL_DEVICE_TYPE; fi
     - if [ -n "${SYCL_DEVICE_FILTER}" ]; then unset SYCL_DEVICE_FILTER; fi
     - if [ "${EXPORT_BUILD_DIR}" == "ON" ]; then ninja test_exportbuild; fi
   dependencies: []
@@ -718,7 +720,7 @@ build/dpcpp/cpu/release/static:
     BUILD_DPCPP: "ON"
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "ON"
-    SYCL_DEVICE_FILTER: "CPU"
+    SYCL_DEVICE_TYPE: "CPU"
 
 build/dpcpp/igpu/release/static:
   <<: *default_build_with_test
@@ -733,7 +735,7 @@ build/dpcpp/igpu/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     DPCPP_SINGLE_MODE: "ON"
-    SYCL_DEVICE_FILTER: "GPU"
+    SYCL_DEVICE_FILTER: "Level_Zero:GPU"
 
 build/dpcpp/dgpu/debug/shared:
   <<: *default_build_with_test

From cb8e046a1605f01c6affdd99a4e88543885126ae Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 11 Jun 2021 22:50:28 +0200
Subject: [PATCH 0840/1008] remove OpenMP collapse(2)

---
 omp/base/kernel_launch.hpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index c1d46680e3f..532fb555106 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -202,12 +202,11 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
 {
     const auto rows = size[0];
     const auto cols = size[1];
-#pragma omp parallel for collapse(2)
+#pragma omp parallel for
     for (size_type row = 0; row < rows; row++) {
         for (size_type col = 0; col < cols; col++) {
             [&]() {
-                fn(row, col,
-                   kernels::omp::map_unpack_to_device(args, size[1])...);
+                fn(row, col, kernels::omp::map_unpack_to_device(args, cols)...);
             }();
         }
     }

From 4e2c2f308aa9bfa353aa6aa72b7567f0ce7f3a63 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 11 Jun 2021 23:11:35 +0200
Subject: [PATCH 0841/1008] explicitly unroll OpenMP simple kernels

---
 omp/base/kernel_launch.hpp | 134 +++++++++++++++++++++++++++++++++++--
 1 file changed, 130 insertions(+), 4 deletions(-)

diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index 532fb555106..c0580816bf3 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -196,14 +196,15 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
     }
 }
 
-template <typename KernelFunction, typename... KernelArgs>
-void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
-                dim<2> size, KernelArgs &&... args)
+template <size_type cols, typename KernelFunction, typename... KernelArgs>
+void run_kernel_fixed_cols(std::shared_ptr<const OmpExecutor> exec,
+                           KernelFunction fn, dim<2> size,
+                           KernelArgs &&... args)
 {
     const auto rows = size[0];
-    const auto cols = size[1];
 #pragma omp parallel for
     for (size_type row = 0; row < rows; row++) {
+#pragma unroll
         for (size_type col = 0; col < cols; col++) {
             [&]() {
                 fn(row, col, kernels::omp::map_unpack_to_device(args, cols)...);
@@ -212,6 +213,131 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
     }
 }
 
+template <size_type remainder_cols, typename KernelFunction,
+          typename... KernelArgs>
+void run_kernel_blocked_cols(std::shared_ptr<const OmpExecutor> exec,
+                             KernelFunction fn, dim<2> size,
+                             KernelArgs &&... args)
+{
+    const auto rows = size[0];
+    const auto cols = size[1];
+    const auto rounded_cols = cols / 8 * 8;
+    GKO_ASSERT(rounded_cols + remainder_cols == cols);
+#pragma omp parallel for
+    for (size_type row = 0; row < rows; row++) {
+        for (size_type base_col = 0; base_col < rounded_cols; base_col += 8) {
+#pragma unroll
+            for (size_type i = 0; i < 8; i++) {
+                [&]() {
+                    fn(row, base_col + i,
+                       kernels::omp::map_unpack_to_device(args, cols)...);
+                }();
+            }
+        }
+#pragma unroll
+        for (size_type i = 0; i < remainder_cols; i++) {
+            [&]() {
+                fn(row, rounded_cols + i,
+                   kernels::omp::map_unpack_to_device(args, cols)...);
+            }();
+        }
+    }
+}
+
+template <typename KernelFunction, typename... KernelArgs>
+void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
+                dim<2> size, KernelArgs &&... args)
+{
+    const auto rows = size[0];
+    const auto cols = size[1];
+    if (cols <= 0) {
+        return;
+    }
+    if (cols == 1) {
+        run_kernel_fixed_cols<1>(exec, fn, size,
+                                 std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (cols == 2) {
+        run_kernel_fixed_cols<2>(exec, fn, size,
+                                 std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (cols == 3) {
+        run_kernel_fixed_cols<3>(exec, fn, size,
+                                 std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (cols == 4) {
+        run_kernel_fixed_cols<4>(exec, fn, size,
+                                 std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (cols == 5) {
+        run_kernel_fixed_cols<5>(exec, fn, size,
+                                 std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (cols == 6) {
+        run_kernel_fixed_cols<6>(exec, fn, size,
+                                 std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (cols == 7) {
+        run_kernel_fixed_cols<7>(exec, fn, size,
+                                 std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (cols == 8) {
+        run_kernel_fixed_cols<8>(exec, fn, size,
+                                 std::forward<KernelArgs>(args)...);
+        return;
+    }
+    const auto rem_cols = cols % 8;
+    if (rem_cols == 0) {
+        run_kernel_blocked_cols<0>(exec, fn, size,
+                                   std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (rem_cols == 1) {
+        run_kernel_blocked_cols<1>(exec, fn, size,
+                                   std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (rem_cols == 2) {
+        run_kernel_blocked_cols<2>(exec, fn, size,
+                                   std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (rem_cols == 3) {
+        run_kernel_blocked_cols<3>(exec, fn, size,
+                                   std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (rem_cols == 4) {
+        run_kernel_blocked_cols<4>(exec, fn, size,
+                                   std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (rem_cols == 5) {
+        run_kernel_blocked_cols<5>(exec, fn, size,
+                                   std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (rem_cols == 6) {
+        run_kernel_blocked_cols<6>(exec, fn, size,
+                                   std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (rem_cols == 7) {
+        run_kernel_blocked_cols<7>(exec, fn, size,
+                                   std::forward<KernelArgs>(args)...);
+        return;
+    }
+    // should be unreachable
+    GKO_ASSERT(false);
+}
+
 
 }  // namespace gko
 

From 2d48a77c55291b8d07a5c064698d391bea677d89 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 11 Jun 2021 23:27:30 +0200
Subject: [PATCH 0842/1008] avoid unrolling remainder loop

---
 omp/base/kernel_launch.hpp | 49 +++-----------------------------------
 1 file changed, 3 insertions(+), 46 deletions(-)

diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index c0580816bf3..937ff277d14 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -213,8 +213,7 @@ void run_kernel_fixed_cols(std::shared_ptr<const OmpExecutor> exec,
     }
 }
 
-template <size_type remainder_cols, typename KernelFunction,
-          typename... KernelArgs>
+template <typename KernelFunction, typename... KernelArgs>
 void run_kernel_blocked_cols(std::shared_ptr<const OmpExecutor> exec,
                              KernelFunction fn, dim<2> size,
                              KernelArgs &&... args)
@@ -222,6 +221,7 @@ void run_kernel_blocked_cols(std::shared_ptr<const OmpExecutor> exec,
     const auto rows = size[0];
     const auto cols = size[1];
     const auto rounded_cols = cols / 8 * 8;
+    const auto remainder_cols = cols % 8;
     GKO_ASSERT(rounded_cols + remainder_cols == cols);
 #pragma omp parallel for
     for (size_type row = 0; row < rows; row++) {
@@ -234,7 +234,6 @@ void run_kernel_blocked_cols(std::shared_ptr<const OmpExecutor> exec,
                 }();
             }
         }
-#pragma unroll
         for (size_type i = 0; i < remainder_cols; i++) {
             [&]() {
                 fn(row, rounded_cols + i,
@@ -293,49 +292,7 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
                                  std::forward<KernelArgs>(args)...);
         return;
     }
-    const auto rem_cols = cols % 8;
-    if (rem_cols == 0) {
-        run_kernel_blocked_cols<0>(exec, fn, size,
-                                   std::forward<KernelArgs>(args)...);
-        return;
-    }
-    if (rem_cols == 1) {
-        run_kernel_blocked_cols<1>(exec, fn, size,
-                                   std::forward<KernelArgs>(args)...);
-        return;
-    }
-    if (rem_cols == 2) {
-        run_kernel_blocked_cols<2>(exec, fn, size,
-                                   std::forward<KernelArgs>(args)...);
-        return;
-    }
-    if (rem_cols == 3) {
-        run_kernel_blocked_cols<3>(exec, fn, size,
-                                   std::forward<KernelArgs>(args)...);
-        return;
-    }
-    if (rem_cols == 4) {
-        run_kernel_blocked_cols<4>(exec, fn, size,
-                                   std::forward<KernelArgs>(args)...);
-        return;
-    }
-    if (rem_cols == 5) {
-        run_kernel_blocked_cols<5>(exec, fn, size,
-                                   std::forward<KernelArgs>(args)...);
-        return;
-    }
-    if (rem_cols == 6) {
-        run_kernel_blocked_cols<6>(exec, fn, size,
-                                   std::forward<KernelArgs>(args)...);
-        return;
-    }
-    if (rem_cols == 7) {
-        run_kernel_blocked_cols<7>(exec, fn, size,
-                                   std::forward<KernelArgs>(args)...);
-        return;
-    }
-    // should be unreachable
-    GKO_ASSERT(false);
+    run_kernel_blocked_cols(exec, fn, size, std::forward<KernelArgs>(args)...);
 }
 
 

From fbc20fd910c56e8d033c937444e1bcc9859eb4cd Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 12 Jun 2021 00:32:59 +0200
Subject: [PATCH 0843/1008] unroll width 4

---
 omp/base/kernel_launch.hpp | 28 ++++------------------------
 1 file changed, 4 insertions(+), 24 deletions(-)

diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index 937ff277d14..8a309030e74 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -220,14 +220,14 @@ void run_kernel_blocked_cols(std::shared_ptr<const OmpExecutor> exec,
 {
     const auto rows = size[0];
     const auto cols = size[1];
-    const auto rounded_cols = cols / 8 * 8;
-    const auto remainder_cols = cols % 8;
+    const auto rounded_cols = cols / 4 * 4;
+    const auto remainder_cols = cols % 4;
     GKO_ASSERT(rounded_cols + remainder_cols == cols);
 #pragma omp parallel for
     for (size_type row = 0; row < rows; row++) {
-        for (size_type base_col = 0; base_col < rounded_cols; base_col += 8) {
+        for (size_type base_col = 0; base_col < rounded_cols; base_col += 4) {
 #pragma unroll
-            for (size_type i = 0; i < 8; i++) {
+            for (size_type i = 0; i < 4; i++) {
                 [&]() {
                     fn(row, base_col + i,
                        kernels::omp::map_unpack_to_device(args, cols)...);
@@ -272,26 +272,6 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
                                  std::forward<KernelArgs>(args)...);
         return;
     }
-    if (cols == 5) {
-        run_kernel_fixed_cols<5>(exec, fn, size,
-                                 std::forward<KernelArgs>(args)...);
-        return;
-    }
-    if (cols == 6) {
-        run_kernel_fixed_cols<6>(exec, fn, size,
-                                 std::forward<KernelArgs>(args)...);
-        return;
-    }
-    if (cols == 7) {
-        run_kernel_fixed_cols<7>(exec, fn, size,
-                                 std::forward<KernelArgs>(args)...);
-        return;
-    }
-    if (cols == 8) {
-        run_kernel_fixed_cols<8>(exec, fn, size,
-                                 std::forward<KernelArgs>(args)...);
-        return;
-    }
     run_kernel_blocked_cols(exec, fn, size, std::forward<KernelArgs>(args)...);
 }
 

From 4f1ba9db1772586a6a054f17237090bec597f7db Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 12 Jun 2021 00:53:54 +0200
Subject: [PATCH 0844/1008] explicitly unroll remainder with width 4

---
 omp/base/kernel_launch.hpp | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)

diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index 8a309030e74..9b466250886 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -213,7 +213,8 @@ void run_kernel_fixed_cols(std::shared_ptr<const OmpExecutor> exec,
     }
 }
 
-template <typename KernelFunction, typename... KernelArgs>
+template <size_type remainder_cols, typename KernelFunction,
+          typename... KernelArgs>
 void run_kernel_blocked_cols(std::shared_ptr<const OmpExecutor> exec,
                              KernelFunction fn, dim<2> size,
                              KernelArgs &&... args)
@@ -221,7 +222,6 @@ void run_kernel_blocked_cols(std::shared_ptr<const OmpExecutor> exec,
     const auto rows = size[0];
     const auto cols = size[1];
     const auto rounded_cols = cols / 4 * 4;
-    const auto remainder_cols = cols % 4;
     GKO_ASSERT(rounded_cols + remainder_cols == cols);
 #pragma omp parallel for
     for (size_type row = 0; row < rows; row++) {
@@ -234,6 +234,7 @@ void run_kernel_blocked_cols(std::shared_ptr<const OmpExecutor> exec,
                 }();
             }
         }
+#pragma unroll
         for (size_type i = 0; i < remainder_cols; i++) {
             [&]() {
                 fn(row, rounded_cols + i,
@@ -272,7 +273,29 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
                                  std::forward<KernelArgs>(args)...);
         return;
     }
-    run_kernel_blocked_cols(exec, fn, size, std::forward<KernelArgs>(args)...);
+    const auto rem_cols = cols % 4;
+    if (rem_cols == 0) {
+        run_kernel_blocked_cols<0>(exec, fn, size,
+                                   std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (rem_cols == 1) {
+        run_kernel_blocked_cols<1>(exec, fn, size,
+                                   std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (rem_cols == 2) {
+        run_kernel_blocked_cols<2>(exec, fn, size,
+                                   std::forward<KernelArgs>(args)...);
+        return;
+    }
+    if (rem_cols == 3) {
+        run_kernel_blocked_cols<3>(exec, fn, size,
+                                   std::forward<KernelArgs>(args)...);
+        return;
+    }
+    // should be unreachable
+    GKO_ASSERT(false);
 }
 
 

From d654ead666f96e81fdd0a81de44c3f6bfd4010eb Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Sat, 12 Jun 2021 01:32:15 +0200
Subject: [PATCH 0845/1008] add kernel_launch to format_header.sh config

---
 dev_tools/scripts/config | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/dev_tools/scripts/config b/dev_tools/scripts/config
index 10ea75302e6..f856d97dae2 100644
--- a/dev_tools/scripts/config
+++ b/dev_tools/scripts/config
@@ -41,6 +41,10 @@
     - RemoveTest: "true"
     - PathIgnore: "1"
     - PathPrefix: "core"
+- "test/base/kernel_launch"
+    - RemoveTest: "true"
+    - PathIgnore: "1"
+    - PathPrefix: "(cuda|hip|omp|dpcpp)"
 - "test/utils"
     - CoreSuffix: "_test"
     - PathIgnore: "1"

From 36beda7a2af75c462867703f9a704cd0b680bf34 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 17 Jun 2021 11:44:08 +0200
Subject: [PATCH 0846/1008] keep stride in solver create_with_same_config

This means we need to provide a default stride
to the simple solver kernels, which we implement using
a separate run_kernel_solver function
---
 common/matrix/dense_kernels.cpp        |  10 +-
 common/solver/bicg_kernels.cpp         |  30 +++--
 common/solver/bicgstab_kernels.cpp     |  42 ++++---
 common/solver/cg_kernels.cpp           |  25 ++--
 common/solver/cgs_kernels.cpp          |  36 +++---
 common/solver/fcg_kernels.cpp          |  27 +++--
 core/base/simple_kernels_solver.hpp    |  64 ++++++++++
 cuda/base/kernel_launch.cuh            |  65 +---------
 cuda/base/kernel_launch_solver.cuh     | 155 ++++++++++++++++++++++++
 cuda/test/base/kernel_launch.cu        |  27 +++--
 dpcpp/base/kernel_launch.dp.hpp        |  61 +---------
 dpcpp/base/kernel_launch_solver.dp.hpp | 148 +++++++++++++++++++++++
 dpcpp/test/base/kernel_launch.dp.cpp   |  31 ++---
 hip/base/kernel_launch.hip.hpp         |  65 +---------
 hip/base/kernel_launch_solver.hip.hpp  | 158 +++++++++++++++++++++++++
 hip/test/base/kernel_launch.hip.cpp    |  27 +++--
 include/ginkgo/core/matrix/dense.hpp   |   3 +-
 omp/base/kernel_launch.hpp             | 136 +++++----------------
 omp/base/kernel_launch_solver.hpp      | 138 +++++++++++++++++++++
 omp/test/base/kernel_launch.cpp        |  33 +++---
 test/solver/bicg_kernels.cpp           |  63 +++++-----
 test/solver/bicgstab_kernels.cpp       |  55 +++++----
 test/solver/cg_kernels.cpp             |  36 +++---
 test/solver/cgs_kernels.cpp            |  53 ++++-----
 test/solver/fcg_kernels.cpp            |  41 ++++---
 test/solver/ir_kernels.cpp             |  32 ++---
 26 files changed, 1002 insertions(+), 559 deletions(-)
 create mode 100644 core/base/simple_kernels_solver.hpp
 create mode 100644 cuda/base/kernel_launch_solver.cuh
 create mode 100644 dpcpp/base/kernel_launch_solver.dp.hpp
 create mode 100644 hip/base/kernel_launch_solver.hip.hpp
 create mode 100644 omp/base/kernel_launch_solver.hpp

diff --git a/common/matrix/dense_kernels.cpp b/common/matrix/dense_kernels.cpp
index dfc92a150f8..3b53a538fa8 100644
--- a/common/matrix/dense_kernels.cpp
+++ b/common/matrix/dense_kernels.cpp
@@ -92,14 +92,14 @@ void scale(std::shared_ptr<const DefaultExecutor> exec,
             [] GKO_KERNEL(auto row, auto col, auto alpha, auto x) {
                 x(row, col) *= alpha[col];
             },
-            x->get_size(), vector(alpha), x);
+            x->get_size(), alpha->get_const_values(), x);
     } else {
         run_kernel(
             exec,
             [] GKO_KERNEL(auto row, auto col, auto alpha, auto x) {
                 x(row, col) *= alpha[0];
             },
-            x->get_size(), vector(alpha), x);
+            x->get_size(), alpha->get_const_values(), x);
     }
 }
 
@@ -117,14 +117,14 @@ void add_scaled(std::shared_ptr<const DefaultExecutor> exec,
             [] GKO_KERNEL(auto row, auto col, auto alpha, auto x, auto y) {
                 y(row, col) += alpha[col] * x(row, col);
             },
-            x->get_size(), vector(alpha), x, y);
+            x->get_size(), alpha->get_const_values(), x, y);
     } else {
         run_kernel(
             exec,
             [] GKO_KERNEL(auto row, auto col, auto alpha, auto x, auto y) {
                 y(row, col) += alpha[0] * x(row, col);
             },
-            x->get_size(), vector(alpha), x, y);
+            x->get_size(), alpha->get_const_values(), x, y);
     }
 }
 
@@ -143,7 +143,7 @@ void add_scaled_diag(std::shared_ptr<const DefaultExecutor> exec,
         [] GKO_KERNEL(auto i, auto alpha, auto diag, auto y) {
             y(i, i) += alpha[0] * diag[i];
         },
-        x->get_size()[0], vector(alpha), x->get_const_values(), y);
+        x->get_size()[0], alpha->get_const_values(), x->get_const_values(), y);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL);
diff --git a/common/solver/bicg_kernels.cpp b/common/solver/bicg_kernels.cpp
index e84ae5d9ec2..99748e9839f 100644
--- a/common/solver/bicg_kernels.cpp
+++ b/common/solver/bicg_kernels.cpp
@@ -36,7 +36,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "core/base/simple_kernels.hpp"
+#include "core/base/simple_kernels_solver.hpp"
+
+
+using gko::solver::default_stride;
+using gko::solver::rowvector;
 
 
 namespace gko {
@@ -60,7 +64,7 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 matrix::Dense<ValueType> *q2,
                 Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto z, auto p,
                       auto q, auto prev_rho, auto rho, auto r2, auto z2,
@@ -75,9 +79,11 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             z(row, col) = p(row, col) = q(row, col) = z2(row, col) =
                 p2(row, col) = q2(row, col) = zero(z(row, col));
         },
-        b->get_size(), b, compact(r), compact(z), compact(p), compact(q),
-        vector(prev_rho), vector(rho), compact(r2), compact(z2), compact(p2),
-        compact(q2), *stop_status);
+        b->get_size(), b->get_stride(), default_stride(b), default_stride(r),
+        default_stride(z), default_stride(p), default_stride(q),
+        rowvector(prev_rho), rowvector(rho), default_stride(r2),
+        default_stride(z2), default_stride(p2), default_stride(q2),
+        *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_INITIALIZE_KERNEL);
@@ -91,7 +97,7 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *prev_rho,
             const Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto p, auto z, auto p2, auto z2,
                       auto rho, auto prev_rho, auto stop) {
@@ -101,8 +107,9 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
                 p2(row, col) = z2(row, col) + tmp * p2(row, col);
             }
         },
-        p->get_size(), compact(p), compact(z), compact(p2), compact(z2),
-        vector(rho), vector(prev_rho), *stop_status);
+        p->get_size(), p->get_stride(), default_stride(p), default_stride(z),
+        default_stride(p2), default_stride(z2), rowvector(rho),
+        rowvector(prev_rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_1_KERNEL);
@@ -118,7 +125,7 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *rho,
             const Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto r2, auto p,
                       auto q, auto q2, auto beta, auto rho, auto stop) {
@@ -129,8 +136,9 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
                 r2(row, col) -= tmp * q2(row, col);
             }
         },
-        x->get_size(), x, compact(r), compact(r2), compact(p), compact(q),
-        compact(q2), vector(beta), vector(rho), *stop_status);
+        x->get_size(), r->get_stride(), x, default_stride(r),
+        default_stride(r2), default_stride(p), default_stride(q),
+        default_stride(q2), rowvector(beta), rowvector(rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_2_KERNEL);
diff --git a/common/solver/bicgstab_kernels.cpp b/common/solver/bicgstab_kernels.cpp
index 803089c3a0c..5fbfc7b7262 100644
--- a/common/solver/bicgstab_kernels.cpp
+++ b/common/solver/bicgstab_kernels.cpp
@@ -36,7 +36,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "core/base/simple_kernels.hpp"
+#include "core/base/simple_kernels_solver.hpp"
+
+
+using gko::solver::default_stride;
+using gko::solver::rowvector;
 
 
 namespace gko {
@@ -62,7 +66,7 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 matrix::Dense<ValueType> *omega,
                 Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto rr, auto y,
                       auto s, auto t, auto z, auto v, auto p, auto prev_rho,
@@ -77,9 +81,11 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             rr(row, col) = z(row, col) = v(row, col) = s(row, col) =
                 t(row, col) = y(row, col) = p(row, col) = zero(rr(row, col));
         },
-        b->get_size(), b, compact(r), compact(rr), compact(y), compact(s),
-        compact(t), compact(z), compact(v), compact(p), vector(prev_rho),
-        vector(rho), vector(alpha), vector(beta), vector(gamma), vector(omega),
+        b->get_size(), b->get_stride(), default_stride(b), default_stride(r),
+        default_stride(rr), default_stride(y), default_stride(s),
+        default_stride(t), default_stride(z), default_stride(v),
+        default_stride(p), rowvector(prev_rho), rowvector(rho),
+        rowvector(alpha), rowvector(beta), rowvector(gamma), rowvector(omega),
         *stop_status);
 }
 
@@ -96,7 +102,7 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *omega,
             const Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto r, auto p, auto v, auto rho,
                       auto prev_rho, auto alpha, auto omega, auto stop) {
@@ -107,8 +113,9 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
                               tmp * (p(row, col) - omega[col] * v(row, col));
             }
         },
-        r->get_size(), compact(r), compact(p), compact(v), vector(rho),
-        vector(prev_rho), vector(alpha), vector(omega), *stop_status);
+        r->get_size(), r->get_stride(), default_stride(r), default_stride(p),
+        default_stride(v), rowvector(rho), rowvector(prev_rho),
+        rowvector(alpha), rowvector(omega), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_1_KERNEL);
@@ -123,7 +130,7 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *beta,
             const Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto r, auto s, auto v, auto rho,
                       auto alpha, auto beta, auto stop) {
@@ -135,8 +142,9 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
                 s(row, col) = r(row, col) - tmp * v(row, col);
             }
         },
-        r->get_size(), compact(r), compact(s), compact(v), vector(rho),
-        vector(alpha), vector(beta), *stop_status);
+        r->get_size(), r->get_stride(), default_stride(r), default_stride(s),
+        default_stride(v), rowvector(rho), rowvector(alpha), rowvector(beta),
+        *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_2_KERNEL);
@@ -151,7 +159,7 @@ void step_3(
     const matrix::Dense<ValueType> *beta, const matrix::Dense<ValueType> *gamma,
     matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto s, auto t,
                       auto y, auto z, auto alpha, auto beta, auto gamma,
@@ -165,8 +173,9 @@ void step_3(
                 r(row, col) = s(row, col) - tmp * t(row, col);
             }
         },
-        x->get_size(), x, compact(r), compact(s), compact(t), compact(y),
-        compact(z), vector(alpha), vector(beta), vector(gamma), vector(omega),
+        x->get_size(), r->get_stride(), x, default_stride(r), default_stride(s),
+        default_stride(t), default_stride(y), default_stride(z),
+        rowvector(alpha), rowvector(beta), rowvector(gamma), rowvector(omega),
         *stop_status);
 }
 
@@ -179,7 +188,7 @@ void finalize(std::shared_ptr<const DefaultExecutor> exec,
               const matrix::Dense<ValueType> *alpha,
               Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto x, auto y, auto alpha,
                       auto stop) {
@@ -188,7 +197,8 @@ void finalize(std::shared_ptr<const DefaultExecutor> exec,
                 stop[col].finalize();
             }
         },
-        x->get_size(), x, compact(y), vector(alpha), *stop_status);
+        x->get_size(), y->get_stride(), x, default_stride(y), rowvector(alpha),
+        *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_FINALIZE_KERNEL);
diff --git a/common/solver/cg_kernels.cpp b/common/solver/cg_kernels.cpp
index 82ed384df31..e8cbc8f691f 100644
--- a/common/solver/cg_kernels.cpp
+++ b/common/solver/cg_kernels.cpp
@@ -36,7 +36,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "core/base/simple_kernels.hpp"
+#include "core/base/simple_kernels_solver.hpp"
+
+
+using gko::solver::default_stride;
+using gko::solver::rowvector;
 
 
 namespace gko {
@@ -58,7 +62,7 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 matrix::Dense<ValueType> *rho,
                 Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto z, auto p,
                       auto q, auto prev_rho, auto rho, auto stop) {
@@ -70,8 +74,9 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             r(row, col) = b(row, col);
             z(row, col) = p(row, col) = q(row, col) = zero(z(row, col));
         },
-        b->get_size(), b, compact(r), compact(z), compact(p), compact(q),
-        vector(prev_rho), vector(rho), *stop_status);
+        b->get_size(), b->get_stride(), b, default_stride(r), default_stride(z),
+        default_stride(p), default_stride(q), rowvector(prev_rho),
+        rowvector(rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
@@ -84,7 +89,7 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *prev_rho,
             const Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto p, auto z, auto rho,
                       auto prev_rho, auto stop) {
@@ -93,8 +98,8 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
                 p(row, col) = z(row, col) + tmp * p(row, col);
             }
         },
-        p->get_size(), compact(p), compact(z), vector(rho), vector(prev_rho),
-        *stop_status);
+        p->get_size(), p->get_stride(), default_stride(p), default_stride(z),
+        rowvector(rho), rowvector(prev_rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
@@ -109,7 +114,7 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *rho,
             const Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto p, auto q,
                       auto beta, auto rho, auto stop) {
@@ -119,8 +124,8 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
                 r(row, col) -= tmp * q(row, col);
             }
         },
-        x->get_size(), x, compact(r), compact(p), compact(q), vector(beta),
-        vector(rho), *stop_status);
+        x->get_size(), r->get_stride(), x, default_stride(r), default_stride(p),
+        default_stride(q), rowvector(beta), rowvector(rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
diff --git a/common/solver/cgs_kernels.cpp b/common/solver/cgs_kernels.cpp
index df96ab4976c..51d93bd5ca1 100644
--- a/common/solver/cgs_kernels.cpp
+++ b/common/solver/cgs_kernels.cpp
@@ -36,7 +36,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "core/base/simple_kernels.hpp"
+#include "core/base/simple_kernels_solver.hpp"
+
+
+using gko::solver::default_stride;
+using gko::solver::rowvector;
 
 
 namespace gko {
@@ -63,7 +67,7 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 matrix::Dense<ValueType> *rho,
                 Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto r_tld, auto p,
                       auto q, auto u, auto u_hat, auto v_hat, auto t,
@@ -79,10 +83,11 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             u(row, col) = u_hat(row, col) = p(row, col) = q(row, col) =
                 v_hat(row, col) = t(row, col) = zero(u(row, col));
         },
-        b->get_size(), b, compact(r), compact(r_tld), compact(p), compact(q),
-        compact(u), compact(u_hat), compact(v_hat), compact(t), vector(alpha),
-        vector(beta), vector(gamma), vector(prev_rho), vector(rho),
-        *stop_status);
+        b->get_size(), b->get_stride(), default_stride(b), default_stride(r),
+        default_stride(r_tld), default_stride(p), default_stride(q),
+        default_stride(u), default_stride(u_hat), default_stride(v_hat),
+        default_stride(t), rowvector(alpha), rowvector(beta), rowvector(gamma),
+        rowvector(prev_rho), rowvector(rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_INITIALIZE_KERNEL);
@@ -96,7 +101,7 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *prev_rho,
             const Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto r, auto u, auto p, auto q,
                       auto beta, auto rho, auto prev_rho, auto stop) {
@@ -111,8 +116,9 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
                     u(row, col) + tmp * (q(row, col) + tmp * p(row, col));
             }
         },
-        r->get_size(), compact(r), compact(u), compact(p), compact(q),
-        vector(beta), vector(rho), vector(prev_rho), *stop_status);
+        r->get_size(), r->get_stride(), default_stride(r), default_stride(u),
+        default_stride(p), default_stride(q), rowvector(beta), rowvector(rho),
+        rowvector(prev_rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_1_KERNEL);
@@ -127,7 +133,7 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *gamma,
             const Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto u, auto v_hat, auto q, auto t,
                       auto alpha, auto rho, auto gamma, auto stop) {
@@ -141,8 +147,9 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
                 t(row, col) = u(row, col) + q(row, col);
             }
         },
-        u->get_size(), compact(u), compact(v_hat), compact(q), compact(t),
-        vector(alpha), vector(rho), vector(gamma), *stop_status);
+        u->get_size(), u->get_stride(), default_stride(u),
+        default_stride(v_hat), default_stride(q), default_stride(t),
+        rowvector(alpha), rowvector(rho), rowvector(gamma), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_2_KERNEL);
@@ -154,7 +161,7 @@ void step_3(std::shared_ptr<const DefaultExecutor> exec,
             matrix::Dense<ValueType> *x, const matrix::Dense<ValueType> *alpha,
             const Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto t, auto u_hat, auto r, auto x,
                       auto alpha, auto stop) {
@@ -163,7 +170,8 @@ void step_3(std::shared_ptr<const DefaultExecutor> exec,
                 r(row, col) -= alpha[col] * t(row, col);
             }
         },
-        t->get_size(), compact(t), compact(u_hat), compact(r), x, vector(alpha),
+        t->get_size(), t->get_stride(), default_stride(t),
+        default_stride(u_hat), default_stride(r), x, rowvector(alpha),
         *stop_status);
 }
 
diff --git a/common/solver/fcg_kernels.cpp b/common/solver/fcg_kernels.cpp
index 1a4c92ac3bf..b33a7dc90a4 100644
--- a/common/solver/fcg_kernels.cpp
+++ b/common/solver/fcg_kernels.cpp
@@ -36,7 +36,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "core/base/simple_kernels.hpp"
+#include "core/base/simple_kernels_solver.hpp"
+
+
+using gko::solver::default_stride;
+using gko::solver::rowvector;
 
 
 namespace gko {
@@ -59,7 +63,7 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
                 matrix::Dense<ValueType> *rho, matrix::Dense<ValueType> *rho_t,
                 Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto b, auto r, auto z, auto p,
                       auto q, auto t, auto prev_rho, auto rho, auto rho_t,
@@ -72,8 +76,10 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             t(row, col) = r(row, col) = b(row, col);
             z(row, col) = p(row, col) = q(row, col) = zero(z(row, col));
         },
-        b->get_size(), b, compact(r), compact(z), compact(p), compact(q),
-        compact(t), vector(prev_rho), vector(rho), vector(rho_t), *stop_status);
+        b->get_size(), b->get_stride(), default_stride(b), default_stride(r),
+        default_stride(z), default_stride(p), default_stride(q),
+        default_stride(t), rowvector(prev_rho), rowvector(rho),
+        rowvector(rho_t), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_INITIALIZE_KERNEL);
@@ -86,7 +92,7 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *prev_rho,
             const Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto p, auto z, auto rho_t,
                       auto prev_rho, auto stop) {
@@ -95,8 +101,8 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
                 p(row, col) = z(row, col) + tmp * p(row, col);
             }
         },
-        p->get_size(), compact(p), compact(z), vector(rho_t), vector(prev_rho),
-        *stop_status);
+        p->get_size(), p->get_stride(), default_stride(p), default_stride(z),
+        rowvector(rho_t), rowvector(prev_rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_1_KERNEL);
@@ -111,7 +117,7 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *rho,
             const Array<stopping_status> *stop_status)
 {
-    run_kernel(
+    run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto row, auto col, auto x, auto r, auto t, auto p,
                       auto q, auto beta, auto rho, auto stop) {
@@ -123,8 +129,9 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
                 t(row, col) = r(row, col) - prev_r;
             }
         },
-        x->get_size(), x, compact(r), compact(t), compact(p), compact(q),
-        vector(beta), vector(rho), *stop_status);
+        x->get_size(), r->get_stride(), x, default_stride(r), default_stride(t),
+        default_stride(p), default_stride(q), rowvector(beta), rowvector(rho),
+        *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_2_KERNEL);
diff --git a/core/base/simple_kernels_solver.hpp b/core/base/simple_kernels_solver.hpp
new file mode 100644
index 00000000000..b6b8c3adf36
--- /dev/null
+++ b/core/base/simple_kernels_solver.hpp
@@ -0,0 +1,64 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_BASE_SIMPLE_KERNELS_SOLVER_HPP_
+#define GKO_CORE_BASE_SIMPLE_KERNELS_SOLVER_HPP_
+
+
+#if defined(GKO_COMPILING_CUDA)
+
+#include "cuda/base/kernel_launch_solver.cuh"
+#define GKO_DEVICE_NAMESPACE cuda
+
+#elif defined(GKO_COMPILING_HIP)
+
+#include "hip/base/kernel_launch_solver.hip.hpp"
+#define GKO_DEVICE_NAMESPACE hip
+
+#elif defined(GKO_COMPILING_DPCPP)
+
+#include "dpcpp/base/kernel_launch_solver.dp.hpp"
+#define GKO_DEVICE_NAMESPACE dpcpp
+
+#elif defined(GKO_COMPILING_OMP)
+
+#include "omp/base/kernel_launch_solver.hpp"
+#define GKO_DEVICE_NAMESPACE omp
+
+#else
+
+#error "This file should only be used inside Ginkgo device compilation"
+
+#endif
+
+
+#endif  // GKO_CORE_BASE_SIMPLE_KERNELS_SOLVER_HPP_
diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index ea62c049ccf..500ae9bf68f 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -122,32 +122,6 @@ typename to_device_type_impl<T>::type map_to_device(T &&param)
 }
 
 
-template <typename ValueType>
-struct compact_dense_wrapper {
-    ValueType *data;
-};
-
-
-template <typename T>
-struct device_unpack_2d_impl {
-    using type = T;
-    static __device__ __forceinline__ type unpack(T param, size_type)
-    {
-        return param;
-    }
-};
-
-template <typename ValueType>
-struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
-    using type = matrix_accessor<ValueType>;
-    static __device__ __forceinline__ type
-    unpack(compact_dense_wrapper<ValueType> param, size_type num_cols)
-    {
-        return {param.data, num_cols};
-    }
-};
-
-
 template <typename KernelFunction, typename... KernelArgs>
 __global__ __launch_bounds__(default_block_size) void generic_kernel_1d(
     size_type size, KernelFunction fn, KernelArgs... args)
@@ -170,7 +144,7 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
     if (row >= rows) {
         return;
     }
-    fn(row, col, device_unpack_2d_impl<KernelArgs>::unpack(args, cols)...);
+    fn(row, col, args...);
 }
 
 
@@ -178,43 +152,6 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
 }  // namespace kernels
 
 
-template <typename ValueType>
-kernels::cuda::compact_dense_wrapper<kernels::cuda::cuda_type<ValueType>>
-compact(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {kernels::cuda::as_cuda_type(mtx->get_values())};
-}
-
-
-template <typename ValueType>
-kernels::cuda::compact_dense_wrapper<const kernels::cuda::cuda_type<ValueType>>
-compact(const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {kernels::cuda::as_cuda_type(mtx->get_const_values())};
-}
-
-
-template <typename ValueType>
-kernels::cuda::cuda_type<ValueType> *vector(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return kernels::cuda::as_cuda_type(mtx->get_values());
-}
-
-
-template <typename ValueType>
-const kernels::cuda::cuda_type<ValueType> *vector(
-    const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return kernels::cuda::as_cuda_type(mtx->get_const_values());
-}
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const CudaExecutor> exec, KernelFunction fn,
                 size_type size, KernelArgs &&... args)
diff --git a/cuda/base/kernel_launch_solver.cuh b/cuda/base/kernel_launch_solver.cuh
new file mode 100644
index 00000000000..cc60cda5b00
--- /dev/null
+++ b/cuda/base/kernel_launch_solver.cuh
@@ -0,0 +1,155 @@
+
+
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CUDA_BASE_KERNEL_LAUNCH_SOLVER_CUH_
+#define GKO_CUDA_BASE_KERNEL_LAUNCH_SOLVER_CUH_
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "cuda/base/kernel_launch.cuh"
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+
+
+template <typename ValueType>
+struct default_stride_dense_wrapper {
+    ValueType *data;
+};
+
+
+template <typename T>
+struct device_unpack_solver_impl {
+    using type = T;
+    static __device__ __forceinline__ type unpack(T param, size_type)
+    {
+        return param;
+    }
+};
+
+template <typename ValueType>
+struct device_unpack_solver_impl<default_stride_dense_wrapper<ValueType>> {
+    using type = matrix_accessor<ValueType>;
+    static __device__ __forceinline__ type unpack(
+        default_stride_dense_wrapper<ValueType> param, size_type default_stride)
+    {
+        return {param.data, default_stride};
+    }
+};
+
+
+template <typename KernelFunction, typename... KernelArgs>
+__global__ __launch_bounds__(default_block_size) void generic_kernel_2d_solver(
+    size_type rows, size_type cols, size_type default_stride, KernelFunction fn,
+    KernelArgs... args)
+{
+    auto tidx = thread::get_thread_id_flat();
+    auto col = tidx % cols;
+    auto row = tidx / cols;
+    if (row >= rows) {
+        return;
+    }
+    fn(row, col,
+       device_unpack_solver_impl<KernelArgs>::unpack(args, default_stride)...);
+}
+
+
+}  // namespace cuda
+}  // namespace kernels
+
+
+namespace solver {
+
+
+template <typename ValueType>
+kernels::cuda::default_stride_dense_wrapper<kernels::cuda::cuda_type<ValueType>>
+default_stride(matrix::Dense<ValueType> *mtx)
+{
+    return {kernels::cuda::as_cuda_type(mtx->get_values())};
+}
+
+
+template <typename ValueType>
+kernels::cuda::default_stride_dense_wrapper<
+    const kernels::cuda::cuda_type<ValueType>>
+default_stride(const matrix::Dense<ValueType> *mtx)
+{
+    return {kernels::cuda::as_cuda_type(mtx->get_const_values())};
+}
+
+
+template <typename ValueType>
+kernels::cuda::cuda_type<ValueType> *rowvector(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return kernels::cuda::as_cuda_type(mtx->get_values());
+}
+
+
+template <typename ValueType>
+const kernels::cuda::cuda_type<ValueType> *rowvector(
+    const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return kernels::cuda::as_cuda_type(mtx->get_const_values());
+}
+
+
+}  // namespace solver
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void run_kernel_solver(std::shared_ptr<const CudaExecutor> exec,
+                       KernelFunction fn, dim<2> size, size_type default_stride,
+                       KernelArgs &&... args)
+{
+    cuda::device_guard guard{exec->get_device_id()};
+    constexpr auto block_size = kernels::cuda::default_block_size;
+    auto num_blocks = ceildiv(size[0] * size[1], block_size);
+    kernels::cuda::generic_kernel_2d_solver<<<num_blocks, block_size>>>(
+        size[0], size[1], default_stride, fn,
+        kernels::cuda::map_to_device(args)...);
+}
+
+
+}  // namespace gko
+
+#endif  // GKO_CUDA_BASE_KERNEL_LAUNCH_SOLVER_CUH_
diff --git a/cuda/test/base/kernel_launch.cu b/cuda/test/base/kernel_launch.cu
index 801e93d6485..3b01ea37804 100644
--- a/cuda/test/base/kernel_launch.cu
+++ b/cuda/test/base/kernel_launch.cu
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/test/utils.hpp"
+#include "cuda/base/kernel_launch_solver.cuh"
 
 
 namespace {
@@ -60,9 +61,8 @@ protected:
           iota_array(exec->get_master(), 16),
           iota_transp_array(exec->get_master(), 16),
           iota_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
-          zero_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
-          zero_dense_padded(
-              gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 5)),
+          zero_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 6)),
+          zero_dense2(gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 5)),
           vec_dense(gko::matrix::Dense<>::create(exec, dim<2>{1, 4}))
     {
         auto ref_iota_dense =
@@ -74,7 +74,7 @@ protected:
             ref_iota_dense->at(i / 4, i % 4) = i;
         }
         zero_dense->fill(0.0);
-        zero_dense_padded->fill(0.0);
+        zero_dense2->fill(0.0);
         iota_dense->copy_from(ref_iota_dense.get());
         zero_array.set_executor(exec);
         iota_array.set_executor(exec);
@@ -87,7 +87,7 @@ protected:
     gko::Array<int> iota_transp_array;
     std::unique_ptr<gko::matrix::Dense<>> iota_dense;
     std::unique_ptr<gko::matrix::Dense<>> zero_dense;
-    std::unique_ptr<gko::matrix::Dense<>> zero_dense_padded;
+    std::unique_ptr<gko::matrix::Dense<>> zero_dense2;
     std::unique_ptr<gko::matrix::Dense<>> vec_dense;
 };
 
@@ -168,9 +168,9 @@ void run1d(std::shared_ptr<gko::CudaExecutor> exec, gko::matrix::Dense<> *m)
 
 TEST_F(KernelLaunch, Runs1DDense)
 {
-    run1d(exec, zero_dense_padded.get());
+    run1d(exec, zero_dense2.get());
 
-    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+    GKO_ASSERT_MTX_NEAR(zero_dense2, iota_dense, 0.0);
 }
 
 
@@ -224,7 +224,7 @@ TEST_F(KernelLaunch, Runs2DArray)
 void run2d(std::shared_ptr<gko::CudaExecutor> exec, gko::matrix::Dense<> *m1,
            gko::matrix::Dense<> *m2, gko::matrix::Dense<> *m3)
 {
-    gko::run_kernel(
+    gko::run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d, auto d2, auto d_ptr, auto d3,
                       auto d4, auto d2_ptr, auto d3_ptr) {
@@ -241,7 +241,7 @@ void run2d(std::shared_ptr<gko::CudaExecutor> exec, gko::matrix::Dense<> *m1,
             bool pointers_correct = d.data == d_ptr && d2.data == d_ptr &&
                                     d3.data == d2_ptr && d4 == d3_ptr;
             bool strides_correct =
-                d.stride == 5 && d2.stride == 5 && d3.stride == 4;
+                d.stride == 5 && d2.stride == 5 && d3.stride == 6;
             bool accessors_2d_correct =
                 &d(0, 0) == d_ptr && &d(1, 0) == d_ptr + d.stride &&
                 &d2(0, 0) == d_ptr && &d2(1, 0) == d_ptr + d2.stride &&
@@ -255,16 +255,17 @@ void run2d(std::shared_ptr<gko::CudaExecutor> exec, gko::matrix::Dense<> *m1,
                 d(i, j) = 0;
             }
         },
-        dim<2>{4, 4}, m1, static_cast<const gko::matrix::Dense<> *>(m1),
-        m1->get_const_values(), gko::compact(m2), gko::vector(m3),
+        dim<2>{4, 4}, m2->get_stride(), m1,
+        static_cast<const gko::matrix::Dense<> *>(m1), m1->get_const_values(),
+        gko::solver::default_stride(m2), gko::solver::rowvector(m3),
         m2->get_values(), m3->get_values());
 }
 
 TEST_F(KernelLaunch, Runs2DDense)
 {
-    run2d(exec, zero_dense_padded.get(), zero_dense.get(), vec_dense.get());
+    run2d(exec, zero_dense2.get(), zero_dense.get(), vec_dense.get());
 
-    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+    GKO_ASSERT_MTX_NEAR(zero_dense2, iota_dense, 0.0);
 }
 
 
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
index 03b8f659d8c..53f2c27b09a 100644
--- a/dpcpp/base/kernel_launch.dp.hpp
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -117,29 +117,6 @@ typename to_device_type_impl<T>::type map_to_device(T &&param)
 }
 
 
-template <typename ValueType>
-struct compact_dense_wrapper {
-    ValueType *data;
-};
-
-
-template <typename T>
-struct device_unpack_2d_impl {
-    using type = T;
-    static type unpack(T param, size_type) { return param; }
-};
-
-template <typename ValueType>
-struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
-    using type = matrix_accessor<ValueType>;
-    static type unpack(compact_dense_wrapper<ValueType> param,
-                       size_type num_cols)
-    {
-        return {param.data, num_cols};
-    }
-};
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void generic_kernel_1d(sycl::handler &cgh, size_type size, KernelFunction fn,
                        KernelArgs... args)
@@ -158,7 +135,7 @@ void generic_kernel_2d(sycl::handler &cgh, size_type rows, size_type cols,
     cgh.parallel_for(sycl::range<2>{rows, cols}, [=](sycl::id<2> idx) {
         auto row = static_cast<size_type>(idx[0]);
         auto col = static_cast<size_type>(idx[1]);
-        fn(row, col, device_unpack_2d_impl<KernelArgs>::unpack(args, cols)...);
+        fn(row, col, args...);
     });
 }
 
@@ -167,42 +144,6 @@ void generic_kernel_2d(sycl::handler &cgh, size_type rows, size_type cols,
 }  // namespace kernels
 
 
-template <typename ValueType>
-kernels::dpcpp::compact_dense_wrapper<ValueType> compact(
-    matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {mtx->get_values()};
-}
-
-
-template <typename ValueType>
-kernels::dpcpp::compact_dense_wrapper<const ValueType> compact(
-    const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {mtx->get_const_values()};
-}
-
-
-template <typename ValueType>
-ValueType *vector(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return mtx->get_values();
-}
-
-
-template <typename ValueType>
-const ValueType *vector(const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return mtx->get_const_values();
-}
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const DpcppExecutor> exec, KernelFunction fn,
                 size_type size, KernelArgs &&... args)
diff --git a/dpcpp/base/kernel_launch_solver.dp.hpp b/dpcpp/base/kernel_launch_solver.dp.hpp
new file mode 100644
index 00000000000..919d06bf150
--- /dev/null
+++ b/dpcpp/base/kernel_launch_solver.dp.hpp
@@ -0,0 +1,148 @@
+
+
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_BASE_KERNEL_LAUNCH_SOLVER_DP_HPP_
+#define GKO_DPCPP_BASE_KERNEL_LAUNCH_SOLVER_DP_HPP_
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "dpcpp/base/kernel_launch.dp.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+template <typename ValueType>
+struct default_stride_dense_wrapper {
+    ValueType *data;
+};
+
+
+template <typename T>
+struct device_unpack_solver_impl {
+    using type = T;
+    static type unpack(T param, size_type) { return param; }
+};
+
+template <typename ValueType>
+struct device_unpack_solver_impl<default_stride_dense_wrapper<ValueType>> {
+    using type = matrix_accessor<ValueType>;
+    static type unpack(default_stride_dense_wrapper<ValueType> param,
+                       size_type default_stride)
+    {
+        return {param.data, default_stride};
+    }
+};
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void generic_kernel_2d_solver(sycl::handler &cgh, size_type rows,
+                              size_type cols, size_type default_stride,
+                              KernelFunction fn, KernelArgs... args)
+{
+    cgh.parallel_for(sycl::range<2>{rows, cols}, [=](sycl::id<2> idx) {
+        auto row = static_cast<size_type>(idx[0]);
+        auto col = static_cast<size_type>(idx[1]);
+        fn(row, col,
+           device_unpack_solver_impl<KernelArgs>::unpack(args,
+                                                         default_stride)...);
+    });
+}
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+
+
+namespace solver {
+
+
+template <typename ValueType>
+kernels::dpcpp::default_stride_dense_wrapper<ValueType> default_stride(
+    matrix::Dense<ValueType> *mtx)
+{
+    return {mtx->get_values()};
+}
+
+
+template <typename ValueType>
+kernels::dpcpp::default_stride_dense_wrapper<const ValueType> default_stride(
+    const matrix::Dense<ValueType> *mtx)
+{
+    return {mtx->get_const_values()};
+}
+
+
+template <typename ValueType>
+ValueType *rowvector(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return mtx->get_values();
+}
+
+
+template <typename ValueType>
+const ValueType *rowvector(const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return mtx->get_const_values();
+}
+
+
+}  // namespace solver
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void run_kernel_solver(std::shared_ptr<const DpcppExecutor> exec,
+                       KernelFunction fn, dim<2> size, size_type default_stride,
+                       KernelArgs &&... args)
+{
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        kernels::dpcpp::generic_kernel_2d_solver(
+            cgh, size[0], size[1], default_stride, fn,
+            kernels::dpcpp::map_to_device(args)...);
+    });
+}
+
+
+}  // namespace gko
+
+#endif  // GKO_HIP_BASE_KERNEL_LAUNCH_SOLVER_HIP_HPP_
diff --git a/dpcpp/test/base/kernel_launch.dp.cpp b/dpcpp/test/base/kernel_launch.dp.cpp
index ab03967e872..6572b34e5a1 100644
--- a/dpcpp/test/base/kernel_launch.dp.cpp
+++ b/dpcpp/test/base/kernel_launch.dp.cpp
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/test/utils.hpp"
+#include "dpcpp/base/kernel_launch_solver.dp.hpp"
 
 
 namespace {
@@ -68,8 +69,8 @@ class KernelLaunch : public ::testing::Test {
           iota_array(exec->get_master(), 16),
           iota_transp_array(exec->get_master(), 16),
           iota_dense(Mtx::create(exec, dim<2>{4, 4})),
-          zero_dense(Mtx::create(exec, dim<2>{4, 4})),
-          zero_dense_padded(Mtx::create(exec, dim<2>{4, 4}, 5)),
+          zero_dense(Mtx::create(exec, dim<2>{4, 4}, 6)),
+          zero_dense2(Mtx::create(exec, dim<2>{4, 4}, 5)),
           vec_dense(Mtx::create(exec, dim<2>{1, 4}))
     {
         auto ref_iota_dense = Mtx::create(exec->get_master(), dim<2>{4, 4});
@@ -80,7 +81,7 @@ class KernelLaunch : public ::testing::Test {
             ref_iota_dense->at(i / 4, i % 4) = i;
         }
         zero_dense->fill(0.0);
-        zero_dense_padded->fill(0.0);
+        zero_dense2->fill(0.0);
         iota_dense->copy_from(ref_iota_dense.get());
         zero_array.set_executor(exec);
         iota_array.set_executor(exec);
@@ -93,7 +94,7 @@ class KernelLaunch : public ::testing::Test {
     gko::Array<int> iota_transp_array;
     std::unique_ptr<Mtx> iota_dense;
     std::unique_ptr<Mtx> zero_dense;
-    std::unique_ptr<Mtx> zero_dense_padded;
+    std::unique_ptr<Mtx> zero_dense2;
     std::unique_ptr<Mtx> vec_dense;
 };
 
@@ -158,11 +159,10 @@ TEST_F(KernelLaunch, Runs1DDense)
                 d(i / 4, i % 4) = 0;
             }
         },
-        16, zero_dense_padded.get(),
-        static_cast<const Mtx *>(zero_dense_padded.get()),
-        zero_dense_padded->get_const_values());
+        16, zero_dense2.get(), static_cast<const Mtx *>(zero_dense2.get()),
+        zero_dense2->get_const_values());
 
-    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+    GKO_ASSERT_MTX_NEAR(zero_dense2, iota_dense, 0.0);
 }
 
 
@@ -205,7 +205,7 @@ TEST_F(KernelLaunch, Runs2DArray)
 
 TEST_F(KernelLaunch, Runs2DDense)
 {
-    gko::run_kernel(
+    gko::run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d, auto d2, auto d_ptr, auto d3,
                       auto d4, auto d2_ptr, auto d3_ptr) {
@@ -226,7 +226,7 @@ TEST_F(KernelLaunch, Runs2DDense)
             bool pointers_correct = d.data == d_ptr && d2.data == d_ptr &&
                                     d3.data == d2_ptr && d4 == d3_ptr;
             bool strides_correct =
-                d.stride == 5 && d2.stride == 5 && d3.stride == 4;
+                d.stride == 5 && d2.stride == 5 && d3.stride == 6;
             bool accessors_2d_correct =
                 &d(0, 0) == d_ptr && &d(1, 0) == d_ptr + d.stride &&
                 &d2(0, 0) == d_ptr && &d2(1, 0) == d_ptr + d2.stride &&
@@ -240,13 +240,14 @@ TEST_F(KernelLaunch, Runs2DDense)
                 d(i, j) = 0;
             }
         },
-        dim<2>{4, 4}, zero_dense_padded.get(),
-        static_cast<const Mtx *>(zero_dense_padded.get()),
-        zero_dense_padded->get_const_values(), gko::compact(zero_dense.get()),
-        gko::vector(vec_dense.get()), zero_dense->get_values(),
+        dim<2>{4, 4}, zero_dense->get_stride(), zero_dense2.get(),
+        static_cast<const Mtx *>(zero_dense2.get()),
+        zero_dense2->get_const_values(),
+        gko::solver::default_stride(zero_dense.get()),
+        gko::solver::rowvector(vec_dense.get()), zero_dense->get_values(),
         vec_dense->get_values());
 
-    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+    GKO_ASSERT_MTX_NEAR(zero_dense2, iota_dense, 0.0);
 }
 
 
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index 0bf70a515f4..4687272de0f 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -125,32 +125,6 @@ typename to_device_type_impl<T>::type map_to_device(T &&param)
 }
 
 
-template <typename ValueType>
-struct compact_dense_wrapper {
-    ValueType *data;
-};
-
-
-template <typename T>
-struct device_unpack_2d_impl {
-    using type = T;
-    static __device__ __forceinline__ type unpack(T param, size_type)
-    {
-        return param;
-    }
-};
-
-template <typename ValueType>
-struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
-    using type = matrix_accessor<ValueType>;
-    static __device__ __forceinline__ type
-    unpack(compact_dense_wrapper<ValueType> param, size_type num_cols)
-    {
-        return {param.data, num_cols};
-    }
-};
-
-
 template <typename KernelFunction, typename... KernelArgs>
 __global__ __launch_bounds__(default_block_size) void generic_kernel_1d(
     size_type size, KernelFunction fn, KernelArgs... args)
@@ -173,7 +147,7 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
     if (row >= rows) {
         return;
     }
-    fn(row, col, device_unpack_2d_impl<KernelArgs>::unpack(args, cols)...);
+    fn(row, col, args...);
 }
 
 
@@ -181,43 +155,6 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
 }  // namespace kernels
 
 
-template <typename ValueType>
-kernels::hip::compact_dense_wrapper<kernels::hip::hip_type<ValueType>> compact(
-    matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {kernels::hip::as_hip_type(mtx->get_values())};
-}
-
-
-template <typename ValueType>
-kernels::hip::compact_dense_wrapper<const kernels::hip::hip_type<ValueType>>
-compact(const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {kernels::hip::as_hip_type(mtx->get_const_values())};
-}
-
-
-template <typename ValueType>
-kernels::hip::hip_type<ValueType> *vector(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return kernels::hip::as_hip_type(mtx->get_values());
-}
-
-
-template <typename ValueType>
-const kernels::hip::hip_type<ValueType> *vector(
-    const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return kernels::hip::as_hip_type(mtx->get_const_values());
-}
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const HipExecutor> exec, KernelFunction fn,
                 size_type size, KernelArgs &&... args)
diff --git a/hip/base/kernel_launch_solver.hip.hpp b/hip/base/kernel_launch_solver.hip.hpp
new file mode 100644
index 00000000000..823cf72afba
--- /dev/null
+++ b/hip/base/kernel_launch_solver.hip.hpp
@@ -0,0 +1,158 @@
+
+
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_HIP_BASE_KERNEL_LAUNCH_SOLVER_HIP_HPP_
+#define GKO_HIP_BASE_KERNEL_LAUNCH_SOLVER_HIP_HPP_
+
+
+#include <hip/hip_runtime.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "hip/base/kernel_launch.hip.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace hip {
+
+
+template <typename ValueType>
+struct default_stride_dense_wrapper {
+    ValueType *data;
+};
+
+
+template <typename T>
+struct device_unpack_solver_impl {
+    using type = T;
+    static __device__ __forceinline__ type unpack(T param, size_type)
+    {
+        return param;
+    }
+};
+
+template <typename ValueType>
+struct device_unpack_solver_impl<default_stride_dense_wrapper<ValueType>> {
+    using type = matrix_accessor<ValueType>;
+    static __device__ __forceinline__ type unpack(
+        default_stride_dense_wrapper<ValueType> param, size_type default_stride)
+    {
+        return {param.data, default_stride};
+    }
+};
+
+
+template <typename KernelFunction, typename... KernelArgs>
+__global__ __launch_bounds__(default_block_size) void generic_kernel_2d_solver(
+    size_type rows, size_type cols, size_type default_stride, KernelFunction fn,
+    KernelArgs... args)
+{
+    auto tidx = thread::get_thread_id_flat();
+    auto col = tidx % cols;
+    auto row = tidx / cols;
+    if (row >= rows) {
+        return;
+    }
+    fn(row, col,
+       device_unpack_solver_impl<KernelArgs>::unpack(args, default_stride)...);
+}
+
+
+}  // namespace hip
+}  // namespace kernels
+
+
+namespace solver {
+
+
+template <typename ValueType>
+kernels::hip::default_stride_dense_wrapper<kernels::hip::hip_type<ValueType>>
+default_stride(matrix::Dense<ValueType> *mtx)
+{
+    return {kernels::hip::as_hip_type(mtx->get_values())};
+}
+
+
+template <typename ValueType>
+kernels::hip::default_stride_dense_wrapper<
+    const kernels::hip::hip_type<ValueType>>
+default_stride(const matrix::Dense<ValueType> *mtx)
+{
+    return {kernels::hip::as_hip_type(mtx->get_const_values())};
+}
+
+
+template <typename ValueType>
+kernels::hip::hip_type<ValueType> *rowvector(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return kernels::hip::as_hip_type(mtx->get_values());
+}
+
+
+template <typename ValueType>
+const kernels::hip::hip_type<ValueType> *rowvector(
+    const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return kernels::hip::as_hip_type(mtx->get_const_values());
+}
+
+
+}  // namespace solver
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void run_kernel_solver(std::shared_ptr<const HipExecutor> exec,
+                       KernelFunction fn, dim<2> size, size_type default_stride,
+                       KernelArgs &&... args)
+{
+    hip::device_guard guard{exec->get_device_id()};
+    constexpr auto block_size = kernels::hip::default_block_size;
+    auto num_blocks = ceildiv(size[0] * size[1], block_size);
+    hipLaunchKernelGGL(kernels::hip::generic_kernel_2d_solver, num_blocks,
+                       block_size, 0, 0, size[0], size[1], default_stride, fn,
+                       kernels::hip::map_to_device(args)...);
+}
+
+
+}  // namespace gko
+
+#endif  // GKO_HIP_BASE_KERNEL_LAUNCH_SOLVER_HIP_HPP_
diff --git a/hip/test/base/kernel_launch.hip.cpp b/hip/test/base/kernel_launch.hip.cpp
index d35fd92460e..65b901841f7 100644
--- a/hip/test/base/kernel_launch.hip.cpp
+++ b/hip/test/base/kernel_launch.hip.cpp
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/test/utils.hpp"
+#include "hip/base/kernel_launch_solver.hip.hpp"
 
 
 namespace {
@@ -59,9 +60,8 @@ class KernelLaunch : public ::testing::Test {
           iota_array(exec->get_master(), 16),
           iota_transp_array(exec->get_master(), 16),
           iota_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
-          zero_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
-          zero_dense_padded(
-              gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 5)),
+          zero_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 6)),
+          zero_dense2(gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 5)),
           vec_dense(gko::matrix::Dense<>::create(exec, dim<2>{1, 4}))
     {
         auto ref_iota_dense =
@@ -73,7 +73,7 @@ class KernelLaunch : public ::testing::Test {
             ref_iota_dense->at(i / 4, i % 4) = i;
         }
         zero_dense->fill(0.0);
-        zero_dense_padded->fill(0.0);
+        zero_dense2->fill(0.0);
         iota_dense->copy_from(ref_iota_dense.get());
         zero_array.set_executor(exec);
         iota_array.set_executor(exec);
@@ -86,7 +86,7 @@ class KernelLaunch : public ::testing::Test {
     gko::Array<int> iota_transp_array;
     std::unique_ptr<gko::matrix::Dense<>> iota_dense;
     std::unique_ptr<gko::matrix::Dense<>> zero_dense;
-    std::unique_ptr<gko::matrix::Dense<>> zero_dense_padded;
+    std::unique_ptr<gko::matrix::Dense<>> zero_dense2;
     std::unique_ptr<gko::matrix::Dense<>> vec_dense;
 };
 
@@ -167,9 +167,9 @@ void run1d(std::shared_ptr<gko::HipExecutor> exec, gko::matrix::Dense<> *m)
 
 TEST_F(KernelLaunch, Runs1DDense)
 {
-    run1d(exec, zero_dense_padded.get());
+    run1d(exec, zero_dense2.get());
 
-    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+    GKO_ASSERT_MTX_NEAR(zero_dense2, iota_dense, 0.0);
 }
 
 
@@ -223,7 +223,7 @@ TEST_F(KernelLaunch, Runs2DArray)
 void run2d(std::shared_ptr<gko::HipExecutor> exec, gko::matrix::Dense<> *m1,
            gko::matrix::Dense<> *m2, gko::matrix::Dense<> *m3)
 {
-    gko::run_kernel(
+    gko::run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d, auto d2, auto d_ptr, auto d3,
                       auto d4, auto d2_ptr, auto d3_ptr) {
@@ -240,7 +240,7 @@ void run2d(std::shared_ptr<gko::HipExecutor> exec, gko::matrix::Dense<> *m1,
             bool pointers_correct = d.data == d_ptr && d2.data == d_ptr &&
                                     d3.data == d2_ptr && d4 == d3_ptr;
             bool strides_correct =
-                d.stride == 5 && d2.stride == 5 && d3.stride == 4;
+                d.stride == 5 && d2.stride == 5 && d3.stride == 6;
             bool accessors_2d_correct =
                 &d(0, 0) == d_ptr && &d(1, 0) == d_ptr + d.stride &&
                 &d2(0, 0) == d_ptr && &d2(1, 0) == d_ptr + d2.stride &&
@@ -254,16 +254,17 @@ void run2d(std::shared_ptr<gko::HipExecutor> exec, gko::matrix::Dense<> *m1,
                 d(i, j) = 0;
             }
         },
-        dim<2>{4, 4}, m1, static_cast<const gko::matrix::Dense<> *>(m1),
-        m1->get_const_values(), gko::compact(m2), gko::vector(m3),
+        dim<2>{4, 4}, m2->get_stride(), m1,
+        static_cast<const gko::matrix::Dense<> *>(m1), m1->get_const_values(),
+        gko::solver::default_stride(m2), gko::solver::rowvector(m3),
         m2->get_values(), m3->get_values());
 }
 
 TEST_F(KernelLaunch, Runs2DDense)
 {
-    run2d(exec, zero_dense_padded.get(), zero_dense.get(), vec_dense.get());
+    run2d(exec, zero_dense2.get(), zero_dense.get(), vec_dense.get());
 
-    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+    GKO_ASSERT_MTX_NEAR(zero_dense2, iota_dense, 0.0);
 }
 
 
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 41fcb32cd4f..45ef7ef182c 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -834,7 +834,8 @@ class Dense
      */
     virtual std::unique_ptr<Dense> create_with_same_config() const
     {
-        return Dense::create(this->get_executor(), this->get_size());
+        return Dense::create(this->get_executor(), this->get_size(),
+                             this->get_stride());
     }
 
     /**
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index 9b466250886..dcad0da9b07 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -114,78 +114,10 @@ typename to_device_type_impl<T>::type map_to_device(T &&param)
 }
 
 
-template <typename ValueType>
-struct compact_dense_wrapper {
-    ValueType *data;
-};
-
-
-template <typename T>
-struct device_unpack_2d_impl {
-    using type = T;
-    static type unpack(T param, size_type) { return param; }
-};
-
-template <typename ValueType>
-struct device_unpack_2d_impl<compact_dense_wrapper<ValueType>> {
-    using type = matrix_accessor<ValueType>;
-    static type unpack(compact_dense_wrapper<ValueType> param,
-                       size_type num_cols)
-    {
-        return {param.data, num_cols};
-    }
-};
-
-
-template <typename T>
-typename device_unpack_2d_impl<typename to_device_type_impl<T>::type>::type
-map_unpack_to_device(T &&param, size_type num_cols)
-{
-    return device_unpack_2d_impl<typename to_device_type_impl<T>::type>::unpack(
-        to_device_type_impl<T>::map_to_device(param), num_cols);
-}
-
-
 }  // namespace omp
 }  // namespace kernels
 
 
-template <typename ValueType>
-kernels::omp::compact_dense_wrapper<ValueType> compact(
-    matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {mtx->get_values()};
-}
-
-
-template <typename ValueType>
-kernels::omp::compact_dense_wrapper<const ValueType> compact(
-    const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_stride() == mtx->get_size()[1]);
-    return {mtx->get_const_values()};
-}
-
-
-template <typename ValueType>
-ValueType *vector(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return mtx->get_values();
-}
-
-
-template <typename ValueType>
-const ValueType *vector(const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
-    return mtx->get_const_values();
-}
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
                 size_type size, KernelArgs &&... args)
@@ -196,28 +128,26 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
     }
 }
 
-template <size_type cols, typename KernelFunction, typename... KernelArgs>
-void run_kernel_fixed_cols(std::shared_ptr<const OmpExecutor> exec,
-                           KernelFunction fn, dim<2> size,
-                           KernelArgs &&... args)
+template <size_type cols, typename KernelFunction, typename... MappedKernelArgs>
+void run_kernel_fixed_cols_impl(std::shared_ptr<const OmpExecutor> exec,
+                                KernelFunction fn, dim<2> size,
+                                MappedKernelArgs... args)
 {
     const auto rows = size[0];
 #pragma omp parallel for
     for (size_type row = 0; row < rows; row++) {
 #pragma unroll
         for (size_type col = 0; col < cols; col++) {
-            [&]() {
-                fn(row, col, kernels::omp::map_unpack_to_device(args, cols)...);
-            }();
+            [&]() { fn(row, col, args...); }();
         }
     }
 }
 
 template <size_type remainder_cols, typename KernelFunction,
-          typename... KernelArgs>
-void run_kernel_blocked_cols(std::shared_ptr<const OmpExecutor> exec,
-                             KernelFunction fn, dim<2> size,
-                             KernelArgs &&... args)
+          typename... MappedKernelArgs>
+void run_kernel_blocked_cols_impl(std::shared_ptr<const OmpExecutor> exec,
+                                  KernelFunction fn, dim<2> size,
+                                  MappedKernelArgs... args)
 {
     const auto rows = size[0];
     const auto cols = size[1];
@@ -228,25 +158,19 @@ void run_kernel_blocked_cols(std::shared_ptr<const OmpExecutor> exec,
         for (size_type base_col = 0; base_col < rounded_cols; base_col += 4) {
 #pragma unroll
             for (size_type i = 0; i < 4; i++) {
-                [&]() {
-                    fn(row, base_col + i,
-                       kernels::omp::map_unpack_to_device(args, cols)...);
-                }();
+                [&]() { fn(row, base_col + i, args...); }();
             }
         }
 #pragma unroll
         for (size_type i = 0; i < remainder_cols; i++) {
-            [&]() {
-                fn(row, rounded_cols + i,
-                   kernels::omp::map_unpack_to_device(args, cols)...);
-            }();
+            [&]() { fn(row, rounded_cols + i, args...); }();
         }
     }
 }
 
-template <typename KernelFunction, typename... KernelArgs>
-void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
-                dim<2> size, KernelArgs &&... args)
+template <typename KernelFunction, typename... MappedKernelArgs>
+void run_kernel_impl(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
+                     dim<2> size, MappedKernelArgs... args)
 {
     const auto rows = size[0];
     const auto cols = size[1];
@@ -254,44 +178,36 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
         return;
     }
     if (cols == 1) {
-        run_kernel_fixed_cols<1>(exec, fn, size,
-                                 std::forward<KernelArgs>(args)...);
+        run_kernel_fixed_cols_impl<1>(exec, fn, size, args...);
         return;
     }
     if (cols == 2) {
-        run_kernel_fixed_cols<2>(exec, fn, size,
-                                 std::forward<KernelArgs>(args)...);
+        run_kernel_fixed_cols_impl<2>(exec, fn, size, args...);
         return;
     }
     if (cols == 3) {
-        run_kernel_fixed_cols<3>(exec, fn, size,
-                                 std::forward<KernelArgs>(args)...);
+        run_kernel_fixed_cols_impl<3>(exec, fn, size, args...);
         return;
     }
     if (cols == 4) {
-        run_kernel_fixed_cols<4>(exec, fn, size,
-                                 std::forward<KernelArgs>(args)...);
+        run_kernel_fixed_cols_impl<4>(exec, fn, size, args...);
         return;
     }
     const auto rem_cols = cols % 4;
     if (rem_cols == 0) {
-        run_kernel_blocked_cols<0>(exec, fn, size,
-                                   std::forward<KernelArgs>(args)...);
+        run_kernel_blocked_cols_impl<0>(exec, fn, size, args...);
         return;
     }
     if (rem_cols == 1) {
-        run_kernel_blocked_cols<1>(exec, fn, size,
-                                   std::forward<KernelArgs>(args)...);
+        run_kernel_blocked_cols_impl<1>(exec, fn, size, args...);
         return;
     }
     if (rem_cols == 2) {
-        run_kernel_blocked_cols<2>(exec, fn, size,
-                                   std::forward<KernelArgs>(args)...);
+        run_kernel_blocked_cols_impl<2>(exec, fn, size, args...);
         return;
     }
     if (rem_cols == 3) {
-        run_kernel_blocked_cols<3>(exec, fn, size,
-                                   std::forward<KernelArgs>(args)...);
+        run_kernel_blocked_cols_impl<3>(exec, fn, size, args...);
         return;
     }
     // should be unreachable
@@ -299,6 +215,14 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
 }
 
 
+template <typename KernelFunction, typename... KernelArgs>
+void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
+                dim<2> size, KernelArgs &&... args)
+{
+    run_kernel_impl(exec, fn, size, kernels::omp::map_to_device(args)...);
+}
+
+
 }  // namespace gko
 
 #endif  // GKO_OMP_BASE_KERNEL_LAUNCH_HPP_
diff --git a/omp/base/kernel_launch_solver.hpp b/omp/base/kernel_launch_solver.hpp
new file mode 100644
index 00000000000..63ede5fd7a3
--- /dev/null
+++ b/omp/base/kernel_launch_solver.hpp
@@ -0,0 +1,138 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_OMP_BASE_KERNEL_LAUNCH_SOLVER_HPP_
+#define GKO_OMP_BASE_KERNEL_LAUNCH_SOLVER_HPP_
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "omp/base/kernel_launch.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+
+
+template <typename ValueType>
+struct default_stride_dense_wrapper {
+    ValueType *data;
+};
+
+
+template <typename T>
+struct device_unpack_solver_impl {
+    using type = T;
+    static type unpack(T param, size_type) { return param; }
+};
+
+template <typename ValueType>
+struct device_unpack_solver_impl<default_stride_dense_wrapper<ValueType>> {
+    using type = matrix_accessor<ValueType>;
+    static type unpack(default_stride_dense_wrapper<ValueType> param,
+                       size_type default_stride)
+    {
+        return {param.data, default_stride};
+    }
+};
+
+
+template <typename T>
+typename device_unpack_solver_impl<typename to_device_type_impl<T>::type>::type
+map_to_device_solver(T &&param, size_type default_stride)
+{
+    return device_unpack_solver_impl<typename to_device_type_impl<T>::type>::
+        unpack(to_device_type_impl<T>::map_to_device(param), default_stride);
+}
+
+
+}  // namespace omp
+}  // namespace kernels
+
+
+namespace solver {
+
+
+template <typename ValueType>
+kernels::omp::default_stride_dense_wrapper<ValueType> default_stride(
+    matrix::Dense<ValueType> *mtx)
+{
+    return {mtx->get_values()};
+}
+
+
+template <typename ValueType>
+kernels::omp::default_stride_dense_wrapper<const ValueType> default_stride(
+    const matrix::Dense<ValueType> *mtx)
+{
+    return {mtx->get_const_values()};
+}
+
+
+template <typename ValueType>
+ValueType *rowvector(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return mtx->get_values();
+}
+
+
+template <typename ValueType>
+const ValueType *rowvector(const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1 ||
+               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    return mtx->get_const_values();
+}
+
+
+}  // namespace solver
+
+
+template <typename KernelFunction, typename... KernelArgs>
+void run_kernel_solver(std::shared_ptr<const OmpExecutor> exec,
+                       KernelFunction fn, dim<2> size, size_type default_stride,
+                       KernelArgs &&... args)
+{
+    run_kernel_impl(
+        exec, fn, size,
+        kernels::omp::map_to_device_solver(args, default_stride)...);
+}
+
+
+}  // namespace gko
+
+#endif  // GKO_OMP_BASE_KERNEL_LAUNCH_SOLVER_HPP_
diff --git a/omp/test/base/kernel_launch.cpp b/omp/test/base/kernel_launch.cpp
index 5a82b90bcf2..d03a5a26c3a 100644
--- a/omp/test/base/kernel_launch.cpp
+++ b/omp/test/base/kernel_launch.cpp
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/test/utils.hpp"
+#include "omp/base/kernel_launch_solver.hpp"
 
 
 namespace {
@@ -59,9 +60,8 @@ class KernelLaunch : public ::testing::Test {
           iota_array(exec, 16),
           iota_transp_array(exec, 16),
           iota_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
-          zero_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4})),
-          zero_dense_padded(
-              gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 5)),
+          zero_dense(gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 6)),
+          zero_dense2(gko::matrix::Dense<>::create(exec, dim<2>{4, 4}, 5)),
           vec_dense(gko::matrix::Dense<>::create(exec, dim<2>{1, 4}))
     {
         for (int i = 0; i < 16; i++) {
@@ -70,7 +70,7 @@ class KernelLaunch : public ::testing::Test {
             iota_transp_array.get_data()[i] = (i % 4 * 4) + i / 4;
             iota_dense->at(i / 4, i % 4) = i;
             zero_dense->at(i / 4, i % 4) = 0.0;
-            zero_dense_padded->at(i / 4, i % 4) = 0.0;
+            zero_dense2->at(i / 4, i % 4) = 0.0;
         }
     }
 
@@ -80,7 +80,7 @@ class KernelLaunch : public ::testing::Test {
     gko::Array<int> iota_transp_array;
     std::unique_ptr<gko::matrix::Dense<>> iota_dense;
     std::unique_ptr<gko::matrix::Dense<>> zero_dense;
-    std::unique_ptr<gko::matrix::Dense<>> zero_dense_padded;
+    std::unique_ptr<gko::matrix::Dense<>> zero_dense2;
     std::unique_ptr<gko::matrix::Dense<>> vec_dense;
 };
 
@@ -144,11 +144,11 @@ TEST_F(KernelLaunch, Runs1DDense)
                 d(i / 4, i % 4) = 0;
             }
         },
-        16, zero_dense_padded.get(),
-        static_cast<const gko::matrix::Dense<> *>(zero_dense_padded.get()),
-        zero_dense_padded->get_const_values());
+        16, zero_dense2.get(),
+        static_cast<const gko::matrix::Dense<> *>(zero_dense2.get()),
+        zero_dense2->get_const_values());
 
-    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+    GKO_ASSERT_MTX_NEAR(zero_dense2, iota_dense, 0.0);
 }
 
 
@@ -191,7 +191,7 @@ TEST_F(KernelLaunch, Runs2DArray)
 
 TEST_F(KernelLaunch, Runs2DDense)
 {
-    gko::run_kernel(
+    gko::run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d, auto d2, auto d_ptr, auto d3,
                       auto d4, auto d2_ptr, auto d3_ptr) {
@@ -208,7 +208,7 @@ TEST_F(KernelLaunch, Runs2DDense)
             bool pointers_correct = d.data == d_ptr && d2.data == d_ptr &&
                                     d3.data == d2_ptr && d4 == d3_ptr;
             bool strides_correct =
-                d.stride == 5 && d2.stride == 5 && d3.stride == 4;
+                d.stride == 5 && d2.stride == 5 && d3.stride == 6;
             bool accessors_2d_correct =
                 &d(0, 0) == d_ptr && &d(1, 0) == d_ptr + d.stride &&
                 &d2(0, 0) == d_ptr && &d2(1, 0) == d_ptr + d2.stride &&
@@ -222,13 +222,14 @@ TEST_F(KernelLaunch, Runs2DDense)
                 d(i, j) = 0;
             }
         },
-        dim<2>{4, 4}, zero_dense_padded.get(),
-        static_cast<const gko::matrix::Dense<> *>(zero_dense_padded.get()),
-        zero_dense_padded->get_const_values(), gko::compact(zero_dense.get()),
-        gko::vector(vec_dense.get()), zero_dense->get_values(),
+        dim<2>{4, 4}, zero_dense->get_stride(), zero_dense2.get(),
+        static_cast<const gko::matrix::Dense<> *>(zero_dense2.get()),
+        zero_dense2->get_const_values(),
+        gko::solver::default_stride(zero_dense.get()),
+        gko::solver::rowvector(vec_dense.get()), zero_dense->get_values(),
         vec_dense->get_values());
 
-    GKO_ASSERT_MTX_NEAR(zero_dense_padded, iota_dense, 0.0);
+    GKO_ASSERT_MTX_NEAR(zero_dense2, iota_dense, 0.0);
 }
 
 
diff --git a/test/solver/bicg_kernels.cpp b/test/solver/bicg_kernels.cpp
index 4d087981d60..06b1726cdc4 100644
--- a/test/solver/bicg_kernels.cpp
+++ b/test/solver/bicg_kernels.cpp
@@ -65,7 +65,6 @@ class Bicg : public ::testing::Test {
 #endif
     using Mtx = gko::matrix::Dense<value_type>;
     using index_type = gko::int32;
-    using Csr = gko::matrix::Csr<value_type, index_type>;
 
     Bicg() : rand_engine(30) {}
 
@@ -80,10 +79,8 @@ class Bicg : public ::testing::Test {
             FAIL() << "Could not find the file \"" << file_name
                    << "\", which is required for this test.\n";
         }
-        csr = gko::read<Csr>(input_file, ref);
-        auto csr_hip_temp = Csr::create(exec);
-        csr_hip_temp->copy_from(gko::lend(csr));
-        d_csr = gko::give(csr_hip_temp);
+        mtx_ani = gko::read<Mtx>(input_file, ref);
+        d_mtx_ani = gko::clone(exec, mtx_ani.get());
     }
 
     void TearDown()
@@ -94,33 +91,35 @@ class Bicg : public ::testing::Test {
     }
 
     std::unique_ptr<Mtx> gen_mtx(gko::size_type num_rows,
-                                 gko::size_type num_cols)
+                                 gko::size_type num_cols, gko::size_type stride)
     {
-        return gko::test::generate_random_matrix<Mtx>(
+        auto tmp_mtx = gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(num_cols, num_cols),
             std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
+        auto result = Mtx::create(ref, gko::dim<2>{num_rows, num_cols}, stride);
+        result->copy_from(tmp_mtx.get());
+        return result;
     }
 
     void initialize_data()
     {
         gko::size_type m = 597;
         gko::size_type n = 43;
-        b_full = gen_mtx(m, n + 2);
-        b = b_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
-        r = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        q = gen_mtx(m, n);
-        r2 = gen_mtx(m, n);
-        z2 = gen_mtx(m, n);
-        p2 = gen_mtx(m, n);
-        q2 = gen_mtx(m, n);
-        x_full = gen_mtx(m, n + 3);
-        x = x_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
-        beta = gen_mtx(1, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
+        // all vectors need the same stride as b, except x
+        b = gen_mtx(m, n, n + 2);
+        r = gen_mtx(m, n, n + 2);
+        z = gen_mtx(m, n, n + 2);
+        p = gen_mtx(m, n, n + 2);
+        q = gen_mtx(m, n, n + 2);
+        r2 = gen_mtx(m, n, n + 2);
+        z2 = gen_mtx(m, n, n + 2);
+        p2 = gen_mtx(m, n, n + 2);
+        q2 = gen_mtx(m, n, n + 2);
+        x = gen_mtx(m, n, n + 3);
+        beta = gen_mtx(1, n, n);
+        prev_rho = gen_mtx(1, n, n);
+        rho = gen_mtx(1, n, n);
         // check correct handling for zero values
         beta->at(2) = 0.0;
         prev_rho->at(2) = 0.0;
@@ -168,7 +167,6 @@ class Bicg : public ::testing::Test {
 
     std::ranlux48 rand_engine;
 
-    std::unique_ptr<Mtx> b_full;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
     std::unique_ptr<Mtx> z;
@@ -178,12 +176,11 @@ class Bicg : public ::testing::Test {
     std::unique_ptr<Mtx> z2;
     std::unique_ptr<Mtx> p2;
     std::unique_ptr<Mtx> q2;
-    std::unique_ptr<Mtx> x_full;
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> beta;
     std::unique_ptr<Mtx> prev_rho;
     std::unique_ptr<Mtx> rho;
-    std::shared_ptr<Csr> csr;
+    std::shared_ptr<Mtx> mtx_ani;
     std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
 
     std::unique_ptr<Mtx> d_b;
@@ -199,7 +196,7 @@ class Bicg : public ::testing::Test {
     std::unique_ptr<Mtx> d_beta;
     std::unique_ptr<Mtx> d_prev_rho;
     std::unique_ptr<Mtx> d_rho;
-    std::shared_ptr<Csr> d_csr;
+    std::shared_ptr<Mtx> d_mtx_ani;
     std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
 };
 
@@ -270,10 +267,10 @@ TEST_F(Bicg, BicgStep2IsEquivalentToRef)
 
 TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
 {
-    auto mtx = gen_mtx(50, 50);
+    auto mtx = gen_mtx(50, 50, 53);
     gko::test::make_hpd(mtx.get());
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
+    auto x = gen_mtx(50, 3, 5);
+    auto b = gen_mtx(50, 3, 4);
     auto d_mtx = Mtx::create(exec);
     d_mtx->copy_from(mtx.get());
     auto d_x = Mtx::create(exec);
@@ -308,8 +305,8 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
 
 TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
 {
-    auto x = gen_mtx(36, 1);
-    auto b = gen_mtx(36, 1);
+    auto x = gen_mtx(36, 1, 2);
+    auto b = gen_mtx(36, 1, 3);
     auto d_x = Mtx::create(exec);
     d_x->copy_from(x.get());
     auto d_b = Mtx::create(exec);
@@ -330,8 +327,8 @@ TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
                     .with_reduction_factor(::r<value_type>::value)
                     .on(exec))
             .on(exec);
-    auto solver = bicg_factory->generate(std::move(csr));
-    auto d_solver = d_bicg_factory->generate(std::move(d_csr));
+    auto solver = bicg_factory->generate(std::move(mtx_ani));
+    auto d_solver = d_bicg_factory->generate(std::move(d_mtx_ani));
 
     solver->apply(b.get(), x.get());
     d_solver->apply(d_b.get(), d_x.get());
diff --git a/test/solver/bicgstab_kernels.cpp b/test/solver/bicgstab_kernels.cpp
index 5a1403b0e35..4d8cd82700e 100644
--- a/test/solver/bicgstab_kernels.cpp
+++ b/test/solver/bicgstab_kernels.cpp
@@ -74,7 +74,7 @@ class Bicgstab : public ::testing::Test {
         ref = gko::ReferenceExecutor::create();
         init_executor(ref, exec);
 
-        mtx = gen_mtx(123, 123);
+        mtx = gen_mtx(123, 123, 125);
         gko::test::make_diag_dominant(mtx.get());
         d_mtx = Mtx::create(exec);
         d_mtx->copy_from(mtx.get());
@@ -105,36 +105,37 @@ class Bicgstab : public ::testing::Test {
     }
 
     std::unique_ptr<Mtx> gen_mtx(gko::size_type num_rows,
-                                 gko::size_type num_cols)
+                                 gko::size_type num_cols, gko::size_type stride)
     {
-        return gko::test::generate_random_matrix<Mtx>(
+        auto tmp_mtx = gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<value_type>(0.0, 1.0), rand_engine, ref);
+            std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
+        auto result = Mtx::create(ref, gko::dim<2>{num_rows, num_cols}, stride);
+        result->copy_from(tmp_mtx.get());
+        return result;
     }
 
     void initialize_data()
     {
         gko::size_type m = 597;
         gko::size_type n = 17;
-        x_full = gen_mtx(m, n + 3);
-        b_full = gen_mtx(m, n + 2);
-        x = x_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
-        b = b_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
-        r = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        rr = gen_mtx(m, n);
-        s = gen_mtx(m, n);
-        t = gen_mtx(m, n);
-        y = gen_mtx(m, n);
-        v = gen_mtx(m, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        alpha = gen_mtx(1, n);
-        beta = gen_mtx(1, n);
-        gamma = gen_mtx(1, n);
-        omega = gen_mtx(1, n);
+        x = gen_mtx(m, n, n + 3);
+        b = gen_mtx(m, n, n + 2);
+        r = gen_mtx(m, n, n + 2);
+        z = gen_mtx(m, n, n + 2);
+        p = gen_mtx(m, n, n + 2);
+        rr = gen_mtx(m, n, n + 2);
+        s = gen_mtx(m, n, n + 2);
+        t = gen_mtx(m, n, n + 2);
+        y = gen_mtx(m, n, n + 2);
+        v = gen_mtx(m, n, n + 2);
+        prev_rho = gen_mtx(1, n, n);
+        rho = gen_mtx(1, n, n);
+        alpha = gen_mtx(1, n, n);
+        beta = gen_mtx(1, n, n);
+        gamma = gen_mtx(1, n, n);
+        omega = gen_mtx(1, n, n);
         // check correct handling for zero values
         prev_rho->at(2) = 0.0;
         prev_rho->at(4) = 0.0;
@@ -198,8 +199,6 @@ class Bicgstab : public ::testing::Test {
     std::unique_ptr<Solver::Factory> exec_bicgstab_factory;
     std::unique_ptr<Solver::Factory> ref_bicgstab_factory;
 
-    std::unique_ptr<Mtx> x_full;
-    std::unique_ptr<Mtx> b_full;
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
@@ -325,8 +324,8 @@ TEST_F(Bicgstab, BicgstabApplyOneRHSIsEquivalentToRef)
     int n = 1;
     auto ref_solver = ref_bicgstab_factory->generate(mtx);
     auto exec_solver = exec_bicgstab_factory->generate(d_mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
+    auto b = gen_mtx(m, n, n + 2);
+    auto x = gen_mtx(m, n, n + 3);
     auto d_b = Mtx::create(exec);
     auto d_x = Mtx::create(exec);
     d_b->copy_from(b.get());
@@ -346,8 +345,8 @@ TEST_F(Bicgstab, BicgstabApplyMultipleRHSIsEquivalentToRef)
     int n = 16;
     auto exec_solver = exec_bicgstab_factory->generate(d_mtx);
     auto ref_solver = ref_bicgstab_factory->generate(mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
+    auto b = gen_mtx(m, n, n + 4);
+    auto x = gen_mtx(m, n, n + 3);
     auto d_b = Mtx::create(exec);
     auto d_x = Mtx::create(exec);
     d_b->copy_from(b.get());
diff --git a/test/solver/cg_kernels.cpp b/test/solver/cg_kernels.cpp
index ac182fb2090..edd9d33a78d 100644
--- a/test/solver/cg_kernels.cpp
+++ b/test/solver/cg_kernels.cpp
@@ -80,29 +80,31 @@ class Cg : public ::testing::Test {
     }
 
     std::unique_ptr<Mtx> gen_mtx(gko::size_type num_rows,
-                                 gko::size_type num_cols)
+                                 gko::size_type num_cols, gko::size_type stride)
     {
-        return gko::test::generate_random_matrix<Mtx>(
+        auto tmp_mtx = gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(num_cols, num_cols),
             std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
+        auto result = Mtx::create(ref, gko::dim<2>{num_rows, num_cols}, stride);
+        result->copy_from(tmp_mtx.get());
+        return result;
     }
 
     void initialize_data()
     {
         gko::size_type m = 597;
         gko::size_type n = 43;
-        b_full = gen_mtx(m, n + 2);
-        b = b_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
-        r = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        q = gen_mtx(m, n);
-        x_full = gen_mtx(m, n + 3);
-        x = x_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
-        beta = gen_mtx(1, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
+        // all vectors need the same stride as b, except x
+        b = gen_mtx(m, n, n + 2);
+        r = gen_mtx(m, n, n + 2);
+        z = gen_mtx(m, n, n + 2);
+        p = gen_mtx(m, n, n + 2);
+        q = gen_mtx(m, n, n + 2);
+        x = gen_mtx(m, n, n + 3);
+        beta = gen_mtx(1, n, n);
+        prev_rho = gen_mtx(1, n, n);
+        rho = gen_mtx(1, n, n);
         // check correct handling for zero values
         beta->at(2) = 0.0;
         prev_rho->at(2) = 0.0;
@@ -142,13 +144,11 @@ class Cg : public ::testing::Test {
 
     std::ranlux48 rand_engine;
 
-    std::unique_ptr<Mtx> b_full;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
     std::unique_ptr<Mtx> z;
     std::unique_ptr<Mtx> p;
     std::unique_ptr<Mtx> q;
-    std::unique_ptr<Mtx> x_full;
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> beta;
     std::unique_ptr<Mtx> prev_rho;
@@ -223,10 +223,10 @@ TEST_F(Cg, CgStep2IsEquivalentToRef)
 
 TEST_F(Cg, ApplyIsEquivalentToRef)
 {
-    auto mtx = gen_mtx(50, 50);
+    auto mtx = gen_mtx(50, 50, 53);
     gko::test::make_hpd(mtx.get());
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
+    auto x = gen_mtx(50, 3, 5);
+    auto b = gen_mtx(50, 3, 4);
     auto d_mtx = Mtx::create(exec);
     d_mtx->copy_from(mtx.get());
     auto d_x = Mtx::create(exec);
diff --git a/test/solver/cgs_kernels.cpp b/test/solver/cgs_kernels.cpp
index b25ced0351d..cd4f0c10437 100644
--- a/test/solver/cgs_kernels.cpp
+++ b/test/solver/cgs_kernels.cpp
@@ -72,7 +72,7 @@ class Cgs : public ::testing::Test {
         ref = gko::ReferenceExecutor::create();
         init_executor(ref, exec);
 
-        mtx = gen_mtx(123, 123);
+        mtx = gen_mtx(123, 123, 125);
         gko::test::make_diag_dominant(mtx.get());
         d_mtx = Mtx::create(exec);
         d_mtx->copy_from(mtx.get());
@@ -102,35 +102,36 @@ class Cgs : public ::testing::Test {
     }
 
     std::unique_ptr<Mtx> gen_mtx(gko::size_type num_rows,
-                                 gko::size_type num_cols)
+                                 gko::size_type num_cols, gko::size_type stride)
     {
-        return gko::test::generate_random_matrix<Mtx>(
+        auto tmp_mtx = gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<value_type>(0.0, 1.0), rand_engine, ref);
+            std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
+        auto result = Mtx::create(ref, gko::dim<2>{num_rows, num_cols}, stride);
+        result->copy_from(tmp_mtx.get());
+        return result;
     }
 
     void initialize_data()
     {
         gko::size_type m = 597;
         gko::size_type n = 43;
-        b_full = gen_mtx(m, n + 2);
-        b = b_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
-        r = gen_mtx(m, n);
-        r_tld = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        q = gen_mtx(m, n);
-        u = gen_mtx(m, n);
-        u_hat = gen_mtx(m, n);
-        v_hat = gen_mtx(m, n);
-        t = gen_mtx(m, n);
-        x_full = gen_mtx(m, n + 3);
-        x = x_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
-        alpha = gen_mtx(1, n);
-        beta = gen_mtx(1, n);
-        gamma = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        rho_prev = gen_mtx(1, n);
+        b = gen_mtx(m, n, n + 2);
+        r = gen_mtx(m, n, n + 2);
+        r_tld = gen_mtx(m, n, n + 2);
+        p = gen_mtx(m, n, n + 2);
+        q = gen_mtx(m, n, n + 2);
+        u = gen_mtx(m, n, n + 2);
+        u_hat = gen_mtx(m, n, n + 2);
+        v_hat = gen_mtx(m, n, n + 2);
+        t = gen_mtx(m, n, n + 2);
+        x = gen_mtx(m, n, n + 3);
+        alpha = gen_mtx(1, n, n);
+        beta = gen_mtx(1, n, n);
+        gamma = gen_mtx(1, n, n);
+        rho = gen_mtx(1, n, n);
+        rho_prev = gen_mtx(1, n, n);
         // check correct handling for zero values
         gamma->at(2) = 0.0;
         rho_prev->at(2) = 0.0;
@@ -189,7 +190,6 @@ class Cgs : public ::testing::Test {
     std::unique_ptr<Solver::Factory> exec_cgs_factory;
     std::unique_ptr<Solver::Factory> ref_cgs_factory;
 
-    std::unique_ptr<Mtx> b_full;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
     std::unique_ptr<Mtx> r_tld;
@@ -199,7 +199,6 @@ class Cgs : public ::testing::Test {
     std::unique_ptr<Mtx> u;
     std::unique_ptr<Mtx> u_hat;
     std::unique_ptr<Mtx> v_hat;
-    std::unique_ptr<Mtx> x_full;
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> alpha;
     std::unique_ptr<Mtx> beta;
@@ -314,8 +313,8 @@ TEST_F(Cgs, CgsApplyOneRHSIsEquivalentToRef)
     int n = 1;
     auto ref_solver = ref_cgs_factory->generate(mtx);
     auto exec_solver = exec_cgs_factory->generate(d_mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
+    auto b = gen_mtx(m, n, n + 4);
+    auto x = gen_mtx(m, n, n + 2);
     auto d_b = Mtx::create(exec);
     auto d_x = Mtx::create(exec);
     d_b->copy_from(b.get());
@@ -335,8 +334,8 @@ TEST_F(Cgs, CgsApplyMultipleRHSIsEquivalentToRef)
     int n = 16;
     auto exec_solver = exec_cgs_factory->generate(d_mtx);
     auto ref_solver = ref_cgs_factory->generate(mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
+    auto b = gen_mtx(m, n, n + 1);
+    auto x = gen_mtx(m, n, n);
     auto d_b = Mtx::create(exec);
     auto d_x = Mtx::create(exec);
     d_b->copy_from(b.get());
diff --git a/test/solver/fcg_kernels.cpp b/test/solver/fcg_kernels.cpp
index e65fab5bd6e..46fbb30cdf6 100644
--- a/test/solver/fcg_kernels.cpp
+++ b/test/solver/fcg_kernels.cpp
@@ -81,31 +81,32 @@ class Fcg : public ::testing::Test {
     }
 
     std::unique_ptr<Mtx> gen_mtx(gko::size_type num_rows,
-                                 gko::size_type num_cols)
+                                 gko::size_type num_cols, gko::size_type stride)
     {
-        return gko::test::generate_random_matrix<Mtx>(
+        auto tmp_mtx = gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<value_type>(0.0, 1.0), rand_engine, ref);
+            std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
+        auto result = Mtx::create(ref, gko::dim<2>{num_rows, num_cols}, stride);
+        result->copy_from(tmp_mtx.get());
+        return result;
     }
 
     void initialize_data()
     {
         gko::size_type m = 597;
         gko::size_type n = 43;
-        b_full = gen_mtx(m, n + 2);
-        b = b_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
-        r = gen_mtx(m, n);
-        t = gen_mtx(m, n);
-        z = gen_mtx(m, n);
-        p = gen_mtx(m, n);
-        q = gen_mtx(m, n);
-        x_full = gen_mtx(m, n + 3);
-        x = x_full->create_submatrix(gko::span{0, m}, gko::span{0, n});
-        beta = gen_mtx(1, n);
-        prev_rho = gen_mtx(1, n);
-        rho = gen_mtx(1, n);
-        rho_t = gen_mtx(1, n);
+        b = gen_mtx(m, n, n + 2);
+        r = gen_mtx(m, n, n + 2);
+        t = gen_mtx(m, n, n + 2);
+        z = gen_mtx(m, n, n + 2);
+        p = gen_mtx(m, n, n + 2);
+        q = gen_mtx(m, n, n + 2);
+        x = gen_mtx(m, n, n + 3);
+        beta = gen_mtx(1, n, n);
+        prev_rho = gen_mtx(1, n, n);
+        rho = gen_mtx(1, n, n);
+        rho_t = gen_mtx(1, n, n);
         // check correct handling for zero values
         beta->at(2) = 0.0;
         prev_rho->at(2) = 0.0;
@@ -149,14 +150,12 @@ class Fcg : public ::testing::Test {
 
     std::ranlux48 rand_engine;
 
-    std::unique_ptr<Mtx> b_full;
     std::unique_ptr<Mtx> b;
     std::unique_ptr<Mtx> r;
     std::unique_ptr<Mtx> t;
     std::unique_ptr<Mtx> z;
     std::unique_ptr<Mtx> p;
     std::unique_ptr<Mtx> q;
-    std::unique_ptr<Mtx> x_full;
     std::unique_ptr<Mtx> x;
     std::unique_ptr<Mtx> beta;
     std::unique_ptr<Mtx> prev_rho;
@@ -235,10 +234,10 @@ TEST_F(Fcg, FcgStep2IsEquivalentToRef)
 
 TEST_F(Fcg, ApplyIsEquivalentToRef)
 {
-    auto mtx = gen_mtx(50, 50);
+    auto mtx = gen_mtx(50, 50, 53);
     gko::test::make_hpd(mtx.get());
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
+    auto x = gen_mtx(50, 3, 4);
+    auto b = gen_mtx(50, 3, 5);
     auto d_mtx = Mtx::create(exec);
     d_mtx->copy_from(mtx.get());
     auto d_x = Mtx::create(exec);
diff --git a/test/solver/ir_kernels.cpp b/test/solver/ir_kernels.cpp
index 6ce9d98f7a2..769ce6881be 100644
--- a/test/solver/ir_kernels.cpp
+++ b/test/solver/ir_kernels.cpp
@@ -72,12 +72,16 @@ class Ir : public ::testing::Test {
         init_executor(ref, exec);
     }
 
-    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    std::unique_ptr<Mtx> gen_mtx(gko::size_type num_rows,
+                                 gko::size_type num_cols, gko::size_type stride)
     {
-        return gko::test::generate_random_matrix<Mtx>(
+        auto tmp_mtx = gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(num_cols, num_cols),
             std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
+        auto result = Mtx::create(ref, gko::dim<2>{num_rows, num_cols}, stride);
+        result->copy_from(tmp_mtx.get());
+        return result;
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -107,9 +111,9 @@ TEST_F(Ir, InitializeIsEquivalentToRef)
 
 TEST_F(Ir, ApplyIsEquivalentToRef)
 {
-    auto mtx = gen_mtx(50, 50);
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
+    auto mtx = gen_mtx(50, 50, 52);
+    auto x = gen_mtx(50, 3, 8);
+    auto b = gen_mtx(50, 3, 5);
     auto d_mtx = clone(exec, mtx);
     auto d_x = clone(exec, x);
     auto d_b = clone(exec, b);
@@ -138,9 +142,9 @@ TEST_F(Ir, ApplyIsEquivalentToRef)
 
 TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef)
 {
-    auto mtx = gen_mtx(50, 50);
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
+    auto mtx = gen_mtx(50, 50, 54);
+    auto x = gen_mtx(50, 3, 6);
+    auto b = gen_mtx(50, 3, 10);
     auto d_mtx = clone(exec, mtx);
     auto d_x = clone(exec, x);
     auto d_b = clone(exec, b);
@@ -182,9 +186,9 @@ TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef)
 
 TEST_F(Ir, RichardsonApplyIsEquivalentToRef)
 {
-    auto mtx = gen_mtx(50, 50);
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
+    auto mtx = gen_mtx(50, 50, 54);
+    auto x = gen_mtx(50, 3, 4);
+    auto b = gen_mtx(50, 3, 3);
     auto d_mtx = clone(exec, mtx);
     auto d_x = clone(exec, x);
     auto d_b = clone(exec, b);
@@ -215,9 +219,9 @@ TEST_F(Ir, RichardsonApplyIsEquivalentToRef)
 
 TEST_F(Ir, RichardsonApplyWithIterativeInnerSolverIsEquivalentToRef)
 {
-    auto mtx = gen_mtx(50, 50);
-    auto x = gen_mtx(50, 3);
-    auto b = gen_mtx(50, 3);
+    auto mtx = gen_mtx(50, 50, 52);
+    auto x = gen_mtx(50, 3, 4);
+    auto b = gen_mtx(50, 3, 7);
     auto d_mtx = clone(exec, mtx);
     auto d_x = clone(exec, x);
     auto d_b = clone(exec, b);

From aa86ad99b1b5247ff7a9368ad43904d7d02d548a Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 17 Jun 2021 12:10:55 +0200
Subject: [PATCH 0847/1008] update documentation and assertions

---
 cuda/base/kernel_launch_solver.cuh     | 8 ++------
 dpcpp/base/kernel_launch_solver.dp.hpp | 8 ++------
 hip/base/kernel_launch_solver.hip.hpp  | 8 ++------
 include/ginkgo/core/matrix/dense.hpp   | 1 -
 omp/base/kernel_launch_solver.hpp      | 6 ++----
 5 files changed, 8 insertions(+), 23 deletions(-)

diff --git a/cuda/base/kernel_launch_solver.cuh b/cuda/base/kernel_launch_solver.cuh
index cc60cda5b00..91da455ce87 100644
--- a/cuda/base/kernel_launch_solver.cuh
+++ b/cuda/base/kernel_launch_solver.cuh
@@ -1,5 +1,3 @@
-
-
 /*******************************<GINKGO LICENSE>******************************
 Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
@@ -117,8 +115,7 @@ default_stride(const matrix::Dense<ValueType> *mtx)
 template <typename ValueType>
 kernels::cuda::cuda_type<ValueType> *rowvector(matrix::Dense<ValueType> *mtx)
 {
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    GKO_ASSERT(mtx->get_size()[0] == 1);
     return kernels::cuda::as_cuda_type(mtx->get_values());
 }
 
@@ -127,8 +124,7 @@ template <typename ValueType>
 const kernels::cuda::cuda_type<ValueType> *rowvector(
     const matrix::Dense<ValueType> *mtx)
 {
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    GKO_ASSERT(mtx->get_size()[0] == 1);
     return kernels::cuda::as_cuda_type(mtx->get_const_values());
 }
 
diff --git a/dpcpp/base/kernel_launch_solver.dp.hpp b/dpcpp/base/kernel_launch_solver.dp.hpp
index 919d06bf150..d1221d6f04a 100644
--- a/dpcpp/base/kernel_launch_solver.dp.hpp
+++ b/dpcpp/base/kernel_launch_solver.dp.hpp
@@ -1,5 +1,3 @@
-
-
 /*******************************<GINKGO LICENSE>******************************
 Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
@@ -112,8 +110,7 @@ kernels::dpcpp::default_stride_dense_wrapper<const ValueType> default_stride(
 template <typename ValueType>
 ValueType *rowvector(matrix::Dense<ValueType> *mtx)
 {
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    GKO_ASSERT(mtx->get_size()[0] == 1);
     return mtx->get_values();
 }
 
@@ -121,8 +118,7 @@ ValueType *rowvector(matrix::Dense<ValueType> *mtx)
 template <typename ValueType>
 const ValueType *rowvector(const matrix::Dense<ValueType> *mtx)
 {
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    GKO_ASSERT(mtx->get_size()[0] == 1);
     return mtx->get_const_values();
 }
 
diff --git a/hip/base/kernel_launch_solver.hip.hpp b/hip/base/kernel_launch_solver.hip.hpp
index 823cf72afba..a97fb8d76ed 100644
--- a/hip/base/kernel_launch_solver.hip.hpp
+++ b/hip/base/kernel_launch_solver.hip.hpp
@@ -1,5 +1,3 @@
-
-
 /*******************************<GINKGO LICENSE>******************************
 Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
@@ -120,8 +118,7 @@ default_stride(const matrix::Dense<ValueType> *mtx)
 template <typename ValueType>
 kernels::hip::hip_type<ValueType> *rowvector(matrix::Dense<ValueType> *mtx)
 {
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    GKO_ASSERT(mtx->get_size()[0] == 1);
     return kernels::hip::as_hip_type(mtx->get_values());
 }
 
@@ -130,8 +127,7 @@ template <typename ValueType>
 const kernels::hip::hip_type<ValueType> *rowvector(
     const matrix::Dense<ValueType> *mtx)
 {
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    GKO_ASSERT(mtx->get_size()[0] == 1);
     return kernels::hip::as_hip_type(mtx->get_const_values());
 }
 
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 45ef7ef182c..652c6357af2 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -828,7 +828,6 @@ class Dense
 
     /**
      * Creates a Dense matrix with the same size as the callers matrix.
-     * Note that the stride is not preserved!
      *
      * @returns a Dense matrix with the same size as the caller.
      */
diff --git a/omp/base/kernel_launch_solver.hpp b/omp/base/kernel_launch_solver.hpp
index 63ede5fd7a3..38e88cb79d3 100644
--- a/omp/base/kernel_launch_solver.hpp
+++ b/omp/base/kernel_launch_solver.hpp
@@ -104,8 +104,7 @@ kernels::omp::default_stride_dense_wrapper<const ValueType> default_stride(
 template <typename ValueType>
 ValueType *rowvector(matrix::Dense<ValueType> *mtx)
 {
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    GKO_ASSERT(mtx->get_size()[0] == 1);
     return mtx->get_values();
 }
 
@@ -113,8 +112,7 @@ ValueType *rowvector(matrix::Dense<ValueType> *mtx)
 template <typename ValueType>
 const ValueType *rowvector(const matrix::Dense<ValueType> *mtx)
 {
-    GKO_ASSERT(mtx->get_size()[0] == 1 ||
-               (mtx->get_size()[1] == 1 && mtx->get_stride() == 1));
+    GKO_ASSERT(mtx->get_size()[0] == 1);
     return mtx->get_const_values();
 }
 

From 4fc79cc9fe021b012bd05bc95529ec32afe61a47 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 17 Jun 2021 15:54:50 +0200
Subject: [PATCH 0848/1008] fix remaining create_with_same_config test

---
 core/test/matrix/dense.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/test/matrix/dense.cpp b/core/test/matrix/dense.cpp
index 3a809bae7b1..77dd543ac68 100644
--- a/core/test/matrix/dense.cpp
+++ b/core/test/matrix/dense.cpp
@@ -138,15 +138,15 @@ TYPED_TEST(Dense, CanBeConstructedFromExistingData)
 }
 
 
-TYPED_TEST(Dense, CreateWithSameConfigDoesntKeepStride)
+TYPED_TEST(Dense, CreateWithSameConfigKeepsStride)
 {
     auto m =
         gko::matrix::Dense<TypeParam>::create(this->exec, gko::dim<2>{2, 3}, 4);
     auto m2 = gko::matrix::Dense<TypeParam>::create_with_config_of(m.get());
 
     ASSERT_EQ(m2->get_size(), gko::dim<2>(2, 3));
-    EXPECT_EQ(m2->get_stride(), 3);
-    ASSERT_EQ(m2->get_num_stored_elements(), 6);
+    EXPECT_EQ(m2->get_stride(), 4);
+    ASSERT_EQ(m2->get_num_stored_elements(), 8);
 }
 
 

From d0a6371c3746864e2d830079cf90346245944d98 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 23 Jun 2021 08:38:55 +0200
Subject: [PATCH 0849/1008] review updates

---
 common/solver/bicg_kernels.cpp         | 10 +++++-----
 common/solver/bicgstab_kernels.cpp     | 20 ++++++++++----------
 common/solver/cg_kernels.cpp           | 10 +++++-----
 common/solver/cgs_kernels.cpp          | 14 +++++++-------
 common/solver/fcg_kernels.cpp          | 10 +++++-----
 cuda/base/kernel_launch.cuh            |  2 +-
 cuda/base/kernel_launch_solver.cuh     |  4 ++--
 cuda/test/base/kernel_launch.cu        |  2 +-
 dpcpp/base/kernel_launch_solver.dp.hpp |  4 ++--
 dpcpp/test/base/kernel_launch.dp.cpp   |  2 +-
 hip/base/kernel_launch.hip.hpp         |  2 +-
 hip/base/kernel_launch_solver.hip.hpp  |  4 ++--
 hip/test/base/kernel_launch.hip.cpp    |  2 +-
 omp/base/kernel_launch_solver.hpp      |  4 ++--
 omp/test/base/kernel_launch.cpp        |  2 +-
 15 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/common/solver/bicg_kernels.cpp b/common/solver/bicg_kernels.cpp
index 99748e9839f..8353956f5d5 100644
--- a/common/solver/bicg_kernels.cpp
+++ b/common/solver/bicg_kernels.cpp
@@ -40,7 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 using gko::solver::default_stride;
-using gko::solver::rowvector;
+using gko::solver::row_vector;
 
 
 namespace gko {
@@ -81,7 +81,7 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
         },
         b->get_size(), b->get_stride(), default_stride(b), default_stride(r),
         default_stride(z), default_stride(p), default_stride(q),
-        rowvector(prev_rho), rowvector(rho), default_stride(r2),
+        row_vector(prev_rho), row_vector(rho), default_stride(r2),
         default_stride(z2), default_stride(p2), default_stride(q2),
         *stop_status);
 }
@@ -108,8 +108,8 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
             }
         },
         p->get_size(), p->get_stride(), default_stride(p), default_stride(z),
-        default_stride(p2), default_stride(z2), rowvector(rho),
-        rowvector(prev_rho), *stop_status);
+        default_stride(p2), default_stride(z2), row_vector(rho),
+        row_vector(prev_rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_1_KERNEL);
@@ -138,7 +138,7 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
         },
         x->get_size(), r->get_stride(), x, default_stride(r),
         default_stride(r2), default_stride(p), default_stride(q),
-        default_stride(q2), rowvector(beta), rowvector(rho), *stop_status);
+        default_stride(q2), row_vector(beta), row_vector(rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICG_STEP_2_KERNEL);
diff --git a/common/solver/bicgstab_kernels.cpp b/common/solver/bicgstab_kernels.cpp
index 5fbfc7b7262..4bb9b254bff 100644
--- a/common/solver/bicgstab_kernels.cpp
+++ b/common/solver/bicgstab_kernels.cpp
@@ -40,7 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 using gko::solver::default_stride;
-using gko::solver::rowvector;
+using gko::solver::row_vector;
 
 
 namespace gko {
@@ -84,9 +84,9 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
         b->get_size(), b->get_stride(), default_stride(b), default_stride(r),
         default_stride(rr), default_stride(y), default_stride(s),
         default_stride(t), default_stride(z), default_stride(v),
-        default_stride(p), rowvector(prev_rho), rowvector(rho),
-        rowvector(alpha), rowvector(beta), rowvector(gamma), rowvector(omega),
-        *stop_status);
+        default_stride(p), row_vector(prev_rho), row_vector(rho),
+        row_vector(alpha), row_vector(beta), row_vector(gamma),
+        row_vector(omega), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_INITIALIZE_KERNEL);
@@ -114,8 +114,8 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
             }
         },
         r->get_size(), r->get_stride(), default_stride(r), default_stride(p),
-        default_stride(v), rowvector(rho), rowvector(prev_rho),
-        rowvector(alpha), rowvector(omega), *stop_status);
+        default_stride(v), row_vector(rho), row_vector(prev_rho),
+        row_vector(alpha), row_vector(omega), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_1_KERNEL);
@@ -143,7 +143,7 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
             }
         },
         r->get_size(), r->get_stride(), default_stride(r), default_stride(s),
-        default_stride(v), rowvector(rho), rowvector(alpha), rowvector(beta),
+        default_stride(v), row_vector(rho), row_vector(alpha), row_vector(beta),
         *stop_status);
 }
 
@@ -175,8 +175,8 @@ void step_3(
         },
         x->get_size(), r->get_stride(), x, default_stride(r), default_stride(s),
         default_stride(t), default_stride(y), default_stride(z),
-        rowvector(alpha), rowvector(beta), rowvector(gamma), rowvector(omega),
-        *stop_status);
+        row_vector(alpha), row_vector(beta), row_vector(gamma),
+        row_vector(omega), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_BICGSTAB_STEP_3_KERNEL);
@@ -197,7 +197,7 @@ void finalize(std::shared_ptr<const DefaultExecutor> exec,
                 stop[col].finalize();
             }
         },
-        x->get_size(), y->get_stride(), x, default_stride(y), rowvector(alpha),
+        x->get_size(), y->get_stride(), x, default_stride(y), row_vector(alpha),
         *stop_status);
 }
 
diff --git a/common/solver/cg_kernels.cpp b/common/solver/cg_kernels.cpp
index e8cbc8f691f..6f270c5ac95 100644
--- a/common/solver/cg_kernels.cpp
+++ b/common/solver/cg_kernels.cpp
@@ -40,7 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 using gko::solver::default_stride;
-using gko::solver::rowvector;
+using gko::solver::row_vector;
 
 
 namespace gko {
@@ -75,8 +75,8 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
             z(row, col) = p(row, col) = q(row, col) = zero(z(row, col));
         },
         b->get_size(), b->get_stride(), b, default_stride(r), default_stride(z),
-        default_stride(p), default_stride(q), rowvector(prev_rho),
-        rowvector(rho), *stop_status);
+        default_stride(p), default_stride(q), row_vector(prev_rho),
+        row_vector(rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_INITIALIZE_KERNEL);
@@ -99,7 +99,7 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
             }
         },
         p->get_size(), p->get_stride(), default_stride(p), default_stride(z),
-        rowvector(rho), rowvector(prev_rho), *stop_status);
+        row_vector(rho), row_vector(prev_rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_1_KERNEL);
@@ -125,7 +125,7 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
             }
         },
         x->get_size(), r->get_stride(), x, default_stride(r), default_stride(p),
-        default_stride(q), rowvector(beta), rowvector(rho), *stop_status);
+        default_stride(q), row_vector(beta), row_vector(rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CG_STEP_2_KERNEL);
diff --git a/common/solver/cgs_kernels.cpp b/common/solver/cgs_kernels.cpp
index 51d93bd5ca1..74d654ef70e 100644
--- a/common/solver/cgs_kernels.cpp
+++ b/common/solver/cgs_kernels.cpp
@@ -40,7 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 using gko::solver::default_stride;
-using gko::solver::rowvector;
+using gko::solver::row_vector;
 
 
 namespace gko {
@@ -86,8 +86,8 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
         b->get_size(), b->get_stride(), default_stride(b), default_stride(r),
         default_stride(r_tld), default_stride(p), default_stride(q),
         default_stride(u), default_stride(u_hat), default_stride(v_hat),
-        default_stride(t), rowvector(alpha), rowvector(beta), rowvector(gamma),
-        rowvector(prev_rho), rowvector(rho), *stop_status);
+        default_stride(t), row_vector(alpha), row_vector(beta),
+        row_vector(gamma), row_vector(prev_rho), row_vector(rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_INITIALIZE_KERNEL);
@@ -117,8 +117,8 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
             }
         },
         r->get_size(), r->get_stride(), default_stride(r), default_stride(u),
-        default_stride(p), default_stride(q), rowvector(beta), rowvector(rho),
-        rowvector(prev_rho), *stop_status);
+        default_stride(p), default_stride(q), row_vector(beta), row_vector(rho),
+        row_vector(prev_rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_1_KERNEL);
@@ -149,7 +149,7 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
         },
         u->get_size(), u->get_stride(), default_stride(u),
         default_stride(v_hat), default_stride(q), default_stride(t),
-        rowvector(alpha), rowvector(rho), rowvector(gamma), *stop_status);
+        row_vector(alpha), row_vector(rho), row_vector(gamma), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CGS_STEP_2_KERNEL);
@@ -171,7 +171,7 @@ void step_3(std::shared_ptr<const DefaultExecutor> exec,
             }
         },
         t->get_size(), t->get_stride(), default_stride(t),
-        default_stride(u_hat), default_stride(r), x, rowvector(alpha),
+        default_stride(u_hat), default_stride(r), x, row_vector(alpha),
         *stop_status);
 }
 
diff --git a/common/solver/fcg_kernels.cpp b/common/solver/fcg_kernels.cpp
index b33a7dc90a4..78e496e1060 100644
--- a/common/solver/fcg_kernels.cpp
+++ b/common/solver/fcg_kernels.cpp
@@ -40,7 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 using gko::solver::default_stride;
-using gko::solver::rowvector;
+using gko::solver::row_vector;
 
 
 namespace gko {
@@ -78,8 +78,8 @@ void initialize(std::shared_ptr<const DefaultExecutor> exec,
         },
         b->get_size(), b->get_stride(), default_stride(b), default_stride(r),
         default_stride(z), default_stride(p), default_stride(q),
-        default_stride(t), rowvector(prev_rho), rowvector(rho),
-        rowvector(rho_t), *stop_status);
+        default_stride(t), row_vector(prev_rho), row_vector(rho),
+        row_vector(rho_t), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_INITIALIZE_KERNEL);
@@ -102,7 +102,7 @@ void step_1(std::shared_ptr<const DefaultExecutor> exec,
             }
         },
         p->get_size(), p->get_stride(), default_stride(p), default_stride(z),
-        rowvector(rho_t), rowvector(prev_rho), *stop_status);
+        row_vector(rho_t), row_vector(prev_rho), *stop_status);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_FCG_STEP_1_KERNEL);
@@ -130,7 +130,7 @@ void step_2(std::shared_ptr<const DefaultExecutor> exec,
             }
         },
         x->get_size(), r->get_stride(), x, default_stride(r), default_stride(t),
-        default_stride(p), default_stride(q), rowvector(beta), rowvector(rho),
+        default_stride(p), default_stride(q), row_vector(beta), row_vector(rho),
         *stop_status);
 }
 
diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index 500ae9bf68f..5ad9842d2f2 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -55,7 +55,7 @@ namespace kernels {
 namespace cuda {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 template <typename ValueType>
diff --git a/cuda/base/kernel_launch_solver.cuh b/cuda/base/kernel_launch_solver.cuh
index 91da455ce87..738d2d0cda8 100644
--- a/cuda/base/kernel_launch_solver.cuh
+++ b/cuda/base/kernel_launch_solver.cuh
@@ -113,7 +113,7 @@ default_stride(const matrix::Dense<ValueType> *mtx)
 
 
 template <typename ValueType>
-kernels::cuda::cuda_type<ValueType> *rowvector(matrix::Dense<ValueType> *mtx)
+kernels::cuda::cuda_type<ValueType> *row_vector(matrix::Dense<ValueType> *mtx)
 {
     GKO_ASSERT(mtx->get_size()[0] == 1);
     return kernels::cuda::as_cuda_type(mtx->get_values());
@@ -121,7 +121,7 @@ kernels::cuda::cuda_type<ValueType> *rowvector(matrix::Dense<ValueType> *mtx)
 
 
 template <typename ValueType>
-const kernels::cuda::cuda_type<ValueType> *rowvector(
+const kernels::cuda::cuda_type<ValueType> *row_vector(
     const matrix::Dense<ValueType> *mtx)
 {
     GKO_ASSERT(mtx->get_size()[0] == 1);
diff --git a/cuda/test/base/kernel_launch.cu b/cuda/test/base/kernel_launch.cu
index 3b01ea37804..fa0bbb8aca1 100644
--- a/cuda/test/base/kernel_launch.cu
+++ b/cuda/test/base/kernel_launch.cu
@@ -257,7 +257,7 @@ void run2d(std::shared_ptr<gko::CudaExecutor> exec, gko::matrix::Dense<> *m1,
         },
         dim<2>{4, 4}, m2->get_stride(), m1,
         static_cast<const gko::matrix::Dense<> *>(m1), m1->get_const_values(),
-        gko::solver::default_stride(m2), gko::solver::rowvector(m3),
+        gko::solver::default_stride(m2), gko::solver::row_vector(m3),
         m2->get_values(), m3->get_values());
 }
 
diff --git a/dpcpp/base/kernel_launch_solver.dp.hpp b/dpcpp/base/kernel_launch_solver.dp.hpp
index d1221d6f04a..9824d0ac567 100644
--- a/dpcpp/base/kernel_launch_solver.dp.hpp
+++ b/dpcpp/base/kernel_launch_solver.dp.hpp
@@ -108,7 +108,7 @@ kernels::dpcpp::default_stride_dense_wrapper<const ValueType> default_stride(
 
 
 template <typename ValueType>
-ValueType *rowvector(matrix::Dense<ValueType> *mtx)
+ValueType *row_vector(matrix::Dense<ValueType> *mtx)
 {
     GKO_ASSERT(mtx->get_size()[0] == 1);
     return mtx->get_values();
@@ -116,7 +116,7 @@ ValueType *rowvector(matrix::Dense<ValueType> *mtx)
 
 
 template <typename ValueType>
-const ValueType *rowvector(const matrix::Dense<ValueType> *mtx)
+const ValueType *row_vector(const matrix::Dense<ValueType> *mtx)
 {
     GKO_ASSERT(mtx->get_size()[0] == 1);
     return mtx->get_const_values();
diff --git a/dpcpp/test/base/kernel_launch.dp.cpp b/dpcpp/test/base/kernel_launch.dp.cpp
index 6572b34e5a1..3a589dfc8ce 100644
--- a/dpcpp/test/base/kernel_launch.dp.cpp
+++ b/dpcpp/test/base/kernel_launch.dp.cpp
@@ -244,7 +244,7 @@ TEST_F(KernelLaunch, Runs2DDense)
         static_cast<const Mtx *>(zero_dense2.get()),
         zero_dense2->get_const_values(),
         gko::solver::default_stride(zero_dense.get()),
-        gko::solver::rowvector(vec_dense.get()), zero_dense->get_values(),
+        gko::solver::row_vector(vec_dense.get()), zero_dense->get_values(),
         vec_dense->get_values());
 
     GKO_ASSERT_MTX_NEAR(zero_dense2, iota_dense, 0.0);
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index 4687272de0f..acef2c8cfab 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -58,7 +58,7 @@ namespace kernels {
 namespace hip {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 template <typename ValueType>
diff --git a/hip/base/kernel_launch_solver.hip.hpp b/hip/base/kernel_launch_solver.hip.hpp
index a97fb8d76ed..cfacf34a412 100644
--- a/hip/base/kernel_launch_solver.hip.hpp
+++ b/hip/base/kernel_launch_solver.hip.hpp
@@ -116,7 +116,7 @@ default_stride(const matrix::Dense<ValueType> *mtx)
 
 
 template <typename ValueType>
-kernels::hip::hip_type<ValueType> *rowvector(matrix::Dense<ValueType> *mtx)
+kernels::hip::hip_type<ValueType> *row_vector(matrix::Dense<ValueType> *mtx)
 {
     GKO_ASSERT(mtx->get_size()[0] == 1);
     return kernels::hip::as_hip_type(mtx->get_values());
@@ -124,7 +124,7 @@ kernels::hip::hip_type<ValueType> *rowvector(matrix::Dense<ValueType> *mtx)
 
 
 template <typename ValueType>
-const kernels::hip::hip_type<ValueType> *rowvector(
+const kernels::hip::hip_type<ValueType> *row_vector(
     const matrix::Dense<ValueType> *mtx)
 {
     GKO_ASSERT(mtx->get_size()[0] == 1);
diff --git a/hip/test/base/kernel_launch.hip.cpp b/hip/test/base/kernel_launch.hip.cpp
index 65b901841f7..91d03d9c00e 100644
--- a/hip/test/base/kernel_launch.hip.cpp
+++ b/hip/test/base/kernel_launch.hip.cpp
@@ -256,7 +256,7 @@ void run2d(std::shared_ptr<gko::HipExecutor> exec, gko::matrix::Dense<> *m1,
         },
         dim<2>{4, 4}, m2->get_stride(), m1,
         static_cast<const gko::matrix::Dense<> *>(m1), m1->get_const_values(),
-        gko::solver::default_stride(m2), gko::solver::rowvector(m3),
+        gko::solver::default_stride(m2), gko::solver::row_vector(m3),
         m2->get_values(), m3->get_values());
 }
 
diff --git a/omp/base/kernel_launch_solver.hpp b/omp/base/kernel_launch_solver.hpp
index 38e88cb79d3..72df8aa835a 100644
--- a/omp/base/kernel_launch_solver.hpp
+++ b/omp/base/kernel_launch_solver.hpp
@@ -102,7 +102,7 @@ kernels::omp::default_stride_dense_wrapper<const ValueType> default_stride(
 
 
 template <typename ValueType>
-ValueType *rowvector(matrix::Dense<ValueType> *mtx)
+ValueType *row_vector(matrix::Dense<ValueType> *mtx)
 {
     GKO_ASSERT(mtx->get_size()[0] == 1);
     return mtx->get_values();
@@ -110,7 +110,7 @@ ValueType *rowvector(matrix::Dense<ValueType> *mtx)
 
 
 template <typename ValueType>
-const ValueType *rowvector(const matrix::Dense<ValueType> *mtx)
+const ValueType *row_vector(const matrix::Dense<ValueType> *mtx)
 {
     GKO_ASSERT(mtx->get_size()[0] == 1);
     return mtx->get_const_values();
diff --git a/omp/test/base/kernel_launch.cpp b/omp/test/base/kernel_launch.cpp
index d03a5a26c3a..30fd82e5002 100644
--- a/omp/test/base/kernel_launch.cpp
+++ b/omp/test/base/kernel_launch.cpp
@@ -226,7 +226,7 @@ TEST_F(KernelLaunch, Runs2DDense)
         static_cast<const gko::matrix::Dense<> *>(zero_dense2.get()),
         zero_dense2->get_const_values(),
         gko::solver::default_stride(zero_dense.get()),
-        gko::solver::rowvector(vec_dense.get()), zero_dense->get_values(),
+        gko::solver::row_vector(vec_dense.get()), zero_dense->get_values(),
         vec_dense->get_values());
 
     GKO_ASSERT_MTX_NEAR(zero_dense2, iota_dense, 0.0);

From 4f651528996f03eb26d419e97499cb9e5d1e04ea Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 23 Jun 2021 22:24:32 +0200
Subject: [PATCH 0850/1008] extract common code for simple kernel launch

---
 cmake/build_helpers.cmake                     |  11 +-
 common/base/kernel_launch.hpp                 | 243 ++++++++++++++++++
 .../base/kernel_launch_solver.hpp             |  89 +++++--
 common/components/precision_conversion.cpp    |   2 +-
 common/matrix/dense_kernels.cpp               |   2 +-
 common/solver/bicg_kernels.cpp                |   6 +-
 common/solver/bicgstab_kernels.cpp            |   6 +-
 common/solver/cg_kernels.cpp                  |   6 +-
 common/solver/cgs_kernels.cpp                 |   6 +-
 common/solver/fcg_kernels.cpp                 |   6 +-
 common/solver/ir_kernels.cpp                  |   2 +-
 core/CMakeLists.txt                           |   2 +-
 core/base/simple_kernels.hpp                  |  64 -----
 cuda/CMakeLists.txt                           |   2 +-
 cuda/base/kernel_launch.cuh                   |  96 +------
 cuda/base/kernel_launch_solver.cuh            |  87 +------
 cuda/test/base/kernel_launch.cu               |  17 +-
 dpcpp/CMakeLists.txt                          |   2 +-
 dpcpp/base/kernel_launch.dp.hpp               |  88 +------
 dpcpp/base/kernel_launch_solver.dp.hpp        |  73 +-----
 dpcpp/components/format_conversion.dp.hpp     |   1 +
 dpcpp/test/base/kernel_launch.dp.cpp          |  19 +-
 hip/CMakeLists.txt                            |   2 +-
 hip/base/kernel_launch.hip.hpp                |  99 +------
 hip/base/kernel_launch_solver.hip.hpp         |  82 +-----
 hip/test/base/kernel_launch.hip.cpp           |  17 +-
 include/CMakeLists.txt                        |   2 +-
 omp/CMakeLists.txt                            |   2 +-
 omp/base/kernel_launch.hpp                    |  84 +-----
 omp/base/kernel_launch_solver.hpp             |  78 +-----
 omp/test/base/kernel_launch.cpp               |  19 +-
 reference/CMakeLists.txt                      |   2 +-
 32 files changed, 430 insertions(+), 787 deletions(-)
 create mode 100644 common/base/kernel_launch.hpp
 rename core/base/simple_kernels_solver.hpp => common/base/kernel_launch_solver.hpp (53%)
 delete mode 100644 core/base/simple_kernels.hpp

diff --git a/cmake/build_helpers.cmake b/cmake/build_helpers.cmake
index ac1574939af..f0e4dcbd698 100644
--- a/cmake/build_helpers.cmake
+++ b/cmake/build_helpers.cmake
@@ -43,7 +43,7 @@ function(ginkgo_compile_features name)
     set_target_properties("${name}" PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endfunction()
 
-function(ginkgo_check_headers target)
+function(ginkgo_check_headers target defines)
     # build object library used to "compile" the headers
     # add a proxy source file for each header in the target source list
     file(GLOB_RECURSE CUDA_HEADERS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" CONFIGURE_DEPENDS "*.cuh")
@@ -56,7 +56,7 @@ function(ginkgo_check_headers target)
 
     set(SOURCES "")
     # if we have any CUDA files in there, compile everything as CUDA
-    if (CUDA_HEADERS)
+    if(CUDA_HEADERS)
         set(CUDA_HEADERS ${CUDA_HEADERS} ${CXX_HEADERS})
         set(CXX_HEADERS "")
         if (HIP_HEADERS) 
@@ -74,10 +74,13 @@ function(ginkgo_check_headers target)
         file(WRITE "${HEADER_SOURCEFILE}" "#include \"${HEADER}\"")
         list(APPEND SOURCES "${HEADER_SOURCEFILE}")
     endforeach()
-    if (SOURCES)
+    if(SOURCES)
         add_library(${target}_headers OBJECT ${SOURCES})
         target_link_libraries(${target}_headers PRIVATE ${target})
         target_include_directories(${target}_headers PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}")
+        if(defines)
+            target_compile_definitions(${target}_headers PRIVATE ${defines})
+        endif()
     endif()
 
     set(HIP_SOURCES "")
@@ -86,7 +89,7 @@ function(ginkgo_check_headers target)
         file(WRITE "${HEADER_SOURCEFILE}" "#include \"${HEADER}\"")
         list(APPEND HIP_SOURCES "${HEADER_SOURCEFILE}")
     endforeach()
-    if (HIP_SOURCES)
+    if(HIP_SOURCES)
         set_source_files_properties(${HIP_SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
         hip_add_library(${target}_headers_hip ${HIP_SOURCES}) # the compiler options get set by linking to ginkgo_hip
         target_link_libraries(${target}_headers_hip PRIVATE ${target} roc::hipblas roc::hipsparse hip::hiprand roc::rocrand)
diff --git a/common/base/kernel_launch.hpp b/common/base/kernel_launch.hpp
new file mode 100644
index 00000000000..289df89eae5
--- /dev/null
+++ b/common/base/kernel_launch.hpp
@@ -0,0 +1,243 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_BASE_KERNEL_LAUNCH_HPP_
+#define GKO_CORE_BASE_KERNEL_LAUNCH_HPP_
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#if defined(GKO_COMPILING_CUDA)
+
+#define GKO_DEVICE_NAMESPACE cuda
+#define GKO_KERNEL __device__
+#include "cuda/base/types.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+
+
+template <typename T>
+using device_type = typename detail::cuda_type_impl<T>::type;
+
+template <typename T>
+device_type<T> as_device_type(T value)
+{
+    return as_cuda_type(value);
+}
+
+
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
+
+
+#elif defined(GKO_COMPILING_HIP)
+
+#define GKO_DEVICE_NAMESPACE hip
+#define GKO_KERNEL __device__
+#include "hip/base/types.hip.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace hip {
+
+
+template <typename T>
+using device_type = typename detail::hip_type_impl<T>::type;
+
+template <typename T>
+device_type<T> as_device_type(T value)
+{
+    return as_hip_type(value);
+}
+
+
+}  // namespace hip
+}  // namespace kernels
+}  // namespace gko
+
+
+#elif defined(GKO_COMPILING_DPCPP)
+
+#define GKO_DEVICE_NAMESPACE dpcpp
+#define GKO_KERNEL
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+template <typename T>
+using device_type = T;
+
+template <typename T>
+device_type<T> as_device_type(T value)
+{
+    return value;
+}
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+#elif defined(GKO_COMPILING_OMP)
+
+#define GKO_DEVICE_NAMESPACE omp
+#define GKO_KERNEL
+
+
+namespace gko {
+namespace kernels {
+namespace omp {
+
+
+template <typename T>
+using device_type = T;
+
+template <typename T>
+device_type<T> as_device_type(T value)
+{
+    return value;
+}
+
+
+}  // namespace omp
+}  // namespace kernels
+}  // namespace gko
+
+
+#else
+
+#error "This file should only be used inside Ginkgo device compilation"
+
+#endif
+
+
+namespace gko {
+namespace kernels {
+namespace GKO_DEVICE_NAMESPACE {
+
+
+template <typename ValueType>
+struct matrix_accessor {
+    ValueType *data;
+    size_type stride;
+
+    GKO_INLINE GKO_ATTRIBUTES ValueType &operator()(size_type row,
+                                                    size_type col)
+    {
+        return data[row * stride + col];
+    }
+
+    GKO_INLINE GKO_ATTRIBUTES ValueType &operator[](size_type idx)
+    {
+        return data[idx];
+    }
+};
+
+
+template <typename T>
+struct to_device_type_impl {
+    using type = std::decay_t<device_type<T>>;
+    static type map_to_device(T in) { return as_device_type(in); }
+};
+
+template <typename ValueType>
+struct to_device_type_impl<matrix::Dense<ValueType> *&> {
+    using type = matrix_accessor<device_type<ValueType>>;
+    static type map_to_device(matrix::Dense<ValueType> *mtx)
+    {
+        return {as_device_type(mtx->get_values()), mtx->get_stride()};
+    }
+};
+
+template <typename ValueType>
+struct to_device_type_impl<const matrix::Dense<ValueType> *&> {
+    using type = matrix_accessor<const device_type<ValueType>>;
+    static type map_to_device(const matrix::Dense<ValueType> *mtx)
+    {
+        return {as_device_type(mtx->get_const_values()), mtx->get_stride()};
+    }
+};
+
+template <typename ValueType>
+struct to_device_type_impl<Array<ValueType> &> {
+    using type = device_type<ValueType> *;
+    static type map_to_device(Array<ValueType> &array)
+    {
+        return as_device_type(array.get_data());
+    }
+};
+
+template <typename ValueType>
+struct to_device_type_impl<const Array<ValueType> &> {
+    using type = const device_type<ValueType> *;
+    static type map_to_device(const Array<ValueType> &array)
+    {
+        return as_device_type(array.get_const_data());
+    }
+};
+
+
+template <typename T>
+typename to_device_type_impl<T>::type map_to_device(T &&param)
+{
+    return to_device_type_impl<T>::map_to_device(param);
+}
+
+
+}  // namespace GKO_DEVICE_NAMESPACE
+}  // namespace kernels
+}  // namespace gko
+
+
+#if defined(GKO_COMPILING_CUDA)
+#include "cuda/base/kernel_launch.cuh"
+#elif defined(GKO_COMPILING_HIP)
+#include "hip/base/kernel_launch.hip.hpp"
+#elif defined(GKO_COMPILING_DPCPP)
+#include "dpcpp/base/kernel_launch.dp.hpp"
+#elif defined(GKO_COMPILING_OMP)
+#include "omp/base/kernel_launch.hpp"
+#endif
+
+
+#endif  // GKO_CORE_BASE_KERNEL_LAUNCH_HPP_
diff --git a/core/base/simple_kernels_solver.hpp b/common/base/kernel_launch_solver.hpp
similarity index 53%
rename from core/base/simple_kernels_solver.hpp
rename to common/base/kernel_launch_solver.hpp
index b6b8c3adf36..fc10e508807 100644
--- a/core/base/simple_kernels_solver.hpp
+++ b/common/base/kernel_launch_solver.hpp
@@ -30,35 +30,90 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_SIMPLE_KERNELS_SOLVER_HPP_
-#define GKO_CORE_BASE_SIMPLE_KERNELS_SOLVER_HPP_
+#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
+#define GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
 
 
-#if defined(GKO_COMPILING_CUDA)
+#include "common/base/kernel_launch.hpp"
 
-#include "cuda/base/kernel_launch_solver.cuh"
-#define GKO_DEVICE_NAMESPACE cuda
 
-#elif defined(GKO_COMPILING_HIP)
+namespace gko {
+namespace kernels {
+namespace GKO_DEVICE_NAMESPACE {
 
-#include "hip/base/kernel_launch_solver.hip.hpp"
-#define GKO_DEVICE_NAMESPACE hip
 
-#elif defined(GKO_COMPILING_DPCPP)
+template <typename ValueType>
+struct default_stride_dense_wrapper {
+    ValueType *data;
+};
 
-#include "dpcpp/base/kernel_launch_solver.dp.hpp"
-#define GKO_DEVICE_NAMESPACE dpcpp
 
-#elif defined(GKO_COMPILING_OMP)
+template <typename T>
+struct device_unpack_solver_impl {
+    using type = T;
+    static GKO_INLINE GKO_ATTRIBUTES type unpack(T param, size_type)
+    {
+        return param;
+    }
+};
+
+template <typename ValueType>
+struct device_unpack_solver_impl<default_stride_dense_wrapper<ValueType>> {
+    using type = matrix_accessor<ValueType>;
+    static GKO_INLINE GKO_ATTRIBUTES type unpack(
+        default_stride_dense_wrapper<ValueType> param, size_type default_stride)
+    {
+        return {param.data, default_stride};
+    }
+};
+
+
+template <typename ValueType>
+default_stride_dense_wrapper<device_type<ValueType>> default_stride(
+    matrix::Dense<ValueType> *mtx)
+{
+    return {as_device_type(mtx->get_values())};
+}
+
+
+template <typename ValueType>
+default_stride_dense_wrapper<const device_type<ValueType>> default_stride(
+    const matrix::Dense<ValueType> *mtx)
+{
+    return {as_device_type(mtx->get_const_values())};
+}
 
-#include "omp/base/kernel_launch_solver.hpp"
-#define GKO_DEVICE_NAMESPACE omp
 
-#else
+template <typename ValueType>
+device_type<ValueType> *row_vector(matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1);
+    return as_device_type(mtx->get_values());
+}
 
-#error "This file should only be used inside Ginkgo device compilation"
 
+template <typename ValueType>
+const device_type<ValueType> *row_vector(const matrix::Dense<ValueType> *mtx)
+{
+    GKO_ASSERT(mtx->get_size()[0] == 1);
+    return as_device_type(mtx->get_const_values());
+}
+
+
+}  // namespace GKO_DEVICE_NAMESPACE
+}  // namespace kernels
+}  // namespace gko
+
+
+#if defined(GKO_COMPILING_CUDA)
+#include "cuda/base/kernel_launch_solver.cuh"
+#elif defined(GKO_COMPILING_HIP)
+#include "hip/base/kernel_launch_solver.hip.hpp"
+#elif defined(GKO_COMPILING_DPCPP)
+#include "dpcpp/base/kernel_launch_solver.dp.hpp"
+#elif defined(GKO_COMPILING_OMP)
+#include "omp/base/kernel_launch_solver.hpp"
 #endif
 
 
-#endif  // GKO_CORE_BASE_SIMPLE_KERNELS_SOLVER_HPP_
+#endif  // GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
diff --git a/common/components/precision_conversion.cpp b/common/components/precision_conversion.cpp
index 9982ea71c10..89ec288e6af 100644
--- a/common/components/precision_conversion.cpp
+++ b/common/components/precision_conversion.cpp
@@ -33,7 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/precision_conversion.hpp"
 
 
-#include "core/base/simple_kernels.hpp"
+#include "common/base/kernel_launch.hpp"
 
 
 namespace gko {
diff --git a/common/matrix/dense_kernels.cpp b/common/matrix/dense_kernels.cpp
index 3b53a538fa8..446090b8121 100644
--- a/common/matrix/dense_kernels.cpp
+++ b/common/matrix/dense_kernels.cpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "core/base/simple_kernels.hpp"
+#include "common/base/kernel_launch.hpp"
 
 
 namespace gko {
diff --git a/common/solver/bicg_kernels.cpp b/common/solver/bicg_kernels.cpp
index 8353956f5d5..11c3feda57b 100644
--- a/common/solver/bicg_kernels.cpp
+++ b/common/solver/bicg_kernels.cpp
@@ -36,11 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "core/base/simple_kernels_solver.hpp"
-
-
-using gko::solver::default_stride;
-using gko::solver::row_vector;
+#include "common/base/kernel_launch_solver.hpp"
 
 
 namespace gko {
diff --git a/common/solver/bicgstab_kernels.cpp b/common/solver/bicgstab_kernels.cpp
index 4bb9b254bff..c9263e018dc 100644
--- a/common/solver/bicgstab_kernels.cpp
+++ b/common/solver/bicgstab_kernels.cpp
@@ -36,11 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "core/base/simple_kernels_solver.hpp"
-
-
-using gko::solver::default_stride;
-using gko::solver::row_vector;
+#include "common/base/kernel_launch_solver.hpp"
 
 
 namespace gko {
diff --git a/common/solver/cg_kernels.cpp b/common/solver/cg_kernels.cpp
index 6f270c5ac95..f62a444ab1a 100644
--- a/common/solver/cg_kernels.cpp
+++ b/common/solver/cg_kernels.cpp
@@ -36,11 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "core/base/simple_kernels_solver.hpp"
-
-
-using gko::solver::default_stride;
-using gko::solver::row_vector;
+#include "common/base/kernel_launch_solver.hpp"
 
 
 namespace gko {
diff --git a/common/solver/cgs_kernels.cpp b/common/solver/cgs_kernels.cpp
index 74d654ef70e..b46731aeadf 100644
--- a/common/solver/cgs_kernels.cpp
+++ b/common/solver/cgs_kernels.cpp
@@ -36,11 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "core/base/simple_kernels_solver.hpp"
-
-
-using gko::solver::default_stride;
-using gko::solver::row_vector;
+#include "common/base/kernel_launch_solver.hpp"
 
 
 namespace gko {
diff --git a/common/solver/fcg_kernels.cpp b/common/solver/fcg_kernels.cpp
index 78e496e1060..66b26d946f0 100644
--- a/common/solver/fcg_kernels.cpp
+++ b/common/solver/fcg_kernels.cpp
@@ -36,11 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "core/base/simple_kernels_solver.hpp"
-
-
-using gko::solver::default_stride;
-using gko::solver::row_vector;
+#include "common/base/kernel_launch_solver.hpp"
 
 
 namespace gko {
diff --git a/common/solver/ir_kernels.cpp b/common/solver/ir_kernels.cpp
index 6fe9e5721d2..f30fb066588 100644
--- a/common/solver/ir_kernels.cpp
+++ b/common/solver/ir_kernels.cpp
@@ -33,7 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/ir_kernels.hpp"
 
 
-#include "core/base/simple_kernels.hpp"
+#include "common/base/kernel_launch.hpp"
 
 
 namespace gko {
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 700b713b129..f5388785d6a 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -83,7 +83,7 @@ ginkgo_default_includes(ginkgo)
 ginkgo_install_library(ginkgo "${GKO_RPATH_ADDITIONS}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
-    ginkgo_check_headers(ginkgo)
+    ginkgo_check_headers(ginkgo "")
 endif()
 
 if(GINKGO_BUILD_TESTS)
diff --git a/core/base/simple_kernels.hpp b/core/base/simple_kernels.hpp
deleted file mode 100644
index de90bbb6233..00000000000
--- a/core/base/simple_kernels.hpp
+++ /dev/null
@@ -1,64 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GKO_CORE_BASE_SIMPLE_KERNELS_HPP_
-#define GKO_CORE_BASE_SIMPLE_KERNELS_HPP_
-
-
-#if defined(GKO_COMPILING_CUDA)
-
-#include "cuda/base/kernel_launch.cuh"
-#define GKO_DEVICE_NAMESPACE cuda
-
-#elif defined(GKO_COMPILING_HIP)
-
-#include "hip/base/kernel_launch.hip.hpp"
-#define GKO_DEVICE_NAMESPACE hip
-
-#elif defined(GKO_COMPILING_DPCPP)
-
-#include "dpcpp/base/kernel_launch.dp.hpp"
-#define GKO_DEVICE_NAMESPACE dpcpp
-
-#elif defined(GKO_COMPILING_OMP)
-
-#include "omp/base/kernel_launch.hpp"
-#define GKO_DEVICE_NAMESPACE omp
-
-#else
-
-#error "This file should only be used inside Ginkgo device compilation"
-
-#endif
-
-
-#endif  // GKO_CORE_BASE_SIMPLE_KERNELS_HPP_
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 99301a6afe3..6f89ca45f79 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -187,7 +187,7 @@ ginkgo_default_includes(ginkgo_cuda)
 ginkgo_install_library(ginkgo_cuda "${GKO_CUDA_LIBDIR}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
-    ginkgo_check_headers(ginkgo_cuda)
+    ginkgo_check_headers(ginkgo_cuda GKO_COMPILING_CUDA)
 endif()
 
 if(GINKGO_BUILD_TESTS)
diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index 5ad9842d2f2..a4ee9296445 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -34,8 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CUDA_BASE_KERNEL_LAUNCH_CUH_
 
 
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
+#include "common/base/kernel_launch.hpp"
 
 
 #include "cuda/base/device_guard.hpp"
@@ -43,13 +42,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "cuda/components/thread_ids.cuh"
 
 
-#ifdef GKO_KERNEL
-#error "Only one kernel_launch.hpp file can be included at a time."
-#else
-#define GKO_KERNEL __device__
-#endif
-
-
 namespace gko {
 namespace kernels {
 namespace cuda {
@@ -58,70 +50,6 @@ namespace cuda {
 constexpr int default_block_size = 512;
 
 
-template <typename ValueType>
-struct matrix_accessor {
-    ValueType *data;
-    size_type stride;
-
-    __device__ ValueType &operator()(size_type row, size_type col)
-    {
-        return data[row * stride + col];
-    }
-
-    __device__ ValueType &operator[](size_type idx) { return data[idx]; }
-};
-
-
-template <typename T>
-struct to_device_type_impl {
-    using type = std::decay_t<cuda_type<T>>;
-    static type map_to_device(T in) { return as_cuda_type(in); }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<matrix::Dense<ValueType> *&> {
-    using type = matrix_accessor<cuda_type<ValueType>>;
-    static type map_to_device(matrix::Dense<ValueType> *mtx)
-    {
-        return {as_cuda_type(mtx->get_values()), mtx->get_stride()};
-    }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<const matrix::Dense<ValueType> *&> {
-    using type = matrix_accessor<const cuda_type<ValueType>>;
-    static type map_to_device(const matrix::Dense<ValueType> *mtx)
-    {
-        return {as_cuda_type(mtx->get_const_values()), mtx->get_stride()};
-    }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<Array<ValueType> &> {
-    using type = cuda_type<ValueType> *;
-    static type map_to_device(Array<ValueType> &array)
-    {
-        return as_cuda_type(array.get_data());
-    }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<const Array<ValueType> &> {
-    using type = const cuda_type<ValueType> *;
-    static type map_to_device(const Array<ValueType> &array)
-    {
-        return as_cuda_type(array.get_const_data());
-    }
-};
-
-
-template <typename T>
-typename to_device_type_impl<T>::type map_to_device(T &&param)
-{
-    return to_device_type_impl<T>::map_to_device(param);
-}
-
-
 template <typename KernelFunction, typename... KernelArgs>
 __global__ __launch_bounds__(default_block_size) void generic_kernel_1d(
     size_type size, KernelFunction fn, KernelArgs... args)
@@ -148,33 +76,31 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
 }
 
 
-}  // namespace cuda
-}  // namespace kernels
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const CudaExecutor> exec, KernelFunction fn,
                 size_type size, KernelArgs &&... args)
 {
-    cuda::device_guard guard{exec->get_device_id()};
-    constexpr auto block_size = kernels::cuda::default_block_size;
+    gko::cuda::device_guard guard{exec->get_device_id()};
+    constexpr auto block_size = default_block_size;
     auto num_blocks = ceildiv(size, block_size);
-    kernels::cuda::generic_kernel_1d<<<num_blocks, block_size>>>(
-        size, fn, kernels::cuda::map_to_device(args)...);
+    generic_kernel_1d<<<num_blocks, block_size>>>(size, fn,
+                                                  map_to_device(args)...);
 }
 
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const CudaExecutor> exec, KernelFunction fn,
                 dim<2> size, KernelArgs &&... args)
 {
-    cuda::device_guard guard{exec->get_device_id()};
-    constexpr auto block_size = kernels::cuda::default_block_size;
+    gko::cuda::device_guard guard{exec->get_device_id()};
+    constexpr auto block_size = default_block_size;
     auto num_blocks = ceildiv(size[0] * size[1], block_size);
-    kernels::cuda::generic_kernel_2d<<<num_blocks, block_size>>>(
-        size[0], size[1], fn, kernels::cuda::map_to_device(args)...);
+    generic_kernel_2d<<<num_blocks, block_size>>>(size[0], size[1], fn,
+                                                  map_to_device(args)...);
 }
 
 
+}  // namespace cuda
+}  // namespace kernels
 }  // namespace gko
 
 #endif  // GKO_CUDA_BASE_KERNEL_LAUNCH_CUH_
diff --git a/cuda/base/kernel_launch_solver.cuh b/cuda/base/kernel_launch_solver.cuh
index 738d2d0cda8..eaebcc62dcb 100644
--- a/cuda/base/kernel_launch_solver.cuh
+++ b/cuda/base/kernel_launch_solver.cuh
@@ -34,11 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CUDA_BASE_KERNEL_LAUNCH_SOLVER_CUH_
 
 
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-
-
-#include "cuda/base/kernel_launch.cuh"
+#include "common/base/kernel_launch_solver.hpp"
 
 
 namespace gko {
@@ -46,32 +42,6 @@ namespace kernels {
 namespace cuda {
 
 
-template <typename ValueType>
-struct default_stride_dense_wrapper {
-    ValueType *data;
-};
-
-
-template <typename T>
-struct device_unpack_solver_impl {
-    using type = T;
-    static __device__ __forceinline__ type unpack(T param, size_type)
-    {
-        return param;
-    }
-};
-
-template <typename ValueType>
-struct device_unpack_solver_impl<default_stride_dense_wrapper<ValueType>> {
-    using type = matrix_accessor<ValueType>;
-    static __device__ __forceinline__ type unpack(
-        default_stride_dense_wrapper<ValueType> param, size_type default_stride)
-    {
-        return {param.data, default_stride};
-    }
-};
-
-
 template <typename KernelFunction, typename... KernelArgs>
 __global__ __launch_bounds__(default_block_size) void generic_kernel_2d_solver(
     size_type rows, size_type cols, size_type default_stride, KernelFunction fn,
@@ -88,64 +58,21 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_2d_solver(
 }
 
 
-}  // namespace cuda
-}  // namespace kernels
-
-
-namespace solver {
-
-
-template <typename ValueType>
-kernels::cuda::default_stride_dense_wrapper<kernels::cuda::cuda_type<ValueType>>
-default_stride(matrix::Dense<ValueType> *mtx)
-{
-    return {kernels::cuda::as_cuda_type(mtx->get_values())};
-}
-
-
-template <typename ValueType>
-kernels::cuda::default_stride_dense_wrapper<
-    const kernels::cuda::cuda_type<ValueType>>
-default_stride(const matrix::Dense<ValueType> *mtx)
-{
-    return {kernels::cuda::as_cuda_type(mtx->get_const_values())};
-}
-
-
-template <typename ValueType>
-kernels::cuda::cuda_type<ValueType> *row_vector(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1);
-    return kernels::cuda::as_cuda_type(mtx->get_values());
-}
-
-
-template <typename ValueType>
-const kernels::cuda::cuda_type<ValueType> *row_vector(
-    const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1);
-    return kernels::cuda::as_cuda_type(mtx->get_const_values());
-}
-
-
-}  // namespace solver
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel_solver(std::shared_ptr<const CudaExecutor> exec,
                        KernelFunction fn, dim<2> size, size_type default_stride,
                        KernelArgs &&... args)
 {
-    cuda::device_guard guard{exec->get_device_id()};
-    constexpr auto block_size = kernels::cuda::default_block_size;
+    gko::cuda::device_guard guard{exec->get_device_id()};
+    constexpr auto block_size = default_block_size;
     auto num_blocks = ceildiv(size[0] * size[1], block_size);
-    kernels::cuda::generic_kernel_2d_solver<<<num_blocks, block_size>>>(
-        size[0], size[1], default_stride, fn,
-        kernels::cuda::map_to_device(args)...);
+    generic_kernel_2d_solver<<<num_blocks, block_size>>>(
+        size[0], size[1], default_stride, fn, map_to_device(args)...);
 }
 
 
+}  // namespace cuda
+}  // namespace kernels
 }  // namespace gko
 
 #endif  // GKO_CUDA_BASE_KERNEL_LAUNCH_SOLVER_CUH_
diff --git a/cuda/test/base/kernel_launch.cu b/cuda/test/base/kernel_launch.cu
index fa0bbb8aca1..7f2a67de50f 100644
--- a/cuda/test/base/kernel_launch.cu
+++ b/cuda/test/base/kernel_launch.cu
@@ -30,6 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#define GKO_COMPILING_CUDA
 #include "cuda/base/kernel_launch.cuh"
 
 
@@ -95,7 +96,7 @@ protected:
 // nvcc doesn't like device lambdas declared in complex classes, move it out
 void run1d(std::shared_ptr<gko::CudaExecutor> exec, size_type dim, int *data)
 {
-    gko::run_kernel(
+    gko::kernels::cuda::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto d) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -115,7 +116,7 @@ TEST_F(KernelLaunch, Runs1D)
 
 void run1d(std::shared_ptr<gko::CudaExecutor> exec, gko::Array<int> &data)
 {
-    gko::run_kernel(
+    gko::kernels::cuda::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto d, auto d_ptr) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -140,7 +141,7 @@ TEST_F(KernelLaunch, Runs1DArray)
 
 void run1d(std::shared_ptr<gko::CudaExecutor> exec, gko::matrix::Dense<> *m)
 {
-    gko::run_kernel(
+    gko::kernels::cuda::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto d, auto d2, auto d_ptr) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -176,7 +177,7 @@ TEST_F(KernelLaunch, Runs1DDense)
 
 void run2d(std::shared_ptr<gko::CudaExecutor> exec, int *data)
 {
-    gko::run_kernel(
+    gko::kernels::cuda::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -197,7 +198,7 @@ TEST_F(KernelLaunch, Runs2D)
 
 void run2d(std::shared_ptr<gko::CudaExecutor> exec, gko::Array<int> &data)
 {
-    gko::run_kernel(
+    gko::kernels::cuda::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d, auto d_ptr) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -224,7 +225,7 @@ TEST_F(KernelLaunch, Runs2DArray)
 void run2d(std::shared_ptr<gko::CudaExecutor> exec, gko::matrix::Dense<> *m1,
            gko::matrix::Dense<> *m2, gko::matrix::Dense<> *m3)
 {
-    gko::run_kernel_solver(
+    gko::kernels::cuda::run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d, auto d2, auto d_ptr, auto d3,
                       auto d4, auto d2_ptr, auto d3_ptr) {
@@ -257,8 +258,8 @@ void run2d(std::shared_ptr<gko::CudaExecutor> exec, gko::matrix::Dense<> *m1,
         },
         dim<2>{4, 4}, m2->get_stride(), m1,
         static_cast<const gko::matrix::Dense<> *>(m1), m1->get_const_values(),
-        gko::solver::default_stride(m2), gko::solver::row_vector(m3),
-        m2->get_values(), m3->get_values());
+        gko::kernels::cuda::default_stride(m2),
+        gko::kernels::cuda::row_vector(m3), m2->get_values(), m3->get_values());
 }
 
 TEST_F(KernelLaunch, Runs2DDense)
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index e90777c4ad3..97cf8a5daf6 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -69,7 +69,7 @@ ginkgo_default_includes(ginkgo_dpcpp)
 ginkgo_install_library(ginkgo_dpcpp)
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
-    ginkgo_check_headers(ginkgo_dpcpp)
+    ginkgo_check_headers(ginkgo_dpcpp GKO_COMPILING_DPCPP)
 endif()
 
 if(GINKGO_BUILD_TESTS)
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
index 53f2c27b09a..3630e8f326a 100644
--- a/dpcpp/base/kernel_launch.dp.hpp
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -34,18 +34,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_DPCPP_BASE_KERNEL_LAUNCH_DP_HPP_
 
 
-#include <CL/sycl.hpp>
-
+#include "common/base/kernel_launch.hpp"
 
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
 
-
-#ifdef GKO_KERNEL
-#error "Only one kernel_launch.hpp file can be included at a time."
-#else
-#define GKO_KERNEL
-#endif
+#include <CL/sycl.hpp>
 
 
 namespace gko {
@@ -53,70 +45,6 @@ namespace kernels {
 namespace dpcpp {
 
 
-template <typename ValueType>
-struct matrix_accessor {
-    ValueType *data;
-    size_type stride;
-
-    ValueType &operator()(size_type row, size_type col)
-    {
-        return data[row * stride + col];
-    }
-
-    ValueType &operator[](size_type idx) { return data[idx]; }
-};
-
-
-template <typename T>
-struct to_device_type_impl {
-    using type = std::decay_t<T>;
-    static type map_to_device(T in) { return in; }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<matrix::Dense<ValueType> *&> {
-    using type = matrix_accessor<ValueType>;
-    static type map_to_device(matrix::Dense<ValueType> *mtx)
-    {
-        return {mtx->get_values(), mtx->get_stride()};
-    }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<const matrix::Dense<ValueType> *&> {
-    using type = matrix_accessor<const ValueType>;
-    static type map_to_device(const matrix::Dense<ValueType> *mtx)
-    {
-        return {mtx->get_const_values(), mtx->get_stride()};
-    }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<Array<ValueType> &> {
-    using type = ValueType *;
-    static type map_to_device(Array<ValueType> &array)
-    {
-        return array.get_data();
-    }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<const Array<ValueType> &> {
-    using type = const ValueType *;
-    static type map_to_device(const Array<ValueType> &array)
-    {
-        return array.get_const_data();
-    }
-};
-
-
-template <typename T>
-typename to_device_type_impl<T>::type map_to_device(T &&param)
-{
-    return to_device_type_impl<T>::map_to_device(param);
-}
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void generic_kernel_1d(sycl::handler &cgh, size_type size, KernelFunction fn,
                        KernelArgs... args)
@@ -140,17 +68,12 @@ void generic_kernel_2d(sycl::handler &cgh, size_type rows, size_type cols,
 }
 
 
-}  // namespace dpcpp
-}  // namespace kernels
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const DpcppExecutor> exec, KernelFunction fn,
                 size_type size, KernelArgs &&... args)
 {
     exec->get_queue()->submit([&](sycl::handler &cgh) {
-        kernels::dpcpp::generic_kernel_1d(
-            cgh, size, fn, kernels::dpcpp::map_to_device(args)...);
+        generic_kernel_1d(cgh, size, fn, map_to_device(args)...);
     });
 }
 
@@ -159,12 +82,13 @@ void run_kernel(std::shared_ptr<const DpcppExecutor> exec, KernelFunction fn,
                 dim<2> size, KernelArgs &&... args)
 {
     exec->get_queue()->submit([&](sycl::handler &cgh) {
-        kernels::dpcpp::generic_kernel_2d(
-            cgh, size[0], size[1], fn, kernels::dpcpp::map_to_device(args)...);
+        generic_kernel_2d(cgh, size[0], size[1], fn, map_to_device(args)...);
     });
 }
 
 
+}  // namespace dpcpp
+}  // namespace kernels
 }  // namespace gko
 
 
diff --git a/dpcpp/base/kernel_launch_solver.dp.hpp b/dpcpp/base/kernel_launch_solver.dp.hpp
index 9824d0ac567..62c4c325ac5 100644
--- a/dpcpp/base/kernel_launch_solver.dp.hpp
+++ b/dpcpp/base/kernel_launch_solver.dp.hpp
@@ -34,11 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_DPCPP_BASE_KERNEL_LAUNCH_SOLVER_DP_HPP_
 
 
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-
-
-#include "dpcpp/base/kernel_launch.dp.hpp"
+#include "common/base/kernel_launch_solver.hpp"
 
 
 namespace gko {
@@ -46,29 +42,6 @@ namespace kernels {
 namespace dpcpp {
 
 
-template <typename ValueType>
-struct default_stride_dense_wrapper {
-    ValueType *data;
-};
-
-
-template <typename T>
-struct device_unpack_solver_impl {
-    using type = T;
-    static type unpack(T param, size_type) { return param; }
-};
-
-template <typename ValueType>
-struct device_unpack_solver_impl<default_stride_dense_wrapper<ValueType>> {
-    using type = matrix_accessor<ValueType>;
-    static type unpack(default_stride_dense_wrapper<ValueType> param,
-                       size_type default_stride)
-    {
-        return {param.data, default_stride};
-    }
-};
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void generic_kernel_2d_solver(sycl::handler &cgh, size_type rows,
                               size_type cols, size_type default_stride,
@@ -84,48 +57,6 @@ void generic_kernel_2d_solver(sycl::handler &cgh, size_type rows,
 }
 
 
-}  // namespace dpcpp
-}  // namespace kernels
-
-
-namespace solver {
-
-
-template <typename ValueType>
-kernels::dpcpp::default_stride_dense_wrapper<ValueType> default_stride(
-    matrix::Dense<ValueType> *mtx)
-{
-    return {mtx->get_values()};
-}
-
-
-template <typename ValueType>
-kernels::dpcpp::default_stride_dense_wrapper<const ValueType> default_stride(
-    const matrix::Dense<ValueType> *mtx)
-{
-    return {mtx->get_const_values()};
-}
-
-
-template <typename ValueType>
-ValueType *row_vector(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1);
-    return mtx->get_values();
-}
-
-
-template <typename ValueType>
-const ValueType *row_vector(const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1);
-    return mtx->get_const_values();
-}
-
-
-}  // namespace solver
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel_solver(std::shared_ptr<const DpcppExecutor> exec,
                        KernelFunction fn, dim<2> size, size_type default_stride,
@@ -139,6 +70,8 @@ void run_kernel_solver(std::shared_ptr<const DpcppExecutor> exec,
 }
 
 
+}  // namespace dpcpp
+}  // namespace kernels
 }  // namespace gko
 
 #endif  // GKO_HIP_BASE_KERNEL_LAUNCH_SOLVER_HIP_HPP_
diff --git a/dpcpp/components/format_conversion.dp.hpp b/dpcpp/components/format_conversion.dp.hpp
index 404637daed2..c40f765448f 100644
--- a/dpcpp/components/format_conversion.dp.hpp
+++ b/dpcpp/components/format_conversion.dp.hpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <CL/sycl.hpp>
 
 
+#include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/types.hpp>
 
 
diff --git a/dpcpp/test/base/kernel_launch.dp.cpp b/dpcpp/test/base/kernel_launch.dp.cpp
index 3a589dfc8ce..82cc6c2b320 100644
--- a/dpcpp/test/base/kernel_launch.dp.cpp
+++ b/dpcpp/test/base/kernel_launch.dp.cpp
@@ -30,6 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#define GKO_COMPILING_DPCPP
 #include "dpcpp/base/kernel_launch.dp.hpp"
 
 
@@ -101,7 +102,7 @@ class KernelLaunch : public ::testing::Test {
 
 TEST_F(KernelLaunch, Runs1D)
 {
-    gko::run_kernel(
+    gko::kernels::dpcpp::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto d) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -116,7 +117,7 @@ TEST_F(KernelLaunch, Runs1D)
 
 TEST_F(KernelLaunch, Runs1DArray)
 {
-    gko::run_kernel(
+    gko::kernels::dpcpp::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto d, auto d_ptr) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -136,7 +137,7 @@ TEST_F(KernelLaunch, Runs1DArray)
 
 TEST_F(KernelLaunch, Runs1DDense)
 {
-    gko::run_kernel(
+    gko::kernels::dpcpp::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto d, auto d2, auto d_ptr) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -168,7 +169,7 @@ TEST_F(KernelLaunch, Runs1DDense)
 
 TEST_F(KernelLaunch, Runs2D)
 {
-    gko::run_kernel(
+    gko::kernels::dpcpp::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -184,7 +185,7 @@ TEST_F(KernelLaunch, Runs2D)
 
 TEST_F(KernelLaunch, Runs2DArray)
 {
-    gko::run_kernel(
+    gko::kernels::dpcpp::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d, auto d_ptr) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -205,7 +206,7 @@ TEST_F(KernelLaunch, Runs2DArray)
 
 TEST_F(KernelLaunch, Runs2DDense)
 {
-    gko::run_kernel_solver(
+    gko::kernels::dpcpp::run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d, auto d2, auto d_ptr, auto d3,
                       auto d4, auto d2_ptr, auto d3_ptr) {
@@ -243,9 +244,9 @@ TEST_F(KernelLaunch, Runs2DDense)
         dim<2>{4, 4}, zero_dense->get_stride(), zero_dense2.get(),
         static_cast<const Mtx *>(zero_dense2.get()),
         zero_dense2->get_const_values(),
-        gko::solver::default_stride(zero_dense.get()),
-        gko::solver::row_vector(vec_dense.get()), zero_dense->get_values(),
-        vec_dense->get_values());
+        gko::kernels::dpcpp::default_stride(zero_dense.get()),
+        gko::kernels::dpcpp::row_vector(vec_dense.get()),
+        zero_dense->get_values(), vec_dense->get_values());
 
     GKO_ASSERT_MTX_NEAR(zero_dense2, iota_dense, 0.0);
 }
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index e6f6199bc48..61870d03aa7 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -296,7 +296,7 @@ ginkgo_default_includes(ginkgo_hip)
 ginkgo_install_library(ginkgo_hip "${GKO_HIP_RPATH}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
-    ginkgo_check_headers(ginkgo_hip)
+    ginkgo_check_headers(ginkgo_hip GKO_COMPILING_HIP)
 endif()
 
 if(GINKGO_BUILD_TESTS)
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index acef2c8cfab..991b0cc1f98 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -34,11 +34,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_HIP_BASE_KERNEL_LAUNCH_HIP_HPP_
 
 
-#include <hip/hip_runtime.h>
+#include "common/base/kernel_launch.hpp"
 
 
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
+#include <hip/hip_runtime.h>
 
 
 #include "hip/base/device_guard.hip.hpp"
@@ -46,13 +45,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "hip/components/thread_ids.hip.hpp"
 
 
-#ifdef GKO_KERNEL
-#error "Only one kernel_launch.hpp file can be included at a time."
-#else
-#define GKO_KERNEL __device__
-#endif
-
-
 namespace gko {
 namespace kernels {
 namespace hip {
@@ -61,70 +53,6 @@ namespace hip {
 constexpr int default_block_size = 512;
 
 
-template <typename ValueType>
-struct matrix_accessor {
-    ValueType *data;
-    size_type stride;
-
-    __device__ ValueType &operator()(size_type row, size_type col)
-    {
-        return data[row * stride + col];
-    }
-
-    __device__ ValueType &operator[](size_type idx) { return data[idx]; }
-};
-
-
-template <typename T>
-struct to_device_type_impl {
-    using type = std::decay_t<hip_type<T>>;
-    static type map_to_device(T in) { return as_hip_type(in); }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<matrix::Dense<ValueType> *&> {
-    using type = matrix_accessor<hip_type<ValueType>>;
-    static type map_to_device(matrix::Dense<ValueType> *mtx)
-    {
-        return {as_hip_type(mtx->get_values()), mtx->get_stride()};
-    }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<const matrix::Dense<ValueType> *&> {
-    using type = matrix_accessor<const hip_type<ValueType>>;
-    static type map_to_device(const matrix::Dense<ValueType> *mtx)
-    {
-        return {as_hip_type(mtx->get_const_values()), mtx->get_stride()};
-    }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<Array<ValueType> &> {
-    using type = hip_type<ValueType> *;
-    static type map_to_device(Array<ValueType> &array)
-    {
-        return as_hip_type(array.get_data());
-    }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<const Array<ValueType> &> {
-    using type = const hip_type<ValueType> *;
-    static type map_to_device(const Array<ValueType> &array)
-    {
-        return as_hip_type(array.get_const_data());
-    }
-};
-
-
-template <typename T>
-typename to_device_type_impl<T>::type map_to_device(T &&param)
-{
-    return to_device_type_impl<T>::map_to_device(param);
-}
-
-
 template <typename KernelFunction, typename... KernelArgs>
 __global__ __launch_bounds__(default_block_size) void generic_kernel_1d(
     size_type size, KernelFunction fn, KernelArgs... args)
@@ -151,34 +79,31 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_2d(
 }
 
 
-}  // namespace hip
-}  // namespace kernels
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const HipExecutor> exec, KernelFunction fn,
                 size_type size, KernelArgs &&... args)
 {
-    hip::device_guard guard{exec->get_device_id()};
-    constexpr auto block_size = kernels::hip::default_block_size;
+    gko::hip::device_guard guard{exec->get_device_id()};
+    constexpr auto block_size = default_block_size;
     auto num_blocks = ceildiv(size, block_size);
-    hipLaunchKernelGGL(kernels::hip::generic_kernel_1d, num_blocks, block_size,
-                       0, 0, size, fn, kernels::hip::map_to_device(args)...);
+    hipLaunchKernelGGL(generic_kernel_1d, num_blocks, block_size, 0, 0, size,
+                       fn, map_to_device(args)...);
 }
 
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const HipExecutor> exec, KernelFunction fn,
                 dim<2> size, KernelArgs &&... args)
 {
-    hip::device_guard guard{exec->get_device_id()};
-    constexpr auto block_size = kernels::hip::default_block_size;
+    gko::hip::device_guard guard{exec->get_device_id()};
+    constexpr auto block_size = default_block_size;
     auto num_blocks = ceildiv(size[0] * size[1], block_size);
-    hipLaunchKernelGGL(kernels::hip::generic_kernel_2d, num_blocks, block_size,
-                       0, 0, size[0], size[1], fn,
-                       kernels::hip::map_to_device(args)...);
+    hipLaunchKernelGGL(generic_kernel_2d, num_blocks, block_size, 0, 0, size[0],
+                       size[1], fn, map_to_device(args)...);
 }
 
 
+}  // namespace hip
+}  // namespace kernels
 }  // namespace gko
 
 #endif  // GKO_HIP_BASE_KERNEL_LAUNCH_HIP_HPP_
diff --git a/hip/base/kernel_launch_solver.hip.hpp b/hip/base/kernel_launch_solver.hip.hpp
index cfacf34a412..9f8a5e6e5f2 100644
--- a/hip/base/kernel_launch_solver.hip.hpp
+++ b/hip/base/kernel_launch_solver.hip.hpp
@@ -34,14 +34,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_HIP_BASE_KERNEL_LAUNCH_SOLVER_HIP_HPP_
 
 
-#include <hip/hip_runtime.h>
-
-
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
+#include "common/base/kernel_launch_solver.hpp"
 
 
-#include "hip/base/kernel_launch.hip.hpp"
+#include <hip/hip_runtime.h>
 
 
 namespace gko {
@@ -49,32 +45,6 @@ namespace kernels {
 namespace hip {
 
 
-template <typename ValueType>
-struct default_stride_dense_wrapper {
-    ValueType *data;
-};
-
-
-template <typename T>
-struct device_unpack_solver_impl {
-    using type = T;
-    static __device__ __forceinline__ type unpack(T param, size_type)
-    {
-        return param;
-    }
-};
-
-template <typename ValueType>
-struct device_unpack_solver_impl<default_stride_dense_wrapper<ValueType>> {
-    using type = matrix_accessor<ValueType>;
-    static __device__ __forceinline__ type unpack(
-        default_stride_dense_wrapper<ValueType> param, size_type default_stride)
-    {
-        return {param.data, default_stride};
-    }
-};
-
-
 template <typename KernelFunction, typename... KernelArgs>
 __global__ __launch_bounds__(default_block_size) void generic_kernel_2d_solver(
     size_type rows, size_type cols, size_type default_stride, KernelFunction fn,
@@ -91,56 +61,12 @@ __global__ __launch_bounds__(default_block_size) void generic_kernel_2d_solver(
 }
 
 
-}  // namespace hip
-}  // namespace kernels
-
-
-namespace solver {
-
-
-template <typename ValueType>
-kernels::hip::default_stride_dense_wrapper<kernels::hip::hip_type<ValueType>>
-default_stride(matrix::Dense<ValueType> *mtx)
-{
-    return {kernels::hip::as_hip_type(mtx->get_values())};
-}
-
-
-template <typename ValueType>
-kernels::hip::default_stride_dense_wrapper<
-    const kernels::hip::hip_type<ValueType>>
-default_stride(const matrix::Dense<ValueType> *mtx)
-{
-    return {kernels::hip::as_hip_type(mtx->get_const_values())};
-}
-
-
-template <typename ValueType>
-kernels::hip::hip_type<ValueType> *row_vector(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1);
-    return kernels::hip::as_hip_type(mtx->get_values());
-}
-
-
-template <typename ValueType>
-const kernels::hip::hip_type<ValueType> *row_vector(
-    const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1);
-    return kernels::hip::as_hip_type(mtx->get_const_values());
-}
-
-
-}  // namespace solver
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel_solver(std::shared_ptr<const HipExecutor> exec,
                        KernelFunction fn, dim<2> size, size_type default_stride,
                        KernelArgs &&... args)
 {
-    hip::device_guard guard{exec->get_device_id()};
+    gko::hip::device_guard guard{exec->get_device_id()};
     constexpr auto block_size = kernels::hip::default_block_size;
     auto num_blocks = ceildiv(size[0] * size[1], block_size);
     hipLaunchKernelGGL(kernels::hip::generic_kernel_2d_solver, num_blocks,
@@ -149,6 +75,8 @@ void run_kernel_solver(std::shared_ptr<const HipExecutor> exec,
 }
 
 
+}  // namespace hip
+}  // namespace kernels
 }  // namespace gko
 
 #endif  // GKO_HIP_BASE_KERNEL_LAUNCH_SOLVER_HIP_HPP_
diff --git a/hip/test/base/kernel_launch.hip.cpp b/hip/test/base/kernel_launch.hip.cpp
index 91d03d9c00e..968f5a47424 100644
--- a/hip/test/base/kernel_launch.hip.cpp
+++ b/hip/test/base/kernel_launch.hip.cpp
@@ -30,6 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+// no need for #define GKO_COMPILING_HIP here, it gets set by ginkgo_hip
 #include "hip/base/kernel_launch.hip.hpp"
 
 
@@ -94,7 +95,7 @@ class KernelLaunch : public ::testing::Test {
 // nvcc doesn't like device lambdas declared in complex classes, move it out
 void run1d(std::shared_ptr<gko::HipExecutor> exec, size_type dim, int *data)
 {
-    gko::run_kernel(
+    gko::kernels::hip::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto d) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -114,7 +115,7 @@ TEST_F(KernelLaunch, Runs1D)
 
 void run1d(std::shared_ptr<gko::HipExecutor> exec, gko::Array<int> &data)
 {
-    gko::run_kernel(
+    gko::kernels::hip::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto d, auto d_ptr) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -139,7 +140,7 @@ TEST_F(KernelLaunch, Runs1DArray)
 
 void run1d(std::shared_ptr<gko::HipExecutor> exec, gko::matrix::Dense<> *m)
 {
-    gko::run_kernel(
+    gko::kernels::hip::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto d, auto d2, auto d_ptr) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -175,7 +176,7 @@ TEST_F(KernelLaunch, Runs1DDense)
 
 void run2d(std::shared_ptr<gko::HipExecutor> exec, int *data)
 {
-    gko::run_kernel(
+    gko::kernels::hip::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -196,7 +197,7 @@ TEST_F(KernelLaunch, Runs2D)
 
 void run2d(std::shared_ptr<gko::HipExecutor> exec, gko::Array<int> &data)
 {
-    gko::run_kernel(
+    gko::kernels::hip::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d, auto d_ptr) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -223,7 +224,7 @@ TEST_F(KernelLaunch, Runs2DArray)
 void run2d(std::shared_ptr<gko::HipExecutor> exec, gko::matrix::Dense<> *m1,
            gko::matrix::Dense<> *m2, gko::matrix::Dense<> *m3)
 {
-    gko::run_kernel_solver(
+    gko::kernels::hip::run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d, auto d2, auto d_ptr, auto d3,
                       auto d4, auto d2_ptr, auto d3_ptr) {
@@ -256,8 +257,8 @@ void run2d(std::shared_ptr<gko::HipExecutor> exec, gko::matrix::Dense<> *m1,
         },
         dim<2>{4, 4}, m2->get_stride(), m1,
         static_cast<const gko::matrix::Dense<> *>(m1), m1->get_const_values(),
-        gko::solver::default_stride(m2), gko::solver::row_vector(m3),
-        m2->get_values(), m3->get_values());
+        gko::kernels::hip::default_stride(m2),
+        gko::kernels::hip::row_vector(m3), m2->get_values(), m3->get_values());
 }
 
 TEST_F(KernelLaunch, Runs2DDense)
diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt
index 004b7b359ad..1f103460c2a 100644
--- a/include/CMakeLists.txt
+++ b/include/CMakeLists.txt
@@ -2,5 +2,5 @@ if (GINKGO_CHECK_CIRCULAR_DEPS)
     add_library(ginkgo_public_api INTERFACE) # dummy target
     set_property(TARGET ginkgo_public_api APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}")
     set_property(TARGET ginkgo_public_api APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_BINARY_DIR}")
-    ginkgo_check_headers(ginkgo_public_api)
+    ginkgo_check_headers(ginkgo_public_api "")
 endif()
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index 097ceab9695..c58709bc904 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -78,7 +78,7 @@ ginkgo_default_includes(ginkgo_omp)
 ginkgo_install_library(ginkgo_omp)
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
-    ginkgo_check_headers(ginkgo_omp)
+    ginkgo_check_headers(ginkgo_omp GKO_COMPILING_OMP)
 endif()
 
 if(GINKGO_BUILD_TESTS)
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index dcad0da9b07..0ff866e7966 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -34,15 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_OMP_BASE_KERNEL_LAUNCH_HPP_
 
 
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-
-
-#ifdef GKO_KERNEL
-#error "Only one kernel_launch.hpp file can be included at a time."
-#else
-#define GKO_KERNEL
-#endif
+#include "common/base/kernel_launch.hpp"
 
 
 namespace gko {
@@ -50,81 +42,13 @@ namespace kernels {
 namespace omp {
 
 
-template <typename ValueType>
-struct matrix_accessor {
-    ValueType *data;
-    size_type stride;
-
-    ValueType &operator()(size_type row, size_type col)
-    {
-        return data[row * stride + col];
-    }
-
-    ValueType &operator[](size_type idx) { return data[idx]; }
-};
-
-
-template <typename T>
-struct to_device_type_impl {
-    using type = std::decay_t<T>;
-    static type map_to_device(T in) { return in; }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<matrix::Dense<ValueType> *&> {
-    using type = matrix_accessor<ValueType>;
-    static type map_to_device(matrix::Dense<ValueType> *mtx)
-    {
-        return {mtx->get_values(), mtx->get_stride()};
-    }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<const matrix::Dense<ValueType> *&> {
-    using type = matrix_accessor<const ValueType>;
-    static type map_to_device(const matrix::Dense<ValueType> *mtx)
-    {
-        return {mtx->get_const_values(), mtx->get_stride()};
-    }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<Array<ValueType> &> {
-    using type = ValueType *;
-    static type map_to_device(Array<ValueType> &array)
-    {
-        return array.get_data();
-    }
-};
-
-template <typename ValueType>
-struct to_device_type_impl<const Array<ValueType> &> {
-    using type = const ValueType *;
-    static type map_to_device(const Array<ValueType> &array)
-    {
-        return array.get_const_data();
-    }
-};
-
-
-template <typename T>
-typename to_device_type_impl<T>::type map_to_device(T &&param)
-{
-    return to_device_type_impl<T>::map_to_device(param);
-}
-
-
-}  // namespace omp
-}  // namespace kernels
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
                 size_type size, KernelArgs &&... args)
 {
 #pragma omp parallel for
     for (size_type i = 0; i < size; i++) {
-        [&]() { fn(i, kernels::omp::map_to_device(args)...); }();
+        [&]() { fn(i, map_to_device(args)...); }();
     }
 }
 
@@ -219,10 +143,12 @@ template <typename KernelFunction, typename... KernelArgs>
 void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
                 dim<2> size, KernelArgs &&... args)
 {
-    run_kernel_impl(exec, fn, size, kernels::omp::map_to_device(args)...);
+    run_kernel_impl(exec, fn, size, map_to_device(args)...);
 }
 
 
+}  // namespace omp
+}  // namespace kernels
 }  // namespace gko
 
 #endif  // GKO_OMP_BASE_KERNEL_LAUNCH_HPP_
diff --git a/omp/base/kernel_launch_solver.hpp b/omp/base/kernel_launch_solver.hpp
index 72df8aa835a..f3607a063ee 100644
--- a/omp/base/kernel_launch_solver.hpp
+++ b/omp/base/kernel_launch_solver.hpp
@@ -34,11 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_OMP_BASE_KERNEL_LAUNCH_SOLVER_HPP_
 
 
-#include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-
-
-#include "omp/base/kernel_launch.hpp"
+#include "common/base/kernel_launch_solver.hpp"
 
 
 namespace gko {
@@ -46,29 +42,6 @@ namespace kernels {
 namespace omp {
 
 
-template <typename ValueType>
-struct default_stride_dense_wrapper {
-    ValueType *data;
-};
-
-
-template <typename T>
-struct device_unpack_solver_impl {
-    using type = T;
-    static type unpack(T param, size_type) { return param; }
-};
-
-template <typename ValueType>
-struct device_unpack_solver_impl<default_stride_dense_wrapper<ValueType>> {
-    using type = matrix_accessor<ValueType>;
-    static type unpack(default_stride_dense_wrapper<ValueType> param,
-                       size_type default_stride)
-    {
-        return {param.data, default_stride};
-    }
-};
-
-
 template <typename T>
 typename device_unpack_solver_impl<typename to_device_type_impl<T>::type>::type
 map_to_device_solver(T &&param, size_type default_stride)
@@ -78,59 +51,18 @@ map_to_device_solver(T &&param, size_type default_stride)
 }
 
 
-}  // namespace omp
-}  // namespace kernels
-
-
-namespace solver {
-
-
-template <typename ValueType>
-kernels::omp::default_stride_dense_wrapper<ValueType> default_stride(
-    matrix::Dense<ValueType> *mtx)
-{
-    return {mtx->get_values()};
-}
-
-
-template <typename ValueType>
-kernels::omp::default_stride_dense_wrapper<const ValueType> default_stride(
-    const matrix::Dense<ValueType> *mtx)
-{
-    return {mtx->get_const_values()};
-}
-
-
-template <typename ValueType>
-ValueType *row_vector(matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1);
-    return mtx->get_values();
-}
-
-
-template <typename ValueType>
-const ValueType *row_vector(const matrix::Dense<ValueType> *mtx)
-{
-    GKO_ASSERT(mtx->get_size()[0] == 1);
-    return mtx->get_const_values();
-}
-
-
-}  // namespace solver
-
-
 template <typename KernelFunction, typename... KernelArgs>
 void run_kernel_solver(std::shared_ptr<const OmpExecutor> exec,
                        KernelFunction fn, dim<2> size, size_type default_stride,
                        KernelArgs &&... args)
 {
-    run_kernel_impl(
-        exec, fn, size,
-        kernels::omp::map_to_device_solver(args, default_stride)...);
+    run_kernel_impl(exec, fn, size,
+                    map_to_device_solver(args, default_stride)...);
 }
 
 
+}  // namespace omp
+}  // namespace kernels
 }  // namespace gko
 
 #endif  // GKO_OMP_BASE_KERNEL_LAUNCH_SOLVER_HPP_
diff --git a/omp/test/base/kernel_launch.cpp b/omp/test/base/kernel_launch.cpp
index 30fd82e5002..fb40a68810d 100644
--- a/omp/test/base/kernel_launch.cpp
+++ b/omp/test/base/kernel_launch.cpp
@@ -30,6 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#define GKO_COMPILING_OMP
 #include "omp/base/kernel_launch.hpp"
 
 
@@ -87,7 +88,7 @@ class KernelLaunch : public ::testing::Test {
 
 TEST_F(KernelLaunch, Runs1D)
 {
-    gko::run_kernel(
+    gko::kernels::omp::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto d) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -102,7 +103,7 @@ TEST_F(KernelLaunch, Runs1D)
 
 TEST_F(KernelLaunch, Runs1DArray)
 {
-    gko::run_kernel(
+    gko::kernels::omp::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto d, auto d_ptr) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -122,7 +123,7 @@ TEST_F(KernelLaunch, Runs1DArray)
 
 TEST_F(KernelLaunch, Runs1DDense)
 {
-    gko::run_kernel(
+    gko::kernels::omp::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto d, auto d2, auto d_ptr) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -154,7 +155,7 @@ TEST_F(KernelLaunch, Runs1DDense)
 
 TEST_F(KernelLaunch, Runs2D)
 {
-    gko::run_kernel(
+    gko::kernels::omp::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -170,7 +171,7 @@ TEST_F(KernelLaunch, Runs2D)
 
 TEST_F(KernelLaunch, Runs2DArray)
 {
-    gko::run_kernel(
+    gko::kernels::omp::run_kernel(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d, auto d_ptr) {
             static_assert(is_same<decltype(i), size_type>::value, "index");
@@ -191,7 +192,7 @@ TEST_F(KernelLaunch, Runs2DArray)
 
 TEST_F(KernelLaunch, Runs2DDense)
 {
-    gko::run_kernel_solver(
+    gko::kernels::omp::run_kernel_solver(
         exec,
         [] GKO_KERNEL(auto i, auto j, auto d, auto d2, auto d_ptr, auto d3,
                       auto d4, auto d2_ptr, auto d3_ptr) {
@@ -225,9 +226,9 @@ TEST_F(KernelLaunch, Runs2DDense)
         dim<2>{4, 4}, zero_dense->get_stride(), zero_dense2.get(),
         static_cast<const gko::matrix::Dense<> *>(zero_dense2.get()),
         zero_dense2->get_const_values(),
-        gko::solver::default_stride(zero_dense.get()),
-        gko::solver::row_vector(vec_dense.get()), zero_dense->get_values(),
-        vec_dense->get_values());
+        gko::kernels::omp::default_stride(zero_dense.get()),
+        gko::kernels::omp::row_vector(vec_dense.get()),
+        zero_dense->get_values(), vec_dense->get_values());
 
     GKO_ASSERT_MTX_NEAR(zero_dense2, iota_dense, 0.0);
 }
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index e84453b8664..c008ad1a87c 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -47,7 +47,7 @@ ginkgo_install_library(ginkgo_reference)
 target_compile_options(ginkgo_reference PRIVATE "${GINKGO_COMPILER_FLAGS}")
 
 if (GINKGO_CHECK_CIRCULAR_DEPS)
-    ginkgo_check_headers(ginkgo_reference)
+    ginkgo_check_headers(ginkgo_reference "")
 endif()
 
 if(GINKGO_BUILD_TESTS)

From f19dc74cf6b6f44c6167a0c8f7cd5dc540f812e7 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 23 Jun 2021 22:25:07 +0200
Subject: [PATCH 0851/1008] work around DPC++ use-after-free

---
 dpcpp/base/executor.dp.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 9130fcf7ca5..39c3e4557e6 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -102,6 +102,7 @@ void DpcppExecutor::populate_exec_info(const MachineTopology *mach_topo)
 
 void DpcppExecutor::raw_free(void *ptr) const noexcept
 {
+    queue_->wait_and_throw();
     sycl::free(ptr, queue_->get_context());
 }
 

From 61edc0eca50d2c5adc67af738a2a6c71f28a9b09 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 24 Jun 2021 12:13:04 +0200
Subject: [PATCH 0852/1008] improve documentation for kernel_launch

---
 common/base/kernel_launch.hpp        | 36 ++++++++++++++++++++++++++++
 common/base/kernel_launch_solver.hpp | 34 ++++++++++++++++++++++++--
 2 files changed, 68 insertions(+), 2 deletions(-)

diff --git a/common/base/kernel_launch.hpp b/common/base/kernel_launch.hpp
index 289df89eae5..bb127ea8ece 100644
--- a/common/base/kernel_launch.hpp
+++ b/common/base/kernel_launch.hpp
@@ -156,17 +156,33 @@ namespace kernels {
 namespace GKO_DEVICE_NAMESPACE {
 
 
+/**
+ * @internal
+ * A simple row-major accessor as a device representation of gko::matrix::Dense
+ * objects.
+ *
+ * @tparam ValueType  the value type of the underlying matrix.
+ */
 template <typename ValueType>
 struct matrix_accessor {
     ValueType *data;
     size_type stride;
 
+    /**
+     * @internal
+     * Returns a reference to the element at position (row, col).
+     */
     GKO_INLINE GKO_ATTRIBUTES ValueType &operator()(size_type row,
                                                     size_type col)
     {
         return data[row * stride + col];
     }
 
+    /**
+     * @internal
+     * Returns a reference to the element at position idx in the underlying
+     * storage.
+     */
     GKO_INLINE GKO_ATTRIBUTES ValueType &operator[](size_type idx)
     {
         return data[idx];
@@ -174,6 +190,24 @@ struct matrix_accessor {
 };
 
 
+/**
+ * @internal
+ * This struct is used to provide mappings from host types like
+ * gko::matrix::Dense to device representations of the same data, like an
+ * accessor storing only data pointer and stride.
+ *
+ * By default, it only maps std::complex to the corresponding device
+ * representation of the complex type. There are specializations for dealing
+ * with gko::Array and gko::matrix::Dense (both const and mutable) that map them
+ * to plain pointers or matrix_accessor objects.
+ *
+ * @tparam T  the type being mapped. It will be used based on a
+ *            forwarding-reference, i.e. preserve references in the input
+ *            parameter, so special care must be taken to only return types that
+ *            can be passed to the device, i.e. (structs containing) device
+ *            pointers or values. This means that T will be either a r-value or
+ *            l-value reference.
+ */
 template <typename T>
 struct to_device_type_impl {
     using type = std::decay_t<device_type<T>>;
@@ -229,6 +263,8 @@ typename to_device_type_impl<T>::type map_to_device(T &&param)
 }  // namespace gko
 
 
+// these files include this file again to make inclusion work from both sides,
+// this does not lead to issues due to the header guards.
 #if defined(GKO_COMPILING_CUDA)
 #include "cuda/base/kernel_launch.cuh"
 #elif defined(GKO_COMPILING_HIP)
diff --git a/common/base/kernel_launch_solver.hpp b/common/base/kernel_launch_solver.hpp
index fc10e508807..b67813c1e1a 100644
--- a/common/base/kernel_launch_solver.hpp
+++ b/common/base/kernel_launch_solver.hpp
@@ -42,12 +42,24 @@ namespace kernels {
 namespace GKO_DEVICE_NAMESPACE {
 
 
+/**
+ * @internal
+ * Wrapper class used by default_stride(matrix::Dense<ValueType>*) to wrap a
+ * dense matrix using the default stride.
+ */
 template <typename ValueType>
 struct default_stride_dense_wrapper {
     ValueType *data;
 };
 
 
+/**
+ * @internal
+ * Helper that creates a device representation of the input object based on the
+ * default stride that was passed to run_kernel_solver.
+ * @see default_stride_dense_wrapper
+ * @see default_stride(matrix::Dense<ValueType>*)
+ */
 template <typename T>
 struct device_unpack_solver_impl {
     using type = T;
@@ -68,6 +80,12 @@ struct device_unpack_solver_impl<default_stride_dense_wrapper<ValueType>> {
 };
 
 
+/**
+ * @internal
+ * Wraps the given matrix in a wrapper signifying that it has the default stride
+ * that was provided to run_kernel_solver. This avoids having individual stride
+ * parameters for all dense matrix parameters.
+ */
 template <typename ValueType>
 default_stride_dense_wrapper<device_type<ValueType>> default_stride(
     matrix::Dense<ValueType> *mtx)
@@ -75,7 +93,10 @@ default_stride_dense_wrapper<device_type<ValueType>> default_stride(
     return {as_device_type(mtx->get_values())};
 }
 
-
+/**
+ * @internal
+ * @copydoc default_stride(matrix::Dense<ValueType>*)
+ */
 template <typename ValueType>
 default_stride_dense_wrapper<const device_type<ValueType>> default_stride(
     const matrix::Dense<ValueType> *mtx)
@@ -84,6 +105,12 @@ default_stride_dense_wrapper<const device_type<ValueType>> default_stride(
 }
 
 
+/**
+ * @internal
+ * Wraps the given matrix in a wrapper signifying that it is a row vector, i.e.
+ * we don't need to pass a stride parameter, but can access it directly as a
+ * pointer.
+ */
 template <typename ValueType>
 device_type<ValueType> *row_vector(matrix::Dense<ValueType> *mtx)
 {
@@ -91,7 +118,10 @@ device_type<ValueType> *row_vector(matrix::Dense<ValueType> *mtx)
     return as_device_type(mtx->get_values());
 }
 
-
+/**
+ * @internal
+ * @copydoc row_vector(matrix::Dense<ValueType>*)
+ */
 template <typename ValueType>
 const device_type<ValueType> *row_vector(const matrix::Dense<ValueType> *mtx)
 {

From 065a64830f58f476ffe3fdacdab197b56f74fb27 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 24 Jun 2021 12:33:57 +0200
Subject: [PATCH 0853/1008] move simple dense kernel tests to common tests

---
 cuda/test/matrix/dense_kernels.cpp    | 487 ------------------
 hip/test/matrix/dense_kernels.hip.cpp | 487 ------------------
 omp/test/matrix/dense_kernels.cpp     | 491 ------------------
 test/CMakeLists.txt                   |   1 +
 test/matrix/CMakeLists.txt            |   1 +
 test/matrix/dense_kernels.cpp         | 704 ++++++++++++++++++++++++++
 6 files changed, 706 insertions(+), 1465 deletions(-)
 create mode 100644 test/matrix/CMakeLists.txt
 create mode 100644 test/matrix/dense_kernels.cpp

diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index 23d394ad546..6e40ce5b5a3 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -191,146 +191,6 @@ class Dense : public ::testing::Test {
 };
 
 
-TEST_F(Dense, CudaCopyRespectsStride)
-{
-    set_up_vector_data(3);
-    auto stride = dx->get_size()[1] + 1;
-    auto result = Mtx::create(cuda, dx->get_size(), stride);
-    double val = 123456789.0;
-    auto original_data = result->get_values();
-    auto padding_ptr = original_data + dx->get_size()[1];
-    cuda->copy_from(ref.get(), 1, &val, padding_ptr);
-
-    dx->convert_to(result.get());
-
-    GKO_ASSERT_MTX_NEAR(result, dx, 0);
-    ASSERT_EQ(result->get_stride(), stride);
-    ASSERT_EQ(cuda->copy_val_to_host(padding_ptr), val);
-    ASSERT_EQ(result->get_values(), original_data);
-}
-
-
-TEST_F(Dense, CudaFillIsEquivalentToRef)
-{
-    set_up_vector_data(3);
-    auto result = Mtx::create(ref);
-
-    x->fill(42);
-    dx->fill(42);
-    result->copy_from(dx.get());
-
-    GKO_ASSERT_MTX_NEAR(result, x, 0);
-}
-
-
-TEST_F(Dense, CudaStridedFillIsEquivalentToRef)
-{
-    using T = double;
-    auto x = gko::initialize<gko::matrix::Dense<T>>(
-        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, ref);
-    auto dx = gko::initialize<gko::matrix::Dense<T>>(
-        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, cuda);
-    auto result = Mtx::create(ref);
-
-    x->fill(42);
-    dx->fill(42);
-    result->copy_from(dx.get());
-
-    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
-}
-
-
-TEST_F(Dense, SingleVectorCudaScaleIsEquivalentToRef)
-{
-    set_up_vector_data(1);
-    auto result = Mtx::create(ref);
-
-    x->scale(alpha.get());
-    dx->scale(dalpha.get());
-    result->copy_from(dx.get());
-
-    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
-}
-
-
-TEST_F(Dense, MultipleVectorCudaScaleIsEquivalentToRef)
-{
-    set_up_vector_data(20);
-
-    x->scale(alpha.get());
-    dx->scale(dalpha.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, MultipleVectorCudaScaleWithDifferentAlphaIsEquivalentToRef)
-{
-    set_up_vector_data(20, true);
-
-    x->scale(alpha.get());
-    dx->scale(dalpha.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, SingleVectorCudaAddScaledIsEquivalentToRef)
-{
-    set_up_vector_data(1);
-
-    x->add_scaled(alpha.get(), y.get());
-    dx->add_scaled(dalpha.get(), dy.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, MultipleVectorCudaAddScaledIsEquivalentToRef)
-{
-    set_up_vector_data(20);
-
-    x->add_scaled(alpha.get(), y.get());
-    dx->add_scaled(dalpha.get(), dy.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, MultipleVectorCudaAddScaledWithDifferentAlphaIsEquivalentToRef)
-{
-    set_up_vector_data(20);
-
-    x->add_scaled(alpha.get(), y.get());
-    dx->add_scaled(dalpha.get(), dy.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, AddsScaledDiagIsEquivalentToRef)
-{
-    auto mat = gen_mtx<Mtx>(532, 532);
-    gko::Array<Mtx::value_type> diag_values(ref, 532);
-    gko::kernels::reference::components::fill_array(ref, diag_values.get_data(),
-                                                    532, Mtx::value_type{2.0});
-    auto diag =
-        gko::matrix::Diagonal<Mtx::value_type>::create(ref, 532, diag_values);
-    alpha = gko::initialize<Mtx>({2.0}, ref);
-    auto dmat = Mtx::create(cuda);
-    dmat->copy_from(mat.get());
-    auto ddiag = gko::matrix::Diagonal<Mtx::value_type>::create(cuda);
-    ddiag->copy_from(diag.get());
-    dalpha = Mtx::create(cuda);
-    dalpha->copy_from(alpha.get());
-
-    mat->add_scaled(alpha.get(), diag.get());
-    dmat->add_scaled(dalpha.get(), ddiag.get());
-
-    GKO_ASSERT_MTX_NEAR(mat, dmat, 1e-14);
-}
-
-
 TEST_F(Dense, SingleVectorCudaComputeDotIsEquivalentToRef)
 {
     set_up_vector_data(1);
@@ -790,351 +650,4 @@ TEST_F(Dense, IsConjugateTransposableIntoDenseCrossExecutor)
 }
 
 
-TEST_F(Dense, CanGatherRows)
-{
-    set_up_apply_data();
-
-    auto r_gather = x->row_gather(rgather_idxs.get());
-    auto dr_gather = dx->row_gather(rgather_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
-}
-
-
-TEST_F(Dense, CanGatherRowsIntoDense)
-{
-    set_up_apply_data();
-    auto gather_size =
-        gko::dim<2>{rgather_idxs->get_num_elems(), x->get_size()[1]};
-    auto r_gather = Mtx::create(ref, gather_size);
-    // test make_temporary_clone and non-default stride
-    auto dr_gather = Mtx::create(ref, gather_size, x->get_size()[1] + 2);
-
-    x->row_gather(rgather_idxs.get(), r_gather.get());
-    dx->row_gather(rgather_idxs.get(), dr_gather.get());
-
-    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
-}
-
-
-TEST_F(Dense, IsPermutable)
-{
-    set_up_apply_data();
-
-    auto permuted = square->permute(rpermute_idxs.get());
-    auto dpermuted = dsquare->permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
-                        static_cast<Mtx *>(dpermuted.get()), 0);
-}
-
-
-TEST_F(Dense, IsPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, square->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted =
-        Mtx::create(ref, square->get_size(), square->get_size()[1] + 2);
-
-    square->permute(rpermute_idxs.get(), permuted.get());
-    dsquare->permute(rpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsInversePermutable)
-{
-    set_up_apply_data();
-
-    auto permuted = square->inverse_permute(rpermute_idxs.get());
-    auto dpermuted = dsquare->inverse_permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
-                        static_cast<Mtx *>(dpermuted.get()), 0);
-}
-
-
-TEST_F(Dense, IsInversePermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, square->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted =
-        Mtx::create(ref, square->get_size(), square->get_size()[1] + 2);
-
-    square->inverse_permute(rpermute_idxs.get(), permuted.get());
-    dsquare->inverse_permute(rpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsRowPermutable)
-{
-    set_up_apply_data();
-
-    auto r_permute = x->row_permute(rpermute_idxs.get());
-    auto dr_permute = dx->row_permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(r_permute.get()),
-                        static_cast<Mtx *>(dr_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsRowPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->row_permute(rpermute_idxs.get(), permuted.get());
-    dx->row_permute(rpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsColPermutable)
-{
-    set_up_apply_data();
-
-    auto c_permute = x->column_permute(cpermute_idxs.get());
-    auto dc_permute = dx->column_permute(cpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(c_permute.get()),
-                        static_cast<Mtx *>(dc_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsColPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->column_permute(cpermute_idxs.get(), permuted.get());
-    dx->column_permute(cpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsInverseRowPermutable)
-{
-    set_up_apply_data();
-
-    auto inverse_r_permute = x->inverse_row_permute(rpermute_idxs.get());
-    auto d_inverse_r_permute = dx->inverse_row_permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_r_permute.get()),
-                        static_cast<Mtx *>(d_inverse_r_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsInverseRowPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->inverse_row_permute(rpermute_idxs.get(), permuted.get());
-    dx->inverse_row_permute(rpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsInverseColPermutable)
-{
-    set_up_apply_data();
-
-    auto inverse_c_permute = x->inverse_column_permute(cpermute_idxs.get());
-    auto d_inverse_c_permute = dx->inverse_column_permute(cpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_c_permute.get()),
-                        static_cast<Mtx *>(d_inverse_c_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsInverseColPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->inverse_column_permute(cpermute_idxs.get(), permuted.get());
-    dx->inverse_column_permute(cpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto diag = x->extract_diagonal();
-    auto ddiag = dx->extract_diagonal();
-
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-}
-
-
-TEST_F(Dense, ExtractDiagonalOnTallSkinnyIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto diag = Diagonal::create(ref, x->get_size()[1]);
-    // test make_temporary_clone
-    auto ddiag = Diagonal::create(ref, x->get_size()[1]);
-
-    x->extract_diagonal(diag.get());
-    dx->extract_diagonal(ddiag.get());
-
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-}
-
-
-TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto diag = y->extract_diagonal();
-    auto ddiag = dy->extract_diagonal();
-
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-}
-
-
-TEST_F(Dense, ExtractDiagonalOnShortFatIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto diag = Diagonal::create(ref, y->get_size()[0]);
-    // test make_temporary_clone
-    auto ddiag = Diagonal::create(ref, y->get_size()[0]);
-
-    y->extract_diagonal(diag.get());
-    dy->extract_diagonal(ddiag.get());
-
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-}
-
-
-TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    x->compute_absolute_inplace();
-    dx->compute_absolute_inplace();
-
-    GKO_ASSERT_MTX_NEAR(x, dx, 1e-14);
-}
-
-
-TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto abs_x = x->compute_absolute();
-    auto dabs_x = dx->compute_absolute();
-
-    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, 1e-14);
-}
-
-
-TEST_F(Dense, OutplaceAbsoluteMatrixIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto abs_x = NormVector::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dabs_x = NormVector::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->compute_absolute(abs_x.get());
-    dx->compute_absolute(dabs_x.get());
-
-    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, 1e-14);
-}
-
-
-TEST_F(Dense, MakeComplexIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto complex_x = x->make_complex();
-    auto dcomplex_x = dx->make_complex();
-
-    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
-}
-
-
-TEST_F(Dense, MakeComplexIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto complex_x = ComplexMtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dcomplex_x =
-        ComplexMtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->make_complex(complex_x.get());
-    dx->make_complex(dcomplex_x.get());
-
-    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
-}
-
-
-TEST_F(Dense, GetRealIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto real_x = x->get_real();
-    auto dreal_x = dx->get_real();
-
-    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
-}
-
-
-TEST_F(Dense, GetRealIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto real_x = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dreal_x = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->get_real(real_x.get());
-    dx->get_real(dreal_x.get());
-
-    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
-}
-
-
-TEST_F(Dense, GetImagIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto imag_x = x->get_imag();
-    auto dimag_x = dx->get_imag();
-
-    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
-}
-
-
-TEST_F(Dense, GetImagIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto imag_x = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dimag_x = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->get_imag(imag_x.get());
-    dx->get_imag(dimag_x.get());
-
-    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
-}
-
-
 }  // namespace
diff --git a/hip/test/matrix/dense_kernels.hip.cpp b/hip/test/matrix/dense_kernels.hip.cpp
index 18c5a6dc0b5..b736dc30117 100644
--- a/hip/test/matrix/dense_kernels.hip.cpp
+++ b/hip/test/matrix/dense_kernels.hip.cpp
@@ -186,146 +186,6 @@ class Dense : public ::testing::Test {
 };
 
 
-TEST_F(Dense, HipCopyRespectsStride)
-{
-    set_up_vector_data(3);
-    auto stride = dx->get_size()[1] + 1;
-    auto result = Mtx::create(hip, dx->get_size(), stride);
-    double val = 123456789.0;
-    auto original_data = result->get_values();
-    auto padding_ptr = original_data + dx->get_size()[1];
-    hip->copy_from(ref.get(), 1, &val, padding_ptr);
-
-    dx->convert_to(result.get());
-
-    GKO_ASSERT_MTX_NEAR(result, dx, 0);
-    ASSERT_EQ(result->get_stride(), stride);
-    ASSERT_EQ(hip->copy_val_to_host(padding_ptr), val);
-    ASSERT_EQ(result->get_values(), original_data);
-}
-
-
-TEST_F(Dense, HipFillIsEquivalentToRef)
-{
-    set_up_vector_data(3);
-    auto result = Mtx::create(ref);
-
-    x->fill(42);
-    dx->fill(42);
-    result->copy_from(dx.get());
-
-    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
-}
-
-
-TEST_F(Dense, HipStridedFillIsEquivalentToRef)
-{
-    using T = double;
-    auto x = gko::initialize<gko::matrix::Dense<T>>(
-        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, ref);
-    auto dx = gko::initialize<gko::matrix::Dense<T>>(
-        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, hip);
-    auto result = Mtx::create(ref);
-
-    x->fill(42);
-    dx->fill(42);
-    result->copy_from(dx.get());
-
-    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
-}
-
-
-TEST_F(Dense, SingleVectorHipScaleIsEquivalentToRef)
-{
-    set_up_vector_data(1);
-    auto result = Mtx::create(ref);
-
-    x->scale(alpha.get());
-    dx->scale(dalpha.get());
-    result->copy_from(dx.get());
-
-    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
-}
-
-
-TEST_F(Dense, MultipleVectorHipScaleIsEquivalentToRef)
-{
-    set_up_vector_data(20);
-
-    x->scale(alpha.get());
-    dx->scale(dalpha.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, MultipleVectorHipScaleWithDifferentAlphaIsEquivalentToRef)
-{
-    set_up_vector_data(20, true);
-
-    x->scale(alpha.get());
-    dx->scale(dalpha.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, SingleVectorHipAddScaledIsEquivalentToRef)
-{
-    set_up_vector_data(1);
-
-    x->add_scaled(alpha.get(), y.get());
-    dx->add_scaled(dalpha.get(), dy.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, MultipleVectorHipAddScaledIsEquivalentToRef)
-{
-    set_up_vector_data(20);
-
-    x->add_scaled(alpha.get(), y.get());
-    dx->add_scaled(dalpha.get(), dy.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, MultipleVectorHipAddScaledWithDifferentAlphaIsEquivalentToRef)
-{
-    set_up_vector_data(20);
-
-    x->add_scaled(alpha.get(), y.get());
-    dx->add_scaled(dalpha.get(), dy.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, AddsScaledDiagIsEquivalentToRef)
-{
-    auto mat = gen_mtx<Mtx>(532, 532);
-    gko::Array<Mtx::value_type> diag_values(ref, 532);
-    gko::kernels::reference::components::fill_array(ref, diag_values.get_data(),
-                                                    532, Mtx::value_type{2.0});
-    auto diag =
-        gko::matrix::Diagonal<Mtx::value_type>::create(ref, 532, diag_values);
-    alpha = gko::initialize<Mtx>({2.0}, ref);
-    auto dmat = Mtx::create(hip);
-    dmat->copy_from(mat.get());
-    auto ddiag = gko::matrix::Diagonal<Mtx::value_type>::create(hip);
-    ddiag->copy_from(diag.get());
-    dalpha = Mtx::create(hip);
-    dalpha->copy_from(alpha.get());
-
-    mat->add_scaled(alpha.get(), diag.get());
-    dmat->add_scaled(dalpha.get(), ddiag.get());
-
-    GKO_ASSERT_MTX_NEAR(mat, dmat, 1e-14);
-}
-
-
 TEST_F(Dense, SingleVectorHipComputeDotIsEquivalentToRef)
 {
     set_up_vector_data(1);
@@ -753,351 +613,4 @@ TEST_F(Dense, IsTransposableIntoDenseCrossExecutor)
 }
 
 
-TEST_F(Dense, CanGatherRows)
-{
-    set_up_apply_data();
-
-    auto r_gather = x->row_gather(rgather_idxs.get());
-    auto dr_gather = dx->row_gather(rgather_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
-}
-
-
-TEST_F(Dense, CanGatherRowsIntoDense)
-{
-    set_up_apply_data();
-    auto gather_size =
-        gko::dim<2>{rgather_idxs->get_num_elems(), x->get_size()[1]};
-    auto r_gather = Mtx::create(ref, gather_size);
-    // test make_temporary_clone and non-default stride
-    auto dr_gather = Mtx::create(ref, gather_size, x->get_size()[1] + 2);
-
-    x->row_gather(rgather_idxs.get(), r_gather.get());
-    dx->row_gather(rgather_idxs.get(), dr_gather.get());
-
-    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
-}
-
-
-TEST_F(Dense, IsPermutable)
-{
-    set_up_apply_data();
-
-    auto permuted = square->permute(rpermute_idxs.get());
-    auto dpermuted = dsquare->permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
-                        static_cast<Mtx *>(dpermuted.get()), 0);
-}
-
-
-TEST_F(Dense, IsPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, square->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted =
-        Mtx::create(ref, square->get_size(), square->get_size()[1] + 2);
-
-    square->permute(rpermute_idxs.get(), permuted.get());
-    dsquare->permute(rpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsInversePermutable)
-{
-    set_up_apply_data();
-
-    auto permuted = square->inverse_permute(rpermute_idxs.get());
-    auto dpermuted = dsquare->inverse_permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
-                        static_cast<Mtx *>(dpermuted.get()), 0);
-}
-
-
-TEST_F(Dense, IsInversePermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, square->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted =
-        Mtx::create(ref, square->get_size(), square->get_size()[1] + 2);
-
-    square->inverse_permute(rpermute_idxs.get(), permuted.get());
-    dsquare->inverse_permute(rpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsRowPermutable)
-{
-    set_up_apply_data();
-
-    auto r_permute = x->row_permute(rpermute_idxs.get());
-    auto dr_permute = dx->row_permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(r_permute.get()),
-                        static_cast<Mtx *>(dr_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsRowPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->row_permute(rpermute_idxs.get(), permuted.get());
-    dx->row_permute(rpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsColPermutable)
-{
-    set_up_apply_data();
-
-    auto c_permute = x->column_permute(cpermute_idxs.get());
-    auto dc_permute = dx->column_permute(cpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(c_permute.get()),
-                        static_cast<Mtx *>(dc_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsColPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->column_permute(cpermute_idxs.get(), permuted.get());
-    dx->column_permute(cpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsInverseRowPermutable)
-{
-    set_up_apply_data();
-
-    auto inverse_r_permute = x->inverse_row_permute(rpermute_idxs.get());
-    auto d_inverse_r_permute = dx->inverse_row_permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_r_permute.get()),
-                        static_cast<Mtx *>(d_inverse_r_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsInverseRowPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->inverse_row_permute(rpermute_idxs.get(), permuted.get());
-    dx->inverse_row_permute(rpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsInverseColPermutable)
-{
-    set_up_apply_data();
-
-    auto inverse_c_permute = x->inverse_column_permute(cpermute_idxs.get());
-    auto d_inverse_c_permute = dx->inverse_column_permute(cpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_c_permute.get()),
-                        static_cast<Mtx *>(d_inverse_c_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsInverseColPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->inverse_column_permute(cpermute_idxs.get(), permuted.get());
-    dx->inverse_column_permute(cpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto diag = x->extract_diagonal();
-    auto ddiag = dx->extract_diagonal();
-
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-}
-
-
-TEST_F(Dense, ExtractDiagonalOnTallSkinnyIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto diag = Diagonal::create(ref, x->get_size()[1]);
-    // test make_temporary_clone
-    auto ddiag = Diagonal::create(ref, x->get_size()[1]);
-
-    x->extract_diagonal(diag.get());
-    dx->extract_diagonal(ddiag.get());
-
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-}
-
-
-TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto diag = y->extract_diagonal();
-    auto ddiag = dy->extract_diagonal();
-
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-}
-
-
-TEST_F(Dense, ExtractDiagonalOnShortFatIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto diag = Diagonal::create(ref, y->get_size()[0]);
-    // test make_temporary_clone
-    auto ddiag = Diagonal::create(ref, y->get_size()[0]);
-
-    y->extract_diagonal(diag.get());
-    dy->extract_diagonal(ddiag.get());
-
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-}
-
-
-TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    x->compute_absolute_inplace();
-    dx->compute_absolute_inplace();
-
-    GKO_ASSERT_MTX_NEAR(x, dx, 1e-14);
-}
-
-
-TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto abs_x = x->compute_absolute();
-    auto dabs_x = dx->compute_absolute();
-
-    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, 1e-14);
-}
-
-
-TEST_F(Dense, OutplaceAbsoluteMatrixIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto abs_x = NormVector::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dabs_x = NormVector::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->compute_absolute(abs_x.get());
-    dx->compute_absolute(dabs_x.get());
-
-    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, 1e-14);
-}
-
-
-TEST_F(Dense, MakeComplexIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto complex_x = x->make_complex();
-    auto dcomplex_x = dx->make_complex();
-
-    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
-}
-
-
-TEST_F(Dense, MakeComplexIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto complex_x = ComplexMtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dcomplex_x =
-        ComplexMtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->make_complex(complex_x.get());
-    dx->make_complex(dcomplex_x.get());
-
-    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
-}
-
-
-TEST_F(Dense, GetRealIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto real_x = x->get_real();
-    auto dreal_x = dx->get_real();
-
-    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
-}
-
-
-TEST_F(Dense, GetRealIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto real_x = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dreal_x = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->get_real(real_x.get());
-    dx->get_real(dreal_x.get());
-
-    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
-}
-
-
-TEST_F(Dense, GetImagIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto imag_x = x->get_imag();
-    auto dimag_x = dx->get_imag();
-
-    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
-}
-
-
-TEST_F(Dense, GetImagIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto imag_x = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dimag_x = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->get_imag(imag_x.get());
-    dx->get_imag(dimag_x.get());
-
-    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
-}
-
-
 }  // namespace
diff --git a/omp/test/matrix/dense_kernels.cpp b/omp/test/matrix/dense_kernels.cpp
index 8ade4b939af..42b1151e5e3 100644
--- a/omp/test/matrix/dense_kernels.cpp
+++ b/omp/test/matrix/dense_kernels.cpp
@@ -205,146 +205,6 @@ class Dense : public ::testing::Test {
 };
 
 
-TEST_F(Dense, OmpCopyRespectsStride)
-{
-    set_up_vector_data(3);
-    auto stride = dx->get_size()[1] + 1;
-    auto result = Mtx::create(omp, dx->get_size(), stride);
-    double val = 123456789.0;
-    auto original_data = result->get_values();
-    auto padding_ptr = original_data + dx->get_size()[1];
-    omp->copy_from(ref.get(), 1, &val, padding_ptr);
-
-    dx->convert_to(result.get());
-
-    GKO_ASSERT_MTX_NEAR(result, dx, 0);
-    ASSERT_EQ(result->get_stride(), stride);
-    ASSERT_EQ(omp->copy_val_to_host(padding_ptr), val);
-    ASSERT_EQ(result->get_values(), original_data);
-}
-
-
-TEST_F(Dense, OmpFillIsEquivalentToRef)
-{
-    set_up_vector_data(3);
-    auto result = Mtx::create(ref);
-
-    x->fill(42);
-    dx->fill(42);
-    result->copy_from(dx.get());
-
-    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
-}
-
-
-TEST_F(Dense, OmpStridedFillIsEquivalentToRef)
-{
-    using T = double;
-    auto x = gko::initialize<gko::matrix::Dense<T>>(
-        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, ref);
-    auto dx = gko::initialize<gko::matrix::Dense<T>>(
-        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, omp);
-    auto result = Mtx::create(ref);
-
-    x->fill(42);
-    dx->fill(42);
-    result->copy_from(dx.get());
-
-    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
-}
-
-
-TEST_F(Dense, SingleVectorOmpScaleIsEquivalentToRef)
-{
-    set_up_vector_data(1);
-
-    x->scale(alpha.get());
-    dx->scale(dalpha.get());
-
-    auto result = Mtx::create(ref);
-    result->copy_from(dx.get());
-    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
-}
-
-
-TEST_F(Dense, MultipleVectorOmpScaleIsEquivalentToRef)
-{
-    set_up_vector_data(20);
-
-    x->scale(alpha.get());
-    dx->scale(dalpha.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, MultipleVectorOmpScaleWithDifferentAlphaIsEquivalentToRef)
-{
-    set_up_vector_data(20, true);
-
-    x->scale(alpha.get());
-    dx->scale(dalpha.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, SingleVectorOmpAddScaledIsEquivalentToRef)
-{
-    set_up_vector_data(1);
-
-    x->add_scaled(alpha.get(), y.get());
-    dx->add_scaled(dalpha.get(), dy.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, MultipleVectorOmpAddScaledIsEquivalentToRef)
-{
-    set_up_vector_data(20);
-
-    x->add_scaled(alpha.get(), y.get());
-    dx->add_scaled(dalpha.get(), dy.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, MultipleVectorOmpAddScaledWithDifferentAlphaIsEquivalentToRef)
-{
-    set_up_vector_data(20);
-
-    x->add_scaled(alpha.get(), y.get());
-    dx->add_scaled(dalpha.get(), dy.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
-}
-
-
-TEST_F(Dense, AddsScaledDiagIsEquivalentToRef)
-{
-    auto mat = gen_mtx<Mtx>(532, 532);
-    gko::Array<Mtx::value_type> diag_values(this->ref, 532);
-    gko::kernels::reference::components::fill_array(
-        this->ref, diag_values.get_data(), 532, Mtx::value_type{2.0});
-    auto diag = gko::matrix::Diagonal<Mtx::value_type>::create(this->ref, 532,
-                                                               diag_values);
-    auto alpha = gko::initialize<Mtx>({2.0}, this->ref);
-    auto dmat = Mtx::create(this->omp);
-    dmat->copy_from(mat.get());
-    auto ddiag = gko::matrix::Diagonal<Mtx::value_type>::create(this->omp);
-    ddiag->copy_from(diag.get());
-    auto dalpha = Mtx::create(this->omp);
-    dalpha->copy_from(alpha.get());
-
-    mat->add_scaled(alpha.get(), diag.get());
-    dmat->add_scaled(dalpha.get(), ddiag.get());
-
-    GKO_ASSERT_MTX_NEAR(mat, dmat, 1e-14);
-}
-
-
 TEST_F(Dense, SingleVectorOmpComputeDotIsEquivalentToRef)
 {
     set_up_vector_data(1);
@@ -928,355 +788,4 @@ TEST_F(Dense, IsConjugateTransposableIntoDenseCrossExecutor)
 }
 
 
-TEST_F(Dense, CanGatherRows)
-{
-    set_up_apply_data();
-
-    auto r_gather = x->row_gather(rgather_idxs.get());
-    auto dr_gather = dx->row_gather(rgather_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
-}
-
-
-TEST_F(Dense, CanGatherRowsIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto row_span = gko::span{0, x->get_size()[0]};
-    auto col_span = gko::span{0, x->get_size()[1] - 2};
-    auto sub_x = x->create_submatrix(row_span, col_span);
-    auto sub_dx = dx->create_submatrix(row_span, col_span);
-    auto gather_size =
-        gko::dim<2>{rgather_idxs->get_num_elems(), sub_x->get_size()[1]};
-    auto r_gather = Mtx::create(ref, gather_size);
-    // test make_temporary_clone and non-default stride
-    auto dr_gather = Mtx::create(ref, gather_size, sub_x->get_size()[1] + 2);
-
-    sub_x->row_gather(rgather_idxs.get(), r_gather.get());
-    sub_dx->row_gather(rgather_idxs.get(), dr_gather.get());
-
-    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
-}
-
-
-TEST_F(Dense, IsPermutable)
-{
-    set_up_apply_data();
-
-    auto permuted = square->permute(rpermute_idxs.get());
-    auto dpermuted = dsquare->permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
-                        static_cast<Mtx *>(dpermuted.get()), 0);
-}
-
-
-TEST_F(Dense, IsPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, square->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted =
-        Mtx::create(ref, square->get_size(), square->get_size()[1] + 2);
-
-    square->permute(rpermute_idxs.get(), permuted.get());
-    dsquare->permute(rpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsInversePermutable)
-{
-    set_up_apply_data();
-
-    auto permuted = square->inverse_permute(rpermute_idxs.get());
-    auto dpermuted = dsquare->inverse_permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
-                        static_cast<Mtx *>(dpermuted.get()), 0);
-}
-
-
-TEST_F(Dense, IsInversePermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, square->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted =
-        Mtx::create(ref, square->get_size(), square->get_size()[1] + 2);
-
-    square->inverse_permute(rpermute_idxs.get(), permuted.get());
-    dsquare->inverse_permute(rpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsRowPermutable)
-{
-    set_up_apply_data();
-
-    auto r_permute = x->row_permute(rpermute_idxs.get());
-    auto dr_permute = dx->row_permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(r_permute.get()),
-                        static_cast<Mtx *>(dr_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsRowPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->row_permute(rpermute_idxs.get(), permuted.get());
-    dx->row_permute(rpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsColPermutable)
-{
-    set_up_apply_data();
-
-    auto c_permute = x->column_permute(cpermute_idxs.get());
-    auto dc_permute = dx->column_permute(cpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(c_permute.get()),
-                        static_cast<Mtx *>(dc_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsColPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->column_permute(cpermute_idxs.get(), permuted.get());
-    dx->column_permute(cpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsInverseRowPermutable)
-{
-    set_up_apply_data();
-
-    auto inverse_r_permute = x->inverse_row_permute(rpermute_idxs.get());
-    auto d_inverse_r_permute = dx->inverse_row_permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_r_permute.get()),
-                        static_cast<Mtx *>(d_inverse_r_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsInverseRowPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->inverse_row_permute(rpermute_idxs.get(), permuted.get());
-    dx->inverse_row_permute(rpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, IsInverseColPermutable)
-{
-    set_up_apply_data();
-
-    auto inverse_c_permute = x->inverse_column_permute(cpermute_idxs.get());
-    auto d_inverse_c_permute = dx->inverse_column_permute(cpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_c_permute.get()),
-                        static_cast<Mtx *>(d_inverse_c_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsInverseColPermutableIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto permuted = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->inverse_column_permute(cpermute_idxs.get(), permuted.get());
-    dx->inverse_column_permute(cpermute_idxs.get(), dpermuted.get());
-
-    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
-}
-
-
-TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto diag = x->extract_diagonal();
-    auto ddiag = dx->extract_diagonal();
-
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-}
-
-
-TEST_F(Dense, ExtractDiagonalOnTallSkinnyIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto diag = Diagonal::create(ref, x->get_size()[1]);
-    // test make_temporary_clone
-    auto ddiag = Diagonal::create(ref, x->get_size()[1]);
-
-    x->extract_diagonal(diag.get());
-    dx->extract_diagonal(ddiag.get());
-
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-}
-
-
-TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto diag = y->extract_diagonal();
-    auto ddiag = dy->extract_diagonal();
-
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-}
-
-
-TEST_F(Dense, ExtractDiagonalOnShortFatIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto diag = Diagonal::create(ref, y->get_size()[0]);
-    // test make_temporary_clone
-    auto ddiag = Diagonal::create(ref, y->get_size()[0]);
-
-    y->extract_diagonal(diag.get());
-    dy->extract_diagonal(ddiag.get());
-
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-}
-
-
-TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    x->compute_absolute_inplace();
-    dx->compute_absolute_inplace();
-
-    GKO_ASSERT_MTX_NEAR(x, dx, 1e-14);
-}
-
-
-TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto abs_x = x->compute_absolute();
-    auto dabs_x = dx->compute_absolute();
-
-    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, 1e-14);
-}
-
-
-TEST_F(Dense, OutplaceAbsoluteMatrixIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto abs_x = NormVector::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dabs_x = NormVector::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->compute_absolute(abs_x.get());
-    dx->compute_absolute(dabs_x.get());
-
-    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, 1e-14);
-}
-
-
-TEST_F(Dense, MakeComplexIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto complex_x = x->make_complex();
-    auto dcomplex_x = dx->make_complex();
-
-    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
-}
-
-
-TEST_F(Dense, MakeComplexIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto complex_x = ComplexMtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dcomplex_x =
-        ComplexMtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->make_complex(complex_x.get());
-    dx->make_complex(dcomplex_x.get());
-
-    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
-}
-
-
-TEST_F(Dense, GetRealIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto real_x = x->get_real();
-    auto dreal_x = dx->get_real();
-
-    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
-}
-
-
-TEST_F(Dense, GetRealIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto real_x = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dreal_x = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->get_real(real_x.get());
-    dx->get_real(dreal_x.get());
-
-    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
-}
-
-
-TEST_F(Dense, GetImagIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto imag_x = x->get_imag();
-    auto dimag_x = dx->get_imag();
-
-    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
-}
-
-
-TEST_F(Dense, GetImagIntoDenseCrossExecutor)
-{
-    set_up_apply_data();
-    auto imag_x = Mtx::create(ref, x->get_size());
-    // test make_temporary_clone and non-default stride
-    auto dimag_x = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
-
-    x->get_imag(imag_x.get());
-    dx->get_imag(dimag_x.get());
-
-    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
-}
-
-
 }  // namespace
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 8a32b18862d..c52ac51d956 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -1,3 +1,4 @@
 include(${PROJECT_SOURCE_DIR}/cmake/create_test.cmake)
 
+add_subdirectory(matrix)
 add_subdirectory(solver)
\ No newline at end of file
diff --git a/test/matrix/CMakeLists.txt b/test/matrix/CMakeLists.txt
new file mode 100644
index 00000000000..45bd0ec364d
--- /dev/null
+++ b/test/matrix/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_common_test(dense_kernels)
\ No newline at end of file
diff --git a/test/matrix/dense_kernels.cpp b/test/matrix/dense_kernels.cpp
new file mode 100644
index 00000000000..d699dd93f17
--- /dev/null
+++ b/test/matrix/dense_kernels.cpp
@@ -0,0 +1,704 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+#include <ginkgo/core/matrix/ell.hpp>
+#include <ginkgo/core/matrix/hybrid.hpp>
+#include <ginkgo/core/matrix/sellp.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
+
+
+#include "core/components/fill_array.hpp"
+#include "core/matrix/dense_kernels.hpp"
+#include "core/test/utils.hpp"
+#include "test/utils/executor.hpp"
+
+
+namespace {
+
+
+class Dense : public ::testing::Test {
+protected:
+    using itype = int;
+#if GINKGO_COMMON_SINGLE_MODE
+    using vtype = float;
+#else
+    using vtype = double;
+#endif
+    using Mtx = gko::matrix::Dense<vtype>;
+    using MixedMtx = gko::matrix::Dense<gko::next_precision<vtype>>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<vtype>>;
+    using Arr = gko::Array<itype>;
+    using ComplexMtx = gko::matrix::Dense<std::complex<vtype>>;
+    using Diagonal = gko::matrix::Diagonal<vtype>;
+    using MixedComplexMtx =
+        gko::matrix::Dense<gko::next_precision<std::complex<vtype>>>;
+
+    Dense() : rand_engine(15) {}
+
+    void SetUp()
+    {
+        ref = gko::ReferenceExecutor::create();
+        init_executor(ref, exec);
+    }
+
+    void TearDown()
+    {
+        if (exec != nullptr) {
+            ASSERT_NO_THROW(exec->synchronize());
+        }
+    }
+
+    template <typename MtxType>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<MtxType>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(num_cols, num_cols),
+            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
+    }
+
+    template <typename MtxType>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row)
+    {
+        return gko::test::generate_random_matrix<MtxType>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(min_nnz_row, num_cols),
+            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    void set_up_vector_data(gko::size_type num_vecs,
+                            bool different_alpha = false)
+    {
+        x = gen_mtx<Mtx>(1000, num_vecs);
+        y = gen_mtx<Mtx>(1000, num_vecs);
+        if (different_alpha) {
+            alpha = gen_mtx<Mtx>(1, num_vecs);
+        } else {
+            alpha = gko::initialize<Mtx>({2.0}, ref);
+        }
+        dx = Mtx::create(exec);
+        dx->copy_from(x.get());
+        dy = Mtx::create(exec);
+        dy->copy_from(y.get());
+        dalpha = Mtx::create(exec);
+        dalpha->copy_from(alpha.get());
+        expected = Mtx::create(ref, gko::dim<2>{1, num_vecs});
+        dresult = Mtx::create(exec, gko::dim<2>{1, num_vecs});
+    }
+
+    void set_up_apply_data()
+    {
+        x = gen_mtx<Mtx>(40, 25);
+        c_x = gen_mtx<ComplexMtx>(40, 25);
+        y = gen_mtx<Mtx>(25, 35);
+        expected = gen_mtx<Mtx>(40, 35);
+        alpha = gko::initialize<Mtx>({2.0}, ref);
+        beta = gko::initialize<Mtx>({-1.0}, ref);
+        square = gen_mtx<Mtx>(x->get_size()[0], x->get_size()[0]);
+        dx = Mtx::create(exec);
+        dx->copy_from(x.get());
+        dc_x = ComplexMtx::create(exec);
+        dc_x->copy_from(c_x.get());
+        dy = Mtx::create(exec);
+        dy->copy_from(y.get());
+        dresult = Mtx::create(exec);
+        dresult->copy_from(expected.get());
+        dalpha = Mtx::create(exec);
+        dalpha->copy_from(alpha.get());
+        dbeta = Mtx::create(exec);
+        dbeta->copy_from(beta.get());
+        dsquare = Mtx::create(exec);
+        dsquare->copy_from(square.get());
+
+        std::vector<int> tmp(x->get_size()[0], 0);
+        auto rng = std::default_random_engine{};
+        std::iota(tmp.begin(), tmp.end(), 0);
+        std::shuffle(tmp.begin(), tmp.end(), rng);
+        std::vector<int> tmp2(x->get_size()[1], 0);
+        std::iota(tmp2.begin(), tmp2.end(), 0);
+        std::shuffle(tmp2.begin(), tmp2.end(), rng);
+        std::vector<int> tmp3(x->get_size()[0] / 10);
+        std::uniform_int_distribution<int> row_dist(0, x->get_size()[0] - 1);
+        for (auto &i : tmp3) {
+            i = row_dist(rng);
+        }
+        rpermute_idxs =
+            std::unique_ptr<Arr>(new Arr{ref, tmp.begin(), tmp.end()});
+        cpermute_idxs =
+            std::unique_ptr<Arr>(new Arr{ref, tmp2.begin(), tmp2.end()});
+        rgather_idxs =
+            std::unique_ptr<Arr>(new Arr{ref, tmp3.begin(), tmp3.end()});
+    }
+
+    template <typename ConvertedType, typename InputType>
+    std::unique_ptr<ConvertedType> convert(InputType &&input)
+    {
+        auto result = ConvertedType::create(input->get_executor());
+        input->convert_to(result.get());
+        return result;
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<gko::EXEC_TYPE> exec;
+
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Mtx> x;
+    std::unique_ptr<ComplexMtx> c_x;
+    std::unique_ptr<Mtx> y;
+    std::unique_ptr<Mtx> alpha;
+    std::unique_ptr<Mtx> beta;
+    std::unique_ptr<Mtx> expected;
+    std::unique_ptr<Mtx> square;
+    std::unique_ptr<Mtx> dresult;
+    std::unique_ptr<Mtx> dx;
+    std::unique_ptr<ComplexMtx> dc_x;
+    std::unique_ptr<Mtx> dy;
+    std::unique_ptr<Mtx> dalpha;
+    std::unique_ptr<Mtx> dbeta;
+    std::unique_ptr<Mtx> dsquare;
+    std::unique_ptr<Arr> rpermute_idxs;
+    std::unique_ptr<Arr> cpermute_idxs;
+    std::unique_ptr<Arr> rgather_idxs;
+};
+
+
+TEST_F(Dense, CopyRespectsStride)
+{
+    set_up_vector_data(3);
+    auto stride = dx->get_size()[1] + 1;
+    auto result = Mtx::create(exec, dx->get_size(), stride);
+    double val = 123456789.0;
+    auto original_data = result->get_values();
+    auto padding_ptr = original_data + dx->get_size()[1];
+    exec->copy_from(ref.get(), 1, &val, padding_ptr);
+
+    dx->convert_to(result.get());
+
+    GKO_ASSERT_MTX_NEAR(result, dx, 0);
+    ASSERT_EQ(result->get_stride(), stride);
+    ASSERT_EQ(exec->copy_val_to_host(padding_ptr), val);
+    ASSERT_EQ(result->get_values(), original_data);
+}
+
+
+TEST_F(Dense, FillIsEquivalentToRef)
+{
+    set_up_vector_data(3);
+    auto result = Mtx::create(ref);
+
+    x->fill(42);
+    dx->fill(42);
+    result->copy_from(dx.get());
+
+    GKO_ASSERT_MTX_NEAR(result, x, 0);
+}
+
+
+TEST_F(Dense, StridedFillIsEquivalentToRef)
+{
+    using T = double;
+    auto x = gko::initialize<gko::matrix::Dense<T>>(
+        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, ref);
+    auto dx = gko::initialize<gko::matrix::Dense<T>>(
+        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, exec);
+    auto result = Mtx::create(ref);
+
+    x->fill(42);
+    dx->fill(42);
+    result->copy_from(dx.get());
+
+    GKO_ASSERT_MTX_NEAR(result, x, 0);
+}
+
+
+TEST_F(Dense, SingleVectorScaleIsEquivalentToRef)
+{
+    set_up_vector_data(1);
+
+    x->scale(alpha.get());
+    dx->scale(dalpha.get());
+
+    auto result = Mtx::create(ref);
+    result->copy_from(dx.get());
+    GKO_ASSERT_MTX_NEAR(result, x, r<vtype>::value);
+}
+
+
+TEST_F(Dense, MultipleVectorScaleIsEquivalentToRef)
+{
+    set_up_vector_data(20);
+
+    x->scale(alpha.get());
+    dx->scale(dalpha.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
+}
+
+
+TEST_F(Dense, MultipleVectorScaleWithDifferentAlphaIsEquivalentToRef)
+{
+    set_up_vector_data(20, true);
+
+    x->scale(alpha.get());
+    dx->scale(dalpha.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
+}
+
+
+TEST_F(Dense, SingleVectorAddScaledIsEquivalentToRef)
+{
+    set_up_vector_data(1);
+
+    x->add_scaled(alpha.get(), y.get());
+    dx->add_scaled(dalpha.get(), dy.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
+}
+
+
+TEST_F(Dense, MultipleVectorAddScaledIsEquivalentToRef)
+{
+    set_up_vector_data(20);
+
+    x->add_scaled(alpha.get(), y.get());
+    dx->add_scaled(dalpha.get(), dy.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
+}
+
+
+TEST_F(Dense, MultipleVectorAddScaledWithDifferentAlphaIsEquivalentToRef)
+{
+    set_up_vector_data(20);
+
+    x->add_scaled(alpha.get(), y.get());
+    dx->add_scaled(dalpha.get(), dy.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
+}
+
+
+TEST_F(Dense, AddsScaledDiagIsEquivalentToRef)
+{
+    auto mat = gen_mtx<Mtx>(532, 532);
+    gko::Array<Mtx::value_type> diag_values(this->ref, 532);
+    gko::kernels::reference::components::fill_array(
+        this->ref, diag_values.get_data(), 532, Mtx::value_type{2.0});
+    auto diag = gko::matrix::Diagonal<Mtx::value_type>::create(this->ref, 532,
+                                                               diag_values);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->ref);
+    auto dmat = Mtx::create(this->exec);
+    dmat->copy_from(mat.get());
+    auto ddiag = gko::matrix::Diagonal<Mtx::value_type>::create(this->exec);
+    ddiag->copy_from(diag.get());
+    auto dalpha = Mtx::create(this->exec);
+    dalpha->copy_from(alpha.get());
+
+    mat->add_scaled(alpha.get(), diag.get());
+    dmat->add_scaled(dalpha.get(), ddiag.get());
+
+    GKO_ASSERT_MTX_NEAR(mat, dmat, r<vtype>::value);
+}
+
+
+TEST_F(Dense, CanGatherRows)
+{
+    set_up_apply_data();
+
+    auto r_gather = x->row_gather(rgather_idxs.get());
+    auto dr_gather = dx->row_gather(rgather_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
+}
+
+
+TEST_F(Dense, CanGatherRowsIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto row_span = gko::span{0, x->get_size()[0]};
+    auto col_span = gko::span{0, x->get_size()[1] - 2};
+    auto sub_x = x->create_submatrix(row_span, col_span);
+    auto sub_dx = dx->create_submatrix(row_span, col_span);
+    auto gather_size =
+        gko::dim<2>{rgather_idxs->get_num_elems(), sub_x->get_size()[1]};
+    auto r_gather = Mtx::create(ref, gather_size);
+    // test make_temporary_clone and non-default stride
+    auto dr_gather = Mtx::create(ref, gather_size, sub_x->get_size()[1] + 2);
+
+    sub_x->row_gather(rgather_idxs.get(), r_gather.get());
+    sub_dx->row_gather(rgather_idxs.get(), dr_gather.get());
+
+    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
+}
+
+
+TEST_F(Dense, IsPermutable)
+{
+    set_up_apply_data();
+
+    auto permuted = square->permute(rpermute_idxs.get());
+    auto dpermuted = dsquare->permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
+                        static_cast<Mtx *>(dpermuted.get()), 0);
+}
+
+
+TEST_F(Dense, IsPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, square->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted =
+        Mtx::create(ref, square->get_size(), square->get_size()[1] + 2);
+
+    square->permute(rpermute_idxs.get(), permuted.get());
+    dsquare->permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
+TEST_F(Dense, IsInversePermutable)
+{
+    set_up_apply_data();
+
+    auto permuted = square->inverse_permute(rpermute_idxs.get());
+    auto dpermuted = dsquare->inverse_permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
+                        static_cast<Mtx *>(dpermuted.get()), 0);
+}
+
+
+TEST_F(Dense, IsInversePermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, square->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted =
+        Mtx::create(ref, square->get_size(), square->get_size()[1] + 2);
+
+    square->inverse_permute(rpermute_idxs.get(), permuted.get());
+    dsquare->inverse_permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
+TEST_F(Dense, IsRowPermutable)
+{
+    set_up_apply_data();
+
+    auto r_permute = x->row_permute(rpermute_idxs.get());
+    auto dr_permute = dx->row_permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(r_permute.get()),
+                        static_cast<Mtx *>(dr_permute.get()), 0);
+}
+
+
+TEST_F(Dense, IsRowPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->row_permute(rpermute_idxs.get(), permuted.get());
+    dx->row_permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
+TEST_F(Dense, IsColPermutable)
+{
+    set_up_apply_data();
+
+    auto c_permute = x->column_permute(cpermute_idxs.get());
+    auto dc_permute = dx->column_permute(cpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(c_permute.get()),
+                        static_cast<Mtx *>(dc_permute.get()), 0);
+}
+
+
+TEST_F(Dense, IsColPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->column_permute(cpermute_idxs.get(), permuted.get());
+    dx->column_permute(cpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
+TEST_F(Dense, IsInverseRowPermutable)
+{
+    set_up_apply_data();
+
+    auto inverse_r_permute = x->inverse_row_permute(rpermute_idxs.get());
+    auto d_inverse_r_permute = dx->inverse_row_permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_r_permute.get()),
+                        static_cast<Mtx *>(d_inverse_r_permute.get()), 0);
+}
+
+
+TEST_F(Dense, IsInverseRowPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->inverse_row_permute(rpermute_idxs.get(), permuted.get());
+    dx->inverse_row_permute(rpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
+TEST_F(Dense, IsInverseColPermutable)
+{
+    set_up_apply_data();
+
+    auto inverse_c_permute = x->inverse_column_permute(cpermute_idxs.get());
+    auto d_inverse_c_permute = dx->inverse_column_permute(cpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_c_permute.get()),
+                        static_cast<Mtx *>(d_inverse_c_permute.get()), 0);
+}
+
+
+TEST_F(Dense, IsInverseColPermutableIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto permuted = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dpermuted = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->inverse_column_permute(cpermute_idxs.get(), permuted.get());
+    dx->inverse_column_permute(cpermute_idxs.get(), dpermuted.get());
+
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
+TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto diag = x->extract_diagonal();
+    auto ddiag = dx->extract_diagonal();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
+TEST_F(Dense, ExtractDiagonalOnTallSkinnyIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto diag = Diagonal::create(ref, x->get_size()[1]);
+    // test make_temporary_clone
+    auto ddiag = Diagonal::create(ref, x->get_size()[1]);
+
+    x->extract_diagonal(diag.get());
+    dx->extract_diagonal(ddiag.get());
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
+TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto diag = y->extract_diagonal();
+    auto ddiag = dy->extract_diagonal();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
+TEST_F(Dense, ExtractDiagonalOnShortFatIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto diag = Diagonal::create(ref, y->get_size()[0]);
+    // test make_temporary_clone
+    auto ddiag = Diagonal::create(ref, y->get_size()[0]);
+
+    y->extract_diagonal(diag.get());
+    dy->extract_diagonal(ddiag.get());
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
+TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    x->compute_absolute_inplace();
+    dx->compute_absolute_inplace();
+
+    GKO_ASSERT_MTX_NEAR(x, dx, r<vtype>::value);
+}
+
+
+TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_x = x->compute_absolute();
+    auto dabs_x = dx->compute_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, r<vtype>::value);
+}
+
+
+TEST_F(Dense, OutplaceAbsoluteMatrixIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto abs_x = NormVector::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dabs_x = NormVector::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->compute_absolute(abs_x.get());
+    dx->compute_absolute(dabs_x.get());
+
+    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, r<vtype>::value);
+}
+
+
+TEST_F(Dense, MakeComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto complex_x = x->make_complex();
+    auto dcomplex_x = dx->make_complex();
+
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
+}
+
+
+TEST_F(Dense, MakeComplexIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto complex_x = ComplexMtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dcomplex_x =
+        ComplexMtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->make_complex(complex_x.get());
+    dx->make_complex(dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
+}
+
+
+TEST_F(Dense, GetRealIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto real_x = x->get_real();
+    auto dreal_x = dx->get_real();
+
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
+}
+
+
+TEST_F(Dense, GetRealIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto real_x = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dreal_x = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->get_real(real_x.get());
+    dx->get_real(dreal_x.get());
+
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
+}
+
+
+TEST_F(Dense, GetImagIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto imag_x = x->get_imag();
+    auto dimag_x = dx->get_imag();
+
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
+}
+
+
+TEST_F(Dense, GetImagIntoDenseCrossExecutor)
+{
+    set_up_apply_data();
+    auto imag_x = Mtx::create(ref, x->get_size());
+    // test make_temporary_clone and non-default stride
+    auto dimag_x = Mtx::create(ref, x->get_size(), x->get_size()[1] + 2);
+
+    x->get_imag(imag_x.get());
+    dx->get_imag(dimag_x.get());
+
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
+}
+
+
+}  // namespace

From 08a2256cd6107e19d608c202ce09e8f23ae4c109 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 24 Jun 2021 12:35:34 +0200
Subject: [PATCH 0854/1008] increase BiCGSTAB test tolerances

---
 test/solver/bicgstab_kernels.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/solver/bicgstab_kernels.cpp b/test/solver/bicgstab_kernels.cpp
index 4d8cd82700e..9b5a8457cc6 100644
--- a/test/solver/bicgstab_kernels.cpp
+++ b/test/solver/bicgstab_kernels.cpp
@@ -334,8 +334,8 @@ TEST_F(Bicgstab, BicgstabApplyOneRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     exec_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 100);
-    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 100);
+    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 500);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 500);
 }
 
 
@@ -355,8 +355,8 @@ TEST_F(Bicgstab, BicgstabApplyMultipleRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     exec_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 100);
-    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 100);
+    GKO_ASSERT_MTX_NEAR(d_b, b, ::r<value_type>::value * 500);
+    GKO_ASSERT_MTX_NEAR(d_x, x, ::r<value_type>::value * 500);
 }
 
 

From 1e55da970fd54a05bddf154c278a965d5c12a5d1 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 24 Jun 2021 12:47:47 +0200
Subject: [PATCH 0855/1008] fix formatting

---
 dev_tools/scripts/config               | 4 ++++
 dpcpp/base/kernel_launch_solver.dp.hpp | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/dev_tools/scripts/config b/dev_tools/scripts/config
index f856d97dae2..9e333e66762 100644
--- a/dev_tools/scripts/config
+++ b/dev_tools/scripts/config
@@ -22,6 +22,10 @@
     - FixInclude: "core/factorization/par_ict_kernels.hpp"
 - "(cuda|hip|dpcpp)/preconditioner/jacobi_"
     - FixInclude: "core/preconditioner/jacobi_kernels.hpp"
+- "(cuda|hip|dpcpp|omp)/base/kernel_launch\."
+    - FixInclude: "common/base/kernel_launch.hpp"
+- "(cuda|hip|dpcpp|omp)/base/kernel_launch_solver\."
+    - FixInclude: "common/base/kernel_launch_solver.hpp"
 - "core/test/base/(extended_float|iterator_factory)"
     - RemoveTest: "true"
 - "core/test/base/allocator"
diff --git a/dpcpp/base/kernel_launch_solver.dp.hpp b/dpcpp/base/kernel_launch_solver.dp.hpp
index 62c4c325ac5..fec21f799e6 100644
--- a/dpcpp/base/kernel_launch_solver.dp.hpp
+++ b/dpcpp/base/kernel_launch_solver.dp.hpp
@@ -74,4 +74,4 @@ void run_kernel_solver(std::shared_ptr<const DpcppExecutor> exec,
 }  // namespace kernels
 }  // namespace gko
 
-#endif  // GKO_HIP_BASE_KERNEL_LAUNCH_SOLVER_HIP_HPP_
+#endif  // GKO_DPCPP_BASE_KERNEL_LAUNCH_SOLVER_DP_HPP_

From 6a52ee045f76e5fcaab57217fa4d156b88522f0f Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 24 Jun 2021 13:01:49 +0200
Subject: [PATCH 0856/1008] move kernel_launch defines into test CMakeLists

---
 cuda/test/base/CMakeLists.txt        | 2 ++
 cuda/test/base/kernel_launch.cu      | 1 -
 dpcpp/test/base/CMakeLists.txt       | 2 ++
 dpcpp/test/base/kernel_launch.dp.cpp | 1 -
 hip/test/base/CMakeLists.txt         | 1 +
 hip/test/base/kernel_launch.hip.cpp  | 1 -
 omp/test/base/CMakeLists.txt         | 1 +
 omp/test/base/kernel_launch.cpp      | 1 -
 8 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/cuda/test/base/CMakeLists.txt b/cuda/test/base/CMakeLists.txt
index 63318636a2b..cedc2d8d328 100644
--- a/cuda/test/base/CMakeLists.txt
+++ b/cuda/test/base/CMakeLists.txt
@@ -6,5 +6,7 @@ if(GINKGO_HAVE_HWLOC)
 endif()
 ginkgo_create_cuda_test(exception_helpers)
 ginkgo_create_cuda_test(kernel_launch)
+# set correct flags for kernel_launch.hpp
+target_compile_definitions(cuda_test_base_kernel_launch PRIVATE GKO_COMPILING_CUDA)
 ginkgo_create_cuda_test(lin_op)
 ginkgo_create_cuda_test(math)
diff --git a/cuda/test/base/kernel_launch.cu b/cuda/test/base/kernel_launch.cu
index 7f2a67de50f..23f1a1b2e62 100644
--- a/cuda/test/base/kernel_launch.cu
+++ b/cuda/test/base/kernel_launch.cu
@@ -30,7 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#define GKO_COMPILING_CUDA
 #include "cuda/base/kernel_launch.cuh"
 
 
diff --git a/dpcpp/test/base/CMakeLists.txt b/dpcpp/test/base/CMakeLists.txt
index 8ed2d944eea..bb9c8a75050 100644
--- a/dpcpp/test/base/CMakeLists.txt
+++ b/dpcpp/test/base/CMakeLists.txt
@@ -1,3 +1,5 @@
 ginkgo_create_dpcpp_test(executor)
 ginkgo_create_dpcpp_test(dim3)
 ginkgo_create_dpcpp_test(kernel_launch)
+# set correct flags for kernel_launch.hpp
+target_compile_definitions(dpcpp_test_base_kernel_launch PRIVATE GKO_COMPILING_DPCPP)
diff --git a/dpcpp/test/base/kernel_launch.dp.cpp b/dpcpp/test/base/kernel_launch.dp.cpp
index 82cc6c2b320..e9aa1e17e9d 100644
--- a/dpcpp/test/base/kernel_launch.dp.cpp
+++ b/dpcpp/test/base/kernel_launch.dp.cpp
@@ -30,7 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#define GKO_COMPILING_DPCPP
 #include "dpcpp/base/kernel_launch.dp.hpp"
 
 
diff --git a/hip/test/base/CMakeLists.txt b/hip/test/base/CMakeLists.txt
index 9659cf465aa..c2f694fdf8a 100644
--- a/hip/test/base/CMakeLists.txt
+++ b/hip/test/base/CMakeLists.txt
@@ -4,6 +4,7 @@ if(GINKGO_HAVE_HWLOC)
     ginkgo_create_hip_test(hip_executor_topology NUMA::NUMA)
 endif()
 ginkgo_create_hip_test(kernel_launch)
+# correct flags for kernel_launch.hpp are set in GINKGO_HIPCC_OPTIONS
 ginkgo_create_hip_test(lin_op)
 ginkgo_create_hip_test(math)
 # Only hcc needs the libraries. nvcc only requires the headers.
diff --git a/hip/test/base/kernel_launch.hip.cpp b/hip/test/base/kernel_launch.hip.cpp
index 968f5a47424..b5649894819 100644
--- a/hip/test/base/kernel_launch.hip.cpp
+++ b/hip/test/base/kernel_launch.hip.cpp
@@ -30,7 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-// no need for #define GKO_COMPILING_HIP here, it gets set by ginkgo_hip
 #include "hip/base/kernel_launch.hip.hpp"
 
 
diff --git a/omp/test/base/CMakeLists.txt b/omp/test/base/CMakeLists.txt
index 475e27ff3df..369bb5dc7d6 100644
--- a/omp/test/base/CMakeLists.txt
+++ b/omp/test/base/CMakeLists.txt
@@ -1,3 +1,4 @@
 ginkgo_create_test(kernel_launch)
 find_package(OpenMP 3.0 REQUIRED)
+target_compile_definitions(omp_test_base_kernel_launch PRIVATE GKO_COMPILING_OMP)
 target_link_libraries(omp_test_base_kernel_launch PRIVATE OpenMP::OpenMP_CXX)
\ No newline at end of file
diff --git a/omp/test/base/kernel_launch.cpp b/omp/test/base/kernel_launch.cpp
index fb40a68810d..7b9388b9fed 100644
--- a/omp/test/base/kernel_launch.cpp
+++ b/omp/test/base/kernel_launch.cpp
@@ -30,7 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#define GKO_COMPILING_OMP
 #include "omp/base/kernel_launch.hpp"
 
 

From 05fabc3bc668eb89c66e3ff2903c8ed77026b4b4 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 29 Jun 2021 14:11:50 +0200
Subject: [PATCH 0857/1008] update test_install path for formatting job

---
 .github/bot-pr-format-base.sh | 2 +-
 common/base/kernel_launch.hpp | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/bot-pr-format-base.sh b/.github/bot-pr-format-base.sh
index 6821306322c..e72539a9d61 100644
--- a/.github/bot-pr-format-base.sh
+++ b/.github/bot-pr-format-base.sh
@@ -4,7 +4,7 @@ source .github/bot-pr-base.sh
 
 EXTENSION_REGEX='\.(cuh?|hpp|hpp\.inc?|cpp)$'
 FORMAT_HEADER_REGEX='^(benchmark|core|cuda|hip|include/ginkgo/core|omp|reference|dpcpp)/'
-FORMAT_REGEX='^(common|examples|test_install)/'
+FORMAT_REGEX='^(common|examples|test)/'
 
 echo "Retrieving PR file list"
 PR_FILES=$(bot_get_all_changed_files ${PR_URL})
diff --git a/common/base/kernel_launch.hpp b/common/base/kernel_launch.hpp
index bb127ea8ece..bd15b6f704b 100644
--- a/common/base/kernel_launch.hpp
+++ b/common/base/kernel_launch.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CORE_BASE_KERNEL_LAUNCH_HPP_
-#define GKO_CORE_BASE_KERNEL_LAUNCH_HPP_
+#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
+#define GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
 
 
 #include <ginkgo/core/base/executor.hpp>
@@ -276,4 +276,4 @@ typename to_device_type_impl<T>::type map_to_device(T &&param)
 #endif
 
 
-#endif  // GKO_CORE_BASE_KERNEL_LAUNCH_HPP_
+#endif  // GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_

From 15c0b0f60cec14d700fd6f3b3014b25eb57f5386 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 29 Jun 2021 14:25:20 +0200
Subject: [PATCH 0858/1008] rename compile tests inside test/

---
 .gitlab-ci.yml                                         | 2 +-
 CMakeLists.txt                                         | 8 ++++----
 test/{build_export => test_exportbuild}/CMakeLists.txt | 2 +-
 test/{build_install => test_install}/CMakeLists.txt    | 0
 test/{build_install => test_install}/data/A.mtx        | 0
 test/{build_install => test_install}/data/b.mtx        | 0
 test/{build_install => test_install}/data/x0.mtx       | 0
 test/{build_install => test_install}/test_install.cpp  | 0
 test/{build_subdir => test_subdir}/CMakeLists.txt      | 2 +-
 9 files changed, 7 insertions(+), 7 deletions(-)
 rename test/{build_export => test_exportbuild}/CMakeLists.txt (82%)
 rename test/{build_install => test_install}/CMakeLists.txt (100%)
 rename test/{build_install => test_install}/data/A.mtx (100%)
 rename test/{build_install => test_install}/data/b.mtx (100%)
 rename test/{build_install => test_install}/data/x0.mtx (100%)
 rename test/{build_install => test_install}/test_install.cpp (100%)
 rename test/{build_subdir => test_subdir}/CMakeLists.txt (84%)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0b4f96f4371..cbb9931f3b5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -122,7 +122,7 @@ include:
         (( $(ctest -N | tail -1 | sed 's/Total Tests: //') != 0 )) || exit 1
     - ctest -V
     - ninja test_install
-    - pushd test/build_install
+    - pushd test/test_install
     - ninja install
     - popd
     - |
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c9691c27325..e4aebe07cdf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -333,10 +333,10 @@ ginkgo_modify_flags(CMAKE_CUDA_FLAGS_DEBUG)
 ginkgo_modify_flags(CMAKE_CUDA_FLAGS_RELEASE)
 ginkgo_install()
 
-set(GINKGO_TEST_INSTALL_SRC_DIR "${Ginkgo_SOURCE_DIR}/test/build_install/")
-set(GINKGO_TEST_INSTALL_BIN_DIR "${Ginkgo_BINARY_DIR}/test/build_install/")
-set(GINKGO_TEST_EXPORTBUILD_SRC_DIR "${Ginkgo_SOURCE_DIR}/test/build_export/")
-set(GINKGO_TEST_EXPORTBUILD_BIN_DIR "${Ginkgo_BINARY_DIR}/test/build_export/")
+set(GINKGO_TEST_INSTALL_SRC_DIR "${Ginkgo_SOURCE_DIR}/test/test_install/")
+set(GINKGO_TEST_INSTALL_BIN_DIR "${Ginkgo_BINARY_DIR}/test/test_install/")
+set(GINKGO_TEST_EXPORTBUILD_SRC_DIR "${Ginkgo_SOURCE_DIR}/test/test_exportbuild/")
+set(GINKGO_TEST_EXPORTBUILD_BIN_DIR "${Ginkgo_BINARY_DIR}/test/test_exportbuild/")
 if(MSVC)
     set(GINKGO_TEST_INSTALL_CMD ${GINKGO_TEST_INSTALL_BIN_DIR}/$<CONFIG>/test_install)
     set(GINKGO_TEST_EXPORTBUILD_CMD ${GINKGO_TEST_EXPORTBUILD_BIN_DIR}/$<CONFIG>/test_exportbuild)
diff --git a/test/build_export/CMakeLists.txt b/test/test_exportbuild/CMakeLists.txt
similarity index 82%
rename from test/build_export/CMakeLists.txt
rename to test/test_exportbuild/CMakeLists.txt
index 8588b63a471..52a8d3851cd 100644
--- a/test/build_export/CMakeLists.txt
+++ b/test/test_exportbuild/CMakeLists.txt
@@ -5,6 +5,6 @@ find_package(Ginkgo REQUIRED)
 
 # Here, we use test install without any data. We instantiate the
 # interface only.
-add_executable(test_exportbuild ../build_install/test_install.cpp)
+add_executable(test_exportbuild ../test_install/test_install.cpp)
 target_compile_features(test_exportbuild PUBLIC cxx_std_14)
 target_link_libraries(test_exportbuild PRIVATE Ginkgo::ginkgo)
diff --git a/test/build_install/CMakeLists.txt b/test/test_install/CMakeLists.txt
similarity index 100%
rename from test/build_install/CMakeLists.txt
rename to test/test_install/CMakeLists.txt
diff --git a/test/build_install/data/A.mtx b/test/test_install/data/A.mtx
similarity index 100%
rename from test/build_install/data/A.mtx
rename to test/test_install/data/A.mtx
diff --git a/test/build_install/data/b.mtx b/test/test_install/data/b.mtx
similarity index 100%
rename from test/build_install/data/b.mtx
rename to test/test_install/data/b.mtx
diff --git a/test/build_install/data/x0.mtx b/test/test_install/data/x0.mtx
similarity index 100%
rename from test/build_install/data/x0.mtx
rename to test/test_install/data/x0.mtx
diff --git a/test/build_install/test_install.cpp b/test/test_install/test_install.cpp
similarity index 100%
rename from test/build_install/test_install.cpp
rename to test/test_install/test_install.cpp
diff --git a/test/build_subdir/CMakeLists.txt b/test/test_subdir/CMakeLists.txt
similarity index 84%
rename from test/build_subdir/CMakeLists.txt
rename to test/test_subdir/CMakeLists.txt
index 811cc61f430..2017b69366f 100644
--- a/test/build_subdir/CMakeLists.txt
+++ b/test/test_subdir/CMakeLists.txt
@@ -4,6 +4,6 @@ file(CREATE_LINK "${CMAKE_CURRENT_SOURCE_DIR}/../.." "${CMAKE_CURRENT_BINARY_DIR
 
 add_subdirectory("${CMAKE_CURRENT_BINARY_DIR}/ginkgo")
 
-add_executable(test_subdir ../build_install/test_install.cpp)
+add_executable(test_subdir ../test_install/test_install.cpp)
 target_compile_features(test_subdir PUBLIC cxx_std_14)
 target_link_libraries(test_subdir PRIVATE Ginkgo::ginkgo)

From 29d0e4be5ba59885dc085de38e86553b50af3ac3 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 29 Jun 2021 14:37:59 +0200
Subject: [PATCH 0859/1008] remove circular dependency in kernel_launch header

---
 cmake/build_helpers.cmake              |  3 +++
 cuda/base/kernel_launch.cuh            | 10 +++-------
 cuda/base/kernel_launch_solver.cuh     | 11 ++++-------
 cuda/test/base/kernel_launch.cu        |  4 ++--
 dpcpp/base/kernel_launch.dp.hpp        | 11 +++--------
 dpcpp/base/kernel_launch_solver.dp.hpp | 11 ++++-------
 dpcpp/test/base/kernel_launch.dp.cpp   |  4 ++--
 hip/base/kernel_launch.hip.hpp         | 10 +++-------
 hip/base/kernel_launch_solver.hip.hpp  | 11 ++++-------
 hip/test/base/kernel_launch.hip.cpp    |  4 ++--
 omp/base/kernel_launch.hpp             | 10 +++-------
 omp/base/kernel_launch_solver.hpp      | 11 ++++-------
 omp/test/base/kernel_launch.cpp        |  4 ++--
 13 files changed, 39 insertions(+), 65 deletions(-)

diff --git a/cmake/build_helpers.cmake b/cmake/build_helpers.cmake
index f0e4dcbd698..4b89cf451c4 100644
--- a/cmake/build_helpers.cmake
+++ b/cmake/build_helpers.cmake
@@ -51,8 +51,11 @@ function(ginkgo_check_headers target defines)
     file(GLOB_RECURSE CXX_HEADERS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" CONFIGURE_DEPENDS "*.hpp")
     list(FILTER CXX_HEADERS EXCLUDE REGEX ".*\.hip\.hpp$")
     list(FILTER CXX_HEADERS EXCLUDE REGEX "^test.*")
+    list(FILTER CXX_HEADERS EXCLUDE REGEX "^base/kernel_launch.*")
     list(FILTER CUDA_HEADERS EXCLUDE REGEX "^test.*")
+    list(FILTER CUDA_HEADERS EXCLUDE REGEX "^base/kernel_launch.*")
     list(FILTER HIP_HEADERS EXCLUDE REGEX "^test.*")
+    list(FILTER HIP_HEADERS EXCLUDE REGEX "^base/kernel_launch.*")
 
     set(SOURCES "")
     # if we have any CUDA files in there, compile everything as CUDA
diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index a4ee9296445..905dbf5d0b7 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -30,11 +30,9 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CUDA_BASE_KERNEL_LAUNCH_CUH_
-#define GKO_CUDA_BASE_KERNEL_LAUNCH_CUH_
-
-
-#include "common/base/kernel_launch.hpp"
+#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
+#error "This file can only be used from inside common/base/kernel_launch.hpp"
+#endif
 
 
 #include "cuda/base/device_guard.hpp"
@@ -102,5 +100,3 @@ void run_kernel(std::shared_ptr<const CudaExecutor> exec, KernelFunction fn,
 }  // namespace cuda
 }  // namespace kernels
 }  // namespace gko
-
-#endif  // GKO_CUDA_BASE_KERNEL_LAUNCH_CUH_
diff --git a/cuda/base/kernel_launch_solver.cuh b/cuda/base/kernel_launch_solver.cuh
index eaebcc62dcb..2424e6a4fa2 100644
--- a/cuda/base/kernel_launch_solver.cuh
+++ b/cuda/base/kernel_launch_solver.cuh
@@ -30,11 +30,10 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_CUDA_BASE_KERNEL_LAUNCH_SOLVER_CUH_
-#define GKO_CUDA_BASE_KERNEL_LAUNCH_SOLVER_CUH_
-
-
-#include "common/base/kernel_launch_solver.hpp"
+#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
+#error \
+    "This file can only be used from inside common/base/kernel_launch_solver.hpp"
+#endif
 
 
 namespace gko {
@@ -74,5 +73,3 @@ void run_kernel_solver(std::shared_ptr<const CudaExecutor> exec,
 }  // namespace cuda
 }  // namespace kernels
 }  // namespace gko
-
-#endif  // GKO_CUDA_BASE_KERNEL_LAUNCH_SOLVER_CUH_
diff --git a/cuda/test/base/kernel_launch.cu b/cuda/test/base/kernel_launch.cu
index 23f1a1b2e62..4c1cc80155f 100644
--- a/cuda/test/base/kernel_launch.cu
+++ b/cuda/test/base/kernel_launch.cu
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "cuda/base/kernel_launch.cuh"
+#include "common/base/kernel_launch.hpp"
 
 
 #include <gtest/gtest.h>
@@ -40,8 +40,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "common/base/kernel_launch_solver.hpp"
 #include "core/test/utils.hpp"
-#include "cuda/base/kernel_launch_solver.cuh"
 
 
 namespace {
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
index 3630e8f326a..df41eee5e0f 100644
--- a/dpcpp/base/kernel_launch.dp.hpp
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -30,11 +30,9 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_DPCPP_BASE_KERNEL_LAUNCH_DP_HPP_
-#define GKO_DPCPP_BASE_KERNEL_LAUNCH_DP_HPP_
-
-
-#include "common/base/kernel_launch.hpp"
+#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
+#error "This file can only be used from inside common/base/kernel_launch.hpp"
+#endif
 
 
 #include <CL/sycl.hpp>
@@ -90,6 +88,3 @@ void run_kernel(std::shared_ptr<const DpcppExecutor> exec, KernelFunction fn,
 }  // namespace dpcpp
 }  // namespace kernels
 }  // namespace gko
-
-
-#endif  // GKO_DPCPP_BASE_KERNEL_LAUNCH_DP_HPP_
diff --git a/dpcpp/base/kernel_launch_solver.dp.hpp b/dpcpp/base/kernel_launch_solver.dp.hpp
index fec21f799e6..81c46b1f503 100644
--- a/dpcpp/base/kernel_launch_solver.dp.hpp
+++ b/dpcpp/base/kernel_launch_solver.dp.hpp
@@ -30,11 +30,10 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_DPCPP_BASE_KERNEL_LAUNCH_SOLVER_DP_HPP_
-#define GKO_DPCPP_BASE_KERNEL_LAUNCH_SOLVER_DP_HPP_
-
-
-#include "common/base/kernel_launch_solver.hpp"
+#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
+#error \
+    "This file can only be used from inside common/base/kernel_launch_solver.hpp"
+#endif
 
 
 namespace gko {
@@ -73,5 +72,3 @@ void run_kernel_solver(std::shared_ptr<const DpcppExecutor> exec,
 }  // namespace dpcpp
 }  // namespace kernels
 }  // namespace gko
-
-#endif  // GKO_DPCPP_BASE_KERNEL_LAUNCH_SOLVER_DP_HPP_
diff --git a/dpcpp/test/base/kernel_launch.dp.cpp b/dpcpp/test/base/kernel_launch.dp.cpp
index e9aa1e17e9d..3bda50b6175 100644
--- a/dpcpp/test/base/kernel_launch.dp.cpp
+++ b/dpcpp/test/base/kernel_launch.dp.cpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "dpcpp/base/kernel_launch.dp.hpp"
+#include "common/base/kernel_launch.hpp"
 
 
 #include <gtest/gtest.h>
@@ -40,8 +40,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "common/base/kernel_launch_solver.hpp"
 #include "core/test/utils.hpp"
-#include "dpcpp/base/kernel_launch_solver.dp.hpp"
 
 
 namespace {
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index 991b0cc1f98..1267fe69695 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -30,11 +30,9 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_HIP_BASE_KERNEL_LAUNCH_HIP_HPP_
-#define GKO_HIP_BASE_KERNEL_LAUNCH_HIP_HPP_
-
-
-#include "common/base/kernel_launch.hpp"
+#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
+#error "This file can only be used from inside common/base/kernel_launch.hpp"
+#endif
 
 
 #include <hip/hip_runtime.h>
@@ -105,5 +103,3 @@ void run_kernel(std::shared_ptr<const HipExecutor> exec, KernelFunction fn,
 }  // namespace hip
 }  // namespace kernels
 }  // namespace gko
-
-#endif  // GKO_HIP_BASE_KERNEL_LAUNCH_HIP_HPP_
diff --git a/hip/base/kernel_launch_solver.hip.hpp b/hip/base/kernel_launch_solver.hip.hpp
index 9f8a5e6e5f2..f2e117655bc 100644
--- a/hip/base/kernel_launch_solver.hip.hpp
+++ b/hip/base/kernel_launch_solver.hip.hpp
@@ -30,11 +30,10 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_HIP_BASE_KERNEL_LAUNCH_SOLVER_HIP_HPP_
-#define GKO_HIP_BASE_KERNEL_LAUNCH_SOLVER_HIP_HPP_
-
-
-#include "common/base/kernel_launch_solver.hpp"
+#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
+#error \
+    "This file can only be used from inside common/base/kernel_launch_solver.hpp"
+#endif
 
 
 #include <hip/hip_runtime.h>
@@ -78,5 +77,3 @@ void run_kernel_solver(std::shared_ptr<const HipExecutor> exec,
 }  // namespace hip
 }  // namespace kernels
 }  // namespace gko
-
-#endif  // GKO_HIP_BASE_KERNEL_LAUNCH_SOLVER_HIP_HPP_
diff --git a/hip/test/base/kernel_launch.hip.cpp b/hip/test/base/kernel_launch.hip.cpp
index b5649894819..386f3b9c3c9 100644
--- a/hip/test/base/kernel_launch.hip.cpp
+++ b/hip/test/base/kernel_launch.hip.cpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "hip/base/kernel_launch.hip.hpp"
+#include "common/base/kernel_launch.hpp"
 
 
 #include <gtest/gtest.h>
@@ -40,8 +40,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "common/base/kernel_launch_solver.hpp"
 #include "core/test/utils.hpp"
-#include "hip/base/kernel_launch_solver.hip.hpp"
 
 
 namespace {
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index 0ff866e7966..8de7f71581a 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -30,11 +30,9 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_OMP_BASE_KERNEL_LAUNCH_HPP_
-#define GKO_OMP_BASE_KERNEL_LAUNCH_HPP_
-
-
-#include "common/base/kernel_launch.hpp"
+#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
+#error "This file can only be used from inside common/base/kernel_launch.hpp"
+#endif
 
 
 namespace gko {
@@ -150,5 +148,3 @@ void run_kernel(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
 }  // namespace omp
 }  // namespace kernels
 }  // namespace gko
-
-#endif  // GKO_OMP_BASE_KERNEL_LAUNCH_HPP_
diff --git a/omp/base/kernel_launch_solver.hpp b/omp/base/kernel_launch_solver.hpp
index f3607a063ee..69e20ac4faa 100644
--- a/omp/base/kernel_launch_solver.hpp
+++ b/omp/base/kernel_launch_solver.hpp
@@ -30,11 +30,10 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_OMP_BASE_KERNEL_LAUNCH_SOLVER_HPP_
-#define GKO_OMP_BASE_KERNEL_LAUNCH_SOLVER_HPP_
-
-
-#include "common/base/kernel_launch_solver.hpp"
+#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
+#error \
+    "This file can only be used from inside common/base/kernel_launch_solver.hpp"
+#endif
 
 
 namespace gko {
@@ -64,5 +63,3 @@ void run_kernel_solver(std::shared_ptr<const OmpExecutor> exec,
 }  // namespace omp
 }  // namespace kernels
 }  // namespace gko
-
-#endif  // GKO_OMP_BASE_KERNEL_LAUNCH_SOLVER_HPP_
diff --git a/omp/test/base/kernel_launch.cpp b/omp/test/base/kernel_launch.cpp
index 7b9388b9fed..5fc08bdcd08 100644
--- a/omp/test/base/kernel_launch.cpp
+++ b/omp/test/base/kernel_launch.cpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "omp/base/kernel_launch.hpp"
+#include "common/base/kernel_launch.hpp"
 
 
 #include <gtest/gtest.h>
@@ -40,8 +40,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "common/base/kernel_launch_solver.hpp"
 #include "core/test/utils.hpp"
-#include "omp/base/kernel_launch_solver.hpp"
 
 
 namespace {

From fd907949e44920d2e38fe939109125a8d6b6e331 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 29 Jun 2021 16:15:22 +0200
Subject: [PATCH 0860/1008] review updates
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

add missing includes for more IWYU-like structure

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 common/base/kernel_launch.hpp        | 4 ++++
 core/matrix/dense_kernels.hpp        | 3 +++
 core/solver/bicg_kernels.hpp         | 3 +++
 core/solver/bicgstab_kernels.hpp     | 3 +++
 core/solver/cg_kernels.hpp           | 3 +++
 core/solver/cgs_kernels.hpp          | 3 +++
 core/solver/fcg_kernels.hpp          | 3 +++
 core/solver/ir_kernels.hpp           | 3 +++
 cuda/test/base/kernel_launch.cu      | 6 ++++++
 dpcpp/test/base/kernel_launch.dp.cpp | 6 ++++++
 hip/test/base/kernel_launch.hip.cpp  | 6 ++++++
 include/ginkgo/core/matrix/dense.hpp | 8 +++++---
 omp/test/base/kernel_launch.cpp      | 6 ++++++
 test/utils/executor.hpp              | 3 +++
 14 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/common/base/kernel_launch.hpp b/common/base/kernel_launch.hpp
index bd15b6f704b..837f7f9e65a 100644
--- a/common/base/kernel_launch.hpp
+++ b/common/base/kernel_launch.hpp
@@ -34,6 +34,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
 
 
+#include <type_traits>
+
+
+#include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
diff --git a/core/matrix/dense_kernels.hpp b/core/matrix/dense_kernels.hpp
index 6b1e57f0a27..014c58f1b45 100644
--- a/core/matrix/dense_kernels.hpp
+++ b/core/matrix/dense_kernels.hpp
@@ -37,6 +37,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include <memory>
+
+
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/diagonal.hpp>
diff --git a/core/solver/bicg_kernels.hpp b/core/solver/bicg_kernels.hpp
index e715694f6e3..87bac70421c 100644
--- a/core/solver/bicg_kernels.hpp
+++ b/core/solver/bicg_kernels.hpp
@@ -34,6 +34,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_SOLVER_BICG_KERNELS_HPP_
 
 
+#include <memory>
+
+
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
diff --git a/core/solver/bicgstab_kernels.hpp b/core/solver/bicgstab_kernels.hpp
index b6b5072be26..74b1208d4d7 100644
--- a/core/solver/bicgstab_kernels.hpp
+++ b/core/solver/bicgstab_kernels.hpp
@@ -34,6 +34,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_SOLVER_BICGSTAB_KERNELS_HPP_
 
 
+#include <memory>
+
+
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
diff --git a/core/solver/cg_kernels.hpp b/core/solver/cg_kernels.hpp
index fed7d5b97bc..2cfa476c744 100644
--- a/core/solver/cg_kernels.hpp
+++ b/core/solver/cg_kernels.hpp
@@ -34,6 +34,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_SOLVER_CG_KERNELS_HPP_
 
 
+#include <memory>
+
+
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
diff --git a/core/solver/cgs_kernels.hpp b/core/solver/cgs_kernels.hpp
index 210aacbc260..3c10d782062 100644
--- a/core/solver/cgs_kernels.hpp
+++ b/core/solver/cgs_kernels.hpp
@@ -34,6 +34,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_SOLVER_CGS_KERNELS_HPP_
 
 
+#include <memory>
+
+
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/types.hpp>
diff --git a/core/solver/fcg_kernels.hpp b/core/solver/fcg_kernels.hpp
index 9baa62e9e91..04c385f0e3e 100644
--- a/core/solver/fcg_kernels.hpp
+++ b/core/solver/fcg_kernels.hpp
@@ -34,6 +34,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_SOLVER_FCG_KERNELS_HPP_
 
 
+#include <memory>
+
+
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
diff --git a/core/solver/ir_kernels.hpp b/core/solver/ir_kernels.hpp
index da3fc5c0997..2f96a445e3d 100644
--- a/core/solver/ir_kernels.hpp
+++ b/core/solver/ir_kernels.hpp
@@ -34,6 +34,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_CORE_SOLVER_IR_KERNELS_HPP_
 
 
+#include <memory>
+
+
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/types.hpp>
diff --git a/cuda/test/base/kernel_launch.cu b/cuda/test/base/kernel_launch.cu
index 4c1cc80155f..18244836674 100644
--- a/cuda/test/base/kernel_launch.cu
+++ b/cuda/test/base/kernel_launch.cu
@@ -33,10 +33,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common/base/kernel_launch.hpp"
 
 
+#include <memory>
+#include <type_traits>
+
+
 #include <gtest/gtest.h>
 
 
 #include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
diff --git a/dpcpp/test/base/kernel_launch.dp.cpp b/dpcpp/test/base/kernel_launch.dp.cpp
index 3bda50b6175..c148721fb48 100644
--- a/dpcpp/test/base/kernel_launch.dp.cpp
+++ b/dpcpp/test/base/kernel_launch.dp.cpp
@@ -33,10 +33,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common/base/kernel_launch.hpp"
 
 
+#include <memory>
+#include <type_traits>
+
+
 #include <gtest/gtest.h>
 
 
 #include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
diff --git a/hip/test/base/kernel_launch.hip.cpp b/hip/test/base/kernel_launch.hip.cpp
index 386f3b9c3c9..90073a00554 100644
--- a/hip/test/base/kernel_launch.hip.cpp
+++ b/hip/test/base/kernel_launch.hip.cpp
@@ -33,10 +33,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common/base/kernel_launch.hpp"
 
 
+#include <memory>
+#include <type_traits>
+
+
 #include <gtest/gtest.h>
 
 
 #include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 652c6357af2..95ca95e720b 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -149,7 +149,8 @@ class Dense
     using row_major_range = gko::range<gko::accessor::row_major<ValueType, 2>>;
 
     /**
-     * Creates a Dense matrix with the configuration of another Dense matrix.
+     * Creates a Dense matrix with the same size and stride as another Dense
+     * matrix.
      *
      * @param other  The other matrix whose configuration needs to copied.
      */
@@ -827,9 +828,10 @@ class Dense
     }
 
     /**
-     * Creates a Dense matrix with the same size as the callers matrix.
+     * Creates a Dense matrix with the same size and stride as the callers
+     * matrix.
      *
-     * @returns a Dense matrix with the same size as the caller.
+     * @returns a Dense matrix with the same size and stride as the caller.
      */
     virtual std::unique_ptr<Dense> create_with_same_config() const
     {
diff --git a/omp/test/base/kernel_launch.cpp b/omp/test/base/kernel_launch.cpp
index 5fc08bdcd08..6e8c5fc3f9e 100644
--- a/omp/test/base/kernel_launch.cpp
+++ b/omp/test/base/kernel_launch.cpp
@@ -33,10 +33,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common/base/kernel_launch.hpp"
 
 
+#include <memory>
+#include <type_traits>
+
+
 #include <gtest/gtest.h>
 
 
 #include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/dim.hpp>
+#include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
diff --git a/test/utils/executor.hpp b/test/utils/executor.hpp
index c30add581cf..9cb4238aed6 100644
--- a/test/utils/executor.hpp
+++ b/test/utils/executor.hpp
@@ -36,6 +36,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
+#include <memory>
+
+
 #include <gtest/gtest.h>
 
 

From 642c2a22a3070cfe7bf7cf696b3822031a41856c Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 29 Jun 2021 17:25:02 +0200
Subject: [PATCH 0861/1008] remove magic numbers for OpenMP unrolling

---
 omp/base/kernel_launch.hpp | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index 8de7f71581a..b8540b382b2 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -65,21 +65,23 @@ void run_kernel_fixed_cols_impl(std::shared_ptr<const OmpExecutor> exec,
     }
 }
 
-template <size_type remainder_cols, typename KernelFunction,
-          typename... MappedKernelArgs>
+template <size_type remainder_cols, size_type block_size,
+          typename KernelFunction, typename... MappedKernelArgs>
 void run_kernel_blocked_cols_impl(std::shared_ptr<const OmpExecutor> exec,
                                   KernelFunction fn, dim<2> size,
                                   MappedKernelArgs... args)
 {
+    static_assert(remainder_cols < block_size, "remainder too large");
     const auto rows = size[0];
     const auto cols = size[1];
-    const auto rounded_cols = cols / 4 * 4;
+    const auto rounded_cols = cols / block_size * block_size;
     GKO_ASSERT(rounded_cols + remainder_cols == cols);
 #pragma omp parallel for
     for (size_type row = 0; row < rows; row++) {
-        for (size_type base_col = 0; base_col < rounded_cols; base_col += 4) {
+        for (size_type base_col = 0; base_col < rounded_cols;
+             base_col += block_size) {
 #pragma unroll
-            for (size_type i = 0; i < 4; i++) {
+            for (size_type i = 0; i < block_size; i++) {
                 [&]() { fn(row, base_col + i, args...); }();
             }
         }
@@ -96,6 +98,7 @@ void run_kernel_impl(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
 {
     const auto rows = size[0];
     const auto cols = size[1];
+    constexpr size_type block_size = 4;
     if (cols <= 0) {
         return;
     }
@@ -115,21 +118,21 @@ void run_kernel_impl(std::shared_ptr<const OmpExecutor> exec, KernelFunction fn,
         run_kernel_fixed_cols_impl<4>(exec, fn, size, args...);
         return;
     }
-    const auto rem_cols = cols % 4;
+    const auto rem_cols = cols % block_size;
     if (rem_cols == 0) {
-        run_kernel_blocked_cols_impl<0>(exec, fn, size, args...);
+        run_kernel_blocked_cols_impl<0, block_size>(exec, fn, size, args...);
         return;
     }
     if (rem_cols == 1) {
-        run_kernel_blocked_cols_impl<1>(exec, fn, size, args...);
+        run_kernel_blocked_cols_impl<1, block_size>(exec, fn, size, args...);
         return;
     }
     if (rem_cols == 2) {
-        run_kernel_blocked_cols_impl<2>(exec, fn, size, args...);
+        run_kernel_blocked_cols_impl<2, block_size>(exec, fn, size, args...);
         return;
     }
     if (rem_cols == 3) {
-        run_kernel_blocked_cols_impl<3>(exec, fn, size, args...);
+        run_kernel_blocked_cols_impl<3, block_size>(exec, fn, size, args...);
         return;
     }
     // should be unreachable

From 595ea65bb0d949f373cd34bde39c4b487aecb1f8 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 29 Jun 2021 18:49:34 +0200
Subject: [PATCH 0862/1008] fix include position for kernel_launch tests

---
 dev_tools/scripts/config | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/dev_tools/scripts/config b/dev_tools/scripts/config
index 9e333e66762..37656df0872 100644
--- a/dev_tools/scripts/config
+++ b/dev_tools/scripts/config
@@ -24,6 +24,8 @@
     - FixInclude: "core/preconditioner/jacobi_kernels.hpp"
 - "(cuda|hip|dpcpp|omp)/base/kernel_launch\."
     - FixInclude: "common/base/kernel_launch.hpp"
+- "(cuda|hip|dpcpp|omp)/test/base/kernel_launch\."
+    - FixInclude: "common/base/kernel_launch.hpp"
 - "(cuda|hip|dpcpp|omp)/base/kernel_launch_solver\."
     - FixInclude: "common/base/kernel_launch_solver.hpp"
 - "core/test/base/(extended_float|iterator_factory)"

From c2972698b4675dcb2901d11278bdbcb3ef03dd5f Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Tue, 29 Jun 2021 19:36:36 +0200
Subject: [PATCH 0863/1008] fix single-precision dpcpp dense kernel tests

---
 test/matrix/dense_kernels.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/matrix/dense_kernels.cpp b/test/matrix/dense_kernels.cpp
index d699dd93f17..ddccd148d02 100644
--- a/test/matrix/dense_kernels.cpp
+++ b/test/matrix/dense_kernels.cpp
@@ -215,7 +215,7 @@ TEST_F(Dense, CopyRespectsStride)
     set_up_vector_data(3);
     auto stride = dx->get_size()[1] + 1;
     auto result = Mtx::create(exec, dx->get_size(), stride);
-    double val = 123456789.0;
+    vtype val = 1234567.0;
     auto original_data = result->get_values();
     auto padding_ptr = original_data + dx->get_size()[1];
     exec->copy_from(ref.get(), 1, &val, padding_ptr);
@@ -244,7 +244,7 @@ TEST_F(Dense, FillIsEquivalentToRef)
 
 TEST_F(Dense, StridedFillIsEquivalentToRef)
 {
-    using T = double;
+    using T = vtype;
     auto x = gko::initialize<gko::matrix::Dense<T>>(
         4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, ref);
     auto dx = gko::initialize<gko::matrix::Dense<T>>(

From 147b4fdcf36e8417696f6f3da34268e40db3f52f Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 9 Jul 2021 22:45:17 +0200
Subject: [PATCH 0864/1008] skip DPCPP solver apply tests

---
 test/solver/bicg_kernels.cpp     |  6 ++++++
 test/solver/bicgstab_kernels.cpp |  6 ++++++
 test/solver/cg_kernels.cpp       |  3 +++
 test/solver/cgs_kernels.cpp      |  6 ++++++
 test/solver/fcg_kernels.cpp      |  3 +++
 test/solver/ir_kernels.cpp       | 12 ++++++++++++
 6 files changed, 36 insertions(+)

diff --git a/test/solver/bicg_kernels.cpp b/test/solver/bicg_kernels.cpp
index 06b1726cdc4..b19f61bbd55 100644
--- a/test/solver/bicg_kernels.cpp
+++ b/test/solver/bicg_kernels.cpp
@@ -267,6 +267,9 @@ TEST_F(Bicg, BicgStep2IsEquivalentToRef)
 
 TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
 {
+    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
+        GTEST_SKIP();
+    }
     auto mtx = gen_mtx(50, 50, 53);
     gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3, 5);
@@ -305,6 +308,9 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
 
 TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
 {
+    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
+        GTEST_SKIP();
+    }
     auto x = gen_mtx(36, 1, 2);
     auto b = gen_mtx(36, 1, 3);
     auto d_x = Mtx::create(exec);
diff --git a/test/solver/bicgstab_kernels.cpp b/test/solver/bicgstab_kernels.cpp
index 9b5a8457cc6..c0e75432fc9 100644
--- a/test/solver/bicgstab_kernels.cpp
+++ b/test/solver/bicgstab_kernels.cpp
@@ -320,6 +320,9 @@ TEST_F(Bicgstab, BicgstabStep3IsEquivalentToRef)
 
 TEST_F(Bicgstab, BicgstabApplyOneRHSIsEquivalentToRef)
 {
+    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
+        GTEST_SKIP();
+    }
     int m = 123;
     int n = 1;
     auto ref_solver = ref_bicgstab_factory->generate(mtx);
@@ -341,6 +344,9 @@ TEST_F(Bicgstab, BicgstabApplyOneRHSIsEquivalentToRef)
 
 TEST_F(Bicgstab, BicgstabApplyMultipleRHSIsEquivalentToRef)
 {
+    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
+        GTEST_SKIP();
+    }
     int m = 123;
     int n = 16;
     auto exec_solver = exec_bicgstab_factory->generate(d_mtx);
diff --git a/test/solver/cg_kernels.cpp b/test/solver/cg_kernels.cpp
index edd9d33a78d..3125003474c 100644
--- a/test/solver/cg_kernels.cpp
+++ b/test/solver/cg_kernels.cpp
@@ -223,6 +223,9 @@ TEST_F(Cg, CgStep2IsEquivalentToRef)
 
 TEST_F(Cg, ApplyIsEquivalentToRef)
 {
+    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
+        GTEST_SKIP();
+    }
     auto mtx = gen_mtx(50, 50, 53);
     gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3, 5);
diff --git a/test/solver/cgs_kernels.cpp b/test/solver/cgs_kernels.cpp
index cd4f0c10437..bd5400e77c3 100644
--- a/test/solver/cgs_kernels.cpp
+++ b/test/solver/cgs_kernels.cpp
@@ -309,6 +309,9 @@ TEST_F(Cgs, CgsStep3IsEquivalentToRef)
 
 TEST_F(Cgs, CgsApplyOneRHSIsEquivalentToRef)
 {
+    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
+        GTEST_SKIP();
+    }
     int m = 123;
     int n = 1;
     auto ref_solver = ref_cgs_factory->generate(mtx);
@@ -330,6 +333,9 @@ TEST_F(Cgs, CgsApplyOneRHSIsEquivalentToRef)
 
 TEST_F(Cgs, CgsApplyMultipleRHSIsEquivalentToRef)
 {
+    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
+        GTEST_SKIP();
+    }
     int m = 123;
     int n = 16;
     auto exec_solver = exec_cgs_factory->generate(d_mtx);
diff --git a/test/solver/fcg_kernels.cpp b/test/solver/fcg_kernels.cpp
index 46fbb30cdf6..837a5cfa7e9 100644
--- a/test/solver/fcg_kernels.cpp
+++ b/test/solver/fcg_kernels.cpp
@@ -234,6 +234,9 @@ TEST_F(Fcg, FcgStep2IsEquivalentToRef)
 
 TEST_F(Fcg, ApplyIsEquivalentToRef)
 {
+    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
+        GTEST_SKIP();
+    }
     auto mtx = gen_mtx(50, 50, 53);
     gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3, 4);
diff --git a/test/solver/ir_kernels.cpp b/test/solver/ir_kernels.cpp
index 769ce6881be..3844044138b 100644
--- a/test/solver/ir_kernels.cpp
+++ b/test/solver/ir_kernels.cpp
@@ -111,6 +111,9 @@ TEST_F(Ir, InitializeIsEquivalentToRef)
 
 TEST_F(Ir, ApplyIsEquivalentToRef)
 {
+    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
+        GTEST_SKIP();
+    }
     auto mtx = gen_mtx(50, 50, 52);
     auto x = gen_mtx(50, 3, 8);
     auto b = gen_mtx(50, 3, 5);
@@ -142,6 +145,9 @@ TEST_F(Ir, ApplyIsEquivalentToRef)
 
 TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef)
 {
+    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
+        GTEST_SKIP();
+    }
     auto mtx = gen_mtx(50, 50, 54);
     auto x = gen_mtx(50, 3, 6);
     auto b = gen_mtx(50, 3, 10);
@@ -186,6 +192,9 @@ TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef)
 
 TEST_F(Ir, RichardsonApplyIsEquivalentToRef)
 {
+    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
+        GTEST_SKIP();
+    }
     auto mtx = gen_mtx(50, 50, 54);
     auto x = gen_mtx(50, 3, 4);
     auto b = gen_mtx(50, 3, 3);
@@ -219,6 +228,9 @@ TEST_F(Ir, RichardsonApplyIsEquivalentToRef)
 
 TEST_F(Ir, RichardsonApplyWithIterativeInnerSolverIsEquivalentToRef)
 {
+    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
+        GTEST_SKIP();
+    }
     auto mtx = gen_mtx(50, 50, 52);
     auto x = gen_mtx(50, 3, 4);
     auto b = gen_mtx(50, 3, 7);

From 9d8b6eae730e7483115cbeccc517bbbeac57ae43 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 6 Jul 2021 11:15:10 +0200
Subject: [PATCH 0865/1008] Add ostream overload for dim.

---
 core/test/base/dim.cpp           | 20 ++++++++++++++++++++
 include/ginkgo/core/base/dim.hpp | 20 ++++++++++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/core/test/base/dim.cpp b/core/test/base/dim.cpp
index 6d48bcab794..c6340f38335 100644
--- a/core/test/base/dim.cpp
+++ b/core/test/base/dim.cpp
@@ -117,6 +117,26 @@ TEST(Dim, ConvertsToBool)
 }
 
 
+TEST(Dim, CanAppendToStream1)
+{
+    gko::dim<2> d2{2, 3};
+
+    std::ostringstream os;
+    os << d2;
+    ASSERT_EQ(os.str(), "(2 , 3)");
+}
+
+
+TEST(Dim, CanAppendToStream2)
+{
+    gko::dim<3> d2{2, 3, 4};
+
+    std::ostringstream os;
+    os << d2;
+    ASSERT_EQ(os.str(), "(2 , (3 , 4))");
+}
+
+
 TEST(Dim, EqualityReturnsTrueWhenEqual)
 {
     ASSERT_TRUE(gko::dim<2>(2, 3) == gko::dim<2>(2, 3));
diff --git a/include/ginkgo/core/base/dim.hpp b/include/ginkgo/core/base/dim.hpp
index 0881d954aa0..b094c479506 100644
--- a/include/ginkgo/core/base/dim.hpp
+++ b/include/ginkgo/core/base/dim.hpp
@@ -152,6 +152,20 @@ struct dim {
         return dim(x.first_ * y.first_, x.rest_ * y.rest_);
     }
 
+    /**
+     * A stream operator overload for dim
+     *
+     * @param os  stream object
+     * @param x  dim object
+     *
+     * @return a stream object appended with the dim output
+     */
+    friend GKO_ATTRIBUTES std::ostream &operator<<(std::ostream &os,
+                                                   const dim &x)
+    {
+        return os << "(" << x.first_ << " , " << x.rest_ << ")";
+    }
+
 private:
     constexpr GKO_ATTRIBUTES dim(const dimension_type first,
                                  dim<dimensionality - 1> rest)
@@ -200,6 +214,12 @@ struct dim<1u, DimensionType> {
         return dim(x.first_ * y.first_);
     }
 
+    friend GKO_ATTRIBUTES std::ostream &operator<<(std::ostream &os,
+                                                   const dim &x)
+    {
+        return os << x.first_;
+    }
+
 private:
     dimension_type first_;
 };

From 9a06059143cbe7d1b5070b27746823650adbc77d Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Tue, 6 Jul 2021 11:15:57 +0200
Subject: [PATCH 0866/1008] Add span length getter

---
 core/test/base/range.cpp           | 15 +++++++++++----
 include/ginkgo/core/base/range.hpp | 11 +++++++++++
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/core/test/base/range.cpp b/core/test/base/range.cpp
index fbcc55b4f09..c15c9c412da 100644
--- a/core/test/base/range.cpp
+++ b/core/test/base/range.cpp
@@ -60,6 +60,13 @@ TEST(Span, CreatesPoint)
 }
 
 
+TEST(Span, KnowsItsLength)
+{
+    gko::span s{3, 5};
+    ASSERT_EQ(2, s.length());
+}
+
+
 TEST(Span, LessThanEvaluatesToTrue)
 {
     ASSERT_TRUE(gko::span(2, 3) < gko::span(4, 7));
@@ -645,7 +652,7 @@ TEST(Range, DividesScalarAndRange)
 }
 
 
-TEST(Range, AddsRangeAndSclar)
+TEST(Range, AddsRangeAndScalar)
 {
     dummy_range r{5u, 1, 2};
 
@@ -656,7 +663,7 @@ TEST(Range, AddsRangeAndSclar)
 }
 
 
-TEST(Range, SubtractsRangeAndSclar)
+TEST(Range, SubtractsRangeAndScalar)
 {
     dummy_range r{5u, 1, 2};
 
@@ -667,7 +674,7 @@ TEST(Range, SubtractsRangeAndSclar)
 }
 
 
-TEST(Range, MultipliesRangeAndSclar)
+TEST(Range, MultipliesRangeAndScalar)
 {
     dummy_range r{5u, 1, 2};
 
@@ -678,7 +685,7 @@ TEST(Range, MultipliesRangeAndSclar)
 }
 
 
-TEST(Range, DividesRangeAndSclar)
+TEST(Range, DividesRangeAndScalar)
 {
     dummy_range r{5u, 1, 2};
 
diff --git a/include/ginkgo/core/base/range.hpp b/include/ginkgo/core/base/range.hpp
index e0ea8bcf284..d4a2cb3d3ac 100644
--- a/include/ginkgo/core/base/range.hpp
+++ b/include/ginkgo/core/base/range.hpp
@@ -101,6 +101,17 @@ struct span {
      */
     constexpr bool is_valid() const { return begin < end; }
 
+    /**
+     * Returns the length of a span.
+     *
+     * @return `this->end - this->begin`
+     */
+    constexpr size_type length() const
+    {
+        GKO_ASSERT(is_valid());
+        return end - begin;
+    }
+
     /**
      * Beginning of the span.
      */

From efb1a88ee383100e4361bd13345a5f119b35e91e Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Wed, 7 Jul 2021 11:11:54 +0200
Subject: [PATCH 0867/1008] A cleaner ostream output.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 core/test/base/dim.cpp           |  4 ++--
 include/ginkgo/core/base/dim.hpp | 24 ++++++++++++++++++++++--
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/core/test/base/dim.cpp b/core/test/base/dim.cpp
index c6340f38335..f78191967d9 100644
--- a/core/test/base/dim.cpp
+++ b/core/test/base/dim.cpp
@@ -123,7 +123,7 @@ TEST(Dim, CanAppendToStream1)
 
     std::ostringstream os;
     os << d2;
-    ASSERT_EQ(os.str(), "(2 , 3)");
+    ASSERT_EQ(os.str(), "(2, 3)");
 }
 
 
@@ -133,7 +133,7 @@ TEST(Dim, CanAppendToStream2)
 
     std::ostringstream os;
     os << d2;
-    ASSERT_EQ(os.str(), "(2 , (3 , 4))");
+    ASSERT_EQ(os.str(), "(2, 3, 4)");
 }
 
 
diff --git a/include/ginkgo/core/base/dim.hpp b/include/ginkgo/core/base/dim.hpp
index b094c479506..1ec13475831 100644
--- a/include/ginkgo/core/base/dim.hpp
+++ b/include/ginkgo/core/base/dim.hpp
@@ -34,6 +34,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_PUBLIC_CORE_BASE_DIM_HPP_
 
 
+#include <iostream>
+
+
 #include <ginkgo/core/base/types.hpp>
 
 
@@ -51,6 +54,7 @@ namespace gko {
 template <size_type Dimensionality, typename DimensionType = size_type>
 struct dim {
     static constexpr size_type dimensionality = Dimensionality;
+    friend class dim<dimensionality + 1>;
 
     using dimension_type = DimensionType;
 
@@ -163,10 +167,20 @@ struct dim {
     friend GKO_ATTRIBUTES std::ostream &operator<<(std::ostream &os,
                                                    const dim &x)
     {
-        return os << "(" << x.first_ << " , " << x.rest_ << ")";
+        os << "(";
+        x.print_to(os);
+        os << ")";
+        return os;
     }
 
 private:
+    void inline print_to(std::ostream &os) const
+    {
+        os << first_ << ", ";
+        rest_.print_to(os);
+    }
+
+
     constexpr GKO_ATTRIBUTES dim(const dimension_type first,
                                  dim<dimensionality - 1> rest)
         : first_{first}, rest_{rest}
@@ -181,6 +195,7 @@ struct dim {
 template <typename DimensionType>
 struct dim<1u, DimensionType> {
     static constexpr size_type dimensionality = 1u;
+    friend class dim<2>;
 
     using dimension_type = DimensionType;
 
@@ -217,10 +232,15 @@ struct dim<1u, DimensionType> {
     friend GKO_ATTRIBUTES std::ostream &operator<<(std::ostream &os,
                                                    const dim &x)
     {
-        return os << x.first_;
+        os << "(";
+        x.print_to(os);
+        os << ")";
+        return os;
     }
 
 private:
+    void inline print_to(std::ostream &os) const { os << first_; }
+
     dimension_type first_;
 };
 

From 804382572553b11bf2cc9b5919922555308189bf Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 8 Jul 2021 10:54:33 +0200
Subject: [PATCH 0868/1008] Review update

---
 include/ginkgo/core/base/dim.hpp   | 6 ++----
 include/ginkgo/core/base/range.hpp | 4 ++--
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/include/ginkgo/core/base/dim.hpp b/include/ginkgo/core/base/dim.hpp
index 1ec13475831..726b899ba60 100644
--- a/include/ginkgo/core/base/dim.hpp
+++ b/include/ginkgo/core/base/dim.hpp
@@ -164,8 +164,7 @@ struct dim {
      *
      * @return a stream object appended with the dim output
      */
-    friend GKO_ATTRIBUTES std::ostream &operator<<(std::ostream &os,
-                                                   const dim &x)
+    friend std::ostream &operator<<(std::ostream &os, const dim &x)
     {
         os << "(";
         x.print_to(os);
@@ -229,8 +228,7 @@ struct dim<1u, DimensionType> {
         return dim(x.first_ * y.first_);
     }
 
-    friend GKO_ATTRIBUTES std::ostream &operator<<(std::ostream &os,
-                                                   const dim &x)
+    friend std::ostream &operator<<(std::ostream &os, const dim &x)
     {
         os << "(";
         x.print_to(os);
diff --git a/include/ginkgo/core/base/range.hpp b/include/ginkgo/core/base/range.hpp
index d4a2cb3d3ac..3942a1d218d 100644
--- a/include/ginkgo/core/base/range.hpp
+++ b/include/ginkgo/core/base/range.hpp
@@ -99,14 +99,14 @@ struct span {
      *
      * @return true if and only if `this->begin < this->end`
      */
-    constexpr bool is_valid() const { return begin < end; }
+    GKO_ATTRIBUTES constexpr bool is_valid() const { return begin < end; }
 
     /**
      * Returns the length of a span.
      *
      * @return `this->end - this->begin`
      */
-    constexpr size_type length() const
+    GKO_ATTRIBUTES constexpr size_type length() const
     {
         GKO_ASSERT(is_valid());
         return end - begin;

From 5758d410e4a73bd1e55422f41215b092ebbb42c4 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 7 Jul 2021 13:34:12 +0200
Subject: [PATCH 0869/1008] use consistent matrix index type in benchmark

---
 benchmark/conversions/conversions.cpp         |   6 +-
 .../matrix_generator/matrix_generator.cpp     |  12 +-
 benchmark/preconditioner/preconditioner.cpp   |   2 +-
 benchmark/solver/solver.cpp                   |   2 +-
 benchmark/spmv/spmv.cpp                       |   4 +-
 benchmark/utils/cuda_linops.hpp               |  21 +-
 benchmark/utils/formats.hpp                   |  41 ++--
 benchmark/utils/general.hpp                   |   7 +-
 benchmark/utils/hip_linops.hip.hpp            |  10 +-
 benchmark/utils/preconditioners.hpp           | 182 +++++++++---------
 benchmark/utils/types.hpp                     |   3 +
 11 files changed, 150 insertions(+), 140 deletions(-)

diff --git a/benchmark/conversions/conversions.cpp b/benchmark/conversions/conversions.cpp
index 18313bf26db..111470a6f0f 100644
--- a/benchmark/conversions/conversions.cpp
+++ b/benchmark/conversions/conversions.cpp
@@ -69,7 +69,7 @@ void convert_matrix(const gko::LinOp *matrix_from, const char *format_to,
         add_or_set_member(conversion_case, conversion_name,
                           rapidjson::Value(rapidjson::kObjectType), allocator);
 
-        gko::matrix_data<etype> data{gko::dim<2>{1, 1}, 1};
+        gko::matrix_data<etype, itype> data{gko::dim<2>{1, 1}, 1};
         auto matrix_to =
             share(formats::matrix_factory.at(format_to)(exec, data));
 
@@ -148,9 +148,9 @@ int main(int argc, char *argv[])
 
         std::clog << "Running test case: " << test_case << std::endl;
         std::ifstream mtx_fd(test_case["filename"].GetString());
-        gko::matrix_data<etype> data;
+        gko::matrix_data<etype, itype> data;
         try {
-            data = gko::read_raw<etype>(mtx_fd);
+            data = gko::read_raw<etype, itype>(mtx_fd);
         } catch (std::exception &e) {
             std::cerr << "Error setting up matrix data, what(): " << e.what()
                       << std::endl;
diff --git a/benchmark/matrix_generator/matrix_generator.cpp b/benchmark/matrix_generator/matrix_generator.cpp
index 308c57de421..241678118f4 100644
--- a/benchmark/matrix_generator/matrix_generator.cpp
+++ b/benchmark/matrix_generator/matrix_generator.cpp
@@ -96,13 +96,13 @@ void validate_option_object(const rapidjson::Value &value)
 }
 
 
-using generator_function =
-    std::function<gko::matrix_data<etype>(rapidjson::Value &, std::ranlux24 &)>;
+using generator_function = std::function<gko::matrix_data<etype, itype>(
+    rapidjson::Value &, std::ranlux24 &)>;
 
 
 // matrix generators
-gko::matrix_data<etype> generate_block_diagonal(rapidjson::Value &config,
-                                                std::ranlux24 &engine)
+gko::matrix_data<etype, itype> generate_block_diagonal(rapidjson::Value &config,
+                                                       std::ranlux24 &engine)
 {
     if (!config.HasMember("num_blocks") || !config["num_blocks"].IsUint() ||
         !config.HasMember("block_size") || !config["block_size"].IsUint()) {
@@ -110,10 +110,10 @@ gko::matrix_data<etype> generate_block_diagonal(rapidjson::Value &config,
     }
     auto num_blocks = config["num_blocks"].GetUint();
     auto block_size = config["block_size"].GetUint();
-    auto block = gko::matrix_data<etype>(
+    auto block = gko::matrix_data<etype, itype>(
         gko::dim<2>(block_size),
         std::uniform_real_distribution<rc_etype>(-1.0, 1.0), engine);
-    return gko::matrix_data<etype>::diag(num_blocks, block);
+    return gko::matrix_data<etype, itype>::diag(num_blocks, block);
 }
 
 
diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp
index 002faeeddff..2225af0ee82 100644
--- a/benchmark/preconditioner/preconditioner.cpp
+++ b/benchmark/preconditioner/preconditioner.cpp
@@ -296,7 +296,7 @@ int main(int argc, char *argv[])
             std::clog << "Running test case: " << test_case << std::endl;
 
             std::ifstream mtx_fd(test_case["filename"].GetString());
-            auto data = gko::read_raw<etype>(mtx_fd);
+            auto data = gko::read_raw<etype, itype>(mtx_fd);
 
             auto system_matrix =
                 share(formats::matrix_factory.at(FLAGS_formats)(exec, data));
diff --git a/benchmark/solver/solver.cpp b/benchmark/solver/solver.cpp
index e4df2948cb9..77d72c94bdf 100644
--- a/benchmark/solver/solver.cpp
+++ b/benchmark/solver/solver.cpp
@@ -609,7 +609,7 @@ int main(int argc, char *argv[])
                     {std::numeric_limits<rc_etype>::quiet_NaN()}, exec);
                 x = gko::initialize<Vec>({0.0}, exec);
             } else {
-                auto data = gko::read_raw<etype>(mtx_fd);
+                auto data = gko::read_raw<etype, itype>(mtx_fd);
                 system_matrix = share(formats::matrix_factory.at(
                     test_case["optimal"]["spmv"].GetString())(exec, data));
                 if (test_case.HasMember("rhs")) {
diff --git a/benchmark/spmv/spmv.cpp b/benchmark/spmv/spmv.cpp
index 6d21ddd88e6..ac7562e4ed6 100644
--- a/benchmark/spmv/spmv.cpp
+++ b/benchmark/spmv/spmv.cpp
@@ -63,7 +63,7 @@ DEFINE_uint32(nrhs, 1, "The number of right hand sides");
 // This function supposes that management of `FLAGS_overwrite` is done before
 // calling it
 void apply_spmv(const char *format_name, std::shared_ptr<gko::Executor> exec,
-                const gko::matrix_data<etype> &data, const vec<etype> *b,
+                const gko::matrix_data<etype, itype> &data, const vec<etype> *b,
                 const vec<etype> *x, const vec<etype> *answer,
                 rapidjson::Value &test_case,
                 rapidjson::MemoryPoolAllocator<> &allocator)
@@ -212,7 +212,7 @@ int main(int argc, char *argv[])
             }
             std::clog << "Running test case: " << test_case << std::endl;
             std::ifstream mtx_fd(test_case["filename"].GetString());
-            auto data = gko::read_raw<etype>(mtx_fd);
+            auto data = gko::read_raw<etype, itype>(mtx_fd);
 
             auto nrhs = FLAGS_nrhs;
             auto b = create_matrix<etype>(exec, gko::dim<2>{data.size[1], nrhs},
diff --git a/benchmark/utils/cuda_linops.hpp b/benchmark/utils/cuda_linops.hpp
index 61dbfc23396..d64ec8b3ebd 100644
--- a/benchmark/utils/cuda_linops.hpp
+++ b/benchmark/utils/cuda_linops.hpp
@@ -685,11 +685,11 @@ class CuspGenericCoo
 
 
 // Some shortcuts
-using cusp_csrex = detail::CuspCsrEx<etype>;
+using cusp_csrex = detail::CuspCsrEx<etype, itype>;
 #if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
-using cusp_csr = detail::CuspCsr<etype>;
-using cusp_csrmp = detail::CuspCsrmp<etype>;
-using cusp_csrmm = detail::CuspCsrmm<etype>;
+using cusp_csr = detail::CuspCsr<etype, itype>;
+using cusp_csrmp = detail::CuspCsrmp<etype, itype>;
+using cusp_csrmm = detail::CuspCsrmm<etype, itype>;
 #endif  // defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
 
 
@@ -698,10 +698,9 @@ using cusp_csrmm = detail::CuspCsrmm<etype>;
      ((CUDA_VERSION >= 10020) && !(defined(_WIN32) || defined(__CYGWIN__))))
 
 
-using cusp_gcsr = detail::CuspGenericCsr<etype>;
-using cusp_gcsr2 =
-    detail::CuspGenericCsr<etype, gko::int32, CUSPARSE_CSRMV_ALG2>;
-using cusp_gcoo = detail::CuspGenericCoo<etype>;
+using cusp_gcsr = detail::CuspGenericCsr<etype, itype>;
+using cusp_gcsr2 = detail::CuspGenericCsr<etype, itype, CUSPARSE_CSRMV_ALG2>;
+using cusp_gcoo = detail::CuspGenericCoo<etype, itype>;
 
 
 #endif  // defined(CUDA_VERSION) && (CUDA_VERSION >= 11000 || ((CUDA_VERSION >=
@@ -710,10 +709,10 @@ using cusp_gcoo = detail::CuspGenericCoo<etype>;
 
 #if defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
 using cusp_coo =
-    detail::CuspHybrid<etype, gko::int32, CUSPARSE_HYB_PARTITION_USER, 0>;
+    detail::CuspHybrid<etype, itype, CUSPARSE_HYB_PARTITION_USER, 0>;
 using cusp_ell =
-    detail::CuspHybrid<etype, gko::int32, CUSPARSE_HYB_PARTITION_MAX, 0>;
-using cusp_hybrid = detail::CuspHybrid<etype>;
+    detail::CuspHybrid<etype, itype, CUSPARSE_HYB_PARTITION_MAX, 0>;
+using cusp_hybrid = detail::CuspHybrid<etype, itype>;
 #endif  // defined(CUDA_VERSION) && (CUDA_VERSION < 11000)
 
 
diff --git a/benchmark/utils/formats.hpp b/benchmark/utils/formats.hpp
index bbce4f25cf2..8ae4ad45e7a 100644
--- a/benchmark/utils/formats.hpp
+++ b/benchmark/utils/formats.hpp
@@ -162,8 +162,8 @@ namespace formats {
 
 
 // some shortcuts
-using hybrid = gko::matrix::Hybrid<etype>;
-using csr = gko::matrix::Csr<etype>;
+using hybrid = gko::matrix::Hybrid<etype, itype>;
+using csr = gko::matrix::Csr<etype, itype>;
 
 /**
  * Creates a Ginkgo matrix from the intermediate data representation format
@@ -179,7 +179,7 @@ using csr = gko::matrix::Csr<etype>;
 template <typename MatrixType>
 std::unique_ptr<MatrixType> read_matrix_from_data(
     std::shared_ptr<const gko::Executor> exec,
-    const gko::matrix_data<etype> &data)
+    const gko::matrix_data<etype, itype> &data)
 {
     auto mat = MatrixType::create(std::move(exec));
     mat->read(data);
@@ -213,7 +213,7 @@ std::shared_ptr<csr::strategy_type> create_gpu_strategy(
  *
  * @throws gko::Error if the imbalance limit is exceeded
  */
-void check_ell_admissibility(const gko::matrix_data<etype> &data)
+void check_ell_admissibility(const gko::matrix_data<etype, itype> &data)
 {
     if (data.size[0] == 0 || FLAGS_ell_imbalance_limit < 0) {
         return;
@@ -237,23 +237,24 @@ void check_ell_admissibility(const gko::matrix_data<etype> &data)
  *
  * @param MATRIX_TYPE  the Ginkgo matrix type (such as `gko::matrix::Csr<>`)
  */
-#define READ_MATRIX(MATRIX_TYPE, ...)                                         \
-    [](std::shared_ptr<const gko::Executor> exec,                             \
-       const gko::matrix_data<etype> &data) -> std::unique_ptr<MATRIX_TYPE> { \
-        auto mat = MATRIX_TYPE::create(std::move(exec), __VA_ARGS__);         \
-        mat->read(data);                                                      \
-        return mat;                                                           \
+#define READ_MATRIX(MATRIX_TYPE, ...)                                 \
+    [](std::shared_ptr<const gko::Executor> exec,                     \
+       const gko::matrix_data<etype, itype> &data)                    \
+        -> std::unique_ptr<MATRIX_TYPE> {                             \
+        auto mat = MATRIX_TYPE::create(std::move(exec), __VA_ARGS__); \
+        mat->read(data);                                              \
+        return mat;                                                   \
     }
 
 
 // clang-format off
 const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
                                 std::shared_ptr<const gko::Executor>,
-                                const gko::matrix_data<etype> &)>>
+                                const gko::matrix_data<etype, itype> &)>>
     matrix_factory{
         {"csr",
          [](std::shared_ptr<const gko::Executor> exec,
-            const gko::matrix_data<etype> &data) -> std::unique_ptr<csr> {
+            const gko::matrix_data<etype, itype> &data) -> std::unique_ptr<csr> {
             auto mat =
                 csr::create(exec, create_gpu_strategy<csr::automatical>(exec));
             mat->read(data);
@@ -261,7 +262,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
          }},
         {"csri",
          [](std::shared_ptr<const gko::Executor> exec,
-            const gko::matrix_data<etype> &data) -> std::unique_ptr<csr> {
+            const gko::matrix_data<etype, itype> &data) -> std::unique_ptr<csr> {
              auto mat = csr::create(
                  exec, create_gpu_strategy<csr::load_balance>(exec));
              mat->read(data);
@@ -269,19 +270,19 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
          }},
         {"csrm", READ_MATRIX(csr, std::make_shared<csr::merge_path>())},
         {"csrc", READ_MATRIX(csr, std::make_shared<csr::classical>())},
-        {"coo", read_matrix_from_data<gko::matrix::Coo<etype>>},
+        {"coo", read_matrix_from_data<gko::matrix::Coo<etype, itype>>},
         {"ell", [](std::shared_ptr<const gko::Executor> exec,
-            const gko::matrix_data<etype> &data) {
+            const gko::matrix_data<etype, itype> &data) {
              check_ell_admissibility(data);
-             auto mat = gko::matrix::Ell<etype>::create(exec);
+             auto mat = gko::matrix::Ell<etype, itype>::create(exec);
              mat->read(data);
              return mat;
          }},
         {"ell-mixed",
          [](std::shared_ptr<const gko::Executor> exec,
-            const gko::matrix_data<etype> &data) {
+            const gko::matrix_data<etype, itype> &data) {
              check_ell_admissibility(data);
-             gko::matrix_data<gko::next_precision<etype>> conv_data;
+             gko::matrix_data<gko::next_precision<etype>, itype> conv_data;
              conv_data.size = data.size;
              conv_data.nonzeros.resize(data.nonzeros.size());
              auto it = conv_data.nonzeros.begin();
@@ -291,7 +292,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
                  it->value = el.value;
                  ++it;
              }
-             auto mat = gko::matrix::Ell<gko::next_precision<etype>>::create(
+             auto mat = gko::matrix::Ell<gko::next_precision<etype>, itype>::create(
                  std::move(exec));
              mat->read(conv_data);
              return mat;
@@ -352,7 +353,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
         {"hybridminstorage",
          READ_MATRIX(hybrid,
                      std::make_shared<hybrid::minimal_storage_limit>())},
-        {"sellp", read_matrix_from_data<gko::matrix::Sellp<etype>>}
+        {"sellp", read_matrix_from_data<gko::matrix::Sellp<etype, itype>>}
 };
 // clang-format on
 
diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp
index dca22e7fe27..d4f6a6853a8 100644
--- a/benchmark/utils/general.hpp
+++ b/benchmark/utils/general.hpp
@@ -59,6 +59,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "benchmark/utils/timer.hpp"
+#include "benchmark/utils/types.hpp"
 
 
 // Global command-line arguments
@@ -366,7 +367,7 @@ std::unique_ptr<vec<ValueType>> create_matrix(
     ValueType value)
 {
     auto res = vec<ValueType>::create(exec);
-    res->read(gko::matrix_data<ValueType>(size, value));
+    res->read(gko::matrix_data<ValueType, itype>(size, value));
     return res;
 }
 
@@ -378,7 +379,7 @@ std::unique_ptr<vec<ValueType>> create_matrix(
     RandomEngine &engine)
 {
     auto res = vec<ValueType>::create(exec);
-    res->read(gko::matrix_data<ValueType>(
+    res->read(gko::matrix_data<ValueType, itype>(
         size,
         std::uniform_real_distribution<gko::remove_complex<ValueType>>(-1.0,
                                                                        1.0),
@@ -393,7 +394,7 @@ std::unique_ptr<vec<ValueType>> create_vector(
     std::shared_ptr<const gko::Executor> exec, gko::size_type size)
 {
     auto res = vec<ValueType>::create(exec);
-    res->read(gko::matrix_data<ValueType>(gko::dim<2>{size, 1}));
+    res->read(gko::matrix_data<ValueType, itype>(gko::dim<2>{size, 1}));
     return res;
 }
 
diff --git a/benchmark/utils/hip_linops.hip.hpp b/benchmark/utils/hip_linops.hip.hpp
index b46d85646f4..f9b2066aab5 100644
--- a/benchmark/utils/hip_linops.hip.hpp
+++ b/benchmark/utils/hip_linops.hip.hpp
@@ -322,14 +322,14 @@ class HipspHybrid
 
 
 // Some shortcuts
-using hipsp_csr = detail::HipspCsr<etype>;
-using hipsp_csrmm = detail::HipspCsrmm<etype>;
+using hipsp_csr = detail::HipspCsr<etype, itype>;
+using hipsp_csrmm = detail::HipspCsrmm<etype, itype>;
 
 
 using hipsp_coo =
-    detail::HipspHybrid<etype, gko::int32, HIPSPARSE_HYB_PARTITION_USER, 0>;
+    detail::HipspHybrid<etype, itype, HIPSPARSE_HYB_PARTITION_USER, 0>;
 using hipsp_ell =
-    detail::HipspHybrid<etype, gko::int32, HIPSPARSE_HYB_PARTITION_MAX, 0>;
-using hipsp_hybrid = detail::HipspHybrid<etype>;
+    detail::HipspHybrid<etype, itype, HIPSPARSE_HYB_PARTITION_MAX, 0>;
+using hipsp_hybrid = detail::HipspHybrid<etype, itype>;
 
 #endif  // GKO_BENCHMARK_UTILS_HIP_LINOPS_HIP_HPP_
diff --git a/benchmark/utils/preconditioners.hpp b/benchmark/utils/preconditioners.hpp
index 6af0d2d7a12..b9f08348b6d 100644
--- a/benchmark/utils/preconditioners.hpp
+++ b/benchmark/utils/preconditioners.hpp
@@ -105,7 +105,7 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
          }},
         {"jacobi",
          [](std::shared_ptr<const gko::Executor> exec) {
-             return gko::preconditioner::Jacobi<etype>::build()
+             return gko::preconditioner::Jacobi<etype, itype>::build()
                  .with_max_block_size(FLAGS_jacobi_max_block_size)
                  .with_storage_optimization(
                      parse_storage_optimization(FLAGS_jacobi_storage))
@@ -116,90 +116,91 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"paric",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact =
-                 gko::share(gko::factorization::ParIc<etype>::build()
+                 gko::share(gko::factorization::ParIc<etype, itype>::build()
                                 .with_iterations(FLAGS_parilu_iterations)
                                 .with_skip_sorting(true)
                                 .on(exec));
-             return gko::preconditioner::Ic<
-                        gko::solver::LowerTrs<etype>>::build()
+             return gko::preconditioner::Ic<gko::solver::LowerTrs<etype, itype>,
+                                            itype>::build()
                  .with_factorization_factory(fact)
                  .on(exec);
          }},
         {"parict",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact = gko::share(
-                 gko::factorization::ParIct<etype>::build()
+                 gko::factorization::ParIct<etype, itype>::build()
                      .with_iterations(FLAGS_parilu_iterations)
                      .with_approximate_select(FLAGS_parilut_approx_select)
                      .with_fill_in_limit(FLAGS_parilut_limit)
                      .with_skip_sorting(true)
                      .on(exec));
-             return gko::preconditioner::Ilu<
-                        gko::solver::LowerTrs<etype>,
-                        gko::solver::UpperTrs<etype>>::build()
-                 .with_factorization_factory(fact)
-                 .on(exec);
+             return gko::preconditioner::
+                 Ilu<gko::solver::LowerTrs<etype, itype>,
+                     gko::solver::UpperTrs<etype, itype>, false, itype>::build()
+                     .with_factorization_factory(fact)
+                     .on(exec);
          }},
         {"parilu",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact =
-                 gko::share(gko::factorization::ParIlu<etype>::build()
+                 gko::share(gko::factorization::ParIlu<etype, itype>::build()
                                 .with_iterations(FLAGS_parilu_iterations)
                                 .with_skip_sorting(true)
                                 .on(exec));
-             return gko::preconditioner::Ilu<
-                        gko::solver::LowerTrs<etype>,
-                        gko::solver::UpperTrs<etype>>::build()
-                 .with_factorization_factory(fact)
-                 .on(exec);
+             return gko::preconditioner::
+                 Ilu<gko::solver::LowerTrs<etype, itype>,
+                     gko::solver::UpperTrs<etype, itype>, false, itype>::build()
+                     .with_factorization_factory(fact)
+                     .on(exec);
          }},
         {"parilut",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact = gko::share(
-                 gko::factorization::ParIlut<etype>::build()
+                 gko::factorization::ParIlut<etype, itype>::build()
                      .with_iterations(FLAGS_parilu_iterations)
                      .with_approximate_select(FLAGS_parilut_approx_select)
                      .with_fill_in_limit(FLAGS_parilut_limit)
                      .with_skip_sorting(true)
                      .on(exec));
-             return gko::preconditioner::Ilu<
-                        gko::solver::LowerTrs<etype>,
-                        gko::solver::UpperTrs<etype>>::build()
-                 .with_factorization_factory(fact)
-                 .on(exec);
+             return gko::preconditioner::
+                 Ilu<gko::solver::LowerTrs<etype, itype>,
+                     gko::solver::UpperTrs<etype, itype>, false, itype>::build()
+                     .with_factorization_factory(fact)
+                     .on(exec);
          }},
         {"ic",
          [](std::shared_ptr<const gko::Executor> exec) {
-             auto fact =
-                 gko::share(gko::factorization::Ic<etype>::build().on(exec));
-             return gko::preconditioner::Ic<
-                        gko::solver::LowerTrs<etype>>::build()
+             auto fact = gko::share(
+                 gko::factorization::Ic<etype, itype>::build().on(exec));
+             return gko::preconditioner::Ic<gko::solver::LowerTrs<etype, itype>,
+                                            itype>::build()
                  .with_factorization_factory(fact)
                  .on(exec);
          }},
         {"ilu",
          [](std::shared_ptr<const gko::Executor> exec) {
-             auto fact =
-                 gko::share(gko::factorization::Ilu<etype>::build().on(exec));
-             return gko::preconditioner::Ilu<
-                        gko::solver::LowerTrs<etype>,
-                        gko::solver::UpperTrs<etype>>::build()
-                 .with_factorization_factory(fact)
-                 .on(exec);
+             auto fact = gko::share(
+                 gko::factorization::Ilu<etype, itype>::build().on(exec));
+             return gko::preconditioner::
+                 Ilu<gko::solver::LowerTrs<etype, itype>,
+                     gko::solver::UpperTrs<etype, itype>, false, itype>::build()
+                     .with_factorization_factory(fact)
+                     .on(exec);
          }},
         {"paric-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact =
-                 gko::share(gko::factorization::ParIc<etype>::build()
+                 gko::share(gko::factorization::ParIc<etype, itype>::build()
                                 .with_iterations(FLAGS_parilu_iterations)
                                 .with_skip_sorting(true)
                                 .on(exec));
-             auto lisai =
-                 gko::share(gko::preconditioner::LowerIsai<etype>::build()
-                                .with_sparsity_power(FLAGS_isai_power)
-                                .on(exec));
+             auto lisai = gko::share(
+                 gko::preconditioner::LowerIsai<etype, itype>::build()
+                     .with_sparsity_power(FLAGS_isai_power)
+                     .on(exec));
              return gko::preconditioner::Ic<
-                        gko::preconditioner::LowerIsai<etype>>::build()
+                        gko::preconditioner::LowerIsai<etype, itype>,
+                        itype>::build()
                  .with_factorization_factory(fact)
                  .with_l_solver_factory(lisai)
                  .on(exec);
@@ -207,18 +208,19 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"parict-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact = gko::share(
-                 gko::factorization::ParIct<etype>::build()
+                 gko::factorization::ParIct<etype, itype>::build()
                      .with_iterations(FLAGS_parilu_iterations)
                      .with_approximate_select(FLAGS_parilut_approx_select)
                      .with_fill_in_limit(FLAGS_parilut_limit)
                      .with_skip_sorting(true)
                      .on(exec));
-             auto lisai =
-                 gko::share(gko::preconditioner::LowerIsai<etype>::build()
-                                .with_sparsity_power(FLAGS_isai_power)
-                                .on(exec));
+             auto lisai = gko::share(
+                 gko::preconditioner::LowerIsai<etype, itype>::build()
+                     .with_sparsity_power(FLAGS_isai_power)
+                     .on(exec));
              return gko::preconditioner::Ic<
-                        gko::preconditioner::LowerIsai<etype>>::build()
+                        gko::preconditioner::LowerIsai<etype, itype>,
+                        itype>::build()
                  .with_factorization_factory(fact)
                  .with_l_solver_factory(lisai)
                  .on(exec);
@@ -226,21 +228,22 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"parilu-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact =
-                 gko::share(gko::factorization::ParIlu<etype>::build()
+                 gko::share(gko::factorization::ParIlu<etype, itype>::build()
                                 .with_iterations(FLAGS_parilu_iterations)
                                 .with_skip_sorting(true)
                                 .on(exec));
-             auto lisai =
-                 gko::share(gko::preconditioner::LowerIsai<etype>::build()
-                                .with_sparsity_power(FLAGS_isai_power)
-                                .on(exec));
-             auto uisai =
-                 gko::share(gko::preconditioner::UpperIsai<etype>::build()
-                                .with_sparsity_power(FLAGS_isai_power)
-                                .on(exec));
+             auto lisai = gko::share(
+                 gko::preconditioner::LowerIsai<etype, itype>::build()
+                     .with_sparsity_power(FLAGS_isai_power)
+                     .on(exec));
+             auto uisai = gko::share(
+                 gko::preconditioner::UpperIsai<etype, itype>::build()
+                     .with_sparsity_power(FLAGS_isai_power)
+                     .on(exec));
              return gko::preconditioner::Ilu<
-                        gko::preconditioner::LowerIsai<etype>,
-                        gko::preconditioner::UpperIsai<etype>>::build()
+                        gko::preconditioner::LowerIsai<etype, itype>,
+                        gko::preconditioner::UpperIsai<etype, itype>, false,
+                        itype>::build()
                  .with_factorization_factory(fact)
                  .with_l_solver_factory(lisai)
                  .with_u_solver_factory(uisai)
@@ -249,23 +252,24 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
         {"parilut-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
              auto fact = gko::share(
-                 gko::factorization::ParIlut<etype>::build()
+                 gko::factorization::ParIlut<etype, itype>::build()
                      .with_iterations(FLAGS_parilu_iterations)
                      .with_approximate_select(FLAGS_parilut_approx_select)
                      .with_fill_in_limit(FLAGS_parilut_limit)
                      .with_skip_sorting(true)
                      .on(exec));
-             auto lisai =
-                 gko::share(gko::preconditioner::LowerIsai<etype>::build()
-                                .with_sparsity_power(FLAGS_isai_power)
-                                .on(exec));
-             auto uisai =
-                 gko::share(gko::preconditioner::UpperIsai<etype>::build()
-                                .with_sparsity_power(FLAGS_isai_power)
-                                .on(exec));
+             auto lisai = gko::share(
+                 gko::preconditioner::LowerIsai<etype, itype>::build()
+                     .with_sparsity_power(FLAGS_isai_power)
+                     .on(exec));
+             auto uisai = gko::share(
+                 gko::preconditioner::UpperIsai<etype, itype>::build()
+                     .with_sparsity_power(FLAGS_isai_power)
+                     .on(exec));
              return gko::preconditioner::Ilu<
-                        gko::preconditioner::LowerIsai<etype>,
-                        gko::preconditioner::UpperIsai<etype>>::build()
+                        gko::preconditioner::LowerIsai<etype, itype>,
+                        gko::preconditioner::UpperIsai<etype, itype>, false,
+                        itype>::build()
                  .with_factorization_factory(fact)
                  .with_l_solver_factory(lisai)
                  .with_u_solver_factory(uisai)
@@ -273,33 +277,35 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
          }},
         {"ic-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
-             auto fact =
-                 gko::share(gko::factorization::Ic<etype>::build().on(exec));
-             auto lisai =
-                 gko::share(gko::preconditioner::LowerIsai<etype>::build()
-                                .with_sparsity_power(FLAGS_isai_power)
-                                .on(exec));
+             auto fact = gko::share(
+                 gko::factorization::Ic<etype, itype>::build().on(exec));
+             auto lisai = gko::share(
+                 gko::preconditioner::LowerIsai<etype, itype>::build()
+                     .with_sparsity_power(FLAGS_isai_power)
+                     .on(exec));
              return gko::preconditioner::Ic<
-                        gko::preconditioner::LowerIsai<etype>>::build()
+                        gko::preconditioner::LowerIsai<etype, itype>,
+                        itype>::build()
                  .with_factorization_factory(fact)
                  .with_l_solver_factory(lisai)
                  .on(exec);
          }},
         {"ilu-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
-             auto fact =
-                 gko::share(gko::factorization::Ilu<etype>::build().on(exec));
-             auto lisai =
-                 gko::share(gko::preconditioner::LowerIsai<etype>::build()
-                                .with_sparsity_power(FLAGS_isai_power)
-                                .on(exec));
-             auto uisai =
-                 gko::share(gko::preconditioner::UpperIsai<etype>::build()
-                                .with_sparsity_power(FLAGS_isai_power)
-                                .on(exec));
+             auto fact = gko::share(
+                 gko::factorization::Ilu<etype, itype>::build().on(exec));
+             auto lisai = gko::share(
+                 gko::preconditioner::LowerIsai<etype, itype>::build()
+                     .with_sparsity_power(FLAGS_isai_power)
+                     .on(exec));
+             auto uisai = gko::share(
+                 gko::preconditioner::UpperIsai<etype, itype>::build()
+                     .with_sparsity_power(FLAGS_isai_power)
+                     .on(exec));
              return gko::preconditioner::Ilu<
-                        gko::preconditioner::LowerIsai<etype>,
-                        gko::preconditioner::UpperIsai<etype>>::build()
+                        gko::preconditioner::LowerIsai<etype, itype>,
+                        gko::preconditioner::UpperIsai<etype, itype>, false,
+                        itype>::build()
                  .with_factorization_factory(fact)
                  .with_l_solver_factory(lisai)
                  .with_u_solver_factory(uisai)
@@ -307,13 +313,13 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOpFactory>(
          }},
         {"general-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
-             return gko::preconditioner::GeneralIsai<>::build()
+             return gko::preconditioner::GeneralIsai<etype, itype>::build()
                  .with_sparsity_power(FLAGS_isai_power)
                  .on(exec);
          }},
         {"spd-isai",
          [](std::shared_ptr<const gko::Executor> exec) {
-             return gko::preconditioner::SpdIsai<>::build()
+             return gko::preconditioner::SpdIsai<etype, itype>::build()
                  .with_sparsity_power(FLAGS_isai_power)
                  .on(exec);
          }},
diff --git a/benchmark/utils/types.hpp b/benchmark/utils/types.hpp
index c819ba30f92..61ee056c5ff 100644
--- a/benchmark/utils/types.hpp
+++ b/benchmark/utils/types.hpp
@@ -40,6 +40,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
+using itype = gko::int32;
+
+
 #if defined(GKO_BENCHMARK_USE_DOUBLE_PRECISION)
 using etype = double;
 #elif defined(GKO_BENCHMARK_USE_SINGLE_PRECISION)

From 58e2ede63ba9cfbca11f94558d4b98fe9da0f7c9 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 17 Jun 2021 21:35:58 +0200
Subject: [PATCH 0870/1008] add DPC++ SPGEMM and SPGEAM kernels

---
 dpcpp/components/prefix_sum.dp.cpp |  13 +-
 dpcpp/matrix/csr_kernels.dp.cpp    | 460 +++++++++++++++++++++++++++--
 dpcpp/test/CMakeLists.txt          |   1 +
 dpcpp/test/matrix/CMakeLists.txt   |   1 +
 dpcpp/test/matrix/csr_kernels.cpp  | 242 +++++++++++++++
 5 files changed, 697 insertions(+), 20 deletions(-)
 create mode 100644 dpcpp/test/matrix/CMakeLists.txt
 create mode 100644 dpcpp/test/matrix/csr_kernels.cpp

diff --git a/dpcpp/components/prefix_sum.dp.cpp b/dpcpp/components/prefix_sum.dp.cpp
index 7b1a5a2bf6d..4b7f816b381 100644
--- a/dpcpp/components/prefix_sum.dp.cpp
+++ b/dpcpp/components/prefix_sum.dp.cpp
@@ -47,7 +47,18 @@ namespace components {
 
 template <typename IndexType>
 void prefix_sum(std::shared_ptr<const DefaultExecutor> exec, IndexType *counts,
-                size_type num_entries) GKO_NOT_IMPLEMENTED;
+                size_type num_entries)
+{
+    // TODO actually implement parallel prefix sum
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{1}, [=](sycl::id<1> idx) {
+            IndexType sum{};
+            for (size_type i = 0; i < num_entries; i++) {
+                sum += std::exchange(counts[i], sum);
+            }
+        });
+    });
+}
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_PREFIX_SUM_KERNEL);
 // instantiate for size_type as well, as this is used in the Sellp format
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index 9148acb4743..36a8a7e236f 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -51,6 +51,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/base/allocator.hpp"
 #include "core/base/iterator_factory.hpp"
+#include "core/base/utils.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/csr_builder.hpp"
 #include "dpcpp/components/format_conversion.dp.hpp"
@@ -88,37 +89,263 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_ADVANCED_SPMV_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void spgemm_insert_row(unordered_set<IndexType> &cols,
-                       const matrix::Csr<ValueType, IndexType> *c,
-                       size_type row) GKO_NOT_IMPLEMENTED;
+namespace {
 
 
+/**
+ * @internal
+ *
+ * Entry in a heap storing a column index and associated non-zero index
+ * (and row end) from a matrix.
+ *
+ * @tparam ValueType  The value type for matrices.
+ * @tparam IndexType  The index type for matrices.
+ */
 template <typename ValueType, typename IndexType>
-void spgemm_insert_row2(unordered_set<IndexType> &cols,
-                        const matrix::Csr<ValueType, IndexType> *a,
-                        const matrix::Csr<ValueType, IndexType> *b,
-                        size_type row) GKO_NOT_IMPLEMENTED;
+struct col_heap_element {
+    using value_type = ValueType;
+    using index_type = IndexType;
 
+    IndexType idx;
+    IndexType end;
+    IndexType col;
 
-template <typename ValueType, typename IndexType>
-void spgemm_accumulate_row(map<IndexType, ValueType> &cols,
-                           const matrix::Csr<ValueType, IndexType> *c,
-                           ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
+    ValueType val() const { return zero<ValueType>(); }
+
+    col_heap_element(IndexType idx, IndexType end, IndexType col, ValueType)
+        : idx{idx}, end{end}, col{col}
+    {}
+};
 
 
+/**
+ * @internal
+ *
+ * Entry in a heap storing an entry (value and column index) and associated
+ * non-zero index (and row end) from a matrix.
+ *
+ * @tparam ValueType  The value type for matrices.
+ * @tparam IndexType  The index type for matrices.
+ */
 template <typename ValueType, typename IndexType>
-void spgemm_accumulate_row2(map<IndexType, ValueType> &cols,
-                            const matrix::Csr<ValueType, IndexType> *a,
-                            const matrix::Csr<ValueType, IndexType> *b,
-                            ValueType scale, size_type row) GKO_NOT_IMPLEMENTED;
+struct val_heap_element {
+    using value_type = ValueType;
+    using index_type = IndexType;
+
+    IndexType idx;
+    IndexType end;
+    IndexType col;
+    ValueType val_;
+
+    ValueType val() const { return val_; }
+
+    val_heap_element(IndexType idx, IndexType end, IndexType col, ValueType val)
+        : idx{idx}, end{end}, col{col}, val_{val}
+    {}
+};
+
+
+/**
+ * @internal
+ *
+ * Restores the binary heap condition downwards from a given index.
+ *
+ * The heap condition is: col(child) >= col(parent)
+ *
+ * @param heap  a pointer to the array containing the heap elements.
+ * @param idx  the index of the starting heap node that potentially
+ *             violates the heap condition.
+ * @param size  the number of elements in the heap.
+ * @tparam HeapElement  the element type in the heap. See col_heap_element and
+ *                      val_heap_element
+ */
+template <typename HeapElement>
+void sift_down(HeapElement *heap, typename HeapElement::index_type idx,
+               typename HeapElement::index_type size)
+{
+    auto curcol = heap[idx].col;
+    while (idx * 2 + 1 < size) {
+        auto lchild = idx * 2 + 1;
+        auto rchild = min(lchild + 1, size - 1);
+        auto lcol = heap[lchild].col;
+        auto rcol = heap[rchild].col;
+        auto mincol = min(lcol, rcol);
+        if (mincol >= curcol) {
+            break;
+        }
+        auto minchild = lcol == mincol ? lchild : rchild;
+        std::swap(heap[minchild], heap[idx]);
+        idx = minchild;
+    }
+}
+
+
+/**
+ * @internal
+ *
+ * Generic SpGEMM implementation for a single output row of A * B using binary
+ * heap-based multiway merging.
+ *
+ * @param row  The row for which to compute the SpGEMM
+ * @param a  The input matrix A
+ * @param b  The input matrix B (its column indices must be sorted within each
+ *           row!)
+ * @param heap  The heap to use for this implementation. It must have as many
+ *              entries as the input row has non-zeros.
+ * @param init_cb  function to initialize the state for a single row. Its return
+ *                 value will be updated by subsequent calls of other callbacks,
+ *                 and then returned by this function. Its signature must be
+ *                 compatible with `return_type state = init_cb(row)`.
+ * @param step_cb  function that will be called for each accumulation from an
+ *                 entry of B into the output state. Its signature must be
+ *                 compatible with `step_cb(value, column, state)`.
+ * @param col_cb  function that will be called once for each output column after
+ *                all accumulations into it are completed. Its signature must be
+ *                compatible with `col_cb(column, state)`.
+ * @return the value initialized by init_cb and updated by step_cb and col_cb
+ * @note If the columns of B are not sorted, the output may have duplicate
+ *       column entries.
+ *
+ * @tparam HeapElement  the heap element type. See col_heap_element and
+ *                      val_heap_element
+ * @tparam InitCallback  functor type for init_cb
+ * @tparam StepCallback  functor type for step_cb
+ * @tparam ColCallback  functor type for col_cb
+ */
+template <typename HeapElement, typename InitCallback, typename StepCallback,
+          typename ColCallback>
+auto spgemm_multiway_merge(size_type row,
+                           const typename HeapElement::index_type *a_row_ptrs,
+                           const typename HeapElement::index_type *a_cols,
+                           const typename HeapElement::value_type *a_vals,
+                           const typename HeapElement::index_type *b_row_ptrs,
+                           const typename HeapElement::index_type *b_cols,
+                           const typename HeapElement::value_type *b_vals,
+                           HeapElement *heap, InitCallback init_cb,
+                           StepCallback step_cb, ColCallback col_cb)
+    -> decltype(init_cb(0))
+{
+    auto a_begin = a_row_ptrs[row];
+    auto a_end = a_row_ptrs[row + 1];
+
+    using index_type = typename HeapElement::index_type;
+    constexpr auto sentinel = std::numeric_limits<index_type>::max();
+
+    auto state = init_cb(row);
+
+    // initialize the heap
+    for (auto a_nz = a_begin; a_nz < a_end; ++a_nz) {
+        auto b_row = a_cols[a_nz];
+        auto b_begin = b_row_ptrs[b_row];
+        auto b_end = b_row_ptrs[b_row + 1];
+        heap[a_nz] = {b_begin, b_end,
+                      checked_load(b_cols, b_begin, b_end, sentinel),
+                      a_vals[a_nz]};
+    }
+
+    if (a_begin != a_end) {
+        // make heap:
+        auto a_size = a_end - a_begin;
+        for (auto i = (a_size - 2) / 2; i >= 0; --i) {
+            sift_down(heap + a_begin, i, a_size);
+        }
+        auto &top = heap[a_begin];
+        auto &bot = heap[a_end - 1];
+        auto col = top.col;
+
+        while (top.col != sentinel) {
+            step_cb(b_vals[top.idx] * top.val(), top.col, state);
+            // move to the next element
+            top.idx++;
+            top.col = checked_load(b_cols, top.idx, top.end, sentinel);
+            // restore heap property
+            // pop_heap swaps top and bot, we need to prevent that
+            // so that we do a simple sift_down instead
+            sift_down(heap + a_begin, index_type{}, a_size);
+            if (top.col != col) {
+                col_cb(col, state);
+            }
+            col = top.col;
+        }
+    }
+
+    return state;
+}
+
+
+}  // namespace
 
 
 template <typename ValueType, typename IndexType>
 void spgemm(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Csr<ValueType, IndexType> *a,
             const matrix::Csr<ValueType, IndexType> *b,
-            matrix::Csr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+            matrix::Csr<ValueType, IndexType> *c)
+{
+    auto num_rows = a->get_size()[0];
+    const auto a_row_ptrs = a->get_const_row_ptrs();
+    const auto a_cols = a->get_const_col_idxs();
+    const auto a_vals = a->get_const_values();
+    const auto b_row_ptrs = b->get_const_row_ptrs();
+    const auto b_cols = b->get_const_col_idxs();
+    const auto b_vals = b->get_const_values();
+    auto c_row_ptrs = c->get_row_ptrs();
+    auto queue = exec->get_queue();
+
+    Array<val_heap_element<ValueType, IndexType>> heap_array(
+        exec, a->get_num_stored_elements());
+
+    auto heap = heap_array.get_data();
+    auto col_heap =
+        reinterpret_cast<col_heap_element<ValueType, IndexType> *>(heap);
+
+    // first sweep: count nnz for each row
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto a_row = static_cast<size_type>(idx[0]);
+            c_row_ptrs[a_row] = spgemm_multiway_merge(
+                a_row, a_row_ptrs, a_cols, a_vals, b_row_ptrs, b_cols, b_vals,
+                col_heap, [](size_type) { return IndexType{}; },
+                [](ValueType, IndexType, IndexType &) {},
+                [](IndexType, IndexType &nnz) { nnz++; });
+        });
+    });
+
+    // build row pointers
+    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
+
+    // second sweep: accumulate non-zeros
+    const auto new_nnz = exec->copy_val_to_host(c_row_ptrs + num_rows);
+    matrix::CsrBuilder<ValueType, IndexType> c_builder{c};
+    auto &c_col_idxs_array = c_builder.get_col_idx_array();
+    auto &c_vals_array = c_builder.get_value_array();
+    c_col_idxs_array.resize_and_reset(new_nnz);
+    c_vals_array.resize_and_reset(new_nnz);
+    auto c_col_idxs = c_col_idxs_array.get_data();
+    auto c_vals = c_vals_array.get_data();
+
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto a_row = static_cast<size_type>(idx[0]);
+            spgemm_multiway_merge(
+                a_row, a_row_ptrs, a_cols, a_vals, b_row_ptrs, b_cols, b_vals,
+                heap,
+                [&](size_type row) {
+                    return std::make_pair(zero<ValueType>(), c_row_ptrs[row]);
+                },
+                [](ValueType val, IndexType,
+                   std::pair<ValueType, IndexType> &state) {
+                    state.first += val;
+                },
+                [&](IndexType col, std::pair<ValueType, IndexType> &state) {
+                    c_col_idxs[state.second] = col;
+                    c_vals[state.second] = state.first;
+                    state.first = zero<ValueType>();
+                    state.second++;
+                });
+        });
+    });
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEMM_KERNEL);
 
@@ -130,7 +357,128 @@ void advanced_spgemm(std::shared_ptr<const DpcppExecutor> exec,
                      const matrix::Csr<ValueType, IndexType> *b,
                      const matrix::Dense<ValueType> *beta,
                      const matrix::Csr<ValueType, IndexType> *d,
-                     matrix::Csr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+                     matrix::Csr<ValueType, IndexType> *c)
+{
+    auto num_rows = a->get_size()[0];
+    const auto a_row_ptrs = a->get_const_row_ptrs();
+    const auto a_cols = a->get_const_col_idxs();
+    const auto a_vals = a->get_const_values();
+    const auto b_row_ptrs = b->get_const_row_ptrs();
+    const auto b_cols = b->get_const_col_idxs();
+    const auto b_vals = b->get_const_values();
+    const auto d_row_ptrs = d->get_const_row_ptrs();
+    const auto d_cols = d->get_const_col_idxs();
+    const auto d_vals = d->get_const_values();
+    auto c_row_ptrs = c->get_row_ptrs();
+    const auto alpha_vals = alpha->get_const_values();
+    const auto beta_vals = beta->get_const_values();
+    constexpr auto sentinel = std::numeric_limits<IndexType>::max();
+    auto queue = exec->get_queue();
+
+    // first sweep: count nnz for each row
+
+    Array<val_heap_element<ValueType, IndexType>> heap_array(
+        exec, a->get_num_stored_elements());
+
+    auto heap = heap_array.get_data();
+    auto col_heap =
+        reinterpret_cast<col_heap_element<ValueType, IndexType> *>(heap);
+
+    // first sweep: count nnz for each row
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto a_row = static_cast<size_type>(idx[0]);
+            auto d_nz = d_row_ptrs[a_row];
+            const auto d_end = d_row_ptrs[a_row + 1];
+            auto d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+            c_row_ptrs[a_row] = spgemm_multiway_merge(
+                a_row, a_row_ptrs, a_cols, a_vals, b_row_ptrs, b_cols, b_vals,
+                col_heap, [](size_type row) { return IndexType{}; },
+                [](ValueType, IndexType, IndexType &) {},
+                [&](IndexType col, IndexType &nnz) {
+                    // skip smaller elements from d
+                    while (d_col <= col) {
+                        d_nz++;
+                        nnz += d_col != col;
+                        d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+                    }
+                    nnz++;
+                });
+            // handle the remaining columns from d
+            c_row_ptrs[a_row] += d_end - d_nz;
+        });
+    });
+
+    // build row pointers
+    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
+
+    // second sweep: accumulate non-zeros
+    const auto new_nnz = exec->copy_val_to_host(c_row_ptrs + num_rows);
+    matrix::CsrBuilder<ValueType, IndexType> c_builder{c};
+    auto &c_col_idxs_array = c_builder.get_col_idx_array();
+    auto &c_vals_array = c_builder.get_value_array();
+    c_col_idxs_array.resize_and_reset(new_nnz);
+    c_vals_array.resize_and_reset(new_nnz);
+
+    auto c_col_idxs = c_col_idxs_array.get_data();
+    auto c_vals = c_vals_array.get_data();
+
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto a_row = static_cast<size_type>(idx[0]);
+            auto d_nz = d_row_ptrs[a_row];
+            const auto d_end = d_row_ptrs[a_row + 1];
+            auto d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+            auto d_val = checked_load(d_vals, d_nz, d_end, zero<ValueType>());
+            const auto valpha = alpha_vals[0];
+            const auto vbeta = beta_vals[0];
+            auto c_nz =
+                spgemm_multiway_merge(
+                    a_row, a_row_ptrs, a_cols, a_vals, b_row_ptrs, b_cols,
+                    b_vals, heap,
+                    [&](size_type row) {
+                        return std::make_pair(zero<ValueType>(),
+                                              c_row_ptrs[row]);
+                    },
+                    [](ValueType val, IndexType,
+                       std::pair<ValueType, IndexType> &state) {
+                        state.first += val;
+                    },
+                    [&](IndexType col, std::pair<ValueType, IndexType> &state) {
+                        // handle smaller elements from d
+                        ValueType part_d_val{};
+                        while (d_col <= col) {
+                            if (d_col == col) {
+                                part_d_val = d_val;
+                            } else {
+                                c_col_idxs[state.second] = d_col;
+                                c_vals[state.second] = vbeta * d_val;
+                                state.second++;
+                            }
+                            d_nz++;
+                            d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+                            d_val = checked_load(d_vals, d_nz, d_end,
+                                                 zero<ValueType>());
+                        }
+                        c_col_idxs[state.second] = col;
+                        c_vals[state.second] =
+                            vbeta * part_d_val + valpha * state.first;
+                        state.first = zero<ValueType>();
+                        state.second++;
+                    })
+                    .second;
+            // handle remaining elements from d
+            while (d_col < sentinel) {
+                c_col_idxs[c_nz] = d_col;
+                c_vals[c_nz] = vbeta * d_val;
+                c_nz++;
+                d_nz++;
+                d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+                d_val = checked_load(d_vals, d_nz, d_end, zero<ValueType>());
+            }
+        });
+    });
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_ADVANCED_SPGEMM_KERNEL);
@@ -142,7 +490,81 @@ void spgeam(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Csr<ValueType, IndexType> *a,
             const matrix::Dense<ValueType> *beta,
             const matrix::Csr<ValueType, IndexType> *b,
-            matrix::Csr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+            matrix::Csr<ValueType, IndexType> *c)
+{
+    constexpr auto sentinel = std::numeric_limits<IndexType>::max();
+    const auto num_rows = a->get_size()[0];
+    const auto a_row_ptrs = a->get_const_row_ptrs();
+    const auto a_cols = a->get_const_col_idxs();
+    const auto b_row_ptrs = b->get_const_row_ptrs();
+    const auto b_cols = b->get_const_col_idxs();
+    auto c_row_ptrs = c->get_row_ptrs();
+    auto queue = exec->get_queue();
+
+    // count number of non-zeros per row
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto row = static_cast<size_type>(idx[0]);
+            auto a_idx = a_row_ptrs[row];
+            const auto a_end = a_row_ptrs[row + 1];
+            auto b_idx = b_row_ptrs[row];
+            const auto b_end = b_row_ptrs[row + 1];
+            IndexType row_nnz{};
+            while (a_idx < a_end || b_idx < b_end) {
+                const auto a_col = checked_load(a_cols, a_idx, a_end, sentinel);
+                const auto b_col = checked_load(b_cols, b_idx, b_end, sentinel);
+                row_nnz++;
+                a_idx += (a_col <= b_col) ? 1 : 0;
+                b_idx += (b_col <= a_col) ? 1 : 0;
+            }
+            c_row_ptrs[row] = row_nnz;
+        });
+    });
+
+    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
+
+    // second sweep: accumulate non-zeros
+    const auto new_nnz = exec->copy_val_to_host(c_row_ptrs + num_rows);
+    matrix::CsrBuilder<ValueType, IndexType> c_builder{c};
+    auto &c_col_idxs_array = c_builder.get_col_idx_array();
+    auto &c_vals_array = c_builder.get_value_array();
+    c_col_idxs_array.resize_and_reset(new_nnz);
+    c_vals_array.resize_and_reset(new_nnz);
+    auto c_cols = c_col_idxs_array.get_data();
+    auto c_vals = c_vals_array.get_data();
+
+    const auto a_vals = a->get_const_values();
+    const auto b_vals = b->get_const_values();
+    const auto alpha_vals = alpha->get_const_values();
+    const auto beta_vals = beta->get_const_values();
+
+    // count number of non-zeros per row
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto row = static_cast<size_type>(idx[0]);
+            auto a_idx = a_row_ptrs[row];
+            const auto a_end = a_row_ptrs[row + 1];
+            auto b_idx = b_row_ptrs[row];
+            const auto b_end = b_row_ptrs[row + 1];
+            const auto alpha = alpha_vals[0];
+            const auto beta = beta_vals[0];
+            auto c_nz = c_row_ptrs[row];
+            while (a_idx < a_end || b_idx < b_end) {
+                const auto a_col = checked_load(a_cols, a_idx, a_end, sentinel);
+                const auto b_col = checked_load(b_cols, b_idx, b_end, sentinel);
+                const bool use_a = a_col <= b_col;
+                const bool use_b = b_col <= a_col;
+                const auto a_val = use_a ? a_vals[a_idx] : zero<ValueType>();
+                const auto b_val = use_b ? b_vals[b_idx] : zero<ValueType>();
+                c_cols[c_nz] = std::min(a_col, b_col);
+                c_vals[c_nz] = alpha * a_val + beta * b_val;
+                c_nz++;
+                a_idx += use_a ? 1 : 0;
+                b_idx += use_b ? 1 : 0;
+            }
+        });
+    });
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEAM_KERNEL);
 
diff --git a/dpcpp/test/CMakeLists.txt b/dpcpp/test/CMakeLists.txt
index daa5367f268..3af602c259e 100644
--- a/dpcpp/test/CMakeLists.txt
+++ b/dpcpp/test/CMakeLists.txt
@@ -3,4 +3,5 @@ set(GINKGO_COMPILING_DPCPP_TEST ON)
 
 add_subdirectory(base)
 add_subdirectory(components)
+add_subdirectory(matrix)
 add_subdirectory(stop)
diff --git a/dpcpp/test/matrix/CMakeLists.txt b/dpcpp/test/matrix/CMakeLists.txt
new file mode 100644
index 00000000000..af64b693718
--- /dev/null
+++ b/dpcpp/test/matrix/CMakeLists.txt
@@ -0,0 +1 @@
+ginkgo_create_test(csr_kernels)
\ No newline at end of file
diff --git a/dpcpp/test/matrix/csr_kernels.cpp b/dpcpp/test/matrix/csr_kernels.cpp
new file mode 100644
index 00000000000..07e83830989
--- /dev/null
+++ b/dpcpp/test/matrix/csr_kernels.cpp
@@ -0,0 +1,242 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/matrix/csr.hpp>
+
+
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/identity.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class Csr : public ::testing::Test {
+protected:
+#if GINKGO_DPCPP_SINGLE_MODE
+    using value_type = float;
+#else
+    using value_type = double;
+#endif
+    using Arr = gko::Array<int>;
+    using Mtx = gko::matrix::Csr<value_type>;
+    using Vec = gko::matrix::Dense<value_type>;
+
+    Csr() : mtx_size(532, 231), rand_engine(42) {}
+
+    void SetUp()
+    {
+        ref = gko::ReferenceExecutor::create();
+        dpcpp = gko::DpcppExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (dpcpp != nullptr) {
+            ASSERT_NO_THROW(dpcpp->synchronize());
+        }
+    }
+
+    template <typename MtxType>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row, int max_nnz_row)
+    {
+        return gko::test::generate_random_matrix<MtxType>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(min_nnz_row, max_nnz_row),
+            std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    template <typename MtxType>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row)
+    {
+        return gen_mtx<MtxType>(num_rows, num_cols, min_nnz_row, num_cols);
+    }
+
+    void set_up_apply_data(int num_vectors = 1)
+    {
+        mtx = Mtx::create(ref);
+        mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[1], 1));
+        square_mtx = Mtx::create(ref);
+        square_mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[0], 1));
+        alpha = gko::initialize<Vec>({2.0}, ref);
+        beta = gko::initialize<Vec>({-1.0}, ref);
+        dmtx = Mtx::create(dpcpp);
+        dmtx->copy_from(mtx.get());
+        square_dmtx = Mtx::create(dpcpp);
+        square_dmtx->copy_from(square_mtx.get());
+        dalpha = Vec::create(dpcpp);
+        dalpha->copy_from(alpha.get());
+        dbeta = Vec::create(dpcpp);
+        dbeta->copy_from(beta.get());
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
+
+    const gko::dim<2> mtx_size;
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Mtx> mtx;
+    std::unique_ptr<Mtx> square_mtx;
+    std::unique_ptr<Vec> alpha;
+    std::unique_ptr<Vec> beta;
+
+    std::unique_ptr<Mtx> dmtx;
+    std::unique_ptr<Mtx> square_dmtx;
+    std::unique_ptr<Vec> dalpha;
+    std::unique_ptr<Vec> dbeta;
+};
+
+
+TEST_F(Csr, AdvancedApplyToCsrMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto trans = mtx->transpose();
+    auto d_trans = gko::clone(dpcpp, trans);
+
+    mtx->apply(alpha.get(), trans.get(), beta.get(), square_mtx.get());
+    dmtx->apply(dalpha.get(), d_trans.get(), dbeta.get(), square_dmtx.get());
+
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
+    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+    ASSERT_TRUE(gko::clone(ref, square_dmtx)->is_sorted_by_column_index());
+}
+
+
+TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto trans = mtx->transpose();
+    auto d_trans = gko::clone(dpcpp, trans);
+
+    mtx->apply(trans.get(), square_mtx.get());
+    dmtx->apply(d_trans.get(), square_dmtx.get());
+
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
+    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+    ASSERT_TRUE(gko::clone(ref, square_dmtx)->is_sorted_by_column_index());
+}
+
+
+TEST_F(Csr, SimpleApplyToSparseCsrMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto mtx2 =
+        gen_mtx<Mtx>(mtx->get_size()[1], square_mtx->get_size()[1], 0, 10);
+    auto dmtx2 = Mtx::create(dpcpp, mtx2->get_size());
+    dmtx2->copy_from(mtx2.get());
+
+    mtx->apply(mtx2.get(), square_mtx.get());
+    dmtx->apply(dmtx2.get(), square_dmtx.get());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
+    ASSERT_TRUE(gko::clone(ref, square_dmtx)->is_sorted_by_column_index());
+}
+
+
+TEST_F(Csr, SimpleApplySparseToSparseCsrMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto mtx1 = gen_mtx<Mtx>(mtx->get_size()[0], mtx->get_size()[1], 0, 10);
+    auto mtx2 =
+        gen_mtx<Mtx>(mtx->get_size()[1], square_mtx->get_size()[1], 0, 10);
+    auto dmtx1 = Mtx::create(dpcpp, mtx1->get_size());
+    auto dmtx2 = Mtx::create(dpcpp, mtx2->get_size());
+    dmtx1->copy_from(mtx1.get());
+    dmtx2->copy_from(mtx2.get());
+
+    mtx1->apply(mtx2.get(), square_mtx.get());
+    dmtx1->apply(dmtx2.get(), square_dmtx.get());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
+    ASSERT_TRUE(gko::clone(ref, square_dmtx)->is_sorted_by_column_index());
+}
+
+
+TEST_F(Csr, SimpleApplyToEmptyCsrMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto mtx2 =
+        gen_mtx<Mtx>(mtx->get_size()[1], square_mtx->get_size()[1], 0, 0);
+    auto dmtx2 = Mtx::create(dpcpp, mtx2->get_size());
+    dmtx2->copy_from(mtx2.get());
+
+    mtx->apply(mtx2.get(), square_mtx.get());
+    dmtx->apply(dmtx2.get(), square_dmtx.get());
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
+    ASSERT_TRUE(gko::clone(ref, square_dmtx)->is_sorted_by_column_index());
+}
+
+
+TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto a = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
+    auto b = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
+    auto da = Mtx::create(dpcpp);
+    auto db = Mtx::create(dpcpp);
+    da->copy_from(a.get());
+    db->copy_from(b.get());
+    auto id = gko::matrix::Identity<Mtx::value_type>::create(ref, mtx_size[1]);
+    auto did =
+        gko::matrix::Identity<Mtx::value_type>::create(dpcpp, mtx_size[1]);
+
+    a->apply(alpha.get(), id.get(), beta.get(), b.get());
+    da->apply(dalpha.get(), did.get(), dbeta.get(), db.get());
+
+    GKO_ASSERT_MTX_NEAR(b, db, r<value_type>::value);
+    GKO_ASSERT_MTX_EQ_SPARSITY(b, db);
+    ASSERT_TRUE(gko::clone(ref, db)->is_sorted_by_column_index());
+}
+
+
+}  // namespace

From bbfd9b4c9cb7e6daf2600763c31b062e18d91242 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 18 Jun 2021 12:29:42 +0200
Subject: [PATCH 0871/1008] add dpcpp is_sorted_by_col_idxs kernel

---
 dpcpp/matrix/csr_kernels.dp.cpp   | 26 ++++++++++++--
 dpcpp/test/matrix/csr_kernels.cpp | 57 +++++++++++++++++++++++++++----
 2 files changed, 75 insertions(+), 8 deletions(-)

diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index 36a8a7e236f..31cdd4a1142 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -745,8 +745,30 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void is_sorted_by_column_index(
     std::shared_ptr<const DpcppExecutor> exec,
-    const matrix::Csr<ValueType, IndexType> *to_check,
-    bool *is_sorted) GKO_NOT_IMPLEMENTED;
+    const matrix::Csr<ValueType, IndexType> *to_check, bool *is_sorted)
+{
+    Array<bool> is_sorted_device_array{exec, {true}};
+    const auto num_rows = to_check->get_size()[0];
+    const auto row_ptrs = to_check->get_const_row_ptrs();
+    const auto cols = to_check->get_const_col_idxs();
+    auto is_sorted_device = is_sorted_device_array.get_data();
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto row = static_cast<size_type>(idx[0]);
+            const auto begin = row_ptrs[row];
+            const auto end = row_ptrs[row + 1];
+            if (*is_sorted_device) {
+                for (auto i = begin; i < end - 1; i++) {
+                    if (cols[i] > cols[i + 1]) {
+                        *is_sorted_device = false;
+                        break;
+                    }
+                }
+            }
+        });
+    });
+    exec->get_master()->copy_from(exec.get(), 1, is_sorted_device, is_sorted);
+};
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_IS_SORTED_BY_COLUMN_INDEX);
diff --git a/dpcpp/test/matrix/csr_kernels.cpp b/dpcpp/test/matrix/csr_kernels.cpp
index 07e83830989..d74a4c802e8 100644
--- a/dpcpp/test/matrix/csr_kernels.cpp
+++ b/dpcpp/test/matrix/csr_kernels.cpp
@@ -49,6 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "core/test/utils.hpp"
+#include "core/test/utils/unsort_matrix.hpp"
 
 
 namespace {
@@ -115,6 +116,24 @@ class Csr : public ::testing::Test {
         dbeta->copy_from(beta.get());
     }
 
+    struct matrix_pair {
+        std::unique_ptr<Mtx> ref;
+        std::unique_ptr<Mtx> dpcpp;
+    };
+
+    matrix_pair gen_unsorted_mtx()
+    {
+        constexpr int min_nnz_per_row{2};
+        auto local_mtx_ref =
+            gen_mtx<Mtx>(mtx_size[0], mtx_size[1], min_nnz_per_row);
+        gko::test::unsort_matrix(gko::lend(local_mtx_ref), rand_engine);
+
+        auto local_mtx_dpcpp = Mtx::create(dpcpp);
+        local_mtx_dpcpp->copy_from(local_mtx_ref.get());
+
+        return {std::move(local_mtx_ref), std::move(local_mtx_dpcpp)};
+    }
+
     std::shared_ptr<gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::DpcppExecutor> dpcpp;
 
@@ -144,7 +163,7 @@ TEST_F(Csr, AdvancedApplyToCsrMatrixIsEquivalentToRef)
 
     GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
     GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-    ASSERT_TRUE(gko::clone(ref, square_dmtx)->is_sorted_by_column_index());
+    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
 }
 
 
@@ -159,7 +178,7 @@ TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRef)
 
     GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
     GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-    ASSERT_TRUE(gko::clone(ref, square_dmtx)->is_sorted_by_column_index());
+    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
 }
 
 
@@ -176,7 +195,7 @@ TEST_F(Csr, SimpleApplyToSparseCsrMatrixIsEquivalentToRef)
 
     GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
     GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
-    ASSERT_TRUE(gko::clone(ref, square_dmtx)->is_sorted_by_column_index());
+    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
 }
 
 
@@ -196,7 +215,7 @@ TEST_F(Csr, SimpleApplySparseToSparseCsrMatrixIsEquivalentToRef)
 
     GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
     GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
-    ASSERT_TRUE(gko::clone(ref, square_dmtx)->is_sorted_by_column_index());
+    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
 }
 
 
@@ -213,7 +232,7 @@ TEST_F(Csr, SimpleApplyToEmptyCsrMatrixIsEquivalentToRef)
 
     GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
     GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
-    ASSERT_TRUE(gko::clone(ref, square_dmtx)->is_sorted_by_column_index());
+    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
 }
 
 
@@ -235,7 +254,33 @@ TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
 
     GKO_ASSERT_MTX_NEAR(b, db, r<value_type>::value);
     GKO_ASSERT_MTX_EQ_SPARSITY(b, db);
-    ASSERT_TRUE(gko::clone(ref, db)->is_sorted_by_column_index());
+    ASSERT_TRUE(db->is_sorted_by_column_index());
+}
+
+
+TEST_F(Csr, RecognizeSortedMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+    bool is_sorted_dpcpp{};
+    bool is_sorted_ref{};
+
+    is_sorted_ref = mtx->is_sorted_by_column_index();
+    is_sorted_dpcpp = dmtx->is_sorted_by_column_index();
+
+    ASSERT_EQ(is_sorted_ref, is_sorted_dpcpp);
+}
+
+
+TEST_F(Csr, RecognizeUnsortedMatrixIsEquivalentToRef)
+{
+    auto uns_mtx = gen_unsorted_mtx();
+    bool is_sorted_dpcpp{};
+    bool is_sorted_ref{};
+
+    is_sorted_ref = uns_mtx.ref->is_sorted_by_column_index();
+    is_sorted_dpcpp = uns_mtx.dpcpp->is_sorted_by_column_index();
+
+    ASSERT_EQ(is_sorted_ref, is_sorted_dpcpp);
 }
 
 

From dbe62519abe3c1cc635dc203086d8e3d27ba45ad Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 18 Jun 2021 15:38:43 +0200
Subject: [PATCH 0872/1008] add dpcpp csrsort

---
 dpcpp/matrix/csr_kernels.dp.cpp   | 51 ++++++++++++++++++++++++++++++-
 dpcpp/test/matrix/csr_kernels.cpp | 28 +++++++++++++++++
 2 files changed, 78 insertions(+), 1 deletion(-)

diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index 31cdd4a1142..0f335c745a2 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -736,7 +736,56 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void sort_by_column_index(std::shared_ptr<const DpcppExecutor> exec,
                           matrix::Csr<ValueType, IndexType> *to_sort)
-    GKO_NOT_IMPLEMENTED;
+{
+    const auto num_rows = to_sort->get_size()[0];
+    const auto row_ptrs = to_sort->get_const_row_ptrs();
+    auto cols = to_sort->get_col_idxs();
+    auto vals = to_sort->get_values();
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto row = static_cast<size_type>(idx[0]);
+            const auto begin = row_ptrs[row];
+            auto size = row_ptrs[row + 1] - begin;
+            if (size <= 1) {
+                return;
+            }
+            auto swap = [&](IndexType i, IndexType j) {
+                std::swap(cols[i + begin], cols[j + begin]);
+                std::swap(vals[i + begin], vals[j + begin]);
+            };
+            auto lchild = [](IndexType i) { return 2 * i + 1; };
+            auto rchild = [](IndexType i) { return 2 * i + 2; };
+            auto parent = [](IndexType i) { return (i - 1) / 2; };
+            auto sift_down = [&](IndexType i) {
+                const auto col = cols[i + begin];
+                while (lchild(i) < size) {
+                    const auto lcol = cols[lchild(i) + begin];
+                    // -1 as sentinel, since we are building a max heap
+                    const auto rcol = checked_load(cols + begin, rchild(i),
+                                                   size, IndexType{-1});
+                    if (col >= std::max(lcol, rcol)) {
+                        return;
+                    }
+                    const auto maxchild = lcol > rcol ? lchild(i) : rchild(i);
+                    swap(i, maxchild);
+                    i = maxchild;
+                }
+            };
+            // heapify / sift_down for max-heap
+            for (auto i = (size - 2) / 2; i >= 0; i--) {
+                sift_down(i);
+            }
+            // heapsort: swap maximum to the end, shrink heap
+            swap(0, size - 1);
+            size--;
+            for (; size > 1; size--) {
+                // restore heap property and repeat
+                sift_down(0);
+                swap(0, size - 1);
+            }
+        });
+    });
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_SORT_BY_COLUMN_INDEX);
diff --git a/dpcpp/test/matrix/csr_kernels.cpp b/dpcpp/test/matrix/csr_kernels.cpp
index d74a4c802e8..fb8ae74d685 100644
--- a/dpcpp/test/matrix/csr_kernels.cpp
+++ b/dpcpp/test/matrix/csr_kernels.cpp
@@ -284,4 +284,32 @@ TEST_F(Csr, RecognizeUnsortedMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, SortSortedMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->sort_by_column_index();
+    dmtx->sort_by_column_index();
+
+    // Values must be unchanged, therefore, tolerance is `0`
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 0);
+    ASSERT_TRUE(mtx->is_sorted_by_column_index());
+    ASSERT_TRUE(dmtx->is_sorted_by_column_index());
+}
+
+
+TEST_F(Csr, SortUnsortedMatrixIsEquivalentToRef)
+{
+    auto uns_mtx = gen_unsorted_mtx();
+
+    uns_mtx.ref->sort_by_column_index();
+    uns_mtx.dpcpp->sort_by_column_index();
+
+    // Values must be unchanged, therefore, tolerance is `0`
+    GKO_ASSERT_MTX_NEAR(uns_mtx.ref, uns_mtx.dpcpp, 0);
+    ASSERT_TRUE(uns_mtx.ref->is_sorted_by_column_index());
+    ASSERT_TRUE(uns_mtx.dpcpp->is_sorted_by_column_index());
+}
+
+
 }  // namespace

From cfb2048e4c1674206fdf57f1576b90bc77954100 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 18 Jun 2021 16:34:08 +0200
Subject: [PATCH 0873/1008] add dpcpp transpose kernels

---
 dpcpp/components/atomic.dp.hpp    | 214 ++++++++++++++++++++++++++++++
 dpcpp/matrix/csr_kernels.dp.cpp   |  62 ++++++++-
 dpcpp/test/matrix/csr_kernels.cpp |  33 +++++
 3 files changed, 307 insertions(+), 2 deletions(-)
 create mode 100644 dpcpp/components/atomic.dp.hpp

diff --git a/dpcpp/components/atomic.dp.hpp b/dpcpp/components/atomic.dp.hpp
new file mode 100644
index 00000000000..7bedf2cbe48
--- /dev/null
+++ b/dpcpp/components/atomic.dp.hpp
@@ -0,0 +1,214 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_COMPONENTS_ATOMIC_DP_HPP_
+#define GKO_DPCPP_COMPONENTS_ATOMIC_DP_HPP_
+
+
+#include <type_traits>
+
+
+#include <CL/sycl.hpp>
+
+
+#include "dpcpp/base/dpct.hpp"
+
+// #include "dpct/atomic.hpp"
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+namespace atomic {
+constexpr cl::sycl::access::address_space local_space =
+    cl::sycl::access::address_space::local_space;
+constexpr cl::sycl::access::address_space global_space =
+    cl::sycl::access::address_space::global_space;
+}  // namespace atomic
+
+namespace {
+
+
+template <cl::sycl::access::address_space addressSpace = atomic::global_space,
+          typename T>
+T atomic_compare_exchange_strong(
+    cl::sycl::multi_ptr<T, addressSpace> addr, T expected, T desired,
+    cl::sycl::memory_order success = cl::sycl::memory_order::relaxed,
+    cl::sycl::memory_order fail = cl::sycl::memory_order::relaxed)
+{
+    cl::sycl::atomic<T, addressSpace> obj(addr);
+    obj.compare_exchange_strong(expected, desired, success, fail);
+    return expected;
+}
+
+template <cl::sycl::access::address_space addressSpace = atomic::global_space,
+          typename T>
+T atomic_compare_exchange_strong(
+    T *addr, T expected, T desired,
+    cl::sycl::memory_order success = cl::sycl::memory_order::relaxed,
+    cl::sycl::memory_order fail = cl::sycl::memory_order::relaxed)
+{
+    return atomic_compare_exchange_strong(
+        cl::sycl::multi_ptr<T, addressSpace>(addr), expected, desired, success,
+        fail);
+}
+
+
+template <cl::sycl::access::address_space addressSpace = atomic::global_space,
+          typename T>
+inline T atomic_fetch_add(
+    T *addr, T operand,
+    cl::sycl::memory_order memoryOrder = cl::sycl::memory_order::relaxed)
+{
+    cl::sycl::atomic<T, addressSpace> obj(
+        (cl::sycl::multi_ptr<T, addressSpace>(addr)));
+    return cl::sycl::atomic_fetch_add(obj, operand, memoryOrder);
+}
+
+template <typename T>
+struct fake_complex {
+    T x;
+    T y;
+};
+
+}  // namespace
+
+
+namespace detail {
+
+
+template <cl::sycl::access::address_space addressSpace, typename ValueType,
+          typename = void>
+struct atomic_helper {
+    __dpct_inline__ static ValueType atomic_add(ValueType *, ValueType)
+    {
+        static_assert(sizeof(ValueType) == 0,
+                      "This default function is not implemented, only the "
+                      "specializations are.");
+        // TODO: add proper implementation of generic atomic add
+    }
+};
+
+
+template <typename ResultType, typename ValueType>
+__dpct_inline__ ResultType reinterpret(ValueType val)
+{
+    static_assert(sizeof(ValueType) == sizeof(ResultType),
+                  "The type to reinterpret to must be of the same size as the "
+                  "original type.");
+    return reinterpret_cast<ResultType &>(val);
+}
+
+
+#define GKO_BIND_ATOMIC_HELPER_STRUCTURE(CONVERTER_TYPE)                   \
+    template <cl::sycl::access::address_space addressSpace,                \
+              typename ValueType>                                          \
+    struct atomic_helper<                                                  \
+        addressSpace, ValueType,                                           \
+        std::enable_if_t<(sizeof(ValueType) == sizeof(CONVERTER_TYPE))>> { \
+        __dpct_inline__ static ValueType atomic_add(                       \
+            ValueType *__restrict__ addr, ValueType val)                   \
+        {                                                                  \
+            CONVERTER_TYPE *address_as_converter =                         \
+                reinterpret_cast<CONVERTER_TYPE *>(addr);                  \
+            CONVERTER_TYPE old = *address_as_converter;                    \
+            CONVERTER_TYPE assumed;                                        \
+            do {                                                           \
+                assumed = old;                                             \
+                old = atomic_compare_exchange_strong<addressSpace>(        \
+                    address_as_converter, assumed,                         \
+                    reinterpret<CONVERTER_TYPE>(                           \
+                        val + reinterpret<ValueType>(assumed)));           \
+            } while (assumed != old);                                      \
+            return reinterpret<ValueType>(old);                            \
+        }                                                                  \
+    };
+
+// Support 64-bit ATOMIC_ADD
+GKO_BIND_ATOMIC_HELPER_STRUCTURE(unsigned long long int);
+// Support 32-bit ATOMIC_ADD
+GKO_BIND_ATOMIC_HELPER_STRUCTURE(unsigned int);
+
+
+#undef GKO_BIND_ATOMIC_HELPER_STRUCTURE
+
+#define GKO_BIND_ATOMIC_HELPER_ValueType(ValueType)                         \
+    template <cl::sycl::access::address_space addressSpace>                 \
+    struct atomic_helper<addressSpace, ValueType, std::enable_if_t<true>> { \
+        __dpct_inline__ static ValueType atomic_add(                        \
+            ValueType *__restrict__ addr, ValueType val)                    \
+        {                                                                   \
+            return atomic_fetch_add<addressSpace>(addr, val);               \
+        }                                                                   \
+    };
+
+GKO_BIND_ATOMIC_HELPER_ValueType(int);
+GKO_BIND_ATOMIC_HELPER_ValueType(unsigned int);
+GKO_BIND_ATOMIC_HELPER_ValueType(unsigned long long int);
+
+
+template <cl::sycl::access::address_space addressSpace, typename ValueType>
+struct atomic_helper<
+    addressSpace, ValueType,
+    std::enable_if_t<is_complex<ValueType>() && sizeof(ValueType) >= 16>> {
+    __dpct_inline__ static ValueType atomic_add(ValueType *__restrict__ addr,
+                                                ValueType val)
+    {
+        using real_type = remove_complex<ValueType>;
+        fake_complex<real_type> *fake_addr =
+            reinterpret_cast<fake_complex<real_type> *>(addr);
+        // Separate to real part and imag part
+        auto real = atomic_helper<addressSpace, real_type>::atomic_add(
+            &(fake_addr->x), val.real());
+        auto imag = atomic_helper<addressSpace, real_type>::atomic_add(
+            &(fake_addr->y), val.imag());
+        return {real, imag};
+    }
+};
+
+
+}  // namespace detail
+
+
+template <cl::sycl::access::address_space addressSpace = atomic::global_space,
+          typename T>
+__dpct_inline__ T atomic_add(T *__restrict__ addr, T val)
+{
+    return detail::atomic_helper<addressSpace, T>::atomic_add(addr, val);
+}
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_DPCPP_COMPONENTS_ATOMIC_DP_HPP_
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index 0f335c745a2..57e91c12c63 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -52,8 +52,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/base/allocator.hpp"
 #include "core/base/iterator_factory.hpp"
 #include "core/base/utils.hpp"
+#include "core/components/fill_array.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/csr_builder.hpp"
+#include "dpcpp/components/atomic.dp.hpp"
 #include "dpcpp/components/format_conversion.dp.hpp"
 
 
@@ -629,10 +631,64 @@ void transpose_and_transform(std::shared_ptr<const DpcppExecutor> exec,
                              UnaryOperator op) GKO_NOT_IMPLEMENTED;
 
 
+template <bool conjugate, typename ValueType, typename IndexType>
+void generic_transpose(std::shared_ptr<const DpcppExecutor> exec,
+                       const matrix::Csr<ValueType, IndexType> *orig,
+                       matrix::Csr<ValueType, IndexType> *trans)
+{
+    const auto num_rows = orig->get_size()[0];
+    const auto num_cols = orig->get_size()[1];
+    auto queue = exec->get_queue();
+    const auto row_ptrs = orig->get_const_row_ptrs();
+    const auto cols = orig->get_const_col_idxs();
+    const auto vals = orig->get_const_values();
+
+    Array<IndexType> counts{exec, num_cols + 1};
+    auto tmp_counts = counts.get_data();
+    auto out_row_ptrs = trans->get_row_ptrs();
+    auto out_cols = trans->get_col_idxs();
+    auto out_vals = trans->get_values();
+    components::fill_array(exec, tmp_counts, num_cols, IndexType{});
+
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto row = static_cast<size_type>(idx[0]);
+            const auto begin = row_ptrs[row];
+            const auto end = row_ptrs[row + 1];
+            for (auto i = begin; i < end; i++) {
+                atomic_fetch_add(tmp_counts + cols[i], IndexType{1});
+            }
+        });
+    });
+
+    components::prefix_sum(exec, tmp_counts, num_cols + 1);
+    exec->copy(num_cols + 1, tmp_counts, out_row_ptrs);
+
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto row = static_cast<size_type>(idx[0]);
+            const auto begin = row_ptrs[row];
+            const auto end = row_ptrs[row + 1];
+            for (auto i = begin; i < end; i++) {
+                auto out_nz =
+                    atomic_fetch_add(tmp_counts + cols[i], IndexType{1});
+                out_cols[out_nz] = row;
+                out_vals[out_nz] = conjugate ? conj(vals[i]) : vals[i];
+            }
+        });
+    });
+
+    sort_by_column_index(exec, trans);
+}
+
+
 template <typename ValueType, typename IndexType>
 void transpose(std::shared_ptr<const DpcppExecutor> exec,
                const matrix::Csr<ValueType, IndexType> *orig,
-               matrix::Csr<ValueType, IndexType> *trans) GKO_NOT_IMPLEMENTED;
+               matrix::Csr<ValueType, IndexType> *trans)
+{
+    generic_transpose<false>(exec, orig, trans);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_TRANSPOSE_KERNEL);
 
@@ -641,7 +697,9 @@ template <typename ValueType, typename IndexType>
 void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Csr<ValueType, IndexType> *orig,
                     matrix::Csr<ValueType, IndexType> *trans)
-    GKO_NOT_IMPLEMENTED;
+{
+    generic_transpose<true>(exec, orig, trans);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONJ_TRANSPOSE_KERNEL);
diff --git a/dpcpp/test/matrix/csr_kernels.cpp b/dpcpp/test/matrix/csr_kernels.cpp
index fb8ae74d685..3c76f2760b1 100644
--- a/dpcpp/test/matrix/csr_kernels.cpp
+++ b/dpcpp/test/matrix/csr_kernels.cpp
@@ -65,6 +65,8 @@ class Csr : public ::testing::Test {
     using Arr = gko::Array<int>;
     using Mtx = gko::matrix::Csr<value_type>;
     using Vec = gko::matrix::Dense<value_type>;
+    using ComplexVec = gko::matrix::Dense<std::complex<value_type>>;
+    using ComplexMtx = gko::matrix::Csr<std::complex<value_type>>;
 
     Csr() : mtx_size(532, 231), rand_engine(42) {}
 
@@ -102,12 +104,17 @@ class Csr : public ::testing::Test {
     {
         mtx = Mtx::create(ref);
         mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[1], 1));
+        complex_mtx = ComplexMtx::create(ref);
+        complex_mtx->copy_from(
+            gen_mtx<ComplexVec>(mtx_size[0], mtx_size[1], 1));
         square_mtx = Mtx::create(ref);
         square_mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[0], 1));
         alpha = gko::initialize<Vec>({2.0}, ref);
         beta = gko::initialize<Vec>({-1.0}, ref);
         dmtx = Mtx::create(dpcpp);
         dmtx->copy_from(mtx.get());
+        complex_dmtx = ComplexMtx::create(dpcpp);
+        complex_dmtx->copy_from(complex_mtx.get());
         square_dmtx = Mtx::create(dpcpp);
         square_dmtx->copy_from(square_mtx.get());
         dalpha = Vec::create(dpcpp);
@@ -141,11 +148,13 @@ class Csr : public ::testing::Test {
     std::ranlux48 rand_engine;
 
     std::unique_ptr<Mtx> mtx;
+    std::unique_ptr<ComplexMtx> complex_mtx;
     std::unique_ptr<Mtx> square_mtx;
     std::unique_ptr<Vec> alpha;
     std::unique_ptr<Vec> beta;
 
     std::unique_ptr<Mtx> dmtx;
+    std::unique_ptr<ComplexMtx> complex_dmtx;
     std::unique_ptr<Mtx> square_dmtx;
     std::unique_ptr<Vec> dalpha;
     std::unique_ptr<Vec> dbeta;
@@ -258,6 +267,30 @@ TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, TransposeIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto trans = gko::as<Mtx>(mtx->transpose());
+    auto d_trans = gko::as<Mtx>(dmtx->transpose());
+
+    GKO_ASSERT_MTX_NEAR(d_trans, trans, 0.0);
+    ASSERT_TRUE(d_trans->is_sorted_by_column_index());
+}
+
+
+TEST_F(Csr, ConjugateTransposeIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto trans = gko::as<ComplexMtx>(complex_mtx->conj_transpose());
+    auto d_trans = gko::as<ComplexMtx>(complex_dmtx->conj_transpose());
+
+    GKO_ASSERT_MTX_NEAR(d_trans, trans, 0.0);
+    ASSERT_TRUE(d_trans->is_sorted_by_column_index());
+}
+
+
 TEST_F(Csr, RecognizeSortedMatrixIsEquivalentToRef)
 {
     set_up_apply_data();

From 668e655ab43751a61d0a1a0c0fab03f68ffe46e9 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Wed, 14 Jul 2021 09:29:10 +0200
Subject: [PATCH 0874/1008] review updates
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 dpcpp/components/atomic.dp.hpp  | 33 +++++++++++++++------------------
 dpcpp/matrix/csr_kernels.dp.cpp |  3 ++-
 2 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/dpcpp/components/atomic.dp.hpp b/dpcpp/components/atomic.dp.hpp
index 7bedf2cbe48..78191d4da4b 100644
--- a/dpcpp/components/atomic.dp.hpp
+++ b/dpcpp/components/atomic.dp.hpp
@@ -42,16 +42,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "dpcpp/base/dpct.hpp"
 
-// #include "dpct/atomic.hpp"
 
 namespace gko {
 namespace kernels {
 namespace dpcpp {
 namespace atomic {
-constexpr cl::sycl::access::address_space local_space =
-    cl::sycl::access::address_space::local_space;
-constexpr cl::sycl::access::address_space global_space =
-    cl::sycl::access::address_space::global_space;
+
+
+constexpr auto local_space = cl::sycl::access::address_space::local_space;
+constexpr auto global_space = cl::sycl::access::address_space::global_space;
+
+
 }  // namespace atomic
 
 namespace {
@@ -93,11 +94,6 @@ inline T atomic_fetch_add(
     return cl::sycl::atomic_fetch_add(obj, operand, memoryOrder);
 }
 
-template <typename T>
-struct fake_complex {
-    T x;
-    T y;
-};
 
 }  // namespace
 
@@ -160,7 +156,7 @@ GKO_BIND_ATOMIC_HELPER_STRUCTURE(unsigned int);
 
 #undef GKO_BIND_ATOMIC_HELPER_STRUCTURE
 
-#define GKO_BIND_ATOMIC_HELPER_ValueType(ValueType)                         \
+#define GKO_BIND_ATOMIC_HELPER_VALUETYPE(ValueType)                         \
     template <cl::sycl::access::address_space addressSpace>                 \
     struct atomic_helper<addressSpace, ValueType, std::enable_if_t<true>> { \
         __dpct_inline__ static ValueType atomic_add(                        \
@@ -170,9 +166,11 @@ GKO_BIND_ATOMIC_HELPER_STRUCTURE(unsigned int);
         }                                                                   \
     };
 
-GKO_BIND_ATOMIC_HELPER_ValueType(int);
-GKO_BIND_ATOMIC_HELPER_ValueType(unsigned int);
-GKO_BIND_ATOMIC_HELPER_ValueType(unsigned long long int);
+GKO_BIND_ATOMIC_HELPER_VALUETYPE(int);
+GKO_BIND_ATOMIC_HELPER_VALUETYPE(unsigned int);
+GKO_BIND_ATOMIC_HELPER_VALUETYPE(unsigned long long int);
+
+#undef GKO_BIND_ATOMIC_HELPER_VALUETYPE
 
 
 template <cl::sycl::access::address_space addressSpace, typename ValueType>
@@ -183,13 +181,12 @@ struct atomic_helper<
                                                 ValueType val)
     {
         using real_type = remove_complex<ValueType>;
-        fake_complex<real_type> *fake_addr =
-            reinterpret_cast<fake_complex<real_type> *>(addr);
+        real_type *real_addr = reinterpret_cast<real_type *>(addr);
         // Separate to real part and imag part
         auto real = atomic_helper<addressSpace, real_type>::atomic_add(
-            &(fake_addr->x), val.real());
+            &real_addr[0], val.real());
         auto imag = atomic_helper<addressSpace, real_type>::atomic_add(
-            &(fake_addr->y), val.imag());
+            &real_addr[1], val.imag());
         return {real, imag};
     }
 };
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index 57e91c12c63..747844e21e9 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
+#include <limits>
 #include <numeric>
 #include <utility>
 
@@ -874,7 +875,7 @@ void is_sorted_by_column_index(
             }
         });
     });
-    exec->get_master()->copy_from(exec.get(), 1, is_sorted_device, is_sorted);
+    *is_sorted = exec->copy_val_to_host(is_sorted_device);
 };
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(

From a618e89e6fd41525ac347f99287afa0ed62194f2 Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Fri, 9 Jul 2021 11:38:59 +0200
Subject: [PATCH 0875/1008] fixes dummy-hook's .git directory for multiple work
 trees

---
 third_party/dummy-hook/CMakeLists.txt | 37 +++++++++++++++++----------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/third_party/dummy-hook/CMakeLists.txt b/third_party/dummy-hook/CMakeLists.txt
index 043ae8da917..0a64b65515b 100644
--- a/third_party/dummy-hook/CMakeLists.txt
+++ b/third_party/dummy-hook/CMakeLists.txt
@@ -1,18 +1,27 @@
-if(EXISTS "${Ginkgo_SOURCE_DIR}/.git")
-    set(ADD_HOOK FALSE)
-    set(HOOK_LOCATION "${Ginkgo_SOURCE_DIR}/.git/hooks/pre-commit")
-    if(NOT EXISTS "${HOOK_LOCATION}")
-        set(ADD_HOOK TRUE)
-    else()
-        # check if the correct hook is installed
-        execute_process(COMMAND grep git-cmake-format.py "${HOOK_LOCATION}"
-                        RESULT_VARIABLE res OUTPUT_QUIET)
-        # return value =/= 0 means the pattern was not found
-        if(NOT res EQUAL 0)
+find_package(Git QUIET)
+if(GIT_FOUND)
+    execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse --git-path hooks
+      WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+      OUTPUT_VARIABLE Ginkgo_GIT_HOOKS_DIR
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+    get_filename_component(Ginkgo_GIT_HOOKS_DIR ${Ginkgo_GIT_HOOKS_DIR} REALPATH BASE_DIR ${CMAKE_SOURCE_DIR})
+
+    if(EXISTS "${Ginkgo_GIT_HOOKS_DIR}")
+        set(ADD_HOOK FALSE)
+        set(HOOK_LOCATION "${Ginkgo_GIT_HOOKS_DIR}/pre-commit")
+        if(NOT EXISTS "${HOOK_LOCATION}")
             set(ADD_HOOK TRUE)
+        else()
+            # check if the correct hook is installed
+            execute_process(COMMAND grep git-cmake-format.py "${HOOK_LOCATION}"
+                            RESULT_VARIABLE res OUTPUT_QUIET)
+            # return value =/= 0 means the pattern was not found
+            if(NOT res EQUAL 0)
+                set(ADD_HOOK TRUE)
+            endif()
+        endif()
+        if(ADD_HOOK)
+            configure_file(dummy_hook "${HOOK_LOCATION}" COPYONLY)
         endif()
-    endif()
-    if(ADD_HOOK)
-        configure_file(dummy_hook "${HOOK_LOCATION}" COPYONLY)
     endif()
 endif()

From 10bee50c713c1de8a5a5b4ddc9f6207ff1ee092e Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Fri, 9 Jul 2021 15:20:04 +0200
Subject: [PATCH 0876/1008] includes git-cmake-format fix

---
 third_party/git-cmake-format/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/third_party/git-cmake-format/CMakeLists.txt b/third_party/git-cmake-format/CMakeLists.txt
index fe1a44ad15a..31d333b351d 100644
--- a/third_party/git-cmake-format/CMakeLists.txt
+++ b/third_party/git-cmake-format/CMakeLists.txt
@@ -3,7 +3,7 @@ include(FetchContent)
 FetchContent_Declare(
     git_cmake_format
     GIT_REPOSITORY https://github.com/ginkgo-project/git-cmake-format.git
-    GIT_TAG        be9554a5d71030dcc23f1eb355c41ab9d876e776
+    GIT_TAG        069338e8afd47e6a27e661940cd36e940253f81c
 )
 FetchContent_GetProperties(git_cmake_format)
 if(NOT git_cmake_format_POPULATED)

From 67f7c16fedcada83bcba3672b570f07a0b795c23 Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Wed, 14 Jul 2021 14:20:48 +0200
Subject: [PATCH 0877/1008] enables overwriting hook by default

user can still define GCF_FORCE_OVERWRITE to skip overwrite
---
 third_party/git-cmake-format/CMakeLists.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/third_party/git-cmake-format/CMakeLists.txt b/third_party/git-cmake-format/CMakeLists.txt
index 31d333b351d..98f175495d9 100644
--- a/third_party/git-cmake-format/CMakeLists.txt
+++ b/third_party/git-cmake-format/CMakeLists.txt
@@ -8,5 +8,9 @@ FetchContent_Declare(
 FetchContent_GetProperties(git_cmake_format)
 if(NOT git_cmake_format_POPULATED)
     FetchContent_Populate(git_cmake_format)
+
+    if(NOT DEFINED GCF_FORCE_OVERWRITE)
+        set(GCF_FORCE_OVERWRITE ON CACHE INTERNAL "If true, always overwrite pre-commit hook and script")
+    endif()
     add_subdirectory(${git_cmake_format_SOURCE_DIR} ${git_cmake_format_BINARY_DIR} EXCLUDE_FROM_ALL)
 endif()

From dddaad04deb4b7c14831f9cdce3669834aa0613e Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Tue, 13 Jul 2021 17:36:15 +0200
Subject: [PATCH 0878/1008] clarify that git hooks may be overwritten

---
 INSTALL.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/INSTALL.md b/INSTALL.md
index 377014512ff..34b44108fe4 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -17,7 +17,9 @@ Ginkgo adds the following additional switches to control what is being built:
 
 *   `-DGINKGO_DEVEL_TOOLS={ON, OFF}` sets up the build system for development
     (requires clang-format, will also download git-cmake-format),
-    default is `OFF`.
+    default is `OFF`. If it is set to `ON`, a pre-commit hook for formatting
+    will be installed. This hook may overwrite a user defined pre-commit hook
+    when Ginkgo is used as a submodule.
 *   `-DGINKGO_MIXED_PRECISION={ON, OFF}` compiles true mixed-precision kernels
     instead of converting data on the fly, default is `OFF`.
     Enabling this flag increases the library size, but improves performance of

From f3ddf84a7ed1c4630d4298d1bd41466e1ee584a3 Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Thu, 15 Jul 2021 09:22:32 +0200
Subject: [PATCH 0879/1008] Review update:

- extend documentation to cover dummy hook

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 INSTALL.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index 34b44108fe4..0b7e21acf16 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -17,9 +17,11 @@ Ginkgo adds the following additional switches to control what is being built:
 
 *   `-DGINKGO_DEVEL_TOOLS={ON, OFF}` sets up the build system for development
     (requires clang-format, will also download git-cmake-format),
-    default is `OFF`. If it is set to `ON`, a pre-commit hook for formatting
-    will be installed. This hook may overwrite a user defined pre-commit hook
-    when Ginkgo is used as a submodule.
+    default is `OFF`. The default behavior installs a pre-commit hook, which
+    disables git commits.  If it is set to `ON`, a new pre-commit hook for
+    formatting will be installed (enabling commits again). In both cases the
+    hook may overwrite a user defined pre-commit hook when Ginkgo is used as
+    a submodule.
 *   `-DGINKGO_MIXED_PRECISION={ON, OFF}` compiles true mixed-precision kernels
     instead of converting data on the fly, default is `OFF`.
     Enabling this flag increases the library size, but improves performance of

From 9b41d5674b9eac236a9c82f5dbe530080b8028b0 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 15 Jul 2021 00:22:41 +0200
Subject: [PATCH 0880/1008] ResidualNorm stop can compute the needed info in
 gen/check - ResidualNormRedouction can get initial_res from given args in
 generate - others can use solution to compute residual in check

---
 core/stop/residual_norm.cpp                   |  21 ++
 include/ginkgo/core/stop/residual_norm.hpp    |  48 +++-
 reference/test/stop/residual_norm_kernels.cpp | 249 ++++++++++++++++++
 3 files changed, 308 insertions(+), 10 deletions(-)

diff --git a/core/stop/residual_norm.cpp b/core/stop/residual_norm.cpp
index c4c37162408..7976d035a5c 100644
--- a/core/stop/residual_norm.cpp
+++ b/core/stop/residual_norm.cpp
@@ -75,6 +75,27 @@ bool ResidualNormBase<ValueType>::check_impl(
             dense_r->compute_norm2(u_dense_tau_.get());
         }
         dense_tau = u_dense_tau_.get();
+    } else if (updater.solution_ != nullptr && system_matrix_ != nullptr &&
+               b_ != nullptr) {
+        auto exec = this->get_executor();
+        if (auto vec_b = std::dynamic_pointer_cast<const ComplexVector>(b_)) {
+            auto dense_r = vec_b->clone();
+            auto neg_one = gko::initialize<ComplexVector>({-1}, exec);
+            auto one = gko::initialize<ComplexVector>({1}, exec);
+            system_matrix_->apply(neg_one.get(), updater.solution_, one.get(),
+                                  dense_r.get());
+            dense_r->compute_norm2(u_dense_tau_.get());
+        } else if (auto vec_b = std::dynamic_pointer_cast<const Vector>(b_)) {
+            auto dense_r = vec_b->clone();
+            auto neg_one = gko::initialize<Vector>({-1}, exec);
+            auto one = gko::initialize<Vector>({1}, exec);
+            system_matrix_->apply(neg_one.get(), updater.solution_, one.get(),
+                                  dense_r.get());
+            dense_r->compute_norm2(u_dense_tau_.get());
+        } else {
+            GKO_NOT_SUPPORTED(nullptr);
+        }
+        dense_tau = u_dense_tau_.get();
     } else {
         GKO_NOT_SUPPORTED(nullptr);
     }
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index e2456403643..94b91b2bcdb 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -100,21 +100,47 @@ class ResidualNormBase
         : EnablePolymorphicObject<ResidualNormBase, Criterion>(exec),
           device_storage_{exec, 2},
           reduction_factor_{reduction_factor},
-          baseline_{baseline}
+          baseline_{baseline},
+          system_matrix_{args.system_matrix},
+          b_{args.b}
     {
         switch (baseline_) {
         case mode::initial_resnorm: {
             if (args.initial_residual == nullptr) {
-                GKO_NOT_SUPPORTED(nullptr);
-            }
-            this->starting_tau_ = NormVector::create(
-                exec, dim<2>{1, args.initial_residual->get_size()[1]});
-            if (dynamic_cast<const ComplexVector *>(args.initial_residual)) {
-                auto dense_r = as<ComplexVector>(args.initial_residual);
-                dense_r->compute_norm2(this->starting_tau_.get());
+                if (args.system_matrix == nullptr || args.b == nullptr ||
+                    args.x == nullptr) {
+                    GKO_NOT_SUPPORTED(nullptr);
+                } else {
+                    this->starting_tau_ = NormVector::create(
+                        exec, dim<2>{1, args.b->get_size()[1]});
+                    auto neg_one = share(initialize<Vector>({-1}, exec));
+                    auto one = share(initialize<Vector>({1}, exec));
+                    auto b_clone = share(args.b->clone());
+                    args.system_matrix->apply(neg_one.get(), args.x, one.get(),
+                                              b_clone.get());
+                    if (auto vec =
+                            std::dynamic_pointer_cast<const ComplexVector>(
+                                b_clone)) {
+                        vec->compute_norm2(this->starting_tau_.get());
+                    } else if (auto vec =
+                                   std::dynamic_pointer_cast<const Vector>(
+                                       b_clone)) {
+                        vec->compute_norm2(this->starting_tau_.get());
+                    } else {
+                        GKO_NOT_SUPPORTED(nullptr);
+                    }
+                }
             } else {
-                auto dense_r = as<Vector>(args.initial_residual);
-                dense_r->compute_norm2(this->starting_tau_.get());
+                this->starting_tau_ = NormVector::create(
+                    exec, dim<2>{1, args.initial_residual->get_size()[1]});
+                if (dynamic_cast<const ComplexVector *>(
+                        args.initial_residual)) {
+                    auto dense_r = as<ComplexVector>(args.initial_residual);
+                    dense_r->compute_norm2(this->starting_tau_.get());
+                } else {
+                    auto dense_r = as<Vector>(args.initial_residual);
+                    dense_r->compute_norm2(this->starting_tau_.get());
+                }
             }
             break;
         }
@@ -152,6 +178,8 @@ class ResidualNormBase
     remove_complex<ValueType> reduction_factor_{};
     std::unique_ptr<NormVector> starting_tau_{};
     std::unique_ptr<NormVector> u_dense_tau_{};
+    std::shared_ptr<const LinOp> system_matrix_{};
+    std::shared_ptr<const LinOp> b_{};
     /* Contains device side: all_converged and one_changed booleans */
     Array<bool> device_storage_;
 
diff --git a/reference/test/stop/residual_norm_kernels.cpp b/reference/test/stop/residual_norm_kernels.cpp
index b7676c807ee..78721d0f1ae 100644
--- a/reference/test/stop/residual_norm_kernels.cpp
+++ b/reference/test/stop/residual_norm_kernels.cpp
@@ -237,6 +237,205 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoal)
 }
 
 
+TYPED_TEST(ResidualNorm, SelfCalulatesThrowWithoutMatrix)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using NormVector = typename TestFixture::NormVector;
+    using T = TypeParam;
+    using T_nc = gko::remove_complex<TypeParam>;
+    auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
+
+    T rhs_val = 10.0;
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({rhs_val}, this->exec_);
+    auto rhs_criterion =
+        this->rhs_factory_->generate(nullptr, rhs, nullptr, initial_res.get());
+    auto rel_criterion =
+        this->rel_factory_->generate(nullptr, rhs, nullptr, initial_res.get());
+    auto abs_criterion =
+        this->abs_factory_->generate(nullptr, rhs, nullptr, initial_res.get());
+    {
+        auto solution = gko::initialize<Mtx>({rhs_val - T{10.0}}, this->exec_);
+        auto rhs_norm = gko::initialize<NormVector>({100.0}, this->exec_);
+        gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+        constexpr gko::uint8 RelativeStoppingId{1};
+        bool one_changed{};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+        stop_status.get_data()[0].reset();
+
+        ASSERT_THROW(
+            rhs_criterion->update()
+                .solution(solution.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed),
+            gko::NotSupported);
+    }
+    {
+        T initial_norm = 100.0;
+        auto solution =
+            gko::initialize<Mtx>({rhs_val - initial_norm}, this->exec_);
+        constexpr gko::uint8 RelativeStoppingId{1};
+        bool one_changed{};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+        stop_status.get_data()[0].reset();
+
+        ASSERT_THROW(
+            rel_criterion->update()
+                .solution(solution.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed),
+            gko::NotSupported);
+    }
+    {
+        auto solution = gko::initialize<Mtx>({rhs_val - T{100.0}}, this->exec_);
+        constexpr gko::uint8 RelativeStoppingId{1};
+        bool one_changed{};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+        stop_status.get_data()[0].reset();
+
+        ASSERT_THROW(
+            abs_criterion->update()
+                .solution(solution.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed),
+            gko::NotSupported);
+    }
+}
+
+
+TYPED_TEST(ResidualNorm, RelativeSelfCalulatesThrowWithoutRhs)
+{
+    // only relative residual norm allows generation without rhs.
+    using Mtx = typename TestFixture::Mtx;
+    using NormVector = typename TestFixture::NormVector;
+    using T = TypeParam;
+    using T_nc = gko::remove_complex<TypeParam>;
+    auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
+
+    T rhs_val = 10.0;
+    auto rel_criterion = this->rel_factory_->generate(nullptr, nullptr, nullptr,
+                                                      initial_res.get());
+    T initial_norm = 100.0;
+    auto solution = gko::initialize<Mtx>({rhs_val - initial_norm}, this->exec_);
+    constexpr gko::uint8 RelativeStoppingId{1};
+    bool one_changed{};
+    gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+    stop_status.get_data()[0].reset();
+
+    ASSERT_THROW(
+        rel_criterion->update()
+            .solution(solution.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed),
+        gko::NotSupported);
+}
+
+
+TYPED_TEST(ResidualNorm, SelfCalulatesAndWaitsTillResidualGoal)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using NormVector = typename TestFixture::NormVector;
+    using T = TypeParam;
+    using T_nc = gko::remove_complex<TypeParam>;
+    auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
+    auto system_mtx = share(gko::initialize<Mtx>({1.0}, this->exec_));
+
+    T rhs_val = 10.0;
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({rhs_val}, this->exec_);
+    auto rhs_criterion = this->rhs_factory_->generate(system_mtx, rhs, nullptr,
+                                                      initial_res.get());
+    auto rel_criterion = this->rel_factory_->generate(system_mtx, rhs, nullptr,
+                                                      initial_res.get());
+    auto abs_criterion = this->abs_factory_->generate(system_mtx, rhs, nullptr,
+                                                      initial_res.get());
+    {
+        auto solution = gko::initialize<Mtx>({rhs_val - T{10.0}}, this->exec_);
+        auto rhs_norm = gko::initialize<NormVector>({100.0}, this->exec_);
+        gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+        constexpr gko::uint8 RelativeStoppingId{1};
+        bool one_changed{};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+        stop_status.get_data()[0].reset();
+
+        ASSERT_FALSE(
+            rhs_criterion->update()
+                .solution(solution.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        solution->at(0) = rhs_val - r<T>::value * T{1.1} * rhs_norm->at(0);
+        ASSERT_FALSE(
+            rhs_criterion->update()
+                .solution(solution.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
+        ASSERT_EQ(one_changed, false);
+
+        solution->at(0) = rhs_val - r<T>::value * T{0.9} * rhs_norm->at(0);
+        ASSERT_TRUE(
+            rhs_criterion->update()
+                .solution(solution.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+    {
+        T initial_norm = 100.0;
+        auto solution =
+            gko::initialize<Mtx>({rhs_val - initial_norm}, this->exec_);
+        constexpr gko::uint8 RelativeStoppingId{1};
+        bool one_changed{};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+        stop_status.get_data()[0].reset();
+
+        ASSERT_FALSE(
+            rel_criterion->update()
+                .solution(solution.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        solution->at(0) = rhs_val - r<T>::value * T{1.1} * initial_norm;
+        ASSERT_FALSE(
+            rel_criterion->update()
+                .solution(solution.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
+        ASSERT_EQ(one_changed, false);
+
+        solution->at(0) = rhs_val - r<T>::value * T{0.9} * initial_norm;
+        ASSERT_TRUE(
+            rel_criterion->update()
+                .solution(solution.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+    {
+        auto solution = gko::initialize<Mtx>({rhs_val - T{100.0}}, this->exec_);
+        constexpr gko::uint8 RelativeStoppingId{1};
+        bool one_changed{};
+        gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+        stop_status.get_data()[0].reset();
+
+        ASSERT_FALSE(
+            abs_criterion->update()
+                .solution(solution.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+        solution->at(0) = rhs_val - r<T>::value * T{1.2};
+        ASSERT_FALSE(
+            abs_criterion->update()
+                .solution(solution.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
+        ASSERT_EQ(one_changed, false);
+
+        solution->at(0) = rhs_val - r<T>::value * T{0.9};
+        ASSERT_TRUE(
+            abs_criterion->update()
+                .solution(solution.get())
+                .check(RelativeStoppingId, true, &stop_status, &one_changed));
+        ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+        ASSERT_EQ(one_changed, true);
+    }
+}
+
+
 TYPED_TEST(ResidualNorm, WaitsTillResidualGoalMultipleRHS)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -370,6 +569,20 @@ class ResidualNormReduction : public ::testing::Test {
 TYPED_TEST_SUITE(ResidualNormReduction, gko::test::ValueTypes);
 
 
+TYPED_TEST(ResidualNormReduction,
+           CanCreateCriterionWithMtxRhsXWithoutInitialRes)
+{
+    using Mtx = typename TestFixture::Mtx;
+    std::shared_ptr<gko::LinOp> x = gko::initialize<Mtx>({100.0}, this->exec_);
+    std::shared_ptr<gko::LinOp> mtx = gko::initialize<Mtx>({1.0}, this->exec_);
+    std::shared_ptr<gko::LinOp> b = gko::initialize<Mtx>({10.0}, this->exec_);
+
+    auto criterion = this->factory_->generate(mtx, b, x.get());
+
+    ASSERT_NE(criterion, nullptr);
+}
+
+
 TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoal)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -407,6 +620,42 @@ TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoal)
 }
 
 
+TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoalWithoutInitialRes)
+{
+    using T = TypeParam;
+    using Mtx = typename TestFixture::Mtx;
+    using NormVector = typename TestFixture::NormVector;
+    T initial_res = 100;
+    T rhs_val = 10;
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({rhs_val}, this->exec_);
+    std::shared_ptr<Mtx> x =
+        gko::initialize<Mtx>({rhs_val - initial_res}, this->exec_);
+    std::shared_ptr<gko::LinOp> mtx = gko::initialize<Mtx>({1.0}, this->exec_);
+
+    auto criterion = this->factory_->generate(mtx, rhs, x.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
+    stop_status.get_data()[0].reset();
+
+    ASSERT_FALSE(criterion->update().solution(x.get()).check(
+        RelativeStoppingId, true, &stop_status, &one_changed));
+
+    x->at(0) = rhs_val - r<T>::value * T{1.1} * initial_res;
+    ASSERT_FALSE(criterion->update().solution(x.get()).check(
+        RelativeStoppingId, true, &stop_status, &one_changed));
+    ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
+    ASSERT_EQ(one_changed, false);
+
+    x->at(0) = rhs_val - r<T>::value * T{0.9} * initial_res;
+    ASSERT_TRUE(criterion->update().solution(x.get()).check(
+        RelativeStoppingId, true, &stop_status, &one_changed));
+    ASSERT_EQ(stop_status.get_data()[0].has_converged(), true);
+    ASSERT_EQ(one_changed, true);
+}
+
+
 TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
 {
     using Mtx = typename TestFixture::Mtx;

From 4f1a03f0a4a6ce04087e1c79c5c16162b79e5545 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 15 Jul 2021 21:57:51 +0200
Subject: [PATCH 0881/1008] allocate one/neg_one in the class

move to private by sonar
---
 core/stop/residual_norm.cpp                | 15 +++++++--------
 include/ginkgo/core/stop/residual_norm.hpp | 17 ++++++++++-------
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/core/stop/residual_norm.cpp b/core/stop/residual_norm.cpp
index 7976d035a5c..73630204a22 100644
--- a/core/stop/residual_norm.cpp
+++ b/core/stop/residual_norm.cpp
@@ -78,18 +78,17 @@ bool ResidualNormBase<ValueType>::check_impl(
     } else if (updater.solution_ != nullptr && system_matrix_ != nullptr &&
                b_ != nullptr) {
         auto exec = this->get_executor();
-        if (auto vec_b = std::dynamic_pointer_cast<const ComplexVector>(b_)) {
+        // when LinOp is real but rhs is complex, we use real view on complex,
+        // so it still uses the same type of scalar in apply.
+        if (auto vec_b = std::dynamic_pointer_cast<const Vector>(b_)) {
             auto dense_r = vec_b->clone();
-            auto neg_one = gko::initialize<ComplexVector>({-1}, exec);
-            auto one = gko::initialize<ComplexVector>({1}, exec);
-            system_matrix_->apply(neg_one.get(), updater.solution_, one.get(),
+            system_matrix_->apply(neg_one_.get(), updater.solution_, one_.get(),
                                   dense_r.get());
             dense_r->compute_norm2(u_dense_tau_.get());
-        } else if (auto vec_b = std::dynamic_pointer_cast<const Vector>(b_)) {
+        } else if (auto vec_b =
+                       std::dynamic_pointer_cast<const ComplexVector>(b_)) {
             auto dense_r = vec_b->clone();
-            auto neg_one = gko::initialize<Vector>({-1}, exec);
-            auto one = gko::initialize<Vector>({1}, exec);
-            system_matrix_->apply(neg_one.get(), updater.solution_, one.get(),
+            system_matrix_->apply(neg_one_.get(), updater.solution_, one_.get(),
                                   dense_r.get());
             dense_r->compute_norm2(u_dense_tau_.get());
         } else {
diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp
index 94b91b2bcdb..8dbbdc88544 100644
--- a/include/ginkgo/core/stop/residual_norm.hpp
+++ b/include/ginkgo/core/stop/residual_norm.hpp
@@ -102,7 +102,9 @@ class ResidualNormBase
           reduction_factor_{reduction_factor},
           baseline_{baseline},
           system_matrix_{args.system_matrix},
-          b_{args.b}
+          b_{args.b},
+          one_{gko::initialize<Vector>({1}, exec)},
+          neg_one_{gko::initialize<Vector>({-1}, exec)}
     {
         switch (baseline_) {
         case mode::initial_resnorm: {
@@ -113,11 +115,9 @@ class ResidualNormBase
                 } else {
                     this->starting_tau_ = NormVector::create(
                         exec, dim<2>{1, args.b->get_size()[1]});
-                    auto neg_one = share(initialize<Vector>({-1}, exec));
-                    auto one = share(initialize<Vector>({1}, exec));
                     auto b_clone = share(args.b->clone());
-                    args.system_matrix->apply(neg_one.get(), args.x, one.get(),
-                                              b_clone.get());
+                    args.system_matrix->apply(neg_one_.get(), args.x,
+                                              one_.get(), b_clone.get());
                     if (auto vec =
                             std::dynamic_pointer_cast<const ComplexVector>(
                                 b_clone)) {
@@ -178,13 +178,16 @@ class ResidualNormBase
     remove_complex<ValueType> reduction_factor_{};
     std::unique_ptr<NormVector> starting_tau_{};
     std::unique_ptr<NormVector> u_dense_tau_{};
-    std::shared_ptr<const LinOp> system_matrix_{};
-    std::shared_ptr<const LinOp> b_{};
     /* Contains device side: all_converged and one_changed booleans */
     Array<bool> device_storage_;
 
 private:
     mode baseline_{mode::rhs_norm};
+    std::shared_ptr<const LinOp> system_matrix_{};
+    std::shared_ptr<const LinOp> b_{};
+    /* one/neg_one for residual computation */
+    std::shared_ptr<const Vector> one_{};
+    std::shared_ptr<const Vector> neg_one_{};
 };
 
 

From a2d0488f55f4b8440a4a462db9db94de88a2cf89 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 9 Jul 2021 21:01:36 +0200
Subject: [PATCH 0882/1008] cp error between diff backend on the same device

---
 .gitlab-ci.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cbb9931f3b5..35d25dc9d61 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -100,6 +100,7 @@ include:
       fi
     - if [ -n "${SYCL_DEVICE_TYPE}" ]; then export SYCL_DEVICE_TYPE; fi
     - if [ -n "${SYCL_DEVICE_FILTER}" ]; then export SYCL_DEVICE_FILTER; fi
+    - if [ -n "${SYCL_DEVICE_TYPE}" ]; then export SYCL_DEVICE_TYPE; fi
     - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
         -GNinja
         -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${CXX_COMPILER}
@@ -148,6 +149,7 @@ include:
       fi
     - if [ -n "${SYCL_DEVICE_TYPE}" ]; then unset SYCL_DEVICE_TYPE; fi
     - if [ -n "${SYCL_DEVICE_FILTER}" ]; then unset SYCL_DEVICE_FILTER; fi
+    - if [ -n "${SYCL_DEVICE_TYPE}" ]; then unset SYCL_DEVICE_TYPE; fi
     - if [ "${EXPORT_BUILD_DIR}" == "ON" ]; then ninja test_exportbuild; fi
   dependencies: []
   except:
@@ -750,7 +752,7 @@ build/dpcpp/dgpu/debug/shared:
     BUILD_DPCPP: "ON"
     BUILD_TYPE: "Debug"
     DPCPP_SINGLE_MODE: "ON"
-    SYCL_DEVICE_FILTER: "Level_Zero:GPU"
+    SYCL_DEVICE_TYPE: "GPU"
 
 
 # Job with important warnings as error

From 3b21de1b5abcdc667ba67a98506710c91ddefcad Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Sat, 10 Jul 2021 00:58:22 +0200
Subject: [PATCH 0883/1008] memcpy only when their device are the same note the
 device here already consider backend

---
 dpcpp/base/executor.dp.cpp        | 20 +++++++++++---------
 include/ginkgo/core/base/math.hpp |  2 +-
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 39c3e4557e6..7bcffd8e78d 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -144,7 +144,13 @@ void DpcppExecutor::raw_copy_to(const DpcppExecutor *dest, size_type num_bytes,
                                 const void *src_ptr, void *dest_ptr) const
 {
     if (num_bytes > 0) {
-        dest->get_queue()->memcpy(dest_ptr, src_ptr, num_bytes).wait();
+        if (this->get_queue()->get_device() !=
+            dest->get_queue()->get_device()) {
+            // the memcpy only support host<->device or itself memcpy
+            GKO_NOT_SUPPORTED(dest);
+        } else {
+            dest->get_queue()->memcpy(dest_ptr, src_ptr, num_bytes).wait();
+        }
     }
 }
 
@@ -176,15 +182,11 @@ bool DpcppExecutor::verify_memory_to(const OmpExecutor *dest_exec) const
 
 bool DpcppExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
 {
-    auto device = detail::get_devices(
-        get_exec_info().device_type)[get_exec_info().device_id];
-    auto other_device = detail::get_devices(
-        dest_exec->get_device_type())[dest_exec->get_device_id()];
+    auto device = this->get_queue()->get_device();
+    auto dest_device = dest_exec->get_queue()->get_device();
     return ((device.is_host() || device.is_cpu()) &&
-            (other_device.is_host() || other_device.is_cpu())) ||
-           (device.get_info<cl::sycl::info::device::device_type>() ==
-                other_device.get_info<cl::sycl::info::device::device_type>() &&
-            device.get() == other_device.get());
+            (dest_device.is_host() || dest_device.is_cpu())) ||
+           (device == dest_device);
 }
 
 
diff --git a/include/ginkgo/core/base/math.hpp b/include/ginkgo/core/base/math.hpp
index 2cd1784b28e..81f7349daa8 100644
--- a/include/ginkgo/core/base/math.hpp
+++ b/include/ginkgo/core/base/math.hpp
@@ -41,7 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <type_traits>
 
 
-#ifdef CL_SYCL_LANGUAGE_VERSION
+#ifdef SYCL_LANGUAGE_VERSION
 #include <CL/sycl.hpp>
 #endif
 

From 5da5dd759f4df386b9be169f3aff06467a7efc2c Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Sun, 11 Jul 2021 14:02:55 +0200
Subject: [PATCH 0884/1008] add a test to check free after kernel

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 dpcpp/base/executor.dp.cpp      |  1 -
 dpcpp/test/base/executor.dp.cpp | 59 ++++++++++++++++++++++++---------
 2 files changed, 43 insertions(+), 17 deletions(-)

diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 7bcffd8e78d..035843500c2 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -102,7 +102,6 @@ void DpcppExecutor::populate_exec_info(const MachineTopology *mach_topo)
 
 void DpcppExecutor::raw_free(void *ptr) const noexcept
 {
-    queue_->wait_and_throw();
     sycl::free(ptr, queue_->get_context());
 }
 
diff --git a/dpcpp/test/base/executor.dp.cpp b/dpcpp/test/base/executor.dp.cpp
index 39d9e1e3313..830543349d5 100644
--- a/dpcpp/test/base/executor.dp.cpp
+++ b/dpcpp/test/base/executor.dp.cpp
@@ -55,20 +55,20 @@ namespace {
 class DpcppExecutor : public ::testing::Test {
 protected:
     DpcppExecutor()
-        : omp(gko::OmpExecutor::create()), dpcpp(nullptr), dpcpp2(nullptr)
+        : ref(gko::ReferenceExecutor::create()), dpcpp(nullptr), dpcpp2(nullptr)
     {}
 
     void SetUp()
     {
         if (gko::DpcppExecutor::get_num_devices("gpu") > 0) {
-            dpcpp = gko::DpcppExecutor::create(0, omp, "gpu");
+            dpcpp = gko::DpcppExecutor::create(0, ref, "gpu");
             if (gko::DpcppExecutor::get_num_devices("gpu") > 1) {
-                dpcpp2 = gko::DpcppExecutor::create(1, omp, "gpu");
+                dpcpp2 = gko::DpcppExecutor::create(1, ref, "gpu");
             }
         } else if (gko::DpcppExecutor::get_num_devices("cpu") > 0) {
-            dpcpp = gko::DpcppExecutor::create(0, omp, "cpu");
+            dpcpp = gko::DpcppExecutor::create(0, ref, "cpu");
             if (gko::DpcppExecutor::get_num_devices("cpu") > 1) {
-                dpcpp2 = gko::DpcppExecutor::create(1, omp, "cpu");
+                dpcpp2 = gko::DpcppExecutor::create(1, ref, "cpu");
             }
         } else {
             GKO_NOT_IMPLEMENTED;
@@ -84,7 +84,7 @@ class DpcppExecutor : public ::testing::Test {
         }
     }
 
-    std::shared_ptr<gko::Executor> omp{};
+    std::shared_ptr<gko::Executor> ref{};
     std::shared_ptr<const gko::DpcppExecutor> dpcpp{};
     std::shared_ptr<const gko::DpcppExecutor> dpcpp2{};
 };
@@ -92,9 +92,9 @@ class DpcppExecutor : public ::testing::Test {
 
 TEST_F(DpcppExecutor, CanInstantiateTwoExecutorsOnOneDevice)
 {
-    auto dpcpp = gko::DpcppExecutor::create(0, omp);
+    auto dpcpp = gko::DpcppExecutor::create(0, ref);
     if (dpcpp2 != nullptr) {
-        auto dpcpp2 = gko::DpcppExecutor::create(0, omp);
+        auto dpcpp2 = gko::DpcppExecutor::create(0, ref);
     }
 
     // We want automatic deinitialization to not create any error
@@ -103,7 +103,7 @@ TEST_F(DpcppExecutor, CanInstantiateTwoExecutorsOnOneDevice)
 
 TEST_F(DpcppExecutor, CanGetExecInfo)
 {
-    dpcpp = gko::DpcppExecutor::create(0, omp);
+    dpcpp = gko::DpcppExecutor::create(0, ref);
 
     ASSERT_TRUE(dpcpp->get_num_computing_units() > 0);
     ASSERT_TRUE(dpcpp->get_subgroup_sizes().size() > 0);
@@ -191,9 +191,9 @@ TEST_F(DpcppExecutor, CopiesDataToCPU)
 {
     int orig[] = {3, 8};
     auto *copy = dpcpp->alloc<int>(2);
-    gko::Array<bool> is_set(omp, 1);
+    gko::Array<bool> is_set(ref, 1);
 
-    dpcpp->copy_from(omp.get(), 2, orig, copy);
+    dpcpp->copy_from(ref.get(), 2, orig, copy);
 
     is_set.set_executor(dpcpp);
     ASSERT_NO_THROW(dpcpp->synchronize());
@@ -201,7 +201,7 @@ TEST_F(DpcppExecutor, CopiesDataToCPU)
         auto *is_set_ptr = is_set.get_data();
         cgh.single_task([=]() { check_data(copy, is_set_ptr); });
     }));
-    is_set.set_executor(omp);
+    is_set.set_executor(ref);
     ASSERT_EQ(*is_set.get_data(), true);
     ASSERT_NO_THROW(dpcpp->synchronize());
     dpcpp->free(copy);
@@ -221,7 +221,7 @@ TEST_F(DpcppExecutor, CopiesDataFromCPU)
         cgh.single_task([=]() { init_data(orig); });
     });
 
-    omp->copy_from(dpcpp.get(), 2, orig, copy);
+    ref->copy_from(dpcpp.get(), 2, orig, copy);
 
     EXPECT_EQ(3, copy[0]);
     ASSERT_EQ(8, copy[1]);
@@ -236,7 +236,7 @@ TEST_F(DpcppExecutor, CopiesDataFromDpcppToDpcpp)
     }
 
     int copy[2];
-    gko::Array<bool> is_set(omp, 1);
+    gko::Array<bool> is_set(ref, 1);
     auto orig = dpcpp->alloc<int>(2);
     dpcpp->get_queue()->submit([&](sycl::handler &cgh) {
         cgh.single_task([=]() { init_data(orig); });
@@ -250,11 +250,11 @@ TEST_F(DpcppExecutor, CopiesDataFromDpcppToDpcpp)
         auto *is_set_ptr = is_set.get_data();
         cgh.single_task([=]() { check_data(copy_dpcpp2, is_set_ptr); });
     }));
-    is_set.set_executor(omp);
+    is_set.set_executor(ref);
     ASSERT_EQ(*is_set.get_data(), true);
 
     // Put the results on OpenMP and run CPU side assertions
-    omp->copy_from(dpcpp2.get(), 2, copy_dpcpp2, copy);
+    ref->copy_from(dpcpp2.get(), 2, copy_dpcpp2, copy);
     EXPECT_EQ(3, copy[0]);
     ASSERT_EQ(8, copy[1]);
     dpcpp2->free(copy_dpcpp2);
@@ -269,4 +269,31 @@ TEST_F(DpcppExecutor, Synchronizes)
 }
 
 
+#define GTEST_ASSERT_NO_EXIT(statement) \
+    ASSERT_EXIT({ {statement} exit(0); }, ::testing::ExitedWithCode(0), "")
+
+
+TEST_F(DpcppExecutor, FreeAfterKernel)
+{
+    testing::FLAGS_gtest_death_test_style = "threadsafe";
+    GTEST_ASSERT_NO_EXIT({
+        size_t length = 10000;
+        auto dpcpp =
+            gko::DpcppExecutor::create(0, gko::ReferenceExecutor::create());
+        {
+            gko::Array<float> x(dpcpp, length);
+            gko::Array<float> y(dpcpp, length);
+            auto x_val = x.get_data();
+            auto y_val = y.get_data();
+            dpcpp->get_queue()->submit([&](sycl::handler &cgh) {
+                cgh.parallel_for(sycl::range<1>{length},
+                                 [=](sycl::id<1> i) { y_val[i] += x_val[i]; });
+            });
+        }
+        // to ensure everything on queue is finished.
+        dpcpp->synchronize();
+    });
+}
+
+
 }  // namespace

From cb924d4269054c135cc1071abf2b0c6947f832bd Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 13 Jul 2021 15:48:02 +0200
Subject: [PATCH 0885/1008] add wait_and_throw, capture exception,add job/test

---
 .gitlab-ci.yml              | 69 +++++++++++++++++++++++++++++++++----
 core/test/base/executor.cpp | 12 +++++++
 dpcpp/base/executor.dp.cpp  | 51 ++++++++++++++++++++++-----
 3 files changed, 118 insertions(+), 14 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 35d25dc9d61..750c209fd67 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -100,7 +100,6 @@ include:
       fi
     - if [ -n "${SYCL_DEVICE_TYPE}" ]; then export SYCL_DEVICE_TYPE; fi
     - if [ -n "${SYCL_DEVICE_FILTER}" ]; then export SYCL_DEVICE_FILTER; fi
-    - if [ -n "${SYCL_DEVICE_TYPE}" ]; then export SYCL_DEVICE_TYPE; fi
     - cmake ${CI_PROJECT_DIR}${CI_PROJECT_DIR_SUFFIX}
         -GNinja
         -DCMAKE_C_COMPILER=${C_COMPILER} -DCMAKE_CXX_COMPILER=${CXX_COMPILER}
@@ -149,7 +148,6 @@ include:
       fi
     - if [ -n "${SYCL_DEVICE_TYPE}" ]; then unset SYCL_DEVICE_TYPE; fi
     - if [ -n "${SYCL_DEVICE_FILTER}" ]; then unset SYCL_DEVICE_FILTER; fi
-    - if [ -n "${SYCL_DEVICE_TYPE}" ]; then unset SYCL_DEVICE_TYPE; fi
     - if [ "${EXPORT_BUILD_DIR}" == "ON" ]; then ninja test_exportbuild; fi
   dependencies: []
   except:
@@ -724,11 +722,26 @@ build/dpcpp/cpu/release/static:
     BUILD_SHARED_LIBS: "ON"
     SYCL_DEVICE_TYPE: "CPU"
 
-build/dpcpp/igpu/release/static:
+build/dpcpp/igpu/release/shared:
   <<: *default_build_with_test
   extends:
     - .full_test_condition
     - .use_gko-oneapi-igpu
+  variables:
+    <<: *default_variables
+    C_COMPILER: "gcc"
+    CXX_COMPILER: "dpcpp"
+    BUILD_DPCPP: "ON"
+    BUILD_TYPE: "Release"
+    BUILD_SHARED_LIBS: "ON"
+    DPCPP_SINGLE_MODE: "ON"
+    SYCL_DEVICE_TYPE: "GPU"
+
+build/dpcpp/opencl_igpu/release/static:
+  <<: *default_build_with_test
+  extends:
+    - .quick_test_condition
+    - .use_gko-oneapi-igpu
   variables:
     <<: *default_variables
     C_COMPILER: "gcc"
@@ -737,14 +750,58 @@ build/dpcpp/igpu/release/static:
     BUILD_TYPE: "Release"
     BUILD_SHARED_LIBS: "OFF"
     DPCPP_SINGLE_MODE: "ON"
+    SYCL_DEVICE_FILTER: "OpenCL"
+    SYCL_DEVICE_TYPE: "GPU"
+
+build/dpcpp/level_zero_igpu/debug/static:
+  <<: *default_build_with_test
+  extends:
+    - .full_test_condition
+    - .use_gko-oneapi-igpu
+  variables:
+    <<: *default_variables
+    C_COMPILER: "gcc"
+    CXX_COMPILER: "dpcpp"
+    BUILD_DPCPP: "ON"
+    BUILD_TYPE: "Debug"
+    BUILD_SHARED_LIBS: "OFF"
+    DPCPP_SINGLE_MODE: "ON"
     SYCL_DEVICE_FILTER: "Level_Zero:GPU"
 
-build/dpcpp/dgpu/debug/shared:
+build/dpcpp/dgpu/release/static:
+  <<: *default_build_with_test
+  extends:
+    - .quick_test_condition
+    - .use_gko-oneapi-igpu
+  variables:
+    <<: *default_variables
+    C_COMPILER: "gcc"
+    CXX_COMPILER: "dpcpp"
+    BUILD_DPCPP: "ON"
+    BUILD_TYPE: "Release"
+    BUILD_SHARED_LIBS: "OF"
+    DPCPP_SINGLE_MODE: "ON"
+    SYCL_DEVICE_TYPE: "GPU"
+
+build/dpcpp/level_zero_dgpu/release/shared:
   <<: *default_build_with_test
-  image: localhost:5000/gko-oneapi
   extends:
     - .quick_test_condition
     - .use_gko-oneapi-dgpu
+  variables:
+    <<: *default_variables
+    C_COMPILER: "gcc"
+    CXX_COMPILER: "dpcpp"
+    BUILD_DPCPP: "ON"
+    BUILD_TYPE: "Release"
+    DPCPP_SINGLE_MODE: "ON"
+    SYCL_DEVICE_FILTER: "Level_Zero:GPU"
+
+build/dpcpp/opencl_dgpu/debug/shared:
+  <<: *default_build_with_test
+  extends:
+    - .full_test_condition
+    - .use_gko-oneapi-dgpu
   variables:
     <<: *default_variables
     C_COMPILER: "gcc"
@@ -752,9 +809,9 @@ build/dpcpp/dgpu/debug/shared:
     BUILD_DPCPP: "ON"
     BUILD_TYPE: "Debug"
     DPCPP_SINGLE_MODE: "ON"
+    SYCL_DEVICE_FILTER: "OpenCL"
     SYCL_DEVICE_TYPE: "GPU"
 
-
 # Job with important warnings as error
 warnings:
   <<: *default_build
diff --git a/core/test/base/executor.cpp b/core/test/base/executor.cpp
index 39ea82f6b6d..5f1ec600cd2 100644
--- a/core/test/base/executor.cpp
+++ b/core/test/base/executor.cpp
@@ -541,14 +541,20 @@ TEST(Executor, CanVerifyMemory)
     std::shared_ptr<gko::DpcppExecutor> host_dpcpp;
     std::shared_ptr<gko::DpcppExecutor> cpu_dpcpp;
     std::shared_ptr<gko::DpcppExecutor> gpu_dpcpp;
+    std::shared_ptr<gko::DpcppExecutor> host_dpcpp_dup;
+    std::shared_ptr<gko::DpcppExecutor> cpu_dpcpp_dup;
+    std::shared_ptr<gko::DpcppExecutor> gpu_dpcpp_dup;
     if (gko::DpcppExecutor::get_num_devices("host")) {
         host_dpcpp = gko::DpcppExecutor::create(0, omp, "host");
+        host_dpcpp_dup = gko::DpcppExecutor::create(0, omp, "host");
     }
     if (gko::DpcppExecutor::get_num_devices("cpu")) {
         cpu_dpcpp = gko::DpcppExecutor::create(0, omp, "cpu");
+        cpu_dpcpp_dup = gko::DpcppExecutor::create(0, omp, "cpu");
     }
     if (gko::DpcppExecutor::get_num_devices("gpu")) {
         gpu_dpcpp = gko::DpcppExecutor::create(0, omp, "gpu");
+        gpu_dpcpp_dup = gko::DpcppExecutor::create(0, omp, "gpu");
     }
 
     ASSERT_EQ(false, ref->memory_accessible(omp));
@@ -566,18 +572,24 @@ TEST(Executor, CanVerifyMemory)
         ASSERT_EQ(false, ref->memory_accessible(host_dpcpp));
         ASSERT_EQ(true, host_dpcpp->memory_accessible(omp));
         ASSERT_EQ(true, omp->memory_accessible(host_dpcpp));
+        ASSERT_EQ(true, host_dpcpp->memory_accessible(host_dpcpp_dup));
+        ASSERT_EQ(true, host_dpcpp_dup->memory_accessible(host_dpcpp));
     }
     if (gko::DpcppExecutor::get_num_devices("cpu")) {
         ASSERT_EQ(false, ref->memory_accessible(cpu_dpcpp));
         ASSERT_EQ(false, cpu_dpcpp->memory_accessible(ref));
         ASSERT_EQ(true, cpu_dpcpp->memory_accessible(omp));
         ASSERT_EQ(true, omp->memory_accessible(cpu_dpcpp));
+        ASSERT_EQ(true, cpu_dpcpp->memory_accessible(cpu_dpcpp_dup));
+        ASSERT_EQ(true, cpu_dpcpp_dup->memory_accessible(cpu_dpcpp));
     }
     if (gko::DpcppExecutor::get_num_devices("gpu")) {
         ASSERT_EQ(false, gpu_dpcpp->memory_accessible(ref));
         ASSERT_EQ(false, ref->memory_accessible(gpu_dpcpp));
         ASSERT_EQ(false, gpu_dpcpp->memory_accessible(omp));
         ASSERT_EQ(false, omp->memory_accessible(gpu_dpcpp));
+        ASSERT_EQ(false, gpu_dpcpp->memory_accessible(gpu_dpcpp_dup));
+        ASSERT_EQ(false, gpu_dpcpp_dup->memory_accessible(gpu_dpcpp));
     }
 #if GINKGO_HIP_PLATFORM_NVCC
     ASSERT_EQ(true, hip->memory_accessible(cuda));
diff --git a/dpcpp/base/executor.dp.cpp b/dpcpp/base/executor.dp.cpp
index 035843500c2..d0380127732 100644
--- a/dpcpp/base/executor.dp.cpp
+++ b/dpcpp/base/executor.dp.cpp
@@ -102,7 +102,29 @@ void DpcppExecutor::populate_exec_info(const MachineTopology *mach_topo)
 
 void DpcppExecutor::raw_free(void *ptr) const noexcept
 {
-    sycl::free(ptr, queue_->get_context());
+    // the free function may syncronize excution or not, which depends on
+    // implementation or backend, so it is not guaranteed.
+    // TODO: maybe a light wait implementation?
+    try {
+        queue_->wait_and_throw();
+        sycl::free(ptr, queue_->get_context());
+    } catch (cl::sycl::exception &err) {
+#if GKO_VERBOSE_LEVEL >= 1
+        // Unfortunately, if memory free fails, there's not much we can do
+        std::cerr << "Unrecoverable Dpcpp error on device "
+                  << this->get_device_id() << " in " << __func__ << ": "
+                  << err.what() << std::endl
+                  << "Exiting program" << std::endl;
+#endif  // GKO_VERBOSE_LEVEL >= 1
+        // OpenCL error code use 0 for CL_SUCCESS and negative number for others
+        // error. if the error is not from OpenCL, it will return CL_SUCCESS.
+        int err_code = err.get_cl_code();
+        // if return CL_SUCCESS, exit 1 as DPCPP error.
+        if (err_code == 0) {
+            err_code = 1;
+        }
+        std::exit(err_code);
+    }
 }
 
 
@@ -143,12 +165,20 @@ void DpcppExecutor::raw_copy_to(const DpcppExecutor *dest, size_type num_bytes,
                                 const void *src_ptr, void *dest_ptr) const
 {
     if (num_bytes > 0) {
-        if (this->get_queue()->get_device() !=
-            dest->get_queue()->get_device()) {
+        // If the queue is different and is not cpu/host, the queue can not
+        // transfer the data to another queue (on the same device)
+        // Note. it could be changed when we ensure the behavior is expected.
+        auto queue = this->get_queue();
+        auto dest_queue = dest->get_queue();
+        auto device = queue->get_device();
+        auto dest_device = dest_queue->get_device();
+        if (((device.is_host() || device.is_cpu()) &&
+             (dest_device.is_host() || dest_device.is_cpu())) ||
+            (queue == dest_queue)) {
+            dest->get_queue()->memcpy(dest_ptr, src_ptr, num_bytes).wait();
+        } else {
             // the memcpy only support host<->device or itself memcpy
             GKO_NOT_SUPPORTED(dest);
-        } else {
-            dest->get_queue()->memcpy(dest_ptr, src_ptr, num_bytes).wait();
         }
     }
 }
@@ -181,11 +211,16 @@ bool DpcppExecutor::verify_memory_to(const OmpExecutor *dest_exec) const
 
 bool DpcppExecutor::verify_memory_to(const DpcppExecutor *dest_exec) const
 {
-    auto device = this->get_queue()->get_device();
-    auto dest_device = dest_exec->get_queue()->get_device();
+    // If the queue is different and is not cpu/host, the queue can not access
+    // the data from another queue (on the same device)
+    // Note. it could be changed when we ensure the behavior is expected.
+    auto queue = this->get_queue();
+    auto dest_queue = dest_exec->get_queue();
+    auto device = queue->get_device();
+    auto dest_device = dest_queue->get_device();
     return ((device.is_host() || device.is_cpu()) &&
             (dest_device.is_host() || dest_device.is_cpu())) ||
-           (device == dest_device);
+           (queue == dest_queue);
 }
 
 

From d933b3ea5739f9a894b931b4ef1be697f32d1665 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 15 Jul 2021 14:55:22 +0200
Subject: [PATCH 0886/1008] clog for copy via host when verbose>=1, and debug

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 .gitlab-ci.yml                        | 2 ++
 include/ginkgo/core/base/executor.hpp | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 750c209fd67..df3265cb58c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -722,6 +722,7 @@ build/dpcpp/cpu/release/static:
     BUILD_SHARED_LIBS: "ON"
     SYCL_DEVICE_TYPE: "CPU"
 
+# It gives two available backends of GPU on tests
 build/dpcpp/igpu/release/shared:
   <<: *default_build_with_test
   extends:
@@ -768,6 +769,7 @@ build/dpcpp/level_zero_igpu/debug/static:
     DPCPP_SINGLE_MODE: "ON"
     SYCL_DEVICE_FILTER: "Level_Zero:GPU"
 
+# It gives two available backends of GPU on tests
 build/dpcpp/dgpu/release/static:
   <<: *default_build_with_test
   extends:
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index d9ca2e3af32..0d96cb4cd66 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <array>
+#include <iostream>
 #include <memory>
 #include <mutex>
 #include <sstream>
@@ -621,7 +622,12 @@ class Executor : public log::EnableLogging<Executor> {
             this->raw_copy_from(src_exec, num_elems * sizeof(T), src_ptr,
                                 dest_ptr);
         } catch (NotSupported &) {
+#if (GKO_VERBOSE_LEVEL >= 1) && !defined(NDEBUG)
             // Unoptimized copy. Try to go through the masters.
+            // output to log when verbose >= 1 and debug build
+            std::clog << "Not direct copy. Try to copy data from the masters."
+                      << std::endl;
+#endif
             auto src_master = src_exec->get_master().get();
             if (num_elems > 0 && src_master != src_exec) {
                 auto *master_ptr = src_exec->get_master()->alloc<T>(num_elems);

From b155ccf0094c1c3b7683b5ee182e2a2519e1eca8 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 16 Jul 2021 15:02:25 +0200
Subject: [PATCH 0887/1008] check in git then get abs_path, avoid add_dep

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 CMakeLists.txt                              | 5 ++++-
 third_party/dummy-hook/CMakeLists.txt       | 6 +++++-
 third_party/git-cmake-format/CMakeLists.txt | 2 +-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e4aebe07cdf..3a4b8ae08cd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -278,7 +278,10 @@ if(GINKGO_DEVEL_TOOLS)
     add_custom_target(add_license
         COMMAND ${Ginkgo_SOURCE_DIR}/dev_tools/scripts/add_license.sh
         WORKING_DIRECTORY ${Ginkgo_SOURCE_DIR})
-    add_dependencies(format add_license)
+    # if git-cmake-format can not build format target, do not add the dependencies
+    if(TARGET format)
+        add_dependencies(format add_license)
+    endif()
 endif()
 
 # MacOS needs to install bash, gnu-sed, findutils and coreutils
diff --git a/third_party/dummy-hook/CMakeLists.txt b/third_party/dummy-hook/CMakeLists.txt
index 0a64b65515b..7c6b32cc208 100644
--- a/third_party/dummy-hook/CMakeLists.txt
+++ b/third_party/dummy-hook/CMakeLists.txt
@@ -4,8 +4,12 @@ if(GIT_FOUND)
       WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
       OUTPUT_VARIABLE Ginkgo_GIT_HOOKS_DIR
       OUTPUT_STRIP_TRAILING_WHITESPACE)
+    # If it is not in git, give an error and return
+    if(NOT Ginkgo_GIT_HOOKS_DIR)
+        message(STATUS "Ginkgo is not in a git repository, so no git hook was installed")
+        return()
+    endif()
     get_filename_component(Ginkgo_GIT_HOOKS_DIR ${Ginkgo_GIT_HOOKS_DIR} REALPATH BASE_DIR ${CMAKE_SOURCE_DIR})
-
     if(EXISTS "${Ginkgo_GIT_HOOKS_DIR}")
         set(ADD_HOOK FALSE)
         set(HOOK_LOCATION "${Ginkgo_GIT_HOOKS_DIR}/pre-commit")
diff --git a/third_party/git-cmake-format/CMakeLists.txt b/third_party/git-cmake-format/CMakeLists.txt
index 98f175495d9..bc54159d64c 100644
--- a/third_party/git-cmake-format/CMakeLists.txt
+++ b/third_party/git-cmake-format/CMakeLists.txt
@@ -3,7 +3,7 @@ include(FetchContent)
 FetchContent_Declare(
     git_cmake_format
     GIT_REPOSITORY https://github.com/ginkgo-project/git-cmake-format.git
-    GIT_TAG        069338e8afd47e6a27e661940cd36e940253f81c
+    GIT_TAG        e9a82f20d36f1a92d4c52910a8d0a694e7673c54
 )
 FetchContent_GetProperties(git_cmake_format)
 if(NOT git_cmake_format_POPULATED)

From 928103be43a87e0e14e0e6a68d8845888eb184a4 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 19 Feb 2021 21:20:50 +0800
Subject: [PATCH 0888/1008] update format header

---
 dev_tools/scripts/format_header.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/dev_tools/scripts/format_header.sh b/dev_tools/scripts/format_header.sh
index cbe1b2e8ab9..7a24f50bef6 100755
--- a/dev_tools/scripts/format_header.sh
+++ b/dev_tools/scripts/format_header.sh
@@ -121,7 +121,6 @@ GINKGO_LICENSE_BEACON="******************************<GINKGO LICENSE>***********
 
 CONTENT="content.cpp" # Store the residual part (start from namespace)
 BEFORE="before.cpp" # Store the main header and the #ifdef/#define of header file
-BEGIN="begin.cpp" # Store the header before license
 HAS_HIP_RUNTIME="false"
 DURING_LICENSE="false"
 INCLUDE_REGEX="^#include.*"

From 7228f94fd6318b1ba1e500e79e3a12358f632dde Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Sat, 20 Feb 2021 00:28:42 +0800
Subject: [PATCH 0889/1008] auto

---
 dpcpp/components/reduction.dp.hpp        | 278 ++++++++++++++++++++
 dpcpp/components/thread_ids.dp.hpp       | 317 +++++++++++++++++++++++
 dpcpp/components/uninitialized_array.hpp | 113 ++++++++
 3 files changed, 708 insertions(+)
 create mode 100644 dpcpp/components/reduction.dp.hpp
 create mode 100644 dpcpp/components/thread_ids.dp.hpp
 create mode 100644 dpcpp/components/uninitialized_array.hpp

diff --git a/dpcpp/components/reduction.dp.hpp b/dpcpp/components/reduction.dp.hpp
new file mode 100644
index 00000000000..2fe6c516e9c
--- /dev/null
+++ b/dpcpp/components/reduction.dp.hpp
@@ -0,0 +1,278 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_COMPONENTS_REDUCTION_DP_HPP_
+#define GKO_DPCPP_COMPONENTS_REDUCTION_DP_HPP_
+
+
+#include <type_traits>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include "dpcpp/base/config.hpp"
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/base/dpct.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
+#include "dpcpp/components/uninitialized_array.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+constexpr int default_block_size = 512;
+
+
+// #include "common/components/reduction.hpp.inc"
+/**
+ * @internal
+ *
+ * Computes a reduction using the binary operation `reduce_op` on a group
+ * `group`. Each thread contributes with one element `local_data`. The local
+ * thread element is always passed as the first parameter to the `reduce_op`.
+ * The function returns the result of the reduction on all threads.
+ *
+ * @note The function is guaranteed to return the correct value on all threads
+ *       only if `reduce_op` is commutative (in addition to being associative).
+ *       Otherwise, the correct value is returned only to the thread with
+ *       subwarp index 0.
+ */
+template <
+    typename Group, typename ValueType, typename Operator,
+    typename = std::enable_if_t<group::is_communicator_group<Group>::value>>
+__dpct_inline__ ValueType reduce(const Group &group, ValueType local_data,
+                                 Operator reduce_op = Operator{})
+{
+#pragma unroll
+    for (int32 bitmask = 1; bitmask < group.size(); bitmask <<= 1) {
+        const auto remote_data = group.shfl_xor(local_data, bitmask);
+        local_data = reduce_op(local_data, remote_data);
+    }
+    return local_data;
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the index of the thread that has the element with the largest
+ * magnitude among all the threads in the group.
+ * Only the values from threads which set `is_pivoted` to `false` will be
+ * considered.
+ */
+template <
+    typename Group, typename ValueType,
+    typename = std::enable_if_t<group::is_communicator_group<Group>::value>>
+__dpct_inline__ int choose_pivot(const Group &group, ValueType local_data,
+                                 bool is_pivoted)
+{
+    using real = remove_complex<ValueType>;
+    real lmag = is_pivoted ? -one<real>() : abs(local_data);
+    const auto pivot = std::reduce(
+        oneapi::dpl::execution::make_device_policy(dpct::get_default_queue()),
+        group, group.thread_rank(), [&](int lidx, int ridx) {
+            const auto rmag = group.shfl(lmag, ridx);
+            if (rmag > lmag) {
+                lmag = rmag;
+                lidx = ridx;
+            }
+            return lidx;
+        });
+    // pivot operator not commutative, make sure everyone has the same pivot
+    return group.shfl(pivot, 0);
+}
+
+
+/**
+ * @internal
+ *
+ * Computes a reduction using the binary operation `reduce_op` on entire block.
+ * The data for the reduction is taken from the `data` array which has to be of
+ * size `block_size` and accessible from all threads. The `data` array is also
+ * used as work space (so its content will be destroyed in the process), as well
+ * as to store the return value - which is stored in the 0-th position of the
+ * array.
+ */
+template <
+    typename Group, typename ValueType, typename Operator,
+    typename = std::enable_if_t<group::is_synchronizable_group<Group>::value>>
+void reduce(const Group &__restrict__ group, ValueType *__restrict__ data,
+            Operator reduce_op = Operator{})
+{
+    const auto local_id = group.thread_rank();
+
+    for (int k = group.size() / 2; k >= config::warp_size; k /= 2) {
+        group.sync();
+        if (local_id < k) {
+            data[local_id] = reduce_op(data[local_id], data[local_id + k]);
+        }
+    }
+
+    const auto warp = group::tiled_partition<config::warp_size>(group);
+    const auto warp_id = group.thread_rank() / warp.size();
+    if (warp_id > 0) {
+        return;
+    }
+    auto result = std::reduce(
+        oneapi::dpl::execution::make_device_policy(dpct::get_default_queue()),
+        warp, data[warp.thread_rank()], reduce_op);
+    if (warp.thread_rank() == 0) {
+        data[0] = result;
+    }
+}
+
+
+/**
+ * @internal
+ *
+ * Computes a reduction using the binary operation `reduce_op` on an array
+ * `source` of any size. Has to be called a second time on `result` to reduce
+ * an array larger than `block_size`.
+ */
+template <typename Operator, typename ValueType>
+void reduce_array(size_type size, const ValueType *__restrict__ source,
+                  ValueType *__restrict__ result, sycl::nd_item<3> item_ct1,
+                  Operator reduce_op = Operator{})
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    auto thread_result = zero<ValueType>();
+    for (auto i = tidx; i < size;
+         i += item_ct1.get_local_range().get(2) * item_ct1.get_group_range(2)) {
+        thread_result = reduce_op(thread_result, source[i]);
+    }
+    result[item_ct1.get_local_id(2)] = thread_result;
+
+    group::this_thread_block(item_ct1).sync();
+
+    // Stores the result of the reduction inside `result[0]`
+    reduce(group::this_thread_block(item_ct1), result, reduce_op);
+}
+
+
+/**
+ * @internal
+ *
+ * Computes a reduction using the add operation (+) on an array
+ * `source` of any size. Has to be called a second time on `result` to reduce
+ * an array larger than `default_block_size`.
+ */
+template <typename ValueType>
+void reduce_add_array(
+    size_type size, const ValueType *__restrict__ source,
+    ValueType *__restrict__ result, sycl::nd_item<3> item_ct1,
+    UninitializedArray<ValueType, default_block_size> *block_sum)
+{
+    reduce_array(size, source, static_cast<ValueType *>((*block_sum)), item_ct1,
+                 [](const ValueType &x, const ValueType &y) { return x + y; });
+
+    if (item_ct1.get_local_id(2) == 0) {
+        result[item_ct1.get_group(2)] = (*block_sum)[0];
+    }
+}
+
+template <typename ValueType>
+void reduce_add_array(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                      sycl::queue *stream, size_type size,
+                      const ValueType *source, ValueType *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<ValueType, default_block_size>, 0,
+                       sycl::access::mode::read_write,
+                       sycl::access::target::local>
+            block_sum_acc_ct1(cgh);
+
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(
+            sycl::nd_range<3>(global_range, local_range),
+            [=](sycl::nd_item<3> item_ct1) {
+                reduce_add_array(
+                    size, source, result, item_ct1,
+                    (UninitializedArray<ValueType, default_block_size> *)
+                        block_sum_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+/**
+ * Compute a reduction using add operation (+).
+ *
+ * @param exec  Executor associated to the array
+ * @param size  size of the array
+ * @param source  the pointer of the array
+ *
+ * @return the reduction result
+ */
+template <typename ValueType>
+ValueType reduce_add_array(std::shared_ptr<const DpcppExecutor> exec,
+                           size_type size, const ValueType *source)
+{
+    auto block_results_val = source;
+    size_type grid_dim = size;
+    auto block_results = Array<ValueType>(exec);
+    if (size > default_block_size) {
+        const auto n = ceildiv(size, default_block_size);
+        grid_dim = (n <= default_block_size) ? n : default_block_size;
+
+        block_results.resize_and_reset(grid_dim);
+
+        reduce_add_array(grid_dim, default_block_size, 0, exec->get_queue(),
+                         size, source, block_results.get_data());
+
+        block_results_val = block_results.get_const_data();
+    }
+
+    auto d_result = Array<ValueType>(exec, 1);
+
+    reduce_add_array(1, default_block_size, 0, exec->get_queue(), grid_dim,
+                     block_results_val, d_result.get_data());
+    auto answer = exec->copy_val_to_host(d_result.get_const_data());
+    return answer;
+}
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_DPCPP_COMPONENTS_REDUCTION_DP_HPP_
diff --git a/dpcpp/components/thread_ids.dp.hpp b/dpcpp/components/thread_ids.dp.hpp
new file mode 100644
index 00000000000..4f27302dbc5
--- /dev/null
+++ b/dpcpp/components/thread_ids.dp.hpp
@@ -0,0 +1,317 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_COMPONENTS_THREAD_IDS_DP_HPP_
+#define GKO_DPCPP_COMPONENTS_THREAD_IDS_DP_HPP_
+
+
+#include <CL/sycl.hpp>
+
+
+#include "dpcpp/base/config.hpp"
+#include "dpcpp/base/dpct.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The DPCPP thread namespace.
+ *
+ * @ingroup dpcpp_thread
+ */
+namespace thread {
+
+
+// #include "common/components/thread_ids.hpp.inc"
+/**
+ * @internal
+ *
+ * Returns the ID of the block group this thread belongs to.
+ *
+ * @return the ID of the block group this thread belongs to
+ *
+ * @note Assumes that grid dimensions are in standard format:
+ *       `(block_group_size, first_grid_dimension, second grid_dimension)`
+ */
+__dpct_inline__ size_type get_block_group_id(sycl::nd_item<3> item_ct1)
+{
+    return static_cast<size_type>(item_ct1.get_group(0)) *
+               item_ct1.get_group_range(1) +
+           item_ct1.get_group(1);
+}
+
+/**
+ * @internal
+ *
+ * Returns the ID of the block this thread belongs to.
+ *
+ * @return the ID of the block this thread belongs to
+ *
+ * @note Assumes that grid dimensions are in standard format:
+ *       `(block_group_size, first_grid_dimension, second grid_dimension)`
+ */
+__dpct_inline__ size_type get_block_id(sycl::nd_item<3> item_ct1)
+{
+    return get_block_group_id(item_ct1) * item_ct1.get_group_range(2) +
+           item_ct1.get_group(2);
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the local ID of the warp (relative to the block) this thread belongs
+ * to.
+ *
+ * @return the local ID of the warp (relative to the block) this thread belongs
+ *         to
+ *
+ * @note Assumes that block dimensions are in standard format:
+ *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
+ *         config::warp_size)`
+ */
+__dpct_inline__ size_type get_local_warp_id(sycl::nd_item<3> item_ct1)
+{
+    return static_cast<size_type>(item_ct1.get_local_id(0));
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the local ID of the sub-warp (relative to the block) this thread
+ * belongs to.
+ *
+ * @tparam subwarp_size  size of the subwarp
+ *
+ * @return the local ID of the sub-warp (relative to the block) this thread
+ *         belongs to
+ *
+ * @note Assumes that block dimensions are in standard format:
+ *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
+ *         config::warp_size)`
+ */
+template <int subwarp_size>
+__dpct_inline__ size_type get_local_subwarp_id(sycl::nd_item<3> item_ct1)
+{
+    constexpr auto subwarps_per_warp = config::warp_size / subwarp_size;
+    return get_local_warp_id(item_ct1) * subwarps_per_warp +
+           item_ct1.get_local_id(1);
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the local ID of the thread (relative to the block).
+ * to.
+ *
+ * @tparam subwarp_size  size of the subwarp
+ *
+ * @return the local ID of the thread (relative to the block)
+ *
+ * @note Assumes that block dimensions are in standard format:
+ *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
+ *         config::warp_size)`
+ */
+template <int subwarp_size>
+__dpct_inline__ size_type get_local_thread_id(sycl::nd_item<3> item_ct1)
+{
+    return get_local_subwarp_id<subwarp_size>(item_ct1) * subwarp_size +
+           item_ct1.get_local_id(2);
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the global ID of the warp this thread belongs to.
+ *
+ * @tparam warps_per_block  number of warps within each block
+ *
+ * @return the global ID of the warp this thread belongs to.
+ *
+ * @note Assumes that block dimensions and grid dimensions are in standard
+ *       format:
+ *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
+ *         config::warp_size)` and
+ *       `(block_group_size, first_grid_dimension, second grid_dimension)`,
+ *       respectively.
+ */
+template <int warps_per_block>
+__dpct_inline__ size_type get_warp_id(sycl::nd_item<3> item_ct1)
+{
+    return get_block_id(item_ct1) * warps_per_block +
+           get_local_warp_id(item_ct1);
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the global ID of the sub-warp this thread belongs to.
+ *
+ * @tparam subwarp_size  size of the subwarp
+ *
+ * @return the global ID of the sub-warp this thread belongs to.
+ *
+ * @note Assumes that block dimensions and grid dimensions are in standard
+ *       format:
+ *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
+ *         config::warp_size)` and
+ *       `(block_group_size, first_grid_dimension, second grid_dimension)`,
+ *       respectively.
+ */
+template <int subwarp_size, int warps_per_block>
+__dpct_inline__ size_type get_subwarp_id(sycl::nd_item<3> item_ct1)
+{
+    constexpr auto subwarps_per_warp = config::warp_size / subwarp_size;
+    return get_warp_id<warps_per_block>(item_ct1) * subwarps_per_warp +
+           item_ct1.get_local_id(1);
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the global ID of the thread.
+ *
+ * @return the global ID of the thread.
+ *
+ * @tparam subwarp_size  size of the subwarp
+ *
+ * @note Assumes that block dimensions and grid dimensions are in standard
+ *       format:
+ *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
+ *         config::warp_size)` and
+ *       `(block_group_size, first_grid_dimension, second grid_dimension)`,
+ *       respectively.
+ */
+template <int subwarp_size, int warps_per_block>
+__dpct_inline__ size_type get_thread_id(sycl::nd_item<3> item_ct1)
+{
+    return get_subwarp_id<subwarp_size, warps_per_block>(item_ct1) *
+               subwarp_size +
+           item_ct1.get_local_id(2);
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the global ID of the thread in the given index type.
+ * This function assumes one-dimensional thread and block indexing.
+ *
+ * @return the global ID of the thread in the given index type.
+ *
+ * @tparam IndexType  the index type
+ */
+template <typename IndexType = size_type>
+__dpct_inline__ IndexType get_thread_id_flat(sycl::nd_item<3> item_ct1)
+{
+    return item_ct1.get_local_id(2) +
+           static_cast<IndexType>(item_ct1.get_local_range().get(2)) *
+               item_ct1.get_group(2);
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the total number of threads in the given index type.
+ * This function assumes one-dimensional thread and block indexing.
+ *
+ * @return the total number of threads in the given index type.
+ *
+ * @tparam IndexType  the index type
+ */
+template <typename IndexType = size_type>
+__dpct_inline__ IndexType get_thread_num_flat(sycl::nd_item<3> item_ct1)
+{
+    return item_ct1.get_local_range().get(2) *
+           static_cast<IndexType>(item_ct1.get_group_range(2));
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the global ID of the subwarp in the given index type.
+ * This function assumes one-dimensional thread and block indexing
+ * with a power of two block size of at least subwarp_size.
+ *
+ * @return the global ID of the subwarp in the given index type.
+ *
+ * @tparam subwarp_size  the size of the subwarp. Must be a power of two!
+ * @tparam IndexType  the index type
+ */
+template <int subwarp_size, typename IndexType = size_type>
+__dpct_inline__ IndexType get_subwarp_id_flat(sycl::nd_item<3> item_ct1)
+{
+    static_assert(!(subwarp_size & (subwarp_size - 1)),
+                  "subwarp_size must be a power of two");
+    return item_ct1.get_local_id(2) / subwarp_size +
+           static_cast<IndexType>(item_ct1.get_local_range().get(2) /
+                                  subwarp_size) *
+               item_ct1.get_group(2);
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the total number of subwarps in the given index type.
+ * This function assumes one-dimensional thread and block indexing
+ * with a power of two block size of at least subwarp_size.
+ *
+ * @return the total number of subwarps in the given index type.
+ *
+ * @tparam subwarp_size  the size of the subwarp. Must be a power of two!
+ * @tparam IndexType  the index type
+ */
+template <int subwarp_size, typename IndexType = size_type>
+__dpct_inline__ IndexType get_subwarp_num_flat(sycl::nd_item<3> item_ct1)
+{
+    static_assert(!(subwarp_size & (subwarp_size - 1)),
+                  "subwarp_size must be a power of two");
+    return item_ct1.get_local_range().get(2) / subwarp_size *
+           static_cast<IndexType>(item_ct1.get_group_range(2));
+}
+
+}  // namespace thread
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_DPCPP_COMPONENTS_THREAD_IDS_DP_HPP_
diff --git a/dpcpp/components/uninitialized_array.hpp b/dpcpp/components/uninitialized_array.hpp
new file mode 100644
index 00000000000..b8d3006007d
--- /dev/null
+++ b/dpcpp/components/uninitialized_array.hpp
@@ -0,0 +1,113 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_COMPONENTS_UNINITIALIZED_ARRAY_HPP_
+#define GKO_DPCPP_COMPONENTS_UNINITIALIZED_ARRAY_HPP_
+
+
+#include <ginkgo/core/base/types.hpp>
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+// #include "common/components/uninitialized_array.hpp.inc"
+/**
+ * Stores an array with uninitialized contents.
+ *
+ * This class needed for datatypes that do have a non-empty constructor when`
+ * using them as shared memory, for example `thrust::complex<float>`.
+ *
+ * @tparam ValueType the type of values
+ * @tparam size the size of the array
+ */
+template <typename ValueType, size_type size>
+class UninitializedArray {
+public:
+    /**
+     * Operator for casting an UninitializedArray into its constexpr value
+     * pointer.
+     *
+     * @return the constexpr pointer to the first entry of the array.
+     */
+    constexpr GKO_ATTRIBUTES operator ValueType *() const noexcept
+    {
+        return &(*this)[0];
+    }
+
+    /**
+     * Operator for casting an UninitializedArray into its non-const value
+     * pointer.
+     *
+     * @return the non-const pointer to the first entry of the array.
+     */
+    GKO_ATTRIBUTES operator ValueType *() noexcept { return &(*this)[0]; }
+
+    /**
+     * constexpr array access operator.
+     *
+     * @param pos The array index. Using a value outside [0, size) is undefined
+     * behavior.
+     *
+     * @return a reference to the array entry at the given index.
+     */
+    constexpr GKO_ATTRIBUTES ValueType &operator[](size_type pos) const noexcept
+    {
+        return reinterpret_cast<const ValueType *>(data_)[pos];
+    }
+
+    /**
+     * Non-const array access operator.
+     *
+     * @param pos The array index. Using a value outside [0, size) is undefined
+     * behavior.
+     *
+     * @return a reference to the array entry at the given index.
+     */
+    GKO_ATTRIBUTES ValueType &operator[](size_type pos) noexcept
+    {
+        return reinterpret_cast<ValueType *>(data_)[pos];
+    }
+
+private:
+    unsigned char data_[sizeof(ValueType) / sizeof(unsigned char) * size];
+};
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_DPCPP_COMPONENTS_UNINITIALIZED_ARRAY_HPP_

From ffe1230d0b9f673f061009a42c83744fe73b99de Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Sat, 20 Feb 2021 00:37:10 +0800
Subject: [PATCH 0890/1008] manual modification

---
 dpcpp/components/reduction.dp.hpp        | 12 +++++-------
 dpcpp/components/thread_ids.dp.hpp       |  1 +
 dpcpp/components/uninitialized_array.hpp | 10 +++++++---
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/dpcpp/components/reduction.dp.hpp b/dpcpp/components/reduction.dp.hpp
index 2fe6c516e9c..4caf46229c8 100644
--- a/dpcpp/components/reduction.dp.hpp
+++ b/dpcpp/components/reduction.dp.hpp
@@ -57,7 +57,7 @@ namespace kernels {
 namespace dpcpp {
 
 
-constexpr int default_block_size = 512;
+constexpr int default_block_size = 256;
 
 
 // #include "common/components/reduction.hpp.inc"
@@ -105,9 +105,8 @@ __dpct_inline__ int choose_pivot(const Group &group, ValueType local_data,
 {
     using real = remove_complex<ValueType>;
     real lmag = is_pivoted ? -one<real>() : abs(local_data);
-    const auto pivot = std::reduce(
-        oneapi::dpl::execution::make_device_policy(dpct::get_default_queue()),
-        group, group.thread_rank(), [&](int lidx, int ridx) {
+    const auto pivot =
+        reduce(group, group.thread_rank(), [&](int lidx, int ridx) {
             const auto rmag = group.shfl(lmag, ridx);
             if (rmag > lmag) {
                 lmag = rmag;
@@ -150,9 +149,8 @@ void reduce(const Group &__restrict__ group, ValueType *__restrict__ data,
     if (warp_id > 0) {
         return;
     }
-    auto result = std::reduce(
-        oneapi::dpl::execution::make_device_policy(dpct::get_default_queue()),
-        warp, data[warp.thread_rank()], reduce_op);
+    auto result = ::gko::kernels::dpcpp::reduce(warp, data[warp.thread_rank()],
+                                                reduce_op);
     if (warp.thread_rank() == 0) {
         data[0] = result;
     }
diff --git a/dpcpp/components/thread_ids.dp.hpp b/dpcpp/components/thread_ids.dp.hpp
index 4f27302dbc5..8694d6a88c9 100644
--- a/dpcpp/components/thread_ids.dp.hpp
+++ b/dpcpp/components/thread_ids.dp.hpp
@@ -308,6 +308,7 @@ __dpct_inline__ IndexType get_subwarp_num_flat(sycl::nd_item<3> item_ct1)
            static_cast<IndexType>(item_ct1.get_group_range(2));
 }
 
+
 }  // namespace thread
 }  // namespace dpcpp
 }  // namespace kernels
diff --git a/dpcpp/components/uninitialized_array.hpp b/dpcpp/components/uninitialized_array.hpp
index b8d3006007d..fb7575bc202 100644
--- a/dpcpp/components/uninitialized_array.hpp
+++ b/dpcpp/components/uninitialized_array.hpp
@@ -37,6 +37,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
+#include "dpcpp/base/dpct.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace dpcpp {
@@ -61,7 +64,7 @@ class UninitializedArray {
      *
      * @return the constexpr pointer to the first entry of the array.
      */
-    constexpr GKO_ATTRIBUTES operator ValueType *() const noexcept
+    constexpr __dpct_inline__ operator ValueType *() const noexcept
     {
         return &(*this)[0];
     }
@@ -72,7 +75,7 @@ class UninitializedArray {
      *
      * @return the non-const pointer to the first entry of the array.
      */
-    GKO_ATTRIBUTES operator ValueType *() noexcept { return &(*this)[0]; }
+    __dpct_inline__ operator ValueType *() noexcept { return &(*this)[0]; }
 
     /**
      * constexpr array access operator.
@@ -82,7 +85,8 @@ class UninitializedArray {
      *
      * @return a reference to the array entry at the given index.
      */
-    constexpr GKO_ATTRIBUTES ValueType &operator[](size_type pos) const noexcept
+    constexpr __dpct_inline__ ValueType &operator[](size_type pos) const
+        noexcept
     {
         return reinterpret_cast<const ValueType *>(data_)[pos];
     }

From 6370f76fdb8f3b5f5b01b4e5fd4cfbf3f8ac0387 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Sat, 20 Feb 2021 01:04:38 +0800
Subject: [PATCH 0891/1008] auto dense

---
 dpcpp/matrix/dense_kernels.dp.cpp | 1574 ++++++++++++++++++++++++++++-
 1 file changed, 1555 insertions(+), 19 deletions(-)

diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 531cfd52259..36935b1a4d6 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -33,25 +33,30 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/dense_kernels.hpp"
 
 
-#include <algorithm>
+#include <dpcpp/base/cublas_bindings.hpp>
+#include <dpcpp/base/pointer_mode_guard.hpp>
 
 
 #include <CL/sycl.hpp>
 
 
-#include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/range_accessors.hpp>
 #include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/diagonal.hpp>
 #include <ginkgo/core/matrix/ell.hpp>
-#include <ginkgo/core/matrix/hybrid.hpp>
 #include <ginkgo/core/matrix/sellp.hpp>
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
 #include "core/components/prefix_sum.hpp"
+#include "dpcpp/base/config.hpp"
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
+#include "dpcpp/components/reduction.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
+#include "dpcpp/components/uninitialized_array.hpp"
 
 
 namespace gko {
@@ -65,11 +70,1245 @@ namespace dpcpp {
 namespace dense {
 
 
+constexpr auto default_block_size = 512;
+
+
+// #include "common/matrix/dense_kernels.hpp.inc"
+namespace kernel {
+
+
+template <typename ValueType>
+void strided_fill(size_type num_rows, size_type num_cols, size_type stride,
+                  ValueType *__restrict__ mat, ValueType value,
+                  sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_id = global_id / num_cols;
+    const auto col_id = global_id % num_cols;
+    if (row_id < num_rows) {
+        mat[row_id * stride + col_id] = value;
+    }
+}
+
+template <typename ValueType>
+void strided_fill(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                  sycl::queue *stream, size_type num_rows, size_type num_cols,
+                  size_type stride, ValueType *mat, ValueType value)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             strided_fill(num_rows, num_cols, stride, mat,
+                                          value, item_ct1);
+                         });
+    });
+}
+
+
+template <size_type block_size, typename ValueType>
+void scale(size_type num_rows, size_type num_cols, size_type num_alpha_cols,
+           const ValueType *__restrict__ alpha, ValueType *__restrict__ x,
+           size_type stride_x, sycl::nd_item<3> item_ct1)
+{
+    constexpr auto warps_per_block = block_size / config::warp_size;
+    const auto global_id =
+        thread::get_thread_id<config::warp_size, warps_per_block>(item_ct1);
+    const auto row_id = global_id / num_cols;
+    const auto col_id = global_id % num_cols;
+    const auto alpha_id = num_alpha_cols == 1 ? 0 : col_id;
+    if (row_id < num_rows) {
+        x[row_id * stride_x + col_id] =
+            alpha[alpha_id] == zero<ValueType>()
+                ? zero<ValueType>()
+                : x[row_id * stride_x + col_id] * alpha[alpha_id];
+    }
+}
+
+template <size_type block_size, typename ValueType>
+void scale(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+           sycl::queue *stream, size_type num_rows, size_type num_cols,
+           size_type num_alpha_cols, const ValueType *alpha, ValueType *x,
+           size_type stride_x)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             scale<block_size>(num_rows, num_cols,
+                                               num_alpha_cols, alpha, x,
+                                               stride_x, item_ct1);
+                         });
+    });
+}
+
+
+template <size_type block_size, typename ValueType>
+void add_scaled(size_type num_rows, size_type num_cols,
+                size_type num_alpha_cols, const ValueType *__restrict__ alpha,
+                const ValueType *__restrict__ x, size_type stride_x,
+                ValueType *__restrict__ y, size_type stride_y,
+                sycl::nd_item<3> item_ct1)
+{
+    constexpr auto warps_per_block = block_size / config::warp_size;
+    const auto global_id =
+        thread::get_thread_id<config::warp_size, warps_per_block>(item_ct1);
+    const auto row_id = global_id / num_cols;
+    const auto col_id = global_id % num_cols;
+    const auto alpha_id = num_alpha_cols == 1 ? 0 : col_id;
+    if (row_id < num_rows && alpha[alpha_id] != zero<ValueType>()) {
+        y[row_id * stride_y + col_id] +=
+            x[row_id * stride_x + col_id] * alpha[alpha_id];
+    }
+}
+
+template <size_type block_size, typename ValueType>
+void add_scaled(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                sycl::queue *stream, size_type num_rows, size_type num_cols,
+                size_type num_alpha_cols, const ValueType *alpha,
+                const ValueType *x, size_type stride_x, ValueType *y,
+                size_type stride_y)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             add_scaled<block_size>(
+                                 num_rows, num_cols, num_alpha_cols, alpha, x,
+                                 stride_x, y, stride_y, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType>
+void add_scaled_diag(size_type size, const ValueType *__restrict__ alpha,
+                     const ValueType *__restrict__ diag,
+                     ValueType *__restrict__ y, size_type stride_y,
+                     sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+
+    if (tidx >= size) {
+        return;
+    }
+
+    y[tidx * stride_y + tidx] += alpha[0] * diag[tidx];
+}
+
+template <typename ValueType>
+void add_scaled_diag(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                     sycl::queue *stream, size_type size,
+                     const ValueType *alpha, const ValueType *diag,
+                     ValueType *y, size_type stride_y)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             add_scaled_diag(size, alpha, diag, y, stride_y,
+                                             item_ct1);
+                         });
+    });
+}
+
+
+template <size_type block_size, typename OutType, typename CallableGetValue,
+          typename CallableReduce>
+void compute_partial_reduce(size_type num_rows, OutType *__restrict__ work,
+                            CallableGetValue get_value,
+                            CallableReduce reduce_op, sycl::nd_item<3> item_ct1,
+                            UninitializedArray<OutType, block_size> *tmp_work)
+{
+    constexpr auto warps_per_block = block_size / config::warp_size;
+
+    const auto num_blocks = item_ct1.get_group_range(2);
+    const auto local_id =
+        thread::get_local_thread_id<config::warp_size>(item_ct1);
+    const auto global_id =
+        thread::get_thread_id<config::warp_size, warps_per_block>(item_ct1);
+
+    auto tmp = zero<OutType>();
+    for (auto i = global_id; i < num_rows; i += block_size * num_blocks) {
+        tmp = reduce_op(tmp, get_value(i));
+    }
+
+    (*tmp_work)[local_id] = tmp;
+
+    reduce(group::this_thread_block(item_ct1),
+           static_cast<OutType *>((*tmp_work)), reduce_op);
+
+    if (local_id == 0) {
+        work[thread::get_block_id(item_ct1)] = (*tmp_work)[0];
+    }
+}
+
+
+template <size_type block_size, typename ValueType, typename CallableReduce,
+          typename CallableFinalize>
+void finalize_reduce_computation(
+    size_type size, const ValueType *work, ValueType *result,
+    CallableReduce reduce_op, CallableFinalize finalize_op,
+    sycl::nd_item<3> item_ct1,
+    UninitializedArray<ValueType, block_size> *tmp_work)
+{
+    const auto local_id =
+        thread::get_local_thread_id<config::warp_size>(item_ct1);
+
+    ValueType tmp = zero<ValueType>();
+    for (auto i = local_id; i < size; i += block_size) {
+        tmp = reduce_op(tmp, work[i]);
+    }
+
+    (*tmp_work)[local_id] = tmp;
+
+    reduce(group::this_thread_block(item_ct1),
+           static_cast<ValueType *>((*tmp_work)), reduce_op);
+
+    if (local_id == 0) {
+        *result = finalize_op((*tmp_work)[0]);
+    }
+}
+
+
+template <size_type block_size, typename ValueType>
+void compute_partial_dot(size_type num_rows, const ValueType *__restrict__ x,
+                         size_type stride_x, const ValueType *__restrict__ y,
+                         size_type stride_y, ValueType *__restrict__ work,
+                         sycl::nd_item<3> item_ct1,
+                         UninitializedArray<ValueType, block_size> *tmp_work)
+{
+    compute_partial_reduce<block_size>(
+        /*
+        DPCT1007:4: Migration of this CUDA API is not supported by the Intel(R)
+        DPC++ Compatibility Tool.
+        */
+        num_rows,
+        work,
+        [x, stride_x, y, stride_y](size_type i) {
+            return x[i * stride_x] * conj(y[i * stride_y]);
+        },
+        [](const ValueType &x, const ValueType &y) { return x + y; }, item_ct1,
+        tmp_work);
+}
+
+template <size_type block_size, typename ValueType>
+void compute_partial_dot(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                         sycl::queue *stream, size_type num_rows,
+                         const ValueType *x, size_type stride_x,
+                         const ValueType *y, size_type stride_y,
+                         ValueType *work)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<ValueType, block_size>, 0,
+                       sycl::access::mode::read_write,
+                       sycl::access::target::local>
+            tmp_work_acc_ct1(cgh);
+
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             compute_partial_dot<block_size>(
+                                 num_rows, x, stride_x, y, stride_y, work,
+                                 item_ct1,
+                                 (UninitializedArray<ValueType, block_size> *)
+                                     tmp_work_acc_ct1.get_pointer());
+                         });
+    });
+}
+
+
+template <size_type block_size, typename ValueType>
+void finalize_dot_computation(
+    size_type size, const ValueType *work, ValueType *result,
+    sycl::nd_item<3> item_ct1,
+    UninitializedArray<ValueType, block_size> *tmp_work)
+{
+    finalize_reduce_computation<block_size>(
+        size, work, result,
+        [](const ValueType &x, const ValueType &y) { return x + y; },
+        [](const ValueType &x) { return x; }, item_ct1, tmp_work);
+}
+
+template <size_type block_size, typename ValueType>
+void finalize_dot_computation(dim3 grid, dim3 block,
+                              size_t dynamic_shared_memory, sycl::queue *stream,
+                              size_type size, const ValueType *work,
+                              ValueType *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<ValueType, block_size>, 0,
+                       sycl::access::mode::read_write,
+                       sycl::access::target::local>
+            tmp_work_acc_ct1(cgh);
+
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             finalize_dot_computation<block_size>(
+                                 size, work, result, item_ct1,
+                                 (UninitializedArray<ValueType, block_size> *)
+                                     tmp_work_acc_ct1.get_pointer());
+                         });
+    });
+}
+
+
+template <size_type block_size, typename ValueType>
+void compute_partial_norm2(
+    size_type num_rows, const ValueType *__restrict__ x, size_type stride_x,
+    remove_complex<ValueType> *__restrict__ work, sycl::nd_item<3> item_ct1,
+    UninitializedArray<remove_complex<ValueType>, block_size> *tmp_work)
+{
+    using norm_type = remove_complex<ValueType>;
+    compute_partial_reduce<block_size>(
+        num_rows, work,
+        [x, stride_x](size_type i) { return squared_norm(x[i * stride_x]); },
+        [](const norm_type &x, const norm_type &y) { return x + y; }, item_ct1,
+        tmp_work);
+}
+
+template <size_type block_size, typename ValueType>
+void compute_partial_norm2(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                           sycl::queue *stream, size_type num_rows,
+                           const ValueType *x, size_type stride_x,
+                           remove_complex<ValueType> *work)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<
+            UninitializedArray<remove_complex<ValueType>, block_size>, 0,
+            sycl::access::mode::read_write, sycl::access::target::local>
+            tmp_work_acc_ct1(cgh);
+
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(
+            sycl::nd_range<3>(global_range, local_range),
+            [=](sycl::nd_item<3> item_ct1) {
+                compute_partial_norm2<block_size>(
+                    num_rows, x, stride_x, work, item_ct1,
+                    (UninitializedArray<remove_complex<ValueType>, block_size>
+                         *)tmp_work_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+template <size_type block_size, typename ValueType>
+void finalize_norm2_computation(
+    size_type size, const ValueType *work, ValueType *result,
+    sycl::nd_item<3> item_ct1,
+    UninitializedArray<ValueType, block_size> *tmp_work)
+{
+    finalize_reduce_computation<block_size>(
+        size, work, result,
+        [](const ValueType &x, const ValueType &y) { return x + y; },
+        [](const ValueType &x) { return sycl::sqrt((float)x); }, item_ct1,
+        tmp_work);
+}
+
+template <size_type block_size, typename ValueType>
+void finalize_norm2_computation(dim3 grid, dim3 block,
+                                size_t dynamic_shared_memory,
+                                sycl::queue *stream, size_type size,
+                                const ValueType *work, ValueType *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<ValueType, block_size>, 0,
+                       sycl::access::mode::read_write,
+                       sycl::access::target::local>
+            tmp_work_acc_ct1(cgh);
+
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             finalize_norm2_computation<block_size>(
+                                 size, work, result, item_ct1,
+                                 (UninitializedArray<ValueType, block_size> *)
+                                     tmp_work_acc_ct1.get_pointer());
+                         });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void fill_in_coo(size_type num_rows, size_type num_cols, size_type stride,
+                 const size_type *__restrict__ row_ptrs,
+                 const ValueType *__restrict__ source,
+                 IndexType *__restrict__ row_idxs,
+                 IndexType *__restrict__ col_idxs,
+                 ValueType *__restrict__ values, sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    if (tidx < num_rows) {
+        size_type write_to = row_ptrs[tidx];
+
+        for (size_type i = 0; i < num_cols; i++) {
+            if (source[stride * tidx + i] != zero<ValueType>()) {
+                values[write_to] = source[stride * tidx + i];
+                col_idxs[write_to] = i;
+                row_idxs[write_to] = tidx;
+                write_to++;
+            }
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void fill_in_coo(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                 sycl::queue *stream, size_type num_rows, size_type num_cols,
+                 size_type stride, const size_type *row_ptrs,
+                 const ValueType *source, IndexType *row_idxs,
+                 IndexType *col_idxs, ValueType *values)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             fill_in_coo(num_rows, num_cols, stride, row_ptrs,
+                                         source, row_idxs, col_idxs, values,
+                                         item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void count_nnz_per_row(size_type num_rows, size_type num_cols, size_type stride,
+                       const ValueType *__restrict__ work,
+                       IndexType *__restrict__ result,
+                       sycl::nd_item<3> item_ct1)
+{
+    constexpr auto warp_size = config::warp_size;
+    const auto row_idx = thread::get_subwarp_id_flat<warp_size>(item_ct1);
+    auto warp_tile =
+        group::tiled_partition<warp_size>(group::this_thread_block(item_ct1));
+
+    if (row_idx < num_rows) {
+        IndexType part_result{};
+        for (auto i = warp_tile.thread_rank(); i < num_cols; i += warp_size) {
+            if (work[stride * row_idx + i] != zero<ValueType>()) {
+                part_result += 1;
+            }
+        }
+        result[row_idx] = std::reduce(
+            oneapi::dpl::execution::make_device_policy(
+                dpct::get_default_queue()),
+            warp_tile, part_result,
+            [](const size_type &a, const size_type &b) { return a + b; });
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void count_nnz_per_row(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                       sycl::queue *stream, size_type num_rows,
+                       size_type num_cols, size_type stride,
+                       const ValueType *work, IndexType *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             count_nnz_per_row(num_rows, num_cols, stride, work,
+                                               result, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void fill_in_csr(size_type num_rows, size_type num_cols, size_type stride,
+                 const ValueType *__restrict__ source,
+                 IndexType *__restrict__ row_ptrs,
+                 IndexType *__restrict__ col_idxs,
+                 ValueType *__restrict__ values, sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+
+    if (tidx < num_rows) {
+        auto write_to = row_ptrs[tidx];
+        for (auto i = 0; i < num_cols; i++) {
+            if (source[stride * tidx + i] != zero<ValueType>()) {
+                values[write_to] = source[stride * tidx + i];
+                col_idxs[write_to] = i;
+                write_to++;
+            }
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void fill_in_csr(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                 sycl::queue *stream, size_type num_rows, size_type num_cols,
+                 size_type stride, const ValueType *source, IndexType *row_ptrs,
+                 IndexType *col_idxs, ValueType *values)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             fill_in_csr(num_rows, num_cols, stride, source,
+                                         row_ptrs, col_idxs, values, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void fill_in_ell(size_type num_rows, size_type num_cols,
+                 size_type source_stride, const ValueType *__restrict__ source,
+                 size_type max_nnz_per_row, size_type result_stride,
+                 IndexType *__restrict__ col_ptrs,
+                 ValueType *__restrict__ values, sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    if (tidx < num_rows) {
+        IndexType col_idx = 0;
+        for (size_type col = 0; col < num_cols; col++) {
+            if (source[tidx * source_stride + col] != zero<ValueType>()) {
+                col_ptrs[col_idx * result_stride + tidx] = col;
+                values[col_idx * result_stride + tidx] =
+                    source[tidx * source_stride + col];
+                col_idx++;
+            }
+        }
+        for (size_type j = col_idx; j < max_nnz_per_row; j++) {
+            col_ptrs[j * result_stride + tidx] = 0;
+            values[j * result_stride + tidx] = zero<ValueType>();
+        }
+    } else if (tidx < result_stride) {
+        for (size_type j = 0; j < max_nnz_per_row; j++) {
+            col_ptrs[j * result_stride + tidx] = 0;
+            values[j * result_stride + tidx] = zero<ValueType>();
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void fill_in_ell(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                 sycl::queue *stream, size_type num_rows, size_type num_cols,
+                 size_type source_stride, const ValueType *source,
+                 size_type max_nnz_per_row, size_type result_stride,
+                 IndexType *col_ptrs, ValueType *values)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             fill_in_ell(num_rows, num_cols, source_stride,
+                                         source, max_nnz_per_row, result_stride,
+                                         col_ptrs, values, item_ct1);
+                         });
+    });
+}
+
+
+void calculate_slice_lengths(size_type num_rows, size_type slice_size,
+                             int slice_num, size_type stride_factor,
+                             const size_type *__restrict__ nnz_per_row,
+                             size_type *__restrict__ slice_lengths,
+                             size_type *__restrict__ slice_sets,
+                             sycl::nd_item<3> item_ct1)
+{
+    constexpr auto warp_size = config::warp_size;
+    const auto sliceid = item_ct1.get_group(2);
+    const auto tid_in_warp = item_ct1.get_local_id(2);
+
+    if (sliceid * slice_size + tid_in_warp < num_rows) {
+        size_type thread_result = 0;
+        for (size_type i = tid_in_warp; i < slice_size; i += warp_size) {
+            thread_result =
+                (i + slice_size * sliceid < num_rows)
+                    ? max(thread_result, nnz_per_row[sliceid * slice_size + i])
+                    : thread_result;
+        }
+
+        auto warp_tile = group::tiled_partition<warp_size>(
+            group::this_thread_block(item_ct1));
+        auto warp_result = reduce(
+            warp_tile, thread_result,
+            [](const size_type &a, const size_type &b) { return max(a, b); });
+
+        if (tid_in_warp == 0) {
+            auto slice_length =
+                ceildiv(warp_result, stride_factor) * stride_factor;
+            slice_lengths[sliceid] = slice_length;
+            slice_sets[sliceid] = slice_length;
+        }
+    }
+}
+
+void calculate_slice_lengths(dim3 grid, dim3 block,
+                             size_t dynamic_shared_memory, sycl::queue *stream,
+                             size_type num_rows, size_type slice_size,
+                             int slice_num, size_type stride_factor,
+                             const size_type *nnz_per_row,
+                             size_type *slice_lengths, size_type *slice_sets)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             calculate_slice_lengths(num_rows, slice_size,
+                                                     slice_num, stride_factor,
+                                                     nnz_per_row, slice_lengths,
+                                                     slice_sets, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void fill_in_sellp(size_type num_rows, size_type num_cols, size_type slice_size,
+                   size_type stride, const ValueType *__restrict__ source,
+                   size_type *__restrict__ slice_lengths,
+                   size_type *__restrict__ slice_sets,
+                   IndexType *__restrict__ col_idxs,
+                   ValueType *__restrict__ vals, sycl::nd_item<3> item_ct1)
+{
+    const auto global_row = thread::get_thread_id_flat(item_ct1);
+    const auto row = global_row % slice_size;
+    const auto sliceid = global_row / slice_size;
+
+    if (global_row < num_rows) {
+        size_type sellp_ind = slice_sets[sliceid] * slice_size + row;
+
+        for (size_type col = 0; col < num_cols; col++) {
+            auto val = source[global_row * stride + col];
+            if (val != zero<ValueType>()) {
+                col_idxs[sellp_ind] = col;
+                vals[sellp_ind] = val;
+                sellp_ind += slice_size;
+            }
+        }
+        for (size_type i = sellp_ind;
+             i <
+             (slice_sets[sliceid] + slice_lengths[sliceid]) * slice_size + row;
+             i += slice_size) {
+            col_idxs[i] = 0;
+            vals[i] = zero<ValueType>();
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void fill_in_sellp(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                   sycl::queue *stream, size_type num_rows, size_type num_cols,
+                   size_type slice_size, size_type stride,
+                   const ValueType *source, size_type *slice_lengths,
+                   size_type *slice_sets, IndexType *col_idxs, ValueType *vals)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             fill_in_sellp(num_rows, num_cols, slice_size,
+                                           stride, source, slice_lengths,
+                                           slice_sets, col_idxs, vals,
+                                           item_ct1);
+                         });
+    });
+}
+
+
+void reduce_max_nnz(size_type size, const size_type *__restrict__ nnz_per_row,
+                    size_type *__restrict__ result, sycl::nd_item<3> item_ct1,
+                    uint8_t *dpct_local)
+{
+    auto block_max = (size_type *)dpct_local;
+
+    reduce_array(
+        size, nnz_per_row, block_max,
+        [](const size_type &x, const size_type &y) { return max(x, y); });
+
+    if (item_ct1.get_local_id(2) == 0) {
+        result[item_ct1.get_group(2)] = block_max[0];
+    }
+}
+
+void reduce_max_nnz(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                    sycl::queue *stream, size_type size,
+                    const size_type *nnz_per_row, size_type *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<uint8_t, 1, sycl::access::mode::read_write,
+                       sycl::access::target::local>
+            dpct_local_acc_ct1(sycl::range<1>(dynamic_shared_memory), cgh);
+
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             reduce_max_nnz(size, nnz_per_row, result, item_ct1,
+                                            dpct_local_acc_ct1.get_pointer());
+                         });
+    });
+}
+
+
+void reduce_max_nnz_per_slice(size_type num_rows, size_type slice_size,
+                              size_type stride_factor,
+                              const size_type *__restrict__ nnz_per_row,
+                              size_type *__restrict__ result,
+                              sycl::nd_item<3> item_ct1)
+{
+    constexpr auto warp_size = config::warp_size;
+    auto warp_tile =
+        group::tiled_partition<warp_size>(group::this_thread_block(item_ct1));
+    const auto warpid = thread::get_subwarp_id_flat<warp_size>(item_ct1);
+    const auto tid_in_warp = warp_tile.thread_rank();
+    const auto slice_num = ceildiv(num_rows, slice_size);
+
+    size_type thread_result = 0;
+    for (size_type i = tid_in_warp; i < slice_size; i += warp_size) {
+        if (warpid * slice_size + i < num_rows) {
+            thread_result =
+                max(thread_result, nnz_per_row[warpid * slice_size + i]);
+        }
+    }
+
+    auto warp_result = reduce(
+        warp_tile, thread_result,
+        [](const size_type &a, const size_type &b) { return max(a, b); });
+
+    if (tid_in_warp == 0 && warpid < slice_num) {
+        result[warpid] = ceildiv(warp_result, stride_factor) * stride_factor;
+    }
+}
+
+void reduce_max_nnz_per_slice(dim3 grid, dim3 block,
+                              size_t dynamic_shared_memory, sycl::queue *stream,
+                              size_type num_rows, size_type slice_size,
+                              size_type stride_factor,
+                              const size_type *nnz_per_row, size_type *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             reduce_max_nnz_per_slice(
+                                 num_rows, slice_size, stride_factor,
+                                 nnz_per_row, result, item_ct1);
+                         });
+    });
+}
+
+
+void reduce_total_cols(size_type num_slices,
+                       const size_type *__restrict__ max_nnz_per_slice,
+                       size_type *__restrict__ result,
+                       sycl::nd_item<3> item_ct1, uint8_t *dpct_local)
+{
+    auto block_result = (size_type *)dpct_local;
+
+    reduce_array(num_slices, max_nnz_per_slice, block_result,
+                 [](const size_type &x, const size_type &y) { return x + y; });
+
+    if (item_ct1.get_local_id(2) == 0) {
+        result[item_ct1.get_group(2)] = block_result[0];
+    }
+}
+
+void reduce_total_cols(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                       sycl::queue *stream, size_type num_slices,
+                       const size_type *max_nnz_per_slice, size_type *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<uint8_t, 1, sycl::access::mode::read_write,
+                       sycl::access::target::local>
+            dpct_local_acc_ct1(sycl::range<1>(dynamic_shared_memory), cgh);
+
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             reduce_total_cols(
+                                 num_slices, max_nnz_per_slice, result,
+                                 item_ct1, dpct_local_acc_ct1.get_pointer());
+                         });
+    });
+}
+
+
+template <typename IndexType, typename ValueType>
+void symm_permute(size_type num_rows, size_type num_cols,
+                  const IndexType *__restrict__ perm_idxs,
+                  const ValueType *__restrict__ orig, size_type stride_orig,
+                  ValueType *__restrict__ result, size_type stride_result,
+                  sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_id = global_id / num_cols;
+    const auto col_id = global_id % num_cols;
+    if (row_id < num_rows) {
+        result[row_id * stride_result + col_id] =
+            orig[perm_idxs[row_id] * stride_orig + perm_idxs[col_id]];
+    }
+}
+
+template <typename IndexType, typename ValueType>
+void symm_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                  sycl::queue *stream, size_type num_rows, size_type num_cols,
+                  const IndexType *perm_idxs, const ValueType *orig,
+                  size_type stride_orig, ValueType *result,
+                  size_type stride_result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             symm_permute(num_rows, num_cols, perm_idxs, orig,
+                                          stride_orig, result, stride_result,
+                                          item_ct1);
+                         });
+    });
+}
+
+
+template <typename IndexType, typename ValueType>
+void inv_symm_permute(size_type num_rows, size_type num_cols,
+                      const IndexType *__restrict__ perm_idxs,
+                      const ValueType *__restrict__ orig, size_type stride_orig,
+                      ValueType *__restrict__ result, size_type stride_result,
+                      sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_id = global_id / num_cols;
+    const auto col_id = global_id % num_cols;
+    if (row_id < num_rows) {
+        result[perm_idxs[row_id] * stride_result + perm_idxs[col_id]] =
+            orig[row_id * stride_orig + col_id];
+    }
+}
+
+template <typename IndexType, typename ValueType>
+void inv_symm_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                      sycl::queue *stream, size_type num_rows,
+                      size_type num_cols, const IndexType *perm_idxs,
+                      const ValueType *orig, size_type stride_orig,
+                      ValueType *result, size_type stride_result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             inv_symm_permute(num_rows, num_cols, perm_idxs,
+                                              orig, stride_orig, result,
+                                              stride_result, item_ct1);
+                         });
+    });
+}
+
+
+template <typename IndexType, typename ValueType>
+void row_gather(size_type num_rows, size_type num_cols,
+                const IndexType *__restrict__ perm_idxs,
+                const ValueType *__restrict__ orig, size_type stride_orig,
+                ValueType *__restrict__ result, size_type stride_result,
+                sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_id = global_id / num_cols;
+    const auto col_id = global_id % num_cols;
+    if (row_id < num_rows) {
+        result[row_id * stride_result + col_id] =
+            orig[perm_idxs[row_id] * stride_orig + col_id];
+    }
+}
+
+template <typename IndexType, typename ValueType>
+void row_gather(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                sycl::queue *stream, size_type num_rows, size_type num_cols,
+                const IndexType *perm_idxs, const ValueType *orig,
+                size_type stride_orig, ValueType *result,
+                size_type stride_result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             row_gather(num_rows, num_cols, perm_idxs, orig,
+                                        stride_orig, result, stride_result,
+                                        item_ct1);
+                         });
+    });
+}
+
+
+template <typename IndexType, typename ValueType>
+void column_permute(size_type num_rows, size_type num_cols,
+                    const IndexType *__restrict__ perm_idxs,
+                    const ValueType *__restrict__ orig, size_type stride_orig,
+                    ValueType *__restrict__ result, size_type stride_result,
+                    sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_id = global_id / num_cols;
+    const auto col_id = global_id % num_cols;
+    if (row_id < num_rows) {
+        result[row_id * stride_result + col_id] =
+            orig[row_id * stride_orig + perm_idxs[col_id]];
+    }
+}
+
+template <typename IndexType, typename ValueType>
+void column_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                    sycl::queue *stream, size_type num_rows, size_type num_cols,
+                    const IndexType *perm_idxs, const ValueType *orig,
+                    size_type stride_orig, ValueType *result,
+                    size_type stride_result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             column_permute(num_rows, num_cols, perm_idxs, orig,
+                                            stride_orig, result, stride_result,
+                                            item_ct1);
+                         });
+    });
+}
+
+
+template <typename IndexType, typename ValueType>
+void inverse_row_permute(size_type num_rows, size_type num_cols,
+                         const IndexType *__restrict__ perm_idxs,
+                         const ValueType *__restrict__ orig,
+                         size_type stride_orig, ValueType *__restrict__ result,
+                         size_type stride_result, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_id = global_id / num_cols;
+    const auto col_id = global_id % num_cols;
+    if (row_id < num_rows) {
+        result[perm_idxs[row_id] * stride_result + col_id] =
+            orig[row_id * stride_orig + col_id];
+    }
+}
+
+template <typename IndexType, typename ValueType>
+void inverse_row_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                         sycl::queue *stream, size_type num_rows,
+                         size_type num_cols, const IndexType *perm_idxs,
+                         const ValueType *orig, size_type stride_orig,
+                         ValueType *result, size_type stride_result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             inverse_row_permute(num_rows, num_cols, perm_idxs,
+                                                 orig, stride_orig, result,
+                                                 stride_result, item_ct1);
+                         });
+    });
+}
+
+
+template <typename IndexType, typename ValueType>
+void inverse_column_permute(size_type num_rows, size_type num_cols,
+                            const IndexType *__restrict__ perm_idxs,
+                            const ValueType *__restrict__ orig,
+                            size_type stride_orig,
+                            ValueType *__restrict__ result,
+                            size_type stride_result, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_id = global_id / num_cols;
+    const auto col_id = global_id % num_cols;
+    if (row_id < num_rows) {
+        result[row_id * stride_result + perm_idxs[col_id]] =
+            orig[row_id * stride_orig + col_id];
+    }
+}
+
+template <typename IndexType, typename ValueType>
+void inverse_column_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                            sycl::queue *stream, size_type num_rows,
+                            size_type num_cols, const IndexType *perm_idxs,
+                            const ValueType *orig, size_type stride_orig,
+                            ValueType *result, size_type stride_result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             inverse_column_permute(
+                                 num_rows, num_cols, perm_idxs, orig,
+                                 stride_orig, result, stride_result, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType>
+void extract_diagonal(size_type problem_size,
+                      const ValueType *__restrict__ orig, size_type stride_orig,
+                      ValueType *__restrict__ diag, sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+
+    if (tidx < problem_size) {
+        diag[tidx] = orig[tidx * stride_orig + tidx];
+    }
+}
+
+template <typename ValueType>
+void extract_diagonal(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                      sycl::queue *stream, size_type problem_size,
+                      const ValueType *orig, size_type stride_orig,
+                      ValueType *diag)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             extract_diagonal(problem_size, orig, stride_orig,
+                                              diag, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType>
+void inplace_absolute_dense(size_type num_rows, size_type num_cols,
+                            ValueType *__restrict__ data, size_type stride,
+                            sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    auto row = tidx / num_cols;
+    auto col = tidx % num_cols;
+    if (row < num_rows) {
+        data[row * stride + col] = dpcpp::abs(data[row * stride + col]);
+    }
+}
+
+template <typename ValueType>
+void inplace_absolute_dense(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                            sycl::queue *stream, size_type num_rows,
+                            size_type num_cols, ValueType *data,
+                            size_type stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             inplace_absolute_dense(num_rows, num_cols, data,
+                                                    stride, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType>
+void outplace_absolute_dense(size_type num_rows, size_type num_cols,
+                             const ValueType *__restrict__ in,
+                             size_type stride_in,
+                             remove_complex<ValueType> *__restrict__ out,
+                             size_type stride_out, sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    auto row = tidx / num_cols;
+    auto col = tidx % num_cols;
+    if (row < num_rows) {
+        out[row * stride_out + col] = dpcpp::abs(in[row * stride_in + col]);
+    }
+}
+
+template <typename ValueType>
+void outplace_absolute_dense(dim3 grid, dim3 block,
+                             size_t dynamic_shared_memory, sycl::queue *stream,
+                             size_type num_rows, size_type num_cols,
+                             const ValueType *in, size_type stride_in,
+                             remove_complex<ValueType> *out,
+                             size_type stride_out)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             outplace_absolute_dense(num_rows, num_cols, in,
+                                                     stride_in, out, stride_out,
+                                                     item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType, typename ComplexType>
+void make_complex(size_type num_rows, size_type num_cols,
+                  const ValueType *__restrict__ in, size_type stride_in,
+                  ComplexType *__restrict__ out, size_type stride_out,
+                  sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    auto row = tidx / num_cols;
+    auto col = tidx % num_cols;
+    if (row < num_rows) {
+        out[row * stride_out + col] = in[row * stride_in + col];
+    }
+}
+
+template <typename ValueType, typename ComplexType>
+void make_complex(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                  sycl::queue *stream, size_type num_rows, size_type num_cols,
+                  const ValueType *in, size_type stride_in, ComplexType *out,
+                  size_type stride_out)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             make_complex(num_rows, num_cols, in, stride_in,
+                                          out, stride_out, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType>
+void get_real(size_type num_rows, size_type num_cols,
+              const ValueType *__restrict__ in, size_type stride_in,
+              remove_complex<ValueType> *__restrict__ out, size_type stride_out,
+              sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    auto row = tidx / num_cols;
+    auto col = tidx % num_cols;
+    if (row < num_rows) {
+        out[row * stride_out + col] = real(in[row * stride_in + col]);
+    }
+}
+
+template <typename ValueType>
+void get_real(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+              sycl::queue *stream, size_type num_rows, size_type num_cols,
+              const ValueType *in, size_type stride_in,
+              remove_complex<ValueType> *out, size_type stride_out)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             get_real(num_rows, num_cols, in, stride_in, out,
+                                      stride_out, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType>
+void get_imag(size_type num_rows, size_type num_cols,
+              const ValueType *__restrict__ in, size_type stride_in,
+              remove_complex<ValueType> *__restrict__ out, size_type stride_out,
+              sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    auto row = tidx / num_cols;
+    auto col = tidx % num_cols;
+    if (row < num_rows) {
+        out[row * stride_out + col] = imag(in[row * stride_in + col]);
+    }
+}
+
+template <typename ValueType>
+void get_imag(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+              sycl::queue *stream, size_type num_rows, size_type num_cols,
+              const ValueType *in, size_type stride_in,
+              remove_complex<ValueType> *out, size_type stride_out)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             get_imag(num_rows, num_cols, in, stride_in, out,
+                                      stride_out, item_ct1);
+                         });
+    });
+}
+
+
+}  // namespace kernel
+
+
 template <typename ValueType>
 void simple_apply(std::shared_ptr<const DpcppExecutor> exec,
                   const matrix::Dense<ValueType> *a,
                   const matrix::Dense<ValueType> *b,
-                  matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+                  matrix::Dense<ValueType> *c)
+{
+    if (cublas::is_supported<ValueType>::value) {
+        auto handle = exec->get_cublas_handle();
+        {
+            cublas::pointer_mode_guard pm_guard(handle);
+            auto alpha = one<ValueType>();
+            auto beta = zero<ValueType>();
+            cublas::gemm(handle, oneapi::mkl::transpose::nontrans,
+                         oneapi::mkl::transpose::nontrans, c->get_size()[1],
+                         c->get_size()[0], a->get_size()[1], &alpha,
+                         b->get_const_values(), b->get_stride(),
+                         a->get_const_values(), a->get_stride(), &beta,
+                         c->get_values(), c->get_stride());
+        }
+    } else {
+        GKO_NOT_IMPLEMENTED;
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL);
 
@@ -78,8 +1317,20 @@ template <typename ValueType>
 void apply(std::shared_ptr<const DpcppExecutor> exec,
            const matrix::Dense<ValueType> *alpha,
            const matrix::Dense<ValueType> *a, const matrix::Dense<ValueType> *b,
-           const matrix::Dense<ValueType> *beta,
-           matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+           const matrix::Dense<ValueType> *beta, matrix::Dense<ValueType> *c)
+{
+    if (cublas::is_supported<ValueType>::value) {
+        cublas::gemm(
+            exec->get_cublas_handle(), oneapi::mkl::transpose::nontrans,
+            oneapi::mkl::transpose::nontrans, c->get_size()[1],
+            c->get_size()[0], a->get_size()[1], alpha->get_const_values(),
+            b->get_const_values(), b->get_stride(), a->get_const_values(),
+            a->get_stride(), beta->get_const_values(), c->get_values(),
+            c->get_stride());
+    } else {
+        GKO_NOT_IMPLEMENTED;
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
@@ -88,7 +1339,40 @@ template <typename ValueType>
 void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
                  const matrix::Dense<ValueType> *x,
                  const matrix::Dense<ValueType> *y,
-                 matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
+                 matrix::Dense<ValueType> *result)
+{
+    if (cublas::is_supported<ValueType>::value) {
+        // TODO: write a custom kernel which does this more efficiently
+        for (size_type col = 0; col < x->get_size()[1]; ++col) {
+            cublas::dot(exec->get_cublas_handle(), x->get_size()[0],
+                        x->get_const_values() + col, x->get_stride(),
+                        y->get_const_values() + col, y->get_stride(),
+                        result->get_values() + col);
+        }
+    } else {
+        // TODO: these are tuning parameters obtained experimentally, once
+        // we decide how to handle this uniformly, they should be modified
+        // appropriately
+        constexpr auto work_per_thread = 32;
+        constexpr auto block_size = 1024;
+
+        constexpr auto work_per_block = work_per_thread * block_size;
+        const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
+        const dim3 block_dim{config::warp_size, 1,
+                             block_size / config::warp_size};
+        Array<ValueType> work(exec, grid_dim.x);
+        // TODO: write a kernel which does this more efficiently
+        for (size_type col = 0; col < x->get_size()[1]; ++col) {
+            kernel::compute_partial_dot<block_size>(
+                grid_dim, block_dim, 0, exec->get_queue(), x->get_size()[0],
+                x->get_const_values() + col, x->get_stride(),
+                y->get_const_values() + col, y->get_stride(), work.get_data());
+            kernel::finalize_dot_computation<block_size>(
+                1, block_dim, 0, exec->get_queue(), grid_dim.x,
+                work.get_const_data(), result->get_values() + col);
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL);
 
@@ -106,7 +1390,37 @@ template <typename ValueType>
 void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::Dense<ValueType> *x,
                    matrix::Dense<remove_complex<ValueType>> *result)
-    GKO_NOT_IMPLEMENTED;
+{
+    if (cublas::is_supported<ValueType>::value) {
+        for (size_type col = 0; col < x->get_size()[1]; ++col) {
+            cublas::norm2(exec->get_cublas_handle(), x->get_size()[0],
+                          x->get_const_values() + col, x->get_stride(),
+                          result->get_values() + col);
+        }
+    } else {
+        using norm_type = remove_complex<ValueType>;
+        // TODO: these are tuning parameters obtained experimentally, once
+        // we decide how to handle this uniformly, they should be modified
+        // appropriately
+        constexpr auto work_per_thread = 32;
+        constexpr auto block_size = 1024;
+
+        constexpr auto work_per_block = work_per_thread * block_size;
+        const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
+        const dim3 block_dim{config::warp_size, 1,
+                             block_size / config::warp_size};
+        Array<norm_type> work(exec, grid_dim.x);
+        // TODO: write a kernel which does this more efficiently
+        for (size_type col = 0; col < x->get_size()[1]; ++col) {
+            kernel::compute_partial_norm2<block_size>(
+                grid_dim, block_dim, 0, exec->get_queue(), x->get_size()[0],
+                x->get_const_values() + col, x->get_stride(), work.get_data());
+            kernel::finalize_norm2_computation<block_size>(
+                1, block_dim, 0, exec->get_queue(), grid_dim.x,
+                work.get_const_data(), result->get_values() + col);
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_NORM2_KERNEL);
 
@@ -115,7 +1429,28 @@ template <typename ValueType, typename IndexType>
 void convert_to_coo(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Dense<ValueType> *source,
                     matrix::Coo<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
+{
+    auto num_rows = result->get_size()[0];
+    auto num_cols = result->get_size()[1];
+
+    auto row_idxs = result->get_row_idxs();
+    auto col_idxs = result->get_col_idxs();
+    auto values = result->get_values();
+
+    auto stride = source->get_stride();
+
+    auto nnz_prefix_sum = Array<size_type>(exec, num_rows);
+    calculate_nonzeros_per_row(exec, source, &nnz_prefix_sum);
+
+    components::prefix_sum(exec, nnz_prefix_sum.get_data(), num_rows);
+
+    size_type grid_dim = ceildiv(num_rows, default_block_size);
+
+    kernel::fill_in_coo(grid_dim, default_block_size, 0, exec->get_queue(),
+                        num_rows, num_cols, stride,
+                        nnz_prefix_sum.get_const_data(),
+                        source->get_const_values(), row_idxs, col_idxs, values);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DENSE_CONVERT_TO_COO_KERNEL);
@@ -125,7 +1460,31 @@ template <typename ValueType, typename IndexType>
 void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Dense<ValueType> *source,
                     matrix::Csr<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
+{
+    auto num_rows = result->get_size()[0];
+    auto num_cols = result->get_size()[1];
+
+    auto row_ptrs = result->get_row_ptrs();
+    auto col_idxs = result->get_col_idxs();
+    auto values = result->get_values();
+
+    auto stride = source->get_stride();
+
+    const auto rows_per_block = ceildiv(default_block_size, config::warp_size);
+    const auto grid_dim_nnz = ceildiv(source->get_size()[0], rows_per_block);
+
+    kernel::count_nnz_per_row(grid_dim_nnz, default_block_size, 0,
+                              exec->get_queue(), num_rows, num_cols, stride,
+                              source->get_const_values(), row_ptrs);
+
+    components::prefix_sum(exec, row_ptrs, num_rows + 1);
+
+    size_type grid_dim = ceildiv(num_rows, default_block_size);
+
+    kernel::fill_in_csr(grid_dim, default_block_size, 0, exec->get_queue(),
+                        num_rows, num_cols, stride, source->get_const_values(),
+                        row_ptrs, col_idxs, values);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DENSE_CONVERT_TO_CSR_KERNEL);
@@ -135,7 +1494,23 @@ template <typename ValueType, typename IndexType>
 void convert_to_ell(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Dense<ValueType> *source,
                     matrix::Ell<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
+{
+    auto num_rows = result->get_size()[0];
+    auto num_cols = result->get_size()[1];
+    auto max_nnz_per_row = result->get_num_stored_elements_per_row();
+
+    auto col_ptrs = result->get_col_idxs();
+    auto values = result->get_values();
+
+    auto source_stride = source->get_stride();
+    auto result_stride = result->get_stride();
+
+    auto grid_dim = ceildiv(result_stride, default_block_size);
+    kernel::fill_in_ell(grid_dim, default_block_size, 0, exec->get_queue(),
+                        num_rows, num_cols, source_stride,
+                        source->get_const_values(), max_nnz_per_row,
+                        result_stride, col_ptrs, values);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DENSE_CONVERT_TO_ELL_KERNEL);
@@ -155,7 +1530,46 @@ template <typename ValueType, typename IndexType>
 void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Dense<ValueType> *source,
                       matrix::Sellp<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
+{
+    const auto stride = source->get_stride();
+    const auto num_rows = result->get_size()[0];
+    const auto num_cols = result->get_size()[1];
+
+    auto vals = result->get_values();
+    auto col_idxs = result->get_col_idxs();
+    auto slice_lengths = result->get_slice_lengths();
+    auto slice_sets = result->get_slice_sets();
+
+    const auto slice_size = (result->get_slice_size() == 0)
+                                ? matrix::default_slice_size
+                                : result->get_slice_size();
+    const auto stride_factor = (result->get_stride_factor() == 0)
+                                   ? matrix::default_stride_factor
+                                   : result->get_stride_factor();
+    const int slice_num = ceildiv(num_rows, slice_size);
+
+    auto nnz_per_row = Array<size_type>(exec, num_rows);
+    calculate_nonzeros_per_row(exec, source, &nnz_per_row);
+
+    auto grid_dim = slice_num;
+
+    if (grid_dim > 0) {
+        kernel::calculate_slice_lengths(
+            grid_dim, config::warp_size, 0, exec->get_queue(), num_rows,
+            slice_size, slice_num, stride_factor, nnz_per_row.get_const_data(),
+            slice_lengths, slice_sets);
+    }
+
+    components::prefix_sum(exec, slice_sets, slice_num + 1);
+
+    grid_dim = ceildiv(num_rows, default_block_size);
+    if (grid_dim > 0) {
+        kernel::fill_in_sellp(grid_dim, default_block_size, 0,
+                              exec->get_queue(), num_rows, num_cols, slice_size,
+                              stride, source->get_const_values(), slice_lengths,
+                              slice_sets, col_idxs, vals);
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DENSE_CONVERT_TO_SELLP_KERNEL);
@@ -173,8 +1587,15 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 template <typename ValueType>
 void count_nonzeros(std::shared_ptr<const DpcppExecutor> exec,
-                    const matrix::Dense<ValueType> *source,
-                    size_type *result) GKO_NOT_IMPLEMENTED;
+                    const matrix::Dense<ValueType> *source, size_type *result)
+{
+    const auto num_rows = source->get_size()[0];
+    auto nnz_per_row = Array<size_type>(exec, num_rows);
+
+    calculate_nonzeros_per_row(exec, source, &nnz_per_row);
+
+    *result = reduce_add_array(exec, num_rows, nnz_per_row.get_const_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COUNT_NONZEROS_KERNEL);
 
@@ -182,7 +1603,33 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COUNT_NONZEROS_KERNEL);
 template <typename ValueType>
 void calculate_max_nnz_per_row(std::shared_ptr<const DpcppExecutor> exec,
                                const matrix::Dense<ValueType> *source,
-                               size_type *result) GKO_NOT_IMPLEMENTED;
+                               size_type *result)
+{
+    const auto num_rows = source->get_size()[0];
+    auto nnz_per_row = Array<size_type>(exec, num_rows);
+
+    calculate_nonzeros_per_row(exec, source, &nnz_per_row);
+
+    const auto n = ceildiv(num_rows, default_block_size);
+    const size_type grid_dim =
+        (n <= default_block_size) ? n : default_block_size;
+
+    auto block_results = Array<size_type>(exec, grid_dim);
+
+    kernel::reduce_max_nnz(
+        grid_dim, default_block_size, default_block_size * sizeof(size_type),
+        exec->get_queue(), num_rows, nnz_per_row.get_const_data(),
+        block_results.get_data());
+
+    auto d_result = Array<size_type>(exec, 1);
+
+    kernel::reduce_max_nnz(1, default_block_size,
+                           default_block_size * sizeof(size_type),
+                           exec->get_queue(), grid_dim,
+                           block_results.get_const_data(), d_result.get_data());
+
+    *result = exec->copy_val_to_host(d_result.get_const_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_DENSE_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
@@ -191,7 +1638,19 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
 template <typename ValueType>
 void calculate_nonzeros_per_row(std::shared_ptr<const DpcppExecutor> exec,
                                 const matrix::Dense<ValueType> *source,
-                                Array<size_type> *result) GKO_NOT_IMPLEMENTED;
+                                Array<size_type> *result)
+{
+    const dim3 block_size(default_block_size, 1, 1);
+    auto rows_per_block = ceildiv(default_block_size, config::warp_size);
+    const size_t grid_x = ceildiv(source->get_size()[0], rows_per_block);
+    const dim3 grid_size(grid_x, 1, 1);
+    if (grid_x > 0) {
+        kernel::count_nnz_per_row(
+            grid_size, block_size, 0, exec->get_queue(), source->get_size()[0],
+            source->get_size()[1], source->get_stride(),
+            source->get_const_values(), result->get_data());
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_DENSE_CALCULATE_NONZEROS_PER_ROW_KERNEL);
@@ -201,7 +1660,48 @@ template <typename ValueType>
 void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
                           const matrix::Dense<ValueType> *source,
                           size_type *result, size_type stride_factor,
-                          size_type slice_size) GKO_NOT_IMPLEMENTED;
+                          size_type slice_size)
+{
+    const auto num_rows = source->get_size()[0];
+
+    if (num_rows == 0) {
+        *result = 0;
+        return;
+    }
+
+    const auto num_cols = source->get_size()[1];
+    const auto slice_num = ceildiv(num_rows, slice_size);
+
+    auto nnz_per_row = Array<size_type>(exec, num_rows);
+
+    calculate_nonzeros_per_row(exec, source, &nnz_per_row);
+
+    auto max_nnz_per_slice = Array<size_type>(exec, slice_num);
+
+    auto grid_dim = ceildiv(slice_num * config::warp_size, default_block_size);
+
+    kernel::reduce_max_nnz_per_slice(
+        grid_dim, default_block_size, 0, exec->get_queue(), num_rows,
+        slice_size, stride_factor, nnz_per_row.get_const_data(),
+        max_nnz_per_slice.get_data());
+
+    grid_dim = ceildiv(slice_num, default_block_size);
+    auto block_results = Array<size_type>(exec, grid_dim);
+
+    kernel::reduce_total_cols(
+        grid_dim, default_block_size, default_block_size * sizeof(size_type),
+        exec->get_queue(), slice_num, max_nnz_per_slice.get_const_data(),
+        block_results.get_data());
+
+    auto d_result = Array<size_type>(exec, 1);
+
+    kernel::reduce_total_cols(
+        1, default_block_size, default_block_size * sizeof(size_type),
+        exec->get_queue(), grid_dim, block_results.get_const_data(),
+        d_result.get_data());
+
+    *result = exec->copy_val_to_host(d_result.get_const_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_DENSE_CALCULATE_TOTAL_COLS_KERNEL);
@@ -210,7 +1710,25 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
 template <typename ValueType>
 void transpose(std::shared_ptr<const DpcppExecutor> exec,
                const matrix::Dense<ValueType> *orig,
-               matrix::Dense<ValueType> *trans) GKO_NOT_IMPLEMENTED;
+               matrix::Dense<ValueType> *trans)
+{
+    if (cublas::is_supported<ValueType>::value) {
+        auto handle = exec->get_cublas_handle();
+        {
+            cublas::pointer_mode_guard pm_guard(handle);
+            auto alpha = one<ValueType>();
+            auto beta = zero<ValueType>();
+            cublas::geam(
+                handle, oneapi::mkl::transpose::trans,
+                oneapi::mkl::transpose::nontrans, orig->get_size()[0],
+                orig->get_size()[1], &alpha, orig->get_const_values(),
+                orig->get_stride(), &beta, static_cast<ValueType *>(nullptr),
+                trans->get_size()[1], trans->get_values(), trans->get_stride());
+        }
+    } else {
+        GKO_NOT_IMPLEMENTED;
+    }
+};
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_TRANSPOSE_KERNEL);
 
@@ -218,7 +1736,25 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_TRANSPOSE_KERNEL);
 template <typename ValueType>
 void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Dense<ValueType> *orig,
-                    matrix::Dense<ValueType> *trans) GKO_NOT_IMPLEMENTED;
+                    matrix::Dense<ValueType> *trans)
+{
+    if (cublas::is_supported<ValueType>::value) {
+        auto handle = exec->get_cublas_handle();
+        {
+            cublas::pointer_mode_guard pm_guard(handle);
+            auto alpha = one<ValueType>();
+            auto beta = zero<ValueType>();
+            cublas::geam(
+                handle, oneapi::mkl::transpose::conjtrans,
+                oneapi::mkl::transpose::nontrans, orig->get_size()[0],
+                orig->get_size()[1], &alpha, orig->get_const_values(),
+                orig->get_stride(), &beta, static_cast<ValueType *>(nullptr),
+                trans->get_size()[1], trans->get_values(), trans->get_stride());
+        }
+    } else {
+        GKO_NOT_IMPLEMENTED;
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 

From fa28497b88bfec2d58ee7a4c8e1550f46f9feda3 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 24 Feb 2021 16:09:26 +0800
Subject: [PATCH 0892/1008] dense, prefix_sum and uninitialized_array

---
 dpcpp/CMakeLists.txt                     |   1 +
 dpcpp/components/prefix_sum.dp.cpp       |  33 +-
 dpcpp/components/prefix_sum.dp.hpp       | 260 ++++++++
 dpcpp/components/uninitialized_array.hpp |   9 +-
 dpcpp/matrix/dense_kernels.dp.cpp        | 247 +++----
 dpcpp/test/components/CMakeLists.txt     |   1 +
 dpcpp/test/components/prefix_sum.cpp     |  96 +++
 dpcpp/test/matrix/CMakeLists.txt         |   3 +-
 dpcpp/test/matrix/dense_kernels.cpp      | 806 +++++++++++++++++++++++
 dpcpp/test/utils.hpp                     |  54 ++
 10 files changed, 1372 insertions(+), 138 deletions(-)
 create mode 100644 dpcpp/components/prefix_sum.dp.hpp
 create mode 100644 dpcpp/test/components/prefix_sum.cpp
 create mode 100644 dpcpp/test/matrix/dense_kernels.cpp
 create mode 100644 dpcpp/test/utils.hpp

diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 97cf8a5daf6..b3101d8b2e2 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -60,6 +60,7 @@ target_compile_options(ginkgo_dpcpp PRIVATE "${GINKGO_DPCPP_FLAGS}")
 target_compile_features(ginkgo_dpcpp PRIVATE cxx_std_17)
 target_link_options(ginkgo_dpcpp PRIVATE -fsycl-device-lib=all)
 target_link_options(ginkgo_dpcpp PRIVATE -fsycl-device-code-split=per_kernel)
+target_link_libraries(ginkgo_dpcpp PRIVATE "mkl_sycl;mkl_intel_ilp64;mkl_sequential;mkl_core")
 target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
 if (GINKGO_DPCPP_SINGLE_MODE)
     target_compile_definitions(ginkgo_dpcpp PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
diff --git a/dpcpp/components/prefix_sum.dp.cpp b/dpcpp/components/prefix_sum.dp.cpp
index 4b7f816b381..b4961809a8b 100644
--- a/dpcpp/components/prefix_sum.dp.cpp
+++ b/dpcpp/components/prefix_sum.dp.cpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <CL/sycl.hpp>
 
 
-#include <ginkgo/core/base/exception_helpers.hpp>
+#include "dpcpp/components/prefix_sum.dp.hpp"
 
 
 namespace gko {
@@ -45,22 +45,33 @@ namespace dpcpp {
 namespace components {
 
 
+constexpr int prefix_sum_block_size = 256;
+
+
 template <typename IndexType>
-void prefix_sum(std::shared_ptr<const DefaultExecutor> exec, IndexType *counts,
+void prefix_sum(std::shared_ptr<const DpcppExecutor> exec, IndexType *counts,
                 size_type num_entries)
 {
-    // TODO actually implement parallel prefix sum
-    exec->get_queue()->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl::range<1>{1}, [=](sycl::id<1> idx) {
-            IndexType sum{};
-            for (size_type i = 0; i < num_entries; i++) {
-                sum += std::exchange(counts[i], sum);
-            }
-        });
-    });
+    // prefix_sum should be on the valid array
+    if (num_entries > 0) {
+        auto num_blocks = ceildiv(num_entries, prefix_sum_block_size);
+        Array<IndexType> block_sum_array(exec, num_blocks - 1);
+        auto block_sums = block_sum_array.get_data();
+        start_prefix_sum<prefix_sum_block_size>(
+            num_blocks, prefix_sum_block_size, 0, exec->get_queue(),
+            num_entries, counts, block_sums);
+        // add the total sum of the previous block only when the number of block
+        // is larger than 1.
+        if (num_blocks > 1) {
+            finalize_prefix_sum<prefix_sum_block_size>(
+                num_blocks, prefix_sum_block_size, 0, exec->get_queue(),
+                num_entries, counts, block_sums);
+        }
+    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_PREFIX_SUM_KERNEL);
+
 // instantiate for size_type as well, as this is used in the Sellp format
 template GKO_DECLARE_PREFIX_SUM_KERNEL(size_type);
 
diff --git a/dpcpp/components/prefix_sum.dp.hpp b/dpcpp/components/prefix_sum.dp.hpp
new file mode 100644
index 00000000000..6b3498d1dea
--- /dev/null
+++ b/dpcpp/components/prefix_sum.dp.hpp
@@ -0,0 +1,260 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_COMPONENTS_PREFIX_SUM_DP_HPP_
+#define GKO_DPCPP_COMPONENTS_PREFIX_SUM_DP_HPP_
+
+
+#include <type_traits>
+
+
+#include <CL/sycl.hpp>
+
+
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/base/dpct.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
+#include "dpcpp/components/reduction.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+// #include "common/components/prefix_sum.hpp.inc"
+/**
+ * @internal
+ * Computes the prefix sum and total sum of `element` over a subwarp.
+ *
+ * @param element     the element over which we compute the prefix sum.
+ * @param prefix_sum  will be set to the sum of all `element`s from lower
+ *                    lanes, plus the local `element` if `inclusive` is `true`.
+ * @param total_sum   will be set to the total sum of `element` in this subwarp.
+ * @param subwarp     the cooperative group representing the subwarp.
+ *
+ * @tparam inclusive  if this is true, the computed prefix sum will be
+ *                    inclusive, otherwise it will be exclusive.
+ *
+ * @note For this function to work on architectures with independent thread
+ * scheduling, all threads of the subwarp have to execute it.
+ */
+template <bool inclusive, typename ValueType, typename Group>
+__dpct_inline__ void subwarp_prefix_sum(ValueType element,
+                                        ValueType &prefix_sum,
+                                        ValueType &total_sum, Group subwarp)
+{
+    prefix_sum = inclusive ? element : zero<ValueType>();
+    total_sum = element;
+#pragma unroll
+    // hypercube prefix sum
+    for (auto step = 1; step < subwarp.size(); step *= 2) {
+        auto neighbor = subwarp.shfl_xor(total_sum, step);
+        total_sum += neighbor;
+        prefix_sum += bool(subwarp.thread_rank() & step) ? neighbor : 0;
+    }
+}
+
+/**
+ * @internal
+ * Computes the prefix sum of `element` over a subwarp.
+ *
+ * @param element     the element over which we compute the prefix sum.
+ * @param prefix_sum  will be set to the sum of all `element`s from lower
+ *                    lanes, plus the local `element` if `inclusive` is `true`.
+ * @param subwarp     the cooperative group representing the subwarp.
+ *
+ * @tparam inclusive  if this is true, the computed prefix sum will be
+ *                    inclusive, otherwise it will be exclusive.
+ *
+ * @note All threads of the subwarp have to execute this function for it to work
+ *       (and not dead-lock on newer architectures).
+ */
+template <bool inclusive, typename ValueType, typename Group>
+__dpct_inline__ void subwarp_prefix_sum(ValueType element,
+                                        ValueType &prefix_sum, Group subwarp)
+{
+    ValueType tmp{};
+    subwarp_prefix_sum<inclusive>(element, prefix_sum, tmp, subwarp);
+}
+
+
+/**
+ * @internal
+ * First step of the calculation of a prefix sum. Calculates the prefix sum
+ * in-place on parts of the array `elements`.
+ *
+ * @param elements  array on which the prefix sum is to be calculated
+ * @param block_sum  array which stores the total sum of each block, requires at
+ *                   least `ceildiv(num_elements, block_size) - 1` elements
+ * @param num_elements  total number of entries in `elements`
+ *
+ * @tparam block_size  thread block size for this kernel, also size of blocks on
+ *                     which this kernel calculates the prefix sum in-place
+ *
+ * @note To calculate the prefix sum over an array of size bigger than
+ *       `block_size`, `finalize_prefix_sum` has to be used as well.
+ */
+template <int block_size, typename ValueType>
+void start_prefix_sum(size_type num_elements, ValueType *__restrict__ elements,
+                      ValueType *__restrict__ block_sum,
+                      sycl::nd_item<3> item_ct1,
+                      UninitializedArray<ValueType, block_size> *prefix_helper)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    const auto element_id = item_ct1.get_local_id(2);
+
+    // do not need to access the last element when exclusive prefix sum
+    (*prefix_helper)[element_id] =
+        (tidx + 1 < num_elements) ? elements[tidx] : zero<ValueType>();
+    auto this_block = group::this_thread_block(item_ct1);
+    this_block.sync();
+
+    // Do a normal reduction
+#pragma unroll
+    for (int i = 1; i < block_size; i <<= 1) {
+        const auto ai = i * (2 * element_id + 1) - 1;
+        const auto bi = i * (2 * element_id + 2) - 1;
+        if (bi < block_size) {
+            (*prefix_helper)[bi] += (*prefix_helper)[ai];
+        }
+        this_block.sync();
+    }
+
+    if (element_id == 0) {
+        // Store the total sum except the last block
+        if (item_ct1.get_group(2) + 1 < item_ct1.get_group_range(2)) {
+            block_sum[item_ct1.get_group(2)] = (*prefix_helper)[block_size - 1];
+        }
+        (*prefix_helper)[block_size - 1] = zero<ValueType>();
+    }
+
+    this_block.sync();
+
+    // Perform the down-sweep phase to get the true prefix sum
+#pragma unroll
+    for (int i = block_size >> 1; i > 0; i >>= 1) {
+        const auto ai = i * (2 * element_id + 1) - 1;
+        const auto bi = i * (2 * element_id + 2) - 1;
+        if (bi < block_size) {
+            auto tmp = (*prefix_helper)[ai];
+            (*prefix_helper)[ai] = (*prefix_helper)[bi];
+            (*prefix_helper)[bi] += tmp;
+        }
+        this_block.sync();
+    }
+    if (tidx < num_elements) {
+        elements[tidx] = (*prefix_helper)[element_id];
+    }
+}
+
+template <int block_size, typename ValueType>
+void start_prefix_sum(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                      sycl::queue *stream, size_type num_elements,
+                      ValueType *elements, ValueType *block_sum)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<ValueType, block_size>, 0,
+                       sycl::access::mode::read_write,
+                       sycl::access::target::local>
+            prefix_helper_acc_ct1(cgh);
+
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             start_prefix_sum<block_size>(
+                                 num_elements, elements, block_sum, item_ct1,
+                                 (UninitializedArray<ValueType, block_size> *)
+                                     prefix_helper_acc_ct1.get_pointer());
+                         });
+    });
+}
+
+
+/**
+ * @internal
+ * Second step of the calculation of a prefix sum. Increases the value of each
+ * entry of `elements` by the total sum of all preceding blocks.
+ *
+ * @param elements  array on which the prefix sum is to be calculated
+ * @param block_sum  array storing the total sum of each block
+ * @param num_elements  total number of entries in `elements`
+ *
+ * @tparam block_size  thread block size for this kernel, has to be the same as
+ *                    for `start_prefix_sum`
+ *
+ * @note To calculate a prefix sum, first `start_prefix_sum` has to be called.
+ */
+template <int block_size, typename ValueType>
+void finalize_prefix_sum(size_type num_elements,
+                         ValueType *__restrict__ elements,
+                         const ValueType *__restrict__ block_sum,
+                         sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+
+    if (tidx < num_elements) {
+        ValueType prefix_block_sum = zero<ValueType>();
+        for (size_type i = 0; i < item_ct1.get_group(2); i++) {
+            prefix_block_sum += block_sum[i];
+        }
+        elements[tidx] += prefix_block_sum;
+    }
+}
+
+template <int block_size, typename ValueType>
+void finalize_prefix_sum(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                         sycl::queue *stream, size_type num_elements,
+                         ValueType *elements, const ValueType *block_sum)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        auto local_range = block.reverse();
+        auto global_range = grid.reverse() * local_range;
+
+        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             finalize_prefix_sum<block_size>(
+                                 num_elements, elements, block_sum, item_ct1);
+                         });
+    });
+}
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_DPCPP_COMPONENTS_PREFIX_SUM_DP_HPP_
diff --git a/dpcpp/components/uninitialized_array.hpp b/dpcpp/components/uninitialized_array.hpp
index fb7575bc202..415126b8ed3 100644
--- a/dpcpp/components/uninitialized_array.hpp
+++ b/dpcpp/components/uninitialized_array.hpp
@@ -88,7 +88,7 @@ class UninitializedArray {
     constexpr __dpct_inline__ ValueType &operator[](size_type pos) const
         noexcept
     {
-        return reinterpret_cast<const ValueType *>(data_)[pos];
+        return data_[pos];
     }
 
     /**
@@ -99,13 +99,14 @@ class UninitializedArray {
      *
      * @return a reference to the array entry at the given index.
      */
-    GKO_ATTRIBUTES ValueType &operator[](size_type pos) noexcept
+    __dpct_inline__ ValueType &operator[](size_type pos) noexcept
     {
-        return reinterpret_cast<ValueType *>(data_)[pos];
+        return data_[pos];
     }
 
 private:
-    unsigned char data_[sizeof(ValueType) / sizeof(unsigned char) * size];
+    // unsigned char data_[sizeof(ValueType) / sizeof(unsigned char) * size];
+    ValueType data_[size];
 };
 
 
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 36935b1a4d6..c5074b5cc38 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -33,11 +33,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/dense_kernels.hpp"
 
 
-#include <dpcpp/base/cublas_bindings.hpp>
-#include <dpcpp/base/pointer_mode_guard.hpp>
-
-
 #include <CL/sycl.hpp>
+#include <oneapi/mkl.hpp>
 
 
 #include <ginkgo/core/base/math.hpp>
@@ -48,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/ell.hpp>
 #include <ginkgo/core/matrix/sellp.hpp>
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
+#include <iostream>
 
 
 #include "core/components/prefix_sum.hpp"
@@ -70,7 +68,7 @@ namespace dpcpp {
 namespace dense {
 
 
-constexpr auto default_block_size = 512;
+constexpr auto default_block_size = 256;
 
 
 // #include "common/matrix/dense_kernels.hpp.inc"
@@ -236,18 +234,20 @@ void compute_partial_reduce(size_type num_rows, OutType *__restrict__ work,
     const auto global_id =
         thread::get_thread_id<config::warp_size, warps_per_block>(item_ct1);
 
+    OutType *tmp_work_array=*tmp_work;
     auto tmp = zero<OutType>();
     for (auto i = global_id; i < num_rows; i += block_size * num_blocks) {
         tmp = reduce_op(tmp, get_value(i));
     }
 
-    (*tmp_work)[local_id] = tmp;
+    tmp_work_array[local_id] = tmp;
 
-    reduce(group::this_thread_block(item_ct1),
-           static_cast<OutType *>((*tmp_work)), reduce_op);
+    ::gko::kernels::dpcpp::reduce(group::this_thread_block(item_ct1),
+                                  tmp_work_array,
+                                  reduce_op);
 
     if (local_id == 0) {
-        work[thread::get_block_id(item_ct1)] = (*tmp_work)[0];
+        work[thread::get_block_id(item_ct1)] = tmp_work_array[0];
     }
 }
 
@@ -267,14 +267,15 @@ void finalize_reduce_computation(
     for (auto i = local_id; i < size; i += block_size) {
         tmp = reduce_op(tmp, work[i]);
     }
+    ValueType *tmp_work_array=*tmp_work;
+    tmp_work_array[local_id] = tmp;
 
-    (*tmp_work)[local_id] = tmp;
-
-    reduce(group::this_thread_block(item_ct1),
-           static_cast<ValueType *>((*tmp_work)), reduce_op);
+    ::gko::kernels::dpcpp::reduce(group::this_thread_block(item_ct1),
+                                  tmp_work_array,
+                                  reduce_op);
 
     if (local_id == 0) {
-        *result = finalize_op((*tmp_work)[0]);
+        *result = finalize_op(tmp_work_array[0]);
     }
 }
 
@@ -287,12 +288,7 @@ void compute_partial_dot(size_type num_rows, const ValueType *__restrict__ x,
                          UninitializedArray<ValueType, block_size> *tmp_work)
 {
     compute_partial_reduce<block_size>(
-        /*
-        DPCT1007:4: Migration of this CUDA API is not supported by the Intel(R)
-        DPC++ Compatibility Tool.
-        */
-        num_rows,
-        work,
+        num_rows, work,
         [x, stride_x, y, stride_y](size_type i) {
             return x[i * stride_x] * conj(y[i * stride_y]);
         },
@@ -416,8 +412,7 @@ void finalize_norm2_computation(
     finalize_reduce_computation<block_size>(
         size, work, result,
         [](const ValueType &x, const ValueType &y) { return x + y; },
-        [](const ValueType &x) { return sycl::sqrt((float)x); }, item_ct1,
-        tmp_work);
+        [](const ValueType &x) { return sqrt(x); }, item_ct1, tmp_work);
 }
 
 template <size_type block_size, typename ValueType>
@@ -508,9 +503,7 @@ void count_nnz_per_row(size_type num_rows, size_type num_cols, size_type stride,
                 part_result += 1;
             }
         }
-        result[row_idx] = std::reduce(
-            oneapi::dpl::execution::make_device_policy(
-                dpct::get_default_queue()),
+        result[row_idx] = ::gko::kernels::dpcpp::reduce(
             warp_tile, part_result,
             [](const size_type &a, const size_type &b) { return a + b; });
     }
@@ -648,7 +641,7 @@ void calculate_slice_lengths(size_type num_rows, size_type slice_size,
 
         auto warp_tile = group::tiled_partition<warp_size>(
             group::this_thread_block(item_ct1));
-        auto warp_result = reduce(
+        auto warp_result = ::gko::kernels::dpcpp::reduce(
             warp_tile, thread_result,
             [](const size_type &a, const size_type &b) { return max(a, b); });
 
@@ -745,7 +738,7 @@ void reduce_max_nnz(size_type size, const size_type *__restrict__ nnz_per_row,
     auto block_max = (size_type *)dpct_local;
 
     reduce_array(
-        size, nnz_per_row, block_max,
+        size, nnz_per_row, block_max, item_ct1,
         [](const size_type &x, const size_type &y) { return max(x, y); });
 
     if (item_ct1.get_local_id(2) == 0) {
@@ -795,7 +788,7 @@ void reduce_max_nnz_per_slice(size_type num_rows, size_type slice_size,
         }
     }
 
-    auto warp_result = reduce(
+    auto warp_result = ::gko::kernels::dpcpp::reduce(
         warp_tile, thread_result,
         [](const size_type &a, const size_type &b) { return max(a, b); });
 
@@ -831,7 +824,7 @@ void reduce_total_cols(size_type num_slices,
 {
     auto block_result = (size_type *)dpct_local;
 
-    reduce_array(num_slices, max_nnz_per_slice, block_result,
+    reduce_array(num_slices, max_nnz_per_slice, block_result, item_ct1,
                  [](const size_type &x, const size_type &y) { return x + y; });
 
     if (item_ct1.get_local_id(2) == 0) {
@@ -1292,22 +1285,13 @@ void simple_apply(std::shared_ptr<const DpcppExecutor> exec,
                   const matrix::Dense<ValueType> *b,
                   matrix::Dense<ValueType> *c)
 {
-    if (cublas::is_supported<ValueType>::value) {
-        auto handle = exec->get_cublas_handle();
-        {
-            cublas::pointer_mode_guard pm_guard(handle);
-            auto alpha = one<ValueType>();
-            auto beta = zero<ValueType>();
-            cublas::gemm(handle, oneapi::mkl::transpose::nontrans,
-                         oneapi::mkl::transpose::nontrans, c->get_size()[1],
-                         c->get_size()[0], a->get_size()[1], &alpha,
-                         b->get_const_values(), b->get_stride(),
-                         a->get_const_values(), a->get_stride(), &beta,
-                         c->get_values(), c->get_stride());
-        }
-    } else {
-        GKO_NOT_IMPLEMENTED;
-    }
+    using namespace oneapi::mkl;
+    oneapi::mkl::blas::row_major::gemm(
+        *exec->get_queue(), transpose::nontrans, transpose::nontrans,
+        c->get_size()[0], c->get_size()[1], a->get_size()[1],
+        one<ValueType>(), a->get_const_values(), a->get_stride(),
+        b->get_const_values(), b->get_stride(), zero<ValueType>(),
+        c->get_values(), c->get_stride());
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL);
@@ -1319,17 +1303,14 @@ void apply(std::shared_ptr<const DpcppExecutor> exec,
            const matrix::Dense<ValueType> *a, const matrix::Dense<ValueType> *b,
            const matrix::Dense<ValueType> *beta, matrix::Dense<ValueType> *c)
 {
-    if (cublas::is_supported<ValueType>::value) {
-        cublas::gemm(
-            exec->get_cublas_handle(), oneapi::mkl::transpose::nontrans,
-            oneapi::mkl::transpose::nontrans, c->get_size()[1],
-            c->get_size()[0], a->get_size()[1], alpha->get_const_values(),
-            b->get_const_values(), b->get_stride(), a->get_const_values(),
-            a->get_stride(), beta->get_const_values(), c->get_values(),
-            c->get_stride());
-    } else {
-        GKO_NOT_IMPLEMENTED;
-    }
+    using namespace oneapi::mkl;
+    oneapi::mkl::blas::row_major::gemm(
+        *exec->get_queue(), transpose::nontrans, transpose::nontrans,
+        c->get_size()[0], c->get_size()[1], a->get_size()[1],
+        exec->copy_val_to_host(alpha->get_const_values()),
+        a->get_const_values(), a->get_stride(), b->get_const_values(),
+        b->get_stride(), exec->copy_val_to_host(beta->get_const_values()),
+        c->get_values(), c->get_stride());
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
@@ -1341,20 +1322,20 @@ void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
                  const matrix::Dense<ValueType> *y,
                  matrix::Dense<ValueType> *result)
 {
-    if (cublas::is_supported<ValueType>::value) {
+    if (0) {
         // TODO: write a custom kernel which does this more efficiently
         for (size_type col = 0; col < x->get_size()[1]; ++col) {
-            cublas::dot(exec->get_cublas_handle(), x->get_size()[0],
-                        x->get_const_values() + col, x->get_stride(),
-                        y->get_const_values() + col, y->get_stride(),
-                        result->get_values() + col);
+            dot(*exec->get_queue(), x->get_size()[0],
+                x->get_const_values() + col, x->get_stride(),
+                y->get_const_values() + col, y->get_stride(),
+                result->get_values() + col);
         }
     } else {
         // TODO: these are tuning parameters obtained experimentally, once
         // we decide how to handle this uniformly, they should be modified
         // appropriately
         constexpr auto work_per_thread = 32;
-        constexpr auto block_size = 1024;
+        constexpr auto block_size = default_block_size;
 
         constexpr auto work_per_block = work_per_thread * block_size;
         const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
@@ -1366,7 +1347,8 @@ void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
             kernel::compute_partial_dot<block_size>(
                 grid_dim, block_dim, 0, exec->get_queue(), x->get_size()[0],
                 x->get_const_values() + col, x->get_stride(),
-                y->get_const_values() + col, y->get_stride(), work.get_data());
+                y->get_const_values() + col, y->get_stride(),
+                work.get_data());
             kernel::finalize_dot_computation<block_size>(
                 1, block_dim, 0, exec->get_queue(), grid_dim.x,
                 work.get_const_data(), result->get_values() + col);
@@ -1391,19 +1373,20 @@ void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::Dense<ValueType> *x,
                    matrix::Dense<remove_complex<ValueType>> *result)
 {
-    if (cublas::is_supported<ValueType>::value) {
+    if (0) {
         for (size_type col = 0; col < x->get_size()[1]; ++col) {
-            cublas::norm2(exec->get_cublas_handle(), x->get_size()[0],
-                          x->get_const_values() + col, x->get_stride(),
-                          result->get_values() + col);
+            oneapi::mkl::blas::row_major::nrm2(
+                *exec->get_queue(), x->get_size()[0],
+                x->get_const_values() + col, x->get_stride(),
+                result->get_values() + col);
         }
     } else {
         using norm_type = remove_complex<ValueType>;
-        // TODO: these are tuning parameters obtained experimentally, once
-        // we decide how to handle this uniformly, they should be modified
-        // appropriately
+        // // TODO: these are tuning parameters obtained experimentally, once
+        // // we decide how to handle this uniformly, they should be modified
+        // // appropriately
         constexpr auto work_per_thread = 32;
-        constexpr auto block_size = 1024;
+        constexpr auto block_size = default_block_size;
 
         constexpr auto work_per_block = work_per_thread * block_size;
         const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
@@ -1414,7 +1397,8 @@ void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
         for (size_type col = 0; col < x->get_size()[1]; ++col) {
             kernel::compute_partial_norm2<block_size>(
                 grid_dim, block_dim, 0, exec->get_queue(), x->get_size()[0],
-                x->get_const_values() + col, x->get_stride(), work.get_data());
+                x->get_const_values() + col, x->get_stride(),
+                work.get_data());
             kernel::finalize_norm2_computation<block_size>(
                 1, block_dim, 0, exec->get_queue(), grid_dim.x,
                 work.get_const_data(), result->get_values() + col);
@@ -1449,7 +1433,8 @@ void convert_to_coo(std::shared_ptr<const DpcppExecutor> exec,
     kernel::fill_in_coo(grid_dim, default_block_size, 0, exec->get_queue(),
                         num_rows, num_cols, stride,
                         nnz_prefix_sum.get_const_data(),
-                        source->get_const_values(), row_idxs, col_idxs, values);
+                        source->get_const_values(), row_idxs, col_idxs,
+                        values);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -1470,8 +1455,9 @@ void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
 
     auto stride = source->get_stride();
 
-    const auto rows_per_block = ceildiv(default_block_size, config::warp_size);
-    const auto grid_dim_nnz = ceildiv(source->get_size()[0], rows_per_block);
+    const auto rows_per_block = ceildiv(default_block_size,
+    config::warp_size); const auto grid_dim_nnz =
+    ceildiv(source->get_size()[0], rows_per_block);
 
     kernel::count_nnz_per_row(grid_dim_nnz, default_block_size, 0,
                               exec->get_queue(), num_rows, num_cols, stride,
@@ -1482,8 +1468,9 @@ void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
     size_type grid_dim = ceildiv(num_rows, default_block_size);
 
     kernel::fill_in_csr(grid_dim, default_block_size, 0, exec->get_queue(),
-                        num_rows, num_cols, stride, source->get_const_values(),
-                        row_ptrs, col_idxs, values);
+                        num_rows, num_cols, stride,
+                        source->get_const_values(), row_ptrs, col_idxs,
+                        values);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -1549,25 +1536,34 @@ void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
     const int slice_num = ceildiv(num_rows, slice_size);
 
     auto nnz_per_row = Array<size_type>(exec, num_rows);
+    std::cout << "calculate_nonzeros_per_row" << std::endl;
     calculate_nonzeros_per_row(exec, source, &nnz_per_row);
-
+    exec->synchronize();
+    std::cout << "calculate_nonzeros_per_row finish" << std::endl;
     auto grid_dim = slice_num;
 
     if (grid_dim > 0) {
+        std::cout << "calculate_slice_lengths" << std::endl;
         kernel::calculate_slice_lengths(
             grid_dim, config::warp_size, 0, exec->get_queue(), num_rows,
-            slice_size, slice_num, stride_factor, nnz_per_row.get_const_data(),
-            slice_lengths, slice_sets);
+            slice_size, slice_num, stride_factor,
+            nnz_per_row.get_const_data(), slice_lengths, slice_sets);
+        exec->synchronize();
+        std::cout << "calculate_slice_lengths finish" << std::endl;
     }
-
+    std::cout << "prefix_sum" << std::endl;
     components::prefix_sum(exec, slice_sets, slice_num + 1);
-
+    // exec->synchronize();
+    std::cout << "prefix_sum finish" << std::endl;
     grid_dim = ceildiv(num_rows, default_block_size);
     if (grid_dim > 0) {
+        std::cout << "fill_in_sellp" << std::endl;
         kernel::fill_in_sellp(grid_dim, default_block_size, 0,
-                              exec->get_queue(), num_rows, num_cols, slice_size,
-                              stride, source->get_const_values(), slice_lengths,
-                              slice_sets, col_idxs, vals);
+                              exec->get_queue(), num_rows, num_cols,
+                              slice_size, stride, source->get_const_values(),
+                              slice_lengths, slice_sets, col_idxs, vals);
+        exec->synchronize();
+        std::cout << "fill_in_sellp finish" << std::endl;
     }
 }
 
@@ -1626,7 +1622,8 @@ void calculate_max_nnz_per_row(std::shared_ptr<const DpcppExecutor> exec,
     kernel::reduce_max_nnz(1, default_block_size,
                            default_block_size * sizeof(size_type),
                            exec->get_queue(), grid_dim,
-                           block_results.get_const_data(), d_result.get_data());
+                           block_results.get_const_data(),
+                           d_result.get_data());
 
     *result = exec->copy_val_to_host(d_result.get_const_data());
 }
@@ -1646,9 +1643,10 @@ void calculate_nonzeros_per_row(std::shared_ptr<const DpcppExecutor> exec,
     const dim3 grid_size(grid_x, 1, 1);
     if (grid_x > 0) {
         kernel::count_nnz_per_row(
-            grid_size, block_size, 0, exec->get_queue(), source->get_size()[0],
-            source->get_size()[1], source->get_stride(),
-            source->get_const_values(), result->get_data());
+            grid_size, block_size, 0, exec->get_queue(),
+            source->get_size()[0], source->get_size()[1],
+            source->get_stride(), source->get_const_values(),
+            result->get_data());
     }
 }
 
@@ -1678,7 +1676,8 @@ void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
 
     auto max_nnz_per_slice = Array<size_type>(exec, slice_num);
 
-    auto grid_dim = ceildiv(slice_num * config::warp_size, default_block_size);
+    auto grid_dim = ceildiv(slice_num * config::warp_size,
+    default_block_size);
 
     kernel::reduce_max_nnz_per_slice(
         grid_dim, default_block_size, 0, exec->get_queue(), num_rows,
@@ -1712,22 +1711,24 @@ void transpose(std::shared_ptr<const DpcppExecutor> exec,
                const matrix::Dense<ValueType> *orig,
                matrix::Dense<ValueType> *trans)
 {
-    if (cublas::is_supported<ValueType>::value) {
-        auto handle = exec->get_cublas_handle();
-        {
-            cublas::pointer_mode_guard pm_guard(handle);
-            auto alpha = one<ValueType>();
-            auto beta = zero<ValueType>();
-            cublas::geam(
-                handle, oneapi::mkl::transpose::trans,
-                oneapi::mkl::transpose::nontrans, orig->get_size()[0],
-                orig->get_size()[1], &alpha, orig->get_const_values(),
-                orig->get_stride(), &beta, static_cast<ValueType *>(nullptr),
-                trans->get_size()[1], trans->get_values(), trans->get_stride());
-        }
-    } else {
-        GKO_NOT_IMPLEMENTED;
-    }
+    // if (cublas::is_supported<ValueType>::value) {
+    //     auto handle = exec->get_cublas_handle();
+    //     {
+    //         cublas::pointer_mode_guard pm_guard(handle);
+    //         auto alpha = one<ValueType>();
+    //         auto beta = zero<ValueType>();
+    //         cublas::geam(
+    //             handle, oneapi::mkl::transpose::trans,
+    //             oneapi::mkl::transpose::nontrans, orig->get_size()[0],
+    //             orig->get_size()[1], &alpha, orig->get_const_values(),
+    //             orig->get_stride(), &beta, static_cast<ValueType
+    //             *>(nullptr), trans->get_size()[1], trans->get_values(),
+    //             trans->get_stride());
+    //     }
+    // } else {
+    //     GKO_NOT_IMPLEMENTED;
+    // }
+    GKO_NOT_IMPLEMENTED;
 };
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_TRANSPOSE_KERNEL);
@@ -1738,22 +1739,24 @@ void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Dense<ValueType> *orig,
                     matrix::Dense<ValueType> *trans)
 {
-    if (cublas::is_supported<ValueType>::value) {
-        auto handle = exec->get_cublas_handle();
-        {
-            cublas::pointer_mode_guard pm_guard(handle);
-            auto alpha = one<ValueType>();
-            auto beta = zero<ValueType>();
-            cublas::geam(
-                handle, oneapi::mkl::transpose::conjtrans,
-                oneapi::mkl::transpose::nontrans, orig->get_size()[0],
-                orig->get_size()[1], &alpha, orig->get_const_values(),
-                orig->get_stride(), &beta, static_cast<ValueType *>(nullptr),
-                trans->get_size()[1], trans->get_values(), trans->get_stride());
-        }
-    } else {
-        GKO_NOT_IMPLEMENTED;
-    }
+    // if (cublas::is_supported<ValueType>::value) {
+    //     auto handle = exec->get_cublas_handle();
+    //     {
+    //         cublas::pointer_mode_guard pm_guard(handle);
+    //         auto alpha = one<ValueType>();
+    //         auto beta = zero<ValueType>();
+    //         cublas::geam(
+    //             handle, oneapi::mkl::transpose::conjtrans,
+    //             oneapi::mkl::transpose::nontrans, orig->get_size()[0],
+    //             orig->get_size()[1], &alpha, orig->get_const_values(),
+    //             orig->get_stride(), &beta, static_cast<ValueType
+    //             *>(nullptr), trans->get_size()[1], trans->get_values(),
+    //             trans->get_stride());
+    //     }
+    // } else {
+    //     GKO_NOT_IMPLEMENTED;
+    // }
+    GKO_NOT_IMPLEMENTED;
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
diff --git a/dpcpp/test/components/CMakeLists.txt b/dpcpp/test/components/CMakeLists.txt
index 87a034a64df..77ad6684840 100644
--- a/dpcpp/test/components/CMakeLists.txt
+++ b/dpcpp/test/components/CMakeLists.txt
@@ -2,3 +2,4 @@ ginkgo_create_test(absolute_array)
 ginkgo_create_dpcpp_test(cooperative_groups_kernels)
 ginkgo_create_test(fill_array)
 ginkgo_create_test(precision_conversion)
+ginkgo_create_test(prefix_sum)
diff --git a/dpcpp/test/components/prefix_sum.cpp b/dpcpp/test/components/prefix_sum.cpp
new file mode 100644
index 00000000000..3e2e7ca9d64
--- /dev/null
+++ b/dpcpp/test/components/prefix_sum.cpp
@@ -0,0 +1,96 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/prefix_sum.hpp"
+
+
+#include <memory>
+#include <random>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+
+
+#include "dpcpp/test/utils.hpp"
+
+
+namespace {
+
+
+class PrefixSum : public ::testing::Test {
+protected:
+    using index_type = gko::int32;
+    PrefixSum()
+        : ref(gko::ReferenceExecutor::create()),
+          exec(gko::DpcppExecutor::create(0, ref)),
+          rand(293),
+          total_size(42793),
+          vals(ref, total_size),
+          dvals(exec)
+    {
+        std::uniform_int_distribution<index_type> dist(0, 1000);
+        for (gko::size_type i = 0; i < total_size; ++i) {
+            vals.get_data()[i] = dist(rand);
+        }
+        dvals = vals;
+    }
+
+    void test(gko::size_type size)
+    {
+        gko::kernels::reference::components::prefix_sum(ref, vals.get_data(),
+                                                        size);
+        gko::kernels::dpcpp::components::prefix_sum(exec, dvals.get_data(),
+                                                    size);
+
+        GKO_ASSERT_ARRAY_EQ(vals, dvals);
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<gko::DpcppExecutor> exec;
+    std::default_random_engine rand;
+    gko::size_type total_size;
+    gko::Array<index_type> vals;
+    gko::Array<index_type> dvals;
+};
+
+
+TEST_F(PrefixSum, SmallEqualsReference) { test(100); }
+
+
+TEST_F(PrefixSum, BigEqualsReference) { test(total_size); }
+
+
+}  // namespace
diff --git a/dpcpp/test/matrix/CMakeLists.txt b/dpcpp/test/matrix/CMakeLists.txt
index af64b693718..ba8f0fb70fe 100644
--- a/dpcpp/test/matrix/CMakeLists.txt
+++ b/dpcpp/test/matrix/CMakeLists.txt
@@ -1 +1,2 @@
-ginkgo_create_test(csr_kernels)
\ No newline at end of file
+ginkgo_create_test(csr_kernels)
+ginkgo_create_test(dense_kernels)
diff --git a/dpcpp/test/matrix/dense_kernels.cpp b/dpcpp/test/matrix/dense_kernels.cpp
new file mode 100644
index 00000000000..7c65e8b0f84
--- /dev/null
+++ b/dpcpp/test/matrix/dense_kernels.cpp
@@ -0,0 +1,806 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+#include <ginkgo/core/matrix/ell.hpp>
+#include <ginkgo/core/matrix/sellp.hpp>
+
+
+#include "core/components/fill_array.hpp"
+#include "core/matrix/dense_kernels.hpp"
+#include "dpcpp/test/utils.hpp"
+
+
+namespace {
+
+
+class Dense : public ::testing::Test {
+protected:
+    using itype = int;
+    using vtype = double;
+    using Mtx = gko::matrix::Dense<vtype>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<vtype>>;
+    using Arr = gko::Array<itype>;
+    using ComplexMtx = gko::matrix::Dense<std::complex<vtype>>;
+
+    Dense() : rand_engine(15) {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices("gpu"), 0);
+        ref = gko::ReferenceExecutor::create();
+        dpcpp = gko::DpcppExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (dpcpp != nullptr) {
+            ASSERT_NO_THROW(dpcpp->synchronize());
+        }
+    }
+
+    template <typename MtxType>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<MtxType>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(num_cols, num_cols),
+            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
+    }
+
+    void set_up_vector_data(gko::size_type num_vecs,
+                            bool different_alpha = false)
+    {
+        x = gen_mtx<Mtx>(1000, num_vecs);
+        y = gen_mtx<Mtx>(1000, num_vecs);
+        if (different_alpha) {
+            alpha = gen_mtx<Mtx>(1, num_vecs);
+        } else {
+            alpha = gko::initialize<Mtx>({2.0}, ref);
+        }
+        dx = Mtx::create(dpcpp);
+        dx->copy_from(x.get());
+        dy = Mtx::create(dpcpp);
+        dy->copy_from(y.get());
+        dalpha = Mtx::create(dpcpp);
+        dalpha->copy_from(alpha.get());
+        expected = Mtx::create(ref, gko::dim<2>{1, num_vecs});
+        dresult = Mtx::create(dpcpp, gko::dim<2>{1, num_vecs});
+    }
+
+    void set_up_apply_data()
+    {
+        x = gen_mtx<Mtx>(65, 25);
+        c_x = gen_mtx<ComplexMtx>(65, 25);
+        y = gen_mtx<Mtx>(25, 35);
+        expected = gen_mtx<Mtx>(65, 35);
+        alpha = gko::initialize<Mtx>({2.0}, ref);
+        beta = gko::initialize<Mtx>({-1.0}, ref);
+        square = gen_mtx<Mtx>(x->get_size()[0], x->get_size()[0]);
+        dx = Mtx::create(dpcpp);
+        dx->copy_from(x.get());
+        dc_x = ComplexMtx::create(dpcpp);
+        dc_x->copy_from(c_x.get());
+        dy = Mtx::create(dpcpp);
+        dy->copy_from(y.get());
+        dresult = Mtx::create(dpcpp);
+        dresult->copy_from(expected.get());
+        dalpha = Mtx::create(dpcpp);
+        dalpha->copy_from(alpha.get());
+        dbeta = Mtx::create(dpcpp);
+        dbeta->copy_from(beta.get());
+        dsquare = Mtx::create(dpcpp);
+        dsquare->copy_from(square.get());
+
+        std::vector<itype> tmp(x->get_size()[0], 0);
+        auto rng = std::default_random_engine{};
+        std::iota(tmp.begin(), tmp.end(), 0);
+        std::shuffle(tmp.begin(), tmp.end(), rng);
+        std::vector<itype> tmp2(x->get_size()[1], 0);
+        std::iota(tmp2.begin(), tmp2.end(), 0);
+        std::shuffle(tmp2.begin(), tmp2.end(), rng);
+        std::vector<itype> tmp3(x->get_size()[0] / 10);
+        std::uniform_int_distribution<itype> row_dist(0, x->get_size()[0] - 1);
+        for (auto &i : tmp3) {
+            i = row_dist(rng);
+        }
+        rpermute_idxs =
+            std::unique_ptr<Arr>(new Arr{ref, tmp.begin(), tmp.end()});
+        cpermute_idxs =
+            std::unique_ptr<Arr>(new Arr{ref, tmp2.begin(), tmp2.end()});
+        rgather_idxs =
+            std::unique_ptr<Arr>(new Arr{ref, tmp3.begin(), tmp3.end()});
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
+
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Mtx> x;
+    std::unique_ptr<ComplexMtx> c_x;
+    std::unique_ptr<Mtx> y;
+    std::unique_ptr<Mtx> alpha;
+    std::unique_ptr<Mtx> beta;
+    std::unique_ptr<Mtx> expected;
+    std::unique_ptr<Mtx> square;
+    std::unique_ptr<Mtx> dresult;
+    std::unique_ptr<Mtx> dx;
+    std::unique_ptr<ComplexMtx> dc_x;
+    std::unique_ptr<Mtx> dy;
+    std::unique_ptr<Mtx> dalpha;
+    std::unique_ptr<Mtx> dbeta;
+    std::unique_ptr<Mtx> dsquare;
+    std::unique_ptr<Arr> rpermute_idxs;
+    std::unique_ptr<Arr> cpermute_idxs;
+    std::unique_ptr<Arr> rgather_idxs;
+};
+
+
+TEST_F(Dense, DpcppFillIsEquivalentToRef)
+{
+    set_up_vector_data(3);
+    auto result = Mtx::create(ref);
+
+    x->fill(42);
+    dx->fill(42);
+    result->copy_from(dx.get());
+
+    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
+}
+
+
+TEST_F(Dense, DpcppStridedFillIsEquivalentToRef)
+{
+    using T = double;
+    auto x = gko::initialize<gko::matrix::Dense<T>>(
+        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, ref);
+    auto dx = gko::initialize<gko::matrix::Dense<T>>(
+        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, dpcpp);
+    auto result = Mtx::create(ref);
+
+    x->fill(42);
+    dx->fill(42);
+    result->copy_from(dx.get());
+
+    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
+}
+
+
+TEST_F(Dense, SingleVectorDpcppScaleIsEquivalentToRef)
+{
+    set_up_vector_data(1);
+    auto result = Mtx::create(ref);
+
+    x->scale(alpha.get());
+    dx->scale(dalpha.get());
+    result->copy_from(dx.get());
+
+    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
+}
+
+
+TEST_F(Dense, MultipleVectorDpcppScaleIsEquivalentToRef)
+{
+    set_up_vector_data(20);
+
+    x->scale(alpha.get());
+    dx->scale(dalpha.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
+}
+
+
+TEST_F(Dense, MultipleVectorDpcppScaleWithDifferentAlphaIsEquivalentToRef)
+{
+    set_up_vector_data(20, true);
+
+    x->scale(alpha.get());
+    dx->scale(dalpha.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
+}
+
+
+TEST_F(Dense, SingleVectorDpcppAddScaledIsEquivalentToRef)
+{
+    set_up_vector_data(1);
+
+    x->add_scaled(alpha.get(), y.get());
+    dx->add_scaled(dalpha.get(), dy.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
+}
+
+
+TEST_F(Dense, MultipleVectorDpcppAddScaledIsEquivalentToRef)
+{
+    set_up_vector_data(20);
+
+    x->add_scaled(alpha.get(), y.get());
+    dx->add_scaled(dalpha.get(), dy.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
+}
+
+
+TEST_F(Dense, MultipleVectorDpcppAddScaledWithDifferentAlphaIsEquivalentToRef)
+{
+    set_up_vector_data(20);
+
+    x->add_scaled(alpha.get(), y.get());
+    dx->add_scaled(dalpha.get(), dy.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
+}
+
+
+TEST_F(Dense, AddsScaledDiagIsEquivalentToRef)
+{
+    auto mat = gen_mtx<Mtx>(532, 532);
+    gko::Array<Mtx::value_type> diag_values(ref, 532);
+    gko::kernels::reference::components::fill_array(ref, diag_values.get_data(),
+                                                    532, Mtx::value_type{2.0});
+    auto diag =
+        gko::matrix::Diagonal<Mtx::value_type>::create(ref, 532, diag_values);
+    alpha = gko::initialize<Mtx>({2.0}, ref);
+    auto dmat = Mtx::create(dpcpp);
+    dmat->copy_from(mat.get());
+    auto ddiag = gko::matrix::Diagonal<Mtx::value_type>::create(dpcpp);
+    ddiag->copy_from(diag.get());
+    dalpha = Mtx::create(dpcpp);
+    dalpha->copy_from(alpha.get());
+
+    mat->add_scaled(alpha.get(), diag.get());
+    dmat->add_scaled(dalpha.get(), ddiag.get());
+
+    GKO_ASSERT_MTX_NEAR(mat, dmat, 1e-14);
+}
+
+
+TEST_F(Dense, SingleVectorDpcppComputeDotIsEquivalentToRef)
+{
+    set_up_vector_data(1);
+
+    x->compute_dot(y.get(), expected.get());
+    dx->compute_dot(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Dense, MultipleVectorDpcppComputeDotIsEquivalentToRef)
+{
+    set_up_vector_data(20);
+
+    x->compute_dot(y.get(), expected.get());
+    dx->compute_dot(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Dense, DpcppComputeNorm2IsEquivalentToRef)
+{
+    set_up_vector_data(20);
+    auto norm_size = gko::dim<2>{1, x->get_size()[1]};
+    auto norm_expected = NormVector::create(this->ref, norm_size);
+    auto dnorm = NormVector::create(this->dpcpp, norm_size);
+
+    x->compute_norm2(norm_expected.get());
+    dx->compute_norm2(dnorm.get());
+
+    GKO_ASSERT_MTX_NEAR(norm_expected, dnorm, 1e-14);
+}
+
+
+TEST_F(Dense, SimpleApplyIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    x->apply(y.get(), expected.get());
+    dx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    x->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(25, 1);
+    auto dcomplex_b = ComplexMtx::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(65, 1);
+    auto dcomplex_x = ComplexMtx::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(complex_b.get(), complex_x.get());
+    dx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+
+TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(25, 1);
+    auto dcomplex_b = ComplexMtx::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(65, 1);
+    auto dcomplex_x = ComplexMtx::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+
+TEST_F(Dense, IsTransposable)
+{
+    set_up_apply_data();
+
+    auto trans = x->transpose();
+    auto dtrans = dx->transpose();
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(dtrans.get()),
+                        static_cast<Mtx *>(trans.get()), 0);
+}
+
+
+TEST_F(Dense, IsConjugateTransposable)
+{
+    set_up_apply_data();
+
+    auto trans = c_x->conj_transpose();
+    auto dtrans = dc_x->conj_transpose();
+
+    GKO_ASSERT_MTX_NEAR(static_cast<ComplexMtx *>(dtrans.get()),
+                        static_cast<ComplexMtx *>(trans.get()), 0);
+}
+
+
+TEST_F(Dense, ConvertToCooIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto coo_mtx = gko::matrix::Coo<>::create(ref);
+    auto dcoo_mtx = gko::matrix::Coo<>::create(dpcpp);
+
+    x->convert_to(coo_mtx.get());
+    dx->convert_to(dcoo_mtx.get());
+
+    ASSERT_EQ(dcoo_mtx->get_num_stored_elements(),
+              coo_mtx->get_num_stored_elements());
+    GKO_ASSERT_MTX_NEAR(dcoo_mtx.get(), coo_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Dense, MoveToCooIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto coo_mtx = gko::matrix::Coo<>::create(ref);
+    auto dcoo_mtx = gko::matrix::Coo<>::create(dpcpp);
+
+    x->move_to(coo_mtx.get());
+    dx->move_to(dcoo_mtx.get());
+
+    ASSERT_EQ(dcoo_mtx->get_num_stored_elements(),
+              coo_mtx->get_num_stored_elements());
+    GKO_ASSERT_MTX_NEAR(dcoo_mtx.get(), coo_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Dense, ConvertToCsrIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto csr_mtx = gko::matrix::Csr<>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<>::create(dpcpp);
+
+    x->convert_to(csr_mtx.get());
+    dx->convert_to(dcsr_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(dcsr_mtx.get(), csr_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Dense, MoveToCsrIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto csr_mtx = gko::matrix::Csr<>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<>::create(dpcpp);
+
+    x->move_to(csr_mtx.get());
+    dx->move_to(dcsr_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(dcsr_mtx.get(), csr_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Dense, ConvertToEllIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto ell_mtx = gko::matrix::Ell<>::create(ref);
+    auto dell_mtx = gko::matrix::Ell<>::create(dpcpp);
+
+    x->convert_to(ell_mtx.get());
+    dx->convert_to(dell_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(dell_mtx.get(), ell_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Dense, MoveToEllIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto ell_mtx = gko::matrix::Ell<>::create(ref);
+    auto dell_mtx = gko::matrix::Ell<>::create(dpcpp);
+
+    x->move_to(ell_mtx.get());
+    dx->move_to(dell_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(dell_mtx.get(), ell_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Dense, ConvertToSellpIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
+    auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+
+    x->convert_to(sellp_mtx.get());
+    dx->convert_to(dsellp_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(sellp_mtx, dsellp_mtx, 1e-14);
+}
+
+
+TEST_F(Dense, MoveToSellpIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
+    auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+
+    x->move_to(sellp_mtx.get());
+    dx->move_to(dsellp_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(sellp_mtx, dsellp_mtx, 1e-14);
+}
+
+
+TEST_F(Dense, ConvertsEmptyToSellp)
+{
+    auto dempty_mtx = Mtx::create(dpcpp);
+    auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+
+    dempty_mtx->convert_to(dsellp_mtx.get());
+
+    ASSERT_EQ(dpcpp->copy_val_to_host(dsellp_mtx->get_const_slice_sets()), 0);
+    ASSERT_FALSE(dsellp_mtx->get_size());
+}
+
+
+TEST_F(Dense, CountNNZIsEquivalentToRef)
+{
+    set_up_apply_data();
+    gko::size_type nnz;
+    gko::size_type dnnz;
+
+    gko::kernels::reference::dense::count_nonzeros(ref, x.get(), &nnz);
+    gko::kernels::dpcpp::dense::count_nonzeros(dpcpp, dx.get(), &dnnz);
+
+    ASSERT_EQ(nnz, dnnz);
+}
+
+
+TEST_F(Dense, CalculateNNZPerRowIsEquivalentToRef)
+{
+    set_up_apply_data();
+    gko::Array<gko::size_type> nnz_per_row(ref);
+    nnz_per_row.resize_and_reset(x->get_size()[0]);
+    gko::Array<gko::size_type> dnnz_per_row(dpcpp);
+    dnnz_per_row.resize_and_reset(dx->get_size()[0]);
+
+    gko::kernels::reference::dense::calculate_nonzeros_per_row(ref, x.get(),
+                                                               &nnz_per_row);
+    gko::kernels::dpcpp::dense::calculate_nonzeros_per_row(dpcpp, dx.get(),
+                                                           &dnnz_per_row);
+
+    auto tmp = gko::Array<gko::size_type>(ref, dnnz_per_row);
+    for (auto i = 0; i < nnz_per_row.get_num_elems(); i++) {
+        ASSERT_EQ(nnz_per_row.get_const_data()[i], tmp.get_const_data()[i]);
+    }
+}
+
+
+TEST_F(Dense, CalculateMaxNNZPerRowIsEquivalentToRef)
+{
+    set_up_apply_data();
+    gko::size_type max_nnz;
+    gko::size_type dmax_nnz;
+
+    gko::kernels::reference::dense::calculate_max_nnz_per_row(ref, x.get(),
+                                                              &max_nnz);
+    gko::kernels::dpcpp::dense::calculate_max_nnz_per_row(dpcpp, dx.get(),
+                                                          &dmax_nnz);
+
+    ASSERT_EQ(max_nnz, dmax_nnz);
+}
+
+
+TEST_F(Dense, CalculateTotalColsIsEquivalentToRef)
+{
+    set_up_apply_data();
+    gko::size_type total_cols;
+    gko::size_type dtotal_cols;
+
+    gko::kernels::reference::dense::calculate_total_cols(
+        ref, x.get(), &total_cols, 2, gko::matrix::default_slice_size);
+    gko::kernels::dpcpp::dense::calculate_total_cols(
+        dpcpp, dx.get(), &dtotal_cols, 2, gko::matrix::default_slice_size);
+
+    ASSERT_EQ(total_cols, dtotal_cols);
+}
+
+
+TEST_F(Dense, CanGatherRows)
+{
+    set_up_apply_data();
+
+    auto r_gather = x->row_gather(rgather_idxs.get());
+    auto dr_gather = dx->row_gather(rgather_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
+}
+
+
+TEST_F(Dense, CanGatherRowsIntoDense)
+{
+    set_up_apply_data();
+    auto gather_size =
+        gko::dim<2>{rgather_idxs->get_num_elems(), x->get_size()[1]};
+    auto r_gather = Mtx::create(ref, gather_size);
+    // test make_temporary_clone and non-default stride
+    auto dr_gather = Mtx::create(ref, gather_size, x->get_size()[1] + 2);
+
+    x->row_gather(rgather_idxs.get(), r_gather.get());
+    dx->row_gather(rgather_idxs.get(), dr_gather.get());
+
+    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
+}
+
+
+TEST_F(Dense, IsPermutable)
+{
+    set_up_apply_data();
+
+    auto permuted = square->permute(rpermute_idxs.get());
+    auto dpermuted = dsquare->permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
+                        static_cast<Mtx *>(dpermuted.get()), 0);
+}
+
+
+TEST_F(Dense, IsInversePermutable)
+{
+    set_up_apply_data();
+
+    auto permuted = square->inverse_permute(rpermute_idxs.get());
+    auto dpermuted = dsquare->inverse_permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
+                        static_cast<Mtx *>(dpermuted.get()), 0);
+}
+
+
+TEST_F(Dense, IsRowPermutable)
+{
+    set_up_apply_data();
+
+    auto r_permute = x->row_permute(rpermute_idxs.get());
+    auto dr_permute = dx->row_permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(r_permute.get()),
+                        static_cast<Mtx *>(dr_permute.get()), 0);
+}
+
+
+TEST_F(Dense, IsColPermutable)
+{
+    set_up_apply_data();
+
+    auto c_permute = x->column_permute(cpermute_idxs.get());
+    auto dc_permute = dx->column_permute(cpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(c_permute.get()),
+                        static_cast<Mtx *>(dc_permute.get()), 0);
+}
+
+
+TEST_F(Dense, IsInverseRowPermutable)
+{
+    set_up_apply_data();
+
+    auto inverse_r_permute = x->inverse_row_permute(rpermute_idxs.get());
+    auto d_inverse_r_permute = dx->inverse_row_permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_r_permute.get()),
+                        static_cast<Mtx *>(d_inverse_r_permute.get()), 0);
+}
+
+
+TEST_F(Dense, IsInverseColPermutable)
+{
+    set_up_apply_data();
+
+    auto inverse_c_permute = x->inverse_column_permute(cpermute_idxs.get());
+    auto d_inverse_c_permute = dx->inverse_column_permute(cpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_c_permute.get()),
+                        static_cast<Mtx *>(d_inverse_c_permute.get()), 0);
+}
+
+
+TEST_F(Dense, ExtractDiagonalIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto diag = x->extract_diagonal();
+    auto ddiag = dx->extract_diagonal();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
+TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    x->compute_absolute_inplace();
+    dx->compute_absolute_inplace();
+
+    GKO_ASSERT_MTX_NEAR(x, dx, 1e-14);
+}
+
+
+TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_x = x->compute_absolute();
+    auto dabs_x = dx->compute_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, 1e-14);
+}
+
+
+TEST_F(Dense, MakeComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto complex_x = x->make_complex();
+    auto dcomplex_x = dx->make_complex();
+
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
+}
+
+
+TEST_F(Dense, MakeComplexWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto complex_x = ComplexMtx::create(ref, x->get_size());
+    x->make_complex(complex_x.get());
+    auto dcomplex_x = ComplexMtx::create(dpcpp, x->get_size());
+    dx->make_complex(dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
+}
+
+
+TEST_F(Dense, GetRealIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto real_x = x->get_real();
+    auto dreal_x = dx->get_real();
+
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
+}
+
+
+TEST_F(Dense, GetRealWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto real_x = Mtx::create(ref, x->get_size());
+    x->get_real(real_x.get());
+    auto dreal_x = Mtx::create(dpcpp, dx->get_size());
+    dx->get_real(dreal_x.get());
+
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
+}
+
+
+TEST_F(Dense, GetImagIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto imag_x = x->get_imag();
+    auto dimag_x = dx->get_imag();
+
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
+}
+
+
+TEST_F(Dense, GetImagWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto imag_x = Mtx::create(ref, x->get_size());
+    x->get_imag(imag_x.get());
+    auto dimag_x = Mtx::create(dpcpp, dx->get_size());
+    dx->get_imag(dimag_x.get());
+
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
+}
+
+
+}  // namespace
diff --git a/dpcpp/test/utils.hpp b/dpcpp/test/utils.hpp
new file mode 100644
index 00000000000..88d98f0d9f6
--- /dev/null
+++ b/dpcpp/test/utils.hpp
@@ -0,0 +1,54 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_TEST_UTILS_HPP_
+#define GKO_DPCPP_TEST_UTILS_HPP_
+
+
+#include "core/test/utils.hpp"
+
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+namespace {
+
+
+// prevent device reset after each test
+auto no_reset_exec =
+    gko::DpcppExecutor::create(0, gko::ReferenceExecutor::create());
+
+
+}  // namespace
+
+
+#endif  // GKO_DPCPP_TEST_UTILS_HPP_

From c4e81dced94cee1360801b7ea85dc1cba0b2f483 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 20 May 2021 11:42:51 +0200
Subject: [PATCH 0893/1008] use warp_size 32 to check

---
 dpcpp/base/config.hpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/dpcpp/base/config.hpp b/dpcpp/base/config.hpp
index 78fe25978a7..78fdcc2b819 100644
--- a/dpcpp/base/config.hpp
+++ b/dpcpp/base/config.hpp
@@ -49,6 +49,12 @@ struct config {
      */
     using lane_mask_type = uint64;
 
+
+    /**
+     * The number of threads within a CUDA warp.
+     */
+    static constexpr uint32 warp_size = 32;
+
     /**
      * The bitmask of the entire warp.
      */

From 76e0294782941cdfd4ce5d9819f9b4b51aa9107e Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 20 May 2021 16:06:36 +0200
Subject: [PATCH 0894/1008] fix extract_diag, try default, and single test

---
 dpcpp/base/helper.hpp               |  22 ++
 dpcpp/components/prefix_sum.dp.hpp  |   8 +-
 dpcpp/components/reduction.dp.hpp   |   4 +-
 dpcpp/matrix/dense_kernels.dp.cpp   | 363 +++++++++------------------
 dpcpp/test/matrix/dense_kernels.cpp | 374 +++++++++-------------------
 5 files changed, 274 insertions(+), 497 deletions(-)

diff --git a/dpcpp/base/helper.hpp b/dpcpp/base/helper.hpp
index c888eb9d99d..f8eee93f25b 100644
--- a/dpcpp/base/helper.hpp
+++ b/dpcpp/base/helper.hpp
@@ -46,6 +46,28 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "dpcpp/base/dim3.dp.hpp"
 
 
+/**
+ * GKO_ENABLE_DEFAULT_HOST gives a default host implementation for those
+ * kernels which require encoded config but do not need explicit template
+ * parameter and share memory
+ *
+ * @param name_  the name of the host function with config
+ * @param kernel_  the kernel name
+ */
+#define GKO_ENABLE_DEFAULT_HOST(name_, kernel_)                     \
+    template <typename... InferredArgs>                             \
+    void name_(dim3 grid, dim3 block, size_t dynamic_shared_memory, \
+               sycl::queue *queue, InferredArgs... args)            \
+    {                                                               \
+        queue->submit([&](sycl::handler &cgh) {                     \
+            cgh.parallel_for(sycl_nd_range(grid, block),            \
+                             [=](sycl::nd_item<3> item_ct1) {       \
+                                 kernel_(args..., item_ct1);        \
+                             });                                    \
+        });                                                         \
+    }
+
+
 /**
  * GKO_ENABLE_DEFAULT_HOST_CONFIG gives a default host implementation for those
  * kernels which require encoded config but do not need explicit template
diff --git a/dpcpp/components/prefix_sum.dp.hpp b/dpcpp/components/prefix_sum.dp.hpp
index 6b3498d1dea..c6f7c7cfb20 100644
--- a/dpcpp/components/prefix_sum.dp.hpp
+++ b/dpcpp/components/prefix_sum.dp.hpp
@@ -189,8 +189,8 @@ void start_prefix_sum(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                        sycl::access::target::local>
             prefix_helper_acc_ct1(cgh);
 
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
+        auto local_range = block.get_range();
+        auto global_range = grid.get_range() * local_range;
 
         cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
                          [=](sycl::nd_item<3> item_ct1) {
@@ -240,8 +240,8 @@ void finalize_prefix_sum(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                          ValueType *elements, const ValueType *block_sum)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
+        auto local_range = block.get_range();
+        auto global_range = grid.get_range() * local_range;
 
         cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
                          [=](sycl::nd_item<3> item_ct1) {
diff --git a/dpcpp/components/reduction.dp.hpp b/dpcpp/components/reduction.dp.hpp
index 4caf46229c8..e47d9038af3 100644
--- a/dpcpp/components/reduction.dp.hpp
+++ b/dpcpp/components/reduction.dp.hpp
@@ -216,8 +216,8 @@ void reduce_add_array(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                        sycl::access::target::local>
             block_sum_acc_ct1(cgh);
 
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
+        auto local_range = block.get_range();
+        auto global_range = grid.get_range() * local_range;
 
         cgh.parallel_for(
             sycl::nd_range<3>(global_range, local_range),
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index c5074b5cc38..494caff94c2 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -51,6 +51,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/prefix_sum.hpp"
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/base/helper.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/reduction.dp.hpp"
 #include "dpcpp/components/thread_ids.dp.hpp"
@@ -88,22 +89,7 @@ void strided_fill(size_type num_rows, size_type num_cols, size_type stride,
     }
 }
 
-template <typename ValueType>
-void strided_fill(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                  sycl::queue *stream, size_type num_rows, size_type num_cols,
-                  size_type stride, ValueType *mat, ValueType value)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             strided_fill(num_rows, num_cols, stride, mat,
-                                          value, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(strided_fill, strided_fill)
 
 
 template <size_type block_size, typename ValueType>
@@ -132,15 +118,11 @@ void scale(dim3 grid, dim3 block, size_t dynamic_shared_memory,
            size_type stride_x)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             scale<block_size>(num_rows, num_cols,
-                                               num_alpha_cols, alpha, x,
-                                               stride_x, item_ct1);
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                scale<block_size>(num_rows, num_cols, num_alpha_cols, alpha, x,
+                                  stride_x, item_ct1);
+            });
     });
 }
 
@@ -172,15 +154,12 @@ void add_scaled(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                 size_type stride_y)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             add_scaled<block_size>(
-                                 num_rows, num_cols, num_alpha_cols, alpha, x,
-                                 stride_x, y, stride_y, item_ct1);
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                add_scaled<block_size>(num_rows, num_cols, num_alpha_cols,
+                                       alpha, x, stride_x, y, stride_y,
+                                       item_ct1);
+            });
     });
 }
 
@@ -207,14 +186,10 @@ void add_scaled_diag(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                      ValueType *y, size_type stride_y)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             add_scaled_diag(size, alpha, diag, y, stride_y,
-                                             item_ct1);
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                add_scaled_diag(size, alpha, diag, y, stride_y, item_ct1);
+            });
     });
 }
 
@@ -234,7 +209,7 @@ void compute_partial_reduce(size_type num_rows, OutType *__restrict__ work,
     const auto global_id =
         thread::get_thread_id<config::warp_size, warps_per_block>(item_ct1);
 
-    OutType *tmp_work_array=*tmp_work;
+    OutType *tmp_work_array = *tmp_work;
     auto tmp = zero<OutType>();
     for (auto i = global_id; i < num_rows; i += block_size * num_blocks) {
         tmp = reduce_op(tmp, get_value(i));
@@ -243,8 +218,7 @@ void compute_partial_reduce(size_type num_rows, OutType *__restrict__ work,
     tmp_work_array[local_id] = tmp;
 
     ::gko::kernels::dpcpp::reduce(group::this_thread_block(item_ct1),
-                                  tmp_work_array,
-                                  reduce_op);
+                                  tmp_work_array, reduce_op);
 
     if (local_id == 0) {
         work[thread::get_block_id(item_ct1)] = tmp_work_array[0];
@@ -267,12 +241,11 @@ void finalize_reduce_computation(
     for (auto i = local_id; i < size; i += block_size) {
         tmp = reduce_op(tmp, work[i]);
     }
-    ValueType *tmp_work_array=*tmp_work;
+    ValueType *tmp_work_array = *tmp_work;
     tmp_work_array[local_id] = tmp;
 
     ::gko::kernels::dpcpp::reduce(group::this_thread_block(item_ct1),
-                                  tmp_work_array,
-                                  reduce_op);
+                                  tmp_work_array, reduce_op);
 
     if (local_id == 0) {
         *result = finalize_op(tmp_work_array[0]);
@@ -309,17 +282,14 @@ void compute_partial_dot(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                        sycl::access::target::local>
             tmp_work_acc_ct1(cgh);
 
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
 
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             compute_partial_dot<block_size>(
-                                 num_rows, x, stride_x, y, stride_y, work,
-                                 item_ct1,
-                                 (UninitializedArray<ValueType, block_size> *)
-                                     tmp_work_acc_ct1.get_pointer());
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                compute_partial_dot<block_size>(
+                    num_rows, x, stride_x, y, stride_y, work, item_ct1,
+                    (UninitializedArray<ValueType, block_size> *)
+                        tmp_work_acc_ct1.get_pointer());
+            });
     });
 }
 
@@ -348,10 +318,8 @@ void finalize_dot_computation(dim3 grid, dim3 block,
                        sycl::access::target::local>
             tmp_work_acc_ct1(cgh);
 
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
 
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+        cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              finalize_dot_computation<block_size>(
                                  size, work, result, item_ct1,
@@ -388,12 +356,9 @@ void compute_partial_norm2(dim3 grid, dim3 block, size_t dynamic_shared_memory,
             sycl::access::mode::read_write, sycl::access::target::local>
             tmp_work_acc_ct1(cgh);
 
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
 
         cgh.parallel_for(
-            sycl::nd_range<3>(global_range, local_range),
-            [=](sycl::nd_item<3> item_ct1) {
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 compute_partial_norm2<block_size>(
                     num_rows, x, stride_x, work, item_ct1,
                     (UninitializedArray<remove_complex<ValueType>, block_size>
@@ -427,10 +392,8 @@ void finalize_norm2_computation(dim3 grid, dim3 block,
                        sycl::access::target::local>
             tmp_work_acc_ct1(cgh);
 
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
 
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+        cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              finalize_norm2_computation<block_size>(
                                  size, work, result, item_ct1,
@@ -472,15 +435,11 @@ void fill_in_coo(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                  IndexType *col_idxs, ValueType *values)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             fill_in_coo(num_rows, num_cols, stride, row_ptrs,
-                                         source, row_idxs, col_idxs, values,
-                                         item_ct1);
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                fill_in_coo(num_rows, num_cols, stride, row_ptrs, source,
+                            row_idxs, col_idxs, values, item_ct1);
+            });
     });
 }
 
@@ -516,10 +475,7 @@ void count_nnz_per_row(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                        const ValueType *work, IndexType *result)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+        cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              count_nnz_per_row(num_rows, num_cols, stride, work,
                                                result, item_ct1);
@@ -556,10 +512,7 @@ void fill_in_csr(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                  IndexType *col_idxs, ValueType *values)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+        cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              fill_in_csr(num_rows, num_cols, stride, source,
                                          row_ptrs, col_idxs, values, item_ct1);
@@ -606,10 +559,7 @@ void fill_in_ell(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                  IndexType *col_ptrs, ValueType *values)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+        cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              fill_in_ell(num_rows, num_cols, source_stride,
                                          source, max_nnz_per_row, result_stride,
@@ -662,16 +612,12 @@ void calculate_slice_lengths(dim3 grid, dim3 block,
                              size_type *slice_lengths, size_type *slice_sets)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             calculate_slice_lengths(num_rows, slice_size,
-                                                     slice_num, stride_factor,
-                                                     nnz_per_row, slice_lengths,
-                                                     slice_sets, item_ct1);
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                calculate_slice_lengths(num_rows, slice_size, slice_num,
+                                        stride_factor, nnz_per_row,
+                                        slice_lengths, slice_sets, item_ct1);
+            });
     });
 }
 
@@ -717,16 +663,12 @@ void fill_in_sellp(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                    size_type *slice_sets, IndexType *col_idxs, ValueType *vals)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             fill_in_sellp(num_rows, num_cols, slice_size,
-                                           stride, source, slice_lengths,
-                                           slice_sets, col_idxs, vals,
-                                           item_ct1);
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                fill_in_sellp(num_rows, num_cols, slice_size, stride, source,
+                              slice_lengths, slice_sets, col_idxs, vals,
+                              item_ct1);
+            });
     });
 }
 
@@ -755,10 +697,8 @@ void reduce_max_nnz(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                        sycl::access::target::local>
             dpct_local_acc_ct1(sycl::range<1>(dynamic_shared_memory), cgh);
 
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
 
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+        cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              reduce_max_nnz(size, nnz_per_row, result, item_ct1,
                                             dpct_local_acc_ct1.get_pointer());
@@ -804,15 +744,11 @@ void reduce_max_nnz_per_slice(dim3 grid, dim3 block,
                               const size_type *nnz_per_row, size_type *result)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             reduce_max_nnz_per_slice(
-                                 num_rows, slice_size, stride_factor,
-                                 nnz_per_row, result, item_ct1);
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                reduce_max_nnz_per_slice(num_rows, slice_size, stride_factor,
+                                         nnz_per_row, result, item_ct1);
+            });
     });
 }
 
@@ -841,15 +777,12 @@ void reduce_total_cols(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                        sycl::access::target::local>
             dpct_local_acc_ct1(sycl::range<1>(dynamic_shared_memory), cgh);
 
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
 
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             reduce_total_cols(
-                                 num_slices, max_nnz_per_slice, result,
-                                 item_ct1, dpct_local_acc_ct1.get_pointer());
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                reduce_total_cols(num_slices, max_nnz_per_slice, result,
+                                  item_ct1, dpct_local_acc_ct1.get_pointer());
+            });
     });
 }
 
@@ -878,15 +811,11 @@ void symm_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                   size_type stride_result)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             symm_permute(num_rows, num_cols, perm_idxs, orig,
-                                          stride_orig, result, stride_result,
-                                          item_ct1);
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                symm_permute(num_rows, num_cols, perm_idxs, orig, stride_orig,
+                             result, stride_result, item_ct1);
+            });
     });
 }
 
@@ -915,15 +844,11 @@ void inv_symm_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                       ValueType *result, size_type stride_result)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             inv_symm_permute(num_rows, num_cols, perm_idxs,
-                                              orig, stride_orig, result,
-                                              stride_result, item_ct1);
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                inv_symm_permute(num_rows, num_cols, perm_idxs, orig,
+                                 stride_orig, result, stride_result, item_ct1);
+            });
     });
 }
 
@@ -952,15 +877,11 @@ void row_gather(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                 size_type stride_result)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             row_gather(num_rows, num_cols, perm_idxs, orig,
-                                        stride_orig, result, stride_result,
-                                        item_ct1);
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                row_gather(num_rows, num_cols, perm_idxs, orig, stride_orig,
+                           result, stride_result, item_ct1);
+            });
     });
 }
 
@@ -989,15 +910,11 @@ void column_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                     size_type stride_result)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             column_permute(num_rows, num_cols, perm_idxs, orig,
-                                            stride_orig, result, stride_result,
-                                            item_ct1);
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                column_permute(num_rows, num_cols, perm_idxs, orig, stride_orig,
+                               result, stride_result, item_ct1);
+            });
     });
 }
 
@@ -1026,10 +943,7 @@ void inverse_row_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                          ValueType *result, size_type stride_result)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+        cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              inverse_row_permute(num_rows, num_cols, perm_idxs,
                                                  orig, stride_orig, result,
@@ -1064,26 +978,21 @@ void inverse_column_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                             ValueType *result, size_type stride_result)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             inverse_column_permute(
-                                 num_rows, num_cols, perm_idxs, orig,
-                                 stride_orig, result, stride_result, item_ct1);
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                inverse_column_permute(num_rows, num_cols, perm_idxs, orig,
+                                       stride_orig, result, stride_result,
+                                       item_ct1);
+            });
     });
 }
 
-
 template <typename ValueType>
 void extract_diagonal(size_type problem_size,
                       const ValueType *__restrict__ orig, size_type stride_orig,
                       ValueType *__restrict__ diag, sycl::nd_item<3> item_ct1)
 {
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-
+    const auto tidx = thread::get_thread_id_flat<int>(item_ct1);
     if (tidx < problem_size) {
         diag[tidx] = orig[tidx * stride_orig + tidx];
     }
@@ -1096,10 +1005,7 @@ void extract_diagonal(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                       ValueType *diag)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+        cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              extract_diagonal(problem_size, orig, stride_orig,
                                               diag, item_ct1);
@@ -1117,7 +1023,7 @@ void inplace_absolute_dense(size_type num_rows, size_type num_cols,
     auto row = tidx / num_cols;
     auto col = tidx % num_cols;
     if (row < num_rows) {
-        data[row * stride + col] = dpcpp::abs(data[row * stride + col]);
+        data[row * stride + col] = std::abs(data[row * stride + col]);
     }
 }
 
@@ -1128,10 +1034,7 @@ void inplace_absolute_dense(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                             size_type stride)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+        cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              inplace_absolute_dense(num_rows, num_cols, data,
                                                     stride, item_ct1);
@@ -1151,7 +1054,7 @@ void outplace_absolute_dense(size_type num_rows, size_type num_cols,
     auto row = tidx / num_cols;
     auto col = tidx % num_cols;
     if (row < num_rows) {
-        out[row * stride_out + col] = dpcpp::abs(in[row * stride_in + col]);
+        out[row * stride_out + col] = std::abs(in[row * stride_in + col]);
     }
 }
 
@@ -1164,15 +1067,11 @@ void outplace_absolute_dense(dim3 grid, dim3 block,
                              size_type stride_out)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             outplace_absolute_dense(num_rows, num_cols, in,
-                                                     stride_in, out, stride_out,
-                                                     item_ct1);
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                outplace_absolute_dense(num_rows, num_cols, in, stride_in, out,
+                                        stride_out, item_ct1);
+            });
     });
 }
 
@@ -1198,10 +1097,7 @@ void make_complex(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                   size_type stride_out)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+        cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              make_complex(num_rows, num_cols, in, stride_in,
                                           out, stride_out, item_ct1);
@@ -1231,10 +1127,7 @@ void get_real(dim3 grid, dim3 block, size_t dynamic_shared_memory,
               remove_complex<ValueType> *out, size_type stride_out)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+        cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              get_real(num_rows, num_cols, in, stride_in, out,
                                       stride_out, item_ct1);
@@ -1264,10 +1157,7 @@ void get_imag(dim3 grid, dim3 block, size_t dynamic_shared_memory,
               remove_complex<ValueType> *out, size_type stride_out)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.reverse();
-        auto global_range = grid.reverse() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+        cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              get_imag(num_rows, num_cols, in, stride_in, out,
                                       stride_out, item_ct1);
@@ -1288,10 +1178,9 @@ void simple_apply(std::shared_ptr<const DpcppExecutor> exec,
     using namespace oneapi::mkl;
     oneapi::mkl::blas::row_major::gemm(
         *exec->get_queue(), transpose::nontrans, transpose::nontrans,
-        c->get_size()[0], c->get_size()[1], a->get_size()[1],
-        one<ValueType>(), a->get_const_values(), a->get_stride(),
-        b->get_const_values(), b->get_stride(), zero<ValueType>(),
-        c->get_values(), c->get_stride());
+        c->get_size()[0], c->get_size()[1], a->get_size()[1], one<ValueType>(),
+        a->get_const_values(), a->get_stride(), b->get_const_values(),
+        b->get_stride(), zero<ValueType>(), c->get_values(), c->get_stride());
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL);
@@ -1347,8 +1236,7 @@ void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
             kernel::compute_partial_dot<block_size>(
                 grid_dim, block_dim, 0, exec->get_queue(), x->get_size()[0],
                 x->get_const_values() + col, x->get_stride(),
-                y->get_const_values() + col, y->get_stride(),
-                work.get_data());
+                y->get_const_values() + col, y->get_stride(), work.get_data());
             kernel::finalize_dot_computation<block_size>(
                 1, block_dim, 0, exec->get_queue(), grid_dim.x,
                 work.get_const_data(), result->get_values() + col);
@@ -1397,8 +1285,7 @@ void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
         for (size_type col = 0; col < x->get_size()[1]; ++col) {
             kernel::compute_partial_norm2<block_size>(
                 grid_dim, block_dim, 0, exec->get_queue(), x->get_size()[0],
-                x->get_const_values() + col, x->get_stride(),
-                work.get_data());
+                x->get_const_values() + col, x->get_stride(), work.get_data());
             kernel::finalize_norm2_computation<block_size>(
                 1, block_dim, 0, exec->get_queue(), grid_dim.x,
                 work.get_const_data(), result->get_values() + col);
@@ -1433,8 +1320,7 @@ void convert_to_coo(std::shared_ptr<const DpcppExecutor> exec,
     kernel::fill_in_coo(grid_dim, default_block_size, 0, exec->get_queue(),
                         num_rows, num_cols, stride,
                         nnz_prefix_sum.get_const_data(),
-                        source->get_const_values(), row_idxs, col_idxs,
-                        values);
+                        source->get_const_values(), row_idxs, col_idxs, values);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -1455,9 +1341,8 @@ void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
 
     auto stride = source->get_stride();
 
-    const auto rows_per_block = ceildiv(default_block_size,
-    config::warp_size); const auto grid_dim_nnz =
-    ceildiv(source->get_size()[0], rows_per_block);
+    const auto rows_per_block = ceildiv(default_block_size, config::warp_size);
+    const auto grid_dim_nnz = ceildiv(source->get_size()[0], rows_per_block);
 
     kernel::count_nnz_per_row(grid_dim_nnz, default_block_size, 0,
                               exec->get_queue(), num_rows, num_cols, stride,
@@ -1468,9 +1353,8 @@ void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
     size_type grid_dim = ceildiv(num_rows, default_block_size);
 
     kernel::fill_in_csr(grid_dim, default_block_size, 0, exec->get_queue(),
-                        num_rows, num_cols, stride,
-                        source->get_const_values(), row_ptrs, col_idxs,
-                        values);
+                        num_rows, num_cols, stride, source->get_const_values(),
+                        row_ptrs, col_idxs, values);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -1546,8 +1430,8 @@ void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
         std::cout << "calculate_slice_lengths" << std::endl;
         kernel::calculate_slice_lengths(
             grid_dim, config::warp_size, 0, exec->get_queue(), num_rows,
-            slice_size, slice_num, stride_factor,
-            nnz_per_row.get_const_data(), slice_lengths, slice_sets);
+            slice_size, slice_num, stride_factor, nnz_per_row.get_const_data(),
+            slice_lengths, slice_sets);
         exec->synchronize();
         std::cout << "calculate_slice_lengths finish" << std::endl;
     }
@@ -1559,9 +1443,9 @@ void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
     if (grid_dim > 0) {
         std::cout << "fill_in_sellp" << std::endl;
         kernel::fill_in_sellp(grid_dim, default_block_size, 0,
-                              exec->get_queue(), num_rows, num_cols,
-                              slice_size, stride, source->get_const_values(),
-                              slice_lengths, slice_sets, col_idxs, vals);
+                              exec->get_queue(), num_rows, num_cols, slice_size,
+                              stride, source->get_const_values(), slice_lengths,
+                              slice_sets, col_idxs, vals);
         exec->synchronize();
         std::cout << "fill_in_sellp finish" << std::endl;
     }
@@ -1622,8 +1506,7 @@ void calculate_max_nnz_per_row(std::shared_ptr<const DpcppExecutor> exec,
     kernel::reduce_max_nnz(1, default_block_size,
                            default_block_size * sizeof(size_type),
                            exec->get_queue(), grid_dim,
-                           block_results.get_const_data(),
-                           d_result.get_data());
+                           block_results.get_const_data(), d_result.get_data());
 
     *result = exec->copy_val_to_host(d_result.get_const_data());
 }
@@ -1643,10 +1526,9 @@ void calculate_nonzeros_per_row(std::shared_ptr<const DpcppExecutor> exec,
     const dim3 grid_size(grid_x, 1, 1);
     if (grid_x > 0) {
         kernel::count_nnz_per_row(
-            grid_size, block_size, 0, exec->get_queue(),
-            source->get_size()[0], source->get_size()[1],
-            source->get_stride(), source->get_const_values(),
-            result->get_data());
+            grid_size, block_size, 0, exec->get_queue(), source->get_size()[0],
+            source->get_size()[1], source->get_stride(),
+            source->get_const_values(), result->get_data());
     }
 }
 
@@ -1676,8 +1558,7 @@ void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
 
     auto max_nnz_per_slice = Array<size_type>(exec, slice_num);
 
-    auto grid_dim = ceildiv(slice_num * config::warp_size,
-    default_block_size);
+    auto grid_dim = ceildiv(slice_num * config::warp_size, default_block_size);
 
     kernel::reduce_max_nnz_per_slice(
         grid_dim, default_block_size, 0, exec->get_queue(), num_rows,
diff --git a/dpcpp/test/matrix/dense_kernels.cpp b/dpcpp/test/matrix/dense_kernels.cpp
index 7c65e8b0f84..3cd080313cf 100644
--- a/dpcpp/test/matrix/dense_kernels.cpp
+++ b/dpcpp/test/matrix/dense_kernels.cpp
@@ -59,17 +59,21 @@ namespace {
 class Dense : public ::testing::Test {
 protected:
     using itype = int;
+#if GINKGO_DPCPP_SINGLE_MODE
+    using vtype = float;
+#else
     using vtype = double;
+#endif  // GINKGO_DPCPP_SINGLE_MODE
     using Mtx = gko::matrix::Dense<vtype>;
     using NormVector = gko::matrix::Dense<gko::remove_complex<vtype>>;
     using Arr = gko::Array<itype>;
-    using ComplexMtx = gko::matrix::Dense<std::complex<vtype>>;
+    // using ComplexMtx = gko::matrix::Dense<std::complex<vtype>>;
 
     Dense() : rand_engine(15) {}
 
     void SetUp()
     {
-        ASSERT_GT(gko::DpcppExecutor::get_num_devices("gpu"), 0);
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices("all"), 0);
         ref = gko::ReferenceExecutor::create();
         dpcpp = gko::DpcppExecutor::create(0, ref);
     }
@@ -113,16 +117,15 @@ class Dense : public ::testing::Test {
     void set_up_apply_data()
     {
         x = gen_mtx<Mtx>(65, 25);
-        c_x = gen_mtx<ComplexMtx>(65, 25);
+        // c_x = gen_mtx<ComplexMtx>(65, 25);
         y = gen_mtx<Mtx>(25, 35);
         expected = gen_mtx<Mtx>(65, 35);
         alpha = gko::initialize<Mtx>({2.0}, ref);
         beta = gko::initialize<Mtx>({-1.0}, ref);
-        square = gen_mtx<Mtx>(x->get_size()[0], x->get_size()[0]);
         dx = Mtx::create(dpcpp);
         dx->copy_from(x.get());
-        dc_x = ComplexMtx::create(dpcpp);
-        dc_x->copy_from(c_x.get());
+        // dc_x = ComplexMtx::create(dpcpp);
+        // dc_x->copy_from(c_x.get());
         dy = Mtx::create(dpcpp);
         dy->copy_from(y.get());
         dresult = Mtx::create(dpcpp);
@@ -131,8 +134,6 @@ class Dense : public ::testing::Test {
         dalpha->copy_from(alpha.get());
         dbeta = Mtx::create(dpcpp);
         dbeta->copy_from(beta.get());
-        dsquare = Mtx::create(dpcpp);
-        dsquare->copy_from(square.get());
 
         std::vector<itype> tmp(x->get_size()[0], 0);
         auto rng = std::default_random_engine{};
@@ -141,17 +142,14 @@ class Dense : public ::testing::Test {
         std::vector<itype> tmp2(x->get_size()[1], 0);
         std::iota(tmp2.begin(), tmp2.end(), 0);
         std::shuffle(tmp2.begin(), tmp2.end(), rng);
-        std::vector<itype> tmp3(x->get_size()[0] / 10);
-        std::uniform_int_distribution<itype> row_dist(0, x->get_size()[0] - 1);
-        for (auto &i : tmp3) {
-            i = row_dist(rng);
-        }
         rpermute_idxs =
             std::unique_ptr<Arr>(new Arr{ref, tmp.begin(), tmp.end()});
+        drpermute_idxs =
+            std::unique_ptr<Arr>(new Arr{dpcpp, tmp.begin(), tmp.end()});
         cpermute_idxs =
             std::unique_ptr<Arr>(new Arr{ref, tmp2.begin(), tmp2.end()});
-        rgather_idxs =
-            std::unique_ptr<Arr>(new Arr{ref, tmp3.begin(), tmp3.end()});
+        dcpermute_idxs =
+            std::unique_ptr<Arr>(new Arr{dpcpp, tmp2.begin(), tmp2.end()});
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -160,22 +158,21 @@ class Dense : public ::testing::Test {
     std::ranlux48 rand_engine;
 
     std::unique_ptr<Mtx> x;
-    std::unique_ptr<ComplexMtx> c_x;
+    // std::unique_ptr<ComplexMtx> c_x;
     std::unique_ptr<Mtx> y;
     std::unique_ptr<Mtx> alpha;
     std::unique_ptr<Mtx> beta;
     std::unique_ptr<Mtx> expected;
-    std::unique_ptr<Mtx> square;
     std::unique_ptr<Mtx> dresult;
     std::unique_ptr<Mtx> dx;
-    std::unique_ptr<ComplexMtx> dc_x;
+    // std::unique_ptr<ComplexMtx> dc_x;
     std::unique_ptr<Mtx> dy;
     std::unique_ptr<Mtx> dalpha;
     std::unique_ptr<Mtx> dbeta;
-    std::unique_ptr<Mtx> dsquare;
     std::unique_ptr<Arr> rpermute_idxs;
+    std::unique_ptr<Arr> drpermute_idxs;
     std::unique_ptr<Arr> cpermute_idxs;
-    std::unique_ptr<Arr> rgather_idxs;
+    std::unique_ptr<Arr> dcpermute_idxs;
 };
 
 
@@ -188,13 +185,13 @@ TEST_F(Dense, DpcppFillIsEquivalentToRef)
     dx->fill(42);
     result->copy_from(dx.get());
 
-    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(result, x, r<vtype>::value);
 }
 
 
 TEST_F(Dense, DpcppStridedFillIsEquivalentToRef)
 {
-    using T = double;
+    using T = vtype;
     auto x = gko::initialize<gko::matrix::Dense<T>>(
         4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, ref);
     auto dx = gko::initialize<gko::matrix::Dense<T>>(
@@ -205,7 +202,7 @@ TEST_F(Dense, DpcppStridedFillIsEquivalentToRef)
     dx->fill(42);
     result->copy_from(dx.get());
 
-    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(result, x, r<T>::value);
 }
 
 
@@ -218,7 +215,7 @@ TEST_F(Dense, SingleVectorDpcppScaleIsEquivalentToRef)
     dx->scale(dalpha.get());
     result->copy_from(dx.get());
 
-    GKO_ASSERT_MTX_NEAR(result, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(result, x, r<vtype>::value);
 }
 
 
@@ -229,7 +226,7 @@ TEST_F(Dense, MultipleVectorDpcppScaleIsEquivalentToRef)
     x->scale(alpha.get());
     dx->scale(dalpha.get());
 
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
 }
 
 
@@ -240,7 +237,7 @@ TEST_F(Dense, MultipleVectorDpcppScaleWithDifferentAlphaIsEquivalentToRef)
     x->scale(alpha.get());
     dx->scale(dalpha.get());
 
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
 }
 
 
@@ -251,7 +248,7 @@ TEST_F(Dense, SingleVectorDpcppAddScaledIsEquivalentToRef)
     x->add_scaled(alpha.get(), y.get());
     dx->add_scaled(dalpha.get(), dy.get());
 
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
 }
 
 
@@ -262,7 +259,7 @@ TEST_F(Dense, MultipleVectorDpcppAddScaledIsEquivalentToRef)
     x->add_scaled(alpha.get(), y.get());
     dx->add_scaled(dalpha.get(), dy.get());
 
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
 }
 
 
@@ -273,7 +270,7 @@ TEST_F(Dense, MultipleVectorDpcppAddScaledWithDifferentAlphaIsEquivalentToRef)
     x->add_scaled(alpha.get(), y.get());
     dx->add_scaled(dalpha.get(), dy.get());
 
-    GKO_ASSERT_MTX_NEAR(dx, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
 }
 
 
@@ -296,7 +293,7 @@ TEST_F(Dense, AddsScaledDiagIsEquivalentToRef)
     mat->add_scaled(alpha.get(), diag.get());
     dmat->add_scaled(dalpha.get(), ddiag.get());
 
-    GKO_ASSERT_MTX_NEAR(mat, dmat, 1e-14);
+    GKO_ASSERT_MTX_NEAR(mat, dmat, r<vtype>::value);
 }
 
 
@@ -307,7 +304,7 @@ TEST_F(Dense, SingleVectorDpcppComputeDotIsEquivalentToRef)
     x->compute_dot(y.get(), expected.get());
     dx->compute_dot(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -318,7 +315,7 @@ TEST_F(Dense, MultipleVectorDpcppComputeDotIsEquivalentToRef)
     x->compute_dot(y.get(), expected.get());
     dx->compute_dot(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -332,7 +329,7 @@ TEST_F(Dense, DpcppComputeNorm2IsEquivalentToRef)
     x->compute_norm2(norm_expected.get());
     dx->compute_norm2(dnorm.get());
 
-    GKO_ASSERT_MTX_NEAR(norm_expected, dnorm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(norm_expected, dnorm, r<vtype>::value);
 }
 
 
@@ -343,7 +340,7 @@ TEST_F(Dense, SimpleApplyIsEquivalentToRef)
     x->apply(y.get(), expected.get());
     dx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -354,186 +351,186 @@ TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
     x->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
-TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
-{
-    set_up_apply_data();
-    auto complex_b = gen_mtx<ComplexMtx>(25, 1);
-    auto dcomplex_b = ComplexMtx::create(dpcpp);
-    dcomplex_b->copy_from(complex_b.get());
-    auto complex_x = gen_mtx<ComplexMtx>(65, 1);
-    auto dcomplex_x = ComplexMtx::create(dpcpp);
-    dcomplex_x->copy_from(complex_x.get());
+// TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
+// {
+//     set_up_apply_data();
+//     auto complex_b = gen_mtx<ComplexMtx>(25, 1);
+//     auto dcomplex_b = ComplexMtx::create(dpcpp);
+//     dcomplex_b->copy_from(complex_b.get());
+//     auto complex_x = gen_mtx<ComplexMtx>(65, 1);
+//     auto dcomplex_x = ComplexMtx::create(dpcpp);
+//     dcomplex_x->copy_from(complex_x.get());
 
-    x->apply(complex_b.get(), complex_x.get());
-    dx->apply(dcomplex_b.get(), dcomplex_x.get());
+//     x->apply(complex_b.get(), complex_x.get());
+//     dx->apply(dcomplex_b.get(), dcomplex_x.get());
 
-    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
-}
+//     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+// }
 
 
-TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
-{
-    set_up_apply_data();
-    auto complex_b = gen_mtx<ComplexMtx>(25, 1);
-    auto dcomplex_b = ComplexMtx::create(dpcpp);
-    dcomplex_b->copy_from(complex_b.get());
-    auto complex_x = gen_mtx<ComplexMtx>(65, 1);
-    auto dcomplex_x = ComplexMtx::create(dpcpp);
-    dcomplex_x->copy_from(complex_x.get());
+// TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
+// {
+//     set_up_apply_data();
+//     auto complex_b = gen_mtx<ComplexMtx>(25, 1);
+//     auto dcomplex_b = ComplexMtx::create(dpcpp);
+//     dcomplex_b->copy_from(complex_b.get());
+//     auto complex_x = gen_mtx<ComplexMtx>(65, 1);
+//     auto dcomplex_x = ComplexMtx::create(dpcpp);
+//     dcomplex_x->copy_from(complex_x.get());
 
-    x->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
-    dx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+//     x->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+//     dx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
 
-    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
-}
+//     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+// }
 
 
-TEST_F(Dense, IsTransposable)
-{
-    set_up_apply_data();
+// TEST_F(Dense, IsTransposable)
+// {
+//     set_up_apply_data();
 
-    auto trans = x->transpose();
-    auto dtrans = dx->transpose();
+//     auto trans = x->transpose();
+//     auto dtrans = dx->transpose();
 
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(dtrans.get()),
-                        static_cast<Mtx *>(trans.get()), 0);
-}
+//     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(dtrans.get()),
+//                         static_cast<Mtx *>(trans.get()), 0);
+// }
 
 
-TEST_F(Dense, IsConjugateTransposable)
-{
-    set_up_apply_data();
+// TEST_F(Dense, IsConjugateTransposable)
+// {
+//     set_up_apply_data();
 
-    auto trans = c_x->conj_transpose();
-    auto dtrans = dc_x->conj_transpose();
+//     auto trans = c_x->conj_transpose();
+//     auto dtrans = dc_x->conj_transpose();
 
-    GKO_ASSERT_MTX_NEAR(static_cast<ComplexMtx *>(dtrans.get()),
-                        static_cast<ComplexMtx *>(trans.get()), 0);
-}
+//     GKO_ASSERT_MTX_NEAR(static_cast<ComplexMtx *>(dtrans.get()),
+//                         static_cast<ComplexMtx *>(trans.get()), 0);
+// }
 
 
 TEST_F(Dense, ConvertToCooIsEquivalentToRef)
 {
     set_up_apply_data();
-    auto coo_mtx = gko::matrix::Coo<>::create(ref);
-    auto dcoo_mtx = gko::matrix::Coo<>::create(dpcpp);
+    auto coo_mtx = gko::matrix::Coo<vtype>::create(ref);
+    auto dcoo_mtx = gko::matrix::Coo<vtype>::create(dpcpp);
 
     x->convert_to(coo_mtx.get());
     dx->convert_to(dcoo_mtx.get());
 
     ASSERT_EQ(dcoo_mtx->get_num_stored_elements(),
               coo_mtx->get_num_stored_elements());
-    GKO_ASSERT_MTX_NEAR(dcoo_mtx.get(), coo_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcoo_mtx.get(), coo_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Dense, MoveToCooIsEquivalentToRef)
 {
     set_up_apply_data();
-    auto coo_mtx = gko::matrix::Coo<>::create(ref);
-    auto dcoo_mtx = gko::matrix::Coo<>::create(dpcpp);
+    auto coo_mtx = gko::matrix::Coo<vtype>::create(ref);
+    auto dcoo_mtx = gko::matrix::Coo<vtype>::create(dpcpp);
 
     x->move_to(coo_mtx.get());
     dx->move_to(dcoo_mtx.get());
 
     ASSERT_EQ(dcoo_mtx->get_num_stored_elements(),
               coo_mtx->get_num_stored_elements());
-    GKO_ASSERT_MTX_NEAR(dcoo_mtx.get(), coo_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcoo_mtx.get(), coo_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Dense, ConvertToCsrIsEquivalentToRef)
 {
     set_up_apply_data();
-    auto csr_mtx = gko::matrix::Csr<>::create(ref);
-    auto dcsr_mtx = gko::matrix::Csr<>::create(dpcpp);
+    auto csr_mtx = gko::matrix::Csr<vtype>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<vtype>::create(dpcpp);
 
     x->convert_to(csr_mtx.get());
     dx->convert_to(dcsr_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(dcsr_mtx.get(), csr_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcsr_mtx.get(), csr_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Dense, MoveToCsrIsEquivalentToRef)
 {
     set_up_apply_data();
-    auto csr_mtx = gko::matrix::Csr<>::create(ref);
-    auto dcsr_mtx = gko::matrix::Csr<>::create(dpcpp);
+    auto csr_mtx = gko::matrix::Csr<vtype>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<vtype>::create(dpcpp);
 
     x->move_to(csr_mtx.get());
     dx->move_to(dcsr_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(dcsr_mtx.get(), csr_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcsr_mtx.get(), csr_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Dense, ConvertToEllIsEquivalentToRef)
 {
     set_up_apply_data();
-    auto ell_mtx = gko::matrix::Ell<>::create(ref);
-    auto dell_mtx = gko::matrix::Ell<>::create(dpcpp);
+    auto ell_mtx = gko::matrix::Ell<vtype>::create(ref);
+    auto dell_mtx = gko::matrix::Ell<vtype>::create(dpcpp);
 
     x->convert_to(ell_mtx.get());
     dx->convert_to(dell_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(dell_mtx.get(), ell_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(dell_mtx.get(), ell_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Dense, MoveToEllIsEquivalentToRef)
 {
     set_up_apply_data();
-    auto ell_mtx = gko::matrix::Ell<>::create(ref);
-    auto dell_mtx = gko::matrix::Ell<>::create(dpcpp);
+    auto ell_mtx = gko::matrix::Ell<vtype>::create(ref);
+    auto dell_mtx = gko::matrix::Ell<vtype>::create(dpcpp);
 
     x->move_to(ell_mtx.get());
     dx->move_to(dell_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(dell_mtx.get(), ell_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(dell_mtx.get(), ell_mtx.get(), r<vtype>::value);
 }
 
 
-TEST_F(Dense, ConvertToSellpIsEquivalentToRef)
-{
-    set_up_apply_data();
-    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
-    auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+// TEST_F(Dense, ConvertToSellpIsEquivalentToRef)
+// {
+//     set_up_apply_data();
+//     auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
+//     auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
 
-    x->convert_to(sellp_mtx.get());
-    dx->convert_to(dsellp_mtx.get());
+//     x->convert_to(sellp_mtx.get());
+//     dx->convert_to(dsellp_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(sellp_mtx, dsellp_mtx, 1e-14);
-}
+//     GKO_ASSERT_MTX_NEAR(sellp_mtx, dsellp_mtx, 1e-6);
+// }
 
 
-TEST_F(Dense, MoveToSellpIsEquivalentToRef)
-{
-    set_up_apply_data();
-    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
-    auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+// TEST_F(Dense, MoveToSellpIsEquivalentToRef)
+// {
+//     set_up_apply_data();
+//     auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
+//     auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
 
-    x->move_to(sellp_mtx.get());
-    dx->move_to(dsellp_mtx.get());
+//     x->move_to(sellp_mtx.get());
+//     dx->move_to(dsellp_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(sellp_mtx, dsellp_mtx, 1e-14);
-}
+//     GKO_ASSERT_MTX_NEAR(sellp_mtx, dsellp_mtx, 1e-6);
+// }
 
 
-TEST_F(Dense, ConvertsEmptyToSellp)
-{
-    auto dempty_mtx = Mtx::create(dpcpp);
-    auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+// TEST_F(Dense, ConvertsEmptyToSellp)
+// {
+//     auto dempty_mtx = Mtx::create(dpcpp);
+//     auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
 
-    dempty_mtx->convert_to(dsellp_mtx.get());
+//     dempty_mtx->convert_to(dsellp_mtx.get());
 
-    ASSERT_EQ(dpcpp->copy_val_to_host(dsellp_mtx->get_const_slice_sets()), 0);
-    ASSERT_FALSE(dsellp_mtx->get_size());
-}
+//     ASSERT_EQ(dpcpp->copy_val_to_host(dsellp_mtx->get_const_slice_sets()),
+//     0); ASSERT_FALSE(dsellp_mtx->get_size());
+// }
 
 
 TEST_F(Dense, CountNNZIsEquivalentToRef)
@@ -599,63 +596,12 @@ TEST_F(Dense, CalculateTotalColsIsEquivalentToRef)
 }
 
 
-TEST_F(Dense, CanGatherRows)
-{
-    set_up_apply_data();
-
-    auto r_gather = x->row_gather(rgather_idxs.get());
-    auto dr_gather = dx->row_gather(rgather_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
-}
-
-
-TEST_F(Dense, CanGatherRowsIntoDense)
-{
-    set_up_apply_data();
-    auto gather_size =
-        gko::dim<2>{rgather_idxs->get_num_elems(), x->get_size()[1]};
-    auto r_gather = Mtx::create(ref, gather_size);
-    // test make_temporary_clone and non-default stride
-    auto dr_gather = Mtx::create(ref, gather_size, x->get_size()[1] + 2);
-
-    x->row_gather(rgather_idxs.get(), r_gather.get());
-    dx->row_gather(rgather_idxs.get(), dr_gather.get());
-
-    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
-}
-
-
-TEST_F(Dense, IsPermutable)
-{
-    set_up_apply_data();
-
-    auto permuted = square->permute(rpermute_idxs.get());
-    auto dpermuted = dsquare->permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
-                        static_cast<Mtx *>(dpermuted.get()), 0);
-}
-
-
-TEST_F(Dense, IsInversePermutable)
-{
-    set_up_apply_data();
-
-    auto permuted = square->inverse_permute(rpermute_idxs.get());
-    auto dpermuted = dsquare->inverse_permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
-                        static_cast<Mtx *>(dpermuted.get()), 0);
-}
-
-
 TEST_F(Dense, IsRowPermutable)
 {
     set_up_apply_data();
 
     auto r_permute = x->row_permute(rpermute_idxs.get());
-    auto dr_permute = dx->row_permute(rpermute_idxs.get());
+    auto dr_permute = dx->row_permute(drpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(r_permute.get()),
                         static_cast<Mtx *>(dr_permute.get()), 0);
@@ -667,7 +613,7 @@ TEST_F(Dense, IsColPermutable)
     set_up_apply_data();
 
     auto c_permute = x->column_permute(cpermute_idxs.get());
-    auto dc_permute = dx->column_permute(cpermute_idxs.get());
+    auto dc_permute = dx->column_permute(dcpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(c_permute.get()),
                         static_cast<Mtx *>(dc_permute.get()), 0);
@@ -679,7 +625,7 @@ TEST_F(Dense, IsInverseRowPermutable)
     set_up_apply_data();
 
     auto inverse_r_permute = x->inverse_row_permute(rpermute_idxs.get());
-    auto d_inverse_r_permute = dx->inverse_row_permute(rpermute_idxs.get());
+    auto d_inverse_r_permute = dx->inverse_row_permute(drpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_r_permute.get()),
                         static_cast<Mtx *>(d_inverse_r_permute.get()), 0);
@@ -691,7 +637,7 @@ TEST_F(Dense, IsInverseColPermutable)
     set_up_apply_data();
 
     auto inverse_c_permute = x->inverse_column_permute(cpermute_idxs.get());
-    auto d_inverse_c_permute = dx->inverse_column_permute(cpermute_idxs.get());
+    auto d_inverse_c_permute = dx->inverse_column_permute(dcpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_c_permute.get()),
                         static_cast<Mtx *>(d_inverse_c_permute.get()), 0);
@@ -716,7 +662,7 @@ TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
     x->compute_absolute_inplace();
     dx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(x, dx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(x, dx, r<vtype>::value);
 }
 
 
@@ -727,79 +673,7 @@ TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_x = x->compute_absolute();
     auto dabs_x = dx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, 1e-14);
-}
-
-
-TEST_F(Dense, MakeComplexIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto complex_x = x->make_complex();
-    auto dcomplex_x = dx->make_complex();
-
-    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
-}
-
-
-TEST_F(Dense, MakeComplexWithGivenResultIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto complex_x = ComplexMtx::create(ref, x->get_size());
-    x->make_complex(complex_x.get());
-    auto dcomplex_x = ComplexMtx::create(dpcpp, x->get_size());
-    dx->make_complex(dcomplex_x.get());
-
-    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
-}
-
-
-TEST_F(Dense, GetRealIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto real_x = x->get_real();
-    auto dreal_x = dx->get_real();
-
-    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
-}
-
-
-TEST_F(Dense, GetRealWithGivenResultIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto real_x = Mtx::create(ref, x->get_size());
-    x->get_real(real_x.get());
-    auto dreal_x = Mtx::create(dpcpp, dx->get_size());
-    dx->get_real(dreal_x.get());
-
-    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
-}
-
-
-TEST_F(Dense, GetImagIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto imag_x = x->get_imag();
-    auto dimag_x = dx->get_imag();
-
-    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
-}
-
-
-TEST_F(Dense, GetImagWithGivenResultIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto imag_x = Mtx::create(ref, x->get_size());
-    x->get_imag(imag_x.get());
-    auto dimag_x = Mtx::create(dpcpp, dx->get_size());
-    dx->get_imag(dimag_x.get());
-
-    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
+    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, r<vtype>::value);
 }
 
 

From ebbf4a91521055a74f862374fc407918e10c4031 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 21 May 2021 14:31:26 +0200
Subject: [PATCH 0895/1008] use simple macro

---
 dpcpp/matrix/dense_kernels.dp.cpp | 319 +++---------------------------
 1 file changed, 24 insertions(+), 295 deletions(-)

diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 494caff94c2..58199a221d6 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -92,14 +92,12 @@ void strided_fill(size_type num_rows, size_type num_cols, size_type stride,
 GKO_ENABLE_DEFAULT_HOST(strided_fill, strided_fill)
 
 
-template <size_type block_size, typename ValueType>
+template <typename ValueType>
 void scale(size_type num_rows, size_type num_cols, size_type num_alpha_cols,
            const ValueType *__restrict__ alpha, ValueType *__restrict__ x,
            size_type stride_x, sycl::nd_item<3> item_ct1)
 {
-    constexpr auto warps_per_block = block_size / config::warp_size;
-    const auto global_id =
-        thread::get_thread_id<config::warp_size, warps_per_block>(item_ct1);
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
     const auto row_id = global_id / num_cols;
     const auto col_id = global_id % num_cols;
     const auto alpha_id = num_alpha_cols == 1 ? 0 : col_id;
@@ -111,32 +109,16 @@ void scale(size_type num_rows, size_type num_cols, size_type num_alpha_cols,
     }
 }
 
-template <size_type block_size, typename ValueType>
-void scale(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-           sycl::queue *stream, size_type num_rows, size_type num_cols,
-           size_type num_alpha_cols, const ValueType *alpha, ValueType *x,
-           size_type stride_x)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                scale<block_size>(num_rows, num_cols, num_alpha_cols, alpha, x,
-                                  stride_x, item_ct1);
-            });
-    });
-}
-
+GKO_ENABLE_DEFAULT_HOST(scale, scale)
 
-template <size_type block_size, typename ValueType>
+template <typename ValueType>
 void add_scaled(size_type num_rows, size_type num_cols,
                 size_type num_alpha_cols, const ValueType *__restrict__ alpha,
                 const ValueType *__restrict__ x, size_type stride_x,
                 ValueType *__restrict__ y, size_type stride_y,
                 sycl::nd_item<3> item_ct1)
 {
-    constexpr auto warps_per_block = block_size / config::warp_size;
-    const auto global_id =
-        thread::get_thread_id<config::warp_size, warps_per_block>(item_ct1);
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
     const auto row_id = global_id / num_cols;
     const auto col_id = global_id % num_cols;
     const auto alpha_id = num_alpha_cols == 1 ? 0 : col_id;
@@ -146,22 +128,7 @@ void add_scaled(size_type num_rows, size_type num_cols,
     }
 }
 
-template <size_type block_size, typename ValueType>
-void add_scaled(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                sycl::queue *stream, size_type num_rows, size_type num_cols,
-                size_type num_alpha_cols, const ValueType *alpha,
-                const ValueType *x, size_type stride_x, ValueType *y,
-                size_type stride_y)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                add_scaled<block_size>(num_rows, num_cols, num_alpha_cols,
-                                       alpha, x, stride_x, y, stride_y,
-                                       item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(add_scaled, add_scaled)
 
 
 template <typename ValueType>
@@ -179,19 +146,7 @@ void add_scaled_diag(size_type size, const ValueType *__restrict__ alpha,
     y[tidx * stride_y + tidx] += alpha[0] * diag[tidx];
 }
 
-template <typename ValueType>
-void add_scaled_diag(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                     sycl::queue *stream, size_type size,
-                     const ValueType *alpha, const ValueType *diag,
-                     ValueType *y, size_type stride_y)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                add_scaled_diag(size, alpha, diag, y, stride_y, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(add_scaled_diag, add_scaled_diag)
 
 
 template <size_type block_size, typename OutType, typename CallableGetValue,
@@ -282,7 +237,6 @@ void compute_partial_dot(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                        sycl::access::target::local>
             tmp_work_acc_ct1(cgh);
 
-
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 compute_partial_dot<block_size>(
@@ -318,7 +272,6 @@ void finalize_dot_computation(dim3 grid, dim3 block,
                        sycl::access::target::local>
             tmp_work_acc_ct1(cgh);
 
-
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              finalize_dot_computation<block_size>(
@@ -356,7 +309,6 @@ void compute_partial_norm2(dim3 grid, dim3 block, size_t dynamic_shared_memory,
             sycl::access::mode::read_write, sycl::access::target::local>
             tmp_work_acc_ct1(cgh);
 
-
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 compute_partial_norm2<block_size>(
@@ -427,21 +379,7 @@ void fill_in_coo(size_type num_rows, size_type num_cols, size_type stride,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void fill_in_coo(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                 sycl::queue *stream, size_type num_rows, size_type num_cols,
-                 size_type stride, const size_type *row_ptrs,
-                 const ValueType *source, IndexType *row_idxs,
-                 IndexType *col_idxs, ValueType *values)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                fill_in_coo(num_rows, num_cols, stride, row_ptrs, source,
-                            row_idxs, col_idxs, values, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(fill_in_coo, fill_in_coo)
 
 
 template <typename ValueType, typename IndexType>
@@ -505,20 +443,7 @@ void fill_in_csr(size_type num_rows, size_type num_cols, size_type stride,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void fill_in_csr(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                 sycl::queue *stream, size_type num_rows, size_type num_cols,
-                 size_type stride, const ValueType *source, IndexType *row_ptrs,
-                 IndexType *col_idxs, ValueType *values)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             fill_in_csr(num_rows, num_cols, stride, source,
-                                         row_ptrs, col_idxs, values, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(fill_in_csr, fill_in_csr)
 
 
 template <typename ValueType, typename IndexType>
@@ -551,22 +476,7 @@ void fill_in_ell(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void fill_in_ell(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                 sycl::queue *stream, size_type num_rows, size_type num_cols,
-                 size_type source_stride, const ValueType *source,
-                 size_type max_nnz_per_row, size_type result_stride,
-                 IndexType *col_ptrs, ValueType *values)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             fill_in_ell(num_rows, num_cols, source_stride,
-                                         source, max_nnz_per_row, result_stride,
-                                         col_ptrs, values, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(fill_in_ell, fill_in_ell)
 
 
 void calculate_slice_lengths(size_type num_rows, size_type slice_size,
@@ -655,22 +565,7 @@ void fill_in_sellp(size_type num_rows, size_type num_cols, size_type slice_size,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void fill_in_sellp(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                   sycl::queue *stream, size_type num_rows, size_type num_cols,
-                   size_type slice_size, size_type stride,
-                   const ValueType *source, size_type *slice_lengths,
-                   size_type *slice_sets, IndexType *col_idxs, ValueType *vals)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                fill_in_sellp(num_rows, num_cols, slice_size, stride, source,
-                              slice_lengths, slice_sets, col_idxs, vals,
-                              item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(fill_in_sellp, fill_in_sellp)
 
 
 void reduce_max_nnz(size_type size, const size_type *__restrict__ nnz_per_row,
@@ -777,7 +672,6 @@ void reduce_total_cols(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                        sycl::access::target::local>
             dpct_local_acc_ct1(sycl::range<1>(dynamic_shared_memory), cgh);
 
-
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 reduce_total_cols(num_slices, max_nnz_per_slice, result,
@@ -803,21 +697,7 @@ void symm_permute(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename IndexType, typename ValueType>
-void symm_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                  sycl::queue *stream, size_type num_rows, size_type num_cols,
-                  const IndexType *perm_idxs, const ValueType *orig,
-                  size_type stride_orig, ValueType *result,
-                  size_type stride_result)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                symm_permute(num_rows, num_cols, perm_idxs, orig, stride_orig,
-                             result, stride_result, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(symm_permute, symm_permute)
 
 
 template <typename IndexType, typename ValueType>
@@ -836,21 +716,7 @@ void inv_symm_permute(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename IndexType, typename ValueType>
-void inv_symm_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                      sycl::queue *stream, size_type num_rows,
-                      size_type num_cols, const IndexType *perm_idxs,
-                      const ValueType *orig, size_type stride_orig,
-                      ValueType *result, size_type stride_result)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                inv_symm_permute(num_rows, num_cols, perm_idxs, orig,
-                                 stride_orig, result, stride_result, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(inv_symm_permute, inv_symm_permute)
 
 
 template <typename IndexType, typename ValueType>
@@ -869,21 +735,7 @@ void row_gather(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename IndexType, typename ValueType>
-void row_gather(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                sycl::queue *stream, size_type num_rows, size_type num_cols,
-                const IndexType *perm_idxs, const ValueType *orig,
-                size_type stride_orig, ValueType *result,
-                size_type stride_result)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                row_gather(num_rows, num_cols, perm_idxs, orig, stride_orig,
-                           result, stride_result, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(row_gather, row_gather)
 
 
 template <typename IndexType, typename ValueType>
@@ -902,21 +754,7 @@ void column_permute(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename IndexType, typename ValueType>
-void column_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                    sycl::queue *stream, size_type num_rows, size_type num_cols,
-                    const IndexType *perm_idxs, const ValueType *orig,
-                    size_type stride_orig, ValueType *result,
-                    size_type stride_result)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                column_permute(num_rows, num_cols, perm_idxs, orig, stride_orig,
-                               result, stride_result, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(column_permute, column_permute)
 
 
 template <typename IndexType, typename ValueType>
@@ -935,22 +773,7 @@ void inverse_row_permute(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename IndexType, typename ValueType>
-void inverse_row_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                         sycl::queue *stream, size_type num_rows,
-                         size_type num_cols, const IndexType *perm_idxs,
-                         const ValueType *orig, size_type stride_orig,
-                         ValueType *result, size_type stride_result)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             inverse_row_permute(num_rows, num_cols, perm_idxs,
-                                                 orig, stride_orig, result,
-                                                 stride_result, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(inverse_row_permute, inverse_row_permute)
 
 
 template <typename IndexType, typename ValueType>
@@ -970,22 +793,8 @@ void inverse_column_permute(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename IndexType, typename ValueType>
-void inverse_column_permute(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                            sycl::queue *stream, size_type num_rows,
-                            size_type num_cols, const IndexType *perm_idxs,
-                            const ValueType *orig, size_type stride_orig,
-                            ValueType *result, size_type stride_result)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                inverse_column_permute(num_rows, num_cols, perm_idxs, orig,
-                                       stride_orig, result, stride_result,
-                                       item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(inverse_column_permute, inverse_column_permute)
+
 
 template <typename ValueType>
 void extract_diagonal(size_type problem_size,
@@ -998,20 +807,7 @@ void extract_diagonal(size_type problem_size,
     }
 }
 
-template <typename ValueType>
-void extract_diagonal(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                      sycl::queue *stream, size_type problem_size,
-                      const ValueType *orig, size_type stride_orig,
-                      ValueType *diag)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             extract_diagonal(problem_size, orig, stride_orig,
-                                              diag, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(extract_diagonal, extract_diagonal)
 
 
 template <typename ValueType>
@@ -1027,20 +823,7 @@ void inplace_absolute_dense(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename ValueType>
-void inplace_absolute_dense(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                            sycl::queue *stream, size_type num_rows,
-                            size_type num_cols, ValueType *data,
-                            size_type stride)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             inplace_absolute_dense(num_rows, num_cols, data,
-                                                    stride, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(inplace_absolute_dense, inplace_absolute_dense)
 
 
 template <typename ValueType>
@@ -1058,22 +841,7 @@ void outplace_absolute_dense(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename ValueType>
-void outplace_absolute_dense(dim3 grid, dim3 block,
-                             size_t dynamic_shared_memory, sycl::queue *stream,
-                             size_type num_rows, size_type num_cols,
-                             const ValueType *in, size_type stride_in,
-                             remove_complex<ValueType> *out,
-                             size_type stride_out)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                outplace_absolute_dense(num_rows, num_cols, in, stride_in, out,
-                                        stride_out, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(outplace_absolute_dense, outplace_absolute_dense)
 
 
 template <typename ValueType, typename ComplexType>
@@ -1090,20 +858,7 @@ void make_complex(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename ValueType, typename ComplexType>
-void make_complex(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                  sycl::queue *stream, size_type num_rows, size_type num_cols,
-                  const ValueType *in, size_type stride_in, ComplexType *out,
-                  size_type stride_out)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             make_complex(num_rows, num_cols, in, stride_in,
-                                          out, stride_out, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(make_complex, make_complex)
 
 
 template <typename ValueType>
@@ -1120,20 +875,7 @@ void get_real(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename ValueType>
-void get_real(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-              sycl::queue *stream, size_type num_rows, size_type num_cols,
-              const ValueType *in, size_type stride_in,
-              remove_complex<ValueType> *out, size_type stride_out)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             get_real(num_rows, num_cols, in, stride_in, out,
-                                      stride_out, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(get_real, get_real)
 
 
 template <typename ValueType>
@@ -1150,20 +892,7 @@ void get_imag(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename ValueType>
-void get_imag(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-              sycl::queue *stream, size_type num_rows, size_type num_cols,
-              const ValueType *in, size_type stride_in,
-              remove_complex<ValueType> *out, size_type stride_out)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             get_imag(num_rows, num_cols, in, stride_in, out,
-                                      stride_out, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(get_imag, get_imag)
 
 
 }  // namespace kernel

From 9d548f154fd680db7522ba23ca430c3e38fb64de Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Sun, 23 May 2021 00:08:32 +0200
Subject: [PATCH 0896/1008] add as_array and reduce_add_array for config

---
 dpcpp/components/reduction.dp.hpp             | 89 ++++++++++++-------
 .../ginkgo/core/synthesizer/containers.hpp    |  8 ++
 2 files changed, 63 insertions(+), 34 deletions(-)

diff --git a/dpcpp/components/reduction.dp.hpp b/dpcpp/components/reduction.dp.hpp
index e47d9038af3..4f1835dfea7 100644
--- a/dpcpp/components/reduction.dp.hpp
+++ b/dpcpp/components/reduction.dp.hpp
@@ -44,21 +44,29 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
+#include <ginkgo/core/synthesizer/containers.hpp>
+#include "core/synthesizer/implementation_selection.hpp"
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
 #include "dpcpp/base/dpct.hpp"
+#include "dpcpp/base/helper.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/thread_ids.dp.hpp"
 #include "dpcpp/components/uninitialized_array.hpp"
 
-
 namespace gko {
 namespace kernels {
 namespace dpcpp {
 
 
 constexpr int default_block_size = 256;
-
+using KCFG_1D = ConfigSet<11, 7>;
+constexpr auto kcfg_1d_list =
+    syn::value_list<ConfigSetType, KCFG_1D::encode(512, 64),
+                    KCFG_1D::encode(512, 32), KCFG_1D::encode(512, 16),
+                    KCFG_1D::encode(256, 32), KCFG_1D::encode(256, 16),
+                    KCFG_1D::encode(256, 8)>();
+constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
 
 // #include "common/components/reduction.hpp.inc"
 /**
@@ -130,21 +138,22 @@ __dpct_inline__ int choose_pivot(const Group &group, ValueType local_data,
  * array.
  */
 template <
-    typename Group, typename ValueType, typename Operator,
+    unsigned int sg_size = 32, typename Group, typename ValueType,
+    typename Operator,
     typename = std::enable_if_t<group::is_synchronizable_group<Group>::value>>
 void reduce(const Group &__restrict__ group, ValueType *__restrict__ data,
             Operator reduce_op = Operator{})
 {
     const auto local_id = group.thread_rank();
 
-    for (int k = group.size() / 2; k >= config::warp_size; k /= 2) {
+    for (int k = group.size() / 2; k >= sg_size; k /= 2) {
         group.sync();
         if (local_id < k) {
             data[local_id] = reduce_op(data[local_id], data[local_id + k]);
         }
     }
 
-    const auto warp = group::tiled_partition<config::warp_size>(group);
+    const auto warp = group::tiled_partition<sg_size>(group);
     const auto warp_id = group.thread_rank() / warp.size();
     if (warp_id > 0) {
         return;
@@ -164,7 +173,7 @@ void reduce(const Group &__restrict__ group, ValueType *__restrict__ data,
  * `source` of any size. Has to be called a second time on `result` to reduce
  * an array larger than `block_size`.
  */
-template <typename Operator, typename ValueType>
+template <unsigned int sg_size = 32, typename Operator, typename ValueType>
 void reduce_array(size_type size, const ValueType *__restrict__ source,
                   ValueType *__restrict__ result, sycl::nd_item<3> item_ct1,
                   Operator reduce_op = Operator{})
@@ -180,7 +189,7 @@ void reduce_array(size_type size, const ValueType *__restrict__ source,
     group::this_thread_block(item_ct1).sync();
 
     // Stores the result of the reduction inside `result[0]`
-    reduce(group::this_thread_block(item_ct1), result, reduce_op);
+    reduce<sg_size>(group::this_thread_block(item_ct1), result, reduce_op);
 }
 
 
@@ -189,47 +198,50 @@ void reduce_array(size_type size, const ValueType *__restrict__ source,
  *
  * Computes a reduction using the add operation (+) on an array
  * `source` of any size. Has to be called a second time on `result` to reduce
- * an array larger than `default_block_size`.
+ * an array larger than `block_size`.
  */
-template <typename ValueType>
+template <ConfigSetType cfg, typename ValueType>
 void reduce_add_array(
     size_type size, const ValueType *__restrict__ source,
     ValueType *__restrict__ result, sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, default_block_size> *block_sum)
+    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *block_sum)
 {
-    reduce_array(size, source, static_cast<ValueType *>((*block_sum)), item_ct1,
-                 [](const ValueType &x, const ValueType &y) { return x + y; });
+    reduce_array<KCFG_1D::decode<1>(cfg)>(
+        size, source, static_cast<ValueType *>((*block_sum)), item_ct1,
+        [](const ValueType &x, const ValueType &y) { return x + y; });
 
     if (item_ct1.get_local_id(2) == 0) {
         result[item_ct1.get_group(2)] = (*block_sum)[0];
     }
 }
 
-template <typename ValueType>
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void reduce_add_array(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                       sycl::queue *stream, size_type size,
                       const ValueType *source, ValueType *result)
 {
     stream->submit([&](sycl::handler &cgh) {
-        sycl::accessor<UninitializedArray<ValueType, default_block_size>, 0,
-                       sycl::access::mode::read_write,
+        sycl::accessor<UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)>,
+                       0, sycl::access::mode::read_write,
                        sycl::access::target::local>
             block_sum_acc_ct1(cgh);
 
-        auto local_range = block.get_range();
-        auto global_range = grid.get_range() * local_range;
-
         cgh.parallel_for(
-            sycl::nd_range<3>(global_range, local_range),
-            [=](sycl::nd_item<3> item_ct1) {
-                reduce_add_array(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                reduce_add_array<cfg>(
                     size, source, result, item_ct1,
-                    (UninitializedArray<ValueType, default_block_size> *)
+                    (UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *)
                         block_sum_acc_ct1.get_pointer());
             });
     });
 }
 
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(reduce_add_array_config,
+                                           reduce_add_array);
+
+GKO_ENABLE_DEFAULT_CONFIG_CALL(reduce_add_array_call, reduce_add_array_config,
+                               KCFG_1D, kcfg_1d_list);
+
 
 /**
  * Compute a reduction using add operation (+).
@@ -247,23 +259,32 @@ ValueType reduce_add_array(std::shared_ptr<const DpcppExecutor> exec,
     auto block_results_val = source;
     size_type grid_dim = size;
     auto block_results = Array<ValueType>(exec);
-    if (size > default_block_size) {
-        const auto n = ceildiv(size, default_block_size);
-        grid_dim = (n <= default_block_size) ? n : default_block_size;
+    ValueType answer = zero<ValueType>();
+    for (auto &cfg : kcfg_1d_array) {
+        const auto block_size = KCFG_1D::decode<0>(cfg);
+        const auto warp_size = KCFG_1D::decode<1>(cfg);
+        if (!validate(exec->get_queue(), block_size, warp_size)) {
+            continue;
+        }
+        if (size > block_size) {
+            const auto n = ceildiv(size, block_size);
+            grid_dim = (n <= block_size) ? n : block_size;
 
-        block_results.resize_and_reset(grid_dim);
+            block_results.resize_and_reset(grid_dim);
 
-        reduce_add_array(grid_dim, default_block_size, 0, exec->get_queue(),
-                         size, source, block_results.get_data());
+            reduce_add_array_call(grid_dim, block_size, 0, exec->get_queue(),
+                                  size, source, block_results.get_data());
 
-        block_results_val = block_results.get_const_data();
-    }
+            block_results_val = block_results.get_const_data();
+        }
 
-    auto d_result = Array<ValueType>(exec, 1);
+        auto d_result = Array<ValueType>(exec, 1);
 
-    reduce_add_array(1, default_block_size, 0, exec->get_queue(), grid_dim,
-                     block_results_val, d_result.get_data());
-    auto answer = exec->copy_val_to_host(d_result.get_const_data());
+        reduce_add_array_call(1, block_size, 0, exec->get_queue(), grid_dim,
+                              block_results_val, d_result.get_data());
+        answer = exec->copy_val_to_host(d_result.get_const_data());
+        break;
+    }
     return answer;
 }
 
diff --git a/include/ginkgo/core/synthesizer/containers.hpp b/include/ginkgo/core/synthesizer/containers.hpp
index ebd5b441f6c..3c79c7b7455 100644
--- a/include/ginkgo/core/synthesizer/containers.hpp
+++ b/include/ginkgo/core/synthesizer/containers.hpp
@@ -34,6 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_PUBLIC_CORE_SYNTHESIZER_CONTAINERS_HPP_
 
 
+#include <array>
 #include <type_traits>
 
 
@@ -113,6 +114,13 @@ template <typename T>
 using as_list = typename detail::as_list_impl<T>::type;
 
 
+template <typename T, T... Value>
+constexpr std::array<T, sizeof...(Value)> as_array(value_list<T, Value...> vl)
+{
+    return std::array<T, sizeof...(Value)>{Value...};
+}
+
+
 }  // namespace syn
 }  // namespace gko
 

From ac6258818de465e47cac55b6a40a7deb9c735c9b Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 25 May 2021 00:46:29 +0200
Subject: [PATCH 0897/1008] fix dpcpp doesn't have subwarp and use ConfigSet

---
 dpcpp/CMakeLists.txt               |   1 +
 dpcpp/base/helper.dp.cpp           |  31 ++
 dpcpp/base/helper.hpp              |  22 +-
 dpcpp/components/reduction.dp.hpp  |   1 +
 dpcpp/components/thread_ids.dp.hpp |   6 +-
 dpcpp/matrix/dense_kernels.dp.cpp  | 485 +++++++++++++++++------------
 6 files changed, 336 insertions(+), 210 deletions(-)
 create mode 100644 dpcpp/base/helper.dp.cpp

diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index b3101d8b2e2..48addebaf5f 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -11,6 +11,7 @@ target_sources(ginkgo_dpcpp
     PRIVATE
     base/version.dp.cpp
     base/executor.dp.cpp
+    base/helper.dp.cpp
     components/absolute_array.dp.cpp
     components/fill_array.dp.cpp
     components/prefix_sum.dp.cpp
diff --git a/dpcpp/base/helper.dp.cpp b/dpcpp/base/helper.dp.cpp
new file mode 100644
index 00000000000..fe4395e2534
--- /dev/null
+++ b/dpcpp/base/helper.dp.cpp
@@ -0,0 +1,31 @@
+#include <CL/sycl.hpp>
+
+#include "dpcpp/base/helper.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+bool validate(sycl::queue *queue, unsigned int workgroup_size,
+              unsigned int subgroup_size)
+{
+    {
+        auto device = queue->get_device();
+        auto subgroup_size_list =
+            device.get_info<cl::sycl::info::device::sub_group_sizes>();
+        auto max_workgroup_size =
+            device.get_info<sycl::info::device::max_work_group_size>();
+        bool allowed = false;
+        for (auto &i : subgroup_size_list) {
+            allowed |= (i == subgroup_size);
+        }
+        return allowed && (workgroup_size <= max_workgroup_size);
+    }
+}
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/base/helper.hpp b/dpcpp/base/helper.hpp
index f8eee93f25b..3979caa905c 100644
--- a/dpcpp/base/helper.hpp
+++ b/dpcpp/base/helper.hpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <CL/sycl.hpp>
 
 
+#include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/types.hpp>
 
 
@@ -137,18 +138,19 @@ namespace dpcpp {
 
 
 bool validate(sycl::queue *queue, unsigned workgroup_size,
-              unsigned subgroup_size)
+              unsigned subgroup_size);
+
+
+template <typename IterArr, typename Validate>
+ConfigSetType get_first_cfg(IterArr &arr, Validate verify)
 {
-    auto device = queue->get_device();
-    auto subgroup_size_list =
-        device.get_info<cl::sycl::info::device::sub_group_sizes>();
-    auto max_workgroup_size =
-        device.get_info<sycl::info::device::max_work_group_size>();
-    bool allowed = false;
-    for (auto &i : subgroup_size_list) {
-        allowed |= (i == subgroup_size);
+    for (auto &cfg : arr) {
+        if (verify(cfg)) {
+            return cfg;
+        }
     }
-    return allowed && (workgroup_size <= max_workgroup_size);
+    GKO_NOT_SUPPORTED(arr);
+    return 0;
 }
 
 
diff --git a/dpcpp/components/reduction.dp.hpp b/dpcpp/components/reduction.dp.hpp
index 4f1835dfea7..bc9937ddb1a 100644
--- a/dpcpp/components/reduction.dp.hpp
+++ b/dpcpp/components/reduction.dp.hpp
@@ -54,6 +54,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "dpcpp/components/thread_ids.dp.hpp"
 #include "dpcpp/components/uninitialized_array.hpp"
 
+
 namespace gko {
 namespace kernels {
 namespace dpcpp {
diff --git a/dpcpp/components/thread_ids.dp.hpp b/dpcpp/components/thread_ids.dp.hpp
index 8694d6a88c9..5b656c5e0db 100644
--- a/dpcpp/components/thread_ids.dp.hpp
+++ b/dpcpp/components/thread_ids.dp.hpp
@@ -124,7 +124,8 @@ __dpct_inline__ size_type get_local_warp_id(sycl::nd_item<3> item_ct1)
 template <int subwarp_size>
 __dpct_inline__ size_type get_local_subwarp_id(sycl::nd_item<3> item_ct1)
 {
-    constexpr auto subwarps_per_warp = config::warp_size / subwarp_size;
+    // dpcpp does not have subwarp.
+    constexpr auto subwarps_per_warp = subwarp_size / subwarp_size;
     return get_local_warp_id(item_ct1) * subwarps_per_warp +
            item_ct1.get_local_id(1);
 }
@@ -195,7 +196,8 @@ __dpct_inline__ size_type get_warp_id(sycl::nd_item<3> item_ct1)
 template <int subwarp_size, int warps_per_block>
 __dpct_inline__ size_type get_subwarp_id(sycl::nd_item<3> item_ct1)
 {
-    constexpr auto subwarps_per_warp = config::warp_size / subwarp_size;
+    // dpcpp dose not have subwarp
+    constexpr auto subwarps_per_warp = subwarp_size / subwarp_size;
     return get_warp_id<warps_per_block>(item_ct1) * subwarps_per_warp +
            item_ct1.get_local_id(1);
 }
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 58199a221d6..0d1be735e17 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -68,7 +68,13 @@ namespace dpcpp {
  */
 namespace dense {
 
-
+using KCFG_1D = ConfigSet<11, 7>;
+constexpr auto kcfg_1d_list =
+    syn::value_list<ConfigSetType, KCFG_1D::encode(512, 64),
+                    KCFG_1D::encode(512, 32), KCFG_1D::encode(512, 16),
+                    KCFG_1D::encode(256, 32), KCFG_1D::encode(256, 16),
+                    KCFG_1D::encode(256, 8)>();
+constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
 constexpr auto default_block_size = 256;
 
 
@@ -149,58 +155,67 @@ void add_scaled_diag(size_type size, const ValueType *__restrict__ alpha,
 GKO_ENABLE_DEFAULT_HOST(add_scaled_diag, add_scaled_diag)
 
 
-template <size_type block_size, typename OutType, typename CallableGetValue,
-          typename CallableReduce>
-void compute_partial_reduce(size_type num_rows, OutType *__restrict__ work,
-                            CallableGetValue get_value,
-                            CallableReduce reduce_op, sycl::nd_item<3> item_ct1,
-                            UninitializedArray<OutType, block_size> *tmp_work)
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename OutType,
+          typename CallableGetValue, typename CallableReduce>
+void compute_partial_reduce(
+    size_type num_rows, OutType *__restrict__ work, CallableGetValue get_value,
+    CallableReduce reduce_op, sycl::nd_item<3> item_ct1,
+    UninitializedArray<OutType, KCFG_1D::decode<0>(cfg)> *tmp_work)
 {
-    constexpr auto warps_per_block = block_size / config::warp_size;
+    constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
+    constexpr auto sg_size = KCFG_1D::decode<1>(cfg);
+
+    constexpr auto warps_per_block = wg_size / sg_size;
 
     const auto num_blocks = item_ct1.get_group_range(2);
-    const auto local_id =
-        thread::get_local_thread_id<config::warp_size>(item_ct1);
+    const auto local_id = thread::get_local_thread_id<sg_size>(item_ct1);
     const auto global_id =
-        thread::get_thread_id<config::warp_size, warps_per_block>(item_ct1);
+        thread::get_thread_id<sg_size, warps_per_block>(item_ct1);
 
     OutType *tmp_work_array = *tmp_work;
     auto tmp = zero<OutType>();
-    for (auto i = global_id; i < num_rows; i += block_size * num_blocks) {
+    for (auto i = global_id; i < num_rows; i += wg_size * num_blocks) {
         tmp = reduce_op(tmp, get_value(i));
     }
 
     tmp_work_array[local_id] = tmp;
 
-    ::gko::kernels::dpcpp::reduce(group::this_thread_block(item_ct1),
-                                  tmp_work_array, reduce_op);
+    ::gko::kernels::dpcpp::reduce<sg_size>(group::this_thread_block(item_ct1),
+                                           tmp_work_array, reduce_op);
 
     if (local_id == 0) {
         work[thread::get_block_id(item_ct1)] = tmp_work_array[0];
     }
 }
 
+// GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(compute_partial_reduce_config,
+//                                            compute_partial_reduce);
+// GKO_ENABLE_DEFAULT_CONFIG_CALL(compute_partial_reduce_call,
+// compute_partial_reduce_config,
+//                                KCFG_1D, kcfg_1d_list);
 
-template <size_type block_size, typename ValueType, typename CallableReduce,
-          typename CallableFinalize>
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType,
+          typename CallableReduce, typename CallableFinalize>
 void finalize_reduce_computation(
     size_type size, const ValueType *work, ValueType *result,
     CallableReduce reduce_op, CallableFinalize finalize_op,
     sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, block_size> *tmp_work)
+    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *tmp_work)
 {
-    const auto local_id =
-        thread::get_local_thread_id<config::warp_size>(item_ct1);
+    constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
+    constexpr auto sg_size = KCFG_1D::decode<1>(cfg);
+
+    const auto local_id = thread::get_local_thread_id<sg_size>(item_ct1);
 
     ValueType tmp = zero<ValueType>();
-    for (auto i = local_id; i < size; i += block_size) {
+    for (auto i = local_id; i < size; i += wg_size) {
         tmp = reduce_op(tmp, work[i]);
     }
     ValueType *tmp_work_array = *tmp_work;
     tmp_work_array[local_id] = tmp;
 
-    ::gko::kernels::dpcpp::reduce(group::this_thread_block(item_ct1),
-                                  tmp_work_array, reduce_op);
+    ::gko::kernels::dpcpp::reduce<sg_size>(group::this_thread_block(item_ct1),
+                                           tmp_work_array, reduce_op);
 
     if (local_id == 0) {
         *result = finalize_op(tmp_work_array[0]);
@@ -208,14 +223,14 @@ void finalize_reduce_computation(
 }
 
 
-template <size_type block_size, typename ValueType>
-void compute_partial_dot(size_type num_rows, const ValueType *__restrict__ x,
-                         size_type stride_x, const ValueType *__restrict__ y,
-                         size_type stride_y, ValueType *__restrict__ work,
-                         sycl::nd_item<3> item_ct1,
-                         UninitializedArray<ValueType, block_size> *tmp_work)
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
+void compute_partial_dot(
+    size_type num_rows, const ValueType *__restrict__ x, size_type stride_x,
+    const ValueType *__restrict__ y, size_type stride_y,
+    ValueType *__restrict__ work, sycl::nd_item<3> item_ct1,
+    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *tmp_work)
 {
-    compute_partial_reduce<block_size>(
+    compute_partial_reduce<cfg>(
         num_rows, work,
         [x, stride_x, y, stride_y](size_type i) {
             return x[i * stride_x] * conj(y[i * stride_y]);
@@ -224,122 +239,175 @@ void compute_partial_dot(size_type num_rows, const ValueType *__restrict__ x,
         tmp_work);
 }
 
-template <size_type block_size, typename ValueType>
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void compute_partial_dot(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                          sycl::queue *stream, size_type num_rows,
                          const ValueType *x, size_type stride_x,
                          const ValueType *y, size_type stride_y,
                          ValueType *work)
 {
+    constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
+    std::cout << "partial " << cfg << std::endl;
     stream->submit([&](sycl::handler &cgh) {
-        sycl::accessor<UninitializedArray<ValueType, block_size>, 0,
+        sycl::accessor<UninitializedArray<ValueType, wg_size>, 0,
                        sycl::access::mode::read_write,
                        sycl::access::target::local>
             tmp_work_acc_ct1(cgh);
 
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                compute_partial_dot<block_size>(
+                compute_partial_dot<cfg>(
                     num_rows, x, stride_x, y, stride_y, work, item_ct1,
-                    (UninitializedArray<ValueType, block_size> *)
+                    (UninitializedArray<ValueType, wg_size> *)
                         tmp_work_acc_ct1.get_pointer());
             });
     });
 }
 
 
-template <size_type block_size, typename ValueType>
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void finalize_dot_computation(
     size_type size, const ValueType *work, ValueType *result,
     sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, block_size> *tmp_work)
+    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *tmp_work)
 {
-    finalize_reduce_computation<block_size>(
+    finalize_reduce_computation<cfg>(
         size, work, result,
         [](const ValueType &x, const ValueType &y) { return x + y; },
         [](const ValueType &x) { return x; }, item_ct1, tmp_work);
 }
 
-template <size_type block_size, typename ValueType>
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void finalize_dot_computation(dim3 grid, dim3 block,
                               size_t dynamic_shared_memory, sycl::queue *stream,
                               size_type size, const ValueType *work,
                               ValueType *result)
 {
+    constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
+    std::cout << "finalize " << cfg << std::endl;
     stream->submit([&](sycl::handler &cgh) {
-        sycl::accessor<UninitializedArray<ValueType, block_size>, 0,
+        sycl::accessor<UninitializedArray<ValueType, wg_size>, 0,
                        sycl::access::mode::read_write,
                        sycl::access::target::local>
             tmp_work_acc_ct1(cgh);
 
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
-                             finalize_dot_computation<block_size>(
+                             finalize_dot_computation<cfg>(
                                  size, work, result, item_ct1,
-                                 (UninitializedArray<ValueType, block_size> *)
+                                 (UninitializedArray<ValueType, wg_size> *)
                                      tmp_work_acc_ct1.get_pointer());
                          });
     });
 }
 
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
+void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
+                 const matrix::Dense<ValueType> *x,
+                 const matrix::Dense<ValueType> *y,
+                 matrix::Dense<ValueType> *result)
+{
+    constexpr auto work_per_thread = 32;
+    constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
+    constexpr auto sg_size = KCFG_1D::decode<1>(cfg);
+    std::cout << "dot " << cfg << " " << wg_size << " " << sg_size << std::endl;
+    constexpr auto work_per_block = work_per_thread * wg_size;
+    const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
+    const dim3 block_dim{sg_size, 1, wg_size / sg_size};
+    Array<ValueType> work(exec, grid_dim.x);
+    // TODO: write a kernel which does this more efficiently
+    for (size_type col = 0; col < x->get_size()[1]; ++col) {
+        compute_partial_dot<cfg>(grid_dim, block_dim, 0, exec->get_queue(),
+                                 x->get_size()[0], x->get_const_values() + col,
+                                 x->get_stride(), y->get_const_values() + col,
+                                 y->get_stride(), work.get_data());
+        finalize_dot_computation<cfg>(1, block_dim, 0, exec->get_queue(),
+                                      grid_dim.x, work.get_const_data(),
+                                      result->get_values() + col);
+    }
+}
 
-template <size_type block_size, typename ValueType>
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(compute_dot_config, compute_dot)
+
+template <typename ValueType>
+void compute_dot_call(std::shared_ptr<const DpcppExecutor> exec,
+                      const matrix::Dense<ValueType> *x,
+                      const matrix::Dense<ValueType> *y,
+                      matrix::Dense<ValueType> *result)
+{
+    auto queue = exec->get_queue();
+    compute_dot_config(
+        kcfg_1d_list,
+        [&queue](::gko::ConfigSetType cfg) {
+            return validate(queue, KCFG_1D::decode<0>(cfg),
+                            KCFG_1D::decode<1>(cfg));
+        },
+        ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
+        ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
+        exec, x, y, result);
+}
+
+
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void compute_partial_norm2(
     size_type num_rows, const ValueType *__restrict__ x, size_type stride_x,
     remove_complex<ValueType> *__restrict__ work, sycl::nd_item<3> item_ct1,
-    UninitializedArray<remove_complex<ValueType>, block_size> *tmp_work)
+    UninitializedArray<remove_complex<ValueType>, KCFG_1D::decode<0>(cfg)>
+        *tmp_work)
 {
     using norm_type = remove_complex<ValueType>;
-    compute_partial_reduce<block_size>(
+    compute_partial_reduce<cfg>(
         num_rows, work,
         [x, stride_x](size_type i) { return squared_norm(x[i * stride_x]); },
         [](const norm_type &x, const norm_type &y) { return x + y; }, item_ct1,
         tmp_work);
 }
 
-template <size_type block_size, typename ValueType>
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void compute_partial_norm2(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                            sycl::queue *stream, size_type num_rows,
                            const ValueType *x, size_type stride_x,
                            remove_complex<ValueType> *work)
 {
+    constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
     stream->submit([&](sycl::handler &cgh) {
-        sycl::accessor<
-            UninitializedArray<remove_complex<ValueType>, block_size>, 0,
-            sycl::access::mode::read_write, sycl::access::target::local>
+        sycl::accessor<UninitializedArray<remove_complex<ValueType>, wg_size>,
+                       0, sycl::access::mode::read_write,
+                       sycl::access::target::local>
             tmp_work_acc_ct1(cgh);
 
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                compute_partial_norm2<block_size>(
+                compute_partial_norm2<cfg>(
                     num_rows, x, stride_x, work, item_ct1,
-                    (UninitializedArray<remove_complex<ValueType>, block_size>
-                         *)tmp_work_acc_ct1.get_pointer());
+                    (UninitializedArray<remove_complex<ValueType>, wg_size> *)
+                        tmp_work_acc_ct1.get_pointer());
             });
     });
 }
 
 
-template <size_type block_size, typename ValueType>
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void finalize_norm2_computation(
     size_type size, const ValueType *work, ValueType *result,
     sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, block_size> *tmp_work)
+    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *tmp_work)
 {
-    finalize_reduce_computation<block_size>(
+    finalize_reduce_computation<cfg>(
         size, work, result,
         [](const ValueType &x, const ValueType &y) { return x + y; },
         [](const ValueType &x) { return sqrt(x); }, item_ct1, tmp_work);
 }
 
-template <size_type block_size, typename ValueType>
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void finalize_norm2_computation(dim3 grid, dim3 block,
                                 size_t dynamic_shared_memory,
                                 sycl::queue *stream, size_type size,
                                 const ValueType *work, ValueType *result)
 {
+    constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
     stream->submit([&](sycl::handler &cgh) {
-        sycl::accessor<UninitializedArray<ValueType, block_size>, 0,
+        sycl::accessor<UninitializedArray<ValueType, wg_size>, 0,
                        sycl::access::mode::read_write,
                        sycl::access::target::local>
             tmp_work_acc_ct1(cgh);
@@ -347,15 +415,63 @@ void finalize_norm2_computation(dim3 grid, dim3 block,
 
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
-                             finalize_norm2_computation<block_size>(
+                             finalize_norm2_computation<cfg>(
                                  size, work, result, item_ct1,
-                                 (UninitializedArray<ValueType, block_size> *)
+                                 (UninitializedArray<ValueType, wg_size> *)
                                      tmp_work_acc_ct1.get_pointer());
                          });
     });
 }
 
 
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
+void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
+                   const matrix::Dense<ValueType> *x,
+                   matrix::Dense<remove_complex<ValueType>> *result)
+{
+    using norm_type = remove_complex<ValueType>;
+    // // TODO: these are tuning parameters obtained experimentally, once
+    // // we decide how to handle this uniformly, they should be modified
+    // // appropriately
+    constexpr auto work_per_thread = 32;
+    constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
+    constexpr auto sg_size = KCFG_1D::decode<1>(cfg);
+
+    constexpr auto work_per_block = work_per_thread * wg_size;
+    const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
+    const dim3 block_dim{sg_size, 1, wg_size / sg_size};
+    Array<norm_type> work(exec, grid_dim.x);
+    // TODO: write a kernel which does this more efficiently
+    for (size_type col = 0; col < x->get_size()[1]; ++col) {
+        compute_partial_norm2<cfg>(
+            grid_dim, block_dim, 0, exec->get_queue(), x->get_size()[0],
+            x->get_const_values() + col, x->get_stride(), work.get_data());
+        finalize_norm2_computation<cfg>(1, block_dim, 0, exec->get_queue(),
+                                        grid_dim.x, work.get_const_data(),
+                                        result->get_values() + col);
+    }
+}
+
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(compute_norm2_config, compute_norm2)
+
+template <typename ValueType>
+void compute_norm2_call(std::shared_ptr<const DpcppExecutor> exec,
+                        const matrix::Dense<ValueType> *x,
+                        matrix::Dense<remove_complex<ValueType>> *result)
+{
+    auto queue = exec->get_queue();
+    compute_norm2_config(
+        kcfg_1d_list,
+        [&queue](::gko::ConfigSetType cfg) {
+            return validate(queue, KCFG_1D::decode<0>(cfg),
+                            KCFG_1D::decode<1>(cfg));
+        },
+        ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
+        ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
+        exec, x, result);
+}
+
+
 template <typename ValueType, typename IndexType>
 void fill_in_coo(size_type num_rows, size_type num_cols, size_type stride,
                  const size_type *__restrict__ row_ptrs,
@@ -382,20 +498,20 @@ void fill_in_coo(size_type num_rows, size_type num_cols, size_type stride,
 GKO_ENABLE_DEFAULT_HOST(fill_in_coo, fill_in_coo)
 
 
-template <typename ValueType, typename IndexType>
+template <ConfigSetType cfg, typename ValueType, typename IndexType>
 void count_nnz_per_row(size_type num_rows, size_type num_cols, size_type stride,
                        const ValueType *__restrict__ work,
                        IndexType *__restrict__ result,
                        sycl::nd_item<3> item_ct1)
 {
-    constexpr auto warp_size = config::warp_size;
-    const auto row_idx = thread::get_subwarp_id_flat<warp_size>(item_ct1);
+    constexpr auto sg_size = KCFG_1D::decode<1>(cfg);
+    const auto row_idx = thread::get_subwarp_id_flat<sg_size>(item_ct1);
     auto warp_tile =
-        group::tiled_partition<warp_size>(group::this_thread_block(item_ct1));
+        group::tiled_partition<sg_size>(group::this_thread_block(item_ct1));
 
     if (row_idx < num_rows) {
         IndexType part_result{};
-        for (auto i = warp_tile.thread_rank(); i < num_cols; i += warp_size) {
+        for (auto i = warp_tile.thread_rank(); i < num_cols; i += sg_size) {
             if (work[stride * row_idx + i] != zero<ValueType>()) {
                 part_result += 1;
             }
@@ -406,20 +522,10 @@ void count_nnz_per_row(size_type num_rows, size_type num_cols, size_type stride,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void count_nnz_per_row(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                       sycl::queue *stream, size_type num_rows,
-                       size_type num_cols, size_type stride,
-                       const ValueType *work, IndexType *result)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             count_nnz_per_row(num_rows, num_cols, stride, work,
-                                               result, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST_CONFIG(count_nnz_per_row, count_nnz_per_row)
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(count_nnz_per_row, count_nnz_per_row)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(count_nnz_per_row_call, count_nnz_per_row,
+                               KCFG_1D, kcfg_1d_list)
 
 
 template <typename ValueType, typename IndexType>
@@ -479,6 +585,7 @@ void fill_in_ell(size_type num_rows, size_type num_cols,
 GKO_ENABLE_DEFAULT_HOST(fill_in_ell, fill_in_ell)
 
 
+template <ConfigSetType cfg>
 void calculate_slice_lengths(size_type num_rows, size_type slice_size,
                              int slice_num, size_type stride_factor,
                              const size_type *__restrict__ nnz_per_row,
@@ -486,21 +593,21 @@ void calculate_slice_lengths(size_type num_rows, size_type slice_size,
                              size_type *__restrict__ slice_sets,
                              sycl::nd_item<3> item_ct1)
 {
-    constexpr auto warp_size = config::warp_size;
+    constexpr auto sg_size = KCFG_1D::decode<1>(cfg);
     const auto sliceid = item_ct1.get_group(2);
     const auto tid_in_warp = item_ct1.get_local_id(2);
 
     if (sliceid * slice_size + tid_in_warp < num_rows) {
         size_type thread_result = 0;
-        for (size_type i = tid_in_warp; i < slice_size; i += warp_size) {
+        for (size_type i = tid_in_warp; i < slice_size; i += sg_size) {
             thread_result =
                 (i + slice_size * sliceid < num_rows)
                     ? max(thread_result, nnz_per_row[sliceid * slice_size + i])
                     : thread_result;
         }
 
-        auto warp_tile = group::tiled_partition<warp_size>(
-            group::this_thread_block(item_ct1));
+        auto warp_tile =
+            group::tiled_partition<sg_size>(group::this_thread_block(item_ct1));
         auto warp_result = ::gko::kernels::dpcpp::reduce(
             warp_tile, thread_result,
             [](const size_type &a, const size_type &b) { return max(a, b); });
@@ -514,22 +621,11 @@ void calculate_slice_lengths(size_type num_rows, size_type slice_size,
     }
 }
 
-void calculate_slice_lengths(dim3 grid, dim3 block,
-                             size_t dynamic_shared_memory, sycl::queue *stream,
-                             size_type num_rows, size_type slice_size,
-                             int slice_num, size_type stride_factor,
-                             const size_type *nnz_per_row,
-                             size_type *slice_lengths, size_type *slice_sets)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                calculate_slice_lengths(num_rows, slice_size, slice_num,
-                                        stride_factor, nnz_per_row,
-                                        slice_lengths, slice_sets, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST_CONFIG(calculate_slice_lengths, calculate_slice_lengths)
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(calculate_slice_lengths,
+                                           calculate_slice_lengths)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(calculate_slice_lengths_call,
+                               calculate_slice_lengths, KCFG_1D, kcfg_1d_list)
 
 
 template <typename ValueType, typename IndexType>
@@ -567,14 +663,15 @@ void fill_in_sellp(size_type num_rows, size_type num_cols, size_type slice_size,
 
 GKO_ENABLE_DEFAULT_HOST(fill_in_sellp, fill_in_sellp)
 
-
+template <ConfigSetType cfg>
 void reduce_max_nnz(size_type size, const size_type *__restrict__ nnz_per_row,
                     size_type *__restrict__ result, sycl::nd_item<3> item_ct1,
                     uint8_t *dpct_local)
 {
+    constexpr auto sg_size = KCFG_1D::decode<1>(cfg);
     auto block_max = (size_type *)dpct_local;
 
-    reduce_array(
+    reduce_array<sg_size>(
         size, nnz_per_row, block_max, item_ct1,
         [](const size_type &x, const size_type &y) { return max(x, y); });
 
@@ -583,6 +680,7 @@ void reduce_max_nnz(size_type size, const size_type *__restrict__ nnz_per_row,
     }
 }
 
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32)>
 void reduce_max_nnz(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                     sycl::queue *stream, size_type size,
                     const size_type *nnz_per_row, size_type *result)
@@ -593,30 +691,34 @@ void reduce_max_nnz(dim3 grid, dim3 block, size_t dynamic_shared_memory,
             dpct_local_acc_ct1(sycl::range<1>(dynamic_shared_memory), cgh);
 
 
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             reduce_max_nnz(size, nnz_per_row, result, item_ct1,
-                                            dpct_local_acc_ct1.get_pointer());
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                reduce_max_nnz<cfg>(size, nnz_per_row, result, item_ct1,
+                                    dpct_local_acc_ct1.get_pointer());
+            });
     });
 }
 
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(reduce_max_nnz, reduce_max_nnz);
+GKO_ENABLE_DEFAULT_CONFIG_CALL(reduce_max_nnz_call, reduce_max_nnz, KCFG_1D,
+                               kcfg_1d_list)
 
+template <ConfigSetType cfg>
 void reduce_max_nnz_per_slice(size_type num_rows, size_type slice_size,
                               size_type stride_factor,
                               const size_type *__restrict__ nnz_per_row,
                               size_type *__restrict__ result,
                               sycl::nd_item<3> item_ct1)
 {
-    constexpr auto warp_size = config::warp_size;
+    constexpr auto sg_size = KCFG_1D::decode<1>(cfg);
     auto warp_tile =
-        group::tiled_partition<warp_size>(group::this_thread_block(item_ct1));
-    const auto warpid = thread::get_subwarp_id_flat<warp_size>(item_ct1);
+        group::tiled_partition<sg_size>(group::this_thread_block(item_ct1));
+    const auto warpid = thread::get_subwarp_id_flat<sg_size>(item_ct1);
     const auto tid_in_warp = warp_tile.thread_rank();
     const auto slice_num = ceildiv(num_rows, slice_size);
 
     size_type thread_result = 0;
-    for (size_type i = tid_in_warp; i < slice_size; i += warp_size) {
+    for (size_type i = tid_in_warp; i < slice_size; i += sg_size) {
         if (warpid * slice_size + i < num_rows) {
             thread_result =
                 max(thread_result, nnz_per_row[warpid * slice_size + i]);
@@ -632,37 +734,32 @@ void reduce_max_nnz_per_slice(size_type num_rows, size_type slice_size,
     }
 }
 
-void reduce_max_nnz_per_slice(dim3 grid, dim3 block,
-                              size_t dynamic_shared_memory, sycl::queue *stream,
-                              size_type num_rows, size_type slice_size,
-                              size_type stride_factor,
-                              const size_type *nnz_per_row, size_type *result)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                reduce_max_nnz_per_slice(num_rows, slice_size, stride_factor,
-                                         nnz_per_row, result, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST_CONFIG(reduce_max_nnz_per_slice,
+                               reduce_max_nnz_per_slice)
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(reduce_max_nnz_per_slice,
+                                           reduce_max_nnz_per_slice)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(reduce_max_nnz_per_slice_call,
+                               reduce_max_nnz_per_slice, KCFG_1D, kcfg_1d_list)
 
 
+template <ConfigSetType cfg>
 void reduce_total_cols(size_type num_slices,
                        const size_type *__restrict__ max_nnz_per_slice,
                        size_type *__restrict__ result,
                        sycl::nd_item<3> item_ct1, uint8_t *dpct_local)
 {
     auto block_result = (size_type *)dpct_local;
-
-    reduce_array(num_slices, max_nnz_per_slice, block_result, item_ct1,
-                 [](const size_type &x, const size_type &y) { return x + y; });
+    constexpr auto sg_size = KCFG_1D::decode<1>(cfg);
+    reduce_array<sg_size>(
+        num_slices, max_nnz_per_slice, block_result, item_ct1,
+        [](const size_type &x, const size_type &y) { return x + y; });
 
     if (item_ct1.get_local_id(2) == 0) {
         result[item_ct1.get_group(2)] = block_result[0];
     }
 }
 
+template <ConfigSetType cfg = KCFG_1D::encode(256, 32)>
 void reduce_total_cols(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                        sycl::queue *stream, size_type num_slices,
                        const size_type *max_nnz_per_slice, size_type *result)
@@ -674,12 +771,18 @@ void reduce_total_cols(dim3 grid, dim3 block, size_t dynamic_shared_memory,
 
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                reduce_total_cols(num_slices, max_nnz_per_slice, result,
-                                  item_ct1, dpct_local_acc_ct1.get_pointer());
+                reduce_total_cols<cfg>(num_slices, max_nnz_per_slice, result,
+                                       item_ct1,
+                                       dpct_local_acc_ct1.get_pointer());
             });
     });
 }
 
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(reduce_total_cols,
+                                           reduce_total_cols);
+GKO_ENABLE_DEFAULT_CONFIG_CALL(reduce_total_cols_call, reduce_total_cols,
+                               KCFG_1D, kcfg_1d_list)
+
 
 template <typename IndexType, typename ValueType>
 void symm_permute(size_type num_rows, size_type num_cols,
@@ -952,24 +1055,7 @@ void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
         // TODO: these are tuning parameters obtained experimentally, once
         // we decide how to handle this uniformly, they should be modified
         // appropriately
-        constexpr auto work_per_thread = 32;
-        constexpr auto block_size = default_block_size;
-
-        constexpr auto work_per_block = work_per_thread * block_size;
-        const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
-        const dim3 block_dim{config::warp_size, 1,
-                             block_size / config::warp_size};
-        Array<ValueType> work(exec, grid_dim.x);
-        // TODO: write a kernel which does this more efficiently
-        for (size_type col = 0; col < x->get_size()[1]; ++col) {
-            kernel::compute_partial_dot<block_size>(
-                grid_dim, block_dim, 0, exec->get_queue(), x->get_size()[0],
-                x->get_const_values() + col, x->get_stride(),
-                y->get_const_values() + col, y->get_stride(), work.get_data());
-            kernel::finalize_dot_computation<block_size>(
-                1, block_dim, 0, exec->get_queue(), grid_dim.x,
-                work.get_const_data(), result->get_values() + col);
-        }
+        kernel::compute_dot_call(exec, x, y, result);
     }
 }
 
@@ -998,27 +1084,7 @@ void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
                 result->get_values() + col);
         }
     } else {
-        using norm_type = remove_complex<ValueType>;
-        // // TODO: these are tuning parameters obtained experimentally, once
-        // // we decide how to handle this uniformly, they should be modified
-        // // appropriately
-        constexpr auto work_per_thread = 32;
-        constexpr auto block_size = default_block_size;
-
-        constexpr auto work_per_block = work_per_thread * block_size;
-        const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
-        const dim3 block_dim{config::warp_size, 1,
-                             block_size / config::warp_size};
-        Array<norm_type> work(exec, grid_dim.x);
-        // TODO: write a kernel which does this more efficiently
-        for (size_type col = 0; col < x->get_size()[1]; ++col) {
-            kernel::compute_partial_norm2<block_size>(
-                grid_dim, block_dim, 0, exec->get_queue(), x->get_size()[0],
-                x->get_const_values() + col, x->get_stride(), work.get_data());
-            kernel::finalize_norm2_computation<block_size>(
-                1, block_dim, 0, exec->get_queue(), grid_dim.x,
-                work.get_const_data(), result->get_values() + col);
-        }
+        kernel::compute_norm2_call(exec, x, result);
     }
 }
 
@@ -1073,9 +1139,9 @@ void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
     const auto rows_per_block = ceildiv(default_block_size, config::warp_size);
     const auto grid_dim_nnz = ceildiv(source->get_size()[0], rows_per_block);
 
-    kernel::count_nnz_per_row(grid_dim_nnz, default_block_size, 0,
-                              exec->get_queue(), num_rows, num_cols, stride,
-                              source->get_const_values(), row_ptrs);
+    kernel::count_nnz_per_row_call(
+        grid_dim_nnz, default_block_size, 0, exec->get_queue(), num_rows,
+        num_cols, stride, source->get_const_values(), row_ptrs);
 
     components::prefix_sum(exec, row_ptrs, num_rows + 1);
 
@@ -1157,7 +1223,7 @@ void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
 
     if (grid_dim > 0) {
         std::cout << "calculate_slice_lengths" << std::endl;
-        kernel::calculate_slice_lengths(
+        kernel::calculate_slice_lengths_call(
             grid_dim, config::warp_size, 0, exec->get_queue(), num_rows,
             slice_size, slice_num, stride_factor, nnz_per_row.get_const_data(),
             slice_lengths, slice_sets);
@@ -1218,24 +1284,30 @@ void calculate_max_nnz_per_row(std::shared_ptr<const DpcppExecutor> exec,
     auto nnz_per_row = Array<size_type>(exec, num_rows);
 
     calculate_nonzeros_per_row(exec, source, &nnz_per_row);
-
-    const auto n = ceildiv(num_rows, default_block_size);
-    const size_type grid_dim =
-        (n <= default_block_size) ? n : default_block_size;
+    auto queue = exec->get_queue();
+    constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
+    const ConfigSetType cfg =
+        get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+            return validate(queue, KCFG_1D::decode<0>(cfg),
+                            KCFG_1D::decode<1>(cfg));
+        });
+    const auto wg_size = KCFG_1D::decode<0>(cfg);
+    std::cout << "wg_size " << wg_size << "sg_size " << KCFG_1D::decode<1>(cfg)
+              << std::endl;
+    const auto n = ceildiv(num_rows, wg_size);
+    const size_type grid_dim = (n <= wg_size) ? n : wg_size;
 
     auto block_results = Array<size_type>(exec, grid_dim);
 
-    kernel::reduce_max_nnz(
-        grid_dim, default_block_size, default_block_size * sizeof(size_type),
-        exec->get_queue(), num_rows, nnz_per_row.get_const_data(),
-        block_results.get_data());
+    kernel::reduce_max_nnz_call(
+        grid_dim, wg_size, wg_size * sizeof(size_type), exec->get_queue(),
+        num_rows, nnz_per_row.get_const_data(), block_results.get_data());
 
     auto d_result = Array<size_type>(exec, 1);
 
-    kernel::reduce_max_nnz(1, default_block_size,
-                           default_block_size * sizeof(size_type),
-                           exec->get_queue(), grid_dim,
-                           block_results.get_const_data(), d_result.get_data());
+    kernel::reduce_max_nnz_call(
+        1, wg_size, wg_size * sizeof(size_type), exec->get_queue(), grid_dim,
+        block_results.get_const_data(), d_result.get_data());
 
     *result = exec->copy_val_to_host(d_result.get_const_data());
 }
@@ -1249,12 +1321,21 @@ void calculate_nonzeros_per_row(std::shared_ptr<const DpcppExecutor> exec,
                                 const matrix::Dense<ValueType> *source,
                                 Array<size_type> *result)
 {
-    const dim3 block_size(default_block_size, 1, 1);
-    auto rows_per_block = ceildiv(default_block_size, config::warp_size);
+    auto queue = exec->get_queue();
+    constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
+    const ConfigSetType cfg =
+        get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+            return validate(queue, KCFG_1D::decode<0>(cfg),
+                            KCFG_1D::decode<1>(cfg));
+        });
+    const auto wg_size = KCFG_1D::decode<0>(cfg);
+    const auto sg_size = KCFG_1D::decode<1>(cfg);
+    const dim3 block_size(wg_size, 1, 1);
+    auto rows_per_block = ceildiv(wg_size, sg_size);
     const size_t grid_x = ceildiv(source->get_size()[0], rows_per_block);
     const dim3 grid_size(grid_x, 1, 1);
     if (grid_x > 0) {
-        kernel::count_nnz_per_row(
+        kernel::count_nnz_per_row_call(
             grid_size, block_size, 0, exec->get_queue(), source->get_size()[0],
             source->get_size()[1], source->get_stride(),
             source->get_const_values(), result->get_data());
@@ -1286,28 +1367,36 @@ void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
     calculate_nonzeros_per_row(exec, source, &nnz_per_row);
 
     auto max_nnz_per_slice = Array<size_type>(exec, slice_num);
-
-    auto grid_dim = ceildiv(slice_num * config::warp_size, default_block_size);
-
-    kernel::reduce_max_nnz_per_slice(
-        grid_dim, default_block_size, 0, exec->get_queue(), num_rows,
-        slice_size, stride_factor, nnz_per_row.get_const_data(),
+    auto queue = exec->get_queue();
+    constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
+    const ConfigSetType cfg =
+        get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+            return validate(queue, KCFG_1D::decode<0>(cfg),
+                            KCFG_1D::decode<1>(cfg));
+        });
+    const auto wg_size = KCFG_1D::decode<0>(cfg);
+    const auto sg_size = KCFG_1D::decode<1>(cfg);
+
+    auto grid_dim = ceildiv(slice_num * sg_size, wg_size);
+
+    kernel::reduce_max_nnz_per_slice_call(
+        grid_dim, wg_size, 0, exec->get_queue(), num_rows, slice_size,
+        stride_factor, nnz_per_row.get_const_data(),
         max_nnz_per_slice.get_data());
 
-    grid_dim = ceildiv(slice_num, default_block_size);
+    grid_dim = ceildiv(slice_num, wg_size);
     auto block_results = Array<size_type>(exec, grid_dim);
 
-    kernel::reduce_total_cols(
-        grid_dim, default_block_size, default_block_size * sizeof(size_type),
-        exec->get_queue(), slice_num, max_nnz_per_slice.get_const_data(),
-        block_results.get_data());
+    kernel::reduce_total_cols(grid_dim, wg_size, wg_size * sizeof(size_type),
+                              exec->get_queue(), slice_num,
+                              max_nnz_per_slice.get_const_data(),
+                              block_results.get_data());
 
     auto d_result = Array<size_type>(exec, 1);
 
     kernel::reduce_total_cols(
-        1, default_block_size, default_block_size * sizeof(size_type),
-        exec->get_queue(), grid_dim, block_results.get_const_data(),
-        d_result.get_data());
+        1, wg_size, wg_size * sizeof(size_type), exec->get_queue(), grid_dim,
+        block_results.get_const_data(), d_result.get_data());
 
     *result = exec->copy_val_to_host(d_result.get_const_data());
 }

From 22ab0a01125221be65ad69995e88d8e6dc4bbc22 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 25 May 2021 22:02:18 +0200
Subject: [PATCH 0898/1008] move to new style for config selection

---
 dpcpp/components/prefix_sum.dp.cpp |  38 ++++-
 dpcpp/components/prefix_sum.dp.hpp |  18 +-
 dpcpp/components/reduction.dp.hpp  |  42 ++---
 dpcpp/matrix/dense_kernels.dp.cpp  | 256 ++++++++++++++---------------
 4 files changed, 181 insertions(+), 173 deletions(-)

diff --git a/dpcpp/components/prefix_sum.dp.cpp b/dpcpp/components/prefix_sum.dp.cpp
index b4961809a8b..62a9700473d 100644
--- a/dpcpp/components/prefix_sum.dp.cpp
+++ b/dpcpp/components/prefix_sum.dp.cpp
@@ -36,6 +36,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <CL/sycl.hpp>
 
 
+#include <include/ginkgo/core/base/types.hpp>
+
+
+#include "dpcpp/base/helper.hpp"
 #include "dpcpp/components/prefix_sum.dp.hpp"
 
 
@@ -45,7 +49,20 @@ namespace dpcpp {
 namespace components {
 
 
-constexpr int prefix_sum_block_size = 256;
+using BlockCfg = ConfigSet<11>;
+
+constexpr auto block_cfg_list =
+    ::gko::syn::value_list<ConfigSetType, BlockCfg::encode(512),
+                           BlockCfg::encode(256), BlockCfg::encode(128)>();
+
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(start_prefix_sum, start_prefix_sum)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(start_prefix_sum_call, start_prefix_sum,
+                               BlockCfg, block_cfg_list)
+
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(finalize_prefix_sum,
+                                           finalize_prefix_sum)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(finalize_prefix_sum_call, finalize_prefix_sum,
+                               BlockCfg, block_cfg_list)
 
 
 template <typename IndexType>
@@ -54,18 +71,23 @@ void prefix_sum(std::shared_ptr<const DpcppExecutor> exec, IndexType *counts,
 {
     // prefix_sum should be on the valid array
     if (num_entries > 0) {
-        auto num_blocks = ceildiv(num_entries, prefix_sum_block_size);
+        auto queue = exec->get_queue();
+        constexpr auto block_cfg_array = as_array(block_cfg_list);
+        const ConfigSetType cfg =
+            get_first_cfg(block_cfg_array, [&queue](ConfigSetType cfg) {
+                return validate(queue, BlockCfg::decode<0>(cfg), 16);
+            });
+        const auto wg_size = BlockCfg::decode<0>(cfg);
+        auto num_blocks = ceildiv(num_entries, wg_size);
         Array<IndexType> block_sum_array(exec, num_blocks - 1);
         auto block_sums = block_sum_array.get_data();
-        start_prefix_sum<prefix_sum_block_size>(
-            num_blocks, prefix_sum_block_size, 0, exec->get_queue(),
-            num_entries, counts, block_sums);
+        start_prefix_sum_call(num_blocks, wg_size, 0, exec->get_queue(), cfg,
+                              num_entries, counts, block_sums);
         // add the total sum of the previous block only when the number of block
         // is larger than 1.
         if (num_blocks > 1) {
-            finalize_prefix_sum<prefix_sum_block_size>(
-                num_blocks, prefix_sum_block_size, 0, exec->get_queue(),
-                num_entries, counts, block_sums);
+            finalize_prefix_sum_call(num_blocks, wg_size, 0, exec->get_queue(),
+                                     cfg, num_entries, counts, block_sums);
         }
     }
 }
diff --git a/dpcpp/components/prefix_sum.dp.hpp b/dpcpp/components/prefix_sum.dp.hpp
index c6f7c7cfb20..fd9ff2ac263 100644
--- a/dpcpp/components/prefix_sum.dp.hpp
+++ b/dpcpp/components/prefix_sum.dp.hpp
@@ -125,7 +125,7 @@ __dpct_inline__ void subwarp_prefix_sum(ValueType element,
  * @note To calculate the prefix sum over an array of size bigger than
  *       `block_size`, `finalize_prefix_sum` has to be used as well.
  */
-template <int block_size, typename ValueType>
+template <ConfigSetType block_size, typename ValueType>
 void start_prefix_sum(size_type num_elements, ValueType *__restrict__ elements,
                       ValueType *__restrict__ block_sum,
                       sycl::nd_item<3> item_ct1,
@@ -178,7 +178,7 @@ void start_prefix_sum(size_type num_elements, ValueType *__restrict__ elements,
     }
 }
 
-template <int block_size, typename ValueType>
+template <ConfigSetType block_size, typename ValueType>
 void start_prefix_sum(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                       sycl::queue *stream, size_type num_elements,
                       ValueType *elements, ValueType *block_sum)
@@ -189,10 +189,7 @@ void start_prefix_sum(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                        sycl::access::target::local>
             prefix_helper_acc_ct1(cgh);
 
-        auto local_range = block.get_range();
-        auto global_range = grid.get_range() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+        cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              start_prefix_sum<block_size>(
                                  num_elements, elements, block_sum, item_ct1,
@@ -217,7 +214,7 @@ void start_prefix_sum(dim3 grid, dim3 block, size_t dynamic_shared_memory,
  *
  * @note To calculate a prefix sum, first `start_prefix_sum` has to be called.
  */
-template <int block_size, typename ValueType>
+template <ConfigSetType block_size, typename ValueType>
 void finalize_prefix_sum(size_type num_elements,
                          ValueType *__restrict__ elements,
                          const ValueType *__restrict__ block_sum,
@@ -234,16 +231,13 @@ void finalize_prefix_sum(size_type num_elements,
     }
 }
 
-template <int block_size, typename ValueType>
+template <ConfigSetType block_size, typename ValueType>
 void finalize_prefix_sum(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                          sycl::queue *stream, size_type num_elements,
                          ValueType *elements, const ValueType *block_sum)
 {
     stream->submit([&](sycl::handler &cgh) {
-        auto local_range = block.get_range();
-        auto global_range = grid.get_range() * local_range;
-
-        cgh.parallel_for(sycl::nd_range<3>(global_range, local_range),
+        cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              finalize_prefix_sum<block_size>(
                                  num_elements, elements, block_sum, item_ct1);
diff --git a/dpcpp/components/reduction.dp.hpp b/dpcpp/components/reduction.dp.hpp
index bc9937ddb1a..2626d40b314 100644
--- a/dpcpp/components/reduction.dp.hpp
+++ b/dpcpp/components/reduction.dp.hpp
@@ -261,31 +261,33 @@ ValueType reduce_add_array(std::shared_ptr<const DpcppExecutor> exec,
     size_type grid_dim = size;
     auto block_results = Array<ValueType>(exec);
     ValueType answer = zero<ValueType>();
-    for (auto &cfg : kcfg_1d_array) {
-        const auto block_size = KCFG_1D::decode<0>(cfg);
-        const auto warp_size = KCFG_1D::decode<1>(cfg);
-        if (!validate(exec->get_queue(), block_size, warp_size)) {
-            continue;
-        }
-        if (size > block_size) {
-            const auto n = ceildiv(size, block_size);
-            grid_dim = (n <= block_size) ? n : block_size;
-
-            block_results.resize_and_reset(grid_dim);
+    auto queue = exec->get_queue();
+    constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
+    const ConfigSetType cfg =
+        get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+            return validate(queue, KCFG_1D::decode<0>(cfg),
+                            KCFG_1D::decode<1>(cfg));
+        });
+    const auto wg_size = KCFG_1D::decode<0>(cfg);
+    const auto sg_size = KCFG_1D::decode<1>(cfg);
 
-            reduce_add_array_call(grid_dim, block_size, 0, exec->get_queue(),
-                                  size, source, block_results.get_data());
+    if (size > wg_size) {
+        const auto n = ceildiv(size, wg_size);
+        grid_dim = (n <= wg_size) ? n : wg_size;
 
-            block_results_val = block_results.get_const_data();
-        }
+        block_results.resize_and_reset(grid_dim);
 
-        auto d_result = Array<ValueType>(exec, 1);
+        reduce_add_array_call(grid_dim, wg_size, 0, exec->get_queue(), cfg,
+                              size, source, block_results.get_data());
 
-        reduce_add_array_call(1, block_size, 0, exec->get_queue(), grid_dim,
-                              block_results_val, d_result.get_data());
-        answer = exec->copy_val_to_host(d_result.get_const_data());
-        break;
+        block_results_val = block_results.get_const_data();
     }
+
+    auto d_result = Array<ValueType>(exec, 1);
+
+    reduce_add_array_call(1, wg_size, 0, exec->get_queue(), cfg, grid_dim,
+                          block_results_val, d_result.get_data());
+    answer = exec->copy_val_to_host(d_result.get_const_data());
     return answer;
 }
 
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 0d1be735e17..3e4fee0a2d5 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -264,6 +264,11 @@ void compute_partial_dot(dim3 grid, dim3 block, size_t dynamic_shared_memory,
     });
 }
 
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(compute_partial_dot,
+                                           compute_partial_dot)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(compute_partial_dot_call, compute_partial_dot,
+                               KCFG_1D, kcfg_1d_list)
+
 
 template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void finalize_dot_computation(
@@ -301,51 +306,10 @@ void finalize_dot_computation(dim3 grid, dim3 block,
     });
 }
 
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
-void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
-                 const matrix::Dense<ValueType> *x,
-                 const matrix::Dense<ValueType> *y,
-                 matrix::Dense<ValueType> *result)
-{
-    constexpr auto work_per_thread = 32;
-    constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
-    constexpr auto sg_size = KCFG_1D::decode<1>(cfg);
-    std::cout << "dot " << cfg << " " << wg_size << " " << sg_size << std::endl;
-    constexpr auto work_per_block = work_per_thread * wg_size;
-    const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
-    const dim3 block_dim{sg_size, 1, wg_size / sg_size};
-    Array<ValueType> work(exec, grid_dim.x);
-    // TODO: write a kernel which does this more efficiently
-    for (size_type col = 0; col < x->get_size()[1]; ++col) {
-        compute_partial_dot<cfg>(grid_dim, block_dim, 0, exec->get_queue(),
-                                 x->get_size()[0], x->get_const_values() + col,
-                                 x->get_stride(), y->get_const_values() + col,
-                                 y->get_stride(), work.get_data());
-        finalize_dot_computation<cfg>(1, block_dim, 0, exec->get_queue(),
-                                      grid_dim.x, work.get_const_data(),
-                                      result->get_values() + col);
-    }
-}
-
-GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(compute_dot_config, compute_dot)
-
-template <typename ValueType>
-void compute_dot_call(std::shared_ptr<const DpcppExecutor> exec,
-                      const matrix::Dense<ValueType> *x,
-                      const matrix::Dense<ValueType> *y,
-                      matrix::Dense<ValueType> *result)
-{
-    auto queue = exec->get_queue();
-    compute_dot_config(
-        kcfg_1d_list,
-        [&queue](::gko::ConfigSetType cfg) {
-            return validate(queue, KCFG_1D::decode<0>(cfg),
-                            KCFG_1D::decode<1>(cfg));
-        },
-        ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
-        ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
-        exec, x, y, result);
-}
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(finalize_dot_computation,
+                                           finalize_dot_computation)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(finalize_dot_computation_call,
+                               finalize_dot_computation, KCFG_1D, kcfg_1d_list)
 
 
 template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
@@ -386,6 +350,11 @@ void compute_partial_norm2(dim3 grid, dim3 block, size_t dynamic_shared_memory,
     });
 }
 
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(compute_partial_norm2,
+                                           compute_partial_norm2)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(compute_partial_norm2_call,
+                               compute_partial_norm2, KCFG_1D, kcfg_1d_list)
+
 
 template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void finalize_norm2_computation(
@@ -423,53 +392,11 @@ void finalize_norm2_computation(dim3 grid, dim3 block,
     });
 }
 
-
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
-void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
-                   const matrix::Dense<ValueType> *x,
-                   matrix::Dense<remove_complex<ValueType>> *result)
-{
-    using norm_type = remove_complex<ValueType>;
-    // // TODO: these are tuning parameters obtained experimentally, once
-    // // we decide how to handle this uniformly, they should be modified
-    // // appropriately
-    constexpr auto work_per_thread = 32;
-    constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
-    constexpr auto sg_size = KCFG_1D::decode<1>(cfg);
-
-    constexpr auto work_per_block = work_per_thread * wg_size;
-    const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
-    const dim3 block_dim{sg_size, 1, wg_size / sg_size};
-    Array<norm_type> work(exec, grid_dim.x);
-    // TODO: write a kernel which does this more efficiently
-    for (size_type col = 0; col < x->get_size()[1]; ++col) {
-        compute_partial_norm2<cfg>(
-            grid_dim, block_dim, 0, exec->get_queue(), x->get_size()[0],
-            x->get_const_values() + col, x->get_stride(), work.get_data());
-        finalize_norm2_computation<cfg>(1, block_dim, 0, exec->get_queue(),
-                                        grid_dim.x, work.get_const_data(),
-                                        result->get_values() + col);
-    }
-}
-
-GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(compute_norm2_config, compute_norm2)
-
-template <typename ValueType>
-void compute_norm2_call(std::shared_ptr<const DpcppExecutor> exec,
-                        const matrix::Dense<ValueType> *x,
-                        matrix::Dense<remove_complex<ValueType>> *result)
-{
-    auto queue = exec->get_queue();
-    compute_norm2_config(
-        kcfg_1d_list,
-        [&queue](::gko::ConfigSetType cfg) {
-            return validate(queue, KCFG_1D::decode<0>(cfg),
-                            KCFG_1D::decode<1>(cfg));
-        },
-        ::gko::syn::value_list<bool>(), ::gko::syn::value_list<int>(),
-        ::gko::syn::value_list<gko::size_type>(), ::gko::syn::type_list<>(),
-        exec, x, result);
-}
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(finalize_norm2_computation,
+                                           finalize_norm2_computation)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(finalize_norm2_computation_call,
+                               finalize_norm2_computation, KCFG_1D,
+                               kcfg_1d_list)
 
 
 template <typename ValueType, typename IndexType>
@@ -1055,7 +982,32 @@ void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
         // TODO: these are tuning parameters obtained experimentally, once
         // we decide how to handle this uniformly, they should be modified
         // appropriately
-        kernel::compute_dot_call(exec, x, y, result);
+        constexpr auto work_per_thread = 32;
+        auto queue = exec->get_queue();
+        constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
+        const ConfigSetType cfg =
+            get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+                return validate(queue, KCFG_1D::decode<0>(cfg),
+                                KCFG_1D::decode<1>(cfg));
+            });
+        const auto wg_size = KCFG_1D::decode<0>(cfg);
+        const auto sg_size = KCFG_1D::decode<1>(cfg);
+        std::cout << "dot " << cfg << " " << wg_size << " " << sg_size
+                  << std::endl;
+        const auto work_per_block = work_per_thread * wg_size;
+        const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
+        const dim3 block_dim{sg_size, 1, wg_size / sg_size};
+        Array<ValueType> work(exec, grid_dim.x);
+        // TODO: write a kernel which does this more efficiently
+        for (size_type col = 0; col < x->get_size()[1]; ++col) {
+            kernel::compute_partial_dot_call(
+                grid_dim, block_dim, 0, exec->get_queue(), cfg,
+                x->get_size()[0], x->get_const_values() + col, x->get_stride(),
+                y->get_const_values() + col, y->get_stride(), work.get_data());
+            kernel::finalize_dot_computation_call(
+                1, block_dim, 0, exec->get_queue(), cfg, grid_dim.x,
+                work.get_const_data(), result->get_values() + col);
+        }
     }
 }
 
@@ -1084,7 +1036,35 @@ void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
                 result->get_values() + col);
         }
     } else {
-        kernel::compute_norm2_call(exec, x, result);
+        using norm_type = remove_complex<ValueType>;
+        // TODO: these are tuning parameters obtained experimentally, once
+        // we decide how to handle this uniformly, they should be modified
+        // appropriately
+        constexpr auto work_per_thread = 32;
+        auto queue = exec->get_queue();
+        constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
+        const ConfigSetType cfg =
+            get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+                return validate(queue, KCFG_1D::decode<0>(cfg),
+                                KCFG_1D::decode<1>(cfg));
+            });
+        const auto wg_size = KCFG_1D::decode<0>(cfg);
+        const auto sg_size = KCFG_1D::decode<1>(cfg);
+
+        const auto work_per_block = work_per_thread * wg_size;
+        const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
+        const dim3 block_dim{sg_size, 1, wg_size / sg_size};
+        Array<norm_type> work(exec, grid_dim.x);
+        // TODO: write a kernel which does this more efficiently
+        for (size_type col = 0; col < x->get_size()[1]; ++col) {
+            kernel::compute_partial_norm2_call(
+                grid_dim, block_dim, 0, exec->get_queue(), cfg,
+                x->get_size()[0], x->get_const_values() + col, x->get_stride(),
+                work.get_data());
+            kernel::finalize_norm2_computation_call(
+                1, block_dim, 0, exec->get_queue(), cfg, grid_dim.x,
+                work.get_const_data(), result->get_values() + col);
+        }
     }
 }
 
@@ -1127,6 +1107,16 @@ void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Dense<ValueType> *source,
                     matrix::Csr<ValueType, IndexType> *result)
 {
+    auto queue = exec->get_queue();
+    constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
+    const ConfigSetType cfg =
+        get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+            return validate(queue, KCFG_1D::decode<0>(cfg),
+                            KCFG_1D::decode<1>(cfg));
+        });
+    const auto wg_size = KCFG_1D::decode<0>(cfg);
+    const auto sg_size = KCFG_1D::decode<1>(cfg);
+
     auto num_rows = result->get_size()[0];
     auto num_cols = result->get_size()[1];
 
@@ -1136,20 +1126,20 @@ void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
 
     auto stride = source->get_stride();
 
-    const auto rows_per_block = ceildiv(default_block_size, config::warp_size);
+    const auto rows_per_block = ceildiv(wg_size, sg_size);
     const auto grid_dim_nnz = ceildiv(source->get_size()[0], rows_per_block);
 
-    kernel::count_nnz_per_row_call(
-        grid_dim_nnz, default_block_size, 0, exec->get_queue(), num_rows,
-        num_cols, stride, source->get_const_values(), row_ptrs);
+    kernel::count_nnz_per_row_call(grid_dim_nnz, wg_size, 0, exec->get_queue(),
+                                   cfg, num_rows, num_cols, stride,
+                                   source->get_const_values(), row_ptrs);
 
     components::prefix_sum(exec, row_ptrs, num_rows + 1);
 
-    size_type grid_dim = ceildiv(num_rows, default_block_size);
+    size_type grid_dim = ceildiv(num_rows, wg_size);
 
-    kernel::fill_in_csr(grid_dim, default_block_size, 0, exec->get_queue(),
-                        num_rows, num_cols, stride, source->get_const_values(),
-                        row_ptrs, col_idxs, values);
+    kernel::fill_in_csr(grid_dim, wg_size, 0, exec->get_queue(), num_rows,
+                        num_cols, stride, source->get_const_values(), row_ptrs,
+                        col_idxs, values);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -1197,6 +1187,16 @@ void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Dense<ValueType> *source,
                       matrix::Sellp<ValueType, IndexType> *result)
 {
+    auto queue = exec->get_queue();
+    constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
+    const ConfigSetType cfg =
+        get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+            return validate(queue, KCFG_1D::decode<0>(cfg),
+                            KCFG_1D::decode<1>(cfg));
+        });
+    const auto wg_size = KCFG_1D::decode<0>(cfg);
+    const auto sg_size = KCFG_1D::decode<1>(cfg);
+
     const auto stride = source->get_stride();
     const auto num_rows = result->get_size()[0];
     const auto num_cols = result->get_size()[1];
@@ -1215,34 +1215,24 @@ void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
     const int slice_num = ceildiv(num_rows, slice_size);
 
     auto nnz_per_row = Array<size_type>(exec, num_rows);
-    std::cout << "calculate_nonzeros_per_row" << std::endl;
+
     calculate_nonzeros_per_row(exec, source, &nnz_per_row);
-    exec->synchronize();
-    std::cout << "calculate_nonzeros_per_row finish" << std::endl;
+
     auto grid_dim = slice_num;
 
     if (grid_dim > 0) {
-        std::cout << "calculate_slice_lengths" << std::endl;
         kernel::calculate_slice_lengths_call(
-            grid_dim, config::warp_size, 0, exec->get_queue(), num_rows,
-            slice_size, slice_num, stride_factor, nnz_per_row.get_const_data(),
+            grid_dim, sg_size, 0, exec->get_queue(), cfg, num_rows, slice_size,
+            slice_num, stride_factor, nnz_per_row.get_const_data(),
             slice_lengths, slice_sets);
-        exec->synchronize();
-        std::cout << "calculate_slice_lengths finish" << std::endl;
     }
-    std::cout << "prefix_sum" << std::endl;
     components::prefix_sum(exec, slice_sets, slice_num + 1);
-    // exec->synchronize();
-    std::cout << "prefix_sum finish" << std::endl;
-    grid_dim = ceildiv(num_rows, default_block_size);
+    grid_dim = ceildiv(num_rows, wg_size);
     if (grid_dim > 0) {
-        std::cout << "fill_in_sellp" << std::endl;
-        kernel::fill_in_sellp(grid_dim, default_block_size, 0,
-                              exec->get_queue(), num_rows, num_cols, slice_size,
-                              stride, source->get_const_values(), slice_lengths,
+        kernel::fill_in_sellp(grid_dim, wg_size, 0, exec->get_queue(), num_rows,
+                              num_cols, slice_size, stride,
+                              source->get_const_values(), slice_lengths,
                               slice_sets, col_idxs, vals);
-        exec->synchronize();
-        std::cout << "fill_in_sellp finish" << std::endl;
     }
 }
 
@@ -1300,14 +1290,14 @@ void calculate_max_nnz_per_row(std::shared_ptr<const DpcppExecutor> exec,
     auto block_results = Array<size_type>(exec, grid_dim);
 
     kernel::reduce_max_nnz_call(
-        grid_dim, wg_size, wg_size * sizeof(size_type), exec->get_queue(),
+        grid_dim, wg_size, wg_size * sizeof(size_type), exec->get_queue(), cfg,
         num_rows, nnz_per_row.get_const_data(), block_results.get_data());
 
     auto d_result = Array<size_type>(exec, 1);
 
     kernel::reduce_max_nnz_call(
-        1, wg_size, wg_size * sizeof(size_type), exec->get_queue(), grid_dim,
-        block_results.get_const_data(), d_result.get_data());
+        1, wg_size, wg_size * sizeof(size_type), exec->get_queue(), cfg,
+        grid_dim, block_results.get_const_data(), d_result.get_data());
 
     *result = exec->copy_val_to_host(d_result.get_const_data());
 }
@@ -1336,8 +1326,8 @@ void calculate_nonzeros_per_row(std::shared_ptr<const DpcppExecutor> exec,
     const dim3 grid_size(grid_x, 1, 1);
     if (grid_x > 0) {
         kernel::count_nnz_per_row_call(
-            grid_size, block_size, 0, exec->get_queue(), source->get_size()[0],
-            source->get_size()[1], source->get_stride(),
+            grid_size, block_size, 0, exec->get_queue(), cfg,
+            source->get_size()[0], source->get_size()[1], source->get_stride(),
             source->get_const_values(), result->get_data());
     }
 }
@@ -1380,23 +1370,23 @@ void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
     auto grid_dim = ceildiv(slice_num * sg_size, wg_size);
 
     kernel::reduce_max_nnz_per_slice_call(
-        grid_dim, wg_size, 0, exec->get_queue(), num_rows, slice_size,
+        grid_dim, wg_size, 0, exec->get_queue(), cfg, num_rows, slice_size,
         stride_factor, nnz_per_row.get_const_data(),
         max_nnz_per_slice.get_data());
 
     grid_dim = ceildiv(slice_num, wg_size);
     auto block_results = Array<size_type>(exec, grid_dim);
 
-    kernel::reduce_total_cols(grid_dim, wg_size, wg_size * sizeof(size_type),
-                              exec->get_queue(), slice_num,
-                              max_nnz_per_slice.get_const_data(),
-                              block_results.get_data());
+    kernel::reduce_total_cols_call(
+        grid_dim, wg_size, wg_size * sizeof(size_type), exec->get_queue(), cfg,
+        slice_num, max_nnz_per_slice.get_const_data(),
+        block_results.get_data());
 
     auto d_result = Array<size_type>(exec, 1);
 
-    kernel::reduce_total_cols(
-        1, wg_size, wg_size * sizeof(size_type), exec->get_queue(), grid_dim,
-        block_results.get_const_data(), d_result.get_data());
+    kernel::reduce_total_cols_call(
+        1, wg_size, wg_size * sizeof(size_type), exec->get_queue(), cfg,
+        grid_dim, block_results.get_const_data(), d_result.get_data());
 
     *result = exec->copy_val_to_host(d_result.get_const_data());
 }

From e3a48650b007fafadda113218f3f182df536476b Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 25 May 2021 23:07:31 +0200
Subject: [PATCH 0899/1008] mv config to the first argument

---
 dev_tools/scripts/regroup            |  2 +-
 dpcpp/base/helper.dp.cpp             | 33 ++++++++++++++++
 dpcpp/components/prefix_sum.dp.cpp   | 13 ++++---
 dpcpp/components/reduction.dp.hpp    |  8 ++--
 dpcpp/matrix/dense_kernels.dp.cpp    | 56 +++++++++++++---------------
 dpcpp/test/components/prefix_sum.cpp |  3 --
 dpcpp/test/matrix/dense_kernels.cpp  |  1 -
 dpcpp/test/utils.hpp                 | 54 ---------------------------
 8 files changed, 71 insertions(+), 99 deletions(-)
 delete mode 100644 dpcpp/test/utils.hpp

diff --git a/dev_tools/scripts/regroup b/dev_tools/scripts/regroup
index 9e8a5172d05..1756481e2e4 100644
--- a/dev_tools/scripts/regroup
+++ b/dev_tools/scripts/regroup
@@ -2,7 +2,7 @@ IncludeBlocks: Regroup
 IncludeCategories:
   - Regex: '^<(rapidjson|gflags|gtest|papi).*'
     Priority: 3
-  - Regex: '^<(omp|cu|hip|thrust|CL/|cooperative).*'
+  - Regex: '^<(omp|cu|hip|thrust|CL/|cooperative|oneapi).*'
     Priority: 2
   - Regex: '^<ginkgo.*'
     Priority: 5
diff --git a/dpcpp/base/helper.dp.cpp b/dpcpp/base/helper.dp.cpp
index fe4395e2534..ae453dd937d 100644
--- a/dpcpp/base/helper.dp.cpp
+++ b/dpcpp/base/helper.dp.cpp
@@ -1,5 +1,38 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
 #include <CL/sycl.hpp>
 
+
 #include "dpcpp/base/helper.hpp"
 
 
diff --git a/dpcpp/components/prefix_sum.dp.cpp b/dpcpp/components/prefix_sum.dp.cpp
index 62a9700473d..330fa297e58 100644
--- a/dpcpp/components/prefix_sum.dp.cpp
+++ b/dpcpp/components/prefix_sum.dp.cpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <CL/sycl.hpp>
 
 
-#include <include/ginkgo/core/base/types.hpp>
+#include <ginkgo/core/base/types.hpp>
 
 
 #include "dpcpp/base/helper.hpp"
@@ -57,12 +57,12 @@ constexpr auto block_cfg_list =
 
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(start_prefix_sum, start_prefix_sum)
 GKO_ENABLE_DEFAULT_CONFIG_CALL(start_prefix_sum_call, start_prefix_sum,
-                               BlockCfg, block_cfg_list)
+                               block_cfg_list)
 
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(finalize_prefix_sum,
                                            finalize_prefix_sum)
 GKO_ENABLE_DEFAULT_CONFIG_CALL(finalize_prefix_sum_call, finalize_prefix_sum,
-                               BlockCfg, block_cfg_list)
+                               block_cfg_list)
 
 
 template <typename IndexType>
@@ -81,13 +81,14 @@ void prefix_sum(std::shared_ptr<const DpcppExecutor> exec, IndexType *counts,
         auto num_blocks = ceildiv(num_entries, wg_size);
         Array<IndexType> block_sum_array(exec, num_blocks - 1);
         auto block_sums = block_sum_array.get_data();
-        start_prefix_sum_call(num_blocks, wg_size, 0, exec->get_queue(), cfg,
+        start_prefix_sum_call(cfg, num_blocks, wg_size, 0, exec->get_queue(),
                               num_entries, counts, block_sums);
         // add the total sum of the previous block only when the number of block
         // is larger than 1.
         if (num_blocks > 1) {
-            finalize_prefix_sum_call(num_blocks, wg_size, 0, exec->get_queue(),
-                                     cfg, num_entries, counts, block_sums);
+            finalize_prefix_sum_call(cfg, num_blocks, wg_size, 0,
+                                     exec->get_queue(), num_entries, counts,
+                                     block_sums);
         }
     }
 }
diff --git a/dpcpp/components/reduction.dp.hpp b/dpcpp/components/reduction.dp.hpp
index 2626d40b314..d3e925ee4ba 100644
--- a/dpcpp/components/reduction.dp.hpp
+++ b/dpcpp/components/reduction.dp.hpp
@@ -42,9 +42,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/synthesizer/containers.hpp>
 
 
-#include <ginkgo/core/synthesizer/containers.hpp>
 #include "core/synthesizer/implementation_selection.hpp"
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
@@ -241,7 +241,7 @@ GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(reduce_add_array_config,
                                            reduce_add_array);
 
 GKO_ENABLE_DEFAULT_CONFIG_CALL(reduce_add_array_call, reduce_add_array_config,
-                               KCFG_1D, kcfg_1d_list);
+                               kcfg_1d_list);
 
 
 /**
@@ -277,7 +277,7 @@ ValueType reduce_add_array(std::shared_ptr<const DpcppExecutor> exec,
 
         block_results.resize_and_reset(grid_dim);
 
-        reduce_add_array_call(grid_dim, wg_size, 0, exec->get_queue(), cfg,
+        reduce_add_array_call(cfg, grid_dim, wg_size, 0, exec->get_queue(),
                               size, source, block_results.get_data());
 
         block_results_val = block_results.get_const_data();
@@ -285,7 +285,7 @@ ValueType reduce_add_array(std::shared_ptr<const DpcppExecutor> exec,
 
     auto d_result = Array<ValueType>(exec, 1);
 
-    reduce_add_array_call(1, wg_size, 0, exec->get_queue(), cfg, grid_dim,
+    reduce_add_array_call(cfg, 1, wg_size, 0, exec->get_queue(), grid_dim,
                           block_results_val, d_result.get_data());
     answer = exec->copy_val_to_host(d_result.get_const_data());
     return answer;
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 3e4fee0a2d5..e83c5dba5cc 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -33,6 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/dense_kernels.hpp"
 
 
+#include <iostream>
+
+
 #include <CL/sycl.hpp>
 #include <oneapi/mkl.hpp>
 
@@ -45,7 +48,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/ell.hpp>
 #include <ginkgo/core/matrix/sellp.hpp>
 #include <ginkgo/core/matrix/sparsity_csr.hpp>
-#include <iostream>
 
 
 #include "core/components/prefix_sum.hpp"
@@ -188,11 +190,6 @@ void compute_partial_reduce(
     }
 }
 
-// GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(compute_partial_reduce_config,
-//                                            compute_partial_reduce);
-// GKO_ENABLE_DEFAULT_CONFIG_CALL(compute_partial_reduce_call,
-// compute_partial_reduce_config,
-//                                KCFG_1D, kcfg_1d_list);
 
 template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType,
           typename CallableReduce, typename CallableFinalize>
@@ -267,7 +264,7 @@ void compute_partial_dot(dim3 grid, dim3 block, size_t dynamic_shared_memory,
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(compute_partial_dot,
                                            compute_partial_dot)
 GKO_ENABLE_DEFAULT_CONFIG_CALL(compute_partial_dot_call, compute_partial_dot,
-                               KCFG_1D, kcfg_1d_list)
+                               kcfg_1d_list)
 
 
 template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
@@ -309,7 +306,7 @@ void finalize_dot_computation(dim3 grid, dim3 block,
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(finalize_dot_computation,
                                            finalize_dot_computation)
 GKO_ENABLE_DEFAULT_CONFIG_CALL(finalize_dot_computation_call,
-                               finalize_dot_computation, KCFG_1D, kcfg_1d_list)
+                               finalize_dot_computation, kcfg_1d_list)
 
 
 template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
@@ -353,7 +350,7 @@ void compute_partial_norm2(dim3 grid, dim3 block, size_t dynamic_shared_memory,
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(compute_partial_norm2,
                                            compute_partial_norm2)
 GKO_ENABLE_DEFAULT_CONFIG_CALL(compute_partial_norm2_call,
-                               compute_partial_norm2, KCFG_1D, kcfg_1d_list)
+                               compute_partial_norm2, kcfg_1d_list)
 
 
 template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
@@ -395,8 +392,7 @@ void finalize_norm2_computation(dim3 grid, dim3 block,
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(finalize_norm2_computation,
                                            finalize_norm2_computation)
 GKO_ENABLE_DEFAULT_CONFIG_CALL(finalize_norm2_computation_call,
-                               finalize_norm2_computation, KCFG_1D,
-                               kcfg_1d_list)
+                               finalize_norm2_computation, kcfg_1d_list)
 
 
 template <typename ValueType, typename IndexType>
@@ -452,7 +448,7 @@ void count_nnz_per_row(size_type num_rows, size_type num_cols, size_type stride,
 GKO_ENABLE_DEFAULT_HOST_CONFIG(count_nnz_per_row, count_nnz_per_row)
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(count_nnz_per_row, count_nnz_per_row)
 GKO_ENABLE_DEFAULT_CONFIG_CALL(count_nnz_per_row_call, count_nnz_per_row,
-                               KCFG_1D, kcfg_1d_list)
+                               kcfg_1d_list)
 
 
 template <typename ValueType, typename IndexType>
@@ -552,7 +548,7 @@ GKO_ENABLE_DEFAULT_HOST_CONFIG(calculate_slice_lengths, calculate_slice_lengths)
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(calculate_slice_lengths,
                                            calculate_slice_lengths)
 GKO_ENABLE_DEFAULT_CONFIG_CALL(calculate_slice_lengths_call,
-                               calculate_slice_lengths, KCFG_1D, kcfg_1d_list)
+                               calculate_slice_lengths, kcfg_1d_list)
 
 
 template <typename ValueType, typename IndexType>
@@ -627,7 +623,7 @@ void reduce_max_nnz(dim3 grid, dim3 block, size_t dynamic_shared_memory,
 }
 
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(reduce_max_nnz, reduce_max_nnz);
-GKO_ENABLE_DEFAULT_CONFIG_CALL(reduce_max_nnz_call, reduce_max_nnz, KCFG_1D,
+GKO_ENABLE_DEFAULT_CONFIG_CALL(reduce_max_nnz_call, reduce_max_nnz,
                                kcfg_1d_list)
 
 template <ConfigSetType cfg>
@@ -666,7 +662,7 @@ GKO_ENABLE_DEFAULT_HOST_CONFIG(reduce_max_nnz_per_slice,
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(reduce_max_nnz_per_slice,
                                            reduce_max_nnz_per_slice)
 GKO_ENABLE_DEFAULT_CONFIG_CALL(reduce_max_nnz_per_slice_call,
-                               reduce_max_nnz_per_slice, KCFG_1D, kcfg_1d_list)
+                               reduce_max_nnz_per_slice, kcfg_1d_list)
 
 
 template <ConfigSetType cfg>
@@ -708,7 +704,7 @@ void reduce_total_cols(dim3 grid, dim3 block, size_t dynamic_shared_memory,
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(reduce_total_cols,
                                            reduce_total_cols);
 GKO_ENABLE_DEFAULT_CONFIG_CALL(reduce_total_cols_call, reduce_total_cols,
-                               KCFG_1D, kcfg_1d_list)
+                               kcfg_1d_list)
 
 
 template <typename IndexType, typename ValueType>
@@ -1001,11 +997,11 @@ void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
         // TODO: write a kernel which does this more efficiently
         for (size_type col = 0; col < x->get_size()[1]; ++col) {
             kernel::compute_partial_dot_call(
-                grid_dim, block_dim, 0, exec->get_queue(), cfg,
+                cfg, grid_dim, block_dim, 0, exec->get_queue(),
                 x->get_size()[0], x->get_const_values() + col, x->get_stride(),
                 y->get_const_values() + col, y->get_stride(), work.get_data());
             kernel::finalize_dot_computation_call(
-                1, block_dim, 0, exec->get_queue(), cfg, grid_dim.x,
+                cfg, 1, block_dim, 0, exec->get_queue(), grid_dim.x,
                 work.get_const_data(), result->get_values() + col);
         }
     }
@@ -1058,11 +1054,11 @@ void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
         // TODO: write a kernel which does this more efficiently
         for (size_type col = 0; col < x->get_size()[1]; ++col) {
             kernel::compute_partial_norm2_call(
-                grid_dim, block_dim, 0, exec->get_queue(), cfg,
+                cfg, grid_dim, block_dim, 0, exec->get_queue(),
                 x->get_size()[0], x->get_const_values() + col, x->get_stride(),
                 work.get_data());
             kernel::finalize_norm2_computation_call(
-                1, block_dim, 0, exec->get_queue(), cfg, grid_dim.x,
+                cfg, 1, block_dim, 0, exec->get_queue(), grid_dim.x,
                 work.get_const_data(), result->get_values() + col);
         }
     }
@@ -1129,9 +1125,9 @@ void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
     const auto rows_per_block = ceildiv(wg_size, sg_size);
     const auto grid_dim_nnz = ceildiv(source->get_size()[0], rows_per_block);
 
-    kernel::count_nnz_per_row_call(grid_dim_nnz, wg_size, 0, exec->get_queue(),
-                                   cfg, num_rows, num_cols, stride,
-                                   source->get_const_values(), row_ptrs);
+    kernel::count_nnz_per_row_call(
+        cfg, grid_dim_nnz, wg_size, 0, exec->get_queue(), num_rows, num_cols,
+        stride, source->get_const_values(), row_ptrs);
 
     components::prefix_sum(exec, row_ptrs, num_rows + 1);
 
@@ -1222,7 +1218,7 @@ void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
 
     if (grid_dim > 0) {
         kernel::calculate_slice_lengths_call(
-            grid_dim, sg_size, 0, exec->get_queue(), cfg, num_rows, slice_size,
+            cfg, grid_dim, sg_size, 0, exec->get_queue(), num_rows, slice_size,
             slice_num, stride_factor, nnz_per_row.get_const_data(),
             slice_lengths, slice_sets);
     }
@@ -1290,13 +1286,13 @@ void calculate_max_nnz_per_row(std::shared_ptr<const DpcppExecutor> exec,
     auto block_results = Array<size_type>(exec, grid_dim);
 
     kernel::reduce_max_nnz_call(
-        grid_dim, wg_size, wg_size * sizeof(size_type), exec->get_queue(), cfg,
+        cfg, grid_dim, wg_size, wg_size * sizeof(size_type), exec->get_queue(),
         num_rows, nnz_per_row.get_const_data(), block_results.get_data());
 
     auto d_result = Array<size_type>(exec, 1);
 
     kernel::reduce_max_nnz_call(
-        1, wg_size, wg_size * sizeof(size_type), exec->get_queue(), cfg,
+        cfg, 1, wg_size, wg_size * sizeof(size_type), exec->get_queue(),
         grid_dim, block_results.get_const_data(), d_result.get_data());
 
     *result = exec->copy_val_to_host(d_result.get_const_data());
@@ -1326,7 +1322,7 @@ void calculate_nonzeros_per_row(std::shared_ptr<const DpcppExecutor> exec,
     const dim3 grid_size(grid_x, 1, 1);
     if (grid_x > 0) {
         kernel::count_nnz_per_row_call(
-            grid_size, block_size, 0, exec->get_queue(), cfg,
+            cfg, grid_size, block_size, 0, exec->get_queue(),
             source->get_size()[0], source->get_size()[1], source->get_stride(),
             source->get_const_values(), result->get_data());
     }
@@ -1370,7 +1366,7 @@ void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
     auto grid_dim = ceildiv(slice_num * sg_size, wg_size);
 
     kernel::reduce_max_nnz_per_slice_call(
-        grid_dim, wg_size, 0, exec->get_queue(), cfg, num_rows, slice_size,
+        cfg, grid_dim, wg_size, 0, exec->get_queue(), num_rows, slice_size,
         stride_factor, nnz_per_row.get_const_data(),
         max_nnz_per_slice.get_data());
 
@@ -1378,14 +1374,14 @@ void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
     auto block_results = Array<size_type>(exec, grid_dim);
 
     kernel::reduce_total_cols_call(
-        grid_dim, wg_size, wg_size * sizeof(size_type), exec->get_queue(), cfg,
+        cfg, grid_dim, wg_size, wg_size * sizeof(size_type), exec->get_queue(),
         slice_num, max_nnz_per_slice.get_const_data(),
         block_results.get_data());
 
     auto d_result = Array<size_type>(exec, 1);
 
     kernel::reduce_total_cols_call(
-        1, wg_size, wg_size * sizeof(size_type), exec->get_queue(), cfg,
+        cfg, 1, wg_size, wg_size * sizeof(size_type), exec->get_queue(),
         grid_dim, block_results.get_const_data(), d_result.get_data());
 
     *result = exec->copy_val_to_host(d_result.get_const_data());
diff --git a/dpcpp/test/components/prefix_sum.cpp b/dpcpp/test/components/prefix_sum.cpp
index 3e2e7ca9d64..2ae72880443 100644
--- a/dpcpp/test/components/prefix_sum.cpp
+++ b/dpcpp/test/components/prefix_sum.cpp
@@ -44,9 +44,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/array.hpp>
 
 
-#include "dpcpp/test/utils.hpp"
-
-
 namespace {
 
 
diff --git a/dpcpp/test/matrix/dense_kernels.cpp b/dpcpp/test/matrix/dense_kernels.cpp
index 3cd080313cf..e47de0a6487 100644
--- a/dpcpp/test/matrix/dense_kernels.cpp
+++ b/dpcpp/test/matrix/dense_kernels.cpp
@@ -50,7 +50,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/components/fill_array.hpp"
 #include "core/matrix/dense_kernels.hpp"
-#include "dpcpp/test/utils.hpp"
 
 
 namespace {
diff --git a/dpcpp/test/utils.hpp b/dpcpp/test/utils.hpp
deleted file mode 100644
index 88d98f0d9f6..00000000000
--- a/dpcpp/test/utils.hpp
+++ /dev/null
@@ -1,54 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GKO_DPCPP_TEST_UTILS_HPP_
-#define GKO_DPCPP_TEST_UTILS_HPP_
-
-
-#include "core/test/utils.hpp"
-
-
-#include <ginkgo/core/base/executor.hpp>
-
-
-namespace {
-
-
-// prevent device reset after each test
-auto no_reset_exec =
-    gko::DpcppExecutor::create(0, gko::ReferenceExecutor::create());
-
-
-}  // namespace
-
-
-#endif  // GKO_DPCPP_TEST_UTILS_HPP_

From ed501f122a74c07594f0290ec2eae6686287e4ec Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 1 Jun 2021 17:40:52 +0200
Subject: [PATCH 0900/1008] update lateset ConfigSet and dense kernel/test

---
 dpcpp/base/helper.hpp                |   3 +-
 dpcpp/components/prefix_sum.dp.cpp   |   7 +-
 dpcpp/components/prefix_sum.dp.hpp   |   9 +-
 dpcpp/components/reduction.dp.hpp    |  11 +-
 dpcpp/matrix/dense_kernels.dp.cpp    | 374 ++++++++++++++---------
 dpcpp/test/components/prefix_sum.cpp |   3 +
 dpcpp/test/matrix/dense_kernels.cpp  | 436 ++++++++++++++++++++++-----
 7 files changed, 622 insertions(+), 221 deletions(-)

diff --git a/dpcpp/base/helper.hpp b/dpcpp/base/helper.hpp
index 3979caa905c..8c7f45e5174 100644
--- a/dpcpp/base/helper.hpp
+++ b/dpcpp/base/helper.hpp
@@ -44,6 +44,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
+#include "core/base/types.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
 
 
@@ -142,7 +143,7 @@ bool validate(sycl::queue *queue, unsigned workgroup_size,
 
 
 template <typename IterArr, typename Validate>
-ConfigSetType get_first_cfg(IterArr &arr, Validate verify)
+std::uint32_t get_first_cfg(IterArr &arr, Validate verify)
 {
     for (auto &cfg : arr) {
         if (verify(cfg)) {
diff --git a/dpcpp/components/prefix_sum.dp.cpp b/dpcpp/components/prefix_sum.dp.cpp
index 330fa297e58..07cdb5b38aa 100644
--- a/dpcpp/components/prefix_sum.dp.cpp
+++ b/dpcpp/components/prefix_sum.dp.cpp
@@ -39,6 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/types.hpp>
 
 
+#include "core/base/types.hpp"
 #include "dpcpp/base/helper.hpp"
 #include "dpcpp/components/prefix_sum.dp.hpp"
 
@@ -52,7 +53,7 @@ namespace components {
 using BlockCfg = ConfigSet<11>;
 
 constexpr auto block_cfg_list =
-    ::gko::syn::value_list<ConfigSetType, BlockCfg::encode(512),
+    ::gko::syn::value_list<std::uint32_t, BlockCfg::encode(512),
                            BlockCfg::encode(256), BlockCfg::encode(128)>();
 
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(start_prefix_sum, start_prefix_sum)
@@ -73,8 +74,8 @@ void prefix_sum(std::shared_ptr<const DpcppExecutor> exec, IndexType *counts,
     if (num_entries > 0) {
         auto queue = exec->get_queue();
         constexpr auto block_cfg_array = as_array(block_cfg_list);
-        const ConfigSetType cfg =
-            get_first_cfg(block_cfg_array, [&queue](ConfigSetType cfg) {
+        const std::uint32_t cfg =
+            get_first_cfg(block_cfg_array, [&queue](std::uint32_t cfg) {
                 return validate(queue, BlockCfg::decode<0>(cfg), 16);
             });
         const auto wg_size = BlockCfg::decode<0>(cfg);
diff --git a/dpcpp/components/prefix_sum.dp.hpp b/dpcpp/components/prefix_sum.dp.hpp
index fd9ff2ac263..f76f85135eb 100644
--- a/dpcpp/components/prefix_sum.dp.hpp
+++ b/dpcpp/components/prefix_sum.dp.hpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <CL/sycl.hpp>
 
 
+#include "core/base/types.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
 #include "dpcpp/base/dpct.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
@@ -125,7 +126,7 @@ __dpct_inline__ void subwarp_prefix_sum(ValueType element,
  * @note To calculate the prefix sum over an array of size bigger than
  *       `block_size`, `finalize_prefix_sum` has to be used as well.
  */
-template <ConfigSetType block_size, typename ValueType>
+template <std::uint32_t block_size, typename ValueType>
 void start_prefix_sum(size_type num_elements, ValueType *__restrict__ elements,
                       ValueType *__restrict__ block_sum,
                       sycl::nd_item<3> item_ct1,
@@ -178,7 +179,7 @@ void start_prefix_sum(size_type num_elements, ValueType *__restrict__ elements,
     }
 }
 
-template <ConfigSetType block_size, typename ValueType>
+template <std::uint32_t block_size, typename ValueType>
 void start_prefix_sum(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                       sycl::queue *stream, size_type num_elements,
                       ValueType *elements, ValueType *block_sum)
@@ -214,7 +215,7 @@ void start_prefix_sum(dim3 grid, dim3 block, size_t dynamic_shared_memory,
  *
  * @note To calculate a prefix sum, first `start_prefix_sum` has to be called.
  */
-template <ConfigSetType block_size, typename ValueType>
+template <std::uint32_t block_size, typename ValueType>
 void finalize_prefix_sum(size_type num_elements,
                          ValueType *__restrict__ elements,
                          const ValueType *__restrict__ block_sum,
@@ -231,7 +232,7 @@ void finalize_prefix_sum(size_type num_elements,
     }
 }
 
-template <ConfigSetType block_size, typename ValueType>
+template <std::uint32_t block_size, typename ValueType>
 void finalize_prefix_sum(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                          sycl::queue *stream, size_type num_elements,
                          ValueType *elements, const ValueType *block_sum)
diff --git a/dpcpp/components/reduction.dp.hpp b/dpcpp/components/reduction.dp.hpp
index d3e925ee4ba..9c2387a7113 100644
--- a/dpcpp/components/reduction.dp.hpp
+++ b/dpcpp/components/reduction.dp.hpp
@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/synthesizer/containers.hpp>
 
 
+#include "core/base/types.hpp"
 #include "core/synthesizer/implementation_selection.hpp"
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
@@ -63,7 +64,7 @@ namespace dpcpp {
 constexpr int default_block_size = 256;
 using KCFG_1D = ConfigSet<11, 7>;
 constexpr auto kcfg_1d_list =
-    syn::value_list<ConfigSetType, KCFG_1D::encode(512, 64),
+    syn::value_list<std::uint32_t, KCFG_1D::encode(512, 64),
                     KCFG_1D::encode(512, 32), KCFG_1D::encode(512, 16),
                     KCFG_1D::encode(256, 32), KCFG_1D::encode(256, 16),
                     KCFG_1D::encode(256, 8)>();
@@ -201,7 +202,7 @@ void reduce_array(size_type size, const ValueType *__restrict__ source,
  * `source` of any size. Has to be called a second time on `result` to reduce
  * an array larger than `block_size`.
  */
-template <ConfigSetType cfg, typename ValueType>
+template <std::uint32_t cfg, typename ValueType>
 void reduce_add_array(
     size_type size, const ValueType *__restrict__ source,
     ValueType *__restrict__ result, sycl::nd_item<3> item_ct1,
@@ -216,7 +217,7 @@ void reduce_add_array(
     }
 }
 
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void reduce_add_array(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                       sycl::queue *stream, size_type size,
                       const ValueType *source, ValueType *result)
@@ -263,8 +264,8 @@ ValueType reduce_add_array(std::shared_ptr<const DpcppExecutor> exec,
     ValueType answer = zero<ValueType>();
     auto queue = exec->get_queue();
     constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
-    const ConfigSetType cfg =
-        get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+    const std::uint32_t cfg =
+        get_first_cfg(kcfg_1d_array, [&queue](std::uint32_t cfg) {
             return validate(queue, KCFG_1D::decode<0>(cfg),
                             KCFG_1D::decode<1>(cfg));
         });
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index e83c5dba5cc..58f688951c9 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/dense_kernels.hpp"
 
 
-#include <iostream>
-
-
 #include <CL/sycl.hpp>
 #include <oneapi/mkl.hpp>
 
@@ -70,12 +67,14 @@ namespace dpcpp {
  */
 namespace dense {
 
+
 using KCFG_1D = ConfigSet<11, 7>;
 constexpr auto kcfg_1d_list =
-    syn::value_list<ConfigSetType, KCFG_1D::encode(512, 64),
+    syn::value_list<std::uint32_t, KCFG_1D::encode(512, 64),
                     KCFG_1D::encode(512, 32), KCFG_1D::encode(512, 16),
                     KCFG_1D::encode(256, 32), KCFG_1D::encode(256, 16),
                     KCFG_1D::encode(256, 8)>();
+constexpr auto subgroup_list = syn::value_list<std::uint32_t, 64, 32, 16, 8>();
 constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
 constexpr auto default_block_size = 256;
 
@@ -119,6 +118,7 @@ void scale(size_type num_rows, size_type num_cols, size_type num_alpha_cols,
 
 GKO_ENABLE_DEFAULT_HOST(scale, scale)
 
+
 template <typename ValueType>
 void add_scaled(size_type num_rows, size_type num_cols,
                 size_type num_alpha_cols, const ValueType *__restrict__ alpha,
@@ -157,7 +157,7 @@ void add_scaled_diag(size_type size, const ValueType *__restrict__ alpha,
 GKO_ENABLE_DEFAULT_HOST(add_scaled_diag, add_scaled_diag)
 
 
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename OutType,
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename OutType,
           typename CallableGetValue, typename CallableReduce>
 void compute_partial_reduce(
     size_type num_rows, OutType *__restrict__ work, CallableGetValue get_value,
@@ -191,7 +191,7 @@ void compute_partial_reduce(
 }
 
 
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType,
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType,
           typename CallableReduce, typename CallableFinalize>
 void finalize_reduce_computation(
     size_type size, const ValueType *work, ValueType *result,
@@ -220,7 +220,7 @@ void finalize_reduce_computation(
 }
 
 
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void compute_partial_dot(
     size_type num_rows, const ValueType *__restrict__ x, size_type stride_x,
     const ValueType *__restrict__ y, size_type stride_y,
@@ -230,13 +230,13 @@ void compute_partial_dot(
     compute_partial_reduce<cfg>(
         num_rows, work,
         [x, stride_x, y, stride_y](size_type i) {
-            return x[i * stride_x] * conj(y[i * stride_y]);
+            return x[i * stride_x] * y[i * stride_y];
         },
         [](const ValueType &x, const ValueType &y) { return x + y; }, item_ct1,
         tmp_work);
 }
 
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void compute_partial_dot(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                          sycl::queue *stream, size_type num_rows,
                          const ValueType *x, size_type stride_x,
@@ -244,7 +244,6 @@ void compute_partial_dot(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                          ValueType *work)
 {
     constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
-    std::cout << "partial " << cfg << std::endl;
     stream->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<ValueType, wg_size>, 0,
                        sycl::access::mode::read_write,
@@ -267,8 +266,54 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(compute_partial_dot_call, compute_partial_dot,
                                kcfg_1d_list)
 
 
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
-void finalize_dot_computation(
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+void compute_partial_conj_dot(
+    size_type num_rows, const ValueType *__restrict__ x, size_type stride_x,
+    const ValueType *__restrict__ y, size_type stride_y,
+    ValueType *__restrict__ work, sycl::nd_item<3> item_ct1,
+    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *tmp_work)
+{
+    compute_partial_reduce<cfg>(
+        num_rows, work,
+        [x, stride_x, y, stride_y](size_type i) {
+            return conj(x[i * stride_x]) * y[i * stride_y];
+        },
+        [](const ValueType &x, const ValueType &y) { return x + y; }, item_ct1,
+        tmp_work);
+}
+
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+void compute_partial_conj_dot(dim3 grid, dim3 block,
+                              size_t dynamic_shared_memory, sycl::queue *stream,
+                              size_type num_rows, const ValueType *x,
+                              size_type stride_x, const ValueType *y,
+                              size_type stride_y, ValueType *work)
+{
+    constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<ValueType, wg_size>, 0,
+                       sycl::access::mode::read_write,
+                       sycl::access::target::local>
+            tmp_work_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                compute_partial_conj_dot<cfg>(
+                    num_rows, x, stride_x, y, stride_y, work, item_ct1,
+                    (UninitializedArray<ValueType, wg_size> *)
+                        tmp_work_acc_ct1.get_pointer());
+            });
+    });
+}
+
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(compute_partial_conj_dot,
+                                           compute_partial_conj_dot)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(compute_partial_conj_dot_call,
+                               compute_partial_conj_dot, kcfg_1d_list)
+
+
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+void finalize_sum_reduce_computation(
     size_type size, const ValueType *work, ValueType *result,
     sycl::nd_item<3> item_ct1,
     UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *tmp_work)
@@ -279,14 +324,13 @@ void finalize_dot_computation(
         [](const ValueType &x) { return x; }, item_ct1, tmp_work);
 }
 
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
-void finalize_dot_computation(dim3 grid, dim3 block,
-                              size_t dynamic_shared_memory, sycl::queue *stream,
-                              size_type size, const ValueType *work,
-                              ValueType *result)
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+void finalize_sum_reduce_computation(dim3 grid, dim3 block,
+                                     size_t dynamic_shared_memory,
+                                     sycl::queue *stream, size_type size,
+                                     const ValueType *work, ValueType *result)
 {
     constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
-    std::cout << "finalize " << cfg << std::endl;
     stream->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<ValueType, wg_size>, 0,
                        sycl::access::mode::read_write,
@@ -295,7 +339,7 @@ void finalize_dot_computation(dim3 grid, dim3 block,
 
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
-                             finalize_dot_computation<cfg>(
+                             finalize_sum_reduce_computation<cfg>(
                                  size, work, result, item_ct1,
                                  (UninitializedArray<ValueType, wg_size> *)
                                      tmp_work_acc_ct1.get_pointer());
@@ -303,13 +347,13 @@ void finalize_dot_computation(dim3 grid, dim3 block,
     });
 }
 
-GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(finalize_dot_computation,
-                                           finalize_dot_computation)
-GKO_ENABLE_DEFAULT_CONFIG_CALL(finalize_dot_computation_call,
-                               finalize_dot_computation, kcfg_1d_list)
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(finalize_sum_reduce_computation,
+                                           finalize_sum_reduce_computation)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(finalize_sum_reduce_computation_call,
+                               finalize_sum_reduce_computation, kcfg_1d_list)
 
 
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void compute_partial_norm2(
     size_type num_rows, const ValueType *__restrict__ x, size_type stride_x,
     remove_complex<ValueType> *__restrict__ work, sycl::nd_item<3> item_ct1,
@@ -324,7 +368,7 @@ void compute_partial_norm2(
         tmp_work);
 }
 
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
 void compute_partial_norm2(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                            sycl::queue *stream, size_type num_rows,
                            const ValueType *x, size_type stride_x,
@@ -353,8 +397,8 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(compute_partial_norm2_call,
                                compute_partial_norm2, kcfg_1d_list)
 
 
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
-void finalize_norm2_computation(
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+void finalize_sqrt_reduce_computation(
     size_type size, const ValueType *work, ValueType *result,
     sycl::nd_item<3> item_ct1,
     UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *tmp_work)
@@ -362,14 +406,14 @@ void finalize_norm2_computation(
     finalize_reduce_computation<cfg>(
         size, work, result,
         [](const ValueType &x, const ValueType &y) { return x + y; },
-        [](const ValueType &x) { return sqrt(x); }, item_ct1, tmp_work);
+        [](const ValueType &x) { return std::sqrt(x); }, item_ct1, tmp_work);
 }
 
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32), typename ValueType>
-void finalize_norm2_computation(dim3 grid, dim3 block,
-                                size_t dynamic_shared_memory,
-                                sycl::queue *stream, size_type size,
-                                const ValueType *work, ValueType *result)
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+void finalize_sqrt_reduce_computation(dim3 grid, dim3 block,
+                                      size_t dynamic_shared_memory,
+                                      sycl::queue *stream, size_type size,
+                                      const ValueType *work, ValueType *result)
 {
     constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
     stream->submit([&](sycl::handler &cgh) {
@@ -381,7 +425,7 @@ void finalize_norm2_computation(dim3 grid, dim3 block,
 
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
-                             finalize_norm2_computation<cfg>(
+                             finalize_sqrt_reduce_computation<cfg>(
                                  size, work, result, item_ct1,
                                  (UninitializedArray<ValueType, wg_size> *)
                                      tmp_work_acc_ct1.get_pointer());
@@ -389,13 +433,13 @@ void finalize_norm2_computation(dim3 grid, dim3 block,
     });
 }
 
-GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(finalize_norm2_computation,
-                                           finalize_norm2_computation)
-GKO_ENABLE_DEFAULT_CONFIG_CALL(finalize_norm2_computation_call,
-                               finalize_norm2_computation, kcfg_1d_list)
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(finalize_sqrt_reduce_computation,
+                                           finalize_sqrt_reduce_computation)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(finalize_sqrt_reduce_computation_call,
+                               finalize_sqrt_reduce_computation, kcfg_1d_list)
 
 
-template <typename ValueType, typename IndexType>
+template <std::uint32_t cfg, typename ValueType, typename IndexType>
 void fill_in_coo(size_type num_rows, size_type num_cols, size_type stride,
                  const size_type *__restrict__ row_ptrs,
                  const ValueType *__restrict__ source,
@@ -418,10 +462,12 @@ void fill_in_coo(size_type num_rows, size_type num_cols, size_type stride,
     }
 }
 
-GKO_ENABLE_DEFAULT_HOST(fill_in_coo, fill_in_coo)
+GKO_ENABLE_DEFAULT_HOST_CONFIG(fill_in_coo, fill_in_coo)
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(fill_in_coo, fill_in_coo)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(fill_in_coo_call, fill_in_coo, kcfg_1d_list)
 
 
-template <ConfigSetType cfg, typename ValueType, typename IndexType>
+template <std::uint32_t cfg, typename ValueType, typename IndexType>
 void count_nnz_per_row(size_type num_rows, size_type num_cols, size_type stride,
                        const ValueType *__restrict__ work,
                        IndexType *__restrict__ result,
@@ -451,7 +497,7 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(count_nnz_per_row_call, count_nnz_per_row,
                                kcfg_1d_list)
 
 
-template <typename ValueType, typename IndexType>
+template <std::uint32_t cfg, typename ValueType, typename IndexType>
 void fill_in_csr(size_type num_rows, size_type num_cols, size_type stride,
                  const ValueType *__restrict__ source,
                  IndexType *__restrict__ row_ptrs,
@@ -472,10 +518,12 @@ void fill_in_csr(size_type num_rows, size_type num_cols, size_type stride,
     }
 }
 
-GKO_ENABLE_DEFAULT_HOST(fill_in_csr, fill_in_csr)
+GKO_ENABLE_DEFAULT_HOST_CONFIG(fill_in_csr, fill_in_csr)
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(fill_in_csr, fill_in_csr)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(fill_in_csr_call, fill_in_csr, kcfg_1d_list)
 
 
-template <typename ValueType, typename IndexType>
+template <std::uint32_t cfg, typename ValueType, typename IndexType>
 void fill_in_ell(size_type num_rows, size_type num_cols,
                  size_type source_stride, const ValueType *__restrict__ source,
                  size_type max_nnz_per_row, size_type result_stride,
@@ -505,10 +553,12 @@ void fill_in_ell(size_type num_rows, size_type num_cols,
     }
 }
 
-GKO_ENABLE_DEFAULT_HOST(fill_in_ell, fill_in_ell)
+GKO_ENABLE_DEFAULT_HOST_CONFIG(fill_in_ell, fill_in_ell)
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(fill_in_ell, fill_in_ell)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(fill_in_ell_call, fill_in_ell, kcfg_1d_list)
 
 
-template <ConfigSetType cfg>
+template <std::uint32_t cfg>
 void calculate_slice_lengths(size_type num_rows, size_type slice_size,
                              int slice_num, size_type stride_factor,
                              const size_type *__restrict__ nnz_per_row,
@@ -516,7 +566,7 @@ void calculate_slice_lengths(size_type num_rows, size_type slice_size,
                              size_type *__restrict__ slice_sets,
                              sycl::nd_item<3> item_ct1)
 {
-    constexpr auto sg_size = KCFG_1D::decode<1>(cfg);
+    constexpr auto sg_size = cfg;
     const auto sliceid = item_ct1.get_group(2);
     const auto tid_in_warp = item_ct1.get_local_id(2);
 
@@ -548,10 +598,10 @@ GKO_ENABLE_DEFAULT_HOST_CONFIG(calculate_slice_lengths, calculate_slice_lengths)
 GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(calculate_slice_lengths,
                                            calculate_slice_lengths)
 GKO_ENABLE_DEFAULT_CONFIG_CALL(calculate_slice_lengths_call,
-                               calculate_slice_lengths, kcfg_1d_list)
+                               calculate_slice_lengths, subgroup_list)
 
 
-template <typename ValueType, typename IndexType>
+template <std::uint32_t cfg, typename ValueType, typename IndexType>
 void fill_in_sellp(size_type num_rows, size_type num_cols, size_type slice_size,
                    size_type stride, const ValueType *__restrict__ source,
                    size_type *__restrict__ slice_lengths,
@@ -584,9 +634,12 @@ void fill_in_sellp(size_type num_rows, size_type num_cols, size_type slice_size,
     }
 }
 
-GKO_ENABLE_DEFAULT_HOST(fill_in_sellp, fill_in_sellp)
+GKO_ENABLE_DEFAULT_HOST_CONFIG(fill_in_sellp, fill_in_sellp)
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(fill_in_sellp, fill_in_sellp)
+GKO_ENABLE_DEFAULT_CONFIG_CALL(fill_in_sellp_call, fill_in_sellp, kcfg_1d_list)
 
-template <ConfigSetType cfg>
+
+template <std::uint32_t cfg>
 void reduce_max_nnz(size_type size, const size_type *__restrict__ nnz_per_row,
                     size_type *__restrict__ result, sycl::nd_item<3> item_ct1,
                     uint8_t *dpct_local)
@@ -603,7 +656,7 @@ void reduce_max_nnz(size_type size, const size_type *__restrict__ nnz_per_row,
     }
 }
 
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32)>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32)>
 void reduce_max_nnz(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                     sycl::queue *stream, size_type size,
                     const size_type *nnz_per_row, size_type *result)
@@ -626,7 +679,8 @@ GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(reduce_max_nnz, reduce_max_nnz);
 GKO_ENABLE_DEFAULT_CONFIG_CALL(reduce_max_nnz_call, reduce_max_nnz,
                                kcfg_1d_list)
 
-template <ConfigSetType cfg>
+
+template <std::uint32_t cfg>
 void reduce_max_nnz_per_slice(size_type num_rows, size_type slice_size,
                               size_type stride_factor,
                               const size_type *__restrict__ nnz_per_row,
@@ -665,7 +719,7 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(reduce_max_nnz_per_slice_call,
                                reduce_max_nnz_per_slice, kcfg_1d_list)
 
 
-template <ConfigSetType cfg>
+template <std::uint32_t cfg>
 void reduce_total_cols(size_type num_slices,
                        const size_type *__restrict__ max_nnz_per_slice,
                        size_type *__restrict__ result,
@@ -682,7 +736,7 @@ void reduce_total_cols(size_type num_slices,
     }
 }
 
-template <ConfigSetType cfg = KCFG_1D::encode(256, 32)>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 32)>
 void reduce_total_cols(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                        sycl::queue *stream, size_type num_slices,
                        const size_type *max_nnz_per_slice, size_type *result)
@@ -960,6 +1014,34 @@ void apply(std::shared_ptr<const DpcppExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
+namespace {
+
+
+#define GKO_BIND_DOT(ValueType, Name, Func)                                  \
+    void Name(::cl::sycl::queue &exec_queue, std::int64_t n,                 \
+              const ValueType *x, std::int64_t incx, const ValueType *y,     \
+              std::int64_t incy, ValueType *result)                          \
+    {                                                                        \
+        Func(exec_queue, n, x, incx, y, incy, result);                       \
+    }                                                                        \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
+                  "semi-colon warnings")
+
+GKO_BIND_DOT(float, dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(double, dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(std::complex<float>, dot, oneapi::mkl::blas::row_major::dotu);
+GKO_BIND_DOT(std::complex<double>, dot, oneapi::mkl::blas::row_major::dotu);
+GKO_BIND_DOT(float, conj_dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(double, conj_dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(std::complex<float>, conj_dot, oneapi::mkl::blas::row_major::dotc);
+GKO_BIND_DOT(std::complex<double>, conj_dot,
+             oneapi::mkl::blas::row_major::dotc);
+
+
+}  // namespace
+
+
 template <typename ValueType>
 void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
                  const matrix::Dense<ValueType> *x,
@@ -981,15 +1063,13 @@ void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
         constexpr auto work_per_thread = 32;
         auto queue = exec->get_queue();
         constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
-        const ConfigSetType cfg =
-            get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+        const std::uint32_t cfg =
+            get_first_cfg(kcfg_1d_array, [&queue](std::uint32_t cfg) {
                 return validate(queue, KCFG_1D::decode<0>(cfg),
                                 KCFG_1D::decode<1>(cfg));
             });
         const auto wg_size = KCFG_1D::decode<0>(cfg);
         const auto sg_size = KCFG_1D::decode<1>(cfg);
-        std::cout << "dot " << cfg << " " << wg_size << " " << sg_size
-                  << std::endl;
         const auto work_per_block = work_per_thread * wg_size;
         const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
         const dim3 block_dim{sg_size, 1, wg_size / sg_size};
@@ -1000,7 +1080,7 @@ void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
                 cfg, grid_dim, block_dim, 0, exec->get_queue(),
                 x->get_size()[0], x->get_const_values() + col, x->get_stride(),
                 y->get_const_values() + col, y->get_stride(), work.get_data());
-            kernel::finalize_dot_computation_call(
+            kernel::finalize_sum_reduce_computation_call(
                 cfg, 1, block_dim, 0, exec->get_queue(), grid_dim.x,
                 work.get_const_data(), result->get_values() + col);
         }
@@ -1014,7 +1094,47 @@ template <typename ValueType>
 void compute_conj_dot(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Dense<ValueType> *x,
                       const matrix::Dense<ValueType> *y,
-                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
+                      matrix::Dense<ValueType> *result)
+{
+    if (0) {
+        // TODO: write a custom kernel which does this more efficiently
+        for (size_type col = 0; col < x->get_size()[1]; ++col) {
+            conj_dot(*exec->get_queue(), x->get_size()[0],
+                     x->get_const_values() + col, x->get_stride(),
+                     y->get_const_values() + col, y->get_stride(),
+                     result->get_values() + col);
+        }
+    } else {
+        // TODO: these are tuning parameters obtained experimentally, once
+        // we decide how to handle this uniformly, they should be modified
+        // appropriately
+        constexpr auto work_per_thread = 32;
+        auto queue = exec->get_queue();
+        constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
+        const std::uint32_t cfg =
+            get_first_cfg(kcfg_1d_array, [&queue](std::uint32_t cfg) {
+                return validate(queue, KCFG_1D::decode<0>(cfg),
+                                KCFG_1D::decode<1>(cfg));
+            });
+        const auto wg_size = KCFG_1D::decode<0>(cfg);
+        const auto sg_size = KCFG_1D::decode<1>(cfg);
+
+        const auto work_per_block = work_per_thread * wg_size;
+        const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
+        const dim3 block_dim{sg_size, 1, wg_size / sg_size};
+        Array<ValueType> work(exec, grid_dim.x);
+        // TODO: write a kernel which does this more efficiently
+        for (size_type col = 0; col < x->get_size()[1]; ++col) {
+            kernel::compute_partial_conj_dot_call(
+                cfg, grid_dim, block_dim, 0, exec->get_queue(),
+                x->get_size()[0], x->get_const_values() + col, x->get_stride(),
+                y->get_const_values() + col, y->get_stride(), work.get_data());
+            kernel::finalize_sum_reduce_computation_call(
+                cfg, 1, block_dim, 0, exec->get_queue(), grid_dim.x,
+                work.get_const_data(), result->get_values() + col);
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_CONJ_DOT_KERNEL);
 
@@ -1039,8 +1159,8 @@ void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
         constexpr auto work_per_thread = 32;
         auto queue = exec->get_queue();
         constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
-        const ConfigSetType cfg =
-            get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+        const std::uint32_t cfg =
+            get_first_cfg(kcfg_1d_array, [&queue](std::uint32_t cfg) {
                 return validate(queue, KCFG_1D::decode<0>(cfg),
                                 KCFG_1D::decode<1>(cfg));
             });
@@ -1057,7 +1177,7 @@ void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
                 cfg, grid_dim, block_dim, 0, exec->get_queue(),
                 x->get_size()[0], x->get_const_values() + col, x->get_stride(),
                 work.get_data());
-            kernel::finalize_norm2_computation_call(
+            kernel::finalize_sqrt_reduce_computation_call(
                 cfg, 1, block_dim, 0, exec->get_queue(), grid_dim.x,
                 work.get_const_data(), result->get_values() + col);
         }
@@ -1086,12 +1206,21 @@ void convert_to_coo(std::shared_ptr<const DpcppExecutor> exec,
 
     components::prefix_sum(exec, nnz_prefix_sum.get_data(), num_rows);
 
-    size_type grid_dim = ceildiv(num_rows, default_block_size);
+    auto queue = exec->get_queue();
+    constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
+    const std::uint32_t cfg =
+        get_first_cfg(kcfg_1d_array, [&queue](std::uint32_t cfg) {
+            return validate(queue, KCFG_1D::decode<0>(cfg),
+                            KCFG_1D::decode<1>(cfg));
+        });
+    const auto wg_size = KCFG_1D::decode<0>(cfg);
+    const auto sg_size = KCFG_1D::decode<1>(cfg);
+    size_type grid_dim = ceildiv(num_rows, wg_size);
 
-    kernel::fill_in_coo(grid_dim, default_block_size, 0, exec->get_queue(),
-                        num_rows, num_cols, stride,
-                        nnz_prefix_sum.get_const_data(),
-                        source->get_const_values(), row_idxs, col_idxs, values);
+    kernel::fill_in_coo_call(
+        cfg, grid_dim, wg_size, 0, exec->get_queue(), num_rows, num_cols,
+        stride, nnz_prefix_sum.get_const_data(), source->get_const_values(),
+        row_idxs, col_idxs, values);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -1105,8 +1234,8 @@ void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
 {
     auto queue = exec->get_queue();
     constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
-    const ConfigSetType cfg =
-        get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+    const std::uint32_t cfg =
+        get_first_cfg(kcfg_1d_array, [&queue](std::uint32_t cfg) {
             return validate(queue, KCFG_1D::decode<0>(cfg),
                             KCFG_1D::decode<1>(cfg));
         });
@@ -1133,9 +1262,10 @@ void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
 
     size_type grid_dim = ceildiv(num_rows, wg_size);
 
-    kernel::fill_in_csr(grid_dim, wg_size, 0, exec->get_queue(), num_rows,
-                        num_cols, stride, source->get_const_values(), row_ptrs,
-                        col_idxs, values);
+    kernel::fill_in_csr_call(cfg, grid_dim, default_block_size, 0,
+                             exec->get_queue(), num_rows, num_cols, stride,
+                             source->get_const_values(), row_ptrs, col_idxs,
+                             values);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -1157,11 +1287,20 @@ void convert_to_ell(std::shared_ptr<const DpcppExecutor> exec,
     auto source_stride = source->get_stride();
     auto result_stride = result->get_stride();
 
-    auto grid_dim = ceildiv(result_stride, default_block_size);
-    kernel::fill_in_ell(grid_dim, default_block_size, 0, exec->get_queue(),
-                        num_rows, num_cols, source_stride,
-                        source->get_const_values(), max_nnz_per_row,
-                        result_stride, col_ptrs, values);
+    auto queue = exec->get_queue();
+    constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
+    const std::uint32_t cfg =
+        get_first_cfg(kcfg_1d_array, [&queue](std::uint32_t cfg) {
+            return validate(queue, KCFG_1D::decode<0>(cfg),
+                            KCFG_1D::decode<1>(cfg));
+        });
+    const auto wg_size = KCFG_1D::decode<0>(cfg);
+    const auto sg_size = KCFG_1D::decode<1>(cfg);
+    auto grid_dim = ceildiv(result_stride, wg_size);
+    kernel::fill_in_ell_call(cfg, grid_dim, wg_size, 0, exec->get_queue(),
+                             num_rows, num_cols, source_stride,
+                             source->get_const_values(), max_nnz_per_row,
+                             result_stride, col_ptrs, values);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -1185,8 +1324,8 @@ void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
 {
     auto queue = exec->get_queue();
     constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
-    const ConfigSetType cfg =
-        get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+    const std::uint32_t cfg =
+        get_first_cfg(kcfg_1d_array, [&queue](std::uint32_t cfg) {
             return validate(queue, KCFG_1D::decode<0>(cfg),
                             KCFG_1D::decode<1>(cfg));
         });
@@ -1211,24 +1350,25 @@ void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
     const int slice_num = ceildiv(num_rows, slice_size);
 
     auto nnz_per_row = Array<size_type>(exec, num_rows);
-
     calculate_nonzeros_per_row(exec, source, &nnz_per_row);
 
     auto grid_dim = slice_num;
 
     if (grid_dim > 0) {
         kernel::calculate_slice_lengths_call(
-            cfg, grid_dim, sg_size, 0, exec->get_queue(), num_rows, slice_size,
-            slice_num, stride_factor, nnz_per_row.get_const_data(),
+            sg_size, grid_dim, sg_size, 0, exec->get_queue(), num_rows,
+            slice_size, slice_num, stride_factor, nnz_per_row.get_const_data(),
             slice_lengths, slice_sets);
     }
+
     components::prefix_sum(exec, slice_sets, slice_num + 1);
+
     grid_dim = ceildiv(num_rows, wg_size);
     if (grid_dim > 0) {
-        kernel::fill_in_sellp(grid_dim, wg_size, 0, exec->get_queue(), num_rows,
-                              num_cols, slice_size, stride,
-                              source->get_const_values(), slice_lengths,
-                              slice_sets, col_idxs, vals);
+        kernel::fill_in_sellp_call(cfg, grid_dim, wg_size, 0, exec->get_queue(),
+                                   num_rows, num_cols, slice_size, stride,
+                                   source->get_const_values(), slice_lengths,
+                                   slice_sets, col_idxs, vals);
     }
 }
 
@@ -1272,14 +1412,12 @@ void calculate_max_nnz_per_row(std::shared_ptr<const DpcppExecutor> exec,
     calculate_nonzeros_per_row(exec, source, &nnz_per_row);
     auto queue = exec->get_queue();
     constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
-    const ConfigSetType cfg =
-        get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+    const std::uint32_t cfg =
+        get_first_cfg(kcfg_1d_array, [&queue](std::uint32_t cfg) {
             return validate(queue, KCFG_1D::decode<0>(cfg),
                             KCFG_1D::decode<1>(cfg));
         });
     const auto wg_size = KCFG_1D::decode<0>(cfg);
-    std::cout << "wg_size " << wg_size << "sg_size " << KCFG_1D::decode<1>(cfg)
-              << std::endl;
     const auto n = ceildiv(num_rows, wg_size);
     const size_type grid_dim = (n <= wg_size) ? n : wg_size;
 
@@ -1309,8 +1447,8 @@ void calculate_nonzeros_per_row(std::shared_ptr<const DpcppExecutor> exec,
 {
     auto queue = exec->get_queue();
     constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
-    const ConfigSetType cfg =
-        get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+    const std::uint32_t cfg =
+        get_first_cfg(kcfg_1d_array, [&queue](std::uint32_t cfg) {
             return validate(queue, KCFG_1D::decode<0>(cfg),
                             KCFG_1D::decode<1>(cfg));
         });
@@ -1355,8 +1493,8 @@ void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
     auto max_nnz_per_slice = Array<size_type>(exec, slice_num);
     auto queue = exec->get_queue();
     constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
-    const ConfigSetType cfg =
-        get_first_cfg(kcfg_1d_array, [&queue](ConfigSetType cfg) {
+    const std::uint32_t cfg =
+        get_first_cfg(kcfg_1d_array, [&queue](std::uint32_t cfg) {
             return validate(queue, KCFG_1D::decode<0>(cfg),
                             KCFG_1D::decode<1>(cfg));
         });
@@ -1394,27 +1532,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
 template <typename ValueType>
 void transpose(std::shared_ptr<const DpcppExecutor> exec,
                const matrix::Dense<ValueType> *orig,
-               matrix::Dense<ValueType> *trans)
-{
-    // if (cublas::is_supported<ValueType>::value) {
-    //     auto handle = exec->get_cublas_handle();
-    //     {
-    //         cublas::pointer_mode_guard pm_guard(handle);
-    //         auto alpha = one<ValueType>();
-    //         auto beta = zero<ValueType>();
-    //         cublas::geam(
-    //             handle, oneapi::mkl::transpose::trans,
-    //             oneapi::mkl::transpose::nontrans, orig->get_size()[0],
-    //             orig->get_size()[1], &alpha, orig->get_const_values(),
-    //             orig->get_stride(), &beta, static_cast<ValueType
-    //             *>(nullptr), trans->get_size()[1], trans->get_values(),
-    //             trans->get_stride());
-    //     }
-    // } else {
-    //     GKO_NOT_IMPLEMENTED;
-    // }
-    GKO_NOT_IMPLEMENTED;
-};
+               matrix::Dense<ValueType> *trans) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_TRANSPOSE_KERNEL);
 
@@ -1422,27 +1540,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_TRANSPOSE_KERNEL);
 template <typename ValueType>
 void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Dense<ValueType> *orig,
-                    matrix::Dense<ValueType> *trans)
-{
-    // if (cublas::is_supported<ValueType>::value) {
-    //     auto handle = exec->get_cublas_handle();
-    //     {
-    //         cublas::pointer_mode_guard pm_guard(handle);
-    //         auto alpha = one<ValueType>();
-    //         auto beta = zero<ValueType>();
-    //         cublas::geam(
-    //             handle, oneapi::mkl::transpose::conjtrans,
-    //             oneapi::mkl::transpose::nontrans, orig->get_size()[0],
-    //             orig->get_size()[1], &alpha, orig->get_const_values(),
-    //             orig->get_stride(), &beta, static_cast<ValueType
-    //             *>(nullptr), trans->get_size()[1], trans->get_values(),
-    //             trans->get_stride());
-    //     }
-    // } else {
-    //     GKO_NOT_IMPLEMENTED;
-    // }
-    GKO_NOT_IMPLEMENTED;
-}
+                    matrix::Dense<ValueType> *trans) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
diff --git a/dpcpp/test/components/prefix_sum.cpp b/dpcpp/test/components/prefix_sum.cpp
index 2ae72880443..402192d0b77 100644
--- a/dpcpp/test/components/prefix_sum.cpp
+++ b/dpcpp/test/components/prefix_sum.cpp
@@ -44,6 +44,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/array.hpp>
 
 
+#include "core/test/utils.hpp"
+
+
 namespace {
 
 
diff --git a/dpcpp/test/matrix/dense_kernels.cpp b/dpcpp/test/matrix/dense_kernels.cpp
index e47de0a6487..2b9af16732a 100644
--- a/dpcpp/test/matrix/dense_kernels.cpp
+++ b/dpcpp/test/matrix/dense_kernels.cpp
@@ -50,6 +50,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/components/fill_array.hpp"
 #include "core/matrix/dense_kernels.hpp"
+#include "core/test/utils.hpp"
 
 
 namespace {
@@ -64,9 +65,12 @@ class Dense : public ::testing::Test {
     using vtype = double;
 #endif  // GINKGO_DPCPP_SINGLE_MODE
     using Mtx = gko::matrix::Dense<vtype>;
+    using MixedMtx = gko::matrix::Dense<gko::next_precision<vtype>>;
     using NormVector = gko::matrix::Dense<gko::remove_complex<vtype>>;
     using Arr = gko::Array<itype>;
-    // using ComplexMtx = gko::matrix::Dense<std::complex<vtype>>;
+    using ComplexMtx = gko::matrix::Dense<std::complex<vtype>>;
+    using MixedComplexMtx =
+        gko::matrix::Dense<gko::next_precision<std::complex<vtype>>>;
 
     Dense() : rand_engine(15) {}
 
@@ -116,15 +120,16 @@ class Dense : public ::testing::Test {
     void set_up_apply_data()
     {
         x = gen_mtx<Mtx>(65, 25);
-        // c_x = gen_mtx<ComplexMtx>(65, 25);
+        c_x = gen_mtx<ComplexMtx>(65, 25);
         y = gen_mtx<Mtx>(25, 35);
         expected = gen_mtx<Mtx>(65, 35);
         alpha = gko::initialize<Mtx>({2.0}, ref);
         beta = gko::initialize<Mtx>({-1.0}, ref);
+        square = gen_mtx<Mtx>(x->get_size()[0], x->get_size()[0]);
         dx = Mtx::create(dpcpp);
         dx->copy_from(x.get());
-        // dc_x = ComplexMtx::create(dpcpp);
-        // dc_x->copy_from(c_x.get());
+        dc_x = ComplexMtx::create(dpcpp);
+        dc_x->copy_from(c_x.get());
         dy = Mtx::create(dpcpp);
         dy->copy_from(y.get());
         dresult = Mtx::create(dpcpp);
@@ -133,6 +138,8 @@ class Dense : public ::testing::Test {
         dalpha->copy_from(alpha.get());
         dbeta = Mtx::create(dpcpp);
         dbeta->copy_from(beta.get());
+        dsquare = Mtx::create(dpcpp);
+        dsquare->copy_from(square.get());
 
         std::vector<itype> tmp(x->get_size()[0], 0);
         auto rng = std::default_random_engine{};
@@ -141,14 +148,25 @@ class Dense : public ::testing::Test {
         std::vector<itype> tmp2(x->get_size()[1], 0);
         std::iota(tmp2.begin(), tmp2.end(), 0);
         std::shuffle(tmp2.begin(), tmp2.end(), rng);
+        std::vector<itype> tmp3(x->get_size()[0] / 10);
+        std::uniform_int_distribution<itype> row_dist(0, x->get_size()[0] - 1);
+        for (auto &i : tmp3) {
+            i = row_dist(rng);
+        }
         rpermute_idxs =
             std::unique_ptr<Arr>(new Arr{ref, tmp.begin(), tmp.end()});
-        drpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{dpcpp, tmp.begin(), tmp.end()});
         cpermute_idxs =
             std::unique_ptr<Arr>(new Arr{ref, tmp2.begin(), tmp2.end()});
-        dcpermute_idxs =
-            std::unique_ptr<Arr>(new Arr{dpcpp, tmp2.begin(), tmp2.end()});
+        rgather_idxs =
+            std::unique_ptr<Arr>(new Arr{ref, tmp3.begin(), tmp3.end()});
+    }
+
+    template <typename ConvertedType, typename InputType>
+    std::unique_ptr<ConvertedType> convert(InputType &&input)
+    {
+        auto result = ConvertedType::create(input->get_executor());
+        input->convert_to(result.get());
+        return result;
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -157,21 +175,22 @@ class Dense : public ::testing::Test {
     std::ranlux48 rand_engine;
 
     std::unique_ptr<Mtx> x;
-    // std::unique_ptr<ComplexMtx> c_x;
+    std::unique_ptr<ComplexMtx> c_x;
     std::unique_ptr<Mtx> y;
     std::unique_ptr<Mtx> alpha;
     std::unique_ptr<Mtx> beta;
     std::unique_ptr<Mtx> expected;
+    std::unique_ptr<Mtx> square;
     std::unique_ptr<Mtx> dresult;
     std::unique_ptr<Mtx> dx;
-    // std::unique_ptr<ComplexMtx> dc_x;
+    std::unique_ptr<ComplexMtx> dc_x;
     std::unique_ptr<Mtx> dy;
     std::unique_ptr<Mtx> dalpha;
     std::unique_ptr<Mtx> dbeta;
+    std::unique_ptr<Mtx> dsquare;
     std::unique_ptr<Arr> rpermute_idxs;
-    std::unique_ptr<Arr> drpermute_idxs;
     std::unique_ptr<Arr> cpermute_idxs;
-    std::unique_ptr<Arr> dcpermute_idxs;
+    std::unique_ptr<Arr> rgather_idxs;
 };
 
 
@@ -201,7 +220,7 @@ TEST_F(Dense, DpcppStridedFillIsEquivalentToRef)
     dx->fill(42);
     result->copy_from(dx.get());
 
-    GKO_ASSERT_MTX_NEAR(result, x, r<T>::value);
+    GKO_ASSERT_MTX_NEAR(result, x, r<vtype>::value);
 }
 
 
@@ -318,6 +337,28 @@ TEST_F(Dense, MultipleVectorDpcppComputeDotIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, SingleVectorDpcppComputeConjDotIsEquivalentToRef)
+{
+    set_up_vector_data(1);
+
+    x->compute_conj_dot(y.get(), expected.get());
+    dx->compute_conj_dot(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
+}
+
+
+TEST_F(Dense, MultipleVectorDpcppComputeConjDotIsEquivalentToRef)
+{
+    set_up_vector_data(20);
+
+    x->compute_conj_dot(y.get(), expected.get());
+    dx->compute_conj_dot(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
+}
+
+
 TEST_F(Dense, DpcppComputeNorm2IsEquivalentToRef)
 {
     set_up_vector_data(20);
@@ -343,6 +384,23 @@ TEST_F(Dense, SimpleApplyIsEquivalentToRef)
 }
 
 
+#if !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Dense, SimpleApplyMixedIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    x->apply(convert<MixedMtx>(y).get(), convert<MixedMtx>(expected).get());
+    dx->apply(convert<MixedMtx>(dy).get(), convert<MixedMtx>(dresult).get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-7);
+}
+
+
+#endif  // !GINKGO_DPCPP_SINGLE_MODE
+
+
 TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -354,38 +412,142 @@ TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
 }
 
 
-// TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
-// {
-//     set_up_apply_data();
-//     auto complex_b = gen_mtx<ComplexMtx>(25, 1);
-//     auto dcomplex_b = ComplexMtx::create(dpcpp);
-//     dcomplex_b->copy_from(complex_b.get());
-//     auto complex_x = gen_mtx<ComplexMtx>(65, 1);
-//     auto dcomplex_x = ComplexMtx::create(dpcpp);
-//     dcomplex_x->copy_from(complex_x.get());
+#if !GINKGO_DPCPP_SINGLE_MODE
 
-//     x->apply(complex_b.get(), complex_x.get());
-//     dx->apply(dcomplex_b.get(), dcomplex_x.get());
 
-//     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
-// }
+TEST_F(Dense, AdvancedApplyMixedIsEquivalentToRef)
+{
+    set_up_apply_data();
 
+    x->apply(convert<MixedMtx>(alpha).get(), convert<MixedMtx>(y).get(),
+             convert<MixedMtx>(beta).get(), convert<MixedMtx>(expected).get());
+    dx->apply(convert<MixedMtx>(dalpha).get(), convert<MixedMtx>(dy).get(),
+              convert<MixedMtx>(dbeta).get(), convert<MixedMtx>(dresult).get());
 
-// TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
-// {
-//     set_up_apply_data();
-//     auto complex_b = gen_mtx<ComplexMtx>(25, 1);
-//     auto dcomplex_b = ComplexMtx::create(dpcpp);
-//     dcomplex_b->copy_from(complex_b.get());
-//     auto complex_x = gen_mtx<ComplexMtx>(65, 1);
-//     auto dcomplex_x = ComplexMtx::create(dpcpp);
-//     dcomplex_x->copy_from(complex_x.get());
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-7);
+}
 
-//     x->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
-//     dx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
 
-//     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
-// }
+#endif  // !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(25, 1);
+    auto dcomplex_b = ComplexMtx::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(65, 1);
+    auto dcomplex_x = ComplexMtx::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(complex_b.get(), complex_x.get());
+    dx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r<vtype>::value);
+}
+
+
+#if !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Dense, ApplyToMixedComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<MixedComplexMtx>(25, 1);
+    auto dcomplex_b = MixedComplexMtx::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<MixedComplexMtx>(65, 1);
+    auto dcomplex_x = MixedComplexMtx::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(complex_b.get(), complex_x.get());
+    dx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-7);
+}
+
+#endif  // !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(25, 1);
+    auto dcomplex_b = ComplexMtx::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(65, 1);
+    auto dcomplex_x = ComplexMtx::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r<vtype>::value);
+}
+
+
+#if !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Dense, AdvancedApplyToMixedComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<MixedComplexMtx>(25, 1);
+    auto dcomplex_b = MixedComplexMtx::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<MixedComplexMtx>(65, 1);
+    auto dcomplex_x = MixedComplexMtx::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    x->apply(convert<MixedMtx>(alpha).get(), complex_b.get(),
+             convert<MixedMtx>(beta).get(), complex_x.get());
+    dx->apply(convert<MixedMtx>(dalpha).get(), dcomplex_b.get(),
+              convert<MixedMtx>(dbeta).get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-7);
+}
+
+
+#endif  // !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Dense, ComputeDotComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_b = ComplexMtx::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_x = ComplexMtx::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+    auto result = ComplexMtx::create(ref, gko::dim<2>{1, 2});
+    auto dresult = ComplexMtx::create(dpcpp, gko::dim<2>{1, 2});
+
+    complex_b->compute_dot(complex_x.get(), result.get());
+    dcomplex_b->compute_dot(dcomplex_x.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(result, dresult, r<vtype>::value);
+}
+
+
+TEST_F(Dense, ComputeConjDotComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_b = ComplexMtx::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexMtx>(1234, 2);
+    auto dcomplex_x = ComplexMtx::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+    auto result = ComplexMtx::create(ref, gko::dim<2>{1, 2});
+    auto dresult = ComplexMtx::create(dpcpp, gko::dim<2>{1, 2});
+
+    complex_b->compute_conj_dot(complex_x.get(), result.get());
+    dcomplex_b->compute_conj_dot(dcomplex_x.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(result, dresult, r<vtype>::value);
+}
 
 
 // TEST_F(Dense, IsTransposable)
@@ -494,42 +656,42 @@ TEST_F(Dense, MoveToEllIsEquivalentToRef)
 }
 
 
-// TEST_F(Dense, ConvertToSellpIsEquivalentToRef)
-// {
-//     set_up_apply_data();
-//     auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
-//     auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+TEST_F(Dense, ConvertToSellpIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto sellp_mtx = gko::matrix::Sellp<vtype>::create(ref);
+    auto dsellp_mtx = gko::matrix::Sellp<vtype>::create(dpcpp);
 
-//     x->convert_to(sellp_mtx.get());
-//     dx->convert_to(dsellp_mtx.get());
+    x->convert_to(sellp_mtx.get());
+    dx->convert_to(dsellp_mtx.get());
 
-//     GKO_ASSERT_MTX_NEAR(sellp_mtx, dsellp_mtx, 1e-6);
-// }
+    GKO_ASSERT_MTX_NEAR(sellp_mtx, dsellp_mtx, r<vtype>::value);
+}
 
 
-// TEST_F(Dense, MoveToSellpIsEquivalentToRef)
-// {
-//     set_up_apply_data();
-//     auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
-//     auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+TEST_F(Dense, MoveToSellpIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto sellp_mtx = gko::matrix::Sellp<vtype>::create(ref);
+    auto dsellp_mtx = gko::matrix::Sellp<vtype>::create(dpcpp);
 
-//     x->move_to(sellp_mtx.get());
-//     dx->move_to(dsellp_mtx.get());
+    x->move_to(sellp_mtx.get());
+    dx->move_to(dsellp_mtx.get());
 
-//     GKO_ASSERT_MTX_NEAR(sellp_mtx, dsellp_mtx, 1e-6);
-// }
+    GKO_ASSERT_MTX_NEAR(sellp_mtx, dsellp_mtx, r<vtype>::value);
+}
 
 
-// TEST_F(Dense, ConvertsEmptyToSellp)
-// {
-//     auto dempty_mtx = Mtx::create(dpcpp);
-//     auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+TEST_F(Dense, ConvertsEmptyToSellp)
+{
+    auto dempty_mtx = Mtx::create(dpcpp);
+    auto dsellp_mtx = gko::matrix::Sellp<vtype>::create(dpcpp);
 
-//     dempty_mtx->convert_to(dsellp_mtx.get());
+    dempty_mtx->convert_to(dsellp_mtx.get());
 
-//     ASSERT_EQ(dpcpp->copy_val_to_host(dsellp_mtx->get_const_slice_sets()),
-//     0); ASSERT_FALSE(dsellp_mtx->get_size());
-// }
+    ASSERT_EQ(dpcpp->copy_val_to_host(dsellp_mtx->get_const_slice_sets()), 0);
+    ASSERT_FALSE(dsellp_mtx->get_size());
+}
 
 
 TEST_F(Dense, CountNNZIsEquivalentToRef)
@@ -595,12 +757,63 @@ TEST_F(Dense, CalculateTotalColsIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, CanGatherRows)
+{
+    set_up_apply_data();
+
+    auto r_gather = x->row_gather(rgather_idxs.get());
+    auto dr_gather = dx->row_gather(rgather_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
+}
+
+
+TEST_F(Dense, CanGatherRowsIntoDense)
+{
+    set_up_apply_data();
+    auto gather_size =
+        gko::dim<2>{rgather_idxs->get_num_elems(), x->get_size()[1]};
+    auto r_gather = Mtx::create(ref, gather_size);
+    // test make_temporary_clone and non-default stride
+    auto dr_gather = Mtx::create(ref, gather_size, x->get_size()[1] + 2);
+
+    x->row_gather(rgather_idxs.get(), r_gather.get());
+    dx->row_gather(rgather_idxs.get(), dr_gather.get());
+
+    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
+}
+
+
+TEST_F(Dense, IsPermutable)
+{
+    set_up_apply_data();
+
+    auto permuted = square->permute(rpermute_idxs.get());
+    auto dpermuted = dsquare->permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
+                        static_cast<Mtx *>(dpermuted.get()), 0);
+}
+
+
+TEST_F(Dense, IsInversePermutable)
+{
+    set_up_apply_data();
+
+    auto permuted = square->inverse_permute(rpermute_idxs.get());
+    auto dpermuted = dsquare->inverse_permute(rpermute_idxs.get());
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
+                        static_cast<Mtx *>(dpermuted.get()), 0);
+}
+
+
 TEST_F(Dense, IsRowPermutable)
 {
     set_up_apply_data();
 
     auto r_permute = x->row_permute(rpermute_idxs.get());
-    auto dr_permute = dx->row_permute(drpermute_idxs.get());
+    auto dr_permute = dx->row_permute(rpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(r_permute.get()),
                         static_cast<Mtx *>(dr_permute.get()), 0);
@@ -612,7 +825,7 @@ TEST_F(Dense, IsColPermutable)
     set_up_apply_data();
 
     auto c_permute = x->column_permute(cpermute_idxs.get());
-    auto dc_permute = dx->column_permute(dcpermute_idxs.get());
+    auto dc_permute = dx->column_permute(cpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(c_permute.get()),
                         static_cast<Mtx *>(dc_permute.get()), 0);
@@ -624,7 +837,7 @@ TEST_F(Dense, IsInverseRowPermutable)
     set_up_apply_data();
 
     auto inverse_r_permute = x->inverse_row_permute(rpermute_idxs.get());
-    auto d_inverse_r_permute = dx->inverse_row_permute(drpermute_idxs.get());
+    auto d_inverse_r_permute = dx->inverse_row_permute(rpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_r_permute.get()),
                         static_cast<Mtx *>(d_inverse_r_permute.get()), 0);
@@ -636,14 +849,14 @@ TEST_F(Dense, IsInverseColPermutable)
     set_up_apply_data();
 
     auto inverse_c_permute = x->inverse_column_permute(cpermute_idxs.get());
-    auto d_inverse_c_permute = dx->inverse_column_permute(dcpermute_idxs.get());
+    auto d_inverse_c_permute = dx->inverse_column_permute(cpermute_idxs.get());
 
     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_c_permute.get()),
                         static_cast<Mtx *>(d_inverse_c_permute.get()), 0);
 }
 
 
-TEST_F(Dense, ExtractDiagonalIsEquivalentToRef)
+TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
 {
     set_up_apply_data();
 
@@ -654,6 +867,17 @@ TEST_F(Dense, ExtractDiagonalIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto diag = y->extract_diagonal();
+    auto ddiag = dy->extract_diagonal();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
 TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -676,4 +900,76 @@ TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, MakeComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto complex_x = x->make_complex();
+    auto dcomplex_x = dx->make_complex();
+
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
+}
+
+
+TEST_F(Dense, MakeComplexWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto complex_x = ComplexMtx::create(ref, x->get_size());
+    x->make_complex(complex_x.get());
+    auto dcomplex_x = ComplexMtx::create(dpcpp, x->get_size());
+    dx->make_complex(dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
+}
+
+
+TEST_F(Dense, GetRealIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto real_x = x->get_real();
+    auto dreal_x = dx->get_real();
+
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
+}
+
+
+TEST_F(Dense, GetRealWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto real_x = Mtx::create(ref, x->get_size());
+    x->get_real(real_x.get());
+    auto dreal_x = Mtx::create(dpcpp, dx->get_size());
+    dx->get_real(dreal_x.get());
+
+    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
+}
+
+
+TEST_F(Dense, GetImagIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto imag_x = x->get_imag();
+    auto dimag_x = dx->get_imag();
+
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
+}
+
+
+TEST_F(Dense, GetImagWithGivenResultIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto imag_x = Mtx::create(ref, x->get_size());
+    x->get_imag(imag_x.get());
+    auto dimag_x = Mtx::create(dpcpp, dx->get_size());
+    dx->get_imag(dimag_x.get());
+
+    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
+}
+
+
 }  // namespace

From 63e1a6948c1d1e7621a0d777f32dc6a7a3db712a Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 2 Jun 2021 19:46:54 +0200
Subject: [PATCH 0901/1008] fix sellp stuck

---
 dpcpp/matrix/dense_kernels.dp.cpp | 37 ++++++++++++++-----------------
 1 file changed, 17 insertions(+), 20 deletions(-)

diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 58f688951c9..41a283d71bd 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -569,28 +569,25 @@ void calculate_slice_lengths(size_type num_rows, size_type slice_size,
     constexpr auto sg_size = cfg;
     const auto sliceid = item_ct1.get_group(2);
     const auto tid_in_warp = item_ct1.get_local_id(2);
+    const bool runable = sliceid * slice_size + tid_in_warp < num_rows;
+    size_type thread_result = 0;
+    for (size_type i = tid_in_warp; i < slice_size; i += sg_size) {
+        thread_result =
+            (i + slice_size * sliceid < num_rows)
+                ? max(thread_result, nnz_per_row[sliceid * slice_size + i])
+                : thread_result;
+    }
 
-    if (sliceid * slice_size + tid_in_warp < num_rows) {
-        size_type thread_result = 0;
-        for (size_type i = tid_in_warp; i < slice_size; i += sg_size) {
-            thread_result =
-                (i + slice_size * sliceid < num_rows)
-                    ? max(thread_result, nnz_per_row[sliceid * slice_size + i])
-                    : thread_result;
-        }
+    auto warp_tile =
+        group::tiled_partition<sg_size>(group::this_thread_block(item_ct1));
+    auto warp_result = ::gko::kernels::dpcpp::reduce(
+        warp_tile, thread_result,
+        [](const size_type &a, const size_type &b) { return max(a, b); });
 
-        auto warp_tile =
-            group::tiled_partition<sg_size>(group::this_thread_block(item_ct1));
-        auto warp_result = ::gko::kernels::dpcpp::reduce(
-            warp_tile, thread_result,
-            [](const size_type &a, const size_type &b) { return max(a, b); });
-
-        if (tid_in_warp == 0) {
-            auto slice_length =
-                ceildiv(warp_result, stride_factor) * stride_factor;
-            slice_lengths[sliceid] = slice_length;
-            slice_sets[sliceid] = slice_length;
-        }
+    if (tid_in_warp == 0 && runable) {
+        auto slice_length = ceildiv(warp_result, stride_factor) * stride_factor;
+        slice_lengths[sliceid] = slice_length;
+        slice_sets[sliceid] = slice_length;
     }
 }
 

From 8542769811b5da30eb89525809955080302d4287 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 8 Jul 2021 13:44:52 +0200
Subject: [PATCH 0902/1008] add cp, update doc, mv mkl bind

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 dpcpp/CMakeLists.txt                          |   4 +-
 dpcpp/base/config.hpp                         |   2 +-
 dpcpp/base/onemkl_bindings.hpp                | 128 ++++++++++++++++++
 dpcpp/components/prefix_sum.dp.hpp            |   1 -
 dpcpp/components/reduction.dp.hpp             |   3 +-
 dpcpp/components/thread_ids.dp.hpp            |   1 -
 dpcpp/components/uninitialized_array.hpp      |   4 +-
 dpcpp/matrix/dense_kernels.dp.cpp             |  63 +++++----
 .../ginkgo/core/synthesizer/containers.hpp    |  85 ++++++++++++
 9 files changed, 259 insertions(+), 32 deletions(-)
 create mode 100644 dpcpp/base/onemkl_bindings.hpp

diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 48addebaf5f..8755b424433 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -55,13 +55,13 @@ target_sources(ginkgo_dpcpp
 ginkgo_compile_features(ginkgo_dpcpp)
 target_compile_definitions(ginkgo_dpcpp PRIVATE GKO_COMPILING_DPCPP)
 
-set(GINKGO_DPCPP_FLAGS ${GINKGO_COMPILER_FLAGS} -fsycl)
+set(GINKGO_DPCPP_FLAGS ${GINKGO_COMPILER_FLAGS} -DMKL_ILP64)
 set(GINKGO_DPCPP_FLAGS ${GINKGO_DPCPP_FLAGS} PARENT_SCOPE)
 target_compile_options(ginkgo_dpcpp PRIVATE "${GINKGO_DPCPP_FLAGS}")
 target_compile_features(ginkgo_dpcpp PRIVATE cxx_std_17)
 target_link_options(ginkgo_dpcpp PRIVATE -fsycl-device-lib=all)
 target_link_options(ginkgo_dpcpp PRIVATE -fsycl-device-code-split=per_kernel)
-target_link_libraries(ginkgo_dpcpp PRIVATE "mkl_sycl;mkl_intel_ilp64;mkl_sequential;mkl_core")
+target_link_libraries(ginkgo_dpcpp PRIVATE "mkl_sycl;mkl_intel_ilp64;mkl_tbb_thread;mkl_core;sycl;OpenCL;pthread;m;dl")
 target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
 if (GINKGO_DPCPP_SINGLE_MODE)
     target_compile_definitions(ginkgo_dpcpp PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
diff --git a/dpcpp/base/config.hpp b/dpcpp/base/config.hpp
index 78fdcc2b819..abb84d9b7ff 100644
--- a/dpcpp/base/config.hpp
+++ b/dpcpp/base/config.hpp
@@ -53,7 +53,7 @@ struct config {
     /**
      * The number of threads within a CUDA warp.
      */
-    static constexpr uint32 warp_size = 32;
+    static constexpr uint32 warp_size = 16;
 
     /**
      * The bitmask of the entire warp.
diff --git a/dpcpp/base/onemkl_bindings.hpp b/dpcpp/base/onemkl_bindings.hpp
new file mode 100644
index 00000000000..6456a048d23
--- /dev/null
+++ b/dpcpp/base/onemkl_bindings.hpp
@@ -0,0 +1,128 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_BASE_ONEMKL_BINDINGS_HPP_
+#define GKO_DPCPP_BASE_ONEMKL_BINDINGS_HPP_
+
+
+#include <CL/sycl.hpp>
+#include <oneapi/mkl.hpp>
+
+
+namespace gko {
+/**
+ * @brief The device specific kernels namespace.
+ *
+ * @ingroup kernels
+ */
+namespace kernels {
+/**
+ * @brief The DPCPP namespace.
+ *
+ * @ingroup dpcpp
+ */
+namespace dpcpp {
+/**
+ * @brief The ONEMKL namespace.
+ *
+ * @ingroup onemkl
+ */
+namespace onemkl {
+/**
+ * @brief The detail namespace.
+ *
+ * @ingroup detail
+ */
+namespace detail {
+
+
+template <typename... Args>
+inline void not_implemented(Args &&...) GKO_NOT_IMPLEMENTED;
+
+
+}  // namespace detail
+
+
+template <typename ValueType>
+struct is_supported : std::false_type {};
+
+template <>
+struct is_supported<float> : std::true_type {};
+
+template <>
+struct is_supported<double> : std::true_type {};
+
+template <>
+struct is_supported<std::complex<float>> : std::true_type {};
+
+template <>
+struct is_supported<std::complex<double>> : std::true_type {};
+
+
+#define GKO_BIND_DOT(ValueType, Name, Func)                                  \
+    void Name(::cl::sycl::queue &exec_queue, std::int64_t n,                 \
+              const ValueType *x, std::int64_t incx, const ValueType *y,     \
+              std::int64_t incy, ValueType *result)                          \
+    {                                                                        \
+        Func(exec_queue, n, x, incx, y, incy, result);                       \
+    }                                                                        \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
+                  "semi-colon warnings")
+
+// Bind the dot for x^T * y
+GKO_BIND_DOT(float, dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(double, dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(std::complex<float>, dot, oneapi::mkl::blas::row_major::dotu);
+GKO_BIND_DOT(std::complex<double>, dot, oneapi::mkl::blas::row_major::dotu);
+template <typename ValueType>
+GKO_BIND_DOT(ValueType, dot, detail::not_implemented);
+
+// Bind the conj_dot for x' * y
+GKO_BIND_DOT(float, conj_dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(double, conj_dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(std::complex<float>, conj_dot, oneapi::mkl::blas::row_major::dotc);
+GKO_BIND_DOT(std::complex<double>, conj_dot,
+             oneapi::mkl::blas::row_major::dotc);
+template <typename ValueType>
+GKO_BIND_DOT(ValueType, conj_dot, detail::not_implemented);
+
+#undef GKO_BIND_DOT
+
+
+}  // namespace onemkl
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_DPCPP_BASE_ONEMKL_BINDINGS_HPP_
diff --git a/dpcpp/components/prefix_sum.dp.hpp b/dpcpp/components/prefix_sum.dp.hpp
index f76f85135eb..22e6139dd84 100644
--- a/dpcpp/components/prefix_sum.dp.hpp
+++ b/dpcpp/components/prefix_sum.dp.hpp
@@ -53,7 +53,6 @@ namespace kernels {
 namespace dpcpp {
 
 
-// #include "common/components/prefix_sum.hpp.inc"
 /**
  * @internal
  * Computes the prefix sum and total sum of `element` over a subwarp.
diff --git a/dpcpp/components/reduction.dp.hpp b/dpcpp/components/reduction.dp.hpp
index 9c2387a7113..e0678f6cf7a 100644
--- a/dpcpp/components/reduction.dp.hpp
+++ b/dpcpp/components/reduction.dp.hpp
@@ -70,7 +70,6 @@ constexpr auto kcfg_1d_list =
                     KCFG_1D::encode(256, 8)>();
 constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
 
-// #include "common/components/reduction.hpp.inc"
 /**
  * @internal
  *
@@ -217,7 +216,7 @@ void reduce_add_array(
     }
 }
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void reduce_add_array(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                       sycl::queue *stream, size_type size,
                       const ValueType *source, ValueType *result)
diff --git a/dpcpp/components/thread_ids.dp.hpp b/dpcpp/components/thread_ids.dp.hpp
index 5b656c5e0db..9eda077381c 100644
--- a/dpcpp/components/thread_ids.dp.hpp
+++ b/dpcpp/components/thread_ids.dp.hpp
@@ -52,7 +52,6 @@ namespace dpcpp {
 namespace thread {
 
 
-// #include "common/components/thread_ids.hpp.inc"
 /**
  * @internal
  *
diff --git a/dpcpp/components/uninitialized_array.hpp b/dpcpp/components/uninitialized_array.hpp
index 415126b8ed3..b10457df217 100644
--- a/dpcpp/components/uninitialized_array.hpp
+++ b/dpcpp/components/uninitialized_array.hpp
@@ -45,7 +45,6 @@ namespace kernels {
 namespace dpcpp {
 
 
-// #include "common/components/uninitialized_array.hpp.inc"
 /**
  * Stores an array with uninitialized contents.
  *
@@ -105,7 +104,8 @@ class UninitializedArray {
     }
 
 private:
-    // unsigned char data_[sizeof(ValueType) / sizeof(unsigned char) * size];
+    // if dpcpp uses char to represent data in char, compiling gives error.
+    // Thanksfully, dpcpp support complex data allocation directly.
     ValueType data_[size];
 };
 
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 41a283d71bd..a4e061c3f98 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -51,6 +51,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
 #include "dpcpp/base/helper.hpp"
+#include "dpcpp/base/onemkl_bindings.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/reduction.dp.hpp"
 #include "dpcpp/components/thread_ids.dp.hpp"
@@ -79,10 +80,26 @@ constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
 constexpr auto default_block_size = 256;
 
 
-// #include "common/matrix/dense_kernels.hpp.inc"
 namespace kernel {
 
 
+template <typename InValueType, typename OutValueType>
+void strided_copy(size_type num_rows, size_type num_cols, size_type in_stride,
+                  size_type out_stride, const InValueType *__restrict__ input,
+                  OutValueType *__restrict__ output, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_id = global_id / num_cols;
+    const auto col_id = global_id % num_cols;
+    if (row_id < num_rows) {
+        output[row_id * out_stride + col_id] =
+            static_cast<OutValueType>(input[row_id * in_stride + col_id]);
+    }
+}
+
+GKO_ENABLE_DEFAULT_HOST(strided_copy, strided_copy)
+
+
 template <typename ValueType>
 void strided_fill(size_type num_rows, size_type num_cols, size_type stride,
                   ValueType *__restrict__ mat, ValueType value,
@@ -157,7 +174,7 @@ void add_scaled_diag(size_type size, const ValueType *__restrict__ alpha,
 GKO_ENABLE_DEFAULT_HOST(add_scaled_diag, add_scaled_diag)
 
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename OutType,
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename OutType,
           typename CallableGetValue, typename CallableReduce>
 void compute_partial_reduce(
     size_type num_rows, OutType *__restrict__ work, CallableGetValue get_value,
@@ -191,7 +208,7 @@ void compute_partial_reduce(
 }
 
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType,
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType,
           typename CallableReduce, typename CallableFinalize>
 void finalize_reduce_computation(
     size_type size, const ValueType *work, ValueType *result,
@@ -220,7 +237,7 @@ void finalize_reduce_computation(
 }
 
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void compute_partial_dot(
     size_type num_rows, const ValueType *__restrict__ x, size_type stride_x,
     const ValueType *__restrict__ y, size_type stride_y,
@@ -236,7 +253,7 @@ void compute_partial_dot(
         tmp_work);
 }
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void compute_partial_dot(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                          sycl::queue *stream, size_type num_rows,
                          const ValueType *x, size_type stride_x,
@@ -266,7 +283,7 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(compute_partial_dot_call, compute_partial_dot,
                                kcfg_1d_list)
 
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void compute_partial_conj_dot(
     size_type num_rows, const ValueType *__restrict__ x, size_type stride_x,
     const ValueType *__restrict__ y, size_type stride_y,
@@ -282,7 +299,7 @@ void compute_partial_conj_dot(
         tmp_work);
 }
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void compute_partial_conj_dot(dim3 grid, dim3 block,
                               size_t dynamic_shared_memory, sycl::queue *stream,
                               size_type num_rows, const ValueType *x,
@@ -312,7 +329,7 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(compute_partial_conj_dot_call,
                                compute_partial_conj_dot, kcfg_1d_list)
 
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void finalize_sum_reduce_computation(
     size_type size, const ValueType *work, ValueType *result,
     sycl::nd_item<3> item_ct1,
@@ -324,7 +341,7 @@ void finalize_sum_reduce_computation(
         [](const ValueType &x) { return x; }, item_ct1, tmp_work);
 }
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void finalize_sum_reduce_computation(dim3 grid, dim3 block,
                                      size_t dynamic_shared_memory,
                                      sycl::queue *stream, size_type size,
@@ -353,7 +370,7 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(finalize_sum_reduce_computation_call,
                                finalize_sum_reduce_computation, kcfg_1d_list)
 
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void compute_partial_norm2(
     size_type num_rows, const ValueType *__restrict__ x, size_type stride_x,
     remove_complex<ValueType> *__restrict__ work, sycl::nd_item<3> item_ct1,
@@ -368,7 +385,7 @@ void compute_partial_norm2(
         tmp_work);
 }
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void compute_partial_norm2(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                            sycl::queue *stream, size_type num_rows,
                            const ValueType *x, size_type stride_x,
@@ -397,7 +414,7 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(compute_partial_norm2_call,
                                compute_partial_norm2, kcfg_1d_list)
 
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void finalize_sqrt_reduce_computation(
     size_type size, const ValueType *work, ValueType *result,
     sycl::nd_item<3> item_ct1,
@@ -409,7 +426,7 @@ void finalize_sqrt_reduce_computation(
         [](const ValueType &x) { return std::sqrt(x); }, item_ct1, tmp_work);
 }
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32), typename ValueType>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void finalize_sqrt_reduce_computation(dim3 grid, dim3 block,
                                       size_t dynamic_shared_memory,
                                       sycl::queue *stream, size_type size,
@@ -653,7 +670,7 @@ void reduce_max_nnz(size_type size, const size_type *__restrict__ nnz_per_row,
     }
 }
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32)>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16)>
 void reduce_max_nnz(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                     sycl::queue *stream, size_type size,
                     const size_type *nnz_per_row, size_type *result)
@@ -733,7 +750,7 @@ void reduce_total_cols(size_type num_slices,
     }
 }
 
-template <std::uint32_t cfg = KCFG_1D::encode(256, 32)>
+template <std::uint32_t cfg = KCFG_1D::encode(256, 16)>
 void reduce_total_cols(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                        sycl::queue *stream, size_type num_slices,
                        const size_type *max_nnz_per_slice, size_type *result)
@@ -1048,10 +1065,10 @@ void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
     if (0) {
         // TODO: write a custom kernel which does this more efficiently
         for (size_type col = 0; col < x->get_size()[1]; ++col) {
-            dot(*exec->get_queue(), x->get_size()[0],
-                x->get_const_values() + col, x->get_stride(),
-                y->get_const_values() + col, y->get_stride(),
-                result->get_values() + col);
+            onemkl::dot(*exec->get_queue(), x->get_size()[0],
+                        x->get_const_values() + col, x->get_stride(),
+                        y->get_const_values() + col, y->get_stride(),
+                        result->get_values() + col);
         }
     } else {
         // TODO: these are tuning parameters obtained experimentally, once
@@ -1096,10 +1113,10 @@ void compute_conj_dot(std::shared_ptr<const DpcppExecutor> exec,
     if (0) {
         // TODO: write a custom kernel which does this more efficiently
         for (size_type col = 0; col < x->get_size()[1]; ++col) {
-            conj_dot(*exec->get_queue(), x->get_size()[0],
-                     x->get_const_values() + col, x->get_stride(),
-                     y->get_const_values() + col, y->get_stride(),
-                     result->get_values() + col);
+            onemkl::conj_dot(*exec->get_queue(), x->get_size()[0],
+                             x->get_const_values() + col, x->get_stride(),
+                             y->get_const_values() + col, y->get_stride(),
+                             result->get_values() + col);
         }
     } else {
         // TODO: these are tuning parameters obtained experimentally, once
diff --git a/include/ginkgo/core/synthesizer/containers.hpp b/include/ginkgo/core/synthesizer/containers.hpp
index 3c79c7b7455..10e8c1031a1 100644
--- a/include/ginkgo/core/synthesizer/containers.hpp
+++ b/include/ginkgo/core/synthesizer/containers.hpp
@@ -47,14 +47,32 @@ namespace gko {
 namespace syn {
 
 
+/**
+ * value_list records several values with the same type in template.
+ *
+ * @tparam T  the value type of the list
+ * @tparam T...  the values in the list
+ */
 template <typename T, T... Values>
 struct value_list {};
 
 
+/**
+ * type_list records several types in template
+ *
+ * @tparam ...Types  the types in the list
+ */
 template <typename... Types>
 struct type_list {};
 
 
+/**
+ * range records start, end, step in template
+ *
+ * @tparam int  start of range
+ * @tparam int  end of range
+ * @tparam int  step of range. default is 1
+ */
 template <int Start, int End, int Step = 1>
 struct range {};
 
@@ -62,9 +80,22 @@ struct range {};
 namespace detail {
 
 
+/**
+ * concatenate_impl base type
+ *
+ * @tparam List1  the first List
+ * @tparam List2  the second List
+ */
 template <typename List1, typename List2>
 struct concatenate_impl;
 
+/**
+ * concatenate_impl specializes for two value_list with the same value type.
+ *
+ * @tparam T  the value type of two value_list
+ * @tparam T...  the values of the first list
+ * @tparam T...  the values of the second list
+ */
 template <typename T, T... Values1, T... Values2>
 struct concatenate_impl<value_list<T, Values1...>, value_list<T, Values2...>> {
     using type = value_list<T, Values1..., Values2...>;
@@ -74,6 +105,12 @@ struct concatenate_impl<value_list<T, Values1...>, value_list<T, Values2...>> {
 }  // namespace detail
 
 
+/**
+ * concatenate combines two value_list into one value_list.
+ *
+ * @tparam List1  the first list
+ * @tparam List2  the second list
+ */
 template <typename List1, typename List2>
 using concatenate = typename detail::concatenate_impl<List1, List2>::type;
 
@@ -81,19 +118,43 @@ using concatenate = typename detail::concatenate_impl<List1, List2>::type;
 namespace detail {
 
 
+/**
+ * as_list_impl base type
+ *
+ * @tparam T  the input template
+ */
 template <typename T, typename = void>
 struct as_list_impl;
 
+/**
+ * as_list_impl specializes for the value_list
+ *
+ * @tparam T  the value_list type
+ * @tparam T...  the values of value_list
+ */
 template <typename T, T... Values>
 struct as_list_impl<value_list<T, Values...>> {
     using type = value_list<T, Values...>;
 };
 
+/**
+ * as_list_impl specializes for the type_list
+ *
+ * @tparam ...Types  the types of type_list
+ */
 template <typename... Types>
 struct as_list_impl<type_list<Types...>> {
     using type = type_list<Types...>;
 };
 
+/**
+ * as_list_impl specializes for the range. This is the recursive case. It will
+ * concatenate Start and range<Start + Step, End, Step>.
+ *
+ * @tparam int  the start of range
+ * @tparam int  the end of range
+ * @tparam int  the step of range
+ */
 template <int Start, int End, int Step>
 struct as_list_impl<range<Start, End, Step>, std::enable_if_t<(Start < End)>> {
     using type = concatenate<
@@ -101,6 +162,13 @@ struct as_list_impl<range<Start, End, Step>, std::enable_if_t<(Start < End)>> {
         typename as_list_impl<range<Start + Step, End, Step>>::type>;
 };
 
+/**
+ * as_list_impl specializes for the range. This is the end case.
+ *
+ * @tparam int  the start of range
+ * @tparam int  the end of range
+ * @tparam int  the step of range
+ */
 template <int Start, int End, int Step>
 struct as_list_impl<range<Start, End, Step>, std::enable_if_t<(Start >= End)>> {
     using type = value_list<int>;
@@ -110,10 +178,27 @@ struct as_list_impl<range<Start, End, Step>, std::enable_if_t<(Start >= End)>> {
 }  // namespace detail
 
 
+/**
+ * as_list<T> gives the alias type of as_list_impl<T>::type. It gives a list
+ * (itself) if input is already a list, or generates list type from range input.
+ *
+ * @tparam T  list or range
+ */
 template <typename T>
 using as_list = typename detail::as_list_impl<T>::type;
 
 
+/**
+ * as_array<T> returns the array from value_list. It will be helpful if using
+ * for in runtime on the array.
+ *
+ * @tparam T  the type of value_list
+ * @tparam T...  the values of value_list
+ *
+ * @param value_list  the input value_list
+ *
+ * @return std::array  the std::array contains the values of value_list
+ */
 template <typename T, T... Value>
 constexpr std::array<T, sizeof...(Value)> as_array(value_list<T, Value...> vl)
 {

From 45ecddbacd39131aa5084cd888963f6531ca4d6b Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 13 Jul 2021 14:06:27 +0200
Subject: [PATCH 0903/1008] MKL cmake, delete unused, simplify func, add job

---
 cmake/create_test.cmake           |   8 +-
 dpcpp/CMakeLists.txt              |   8 +-
 dpcpp/matrix/dense_kernels.dp.cpp | 386 ++----------------------------
 3 files changed, 33 insertions(+), 369 deletions(-)

diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
index ebf70232dd3..9d36b49911e 100644
--- a/cmake/create_test.cmake
+++ b/cmake/create_test.cmake
@@ -42,6 +42,10 @@ function(ginkgo_create_dpcpp_test test_name)
     target_compile_options(${test_target_name} PRIVATE "${GINKGO_DPCPP_FLAGS}")
     target_link_options(${test_target_name} PRIVATE -fsycl-device-code-split=per_kernel)
     ginkgo_set_test_target_properties(${test_name} ${test_target_name})
+    # Note: MKL_ENV is empty on linux. Maybe need to apply MKL_ENV to all test.
+    if (MKL_ENV)
+       set_tests_properties(${test_target_name} PROPERTIES ENVIRONMENT "${MKL_ENV}")
+    endif()
 endfunction(ginkgo_create_dpcpp_test)
 
 function(ginkgo_create_thread_test test_name)
@@ -165,7 +169,7 @@ function(ginkgo_create_common_test test_name)
         # use float for DPC++ if necessary
         if((exec STREQUAL "dpcpp") AND GINKGO_DPCPP_SINGLE_MODE)
             target_compile_definitions(${test_target_name} PRIVATE GINKGO_COMMON_SINGLE_MODE=1)
-        endif() 
+        endif()
         ginkgo_set_test_target_properties(${test_name}_${exec} ${test_target_name})
     endforeach()
-endfunction(ginkgo_create_common_test)
\ No newline at end of file
+endfunction(ginkgo_create_common_test)
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 8755b424433..d30810cf12c 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -6,6 +6,8 @@ endif()
 ginkgo_extract_dpcpp_version(${CMAKE_CXX_COMPILER} GINKGO_DPCPP_VERSION)
 set(GINKGO_DPCPP_VERSION ${GINKGO_DPCPP_VERSION} PARENT_SCOPE)
 
+find_package(MKL CONFIG REQUIRED HINTS "$ENV{MKLROOT}")
+
 add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_dpcpp_device> "")
 target_sources(ginkgo_dpcpp
     PRIVATE
@@ -55,14 +57,16 @@ target_sources(ginkgo_dpcpp
 ginkgo_compile_features(ginkgo_dpcpp)
 target_compile_definitions(ginkgo_dpcpp PRIVATE GKO_COMPILING_DPCPP)
 
-set(GINKGO_DPCPP_FLAGS ${GINKGO_COMPILER_FLAGS} -DMKL_ILP64)
 set(GINKGO_DPCPP_FLAGS ${GINKGO_DPCPP_FLAGS} PARENT_SCOPE)
 target_compile_options(ginkgo_dpcpp PRIVATE "${GINKGO_DPCPP_FLAGS}")
+# Note. add MKL via PRIVATE not PUBLIC (MKL example shows) to avoid find_package(MKL) everywhere when link ginkgo
+target_compile_options(ginkgo_dpcpp PRIVATE $<TARGET_PROPERTY:MKL::MKL_DPCPP,INTERFACE_COMPILE_OPTIONS>)
 target_compile_features(ginkgo_dpcpp PRIVATE cxx_std_17)
+target_include_directories(ginkgo_dpcpp PRIVATE $<TARGET_PROPERTY:MKL::MKL_DPCPP,INTERFACE_INCLUDE_DIRECTORIES>)
 target_link_options(ginkgo_dpcpp PRIVATE -fsycl-device-lib=all)
 target_link_options(ginkgo_dpcpp PRIVATE -fsycl-device-code-split=per_kernel)
-target_link_libraries(ginkgo_dpcpp PRIVATE "mkl_sycl;mkl_intel_ilp64;mkl_tbb_thread;mkl_core;sycl;OpenCL;pthread;m;dl")
 target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
+target_link_libraries(ginkgo_dpcpp PRIVATE $<LINK_ONLY:MKL::MKL_DPCPP>)
 if (GINKGO_DPCPP_SINGLE_MODE)
     target_compile_definitions(ginkgo_dpcpp PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
 endif()
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index a4e061c3f98..f264f970cac 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -83,97 +83,6 @@ constexpr auto default_block_size = 256;
 namespace kernel {
 
 
-template <typename InValueType, typename OutValueType>
-void strided_copy(size_type num_rows, size_type num_cols, size_type in_stride,
-                  size_type out_stride, const InValueType *__restrict__ input,
-                  OutValueType *__restrict__ output, sycl::nd_item<3> item_ct1)
-{
-    const auto global_id = thread::get_thread_id_flat(item_ct1);
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        output[row_id * out_stride + col_id] =
-            static_cast<OutValueType>(input[row_id * in_stride + col_id]);
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(strided_copy, strided_copy)
-
-
-template <typename ValueType>
-void strided_fill(size_type num_rows, size_type num_cols, size_type stride,
-                  ValueType *__restrict__ mat, ValueType value,
-                  sycl::nd_item<3> item_ct1)
-{
-    const auto global_id = thread::get_thread_id_flat(item_ct1);
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        mat[row_id * stride + col_id] = value;
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(strided_fill, strided_fill)
-
-
-template <typename ValueType>
-void scale(size_type num_rows, size_type num_cols, size_type num_alpha_cols,
-           const ValueType *__restrict__ alpha, ValueType *__restrict__ x,
-           size_type stride_x, sycl::nd_item<3> item_ct1)
-{
-    const auto global_id = thread::get_thread_id_flat(item_ct1);
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    const auto alpha_id = num_alpha_cols == 1 ? 0 : col_id;
-    if (row_id < num_rows) {
-        x[row_id * stride_x + col_id] =
-            alpha[alpha_id] == zero<ValueType>()
-                ? zero<ValueType>()
-                : x[row_id * stride_x + col_id] * alpha[alpha_id];
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(scale, scale)
-
-
-template <typename ValueType>
-void add_scaled(size_type num_rows, size_type num_cols,
-                size_type num_alpha_cols, const ValueType *__restrict__ alpha,
-                const ValueType *__restrict__ x, size_type stride_x,
-                ValueType *__restrict__ y, size_type stride_y,
-                sycl::nd_item<3> item_ct1)
-{
-    const auto global_id = thread::get_thread_id_flat(item_ct1);
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    const auto alpha_id = num_alpha_cols == 1 ? 0 : col_id;
-    if (row_id < num_rows && alpha[alpha_id] != zero<ValueType>()) {
-        y[row_id * stride_y + col_id] +=
-            x[row_id * stride_x + col_id] * alpha[alpha_id];
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(add_scaled, add_scaled)
-
-
-template <typename ValueType>
-void add_scaled_diag(size_type size, const ValueType *__restrict__ alpha,
-                     const ValueType *__restrict__ diag,
-                     ValueType *__restrict__ y, size_type stride_y,
-                     sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-
-    if (tidx >= size) {
-        return;
-    }
-
-    y[tidx * stride_y + tidx] += alpha[0] * diag[tidx];
-}
-
-GKO_ENABLE_DEFAULT_HOST(add_scaled_diag, add_scaled_diag)
-
-
 template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename OutType,
           typename CallableGetValue, typename CallableReduce>
 void compute_partial_reduce(
@@ -456,7 +365,7 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(finalize_sqrt_reduce_computation_call,
                                finalize_sqrt_reduce_computation, kcfg_1d_list)
 
 
-template <std::uint32_t cfg, typename ValueType, typename IndexType>
+template <typename ValueType, typename IndexType>
 void fill_in_coo(size_type num_rows, size_type num_cols, size_type stride,
                  const size_type *__restrict__ row_ptrs,
                  const ValueType *__restrict__ source,
@@ -479,9 +388,7 @@ void fill_in_coo(size_type num_rows, size_type num_cols, size_type stride,
     }
 }
 
-GKO_ENABLE_DEFAULT_HOST_CONFIG(fill_in_coo, fill_in_coo)
-GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(fill_in_coo, fill_in_coo)
-GKO_ENABLE_DEFAULT_CONFIG_CALL(fill_in_coo_call, fill_in_coo, kcfg_1d_list)
+GKO_ENABLE_DEFAULT_HOST(fill_in_coo, fill_in_coo)
 
 
 template <std::uint32_t cfg, typename ValueType, typename IndexType>
@@ -514,7 +421,7 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(count_nnz_per_row_call, count_nnz_per_row,
                                kcfg_1d_list)
 
 
-template <std::uint32_t cfg, typename ValueType, typename IndexType>
+template <typename ValueType, typename IndexType>
 void fill_in_csr(size_type num_rows, size_type num_cols, size_type stride,
                  const ValueType *__restrict__ source,
                  IndexType *__restrict__ row_ptrs,
@@ -535,12 +442,10 @@ void fill_in_csr(size_type num_rows, size_type num_cols, size_type stride,
     }
 }
 
-GKO_ENABLE_DEFAULT_HOST_CONFIG(fill_in_csr, fill_in_csr)
-GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(fill_in_csr, fill_in_csr)
-GKO_ENABLE_DEFAULT_CONFIG_CALL(fill_in_csr_call, fill_in_csr, kcfg_1d_list)
+GKO_ENABLE_DEFAULT_HOST(fill_in_csr, fill_in_csr)
 
 
-template <std::uint32_t cfg, typename ValueType, typename IndexType>
+template <typename ValueType, typename IndexType>
 void fill_in_ell(size_type num_rows, size_type num_cols,
                  size_type source_stride, const ValueType *__restrict__ source,
                  size_type max_nnz_per_row, size_type result_stride,
@@ -570,9 +475,7 @@ void fill_in_ell(size_type num_rows, size_type num_cols,
     }
 }
 
-GKO_ENABLE_DEFAULT_HOST_CONFIG(fill_in_ell, fill_in_ell)
-GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(fill_in_ell, fill_in_ell)
-GKO_ENABLE_DEFAULT_CONFIG_CALL(fill_in_ell_call, fill_in_ell, kcfg_1d_list)
+GKO_ENABLE_DEFAULT_HOST(fill_in_ell, fill_in_ell)
 
 
 template <std::uint32_t cfg>
@@ -615,7 +518,7 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(calculate_slice_lengths_call,
                                calculate_slice_lengths, subgroup_list)
 
 
-template <std::uint32_t cfg, typename ValueType, typename IndexType>
+template <typename ValueType, typename IndexType>
 void fill_in_sellp(size_type num_rows, size_type num_cols, size_type slice_size,
                    size_type stride, const ValueType *__restrict__ source,
                    size_type *__restrict__ slice_lengths,
@@ -648,9 +551,7 @@ void fill_in_sellp(size_type num_rows, size_type num_cols, size_type slice_size,
     }
 }
 
-GKO_ENABLE_DEFAULT_HOST_CONFIG(fill_in_sellp, fill_in_sellp)
-GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(fill_in_sellp, fill_in_sellp)
-GKO_ENABLE_DEFAULT_CONFIG_CALL(fill_in_sellp_call, fill_in_sellp, kcfg_1d_list)
+GKO_ENABLE_DEFAULT_HOST(fill_in_sellp, fill_in_sellp)
 
 
 template <std::uint32_t cfg>
@@ -775,220 +676,6 @@ GKO_ENABLE_DEFAULT_CONFIG_CALL(reduce_total_cols_call, reduce_total_cols,
                                kcfg_1d_list)
 
 
-template <typename IndexType, typename ValueType>
-void symm_permute(size_type num_rows, size_type num_cols,
-                  const IndexType *__restrict__ perm_idxs,
-                  const ValueType *__restrict__ orig, size_type stride_orig,
-                  ValueType *__restrict__ result, size_type stride_result,
-                  sycl::nd_item<3> item_ct1)
-{
-    const auto global_id = thread::get_thread_id_flat(item_ct1);
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        result[row_id * stride_result + col_id] =
-            orig[perm_idxs[row_id] * stride_orig + perm_idxs[col_id]];
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(symm_permute, symm_permute)
-
-
-template <typename IndexType, typename ValueType>
-void inv_symm_permute(size_type num_rows, size_type num_cols,
-                      const IndexType *__restrict__ perm_idxs,
-                      const ValueType *__restrict__ orig, size_type stride_orig,
-                      ValueType *__restrict__ result, size_type stride_result,
-                      sycl::nd_item<3> item_ct1)
-{
-    const auto global_id = thread::get_thread_id_flat(item_ct1);
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        result[perm_idxs[row_id] * stride_result + perm_idxs[col_id]] =
-            orig[row_id * stride_orig + col_id];
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(inv_symm_permute, inv_symm_permute)
-
-
-template <typename IndexType, typename ValueType>
-void row_gather(size_type num_rows, size_type num_cols,
-                const IndexType *__restrict__ perm_idxs,
-                const ValueType *__restrict__ orig, size_type stride_orig,
-                ValueType *__restrict__ result, size_type stride_result,
-                sycl::nd_item<3> item_ct1)
-{
-    const auto global_id = thread::get_thread_id_flat(item_ct1);
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        result[row_id * stride_result + col_id] =
-            orig[perm_idxs[row_id] * stride_orig + col_id];
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(row_gather, row_gather)
-
-
-template <typename IndexType, typename ValueType>
-void column_permute(size_type num_rows, size_type num_cols,
-                    const IndexType *__restrict__ perm_idxs,
-                    const ValueType *__restrict__ orig, size_type stride_orig,
-                    ValueType *__restrict__ result, size_type stride_result,
-                    sycl::nd_item<3> item_ct1)
-{
-    const auto global_id = thread::get_thread_id_flat(item_ct1);
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        result[row_id * stride_result + col_id] =
-            orig[row_id * stride_orig + perm_idxs[col_id]];
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(column_permute, column_permute)
-
-
-template <typename IndexType, typename ValueType>
-void inverse_row_permute(size_type num_rows, size_type num_cols,
-                         const IndexType *__restrict__ perm_idxs,
-                         const ValueType *__restrict__ orig,
-                         size_type stride_orig, ValueType *__restrict__ result,
-                         size_type stride_result, sycl::nd_item<3> item_ct1)
-{
-    const auto global_id = thread::get_thread_id_flat(item_ct1);
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        result[perm_idxs[row_id] * stride_result + col_id] =
-            orig[row_id * stride_orig + col_id];
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(inverse_row_permute, inverse_row_permute)
-
-
-template <typename IndexType, typename ValueType>
-void inverse_column_permute(size_type num_rows, size_type num_cols,
-                            const IndexType *__restrict__ perm_idxs,
-                            const ValueType *__restrict__ orig,
-                            size_type stride_orig,
-                            ValueType *__restrict__ result,
-                            size_type stride_result, sycl::nd_item<3> item_ct1)
-{
-    const auto global_id = thread::get_thread_id_flat(item_ct1);
-    const auto row_id = global_id / num_cols;
-    const auto col_id = global_id % num_cols;
-    if (row_id < num_rows) {
-        result[row_id * stride_result + perm_idxs[col_id]] =
-            orig[row_id * stride_orig + col_id];
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(inverse_column_permute, inverse_column_permute)
-
-
-template <typename ValueType>
-void extract_diagonal(size_type problem_size,
-                      const ValueType *__restrict__ orig, size_type stride_orig,
-                      ValueType *__restrict__ diag, sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat<int>(item_ct1);
-    if (tidx < problem_size) {
-        diag[tidx] = orig[tidx * stride_orig + tidx];
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(extract_diagonal, extract_diagonal)
-
-
-template <typename ValueType>
-void inplace_absolute_dense(size_type num_rows, size_type num_cols,
-                            ValueType *__restrict__ data, size_type stride,
-                            sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-    auto row = tidx / num_cols;
-    auto col = tidx % num_cols;
-    if (row < num_rows) {
-        data[row * stride + col] = std::abs(data[row * stride + col]);
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(inplace_absolute_dense, inplace_absolute_dense)
-
-
-template <typename ValueType>
-void outplace_absolute_dense(size_type num_rows, size_type num_cols,
-                             const ValueType *__restrict__ in,
-                             size_type stride_in,
-                             remove_complex<ValueType> *__restrict__ out,
-                             size_type stride_out, sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-    auto row = tidx / num_cols;
-    auto col = tidx % num_cols;
-    if (row < num_rows) {
-        out[row * stride_out + col] = std::abs(in[row * stride_in + col]);
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(outplace_absolute_dense, outplace_absolute_dense)
-
-
-template <typename ValueType, typename ComplexType>
-void make_complex(size_type num_rows, size_type num_cols,
-                  const ValueType *__restrict__ in, size_type stride_in,
-                  ComplexType *__restrict__ out, size_type stride_out,
-                  sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-    auto row = tidx / num_cols;
-    auto col = tidx % num_cols;
-    if (row < num_rows) {
-        out[row * stride_out + col] = in[row * stride_in + col];
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(make_complex, make_complex)
-
-
-template <typename ValueType>
-void get_real(size_type num_rows, size_type num_cols,
-              const ValueType *__restrict__ in, size_type stride_in,
-              remove_complex<ValueType> *__restrict__ out, size_type stride_out,
-              sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-    auto row = tidx / num_cols;
-    auto col = tidx % num_cols;
-    if (row < num_rows) {
-        out[row * stride_out + col] = real(in[row * stride_in + col]);
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(get_real, get_real)
-
-
-template <typename ValueType>
-void get_imag(size_type num_rows, size_type num_cols,
-              const ValueType *__restrict__ in, size_type stride_in,
-              remove_complex<ValueType> *__restrict__ out, size_type stride_out,
-              sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-    auto row = tidx / num_cols;
-    auto col = tidx % num_cols;
-    if (row < num_rows) {
-        out[row * stride_out + col] = imag(in[row * stride_in + col]);
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(get_imag, get_imag)
-
-
 }  // namespace kernel
 
 
@@ -1028,34 +715,6 @@ void apply(std::shared_ptr<const DpcppExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL);
 
 
-namespace {
-
-
-#define GKO_BIND_DOT(ValueType, Name, Func)                                  \
-    void Name(::cl::sycl::queue &exec_queue, std::int64_t n,                 \
-              const ValueType *x, std::int64_t incx, const ValueType *y,     \
-              std::int64_t incy, ValueType *result)                          \
-    {                                                                        \
-        Func(exec_queue, n, x, incx, y, incy, result);                       \
-    }                                                                        \
-    static_assert(true,                                                      \
-                  "This assert is used to counter the false positive extra " \
-                  "semi-colon warnings")
-
-GKO_BIND_DOT(float, dot, oneapi::mkl::blas::row_major::dot);
-GKO_BIND_DOT(double, dot, oneapi::mkl::blas::row_major::dot);
-GKO_BIND_DOT(std::complex<float>, dot, oneapi::mkl::blas::row_major::dotu);
-GKO_BIND_DOT(std::complex<double>, dot, oneapi::mkl::blas::row_major::dotu);
-GKO_BIND_DOT(float, conj_dot, oneapi::mkl::blas::row_major::dot);
-GKO_BIND_DOT(double, conj_dot, oneapi::mkl::blas::row_major::dot);
-GKO_BIND_DOT(std::complex<float>, conj_dot, oneapi::mkl::blas::row_major::dotc);
-GKO_BIND_DOT(std::complex<double>, conj_dot,
-             oneapi::mkl::blas::row_major::dotc);
-
-
-}  // namespace
-
-
 template <typename ValueType>
 void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
                  const matrix::Dense<ValueType> *x,
@@ -1231,10 +890,9 @@ void convert_to_coo(std::shared_ptr<const DpcppExecutor> exec,
     const auto sg_size = KCFG_1D::decode<1>(cfg);
     size_type grid_dim = ceildiv(num_rows, wg_size);
 
-    kernel::fill_in_coo_call(
-        cfg, grid_dim, wg_size, 0, exec->get_queue(), num_rows, num_cols,
-        stride, nnz_prefix_sum.get_const_data(), source->get_const_values(),
-        row_idxs, col_idxs, values);
+    kernel::fill_in_coo(grid_dim, wg_size, 0, exec->get_queue(), num_rows,
+                        num_cols, stride, nnz_prefix_sum.get_const_data(),
+                        source->get_const_values(), row_idxs, col_idxs, values);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -1276,10 +934,9 @@ void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
 
     size_type grid_dim = ceildiv(num_rows, wg_size);
 
-    kernel::fill_in_csr_call(cfg, grid_dim, default_block_size, 0,
-                             exec->get_queue(), num_rows, num_cols, stride,
-                             source->get_const_values(), row_ptrs, col_idxs,
-                             values);
+    kernel::fill_in_csr(grid_dim, default_block_size, 0, exec->get_queue(),
+                        num_rows, num_cols, stride, source->get_const_values(),
+                        row_ptrs, col_idxs, values);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -1311,10 +968,9 @@ void convert_to_ell(std::shared_ptr<const DpcppExecutor> exec,
     const auto wg_size = KCFG_1D::decode<0>(cfg);
     const auto sg_size = KCFG_1D::decode<1>(cfg);
     auto grid_dim = ceildiv(result_stride, wg_size);
-    kernel::fill_in_ell_call(cfg, grid_dim, wg_size, 0, exec->get_queue(),
-                             num_rows, num_cols, source_stride,
-                             source->get_const_values(), max_nnz_per_row,
-                             result_stride, col_ptrs, values);
+    kernel::fill_in_ell(grid_dim, wg_size, 0, exec->get_queue(), num_rows,
+                        num_cols, source_stride, source->get_const_values(),
+                        max_nnz_per_row, result_stride, col_ptrs, values);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -1379,10 +1035,10 @@ void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
 
     grid_dim = ceildiv(num_rows, wg_size);
     if (grid_dim > 0) {
-        kernel::fill_in_sellp_call(cfg, grid_dim, wg_size, 0, exec->get_queue(),
-                                   num_rows, num_cols, slice_size, stride,
-                                   source->get_const_values(), slice_lengths,
-                                   slice_sets, col_idxs, vals);
+        kernel::fill_in_sellp(grid_dim, wg_size, 0, exec->get_queue(), num_rows,
+                              num_cols, slice_size, stride,
+                              source->get_const_values(), slice_lengths,
+                              slice_sets, col_idxs, vals);
     }
 }
 

From b26a2efbb9409df23c232bab3bb8571ef1d6b92e Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 16 Jul 2021 10:46:25 +0200
Subject: [PATCH 0904/1008] MKL static cmake - public per_kernel link option
 when static - add find(MKL) when ginkgo is static

---
 cmake/GinkgoConfig.cmake.in         |  5 +++++
 dpcpp/CMakeLists.txt                |  9 ++++++++-
 dpcpp/matrix/dense_kernels.dp.cpp   |  5 +++--
 dpcpp/test/matrix/dense_kernels.cpp | 24 ------------------------
 4 files changed, 16 insertions(+), 27 deletions(-)

diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 61da405cf6f..4675bcb4781 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -78,6 +78,7 @@ set(GINKGO_AMD_ARCH_FLAGS @GINKGO_AMD_ARCH_FLAGS@)
 
 set(GINKGO_DPCPP_VERSION @GINKGO_DPCPP_VERSION@)
 set(GINKGO_DPCPP_FLAGS @GINKGO_DPCPP_FLAGS@)
+set(GINKGO_MKL_ROOT @GINKGO_MKL_ROOT@)
 
 set(GINKGO_HAVE_PAPI_SDE @GINKGO_HAVE_PAPI_SDE@)
 
@@ -168,4 +169,8 @@ if((NOT GINKGO_BUILD_SHARED_LIBS) AND GINKGO_BUILD_HIP)
     find_package(rocrand REQUIRED)
 endif()
 
+if((NOT GINKGO_BUILD_SHARED_LIBS) AND GINKGO_BUILD_DPCPP)
+    find_package(MKL CONFIG REQUIRED HINTS "${GINKGO_MKL_ROOT}")
+endif()
+
 include(${CMAKE_CURRENT_LIST_DIR}/GinkgoTargets.cmake)
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index d30810cf12c..443d180b172 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -7,6 +7,7 @@ ginkgo_extract_dpcpp_version(${CMAKE_CXX_COMPILER} GINKGO_DPCPP_VERSION)
 set(GINKGO_DPCPP_VERSION ${GINKGO_DPCPP_VERSION} PARENT_SCOPE)
 
 find_package(MKL CONFIG REQUIRED HINTS "$ENV{MKLROOT}")
+set(GINKGO_MKL_ROOT "${MKL_ROOT}" PARENT_SCOPE)
 
 add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_dpcpp_device> "")
 target_sources(ginkgo_dpcpp
@@ -64,7 +65,13 @@ target_compile_options(ginkgo_dpcpp PRIVATE $<TARGET_PROPERTY:MKL::MKL_DPCPP,INT
 target_compile_features(ginkgo_dpcpp PRIVATE cxx_std_17)
 target_include_directories(ginkgo_dpcpp PRIVATE $<TARGET_PROPERTY:MKL::MKL_DPCPP,INTERFACE_INCLUDE_DIRECTORIES>)
 target_link_options(ginkgo_dpcpp PRIVATE -fsycl-device-lib=all)
-target_link_options(ginkgo_dpcpp PRIVATE -fsycl-device-code-split=per_kernel)
+# When building ginkgo as a static library, we need to use dpcpp and per_kernel
+# link option when the program uses dpcpp related function.
+if (BUILD_SHARED_LIBS)
+    target_link_options(ginkgo_dpcpp PRIVATE -fsycl-device-code-split=per_kernel)
+else ()
+    target_link_options(ginkgo_dpcpp PUBLIC -fsycl-device-code-split=per_kernel)
+endif()
 target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
 target_link_libraries(ginkgo_dpcpp PRIVATE $<LINK_ONLY:MKL::MKL_DPCPP>)
 if (GINKGO_DPCPP_SINGLE_MODE)
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index f264f970cac..c2326be9c82 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -75,8 +75,9 @@ constexpr auto kcfg_1d_list =
                     KCFG_1D::encode(512, 32), KCFG_1D::encode(512, 16),
                     KCFG_1D::encode(256, 32), KCFG_1D::encode(256, 16),
                     KCFG_1D::encode(256, 8)>();
-constexpr auto subgroup_list = syn::value_list<std::uint32_t, 64, 32, 16, 8>();
-constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
+constexpr auto subgroup_list =
+    syn::value_list<std::uint32_t, 64, 32, 16, 8, 4>();
+constexpr auto kcfg_1d_array = syn::as_array(kcfg_1d_list);
 constexpr auto default_block_size = 256;
 
 
diff --git a/dpcpp/test/matrix/dense_kernels.cpp b/dpcpp/test/matrix/dense_kernels.cpp
index 2b9af16732a..43ce9bad547 100644
--- a/dpcpp/test/matrix/dense_kernels.cpp
+++ b/dpcpp/test/matrix/dense_kernels.cpp
@@ -550,30 +550,6 @@ TEST_F(Dense, ComputeConjDotComplexIsEquivalentToRef)
 }
 
 
-// TEST_F(Dense, IsTransposable)
-// {
-//     set_up_apply_data();
-
-//     auto trans = x->transpose();
-//     auto dtrans = dx->transpose();
-
-//     GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(dtrans.get()),
-//                         static_cast<Mtx *>(trans.get()), 0);
-// }
-
-
-// TEST_F(Dense, IsConjugateTransposable)
-// {
-//     set_up_apply_data();
-
-//     auto trans = c_x->conj_transpose();
-//     auto dtrans = dc_x->conj_transpose();
-
-//     GKO_ASSERT_MTX_NEAR(static_cast<ComplexMtx *>(dtrans.get()),
-//                         static_cast<ComplexMtx *>(trans.get()), 0);
-// }
-
-
 TEST_F(Dense, ConvertToCooIsEquivalentToRef)
 {
     set_up_apply_data();

From 0ba615ef0c3b64bf41064979a315e78611c7cd0b Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Sun, 18 Jul 2021 23:33:35 +0200
Subject: [PATCH 0905/1008] use mkl in ncols=1, add cuda descp in dpcpp thread

---
 dpcpp/components/thread_ids.dp.hpp | 26 ++++++++++++----------
 dpcpp/matrix/dense_kernels.dp.cpp  | 35 ++++++++++++------------------
 2 files changed, 28 insertions(+), 33 deletions(-)

diff --git a/dpcpp/components/thread_ids.dp.hpp b/dpcpp/components/thread_ids.dp.hpp
index 9eda077381c..47abf3c7b72 100644
--- a/dpcpp/components/thread_ids.dp.hpp
+++ b/dpcpp/components/thread_ids.dp.hpp
@@ -59,7 +59,7 @@ namespace thread {
  *
  * @return the ID of the block group this thread belongs to
  *
- * @note Assumes that grid dimensions are in standard format:
+ * @note Assumes that grid dimensions are in cuda standard format:
  *       `(block_group_size, first_grid_dimension, second grid_dimension)`
  */
 __dpct_inline__ size_type get_block_group_id(sycl::nd_item<3> item_ct1)
@@ -76,7 +76,7 @@ __dpct_inline__ size_type get_block_group_id(sycl::nd_item<3> item_ct1)
  *
  * @return the ID of the block this thread belongs to
  *
- * @note Assumes that grid dimensions are in standard format:
+ * @note Assumes that grid dimensions are in cuda standard format:
  *       `(block_group_size, first_grid_dimension, second grid_dimension)`
  */
 __dpct_inline__ size_type get_block_id(sycl::nd_item<3> item_ct1)
@@ -95,7 +95,7 @@ __dpct_inline__ size_type get_block_id(sycl::nd_item<3> item_ct1)
  * @return the local ID of the warp (relative to the block) this thread belongs
  *         to
  *
- * @note Assumes that block dimensions are in standard format:
+ * @note Assumes that block dimensions are in cuda standard format:
  *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
  *         config::warp_size)`
  */
@@ -116,7 +116,7 @@ __dpct_inline__ size_type get_local_warp_id(sycl::nd_item<3> item_ct1)
  * @return the local ID of the sub-warp (relative to the block) this thread
  *         belongs to
  *
- * @note Assumes that block dimensions are in standard format:
+ * @note Assumes that block dimensions are in cuda standard format:
  *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
  *         config::warp_size)`
  */
@@ -140,7 +140,7 @@ __dpct_inline__ size_type get_local_subwarp_id(sycl::nd_item<3> item_ct1)
  *
  * @return the local ID of the thread (relative to the block)
  *
- * @note Assumes that block dimensions are in standard format:
+ * @note Assumes that block dimensions are in cuda standard format:
  *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
  *         config::warp_size)`
  */
@@ -161,7 +161,7 @@ __dpct_inline__ size_type get_local_thread_id(sycl::nd_item<3> item_ct1)
  *
  * @return the global ID of the warp this thread belongs to.
  *
- * @note Assumes that block dimensions and grid dimensions are in standard
+ * @note Assumes that block dimensions and grid dimensions are in cuda standard
  *       format:
  *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
  *         config::warp_size)` and
@@ -185,7 +185,7 @@ __dpct_inline__ size_type get_warp_id(sycl::nd_item<3> item_ct1)
  *
  * @return the global ID of the sub-warp this thread belongs to.
  *
- * @note Assumes that block dimensions and grid dimensions are in standard
+ * @note Assumes that block dimensions and grid dimensions are in cuda standard
  *       format:
  *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
  *         config::warp_size)` and
@@ -211,7 +211,7 @@ __dpct_inline__ size_type get_subwarp_id(sycl::nd_item<3> item_ct1)
  *
  * @tparam subwarp_size  size of the subwarp
  *
- * @note Assumes that block dimensions and grid dimensions are in standard
+ * @note Assumes that block dimensions and grid dimensions are in cuda standard
  *       format:
  *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
  *         config::warp_size)` and
@@ -231,7 +231,8 @@ __dpct_inline__ size_type get_thread_id(sycl::nd_item<3> item_ct1)
  * @internal
  *
  * Returns the global ID of the thread in the given index type.
- * This function assumes one-dimensional thread and block indexing.
+ * This function assumes one-dimensional thread and block indexing in cuda
+ * sense. It uses the third position infomation to get the information.
  *
  * @return the global ID of the thread in the given index type.
  *
@@ -250,7 +251,8 @@ __dpct_inline__ IndexType get_thread_id_flat(sycl::nd_item<3> item_ct1)
  * @internal
  *
  * Returns the total number of threads in the given index type.
- * This function assumes one-dimensional thread and block indexing.
+ * This function assumes one-dimensional thread and block indexing in cuda
+ * sense. It uses the third position infomation to get the information.
  *
  * @return the total number of threads in the given index type.
  *
@@ -268,7 +270,7 @@ __dpct_inline__ IndexType get_thread_num_flat(sycl::nd_item<3> item_ct1)
  * @internal
  *
  * Returns the global ID of the subwarp in the given index type.
- * This function assumes one-dimensional thread and block indexing
+ * This function assumes one-dimensional thread and block indexing in cuda sense
  * with a power of two block size of at least subwarp_size.
  *
  * @return the global ID of the subwarp in the given index type.
@@ -292,7 +294,7 @@ __dpct_inline__ IndexType get_subwarp_id_flat(sycl::nd_item<3> item_ct1)
  * @internal
  *
  * Returns the total number of subwarps in the given index type.
- * This function assumes one-dimensional thread and block indexing
+ * This function assumes one-dimensional thread and block indexing in cuda sense
  * with a power of two block size of at least subwarp_size.
  *
  * @return the total number of subwarps in the given index type.
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index c2326be9c82..32eef01af63 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -722,14 +722,11 @@ void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
                  const matrix::Dense<ValueType> *y,
                  matrix::Dense<ValueType> *result)
 {
-    if (0) {
+    if (x->get_size()[1] == 1) {
         // TODO: write a custom kernel which does this more efficiently
-        for (size_type col = 0; col < x->get_size()[1]; ++col) {
-            onemkl::dot(*exec->get_queue(), x->get_size()[0],
-                        x->get_const_values() + col, x->get_stride(),
-                        y->get_const_values() + col, y->get_stride(),
-                        result->get_values() + col);
-        }
+        onemkl::dot(*exec->get_queue(), x->get_size()[0], x->get_const_values(),
+                    x->get_stride(), y->get_const_values(), y->get_stride(),
+                    result->get_values());
     } else {
         // TODO: these are tuning parameters obtained experimentally, once
         // we decide how to handle this uniformly, they should be modified
@@ -770,14 +767,13 @@ void compute_conj_dot(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Dense<ValueType> *y,
                       matrix::Dense<ValueType> *result)
 {
-    if (0) {
+    if (x->get_size()[1] == 1) {
         // TODO: write a custom kernel which does this more efficiently
-        for (size_type col = 0; col < x->get_size()[1]; ++col) {
-            onemkl::conj_dot(*exec->get_queue(), x->get_size()[0],
-                             x->get_const_values() + col, x->get_stride(),
-                             y->get_const_values() + col, y->get_stride(),
-                             result->get_values() + col);
-        }
+        onemkl::conj_dot(*exec->get_queue(), x->get_size()[0],
+                         x->get_const_values(), x->get_stride(),
+                         y->get_const_values(), y->get_stride(),
+                         result->get_values());
+
     } else {
         // TODO: these are tuning parameters obtained experimentally, once
         // we decide how to handle this uniformly, they should be modified
@@ -818,13 +814,10 @@ void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::Dense<ValueType> *x,
                    matrix::Dense<remove_complex<ValueType>> *result)
 {
-    if (0) {
-        for (size_type col = 0; col < x->get_size()[1]; ++col) {
-            oneapi::mkl::blas::row_major::nrm2(
-                *exec->get_queue(), x->get_size()[0],
-                x->get_const_values() + col, x->get_stride(),
-                result->get_values() + col);
-        }
+    if (x->get_size()[1] == 1) {
+        oneapi::mkl::blas::row_major::nrm2(
+            *exec->get_queue(), x->get_size()[0], x->get_const_values(),
+            x->get_stride(), result->get_values());
     } else {
         using norm_type = remove_complex<ValueType>;
         // TODO: these are tuning parameters obtained experimentally, once

From 429cc9699b80904dab6bb83f72f19815c8475fc8 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 20 Jul 2021 15:56:24 +0200
Subject: [PATCH 0906/1008] improve document, fix auto usage in for,
 shared_memory usage
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 common/components/prefix_sum.hpp.inc          |  2 +-
 common/components/sorting.hpp.inc             | 12 ++--
 common/components/uninitialized_array.hpp.inc |  7 +-
 .../par_ilut_filter_kernels.hpp.inc           |  4 +-
 .../par_ilut_select_kernels.hpp.inc           | 10 +--
 common/matrix/dense_kernels.hpp.inc           |  2 +-
 common/matrix/ell_kernels.hpp.inc             |  4 +-
 common/matrix/hybrid_kernels.hpp.inc          |  2 +-
 cuda/components/prefix_sum.cu                 |  6 +-
 cuda/test/components/sorting_kernels.cu       |  2 +-
 cuda/test/matrix/dense_kernels.cpp            |  2 +-
 cuda/test/matrix/ell_kernels.cpp              |  2 +-
 dpcpp/CMakeLists.txt                          |  6 +-
 dpcpp/base/config.hpp                         |  6 --
 dpcpp/base/helper.dp.cpp                      | 20 +++---
 dpcpp/base/helper.hpp                         | 71 ++++++++++++-------
 dpcpp/base/onemkl_bindings.hpp                |  3 +
 dpcpp/components/prefix_sum.dp.cpp            |  6 +-
 dpcpp/components/prefix_sum.dp.hpp            |  5 +-
 dpcpp/components/reduction.dp.hpp             |  6 +-
 dpcpp/components/thread_ids.dp.hpp            |  2 +-
 dpcpp/components/uninitialized_array.hpp      |  6 +-
 dpcpp/matrix/dense_kernels.dp.cpp             | 35 ++++-----
 dpcpp/test/matrix/dense_kernels.cpp           |  2 +-
 hip/components/prefix_sum.hip.cpp             |  6 +-
 .../ginkgo/core/synthesizer/containers.hpp    | 18 ++---
 26 files changed, 131 insertions(+), 116 deletions(-)

diff --git a/common/components/prefix_sum.hpp.inc b/common/components/prefix_sum.hpp.inc
index 8f759b1dc95..1d57c20b2e5 100644
--- a/common/components/prefix_sum.hpp.inc
+++ b/common/components/prefix_sum.hpp.inc
@@ -57,7 +57,7 @@ __forceinline__ __device__ void subwarp_prefix_sum(ValueType element,
     total_sum = element;
 #pragma unroll
     // hypercube prefix sum
-    for (auto step = 1; step < subwarp.size(); step *= 2) {
+    for (int step = 1; step < subwarp.size(); step *= 2) {
         auto neighbor = subwarp.shfl_xor(total_sum, step);
         total_sum += neighbor;
         prefix_sum += bool(subwarp.thread_rank() & step) ? neighbor : 0;
diff --git a/common/components/sorting.hpp.inc b/common/components/sorting.hpp.inc
index ef5bd690937..cd772e08adb 100644
--- a/common/components/sorting.hpp.inc
+++ b/common/components/sorting.hpp.inc
@@ -70,7 +70,7 @@ struct bitonic_local {
                                                           bool reverse)
     {
         auto els_mid = els + (num_elements / 2);
-        for (auto i = 0; i < num_elements / 2; ++i) {
+        for (int i = 0; i < num_elements / 2; ++i) {
             bitonic_cas(els[i], els_mid[i], reverse);
         }
         half::merge(els, reverse);
@@ -131,7 +131,7 @@ struct bitonic_warp {
         auto tile =
             group::tiled_partition<num_threads>(group::this_thread_block());
         auto new_reverse = reverse != upper_half();
-        for (auto i = 0; i < num_local; ++i) {
+        for (int i = 0; i < num_local; ++i) {
             auto other = tile.shfl_xor(els[i], num_threads / 2);
             bitonic_cas(els[i], other, new_reverse);
         }
@@ -206,7 +206,7 @@ struct bitonic_global {
         auto upper_shared_els = shared_els + (num_groups * num_threads / 2);
         // only the lower group executes the CAS
         if (!upper_half()) {
-            for (auto i = 0; i < num_local; ++i) {
+            for (int i = 0; i < num_local; ++i) {
                 auto j = shared_idx(i);
                 bitonic_cas(shared_els[j], upper_shared_els[j], reverse);
             }
@@ -241,11 +241,11 @@ struct bitonic_global<ValueType, num_local, num_threads, 1, num_total_threads> {
                                                  bool reverse)
     {
         group::this_thread_block().sync();
-        for (auto i = 0; i < num_local; ++i) {
+        for (int i = 0; i < num_local; ++i) {
             local_els[i] = shared_els[shared_idx(i)];
         }
         warp::merge(local_els, reverse);
-        for (auto i = 0; i < num_local; ++i) {
+        for (int i = 0; i < num_local; ++i) {
             shared_els[shared_idx(i)] = local_els[i];
         }
     }
@@ -258,7 +258,7 @@ struct bitonic_global<ValueType, num_local, num_threads, 1, num_total_threads> {
         // This is the first step, so we don't need to load from shared memory
         warp::sort(local_els, reverse);
         // store the sorted elements in shared memory
-        for (auto i = 0; i < num_local; ++i) {
+        for (int i = 0; i < num_local; ++i) {
             shared_els[shared_idx(i)] = local_els[i];
         }
     }
diff --git a/common/components/uninitialized_array.hpp.inc b/common/components/uninitialized_array.hpp.inc
index 3a8b3796c12..e951cf06860 100644
--- a/common/components/uninitialized_array.hpp.inc
+++ b/common/components/uninitialized_array.hpp.inc
@@ -34,7 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 /**
  * Stores an array with uninitialized contents.
  *
- * This class needed for datatypes that do have a non-empty constructor when`
+ * This class is needed for datatypes that do have a non-empty constructor when
  * using them as shared memory, for example `thrust::complex<float>`.
  *
  * @tparam ValueType the type of values
@@ -49,7 +49,7 @@ public:
      *
      * @return the constexpr pointer to the first entry of the array.
      */
-    constexpr GKO_ATTRIBUTES operator ValueType *() const noexcept
+    constexpr GKO_ATTRIBUTES operator const ValueType *() const noexcept
     {
         return &(*this)[0];
     }
@@ -70,7 +70,8 @@ public:
      *
      * @return a reference to the array entry at the given index.
      */
-    constexpr GKO_ATTRIBUTES ValueType &operator[](size_type pos) const noexcept
+    constexpr GKO_ATTRIBUTES const ValueType &operator[](size_type pos) const
+        noexcept
     {
         return reinterpret_cast<const ValueType *>(data_)[pos];
     }
diff --git a/common/factorization/par_ilut_filter_kernels.hpp.inc b/common/factorization/par_ilut_filter_kernels.hpp.inc
index 25b43e789ee..b5f7d43db67 100644
--- a/common/factorization/par_ilut_filter_kernels.hpp.inc
+++ b/common/factorization/par_ilut_filter_kernels.hpp.inc
@@ -55,7 +55,7 @@ __device__ void abstract_filter_impl(const IndexType *row_ptrs,
     auto end = row_ptrs[row + 1];
     begin_cb(row);
     auto num_steps = ceildiv(end - begin, subwarp_size);
-    for (auto step = 0; step < num_steps; ++step) {
+    for (IndexType step = 0; step < num_steps; ++step) {
         auto idx = begin + lane + step * subwarp_size;
         auto keep = idx < end && pred(idx, begin, end);
         auto mask = subwarp.ballot(keep);
@@ -189,4 +189,4 @@ __global__ __launch_bounds__(default_block_size) void bucket_filter(
 }
 
 
-}  // namespace kernel
\ No newline at end of file
+}  // namespace kernel
diff --git a/common/factorization/par_ilut_select_kernels.hpp.inc b/common/factorization/par_ilut_select_kernels.hpp.inc
index 059069faf41..9b4897d1766 100644
--- a/common/factorization/par_ilut_select_kernels.hpp.inc
+++ b/common/factorization/par_ilut_select_kernels.hpp.inc
@@ -62,7 +62,8 @@ __global__ __launch_bounds__(searchtree_width) void build_searchtree(
     // assuming rounding towards zero
     auto stride = double(size) / sample_size;
 #pragma unroll
-    for (auto i = 0; i < sampleselect_oversampling; ++i) {
+    for (auto i = decltype(sampleselect_oversampling){0};
+         i < sampleselect_oversampling; ++i) {
         auto lidx = idx * sampleselect_oversampling + i;
         auto val = input[static_cast<IndexType>(lidx * stride)];
         samples[i] = abs(val);
@@ -119,7 +120,8 @@ __global__ __launch_bounds__(default_block_size) void count_buckets(
         auto el = abs(input[i]);
         IndexType tree_idx{};
 #pragma unroll
-        for (auto level = 0; level < sampleselect_searchtree_height; ++level) {
+        for (auto level = decltype(sampleselect_searchtree_height){0};
+             level < sampleselect_searchtree_height; ++level) {
             auto cmp = !(el < sh_tree[tree_idx]);
             tree_idx = 2 * tree_idx + 1 + cmp;
         }
@@ -168,7 +170,7 @@ __global__ __launch_bounds__(default_block_size) void block_prefix_sum(
     // compute prefix sum over warp-sized blocks
     IndexType total{};
     auto base_idx = warp_idx * work_per_warp * warp.size();
-    for (auto step = 0; step < work_per_warp; ++step) {
+    for (auto step = decltype(work_per_warp){0}; step < work_per_warp; ++step) {
         auto idx = warp_lane + step * warp.size() + base_idx;
         auto val = idx < num_blocks ? local_counters[idx] : zero<IndexType>();
         IndexType warp_total{};
@@ -207,7 +209,7 @@ __global__ __launch_bounds__(default_block_size) void block_prefix_sum(
     // add block prefix sum to each warp's block of data
     block.sync();
     auto warp_prefixsum = warp_sums[warp_idx];
-    for (auto step = 0; step < work_per_warp; ++step) {
+    for (IndexType step = 0; step < work_per_warp; ++step) {
         auto idx = warp_lane + step * warp.size() + base_idx;
         auto val = idx < num_blocks ? local_counters[idx] : zero<IndexType>();
         if (idx < num_blocks) {
diff --git a/common/matrix/dense_kernels.hpp.inc b/common/matrix/dense_kernels.hpp.inc
index d46b202a8ff..c7ebafd0627 100644
--- a/common/matrix/dense_kernels.hpp.inc
+++ b/common/matrix/dense_kernels.hpp.inc
@@ -211,7 +211,7 @@ __global__ __launch_bounds__(default_block_size) void fill_in_csr(
 
     if (tidx < num_rows) {
         auto write_to = row_ptrs[tidx];
-        for (auto i = 0; i < num_cols; i++) {
+        for (size_type i = 0; i < num_cols; i++) {
             if (source[stride * tidx + i] != zero<ValueType>()) {
                 values[write_to] = source[stride * tidx + i];
                 col_idxs[write_to] = i;
diff --git a/common/matrix/ell_kernels.hpp.inc b/common/matrix/ell_kernels.hpp.inc
index 2323d512258..399dd5070ac 100644
--- a/common/matrix/ell_kernels.hpp.inc
+++ b/common/matrix/ell_kernels.hpp.inc
@@ -179,7 +179,7 @@ __global__ __launch_bounds__(default_block_size) void fill_in_dense(
 {
     const auto tidx = thread::get_thread_id_flat();
     if (tidx < num_rows) {
-        for (auto col = 0; col < nnz; col++) {
+        for (size_type col = 0; col < nnz; col++) {
             result[tidx * result_stride +
                    col_idxs[tidx + col * source_stride]] +=
                 values[tidx + col * source_stride];
@@ -226,7 +226,7 @@ __global__ __launch_bounds__(default_block_size) void fill_in_csr(
 
     if (tidx < num_rows) {
         auto write_to = result_row_ptrs[tidx];
-        for (auto i = 0; i < max_nnz_per_row; i++) {
+        for (size_type i = 0; i < max_nnz_per_row; i++) {
             const auto source_idx = tidx + stride * i;
             if (source_values[source_idx] != zero<ValueType>()) {
                 result_values[write_to] = source_values[source_idx];
diff --git a/common/matrix/hybrid_kernels.hpp.inc b/common/matrix/hybrid_kernels.hpp.inc
index b6af7c2be36..c7c192189e0 100644
--- a/common/matrix/hybrid_kernels.hpp.inc
+++ b/common/matrix/hybrid_kernels.hpp.inc
@@ -108,7 +108,7 @@ __global__ __launch_bounds__(default_block_size) void fill_in_csr(
 
     if (tidx < num_rows) {
         auto write_to = result_row_ptrs[tidx];
-        for (auto i = 0; i < max_nnz_per_row; i++) {
+        for (size_type i = 0; i < max_nnz_per_row; i++) {
             const auto source_idx = tidx + stride * i;
             if (ell_val[source_idx] != zero<ValueType>()) {
                 result_values[write_to] = ell_val[source_idx];
diff --git a/cuda/components/prefix_sum.cu b/cuda/components/prefix_sum.cu
index 54739c783c8..ce108fa8cf9 100644
--- a/cuda/components/prefix_sum.cu
+++ b/cuda/components/prefix_sum.cu
@@ -49,7 +49,7 @@ template <typename IndexType>
 void prefix_sum(std::shared_ptr<const CudaExecutor> exec, IndexType *counts,
                 size_type num_entries)
 {
-    // prefix_sum should be on the valid array
+    // prefix_sum should only be performed on a valid array
     if (num_entries > 0) {
         auto num_blocks = ceildiv(num_entries, prefix_sum_block_size);
         Array<IndexType> block_sum_array(exec, num_blocks - 1);
@@ -57,8 +57,8 @@ void prefix_sum(std::shared_ptr<const CudaExecutor> exec, IndexType *counts,
         start_prefix_sum<prefix_sum_block_size>
             <<<num_blocks, prefix_sum_block_size>>>(num_entries, counts,
                                                     block_sums);
-        // add the total sum of the previous block only when the number of block
-        // is larger than 1.
+        // add the total sum of the previous block only when the number of
+        // blocks is larger than 1.
         if (num_blocks > 1) {
             finalize_prefix_sum<prefix_sum_block_size>
                 <<<num_blocks, prefix_sum_block_size>>>(num_entries, counts,
diff --git a/cuda/test/components/sorting_kernels.cu b/cuda/test/components/sorting_kernels.cu
index e973cc0f650..f61bbd0694e 100644
--- a/cuda/test/components/sorting_kernels.cu
+++ b/cuda/test/components/sorting_kernels.cu
@@ -99,7 +99,7 @@ protected:
     {
         // we want some duplicate elements
         std::uniform_int_distribution<gko::int32> dist(0, num_elements / 2);
-        for (auto i = 0; i < num_elements; ++i) {
+        for (auto i = decltype(num_elements){0}; i < num_elements; ++i) {
             ref_shared.get_data()[i] = dist(rng);
         }
         ddata = gko::Array<gko::int32>{cuda, ref_shared};
diff --git a/cuda/test/matrix/dense_kernels.cpp b/cuda/test/matrix/dense_kernels.cpp
index 6e40ce5b5a3..de96d27d823 100644
--- a/cuda/test/matrix/dense_kernels.cpp
+++ b/cuda/test/matrix/dense_kernels.cpp
@@ -550,7 +550,7 @@ TEST_F(Dense, CalculateNNZPerRowIsEquivalentToRef)
                                                           &dnnz_per_row);
 
     auto tmp = gko::Array<gko::size_type>(ref, dnnz_per_row);
-    for (auto i = 0; i < nnz_per_row.get_num_elems(); i++) {
+    for (gko::size_type i = 0; i < nnz_per_row.get_num_elems(); i++) {
         ASSERT_EQ(nnz_per_row.get_const_data()[i], tmp.get_const_data()[i]);
     }
 }
diff --git a/cuda/test/matrix/ell_kernels.cpp b/cuda/test/matrix/ell_kernels.cpp
index 2df1c397f4c..51c12fab531 100644
--- a/cuda/test/matrix/ell_kernels.cpp
+++ b/cuda/test/matrix/ell_kernels.cpp
@@ -585,7 +585,7 @@ TEST_F(Ell, CalculateNNZPerRowIsEquivalentToRef)
                                                         &dnnz_per_row);
 
     auto tmp = gko::Array<gko::size_type>(ref, dnnz_per_row);
-    for (auto i = 0; i < nnz_per_row.get_num_elems(); i++) {
+    for (gko::size_type i = 0; i < nnz_per_row.get_num_elems(); i++) {
         ASSERT_EQ(nnz_per_row.get_const_data()[i], tmp.get_const_data()[i]);
     }
 }
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 443d180b172..7729588d363 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -60,13 +60,15 @@ target_compile_definitions(ginkgo_dpcpp PRIVATE GKO_COMPILING_DPCPP)
 
 set(GINKGO_DPCPP_FLAGS ${GINKGO_DPCPP_FLAGS} PARENT_SCOPE)
 target_compile_options(ginkgo_dpcpp PRIVATE "${GINKGO_DPCPP_FLAGS}")
-# Note. add MKL via PRIVATE not PUBLIC (MKL example shows) to avoid find_package(MKL) everywhere when link ginkgo
+# Note: add MKL as PRIVATE not PUBLIC (MKL example shows) to avoid propagating
+# find_package(MKL) everywhere when linking ginkgo (see the MKL example
+# https://software.intel.com/content/www/us/en/develop/documentation/onemkl-windows-developer-guide/top/getting-started/cmake-config-for-onemkl.html)
 target_compile_options(ginkgo_dpcpp PRIVATE $<TARGET_PROPERTY:MKL::MKL_DPCPP,INTERFACE_COMPILE_OPTIONS>)
 target_compile_features(ginkgo_dpcpp PRIVATE cxx_std_17)
 target_include_directories(ginkgo_dpcpp PRIVATE $<TARGET_PROPERTY:MKL::MKL_DPCPP,INTERFACE_INCLUDE_DIRECTORIES>)
 target_link_options(ginkgo_dpcpp PRIVATE -fsycl-device-lib=all)
 # When building ginkgo as a static library, we need to use dpcpp and per_kernel
-# link option when the program uses dpcpp related function.
+# link option when the program uses a dpcpp related function.
 if (BUILD_SHARED_LIBS)
     target_link_options(ginkgo_dpcpp PRIVATE -fsycl-device-code-split=per_kernel)
 else ()
diff --git a/dpcpp/base/config.hpp b/dpcpp/base/config.hpp
index abb84d9b7ff..78fe25978a7 100644
--- a/dpcpp/base/config.hpp
+++ b/dpcpp/base/config.hpp
@@ -49,12 +49,6 @@ struct config {
      */
     using lane_mask_type = uint64;
 
-
-    /**
-     * The number of threads within a CUDA warp.
-     */
-    static constexpr uint32 warp_size = 16;
-
     /**
      * The bitmask of the entire warp.
      */
diff --git a/dpcpp/base/helper.dp.cpp b/dpcpp/base/helper.dp.cpp
index ae453dd937d..5e6c1a579f5 100644
--- a/dpcpp/base/helper.dp.cpp
+++ b/dpcpp/base/helper.dp.cpp
@@ -44,18 +44,16 @@ namespace dpcpp {
 bool validate(sycl::queue *queue, unsigned int workgroup_size,
               unsigned int subgroup_size)
 {
-    {
-        auto device = queue->get_device();
-        auto subgroup_size_list =
-            device.get_info<cl::sycl::info::device::sub_group_sizes>();
-        auto max_workgroup_size =
-            device.get_info<sycl::info::device::max_work_group_size>();
-        bool allowed = false;
-        for (auto &i : subgroup_size_list) {
-            allowed |= (i == subgroup_size);
-        }
-        return allowed && (workgroup_size <= max_workgroup_size);
+    auto device = queue->get_device();
+    auto subgroup_size_list =
+        device.get_info<cl::sycl::info::device::sub_group_sizes>();
+    auto max_workgroup_size =
+        device.get_info<sycl::info::device::max_work_group_size>();
+    bool allowed = false;
+    for (auto &i : subgroup_size_list) {
+        allowed |= (i == subgroup_size);
     }
+    return allowed && (workgroup_size <= max_workgroup_size);
 }
 
 
diff --git a/dpcpp/base/helper.hpp b/dpcpp/base/helper.hpp
index 8c7f45e5174..16d91c2ef8d 100644
--- a/dpcpp/base/helper.hpp
+++ b/dpcpp/base/helper.hpp
@@ -51,44 +51,44 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 /**
  * GKO_ENABLE_DEFAULT_HOST gives a default host implementation for those
  * kernels which require encoded config but do not need explicit template
- * parameter and share memory
+ * parameter and shared memory
  *
  * @param name_  the name of the host function with config
  * @param kernel_  the kernel name
  */
-#define GKO_ENABLE_DEFAULT_HOST(name_, kernel_)                     \
-    template <typename... InferredArgs>                             \
-    void name_(dim3 grid, dim3 block, size_t dynamic_shared_memory, \
-               sycl::queue *queue, InferredArgs... args)            \
-    {                                                               \
-        queue->submit([&](sycl::handler &cgh) {                     \
-            cgh.parallel_for(sycl_nd_range(grid, block),            \
-                             [=](sycl::nd_item<3> item_ct1) {       \
-                                 kernel_(args..., item_ct1);        \
-                             });                                    \
-        });                                                         \
+#define GKO_ENABLE_DEFAULT_HOST(name_, kernel_)                   \
+    template <typename... InferredArgs>                           \
+    void name_(dim3 grid, dim3 block, size_t, sycl::queue *queue, \
+               InferredArgs... args)                              \
+    {                                                             \
+        queue->submit([&](sycl::handler &cgh) {                   \
+            cgh.parallel_for(sycl_nd_range(grid, block),          \
+                             [=](sycl::nd_item<3> item_ct1) {     \
+                                 kernel_(args..., item_ct1);      \
+                             });                                  \
+        });                                                       \
     }
 
 
 /**
  * GKO_ENABLE_DEFAULT_HOST_CONFIG gives a default host implementation for those
  * kernels which require encoded config but do not need explicit template
- * parameter and share memory
+ * parameter and shared memory
  *
  * @param name_  the name of the host function with config
  * @param kernel_  the kernel name
  */
-#define GKO_ENABLE_DEFAULT_HOST_CONFIG(name_, kernel_)                     \
-    template <std::uint32_t encoded, typename... InferredArgs>             \
-    inline void name_(dim3 grid, dim3 block, size_t dynamic_shared_memory, \
-                      sycl::queue *queue, InferredArgs... args)            \
-    {                                                                      \
-        queue->submit([&](sycl::handler &cgh) {                            \
-            cgh.parallel_for(sycl_nd_range(grid, block),                   \
-                             [=](sycl::nd_item<3> item_ct1) {              \
-                                 kernel_<encoded>(args..., item_ct1);      \
-                             });                                           \
-        });                                                                \
+#define GKO_ENABLE_DEFAULT_HOST_CONFIG(name_, kernel_)                   \
+    template <std::uint32_t encoded, typename... InferredArgs>           \
+    inline void name_(dim3 grid, dim3 block, size_t, sycl::queue *queue, \
+                      InferredArgs... args)                              \
+    {                                                                    \
+        queue->submit([&](sycl::handler &cgh) {                          \
+            cgh.parallel_for(sycl_nd_range(grid, block),                 \
+                             [=](sycl::nd_item<3> item_ct1) {            \
+                                 kernel_<encoded>(args..., item_ct1);    \
+                             });                                         \
+        });                                                              \
     }
 
 /**
@@ -138,10 +138,33 @@ namespace kernels {
 namespace dpcpp {
 
 
+/**
+ * This is the validate function for common check. It checks the workgroup size
+ * is below device max workgroup size and subgroup size is in the supported
+ * subgroup size.
+ *
+ * @param queue  the sycl queue pointer
+ * @param workgroup_size  the workgroup size (block size in cuda sense)
+ * @param subgroup_size  the subgroup size (warp size in cuda sense)
+ *
+ * @return the given arguments are valid or not in given queue.
+ */
 bool validate(sycl::queue *queue, unsigned workgroup_size,
               unsigned subgroup_size);
 
 
+/**
+ * get_first_cfg will return the first valid config by validate function from
+ * given config array.
+ *
+ * @tparam IterArr  the iteratable array type
+ * @tparam Validate  the validate function type
+ *
+ * @param arr  the config array
+ * @param verify  the validate function
+ *
+ * @return the first valid config
+ */
 template <typename IterArr, typename Validate>
 std::uint32_t get_first_cfg(IterArr &arr, Validate verify)
 {
diff --git a/dpcpp/base/onemkl_bindings.hpp b/dpcpp/base/onemkl_bindings.hpp
index 6456a048d23..1c9a8dabb30 100644
--- a/dpcpp/base/onemkl_bindings.hpp
+++ b/dpcpp/base/onemkl_bindings.hpp
@@ -34,6 +34,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_DPCPP_BASE_ONEMKL_BINDINGS_HPP_
 
 
+#include <type_traits>
+
+
 #include <CL/sycl.hpp>
 #include <oneapi/mkl.hpp>
 
diff --git a/dpcpp/components/prefix_sum.dp.cpp b/dpcpp/components/prefix_sum.dp.cpp
index 07cdb5b38aa..63f33e9ba35 100644
--- a/dpcpp/components/prefix_sum.dp.cpp
+++ b/dpcpp/components/prefix_sum.dp.cpp
@@ -70,7 +70,7 @@ template <typename IndexType>
 void prefix_sum(std::shared_ptr<const DpcppExecutor> exec, IndexType *counts,
                 size_type num_entries)
 {
-    // prefix_sum should be on the valid array
+    // prefix_sum should only be performed on a valid array
     if (num_entries > 0) {
         auto queue = exec->get_queue();
         constexpr auto block_cfg_array = as_array(block_cfg_list);
@@ -84,8 +84,8 @@ void prefix_sum(std::shared_ptr<const DpcppExecutor> exec, IndexType *counts,
         auto block_sums = block_sum_array.get_data();
         start_prefix_sum_call(cfg, num_blocks, wg_size, 0, exec->get_queue(),
                               num_entries, counts, block_sums);
-        // add the total sum of the previous block only when the number of block
-        // is larger than 1.
+        // add the total sum of the previous block only when the number of
+        // blocks is larger than 1.
         if (num_blocks > 1) {
             finalize_prefix_sum_call(cfg, num_blocks, wg_size, 0,
                                      exec->get_queue(), num_entries, counts,
diff --git a/dpcpp/components/prefix_sum.dp.hpp b/dpcpp/components/prefix_sum.dp.hpp
index 22e6139dd84..334d4239c56 100644
--- a/dpcpp/components/prefix_sum.dp.hpp
+++ b/dpcpp/components/prefix_sum.dp.hpp
@@ -78,7 +78,7 @@ __dpct_inline__ void subwarp_prefix_sum(ValueType element,
     total_sum = element;
 #pragma unroll
     // hypercube prefix sum
-    for (auto step = 1; step < subwarp.size(); step *= 2) {
+    for (int step = 1; step < subwarp.size(); step *= 2) {
         auto neighbor = subwarp.shfl_xor(total_sum, step);
         total_sum += neighbor;
         prefix_sum += bool(subwarp.thread_rank() & step) ? neighbor : 0;
@@ -193,8 +193,7 @@ void start_prefix_sum(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                          [=](sycl::nd_item<3> item_ct1) {
                              start_prefix_sum<block_size>(
                                  num_elements, elements, block_sum, item_ct1,
-                                 (UninitializedArray<ValueType, block_size> *)
-                                     prefix_helper_acc_ct1.get_pointer());
+                                 prefix_helper_acc_ct1.get_pointer().get());
                          });
     });
 }
diff --git a/dpcpp/components/reduction.dp.hpp b/dpcpp/components/reduction.dp.hpp
index e0678f6cf7a..094f2093a95 100644
--- a/dpcpp/components/reduction.dp.hpp
+++ b/dpcpp/components/reduction.dp.hpp
@@ -229,10 +229,8 @@ void reduce_add_array(dim3 grid, dim3 block, size_t dynamic_shared_memory,
 
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                reduce_add_array<cfg>(
-                    size, source, result, item_ct1,
-                    (UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *)
-                        block_sum_acc_ct1.get_pointer());
+                reduce_add_array<cfg>(size, source, result, item_ct1,
+                                      block_sum_acc_ct1.get_pointer().get());
             });
     });
 }
diff --git a/dpcpp/components/thread_ids.dp.hpp b/dpcpp/components/thread_ids.dp.hpp
index 47abf3c7b72..f9decfd989d 100644
--- a/dpcpp/components/thread_ids.dp.hpp
+++ b/dpcpp/components/thread_ids.dp.hpp
@@ -195,7 +195,7 @@ __dpct_inline__ size_type get_warp_id(sycl::nd_item<3> item_ct1)
 template <int subwarp_size, int warps_per_block>
 __dpct_inline__ size_type get_subwarp_id(sycl::nd_item<3> item_ct1)
 {
-    // dpcpp dose not have subwarp
+    // dpcpp does not have subwarp
     constexpr auto subwarps_per_warp = subwarp_size / subwarp_size;
     return get_warp_id<warps_per_block>(item_ct1) * subwarps_per_warp +
            item_ct1.get_local_id(1);
diff --git a/dpcpp/components/uninitialized_array.hpp b/dpcpp/components/uninitialized_array.hpp
index b10457df217..d9d423c9c94 100644
--- a/dpcpp/components/uninitialized_array.hpp
+++ b/dpcpp/components/uninitialized_array.hpp
@@ -48,7 +48,7 @@ namespace dpcpp {
 /**
  * Stores an array with uninitialized contents.
  *
- * This class needed for datatypes that do have a non-empty constructor when`
+ * This class is needed for datatypes that do have a non-empty constructor when
  * using them as shared memory, for example `thrust::complex<float>`.
  *
  * @tparam ValueType the type of values
@@ -63,7 +63,7 @@ class UninitializedArray {
      *
      * @return the constexpr pointer to the first entry of the array.
      */
-    constexpr __dpct_inline__ operator ValueType *() const noexcept
+    constexpr __dpct_inline__ operator const ValueType *() const noexcept
     {
         return &(*this)[0];
     }
@@ -84,7 +84,7 @@ class UninitializedArray {
      *
      * @return a reference to the array entry at the given index.
      */
-    constexpr __dpct_inline__ ValueType &operator[](size_type pos) const
+    constexpr __dpct_inline__ const ValueType &operator[](size_type pos) const
         noexcept
     {
         return data_[pos];
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index 32eef01af63..b17e44f9706 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -179,10 +179,9 @@ void compute_partial_dot(dim3 grid, dim3 block, size_t dynamic_shared_memory,
 
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                compute_partial_dot<cfg>(
-                    num_rows, x, stride_x, y, stride_y, work, item_ct1,
-                    (UninitializedArray<ValueType, wg_size> *)
-                        tmp_work_acc_ct1.get_pointer());
+                compute_partial_dot<cfg>(num_rows, x, stride_x, y, stride_y,
+                                         work, item_ct1,
+                                         tmp_work_acc_ct1.get_pointer().get());
             });
     });
 }
@@ -227,8 +226,7 @@ void compute_partial_conj_dot(dim3 grid, dim3 block,
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 compute_partial_conj_dot<cfg>(
                     num_rows, x, stride_x, y, stride_y, work, item_ct1,
-                    (UninitializedArray<ValueType, wg_size> *)
-                        tmp_work_acc_ct1.get_pointer());
+                    tmp_work_acc_ct1.get_pointer().get());
             });
     });
 }
@@ -268,8 +266,7 @@ void finalize_sum_reduce_computation(dim3 grid, dim3 block,
                          [=](sycl::nd_item<3> item_ct1) {
                              finalize_sum_reduce_computation<cfg>(
                                  size, work, result, item_ct1,
-                                 (UninitializedArray<ValueType, wg_size> *)
-                                     tmp_work_acc_ct1.get_pointer());
+                                 tmp_work_acc_ct1.get_pointer().get());
                          });
     });
 }
@@ -308,13 +305,12 @@ void compute_partial_norm2(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                        sycl::access::target::local>
             tmp_work_acc_ct1(cgh);
 
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                compute_partial_norm2<cfg>(
-                    num_rows, x, stride_x, work, item_ct1,
-                    (UninitializedArray<remove_complex<ValueType>, wg_size> *)
-                        tmp_work_acc_ct1.get_pointer());
-            });
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             compute_partial_norm2<cfg>(
+                                 num_rows, x, stride_x, work, item_ct1,
+                                 tmp_work_acc_ct1.get_pointer().get());
+                         });
     });
 }
 
@@ -354,8 +350,7 @@ void finalize_sqrt_reduce_computation(dim3 grid, dim3 block,
                          [=](sycl::nd_item<3> item_ct1) {
                              finalize_sqrt_reduce_computation<cfg>(
                                  size, work, result, item_ct1,
-                                 (UninitializedArray<ValueType, wg_size> *)
-                                     tmp_work_acc_ct1.get_pointer());
+                                 tmp_work_acc_ct1.get_pointer().get());
                          });
     });
 }
@@ -433,7 +428,7 @@ void fill_in_csr(size_type num_rows, size_type num_cols, size_type stride,
 
     if (tidx < num_rows) {
         auto write_to = row_ptrs[tidx];
-        for (auto i = 0; i < num_cols; i++) {
+        for (size_type i = 0; i < num_cols; i++) {
             if (source[stride * tidx + i] != zero<ValueType>()) {
                 values[write_to] = source[stride * tidx + i];
                 col_idxs[write_to] = i;
@@ -586,7 +581,7 @@ void reduce_max_nnz(dim3 grid, dim3 block, size_t dynamic_shared_memory,
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 reduce_max_nnz<cfg>(size, nnz_per_row, result, item_ct1,
-                                    dpct_local_acc_ct1.get_pointer());
+                                    dpct_local_acc_ct1.get_pointer().get());
             });
     });
 }
@@ -666,7 +661,7 @@ void reduce_total_cols(dim3 grid, dim3 block, size_t dynamic_shared_memory,
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 reduce_total_cols<cfg>(num_slices, max_nnz_per_slice, result,
                                        item_ct1,
-                                       dpct_local_acc_ct1.get_pointer());
+                                       dpct_local_acc_ct1.get_pointer().get());
             });
     });
 }
diff --git a/dpcpp/test/matrix/dense_kernels.cpp b/dpcpp/test/matrix/dense_kernels.cpp
index 43ce9bad547..257ee6fbc6a 100644
--- a/dpcpp/test/matrix/dense_kernels.cpp
+++ b/dpcpp/test/matrix/dense_kernels.cpp
@@ -697,7 +697,7 @@ TEST_F(Dense, CalculateNNZPerRowIsEquivalentToRef)
                                                            &dnnz_per_row);
 
     auto tmp = gko::Array<gko::size_type>(ref, dnnz_per_row);
-    for (auto i = 0; i < nnz_per_row.get_num_elems(); i++) {
+    for (gko::size_type i = 0; i < nnz_per_row.get_num_elems(); i++) {
         ASSERT_EQ(nnz_per_row.get_const_data()[i], tmp.get_const_data()[i]);
     }
 }
diff --git a/hip/components/prefix_sum.hip.cpp b/hip/components/prefix_sum.hip.cpp
index 28cd01b4fb5..9302fc07b9a 100644
--- a/hip/components/prefix_sum.hip.cpp
+++ b/hip/components/prefix_sum.hip.cpp
@@ -49,7 +49,7 @@ template <typename IndexType>
 void prefix_sum(std::shared_ptr<const HipExecutor> exec, IndexType *counts,
                 size_type num_entries)
 {
-    // prefix_sum should be on the valid array
+    // prefix_sum should only be performed on a valid array
     if (num_entries > 0) {
         auto num_blocks = ceildiv(num_entries, prefix_sum_block_size);
         Array<IndexType> block_sum_array(exec, num_blocks - 1);
@@ -58,8 +58,8 @@ void prefix_sum(std::shared_ptr<const HipExecutor> exec, IndexType *counts,
             HIP_KERNEL_NAME(start_prefix_sum<prefix_sum_block_size>),
             dim3(num_blocks), dim3(prefix_sum_block_size), 0, 0, num_entries,
             counts, block_sums);
-        // add the total sum of the previous block only when the number of block
-        // is larger than 1.
+        // add the total sum of the previous block only when the number of
+        // blocks is larger than 1.
         if (num_blocks > 1) {
             hipLaunchKernelGGL(
                 HIP_KERNEL_NAME(finalize_prefix_sum<prefix_sum_block_size>),
diff --git a/include/ginkgo/core/synthesizer/containers.hpp b/include/ginkgo/core/synthesizer/containers.hpp
index 10e8c1031a1..0e9570540fa 100644
--- a/include/ginkgo/core/synthesizer/containers.hpp
+++ b/include/ginkgo/core/synthesizer/containers.hpp
@@ -51,7 +51,7 @@ namespace syn {
  * value_list records several values with the same type in template.
  *
  * @tparam T  the value type of the list
- * @tparam T...  the values in the list
+ * @tparam Values  the values in the list
  */
 template <typename T, T... Values>
 struct value_list {};
@@ -60,7 +60,7 @@ struct value_list {};
 /**
  * type_list records several types in template
  *
- * @tparam ...Types  the types in the list
+ * @tparam Types  the types in the list
  */
 template <typename... Types>
 struct type_list {};
@@ -69,9 +69,9 @@ struct type_list {};
 /**
  * range records start, end, step in template
  *
- * @tparam int  start of range
- * @tparam int  end of range
- * @tparam int  step of range. default is 1
+ * @tparam Start  start of range
+ * @tparam End  end of range
+ * @tparam Step  step of range. default is 1
  */
 template <int Start, int End, int Step = 1>
 struct range {};
@@ -93,8 +93,8 @@ struct concatenate_impl;
  * concatenate_impl specializes for two value_list with the same value type.
  *
  * @tparam T  the value type of two value_list
- * @tparam T...  the values of the first list
- * @tparam T...  the values of the second list
+ * @tparam Values  the values of the first list
+ * @tparam Values  the values of the second list
  */
 template <typename T, T... Values1, T... Values2>
 struct concatenate_impl<value_list<T, Values1...>, value_list<T, Values2...>> {
@@ -130,7 +130,7 @@ struct as_list_impl;
  * as_list_impl specializes for the value_list
  *
  * @tparam T  the value_list type
- * @tparam T...  the values of value_list
+ * @tparam Values  the values of value_list
  */
 template <typename T, T... Values>
 struct as_list_impl<value_list<T, Values...>> {
@@ -193,7 +193,7 @@ using as_list = typename detail::as_list_impl<T>::type;
  * for in runtime on the array.
  *
  * @tparam T  the type of value_list
- * @tparam T...  the values of value_list
+ * @tparam Value  the values of value_list
  *
  * @param value_list  the input value_list
  *

From e4551c1da01f86a1e0a85629bd59fdbfc2000f04 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 20 Jul 2021 18:08:59 +0200
Subject: [PATCH 0907/1008] add some note to indicate the porting TODO need to
 revisit these TODO when we are close to fully porting ginkgo
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 dpcpp/base/helper.hpp                    | 47 ++++++++++++------------
 dpcpp/components/thread_ids.dp.hpp       |  6 +++
 dpcpp/components/uninitialized_array.hpp |  3 ++
 3 files changed, 32 insertions(+), 24 deletions(-)

diff --git a/dpcpp/base/helper.hpp b/dpcpp/base/helper.hpp
index 16d91c2ef8d..cb98e4c511e 100644
--- a/dpcpp/base/helper.hpp
+++ b/dpcpp/base/helper.hpp
@@ -56,17 +56,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  * @param name_  the name of the host function with config
  * @param kernel_  the kernel name
  */
-#define GKO_ENABLE_DEFAULT_HOST(name_, kernel_)                   \
-    template <typename... InferredArgs>                           \
-    void name_(dim3 grid, dim3 block, size_t, sycl::queue *queue, \
-               InferredArgs... args)                              \
-    {                                                             \
-        queue->submit([&](sycl::handler &cgh) {                   \
-            cgh.parallel_for(sycl_nd_range(grid, block),          \
-                             [=](sycl::nd_item<3> item_ct1) {     \
-                                 kernel_(args..., item_ct1);      \
-                             });                                  \
-        });                                                       \
+#define GKO_ENABLE_DEFAULT_HOST(name_, kernel_)                           \
+    template <typename... InferredArgs>                                   \
+    void name_(dim3 grid, dim3 block, gko::size_type, sycl::queue *queue, \
+               InferredArgs... args)                                      \
+    {                                                                     \
+        queue->submit([&](sycl::handler &cgh) {                           \
+            cgh.parallel_for(sycl_nd_range(grid, block),                  \
+                             [=](sycl::nd_item<3> item_ct1) {             \
+                                 kernel_(args..., item_ct1);              \
+                             });                                          \
+        });                                                               \
     }
 
 
@@ -78,17 +78,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  * @param name_  the name of the host function with config
  * @param kernel_  the kernel name
  */
-#define GKO_ENABLE_DEFAULT_HOST_CONFIG(name_, kernel_)                   \
-    template <std::uint32_t encoded, typename... InferredArgs>           \
-    inline void name_(dim3 grid, dim3 block, size_t, sycl::queue *queue, \
-                      InferredArgs... args)                              \
-    {                                                                    \
-        queue->submit([&](sycl::handler &cgh) {                          \
-            cgh.parallel_for(sycl_nd_range(grid, block),                 \
-                             [=](sycl::nd_item<3> item_ct1) {            \
-                                 kernel_<encoded>(args..., item_ct1);    \
-                             });                                         \
-        });                                                              \
+#define GKO_ENABLE_DEFAULT_HOST_CONFIG(name_, kernel_)                \
+    template <std::uint32_t encoded, typename... InferredArgs>        \
+    inline void name_(dim3 grid, dim3 block, gko::size_type,          \
+                      sycl::queue *queue, InferredArgs... args)       \
+    {                                                                 \
+        queue->submit([&](sycl::handler &cgh) {                       \
+            cgh.parallel_for(sycl_nd_range(grid, block),              \
+                             [=](sycl::nd_item<3> item_ct1) {         \
+                                 kernel_<encoded>(args..., item_ct1); \
+                             });                                      \
+        });                                                           \
     }
 
 /**
@@ -106,7 +106,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_ENABLE_DEFAULT_CONFIG_CALL(name_, callable_, list_)               \
     template <typename... InferredArgs>                                       \
     void name_(std::uint32_t desired_cfg, dim3 grid, dim3 block,              \
-               size_t dynamic_shared_memory, sycl::queue *queue,              \
+               gko::size_type dynamic_shared_memory, sycl::queue *queue,      \
                InferredArgs... args)                                          \
     {                                                                         \
         callable_(                                                            \
@@ -174,7 +174,6 @@ std::uint32_t get_first_cfg(IterArr &arr, Validate verify)
         }
     }
     GKO_NOT_SUPPORTED(arr);
-    return 0;
 }
 
 
diff --git a/dpcpp/components/thread_ids.dp.hpp b/dpcpp/components/thread_ids.dp.hpp
index f9decfd989d..70ad76d9ccb 100644
--- a/dpcpp/components/thread_ids.dp.hpp
+++ b/dpcpp/components/thread_ids.dp.hpp
@@ -52,6 +52,12 @@ namespace dpcpp {
 namespace thread {
 
 
+// TODO: porting - need to refine functions and their name in this file
+// the grid/block description uses the cuda dim3 to represent. i.e. using dim3
+// to launch dpcpp kernel, the kernel will reverse the ordering to keep the same
+// linear memory usage as cuda.
+
+
 /**
  * @internal
  *
diff --git a/dpcpp/components/uninitialized_array.hpp b/dpcpp/components/uninitialized_array.hpp
index d9d423c9c94..eb8a36770d7 100644
--- a/dpcpp/components/uninitialized_array.hpp
+++ b/dpcpp/components/uninitialized_array.hpp
@@ -45,6 +45,9 @@ namespace kernels {
 namespace dpcpp {
 
 
+// TODO: porting - consider directly use the array as shared memory
+
+
 /**
  * Stores an array with uninitialized contents.
  *

From ce20100d5db65e3a4dcdeaea23d75f3aee2f0121 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 21 Jul 2021 12:04:00 +0200
Subject: [PATCH 0908/1008] explict type in for, use func not macro to skip,
 remove dup test note: gtest_skip can not be in another function call
 Co-authored-by: Tobias Ribizel <ribizel@kit.edu>

---
 .../par_ilut_select_kernels.hpp.inc           |   8 +-
 cuda/test/components/sorting_kernels.cu       |   2 +-
 dpcpp/CMakeLists.txt                          |   2 +-
 dpcpp/test/matrix/dense_kernels.cpp           | 365 +-----------------
 dpcpp/test/utils.hpp                          |  56 +++
 5 files changed, 67 insertions(+), 366 deletions(-)
 create mode 100644 dpcpp/test/utils.hpp

diff --git a/common/factorization/par_ilut_select_kernels.hpp.inc b/common/factorization/par_ilut_select_kernels.hpp.inc
index 9b4897d1766..e443d7b6ba7 100644
--- a/common/factorization/par_ilut_select_kernels.hpp.inc
+++ b/common/factorization/par_ilut_select_kernels.hpp.inc
@@ -62,8 +62,7 @@ __global__ __launch_bounds__(searchtree_width) void build_searchtree(
     // assuming rounding towards zero
     auto stride = double(size) / sample_size;
 #pragma unroll
-    for (auto i = decltype(sampleselect_oversampling){0};
-         i < sampleselect_oversampling; ++i) {
+    for (int i = 0; i < sampleselect_oversampling; ++i) {
         auto lidx = idx * sampleselect_oversampling + i;
         auto val = input[static_cast<IndexType>(lidx * stride)];
         samples[i] = abs(val);
@@ -120,8 +119,7 @@ __global__ __launch_bounds__(default_block_size) void count_buckets(
         auto el = abs(input[i]);
         IndexType tree_idx{};
 #pragma unroll
-        for (auto level = decltype(sampleselect_searchtree_height){0};
-             level < sampleselect_searchtree_height; ++level) {
+        for (int level = 0; level < sampleselect_searchtree_height; ++level) {
             auto cmp = !(el < sh_tree[tree_idx]);
             tree_idx = 2 * tree_idx + 1 + cmp;
         }
@@ -170,7 +168,7 @@ __global__ __launch_bounds__(default_block_size) void block_prefix_sum(
     // compute prefix sum over warp-sized blocks
     IndexType total{};
     auto base_idx = warp_idx * work_per_warp * warp.size();
-    for (auto step = decltype(work_per_warp){0}; step < work_per_warp; ++step) {
+    for (IndexType step = 0; step < work_per_warp; ++step) {
         auto idx = warp_lane + step * warp.size() + base_idx;
         auto val = idx < num_blocks ? local_counters[idx] : zero<IndexType>();
         IndexType warp_total{};
diff --git a/cuda/test/components/sorting_kernels.cu b/cuda/test/components/sorting_kernels.cu
index f61bbd0694e..e2b7abc51d7 100644
--- a/cuda/test/components/sorting_kernels.cu
+++ b/cuda/test/components/sorting_kernels.cu
@@ -99,7 +99,7 @@ protected:
     {
         // we want some duplicate elements
         std::uniform_int_distribution<gko::int32> dist(0, num_elements / 2);
-        for (auto i = decltype(num_elements){0}; i < num_elements; ++i) {
+        for (int i = 0; i < num_elements; ++i) {
             ref_shared.get_data()[i] = dist(rng);
         }
         ddata = gko::Array<gko::int32>{cuda, ref_shared};
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 7729588d363..e2d476164e8 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -75,7 +75,7 @@ else ()
     target_link_options(ginkgo_dpcpp PUBLIC -fsycl-device-code-split=per_kernel)
 endif()
 target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
-target_link_libraries(ginkgo_dpcpp PRIVATE $<LINK_ONLY:MKL::MKL_DPCPP>)
+target_link_libraries(ginkgo_dpcpp PRIVATE MKL::MKL_DPCPP)
 if (GINKGO_DPCPP_SINGLE_MODE)
     target_compile_definitions(ginkgo_dpcpp PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
 endif()
diff --git a/dpcpp/test/matrix/dense_kernels.cpp b/dpcpp/test/matrix/dense_kernels.cpp
index 257ee6fbc6a..cf1bbe26cd4 100644
--- a/dpcpp/test/matrix/dense_kernels.cpp
+++ b/dpcpp/test/matrix/dense_kernels.cpp
@@ -51,6 +51,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/fill_array.hpp"
 #include "core/matrix/dense_kernels.hpp"
 #include "core/test/utils.hpp"
+#include "dpcpp/test/utils.hpp"
 
 
 namespace {
@@ -194,127 +195,6 @@ class Dense : public ::testing::Test {
 };
 
 
-TEST_F(Dense, DpcppFillIsEquivalentToRef)
-{
-    set_up_vector_data(3);
-    auto result = Mtx::create(ref);
-
-    x->fill(42);
-    dx->fill(42);
-    result->copy_from(dx.get());
-
-    GKO_ASSERT_MTX_NEAR(result, x, r<vtype>::value);
-}
-
-
-TEST_F(Dense, DpcppStridedFillIsEquivalentToRef)
-{
-    using T = vtype;
-    auto x = gko::initialize<gko::matrix::Dense<T>>(
-        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, ref);
-    auto dx = gko::initialize<gko::matrix::Dense<T>>(
-        4, {I<T>{1.0, 2.0}, I<T>{3.0, 4.0}, I<T>{5.0, 6.0}}, dpcpp);
-    auto result = Mtx::create(ref);
-
-    x->fill(42);
-    dx->fill(42);
-    result->copy_from(dx.get());
-
-    GKO_ASSERT_MTX_NEAR(result, x, r<vtype>::value);
-}
-
-
-TEST_F(Dense, SingleVectorDpcppScaleIsEquivalentToRef)
-{
-    set_up_vector_data(1);
-    auto result = Mtx::create(ref);
-
-    x->scale(alpha.get());
-    dx->scale(dalpha.get());
-    result->copy_from(dx.get());
-
-    GKO_ASSERT_MTX_NEAR(result, x, r<vtype>::value);
-}
-
-
-TEST_F(Dense, MultipleVectorDpcppScaleIsEquivalentToRef)
-{
-    set_up_vector_data(20);
-
-    x->scale(alpha.get());
-    dx->scale(dalpha.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
-}
-
-
-TEST_F(Dense, MultipleVectorDpcppScaleWithDifferentAlphaIsEquivalentToRef)
-{
-    set_up_vector_data(20, true);
-
-    x->scale(alpha.get());
-    dx->scale(dalpha.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
-}
-
-
-TEST_F(Dense, SingleVectorDpcppAddScaledIsEquivalentToRef)
-{
-    set_up_vector_data(1);
-
-    x->add_scaled(alpha.get(), y.get());
-    dx->add_scaled(dalpha.get(), dy.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
-}
-
-
-TEST_F(Dense, MultipleVectorDpcppAddScaledIsEquivalentToRef)
-{
-    set_up_vector_data(20);
-
-    x->add_scaled(alpha.get(), y.get());
-    dx->add_scaled(dalpha.get(), dy.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
-}
-
-
-TEST_F(Dense, MultipleVectorDpcppAddScaledWithDifferentAlphaIsEquivalentToRef)
-{
-    set_up_vector_data(20);
-
-    x->add_scaled(alpha.get(), y.get());
-    dx->add_scaled(dalpha.get(), dy.get());
-
-    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
-}
-
-
-TEST_F(Dense, AddsScaledDiagIsEquivalentToRef)
-{
-    auto mat = gen_mtx<Mtx>(532, 532);
-    gko::Array<Mtx::value_type> diag_values(ref, 532);
-    gko::kernels::reference::components::fill_array(ref, diag_values.get_data(),
-                                                    532, Mtx::value_type{2.0});
-    auto diag =
-        gko::matrix::Diagonal<Mtx::value_type>::create(ref, 532, diag_values);
-    alpha = gko::initialize<Mtx>({2.0}, ref);
-    auto dmat = Mtx::create(dpcpp);
-    dmat->copy_from(mat.get());
-    auto ddiag = gko::matrix::Diagonal<Mtx::value_type>::create(dpcpp);
-    ddiag->copy_from(diag.get());
-    dalpha = Mtx::create(dpcpp);
-    dalpha->copy_from(alpha.get());
-
-    mat->add_scaled(alpha.get(), diag.get());
-    dmat->add_scaled(dalpha.get(), ddiag.get());
-
-    GKO_ASSERT_MTX_NEAR(mat, dmat, r<vtype>::value);
-}
-
-
 TEST_F(Dense, SingleVectorDpcppComputeDotIsEquivalentToRef)
 {
     set_up_vector_data(1);
@@ -384,11 +264,9 @@ TEST_F(Dense, SimpleApplyIsEquivalentToRef)
 }
 
 
-#if !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Dense, SimpleApplyMixedIsEquivalentToRef)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data();
 
     x->apply(convert<MixedMtx>(y).get(), convert<MixedMtx>(expected).get());
@@ -398,9 +276,6 @@ TEST_F(Dense, SimpleApplyMixedIsEquivalentToRef)
 }
 
 
-#endif  // !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -412,11 +287,9 @@ TEST_F(Dense, AdvancedApplyIsEquivalentToRef)
 }
 
 
-#if !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Dense, AdvancedApplyMixedIsEquivalentToRef)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data();
 
     x->apply(convert<MixedMtx>(alpha).get(), convert<MixedMtx>(y).get(),
@@ -428,11 +301,9 @@ TEST_F(Dense, AdvancedApplyMixedIsEquivalentToRef)
 }
 
 
-#endif  // !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data();
     auto complex_b = gen_mtx<ComplexMtx>(25, 1);
     auto dcomplex_b = ComplexMtx::create(dpcpp);
@@ -448,11 +319,9 @@ TEST_F(Dense, ApplyToComplexIsEquivalentToRef)
 }
 
 
-#if !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Dense, ApplyToMixedComplexIsEquivalentToRef)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data();
     auto complex_b = gen_mtx<MixedComplexMtx>(25, 1);
     auto dcomplex_b = MixedComplexMtx::create(dpcpp);
@@ -467,8 +336,6 @@ TEST_F(Dense, ApplyToMixedComplexIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-7);
 }
 
-#endif  // !GINKGO_DPCPP_SINGLE_MODE
-
 
 TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
 {
@@ -487,11 +354,9 @@ TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef)
 }
 
 
-#if !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Dense, AdvancedApplyToMixedComplexIsEquivalentToRef)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data();
     auto complex_b = gen_mtx<MixedComplexMtx>(25, 1);
     auto dcomplex_b = MixedComplexMtx::create(dpcpp);
@@ -509,9 +374,6 @@ TEST_F(Dense, AdvancedApplyToMixedComplexIsEquivalentToRef)
 }
 
 
-#endif  // !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Dense, ComputeDotComplexIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -733,219 +595,4 @@ TEST_F(Dense, CalculateTotalColsIsEquivalentToRef)
 }
 
 
-TEST_F(Dense, CanGatherRows)
-{
-    set_up_apply_data();
-
-    auto r_gather = x->row_gather(rgather_idxs.get());
-    auto dr_gather = dx->row_gather(rgather_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
-}
-
-
-TEST_F(Dense, CanGatherRowsIntoDense)
-{
-    set_up_apply_data();
-    auto gather_size =
-        gko::dim<2>{rgather_idxs->get_num_elems(), x->get_size()[1]};
-    auto r_gather = Mtx::create(ref, gather_size);
-    // test make_temporary_clone and non-default stride
-    auto dr_gather = Mtx::create(ref, gather_size, x->get_size()[1] + 2);
-
-    x->row_gather(rgather_idxs.get(), r_gather.get());
-    dx->row_gather(rgather_idxs.get(), dr_gather.get());
-
-    GKO_ASSERT_MTX_NEAR(r_gather.get(), dr_gather.get(), 0);
-}
-
-
-TEST_F(Dense, IsPermutable)
-{
-    set_up_apply_data();
-
-    auto permuted = square->permute(rpermute_idxs.get());
-    auto dpermuted = dsquare->permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
-                        static_cast<Mtx *>(dpermuted.get()), 0);
-}
-
-
-TEST_F(Dense, IsInversePermutable)
-{
-    set_up_apply_data();
-
-    auto permuted = square->inverse_permute(rpermute_idxs.get());
-    auto dpermuted = dsquare->inverse_permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(permuted.get()),
-                        static_cast<Mtx *>(dpermuted.get()), 0);
-}
-
-
-TEST_F(Dense, IsRowPermutable)
-{
-    set_up_apply_data();
-
-    auto r_permute = x->row_permute(rpermute_idxs.get());
-    auto dr_permute = dx->row_permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(r_permute.get()),
-                        static_cast<Mtx *>(dr_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsColPermutable)
-{
-    set_up_apply_data();
-
-    auto c_permute = x->column_permute(cpermute_idxs.get());
-    auto dc_permute = dx->column_permute(cpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(c_permute.get()),
-                        static_cast<Mtx *>(dc_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsInverseRowPermutable)
-{
-    set_up_apply_data();
-
-    auto inverse_r_permute = x->inverse_row_permute(rpermute_idxs.get());
-    auto d_inverse_r_permute = dx->inverse_row_permute(rpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_r_permute.get()),
-                        static_cast<Mtx *>(d_inverse_r_permute.get()), 0);
-}
-
-
-TEST_F(Dense, IsInverseColPermutable)
-{
-    set_up_apply_data();
-
-    auto inverse_c_permute = x->inverse_column_permute(cpermute_idxs.get());
-    auto d_inverse_c_permute = dx->inverse_column_permute(cpermute_idxs.get());
-
-    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(inverse_c_permute.get()),
-                        static_cast<Mtx *>(d_inverse_c_permute.get()), 0);
-}
-
-
-TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto diag = x->extract_diagonal();
-    auto ddiag = dx->extract_diagonal();
-
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-}
-
-
-TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto diag = y->extract_diagonal();
-    auto ddiag = dy->extract_diagonal();
-
-    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
-}
-
-
-TEST_F(Dense, InplaceAbsoluteMatrixIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    x->compute_absolute_inplace();
-    dx->compute_absolute_inplace();
-
-    GKO_ASSERT_MTX_NEAR(x, dx, r<vtype>::value);
-}
-
-
-TEST_F(Dense, OutplaceAbsoluteMatrixIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto abs_x = x->compute_absolute();
-    auto dabs_x = dx->compute_absolute();
-
-    GKO_ASSERT_MTX_NEAR(abs_x, dabs_x, r<vtype>::value);
-}
-
-
-TEST_F(Dense, MakeComplexIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto complex_x = x->make_complex();
-    auto dcomplex_x = dx->make_complex();
-
-    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
-}
-
-
-TEST_F(Dense, MakeComplexWithGivenResultIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto complex_x = ComplexMtx::create(ref, x->get_size());
-    x->make_complex(complex_x.get());
-    auto dcomplex_x = ComplexMtx::create(dpcpp, x->get_size());
-    dx->make_complex(dcomplex_x.get());
-
-    GKO_ASSERT_MTX_NEAR(complex_x, dcomplex_x, 0);
-}
-
-
-TEST_F(Dense, GetRealIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto real_x = x->get_real();
-    auto dreal_x = dx->get_real();
-
-    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
-}
-
-
-TEST_F(Dense, GetRealWithGivenResultIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto real_x = Mtx::create(ref, x->get_size());
-    x->get_real(real_x.get());
-    auto dreal_x = Mtx::create(dpcpp, dx->get_size());
-    dx->get_real(dreal_x.get());
-
-    GKO_ASSERT_MTX_NEAR(real_x, dreal_x, 0);
-}
-
-
-TEST_F(Dense, GetImagIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto imag_x = x->get_imag();
-    auto dimag_x = dx->get_imag();
-
-    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
-}
-
-
-TEST_F(Dense, GetImagWithGivenResultIsEquivalentToRef)
-{
-    set_up_apply_data();
-
-    auto imag_x = Mtx::create(ref, x->get_size());
-    x->get_imag(imag_x.get());
-    auto dimag_x = Mtx::create(dpcpp, dx->get_size());
-    dx->get_imag(dimag_x.get());
-
-    GKO_ASSERT_MTX_NEAR(imag_x, dimag_x, 0);
-}
-
-
 }  // namespace
diff --git a/dpcpp/test/utils.hpp b/dpcpp/test/utils.hpp
new file mode 100644
index 00000000000..57e703b8ef1
--- /dev/null
+++ b/dpcpp/test/utils.hpp
@@ -0,0 +1,56 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_TEST_UTILS_HPP_
+#define GKO_DPCPP_TEST_UTILS_HPP_
+
+
+#include <gtest/gtest.h>
+
+
+namespace {
+
+
+#if GINKGO_DPCPP_SINGLE_MODE
+#define SKIP_IF_SINGLE_MODE GTEST_SKIP() << "Skip due to single mode"
+#else
+#define SKIP_IF_SINGLE_MODE                                                  \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
+                  "semi-colon warnings")
+#endif
+
+
+}  // namespace
+
+
+#endif  // GKO_DPCPP_TEST_UTILS_HPP_

From 048570dce77ad3bf208941e09eef5d0cae602820 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 21 Jul 2021 16:44:49 +0200
Subject: [PATCH 0909/1008] debug/static dpcpp -> debug/shared due to memory

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index df3265cb58c..1a884c2b408 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -754,7 +754,7 @@ build/dpcpp/opencl_igpu/release/static:
     SYCL_DEVICE_FILTER: "OpenCL"
     SYCL_DEVICE_TYPE: "GPU"
 
-build/dpcpp/level_zero_igpu/debug/static:
+build/dpcpp/level_zero_igpu/debug/shared:
   <<: *default_build_with_test
   extends:
     - .full_test_condition
@@ -765,7 +765,7 @@ build/dpcpp/level_zero_igpu/debug/static:
     CXX_COMPILER: "dpcpp"
     BUILD_DPCPP: "ON"
     BUILD_TYPE: "Debug"
-    BUILD_SHARED_LIBS: "OFF"
+    BUILD_SHARED_LIBS: "ON"
     DPCPP_SINGLE_MODE: "ON"
     SYCL_DEVICE_FILTER: "Level_Zero:GPU"
 

From c1eea639560071e7c5bd9eee032973a966c7282e Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 27 May 2021 22:14:04 +0200
Subject: [PATCH 0910/1008] initial the dpcpp porting script

---
 dev_tools/oneapi/add_host_function.sh         | 305 +++++++++++++
 dev_tools/oneapi/convert_source.sh            | 429 ++++++++++++++++++
 .../fake_interface/cooperative_groups.cuh     |  57 +++
 .../oneapi/working_directory/.clang-format    | 108 +++++
 .../trick/cooperative_groups.hpp              |  72 +++
 .../oneapi/working_directory/trick/dim3_t.hpp |  48 ++
 .../working_directory/trick/reduction.hpp     |  57 +++
 .../working_directory/trick/sorting.hpp       |  62 +++
 .../working_directory/trick/thread_ids.hpp    | 299 ++++++++++++
 9 files changed, 1437 insertions(+)
 create mode 100755 dev_tools/oneapi/add_host_function.sh
 create mode 100755 dev_tools/oneapi/convert_source.sh
 create mode 100644 dev_tools/oneapi/fake_interface/cooperative_groups.cuh
 create mode 100644 dev_tools/oneapi/working_directory/.clang-format
 create mode 100644 dev_tools/oneapi/working_directory/trick/cooperative_groups.hpp
 create mode 100644 dev_tools/oneapi/working_directory/trick/dim3_t.hpp
 create mode 100644 dev_tools/oneapi/working_directory/trick/reduction.hpp
 create mode 100644 dev_tools/oneapi/working_directory/trick/sorting.hpp
 create mode 100644 dev_tools/oneapi/working_directory/trick/thread_ids.hpp

diff --git a/dev_tools/oneapi/add_host_function.sh b/dev_tools/oneapi/add_host_function.sh
new file mode 100755
index 00000000000..6f5a0dd6574
--- /dev/null
+++ b/dev_tools/oneapi/add_host_function.sh
@@ -0,0 +1,305 @@
+#!/bin/bash
+
+extract_varname() {
+    local str="$1"
+    # local GET_PARAM=" *([^ ]*) *$"
+    # Need to remove the  = ....
+    # note. it only remove the simple one
+    local GET_PARAM=" *([^ =\*]*) *(= *.*)* *$"
+    local parameter=""
+    local temp=""
+    IFS=',' read -ra par <<< "$str"
+    for var in "${par[@]}"; do
+        if [ -n "${temp}" ]; then
+            temp="${temp},"
+        fi
+        temp="${temp}${var}"
+        # only handle one pair <> currently
+        if [[ "${temp}" =~ "<" ]] && [[ ! "${temp}" =~ ">" ]]; then
+            continue
+        fi
+        # If the string contains typename, do not extract it.
+        # It should automatically be decided from argument
+        # Also need to ignore = ...
+        if [[ "${temp}" =~ "typename" ]]; then
+            :
+        elif [[  "${temp}" =~ $GET_PARAM ]]; then
+            if [ -n "${parameter}" ]; then
+                parameter="${parameter}, "
+            fi
+            parameter="${parameter}${BASH_REMATCH[1]}"
+        fi
+        temp=""
+    done
+    echo "$parameter"
+}
+
+check_closed() {
+    local str="$1"
+    str="${str//->}"
+    str_start="${str//[^(<\[]}"
+    str_end="${str//[^>)\]]}"
+    if [[ "${#str_start}" -eq "${#str_end}" ]]; then
+        echo "true"
+    else
+        echo "false"
+    fi
+}
+CONFIG_SELECTION_SUFFIX="_CONFIG"
+generate_config_selection() {
+    local template="$1"
+    local name="$2"
+    local variable="$3"
+    local var_input="$4"
+    template=$(echo ${template} | sed -E 's/^ *template *<(.*)> *$/\1/g')
+    local temp=""
+    IFS=',' read -ra par <<< "$template"
+
+    local id_list=("bool" "int" "size_type" "typename")
+    # Bash 4+ required
+    declare -A rev_id_list
+    local rev_id_list=(["bool"]=0 ["int"]=1 ["size_type"]=2 ["typename"]=3 ["Config"]=4)
+    declare -A list_list
+    list_list=(["bool"]="bool" ["int"]="int" ["size_type"]="size_type" ["typename"]="")
+    local max_num=0;
+    local param_regex="([^ =\*]*) *([^ =\*]*) *(= *.*)* *$"
+    local template_rm_config=""
+    local default_config=""
+    for var in "${par[@]}"; do
+        # echo "varaible - ${var}"
+        if [ -n "${temp}" ]; then
+            temp="${temp},"
+        fi
+        temp="${temp}${var}"
+        is_closed=$(check_closed "$temp")
+        if [[ "${is_closed}" = "false" ]]; then
+            continue
+        fi
+        # echo "temp - ${temp}"
+        # If the string contains typename, do not extract it.
+        # It should automatically be decided from argument
+        local config_regex="Config *([a-zA-Z0-9_]*) *= *([a-zA-Z][a-zA-Z_:,\(\) ]*)"
+        if [[ "${temp}" =~ ${config_regex} ]]; then
+            max_num=${rev_id_list[Config]}
+            default_config="${BASH_REMATCH[2]}"
+            temp=""
+            continue
+        fi
+        if [ -n "${template_rm_config}" ]; then
+                template_rm_config="${template_rm_config}, "
+        fi
+        template_rm_config="${template_rm_config}${temp}"
+        if [[  "${temp}" =~ $param_regex ]]; then
+            identification=${BASH_REMATCH[1]}
+            parameter_name=${BASH_REMATCH[2]}
+            # echo "${identification} ${parameter_name}"
+            if [ "${max_num}" -eq "${rev_id_list[Config]}" ]; then
+                if [ "${identification}" != "typename" ]; then
+                    echo "static_assert(false, \"Only inferred type are allowed after Config\");"
+                fi
+            else
+                if [ "${rev_id_list[${identification}]}" -ge "${max_num}" ]; then
+                    max_num=${rev_id_list[${identification}]}
+                    if [ -n "${list_list[${identification}]}" ]; then
+                        parameter_name=", ${parameter_name}"
+                    fi
+                    list_list[${identification}]="${list_list[${identification}]}${parameter_name}"
+                else
+                    echo "static_assert(false, \"Need to follow bool, int, size_type, type ordering\");"
+                fi
+            fi
+        fi
+        temp=""
+    done
+    # for item in "${!list_list[@]}"; do
+    #     echo "//$item - ${list_list[$item]}"
+    # done
+    config_generation=""
+    if [[ "${default_config}" = *"config_set"* ]]; then
+        # When the config is set in template, use it directly.
+        config_generation="::gko::syn::value_list<Config, ${default_config}>();"
+    else
+        # When config use the constexpr variable, use the variable name. The constexpr variable will be changed to Config_list afterwards.
+        config_generation="${default_config}_list"
+    fi
+    echo "template <${template_rm_config}>
+        void ${name}${CONFIG_SELECTION_SUFFIX}
+        (dim3 grid, dim3 block, size_t dynamic_shared_memory, cudaStream_t stream, std::shared_ptr<const CudaExecutor> exec, ${variable}) {
+        auto config_list = ${config_generation};
+        auto exec_config = exec->get_config();
+        ${name}${CONFIG_SELECTION_SUFFIX}(
+            config_list,
+            [&exec_config] (Config config) {
+                return (get_warp_size(exec_config) == get_warp_size(config)) &&
+                        (get_block_size(exec_config) >= get_block_size(config));
+                },
+            ::gko::syn::value_list<${list_list[bool]}>(),
+            ::gko::syn::value_list<${list_list[int]}>(),
+            ::gko::syn::value_list<${list_list[size_type]}>(),
+            ::gko::syn::type_list<${list_list[typename]}>(),
+            grid, block, dynamic_shared_memory, stream, ${var_input});
+    }
+    "
+}
+
+
+GLOBAL_KEYWORD="__global__"
+TEMPLATE_REGEX="^ *template <*"
+FUNCTION_START="^ *(template *<|${GLOBAL_KEYWORD}|void)"
+FUNCTION_NAME_END=".*\{.*"
+SCOPE_START="${FUNCTION_NAME_END}"
+SCOPE_END=".*\}.*"
+CHECK_GLOBAL_KEYWORD=".*${GLOBAL_KEYWORD}.*"
+FUNCTION_HANDLE=""
+DURING_FUNCNAME="false"
+ANAYSIS_FUNC=" *(template *<(.*)>)?.* (.*)\((.*)\)"
+START_BLOCK_REX="^( *\/\*| *\/\/)"
+END_BLOCK_REX="\*\/$| *\/\/"
+IN_BLOCK=0
+IN_FUNC=0
+STORE_LINE=""
+STORE_REGEX="__ *$"
+HOST_SUFFIX="_AUTOHOSTFUNC"
+EXTRACT_KERNEL="false"
+GINKGO_LICENSE_BEACON="******************************<GINKGO LICENSE>******************************"
+DURING_LICENSE="false"
+SKIP="false"
+MAP_FILE="map_list"
+rm "${MAP_FILE}"
+while IFS='' read -r line || [ -n "$line" ]; do
+    if [ "${EXTRACT_KERNEL}" = "false" ] && ([ "${line}" = "/*${GINKGO_LICENSE_BEACON}" ] ||  [ "${DURING_LICENSE}" = "true" ]); then
+        DURING_LICENSE="true"
+        if [ "${line}" = "${GINKGO_LICENSE_BEACON}*/" ]; then
+            DURING_LICENSE="false"
+            SKIP="true"
+        fi
+        continue
+    fi
+    # When do not need the license, do not need the space between license and other codes, neither.
+    if [ ${SKIP} = "true" ] && [ -z "${line}" ]; then
+        continue
+    fi
+    SKIP="false"
+    if [[ "$line" =~ ${STORE_REGEX} ]]; then
+        STORE_LINE="${STORE_LINE} ${line}"
+    elif [[ -n "${STORE_LINE}" ]]; then
+        echo "${STORE_LINE} ${line}"
+        STORE_LINE=""
+    else
+        echo "${line}"
+    fi
+    # echo "Handle___ ${line}"
+    if [[ "$line" =~ ${START_BLOCK_REX} ]] || [[ "${IN_BLOCK}" -gt 0 ]]; then
+        if [[ "$line" =~ ${START_BLOCK_REX} ]]; then
+            IN_BLOCK=$((IN_BLOCK+1))
+        fi
+        if [[ "$line" =~ ${END_BLOCK_REX} ]]; then
+            IN_BLOCK=$((IN_BLOCK-1))
+        fi
+        # echo ""
+        # echo "IN BLOCK ${IN_BLOCK}"
+        # output to new file
+        continue
+    fi
+    # echo "Handle ${line}"
+    # handle comments
+    if [[ "${line}" =~ $FUNCTION_START ]] || [[ $DURING_FUNCNAME = "true" ]]; then
+        # echo "line ${line}"
+        # echo "${FUNCTION_NAME_END}"
+        DURING_FUNCNAME="true"
+        FUNCTION_HANDLE="${FUNCTION_HANDLE} $line"
+        if [[ "${line}" =~ ${FUNCTION_NAME_END} ]]; then
+            # echo "end"
+            DURING_FUNCNAME="false"
+        fi
+        if [[ "${line}" =~ ${SCOPE_START} ]]; then
+            IN_FUNC=$((IN_FUNC+1))
+        fi
+        if [[ "${line}" =~ ${SCOPE_END} ]]; then
+            IN_FUNC=$((IN_FUNC-1))
+        fi
+        # output to new file
+        continue
+    fi
+    # echo "Handle ${line}"
+    if [ -n "${FUNCTION_HANDLE}" ] && [[ ${DURING_FUNCNAME} = "false" ]]; then
+        if [[ "${line}" =~ ${SCOPE_START} ]]; then
+            IN_FUNC=$((IN_FUNC+1))
+        fi
+        if [[ "${line}" =~ ${SCOPE_END} ]]; then
+            IN_FUNC=$((IN_FUNC-1))
+        fi
+        # echo "IN FUNC ${IN_FUNC}"
+
+        # make sure the function is end
+        if [[ "${IN_FUNC}" -eq 0 ]]; then
+            # echo "check ${FUNCTION_HANDLE}"
+
+            if [[ "${FUNCTION_HANDLE}" =~ $CHECK_GLOBAL_KEYWORD ]]; then
+                echo ""
+                # echo "${FUNCTION_HANDLE}"
+                # remove additional space
+                FUNCTION_HANDLE=$(echo "${FUNCTION_HANDLE}" | sed -E 's/ +/ /g;')
+                # echo "->"
+                # echo "${FUNCTION_HANDLE}"
+                # echo "->"
+
+                if [[ "${FUNCTION_HANDLE}" =~ $ANAYSIS_FUNC ]]; then
+                    TEMPLATE="${BASH_REMATCH[1]}"
+                    TEMPLATE_CONTENT="${BASH_REMATCH[2]}"
+                    NAME="${BASH_REMATCH[3]}"
+                    VARIABLE="${BASH_REMATCH[4]}"
+                    VARIABLE=$(echo ${VARIABLE} | sed 's/__restrict__ //g')
+                    VAR_INPUT=$(extract_varname "${VARIABLE}")
+                    TEMPLATE_INPUT=$(extract_varname "${TEMPLATE_CONTENT}")
+                    if [ -n "${TEMPLATE_INPUT}" ]; then
+                        TEMPLATE_INPUT="<${TEMPLATE_INPUT}>"
+                    fi
+                    echo "${TEMPLATE} void ${NAME}${HOST_SUFFIX} (dim3 grid, dim3 block, size_t dynamic_shared_memory, cudaStream_t stream, ${VARIABLE}) {
+                        /*KEEP*/${NAME}${TEMPLATE_INPUT}<<<grid, block, dynamic_shared_memory, stream>>>(${VAR_INPUT});
+                        }"
+                    if [[ "${FUNCTION_HANDLE}" = *"Config"* ]]; then
+                        # echo "/** DPCPP ONLY // RM_TAG"
+                        echo ""
+                        echo "GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(${NAME}${HOST_SUFFIX}${CONFIG_SELECTION_SUFFIX}, ${NAME}${HOST_SUFFIX})"
+                        echo ""
+                        echo ""
+                        config_function=$(generate_config_selection "${TEMPLATE}" "${NAME}${HOST_SUFFIX}" "${VARIABLE}" "${VAR_INPUT}")
+                        echo "${config_function}"
+                        # echo "**/ // RM_TAG"
+                        echo ""
+                        echo ""
+                        # echo "${TEMPLATE} void ${NAME}${HOST_SUFFIX}${CONFIG_SELECTION_SUFFIX} // RM_TAG
+                        #     (dim3 grid, dim3 block, size_t dynamic_shared_memory, cudaStream_t stream, std::shared_ptr<const CudaExecutor> exec, ${VARIABLE}) { // RM_TAG
+                        #     ${NAME}${TEMPLATE_INPUT}<<<grid, block, dynamic_shared_memory, stream>>>(${VAR_INPUT}); // RM_TAG
+                        #     } // RM_TAG"
+                        # write the map to the file ${NAME} -> ${NAME}${HOST_SUFFIX}${CONFIG_SELECTION_SUFFIX}
+                        echo "${NAME} -> ${NAME}${HOST_SUFFIX}${CONFIG_SELECTION_SUFFIX}" >> ${MAP_FILE}
+                    else
+                        # else, write the map to the file ${NAME} -> ${NAME}${HOST_SUFFIX}
+                        echo "${NAME} -> ${NAME}${HOST_SUFFIX}" >> ${MAP_FILE}
+                    fi
+                fi
+                # echo ""
+                # check the property
+                # extract template
+                # maybe remove any [[ ]]
+                # extract function name
+                # extract function variables
+
+                # check Config
+                # extract bool, int, size_type, typename before Config
+                # add one function like original one
+                # add selection_config macro
+                # need to keep the map
+            fi
+            FUNCTION_HANDLE=""
+        fi
+    fi
+
+
+done < "$1"
+
+# Maybe it only works in Linux
+sort "${MAP_FILE}" | uniq > "${MAP_FILE}_temp"
+mv "${MAP_FILE}_temp"  "${MAP_FILE}"
diff --git a/dev_tools/oneapi/convert_source.sh b/dev_tools/oneapi/convert_source.sh
new file mode 100755
index 00000000000..6b26bffc831
--- /dev/null
+++ b/dev_tools/oneapi/convert_source.sh
@@ -0,0 +1,429 @@
+#!/bin/bash
+
+# the following parameters set by environment
+# MUST:
+#   CUDA_HEADER_DIR: contains the cuda headers
+# OPTIONAL:
+#   ROOT_DIR: the ginkgo folder. The default is current path
+#   BUILD_DIR: the ginkgo build folder, which needs cmake to generate config.hpp and gtest include. The default is "build"
+#              Note. It requires GINKGO_BUILD_TESTS=ON to download gtest but does not need to compile ginkgo.
+#                    If GTEST_HEADER_DIR is from other place, does not require GINKGO_BUILD_TESTS.
+#                    If copy the ginkgo config.hpp from other ginkgo build into "${ROOT_DIR}/include/ginkgo/", do not require cmake.
+#   ROOT_BUILD_DIR: the complete path for build folder. The default is "${ROOT_DIR}/${BUILD_DIR}"
+#   GTEST_HEADER_DIR: the gtest header folder. The default is "${ROOT_BUILD_DIR}/third_party_gtest/src/googletest/include"
+#   CLANG_FORMAT: the clang-format exec. The default is "clang-format"
+CURRENT_DIR="$( pwd )"
+cd "$( dirname "${BASH_SOURCE[0]}" )"
+SCRIPT_DIR="$( pwd )"
+ROOT_DIR="${ROOT_DIR:="${CURRENT_DIR}"}"
+BUILD_DIR="${BUILD_DIR:="build"}"
+ROOT_BUILD_DIR="${ROOT_BUILD_DIR:="${ROOT_DIR}/${BUILD_DIR}"}"
+CUDA_HEADER_DIR="${CUDA_HEADER_DIR}"
+GTEST_HEADER_DIR="${GTEST_HEADER_DIR:="${ROOT_BUILD_DIR}/third_party/gtest/src/googletest/include"}"
+CLANG_FORMAT=${CLANG_FORMAT:="clang-format"}
+if [[ "${VERBOSE}" == 1 ]]; then
+    echo "the current setting is "
+    echo "CURRENT_DIR ${CURRENT_DIR}"
+    echo "SCRIPT_DIR ${SCRIPT_DIR}"
+    echo "ROOT_DIR ${ROOT_DIR}"
+    echo "ROOT_BUILD_DIR ${ROOT_BUILD_DIR}"
+    echo "GTEST_HEADER_DIR ${GTEST_HEADER_DIR}"
+    echo "CUDA_HEADER_DIR ${CUDA_HEADER_DIR}"
+    echo "ROOT_BUILD_DIR ${ROOT_BUILD_DIR}"
+    echo "CLANG_FORMAT ${CLANG_FORMAT}"
+fi
+if [[ "${CUDA_HEADER_DIR}" == "" ]]; then
+    echo "Please set the CUDA_HEADER_DIR in environment"
+    exit 1
+fi
+# move to working_directory
+cd working_directory
+
+
+KERNEL_SYNTAX_START="<<<"
+KERNEL_SYNTAX_END=">>>"
+DEVICE_CODE_SYNTAX="#include \"(common.*)\""
+FUNCTION_END=");"
+SPECIAL_SUFFIX="_AUTOHOSTFUNC"
+EXTRACT_KERNEL="false"
+GLOBAL_FILE="global_kernel"
+CONFIG_SELECTION_SUFFIX="_CONFIG"
+
+check_closed() {
+    local str="$1"
+    str="${str//->}"
+    str_start="${str//[^(<\[]}"
+    str_end="${str//[^>)\]]}"
+    if [[ "${#str_start}" -eq "${#str_end}" ]]; then
+        echo "true"
+    else
+        echo "false"
+    fi
+}
+
+convert_syntax() {
+    local syntax_regex="([^<>]*)(<[^<>]*>)?<<<(.*)>>>(.*)"
+    local str="$1"
+    str=$(echo "${str}" | sed -E 's/ +/ /g')
+    # echo "str ${str}"
+    local temp=""
+    local num=0
+    local var=""
+    if [[ "${str}" =~ ${syntax_regex} ]]; then
+        content="${BASH_REMATCH[3]}"
+        # echo "// content ${content}"
+        IFS=',' read -ra par <<< "$content"
+        for variable in "${par[@]}"; do
+            if [ -n "${temp}" ]; then
+                temp="${temp},"
+            fi
+            temp="${temp}${variable}"
+            # echo "// temp ${temp}"
+            is_closed=$(check_closed "$temp")
+            if [[ "${is_closed}" = "true" ]]; then
+                # echo "// temp ${temp}"
+                num=$((num+1))
+                # the last stream is zero, use the queue from exec
+                if [[ "${num}" -eq 4 ]] && [[ "${temp// }" -eq 0 ]]; then
+                    num=$((num-1))
+                else
+                    if [ -n "${var}" ]; then
+                        var="${var},"
+                    fi
+                    var="${var}${temp}"
+                fi
+                temp=""
+            fi
+        done
+        # echo "// var ${var}"
+        # echo "var ${var}"
+        if [[ "${num}" -lt 2 ]]; then
+            var="Error"
+        else
+            if [[ ${num} -eq 2 ]]; then
+                var="${var}, 0"
+                num=$((num+1))
+            fi
+            if [[ ${num} -eq 3 ]]; then
+                # var="${var}, exec->get_queue()"
+                var="${var}, GET_QUEUE"
+                num=$((num+1))
+            fi
+        fi
+        local suffix=""
+        local function_name=$(echo "${BASH_REMATCH[1]}" | sed -E 's/(.*::)| //g')
+        # echo "// functioname ${function_name}"
+        MAP_FILE="map_list"
+        suffix=$(cat "${MAP_FILE}" | sed -nE "s/${function_name} -> ${function_name}(.*)/\1/p")
+        suffix_matches=$(echo "${suffix}" | wc -l)
+        if [[ "${suffix_matches}" -gt 1 ]]; then
+            echo "static_assert(false, \"Has ${suffix_matches} matches for ${function_name}\");"
+            suffix=$(echo "${suffix}" | head -1)
+        fi
+
+        if [[ "${suffix}" = *"${CONFIG_SELECTION_SUFFIX}"* ]]; then
+            var="${var}, exec"
+        fi
+        local result="${BASH_REMATCH[1]}${suffix}${BASH_REMATCH[2]}(${var})${BASH_REMATCH[4]}"
+        result=$(echo "${result}" | sed -E 's/\) *\(+/,/g;s/, *\)/\)/g')
+        if [[ "${suffix}" = "" ]]; then
+            # if the function does not exist in the file, comment it.
+            echo "//remove//${result}"
+        else
+            echo "${result}"
+        fi
+    fi
+}
+
+
+convert_regex_allowed() {
+    local str="$1"
+    str=$(echo "${str}" | sed -E 's/(\/|\(|\)|\.)/\\\1/g')
+    echo "$str"
+}
+
+# Transfer header file to the correct one
+# common -> dpcpp_common
+# cuda -> dpcpp
+# add sycl header
+
+input="$1"
+filename="${ROOT_DIR}/$input"
+if [[ "${VERBOSE}" == 1 ]]; then
+    echo "Porting file ${filename}"
+fi
+# check file exists
+
+if [ ! -f "${filename}" ]; then
+    echo "${filename} does not exist"
+    exit 1
+fi
+
+
+temp=""
+IN_SYNTAX="false"
+DEVICE_FILE=""
+GINKGO_LICENSE_BEACON="******************************<GINKGO LICENSE>******************************"
+UNFORMAT_FILE="unformat.cpp"
+FORMAT_FILE="format.cpp"
+cp "${filename}" "${UNFORMAT_FILE}"
+OUTPUT_FILE="source.cpp"
+EMBED_FILE="embed.cu"
+EMBED_HOST_FILE="embed_host.cu"
+if [[ "${VERBOSE}" == 1 ]]; then
+    echo "the original file ${UNFORMAT_FILE}"
+    echo "the formatted file ${FORMAT_FILE}"
+    echo "collect common/*.inc in file ${EMBED_FILE}"
+    echo "add autohost func in file ${EMBED_HOST_FILE}"
+    echo "convert original CUDA call in file ${OUTPUT_FILE}, which is the file for dpct"
+fi
+rm "${OUTPUT_FILE}"
+# echo "#include \"trick/dim3.hpp\"" >> "${OUTPUT_FILE}"
+echo "#define GET_QUEUE 0" >> "${OUTPUT_FILE}"
+# add empty ginkgo license such that format_header recognize some header before header def macro
+echo "/*${GINKGO_LICENSE_BEACON}" >> "${OUTPUT_FILE}"
+echo "${GINKGO_LICENSE_BEACON}*/" >> "${OUTPUT_FILE}"
+rm "${GLOBAL_FILE}"
+rm "${EMBED_FILE}"
+while IFS='' read -r line; do
+    if [[ "${line}" =~ $DEVICE_CODE_SYNTAX ]]; then
+        # hold the command to easy replace
+        echo "// ${line}" >> ${EMBED_FILE}
+        device_file="${BASH_REMATCH[1]}"
+        [ "${EXTRACT_KERNEL}" == "true" ] && echo "/**** ${device_file} - start ****/" >> "${EMBED_FILE}"
+        cat "${ROOT_DIR}/${device_file}" >> "${EMBED_FILE}"
+        [ "${EXTRACT_KERNEL}" == "true" ] && echo "/**** ${device_file} - end ****/" >> "${EMBED_FILE}"
+        if [ -n "${DEVICE_FILE}" ]; then
+            DEVICE_FILE="${DEVICE_FILE};"
+        fi
+        DEVICE_FILE="${DEVICE_FILE}${device_file}"
+    else
+        echo "${line}" >> "${EMBED_FILE}"
+    fi
+done < "${UNFORMAT_FILE}"
+${CLANG_FORMAT} -style=file "${EMBED_FILE}" > "${FORMAT_FILE}"
+${SCRIPT_DIR}/add_host_function.sh "${FORMAT_FILE}" > "${EMBED_HOST_FILE}"
+
+CONFIG_REGEX="constexpr *Config *([a-zA-Z0-9_]*) *= *([a-zA-Z][a-zA-Z_:,\(\) ]*)"
+while IFS='' read -r line; do
+    if [[ "${IN_SYNTAX}" = "false" ]] && [[ "${line}" =~ ${CONFIG_REGEX} ]]; then
+        config_name="${BASH_REMATCH[1]}"
+        config_content="${BASH_REMATCH[2]}"
+        echo "${line}" >> "${OUTPUT_FILE}"
+        echo "constexpr auto ${config_name}_list = ::gko::syn::value_list<Config, ${config_name}>();" >> "${OUTPUT_FILE}"
+    elif [[ "${line}" =~ ${KERNEL_SYNTAX_START} ]] || [[ "${IN_SYNTAX}" = "true" ]]; then
+        temp="${temp} ${line}"
+        IN_SYNTAX="true"
+        if [[ "${line}" =~ ${FUNCTION_END} ]]; then
+            IN_SYNTAX="false"
+            modified=""
+            if [[ "${temp}" = *"/*KEEP*/"* ]]; then
+                modified="${temp/\/\*KEEP\*\//}"
+            else
+                # change <<<>>> to (grid, block, dynamic, queue)
+                modified=$(convert_syntax "$temp")
+            fi
+            # echo "temp -> convert"
+            # echo "$temp"
+            # echo "$modified"
+            echo "${modified}" >> "${OUTPUT_FILE}"
+            temp=""
+        fi
+    else
+        echo "${line}" >> "${OUTPUT_FILE}"
+    fi
+done < "${EMBED_HOST_FILE}"
+
+# Other fix on OUTPUT_FILE
+# dim3 -> dim3_t (for easy replace)
+# this_thread_block -> this_thread_block_t
+# tiled_partition -> tiled_partition_t
+# thread_id.cuh -> use local
+# cooperative_group.cuh -> use local
+replace_regex="s/dim3/dim3_t/g"
+replace_regex="${replace_regex};s/this_thread_block/this_thread_block_t/g"
+replace_regex="${replace_regex};s/this_grid/this_grid_t/g"
+replace_regex="${replace_regex};s/tiled_partition/tiled_partition_t/g"
+replace_regex="${replace_regex};s/thread::/thread_t::/g"
+replace_regex="${replace_regex};s/bitonic_sort/bitonic_sort_t/g"
+replace_regex="${replace_regex};s/reduction_array/reduction_array_t/g"
+replace_regex="${replace_regex};s/cuda\/components\/thread_ids.cuh/trick\/thread_ids.hpp/g"
+replace_regex="${replace_regex};s/cuda\/components\/cooperative_groups.cuh/trick\/cooperative_groups.hpp/g"
+replace_regex="${replace_regex};s/cuda\/components\/sorting.cuh/trick\/sorting.hpp/g"
+replace_regex="${replace_regex};s/cuda\/components\/reduction.cuh/trick\/reduction.hpp/g"
+replace_regex="${replace_regex};s/CUH_/DP_HPP_/g"
+replace_regex="${replace_regex};s/(.*template.*)(Config * [a-zA-Z0-9_]*) * = *[a-zA-Z0-9_:]*(\(+[^\)]*\))?/\1\2/g"
+# keep using original xxx.sync(); ->xxx;//.sync();
+replace_regex="${replace_regex};s/(\.sync\(\);)/;\/\/.sync()/g"
+# template macro(); lead std::length_error
+replace_regex="${replace_regex};s/(template GKO.*;)/\/\/ \1/g"
+sed -i -E "${replace_regex}" "${OUTPUT_FILE}"
+if grep -Eq "dim3" ${OUTPUT_FILE}; then
+    # Found
+    sed -i '1 i#include "trick/dim3_t.hpp"' ${OUTPUT_FILE}
+fi
+# add the cooperative group header according to group:: because some sources forget to add it
+if grep -Eq "group::" ${OUTPUT_FILE}; then
+    # Found
+    sed -i '1 i#include "trick/cooperative_groups.hpp"' ${OUTPUT_FILE}
+fi
+# exit 0
+
+OUTPUT_FOLDER="output"
+if [[ "${VERBOSE}" == 1]]; then
+    echo "The dpct calling:"
+    echo "dpct --extra-arg=\"-std=c++14\" --extra-arg=\"-I ${ROOT_DIR}\" --extra-arg=\"-I ${ROOT_DIR}/include\" --extra-arg=\"-I ${ROOT_DIR}/dev_tools/oneapi\" --extra-arg=\"-I ${GTEST_HEADER_DIR}\" --cuda-include-path=\"${CUDA_HEADER_DIR}\" --format-range=none ${OUTPUT_FILE} --out-root=${OUTPUT_FOLDER}"
+fi
+
+# Delete output/source.cpp
+rm "${OUTPUT_FOLDER}/${OUTPUT_FILE}"
+rm "${OUTPUT_FOLDER}/${OUTPUT_FILE}.dp.cpp"
+
+dpct --extra-arg="-std=c++14" --extra-arg="-I ${ROOT_DIR}" --extra-arg="-I ${ROOT_DIR}/include" --extra-arg="-I ${ROOT_DIR}/dev_tools/oneapi" --extra-arg="-I ${GTEST_HEADER_DIR}" --cuda-include-path="${CUDA_HEADER_DIR}" --format-range=none ${OUTPUT_FILE} --out-root=${OUTPUT_FOLDER}
+
+dpct_file=""
+if [ -f "${OUTPUT_FOLDER}/${OUTPUT_FILE}.dp.cpp" ]; then
+    dpct_file="${OUTPUT_FOLDER}/${OUTPUT_FILE}.dp.cpp"
+elif [ -f "${OUTPUT_FOLDER}/${OUTPUT_FILE}" ]; then
+    dpct_file="${OUTPUT_FOLDER}/${OUTPUT_FILE}"
+else
+    echo "No file"
+    exit 1
+fi
+
+cp "${dpct_file}" "${dpct_file}_bkp"
+if [[ "${VERBOSE}" == 1 ]]; then
+    echo "the dpct result ${dpct_file}_bkp"
+    echo "recover the temporary change in the file ${dpct_file}"
+fi
+# global reverse fix
+# dim3_t -> dim3
+# this_thread_block_t -> this_thread_block
+# tiled_partition_t -> tiled_partition
+# thread_id
+# cooperative_group
+# replace_regex="1 i\#include <CL/sycl.hpp>"
+# replace_regex="s/cuda/dpcpp/g"
+# replace_regex="${replace_regex};s/#include \\\"common/#include \\\"dpcpp_code/g"
+replace_regex="s/dim3_t/dim3/g"
+replace_regex="${replace_regex};s/trick\/dim3\.hpp/dpcpp\/base\/dim3\.dp\.hpp/g"
+replace_regex="${replace_regex};s/this_thread_block_t/this_thread_block/g"
+replace_regex="${replace_regex};s/thread_t::/thread::/g"
+replace_regex="${replace_regex};s/this_grid_t/this_grid/g"
+replace_regex="${replace_regex};s/bitonic_sort_t/bitonic_sort/g"
+replace_regex="${replace_regex};s/reduce_array_t/reduce_array/g"
+# beta 09 ()
+# replace_regex="${replace_regex};s/auto dpct_global_range = grid \* block;/auto local_range = block.reverse();/g"
+replace_regex="${replace_regex};s/auto dpct_local_range = block;//g"
+replace_regex="${replace_regex};s/sycl::nd_range<3>.*, *$/sycl_nd_range(grid, block), /g"
+# global_range="sycl::range<3>(dpct_global_range.get(2), dpct_global_range.get(1), dpct_global_range.get(0))"
+# global_range=$(convert_regex_allowed "${global_range}")
+# echo "global_range ${global_range}"
+# local_range="sycl::range<3>(dpct_local_range.get(2), dpct_local_range.get(1), dpct_local_range.get(0))"
+# local_range=$(convert_regex_allowed "${local_range}")
+# echo "local_range ${local_range}"
+# replace_regex="${replace_regex};s/${global_range}/global_range/g"
+# replace_regex="${replace_regex};s/${local_range}/local_range/g"
+replace_regex="${replace_regex};s/tiled_partition_t/tiled_partition/g"
+replace_regex="${replace_regex};s/trick\/thread_ids.hpp/dpcpp\/components\/thread_ids.dp.hpp/g"
+replace_regex="${replace_regex};s/trick\/cooperative_groups.hpp/dpcpp\/components\/cooperative_groups.dp.hpp/g"
+replace_regex="${replace_regex};s/trick\/sorting.hpp/dpcpp\/components\/sorting.dp.hpp/g"
+replace_regex="${replace_regex};s/trick\/reduction.hpp/dpcpp\/components\/reduction.dp.hpp/g"
+replace_regex="${replace_regex};s/#define GET_QUEUE 0//g"
+replace_regex="${replace_regex};s/GET_QUEUE/exec->get_queue()/g"
+replace_regex="${replace_regex};s/cuda/dpcpp/g"
+replace_regex="${replace_regex};s/Cuda/Dpcpp/g"
+replace_regex="${replace_regex};s/CUDA/DPCPP/g"
+replace_regex="${replace_regex};s/\.cuh/\.dp\.hpp/g"
+replace_regex="${replace_regex};s/\.cu/\.dp\.cpp/g"
+replace_regex="${replace_regex};s/${SPECIAL_SUFFIX}//g"
+replace_regex="${replace_regex};s/#include <dpct\/dpct\.hpp>//g"
+replace_regex="${replace_regex};s/#include \"dpcpp\/base\/types\.hpp\"//g"
+replace_regex="${replace_regex};s/#include \"dpcpp\/test\/utils\.hpp\"/#include \"core\/test\/utils\.hpp\"/g"
+# remove as_dpcpp_type\(content\) -> content
+# If the content use some brackets, only allowed one nested bracket now.
+replace_regex="${replace_regex};s/as_dpcpp_type\((([^\(\)]*(\([^\(\)]*\))[^\(\)]*)*)\)/\1/g"
+replace_regex="${replace_regex};s/as_dpcpp_type\(([^\(\)]*)\)/\1/g"
+# dcpt can not convert idx in static_cast
+replace_regex="${replace_regex};s/threadIdx\.x/item_ct1.get_local_id(2)/g"
+replace_regex="${replace_regex};s/threadIdx\.y/item_ct1.get_local_id(1)/g"
+replace_regex="${replace_regex};s/threadIdx\.z/item_ct1.get_local_id(0)/g"
+
+replace_regex="${replace_regex};s/blockIdx\.x/item_ct1.get_group(2)/g"
+replace_regex="${replace_regex};s/blockIdx\.y/item_ct1.get_group(1)/g"
+replace_regex="${replace_regex};s/blockIdx\.z/item_ct1.get_group(0)/g"
+
+replace_regex="${replace_regex};s/blockDim\.x/item_ct1.get_local_range().get(2)/g"
+replace_regex="${replace_regex};s/blockDim\.y/item_ct1.get_local_range().get(1)/g"
+replace_regex="${replace_regex};s/blockDim\.z/item_ct1.get_local_range().get(0)/g"
+
+replace_regex="${replace_regex};s/gridDim\.x/item_ct1.get_group_range(2)/g"
+replace_regex="${replace_regex};s/gridDim\.y/item_ct1.get_group_range(1)/g"
+replace_regex="${replace_regex};s/gridDim\.z/item_ct1.get_group_range(0)/g"
+# Workaround for abs
+replace_regex="${replace_regex};s/sycl::fabs/std::abs/g"
+# Remove unneed warning DPCT1049 - check the block size
+# replace_regex="${replace_regex};/\/\*$/,/DPCT1049/s/^(.*)$/\1-remove/g;/\/\*-remove/,/\*\/$/d"
+replace_regex="${replace_regex};/\/\*$/{N;N;/ *\/\*\n *DPCT1049.*\n *\*\//d}"
+# Recover // template GKO_...
+replace_regex="${replace_regex};s/\/\/ (template GKO.*;)/\1/g"
+# Recover // xxx;//.sync();->xxx.sync();
+replace_regex="${replace_regex};s/;\/\/\.sync\(\)/\.sync\(\);/g"
+replace_regex="${replace_regex};s/\/\/remove\/\///g"
+
+quote="'"
+# echo "replace_regex ${quote}${replace_regex}${quote}"
+sed -i -E "${replace_regex}" "${dpct_file}"
+
+# auto dpct_global_range = grid * block;
+# auto dpct_local_range = block;
+# sycl::nd_range<3>(sycl::range<3>(dpct_global_range.get(2), dpct_global_range.get(1), dpct_global_range.get(0)), sycl::range<3>(dpct_local_range.get(2), dpct_local_range.get(1), dpct_local_range.get(0))),
+
+# check whether containing __dpct_inline__ or __dpct_align__
+need_dpct=$(grep -Eq "__(dpct_align|dpct_inline)__" ${dpct_file})
+if grep -Eq "__(dpct_align|dpct_inline)__" ${dpct_file}; then
+    # Found
+    sed -i '1 i#include "dpcpp/base/dpct.hpp"' ${dpct_file}
+fi
+
+# extract device_code
+if [ "${EXTRACT_KERNEL}" = "true" ]; then
+    IFS=';' read -ra individual_deivce <<< "${DEVICE_FILE}"
+    for variable in "${individual_deivce[@]}"; do
+        device_regex=$(convert_regex_allowed "${variable}")
+        # echo "device_regex ${device_regex}"
+        dpct_device_path=$(echo "${variable}" | sed 's/common/dpcpp_code/g')
+        dpct_device_file=$(echo "${dpct_device_path}" | sed 's/\//@/g')
+        dpct_device_file="output/${dpct_device_file}"
+        # echo "dpct_device_path ${dpct_device_path}"
+        # echo "dpct_device_file ${dpct_device_file}"
+        cat ${dpct_file} | sed -n "/${device_regex} - start/,/${device_regex} - end/p" | sed "1d;\$d" > ${dpct_device_file}
+        sed -i "/${device_regex} - start/,/${device_regex} - end/d;s~// *#include \"${device_regex}\"~#include \"${dpct_device_path}\"~g" ${dpct_file}
+        dpct_dir=$(dirname "${dpct_device_path}")
+        mkdir -p "${ROOT_DIR}/${dpct_dir}"
+        cp "${dpct_device_file}" "${ROOT_DIR}/${dpct_device_path}"
+    done
+fi
+
+target_file=$(echo "${input}" | sed 's/cuda\//dpcpp\//g;s/\.cuh/\.dp\.hpp/g;s/\.cu/\.dp\.cpp/g')
+target_dir=$(dirname "${target_file}")
+
+mkdir -p "${ROOT_DIR}/${target_dir}"
+echo "cp ${dpct_file} ${ROOT_DIR}/${target_file}"
+cp "${dpct_file}" "${ROOT_DIR}/${target_file}"
+# sepecial reverse-fix (maybe not need)
+# for dpct_device_file
+# for variable in "${individual_deivce[@]}"; do
+#     # device_regex=$(convert_regex_allowed "${variable}")
+#     dpct_device_file=$(echo "${variable}" | sed 's/common/dpcpp_code/g;s/\//@/g')
+#     # some syntex reverse or some better cuda format
+#     #
+# done
+# for dpct_file
+# sed -i '1 i\#include <CL/sycl.hpp>;s/cuda/dpcpp/g;s/#include \"common/#include \"dpcpp_code/g' "${dpct_file}"
+# cublas/cusparse->onemkl?
+# reverse some header and change to DPCPP-related files
+
+# Move the file to correct place
+
+# apply format header()?
diff --git a/dev_tools/oneapi/fake_interface/cooperative_groups.cuh b/dev_tools/oneapi/fake_interface/cooperative_groups.cuh
new file mode 100644
index 00000000000..d242ba78461
--- /dev/null
+++ b/dev_tools/oneapi/fake_interface/cooperative_groups.cuh
@@ -0,0 +1,57 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_FAKE_INTERFACE_COOPERATIVE_GROUPS_CUH_
+#define GKO_FAKE_INTERFACE_COOPERATIVE_GROUPS_CUH_
+
+#include <cuda/components/cooperative_groups.cuh>
+
+namespace gko {
+    namespace kernels{
+        namespace cuda{
+            namespace group{
+__device__ __forceinline__ grid_group this_grid_i() { return this_grid(); }
+
+__device__ auto this_thread_block_i() { return this_thread_block(); }
+
+template <unsigned Size, typename Group>
+__device__ __forceinline__ auto tiled_partition_i(const Group &g)
+{
+    return ::gko::kernels::cuda::group::tiled_partition<Size>(g);
+}
+
+            }
+        }
+    }
+}
+
+#endif // GKO_FAKE_INTERFACE_COOPERATIVE_GROUPS_CUH_
diff --git a/dev_tools/oneapi/working_directory/.clang-format b/dev_tools/oneapi/working_directory/.clang-format
new file mode 100644
index 00000000000..ffa29df7461
--- /dev/null
+++ b/dev_tools/oneapi/working_directory/.clang-format
@@ -0,0 +1,108 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Google
+AccessModifierOffset: -4
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Left
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: true
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   true
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+  SplitEmptyFunction: false
+  SplitEmptyRecord: false
+  SplitEmptyNamespace: false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Custom
+BreakBeforeInheritanceComma: false
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     1000
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeCategories:
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: false
+IndentWidth:     4
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 2
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Right
+ReflowComments:  true
+SortIncludes:    true
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Never
+...
+
diff --git a/dev_tools/oneapi/working_directory/trick/cooperative_groups.hpp b/dev_tools/oneapi/working_directory/trick/cooperative_groups.hpp
new file mode 100644
index 00000000000..fd9d83ca326
--- /dev/null
+++ b/dev_tools/oneapi/working_directory/trick/cooperative_groups.hpp
@@ -0,0 +1,72 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef TRICK_COOPERATIVE_GROUPS_HPP_
+#define TRICK_COOPERATIVE_GROUPS_HPP_
+
+
+#include <type_traits>
+
+#include <fake_interface/cooperative_groups.cuh>
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+namespace group {
+
+template <unsigned Size, typename Group>
+__device__ __forceinline__ auto tiled_partition_t(const Group &g)
+{
+    return tiled_partition_i<Size>(g);
+}
+
+__device__ inline grid_group this_grid_t()
+{
+    auto tidx = threadIdx.x;
+    return this_grid_i();
+}
+
+__device__ auto this_thread_block_t()
+{
+    auto tidx = threadIdx.x;
+    return this_thread_block_i();
+}
+
+
+}  // namespace group
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // TRICK_COOPERATIVE_GROUPS_HPP_
diff --git a/dev_tools/oneapi/working_directory/trick/dim3_t.hpp b/dev_tools/oneapi/working_directory/trick/dim3_t.hpp
new file mode 100644
index 00000000000..57e47b519c7
--- /dev/null
+++ b/dev_tools/oneapi/working_directory/trick/dim3_t.hpp
@@ -0,0 +1,48 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef TRICK_DIM3_T_HPP_
+#define TRICK_DIM3_T_HPP_
+
+struct dim3_t {
+    unsigned int x;
+    unsigned int y;
+    unsigned int z;
+    dim3_t(unsigned int xval, unsigned int yval = 1, unsigned int zval = 1)
+        : x(xval), y(yval), z(zval)
+    {}
+    operator dim3() {
+        return dim3{x, y, z};
+    }
+};
+
+#endif // TRICK_DIM3_T_HPP_
diff --git a/dev_tools/oneapi/working_directory/trick/reduction.hpp b/dev_tools/oneapi/working_directory/trick/reduction.hpp
new file mode 100644
index 00000000000..d732f6bce91
--- /dev/null
+++ b/dev_tools/oneapi/working_directory/trick/reduction.hpp
@@ -0,0 +1,57 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef TRICK_REDUCTION_HPP_
+#define TRICK_REDUCTION_HPP_
+
+
+#include <cuda/components/reduction.cuh>
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+
+
+template <typename Operator, typename ValueType>
+void __device__ reduce_array_t(size_type size, const ValueType *__restrict__ source, ValueType *__restrict__ result, Operator reduce_op = Operator{})
+{
+    auto tid = threadIdx.x;
+    reduce_array(size, source, result, reduce_op);
+}
+
+
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
+
+#endif  // TRICK_REDUCTION_HPP_
diff --git a/dev_tools/oneapi/working_directory/trick/sorting.hpp b/dev_tools/oneapi/working_directory/trick/sorting.hpp
new file mode 100644
index 00000000000..3c82b11f315
--- /dev/null
+++ b/dev_tools/oneapi/working_directory/trick/sorting.hpp
@@ -0,0 +1,62 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef TRICK_SORTING_HPP_
+#define TRICK_SORTING_HPP_
+
+
+#include <type_traits>
+
+
+#include <cuda/components/sorting.cuh>
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+
+
+template <int num_elements, int num_local, typename ValueType>
+__forceinline__ __device__ void bitonic_sort_t(ValueType *local_elements,
+                                             ValueType *shared_elements)
+{
+    auto tidx = threadIdx.x;
+    bitonic_sort<num_elements, num_local>(local_elements, shared_elements);
+}
+
+
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // TRICK_SORTING_HPP_
diff --git a/dev_tools/oneapi/working_directory/trick/thread_ids.hpp b/dev_tools/oneapi/working_directory/trick/thread_ids.hpp
new file mode 100644
index 00000000000..3c003dd1b95
--- /dev/null
+++ b/dev_tools/oneapi/working_directory/trick/thread_ids.hpp
@@ -0,0 +1,299 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef TRICK_THREAD_IDS_HPP_
+#define TRICK_THREAD_IDS_HPP_
+
+
+#include "cuda/base/config.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace cuda {
+/**
+ * @brief The CUDA thread namespace.
+ *
+ * @ingroup cuda_thread
+ */
+namespace thread_t {
+
+
+/**
+ * @internal
+ *
+ * Returns the ID of the block group this thread belongs to.
+ *
+ * @return the ID of the block group this thread belongs to
+ *
+ * @note Assumes that grid dimensions are in standard format:
+ *       `(block_group_size, first_grid_dimension, second grid_dimension)`
+ */
+__device__ __forceinline__ size_type get_block_group_id()
+{
+    return static_cast<size_type>(blockIdx.z) * gridDim.y + blockIdx.y;
+}
+
+/**
+ * @internal
+ *
+ * Returns the ID of the block this thread belongs to.
+ *
+ * @return the ID of the block this thread belongs to
+ *
+ * @note Assumes that grid dimensions are in standard format:
+ *       `(block_group_size, first_grid_dimension, second grid_dimension)`
+ */
+__device__ __forceinline__ size_type get_block_id()
+{
+    return get_block_group_id() * gridDim.x + blockIdx.x;
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the local ID of the warp (relative to the block) this thread belongs
+ * to.
+ *
+ * @return the local ID of the warp (relative to the block) this thread belongs
+ *         to
+ *
+ * @note Assumes that block dimensions are in standard format:
+ *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
+ *         config::warp_size)`
+ */
+__device__ __forceinline__ size_type get_local_warp_id()
+{
+    return static_cast<size_type>(threadIdx.z);
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the local ID of the sub-warp (relative to the block) this thread
+ * belongs to.
+ *
+ * @tparam subwarp_size  size of the subwarp
+ *
+ * @return the local ID of the sub-warp (relative to the block) this thread
+ *         belongs to
+ *
+ * @note Assumes that block dimensions are in standard format:
+ *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
+ *         config::warp_size)`
+ */
+template <int subwarp_size>
+__device__ __forceinline__ size_type get_local_subwarp_id()
+{
+    constexpr auto subwarps_per_warp = config::warp_size / subwarp_size;
+    return get_local_warp_id() * subwarps_per_warp + threadIdx.y;
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the local ID of the thread (relative to the block).
+ * to.
+ *
+ * @tparam subwarp_size  size of the subwarp
+ *
+ * @return the local ID of the thread (relative to the block)
+ *
+ * @note Assumes that block dimensions are in standard format:
+ *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
+ *         config::warp_size)`
+ */
+template <int subwarp_size>
+__device__ __forceinline__ size_type get_local_thread_id()
+{
+    return get_local_subwarp_id<subwarp_size>() * subwarp_size + threadIdx.x;
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the global ID of the warp this thread belongs to.
+ *
+ * @tparam warps_per_block  number of warps within each block
+ *
+ * @return the global ID of the warp this thread belongs to.
+ *
+ * @note Assumes that block dimensions and grid dimensions are in standard
+ *       format:
+ *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
+ *         config::warp_size)` and
+ *       `(block_group_size, first_grid_dimension, second grid_dimension)`,
+ *       respectively.
+ */
+template <int warps_per_block>
+__device__ __forceinline__ size_type get_warp_id()
+{
+    return get_block_id() * warps_per_block + get_local_warp_id();
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the global ID of the sub-warp this thread belongs to.
+ *
+ * @tparam subwarp_size  size of the subwarp
+ *
+ * @return the global ID of the sub-warp this thread belongs to.
+ *
+ * @note Assumes that block dimensions and grid dimensions are in standard
+ *       format:
+ *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
+ *         config::warp_size)` and
+ *       `(block_group_size, first_grid_dimension, second grid_dimension)`,
+ *       respectively.
+ */
+template <int subwarp_size, int warps_per_block>
+__device__ __forceinline__ size_type get_subwarp_id()
+{
+    constexpr auto subwarps_per_warp = config::warp_size / subwarp_size;
+    return get_warp_id<warps_per_block>() * subwarps_per_warp + threadIdx.y;
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the global ID of the thread.
+ *
+ * @return the global ID of the thread.
+ *
+ * @tparam subwarp_size  size of the subwarp
+ *
+ * @note Assumes that block dimensions and grid dimensions are in standard
+ *       format:
+ *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
+ *         config::warp_size)` and
+ *       `(block_group_size, first_grid_dimension, second grid_dimension)`,
+ *       respectively.
+ */
+template <int subwarp_size, int warps_per_block>
+__device__ __forceinline__ size_type get_thread_id()
+{
+    return get_subwarp_id<subwarp_size, warps_per_block>() * subwarp_size +
+           threadIdx.x;
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the global ID of the thread in the given index type.
+ * This function assumes one-dimensional thread and block indexing.
+ *
+ * @return the global ID of the thread in the given index type.
+ *
+ * @tparam IndexType  the index type
+ */
+template <typename IndexType = size_type>
+__device__ __forceinline__ IndexType get_thread_id_flat()
+{
+    return threadIdx.x + static_cast<IndexType>(blockDim.x) * blockIdx.x;
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the total number of threads in the given index type.
+ * This function assumes one-dimensional thread and block indexing.
+ *
+ * @return the total number of threads in the given index type.
+ *
+ * @tparam IndexType  the index type
+ */
+template <typename IndexType = size_type>
+__device__ __forceinline__ IndexType get_thread_num_flat()
+{
+    return blockDim.x * static_cast<IndexType>(gridDim.x);
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the global ID of the subwarp in the given index type.
+ * This function assumes one-dimensional thread and block indexing
+ * with a power of two block size of at least subwarp_size.
+ *
+ * @return the global ID of the subwarp in the given index type.
+ *
+ * @tparam subwarp_size  the size of the subwarp. Must be a power of two!
+ * @tparam IndexType  the index type
+ */
+template <int subwarp_size, typename IndexType = size_type>
+__device__ __forceinline__ IndexType get_subwarp_id_flat()
+{
+    static_assert(!(subwarp_size & (subwarp_size - 1)),
+                  "subwarp_size must be a power of two");
+    return threadIdx.x / subwarp_size +
+           static_cast<IndexType>(blockDim.x / subwarp_size) * blockIdx.x;
+}
+
+
+/**
+ * @internal
+ *
+ * Returns the total number of subwarps in the given index type.
+ * This function assumes one-dimensional thread and block indexing
+ * with a power of two block size of at least subwarp_size.
+ *
+ * @return the total number of subwarps in the given index type.
+ *
+ * @tparam subwarp_size  the size of the subwarp. Must be a power of two!
+ * @tparam IndexType  the index type
+ */
+template <int subwarp_size, typename IndexType = size_type>
+__device__ __forceinline__ IndexType get_subwarp_num_flat()
+{
+    static_assert(!(subwarp_size & (subwarp_size - 1)),
+                  "subwarp_size must be a power of two");
+    return blockDim.x / subwarp_size * static_cast<IndexType>(gridDim.x);
+}
+
+
+}  // namespace thread
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // TRICK_THREAD_IDS_HPP_

From 777bf574507688eafc7c6192d784d55ae7ea734f Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 27 May 2021 22:14:37 +0200
Subject: [PATCH 0911/1008] add range nonconst copy constructor

---
 accessor/range.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/accessor/range.hpp b/accessor/range.hpp
index be73d8dbf87..3dc8914ebfa 100644
--- a/accessor/range.hpp
+++ b/accessor/range.hpp
@@ -102,6 +102,8 @@ class range {
 
     range(const range &other) = default;
 
+    range(range &other) = default;
+
     /**
      * Returns the length of the specified dimension of the range.
      *

From d7797292602d009bb85f5e5225b64508518c4b8f Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 27 May 2021 22:22:43 +0200
Subject: [PATCH 0912/1008] porting ell with some fix

---
 .../fake_interface/cooperative_groups.cuh     |   2 +-
 .../trick/cooperative_groups.hpp              |   2 +-
 .../oneapi/working_directory/trick/dim3_t.hpp |   2 +-
 .../working_directory/trick/reduction.hpp     |   2 +-
 .../working_directory/trick/sorting.hpp       |   2 +-
 .../working_directory/trick/thread_ids.hpp    |   2 +-
 dpcpp/base/config.hpp                         |   5 +
 dpcpp/components/format_conversion.dp.hpp     |  76 +-
 dpcpp/matrix/ell_kernels.dp.cpp               | 703 +++++++++++++++++-
 dpcpp/test/matrix/CMakeLists.txt              |   1 +
 dpcpp/test/matrix/ell_kernels.cpp             | 682 +++++++++++++++++
 11 files changed, 1445 insertions(+), 34 deletions(-)
 create mode 100644 dpcpp/test/matrix/ell_kernels.cpp

diff --git a/dev_tools/oneapi/fake_interface/cooperative_groups.cuh b/dev_tools/oneapi/fake_interface/cooperative_groups.cuh
index d242ba78461..05d9b936fbe 100644
--- a/dev_tools/oneapi/fake_interface/cooperative_groups.cuh
+++ b/dev_tools/oneapi/fake_interface/cooperative_groups.cuh
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dev_tools/oneapi/working_directory/trick/cooperative_groups.hpp b/dev_tools/oneapi/working_directory/trick/cooperative_groups.hpp
index fd9d83ca326..d7a8760f2ea 100644
--- a/dev_tools/oneapi/working_directory/trick/cooperative_groups.hpp
+++ b/dev_tools/oneapi/working_directory/trick/cooperative_groups.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dev_tools/oneapi/working_directory/trick/dim3_t.hpp b/dev_tools/oneapi/working_directory/trick/dim3_t.hpp
index 57e47b519c7..78c04f8e356 100644
--- a/dev_tools/oneapi/working_directory/trick/dim3_t.hpp
+++ b/dev_tools/oneapi/working_directory/trick/dim3_t.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dev_tools/oneapi/working_directory/trick/reduction.hpp b/dev_tools/oneapi/working_directory/trick/reduction.hpp
index d732f6bce91..deaf0bb73b9 100644
--- a/dev_tools/oneapi/working_directory/trick/reduction.hpp
+++ b/dev_tools/oneapi/working_directory/trick/reduction.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dev_tools/oneapi/working_directory/trick/sorting.hpp b/dev_tools/oneapi/working_directory/trick/sorting.hpp
index 3c82b11f315..ceccbac71e6 100644
--- a/dev_tools/oneapi/working_directory/trick/sorting.hpp
+++ b/dev_tools/oneapi/working_directory/trick/sorting.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dev_tools/oneapi/working_directory/trick/thread_ids.hpp b/dev_tools/oneapi/working_directory/trick/thread_ids.hpp
index 3c003dd1b95..791d93105a4 100644
--- a/dev_tools/oneapi/working_directory/trick/thread_ids.hpp
+++ b/dev_tools/oneapi/working_directory/trick/thread_ids.hpp
@@ -1,5 +1,5 @@
 /*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2020, the Ginkgo authors
+Copyright (c) 2017-2021, the Ginkgo authors
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
diff --git a/dpcpp/base/config.hpp b/dpcpp/base/config.hpp
index 78fe25978a7..c5bb77e48b3 100644
--- a/dpcpp/base/config.hpp
+++ b/dpcpp/base/config.hpp
@@ -59,6 +59,11 @@ struct config {
      * to maximize GPU occupancy.
      */
     static constexpr uint32 min_warps_per_block = 4;
+
+    /**
+     * The maximal number of threads allowed in a CUDA warp.
+     */
+    static constexpr uint32 max_block_size = 256;
 };
 
 
diff --git a/dpcpp/components/format_conversion.dp.hpp b/dpcpp/components/format_conversion.dp.hpp
index c40f765448f..d92661bce83 100644
--- a/dpcpp/components/format_conversion.dp.hpp
+++ b/dpcpp/components/format_conversion.dp.hpp
@@ -30,55 +30,89 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <algorithm>
-#include <numeric>
+#ifndef GKO_DPCPP_COMPONENTS_FORMAT_CONVERSION_DP_HPP_
+#define GKO_DPCPP_COMPONENTS_FORMAT_CONVERSION_DP_HPP_
 
 
 #include <CL/sycl.hpp>
 
 
-#include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
 
 
 namespace gko {
 namespace kernels {
 namespace dpcpp {
+namespace ell {
+namespace kernel {
 
 
 /**
  * @internal
  *
- * Converts an array of indexes `idxs` in any order to an array of pointers
- * `ptrs`. This is used for transposing a csr matrix when calculating the row
- * pointers of the transposed matrix out of the column indices of the original
- * matrix.
+ * It counts the number of explicit nonzeros per row of Ell.
  */
-template <typename IndexType>
-inline void convert_unsorted_idxs_to_ptrs(const IndexType *idxs,
-                                          size_type num_nonzeros,
-                                          IndexType *ptrs,
-                                          size_type length) GKO_NOT_IMPLEMENTED;
+template <typename ValueType, typename IndexType>
+void count_nnz_per_row(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                       sycl::queue *stream, size_type num_rows,
+                       size_type max_nnz_per_row, size_type stride,
+                       const ValueType *values, IndexType *result);
+
+
+}  // namespace kernel
+}  // namespace ell
+
+
+namespace coo {
+namespace kernel {
 
 
 /**
  * @internal
  *
- * Converts an array of indexes `idxs` which are already stored in an increasing
- * order to an array of pointers `ptrs`. This is used to calculate the row
- * pointers when converting a coo matrix to a csr matrix.
+ * It converts the row index of Coo to the row pointer of Csr.
  */
 template <typename IndexType>
-inline void convert_sorted_idxs_to_ptrs(const IndexType *idxs,
-                                        size_type num_nonzeros, IndexType *ptrs,
-                                        size_type length) GKO_NOT_IMPLEMENTED;
+void convert_row_idxs_to_ptrs(dim3 grid, dim3 block,
+                              size_t dynamic_shared_memory, sycl::queue *stream,
+                              const IndexType *idxs, size_type num_nonzeros,
+                              IndexType *ptrs, size_type length);
 
 
-template <typename IndexType>
-inline void convert_ptrs_to_idxs(const IndexType *ptrs, size_type num_rows,
-                                 IndexType *idxs) GKO_NOT_IMPLEMENTED;
+}  // namespace kernel
 
 
+namespace host_kernel {
+
+
+/**
+ * @internal
+ *
+ * It calculates the number of warps used in Coo Spmv depending on the GPU
+ * architecture and the number of stored elements.
+ */
+template <size_type subwarp_size = config::warp_size>
+size_type calculate_nwarps(std::shared_ptr<const DpcppExecutor> exec,
+                           const size_type nnz)
+{
+    size_type nwarps_in_dpcpp = exec->get_num_computing_units() * 7;
+    size_type multiple = 8;
+    return std::min(multiple * nwarps_in_dpcpp,
+                    size_type(ceildiv(nnz, config::warp_size)));
+}
+
+
+}  // namespace host_kernel
+}  // namespace coo
 }  // namespace dpcpp
 }  // namespace kernels
 }  // namespace gko
+
+
+#endif  // GKO_DPCPP_COMPONENTS_FORMAT_CONVERSION_DP_HPP_
diff --git a/dpcpp/matrix/ell_kernels.dp.cpp b/dpcpp/matrix/ell_kernels.dp.cpp
index 3c1d93eb323..2f27d543e19 100644
--- a/dpcpp/matrix/ell_kernels.dp.cpp
+++ b/dpcpp/matrix/ell_kernels.dp.cpp
@@ -33,17 +33,32 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/ell_kernels.hpp"
 
 
+#include <array>
+
+
 #include <CL/sycl.hpp>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "accessor/reduced_row_major.hpp"
 #include "core/base/mixed_precision_types.hpp"
+#include "core/components/fill_array.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/dense_kernels.hpp"
+#include "core/synthesizer/implementation_selection.hpp"
+#include "dpcpp/base/config.hpp"
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/components/atomic.dp.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/format_conversion.dp.hpp"
+#include "dpcpp/components/reduction.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
 
 
 namespace gko {
@@ -57,12 +72,580 @@ namespace dpcpp {
 namespace ell {
 
 
+constexpr int default_block_size = 256;
+
+
+// TODO: num_threads_per_core and ratio are parameters should be tuned
+/**
+ * num_threads_per_core is the oversubscribing parameter. There are
+ * `num_threads_per_core` threads assigned to each physical core.
+ */
+constexpr int num_threads_per_core = 4;
+
+
+/**
+ * ratio is the parameter to decide when to use threads to do reduction on each
+ * row. (#cols/#rows > ratio)
+ */
+constexpr double ratio = 1e-2;
+
+
+/**
+ * max_thread_per_worker is the max number of thread per worker. The
+ * `compiled_kernels` must be a list <0, 1, 2, ..., max_thread_per_worker>
+ */
+constexpr int max_thread_per_worker = 32;
+
+
+/**
+ * A compile-time list of sub-warp sizes for which the spmv kernels should be
+ * compiled.
+ * 0 is a special case where it uses a sub-warp size of warp_size in
+ * combination with atomic_adds.
+ */
+using compiled_kernels = syn::value_list<int, 0, 8, 16, 32>;
+
+
+// #include "common/matrix/ell_kernels.hpp.inc"
+namespace kernel {
+namespace {
+
+
+template <int num_thread_per_worker, bool atomic, typename b_accessor,
+          typename a_accessor, typename OutputValueType, typename IndexType,
+          typename Closure>
+void spmv_kernel(
+    const size_type num_rows, const int num_worker_per_row,
+    acc::range<a_accessor> val, const IndexType *__restrict__ col,
+    const size_type stride, const size_type num_stored_elements_per_row,
+    acc::range<b_accessor> b, OutputValueType *__restrict__ c,
+    const size_type c_stride, Closure op, sycl::nd_item<3> item_ct1,
+    UninitializedArray<OutputValueType,
+                       default_block_size / num_thread_per_worker> *storage)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    const decltype(tidx) column_id = item_ct1.get_group(1);
+    if (num_thread_per_worker == 1) {
+        // Specialize the num_thread_per_worker = 1. It doesn't need the shared
+        // memory, __syncthreads, and atomic_add
+        if (tidx < num_rows) {
+            auto temp = zero<OutputValueType>();
+            for (size_type idx = 0; idx < num_stored_elements_per_row; idx++) {
+                const auto ind = tidx + idx * stride;
+                const auto col_idx = col[ind];
+                if (col_idx < idx) {
+                    break;
+                } else {
+                    temp += val(ind) * b(col_idx, column_id);
+                }
+            }
+            const auto c_ind = tidx * c_stride + column_id;
+            c[c_ind] = op(temp, c[c_ind]);
+        }
+    } else {
+        bool runnable = tidx < num_worker_per_row * num_rows;
+        const auto idx_in_worker = item_ct1.get_local_id(1);
+        const auto x = tidx % num_rows;
+        const auto worker_id = tidx / num_rows;
+        const auto step_size = num_worker_per_row * num_thread_per_worker;
+
+        if (runnable && idx_in_worker == 0) {
+            (*storage)[item_ct1.get_local_id(2)] = 0;
+        }
+        /*
+        DPCT1065:0: Consider replacing sycl::nd_item::barrier() with
+        sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
+        better performance, if there is no access to global memory.
+        */
+        item_ct1.barrier();
+        auto temp = zero<OutputValueType>();
+        if (runnable) {
+            for (size_type idx =
+                     worker_id * num_thread_per_worker + idx_in_worker;
+                 idx < num_stored_elements_per_row; idx += step_size) {
+                const auto ind = x + idx * stride;
+                const auto col_idx = col[ind];
+                if (col_idx < idx) {
+                    break;
+                } else {
+                    temp += val(ind) * b(col_idx, column_id);
+                }
+            }
+            atomic_add<atomic::local_space>(
+                &(*storage)[item_ct1.get_local_id(2)], temp);
+        }
+        /*
+        DPCT1065:1: Consider replacing sycl::nd_item::barrier() with
+        sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
+        better performance, if there is no access to global memory.
+        */
+        item_ct1.barrier();
+        if (runnable && idx_in_worker == 0) {
+            const auto c_ind = x * c_stride + column_id;
+            if (atomic) {
+                atomic_add(&(c[c_ind]),
+                           op((*storage)[item_ct1.get_local_id(2)], c[c_ind]));
+            } else {
+                c[c_ind] = op((*storage)[item_ct1.get_local_id(2)], c[c_ind]);
+            }
+        }
+    }
+}
+
+
+template <int num_thread_per_worker, bool atomic = false, typename b_accessor,
+          typename a_accessor, typename OutputValueType, typename IndexType>
+void spmv(
+    const size_type num_rows, const int num_worker_per_row,
+    acc::range<a_accessor> val, const IndexType *__restrict__ col,
+    const size_type stride, const size_type num_stored_elements_per_row,
+    acc::range<b_accessor> b, OutputValueType *__restrict__ c,
+    const size_type c_stride, sycl::nd_item<3> item_ct1,
+    UninitializedArray<OutputValueType,
+                       default_block_size / num_thread_per_worker> *storage)
+{
+    spmv_kernel<num_thread_per_worker, atomic>(
+        num_rows, num_worker_per_row, val, col, stride,
+        num_stored_elements_per_row, b, c, c_stride,
+        [](const OutputValueType &x, const OutputValueType &y) { return x; },
+        item_ct1, storage);
+}
+
+template <int num_thread_per_worker, bool atomic = false, typename b_accessor,
+          typename a_accessor, typename OutputValueType, typename IndexType>
+void spmv(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+          sycl::queue *stream, const size_type num_rows,
+          const int num_worker_per_row, acc::range<a_accessor> val,
+          const IndexType *col, const size_type stride,
+          const size_type num_stored_elements_per_row, acc::range<b_accessor> b,
+          OutputValueType *c, const size_type c_stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<
+            UninitializedArray<OutputValueType,
+                               default_block_size / num_thread_per_worker>,
+            0, sycl::access::mode::read_write, sycl::access::target::local>
+            storage_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                spmv<num_thread_per_worker, atomic>(
+                    num_rows, num_worker_per_row, val, col, stride,
+                    num_stored_elements_per_row, b, c, c_stride, item_ct1,
+                    (UninitializedArray<OutputValueType,
+                                        default_block_size /
+                                            num_thread_per_worker> *)
+                        storage_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+template <int num_thread_per_worker, bool atomic = false, typename b_accessor,
+          typename a_accessor, typename OutputValueType, typename IndexType>
+void spmv(
+    const size_type num_rows, const int num_worker_per_row,
+    acc::range<a_accessor> alpha, acc::range<a_accessor> val,
+    const IndexType *__restrict__ col, const size_type stride,
+    const size_type num_stored_elements_per_row, acc::range<b_accessor> b,
+    const OutputValueType *__restrict__ beta, OutputValueType *__restrict__ c,
+    const size_type c_stride, sycl::nd_item<3> item_ct1,
+    UninitializedArray<OutputValueType,
+                       default_block_size / num_thread_per_worker> *storage)
+{
+    const OutputValueType alpha_val = alpha(0);
+    const OutputValueType beta_val = beta[0];
+    // Because the atomic operation changes the values of c during computation,
+    // it can not do the right alpha * a * b + beta * c operation.
+    // Thus, the dpcpp kernel only computes alpha * a * b when it uses atomic
+    // operation.
+    if (atomic) {
+        spmv_kernel<num_thread_per_worker, atomic>(
+            num_rows, num_worker_per_row, val, col, stride,
+            num_stored_elements_per_row, b, c, c_stride,
+            [&alpha_val](const OutputValueType &x, const OutputValueType &y) {
+                return alpha_val * x;
+            },
+            item_ct1, storage);
+    } else {
+        spmv_kernel<num_thread_per_worker, atomic>(
+            num_rows, num_worker_per_row, val, col, stride,
+            num_stored_elements_per_row, b, c, c_stride,
+            [&alpha_val, &beta_val](const OutputValueType &x,
+                                    const OutputValueType &y) {
+                return alpha_val * x + beta_val * y;
+            },
+            item_ct1, storage);
+    }
+}
+
+template <int num_thread_per_worker, bool atomic = false, typename b_accessor,
+          typename a_accessor, typename OutputValueType, typename IndexType>
+void spmv(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+          sycl::queue *stream, const size_type num_rows,
+          const int num_worker_per_row, acc::range<a_accessor> alpha,
+          acc::range<a_accessor> val, const IndexType *col,
+          const size_type stride, const size_type num_stored_elements_per_row,
+          acc::range<b_accessor> b, const OutputValueType *beta,
+          OutputValueType *c, const size_type c_stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<
+            UninitializedArray<OutputValueType,
+                               default_block_size / num_thread_per_worker>,
+            0, sycl::access::mode::read_write, sycl::access::target::local>
+            storage_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                spmv<num_thread_per_worker, atomic>(
+                    num_rows, num_worker_per_row, alpha, val, col, stride,
+                    num_stored_elements_per_row, b, beta, c, c_stride, item_ct1,
+                    (UninitializedArray<OutputValueType,
+                                        default_block_size /
+                                            num_thread_per_worker> *)
+                        storage_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+}  // namespace
+
+
+template <typename ValueType>
+void initialize_zero_dense(size_type num_rows, size_type num_cols,
+                           size_type stride, ValueType *__restrict__ result,
+                           sycl::nd_item<3> item_ct1)
+{
+    const auto tidx_x =
+        item_ct1.get_local_id(2) +
+        item_ct1.get_local_range().get(2) * item_ct1.get_group(2);
+    const auto tidx_y =
+        item_ct1.get_local_id(1) +
+        item_ct1.get_local_range().get(1) * item_ct1.get_group(1);
+    if (tidx_x < num_cols && tidx_y < num_rows) {
+        result[tidx_y * stride + tidx_x] = zero<ValueType>();
+    }
+}
+
+template <typename ValueType>
+void initialize_zero_dense(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                           sycl::queue *stream, size_type num_rows,
+                           size_type num_cols, size_type stride,
+                           ValueType *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             initialize_zero_dense(num_rows, num_cols, stride,
+                                                   result, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void fill_in_dense(size_type num_rows, size_type nnz, size_type source_stride,
+                   const IndexType *__restrict__ col_idxs,
+                   const ValueType *__restrict__ values,
+                   size_type result_stride, ValueType *__restrict__ result,
+                   sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    if (tidx < num_rows) {
+        for (auto col = 0; col < nnz; col++) {
+            result[tidx * result_stride +
+                   col_idxs[tidx + col * source_stride]] +=
+                values[tidx + col * source_stride];
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void fill_in_dense(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                   sycl::queue *stream, size_type num_rows, size_type nnz,
+                   size_type source_stride, const IndexType *col_idxs,
+                   const ValueType *values, size_type result_stride,
+                   ValueType *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                fill_in_dense(num_rows, nnz, source_stride, col_idxs, values,
+                              result_stride, result, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void count_nnz_per_row(size_type num_rows, size_type max_nnz_per_row,
+                       size_type stride, const ValueType *__restrict__ values,
+                       IndexType *__restrict__ result,
+                       sycl::nd_item<3> item_ct1)
+{
+    constexpr auto warp_size = config::warp_size;
+    const auto row_idx = thread::get_subwarp_id_flat<warp_size>(item_ct1);
+    auto warp_tile =
+        group::tiled_partition<warp_size>(group::this_thread_block(item_ct1));
+
+    if (row_idx < num_rows) {
+        IndexType part_result{};
+        for (auto i = warp_tile.thread_rank(); i < max_nnz_per_row;
+             i += warp_size) {
+            if (values[stride * i + row_idx] != zero<ValueType>()) {
+                part_result += 1;
+            }
+        }
+        result[row_idx] = ::gko::kernels::dpcpp::reduce(
+            warp_tile, part_result,
+            [](const size_type &a, const size_type &b) { return a + b; });
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void count_nnz_per_row(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                       sycl::queue *stream, size_type num_rows,
+                       size_type max_nnz_per_row, size_type stride,
+                       const ValueType *values, IndexType *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                count_nnz_per_row(num_rows, max_nnz_per_row, stride, values,
+                                  result, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void fill_in_csr(size_type num_rows, size_type max_nnz_per_row,
+                 size_type stride, const ValueType *__restrict__ source_values,
+                 const IndexType *__restrict__ source_col_idxs,
+                 IndexType *__restrict__ result_row_ptrs,
+                 IndexType *__restrict__ result_col_idxs,
+                 ValueType *__restrict__ result_values,
+                 sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+
+    if (tidx < num_rows) {
+        auto write_to = result_row_ptrs[tidx];
+        for (auto i = 0; i < max_nnz_per_row; i++) {
+            const auto source_idx = tidx + stride * i;
+            if (source_values[source_idx] != zero<ValueType>()) {
+                result_values[write_to] = source_values[source_idx];
+                result_col_idxs[write_to] = source_col_idxs[source_idx];
+                write_to++;
+            }
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void fill_in_csr(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                 sycl::queue *stream, size_type num_rows,
+                 size_type max_nnz_per_row, size_type stride,
+                 const ValueType *source_values,
+                 const IndexType *source_col_idxs, IndexType *result_row_ptrs,
+                 IndexType *result_col_idxs, ValueType *result_values)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                fill_in_csr(num_rows, max_nnz_per_row, stride, source_values,
+                            source_col_idxs, result_row_ptrs, result_col_idxs,
+                            result_values, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(size_type diag_size, size_type max_nnz_per_row,
+                      size_type orig_stride,
+                      const ValueType *__restrict__ orig_values,
+                      const IndexType *__restrict__ orig_col_idxs,
+                      ValueType *__restrict__ diag, sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    const auto row = tidx % diag_size;
+    const auto col = tidx / diag_size;
+    const auto ell_ind = orig_stride * col + row;
+
+    if (col < max_nnz_per_row) {
+        if (orig_col_idxs[ell_ind] == row &&
+            orig_values[ell_ind] != zero<ValueType>()) {
+            diag[row] = orig_values[ell_ind];
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                      sycl::queue *stream, size_type diag_size,
+                      size_type max_nnz_per_row, size_type orig_stride,
+                      const ValueType *orig_values,
+                      const IndexType *orig_col_idxs, ValueType *diag)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                extract_diagonal(diag_size, max_nnz_per_row, orig_stride,
+                                 orig_values, orig_col_idxs, diag, item_ct1);
+            });
+    });
+}
+
+
+}  // namespace kernel
+
+
+namespace {
+
+template <int dim, typename Type1, typename Type2>
+GKO_INLINE auto as_dpcpp_accessor(
+    const acc::range<acc::reduced_row_major<dim, Type1, Type2>> &acc)
+{
+    return acc::range<acc::reduced_row_major<dim, Type1, Type2>>(
+        acc.get_accessor().get_size(), acc.get_accessor().get_stored_data(),
+        acc.get_accessor().get_stride());
+}
+
+
+template <int info, typename InputValueType, typename MatrixValueType,
+          typename OutputValueType, typename IndexType>
+void abstract_spmv(syn::value_list<int, info>,
+                   std::shared_ptr<const DpcppExecutor> exec,
+                   int num_worker_per_row,
+                   const matrix::Ell<MatrixValueType, IndexType> *a,
+                   const matrix::Dense<InputValueType> *b,
+                   matrix::Dense<OutputValueType> *c,
+                   const matrix::Dense<MatrixValueType> *alpha = nullptr,
+                   const matrix::Dense<OutputValueType> *beta = nullptr)
+{
+    using a_accessor =
+        gko::acc::reduced_row_major<1, OutputValueType, const MatrixValueType>;
+    using b_accessor =
+        gko::acc::reduced_row_major<2, OutputValueType, const InputValueType>;
+
+    const auto nrows = a->get_size()[0];
+    const auto stride = a->get_stride();
+    const auto num_stored_elements_per_row =
+        a->get_num_stored_elements_per_row();
+
+    constexpr int num_thread_per_worker =
+        (info == 0) ? max_thread_per_worker : info;
+    constexpr bool atomic = (info == 0);
+    const dim3 block_size(default_block_size / num_thread_per_worker,
+                          num_thread_per_worker, 1);
+    const dim3 grid_size(ceildiv(nrows * num_worker_per_row, block_size.x),
+                         b->get_size()[1], 1);
+
+    const auto a_vals = gko::acc::range<a_accessor>(
+        std::array<size_type, 1>{{num_stored_elements_per_row * stride}},
+        a->get_const_values());
+    const auto b_vals = gko::acc::range<b_accessor>(
+        std::array<size_type, 2>{{b->get_size()[0], b->get_size()[1]}},
+        b->get_const_values(), std::array<size_type, 1>{{b->get_stride()}});
+
+    if (alpha == nullptr && beta == nullptr) {
+        kernel::spmv<num_thread_per_worker, atomic>(
+            grid_size, block_size, 0, exec->get_queue(), nrows,
+            num_worker_per_row, as_dpcpp_accessor(a_vals),
+            a->get_const_col_idxs(), stride, num_stored_elements_per_row,
+            as_dpcpp_accessor(b_vals), c->get_values(), c->get_stride());
+    } else if (alpha != nullptr && beta != nullptr) {
+        const auto alpha_val = gko::acc::range<a_accessor>(
+            std::array<size_type, 1>{1}, alpha->get_const_values());
+        kernel::spmv<num_thread_per_worker, atomic>(
+            grid_size, block_size, 0, exec->get_queue(), nrows,
+            num_worker_per_row, as_dpcpp_accessor(alpha_val),
+            as_dpcpp_accessor(a_vals), a->get_const_col_idxs(), stride,
+            num_stored_elements_per_row, as_dpcpp_accessor(b_vals),
+            beta->get_const_values(), c->get_values(), c->get_stride());
+    } else {
+        GKO_KERNEL_NOT_FOUND;
+    }
+}
+
+GKO_ENABLE_IMPLEMENTATION_SELECTION(select_abstract_spmv, abstract_spmv);
+
+
+template <typename ValueType, typename IndexType>
+std::array<int, 3> compute_thread_worker_and_atomicity(
+    std::shared_ptr<const DpcppExecutor> exec,
+    const matrix::Ell<ValueType, IndexType> *a)
+{
+    int num_thread_per_worker = 8;
+    int atomic = 0;
+    int num_worker_per_row = 1;
+
+    const auto nrows = a->get_size()[0];
+    const auto ell_ncols = a->get_num_stored_elements_per_row();
+    // TODO: num_threads_per_core should be tuned for AMD gpu
+    const auto nwarps = 16 * num_threads_per_core;
+
+    // Use multithreads to perform the reduction on each row when the matrix is
+    // wide.
+    // To make every thread have computation, so pick the value which is the
+    // power of 2 less than max_thread_per_worker and is less than or equal to
+    // ell_ncols. If the num_thread_per_worker is max_thread_per_worker and
+    // allow more than one worker to work on the same row, use atomic add to
+    // handle the worker write the value into the same position. The #worker is
+    // decided according to the number of worker allowed on GPU.
+    if (static_cast<double>(ell_ncols) / nrows > ratio) {
+        while (num_thread_per_worker < max_thread_per_worker &&
+               (num_thread_per_worker << 1) <= ell_ncols) {
+            num_thread_per_worker <<= 1;
+        }
+        if (num_thread_per_worker == max_thread_per_worker) {
+            num_worker_per_row =
+                std::min(ell_ncols / max_thread_per_worker, nwarps / nrows);
+            num_worker_per_row = std::max(num_worker_per_row, 1);
+        }
+        if (num_worker_per_row > 1) {
+            atomic = 1;
+        }
+    }
+    return {num_thread_per_worker, atomic, num_worker_per_row};
+}
+
+
+}  // namespace
+
+
 template <typename InputValueType, typename MatrixValueType,
           typename OutputValueType, typename IndexType>
 void spmv(std::shared_ptr<const DpcppExecutor> exec,
           const matrix::Ell<MatrixValueType, IndexType> *a,
           const matrix::Dense<InputValueType> *b,
-          matrix::Dense<OutputValueType> *c) GKO_NOT_IMPLEMENTED;
+          matrix::Dense<OutputValueType> *c)
+{
+    const auto data = compute_thread_worker_and_atomicity(exec, a);
+    const int num_thread_per_worker = std::get<0>(data);
+    const int atomic = std::get<1>(data);
+    const int num_worker_per_row = std::get<2>(data);
+
+    /**
+     * info is the parameter for selecting the dpcpp kernel.
+     * for info == 0, it uses the kernel by warp_size threads with atomic
+     * operation for other value, it uses the kernel without atomic_add
+     */
+    const int info = (!atomic) * num_thread_per_worker;
+    if (atomic) {
+        components::fill_array(exec, c->get_values(),
+                               c->get_num_stored_elements(),
+                               zero<OutputValueType>());
+    }
+    select_abstract_spmv(
+        compiled_kernels(),
+        [&info](int compiled_info) { return info == compiled_info; },
+        syn::value_list<int>(), syn::type_list<>(), exec, num_worker_per_row, a,
+        b, c);
+}
 
 GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_SPMV_KERNEL);
@@ -75,7 +658,28 @@ void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::Ell<MatrixValueType, IndexType> *a,
                    const matrix::Dense<InputValueType> *b,
                    const matrix::Dense<OutputValueType> *beta,
-                   matrix::Dense<OutputValueType> *c) GKO_NOT_IMPLEMENTED;
+                   matrix::Dense<OutputValueType> *c)
+{
+    const auto data = compute_thread_worker_and_atomicity(exec, a);
+    const int num_thread_per_worker = std::get<0>(data);
+    const int atomic = std::get<1>(data);
+    const int num_worker_per_row = std::get<2>(data);
+
+    /**
+     * info is the parameter for selecting the dpcpp kernel.
+     * for info == 0, it uses the kernel by warp_size threads with atomic
+     * operation for other value, it uses the kernel without atomic_add
+     */
+    const int info = (!atomic) * num_thread_per_worker;
+    if (atomic) {
+        dense::scale(exec, beta, c);
+    }
+    select_abstract_spmv(
+        compiled_kernels(),
+        [&info](int compiled_info) { return info == compiled_info; },
+        syn::value_list<int>(), syn::type_list<>(), exec, num_worker_per_row, a,
+        b, c, alpha, beta);
+}
 
 GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_ADVANCED_SPMV_KERNEL);
@@ -84,7 +688,29 @@ GKO_INSTANTIATE_FOR_EACH_MIXED_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Ell<ValueType, IndexType> *source,
-                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
+                      matrix::Dense<ValueType> *result)
+{
+    const auto num_rows = result->get_size()[0];
+    const auto num_cols = result->get_size()[1];
+    const auto result_stride = result->get_stride();
+    const auto col_idxs = source->get_const_col_idxs();
+    const auto vals = source->get_const_values();
+    const auto source_stride = source->get_stride();
+
+    const dim3 block_size(config::warp_size,
+                          config::max_block_size / config::warp_size, 1);
+    const dim3 init_grid_dim(ceildiv(num_cols, block_size.x),
+                             ceildiv(num_rows, block_size.y), 1);
+    kernel::initialize_zero_dense(init_grid_dim, block_size, 0,
+                                  exec->get_queue(), num_rows, num_cols,
+                                  result_stride, result->get_values());
+
+    const auto grid_dim = ceildiv(num_rows, default_block_size);
+    kernel::fill_in_dense(grid_dim, default_block_size, 0, exec->get_queue(),
+                          num_rows, source->get_num_stored_elements_per_row(),
+                          source_stride, col_idxs, vals, result_stride,
+                          result->get_values());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_CONVERT_TO_DENSE_KERNEL);
@@ -94,15 +720,50 @@ template <typename ValueType, typename IndexType>
 void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Ell<ValueType, IndexType> *source,
                     matrix::Csr<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
+{
+    auto num_rows = result->get_size()[0];
+
+    auto row_ptrs = result->get_row_ptrs();
+    auto col_idxs = result->get_col_idxs();
+    auto values = result->get_values();
+
+    const auto stride = source->get_stride();
+    const auto max_nnz_per_row = source->get_num_stored_elements_per_row();
+
+    constexpr auto rows_per_block =
+        ceildiv(default_block_size, config::warp_size);
+    const auto grid_dim_nnz = ceildiv(source->get_size()[0], rows_per_block);
+
+    kernel::count_nnz_per_row(grid_dim_nnz, default_block_size, 0,
+                              exec->get_queue(), num_rows, max_nnz_per_row,
+                              stride, source->get_const_values(), row_ptrs);
+
+    components::prefix_sum(exec, row_ptrs, num_rows + 1);
+
+    size_type grid_dim = ceildiv(num_rows, default_block_size);
+
+    kernel::fill_in_csr(
+        grid_dim, default_block_size, 0, exec->get_queue(), num_rows,
+        max_nnz_per_row, stride, source->get_const_values(),
+        source->get_const_col_idxs(), row_ptrs, col_idxs, values);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_CONVERT_TO_CSR_KERNEL);
 
+
 template <typename ValueType, typename IndexType>
 void count_nonzeros(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Ell<ValueType, IndexType> *source,
-                    size_type *result) GKO_NOT_IMPLEMENTED;
+                    size_type *result)
+{
+    const auto num_rows = source->get_size()[0];
+    auto nnz_per_row = Array<size_type>(exec, num_rows);
+
+    calculate_nonzeros_per_row(exec, source, &nnz_per_row);
+
+    *result = reduce_add_array(exec, num_rows, nnz_per_row.get_const_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_COUNT_NONZEROS_KERNEL);
@@ -111,7 +772,20 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void calculate_nonzeros_per_row(std::shared_ptr<const DpcppExecutor> exec,
                                 const matrix::Ell<ValueType, IndexType> *source,
-                                Array<size_type> *result) GKO_NOT_IMPLEMENTED;
+                                Array<size_type> *result)
+{
+    const auto num_rows = source->get_size()[0];
+    const auto max_nnz_per_row = source->get_num_stored_elements_per_row();
+    const auto stride = source->get_stride();
+    const auto values = source->get_const_values();
+
+    const auto warp_size = config::warp_size;
+    const auto grid_dim = ceildiv(num_rows * warp_size, default_block_size);
+
+    kernel::count_nnz_per_row(grid_dim, default_block_size, 0,
+                              exec->get_queue(), num_rows, max_nnz_per_row,
+                              stride, values, result->get_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_CALCULATE_NONZEROS_PER_ROW_KERNEL);
@@ -120,7 +794,22 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Ell<ValueType, IndexType> *orig,
-                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
+                      matrix::Diagonal<ValueType> *diag)
+{
+    const auto max_nnz_per_row = orig->get_num_stored_elements_per_row();
+    const auto orig_stride = orig->get_stride();
+    const auto diag_size = diag->get_size()[0];
+    const auto num_blocks =
+        ceildiv(diag_size * max_nnz_per_row, default_block_size);
+
+    const auto orig_values = orig->get_const_values();
+    const auto orig_col_idxs = orig->get_const_col_idxs();
+    auto diag_values = diag->get_values();
+
+    kernel::extract_diagonal(
+        num_blocks, default_block_size, 0, exec->get_queue(), diag_size,
+        max_nnz_per_row, orig_stride, orig_values, orig_col_idxs, diag_values);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ELL_EXTRACT_DIAGONAL_KERNEL);
diff --git a/dpcpp/test/matrix/CMakeLists.txt b/dpcpp/test/matrix/CMakeLists.txt
index ba8f0fb70fe..e905c257493 100644
--- a/dpcpp/test/matrix/CMakeLists.txt
+++ b/dpcpp/test/matrix/CMakeLists.txt
@@ -1,2 +1,3 @@
 ginkgo_create_test(csr_kernels)
 ginkgo_create_test(dense_kernels)
+ginkgo_create_test(ell_kernels)
diff --git a/dpcpp/test/matrix/ell_kernels.cpp b/dpcpp/test/matrix/ell_kernels.cpp
new file mode 100644
index 00000000000..d876bb1b38a
--- /dev/null
+++ b/dpcpp/test/matrix/ell_kernels.cpp
@@ -0,0 +1,682 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/matrix/ell.hpp>
+
+
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+
+
+#include "core/matrix/ell_kernels.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class Ell : public ::testing::Test {
+protected:
+#if GINKGO_DPCPP_SINGLE_MODE
+    using vtype = float;
+#else
+    using vtype = double;
+#endif  // GINKGO_DPCPP_SINGLE_MODE
+    using Mtx = gko::matrix::Ell<vtype>;
+    using Vec = gko::matrix::Dense<vtype>;
+    using Vec2 = gko::matrix::Dense<float>;
+    using ComplexVec = gko::matrix::Dense<std::complex<vtype>>;
+
+    Ell()
+        : rand_engine(42), size{532, 231}, num_els_rowwise{300}, ell_stride{600}
+    {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices("all"), 0);
+        ref = gko::ReferenceExecutor::create();
+        dpcpp = gko::DpcppExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (dpcpp != nullptr) {
+            ASSERT_NO_THROW(dpcpp->synchronize());
+        }
+    }
+
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<MtxType>(
+            num_rows, num_cols, std::uniform_int_distribution<>(1, num_cols),
+            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    void set_up_apply_data(int num_rows = 532, int num_cols = 231,
+                           int num_vectors = 1,
+                           int num_stored_elements_per_row = 0, int stride = 0)
+    {
+        mtx = Mtx::create(ref, gko::dim<2>{}, num_stored_elements_per_row,
+                          stride);
+        mtx->copy_from(gen_mtx(num_rows, num_cols));
+        expected = gen_mtx(num_rows, num_vectors);
+        expected2 = Vec2::create(ref);
+        expected2->copy_from(expected.get());
+        y = gen_mtx(num_cols, num_vectors);
+        y2 = Vec2::create(ref);
+        y2->copy_from(y.get());
+        alpha = gko::initialize<Vec>({2.0}, ref);
+        alpha2 = gko::initialize<Vec2>({2.0}, ref);
+        beta = gko::initialize<Vec>({-1.0}, ref);
+        beta2 = gko::initialize<Vec2>({-1.0}, ref);
+        dmtx = Mtx::create(dpcpp);
+        dmtx->copy_from(mtx.get());
+        dresult = Vec::create(dpcpp);
+        dresult->copy_from(expected.get());
+        dresult2 = Vec2::create(dpcpp);
+        dresult2->copy_from(expected2.get());
+        dy = Vec::create(dpcpp);
+        dy->copy_from(y.get());
+        dy2 = Vec2::create(dpcpp);
+        dy2->copy_from(y2.get());
+        dalpha = Vec::create(dpcpp);
+        dalpha->copy_from(alpha.get());
+        dalpha2 = Vec2::create(dpcpp);
+        dalpha2->copy_from(alpha2.get());
+        dbeta = Vec::create(dpcpp);
+        dbeta->copy_from(beta.get());
+        dbeta2 = Vec2::create(dpcpp);
+        dbeta2->copy_from(beta2.get());
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
+
+    std::ranlux48 rand_engine;
+    gko::dim<2> size;
+    gko::size_type num_els_rowwise;
+    gko::size_type ell_stride;
+
+    std::unique_ptr<Mtx> mtx;
+    std::unique_ptr<Vec> expected;
+    std::unique_ptr<Vec2> expected2;
+    std::unique_ptr<Vec> y;
+    std::unique_ptr<Vec2> y2;
+    std::unique_ptr<Vec> alpha;
+    std::unique_ptr<Vec2> alpha2;
+    std::unique_ptr<Vec> beta;
+    std::unique_ptr<Vec2> beta2;
+
+    std::unique_ptr<Mtx> dmtx;
+    std::unique_ptr<Vec> dresult;
+    std::unique_ptr<Vec2> dresult2;
+    std::unique_ptr<Vec> dy;
+    std::unique_ptr<Vec2> dy2;
+    std::unique_ptr<Vec> dalpha;
+    std::unique_ptr<Vec2> dalpha2;
+    std::unique_ptr<Vec> dbeta;
+    std::unique_ptr<Vec2> dbeta2;
+};
+
+
+TEST_F(Ell, SimpleApplyIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+#if !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef1)
+{
+    set_up_apply_data();
+
+    mtx->apply(y2.get(), expected2.get());
+    dmtx->apply(dy2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef2)
+{
+    set_up_apply_data();
+
+    mtx->apply(y2.get(), expected.get());
+    dmtx->apply(dy2.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef3)
+{
+    set_up_apply_data();
+
+    mtx->apply(y.get(), expected2.get());
+    dmtx->apply(dy.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
+}
+
+
+#endif  // !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Ell, AdvancedApplyIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+#if !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef1)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef2)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef3)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
+}
+
+
+#endif  // !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Ell, SimpleApplyWithStrideIsEquivalentToRef)
+{
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+#if !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef1)
+{
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
+
+    mtx->apply(y2.get(), expected2.get());
+    dmtx->apply(dy2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef2)
+{
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
+
+    mtx->apply(y2.get(), expected.get());
+    dmtx->apply(dy2.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef3)
+{
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
+
+    mtx->apply(y.get(), expected2.get());
+    dmtx->apply(dy.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
+}
+
+
+#endif  // !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Ell, AdvancedApplyWithStrideIsEquivalentToRef)
+{
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+#if !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef1)
+{
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
+
+    mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef2)
+{
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
+
+    mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef3)
+{
+    set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
+
+    mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
+}
+
+
+#endif  // !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Ell, SimpleApplyWithStrideToDenseMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+#if !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef1)
+{
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
+
+    mtx->apply(y2.get(), expected2.get());
+    dmtx->apply(dy2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef2)
+{
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
+
+    mtx->apply(y2.get(), expected.get());
+    dmtx->apply(dy2.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef3)
+{
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
+
+    mtx->apply(y.get(), expected2.get());
+    dmtx->apply(dy.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
+}
+
+
+#endif  // !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Ell, AdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+#if !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef1)
+{
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
+
+    mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef2)
+{
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
+
+    mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha2.get(), dy2.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef3)
+{
+    set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
+
+    mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta2.get(), dresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult2, expected2, 1e-6);
+}
+
+
+#endif  // !GINKGO_DPCPP_SINGLE_MODE
+
+
+TEST_F(Ell, SimpleApplyByAtomicIsEquivalentToRef)
+{
+    set_up_apply_data(10, 10000);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, AdvancedByAtomicApplyIsEquivalentToRef)
+{
+    set_up_apply_data(10, 10000);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, SimpleApplyByAtomicToDenseMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(10, 10000, 3);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, AdvancedByAtomicToDenseMatrixApplyIsEquivalentToRef)
+{
+    set_up_apply_data(10, 10000, 3);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, SimpleApplyOnSmallMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(1, 10);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, AdvancedApplyOnSmallMatrixToDenseMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(1, 10, 3);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, SimpleApplyOnSmallMatrixToDenseMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(1, 10, 3);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, AdvancedApplyOnSmallMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(1, 10);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(size[1], 3);
+    auto dcomplex_b = ComplexVec::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(size[0], 3);
+    auto dcomplex_x = ComplexVec::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+
+TEST_F(Ell, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(size[1], 3);
+    auto dcomplex_b = ComplexVec::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(size[0], 3);
+    auto dcomplex_x = ComplexVec::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+
+TEST_F(Ell, ConvertToDenseIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto dense_mtx = gko::matrix::Dense<vtype>::create(ref);
+    auto ddense_mtx = gko::matrix::Dense<vtype>::create(dpcpp);
+
+    mtx->convert_to(dense_mtx.get());
+    dmtx->convert_to(ddense_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Ell, ConvertToCsrIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto csr_mtx = gko::matrix::Csr<vtype>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<vtype>::create(dpcpp);
+
+    mtx->convert_to(csr_mtx.get());
+    dmtx->convert_to(dcsr_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Ell, CalculateNNZPerRowIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    gko::Array<gko::size_type> nnz_per_row;
+    nnz_per_row.set_executor(ref);
+    nnz_per_row.resize_and_reset(mtx->get_size()[0]);
+
+    gko::Array<gko::size_type> dnnz_per_row;
+    dnnz_per_row.set_executor(dpcpp);
+    dnnz_per_row.resize_and_reset(dmtx->get_size()[0]);
+
+    gko::kernels::reference::ell::calculate_nonzeros_per_row(ref, mtx.get(),
+                                                             &nnz_per_row);
+    gko::kernels::dpcpp::ell::calculate_nonzeros_per_row(dpcpp, dmtx.get(),
+                                                         &dnnz_per_row);
+
+    auto tmp = gko::Array<gko::size_type>(ref, dnnz_per_row);
+    for (auto i = 0; i < nnz_per_row.get_num_elems(); i++) {
+        ASSERT_EQ(nnz_per_row.get_const_data()[i], tmp.get_const_data()[i]);
+    }
+}
+
+
+TEST_F(Ell, CountNNZIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    gko::size_type nnz;
+    gko::size_type dnnz;
+
+    gko::kernels::reference::ell::count_nonzeros(ref, mtx.get(), &nnz);
+    gko::kernels::dpcpp::ell::count_nonzeros(dpcpp, dmtx.get(), &dnnz);
+
+    ASSERT_EQ(nnz, dnnz);
+}
+
+
+TEST_F(Ell, ExtractDiagonalIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto diag = mtx->extract_diagonal();
+    auto ddiag = dmtx->extract_diagonal();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
+TEST_F(Ell, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
+
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
+}
+
+
+TEST_F(Ell, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+}
+
+
+}  // namespace

From ad497dd7cc252422ae7711a873f0f34219f92091 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 7 Jul 2021 14:15:41 +0200
Subject: [PATCH 0913/1008] update some regex and remove auto config add

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 dev_tools/oneapi/add_host_function.sh | 137 +------------
 dev_tools/oneapi/convert_source.sh    | 271 ++++++++++----------------
 dev_tools/oneapi/shared.sh            |  22 +++
 3 files changed, 135 insertions(+), 295 deletions(-)
 create mode 100644 dev_tools/oneapi/shared.sh

diff --git a/dev_tools/oneapi/add_host_function.sh b/dev_tools/oneapi/add_host_function.sh
index 6f5a0dd6574..c5ac8bbaa6c 100755
--- a/dev_tools/oneapi/add_host_function.sh
+++ b/dev_tools/oneapi/add_host_function.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
+SCRIPT_DIR="$( dirname "${BASH_SOURCE[0]}" )"
+source "${SCRIPT_DIR}/shared.sh"
 
 extract_varname() {
     local str="$1"
@@ -34,114 +36,6 @@ extract_varname() {
     echo "$parameter"
 }
 
-check_closed() {
-    local str="$1"
-    str="${str//->}"
-    str_start="${str//[^(<\[]}"
-    str_end="${str//[^>)\]]}"
-    if [[ "${#str_start}" -eq "${#str_end}" ]]; then
-        echo "true"
-    else
-        echo "false"
-    fi
-}
-CONFIG_SELECTION_SUFFIX="_CONFIG"
-generate_config_selection() {
-    local template="$1"
-    local name="$2"
-    local variable="$3"
-    local var_input="$4"
-    template=$(echo ${template} | sed -E 's/^ *template *<(.*)> *$/\1/g')
-    local temp=""
-    IFS=',' read -ra par <<< "$template"
-
-    local id_list=("bool" "int" "size_type" "typename")
-    # Bash 4+ required
-    declare -A rev_id_list
-    local rev_id_list=(["bool"]=0 ["int"]=1 ["size_type"]=2 ["typename"]=3 ["Config"]=4)
-    declare -A list_list
-    list_list=(["bool"]="bool" ["int"]="int" ["size_type"]="size_type" ["typename"]="")
-    local max_num=0;
-    local param_regex="([^ =\*]*) *([^ =\*]*) *(= *.*)* *$"
-    local template_rm_config=""
-    local default_config=""
-    for var in "${par[@]}"; do
-        # echo "varaible - ${var}"
-        if [ -n "${temp}" ]; then
-            temp="${temp},"
-        fi
-        temp="${temp}${var}"
-        is_closed=$(check_closed "$temp")
-        if [[ "${is_closed}" = "false" ]]; then
-            continue
-        fi
-        # echo "temp - ${temp}"
-        # If the string contains typename, do not extract it.
-        # It should automatically be decided from argument
-        local config_regex="Config *([a-zA-Z0-9_]*) *= *([a-zA-Z][a-zA-Z_:,\(\) ]*)"
-        if [[ "${temp}" =~ ${config_regex} ]]; then
-            max_num=${rev_id_list[Config]}
-            default_config="${BASH_REMATCH[2]}"
-            temp=""
-            continue
-        fi
-        if [ -n "${template_rm_config}" ]; then
-                template_rm_config="${template_rm_config}, "
-        fi
-        template_rm_config="${template_rm_config}${temp}"
-        if [[  "${temp}" =~ $param_regex ]]; then
-            identification=${BASH_REMATCH[1]}
-            parameter_name=${BASH_REMATCH[2]}
-            # echo "${identification} ${parameter_name}"
-            if [ "${max_num}" -eq "${rev_id_list[Config]}" ]; then
-                if [ "${identification}" != "typename" ]; then
-                    echo "static_assert(false, \"Only inferred type are allowed after Config\");"
-                fi
-            else
-                if [ "${rev_id_list[${identification}]}" -ge "${max_num}" ]; then
-                    max_num=${rev_id_list[${identification}]}
-                    if [ -n "${list_list[${identification}]}" ]; then
-                        parameter_name=", ${parameter_name}"
-                    fi
-                    list_list[${identification}]="${list_list[${identification}]}${parameter_name}"
-                else
-                    echo "static_assert(false, \"Need to follow bool, int, size_type, type ordering\");"
-                fi
-            fi
-        fi
-        temp=""
-    done
-    # for item in "${!list_list[@]}"; do
-    #     echo "//$item - ${list_list[$item]}"
-    # done
-    config_generation=""
-    if [[ "${default_config}" = *"config_set"* ]]; then
-        # When the config is set in template, use it directly.
-        config_generation="::gko::syn::value_list<Config, ${default_config}>();"
-    else
-        # When config use the constexpr variable, use the variable name. The constexpr variable will be changed to Config_list afterwards.
-        config_generation="${default_config}_list"
-    fi
-    echo "template <${template_rm_config}>
-        void ${name}${CONFIG_SELECTION_SUFFIX}
-        (dim3 grid, dim3 block, size_t dynamic_shared_memory, cudaStream_t stream, std::shared_ptr<const CudaExecutor> exec, ${variable}) {
-        auto config_list = ${config_generation};
-        auto exec_config = exec->get_config();
-        ${name}${CONFIG_SELECTION_SUFFIX}(
-            config_list,
-            [&exec_config] (Config config) {
-                return (get_warp_size(exec_config) == get_warp_size(config)) &&
-                        (get_block_size(exec_config) >= get_block_size(config));
-                },
-            ::gko::syn::value_list<${list_list[bool]}>(),
-            ::gko::syn::value_list<${list_list[int]}>(),
-            ::gko::syn::value_list<${list_list[size_type]}>(),
-            ::gko::syn::type_list<${list_list[typename]}>(),
-            grid, block, dynamic_shared_memory, stream, ${var_input});
-    }
-    "
-}
-
 
 GLOBAL_KEYWORD="__global__"
 TEMPLATE_REGEX="^ *template <*"
@@ -159,12 +53,10 @@ IN_BLOCK=0
 IN_FUNC=0
 STORE_LINE=""
 STORE_REGEX="__ *$"
-HOST_SUFFIX="_AUTOHOSTFUNC"
 EXTRACT_KERNEL="false"
-GINKGO_LICENSE_BEACON="******************************<GINKGO LICENSE>******************************"
 DURING_LICENSE="false"
 SKIP="false"
-MAP_FILE="map_list"
+
 rm "${MAP_FILE}"
 while IFS='' read -r line || [ -n "$line" ]; do
     if [ "${EXTRACT_KERNEL}" = "false" ] && ([ "${line}" = "/*${GINKGO_LICENSE_BEACON}" ] ||  [ "${DURING_LICENSE}" = "true" ]); then
@@ -180,6 +72,7 @@ while IFS='' read -r line || [ -n "$line" ]; do
         continue
     fi
     SKIP="false"
+    # It prints the original text into new file.
     if [[ "$line" =~ ${STORE_REGEX} ]]; then
         STORE_LINE="${STORE_LINE} ${line}"
     elif [[ -n "${STORE_LINE}" ]]; then
@@ -258,27 +151,7 @@ while IFS='' read -r line || [ -n "$line" ]; do
                     echo "${TEMPLATE} void ${NAME}${HOST_SUFFIX} (dim3 grid, dim3 block, size_t dynamic_shared_memory, cudaStream_t stream, ${VARIABLE}) {
                         /*KEEP*/${NAME}${TEMPLATE_INPUT}<<<grid, block, dynamic_shared_memory, stream>>>(${VAR_INPUT});
                         }"
-                    if [[ "${FUNCTION_HANDLE}" = *"Config"* ]]; then
-                        # echo "/** DPCPP ONLY // RM_TAG"
-                        echo ""
-                        echo "GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(${NAME}${HOST_SUFFIX}${CONFIG_SELECTION_SUFFIX}, ${NAME}${HOST_SUFFIX})"
-                        echo ""
-                        echo ""
-                        config_function=$(generate_config_selection "${TEMPLATE}" "${NAME}${HOST_SUFFIX}" "${VARIABLE}" "${VAR_INPUT}")
-                        echo "${config_function}"
-                        # echo "**/ // RM_TAG"
-                        echo ""
-                        echo ""
-                        # echo "${TEMPLATE} void ${NAME}${HOST_SUFFIX}${CONFIG_SELECTION_SUFFIX} // RM_TAG
-                        #     (dim3 grid, dim3 block, size_t dynamic_shared_memory, cudaStream_t stream, std::shared_ptr<const CudaExecutor> exec, ${VARIABLE}) { // RM_TAG
-                        #     ${NAME}${TEMPLATE_INPUT}<<<grid, block, dynamic_shared_memory, stream>>>(${VAR_INPUT}); // RM_TAG
-                        #     } // RM_TAG"
-                        # write the map to the file ${NAME} -> ${NAME}${HOST_SUFFIX}${CONFIG_SELECTION_SUFFIX}
-                        echo "${NAME} -> ${NAME}${HOST_SUFFIX}${CONFIG_SELECTION_SUFFIX}" >> ${MAP_FILE}
-                    else
-                        # else, write the map to the file ${NAME} -> ${NAME}${HOST_SUFFIX}
-                        echo "${NAME} -> ${NAME}${HOST_SUFFIX}" >> ${MAP_FILE}
-                    fi
+                    echo "${NAME} -> ${NAME}${HOST_SUFFIX}" >> ${MAP_FILE}
                 fi
                 # echo ""
                 # check the property
diff --git a/dev_tools/oneapi/convert_source.sh b/dev_tools/oneapi/convert_source.sh
index 6b26bffc831..0b0a505a6e1 100755
--- a/dev_tools/oneapi/convert_source.sh
+++ b/dev_tools/oneapi/convert_source.sh
@@ -1,20 +1,27 @@
 #!/bin/bash
 
-# the following parameters set by environment
-# MUST:
+# Usage:
+# EnvironementSet ./dev_tools/oneapi/convert_source.sh <the_file_in_cuda>
+# <the_file_in_cuda> can be .hpp/.cpp/.cu/.cuh
+
+# the following are parameters set by environment variables
+# REQUIRED:
 #   CUDA_HEADER_DIR: contains the cuda headers
 # OPTIONAL:
 #   ROOT_DIR: the ginkgo folder. The default is current path
 #   BUILD_DIR: the ginkgo build folder, which needs cmake to generate config.hpp and gtest include. The default is "build"
-#              Note. It requires GINKGO_BUILD_TESTS=ON to download gtest but does not need to compile ginkgo.
-#                    If GTEST_HEADER_DIR is from other place, does not require GINKGO_BUILD_TESTS.
-#                    If copy the ginkgo config.hpp from other ginkgo build into "${ROOT_DIR}/include/ginkgo/", do not require cmake.
+#              Note. It requires GINKGO_BUILD_TESTS=ON to download gtest but it is not required to compile ginkgo.
+#                    If GTEST_HEADER_DIR is available elsewhere GINKGO_BUILD_TESTS is not required.
+#                    CMake's step is not required if copying the ginkgo config.hpp from another ginkgo build into "${ROOT_DIR}/include/ginkgo/".
 #   ROOT_BUILD_DIR: the complete path for build folder. The default is "${ROOT_DIR}/${BUILD_DIR}"
 #   GTEST_HEADER_DIR: the gtest header folder. The default is "${ROOT_BUILD_DIR}/third_party_gtest/src/googletest/include"
 #   CLANG_FORMAT: the clang-format exec. The default is "clang-format"
 CURRENT_DIR="$( pwd )"
 cd "$( dirname "${BASH_SOURCE[0]}" )"
 SCRIPT_DIR="$( pwd )"
+
+source "${SCRIPT_DIR}/shared.sh"
+
 ROOT_DIR="${ROOT_DIR:="${CURRENT_DIR}"}"
 BUILD_DIR="${BUILD_DIR:="build"}"
 ROOT_BUILD_DIR="${ROOT_BUILD_DIR:="${ROOT_DIR}/${BUILD_DIR}"}"
@@ -22,18 +29,19 @@ CUDA_HEADER_DIR="${CUDA_HEADER_DIR}"
 GTEST_HEADER_DIR="${GTEST_HEADER_DIR:="${ROOT_BUILD_DIR}/third_party/gtest/src/googletest/include"}"
 CLANG_FORMAT=${CLANG_FORMAT:="clang-format"}
 if [[ "${VERBOSE}" == 1 ]]; then
-    echo "the current setting is "
+    echo "#####################"
+    echo "# Enviroment Setting:"
     echo "CURRENT_DIR ${CURRENT_DIR}"
     echo "SCRIPT_DIR ${SCRIPT_DIR}"
     echo "ROOT_DIR ${ROOT_DIR}"
     echo "ROOT_BUILD_DIR ${ROOT_BUILD_DIR}"
     echo "GTEST_HEADER_DIR ${GTEST_HEADER_DIR}"
     echo "CUDA_HEADER_DIR ${CUDA_HEADER_DIR}"
-    echo "ROOT_BUILD_DIR ${ROOT_BUILD_DIR}"
     echo "CLANG_FORMAT ${CLANG_FORMAT}"
+    echo "#####################"
 fi
 if [[ "${CUDA_HEADER_DIR}" == "" ]]; then
-    echo "Please set the CUDA_HEADER_DIR in environment"
+    echo "Please set the environment variable CUDA_HEADER_DIR"
     exit 1
 fi
 # move to working_directory
@@ -44,24 +52,16 @@ KERNEL_SYNTAX_START="<<<"
 KERNEL_SYNTAX_END=">>>"
 DEVICE_CODE_SYNTAX="#include \"(common.*)\""
 FUNCTION_END=");"
-SPECIAL_SUFFIX="_AUTOHOSTFUNC"
 EXTRACT_KERNEL="false"
 GLOBAL_FILE="global_kernel"
-CONFIG_SELECTION_SUFFIX="_CONFIG"
 
-check_closed() {
-    local str="$1"
-    str="${str//->}"
-    str_start="${str//[^(<\[]}"
-    str_end="${str//[^>)\]]}"
-    if [[ "${#str_start}" -eq "${#str_end}" ]]; then
-        echo "true"
-    else
-        echo "false"
-    fi
-}
-
-convert_syntax() {
+# Converts a CUDA kernel call to the DPC++ equivalent. Also takes care of DPC++ queue manipulation if needed.
+convert_cuda_kernel_call() {
+    # the syntax_regex turn the string into the following
+    # [1]: kernel name
+    # [2]: template parameters if it exists. if template contain template struct, it will fail.
+    # [3]: CUDA parameters
+    # [4]: the rest of the function call (parameters, ...)
     local syntax_regex="([^<>]*)(<[^<>]*>)?<<<(.*)>>>(.*)"
     local str="$1"
     str=$(echo "${str}" | sed -E 's/ +/ /g')
@@ -70,61 +70,45 @@ convert_syntax() {
     local num=0
     local var=""
     if [[ "${str}" =~ ${syntax_regex} ]]; then
-        content="${BASH_REMATCH[3]}"
-        # echo "// content ${content}"
-        IFS=',' read -ra par <<< "$content"
-        for variable in "${par[@]}"; do
-            if [ -n "${temp}" ]; then
-                temp="${temp},"
-            fi
-            temp="${temp}${variable}"
-            # echo "// temp ${temp}"
-            is_closed=$(check_closed "$temp")
-            if [[ "${is_closed}" = "true" ]]; then
-                # echo "// temp ${temp}"
-                num=$((num+1))
-                # the last stream is zero, use the queue from exec
-                if [[ "${num}" -eq 4 ]] && [[ "${temp// }" -eq 0 ]]; then
-                    num=$((num-1))
+        local kernel_name="${BASH_REMATCH[1]}"
+        local template_param="${BASH_REMATCH[2]}"
+        local cuda_arg="${BASH_REMATCH[3]}"
+        local arg="${BASH_REMATCH[4]}"
+        temp=${cuda_arg//\[*\]/}
+        temp=${temp//\(*\)/}
+        temp=${temp//<*>/}
+        temp=${temp//[^,]}
+        num=$((${#temp} + 1))
+        if [[ ${num} -eq 4 ]]; then
+            local split_regex="(.*),([^,]*)$"
+            # avoid overwriting the BASH_REMATCH
+            if [[ "${cuda_arg}" =~ ${split_regex} ]]; then
+                var="${BASH_REMATCH[1]}"
+                if [[ "${BASH_REMATCH[2]}" -eq 0 ]]; then
+                    var="${var}, GET_QUEUE"
                 else
-                    if [ -n "${var}" ]; then
-                        var="${var},"
-                    fi
-                    var="${var}${temp}"
+                    var="${var}, Error"
                 fi
-                temp=""
+            else
+                var="Error"
             fi
-        done
-        # echo "// var ${var}"
-        # echo "var ${var}"
-        if [[ "${num}" -lt 2 ]]; then
+        elif [[ "${num}" -lt 2 ]]; then
             var="Error"
-        else
-            if [[ ${num} -eq 2 ]]; then
-                var="${var}, 0"
-                num=$((num+1))
-            fi
-            if [[ ${num} -eq 3 ]]; then
-                # var="${var}, exec->get_queue()"
-                var="${var}, GET_QUEUE"
-                num=$((num+1))
-            fi
+        elif [[ ${num} -eq 2 ]]; then
+            var="${cuda_arg}, 0, GET_QUEUE"
+        elif [[ ${num} -eq 3 ]]; then
+            var="${cuda_arg}, GET_QUEUE"
         fi
         local suffix=""
-        local function_name=$(echo "${BASH_REMATCH[1]}" | sed -E 's/(.*::)| //g')
-        # echo "// functioname ${function_name}"
-        MAP_FILE="map_list"
+        local function_name=$(echo "${kernel_name}" | sed -E 's/(.*::)| //g')
         suffix=$(cat "${MAP_FILE}" | sed -nE "s/${function_name} -> ${function_name}(.*)/\1/p")
-        suffix_matches=$(echo "${suffix}" | wc -l)
+        local suffix_matches=$(echo "${suffix}" | wc -l)
         if [[ "${suffix_matches}" -gt 1 ]]; then
             echo "static_assert(false, \"Has ${suffix_matches} matches for ${function_name}\");"
             suffix=$(echo "${suffix}" | head -1)
         fi
 
-        if [[ "${suffix}" = *"${CONFIG_SELECTION_SUFFIX}"* ]]; then
-            var="${var}, exec"
-        fi
-        local result="${BASH_REMATCH[1]}${suffix}${BASH_REMATCH[2]}(${var})${BASH_REMATCH[4]}"
+        local result="${kernel_name}${suffix}${template_param}(${var})${arg}"
         result=$(echo "${result}" | sed -E 's/\) *\(+/,/g;s/, *\)/\)/g')
         if [[ "${suffix}" = "" ]]; then
             # if the function does not exist in the file, comment it.
@@ -138,12 +122,11 @@ convert_syntax() {
 
 convert_regex_allowed() {
     local str="$1"
-    str=$(echo "${str}" | sed -E 's/(\/|\(|\)|\.)/\\\1/g')
+    str=$(echo "${str}" | sed -E 's~([/().])~\\\1~g')
     echo "$str"
 }
 
-# Transfer header file to the correct one
-# common -> dpcpp_common
+# Transfer header file to the correct location
 # cuda -> dpcpp
 # add sycl header
 
@@ -152,8 +135,8 @@ filename="${ROOT_DIR}/$input"
 if [[ "${VERBOSE}" == 1 ]]; then
     echo "Porting file ${filename}"
 fi
-# check file exists
 
+# check file exists
 if [ ! -f "${filename}" ]; then
     echo "${filename} does not exist"
     exit 1
@@ -163,7 +146,6 @@ fi
 temp=""
 IN_SYNTAX="false"
 DEVICE_FILE=""
-GINKGO_LICENSE_BEACON="******************************<GINKGO LICENSE>******************************"
 UNFORMAT_FILE="unformat.cpp"
 FORMAT_FILE="format.cpp"
 cp "${filename}" "${UNFORMAT_FILE}"
@@ -171,24 +153,27 @@ OUTPUT_FILE="source.cpp"
 EMBED_FILE="embed.cu"
 EMBED_HOST_FILE="embed_host.cu"
 if [[ "${VERBOSE}" == 1 ]]; then
+    echo "###################################"
+    echo "# Some generated file for debugging"
     echo "the original file ${UNFORMAT_FILE}"
     echo "the formatted file ${FORMAT_FILE}"
     echo "collect common/*.inc in file ${EMBED_FILE}"
     echo "add autohost func in file ${EMBED_HOST_FILE}"
     echo "convert original CUDA call in file ${OUTPUT_FILE}, which is the file for dpct"
+    echo "###################################"
 fi
 rm "${OUTPUT_FILE}"
-# echo "#include \"trick/dim3.hpp\"" >> "${OUTPUT_FILE}"
 echo "#define GET_QUEUE 0" >> "${OUTPUT_FILE}"
 # add empty ginkgo license such that format_header recognize some header before header def macro
 echo "/*${GINKGO_LICENSE_BEACON}" >> "${OUTPUT_FILE}"
 echo "${GINKGO_LICENSE_BEACON}*/" >> "${OUTPUT_FILE}"
 rm "${GLOBAL_FILE}"
 rm "${EMBED_FILE}"
+
+# Embed shared kernels into ${EMBED_FILE}
 while IFS='' read -r line; do
     if [[ "${line}" =~ $DEVICE_CODE_SYNTAX ]]; then
         # hold the command to easy replace
-        echo "// ${line}" >> ${EMBED_FILE}
         device_file="${BASH_REMATCH[1]}"
         [ "${EXTRACT_KERNEL}" == "true" ] && echo "/**** ${device_file} - start ****/" >> "${EMBED_FILE}"
         cat "${ROOT_DIR}/${device_file}" >> "${EMBED_FILE}"
@@ -201,17 +186,15 @@ while IFS='' read -r line; do
         echo "${line}" >> "${EMBED_FILE}"
     fi
 done < "${UNFORMAT_FILE}"
+
+# Call clang-format for better formatting.
 ${CLANG_FORMAT} -style=file "${EMBED_FILE}" > "${FORMAT_FILE}"
+
+# Add an extra host function so that the converted DPC++ code will look like CUDA.
 ${SCRIPT_DIR}/add_host_function.sh "${FORMAT_FILE}" > "${EMBED_HOST_FILE}"
 
-CONFIG_REGEX="constexpr *Config *([a-zA-Z0-9_]*) *= *([a-zA-Z][a-zA-Z_:,\(\) ]*)"
 while IFS='' read -r line; do
-    if [[ "${IN_SYNTAX}" = "false" ]] && [[ "${line}" =~ ${CONFIG_REGEX} ]]; then
-        config_name="${BASH_REMATCH[1]}"
-        config_content="${BASH_REMATCH[2]}"
-        echo "${line}" >> "${OUTPUT_FILE}"
-        echo "constexpr auto ${config_name}_list = ::gko::syn::value_list<Config, ${config_name}>();" >> "${OUTPUT_FILE}"
-    elif [[ "${line}" =~ ${KERNEL_SYNTAX_START} ]] || [[ "${IN_SYNTAX}" = "true" ]]; then
+    if [[ "${line}" =~ ${KERNEL_SYNTAX_START} ]] || [[ "${IN_SYNTAX}" = "true" ]]; then
         temp="${temp} ${line}"
         IN_SYNTAX="true"
         if [[ "${line}" =~ ${FUNCTION_END} ]]; then
@@ -221,11 +204,8 @@ while IFS='' read -r line; do
                 modified="${temp/\/\*KEEP\*\//}"
             else
                 # change <<<>>> to (grid, block, dynamic, queue)
-                modified=$(convert_syntax "$temp")
+                modified=$(convert_cuda_kernel_call "$temp")
             fi
-            # echo "temp -> convert"
-            # echo "$temp"
-            # echo "$modified"
             echo "${modified}" >> "${OUTPUT_FILE}"
             temp=""
         fi
@@ -247,17 +227,18 @@ replace_regex="${replace_regex};s/tiled_partition/tiled_partition_t/g"
 replace_regex="${replace_regex};s/thread::/thread_t::/g"
 replace_regex="${replace_regex};s/bitonic_sort/bitonic_sort_t/g"
 replace_regex="${replace_regex};s/reduction_array/reduction_array_t/g"
-replace_regex="${replace_regex};s/cuda\/components\/thread_ids.cuh/trick\/thread_ids.hpp/g"
-replace_regex="${replace_regex};s/cuda\/components\/cooperative_groups.cuh/trick\/cooperative_groups.hpp/g"
-replace_regex="${replace_regex};s/cuda\/components\/sorting.cuh/trick\/sorting.hpp/g"
-replace_regex="${replace_regex};s/cuda\/components\/reduction.cuh/trick\/reduction.hpp/g"
-replace_regex="${replace_regex};s/CUH_/DP_HPP_/g"
-replace_regex="${replace_regex};s/(.*template.*)(Config * [a-zA-Z0-9_]*) * = *[a-zA-Z0-9_:]*(\(+[^\)]*\))?/\1\2/g"
+replace_regex="${replace_regex};s|cuda/components/thread_ids\.cuh|trick/thread_ids.hpp|g"
+replace_regex="${replace_regex};s|cuda/components/cooperative_groups\.cuh|trick/cooperative_groups.hpp|g"
+replace_regex="${replace_regex};s|cuda/components/sorting\.cuh|trick/sorting.hpp|g"
+replace_regex="${replace_regex};s|cuda/components/reduction\.cuh|trick/reduction.hpp|g"
+replace_regex="${replace_regex};s|CUH_|DP_HPP_|g"
 # keep using original xxx.sync(); ->xxx;//.sync();
-replace_regex="${replace_regex};s/(\.sync\(\);)/;\/\/.sync()/g"
+replace_regex="${replace_regex};s|(\.sync\(\);)|;//.sync()|g"
 # template macro(); lead std::length_error
-replace_regex="${replace_regex};s/(template GKO.*;)/\/\/ \1/g"
+replace_regex="${replace_regex};s|(template GKO.*;)|// \1|g"
 sed -i -E "${replace_regex}" "${OUTPUT_FILE}"
+
+# Add the trick dim3_t header if dim3 is used.
 if grep -Eq "dim3" ${OUTPUT_FILE}; then
     # Found
     sed -i '1 i#include "trick/dim3_t.hpp"' ${OUTPUT_FILE}
@@ -267,19 +248,23 @@ if grep -Eq "group::" ${OUTPUT_FILE}; then
     # Found
     sed -i '1 i#include "trick/cooperative_groups.hpp"' ${OUTPUT_FILE}
 fi
-# exit 0
 
 OUTPUT_FOLDER="output"
-if [[ "${VERBOSE}" == 1]]; then
+if [[ "${VERBOSE}" == 1 ]]; then
     echo "The dpct calling:"
-    echo "dpct --extra-arg=\"-std=c++14\" --extra-arg=\"-I ${ROOT_DIR}\" --extra-arg=\"-I ${ROOT_DIR}/include\" --extra-arg=\"-I ${ROOT_DIR}/dev_tools/oneapi\" --extra-arg=\"-I ${GTEST_HEADER_DIR}\" --cuda-include-path=\"${CUDA_HEADER_DIR}\" --format-range=none ${OUTPUT_FILE} --out-root=${OUTPUT_FOLDER}"
+    echo "dpct --extra-arg=\"-std=c++14\" --extra-arg=\"-I ${ROOT_DIR}\" --extra-arg=\"-I ${ROOT_DIR}/include\" --extra-arg=\"-I ${ROOT_BUILD_DIR}/include\" --extra-arg=\"-I ${ROOT_DIR}/dev_tools/oneapi\" --extra-arg=\"-I ${GTEST_HEADER_DIR}\" --cuda-include-path=\"${CUDA_HEADER_DIR}\" --format-range=none ${OUTPUT_FILE} --suppress-warnings=1049 --out-root=${OUTPUT_FOLDER}"
 fi
 
 # Delete output/source.cpp
 rm "${OUTPUT_FOLDER}/${OUTPUT_FILE}"
 rm "${OUTPUT_FOLDER}/${OUTPUT_FILE}.dp.cpp"
 
-dpct --extra-arg="-std=c++14" --extra-arg="-I ${ROOT_DIR}" --extra-arg="-I ${ROOT_DIR}/include" --extra-arg="-I ${ROOT_DIR}/dev_tools/oneapi" --extra-arg="-I ${GTEST_HEADER_DIR}" --cuda-include-path="${CUDA_HEADER_DIR}" --format-range=none ${OUTPUT_FILE} --out-root=${OUTPUT_FOLDER}
+# Call DPCT
+echo "# Call DPCT on the previosly generated file."
+echo "############################################"
+dpct --extra-arg="-std=c++14" --extra-arg="-I ${ROOT_DIR}" --extra-arg="-I ${ROOT_DIR}/include" --extra-arg="-I ${ROOT_BUILD_DIR}/include" --extra-arg="-I ${ROOT_DIR}/dev_tools/oneapi" --extra-arg="-I ${GTEST_HEADER_DIR}" --cuda-include-path="${CUDA_HEADER_DIR}" --format-range=none ${OUTPUT_FILE} --suppress-warnings=1049 --out-root=${OUTPUT_FOLDER}
+echo "############################################"
+echo "# The DPCT process is end"
 
 dpct_file=""
 if [ -f "${OUTPUT_FOLDER}/${OUTPUT_FILE}.dp.cpp" ]; then
@@ -296,55 +281,40 @@ if [[ "${VERBOSE}" == 1 ]]; then
     echo "the dpct result ${dpct_file}_bkp"
     echo "recover the temporary change in the file ${dpct_file}"
 fi
+
 # global reverse fix
-# dim3_t -> dim3
-# this_thread_block_t -> this_thread_block
-# tiled_partition_t -> tiled_partition
-# thread_id
-# cooperative_group
-# replace_regex="1 i\#include <CL/sycl.hpp>"
-# replace_regex="s/cuda/dpcpp/g"
-# replace_regex="${replace_regex};s/#include \\\"common/#include \\\"dpcpp_code/g"
 replace_regex="s/dim3_t/dim3/g"
-replace_regex="${replace_regex};s/trick\/dim3\.hpp/dpcpp\/base\/dim3\.dp\.hpp/g"
+replace_regex="${replace_regex};s|trick/dim3\.hpp|dpcpp/base/dim3.dp.hpp|g"
 replace_regex="${replace_regex};s/this_thread_block_t/this_thread_block/g"
 replace_regex="${replace_regex};s/thread_t::/thread::/g"
 replace_regex="${replace_regex};s/this_grid_t/this_grid/g"
 replace_regex="${replace_regex};s/bitonic_sort_t/bitonic_sort/g"
 replace_regex="${replace_regex};s/reduce_array_t/reduce_array/g"
-# beta 09 ()
-# replace_regex="${replace_regex};s/auto dpct_global_range = grid \* block;/auto local_range = block.reverse();/g"
 replace_regex="${replace_regex};s/auto dpct_local_range = block;//g"
 replace_regex="${replace_regex};s/sycl::nd_range<3>.*, *$/sycl_nd_range(grid, block), /g"
-# global_range="sycl::range<3>(dpct_global_range.get(2), dpct_global_range.get(1), dpct_global_range.get(0))"
-# global_range=$(convert_regex_allowed "${global_range}")
-# echo "global_range ${global_range}"
-# local_range="sycl::range<3>(dpct_local_range.get(2), dpct_local_range.get(1), dpct_local_range.get(0))"
-# local_range=$(convert_regex_allowed "${local_range}")
-# echo "local_range ${local_range}"
-# replace_regex="${replace_regex};s/${global_range}/global_range/g"
-# replace_regex="${replace_regex};s/${local_range}/local_range/g"
 replace_regex="${replace_regex};s/tiled_partition_t/tiled_partition/g"
-replace_regex="${replace_regex};s/trick\/thread_ids.hpp/dpcpp\/components\/thread_ids.dp.hpp/g"
-replace_regex="${replace_regex};s/trick\/cooperative_groups.hpp/dpcpp\/components\/cooperative_groups.dp.hpp/g"
-replace_regex="${replace_regex};s/trick\/sorting.hpp/dpcpp\/components\/sorting.dp.hpp/g"
-replace_regex="${replace_regex};s/trick\/reduction.hpp/dpcpp\/components\/reduction.dp.hpp/g"
+replace_regex="${replace_regex};s|trick/thread_ids.hpp|dpcpp/components/thread_ids.dp.hpp|g"
+replace_regex="${replace_regex};s|trick/cooperative_groups\.hpp|dpcpp/components/cooperative_groups.dp.hpp|g"
+replace_regex="${replace_regex};s|trick/sorting\.hpp|dpcpp/components/sorting.dp.hpp|g"
+replace_regex="${replace_regex};s|trick/reduction\.hpp|dpcpp/components/reduction.dp.hpp|g"
 replace_regex="${replace_regex};s/#define GET_QUEUE 0//g"
 replace_regex="${replace_regex};s/GET_QUEUE/exec->get_queue()/g"
 replace_regex="${replace_regex};s/cuda/dpcpp/g"
 replace_regex="${replace_regex};s/Cuda/Dpcpp/g"
 replace_regex="${replace_regex};s/CUDA/DPCPP/g"
-replace_regex="${replace_regex};s/\.cuh/\.dp\.hpp/g"
-replace_regex="${replace_regex};s/\.cu/\.dp\.cpp/g"
-replace_regex="${replace_regex};s/${SPECIAL_SUFFIX}//g"
-replace_regex="${replace_regex};s/#include <dpct\/dpct\.hpp>//g"
-replace_regex="${replace_regex};s/#include \"dpcpp\/base\/types\.hpp\"//g"
-replace_regex="${replace_regex};s/#include \"dpcpp\/test\/utils\.hpp\"/#include \"core\/test\/utils\.hpp\"/g"
+replace_regex="${replace_regex};s/(cusparse|cublas)_bindings/onemkl_bindings/g"
+replace_regex="${replace_regex};s/\.cuh/.dp.hpp/g"
+replace_regex="${replace_regex};s/\.cu/.dp.cpp/g"
+replace_regex="${replace_regex};s/${HOST_SUFFIX}//g"
+replace_regex="${replace_regex};s|#include <dpct/dpct\.hpp>||g"
+replace_regex="${replace_regex};s|#include \"dpcpp/base/types\.hpp\"||g"
+replace_regex="${replace_regex};s|#include \"dpcpp/test/utils\.hpp\"|#include \"core/test/utils.hpp\"|g"
 # remove as_dpcpp_type\(content\) -> content
 # If the content use some brackets, only allowed one nested bracket now.
-replace_regex="${replace_regex};s/as_dpcpp_type\((([^\(\)]*(\([^\(\)]*\))[^\(\)]*)*)\)/\1/g"
-replace_regex="${replace_regex};s/as_dpcpp_type\(([^\(\)]*)\)/\1/g"
-# dcpt can not convert idx in static_cast
+replace_regex="${replace_regex};s/as_dpcpp_type\((([^()]*(\([^()]*\))[^()]*)*)\)/\1/g"
+replace_regex="${replace_regex};s/as_dpcpp_type\(([^()]*)\)/\1/g"
+# dcpt can not convert idx in static_cast of nested template
+# It should be fixed already, but let's keep it
 replace_regex="${replace_regex};s/threadIdx\.x/item_ct1.get_local_id(2)/g"
 replace_regex="${replace_regex};s/threadIdx\.y/item_ct1.get_local_id(1)/g"
 replace_regex="${replace_regex};s/threadIdx\.z/item_ct1.get_local_id(0)/g"
@@ -363,23 +333,17 @@ replace_regex="${replace_regex};s/gridDim\.z/item_ct1.get_group_range(0)/g"
 # Workaround for abs
 replace_regex="${replace_regex};s/sycl::fabs/std::abs/g"
 # Remove unneed warning DPCT1049 - check the block size
-# replace_regex="${replace_regex};/\/\*$/,/DPCT1049/s/^(.*)$/\1-remove/g;/\/\*-remove/,/\*\/$/d"
+# It should be properly handled by dpct, but let's keep it without hurt
 replace_regex="${replace_regex};/\/\*$/{N;N;/ *\/\*\n *DPCT1049.*\n *\*\//d}"
 # Recover // template GKO_...
-replace_regex="${replace_regex};s/\/\/ (template GKO.*;)/\1/g"
+replace_regex="${replace_regex};s|// (template GKO.*;)|\1|g"
 # Recover // xxx;//.sync();->xxx.sync();
-replace_regex="${replace_regex};s/;\/\/\.sync\(\)/\.sync\(\);/g"
-replace_regex="${replace_regex};s/\/\/remove\/\///g"
+replace_regex="${replace_regex};s|;//\.sync\(\)|.sync();|g"
+replace_regex="${replace_regex};s|//remove//||g"
 
-quote="'"
-# echo "replace_regex ${quote}${replace_regex}${quote}"
 sed -i -E "${replace_regex}" "${dpct_file}"
 
-# auto dpct_global_range = grid * block;
-# auto dpct_local_range = block;
-# sycl::nd_range<3>(sycl::range<3>(dpct_global_range.get(2), dpct_global_range.get(1), dpct_global_range.get(0)), sycl::range<3>(dpct_local_range.get(2), dpct_local_range.get(1), dpct_local_range.get(0))),
-
-# check whether containing __dpct_inline__ or __dpct_align__
+# Include dpct.hpp if __dpct_inline__ or __dpct_align__ are found
 need_dpct=$(grep -Eq "__(dpct_align|dpct_inline)__" ${dpct_file})
 if grep -Eq "__(dpct_align|dpct_inline)__" ${dpct_file}; then
     # Found
@@ -388,15 +352,12 @@ fi
 
 # extract device_code
 if [ "${EXTRACT_KERNEL}" = "true" ]; then
-    IFS=';' read -ra individual_deivce <<< "${DEVICE_FILE}"
-    for variable in "${individual_deivce[@]}"; do
+    IFS=';' read -ra individual_device <<< "${DEVICE_FILE}"
+    for variable in "${individual_device[@]}"; do
         device_regex=$(convert_regex_allowed "${variable}")
-        # echo "device_regex ${device_regex}"
         dpct_device_path=$(echo "${variable}" | sed 's/common/dpcpp_code/g')
-        dpct_device_file=$(echo "${dpct_device_path}" | sed 's/\//@/g')
+        dpct_device_file=$(echo "${dpct_device_path}" | sed 's|/|@|g')
         dpct_device_file="output/${dpct_device_file}"
-        # echo "dpct_device_path ${dpct_device_path}"
-        # echo "dpct_device_file ${dpct_device_file}"
         cat ${dpct_file} | sed -n "/${device_regex} - start/,/${device_regex} - end/p" | sed "1d;\$d" > ${dpct_device_file}
         sed -i "/${device_regex} - start/,/${device_regex} - end/d;s~// *#include \"${device_regex}\"~#include \"${dpct_device_path}\"~g" ${dpct_file}
         dpct_dir=$(dirname "${dpct_device_path}")
@@ -405,25 +366,9 @@ if [ "${EXTRACT_KERNEL}" = "true" ]; then
     done
 fi
 
-target_file=$(echo "${input}" | sed 's/cuda\//dpcpp\//g;s/\.cuh/\.dp\.hpp/g;s/\.cu/\.dp\.cpp/g')
+# Integrate the new DPC++ file into the correct place
+target_file=$(echo "${input}" | sed 's|cuda/|dpcpp/|g;s/\.cuh/\.dp\.hpp/g;s/\.cu/.dp.cpp/g')
 target_dir=$(dirname "${target_file}")
-
 mkdir -p "${ROOT_DIR}/${target_dir}"
 echo "cp ${dpct_file} ${ROOT_DIR}/${target_file}"
 cp "${dpct_file}" "${ROOT_DIR}/${target_file}"
-# sepecial reverse-fix (maybe not need)
-# for dpct_device_file
-# for variable in "${individual_deivce[@]}"; do
-#     # device_regex=$(convert_regex_allowed "${variable}")
-#     dpct_device_file=$(echo "${variable}" | sed 's/common/dpcpp_code/g;s/\//@/g')
-#     # some syntex reverse or some better cuda format
-#     #
-# done
-# for dpct_file
-# sed -i '1 i\#include <CL/sycl.hpp>;s/cuda/dpcpp/g;s/#include \"common/#include \"dpcpp_code/g' "${dpct_file}"
-# cublas/cusparse->onemkl?
-# reverse some header and change to DPCPP-related files
-
-# Move the file to correct place
-
-# apply format header()?
diff --git a/dev_tools/oneapi/shared.sh b/dev_tools/oneapi/shared.sh
new file mode 100644
index 00000000000..7caee4d0c61
--- /dev/null
+++ b/dev_tools/oneapi/shared.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Checks if $1 is self contained code, that is it does not have an open and
+# unclosed code portion (<>()[]), e.g. `my_struct->my_func(xxx,` should fail.
+check_closed() {
+    local str="$1"
+    # remove -> to avoid the confusion
+    str="${str//->}"
+    # Replace everything except begin or end characters, resp. (<[ and )>]
+    str_start="${str//[^(<\[]}"
+    str_end="${str//[^>)\]]}"
+    # Check that there are as many begin as end characters
+    if [[ "${#str_start}" -eq "${#str_end}" ]]; then
+        echo "true"
+    else
+        echo "false"
+    fi
+}
+
+GINKGO_LICENSE_BEACON="******************************<GINKGO LICENSE>******************************"
+HOST_SUFFIX="_AUTOHOSTFUNC"
+MAP_FILE="map_list"

From cda47e8e9a934dc58652af0e5b751fba622b42e3 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 7 Jul 2021 17:59:21 +0200
Subject: [PATCH 0914/1008] add workaround for MSVC but it disable the dpcpp
 support possiblility in MSVC

---
 accessor/range.hpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/accessor/range.hpp b/accessor/range.hpp
index 3dc8914ebfa..91123a8c617 100644
--- a/accessor/range.hpp
+++ b/accessor/range.hpp
@@ -100,9 +100,11 @@ class range {
         return accessor_(std::forward<DimensionTypes>(dimensions)...);
     }
 
-    range(const range &other) = default;
+    range(const range &) = default;
 
-    range(range &other) = default;
+#ifndef _WIN32
+    range(range &) = default;
+#endif
 
     /**
      * Returns the length of the specified dimension of the range.

From baaa4bbe34ca5ca2c502340be77420be425e9a33 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 7 Jul 2021 22:48:09 +0200
Subject: [PATCH 0915/1008] avoid templated constructor as cp/mv constuctor
 such that compiler use auto generated one.

---
 accessor/range.hpp | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/accessor/range.hpp b/accessor/range.hpp
index 91123a8c617..06ef8b1b480 100644
--- a/accessor/range.hpp
+++ b/accessor/range.hpp
@@ -69,7 +69,8 @@ class range {
      *
      * @param params  parameters forwarded to Accessor constructor.
      */
-    template <typename... AccessorParams>
+    template <typename... AccessorParams,
+              std::enable_if_t<(sizeof...(AccessorParams) > 1), bool> = true>
     GKO_ACC_ATTRIBUTES constexpr explicit range(AccessorParams &&... params)
         : accessor_{std::forward<AccessorParams>(params)...}
     {}
@@ -100,12 +101,6 @@ class range {
         return accessor_(std::forward<DimensionTypes>(dimensions)...);
     }
 
-    range(const range &) = default;
-
-#ifndef _WIN32
-    range(range &) = default;
-#endif
-
     /**
      * Returns the length of the specified dimension of the range.
      *

From 5934bfd59a76c29ccecc6f5a824df47f47cce798 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 13 Jul 2021 23:23:40 +0200
Subject: [PATCH 0916/1008] ell test with single_mode

---
 dpcpp/components/format_conversion.dp.hpp |  2 +-
 dpcpp/test/matrix/ell_kernels.cpp         | 40 +++++++++++------------
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/dpcpp/components/format_conversion.dp.hpp b/dpcpp/components/format_conversion.dp.hpp
index d92661bce83..b75b663bb99 100644
--- a/dpcpp/components/format_conversion.dp.hpp
+++ b/dpcpp/components/format_conversion.dp.hpp
@@ -37,8 +37,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <CL/sycl.hpp>
 
 
-#include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/types.hpp>
 
 
 #include "dpcpp/base/dim3.dp.hpp"
diff --git a/dpcpp/test/matrix/ell_kernels.cpp b/dpcpp/test/matrix/ell_kernels.cpp
index d876bb1b38a..081f46cff51 100644
--- a/dpcpp/test/matrix/ell_kernels.cpp
+++ b/dpcpp/test/matrix/ell_kernels.cpp
@@ -166,7 +166,7 @@ TEST_F(Ell, SimpleApplyIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -216,7 +216,7 @@ TEST_F(Ell, AdvancedApplyIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -266,7 +266,7 @@ TEST_F(Ell, SimpleApplyWithStrideIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -315,7 +315,7 @@ TEST_F(Ell, AdvancedApplyWithStrideIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -365,7 +365,7 @@ TEST_F(Ell, SimpleApplyWithStrideToDenseMatrixIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -415,7 +415,7 @@ TEST_F(Ell, AdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -465,7 +465,7 @@ TEST_F(Ell, SimpleApplyByAtomicIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -476,7 +476,7 @@ TEST_F(Ell, AdvancedByAtomicApplyIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -487,7 +487,7 @@ TEST_F(Ell, SimpleApplyByAtomicToDenseMatrixIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -498,7 +498,7 @@ TEST_F(Ell, AdvancedByAtomicToDenseMatrixApplyIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -509,7 +509,7 @@ TEST_F(Ell, SimpleApplyOnSmallMatrixIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -520,7 +520,7 @@ TEST_F(Ell, AdvancedApplyOnSmallMatrixToDenseMatrixIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -531,7 +531,7 @@ TEST_F(Ell, SimpleApplyOnSmallMatrixToDenseMatrixIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -542,7 +542,7 @@ TEST_F(Ell, AdvancedApplyOnSmallMatrixIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -559,7 +559,7 @@ TEST_F(Ell, ApplyToComplexIsEquivalentToRef)
     mtx->apply(complex_b.get(), complex_x.get());
     dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
 
-    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r<vtype>::value);
 }
 
 
@@ -576,7 +576,7 @@ TEST_F(Ell, AdvancedApplyToComplexIsEquivalentToRef)
     mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
     dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
 
-    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r<vtype>::value);
 }
 
 
@@ -590,7 +590,7 @@ TEST_F(Ell, ConvertToDenseIsEquivalentToRef)
     mtx->convert_to(dense_mtx.get());
     dmtx->convert_to(ddense_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), r<vtype>::value);
 }
 
 
@@ -604,7 +604,7 @@ TEST_F(Ell, ConvertToCsrIsEquivalentToRef)
     mtx->convert_to(csr_mtx.get());
     dmtx->convert_to(dcsr_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), r<vtype>::value);
 }
 
 
@@ -664,7 +664,7 @@ TEST_F(Ell, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, r<vtype>::value);
 }
 
 
@@ -675,7 +675,7 @@ TEST_F(Ell, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, r<vtype>::value);
 }
 
 

From 06438474313085a58dd68ff9e88d0ddf8938b890 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 19 Jul 2021 14:46:34 +0200
Subject: [PATCH 0917/1008] update script

---
 dev_tools/oneapi/convert_source.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev_tools/oneapi/convert_source.sh b/dev_tools/oneapi/convert_source.sh
index 0b0a505a6e1..bf59858ae56 100755
--- a/dev_tools/oneapi/convert_source.sh
+++ b/dev_tools/oneapi/convert_source.sh
@@ -26,7 +26,7 @@ ROOT_DIR="${ROOT_DIR:="${CURRENT_DIR}"}"
 BUILD_DIR="${BUILD_DIR:="build"}"
 ROOT_BUILD_DIR="${ROOT_BUILD_DIR:="${ROOT_DIR}/${BUILD_DIR}"}"
 CUDA_HEADER_DIR="${CUDA_HEADER_DIR}"
-GTEST_HEADER_DIR="${GTEST_HEADER_DIR:="${ROOT_BUILD_DIR}/third_party/gtest/src/googletest/include"}"
+GTEST_HEADER_DIR="${GTEST_HEADER_DIR:="${ROOT_BUILD_DIR}/_deps/googletest-src/googletest/include"}"
 CLANG_FORMAT=${CLANG_FORMAT:="clang-format"}
 if [[ "${VERBOSE}" == 1 ]]; then
     echo "#####################"

From 080f3b830ac293e9d69188ac080f8210017173b0 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 19 Jul 2021 17:24:38 +0200
Subject: [PATCH 0918/1008] delete debug comment, update description, refine
 format

Co-authored-by: Fritz Goebel <goebel.fritz@googlemail.com>
Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 accessor/range.hpp                            |  1 +
 common/matrix/ell_kernels.hpp.inc             |  9 +--
 dev_tools/oneapi/add_host_function.sh         | 54 +++++++++---------
 dev_tools/oneapi/convert_source.sh            |  3 +-
 .../fake_interface/cooperative_groups.cuh     | 25 +++++----
 .../trick/cooperative_groups.hpp              |  1 +
 .../oneapi/working_directory/trick/dim3_t.hpp | 12 ++--
 .../working_directory/trick/sorting.hpp       |  5 +-
 .../working_directory/trick/thread_ids.hpp    | 29 +++-------
 dpcpp/base/config.hpp                         |  7 ++-
 dpcpp/matrix/ell_kernels.dp.cpp               | 20 +++----
 dpcpp/test/matrix/ell_kernels.cpp             | 55 +++++++------------
 12 files changed, 98 insertions(+), 123 deletions(-)

diff --git a/accessor/range.hpp b/accessor/range.hpp
index 06ef8b1b480..1928485fc5c 100644
--- a/accessor/range.hpp
+++ b/accessor/range.hpp
@@ -33,6 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifndef GKO_ACCESSOR_RANGE_HPP_
 #define GKO_ACCESSOR_RANGE_HPP_
 
+
 #include <utility>
 
 
diff --git a/common/matrix/ell_kernels.hpp.inc b/common/matrix/ell_kernels.hpp.inc
index 399dd5070ac..f1845111eec 100644
--- a/common/matrix/ell_kernels.hpp.inc
+++ b/common/matrix/ell_kernels.hpp.inc
@@ -131,11 +131,12 @@ __global__ __launch_bounds__(default_block_size) void spmv(
 {
     const OutputValueType alpha_val = alpha(0);
     const OutputValueType beta_val = beta[0];
-    // Because the atomic operation changes the values of c during computation,
-    // it can not do the right alpha * a * b + beta * c operation.
-    // Thus, the cuda kernel only computes alpha * a * b when it uses atomic
-    // operation.
     if (atomic) {
+        // Because the atomic operation changes the values of c during
+        // computation, it can not directly do alpha * a * b + beta * c
+        // operation. The beta * c needs to be done before calling this kernel.
+        // Then, this kernel only adds alpha * a * b when it uses atomic
+        // operation.
         spmv_kernel<num_thread_per_worker, atomic>(
             num_rows, num_worker_per_row, val, col, stride,
             num_stored_elements_per_row, b, c, c_stride,
diff --git a/dev_tools/oneapi/add_host_function.sh b/dev_tools/oneapi/add_host_function.sh
index c5ac8bbaa6c..86c84d84762 100755
--- a/dev_tools/oneapi/add_host_function.sh
+++ b/dev_tools/oneapi/add_host_function.sh
@@ -1,4 +1,25 @@
 #!/bin/bash
+
+# add_host_function adds a host function to wrap the cuda kernel call with template and parameter configuration.
+#
+# For example
+# ````
+# template <int config = 1, int info, typename ValueType>
+# __global__ kernel(ValueType a) {...}
+# ```
+# add_host_function will add another host function with the same template and calling the cuda call
+# ```
+# template <int config = 1, int info, typename ValueType>
+# void kernel_AUTOHOSTFUNC(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory, cudaStream_t stream, ValueType a) {
+#     /*KEEP*/kernel<config, info><<<grid, block, dynamic_shared_memory, stream>>>(a);
+# }
+# ```
+# _AUTOHOSTFUNC and /*KEEP*/ is internal step and they are removed in the end.
+# It will use the same template as original cuda call and pust the kernel args into input args.
+# Note. This script does not translate original cuda kernel call to corresponding call.
+#       convert_source.sh will handle it later.
+
+
 SCRIPT_DIR="$( dirname "${BASH_SOURCE[0]}" )"
 source "${SCRIPT_DIR}/shared.sh"
 
@@ -81,7 +102,8 @@ while IFS='' read -r line || [ -n "$line" ]; do
     else
         echo "${line}"
     fi
-    # echo "Handle___ ${line}"
+
+    # handle comments
     if [[ "$line" =~ ${START_BLOCK_REX} ]] || [[ "${IN_BLOCK}" -gt 0 ]]; then
         if [[ "$line" =~ ${START_BLOCK_REX} ]]; then
             IN_BLOCK=$((IN_BLOCK+1))
@@ -89,20 +111,14 @@ while IFS='' read -r line || [ -n "$line" ]; do
         if [[ "$line" =~ ${END_BLOCK_REX} ]]; then
             IN_BLOCK=$((IN_BLOCK-1))
         fi
-        # echo ""
-        # echo "IN BLOCK ${IN_BLOCK}"
         # output to new file
         continue
     fi
-    # echo "Handle ${line}"
-    # handle comments
+    # handle functions
     if [[ "${line}" =~ $FUNCTION_START ]] || [[ $DURING_FUNCNAME = "true" ]]; then
-        # echo "line ${line}"
-        # echo "${FUNCTION_NAME_END}"
         DURING_FUNCNAME="true"
         FUNCTION_HANDLE="${FUNCTION_HANDLE} $line"
         if [[ "${line}" =~ ${FUNCTION_NAME_END} ]]; then
-            # echo "end"
             DURING_FUNCNAME="false"
         fi
         if [[ "${line}" =~ ${SCOPE_START} ]]; then
@@ -114,7 +130,7 @@ while IFS='' read -r line || [ -n "$line" ]; do
         # output to new file
         continue
     fi
-    # echo "Handle ${line}"
+
     if [ -n "${FUNCTION_HANDLE}" ] && [[ ${DURING_FUNCNAME} = "false" ]]; then
         if [[ "${line}" =~ ${SCOPE_START} ]]; then
             IN_FUNC=$((IN_FUNC+1))
@@ -122,20 +138,14 @@ while IFS='' read -r line || [ -n "$line" ]; do
         if [[ "${line}" =~ ${SCOPE_END} ]]; then
             IN_FUNC=$((IN_FUNC-1))
         fi
-        # echo "IN FUNC ${IN_FUNC}"
 
         # make sure the function is end
         if [[ "${IN_FUNC}" -eq 0 ]]; then
-            # echo "check ${FUNCTION_HANDLE}"
 
             if [[ "${FUNCTION_HANDLE}" =~ $CHECK_GLOBAL_KEYWORD ]]; then
                 echo ""
-                # echo "${FUNCTION_HANDLE}"
                 # remove additional space
                 FUNCTION_HANDLE=$(echo "${FUNCTION_HANDLE}" | sed -E 's/ +/ /g;')
-                # echo "->"
-                # echo "${FUNCTION_HANDLE}"
-                # echo "->"
 
                 if [[ "${FUNCTION_HANDLE}" =~ $ANAYSIS_FUNC ]]; then
                     TEMPLATE="${BASH_REMATCH[1]}"
@@ -148,23 +158,11 @@ while IFS='' read -r line || [ -n "$line" ]; do
                     if [ -n "${TEMPLATE_INPUT}" ]; then
                         TEMPLATE_INPUT="<${TEMPLATE_INPUT}>"
                     fi
-                    echo "${TEMPLATE} void ${NAME}${HOST_SUFFIX} (dim3 grid, dim3 block, size_t dynamic_shared_memory, cudaStream_t stream, ${VARIABLE}) {
+                    echo "${TEMPLATE} void ${NAME}${HOST_SUFFIX} (dim3 grid, dim3 block, gko::size_type dynamic_shared_memory, cudaStream_t stream, ${VARIABLE}) {
                         /*KEEP*/${NAME}${TEMPLATE_INPUT}<<<grid, block, dynamic_shared_memory, stream>>>(${VAR_INPUT});
                         }"
                     echo "${NAME} -> ${NAME}${HOST_SUFFIX}" >> ${MAP_FILE}
                 fi
-                # echo ""
-                # check the property
-                # extract template
-                # maybe remove any [[ ]]
-                # extract function name
-                # extract function variables
-
-                # check Config
-                # extract bool, int, size_type, typename before Config
-                # add one function like original one
-                # add selection_config macro
-                # need to keep the map
             fi
             FUNCTION_HANDLE=""
         fi
diff --git a/dev_tools/oneapi/convert_source.sh b/dev_tools/oneapi/convert_source.sh
index bf59858ae56..5d2bd757210 100755
--- a/dev_tools/oneapi/convert_source.sh
+++ b/dev_tools/oneapi/convert_source.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+# convert_source.sh converts cuda (and c++ code) to dpcpp code with ginkgo design.
+
 # Usage:
 # EnvironementSet ./dev_tools/oneapi/convert_source.sh <the_file_in_cuda>
 # <the_file_in_cuda> can be .hpp/.cpp/.cu/.cuh
@@ -65,7 +67,6 @@ convert_cuda_kernel_call() {
     local syntax_regex="([^<>]*)(<[^<>]*>)?<<<(.*)>>>(.*)"
     local str="$1"
     str=$(echo "${str}" | sed -E 's/ +/ /g')
-    # echo "str ${str}"
     local temp=""
     local num=0
     local var=""
diff --git a/dev_tools/oneapi/fake_interface/cooperative_groups.cuh b/dev_tools/oneapi/fake_interface/cooperative_groups.cuh
index 05d9b936fbe..752197e63c2 100644
--- a/dev_tools/oneapi/fake_interface/cooperative_groups.cuh
+++ b/dev_tools/oneapi/fake_interface/cooperative_groups.cuh
@@ -30,15 +30,19 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_FAKE_INTERFACE_COOPERATIVE_GROUPS_CUH_
-#define GKO_FAKE_INTERFACE_COOPERATIVE_GROUPS_CUH_
+#ifndef FAKE_INTERFACE_COOPERATIVE_GROUPS_CUH_
+#define FAKE_INTERFACE_COOPERATIVE_GROUPS_CUH_
+
 
 #include <cuda/components/cooperative_groups.cuh>
 
+
 namespace gko {
-    namespace kernels{
-        namespace cuda{
-            namespace group{
+namespace kernels {
+namespace cuda {
+namespace group {
+
+
 __device__ __forceinline__ grid_group this_grid_i() { return this_grid(); }
 
 __device__ auto this_thread_block_i() { return this_thread_block(); }
@@ -49,9 +53,10 @@ __device__ __forceinline__ auto tiled_partition_i(const Group &g)
     return ::gko::kernels::cuda::group::tiled_partition<Size>(g);
 }
 
-            }
-        }
-    }
-}
 
-#endif // GKO_FAKE_INTERFACE_COOPERATIVE_GROUPS_CUH_
+}  // namespace group
+}  // namespace cuda
+}  // namespace kernels
+}  // namespace gko
+
+#endif  // FAKE_INTERFACE_COOPERATIVE_GROUPS_CUH_
diff --git a/dev_tools/oneapi/working_directory/trick/cooperative_groups.hpp b/dev_tools/oneapi/working_directory/trick/cooperative_groups.hpp
index d7a8760f2ea..74f2a6d1656 100644
--- a/dev_tools/oneapi/working_directory/trick/cooperative_groups.hpp
+++ b/dev_tools/oneapi/working_directory/trick/cooperative_groups.hpp
@@ -36,6 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <type_traits>
 
+
 #include <fake_interface/cooperative_groups.cuh>
 
 
diff --git a/dev_tools/oneapi/working_directory/trick/dim3_t.hpp b/dev_tools/oneapi/working_directory/trick/dim3_t.hpp
index 78c04f8e356..b77d597d768 100644
--- a/dev_tools/oneapi/working_directory/trick/dim3_t.hpp
+++ b/dev_tools/oneapi/working_directory/trick/dim3_t.hpp
@@ -37,12 +37,10 @@ struct dim3_t {
     unsigned int x;
     unsigned int y;
     unsigned int z;
-    dim3_t(unsigned int xval, unsigned int yval = 1, unsigned int zval = 1)
-        : x(xval), y(yval), z(zval)
-    {}
-    operator dim3() {
-        return dim3{x, y, z};
-    }
+
+    dim3_t(unsigned int xval, unsigned int yval = 1, unsigned int zval = 1) : x(xval), y(yval), z(zval) {}
+
+    operator dim3() { return dim3{x, y, z}; }
 };
 
-#endif // TRICK_DIM3_T_HPP_
+#endif  // TRICK_DIM3_T_HPP_
diff --git a/dev_tools/oneapi/working_directory/trick/sorting.hpp b/dev_tools/oneapi/working_directory/trick/sorting.hpp
index ceccbac71e6..2d5103a4d9d 100644
--- a/dev_tools/oneapi/working_directory/trick/sorting.hpp
+++ b/dev_tools/oneapi/working_directory/trick/sorting.hpp
@@ -37,7 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <type_traits>
 
 
-#include <cuda/components/sorting.cuh>
+#include "cuda/components/sorting.cuh"
 
 
 namespace gko {
@@ -46,8 +46,7 @@ namespace cuda {
 
 
 template <int num_elements, int num_local, typename ValueType>
-__forceinline__ __device__ void bitonic_sort_t(ValueType *local_elements,
-                                             ValueType *shared_elements)
+__forceinline__ __device__ void bitonic_sort_t(ValueType *local_elements, ValueType *shared_elements)
 {
     auto tidx = threadIdx.x;
     bitonic_sort<num_elements, num_local>(local_elements, shared_elements);
diff --git a/dev_tools/oneapi/working_directory/trick/thread_ids.hpp b/dev_tools/oneapi/working_directory/trick/thread_ids.hpp
index 791d93105a4..9d946c04eff 100644
--- a/dev_tools/oneapi/working_directory/trick/thread_ids.hpp
+++ b/dev_tools/oneapi/working_directory/trick/thread_ids.hpp
@@ -58,10 +58,7 @@ namespace thread_t {
  * @note Assumes that grid dimensions are in standard format:
  *       `(block_group_size, first_grid_dimension, second grid_dimension)`
  */
-__device__ __forceinline__ size_type get_block_group_id()
-{
-    return static_cast<size_type>(blockIdx.z) * gridDim.y + blockIdx.y;
-}
+__device__ __forceinline__ size_type get_block_group_id() { return static_cast<size_type>(blockIdx.z) * gridDim.y + blockIdx.y; }
 
 /**
  * @internal
@@ -73,10 +70,7 @@ __device__ __forceinline__ size_type get_block_group_id()
  * @note Assumes that grid dimensions are in standard format:
  *       `(block_group_size, first_grid_dimension, second grid_dimension)`
  */
-__device__ __forceinline__ size_type get_block_id()
-{
-    return get_block_group_id() * gridDim.x + blockIdx.x;
-}
+__device__ __forceinline__ size_type get_block_id() { return get_block_group_id() * gridDim.x + blockIdx.x; }
 
 
 /**
@@ -92,10 +86,7 @@ __device__ __forceinline__ size_type get_block_id()
  *       `(subwarp_size, config::warp_size / subwarp_size, block_size /
  *         config::warp_size)`
  */
-__device__ __forceinline__ size_type get_local_warp_id()
-{
-    return static_cast<size_type>(threadIdx.z);
-}
+__device__ __forceinline__ size_type get_local_warp_id() { return static_cast<size_type>(threadIdx.z); }
 
 
 /**
@@ -208,8 +199,7 @@ __device__ __forceinline__ size_type get_subwarp_id()
 template <int subwarp_size, int warps_per_block>
 __device__ __forceinline__ size_type get_thread_id()
 {
-    return get_subwarp_id<subwarp_size, warps_per_block>() * subwarp_size +
-           threadIdx.x;
+    return get_subwarp_id<subwarp_size, warps_per_block>() * subwarp_size + threadIdx.x;
 }
 
 
@@ -262,10 +252,8 @@ __device__ __forceinline__ IndexType get_thread_num_flat()
 template <int subwarp_size, typename IndexType = size_type>
 __device__ __forceinline__ IndexType get_subwarp_id_flat()
 {
-    static_assert(!(subwarp_size & (subwarp_size - 1)),
-                  "subwarp_size must be a power of two");
-    return threadIdx.x / subwarp_size +
-           static_cast<IndexType>(blockDim.x / subwarp_size) * blockIdx.x;
+    static_assert(!(subwarp_size & (subwarp_size - 1)), "subwarp_size must be a power of two");
+    return threadIdx.x / subwarp_size + static_cast<IndexType>(blockDim.x / subwarp_size) * blockIdx.x;
 }
 
 
@@ -284,13 +272,12 @@ __device__ __forceinline__ IndexType get_subwarp_id_flat()
 template <int subwarp_size, typename IndexType = size_type>
 __device__ __forceinline__ IndexType get_subwarp_num_flat()
 {
-    static_assert(!(subwarp_size & (subwarp_size - 1)),
-                  "subwarp_size must be a power of two");
+    static_assert(!(subwarp_size & (subwarp_size - 1)), "subwarp_size must be a power of two");
     return blockDim.x / subwarp_size * static_cast<IndexType>(gridDim.x);
 }
 
 
-}  // namespace thread
+}  // namespace thread_t
 }  // namespace cuda
 }  // namespace kernels
 }  // namespace gko
diff --git a/dpcpp/base/config.hpp b/dpcpp/base/config.hpp
index c5bb77e48b3..94e15508d64 100644
--- a/dpcpp/base/config.hpp
+++ b/dpcpp/base/config.hpp
@@ -49,6 +49,11 @@ struct config {
      */
     using lane_mask_type = uint64;
 
+    /**
+     * The number of threads within a Dpcpp subgroup.
+     */
+    static constexpr uint32 warp_size = 16;
+
     /**
      * The bitmask of the entire warp.
      */
@@ -61,7 +66,7 @@ struct config {
     static constexpr uint32 min_warps_per_block = 4;
 
     /**
-     * The maximal number of threads allowed in a CUDA warp.
+     * The default maximal number of threads allowed in DPCPP group
      */
     static constexpr uint32 max_block_size = 256;
 };
diff --git a/dpcpp/matrix/ell_kernels.dp.cpp b/dpcpp/matrix/ell_kernels.dp.cpp
index 2f27d543e19..951831de27c 100644
--- a/dpcpp/matrix/ell_kernels.dp.cpp
+++ b/dpcpp/matrix/ell_kernels.dp.cpp
@@ -106,7 +106,6 @@ constexpr int max_thread_per_worker = 32;
 using compiled_kernels = syn::value_list<int, 0, 8, 16, 32>;
 
 
-// #include "common/matrix/ell_kernels.hpp.inc"
 namespace kernel {
 namespace {
 
@@ -174,12 +173,8 @@ void spmv_kernel(
             atomic_add<atomic::local_space>(
                 &(*storage)[item_ct1.get_local_id(2)], temp);
         }
-        /*
-        DPCT1065:1: Consider replacing sycl::nd_item::barrier() with
-        sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
-        better performance, if there is no access to global memory.
-        */
-        item_ct1.barrier();
+
+        item_ct1.barrier(sycl::access::fence_space::local_space);
         if (runnable && idx_in_worker == 0) {
             const auto c_ind = x * c_stride + column_id;
             if (atomic) {
@@ -255,11 +250,12 @@ void spmv(
 {
     const OutputValueType alpha_val = alpha(0);
     const OutputValueType beta_val = beta[0];
-    // Because the atomic operation changes the values of c during computation,
-    // it can not do the right alpha * a * b + beta * c operation.
-    // Thus, the dpcpp kernel only computes alpha * a * b when it uses atomic
-    // operation.
     if (atomic) {
+        // Because the atomic operation changes the values of c during
+        // computation, it can not directly do alpha * a * b + beta * c
+        // operation. The beta * c needs to be done before calling this kernel.
+        // Then, this kernel only adds alpha * a * b when it uses atomic
+        // operation.
         spmv_kernel<num_thread_per_worker, atomic>(
             num_rows, num_worker_per_row, val, col, stride,
             num_stored_elements_per_row, b, c, c_stride,
@@ -585,7 +581,7 @@ std::array<int, 3> compute_thread_worker_and_atomicity(
 
     const auto nrows = a->get_size()[0];
     const auto ell_ncols = a->get_num_stored_elements_per_row();
-    // TODO: num_threads_per_core should be tuned for AMD gpu
+    // TODO: num_threads_per_core should be tuned for Dpcpp
     const auto nwarps = 16 * num_threads_per_core;
 
     // Use multithreads to perform the reduction on each row when the matrix is
diff --git a/dpcpp/test/matrix/ell_kernels.cpp b/dpcpp/test/matrix/ell_kernels.cpp
index 081f46cff51..37814e946c4 100644
--- a/dpcpp/test/matrix/ell_kernels.cpp
+++ b/dpcpp/test/matrix/ell_kernels.cpp
@@ -49,6 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/matrix/ell_kernels.hpp"
 #include "core/test/utils.hpp"
+#include "dpcpp/test/utils.hpp"
 
 
 namespace {
@@ -170,11 +171,9 @@ TEST_F(Ell, SimpleApplyIsEquivalentToRef)
 }
 
 
-#if !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef1)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data();
 
     mtx->apply(y2.get(), expected2.get());
@@ -186,6 +185,7 @@ TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef1)
 
 TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef2)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data();
 
     mtx->apply(y2.get(), expected.get());
@@ -197,6 +197,7 @@ TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef2)
 
 TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef3)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data();
 
     mtx->apply(y.get(), expected2.get());
@@ -206,9 +207,6 @@ TEST_F(Ell, MixedSimpleApplyIsEquivalentToRef3)
 }
 
 
-#endif  // !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Ell, AdvancedApplyIsEquivalentToRef)
 {
     set_up_apply_data();
@@ -220,11 +218,9 @@ TEST_F(Ell, AdvancedApplyIsEquivalentToRef)
 }
 
 
-#if !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef1)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data();
 
     mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
@@ -236,6 +232,7 @@ TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef1)
 
 TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef2)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data();
 
     mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
@@ -247,6 +244,7 @@ TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef2)
 
 TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef3)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data();
 
     mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
@@ -256,9 +254,6 @@ TEST_F(Ell, MixedAdvancedApplyIsEquivalentToRef3)
 }
 
 
-#endif  // !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Ell, SimpleApplyWithStrideIsEquivalentToRef)
 {
     set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
@@ -270,11 +265,9 @@ TEST_F(Ell, SimpleApplyWithStrideIsEquivalentToRef)
 }
 
 
-#if !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef1)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(y2.get(), expected2.get());
@@ -286,6 +279,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef1)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef2)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(y2.get(), expected.get());
@@ -297,6 +291,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef2)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef3)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(y.get(), expected2.get());
@@ -306,9 +301,6 @@ TEST_F(Ell, MixedSimpleApplyWithStrideIsEquivalentToRef3)
 }
 
 
-#endif  // !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Ell, AdvancedApplyWithStrideIsEquivalentToRef)
 {
     set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
@@ -319,11 +311,9 @@ TEST_F(Ell, AdvancedApplyWithStrideIsEquivalentToRef)
 }
 
 
-#if !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef1)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
@@ -335,6 +325,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef1)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef2)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
@@ -346,6 +337,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef2)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef3)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data(size[0], size[1], 1, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
@@ -355,9 +347,6 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideIsEquivalentToRef3)
 }
 
 
-#endif  // !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Ell, SimpleApplyWithStrideToDenseMatrixIsEquivalentToRef)
 {
     set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
@@ -369,11 +358,9 @@ TEST_F(Ell, SimpleApplyWithStrideToDenseMatrixIsEquivalentToRef)
 }
 
 
-#if !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef1)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(y2.get(), expected2.get());
@@ -385,6 +372,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef1)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef2)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(y2.get(), expected.get());
@@ -396,6 +384,7 @@ TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef2)
 
 TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef3)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(y.get(), expected2.get());
@@ -405,9 +394,6 @@ TEST_F(Ell, MixedSimpleApplyWithStrideToDenseMatrixIsEquivalentToRef3)
 }
 
 
-#endif  // !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Ell, AdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef)
 {
     set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
@@ -419,11 +405,9 @@ TEST_F(Ell, AdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef)
 }
 
 
-#if !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef1)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha2.get(), y2.get(), beta2.get(), expected2.get());
@@ -435,6 +419,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef1)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef2)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha2.get(), y2.get(), beta.get(), expected.get());
@@ -446,6 +431,7 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef2)
 
 TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef3)
 {
+    SKIP_IF_SINGLE_MODE;
     set_up_apply_data(size[0], size[1], 3, num_els_rowwise, ell_stride);
 
     mtx->apply(alpha.get(), y.get(), beta2.get(), expected2.get());
@@ -455,9 +441,6 @@ TEST_F(Ell, MixedAdvancedApplyWithStrideToDenseMatrixIsEquivalentToRef3)
 }
 
 
-#endif  // !GINKGO_DPCPP_SINGLE_MODE
-
-
 TEST_F(Ell, SimpleApplyByAtomicIsEquivalentToRef)
 {
     set_up_apply_data(10, 10000);

From e76167fed1af63bae1f0fc809e0f1d951d783150 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 21 Jul 2021 15:26:26 +0200
Subject: [PATCH 0919/1008] auto replace shared_memory and update ell

---
 dev_tools/oneapi/convert_source.sh |  2 +
 dpcpp/matrix/ell_kernels.dp.cpp    | 60 +++++++++++++-----------------
 2 files changed, 28 insertions(+), 34 deletions(-)

diff --git a/dev_tools/oneapi/convert_source.sh b/dev_tools/oneapi/convert_source.sh
index 5d2bd757210..b0ed62252f7 100755
--- a/dev_tools/oneapi/convert_source.sh
+++ b/dev_tools/oneapi/convert_source.sh
@@ -293,6 +293,8 @@ replace_regex="${replace_regex};s/bitonic_sort_t/bitonic_sort/g"
 replace_regex="${replace_regex};s/reduce_array_t/reduce_array/g"
 replace_regex="${replace_regex};s/auto dpct_local_range = block;//g"
 replace_regex="${replace_regex};s/sycl::nd_range<3>.*, *$/sycl_nd_range(grid, block), /g"
+# do not use c-style casting. use .get() to get the pointer.
+replace_regex="${replace_regex};s/\(UninitializedArray<.*> \*\)(.*\.get_pointer\(\))/\1.get()/g"
 replace_regex="${replace_regex};s/tiled_partition_t/tiled_partition/g"
 replace_regex="${replace_regex};s|trick/thread_ids.hpp|dpcpp/components/thread_ids.dp.hpp|g"
 replace_regex="${replace_regex};s|trick/cooperative_groups\.hpp|dpcpp/components/cooperative_groups.dp.hpp|g"
diff --git a/dpcpp/matrix/ell_kernels.dp.cpp b/dpcpp/matrix/ell_kernels.dp.cpp
index 951831de27c..76fdb26d376 100644
--- a/dpcpp/matrix/ell_kernels.dp.cpp
+++ b/dpcpp/matrix/ell_kernels.dp.cpp
@@ -151,12 +151,8 @@ void spmv_kernel(
         if (runnable && idx_in_worker == 0) {
             (*storage)[item_ct1.get_local_id(2)] = 0;
         }
-        /*
-        DPCT1065:0: Consider replacing sycl::nd_item::barrier() with
-        sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
-        better performance, if there is no access to global memory.
-        */
-        item_ct1.barrier();
+
+        item_ct1.barrier(sycl::access::fence_space::local_space);
         auto temp = zero<OutputValueType>();
         if (runnable) {
             for (size_type idx =
@@ -208,7 +204,7 @@ void spmv(
 
 template <int num_thread_per_worker, bool atomic = false, typename b_accessor,
           typename a_accessor, typename OutputValueType, typename IndexType>
-void spmv(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+void spmv(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
           sycl::queue *stream, const size_type num_rows,
           const int num_worker_per_row, acc::range<a_accessor> val,
           const IndexType *col, const size_type stride,
@@ -219,19 +215,16 @@ void spmv(dim3 grid, dim3 block, size_t dynamic_shared_memory,
         sycl::accessor<
             UninitializedArray<OutputValueType,
                                default_block_size / num_thread_per_worker>,
-            0, sycl::access::mode::read_write, sycl::access::target::local>
+            0, sycl::access_mode::read_write, sycl::access::target::local>
             storage_acc_ct1(cgh);
 
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                spmv<num_thread_per_worker, atomic>(
-                    num_rows, num_worker_per_row, val, col, stride,
-                    num_stored_elements_per_row, b, c, c_stride, item_ct1,
-                    (UninitializedArray<OutputValueType,
-                                        default_block_size /
-                                            num_thread_per_worker> *)
-                        storage_acc_ct1.get_pointer());
-            });
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             spmv<num_thread_per_worker, atomic>(
+                                 num_rows, num_worker_per_row, val, col, stride,
+                                 num_stored_elements_per_row, b, c, c_stride,
+                                 item_ct1, storage_acc_ct1.get_pointer().get());
+                         });
     });
 }
 
@@ -277,7 +270,7 @@ void spmv(
 
 template <int num_thread_per_worker, bool atomic = false, typename b_accessor,
           typename a_accessor, typename OutputValueType, typename IndexType>
-void spmv(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+void spmv(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
           sycl::queue *stream, const size_type num_rows,
           const int num_worker_per_row, acc::range<a_accessor> alpha,
           acc::range<a_accessor> val, const IndexType *col,
@@ -289,7 +282,7 @@ void spmv(dim3 grid, dim3 block, size_t dynamic_shared_memory,
         sycl::accessor<
             UninitializedArray<OutputValueType,
                                default_block_size / num_thread_per_worker>,
-            0, sycl::access::mode::read_write, sycl::access::target::local>
+            0, sycl::access_mode::read_write, sycl::access::target::local>
             storage_acc_ct1(cgh);
 
         cgh.parallel_for(
@@ -297,10 +290,7 @@ void spmv(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                 spmv<num_thread_per_worker, atomic>(
                     num_rows, num_worker_per_row, alpha, val, col, stride,
                     num_stored_elements_per_row, b, beta, c, c_stride, item_ct1,
-                    (UninitializedArray<OutputValueType,
-                                        default_block_size /
-                                            num_thread_per_worker> *)
-                        storage_acc_ct1.get_pointer());
+                    storage_acc_ct1.get_pointer().get());
             });
     });
 }
@@ -326,7 +316,8 @@ void initialize_zero_dense(size_type num_rows, size_type num_cols,
 }
 
 template <typename ValueType>
-void initialize_zero_dense(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+void initialize_zero_dense(dim3 grid, dim3 block,
+                           gko::size_type dynamic_shared_memory,
                            sycl::queue *stream, size_type num_rows,
                            size_type num_cols, size_type stride,
                            ValueType *result)
@@ -350,7 +341,7 @@ void fill_in_dense(size_type num_rows, size_type nnz, size_type source_stride,
 {
     const auto tidx = thread::get_thread_id_flat(item_ct1);
     if (tidx < num_rows) {
-        for (auto col = 0; col < nnz; col++) {
+        for (size_type col = 0; col < nnz; col++) {
             result[tidx * result_stride +
                    col_idxs[tidx + col * source_stride]] +=
                 values[tidx + col * source_stride];
@@ -359,7 +350,7 @@ void fill_in_dense(size_type num_rows, size_type nnz, size_type source_stride,
 }
 
 template <typename ValueType, typename IndexType>
-void fill_in_dense(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+void fill_in_dense(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
                    sycl::queue *stream, size_type num_rows, size_type nnz,
                    size_type source_stride, const IndexType *col_idxs,
                    const ValueType *values, size_type result_stride,
@@ -401,7 +392,8 @@ void count_nnz_per_row(size_type num_rows, size_type max_nnz_per_row,
 }
 
 template <typename ValueType, typename IndexType>
-void count_nnz_per_row(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+void count_nnz_per_row(dim3 grid, dim3 block,
+                       gko::size_type dynamic_shared_memory,
                        sycl::queue *stream, size_type num_rows,
                        size_type max_nnz_per_row, size_type stride,
                        const ValueType *values, IndexType *result)
@@ -429,7 +421,7 @@ void fill_in_csr(size_type num_rows, size_type max_nnz_per_row,
 
     if (tidx < num_rows) {
         auto write_to = result_row_ptrs[tidx];
-        for (auto i = 0; i < max_nnz_per_row; i++) {
+        for (size_type i = 0; i < max_nnz_per_row; i++) {
             const auto source_idx = tidx + stride * i;
             if (source_values[source_idx] != zero<ValueType>()) {
                 result_values[write_to] = source_values[source_idx];
@@ -441,7 +433,7 @@ void fill_in_csr(size_type num_rows, size_type max_nnz_per_row,
 }
 
 template <typename ValueType, typename IndexType>
-void fill_in_csr(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+void fill_in_csr(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
                  sycl::queue *stream, size_type num_rows,
                  size_type max_nnz_per_row, size_type stride,
                  const ValueType *source_values,
@@ -480,10 +472,10 @@ void extract_diagonal(size_type diag_size, size_type max_nnz_per_row,
 }
 
 template <typename ValueType, typename IndexType>
-void extract_diagonal(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                      sycl::queue *stream, size_type diag_size,
-                      size_type max_nnz_per_row, size_type orig_stride,
-                      const ValueType *orig_values,
+void extract_diagonal(dim3 grid, dim3 block,
+                      gko::size_type dynamic_shared_memory, sycl::queue *stream,
+                      size_type diag_size, size_type max_nnz_per_row,
+                      size_type orig_stride, const ValueType *orig_values,
                       const IndexType *orig_col_idxs, ValueType *diag)
 {
     stream->submit([&](sycl::handler &cgh) {

From 01f8c73289e1b41875a8f642e0aaa1176db16536 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 21 Jul 2021 23:33:44 +0200
Subject: [PATCH 0920/1008] alter. to avoid constructor taking range itself
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
---
 accessor/range.hpp | 33 +++++++++++++++++++++++++++++----
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/accessor/range.hpp b/accessor/range.hpp
index 1928485fc5c..ee3eb59c763 100644
--- a/accessor/range.hpp
+++ b/accessor/range.hpp
@@ -46,6 +46,25 @@ namespace acc {
 
 template <typename Accessor>
 class range {
+private:
+    /**
+     * the default check_if_same gives false.
+     *
+     * @tparam Ref  the reference type
+     * @tparam Args  the input type
+     */
+    template <typename Ref, typename... Args>
+    struct check_if_same : public std::false_type {};
+
+    /**
+     * check_if_same gives true if the decay type of input is the same type as
+     * Ref.
+     *
+     * @tparam Ref  the reference type
+     */
+    template <typename Ref>
+    struct check_if_same<Ref, Ref> : public std::true_type {};
+
 public:
     /**
      * The type of the underlying accessor.
@@ -66,14 +85,20 @@ class range {
      * Creates a new range.
      *
      * @tparam AccessorParam  types of parameters forwarded to the accessor
-     *                        constructor
+     *                        constructor.
      *
      * @param params  parameters forwarded to Accessor constructor.
+     *
+     * @note We use SFINAE to allow for a default copy and move constructor to
+     *       be generated, so a `range` is trivially copyable if the `Accessor`
+     *       is trivially copyable.
      */
     template <typename... AccessorParams,
-              std::enable_if_t<(sizeof...(AccessorParams) > 1), bool> = true>
-    GKO_ACC_ATTRIBUTES constexpr explicit range(AccessorParams &&... params)
-        : accessor_{std::forward<AccessorParams>(params)...}
+              std::enable_if_t<
+                  !check_if_same<range, std::decay_t<AccessorParams>...>::value,
+                  int> = 0>
+    GKO_ACC_ATTRIBUTES constexpr explicit range(AccessorParams &&... args)
+        : accessor_{std::forward<AccessorParams>(args)...}
     {}
 
     /**

From dc2b179b3e54174810424c8ddf875819e7637726 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 22 Jul 2021 14:47:10 +0200
Subject: [PATCH 0921/1008] relax the error bound on adv_apply_on_small_mtx

---
 dpcpp/test/matrix/ell_kernels.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dpcpp/test/matrix/ell_kernels.cpp b/dpcpp/test/matrix/ell_kernels.cpp
index 37814e946c4..bf38fb885be 100644
--- a/dpcpp/test/matrix/ell_kernels.cpp
+++ b/dpcpp/test/matrix/ell_kernels.cpp
@@ -525,7 +525,7 @@ TEST_F(Ell, AdvancedApplyOnSmallMatrixIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value * 5);
 }
 
 

From 270c08ae4a96966c07dc8d84f690bd9e680e56c2 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 16 Jul 2021 01:16:45 +0200
Subject: [PATCH 0922/1008] add ILU0 reference kernels

---
 reference/factorization/ic_kernels.cpp       |  50 +-
 reference/factorization/ilu_kernels.cpp      |  46 +-
 reference/test/factorization/CMakeLists.txt  |   2 +
 reference/test/factorization/ic_kernels.cpp  | 239 +++++++++
 reference/test/factorization/ilu_kernels.cpp | 505 +++++++++++++++++++
 5 files changed, 840 insertions(+), 2 deletions(-)
 create mode 100644 reference/test/factorization/ic_kernels.cpp
 create mode 100644 reference/test/factorization/ilu_kernels.cpp

diff --git a/reference/factorization/ic_kernels.cpp b/reference/factorization/ic_kernels.cpp
index b1d075044ad..77bacb7e34a 100644
--- a/reference/factorization/ic_kernels.cpp
+++ b/reference/factorization/ic_kernels.cpp
@@ -33,6 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/factorization/ic_kernels.hpp"
 
 
+#include "core/base/allocator.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace reference {
@@ -46,7 +49,52 @@ namespace ic_factorization {
 
 template <typename ValueType, typename IndexType>
 void compute(std::shared_ptr<const DefaultExecutor> exec,
-             matrix::Csr<ValueType, IndexType> *m) GKO_NOT_IMPLEMENTED;
+             matrix::Csr<ValueType, IndexType> *m)
+{
+    vector<IndexType> diagonals{m->get_size()[0], -1, {exec}};
+    const auto row_ptrs = m->get_const_row_ptrs();
+    const auto col_idxs = m->get_const_col_idxs();
+    const auto values = m->get_values();
+    for (size_type row = 0; row < m->get_size()[0]; row++) {
+        const auto begin = row_ptrs[row];
+        const auto end = row_ptrs[row + 1];
+        for (auto nz = begin; nz < end; nz++) {
+            const auto col = col_idxs[nz];
+            if (col == row) {
+                diagonals[row] = nz;
+            }
+            if (col > row) {
+                continue;
+            }
+            // accumulate l(row,:) * l(col,:) without the last entry l(col, col)
+            ValueType sum{};
+            auto l_idx = begin;
+            const auto l_end = end;
+            auto lh_idx = row_ptrs[col];
+            const auto lh_end = row_ptrs[col + 1];
+            while (l_idx < l_end && lh_idx < lh_end) {
+                const auto l_col = col_idxs[l_idx];
+                const auto lh_row = col_idxs[lh_idx];
+                // only consider lower triangle of L
+                if (max(l_col, lh_row) > row) {
+                    break;
+                }
+                // ignore l(col, col)
+                if (l_col == lh_row && l_col < col) {
+                    sum += values[l_idx] * values[lh_idx];
+                }
+                l_idx += l_col <= lh_row ? 1 : 0;
+                lh_idx += lh_row <= l_col ? 1 : 0;
+            }
+            if (row == col) {
+                values[nz] = sqrt(values[nz] - sum);
+            } else {
+                GKO_ASSERT(diagonals[col] != -1);
+                values[nz] = (values[nz] - sum) / values[diagonals[col]];
+            }
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_IC_COMPUTE_KERNEL);
 
diff --git a/reference/factorization/ilu_kernels.cpp b/reference/factorization/ilu_kernels.cpp
index 73f4cb8579e..68203977cdf 100644
--- a/reference/factorization/ilu_kernels.cpp
+++ b/reference/factorization/ilu_kernels.cpp
@@ -33,6 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/factorization/ilu_kernels.hpp"
 
 
+#include "core/base/allocator.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace reference {
@@ -46,7 +49,48 @@ namespace ilu_factorization {
 
 template <typename ValueType, typename IndexType>
 void compute_lu(std::shared_ptr<const DefaultExecutor> exec,
-                matrix::Csr<ValueType, IndexType> *m) GKO_NOT_IMPLEMENTED;
+                matrix::Csr<ValueType, IndexType> *m)
+{
+    vector<IndexType> diagonals{m->get_size()[0], -1, {exec}};
+    const auto row_ptrs = m->get_const_row_ptrs();
+    const auto col_idxs = m->get_const_col_idxs();
+    const auto values = m->get_values();
+    for (IndexType row = 0; row < static_cast<IndexType>(m->get_size()[0]);
+         row++) {
+        const auto begin = row_ptrs[row];
+        const auto end = row_ptrs[row + 1];
+        for (auto nz = begin; nz < end; nz++) {
+            const auto col = col_idxs[nz];
+            if (col == row) {
+                diagonals[row] = nz;
+            }
+            auto value = values[nz];
+            const auto l_begin = begin;
+            const auto l_end = end;
+            for (auto l_nz = begin; l_nz < end; l_nz++) {
+                // for each lower triangular entry l_ik
+                const auto l_col = col_idxs[l_nz];
+                if (l_col >= std::min(row, col)) {
+                    continue;
+                }
+                // find corresponding entry u_kj
+                const auto u_begin_it = col_idxs + row_ptrs[l_col];
+                const auto u_end_it = col_idxs + row_ptrs[l_col + 1];
+                const auto u_it = std::lower_bound(u_begin_it, u_end_it, col);
+                const auto u_nz = std::distance(col_idxs, u_it);
+                if (u_it != u_end_it && *u_it == col) {
+                    value -= values[l_nz] * values[u_nz];
+                }
+            }
+            if (row <= col) {
+                values[nz] = value;
+            } else {
+                GKO_ASSERT(diagonals[col] != -1);
+                values[nz] = value / values[diagonals[col]];
+            }
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_ILU_COMPUTE_LU_KERNEL);
diff --git a/reference/test/factorization/CMakeLists.txt b/reference/test/factorization/CMakeLists.txt
index d3d66296318..6fb7f5fea43 100644
--- a/reference/test/factorization/CMakeLists.txt
+++ b/reference/test/factorization/CMakeLists.txt
@@ -1,3 +1,5 @@
+ginkgo_create_test(ic_kernels)
+ginkgo_create_test(ilu_kernels)
 ginkgo_create_test(par_ic_kernels)
 ginkgo_create_test(par_ict_kernels)
 ginkgo_create_test(par_ilu_kernels)
diff --git a/reference/test/factorization/ic_kernels.cpp b/reference/test/factorization/ic_kernels.cpp
new file mode 100644
index 00000000000..43823eb3133
--- /dev/null
+++ b/reference/test/factorization/ic_kernels.cpp
@@ -0,0 +1,239 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/factorization/ic.hpp>
+
+
+#include <algorithm>
+#include <memory>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class DummyLinOp : public gko::EnableLinOp<DummyLinOp>,
+                   public gko::EnableCreateMethod<DummyLinOp> {
+public:
+    DummyLinOp(std::shared_ptr<const gko::Executor> exec,
+               gko::dim<2> size = gko::dim<2>{})
+        : EnableLinOp<DummyLinOp>(exec, size)
+    {}
+
+protected:
+    void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override {}
+
+    void apply_impl(const gko::LinOp *alpha, const gko::LinOp *b,
+                    const gko::LinOp *beta, gko::LinOp *x) const override
+    {}
+};
+
+
+template <typename ValueIndexType>
+class Ic : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using factorization_type = gko::factorization::Ic<value_type, index_type>;
+    using Coo = gko::matrix::Coo<value_type, index_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    using Dense = gko::matrix::Dense<value_type>;
+
+    Ic()
+        : ref(gko::ReferenceExecutor::create()),
+          exec(std::static_pointer_cast<const gko::Executor>(ref)),
+          identity(gko::initialize<Csr>(
+              {{1., 0., 0.}, {0., 1., 0.}, {0., 0., 1.}}, ref)),
+          banded(gko::initialize<Csr>(
+              {{1., 1., 0.}, {1., 2., 1.}, {0., 1., 2.}}, ref)),
+          banded_l_expect(gko::initialize<Csr>(
+              {{1., 0., 0.}, {1., 1., 0.}, {0., 1., 1.}}, ref)),
+          mtx_system(gko::initialize<Csr>({{9., 0., -6., 3.},
+                                           {0., 36., 18., 24.},
+                                           {-6., 18., 17., 14.},
+                                           {-3., 24., 14., 18.}},
+                                          ref)),
+          mtx_l_system(gko::initialize<Csr>({{9., 0., 0., 0.},
+                                             {0., 36., 0., 0.},
+                                             {-6., 18., 17., 0.},
+                                             {-3., 24., 14., 18.}},
+                                            ref)),
+          mtx_l_system_coo(Coo::create(exec)),
+          mtx_l_init_expect(gko::initialize<Csr>(
+              {{3., 0., 0., 0.},
+               {0., 6., 0., 0.},
+               {-6., 18., static_cast<value_type>(sqrt(17.)), 0.},
+               {-3., 24., 14., static_cast<value_type>(sqrt(18.))}},
+              ref)),
+          mtx_l_it_expect(gko::initialize<Csr>({{3., 0., 0., 0.},
+                                                {0., 6., 0., 0.},
+                                                {-2., 3., 2., 0.},
+                                                {-1., 4., 0., 1.}},
+                                               ref)),
+          fact_fact(factorization_type::build().on(exec)),
+          tol{r<value_type>::value}
+    {
+        mtx_l_system->convert_to(gko::lend(mtx_l_system_coo));
+    }
+
+    std::shared_ptr<const gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<Csr> identity;
+    std::shared_ptr<Csr> banded;
+    std::shared_ptr<Csr> banded_l_expect;
+    std::shared_ptr<Csr> mtx_system;
+    std::unique_ptr<Csr> mtx_l_system;
+    std::unique_ptr<Coo> mtx_l_system_coo;
+    std::unique_ptr<Csr> mtx_l_init_expect;
+    std::unique_ptr<Csr> mtx_l_it_expect;
+    std::unique_ptr<typename factorization_type::Factory> fact_fact;
+    gko::remove_complex<value_type> tol;
+};
+
+TYPED_TEST_SUITE(Ic, gko::test::ValueIndexTypes);
+
+
+TYPED_TEST(Ic, ThrowNotSupportedForWrongLinOp)
+{
+    auto lin_op = DummyLinOp::create(this->ref);
+
+    ASSERT_THROW(this->fact_fact->generate(gko::share(lin_op)),
+                 gko::NotSupported);
+}
+
+
+TYPED_TEST(Ic, ThrowDimensionMismatch)
+{
+    using Csr = typename TestFixture::Csr;
+    auto matrix = Csr::create(this->ref, gko::dim<2>{2, 3}, 4);
+
+    ASSERT_THROW(this->fact_fact->generate(gko::share(matrix)),
+                 gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(Ic, SetStrategy)
+{
+    using Csr = typename TestFixture::Csr;
+    using factorization_type = typename TestFixture::factorization_type;
+    auto l_strategy = std::make_shared<typename Csr::merge_path>();
+
+    auto factory =
+        factorization_type::build().with_l_strategy(l_strategy).on(this->ref);
+    auto fact = factory->generate(this->mtx_system);
+
+    ASSERT_EQ(factory->get_parameters().l_strategy, l_strategy);
+    ASSERT_EQ(fact->get_l_factor()->get_strategy()->get_name(),
+              l_strategy->get_name());
+    ASSERT_EQ(fact->get_lt_factor()->get_strategy()->get_name(),
+              l_strategy->get_name());
+}
+
+
+TYPED_TEST(Ic, IsConsistentWithComposition)
+{
+    auto fact = this->fact_fact->generate(this->mtx_system);
+
+    auto lin_op_l_factor = gko::as<gko::LinOp>(fact->get_l_factor());
+    auto lin_op_lt_factor = gko::as<gko::LinOp>(fact->get_lt_factor());
+    auto first_operator = fact->get_operators()[0];
+    auto second_operator = fact->get_operators()[1];
+
+    ASSERT_EQ(lin_op_l_factor, first_operator);
+    ASSERT_EQ(lin_op_lt_factor, second_operator);
+}
+
+
+TYPED_TEST(Ic, GenerateIdentity)
+{
+    auto fact = this->fact_fact->generate(this->identity);
+
+    GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->identity, this->tol);
+    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(), this->identity, this->tol);
+}
+
+
+TYPED_TEST(Ic, GenerateDenseIdentity)
+{
+    using Dense = typename TestFixture::Dense;
+    auto dense_id = Dense::create(this->exec, this->identity->get_size());
+    this->identity->convert_to(dense_id.get());
+    auto fact = this->fact_fact->generate(gko::share(dense_id));
+
+    GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->identity, this->tol);
+    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(), this->identity, this->tol);
+}
+
+
+TYPED_TEST(Ic, GenerateBanded)
+{
+    using factorization_type = typename TestFixture::factorization_type;
+    using Csr = typename TestFixture::Csr;
+    auto fact =
+        factorization_type::build().on(this->exec)->generate(this->banded);
+
+    GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->banded_l_expect, this->tol);
+    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(),
+                        gko::as<Csr>(this->banded_l_expect->conj_transpose()),
+                        this->tol);
+}
+
+
+TYPED_TEST(Ic, GenerateGeneral)
+{
+    using factorization_type = typename TestFixture::factorization_type;
+    using Csr = typename TestFixture::Csr;
+    auto fact =
+        factorization_type::build().on(this->exec)->generate(this->mtx_system);
+
+    GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->mtx_l_it_expect, this->tol);
+    GKO_ASSERT_MTX_NEAR(fact->get_lt_factor(),
+                        gko::as<Csr>(this->mtx_l_it_expect->conj_transpose()),
+                        this->tol);
+}
+
+
+}  // namespace
diff --git a/reference/test/factorization/ilu_kernels.cpp b/reference/test/factorization/ilu_kernels.cpp
new file mode 100644
index 00000000000..fd47d1e302c
--- /dev/null
+++ b/reference/test/factorization/ilu_kernels.cpp
@@ -0,0 +1,505 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/factorization/ilu.hpp>
+
+
+#include <algorithm>
+#include <initializer_list>
+#include <memory>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class DummyLinOp : public gko::EnableLinOp<DummyLinOp>,
+                   public gko::EnableCreateMethod<DummyLinOp> {
+public:
+    DummyLinOp(std::shared_ptr<const gko::Executor> exec,
+               gko::dim<2> size = gko::dim<2>{})
+        : EnableLinOp<DummyLinOp>(exec, size)
+    {}
+
+protected:
+    void apply_impl(const gko::LinOp *b, gko::LinOp *x) const override {}
+
+    void apply_impl(const gko::LinOp *alpha, const gko::LinOp *b,
+                    const gko::LinOp *beta, gko::LinOp *x) const override
+    {}
+};
+
+
+template <typename ValueIndexType>
+class Ilu : public ::testing::Test {
+protected:
+    using value_type =
+        typename std::tuple_element<0, decltype(ValueIndexType())>::type;
+    using index_type =
+        typename std::tuple_element<1, decltype(ValueIndexType())>::type;
+    using Dense = gko::matrix::Dense<value_type>;
+    using Coo = gko::matrix::Coo<value_type, index_type>;
+    using Csr = gko::matrix::Csr<value_type, index_type>;
+    using ilu_type = gko::factorization::Ilu<value_type, index_type>;
+    Ilu()
+        : ref(gko::ReferenceExecutor::create()),
+          exec(std::static_pointer_cast<const gko::Executor>(ref)),
+          // clang-format off
+          empty_csr(gko::initialize<Csr>(
+              {{0., 0., 0.},
+               {0., 0., 0.},
+               {0., 0., 0.}}, exec)),
+          identity(gko::initialize<Dense>(
+              {{1., 0., 0.},
+               {0., 1., 0.},
+               {0., 0., 1.}}, exec)),
+          lower_triangular(gko::initialize<Dense>(
+              {{1., 0., 0.},
+               {1., 1., 0.},
+               {1., 1., 1.}}, exec)),
+          upper_triangular(gko::initialize<Dense>(
+              {{1., 1., 1.},
+               {0., 1., 1.},
+               {0., 0., 1.}}, exec)),
+          mtx_small(gko::initialize<Dense>(
+              {{4., 6., 8.},
+               {2., 2., 5.},
+               {1., 1., 1.}}, exec)),
+          mtx_csr_small(nullptr),
+          small_l_expected(gko::initialize<Dense>(
+              {{1., 0., 0.},
+               {0.5, 1., 0.},
+               {0.25, 0.5, 1.}}, exec)),
+          small_u_expected(gko::initialize<Dense>(
+              {{4., 6., 8.},
+               {0., -1., 1.},
+               {0., 0., -1.5}}, exec)),
+          mtx_small2(gko::initialize<Dense>(
+              {{8., 8., 0},
+              {2., 0., 5.},
+              {1., 1., 1}}, exec)),
+          mtx_csr_small2(nullptr),
+          small2_l_expected(gko::initialize<Dense>(
+              {{1., 0., 0},
+              {.25, 1., 0.},
+              {.125, 0., 1}}, exec)),
+          small2_u_expected(gko::initialize<Dense>(
+              {{8., 8., 0},
+              {0., -2., 5.},
+              {0., 0., 1}}, exec)),
+          mtx_big(gko::initialize<Dense>({{1., 1., 1., 0., 1., 3.},
+                                          {1., 2., 2., 0., 2., 0.},
+                                          {0., 2., 3., 3., 3., 5.},
+                                          {1., 0., 3., 4., 4., 4.},
+                                          {1., 2., 0., 4., 5., 6.},
+                                          {0., 2., 3., 4., 5., 8.}},
+                                         exec)),
+          big_l_expected(gko::initialize<Dense>({{1., 0., 0., 0., 0., 0.},
+                                                 {1., 1., 0., 0., 0., 0.},
+                                                 {0., 2., 1., 0., 0., 0.},
+                                                 {1., 0., 2., 1., 0., 0.},
+                                                 {1., 1., 0., -2., 1., 0.},
+                                                 {0., 2., 1., -0.5, 0.5, 1.}},
+                                                exec)),
+          big_u_expected(gko::initialize<Dense>({{1., 1., 1., 0., 1., 3.},
+                                                 {0., 1., 1., 0., 1., 0.},
+                                                 {0., 0., 1., 3., 1., 5.},
+                                                 {0., 0., 0., -2., 1., -9.},
+                                                 {0., 0., 0., 0., 5., -15.},
+                                                 {0., 0., 0., 0., 0., 6.}},
+                                                exec)),
+          mtx_big_nodiag(gko::initialize<Csr>({{1., 1., 1., 0., 1., 3.},
+                                               {1., 2., 2., 0., 2., 0.},
+                                               {0., 2., 0., 3., 3., 5.},
+                                               {1., 0., 3., 4., 4., 4.},
+                                               {1., 2., 0., 4., 1., 6.},
+                                               {0., 2., 3., 4., 5., 8.}},
+                                         exec)),
+          big_nodiag_l_expected(gko::initialize<Dense>(
+            {{1., 0., 0., 0., 0., 0.},
+             {1., 1., 0., 0., 0., 0.},
+             {0., 2., 1., 0., 0., 0.},
+             {1., 0., -1., 1., 0., 0.},
+             {1., 1., 0., 0.571428571428571, 1., 0.},
+             {0., 2., -0.5, 0.785714285714286, -0.108695652173913, 1.}},
+            exec)),
+          big_nodiag_u_expected(gko::initialize<Dense>(
+            {{1., 1., 1., 0., 1., 3.},
+             {0., 1., 1., 0., 1., 0.},
+             {0., 0., -2., 3., 1., 5.},
+             {0., 0., 0., 7., 4., 6.},
+             {0., 0., 0., 0., -3.28571428571429, -0.428571428571429},
+             {0., 0., 0., 0., 0., 5.73913043478261}},
+            exec)),
+          // clang-format on
+          ilu_factory_skip(ilu_type::build().with_skip_sorting(true).on(exec)),
+          ilu_factory_sort(ilu_type::build().with_skip_sorting(false).on(exec))
+    {
+        auto tmp_csr = Csr::create(exec);
+        mtx_small->convert_to(gko::lend(tmp_csr));
+        mtx_csr_small = std::move(tmp_csr);
+        auto tmp_csr2 = Csr::create(exec);
+        mtx_small2->convert_to(gko::lend(tmp_csr2));
+        mtx_csr_small2 = std::move(tmp_csr2);
+    }
+
+    std::shared_ptr<const gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::Executor> exec;
+    std::shared_ptr<const Csr> empty_csr;
+    std::shared_ptr<const Dense> identity;
+    std::shared_ptr<const Dense> lower_triangular;
+    std::shared_ptr<const Dense> upper_triangular;
+    std::shared_ptr<const Dense> mtx_small;
+    std::shared_ptr<const Csr> mtx_csr_small;
+    std::shared_ptr<const Dense> small_l_expected;
+    std::shared_ptr<const Dense> small_u_expected;
+    std::shared_ptr<const Dense> mtx_small2;
+    std::shared_ptr<const Csr> mtx_csr_small2;
+    std::shared_ptr<const Dense> small2_l_expected;
+    std::shared_ptr<const Dense> small2_u_expected;
+    std::shared_ptr<const Dense> mtx_big;
+    std::shared_ptr<const Dense> big_l_expected;
+    std::shared_ptr<const Dense> big_u_expected;
+    std::shared_ptr<const Csr> mtx_big_nodiag;
+    std::shared_ptr<const Dense> big_nodiag_l_expected;
+    std::shared_ptr<const Dense> big_nodiag_u_expected;
+    std::unique_ptr<typename ilu_type::Factory> ilu_factory_skip;
+    std::unique_ptr<typename ilu_type::Factory> ilu_factory_sort;
+};
+
+TYPED_TEST_SUITE(Ilu, gko::test::ValueIndexTypes);
+
+
+TYPED_TEST(Ilu, ThrowNotSupportedForWrongLinOp1)
+{
+    auto linOp = DummyLinOp::create(this->ref);
+
+    ASSERT_THROW(this->ilu_factory_skip->generate(gko::share(linOp)),
+                 gko::NotSupported);
+}
+
+
+TYPED_TEST(Ilu, ThrowNotSupportedForWrongLinOp2)
+{
+    auto linOp = DummyLinOp::create(this->ref);
+
+    ASSERT_THROW(this->ilu_factory_sort->generate(gko::share(linOp)),
+                 gko::NotSupported);
+}
+
+
+TYPED_TEST(Ilu, ThrowDimensionMismatch)
+{
+    using Csr = typename TestFixture::Csr;
+    auto matrix = Csr::create(this->ref, gko::dim<2>{2, 3}, 4);
+
+    ASSERT_THROW(this->ilu_factory_sort->generate(gko::share(matrix)),
+                 gko::DimensionMismatch);
+}
+
+
+TYPED_TEST(Ilu, SetLStrategy)
+{
+    using Csr = typename TestFixture::Csr;
+    using ilu_type = typename TestFixture::ilu_type;
+    auto l_strategy = std::make_shared<typename Csr::classical>();
+
+    auto factory = ilu_type::build().with_l_strategy(l_strategy).on(this->ref);
+    auto ilu = factory->generate(this->mtx_small);
+
+    ASSERT_EQ(factory->get_parameters().l_strategy, l_strategy);
+    ASSERT_EQ(ilu->get_l_factor()->get_strategy()->get_name(),
+              l_strategy->get_name());
+}
+
+
+TYPED_TEST(Ilu, SetUStrategy)
+{
+    using Csr = typename TestFixture::Csr;
+    using ilu_type = typename TestFixture::ilu_type;
+    auto u_strategy = std::make_shared<typename Csr::classical>();
+
+    auto factory = ilu_type::build().with_u_strategy(u_strategy).on(this->ref);
+    auto ilu = factory->generate(this->mtx_small);
+
+    ASSERT_EQ(factory->get_parameters().u_strategy, u_strategy);
+    ASSERT_EQ(ilu->get_u_factor()->get_strategy()->get_name(),
+              u_strategy->get_name());
+}
+
+
+TYPED_TEST(Ilu, LUFactorFunctionsSetProperly)
+{
+    auto factors = this->ilu_factory_skip->generate(this->mtx_small);
+
+    auto lin_op_l_factor =
+        static_cast<const gko::LinOp *>(gko::lend(factors->get_l_factor()));
+    auto lin_op_u_factor =
+        static_cast<const gko::LinOp *>(gko::lend(factors->get_u_factor()));
+    auto first_operator = gko::lend(factors->get_operators()[0]);
+    auto second_operator = gko::lend(factors->get_operators()[1]);
+
+    ASSERT_EQ(lin_op_l_factor, first_operator);
+    ASSERT_EQ(lin_op_u_factor, second_operator);
+}
+
+
+TYPED_TEST(Ilu, GenerateForCooIdentity)
+{
+    using Coo = typename TestFixture::Coo;
+    using value_type = typename TestFixture::value_type;
+    auto coo_mtx = gko::share(Coo::create(this->exec));
+    this->identity->convert_to(gko::lend(coo_mtx));
+
+    auto factors = this->ilu_factory_skip->generate(coo_mtx);
+    auto l_factor = factors->get_l_factor();
+    auto u_factor = factors->get_u_factor();
+
+    GKO_ASSERT_MTX_NEAR(l_factor, this->identity, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u_factor, this->identity, r<value_type>::value);
+}
+
+
+TYPED_TEST(Ilu, GenerateForCsrIdentity)
+{
+    using Csr = typename TestFixture::Csr;
+    using value_type = typename TestFixture::value_type;
+    auto csr_mtx = gko::share(Csr::create(this->exec));
+    this->identity->convert_to(gko::lend(csr_mtx));
+
+    auto factors = this->ilu_factory_skip->generate(csr_mtx);
+    auto l_factor = factors->get_l_factor();
+    auto u_factor = factors->get_u_factor();
+
+    GKO_ASSERT_MTX_NEAR(l_factor, this->identity, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u_factor, this->identity, r<value_type>::value);
+}
+
+
+TYPED_TEST(Ilu, GenerateForDenseIdentity)
+{
+    using value_type = typename TestFixture::value_type;
+    auto factors = this->ilu_factory_skip->generate(this->identity);
+    auto l_factor = factors->get_l_factor();
+    auto u_factor = factors->get_u_factor();
+
+    GKO_ASSERT_MTX_NEAR(l_factor, this->identity, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u_factor, this->identity, r<value_type>::value);
+}
+
+
+TYPED_TEST(Ilu, GenerateForDenseLowerTriangular)
+{
+    using value_type = typename TestFixture::value_type;
+    auto factors = this->ilu_factory_skip->generate(this->lower_triangular);
+    auto l_factor = factors->get_l_factor();
+    auto u_factor = factors->get_u_factor();
+
+    GKO_ASSERT_MTX_NEAR(l_factor, this->lower_triangular, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u_factor, this->identity, r<value_type>::value);
+}
+
+
+TYPED_TEST(Ilu, GenerateForDenseUpperTriangular)
+{
+    using value_type = typename TestFixture::value_type;
+    auto factors = this->ilu_factory_skip->generate(this->upper_triangular);
+    auto l_factor = factors->get_l_factor();
+    auto u_factor = factors->get_u_factor();
+
+    GKO_ASSERT_MTX_NEAR(l_factor, this->identity, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u_factor, this->upper_triangular, r<value_type>::value);
+}
+
+
+TYPED_TEST(Ilu, ApplyMethodDenseSmall)
+{
+    using value_type = typename TestFixture::value_type;
+    using Dense = typename TestFixture::Dense;
+    const auto x = gko::initialize<Dense>({1., 2., 3.}, this->exec);
+    auto b_lu = Dense::create_with_config_of(gko::lend(x));
+    auto b_ref = Dense::create_with_config_of(gko::lend(x));
+
+    auto factors = this->ilu_factory_skip->generate(this->mtx_small);
+    factors->apply(gko::lend(x), gko::lend(b_lu));
+    this->mtx_small->apply(gko::lend(x), gko::lend(b_ref));
+
+    GKO_ASSERT_MTX_NEAR(b_lu, b_ref, r<value_type>::value);
+}
+
+
+TYPED_TEST(Ilu, GenerateForDenseSmall)
+{
+    using value_type = typename TestFixture::value_type;
+    auto factors = this->ilu_factory_skip->generate(this->mtx_small);
+    auto l_factor = factors->get_l_factor();
+    auto u_factor = factors->get_u_factor();
+
+    GKO_ASSERT_MTX_NEAR(l_factor, this->small_l_expected, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u_factor, this->small_u_expected, r<value_type>::value);
+}
+
+
+TYPED_TEST(Ilu, GenerateForCsrSmall)
+{
+    using value_type = typename TestFixture::value_type;
+    auto factors = this->ilu_factory_skip->generate(this->mtx_csr_small);
+    auto l_factor = factors->get_l_factor();
+    auto u_factor = factors->get_u_factor();
+
+    GKO_ASSERT_MTX_NEAR(l_factor, this->small_l_expected, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u_factor, this->small_u_expected, r<value_type>::value);
+}
+
+
+TYPED_TEST(Ilu, GenerateForCsrSmall2ZeroDiagonal)
+{
+    using value_type = typename TestFixture::value_type;
+    auto factors = this->ilu_factory_skip->generate(this->mtx_csr_small2);
+    auto l_factor = factors->get_l_factor();
+    auto u_factor = factors->get_u_factor();
+
+    GKO_ASSERT_MTX_NEAR(l_factor, this->small2_l_expected,
+                        r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u_factor, this->small2_u_expected,
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(Ilu, GenerateForCsrBigWithDiagonalZeros)
+{
+    using value_type = typename TestFixture::value_type;
+    auto factors = this->ilu_factory_skip->generate(this->mtx_big_nodiag);
+    auto l_factor = factors->get_l_factor();
+    auto u_factor = factors->get_u_factor();
+
+    GKO_ASSERT_MTX_NEAR(l_factor, this->big_nodiag_l_expected,
+                        r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u_factor, this->big_nodiag_u_expected,
+                        r<value_type>::value);
+}
+
+
+TYPED_TEST(Ilu, GenerateForDenseBig)
+{
+    using value_type = typename TestFixture::value_type;
+    auto factors = this->ilu_factory_skip->generate(this->mtx_big);
+    auto l_factor = factors->get_l_factor();
+    auto u_factor = factors->get_u_factor();
+
+    GKO_ASSERT_MTX_NEAR(l_factor, this->big_l_expected, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u_factor, this->big_u_expected, r<value_type>::value);
+}
+
+
+TYPED_TEST(Ilu, GenerateForDenseBigSort)
+{
+    using value_type = typename TestFixture::value_type;
+    auto factors = this->ilu_factory_skip->generate(this->mtx_big);
+    auto l_factor = factors->get_l_factor();
+    auto u_factor = factors->get_u_factor();
+
+    GKO_ASSERT_MTX_NEAR(l_factor, this->big_l_expected, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u_factor, this->big_u_expected, r<value_type>::value);
+}
+
+
+TYPED_TEST(Ilu, GenerateForReverseCooSmall)
+{
+    using value_type = typename TestFixture::value_type;
+    using Coo = typename TestFixture::Coo;
+    const auto size = this->mtx_small->get_size();
+    const auto nnz = size[0] * size[1];
+    auto reverse_coo = gko::share(Coo::create(this->exec, size, nnz));
+    // Fill the Coo matrix in reversed row order (right to left)
+    for (size_t i = 0; i < size[0]; ++i) {
+        for (size_t j = 0; j < size[1]; ++j) {
+            const auto coo_idx = i * size[1] + (size[1] - 1 - j);
+            reverse_coo->get_row_idxs()[coo_idx] = i;
+            reverse_coo->get_col_idxs()[coo_idx] = j;
+            reverse_coo->get_values()[coo_idx] = this->mtx_small->at(i, j);
+        }
+    }
+
+    auto factors = this->ilu_factory_sort->generate(reverse_coo);
+    auto l_factor = factors->get_l_factor();
+    auto u_factor = factors->get_u_factor();
+
+    GKO_ASSERT_MTX_NEAR(reverse_coo, this->mtx_small, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(l_factor, this->small_l_expected, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u_factor, this->small_u_expected, r<value_type>::value);
+}
+
+
+TYPED_TEST(Ilu, GenerateForReverseCsrSmall)
+{
+    using value_type = typename TestFixture::value_type;
+    using Csr = typename TestFixture::Csr;
+    const auto size = this->mtx_csr_small->get_size();
+    const auto nnz = size[0] * size[1];
+    auto reverse_csr = gko::share(Csr::create(this->exec));
+    reverse_csr->copy_from(gko::lend(this->mtx_csr_small));
+    // Fill the Csr matrix rows in reverse order
+    for (size_t i = 0; i < size[0]; ++i) {
+        const auto row_start = reverse_csr->get_row_ptrs()[i];
+        const auto row_end = reverse_csr->get_row_ptrs()[i + 1];
+        for (size_t j = row_start; j < row_end; ++j) {
+            const auto reverse_j = row_end - 1 - (j - row_start);
+            reverse_csr->get_values()[reverse_j] =
+                this->mtx_csr_small->get_const_values()[j];
+            reverse_csr->get_col_idxs()[reverse_j] =
+                this->mtx_csr_small->get_const_col_idxs()[j];
+        }
+    }
+
+    auto factors = this->ilu_factory_sort->generate(reverse_csr);
+    auto l_factor = factors->get_l_factor();
+    auto u_factor = factors->get_u_factor();
+
+    GKO_ASSERT_MTX_NEAR(l_factor, this->small_l_expected, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u_factor, this->small_u_expected, r<value_type>::value);
+}
+
+
+}  // namespace

From fc0926ddc1b952e260716629d016ec797730509e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 16 Jul 2021 09:42:12 +0200
Subject: [PATCH 0923/1008] add missing IC conjugate

---
 reference/factorization/ic_kernels.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reference/factorization/ic_kernels.cpp b/reference/factorization/ic_kernels.cpp
index 77bacb7e34a..5426aff0cd0 100644
--- a/reference/factorization/ic_kernels.cpp
+++ b/reference/factorization/ic_kernels.cpp
@@ -81,7 +81,7 @@ void compute(std::shared_ptr<const DefaultExecutor> exec,
                 }
                 // ignore l(col, col)
                 if (l_col == lh_row && l_col < col) {
-                    sum += values[l_idx] * values[lh_idx];
+                    sum += values[l_idx] * conj(values[lh_idx]);
                 }
                 l_idx += l_col <= lh_row ? 1 : 0;
                 lh_idx += lh_row <= l_col ? 1 : 0;

From dd1bd2290a5b911f5036fec59a3014f454beb3b7 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 16 Jul 2021 21:52:44 +0200
Subject: [PATCH 0924/1008] anticipate Thomas' review comments

---
 reference/factorization/ic_kernels.cpp  |  3 +++
 reference/factorization/ilu_kernels.cpp | 10 ++++++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/reference/factorization/ic_kernels.cpp b/reference/factorization/ic_kernels.cpp
index 5426aff0cd0..bc63c074396 100644
--- a/reference/factorization/ic_kernels.cpp
+++ b/reference/factorization/ic_kernels.cpp
@@ -33,6 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/factorization/ic_kernels.hpp"
 
 
+#include <ginkgo/core/base/math.hpp>
+
+
 #include "core/base/allocator.hpp"
 
 
diff --git a/reference/factorization/ilu_kernels.cpp b/reference/factorization/ilu_kernels.cpp
index 68203977cdf..0fed2e9025b 100644
--- a/reference/factorization/ilu_kernels.cpp
+++ b/reference/factorization/ilu_kernels.cpp
@@ -33,6 +33,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/factorization/ilu_kernels.hpp"
 
 
+#include <algorithm>
+
+
+#include <ginkgo/core/base/math.hpp>
+
+
 #include "core/base/allocator.hpp"
 
 
@@ -67,10 +73,10 @@ void compute_lu(std::shared_ptr<const DefaultExecutor> exec,
             auto value = values[nz];
             const auto l_begin = begin;
             const auto l_end = end;
-            for (auto l_nz = begin; l_nz < end; l_nz++) {
+            for (auto l_nz = l_begin; l_nz < l_end; l_nz++) {
                 // for each lower triangular entry l_ik
                 const auto l_col = col_idxs[l_nz];
-                if (l_col >= std::min(row, col)) {
+                if (l_col >= min(row, col)) {
                     continue;
                 }
                 // find corresponding entry u_kj

From ca9916448c7e0a83e03ff382d06595db5fc6134d Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 23 Jul 2021 11:11:17 +0200
Subject: [PATCH 0925/1008] update tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* remove a few unused variables
* fix a ParILU and ILU(0) test

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
Co-authored-by: Marcel Koch <marcel.koch@kit.edu>
---
 reference/factorization/ic_kernels.cpp        |  2 +-
 reference/factorization/ilu_kernels.cpp       |  6 ++---
 reference/test/factorization/ic_kernels.cpp   | 22 ++++---------------
 reference/test/factorization/ilu_kernels.cpp  |  7 +-----
 .../test/factorization/par_ic_kernels.cpp     |  3 +++
 .../test/factorization/par_ilu_kernels.cpp    |  2 +-
 6 files changed, 12 insertions(+), 30 deletions(-)

diff --git a/reference/factorization/ic_kernels.cpp b/reference/factorization/ic_kernels.cpp
index bc63c074396..fff0811ce92 100644
--- a/reference/factorization/ic_kernels.cpp
+++ b/reference/factorization/ic_kernels.cpp
@@ -54,7 +54,7 @@ template <typename ValueType, typename IndexType>
 void compute(std::shared_ptr<const DefaultExecutor> exec,
              matrix::Csr<ValueType, IndexType> *m)
 {
-    vector<IndexType> diagonals{m->get_size()[0], -1, {exec}};
+    vector<IndexType> diagonals{m->get_size()[0], -1, exec};
     const auto row_ptrs = m->get_const_row_ptrs();
     const auto col_idxs = m->get_const_col_idxs();
     const auto values = m->get_values();
diff --git a/reference/factorization/ilu_kernels.cpp b/reference/factorization/ilu_kernels.cpp
index 0fed2e9025b..80c2e61098f 100644
--- a/reference/factorization/ilu_kernels.cpp
+++ b/reference/factorization/ilu_kernels.cpp
@@ -57,7 +57,7 @@ template <typename ValueType, typename IndexType>
 void compute_lu(std::shared_ptr<const DefaultExecutor> exec,
                 matrix::Csr<ValueType, IndexType> *m)
 {
-    vector<IndexType> diagonals{m->get_size()[0], -1, {exec}};
+    vector<IndexType> diagonals{m->get_size()[0], -1, exec};
     const auto row_ptrs = m->get_const_row_ptrs();
     const auto col_idxs = m->get_const_col_idxs();
     const auto values = m->get_values();
@@ -71,9 +71,7 @@ void compute_lu(std::shared_ptr<const DefaultExecutor> exec,
                 diagonals[row] = nz;
             }
             auto value = values[nz];
-            const auto l_begin = begin;
-            const auto l_end = end;
-            for (auto l_nz = l_begin; l_nz < l_end; l_nz++) {
+            for (auto l_nz = begin; l_nz < end; l_nz++) {
                 // for each lower triangular entry l_ik
                 const auto l_col = col_idxs[l_nz];
                 if (l_col >= min(row, col)) {
diff --git a/reference/test/factorization/ic_kernels.cpp b/reference/test/factorization/ic_kernels.cpp
index 43823eb3133..a1ffc505e2f 100644
--- a/reference/test/factorization/ic_kernels.cpp
+++ b/reference/test/factorization/ic_kernels.cpp
@@ -96,18 +96,6 @@ class Ic : public ::testing::Test {
                                            {-6., 18., 17., 14.},
                                            {-3., 24., 14., 18.}},
                                           ref)),
-          mtx_l_system(gko::initialize<Csr>({{9., 0., 0., 0.},
-                                             {0., 36., 0., 0.},
-                                             {-6., 18., 17., 0.},
-                                             {-3., 24., 14., 18.}},
-                                            ref)),
-          mtx_l_system_coo(Coo::create(exec)),
-          mtx_l_init_expect(gko::initialize<Csr>(
-              {{3., 0., 0., 0.},
-               {0., 6., 0., 0.},
-               {-6., 18., static_cast<value_type>(sqrt(17.)), 0.},
-               {-3., 24., 14., static_cast<value_type>(sqrt(18.))}},
-              ref)),
           mtx_l_it_expect(gko::initialize<Csr>({{3., 0., 0., 0.},
                                                 {0., 6., 0., 0.},
                                                 {-2., 3., 2., 0.},
@@ -115,9 +103,7 @@ class Ic : public ::testing::Test {
                                                ref)),
           fact_fact(factorization_type::build().on(exec)),
           tol{r<value_type>::value}
-    {
-        mtx_l_system->convert_to(gko::lend(mtx_l_system_coo));
-    }
+    {}
 
     std::shared_ptr<const gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::Executor> exec;
@@ -125,9 +111,6 @@ class Ic : public ::testing::Test {
     std::shared_ptr<Csr> banded;
     std::shared_ptr<Csr> banded_l_expect;
     std::shared_ptr<Csr> mtx_system;
-    std::unique_ptr<Csr> mtx_l_system;
-    std::unique_ptr<Coo> mtx_l_system_coo;
-    std::unique_ptr<Csr> mtx_l_init_expect;
     std::unique_ptr<Csr> mtx_l_it_expect;
     std::unique_ptr<typename factorization_type::Factory> fact_fact;
     gko::remove_complex<value_type> tol;
@@ -201,6 +184,7 @@ TYPED_TEST(Ic, GenerateDenseIdentity)
     using Dense = typename TestFixture::Dense;
     auto dense_id = Dense::create(this->exec, this->identity->get_size());
     this->identity->convert_to(dense_id.get());
+
     auto fact = this->fact_fact->generate(gko::share(dense_id));
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->identity, this->tol);
@@ -212,6 +196,7 @@ TYPED_TEST(Ic, GenerateBanded)
 {
     using factorization_type = typename TestFixture::factorization_type;
     using Csr = typename TestFixture::Csr;
+
     auto fact =
         factorization_type::build().on(this->exec)->generate(this->banded);
 
@@ -226,6 +211,7 @@ TYPED_TEST(Ic, GenerateGeneral)
 {
     using factorization_type = typename TestFixture::factorization_type;
     using Csr = typename TestFixture::Csr;
+
     auto fact =
         factorization_type::build().on(this->exec)->generate(this->mtx_system);
 
diff --git a/reference/test/factorization/ilu_kernels.cpp b/reference/test/factorization/ilu_kernels.cpp
index fd47d1e302c..24e49be272a 100644
--- a/reference/test/factorization/ilu_kernels.cpp
+++ b/reference/test/factorization/ilu_kernels.cpp
@@ -86,10 +86,6 @@ class Ilu : public ::testing::Test {
         : ref(gko::ReferenceExecutor::create()),
           exec(std::static_pointer_cast<const gko::Executor>(ref)),
           // clang-format off
-          empty_csr(gko::initialize<Csr>(
-              {{0., 0., 0.},
-               {0., 0., 0.},
-               {0., 0., 0.}}, exec)),
           identity(gko::initialize<Dense>(
               {{1., 0., 0.},
                {0., 1., 0.},
@@ -186,7 +182,6 @@ class Ilu : public ::testing::Test {
 
     std::shared_ptr<const gko::ReferenceExecutor> ref;
     std::shared_ptr<const gko::Executor> exec;
-    std::shared_ptr<const Csr> empty_csr;
     std::shared_ptr<const Dense> identity;
     std::shared_ptr<const Dense> lower_triangular;
     std::shared_ptr<const Dense> upper_triangular;
@@ -436,7 +431,7 @@ TYPED_TEST(Ilu, GenerateForDenseBig)
 TYPED_TEST(Ilu, GenerateForDenseBigSort)
 {
     using value_type = typename TestFixture::value_type;
-    auto factors = this->ilu_factory_skip->generate(this->mtx_big);
+    auto factors = this->ilu_factory_sort->generate(this->mtx_big);
     auto l_factor = factors->get_l_factor();
     auto u_factor = factors->get_u_factor();
 
diff --git a/reference/test/factorization/par_ic_kernels.cpp b/reference/test/factorization/par_ic_kernels.cpp
index 899f5e1a34e..6211b7e0a59 100644
--- a/reference/test/factorization/par_ic_kernels.cpp
+++ b/reference/test/factorization/par_ic_kernels.cpp
@@ -241,6 +241,7 @@ TYPED_TEST(ParIc, GenerateDenseIdentity)
     using Dense = typename TestFixture::Dense;
     auto dense_id = Dense::create(this->exec, this->identity->get_size());
     this->identity->convert_to(dense_id.get());
+
     auto fact = this->fact_fact->generate(gko::share(dense_id));
 
     GKO_ASSERT_MTX_NEAR(fact->get_l_factor(), this->identity, this->tol);
@@ -252,6 +253,7 @@ TYPED_TEST(ParIc, GenerateBanded)
 {
     using factorization_type = typename TestFixture::factorization_type;
     using Csr = typename TestFixture::Csr;
+
     auto fact =
         factorization_type::build().on(this->exec)->generate(this->banded);
 
@@ -266,6 +268,7 @@ TYPED_TEST(ParIc, GenerateGeneral)
 {
     using factorization_type = typename TestFixture::factorization_type;
     using Csr = typename TestFixture::Csr;
+
     auto fact =
         factorization_type::build().on(this->exec)->generate(this->mtx_system);
 
diff --git a/reference/test/factorization/par_ilu_kernels.cpp b/reference/test/factorization/par_ilu_kernels.cpp
index 388d54232ad..8efbf77dc4e 100644
--- a/reference/test/factorization/par_ilu_kernels.cpp
+++ b/reference/test/factorization/par_ilu_kernels.cpp
@@ -688,7 +688,7 @@ TYPED_TEST(ParIlu, GenerateForDenseBig)
 TYPED_TEST(ParIlu, GenerateForDenseBigSort)
 {
     using value_type = typename TestFixture::value_type;
-    auto factors = this->ilu_factory_skip->generate(this->mtx_big);
+    auto factors = this->ilu_factory_sort->generate(this->mtx_big);
     auto l_factor = factors->get_l_factor();
     auto u_factor = factors->get_u_factor();
 

From 5f3a95163e702afcbb91eb118610eccd207ce5e6 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 23 Jul 2021 13:47:09 +0200
Subject: [PATCH 0926/1008] fix ILU and IC kernels on CUDA 11.4

---
 cuda/factorization/ic_kernels.cu  | 5 +++++
 cuda/factorization/ilu_kernels.cu | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/cuda/factorization/ic_kernels.cu b/cuda/factorization/ic_kernels.cu
index eb87b79c633..9eb5c906cfb 100644
--- a/cuda/factorization/ic_kernels.cu
+++ b/cuda/factorization/ic_kernels.cu
@@ -81,6 +81,11 @@ void compute(std::shared_ptr<const DefaultExecutor> exec,
                   m->get_const_row_ptrs(), m->get_const_col_idxs(), info,
                   CUSPARSE_SOLVE_POLICY_USE_LEVEL, buffer.get_data());
 
+    // CUDA 11.4 has a use-after-free bug on Turing
+#if (CUDA_VERSION >= 11040)
+    exec->synchronize();
+#endif
+
     cusparse::destroy(info);
     cusparse::destroy(desc);
 }
diff --git a/cuda/factorization/ilu_kernels.cu b/cuda/factorization/ilu_kernels.cu
index fa1b222e26a..954843ebf1e 100644
--- a/cuda/factorization/ilu_kernels.cu
+++ b/cuda/factorization/ilu_kernels.cu
@@ -81,6 +81,11 @@ void compute_lu(std::shared_ptr<const DefaultExecutor> exec,
                    m->get_const_row_ptrs(), m->get_const_col_idxs(), info,
                    CUSPARSE_SOLVE_POLICY_USE_LEVEL, buffer.get_data());
 
+    // CUDA 11.4 has a use-after-free bug on Turing
+#if (CUDA_VERSION >= 11040)
+    exec->synchronize();
+#endif
+
     cusparse::destroy(info);
     cusparse::destroy(desc);
 }

From d0e63aeb1757ffffa3f3a7fdd0dd61001c01e59e Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Fri, 23 Jul 2021 10:43:38 +0200
Subject: [PATCH 0927/1008] add inverse scale and axpy

---
 common/matrix/dense_kernels.cpp          | 68 ++++++++++++++++++
 core/device_hooks/common_kernels.inc.cpp | 15 ++++
 core/matrix/dense.cpp                    | 40 +++++++++++
 core/matrix/dense_kernels.hpp            | 21 ++++++
 include/ginkgo/core/matrix/dense.hpp     | 48 +++++++++++++
 reference/matrix/dense_kernels.cpp       | 61 ++++++++++++++++
 reference/test/matrix/dense_kernels.cpp  | 63 ++++++++++++++++
 test/matrix/dense_kernels.cpp            | 91 ++++++++++++++++++++++++
 8 files changed, 407 insertions(+)

diff --git a/common/matrix/dense_kernels.cpp b/common/matrix/dense_kernels.cpp
index 446090b8121..8c2fd00a8c2 100644
--- a/common/matrix/dense_kernels.cpp
+++ b/common/matrix/dense_kernels.cpp
@@ -106,6 +106,31 @@ void scale(std::shared_ptr<const DefaultExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SCALE_KERNEL);
 
 
+template <typename ValueType>
+void inv_scale(std::shared_ptr<const DefaultExecutor> exec,
+               const matrix::Dense<ValueType> *alpha,
+               matrix::Dense<ValueType> *x)
+{
+    if (alpha->get_size()[1] > 1) {
+        run_kernel(
+            exec,
+            [] GKO_KERNEL(auto row, auto col, auto alpha, auto x) {
+                x(row, col) /= alpha[col];
+            },
+            x->get_size(), alpha->get_const_values(), x);
+    } else {
+        run_kernel(
+            exec,
+            [] GKO_KERNEL(auto row, auto col, auto alpha, auto x) {
+                x(row, col) /= alpha[0];
+            },
+            x->get_size(), alpha->get_const_values(), x);
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_INV_SCALE_KERNEL);
+
+
 template <typename ValueType>
 void add_scaled(std::shared_ptr<const DefaultExecutor> exec,
                 const matrix::Dense<ValueType> *alpha,
@@ -131,6 +156,31 @@ void add_scaled(std::shared_ptr<const DefaultExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_KERNEL);
 
 
+template <typename ValueType>
+void sub_scaled(std::shared_ptr<const DefaultExecutor> exec,
+                const matrix::Dense<ValueType> *alpha,
+                const matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *y)
+{
+    if (alpha->get_size()[1] > 1) {
+        run_kernel(
+            exec,
+            [] GKO_KERNEL(auto row, auto col, auto alpha, auto x, auto y) {
+                y(row, col) -= alpha[col] * x(row, col);
+            },
+            x->get_size(), alpha->get_const_values(), x, y);
+    } else {
+        run_kernel(
+            exec,
+            [] GKO_KERNEL(auto row, auto col, auto alpha, auto x, auto y) {
+                y(row, col) -= alpha[0] * x(row, col);
+            },
+            x->get_size(), alpha->get_const_values(), x, y);
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SUB_SCALED_KERNEL);
+
+
 template <typename ValueType>
 void add_scaled_diag(std::shared_ptr<const DefaultExecutor> exec,
                      const matrix::Dense<ValueType> *alpha,
@@ -149,6 +199,24 @@ void add_scaled_diag(std::shared_ptr<const DefaultExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL);
 
 
+template <typename ValueType>
+void sub_scaled_diag(std::shared_ptr<const DefaultExecutor> exec,
+                     const matrix::Dense<ValueType> *alpha,
+                     const matrix::Diagonal<ValueType> *x,
+                     matrix::Dense<ValueType> *y)
+{
+    const auto diag_values = x->get_const_values();
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto i, auto alpha, auto diag, auto y) {
+            y(i, i) -= alpha[0] * diag[i];
+        },
+        x->get_size()[0], alpha->get_const_values(), x->get_const_values(), y);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SUB_SCALED_DIAG_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void symm_permute(std::shared_ptr<const DefaultExecutor> exec,
                   const Array<IndexType> *permutation_indices,
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 4f09888e88f..0c5f46f6c1f 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -149,16 +149,31 @@ GKO_DECLARE_DENSE_SCALE_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SCALE_KERNEL);
 
+template <typename ValueType>
+GKO_DECLARE_DENSE_INV_SCALE_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_INV_SCALE_KERNEL);
+
 template <typename ValueType>
 GKO_DECLARE_DENSE_ADD_SCALED_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_KERNEL);
 
+template <typename ValueType>
+GKO_DECLARE_DENSE_SUB_SCALED_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SUB_SCALED_KERNEL);
+
 template <typename ValueType>
 GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL);
 
+template <typename ValueType>
+GKO_DECLARE_DENSE_SUB_SCALED_DIAG_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SUB_SCALED_DIAG_KERNEL);
+
 template <typename ValueType>
 GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp
index 09f19e9b360..abf0fe57ac5 100644
--- a/core/matrix/dense.cpp
+++ b/core/matrix/dense.cpp
@@ -66,8 +66,11 @@ GKO_REGISTER_OPERATION(apply, dense::apply);
 GKO_REGISTER_OPERATION(copy, dense::copy);
 GKO_REGISTER_OPERATION(fill, dense::fill);
 GKO_REGISTER_OPERATION(scale, dense::scale);
+GKO_REGISTER_OPERATION(inv_scale, dense::inv_scale);
 GKO_REGISTER_OPERATION(add_scaled, dense::add_scaled);
+GKO_REGISTER_OPERATION(sub_scaled, dense::sub_scaled);
 GKO_REGISTER_OPERATION(add_scaled_diag, dense::add_scaled_diag);
+GKO_REGISTER_OPERATION(sub_scaled_diag, dense::sub_scaled_diag);
 GKO_REGISTER_OPERATION(compute_dot, dense::compute_dot);
 GKO_REGISTER_OPERATION(compute_conj_dot, dense::compute_conj_dot);
 GKO_REGISTER_OPERATION(compute_norm2, dense::compute_norm2);
@@ -254,6 +257,20 @@ void Dense<ValueType>::fill(const ValueType value)
 }
 
 
+template <typename ValueType>
+void Dense<ValueType>::inv_scale_impl(const LinOp *alpha)
+{
+    GKO_ASSERT_EQUAL_ROWS(alpha, dim<2>(1, 1));
+    if (alpha->get_size()[1] != 1) {
+        // different alpha for each column
+        GKO_ASSERT_EQUAL_COLS(this, alpha);
+    }
+    auto exec = this->get_executor();
+    exec->run(dense::make_inv_scale(
+        make_temporary_conversion<ValueType>(alpha).get(), this));
+}
+
+
 template <typename ValueType>
 void Dense<ValueType>::scale_impl(const LinOp *alpha)
 {
@@ -291,6 +308,29 @@ void Dense<ValueType>::add_scaled_impl(const LinOp *alpha, const LinOp *b)
 }
 
 
+template <typename ValueType>
+void Dense<ValueType>::sub_scaled_impl(const LinOp *alpha, const LinOp *b)
+{
+    GKO_ASSERT_EQUAL_ROWS(alpha, dim<2>(1, 1));
+    if (alpha->get_size()[1] != 1) {
+        // different alpha for each column
+        GKO_ASSERT_EQUAL_COLS(this, alpha);
+    }
+    GKO_ASSERT_EQUAL_DIMENSIONS(this, b);
+    auto exec = this->get_executor();
+
+    if (dynamic_cast<const Diagonal<ValueType> *>(b)) {
+        exec->run(dense::make_sub_scaled_diag(
+            as<Dense<ValueType>>(alpha),
+            dynamic_cast<const Diagonal<ValueType> *>(b), this));
+    } else {
+        exec->run(dense::make_sub_scaled(
+            make_temporary_conversion<ValueType>(alpha).get(),
+            make_temporary_conversion<ValueType>(b).get(), this));
+    }
+}
+
+
 template <typename ValueType>
 void Dense<ValueType>::compute_dot_impl(const LinOp *b, LinOp *result) const
 {
diff --git a/core/matrix/dense_kernels.hpp b/core/matrix/dense_kernels.hpp
index 014c58f1b45..74633826cac 100644
--- a/core/matrix/dense_kernels.hpp
+++ b/core/matrix/dense_kernels.hpp
@@ -73,17 +73,32 @@ namespace kernels {
     void scale(std::shared_ptr<const DefaultExecutor> exec, \
                const matrix::Dense<_type> *alpha, matrix::Dense<_type> *x)
 
+#define GKO_DECLARE_DENSE_INV_SCALE_KERNEL(_type)               \
+    void inv_scale(std::shared_ptr<const DefaultExecutor> exec, \
+                   const matrix::Dense<_type> *alpha, matrix::Dense<_type> *x)
+
 #define GKO_DECLARE_DENSE_ADD_SCALED_KERNEL(_type)               \
     void add_scaled(std::shared_ptr<const DefaultExecutor> exec, \
                     const matrix::Dense<_type> *alpha,           \
                     const matrix::Dense<_type> *x, matrix::Dense<_type> *y)
 
+#define GKO_DECLARE_DENSE_SUB_SCALED_KERNEL(_type)               \
+    void sub_scaled(std::shared_ptr<const DefaultExecutor> exec, \
+                    const matrix::Dense<_type> *alpha,           \
+                    const matrix::Dense<_type> *x, matrix::Dense<_type> *y)
+
 #define GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL(_type)               \
     void add_scaled_diag(std::shared_ptr<const DefaultExecutor> exec, \
                          const matrix::Dense<_type> *alpha,           \
                          const matrix::Diagonal<_type> *x,            \
                          matrix::Dense<_type> *y)
 
+#define GKO_DECLARE_DENSE_SUB_SCALED_DIAG_KERNEL(_type)               \
+    void sub_scaled_diag(std::shared_ptr<const DefaultExecutor> exec, \
+                         const matrix::Dense<_type> *alpha,           \
+                         const matrix::Diagonal<_type> *x,            \
+                         matrix::Dense<_type> *y)
+
 #define GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL(_type)               \
     void compute_dot(std::shared_ptr<const DefaultExecutor> exec, \
                      const matrix::Dense<_type> *x,               \
@@ -240,10 +255,16 @@ namespace kernels {
     template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_SCALE_KERNEL(ValueType);                              \
     template <typename ValueType>                                           \
+    GKO_DECLARE_DENSE_INV_SCALE_KERNEL(ValueType);                          \
+    template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_ADD_SCALED_KERNEL(ValueType);                         \
     template <typename ValueType>                                           \
+    GKO_DECLARE_DENSE_SUB_SCALED_KERNEL(ValueType);                         \
+    template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL(ValueType);                    \
     template <typename ValueType>                                           \
+    GKO_DECLARE_DENSE_SUB_SCALED_DIAG_KERNEL(ValueType);                    \
+    template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL(ValueType);                        \
     template <typename ValueType>                                           \
     GKO_DECLARE_DENSE_COMPUTE_CONJ_DOT_KERNEL(ValueType);                   \
diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 95ca95e720b..8e76a036ba3 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -632,6 +632,21 @@ class Dense
         this->scale_impl(make_temporary_clone(exec, alpha).get());
     }
 
+    /**
+     * Scales the matrix with the inverse of a scalar.
+     *
+     * @param alpha  If alpha is 1x1 Dense matrix, the entire matrix is scaled
+     *               by 1 / alpha. If it is a Dense row vector of values,
+     *               then i-th column of the matrix is scaled with the inverse
+     *               of the i-th element of alpha (the number of columns of
+     *               alpha has to match the number of columns of the matrix).
+     */
+    void inv_scale(const LinOp *alpha)
+    {
+        auto exec = this->get_executor();
+        this->inv_scale_impl(make_temporary_clone(exec, alpha).get());
+    }
+
     /**
      * Adds `b` scaled by `alpha` to the matrix (aka: BLAS axpy).
      *
@@ -649,6 +664,23 @@ class Dense
                               make_temporary_clone(exec, b).get());
     }
 
+    /**
+     * Subtracts `b` scaled by `alpha` fron the matrix (aka: BLAS axpy).
+     *
+     * @param alpha  If alpha is 1x1 Dense matrix, b is scaled
+     *               by alpha. If it is a Dense row vector of values,
+     *               then i-th column of b is scaled with the i-th
+     *               element of alpha (the number of columns of alpha has to
+     *               match the number of columns of the matrix).
+     * @param b  a matrix of the same dimension as this
+     */
+    void sub_scaled(const LinOp *alpha, const LinOp *b)
+    {
+        auto exec = this->get_executor();
+        this->sub_scaled_impl(make_temporary_clone(exec, alpha).get(),
+                              make_temporary_clone(exec, b).get());
+    }
+
     /**
      * Computes the column-wise dot product of this matrix and `b`.
      *
@@ -861,6 +893,14 @@ class Dense
      */
     virtual void scale_impl(const LinOp *alpha);
 
+    /**
+     * @copydoc inv_scale(const LinOp *)
+     *
+     * @note  Other implementations of dense should override this function
+     *        instead of inv_scale(const LinOp *alpha).
+     */
+    virtual void inv_scale_impl(const LinOp *alpha);
+
     /**
      * @copydoc add_scaled(const LinOp *, const LinOp *)
      *
@@ -869,6 +909,14 @@ class Dense
      */
     virtual void add_scaled_impl(const LinOp *alpha, const LinOp *b);
 
+    /**
+     * @copydoc sub_scaled(const LinOp *, const LinOp *)
+     *
+     * @note  Other implementations of dense should override this function
+     *        instead of sub_scale(const LinOp *alpha, const LinOp *b).
+     */
+    virtual void sub_scaled_impl(const LinOp *alpha, const LinOp *b);
+
     /**
      * @copydoc compute_dot(const LinOp *, LinOp *) const
      *
diff --git a/reference/matrix/dense_kernels.cpp b/reference/matrix/dense_kernels.cpp
index 49721aab418..5f509ffb7ff 100644
--- a/reference/matrix/dense_kernels.cpp
+++ b/reference/matrix/dense_kernels.cpp
@@ -169,6 +169,29 @@ void scale(std::shared_ptr<const ReferenceExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SCALE_KERNEL);
 
 
+template <typename ValueType>
+void inv_scale(std::shared_ptr<const ReferenceExecutor> exec,
+               const matrix::Dense<ValueType> *alpha,
+               matrix::Dense<ValueType> *x)
+{
+    if (alpha->get_size()[1] == 1) {
+        for (size_type i = 0; i < x->get_size()[0]; ++i) {
+            for (size_type j = 0; j < x->get_size()[1]; ++j) {
+                x->at(i, j) /= alpha->at(0, 0);
+            }
+        }
+    } else {
+        for (size_type i = 0; i < x->get_size()[0]; ++i) {
+            for (size_type j = 0; j < x->get_size()[1]; ++j) {
+                x->at(i, j) /= alpha->at(0, j);
+            }
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_INV_SCALE_KERNEL);
+
+
 template <typename ValueType>
 void add_scaled(std::shared_ptr<const ReferenceExecutor> exec,
                 const matrix::Dense<ValueType> *alpha,
@@ -192,6 +215,29 @@ void add_scaled(std::shared_ptr<const ReferenceExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_KERNEL);
 
 
+template <typename ValueType>
+void sub_scaled(std::shared_ptr<const ReferenceExecutor> exec,
+                const matrix::Dense<ValueType> *alpha,
+                const matrix::Dense<ValueType> *x, matrix::Dense<ValueType> *y)
+{
+    if (alpha->get_size()[1] == 1) {
+        for (size_type i = 0; i < x->get_size()[0]; ++i) {
+            for (size_type j = 0; j < x->get_size()[1]; ++j) {
+                y->at(i, j) -= alpha->at(0, 0) * x->at(i, j);
+            }
+        }
+    } else {
+        for (size_type i = 0; i < x->get_size()[0]; ++i) {
+            for (size_type j = 0; j < x->get_size()[1]; ++j) {
+                y->at(i, j) -= alpha->at(0, j) * x->at(i, j);
+            }
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SUB_SCALED_KERNEL);
+
+
 template <typename ValueType>
 void add_scaled_diag(std::shared_ptr<const ReferenceExecutor> exec,
                      const matrix::Dense<ValueType> *alpha,
@@ -207,6 +253,21 @@ void add_scaled_diag(std::shared_ptr<const ReferenceExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL);
 
 
+template <typename ValueType>
+void sub_scaled_diag(std::shared_ptr<const ReferenceExecutor> exec,
+                     const matrix::Dense<ValueType> *alpha,
+                     const matrix::Diagonal<ValueType> *x,
+                     matrix::Dense<ValueType> *y)
+{
+    const auto diag_values = x->get_const_values();
+    for (size_type i = 0; i < x->get_size()[0]; i++) {
+        y->at(i, i) -= alpha->at(0, 0) * diag_values[i];
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SUB_SCALED_DIAG_KERNEL);
+
+
 template <typename ValueType>
 void compute_dot(std::shared_ptr<const ReferenceExecutor> exec,
                  const matrix::Dense<ValueType> *x,
diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp
index 47703ff8349..0d1e1c89453 100644
--- a/reference/test/matrix/dense_kernels.cpp
+++ b/reference/test/matrix/dense_kernels.cpp
@@ -337,6 +337,21 @@ TYPED_TEST(Dense, ScalesDataMixed)
 }
 
 
+TYPED_TEST(Dense, InvScalesData)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto alpha = gko::initialize<Mtx>({I<T>{0.5, -0.5}}, this->exec);
+
+    this->mtx2->inv_scale(alpha.get());
+
+    EXPECT_EQ(this->mtx2->at(0, 0), T{2.0});
+    EXPECT_EQ(this->mtx2->at(0, 1), T{2.0});
+    EXPECT_EQ(this->mtx2->at(1, 0), T{-4.0});
+    EXPECT_EQ(this->mtx2->at(1, 1), T{-4.0});
+}
+
+
 TYPED_TEST(Dense, ScalesDataWithScalar)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -352,6 +367,21 @@ TYPED_TEST(Dense, ScalesDataWithScalar)
 }
 
 
+TYPED_TEST(Dense, InvScalesDataWithScalar)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto alpha = gko::initialize<Mtx>({0.5}, this->exec);
+
+    this->mtx2->inv_scale(alpha.get());
+
+    EXPECT_EQ(this->mtx2->at(0, 0), T{2.0});
+    EXPECT_EQ(this->mtx2->at(0, 1), T{-2.0});
+    EXPECT_EQ(this->mtx2->at(1, 0), T{-4.0});
+    EXPECT_EQ(this->mtx2->at(1, 1), T{4.0});
+}
+
+
 TYPED_TEST(Dense, ScalesDataWithStride)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -405,6 +435,23 @@ TYPED_TEST(Dense, AddsScaledMixed)
 }
 
 
+TYPED_TEST(Dense, SubtractsScaled)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto alpha = gko::initialize<Mtx>({{-2.0, -1.0, 2.0}}, this->exec);
+
+    this->mtx1->sub_scaled(alpha.get(), this->mtx3.get());
+
+    EXPECT_EQ(this->mtx1->at(0, 0), T{3.0});
+    EXPECT_EQ(this->mtx1->at(0, 1), T{4.0});
+    EXPECT_EQ(this->mtx1->at(0, 2), T{-3.0});
+    EXPECT_EQ(this->mtx1->at(1, 0), T{2.5});
+    EXPECT_EQ(this->mtx1->at(1, 1), T{4.0});
+    ASSERT_EQ(this->mtx1->at(1, 2), T{-1.5});
+}
+
+
 TYPED_TEST(Dense, AddsScaledWithScalar)
 {
     using Mtx = typename TestFixture::Mtx;
@@ -448,6 +495,22 @@ TYPED_TEST(Dense, AddsScaledDiag)
 }
 
 
+TYPED_TEST(Dense, SubtractsScaledDiag)
+{
+    using Mtx = typename TestFixture::Mtx;
+    using T = typename TestFixture::value_type;
+    auto alpha = gko::initialize<Mtx>({-2.0}, this->exec);
+    auto diag = gko::matrix::Diagonal<T>::create(this->exec, 2, I<T>{3.0, 2.0});
+
+    this->mtx2->sub_scaled(alpha.get(), diag.get());
+
+    ASSERT_EQ(this->mtx2->at(0, 0), T{7.0});
+    ASSERT_EQ(this->mtx2->at(0, 1), T{-1.0});
+    ASSERT_EQ(this->mtx2->at(1, 0), T{-2.0});
+    ASSERT_EQ(this->mtx2->at(1, 1), T{6.0});
+}
+
+
 TYPED_TEST(Dense, ComputesDot)
 {
     using Mtx = typename TestFixture::Mtx;
diff --git a/test/matrix/dense_kernels.cpp b/test/matrix/dense_kernels.cpp
index ddccd148d02..306041ea4e1 100644
--- a/test/matrix/dense_kernels.cpp
+++ b/test/matrix/dense_kernels.cpp
@@ -272,6 +272,19 @@ TEST_F(Dense, SingleVectorScaleIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, SingleVectorInvScaleIsEquivalentToRef)
+{
+    set_up_vector_data(1);
+
+    x->inv_scale(alpha.get());
+    dx->inv_scale(dalpha.get());
+
+    auto result = Mtx::create(ref);
+    result->copy_from(dx.get());
+    GKO_ASSERT_MTX_NEAR(result, x, r<vtype>::value);
+}
+
+
 TEST_F(Dense, MultipleVectorScaleIsEquivalentToRef)
 {
     set_up_vector_data(20);
@@ -283,6 +296,17 @@ TEST_F(Dense, MultipleVectorScaleIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, MultipleVectorInvScaleIsEquivalentToRef)
+{
+    set_up_vector_data(20);
+
+    x->inv_scale(alpha.get());
+    dx->inv_scale(dalpha.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
+}
+
+
 TEST_F(Dense, MultipleVectorScaleWithDifferentAlphaIsEquivalentToRef)
 {
     set_up_vector_data(20, true);
@@ -294,6 +318,17 @@ TEST_F(Dense, MultipleVectorScaleWithDifferentAlphaIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, MultipleVectorInvScaleWithDifferentAlphaIsEquivalentToRef)
+{
+    set_up_vector_data(20, true);
+
+    x->inv_scale(alpha.get());
+    dx->inv_scale(dalpha.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
+}
+
+
 TEST_F(Dense, SingleVectorAddScaledIsEquivalentToRef)
 {
     set_up_vector_data(1);
@@ -305,6 +340,17 @@ TEST_F(Dense, SingleVectorAddScaledIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, SingleVectorSubtractScaledIsEquivalentToRef)
+{
+    set_up_vector_data(1);
+
+    x->sub_scaled(alpha.get(), y.get());
+    dx->sub_scaled(dalpha.get(), dy.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
+}
+
+
 TEST_F(Dense, MultipleVectorAddScaledIsEquivalentToRef)
 {
     set_up_vector_data(20);
@@ -316,6 +362,17 @@ TEST_F(Dense, MultipleVectorAddScaledIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, MultipleVectorSubtractScaledIsEquivalentToRef)
+{
+    set_up_vector_data(20);
+
+    x->sub_scaled(alpha.get(), y.get());
+    dx->sub_scaled(dalpha.get(), dy.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
+}
+
+
 TEST_F(Dense, MultipleVectorAddScaledWithDifferentAlphaIsEquivalentToRef)
 {
     set_up_vector_data(20);
@@ -327,6 +384,17 @@ TEST_F(Dense, MultipleVectorAddScaledWithDifferentAlphaIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, MultipleVectorSubtractScaledWithDifferentAlphaIsEquivalentToRef)
+{
+    set_up_vector_data(20);
+
+    x->sub_scaled(alpha.get(), y.get());
+    dx->sub_scaled(dalpha.get(), dy.get());
+
+    GKO_ASSERT_MTX_NEAR(dx, x, r<vtype>::value);
+}
+
+
 TEST_F(Dense, AddsScaledDiagIsEquivalentToRef)
 {
     auto mat = gen_mtx<Mtx>(532, 532);
@@ -350,6 +418,29 @@ TEST_F(Dense, AddsScaledDiagIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, SubtractScaledDiagIsEquivalentToRef)
+{
+    auto mat = gen_mtx<Mtx>(532, 532);
+    gko::Array<Mtx::value_type> diag_values(this->ref, 532);
+    gko::kernels::reference::components::fill_array(
+        this->ref, diag_values.get_data(), 532, Mtx::value_type{2.0});
+    auto diag = gko::matrix::Diagonal<Mtx::value_type>::create(this->ref, 532,
+                                                               diag_values);
+    auto alpha = gko::initialize<Mtx>({2.0}, this->ref);
+    auto dmat = Mtx::create(this->exec);
+    dmat->copy_from(mat.get());
+    auto ddiag = gko::matrix::Diagonal<Mtx::value_type>::create(this->exec);
+    ddiag->copy_from(diag.get());
+    auto dalpha = Mtx::create(this->exec);
+    dalpha->copy_from(alpha.get());
+
+    mat->sub_scaled(alpha.get(), diag.get());
+    dmat->sub_scaled(dalpha.get(), ddiag.get());
+
+    GKO_ASSERT_MTX_NEAR(mat, dmat, r<vtype>::value);
+}
+
+
 TEST_F(Dense, CanGatherRows)
 {
     set_up_apply_data();

From 2d13593b7b3b881bd6bd11fb9554bd3372832d46 Mon Sep 17 00:00:00 2001
From: Fabian Castelli <fabian.castelli@kit.edu>
Date: Wed, 21 Jul 2021 17:35:12 +0200
Subject: [PATCH 0928/1008] Ignore IDE generated files

---
 .gitignore | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.gitignore b/.gitignore
index d8b09ffa944..cb1af9288dd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,17 @@ compile_commands.json
 CTestTestfile.cmake
 build
 
+###IDE
+# clion
+.idea
+
+# eclipse
+.cproject
+.project
+
+# visual studio code
+.vscode
+
 ### Generated header file
 include/config.hpp
 

From 0949a642a488dd47de8da6d3a6af6b5e0a2996a0 Mon Sep 17 00:00:00 2001
From: Fabian Castelli <fabian.castelli@kit.edu>
Date: Thu, 22 Jul 2021 14:02:09 +0200
Subject: [PATCH 0929/1008] Minor update and OS generated files

---
 .gitignore | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/.gitignore b/.gitignore
index cb1af9288dd..af0a88ef513 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,6 @@
 # Created by https://www.gitignore.io/api/
 
-###CMake
+### CMake
 CMakeCache.txt
 CMakeFiles
 CMakeScripts
@@ -12,21 +12,21 @@ compile_commands.json
 CTestTestfile.cmake
 build
 
-###IDE
-# clion
+### IDE
+# Clion
 .idea
 
-# eclipse
+# Eclipse
 .cproject
 .project
 
-# visual studio code
+# Visual studio code
 .vscode
 
 ### Generated header file
 include/config.hpp
 
-###C++
+### C++
 # Prerequisites
 *.d
 
@@ -60,10 +60,25 @@ include/config.hpp
 *.out
 *.app
 
-#other
+# Others
 diff.patch
 GRTAGS
 GSYMS
 GTAGS
 GPATH
 .gitignore
+
+### OS
+# temporary and backup files
+*~
+*#
+*.swp
+
+# MacOS self-generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db

From a94218d57e79377610d0c90cca1b40b275233673 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Sun, 18 Jul 2021 23:49:16 +0200
Subject: [PATCH 0930/1008] dpcpp trans impl

---
 dpcpp/matrix/dense_kernels.dp.cpp   | 137 +++++++++++++++++++++++++++-
 dpcpp/test/matrix/dense_kernels.cpp |  24 +++++
 2 files changed, 159 insertions(+), 2 deletions(-)

diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index b17e44f9706..f8608ad55b9 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -671,6 +671,111 @@ GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(reduce_total_cols,
 GKO_ENABLE_DEFAULT_CONFIG_CALL(reduce_total_cols_call, reduce_total_cols,
                                kcfg_1d_list)
 
+template <std::uint32_t sg_size, typename ValueType, typename Closure>
+void transpose(const size_type nrows, const size_type ncols,
+               const ValueType *__restrict__ in, const size_type in_stride,
+               ValueType *__restrict__ out, const size_type out_stride,
+               Closure op, sycl::nd_item<3> item_ct1,
+               UninitializedArray<ValueType, sg_size *(sg_size + 1)> *space)
+{
+    auto local_x = item_ct1.get_local_id(2);
+    auto local_y = item_ct1.get_local_id(1);
+    auto x = item_ct1.get_group(2) * sg_size + local_x;
+    auto y = item_ct1.get_group(1) * sg_size + local_y;
+    if (y < nrows && x < ncols) {
+        (*space)[local_y * (sg_size + 1) + local_x] = op(in[y * in_stride + x]);
+    }
+
+    item_ct1.barrier(sycl::access::fence_space::local_space);
+    x = item_ct1.get_group(1) * sg_size + local_x;
+    y = item_ct1.get_group(2) * sg_size + local_y;
+    if (y < ncols && x < nrows) {
+        out[y * out_stride + x] = (*space)[local_x * (sg_size + 1) + local_y];
+    }
+}
+
+template <std::uint32_t sg_size, typename ValueType>
+__WG_BOUND__(sg_size, sg_size)
+void transpose(const size_type nrows, const size_type ncols,
+               const ValueType *__restrict__ in, const size_type in_stride,
+               ValueType *__restrict__ out, const size_type out_stride,
+               sycl::nd_item<3> item_ct1,
+               UninitializedArray<ValueType, sg_size *(sg_size + 1)> *space)
+{
+    transpose<sg_size>(
+        nrows, ncols, in, in_stride, out, out_stride,
+        [](ValueType val) { return val; }, item_ct1, space);
+}
+
+template <std::uint32_t sg_size = 32, typename ValueType>
+void transpose(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+               sycl::queue *stream, const size_type nrows,
+               const size_type ncols, const ValueType *in,
+               const size_type in_stride, ValueType *out,
+               const size_type out_stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<ValueType, sg_size *(sg_size + 1)>, 0,
+                       sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            space_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                transpose<sg_size>(
+                    nrows, ncols, in, in_stride, out, out_stride, item_ct1,
+                    (UninitializedArray<ValueType, sg_size *(sg_size + 1)> *)
+                        space_acc_ct1.get_pointer());
+            });
+    });
+}
+
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(transpose, transpose);
+GKO_ENABLE_DEFAULT_CONFIG_CALL(transpose_call, transpose, subgroup_list);
+
+
+template <std::uint32_t sg_size, typename ValueType>
+__WG_BOUND__(sg_size, sg_size)
+void conj_transpose(
+    const size_type nrows, const size_type ncols,
+    const ValueType *__restrict__ in, const size_type in_stride,
+    ValueType *__restrict__ out, const size_type out_stride,
+    sycl::nd_item<3> item_ct1,
+    UninitializedArray<ValueType, sg_size *(sg_size + 1)> *space)
+{
+    transpose<sg_size>(
+        nrows, ncols, in, in_stride, out, out_stride,
+        [](ValueType val) { return conj(val); }, item_ct1, space);
+}
+
+template <std::uint32_t sg_size = 16, typename ValueType>
+void conj_transpose(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                    sycl::queue *stream, const size_type nrows,
+                    const size_type ncols, const ValueType *in,
+                    const size_type in_stride, ValueType *out,
+                    const size_type out_stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<ValueType, sg_size *(sg_size + 1)>, 0,
+                       sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            space_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                conj_transpose<sg_size>(
+                    nrows, ncols, in, in_stride, out, out_stride, item_ct1,
+                    (UninitializedArray<ValueType, sg_size *(sg_size + 1)> *)
+                        space_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(conj_transpose, conj_transpose);
+GKO_ENABLE_DEFAULT_CONFIG_CALL(conj_transpose_call, conj_transpose,
+                               subgroup_list);
+
 
 }  // namespace kernel
 
@@ -1191,7 +1296,21 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
 template <typename ValueType>
 void transpose(std::shared_ptr<const DpcppExecutor> exec,
                const matrix::Dense<ValueType> *orig,
-               matrix::Dense<ValueType> *trans) GKO_NOT_IMPLEMENTED;
+               matrix::Dense<ValueType> *trans)
+{
+    auto size = orig->get_size();
+    auto sg_array = syn::as_array(subgroup_list);
+    auto queue = exec->get_queue();
+    const std::uint32_t cfg =
+        get_first_cfg(sg_array, [&queue](std::uint32_t cfg) {
+            return validate(queue, cfg * cfg, cfg);
+        });
+    dim3 grid(ceildiv(size[1], cfg), ceildiv(size[0], cfg));
+    dim3 block(cfg, cfg);
+    kernel::transpose_call(cfg, grid, block, 0, queue, size[0], size[1],
+                           orig->get_const_values(), orig->get_stride(),
+                           trans->get_values(), trans->get_stride());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_TRANSPOSE_KERNEL);
 
@@ -1199,7 +1318,21 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_TRANSPOSE_KERNEL);
 template <typename ValueType>
 void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Dense<ValueType> *orig,
-                    matrix::Dense<ValueType> *trans) GKO_NOT_IMPLEMENTED;
+                    matrix::Dense<ValueType> *trans)
+{
+    auto size = orig->get_size();
+    auto sg_array = syn::as_array(subgroup_list);
+    auto queue = exec->get_queue();
+    const std::uint32_t cfg =
+        get_first_cfg(sg_array, [&queue](std::uint32_t cfg) {
+            return validate(queue, cfg * cfg, cfg);
+        });
+    dim3 grid(ceildiv(size[1], cfg), ceildiv(size[0], cfg));
+    dim3 block(cfg, cfg);
+    kernel::conj_transpose_call(cfg, grid, block, 0, queue, size[0], size[1],
+                                orig->get_const_values(), orig->get_stride(),
+                                trans->get_values(), trans->get_stride());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_CONJ_TRANSPOSE_KERNEL);
 
diff --git a/dpcpp/test/matrix/dense_kernels.cpp b/dpcpp/test/matrix/dense_kernels.cpp
index cf1bbe26cd4..a21d6fd15fe 100644
--- a/dpcpp/test/matrix/dense_kernels.cpp
+++ b/dpcpp/test/matrix/dense_kernels.cpp
@@ -412,6 +412,30 @@ TEST_F(Dense, ComputeConjDotComplexIsEquivalentToRef)
 }
 
 
+TEST_F(Dense, IsTransposable)
+{
+    set_up_apply_data();
+
+    auto trans = x->transpose();
+    auto dtrans = dx->transpose();
+
+    GKO_ASSERT_MTX_NEAR(static_cast<Mtx *>(dtrans.get()),
+                        static_cast<Mtx *>(trans.get()), 0);
+}
+
+
+TEST_F(Dense, IsConjugateTransposable)
+{
+    set_up_apply_data();
+
+    auto trans = c_x->conj_transpose();
+    auto dtrans = dc_x->conj_transpose();
+
+    GKO_ASSERT_MTX_NEAR(static_cast<ComplexMtx *>(dtrans.get()),
+                        static_cast<ComplexMtx *>(trans.get()), 0);
+}
+
+
 TEST_F(Dense, ConvertToCooIsEquivalentToRef)
 {
     set_up_apply_data();

From f027f38ca67a8dfe49644d6a308addde29642880 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 19 Jul 2021 15:50:54 +0200
Subject: [PATCH 0931/1008] residual norm porting and enable the solver test

---
 dpcpp/stop/residual_norm_kernels.dp.cpp   | 165 ++++-
 dpcpp/test/stop/CMakeLists.txt            |   1 +
 dpcpp/test/stop/residual_norm_kernels.cpp | 816 ++++++++++++++++++++++
 test/solver/bicg_kernels.cpp              |   6 -
 test/solver/bicgstab_kernels.cpp          |   6 -
 test/solver/cg_kernels.cpp                |   3 -
 test/solver/cgs_kernels.cpp               |   6 -
 test/solver/fcg_kernels.cpp               |   3 -
 8 files changed, 977 insertions(+), 29 deletions(-)
 create mode 100644 dpcpp/test/stop/residual_norm_kernels.cpp

diff --git a/dpcpp/stop/residual_norm_kernels.dp.cpp b/dpcpp/stop/residual_norm_kernels.dp.cpp
index 71d8b07574d..2023b6999a4 100644
--- a/dpcpp/stop/residual_norm_kernels.dp.cpp
+++ b/dpcpp/stop/residual_norm_kernels.dp.cpp
@@ -41,6 +41,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/stop/residual_norm.hpp>
 
 
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace dpcpp {
@@ -52,7 +56,66 @@ namespace dpcpp {
 namespace residual_norm {
 
 
-constexpr int default_group_size = 512;
+constexpr int default_block_size = 256;
+
+
+template <typename ValueType>
+void residual_norm_kernel(size_type num_cols, ValueType rel_residual_goal,
+                          const ValueType *__restrict__ tau,
+                          const ValueType *__restrict__ orig_tau,
+                          uint8 stoppingId, bool setFinalized,
+                          stopping_status *__restrict__ stop_status,
+                          bool *__restrict__ device_storage,
+                          sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    if (tidx < num_cols) {
+        if (tau[tidx] < rel_residual_goal * orig_tau[tidx]) {
+            stop_status[tidx].converge(stoppingId, setFinalized);
+            device_storage[1] = true;
+        }
+        // because only false is written to all_converged, write conflicts
+        // should not cause any problem
+        else if (!stop_status[tidx].has_stopped()) {
+            device_storage[0] = false;
+        }
+    }
+}
+
+template <typename ValueType>
+void residual_norm_kernel(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                          sycl::queue *stream, size_type num_cols,
+                          ValueType rel_residual_goal, const ValueType *tau,
+                          const ValueType *orig_tau, uint8 stoppingId,
+                          bool setFinalized, stopping_status *stop_status,
+                          bool *device_storage)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                residual_norm_kernel(num_cols, rel_residual_goal, tau, orig_tau,
+                                     stoppingId, setFinalized, stop_status,
+                                     device_storage, item_ct1);
+            });
+    });
+}
+
+
+void init_kernel(bool *__restrict__ device_storage)
+{
+    device_storage[0] = true;
+    device_storage[1] = false;
+}
+
+void init_kernel(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                 sycl::queue *stream, bool *device_storage)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block),
+            [=](sycl::nd_item<3> item_ct1) { init_kernel(device_storage); });
+    });
+}
 
 
 template <typename ValueType>
@@ -62,7 +125,25 @@ void residual_norm(std::shared_ptr<const DpcppExecutor> exec,
                    ValueType rel_residual_goal, uint8 stoppingId,
                    bool setFinalized, Array<stopping_status> *stop_status,
                    Array<bool> *device_storage, bool *all_converged,
-                   bool *one_changed) GKO_NOT_IMPLEMENTED;
+                   bool *one_changed)
+{
+    static_assert(is_complex_s<ValueType>::value == false,
+                  "ValueType must not be complex in this function!");
+    init_kernel(1, 1, 0, exec->get_queue(), device_storage->get_data());
+
+    const dim3 block_size(default_block_size, 1, 1);
+    const dim3 grid_size(ceildiv(tau->get_size()[1], block_size.x), 1, 1);
+
+    residual_norm_kernel(grid_size, block_size, 0, exec->get_queue(),
+                         tau->get_size()[1], rel_residual_goal,
+                         tau->get_const_values(), orig_tau->get_const_values(),
+                         stoppingId, setFinalized, stop_status->get_data(),
+                         device_storage->get_data());
+
+    /* Represents all_converged, one_changed */
+    *all_converged = exec->copy_val_to_host(device_storage->get_const_data());
+    *one_changed = exec->copy_val_to_host(device_storage->get_const_data() + 1);
+}
 
 GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
     GKO_DECLARE_RESIDUAL_NORM_KERNEL);
@@ -79,7 +160,66 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
 namespace implicit_residual_norm {
 
 
-constexpr int default_group_size = 512;
+constexpr int default_block_size = 256;
+
+
+template <typename ValueType>
+void implicit_residual_norm_kernel(
+    size_type num_cols, remove_complex<ValueType> rel_residual_goal,
+    const ValueType *__restrict__ tau,
+    const remove_complex<ValueType> *__restrict__ orig_tau, uint8 stoppingId,
+    bool setFinalized, stopping_status *__restrict__ stop_status,
+    bool *__restrict__ device_storage, sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    if (tidx < num_cols) {
+        if (sycl::sqrt(std::abs(tau[tidx])) <
+            rel_residual_goal * orig_tau[tidx]) {
+            stop_status[tidx].converge(stoppingId, setFinalized);
+            device_storage[1] = true;
+        }
+        // because only false is written to all_converged, write conflicts
+        // should not cause any problem
+        else if (!stop_status[tidx].has_stopped()) {
+            device_storage[0] = false;
+        }
+    }
+}
+
+template <typename ValueType>
+void implicit_residual_norm_kernel(
+    dim3 grid, dim3 block, size_t dynamic_shared_memory, sycl::queue *stream,
+    size_type num_cols, remove_complex<ValueType> rel_residual_goal,
+    const ValueType *tau, const remove_complex<ValueType> *orig_tau,
+    uint8 stoppingId, bool setFinalized, stopping_status *stop_status,
+    bool *device_storage)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                implicit_residual_norm_kernel(
+                    num_cols, rel_residual_goal, tau, orig_tau, stoppingId,
+                    setFinalized, stop_status, device_storage, item_ct1);
+            });
+    });
+}
+
+
+void init_kernel(bool *__restrict__ device_storage)
+{
+    device_storage[0] = true;
+    device_storage[1] = false;
+}
+
+void init_kernel(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                 sycl::queue *stream, bool *device_storage)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block),
+            [=](sycl::nd_item<3> item_ct1) { init_kernel(device_storage); });
+    });
+}
 
 
 template <typename ValueType>
@@ -89,8 +229,23 @@ void implicit_residual_norm(
     const matrix::Dense<remove_complex<ValueType>> *orig_tau,
     remove_complex<ValueType> rel_residual_goal, uint8 stoppingId,
     bool setFinalized, Array<stopping_status> *stop_status,
-    Array<bool> *device_storage, bool *all_converged,
-    bool *one_changed) GKO_NOT_IMPLEMENTED;
+    Array<bool> *device_storage, bool *all_converged, bool *one_changed)
+{
+    init_kernel(1, 1, 0, exec->get_queue(), device_storage->get_data());
+
+    const dim3 block_size(default_block_size, 1, 1);
+    const dim3 grid_size(ceildiv(tau->get_size()[1], block_size.x), 1, 1);
+
+    implicit_residual_norm_kernel(
+        grid_size, block_size, 0, exec->get_queue(), tau->get_size()[1],
+        rel_residual_goal, tau->get_const_values(),
+        orig_tau->get_const_values(), stoppingId, setFinalized,
+        stop_status->get_data(), device_storage->get_data());
+
+    /* Represents all_converged, one_changed */
+    *all_converged = exec->copy_val_to_host(device_storage->get_const_data());
+    *one_changed = exec->copy_val_to_host(device_storage->get_const_data() + 1);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IMPLICIT_RESIDUAL_NORM_KERNEL);
 
diff --git a/dpcpp/test/stop/CMakeLists.txt b/dpcpp/test/stop/CMakeLists.txt
index dd96e64d469..0ba0781e077 100644
--- a/dpcpp/test/stop/CMakeLists.txt
+++ b/dpcpp/test/stop/CMakeLists.txt
@@ -1 +1,2 @@
 ginkgo_create_test(criterion_kernels)
+ginkgo_create_test(residual_norm_kernels)
diff --git a/dpcpp/test/stop/residual_norm_kernels.cpp b/dpcpp/test/stop/residual_norm_kernels.cpp
new file mode 100644
index 00000000000..d67dfc086c5
--- /dev/null
+++ b/dpcpp/test/stop/residual_norm_kernels.cpp
@@ -0,0 +1,816 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/stop/residual_norm.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+#ifdef GINKGO_DPCPP_SINGLE_MODE
+using value_type = float;
+constexpr auto tol = r<value_type>::value;
+#else
+using value_type = double;
+constexpr value_type tol = r<value_type>::value;
+#endif
+
+
+class ResidualNorm : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<value_type>;
+    using NormVector = gko::remove_complex<Mtx>;
+
+    ResidualNorm()
+    {
+        ref_ = gko::ReferenceExecutor::create();
+        dpcpp_ = gko::DpcppExecutor::create(0, ref_);
+        factory_ = gko::stop::ResidualNorm<value_type>::build()
+                       .with_reduction_factor(tol)
+                       .on(dpcpp_);
+        rel_factory_ = gko::stop::ResidualNorm<value_type>::build()
+                           .with_reduction_factor(tol)
+                           .with_baseline(gko::stop::mode::initial_resnorm)
+                           .on(dpcpp_);
+        abs_factory_ = gko::stop::ResidualNorm<value_type>::build()
+                           .with_reduction_factor(tol)
+                           .with_baseline(gko::stop::mode::absolute)
+                           .on(dpcpp_);
+    }
+
+    std::unique_ptr<gko::stop::ResidualNorm<value_type>::Factory> factory_;
+    std::unique_ptr<gko::stop::ResidualNorm<value_type>::Factory> rel_factory_;
+    std::unique_ptr<gko::stop::ResidualNorm<value_type>::Factory> abs_factory_;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp_;
+    std::shared_ptr<gko::ReferenceExecutor> ref_;
+};
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalForRhsResNorm)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto res_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(dpcpp_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(dpcpp_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(dpcpp_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = tol * 1.1 * rhs_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(dpcpp_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = tol * 0.9 * rhs_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalMultipleRHSForRhsResNorm)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto res_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(dpcpp_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(dpcpp_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(dpcpp_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = tol * 0.9 * rhs_norm->at(0, 0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(dpcpp_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = tol * 0.9 * rhs_norm->at(0, 1);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalForRelResNorm)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto res_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(dpcpp_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(dpcpp_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion =
+        rel_factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(dpcpp_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = tol * 1.1 * res_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(dpcpp_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = tol * 0.9 * res_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalMultipleRHSForRelResNorm)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto res_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(dpcpp_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(dpcpp_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion =
+        rel_factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(dpcpp_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = tol * 0.9 * res_norm->at(0, 0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(dpcpp_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = tol * 0.9 * res_norm->at(0, 1);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalForAbsResNorm)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto res_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(dpcpp_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(dpcpp_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion =
+        abs_factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(dpcpp_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = tol * 1.1;
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(dpcpp_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = tol * 0.9;
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNorm, WaitsTillResidualGoalMultipleRHSForAbsResNorm)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto res_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(dpcpp_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(dpcpp_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion =
+        abs_factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(dpcpp_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = tol * 0.9;
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(dpcpp_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = tol * 0.9;
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+class ResidualNormReduction : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<value_type>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
+
+    ResidualNormReduction()
+    {
+        ref_ = gko::ReferenceExecutor::create();
+        dpcpp_ = gko::DpcppExecutor::create(0, ref_);
+        factory_ = gko::stop::ResidualNormReduction<value_type>::build()
+                       .with_reduction_factor(tol)
+                       .on(dpcpp_);
+    }
+
+    std::unique_ptr<gko::stop::ResidualNormReduction<value_type>::Factory>
+        factory_;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp_;
+    std::shared_ptr<gko::ReferenceExecutor> ref_;
+};
+
+
+TEST_F(ResidualNormReduction, WaitsTillResidualGoal)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto res_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(dpcpp_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(dpcpp_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(dpcpp_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = tol * 1.1 * res_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(dpcpp_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = tol * 0.9 * res_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ResidualNormReduction, WaitsTillResidualGoalMultipleRHS)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto res_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    res->compute_norm2(res_norm.get());
+    auto d_res = Mtx::create(dpcpp_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(dpcpp_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(dpcpp_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = tol * 0.9 * res_norm->at(0, 0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(dpcpp_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = tol * 0.9 * res_norm->at(0, 1);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+class RelativeResidualNorm : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<value_type>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
+
+    RelativeResidualNorm()
+    {
+        ref_ = gko::ReferenceExecutor::create();
+        dpcpp_ = gko::DpcppExecutor::create(0, ref_);
+        factory_ = gko::stop::RelativeResidualNorm<value_type>::build()
+                       .with_tolerance(tol)
+                       .on(dpcpp_);
+    }
+
+    std::unique_ptr<gko::stop::RelativeResidualNorm<value_type>::Factory>
+        factory_;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp_;
+    std::shared_ptr<gko::ReferenceExecutor> ref_;
+};
+
+
+TEST_F(RelativeResidualNorm, WaitsTillResidualGoal)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto d_res = Mtx::create(dpcpp_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(dpcpp_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(dpcpp_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = tol * 1.1 * rhs_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(dpcpp_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = tol * 0.9 * rhs_norm->at(0);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(RelativeResidualNorm, WaitsTillResidualGoalMultipleRHS)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto d_res = Mtx::create(dpcpp_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(dpcpp_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(dpcpp_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = tol * 0.9 * rhs_norm->at(0, 0);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(dpcpp_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = tol * 0.9 * rhs_norm->at(0, 1);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+class ImplicitResidualNorm : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<value_type>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
+
+    ImplicitResidualNorm()
+    {
+        ref_ = gko::ReferenceExecutor::create();
+        dpcpp_ = gko::DpcppExecutor::create(0, ref_);
+        factory_ = gko::stop::ImplicitResidualNorm<value_type>::build()
+                       .with_reduction_factor(tol)
+                       .on(dpcpp_);
+    }
+
+    std::unique_ptr<gko::stop::ImplicitResidualNorm<value_type>::Factory>
+        factory_;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp_;
+    std::shared_ptr<gko::ReferenceExecutor> ref_;
+};
+
+
+TEST_F(ImplicitResidualNorm, WaitsTillResidualGoal)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto d_res = Mtx::create(dpcpp_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({0.0}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(dpcpp_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(dpcpp_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = std::pow(tol * 1.1 * rhs_norm->at(0), 2);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(dpcpp_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = std::pow(tol * 0.9 * rhs_norm->at(0), 2);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(ImplicitResidualNorm, WaitsTillResidualGoalMultipleRHS)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto d_res = Mtx::create(dpcpp_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    auto rhs_norm = gko::initialize<NormVector>({{0.0, 0.0}}, this->ref_);
+    gko::as<Mtx>(rhs)->compute_norm2(rhs_norm.get());
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(dpcpp_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(dpcpp_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = std::pow(tol * 0.9 * rhs_norm->at(0, 0), 2);
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(dpcpp_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = std::pow(tol * 0.9 * rhs_norm->at(0, 1), 2);
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .implicit_sq_residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+class AbsoluteResidualNorm : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<value_type>;
+    using NormVector = gko::matrix::Dense<gko::remove_complex<double>>;
+
+    AbsoluteResidualNorm()
+    {
+        ref_ = gko::ReferenceExecutor::create();
+        dpcpp_ = gko::DpcppExecutor::create(0, ref_);
+        factory_ = gko::stop::AbsoluteResidualNorm<value_type>::build()
+                       .with_tolerance(tol)
+                       .on(dpcpp_);
+    }
+
+    std::unique_ptr<gko::stop::AbsoluteResidualNorm<value_type>::Factory>
+        factory_;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp_;
+    std::shared_ptr<gko::ReferenceExecutor> ref_;
+};
+
+
+TEST_F(AbsoluteResidualNorm, WaitsTillResidualGoal)
+{
+    auto res = gko::initialize<Mtx>({100.0}, ref_);
+    auto d_res = Mtx::create(dpcpp_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(dpcpp_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 1);
+    stop_status.get_data()[0].reset();
+    stop_status.set_executor(dpcpp_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0) = tol * 1.1;
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_FALSE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(dpcpp_);
+    ASSERT_FALSE(one_changed);
+
+    res->at(0) = tol * 0.9;
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+TEST_F(AbsoluteResidualNorm, WaitsTillResidualGoalMultipleRHS)
+{
+    auto res = gko::initialize<Mtx>({{100.0, 100.0}}, ref_);
+    auto d_res = Mtx::create(dpcpp_);
+    d_res->copy_from(res.get());
+    std::shared_ptr<gko::LinOp> rhs =
+        gko::initialize<Mtx>({{10.0, 10.0}}, ref_);
+    std::shared_ptr<gko::LinOp> d_rhs = Mtx::create(dpcpp_);
+    d_rhs->copy_from(rhs.get());
+    auto criterion = factory_->generate(nullptr, d_rhs, nullptr, d_res.get());
+    bool one_changed{};
+    constexpr gko::uint8 RelativeStoppingId{1};
+    gko::Array<gko::stopping_status> stop_status(ref_, 2);
+    stop_status.get_data()[0].reset();
+    stop_status.get_data()[1].reset();
+    stop_status.set_executor(dpcpp_);
+
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+
+    res->at(0, 0) = tol * 0.9;
+    d_res->copy_from(res.get());
+    ASSERT_FALSE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[0].has_converged());
+    stop_status.set_executor(dpcpp_);
+    ASSERT_TRUE(one_changed);
+
+    res->at(0, 1) = tol * 0.9;
+    d_res->copy_from(res.get());
+    ASSERT_TRUE(
+        criterion->update()
+            .residual_norm(d_res.get())
+            .check(RelativeStoppingId, true, &stop_status, &one_changed));
+    stop_status.set_executor(ref_);
+    ASSERT_TRUE(stop_status.get_data()[1].has_converged());
+    ASSERT_TRUE(one_changed);
+}
+
+
+}  // namespace
diff --git a/test/solver/bicg_kernels.cpp b/test/solver/bicg_kernels.cpp
index b19f61bbd55..06b1726cdc4 100644
--- a/test/solver/bicg_kernels.cpp
+++ b/test/solver/bicg_kernels.cpp
@@ -267,9 +267,6 @@ TEST_F(Bicg, BicgStep2IsEquivalentToRef)
 
 TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
 {
-    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
-        GTEST_SKIP();
-    }
     auto mtx = gen_mtx(50, 50, 53);
     gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3, 5);
@@ -308,9 +305,6 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef)
 
 TEST_F(Bicg, ApplyWithSuiteSparseMatrixIsEquivalentToRef)
 {
-    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
-        GTEST_SKIP();
-    }
     auto x = gen_mtx(36, 1, 2);
     auto b = gen_mtx(36, 1, 3);
     auto d_x = Mtx::create(exec);
diff --git a/test/solver/bicgstab_kernels.cpp b/test/solver/bicgstab_kernels.cpp
index c0e75432fc9..9b5a8457cc6 100644
--- a/test/solver/bicgstab_kernels.cpp
+++ b/test/solver/bicgstab_kernels.cpp
@@ -320,9 +320,6 @@ TEST_F(Bicgstab, BicgstabStep3IsEquivalentToRef)
 
 TEST_F(Bicgstab, BicgstabApplyOneRHSIsEquivalentToRef)
 {
-    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
-        GTEST_SKIP();
-    }
     int m = 123;
     int n = 1;
     auto ref_solver = ref_bicgstab_factory->generate(mtx);
@@ -344,9 +341,6 @@ TEST_F(Bicgstab, BicgstabApplyOneRHSIsEquivalentToRef)
 
 TEST_F(Bicgstab, BicgstabApplyMultipleRHSIsEquivalentToRef)
 {
-    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
-        GTEST_SKIP();
-    }
     int m = 123;
     int n = 16;
     auto exec_solver = exec_bicgstab_factory->generate(d_mtx);
diff --git a/test/solver/cg_kernels.cpp b/test/solver/cg_kernels.cpp
index 3125003474c..edd9d33a78d 100644
--- a/test/solver/cg_kernels.cpp
+++ b/test/solver/cg_kernels.cpp
@@ -223,9 +223,6 @@ TEST_F(Cg, CgStep2IsEquivalentToRef)
 
 TEST_F(Cg, ApplyIsEquivalentToRef)
 {
-    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
-        GTEST_SKIP();
-    }
     auto mtx = gen_mtx(50, 50, 53);
     gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3, 5);
diff --git a/test/solver/cgs_kernels.cpp b/test/solver/cgs_kernels.cpp
index bd5400e77c3..cd4f0c10437 100644
--- a/test/solver/cgs_kernels.cpp
+++ b/test/solver/cgs_kernels.cpp
@@ -309,9 +309,6 @@ TEST_F(Cgs, CgsStep3IsEquivalentToRef)
 
 TEST_F(Cgs, CgsApplyOneRHSIsEquivalentToRef)
 {
-    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
-        GTEST_SKIP();
-    }
     int m = 123;
     int n = 1;
     auto ref_solver = ref_cgs_factory->generate(mtx);
@@ -333,9 +330,6 @@ TEST_F(Cgs, CgsApplyOneRHSIsEquivalentToRef)
 
 TEST_F(Cgs, CgsApplyMultipleRHSIsEquivalentToRef)
 {
-    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
-        GTEST_SKIP();
-    }
     int m = 123;
     int n = 16;
     auto exec_solver = exec_cgs_factory->generate(d_mtx);
diff --git a/test/solver/fcg_kernels.cpp b/test/solver/fcg_kernels.cpp
index 837a5cfa7e9..46fbb30cdf6 100644
--- a/test/solver/fcg_kernels.cpp
+++ b/test/solver/fcg_kernels.cpp
@@ -234,9 +234,6 @@ TEST_F(Fcg, FcgStep2IsEquivalentToRef)
 
 TEST_F(Fcg, ApplyIsEquivalentToRef)
 {
-    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
-        GTEST_SKIP();
-    }
     auto mtx = gen_mtx(50, 50, 53);
     gko::test::make_hpd(mtx.get());
     auto x = gen_mtx(50, 3, 4);

From ffd9ee6887b2174a74b50963fada01a299eec172 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 19 Jul 2021 17:05:42 +0200
Subject: [PATCH 0932/1008] porting and formating

---
 dpcpp/solver/cb_gmres_kernels.dp.cpp   | 1556 +++++++++++++++++++++++-
 dpcpp/solver/gmres_kernels.dp.cpp      |  881 +++++++++++++-
 dpcpp/test/solver/cb_gmres_kernels.cpp |  368 ++++++
 dpcpp/test/solver/gmres_kernels.cpp    |  303 +++++
 4 files changed, 3055 insertions(+), 53 deletions(-)
 create mode 100644 dpcpp/test/solver/cb_gmres_kernels.cpp
 create mode 100644 dpcpp/test/solver/gmres_kernels.cpp

diff --git a/dpcpp/solver/cb_gmres_kernels.dp.cpp b/dpcpp/solver/cb_gmres_kernels.dp.cpp
index b30fe70c76b..15bf601cd26 100644
--- a/dpcpp/solver/cb_gmres_kernels.dp.cpp
+++ b/dpcpp/solver/cb_gmres_kernels.dp.cpp
@@ -33,10 +33,32 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/cb_gmres_kernels.hpp"
 
 
+#include <algorithm>
+#include <dpcpp/base/math.hpp>
+
+
 #include <CL/sycl.hpp>
 
 
-#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/range.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/stopping_status.hpp>
+
+
+#include "accessor/range.hpp"
+#include "accessor/reduced_row_major.hpp"
+#include "accessor/scaled_reduced_row_major.hpp"
+#include "core/components/fill_array.hpp"
+#include "core/matrix/dense_kernels.hpp"
+#include "core/solver/cb_gmres_accessor.hpp"
+#include "dpcpp/base/config.hpp"
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/components/atomic.dp.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
+#include "dpcpp/components/reduction.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
+#include "dpcpp/components/uninitialized_array.hpp"
 
 
 namespace gko {
@@ -50,14 +72,1221 @@ namespace dpcpp {
 namespace cb_gmres {
 
 
+constexpr int default_block_size = 512;
+constexpr int default_dot_dim = 32;
+constexpr int default_dot_size = default_dot_dim * default_dot_dim;
+
+
+// Must be called with at least `max(stride_b * num_rows, krylov_dim *
+// num_cols)` threads in total.
+template <size_type block_size, typename ValueType>
+void initialize_1_kernel(
+    size_type num_rows, size_type num_cols, size_type krylov_dim,
+    const ValueType *__restrict__ b, size_type stride_b,
+    ValueType *__restrict__ residual, size_type stride_residual,
+    ValueType *__restrict__ givens_sin, size_type stride_sin,
+    ValueType *__restrict__ givens_cos, size_type stride_cos,
+    stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+
+    const auto row_idx = global_id / stride_b;
+    const auto col_idx = global_id % stride_b;
+
+    if (global_id < num_cols) {
+        stop_status[global_id].reset();
+    }
+
+    if (row_idx < num_rows && col_idx < num_cols) {
+        residual[row_idx * stride_residual + col_idx] =
+            b[row_idx * stride_b + col_idx];
+    }
+
+    if (global_id < krylov_dim * num_cols) {
+        const auto row_givens = global_id / num_cols;
+        const auto col_givens = global_id % num_cols;
+
+        givens_sin[row_givens * stride_sin + col_givens] = zero<ValueType>();
+        givens_cos[row_givens * stride_cos + col_givens] = zero<ValueType>();
+    }
+}
+
+template <size_type block_size, typename ValueType>
+void initialize_1_kernel(dim3 grid, dim3 block,
+                         gko::size_type dynamic_shared_memory,
+                         sycl::queue *stream, size_type num_rows,
+                         size_type num_cols, size_type krylov_dim,
+                         const ValueType *b, size_type stride_b,
+                         ValueType *residual, size_type stride_residual,
+                         ValueType *givens_sin, size_type stride_sin,
+                         ValueType *givens_cos, size_type stride_cos,
+                         stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                initialize_1_kernel<block_size>(
+                    num_rows, num_cols, krylov_dim, b, stride_b, residual,
+                    stride_residual, givens_sin, stride_sin, givens_cos,
+                    stride_cos, stop_status, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType>
+void calculate_sin_and_cos_kernel(size_type col_idx, size_type num_cols,
+                                  size_type iter, const ValueType &this_hess,
+                                  const ValueType &next_hess,
+                                  ValueType *givens_sin, size_type stride_sin,
+                                  ValueType *givens_cos, size_type stride_cos,
+                                  ValueType &register_sin,
+                                  ValueType &register_cos)
+{
+    if (this_hess == zero<ValueType>()) {
+        register_cos = zero<ValueType>();
+        register_sin = one<ValueType>();
+    } else {
+        const auto scale = std::abs(this_hess) + std::abs(next_hess);
+        const auto hypotenuse =
+            scale *
+            sycl::sqrt(
+                std::abs(this_hess / scale) * std::abs(this_hess / scale) +
+                std::abs(next_hess / scale) * std::abs(next_hess / scale));
+        /*
+        DPCT1007:1: Migration of this CUDA API is not supported by the Intel(R)
+        DPC++ Compatibility Tool.
+        */
+        register_cos = conj(this_hess) / hypotenuse;
+        /*
+        DPCT1007:2: Migration of this CUDA API is not supported by the Intel(R)
+        DPC++ Compatibility Tool.
+        */
+        register_sin = conj(next_hess) / hypotenuse;
+    }
+    givens_cos[iter * stride_cos + col_idx] = register_cos;
+    givens_sin[iter * stride_sin + col_idx] = register_sin;
+}
+
+
+template <typename ValueType>
+void calculate_residual_norm_kernel(size_type col_idx, size_type num_cols,
+                                    size_type iter,
+                                    const ValueType &register_sin,
+                                    const ValueType &register_cos,
+                                    remove_complex<ValueType> *residual_norm,
+                                    ValueType *residual_norm_collection,
+                                    size_type stride_residual_norm_collection)
+{
+    const auto this_rnc =
+        residual_norm_collection[iter * stride_residual_norm_collection +
+                                 col_idx];
+    /*
+    DPCT1007:3: Migration of this CUDA API is not supported by the Intel(R)
+    DPC++ Compatibility Tool.
+    */
+    const auto next_rnc = -conj(register_sin) * this_rnc;
+    residual_norm_collection[iter * stride_residual_norm_collection + col_idx] =
+        register_cos * this_rnc;
+    residual_norm[col_idx] = std::abs(next_rnc);
+    residual_norm_collection[(iter + 1) * stride_residual_norm_collection +
+                             col_idx] = next_rnc;
+}
+
+
+// Must be called with at least `num_cols` threads in total.
+template <size_type block_size, typename ValueType>
+void givens_rotation_kernel(
+    size_type num_rows, size_type num_cols, size_type iter,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    ValueType *__restrict__ givens_sin, size_type stride_sin,
+    ValueType *__restrict__ givens_cos, size_type stride_cos,
+    remove_complex<ValueType> *__restrict__ residual_norm,
+    ValueType *__restrict__ residual_norm_collection,
+    size_type stride_residual_norm_collection,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
+{
+    const auto col_idx = thread::get_thread_id_flat(item_ct1);
+
+    if (col_idx >= num_cols || stop_status[col_idx].has_stopped()) {
+        return;
+    }
+
+    auto this_hess = hessenberg_iter[col_idx];
+    auto next_hess = hessenberg_iter[stride_hessenberg + col_idx];
+    for (size_type i = 0; i < iter; ++i) {
+        const auto cos = givens_cos[i * stride_cos + col_idx];
+        const auto sin = givens_sin[i * stride_sin + col_idx];
+        hessenberg_iter[i * stride_hessenberg + col_idx] =
+            cos * this_hess + sin * next_hess;
+        /*
+        DPCT1007:4: Migration of this CUDA API is not supported by the Intel(R)
+        DPC++ Compatibility Tool.
+        */
+        this_hess = conj(cos) * next_hess - conj(sin) * this_hess;
+        next_hess = hessenberg_iter[(i + 2) * stride_hessenberg + col_idx];
+    }
+    // for j in 0:iter - 1
+    //     temp             =  cos(j)*hessenberg(j) +
+    //                         sin(j)*hessenberg(j+1)
+    //     hessenberg(j+1)  = -sin(j)*hessenberg(j) +
+    //                         cos(j)*hessenberg(j+1)
+    //     hessenberg(j)    =  temp;
+    // end
+
+    ValueType register_sin;
+    ValueType register_cos;
+    calculate_sin_and_cos_kernel(col_idx, num_cols, iter, this_hess, next_hess,
+                                 givens_sin, stride_sin, givens_cos, stride_cos,
+                                 register_sin, register_cos);
+    // Calculate sin and cos on hessenberg(iter) and hessenberg(iter+1)
+
+    hessenberg_iter[iter * stride_hessenberg + col_idx] =
+        register_cos * this_hess + register_sin * next_hess;
+    hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] =
+        zero<ValueType>();
+    // hessenberg(iter)   = cos(iter)*hessenberg(iter) +
+    //                      sin(iter)*hessenberg(iter+1)
+    // hessenberg(iter+1) = 0
+
+    calculate_residual_norm_kernel(
+        col_idx, num_cols, iter, register_sin, register_cos, residual_norm,
+        residual_norm_collection, stride_residual_norm_collection);
+    // Calculate residual norm
+}
+
+template <size_type block_size, typename ValueType>
+void givens_rotation_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type num_rows, size_type num_cols, size_type iter,
+    ValueType *hessenberg_iter, size_type stride_hessenberg,
+    ValueType *givens_sin, size_type stride_sin, ValueType *givens_cos,
+    size_type stride_cos, remove_complex<ValueType> *residual_norm,
+    ValueType *residual_norm_collection,
+    size_type stride_residual_norm_collection,
+    const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                givens_rotation_kernel<block_size>(
+                    num_rows, num_cols, iter, hessenberg_iter,
+                    stride_hessenberg, givens_sin, stride_sin, givens_cos,
+                    stride_cos, residual_norm, residual_norm_collection,
+                    stride_residual_norm_collection, stop_status, item_ct1);
+            });
+    });
+}
+
+
+// Must be called with at least `num_rhs` threads in total.
+template <size_type block_size, typename ValueType>
+void solve_upper_triangular_kernel(
+    size_type num_cols, size_type num_rhs,
+    const ValueType *__restrict__ residual_norm_collection,
+    size_type stride_residual_norm_collection,
+    const ValueType *__restrict__ hessenberg, size_type stride_hessenberg,
+    ValueType *__restrict__ y, size_type stride_y,
+    const size_type *__restrict__ final_iter_nums, sycl::nd_item<3> item_ct1)
+{
+    const auto col_idx = thread::get_thread_id_flat(item_ct1);
+
+    if (col_idx >= num_rhs) {
+        return;
+    }
+
+    for (int i = final_iter_nums[col_idx] - 1; i >= 0; --i) {
+        auto temp =
+            residual_norm_collection[i * stride_residual_norm_collection +
+                                     col_idx];
+        for (size_type j = i + 1; j < final_iter_nums[col_idx]; ++j) {
+            temp -= hessenberg[i * stride_hessenberg + j * num_rhs + col_idx] *
+                    y[j * stride_y + col_idx];
+        }
+
+        y[i * stride_y + col_idx] =
+            temp / hessenberg[i * stride_hessenberg + i * num_rhs + col_idx];
+    }
+    // Solve upper triangular.
+    // y = hessenberg \ residual_norm_collection
+}
+
+template <size_type block_size, typename ValueType>
+void solve_upper_triangular_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type num_cols, size_type num_rhs,
+    const ValueType *residual_norm_collection,
+    size_type stride_residual_norm_collection, const ValueType *hessenberg,
+    size_type stride_hessenberg, ValueType *y, size_type stride_y,
+    const size_type *final_iter_nums)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                solve_upper_triangular_kernel<block_size>(
+                    num_cols, num_rhs, residual_norm_collection,
+                    stride_residual_norm_collection, hessenberg,
+                    stride_hessenberg, y, stride_y, final_iter_nums, item_ct1);
+            });
+    });
+}
+
+template <typename ValueType>
+void zero_matrix_kernel(size_type m, size_type n, size_type stride,
+                        ValueType *__restrict__ array,
+                        sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    if (tidx < n) {
+        auto pos = tidx;
+        for (size_type k = 0; k < m; ++k) {
+            array[pos] = zero<ValueType>();
+            pos += stride;
+        }
+    }
+}
+
+template <typename ValueType>
+void zero_matrix_kernel(dim3 grid, dim3 block,
+                        gko::size_type dynamic_shared_memory,
+                        sycl::queue *stream, size_type m, size_type n,
+                        size_type stride, ValueType *array)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             zero_matrix_kernel(m, n, stride, array, item_ct1);
+                         });
+    });
+}
+
+
+// Must be called with at least `num_rows * stride_krylov` threads in total.
+template <size_type block_size, typename ValueType, typename Accessor3d>
+void initialize_2_1_kernel(size_type num_rows, size_type num_rhs,
+                           size_type krylov_dim, Accessor3d krylov_bases,
+                           ValueType *__restrict__ residual_norm_collection,
+                           size_type stride_residual_nc,
+                           sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto krylov_stride =
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::get_stride(
+            krylov_bases);
+    // krylov indices
+    const auto krylov_idx = global_id / krylov_stride[0];
+    const auto reminder = global_id % krylov_stride[0];
+    const auto krylov_row = reminder / krylov_stride[1];
+    const auto rhs = reminder % krylov_stride[1];
+
+    // residual_norm indices (separated for better coalesced access)
+    const auto residual_row = global_id / stride_residual_nc;
+    const auto residual_col = global_id % stride_residual_nc;
+
+    if (krylov_idx < krylov_dim + 1 && krylov_row < num_rows && rhs < num_rhs) {
+        krylov_bases(krylov_idx, krylov_row, rhs) = zero<ValueType>();
+    }
+
+    if (residual_row < krylov_dim + 1 && residual_col < num_rhs) {
+        residual_norm_collection[residual_row * stride_residual_nc +
+                                 residual_col] = zero<ValueType>();
+    }
+}
+
+template <size_type block_size, typename ValueType, typename Accessor3d>
+void initialize_2_1_kernel(dim3 grid, dim3 block,
+                           gko::size_type dynamic_shared_memory,
+                           sycl::queue *stream, size_type num_rows,
+                           size_type num_rhs, size_type krylov_dim,
+                           Accessor3d krylov_bases,
+                           ValueType *residual_norm_collection,
+                           size_type stride_residual_nc)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                initialize_2_1_kernel<block_size>(
+                    num_rows, num_rhs, krylov_dim, krylov_bases,
+                    residual_norm_collection, stride_residual_nc, item_ct1);
+            });
+    });
+}
+
+
+// Must be called with at least `num_rows * num_rhs` threads in total.
+template <size_type block_size, typename ValueType, typename Accessor3d>
+void initialize_2_2_kernel(
+    size_type num_rows, size_type num_rhs,
+    const ValueType *__restrict__ residual, size_type stride_residual,
+    const remove_complex<ValueType> *__restrict__ residual_norm,
+    ValueType *__restrict__ residual_norm_collection, Accessor3d krylov_bases,
+    ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
+    size_type *__restrict__ final_iter_nums, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto krylov_stride =
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::get_stride(
+            krylov_bases);
+    const auto row_idx = global_id / krylov_stride[1];
+    const auto col_idx = global_id % krylov_stride[1];
+
+    if (global_id < num_rhs) {
+        residual_norm_collection[global_id] = residual_norm[global_id];
+        final_iter_nums[global_id] = 0;
+    }
+
+    if (row_idx < num_rows && col_idx < num_rhs) {
+        auto value = residual[row_idx * stride_residual + col_idx] /
+                     residual_norm[col_idx];
+        krylov_bases(0, row_idx, col_idx) = value;
+        next_krylov_basis[row_idx * stride_next_krylov + col_idx] = value;
+    }
+}
+
+template <size_type block_size, typename ValueType, typename Accessor3d>
+void initialize_2_2_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type num_rows, size_type num_rhs,
+    const ValueType *residual, size_type stride_residual,
+    const remove_complex<ValueType> *residual_norm,
+    ValueType *residual_norm_collection, Accessor3d krylov_bases,
+    ValueType *next_krylov_basis, size_type stride_next_krylov,
+    size_type *final_iter_nums)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                initialize_2_2_kernel<block_size>(
+                    num_rows, num_rhs, residual, stride_residual, residual_norm,
+                    residual_norm_collection, krylov_bases, next_krylov_basis,
+                    stride_next_krylov, final_iter_nums, item_ct1);
+            });
+    });
+}
+
+
+void increase_final_iteration_numbers_kernel(
+    size_type *__restrict__ final_iter_nums,
+    const stopping_status *__restrict__ stop_status, size_type total_number,
+    sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    if (global_id < total_number) {
+        final_iter_nums[global_id] += !stop_status[global_id].has_stopped();
+    }
+}
+
+void increase_final_iteration_numbers_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type *final_iter_nums,
+    const stopping_status *stop_status, size_type total_number)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                increase_final_iteration_numbers_kernel(
+                    final_iter_nums, stop_status, total_number, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType>
+void multinorm2_kernel(
+    size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov, remove_complex<ValueType> *__restrict__ norms,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1,
+    UninitializedArray<rc_vtype, default_dot_dim *(default_dot_dim + 1)>
+        *reduction_helper_array)
+{
+    using rc_vtype = remove_complex<ValueType>;
+    const auto tidx = item_ct1.get_local_id(2);
+    const auto tidy = item_ct1.get_local_id(1);
+    const auto col_idx = item_ct1.get_group(2) * default_dot_dim + tidx;
+    const auto num = ceildiv(num_rows, item_ct1.get_group_range(1));
+    const auto start_row = item_ct1.get_group(1) * num;
+    const auto end_row = ((item_ct1.get_group(1) + 1) * num > num_rows)
+                             ? num_rows
+                             : (item_ct1.get_group(1) + 1) * num;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+
+    rc_vtype *__restrict__ reduction_helper = (*reduction_helper_array);
+    rc_vtype local_res = zero<rc_vtype>();
+    if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+        for (size_type i = start_row + tidy; i < end_row;
+             i += default_dot_dim) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            local_res += squared_norm(next_krylov_basis[next_krylov_idx]);
+        }
+    }
+    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
+    group::this_thread_block(item_ct1).sync();
+    local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
+    const auto tile_block = group::tiled_partition<default_dot_dim>(
+        group::this_thread_block(item_ct1));
+    const auto sum =
+        reduce(tile_block, local_res,
+               [](const rc_vtype &a, const rc_vtype &b) { return a + b; });
+    const auto new_col_idx = item_ct1.get_group(2) * default_dot_dim + tidy;
+    if (tidx == 0 && new_col_idx < num_cols &&
+        !stop_status[new_col_idx].has_stopped()) {
+        const auto norms_idx = new_col_idx;
+        atomic_add(norms + norms_idx, sum);
+    }
+}
+
+template <typename ValueType>
+void multinorm2_kernel(dim3 grid, dim3 block,
+                       gko::size_type dynamic_shared_memory,
+                       sycl::queue *stream, size_type num_rows,
+                       size_type num_cols, const ValueType *next_krylov_basis,
+                       size_type stride_next_krylov,
+                       remove_complex<ValueType> *norms,
+                       const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<rc_vtype, default_dot_dim *(
+                                                        default_dot_dim + 1)>,
+                       0, sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            reduction_helper_array_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                multinorm2_kernel(num_rows, num_cols, next_krylov_basis,
+                                  stride_next_krylov, norms, stop_status,
+                                  item_ct1,
+                                  reduction_helper_array_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+template <typename ValueType>
+void multinorminf_without_stop_kernel(
+    size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov, remove_complex<ValueType> *__restrict__ norms,
+    size_type stride_norms, sycl::nd_item<3> item_ct1,
+    UninitializedArray<rc_vtype, default_dot_dim *(default_dot_dim + 1)>
+        *reduction_helper_array)
+{
+    using rc_vtype = remove_complex<ValueType>;
+    const auto tidx = item_ct1.get_local_id(2);
+    const auto tidy = item_ct1.get_local_id(1);
+    const auto col_idx = item_ct1.get_group(2) * default_dot_dim + tidx;
+    const auto num = ceildiv(num_rows, item_ct1.get_group_range(1));
+    const auto start_row = item_ct1.get_group(1) * num;
+    const auto end_row = ((item_ct1.get_group(1) + 1) * num > num_rows)
+                             ? num_rows
+                             : (item_ct1.get_group(1) + 1) * num;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+
+    rc_vtype *__restrict__ reduction_helper = (*reduction_helper_array);
+    rc_vtype local_max = zero<rc_vtype>();
+    if (col_idx < num_cols) {
+        for (size_type i = start_row + tidy; i < end_row;
+             i += default_dot_dim) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            local_max =
+                (local_max >= std::abs(next_krylov_basis[next_krylov_idx]))
+                    ? local_max
+                    : std::abs(next_krylov_basis[next_krylov_idx]);
+        }
+    }
+    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_max;
+    group::this_thread_block(item_ct1).sync();
+    local_max = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
+    const auto tile_block = group::tiled_partition<default_dot_dim>(
+        group::this_thread_block(item_ct1));
+    const auto value =
+        reduce(tile_block, local_max, [](const rc_vtype &a, const rc_vtype &b) {
+            return ((a >= b) ? a : b);
+        });
+    const auto new_col_idx = item_ct1.get_group(2) * default_dot_dim + tidy;
+    if (tidx == 0 && new_col_idx < num_cols) {
+        const auto norms_idx = new_col_idx;
+        atomic_max(norms + norms_idx, value);
+    }
+}
+
+template <typename ValueType>
+void multinorminf_without_stop_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type num_rows, size_type num_cols,
+    const ValueType *next_krylov_basis, size_type stride_next_krylov,
+    remove_complex<ValueType> *norms, size_type stride_norms)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<rc_vtype, default_dot_dim *(
+                                                        default_dot_dim + 1)>,
+                       0, sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            reduction_helper_array_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                multinorminf_without_stop_kernel(
+                    num_rows, num_cols, next_krylov_basis, stride_next_krylov,
+                    norms, stride_norms, item_ct1,
+                    reduction_helper_array_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+// ONLY computes the inf-norm (into norms2) when compute_inf is true
+template <bool compute_inf, typename ValueType>
+void multinorm2_inf_kernel(
+    size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov,
+    remove_complex<ValueType> *__restrict__ norms1,
+    remove_complex<ValueType> *__restrict__ norms2,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1,
+    UninitializedArray<rc_vtype, (1 + compute_inf) *
+                                     default_dot_dim *(default_dot_dim + 1)>
+        *reduction_helper_array)
+{
+    using rc_vtype = remove_complex<ValueType>;
+    const auto tidx = item_ct1.get_local_id(2);
+    const auto tidy = item_ct1.get_local_id(1);
+    const auto col_idx = item_ct1.get_group(2) * default_dot_dim + tidx;
+    const auto num = ceildiv(num_rows, item_ct1.get_group_range(1));
+    const auto start_row = item_ct1.get_group(1) * num;
+    const auto end_row = ((item_ct1.get_group(1) + 1) * num > num_rows)
+                             ? num_rows
+                             : (item_ct1.get_group(1) + 1) * num;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+
+    rc_vtype *__restrict__ reduction_helper_add = (*reduction_helper_array);
+    rc_vtype *__restrict__ reduction_helper_max =
+        static_cast<rc_vtype *>((*reduction_helper_array)) +
+        default_dot_dim * (default_dot_dim + 1);
+    rc_vtype local_res = zero<rc_vtype>();
+    rc_vtype local_max = zero<rc_vtype>();
+    if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+        for (size_type i = start_row + tidy; i < end_row;
+             i += default_dot_dim) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            const auto num = next_krylov_basis[next_krylov_idx];
+            local_res += squared_norm(num);
+            if (compute_inf) {
+                local_max =
+                    ((local_max >= std::abs(num)) ? local_max : std::abs(num));
+            }
+        }
+    }
+    // Add reduction
+    reduction_helper_add[tidx * (default_dot_dim + 1) + tidy] = local_res;
+    if (compute_inf) {
+        reduction_helper_max[tidx * (default_dot_dim + 1) + tidy] = local_max;
+    }
+    group::this_thread_block(item_ct1).sync();
+    local_res = reduction_helper_add[tidy * (default_dot_dim + 1) + tidx];
+    const auto tile_block = group::tiled_partition<default_dot_dim>(
+        group::this_thread_block(item_ct1));
+    const auto sum =
+        reduce(tile_block, local_res,
+               [](const rc_vtype &a, const rc_vtype &b) { return a + b; });
+    rc_vtype reduced_max{};
+    if (compute_inf) {
+        local_max = reduction_helper_max[tidy * (default_dot_dim + 1) + tidx];
+        reduced_max = reduce(tile_block, local_max,
+                             [](const rc_vtype &a, const rc_vtype &b) {
+                                 return ((a >= b) ? a : b);
+                             });
+    }
+    const auto new_col_idx = item_ct1.get_group(2) * default_dot_dim + tidy;
+    if (tidx == 0 && new_col_idx < num_cols &&
+        !stop_status[new_col_idx].has_stopped()) {
+        const auto norms_idx = new_col_idx;
+        atomic_add(norms1 + norms_idx, sum);
+        if (compute_inf) {
+            atomic_max(norms2 + norms_idx, reduced_max);
+        }
+    }
+}
+
+template <bool compute_inf, typename ValueType>
+void multinorm2_inf_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type num_rows, size_type num_cols,
+    const ValueType *next_krylov_basis, size_type stride_next_krylov,
+    remove_complex<ValueType> *norms1, remove_complex<ValueType> *norms2,
+    const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<
+            UninitializedArray<rc_vtype,
+                               (1 + compute_inf) *
+                                   default_dot_dim *(default_dot_dim + 1)>,
+            0, sycl::access_mode::read_write, sycl::access::target::local>
+            reduction_helper_array_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                multinorm2_inf_kernel<compute_inf>(
+                    num_rows, num_cols, next_krylov_basis, stride_next_krylov,
+                    norms1, norms2, stop_status, item_ct1,
+                    reduction_helper_array_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+template <int dot_dim, typename ValueType, typename Accessor3d>
+void multidot_kernel(
+    size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov, const Accessor3d krylov_bases,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1,
+    UninitializedArray<ValueType, dot_dim * dot_dim> *reduction_helper_array)
+{
+    /*
+     * In general in this kernel:
+     * grid_dim
+     *   x: for col_idx (^= which right hand side)
+     *   y: for row_idx
+     *   z: for num_iters (number of krylov vectors)
+     * block_dim
+     *   x: for col_idx (must be < dot_dim)
+     *   y: for row_idx (must be < dot_dim)
+     *   (z not used, must be set to 1 in dim)
+     */
+    const size_type tidx = item_ct1.get_local_id(2);
+    const size_type tidy = item_ct1.get_local_id(1);
+    const size_type col_idx =
+        item_ct1.get_group(2) * item_ct1.get_local_range().get(2) +
+        item_ct1.get_local_id(2);
+    const size_type num_rows_per_thread =
+        ceildiv(num_rows, item_ct1.get_group_range(1));
+    const size_type start_row =
+        item_ct1.get_group(1) * num_rows_per_thread + item_ct1.get_local_id(1);
+    const auto end_row =
+        min((item_ct1.get_group(1) + 1) * num_rows_per_thread, num_rows);
+    const size_type k = item_ct1.get_group(0);
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+
+    ValueType *__restrict__ reduction_helper = (*reduction_helper_array);
+
+    ValueType local_res = zero<ValueType>();
+    if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+        for (size_type i = start_row; i < end_row;
+             i += item_ct1.get_local_range().get(1)) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            ValueType other_basis = krylov_bases(k, i, col_idx);
+            /*
+            DPCT1007:14: Migration of this CUDA API is not supported by the
+            Intel(R) DPC++ Compatibility Tool.
+            */
+            local_res += next_krylov_basis[next_krylov_idx] * conj(other_basis);
+        }
+    }
+    // Transpose local_res, so each warp contains a local_res from the same
+    // right hand side
+    reduction_helper[tidx * dot_dim + tidy] = local_res;
+    auto thread_block = group::this_thread_block(item_ct1);
+    thread_block.sync();
+    local_res = reduction_helper[tidy * dot_dim + tidx];
+    const auto new_col_idx =
+        item_ct1.get_group(2) * item_ct1.get_local_range().get(2) + tidy;
+    const auto tile_block = group::tiled_partition<dot_dim>(thread_block);
+    const auto sum =
+        reduce(tile_block, local_res,
+               [](const ValueType &a, const ValueType &b) { return a + b; });
+    if (tidx == 0 && new_col_idx < num_cols &&
+        !stop_status[new_col_idx].has_stopped()) {
+        const auto hessenberg_idx = k * stride_hessenberg + new_col_idx;
+        atomic_add(hessenberg_iter + hessenberg_idx, sum);
+    }
+}
+
+template <int dot_dim, typename ValueType, typename Accessor3d>
+void multidot_kernel(dim3 grid, dim3 block,
+                     gko::size_type dynamic_shared_memory, sycl::queue *stream,
+                     size_type num_rows, size_type num_cols,
+                     const ValueType *next_krylov_basis,
+                     size_type stride_next_krylov,
+                     const Accessor3d krylov_bases, ValueType *hessenberg_iter,
+                     size_type stride_hessenberg,
+                     const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<ValueType, dot_dim * dot_dim>, 0,
+                       sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            reduction_helper_array_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                multidot_kernel<dot_dim>(
+                    num_rows, num_cols, next_krylov_basis, stride_next_krylov,
+                    krylov_bases, hessenberg_iter, stride_hessenberg,
+                    stop_status, item_ct1,
+                    (UninitializedArray<ValueType, dot_dim * dot_dim> *)
+                        reduction_helper_array_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+template <int block_size, typename ValueType, typename Accessor3d>
+void singledot_kernel(
+    size_type num_rows, const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov, const Accessor3d krylov_bases,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1,
+    UninitializedArray<ValueType, block_size> *reduction_helper_array)
+{
+    /*
+     * In general in this kernel:
+     * grid_dim
+     *   x: for row_idx
+     *   y: for num_iters (number of krylov vectors)
+     * block_dim
+     *   x: for row_idx (must be block_size)
+     *   (y and z not used, must be set to 1 in dim)
+     */
+    const size_type tidx = item_ct1.get_local_id(2);
+    constexpr size_type col_idx{0};
+    const size_type k = item_ct1.get_group(1);
+    const size_type num_rows_per_thread =
+        ceildiv(num_rows, item_ct1.get_group_range(2));
+    const size_type start_row =
+        item_ct1.get_group(2) * num_rows_per_thread + item_ct1.get_local_id(2);
+    const auto end_row =
+        min((item_ct1.get_group(2) + 1) * num_rows_per_thread, num_rows);
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+
+    ValueType *__restrict__ reduction_helper = (*reduction_helper_array);
+
+    ValueType local_res = zero<ValueType>();
+    if (!stop_status[col_idx].has_stopped()) {
+        for (size_type i = start_row; i < end_row; i += block_size) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            ValueType other_basis = krylov_bases(k, i, col_idx);
+            /*
+            DPCT1007:16: Migration of this CUDA API is not supported by the
+            Intel(R) DPC++ Compatibility Tool.
+            */
+            local_res += next_krylov_basis[next_krylov_idx] * conj(other_basis);
+        }
+    }
+    // Transpose local_res, so each warp contains a local_res from the same
+    // right hand side
+    reduction_helper[tidx] = local_res;
+    auto thread_block = group::this_thread_block(item_ct1);
+    thread_block.sync();
+    reduce(thread_block, reduction_helper,
+           [](const ValueType &a, const ValueType &b) { return a + b; });
+    if (tidx == 0 && !stop_status[col_idx].has_stopped()) {
+        const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+        atomic_add(hessenberg_iter + hessenberg_idx, reduction_helper[0]);
+    }
+}
+
+template <int block_size, typename ValueType, typename Accessor3d>
+void singledot_kernel(dim3 grid, dim3 block,
+                      gko::size_type dynamic_shared_memory, sycl::queue *stream,
+                      size_type num_rows, const ValueType *next_krylov_basis,
+                      size_type stride_next_krylov,
+                      const Accessor3d krylov_bases, ValueType *hessenberg_iter,
+                      size_type stride_hessenberg,
+                      const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<ValueType, block_size>, 0,
+                       sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            reduction_helper_array_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                singledot_kernel<block_size>(
+                    num_rows, next_krylov_basis, stride_next_krylov,
+                    krylov_bases, hessenberg_iter, stride_hessenberg,
+                    stop_status, item_ct1,
+                    (UninitializedArray<ValueType, block_size> *)
+                        reduction_helper_array_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename Accessor3d>
+void update_next_krylov_kernel(
+    size_type num_iters, size_type num_rows, size_type num_cols,
+    ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
+    const Accessor3d krylov_bases,
+    const ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_idx = global_id / stride_next_krylov;
+    const auto col_idx = global_id % stride_next_krylov;
+
+    if (row_idx < num_rows && col_idx < num_cols &&
+        !stop_status[col_idx].has_stopped()) {
+        const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+        auto local_res = next_krylov_basis[next_krylov_idx];
+        for (size_type k = 0; k < num_iters; ++k) {
+            const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+
+            local_res -= hessenberg_iter[hessenberg_idx] *
+                         krylov_bases(k, row_idx, col_idx);
+        }
+        next_krylov_basis[next_krylov_idx] = local_res;
+    }
+}
+
+template <int block_size, typename ValueType, typename Accessor3d>
+void update_next_krylov_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type num_iters, size_type num_rows,
+    size_type num_cols, ValueType *next_krylov_basis,
+    size_type stride_next_krylov, const Accessor3d krylov_bases,
+    const ValueType *hessenberg_iter, size_type stride_hessenberg,
+    const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                update_next_krylov_kernel<block_size>(
+                    num_iters, num_rows, num_cols, next_krylov_basis,
+                    stride_next_krylov, krylov_bases, hessenberg_iter,
+                    stride_hessenberg, stop_status, item_ct1);
+            });
+    });
+}
+
+
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename Accessor3d>
+void update_next_krylov_and_add_kernel(
+    size_type num_iters, size_type num_rows, size_type num_cols,
+    ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
+    const Accessor3d krylov_bases, ValueType *__restrict__ hessenberg_iter,
+    size_type stride_hessenberg, const ValueType *__restrict__ buffer_iter,
+    size_type stride_buffer, const stopping_status *__restrict__ stop_status,
+    const stopping_status *__restrict__ reorth_status,
+    sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_idx = global_id / stride_next_krylov;
+    const auto col_idx = global_id % stride_next_krylov;
+
+    if (row_idx < num_rows && col_idx < num_cols &&
+        !stop_status[col_idx].has_stopped() &&
+        !reorth_status[col_idx].has_stopped()) {
+        const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+        auto local_res = next_krylov_basis[next_krylov_idx];
+        for (size_type k = 0; k < num_iters; ++k) {
+            const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+            const auto buffer_idx = k * stride_buffer + col_idx;
+            local_res -=
+                buffer_iter[buffer_idx] * krylov_bases(k, row_idx, col_idx);
+            if ((row_idx == 0) && !reorth_status[col_idx].has_stopped()) {
+                hessenberg_iter[hessenberg_idx] += buffer_iter[buffer_idx];
+            }
+        }
+        next_krylov_basis[next_krylov_idx] = local_res;
+    }
+}
+
+template <int block_size, typename ValueType, typename Accessor3d>
+void update_next_krylov_and_add_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type num_iters, size_type num_rows,
+    size_type num_cols, ValueType *next_krylov_basis,
+    size_type stride_next_krylov, const Accessor3d krylov_bases,
+    ValueType *hessenberg_iter, size_type stride_hessenberg,
+    const ValueType *buffer_iter, size_type stride_buffer,
+    const stopping_status *stop_status, const stopping_status *reorth_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                update_next_krylov_and_add_kernel<block_size>(
+                    num_iters, num_rows, num_cols, next_krylov_basis,
+                    stride_next_krylov, krylov_bases, hessenberg_iter,
+                    stride_hessenberg, buffer_iter, stride_buffer, stop_status,
+                    reorth_status, item_ct1);
+            });
+    });
+}
+
+
+// Must be called with at least `num_rhs` threads
+template <int block_size, typename ValueType, typename Accessor3d>
+void check_arnoldi_norms(
+    size_type num_rhs, remove_complex<ValueType> *__restrict__ arnoldi_norm,
+    size_type stride_norm, ValueType *__restrict__ hessenberg_iter,
+    size_type stride_hessenberg, size_type iter, Accessor3d krylov_bases,
+    const stopping_status *__restrict__ stop_status,
+    stopping_status *__restrict__ reorth_status,
+    size_type *__restrict__ num_reorth, sycl::nd_item<3> item_ct1)
+{
+    const remove_complex<ValueType> eta_squared = 1.0 / 2.0;
+    const auto col_idx = thread::get_thread_id_flat(item_ct1);
+    constexpr bool has_scalar =
+        gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value;
+
+    if (col_idx < num_rhs && !stop_status[col_idx].has_stopped()) {
+        const auto num0 = (sycl::sqrt(eta_squared * arnoldi_norm[col_idx]));
+        const auto num11 =
+            sycl::sqrt((double)(arnoldi_norm[col_idx + stride_norm]));
+        const auto num2 = has_scalar ? (arnoldi_norm[col_idx + 2 * stride_norm])
+                                     : remove_complex<ValueType>{};
+        if (num11 < num0) {
+            reorth_status[col_idx].reset();
+            atomic_add(num_reorth, one<size_type>());
+        } else {
+            reorth_status[col_idx].stop(1);
+        }
+        arnoldi_norm[col_idx] = num0;
+        arnoldi_norm[col_idx + stride_norm] = num11;
+        hessenberg_iter[iter * stride_hessenberg + col_idx] = num11;
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
+            krylov_bases, iter, col_idx, num2 / num11);
+    }
+}
+
+template <int block_size, typename ValueType, typename Accessor3d>
+void check_arnoldi_norms(dim3 grid, dim3 block,
+                         gko::size_type dynamic_shared_memory,
+                         sycl::queue *stream, size_type num_rhs,
+                         remove_complex<ValueType> *arnoldi_norm,
+                         size_type stride_norm, ValueType *hessenberg_iter,
+                         size_type stride_hessenberg, size_type iter,
+                         Accessor3d krylov_bases,
+                         const stopping_status *stop_status,
+                         stopping_status *reorth_status, size_type *num_reorth)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                check_arnoldi_norms<block_size>(
+                    num_rhs, arnoldi_norm, stride_norm, hessenberg_iter,
+                    stride_hessenberg, iter, krylov_bases, stop_status,
+                    reorth_status, num_reorth, item_ct1);
+            });
+    });
+}
+
+
+template <int block_size, typename RealValueType, typename Accessor3d>
+void set_scalar_kernel(size_type num_rhs, size_type num_blocks,
+                       const RealValueType *__restrict__ residual_norm,
+                       size_type stride_residual,
+                       const RealValueType *__restrict__ arnoldi_inf,
+                       size_type stride_inf, Accessor3d krylov_bases,
+                       sycl::nd_item<3> item_ct1)
+{
+    static_assert(!is_complex_s<RealValueType>::value,
+                  "ValueType must not be complex!");
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto krylov_stride =
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::get_stride(
+            krylov_bases);
+    const auto blk_idx = global_id / krylov_stride[1];
+    const auto col_idx = global_id % krylov_stride[1];
+
+    if (blk_idx < num_blocks && col_idx < num_rhs) {
+        if (blk_idx == 0) {
+            const auto num1 = residual_norm[col_idx];
+            const auto num2 = arnoldi_inf[col_idx];
+            gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
+                krylov_bases, {0}, col_idx, num2 / num1);
+        } else {
+            const auto num = one<RealValueType>();
+            gko::cb_gmres::helper_functions_accessor<Accessor3d>::write_scalar(
+                krylov_bases, blk_idx, col_idx, num);
+        }
+    }
+}
+
+template <int block_size, typename RealValueType, typename Accessor3d>
+void set_scalar_kernel(dim3 grid, dim3 block,
+                       gko::size_type dynamic_shared_memory,
+                       sycl::queue *stream, size_type num_rhs,
+                       size_type num_blocks, const RealValueType *residual_norm,
+                       size_type stride_residual,
+                       const RealValueType *arnoldi_inf, size_type stride_inf,
+                       Accessor3d krylov_bases)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                set_scalar_kernel<block_size>(
+                    num_rhs, num_blocks, residual_norm, stride_residual,
+                    arnoldi_inf, stride_inf, krylov_bases, item_ct1);
+            });
+    });
+}
+
+
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType, typename Accessor3d>
+void update_krylov_next_krylov_kernel(
+    size_type iter, size_type num_rows, size_type num_cols,
+    ValueType *__restrict__ next_krylov_basis, size_type stride_next_krylov,
+    Accessor3d krylov_bases, const ValueType *__restrict__ hessenberg_iter,
+    size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_idx = global_id / stride_next_krylov;
+    const auto col_idx = global_id % stride_next_krylov;
+    const auto hessenberg =
+        hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx];
+
+    if (row_idx < num_rows && col_idx < num_cols &&
+        !stop_status[col_idx].has_stopped()) {
+        const auto next_krylov_idx = row_idx * stride_next_krylov + col_idx;
+
+        const auto next_krylov_value =
+            next_krylov_basis[next_krylov_idx] / hessenberg;
+
+        next_krylov_basis[next_krylov_idx] = next_krylov_value;
+        krylov_bases(iter + 1, row_idx, col_idx) = next_krylov_value;
+    }
+}
+
+template <int block_size, typename ValueType, typename Accessor3d>
+void update_krylov_next_krylov_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type iter, size_type num_rows, size_type num_cols,
+    ValueType *next_krylov_basis, size_type stride_next_krylov,
+    Accessor3d krylov_bases, const ValueType *hessenberg_iter,
+    size_type stride_hessenberg, const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                update_krylov_next_krylov_kernel<block_size>(
+                    iter, num_rows, num_cols, next_krylov_basis,
+                    stride_next_krylov, krylov_bases, hessenberg_iter,
+                    stride_hessenberg, stop_status, item_ct1);
+            });
+    });
+}
+
+
+// Must be called with at least `stride_preconditioner * num_rows` threads
+// in total.
+template <size_type block_size, typename ValueType, typename Accessor3d>
+void calculate_Qy_kernel(size_type num_rows, size_type num_cols,
+                         const Accessor3d krylov_bases,
+                         const ValueType *__restrict__ y, size_type stride_y,
+                         ValueType *__restrict__ before_preconditioner,
+                         size_type stride_preconditioner,
+                         const size_type *__restrict__ final_iter_nums,
+                         sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_id = global_id / stride_preconditioner;
+    const auto col_id = global_id % stride_preconditioner;
+
+    if (row_id < num_rows && col_id < num_cols) {
+        ValueType temp = zero<ValueType>();
+        for (size_type j = 0; j < final_iter_nums[col_id]; ++j) {
+            temp += krylov_bases(j, row_id, col_id) * y[j * stride_y + col_id];
+        }
+        before_preconditioner[global_id] = temp;
+    }
+}
+
+template <size_type block_size, typename ValueType, typename Accessor3d>
+void calculate_Qy_kernel(dim3 grid, dim3 block,
+                         gko::size_type dynamic_shared_memory,
+                         sycl::queue *stream, size_type num_rows,
+                         size_type num_cols, const Accessor3d krylov_bases,
+                         const ValueType *y, size_type stride_y,
+                         ValueType *before_preconditioner,
+                         size_type stride_preconditioner,
+                         const size_type *final_iter_nums)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             calculate_Qy_kernel<block_size>(
+                                 num_rows, num_cols, krylov_bases, y, stride_y,
+                                 before_preconditioner, stride_preconditioner,
+                                 final_iter_nums, item_ct1);
+                         });
+    });
+}
+
+
+// Specialization, so the Accessor can use the same function as regular pointers
+template <int dim, typename Type1, typename Type2>
+GKO_INLINE auto as_dpcpp_accessor(
+    const acc::range<acc::reduced_row_major<dim, Type1, Type2>> &acc)
+{
+    return acc::range<
+        acc::reduced_row_major<dim, dpcpp_type<Type1>, dpcpp_type<Type2>>>(
+        acc.get_accessor().get_size(), acc.get_accessor().get_stored_data(),
+        acc.get_accessor().get_stride());
+}
+
+template <int dim, typename Type1, typename Type2, size_type mask>
+GKO_INLINE auto as_dpcpp_accessor(
+    const acc::range<acc::scaled_reduced_row_major<dim, Type1, Type2, mask>>
+        &acc)
+{
+    return acc::range<acc::scaled_reduced_row_major<dim, dpcpp_type<Type1>,
+                                                    dpcpp_type<Type2>, mask>>(
+        acc.get_accessor().get_size(), acc.get_accessor().get_stored_data(),
+        acc.get_accessor().get_storage_stride(),
+        acc.get_accessor().get_scalar(),
+        acc.get_accessor().get_scalar_stride());
+}
+
+
+template <typename ValueType>
+void zero_matrix(size_type m, size_type n, size_type stride, ValueType *array)
+{
+    const dim3 block_size(default_block_size, 1, 1);
+    const dim3 grid_size(ceildiv(n, block_size.x), 1, 1);
+    zero_matrix_kernel(grid_size, block_size, 0, exec->get_queue(), m, n,
+                       stride, array);
+}
+
+
 template <typename ValueType>
 void initialize_1(std::shared_ptr<const DpcppExecutor> exec,
                   const matrix::Dense<ValueType> *b,
                   matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *givens_sin,
                   matrix::Dense<ValueType> *givens_cos,
-                  Array<stopping_status> *stop_status,
-                  size_type krylov_dim) GKO_NOT_IMPLEMENTED;
+                  Array<stopping_status> *stop_status, size_type krylov_dim)
+{
+    const auto num_threads = std::max(b->get_size()[0] * b->get_stride(),
+                                      krylov_dim * b->get_size()[1]);
+    const dim3 grid_dim(ceildiv(num_threads, default_block_size), 1, 1);
+    const dim3 block_dim(default_block_size, 1, 1);
+    constexpr auto block_size = default_block_size;
+
+    initialize_1_kernel<block_size>(
+        grid_dim, block_dim, 0, exec->get_queue(), b->get_size()[0],
+        b->get_size()[1], krylov_dim, b->get_const_values(), b->get_stride(),
+        residual->get_values(), residual->get_stride(),
+        givens_sin->get_values(), givens_sin->get_stride(),
+        givens_cos->get_values(), givens_cos->get_stride(),
+        stop_status->get_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CB_GMRES_INITIALIZE_1_KERNEL);
 
@@ -70,13 +1299,250 @@ void initialize_2(std::shared_ptr<const DpcppExecutor> exec,
                   matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
                   Accessor3d krylov_bases,
                   matrix::Dense<ValueType> *next_krylov_basis,
-                  Array<size_type> *final_iter_nums,
-                  size_type krylov_dim) GKO_NOT_IMPLEMENTED;
+                  Array<size_type> *final_iter_nums, size_type krylov_dim)
+{
+    constexpr bool use_scalar =
+        gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value;
+    const auto num_rows = residual->get_size()[0];
+    const auto num_rhs = residual->get_size()[1];
+    const auto krylov_stride =
+        gko::cb_gmres::helper_functions_accessor<Accessor3d>::get_stride(
+            krylov_bases);
+    const dim3 grid_dim_1(
+        ceildiv((krylov_dim + 1) * krylov_stride[0], default_block_size), 1, 1);
+    const dim3 block_dim(default_block_size, 1, 1);
+    constexpr auto block_size = default_block_size;
+    const auto stride_arnoldi = arnoldi_norm->get_stride();
+
+    initialize_2_1_kernel<block_size>(
+        grid_dim_1, block_dim, 0, exec->get_queue(), residual->get_size()[0],
+        residual->get_size()[1], krylov_dim, as_dpcpp_accessor(krylov_bases),
+        residual_norm_collection->get_values(),
+        residual_norm_collection->get_stride());
+    kernels::dpcpp::dense::compute_norm2(exec, residual, residual_norm);
+
+    if (use_scalar) {
+        components::fill_array(exec,
+                               arnoldi_norm->get_values() + 2 * stride_arnoldi,
+                               num_rhs, zero<remove_complex<ValueType>>());
+        const dim3 grid_size_nrm(ceildiv(num_rhs, default_dot_dim),
+                                 exec->get_num_multiprocessor() * 2);
+        const dim3 block_size_nrm(default_dot_dim, default_dot_dim);
+        multinorminf_without_stop_kernel(
+            grid_size_nrm, block_size_nrm, 0, exec->get_queue(), num_rows,
+            num_rhs, residual->get_const_values(), residual->get_stride(),
+            arnoldi_norm->get_values() + 2 * stride_arnoldi, 0);
+    }
+
+    if (gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value) {
+        set_scalar_kernel<default_block_size>(
+            ceildiv(num_rhs * (krylov_dim + 1), default_block_size),
+            default_block_size, 0, exec->get_queue(), num_rhs, krylov_dim + 1,
+            residual_norm->get_const_values(), residual_norm->get_stride(),
+            arnoldi_norm->get_const_values() + 2 * stride_arnoldi,
+            stride_arnoldi, as_dpcpp_accessor(krylov_bases));
+    }
+
+    const dim3 grid_dim_2(
+        ceildiv(num_rows * krylov_stride[1], default_block_size), 1, 1);
+    initialize_2_2_kernel<block_size>(
+        grid_dim_2, block_dim, 0, exec->get_queue(), residual->get_size()[0],
+        residual->get_size()[1], residual->get_const_values(),
+        residual->get_stride(), residual_norm->get_const_values(),
+        residual_norm_collection->get_values(), as_dpcpp_accessor(krylov_bases),
+        next_krylov_basis->get_values(), next_krylov_basis->get_stride(),
+        final_iter_nums->get_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(
     GKO_DECLARE_CB_GMRES_INITIALIZE_2_KERNEL);
 
 
+template <typename ValueType, typename Accessor3dim>
+void finish_arnoldi_CGS(std::shared_ptr<const DpcppExecutor> exec,
+                        matrix::Dense<ValueType> *next_krylov_basis,
+                        Accessor3dim krylov_bases,
+                        matrix::Dense<ValueType> *hessenberg_iter,
+                        matrix::Dense<ValueType> *buffer_iter,
+                        matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
+                        size_type iter, const stopping_status *stop_status,
+                        stopping_status *reorth_status,
+                        Array<size_type> *num_reorth)
+{
+    using non_complex = remove_complex<ValueType>;
+    // optimization parameter
+    constexpr int singledot_block_size = default_dot_dim;
+    constexpr bool use_scalar =
+        gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3dim>::value;
+    const auto stride_next_krylov = next_krylov_basis->get_stride();
+    const auto stride_hessenberg = hessenberg_iter->get_stride();
+    const auto stride_buffer = buffer_iter->get_stride();
+    const auto stride_arnoldi = arnoldi_norm->get_stride();
+    const auto dim_size = next_krylov_basis->get_size();
+    const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
+                         exec->get_num_multiprocessor() * 2);
+    const dim3 grid_size_num_iters(ceildiv(dim_size[1], default_dot_dim),
+                                   exec->get_num_multiprocessor() * 2,
+                                   iter + 1);
+    const dim3 block_size(default_dot_dim, default_dot_dim);
+    // Note: having iter first (instead of row_idx information) is likely
+    //       beneficial for avoiding atomic_add conflicts, but that needs
+    //       further investigation.
+    const dim3 grid_size_iters_single(exec->get_num_multiprocessor() * 2,
+                                      iter + 1);
+    const dim3 block_size_iters_single(singledot_block_size);
+    size_type num_reorth_host;
+
+    components::fill_array(exec, arnoldi_norm->get_values(), dim_size[1],
+                           zero<non_complex>());
+    multinorm2_kernel(grid_size, block_size, 0, exec->get_queue(), dim_size[0],
+                      dim_size[1], next_krylov_basis->get_const_values(),
+                      stride_next_krylov, arnoldi_norm->get_values(),
+                      stop_status);
+    zero_matrix(iter + 1, dim_size[1], stride_hessenberg,
+                hessenberg_iter->get_values());
+    if (dim_size[1] > 1) {
+        multidot_kernel<default_dot_dim>(
+            grid_size_num_iters, block_size, 0, exec->get_queue(), dim_size[0],
+            dim_size[1], next_krylov_basis->get_const_values(),
+            stride_next_krylov, as_dpcpp_accessor(krylov_bases),
+            hessenberg_iter->get_values(), stride_hessenberg, stop_status);
+    } else {
+        singledot_kernel<singledot_block_size>(
+            grid_size_iters_single, block_size_iters_single, 0,
+            exec->get_queue(), dim_size[0],
+            next_krylov_basis->get_const_values(), stride_next_krylov,
+            as_dpcpp_accessor(krylov_bases), hessenberg_iter->get_values(),
+            stride_hessenberg, stop_status);
+    }
+    // for i in 1:iter
+    //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+    // end
+    update_next_krylov_kernel<default_block_size>(
+        ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+        default_block_size, 0, exec->get_queue(), iter + 1, dim_size[0],
+        dim_size[1], next_krylov_basis->get_values(), stride_next_krylov,
+        as_dpcpp_accessor(krylov_bases), hessenberg_iter->get_const_values(),
+        stride_hessenberg, stop_status);
+
+    // for i in 1:iter
+    //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+    // end
+    components::fill_array(exec, arnoldi_norm->get_values() + stride_arnoldi,
+                           dim_size[1], zero<non_complex>());
+    if (use_scalar) {
+        components::fill_array(exec,
+                               arnoldi_norm->get_values() + 2 * stride_arnoldi,
+                               dim_size[1], zero<non_complex>());
+    }
+    multinorm2_inf_kernel<use_scalar>(
+        grid_size, block_size, 0, exec->get_queue(), dim_size[0], dim_size[1],
+        next_krylov_basis->get_const_values(), stride_next_krylov,
+        arnoldi_norm->get_values() + stride_arnoldi,
+        arnoldi_norm->get_values() + 2 * stride_arnoldi, stop_status);
+    // nrmN = norm(next_krylov_basis)
+    components::fill_array(exec, num_reorth->get_data(), 1, zero<size_type>());
+    check_arnoldi_norms<default_block_size>(
+        ceildiv(dim_size[1], default_block_size), default_block_size, 0,
+        exec->get_queue(), dim_size[1], arnoldi_norm->get_values(),
+        stride_arnoldi, hessenberg_iter->get_values(), stride_hessenberg,
+        iter + 1, as_dpcpp_accessor(krylov_bases), stop_status, reorth_status,
+        num_reorth->get_data());
+    num_reorth_host = exec->copy_val_to_host(num_reorth->get_const_data());
+    // num_reorth_host := number of next_krylov vector to be reorthogonalization
+    for (size_type l = 1; (num_reorth_host > 0) && (l < 3); l++) {
+        zero_matrix(iter + 1, dim_size[1], stride_buffer,
+                    buffer_iter->get_values());
+        if (dim_size[1] > 1) {
+            multidot_kernel<default_dot_dim>(
+                grid_size_num_iters, block_size, 0, exec->get_queue(),
+                dim_size[0], dim_size[1], next_krylov_basis->get_const_values(),
+                stride_next_krylov, as_dpcpp_accessor(krylov_bases),
+                buffer_iter->get_values(), stride_buffer, stop_status);
+        } else {
+            singledot_kernel<singledot_block_size>(
+                grid_size_iters_single, block_size_iters_single, 0,
+                exec->get_queue(), dim_size[0],
+                next_krylov_basis->get_const_values(), stride_next_krylov,
+                as_dpcpp_accessor(krylov_bases), buffer_iter->get_values(),
+                stride_buffer, stop_status);
+        }
+        // for i in 1:iter
+        //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+        // end
+        update_next_krylov_and_add_kernel<default_block_size>(
+            ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+            default_block_size, 0, exec->get_queue(), iter + 1, dim_size[0],
+            dim_size[1], next_krylov_basis->get_values(), stride_next_krylov,
+            as_dpcpp_accessor(krylov_bases), hessenberg_iter->get_values(),
+            stride_hessenberg, buffer_iter->get_const_values(), stride_buffer,
+            stop_status, reorth_status);
+        // for i in 1:iter
+        //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+        // end
+        components::fill_array(exec,
+                               arnoldi_norm->get_values() + stride_arnoldi,
+                               dim_size[1], zero<non_complex>());
+        if (use_scalar) {
+            components::fill_array(
+                exec, arnoldi_norm->get_values() + 2 * stride_arnoldi,
+                dim_size[1], zero<non_complex>());
+        }
+        multinorm2_inf_kernel<use_scalar>(
+            grid_size, block_size, 0, exec->get_queue(), dim_size[0],
+            dim_size[1], next_krylov_basis->get_const_values(),
+            stride_next_krylov, arnoldi_norm->get_values() + stride_arnoldi,
+            arnoldi_norm->get_values() + 2 * stride_arnoldi, stop_status);
+        // nrmN = norm(next_krylov_basis)
+        components::fill_array(exec, num_reorth->get_data(), 1,
+                               zero<size_type>());
+        check_arnoldi_norms<default_block_size>(
+            ceildiv(dim_size[1], default_block_size), default_block_size, 0,
+            exec->get_queue(), dim_size[1], arnoldi_norm->get_values(),
+            stride_arnoldi, hessenberg_iter->get_values(), stride_hessenberg,
+            iter + 1, as_dpcpp_accessor(krylov_bases), stop_status,
+            reorth_status, num_reorth->get_data());
+        num_reorth_host = exec->copy_val_to_host(num_reorth->get_const_data());
+    }
+
+    update_krylov_next_krylov_kernel<default_block_size>(
+        ceildiv(dim_size[0] * stride_next_krylov, default_block_size),
+        default_block_size, 0, exec->get_queue(), iter, dim_size[0],
+        dim_size[1], next_krylov_basis->get_values(), stride_next_krylov,
+        as_dpcpp_accessor(krylov_bases), hessenberg_iter->get_const_values(),
+        stride_hessenberg, stop_status);
+    // next_krylov_basis /= hessenberg(iter, iter + 1)
+    // krylov_bases(:, iter + 1) = next_krylov_basis
+    // End of arnoldi
+}
+
+template <typename ValueType>
+void givens_rotation(std::shared_ptr<const DpcppExecutor> exec,
+                     matrix::Dense<ValueType> *givens_sin,
+                     matrix::Dense<ValueType> *givens_cos,
+                     matrix::Dense<ValueType> *hessenberg_iter,
+                     matrix::Dense<remove_complex<ValueType>> *residual_norm,
+                     matrix::Dense<ValueType> *residual_norm_collection,
+                     size_type iter, const Array<stopping_status> *stop_status)
+{
+    // TODO: tune block_size for optimal performance
+    constexpr auto block_size = default_block_size;
+    const auto num_cols = hessenberg_iter->get_size()[1];
+    const dim3 block_dim{block_size, 1, 1};
+    const dim3 grid_dim{
+        static_cast<unsigned int>(ceildiv(num_cols, block_size)), 1, 1};
+
+    givens_rotation_kernel<block_size>(
+        grid_dim, block_dim, 0, exec->get_queue(),
+        hessenberg_iter->get_size()[0], hessenberg_iter->get_size()[1], iter,
+        hessenberg_iter->get_values(), hessenberg_iter->get_stride(),
+        givens_sin->get_values(), givens_sin->get_stride(),
+        givens_cos->get_values(), givens_cos->get_stride(),
+        residual_norm->get_values(), residual_norm_collection->get_values(),
+        residual_norm_collection->get_stride(), stop_status->get_const_data());
+}
+
+
 template <typename ValueType, typename Accessor3d>
 void step_1(std::shared_ptr<const DpcppExecutor> exec,
             matrix::Dense<ValueType> *next_krylov_basis,
@@ -89,12 +1555,75 @@ void step_1(std::shared_ptr<const DpcppExecutor> exec,
             matrix::Dense<remove_complex<ValueType>> *arnoldi_norm,
             size_type iter, Array<size_type> *final_iter_nums,
             const Array<stopping_status> *stop_status,
-            Array<stopping_status> *reorth_status,
-            Array<size_type> *num_reorth) GKO_NOT_IMPLEMENTED;
+            Array<stopping_status> *reorth_status, Array<size_type> *num_reorth)
+{
+    increase_final_iteration_numbers_kernel(
+        static_cast<unsigned int>(
+            ceildiv(final_iter_nums->get_num_elems(), default_block_size)),
+        default_block_size, 0, exec->get_queue(), final_iter_nums->get_data(),
+        stop_status->get_const_data(), final_iter_nums->get_num_elems());
+    finish_arnoldi_CGS(exec, next_krylov_basis, krylov_bases, hessenberg_iter,
+                       buffer_iter, arnoldi_norm, iter,
+                       stop_status->get_const_data(), reorth_status->get_data(),
+                       num_reorth);
+    givens_rotation(exec, givens_sin, givens_cos, hessenberg_iter,
+                    residual_norm, residual_norm_collection, iter, stop_status);
+}
 
 GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(GKO_DECLARE_CB_GMRES_STEP_1_KERNEL);
 
 
+template <typename ValueType>
+void solve_upper_triangular(
+    const matrix::Dense<ValueType> *residual_norm_collection,
+    const matrix::Dense<ValueType> *hessenberg, matrix::Dense<ValueType> *y,
+    const Array<size_type> *final_iter_nums)
+{
+    // TODO: tune block_size for optimal performance
+    constexpr auto block_size = default_block_size;
+    const auto num_rhs = residual_norm_collection->get_size()[1];
+    const dim3 block_dim{block_size, 1, 1};
+    const dim3 grid_dim{static_cast<unsigned int>(ceildiv(num_rhs, block_size)),
+                        1, 1};
+
+    solve_upper_triangular_kernel<block_size>(
+        grid_dim, block_dim, 0, exec->get_queue(), hessenberg->get_size()[1],
+        num_rhs, residual_norm_collection->get_const_values(),
+        residual_norm_collection->get_stride(), hessenberg->get_const_values(),
+        hessenberg->get_stride(), y->get_values(), y->get_stride(),
+        final_iter_nums->get_const_data());
+}
+
+
+template <typename ValueType, typename ConstAccessor3d>
+void calculate_qy(ConstAccessor3d krylov_bases, size_type num_krylov_bases,
+                  const matrix::Dense<ValueType> *y,
+                  matrix::Dense<ValueType> *before_preconditioner,
+                  const Array<size_type> *final_iter_nums)
+{
+    const auto num_rows = before_preconditioner->get_size()[0];
+    const auto num_cols = before_preconditioner->get_size()[1];
+    const auto stride_before_preconditioner =
+        before_preconditioner->get_stride();
+
+    constexpr auto block_size = default_block_size;
+    const dim3 grid_dim{
+        static_cast<unsigned int>(
+            ceildiv(num_rows * stride_before_preconditioner, block_size)),
+        1, 1};
+    const dim3 block_dim{block_size, 1, 1};
+
+
+    calculate_Qy_kernel<block_size>(
+        grid_dim, block_dim, 0, exec->get_queue(), num_rows, num_cols,
+        as_dpcpp_accessor(krylov_bases), y->get_const_values(), y->get_stride(),
+        before_preconditioner->get_values(), stride_before_preconditioner,
+        final_iter_nums->get_const_data());
+    // Calculate qy
+    // before_preconditioner = krylov_bases * y
+}
+
+
 template <typename ValueType, typename ConstAccessor3d>
 void step_2(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
@@ -102,7 +1631,18 @@ void step_2(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
-            const Array<size_type> *final_iter_nums) GKO_NOT_IMPLEMENTED;
+            const Array<size_type> *final_iter_nums)
+{
+    // since hessenberg has dims:  iters x iters * num_rhs
+    // krylov_bases has dims:  (iters + 1) x sysmtx[0] x num_rhs
+    const auto iters =
+        hessenberg->get_size()[1] / before_preconditioner->get_size()[1];
+    const auto num_krylov_bases = iters + 1;
+    solve_upper_triangular(residual_norm_collection, hessenberg, y,
+                           final_iter_nums);
+    calculate_qy(krylov_bases, num_krylov_bases, y, before_preconditioner,
+                 final_iter_nums);
+}
 
 GKO_INSTANTIATE_FOR_EACH_CB_GMRES_CONST_TYPE(
     GKO_DECLARE_CB_GMRES_STEP_2_KERNEL);
diff --git a/dpcpp/solver/gmres_kernels.dp.cpp b/dpcpp/solver/gmres_kernels.dp.cpp
index 0853eeb1804..6c7097e9e3e 100644
--- a/dpcpp/solver/gmres_kernels.dp.cpp
+++ b/dpcpp/solver/gmres_kernels.dp.cpp
@@ -33,14 +33,28 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/gmres_kernels.hpp"
 
 
+#include <algorithm>
+
+
 #include <CL/sycl.hpp>
 
 
-#include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/base/types.hpp>
-#include <ginkgo/core/solver/gmres.hpp>
+#include <ginkgo/core/base/range_accessors.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/components/fill_array.hpp"
+#include "core/matrix/dense_kernels.hpp"
+#include "dpcpp/base/config.hpp"
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/base/onemkl_bindings.hpp"
+#include "dpcpp/components/atomic.dp.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
+#include "dpcpp/components/reduction.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
+#include "dpcpp/components/uninitialized_array.hpp"
 
 
 namespace gko {
@@ -54,53 +68,634 @@ namespace dpcpp {
 namespace gmres {
 
 
-namespace {
+constexpr int default_block_size = 512;
+// default_dot_dim can not be 64 in hip because 64 * 64 exceeds their max block
+// size limit.
+constexpr int default_dot_dim = 32;
+constexpr int default_dot_size = default_dot_dim * default_dot_dim;
+
+
+// Must be called with at least `max(stride_b * num_rows, krylov_dim *
+// num_cols)` threads in total.
+template <size_type block_size, typename ValueType>
+void initialize_1_kernel(
+    size_type num_rows, size_type num_cols, size_type krylov_dim,
+    const ValueType *__restrict__ b, size_type stride_b,
+    ValueType *__restrict__ residual, size_type stride_residual,
+    ValueType *__restrict__ givens_sin, size_type stride_sin,
+    ValueType *__restrict__ givens_cos, size_type stride_cos,
+    stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+
+    const auto row_idx = global_id / stride_b;
+    const auto col_idx = global_id % stride_b;
+
+    if (global_id < num_cols) {
+        stop_status[global_id].reset();
+    }
+
+    if (row_idx < num_rows && col_idx < num_cols) {
+        residual[row_idx * stride_residual + col_idx] =
+            b[row_idx * stride_b + col_idx];
+    }
+
+    if (global_id < krylov_dim * num_cols) {
+        const auto row_givens = global_id / num_cols;
+        const auto col_givens = global_id % num_cols;
+
+        givens_sin[row_givens * stride_sin + col_givens] = zero<ValueType>();
+        givens_cos[row_givens * stride_cos + col_givens] = zero<ValueType>();
+    }
+}
+
+template <size_type block_size, typename ValueType>
+void initialize_1_kernel(dim3 grid, dim3 block,
+                         gko::size_type dynamic_shared_memory,
+                         sycl::queue *stream, size_type num_rows,
+                         size_type num_cols, size_type krylov_dim,
+                         const ValueType *b, size_type stride_b,
+                         ValueType *residual, size_type stride_residual,
+                         ValueType *givens_sin, size_type stride_sin,
+                         ValueType *givens_cos, size_type stride_cos,
+                         stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                initialize_1_kernel<block_size>(
+                    num_rows, num_cols, krylov_dim, b, stride_b, residual,
+                    stride_residual, givens_sin, stride_sin, givens_cos,
+                    stride_cos, stop_status, item_ct1);
+            });
+    });
+}
 
 
 template <typename ValueType>
-void finish_arnoldi(size_type num_rows, matrix::Dense<ValueType> *krylov_bases,
-                    matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
-                    const stopping_status *stop_status) GKO_NOT_IMPLEMENTED;
+void calculate_sin_and_cos_kernel(size_type col_idx, size_type num_cols,
+                                  size_type iter, const ValueType &this_hess,
+                                  const ValueType &next_hess,
+                                  ValueType *givens_sin, size_type stride_sin,
+                                  ValueType *givens_cos, size_type stride_cos,
+                                  ValueType &register_sin,
+                                  ValueType &register_cos)
+{
+    if (this_hess == zero<ValueType>()) {
+        register_cos = zero<ValueType>();
+        register_sin = one<ValueType>();
+    } else {
+        const auto scale = std::abs(this_hess) + std::abs(next_hess);
+        const auto hypotenuse =
+            scale *
+            sycl::sqrt(
+                std::abs(this_hess / scale) * std::abs(this_hess / scale) +
+                std::abs(next_hess / scale) * std::abs(next_hess / scale));
+        /*
+        DPCT1007:1: Migration of this CUDA API is not supported by the Intel(R)
+        DPC++ Compatibility Tool.
+        */
+        register_cos = conj(this_hess) / hypotenuse;
+        /*
+        DPCT1007:2: Migration of this CUDA API is not supported by the Intel(R)
+        DPC++ Compatibility Tool.
+        */
+        register_sin = conj(next_hess) / hypotenuse;
+    }
+    givens_cos[iter * stride_cos + col_idx] = register_cos;
+    givens_sin[iter * stride_sin + col_idx] = register_sin;
+}
 
 
 template <typename ValueType>
-void calculate_sin_and_cos(matrix::Dense<ValueType> *givens_sin,
-                           matrix::Dense<ValueType> *givens_cos,
-                           matrix::Dense<ValueType> *hessenberg_iter,
-                           size_type iter,
-                           const size_type rhs) GKO_NOT_IMPLEMENTED;
+void calculate_residual_norm_kernel(size_type col_idx, size_type num_cols,
+                                    size_type iter,
+                                    const ValueType &register_sin,
+                                    const ValueType &register_cos,
+                                    remove_complex<ValueType> *residual_norm,
+                                    ValueType *residual_norm_collection,
+                                    size_type stride_residual_norm_collection)
+{
+    const auto this_rnc =
+        residual_norm_collection[iter * stride_residual_norm_collection +
+                                 col_idx];
+    /*
+    DPCT1007:3: Migration of this CUDA API is not supported by the Intel(R)
+    DPC++ Compatibility Tool.
+    */
+    const auto next_rnc = -conj(register_sin) * this_rnc;
+    residual_norm_collection[iter * stride_residual_norm_collection + col_idx] =
+        register_cos * this_rnc;
+    residual_norm[col_idx] = std::abs(next_rnc);
+    residual_norm_collection[(iter + 1) * stride_residual_norm_collection +
+                             col_idx] = next_rnc;
+}
+
+
+// Must be called with at least `num_cols` threads in total.
+template <size_type block_size, typename ValueType>
+void givens_rotation_kernel(
+    size_type num_rows, size_type num_cols, size_type iter,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    ValueType *__restrict__ givens_sin, size_type stride_sin,
+    ValueType *__restrict__ givens_cos, size_type stride_cos,
+    remove_complex<ValueType> *__restrict__ residual_norm,
+    ValueType *__restrict__ residual_norm_collection,
+    size_type stride_residual_norm_collection,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
+{
+    const auto col_idx = thread::get_thread_id_flat(item_ct1);
+
+    if (col_idx >= num_cols || stop_status[col_idx].has_stopped()) {
+        return;
+    }
+
+    auto this_hess = hessenberg_iter[col_idx];
+    auto next_hess = hessenberg_iter[stride_hessenberg + col_idx];
+    for (size_type i = 0; i < iter; ++i) {
+        const auto cos = givens_cos[i * stride_cos + col_idx];
+        const auto sin = givens_sin[i * stride_sin + col_idx];
+        hessenberg_iter[i * stride_hessenberg + col_idx] =
+            cos * this_hess + sin * next_hess;
+        /*
+        DPCT1007:4: Migration of this CUDA API is not supported by the Intel(R)
+        DPC++ Compatibility Tool.
+        */
+        this_hess = conj(cos) * next_hess - conj(sin) * this_hess;
+        next_hess = hessenberg_iter[(i + 2) * stride_hessenberg + col_idx];
+    }
+    // for j in 0:iter - 1
+    //     temp             =  cos(j)*hessenberg(j) +
+    //                         sin(j)*hessenberg(j+1)
+    //     hessenberg(j+1)  = -sin(j)*hessenberg(j) +
+    //                         cos(j)*hessenberg(j+1)
+    //     hessenberg(j)    =  temp;
+    // end
+
+    ValueType register_sin;
+    ValueType register_cos;
+    calculate_sin_and_cos_kernel(col_idx, num_cols, iter, this_hess, next_hess,
+                                 givens_sin, stride_sin, givens_cos, stride_cos,
+                                 register_sin, register_cos);
+    // Calculate sin and cos on hessenberg(iter) and hessenberg(iter+1)
+
+    hessenberg_iter[iter * stride_hessenberg + col_idx] =
+        register_cos * this_hess + register_sin * next_hess;
+    hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] =
+        zero<ValueType>();
+    // hessenberg(iter)   = cos(iter)*hessenberg(iter) +
+    //                      sin(iter)*hessenberg(iter+1)
+    // hessenberg(iter+1) = 0
+
+    calculate_residual_norm_kernel(
+        col_idx, num_cols, iter, register_sin, register_cos, residual_norm,
+        residual_norm_collection, stride_residual_norm_collection);
+    // Calculate residual norm
+}
+
+template <size_type block_size, typename ValueType>
+void givens_rotation_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type num_rows, size_type num_cols, size_type iter,
+    ValueType *hessenberg_iter, size_type stride_hessenberg,
+    ValueType *givens_sin, size_type stride_sin, ValueType *givens_cos,
+    size_type stride_cos, remove_complex<ValueType> *residual_norm,
+    ValueType *residual_norm_collection,
+    size_type stride_residual_norm_collection,
+    const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                givens_rotation_kernel<block_size>(
+                    num_rows, num_cols, iter, hessenberg_iter,
+                    stride_hessenberg, givens_sin, stride_sin, givens_cos,
+                    stride_cos, residual_norm, residual_norm_collection,
+                    stride_residual_norm_collection, stop_status, item_ct1);
+            });
+    });
+}
+
+
+// Must be called with at least `num_rhs` threads in total.
+template <size_type block_size, typename ValueType>
+void solve_upper_triangular_kernel(
+    size_type num_cols, size_type num_rhs,
+    const ValueType *__restrict__ residual_norm_collection,
+    size_type stride_residual_norm_collection,
+    const ValueType *__restrict__ hessenberg, size_type stride_hessenberg,
+    ValueType *__restrict__ y, size_type stride_y,
+    const size_type *__restrict__ final_iter_nums, sycl::nd_item<3> item_ct1)
+{
+    const auto col_idx = thread::get_thread_id_flat(item_ct1);
+
+    if (col_idx >= num_rhs) {
+        return;
+    }
+
+    for (int i = final_iter_nums[col_idx] - 1; i >= 0; --i) {
+        auto temp =
+            residual_norm_collection[i * stride_residual_norm_collection +
+                                     col_idx];
+        for (size_type j = i + 1; j < final_iter_nums[col_idx]; ++j) {
+            temp -= hessenberg[i * stride_hessenberg + j * num_rhs + col_idx] *
+                    y[j * stride_y + col_idx];
+        }
+
+        y[i * stride_y + col_idx] =
+            temp / hessenberg[i * stride_hessenberg + i * num_rhs + col_idx];
+    }
+    // Solve upper triangular.
+    // y = hessenberg \ residual_norm_collection
+}
+
+template <size_type block_size, typename ValueType>
+void solve_upper_triangular_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type num_cols, size_type num_rhs,
+    const ValueType *residual_norm_collection,
+    size_type stride_residual_norm_collection, const ValueType *hessenberg,
+    size_type stride_hessenberg, ValueType *y, size_type stride_y,
+    const size_type *final_iter_nums)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                solve_upper_triangular_kernel<block_size>(
+                    num_cols, num_rhs, residual_norm_collection,
+                    stride_residual_norm_collection, hessenberg,
+                    stride_hessenberg, y, stride_y, final_iter_nums, item_ct1);
+            });
+    });
+}
+
+// Must be called with at least `num_rows * num_rhs` threads in total.
+template <size_type block_size, typename ValueType>
+void initialize_2_2_kernel(
+    size_type num_rows, size_type num_rhs,
+    const ValueType *__restrict__ residual, size_type stride_residual,
+    const remove_complex<ValueType> *__restrict__ residual_norm,
+    ValueType *__restrict__ residual_norm_collection,
+    ValueType *__restrict__ krylov_bases, size_type stride_krylov,
+    size_type *__restrict__ final_iter_nums, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_idx = global_id / num_rhs;
+    const auto col_idx = global_id % num_rhs;
+
+    if (global_id < num_rhs) {
+        residual_norm_collection[global_id] = residual_norm[global_id];
+        final_iter_nums[global_id] = 0;
+    }
+
+    if (row_idx < num_rows && col_idx < num_rhs) {
+        auto value = residual[row_idx * stride_residual + col_idx] /
+                     residual_norm[col_idx];
+        krylov_bases[row_idx * stride_krylov + col_idx] = value;
+    }
+}
+
+template <size_type block_size, typename ValueType>
+void initialize_2_2_kernel(dim3 grid, dim3 block,
+                           gko::size_type dynamic_shared_memory,
+                           sycl::queue *stream, size_type num_rows,
+                           size_type num_rhs, const ValueType *residual,
+                           size_type stride_residual,
+                           const remove_complex<ValueType> *residual_norm,
+                           ValueType *residual_norm_collection,
+                           ValueType *krylov_bases, size_type stride_krylov,
+                           size_type *final_iter_nums)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                initialize_2_2_kernel<block_size>(
+                    num_rows, num_rhs, residual, stride_residual, residual_norm,
+                    residual_norm_collection, krylov_bases, stride_krylov,
+                    final_iter_nums, item_ct1);
+            });
+    });
+}
+
+
+void increase_final_iteration_numbers_kernel(
+    size_type *__restrict__ final_iter_nums,
+    const stopping_status *__restrict__ stop_status, size_type total_number,
+    sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    if (global_id < total_number) {
+        final_iter_nums[global_id] += !stop_status[global_id].has_stopped();
+    }
+}
+
+void increase_final_iteration_numbers_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type *final_iter_nums,
+    const stopping_status *stop_status, size_type total_number)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                increase_final_iteration_numbers_kernel(
+                    final_iter_nums, stop_status, total_number, item_ct1);
+            });
+    });
+}
 
 
 template <typename ValueType>
-void givens_rotation(matrix::Dense<ValueType> *givens_sin,
-                     matrix::Dense<ValueType> *givens_cos,
-                     matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
-                     const stopping_status *stop_status) GKO_NOT_IMPLEMENTED;
-
+void multidot_kernel(
+    size_type k, size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ krylov_bases,
+    const ValueType *__restrict__ next_krylov_basis, size_type stride_krylov,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1,
+    UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
+        *reduction_helper_array)
+{
+    const auto tidx = item_ct1.get_local_id(2);
+    const auto tidy = item_ct1.get_local_id(1);
+    const auto col_idx = item_ct1.get_group(2) * default_dot_dim + tidx;
+    const auto num = ceildiv(num_rows, item_ct1.get_group_range(1));
+    const auto start_row = item_ct1.get_group(1) * num;
+    const auto end_row = ((item_ct1.get_group(1) + 1) * num > num_rows)
+                             ? num_rows
+                             : (item_ct1.get_group(1) + 1) * num;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+
+    ValueType *__restrict__ reduction_helper = (*reduction_helper_array);
+
+    ValueType local_res = zero<ValueType>();
+    if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+        for (size_type i = start_row + tidy; i < end_row;
+             i += default_dot_dim) {
+            const auto krylov_idx = i * stride_krylov + col_idx;
+            /*
+            DPCT1007:10: Migration of this CUDA API is not supported by the
+            Intel(R) DPC++ Compatibility Tool.
+            */
+            local_res +=
+                conj(krylov_bases[krylov_idx]) * next_krylov_basis[krylov_idx];
+        }
+    }
+    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
+    /*
+    DPCT1065:9: Consider replacing sycl::nd_item::barrier() with
+    sycl::nd_item::barrier(sycl::access::fence_space::local_space) for better
+    performance, if there is no access to global memory.
+    */
+    item_ct1.barrier();
+    local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
+    const auto tile_block = group::tiled_partition<default_dot_dim>(
+        group::this_thread_block(item_ct1));
+    const auto sum =
+        reduce(tile_block, local_res,
+               [](const ValueType &a, const ValueType &b) { return a + b; });
+    const auto new_col_idx = item_ct1.get_group(2) * default_dot_dim + tidy;
+    if (tidx == 0 && new_col_idx < num_cols &&
+        !stop_status[new_col_idx].has_stopped()) {
+        const auto hessenberg_idx = k * stride_hessenberg + new_col_idx;
+        atomic_add(hessenberg_iter + hessenberg_idx, sum);
+    }
+}
 
 template <typename ValueType>
-void calculate_next_residual_norm(
-    matrix::Dense<ValueType> *givens_sin, matrix::Dense<ValueType> *givens_cos,
-    matrix::Dense<remove_complex<ValueType>> *residual_norm,
-    matrix::Dense<ValueType> *residual_norm_collection, size_type iter,
-    const stopping_status *stop_status) GKO_NOT_IMPLEMENTED;
-
-
-template <typename ValueType>
-void solve_upper_triangular(
-    const matrix::Dense<ValueType> *residual_norm_collection,
-    const matrix::Dense<ValueType> *hessenberg, matrix::Dense<ValueType> *y,
-    const size_type *final_iter_nums) GKO_NOT_IMPLEMENTED;
-
-
-template <typename ValueType>
-void calculate_qy(const matrix::Dense<ValueType> *krylov_bases,
-                  const matrix::Dense<ValueType> *y,
-                  matrix::Dense<ValueType> *before_preconditioner,
-                  const size_type *final_iter_nums) GKO_NOT_IMPLEMENTED;
-
-
-}  // namespace
+void multidot_kernel(dim3 grid, dim3 block,
+                     gko::size_type dynamic_shared_memory, sycl::queue *stream,
+                     size_type k, size_type num_rows, size_type num_cols,
+                     const ValueType *krylov_bases,
+                     const ValueType *next_krylov_basis,
+                     size_type stride_krylov, ValueType *hessenberg_iter,
+                     size_type stride_hessenberg,
+                     const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<ValueType, default_dot_dim *(
+                                                         default_dot_dim + 1)>,
+                       0, sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            reduction_helper_array_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                multidot_kernel(
+                    k, num_rows, num_cols, krylov_bases, next_krylov_basis,
+                    stride_krylov, hessenberg_iter, stride_hessenberg,
+                    stop_status, item_ct1,
+                    (UninitializedArray<ValueType, default_dot_dim *(
+                                                       default_dot_dim + 1)> *)
+                        reduction_helper_array_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+// Must be called with at least `num_rows * stride_next_krylov` threads in
+// total.
+template <int block_size, typename ValueType>
+void update_next_krylov_kernel(
+    size_type k, size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ krylov_bases,
+    ValueType *__restrict__ next_krylov_basis, size_type stride_krylov,
+    const ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_idx = global_id / stride_krylov;
+    const auto col_idx = global_id % stride_krylov;
+
+    if (row_idx < num_rows && col_idx < num_cols &&
+        !stop_status[col_idx].has_stopped()) {
+        const auto next_krylov_idx = row_idx * stride_krylov + col_idx;
+        const auto krylov_idx = row_idx * stride_krylov + col_idx;
+        const auto hessenberg_idx = k * stride_hessenberg + col_idx;
+
+        next_krylov_basis[next_krylov_idx] -=
+            hessenberg_iter[hessenberg_idx] * krylov_bases[krylov_idx];
+    }
+}
+
+template <int block_size, typename ValueType>
+void update_next_krylov_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type k, size_type num_rows, size_type num_cols,
+    const ValueType *krylov_bases, ValueType *next_krylov_basis,
+    size_type stride_krylov, const ValueType *hessenberg_iter,
+    size_type stride_hessenberg, const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                update_next_krylov_kernel<block_size>(
+                    k, num_rows, num_cols, krylov_bases, next_krylov_basis,
+                    stride_krylov, hessenberg_iter, stride_hessenberg,
+                    stop_status, item_ct1);
+            });
+    });
+}
+
+
+// Must be called with at least `num_cols` blocks, each with `block_size`
+// threads. `block_size` must be a power of 2.
+template <int block_size, typename ValueType>
+void update_hessenberg_2_kernel(
+    size_type iter, size_type num_rows, size_type num_cols,
+    const ValueType *__restrict__ next_krylov_basis,
+    size_type stride_next_krylov, ValueType *__restrict__ hessenberg_iter,
+    size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1,
+    UninitializedArray<ValueType, block_size> *reduction_helper_array)
+{
+    const auto tidx = item_ct1.get_local_id(2);
+    const auto col_idx = item_ct1.get_group(2);
+
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+
+    ValueType *__restrict__ reduction_helper = (*reduction_helper_array);
+
+    if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
+        ValueType local_res{};
+        for (size_type i = tidx; i < num_rows; i += block_size) {
+            const auto next_krylov_idx = i * stride_next_krylov + col_idx;
+            const auto next_krylov_value = next_krylov_basis[next_krylov_idx];
+
+            local_res += next_krylov_value * next_krylov_value;
+        }
+
+        reduction_helper[tidx] = local_res;
+
+        // Perform thread block reduction. Result is in reduction_helper[0]
+        reduce(group::this_thread_block(item_ct1), reduction_helper,
+               [](const ValueType &a, const ValueType &b) { return a + b; });
+
+        if (tidx == 0) {
+            hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] =
+                sycl::sqrt(reduction_helper[0]);
+        }
+    }
+}
+
+template <int block_size, typename ValueType>
+void update_hessenberg_2_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type iter, size_type num_rows, size_type num_cols,
+    const ValueType *next_krylov_basis, size_type stride_next_krylov,
+    ValueType *hessenberg_iter, size_type stride_hessenberg,
+    const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<ValueType, block_size>, 0,
+                       sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            reduction_helper_array_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                update_hessenberg_2_kernel<block_size>(
+                    iter, num_rows, num_cols, next_krylov_basis,
+                    stride_next_krylov, hessenberg_iter, stride_hessenberg,
+                    stop_status, item_ct1,
+                    (UninitializedArray<ValueType, block_size> *)
+                        reduction_helper_array_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+// Must be called with at least `num_rows * stride_krylov` threads in
+// total.
+template <int block_size, typename ValueType>
+void update_krylov_kernel(
+    size_type iter, size_type num_rows, size_type num_cols,
+    ValueType *__restrict__ krylov_bases, size_type stride_krylov,
+    const ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_idx = global_id / stride_krylov;
+    const auto col_idx = global_id % stride_krylov;
+    const auto hessenberg =
+        hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx];
+
+    if (row_idx < num_rows && col_idx < num_cols &&
+        !stop_status[col_idx].has_stopped()) {
+        const auto krylov_idx = row_idx * stride_krylov + col_idx;
+
+        krylov_bases[krylov_idx] /= hessenberg;
+    }
+}
+
+template <int block_size, typename ValueType>
+void update_krylov_kernel(dim3 grid, dim3 block,
+                          gko::size_type dynamic_shared_memory,
+                          sycl::queue *stream, size_type iter,
+                          size_type num_rows, size_type num_cols,
+                          ValueType *krylov_bases, size_type stride_krylov,
+                          const ValueType *hessenberg_iter,
+                          size_type stride_hessenberg,
+                          const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                update_krylov_kernel<block_size>(
+                    iter, num_rows, num_cols, krylov_bases, stride_krylov,
+                    hessenberg_iter, stride_hessenberg, stop_status, item_ct1);
+            });
+    });
+}
+
+
+// Must be called with at least `stride_preconditioner * num_rows` threads in
+// total.
+template <size_type block_size, typename ValueType>
+void calculate_Qy_kernel(size_type num_rows, size_type num_cols,
+                         size_type num_rhs,
+                         const ValueType *__restrict__ krylov_bases,
+                         size_type stride_krylov,
+                         const ValueType *__restrict__ y, size_type stride_y,
+                         ValueType *__restrict__ before_preconditioner,
+                         size_type stride_preconditioner,
+                         const size_type *__restrict__ final_iter_nums,
+                         sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row_id = global_id / stride_preconditioner;
+    const auto col_id = global_id % stride_preconditioner;
+
+    if (row_id < num_rows && col_id < num_cols) {
+        ValueType temp = zero<ValueType>();
+
+        for (size_type j = 0; j < final_iter_nums[col_id]; ++j) {
+            temp +=
+                krylov_bases[(row_id + j * num_rows) * stride_krylov + col_id] *
+                y[j * stride_y + col_id];
+        }
+        before_preconditioner[global_id] = temp;
+    }
+}
+
+template <size_type block_size, typename ValueType>
+void calculate_Qy_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type num_rows, size_type num_cols,
+    size_type num_rhs, const ValueType *krylov_bases, size_type stride_krylov,
+    const ValueType *y, size_type stride_y, ValueType *before_preconditioner,
+    size_type stride_preconditioner, const size_type *final_iter_nums)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                calculate_Qy_kernel<block_size>(
+                    num_rows, num_cols, num_rhs, krylov_bases, stride_krylov, y,
+                    stride_y, before_preconditioner, stride_preconditioner,
+                    final_iter_nums, item_ct1);
+            });
+    });
+}
 
 
 template <typename ValueType>
@@ -109,8 +704,22 @@ void initialize_1(std::shared_ptr<const DpcppExecutor> exec,
                   matrix::Dense<ValueType> *residual,
                   matrix::Dense<ValueType> *givens_sin,
                   matrix::Dense<ValueType> *givens_cos,
-                  Array<stopping_status> *stop_status,
-                  size_type krylov_dim) GKO_NOT_IMPLEMENTED;
+                  Array<stopping_status> *stop_status, size_type krylov_dim)
+{
+    const auto num_threads = std::max(b->get_size()[0] * b->get_stride(),
+                                      krylov_dim * b->get_size()[1]);
+    const dim3 grid_dim(ceildiv(num_threads, default_block_size), 1, 1);
+    const dim3 block_dim(default_block_size, 1, 1);
+    constexpr auto block_size = default_block_size;
+
+    initialize_1_kernel<block_size>(
+        grid_dim, block_dim, 0, exec->get_queue(), b->get_size()[0],
+        b->get_size()[1], krylov_dim, b->get_const_values(), b->get_stride(),
+        residual->get_values(), residual->get_stride(),
+        givens_sin->get_values(), givens_sin->get_stride(),
+        givens_cos->get_values(), givens_cos->get_stride(),
+        stop_status->get_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_INITIALIZE_1_KERNEL);
 
@@ -121,12 +730,125 @@ void initialize_2(std::shared_ptr<const DpcppExecutor> exec,
                   matrix::Dense<remove_complex<ValueType>> *residual_norm,
                   matrix::Dense<ValueType> *residual_norm_collection,
                   matrix::Dense<ValueType> *krylov_bases,
-                  Array<size_type> *final_iter_nums,
-                  size_type krylov_dim) GKO_NOT_IMPLEMENTED;
+                  Array<size_type> *final_iter_nums, size_type krylov_dim)
+{
+    const auto num_rows = residual->get_size()[0];
+    const auto num_rhs = residual->get_size()[1];
+    const dim3 grid_dim_1(
+        ceildiv(krylov_bases->get_size()[0] * krylov_bases->get_stride(),
+                default_block_size),
+        1, 1);
+    const dim3 block_dim(default_block_size, 1, 1);
+    constexpr auto block_size = default_block_size;
+
+    kernels::dpcpp::dense::compute_norm2(exec, residual, residual_norm);
+
+    const dim3 grid_dim_2(ceildiv(num_rows * num_rhs, default_block_size), 1,
+                          1);
+    initialize_2_2_kernel<block_size>(
+        grid_dim_2, block_dim, 0, exec->get_queue(), residual->get_size()[0],
+        residual->get_size()[1], residual->get_const_values(),
+        residual->get_stride(), residual_norm->get_const_values(),
+        residual_norm_collection->get_values(), krylov_bases->get_values(),
+        krylov_bases->get_stride(), final_iter_nums->get_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_INITIALIZE_2_KERNEL);
 
 
+template <typename ValueType>
+void finish_arnoldi(std::shared_ptr<const DpcppExecutor> exec,
+                    size_type num_rows, matrix::Dense<ValueType> *krylov_bases,
+                    matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
+                    const stopping_status *stop_status)
+{
+    const auto stride_krylov = krylov_bases->get_stride();
+    const auto stride_hessenberg = hessenberg_iter->get_stride();
+    auto cublas_handle = exec->get_cublas_handle();
+    const dim3 grid_size(
+        ceildiv(hessenberg_iter->get_size()[1], default_dot_dim),
+        exec->get_num_multiprocessor() * 2);
+    const dim3 block_size(default_dot_dim, default_dot_dim);
+    auto next_krylov_basis =
+        krylov_bases->get_values() +
+        (iter + 1) * num_rows * hessenberg_iter->get_size()[1];
+    for (size_type k = 0; k < iter + 1; ++k) {
+        const auto k_krylov_bases =
+            krylov_bases->get_const_values() +
+            k * num_rows * hessenberg_iter->get_size()[1];
+        if (hessenberg_iter->get_size()[1] > 1) {
+            // TODO: this condition should be tuned
+            // single rhs will use vendor's dot, otherwise, use our own
+            // multidot_kernel which parallelize multiple rhs.
+            components::fill_array(
+                exec, hessenberg_iter->get_values() + k * stride_hessenberg,
+                hessenberg_iter->get_size()[1], zero<ValueType>());
+            multidot_kernel(grid_size, block_size, 0, exec->get_queue(), k,
+                            num_rows, hessenberg_iter->get_size()[1],
+                            k_krylov_bases, next_krylov_basis, stride_krylov,
+                            hessenberg_iter->get_values(), stride_hessenberg,
+                            stop_status);
+        } else {
+            cublas::dot(exec->get_cublas_handle(), num_rows, k_krylov_bases,
+                        stride_krylov, next_krylov_basis, stride_krylov,
+                        hessenberg_iter->get_values() + k * stride_hessenberg);
+        }
+        update_next_krylov_kernel<default_block_size>(
+            ceildiv(num_rows * stride_krylov, default_block_size),
+            default_block_size, 0, exec->get_queue(), k, num_rows,
+            hessenberg_iter->get_size()[1], k_krylov_bases, next_krylov_basis,
+            stride_krylov, hessenberg_iter->get_const_values(),
+            stride_hessenberg, stop_status);
+    }
+    // for i in 1:iter
+    //     hessenberg(iter, i) = next_krylov_basis' * krylov_bases(:, i)
+    //     next_krylov_basis  -= hessenberg(iter, i) * krylov_bases(:, i)
+    // end
+
+
+    update_hessenberg_2_kernel<default_block_size>(
+        hessenberg_iter->get_size()[1], default_block_size, 0,
+        exec->get_queue(), iter, num_rows, hessenberg_iter->get_size()[1],
+        next_krylov_basis, stride_krylov, hessenberg_iter->get_values(),
+        stride_hessenberg, stop_status);
+
+    update_krylov_kernel<default_block_size>(
+        ceildiv(num_rows * stride_krylov, default_block_size),
+        default_block_size, 0, exec->get_queue(), iter, num_rows,
+        hessenberg_iter->get_size()[1], next_krylov_basis, stride_krylov,
+        hessenberg_iter->get_const_values(), stride_hessenberg, stop_status);
+    // next_krylov_basis /= hessenberg(iter, iter + 1)
+    // End of arnoldi
+}
+
+
+template <typename ValueType>
+void givens_rotation(std::shared_ptr<const DpcppExecutor> exec,
+                     matrix::Dense<ValueType> *givens_sin,
+                     matrix::Dense<ValueType> *givens_cos,
+                     matrix::Dense<ValueType> *hessenberg_iter,
+                     matrix::Dense<remove_complex<ValueType>> *residual_norm,
+                     matrix::Dense<ValueType> *residual_norm_collection,
+                     size_type iter, const Array<stopping_status> *stop_status)
+{
+    // TODO: tune block_size for optimal performance
+    constexpr auto block_size = default_block_size;
+    const auto num_cols = hessenberg_iter->get_size()[1];
+    const dim3 block_dim{block_size, 1, 1};
+    const dim3 grid_dim{
+        static_cast<unsigned int>(ceildiv(num_cols, block_size)), 1, 1};
+
+    givens_rotation_kernel<block_size>(
+        grid_dim, block_dim, 0, exec->get_queue(),
+        hessenberg_iter->get_size()[0], hessenberg_iter->get_size()[1], iter,
+        hessenberg_iter->get_values(), hessenberg_iter->get_stride(),
+        givens_sin->get_values(), givens_sin->get_stride(),
+        givens_cos->get_values(), givens_cos->get_stride(),
+        residual_norm->get_values(), residual_norm_collection->get_values(),
+        residual_norm_collection->get_stride(), stop_status->get_const_data());
+}
+
+
 template <typename ValueType>
 void step_1(std::shared_ptr<const DpcppExecutor> exec, size_type num_rows,
             matrix::Dense<ValueType> *givens_sin,
@@ -136,11 +858,75 @@ void step_1(std::shared_ptr<const DpcppExecutor> exec, size_type num_rows,
             matrix::Dense<ValueType> *krylov_bases,
             matrix::Dense<ValueType> *hessenberg_iter, size_type iter,
             Array<size_type> *final_iter_nums,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+            const Array<stopping_status> *stop_status)
+{
+    increase_final_iteration_numbers_kernel(
+        static_cast<unsigned int>(
+            ceildiv(final_iter_nums->get_num_elems(), default_block_size)),
+        default_block_size, 0, exec->get_queue(), final_iter_nums->get_data(),
+        stop_status->get_const_data(), final_iter_nums->get_num_elems());
+    finish_arnoldi(exec, num_rows, krylov_bases, hessenberg_iter, iter,
+                   stop_status->get_const_data());
+    givens_rotation(exec, givens_sin, givens_cos, hessenberg_iter,
+                    residual_norm, residual_norm_collection, iter, stop_status);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_STEP_1_KERNEL);
 
 
+template <typename ValueType>
+void solve_upper_triangular(
+    const matrix::Dense<ValueType> *residual_norm_collection,
+    const matrix::Dense<ValueType> *hessenberg, matrix::Dense<ValueType> *y,
+    const Array<size_type> *final_iter_nums)
+{
+    // TODO: tune block_size for optimal performance
+    constexpr auto block_size = default_block_size;
+    const auto num_rhs = residual_norm_collection->get_size()[1];
+    const dim3 block_dim{block_size, 1, 1};
+    const dim3 grid_dim{static_cast<unsigned int>(ceildiv(num_rhs, block_size)),
+                        1, 1};
+
+    solve_upper_triangular_kernel<block_size>(
+        grid_dim, block_dim, 0, exec->get_queue(), hessenberg->get_size()[1],
+        num_rhs, residual_norm_collection->get_const_values(),
+        residual_norm_collection->get_stride(), hessenberg->get_const_values(),
+        hessenberg->get_stride(), y->get_values(), y->get_stride(),
+        final_iter_nums->get_const_data());
+}
+
+
+template <typename ValueType>
+void calculate_qy(const matrix::Dense<ValueType> *krylov_bases,
+                  const matrix::Dense<ValueType> *y,
+                  matrix::Dense<ValueType> *before_preconditioner,
+                  const Array<size_type> *final_iter_nums)
+{
+    const auto num_rows = before_preconditioner->get_size()[0];
+    const auto num_cols = krylov_bases->get_size()[1];
+    const auto num_rhs = before_preconditioner->get_size()[1];
+    const auto stride_before_preconditioner =
+        before_preconditioner->get_stride();
+
+    constexpr auto block_size = default_block_size;
+    const dim3 grid_dim{
+        static_cast<unsigned int>(
+            ceildiv(num_rows * stride_before_preconditioner, block_size)),
+        1, 1};
+    const dim3 block_dim{block_size, 1, 1};
+
+
+    calculate_Qy_kernel<block_size>(
+        grid_dim, block_dim, 0, exec->get_queue(), num_rows, num_cols, num_rhs,
+        krylov_bases->get_const_values(), krylov_bases->get_stride(),
+        y->get_const_values(), y->get_stride(),
+        before_preconditioner->get_values(), stride_before_preconditioner,
+        final_iter_nums->get_const_data());
+    // Calculate qy
+    // before_preconditioner = krylov_bases * y
+}
+
+
 template <typename ValueType>
 void step_2(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Dense<ValueType> *residual_norm_collection,
@@ -148,7 +934,12 @@ void step_2(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Dense<ValueType> *hessenberg,
             matrix::Dense<ValueType> *y,
             matrix::Dense<ValueType> *before_preconditioner,
-            const Array<size_type> *final_iter_nums) GKO_NOT_IMPLEMENTED;
+            const Array<size_type> *final_iter_nums)
+{
+    solve_upper_triangular(residual_norm_collection, hessenberg, y,
+                           final_iter_nums);
+    calculate_qy(krylov_bases, y, before_preconditioner, final_iter_nums);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_STEP_2_KERNEL);
 
diff --git a/dpcpp/test/solver/cb_gmres_kernels.cpp b/dpcpp/test/solver/cb_gmres_kernels.cpp
new file mode 100644
index 00000000000..e9a5ec97f13
--- /dev/null
+++ b/dpcpp/test/solver/cb_gmres_kernels.cpp
@@ -0,0 +1,368 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/solver/cb_gmres.hpp>
+
+
+#include <cmath>
+#include <random>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
+
+
+#include "core/solver/cb_gmres_accessor.hpp"
+#include "core/solver/cb_gmres_kernels.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class CbGmres : public ::testing::Test {
+protected:
+    using value_type = double;
+    using storage_type = float;
+    using index_type = int;
+    using size_type = gko::size_type;
+    using Range3dHelper =
+        gko::cb_gmres::Range3dHelper<value_type, storage_type>;
+    using Range3d = typename Range3dHelper::Range;
+    using Dense = gko::matrix::Dense<value_type>;
+    using Mtx = Dense;
+    static constexpr unsigned int default_krylov_dim_mixed{100};
+
+    CbGmres() : rand_engine(30) {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        dpcpp = gko::DpcppExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (dpcpp != nullptr) {
+            ASSERT_NO_THROW(dpcpp->synchronize());
+        }
+    }
+
+    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<Mtx>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<index_type>(num_cols, num_cols),
+            std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    Range3dHelper generate_krylov_helper(gko::dim<3> size)
+    {
+        auto helper = Range3dHelper{ref, size};
+        auto &bases = helper.get_bases();
+        const auto num_rows = size[0] * size[1];
+        const auto num_cols = size[2];
+        auto temp_krylov_bases = gko::test::generate_random_matrix<Dense>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<index_type>(num_cols, num_cols),
+            std::normal_distribution<storage_type>(-1.0, 1.0), rand_engine,
+            ref);
+        std::copy_n(temp_krylov_bases->get_const_values(),
+                    bases.get_num_elems(), bases.get_data());
+        // Only useful when the Accessor actually has a scale
+        auto range = helper.get_range();
+        auto dist = std::normal_distribution<value_type>(-1, 1);
+        for (size_type k = 0; k < size[0]; ++k) {
+            for (size_type i = 0; i < size[2]; ++i) {
+                gko::cb_gmres::helper_functions_accessor<Range3d>::write_scalar(
+                    range, k, i, dist(rand_engine));
+            }
+        }
+        return helper;
+    }
+
+    void initialize_data()
+    {
+#ifdef GINKGO_FAST_TESTS
+        int m = 123;
+#else
+        int m = 597;
+#endif
+        int n = 43;
+        x = gen_mtx(m, n);
+        y = gen_mtx(default_krylov_dim_mixed, n);
+        before_preconditioner = Mtx::create_with_config_of(x.get());
+        b = gen_mtx(m, n);
+        arnoldi_norm = gen_mtx(3, n);
+        gko::dim<3> krylov_bases_dim(default_krylov_dim_mixed + 1, m, n);
+        range_helper = generate_krylov_helper(krylov_bases_dim);
+
+        next_krylov_basis = gen_mtx(m, n);
+        hessenberg =
+            gen_mtx(default_krylov_dim_mixed + 1, default_krylov_dim_mixed * n);
+        hessenberg_iter = gen_mtx(default_krylov_dim_mixed + 1, n);
+        buffer_iter = gen_mtx(default_krylov_dim_mixed + 1, n);
+        residual = gen_mtx(m, n);
+        residual_norm = gen_mtx(1, n);
+        residual_norm_collection = gen_mtx(default_krylov_dim_mixed + 1, n);
+        givens_sin = gen_mtx(default_krylov_dim_mixed, n);
+        givens_cos = gen_mtx(default_krylov_dim_mixed, n);
+        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(ref, n));
+        for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
+            stop_status->get_data()[i].reset();
+        }
+        reorth_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(ref, n));
+        for (size_t i = 0; i < reorth_status->get_num_elems(); ++i) {
+            reorth_status->get_data()[i].reset();
+        }
+        final_iter_nums = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(ref, n));
+        for (size_t i = 0; i < final_iter_nums->get_num_elems(); ++i) {
+            final_iter_nums->get_data()[i] = 5;
+        }
+        num_reorth = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(ref, n));
+        for (size_t i = 0; i < num_reorth->get_num_elems(); ++i) {
+            num_reorth->get_data()[i] = 5;
+        }
+
+        d_x = Mtx::create(dpcpp);
+        d_x->copy_from(x.get());
+        d_before_preconditioner = Mtx::create_with_config_of(d_x.get());
+        d_y = Mtx::create(dpcpp);
+        d_y->copy_from(y.get());
+        d_b = Mtx::create(dpcpp);
+        d_b->copy_from(b.get());
+        d_arnoldi_norm = Mtx::create(dpcpp);
+        d_arnoldi_norm->copy_from(arnoldi_norm.get());
+        d_range_helper = Range3dHelper{dpcpp, {}};
+        d_range_helper = range_helper;
+        d_next_krylov_basis = Mtx::create(dpcpp);
+        d_next_krylov_basis->copy_from(next_krylov_basis.get());
+        d_hessenberg = Mtx::create(dpcpp);
+        d_hessenberg->copy_from(hessenberg.get());
+        d_hessenberg_iter = Mtx::create(dpcpp);
+        d_hessenberg_iter->copy_from(hessenberg_iter.get());
+        d_buffer_iter = Mtx::create(dpcpp);
+        d_buffer_iter->copy_from(buffer_iter.get());
+        d_residual = Mtx::create(dpcpp);
+        d_residual->copy_from(residual.get());
+        d_residual_norm = Mtx::create(dpcpp);
+        d_residual_norm->copy_from(residual_norm.get());
+        d_residual_norm_collection = Mtx::create(dpcpp);
+        d_residual_norm_collection->copy_from(residual_norm_collection.get());
+        d_givens_sin = Mtx::create(dpcpp);
+        d_givens_sin->copy_from(givens_sin.get());
+        d_givens_cos = Mtx::create(dpcpp);
+        d_givens_cos->copy_from(givens_cos.get());
+        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(dpcpp, n));
+        *d_stop_status = *stop_status;
+        d_reorth_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(dpcpp, n));
+        *d_reorth_status = *reorth_status;
+        d_final_iter_nums = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(dpcpp, n));
+        *d_final_iter_nums = *final_iter_nums;
+        d_num_reorth = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(dpcpp, n));
+        *d_num_reorth = *num_reorth;
+    }
+
+    void assert_krylov_bases_near()
+    {
+        gko::Array<storage_type> d_to_host{ref};
+        auto &krylov_bases = range_helper.get_bases();
+        d_to_host = d_range_helper.get_bases();
+        const auto tolerance = r<storage_type>::value;
+        using sycl::abs;
+        for (gko::size_type i = 0; i < krylov_bases.get_num_elems(); ++i) {
+            const auto ref_value = krylov_bases.get_const_data()[i];
+            const auto dev_value = d_to_host.get_const_data()[i];
+            ASSERT_LE(fabsf(dev_value - ref_value), tolerance);
+        }
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
+
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Mtx> before_preconditioner;
+    std::unique_ptr<Mtx> x;
+    std::unique_ptr<Mtx> y;
+    std::unique_ptr<Mtx> b;
+    std::unique_ptr<Mtx> arnoldi_norm;
+    Range3dHelper range_helper;
+    std::unique_ptr<Mtx> next_krylov_basis;
+    std::unique_ptr<Mtx> hessenberg;
+    std::unique_ptr<Mtx> hessenberg_iter;
+    std::unique_ptr<Mtx> buffer_iter;
+    std::unique_ptr<Mtx> residual;
+    std::unique_ptr<Mtx> residual_norm;
+    std::unique_ptr<Mtx> residual_norm_collection;
+    std::unique_ptr<Mtx> givens_sin;
+    std::unique_ptr<Mtx> givens_cos;
+    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
+    std::unique_ptr<gko::Array<gko::stopping_status>> reorth_status;
+    std::unique_ptr<gko::Array<gko::size_type>> final_iter_nums;
+    std::unique_ptr<gko::Array<gko::size_type>> num_reorth;
+
+    std::unique_ptr<Mtx> d_x;
+    std::unique_ptr<Mtx> d_before_preconditioner;
+    std::unique_ptr<Mtx> d_y;
+    std::unique_ptr<Mtx> d_b;
+    std::unique_ptr<Mtx> d_arnoldi_norm;
+    Range3dHelper d_range_helper;
+    std::unique_ptr<Mtx> d_next_krylov_basis;
+    std::unique_ptr<Mtx> d_hessenberg;
+    std::unique_ptr<Mtx> d_hessenberg_iter;
+    std::unique_ptr<Mtx> d_buffer_iter;
+    std::unique_ptr<Mtx> d_residual;
+    std::unique_ptr<Mtx> d_residual_norm;
+    std::unique_ptr<Mtx> d_residual_norm_collection;
+    std::unique_ptr<Mtx> d_givens_sin;
+    std::unique_ptr<Mtx> d_givens_cos;
+    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
+    std::unique_ptr<gko::Array<gko::stopping_status>> d_reorth_status;
+    std::unique_ptr<gko::Array<gko::size_type>> d_final_iter_nums;
+    std::unique_ptr<gko::Array<gko::size_type>> d_num_reorth;
+};
+
+
+TEST_F(CbGmres, DpcppCbGmresInitialize1IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::cb_gmres::initialize_1(
+        ref, b.get(), residual.get(), givens_sin.get(), givens_cos.get(),
+        stop_status.get(), default_krylov_dim_mixed);
+    gko::kernels::dpcpp::cb_gmres::initialize_1(
+        dpcpp, d_b.get(), d_residual.get(), d_givens_sin.get(),
+        d_givens_cos.get(), d_stop_status.get(), default_krylov_dim_mixed);
+
+    GKO_ASSERT_MTX_NEAR(d_residual, residual, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
+    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
+}
+
+
+TEST_F(CbGmres, DpcppCbGmresInitialize2IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::cb_gmres::initialize_2(
+        ref, residual.get(), residual_norm.get(),
+        residual_norm_collection.get(), arnoldi_norm.get(),
+        range_helper.get_range(), next_krylov_basis.get(),
+        final_iter_nums.get(), default_krylov_dim_mixed);
+    gko::kernels::dpcpp::cb_gmres::initialize_2(
+        dpcpp, d_residual.get(), d_residual_norm.get(),
+        d_residual_norm_collection.get(), d_arnoldi_norm.get(),
+        d_range_helper.get_range(), d_next_krylov_basis.get(),
+        d_final_iter_nums.get(), default_krylov_dim_mixed);
+
+    GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
+                        1e-14);
+    assert_krylov_bases_near();
+    GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
+}
+
+
+TEST_F(CbGmres, DpcppCbGmresStep1IsEquivalentToRef)
+{
+    initialize_data();
+    int iter = 5;
+
+    gko::kernels::reference::cb_gmres::step_1(
+        ref, next_krylov_basis.get(), givens_sin.get(), givens_cos.get(),
+        residual_norm.get(), residual_norm_collection.get(),
+        range_helper.get_range(), hessenberg_iter.get(), buffer_iter.get(),
+        arnoldi_norm.get(), iter, final_iter_nums.get(), stop_status.get(),
+        reorth_status.get(), num_reorth.get());
+    gko::kernels::dpcpp::cb_gmres::step_1(
+        dpcpp, d_next_krylov_basis.get(), d_givens_sin.get(),
+        d_givens_cos.get(), d_residual_norm.get(),
+        d_residual_norm_collection.get(), d_range_helper.get_range(),
+        d_hessenberg_iter.get(), d_buffer_iter.get(), d_arnoldi_norm.get(),
+        iter, d_final_iter_nums.get(), d_stop_status.get(),
+        d_reorth_status.get(), d_num_reorth.get());
+
+    GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter, 1e-14);
+    assert_krylov_bases_near();
+    GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
+}
+
+
+TEST_F(CbGmres, DpcppCbGmresStep2IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::cb_gmres::step_2(
+        ref, residual_norm_collection.get(),
+        range_helper.get_range().get_accessor().to_const(), hessenberg.get(),
+        y.get(), before_preconditioner.get(), final_iter_nums.get());
+    gko::kernels::dpcpp::cb_gmres::step_2(
+        dpcpp, d_residual_norm_collection.get(),
+        d_range_helper.get_range().get_accessor().to_const(),
+        d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
+        d_final_iter_nums.get());
+
+    GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+}
+
+
+}  // namespace
diff --git a/dpcpp/test/solver/gmres_kernels.cpp b/dpcpp/test/solver/gmres_kernels.cpp
new file mode 100644
index 00000000000..a479d7b51bf
--- /dev/null
+++ b/dpcpp/test/solver/gmres_kernels.cpp
@@ -0,0 +1,303 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/solver/gmres.hpp>
+
+
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
+
+
+#include "core/solver/gmres_kernels.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class Gmres : public ::testing::Test {
+protected:
+    using value_type = gko::default_precision;
+    using index_type = gko::int32;
+    using Mtx = gko::matrix::Dense<value_type>;
+    using norm_type = gko::remove_complex<value_type>;
+    using NormVector = gko::matrix::Dense<norm_type>;
+    template <typename T>
+    using Dense = typename gko::matrix::Dense<T>;
+
+    Gmres() : rand_engine(30) {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        dpcpp = gko::DpcppExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (dpcpp != nullptr) {
+            ASSERT_NO_THROW(dpcpp->synchronize());
+        }
+    }
+
+    template <typename ValueType = value_type, typename IndexType = index_type>
+    std::unique_ptr<Dense<ValueType>> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<Dense<ValueType>>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<IndexType>(num_cols, num_cols),
+            std::normal_distribution<ValueType>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    void initialize_data(int nrhs = 43)
+    {
+#ifdef GINKGO_FAST_TESTS
+        int m = 123;
+#else
+        int m = 597;
+#endif
+        x = gen_mtx(m, nrhs);
+        y = gen_mtx(gko::solver::default_krylov_dim, nrhs);
+        before_preconditioner = Mtx::create_with_config_of(x.get());
+        b = gen_mtx(m, nrhs);
+        krylov_bases = gen_mtx(m * (gko::solver::default_krylov_dim + 1), nrhs);
+        hessenberg = gen_mtx(gko::solver::default_krylov_dim + 1,
+                             gko::solver::default_krylov_dim * nrhs);
+        hessenberg_iter = gen_mtx(gko::solver::default_krylov_dim + 1, nrhs);
+        residual = gen_mtx(m, nrhs);
+        residual_norm = gen_mtx<norm_type>(1, nrhs);
+        residual_norm_collection =
+            gen_mtx(gko::solver::default_krylov_dim + 1, nrhs);
+        givens_sin = gen_mtx(gko::solver::default_krylov_dim, nrhs);
+        givens_cos = gen_mtx(gko::solver::default_krylov_dim, nrhs);
+        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(ref, nrhs));
+        for (size_t i = 0; i < stop_status->get_num_elems(); ++i) {
+            stop_status->get_data()[i].reset();
+        }
+        final_iter_nums = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(ref, nrhs));
+        for (size_t i = 0; i < final_iter_nums->get_num_elems(); ++i) {
+            final_iter_nums->get_data()[i] = 5;
+        }
+
+        d_x = Mtx::create(dpcpp);
+        d_x->copy_from(x.get());
+        d_before_preconditioner = Mtx::create_with_config_of(d_x.get());
+        d_y = Mtx::create(dpcpp);
+        d_y->copy_from(y.get());
+        d_b = Mtx::create(dpcpp);
+        d_b->copy_from(b.get());
+        d_krylov_bases = Mtx::create(dpcpp);
+        d_krylov_bases->copy_from(krylov_bases.get());
+        d_hessenberg = Mtx::create(dpcpp);
+        d_hessenberg->copy_from(hessenberg.get());
+        d_hessenberg_iter = Mtx::create(dpcpp);
+        d_hessenberg_iter->copy_from(hessenberg_iter.get());
+        d_residual = Mtx::create(dpcpp);
+        d_residual->copy_from(residual.get());
+        d_residual_norm = NormVector::create(dpcpp);
+        d_residual_norm->copy_from(residual_norm.get());
+        d_residual_norm_collection = Mtx::create(dpcpp);
+        d_residual_norm_collection->copy_from(residual_norm_collection.get());
+        d_givens_sin = Mtx::create(dpcpp);
+        d_givens_sin->copy_from(givens_sin.get());
+        d_givens_cos = Mtx::create(dpcpp);
+        d_givens_cos->copy_from(givens_cos.get());
+        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(dpcpp, nrhs));
+        *d_stop_status = *stop_status;
+        d_final_iter_nums = std::unique_ptr<gko::Array<gko::size_type>>(
+            new gko::Array<gko::size_type>(dpcpp, nrhs));
+        *d_final_iter_nums = *final_iter_nums;
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
+
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Mtx> before_preconditioner;
+    std::unique_ptr<Mtx> x;
+    std::unique_ptr<Mtx> y;
+    std::unique_ptr<Mtx> b;
+    std::unique_ptr<Mtx> krylov_bases;
+    std::unique_ptr<Mtx> hessenberg;
+    std::unique_ptr<Mtx> hessenberg_iter;
+    std::unique_ptr<Mtx> residual;
+    std::unique_ptr<NormVector> residual_norm;
+    std::unique_ptr<Mtx> residual_norm_collection;
+    std::unique_ptr<Mtx> givens_sin;
+    std::unique_ptr<Mtx> givens_cos;
+    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
+    std::unique_ptr<gko::Array<gko::size_type>> final_iter_nums;
+
+    std::unique_ptr<Mtx> d_x;
+    std::unique_ptr<Mtx> d_before_preconditioner;
+    std::unique_ptr<Mtx> d_y;
+    std::unique_ptr<Mtx> d_b;
+    std::unique_ptr<Mtx> d_krylov_bases;
+    std::unique_ptr<Mtx> d_hessenberg;
+    std::unique_ptr<Mtx> d_hessenberg_iter;
+    std::unique_ptr<Mtx> d_residual;
+    std::unique_ptr<NormVector> d_residual_norm;
+    std::unique_ptr<Mtx> d_residual_norm_collection;
+    std::unique_ptr<Mtx> d_givens_sin;
+    std::unique_ptr<Mtx> d_givens_cos;
+    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
+    std::unique_ptr<gko::Array<gko::size_type>> d_final_iter_nums;
+};
+
+
+TEST_F(Gmres, DpcppGmresInitialize1IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::gmres::initialize_1(
+        ref, b.get(), residual.get(), givens_sin.get(), givens_cos.get(),
+        stop_status.get(), gko::solver::default_krylov_dim);
+    gko::kernels::dpcpp::gmres::initialize_1(
+        dpcpp, d_b.get(), d_residual.get(), d_givens_sin.get(),
+        d_givens_cos.get(), d_stop_status.get(),
+        gko::solver::default_krylov_dim);
+
+    GKO_ASSERT_MTX_NEAR(d_residual, residual, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
+    GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
+}
+
+
+TEST_F(Gmres, DpcppGmresInitialize2IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::gmres::initialize_2(
+        ref, residual.get(), residual_norm.get(),
+        residual_norm_collection.get(), krylov_bases.get(),
+        final_iter_nums.get(), gko::solver::default_krylov_dim);
+    gko::kernels::dpcpp::gmres::initialize_2(
+        dpcpp, d_residual.get(), d_residual_norm.get(),
+        d_residual_norm_collection.get(), d_krylov_bases.get(),
+        d_final_iter_nums.get(), gko::solver::default_krylov_dim);
+
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, 1e-14);
+    GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
+}
+
+
+TEST_F(Gmres, DpcppGmresStep1IsEquivalentToRef)
+{
+    initialize_data();
+    int iter = 5;
+
+    gko::kernels::reference::gmres::step_1(
+        ref, x->get_size()[0], givens_sin.get(), givens_cos.get(),
+        residual_norm.get(), residual_norm_collection.get(), krylov_bases.get(),
+        hessenberg_iter.get(), iter, final_iter_nums.get(), stop_status.get());
+    gko::kernels::dpcpp::gmres::step_1(
+        dpcpp, d_x->get_size()[0], d_givens_sin.get(), d_givens_cos.get(),
+        d_residual_norm.get(), d_residual_norm_collection.get(),
+        d_krylov_bases.get(), d_hessenberg_iter.get(), iter,
+        d_final_iter_nums.get(), d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, 1e-14);
+    GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
+}
+
+
+TEST_F(Gmres, DpcppGmresStep1OnSingleRHSIsEquivalentToRef)
+{
+    initialize_data(1);
+    int iter = 5;
+
+    gko::kernels::reference::gmres::step_1(
+        ref, x->get_size()[0], givens_sin.get(), givens_cos.get(),
+        residual_norm.get(), residual_norm_collection.get(), krylov_bases.get(),
+        hessenberg_iter.get(), iter, final_iter_nums.get(), stop_status.get());
+    gko::kernels::dpcpp::gmres::step_1(
+        dpcpp, d_x->get_size()[0], d_givens_sin.get(), d_givens_cos.get(),
+        d_residual_norm.get(), d_residual_norm_collection.get(),
+        d_krylov_bases.get(), d_hessenberg_iter.get(), iter,
+        d_final_iter_nums.get(), d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
+                        1e-14);
+    GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, 1e-14);
+    GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
+}
+
+
+TEST_F(Gmres, DpcppGmresStep2IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::gmres::step_2(ref, residual_norm_collection.get(),
+                                           krylov_bases.get(), hessenberg.get(),
+                                           y.get(), before_preconditioner.get(),
+                                           final_iter_nums.get());
+    gko::kernels::dpcpp::gmres::step_2(dpcpp, d_residual_norm_collection.get(),
+                                       d_krylov_bases.get(), d_hessenberg.get(),
+                                       d_y.get(), d_before_preconditioner.get(),
+                                       d_final_iter_nums.get());
+
+    GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+}
+
+
+}  // namespace

From a995049541df64a7798dd1ce39c98fe94663005c Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 20 Jul 2021 08:19:23 +0200
Subject: [PATCH 0933/1008] fix issue from porting and add atomic max bind

---
 cmake/GinkgoConfig.cmake.in             |   2 +
 dpcpp/CMakeLists.txt                    |   4 +-
 dpcpp/base/onemkl_bindings.hpp          |   4 +-
 dpcpp/components/atomic.dp.hpp          |  84 +++++++++++++++
 dpcpp/solver/cb_gmres_kernels.dp.cpp    | 129 ++++++++++--------------
 dpcpp/solver/gmres_kernels.dp.cpp       |  39 +++----
 dpcpp/stop/residual_norm_kernels.dp.cpp |   2 +-
 dpcpp/test/CMakeLists.txt               |   1 +
 dpcpp/test/solver/CMakeLists.txt        |   2 +
 dpcpp/test/solver/cb_gmres_kernels.cpp  |  39 ++++---
 dpcpp/test/solver/gmres_kernels.cpp     |  50 +++++----
 test/solver/ir_kernels.cpp              |  12 ---
 12 files changed, 216 insertions(+), 152 deletions(-)
 create mode 100644 dpcpp/test/solver/CMakeLists.txt

diff --git a/cmake/GinkgoConfig.cmake.in b/cmake/GinkgoConfig.cmake.in
index 4675bcb4781..f3e2cf2b430 100644
--- a/cmake/GinkgoConfig.cmake.in
+++ b/cmake/GinkgoConfig.cmake.in
@@ -79,6 +79,7 @@ set(GINKGO_AMD_ARCH_FLAGS @GINKGO_AMD_ARCH_FLAGS@)
 set(GINKGO_DPCPP_VERSION @GINKGO_DPCPP_VERSION@)
 set(GINKGO_DPCPP_FLAGS @GINKGO_DPCPP_FLAGS@)
 set(GINKGO_MKL_ROOT @GINKGO_MKL_ROOT@)
+set(GINKGO_DPL_ROOT @GINKGO_DPL_ROOT@)
 
 set(GINKGO_HAVE_PAPI_SDE @GINKGO_HAVE_PAPI_SDE@)
 
@@ -171,6 +172,7 @@ endif()
 
 if((NOT GINKGO_BUILD_SHARED_LIBS) AND GINKGO_BUILD_DPCPP)
     find_package(MKL CONFIG REQUIRED HINTS "${GINKGO_MKL_ROOT}")
+    find_package(oneDPL REQUIRED HINTS "${GINKGO_DPL_ROOT}")
 endif()
 
 include(${CMAKE_CURRENT_LIST_DIR}/GinkgoTargets.cmake)
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index e2d476164e8..401be197820 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -8,6 +8,8 @@ set(GINKGO_DPCPP_VERSION ${GINKGO_DPCPP_VERSION} PARENT_SCOPE)
 
 find_package(MKL CONFIG REQUIRED HINTS "$ENV{MKLROOT}")
 set(GINKGO_MKL_ROOT "${MKL_ROOT}" PARENT_SCOPE)
+find_package(oneDPL REQUIRED HINTS "$ENV{DPL_ROOT}")
+set(GINKGO_DPL_ROOT "${DPL_ROOT}" PARENT_SCOPE)
 
 add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_dpcpp_device> "")
 target_sources(ginkgo_dpcpp
@@ -75,7 +77,7 @@ else ()
     target_link_options(ginkgo_dpcpp PUBLIC -fsycl-device-code-split=per_kernel)
 endif()
 target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
-target_link_libraries(ginkgo_dpcpp PRIVATE MKL::MKL_DPCPP)
+target_link_libraries(ginkgo_dpcpp PRIVATE MKL::MKL_DPCPP oneDPL)
 if (GINKGO_DPCPP_SINGLE_MODE)
     target_compile_definitions(ginkgo_dpcpp PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
 endif()
diff --git a/dpcpp/base/onemkl_bindings.hpp b/dpcpp/base/onemkl_bindings.hpp
index 1c9a8dabb30..cb1b944d12c 100644
--- a/dpcpp/base/onemkl_bindings.hpp
+++ b/dpcpp/base/onemkl_bindings.hpp
@@ -90,7 +90,7 @@ struct is_supported<std::complex<float>> : std::true_type {};
 template <>
 struct is_supported<std::complex<double>> : std::true_type {};
 
-
+namespace {
 #define GKO_BIND_DOT(ValueType, Name, Func)                                  \
     void Name(::cl::sycl::queue &exec_queue, std::int64_t n,                 \
               const ValueType *x, std::int64_t incx, const ValueType *y,     \
@@ -121,7 +121,7 @@ GKO_BIND_DOT(ValueType, conj_dot, detail::not_implemented);
 
 #undef GKO_BIND_DOT
 
-
+}  // namespace
 }  // namespace onemkl
 }  // namespace dpcpp
 }  // namespace kernels
diff --git a/dpcpp/components/atomic.dp.hpp b/dpcpp/components/atomic.dp.hpp
index 78191d4da4b..c2048b1510b 100644
--- a/dpcpp/components/atomic.dp.hpp
+++ b/dpcpp/components/atomic.dp.hpp
@@ -95,6 +95,18 @@ inline T atomic_fetch_add(
 }
 
 
+template <cl::sycl::access::address_space addressSpace = atomic::global_space,
+          typename T>
+inline T atomic_fetch_max(
+    T *addr, T operand,
+    cl::sycl::memory_order memoryOrder = cl::sycl::memory_order::relaxed)
+{
+    cl::sycl::atomic<T, addressSpace> obj(
+        (cl::sycl::multi_ptr<T, addressSpace>(addr)));
+    return cl::sycl::atomic_fetch_max(obj, operand, memoryOrder);
+}
+
+
 }  // namespace
 
 
@@ -114,6 +126,19 @@ struct atomic_helper {
 };
 
 
+template <cl::sycl::access::address_space addressSpace, typename ValueType,
+          typename = void>
+struct atomic_max_helper {
+    __dpct_inline__ static ValueType atomic_max(ValueType *, ValueType)
+    {
+        static_assert(sizeof(ValueType) == 0,
+                      "This default function is not implemented, only the "
+                      "specializations are.");
+        // TODO: add proper implementation of generic atomic max
+    }
+};
+
+
 template <typename ResultType, typename ValueType>
 __dpct_inline__ ResultType reinterpret(ValueType val)
 {
@@ -192,6 +217,57 @@ struct atomic_helper<
 };
 
 
+#define GKO_BIND_ATOMIC_MAX_STRUCTURE(CONVERTER_TYPE)                      \
+    template <cl::sycl::access::address_space addressSpace,                \
+              typename ValueType>                                          \
+    struct atomic_max_helper<                                              \
+        addressSpace, ValueType,                                           \
+        std::enable_if_t<(sizeof(ValueType) == sizeof(CONVERTER_TYPE))>> { \
+        __dpct_inline__ static ValueType atomic_max(                       \
+            ValueType *__restrict__ addr, ValueType val)                   \
+        {                                                                  \
+            CONVERTER_TYPE *address_as_converter =                         \
+                reinterpret_cast<CONVERTER_TYPE *>(addr);                  \
+            CONVERTER_TYPE old = *address_as_converter;                    \
+            CONVERTER_TYPE assumed;                                        \
+            do {                                                           \
+                assumed = old;                                             \
+                if (reinterpret<ValueType>(assumed) < val) {               \
+                    old = atomic_compare_exchange_strong<addressSpace>(    \
+                        address_as_converter, assumed,                     \
+                        reinterpret<CONVERTER_TYPE>(val));                 \
+                }                                                          \
+            } while (assumed != old);                                      \
+            return reinterpret<ValueType>(old);                            \
+        }                                                                  \
+    };
+
+// Support 64-bit ATOMIC_ADD
+GKO_BIND_ATOMIC_MAX_STRUCTURE(unsigned long long int);
+// Support 32-bit ATOMIC_ADD
+GKO_BIND_ATOMIC_MAX_STRUCTURE(unsigned int);
+
+
+#undef GKO_BIND_ATOMIC_MAX_STRUCTURE
+
+#define GKO_BIND_ATOMIC_MAX_VALUETYPE(ValueType)              \
+    template <cl::sycl::access::address_space addressSpace>   \
+    struct atomic_max_helper<addressSpace, ValueType,         \
+                             std::enable_if_t<true>> {        \
+        __dpct_inline__ static ValueType atomic_max(          \
+            ValueType *__restrict__ addr, ValueType val)      \
+        {                                                     \
+            return atomic_fetch_max<addressSpace>(addr, val); \
+        }                                                     \
+    };
+
+GKO_BIND_ATOMIC_MAX_VALUETYPE(int);
+GKO_BIND_ATOMIC_MAX_VALUETYPE(unsigned int);
+GKO_BIND_ATOMIC_MAX_VALUETYPE(unsigned long long int);
+
+#undef GKO_BIND_ATOMIC_MAX_VALUETYPE
+
+
 }  // namespace detail
 
 
@@ -203,6 +279,14 @@ __dpct_inline__ T atomic_add(T *__restrict__ addr, T val)
 }
 
 
+template <cl::sycl::access::address_space addressSpace = atomic::global_space,
+          typename T>
+__dpct_inline__ T atomic_max(T *__restrict__ addr, T val)
+{
+    return detail::atomic_max_helper<addressSpace, T>::atomic_max(addr, val);
+}
+
+
 }  // namespace dpcpp
 }  // namespace kernels
 }  // namespace gko
diff --git a/dpcpp/solver/cb_gmres_kernels.dp.cpp b/dpcpp/solver/cb_gmres_kernels.dp.cpp
index 15bf601cd26..cf46a44b4b1 100644
--- a/dpcpp/solver/cb_gmres_kernels.dp.cpp
+++ b/dpcpp/solver/cb_gmres_kernels.dp.cpp
@@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <algorithm>
-#include <dpcpp/base/math.hpp>
 
 
 #include <CL/sycl.hpp>
@@ -72,8 +71,8 @@ namespace dpcpp {
 namespace cb_gmres {
 
 
-constexpr int default_block_size = 512;
-constexpr int default_dot_dim = 32;
+constexpr int default_block_size = 256;
+constexpr int default_dot_dim = 16;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
@@ -150,18 +149,10 @@ void calculate_sin_and_cos_kernel(size_type col_idx, size_type num_cols,
         const auto scale = std::abs(this_hess) + std::abs(next_hess);
         const auto hypotenuse =
             scale *
-            sycl::sqrt(
+            std::sqrt(
                 std::abs(this_hess / scale) * std::abs(this_hess / scale) +
                 std::abs(next_hess / scale) * std::abs(next_hess / scale));
-        /*
-        DPCT1007:1: Migration of this CUDA API is not supported by the Intel(R)
-        DPC++ Compatibility Tool.
-        */
         register_cos = conj(this_hess) / hypotenuse;
-        /*
-        DPCT1007:2: Migration of this CUDA API is not supported by the Intel(R)
-        DPC++ Compatibility Tool.
-        */
         register_sin = conj(next_hess) / hypotenuse;
     }
     givens_cos[iter * stride_cos + col_idx] = register_cos;
@@ -181,10 +172,6 @@ void calculate_residual_norm_kernel(size_type col_idx, size_type num_cols,
     const auto this_rnc =
         residual_norm_collection[iter * stride_residual_norm_collection +
                                  col_idx];
-    /*
-    DPCT1007:3: Migration of this CUDA API is not supported by the Intel(R)
-    DPC++ Compatibility Tool.
-    */
     const auto next_rnc = -conj(register_sin) * this_rnc;
     residual_norm_collection[iter * stride_residual_norm_collection + col_idx] =
         register_cos * this_rnc;
@@ -219,10 +206,6 @@ void givens_rotation_kernel(
         const auto sin = givens_sin[i * stride_sin + col_idx];
         hessenberg_iter[i * stride_hessenberg + col_idx] =
             cos * this_hess + sin * next_hess;
-        /*
-        DPCT1007:4: Migration of this CUDA API is not supported by the Intel(R)
-        DPC++ Compatibility Tool.
-        */
         this_hess = conj(cos) * next_hess - conj(sin) * this_hess;
         next_hess = hessenberg_iter[(i + 2) * stride_hessenberg + col_idx];
     }
@@ -497,7 +480,8 @@ void multinorm2_kernel(
     const ValueType *__restrict__ next_krylov_basis,
     size_type stride_next_krylov, remove_complex<ValueType> *__restrict__ norms,
     const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1,
-    UninitializedArray<rc_vtype, default_dot_dim *(default_dot_dim + 1)>
+    UninitializedArray<remove_complex<ValueType>,
+                       default_dot_dim *(default_dot_dim + 1)>
         *reduction_helper_array)
 {
     using rc_vtype = remove_complex<ValueType>;
@@ -526,9 +510,9 @@ void multinorm2_kernel(
     local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
     const auto tile_block = group::tiled_partition<default_dot_dim>(
         group::this_thread_block(item_ct1));
-    const auto sum =
-        reduce(tile_block, local_res,
-               [](const rc_vtype &a, const rc_vtype &b) { return a + b; });
+    const auto sum = ::gko::kernels::dpcpp::reduce(
+        tile_block, local_res,
+        [](const rc_vtype &a, const rc_vtype &b) { return a + b; });
     const auto new_col_idx = item_ct1.get_group(2) * default_dot_dim + tidy;
     if (tidx == 0 && new_col_idx < num_cols &&
         !stop_status[new_col_idx].has_stopped()) {
@@ -547,10 +531,10 @@ void multinorm2_kernel(dim3 grid, dim3 block,
                        const stopping_status *stop_status)
 {
     stream->submit([&](sycl::handler &cgh) {
-        sycl::accessor<UninitializedArray<rc_vtype, default_dot_dim *(
-                                                        default_dot_dim + 1)>,
-                       0, sycl::access_mode::read_write,
-                       sycl::access::target::local>
+        sycl::accessor<
+            UninitializedArray<remove_complex<ValueType>,
+                               default_dot_dim *(default_dot_dim + 1)>,
+            0, sycl::access_mode::read_write, sycl::access::target::local>
             reduction_helper_array_acc_ct1(cgh);
 
         cgh.parallel_for(
@@ -570,7 +554,8 @@ void multinorminf_without_stop_kernel(
     const ValueType *__restrict__ next_krylov_basis,
     size_type stride_next_krylov, remove_complex<ValueType> *__restrict__ norms,
     size_type stride_norms, sycl::nd_item<3> item_ct1,
-    UninitializedArray<rc_vtype, default_dot_dim *(default_dot_dim + 1)>
+    UninitializedArray<remove_complex<ValueType>,
+                       default_dot_dim *(default_dot_dim + 1)>
         *reduction_helper_array)
 {
     using rc_vtype = remove_complex<ValueType>;
@@ -602,8 +587,8 @@ void multinorminf_without_stop_kernel(
     local_max = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
     const auto tile_block = group::tiled_partition<default_dot_dim>(
         group::this_thread_block(item_ct1));
-    const auto value =
-        reduce(tile_block, local_max, [](const rc_vtype &a, const rc_vtype &b) {
+    const auto value = ::gko::kernels::dpcpp::reduce(
+        tile_block, local_max, [](const rc_vtype &a, const rc_vtype &b) {
             return ((a >= b) ? a : b);
         });
     const auto new_col_idx = item_ct1.get_group(2) * default_dot_dim + tidy;
@@ -621,10 +606,10 @@ void multinorminf_without_stop_kernel(
     remove_complex<ValueType> *norms, size_type stride_norms)
 {
     stream->submit([&](sycl::handler &cgh) {
-        sycl::accessor<UninitializedArray<rc_vtype, default_dot_dim *(
-                                                        default_dot_dim + 1)>,
-                       0, sycl::access_mode::read_write,
-                       sycl::access::target::local>
+        sycl::accessor<
+            UninitializedArray<remove_complex<ValueType>,
+                               default_dot_dim *(default_dot_dim + 1)>,
+            0, sycl::access_mode::read_write, sycl::access::target::local>
             reduction_helper_array_acc_ct1(cgh);
 
         cgh.parallel_for(
@@ -647,8 +632,9 @@ void multinorm2_inf_kernel(
     remove_complex<ValueType> *__restrict__ norms1,
     remove_complex<ValueType> *__restrict__ norms2,
     const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1,
-    UninitializedArray<rc_vtype, (1 + compute_inf) *
-                                     default_dot_dim *(default_dot_dim + 1)>
+    UninitializedArray<remove_complex<ValueType>,
+                       (1 + compute_inf) *
+                           default_dot_dim *(default_dot_dim + 1)>
         *reduction_helper_array)
 {
     using rc_vtype = remove_complex<ValueType>;
@@ -690,16 +676,16 @@ void multinorm2_inf_kernel(
     local_res = reduction_helper_add[tidy * (default_dot_dim + 1) + tidx];
     const auto tile_block = group::tiled_partition<default_dot_dim>(
         group::this_thread_block(item_ct1));
-    const auto sum =
-        reduce(tile_block, local_res,
-               [](const rc_vtype &a, const rc_vtype &b) { return a + b; });
+    const auto sum = ::gko::kernels::dpcpp::reduce(
+        tile_block, local_res,
+        [](const rc_vtype &a, const rc_vtype &b) { return a + b; });
     rc_vtype reduced_max{};
     if (compute_inf) {
         local_max = reduction_helper_max[tidy * (default_dot_dim + 1) + tidx];
-        reduced_max = reduce(tile_block, local_max,
-                             [](const rc_vtype &a, const rc_vtype &b) {
-                                 return ((a >= b) ? a : b);
-                             });
+        reduced_max = ::gko::kernels::dpcpp::reduce(
+            tile_block, local_max, [](const rc_vtype &a, const rc_vtype &b) {
+                return ((a >= b) ? a : b);
+            });
     }
     const auto new_col_idx = item_ct1.get_group(2) * default_dot_dim + tidy;
     if (tidx == 0 && new_col_idx < num_cols &&
@@ -722,7 +708,7 @@ void multinorm2_inf_kernel(
 {
     stream->submit([&](sycl::handler &cgh) {
         sycl::accessor<
-            UninitializedArray<rc_vtype,
+            UninitializedArray<remove_complex<ValueType>,
                                (1 + compute_inf) *
                                    default_dot_dim *(default_dot_dim + 1)>,
             0, sycl::access_mode::read_write, sycl::access::target::local>
@@ -782,10 +768,6 @@ void multidot_kernel(
              i += item_ct1.get_local_range().get(1)) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
             ValueType other_basis = krylov_bases(k, i, col_idx);
-            /*
-            DPCT1007:14: Migration of this CUDA API is not supported by the
-            Intel(R) DPC++ Compatibility Tool.
-            */
             local_res += next_krylov_basis[next_krylov_idx] * conj(other_basis);
         }
     }
@@ -798,9 +780,9 @@ void multidot_kernel(
     const auto new_col_idx =
         item_ct1.get_group(2) * item_ct1.get_local_range().get(2) + tidy;
     const auto tile_block = group::tiled_partition<dot_dim>(thread_block);
-    const auto sum =
-        reduce(tile_block, local_res,
-               [](const ValueType &a, const ValueType &b) { return a + b; });
+    const auto sum = ::gko::kernels::dpcpp::reduce(
+        tile_block, local_res,
+        [](const ValueType &a, const ValueType &b) { return a + b; });
     if (tidx == 0 && new_col_idx < num_cols &&
         !stop_status[new_col_idx].has_stopped()) {
         const auto hessenberg_idx = k * stride_hessenberg + new_col_idx;
@@ -873,10 +855,6 @@ void singledot_kernel(
         for (size_type i = start_row; i < end_row; i += block_size) {
             const auto next_krylov_idx = i * stride_next_krylov + col_idx;
             ValueType other_basis = krylov_bases(k, i, col_idx);
-            /*
-            DPCT1007:16: Migration of this CUDA API is not supported by the
-            Intel(R) DPC++ Compatibility Tool.
-            */
             local_res += next_krylov_basis[next_krylov_idx] * conj(other_basis);
         }
     }
@@ -885,8 +863,9 @@ void singledot_kernel(
     reduction_helper[tidx] = local_res;
     auto thread_block = group::this_thread_block(item_ct1);
     thread_block.sync();
-    reduce(thread_block, reduction_helper,
-           [](const ValueType &a, const ValueType &b) { return a + b; });
+    ::gko::kernels::dpcpp::reduce(
+        thread_block, reduction_helper,
+        [](const ValueType &a, const ValueType &b) { return a + b; });
     if (tidx == 0 && !stop_status[col_idx].has_stopped()) {
         const auto hessenberg_idx = k * stride_hessenberg + col_idx;
         atomic_add(hessenberg_iter + hessenberg_idx, reduction_helper[0]);
@@ -1043,9 +1022,8 @@ void check_arnoldi_norms(
         gko::cb_gmres::detail::has_3d_scaled_accessor<Accessor3d>::value;
 
     if (col_idx < num_rhs && !stop_status[col_idx].has_stopped()) {
-        const auto num0 = (sycl::sqrt(eta_squared * arnoldi_norm[col_idx]));
-        const auto num11 =
-            sycl::sqrt((double)(arnoldi_norm[col_idx + stride_norm]));
+        const auto num0 = (std::sqrt(eta_squared * arnoldi_norm[col_idx]));
+        const auto num11 = std::sqrt(arnoldi_norm[col_idx + stride_norm]);
         const auto num2 = has_scalar ? (arnoldi_norm[col_idx + 2 * stride_norm])
                                      : remove_complex<ValueType>{};
         if (num11 < num0) {
@@ -1235,8 +1213,7 @@ template <int dim, typename Type1, typename Type2>
 GKO_INLINE auto as_dpcpp_accessor(
     const acc::range<acc::reduced_row_major<dim, Type1, Type2>> &acc)
 {
-    return acc::range<
-        acc::reduced_row_major<dim, dpcpp_type<Type1>, dpcpp_type<Type2>>>(
+    return acc::range<acc::reduced_row_major<dim, Type1, Type2>>(
         acc.get_accessor().get_size(), acc.get_accessor().get_stored_data(),
         acc.get_accessor().get_stride());
 }
@@ -1246,8 +1223,7 @@ GKO_INLINE auto as_dpcpp_accessor(
     const acc::range<acc::scaled_reduced_row_major<dim, Type1, Type2, mask>>
         &acc)
 {
-    return acc::range<acc::scaled_reduced_row_major<dim, dpcpp_type<Type1>,
-                                                    dpcpp_type<Type2>, mask>>(
+    return acc::range<acc::scaled_reduced_row_major<dim, Type1, Type2, mask>>(
         acc.get_accessor().get_size(), acc.get_accessor().get_stored_data(),
         acc.get_accessor().get_storage_stride(),
         acc.get_accessor().get_scalar(),
@@ -1256,7 +1232,8 @@ GKO_INLINE auto as_dpcpp_accessor(
 
 
 template <typename ValueType>
-void zero_matrix(size_type m, size_type n, size_type stride, ValueType *array)
+void zero_matrix(std::shared_ptr<const DpcppExecutor> exec, size_type m,
+                 size_type n, size_type stride, ValueType *array)
 {
     const dim3 block_size(default_block_size, 1, 1);
     const dim3 grid_size(ceildiv(n, block_size.x), 1, 1);
@@ -1326,7 +1303,7 @@ void initialize_2(std::shared_ptr<const DpcppExecutor> exec,
                                arnoldi_norm->get_values() + 2 * stride_arnoldi,
                                num_rhs, zero<remove_complex<ValueType>>());
         const dim3 grid_size_nrm(ceildiv(num_rhs, default_dot_dim),
-                                 exec->get_num_multiprocessor() * 2);
+                                 exec->get_num_computing_units() * 2);
         const dim3 block_size_nrm(default_dot_dim, default_dot_dim);
         multinorminf_without_stop_kernel(
             grid_size_nrm, block_size_nrm, 0, exec->get_queue(), num_rows,
@@ -1380,15 +1357,15 @@ void finish_arnoldi_CGS(std::shared_ptr<const DpcppExecutor> exec,
     const auto stride_arnoldi = arnoldi_norm->get_stride();
     const auto dim_size = next_krylov_basis->get_size();
     const dim3 grid_size(ceildiv(dim_size[1], default_dot_dim),
-                         exec->get_num_multiprocessor() * 2);
+                         exec->get_num_computing_units() * 2);
     const dim3 grid_size_num_iters(ceildiv(dim_size[1], default_dot_dim),
-                                   exec->get_num_multiprocessor() * 2,
+                                   exec->get_num_computing_units() * 2,
                                    iter + 1);
     const dim3 block_size(default_dot_dim, default_dot_dim);
     // Note: having iter first (instead of row_idx information) is likely
     //       beneficial for avoiding atomic_add conflicts, but that needs
     //       further investigation.
-    const dim3 grid_size_iters_single(exec->get_num_multiprocessor() * 2,
+    const dim3 grid_size_iters_single(exec->get_num_computing_units() * 2,
                                       iter + 1);
     const dim3 block_size_iters_single(singledot_block_size);
     size_type num_reorth_host;
@@ -1399,7 +1376,7 @@ void finish_arnoldi_CGS(std::shared_ptr<const DpcppExecutor> exec,
                       dim_size[1], next_krylov_basis->get_const_values(),
                       stride_next_krylov, arnoldi_norm->get_values(),
                       stop_status);
-    zero_matrix(iter + 1, dim_size[1], stride_hessenberg,
+    zero_matrix(exec, iter + 1, dim_size[1], stride_hessenberg,
                 hessenberg_iter->get_values());
     if (dim_size[1] > 1) {
         multidot_kernel<default_dot_dim>(
@@ -1451,7 +1428,7 @@ void finish_arnoldi_CGS(std::shared_ptr<const DpcppExecutor> exec,
     num_reorth_host = exec->copy_val_to_host(num_reorth->get_const_data());
     // num_reorth_host := number of next_krylov vector to be reorthogonalization
     for (size_type l = 1; (num_reorth_host > 0) && (l < 3); l++) {
-        zero_matrix(iter + 1, dim_size[1], stride_buffer,
+        zero_matrix(exec, iter + 1, dim_size[1], stride_buffer,
                     buffer_iter->get_values());
         if (dim_size[1] > 1) {
             multidot_kernel<default_dot_dim>(
@@ -1575,6 +1552,7 @@ GKO_INSTANTIATE_FOR_EACH_CB_GMRES_TYPE(GKO_DECLARE_CB_GMRES_STEP_1_KERNEL);
 
 template <typename ValueType>
 void solve_upper_triangular(
+    std::shared_ptr<const DpcppExecutor> exec,
     const matrix::Dense<ValueType> *residual_norm_collection,
     const matrix::Dense<ValueType> *hessenberg, matrix::Dense<ValueType> *y,
     const Array<size_type> *final_iter_nums)
@@ -1596,7 +1574,8 @@ void solve_upper_triangular(
 
 
 template <typename ValueType, typename ConstAccessor3d>
-void calculate_qy(ConstAccessor3d krylov_bases, size_type num_krylov_bases,
+void calculate_qy(std::shared_ptr<const DpcppExecutor> exec,
+                  ConstAccessor3d krylov_bases, size_type num_krylov_bases,
                   const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const Array<size_type> *final_iter_nums)
@@ -1638,9 +1617,9 @@ void step_2(std::shared_ptr<const DpcppExecutor> exec,
     const auto iters =
         hessenberg->get_size()[1] / before_preconditioner->get_size()[1];
     const auto num_krylov_bases = iters + 1;
-    solve_upper_triangular(residual_norm_collection, hessenberg, y,
+    solve_upper_triangular(exec, residual_norm_collection, hessenberg, y,
                            final_iter_nums);
-    calculate_qy(krylov_bases, num_krylov_bases, y, before_preconditioner,
+    calculate_qy(exec, krylov_bases, num_krylov_bases, y, before_preconditioner,
                  final_iter_nums);
 }
 
diff --git a/dpcpp/solver/gmres_kernels.dp.cpp b/dpcpp/solver/gmres_kernels.dp.cpp
index 6c7097e9e3e..37274546bfa 100644
--- a/dpcpp/solver/gmres_kernels.dp.cpp
+++ b/dpcpp/solver/gmres_kernels.dp.cpp
@@ -68,10 +68,10 @@ namespace dpcpp {
 namespace gmres {
 
 
-constexpr int default_block_size = 512;
+constexpr int default_block_size = 256;
 // default_dot_dim can not be 64 in hip because 64 * 64 exceeds their max block
 // size limit.
-constexpr int default_dot_dim = 32;
+constexpr int default_dot_dim = 16;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
@@ -148,7 +148,7 @@ void calculate_sin_and_cos_kernel(size_type col_idx, size_type num_cols,
         const auto scale = std::abs(this_hess) + std::abs(next_hess);
         const auto hypotenuse =
             scale *
-            sycl::sqrt(
+            std::sqrt(
                 std::abs(this_hess / scale) * std::abs(this_hess / scale) +
                 std::abs(next_hess / scale) * std::abs(next_hess / scale));
         /*
@@ -432,27 +432,18 @@ void multidot_kernel(
         for (size_type i = start_row + tidy; i < end_row;
              i += default_dot_dim) {
             const auto krylov_idx = i * stride_krylov + col_idx;
-            /*
-            DPCT1007:10: Migration of this CUDA API is not supported by the
-            Intel(R) DPC++ Compatibility Tool.
-            */
             local_res +=
                 conj(krylov_bases[krylov_idx]) * next_krylov_basis[krylov_idx];
         }
     }
     reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
-    /*
-    DPCT1065:9: Consider replacing sycl::nd_item::barrier() with
-    sycl::nd_item::barrier(sycl::access::fence_space::local_space) for better
-    performance, if there is no access to global memory.
-    */
-    item_ct1.barrier();
+    item_ct1.barrier(sycl::access::fence_space::local_space);
     local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
     const auto tile_block = group::tiled_partition<default_dot_dim>(
         group::this_thread_block(item_ct1));
-    const auto sum =
-        reduce(tile_block, local_res,
-               [](const ValueType &a, const ValueType &b) { return a + b; });
+    const auto sum = ::gko::kernels::dpcpp::reduce(
+        tile_block, local_res,
+        [](const ValueType &a, const ValueType &b) { return a + b; });
     const auto new_col_idx = item_ct1.get_group(2) * default_dot_dim + tidy;
     if (tidx == 0 && new_col_idx < num_cols &&
         !stop_status[new_col_idx].has_stopped()) {
@@ -573,7 +564,7 @@ void update_hessenberg_2_kernel(
 
         if (tidx == 0) {
             hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] =
-                sycl::sqrt(reduction_helper[0]);
+                std::sqrt(reduction_helper[0]);
         }
     }
 }
@@ -764,10 +755,10 @@ void finish_arnoldi(std::shared_ptr<const DpcppExecutor> exec,
 {
     const auto stride_krylov = krylov_bases->get_stride();
     const auto stride_hessenberg = hessenberg_iter->get_stride();
-    auto cublas_handle = exec->get_cublas_handle();
+    // auto cublas_handle = exec->get_cublas_handle();
     const dim3 grid_size(
         ceildiv(hessenberg_iter->get_size()[1], default_dot_dim),
-        exec->get_num_multiprocessor() * 2);
+        exec->get_num_computing_units() * 2);
     const dim3 block_size(default_dot_dim, default_dot_dim);
     auto next_krylov_basis =
         krylov_bases->get_values() +
@@ -789,7 +780,7 @@ void finish_arnoldi(std::shared_ptr<const DpcppExecutor> exec,
                             hessenberg_iter->get_values(), stride_hessenberg,
                             stop_status);
         } else {
-            cublas::dot(exec->get_cublas_handle(), num_rows, k_krylov_bases,
+            onemkl::dot(*exec->get_queue(), num_rows, k_krylov_bases,
                         stride_krylov, next_krylov_basis, stride_krylov,
                         hessenberg_iter->get_values() + k * stride_hessenberg);
         }
@@ -876,6 +867,7 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_STEP_1_KERNEL);
 
 template <typename ValueType>
 void solve_upper_triangular(
+    std::shared_ptr<const DpcppExecutor> exec,
     const matrix::Dense<ValueType> *residual_norm_collection,
     const matrix::Dense<ValueType> *hessenberg, matrix::Dense<ValueType> *y,
     const Array<size_type> *final_iter_nums)
@@ -897,7 +889,8 @@ void solve_upper_triangular(
 
 
 template <typename ValueType>
-void calculate_qy(const matrix::Dense<ValueType> *krylov_bases,
+void calculate_qy(std::shared_ptr<const DpcppExecutor> exec,
+                  const matrix::Dense<ValueType> *krylov_bases,
                   const matrix::Dense<ValueType> *y,
                   matrix::Dense<ValueType> *before_preconditioner,
                   const Array<size_type> *final_iter_nums)
@@ -936,9 +929,9 @@ void step_2(std::shared_ptr<const DpcppExecutor> exec,
             matrix::Dense<ValueType> *before_preconditioner,
             const Array<size_type> *final_iter_nums)
 {
-    solve_upper_triangular(residual_norm_collection, hessenberg, y,
+    solve_upper_triangular(exec, residual_norm_collection, hessenberg, y,
                            final_iter_nums);
-    calculate_qy(krylov_bases, y, before_preconditioner, final_iter_nums);
+    calculate_qy(exec, krylov_bases, y, before_preconditioner, final_iter_nums);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GMRES_STEP_2_KERNEL);
diff --git a/dpcpp/stop/residual_norm_kernels.dp.cpp b/dpcpp/stop/residual_norm_kernels.dp.cpp
index 2023b6999a4..d6f76752f77 100644
--- a/dpcpp/stop/residual_norm_kernels.dp.cpp
+++ b/dpcpp/stop/residual_norm_kernels.dp.cpp
@@ -173,7 +173,7 @@ void implicit_residual_norm_kernel(
 {
     const auto tidx = thread::get_thread_id_flat(item_ct1);
     if (tidx < num_cols) {
-        if (sycl::sqrt(std::abs(tau[tidx])) <
+        if (std::sqrt(std::abs(tau[tidx])) <
             rel_residual_goal * orig_tau[tidx]) {
             stop_status[tidx].converge(stoppingId, setFinalized);
             device_storage[1] = true;
diff --git a/dpcpp/test/CMakeLists.txt b/dpcpp/test/CMakeLists.txt
index 3af602c259e..8ccd05b0518 100644
--- a/dpcpp/test/CMakeLists.txt
+++ b/dpcpp/test/CMakeLists.txt
@@ -4,4 +4,5 @@ set(GINKGO_COMPILING_DPCPP_TEST ON)
 add_subdirectory(base)
 add_subdirectory(components)
 add_subdirectory(matrix)
+add_subdirectory(solver)
 add_subdirectory(stop)
diff --git a/dpcpp/test/solver/CMakeLists.txt b/dpcpp/test/solver/CMakeLists.txt
new file mode 100644
index 00000000000..44046002504
--- /dev/null
+++ b/dpcpp/test/solver/CMakeLists.txt
@@ -0,0 +1,2 @@
+ginkgo_create_test(gmres_kernels)
+ginkgo_create_test(cb_gmres_kernels)
diff --git a/dpcpp/test/solver/cb_gmres_kernels.cpp b/dpcpp/test/solver/cb_gmres_kernels.cpp
index e9a5ec97f13..5ebbe3a810e 100644
--- a/dpcpp/test/solver/cb_gmres_kernels.cpp
+++ b/dpcpp/test/solver/cb_gmres_kernels.cpp
@@ -61,7 +61,12 @@ namespace {
 
 class CbGmres : public ::testing::Test {
 protected:
+// the CbGmres in single mode does not give benefit from mixed precision storage
+#if GINKGO_DPCPP_SINGLE_MODE
+    using value_type = float;
+#else
     using value_type = double;
+#endif
     using storage_type = float;
     using index_type = int;
     using size_type = gko::size_type;
@@ -76,7 +81,7 @@ class CbGmres : public ::testing::Test {
 
     void SetUp()
     {
-        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices("all"), 0);
         ref = gko::ReferenceExecutor::create();
         dpcpp = gko::DpcppExecutor::create(0, ref);
     }
@@ -283,9 +288,9 @@ TEST_F(CbGmres, DpcppCbGmresInitialize1IsEquivalentToRef)
         dpcpp, d_b.get(), d_residual.get(), d_givens_sin.get(),
         d_givens_cos.get(), d_stop_status.get(), default_krylov_dim_mixed);
 
-    GKO_ASSERT_MTX_NEAR(d_residual, residual, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual, residual, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, r<value_type>::value);
     GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
 }
 
@@ -305,10 +310,10 @@ TEST_F(CbGmres, DpcppCbGmresInitialize2IsEquivalentToRef)
         d_range_helper.get_range(), d_next_krylov_basis.get(),
         d_final_iter_nums.get(), default_krylov_dim_mixed);
 
-    GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, r<value_type>::value);
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
-                        1e-14);
+                        r<value_type>::value);
     assert_krylov_bases_near();
     GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
 }
@@ -333,14 +338,16 @@ TEST_F(CbGmres, DpcppCbGmresStep1IsEquivalentToRef)
         iter, d_final_iter_nums.get(), d_stop_status.get(),
         d_reorth_status.get(), d_num_reorth.get());
 
-    GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_arnoldi_norm, arnoldi_norm, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_next_krylov_basis, next_krylov_basis,
+                        r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, r<value_type>::value);
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
-                        1e-14);
-    GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter, 1e-14);
+                        r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter,
+                        r<value_type>::value);
     assert_krylov_bases_near();
     GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
 }
@@ -360,8 +367,8 @@ TEST_F(CbGmres, DpcppCbGmresStep2IsEquivalentToRef)
         d_hessenberg.get(), d_y.get(), d_before_preconditioner.get(),
         d_final_iter_nums.get());
 
-    GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_y, y, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_x, x, r<value_type>::value);
 }
 
 
diff --git a/dpcpp/test/solver/gmres_kernels.cpp b/dpcpp/test/solver/gmres_kernels.cpp
index a479d7b51bf..0cefc2e2d3a 100644
--- a/dpcpp/test/solver/gmres_kernels.cpp
+++ b/dpcpp/test/solver/gmres_kernels.cpp
@@ -57,7 +57,11 @@ namespace {
 
 class Gmres : public ::testing::Test {
 protected:
-    using value_type = gko::default_precision;
+#if GINKGO_DPCPP_SINGLE_MODE
+    using value_type = float;
+#else
+    using value_type = double;
+#endif
     using index_type = gko::int32;
     using Mtx = gko::matrix::Dense<value_type>;
     using norm_type = gko::remove_complex<value_type>;
@@ -69,7 +73,7 @@ class Gmres : public ::testing::Test {
 
     void SetUp()
     {
-        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices("all"), 0);
         ref = gko::ReferenceExecutor::create();
         dpcpp = gko::DpcppExecutor::create(0, ref);
     }
@@ -202,9 +206,9 @@ TEST_F(Gmres, DpcppGmresInitialize1IsEquivalentToRef)
         d_givens_cos.get(), d_stop_status.get(),
         gko::solver::default_krylov_dim);
 
-    GKO_ASSERT_MTX_NEAR(d_residual, residual, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual, residual, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, r<value_type>::value);
     GKO_ASSERT_ARRAY_EQ(*d_stop_status, *stop_status);
 }
 
@@ -222,10 +226,10 @@ TEST_F(Gmres, DpcppGmresInitialize2IsEquivalentToRef)
         d_residual_norm_collection.get(), d_krylov_bases.get(),
         d_final_iter_nums.get(), gko::solver::default_krylov_dim);
 
-    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, r<value_type>::value);
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
-                        1e-14);
-    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, 1e-14);
+                        r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, r<value_type>::value);
     GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
 }
 
@@ -245,13 +249,14 @@ TEST_F(Gmres, DpcppGmresStep1IsEquivalentToRef)
         d_krylov_bases.get(), d_hessenberg_iter.get(), iter,
         d_final_iter_nums.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, r<value_type>::value);
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
-                        1e-14);
-    GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, 1e-14);
+                        r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter,
+                        2 * r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, r<value_type>::value);
     GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
 }
 
@@ -271,13 +276,14 @@ TEST_F(Gmres, DpcppGmresStep1OnSingleRHSIsEquivalentToRef)
         d_krylov_bases.get(), d_hessenberg_iter.get(), iter,
         d_final_iter_nums.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_givens_sin, givens_sin, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_givens_cos, givens_cos, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_residual_norm, residual_norm, r<value_type>::value);
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
-                        1e-14);
-    GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, 1e-14);
+                        r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter,
+                        r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, r<value_type>::value);
     GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
 }
 
@@ -295,8 +301,8 @@ TEST_F(Gmres, DpcppGmresStep2IsEquivalentToRef)
                                        d_y.get(), d_before_preconditioner.get(),
                                        d_final_iter_nums.get());
 
-    GKO_ASSERT_MTX_NEAR(d_y, y, 1e-14);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(d_y, y, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(d_x, x, r<value_type>::value);
 }
 
 
diff --git a/test/solver/ir_kernels.cpp b/test/solver/ir_kernels.cpp
index 3844044138b..769ce6881be 100644
--- a/test/solver/ir_kernels.cpp
+++ b/test/solver/ir_kernels.cpp
@@ -111,9 +111,6 @@ TEST_F(Ir, InitializeIsEquivalentToRef)
 
 TEST_F(Ir, ApplyIsEquivalentToRef)
 {
-    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
-        GTEST_SKIP();
-    }
     auto mtx = gen_mtx(50, 50, 52);
     auto x = gen_mtx(50, 3, 8);
     auto b = gen_mtx(50, 3, 5);
@@ -145,9 +142,6 @@ TEST_F(Ir, ApplyIsEquivalentToRef)
 
 TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef)
 {
-    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
-        GTEST_SKIP();
-    }
     auto mtx = gen_mtx(50, 50, 54);
     auto x = gen_mtx(50, 3, 6);
     auto b = gen_mtx(50, 3, 10);
@@ -192,9 +186,6 @@ TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef)
 
 TEST_F(Ir, RichardsonApplyIsEquivalentToRef)
 {
-    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
-        GTEST_SKIP();
-    }
     auto mtx = gen_mtx(50, 50, 54);
     auto x = gen_mtx(50, 3, 4);
     auto b = gen_mtx(50, 3, 3);
@@ -228,9 +219,6 @@ TEST_F(Ir, RichardsonApplyIsEquivalentToRef)
 
 TEST_F(Ir, RichardsonApplyWithIterativeInnerSolverIsEquivalentToRef)
 {
-    if (dynamic_cast<gko::DpcppExecutor *>(exec.get())) {
-        GTEST_SKIP();
-    }
     auto mtx = gen_mtx(50, 50, 52);
     auto x = gen_mtx(50, 3, 4);
     auto b = gen_mtx(50, 3, 7);

From ac9f05decfaa66d5dd587ca152e50af95c845b1a Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 22 Jul 2021 22:11:49 +0200
Subject: [PATCH 0934/1008] move some dpcpp job to full pipeline

---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1a884c2b408..630471838d4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -726,7 +726,7 @@ build/dpcpp/cpu/release/static:
 build/dpcpp/igpu/release/shared:
   <<: *default_build_with_test
   extends:
-    - .full_test_condition
+    - .quick_test_condition
     - .use_gko-oneapi-igpu
   variables:
     <<: *default_variables
@@ -741,7 +741,7 @@ build/dpcpp/igpu/release/shared:
 build/dpcpp/opencl_igpu/release/static:
   <<: *default_build_with_test
   extends:
-    - .quick_test_condition
+    - .full_test_condition
     - .use_gko-oneapi-igpu
   variables:
     <<: *default_variables
@@ -773,7 +773,7 @@ build/dpcpp/level_zero_igpu/debug/shared:
 build/dpcpp/dgpu/release/static:
   <<: *default_build_with_test
   extends:
-    - .quick_test_condition
+    - .full_test_condition
     - .use_gko-oneapi-igpu
   variables:
     <<: *default_variables

From ce66a3c6c500549f4bffc0075f95d45a63001c18 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 23 Jul 2021 16:59:04 +0200
Subject: [PATCH 0935/1008] bind mkl isn't in anymous, collect common gmres

---
 dpcpp/CMakeLists.txt                     |   1 +
 dpcpp/base/onemkl_bindings.dp.cpp        |  94 ++++++++
 dpcpp/base/onemkl_bindings.hpp           |  57 ++---
 dpcpp/solver/cb_gmres_kernels.dp.cpp     | 237 +------------------
 dpcpp/solver/common_gmres_kernels.dp.inc | 276 +++++++++++++++++++++++
 dpcpp/solver/gmres_kernels.dp.cpp        | 253 +--------------------
 6 files changed, 402 insertions(+), 516 deletions(-)
 create mode 100644 dpcpp/base/onemkl_bindings.dp.cpp
 create mode 100644 dpcpp/solver/common_gmres_kernels.dp.inc

diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 401be197820..9eee05f0c3f 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -17,6 +17,7 @@ target_sources(ginkgo_dpcpp
     base/version.dp.cpp
     base/executor.dp.cpp
     base/helper.dp.cpp
+    base/onemkl_bindings.dp.cpp
     components/absolute_array.dp.cpp
     components/fill_array.dp.cpp
     components/prefix_sum.dp.cpp
diff --git a/dpcpp/base/onemkl_bindings.dp.cpp b/dpcpp/base/onemkl_bindings.dp.cpp
new file mode 100644
index 00000000000..db5120928f0
--- /dev/null
+++ b/dpcpp/base/onemkl_bindings.dp.cpp
@@ -0,0 +1,94 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <type_traits>
+
+
+#include <CL/sycl.hpp>
+#include <oneapi/mkl.hpp>
+
+
+#include "dpcpp/base/onemkl_bindings.hpp"
+
+
+namespace gko {
+/**
+ * @brief The device specific kernels namespace.
+ *
+ * @ingroup kernels
+ */
+namespace kernels {
+/**
+ * @brief The DPCPP namespace.
+ *
+ * @ingroup dpcpp
+ */
+namespace dpcpp {
+/**
+ * @brief The ONEMKL namespace.
+ *
+ * @ingroup onemkl
+ */
+namespace onemkl {
+
+
+#define GKO_BIND_DOT(ValueType, Name, Func)                                  \
+    void Name(::cl::sycl::queue &exec_queue, std::int64_t n,                 \
+              const ValueType *x, std::int64_t incx, const ValueType *y,     \
+              std::int64_t incy, ValueType *result)                          \
+    {                                                                        \
+        Func(exec_queue, n, x, incx, y, incy, result);                       \
+    }                                                                        \
+    static_assert(true,                                                      \
+                  "This assert is used to counter the false positive extra " \
+                  "semi-colon warnings")
+
+// Bind the dot for x^T * y
+GKO_BIND_DOT(float, dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(double, dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(std::complex<float>, dot, oneapi::mkl::blas::row_major::dotu);
+GKO_BIND_DOT(std::complex<double>, dot, oneapi::mkl::blas::row_major::dotu);
+
+// Bind the conj_dot for x' * y
+GKO_BIND_DOT(float, conj_dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(double, conj_dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(std::complex<float>, conj_dot, oneapi::mkl::blas::row_major::dotc);
+GKO_BIND_DOT(std::complex<double>, conj_dot,
+             oneapi::mkl::blas::row_major::dotc);
+
+#undef GKO_BIND_DOT
+
+
+}  // namespace onemkl
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
diff --git a/dpcpp/base/onemkl_bindings.hpp b/dpcpp/base/onemkl_bindings.hpp
index cb1b944d12c..39928639798 100644
--- a/dpcpp/base/onemkl_bindings.hpp
+++ b/dpcpp/base/onemkl_bindings.hpp
@@ -41,6 +41,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <oneapi/mkl.hpp>
 
 
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
 namespace gko {
 /**
  * @brief The device specific kernels namespace.
@@ -90,38 +93,36 @@ struct is_supported<std::complex<float>> : std::true_type {};
 template <>
 struct is_supported<std::complex<double>> : std::true_type {};
 
-namespace {
-#define GKO_BIND_DOT(ValueType, Name, Func)                                  \
-    void Name(::cl::sycl::queue &exec_queue, std::int64_t n,                 \
-              const ValueType *x, std::int64_t incx, const ValueType *y,     \
-              std::int64_t incy, ValueType *result)                          \
-    {                                                                        \
-        Func(exec_queue, n, x, incx, y, incy, result);                       \
-    }                                                                        \
-    static_assert(true,                                                      \
-                  "This assert is used to counter the false positive extra " \
-                  "semi-colon warnings")
-
-// Bind the dot for x^T * y
-GKO_BIND_DOT(float, dot, oneapi::mkl::blas::row_major::dot);
-GKO_BIND_DOT(double, dot, oneapi::mkl::blas::row_major::dot);
-GKO_BIND_DOT(std::complex<float>, dot, oneapi::mkl::blas::row_major::dotu);
-GKO_BIND_DOT(std::complex<double>, dot, oneapi::mkl::blas::row_major::dotu);
+
+#define GKO_DECLARE_DOT(ValueType, Name)                                 \
+    void Name(::cl::sycl::queue &exec_queue, std::int64_t n,             \
+              const ValueType *x, std::int64_t incx, const ValueType *y, \
+              std::int64_t incy, ValueType *result)
+
+// Declare the dot for x^T * y
+GKO_DECLARE_DOT(float, dot);
+GKO_DECLARE_DOT(double, dot);
+GKO_DECLARE_DOT(std::complex<float>, dotu);
+GKO_DECLARE_DOT(std::complex<double>, dotu);
 template <typename ValueType>
-GKO_BIND_DOT(ValueType, dot, detail::not_implemented);
-
-// Bind the conj_dot for x' * y
-GKO_BIND_DOT(float, conj_dot, oneapi::mkl::blas::row_major::dot);
-GKO_BIND_DOT(double, conj_dot, oneapi::mkl::blas::row_major::dot);
-GKO_BIND_DOT(std::complex<float>, conj_dot, oneapi::mkl::blas::row_major::dotc);
-GKO_BIND_DOT(std::complex<double>, conj_dot,
-             oneapi::mkl::blas::row_major::dotc);
+GKO_DECLARE_DOT(ValueType, dot)
+{
+    detail::not_implemented(exec_queue, n, x, incx, y, incy, result);
+}
+
+// Declare the conj_dot for x' * y
+GKO_DECLARE_DOT(float, conj_dot);
+GKO_DECLARE_DOT(double, conj_dot);
+GKO_DECLARE_DOT(std::complex<float>, conj_dot);
+GKO_DECLARE_DOT(std::complex<double>, conj_dot);
 template <typename ValueType>
-GKO_BIND_DOT(ValueType, conj_dot, detail::not_implemented);
+GKO_DECLARE_DOT(ValueType, conj_dot)
+{
+    detail::not_implemented(exec_queue, n, x, incx, y, incy, result);
+}
 
-#undef GKO_BIND_DOT
+#undef GKO_DECLARE_DOT
 
-}  // namespace
 }  // namespace onemkl
 }  // namespace dpcpp
 }  // namespace kernels
diff --git a/dpcpp/solver/cb_gmres_kernels.dp.cpp b/dpcpp/solver/cb_gmres_kernels.dp.cpp
index cf46a44b4b1..de7a21d46f5 100644
--- a/dpcpp/solver/cb_gmres_kernels.dp.cpp
+++ b/dpcpp/solver/cb_gmres_kernels.dp.cpp
@@ -76,244 +76,9 @@ constexpr int default_dot_dim = 16;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
-// Must be called with at least `max(stride_b * num_rows, krylov_dim *
-// num_cols)` threads in total.
-template <size_type block_size, typename ValueType>
-void initialize_1_kernel(
-    size_type num_rows, size_type num_cols, size_type krylov_dim,
-    const ValueType *__restrict__ b, size_type stride_b,
-    ValueType *__restrict__ residual, size_type stride_residual,
-    ValueType *__restrict__ givens_sin, size_type stride_sin,
-    ValueType *__restrict__ givens_cos, size_type stride_cos,
-    stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
-{
-    const auto global_id = thread::get_thread_id_flat(item_ct1);
-
-    const auto row_idx = global_id / stride_b;
-    const auto col_idx = global_id % stride_b;
-
-    if (global_id < num_cols) {
-        stop_status[global_id].reset();
-    }
-
-    if (row_idx < num_rows && col_idx < num_cols) {
-        residual[row_idx * stride_residual + col_idx] =
-            b[row_idx * stride_b + col_idx];
-    }
-
-    if (global_id < krylov_dim * num_cols) {
-        const auto row_givens = global_id / num_cols;
-        const auto col_givens = global_id % num_cols;
-
-        givens_sin[row_givens * stride_sin + col_givens] = zero<ValueType>();
-        givens_cos[row_givens * stride_cos + col_givens] = zero<ValueType>();
-    }
-}
-
-template <size_type block_size, typename ValueType>
-void initialize_1_kernel(dim3 grid, dim3 block,
-                         gko::size_type dynamic_shared_memory,
-                         sycl::queue *stream, size_type num_rows,
-                         size_type num_cols, size_type krylov_dim,
-                         const ValueType *b, size_type stride_b,
-                         ValueType *residual, size_type stride_residual,
-                         ValueType *givens_sin, size_type stride_sin,
-                         ValueType *givens_cos, size_type stride_cos,
-                         stopping_status *stop_status)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                initialize_1_kernel<block_size>(
-                    num_rows, num_cols, krylov_dim, b, stride_b, residual,
-                    stride_residual, givens_sin, stride_sin, givens_cos,
-                    stride_cos, stop_status, item_ct1);
-            });
-    });
-}
-
-
-template <typename ValueType>
-void calculate_sin_and_cos_kernel(size_type col_idx, size_type num_cols,
-                                  size_type iter, const ValueType &this_hess,
-                                  const ValueType &next_hess,
-                                  ValueType *givens_sin, size_type stride_sin,
-                                  ValueType *givens_cos, size_type stride_cos,
-                                  ValueType &register_sin,
-                                  ValueType &register_cos)
-{
-    if (this_hess == zero<ValueType>()) {
-        register_cos = zero<ValueType>();
-        register_sin = one<ValueType>();
-    } else {
-        const auto scale = std::abs(this_hess) + std::abs(next_hess);
-        const auto hypotenuse =
-            scale *
-            std::sqrt(
-                std::abs(this_hess / scale) * std::abs(this_hess / scale) +
-                std::abs(next_hess / scale) * std::abs(next_hess / scale));
-        register_cos = conj(this_hess) / hypotenuse;
-        register_sin = conj(next_hess) / hypotenuse;
-    }
-    givens_cos[iter * stride_cos + col_idx] = register_cos;
-    givens_sin[iter * stride_sin + col_idx] = register_sin;
-}
-
-
-template <typename ValueType>
-void calculate_residual_norm_kernel(size_type col_idx, size_type num_cols,
-                                    size_type iter,
-                                    const ValueType &register_sin,
-                                    const ValueType &register_cos,
-                                    remove_complex<ValueType> *residual_norm,
-                                    ValueType *residual_norm_collection,
-                                    size_type stride_residual_norm_collection)
-{
-    const auto this_rnc =
-        residual_norm_collection[iter * stride_residual_norm_collection +
-                                 col_idx];
-    const auto next_rnc = -conj(register_sin) * this_rnc;
-    residual_norm_collection[iter * stride_residual_norm_collection + col_idx] =
-        register_cos * this_rnc;
-    residual_norm[col_idx] = std::abs(next_rnc);
-    residual_norm_collection[(iter + 1) * stride_residual_norm_collection +
-                             col_idx] = next_rnc;
-}
-
-
-// Must be called with at least `num_cols` threads in total.
-template <size_type block_size, typename ValueType>
-void givens_rotation_kernel(
-    size_type num_rows, size_type num_cols, size_type iter,
-    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
-    ValueType *__restrict__ givens_sin, size_type stride_sin,
-    ValueType *__restrict__ givens_cos, size_type stride_cos,
-    remove_complex<ValueType> *__restrict__ residual_norm,
-    ValueType *__restrict__ residual_norm_collection,
-    size_type stride_residual_norm_collection,
-    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
-{
-    const auto col_idx = thread::get_thread_id_flat(item_ct1);
-
-    if (col_idx >= num_cols || stop_status[col_idx].has_stopped()) {
-        return;
-    }
-
-    auto this_hess = hessenberg_iter[col_idx];
-    auto next_hess = hessenberg_iter[stride_hessenberg + col_idx];
-    for (size_type i = 0; i < iter; ++i) {
-        const auto cos = givens_cos[i * stride_cos + col_idx];
-        const auto sin = givens_sin[i * stride_sin + col_idx];
-        hessenberg_iter[i * stride_hessenberg + col_idx] =
-            cos * this_hess + sin * next_hess;
-        this_hess = conj(cos) * next_hess - conj(sin) * this_hess;
-        next_hess = hessenberg_iter[(i + 2) * stride_hessenberg + col_idx];
-    }
-    // for j in 0:iter - 1
-    //     temp             =  cos(j)*hessenberg(j) +
-    //                         sin(j)*hessenberg(j+1)
-    //     hessenberg(j+1)  = -sin(j)*hessenberg(j) +
-    //                         cos(j)*hessenberg(j+1)
-    //     hessenberg(j)    =  temp;
-    // end
-
-    ValueType register_sin;
-    ValueType register_cos;
-    calculate_sin_and_cos_kernel(col_idx, num_cols, iter, this_hess, next_hess,
-                                 givens_sin, stride_sin, givens_cos, stride_cos,
-                                 register_sin, register_cos);
-    // Calculate sin and cos on hessenberg(iter) and hessenberg(iter+1)
-
-    hessenberg_iter[iter * stride_hessenberg + col_idx] =
-        register_cos * this_hess + register_sin * next_hess;
-    hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] =
-        zero<ValueType>();
-    // hessenberg(iter)   = cos(iter)*hessenberg(iter) +
-    //                      sin(iter)*hessenberg(iter+1)
-    // hessenberg(iter+1) = 0
-
-    calculate_residual_norm_kernel(
-        col_idx, num_cols, iter, register_sin, register_cos, residual_norm,
-        residual_norm_collection, stride_residual_norm_collection);
-    // Calculate residual norm
-}
-
-template <size_type block_size, typename ValueType>
-void givens_rotation_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type num_rows, size_type num_cols, size_type iter,
-    ValueType *hessenberg_iter, size_type stride_hessenberg,
-    ValueType *givens_sin, size_type stride_sin, ValueType *givens_cos,
-    size_type stride_cos, remove_complex<ValueType> *residual_norm,
-    ValueType *residual_norm_collection,
-    size_type stride_residual_norm_collection,
-    const stopping_status *stop_status)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                givens_rotation_kernel<block_size>(
-                    num_rows, num_cols, iter, hessenberg_iter,
-                    stride_hessenberg, givens_sin, stride_sin, givens_cos,
-                    stride_cos, residual_norm, residual_norm_collection,
-                    stride_residual_norm_collection, stop_status, item_ct1);
-            });
-    });
-}
+#include "dpcpp/solver/common_gmres_kernels.dp.inc"
 
 
-// Must be called with at least `num_rhs` threads in total.
-template <size_type block_size, typename ValueType>
-void solve_upper_triangular_kernel(
-    size_type num_cols, size_type num_rhs,
-    const ValueType *__restrict__ residual_norm_collection,
-    size_type stride_residual_norm_collection,
-    const ValueType *__restrict__ hessenberg, size_type stride_hessenberg,
-    ValueType *__restrict__ y, size_type stride_y,
-    const size_type *__restrict__ final_iter_nums, sycl::nd_item<3> item_ct1)
-{
-    const auto col_idx = thread::get_thread_id_flat(item_ct1);
-
-    if (col_idx >= num_rhs) {
-        return;
-    }
-
-    for (int i = final_iter_nums[col_idx] - 1; i >= 0; --i) {
-        auto temp =
-            residual_norm_collection[i * stride_residual_norm_collection +
-                                     col_idx];
-        for (size_type j = i + 1; j < final_iter_nums[col_idx]; ++j) {
-            temp -= hessenberg[i * stride_hessenberg + j * num_rhs + col_idx] *
-                    y[j * stride_y + col_idx];
-        }
-
-        y[i * stride_y + col_idx] =
-            temp / hessenberg[i * stride_hessenberg + i * num_rhs + col_idx];
-    }
-    // Solve upper triangular.
-    // y = hessenberg \ residual_norm_collection
-}
-
-template <size_type block_size, typename ValueType>
-void solve_upper_triangular_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type num_cols, size_type num_rhs,
-    const ValueType *residual_norm_collection,
-    size_type stride_residual_norm_collection, const ValueType *hessenberg,
-    size_type stride_hessenberg, ValueType *y, size_type stride_y,
-    const size_type *final_iter_nums)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                solve_upper_triangular_kernel<block_size>(
-                    num_cols, num_rhs, residual_norm_collection,
-                    stride_residual_norm_collection, hessenberg,
-                    stride_hessenberg, y, stride_y, final_iter_nums, item_ct1);
-            });
-    });
-}
-
 template <typename ValueType>
 void zero_matrix_kernel(size_type m, size_type n, size_type stride,
                         ValueType *__restrict__ array,
diff --git a/dpcpp/solver/common_gmres_kernels.dp.inc b/dpcpp/solver/common_gmres_kernels.dp.inc
new file mode 100644
index 00000000000..04714bc82f8
--- /dev/null
+++ b/dpcpp/solver/common_gmres_kernels.dp.inc
@@ -0,0 +1,276 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+
+// Must be called with at least `max(stride_b * num_rows, krylov_dim *
+// num_cols)` threads in total.
+template <size_type block_size, typename ValueType>
+void initialize_1_kernel(
+    size_type num_rows, size_type num_cols, size_type krylov_dim,
+    const ValueType *__restrict__ b, size_type stride_b,
+    ValueType *__restrict__ residual, size_type stride_residual,
+    ValueType *__restrict__ givens_sin, size_type stride_sin,
+    ValueType *__restrict__ givens_cos, size_type stride_cos,
+    stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+
+    const auto row_idx = global_id / stride_b;
+    const auto col_idx = global_id % stride_b;
+
+    if (global_id < num_cols) {
+        stop_status[global_id].reset();
+    }
+
+    if (row_idx < num_rows && col_idx < num_cols) {
+        residual[row_idx * stride_residual + col_idx] =
+            b[row_idx * stride_b + col_idx];
+    }
+
+    if (global_id < krylov_dim * num_cols) {
+        const auto row_givens = global_id / num_cols;
+        const auto col_givens = global_id % num_cols;
+
+        givens_sin[row_givens * stride_sin + col_givens] = zero<ValueType>();
+        givens_cos[row_givens * stride_cos + col_givens] = zero<ValueType>();
+    }
+}
+
+template <size_type block_size, typename ValueType>
+void initialize_1_kernel(dim3 grid, dim3 block,
+                         gko::size_type dynamic_shared_memory,
+                         sycl::queue *stream, size_type num_rows,
+                         size_type num_cols, size_type krylov_dim,
+                         const ValueType *b, size_type stride_b,
+                         ValueType *residual, size_type stride_residual,
+                         ValueType *givens_sin, size_type stride_sin,
+                         ValueType *givens_cos, size_type stride_cos,
+                         stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                initialize_1_kernel<block_size>(
+                    num_rows, num_cols, krylov_dim, b, stride_b, residual,
+                    stride_residual, givens_sin, stride_sin, givens_cos,
+                    stride_cos, stop_status, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType>
+void calculate_sin_and_cos_kernel(size_type col_idx, size_type num_cols,
+                                  size_type iter, const ValueType &this_hess,
+                                  const ValueType &next_hess,
+                                  ValueType *givens_sin, size_type stride_sin,
+                                  ValueType *givens_cos, size_type stride_cos,
+                                  ValueType &register_sin,
+                                  ValueType &register_cos)
+{
+    if (this_hess == zero<ValueType>()) {
+        register_cos = zero<ValueType>();
+        register_sin = one<ValueType>();
+    } else {
+        const auto scale = std::abs(this_hess) + std::abs(next_hess);
+        const auto hypotenuse =
+            scale *
+            std::sqrt(
+                std::abs(this_hess / scale) * std::abs(this_hess / scale) +
+                std::abs(next_hess / scale) * std::abs(next_hess / scale));
+        register_cos = conj(this_hess) / hypotenuse;
+        register_sin = conj(next_hess) / hypotenuse;
+    }
+    givens_cos[iter * stride_cos + col_idx] = register_cos;
+    givens_sin[iter * stride_sin + col_idx] = register_sin;
+}
+
+
+template <typename ValueType>
+void calculate_residual_norm_kernel(size_type col_idx, size_type num_cols,
+                                    size_type iter,
+                                    const ValueType &register_sin,
+                                    const ValueType &register_cos,
+                                    remove_complex<ValueType> *residual_norm,
+                                    ValueType *residual_norm_collection,
+                                    size_type stride_residual_norm_collection)
+{
+    const auto this_rnc =
+        residual_norm_collection[iter * stride_residual_norm_collection +
+                                 col_idx];
+
+    const auto next_rnc = -conj(register_sin) * this_rnc;
+    residual_norm_collection[iter * stride_residual_norm_collection + col_idx] =
+        register_cos * this_rnc;
+    residual_norm[col_idx] = std::abs(next_rnc);
+    residual_norm_collection[(iter + 1) * stride_residual_norm_collection +
+                             col_idx] = next_rnc;
+}
+
+
+// Must be called with at least `num_cols` threads in total.
+template <size_type block_size, typename ValueType>
+void givens_rotation_kernel(
+    size_type num_rows, size_type num_cols, size_type iter,
+    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
+    ValueType *__restrict__ givens_sin, size_type stride_sin,
+    ValueType *__restrict__ givens_cos, size_type stride_cos,
+    remove_complex<ValueType> *__restrict__ residual_norm,
+    ValueType *__restrict__ residual_norm_collection,
+    size_type stride_residual_norm_collection,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
+{
+    const auto col_idx = thread::get_thread_id_flat(item_ct1);
+
+    if (col_idx >= num_cols || stop_status[col_idx].has_stopped()) {
+        return;
+    }
+
+    auto this_hess = hessenberg_iter[col_idx];
+    auto next_hess = hessenberg_iter[stride_hessenberg + col_idx];
+    for (size_type i = 0; i < iter; ++i) {
+        const auto cos = givens_cos[i * stride_cos + col_idx];
+        const auto sin = givens_sin[i * stride_sin + col_idx];
+        hessenberg_iter[i * stride_hessenberg + col_idx] =
+            cos * this_hess + sin * next_hess;
+        /*
+        DPCT1007:4: Migration of this CUDA API is not supported by the Intel(R)
+        DPC++ Compatibility Tool.
+        */
+        this_hess = conj(cos) * next_hess - conj(sin) * this_hess;
+        next_hess = hessenberg_iter[(i + 2) * stride_hessenberg + col_idx];
+    }
+    // for j in 0:iter - 1
+    //     temp             =  cos(j)*hessenberg(j) +
+    //                         sin(j)*hessenberg(j+1)
+    //     hessenberg(j+1)  = -sin(j)*hessenberg(j) +
+    //                         cos(j)*hessenberg(j+1)
+    //     hessenberg(j)    =  temp;
+    // end
+
+    ValueType register_sin;
+    ValueType register_cos;
+    calculate_sin_and_cos_kernel(col_idx, num_cols, iter, this_hess, next_hess,
+                                 givens_sin, stride_sin, givens_cos, stride_cos,
+                                 register_sin, register_cos);
+    // Calculate sin and cos on hessenberg(iter) and hessenberg(iter+1)
+
+    hessenberg_iter[iter * stride_hessenberg + col_idx] =
+        register_cos * this_hess + register_sin * next_hess;
+    hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] =
+        zero<ValueType>();
+    // hessenberg(iter)   = cos(iter)*hessenberg(iter) +
+    //                      sin(iter)*hessenberg(iter+1)
+    // hessenberg(iter+1) = 0
+
+    calculate_residual_norm_kernel(
+        col_idx, num_cols, iter, register_sin, register_cos, residual_norm,
+        residual_norm_collection, stride_residual_norm_collection);
+    // Calculate residual norm
+}
+
+template <size_type block_size, typename ValueType>
+void givens_rotation_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type num_rows, size_type num_cols, size_type iter,
+    ValueType *hessenberg_iter, size_type stride_hessenberg,
+    ValueType *givens_sin, size_type stride_sin, ValueType *givens_cos,
+    size_type stride_cos, remove_complex<ValueType> *residual_norm,
+    ValueType *residual_norm_collection,
+    size_type stride_residual_norm_collection,
+    const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                givens_rotation_kernel<block_size>(
+                    num_rows, num_cols, iter, hessenberg_iter,
+                    stride_hessenberg, givens_sin, stride_sin, givens_cos,
+                    stride_cos, residual_norm, residual_norm_collection,
+                    stride_residual_norm_collection, stop_status, item_ct1);
+            });
+    });
+}
+
+
+// Must be called with at least `num_rhs` threads in total.
+template <size_type block_size, typename ValueType>
+void solve_upper_triangular_kernel(
+    size_type num_cols, size_type num_rhs,
+    const ValueType *__restrict__ residual_norm_collection,
+    size_type stride_residual_norm_collection,
+    const ValueType *__restrict__ hessenberg, size_type stride_hessenberg,
+    ValueType *__restrict__ y, size_type stride_y,
+    const size_type *__restrict__ final_iter_nums, sycl::nd_item<3> item_ct1)
+{
+    const auto col_idx = thread::get_thread_id_flat(item_ct1);
+
+    if (col_idx >= num_rhs) {
+        return;
+    }
+
+    for (int i = final_iter_nums[col_idx] - 1; i >= 0; --i) {
+        auto temp =
+            residual_norm_collection[i * stride_residual_norm_collection +
+                                     col_idx];
+        for (size_type j = i + 1; j < final_iter_nums[col_idx]; ++j) {
+            temp -= hessenberg[i * stride_hessenberg + j * num_rhs + col_idx] *
+                    y[j * stride_y + col_idx];
+        }
+
+        y[i * stride_y + col_idx] =
+            temp / hessenberg[i * stride_hessenberg + i * num_rhs + col_idx];
+    }
+    // Solve upper triangular.
+    // y = hessenberg \ residual_norm_collection
+}
+
+template <size_type block_size, typename ValueType>
+void solve_upper_triangular_kernel(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, size_type num_cols, size_type num_rhs,
+    const ValueType *residual_norm_collection,
+    size_type stride_residual_norm_collection, const ValueType *hessenberg,
+    size_type stride_hessenberg, ValueType *y, size_type stride_y,
+    const size_type *final_iter_nums)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                solve_upper_triangular_kernel<block_size>(
+                    num_cols, num_rhs, residual_norm_collection,
+                    stride_residual_norm_collection, hessenberg,
+                    stride_hessenberg, y, stride_y, final_iter_nums, item_ct1);
+            });
+    });
+}
+
diff --git a/dpcpp/solver/gmres_kernels.dp.cpp b/dpcpp/solver/gmres_kernels.dp.cpp
index 37274546bfa..374dac1b4a7 100644
--- a/dpcpp/solver/gmres_kernels.dp.cpp
+++ b/dpcpp/solver/gmres_kernels.dp.cpp
@@ -75,259 +75,8 @@ constexpr int default_dot_dim = 16;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
-// Must be called with at least `max(stride_b * num_rows, krylov_dim *
-// num_cols)` threads in total.
-template <size_type block_size, typename ValueType>
-void initialize_1_kernel(
-    size_type num_rows, size_type num_cols, size_type krylov_dim,
-    const ValueType *__restrict__ b, size_type stride_b,
-    ValueType *__restrict__ residual, size_type stride_residual,
-    ValueType *__restrict__ givens_sin, size_type stride_sin,
-    ValueType *__restrict__ givens_cos, size_type stride_cos,
-    stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
-{
-    const auto global_id = thread::get_thread_id_flat(item_ct1);
-
-    const auto row_idx = global_id / stride_b;
-    const auto col_idx = global_id % stride_b;
-
-    if (global_id < num_cols) {
-        stop_status[global_id].reset();
-    }
-
-    if (row_idx < num_rows && col_idx < num_cols) {
-        residual[row_idx * stride_residual + col_idx] =
-            b[row_idx * stride_b + col_idx];
-    }
-
-    if (global_id < krylov_dim * num_cols) {
-        const auto row_givens = global_id / num_cols;
-        const auto col_givens = global_id % num_cols;
-
-        givens_sin[row_givens * stride_sin + col_givens] = zero<ValueType>();
-        givens_cos[row_givens * stride_cos + col_givens] = zero<ValueType>();
-    }
-}
-
-template <size_type block_size, typename ValueType>
-void initialize_1_kernel(dim3 grid, dim3 block,
-                         gko::size_type dynamic_shared_memory,
-                         sycl::queue *stream, size_type num_rows,
-                         size_type num_cols, size_type krylov_dim,
-                         const ValueType *b, size_type stride_b,
-                         ValueType *residual, size_type stride_residual,
-                         ValueType *givens_sin, size_type stride_sin,
-                         ValueType *givens_cos, size_type stride_cos,
-                         stopping_status *stop_status)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                initialize_1_kernel<block_size>(
-                    num_rows, num_cols, krylov_dim, b, stride_b, residual,
-                    stride_residual, givens_sin, stride_sin, givens_cos,
-                    stride_cos, stop_status, item_ct1);
-            });
-    });
-}
-
-
-template <typename ValueType>
-void calculate_sin_and_cos_kernel(size_type col_idx, size_type num_cols,
-                                  size_type iter, const ValueType &this_hess,
-                                  const ValueType &next_hess,
-                                  ValueType *givens_sin, size_type stride_sin,
-                                  ValueType *givens_cos, size_type stride_cos,
-                                  ValueType &register_sin,
-                                  ValueType &register_cos)
-{
-    if (this_hess == zero<ValueType>()) {
-        register_cos = zero<ValueType>();
-        register_sin = one<ValueType>();
-    } else {
-        const auto scale = std::abs(this_hess) + std::abs(next_hess);
-        const auto hypotenuse =
-            scale *
-            std::sqrt(
-                std::abs(this_hess / scale) * std::abs(this_hess / scale) +
-                std::abs(next_hess / scale) * std::abs(next_hess / scale));
-        /*
-        DPCT1007:1: Migration of this CUDA API is not supported by the Intel(R)
-        DPC++ Compatibility Tool.
-        */
-        register_cos = conj(this_hess) / hypotenuse;
-        /*
-        DPCT1007:2: Migration of this CUDA API is not supported by the Intel(R)
-        DPC++ Compatibility Tool.
-        */
-        register_sin = conj(next_hess) / hypotenuse;
-    }
-    givens_cos[iter * stride_cos + col_idx] = register_cos;
-    givens_sin[iter * stride_sin + col_idx] = register_sin;
-}
-
-
-template <typename ValueType>
-void calculate_residual_norm_kernel(size_type col_idx, size_type num_cols,
-                                    size_type iter,
-                                    const ValueType &register_sin,
-                                    const ValueType &register_cos,
-                                    remove_complex<ValueType> *residual_norm,
-                                    ValueType *residual_norm_collection,
-                                    size_type stride_residual_norm_collection)
-{
-    const auto this_rnc =
-        residual_norm_collection[iter * stride_residual_norm_collection +
-                                 col_idx];
-    /*
-    DPCT1007:3: Migration of this CUDA API is not supported by the Intel(R)
-    DPC++ Compatibility Tool.
-    */
-    const auto next_rnc = -conj(register_sin) * this_rnc;
-    residual_norm_collection[iter * stride_residual_norm_collection + col_idx] =
-        register_cos * this_rnc;
-    residual_norm[col_idx] = std::abs(next_rnc);
-    residual_norm_collection[(iter + 1) * stride_residual_norm_collection +
-                             col_idx] = next_rnc;
-}
-
-
-// Must be called with at least `num_cols` threads in total.
-template <size_type block_size, typename ValueType>
-void givens_rotation_kernel(
-    size_type num_rows, size_type num_cols, size_type iter,
-    ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
-    ValueType *__restrict__ givens_sin, size_type stride_sin,
-    ValueType *__restrict__ givens_cos, size_type stride_cos,
-    remove_complex<ValueType> *__restrict__ residual_norm,
-    ValueType *__restrict__ residual_norm_collection,
-    size_type stride_residual_norm_collection,
-    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
-{
-    const auto col_idx = thread::get_thread_id_flat(item_ct1);
+#include "dpcpp/solver/common_gmres_kernels.dp.inc"
 
-    if (col_idx >= num_cols || stop_status[col_idx].has_stopped()) {
-        return;
-    }
-
-    auto this_hess = hessenberg_iter[col_idx];
-    auto next_hess = hessenberg_iter[stride_hessenberg + col_idx];
-    for (size_type i = 0; i < iter; ++i) {
-        const auto cos = givens_cos[i * stride_cos + col_idx];
-        const auto sin = givens_sin[i * stride_sin + col_idx];
-        hessenberg_iter[i * stride_hessenberg + col_idx] =
-            cos * this_hess + sin * next_hess;
-        /*
-        DPCT1007:4: Migration of this CUDA API is not supported by the Intel(R)
-        DPC++ Compatibility Tool.
-        */
-        this_hess = conj(cos) * next_hess - conj(sin) * this_hess;
-        next_hess = hessenberg_iter[(i + 2) * stride_hessenberg + col_idx];
-    }
-    // for j in 0:iter - 1
-    //     temp             =  cos(j)*hessenberg(j) +
-    //                         sin(j)*hessenberg(j+1)
-    //     hessenberg(j+1)  = -sin(j)*hessenberg(j) +
-    //                         cos(j)*hessenberg(j+1)
-    //     hessenberg(j)    =  temp;
-    // end
-
-    ValueType register_sin;
-    ValueType register_cos;
-    calculate_sin_and_cos_kernel(col_idx, num_cols, iter, this_hess, next_hess,
-                                 givens_sin, stride_sin, givens_cos, stride_cos,
-                                 register_sin, register_cos);
-    // Calculate sin and cos on hessenberg(iter) and hessenberg(iter+1)
-
-    hessenberg_iter[iter * stride_hessenberg + col_idx] =
-        register_cos * this_hess + register_sin * next_hess;
-    hessenberg_iter[(iter + 1) * stride_hessenberg + col_idx] =
-        zero<ValueType>();
-    // hessenberg(iter)   = cos(iter)*hessenberg(iter) +
-    //                      sin(iter)*hessenberg(iter+1)
-    // hessenberg(iter+1) = 0
-
-    calculate_residual_norm_kernel(
-        col_idx, num_cols, iter, register_sin, register_cos, residual_norm,
-        residual_norm_collection, stride_residual_norm_collection);
-    // Calculate residual norm
-}
-
-template <size_type block_size, typename ValueType>
-void givens_rotation_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type num_rows, size_type num_cols, size_type iter,
-    ValueType *hessenberg_iter, size_type stride_hessenberg,
-    ValueType *givens_sin, size_type stride_sin, ValueType *givens_cos,
-    size_type stride_cos, remove_complex<ValueType> *residual_norm,
-    ValueType *residual_norm_collection,
-    size_type stride_residual_norm_collection,
-    const stopping_status *stop_status)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                givens_rotation_kernel<block_size>(
-                    num_rows, num_cols, iter, hessenberg_iter,
-                    stride_hessenberg, givens_sin, stride_sin, givens_cos,
-                    stride_cos, residual_norm, residual_norm_collection,
-                    stride_residual_norm_collection, stop_status, item_ct1);
-            });
-    });
-}
-
-
-// Must be called with at least `num_rhs` threads in total.
-template <size_type block_size, typename ValueType>
-void solve_upper_triangular_kernel(
-    size_type num_cols, size_type num_rhs,
-    const ValueType *__restrict__ residual_norm_collection,
-    size_type stride_residual_norm_collection,
-    const ValueType *__restrict__ hessenberg, size_type stride_hessenberg,
-    ValueType *__restrict__ y, size_type stride_y,
-    const size_type *__restrict__ final_iter_nums, sycl::nd_item<3> item_ct1)
-{
-    const auto col_idx = thread::get_thread_id_flat(item_ct1);
-
-    if (col_idx >= num_rhs) {
-        return;
-    }
-
-    for (int i = final_iter_nums[col_idx] - 1; i >= 0; --i) {
-        auto temp =
-            residual_norm_collection[i * stride_residual_norm_collection +
-                                     col_idx];
-        for (size_type j = i + 1; j < final_iter_nums[col_idx]; ++j) {
-            temp -= hessenberg[i * stride_hessenberg + j * num_rhs + col_idx] *
-                    y[j * stride_y + col_idx];
-        }
-
-        y[i * stride_y + col_idx] =
-            temp / hessenberg[i * stride_hessenberg + i * num_rhs + col_idx];
-    }
-    // Solve upper triangular.
-    // y = hessenberg \ residual_norm_collection
-}
-
-template <size_type block_size, typename ValueType>
-void solve_upper_triangular_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type num_cols, size_type num_rhs,
-    const ValueType *residual_norm_collection,
-    size_type stride_residual_norm_collection, const ValueType *hessenberg,
-    size_type stride_hessenberg, ValueType *y, size_type stride_y,
-    const size_type *final_iter_nums)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                solve_upper_triangular_kernel<block_size>(
-                    num_cols, num_rhs, residual_norm_collection,
-                    stride_residual_norm_collection, hessenberg,
-                    stride_hessenberg, y, stride_y, final_iter_nums, item_ct1);
-            });
-    });
-}
 
 // Must be called with at least `num_rows * num_rhs` threads in total.
 template <size_type block_size, typename ValueType>

From 34f1248e673d3129e4d9cd6427903a3434f50646 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 27 Jul 2021 15:05:15 +0200
Subject: [PATCH 0936/1008] update the usage, doc, and ci - inline binding -
 use queue, dynamic_shared_memory consistently - delete unused library -
 delete the opencl job because it is handled by both backend
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 .gitlab-ci.yml                                |  31 ----
 dev_tools/oneapi/add_host_function.sh         |   4 +-
 dpcpp/CMakeLists.txt                          |   5 +-
 dpcpp/base/onemkl_bindings.dp.cpp             |  94 ------------
 dpcpp/base/onemkl_bindings.hpp                |  51 +++----
 dpcpp/components/format_conversion.dp.hpp     |  11 +-
 dpcpp/components/prefix_sum.dp.hpp            |  12 +-
 dpcpp/components/reduction.dp.hpp             |   6 +-
 dpcpp/matrix/dense_kernels.dp.cpp             |  79 +++++-----
 dpcpp/matrix/ell_kernels.dp.cpp               |  50 ++++---
 dpcpp/solver/cb_gmres_kernels.dp.cpp          | 136 +++++++++---------
 dpcpp/solver/common_gmres_kernels.dp.inc      |  40 +++---
 dpcpp/solver/gmres_kernels.dp.cpp             |  72 +++++-----
 dpcpp/stop/residual_norm_kernels.dp.cpp       |  30 ++--
 .../cooperative_groups_kernels.dp.cpp         |   9 +-
 test/solver/bicg_kernels.cpp                  |   4 +-
 test/solver/bicgstab_kernels.cpp              |   4 +-
 test/solver/cg_kernels.cpp                    |   4 +-
 test/solver/cgs_kernels.cpp                   |   4 +-
 test/solver/fcg_kernels.cpp                   |   4 +-
 test/solver/ir_kernels.cpp                    |   4 +-
 21 files changed, 256 insertions(+), 398 deletions(-)
 delete mode 100644 dpcpp/base/onemkl_bindings.dp.cpp

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 630471838d4..8bfdcf7e8e8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -738,22 +738,6 @@ build/dpcpp/igpu/release/shared:
     DPCPP_SINGLE_MODE: "ON"
     SYCL_DEVICE_TYPE: "GPU"
 
-build/dpcpp/opencl_igpu/release/static:
-  <<: *default_build_with_test
-  extends:
-    - .full_test_condition
-    - .use_gko-oneapi-igpu
-  variables:
-    <<: *default_variables
-    C_COMPILER: "gcc"
-    CXX_COMPILER: "dpcpp"
-    BUILD_DPCPP: "ON"
-    BUILD_TYPE: "Release"
-    BUILD_SHARED_LIBS: "OFF"
-    DPCPP_SINGLE_MODE: "ON"
-    SYCL_DEVICE_FILTER: "OpenCL"
-    SYCL_DEVICE_TYPE: "GPU"
-
 build/dpcpp/level_zero_igpu/debug/shared:
   <<: *default_build_with_test
   extends:
@@ -799,21 +783,6 @@ build/dpcpp/level_zero_dgpu/release/shared:
     DPCPP_SINGLE_MODE: "ON"
     SYCL_DEVICE_FILTER: "Level_Zero:GPU"
 
-build/dpcpp/opencl_dgpu/debug/shared:
-  <<: *default_build_with_test
-  extends:
-    - .full_test_condition
-    - .use_gko-oneapi-dgpu
-  variables:
-    <<: *default_variables
-    C_COMPILER: "gcc"
-    CXX_COMPILER: "dpcpp"
-    BUILD_DPCPP: "ON"
-    BUILD_TYPE: "Debug"
-    DPCPP_SINGLE_MODE: "ON"
-    SYCL_DEVICE_FILTER: "OpenCL"
-    SYCL_DEVICE_TYPE: "GPU"
-
 # Job with important warnings as error
 warnings:
   <<: *default_build
diff --git a/dev_tools/oneapi/add_host_function.sh b/dev_tools/oneapi/add_host_function.sh
index 86c84d84762..9d6438915e1 100755
--- a/dev_tools/oneapi/add_host_function.sh
+++ b/dev_tools/oneapi/add_host_function.sh
@@ -158,8 +158,8 @@ while IFS='' read -r line || [ -n "$line" ]; do
                     if [ -n "${TEMPLATE_INPUT}" ]; then
                         TEMPLATE_INPUT="<${TEMPLATE_INPUT}>"
                     fi
-                    echo "${TEMPLATE} void ${NAME}${HOST_SUFFIX} (dim3 grid, dim3 block, gko::size_type dynamic_shared_memory, cudaStream_t stream, ${VARIABLE}) {
-                        /*KEEP*/${NAME}${TEMPLATE_INPUT}<<<grid, block, dynamic_shared_memory, stream>>>(${VAR_INPUT});
+                    echo "${TEMPLATE} void ${NAME}${HOST_SUFFIX} (dim3 grid, dim3 block, size_type dynamic_shared_memory, cudaStream_t queue, ${VARIABLE}) {
+                        /*KEEP*/${NAME}${TEMPLATE_INPUT}<<<grid, block, dynamic_shared_memory, queue>>>(${VAR_INPUT});
                         }"
                     echo "${NAME} -> ${NAME}${HOST_SUFFIX}" >> ${MAP_FILE}
                 fi
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 9eee05f0c3f..e2d476164e8 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -8,8 +8,6 @@ set(GINKGO_DPCPP_VERSION ${GINKGO_DPCPP_VERSION} PARENT_SCOPE)
 
 find_package(MKL CONFIG REQUIRED HINTS "$ENV{MKLROOT}")
 set(GINKGO_MKL_ROOT "${MKL_ROOT}" PARENT_SCOPE)
-find_package(oneDPL REQUIRED HINTS "$ENV{DPL_ROOT}")
-set(GINKGO_DPL_ROOT "${DPL_ROOT}" PARENT_SCOPE)
 
 add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_dpcpp_device> "")
 target_sources(ginkgo_dpcpp
@@ -17,7 +15,6 @@ target_sources(ginkgo_dpcpp
     base/version.dp.cpp
     base/executor.dp.cpp
     base/helper.dp.cpp
-    base/onemkl_bindings.dp.cpp
     components/absolute_array.dp.cpp
     components/fill_array.dp.cpp
     components/prefix_sum.dp.cpp
@@ -78,7 +75,7 @@ else ()
     target_link_options(ginkgo_dpcpp PUBLIC -fsycl-device-code-split=per_kernel)
 endif()
 target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
-target_link_libraries(ginkgo_dpcpp PRIVATE MKL::MKL_DPCPP oneDPL)
+target_link_libraries(ginkgo_dpcpp PRIVATE MKL::MKL_DPCPP)
 if (GINKGO_DPCPP_SINGLE_MODE)
     target_compile_definitions(ginkgo_dpcpp PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
 endif()
diff --git a/dpcpp/base/onemkl_bindings.dp.cpp b/dpcpp/base/onemkl_bindings.dp.cpp
deleted file mode 100644
index db5120928f0..00000000000
--- a/dpcpp/base/onemkl_bindings.dp.cpp
+++ /dev/null
@@ -1,94 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <type_traits>
-
-
-#include <CL/sycl.hpp>
-#include <oneapi/mkl.hpp>
-
-
-#include "dpcpp/base/onemkl_bindings.hpp"
-
-
-namespace gko {
-/**
- * @brief The device specific kernels namespace.
- *
- * @ingroup kernels
- */
-namespace kernels {
-/**
- * @brief The DPCPP namespace.
- *
- * @ingroup dpcpp
- */
-namespace dpcpp {
-/**
- * @brief The ONEMKL namespace.
- *
- * @ingroup onemkl
- */
-namespace onemkl {
-
-
-#define GKO_BIND_DOT(ValueType, Name, Func)                                  \
-    void Name(::cl::sycl::queue &exec_queue, std::int64_t n,                 \
-              const ValueType *x, std::int64_t incx, const ValueType *y,     \
-              std::int64_t incy, ValueType *result)                          \
-    {                                                                        \
-        Func(exec_queue, n, x, incx, y, incy, result);                       \
-    }                                                                        \
-    static_assert(true,                                                      \
-                  "This assert is used to counter the false positive extra " \
-                  "semi-colon warnings")
-
-// Bind the dot for x^T * y
-GKO_BIND_DOT(float, dot, oneapi::mkl::blas::row_major::dot);
-GKO_BIND_DOT(double, dot, oneapi::mkl::blas::row_major::dot);
-GKO_BIND_DOT(std::complex<float>, dot, oneapi::mkl::blas::row_major::dotu);
-GKO_BIND_DOT(std::complex<double>, dot, oneapi::mkl::blas::row_major::dotu);
-
-// Bind the conj_dot for x' * y
-GKO_BIND_DOT(float, conj_dot, oneapi::mkl::blas::row_major::dot);
-GKO_BIND_DOT(double, conj_dot, oneapi::mkl::blas::row_major::dot);
-GKO_BIND_DOT(std::complex<float>, conj_dot, oneapi::mkl::blas::row_major::dotc);
-GKO_BIND_DOT(std::complex<double>, conj_dot,
-             oneapi::mkl::blas::row_major::dotc);
-
-#undef GKO_BIND_DOT
-
-
-}  // namespace onemkl
-}  // namespace dpcpp
-}  // namespace kernels
-}  // namespace gko
diff --git a/dpcpp/base/onemkl_bindings.hpp b/dpcpp/base/onemkl_bindings.hpp
index 39928639798..d4503fc8fe0 100644
--- a/dpcpp/base/onemkl_bindings.hpp
+++ b/dpcpp/base/onemkl_bindings.hpp
@@ -94,34 +94,35 @@ template <>
 struct is_supported<std::complex<double>> : std::true_type {};
 
 
-#define GKO_DECLARE_DOT(ValueType, Name)                                 \
-    void Name(::cl::sycl::queue &exec_queue, std::int64_t n,             \
-              const ValueType *x, std::int64_t incx, const ValueType *y, \
-              std::int64_t incy, ValueType *result)
-
-// Declare the dot for x^T * y
-GKO_DECLARE_DOT(float, dot);
-GKO_DECLARE_DOT(double, dot);
-GKO_DECLARE_DOT(std::complex<float>, dotu);
-GKO_DECLARE_DOT(std::complex<double>, dotu);
+#define GKO_BIND_DOT(ValueType, Name, Func)                                    \
+    inline void Name(::cl::sycl::queue &exec_queue, std::int64_t n,            \
+                     const ValueType *x, std::int64_t incx,                    \
+                     const ValueType *y, std::int64_t incy, ValueType *result) \
+    {                                                                          \
+        Func(exec_queue, n, x, incx, y, incy, result);                         \
+    }                                                                          \
+    static_assert(true,                                                        \
+                  "This assert is used to counter the false positive extra "   \
+                  "semi-colon warnings")
+
+// Bind the dot for x^T * y
+GKO_BIND_DOT(float, dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(double, dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(std::complex<float>, dot, oneapi::mkl::blas::row_major::dotu);
+GKO_BIND_DOT(std::complex<double>, dot, oneapi::mkl::blas::row_major::dotu);
 template <typename ValueType>
-GKO_DECLARE_DOT(ValueType, dot)
-{
-    detail::not_implemented(exec_queue, n, x, incx, y, incy, result);
-}
-
-// Declare the conj_dot for x' * y
-GKO_DECLARE_DOT(float, conj_dot);
-GKO_DECLARE_DOT(double, conj_dot);
-GKO_DECLARE_DOT(std::complex<float>, conj_dot);
-GKO_DECLARE_DOT(std::complex<double>, conj_dot);
+GKO_BIND_DOT(ValueType, dot, detail::not_implemented);
+
+// Bind the conj_dot for x' * y
+GKO_BIND_DOT(float, conj_dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(double, conj_dot, oneapi::mkl::blas::row_major::dot);
+GKO_BIND_DOT(std::complex<float>, conj_dot, oneapi::mkl::blas::row_major::dotc);
+GKO_BIND_DOT(std::complex<double>, conj_dot,
+             oneapi::mkl::blas::row_major::dotc);
 template <typename ValueType>
-GKO_DECLARE_DOT(ValueType, conj_dot)
-{
-    detail::not_implemented(exec_queue, n, x, incx, y, incy, result);
-}
+GKO_BIND_DOT(ValueType, conj_dot, detail::not_implemented);
 
-#undef GKO_DECLARE_DOT
+#undef GKO_BIND_DOT
 
 }  // namespace onemkl
 }  // namespace dpcpp
diff --git a/dpcpp/components/format_conversion.dp.hpp b/dpcpp/components/format_conversion.dp.hpp
index b75b663bb99..270824d80e4 100644
--- a/dpcpp/components/format_conversion.dp.hpp
+++ b/dpcpp/components/format_conversion.dp.hpp
@@ -59,8 +59,8 @@ namespace kernel {
  * It counts the number of explicit nonzeros per row of Ell.
  */
 template <typename ValueType, typename IndexType>
-void count_nnz_per_row(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                       sycl::queue *stream, size_type num_rows,
+void count_nnz_per_row(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                       sycl::queue *queue, size_type num_rows,
                        size_type max_nnz_per_row, size_type stride,
                        const ValueType *values, IndexType *result);
 
@@ -80,9 +80,10 @@ namespace kernel {
  */
 template <typename IndexType>
 void convert_row_idxs_to_ptrs(dim3 grid, dim3 block,
-                              size_t dynamic_shared_memory, sycl::queue *stream,
-                              const IndexType *idxs, size_type num_nonzeros,
-                              IndexType *ptrs, size_type length);
+                              size_type dynamic_shared_memory,
+                              sycl::queue *queue, const IndexType *idxs,
+                              size_type num_nonzeros, IndexType *ptrs,
+                              size_type length);
 
 
 }  // namespace kernel
diff --git a/dpcpp/components/prefix_sum.dp.hpp b/dpcpp/components/prefix_sum.dp.hpp
index 334d4239c56..7eaea344641 100644
--- a/dpcpp/components/prefix_sum.dp.hpp
+++ b/dpcpp/components/prefix_sum.dp.hpp
@@ -179,11 +179,11 @@ void start_prefix_sum(size_type num_elements, ValueType *__restrict__ elements,
 }
 
 template <std::uint32_t block_size, typename ValueType>
-void start_prefix_sum(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                      sycl::queue *stream, size_type num_elements,
+void start_prefix_sum(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                      sycl::queue *queue, size_type num_elements,
                       ValueType *elements, ValueType *block_sum)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<ValueType, block_size>, 0,
                        sycl::access::mode::read_write,
                        sycl::access::target::local>
@@ -231,11 +231,11 @@ void finalize_prefix_sum(size_type num_elements,
 }
 
 template <std::uint32_t block_size, typename ValueType>
-void finalize_prefix_sum(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                         sycl::queue *stream, size_type num_elements,
+void finalize_prefix_sum(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                         sycl::queue *queue, size_type num_elements,
                          ValueType *elements, const ValueType *block_sum)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              finalize_prefix_sum<block_size>(
diff --git a/dpcpp/components/reduction.dp.hpp b/dpcpp/components/reduction.dp.hpp
index 094f2093a95..048dfa4f862 100644
--- a/dpcpp/components/reduction.dp.hpp
+++ b/dpcpp/components/reduction.dp.hpp
@@ -217,11 +217,11 @@ void reduce_add_array(
 }
 
 template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
-void reduce_add_array(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                      sycl::queue *stream, size_type size,
+void reduce_add_array(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                      sycl::queue *queue, size_type size,
                       const ValueType *source, ValueType *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)>,
                        0, sycl::access::mode::read_write,
                        sycl::access::target::local>
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index f8608ad55b9..fd14041b645 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -164,14 +164,14 @@ void compute_partial_dot(
 }
 
 template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
-void compute_partial_dot(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                         sycl::queue *stream, size_type num_rows,
+void compute_partial_dot(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                         sycl::queue *queue, size_type num_rows,
                          const ValueType *x, size_type stride_x,
                          const ValueType *y, size_type stride_y,
                          ValueType *work)
 {
     constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<ValueType, wg_size>, 0,
                        sycl::access::mode::read_write,
                        sycl::access::target::local>
@@ -210,13 +210,14 @@ void compute_partial_conj_dot(
 
 template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void compute_partial_conj_dot(dim3 grid, dim3 block,
-                              size_t dynamic_shared_memory, sycl::queue *stream,
-                              size_type num_rows, const ValueType *x,
-                              size_type stride_x, const ValueType *y,
-                              size_type stride_y, ValueType *work)
+                              size_type dynamic_shared_memory,
+                              sycl::queue *queue, size_type num_rows,
+                              const ValueType *x, size_type stride_x,
+                              const ValueType *y, size_type stride_y,
+                              ValueType *work)
 {
     constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<ValueType, wg_size>, 0,
                        sycl::access::mode::read_write,
                        sycl::access::target::local>
@@ -251,12 +252,12 @@ void finalize_sum_reduce_computation(
 
 template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void finalize_sum_reduce_computation(dim3 grid, dim3 block,
-                                     size_t dynamic_shared_memory,
-                                     sycl::queue *stream, size_type size,
+                                     size_type dynamic_shared_memory,
+                                     sycl::queue *queue, size_type size,
                                      const ValueType *work, ValueType *result)
 {
     constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<ValueType, wg_size>, 0,
                        sycl::access::mode::read_write,
                        sycl::access::target::local>
@@ -293,13 +294,13 @@ void compute_partial_norm2(
 }
 
 template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
-void compute_partial_norm2(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                           sycl::queue *stream, size_type num_rows,
-                           const ValueType *x, size_type stride_x,
-                           remove_complex<ValueType> *work)
+void compute_partial_norm2(dim3 grid, dim3 block,
+                           size_type dynamic_shared_memory, sycl::queue *queue,
+                           size_type num_rows, const ValueType *x,
+                           size_type stride_x, remove_complex<ValueType> *work)
 {
     constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<remove_complex<ValueType>, wg_size>,
                        0, sycl::access::mode::read_write,
                        sycl::access::target::local>
@@ -334,12 +335,12 @@ void finalize_sqrt_reduce_computation(
 
 template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void finalize_sqrt_reduce_computation(dim3 grid, dim3 block,
-                                      size_t dynamic_shared_memory,
-                                      sycl::queue *stream, size_type size,
+                                      size_type dynamic_shared_memory,
+                                      sycl::queue *queue, size_type size,
                                       const ValueType *work, ValueType *result)
 {
     constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<ValueType, wg_size>, 0,
                        sycl::access::mode::read_write,
                        sycl::access::target::local>
@@ -568,11 +569,11 @@ void reduce_max_nnz(size_type size, const size_type *__restrict__ nnz_per_row,
 }
 
 template <std::uint32_t cfg = KCFG_1D::encode(256, 16)>
-void reduce_max_nnz(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                    sycl::queue *stream, size_type size,
+void reduce_max_nnz(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                    sycl::queue *queue, size_type size,
                     const size_type *nnz_per_row, size_type *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<uint8_t, 1, sycl::access::mode::read_write,
                        sycl::access::target::local>
             dpct_local_acc_ct1(sycl::range<1>(dynamic_shared_memory), cgh);
@@ -648,11 +649,11 @@ void reduce_total_cols(size_type num_slices,
 }
 
 template <std::uint32_t cfg = KCFG_1D::encode(256, 16)>
-void reduce_total_cols(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                       sycl::queue *stream, size_type num_slices,
+void reduce_total_cols(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                       sycl::queue *queue, size_type num_slices,
                        const size_type *max_nnz_per_slice, size_type *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<uint8_t, 1, sycl::access::mode::read_write,
                        sycl::access::target::local>
             dpct_local_acc_ct1(sycl::range<1>(dynamic_shared_memory), cgh);
@@ -708,13 +709,12 @@ void transpose(const size_type nrows, const size_type ncols,
 }
 
 template <std::uint32_t sg_size = 32, typename ValueType>
-void transpose(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-               sycl::queue *stream, const size_type nrows,
-               const size_type ncols, const ValueType *in,
-               const size_type in_stride, ValueType *out,
+void transpose(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+               sycl::queue *queue, const size_type nrows, const size_type ncols,
+               const ValueType *in, const size_type in_stride, ValueType *out,
                const size_type out_stride)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<ValueType, sg_size *(sg_size + 1)>, 0,
                        sycl::access_mode::read_write,
                        sycl::access::target::local>
@@ -722,10 +722,8 @@ void transpose(dim3 grid, dim3 block, size_t dynamic_shared_memory,
 
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                transpose<sg_size>(
-                    nrows, ncols, in, in_stride, out, out_stride, item_ct1,
-                    (UninitializedArray<ValueType, sg_size *(sg_size + 1)> *)
-                        space_acc_ct1.get_pointer());
+                transpose<sg_size>(nrows, ncols, in, in_stride, out, out_stride,
+                                   item_ct1, space_acc_ct1.get_pointer().get());
             });
     });
 }
@@ -749,13 +747,13 @@ void conj_transpose(
 }
 
 template <std::uint32_t sg_size = 16, typename ValueType>
-void conj_transpose(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                    sycl::queue *stream, const size_type nrows,
+void conj_transpose(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                    sycl::queue *queue, const size_type nrows,
                     const size_type ncols, const ValueType *in,
                     const size_type in_stride, ValueType *out,
                     const size_type out_stride)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<ValueType, sg_size *(sg_size + 1)>, 0,
                        sycl::access_mode::read_write,
                        sycl::access::target::local>
@@ -763,10 +761,9 @@ void conj_transpose(dim3 grid, dim3 block, size_t dynamic_shared_memory,
 
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                conj_transpose<sg_size>(
-                    nrows, ncols, in, in_stride, out, out_stride, item_ct1,
-                    (UninitializedArray<ValueType, sg_size *(sg_size + 1)> *)
-                        space_acc_ct1.get_pointer());
+                conj_transpose<sg_size>(nrows, ncols, in, in_stride, out,
+                                        out_stride, item_ct1,
+                                        space_acc_ct1.get_pointer().get());
             });
     });
 }
diff --git a/dpcpp/matrix/ell_kernels.dp.cpp b/dpcpp/matrix/ell_kernels.dp.cpp
index 76fdb26d376..e69c3402730 100644
--- a/dpcpp/matrix/ell_kernels.dp.cpp
+++ b/dpcpp/matrix/ell_kernels.dp.cpp
@@ -204,14 +204,14 @@ void spmv(
 
 template <int num_thread_per_worker, bool atomic = false, typename b_accessor,
           typename a_accessor, typename OutputValueType, typename IndexType>
-void spmv(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-          sycl::queue *stream, const size_type num_rows,
+void spmv(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+          sycl::queue *queue, const size_type num_rows,
           const int num_worker_per_row, acc::range<a_accessor> val,
           const IndexType *col, const size_type stride,
           const size_type num_stored_elements_per_row, acc::range<b_accessor> b,
           OutputValueType *c, const size_type c_stride)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<
             UninitializedArray<OutputValueType,
                                default_block_size / num_thread_per_worker>,
@@ -270,15 +270,15 @@ void spmv(
 
 template <int num_thread_per_worker, bool atomic = false, typename b_accessor,
           typename a_accessor, typename OutputValueType, typename IndexType>
-void spmv(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-          sycl::queue *stream, const size_type num_rows,
+void spmv(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+          sycl::queue *queue, const size_type num_rows,
           const int num_worker_per_row, acc::range<a_accessor> alpha,
           acc::range<a_accessor> val, const IndexType *col,
           const size_type stride, const size_type num_stored_elements_per_row,
           acc::range<b_accessor> b, const OutputValueType *beta,
           OutputValueType *c, const size_type c_stride)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<
             UninitializedArray<OutputValueType,
                                default_block_size / num_thread_per_worker>,
@@ -317,12 +317,11 @@ void initialize_zero_dense(size_type num_rows, size_type num_cols,
 
 template <typename ValueType>
 void initialize_zero_dense(dim3 grid, dim3 block,
-                           gko::size_type dynamic_shared_memory,
-                           sycl::queue *stream, size_type num_rows,
-                           size_type num_cols, size_type stride,
-                           ValueType *result)
+                           size_type dynamic_shared_memory, sycl::queue *queue,
+                           size_type num_rows, size_type num_cols,
+                           size_type stride, ValueType *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              initialize_zero_dense(num_rows, num_cols, stride,
@@ -350,13 +349,13 @@ void fill_in_dense(size_type num_rows, size_type nnz, size_type source_stride,
 }
 
 template <typename ValueType, typename IndexType>
-void fill_in_dense(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                   sycl::queue *stream, size_type num_rows, size_type nnz,
+void fill_in_dense(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                   sycl::queue *queue, size_type num_rows, size_type nnz,
                    size_type source_stride, const IndexType *col_idxs,
                    const ValueType *values, size_type result_stride,
                    ValueType *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 fill_in_dense(num_rows, nnz, source_stride, col_idxs, values,
@@ -392,13 +391,12 @@ void count_nnz_per_row(size_type num_rows, size_type max_nnz_per_row,
 }
 
 template <typename ValueType, typename IndexType>
-void count_nnz_per_row(dim3 grid, dim3 block,
-                       gko::size_type dynamic_shared_memory,
-                       sycl::queue *stream, size_type num_rows,
+void count_nnz_per_row(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                       sycl::queue *queue, size_type num_rows,
                        size_type max_nnz_per_row, size_type stride,
                        const ValueType *values, IndexType *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 count_nnz_per_row(num_rows, max_nnz_per_row, stride, values,
@@ -433,14 +431,14 @@ void fill_in_csr(size_type num_rows, size_type max_nnz_per_row,
 }
 
 template <typename ValueType, typename IndexType>
-void fill_in_csr(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                 sycl::queue *stream, size_type num_rows,
+void fill_in_csr(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                 sycl::queue *queue, size_type num_rows,
                  size_type max_nnz_per_row, size_type stride,
                  const ValueType *source_values,
                  const IndexType *source_col_idxs, IndexType *result_row_ptrs,
                  IndexType *result_col_idxs, ValueType *result_values)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 fill_in_csr(num_rows, max_nnz_per_row, stride, source_values,
@@ -472,13 +470,13 @@ void extract_diagonal(size_type diag_size, size_type max_nnz_per_row,
 }
 
 template <typename ValueType, typename IndexType>
-void extract_diagonal(dim3 grid, dim3 block,
-                      gko::size_type dynamic_shared_memory, sycl::queue *stream,
-                      size_type diag_size, size_type max_nnz_per_row,
-                      size_type orig_stride, const ValueType *orig_values,
+void extract_diagonal(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                      sycl::queue *queue, size_type diag_size,
+                      size_type max_nnz_per_row, size_type orig_stride,
+                      const ValueType *orig_values,
                       const IndexType *orig_col_idxs, ValueType *diag)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 extract_diagonal(diag_size, max_nnz_per_row, orig_stride,
diff --git a/dpcpp/solver/cb_gmres_kernels.dp.cpp b/dpcpp/solver/cb_gmres_kernels.dp.cpp
index de7a21d46f5..09f85676724 100644
--- a/dpcpp/solver/cb_gmres_kernels.dp.cpp
+++ b/dpcpp/solver/cb_gmres_kernels.dp.cpp
@@ -95,12 +95,11 @@ void zero_matrix_kernel(size_type m, size_type n, size_type stride,
 }
 
 template <typename ValueType>
-void zero_matrix_kernel(dim3 grid, dim3 block,
-                        gko::size_type dynamic_shared_memory,
-                        sycl::queue *stream, size_type m, size_type n,
+void zero_matrix_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                        sycl::queue *queue, size_type m, size_type n,
                         size_type stride, ValueType *array)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              zero_matrix_kernel(m, n, stride, array, item_ct1);
@@ -143,14 +142,13 @@ void initialize_2_1_kernel(size_type num_rows, size_type num_rhs,
 
 template <size_type block_size, typename ValueType, typename Accessor3d>
 void initialize_2_1_kernel(dim3 grid, dim3 block,
-                           gko::size_type dynamic_shared_memory,
-                           sycl::queue *stream, size_type num_rows,
-                           size_type num_rhs, size_type krylov_dim,
-                           Accessor3d krylov_bases,
+                           size_type dynamic_shared_memory, sycl::queue *queue,
+                           size_type num_rows, size_type num_rhs,
+                           size_type krylov_dim, Accessor3d krylov_bases,
                            ValueType *residual_norm_collection,
                            size_type stride_residual_nc)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 initialize_2_1_kernel<block_size>(
@@ -193,15 +191,14 @@ void initialize_2_2_kernel(
 
 template <size_type block_size, typename ValueType, typename Accessor3d>
 void initialize_2_2_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type num_rows, size_type num_rhs,
-    const ValueType *residual, size_type stride_residual,
-    const remove_complex<ValueType> *residual_norm,
+    dim3 grid, dim3 block, size_type dynamic_shared_memory, sycl::queue *queue,
+    size_type num_rows, size_type num_rhs, const ValueType *residual,
+    size_type stride_residual, const remove_complex<ValueType> *residual_norm,
     ValueType *residual_norm_collection, Accessor3d krylov_bases,
     ValueType *next_krylov_basis, size_type stride_next_krylov,
     size_type *final_iter_nums)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 initialize_2_2_kernel<block_size>(
@@ -224,12 +221,14 @@ void increase_final_iteration_numbers_kernel(
     }
 }
 
-void increase_final_iteration_numbers_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type *final_iter_nums,
-    const stopping_status *stop_status, size_type total_number)
+void increase_final_iteration_numbers_kernel(dim3 grid, dim3 block,
+                                             size_type dynamic_shared_memory,
+                                             sycl::queue *queue,
+                                             size_type *final_iter_nums,
+                                             const stopping_status *stop_status,
+                                             size_type total_number)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 increase_final_iteration_numbers_kernel(
@@ -287,15 +286,14 @@ void multinorm2_kernel(
 }
 
 template <typename ValueType>
-void multinorm2_kernel(dim3 grid, dim3 block,
-                       gko::size_type dynamic_shared_memory,
-                       sycl::queue *stream, size_type num_rows,
+void multinorm2_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                       sycl::queue *queue, size_type num_rows,
                        size_type num_cols, const ValueType *next_krylov_basis,
                        size_type stride_next_krylov,
                        remove_complex<ValueType> *norms,
                        const stopping_status *stop_status)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<
             UninitializedArray<remove_complex<ValueType>,
                                default_dot_dim *(default_dot_dim + 1)>,
@@ -365,12 +363,12 @@ void multinorminf_without_stop_kernel(
 
 template <typename ValueType>
 void multinorminf_without_stop_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type num_rows, size_type num_cols,
-    const ValueType *next_krylov_basis, size_type stride_next_krylov,
-    remove_complex<ValueType> *norms, size_type stride_norms)
+    dim3 grid, dim3 block, size_type dynamic_shared_memory, sycl::queue *queue,
+    size_type num_rows, size_type num_cols, const ValueType *next_krylov_basis,
+    size_type stride_next_krylov, remove_complex<ValueType> *norms,
+    size_type stride_norms)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<
             UninitializedArray<remove_complex<ValueType>,
                                default_dot_dim *(default_dot_dim + 1)>,
@@ -465,13 +463,12 @@ void multinorm2_inf_kernel(
 
 template <bool compute_inf, typename ValueType>
 void multinorm2_inf_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type num_rows, size_type num_cols,
-    const ValueType *next_krylov_basis, size_type stride_next_krylov,
-    remove_complex<ValueType> *norms1, remove_complex<ValueType> *norms2,
-    const stopping_status *stop_status)
+    dim3 grid, dim3 block, size_type dynamic_shared_memory, sycl::queue *queue,
+    size_type num_rows, size_type num_cols, const ValueType *next_krylov_basis,
+    size_type stride_next_krylov, remove_complex<ValueType> *norms1,
+    remove_complex<ValueType> *norms2, const stopping_status *stop_status)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<
             UninitializedArray<remove_complex<ValueType>,
                                (1 + compute_inf) *
@@ -556,16 +553,15 @@ void multidot_kernel(
 }
 
 template <int dot_dim, typename ValueType, typename Accessor3d>
-void multidot_kernel(dim3 grid, dim3 block,
-                     gko::size_type dynamic_shared_memory, sycl::queue *stream,
-                     size_type num_rows, size_type num_cols,
+void multidot_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                     sycl::queue *queue, size_type num_rows, size_type num_cols,
                      const ValueType *next_krylov_basis,
                      size_type stride_next_krylov,
                      const Accessor3d krylov_bases, ValueType *hessenberg_iter,
                      size_type stride_hessenberg,
                      const stopping_status *stop_status)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<ValueType, dot_dim * dot_dim>, 0,
                        sycl::access_mode::read_write,
                        sycl::access::target::local>
@@ -638,15 +634,15 @@ void singledot_kernel(
 }
 
 template <int block_size, typename ValueType, typename Accessor3d>
-void singledot_kernel(dim3 grid, dim3 block,
-                      gko::size_type dynamic_shared_memory, sycl::queue *stream,
-                      size_type num_rows, const ValueType *next_krylov_basis,
+void singledot_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                      sycl::queue *queue, size_type num_rows,
+                      const ValueType *next_krylov_basis,
                       size_type stride_next_krylov,
                       const Accessor3d krylov_bases, ValueType *hessenberg_iter,
                       size_type stride_hessenberg,
                       const stopping_status *stop_status)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<ValueType, block_size>, 0,
                        sycl::access_mode::read_write,
                        sycl::access::target::local>
@@ -695,14 +691,13 @@ void update_next_krylov_kernel(
 
 template <int block_size, typename ValueType, typename Accessor3d>
 void update_next_krylov_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type num_iters, size_type num_rows,
-    size_type num_cols, ValueType *next_krylov_basis,
-    size_type stride_next_krylov, const Accessor3d krylov_bases,
-    const ValueType *hessenberg_iter, size_type stride_hessenberg,
-    const stopping_status *stop_status)
+    dim3 grid, dim3 block, size_type dynamic_shared_memory, sycl::queue *queue,
+    size_type num_iters, size_type num_rows, size_type num_cols,
+    ValueType *next_krylov_basis, size_type stride_next_krylov,
+    const Accessor3d krylov_bases, const ValueType *hessenberg_iter,
+    size_type stride_hessenberg, const stopping_status *stop_status)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 update_next_krylov_kernel<block_size>(
@@ -750,15 +745,15 @@ void update_next_krylov_and_add_kernel(
 
 template <int block_size, typename ValueType, typename Accessor3d>
 void update_next_krylov_and_add_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type num_iters, size_type num_rows,
-    size_type num_cols, ValueType *next_krylov_basis,
-    size_type stride_next_krylov, const Accessor3d krylov_bases,
-    ValueType *hessenberg_iter, size_type stride_hessenberg,
-    const ValueType *buffer_iter, size_type stride_buffer,
-    const stopping_status *stop_status, const stopping_status *reorth_status)
+    dim3 grid, dim3 block, size_type dynamic_shared_memory, sycl::queue *queue,
+    size_type num_iters, size_type num_rows, size_type num_cols,
+    ValueType *next_krylov_basis, size_type stride_next_krylov,
+    const Accessor3d krylov_bases, ValueType *hessenberg_iter,
+    size_type stride_hessenberg, const ValueType *buffer_iter,
+    size_type stride_buffer, const stopping_status *stop_status,
+    const stopping_status *reorth_status)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 update_next_krylov_and_add_kernel<block_size>(
@@ -806,9 +801,8 @@ void check_arnoldi_norms(
 }
 
 template <int block_size, typename ValueType, typename Accessor3d>
-void check_arnoldi_norms(dim3 grid, dim3 block,
-                         gko::size_type dynamic_shared_memory,
-                         sycl::queue *stream, size_type num_rhs,
+void check_arnoldi_norms(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                         sycl::queue *queue, size_type num_rhs,
                          remove_complex<ValueType> *arnoldi_norm,
                          size_type stride_norm, ValueType *hessenberg_iter,
                          size_type stride_hessenberg, size_type iter,
@@ -816,7 +810,7 @@ void check_arnoldi_norms(dim3 grid, dim3 block,
                          const stopping_status *stop_status,
                          stopping_status *reorth_status, size_type *num_reorth)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 check_arnoldi_norms<block_size>(
@@ -860,15 +854,14 @@ void set_scalar_kernel(size_type num_rhs, size_type num_blocks,
 }
 
 template <int block_size, typename RealValueType, typename Accessor3d>
-void set_scalar_kernel(dim3 grid, dim3 block,
-                       gko::size_type dynamic_shared_memory,
-                       sycl::queue *stream, size_type num_rhs,
+void set_scalar_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                       sycl::queue *queue, size_type num_rhs,
                        size_type num_blocks, const RealValueType *residual_norm,
                        size_type stride_residual,
                        const RealValueType *arnoldi_inf, size_type stride_inf,
                        Accessor3d krylov_bases)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 set_scalar_kernel<block_size>(
@@ -909,13 +902,13 @@ void update_krylov_next_krylov_kernel(
 
 template <int block_size, typename ValueType, typename Accessor3d>
 void update_krylov_next_krylov_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type iter, size_type num_rows, size_type num_cols,
+    dim3 grid, dim3 block, size_type dynamic_shared_memory, sycl::queue *queue,
+    size_type iter, size_type num_rows, size_type num_cols,
     ValueType *next_krylov_basis, size_type stride_next_krylov,
     Accessor3d krylov_bases, const ValueType *hessenberg_iter,
     size_type stride_hessenberg, const stopping_status *stop_status)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 update_krylov_next_krylov_kernel<block_size>(
@@ -952,16 +945,15 @@ void calculate_Qy_kernel(size_type num_rows, size_type num_cols,
 }
 
 template <size_type block_size, typename ValueType, typename Accessor3d>
-void calculate_Qy_kernel(dim3 grid, dim3 block,
-                         gko::size_type dynamic_shared_memory,
-                         sycl::queue *stream, size_type num_rows,
+void calculate_Qy_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                         sycl::queue *queue, size_type num_rows,
                          size_type num_cols, const Accessor3d krylov_bases,
                          const ValueType *y, size_type stride_y,
                          ValueType *before_preconditioner,
                          size_type stride_preconditioner,
                          const size_type *final_iter_nums)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              calculate_Qy_kernel<block_size>(
diff --git a/dpcpp/solver/common_gmres_kernels.dp.inc b/dpcpp/solver/common_gmres_kernels.dp.inc
index 04714bc82f8..6c2cf483ef2 100644
--- a/dpcpp/solver/common_gmres_kernels.dp.inc
+++ b/dpcpp/solver/common_gmres_kernels.dp.inc
@@ -66,9 +66,8 @@ void initialize_1_kernel(
 }
 
 template <size_type block_size, typename ValueType>
-void initialize_1_kernel(dim3 grid, dim3 block,
-                         gko::size_type dynamic_shared_memory,
-                         sycl::queue *stream, size_type num_rows,
+void initialize_1_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                         sycl::queue *queue, size_type num_rows,
                          size_type num_cols, size_type krylov_dim,
                          const ValueType *b, size_type stride_b,
                          ValueType *residual, size_type stride_residual,
@@ -76,7 +75,7 @@ void initialize_1_kernel(dim3 grid, dim3 block,
                          ValueType *givens_cos, size_type stride_cos,
                          stopping_status *stop_status)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 initialize_1_kernel<block_size>(
@@ -162,10 +161,6 @@ void givens_rotation_kernel(
         const auto sin = givens_sin[i * stride_sin + col_idx];
         hessenberg_iter[i * stride_hessenberg + col_idx] =
             cos * this_hess + sin * next_hess;
-        /*
-        DPCT1007:4: Migration of this CUDA API is not supported by the Intel(R)
-        DPC++ Compatibility Tool.
-        */
         this_hess = conj(cos) * next_hess - conj(sin) * this_hess;
         next_hess = hessenberg_iter[(i + 2) * stride_hessenberg + col_idx];
     }
@@ -199,17 +194,19 @@ void givens_rotation_kernel(
 }
 
 template <size_type block_size, typename ValueType>
-void givens_rotation_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type num_rows, size_type num_cols, size_type iter,
-    ValueType *hessenberg_iter, size_type stride_hessenberg,
-    ValueType *givens_sin, size_type stride_sin, ValueType *givens_cos,
-    size_type stride_cos, remove_complex<ValueType> *residual_norm,
-    ValueType *residual_norm_collection,
-    size_type stride_residual_norm_collection,
-    const stopping_status *stop_status)
+void givens_rotation_kernel(dim3 grid, dim3 block,
+                            size_type dynamic_shared_memory, sycl::queue *queue,
+                            size_type num_rows, size_type num_cols,
+                            size_type iter, ValueType *hessenberg_iter,
+                            size_type stride_hessenberg, ValueType *givens_sin,
+                            size_type stride_sin, ValueType *givens_cos,
+                            size_type stride_cos,
+                            remove_complex<ValueType> *residual_norm,
+                            ValueType *residual_norm_collection,
+                            size_type stride_residual_norm_collection,
+                            const stopping_status *stop_status)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 givens_rotation_kernel<block_size>(
@@ -256,14 +253,14 @@ void solve_upper_triangular_kernel(
 
 template <size_type block_size, typename ValueType>
 void solve_upper_triangular_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type num_cols, size_type num_rhs,
+    dim3 grid, dim3 block, size_type dynamic_shared_memory, sycl::queue *queue,
+    size_type num_cols, size_type num_rhs,
     const ValueType *residual_norm_collection,
     size_type stride_residual_norm_collection, const ValueType *hessenberg,
     size_type stride_hessenberg, ValueType *y, size_type stride_y,
     const size_type *final_iter_nums)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 solve_upper_triangular_kernel<block_size>(
@@ -273,4 +270,3 @@ void solve_upper_triangular_kernel(
             });
     });
 }
-
diff --git a/dpcpp/solver/gmres_kernels.dp.cpp b/dpcpp/solver/gmres_kernels.dp.cpp
index 374dac1b4a7..67bd5f5d561 100644
--- a/dpcpp/solver/gmres_kernels.dp.cpp
+++ b/dpcpp/solver/gmres_kernels.dp.cpp
@@ -69,8 +69,6 @@ namespace gmres {
 
 
 constexpr int default_block_size = 256;
-// default_dot_dim can not be 64 in hip because 64 * 64 exceeds their max block
-// size limit.
 constexpr int default_dot_dim = 16;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
@@ -106,16 +104,15 @@ void initialize_2_2_kernel(
 
 template <size_type block_size, typename ValueType>
 void initialize_2_2_kernel(dim3 grid, dim3 block,
-                           gko::size_type dynamic_shared_memory,
-                           sycl::queue *stream, size_type num_rows,
-                           size_type num_rhs, const ValueType *residual,
-                           size_type stride_residual,
+                           size_type dynamic_shared_memory, sycl::queue *queue,
+                           size_type num_rows, size_type num_rhs,
+                           const ValueType *residual, size_type stride_residual,
                            const remove_complex<ValueType> *residual_norm,
                            ValueType *residual_norm_collection,
                            ValueType *krylov_bases, size_type stride_krylov,
                            size_type *final_iter_nums)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 initialize_2_2_kernel<block_size>(
@@ -138,12 +135,14 @@ void increase_final_iteration_numbers_kernel(
     }
 }
 
-void increase_final_iteration_numbers_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type *final_iter_nums,
-    const stopping_status *stop_status, size_type total_number)
+void increase_final_iteration_numbers_kernel(dim3 grid, dim3 block,
+                                             size_type dynamic_shared_memory,
+                                             sycl::queue *queue,
+                                             size_type *final_iter_nums,
+                                             const stopping_status *stop_status,
+                                             size_type total_number)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 increase_final_iteration_numbers_kernel(
@@ -171,9 +170,9 @@ void multidot_kernel(
     const auto end_row = ((item_ct1.get_group(1) + 1) * num > num_rows)
                              ? num_rows
                              : (item_ct1.get_group(1) + 1) * num;
+
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
-
     ValueType *__restrict__ reduction_helper = (*reduction_helper_array);
 
     ValueType local_res = zero<ValueType>();
@@ -202,16 +201,15 @@ void multidot_kernel(
 }
 
 template <typename ValueType>
-void multidot_kernel(dim3 grid, dim3 block,
-                     gko::size_type dynamic_shared_memory, sycl::queue *stream,
-                     size_type k, size_type num_rows, size_type num_cols,
-                     const ValueType *krylov_bases,
+void multidot_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                     sycl::queue *queue, size_type k, size_type num_rows,
+                     size_type num_cols, const ValueType *krylov_bases,
                      const ValueType *next_krylov_basis,
                      size_type stride_krylov, ValueType *hessenberg_iter,
                      size_type stride_hessenberg,
                      const stopping_status *stop_status)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<ValueType, default_dot_dim *(
                                                          default_dot_dim + 1)>,
                        0, sycl::access_mode::read_write,
@@ -259,13 +257,13 @@ void update_next_krylov_kernel(
 
 template <int block_size, typename ValueType>
 void update_next_krylov_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type k, size_type num_rows, size_type num_cols,
+    dim3 grid, dim3 block, size_type dynamic_shared_memory, sycl::queue *queue,
+    size_type k, size_type num_rows, size_type num_cols,
     const ValueType *krylov_bases, ValueType *next_krylov_basis,
     size_type stride_krylov, const ValueType *hessenberg_iter,
     size_type stride_hessenberg, const stopping_status *stop_status)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 update_next_krylov_kernel<block_size>(
@@ -320,13 +318,13 @@ void update_hessenberg_2_kernel(
 
 template <int block_size, typename ValueType>
 void update_hessenberg_2_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type iter, size_type num_rows, size_type num_cols,
+    dim3 grid, dim3 block, size_type dynamic_shared_memory, sycl::queue *queue,
+    size_type iter, size_type num_rows, size_type num_cols,
     const ValueType *next_krylov_basis, size_type stride_next_krylov,
     ValueType *hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *stop_status)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<ValueType, block_size>, 0,
                        sycl::access_mode::read_write,
                        sycl::access::target::local>
@@ -370,15 +368,15 @@ void update_krylov_kernel(
 
 template <int block_size, typename ValueType>
 void update_krylov_kernel(dim3 grid, dim3 block,
-                          gko::size_type dynamic_shared_memory,
-                          sycl::queue *stream, size_type iter,
-                          size_type num_rows, size_type num_cols,
-                          ValueType *krylov_bases, size_type stride_krylov,
+                          size_type dynamic_shared_memory, sycl::queue *queue,
+                          size_type iter, size_type num_rows,
+                          size_type num_cols, ValueType *krylov_bases,
+                          size_type stride_krylov,
                           const ValueType *hessenberg_iter,
                           size_type stride_hessenberg,
                           const stopping_status *stop_status)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 update_krylov_kernel<block_size>(
@@ -419,14 +417,16 @@ void calculate_Qy_kernel(size_type num_rows, size_type num_cols,
 }
 
 template <size_type block_size, typename ValueType>
-void calculate_Qy_kernel(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, size_type num_rows, size_type num_cols,
-    size_type num_rhs, const ValueType *krylov_bases, size_type stride_krylov,
-    const ValueType *y, size_type stride_y, ValueType *before_preconditioner,
-    size_type stride_preconditioner, const size_type *final_iter_nums)
+void calculate_Qy_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                         sycl::queue *queue, size_type num_rows,
+                         size_type num_cols, size_type num_rhs,
+                         const ValueType *krylov_bases, size_type stride_krylov,
+                         const ValueType *y, size_type stride_y,
+                         ValueType *before_preconditioner,
+                         size_type stride_preconditioner,
+                         const size_type *final_iter_nums)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 calculate_Qy_kernel<block_size>(
diff --git a/dpcpp/stop/residual_norm_kernels.dp.cpp b/dpcpp/stop/residual_norm_kernels.dp.cpp
index d6f76752f77..448e4f3b38f 100644
--- a/dpcpp/stop/residual_norm_kernels.dp.cpp
+++ b/dpcpp/stop/residual_norm_kernels.dp.cpp
@@ -83,14 +83,14 @@ void residual_norm_kernel(size_type num_cols, ValueType rel_residual_goal,
 }
 
 template <typename ValueType>
-void residual_norm_kernel(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                          sycl::queue *stream, size_type num_cols,
-                          ValueType rel_residual_goal, const ValueType *tau,
-                          const ValueType *orig_tau, uint8 stoppingId,
-                          bool setFinalized, stopping_status *stop_status,
-                          bool *device_storage)
+void residual_norm_kernel(dim3 grid, dim3 block,
+                          size_type dynamic_shared_memory, sycl::queue *queue,
+                          size_type num_cols, ValueType rel_residual_goal,
+                          const ValueType *tau, const ValueType *orig_tau,
+                          uint8 stoppingId, bool setFinalized,
+                          stopping_status *stop_status, bool *device_storage)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 residual_norm_kernel(num_cols, rel_residual_goal, tau, orig_tau,
@@ -107,10 +107,10 @@ void init_kernel(bool *__restrict__ device_storage)
     device_storage[1] = false;
 }
 
-void init_kernel(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                 sycl::queue *stream, bool *device_storage)
+void init_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                 sycl::queue *queue, bool *device_storage)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block),
             [=](sycl::nd_item<3> item_ct1) { init_kernel(device_storage); });
@@ -188,13 +188,13 @@ void implicit_residual_norm_kernel(
 
 template <typename ValueType>
 void implicit_residual_norm_kernel(
-    dim3 grid, dim3 block, size_t dynamic_shared_memory, sycl::queue *stream,
+    dim3 grid, dim3 block, size_type dynamic_shared_memory, sycl::queue *queue,
     size_type num_cols, remove_complex<ValueType> rel_residual_goal,
     const ValueType *tau, const remove_complex<ValueType> *orig_tau,
     uint8 stoppingId, bool setFinalized, stopping_status *stop_status,
     bool *device_storage)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 implicit_residual_norm_kernel(
@@ -211,10 +211,10 @@ void init_kernel(bool *__restrict__ device_storage)
     device_storage[1] = false;
 }
 
-void init_kernel(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                 sycl::queue *stream, bool *device_storage)
+void init_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                 sycl::queue *queue, bool *device_storage)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block),
             [=](sycl::nd_item<3> item_ct1) { init_kernel(device_storage); });
diff --git a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
index 9b46e3ba0f3..300b6cac8cf 100644
--- a/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
+++ b/dpcpp/test/components/cooperative_groups_kernels.dp.cpp
@@ -134,8 +134,9 @@ void cg_shuffle(bool *s, sycl::nd_item<3> item_ct1)
 
 // group all kernel things together
 template <int config>
-void cg_shuffle_host(dim3 grid, dim3 block, size_t dynamic_shared_memory,
-                     sycl::queue *queue, bool *s)
+void cg_shuffle_host(dim3 grid, dim3 block,
+                     gko::size_type dynamic_shared_memory, sycl::queue *queue,
+                     bool *s)
 {
     queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
@@ -150,8 +151,8 @@ GKO_ENABLE_IMPLEMENTATION_CONFIG_SELECTION(cg_shuffle_config, cg_shuffle_host)
 
 // the call
 void cg_shuffle_config_call(std::uint32_t desired_cfg, dim3 grid, dim3 block,
-                            size_t dynamic_shared_memory, sycl::queue *queue,
-                            bool *s)
+                            gko::size_type dynamic_shared_memory,
+                            sycl::queue *queue, bool *s)
 {
     cg_shuffle_config(
         default_config_list,
diff --git a/test/solver/bicg_kernels.cpp b/test/solver/bicg_kernels.cpp
index 06b1726cdc4..cd099445bf1 100644
--- a/test/solver/bicg_kernels.cpp
+++ b/test/solver/bicg_kernels.cpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <ginkgo/core/solver/bicg.hpp>
+#include "core/solver/bicg_kernels.hpp"
 
 
 #include <random>
@@ -42,12 +42,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/solver/bicg.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm.hpp>
 
 
-#include "core/solver/bicg_kernels.hpp"
 #include "core/test/utils.hpp"
 #include "matrices/config.hpp"
 #include "test/utils/executor.hpp"
diff --git a/test/solver/bicgstab_kernels.cpp b/test/solver/bicgstab_kernels.cpp
index 9b5a8457cc6..9318d486a28 100644
--- a/test/solver/bicgstab_kernels.cpp
+++ b/test/solver/bicgstab_kernels.cpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <ginkgo/core/solver/bicgstab.hpp>
+#include "core/solver/bicgstab_kernels.hpp"
 
 
 #include <random>
@@ -43,12 +43,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/solver/bicgstab.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm.hpp>
 
 
-#include "core/solver/bicgstab_kernels.hpp"
 #include "core/test/utils.hpp"
 #include "test/utils/executor.hpp"
 
diff --git a/test/solver/cg_kernels.cpp b/test/solver/cg_kernels.cpp
index edd9d33a78d..2a38b3c8b4b 100644
--- a/test/solver/cg_kernels.cpp
+++ b/test/solver/cg_kernels.cpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <ginkgo/core/solver/cg.hpp>
+#include "core/solver/cg_kernels.hpp"
 
 
 #include <random>
@@ -42,12 +42,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/solver/cg.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm.hpp>
 
 
-#include "core/solver/cg_kernels.hpp"
 #include "core/test/utils.hpp"
 #include "test/utils/executor.hpp"
 
diff --git a/test/solver/cgs_kernels.cpp b/test/solver/cgs_kernels.cpp
index cd4f0c10437..ad8aaceb7cb 100644
--- a/test/solver/cgs_kernels.cpp
+++ b/test/solver/cgs_kernels.cpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <ginkgo/core/solver/cgs.hpp>
+#include "core/solver/cgs_kernels.hpp"
 
 
 #include <random>
@@ -42,12 +42,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/solver/cgs.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm.hpp>
 
 
-#include "core/solver/cgs_kernels.hpp"
 #include "core/test/utils.hpp"
 #include "test/utils/executor.hpp"
 
diff --git a/test/solver/fcg_kernels.cpp b/test/solver/fcg_kernels.cpp
index 46fbb30cdf6..7dd0c0a2558 100644
--- a/test/solver/fcg_kernels.cpp
+++ b/test/solver/fcg_kernels.cpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <ginkgo/core/solver/fcg.hpp>
+#include "core/solver/fcg_kernels.hpp"
 
 
 #include <random>
@@ -42,12 +42,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/solver/fcg.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 #include <ginkgo/core/stop/residual_norm.hpp>
 
 
-#include "core/solver/fcg_kernels.hpp"
 #include "core/test/utils.hpp"
 #include "test/utils/executor.hpp"
 
diff --git a/test/solver/ir_kernels.cpp b/test/solver/ir_kernels.cpp
index 769ce6881be..cd2c45f266f 100644
--- a/test/solver/ir_kernels.cpp
+++ b/test/solver/ir_kernels.cpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <ginkgo/core/solver/ir.hpp>
+#include "core/solver/ir_kernels.hpp"
 
 
 #include <random>
@@ -43,11 +43,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/solver/gmres.hpp>
+#include <ginkgo/core/solver/ir.hpp>
 #include <ginkgo/core/stop/combined.hpp>
 #include <ginkgo/core/stop/iteration.hpp>
 
 
-#include "core/solver/ir_kernels.hpp"
 #include "core/test/utils.hpp"
 #include "test/utils/executor.hpp"
 

From dcf5f578c301e4c988cfe83087d8f4acb277df01 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 27 Jul 2021 16:47:54 +0200
Subject: [PATCH 0937/1008] impl residual norm directly and simplify host.

---
 dpcpp/solver/cb_gmres_kernels.dp.cpp    |  30 +---
 dpcpp/solver/gmres_kernels.dp.cpp       |  17 +--
 dpcpp/stop/residual_norm_kernels.dp.cpp | 193 +++++++-----------------
 3 files changed, 56 insertions(+), 184 deletions(-)

diff --git a/dpcpp/solver/cb_gmres_kernels.dp.cpp b/dpcpp/solver/cb_gmres_kernels.dp.cpp
index 09f85676724..0b8359402e6 100644
--- a/dpcpp/solver/cb_gmres_kernels.dp.cpp
+++ b/dpcpp/solver/cb_gmres_kernels.dp.cpp
@@ -94,18 +94,7 @@ void zero_matrix_kernel(size_type m, size_type n, size_type stride,
     }
 }
 
-template <typename ValueType>
-void zero_matrix_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                        sycl::queue *queue, size_type m, size_type n,
-                        size_type stride, ValueType *array)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             zero_matrix_kernel(m, n, stride, array, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(zero_matrix_kernel, zero_matrix_kernel);
 
 
 // Must be called with at least `num_rows * stride_krylov` threads in total.
@@ -221,21 +210,8 @@ void increase_final_iteration_numbers_kernel(
     }
 }
 
-void increase_final_iteration_numbers_kernel(dim3 grid, dim3 block,
-                                             size_type dynamic_shared_memory,
-                                             sycl::queue *queue,
-                                             size_type *final_iter_nums,
-                                             const stopping_status *stop_status,
-                                             size_type total_number)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                increase_final_iteration_numbers_kernel(
-                    final_iter_nums, stop_status, total_number, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(increase_final_iteration_numbers_kernel,
+                        increase_final_iteration_numbers_kernel);
 
 
 template <typename ValueType>
diff --git a/dpcpp/solver/gmres_kernels.dp.cpp b/dpcpp/solver/gmres_kernels.dp.cpp
index 67bd5f5d561..e27a7ef9f65 100644
--- a/dpcpp/solver/gmres_kernels.dp.cpp
+++ b/dpcpp/solver/gmres_kernels.dp.cpp
@@ -135,21 +135,8 @@ void increase_final_iteration_numbers_kernel(
     }
 }
 
-void increase_final_iteration_numbers_kernel(dim3 grid, dim3 block,
-                                             size_type dynamic_shared_memory,
-                                             sycl::queue *queue,
-                                             size_type *final_iter_nums,
-                                             const stopping_status *stop_status,
-                                             size_type total_number)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                increase_final_iteration_numbers_kernel(
-                    final_iter_nums, stop_status, total_number, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(increase_final_iteration_numbers_kernel,
+                        increase_final_iteration_numbers_kernel);
 
 
 template <typename ValueType>
diff --git a/dpcpp/stop/residual_norm_kernels.dp.cpp b/dpcpp/stop/residual_norm_kernels.dp.cpp
index 448e4f3b38f..a527ec4c564 100644
--- a/dpcpp/stop/residual_norm_kernels.dp.cpp
+++ b/dpcpp/stop/residual_norm_kernels.dp.cpp
@@ -56,68 +56,6 @@ namespace dpcpp {
 namespace residual_norm {
 
 
-constexpr int default_block_size = 256;
-
-
-template <typename ValueType>
-void residual_norm_kernel(size_type num_cols, ValueType rel_residual_goal,
-                          const ValueType *__restrict__ tau,
-                          const ValueType *__restrict__ orig_tau,
-                          uint8 stoppingId, bool setFinalized,
-                          stopping_status *__restrict__ stop_status,
-                          bool *__restrict__ device_storage,
-                          sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-    if (tidx < num_cols) {
-        if (tau[tidx] < rel_residual_goal * orig_tau[tidx]) {
-            stop_status[tidx].converge(stoppingId, setFinalized);
-            device_storage[1] = true;
-        }
-        // because only false is written to all_converged, write conflicts
-        // should not cause any problem
-        else if (!stop_status[tidx].has_stopped()) {
-            device_storage[0] = false;
-        }
-    }
-}
-
-template <typename ValueType>
-void residual_norm_kernel(dim3 grid, dim3 block,
-                          size_type dynamic_shared_memory, sycl::queue *queue,
-                          size_type num_cols, ValueType rel_residual_goal,
-                          const ValueType *tau, const ValueType *orig_tau,
-                          uint8 stoppingId, bool setFinalized,
-                          stopping_status *stop_status, bool *device_storage)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                residual_norm_kernel(num_cols, rel_residual_goal, tau, orig_tau,
-                                     stoppingId, setFinalized, stop_status,
-                                     device_storage, item_ct1);
-            });
-    });
-}
-
-
-void init_kernel(bool *__restrict__ device_storage)
-{
-    device_storage[0] = true;
-    device_storage[1] = false;
-}
-
-void init_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                 sycl::queue *queue, bool *device_storage)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block),
-            [=](sycl::nd_item<3> item_ct1) { init_kernel(device_storage); });
-    });
-}
-
-
 template <typename ValueType>
 void residual_norm(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::Dense<ValueType> *tau,
@@ -129,16 +67,32 @@ void residual_norm(std::shared_ptr<const DpcppExecutor> exec,
 {
     static_assert(is_complex_s<ValueType>::value == false,
                   "ValueType must not be complex in this function!");
-    init_kernel(1, 1, 0, exec->get_queue(), device_storage->get_data());
-
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(ceildiv(tau->get_size()[1], block_size.x), 1, 1);
+    auto device_storage_val = device_storage->get_data();
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{1}, [=](sycl::id<1>) {
+            device_storage_val[0] = true;
+            device_storage_val[1] = false;
+        });
+    });
 
-    residual_norm_kernel(grid_size, block_size, 0, exec->get_queue(),
-                         tau->get_size()[1], rel_residual_goal,
-                         tau->get_const_values(), orig_tau->get_const_values(),
-                         stoppingId, setFinalized, stop_status->get_data(),
-                         device_storage->get_data());
+    auto orig_tau_val = orig_tau->get_const_values();
+    auto tau_val = tau->get_const_values();
+    auto stop_status_val = stop_status->get_data();
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl::range<1>{tau->get_size()[1]}, [=](sycl::id<1> idx_id) {
+                const auto tidx = idx_id[0];
+                if (tau_val[tidx] < rel_residual_goal * orig_tau_val[tidx]) {
+                    stop_status_val[tidx].converge(stoppingId, setFinalized);
+                    device_storage_val[1] = true;
+                }
+                // because only false is written to all_converged, write
+                // conflicts should not cause any problem
+                else if (!stop_status_val[tidx].has_stopped()) {
+                    device_storage_val[0] = false;
+                }
+            });
+    });
 
     /* Represents all_converged, one_changed */
     *all_converged = exec->copy_val_to_host(device_storage->get_const_data());
@@ -160,68 +114,6 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE(
 namespace implicit_residual_norm {
 
 
-constexpr int default_block_size = 256;
-
-
-template <typename ValueType>
-void implicit_residual_norm_kernel(
-    size_type num_cols, remove_complex<ValueType> rel_residual_goal,
-    const ValueType *__restrict__ tau,
-    const remove_complex<ValueType> *__restrict__ orig_tau, uint8 stoppingId,
-    bool setFinalized, stopping_status *__restrict__ stop_status,
-    bool *__restrict__ device_storage, sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-    if (tidx < num_cols) {
-        if (std::sqrt(std::abs(tau[tidx])) <
-            rel_residual_goal * orig_tau[tidx]) {
-            stop_status[tidx].converge(stoppingId, setFinalized);
-            device_storage[1] = true;
-        }
-        // because only false is written to all_converged, write conflicts
-        // should not cause any problem
-        else if (!stop_status[tidx].has_stopped()) {
-            device_storage[0] = false;
-        }
-    }
-}
-
-template <typename ValueType>
-void implicit_residual_norm_kernel(
-    dim3 grid, dim3 block, size_type dynamic_shared_memory, sycl::queue *queue,
-    size_type num_cols, remove_complex<ValueType> rel_residual_goal,
-    const ValueType *tau, const remove_complex<ValueType> *orig_tau,
-    uint8 stoppingId, bool setFinalized, stopping_status *stop_status,
-    bool *device_storage)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                implicit_residual_norm_kernel(
-                    num_cols, rel_residual_goal, tau, orig_tau, stoppingId,
-                    setFinalized, stop_status, device_storage, item_ct1);
-            });
-    });
-}
-
-
-void init_kernel(bool *__restrict__ device_storage)
-{
-    device_storage[0] = true;
-    device_storage[1] = false;
-}
-
-void init_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                 sycl::queue *queue, bool *device_storage)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block),
-            [=](sycl::nd_item<3> item_ct1) { init_kernel(device_storage); });
-    });
-}
-
-
 template <typename ValueType>
 void implicit_residual_norm(
     std::shared_ptr<const DpcppExecutor> exec,
@@ -231,16 +123,33 @@ void implicit_residual_norm(
     bool setFinalized, Array<stopping_status> *stop_status,
     Array<bool> *device_storage, bool *all_converged, bool *one_changed)
 {
-    init_kernel(1, 1, 0, exec->get_queue(), device_storage->get_data());
-
-    const dim3 block_size(default_block_size, 1, 1);
-    const dim3 grid_size(ceildiv(tau->get_size()[1], block_size.x), 1, 1);
+    auto device_storage_val = device_storage->get_data();
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{1}, [=](sycl::id<1>) {
+            device_storage_val[0] = true;
+            device_storage_val[1] = false;
+        });
+    });
 
-    implicit_residual_norm_kernel(
-        grid_size, block_size, 0, exec->get_queue(), tau->get_size()[1],
-        rel_residual_goal, tau->get_const_values(),
-        orig_tau->get_const_values(), stoppingId, setFinalized,
-        stop_status->get_data(), device_storage->get_data());
+    auto orig_tau_val = orig_tau->get_const_values();
+    auto tau_val = tau->get_const_values();
+    auto stop_status_val = stop_status->get_data();
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl::range<1>{tau->get_size()[1]}, [=](sycl::id<1> idx_id) {
+                const auto tidx = idx_id[0];
+                if (std::sqrt(std::abs(tau_val[tidx])) <
+                    rel_residual_goal * orig_tau_val[tidx]) {
+                    stop_status_val[tidx].converge(stoppingId, setFinalized);
+                    device_storage_val[1] = true;
+                }
+                // because only false is written to all_converged, write
+                // conflicts should not cause any problem
+                else if (!stop_status_val[tidx].has_stopped()) {
+                    device_storage_val[0] = false;
+                }
+            });
+    });
 
     /* Represents all_converged, one_changed */
     *all_converged = exec->copy_val_to_host(device_storage->get_const_data());

From 4dc0887a2e824faba5fd565f386e51b8833bbdb9 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 9 Jun 2021 19:40:57 +0200
Subject: [PATCH 0938/1008] add the issue test for cuda/hip reset collision

---
 .gitlab-ci.yml                           |  6 +--
 .gitlab/image.yml                        |  3 +-
 cuda/test/base/CMakeLists.txt            |  1 +
 cuda/test/base/cuda_executor_reset.cu    | 65 ++++++++++++++++++++++++
 hip/test/base/CMakeLists.txt             |  1 +
 hip/test/base/hip_executor_reset.hip.cpp | 65 ++++++++++++++++++++++++
 6 files changed, 136 insertions(+), 5 deletions(-)
 create mode 100644 cuda/test/base/cuda_executor_reset.cu
 create mode 100644 hip/test/base/hip_executor_reset.hip.cpp

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8bfdcf7e8e8..2cfb0faf44f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -315,9 +315,9 @@ build/cuda91/clang/all/release/shared:
 
 # cuda 9.2 and friends
 build/cuda92/gcc/all/release/shared:
-  <<: *default_build
+  <<: *default_build_with_test
   extends:
-    - .full_test_condition
+    - .quick_test_condition
     - .use_gko-cuda92-gnu7-llvm50-intel2017
   variables:
     <<: *default_variables
@@ -325,7 +325,7 @@ build/cuda92/gcc/all/release/shared:
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
-    CUDA_ARCH: 35
+    CUDA_ARCH: 61
 
 # cuda 10.0 and friends
 # Make sure that our jobs run when using self-installed
diff --git a/.gitlab/image.yml b/.gitlab/image.yml
index 5ea3889d8df..a2afe0fcd53 100644
--- a/.gitlab/image.yml
+++ b/.gitlab/image.yml
@@ -30,8 +30,7 @@
   image: ginkgohub/cuda:92-gnu7-llvm50-intel2017
   tags:
     - private_ci
-    - controller
-    - cpu
+    - nvidia-gpu
 
 .use_gko-cuda100-gnu7-llvm60-intel2018:
   image: ginkgohub/cuda:100-gnu7-llvm60-intel2018
diff --git a/cuda/test/base/CMakeLists.txt b/cuda/test/base/CMakeLists.txt
index cedc2d8d328..bc9124a30bf 100644
--- a/cuda/test/base/CMakeLists.txt
+++ b/cuda/test/base/CMakeLists.txt
@@ -1,5 +1,6 @@
 ginkgo_create_cuda_test(array)
 ginkgo_create_cuda_test(cuda_executor)
+ginkgo_create_cuda_test(cuda_executor_reset)
 if(GINKGO_HAVE_HWLOC)
     find_package(NUMA REQUIRED)
     ginkgo_create_cuda_test(cuda_executor_topology NUMA::NUMA)
diff --git a/cuda/test/base/cuda_executor_reset.cu b/cuda/test/base/cuda_executor_reset.cu
new file mode 100644
index 00000000000..b63318b9afc
--- /dev/null
+++ b/cuda/test/base/cuda_executor_reset.cu
@@ -0,0 +1,65 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+namespace {
+
+
+#define GTEST_ASSERT_NO_EXIT(statement) \
+    ASSERT_EXIT({ {statement} exit(0); }, ::testing::ExitedWithCode(0), "")
+
+TEST(DeviceReset, HipCuda)
+{
+    GTEST_ASSERT_NO_EXIT({
+        auto ref = gko::ReferenceExecutor::create();
+        auto hip = gko::HipExecutor::create(0, ref, true);
+        auto cuda = gko::CudaExecutor::create(0, ref, true);
+    });
+}
+
+
+TEST(DeviceReset, CudaHip)
+{
+    GTEST_ASSERT_NO_EXIT({
+        auto ref = gko::ReferenceExecutor::create();
+        auto cuda = gko::CudaExecutor::create(0, ref, true);
+        auto hip = gko::HipExecutor::create(0, ref, true);
+    });
+}
+
+
+}  // namespace
diff --git a/hip/test/base/CMakeLists.txt b/hip/test/base/CMakeLists.txt
index c2f694fdf8a..14640d2fb2d 100644
--- a/hip/test/base/CMakeLists.txt
+++ b/hip/test/base/CMakeLists.txt
@@ -1,4 +1,5 @@
 ginkgo_create_hip_test(hip_executor)
+ginkgo_create_hip_test(hip_executor_reset)
 if(GINKGO_HAVE_HWLOC)
     find_package(NUMA REQUIRED)
     ginkgo_create_hip_test(hip_executor_topology NUMA::NUMA)
diff --git a/hip/test/base/hip_executor_reset.hip.cpp b/hip/test/base/hip_executor_reset.hip.cpp
new file mode 100644
index 00000000000..b63318b9afc
--- /dev/null
+++ b/hip/test/base/hip_executor_reset.hip.cpp
@@ -0,0 +1,65 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+namespace {
+
+
+#define GTEST_ASSERT_NO_EXIT(statement) \
+    ASSERT_EXIT({ {statement} exit(0); }, ::testing::ExitedWithCode(0), "")
+
+TEST(DeviceReset, HipCuda)
+{
+    GTEST_ASSERT_NO_EXIT({
+        auto ref = gko::ReferenceExecutor::create();
+        auto hip = gko::HipExecutor::create(0, ref, true);
+        auto cuda = gko::CudaExecutor::create(0, ref, true);
+    });
+}
+
+
+TEST(DeviceReset, CudaHip)
+{
+    GTEST_ASSERT_NO_EXIT({
+        auto ref = gko::ReferenceExecutor::create();
+        auto cuda = gko::CudaExecutor::create(0, ref, true);
+        auto hip = gko::HipExecutor::create(0, ref, true);
+    });
+}
+
+
+}  // namespace

From 198266517dc6be12efc618c2967e17222970c617 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 9 Jun 2021 21:07:38 +0200
Subject: [PATCH 0939/1008] add device

---
 core/device_hooks/CMakeLists.txt      |  5 ++
 cuda/CMakeLists.txt                   |  4 +-
 cuda/base/executor.cpp                |  3 +
 devices/CMakeLists.txt                |  1 +
 devices/cuda/executor.cpp             |  6 --
 devices/device.cpp                    | 54 ++++++++++++++++++
 devices/hip/executor.cpp              |  6 --
 dpcpp/CMakeLists.txt                  |  2 +-
 hip/CMakeLists.txt                    |  2 +-
 hip/base/executor.hip.cpp             |  3 +
 include/ginkgo/core/base/device.hpp   | 82 +++++++++++++++++++++++++++
 include/ginkgo/core/base/executor.hpp | 45 +++++++++------
 include/ginkgo/ginkgo.hpp             |  1 +
 omp/CMakeLists.txt                    |  2 +-
 reference/CMakeLists.txt              |  2 +-
 15 files changed, 182 insertions(+), 36 deletions(-)
 create mode 100644 devices/device.cpp
 create mode 100644 include/ginkgo/core/base/device.hpp

diff --git a/core/device_hooks/CMakeLists.txt b/core/device_hooks/CMakeLists.txt
index fcb370a81a0..a3d00014025 100644
--- a/core/device_hooks/CMakeLists.txt
+++ b/core/device_hooks/CMakeLists.txt
@@ -1,6 +1,7 @@
 if(NOT GINKGO_BUILD_CUDA)
     add_library(ginkgo_cuda
         $<TARGET_OBJECTS:ginkgo_cuda_device>
+        $<TARGET_OBJECTS:ginkgo_device_mutex>
         cuda_hooks.cpp)
     target_link_libraries(ginkgo_cuda PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_cuda)
@@ -11,6 +12,7 @@ endif()
 if (NOT GINKGO_BUILD_DPCPP)
     add_library(ginkgo_dpcpp
         $<TARGET_OBJECTS:ginkgo_dpcpp_device>
+        $<TARGET_OBJECTS:ginkgo_device_mutex>
         dpcpp_hooks.cpp)
     target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_dpcpp)
@@ -21,6 +23,7 @@ endif()
 if(NOT GINKGO_BUILD_HIP)
     add_library(ginkgo_hip
         $<TARGET_OBJECTS:ginkgo_hip_device>
+        $<TARGET_OBJECTS:ginkgo_device_mutex>
         hip_hooks.cpp)
     target_link_libraries(ginkgo_hip PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_hip)
@@ -31,6 +34,7 @@ endif()
 if (NOT GINKGO_BUILD_OMP)
     add_library(ginkgo_omp
         $<TARGET_OBJECTS:ginkgo_omp_device>
+        $<TARGET_OBJECTS:ginkgo_device_mutex>
         omp_hooks.cpp)
     ginkgo_compile_features(ginkgo_omp)
     target_link_libraries(ginkgo_omp PRIVATE ginkgo_cuda)
@@ -44,6 +48,7 @@ endif()
 if (NOT GINKGO_BUILD_REFERENCE)
     add_library(ginkgo_reference
         $<TARGET_OBJECTS:ginkgo_reference_device>
+        $<TARGET_OBJECTS:ginkgo_device_mutex>
         reference_hooks.cpp)
     target_link_libraries(ginkgo_reference PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_reference)
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 6f89ca45f79..0a45231aae1 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -64,8 +64,8 @@ find_library(CUSPARSE cusparse
 find_library(CURAND curand
     HINT ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
 
-add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_cuda_device> "")
-set(GKO_CUDA_COMMON_SOURCES    
+add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_cuda_device> $<TARGET_OBJECTS:ginkgo_device_mutex> "")
+set(GKO_CUDA_COMMON_SOURCES
     ../common/components/precision_conversion.cpp
     ../common/matrix/dense_kernels.cpp
     ../common/solver/bicg_kernels.cpp
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 0c7bbef8f10..38bffb3aa10 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/config.hpp>
+#include <ginkgo/core/base/device.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 
 
@@ -64,6 +65,8 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
                          alloc_mode),
         [device_id](CudaExecutor *exec) {
             auto device_reset = exec->get_device_reset();
+            std::lock_guard<std::recursive_mutex> guard(
+                device_type::mutex[device_id]);
             delete exec;
             if (!CudaExecutor::get_num_execs(device_id) && device_reset) {
                 cuda::device_guard g(device_id);
diff --git a/devices/CMakeLists.txt b/devices/CMakeLists.txt
index 0c0e80febaf..85632345dc2 100644
--- a/devices/CMakeLists.txt
+++ b/devices/CMakeLists.txt
@@ -17,6 +17,7 @@ function(ginkgo_add_library name)
 endfunction()
 
 ginkgo_add_library(ginkgo_device machine_topology.cpp)
+ginkgo_add_object_library(ginkgo_device_mutex device.cpp)
 ginkgo_install_library(ginkgo_device)
 
 add_subdirectory(cuda)
diff --git a/devices/cuda/executor.cpp b/devices/cuda/executor.cpp
index c1efe6e7d40..abdc93a4a08 100644
--- a/devices/cuda/executor.cpp
+++ b/devices/cuda/executor.cpp
@@ -64,10 +64,4 @@ bool CudaExecutor::verify_memory_to(const HipExecutor *dest_exec) const
 }
 
 
-unsigned CudaExecutor::num_execs[max_devices];
-
-
-std::mutex CudaExecutor::mutex[max_devices];
-
-
 }  // namespace gko
diff --git a/devices/device.cpp b/devices/device.cpp
new file mode 100644
index 00000000000..b178de0777f
--- /dev/null
+++ b/devices/device.cpp
@@ -0,0 +1,54 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <atomic>
+#include <memory>
+#include <mutex>
+
+
+#include <ginkgo/core/base/device.hpp>
+
+
+namespace gko {
+
+
+std::recursive_mutex NvidiaDevice::mutex[max_devices];
+
+int NvidiaDevice::num_execs[max_devices];
+
+
+std::recursive_mutex AmdDevice::mutex[max_devices];
+
+int AmdDevice::num_execs[max_devices];
+
+
+}  // namespace gko
diff --git a/devices/hip/executor.cpp b/devices/hip/executor.cpp
index 31325fcd15f..0e755209f91 100644
--- a/devices/hip/executor.cpp
+++ b/devices/hip/executor.cpp
@@ -61,10 +61,4 @@ bool HipExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
 }
 
 
-int HipExecutor::num_execs[max_devices];
-
-
-std::mutex HipExecutor::mutex[max_devices];
-
-
 }  // namespace gko
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index e2d476164e8..0289fd889e1 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -9,7 +9,7 @@ set(GINKGO_DPCPP_VERSION ${GINKGO_DPCPP_VERSION} PARENT_SCOPE)
 find_package(MKL CONFIG REQUIRED HINTS "$ENV{MKLROOT}")
 set(GINKGO_MKL_ROOT "${MKL_ROOT}" PARENT_SCOPE)
 
-add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_dpcpp_device> "")
+add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_dpcpp_device> $<TARGET_OBJECTS:ginkgo_device_mutex> "")
 target_sources(ginkgo_dpcpp
     PRIVATE
     base/version.dp.cpp
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 61870d03aa7..3d98ec4acd5 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -239,7 +239,7 @@ set(GINKGO_HIP_NVCC_OPTIONS ${GINKGO_HIP_NVCC_COMPILER_FLAGS} ${GINKGO_HIP_NVCC_
 set(GINKGO_HIP_CLANG_OPTIONS ${GINKGO_HIP_CLANG_COMPILER_FLAGS} ${GINKGO_AMD_ARCH_FLAGS})
 
 set_source_files_properties(${GINKGO_HIP_SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
-hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_hip_device> ${GINKGO_HIP_SOURCES}
+hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_hip_device> $<TARGET_OBJECTS:ginkgo_device_mutex> ${GINKGO_HIP_SOURCES}
     HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS}
     CLANG_OPTIONS ${GINKGO_HIP_CLANG_OPTIONS}
     NVCC_OPTIONS ${GINKGO_HIP_NVCC_OPTIONS}
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 1e43fe96549..cdb22ce7ee2 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/config.hpp>
+#include <ginkgo/core/base/device.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 
 
@@ -63,6 +64,8 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
         new HipExecutor(device_id, std::move(master), device_reset, alloc_mode),
         [device_id](HipExecutor *exec) {
             auto device_reset = exec->get_device_reset();
+            std::lock_guard<std::recursive_mutex> guard(
+                device_type::mutex[device_id]);
             delete exec;
             if (!HipExecutor::get_num_execs(device_id) && device_reset) {
                 hip::device_guard g(device_id);
diff --git a/include/ginkgo/core/base/device.hpp b/include/ginkgo/core/base/device.hpp
new file mode 100644
index 00000000000..f933d05e1be
--- /dev/null
+++ b/include/ginkgo/core/base/device.hpp
@@ -0,0 +1,82 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_PUBLIC_CORE_BASE_DEVICE_HPP_
+#define GKO_PUBLIC_CORE_BASE_DEVICE_HPP_
+
+
+#include <array>
+#include <cstdint>
+#include <mutex>
+#include <type_traits>
+
+
+#include <ginkgo/config.hpp>
+
+
+namespace gko {
+
+
+class CudaExecutor;
+
+class HipExecutor;
+
+
+class NvidiaDevice {
+    friend class CudaExecutor;
+#if GINKGO_HIP_PLATFORM_NVCC
+    friend class HipExecutor;
+#endif
+
+private:
+    static constexpr int max_devices = 64;
+    static std::recursive_mutex mutex[max_devices];
+    static int num_execs[max_devices];
+};
+
+
+class AmdDevice {
+// to avoid both GINKGO_HIP_PLATFORM_* zero
+#if !GINKGO_HIP_PLATFORM_NVCC
+    friend class HipExecutor;
+#endif
+
+private:
+    static constexpr int max_devices = 64;
+    static std::recursive_mutex mutex[max_devices];
+    static int num_execs[max_devices];
+};
+
+
+}  // namespace gko
+
+#endif  // GKO_PUBLIC_CORE_BASE_DEVICE_HPP_
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 0d96cb4cd66..3e5c47d5278 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <vector>
 
 
+#include <ginkgo/core/base/device.hpp>
 #include <ginkgo/core/base/machine_topology.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/log/logger.hpp>
@@ -1408,6 +1409,8 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     int get_closest_numa() const { return this->get_exec_info().numa_node; }
 
 protected:
+    using device_type = NvidiaDevice;
+
     void set_gpu_property();
 
     void init_handles();
@@ -1450,20 +1453,23 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
     static void increase_num_execs(unsigned device_id)
     {
-        std::lock_guard<std::mutex> guard(mutex[device_id]);
-        num_execs[device_id]++;
+        std::lock_guard<std::recursive_mutex> guard(
+            device_type::mutex[device_id]);
+        device_type::num_execs[device_id]++;
     }
 
     static void decrease_num_execs(unsigned device_id)
     {
-        std::lock_guard<std::mutex> guard(mutex[device_id]);
-        num_execs[device_id]--;
+        std::lock_guard<std::recursive_mutex> guard(
+            device_type::mutex[device_id]);
+        device_type::num_execs[device_id]--;
     }
 
     static unsigned get_num_execs(unsigned device_id)
     {
-        std::lock_guard<std::mutex> guard(mutex[device_id]);
-        return num_execs[device_id];
+        std::lock_guard<std::recursive_mutex> guard(
+            device_type::mutex[device_id]);
+        return device_type::num_execs[device_id];
     }
 
     void populate_exec_info(const MachineTopology *mach_topo) override;
@@ -1476,9 +1482,6 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     handle_manager<cublasContext> cublas_handle_;
     handle_manager<cusparseContext> cusparse_handle_;
 
-    static constexpr int max_devices = 64;
-    static unsigned num_execs[max_devices];
-    static std::mutex mutex[max_devices];
     allocation_mode alloc_mode_;
 };
 
@@ -1625,6 +1628,12 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     }
 
 protected:
+#if (GINKGO_HIP_PLATFORM_NVCC == 1)
+    using device_type = NvidiaDevice;
+#else
+    using device_type = AmdDevice;
+#endif
+
     void set_gpu_property();
 
     void init_handles();
@@ -1667,20 +1676,23 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
     static void increase_num_execs(int device_id)
     {
-        std::lock_guard<std::mutex> guard(mutex[device_id]);
-        num_execs[device_id]++;
+        std::lock_guard<std::recursive_mutex> guard(
+            device_type::mutex[device_id]);
+        device_type::num_execs[device_id]++;
     }
 
     static void decrease_num_execs(int device_id)
     {
-        std::lock_guard<std::mutex> guard(mutex[device_id]);
-        num_execs[device_id]--;
+        std::lock_guard<std::recursive_mutex> guard(
+            device_type::mutex[device_id]);
+        device_type::num_execs[device_id]--;
     }
 
     static int get_num_execs(int device_id)
     {
-        std::lock_guard<std::mutex> guard(mutex[device_id]);
-        return num_execs[device_id];
+        std::lock_guard<std::recursive_mutex> guard(
+            device_type::mutex[device_id]);
+        return device_type::num_execs[device_id];
     }
 
     void populate_exec_info(const MachineTopology *mach_topo) override;
@@ -1693,9 +1705,6 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     handle_manager<hipblasContext> hipblas_handle_;
     handle_manager<hipsparseContext> hipsparse_handle_;
 
-    static constexpr int max_devices = 64;
-    static int num_execs[max_devices];
-    static std::mutex mutex[max_devices];
     allocation_mode alloc_mode_;
 };
 
diff --git a/include/ginkgo/ginkgo.hpp b/include/ginkgo/ginkgo.hpp
index a757a5a6072..a16163edc8a 100644
--- a/include/ginkgo/ginkgo.hpp
+++ b/include/ginkgo/ginkgo.hpp
@@ -41,6 +41,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/combination.hpp>
 #include <ginkgo/core/base/composition.hpp>
+#include <ginkgo/core/base/device.hpp>
 #include <ginkgo/core/base/dim.hpp>
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index c58709bc904..af680f7a39b 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -1,6 +1,6 @@
 find_package(OpenMP 3.0 REQUIRED)
 
-add_library(ginkgo_omp $<TARGET_OBJECTS:ginkgo_omp_device> "")
+add_library(ginkgo_omp $<TARGET_OBJECTS:ginkgo_omp_device> $<TARGET_OBJECTS:ginkgo_device_mutex> "")
 target_sources(ginkgo_omp
     PRIVATE
     base/version.cpp
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index c008ad1a87c..a66c0323d4f 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_library(ginkgo_reference $<TARGET_OBJECTS:ginkgo_reference_device> "")
+add_library(ginkgo_reference $<TARGET_OBJECTS:ginkgo_reference_device> $<TARGET_OBJECTS:ginkgo_device_mutex> "")
 target_sources(ginkgo_reference
     PRIVATE
     base/version.cpp

From 386a00a548b04bb47adb4dad913ac64176a64301 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 1 Jul 2021 14:44:05 +0200
Subject: [PATCH 0940/1008] put device in ginkgo_device not device_mutex

---
 core/device_hooks/CMakeLists.txt    |  5 -----
 cuda/CMakeLists.txt                 |  2 +-
 devices/CMakeLists.txt              |  3 +--
 dpcpp/CMakeLists.txt                |  2 +-
 hip/CMakeLists.txt                  |  2 +-
 include/ginkgo/core/base/device.hpp | 10 ++++++++++
 omp/CMakeLists.txt                  |  2 +-
 reference/CMakeLists.txt            |  2 +-
 8 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/core/device_hooks/CMakeLists.txt b/core/device_hooks/CMakeLists.txt
index a3d00014025..fcb370a81a0 100644
--- a/core/device_hooks/CMakeLists.txt
+++ b/core/device_hooks/CMakeLists.txt
@@ -1,7 +1,6 @@
 if(NOT GINKGO_BUILD_CUDA)
     add_library(ginkgo_cuda
         $<TARGET_OBJECTS:ginkgo_cuda_device>
-        $<TARGET_OBJECTS:ginkgo_device_mutex>
         cuda_hooks.cpp)
     target_link_libraries(ginkgo_cuda PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_cuda)
@@ -12,7 +11,6 @@ endif()
 if (NOT GINKGO_BUILD_DPCPP)
     add_library(ginkgo_dpcpp
         $<TARGET_OBJECTS:ginkgo_dpcpp_device>
-        $<TARGET_OBJECTS:ginkgo_device_mutex>
         dpcpp_hooks.cpp)
     target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_dpcpp)
@@ -23,7 +21,6 @@ endif()
 if(NOT GINKGO_BUILD_HIP)
     add_library(ginkgo_hip
         $<TARGET_OBJECTS:ginkgo_hip_device>
-        $<TARGET_OBJECTS:ginkgo_device_mutex>
         hip_hooks.cpp)
     target_link_libraries(ginkgo_hip PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_hip)
@@ -34,7 +31,6 @@ endif()
 if (NOT GINKGO_BUILD_OMP)
     add_library(ginkgo_omp
         $<TARGET_OBJECTS:ginkgo_omp_device>
-        $<TARGET_OBJECTS:ginkgo_device_mutex>
         omp_hooks.cpp)
     ginkgo_compile_features(ginkgo_omp)
     target_link_libraries(ginkgo_omp PRIVATE ginkgo_cuda)
@@ -48,7 +44,6 @@ endif()
 if (NOT GINKGO_BUILD_REFERENCE)
     add_library(ginkgo_reference
         $<TARGET_OBJECTS:ginkgo_reference_device>
-        $<TARGET_OBJECTS:ginkgo_device_mutex>
         reference_hooks.cpp)
     target_link_libraries(ginkgo_reference PUBLIC ginkgo_device)
     ginkgo_compile_features(ginkgo_reference)
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 0a45231aae1..2e25435e5c5 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -64,7 +64,7 @@ find_library(CUSPARSE cusparse
 find_library(CURAND curand
     HINT ${CMAKE_CUDA_IMPLICIT_LINK_DIRECTORIES})
 
-add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_cuda_device> $<TARGET_OBJECTS:ginkgo_device_mutex> "")
+add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_cuda_device> "")
 set(GKO_CUDA_COMMON_SOURCES
     ../common/components/precision_conversion.cpp
     ../common/matrix/dense_kernels.cpp
diff --git a/devices/CMakeLists.txt b/devices/CMakeLists.txt
index 85632345dc2..09797aafe49 100644
--- a/devices/CMakeLists.txt
+++ b/devices/CMakeLists.txt
@@ -16,8 +16,7 @@ function(ginkgo_add_library name)
     set_target_properties(${name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endfunction()
 
-ginkgo_add_library(ginkgo_device machine_topology.cpp)
-ginkgo_add_object_library(ginkgo_device_mutex device.cpp)
+ginkgo_add_library(ginkgo_device machine_topology.cpp device.cpp)
 ginkgo_install_library(ginkgo_device)
 
 add_subdirectory(cuda)
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 0289fd889e1..e2d476164e8 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -9,7 +9,7 @@ set(GINKGO_DPCPP_VERSION ${GINKGO_DPCPP_VERSION} PARENT_SCOPE)
 find_package(MKL CONFIG REQUIRED HINTS "$ENV{MKLROOT}")
 set(GINKGO_MKL_ROOT "${MKL_ROOT}" PARENT_SCOPE)
 
-add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_dpcpp_device> $<TARGET_OBJECTS:ginkgo_device_mutex> "")
+add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_dpcpp_device> "")
 target_sources(ginkgo_dpcpp
     PRIVATE
     base/version.dp.cpp
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 3d98ec4acd5..61870d03aa7 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -239,7 +239,7 @@ set(GINKGO_HIP_NVCC_OPTIONS ${GINKGO_HIP_NVCC_COMPILER_FLAGS} ${GINKGO_HIP_NVCC_
 set(GINKGO_HIP_CLANG_OPTIONS ${GINKGO_HIP_CLANG_COMPILER_FLAGS} ${GINKGO_AMD_ARCH_FLAGS})
 
 set_source_files_properties(${GINKGO_HIP_SOURCES} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT TRUE)
-hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_hip_device> $<TARGET_OBJECTS:ginkgo_device_mutex> ${GINKGO_HIP_SOURCES}
+hip_add_library(ginkgo_hip $<TARGET_OBJECTS:ginkgo_hip_device> ${GINKGO_HIP_SOURCES}
     HIPCC_OPTIONS ${GINKGO_HIPCC_OPTIONS}
     CLANG_OPTIONS ${GINKGO_HIP_CLANG_OPTIONS}
     NVCC_OPTIONS ${GINKGO_HIP_NVCC_OPTIONS}
diff --git a/include/ginkgo/core/base/device.hpp b/include/ginkgo/core/base/device.hpp
index f933d05e1be..01f0bd7107d 100644
--- a/include/ginkgo/core/base/device.hpp
+++ b/include/ginkgo/core/base/device.hpp
@@ -51,8 +51,13 @@ class CudaExecutor;
 class HipExecutor;
 
 
+/**
+ * NvidiaDevice handles the number of executor on Nvidia devices and have the
+ * corresponding recursive_mutex.
+ */
 class NvidiaDevice {
     friend class CudaExecutor;
+// If Hip is compiled for NVCC, give NvidiaDevice's permission to HipExecutor
 #if GINKGO_HIP_PLATFORM_NVCC
     friend class HipExecutor;
 #endif
@@ -64,8 +69,13 @@ class NvidiaDevice {
 };
 
 
+/**
+ * AmdDevice handles the number of executor on Amd devices and have the
+ * corresponding recursive_mutex.
+ */
 class AmdDevice {
 // to avoid both GINKGO_HIP_PLATFORM_* zero
+// If Hip isn't compiled for NVCC, give AmdDevice's permission to HipExecutor
 #if !GINKGO_HIP_PLATFORM_NVCC
     friend class HipExecutor;
 #endif
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index af680f7a39b..c58709bc904 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -1,6 +1,6 @@
 find_package(OpenMP 3.0 REQUIRED)
 
-add_library(ginkgo_omp $<TARGET_OBJECTS:ginkgo_omp_device> $<TARGET_OBJECTS:ginkgo_device_mutex> "")
+add_library(ginkgo_omp $<TARGET_OBJECTS:ginkgo_omp_device> "")
 target_sources(ginkgo_omp
     PRIVATE
     base/version.cpp
diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt
index a66c0323d4f..c008ad1a87c 100644
--- a/reference/CMakeLists.txt
+++ b/reference/CMakeLists.txt
@@ -1,4 +1,4 @@
-add_library(ginkgo_reference $<TARGET_OBJECTS:ginkgo_reference_device> $<TARGET_OBJECTS:ginkgo_device_mutex> "")
+add_library(ginkgo_reference $<TARGET_OBJECTS:ginkgo_reference_device> "")
 target_sources(ginkgo_reference
     PRIVATE
     base/version.cpp

From 2824f698da795cd5c7ff681f6bd6dce91064bafd Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 1 Jul 2021 18:06:11 +0200
Subject: [PATCH 0941/1008] use static getter to avoid the static issue in MSVC

---
 cuda/base/executor.cpp                |  2 +-
 devices/CMakeLists.txt                |  2 +-
 devices/device.cpp                    | 54 --------------------------
 hip/base/executor.hip.cpp             |  2 +-
 include/ginkgo/core/base/device.hpp   | 56 +++++++++++++++++++++++++--
 include/ginkgo/core/base/executor.hpp | 24 ++++++------
 6 files changed, 67 insertions(+), 73 deletions(-)
 delete mode 100644 devices/device.cpp

diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 38bffb3aa10..47a712c3eba 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -66,7 +66,7 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
         [device_id](CudaExecutor *exec) {
             auto device_reset = exec->get_device_reset();
             std::lock_guard<std::recursive_mutex> guard(
-                device_type::mutex[device_id]);
+                device_type::get_mutex(device_id));
             delete exec;
             if (!CudaExecutor::get_num_execs(device_id) && device_reset) {
                 cuda::device_guard g(device_id);
diff --git a/devices/CMakeLists.txt b/devices/CMakeLists.txt
index 09797aafe49..0c0e80febaf 100644
--- a/devices/CMakeLists.txt
+++ b/devices/CMakeLists.txt
@@ -16,7 +16,7 @@ function(ginkgo_add_library name)
     set_target_properties(${name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endfunction()
 
-ginkgo_add_library(ginkgo_device machine_topology.cpp device.cpp)
+ginkgo_add_library(ginkgo_device machine_topology.cpp)
 ginkgo_install_library(ginkgo_device)
 
 add_subdirectory(cuda)
diff --git a/devices/device.cpp b/devices/device.cpp
deleted file mode 100644
index b178de0777f..00000000000
--- a/devices/device.cpp
+++ /dev/null
@@ -1,54 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#include <atomic>
-#include <memory>
-#include <mutex>
-
-
-#include <ginkgo/core/base/device.hpp>
-
-
-namespace gko {
-
-
-std::recursive_mutex NvidiaDevice::mutex[max_devices];
-
-int NvidiaDevice::num_execs[max_devices];
-
-
-std::recursive_mutex AmdDevice::mutex[max_devices];
-
-int AmdDevice::num_execs[max_devices];
-
-
-}  // namespace gko
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index cdb22ce7ee2..36df8fa7152 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -65,7 +65,7 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
         [device_id](HipExecutor *exec) {
             auto device_reset = exec->get_device_reset();
             std::lock_guard<std::recursive_mutex> guard(
-                device_type::mutex[device_id]);
+                device_type::get_mutex(device_id));
             delete exec;
             if (!HipExecutor::get_num_execs(device_id) && device_reset) {
                 hip::device_guard g(device_id);
diff --git a/include/ginkgo/core/base/device.hpp b/include/ginkgo/core/base/device.hpp
index 01f0bd7107d..f60fecae736 100644
--- a/include/ginkgo/core/base/device.hpp
+++ b/include/ginkgo/core/base/device.hpp
@@ -63,9 +63,33 @@ class NvidiaDevice {
 #endif
 
 private:
+    /**
+     * get_mutex gets the static mutex reference at i.
+     *
+     * @param i  index of mutex
+     *
+     * @return recursive_mutex reference
+     */
+    static std::recursive_mutex &get_mutex(int i)
+    {
+        static std::recursive_mutex mutex[max_devices];
+        return mutex[i];
+    }
+
+    /**
+     * get_num_execs gets the static num_execs reference at i.
+     *
+     * @param i  index of num_execs
+     *
+     * @return int reference
+     */
+    static int &get_num_execs(int i)
+    {
+        static int num_execs[max_devices];
+        return num_execs[i];
+    }
+
     static constexpr int max_devices = 64;
-    static std::recursive_mutex mutex[max_devices];
-    static int num_execs[max_devices];
 };
 
 
@@ -81,9 +105,33 @@ class AmdDevice {
 #endif
 
 private:
+    /**
+     * get_mutex gets the static mutex reference at i.
+     *
+     * @param i  index of mutex
+     *
+     * @return recursive_mutex reference
+     */
+    static std::recursive_mutex &get_mutex(int i)
+    {
+        static std::recursive_mutex mutex[max_devices];
+        return mutex[i];
+    }
+
+    /**
+     * get_num_execs gets the static num_execs reference at i.
+     *
+     * @param i  index of num_execs
+     *
+     * @return int reference
+     */
+    static int &get_num_execs(int i)
+    {
+        static int num_execs[max_devices];
+        return num_execs[i];
+    }
+
     static constexpr int max_devices = 64;
-    static std::recursive_mutex mutex[max_devices];
-    static int num_execs[max_devices];
 };
 
 
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 3e5c47d5278..9e7f12ab74f 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -1454,22 +1454,22 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     static void increase_num_execs(unsigned device_id)
     {
         std::lock_guard<std::recursive_mutex> guard(
-            device_type::mutex[device_id]);
-        device_type::num_execs[device_id]++;
+            device_type::get_mutex(device_id));
+        device_type::get_num_execs(device_id)++;
     }
 
     static void decrease_num_execs(unsigned device_id)
     {
         std::lock_guard<std::recursive_mutex> guard(
-            device_type::mutex[device_id]);
-        device_type::num_execs[device_id]--;
+            device_type::get_mutex(device_id));
+        device_type::get_num_execs(device_id)--;
     }
 
     static unsigned get_num_execs(unsigned device_id)
     {
         std::lock_guard<std::recursive_mutex> guard(
-            device_type::mutex[device_id]);
-        return device_type::num_execs[device_id];
+            device_type::get_mutex(device_id));
+        return device_type::get_num_execs(device_id);
     }
 
     void populate_exec_info(const MachineTopology *mach_topo) override;
@@ -1677,22 +1677,22 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     static void increase_num_execs(int device_id)
     {
         std::lock_guard<std::recursive_mutex> guard(
-            device_type::mutex[device_id]);
-        device_type::num_execs[device_id]++;
+            device_type::get_mutex(device_id));
+        device_type::get_num_execs(device_id)++;
     }
 
     static void decrease_num_execs(int device_id)
     {
         std::lock_guard<std::recursive_mutex> guard(
-            device_type::mutex[device_id]);
-        device_type::num_execs[device_id]--;
+            device_type::get_mutex(device_id));
+        device_type::get_num_execs(device_id)--;
     }
 
     static int get_num_execs(int device_id)
     {
         std::lock_guard<std::recursive_mutex> guard(
-            device_type::mutex[device_id]);
-        return device_type::num_execs[device_id];
+            device_type::get_mutex(device_id));
+        return device_type::get_num_execs(device_id);
     }
 
     void populate_exec_info(const MachineTopology *mach_topo) override;

From ca1e0d24884e99c2012834710fcd6ba07ea3bf09 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 8 Jul 2021 17:01:45 +0200
Subject: [PATCH 0942/1008] add cudacuda and hiphip test

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 cuda/test/base/cuda_executor_reset.cu    | 33 ++++++++++++++++++++++++
 hip/test/base/hip_executor_reset.hip.cpp | 33 ++++++++++++++++++++++++
 2 files changed, 66 insertions(+)

diff --git a/cuda/test/base/cuda_executor_reset.cu b/cuda/test/base/cuda_executor_reset.cu
index b63318b9afc..4807354bdea 100644
--- a/cuda/test/base/cuda_executor_reset.cu
+++ b/cuda/test/base/cuda_executor_reset.cu
@@ -33,6 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
+#include <thread>
+
+
 #include <gtest/gtest.h>
 
 
@@ -62,4 +65,34 @@ TEST(DeviceReset, CudaHip)
 }
 
 
+template <typename Executor>
+void func()
+{
+    auto ref = gko::ReferenceExecutor::create();
+    auto exec = Executor::create(0, ref, true);
+}
+
+
+TEST(DeviceReset, CudaCuda)
+{
+    GTEST_ASSERT_NO_EXIT({
+        std::thread t1(func<gko::CudaExecutor>);
+        std::thread t2(func<gko::CudaExecutor>);
+        t1.join();
+        t2.join();
+    });
+}
+
+
+TEST(DeviceReset, HipHip)
+{
+    GTEST_ASSERT_NO_EXIT({
+        std::thread t1(func<gko::CudaExecutor>);
+        std::thread t2(func<gko::CudaExecutor>);
+        t1.join();
+        t2.join();
+    });
+}
+
+
 }  // namespace
diff --git a/hip/test/base/hip_executor_reset.hip.cpp b/hip/test/base/hip_executor_reset.hip.cpp
index b63318b9afc..4807354bdea 100644
--- a/hip/test/base/hip_executor_reset.hip.cpp
+++ b/hip/test/base/hip_executor_reset.hip.cpp
@@ -33,6 +33,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 
 
+#include <thread>
+
+
 #include <gtest/gtest.h>
 
 
@@ -62,4 +65,34 @@ TEST(DeviceReset, CudaHip)
 }
 
 
+template <typename Executor>
+void func()
+{
+    auto ref = gko::ReferenceExecutor::create();
+    auto exec = Executor::create(0, ref, true);
+}
+
+
+TEST(DeviceReset, CudaCuda)
+{
+    GTEST_ASSERT_NO_EXIT({
+        std::thread t1(func<gko::CudaExecutor>);
+        std::thread t2(func<gko::CudaExecutor>);
+        t1.join();
+        t2.join();
+    });
+}
+
+
+TEST(DeviceReset, HipHip)
+{
+    GTEST_ASSERT_NO_EXIT({
+        std::thread t1(func<gko::CudaExecutor>);
+        std::thread t2(func<gko::CudaExecutor>);
+        t1.join();
+        t2.join();
+    });
+}
+
+
 }  // namespace

From 05adcb50a98061ed456709938c227070f85f5439 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 9 Jul 2021 15:07:12 +0200
Subject: [PATCH 0943/1008] ensure ops affected by Reset after increase_exec

---
 cuda/test/base/CMakeLists.txt                 |  2 +-
 ...cutor_reset.cu => cuda_executor_reset.cpp} | 19 ++++---------------
 hip/test/base/CMakeLists.txt                  |  2 +-
 ...r_reset.hip.cpp => hip_executor_reset.cpp} | 19 ++++---------------
 include/ginkgo/core/base/executor.hpp         | 12 ++++++++++--
 5 files changed, 20 insertions(+), 34 deletions(-)
 rename cuda/test/base/{cuda_executor_reset.cu => cuda_executor_reset.cpp} (87%)
 rename hip/test/base/{hip_executor_reset.hip.cpp => hip_executor_reset.cpp} (87%)

diff --git a/cuda/test/base/CMakeLists.txt b/cuda/test/base/CMakeLists.txt
index bc9124a30bf..8bd6194f134 100644
--- a/cuda/test/base/CMakeLists.txt
+++ b/cuda/test/base/CMakeLists.txt
@@ -1,6 +1,6 @@
 ginkgo_create_cuda_test(array)
 ginkgo_create_cuda_test(cuda_executor)
-ginkgo_create_cuda_test(cuda_executor_reset)
+ginkgo_create_test(cuda_executor_reset)
 if(GINKGO_HAVE_HWLOC)
     find_package(NUMA REQUIRED)
     ginkgo_create_cuda_test(cuda_executor_topology NUMA::NUMA)
diff --git a/cuda/test/base/cuda_executor_reset.cu b/cuda/test/base/cuda_executor_reset.cpp
similarity index 87%
rename from cuda/test/base/cuda_executor_reset.cu
rename to cuda/test/base/cuda_executor_reset.cpp
index 4807354bdea..5d9169be808 100644
--- a/cuda/test/base/cuda_executor_reset.cu
+++ b/cuda/test/base/cuda_executor_reset.cpp
@@ -45,6 +45,7 @@ namespace {
 #define GTEST_ASSERT_NO_EXIT(statement) \
     ASSERT_EXIT({ {statement} exit(0); }, ::testing::ExitedWithCode(0), "")
 
+
 TEST(DeviceReset, HipCuda)
 {
     GTEST_ASSERT_NO_EXIT({
@@ -65,30 +66,18 @@ TEST(DeviceReset, CudaHip)
 }
 
 
-template <typename Executor>
 void func()
 {
     auto ref = gko::ReferenceExecutor::create();
-    auto exec = Executor::create(0, ref, true);
+    auto exec = gko::CudaExecutor::create(0, ref, true);
 }
 
 
 TEST(DeviceReset, CudaCuda)
 {
     GTEST_ASSERT_NO_EXIT({
-        std::thread t1(func<gko::CudaExecutor>);
-        std::thread t2(func<gko::CudaExecutor>);
-        t1.join();
-        t2.join();
-    });
-}
-
-
-TEST(DeviceReset, HipHip)
-{
-    GTEST_ASSERT_NO_EXIT({
-        std::thread t1(func<gko::CudaExecutor>);
-        std::thread t2(func<gko::CudaExecutor>);
+        std::thread t1(func);
+        std::thread t2(func);
         t1.join();
         t2.join();
     });
diff --git a/hip/test/base/CMakeLists.txt b/hip/test/base/CMakeLists.txt
index 14640d2fb2d..dd91d9e3c5f 100644
--- a/hip/test/base/CMakeLists.txt
+++ b/hip/test/base/CMakeLists.txt
@@ -1,5 +1,5 @@
 ginkgo_create_hip_test(hip_executor)
-ginkgo_create_hip_test(hip_executor_reset)
+ginkgo_create_test(hip_executor_reset)
 if(GINKGO_HAVE_HWLOC)
     find_package(NUMA REQUIRED)
     ginkgo_create_hip_test(hip_executor_topology NUMA::NUMA)
diff --git a/hip/test/base/hip_executor_reset.hip.cpp b/hip/test/base/hip_executor_reset.cpp
similarity index 87%
rename from hip/test/base/hip_executor_reset.hip.cpp
rename to hip/test/base/hip_executor_reset.cpp
index 4807354bdea..33a9c8040c0 100644
--- a/hip/test/base/hip_executor_reset.hip.cpp
+++ b/hip/test/base/hip_executor_reset.cpp
@@ -45,6 +45,7 @@ namespace {
 #define GTEST_ASSERT_NO_EXIT(statement) \
     ASSERT_EXIT({ {statement} exit(0); }, ::testing::ExitedWithCode(0), "")
 
+
 TEST(DeviceReset, HipCuda)
 {
     GTEST_ASSERT_NO_EXIT({
@@ -65,30 +66,18 @@ TEST(DeviceReset, CudaHip)
 }
 
 
-template <typename Executor>
 void func()
 {
     auto ref = gko::ReferenceExecutor::create();
-    auto exec = Executor::create(0, ref, true);
-}
-
-
-TEST(DeviceReset, CudaCuda)
-{
-    GTEST_ASSERT_NO_EXIT({
-        std::thread t1(func<gko::CudaExecutor>);
-        std::thread t2(func<gko::CudaExecutor>);
-        t1.join();
-        t2.join();
-    });
+    auto exec = gko::HipExecutor::create(0, ref, true);
 }
 
 
 TEST(DeviceReset, HipHip)
 {
     GTEST_ASSERT_NO_EXIT({
-        std::thread t1(func<gko::CudaExecutor>);
-        std::thread t2(func<gko::CudaExecutor>);
+        std::thread t1(func);
+        std::thread t2(func);
         t1.join();
         t2.join();
     });
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 9e7f12ab74f..fb484871f9b 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -1430,9 +1430,13 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
             MachineTopology::get_instance()->bind_to_pus(
                 this->get_closest_pus());
         }
+        // it only gets attribute from device, so it should not be affected by
+        // DeviceReset.
         this->set_gpu_property();
-        this->init_handles();
+        // increase the number of executor before any operations may be affected
+        // by DeviceReset.
         increase_num_execs(this->get_exec_info().device_id);
+        this->init_handles();
     }
 
     void *raw_alloc(size_type size) const override;
@@ -1653,9 +1657,13 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
             MachineTopology::get_instance()->bind_to_pus(
                 this->get_closest_pus());
         }
+        // it only gets attribute from device, so it should not be affected by
+        // DeviceReset.
         this->set_gpu_property();
-        this->init_handles();
+        // increase the number of executor before any operations may be affected
+        // by DeviceReset.
         increase_num_execs(this->get_exec_info().device_id);
+        this->init_handles();
     }
 
     void *raw_alloc(size_type size) const override;

From bf667d771b4aab4d285a64e06fcff941566d6e2a Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 29 Jul 2021 08:56:36 +0200
Subject: [PATCH 0944/1008] use device_class and move definition to cpp link
 thread in test

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 cuda/base/executor.cpp                |  2 +-
 cuda/test/base/CMakeLists.txt         |  2 +-
 devices/CMakeLists.txt                |  2 +-
 devices/device.cpp                    | 68 +++++++++++++++++++++++++++
 hip/base/executor.hip.cpp             |  2 +-
 hip/test/base/CMakeLists.txt          |  2 +-
 include/ginkgo/core/base/device.hpp   | 24 ++--------
 include/ginkgo/core/base/executor.hpp | 30 ++++++------
 8 files changed, 92 insertions(+), 40 deletions(-)
 create mode 100644 devices/device.cpp

diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 47a712c3eba..4091a251aeb 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -66,7 +66,7 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
         [device_id](CudaExecutor *exec) {
             auto device_reset = exec->get_device_reset();
             std::lock_guard<std::recursive_mutex> guard(
-                device_type::get_mutex(device_id));
+                device_class::get_mutex(device_id));
             delete exec;
             if (!CudaExecutor::get_num_execs(device_id) && device_reset) {
                 cuda::device_guard g(device_id);
diff --git a/cuda/test/base/CMakeLists.txt b/cuda/test/base/CMakeLists.txt
index 8bd6194f134..c23efefa8ed 100644
--- a/cuda/test/base/CMakeLists.txt
+++ b/cuda/test/base/CMakeLists.txt
@@ -1,6 +1,6 @@
 ginkgo_create_cuda_test(array)
 ginkgo_create_cuda_test(cuda_executor)
-ginkgo_create_test(cuda_executor_reset)
+ginkgo_create_thread_test(cuda_executor_reset)
 if(GINKGO_HAVE_HWLOC)
     find_package(NUMA REQUIRED)
     ginkgo_create_cuda_test(cuda_executor_topology NUMA::NUMA)
diff --git a/devices/CMakeLists.txt b/devices/CMakeLists.txt
index 0c0e80febaf..09797aafe49 100644
--- a/devices/CMakeLists.txt
+++ b/devices/CMakeLists.txt
@@ -16,7 +16,7 @@ function(ginkgo_add_library name)
     set_target_properties(${name} PROPERTIES POSITION_INDEPENDENT_CODE ON)
 endfunction()
 
-ginkgo_add_library(ginkgo_device machine_topology.cpp)
+ginkgo_add_library(ginkgo_device machine_topology.cpp device.cpp)
 ginkgo_install_library(ginkgo_device)
 
 add_subdirectory(cuda)
diff --git a/devices/device.cpp b/devices/device.cpp
new file mode 100644
index 00000000000..79f7da20077
--- /dev/null
+++ b/devices/device.cpp
@@ -0,0 +1,68 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <memory>
+#include <mutex>
+
+
+#include <ginkgo/core/base/device.hpp>
+
+
+namespace gko {
+
+
+std::recursive_mutex &NvidiaDevice::get_mutex(int i)
+{
+    static std::recursive_mutex mutex[max_devices];
+    return mutex[i];
+}
+
+int &NvidiaDevice::get_num_execs(int i)
+{
+    static int num_execs[max_devices];
+    return num_execs[i];
+}
+
+std::recursive_mutex &AmdDevice::get_mutex(int i)
+{
+    static std::recursive_mutex mutex[max_devices];
+    return mutex[i];
+}
+
+int &AmdDevice::get_num_execs(int i)
+{
+    static int num_execs[max_devices];
+    return num_execs[i];
+}
+
+
+}  // namespace gko
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 36df8fa7152..d0d64110289 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -65,7 +65,7 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
         [device_id](HipExecutor *exec) {
             auto device_reset = exec->get_device_reset();
             std::lock_guard<std::recursive_mutex> guard(
-                device_type::get_mutex(device_id));
+                device_class::get_mutex(device_id));
             delete exec;
             if (!HipExecutor::get_num_execs(device_id) && device_reset) {
                 hip::device_guard g(device_id);
diff --git a/hip/test/base/CMakeLists.txt b/hip/test/base/CMakeLists.txt
index dd91d9e3c5f..4529a40c862 100644
--- a/hip/test/base/CMakeLists.txt
+++ b/hip/test/base/CMakeLists.txt
@@ -1,5 +1,5 @@
 ginkgo_create_hip_test(hip_executor)
-ginkgo_create_test(hip_executor_reset)
+ginkgo_create_thread_test(hip_executor_reset)
 if(GINKGO_HAVE_HWLOC)
     find_package(NUMA REQUIRED)
     ginkgo_create_hip_test(hip_executor_topology NUMA::NUMA)
diff --git a/include/ginkgo/core/base/device.hpp b/include/ginkgo/core/base/device.hpp
index f60fecae736..025927ee367 100644
--- a/include/ginkgo/core/base/device.hpp
+++ b/include/ginkgo/core/base/device.hpp
@@ -70,11 +70,7 @@ class NvidiaDevice {
      *
      * @return recursive_mutex reference
      */
-    static std::recursive_mutex &get_mutex(int i)
-    {
-        static std::recursive_mutex mutex[max_devices];
-        return mutex[i];
-    }
+    static std::recursive_mutex &get_mutex(int i);
 
     /**
      * get_num_execs gets the static num_execs reference at i.
@@ -83,11 +79,7 @@ class NvidiaDevice {
      *
      * @return int reference
      */
-    static int &get_num_execs(int i)
-    {
-        static int num_execs[max_devices];
-        return num_execs[i];
-    }
+    static int &get_num_execs(int i);
 
     static constexpr int max_devices = 64;
 };
@@ -112,11 +104,7 @@ class AmdDevice {
      *
      * @return recursive_mutex reference
      */
-    static std::recursive_mutex &get_mutex(int i)
-    {
-        static std::recursive_mutex mutex[max_devices];
-        return mutex[i];
-    }
+    static std::recursive_mutex &get_mutex(int i);
 
     /**
      * get_num_execs gets the static num_execs reference at i.
@@ -125,11 +113,7 @@ class AmdDevice {
      *
      * @return int reference
      */
-    static int &get_num_execs(int i)
-    {
-        static int num_execs[max_devices];
-        return num_execs[i];
-    }
+    static int &get_num_execs(int i);
 
     static constexpr int max_devices = 64;
 };
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index fb484871f9b..8c172bc4ed9 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -1409,7 +1409,7 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     int get_closest_numa() const { return this->get_exec_info().numa_node; }
 
 protected:
-    using device_type = NvidiaDevice;
+    using device_class = NvidiaDevice;
 
     void set_gpu_property();
 
@@ -1458,22 +1458,22 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     static void increase_num_execs(unsigned device_id)
     {
         std::lock_guard<std::recursive_mutex> guard(
-            device_type::get_mutex(device_id));
-        device_type::get_num_execs(device_id)++;
+            device_class::get_mutex(device_id));
+        device_class::get_num_execs(device_id)++;
     }
 
     static void decrease_num_execs(unsigned device_id)
     {
         std::lock_guard<std::recursive_mutex> guard(
-            device_type::get_mutex(device_id));
-        device_type::get_num_execs(device_id)--;
+            device_class::get_mutex(device_id));
+        device_class::get_num_execs(device_id)--;
     }
 
     static unsigned get_num_execs(unsigned device_id)
     {
         std::lock_guard<std::recursive_mutex> guard(
-            device_type::get_mutex(device_id));
-        return device_type::get_num_execs(device_id);
+            device_class::get_mutex(device_id));
+        return device_class::get_num_execs(device_id);
     }
 
     void populate_exec_info(const MachineTopology *mach_topo) override;
@@ -1633,9 +1633,9 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
 protected:
 #if (GINKGO_HIP_PLATFORM_NVCC == 1)
-    using device_type = NvidiaDevice;
+    using device_class = NvidiaDevice;
 #else
-    using device_type = AmdDevice;
+    using device_class = AmdDevice;
 #endif
 
     void set_gpu_property();
@@ -1685,22 +1685,22 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     static void increase_num_execs(int device_id)
     {
         std::lock_guard<std::recursive_mutex> guard(
-            device_type::get_mutex(device_id));
-        device_type::get_num_execs(device_id)++;
+            device_class::get_mutex(device_id));
+        device_class::get_num_execs(device_id)++;
     }
 
     static void decrease_num_execs(int device_id)
     {
         std::lock_guard<std::recursive_mutex> guard(
-            device_type::get_mutex(device_id));
-        device_type::get_num_execs(device_id)--;
+            device_class::get_mutex(device_id));
+        device_class::get_num_execs(device_id)--;
     }
 
     static int get_num_execs(int device_id)
     {
         std::lock_guard<std::recursive_mutex> guard(
-            device_type::get_mutex(device_id));
-        return device_type::get_num_execs(device_id);
+            device_class::get_mutex(device_id));
+        return device_class::get_num_execs(device_id);
     }
 
     void populate_exec_info(const MachineTopology *mach_topo) override;

From 7ac9596aa3b450f074c13b670c6eb055d77d642c Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 29 Jul 2021 17:23:30 +0200
Subject: [PATCH 0945/1008] handle deleter in create not destructor of class it
 makes us keep mutex not recursive_mutex

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
---
 cuda/base/executor.cpp                |  6 ++++--
 devices/device.cpp                    |  8 ++++----
 hip/base/executor.hip.cpp             |  6 ++++--
 include/ginkgo/core/base/device.hpp   |  4 ++--
 include/ginkgo/core/base/executor.hpp | 22 ++++++----------------
 5 files changed, 20 insertions(+), 26 deletions(-)

diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 4091a251aeb..2898ff3126c 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -65,10 +65,12 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
                          alloc_mode),
         [device_id](CudaExecutor *exec) {
             auto device_reset = exec->get_device_reset();
-            std::lock_guard<std::recursive_mutex> guard(
+            std::lock_guard<std::mutex> guard(
                 device_class::get_mutex(device_id));
             delete exec;
-            if (!CudaExecutor::get_num_execs(device_id) && device_reset) {
+            auto &num_execs = device_class::get_num_execs(device_id);
+            num_execs--;
+            if (!num_execs && device_reset) {
                 cuda::device_guard g(device_id);
                 cudaDeviceReset();
             }
diff --git a/devices/device.cpp b/devices/device.cpp
index 79f7da20077..c70fdaaffb8 100644
--- a/devices/device.cpp
+++ b/devices/device.cpp
@@ -40,9 +40,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace gko {
 
 
-std::recursive_mutex &NvidiaDevice::get_mutex(int i)
+std::mutex &NvidiaDevice::get_mutex(int i)
 {
-    static std::recursive_mutex mutex[max_devices];
+    static std::mutex mutex[max_devices];
     return mutex[i];
 }
 
@@ -52,9 +52,9 @@ int &NvidiaDevice::get_num_execs(int i)
     return num_execs[i];
 }
 
-std::recursive_mutex &AmdDevice::get_mutex(int i)
+std::mutex &AmdDevice::get_mutex(int i)
 {
-    static std::recursive_mutex mutex[max_devices];
+    static std::mutex mutex[max_devices];
     return mutex[i];
 }
 
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index d0d64110289..c5b1c32920b 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -64,10 +64,12 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
         new HipExecutor(device_id, std::move(master), device_reset, alloc_mode),
         [device_id](HipExecutor *exec) {
             auto device_reset = exec->get_device_reset();
-            std::lock_guard<std::recursive_mutex> guard(
+            std::lock_guard<std::mutex> guard(
                 device_class::get_mutex(device_id));
             delete exec;
-            if (!HipExecutor::get_num_execs(device_id) && device_reset) {
+            auto &num_execs = device_class::get_num_execs(device_id);
+            num_execs--;
+            if (!num_execs && device_reset) {
                 hip::device_guard g(device_id);
                 hipDeviceReset();
             }
diff --git a/include/ginkgo/core/base/device.hpp b/include/ginkgo/core/base/device.hpp
index 025927ee367..d40ad7f6eef 100644
--- a/include/ginkgo/core/base/device.hpp
+++ b/include/ginkgo/core/base/device.hpp
@@ -70,7 +70,7 @@ class NvidiaDevice {
      *
      * @return recursive_mutex reference
      */
-    static std::recursive_mutex &get_mutex(int i);
+    static std::mutex &get_mutex(int i);
 
     /**
      * get_num_execs gets the static num_execs reference at i.
@@ -104,7 +104,7 @@ class AmdDevice {
      *
      * @return recursive_mutex reference
      */
-    static std::recursive_mutex &get_mutex(int i);
+    static std::mutex &get_mutex(int i);
 
     /**
      * get_num_execs gets the static num_execs reference at i.
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 8c172bc4ed9..1b9da581f30 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -1302,8 +1302,6 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
         bool device_reset = false,
         allocation_mode alloc_mode = default_cuda_alloc_mode);
 
-    ~CudaExecutor() { decrease_num_execs(this->get_device_id()); }
-
     std::shared_ptr<Executor> get_master() noexcept override;
 
     std::shared_ptr<const Executor> get_master() const noexcept override;
@@ -1457,22 +1455,19 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
     static void increase_num_execs(unsigned device_id)
     {
-        std::lock_guard<std::recursive_mutex> guard(
-            device_class::get_mutex(device_id));
+        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         device_class::get_num_execs(device_id)++;
     }
 
     static void decrease_num_execs(unsigned device_id)
     {
-        std::lock_guard<std::recursive_mutex> guard(
-            device_class::get_mutex(device_id));
+        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         device_class::get_num_execs(device_id)--;
     }
 
     static unsigned get_num_execs(unsigned device_id)
     {
-        std::lock_guard<std::recursive_mutex> guard(
-            device_class::get_mutex(device_id));
+        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         return device_class::get_num_execs(device_id);
     }
 
@@ -1525,8 +1520,6 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
         bool device_reset = false,
         allocation_mode alloc_mode = default_hip_alloc_mode);
 
-    ~HipExecutor() { decrease_num_execs(this->get_device_id()); }
-
     std::shared_ptr<Executor> get_master() noexcept override;
 
     std::shared_ptr<const Executor> get_master() const noexcept override;
@@ -1684,22 +1677,19 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
     static void increase_num_execs(int device_id)
     {
-        std::lock_guard<std::recursive_mutex> guard(
-            device_class::get_mutex(device_id));
+        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         device_class::get_num_execs(device_id)++;
     }
 
     static void decrease_num_execs(int device_id)
     {
-        std::lock_guard<std::recursive_mutex> guard(
-            device_class::get_mutex(device_id));
+        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         device_class::get_num_execs(device_id)--;
     }
 
     static int get_num_execs(int device_id)
     {
-        std::lock_guard<std::recursive_mutex> guard(
-            device_class::get_mutex(device_id));
+        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         return device_class::get_num_execs(device_id);
     }
 

From db4378323ee85def89afe93e336741e1fff024d3 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 30 Jul 2021 11:43:01 +0200
Subject: [PATCH 0946/1008] increase/decrease num_exec when enabling module

---
 devices/device.cpp                    |  3 +++
 include/ginkgo/config.hpp.in          |  8 ++++++++
 include/ginkgo/core/base/executor.hpp | 12 ++++++++++++
 3 files changed, 23 insertions(+)

diff --git a/devices/device.cpp b/devices/device.cpp
index c70fdaaffb8..3080044f675 100644
--- a/devices/device.cpp
+++ b/devices/device.cpp
@@ -46,18 +46,21 @@ std::mutex &NvidiaDevice::get_mutex(int i)
     return mutex[i];
 }
 
+
 int &NvidiaDevice::get_num_execs(int i)
 {
     static int num_execs[max_devices];
     return num_execs[i];
 }
 
+
 std::mutex &AmdDevice::get_mutex(int i)
 {
     static std::mutex mutex[max_devices];
     return mutex[i];
 }
 
+
 int &AmdDevice::get_num_execs(int i)
 {
     static int num_execs[max_devices];
diff --git a/include/ginkgo/config.hpp.in b/include/ginkgo/config.hpp.in
index 1c6a31ea481..c232f481fac 100644
--- a/include/ginkgo/config.hpp.in
+++ b/include/ginkgo/config.hpp.in
@@ -67,6 +67,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #cmakedefine GINKGO_MIXED_PRECISION
 
 
+/* Should we compile cuda kenrels for Ginkgo? */
+#cmakedefine GINKGO_BUILD_CUDA
+
+
+/* Should we compile hip kenrels for Ginkgo? */
+#cmakedefine GINKGO_BUILD_HIP
+
+
 /* What is HIP compiled for, hcc or nvcc? */
 // clang-format off
 #define GINKGO_HIP_PLATFORM_HCC @GINKGO_HIP_PLATFORM_HCC@
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 1b9da581f30..45d42f0b0f1 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -1455,14 +1455,20 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
     static void increase_num_execs(unsigned device_id)
     {
+#ifdef GINKGO_BUILD_CUDA
+        // increase the Cuda Device count only when ginkgo build cuda
         std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         device_class::get_num_execs(device_id)++;
+#endif  // GINKGO_BUILD_CUDA
     }
 
     static void decrease_num_execs(unsigned device_id)
     {
+#ifdef GINKGO_BUILD_CUDA
+        // increase the Cuda Device count only when ginkgo build cuda
         std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         device_class::get_num_execs(device_id)--;
+#endif  // GINKGO_BUILD_CUDA
     }
 
     static unsigned get_num_execs(unsigned device_id)
@@ -1677,14 +1683,20 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
     static void increase_num_execs(int device_id)
     {
+#ifdef GINKGO_BUILD_HIP
+        // increase the HIP Device count only when ginkgo build hip
         std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         device_class::get_num_execs(device_id)++;
+#endif  // GINKGO_BUILD_HIP
     }
 
     static void decrease_num_execs(int device_id)
     {
+#ifdef GINKGO_BUILD_HIP
+        // increase the HIP Device count only when ginkgo build hip
         std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
         device_class::get_num_execs(device_id)--;
+#endif  // GINKGO_BUILD_HIP
     }
 
     static int get_num_execs(int device_id)

From 13bb087ef1ae7cc07f15118cf0eab9e551b89d5e Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Sun, 1 Aug 2021 18:10:30 +0200
Subject: [PATCH 0947/1008] move all related to cmake into cpp not hpp

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 cuda/base/executor.cpp                |  4 +-
 devices/cuda/executor.cpp             | 27 +++++++++++++
 devices/hip/executor.cpp              | 34 ++++++++++++++++
 hip/base/executor.hip.cpp             | 11 +++++-
 include/ginkgo/config.hpp.in          |  4 +-
 include/ginkgo/core/base/device.hpp   |  7 ----
 include/ginkgo/core/base/executor.hpp | 56 +++------------------------
 7 files changed, 80 insertions(+), 63 deletions(-)

diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 2898ff3126c..1f179dc1224 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -66,9 +66,9 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
         [device_id](CudaExecutor *exec) {
             auto device_reset = exec->get_device_reset();
             std::lock_guard<std::mutex> guard(
-                device_class::get_mutex(device_id));
+                NvidiaDevice::get_mutex(device_id));
             delete exec;
-            auto &num_execs = device_class::get_num_execs(device_id);
+            auto &num_execs = NvidiaDevice::get_num_execs(device_id);
             num_execs--;
             if (!num_execs && device_reset) {
                 cuda::device_guard g(device_id);
diff --git a/devices/cuda/executor.cpp b/devices/cuda/executor.cpp
index abdc93a4a08..708fe02dcd6 100644
--- a/devices/cuda/executor.cpp
+++ b/devices/cuda/executor.cpp
@@ -64,4 +64,31 @@ bool CudaExecutor::verify_memory_to(const HipExecutor *dest_exec) const
 }
 
 
+void CudaExecutor::increase_num_execs(unsigned device_id)
+{
+#ifdef GINKGO_BUILD_CUDA
+    // increase the Cuda Device count only when ginkgo build cuda
+    std::lock_guard<std::mutex> guard(NvidiaDevice::get_mutex(device_id));
+    NvidiaDevice::get_num_execs(device_id)++;
+#endif  // GINKGO_BUILD_CUDA
+}
+
+
+void CudaExecutor::decrease_num_execs(unsigned device_id)
+{
+#ifdef GINKGO_BUILD_CUDA
+    // increase the Cuda Device count only when ginkgo build cuda
+    std::lock_guard<std::mutex> guard(NvidiaDevice::get_mutex(device_id));
+    NvidiaDevice::get_num_execs(device_id)--;
+#endif  // GINKGO_BUILD_CUDA
+}
+
+
+unsigned CudaExecutor::get_num_execs(unsigned device_id)
+{
+    std::lock_guard<std::mutex> guard(NvidiaDevice::get_mutex(device_id));
+    return NvidiaDevice::get_num_execs(device_id);
+}
+
+
 }  // namespace gko
diff --git a/devices/hip/executor.cpp b/devices/hip/executor.cpp
index 0e755209f91..daf8c14ec58 100644
--- a/devices/hip/executor.cpp
+++ b/devices/hip/executor.cpp
@@ -61,4 +61,38 @@ bool HipExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
 }
 
 
+#if (GINKGO_HIP_PLATFORM_NVCC == 1)
+using hip_device_class = NvidiaDevice;
+#else
+using hip_device_class = AmdDevice;
+#endif
+
+
+void HipExecutor::increase_num_execs(int device_id)
+{
+#ifdef GINKGO_BUILD_HIP
+    // increase the HIP Device count only when ginkgo build hip
+    std::lock_guard<std::mutex> guard(hip_device_class::get_mutex(device_id));
+    hip_device_class::get_num_execs(device_id)++;
+#endif  // GINKGO_BUILD_HIP
+}
+
+
+void HipExecutor::decrease_num_execs(int device_id)
+{
+#ifdef GINKGO_BUILD_HIP
+    // increase the HIP Device count only when ginkgo build hip
+    std::lock_guard<std::mutex> guard(hip_device_class::get_mutex(device_id));
+    hip_device_class::get_num_execs(device_id)--;
+#endif  // GINKGO_BUILD_HIP
+}
+
+
+int HipExecutor::get_num_execs(int device_id)
+{
+    std::lock_guard<std::mutex> guard(hip_device_class::get_mutex(device_id));
+    return hip_device_class::get_num_execs(device_id);
+}
+
+
 }  // namespace gko
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index c5b1c32920b..11ea4ab8ead 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -56,6 +56,13 @@ namespace gko {
 #include "common/base/executor.hpp.inc"
 
 
+#if (GINKGO_HIP_PLATFORM_NVCC == 1)
+using hip_device_class = NvidiaDevice;
+#else
+using hip_device_class = AmdDevice;
+#endif
+
+
 std::shared_ptr<HipExecutor> HipExecutor::create(
     int device_id, std::shared_ptr<Executor> master, bool device_reset,
     allocation_mode alloc_mode)
@@ -65,9 +72,9 @@ std::shared_ptr<HipExecutor> HipExecutor::create(
         [device_id](HipExecutor *exec) {
             auto device_reset = exec->get_device_reset();
             std::lock_guard<std::mutex> guard(
-                device_class::get_mutex(device_id));
+                hip_device_class::get_mutex(device_id));
             delete exec;
-            auto &num_execs = device_class::get_num_execs(device_id);
+            auto &num_execs = hip_device_class::get_num_execs(device_id);
             num_execs--;
             if (!num_execs && device_reset) {
                 hip::device_guard g(device_id);
diff --git a/include/ginkgo/config.hpp.in b/include/ginkgo/config.hpp.in
index c232f481fac..42f2c56f44d 100644
--- a/include/ginkgo/config.hpp.in
+++ b/include/ginkgo/config.hpp.in
@@ -67,11 +67,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #cmakedefine GINKGO_MIXED_PRECISION
 
 
-/* Should we compile cuda kenrels for Ginkgo? */
+/* Should we compile cuda kernels for Ginkgo? */
 #cmakedefine GINKGO_BUILD_CUDA
 
 
-/* Should we compile hip kenrels for Ginkgo? */
+/* Should we compile hip kernels for Ginkgo? */
 #cmakedefine GINKGO_BUILD_HIP
 
 
diff --git a/include/ginkgo/core/base/device.hpp b/include/ginkgo/core/base/device.hpp
index d40ad7f6eef..981191ba73c 100644
--- a/include/ginkgo/core/base/device.hpp
+++ b/include/ginkgo/core/base/device.hpp
@@ -57,10 +57,7 @@ class HipExecutor;
  */
 class NvidiaDevice {
     friend class CudaExecutor;
-// If Hip is compiled for NVCC, give NvidiaDevice's permission to HipExecutor
-#if GINKGO_HIP_PLATFORM_NVCC
     friend class HipExecutor;
-#endif
 
 private:
     /**
@@ -90,11 +87,7 @@ class NvidiaDevice {
  * corresponding recursive_mutex.
  */
 class AmdDevice {
-// to avoid both GINKGO_HIP_PLATFORM_* zero
-// If Hip isn't compiled for NVCC, give AmdDevice's permission to HipExecutor
-#if !GINKGO_HIP_PLATFORM_NVCC
     friend class HipExecutor;
-#endif
 
 private:
     /**
diff --git a/include/ginkgo/core/base/executor.hpp b/include/ginkgo/core/base/executor.hpp
index 45d42f0b0f1..59876d620cc 100644
--- a/include/ginkgo/core/base/executor.hpp
+++ b/include/ginkgo/core/base/executor.hpp
@@ -1407,8 +1407,6 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
     int get_closest_numa() const { return this->get_exec_info().numa_node; }
 
 protected:
-    using device_class = NvidiaDevice;
-
     void set_gpu_property();
 
     void init_handles();
@@ -1453,29 +1451,11 @@ class CudaExecutor : public detail::ExecutorBase<CudaExecutor>,
 
     bool verify_memory_to(const CudaExecutor *dest_exec) const override;
 
-    static void increase_num_execs(unsigned device_id)
-    {
-#ifdef GINKGO_BUILD_CUDA
-        // increase the Cuda Device count only when ginkgo build cuda
-        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
-        device_class::get_num_execs(device_id)++;
-#endif  // GINKGO_BUILD_CUDA
-    }
+    static void increase_num_execs(unsigned device_id);
 
-    static void decrease_num_execs(unsigned device_id)
-    {
-#ifdef GINKGO_BUILD_CUDA
-        // increase the Cuda Device count only when ginkgo build cuda
-        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
-        device_class::get_num_execs(device_id)--;
-#endif  // GINKGO_BUILD_CUDA
-    }
+    static void decrease_num_execs(unsigned device_id);
 
-    static unsigned get_num_execs(unsigned device_id)
-    {
-        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
-        return device_class::get_num_execs(device_id);
-    }
+    static unsigned get_num_execs(unsigned device_id);
 
     void populate_exec_info(const MachineTopology *mach_topo) override;
 
@@ -1631,12 +1611,6 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
     }
 
 protected:
-#if (GINKGO_HIP_PLATFORM_NVCC == 1)
-    using device_class = NvidiaDevice;
-#else
-    using device_class = AmdDevice;
-#endif
-
     void set_gpu_property();
 
     void init_handles();
@@ -1681,29 +1655,11 @@ class HipExecutor : public detail::ExecutorBase<HipExecutor>,
 
     bool verify_memory_to(const HipExecutor *dest_exec) const override;
 
-    static void increase_num_execs(int device_id)
-    {
-#ifdef GINKGO_BUILD_HIP
-        // increase the HIP Device count only when ginkgo build hip
-        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
-        device_class::get_num_execs(device_id)++;
-#endif  // GINKGO_BUILD_HIP
-    }
+    static void increase_num_execs(int device_id);
 
-    static void decrease_num_execs(int device_id)
-    {
-#ifdef GINKGO_BUILD_HIP
-        // increase the HIP Device count only when ginkgo build hip
-        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
-        device_class::get_num_execs(device_id)--;
-#endif  // GINKGO_BUILD_HIP
-    }
+    static void decrease_num_execs(int device_id);
 
-    static int get_num_execs(int device_id)
-    {
-        std::lock_guard<std::mutex> guard(device_class::get_mutex(device_id));
-        return device_class::get_num_execs(device_id);
-    }
+    static int get_num_execs(int device_id);
 
     void populate_exec_info(const MachineTopology *mach_topo) override;
 

From b58e746d3f0fae93969049f167808d4e0bb456d8 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 2 Aug 2021 14:19:19 +0200
Subject: [PATCH 0948/1008] use snake_case and move the compile_definitions

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 cuda/base/executor.cpp              |  4 ++--
 devices/cuda/CMakeLists.txt         |  3 +++
 devices/cuda/executor.cpp           | 20 ++++++++++----------
 devices/device.cpp                  |  8 ++++----
 devices/hip/CMakeLists.txt          |  3 +++
 devices/hip/executor.cpp            | 12 ++++++------
 hip/base/executor.hip.cpp           |  4 ++--
 include/ginkgo/config.hpp.in        |  8 --------
 include/ginkgo/core/base/device.hpp |  8 ++++----
 9 files changed, 34 insertions(+), 36 deletions(-)

diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 1f179dc1224..1c35af4d390 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -66,9 +66,9 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
         [device_id](CudaExecutor *exec) {
             auto device_reset = exec->get_device_reset();
             std::lock_guard<std::mutex> guard(
-                NvidiaDevice::get_mutex(device_id));
+                nvidia_device::get_mutex(device_id));
             delete exec;
-            auto &num_execs = NvidiaDevice::get_num_execs(device_id);
+            auto &num_execs = nvidia_device::get_num_execs(device_id);
             num_execs--;
             if (!num_execs && device_reset) {
                 cuda::device_guard g(device_id);
diff --git a/devices/cuda/CMakeLists.txt b/devices/cuda/CMakeLists.txt
index acd0455ff2a..b9d1bca187b 100644
--- a/devices/cuda/CMakeLists.txt
+++ b/devices/cuda/CMakeLists.txt
@@ -1,2 +1,5 @@
 ginkgo_add_object_library(ginkgo_cuda_device
     executor.cpp)
+if(GINKGO_BUILD_CUDA)
+    target_compile_definitions(ginkgo_cuda_device PRIVATE GKO_COMPILING_CUDA_DEVICE)
+endif()
diff --git a/devices/cuda/executor.cpp b/devices/cuda/executor.cpp
index 708fe02dcd6..06e8d504097 100644
--- a/devices/cuda/executor.cpp
+++ b/devices/cuda/executor.cpp
@@ -66,28 +66,28 @@ bool CudaExecutor::verify_memory_to(const HipExecutor *dest_exec) const
 
 void CudaExecutor::increase_num_execs(unsigned device_id)
 {
-#ifdef GINKGO_BUILD_CUDA
+#ifdef GKO_COMPILING_CUDA_DEVICE
     // increase the Cuda Device count only when ginkgo build cuda
-    std::lock_guard<std::mutex> guard(NvidiaDevice::get_mutex(device_id));
-    NvidiaDevice::get_num_execs(device_id)++;
-#endif  // GINKGO_BUILD_CUDA
+    std::lock_guard<std::mutex> guard(nvidia_device::get_mutex(device_id));
+    nvidia_device::get_num_execs(device_id)++;
+#endif  // GKO_COMPILING_CUDA_DEVICE
 }
 
 
 void CudaExecutor::decrease_num_execs(unsigned device_id)
 {
-#ifdef GINKGO_BUILD_CUDA
+#ifdef GKO_COMPILING_CUDA_DEVICE
     // increase the Cuda Device count only when ginkgo build cuda
-    std::lock_guard<std::mutex> guard(NvidiaDevice::get_mutex(device_id));
-    NvidiaDevice::get_num_execs(device_id)--;
-#endif  // GINKGO_BUILD_CUDA
+    std::lock_guard<std::mutex> guard(nvidia_device::get_mutex(device_id));
+    nvidia_device::get_num_execs(device_id)--;
+#endif  // GKO_COMPILING_CUDA_DEVICE
 }
 
 
 unsigned CudaExecutor::get_num_execs(unsigned device_id)
 {
-    std::lock_guard<std::mutex> guard(NvidiaDevice::get_mutex(device_id));
-    return NvidiaDevice::get_num_execs(device_id);
+    std::lock_guard<std::mutex> guard(nvidia_device::get_mutex(device_id));
+    return nvidia_device::get_num_execs(device_id);
 }
 
 
diff --git a/devices/device.cpp b/devices/device.cpp
index 3080044f675..ca6597d652d 100644
--- a/devices/device.cpp
+++ b/devices/device.cpp
@@ -40,28 +40,28 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace gko {
 
 
-std::mutex &NvidiaDevice::get_mutex(int i)
+std::mutex &nvidia_device::get_mutex(int i)
 {
     static std::mutex mutex[max_devices];
     return mutex[i];
 }
 
 
-int &NvidiaDevice::get_num_execs(int i)
+int &nvidia_device::get_num_execs(int i)
 {
     static int num_execs[max_devices];
     return num_execs[i];
 }
 
 
-std::mutex &AmdDevice::get_mutex(int i)
+std::mutex &amd_device::get_mutex(int i)
 {
     static std::mutex mutex[max_devices];
     return mutex[i];
 }
 
 
-int &AmdDevice::get_num_execs(int i)
+int &amd_device::get_num_execs(int i)
 {
     static int num_execs[max_devices];
     return num_execs[i];
diff --git a/devices/hip/CMakeLists.txt b/devices/hip/CMakeLists.txt
index 9995e51c5e7..d05db433984 100644
--- a/devices/hip/CMakeLists.txt
+++ b/devices/hip/CMakeLists.txt
@@ -1,2 +1,5 @@
 ginkgo_add_object_library(ginkgo_hip_device
     executor.cpp)
+if(GINKGO_BUILD_HIP)
+    target_compile_definitions(ginkgo_hip_device PRIVATE GKO_COMPILING_HIP_DEVICE)
+endif()
diff --git a/devices/hip/executor.cpp b/devices/hip/executor.cpp
index daf8c14ec58..53da0db72c2 100644
--- a/devices/hip/executor.cpp
+++ b/devices/hip/executor.cpp
@@ -62,29 +62,29 @@ bool HipExecutor::verify_memory_to(const CudaExecutor *dest_exec) const
 
 
 #if (GINKGO_HIP_PLATFORM_NVCC == 1)
-using hip_device_class = NvidiaDevice;
+using hip_device_class = nvidia_device;
 #else
-using hip_device_class = AmdDevice;
+using hip_device_class = amd_device;
 #endif
 
 
 void HipExecutor::increase_num_execs(int device_id)
 {
-#ifdef GINKGO_BUILD_HIP
+#ifdef GKO_COMPILING_HIP_DEVICE
     // increase the HIP Device count only when ginkgo build hip
     std::lock_guard<std::mutex> guard(hip_device_class::get_mutex(device_id));
     hip_device_class::get_num_execs(device_id)++;
-#endif  // GINKGO_BUILD_HIP
+#endif  // GKO_COMPILING_HIP_DEVICE
 }
 
 
 void HipExecutor::decrease_num_execs(int device_id)
 {
-#ifdef GINKGO_BUILD_HIP
+#ifdef GKO_COMPILING_HIP_DEVICE
     // increase the HIP Device count only when ginkgo build hip
     std::lock_guard<std::mutex> guard(hip_device_class::get_mutex(device_id));
     hip_device_class::get_num_execs(device_id)--;
-#endif  // GINKGO_BUILD_HIP
+#endif  // GKO_COMPILING_HIP_DEVICE
 }
 
 
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 11ea4ab8ead..3875ad95910 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -57,9 +57,9 @@ namespace gko {
 
 
 #if (GINKGO_HIP_PLATFORM_NVCC == 1)
-using hip_device_class = NvidiaDevice;
+using hip_device_class = nvidia_device;
 #else
-using hip_device_class = AmdDevice;
+using hip_device_class = amd_device;
 #endif
 
 
diff --git a/include/ginkgo/config.hpp.in b/include/ginkgo/config.hpp.in
index 42f2c56f44d..1c6a31ea481 100644
--- a/include/ginkgo/config.hpp.in
+++ b/include/ginkgo/config.hpp.in
@@ -67,14 +67,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #cmakedefine GINKGO_MIXED_PRECISION
 
 
-/* Should we compile cuda kernels for Ginkgo? */
-#cmakedefine GINKGO_BUILD_CUDA
-
-
-/* Should we compile hip kernels for Ginkgo? */
-#cmakedefine GINKGO_BUILD_HIP
-
-
 /* What is HIP compiled for, hcc or nvcc? */
 // clang-format off
 #define GINKGO_HIP_PLATFORM_HCC @GINKGO_HIP_PLATFORM_HCC@
diff --git a/include/ginkgo/core/base/device.hpp b/include/ginkgo/core/base/device.hpp
index 981191ba73c..5eaf892789d 100644
--- a/include/ginkgo/core/base/device.hpp
+++ b/include/ginkgo/core/base/device.hpp
@@ -52,10 +52,10 @@ class HipExecutor;
 
 
 /**
- * NvidiaDevice handles the number of executor on Nvidia devices and have the
+ * nvidia_device handles the number of executor on Nvidia devices and have the
  * corresponding recursive_mutex.
  */
-class NvidiaDevice {
+class nvidia_device {
     friend class CudaExecutor;
     friend class HipExecutor;
 
@@ -83,10 +83,10 @@ class NvidiaDevice {
 
 
 /**
- * AmdDevice handles the number of executor on Amd devices and have the
+ * amd_device handles the number of executor on Amd devices and have the
  * corresponding recursive_mutex.
  */
-class AmdDevice {
+class amd_device {
     friend class HipExecutor;
 
 private:

From 7db5c1caf36a353cf47eb11ed7508611a93efe05 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 14 Jul 2021 14:29:53 +0200
Subject: [PATCH 0949/1008] scalar_jacobi on diff precision csr

---
 reference/test/preconditioner/jacobi.cpp | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/reference/test/preconditioner/jacobi.cpp b/reference/test/preconditioner/jacobi.cpp
index 1dfc17c5569..6b34dd2afde 100644
--- a/reference/test/preconditioner/jacobi.cpp
+++ b/reference/test/preconditioner/jacobi.cpp
@@ -65,6 +65,7 @@ class Jacobi : public ::testing::Test {
     Jacobi()
         : exec(gko::ReferenceExecutor::create()),
           bj_factory(Bj::build().with_max_block_size(3u).on(exec)),
+          scalar_j_factory(Bj::build().with_max_block_size(1u).on(exec)),
           block_pointers(exec, 3),
           block_precisions(exec, 2),
           mtx(Mtx::create(exec, gko::dim<2>{5}, 13))
@@ -168,6 +169,7 @@ class Jacobi : public ::testing::Test {
 
     std::shared_ptr<const gko::Executor> exec;
     std::unique_ptr<typename Bj::Factory> bj_factory;
+    std::unique_ptr<typename Bj::Factory> scalar_j_factory;
     std::unique_ptr<typename Bj::Factory> adaptive_bj_factory;
     gko::Array<index_type> block_pointers;
     gko::Array<gko::precision_reduction> block_precisions;
@@ -378,4 +380,17 @@ TYPED_TEST(Jacobi, GeneratesCorrectMatrixDataWithAdaptivePrecision)
 }
 
 
+TYPED_TEST(Jacobi, ScalarJacobiGeneratesOnDifferentPrecision)
+{
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using next_type = gko::next_precision<value_type>;
+    auto csr =
+        gko::share(gko::matrix::Csr<next_type, index_type>::create(this->exec));
+    csr->copy_from(gko::lend(this->mtx));
+
+    ASSERT_NO_THROW(this->scalar_j_factory->generate(csr));
+}
+
+
 }  // namespace

From fc1c35feae0babfda3a7a5d316953952df93ffe8 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 14 Jul 2021 15:09:44 +0200
Subject: [PATCH 0950/1008] scalarjacobi allows diff system matrix and add
 DiagonalLinOpExtractable

---
 core/preconditioner/jacobi.cpp          | 22 ++++++++++++++++-----
 include/ginkgo/core/base/lin_op.hpp     | 26 ++++++++++++++++++++++++-
 include/ginkgo/core/matrix/diagonal.hpp | 13 +++++++++++++
 3 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index 8270265c2ec..cc60dcc4b76 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -231,11 +231,23 @@ void Jacobi<ValueType, IndexType>::generate(const LinOp *system_matrix,
     using csr_type = matrix::Csr<ValueType, IndexType>;
     const auto exec = this->get_executor();
     if (parameters_.max_block_size == 1) {
-        auto diag = as<DiagonalExtractable<ValueType>>(system_matrix)
-                        ->extract_diagonal();
-        auto temp = gko::Array<ValueType>::view(
-            exec, system_matrix->get_size()[0], diag->get_values());
-        this->blocks_ = temp;
+        auto diag = share(as<DiagonalLinOpExtractable>(system_matrix)
+                              ->extract_diagonal_linop());
+        using next_type = next_precision<ValueType>;
+        if (auto diag_dense =
+                std::dynamic_pointer_cast<matrix::Diagonal<ValueType>>(diag)) {
+            auto temp = gko::Array<ValueType>::view(
+                exec, system_matrix->get_size()[0], diag_dense->get_values());
+            this->blocks_ = temp;
+        } else if (auto diag_dense =
+                       std::dynamic_pointer_cast<matrix::Diagonal<next_type>>(
+                           diag)) {
+            auto temp = gko::Array<next_type>::view(
+                exec, system_matrix->get_size()[0], diag_dense->get_values());
+            this->blocks_ = temp;
+        } else {
+            GKO_NOT_SUPPORTED(system_matrix);
+        }
     } else {
         auto csr_mtx = convert_to_with_sorting<csr_type>(exec, system_matrix,
                                                          skip_sorting);
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index cde3840c30f..fd43bd85136 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -654,6 +654,28 @@ class Preconditionable {
 };
 
 
+/**
+ * The diagonal of a LinOp can be extracted. It will be implemented by
+ * DiagonalExtractable<ValueType>, so the class does not need to implement it.
+ * extract_diagonal_linop returns a linop which extracts the elements whose col
+ * and row index are the same and stores the result in a min(nrows, ncols) x 1
+ * dense matrix.
+ *
+ * @ingroup LinOp
+ */
+class DiagonalLinOpExtractable {
+public:
+    virtual ~DiagonalLinOpExtractable() = default;
+
+    /**
+     * Extracts the diagonal entries of the matrix into a vector.
+     *
+     * @return linop  the linop of diagonal format
+     */
+    virtual std::unique_ptr<LinOp> extract_diagonal_linop() const = 0;
+};
+
+
 /**
  * The diagonal of a LinOp implementing this interface can be extracted.
  * extract_diagonal extracts the elements whose col and row index are the
@@ -662,12 +684,14 @@ class Preconditionable {
  * @ingroup LinOp
  */
 template <typename ValueType>
-class DiagonalExtractable {
+class DiagonalExtractable : public DiagonalLinOpExtractable {
 public:
     using value_type = ValueType;
 
     virtual ~DiagonalExtractable() = default;
 
+    std::unique_ptr<LinOp> extract_diagonal_linop() const override;
+
     /**
      * Extracts the diagonal entries of the matrix into a vector.
      *
diff --git a/include/ginkgo/core/matrix/diagonal.hpp b/include/ginkgo/core/matrix/diagonal.hpp
index ddd4aa73bf0..090ef9a214c 100644
--- a/include/ginkgo/core/matrix/diagonal.hpp
+++ b/include/ginkgo/core/matrix/diagonal.hpp
@@ -209,6 +209,19 @@ class Diagonal
 
 
 }  // namespace matrix
+
+
+template <typename ValueType>
+std::unique_ptr<LinOp> DiagonalExtractable<ValueType>::extract_diagonal_linop()
+    const
+{
+    auto diag = this->extract_diagonal();
+    auto p = dynamic_cast<LinOp *>(diag.get());
+    diag.release();
+    return std::unique_ptr<LinOp>{p};
+}
+
+
 }  // namespace gko
 
 

From b831d603bdc99eb6b7cce8d56a633fed74bc19b9 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 16 Jul 2021 16:25:36 +0200
Subject: [PATCH 0951/1008] move diagonalextrable impl with instantiate

---
 core/matrix/diagonal.cpp                | 20 ++++++++++++++++++++
 include/ginkgo/core/matrix/diagonal.hpp | 13 -------------
 2 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/core/matrix/diagonal.cpp b/core/matrix/diagonal.cpp
index dab0b3fa35a..39bea38e3af 100644
--- a/core/matrix/diagonal.cpp
+++ b/core/matrix/diagonal.cpp
@@ -302,4 +302,24 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_MATRIX);
 
 
 }  // namespace matrix
+
+
+// Implement DiagonalExtractable for LinOp when Diagonal is complete class
+template <typename ValueType>
+std::unique_ptr<LinOp> DiagonalExtractable<ValueType>::extract_diagonal_linop()
+    const
+{
+    auto diag = this->extract_diagonal();
+    auto p = dynamic_cast<LinOp *>(diag.get());
+    diag.release();
+    return std::unique_ptr<LinOp>{p};
+}
+
+
+#define GKO_DECLARE_DIAGONAL_EXTRACTABLE(value_type) \
+    std::unique_ptr<LinOp>                           \
+    DiagonalExtractable<value_type>::extract_diagonal_linop() const
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_EXTRACTABLE);
+
+
 }  // namespace gko
diff --git a/include/ginkgo/core/matrix/diagonal.hpp b/include/ginkgo/core/matrix/diagonal.hpp
index 090ef9a214c..ddd4aa73bf0 100644
--- a/include/ginkgo/core/matrix/diagonal.hpp
+++ b/include/ginkgo/core/matrix/diagonal.hpp
@@ -209,19 +209,6 @@ class Diagonal
 
 
 }  // namespace matrix
-
-
-template <typename ValueType>
-std::unique_ptr<LinOp> DiagonalExtractable<ValueType>::extract_diagonal_linop()
-    const
-{
-    auto diag = this->extract_diagonal();
-    auto p = dynamic_cast<LinOp *>(diag.get());
-    diag.release();
-    return std::unique_ptr<LinOp>{p};
-}
-
-
 }  // namespace gko
 
 

From d0cd30620f0c5bf6dfc9e5adcb6e95748cc735e9 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 19 Jul 2021 14:17:15 +0200
Subject: [PATCH 0952/1008] fix executor issue

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/preconditioner/jacobi.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index cc60dcc4b76..51772a60df4 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -237,13 +237,15 @@ void Jacobi<ValueType, IndexType>::generate(const LinOp *system_matrix,
         if (auto diag_dense =
                 std::dynamic_pointer_cast<matrix::Diagonal<ValueType>>(diag)) {
             auto temp = gko::Array<ValueType>::view(
-                exec, system_matrix->get_size()[0], diag_dense->get_values());
+                diag_dense->get_executor(), system_matrix->get_size()[0],
+                diag_dense->get_values());
             this->blocks_ = temp;
         } else if (auto diag_dense =
                        std::dynamic_pointer_cast<matrix::Diagonal<next_type>>(
                            diag)) {
             auto temp = gko::Array<next_type>::view(
-                exec, system_matrix->get_size()[0], diag_dense->get_values());
+                diag_dense->get_executor(), system_matrix->get_size()[0],
+                diag_dense->get_values());
             this->blocks_ = temp;
         } else {
             GKO_NOT_SUPPORTED(system_matrix);

From aad5bdf0ec0e5d5a39cf24b1231ef08aa7c40de1 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 2 Aug 2021 17:21:46 +0200
Subject: [PATCH 0953/1008] use template_conversion, add diagonal precision
 convert

Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/matrix/diagonal.cpp                   | 16 +++++++++
 core/preconditioner/jacobi.cpp             | 26 ++++++--------
 include/ginkgo/core/matrix/diagonal.hpp    |  7 ++++
 reference/test/matrix/diagonal_kernels.cpp | 40 ++++++++++++++++++++++
 reference/test/preconditioner/jacobi.cpp   | 12 +++++--
 5 files changed, 84 insertions(+), 17 deletions(-)

diff --git a/core/matrix/diagonal.cpp b/core/matrix/diagonal.cpp
index 39bea38e3af..8b2800a4ee8 100644
--- a/core/matrix/diagonal.cpp
+++ b/core/matrix/diagonal.cpp
@@ -146,6 +146,22 @@ std::unique_ptr<LinOp> Diagonal<ValueType>::conj_transpose() const
 }
 
 
+template <typename ValueType>
+void Diagonal<ValueType>::convert_to(
+    Diagonal<next_precision<ValueType>> *result) const
+{
+    result->values_ = this->values_;
+    result->set_size(this->get_size());
+}
+
+
+template <typename ValueType>
+void Diagonal<ValueType>::move_to(Diagonal<next_precision<ValueType>> *result)
+{
+    this->convert_to(result);
+}
+
+
 template <typename ValueType>
 void Diagonal<ValueType>::convert_to(Csr<ValueType, int32> *result) const
 {
diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index 51772a60df4..66809d616ae 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -40,6 +40,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
 #include <ginkgo/core/base/precision_dispatch.hpp>
+#include <ginkgo/core/base/temporary_conversion.hpp>
 #include <ginkgo/core/base/utils.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
@@ -233,23 +234,18 @@ void Jacobi<ValueType, IndexType>::generate(const LinOp *system_matrix,
     if (parameters_.max_block_size == 1) {
         auto diag = share(as<DiagonalLinOpExtractable>(system_matrix)
                               ->extract_diagonal_linop());
-        using next_type = next_precision<ValueType>;
-        if (auto diag_dense =
-                std::dynamic_pointer_cast<matrix::Diagonal<ValueType>>(diag)) {
-            auto temp = gko::Array<ValueType>::view(
-                diag_dense->get_executor(), system_matrix->get_size()[0],
-                diag_dense->get_values());
-            this->blocks_ = temp;
-        } else if (auto diag_dense =
-                       std::dynamic_pointer_cast<matrix::Diagonal<next_type>>(
-                           diag)) {
-            auto temp = gko::Array<next_type>::view(
-                diag_dense->get_executor(), system_matrix->get_size()[0],
-                diag_dense->get_values());
-            this->blocks_ = temp;
-        } else {
+        auto diag_vt =
+            ::gko::detail::temporary_conversion<matrix::Diagonal<ValueType>>::
+                template create<matrix::Diagonal<next_precision<ValueType>>>(
+                    diag.get());
+        if (!diag_vt) {
             GKO_NOT_SUPPORTED(system_matrix);
         }
+        auto temp = gko::Array<ValueType>::view(diag_vt->get_executor(),
+                                                diag_vt->get_size()[0],
+                                                diag_vt->get_values());
+        this->blocks_ = temp;
+        this->num_blocks_ = diag_vt->get_size()[0];
     } else {
         auto csr_mtx = convert_to_with_sorting<csr_type>(exec, system_matrix,
                                                          skip_sorting);
diff --git a/include/ginkgo/core/matrix/diagonal.hpp b/include/ginkgo/core/matrix/diagonal.hpp
index ddd4aa73bf0..4215a08cad6 100644
--- a/include/ginkgo/core/matrix/diagonal.hpp
+++ b/include/ginkgo/core/matrix/diagonal.hpp
@@ -70,6 +70,7 @@ class Diagonal
       public EnableCreateMethod<Diagonal<ValueType>>,
       public ConvertibleTo<Csr<ValueType, int32>>,
       public ConvertibleTo<Csr<ValueType, int64>>,
+      public ConvertibleTo<Diagonal<next_precision<ValueType>>>,
       public Transposable,
       public WritableToMatrixData<ValueType, int32>,
       public WritableToMatrixData<ValueType, int64>,
@@ -92,10 +93,16 @@ class Diagonal
     using mat_data32 = gko::matrix_data<ValueType, int32>;
     using absolute_type = remove_complex<Diagonal>;
 
+    friend class Diagonal<next_precision<ValueType>>;
+
     std::unique_ptr<LinOp> transpose() const override;
 
     std::unique_ptr<LinOp> conj_transpose() const override;
 
+    void convert_to(Diagonal<next_precision<ValueType>> *result) const override;
+
+    void move_to(Diagonal<next_precision<ValueType>> *result) override;
+
     void convert_to(Csr<ValueType, int32> *result) const override;
 
     void move_to(Csr<ValueType, int32> *result) override;
diff --git a/reference/test/matrix/diagonal_kernels.cpp b/reference/test/matrix/diagonal_kernels.cpp
index e772f07ebc2..73c86376a68 100644
--- a/reference/test/matrix/diagonal_kernels.cpp
+++ b/reference/test/matrix/diagonal_kernels.cpp
@@ -108,6 +108,46 @@ class Diagonal : public ::testing::Test {
 TYPED_TEST_SUITE(Diagonal, gko::test::ValueTypes);
 
 
+TYPED_TEST(Diagonal, ConvertsToPrecision)
+{
+    using ValueType = typename TestFixture::value_type;
+    using OtherType = typename gko::next_precision<ValueType>;
+    using Diagonal = typename TestFixture::Diag;
+    using OtherDiagonal = gko::matrix::Diagonal<OtherType>;
+    auto tmp = OtherDiagonal::create(this->exec);
+    auto res = Diagonal::create(this->exec);
+    // If OtherType is more precise: 0, otherwise r
+    auto residual = r<OtherType>::value < r<ValueType>::value
+                        ? gko::remove_complex<ValueType>{0}
+                        : gko::remove_complex<ValueType>{r<OtherType>::value};
+
+    this->diag1->convert_to(tmp.get());
+    tmp->convert_to(res.get());
+
+    GKO_ASSERT_MTX_NEAR(this->diag1, res, residual);
+}
+
+
+TYPED_TEST(Diagonal, MovesToPrecision)
+{
+    using ValueType = typename TestFixture::value_type;
+    using OtherType = typename gko::next_precision<ValueType>;
+    using Diagonal = typename TestFixture::Diag;
+    using OtherDiagonal = gko::matrix::Diagonal<OtherType>;
+    auto tmp = OtherDiagonal::create(this->exec);
+    auto res = Diagonal::create(this->exec);
+    // If OtherType is more precise: 0, otherwise r
+    auto residual = r<OtherType>::value < r<ValueType>::value
+                        ? gko::remove_complex<ValueType>{0}
+                        : gko::remove_complex<ValueType>{r<OtherType>::value};
+
+    this->diag1->move_to(tmp.get());
+    tmp->move_to(res.get());
+
+    GKO_ASSERT_MTX_NEAR(this->diag1, res, residual);
+}
+
+
 TYPED_TEST(Diagonal, AppliesToDense)
 {
     using value_type = typename TestFixture::value_type;
diff --git a/reference/test/preconditioner/jacobi.cpp b/reference/test/preconditioner/jacobi.cpp
index 6b34dd2afde..8f79e8976fb 100644
--- a/reference/test/preconditioner/jacobi.cpp
+++ b/reference/test/preconditioner/jacobi.cpp
@@ -385,11 +385,19 @@ TYPED_TEST(Jacobi, ScalarJacobiGeneratesOnDifferentPrecision)
     using value_type = typename TestFixture::value_type;
     using index_type = typename TestFixture::index_type;
     using next_type = gko::next_precision<value_type>;
+    using Bj = typename TestFixture::Bj;
     auto csr =
         gko::share(gko::matrix::Csr<next_type, index_type>::create(this->exec));
     csr->copy_from(gko::lend(this->mtx));
-
-    ASSERT_NO_THROW(this->scalar_j_factory->generate(csr));
+    std::shared_ptr<Bj> bj{};
+
+    ASSERT_NO_THROW(bj = this->scalar_j_factory->generate(csr));
+    ASSERT_EQ(bj->get_num_blocks(), 5u);
+    ASSERT_EQ(bj->get_blocks()[0], value_type{4});
+    ASSERT_EQ(bj->get_blocks()[1], value_type{4});
+    ASSERT_EQ(bj->get_blocks()[2], value_type{4});
+    ASSERT_EQ(bj->get_blocks()[3], value_type{4});
+    ASSERT_EQ(bj->get_blocks()[4], value_type{4});
 }
 
 

From cd7555947db13ac248b7131685d3dd438bc585f2 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 3 Aug 2021 12:11:32 +0200
Subject: [PATCH 0954/1008] review update

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 core/matrix/diagonal.cpp            | 5 +----
 core/preconditioner/jacobi.cpp      | 6 +++---
 include/ginkgo/core/base/lin_op.hpp | 1 +
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/core/matrix/diagonal.cpp b/core/matrix/diagonal.cpp
index 8b2800a4ee8..8a6d36a8759 100644
--- a/core/matrix/diagonal.cpp
+++ b/core/matrix/diagonal.cpp
@@ -325,10 +325,7 @@ template <typename ValueType>
 std::unique_ptr<LinOp> DiagonalExtractable<ValueType>::extract_diagonal_linop()
     const
 {
-    auto diag = this->extract_diagonal();
-    auto p = dynamic_cast<LinOp *>(diag.get());
-    diag.release();
-    return std::unique_ptr<LinOp>{p};
+    return this->extract_diagonal();
 }
 
 
diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index 66809d616ae..da830f011a7 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -241,9 +241,9 @@ void Jacobi<ValueType, IndexType>::generate(const LinOp *system_matrix,
         if (!diag_vt) {
             GKO_NOT_SUPPORTED(system_matrix);
         }
-        auto temp = gko::Array<ValueType>::view(diag_vt->get_executor(),
-                                                diag_vt->get_size()[0],
-                                                diag_vt->get_values());
+        auto temp = Array<ValueType>::view(diag_vt->get_executor(),
+                                           diag_vt->get_size()[0],
+                                           diag_vt->get_values());
         this->blocks_ = temp;
         this->num_blocks_ = diag_vt->get_size()[0];
     } else {
diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp
index fd43bd85136..50153c78ecb 100644
--- a/include/ginkgo/core/base/lin_op.hpp
+++ b/include/ginkgo/core/base/lin_op.hpp
@@ -661,6 +661,7 @@ class Preconditionable {
  * and row index are the same and stores the result in a min(nrows, ncols) x 1
  * dense matrix.
  *
+ * @ingroup diagonal
  * @ingroup LinOp
  */
 class DiagonalLinOpExtractable {

From f50226837330641e1badc2aab20ec1b41323bbf5 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 3 Aug 2021 23:45:11 +0200
Subject: [PATCH 0955/1008] use the baseline directly

---
 reference/test/stop/residual_norm_kernels.cpp | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/reference/test/stop/residual_norm_kernels.cpp b/reference/test/stop/residual_norm_kernels.cpp
index 78721d0f1ae..2bc5d15aaad 100644
--- a/reference/test/stop/residual_norm_kernels.cpp
+++ b/reference/test/stop/residual_norm_kernels.cpp
@@ -162,7 +162,7 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoal)
                 .residual_norm(res_norm.get())
                 .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-        res_norm->at(0) = r<TypeParam>::value * 1.1 * res_norm->at(0);
+        res_norm->at(0) = r<TypeParam>::value * 1.1 * rhs_norm->at(0);
         ASSERT_FALSE(
             rhs_criterion->update()
                 .residual_norm(res_norm.get())
@@ -170,7 +170,7 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoal)
         ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
         ASSERT_EQ(one_changed, false);
 
-        res_norm->at(0) = r<TypeParam>::value * 0.9 * res_norm->at(0);
+        res_norm->at(0) = r<TypeParam>::value * 0.9 * rhs_norm->at(0);
         ASSERT_TRUE(
             rhs_criterion->update()
                 .residual_norm(res_norm.get())
@@ -180,6 +180,7 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoal)
     }
     {
         auto res_norm = gko::initialize<NormVector>({100.0}, this->exec_);
+        auto init_res_val = res_norm->at(0, 0);
         constexpr gko::uint8 RelativeStoppingId{1};
         bool one_changed{};
         gko::Array<gko::stopping_status> stop_status(this->exec_, 1);
@@ -190,7 +191,7 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoal)
                 .residual_norm(res_norm.get())
                 .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-        res_norm->at(0) = r<TypeParam>::value * 1.1 * res_norm->at(0);
+        res_norm->at(0) = r<TypeParam>::value * 1.1 * init_res_val;
         ASSERT_FALSE(
             rel_criterion->update()
                 .residual_norm(res_norm.get())
@@ -198,7 +199,7 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoal)
         ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
         ASSERT_EQ(one_changed, false);
 
-        res_norm->at(0) = r<TypeParam>::value * 0.9 * res_norm->at(0);
+        res_norm->at(0) = r<TypeParam>::value * 0.9 * init_res_val;
         ASSERT_TRUE(
             rel_criterion->update()
                 .residual_norm(res_norm.get())
@@ -590,6 +591,7 @@ TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoal)
     auto initial_res = gko::initialize<Mtx>({100.0}, this->exec_);
     std::shared_ptr<gko::LinOp> rhs = gko::initialize<Mtx>({10.0}, this->exec_);
     auto res_norm = gko::initialize<NormVector>({100.0}, this->exec_);
+    auto init_res_val = res_norm->at(0, 0);
     auto criterion =
         this->factory_->generate(nullptr, rhs, nullptr, initial_res.get());
     bool one_changed{};
@@ -602,7 +604,7 @@ TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoal)
             .residual_norm(res_norm.get())
             .check(RelativeStoppingId, true, &stop_status, &one_changed));
 
-    res_norm->at(0) = r<TypeParam>::value * 1.1 * res_norm->at(0);
+    res_norm->at(0) = r<TypeParam>::value * 1.1 * init_res_val;
     ASSERT_FALSE(
         criterion->update()
             .residual_norm(res_norm.get())
@@ -610,7 +612,7 @@ TYPED_TEST(ResidualNormReduction, WaitsTillResidualGoal)
     ASSERT_EQ(stop_status.get_data()[0].has_converged(), false);
     ASSERT_EQ(one_changed, false);
 
-    res_norm->at(0) = r<TypeParam>::value * 0.9 * res_norm->at(0);
+    res_norm->at(0) = r<TypeParam>::value * 0.9 * init_res_val;
     ASSERT_TRUE(
         criterion->update()
             .residual_norm(res_norm.get())

From c1feeb640cd3453cbe8f3a78015db33c152505cf Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 4 Aug 2021 14:33:51 +0200
Subject: [PATCH 0956/1008] Fix README.md and INSTALL.md.

Update to the new package status. Simplify the HIP related INSTALL.md section.
---
 INSTALL.md | 19 +------------------
 README.md  | 27 +++++++++++++++++----------
 2 files changed, 18 insertions(+), 28 deletions(-)

diff --git a/INSTALL.md b/INSTALL.md
index 0b7e21acf16..787508b720c 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -2,7 +2,7 @@ Installation Instructions                      {#install_ginkgo}
 -------------------------------------
 ### Building
 
-Use the standard cmake build procedure:
+Use the standard CMake build procedure:
 
 ```sh
 mkdir build; cd build
@@ -152,15 +152,6 @@ of HIP either at `/opt/rocm/hip` or at the path specified by `HIP_PATH` as a
 CMake parameter (`-DHIP_PATH=`) or environment variable (`export HIP_PATH=`),
 unless `-DGINKGO_BUILD_HIP=ON/OFF` is set explicitly.
 
-#### HIP dependencies for Ginkgo
-Ginkgo's HIP backend adds a dependency to the following packages:
-+ HIP,
-+ hipBLAS,
-+ hipSPARSE,
-+ rocRAND,
-+ rocThrust.
-
-
 #### Changing the paths to search for HIP and other packages
 All HIP installation paths can be configured through the use of environment
 variables or CMake variables. This way of configuring the paths is currently
@@ -195,14 +186,6 @@ environment variable `HIP_PLATFORM` like so:
 export HIP_PLATFORM=nvcc # or nvidia for ROCM >= 4.1
 ```
 
-When using `HIP_PLATFORM=hcc` (or `amd`), note that two `HIP` compilers can be
-set: the old `hcc`, or since ROCm 3.5, `clang`. Ginkgo is only compatible with
-the `clang` based installations. Although this setting should be automatically
-done, it is also possible to manually set the `HIP` compiler to `clang`:
-```
-export HIP_COMPILER=clang
-```
-
 #### Setting platform specific compilation flags
 Platform specific compilation flags can be given through the following CMake
 variables:
diff --git a/README.md b/README.md
index 5f66f2fe6d6..bd7078df8be 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ Prerequisites
 
 For Ginkgo core library:
 
-*   _cmake 3.9+_
+*   _cmake 3.13+_
 *   C++14 compliant compiler, one of:
     *   _gcc 5.3+, 6.3+, 7.3+, all versions after 8.1+_
     *   _clang 3.9+_
@@ -51,6 +51,20 @@ The Ginkgo CUDA module has the following __additional__ requirements:
     [CUDA installation guide for Linux](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html)
     or [CUDA installation guide for Mac Os X](https://docs.nvidia.com/cuda/cuda-installation-guide-mac-os-x/index.html)
 
+The Ginkgo HIP module has the following __additional__ requirements:
+
+* _ROCm 3.5+_
+*    the HIP, hipBLAS, hipSPARSE, hip/rocRAND and rocThrust packages compiled with either:
+    * _AMD_ backend (using the `clang` compiler)
+    * _9.2 <= CUDA < 11_ backend
+
+The Ginkgo DPC++ module has the following __additional__ requirements:
+
+* _OneAPI 2021.3+_
+* Set `dpcpp` as the `CMAKE_CXX_COMPILER`
+* The following oneAPI packages should be available:
+    * oneMKL
+    * oneDPL
 
 In addition, if you want to contribute code to Ginkgo, you will also need the
 following:
@@ -59,21 +73,14 @@ following:
 *   _clang-tidy_ (optional, when setting the flag `-DGINKGO_WITH_CLANG_TIDY=ON`)
 *   _iwyu_ (Include What You Use, optional, when setting the flag `-DGINKGO_WITH_IWYU=ON`)
 
-The Ginkgo HIP module has the following __additional__ requirements:
-
-* _ROCm 3.5+_
-*    the HIP, hipBLAS, hipSPARSE, hip/rocRAND packages compiled with either:
-    * _AMD_ backend (using the `clang` compiler)
-    * _CUDA 9.2+_ backend. When using CUDA 10+, _cmake 3.12.2+_ is required.
-
 ### Windows
 
 The prequirement needs to be verified
-*   _cmake 3.9+_
+*   _cmake 3.13+_
 *   C++14 compliant 64-bit compiler:
     *   _MinGW : gcc 5.3+, 6.3+, 7.3+, all versions after 8.1+_
     *   _Cygwin : gcc 5.3+, 6.3+, 7.3+, all versions after 8.1+_
-    *   _Microsoft Visual Studio : VS 2017 15.7+_
+    *   _Microsoft Visual Studio : VS 2019+_
 
 __NOTE:__ Need to add `--autocrlf=input` after `git clone` in _Cygwin_.
 

From 106a0efad59f66d755b5667785b41b2206eec49d Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 4 Aug 2021 14:34:26 +0200
Subject: [PATCH 0957/1008] Disable the MSVC-CUDA 10.2 job.

There is an issue with the CUDA implementation which prevents a proper
execution. See https://github.com/pytorch/pytorch/issues/25393. Tweaking
the compiler settings would allow to get less errors, but it seems
impossible to prevent the errors altogether.
---
 .github/workflows/windows-msvc-cuda.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/windows-msvc-cuda.yml b/.github/workflows/windows-msvc-cuda.yml
index 3eb26f008e5..31eaba20d77 100644
--- a/.github/workflows/windows-msvc-cuda.yml
+++ b/.github/workflows/windows-msvc-cuda.yml
@@ -15,7 +15,6 @@ jobs:
       fail-fast: false
       matrix:
         config:
-        - {version: "10.2.89.20191206", name: "cuda102/release/shared", "mixed": "ON"}
         - {version: "latest", name: "cuda-latest/release/shared", "mixed": "OFF"}
     name: msvc/${{ matrix.config.name }} (only compile)
     runs-on: [windows-latest]

From 65b464ab915d4bf536454d13d0de8be52c60894a Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 4 Aug 2021 15:39:42 +0200
Subject: [PATCH 0958/1008] Review Updates.

+ Simplify GCC versions to an approximation of what we know for sure
works/is tested.
+ Test `mixed` in MSVC-CUDA.

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 .github/workflows/windows-msvc-cuda.yml | 2 +-
 README.md                               | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/windows-msvc-cuda.yml b/.github/workflows/windows-msvc-cuda.yml
index 31eaba20d77..90b000bfc39 100644
--- a/.github/workflows/windows-msvc-cuda.yml
+++ b/.github/workflows/windows-msvc-cuda.yml
@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         config:
-        - {version: "latest", name: "cuda-latest/release/shared", "mixed": "OFF"}
+        - {version: "latest", name: "cuda-latest/release/shared", "mixed": "ON"}
     name: msvc/${{ matrix.config.name }} (only compile)
     runs-on: [windows-latest]
 
diff --git a/README.md b/README.md
index bd7078df8be..16f8894d106 100644
--- a/README.md
+++ b/README.md
@@ -38,7 +38,7 @@ For Ginkgo core library:
 
 *   _cmake 3.13+_
 *   C++14 compliant compiler, one of:
-    *   _gcc 5.3+, 6.3+, 7.3+, all versions after 8.1+_
+    *   _gcc 5.5+_
     *   _clang 3.9+_
     *   _Intel compiler 2018+_
     *   _Apple LLVM 8.0+_
@@ -75,11 +75,10 @@ following:
 
 ### Windows
 
-The prequirement needs to be verified
 *   _cmake 3.13+_
 *   C++14 compliant 64-bit compiler:
-    *   _MinGW : gcc 5.3+, 6.3+, 7.3+, all versions after 8.1+_
-    *   _Cygwin : gcc 5.3+, 6.3+, 7.3+, all versions after 8.1+_
+    *   _MinGW : gcc 5.5+_
+    *   _Cygwin : gcc 5.5+_
     *   _Microsoft Visual Studio : VS 2019+_
 
 __NOTE:__ Need to add `--autocrlf=input` after `git clone` in _Cygwin_.

From 8d667fe57bc6a833f32cc3e683ad0592235431c9 Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Wed, 21 Jul 2021 17:56:13 +0200
Subject: [PATCH 0959/1008] correctly derive enablelogging from template
 parameter

---
 include/ginkgo/core/log/logger.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp
index 9ae4298aab0..a55f7c8b432 100644
--- a/include/ginkgo/core/log/logger.hpp
+++ b/include/ginkgo/core/log/logger.hpp
@@ -584,7 +584,7 @@ class Loggable {
  *                          class.
  */
 template <typename ConcreteLoggable, typename PolymorphicBase = Loggable>
-class EnableLogging : public Loggable {
+class EnableLogging : public PolymorphicBase {
 public:
     void add_logger(std::shared_ptr<const Logger> logger) override
     {

From fc13249509a6d8333e98e71d351751a7b35c569b Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Wed, 21 Jul 2021 18:02:00 +0200
Subject: [PATCH 0960/1008] remove copy of shared_ptr in `remove_logger`

---
 include/ginkgo/core/log/logger.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp
index a55f7c8b432..28cb33e75bc 100644
--- a/include/ginkgo/core/log/logger.hpp
+++ b/include/ginkgo/core/log/logger.hpp
@@ -594,7 +594,7 @@ class EnableLogging : public PolymorphicBase {
     void remove_logger(const Logger *logger) override
     {
         auto idx = find_if(begin(loggers_), end(loggers_),
-                           [&logger](std::shared_ptr<const Logger> l) {
+                           [&logger](std::shared_ptr<const Logger> &l) {
                                return lend(l) == logger;
                            });
         if (idx != end(loggers_)) {

From 5432ee9c12db9654ea62b0c931fc085a4188a761 Mon Sep 17 00:00:00 2001
From: Marcel Koch <marcel.koch@kit.edu>
Date: Thu, 22 Jul 2021 08:56:16 +0200
Subject: [PATCH 0961/1008] Review update

- use auto in lambda

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 include/ginkgo/core/log/logger.hpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp
index 28cb33e75bc..4e0a033189d 100644
--- a/include/ginkgo/core/log/logger.hpp
+++ b/include/ginkgo/core/log/logger.hpp
@@ -593,10 +593,9 @@ class EnableLogging : public PolymorphicBase {
 
     void remove_logger(const Logger *logger) override
     {
-        auto idx = find_if(begin(loggers_), end(loggers_),
-                           [&logger](std::shared_ptr<const Logger> &l) {
-                               return lend(l) == logger;
-                           });
+        auto idx =
+            find_if(begin(loggers_), end(loggers_),
+                    [&logger](const auto &l) { return lend(l) == logger; });
         if (idx != end(loggers_)) {
             loggers_.erase(idx);
         } else {

From 98ac37543d6d26a4e4332aee74f9bf53c9af2394 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 5 Aug 2021 12:07:23 +0200
Subject: [PATCH 0962/1008] use shallow clones for Actions where possible

---
 .github/workflows/bot-pr-comment.yml    | 12 ++++++------
 .github/workflows/bot-pr-created.yml    |  4 ++--
 .github/workflows/bot-pr-updated.yml    |  4 ++--
 .github/workflows/joss.yml              |  6 +++++-
 .github/workflows/windows-cygwin.yml    |  5 ++++-
 .github/workflows/windows-mingw.yml     |  5 ++++-
 .github/workflows/windows-msvc-cuda.yml |  5 ++++-
 .github/workflows/windows-msvc-ref.yml  |  5 ++++-
 8 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/bot-pr-comment.yml b/.github/workflows/bot-pr-comment.yml
index 61952428f3e..fa3aafecf0e 100644
--- a/.github/workflows/bot-pr-comment.yml
+++ b/.github/workflows/bot-pr-comment.yml
@@ -7,11 +7,11 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event.issue.pull_request != '' && github.event.comment.body == 'label!' && (github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER')
     steps:
-    - name: Checkout the latest code
+    - name: Checkout the latest code (shallow clone)
       uses: actions/checkout@v2
       with:
         ref: develop
-        fetch-depth: 0
+        fetch-depth: 1
     - name: Add appropriate labels
       env:
         GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
@@ -21,11 +21,11 @@ jobs:
     runs-on: ubuntu-18.04
     if: github.event.issue.pull_request != '' && github.event.comment.body == 'check-format!' && (github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER')
     steps:
-    - name: Checkout the latest code
+    - name: Checkout the latest code (shallow clone)
       uses: actions/checkout@v2
       with:
         ref: develop
-        fetch-depth: 0
+        fetch-depth: 1
     - name: Check for formatting changes
       env:
         GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
@@ -41,11 +41,11 @@ jobs:
     runs-on: ubuntu-18.04
     if: github.event.issue.pull_request != '' && github.event.comment.body == 'format!' && (github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER')
     steps:
-    - name: Checkout the latest code
+    - name: Checkout the latest code (shallow clone)
       uses: actions/checkout@v2
       with:
         ref: develop
-        fetch-depth: 0
+        fetch-depth: 1
     - name: Commit formatting changes
       env:
         GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
diff --git a/.github/workflows/bot-pr-created.yml b/.github/workflows/bot-pr-created.yml
index d86d6f5c869..5ef9de9b950 100644
--- a/.github/workflows/bot-pr-created.yml
+++ b/.github/workflows/bot-pr-created.yml
@@ -8,11 +8,11 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event.pull_request.author_association == 'COLLABORATOR' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'OWNER'
     steps:
-      - name: Checkout the latest code
+      - name: Checkout the latest code (shallow clone)
         uses: actions/checkout@v2
         with:
           ref: develop
-          fetch-depth: 0
+          fetch-depth: 1
       - name: Add appropriate labels
         env:
           GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
diff --git a/.github/workflows/bot-pr-updated.yml b/.github/workflows/bot-pr-updated.yml
index b0f0c2e5570..6db634d5129 100644
--- a/.github/workflows/bot-pr-updated.yml
+++ b/.github/workflows/bot-pr-updated.yml
@@ -8,11 +8,11 @@ jobs:
     runs-on: ubuntu-18.04
     if: github.event.pull_request.author_association == 'COLLABORATOR' || github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'OWNER'
     steps:
-      - name: Checkout the latest code
+      - name: Checkout the latest code (shallow clone)
         uses: actions/checkout@v2
         with:
           ref: develop
-          fetch-depth: 0
+          fetch-depth: 1
       - name: Check for formatting changes
         env:
           GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
diff --git a/.github/workflows/joss.yml b/.github/workflows/joss.yml
index 67d52410466..d278c5f97b3 100644
--- a/.github/workflows/joss.yml
+++ b/.github/workflows/joss.yml
@@ -11,7 +11,11 @@ jobs:
     runs-on: [ubuntu-latest] 
 
     steps:
-    - uses: actions/checkout@v2
+    - name: Checkout the latest code (shallow clone)
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 1
+
     - name: setup
       run: sudo apt-get install texlive-xetex pandoc pandoc-citeproc
     - name: info
diff --git a/.github/workflows/windows-cygwin.yml b/.github/workflows/windows-cygwin.yml
index 317e3608253..326e6c13e23 100644
--- a/.github/workflows/windows-cygwin.yml
+++ b/.github/workflows/windows-cygwin.yml
@@ -20,7 +20,10 @@ jobs:
     runs-on: [windows-latest]
     steps:
     - run: git config --global core.autocrlf input
-    - uses: actions/checkout@v2
+    - name: Checkout the latest code (shallow clone)
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 1
 
     - name: setup
       run: |
diff --git a/.github/workflows/windows-mingw.yml b/.github/workflows/windows-mingw.yml
index beeeb34d840..d88b6cc322c 100644
--- a/.github/workflows/windows-mingw.yml
+++ b/.github/workflows/windows-mingw.yml
@@ -19,7 +19,10 @@ jobs:
     name: mingw/${{ matrix.config.name }}
     runs-on: [windows-latest]
     steps:
-    - uses: actions/checkout@v2
+    - name: Checkout the latest code (shallow clone)
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 1
 
     - name: Debug over SSH (tmate)
       uses: mxschmitt/action-tmate@v3.5
diff --git a/.github/workflows/windows-msvc-cuda.yml b/.github/workflows/windows-msvc-cuda.yml
index 90b000bfc39..6ee48359f86 100644
--- a/.github/workflows/windows-msvc-cuda.yml
+++ b/.github/workflows/windows-msvc-cuda.yml
@@ -20,7 +20,10 @@ jobs:
     runs-on: [windows-latest]
 
     steps:
-    - uses: actions/checkout@v2
+    - name: Checkout the latest code (shallow clone)
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 1
     - name: setup (versioned)
       if: matrix.config.version != 'latest'
       run: |
diff --git a/.github/workflows/windows-msvc-ref.yml b/.github/workflows/windows-msvc-ref.yml
index c2df41390c1..7783233ffec 100644
--- a/.github/workflows/windows-msvc-ref.yml
+++ b/.github/workflows/windows-msvc-ref.yml
@@ -22,7 +22,10 @@ jobs:
     name: msvc/${{ matrix.config.name }}
     runs-on: [windows-latest]
     steps:
-    - uses: actions/checkout@v2
+    - name: Checkout the latest code (shallow clone)
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 1
 
     - name: Debug over SSH (tmate)
       uses: mxschmitt/action-tmate@v3.5

From 5b7244d03ec2a7344c5b6bb6f52df7fc96fc33c4 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 5 Aug 2021 12:08:38 +0200
Subject: [PATCH 0963/1008] allow bot pushes to trigger workflows

this is due to actions/checkout@v2 persisting git credentials in the
git credential helper. By removing them, we can instead use the ones
we provide in the remote URLs.
---
 .github/workflows/bot-pr-comment.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/bot-pr-comment.yml b/.github/workflows/bot-pr-comment.yml
index fa3aafecf0e..a0850939ec0 100644
--- a/.github/workflows/bot-pr-comment.yml
+++ b/.github/workflows/bot-pr-comment.yml
@@ -46,6 +46,7 @@ jobs:
       with:
         ref: develop
         fetch-depth: 1
+        persist-credentials: false
     - name: Commit formatting changes
       env:
         GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
@@ -60,6 +61,7 @@ jobs:
       with:
         ref: develop
         fetch-depth: 0
+        persist-credentials: false
     - name: Automatic Rebase
       env:
         GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}

From e9578ffbf93bd0997c0644725f810bc733dec560 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 5 Aug 2021 12:17:45 +0200
Subject: [PATCH 0964/1008] use shallow clone also for OS X

---
 .github/workflows/osx.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml
index 79caf2169a4..3cd3d579e51 100644
--- a/.github/workflows/osx.yml
+++ b/.github/workflows/osx.yml
@@ -21,7 +21,10 @@ jobs:
     runs-on: [macos-latest]
 
     steps:
-    - uses: actions/checkout@v2
+    - name: Checkout the latest code (shallow clone)
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 1
 
     - name: setup
       run: |

From dea560b8bf82323433c317b5f2f87d7ee1d0fa81 Mon Sep 17 00:00:00 2001
From: Tobias Ribizel <ribizel@kit.edu>
Date: Thu, 5 Aug 2021 12:51:58 +0200
Subject: [PATCH 0965/1008] remove fetch-depth default

---
 .github/workflows/bot-pr-comment.yml    | 3 ---
 .github/workflows/bot-pr-created.yml    | 1 -
 .github/workflows/bot-pr-updated.yml    | 1 -
 .github/workflows/joss.yml              | 2 --
 .github/workflows/osx.yml               | 2 --
 .github/workflows/windows-cygwin.yml    | 2 --
 .github/workflows/windows-mingw.yml     | 2 --
 .github/workflows/windows-msvc-cuda.yml | 2 --
 .github/workflows/windows-msvc-ref.yml  | 2 --
 9 files changed, 17 deletions(-)

diff --git a/.github/workflows/bot-pr-comment.yml b/.github/workflows/bot-pr-comment.yml
index a0850939ec0..0058e25208c 100644
--- a/.github/workflows/bot-pr-comment.yml
+++ b/.github/workflows/bot-pr-comment.yml
@@ -11,7 +11,6 @@ jobs:
       uses: actions/checkout@v2
       with:
         ref: develop
-        fetch-depth: 1
     - name: Add appropriate labels
       env:
         GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
@@ -25,7 +24,6 @@ jobs:
       uses: actions/checkout@v2
       with:
         ref: develop
-        fetch-depth: 1
     - name: Check for formatting changes
       env:
         GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
@@ -45,7 +43,6 @@ jobs:
       uses: actions/checkout@v2
       with:
         ref: develop
-        fetch-depth: 1
         persist-credentials: false
     - name: Commit formatting changes
       env:
diff --git a/.github/workflows/bot-pr-created.yml b/.github/workflows/bot-pr-created.yml
index 5ef9de9b950..aa306e1f3aa 100644
--- a/.github/workflows/bot-pr-created.yml
+++ b/.github/workflows/bot-pr-created.yml
@@ -12,7 +12,6 @@ jobs:
         uses: actions/checkout@v2
         with:
           ref: develop
-          fetch-depth: 1
       - name: Add appropriate labels
         env:
           GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
diff --git a/.github/workflows/bot-pr-updated.yml b/.github/workflows/bot-pr-updated.yml
index 6db634d5129..c83d1866b57 100644
--- a/.github/workflows/bot-pr-updated.yml
+++ b/.github/workflows/bot-pr-updated.yml
@@ -12,7 +12,6 @@ jobs:
         uses: actions/checkout@v2
         with:
           ref: develop
-          fetch-depth: 1
       - name: Check for formatting changes
         env:
           GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
diff --git a/.github/workflows/joss.yml b/.github/workflows/joss.yml
index d278c5f97b3..92aca560633 100644
--- a/.github/workflows/joss.yml
+++ b/.github/workflows/joss.yml
@@ -13,8 +13,6 @@ jobs:
     steps:
     - name: Checkout the latest code (shallow clone)
       uses: actions/checkout@v2
-      with:
-        fetch-depth: 1
 
     - name: setup
       run: sudo apt-get install texlive-xetex pandoc pandoc-citeproc
diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml
index 3cd3d579e51..5807599e800 100644
--- a/.github/workflows/osx.yml
+++ b/.github/workflows/osx.yml
@@ -23,8 +23,6 @@ jobs:
     steps:
     - name: Checkout the latest code (shallow clone)
       uses: actions/checkout@v2
-      with:
-        fetch-depth: 1
 
     - name: setup
       run: |
diff --git a/.github/workflows/windows-cygwin.yml b/.github/workflows/windows-cygwin.yml
index 326e6c13e23..253a5dd38f6 100644
--- a/.github/workflows/windows-cygwin.yml
+++ b/.github/workflows/windows-cygwin.yml
@@ -22,8 +22,6 @@ jobs:
     - run: git config --global core.autocrlf input
     - name: Checkout the latest code (shallow clone)
       uses: actions/checkout@v2
-      with:
-        fetch-depth: 1
 
     - name: setup
       run: |
diff --git a/.github/workflows/windows-mingw.yml b/.github/workflows/windows-mingw.yml
index d88b6cc322c..7e774c3cc21 100644
--- a/.github/workflows/windows-mingw.yml
+++ b/.github/workflows/windows-mingw.yml
@@ -21,8 +21,6 @@ jobs:
     steps:
     - name: Checkout the latest code (shallow clone)
       uses: actions/checkout@v2
-      with:
-        fetch-depth: 1
 
     - name: Debug over SSH (tmate)
       uses: mxschmitt/action-tmate@v3.5
diff --git a/.github/workflows/windows-msvc-cuda.yml b/.github/workflows/windows-msvc-cuda.yml
index 6ee48359f86..e2c0a3b8c43 100644
--- a/.github/workflows/windows-msvc-cuda.yml
+++ b/.github/workflows/windows-msvc-cuda.yml
@@ -22,8 +22,6 @@ jobs:
     steps:
     - name: Checkout the latest code (shallow clone)
       uses: actions/checkout@v2
-      with:
-        fetch-depth: 1
     - name: setup (versioned)
       if: matrix.config.version != 'latest'
       run: |
diff --git a/.github/workflows/windows-msvc-ref.yml b/.github/workflows/windows-msvc-ref.yml
index 7783233ffec..aa11a2ffba1 100644
--- a/.github/workflows/windows-msvc-ref.yml
+++ b/.github/workflows/windows-msvc-ref.yml
@@ -24,8 +24,6 @@ jobs:
     steps:
     - name: Checkout the latest code (shallow clone)
       uses: actions/checkout@v2
-      with:
-        fetch-depth: 1
 
     - name: Debug over SSH (tmate)
       uses: mxschmitt/action-tmate@v3.5

From 185f61e00f8989f4c1ac2991dd4f286e39e8e814 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 19 Jul 2021 17:05:42 +0200
Subject: [PATCH 0966/1008] porting and formating

---
 dpcpp/solver/idr_kernels.dp.cpp   | 804 +++++++++++++++++++++++++++++-
 dpcpp/test/solver/idr_kernels.cpp | 404 +++++++++++++++
 2 files changed, 1197 insertions(+), 11 deletions(-)
 create mode 100644 dpcpp/test/solver/idr_kernels.cpp

diff --git a/dpcpp/solver/idr_kernels.dp.cpp b/dpcpp/solver/idr_kernels.dp.cpp
index 9bb956f3807..7618c8b70d1 100644
--- a/dpcpp/solver/idr_kernels.dp.cpp
+++ b/dpcpp/solver/idr_kernels.dp.cpp
@@ -33,15 +33,27 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/idr_kernels.hpp"
 
 
-#include <memory>
+#include <ctime>
+#include <dpcpp/base/curand_bindings.hpp>
+#include <dpcpp/base/math.hpp>
+#include <random>
+
+
+#include <CL/sycl.hpp>
 
 
-#include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/executor.hpp>
 #include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/stop/stopping_status.hpp>
+
+
+#include "core/components/fill_array.hpp"
+#include "dpcpp/base/config.hpp"
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/base/onemkl_bindings.hpp"
+#include "dpcpp/components/atomic.dp.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
+#include "dpcpp/components/reduction.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
 
 
 namespace gko {
@@ -55,11 +67,744 @@ namespace dpcpp {
 namespace idr {
 
 
+constexpr int default_block_size = 512;
+constexpr int default_dot_dim = 32;
+constexpr int default_dot_size = default_dot_dim * default_dot_dim;
+
+
+template <typename ValueType>
+void initialize_m_kernel(size_type subspace_dim, size_type nrhs,
+                         ValueType *__restrict__ m_values, size_type m_stride,
+                         stopping_status *__restrict__ stop_status,
+                         sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row = global_id / m_stride;
+    const auto col = global_id % m_stride;
+
+    if (global_id < nrhs) {
+        stop_status[global_id].reset();
+    }
+
+    if (row < subspace_dim && col < nrhs * subspace_dim) {
+        m_values[row * m_stride + col] =
+            (row == col / nrhs) ? one<ValueType>() : zero<ValueType>();
+    }
+}
+
+template <typename ValueType>
+void initialize_m_kernel(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                         sycl::queue *stream, size_type subspace_dim,
+                         size_type nrhs, ValueType *m_values,
+                         size_type m_stride, stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                initialize_m_kernel(subspace_dim, nrhs, m_values, m_stride,
+                                    stop_status, item_ct1);
+            });
+    });
+}
+
+
+template <size_type block_size, typename ValueType>
+void orthonormalize_subspace_vectors_kernel(
+    size_type num_rows, size_type num_cols, ValueType *__restrict__ values,
+    size_type stride, sycl::nd_item<3> item_ct1,
+    UninitializedArray<ValueType, block_size> *reduction_helper_array,
+    remove_complex<ValueType> *reduction_helper_real)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+
+
+    ValueType *__restrict__ reduction_helper = (*reduction_helper_array);
+
+
+    for (size_type row = 0; row < num_rows; row++) {
+        for (size_type i = 0; i < row; i++) {
+            auto dot = zero<ValueType>();
+            for (size_type j = tidx; j < num_cols; j += block_size) {
+                /*
+                DPCT1007:5: Migration of this CUDA API is not supported by the
+                Intel(R) DPC++ Compatibility Tool.
+                */
+                dot += values[row * stride + j] * conj(values[i * stride + j]);
+            }
+
+            reduction_helper[tidx] = dot;
+
+            /*
+            DPCT1065:3: Consider replacing sycl::nd_item::barrier() with
+            sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
+            better performance, if there is no access to global memory.
+            */
+            item_ct1.barrier();
+            reduce(
+                group::this_thread_block(item_ct1), reduction_helper,
+                [](const ValueType &a, const ValueType &b) { return a + b; });
+            /*
+            DPCT1065:4: Consider replacing sycl::nd_item::barrier() with
+            sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
+            better performance, if there is no access to global memory.
+            */
+            item_ct1.barrier();
+
+            dot = reduction_helper[0];
+            for (size_type j = tidx; j < num_cols; j += block_size) {
+                values[row * stride + j] -= dot * values[i * stride + j];
+            }
+        }
+
+        auto norm = zero<remove_complex<ValueType>>();
+        for (size_type j = tidx; j < num_cols; j += block_size) {
+            norm += squared_norm(values[row * stride + j]);
+        }
+
+        reduction_helper_real[tidx] = norm;
+
+        /*
+        DPCT1065:1: Consider replacing sycl::nd_item::barrier() with
+        sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
+        better performance, if there is no access to global memory.
+        */
+        item_ct1.barrier();
+        reduce(group::this_thread_block(item_ct1), reduction_helper_real,
+               [](const remove_complex<ValueType> &a,
+                  const remove_complex<ValueType> &b) { return a + b; });
+        /*
+        DPCT1065:2: Consider replacing sycl::nd_item::barrier() with
+        sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
+        better performance, if there is no access to global memory.
+        */
+        item_ct1.barrier();
+
+        norm = sycl::sqrt((float)(reduction_helper_real[0]));
+        for (size_type j = tidx; j < num_cols; j += block_size) {
+            values[row * stride + j] /= norm;
+        }
+    }
+}
+
+template <size_type block_size, typename ValueType>
+void orthonormalize_subspace_vectors_kernel(
+    dim3 grid, dim3 block, size_t dynamic_shared_memory, sycl::queue *stream,
+    size_type num_rows, size_type num_cols, ValueType *values, size_type stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<ValueType, block_size>, 0,
+                       sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            reduction_helper_array_acc_ct1(cgh);
+        sycl::accessor<remove_complex<ValueType>, 1,
+                       sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            reduction_helper_real_acc_ct1(sycl::range<1>(block_size), cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                orthonormalize_subspace_vectors_kernel<block_size>(
+                    num_rows, num_cols, values, stride, item_ct1,
+                    (UninitializedArray<ValueType, block_size> *)
+                        reduction_helper_array_acc_ct1.get_pointer(),
+                    (remove_complex<ValueType> *)
+                        reduction_helper_real_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+template <typename ValueType>
+void solve_lower_triangular_kernel(
+    size_type subspace_dim, size_type nrhs,
+    const ValueType *__restrict__ m_values, size_type m_stride,
+    const ValueType *__restrict__ f_values, size_type f_stride,
+    ValueType *__restrict__ c_values, size_type c_stride,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+
+    if (global_id >= nrhs) {
+        return;
+    }
+
+    if (!stop_status[global_id].has_stopped()) {
+        for (size_type row = 0; row < subspace_dim; row++) {
+            auto temp = f_values[row * f_stride + global_id];
+            for (size_type col = 0; col < row; col++) {
+                temp -= m_values[row * m_stride + col * nrhs + global_id] *
+                        c_values[col * c_stride + global_id];
+            }
+            c_values[row * c_stride + global_id] =
+                temp / m_values[row * m_stride + row * nrhs + global_id];
+        }
+    }
+}
+
+template <typename ValueType>
+void solve_lower_triangular_kernel(
+    dim3 grid, dim3 block, size_t dynamic_shared_memory, sycl::queue *stream,
+    size_type subspace_dim, size_type nrhs, const ValueType *m_values,
+    size_type m_stride, const ValueType *f_values, size_type f_stride,
+    ValueType *c_values, size_type c_stride, const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                solve_lower_triangular_kernel(
+                    subspace_dim, nrhs, m_values, m_stride, f_values, f_stride,
+                    c_values, c_stride, stop_status, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType>
+void step_1_kernel(size_type k, size_type num_rows, size_type subspace_dim,
+                   size_type nrhs,
+                   const ValueType *__restrict__ residual_values,
+                   size_type residual_stride,
+                   const ValueType *__restrict__ c_values, size_type c_stride,
+                   const ValueType *__restrict__ g_values, size_type g_stride,
+                   ValueType *__restrict__ v_values, size_type v_stride,
+                   const stopping_status *__restrict__ stop_status,
+                   sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row = global_id / nrhs;
+    const auto col = global_id % nrhs;
+
+    if (row >= num_rows) {
+        return;
+    }
+
+    if (!stop_status[col].has_stopped()) {
+        auto temp = residual_values[row * residual_stride + col];
+        for (size_type j = k; j < subspace_dim; j++) {
+            temp -= c_values[j * c_stride + col] *
+                    g_values[row * g_stride + j * nrhs + col];
+        }
+        v_values[row * v_stride + col] = temp;
+    }
+}
+
+template <typename ValueType>
+void step_1_kernel(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                   sycl::queue *stream, size_type k, size_type num_rows,
+                   size_type subspace_dim, size_type nrhs,
+                   const ValueType *residual_values, size_type residual_stride,
+                   const ValueType *c_values, size_type c_stride,
+                   const ValueType *g_values, size_type g_stride,
+                   ValueType *v_values, size_type v_stride,
+                   const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                step_1_kernel(k, num_rows, subspace_dim, nrhs, residual_values,
+                              residual_stride, c_values, c_stride, g_values,
+                              g_stride, v_values, v_stride, stop_status,
+                              item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType>
+void step_2_kernel(size_type k, size_type num_rows, size_type subspace_dim,
+                   size_type nrhs, const ValueType *__restrict__ omega_values,
+                   const ValueType *__restrict__ v_values, size_type v_stride,
+                   const ValueType *__restrict__ c_values, size_type c_stride,
+                   ValueType *__restrict__ u_values, size_type u_stride,
+                   const stopping_status *__restrict__ stop_status,
+                   sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row = global_id / nrhs;
+    const auto col = global_id % nrhs;
+
+    if (row >= num_rows) {
+        return;
+    }
+
+    if (!stop_status[col].has_stopped()) {
+        auto temp = omega_values[col] * v_values[row * v_stride + col];
+        for (size_type j = k; j < subspace_dim; j++) {
+            temp += c_values[j * c_stride + col] *
+                    u_values[row * u_stride + j * nrhs + col];
+        }
+        u_values[row * u_stride + k * nrhs + col] = temp;
+    }
+}
+
+template <typename ValueType>
+void step_2_kernel(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                   sycl::queue *stream, size_type k, size_type num_rows,
+                   size_type subspace_dim, size_type nrhs,
+                   const ValueType *omega_values, const ValueType *v_values,
+                   size_type v_stride, const ValueType *c_values,
+                   size_type c_stride, ValueType *u_values, size_type u_stride,
+                   const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                step_2_kernel(k, num_rows, subspace_dim, nrhs, omega_values,
+                              v_values, v_stride, c_values, c_stride, u_values,
+                              u_stride, stop_status, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType>
+void multidot_kernel(
+    size_type num_rows, size_type nrhs, const ValueType *__restrict__ p_i,
+    const ValueType *__restrict__ g_k, size_type g_k_stride,
+    ValueType *__restrict__ alpha,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1,
+    UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
+        *reduction_helper_array)
+{
+    const auto tidx = item_ct1.get_local_id(2);
+    const auto tidy = item_ct1.get_local_id(1);
+    const auto rhs = item_ct1.get_group(2) * default_dot_dim + tidx;
+    const auto num = ceildiv(num_rows, item_ct1.get_group_range(1));
+    const auto start_row = item_ct1.get_group(1) * num;
+    const auto end_row = ((item_ct1.get_group(1) + 1) * num > num_rows)
+                             ? num_rows
+                             : (item_ct1.get_group(1) + 1) * num;
+    // Used that way to get around dynamic initialization warning and
+    // template error when using `reduction_helper_array` directly in `reduce`
+
+    ValueType *__restrict__ reduction_helper = (*reduction_helper_array);
+
+    ValueType local_res = zero<ValueType>();
+    if (rhs < nrhs && !stop_status[rhs].has_stopped()) {
+        for (size_type i = start_row + tidy; i < end_row;
+             i += default_dot_dim) {
+            const auto g_idx = i * g_k_stride + rhs;
+            local_res += p_i[i] * g_k[g_idx];
+        }
+    }
+    reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
+    /*
+    DPCT1065:10: Consider replacing sycl::nd_item::barrier() with
+    sycl::nd_item::barrier(sycl::access::fence_space::local_space) for better
+    performance, if there is no access to global memory.
+    */
+    item_ct1.barrier();
+    local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
+    const auto tile_block = group::tiled_partition<default_dot_dim>(
+        group::this_thread_block(item_ct1));
+    const auto sum =
+        reduce(tile_block, local_res,
+               [](const ValueType &a, const ValueType &b) { return a + b; });
+    const auto new_rhs = item_ct1.get_group(2) * default_dot_dim + tidy;
+    if (tidx == 0 && new_rhs < nrhs && !stop_status[new_rhs].has_stopped()) {
+        atomic_add(alpha + new_rhs, sum);
+    }
+}
+
+template <typename ValueType>
+void multidot_kernel(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                     sycl::queue *stream, size_type num_rows, size_type nrhs,
+                     const ValueType *p_i, const ValueType *g_k,
+                     size_type g_k_stride, ValueType *alpha,
+                     const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<ValueType, default_dot_dim *(
+                                                         default_dot_dim + 1)>,
+                       0, sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            reduction_helper_array_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                multidot_kernel(
+                    num_rows, nrhs, p_i, g_k, g_k_stride, alpha, stop_status,
+                    item_ct1,
+                    (UninitializedArray<ValueType, default_dot_dim *(
+                                                       default_dot_dim + 1)> *)
+                        reduction_helper_array_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+template <size_type block_size, typename ValueType>
+void update_g_k_and_u_kernel(
+    size_type k, size_type i, size_type size, size_type nrhs,
+    const ValueType *__restrict__ alpha, const ValueType *__restrict__ m_values,
+    size_type m_stride, const ValueType *__restrict__ g_values,
+    size_type g_stride, ValueType *__restrict__ g_k_values,
+    size_type g_k_stride, ValueType *__restrict__ u_values, size_type u_stride,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    const auto row = tidx / g_k_stride;
+    const auto rhs = tidx % g_k_stride;
+
+    if (row >= size || rhs >= nrhs) {
+        return;
+    }
+
+    if (!stop_status[rhs].has_stopped()) {
+        const auto fact = alpha[rhs] / m_values[i * m_stride + i * nrhs + rhs];
+        g_k_values[row * g_k_stride + rhs] -=
+            fact * g_values[row * g_stride + i * nrhs + rhs];
+        u_values[row * u_stride + k * nrhs + rhs] -=
+            fact * u_values[row * u_stride + i * nrhs + rhs];
+    }
+}
+
+template <size_type block_size, typename ValueType>
+void update_g_k_and_u_kernel(dim3 grid, dim3 block,
+                             size_t dynamic_shared_memory, sycl::queue *stream,
+                             size_type k, size_type i, size_type size,
+                             size_type nrhs, const ValueType *alpha,
+                             const ValueType *m_values, size_type m_stride,
+                             const ValueType *g_values, size_type g_stride,
+                             ValueType *g_k_values, size_type g_k_stride,
+                             ValueType *u_values, size_type u_stride,
+                             const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             update_g_k_and_u_kernel<block_size>(
+                                 k, i, size, nrhs, alpha, m_values, m_stride,
+                                 g_values, g_stride, g_k_values, g_k_stride,
+                                 u_values, u_stride, stop_status, item_ct1);
+                         });
+    });
+}
+
+
+template <size_type block_size, typename ValueType>
+void update_g_kernel(size_type k, size_type size, size_type nrhs,
+                     const ValueType *__restrict__ g_k_values,
+                     size_type g_k_stride, ValueType *__restrict__ g_values,
+                     size_type g_stride,
+                     const stopping_status *__restrict__ stop_status,
+                     sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    const auto row = tidx / g_k_stride;
+    const auto rhs = tidx % nrhs;
+
+    if (row >= size || rhs >= nrhs) {
+        return;
+    }
+
+    if (!stop_status[rhs].has_stopped()) {
+        g_values[row * g_stride + k * nrhs + rhs] =
+            g_k_values[row * g_k_stride + rhs];
+    }
+}
+
+template <size_type block_size, typename ValueType>
+void update_g_kernel(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                     sycl::queue *stream, size_type k, size_type size,
+                     size_type nrhs, const ValueType *g_k_values,
+                     size_type g_k_stride, ValueType *g_values,
+                     size_type g_stride, const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                update_g_kernel<block_size>(k, size, nrhs, g_k_values,
+                                            g_k_stride, g_values, g_stride,
+                                            stop_status, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType>
+void update_x_r_and_f_kernel(
+    size_type k, size_type size, size_type subspace_dim, size_type nrhs,
+    const ValueType *__restrict__ m_values, size_type m_stride,
+    const ValueType *__restrict__ g_values, size_type g_stride,
+    const ValueType *__restrict__ u_values, size_type u_stride,
+    ValueType *__restrict__ f_values, size_type f_stride,
+    ValueType *__restrict__ r_values, size_type r_stride,
+    ValueType *__restrict__ x_values, size_type x_stride,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+    const auto row = global_id / x_stride;
+    const auto col = global_id % x_stride;
+
+    if (row >= size || col >= nrhs) {
+        return;
+    }
+
+    if (!stop_status[col].has_stopped()) {
+        const auto beta = f_values[k * f_stride + col] /
+                          m_values[k * m_stride + k * nrhs + col];
+        r_values[row * r_stride + col] -=
+            beta * g_values[row * g_stride + k * nrhs + col];
+        x_values[row * x_stride + col] +=
+            beta * u_values[row * u_stride + k * nrhs + col];
+
+        if (k < row && k + 1 < subspace_dim && row < subspace_dim) {
+            f_values[row * f_stride + col] -=
+                beta * m_values[row * m_stride + k * nrhs + col];
+        }
+    }
+}
+
+template <typename ValueType>
+void update_x_r_and_f_kernel(
+    dim3 grid, dim3 block, size_t dynamic_shared_memory, sycl::queue *stream,
+    size_type k, size_type size, size_type subspace_dim, size_type nrhs,
+    const ValueType *m_values, size_type m_stride, const ValueType *g_values,
+    size_type g_stride, const ValueType *u_values, size_type u_stride,
+    ValueType *f_values, size_type f_stride, ValueType *r_values,
+    size_type r_stride, ValueType *x_values, size_type x_stride,
+    const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                update_x_r_and_f_kernel(
+                    k, size, subspace_dim, nrhs, m_values, m_stride, g_values,
+                    g_stride, u_values, u_stride, f_values, f_stride, r_values,
+                    r_stride, x_values, x_stride, stop_status, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType>
+void compute_omega_kernel(
+    size_type nrhs, const remove_complex<ValueType> kappa,
+    const ValueType *__restrict__ tht,
+    const remove_complex<ValueType> *__restrict__ residual_norm,
+    ValueType *__restrict__ omega,
+    const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1)
+{
+    const auto global_id = thread::get_thread_id_flat(item_ct1);
+
+    if (global_id >= nrhs) {
+        return;
+    }
+
+    if (!stop_status[global_id].has_stopped()) {
+        auto thr = omega[global_id];
+        omega[global_id] /= tht[global_id];
+        auto absrho =
+            std::abs(thr / (sycl::sqrt((float)(real(tht[global_id]))) *
+                            residual_norm[global_id]));
+
+        if (absrho < kappa) {
+            omega[global_id] *= kappa / absrho;
+        }
+    }
+}
+
+template <typename ValueType>
+void compute_omega_kernel(dim3 grid, dim3 block, size_t dynamic_shared_memory,
+                          sycl::queue *stream, size_type nrhs,
+                          const remove_complex<ValueType> kappa,
+                          const ValueType *tht,
+                          const remove_complex<ValueType> *residual_norm,
+                          ValueType *omega, const stopping_status *stop_status)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                compute_omega_kernel(nrhs, kappa, tht, residual_norm, omega,
+                                     stop_status, item_ct1);
+            });
+    });
+}
+
+
+namespace {
+
+
+template <typename ValueType>
+void initialize_m(const size_type nrhs, matrix::Dense<ValueType> *m,
+                  Array<stopping_status> *stop_status)
+{
+    const auto subspace_dim = m->get_size()[0];
+    const auto m_stride = m->get_stride();
+
+    const auto grid_dim = ceildiv(m_stride * subspace_dim, default_block_size);
+    initialize_m_kernel(grid_dim, default_block_size, 0, exec->get_queue(),
+                        subspace_dim, nrhs, m->get_values(), m_stride,
+                        stop_status->get_data());
+}
+
+
+template <typename ValueType>
+void initialize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors,
+                                 bool deterministic)
+{
+    if (deterministic) {
+        auto subspace_vectors_data = matrix_data<ValueType>(
+            subspace_vectors->get_size(), std::normal_distribution<>(0.0, 1.0),
+            std::ranlux48(15));
+        subspace_vectors->read(subspace_vectors_data);
+    } else {
+        auto gen =
+            curand::rand_generator(time(NULL), CURAND_RNG_PSEUDO_DEFAULT);
+        curand::rand_vector(
+            gen,
+            subspace_vectors->get_size()[0] * subspace_vectors->get_stride(),
+            0.0, 1.0, subspace_vectors->get_values());
+    }
+}
+
+
+template <typename ValueType>
+void orthonormalize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors)
+{
+    orthonormalize_subspace_vectors_kernel<default_block_size>(
+        1, default_block_size, 0, exec->get_queue(),
+        subspace_vectors->get_size()[0], subspace_vectors->get_size()[1],
+        subspace_vectors->get_values(), subspace_vectors->get_stride());
+}
+
+
+template <typename ValueType>
+void solve_lower_triangular(const size_type nrhs,
+                            const matrix::Dense<ValueType> *m,
+                            const matrix::Dense<ValueType> *f,
+                            matrix::Dense<ValueType> *c,
+                            const Array<stopping_status> *stop_status)
+{
+    const auto subspace_dim = m->get_size()[0];
+
+    const auto grid_dim = ceildiv(nrhs, default_block_size);
+    solve_lower_triangular_kernel(
+        grid_dim, default_block_size, 0, exec->get_queue(), subspace_dim, nrhs,
+        m->get_const_values(), m->get_stride(), f->get_const_values(),
+        f->get_stride(), c->get_values(), c->get_stride(),
+        stop_status->get_const_data());
+}
+
+
+template <typename ValueType>
+void update_g_and_u(std::shared_ptr<const DpcppExecutor> exec,
+                    const size_type nrhs, const size_type k,
+                    const matrix::Dense<ValueType> *p,
+                    const matrix::Dense<ValueType> *m,
+                    matrix::Dense<ValueType> *alpha,
+                    matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *g_k,
+                    matrix::Dense<ValueType> *u,
+                    const Array<stopping_status> *stop_status)
+{
+    const auto size = g->get_size()[0];
+    const auto p_stride = p->get_stride();
+
+    const dim3 grid_dim(ceildiv(nrhs, default_dot_dim),
+                        exec->get_num_multiprocessor() * 2);
+    const dim3 block_dim(default_dot_dim, default_dot_dim);
+
+    for (size_type i = 0; i < k; i++) {
+        const auto p_i = p->get_const_values() + i * p_stride;
+        if (nrhs > 1 || is_complex<ValueType>()) {
+            components::fill_array(exec, alpha->get_values(), nrhs,
+                                   zero<ValueType>());
+            multidot_kernel(grid_dim, block_dim, 0, exec->get_queue(), size,
+                            nrhs, p_i, g_k->get_values(), g_k->get_stride(),
+                            alpha->get_values(), stop_status->get_const_data());
+        } else {
+            cublas::dot(exec->get_cublas_handle(), size, p_i, 1,
+                        g_k->get_values(), g_k->get_stride(),
+                        alpha->get_values());
+        }
+        update_g_k_and_u_kernel<default_block_size>(
+            ceildiv(size * g_k->get_stride(), default_block_size),
+            default_block_size, 0, exec->get_queue(), k, i, size, nrhs,
+            alpha->get_const_values(), m->get_const_values(), m->get_stride(),
+            g->get_const_values(), g->get_stride(), g_k->get_values(),
+            g_k->get_stride(), u->get_values(), u->get_stride(),
+            stop_status->get_const_data());
+    }
+    update_g_kernel<default_block_size>(
+        ceildiv(size * g_k->get_stride(), default_block_size),
+        default_block_size, 0, exec->get_queue(), k, size, nrhs,
+        g_k->get_const_values(), g_k->get_stride(), g->get_values(),
+        g->get_stride(), stop_status->get_const_data());
+}
+
+
+template <typename ValueType>
+void update_m(std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
+              const size_type k, const matrix::Dense<ValueType> *p,
+              const matrix::Dense<ValueType> *g_k, matrix::Dense<ValueType> *m,
+              const Array<stopping_status> *stop_status)
+{
+    const auto size = g_k->get_size()[0];
+    const auto subspace_dim = m->get_size()[0];
+    const auto p_stride = p->get_stride();
+    const auto m_stride = m->get_stride();
+
+    const dim3 grid_dim(ceildiv(nrhs, default_dot_dim),
+                        exec->get_num_multiprocessor() * 2);
+    const dim3 block_dim(default_dot_dim, default_dot_dim);
+
+    for (size_type i = k; i < subspace_dim; i++) {
+        const auto p_i = p->get_const_values() + i * p_stride;
+        auto m_i = m->get_values() + i * m_stride + k * nrhs;
+        if (nrhs > 1 || is_complex<ValueType>()) {
+            components::fill_array(exec, m_i, nrhs, zero<ValueType>());
+            multidot_kernel(grid_dim, block_dim, 0, exec->get_queue(), size,
+                            nrhs, p_i, g_k->get_const_values(),
+                            g_k->get_stride(), m_i,
+                            stop_status->get_const_data());
+        } else {
+            cublas::dot(exec->get_cublas_handle(), size, p_i, 1,
+                        g_k->get_const_values(), g_k->get_stride(), m_i);
+        }
+    }
+}
+
+
+template <typename ValueType>
+void update_x_r_and_f(std::shared_ptr<const DpcppExecutor> exec,
+                      const size_type nrhs, const size_type k,
+                      const matrix::Dense<ValueType> *m,
+                      const matrix::Dense<ValueType> *g,
+                      const matrix::Dense<ValueType> *u,
+                      matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *r,
+                      matrix::Dense<ValueType> *x,
+                      const Array<stopping_status> *stop_status)
+{
+    const auto size = x->get_size()[0];
+    const auto subspace_dim = m->get_size()[0];
+
+    const auto grid_dim = ceildiv(size * x->get_stride(), default_block_size);
+    update_x_r_and_f_kernel(grid_dim, default_block_size, 0, exec->get_queue(),
+                            k, size, subspace_dim, nrhs, m->get_const_values(),
+                            m->get_stride(), g->get_const_values(),
+                            g->get_stride(), u->get_const_values(),
+                            u->get_stride(), f->get_values(), f->get_stride(),
+                            r->get_values(), r->get_stride(), x->get_values(),
+                            x->get_stride(), stop_status->get_const_data());
+    components::fill_array(exec, f->get_values() + k * f->get_stride(), nrhs,
+                           zero<ValueType>());
+}
+
+
+}  // namespace
+
+
 template <typename ValueType>
 void initialize(std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
                 matrix::Dense<ValueType> *m,
                 matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
-                Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+                Array<stopping_status> *stop_status)
+{
+    initialize_m(nrhs, m, stop_status);
+    initialize_subspace_vectors(subspace_vectors, deterministic);
+    orthonormalize_subspace_vectors(subspace_vectors);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
 
@@ -71,7 +816,21 @@ void step_1(std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
             const matrix::Dense<ValueType> *residual,
             const matrix::Dense<ValueType> *g, matrix::Dense<ValueType> *c,
             matrix::Dense<ValueType> *v,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+            const Array<stopping_status> *stop_status)
+{
+    solve_lower_triangular(nrhs, m, f, c, stop_status);
+
+    const auto num_rows = v->get_size()[0];
+    const auto subspace_dim = m->get_size()[0];
+
+    const auto grid_dim = ceildiv(nrhs * num_rows, default_block_size);
+    step_1_kernel(grid_dim, default_block_size, 0, exec->get_queue(), k,
+                  num_rows, subspace_dim, nrhs, residual->get_const_values(),
+                  residual->get_stride(), c->get_const_values(),
+                  c->get_stride(), g->get_const_values(), g->get_stride(),
+                  v->get_values(), v->get_stride(),
+                  stop_status->get_const_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_1_KERNEL);
 
@@ -81,7 +840,19 @@ void step_2(std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
             const size_type k, const matrix::Dense<ValueType> *omega,
             const matrix::Dense<ValueType> *preconditioned_vector,
             const matrix::Dense<ValueType> *c, matrix::Dense<ValueType> *u,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+            const Array<stopping_status> *stop_status)
+{
+    const auto num_rows = preconditioned_vector->get_size()[0];
+    const auto subspace_dim = u->get_size()[1] / nrhs;
+
+    const auto grid_dim = ceildiv(nrhs * num_rows, default_block_size);
+    step_2_kernel(grid_dim, default_block_size, 0, exec->get_queue(), k,
+                  num_rows, subspace_dim, nrhs, omega->get_const_values(),
+                  preconditioned_vector->get_const_values(),
+                  preconditioned_vector->get_stride(), c->get_const_values(),
+                  c->get_stride(), u->get_values(), u->get_stride(),
+                  stop_status->get_const_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_2_KERNEL);
 
@@ -93,7 +864,12 @@ void step_3(std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
             matrix::Dense<ValueType> *u, matrix::Dense<ValueType> *m,
             matrix::Dense<ValueType> *f, matrix::Dense<ValueType> *alpha,
             matrix::Dense<ValueType> *residual, matrix::Dense<ValueType> *x,
-            const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+            const Array<stopping_status> *stop_status)
+{
+    update_g_and_u(exec, nrhs, k, p, m, alpha, g, g_k, u, stop_status);
+    update_m(exec, nrhs, k, p, g_k, m, stop_status);
+    update_x_r_and_f(exec, nrhs, k, m, g, u, f, residual, x, stop_status);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_STEP_3_KERNEL);
 
@@ -103,8 +879,14 @@ void compute_omega(
     std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
     const remove_complex<ValueType> kappa, const matrix::Dense<ValueType> *tht,
     const matrix::Dense<remove_complex<ValueType>> *residual_norm,
-    matrix::Dense<ValueType> *omega,
-    const Array<stopping_status> *stop_status) GKO_NOT_IMPLEMENTED;
+    matrix::Dense<ValueType> *omega, const Array<stopping_status> *stop_status)
+{
+    const auto grid_dim = ceildiv(nrhs, config::warp_size);
+    compute_omega_kernel(grid_dim, config::warp_size, 0, exec->get_queue(),
+                         nrhs, kappa, tht->get_const_values(),
+                         residual_norm->get_const_values(), omega->get_values(),
+                         stop_status->get_const_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_COMPUTE_OMEGA_KERNEL);
 
diff --git a/dpcpp/test/solver/idr_kernels.cpp b/dpcpp/test/solver/idr_kernels.cpp
new file mode 100644
index 00000000000..9316ec02e7d
--- /dev/null
+++ b/dpcpp/test/solver/idr_kernels.cpp
@@ -0,0 +1,404 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/solver/idr.hpp>
+
+
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/stop/combined.hpp>
+#include <ginkgo/core/stop/iteration.hpp>
+#include <ginkgo/core/stop/residual_norm.hpp>
+
+
+#include "core/solver/idr_kernels.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class Idr : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Dense<>;
+    using Solver = gko::solver::Idr<>;
+
+    Idr() : rand_engine(30) {}
+
+    void SetUp()
+    {
+        ref = gko::ReferenceExecutor::create();
+        dpcpp = gko::DpcppExecutor::create(0, ref);
+
+        mtx = gen_mtx(123, 123);
+        d_mtx = Mtx::create(dpcpp);
+        d_mtx->copy_from(mtx.get());
+        dpcpp_idr_factory =
+            Solver::build()
+                .with_deterministic(true)
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(1u).on(dpcpp))
+                .on(dpcpp);
+
+        ref_idr_factory =
+            Solver::build()
+                .with_deterministic(true)
+                .with_criteria(
+                    gko::stop::Iteration::build().with_max_iters(1u).on(ref))
+                .on(ref);
+    }
+
+    void TearDown()
+    {
+        if (dpcpp != nullptr) {
+            ASSERT_NO_THROW(dpcpp->synchronize());
+        }
+    }
+
+    std::unique_ptr<Mtx> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<Mtx>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(num_cols, num_cols),
+            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
+    }
+
+    void initialize_data()
+    {
+        int size = 597;
+        nrhs = 17;
+        int s = 4;
+        x = gen_mtx(size, nrhs);
+        b = gen_mtx(size, nrhs);
+        r = gen_mtx(size, nrhs);
+        m = gen_mtx(s, nrhs * s);
+        f = gen_mtx(s, nrhs);
+        g = gen_mtx(size, nrhs * s);
+        u = gen_mtx(size, nrhs * s);
+        c = gen_mtx(s, nrhs);
+        v = gen_mtx(size, nrhs);
+        p = gen_mtx(s, size);
+        alpha = gen_mtx(1, nrhs);
+        omega = gen_mtx(1, nrhs);
+        tht = gen_mtx(1, nrhs);
+        residual_norm = gen_mtx(1, nrhs);
+        stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(ref, nrhs));
+        for (size_t i = 0; i < nrhs; ++i) {
+            stop_status->get_data()[i].reset();
+        }
+
+        d_x = Mtx::create(dpcpp);
+        d_b = Mtx::create(dpcpp);
+        d_r = Mtx::create(dpcpp);
+        d_m = Mtx::create(dpcpp);
+        d_f = Mtx::create(dpcpp);
+        d_g = Mtx::create(dpcpp);
+        d_u = Mtx::create(dpcpp);
+        d_c = Mtx::create(dpcpp);
+        d_v = Mtx::create(dpcpp);
+        d_p = Mtx::create(dpcpp);
+        d_alpha = Mtx::create(dpcpp);
+        d_omega = Mtx::create(dpcpp);
+        d_tht = Mtx::create(dpcpp);
+        d_residual_norm = Mtx::create(dpcpp);
+        d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
+            new gko::Array<gko::stopping_status>(dpcpp));
+
+        d_x->copy_from(x.get());
+        d_b->copy_from(b.get());
+        d_r->copy_from(r.get());
+        d_m->copy_from(m.get());
+        d_f->copy_from(f.get());
+        d_g->copy_from(g.get());
+        d_u->copy_from(u.get());
+        d_c->copy_from(c.get());
+        d_v->copy_from(v.get());
+        d_p->copy_from(p.get());
+        d_alpha->copy_from(alpha.get());
+        d_omega->copy_from(omega.get());
+        d_tht->copy_from(tht.get());
+        d_residual_norm->copy_from(residual_norm.get());
+        *d_stop_status =
+            *stop_status;  // copy_from is not a public member function of Array
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
+
+    std::ranlux48 rand_engine;
+
+    std::shared_ptr<Mtx> mtx;
+    std::shared_ptr<Mtx> d_mtx;
+    std::unique_ptr<Solver::Factory> dpcpp_idr_factory;
+    std::unique_ptr<Solver::Factory> ref_idr_factory;
+
+    gko::size_type nrhs;
+
+    std::unique_ptr<Mtx> x;
+    std::unique_ptr<Mtx> b;
+    std::unique_ptr<Mtx> r;
+    std::unique_ptr<Mtx> m;
+    std::unique_ptr<Mtx> f;
+    std::unique_ptr<Mtx> g;
+    std::unique_ptr<Mtx> u;
+    std::unique_ptr<Mtx> c;
+    std::unique_ptr<Mtx> v;
+    std::unique_ptr<Mtx> p;
+    std::unique_ptr<Mtx> alpha;
+    std::unique_ptr<Mtx> omega;
+    std::unique_ptr<Mtx> tht;
+    std::unique_ptr<Mtx> residual_norm;
+    std::unique_ptr<gko::Array<gko::stopping_status>> stop_status;
+
+    std::unique_ptr<Mtx> d_x;
+    std::unique_ptr<Mtx> d_b;
+    std::unique_ptr<Mtx> d_r;
+    std::unique_ptr<Mtx> d_m;
+    std::unique_ptr<Mtx> d_f;
+    std::unique_ptr<Mtx> d_g;
+    std::unique_ptr<Mtx> d_u;
+    std::unique_ptr<Mtx> d_c;
+    std::unique_ptr<Mtx> d_v;
+    std::unique_ptr<Mtx> d_p;
+    std::unique_ptr<Mtx> d_alpha;
+    std::unique_ptr<Mtx> d_omega;
+    std::unique_ptr<Mtx> d_tht;
+    std::unique_ptr<Mtx> d_residual_norm;
+    std::unique_ptr<gko::Array<gko::stopping_status>> d_stop_status;
+};
+
+
+TEST_F(Idr, IdrInitializeIsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::kernels::reference::idr::initialize(ref, nrhs, m.get(), p.get(), true,
+                                             stop_status.get());
+    gko::kernels::dpcpp::idr::initialize(dpcpp, nrhs, d_m.get(), d_p.get(),
+                                         true, d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
+    GKO_ASSERT_MTX_NEAR(p, d_p, 1e-14);
+}
+
+
+TEST_F(Idr, IdrStep1IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::size_type k = 2;
+    gko::kernels::reference::idr::step_1(ref, nrhs, k, m.get(), f.get(),
+                                         r.get(), g.get(), c.get(), v.get(),
+                                         stop_status.get());
+    gko::kernels::dpcpp::idr::step_1(dpcpp, nrhs, k, d_m.get(), d_f.get(),
+                                     d_r.get(), d_g.get(), d_c.get(), d_v.get(),
+                                     d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(c, d_c, 1e-14);
+    GKO_ASSERT_MTX_NEAR(v, d_v, 1e-14);
+}
+
+
+TEST_F(Idr, IdrStep2IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::size_type k = 2;
+    gko::kernels::reference::idr::step_2(ref, nrhs, k, omega.get(), v.get(),
+                                         c.get(), u.get(), stop_status.get());
+    gko::kernels::dpcpp::idr::step_2(dpcpp, nrhs, k, d_omega.get(), d_v.get(),
+                                     d_c.get(), d_u.get(), d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
+}
+
+
+TEST_F(Idr, IdrStep3IsEquivalentToRef)
+{
+    initialize_data();
+
+    gko::size_type k = 2;
+    gko::kernels::reference::idr::step_3(
+        ref, nrhs, k, p.get(), g.get(), v.get(), u.get(), m.get(), f.get(),
+        alpha.get(), r.get(), x.get(), stop_status.get());
+    gko::kernels::dpcpp::idr::step_3(
+        dpcpp, nrhs, k, d_p.get(), d_g.get(), d_v.get(), d_u.get(), d_m.get(),
+        d_f.get(), d_alpha.get(), d_r.get(), d_x.get(), d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(g, d_g, 1e-14);
+    GKO_ASSERT_MTX_NEAR(v, d_v, 1e-14);
+    GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
+    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
+    GKO_ASSERT_MTX_NEAR(f, d_f, 1e-14);
+    GKO_ASSERT_MTX_NEAR(r, d_r, 1e-14);
+    GKO_ASSERT_MTX_NEAR(x, d_x, 1e-14);
+}
+
+
+TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
+{
+    initialize_data();
+
+    double kappa = 0.7;
+    gko::kernels::reference::idr::compute_omega(ref, nrhs, kappa, tht.get(),
+                                                residual_norm.get(),
+                                                omega.get(), stop_status.get());
+    gko::kernels::dpcpp::idr::compute_omega(dpcpp, nrhs, kappa, d_tht.get(),
+                                            d_residual_norm.get(),
+                                            d_omega.get(), d_stop_status.get());
+
+    GKO_ASSERT_MTX_NEAR(omega, d_omega, 1e-14);
+}
+
+
+TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
+{
+    int m = 123;
+    int n = 1;
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto dpcpp_solver = dpcpp_idr_factory->generate(d_mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(dpcpp);
+    auto d_x = Mtx::create(dpcpp);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    dpcpp_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+}
+
+
+TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
+{
+    int m = 123;
+    int n = 1;
+    dpcpp_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(dpcpp))
+            .on(dpcpp);
+    ref_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(ref))
+            .on(ref);
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto dpcpp_solver = dpcpp_idr_factory->generate(d_mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(dpcpp);
+    auto d_x = Mtx::create(dpcpp);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    dpcpp_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+}
+
+
+TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
+{
+    int m = 123;
+    int n = 16;
+    auto dpcpp_solver = dpcpp_idr_factory->generate(d_mtx);
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(dpcpp);
+    auto d_x = Mtx::create(dpcpp);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    dpcpp_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-12);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
+}
+
+
+TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
+{
+    int m = 123;
+    int n = 16;
+    dpcpp_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(dpcpp))
+            .on(dpcpp);
+    ref_idr_factory =
+        Solver::build()
+            .with_deterministic(true)
+            .with_complex_subspace(true)
+            .with_criteria(
+                gko::stop::Iteration::build().with_max_iters(1u).on(ref))
+            .on(ref);
+    auto dpcpp_solver = dpcpp_idr_factory->generate(d_mtx);
+    auto ref_solver = ref_idr_factory->generate(mtx);
+    auto b = gen_mtx(m, n);
+    auto x = gen_mtx(m, n);
+    auto d_b = Mtx::create(dpcpp);
+    auto d_x = Mtx::create(dpcpp);
+    d_b->copy_from(b.get());
+    d_x->copy_from(x.get());
+
+    ref_solver->apply(b.get(), x.get());
+    dpcpp_solver->apply(d_b.get(), d_x.get());
+
+    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+}
+
+
+}  // namespace

From 778a91ffd0cd778ee1961ad7f040e4351efd0f97 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 20 Jul 2021 08:19:23 +0200
Subject: [PATCH 0967/1008] compile but idr still has issue on initial

---
 dpcpp/CMakeLists.txt              |  4 +-
 dpcpp/solver/idr_kernels.dp.cpp   | 92 +++++++++++++++++++------------
 dpcpp/test/solver/CMakeLists.txt  |  1 +
 dpcpp/test/solver/idr_kernels.cpp | 54 ++++++++++--------
 4 files changed, 92 insertions(+), 59 deletions(-)

diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index e2d476164e8..401be197820 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -8,6 +8,8 @@ set(GINKGO_DPCPP_VERSION ${GINKGO_DPCPP_VERSION} PARENT_SCOPE)
 
 find_package(MKL CONFIG REQUIRED HINTS "$ENV{MKLROOT}")
 set(GINKGO_MKL_ROOT "${MKL_ROOT}" PARENT_SCOPE)
+find_package(oneDPL REQUIRED HINTS "$ENV{DPL_ROOT}")
+set(GINKGO_DPL_ROOT "${DPL_ROOT}" PARENT_SCOPE)
 
 add_library(ginkgo_dpcpp $<TARGET_OBJECTS:ginkgo_dpcpp_device> "")
 target_sources(ginkgo_dpcpp
@@ -75,7 +77,7 @@ else ()
     target_link_options(ginkgo_dpcpp PUBLIC -fsycl-device-code-split=per_kernel)
 endif()
 target_link_libraries(ginkgo_dpcpp PUBLIC ginkgo_device)
-target_link_libraries(ginkgo_dpcpp PRIVATE MKL::MKL_DPCPP)
+target_link_libraries(ginkgo_dpcpp PRIVATE MKL::MKL_DPCPP oneDPL)
 if (GINKGO_DPCPP_SINGLE_MODE)
     target_compile_definitions(ginkgo_dpcpp PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
 endif()
diff --git a/dpcpp/solver/idr_kernels.dp.cpp b/dpcpp/solver/idr_kernels.dp.cpp
index 7618c8b70d1..2acfdc953c1 100644
--- a/dpcpp/solver/idr_kernels.dp.cpp
+++ b/dpcpp/solver/idr_kernels.dp.cpp
@@ -34,12 +34,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ctime>
-#include <dpcpp/base/curand_bindings.hpp>
-#include <dpcpp/base/math.hpp>
 #include <random>
 
 
 #include <CL/sycl.hpp>
+#include <oneapi/dpl/random>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
@@ -67,8 +66,8 @@ namespace dpcpp {
 namespace idr {
 
 
-constexpr int default_block_size = 512;
-constexpr int default_dot_dim = 32;
+constexpr int default_block_size = 256;
+constexpr int default_dot_dim = 16;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
@@ -140,7 +139,7 @@ void orthonormalize_subspace_vectors_kernel(
             better performance, if there is no access to global memory.
             */
             item_ct1.barrier();
-            reduce(
+            ::gko::kernels::dpcpp::reduce(
                 group::this_thread_block(item_ct1), reduction_helper,
                 [](const ValueType &a, const ValueType &b) { return a + b; });
             /*
@@ -169,9 +168,10 @@ void orthonormalize_subspace_vectors_kernel(
         better performance, if there is no access to global memory.
         */
         item_ct1.barrier();
-        reduce(group::this_thread_block(item_ct1), reduction_helper_real,
-               [](const remove_complex<ValueType> &a,
-                  const remove_complex<ValueType> &b) { return a + b; });
+        ::gko::kernels::dpcpp::reduce(
+            group::this_thread_block(item_ct1), reduction_helper_real,
+            [](const remove_complex<ValueType> &a,
+               const remove_complex<ValueType> &b) { return a + b; });
         /*
         DPCT1065:2: Consider replacing sycl::nd_item::barrier() with
         sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
@@ -179,7 +179,7 @@ void orthonormalize_subspace_vectors_kernel(
         */
         item_ct1.barrier();
 
-        norm = sycl::sqrt((float)(reduction_helper_real[0]));
+        norm = std::sqrt(reduction_helper_real[0]);
         for (size_type j = tidx; j < num_cols; j += block_size) {
             values[row * stride + j] /= norm;
         }
@@ -397,9 +397,9 @@ void multidot_kernel(
     local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
     const auto tile_block = group::tiled_partition<default_dot_dim>(
         group::this_thread_block(item_ct1));
-    const auto sum =
-        reduce(tile_block, local_res,
-               [](const ValueType &a, const ValueType &b) { return a + b; });
+    const auto sum = ::gko::kernels::dpcpp::reduce(
+        tile_block, local_res,
+        [](const ValueType &a, const ValueType &b) { return a + b; });
     const auto new_rhs = item_ct1.get_group(2) * default_dot_dim + tidy;
     if (tidx == 0 && new_rhs < nrhs && !stop_status[new_rhs].has_stopped()) {
         atomic_add(alpha + new_rhs, sum);
@@ -595,9 +595,8 @@ void compute_omega_kernel(
     if (!stop_status[global_id].has_stopped()) {
         auto thr = omega[global_id];
         omega[global_id] /= tht[global_id];
-        auto absrho =
-            std::abs(thr / (sycl::sqrt((float)(real(tht[global_id]))) *
-                            residual_norm[global_id]));
+        auto absrho = std::abs(thr / (std::sqrt((float)(real(tht[global_id]))) *
+                                      residual_norm[global_id]));
 
         if (absrho < kappa) {
             omega[global_id] *= kappa / absrho;
@@ -627,7 +626,8 @@ namespace {
 
 
 template <typename ValueType>
-void initialize_m(const size_type nrhs, matrix::Dense<ValueType> *m,
+void initialize_m(std::shared_ptr<const DpcppExecutor> exec,
+                  const size_type nrhs, matrix::Dense<ValueType> *m,
                   Array<stopping_status> *stop_status)
 {
     const auto subspace_dim = m->get_size()[0];
@@ -641,27 +641,47 @@ void initialize_m(const size_type nrhs, matrix::Dense<ValueType> *m,
 
 
 template <typename ValueType>
-void initialize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors,
+void initialize_subspace_vectors(std::shared_ptr<const DpcppExecutor> exec,
+                                 matrix::Dense<ValueType> *subspace_vectors,
                                  bool deterministic)
 {
     if (deterministic) {
         auto subspace_vectors_data = matrix_data<ValueType>(
-            subspace_vectors->get_size(), std::normal_distribution<>(0.0, 1.0),
+            subspace_vectors->get_size(),
+            std::normal_distribution<remove_complex<ValueType>>(0.0, 1.0),
             std::ranlux48(15));
         subspace_vectors->read(subspace_vectors_data);
     } else {
-        auto gen =
-            curand::rand_generator(time(NULL), CURAND_RNG_PSEUDO_DEFAULT);
-        curand::rand_vector(
-            gen,
-            subspace_vectors->get_size()[0] * subspace_vectors->get_stride(),
-            0.0, 1.0, subspace_vectors->get_values());
+        auto size = subspace_vectors->get_size();
+        auto stride = subspace_vectors->get_stride();
+        auto values = subspace_vectors->get_values();
+        exec->get_queue()->submit([&](sycl::handler &cgh) {
+            cgh.parallel_for(
+                sycl::range<1>(size[0] * size[1]), [=](sycl::item<1> idx) {
+                    std::uint64_t offset = idx.get_linear_id();
+
+                    // Create minstd_rand engine
+                    oneapi::dpl::minstd_rand engine(132, offset);
+
+                    // Create uniform_real_distribution distribution
+                    oneapi::dpl::uniform_real_distribution<
+                        remove_complex<ValueType>>
+                        distr;
+
+                    // Generate random number
+                    auto res = distr(engine);
+
+                    // Store results to x_acc
+                    values[idx / size[1] * stride + idx % size[1]] = res;
+                });
+        });
     }
 }
 
 
 template <typename ValueType>
-void orthonormalize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors)
+void orthonormalize_subspace_vectors(std::shared_ptr<const DpcppExecutor> exec,
+                                     matrix::Dense<ValueType> *subspace_vectors)
 {
     orthonormalize_subspace_vectors_kernel<default_block_size>(
         1, default_block_size, 0, exec->get_queue(),
@@ -671,7 +691,8 @@ void orthonormalize_subspace_vectors(matrix::Dense<ValueType> *subspace_vectors)
 
 
 template <typename ValueType>
-void solve_lower_triangular(const size_type nrhs,
+void solve_lower_triangular(std::shared_ptr<const DpcppExecutor> exec,
+                            const size_type nrhs,
                             const matrix::Dense<ValueType> *m,
                             const matrix::Dense<ValueType> *f,
                             matrix::Dense<ValueType> *c,
@@ -702,7 +723,7 @@ void update_g_and_u(std::shared_ptr<const DpcppExecutor> exec,
     const auto p_stride = p->get_stride();
 
     const dim3 grid_dim(ceildiv(nrhs, default_dot_dim),
-                        exec->get_num_multiprocessor() * 2);
+                        exec->get_num_computing_units() * 2);
     const dim3 block_dim(default_dot_dim, default_dot_dim);
 
     for (size_type i = 0; i < k; i++) {
@@ -714,9 +735,8 @@ void update_g_and_u(std::shared_ptr<const DpcppExecutor> exec,
                             nrhs, p_i, g_k->get_values(), g_k->get_stride(),
                             alpha->get_values(), stop_status->get_const_data());
         } else {
-            cublas::dot(exec->get_cublas_handle(), size, p_i, 1,
-                        g_k->get_values(), g_k->get_stride(),
-                        alpha->get_values());
+            onemkl::dot(*exec->get_queue(), size, p_i, 1, g_k->get_values(),
+                        g_k->get_stride(), alpha->get_values());
         }
         update_g_k_and_u_kernel<default_block_size>(
             ceildiv(size * g_k->get_stride(), default_block_size),
@@ -746,7 +766,7 @@ void update_m(std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
     const auto m_stride = m->get_stride();
 
     const dim3 grid_dim(ceildiv(nrhs, default_dot_dim),
-                        exec->get_num_multiprocessor() * 2);
+                        exec->get_num_computing_units() * 2);
     const dim3 block_dim(default_dot_dim, default_dot_dim);
 
     for (size_type i = k; i < subspace_dim; i++) {
@@ -759,7 +779,7 @@ void update_m(std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
                             g_k->get_stride(), m_i,
                             stop_status->get_const_data());
         } else {
-            cublas::dot(exec->get_cublas_handle(), size, p_i, 1,
+            onemkl::dot(*exec->get_queue(), size, p_i, 1,
                         g_k->get_const_values(), g_k->get_stride(), m_i);
         }
     }
@@ -801,9 +821,9 @@ void initialize(std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
                 matrix::Dense<ValueType> *subspace_vectors, bool deterministic,
                 Array<stopping_status> *stop_status)
 {
-    initialize_m(nrhs, m, stop_status);
-    initialize_subspace_vectors(subspace_vectors, deterministic);
-    orthonormalize_subspace_vectors(subspace_vectors);
+    initialize_m(exec, nrhs, m, stop_status);
+    initialize_subspace_vectors(exec, subspace_vectors, deterministic);
+    orthonormalize_subspace_vectors(exec, subspace_vectors);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_IDR_INITIALIZE_KERNEL);
@@ -818,7 +838,7 @@ void step_1(std::shared_ptr<const DpcppExecutor> exec, const size_type nrhs,
             matrix::Dense<ValueType> *v,
             const Array<stopping_status> *stop_status)
 {
-    solve_lower_triangular(nrhs, m, f, c, stop_status);
+    solve_lower_triangular(exec, nrhs, m, f, c, stop_status);
 
     const auto num_rows = v->get_size()[0];
     const auto subspace_dim = m->get_size()[0];
diff --git a/dpcpp/test/solver/CMakeLists.txt b/dpcpp/test/solver/CMakeLists.txt
index 44046002504..b4f0e4059ca 100644
--- a/dpcpp/test/solver/CMakeLists.txt
+++ b/dpcpp/test/solver/CMakeLists.txt
@@ -1,2 +1,3 @@
 ginkgo_create_test(gmres_kernels)
 ginkgo_create_test(cb_gmres_kernels)
+ginkgo_create_test(idr_kernels)
diff --git a/dpcpp/test/solver/idr_kernels.cpp b/dpcpp/test/solver/idr_kernels.cpp
index 9316ec02e7d..e7202ea5a27 100644
--- a/dpcpp/test/solver/idr_kernels.cpp
+++ b/dpcpp/test/solver/idr_kernels.cpp
@@ -55,10 +55,19 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace {
 
 
+// use another alias to avoid conflict name in the Idr
+template <typename Precision, typename OutputType = Precision>
+using rr = typename gko::test::reduction_factor<Precision, OutputType>;
+
 class Idr : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Dense<>;
-    using Solver = gko::solver::Idr<>;
+#if GINKGO_DPCPP_SINGLE_MODE
+    using value_type = float;
+#else
+    using value_type = double;
+#endif
+    using Mtx = gko::matrix::Dense<value_type>;
+    using Solver = gko::solver::Idr<value_type>;
 
     Idr() : rand_engine(30) {}
 
@@ -97,7 +106,8 @@ class Idr : public ::testing::Test {
         return gko::test::generate_random_matrix<Mtx>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(num_cols, num_cols),
-            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
+            std::normal_distribution<gko::remove_complex<value_type>>(0.0, 1.0),
+            rand_engine, ref);
     }
 
     void initialize_data()
@@ -215,8 +225,8 @@ TEST_F(Idr, IdrInitializeIsEquivalentToRef)
     gko::kernels::dpcpp::idr::initialize(dpcpp, nrhs, d_m.get(), d_p.get(),
                                          true, d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
-    GKO_ASSERT_MTX_NEAR(p, d_p, 1e-14);
+    GKO_ASSERT_MTX_NEAR(m, d_m, rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(p, d_p, rr<value_type>::value);
 }
 
 
@@ -232,8 +242,8 @@ TEST_F(Idr, IdrStep1IsEquivalentToRef)
                                      d_r.get(), d_g.get(), d_c.get(), d_v.get(),
                                      d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(c, d_c, 1e-14);
-    GKO_ASSERT_MTX_NEAR(v, d_v, 1e-14);
+    GKO_ASSERT_MTX_NEAR(c, d_c, rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(v, d_v, rr<value_type>::value);
 }
 
 
@@ -247,7 +257,7 @@ TEST_F(Idr, IdrStep2IsEquivalentToRef)
     gko::kernels::dpcpp::idr::step_2(dpcpp, nrhs, k, d_omega.get(), d_v.get(),
                                      d_c.get(), d_u.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
+    GKO_ASSERT_MTX_NEAR(u, d_u, rr<value_type>::value);
 }
 
 
@@ -263,13 +273,13 @@ TEST_F(Idr, IdrStep3IsEquivalentToRef)
         dpcpp, nrhs, k, d_p.get(), d_g.get(), d_v.get(), d_u.get(), d_m.get(),
         d_f.get(), d_alpha.get(), d_r.get(), d_x.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(g, d_g, 1e-14);
-    GKO_ASSERT_MTX_NEAR(v, d_v, 1e-14);
-    GKO_ASSERT_MTX_NEAR(u, d_u, 1e-14);
-    GKO_ASSERT_MTX_NEAR(m, d_m, 1e-14);
-    GKO_ASSERT_MTX_NEAR(f, d_f, 1e-14);
-    GKO_ASSERT_MTX_NEAR(r, d_r, 1e-14);
-    GKO_ASSERT_MTX_NEAR(x, d_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(g, d_g, rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(v, d_v, rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u, d_u, rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(m, d_m, rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(f, d_f, rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(r, d_r, rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(x, d_x, rr<value_type>::value);
 }
 
 
@@ -285,7 +295,7 @@ TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
                                             d_residual_norm.get(),
                                             d_omega.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(omega, d_omega, 1e-14);
+    GKO_ASSERT_MTX_NEAR(omega, d_omega, rr<value_type>::value);
 }
 
 
@@ -305,8 +315,8 @@ TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     dpcpp_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_b, b, rr<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_x, x, rr<value_type>::value * 10);
 }
 
 
@@ -340,8 +350,8 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     dpcpp_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_b, b, rr<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_x, x, rr<value_type>::value * 10);
 }
 
 
@@ -396,8 +406,8 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     dpcpp_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-13);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-13);
+    GKO_ASSERT_MTX_NEAR(d_b, b, rr<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_x, x, rr<value_type>::value * 10);
 }
 
 

From 4a5434f9662eecc9aa6b1c34e6e47a84052068aa Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 28 Jul 2021 22:18:34 +0200
Subject: [PATCH 0968/1008] add weird barrier in idr and fix precision

---
 dpcpp/solver/idr_kernels.dp.cpp   |  7 ++++---
 dpcpp/test/solver/idr_kernels.cpp | 14 +++++++-------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/dpcpp/solver/idr_kernels.dp.cpp b/dpcpp/solver/idr_kernels.dp.cpp
index 2acfdc953c1..dfd8ffd9af4 100644
--- a/dpcpp/solver/idr_kernels.dp.cpp
+++ b/dpcpp/solver/idr_kernels.dp.cpp
@@ -130,7 +130,8 @@ void orthonormalize_subspace_vectors_kernel(
                 */
                 dot += values[row * stride + j] * conj(values[i * stride + j]);
             }
-
+            // TODO: check with intel why we need this here.
+            item_ct1.barrier();
             reduction_helper[tidx] = dot;
 
             /*
@@ -595,8 +596,8 @@ void compute_omega_kernel(
     if (!stop_status[global_id].has_stopped()) {
         auto thr = omega[global_id];
         omega[global_id] /= tht[global_id];
-        auto absrho = std::abs(thr / (std::sqrt((float)(real(tht[global_id]))) *
-                                      residual_norm[global_id]));
+        auto absrho = std::abs(
+            thr / (std::sqrt(real(tht[global_id])) * residual_norm[global_id]));
 
         if (absrho < kappa) {
             omega[global_id] *= kappa / absrho;
diff --git a/dpcpp/test/solver/idr_kernels.cpp b/dpcpp/test/solver/idr_kernels.cpp
index e7202ea5a27..8fb08303ed8 100644
--- a/dpcpp/test/solver/idr_kernels.cpp
+++ b/dpcpp/test/solver/idr_kernels.cpp
@@ -273,11 +273,11 @@ TEST_F(Idr, IdrStep3IsEquivalentToRef)
         dpcpp, nrhs, k, d_p.get(), d_g.get(), d_v.get(), d_u.get(), d_m.get(),
         d_f.get(), d_alpha.get(), d_r.get(), d_x.get(), d_stop_status.get());
 
-    GKO_ASSERT_MTX_NEAR(g, d_g, rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(g, d_g, 2 * rr<value_type>::value);
     GKO_ASSERT_MTX_NEAR(v, d_v, rr<value_type>::value);
     GKO_ASSERT_MTX_NEAR(u, d_u, rr<value_type>::value);
     GKO_ASSERT_MTX_NEAR(m, d_m, rr<value_type>::value);
-    GKO_ASSERT_MTX_NEAR(f, d_f, rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(f, d_f, 2 * rr<value_type>::value);
     GKO_ASSERT_MTX_NEAR(r, d_r, rr<value_type>::value);
     GKO_ASSERT_MTX_NEAR(x, d_x, rr<value_type>::value);
 }
@@ -287,7 +287,7 @@ TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
 {
     initialize_data();
 
-    double kappa = 0.7;
+    value_type kappa = 0.7;
     gko::kernels::reference::idr::compute_omega(ref, nrhs, kappa, tht.get(),
                                                 residual_norm.get(),
                                                 omega.get(), stop_status.get());
@@ -350,8 +350,8 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     dpcpp_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, rr<value_type>::value * 10);
-    GKO_ASSERT_MTX_NEAR(d_x, x, rr<value_type>::value * 10);
+    GKO_ASSERT_MTX_NEAR(d_b, b, rr<value_type>::value * 100);
+    GKO_ASSERT_MTX_NEAR(d_x, x, rr<value_type>::value * 100);
 }
 
 
@@ -371,8 +371,8 @@ TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
     ref_solver->apply(b.get(), x.get());
     dpcpp_solver->apply(d_b.get(), d_x.get());
 
-    GKO_ASSERT_MTX_NEAR(d_b, b, 1e-12);
-    GKO_ASSERT_MTX_NEAR(d_x, x, 1e-12);
+    GKO_ASSERT_MTX_NEAR(d_b, b, rr<value_type>::value * 500);
+    GKO_ASSERT_MTX_NEAR(d_x, x, rr<value_type>::value * 500);
 }
 
 

From 11a0606613f9847e619b870ae4bd1ddf1716309e Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 29 Jul 2021 15:09:05 +0200
Subject: [PATCH 0969/1008] oneDPL for complex generation

---
 dpcpp/solver/idr_kernels.dp.cpp | 36 ++++++++++++++-------------------
 1 file changed, 15 insertions(+), 21 deletions(-)

diff --git a/dpcpp/solver/idr_kernels.dp.cpp b/dpcpp/solver/idr_kernels.dp.cpp
index dfd8ffd9af4..b9034842ceb 100644
--- a/dpcpp/solver/idr_kernels.dp.cpp
+++ b/dpcpp/solver/idr_kernels.dp.cpp
@@ -653,28 +653,22 @@ void initialize_subspace_vectors(std::shared_ptr<const DpcppExecutor> exec,
             std::ranlux48(15));
         subspace_vectors->read(subspace_vectors_data);
     } else {
-        auto size = subspace_vectors->get_size();
-        auto stride = subspace_vectors->get_stride();
-        auto values = subspace_vectors->get_values();
+        auto seed = time(NULL);
+        auto work = reinterpret_cast<remove_complex<ValueType> *>(
+            subspace_vectors->get_values());
+        auto n =
+            subspace_vectors->get_size()[0] * subspace_vectors->get_stride();
+        n = is_complex<ValueType>() ? 2 * n : n;
         exec->get_queue()->submit([&](sycl::handler &cgh) {
-            cgh.parallel_for(
-                sycl::range<1>(size[0] * size[1]), [=](sycl::item<1> idx) {
-                    std::uint64_t offset = idx.get_linear_id();
-
-                    // Create minstd_rand engine
-                    oneapi::dpl::minstd_rand engine(132, offset);
-
-                    // Create uniform_real_distribution distribution
-                    oneapi::dpl::uniform_real_distribution<
-                        remove_complex<ValueType>>
-                        distr;
-
-                    // Generate random number
-                    auto res = distr(engine);
-
-                    // Store results to x_acc
-                    values[idx / size[1] * stride + idx % size[1]] = res;
-                });
+            cgh.parallel_for(sycl::range<1>(n), [=](sycl::item<1> idx) {
+                std::uint64_t offset = idx.get_linear_id();
+                oneapi::dpl::minstd_rand engine(seed, offset);
+                oneapi::dpl::normal_distribution<remove_complex<ValueType>>
+                    distr(0, 1);
+                auto res = distr(engine);
+
+                work[idx] = res;
+            });
         });
     }
 }

From b9e62443a88c514edc0247af71a6ff362021b3ce Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 3 Aug 2021 12:01:43 +0200
Subject: [PATCH 0970/1008] relax the error bound of hessenberg

---
 dpcpp/test/solver/gmres_kernels.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dpcpp/test/solver/gmres_kernels.cpp b/dpcpp/test/solver/gmres_kernels.cpp
index 0cefc2e2d3a..72f7ba0ae06 100644
--- a/dpcpp/test/solver/gmres_kernels.cpp
+++ b/dpcpp/test/solver/gmres_kernels.cpp
@@ -282,7 +282,7 @@ TEST_F(Gmres, DpcppGmresStep1OnSingleRHSIsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(d_residual_norm_collection, residual_norm_collection,
                         r<value_type>::value);
     GKO_ASSERT_MTX_NEAR(d_hessenberg_iter, hessenberg_iter,
-                        r<value_type>::value);
+                        2 * r<value_type>::value);
     GKO_ASSERT_MTX_NEAR(d_krylov_bases, krylov_bases, r<value_type>::value);
     GKO_ASSERT_ARRAY_EQ(*d_final_iter_nums, *final_iter_nums);
 }

From d49c7be71d00558316ff2a06feeb6243d967ede9 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 3 Aug 2021 20:41:33 +0200
Subject: [PATCH 0971/1008] rm dpct warning, use ref of uninitialized_array,
 reduce test code dup

Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 cuda/test/solver/idr_kernels.cpp  | 47 +++++------------------
 dpcpp/solver/idr_kernels.dp.cpp   | 63 ++++++++-----------------------
 dpcpp/test/solver/idr_kernels.cpp | 47 +++++------------------
 hip/test/solver/idr_kernels.cpp   | 47 +++++------------------
 omp/test/solver/idr_kernels.cpp   | 47 +++++------------------
 5 files changed, 51 insertions(+), 200 deletions(-)

diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index 9aa925187a4..cc075f9191d 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -67,9 +67,6 @@ class Idr : public ::testing::Test {
         ref = gko::ReferenceExecutor::create();
         cuda = gko::CudaExecutor::create(0, ref);
 
-        mtx = gen_mtx(123, 123);
-        d_mtx = Mtx::create(cuda);
-        d_mtx->copy_from(mtx.get());
         cuda_idr_factory =
             Solver::build()
                 .with_deterministic(true)
@@ -100,11 +97,11 @@ class Idr : public ::testing::Test {
             std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
     }
 
-    void initialize_data()
+    void initialize_data(int size = 597, int input_nrhs = 17)
     {
-        int size = 597;
-        nrhs = 17;
+        nrhs = input_nrhs;
         int s = 4;
+        mtx = gen_mtx(size, size);
         x = gen_mtx(size, nrhs);
         b = gen_mtx(size, nrhs);
         r = gen_mtx(size, nrhs);
@@ -125,6 +122,7 @@ class Idr : public ::testing::Test {
             stop_status->get_data()[i].reset();
         }
 
+        d_mtx = Mtx::create(cuda);
         d_x = Mtx::create(cuda);
         d_b = Mtx::create(cuda);
         d_r = Mtx::create(cuda);
@@ -142,6 +140,7 @@ class Idr : public ::testing::Test {
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(cuda));
 
+        d_mtx->copy_from(mtx.get());
         d_x->copy_from(x.get());
         d_b->copy_from(b.get());
         d_r->copy_from(r.get());
@@ -291,16 +290,9 @@ TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 1;
+    initialize_data(123, 1);
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto cuda_solver = cuda_idr_factory->generate(d_mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(cuda);
-    auto d_x = Mtx::create(cuda);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     cuda_solver->apply(d_b.get(), d_x.get());
@@ -312,8 +304,7 @@ TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 1;
+    initialize_data(123, 1);
     cuda_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -330,12 +321,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .on(ref);
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto cuda_solver = cuda_idr_factory->generate(d_mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(cuda);
-    auto d_x = Mtx::create(cuda);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     cuda_solver->apply(d_b.get(), d_x.get());
@@ -347,16 +332,9 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 16;
+    initialize_data(123, 16);
     auto cuda_solver = cuda_idr_factory->generate(d_mtx);
     auto ref_solver = ref_idr_factory->generate(mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(cuda);
-    auto d_x = Mtx::create(cuda);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     cuda_solver->apply(d_b.get(), d_x.get());
@@ -368,8 +346,7 @@ TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 16;
+    initialize_data(123, 16);
     cuda_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -386,12 +363,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
             .on(ref);
     auto cuda_solver = cuda_idr_factory->generate(d_mtx);
     auto ref_solver = ref_idr_factory->generate(mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(cuda);
-    auto d_x = Mtx::create(cuda);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     cuda_solver->apply(d_b.get(), d_x.get());
diff --git a/dpcpp/solver/idr_kernels.dp.cpp b/dpcpp/solver/idr_kernels.dp.cpp
index b9034842ceb..05d63a840ff 100644
--- a/dpcpp/solver/idr_kernels.dp.cpp
+++ b/dpcpp/solver/idr_kernels.dp.cpp
@@ -111,44 +111,31 @@ template <size_type block_size, typename ValueType>
 void orthonormalize_subspace_vectors_kernel(
     size_type num_rows, size_type num_cols, ValueType *__restrict__ values,
     size_type stride, sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, block_size> *reduction_helper_array,
+    UninitializedArray<ValueType, block_size> &reduction_helper_array,
     remove_complex<ValueType> *reduction_helper_real)
 {
     const auto tidx = thread::get_thread_id_flat(item_ct1);
 
-
-    ValueType *__restrict__ reduction_helper = (*reduction_helper_array);
-
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
 
     for (size_type row = 0; row < num_rows; row++) {
         for (size_type i = 0; i < row; i++) {
             auto dot = zero<ValueType>();
+            // TODO: check with intel why we need this here.
+            // Is it from we use updated the value even if it is on the same
+            // thread?
+            item_ct1.barrier();
             for (size_type j = tidx; j < num_cols; j += block_size) {
-                /*
-                DPCT1007:5: Migration of this CUDA API is not supported by the
-                Intel(R) DPC++ Compatibility Tool.
-                */
                 dot += values[row * stride + j] * conj(values[i * stride + j]);
             }
-            // TODO: check with intel why we need this here.
-            item_ct1.barrier();
+
             reduction_helper[tidx] = dot;
 
-            /*
-            DPCT1065:3: Consider replacing sycl::nd_item::barrier() with
-            sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
-            better performance, if there is no access to global memory.
-            */
-            item_ct1.barrier();
+            item_ct1.barrier(sycl::access::fence_space::local_space);
             ::gko::kernels::dpcpp::reduce(
                 group::this_thread_block(item_ct1), reduction_helper,
                 [](const ValueType &a, const ValueType &b) { return a + b; });
-            /*
-            DPCT1065:4: Consider replacing sycl::nd_item::barrier() with
-            sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
-            better performance, if there is no access to global memory.
-            */
-            item_ct1.barrier();
+            item_ct1.barrier(sycl::access::fence_space::local_space);
 
             dot = reduction_helper[0];
             for (size_type j = tidx; j < num_cols; j += block_size) {
@@ -163,22 +150,12 @@ void orthonormalize_subspace_vectors_kernel(
 
         reduction_helper_real[tidx] = norm;
 
-        /*
-        DPCT1065:1: Consider replacing sycl::nd_item::barrier() with
-        sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
-        better performance, if there is no access to global memory.
-        */
-        item_ct1.barrier();
+        item_ct1.barrier(sycl::access::fence_space::local_space);
         ::gko::kernels::dpcpp::reduce(
             group::this_thread_block(item_ct1), reduction_helper_real,
             [](const remove_complex<ValueType> &a,
                const remove_complex<ValueType> &b) { return a + b; });
-        /*
-        DPCT1065:2: Consider replacing sycl::nd_item::barrier() with
-        sycl::nd_item::barrier(sycl::access::fence_space::local_space) for
-        better performance, if there is no access to global memory.
-        */
-        item_ct1.barrier();
+        item_ct1.barrier(sycl::access::fence_space::local_space);
 
         norm = std::sqrt(reduction_helper_real[0]);
         for (size_type j = tidx; j < num_cols; j += block_size) {
@@ -206,10 +183,8 @@ void orthonormalize_subspace_vectors_kernel(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 orthonormalize_subspace_vectors_kernel<block_size>(
                     num_rows, num_cols, values, stride, item_ct1,
-                    (UninitializedArray<ValueType, block_size> *)
-                        reduction_helper_array_acc_ct1.get_pointer(),
-                    (remove_complex<ValueType> *)
-                        reduction_helper_real_acc_ct1.get_pointer());
+                    *reduction_helper_array_acc_ct1.get_pointer(),
+                    reduction_helper_real_acc_ct1.get_pointer().get());
             });
     });
 }
@@ -377,7 +352,6 @@ void multidot_kernel(
                              : (item_ct1.get_group(1) + 1) * num;
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
-
     ValueType *__restrict__ reduction_helper = (*reduction_helper_array);
 
     ValueType local_res = zero<ValueType>();
@@ -389,12 +363,7 @@ void multidot_kernel(
         }
     }
     reduction_helper[tidx * (default_dot_dim + 1) + tidy] = local_res;
-    /*
-    DPCT1065:10: Consider replacing sycl::nd_item::barrier() with
-    sycl::nd_item::barrier(sycl::access::fence_space::local_space) for better
-    performance, if there is no access to global memory.
-    */
-    item_ct1.barrier();
+    item_ct1.barrier(sycl::access::fence_space::local_space);
     local_res = reduction_helper[tidy * (default_dot_dim + 1) + tidx];
     const auto tile_block = group::tiled_partition<default_dot_dim>(
         group::this_thread_block(item_ct1));
@@ -426,9 +395,7 @@ void multidot_kernel(dim3 grid, dim3 block, size_t dynamic_shared_memory,
                 multidot_kernel(
                     num_rows, nrhs, p_i, g_k, g_k_stride, alpha, stop_status,
                     item_ct1,
-                    (UninitializedArray<ValueType, default_dot_dim *(
-                                                       default_dot_dim + 1)> *)
-                        reduction_helper_array_acc_ct1.get_pointer());
+                    reduction_helper_array_acc_ct1.get_pointer().get());
             });
     });
 }
diff --git a/dpcpp/test/solver/idr_kernels.cpp b/dpcpp/test/solver/idr_kernels.cpp
index 8fb08303ed8..a9a86b941e6 100644
--- a/dpcpp/test/solver/idr_kernels.cpp
+++ b/dpcpp/test/solver/idr_kernels.cpp
@@ -76,9 +76,6 @@ class Idr : public ::testing::Test {
         ref = gko::ReferenceExecutor::create();
         dpcpp = gko::DpcppExecutor::create(0, ref);
 
-        mtx = gen_mtx(123, 123);
-        d_mtx = Mtx::create(dpcpp);
-        d_mtx->copy_from(mtx.get());
         dpcpp_idr_factory =
             Solver::build()
                 .with_deterministic(true)
@@ -110,11 +107,11 @@ class Idr : public ::testing::Test {
             rand_engine, ref);
     }
 
-    void initialize_data()
+    void initialize_data(int size = 597, int input_nrhs = 17)
     {
-        int size = 597;
-        nrhs = 17;
+        nrhs = input_nrhs;
         int s = 4;
+        mtx = gen_mtx(size, size);
         x = gen_mtx(size, nrhs);
         b = gen_mtx(size, nrhs);
         r = gen_mtx(size, nrhs);
@@ -135,6 +132,7 @@ class Idr : public ::testing::Test {
             stop_status->get_data()[i].reset();
         }
 
+        d_mtx = Mtx::create(dpcpp);
         d_x = Mtx::create(dpcpp);
         d_b = Mtx::create(dpcpp);
         d_r = Mtx::create(dpcpp);
@@ -152,6 +150,7 @@ class Idr : public ::testing::Test {
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(dpcpp));
 
+        d_mtx->copy_from(mtx.get());
         d_x->copy_from(x.get());
         d_b->copy_from(b.get());
         d_r->copy_from(r.get());
@@ -301,16 +300,9 @@ TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 1;
+    initialize_data(123, 1);
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto dpcpp_solver = dpcpp_idr_factory->generate(d_mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(dpcpp);
-    auto d_x = Mtx::create(dpcpp);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     dpcpp_solver->apply(d_b.get(), d_x.get());
@@ -322,8 +314,7 @@ TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 1;
+    initialize_data(123, 1);
     dpcpp_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -340,12 +331,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .on(ref);
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto dpcpp_solver = dpcpp_idr_factory->generate(d_mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(dpcpp);
-    auto d_x = Mtx::create(dpcpp);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     dpcpp_solver->apply(d_b.get(), d_x.get());
@@ -357,16 +342,9 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 16;
+    initialize_data(123, 16);
     auto dpcpp_solver = dpcpp_idr_factory->generate(d_mtx);
     auto ref_solver = ref_idr_factory->generate(mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(dpcpp);
-    auto d_x = Mtx::create(dpcpp);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     dpcpp_solver->apply(d_b.get(), d_x.get());
@@ -378,8 +356,7 @@ TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 16;
+    initialize_data(123, 6);
     dpcpp_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -396,12 +373,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
             .on(ref);
     auto dpcpp_solver = dpcpp_idr_factory->generate(d_mtx);
     auto ref_solver = ref_idr_factory->generate(mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(dpcpp);
-    auto d_x = Mtx::create(dpcpp);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     dpcpp_solver->apply(d_b.get(), d_x.get());
diff --git a/hip/test/solver/idr_kernels.cpp b/hip/test/solver/idr_kernels.cpp
index de7f5a14125..4f9cebeefa0 100644
--- a/hip/test/solver/idr_kernels.cpp
+++ b/hip/test/solver/idr_kernels.cpp
@@ -67,9 +67,6 @@ class Idr : public ::testing::Test {
         ref = gko::ReferenceExecutor::create();
         hip = gko::HipExecutor::create(0, ref);
 
-        mtx = gen_mtx(123, 123);
-        d_mtx = Mtx::create(hip);
-        d_mtx->copy_from(mtx.get());
         hip_idr_factory =
             Solver::build()
                 .with_deterministic(true)
@@ -100,11 +97,11 @@ class Idr : public ::testing::Test {
             std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
     }
 
-    void initialize_data()
+    void initialize_data(int size = 597, int input_nrhs = 17)
     {
-        int size = 597;
-        nrhs = 17;
+        nrhs = input_nrhs;
         int s = 4;
+        mtx = gen_mtx(size, size);
         x = gen_mtx(size, nrhs);
         b = gen_mtx(size, nrhs);
         r = gen_mtx(size, nrhs);
@@ -125,6 +122,7 @@ class Idr : public ::testing::Test {
             stop_status->get_data()[i].reset();
         }
 
+        d_mtx = Mtx::create(hip);
         d_x = Mtx::create(hip);
         d_b = Mtx::create(hip);
         d_r = Mtx::create(hip);
@@ -142,6 +140,7 @@ class Idr : public ::testing::Test {
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(hip));
 
+        d_mtx->copy_from(mtx.get());
         d_x->copy_from(x.get());
         d_b->copy_from(b.get());
         d_r->copy_from(r.get());
@@ -291,16 +290,9 @@ TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 1;
+    initialize_data(123, 1);
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto hip_solver = hip_idr_factory->generate(d_mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(hip);
-    auto d_x = Mtx::create(hip);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     hip_solver->apply(d_b.get(), d_x.get());
@@ -312,8 +304,7 @@ TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 1;
+    initialize_data(123, 1);
     hip_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -330,12 +321,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .on(ref);
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto hip_solver = hip_idr_factory->generate(d_mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(hip);
-    auto d_x = Mtx::create(hip);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     hip_solver->apply(d_b.get(), d_x.get());
@@ -347,16 +332,9 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 16;
+    initialize_data(123, 16);
     auto hip_solver = hip_idr_factory->generate(d_mtx);
     auto ref_solver = ref_idr_factory->generate(mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(hip);
-    auto d_x = Mtx::create(hip);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     hip_solver->apply(d_b.get(), d_x.get());
@@ -368,8 +346,7 @@ TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 16;
+    initialize_data(123, 16);
     hip_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -386,12 +363,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
             .on(ref);
     auto hip_solver = hip_idr_factory->generate(d_mtx);
     auto ref_solver = ref_idr_factory->generate(mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(hip);
-    auto d_x = Mtx::create(hip);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     hip_solver->apply(d_b.get(), d_x.get());
diff --git a/omp/test/solver/idr_kernels.cpp b/omp/test/solver/idr_kernels.cpp
index 794d68cf3f2..bf2ec5d8e65 100644
--- a/omp/test/solver/idr_kernels.cpp
+++ b/omp/test/solver/idr_kernels.cpp
@@ -73,9 +73,6 @@ class Idr : public ::testing::Test {
         ref = gko::ReferenceExecutor::create();
         omp = gko::OmpExecutor::create();
 
-        mtx = gen_mtx(123, 123);
-        d_mtx = Mtx::create(omp);
-        d_mtx->copy_from(mtx.get());
         omp_idr_factory =
             Solver::build()
                 .with_deterministic(true)
@@ -106,11 +103,11 @@ class Idr : public ::testing::Test {
             std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
     }
 
-    void initialize_data()
+    void initialize_data(int size = 597, int input_nrhs = 17)
     {
-        int size = 597;
-        nrhs = 17;
+        nrhs = input_nrhs;
         int s = 4;
+        mtx = gen_mtx(size, size);
         x = gen_mtx(size, nrhs);
         b = gen_mtx(size, nrhs);
         r = gen_mtx(size, nrhs);
@@ -131,6 +128,7 @@ class Idr : public ::testing::Test {
             stop_status->get_data()[i].reset();
         }
 
+        d_mtx = Mtx::create(omp);
         d_x = Mtx::create(omp);
         d_b = Mtx::create(omp);
         d_r = Mtx::create(omp);
@@ -148,6 +146,7 @@ class Idr : public ::testing::Test {
         d_stop_status = std::unique_ptr<gko::Array<gko::stopping_status>>(
             new gko::Array<gko::stopping_status>(omp));
 
+        d_mtx->copy_from(mtx.get());
         d_x->copy_from(x.get());
         d_b->copy_from(b.get());
         d_r->copy_from(r.get());
@@ -297,16 +296,9 @@ TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 1;
+    initialize_data(123, 1);
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto omp_solver = omp_idr_factory->generate(d_mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(omp);
-    auto d_x = Mtx::create(omp);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     omp_solver->apply(d_b.get(), d_x.get());
@@ -318,8 +310,7 @@ TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 1;
+    initialize_data(123, 1);
     omp_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -336,12 +327,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
             .on(ref);
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto omp_solver = omp_idr_factory->generate(d_mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(omp);
-    auto d_x = Mtx::create(omp);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     omp_solver->apply(d_b.get(), d_x.get());
@@ -353,16 +338,9 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceOneRHSIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 16;
+    initialize_data(123, 16);
     auto omp_solver = omp_idr_factory->generate(d_mtx);
     auto ref_solver = ref_idr_factory->generate(mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(omp);
-    auto d_x = Mtx::create(omp);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     omp_solver->apply(d_b.get(), d_x.get());
@@ -374,8 +352,7 @@ TEST_F(Idr, IdrIterationMultipleRHSIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
 {
-    int m = 123;
-    int n = 16;
+    initialize_data(123, 16);
     omp_idr_factory =
         Solver::build()
             .with_deterministic(true)
@@ -392,12 +369,6 @@ TEST_F(Idr, IdrIterationWithComplexSubspaceMultipleRHSIsEquivalentToRef)
             .on(ref);
     auto omp_solver = omp_idr_factory->generate(d_mtx);
     auto ref_solver = ref_idr_factory->generate(mtx);
-    auto b = gen_mtx(m, n);
-    auto x = gen_mtx(m, n);
-    auto d_b = Mtx::create(omp);
-    auto d_x = Mtx::create(omp);
-    d_b->copy_from(b.get());
-    d_x->copy_from(x.get());
 
     ref_solver->apply(b.get(), x.get());
     omp_solver->apply(d_b.get(), d_x.get());

From c525aab3ed7f868e06482ccb205655bde3a67318 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 4 Aug 2021 19:19:17 +0200
Subject: [PATCH 0972/1008] pass reference not pointer of UA, overlap shared

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 common/solver/idr_kernels.hpp.inc    |  7 ++--
 dpcpp/components/prefix_sum.dp.hpp   | 20 +++++-----
 dpcpp/components/reduction.dp.hpp    |  8 ++--
 dpcpp/matrix/dense_kernels.dp.cpp    | 56 ++++++++++++++--------------
 dpcpp/solver/cb_gmres_kernels.dp.cpp | 15 +++-----
 dpcpp/solver/gmres_kernels.dp.cpp    | 21 +++++------
 dpcpp/solver/idr_kernels.dp.cpp      | 43 ++++++++++-----------
 dpcpp/test/solver/idr_kernels.cpp    |  4 +-
 8 files changed, 83 insertions(+), 91 deletions(-)

diff --git a/common/solver/idr_kernels.hpp.inc b/common/solver/idr_kernels.hpp.inc
index fd57a31afc7..ee6f0e58d67 100644
--- a/common/solver/idr_kernels.hpp.inc
+++ b/common/solver/idr_kernels.hpp.inc
@@ -59,9 +59,10 @@ __global__
     const auto tidx = thread::get_thread_id_flat();
 
     __shared__ UninitializedArray<ValueType, block_size> reduction_helper_array;
-    ValueType *__restrict__ reduction_helper = reduction_helper_array;
-
-    __shared__ remove_complex<ValueType> reduction_helper_real[block_size];
+    // they are not be used in the same time.
+    ValueType *reduction_helper = reduction_helper_array;
+    auto reduction_helper_real =
+        reinterpret_cast<remove_complex<ValueType> *>(reduction_helper);
 
     for (size_type row = 0; row < num_rows; row++) {
         for (size_type i = 0; i < row; i++) {
diff --git a/dpcpp/components/prefix_sum.dp.hpp b/dpcpp/components/prefix_sum.dp.hpp
index 7eaea344641..0eaf5da3f76 100644
--- a/dpcpp/components/prefix_sum.dp.hpp
+++ b/dpcpp/components/prefix_sum.dp.hpp
@@ -129,13 +129,13 @@ template <std::uint32_t block_size, typename ValueType>
 void start_prefix_sum(size_type num_elements, ValueType *__restrict__ elements,
                       ValueType *__restrict__ block_sum,
                       sycl::nd_item<3> item_ct1,
-                      UninitializedArray<ValueType, block_size> *prefix_helper)
+                      UninitializedArray<ValueType, block_size> &prefix_helper)
 {
     const auto tidx = thread::get_thread_id_flat(item_ct1);
     const auto element_id = item_ct1.get_local_id(2);
 
     // do not need to access the last element when exclusive prefix sum
-    (*prefix_helper)[element_id] =
+    prefix_helper[element_id] =
         (tidx + 1 < num_elements) ? elements[tidx] : zero<ValueType>();
     auto this_block = group::this_thread_block(item_ct1);
     this_block.sync();
@@ -146,7 +146,7 @@ void start_prefix_sum(size_type num_elements, ValueType *__restrict__ elements,
         const auto ai = i * (2 * element_id + 1) - 1;
         const auto bi = i * (2 * element_id + 2) - 1;
         if (bi < block_size) {
-            (*prefix_helper)[bi] += (*prefix_helper)[ai];
+            prefix_helper[bi] += prefix_helper[ai];
         }
         this_block.sync();
     }
@@ -154,9 +154,9 @@ void start_prefix_sum(size_type num_elements, ValueType *__restrict__ elements,
     if (element_id == 0) {
         // Store the total sum except the last block
         if (item_ct1.get_group(2) + 1 < item_ct1.get_group_range(2)) {
-            block_sum[item_ct1.get_group(2)] = (*prefix_helper)[block_size - 1];
+            block_sum[item_ct1.get_group(2)] = prefix_helper[block_size - 1];
         }
-        (*prefix_helper)[block_size - 1] = zero<ValueType>();
+        prefix_helper[block_size - 1] = zero<ValueType>();
     }
 
     this_block.sync();
@@ -167,14 +167,14 @@ void start_prefix_sum(size_type num_elements, ValueType *__restrict__ elements,
         const auto ai = i * (2 * element_id + 1) - 1;
         const auto bi = i * (2 * element_id + 2) - 1;
         if (bi < block_size) {
-            auto tmp = (*prefix_helper)[ai];
-            (*prefix_helper)[ai] = (*prefix_helper)[bi];
-            (*prefix_helper)[bi] += tmp;
+            auto tmp = prefix_helper[ai];
+            prefix_helper[ai] = prefix_helper[bi];
+            prefix_helper[bi] += tmp;
         }
         this_block.sync();
     }
     if (tidx < num_elements) {
-        elements[tidx] = (*prefix_helper)[element_id];
+        elements[tidx] = prefix_helper[element_id];
     }
 }
 
@@ -193,7 +193,7 @@ void start_prefix_sum(dim3 grid, dim3 block, size_type dynamic_shared_memory,
                          [=](sycl::nd_item<3> item_ct1) {
                              start_prefix_sum<block_size>(
                                  num_elements, elements, block_sum, item_ct1,
-                                 prefix_helper_acc_ct1.get_pointer().get());
+                                 *prefix_helper_acc_ct1.get_pointer());
                          });
     });
 }
diff --git a/dpcpp/components/reduction.dp.hpp b/dpcpp/components/reduction.dp.hpp
index 048dfa4f862..0d47da1490b 100644
--- a/dpcpp/components/reduction.dp.hpp
+++ b/dpcpp/components/reduction.dp.hpp
@@ -205,14 +205,14 @@ template <std::uint32_t cfg, typename ValueType>
 void reduce_add_array(
     size_type size, const ValueType *__restrict__ source,
     ValueType *__restrict__ result, sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *block_sum)
+    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> &block_sum)
 {
     reduce_array<KCFG_1D::decode<1>(cfg)>(
-        size, source, static_cast<ValueType *>((*block_sum)), item_ct1,
+        size, source, static_cast<ValueType *>(block_sum), item_ct1,
         [](const ValueType &x, const ValueType &y) { return x + y; });
 
     if (item_ct1.get_local_id(2) == 0) {
-        result[item_ct1.get_group(2)] = (*block_sum)[0];
+        result[item_ct1.get_group(2)] = block_sum[0];
     }
 }
 
@@ -230,7 +230,7 @@ void reduce_add_array(dim3 grid, dim3 block, size_type dynamic_shared_memory,
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 reduce_add_array<cfg>(size, source, result, item_ct1,
-                                      block_sum_acc_ct1.get_pointer().get());
+                                      *block_sum_acc_ct1.get_pointer());
             });
     });
 }
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index fd14041b645..ea2f24c990e 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -89,7 +89,7 @@ template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename OutType,
 void compute_partial_reduce(
     size_type num_rows, OutType *__restrict__ work, CallableGetValue get_value,
     CallableReduce reduce_op, sycl::nd_item<3> item_ct1,
-    UninitializedArray<OutType, KCFG_1D::decode<0>(cfg)> *tmp_work)
+    UninitializedArray<OutType, KCFG_1D::decode<0>(cfg)> &tmp_work)
 {
     constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
     constexpr auto sg_size = KCFG_1D::decode<1>(cfg);
@@ -101,7 +101,7 @@ void compute_partial_reduce(
     const auto global_id =
         thread::get_thread_id<sg_size, warps_per_block>(item_ct1);
 
-    OutType *tmp_work_array = *tmp_work;
+    OutType *tmp_work_array = tmp_work;
     auto tmp = zero<OutType>();
     for (auto i = global_id; i < num_rows; i += wg_size * num_blocks) {
         tmp = reduce_op(tmp, get_value(i));
@@ -124,7 +124,7 @@ void finalize_reduce_computation(
     size_type size, const ValueType *work, ValueType *result,
     CallableReduce reduce_op, CallableFinalize finalize_op,
     sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *tmp_work)
+    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> &tmp_work)
 {
     constexpr auto wg_size = KCFG_1D::decode<0>(cfg);
     constexpr auto sg_size = KCFG_1D::decode<1>(cfg);
@@ -135,7 +135,7 @@ void finalize_reduce_computation(
     for (auto i = local_id; i < size; i += wg_size) {
         tmp = reduce_op(tmp, work[i]);
     }
-    ValueType *tmp_work_array = *tmp_work;
+    ValueType *tmp_work_array = tmp_work;
     tmp_work_array[local_id] = tmp;
 
     ::gko::kernels::dpcpp::reduce<sg_size>(group::this_thread_block(item_ct1),
@@ -152,7 +152,7 @@ void compute_partial_dot(
     size_type num_rows, const ValueType *__restrict__ x, size_type stride_x,
     const ValueType *__restrict__ y, size_type stride_y,
     ValueType *__restrict__ work, sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *tmp_work)
+    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> &tmp_work)
 {
     compute_partial_reduce<cfg>(
         num_rows, work,
@@ -181,7 +181,7 @@ void compute_partial_dot(dim3 grid, dim3 block, size_type dynamic_shared_memory,
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 compute_partial_dot<cfg>(num_rows, x, stride_x, y, stride_y,
                                          work, item_ct1,
-                                         tmp_work_acc_ct1.get_pointer().get());
+                                         *tmp_work_acc_ct1.get_pointer());
             });
     });
 }
@@ -197,7 +197,7 @@ void compute_partial_conj_dot(
     size_type num_rows, const ValueType *__restrict__ x, size_type stride_x,
     const ValueType *__restrict__ y, size_type stride_y,
     ValueType *__restrict__ work, sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *tmp_work)
+    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> &tmp_work)
 {
     compute_partial_reduce<cfg>(
         num_rows, work,
@@ -225,9 +225,9 @@ void compute_partial_conj_dot(dim3 grid, dim3 block,
 
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                compute_partial_conj_dot<cfg>(
-                    num_rows, x, stride_x, y, stride_y, work, item_ct1,
-                    tmp_work_acc_ct1.get_pointer().get());
+                compute_partial_conj_dot<cfg>(num_rows, x, stride_x, y,
+                                              stride_y, work, item_ct1,
+                                              *tmp_work_acc_ct1.get_pointer());
             });
     });
 }
@@ -242,7 +242,7 @@ template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void finalize_sum_reduce_computation(
     size_type size, const ValueType *work, ValueType *result,
     sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *tmp_work)
+    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> &tmp_work)
 {
     finalize_reduce_computation<cfg>(
         size, work, result,
@@ -267,7 +267,7 @@ void finalize_sum_reduce_computation(dim3 grid, dim3 block,
                          [=](sycl::nd_item<3> item_ct1) {
                              finalize_sum_reduce_computation<cfg>(
                                  size, work, result, item_ct1,
-                                 tmp_work_acc_ct1.get_pointer().get());
+                                 *tmp_work_acc_ct1.get_pointer());
                          });
     });
 }
@@ -283,7 +283,7 @@ void compute_partial_norm2(
     size_type num_rows, const ValueType *__restrict__ x, size_type stride_x,
     remove_complex<ValueType> *__restrict__ work, sycl::nd_item<3> item_ct1,
     UninitializedArray<remove_complex<ValueType>, KCFG_1D::decode<0>(cfg)>
-        *tmp_work)
+        &tmp_work)
 {
     using norm_type = remove_complex<ValueType>;
     compute_partial_reduce<cfg>(
@@ -306,12 +306,12 @@ void compute_partial_norm2(dim3 grid, dim3 block,
                        sycl::access::target::local>
             tmp_work_acc_ct1(cgh);
 
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             compute_partial_norm2<cfg>(
-                                 num_rows, x, stride_x, work, item_ct1,
-                                 tmp_work_acc_ct1.get_pointer().get());
-                         });
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                compute_partial_norm2<cfg>(num_rows, x, stride_x, work,
+                                           item_ct1,
+                                           *tmp_work_acc_ct1.get_pointer());
+            });
     });
 }
 
@@ -325,7 +325,7 @@ template <std::uint32_t cfg = KCFG_1D::encode(256, 16), typename ValueType>
 void finalize_sqrt_reduce_computation(
     size_type size, const ValueType *work, ValueType *result,
     sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> *tmp_work)
+    UninitializedArray<ValueType, KCFG_1D::decode<0>(cfg)> &tmp_work)
 {
     finalize_reduce_computation<cfg>(
         size, work, result,
@@ -351,7 +351,7 @@ void finalize_sqrt_reduce_computation(dim3 grid, dim3 block,
                          [=](sycl::nd_item<3> item_ct1) {
                              finalize_sqrt_reduce_computation<cfg>(
                                  size, work, result, item_ct1,
-                                 tmp_work_acc_ct1.get_pointer().get());
+                                 *tmp_work_acc_ct1.get_pointer());
                          });
     });
 }
@@ -677,21 +677,21 @@ void transpose(const size_type nrows, const size_type ncols,
                const ValueType *__restrict__ in, const size_type in_stride,
                ValueType *__restrict__ out, const size_type out_stride,
                Closure op, sycl::nd_item<3> item_ct1,
-               UninitializedArray<ValueType, sg_size *(sg_size + 1)> *space)
+               UninitializedArray<ValueType, sg_size *(sg_size + 1)> &space)
 {
     auto local_x = item_ct1.get_local_id(2);
     auto local_y = item_ct1.get_local_id(1);
     auto x = item_ct1.get_group(2) * sg_size + local_x;
     auto y = item_ct1.get_group(1) * sg_size + local_y;
     if (y < nrows && x < ncols) {
-        (*space)[local_y * (sg_size + 1) + local_x] = op(in[y * in_stride + x]);
+        space[local_y * (sg_size + 1) + local_x] = op(in[y * in_stride + x]);
     }
 
     item_ct1.barrier(sycl::access::fence_space::local_space);
     x = item_ct1.get_group(1) * sg_size + local_x;
     y = item_ct1.get_group(2) * sg_size + local_y;
     if (y < ncols && x < nrows) {
-        out[y * out_stride + x] = (*space)[local_x * (sg_size + 1) + local_y];
+        out[y * out_stride + x] = space[local_x * (sg_size + 1) + local_y];
     }
 }
 
@@ -701,7 +701,7 @@ void transpose(const size_type nrows, const size_type ncols,
                const ValueType *__restrict__ in, const size_type in_stride,
                ValueType *__restrict__ out, const size_type out_stride,
                sycl::nd_item<3> item_ct1,
-               UninitializedArray<ValueType, sg_size *(sg_size + 1)> *space)
+               UninitializedArray<ValueType, sg_size *(sg_size + 1)> &space)
 {
     transpose<sg_size>(
         nrows, ncols, in, in_stride, out, out_stride,
@@ -723,7 +723,7 @@ void transpose(dim3 grid, dim3 block, size_type dynamic_shared_memory,
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 transpose<sg_size>(nrows, ncols, in, in_stride, out, out_stride,
-                                   item_ct1, space_acc_ct1.get_pointer().get());
+                                   item_ct1, *space_acc_ct1.get_pointer());
             });
     });
 }
@@ -739,7 +739,7 @@ void conj_transpose(
     const ValueType *__restrict__ in, const size_type in_stride,
     ValueType *__restrict__ out, const size_type out_stride,
     sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, sg_size *(sg_size + 1)> *space)
+    UninitializedArray<ValueType, sg_size *(sg_size + 1)> &space)
 {
     transpose<sg_size>(
         nrows, ncols, in, in_stride, out, out_stride,
@@ -763,7 +763,7 @@ void conj_transpose(dim3 grid, dim3 block, size_type dynamic_shared_memory,
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 conj_transpose<sg_size>(nrows, ncols, in, in_stride, out,
                                         out_stride, item_ct1,
-                                        space_acc_ct1.get_pointer().get());
+                                        *space_acc_ct1.get_pointer());
             });
     });
 }
diff --git a/dpcpp/solver/cb_gmres_kernels.dp.cpp b/dpcpp/solver/cb_gmres_kernels.dp.cpp
index 0b8359402e6..2dc9d8b84b5 100644
--- a/dpcpp/solver/cb_gmres_kernels.dp.cpp
+++ b/dpcpp/solver/cb_gmres_kernels.dp.cpp
@@ -470,7 +470,7 @@ void multidot_kernel(
     size_type stride_next_krylov, const Accessor3d krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, dot_dim * dot_dim> *reduction_helper_array)
+    UninitializedArray<ValueType, dot_dim * dot_dim> &reduction_helper_array)
 {
     /*
      * In general in this kernel:
@@ -497,8 +497,7 @@ void multidot_kernel(
     const size_type k = item_ct1.get_group(0);
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
-
-    ValueType *__restrict__ reduction_helper = (*reduction_helper_array);
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
 
     ValueType local_res = zero<ValueType>();
     if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
@@ -549,8 +548,7 @@ void multidot_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
                     num_rows, num_cols, next_krylov_basis, stride_next_krylov,
                     krylov_bases, hessenberg_iter, stride_hessenberg,
                     stop_status, item_ct1,
-                    (UninitializedArray<ValueType, dot_dim * dot_dim> *)
-                        reduction_helper_array_acc_ct1.get_pointer());
+                    *reduction_helper_array_acc_ct1.get_pointer());
             });
     });
 }
@@ -562,7 +560,7 @@ void singledot_kernel(
     size_type stride_next_krylov, const Accessor3d krylov_bases,
     ValueType *__restrict__ hessenberg_iter, size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, block_size> *reduction_helper_array)
+    UninitializedArray<ValueType, block_size> &reduction_helper_array)
 {
     /*
      * In general in this kernel:
@@ -585,7 +583,7 @@ void singledot_kernel(
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
 
-    ValueType *__restrict__ reduction_helper = (*reduction_helper_array);
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
 
     ValueType local_res = zero<ValueType>();
     if (!stop_status[col_idx].has_stopped()) {
@@ -630,8 +628,7 @@ void singledot_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
                     num_rows, next_krylov_basis, stride_next_krylov,
                     krylov_bases, hessenberg_iter, stride_hessenberg,
                     stop_status, item_ct1,
-                    (UninitializedArray<ValueType, block_size> *)
-                        reduction_helper_array_acc_ct1.get_pointer());
+                    *reduction_helper_array_acc_ct1.get_pointer());
             });
     });
 }
diff --git a/dpcpp/solver/gmres_kernels.dp.cpp b/dpcpp/solver/gmres_kernels.dp.cpp
index e27a7ef9f65..55369b09e9a 100644
--- a/dpcpp/solver/gmres_kernels.dp.cpp
+++ b/dpcpp/solver/gmres_kernels.dp.cpp
@@ -271,7 +271,7 @@ void update_hessenberg_2_kernel(
     size_type stride_next_krylov, ValueType *__restrict__ hessenberg_iter,
     size_type stride_hessenberg,
     const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, block_size> *reduction_helper_array)
+    UninitializedArray<ValueType, block_size> &reduction_helper_array)
 {
     const auto tidx = item_ct1.get_local_id(2);
     const auto col_idx = item_ct1.get_group(2);
@@ -279,7 +279,7 @@ void update_hessenberg_2_kernel(
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
 
-    ValueType *__restrict__ reduction_helper = (*reduction_helper_array);
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
 
     if (col_idx < num_cols && !stop_status[col_idx].has_stopped()) {
         ValueType local_res{};
@@ -317,15 +317,14 @@ void update_hessenberg_2_kernel(
                        sycl::access::target::local>
             reduction_helper_array_acc_ct1(cgh);
 
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                update_hessenberg_2_kernel<block_size>(
-                    iter, num_rows, num_cols, next_krylov_basis,
-                    stride_next_krylov, hessenberg_iter, stride_hessenberg,
-                    stop_status, item_ct1,
-                    (UninitializedArray<ValueType, block_size> *)
-                        reduction_helper_array_acc_ct1.get_pointer());
-            });
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             update_hessenberg_2_kernel<block_size>(
+                                 iter, num_rows, num_cols, next_krylov_basis,
+                                 stride_next_krylov, hessenberg_iter,
+                                 stride_hessenberg, stop_status, item_ct1,
+                                 *reduction_helper_array_acc_ct1.get_pointer());
+                         });
     });
 }
 
diff --git a/dpcpp/solver/idr_kernels.dp.cpp b/dpcpp/solver/idr_kernels.dp.cpp
index 05d63a840ff..782fb04a8f9 100644
--- a/dpcpp/solver/idr_kernels.dp.cpp
+++ b/dpcpp/solver/idr_kernels.dp.cpp
@@ -111,24 +111,25 @@ template <size_type block_size, typename ValueType>
 void orthonormalize_subspace_vectors_kernel(
     size_type num_rows, size_type num_cols, ValueType *__restrict__ values,
     size_type stride, sycl::nd_item<3> item_ct1,
-    UninitializedArray<ValueType, block_size> &reduction_helper_array,
-    remove_complex<ValueType> *reduction_helper_real)
+    UninitializedArray<ValueType, block_size> &reduction_helper_array)
 {
     const auto tidx = thread::get_thread_id_flat(item_ct1);
 
-    ValueType *__restrict__ reduction_helper = reduction_helper_array;
+    // they are not be used in the same time.
+    ValueType *reduction_helper = reduction_helper_array;
+    auto reduction_helper_real =
+        reinterpret_cast<remove_complex<ValueType> *>(reduction_helper);
 
     for (size_type row = 0; row < num_rows; row++) {
         for (size_type i = 0; i < row; i++) {
             auto dot = zero<ValueType>();
+            for (size_type j = tidx; j < num_cols; j += block_size) {
+                dot += values[row * stride + j] * conj(values[i * stride + j]);
+            }
             // TODO: check with intel why we need this here.
             // Is it from we use updated the value even if it is on the same
             // thread?
             item_ct1.barrier();
-            for (size_type j = tidx; j < num_cols; j += block_size) {
-                dot += values[row * stride + j] * conj(values[i * stride + j]);
-            }
-
             reduction_helper[tidx] = dot;
 
             item_ct1.barrier(sycl::access::fence_space::local_space);
@@ -174,18 +175,13 @@ void orthonormalize_subspace_vectors_kernel(
                        sycl::access_mode::read_write,
                        sycl::access::target::local>
             reduction_helper_array_acc_ct1(cgh);
-        sycl::accessor<remove_complex<ValueType>, 1,
-                       sycl::access_mode::read_write,
-                       sycl::access::target::local>
-            reduction_helper_real_acc_ct1(sycl::range<1>(block_size), cgh);
 
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                orthonormalize_subspace_vectors_kernel<block_size>(
-                    num_rows, num_cols, values, stride, item_ct1,
-                    *reduction_helper_array_acc_ct1.get_pointer(),
-                    reduction_helper_real_acc_ct1.get_pointer().get());
-            });
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             orthonormalize_subspace_vectors_kernel<block_size>(
+                                 num_rows, num_cols, values, stride, item_ct1,
+                                 *reduction_helper_array_acc_ct1.get_pointer());
+                         });
     });
 }
 
@@ -340,7 +336,7 @@ void multidot_kernel(
     ValueType *__restrict__ alpha,
     const stopping_status *__restrict__ stop_status, sycl::nd_item<3> item_ct1,
     UninitializedArray<ValueType, default_dot_dim *(default_dot_dim + 1)>
-        *reduction_helper_array)
+        &reduction_helper_array)
 {
     const auto tidx = item_ct1.get_local_id(2);
     const auto tidy = item_ct1.get_local_id(1);
@@ -352,7 +348,7 @@ void multidot_kernel(
                              : (item_ct1.get_group(1) + 1) * num;
     // Used that way to get around dynamic initialization warning and
     // template error when using `reduction_helper_array` directly in `reduce`
-    ValueType *__restrict__ reduction_helper = (*reduction_helper_array);
+    ValueType *__restrict__ reduction_helper = reduction_helper_array;
 
     ValueType local_res = zero<ValueType>();
     if (rhs < nrhs && !stop_status[rhs].has_stopped()) {
@@ -392,10 +388,9 @@ void multidot_kernel(dim3 grid, dim3 block, size_t dynamic_shared_memory,
 
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                multidot_kernel(
-                    num_rows, nrhs, p_i, g_k, g_k_stride, alpha, stop_status,
-                    item_ct1,
-                    reduction_helper_array_acc_ct1.get_pointer().get());
+                multidot_kernel(num_rows, nrhs, p_i, g_k, g_k_stride, alpha,
+                                stop_status, item_ct1,
+                                *reduction_helper_array_acc_ct1.get_pointer());
             });
     });
 }
diff --git a/dpcpp/test/solver/idr_kernels.cpp b/dpcpp/test/solver/idr_kernels.cpp
index a9a86b941e6..ef5afb132b1 100644
--- a/dpcpp/test/solver/idr_kernels.cpp
+++ b/dpcpp/test/solver/idr_kernels.cpp
@@ -276,8 +276,8 @@ TEST_F(Idr, IdrStep3IsEquivalentToRef)
     GKO_ASSERT_MTX_NEAR(v, d_v, rr<value_type>::value);
     GKO_ASSERT_MTX_NEAR(u, d_u, rr<value_type>::value);
     GKO_ASSERT_MTX_NEAR(m, d_m, rr<value_type>::value);
-    GKO_ASSERT_MTX_NEAR(f, d_f, 2 * rr<value_type>::value);
-    GKO_ASSERT_MTX_NEAR(r, d_r, rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(f, d_f, 13 * rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(r, d_r, 2 * rr<value_type>::value);
     GKO_ASSERT_MTX_NEAR(x, d_x, rr<value_type>::value);
 }
 

From f0892e28f5e10a683d9a437b09a28a621e06d524 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 5 Aug 2021 20:06:27 +0200
Subject: [PATCH 0973/1008] fix idr race condition

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 common/solver/idr_kernels.hpp.inc |  9 ++++-----
 dpcpp/solver/idr_kernels.dp.cpp   | 11 ++++-------
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/common/solver/idr_kernels.hpp.inc b/common/solver/idr_kernels.hpp.inc
index ee6f0e58d67..52e9a3313a7 100644
--- a/common/solver/idr_kernels.hpp.inc
+++ b/common/solver/idr_kernels.hpp.inc
@@ -71,9 +71,9 @@ __global__
                 dot += values[row * stride + j] * conj(values[i * stride + j]);
             }
 
-            reduction_helper[tidx] = dot;
-
+            // Ensure already finish reading this shared memory
             __syncthreads();
+            reduction_helper[tidx] = dot;
             reduce(
                 group::this_thread_block(), reduction_helper,
                 [](const ValueType &a, const ValueType &b) { return a + b; });
@@ -89,10 +89,9 @@ __global__
         for (size_type j = tidx; j < num_cols; j += block_size) {
             norm += squared_norm(values[row * stride + j]);
         }
-
-        reduction_helper_real[tidx] = norm;
-
+        // Ensure already finish reading this shared memory
         __syncthreads();
+        reduction_helper_real[tidx] = norm;
         reduce(group::this_thread_block(), reduction_helper_real,
                [](const remove_complex<ValueType> &a,
                   const remove_complex<ValueType> &b) { return a + b; });
diff --git a/dpcpp/solver/idr_kernels.dp.cpp b/dpcpp/solver/idr_kernels.dp.cpp
index 782fb04a8f9..a4f18019128 100644
--- a/dpcpp/solver/idr_kernels.dp.cpp
+++ b/dpcpp/solver/idr_kernels.dp.cpp
@@ -126,13 +126,10 @@ void orthonormalize_subspace_vectors_kernel(
             for (size_type j = tidx; j < num_cols; j += block_size) {
                 dot += values[row * stride + j] * conj(values[i * stride + j]);
             }
-            // TODO: check with intel why we need this here.
-            // Is it from we use updated the value even if it is on the same
-            // thread?
-            item_ct1.barrier();
-            reduction_helper[tidx] = dot;
 
+            // Ensure already finish reading this shared memory
             item_ct1.barrier(sycl::access::fence_space::local_space);
+            reduction_helper[tidx] = dot;
             ::gko::kernels::dpcpp::reduce(
                 group::this_thread_block(item_ct1), reduction_helper,
                 [](const ValueType &a, const ValueType &b) { return a + b; });
@@ -149,9 +146,9 @@ void orthonormalize_subspace_vectors_kernel(
             norm += squared_norm(values[row * stride + j]);
         }
 
-        reduction_helper_real[tidx] = norm;
-
+        // Ensure already finish reading this shared memory
         item_ct1.barrier(sycl::access::fence_space::local_space);
+        reduction_helper_real[tidx] = norm;
         ::gko::kernels::dpcpp::reduce(
             group::this_thread_block(item_ct1), reduction_helper_real,
             [](const remove_complex<ValueType> &a,

From 5860216690b3079125b8eb5207e8e5a5d691aa34 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 22 Jul 2021 00:32:32 +0200
Subject: [PATCH 0974/1008] auto

---
 dpcpp/matrix/coo_kernels.dp.cpp          |  568 +++-
 dpcpp/matrix/csr_kernels.dp.cpp          | 3571 ++++++++++++++++++----
 dpcpp/matrix/diagonal_kernels.dp.cpp     |  320 +-
 dpcpp/matrix/fbcsr_kernels.dp.cpp        |   24 +-
 dpcpp/matrix/hybrid_kernels.dp.cpp       |  267 +-
 dpcpp/matrix/sellp_kernels.dp.cpp        |  499 ++-
 dpcpp/matrix/sparsity_csr_kernels.dp.cpp |   34 +-
 dpcpp/test/matrix/coo_kernels.cpp        |  405 +++
 dpcpp/test/matrix/csr_kernels.cpp        |  839 ++++-
 dpcpp/test/matrix/diagonal_kernels.cpp   |  278 ++
 dpcpp/test/matrix/fbcsr_kernels.cpp      |   90 +
 dpcpp/test/matrix/hybrid_kernels.cpp     |  297 ++
 dpcpp/test/matrix/sellp_kernels.cpp      |  407 +++
 13 files changed, 6746 insertions(+), 853 deletions(-)
 create mode 100644 dpcpp/test/matrix/coo_kernels.cpp
 create mode 100644 dpcpp/test/matrix/diagonal_kernels.cpp
 create mode 100644 dpcpp/test/matrix/fbcsr_kernels.cpp
 create mode 100644 dpcpp/test/matrix/hybrid_kernels.cpp
 create mode 100644 dpcpp/test/matrix/sellp_kernels.cpp

diff --git a/dpcpp/matrix/coo_kernels.dp.cpp b/dpcpp/matrix/coo_kernels.dp.cpp
index 2bf6987213e..593752c452b 100644
--- a/dpcpp/matrix/coo_kernels.dp.cpp
+++ b/dpcpp/matrix/coo_kernels.dp.cpp
@@ -33,22 +33,34 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/coo_kernels.hpp"
 
 
+#include <dpcpp/base/math.hpp>
+#include <dpcpp/components/segment_scan.dp.hpp>
+
+
 #include <CL/sycl.hpp>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "core/matrix/dense_kernels.hpp"
+#include "dpcpp/base/config.hpp"
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/base/onemkl_bindings.hpp"
+#include "dpcpp/components/atomic.dp.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/format_conversion.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
 
 
 namespace gko {
 namespace kernels {
 /**
- * @brief DPCPP namespace.
+ * @brief The DPCPP namespace.
  *
  * @ingroup dpcpp
  */
@@ -61,11 +73,432 @@ namespace dpcpp {
 namespace coo {
 
 
+constexpr int default_block_size = 512;
+constexpr int warps_in_block = 4;
+constexpr int spmv_block_size = warps_in_block * config::warp_size;
+
+
+namespace {
+
+
+/**
+ * The device function of COO spmv
+ *
+ * @param nnz  the number of nonzeros in the matrix
+ * @param num_lines  the maximum round of each warp
+ * @param val  the value array of the matrix
+ * @param col  the column index array of the matrix
+ * @param row  the row index array of the matrix
+ * @param b  the input dense vector
+ * @param b_stride  the stride of the input dense vector
+ * @param c  the output dense vector
+ * @param c_stride  the stride of the output dense vector
+ * @param scale  the function on the added value
+ *
+ * @tparam ValueType  type of values stored in the matrix
+ * @tparam IndexType  type of matrix indexes stored in the structure
+ * @tparam Closure  type of the function used to write the result
+ */
+template <int subwarp_size = config::warp_size, typename ValueType,
+          typename IndexType, typename Closure>
+void spmv_kernel(const size_type nnz, const size_type num_lines,
+                 const ValueType *__restrict__ val,
+                 const IndexType *__restrict__ col,
+                 const IndexType *__restrict__ row,
+                 const ValueType *__restrict__ b, const size_type b_stride,
+                 ValueType *__restrict__ c, const size_type c_stride,
+                 Closure scale, sycl::nd_item<3> item_ct1)
+{
+    ValueType temp_val = zero<ValueType>();
+    const auto start =
+        static_cast<size_type>(item_ct1.get_local_range().get(2)) *
+            item_ct1.get_group(2) * item_ct1.get_local_range().get(1) *
+            num_lines +
+        item_ct1.get_local_id(1) * item_ct1.get_local_range().get(2) *
+            num_lines;
+    const auto column_id = item_ct1.get_group(1);
+    size_type num = (nnz > start) * ceildiv(nnz - start, subwarp_size);
+    num = min(num, num_lines);
+    const IndexType ind_start = start + item_ct1.get_local_id(2);
+    const IndexType ind_end = ind_start + (num - 1) * subwarp_size;
+    IndexType ind = ind_start;
+    IndexType curr_row = (ind < nnz) ? row[ind] : 0;
+    const auto tile_block = group::tiled_partition<subwarp_size>(
+        group::this_thread_block(item_ct1));
+    for (; ind < ind_end; ind += subwarp_size) {
+        temp_val += (ind < nnz) ? val[ind] * b[col[ind] * b_stride + column_id]
+                                : zero<ValueType>();
+        auto next_row =
+            (ind + subwarp_size < nnz) ? row[ind + subwarp_size] : row[nnz - 1];
+        // segmented scan
+        if (tile_block.any(curr_row != next_row)) {
+            bool is_first_in_segment =
+                segment_scan<subwarp_size>(tile_block, curr_row, &temp_val);
+            if (is_first_in_segment) {
+                atomic_add(&(c[curr_row * c_stride + column_id]),
+                           scale(temp_val));
+            }
+            temp_val = zero<ValueType>();
+        }
+        curr_row = next_row;
+    }
+    if (num > 0) {
+        ind = ind_end;
+        temp_val += (ind < nnz) ? val[ind] * b[col[ind] * b_stride + column_id]
+                                : zero<ValueType>();
+        // segmented scan
+        bool is_first_in_segment =
+            segment_scan<subwarp_size>(tile_block, curr_row, &temp_val);
+        if (is_first_in_segment) {
+            atomic_add(&(c[curr_row * c_stride + column_id]), scale(temp_val));
+        }
+    }
+}
+
+
+template <typename ValueType, typename IndexType>
+void abstract_spmv(const size_type nnz, const size_type num_lines,
+                   const ValueType *__restrict__ val,
+                   const IndexType *__restrict__ col,
+                   const IndexType *__restrict__ row,
+                   const ValueType *__restrict__ b, const size_type b_stride,
+                   ValueType *__restrict__ c, const size_type c_stride,
+                   sycl::nd_item<3> item_ct1)
+{
+    spmv_kernel(
+        nnz, num_lines, val, col, row, b, b_stride, c, c_stride,
+        [](const ValueType &x) { return x; }, item_ct1);
+}
+
+template <typename ValueType, typename IndexType>
+void abstract_spmv(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                   sycl::queue *stream, const size_type nnz,
+                   const size_type num_lines, const ValueType *val,
+                   const IndexType *col, const IndexType *row,
+                   const ValueType *b, const size_type b_stride, ValueType *c,
+                   const size_type c_stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             abstract_spmv(nnz, num_lines, val, col, row, b,
+                                           b_stride, c, c_stride, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void abstract_spmv(const size_type nnz, const size_type num_lines,
+                   const ValueType *__restrict__ alpha,
+                   const ValueType *__restrict__ val,
+                   const IndexType *__restrict__ col,
+                   const IndexType *__restrict__ row,
+                   const ValueType *__restrict__ b, const size_type b_stride,
+                   ValueType *__restrict__ c, const size_type c_stride,
+                   sycl::nd_item<3> item_ct1)
+{
+    ValueType scale_factor = alpha[0];
+    spmv_kernel(
+        nnz, num_lines, val, col, row, b, b_stride, c, c_stride,
+        [&scale_factor](const ValueType &x) { return scale_factor * x; },
+        item_ct1);
+}
+
+template <typename ValueType, typename IndexType>
+void abstract_spmv(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                   sycl::queue *stream, const size_type nnz,
+                   const size_type num_lines, const ValueType *alpha,
+                   const ValueType *val, const IndexType *col,
+                   const IndexType *row, const ValueType *b,
+                   const size_type b_stride, ValueType *c,
+                   const size_type c_stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             abstract_spmv(nnz, num_lines, alpha, val, col, row,
+                                           b, b_stride, c, c_stride, item_ct1);
+                         });
+    });
+}
+
+
+/**
+ * The device function of COO spmm
+ *
+ * @param nnz  the number of nonzeros in the matrix
+ * @param num_elems  the maximum number of nonzeros in each warp
+ * @param val  the value array of the matrix
+ * @param col  the column index array of the matrix
+ * @param row  the row index array of the matrix
+ * @param num_cols the number of columns of the matrix
+ * @param b  the input dense vector
+ * @param b_stride  the stride of the input dense vector
+ * @param c  the output dense vector
+ * @param c_stride  the stride of the output dense vector
+ * @param scale  the function on the added value
+ *
+ * @tparam ValueType  type of values stored in the matrix
+ * @tparam IndexType  type of matrix indexes stored in the structure
+ * @tparam Closure  type of the function used to write the result
+ */
+template <typename ValueType, typename IndexType, typename Closure>
+void spmm_kernel(const size_type nnz, const size_type num_elems,
+                 const ValueType *__restrict__ val,
+                 const IndexType *__restrict__ col,
+                 const IndexType *__restrict__ row, const size_type num_cols,
+                 const ValueType *__restrict__ b, const size_type b_stride,
+                 ValueType *__restrict__ c, const size_type c_stride,
+                 Closure scale, sycl::nd_item<3> item_ct1)
+{
+    ValueType temp = zero<ValueType>();
+    const auto coo_idx =
+        (static_cast<size_type>(item_ct1.get_local_range().get(1)) *
+             item_ct1.get_group(2) +
+         item_ct1.get_local_id(1)) *
+        num_elems;
+    const auto column_id =
+        item_ct1.get_group(1) * item_ct1.get_local_range().get(2) +
+        item_ct1.get_local_id(2);
+    const auto coo_end =
+        (coo_idx + num_elems > nnz) ? nnz : coo_idx + num_elems;
+    if (column_id < num_cols && coo_idx < nnz) {
+        auto curr_row = row[coo_idx];
+        auto idx = coo_idx;
+        for (; idx < coo_end - 1; idx++) {
+            temp += val[idx] * b[col[idx] * b_stride + column_id];
+            const auto next_row = row[idx + 1];
+            if (next_row != curr_row) {
+                atomic_add(&(c[curr_row * c_stride + column_id]), scale(temp));
+                curr_row = next_row;
+                temp = zero<ValueType>();
+            }
+        }
+        temp += val[idx] * b[col[idx] * b_stride + column_id];
+        atomic_add(&(c[curr_row * c_stride + column_id]), scale(temp));
+    }
+}
+
+
+template <typename ValueType, typename IndexType>
+void abstract_spmm(const size_type nnz, const size_type num_elems,
+                   const ValueType *__restrict__ val,
+                   const IndexType *__restrict__ col,
+                   const IndexType *__restrict__ row, const size_type num_cols,
+                   const ValueType *__restrict__ b, const size_type b_stride,
+                   ValueType *__restrict__ c, const size_type c_stride,
+                   sycl::nd_item<3> item_ct1)
+{
+    spmm_kernel(
+        nnz, num_elems, val, col, row, num_cols, b, b_stride, c, c_stride,
+        [](const ValueType &x) { return x; }, item_ct1);
+}
+
+template <typename ValueType, typename IndexType>
+void abstract_spmm(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                   sycl::queue *stream, const size_type nnz,
+                   const size_type num_elems, const ValueType *val,
+                   const IndexType *col, const IndexType *row,
+                   const size_type num_cols, const ValueType *b,
+                   const size_type b_stride, ValueType *c,
+                   const size_type c_stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                abstract_spmm(nnz, num_elems, val, col, row, num_cols, b,
+                              b_stride, c, c_stride, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void abstract_spmm(const size_type nnz, const size_type num_elems,
+                   const ValueType *__restrict__ alpha,
+                   const ValueType *__restrict__ val,
+                   const IndexType *__restrict__ col,
+                   const IndexType *__restrict__ row, const size_type num_cols,
+                   const ValueType *__restrict__ b, const size_type b_stride,
+                   ValueType *__restrict__ c, const size_type c_stride,
+                   sycl::nd_item<3> item_ct1)
+{
+    ValueType scale_factor = alpha[0];
+    spmm_kernel(
+        nnz, num_elems, val, col, row, num_cols, b, b_stride, c, c_stride,
+        [&scale_factor](const ValueType &x) { return scale_factor * x; },
+        item_ct1);
+}
+
+template <typename ValueType, typename IndexType>
+void abstract_spmm(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                   sycl::queue *stream, const size_type nnz,
+                   const size_type num_elems, const ValueType *alpha,
+                   const ValueType *val, const IndexType *col,
+                   const IndexType *row, const size_type num_cols,
+                   const ValueType *b, const size_type b_stride, ValueType *c,
+                   const size_type c_stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                abstract_spmm(nnz, num_elems, alpha, val, col, row, num_cols, b,
+                              b_stride, c, c_stride, item_ct1);
+            });
+    });
+}
+
+
+}  // namespace
+
+
+namespace kernel {
+
+
+template <typename IndexType>
+void convert_row_idxs_to_ptrs(const IndexType *__restrict__ idxs,
+                              size_type num_nonzeros,
+                              IndexType *__restrict__ ptrs, size_type length,
+                              sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+
+    if (tidx == 0) {
+        ptrs[0] = 0;
+        ptrs[length - 1] = num_nonzeros;
+    }
+
+    if (0 < tidx && tidx < num_nonzeros) {
+        if (idxs[tidx - 1] < idxs[tidx]) {
+            for (auto i = idxs[tidx - 1] + 1; i <= idxs[tidx]; i++) {
+                ptrs[i] = tidx;
+            }
+        }
+    }
+}
+
+template <typename IndexType>
+void convert_row_idxs_to_ptrs(dim3 grid, dim3 block,
+                              gko::size_type dynamic_shared_memory,
+                              sycl::queue *stream, const IndexType *idxs,
+                              size_type num_nonzeros, IndexType *ptrs,
+                              size_type length)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             convert_row_idxs_to_ptrs(idxs, num_nonzeros, ptrs,
+                                                      length, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType>
+void initialize_zero_dense(size_type num_rows, size_type num_cols,
+                           size_type stride, ValueType *__restrict__ result,
+                           sycl::nd_item<3> item_ct1)
+{
+    const auto tidx_x =
+        item_ct1.get_local_id(2) +
+        item_ct1.get_local_range().get(2) * item_ct1.get_group(2);
+    const auto tidx_y =
+        item_ct1.get_local_id(1) +
+        item_ct1.get_local_range().get(1) * item_ct1.get_group(1);
+    if (tidx_x < num_cols && tidx_y < num_rows) {
+        result[tidx_y * stride + tidx_x] = zero<ValueType>();
+    }
+}
+
+template <typename ValueType>
+void initialize_zero_dense(dim3 grid, dim3 block,
+                           gko::size_type dynamic_shared_memory,
+                           sycl::queue *stream, size_type num_rows,
+                           size_type num_cols, size_type stride,
+                           ValueType *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             initialize_zero_dense(num_rows, num_cols, stride,
+                                                   result, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void fill_in_dense(size_type nnz, const IndexType *__restrict__ row_idxs,
+                   const IndexType *__restrict__ col_idxs,
+                   const ValueType *__restrict__ values, size_type stride,
+                   ValueType *__restrict__ result, sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    if (tidx < nnz) {
+        result[stride * row_idxs[tidx] + col_idxs[tidx]] = values[tidx];
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void fill_in_dense(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                   sycl::queue *stream, size_type nnz,
+                   const IndexType *row_idxs, const IndexType *col_idxs,
+                   const ValueType *values, size_type stride, ValueType *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             fill_in_dense(nnz, row_idxs, col_idxs, values,
+                                           stride, result, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(size_type nnz, const ValueType *__restrict__ orig_values,
+                      const IndexType *__restrict__ orig_row_idxs,
+                      const IndexType *__restrict__ orig_col_idxs,
+                      ValueType *__restrict__ diag, sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+
+    if (tidx < nnz) {
+        if (orig_row_idxs[tidx] == orig_col_idxs[tidx]) {
+            diag[orig_row_idxs[tidx]] = orig_values[tidx];
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(dim3 grid, dim3 block,
+                      gko::size_type dynamic_shared_memory, sycl::queue *stream,
+                      size_type nnz, const ValueType *orig_values,
+                      const IndexType *orig_row_idxs,
+                      const IndexType *orig_col_idxs, ValueType *diag)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             extract_diagonal(nnz, orig_values, orig_row_idxs,
+                                              orig_col_idxs, diag, item_ct1);
+                         });
+    });
+}
+
+
+}  // namespace kernel
+
+
 template <typename ValueType, typename IndexType>
 void spmv(std::shared_ptr<const DpcppExecutor> exec,
           const matrix::Coo<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b,
-          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+{
+    dense::fill(exec, c, zero<ValueType>());
+    spmv2(exec, a, b, c);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_SPMV_KERNEL);
 
@@ -76,7 +509,11 @@ void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::Coo<ValueType, IndexType> *a,
                    const matrix::Dense<ValueType> *b,
                    const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+                   matrix::Dense<ValueType> *c)
+{
+    dense::scale(exec, beta, c);
+    advanced_spmv2(exec, alpha, a, b, c);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_ADVANCED_SPMV_KERNEL);
@@ -85,8 +522,35 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void spmv2(std::shared_ptr<const DpcppExecutor> exec,
            const matrix::Coo<ValueType, IndexType> *a,
-           const matrix::Dense<ValueType> *b,
-           matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+           const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+{
+    const auto nnz = a->get_num_stored_elements();
+    const auto b_ncols = b->get_size()[1];
+    const dim3 coo_block(config::warp_size, warps_in_block, 1);
+    const auto nwarps = host_kernel::calculate_nwarps(exec, nnz);
+
+    if (nwarps > 0) {
+        if (b_ncols < 4) {
+            const dim3 coo_grid(ceildiv(nwarps, warps_in_block), b_ncols);
+            int num_lines = ceildiv(nnz, nwarps * config::warp_size);
+            abstract_spmv(coo_grid, coo_block, 0, exec->get_queue(), nnz,
+                          num_lines, a->get_const_values(),
+                          a->get_const_col_idxs(), a->get_const_row_idxs(),
+                          b->get_const_values(), b->get_stride(),
+                          c->get_values(), c->get_stride());
+        } else {
+            int num_elems =
+                ceildiv(nnz, nwarps * config::warp_size) * config::warp_size;
+            const dim3 coo_grid(ceildiv(nwarps, warps_in_block),
+                                ceildiv(b_ncols, config::warp_size));
+            abstract_spmm(coo_grid, coo_block, 0, exec->get_queue(), nnz,
+                          num_elems, a->get_const_values(),
+                          a->get_const_col_idxs(), a->get_const_row_idxs(),
+                          b_ncols, b->get_const_values(), b->get_stride(),
+                          c->get_values(), c->get_stride());
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_SPMV2_KERNEL);
 
@@ -96,7 +560,36 @@ void advanced_spmv2(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Dense<ValueType> *alpha,
                     const matrix::Coo<ValueType, IndexType> *a,
                     const matrix::Dense<ValueType> *b,
-                    matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+                    matrix::Dense<ValueType> *c)
+{
+    const auto nnz = a->get_num_stored_elements();
+    const auto nwarps = host_kernel::calculate_nwarps(exec, nnz);
+    const dim3 coo_block(config::warp_size, warps_in_block, 1);
+    const auto b_ncols = b->get_size()[1];
+
+    if (nwarps > 0) {
+        if (b_ncols < 4) {
+            int num_lines = ceildiv(nnz, nwarps * config::warp_size);
+            const dim3 coo_grid(ceildiv(nwarps, warps_in_block), b_ncols);
+            abstract_spmv(coo_grid, coo_block, 0, exec->get_queue(), nnz,
+                          num_lines, alpha->get_const_values(),
+                          a->get_const_values(), a->get_const_col_idxs(),
+                          a->get_const_row_idxs(), b->get_const_values(),
+                          b->get_stride(), c->get_values(), c->get_stride());
+        } else {
+            int num_elems =
+                ceildiv(nnz, nwarps * config::warp_size) * config::warp_size;
+            const dim3 coo_grid(ceildiv(nwarps, warps_in_block),
+                                ceildiv(b_ncols, config::warp_size));
+            abstract_spmm(coo_grid, coo_block, 0, exec->get_queue(), nnz,
+                          num_elems, alpha->get_const_values(),
+                          a->get_const_values(), a->get_const_col_idxs(),
+                          a->get_const_row_idxs(), b_ncols,
+                          b->get_const_values(), b->get_stride(),
+                          c->get_values(), c->get_stride());
+        }
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_ADVANCED_SPMV2_KERNEL);
@@ -105,15 +598,31 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename IndexType>
 void convert_row_idxs_to_ptrs(std::shared_ptr<const DpcppExecutor> exec,
                               const IndexType *idxs, size_type num_nonzeros,
-                              IndexType *ptrs,
-                              size_type length) GKO_NOT_IMPLEMENTED;
+                              IndexType *ptrs, size_type length)
+{
+    const auto grid_dim = ceildiv(num_nonzeros, default_block_size);
+
+    kernel::convert_row_idxs_to_ptrs(grid_dim, default_block_size, 0,
+                                     exec->get_queue(), idxs, num_nonzeros,
+                                     ptrs, length);
+}
 
 
 template <typename ValueType, typename IndexType>
 void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Coo<ValueType, IndexType> *source,
                     matrix::Csr<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
+{
+    auto num_rows = result->get_size()[0];
+
+    auto row_ptrs = result->get_row_ptrs();
+    const auto nnz = result->get_num_stored_elements();
+
+    const auto source_row_idxs = source->get_const_row_idxs();
+
+    convert_row_idxs_to_ptrs(exec, source_row_idxs, nnz, row_ptrs,
+                             num_rows + 1);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_CONVERT_TO_CSR_KERNEL);
@@ -122,7 +631,28 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Coo<ValueType, IndexType> *source,
-                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
+                      matrix::Dense<ValueType> *result)
+{
+    const auto num_rows = result->get_size()[0];
+    const auto num_cols = result->get_size()[1];
+    const auto stride = result->get_stride();
+
+    const auto nnz = source->get_num_stored_elements();
+
+    const dim3 block_size(config::warp_size,
+                          config::max_block_size / config::warp_size, 1);
+    const dim3 init_grid_dim(ceildiv(num_cols, block_size.x),
+                             ceildiv(num_rows, block_size.y), 1);
+    kernel::initialize_zero_dense(init_grid_dim, block_size, 0,
+                                  exec->get_queue(), num_rows, num_cols, stride,
+                                  result->get_values());
+
+    const auto grid_dim = ceildiv(nnz, default_block_size);
+    kernel::fill_in_dense(
+        grid_dim, default_block_size, 0, exec->get_queue(), nnz,
+        source->get_const_row_idxs(), source->get_const_col_idxs(),
+        source->get_const_values(), stride, result->get_values());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_CONVERT_TO_DENSE_KERNEL);
@@ -131,7 +661,21 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Coo<ValueType, IndexType> *orig,
-                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
+                      matrix::Diagonal<ValueType> *diag)
+{
+    const auto nnz = orig->get_num_stored_elements();
+    const auto diag_size = diag->get_size()[0];
+    const auto num_blocks = ceildiv(nnz, default_block_size);
+
+    const auto orig_values = orig->get_const_values();
+    const auto orig_row_idxs = orig->get_const_row_idxs();
+    const auto orig_col_idxs = orig->get_const_col_idxs();
+    auto diag_values = diag->get_values();
+
+    kernel::extract_diagonal(num_blocks, default_block_size, 0,
+                             exec->get_queue(), nnz, orig_values, orig_row_idxs,
+                             orig_col_idxs, diag_values);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_EXTRACT_DIAGONAL_KERNEL);
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index 747844e21e9..3cc17ae6d98 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -30,52 +30,2064 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/matrix/csr_kernels.hpp"
+#include <algorithm>
+#include <dpcpp/base/math.hpp>
+#include <dpcpp/base/pointer_mode_guard.hpp>
+#include <dpcpp/components/intrinsics.dp.hpp>
+#include <dpcpp/components/merging.dp.hpp>
+#include <dpcpp/components/segment_scan.dp.hpp>
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/array.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/ell.hpp>
+#include <ginkgo/core/matrix/hybrid.hpp>
+#include <ginkgo/core/matrix/sellp.hpp>
+
+
+#include "core/components/fill_array.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/csr_builder.hpp"
+#include "core/matrix/dense_kernels.hpp"
+#include "core/synthesizer/implementation_selection.hpp"
+#include "dpcpp/base/config.hpp"
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/base/dpct.hpp"
+#include "dpcpp/base/onemkl_bindings.hpp"
+#include "dpcpp/components/atomic.dp.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
+#include "dpcpp/components/reduction.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
+#include "dpcpp/components/uninitialized_array.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+/**
+ * @brief The Compressed sparse row matrix format namespace.
+ *
+ * @ingroup csr
+ */
+namespace csr {
+
+
+constexpr int default_block_size = 512;
+constexpr int warps_in_block = 4;
+constexpr int spmv_block_size = warps_in_block * config::warp_size;
+constexpr int wsize = config::warp_size;
+constexpr int classical_overweight = 32;
+
+
+/**
+ * A compile-time list of the number items per threads for which spmv kernel
+ * should be compiled.
+ */
+using compiled_kernels = syn::value_list<int, 3, 4, 6, 7, 8, 12, 14>;
+
+using classical_kernels =
+    syn::value_list<int, config::warp_size, 32, 16, 8, 4, 2, 1>;
+
+using spgeam_kernels =
+    syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
+
+
+namespace kernel {
+
+
+template <typename T>
+__dpct_inline__ T ceildivT(T nom, T denom)
+{
+    return (nom + denom - 1ll) / denom;
+}
+
+
+template <typename ValueType, typename IndexType>
+__dpct_inline__ bool block_segment_scan_reverse(
+    const IndexType *__restrict__ ind, ValueType *__restrict__ val,
+    sycl::nd_item<3> item_ct1)
+{
+    bool last = true;
+    const auto reg_ind = ind[item_ct1.get_local_id(2)];
+#pragma unroll
+    for (int i = 1; i < spmv_block_size; i <<= 1) {
+        if (i == 1 && item_ct1.get_local_id(2) < spmv_block_size - 1 &&
+            reg_ind == ind[item_ct1.get_local_id(2) + 1]) {
+            last = false;
+        }
+        auto temp = zero<ValueType>();
+        if (item_ct1.get_local_id(2) >= i &&
+            reg_ind == ind[item_ct1.get_local_id(2) - i]) {
+            temp = val[item_ct1.get_local_id(2) - i];
+        }
+        group::this_thread_block(item_ct1).sync();
+        val[item_ct1.get_local_id(2)] += temp;
+        group::this_thread_block(item_ct1).sync();
+    }
+
+    return last;
+}
+
+
+template <bool overflow, typename IndexType>
+__dpct_inline__ void find_next_row(
+    const IndexType num_rows, const IndexType data_size, const IndexType ind,
+    IndexType *__restrict__ row, IndexType *__restrict__ row_end,
+    const IndexType row_predict, const IndexType row_predict_end,
+    const IndexType *__restrict__ row_ptr)
+{
+    if (!overflow || ind < data_size) {
+        if (ind >= *row_end) {
+            *row = row_predict;
+            *row_end = row_predict_end;
+            for (; ind >= *row_end; *row_end = row_ptr[++*row + 1])
+                ;
+        }
+
+    } else {
+        *row = num_rows - 1;
+        *row_end = data_size;
+    }
+}
+
+
+template <unsigned subwarp_size, typename ValueType, typename IndexType,
+          typename Closure>
+__dpct_inline__ void warp_atomic_add(
+    const group::thread_block_tile<subwarp_size> &group, bool force_write,
+    ValueType *__restrict__ val, const IndexType row, ValueType *__restrict__ c,
+    const size_type c_stride, const IndexType column_id, Closure scale)
+{
+    // do a local scan to avoid atomic collisions
+    const bool need_write = segment_scan(group, row, val);
+    if (need_write && force_write) {
+        atomic_add(&(c[row * c_stride + column_id]), scale(*val));
+    }
+    if (!need_write || force_write) {
+        *val = zero<ValueType>();
+    }
+}
+
+
+template <bool last, unsigned subwarp_size, typename ValueType,
+          typename IndexType, typename Closure>
+__dpct_inline__ void process_window(
+    const group::thread_block_tile<subwarp_size> &group,
+    const IndexType num_rows, const IndexType data_size, const IndexType ind,
+    IndexType *__restrict__ row, IndexType *__restrict__ row_end,
+    IndexType *__restrict__ nrow, IndexType *__restrict__ nrow_end,
+    ValueType *__restrict__ temp_val, const ValueType *__restrict__ val,
+    const IndexType *__restrict__ col_idxs,
+    const IndexType *__restrict__ row_ptrs, const ValueType *__restrict__ b,
+    const size_type b_stride, ValueType *__restrict__ c,
+    const size_type c_stride, const IndexType column_id, Closure scale)
+{
+    const IndexType curr_row = *row;
+    find_next_row<last>(num_rows, data_size, ind, row, row_end, *nrow,
+                        *nrow_end, row_ptrs);
+    // segmented scan
+    if (group.any(curr_row != *row)) {
+        warp_atomic_add(group, curr_row != *row, temp_val, curr_row, c,
+                        c_stride, column_id, scale);
+        *nrow = group.shfl(*row, subwarp_size - 1);
+        *nrow_end = group.shfl(*row_end, subwarp_size - 1);
+    }
+
+    if (!last || ind < data_size) {
+        const auto col = col_idxs[ind];
+        *temp_val += val[ind] * b[col * b_stride + column_id];
+    }
+}
+
+
+template <typename IndexType>
+__dpct_inline__ IndexType get_warp_start_idx(const IndexType nwarps,
+                                             const IndexType nnz,
+                                             const IndexType warp_idx)
+{
+    const long long cache_lines = ceildivT<IndexType>(nnz, wsize);
+    return (warp_idx * cache_lines / nwarps) * wsize;
+}
+
+
+template <typename ValueType, typename IndexType, typename Closure>
+__dpct_inline__ void spmv_kernel(
+    const IndexType nwarps, const IndexType num_rows,
+    const ValueType *__restrict__ val, const IndexType *__restrict__ col_idxs,
+    const IndexType *__restrict__ row_ptrs, const IndexType *__restrict__ srow,
+    const ValueType *__restrict__ b, const size_type b_stride,
+    ValueType *__restrict__ c, const size_type c_stride, Closure scale,
+    sycl::nd_item<3> item_ct1)
+{
+    const IndexType warp_idx =
+        item_ct1.get_group(2) * warps_in_block + item_ct1.get_local_id(1);
+    const IndexType column_id = item_ct1.get_group(1);
+    if (warp_idx >= nwarps) {
+        return;
+    }
+    const IndexType data_size = row_ptrs[num_rows];
+    const IndexType start = get_warp_start_idx(nwarps, data_size, warp_idx);
+    const IndexType end =
+        min(get_warp_start_idx(nwarps, data_size, warp_idx + 1),
+            ceildivT<IndexType>(data_size, wsize) * wsize);
+    auto row = srow[warp_idx];
+    auto row_end = row_ptrs[row + 1];
+    auto nrow = row;
+    auto nrow_end = row_end;
+    ValueType temp_val = zero<ValueType>();
+    IndexType ind = start + item_ct1.get_local_id(2);
+    find_next_row<true>(num_rows, data_size, ind, &row, &row_end, nrow,
+                        nrow_end, row_ptrs);
+    const IndexType ind_end = end - wsize;
+    const auto tile_block =
+        group::tiled_partition<wsize>(group::this_thread_block(item_ct1));
+    for (; ind < ind_end; ind += wsize) {
+        process_window<false>(tile_block, num_rows, data_size, ind, &row,
+                              &row_end, &nrow, &nrow_end, &temp_val, val,
+                              col_idxs, row_ptrs, b, b_stride, c, c_stride,
+                              column_id, scale);
+    }
+    process_window<true>(tile_block, num_rows, data_size, ind, &row, &row_end,
+                         &nrow, &nrow_end, &temp_val, val, col_idxs, row_ptrs,
+                         b, b_stride, c, c_stride, column_id, scale);
+    warp_atomic_add(tile_block, true, &temp_val, row, c, c_stride, column_id,
+                    scale);
+}
+
+
+template <typename ValueType, typename IndexType>
+void abstract_spmv(const IndexType nwarps, const IndexType num_rows,
+                   const ValueType *__restrict__ val,
+                   const IndexType *__restrict__ col_idxs,
+                   const IndexType *__restrict__ row_ptrs,
+                   const IndexType *__restrict__ srow,
+                   const ValueType *__restrict__ b, const size_type b_stride,
+                   ValueType *__restrict__ c, const size_type c_stride,
+                   sycl::nd_item<3> item_ct1)
+{
+    spmv_kernel(
+        nwarps, num_rows, val, col_idxs, row_ptrs, srow, b, b_stride, c,
+        c_stride, [](const ValueType &x) { return x; }, item_ct1);
+}
+
+template <typename ValueType, typename IndexType>
+void abstract_spmv(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                   sycl::queue *stream, const IndexType nwarps,
+                   const IndexType num_rows, const ValueType *val,
+                   const IndexType *col_idxs, const IndexType *row_ptrs,
+                   const IndexType *srow, const ValueType *b,
+                   const size_type b_stride, ValueType *c,
+                   const size_type c_stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                abstract_spmv(nwarps, num_rows, val, col_idxs, row_ptrs, srow,
+                              b, b_stride, c, c_stride, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void abstract_spmv(const IndexType nwarps, const IndexType num_rows,
+                   const ValueType *__restrict__ alpha,
+                   const ValueType *__restrict__ val,
+                   const IndexType *__restrict__ col_idxs,
+                   const IndexType *__restrict__ row_ptrs,
+                   const IndexType *__restrict__ srow,
+                   const ValueType *__restrict__ b, const size_type b_stride,
+                   ValueType *__restrict__ c, const size_type c_stride,
+                   sycl::nd_item<3> item_ct1)
+{
+    ValueType scale_factor = alpha[0];
+    spmv_kernel(
+        nwarps, num_rows, val, col_idxs, row_ptrs, srow, b, b_stride, c,
+        c_stride,
+        [&scale_factor](const ValueType &x) { return scale_factor * x; },
+        item_ct1);
+}
+
+template <typename ValueType, typename IndexType>
+void abstract_spmv(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                   sycl::queue *stream, const IndexType nwarps,
+                   const IndexType num_rows, const ValueType *alpha,
+                   const ValueType *val, const IndexType *col_idxs,
+                   const IndexType *row_ptrs, const IndexType *srow,
+                   const ValueType *b, const size_type b_stride, ValueType *c,
+                   const size_type c_stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                abstract_spmv(nwarps, num_rows, alpha, val, col_idxs, row_ptrs,
+                              srow, b, b_stride, c, c_stride, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType>
+void set_zero(const size_type nnz, ValueType *__restrict__ val,
+              sycl::nd_item<3> item_ct1)
+{
+    const auto ind = thread::get_thread_id_flat(item_ct1);
+    if (ind < nnz) {
+        val[ind] = zero<ValueType>();
+    }
+}
+
+template <typename ValueType>
+void set_zero(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+              sycl::queue *stream, const size_type nnz, ValueType *val)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block),
+            [=](sycl::nd_item<3> item_ct1) { set_zero(nnz, val, item_ct1); });
+    });
+}
+
+
+template <typename IndexType>
+__dpct_inline__ void merge_path_search(
+    const IndexType diagonal, const IndexType a_len, const IndexType b_len,
+    const IndexType *__restrict__ a, const IndexType offset_b,
+    IndexType *__restrict__ x, IndexType *__restrict__ y)
+{
+    auto x_min = max(diagonal - b_len, zero<IndexType>());
+    auto x_max = min(diagonal, a_len);
+    while (x_min < x_max) {
+        auto pivot = x_min + (x_max - x_min) / 2;
+        if (a[pivot] <= offset_b + diagonal - pivot - 1) {
+            x_min = pivot + 1;
+        } else {
+            x_max = pivot;
+        }
+    }
+
+    *x = min(x_min, a_len);
+    *y = diagonal - x_min;
+}
+
+
+template <typename ValueType, typename IndexType, typename Alpha_op>
+void merge_path_reduce(const IndexType nwarps,
+                       const ValueType *__restrict__ last_val,
+                       const IndexType *__restrict__ last_row,
+                       ValueType *__restrict__ c, const size_type c_stride,
+                       Alpha_op alpha_op, sycl::nd_item<3> item_ct1,
+                       UninitializedArray<IndexType, spmv_block_size> *tmp_ind,
+                       UninitializedArray<ValueType, spmv_block_size> *tmp_val)
+{
+    const IndexType cache_lines = ceildivT<IndexType>(nwarps, spmv_block_size);
+    const IndexType tid = item_ct1.get_local_id(2);
+    const IndexType start = min(tid * cache_lines, nwarps);
+    const IndexType end = min((tid + 1) * cache_lines, nwarps);
+    ValueType value = zero<ValueType>();
+    IndexType row = last_row[nwarps - 1];
+    if (start < nwarps) {
+        value = last_val[start];
+        row = last_row[start];
+        for (IndexType i = start + 1; i < end; i++) {
+            if (last_row[i] != row) {
+                c[row * c_stride] += alpha_op(value);
+                row = last_row[i];
+                value = last_val[i];
+            } else {
+                value += last_val[i];
+            }
+        }
+    }
+
+
+    (*tmp_val)[item_ct1.get_local_id(2)] = value;
+    (*tmp_ind)[item_ct1.get_local_id(2)] = row;
+    group::this_thread_block(item_ct1).sync();
+    bool last = block_segment_scan_reverse(static_cast<IndexType *>((*tmp_ind)),
+                                           static_cast<ValueType *>((*tmp_val)),
+                                           item_ct1);
+    group::this_thread_block(item_ct1).sync();
+    if (last) {
+        c[row * c_stride] += alpha_op((*tmp_val)[item_ct1.get_local_id(2)]);
+    }
+}
+
+
+template <int items_per_thread, typename ValueType, typename IndexType,
+          typename Alpha_op, typename Beta_op>
+void merge_path_spmv(const IndexType num_rows,
+                     const ValueType *__restrict__ val,
+                     const IndexType *__restrict__ col_idxs,
+                     const IndexType *__restrict__ row_ptrs,
+                     const IndexType *__restrict__ srow,
+                     const ValueType *__restrict__ b, const size_type b_stride,
+                     ValueType *__restrict__ c, const size_type c_stride,
+                     IndexType *__restrict__ row_out,
+                     ValueType *__restrict__ val_out, Alpha_op alpha_op,
+                     Beta_op beta_op, sycl::nd_item<3> item_ct1,
+                     IndexType *shared_row_ptrs)
+{
+    const auto *row_end_ptrs = row_ptrs + 1;
+    const auto nnz = row_ptrs[num_rows];
+    const IndexType num_merge_items = num_rows + nnz;
+    const auto block_items = spmv_block_size * items_per_thread;
+
+    const IndexType diagonal =
+        min(IndexType(block_items * item_ct1.get_group(2)), num_merge_items);
+    const IndexType diagonal_end = min(diagonal + block_items, num_merge_items);
+    IndexType block_start_x;
+    IndexType block_start_y;
+    IndexType end_x;
+    IndexType end_y;
+    merge_path_search(diagonal, num_rows, nnz, row_end_ptrs, zero<IndexType>(),
+                      &block_start_x, &block_start_y);
+    merge_path_search(diagonal_end, num_rows, nnz, row_end_ptrs,
+                      zero<IndexType>(), &end_x, &end_y);
+    const IndexType block_num_rows = end_x - block_start_x;
+    const IndexType block_num_nonzeros = end_y - block_start_y;
+    for (int i = item_ct1.get_local_id(2);
+         i < block_num_rows && block_start_x + i < num_rows;
+         i += spmv_block_size) {
+        shared_row_ptrs[i] = row_end_ptrs[block_start_x + i];
+    }
+    group::this_thread_block(item_ct1).sync();
+
+    IndexType start_x;
+    IndexType start_y;
+    merge_path_search(IndexType(items_per_thread * item_ct1.get_local_id(2)),
+                      block_num_rows, block_num_nonzeros, shared_row_ptrs,
+                      block_start_y, &start_x, &start_y);
+
+
+    IndexType ind = block_start_y + start_y;
+    IndexType row_i = block_start_x + start_x;
+    ValueType value = zero<ValueType>();
+#pragma unroll
+    for (IndexType i = 0; i < items_per_thread; i++) {
+        if (row_i < num_rows) {
+            if (start_x == block_num_rows || ind < shared_row_ptrs[start_x]) {
+                value += val[ind] * b[col_idxs[ind] * b_stride];
+                ind++;
+            } else {
+                c[row_i * c_stride] =
+                    alpha_op(value) + beta_op(c[row_i * c_stride]);
+                start_x++;
+                row_i++;
+                value = zero<ValueType>();
+            }
+        }
+    }
+    group::this_thread_block(item_ct1).sync();
+    IndexType *tmp_ind = shared_row_ptrs;
+    ValueType *tmp_val =
+        reinterpret_cast<ValueType *>(shared_row_ptrs + spmv_block_size);
+    tmp_val[item_ct1.get_local_id(2)] = value;
+    tmp_ind[item_ct1.get_local_id(2)] = row_i;
+    group::this_thread_block(item_ct1).sync();
+    bool last =
+        block_segment_scan_reverse(static_cast<IndexType *>(tmp_ind),
+                                   static_cast<ValueType *>(tmp_val), item_ct1);
+    if (item_ct1.get_local_id(2) == spmv_block_size - 1) {
+        row_out[item_ct1.get_group(2)] = min(end_x, num_rows - 1);
+        val_out[item_ct1.get_group(2)] = tmp_val[item_ct1.get_local_id(2)];
+    } else if (last) {
+        c[row_i * c_stride] += alpha_op(tmp_val[item_ct1.get_local_id(2)]);
+    }
+}
+
+template <int items_per_thread, typename ValueType, typename IndexType>
+void abstract_merge_path_spmv(
+    const IndexType num_rows, const ValueType *__restrict__ val,
+    const IndexType *__restrict__ col_idxs,
+    const IndexType *__restrict__ row_ptrs, const IndexType *__restrict__ srow,
+    const ValueType *__restrict__ b, const size_type b_stride,
+    ValueType *__restrict__ c, const size_type c_stride,
+    IndexType *__restrict__ row_out, ValueType *__restrict__ val_out,
+    sycl::nd_item<3> item_ct1, IndexType *shared_row_ptrs)
+{
+    merge_path_spmv<items_per_thread>(
+        num_rows, val, col_idxs, row_ptrs, srow, b, b_stride, c, c_stride,
+        row_out, val_out, [](ValueType &x) { return x; },
+        [](ValueType &x) { return zero<ValueType>(); }, item_ct1,
+        shared_row_ptrs);
+}
+
+template <int items_per_thread, typename ValueType, typename IndexType>
+void abstract_merge_path_spmv(dim3 grid, dim3 block,
+                              gko::size_type dynamic_shared_memory,
+                              sycl::queue *stream, const IndexType num_rows,
+                              const ValueType *val, const IndexType *col_idxs,
+                              const IndexType *row_ptrs, const IndexType *srow,
+                              const ValueType *b, const size_type b_stride,
+                              ValueType *c, const size_type c_stride,
+                              IndexType *row_out, ValueType *val_out)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<IndexType, 1, sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            shared_row_ptrs_acc_ct1(sycl::range<1>(block_items), cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                abstract_merge_path_spmv<items_per_thread>(
+                    num_rows, val, col_idxs, row_ptrs, srow, b, b_stride, c,
+                    c_stride, row_out, val_out, item_ct1,
+                    (IndexType *)shared_row_ptrs_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+template <int items_per_thread, typename ValueType, typename IndexType>
+void abstract_merge_path_spmv(
+    const IndexType num_rows, const ValueType *__restrict__ alpha,
+    const ValueType *__restrict__ val, const IndexType *__restrict__ col_idxs,
+    const IndexType *__restrict__ row_ptrs, const IndexType *__restrict__ srow,
+    const ValueType *__restrict__ b, const size_type b_stride,
+    const ValueType *__restrict__ beta, ValueType *__restrict__ c,
+    const size_type c_stride, IndexType *__restrict__ row_out,
+    ValueType *__restrict__ val_out, sycl::nd_item<3> item_ct1,
+    IndexType *shared_row_ptrs)
+{
+    const auto alpha_val = alpha[0];
+    const auto beta_val = beta[0];
+    merge_path_spmv<items_per_thread>(
+        num_rows, val, col_idxs, row_ptrs, srow, b, b_stride, c, c_stride,
+        row_out, val_out, [&alpha_val](ValueType &x) { return alpha_val * x; },
+        [&beta_val](ValueType &x) { return beta_val * x; }, item_ct1,
+        shared_row_ptrs);
+}
+
+template <int items_per_thread, typename ValueType, typename IndexType>
+void abstract_merge_path_spmv(
+    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+    sycl::queue *stream, const IndexType num_rows, const ValueType *alpha,
+    const ValueType *val, const IndexType *col_idxs, const IndexType *row_ptrs,
+    const IndexType *srow, const ValueType *b, const size_type b_stride,
+    const ValueType *beta, ValueType *c, const size_type c_stride,
+    IndexType *row_out, ValueType *val_out)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<IndexType, 1, sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            shared_row_ptrs_acc_ct1(sycl::range<1>(block_items), cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                abstract_merge_path_spmv<items_per_thread>(
+                    num_rows, alpha, val, col_idxs, row_ptrs, srow, b, b_stride,
+                    beta, c, c_stride, row_out, val_out, item_ct1,
+                    (IndexType *)shared_row_ptrs_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void abstract_reduce(const IndexType nwarps,
+                     const ValueType *__restrict__ last_val,
+                     const IndexType *__restrict__ last_row,
+                     ValueType *__restrict__ c, const size_type c_stride,
+                     sycl::nd_item<3> item_ct1,
+                     UninitializedArray<IndexType, spmv_block_size> *tmp_ind,
+                     UninitializedArray<ValueType, spmv_block_size> *tmp_val)
+{
+    merge_path_reduce(
+        nwarps, last_val, last_row, c, c_stride, [](ValueType &x) { return x; },
+        item_ct1, tmp_ind, tmp_val);
+}
+
+template <typename ValueType, typename IndexType>
+void abstract_reduce(dim3 grid, dim3 block,
+                     gko::size_type dynamic_shared_memory, sycl::queue *stream,
+                     const IndexType nwarps, const ValueType *last_val,
+                     const IndexType *last_row, ValueType *c,
+                     const size_type c_stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<IndexType, spmv_block_size>, 0,
+                       sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            tmp_ind_acc_ct1(cgh);
+        sycl::accessor<UninitializedArray<ValueType, spmv_block_size>, 0,
+                       sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            tmp_val_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                abstract_reduce(nwarps, last_val, last_row, c, c_stride,
+                                item_ct1, tmp_val_acc_ct1.get_pointer().get());
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void abstract_reduce(const IndexType nwarps,
+                     const ValueType *__restrict__ last_val,
+                     const IndexType *__restrict__ last_row,
+                     const ValueType *__restrict__ alpha,
+                     ValueType *__restrict__ c, const size_type c_stride,
+                     sycl::nd_item<3> item_ct1,
+                     UninitializedArray<IndexType, spmv_block_size> *tmp_ind,
+                     UninitializedArray<ValueType, spmv_block_size> *tmp_val)
+{
+    const auto alpha_val = alpha[0];
+    merge_path_reduce(
+        nwarps, last_val, last_row, c, c_stride,
+        [&alpha_val](ValueType &x) { return alpha_val * x; }, item_ct1, tmp_ind,
+        tmp_val);
+}
+
+template <typename ValueType, typename IndexType>
+void abstract_reduce(dim3 grid, dim3 block,
+                     gko::size_type dynamic_shared_memory, sycl::queue *stream,
+                     const IndexType nwarps, const ValueType *last_val,
+                     const IndexType *last_row, const ValueType *alpha,
+                     ValueType *c, const size_type c_stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<UninitializedArray<IndexType, spmv_block_size>, 0,
+                       sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            tmp_ind_acc_ct1(cgh);
+        sycl::accessor<UninitializedArray<ValueType, spmv_block_size>, 0,
+                       sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            tmp_val_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                abstract_reduce(nwarps, last_val, last_row, alpha, c, c_stride,
+                                item_ct1, tmp_val_acc_ct1.get_pointer().get());
+            });
+    });
+}
+
+
+template <size_type subwarp_size, typename ValueType, typename IndexType,
+          typename Closure>
+void device_classical_spmv(const size_type num_rows,
+                           const ValueType *__restrict__ val,
+                           const IndexType *__restrict__ col_idxs,
+                           const IndexType *__restrict__ row_ptrs,
+                           const ValueType *__restrict__ b,
+                           const size_type b_stride, ValueType *__restrict__ c,
+                           const size_type c_stride, Closure scale,
+                           sycl::nd_item<3> item_ct1)
+{
+    auto subwarp_tile = group::tiled_partition<subwarp_size>(
+        group::this_thread_block(item_ct1));
+    const auto subrow = thread::get_subwarp_num_flat<subwarp_size>(item_ct1);
+    const auto subid = subwarp_tile.thread_rank();
+    const auto column_id = item_ct1.get_group(1);
+    auto row = thread::get_subwarp_id_flat<subwarp_size>(item_ct1);
+    for (; row < num_rows; row += subrow) {
+        const auto ind_end = row_ptrs[row + 1];
+        ValueType temp_val = zero<ValueType>();
+        for (auto ind = row_ptrs[row] + subid; ind < ind_end;
+             ind += subwarp_size) {
+            temp_val += val[ind] * b[col_idxs[ind] * b_stride + column_id];
+        }
+        auto subwarp_result = reduce(
+            subwarp_tile, temp_val,
+            [](const ValueType &a, const ValueType &b) { return a + b; });
+        if (subid == 0) {
+            c[row * c_stride + column_id] =
+                scale(subwarp_result, c[row * c_stride + column_id]);
+        }
+    }
+}
+
+
+template <size_type subwarp_size, typename ValueType, typename IndexType>
+void abstract_classical_spmv(
+    const size_type num_rows, const ValueType *__restrict__ val,
+    const IndexType *__restrict__ col_idxs,
+    const IndexType *__restrict__ row_ptrs, const ValueType *__restrict__ b,
+    const size_type b_stride, ValueType *__restrict__ c,
+    const size_type c_stride, sycl::nd_item<3> item_ct1)
+{
+    device_classical_spmv<subwarp_size>(
+        num_rows, val, col_idxs, row_ptrs, b, b_stride, c, c_stride,
+        [](const ValueType &x, const ValueType &y) { return x; }, item_ct1);
+}
+
+template <size_type subwarp_size, typename ValueType, typename IndexType>
+void abstract_classical_spmv(dim3 grid, dim3 block,
+                             gko::size_type dynamic_shared_memory,
+                             sycl::queue *stream, const size_type num_rows,
+                             const ValueType *val, const IndexType *col_idxs,
+                             const IndexType *row_ptrs, const ValueType *b,
+                             const size_type b_stride, ValueType *c,
+                             const size_type c_stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                abstract_classical_spmv<subwarp_size>(num_rows, val, col_idxs,
+                                                      row_ptrs, b, b_stride, c,
+                                                      c_stride, item_ct1);
+            });
+    });
+}
+
+
+template <size_type subwarp_size, typename ValueType, typename IndexType>
+void abstract_classical_spmv(
+    const size_type num_rows, const ValueType *__restrict__ alpha,
+    const ValueType *__restrict__ val, const IndexType *__restrict__ col_idxs,
+    const IndexType *__restrict__ row_ptrs, const ValueType *__restrict__ b,
+    const size_type b_stride, const ValueType *__restrict__ beta,
+    ValueType *__restrict__ c, const size_type c_stride,
+    sycl::nd_item<3> item_ct1)
+{
+    const auto alpha_val = alpha[0];
+    const auto beta_val = beta[0];
+    device_classical_spmv<subwarp_size>(
+        num_rows, val, col_idxs, row_ptrs, b, b_stride, c, c_stride,
+        [&alpha_val, &beta_val](const ValueType &x, const ValueType &y) {
+            return alpha_val * x + beta_val * y;
+        },
+        item_ct1);
+}
+
+template <size_type subwarp_size, typename ValueType, typename IndexType>
+void abstract_classical_spmv(dim3 grid, dim3 block,
+                             gko::size_type dynamic_shared_memory,
+                             sycl::queue *stream, const size_type num_rows,
+                             const ValueType *alpha, const ValueType *val,
+                             const IndexType *col_idxs,
+                             const IndexType *row_ptrs, const ValueType *b,
+                             const size_type b_stride, const ValueType *beta,
+                             ValueType *c, const size_type c_stride)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             abstract_classical_spmv<subwarp_size>(
+                                 num_rows, alpha, val, col_idxs, row_ptrs, b,
+                                 b_stride, beta, c, c_stride, item_ct1);
+                         });
+    });
+}
+
+
+template <int subwarp_size, typename IndexType>
+void spgeam_nnz(const IndexType *__restrict__ a_row_ptrs,
+                const IndexType *__restrict__ a_col_idxs,
+                const IndexType *__restrict__ b_row_ptrs,
+                const IndexType *__restrict__ b_col_idxs, IndexType num_rows,
+                IndexType *__restrict__ nnz, sycl::nd_item<3> item_ct1)
+{
+    const auto row =
+        thread::get_subwarp_id_flat<subwarp_size, IndexType>(item_ct1);
+    auto subwarp = group::tiled_partition<subwarp_size>(
+        group::this_thread_block(item_ct1));
+    if (row >= num_rows) {
+        return;
+    }
+
+    const auto a_begin = a_row_ptrs[row];
+    const auto b_begin = b_row_ptrs[row];
+    const auto a_size = a_row_ptrs[row + 1] - a_begin;
+    const auto b_size = b_row_ptrs[row + 1] - b_begin;
+    IndexType count{};
+    group_merge<subwarp_size>(
+        a_col_idxs + a_begin, a_size, b_col_idxs + b_begin, b_size, subwarp,
+        [&](IndexType, IndexType a_col, IndexType, IndexType b_col, IndexType,
+            bool valid) {
+            count += popcnt(subwarp.ballot(a_col != b_col && valid));
+            return true;
+        });
+
+    if (subwarp.thread_rank() == 0) {
+        nnz[row] = count;
+    }
+}
+
+template <int subwarp_size, typename IndexType>
+void spgeam_nnz(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                sycl::queue *stream, const IndexType *a_row_ptrs,
+                const IndexType *a_col_idxs, const IndexType *b_row_ptrs,
+                const IndexType *b_col_idxs, IndexType num_rows, IndexType *nnz)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                spgeam_nnz<subwarp_size>(a_row_ptrs, a_col_idxs, b_row_ptrs,
+                                         b_col_idxs, num_rows, nnz, item_ct1);
+            });
+    });
+}
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+void spgeam(const ValueType *__restrict__ palpha,
+            const IndexType *__restrict__ a_row_ptrs,
+            const IndexType *__restrict__ a_col_idxs,
+            const ValueType *__restrict__ a_vals,
+            const ValueType *__restrict__ pbeta,
+            const IndexType *__restrict__ b_row_ptrs,
+            const IndexType *__restrict__ b_col_idxs,
+            const ValueType *__restrict__ b_vals, IndexType num_rows,
+            const IndexType *__restrict__ c_row_ptrs,
+            IndexType *__restrict__ c_col_idxs, ValueType *__restrict__ c_vals,
+            sycl::nd_item<3> item_ct1)
+{
+    const auto row =
+        thread::get_subwarp_id_flat<subwarp_size, IndexType>(item_ct1);
+    auto subwarp = group::tiled_partition<subwarp_size>(
+        group::this_thread_block(item_ct1));
+    if (row >= num_rows) {
+        return;
+    }
+
+    const auto alpha = palpha[0];
+    const auto beta = pbeta[0];
+    const auto lane = static_cast<IndexType>(subwarp.thread_rank());
+    constexpr auto lanemask_full =
+        ~config::lane_mask_type{} >> (config::warp_size - subwarp_size);
+    const auto lanemask_eq = config::lane_mask_type{1} << lane;
+    const auto lanemask_lt = lanemask_eq - 1;
+
+    const auto a_begin = a_row_ptrs[row];
+    const auto b_begin = b_row_ptrs[row];
+    const auto a_size = a_row_ptrs[row + 1] - a_begin;
+    const auto b_size = b_row_ptrs[row + 1] - b_begin;
+    auto c_begin = c_row_ptrs[row];
+    bool skip_first{};
+    group_merge<subwarp_size>(
+        a_col_idxs + a_begin, a_size, b_col_idxs + b_begin, b_size, subwarp,
+        [&](IndexType a_nz, IndexType a_col, IndexType b_nz, IndexType b_col,
+            IndexType, bool valid) {
+            auto c_col = min(a_col, b_col);
+            auto equal_mask = subwarp.ballot(a_col == b_col && valid);
+            // check if the elements in the previous merge step are
+            // equal
+            auto prev_equal_mask = equal_mask << 1 | skip_first;
+            // store the highest bit for the next group_merge_step
+            skip_first = bool(equal_mask >> (subwarp_size - 1));
+            auto prev_equal = bool(prev_equal_mask & lanemask_eq);
+            // only output an entry if the previous cols weren't equal.
+            // if they were equal, they were both handled in the
+            // previous step
+            if (valid && !prev_equal) {
+                auto c_ofs = popcnt(~prev_equal_mask & lanemask_lt);
+                c_col_idxs[c_begin + c_ofs] = c_col;
+                auto a_val =
+                    a_col <= b_col ? a_vals[a_nz + a_begin] : zero<ValueType>();
+                auto b_val =
+                    b_col <= a_col ? b_vals[b_nz + b_begin] : zero<ValueType>();
+                c_vals[c_begin + c_ofs] = alpha * a_val + beta * b_val;
+            }
+            // advance by the number of merged elements
+            // in theory, we would need to mask by `valid`, but this
+            // would only be false somwhere in the last iteration, where
+            // we don't need the value of c_begin afterwards, anyways.
+            c_begin += popcnt(~prev_equal_mask & lanemask_full);
+            return true;
+        });
+}
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+void spgeam(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+            sycl::queue *stream, const ValueType *palpha,
+            const IndexType *a_row_ptrs, const IndexType *a_col_idxs,
+            const ValueType *a_vals, const ValueType *pbeta,
+            const IndexType *b_row_ptrs, const IndexType *b_col_idxs,
+            const ValueType *b_vals, IndexType num_rows,
+            const IndexType *c_row_ptrs, IndexType *c_col_idxs,
+            ValueType *c_vals)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                spgeam<subwarp_size>(palpha, a_row_ptrs, a_col_idxs, a_vals,
+                                     pbeta, b_row_ptrs, b_col_idxs, b_vals,
+                                     num_rows, c_row_ptrs, c_col_idxs, c_vals,
+                                     item_ct1);
+            });
+    });
+}
+
+
+template <typename IndexType>
+void convert_row_ptrs_to_idxs(size_type num_rows,
+                              const IndexType *__restrict__ ptrs,
+                              IndexType *__restrict__ idxs,
+                              sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    if (tidx < num_rows) {
+        for (auto i = ptrs[tidx]; i < ptrs[tidx + 1]; i++) {
+            idxs[i] = tidx;
+        }
+    }
+}
+
+template <typename IndexType>
+void convert_row_ptrs_to_idxs(dim3 grid, dim3 block,
+                              gko::size_type dynamic_shared_memory,
+                              sycl::queue *stream, size_type num_rows,
+                              const IndexType *ptrs, IndexType *idxs)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                convert_row_ptrs_to_idxs(num_rows, ptrs, idxs, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType>
+void initialize_zero_dense(size_type num_rows, size_type num_cols,
+                           size_type stride, ValueType *__restrict__ result,
+                           sycl::nd_item<3> item_ct1)
+{
+    const auto tidx_x =
+        item_ct1.get_local_id(2) +
+        item_ct1.get_local_range().get(2) * item_ct1.get_group(2);
+    const auto tidx_y =
+        item_ct1.get_local_id(1) +
+        item_ct1.get_local_range().get(1) * item_ct1.get_group(1);
+    if (tidx_x < num_cols && tidx_y < num_rows) {
+        result[tidx_y * stride + tidx_x] = zero<ValueType>();
+    }
+}
+
+template <typename ValueType>
+void initialize_zero_dense(dim3 grid, dim3 block,
+                           gko::size_type dynamic_shared_memory,
+                           sycl::queue *stream, size_type num_rows,
+                           size_type num_cols, size_type stride,
+                           ValueType *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             initialize_zero_dense(num_rows, num_cols, stride,
+                                                   result, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void fill_in_dense(size_type num_rows, const IndexType *__restrict__ row_ptrs,
+                   const IndexType *__restrict__ col_idxs,
+                   const ValueType *__restrict__ values, size_type stride,
+                   ValueType *__restrict__ result, sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    if (tidx < num_rows) {
+        for (auto i = row_ptrs[tidx]; i < row_ptrs[tidx + 1]; i++) {
+            result[stride * tidx + col_idxs[i]] = values[i];
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void fill_in_dense(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                   sycl::queue *stream, size_type num_rows,
+                   const IndexType *row_ptrs, const IndexType *col_idxs,
+                   const ValueType *values, size_type stride, ValueType *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             fill_in_dense(num_rows, row_ptrs, col_idxs, values,
+                                           stride, result, item_ct1);
+                         });
+    });
+}
+
+
+template <typename IndexType>
+void calculate_nnz_per_row(size_type num_rows,
+                           const IndexType *__restrict__ row_ptrs,
+                           size_type *__restrict__ nnz_per_row,
+                           sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    if (tidx < num_rows) {
+        nnz_per_row[tidx] = row_ptrs[tidx + 1] - row_ptrs[tidx];
+    }
+}
+
+template <typename IndexType>
+void calculate_nnz_per_row(dim3 grid, dim3 block,
+                           gko::size_type dynamic_shared_memory,
+                           sycl::queue *stream, size_type num_rows,
+                           const IndexType *row_ptrs, size_type *nnz_per_row)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             calculate_nnz_per_row(num_rows, row_ptrs,
+                                                   nnz_per_row, item_ct1);
+                         });
+    });
+}
+
+
+void calculate_slice_lengths(size_type num_rows, size_type slice_size,
+                             size_type stride_factor,
+                             const size_type *__restrict__ nnz_per_row,
+                             size_type *__restrict__ slice_lengths,
+                             size_type *__restrict__ slice_sets,
+                             sycl::nd_item<3> item_ct1)
+{
+    constexpr auto warp_size = config::warp_size;
+    const auto sliceid = item_ct1.get_group(2);
+    const auto tid_in_warp = item_ct1.get_local_id(2);
+
+    if (sliceid * slice_size + tid_in_warp < num_rows) {
+        size_type thread_result = 0;
+        for (int i = tid_in_warp; i < slice_size; i += warp_size) {
+            thread_result =
+                (i + slice_size * sliceid < num_rows)
+                    ? max(thread_result, nnz_per_row[sliceid * slice_size + i])
+                    : thread_result;
+        }
+
+        auto warp_tile = group::tiled_partition<warp_size>(
+            group::this_thread_block(item_ct1));
+        auto warp_result = reduce(
+            warp_tile, thread_result,
+            [](const size_type &a, const size_type &b) { return max(a, b); });
+
+        if (tid_in_warp == 0) {
+            auto slice_length =
+                ceildiv(warp_result, stride_factor) * stride_factor;
+            slice_lengths[sliceid] = slice_length;
+            slice_sets[sliceid] = slice_length;
+        }
+    }
+}
+
+void calculate_slice_lengths(dim3 grid, dim3 block,
+                             gko::size_type dynamic_shared_memory,
+                             sycl::queue *stream, size_type num_rows,
+                             size_type slice_size, size_type stride_factor,
+                             const size_type *nnz_per_row,
+                             size_type *slice_lengths, size_type *slice_sets)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                calculate_slice_lengths(num_rows, slice_size, stride_factor,
+                                        nnz_per_row, slice_lengths, slice_sets,
+                                        item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void fill_in_sellp(size_type num_rows, size_type slice_size,
+                   const ValueType *__restrict__ source_values,
+                   const IndexType *__restrict__ source_row_ptrs,
+                   const IndexType *__restrict__ source_col_idxs,
+                   size_type *__restrict__ slice_lengths,
+                   size_type *__restrict__ slice_sets,
+                   IndexType *__restrict__ result_col_idxs,
+                   ValueType *__restrict__ result_values,
+                   sycl::nd_item<3> item_ct1)
+{
+    const auto global_row = thread::get_thread_id_flat(item_ct1);
+    const auto row = global_row % slice_size;
+    const auto sliceid = global_row / slice_size;
+
+    if (global_row < num_rows) {
+        size_type sellp_ind = slice_sets[sliceid] * slice_size + row;
+
+        for (size_type csr_ind = source_row_ptrs[global_row];
+             csr_ind < source_row_ptrs[global_row + 1]; csr_ind++) {
+            result_values[sellp_ind] = source_values[csr_ind];
+            result_col_idxs[sellp_ind] = source_col_idxs[csr_ind];
+            sellp_ind += slice_size;
+        }
+        for (size_type i = sellp_ind;
+             i <
+             (slice_sets[sliceid] + slice_lengths[sliceid]) * slice_size + row;
+             i += slice_size) {
+            result_col_idxs[i] = 0;
+            result_values[i] = zero<ValueType>();
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void fill_in_sellp(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                   sycl::queue *stream, size_type num_rows,
+                   size_type slice_size, const ValueType *source_values,
+                   const IndexType *source_row_ptrs,
+                   const IndexType *source_col_idxs, size_type *slice_lengths,
+                   size_type *slice_sets, IndexType *result_col_idxs,
+                   ValueType *result_values)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                fill_in_sellp(num_rows, slice_size, source_values,
+                              source_row_ptrs, source_col_idxs, slice_lengths,
+                              slice_sets, result_col_idxs, result_values,
+                              item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void initialize_zero_ell(size_type max_nnz_per_row, size_type stride,
+                         ValueType *__restrict__ values,
+                         IndexType *__restrict__ col_idxs,
+                         sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+
+    if (tidx < stride * max_nnz_per_row) {
+        values[tidx] = zero<ValueType>();
+        col_idxs[tidx] = 0;
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void initialize_zero_ell(dim3 grid, dim3 block,
+                         gko::size_type dynamic_shared_memory,
+                         sycl::queue *stream, size_type max_nnz_per_row,
+                         size_type stride, ValueType *values,
+                         IndexType *col_idxs)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             initialize_zero_ell(max_nnz_per_row, stride,
+                                                 values, col_idxs, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void fill_in_ell(size_type num_rows, size_type stride,
+                 const ValueType *__restrict__ source_values,
+                 const IndexType *__restrict__ source_row_ptrs,
+                 const IndexType *__restrict__ source_col_idxs,
+                 ValueType *__restrict__ result_values,
+                 IndexType *__restrict__ result_col_idxs,
+                 sycl::nd_item<3> item_ct1)
+{
+    constexpr auto warp_size = config::warp_size;
+    const auto row = thread::get_subwarp_id_flat<warp_size>(item_ct1);
+    const auto local_tidx = item_ct1.get_local_id(2) % warp_size;
+
+    if (row < num_rows) {
+        for (size_type i = local_tidx;
+             i < source_row_ptrs[row + 1] - source_row_ptrs[row];
+             i += warp_size) {
+            const auto result_idx = row + stride * i;
+            const auto source_idx = i + source_row_ptrs[row];
+            result_values[result_idx] = source_values[source_idx];
+            result_col_idxs[result_idx] = source_col_idxs[source_idx];
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void fill_in_ell(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                 sycl::queue *stream, size_type num_rows, size_type stride,
+                 const ValueType *source_values,
+                 const IndexType *source_row_ptrs,
+                 const IndexType *source_col_idxs, ValueType *result_values,
+                 IndexType *result_col_idxs)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                fill_in_ell(num_rows, stride, source_values, source_row_ptrs,
+                            source_col_idxs, result_values, result_col_idxs,
+                            item_ct1);
+            });
+    });
+}
+
+
+void reduce_max_nnz_per_slice(size_type num_rows, size_type slice_size,
+                              size_type stride_factor,
+                              const size_type *__restrict__ nnz_per_row,
+                              size_type *__restrict__ result,
+                              sycl::nd_item<3> item_ct1)
+{
+    constexpr auto warp_size = config::warp_size;
+    auto warp_tile =
+        group::tiled_partition<warp_size>(group::this_thread_block(item_ct1));
+    const auto warpid = thread::get_subwarp_id_flat<warp_size>(item_ct1);
+    const auto tid_in_warp = warp_tile.thread_rank();
+    const auto slice_num = ceildiv(num_rows, slice_size);
+
+    size_type thread_result = 0;
+    for (auto i = tid_in_warp; i < slice_size; i += warp_size) {
+        if (warpid * slice_size + i < num_rows) {
+            thread_result =
+                max(thread_result, nnz_per_row[warpid * slice_size + i]);
+        }
+    }
+    auto warp_result = reduce(
+        warp_tile, thread_result,
+        [](const size_type &a, const size_type &b) { return max(a, b); });
+
+    if (tid_in_warp == 0 && warpid < slice_num) {
+        result[warpid] = ceildiv(warp_result, stride_factor) * stride_factor;
+    }
+}
+
+void reduce_max_nnz_per_slice(dim3 grid, dim3 block,
+                              gko::size_type dynamic_shared_memory,
+                              sycl::queue *stream, size_type num_rows,
+                              size_type slice_size, size_type stride_factor,
+                              const size_type *nnz_per_row, size_type *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                reduce_max_nnz_per_slice(num_rows, slice_size, stride_factor,
+                                         nnz_per_row, result, item_ct1);
+            });
+    });
+}
+
+
+void reduce_total_cols(size_type num_slices,
+                       const size_type *__restrict__ max_nnz_per_slice,
+                       size_type *__restrict__ result,
+                       sycl::nd_item<3> item_ct1, size_type *block_result)
+{
+    reduce_array(num_slices, max_nnz_per_slice, block_result,
+                 [](const size_type &x, const size_type &y) { return x + y; });
+
+    if (item_ct1.get_local_id(2) == 0) {
+        result[item_ct1.get_group(2)] = block_result[0];
+    }
+}
+
+void reduce_total_cols(dim3 grid, dim3 block,
+                       gko::size_type dynamic_shared_memory,
+                       sycl::queue *stream, size_type num_slices,
+                       const size_type *max_nnz_per_slice, size_type *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<size_type, 1, sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            block_result_acc_ct1(sycl::range<1>(512 /*default_block_size*/),
+                                 cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                reduce_total_cols(num_slices, max_nnz_per_slice, result,
+                                  item_ct1, block_result_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+void reduce_max_nnz(size_type size, const size_type *__restrict__ nnz_per_row,
+                    size_type *__restrict__ result, sycl::nd_item<3> item_ct1,
+                    size_type *block_max)
+{
+    reduce_array(
+        size, nnz_per_row, block_max,
+        [](const size_type &x, const size_type &y) { return max(x, y); });
+
+    if (item_ct1.get_local_id(2) == 0) {
+        result[item_ct1.get_group(2)] = block_max[0];
+    }
+}
+
+void reduce_max_nnz(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                    sycl::queue *stream, size_type size,
+                    const size_type *nnz_per_row, size_type *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<size_type, 1, sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            block_max_acc_ct1(sycl::range<1>(512 /*default_block_size*/), cgh);
+
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             reduce_max_nnz(size, nnz_per_row, result, item_ct1,
+                                            block_max_acc_ct1.get_pointer());
+                         });
+    });
+}
+
+
+template <typename IndexType>
+void calculate_hybrid_coo_row_nnz(size_type num_rows,
+                                  size_type ell_max_nnz_per_row,
+                                  IndexType *__restrict__ csr_row_idxs,
+                                  size_type *__restrict__ coo_row_nnz,
+                                  sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    if (tidx < num_rows) {
+        const size_type csr_nnz = csr_row_idxs[tidx + 1] - csr_row_idxs[tidx];
+        coo_row_nnz[tidx] =
+            (csr_nnz > ell_max_nnz_per_row) * (csr_nnz - ell_max_nnz_per_row);
+    }
+}
+
+template <typename IndexType>
+void calculate_hybrid_coo_row_nnz(dim3 grid, dim3 block,
+                                  gko::size_type dynamic_shared_memory,
+                                  sycl::queue *stream, size_type num_rows,
+                                  size_type ell_max_nnz_per_row,
+                                  IndexType *csr_row_idxs,
+                                  size_type *coo_row_nnz)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                calculate_hybrid_coo_row_nnz(num_rows, ell_max_nnz_per_row,
+                                             csr_row_idxs, coo_row_nnz,
+                                             item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void fill_in_hybrid(size_type num_rows, size_type stride,
+                    size_type ell_max_nnz_per_row,
+                    const ValueType *__restrict__ source_values,
+                    const IndexType *__restrict__ source_row_ptrs,
+                    const IndexType *__restrict__ source_col_idxs,
+                    const size_type *__restrict__ coo_offset,
+                    ValueType *__restrict__ result_ell_val,
+                    IndexType *__restrict__ result_ell_col,
+                    ValueType *__restrict__ result_coo_val,
+                    IndexType *__restrict__ result_coo_col,
+                    IndexType *__restrict__ result_coo_row,
+                    sycl::nd_item<3> item_ct1)
+{
+    constexpr auto warp_size = config::warp_size;
+    const auto row = thread::get_subwarp_id_flat<warp_size>(item_ct1);
+    const auto local_tidx = item_ct1.get_local_id(2) % warp_size;
+
+    if (row < num_rows) {
+        for (size_type i = local_tidx;
+             i < source_row_ptrs[row + 1] - source_row_ptrs[row];
+             i += warp_size) {
+            const auto source_idx = i + source_row_ptrs[row];
+            if (i < ell_max_nnz_per_row) {
+                const auto result_idx = row + stride * i;
+                result_ell_val[result_idx] = source_values[source_idx];
+                result_ell_col[result_idx] = source_col_idxs[source_idx];
+            } else {
+                const auto result_idx =
+                    coo_offset[row] + i - ell_max_nnz_per_row;
+                result_coo_val[result_idx] = source_values[source_idx];
+                result_coo_col[result_idx] = source_col_idxs[source_idx];
+                result_coo_row[result_idx] = row;
+            }
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void fill_in_hybrid(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                    sycl::queue *stream, size_type num_rows, size_type stride,
+                    size_type ell_max_nnz_per_row,
+                    const ValueType *source_values,
+                    const IndexType *source_row_ptrs,
+                    const IndexType *source_col_idxs,
+                    const size_type *coo_offset, ValueType *result_ell_val,
+                    IndexType *result_ell_col, ValueType *result_coo_val,
+                    IndexType *result_coo_col, IndexType *result_coo_row)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                fill_in_hybrid(num_rows, stride, ell_max_nnz_per_row,
+                               source_values, source_row_ptrs, source_col_idxs,
+                               coo_offset, result_ell_val, result_ell_col,
+                               result_coo_val, result_coo_col, result_coo_row,
+                               item_ct1);
+            });
+    });
+}
+
+
+template <typename IndexType>
+void check_unsorted(const IndexType *__restrict__ row_ptrs,
+                    const IndexType *__restrict__ col_idxs, IndexType num_rows,
+                    bool *flag, sycl::nd_item<3> item_ct1, bool *sh_flag)
+{
+    auto block = group::this_thread_block(item_ct1);
+    if (block.thread_rank() == 0) {
+        *sh_flag = *flag;
+    }
+    block.sync();
+
+    auto row = thread::get_thread_id_flat<IndexType>(item_ct1);
+    if (row >= num_rows) {
+        return;
+    }
+
+    // fail early
+    if ((*sh_flag)) {
+        for (auto nz = row_ptrs[row]; nz < row_ptrs[row + 1] - 1; ++nz) {
+            if (col_idxs[nz] > col_idxs[nz + 1]) {
+                *flag = false;
+                *sh_flag = false;
+                return;
+            }
+        }
+    }
+}
+
+template <typename IndexType>
+void check_unsorted(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                    sycl::queue *stream, const IndexType *row_ptrs,
+                    const IndexType *col_idxs, IndexType num_rows, bool *flag)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        sycl::accessor<bool, 0, sycl::access_mode::read_write,
+                       sycl::access::target::local>
+            sh_flag_acc_ct1(cgh);
+
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                check_unsorted(row_ptrs, col_idxs, num_rows, flag, item_ct1,
+                               sh_flag_acc_ct1.get_pointer());
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(size_type diag_size, size_type nnz,
+                      const ValueType *__restrict__ orig_values,
+                      const IndexType *__restrict__ orig_row_ptrs,
+                      const IndexType *__restrict__ orig_col_idxs,
+                      ValueType *__restrict__ diag, sycl::nd_item<3> item_ct1)
+{
+    constexpr auto warp_size = config::warp_size;
+    const auto row = thread::get_subwarp_id_flat<warp_size>(item_ct1);
+    const auto local_tidx = item_ct1.get_local_id(2) % warp_size;
+
+    if (row < diag_size) {
+        for (size_type i = local_tidx;
+             i < orig_row_ptrs[row + 1] - orig_row_ptrs[row]; i += warp_size) {
+            const auto orig_idx = i + orig_row_ptrs[row];
+            if (orig_idx < nnz) {
+                if (orig_col_idxs[orig_idx] == row) {
+                    diag[row] = orig_values[orig_idx];
+                    return;
+                }
+            }
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(dim3 grid, dim3 block,
+                      gko::size_type dynamic_shared_memory, sycl::queue *stream,
+                      size_type diag_size, size_type nnz,
+                      const ValueType *orig_values,
+                      const IndexType *orig_row_ptrs,
+                      const IndexType *orig_col_idxs, ValueType *diag)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                extract_diagonal(diag_size, nnz, orig_values, orig_row_ptrs,
+                                 orig_col_idxs, diag, item_ct1);
+            });
+    });
+}
+
+
+}  // namespace kernel
+
+
+namespace {
+
+
+template <typename ValueType>
+void conjugate_kernel(size_type num_nonzeros, ValueType *__restrict__ val,
+                      sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+
+    if (tidx < num_nonzeros) {
+        /*
+        DPCT1007:26: Migration of this CUDA API is not supported by the Intel(R)
+        DPC++ Compatibility Tool.
+        */
+        val[tidx] = conj(val[tidx]);
+    }
+}
+
+template <typename ValueType>
+void conjugate_kernel(dim3 grid, dim3 block,
+                      gko::size_type dynamic_shared_memory, sycl::queue *stream,
+                      size_type num_nonzeros, ValueType *val)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             conjugate_kernel(num_nonzeros, val, item_ct1);
+                         });
+    });
+}
+
+
+}  //  namespace
+
+
+template <typename IndexType>
+void inv_permutation_kernel(size_type size,
+                            const IndexType *__restrict__ permutation,
+                            IndexType *__restrict__ inv_permutation,
+                            sycl::nd_item<3> item_ct1)
+{
+    auto tid = thread::get_thread_id_flat(item_ct1);
+    if (tid >= size) {
+        return;
+    }
+    inv_permutation[permutation[tid]] = tid;
+}
+
+template <typename IndexType>
+void inv_permutation_kernel(dim3 grid, dim3 block,
+                            gko::size_type dynamic_shared_memory,
+                            sycl::queue *stream, size_type size,
+                            const IndexType *permutation,
+                            IndexType *inv_permutation)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             inv_permutation_kernel(size, permutation,
+                                                    inv_permutation, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void col_permute_kernel(size_type num_rows, size_type num_nonzeros,
+                        const IndexType *__restrict__ permutation,
+                        const IndexType *__restrict__ in_row_ptrs,
+                        const IndexType *__restrict__ in_cols,
+                        const ValueType *__restrict__ in_vals,
+                        IndexType *__restrict__ out_row_ptrs,
+                        IndexType *__restrict__ out_cols,
+                        ValueType *__restrict__ out_vals,
+                        sycl::nd_item<3> item_ct1)
+{
+    auto tid = thread::get_thread_id_flat(item_ct1);
+    if (tid < num_nonzeros) {
+        out_cols[tid] = permutation[in_cols[tid]];
+        out_vals[tid] = in_vals[tid];
+    }
+    if (tid <= num_rows) {
+        out_row_ptrs[tid] = in_row_ptrs[tid];
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void col_permute_kernel(dim3 grid, dim3 block,
+                        gko::size_type dynamic_shared_memory,
+                        sycl::queue *stream, size_type num_rows,
+                        size_type num_nonzeros, const IndexType *permutation,
+                        const IndexType *in_row_ptrs, const IndexType *in_cols,
+                        const ValueType *in_vals, IndexType *out_row_ptrs,
+                        IndexType *out_cols, ValueType *out_vals)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                col_permute_kernel(num_rows, num_nonzeros, permutation,
+                                   in_row_ptrs, in_cols, in_vals, out_row_ptrs,
+                                   out_cols, out_vals, item_ct1);
+            });
+    });
+}
+
+
+template <typename IndexType>
+void row_ptr_permute_kernel(size_type num_rows,
+                            const IndexType *__restrict__ permutation,
+                            const IndexType *__restrict__ in_row_ptrs,
+                            IndexType *__restrict__ out_nnz,
+                            sycl::nd_item<3> item_ct1)
+{
+    auto tid = thread::get_thread_id_flat(item_ct1);
+    if (tid >= num_rows) {
+        return;
+    }
+    auto in_row = permutation[tid];
+    auto out_row = tid;
+    out_nnz[out_row] = in_row_ptrs[in_row + 1] - in_row_ptrs[in_row];
+}
+
+template <typename IndexType>
+void row_ptr_permute_kernel(dim3 grid, dim3 block,
+                            gko::size_type dynamic_shared_memory,
+                            sycl::queue *stream, size_type num_rows,
+                            const IndexType *permutation,
+                            const IndexType *in_row_ptrs, IndexType *out_nnz)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                row_ptr_permute_kernel(num_rows, permutation, in_row_ptrs,
+                                       out_nnz, item_ct1);
+            });
+    });
+}
+
+
+template <typename IndexType>
+void inv_row_ptr_permute_kernel(size_type num_rows,
+                                const IndexType *__restrict__ permutation,
+                                const IndexType *__restrict__ in_row_ptrs,
+                                IndexType *__restrict__ out_nnz,
+                                sycl::nd_item<3> item_ct1)
+{
+    auto tid = thread::get_thread_id_flat(item_ct1);
+    if (tid >= num_rows) {
+        return;
+    }
+    auto in_row = tid;
+    auto out_row = permutation[tid];
+    out_nnz[out_row] = in_row_ptrs[in_row + 1] - in_row_ptrs[in_row];
+}
+
+template <typename IndexType>
+void inv_row_ptr_permute_kernel(dim3 grid, dim3 block,
+                                gko::size_type dynamic_shared_memory,
+                                sycl::queue *stream, size_type num_rows,
+                                const IndexType *permutation,
+                                const IndexType *in_row_ptrs,
+                                IndexType *out_nnz)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                inv_row_ptr_permute_kernel(num_rows, permutation, in_row_ptrs,
+                                           out_nnz, item_ct1);
+            });
+    });
+}
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+void row_permute_kernel(size_type num_rows,
+                        const IndexType *__restrict__ permutation,
+                        const IndexType *__restrict__ in_row_ptrs,
+                        const IndexType *__restrict__ in_cols,
+                        const ValueType *__restrict__ in_vals,
+                        const IndexType *__restrict__ out_row_ptrs,
+                        IndexType *__restrict__ out_cols,
+                        ValueType *__restrict__ out_vals,
+                        sycl::nd_item<3> item_ct1)
+{
+    auto tid = thread::get_subwarp_id_flat<subwarp_size>(item_ct1);
+    if (tid >= num_rows) {
+        return;
+    }
+    auto lane = item_ct1.get_local_id(2) % subwarp_size;
+    auto in_row = permutation[tid];
+    auto out_row = tid;
+    auto in_begin = in_row_ptrs[in_row];
+    auto in_size = in_row_ptrs[in_row + 1] - in_begin;
+    auto out_begin = out_row_ptrs[out_row];
+    for (IndexType i = lane; i < in_size; i += subwarp_size) {
+        out_cols[out_begin + i] = in_cols[in_begin + i];
+        out_vals[out_begin + i] = in_vals[in_begin + i];
+    }
+}
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+void row_permute_kernel(dim3 grid, dim3 block,
+                        gko::size_type dynamic_shared_memory,
+                        sycl::queue *stream, size_type num_rows,
+                        const IndexType *permutation,
+                        const IndexType *in_row_ptrs, const IndexType *in_cols,
+                        const ValueType *in_vals, const IndexType *out_row_ptrs,
+                        IndexType *out_cols, ValueType *out_vals)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                row_permute_kernel<subwarp_size>(
+                    num_rows, permutation, in_row_ptrs, in_cols, in_vals,
+                    out_row_ptrs, out_cols, out_vals, item_ct1);
+            });
+    });
+}
+
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+void inv_row_permute_kernel(size_type num_rows,
+                            const IndexType *__restrict__ permutation,
+                            const IndexType *__restrict__ in_row_ptrs,
+                            const IndexType *__restrict__ in_cols,
+                            const ValueType *__restrict__ in_vals,
+                            const IndexType *__restrict__ out_row_ptrs,
+                            IndexType *__restrict__ out_cols,
+                            ValueType *__restrict__ out_vals,
+                            sycl::nd_item<3> item_ct1)
+{
+    auto tid = thread::get_subwarp_id_flat<subwarp_size>(item_ct1);
+    if (tid >= num_rows) {
+        return;
+    }
+    auto lane = item_ct1.get_local_id(2) % subwarp_size;
+    auto in_row = tid;
+    auto out_row = permutation[tid];
+    auto in_begin = in_row_ptrs[in_row];
+    auto in_size = in_row_ptrs[in_row + 1] - in_begin;
+    auto out_begin = out_row_ptrs[out_row];
+    for (IndexType i = lane; i < in_size; i += subwarp_size) {
+        out_cols[out_begin + i] = in_cols[in_begin + i];
+        out_vals[out_begin + i] = in_vals[in_begin + i];
+    }
+}
+
+template <int subwarp_size, typename ValueType, typename IndexType>
+void inv_row_permute_kernel(dim3 grid, dim3 block,
+                            gko::size_type dynamic_shared_memory,
+                            sycl::queue *stream, size_type num_rows,
+                            const IndexType *permutation,
+                            const IndexType *in_row_ptrs,
+                            const IndexType *in_cols, const ValueType *in_vals,
+                            const IndexType *out_row_ptrs, IndexType *out_cols,
+                            ValueType *out_vals)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                inv_row_permute_kernel<subwarp_size>(
+                    num_rows, permutation, in_row_ptrs, in_cols, in_vals,
+                    out_row_ptrs, out_cols, out_vals, item_ct1);
+            });
+    });
+}
+
 
+template <int subwarp_size, typename ValueType, typename IndexType>
+void inv_symm_permute_kernel(size_type num_rows,
+                             const IndexType *__restrict__ permutation,
+                             const IndexType *__restrict__ in_row_ptrs,
+                             const IndexType *__restrict__ in_cols,
+                             const ValueType *__restrict__ in_vals,
+                             const IndexType *__restrict__ out_row_ptrs,
+                             IndexType *__restrict__ out_cols,
+                             ValueType *__restrict__ out_vals,
+                             sycl::nd_item<3> item_ct1)
+{
+    auto tid = thread::get_subwarp_id_flat<subwarp_size>(item_ct1);
+    if (tid >= num_rows) {
+        return;
+    }
+    auto lane = item_ct1.get_local_id(2) % subwarp_size;
+    auto in_row = tid;
+    auto out_row = permutation[tid];
+    auto in_begin = in_row_ptrs[in_row];
+    auto in_size = in_row_ptrs[in_row + 1] - in_begin;
+    auto out_begin = out_row_ptrs[out_row];
+    for (IndexType i = lane; i < in_size; i += subwarp_size) {
+        out_cols[out_begin + i] = permutation[in_cols[in_begin + i]];
+        out_vals[out_begin + i] = in_vals[in_begin + i];
+    }
+}
 
-#include <algorithm>
-#include <limits>
-#include <numeric>
-#include <utility>
+template <int subwarp_size, typename ValueType, typename IndexType>
+void inv_symm_permute_kernel(dim3 grid, dim3 block,
+                             gko::size_type dynamic_shared_memory,
+                             sycl::queue *stream, size_type num_rows,
+                             const IndexType *permutation,
+                             const IndexType *in_row_ptrs,
+                             const IndexType *in_cols, const ValueType *in_vals,
+                             const IndexType *out_row_ptrs, IndexType *out_cols,
+                             ValueType *out_vals)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                inv_symm_permute_kernel<subwarp_size>(
+                    num_rows, permutation, in_row_ptrs, in_cols, in_vals,
+                    out_row_ptrs, out_cols, out_vals, item_ct1);
+            });
+    });
+}
 
+namespace host_kernel {
 
-#include <CL/sycl.hpp>
 
+template <int items_per_thread, typename ValueType, typename IndexType>
+void merge_path_spmv(syn::value_list<int, items_per_thread>,
+                     std::shared_ptr<const DpcppExecutor> exec,
+                     const matrix::Csr<ValueType, IndexType> *a,
+                     const matrix::Dense<ValueType> *b,
+                     matrix::Dense<ValueType> *c,
+                     const matrix::Dense<ValueType> *alpha = nullptr,
+                     const matrix::Dense<ValueType> *beta = nullptr)
+{
+    const IndexType total = a->get_size()[0] + a->get_num_stored_elements();
+    const IndexType grid_num =
+        ceildiv(total, spmv_block_size * items_per_thread);
+    const dim3 grid(grid_num);
+    const dim3 block(spmv_block_size);
+    Array<IndexType> row_out(exec, grid_num);
+    Array<ValueType> val_out(exec, grid_num);
+
+    for (IndexType column_id = 0; column_id < b->get_size()[1]; column_id++) {
+        if (alpha == nullptr && beta == nullptr) {
+            const auto b_vals = b->get_const_values() + column_id;
+            auto c_vals = c->get_values() + column_id;
+            kernel::abstract_merge_path_spmv<items_per_thread>(
+                grid, block, 0, exec->get_queue(),
+                static_cast<IndexType>(a->get_size()[0]), a->get_const_values(),
+                a->get_const_col_idxs(), a->get_const_row_ptrs(),
+                a->get_const_srow(), b_vals, b->get_stride(), c_vals,
+                c->get_stride(), row_out.get_data(), val_out.get_data());
+            kernel::abstract_reduce(1, spmv_block_size, 0, exec->get_queue(),
+                                    grid_num, val_out.get_data(),
+                                    row_out.get_data(), c_vals,
+                                    c->get_stride());
+
+        } else if (alpha != nullptr && beta != nullptr) {
+            const auto b_vals = b->get_const_values() + column_id;
+            auto c_vals = c->get_values() + column_id;
+            kernel::abstract_merge_path_spmv<items_per_thread>(
+                grid, block, 0, exec->get_queue(),
+                static_cast<IndexType>(a->get_size()[0]),
+                alpha->get_const_values(), a->get_const_values(),
+                a->get_const_col_idxs(), a->get_const_row_ptrs(),
+                a->get_const_srow(), b_vals, b->get_stride(),
+                beta->get_const_values(), c_vals, c->get_stride(),
+                row_out.get_data(), val_out.get_data());
+            kernel::abstract_reduce(
+                1, spmv_block_size, 0, exec->get_queue(), grid_num,
+                val_out.get_data(), row_out.get_data(),
+                alpha->get_const_values(), c_vals, c->get_stride());
+        } else {
+            GKO_KERNEL_NOT_FOUND;
+        }
+    }
+}
 
-#include <ginkgo/core/base/array.hpp>
-#include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/matrix/coo.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-#include <ginkgo/core/matrix/hybrid.hpp>
+GKO_ENABLE_IMPLEMENTATION_SELECTION(select_merge_path_spmv, merge_path_spmv);
 
 
-#include "core/base/allocator.hpp"
-#include "core/base/iterator_factory.hpp"
-#include "core/base/utils.hpp"
-#include "core/components/fill_array.hpp"
-#include "core/components/prefix_sum.hpp"
-#include "core/matrix/csr_builder.hpp"
-#include "dpcpp/components/atomic.dp.hpp"
-#include "dpcpp/components/format_conversion.dp.hpp"
+template <typename ValueType, typename IndexType>
+int compute_items_per_thread(std::shared_ptr<const DpcppExecutor> exec)
+{
+    const int version =
+        (exec->get_major_version() << 4) + exec->get_minor_version();
+    // The num_item is decided to make the occupancy 100%
+    // TODO: Extend this list when new GPU is released
+    //       Tune this parameter
+    // 128 threads/block the number of items per threads
+    // 3.0 3.5: 6
+    // 3.7: 14
+    // 5.0, 5.3, 6.0, 6.2: 8
+    // 5.2, 6.1, 7.0: 12
+    int num_item = 6;
+    switch (version) {
+    case 0x50:
+    case 0x53:
+    case 0x60:
+    case 0x62:
+        num_item = 8;
+        break;
+    case 0x52:
+    case 0x61:
+    case 0x70:
+        num_item = 12;
+        break;
+    case 0x37:
+        num_item = 14;
+    }
+    // Ensure that the following is satisfied:
+    // sizeof(IndexType) + sizeof(ValueType)
+    // <= items_per_thread * sizeof(IndexType)
+    constexpr int minimal_num =
+        ceildiv(sizeof(IndexType) + sizeof(ValueType), sizeof(IndexType));
+    int items_per_thread = num_item * 4 / sizeof(IndexType);
+    return std::max(minimal_num, items_per_thread);
+}
 
 
-namespace gko {
-namespace kernels {
-namespace dpcpp {
-/**
- * @brief The Compressed sparse row matrix format namespace.
- *
- * @ingroup csr
- */
-namespace csr {
+template <int subwarp_size, typename ValueType, typename IndexType>
+void classical_spmv(syn::value_list<int, subwarp_size>,
+                    std::shared_ptr<const DpcppExecutor> exec,
+                    const matrix::Csr<ValueType, IndexType> *a,
+                    const matrix::Dense<ValueType> *b,
+                    matrix::Dense<ValueType> *c,
+                    const matrix::Dense<ValueType> *alpha = nullptr,
+                    const matrix::Dense<ValueType> *beta = nullptr)
+{
+    const auto nwarps = exec->get_num_warps_per_sm() *
+                        exec->get_num_multiprocessor() * classical_overweight;
+    const auto gridx =
+        std::min(ceildiv(a->get_size()[0], spmv_block_size / subwarp_size),
+                 int64(nwarps / warps_in_block));
+    const dim3 grid(gridx, b->get_size()[1]);
+    const dim3 block(spmv_block_size);
+
+    if (alpha == nullptr && beta == nullptr) {
+        kernel::abstract_classical_spmv<subwarp_size>(
+            grid, block, 0, exec->get_queue(), a->get_size()[0],
+            a->get_const_values(), a->get_const_col_idxs(),
+            a->get_const_row_ptrs(), b->get_const_values(), b->get_stride(),
+            c->get_values(), c->get_stride());
+
+    } else if (alpha != nullptr && beta != nullptr) {
+        kernel::abstract_classical_spmv<subwarp_size>(
+            grid, block, 0, exec->get_queue(), a->get_size()[0],
+            alpha->get_const_values(), a->get_const_values(),
+            a->get_const_col_idxs(), a->get_const_row_ptrs(),
+            b->get_const_values(), b->get_stride(), beta->get_const_values(),
+            c->get_values(), c->get_stride());
+    } else {
+        GKO_KERNEL_NOT_FOUND;
+    }
+}
+
+GKO_ENABLE_IMPLEMENTATION_SELECTION(select_classical_spmv, classical_spmv);
+
+
+}  // namespace host_kernel
 
 
 template <typename ValueType, typename IndexType>
 void spmv(std::shared_ptr<const DpcppExecutor> exec,
           const matrix::Csr<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b,
-          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+{
+    if (a->get_strategy()->get_name() == "load_balance") {
+        components::fill_array(exec, c->get_values(),
+                               c->get_num_stored_elements(), zero<ValueType>());
+        const IndexType nwarps = a->get_num_srow_elements();
+        if (nwarps > 0) {
+            const dim3 csr_block(config::warp_size, warps_in_block, 1);
+            const dim3 csr_grid(ceildiv(nwarps, warps_in_block),
+                                b->get_size()[1]);
+            kernel::abstract_spmv(
+                csr_grid, csr_block, 0, exec->get_queue(), nwarps,
+                static_cast<IndexType>(a->get_size()[0]), a->get_const_values(),
+                a->get_const_col_idxs(), a->get_const_row_ptrs(),
+                a->get_const_srow(), b->get_const_values(), b->get_stride(),
+                c->get_values(), c->get_stride());
+        } else {
+            GKO_NOT_SUPPORTED(nwarps);
+        }
+    } else if (a->get_strategy()->get_name() == "merge_path") {
+        int items_per_thread =
+            host_kernel::compute_items_per_thread<ValueType, IndexType>(exec);
+        host_kernel::select_merge_path_spmv(
+            compiled_kernels(),
+            [&items_per_thread](int compiled_info) {
+                return items_per_thread == compiled_info;
+            },
+            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c);
+    } else if (a->get_strategy()->get_name() == "classical") {
+        IndexType max_length_per_row = 0;
+        using Tcsr = matrix::Csr<ValueType, IndexType>;
+        if (auto strategy =
+                std::dynamic_pointer_cast<const typename Tcsr::classical>(
+                    a->get_strategy())) {
+            max_length_per_row = strategy->get_max_length_per_row();
+        } else if (auto strategy = std::dynamic_pointer_cast<
+                       const typename Tcsr::automatical>(a->get_strategy())) {
+            max_length_per_row = strategy->get_max_length_per_row();
+        } else {
+            GKO_NOT_SUPPORTED(a->get_strategy());
+        }
+        host_kernel::select_classical_spmv(
+            classical_kernels(),
+            [&max_length_per_row](int compiled_info) {
+                return max_length_per_row >= compiled_info;
+            },
+            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c);
+    } else if (a->get_strategy()->get_name() == "sparselib" ||
+               a->get_strategy()->get_name() == "cusparse") {
+        if (cusparse::is_supported<ValueType, IndexType>::value) {
+            // TODO: add implementation for int64 and multiple RHS
+            auto handle = exec->get_cusparse_handle();
+            {
+                cusparse::pointer_mode_guard pm_guard(handle);
+                const auto alpha = one<ValueType>();
+                const auto beta = zero<ValueType>();
+                // TODO: add implementation for int64 and multiple RHS
+                if (b->get_stride() != 1 || c->get_stride() != 1)
+                    GKO_NOT_IMPLEMENTED;
+
+#if defined(DPCPP_VERSION) && (DPCPP_VERSION < 11000)
+                auto descr = cusparse::create_mat_descr();
+                auto row_ptrs = a->get_const_row_ptrs();
+                auto col_idxs = a->get_const_col_idxs();
+                cusparse::spmv(handle, oneapi::mkl::transpose::nontrans,
+                               a->get_size()[0], a->get_size()[1],
+                               a->get_num_stored_elements(), &alpha, descr,
+                               a->get_const_values(), row_ptrs, col_idxs,
+                               b->get_const_values(), &beta, c->get_values());
+
+                cusparse::destroy(descr);
+#else  // DPCPP_VERSION >= 11000
+                cusparseOperation_t trans = CUSPARSE_OPERATION_NON_TRANSPOSE;
+                cusparseSpMVAlg_t alg = CUSPARSE_CSRMV_ALG1;
+                auto row_ptrs =
+                    const_cast<IndexType *>(a->get_const_row_ptrs());
+                auto col_idxs =
+                    const_cast<IndexType *>(a->get_const_col_idxs());
+                auto values = const_cast<ValueType *>(a->get_const_values());
+                auto mat = cusparse::create_csr(
+                    a->get_size()[0], a->get_size()[1],
+                    a->get_num_stored_elements(), row_ptrs, col_idxs, values);
+                auto b_val = const_cast<ValueType *>(b->get_const_values());
+                auto c_val = c->get_values();
+                auto vecb =
+                    cusparse::create_dnvec(b->get_num_stored_elements(), b_val);
+                auto vecc =
+                    cusparse::create_dnvec(c->get_num_stored_elements(), c_val);
+                size_type buffer_size = 0;
+                cusparse::spmv_buffersize<ValueType>(handle, trans, &alpha, mat,
+                                                     vecb, &beta, vecc, alg,
+                                                     &buffer_size);
+
+                gko::Array<char> buffer_array(exec, buffer_size);
+                auto buffer = buffer_array.get_data();
+                cusparse::spmv<ValueType>(handle, trans, &alpha, mat, vecb,
+                                          &beta, vecc, alg, buffer);
+                cusparse::destroy(vecb);
+                cusparse::destroy(vecc);
+                cusparse::destroy(mat);
+#endif
+            }
+        } else {
+            GKO_NOT_IMPLEMENTED;
+        }
+    } else {
+        GKO_NOT_IMPLEMENTED;
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPMV_KERNEL);
 
@@ -86,271 +2098,286 @@ void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::Csr<ValueType, IndexType> *a,
                    const matrix::Dense<ValueType> *b,
                    const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+                   matrix::Dense<ValueType> *c)
+{
+    if (a->get_strategy()->get_name() == "load_balance") {
+        dense::scale(exec, beta, c);
+
+        const IndexType nwarps = a->get_num_srow_elements();
+
+        if (nwarps > 0) {
+            const dim3 csr_block(config::warp_size, warps_in_block, 1);
+            const dim3 csr_grid(ceildiv(nwarps, warps_in_block),
+                                b->get_size()[1]);
+            kernel::abstract_spmv(
+                csr_grid, csr_block, 0, exec->get_queue(), nwarps,
+                static_cast<IndexType>(a->get_size()[0]),
+                alpha->get_const_values(), a->get_const_values(),
+                a->get_const_col_idxs(), a->get_const_row_ptrs(),
+                a->get_const_srow(), b->get_const_values(), b->get_stride(),
+                c->get_values(), c->get_stride());
+        } else {
+            GKO_NOT_SUPPORTED(nwarps);
+        }
+    } else if (a->get_strategy()->get_name() == "sparselib" ||
+               a->get_strategy()->get_name() == "cusparse") {
+        if (cusparse::is_supported<ValueType, IndexType>::value) {
+            // TODO: add implementation for int64 and multiple RHS
+            if (b->get_stride() != 1 || c->get_stride() != 1)
+                GKO_NOT_IMPLEMENTED;
+
+#if defined(DPCPP_VERSION) && (DPCPP_VERSION < 11000)
+            auto descr = cusparse::create_mat_descr();
+            auto row_ptrs = a->get_const_row_ptrs();
+            auto col_idxs = a->get_const_col_idxs();
+            cusparse::spmv(exec->get_cusparse_handle(),
+                           oneapi::mkl::transpose::nontrans, a->get_size()[0],
+                           a->get_size()[1], a->get_num_stored_elements(),
+                           alpha->get_const_values(), descr,
+                           a->get_const_values(), row_ptrs, col_idxs,
+                           b->get_const_values(), beta->get_const_values(),
+                           c->get_values());
+
+            cusparse::destroy(descr);
+#else  // DPCPP_VERSION >= 11000
+            cusparseOperation_t trans = CUSPARSE_OPERATION_NON_TRANSPOSE;
+            cusparseSpMVAlg_t alg = CUSPARSE_CSRMV_ALG1;
+            auto row_ptrs = const_cast<IndexType *>(a->get_const_row_ptrs());
+            auto col_idxs = const_cast<IndexType *>(a->get_const_col_idxs());
+            auto values = const_cast<ValueType *>(a->get_const_values());
+            auto mat = cusparse::create_csr(a->get_size()[0], a->get_size()[1],
+                                            a->get_num_stored_elements(),
+                                            row_ptrs, col_idxs, values);
+            auto b_val = const_cast<ValueType *>(b->get_const_values());
+            auto c_val = c->get_values();
+            auto vecb =
+                cusparse::create_dnvec(b->get_num_stored_elements(), b_val);
+            auto vecc =
+                cusparse::create_dnvec(c->get_num_stored_elements(), c_val);
+            size_type buffer_size = 0;
+            cusparse::spmv_buffersize<ValueType>(
+                exec->get_cusparse_handle(), trans, alpha->get_const_values(),
+                mat, vecb, beta->get_const_values(), vecc, alg, &buffer_size);
+            gko::Array<char> buffer_array(exec, buffer_size);
+            auto buffer = buffer_array.get_data();
+            cusparse::spmv<ValueType>(
+                exec->get_cusparse_handle(), trans, alpha->get_const_values(),
+                mat, vecb, beta->get_const_values(), vecc, alg, buffer);
+            cusparse::destroy(vecb);
+            cusparse::destroy(vecc);
+            cusparse::destroy(mat);
+#endif
+        } else {
+            GKO_NOT_IMPLEMENTED;
+        }
+    } else if (a->get_strategy()->get_name() == "classical") {
+        IndexType max_length_per_row = 0;
+        using Tcsr = matrix::Csr<ValueType, IndexType>;
+        if (auto strategy =
+                std::dynamic_pointer_cast<const typename Tcsr::classical>(
+                    a->get_strategy())) {
+            max_length_per_row = strategy->get_max_length_per_row();
+        } else if (auto strategy = std::dynamic_pointer_cast<
+                       const typename Tcsr::automatical>(a->get_strategy())) {
+            max_length_per_row = strategy->get_max_length_per_row();
+        } else {
+            GKO_NOT_SUPPORTED(a->get_strategy());
+        }
+        host_kernel::select_classical_spmv(
+            classical_kernels(),
+            [&max_length_per_row](int compiled_info) {
+                return max_length_per_row >= compiled_info;
+            },
+            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c, alpha,
+            beta);
+    } else if (a->get_strategy()->get_name() == "merge_path") {
+        int items_per_thread =
+            host_kernel::compute_items_per_thread<ValueType, IndexType>(exec);
+        host_kernel::select_merge_path_spmv(
+            compiled_kernels(),
+            [&items_per_thread](int compiled_info) {
+                return items_per_thread == compiled_info;
+            },
+            syn::value_list<int>(), syn::type_list<>(), exec, a, b, c, alpha,
+            beta);
+    } else {
+        GKO_NOT_IMPLEMENTED;
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_ADVANCED_SPMV_KERNEL);
 
 
-namespace {
-
-
-/**
- * @internal
- *
- * Entry in a heap storing a column index and associated non-zero index
- * (and row end) from a matrix.
- *
- * @tparam ValueType  The value type for matrices.
- * @tparam IndexType  The index type for matrices.
- */
-template <typename ValueType, typename IndexType>
-struct col_heap_element {
-    using value_type = ValueType;
-    using index_type = IndexType;
-
-    IndexType idx;
-    IndexType end;
-    IndexType col;
-
-    ValueType val() const { return zero<ValueType>(); }
-
-    col_heap_element(IndexType idx, IndexType end, IndexType col, ValueType)
-        : idx{idx}, end{end}, col{col}
-    {}
-};
-
-
-/**
- * @internal
- *
- * Entry in a heap storing an entry (value and column index) and associated
- * non-zero index (and row end) from a matrix.
- *
- * @tparam ValueType  The value type for matrices.
- * @tparam IndexType  The index type for matrices.
- */
 template <typename ValueType, typename IndexType>
-struct val_heap_element {
-    using value_type = ValueType;
-    using index_type = IndexType;
-
-    IndexType idx;
-    IndexType end;
-    IndexType col;
-    ValueType val_;
-
-    ValueType val() const { return val_; }
-
-    val_heap_element(IndexType idx, IndexType end, IndexType col, ValueType val)
-        : idx{idx}, end{end}, col{col}, val_{val}
-    {}
-};
-
+void spgemm(std::shared_ptr<const DpcppExecutor> exec,
+            const matrix::Csr<ValueType, IndexType> *a,
+            const matrix::Csr<ValueType, IndexType> *b,
+            matrix::Csr<ValueType, IndexType> *c)
+{
+    auto a_nnz = IndexType(a->get_num_stored_elements());
+    auto a_vals = a->get_const_values();
+    auto a_row_ptrs = a->get_const_row_ptrs();
+    auto a_col_idxs = a->get_const_col_idxs();
+    auto b_vals = b->get_const_values();
+    auto b_row_ptrs = b->get_const_row_ptrs();
+    auto b_col_idxs = b->get_const_col_idxs();
+    auto c_row_ptrs = c->get_row_ptrs();
 
-/**
- * @internal
- *
- * Restores the binary heap condition downwards from a given index.
- *
- * The heap condition is: col(child) >= col(parent)
- *
- * @param heap  a pointer to the array containing the heap elements.
- * @param idx  the index of the starting heap node that potentially
- *             violates the heap condition.
- * @param size  the number of elements in the heap.
- * @tparam HeapElement  the element type in the heap. See col_heap_element and
- *                      val_heap_element
- */
-template <typename HeapElement>
-void sift_down(HeapElement *heap, typename HeapElement::index_type idx,
-               typename HeapElement::index_type size)
-{
-    auto curcol = heap[idx].col;
-    while (idx * 2 + 1 < size) {
-        auto lchild = idx * 2 + 1;
-        auto rchild = min(lchild + 1, size - 1);
-        auto lcol = heap[lchild].col;
-        auto rcol = heap[rchild].col;
-        auto mincol = min(lcol, rcol);
-        if (mincol >= curcol) {
-            break;
-        }
-        auto minchild = lcol == mincol ? lchild : rchild;
-        std::swap(heap[minchild], heap[idx]);
-        idx = minchild;
+    if (cusparse::is_supported<ValueType, IndexType>::value) {
+        auto handle = exec->get_cusparse_handle();
+        cusparse::pointer_mode_guard pm_guard(handle);
+
+        auto alpha = one<ValueType>();
+        auto a_nnz = static_cast<IndexType>(a->get_num_stored_elements());
+        auto b_nnz = static_cast<IndexType>(b->get_num_stored_elements());
+        auto null_value = static_cast<ValueType *>(nullptr);
+        auto null_index = static_cast<IndexType *>(nullptr);
+        auto zero_nnz = IndexType{};
+        auto m = IndexType(a->get_size()[0]);
+        auto n = IndexType(b->get_size()[1]);
+        auto k = IndexType(a->get_size()[1]);
+        matrix::CsrBuilder<ValueType, IndexType> c_builder{c};
+        auto &c_col_idxs_array = c_builder.get_col_idx_array();
+        auto &c_vals_array = c_builder.get_value_array();
+
+#if defined(DPCPP_VERSION) && (DPCPP_VERSION < 11000)
+        auto a_descr = cusparse::create_mat_descr();
+        auto b_descr = cusparse::create_mat_descr();
+        auto c_descr = cusparse::create_mat_descr();
+        auto d_descr = cusparse::create_mat_descr();
+        auto info = cusparse::create_spgemm_info();
+        // allocate buffer
+        size_type buffer_size{};
+        cusparse::spgemm_buffer_size(
+            handle, m, n, k, &alpha, a_descr, a_nnz, a_row_ptrs, a_col_idxs,
+            b_descr, b_nnz, b_row_ptrs, b_col_idxs, null_value, d_descr,
+            zero_nnz, null_index, null_index, info, buffer_size);
+        Array<char> buffer_array(exec, buffer_size);
+        auto buffer = buffer_array.get_data();
+
+        // count nnz
+        IndexType c_nnz{};
+        cusparse::spgemm_nnz(handle, m, n, k, a_descr, a_nnz, a_row_ptrs,
+                             a_col_idxs, b_descr, b_nnz, b_row_ptrs, b_col_idxs,
+                             d_descr, zero_nnz, null_index, null_index, c_descr,
+                             c_row_ptrs, &c_nnz, info, buffer);
+
+        // accumulate non-zeros
+        c_col_idxs_array.resize_and_reset(c_nnz);
+        c_vals_array.resize_and_reset(c_nnz);
+        auto c_col_idxs = c_col_idxs_array.get_data();
+        auto c_vals = c_vals_array.get_data();
+        cusparse::spgemm(handle, m, n, k, &alpha, a_descr, a_nnz, a_vals,
+                         a_row_ptrs, a_col_idxs, b_descr, b_nnz, b_vals,
+                         b_row_ptrs, b_col_idxs, null_value, d_descr, zero_nnz,
+                         null_value, null_index, null_index, c_descr, c_vals,
+                         c_row_ptrs, c_col_idxs, info, buffer);
+
+        cusparse::destroy(info);
+        cusparse::destroy(d_descr);
+        cusparse::destroy(c_descr);
+        cusparse::destroy(b_descr);
+        cusparse::destroy(a_descr);
+
+#else   // DPCPP_VERSION >= 11000
+        const auto beta = zero<ValueType>();
+        auto spgemm_descr = cusparse::create_spgemm_descr();
+        auto a_descr = cusparse::create_csr(m, k, a_nnz,
+                                            const_cast<IndexType *>(a_row_ptrs),
+                                            const_cast<IndexType *>(a_col_idxs),
+                                            const_cast<ValueType *>(a_vals));
+        auto b_descr = cusparse::create_csr(k, n, b_nnz,
+                                            const_cast<IndexType *>(b_row_ptrs),
+                                            const_cast<IndexType *>(b_col_idxs),
+                                            const_cast<ValueType *>(b_vals));
+        auto c_descr = cusparse::create_csr(m, n, zero_nnz, null_index,
+                                            null_index, null_value);
+
+        // estimate work
+        size_type buffer1_size{};
+        cusparse::spgemm_work_estimation(handle, &alpha, a_descr, b_descr,
+                                         &beta, c_descr, spgemm_descr,
+                                         buffer1_size, nullptr);
+        Array<char> buffer1{exec, buffer1_size};
+        cusparse::spgemm_work_estimation(handle, &alpha, a_descr, b_descr,
+                                         &beta, c_descr, spgemm_descr,
+                                         buffer1_size, buffer1.get_data());
+
+        // compute spgemm
+        size_type buffer2_size{};
+        cusparse::spgemm_compute(handle, &alpha, a_descr, b_descr, &beta,
+                                 c_descr, spgemm_descr, buffer1.get_data(),
+                                 buffer2_size, nullptr);
+        Array<char> buffer2{exec, buffer2_size};
+        cusparse::spgemm_compute(handle, &alpha, a_descr, b_descr, &beta,
+                                 c_descr, spgemm_descr, buffer1.get_data(),
+                                 buffer2_size, buffer2.get_data());
+
+        // copy data to result
+        auto c_nnz = cusparse::sparse_matrix_nnz(c_descr);
+        c_col_idxs_array.resize_and_reset(c_nnz);
+        c_vals_array.resize_and_reset(c_nnz);
+        cusparse::csr_set_pointers(c_descr, c_row_ptrs,
+                                   c_col_idxs_array.get_data(),
+                                   c_vals_array.get_data());
+
+        cusparse::spgemm_copy(handle, &alpha, a_descr, b_descr, &beta, c_descr,
+                              spgemm_descr);
+
+        cusparse::destroy(c_descr);
+        cusparse::destroy(b_descr);
+        cusparse::destroy(a_descr);
+        cusparse::destroy(spgemm_descr);
+#endif  // DPCPP_VERSION >= 11000
+    } else {
+        GKO_NOT_IMPLEMENTED;
     }
 }
 
-
-/**
- * @internal
- *
- * Generic SpGEMM implementation for a single output row of A * B using binary
- * heap-based multiway merging.
- *
- * @param row  The row for which to compute the SpGEMM
- * @param a  The input matrix A
- * @param b  The input matrix B (its column indices must be sorted within each
- *           row!)
- * @param heap  The heap to use for this implementation. It must have as many
- *              entries as the input row has non-zeros.
- * @param init_cb  function to initialize the state for a single row. Its return
- *                 value will be updated by subsequent calls of other callbacks,
- *                 and then returned by this function. Its signature must be
- *                 compatible with `return_type state = init_cb(row)`.
- * @param step_cb  function that will be called for each accumulation from an
- *                 entry of B into the output state. Its signature must be
- *                 compatible with `step_cb(value, column, state)`.
- * @param col_cb  function that will be called once for each output column after
- *                all accumulations into it are completed. Its signature must be
- *                compatible with `col_cb(column, state)`.
- * @return the value initialized by init_cb and updated by step_cb and col_cb
- * @note If the columns of B are not sorted, the output may have duplicate
- *       column entries.
- *
- * @tparam HeapElement  the heap element type. See col_heap_element and
- *                      val_heap_element
- * @tparam InitCallback  functor type for init_cb
- * @tparam StepCallback  functor type for step_cb
- * @tparam ColCallback  functor type for col_cb
- */
-template <typename HeapElement, typename InitCallback, typename StepCallback,
-          typename ColCallback>
-auto spgemm_multiway_merge(size_type row,
-                           const typename HeapElement::index_type *a_row_ptrs,
-                           const typename HeapElement::index_type *a_cols,
-                           const typename HeapElement::value_type *a_vals,
-                           const typename HeapElement::index_type *b_row_ptrs,
-                           const typename HeapElement::index_type *b_cols,
-                           const typename HeapElement::value_type *b_vals,
-                           HeapElement *heap, InitCallback init_cb,
-                           StepCallback step_cb, ColCallback col_cb)
-    -> decltype(init_cb(0))
-{
-    auto a_begin = a_row_ptrs[row];
-    auto a_end = a_row_ptrs[row + 1];
-
-    using index_type = typename HeapElement::index_type;
-    constexpr auto sentinel = std::numeric_limits<index_type>::max();
-
-    auto state = init_cb(row);
-
-    // initialize the heap
-    for (auto a_nz = a_begin; a_nz < a_end; ++a_nz) {
-        auto b_row = a_cols[a_nz];
-        auto b_begin = b_row_ptrs[b_row];
-        auto b_end = b_row_ptrs[b_row + 1];
-        heap[a_nz] = {b_begin, b_end,
-                      checked_load(b_cols, b_begin, b_end, sentinel),
-                      a_vals[a_nz]};
-    }
-
-    if (a_begin != a_end) {
-        // make heap:
-        auto a_size = a_end - a_begin;
-        for (auto i = (a_size - 2) / 2; i >= 0; --i) {
-            sift_down(heap + a_begin, i, a_size);
-        }
-        auto &top = heap[a_begin];
-        auto &bot = heap[a_end - 1];
-        auto col = top.col;
-
-        while (top.col != sentinel) {
-            step_cb(b_vals[top.idx] * top.val(), top.col, state);
-            // move to the next element
-            top.idx++;
-            top.col = checked_load(b_cols, top.idx, top.end, sentinel);
-            // restore heap property
-            // pop_heap swaps top and bot, we need to prevent that
-            // so that we do a simple sift_down instead
-            sift_down(heap + a_begin, index_type{}, a_size);
-            if (top.col != col) {
-                col_cb(col, state);
-            }
-            col = top.col;
-        }
-    }
-
-    return state;
-}
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEMM_KERNEL);
 
 
-}  // namespace
+namespace {
 
 
-template <typename ValueType, typename IndexType>
-void spgemm(std::shared_ptr<const DpcppExecutor> exec,
-            const matrix::Csr<ValueType, IndexType> *a,
-            const matrix::Csr<ValueType, IndexType> *b,
-            matrix::Csr<ValueType, IndexType> *c)
+template <int subwarp_size, typename ValueType, typename IndexType>
+void spgeam(syn::value_list<int, subwarp_size>,
+            std::shared_ptr<const DefaultExecutor> exec, const ValueType *alpha,
+            const IndexType *a_row_ptrs, const IndexType *a_col_idxs,
+            const ValueType *a_vals, const ValueType *beta,
+            const IndexType *b_row_ptrs, const IndexType *b_col_idxs,
+            const ValueType *b_vals, matrix::Csr<ValueType, IndexType> *c)
 {
-    auto num_rows = a->get_size()[0];
-    const auto a_row_ptrs = a->get_const_row_ptrs();
-    const auto a_cols = a->get_const_col_idxs();
-    const auto a_vals = a->get_const_values();
-    const auto b_row_ptrs = b->get_const_row_ptrs();
-    const auto b_cols = b->get_const_col_idxs();
-    const auto b_vals = b->get_const_values();
+    auto m = static_cast<IndexType>(c->get_size()[0]);
     auto c_row_ptrs = c->get_row_ptrs();
-    auto queue = exec->get_queue();
-
-    Array<val_heap_element<ValueType, IndexType>> heap_array(
-        exec, a->get_num_stored_elements());
-
-    auto heap = heap_array.get_data();
-    auto col_heap =
-        reinterpret_cast<col_heap_element<ValueType, IndexType> *>(heap);
-
-    // first sweep: count nnz for each row
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
-            const auto a_row = static_cast<size_type>(idx[0]);
-            c_row_ptrs[a_row] = spgemm_multiway_merge(
-                a_row, a_row_ptrs, a_cols, a_vals, b_row_ptrs, b_cols, b_vals,
-                col_heap, [](size_type) { return IndexType{}; },
-                [](ValueType, IndexType, IndexType &) {},
-                [](IndexType, IndexType &nnz) { nnz++; });
-        });
-    });
+    // count nnz for alpha * A + beta * B
+    auto subwarps_per_block = default_block_size / subwarp_size;
+    auto num_blocks = ceildiv(m, subwarps_per_block);
+    kernel::spgeam_nnz<subwarp_size>(num_blocks, default_block_size, 0,
+                                     exec->get_queue(), a_row_ptrs, a_col_idxs,
+                                     b_row_ptrs, b_col_idxs, m, c_row_ptrs);
 
     // build row pointers
-    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
+    components::prefix_sum(exec, c_row_ptrs, m + 1);
 
-    // second sweep: accumulate non-zeros
-    const auto new_nnz = exec->copy_val_to_host(c_row_ptrs + num_rows);
+    // accumulate non-zeros for alpha * A + beta * B
     matrix::CsrBuilder<ValueType, IndexType> c_builder{c};
-    auto &c_col_idxs_array = c_builder.get_col_idx_array();
-    auto &c_vals_array = c_builder.get_value_array();
-    c_col_idxs_array.resize_and_reset(new_nnz);
-    c_vals_array.resize_and_reset(new_nnz);
-    auto c_col_idxs = c_col_idxs_array.get_data();
-    auto c_vals = c_vals_array.get_data();
-
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
-            const auto a_row = static_cast<size_type>(idx[0]);
-            spgemm_multiway_merge(
-                a_row, a_row_ptrs, a_cols, a_vals, b_row_ptrs, b_cols, b_vals,
-                heap,
-                [&](size_type row) {
-                    return std::make_pair(zero<ValueType>(), c_row_ptrs[row]);
-                },
-                [](ValueType val, IndexType,
-                   std::pair<ValueType, IndexType> &state) {
-                    state.first += val;
-                },
-                [&](IndexType col, std::pair<ValueType, IndexType> &state) {
-                    c_col_idxs[state.second] = col;
-                    c_vals[state.second] = state.first;
-                    state.first = zero<ValueType>();
-                    state.second++;
-                });
-        });
-    });
+    auto c_nnz = exec->copy_val_to_host(c_row_ptrs + m);
+    c_builder.get_col_idx_array().resize_and_reset(c_nnz);
+    c_builder.get_value_array().resize_and_reset(c_nnz);
+    auto c_col_idxs = c->get_col_idxs();
+    auto c_vals = c->get_values();
+    kernel::spgeam<subwarp_size>(
+        num_blocks, default_block_size, 0, exec->get_queue(), alpha, a_row_ptrs,
+        a_col_idxs, a_vals, beta, b_row_ptrs, b_col_idxs, b_vals, m, c_row_ptrs,
+        c_col_idxs, c_vals);
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEMM_KERNEL);
+GKO_ENABLE_IMPLEMENTATION_SELECTION(select_spgeam, spgeam);
+
+
+}  // namespace
 
 
 template <typename ValueType, typename IndexType>
@@ -362,125 +2389,142 @@ void advanced_spgemm(std::shared_ptr<const DpcppExecutor> exec,
                      const matrix::Csr<ValueType, IndexType> *d,
                      matrix::Csr<ValueType, IndexType> *c)
 {
-    auto num_rows = a->get_size()[0];
-    const auto a_row_ptrs = a->get_const_row_ptrs();
-    const auto a_cols = a->get_const_col_idxs();
-    const auto a_vals = a->get_const_values();
-    const auto b_row_ptrs = b->get_const_row_ptrs();
-    const auto b_cols = b->get_const_col_idxs();
-    const auto b_vals = b->get_const_values();
-    const auto d_row_ptrs = d->get_const_row_ptrs();
-    const auto d_cols = d->get_const_col_idxs();
-    const auto d_vals = d->get_const_values();
-    auto c_row_ptrs = c->get_row_ptrs();
-    const auto alpha_vals = alpha->get_const_values();
-    const auto beta_vals = beta->get_const_values();
-    constexpr auto sentinel = std::numeric_limits<IndexType>::max();
-    auto queue = exec->get_queue();
-
-    // first sweep: count nnz for each row
-
-    Array<val_heap_element<ValueType, IndexType>> heap_array(
-        exec, a->get_num_stored_elements());
-
-    auto heap = heap_array.get_data();
-    auto col_heap =
-        reinterpret_cast<col_heap_element<ValueType, IndexType> *>(heap);
-
-    // first sweep: count nnz for each row
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
-            const auto a_row = static_cast<size_type>(idx[0]);
-            auto d_nz = d_row_ptrs[a_row];
-            const auto d_end = d_row_ptrs[a_row + 1];
-            auto d_col = checked_load(d_cols, d_nz, d_end, sentinel);
-            c_row_ptrs[a_row] = spgemm_multiway_merge(
-                a_row, a_row_ptrs, a_cols, a_vals, b_row_ptrs, b_cols, b_vals,
-                col_heap, [](size_type row) { return IndexType{}; },
-                [](ValueType, IndexType, IndexType &) {},
-                [&](IndexType col, IndexType &nnz) {
-                    // skip smaller elements from d
-                    while (d_col <= col) {
-                        d_nz++;
-                        nnz += d_col != col;
-                        d_col = checked_load(d_cols, d_nz, d_end, sentinel);
-                    }
-                    nnz++;
-                });
-            // handle the remaining columns from d
-            c_row_ptrs[a_row] += d_end - d_nz;
-        });
-    });
-
-    // build row pointers
-    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
-
-    // second sweep: accumulate non-zeros
-    const auto new_nnz = exec->copy_val_to_host(c_row_ptrs + num_rows);
-    matrix::CsrBuilder<ValueType, IndexType> c_builder{c};
-    auto &c_col_idxs_array = c_builder.get_col_idx_array();
-    auto &c_vals_array = c_builder.get_value_array();
-    c_col_idxs_array.resize_and_reset(new_nnz);
-    c_vals_array.resize_and_reset(new_nnz);
-
-    auto c_col_idxs = c_col_idxs_array.get_data();
-    auto c_vals = c_vals_array.get_data();
-
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
-            const auto a_row = static_cast<size_type>(idx[0]);
-            auto d_nz = d_row_ptrs[a_row];
-            const auto d_end = d_row_ptrs[a_row + 1];
-            auto d_col = checked_load(d_cols, d_nz, d_end, sentinel);
-            auto d_val = checked_load(d_vals, d_nz, d_end, zero<ValueType>());
-            const auto valpha = alpha_vals[0];
-            const auto vbeta = beta_vals[0];
-            auto c_nz =
-                spgemm_multiway_merge(
-                    a_row, a_row_ptrs, a_cols, a_vals, b_row_ptrs, b_cols,
-                    b_vals, heap,
-                    [&](size_type row) {
-                        return std::make_pair(zero<ValueType>(),
-                                              c_row_ptrs[row]);
-                    },
-                    [](ValueType val, IndexType,
-                       std::pair<ValueType, IndexType> &state) {
-                        state.first += val;
-                    },
-                    [&](IndexType col, std::pair<ValueType, IndexType> &state) {
-                        // handle smaller elements from d
-                        ValueType part_d_val{};
-                        while (d_col <= col) {
-                            if (d_col == col) {
-                                part_d_val = d_val;
-                            } else {
-                                c_col_idxs[state.second] = d_col;
-                                c_vals[state.second] = vbeta * d_val;
-                                state.second++;
-                            }
-                            d_nz++;
-                            d_col = checked_load(d_cols, d_nz, d_end, sentinel);
-                            d_val = checked_load(d_vals, d_nz, d_end,
-                                                 zero<ValueType>());
-                        }
-                        c_col_idxs[state.second] = col;
-                        c_vals[state.second] =
-                            vbeta * part_d_val + valpha * state.first;
-                        state.first = zero<ValueType>();
-                        state.second++;
-                    })
-                    .second;
-            // handle remaining elements from d
-            while (d_col < sentinel) {
-                c_col_idxs[c_nz] = d_col;
-                c_vals[c_nz] = vbeta * d_val;
-                c_nz++;
-                d_nz++;
-                d_col = checked_load(d_cols, d_nz, d_end, sentinel);
-                d_val = checked_load(d_vals, d_nz, d_end, zero<ValueType>());
-            }
-        });
-    });
+    if (cusparse::is_supported<ValueType, IndexType>::value) {
+        auto handle = exec->get_cusparse_handle();
+        cusparse::pointer_mode_guard pm_guard(handle);
+
+        auto valpha = exec->copy_val_to_host(alpha->get_const_values());
+        auto a_nnz = IndexType(a->get_num_stored_elements());
+        auto a_vals = a->get_const_values();
+        auto a_row_ptrs = a->get_const_row_ptrs();
+        auto a_col_idxs = a->get_const_col_idxs();
+        auto b_nnz = IndexType(b->get_num_stored_elements());
+        auto b_vals = b->get_const_values();
+        auto b_row_ptrs = b->get_const_row_ptrs();
+        auto b_col_idxs = b->get_const_col_idxs();
+        auto vbeta = exec->copy_val_to_host(beta->get_const_values());
+        auto d_nnz = IndexType(d->get_num_stored_elements());
+        auto d_vals = d->get_const_values();
+        auto d_row_ptrs = d->get_const_row_ptrs();
+        auto d_col_idxs = d->get_const_col_idxs();
+        auto m = IndexType(a->get_size()[0]);
+        auto n = IndexType(b->get_size()[1]);
+        auto k = IndexType(a->get_size()[1]);
+        auto c_row_ptrs = c->get_row_ptrs();
+
+#if defined(DPCPP_VERSION) && (DPCPP_VERSION < 11000)
+        matrix::CsrBuilder<ValueType, IndexType> c_builder{c};
+        auto &c_col_idxs_array = c_builder.get_col_idx_array();
+        auto &c_vals_array = c_builder.get_value_array();
+        auto a_descr = cusparse::create_mat_descr();
+        auto b_descr = cusparse::create_mat_descr();
+        auto c_descr = cusparse::create_mat_descr();
+        auto d_descr = cusparse::create_mat_descr();
+        auto info = cusparse::create_spgemm_info();
+        // allocate buffer
+        size_type buffer_size{};
+        cusparse::spgemm_buffer_size(
+            handle, m, n, k, &valpha, a_descr, a_nnz, a_row_ptrs, a_col_idxs,
+            b_descr, b_nnz, b_row_ptrs, b_col_idxs, &vbeta, d_descr, d_nnz,
+            d_row_ptrs, d_col_idxs, info, buffer_size);
+        Array<char> buffer_array(exec, buffer_size);
+        auto buffer = buffer_array.get_data();
+
+        // count nnz
+        IndexType c_nnz{};
+        cusparse::spgemm_nnz(handle, m, n, k, a_descr, a_nnz, a_row_ptrs,
+                             a_col_idxs, b_descr, b_nnz, b_row_ptrs, b_col_idxs,
+                             d_descr, d_nnz, d_row_ptrs, d_col_idxs, c_descr,
+                             c_row_ptrs, &c_nnz, info, buffer);
+
+        // accumulate non-zeros
+        c_col_idxs_array.resize_and_reset(c_nnz);
+        c_vals_array.resize_and_reset(c_nnz);
+        auto c_col_idxs = c_col_idxs_array.get_data();
+        auto c_vals = c_vals_array.get_data();
+        cusparse::spgemm(handle, m, n, k, &valpha, a_descr, a_nnz, a_vals,
+                         a_row_ptrs, a_col_idxs, b_descr, b_nnz, b_vals,
+                         b_row_ptrs, b_col_idxs, &vbeta, d_descr, d_nnz, d_vals,
+                         d_row_ptrs, d_col_idxs, c_descr, c_vals, c_row_ptrs,
+                         c_col_idxs, info, buffer);
+
+        cusparse::destroy(info);
+        cusparse::destroy(d_descr);
+        cusparse::destroy(c_descr);
+        cusparse::destroy(b_descr);
+        cusparse::destroy(a_descr);
+#else   // DPCPP_VERSION >= 11000
+        auto null_value = static_cast<ValueType *>(nullptr);
+        auto null_index = static_cast<IndexType *>(nullptr);
+        auto one_val = one<ValueType>();
+        auto zero_val = zero<ValueType>();
+        auto zero_nnz = IndexType{};
+        auto spgemm_descr = cusparse::create_spgemm_descr();
+        auto a_descr = cusparse::create_csr(m, k, a_nnz,
+                                            const_cast<IndexType *>(a_row_ptrs),
+                                            const_cast<IndexType *>(a_col_idxs),
+                                            const_cast<ValueType *>(a_vals));
+        auto b_descr = cusparse::create_csr(k, n, b_nnz,
+                                            const_cast<IndexType *>(b_row_ptrs),
+                                            const_cast<IndexType *>(b_col_idxs),
+                                            const_cast<ValueType *>(b_vals));
+        auto c_descr = cusparse::create_csr(m, n, zero_nnz, null_index,
+                                            null_index, null_value);
+
+        // estimate work
+        size_type buffer1_size{};
+        cusparse::spgemm_work_estimation(handle, &one_val, a_descr, b_descr,
+                                         &zero_val, c_descr, spgemm_descr,
+                                         buffer1_size, nullptr);
+        Array<char> buffer1{exec, buffer1_size};
+        cusparse::spgemm_work_estimation(handle, &one_val, a_descr, b_descr,
+                                         &zero_val, c_descr, spgemm_descr,
+                                         buffer1_size, buffer1.get_data());
+
+        // compute spgemm
+        size_type buffer2_size{};
+        cusparse::spgemm_compute(handle, &one_val, a_descr, b_descr, &zero_val,
+                                 c_descr, spgemm_descr, buffer1.get_data(),
+                                 buffer2_size, nullptr);
+        Array<char> buffer2{exec, buffer2_size};
+        cusparse::spgemm_compute(handle, &one_val, a_descr, b_descr, &zero_val,
+                                 c_descr, spgemm_descr, buffer1.get_data(),
+                                 buffer2_size, buffer2.get_data());
+
+        // write result to temporary storage
+        auto c_tmp_nnz = cusparse::sparse_matrix_nnz(c_descr);
+        Array<IndexType> c_tmp_row_ptrs_array(exec, m + 1);
+        Array<IndexType> c_tmp_col_idxs_array(exec, c_tmp_nnz);
+        Array<ValueType> c_tmp_vals_array(exec, c_tmp_nnz);
+        cusparse::csr_set_pointers(c_descr, c_tmp_row_ptrs_array.get_data(),
+                                   c_tmp_col_idxs_array.get_data(),
+                                   c_tmp_vals_array.get_data());
+
+        cusparse::spgemm_copy(handle, &one_val, a_descr, b_descr, &zero_val,
+                              c_descr, spgemm_descr);
+
+        cusparse::destroy(c_descr);
+        cusparse::destroy(b_descr);
+        cusparse::destroy(a_descr);
+        cusparse::destroy(spgemm_descr);
+
+        auto spgeam_total_nnz = c_tmp_nnz + d->get_num_stored_elements();
+        auto nnz_per_row = spgeam_total_nnz / m;
+        select_spgeam(
+            spgeam_kernels(),
+            [&](int compiled_subwarp_size) {
+                return compiled_subwarp_size >= nnz_per_row ||
+                       compiled_subwarp_size == config::warp_size;
+            },
+            syn::value_list<int>(), syn::type_list<>(), exec,
+            alpha->get_const_values(), c_tmp_row_ptrs_array.get_const_data(),
+            c_tmp_col_idxs_array.get_const_data(),
+            c_tmp_vals_array.get_const_data(), beta->get_const_values(),
+            d_row_ptrs, d_col_idxs, d_vals, c);
+#endif  // DPCPP_VERSION >= 11000
+    } else {
+        GKO_NOT_IMPLEMENTED;
+    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -488,85 +2532,27 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void spgeam(std::shared_ptr<const DpcppExecutor> exec,
+void spgeam(std::shared_ptr<const DefaultExecutor> exec,
             const matrix::Dense<ValueType> *alpha,
             const matrix::Csr<ValueType, IndexType> *a,
             const matrix::Dense<ValueType> *beta,
             const matrix::Csr<ValueType, IndexType> *b,
             matrix::Csr<ValueType, IndexType> *c)
 {
-    constexpr auto sentinel = std::numeric_limits<IndexType>::max();
-    const auto num_rows = a->get_size()[0];
-    const auto a_row_ptrs = a->get_const_row_ptrs();
-    const auto a_cols = a->get_const_col_idxs();
-    const auto b_row_ptrs = b->get_const_row_ptrs();
-    const auto b_cols = b->get_const_col_idxs();
-    auto c_row_ptrs = c->get_row_ptrs();
-    auto queue = exec->get_queue();
-
-    // count number of non-zeros per row
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
-            const auto row = static_cast<size_type>(idx[0]);
-            auto a_idx = a_row_ptrs[row];
-            const auto a_end = a_row_ptrs[row + 1];
-            auto b_idx = b_row_ptrs[row];
-            const auto b_end = b_row_ptrs[row + 1];
-            IndexType row_nnz{};
-            while (a_idx < a_end || b_idx < b_end) {
-                const auto a_col = checked_load(a_cols, a_idx, a_end, sentinel);
-                const auto b_col = checked_load(b_cols, b_idx, b_end, sentinel);
-                row_nnz++;
-                a_idx += (a_col <= b_col) ? 1 : 0;
-                b_idx += (b_col <= a_col) ? 1 : 0;
-            }
-            c_row_ptrs[row] = row_nnz;
-        });
-    });
-
-    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
-
-    // second sweep: accumulate non-zeros
-    const auto new_nnz = exec->copy_val_to_host(c_row_ptrs + num_rows);
-    matrix::CsrBuilder<ValueType, IndexType> c_builder{c};
-    auto &c_col_idxs_array = c_builder.get_col_idx_array();
-    auto &c_vals_array = c_builder.get_value_array();
-    c_col_idxs_array.resize_and_reset(new_nnz);
-    c_vals_array.resize_and_reset(new_nnz);
-    auto c_cols = c_col_idxs_array.get_data();
-    auto c_vals = c_vals_array.get_data();
-
-    const auto a_vals = a->get_const_values();
-    const auto b_vals = b->get_const_values();
-    const auto alpha_vals = alpha->get_const_values();
-    const auto beta_vals = beta->get_const_values();
-
-    // count number of non-zeros per row
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
-            const auto row = static_cast<size_type>(idx[0]);
-            auto a_idx = a_row_ptrs[row];
-            const auto a_end = a_row_ptrs[row + 1];
-            auto b_idx = b_row_ptrs[row];
-            const auto b_end = b_row_ptrs[row + 1];
-            const auto alpha = alpha_vals[0];
-            const auto beta = beta_vals[0];
-            auto c_nz = c_row_ptrs[row];
-            while (a_idx < a_end || b_idx < b_end) {
-                const auto a_col = checked_load(a_cols, a_idx, a_end, sentinel);
-                const auto b_col = checked_load(b_cols, b_idx, b_end, sentinel);
-                const bool use_a = a_col <= b_col;
-                const bool use_b = b_col <= a_col;
-                const auto a_val = use_a ? a_vals[a_idx] : zero<ValueType>();
-                const auto b_val = use_b ? b_vals[b_idx] : zero<ValueType>();
-                c_cols[c_nz] = std::min(a_col, b_col);
-                c_vals[c_nz] = alpha * a_val + beta * b_val;
-                c_nz++;
-                a_idx += use_a ? 1 : 0;
-                b_idx += use_b ? 1 : 0;
-            }
-        });
-    });
+    auto total_nnz =
+        a->get_num_stored_elements() + b->get_num_stored_elements();
+    auto nnz_per_row = total_nnz / a->get_size()[0];
+    select_spgeam(
+        spgeam_kernels(),
+        [&](int compiled_subwarp_size) {
+            return compiled_subwarp_size >= nnz_per_row ||
+                   compiled_subwarp_size == config::warp_size;
+        },
+        syn::value_list<int>(), syn::type_list<>(), exec,
+        alpha->get_const_values(), a->get_const_row_ptrs(),
+        a->get_const_col_idxs(), a->get_const_values(),
+        beta->get_const_values(), b->get_const_row_ptrs(),
+        b->get_const_col_idxs(), b->get_const_values(), c);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEAM_KERNEL);
@@ -575,14 +2561,27 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEAM_KERNEL);
 template <typename IndexType>
 void convert_row_ptrs_to_idxs(std::shared_ptr<const DpcppExecutor> exec,
                               const IndexType *ptrs, size_type num_rows,
-                              IndexType *idxs) GKO_NOT_IMPLEMENTED;
+                              IndexType *idxs)
+{
+    const auto grid_dim = ceildiv(num_rows, default_block_size);
+
+    kernel::convert_row_ptrs_to_idxs(grid_dim, default_block_size, 0,
+                                     exec->get_queue(), num_rows, ptrs, idxs);
+}
 
 
 template <typename ValueType, typename IndexType>
 void convert_to_coo(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Csr<ValueType, IndexType> *source,
                     matrix::Coo<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
+{
+    auto num_rows = result->get_size()[0];
+
+    auto row_idxs = result->get_row_idxs();
+    const auto source_row_ptrs = source->get_const_row_ptrs();
+
+    convert_row_ptrs_to_idxs(exec, source_row_ptrs, num_rows, row_idxs);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONVERT_TO_COO_KERNEL);
@@ -591,7 +2590,28 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Csr<ValueType, IndexType> *source,
-                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
+                      matrix::Dense<ValueType> *result)
+{
+    const auto num_rows = result->get_size()[0];
+    const auto num_cols = result->get_size()[1];
+    const auto stride = result->get_stride();
+    const auto row_ptrs = source->get_const_row_ptrs();
+    const auto col_idxs = source->get_const_col_idxs();
+    const auto vals = source->get_const_values();
+
+    const dim3 block_size(config::warp_size,
+                          config::max_block_size / config::warp_size, 1);
+    const dim3 init_grid_dim(ceildiv(num_cols, block_size.x),
+                             ceildiv(num_rows, block_size.y), 1);
+    kernel::initialize_zero_dense(init_grid_dim, block_size, 0,
+                                  exec->get_queue(), num_rows, num_cols, stride,
+                                  result->get_values());
+
+    auto grid_dim = ceildiv(num_rows, default_block_size);
+    kernel::fill_in_dense(grid_dim, default_block_size, 0, exec->get_queue(),
+                          num_rows, row_ptrs, col_idxs, vals, stride,
+                          result->get_values());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONVERT_TO_DENSE_KERNEL);
@@ -601,7 +2621,55 @@ template <typename ValueType, typename IndexType>
 void convert_to_sellp(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Csr<ValueType, IndexType> *source,
                       matrix::Sellp<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
+{
+    const auto num_rows = result->get_size()[0];
+    const auto num_cols = result->get_size()[1];
+
+    auto result_values = result->get_values();
+    auto result_col_idxs = result->get_col_idxs();
+    auto slice_lengths = result->get_slice_lengths();
+    auto slice_sets = result->get_slice_sets();
+
+    const auto slice_size = (result->get_slice_size() == 0)
+                                ? matrix::default_slice_size
+                                : result->get_slice_size();
+    const auto stride_factor = (result->get_stride_factor() == 0)
+                                   ? matrix::default_stride_factor
+                                   : result->get_stride_factor();
+    const int slice_num = ceildiv(num_rows, slice_size);
+
+    const auto source_values = source->get_const_values();
+    const auto source_row_ptrs = source->get_const_row_ptrs();
+    const auto source_col_idxs = source->get_const_col_idxs();
+
+    auto nnz_per_row = Array<size_type>(exec, num_rows);
+    auto grid_dim = ceildiv(num_rows, default_block_size);
+
+    if (grid_dim > 0) {
+        kernel::calculate_nnz_per_row(grid_dim, default_block_size, 0,
+                                      exec->get_queue(), num_rows,
+                                      source_row_ptrs, nnz_per_row.get_data());
+    }
+
+    grid_dim = slice_num;
+
+    if (grid_dim > 0) {
+        kernel::calculate_slice_lengths(
+            grid_dim, config::warp_size, 0, exec->get_queue(), num_rows,
+            slice_size, stride_factor, nnz_per_row.get_const_data(),
+            slice_lengths, slice_sets);
+    }
+
+    components::prefix_sum(exec, slice_sets, slice_num + 1);
+
+    grid_dim = ceildiv(num_rows, default_block_size);
+    if (grid_dim > 0) {
+        kernel::fill_in_sellp(
+            grid_dim, default_block_size, 0, exec->get_queue(), num_rows,
+            slice_size, source_values, source_row_ptrs, source_col_idxs,
+            slice_lengths, slice_sets, result_col_idxs, result_values);
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONVERT_TO_SELLP_KERNEL);
@@ -611,84 +2679,131 @@ template <typename ValueType, typename IndexType>
 void convert_to_ell(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Csr<ValueType, IndexType> *source,
                     matrix::Ell<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
+{
+    const auto source_values = source->get_const_values();
+    const auto source_row_ptrs = source->get_const_row_ptrs();
+    const auto source_col_idxs = source->get_const_col_idxs();
+
+    auto result_values = result->get_values();
+    auto result_col_idxs = result->get_col_idxs();
+    const auto stride = result->get_stride();
+    const auto max_nnz_per_row = result->get_num_stored_elements_per_row();
+    const auto num_rows = result->get_size()[0];
+    const auto num_cols = result->get_size()[1];
+
+    const auto init_grid_dim =
+        ceildiv(max_nnz_per_row * num_rows, default_block_size);
+
+    kernel::initialize_zero_ell(init_grid_dim, default_block_size, 0,
+                                exec->get_queue(), max_nnz_per_row, stride,
+                                result_values, result_col_idxs);
+
+    const auto grid_dim =
+        ceildiv(num_rows * config::warp_size, default_block_size);
+
+    kernel::fill_in_ell(grid_dim, default_block_size, 0, exec->get_queue(),
+                        num_rows, stride, source_values, source_row_ptrs,
+                        source_col_idxs, result_values, result_col_idxs);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONVERT_TO_ELL_KERNEL);
 
 
-template <typename ValueType, typename IndexType, typename UnaryOperator>
-inline void convert_csr_to_csc(size_type num_rows, const IndexType *row_ptrs,
-                               const IndexType *col_idxs,
-                               const ValueType *csr_vals, IndexType *row_idxs,
-                               IndexType *col_ptrs, ValueType *csc_vals,
-                               UnaryOperator op) GKO_NOT_IMPLEMENTED;
-
-
-template <typename ValueType, typename IndexType, typename UnaryOperator>
-void transpose_and_transform(std::shared_ptr<const DpcppExecutor> exec,
-                             matrix::Csr<ValueType, IndexType> *trans,
-                             const matrix::Csr<ValueType, IndexType> *orig,
-                             UnaryOperator op) GKO_NOT_IMPLEMENTED;
-
-
-template <bool conjugate, typename ValueType, typename IndexType>
-void generic_transpose(std::shared_ptr<const DpcppExecutor> exec,
-                       const matrix::Csr<ValueType, IndexType> *orig,
-                       matrix::Csr<ValueType, IndexType> *trans)
-{
-    const auto num_rows = orig->get_size()[0];
-    const auto num_cols = orig->get_size()[1];
-    auto queue = exec->get_queue();
-    const auto row_ptrs = orig->get_const_row_ptrs();
-    const auto cols = orig->get_const_col_idxs();
-    const auto vals = orig->get_const_values();
-
-    Array<IndexType> counts{exec, num_cols + 1};
-    auto tmp_counts = counts.get_data();
-    auto out_row_ptrs = trans->get_row_ptrs();
-    auto out_cols = trans->get_col_idxs();
-    auto out_vals = trans->get_values();
-    components::fill_array(exec, tmp_counts, num_cols, IndexType{});
-
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
-            const auto row = static_cast<size_type>(idx[0]);
-            const auto begin = row_ptrs[row];
-            const auto end = row_ptrs[row + 1];
-            for (auto i = begin; i < end; i++) {
-                atomic_fetch_add(tmp_counts + cols[i], IndexType{1});
-            }
-        });
-    });
+template <typename ValueType, typename IndexType>
+void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
+                          const matrix::Csr<ValueType, IndexType> *source,
+                          size_type *result, size_type stride_factor,
+                          size_type slice_size)
+{
+    const auto num_rows = source->get_size()[0];
 
-    components::prefix_sum(exec, tmp_counts, num_cols + 1);
-    exec->copy(num_cols + 1, tmp_counts, out_row_ptrs);
+    if (num_rows == 0) {
+        *result = 0;
+        return;
+    }
 
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
-            const auto row = static_cast<size_type>(idx[0]);
-            const auto begin = row_ptrs[row];
-            const auto end = row_ptrs[row + 1];
-            for (auto i = begin; i < end; i++) {
-                auto out_nz =
-                    atomic_fetch_add(tmp_counts + cols[i], IndexType{1});
-                out_cols[out_nz] = row;
-                out_vals[out_nz] = conjugate ? conj(vals[i]) : vals[i];
-            }
-        });
-    });
+    const auto slice_num = ceildiv(num_rows, slice_size);
+    const auto row_ptrs = source->get_const_row_ptrs();
+
+    auto nnz_per_row = Array<size_type>(exec, num_rows);
+    auto grid_dim = ceildiv(num_rows, default_block_size);
+
+    kernel::calculate_nnz_per_row(grid_dim, default_block_size, 0,
+                                  exec->get_queue(), num_rows, row_ptrs,
+                                  nnz_per_row.get_data());
+
+    grid_dim = ceildiv(slice_num * config::warp_size, default_block_size);
+    auto max_nnz_per_slice = Array<size_type>(exec, slice_num);
+
+    kernel::reduce_max_nnz_per_slice(
+        grid_dim, default_block_size, 0, exec->get_queue(), num_rows,
+        slice_size, stride_factor, nnz_per_row.get_const_data(),
+        max_nnz_per_slice.get_data());
+
+    grid_dim = ceildiv(slice_num, default_block_size);
+    auto block_results = Array<size_type>(exec, grid_dim);
+
+    kernel::reduce_total_cols(
+        grid_dim, default_block_size, 0, exec->get_queue(), slice_num,
+        max_nnz_per_slice.get_const_data(), block_results.get_data());
 
-    sort_by_column_index(exec, trans);
+    auto d_result = Array<size_type>(exec, 1);
+
+    kernel::reduce_total_cols(1, default_block_size, 0, exec->get_queue(),
+                              grid_dim, block_results.get_const_data(),
+                              d_result.get_data());
+
+    *result = exec->copy_val_to_host(d_result.get_const_data());
 }
 
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_CALCULATE_TOTAL_COLS_KERNEL);
+
 
 template <typename ValueType, typename IndexType>
 void transpose(std::shared_ptr<const DpcppExecutor> exec,
                const matrix::Csr<ValueType, IndexType> *orig,
                matrix::Csr<ValueType, IndexType> *trans)
 {
-    generic_transpose<false>(exec, orig, trans);
+    if (cusparse::is_supported<ValueType, IndexType>::value) {
+#if defined(DPCPP_VERSION) && (DPCPP_VERSION < 11000)
+        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
+        oneapi::mkl::index_base idxBase = oneapi::mkl::index_base::zero;
+
+        cusparse::transpose(
+            exec->get_cusparse_handle(), orig->get_size()[0],
+            orig->get_size()[1], orig->get_num_stored_elements(),
+            orig->get_const_values(), orig->get_const_row_ptrs(),
+            orig->get_const_col_idxs(), trans->get_values(),
+            trans->get_row_ptrs(), trans->get_col_idxs(), copyValues, idxBase);
+#else  // DPCPP_VERSION >= 11000
+        dpcppDataType_t cu_value =
+            gko::kernels::dpcpp::dpcpp_data_type<ValueType>();
+        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
+        cusparseIndexBase_t idxBase = CUSPARSE_INDEX_BASE_ZERO;
+        cusparseCsr2CscAlg_t alg = CUSPARSE_CSR2CSC_ALG1;
+        size_type buffer_size = 0;
+        cusparse::transpose_buffersize(
+            exec->get_cusparse_handle(), orig->get_size()[0],
+            orig->get_size()[1], orig->get_num_stored_elements(),
+            orig->get_const_values(), orig->get_const_row_ptrs(),
+            orig->get_const_col_idxs(), trans->get_values(),
+            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value, copyValues,
+            idxBase, alg, &buffer_size);
+        Array<char> buffer_array(exec, buffer_size);
+        auto buffer = buffer_array.get_data();
+        cusparse::transpose(
+            exec->get_cusparse_handle(), orig->get_size()[0],
+            orig->get_size()[1], orig->get_num_stored_elements(),
+            orig->get_const_values(), orig->get_const_row_ptrs(),
+            orig->get_const_col_idxs(), trans->get_values(),
+            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value, copyValues,
+            idxBase, alg, buffer);
+#endif
+    } else {
+        GKO_NOT_IMPLEMENTED;
+    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_TRANSPOSE_KERNEL);
@@ -699,94 +2814,252 @@ void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Csr<ValueType, IndexType> *orig,
                     matrix::Csr<ValueType, IndexType> *trans)
 {
-    generic_transpose<true>(exec, orig, trans);
+    if (cusparse::is_supported<ValueType, IndexType>::value) {
+        const dim3 block_size(default_block_size, 1, 1);
+        const dim3 grid_size(
+            ceildiv(trans->get_num_stored_elements(), block_size.x), 1, 1);
+
+#if defined(DPCPP_VERSION) && (DPCPP_VERSION < 11000)
+        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
+        oneapi::mkl::index_base idxBase = oneapi::mkl::index_base::zero;
+
+        cusparse::transpose(
+            exec->get_cusparse_handle(), orig->get_size()[0],
+            orig->get_size()[1], orig->get_num_stored_elements(),
+            orig->get_const_values(), orig->get_const_row_ptrs(),
+            orig->get_const_col_idxs(), trans->get_values(),
+            trans->get_row_ptrs(), trans->get_col_idxs(), copyValues, idxBase);
+#else  // DPCPP_VERSION >= 11000
+        dpcppDataType_t cu_value =
+            gko::kernels::dpcpp::dpcpp_data_type<ValueType>();
+        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
+        cusparseIndexBase_t idxBase = CUSPARSE_INDEX_BASE_ZERO;
+        cusparseCsr2CscAlg_t alg = CUSPARSE_CSR2CSC_ALG1;
+        size_type buffer_size = 0;
+        cusparse::transpose_buffersize(
+            exec->get_cusparse_handle(), orig->get_size()[0],
+            orig->get_size()[1], orig->get_num_stored_elements(),
+            orig->get_const_values(), orig->get_const_row_ptrs(),
+            orig->get_const_col_idxs(), trans->get_values(),
+            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value, copyValues,
+            idxBase, alg, &buffer_size);
+        Array<char> buffer_array(exec, buffer_size);
+        auto buffer = buffer_array.get_data();
+        cusparse::transpose(
+            exec->get_cusparse_handle(), orig->get_size()[0],
+            orig->get_size()[1], orig->get_num_stored_elements(),
+            orig->get_const_values(), orig->get_const_row_ptrs(),
+            orig->get_const_col_idxs(), trans->get_values(),
+            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value, copyValues,
+            idxBase, alg, buffer);
+#endif
+
+        conjugate_kernel(grid_size, block_size, 0, exec->get_queue(),
+                         trans->get_num_stored_elements(), trans->get_values());
+    } else {
+        GKO_NOT_IMPLEMENTED;
+    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void calculate_total_cols(std::shared_ptr<const DpcppExecutor> exec,
-                          const matrix::Csr<ValueType, IndexType> *source,
-                          size_type *result, size_type stride_factor,
-                          size_type slice_size) GKO_NOT_IMPLEMENTED;
+template <typename IndexType>
+void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
+                        size_type size, const IndexType *permutation_indices,
+                        IndexType *inv_permutation)
+{
+    auto num_blocks = ceildiv(size, default_block_size);
+    inv_permutation_kernel(num_blocks, default_block_size, 0, exec->get_queue(),
+                           size, permutation_indices, inv_permutation);
+}
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_CALCULATE_TOTAL_COLS_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void calculate_max_nnz_per_row(std::shared_ptr<const DpcppExecutor> exec,
-                               const matrix::Csr<ValueType, IndexType> *source,
-                               size_type *result) GKO_NOT_IMPLEMENTED;
+void inv_symm_permute(std::shared_ptr<const DpcppExecutor> exec,
+                      const IndexType *perm,
+                      const matrix::Csr<ValueType, IndexType> *orig,
+                      matrix::Csr<ValueType, IndexType> *permuted)
+{
+    auto num_rows = orig->get_size()[0];
+    auto count_num_blocks = ceildiv(num_rows, default_block_size);
+    inv_row_ptr_permute_kernel(
+        count_num_blocks, default_block_size, 0, exec->get_queue(), num_rows,
+        perm, orig->get_const_row_ptrs(), permuted->get_row_ptrs());
+    components::prefix_sum(exec, permuted->get_row_ptrs(), num_rows + 1);
+    auto copy_num_blocks =
+        ceildiv(num_rows, default_block_size / config::warp_size);
+    inv_symm_permute_kernel<config::warp_size>(
+        copy_num_blocks, default_block_size, 0, exec->get_queue(), num_rows,
+        perm, orig->get_const_row_ptrs(), orig->get_const_col_idxs(),
+        orig->get_const_values(), permuted->get_row_ptrs(),
+        permuted->get_col_idxs(), permuted->get_values());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
+    GKO_DECLARE_CSR_INV_SYMM_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void convert_to_hybrid(std::shared_ptr<const DpcppExecutor> exec,
-                       const matrix::Csr<ValueType, IndexType> *source,
-                       matrix::Hybrid<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
+void row_permute(std::shared_ptr<const DpcppExecutor> exec,
+                 const IndexType *perm,
+                 const matrix::Csr<ValueType, IndexType> *orig,
+                 matrix::Csr<ValueType, IndexType> *row_permuted)
+{
+    auto num_rows = orig->get_size()[0];
+    auto count_num_blocks = ceildiv(num_rows, default_block_size);
+    row_ptr_permute_kernel(
+        count_num_blocks, default_block_size, 0, exec->get_queue(), num_rows,
+        perm, orig->get_const_row_ptrs(), row_permuted->get_row_ptrs());
+    components::prefix_sum(exec, row_permuted->get_row_ptrs(), num_rows + 1);
+    auto copy_num_blocks =
+        ceildiv(num_rows, default_block_size / config::warp_size);
+    row_permute_kernel<config::warp_size>(
+        copy_num_blocks, default_block_size, 0, exec->get_queue(), num_rows,
+        perm, orig->get_const_row_ptrs(), orig->get_const_col_idxs(),
+        orig->get_const_values(), row_permuted->get_row_ptrs(),
+        row_permuted->get_col_idxs(), row_permuted->get_values());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_CONVERT_TO_HYBRID_KERNEL);
+    GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL);
 
 
-template <typename IndexType>
-void invert_permutation(std::shared_ptr<const DpcppExecutor> exec,
-                        size_type size, const IndexType *permutation_indices,
-                        IndexType *inv_permutation) GKO_NOT_IMPLEMENTED;
+template <typename ValueType, typename IndexType>
+void inverse_row_permute(std::shared_ptr<const DpcppExecutor> exec,
+                         const IndexType *perm,
+                         const matrix::Csr<ValueType, IndexType> *orig,
+                         matrix::Csr<ValueType, IndexType> *row_permuted)
+{
+    auto num_rows = orig->get_size()[0];
+    auto count_num_blocks = ceildiv(num_rows, default_block_size);
+    inv_row_ptr_permute_kernel(
+        count_num_blocks, default_block_size, 0, exec->get_queue(), num_rows,
+        perm, orig->get_const_row_ptrs(), row_permuted->get_row_ptrs());
+    components::prefix_sum(exec, row_permuted->get_row_ptrs(), num_rows + 1);
+    auto copy_num_blocks =
+        ceildiv(num_rows, default_block_size / config::warp_size);
+    inv_row_permute_kernel<config::warp_size>(
+        copy_num_blocks, default_block_size, 0, exec->get_queue(), num_rows,
+        perm, orig->get_const_row_ptrs(), orig->get_const_col_idxs(),
+        orig->get_const_values(), row_permuted->get_row_ptrs(),
+        row_permuted->get_col_idxs(), row_permuted->get_values());
+}
 
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void inv_symm_permute(
-    std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
-    const matrix::Csr<ValueType, IndexType> *orig,
-    matrix::Csr<ValueType, IndexType> *permuted) GKO_NOT_IMPLEMENTED;
+void inverse_column_permute(std::shared_ptr<const DpcppExecutor> exec,
+                            const IndexType *perm,
+                            const matrix::Csr<ValueType, IndexType> *orig,
+                            matrix::Csr<ValueType, IndexType> *column_permuted)
+{
+    auto num_rows = orig->get_size()[0];
+    auto nnz = orig->get_num_stored_elements();
+    auto num_blocks = ceildiv(std::max(num_rows, nnz), default_block_size);
+    col_permute_kernel(
+        num_blocks, default_block_size, 0, exec->get_queue(), num_rows, nnz,
+        perm, orig->get_const_row_ptrs(), orig->get_const_col_idxs(),
+        orig->get_const_values(), column_permuted->get_row_ptrs(),
+        column_permuted->get_col_idxs(), column_permuted->get_values());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_INV_SYMM_PERMUTE_KERNEL);
+    GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void row_permute(
-    std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
-    const matrix::Csr<ValueType, IndexType> *orig,
-    matrix::Csr<ValueType, IndexType> *row_permuted) GKO_NOT_IMPLEMENTED;
+void calculate_max_nnz_per_row(std::shared_ptr<const DpcppExecutor> exec,
+                               const matrix::Csr<ValueType, IndexType> *source,
+                               size_type *result)
+{
+    const auto num_rows = source->get_size()[0];
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_ROW_PERMUTE_KERNEL);
+    auto nnz_per_row = Array<size_type>(exec, num_rows);
+    auto block_results = Array<size_type>(exec, default_block_size);
+    auto d_result = Array<size_type>(exec, 1);
 
+    const auto grid_dim = ceildiv(num_rows, default_block_size);
+    kernel::calculate_nnz_per_row(
+        grid_dim, default_block_size, 0, exec->get_queue(), num_rows,
+        source->get_const_row_ptrs(), nnz_per_row.get_data());
 
-template <typename ValueType, typename IndexType>
-void inverse_row_permute(
-    std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
-    const matrix::Csr<ValueType, IndexType> *orig,
-    matrix::Csr<ValueType, IndexType> *row_permuted) GKO_NOT_IMPLEMENTED;
+    const auto n = ceildiv(num_rows, default_block_size);
+    const auto reduce_dim = n <= default_block_size ? n : default_block_size;
+    kernel::reduce_max_nnz(reduce_dim, default_block_size, 0, exec->get_queue(),
+                           num_rows, nnz_per_row.get_const_data(),
+                           block_results.get_data());
+
+    kernel::reduce_max_nnz(1, default_block_size, 0, exec->get_queue(),
+                           reduce_dim, block_results.get_const_data(),
+                           d_result.get_data());
+
+    *result = exec->copy_val_to_host(d_result.get_const_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL);
+    GKO_DECLARE_CSR_CALCULATE_MAX_NNZ_PER_ROW_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
-void inverse_column_permute(
-    std::shared_ptr<const DpcppExecutor> exec, const IndexType *perm,
-    const matrix::Csr<ValueType, IndexType> *orig,
-    matrix::Csr<ValueType, IndexType> *column_permuted) GKO_NOT_IMPLEMENTED;
+void convert_to_hybrid(std::shared_ptr<const DpcppExecutor> exec,
+                       const matrix::Csr<ValueType, IndexType> *source,
+                       matrix::Hybrid<ValueType, IndexType> *result)
+{
+    auto ell_val = result->get_ell_values();
+    auto ell_col = result->get_ell_col_idxs();
+    auto coo_val = result->get_coo_values();
+    auto coo_col = result->get_coo_col_idxs();
+    auto coo_row = result->get_coo_row_idxs();
+    const auto stride = result->get_ell_stride();
+    const auto max_nnz_per_row = result->get_ell_num_stored_elements_per_row();
+    const auto num_rows = result->get_size()[0];
+    const auto coo_num_stored_elements = result->get_coo_num_stored_elements();
+    auto grid_dim = ceildiv(max_nnz_per_row * num_rows, default_block_size);
+
+    kernel::initialize_zero_ell(grid_dim, default_block_size, 0,
+                                exec->get_queue(), max_nnz_per_row, stride,
+                                ell_val, ell_col);
+
+    grid_dim = ceildiv(num_rows, default_block_size);
+    auto coo_offset = Array<size_type>(exec, num_rows);
+    kernel::calculate_hybrid_coo_row_nnz(
+        grid_dim, default_block_size, 0, exec->get_queue(), num_rows,
+        max_nnz_per_row, source->get_const_row_ptrs(), coo_offset.get_data());
+
+    components::prefix_sum(exec, coo_offset.get_data(), num_rows);
+
+    grid_dim = ceildiv(num_rows * config::warp_size, default_block_size);
+    kernel::fill_in_hybrid(
+        grid_dim, default_block_size, 0, exec->get_queue(), num_rows, stride,
+        max_nnz_per_row, source->get_const_values(),
+        source->get_const_row_ptrs(), source->get_const_col_idxs(),
+        coo_offset.get_const_data(), ell_val, ell_col, coo_val, coo_col,
+        coo_row);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
+    GKO_DECLARE_CSR_CONVERT_TO_HYBRID_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
 void calculate_nonzeros_per_row(std::shared_ptr<const DpcppExecutor> exec,
                                 const matrix::Csr<ValueType, IndexType> *source,
-                                Array<size_type> *result) GKO_NOT_IMPLEMENTED;
+                                Array<size_type> *result)
+{
+    const auto num_rows = source->get_size()[0];
+    auto row_ptrs = source->get_const_row_ptrs();
+    auto grid_dim = ceildiv(num_rows, default_block_size);
+
+    kernel::calculate_nnz_per_row(grid_dim, default_block_size, 0,
+                                  exec->get_queue(), num_rows, row_ptrs,
+                                  result->get_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CALCULATE_NONZEROS_PER_ROW_KERNEL);
@@ -796,54 +3069,51 @@ template <typename ValueType, typename IndexType>
 void sort_by_column_index(std::shared_ptr<const DpcppExecutor> exec,
                           matrix::Csr<ValueType, IndexType> *to_sort)
 {
-    const auto num_rows = to_sort->get_size()[0];
-    const auto row_ptrs = to_sort->get_const_row_ptrs();
-    auto cols = to_sort->get_col_idxs();
-    auto vals = to_sort->get_values();
-    exec->get_queue()->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
-            const auto row = static_cast<size_type>(idx[0]);
-            const auto begin = row_ptrs[row];
-            auto size = row_ptrs[row + 1] - begin;
-            if (size <= 1) {
-                return;
-            }
-            auto swap = [&](IndexType i, IndexType j) {
-                std::swap(cols[i + begin], cols[j + begin]);
-                std::swap(vals[i + begin], vals[j + begin]);
-            };
-            auto lchild = [](IndexType i) { return 2 * i + 1; };
-            auto rchild = [](IndexType i) { return 2 * i + 2; };
-            auto parent = [](IndexType i) { return (i - 1) / 2; };
-            auto sift_down = [&](IndexType i) {
-                const auto col = cols[i + begin];
-                while (lchild(i) < size) {
-                    const auto lcol = cols[lchild(i) + begin];
-                    // -1 as sentinel, since we are building a max heap
-                    const auto rcol = checked_load(cols + begin, rchild(i),
-                                                   size, IndexType{-1});
-                    if (col >= std::max(lcol, rcol)) {
-                        return;
-                    }
-                    const auto maxchild = lcol > rcol ? lchild(i) : rchild(i);
-                    swap(i, maxchild);
-                    i = maxchild;
-                }
-            };
-            // heapify / sift_down for max-heap
-            for (auto i = (size - 2) / 2; i >= 0; i--) {
-                sift_down(i);
-            }
-            // heapsort: swap maximum to the end, shrink heap
-            swap(0, size - 1);
-            size--;
-            for (; size > 1; size--) {
-                // restore heap property and repeat
-                sift_down(0);
-                swap(0, size - 1);
-            }
-        });
-    });
+    if (cusparse::is_supported<ValueType, IndexType>::value) {
+        auto handle = exec->get_cusparse_handle();
+        auto descr = cusparse::create_mat_descr();
+        auto m = IndexType(to_sort->get_size()[0]);
+        auto n = IndexType(to_sort->get_size()[1]);
+        auto nnz = IndexType(to_sort->get_num_stored_elements());
+        auto row_ptrs = to_sort->get_const_row_ptrs();
+        auto col_idxs = to_sort->get_col_idxs();
+        auto vals = to_sort->get_values();
+
+        // copy values
+        Array<ValueType> tmp_vals_array(exec, nnz);
+        exec->copy(nnz, vals, tmp_vals_array.get_data());
+        auto tmp_vals = tmp_vals_array.get_const_data();
+
+        // init identity permutation
+        Array<IndexType> permutation_array(exec, nnz);
+        auto permutation = permutation_array.get_data();
+        cusparse::create_identity_permutation(handle, nnz, permutation);
+
+        // allocate buffer
+        size_type buffer_size{};
+        cusparse::csrsort_buffer_size(handle, m, n, nnz, row_ptrs, col_idxs,
+                                      buffer_size);
+        Array<char> buffer_array{exec, buffer_size};
+        auto buffer = buffer_array.get_data();
+
+        // sort column indices
+        cusparse::csrsort(handle, m, n, nnz, descr, row_ptrs, col_idxs,
+                          permutation, buffer);
+
+        // sort values
+#if defined(DPCPP_VERSION) && (DPCPP_VERSION < 11000)
+        cusparse::gather(handle, nnz, tmp_vals, vals, permutation);
+#else  // DPCPP_VERSION >= 11000
+        auto val_vec = cusparse::create_spvec(nnz, nnz, permutation, vals);
+        auto tmp_vec =
+            cusparse::create_dnvec(nnz, const_cast<ValueType *>(tmp_vals));
+        cusparse::gather(handle, tmp_vec, val_vec);
+#endif
+
+        cusparse::destroy(descr);
+    } else {
+        GKO_NOT_IMPLEMENTED;
+    }
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -855,28 +3125,18 @@ void is_sorted_by_column_index(
     std::shared_ptr<const DpcppExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *to_check, bool *is_sorted)
 {
-    Array<bool> is_sorted_device_array{exec, {true}};
-    const auto num_rows = to_check->get_size()[0];
-    const auto row_ptrs = to_check->get_const_row_ptrs();
-    const auto cols = to_check->get_const_col_idxs();
-    auto is_sorted_device = is_sorted_device_array.get_data();
-    exec->get_queue()->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
-            const auto row = static_cast<size_type>(idx[0]);
-            const auto begin = row_ptrs[row];
-            const auto end = row_ptrs[row + 1];
-            if (*is_sorted_device) {
-                for (auto i = begin; i < end - 1; i++) {
-                    if (cols[i] > cols[i + 1]) {
-                        *is_sorted_device = false;
-                        break;
-                    }
-                }
-            }
-        });
-    });
-    *is_sorted = exec->copy_val_to_host(is_sorted_device);
-};
+    *is_sorted = true;
+    auto cpu_array = Array<bool>::view(exec->get_master(), 1, is_sorted);
+    auto gpu_array = Array<bool>{exec, cpu_array};
+    auto block_size = default_block_size;
+    auto num_rows = static_cast<IndexType>(to_check->get_size()[0]);
+    auto num_blocks = ceildiv(num_rows, block_size);
+    kernel::check_unsorted(num_blocks, block_size, 0, exec->get_queue(),
+                           to_check->get_const_row_ptrs(),
+                           to_check->get_const_col_idxs(), num_rows,
+                           gpu_array.get_data());
+    cpu_array = gpu_array;
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_IS_SORTED_BY_COLUMN_INDEX);
@@ -885,7 +3145,22 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Csr<ValueType, IndexType> *orig,
-                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
+                      matrix::Diagonal<ValueType> *diag)
+{
+    const auto nnz = orig->get_num_stored_elements();
+    const auto diag_size = diag->get_size()[0];
+    const auto num_blocks =
+        ceildiv(config::warp_size * diag_size, default_block_size);
+
+    const auto orig_values = orig->get_const_values();
+    const auto orig_row_ptrs = orig->get_const_row_ptrs();
+    const auto orig_col_idxs = orig->get_const_col_idxs();
+    auto diag_values = diag->get_values();
+
+    kernel::extract_diagonal(num_blocks, default_block_size, 0,
+                             exec->get_queue(), diag_size, nnz, orig_values,
+                             orig_row_ptrs, orig_col_idxs, diag_values);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_EXTRACT_DIAGONAL);
 
diff --git a/dpcpp/matrix/diagonal_kernels.dp.cpp b/dpcpp/matrix/diagonal_kernels.dp.cpp
index c2910c60f0a..8bd2b937a35 100644
--- a/dpcpp/matrix/diagonal_kernels.dp.cpp
+++ b/dpcpp/matrix/diagonal_kernels.dp.cpp
@@ -30,9 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/matrix/diagonal_kernels.hpp"
-
-
 #include <CL/sycl.hpp>
 
 
@@ -40,6 +37,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "dpcpp/base/config.hpp"
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace dpcpp {
@@ -51,11 +54,251 @@ namespace dpcpp {
 namespace diagonal {
 
 
+constexpr auto default_block_size = 512;
+
+
+namespace kernel {
+
+
+template <typename ValueType>
+void apply_to_dense(size_type num_rows, size_type num_cols,
+                    const ValueType *__restrict__ diag, size_type source_stride,
+                    const ValueType *__restrict__ source_values,
+                    size_type result_stride,
+                    ValueType *__restrict__ result_values,
+                    sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    const auto row = tidx / num_cols;
+    const auto col = tidx % num_cols;
+
+    if (row < num_rows) {
+        result_values[row * result_stride + col] =
+            source_values[row * source_stride + col] * diag[row];
+    }
+}
+
+template <typename ValueType>
+void apply_to_dense(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                    sycl::queue *stream, size_type num_rows, size_type num_cols,
+                    const ValueType *diag, size_type source_stride,
+                    const ValueType *source_values, size_type result_stride,
+                    ValueType *result_values)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                apply_to_dense(num_rows, num_cols, diag, source_stride,
+                               source_values, result_stride, result_values,
+                               item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType>
+void right_apply_to_dense(size_type num_rows, size_type num_cols,
+                          const ValueType *__restrict__ diag,
+                          size_type source_stride,
+                          const ValueType *__restrict__ source_values,
+                          size_type result_stride,
+                          ValueType *__restrict__ result_values,
+                          sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    const auto row = tidx / num_cols;
+    const auto col = tidx % num_cols;
+
+    if (row < num_rows) {
+        result_values[row * result_stride + col] =
+            source_values[row * source_stride + col] * diag[col];
+    }
+}
+
+template <typename ValueType>
+void right_apply_to_dense(dim3 grid, dim3 block,
+                          gko::size_type dynamic_shared_memory,
+                          sycl::queue *stream, size_type num_rows,
+                          size_type num_cols, const ValueType *diag,
+                          size_type source_stride,
+                          const ValueType *source_values,
+                          size_type result_stride, ValueType *result_values)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                right_apply_to_dense(num_rows, num_cols, diag, source_stride,
+                                     source_values, result_stride,
+                                     result_values, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void apply_to_csr(size_type num_rows, const ValueType *__restrict__ diag,
+                  const IndexType *__restrict__ row_ptrs,
+                  ValueType *__restrict__ result_values,
+                  sycl::nd_item<3> item_ct1)
+{
+    constexpr auto warp_size = config::warp_size;
+    auto warp_tile =
+        group::tiled_partition<warp_size>(group::this_thread_block(item_ct1));
+    const auto row = thread::get_subwarp_id_flat<warp_size>(item_ct1);
+    const auto tid_in_warp = warp_tile.thread_rank();
+
+    if (row >= num_rows) {
+        return;
+    }
+
+    const auto diag_val = diag[row];
+
+    for (size_type idx = row_ptrs[row] + tid_in_warp; idx < row_ptrs[row + 1];
+         idx += warp_size) {
+        result_values[idx] *= diag_val;
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void apply_to_csr(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                  sycl::queue *stream, size_type num_rows,
+                  const ValueType *diag, const IndexType *row_ptrs,
+                  ValueType *result_values)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                apply_to_csr(num_rows, diag, row_ptrs, result_values, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void right_apply_to_csr(size_type num_nnz, const ValueType *__restrict__ diag,
+                        const IndexType *__restrict__ col_idxs,
+                        ValueType *__restrict__ result_values,
+                        sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+
+    if (tidx >= num_nnz) {
+        return;
+    }
+
+    result_values[tidx] *= diag[col_idxs[tidx]];
+}
+
+template <typename ValueType, typename IndexType>
+void right_apply_to_csr(dim3 grid, dim3 block,
+                        gko::size_type dynamic_shared_memory,
+                        sycl::queue *stream, size_type num_nnz,
+                        const ValueType *diag, const IndexType *col_idxs,
+                        ValueType *result_values)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             right_apply_to_csr(num_nnz, diag, col_idxs,
+                                                result_values, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_csr(size_type size, const ValueType *__restrict__ diag_values,
+                    IndexType *__restrict__ row_ptrs,
+                    IndexType *__restrict__ col_idxs,
+                    ValueType *__restrict__ csr_values,
+                    sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+
+    if (tidx >= size) {
+        return;
+    }
+    if (tidx == 0) {
+        row_ptrs[size] = size;
+    }
+
+    row_ptrs[tidx] = tidx;
+    col_idxs[tidx] = tidx;
+    csr_values[tidx] = diag_values[tidx];
+}
+
+template <typename ValueType, typename IndexType>
+void convert_to_csr(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                    sycl::queue *stream, size_type size,
+                    const ValueType *diag_values, IndexType *row_ptrs,
+                    IndexType *col_idxs, ValueType *csr_values)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             convert_to_csr(size, diag_values, row_ptrs,
+                                            col_idxs, csr_values, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType>
+void conj_transpose(size_type size, const ValueType *__restrict__ orig_values,
+                    ValueType *__restrict__ trans_values,
+                    sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+
+    if (tidx >= size) {
+        return;
+    }
+
+    /*
+    DPCT1007:5: Migration of this CUDA API is not supported by the Intel(R)
+    DPC++ Compatibility Tool.
+    */
+    trans_values[tidx] = conj(orig_values[tidx]);
+}
+
+template <typename ValueType>
+void conj_transpose(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                    sycl::queue *stream, size_type size,
+                    const ValueType *orig_values, ValueType *trans_values)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                conj_transpose(size, orig_values, trans_values, item_ct1);
+            });
+    });
+}
+
+
+}  // namespace kernel
+
+
 template <typename ValueType>
 void apply_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Diagonal<ValueType> *a,
                     const matrix::Dense<ValueType> *b,
-                    matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+                    matrix::Dense<ValueType> *c)
+{
+    const auto b_size = b->get_size();
+    const auto num_rows = b_size[0];
+    const auto num_cols = b_size[1];
+    const auto b_stride = b->get_stride();
+    const auto c_stride = c->get_stride();
+    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
+
+    const auto diag_values = a->get_const_values();
+    const auto b_values = b->get_const_values();
+    auto c_values = c->get_values();
+
+    kernel::apply_to_dense(grid_dim, default_block_size, 0, exec->get_queue(),
+                           num_rows, num_cols, diag_values, b_stride, b_values,
+                           c_stride, c_values);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_APPLY_TO_DENSE_KERNEL);
 
@@ -64,7 +307,23 @@ template <typename ValueType>
 void right_apply_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                           const matrix::Diagonal<ValueType> *a,
                           const matrix::Dense<ValueType> *b,
-                          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+                          matrix::Dense<ValueType> *c)
+{
+    const auto b_size = b->get_size();
+    const auto num_rows = b_size[0];
+    const auto num_cols = b_size[1];
+    const auto b_stride = b->get_stride();
+    const auto c_stride = c->get_stride();
+    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
+
+    const auto diag_values = a->get_const_values();
+    const auto b_values = b->get_const_values();
+    auto c_values = c->get_values();
+
+    kernel::right_apply_to_dense(
+        grid_dim, default_block_size, 0, exec->get_queue(), num_rows, num_cols,
+        diag_values, b_stride, b_values, c_stride, c_values);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_DENSE_KERNEL);
@@ -74,7 +333,19 @@ template <typename ValueType, typename IndexType>
 void apply_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                   const matrix::Diagonal<ValueType> *a,
                   const matrix::Csr<ValueType, IndexType> *b,
-                  matrix::Csr<ValueType, IndexType> *c) GKO_NOT_IMPLEMENTED;
+                  matrix::Csr<ValueType, IndexType> *c)
+{
+    const auto num_rows = b->get_size()[0];
+    const auto diag_values = a->get_const_values();
+    c->copy_from(b);
+    auto csr_values = c->get_values();
+    const auto csr_row_ptrs = c->get_const_row_ptrs();
+
+    const auto grid_dim =
+        ceildiv(num_rows * config::warp_size, default_block_size);
+    kernel::apply_to_csr(grid_dim, default_block_size, 0, exec->get_queue(),
+                         num_rows, diag_values, csr_row_ptrs, csr_values);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DIAGONAL_APPLY_TO_CSR_KERNEL);
@@ -85,7 +356,18 @@ void right_apply_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                         const matrix::Diagonal<ValueType> *a,
                         const matrix::Csr<ValueType, IndexType> *b,
                         matrix::Csr<ValueType, IndexType> *c)
-    GKO_NOT_IMPLEMENTED;
+{
+    const auto num_nnz = b->get_num_stored_elements();
+    const auto diag_values = a->get_const_values();
+    c->copy_from(b);
+    auto csr_values = c->get_values();
+    const auto csr_col_idxs = c->get_const_col_idxs();
+
+    const auto grid_dim = ceildiv(num_nnz, default_block_size);
+    kernel::right_apply_to_csr(grid_dim, default_block_size, 0,
+                               exec->get_queue(), num_nnz, diag_values,
+                               csr_col_idxs, csr_values);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_CSR_KERNEL);
@@ -95,7 +377,18 @@ template <typename ValueType, typename IndexType>
 void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Diagonal<ValueType> *source,
                     matrix::Csr<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
+{
+    const auto size = source->get_size()[0];
+    const auto grid_dim = ceildiv(size, default_block_size);
+
+    const auto diag_values = source->get_const_values();
+    auto row_ptrs = result->get_row_ptrs();
+    auto col_idxs = result->get_col_idxs();
+    auto csr_values = result->get_values();
+
+    kernel::convert_to_csr(grid_dim, default_block_size, 0, exec->get_queue(),
+                           size, diag_values, row_ptrs, col_idxs, csr_values);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DIAGONAL_CONVERT_TO_CSR_KERNEL);
@@ -104,7 +397,16 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType>
 void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Diagonal<ValueType> *orig,
-                    matrix::Diagonal<ValueType> *trans) GKO_NOT_IMPLEMENTED;
+                    matrix::Diagonal<ValueType> *trans)
+{
+    const auto size = orig->get_size()[0];
+    const auto grid_dim = ceildiv(size, default_block_size);
+    const auto orig_values = orig->get_const_values();
+    auto trans_values = trans->get_values();
+
+    kernel::conj_transpose(grid_dim, default_block_size, 0, exec->get_queue(),
+                           size, orig_values, trans_values);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_CONJ_TRANSPOSE_KERNEL);
 
diff --git a/dpcpp/matrix/fbcsr_kernels.dp.cpp b/dpcpp/matrix/fbcsr_kernels.dp.cpp
index 32244099d6b..e8846f7229c 100644
--- a/dpcpp/matrix/fbcsr_kernels.dp.cpp
+++ b/dpcpp/matrix/fbcsr_kernels.dp.cpp
@@ -30,10 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/matrix/fbcsr_kernels.hpp"
-
-
-#include <CL/sycl.hpp>
+#include <algorithm>
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -43,6 +40,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
+#include "dpcpp/base/config.hpp"
+
+
 namespace gko {
 namespace kernels {
 namespace dpcpp {
@@ -100,22 +100,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_FBCSR_CONVERT_TO_CSR_KERNEL);
 
 
-template <typename ValueType, typename IndexType, typename UnaryOperator>
-inline void convert_fbcsr_to_csc(size_type num_rows, const IndexType *row_ptrs,
-                                 const IndexType *col_idxs,
-                                 const ValueType *fbcsr_vals,
-                                 IndexType *row_idxs, IndexType *col_ptrs,
-                                 ValueType *csc_vals,
-                                 UnaryOperator op) GKO_NOT_IMPLEMENTED;
-
-
-template <typename ValueType, typename IndexType, typename UnaryOperator>
-void transpose_and_transform(std::shared_ptr<const DpcppExecutor> exec,
-                             matrix::Fbcsr<ValueType, IndexType> *trans,
-                             const matrix::Fbcsr<ValueType, IndexType> *orig,
-                             UnaryOperator op) GKO_NOT_IMPLEMENTED;
-
-
 template <typename ValueType, typename IndexType>
 void transpose(std::shared_ptr<const DpcppExecutor> exec,
                const matrix::Fbcsr<ValueType, IndexType> *orig,
diff --git a/dpcpp/matrix/hybrid_kernels.dp.cpp b/dpcpp/matrix/hybrid_kernels.dp.cpp
index b5d9377f990..25d453a623d 100644
--- a/dpcpp/matrix/hybrid_kernels.dp.cpp
+++ b/dpcpp/matrix/hybrid_kernels.dp.cpp
@@ -30,20 +30,27 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/matrix/hybrid_kernels.hpp"
+#include <dpcpp/components/segment_scan.dp.hpp>
 
 
 #include <CL/sycl.hpp>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/matrix/csr.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/ell.hpp>
 
 
+#include "core/components/fill_array.hpp"
+#include "core/components/prefix_sum.hpp"
+#include "core/matrix/coo_kernels.hpp"
 #include "core/matrix/ell_kernels.hpp"
+#include "dpcpp/base/config.hpp"
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/components/atomic.dp.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/format_conversion.dp.hpp"
+#include "dpcpp/components/reduction.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
 
 
 namespace gko {
@@ -57,6 +64,173 @@ namespace dpcpp {
 namespace hybrid {
 
 
+constexpr int default_block_size = 512;
+constexpr int warps_in_block = 4;
+
+
+namespace kernel {
+
+
+/**
+ * The global function for counting the number of nonzeros per row of COO.
+ * It is almost like COO spmv routine.
+ * It performs is_nonzeros(Coo) times the vector whose values are one
+ *
+ * @param nnz  the number of nonzeros in the matrix
+ * @param num_line  the maximum round of each warp
+ * @param val  the value array of the matrix
+ * @param row  the row index array of the matrix
+ * @param nnz_per_row  the output nonzeros per row
+ */
+template <int subwarp_size = config::warp_size, typename ValueType,
+          typename IndexType>
+void count_coo_row_nnz(const size_type nnz, const size_type num_lines,
+                       const ValueType *__restrict__ val,
+                       const IndexType *__restrict__ row,
+                       IndexType *__restrict__ nnz_per_row,
+                       sycl::nd_item<3> item_ct1)
+{
+    IndexType temp_val = 0;
+    const auto start =
+        static_cast<size_type>(item_ct1.get_local_range().get(2)) *
+            item_ct1.get_group(2) * item_ct1.get_local_range().get(1) *
+            num_lines +
+        item_ct1.get_local_id(1) * item_ct1.get_local_range().get(2) *
+            num_lines;
+    size_type num = (nnz > start) * ceildiv(nnz - start, subwarp_size);
+    num = min(num, num_lines);
+    const IndexType ind_start = start + item_ct1.get_local_id(2);
+    const IndexType ind_end = ind_start + (num - 1) * subwarp_size;
+    IndexType ind = ind_start;
+    IndexType curr_row = (ind < nnz) ? row[ind] : 0;
+    const auto tile_block = group::tiled_partition<subwarp_size>(
+        group::this_thread_block(item_ct1));
+    for (; ind < ind_end; ind += subwarp_size) {
+        temp_val += ind < nnz && val[ind] != zero<ValueType>();
+        auto next_row =
+            (ind + subwarp_size < nnz) ? row[ind + subwarp_size] : row[nnz - 1];
+        // segmented scan
+        if (tile_block.any(curr_row != next_row)) {
+            bool is_first_in_segment =
+                segment_scan<subwarp_size>(tile_block, curr_row, &temp_val);
+            if (is_first_in_segment) {
+                atomic_add(&(nnz_per_row[curr_row]), temp_val);
+            }
+            temp_val = 0;
+        }
+        curr_row = next_row;
+    }
+    if (num > 0) {
+        ind = ind_end;
+        temp_val += ind < nnz && val[ind] != zero<ValueType>();
+        // segmented scan
+
+        bool is_first_in_segment =
+            segment_scan<subwarp_size>(tile_block, curr_row, &temp_val);
+        if (is_first_in_segment) {
+            atomic_add(&(nnz_per_row[curr_row]), temp_val);
+        }
+    }
+}
+
+template <int subwarp_size = config::warp_size, typename ValueType,
+          typename IndexType>
+void count_coo_row_nnz(dim3 grid, dim3 block,
+                       gko::size_type dynamic_shared_memory,
+                       sycl::queue *stream, const size_type nnz,
+                       const size_type num_lines, const ValueType *val,
+                       const IndexType *row, IndexType *nnz_per_row)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                count_coo_row_nnz<subwarp_size>(nnz, num_lines, val, row,
+                                                nnz_per_row, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void fill_in_csr(size_type num_rows, size_type max_nnz_per_row,
+                 size_type stride, const ValueType *__restrict__ ell_val,
+                 const IndexType *__restrict__ ell_col,
+                 const ValueType *__restrict__ coo_val,
+                 const IndexType *__restrict__ coo_col,
+                 const IndexType *__restrict__ coo_offset,
+                 IndexType *__restrict__ result_row_ptrs,
+                 IndexType *__restrict__ result_col_idxs,
+                 ValueType *__restrict__ result_values,
+                 sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+
+    if (tidx < num_rows) {
+        auto write_to = result_row_ptrs[tidx];
+        for (size_type i = 0; i < max_nnz_per_row; i++) {
+            const auto source_idx = tidx + stride * i;
+            if (ell_val[source_idx] != zero<ValueType>()) {
+                result_values[write_to] = ell_val[source_idx];
+                result_col_idxs[write_to] = ell_col[source_idx];
+                write_to++;
+            }
+        }
+        for (auto i = coo_offset[tidx]; i < coo_offset[tidx + 1]; i++) {
+            if (coo_val[i] != zero<ValueType>()) {
+                result_values[write_to] = coo_val[i];
+                result_col_idxs[write_to] = coo_col[i];
+                write_to++;
+            }
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void fill_in_csr(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                 sycl::queue *stream, size_type num_rows,
+                 size_type max_nnz_per_row, size_type stride,
+                 const ValueType *ell_val, const IndexType *ell_col,
+                 const ValueType *coo_val, const IndexType *coo_col,
+                 const IndexType *coo_offset, IndexType *result_row_ptrs,
+                 IndexType *result_col_idxs, ValueType *result_values)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                fill_in_csr(num_rows, max_nnz_per_row, stride, ell_val, ell_col,
+                            coo_val, coo_col, coo_offset, result_row_ptrs,
+                            result_col_idxs, result_values, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType1, typename ValueType2>
+void add(size_type num, ValueType1 *__restrict__ val1,
+         const ValueType2 *__restrict__ val2, sycl::nd_item<3> item_ct1)
+{
+    const auto tidx = thread::get_thread_id_flat(item_ct1);
+    if (tidx < num) {
+        val1[tidx] += val2[tidx];
+    }
+}
+
+template <typename ValueType1, typename ValueType2>
+void add(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+         sycl::queue *stream, size_type num, ValueType1 *val1,
+         const ValueType2 *val2)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block),
+            [=](sycl::nd_item<3> item_ct1) { add(num, val1, val2, item_ct1); });
+    });
+}
+
+
+}  // namespace kernel
+
+
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Hybrid<ValueType, IndexType> *source,
@@ -70,7 +244,62 @@ template <typename ValueType, typename IndexType>
 void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Hybrid<ValueType, IndexType> *source,
                     matrix::Csr<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
+{
+    const auto num_rows = source->get_size()[0];
+    auto coo_offset = Array<IndexType>(exec, num_rows + 1);
+    auto coo_val = source->get_const_coo_values();
+    auto coo_col = source->get_const_coo_col_idxs();
+    auto coo_row = source->get_const_coo_row_idxs();
+    auto ell_val = source->get_const_ell_values();
+    auto ell_col = source->get_const_ell_col_idxs();
+    const auto stride = source->get_ell_stride();
+    const auto max_nnz_per_row = source->get_ell_num_stored_elements_per_row();
+    const auto coo_num_stored_elements = source->get_coo_num_stored_elements();
+
+    // Compute the row offset of Coo without zeros
+    size_type grid_num = ceildiv(coo_num_stored_elements, default_block_size);
+    coo::kernel::convert_row_idxs_to_ptrs(
+        grid_num, default_block_size, 0, exec->get_queue(), coo_row,
+        coo_num_stored_elements, coo_offset.get_data(), num_rows + 1);
+
+    // Compute the row ptrs of Csr
+    auto row_ptrs = result->get_row_ptrs();
+    auto coo_row_ptrs = Array<IndexType>(exec, num_rows);
+
+    components::fill_array(exec, row_ptrs, num_rows + 1, zero<IndexType>());
+    grid_num = ceildiv(num_rows, warps_in_block);
+    ell::kernel::count_nnz_per_row(grid_num, default_block_size, 0,
+                                   exec->get_queue(), num_rows, max_nnz_per_row,
+                                   stride, ell_val, row_ptrs);
+
+    components::fill_array(exec, coo_row_ptrs.get_data(), num_rows,
+                           zero<IndexType>());
+
+    auto nwarps =
+        coo::host_kernel::calculate_nwarps(exec, coo_num_stored_elements);
+    if (nwarps > 0) {
+        int num_lines =
+            ceildiv(coo_num_stored_elements, nwarps * config::warp_size);
+        const dim3 coo_block(config::warp_size, warps_in_block, 1);
+        const dim3 coo_grid(ceildiv(nwarps, warps_in_block), 1);
+
+        kernel::count_coo_row_nnz(coo_grid, coo_block, 0, exec->get_queue(),
+                                  coo_num_stored_elements, num_lines, coo_val,
+                                  coo_row, coo_row_ptrs.get_data());
+    }
+
+    kernel::add(grid_num, default_block_size, 0, exec->get_queue(), num_rows,
+                row_ptrs, coo_row_ptrs.get_const_data());
+
+    components::prefix_sum(exec, row_ptrs, num_rows + 1);
+
+    // Fill the value
+    grid_num = ceildiv(num_rows, default_block_size);
+    kernel::fill_in_csr(grid_num, default_block_size, 0, exec->get_queue(),
+                        num_rows, max_nnz_per_row, stride, ell_val, ell_col,
+                        coo_val, coo_col, coo_offset.get_const_data(), row_ptrs,
+                        result->get_col_idxs(), result->get_values());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_HYBRID_CONVERT_TO_CSR_KERNEL);
@@ -79,7 +308,33 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void count_nonzeros(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Hybrid<ValueType, IndexType> *source,
-                    size_type *result) GKO_NOT_IMPLEMENTED;
+                    size_type *result)
+{
+    size_type ell_nnz = 0;
+    size_type coo_nnz = 0;
+    ell::count_nonzeros(exec, source->get_ell(), &ell_nnz);
+
+    auto nnz = source->get_coo_num_stored_elements();
+    auto nwarps = coo::host_kernel::calculate_nwarps(exec, nnz);
+    if (nwarps > 0) {
+        int num_lines = ceildiv(nnz, nwarps * config::warp_size);
+        const dim3 coo_block(config::warp_size, warps_in_block, 1);
+        const dim3 coo_grid(ceildiv(nwarps, warps_in_block), 1);
+        const auto num_rows = source->get_size()[0];
+        auto nnz_per_row = Array<IndexType>(exec, num_rows);
+        components::fill_array(exec, nnz_per_row.get_data(), num_rows,
+                               zero<IndexType>());
+        kernel::count_coo_row_nnz(
+            coo_grid, coo_block, 0, exec->get_queue(), nnz, num_lines,
+            source->get_coo()->get_const_values(),
+            source->get_coo()->get_const_row_idxs(), nnz_per_row.get_data());
+
+        coo_nnz =
+            reduce_add_array(exec, num_rows, nnz_per_row.get_const_data());
+    }
+
+    *result = ell_nnz + coo_nnz;
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_HYBRID_COUNT_NONZEROS_KERNEL);
diff --git a/dpcpp/matrix/sellp_kernels.dp.cpp b/dpcpp/matrix/sellp_kernels.dp.cpp
index 4547accd805..0cc5973975b 100644
--- a/dpcpp/matrix/sellp_kernels.dp.cpp
+++ b/dpcpp/matrix/sellp_kernels.dp.cpp
@@ -30,13 +30,23 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/matrix/sellp_kernels.hpp"
-
-
 #include <CL/sycl.hpp>
 
 
 #include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/math.hpp>
+#include <ginkgo/core/base/types.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/components/prefix_sum.hpp"
+#include "dpcpp/base/config.hpp"
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/base/onemkl_bindings.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
+#include "dpcpp/components/reduction.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
 
 
 namespace gko {
@@ -50,11 +60,361 @@ namespace dpcpp {
 namespace sellp {
 
 
+constexpr auto default_block_size = 512;
+
+
+namespace {
+
+
+template <typename ValueType, typename IndexType>
+void spmv_kernel(size_type num_rows, size_type num_right_hand_sides,
+                 size_type b_stride, size_type c_stride,
+                 const size_type *__restrict__ slice_lengths,
+                 const size_type *__restrict__ slice_sets,
+                 const ValueType *__restrict__ a,
+                 const IndexType *__restrict__ col,
+                 const ValueType *__restrict__ b, ValueType *__restrict__ c,
+                 sycl::nd_item<3> item_ct1)
+{
+    const auto slice_id = item_ct1.get_group(2);
+    const auto slice_size = item_ct1.get_local_range().get(2);
+    const auto row_in_slice = item_ct1.get_local_id(2);
+    const auto global_row =
+        static_cast<size_type>(slice_size) * slice_id + row_in_slice;
+    const auto column_id = item_ct1.get_group(1);
+    ValueType val = 0;
+    IndexType ind = 0;
+    if (global_row < num_rows && column_id < num_right_hand_sides) {
+        for (size_type i = 0; i < slice_lengths[slice_id]; i++) {
+            ind = row_in_slice + (slice_sets[slice_id] + i) * slice_size;
+            val += a[ind] * b[col[ind] * b_stride + column_id];
+        }
+        c[global_row * c_stride + column_id] = val;
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void spmv_kernel(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                 sycl::queue *stream, size_type num_rows,
+                 size_type num_right_hand_sides, size_type b_stride,
+                 size_type c_stride, const size_type *slice_lengths,
+                 const size_type *slice_sets, const ValueType *a,
+                 const IndexType *col, const ValueType *b, ValueType *c)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                spmv_kernel(num_rows, num_right_hand_sides, b_stride, c_stride,
+                            slice_lengths, slice_sets, a, col, b, c, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void advanced_spmv_kernel(size_type num_rows, size_type num_right_hand_sides,
+                          size_type b_stride, size_type c_stride,
+                          const size_type *__restrict__ slice_lengths,
+                          const size_type *__restrict__ slice_sets,
+                          const ValueType *__restrict__ alpha,
+                          const ValueType *__restrict__ a,
+                          const IndexType *__restrict__ col,
+                          const ValueType *__restrict__ b,
+                          const ValueType *__restrict__ beta,
+                          ValueType *__restrict__ c, sycl::nd_item<3> item_ct1)
+{
+    const auto slice_id = item_ct1.get_group(2);
+    const auto slice_size = item_ct1.get_local_range().get(2);
+    const auto row_in_slice = item_ct1.get_local_id(2);
+    const auto global_row =
+        static_cast<size_type>(slice_size) * slice_id + row_in_slice;
+    const auto column_id = item_ct1.get_group(1);
+    ValueType val = 0;
+    IndexType ind = 0;
+    if (global_row < num_rows && column_id < num_right_hand_sides) {
+        for (size_type i = 0; i < slice_lengths[slice_id]; i++) {
+            ind = row_in_slice + (slice_sets[slice_id] + i) * slice_size;
+            val += alpha[0] * a[ind] * b[col[ind] * b_stride + column_id];
+        }
+        c[global_row * c_stride + column_id] =
+            beta[0] * c[global_row * c_stride + column_id] + val;
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void advanced_spmv_kernel(dim3 grid, dim3 block,
+                          gko::size_type dynamic_shared_memory,
+                          sycl::queue *stream, size_type num_rows,
+                          size_type num_right_hand_sides, size_type b_stride,
+                          size_type c_stride, const size_type *slice_lengths,
+                          const size_type *slice_sets, const ValueType *alpha,
+                          const ValueType *a, const IndexType *col,
+                          const ValueType *b, const ValueType *beta,
+                          ValueType *c)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                advanced_spmv_kernel(num_rows, num_right_hand_sides, b_stride,
+                                     c_stride, slice_lengths, slice_sets, alpha,
+                                     a, col, b, beta, c, item_ct1);
+            });
+    });
+}
+
+
+}  // namespace
+
+
+namespace kernel {
+
+
+template <typename ValueType>
+void initialize_zero_dense(size_type num_rows, size_type num_cols,
+                           size_type stride, ValueType *__restrict__ result,
+                           sycl::nd_item<3> item_ct1)
+{
+    const auto tidx_x =
+        item_ct1.get_local_id(2) +
+        item_ct1.get_local_range().get(2) * item_ct1.get_group(2);
+    const auto tidx_y =
+        item_ct1.get_local_id(1) +
+        item_ct1.get_local_range().get(1) * item_ct1.get_group(1);
+    if (tidx_x < num_cols && tidx_y < num_rows) {
+        result[tidx_y * stride + tidx_x] = zero<ValueType>();
+    }
+}
+
+template <typename ValueType>
+void initialize_zero_dense(dim3 grid, dim3 block,
+                           gko::size_type dynamic_shared_memory,
+                           sycl::queue *stream, size_type num_rows,
+                           size_type num_cols, size_type stride,
+                           ValueType *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             initialize_zero_dense(num_rows, num_cols, stride,
+                                                   result, item_ct1);
+                         });
+    });
+}
+
+
+template <unsigned int threads_per_row, typename ValueType, typename IndexType>
+void fill_in_dense(size_type num_rows, size_type num_cols, size_type stride,
+                   size_type slice_size,
+                   const size_type *__restrict__ slice_lengths,
+                   const size_type *__restrict__ slice_sets,
+                   const IndexType *__restrict__ col_idxs,
+                   const ValueType *__restrict__ values,
+                   ValueType *__restrict__ result, sycl::nd_item<3> item_ct1)
+{
+    const auto global_row =
+        thread::get_subwarp_id_flat<threads_per_row>(item_ct1);
+    const auto row = global_row % slice_size;
+    const auto slice = global_row / slice_size;
+    const auto start_index = item_ct1.get_local_id(2) % threads_per_row;
+
+    if (global_row < num_rows) {
+        for (auto i = start_index; i < slice_lengths[slice];
+             i += threads_per_row) {
+            if (values[(slice_sets[slice] + i) * slice_size + row] !=
+                zero<ValueType>()) {
+                result[global_row * stride +
+                       col_idxs[(slice_sets[slice] + i) * slice_size + row]] =
+                    values[(slice_sets[slice] + i) * slice_size + row];
+            }
+        }
+    }
+}
+
+template <unsigned int threads_per_row, typename ValueType, typename IndexType>
+void fill_in_dense(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                   sycl::queue *stream, size_type num_rows, size_type num_cols,
+                   size_type stride, size_type slice_size,
+                   const size_type *slice_lengths, const size_type *slice_sets,
+                   const IndexType *col_idxs, const ValueType *values,
+                   ValueType *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                fill_in_dense<threads_per_row>(
+                    num_rows, num_cols, stride, slice_size, slice_lengths,
+                    slice_sets, col_idxs, values, result, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void count_nnz_per_row(size_type num_rows, size_type slice_size,
+                       const size_type *__restrict__ slice_sets,
+                       const ValueType *__restrict__ values,
+                       IndexType *__restrict__ result,
+                       sycl::nd_item<3> item_ct1)
+{
+    constexpr auto warp_size = config::warp_size;
+    auto warp_tile =
+        group::tiled_partition<warp_size>(group::this_thread_block(item_ct1));
+    const auto row_idx = thread::get_subwarp_id_flat<warp_size>(item_ct1);
+    const auto slice_id = row_idx / slice_size;
+    const auto tid_in_warp = warp_tile.thread_rank();
+    const auto row_in_slice = row_idx % slice_size;
+
+    if (row_idx < num_rows) {
+        IndexType part_result{};
+        for (size_type sellp_ind =
+                 (slice_sets[slice_id] + tid_in_warp) * slice_size +
+                 row_in_slice;
+             sellp_ind < slice_sets[slice_id + 1] * slice_size;
+             sellp_ind += warp_size * slice_size) {
+            if (values[sellp_ind] != zero<ValueType>()) {
+                part_result += 1;
+            }
+        }
+        result[row_idx] = reduce(
+            warp_tile, part_result,
+            [](const size_type &a, const size_type &b) { return a + b; });
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void count_nnz_per_row(dim3 grid, dim3 block,
+                       gko::size_type dynamic_shared_memory,
+                       sycl::queue *stream, size_type num_rows,
+                       size_type slice_size, const size_type *slice_sets,
+                       const ValueType *values, IndexType *result)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl_nd_range(grid, block),
+                         [=](sycl::nd_item<3> item_ct1) {
+                             count_nnz_per_row(num_rows, slice_size, slice_sets,
+                                               values, result, item_ct1);
+                         });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void fill_in_csr(size_type num_rows, size_type slice_size,
+                 const size_type *__restrict__ source_slice_sets,
+                 const IndexType *__restrict__ source_col_idxs,
+                 const ValueType *__restrict__ source_values,
+                 IndexType *__restrict__ result_row_ptrs,
+                 IndexType *__restrict__ result_col_idxs,
+                 ValueType *__restrict__ result_values,
+                 sycl::nd_item<3> item_ct1)
+{
+    const auto row = thread::get_thread_id_flat(item_ct1);
+    const auto slice_id = row / slice_size;
+    const auto row_in_slice = row % slice_size;
+
+    if (row < num_rows) {
+        size_type csr_ind = result_row_ptrs[row];
+        for (size_type sellp_ind =
+                 source_slice_sets[slice_id] * slice_size + row_in_slice;
+             sellp_ind < source_slice_sets[slice_id + 1] * slice_size;
+             sellp_ind += slice_size) {
+            if (source_values[sellp_ind] != zero<ValueType>()) {
+                result_values[csr_ind] = source_values[sellp_ind];
+                result_col_idxs[csr_ind] = source_col_idxs[sellp_ind];
+                csr_ind++;
+            }
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void fill_in_csr(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
+                 sycl::queue *stream, size_type num_rows, size_type slice_size,
+                 const size_type *source_slice_sets,
+                 const IndexType *source_col_idxs,
+                 const ValueType *source_values, IndexType *result_row_ptrs,
+                 IndexType *result_col_idxs, ValueType *result_values)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                fill_in_csr(num_rows, slice_size, source_slice_sets,
+                            source_col_idxs, source_values, result_row_ptrs,
+                            result_col_idxs, result_values, item_ct1);
+            });
+    });
+}
+
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(size_type diag_size, size_type slice_size,
+                      const size_type *__restrict__ orig_slice_sets,
+                      const ValueType *__restrict__ orig_values,
+                      const IndexType *__restrict__ orig_col_idxs,
+                      ValueType *__restrict__ diag, sycl::nd_item<3> item_ct1)
+{
+    constexpr auto warp_size = config::warp_size;
+    auto warp_tile =
+        group::tiled_partition<warp_size>(group::this_thread_block(item_ct1));
+    const auto slice_id = thread::get_subwarp_id_flat<warp_size>(item_ct1);
+    const auto tid_in_warp = warp_tile.thread_rank();
+    const auto slice_num = ceildiv(diag_size, slice_size);
+
+    if (slice_id >= slice_num) {
+        return;
+    }
+
+    const auto start_ind = orig_slice_sets[slice_id] * slice_size + tid_in_warp;
+    const auto end_ind = orig_slice_sets[slice_id + 1] * slice_size;
+
+    for (auto sellp_ind = start_ind; sellp_ind < end_ind;
+         sellp_ind += warp_size) {
+        auto global_row = slice_id * slice_size + sellp_ind % slice_size;
+        if (global_row < diag_size) {
+            if (orig_col_idxs[sellp_ind] == global_row &&
+                orig_values[sellp_ind] != zero<ValueType>()) {
+                diag[global_row] = orig_values[sellp_ind];
+            }
+        }
+    }
+}
+
+template <typename ValueType, typename IndexType>
+void extract_diagonal(dim3 grid, dim3 block,
+                      gko::size_type dynamic_shared_memory, sycl::queue *stream,
+                      size_type diag_size, size_type slice_size,
+                      const size_type *orig_slice_sets,
+                      const ValueType *orig_values,
+                      const IndexType *orig_col_idxs, ValueType *diag)
+{
+    stream->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(
+            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
+                extract_diagonal(diag_size, slice_size, orig_slice_sets,
+                                 orig_values, orig_col_idxs, diag, item_ct1);
+            });
+    });
+}
+
+
+}  // namespace kernel
+
+
 template <typename ValueType, typename IndexType>
 void spmv(std::shared_ptr<const DpcppExecutor> exec,
           const matrix::Sellp<ValueType, IndexType> *a,
-          const matrix::Dense<ValueType> *b,
-          matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+          const matrix::Dense<ValueType> *b, matrix::Dense<ValueType> *c)
+{
+    const dim3 blockSize(matrix::default_slice_size);
+    const dim3 gridSize(ceildiv(a->get_size()[0], matrix::default_slice_size),
+                        b->get_size()[1]);
+
+    spmv_kernel(gridSize, blockSize, 0, exec->get_queue(), a->get_size()[0],
+                b->get_size()[1], b->get_stride(), c->get_stride(),
+                a->get_const_slice_lengths(), a->get_const_slice_sets(),
+                a->get_const_values(), a->get_const_col_idxs(),
+                b->get_const_values(), c->get_values());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_SELLP_SPMV_KERNEL);
 
@@ -65,7 +425,20 @@ void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
                    const matrix::Sellp<ValueType, IndexType> *a,
                    const matrix::Dense<ValueType> *b,
                    const matrix::Dense<ValueType> *beta,
-                   matrix::Dense<ValueType> *c) GKO_NOT_IMPLEMENTED;
+                   matrix::Dense<ValueType> *c)
+{
+    const dim3 blockSize(matrix::default_slice_size);
+    const dim3 gridSize(ceildiv(a->get_size()[0], matrix::default_slice_size),
+                        b->get_size()[1]);
+
+    advanced_spmv_kernel(gridSize, blockSize, 0, exec->get_queue(),
+                         a->get_size()[0], b->get_size()[1], b->get_stride(),
+                         c->get_stride(), a->get_const_slice_lengths(),
+                         a->get_const_slice_sets(), alpha->get_const_values(),
+                         a->get_const_values(), a->get_const_col_idxs(),
+                         b->get_const_values(), beta->get_const_values(),
+                         c->get_values());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SELLP_ADVANCED_SPMV_KERNEL);
@@ -74,7 +447,40 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void convert_to_dense(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Sellp<ValueType, IndexType> *source,
-                      matrix::Dense<ValueType> *result) GKO_NOT_IMPLEMENTED;
+                      matrix::Dense<ValueType> *result)
+{
+    const auto num_rows = source->get_size()[0];
+    const auto num_cols = source->get_size()[1];
+    const auto vals = source->get_const_values();
+    const auto col_idxs = source->get_const_col_idxs();
+    const auto slice_lengths = source->get_const_slice_lengths();
+    const auto slice_sets = source->get_const_slice_sets();
+    const auto slice_size = source->get_slice_size();
+
+    const auto slice_num = ceildiv(num_rows, slice_size);
+
+    const dim3 block_size(config::warp_size,
+                          config::max_block_size / config::warp_size, 1);
+    const dim3 init_grid_dim(ceildiv(num_cols, block_size.x),
+                             ceildiv(num_rows, block_size.y), 1);
+
+    if (num_rows > 0 && result->get_stride() > 0) {
+        kernel::initialize_zero_dense(
+            init_grid_dim, block_size, 0, exec->get_queue(), num_rows, num_cols,
+            result->get_stride(), result->get_values());
+    }
+
+    constexpr auto threads_per_row = config::warp_size;
+    const auto grid_dim =
+        ceildiv(slice_size * slice_num * threads_per_row, default_block_size);
+
+    if (grid_dim > 0) {
+        kernel::fill_in_dense<threads_per_row>(
+            grid_dim, default_block_size, 0, exec->get_queue(), num_rows,
+            num_cols, result->get_stride(), slice_size, slice_lengths,
+            slice_sets, col_idxs, vals, result->get_values());
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SELLP_CONVERT_TO_DENSE_KERNEL);
@@ -84,7 +490,42 @@ template <typename ValueType, typename IndexType>
 void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Sellp<ValueType, IndexType> *source,
                     matrix::Csr<ValueType, IndexType> *result)
-    GKO_NOT_IMPLEMENTED;
+{
+    const auto num_rows = source->get_size()[0];
+    const auto slice_size = source->get_slice_size();
+    const auto slice_num = ceildiv(num_rows, slice_size);
+
+    const auto source_values = source->get_const_values();
+    const auto source_slice_lengths = source->get_const_slice_lengths();
+    const auto source_slice_sets = source->get_const_slice_sets();
+    const auto source_col_idxs = source->get_const_col_idxs();
+
+    auto result_values = result->get_values();
+    auto result_col_idxs = result->get_col_idxs();
+    auto result_row_ptrs = result->get_row_ptrs();
+
+    auto grid_dim = ceildiv(num_rows * config::warp_size, default_block_size);
+
+    if (grid_dim > 0) {
+        kernel::count_nnz_per_row(
+            grid_dim, default_block_size, 0, exec->get_queue(), num_rows,
+            slice_size, source_slice_sets, source_values, result_row_ptrs);
+    }
+
+    grid_dim = ceildiv(num_rows + 1, default_block_size);
+    auto add_values = Array<IndexType>(exec, grid_dim);
+
+    components::prefix_sum(exec, result_row_ptrs, num_rows + 1);
+
+    grid_dim = ceildiv(num_rows, default_block_size);
+
+    if (grid_dim > 0) {
+        kernel::fill_in_csr(grid_dim, default_block_size, 0, exec->get_queue(),
+                            num_rows, slice_size, source_slice_sets,
+                            source_col_idxs, source_values, result_row_ptrs,
+                            result_col_idxs, result_values);
+    }
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SELLP_CONVERT_TO_CSR_KERNEL);
@@ -93,7 +534,29 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void count_nonzeros(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Sellp<ValueType, IndexType> *source,
-                    size_type *result) GKO_NOT_IMPLEMENTED;
+                    size_type *result)
+{
+    const auto num_rows = source->get_size()[0];
+
+    if (num_rows <= 0) {
+        *result = 0;
+        return;
+    }
+
+    const auto slice_size = source->get_slice_size();
+    const auto slice_sets = source->get_const_slice_sets();
+    const auto values = source->get_const_values();
+
+    auto nnz_per_row = Array<size_type>(exec, num_rows);
+
+    auto grid_dim = ceildiv(num_rows * config::warp_size, default_block_size);
+
+    kernel::count_nnz_per_row(grid_dim, default_block_size, 0,
+                              exec->get_queue(), num_rows, slice_size,
+                              slice_sets, values, nnz_per_row.get_data());
+
+    *result = reduce_add_array(exec, num_rows, nnz_per_row.get_const_data());
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SELLP_COUNT_NONZEROS_KERNEL);
@@ -102,7 +565,23 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType, typename IndexType>
 void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
                       const matrix::Sellp<ValueType, IndexType> *orig,
-                      matrix::Diagonal<ValueType> *diag) GKO_NOT_IMPLEMENTED;
+                      matrix::Diagonal<ValueType> *diag)
+{
+    const auto diag_size = diag->get_size()[0];
+    const auto slice_size = orig->get_slice_size();
+    const auto slice_num = ceildiv(diag_size, slice_size);
+    const auto num_blocks =
+        ceildiv(slice_num * config::warp_size, default_block_size);
+
+    const auto orig_slice_sets = orig->get_const_slice_sets();
+    const auto orig_values = orig->get_const_values();
+    const auto orig_col_idxs = orig->get_const_col_idxs();
+    auto diag_values = diag->get_values();
+
+    kernel::extract_diagonal(
+        num_blocks, default_block_size, 0, exec->get_queue(), diag_size,
+        slice_size, orig_slice_sets, orig_values, orig_col_idxs, diag_values);
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SELLP_EXTRACT_DIAGONAL_KERNEL);
diff --git a/dpcpp/matrix/sparsity_csr_kernels.dp.cpp b/dpcpp/matrix/sparsity_csr_kernels.dp.cpp
index 39203fcd3b6..9d0afae20b7 100644
--- a/dpcpp/matrix/sparsity_csr_kernels.dp.cpp
+++ b/dpcpp/matrix/sparsity_csr_kernels.dp.cpp
@@ -30,31 +30,14 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "core/matrix/sparsity_csr_kernels.hpp"
-
-
-#include <algorithm>
-#include <numeric>
-#include <utility>
-
-
-#include <CL/sycl.hpp>
-
-
 #include <ginkgo/core/base/exception_helpers.hpp>
-#include <ginkgo/core/base/math.hpp>
-#include <ginkgo/core/matrix/dense.hpp>
-
-
-#include "core/base/iterator_factory.hpp"
-#include "dpcpp/components/format_conversion.dp.hpp"
 
 
 namespace gko {
 namespace kernels {
 namespace dpcpp {
 /**
- * @brief The SparsityCsr pattern format namespace.
+ * @brief The Compressed sparse row matrix format namespace.
  *
  * @ingroup sparsity
  */
@@ -103,21 +86,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_SPARSITY_CSR_REMOVE_DIAGONAL_ELEMENTS_KERNEL);
 
 
-template <typename IndexType>
-inline void convert_sparsity_to_csc(size_type num_rows,
-                                    const IndexType *row_ptrs,
-                                    const IndexType *col_idxs,
-                                    IndexType *row_idxs,
-                                    IndexType *col_ptrs) GKO_NOT_IMPLEMENTED;
-
-
-template <typename ValueType, typename IndexType>
-void transpose_and_transform(
-    std::shared_ptr<const DpcppExecutor> exec,
-    matrix::SparsityCsr<ValueType, IndexType> *trans,
-    const matrix::SparsityCsr<ValueType, IndexType> *orig) GKO_NOT_IMPLEMENTED;
-
-
 template <typename ValueType, typename IndexType>
 void transpose(std::shared_ptr<const DpcppExecutor> exec,
                const matrix::SparsityCsr<ValueType, IndexType> *orig,
diff --git a/dpcpp/test/matrix/coo_kernels.cpp b/dpcpp/test/matrix/coo_kernels.cpp
new file mode 100644
index 00000000000..6dc8e2729e3
--- /dev/null
+++ b/dpcpp/test/matrix/coo_kernels.cpp
@@ -0,0 +1,405 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/matrix/coo.hpp>
+
+
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+
+
+#include "core/matrix/coo_kernels.hpp"
+#include "core/test/utils.hpp"
+#include "core/test/utils/unsort_matrix.hpp"
+
+
+namespace {
+
+
+class Coo : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Coo<>;
+    using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
+
+    Coo() : rand_engine(42) {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        dpcpp = gko::DpcppExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (dpcpp != nullptr) {
+            ASSERT_NO_THROW(dpcpp->synchronize());
+        }
+    }
+
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<MtxType>(
+            num_rows, num_cols, std::uniform_int_distribution<>(1, num_cols),
+            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    void set_up_apply_data(int num_vectors = 1)
+    {
+        mtx = Mtx::create(ref);
+        mtx->copy_from(gen_mtx(532, 231));
+        expected = gen_mtx(532, num_vectors);
+        y = gen_mtx(231, num_vectors);
+        alpha = gko::initialize<Vec>({2.0}, ref);
+        beta = gko::initialize<Vec>({-1.0}, ref);
+        dmtx = Mtx::create(dpcpp);
+        dmtx->copy_from(mtx.get());
+        dresult = Vec::create(dpcpp);
+        dresult->copy_from(expected.get());
+        dy = Vec::create(dpcpp);
+        dy->copy_from(y.get());
+        dalpha = Vec::create(dpcpp);
+        dalpha->copy_from(alpha.get());
+        dbeta = Vec::create(dpcpp);
+        dbeta->copy_from(beta.get());
+    }
+
+    void unsort_mtx()
+    {
+        gko::test::unsort_matrix(mtx.get(), rand_engine);
+        dmtx->copy_from(mtx.get());
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
+
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Mtx> mtx;
+    std::unique_ptr<Vec> expected;
+    std::unique_ptr<Vec> y;
+    std::unique_ptr<Vec> alpha;
+    std::unique_ptr<Vec> beta;
+
+    std::unique_ptr<Mtx> dmtx;
+    std::unique_ptr<Vec> dresult;
+    std::unique_ptr<Vec> dy;
+    std::unique_ptr<Vec> dalpha;
+    std::unique_ptr<Vec> dbeta;
+};
+
+
+TEST_F(Coo, SimpleApplyIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Coo, SimpleApplyDoesntOverwritePadding)
+{
+    set_up_apply_data();
+    auto dresult_padded =
+        Vec::create(dpcpp, dresult->get_size(), dresult->get_stride() + 1);
+    dresult_padded->copy_from(dresult.get());
+    double padding_val{1234.0};
+    dpcpp->copy_from(dpcpp->get_master().get(), 1, &padding_val,
+                     dresult_padded->get_values() + 1);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult_padded.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult_padded, expected, 1e-14);
+    ASSERT_EQ(dpcpp->copy_val_to_host(dresult_padded->get_values() + 1),
+              1234.0);
+}
+
+
+TEST_F(Coo, SimpleApplyIsEquivalentToRefUnsorted)
+{
+    set_up_apply_data();
+    unsort_mtx();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Coo, AdvancedApplyIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Coo, AdvancedApplyDoesntOverwritePadding)
+{
+    set_up_apply_data();
+    auto dresult_padded =
+        Vec::create(dpcpp, dresult->get_size(), dresult->get_stride() + 1);
+    dresult_padded->copy_from(dresult.get());
+    double padding_val{1234.0};
+    dpcpp->copy_from(dpcpp->get_master().get(), 1, &padding_val,
+                     dresult_padded->get_values() + 1);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult_padded.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult_padded, expected, 1e-14);
+    ASSERT_EQ(dpcpp->copy_val_to_host(dresult_padded->get_values() + 1),
+              1234.0);
+}
+
+
+TEST_F(Coo, SimpleApplyAddIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->apply2(y.get(), expected.get());
+    dmtx->apply2(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Coo, AdvancedApplyAddIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->apply2(alpha.get(), y.get(), expected.get());
+    dmtx->apply2(dalpha.get(), dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Coo, SimpleApplyToDenseMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(3);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Coo, AdvancedApplyToDenseMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(3);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Coo, SimpleApplyAddToDenseMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(3);
+
+    mtx->apply2(y.get(), expected.get());
+    dmtx->apply2(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Coo, SimpleApplyAddToLargeDenseMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(33);
+
+    mtx->apply2(y.get(), expected.get());
+    dmtx->apply2(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Coo, AdvancedApplyAddToDenseMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(3);
+
+    mtx->apply2(alpha.get(), y.get(), expected.get());
+    dmtx->apply2(dalpha.get(), dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Coo, AdvancedApplyAddToLargeDenseMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(33);
+
+    mtx->apply2(y.get(), expected.get());
+    dmtx->apply2(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Coo, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+
+TEST_F(Coo, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+
+TEST_F(Coo, ApplyAddToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply2(alpha.get(), complex_b.get(), complex_x.get());
+    dmtx->apply2(dalpha.get(), dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+
+TEST_F(Coo, ConvertToDenseIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto dense_mtx = gko::matrix::Dense<>::create(ref);
+    auto ddense_mtx = gko::matrix::Dense<>::create(dpcpp);
+
+    mtx->convert_to(dense_mtx.get());
+    dmtx->convert_to(ddense_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Coo, ConvertToCsrIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto dense_mtx = gko::matrix::Dense<>::create(ref);
+    auto csr_mtx = gko::matrix::Csr<>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<>::create(dpcpp);
+
+    mtx->convert_to(dense_mtx.get());
+    dense_mtx->convert_to(csr_mtx.get());
+    dmtx->convert_to(dcsr_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Coo, ExtractDiagonalIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto diag = mtx->extract_diagonal();
+    auto ddiag = dmtx->extract_diagonal();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
+TEST_F(Coo, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
+
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
+}
+
+
+TEST_F(Coo, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+}
+
+
+}  // namespace
diff --git a/dpcpp/test/matrix/csr_kernels.cpp b/dpcpp/test/matrix/csr_kernels.cpp
index 3c76f2760b1..3270725fbe4 100644
--- a/dpcpp/test/matrix/csr_kernels.cpp
+++ b/dpcpp/test/matrix/csr_kernels.cpp
@@ -30,13 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <ginkgo/core/matrix/csr.hpp>
-
-
-#include <algorithm>
-#include <numeric>
 #include <random>
-#include <vector>
 
 
 #include <gtest/gtest.h>
@@ -44,10 +38,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/coo.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+#include <ginkgo/core/matrix/ell.hpp>
+#include <ginkgo/core/matrix/hybrid.hpp>
 #include <ginkgo/core/matrix/identity.hpp>
+#include <ginkgo/core/matrix/sellp.hpp>
+#include <ginkgo/core/matrix/sparsity_csr.hpp>
 
 
+#include "core/matrix/csr_kernels.hpp"
 #include "core/test/utils.hpp"
 #include "core/test/utils/unsort_matrix.hpp"
 
@@ -57,21 +58,24 @@ namespace {
 
 class Csr : public ::testing::Test {
 protected:
-#if GINKGO_DPCPP_SINGLE_MODE
-    using value_type = float;
+    using Arr = gko::Array<int>;
+    using Vec = gko::matrix::Dense<>;
+    using Mtx = gko::matrix::Csr<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
+    using ComplexMtx = gko::matrix::Csr<std::complex<double>>;
+
+    Csr()
+#ifdef GINKGO_FAST_TESTS
+        : mtx_size(152, 231),
 #else
-    using value_type = double;
+        : mtx_size(532, 231),
 #endif
-    using Arr = gko::Array<int>;
-    using Mtx = gko::matrix::Csr<value_type>;
-    using Vec = gko::matrix::Dense<value_type>;
-    using ComplexVec = gko::matrix::Dense<std::complex<value_type>>;
-    using ComplexMtx = gko::matrix::Csr<std::complex<value_type>>;
-
-    Csr() : mtx_size(532, 231), rand_engine(42) {}
+          rand_engine(42)
+    {}
 
     void SetUp()
     {
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
         ref = gko::ReferenceExecutor::create();
         dpcpp = gko::DpcppExecutor::create(0, ref);
     }
@@ -85,60 +89,63 @@ class Csr : public ::testing::Test {
 
     template <typename MtxType>
     std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
-                                     int min_nnz_row, int max_nnz_row)
+                                     int min_nnz_row)
     {
         return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols,
-            std::uniform_int_distribution<>(min_nnz_row, max_nnz_row),
-            std::normal_distribution<value_type>(-1.0, 1.0), rand_engine, ref);
+            std::uniform_int_distribution<>(min_nnz_row, num_cols),
+            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
     }
 
-    template <typename MtxType>
-    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
-                                     int min_nnz_row)
+    void set_up_apply_data(std::shared_ptr<Mtx::strategy_type> strategy,
+                           int num_vectors = 1)
     {
-        return gen_mtx<MtxType>(num_rows, num_cols, min_nnz_row, num_cols);
-    }
-
-    void set_up_apply_data(int num_vectors = 1)
-    {
-        mtx = Mtx::create(ref);
+        mtx = Mtx::create(ref, strategy);
         mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[1], 1));
-        complex_mtx = ComplexMtx::create(ref);
-        complex_mtx->copy_from(
-            gen_mtx<ComplexVec>(mtx_size[0], mtx_size[1], 1));
-        square_mtx = Mtx::create(ref);
+        square_mtx = Mtx::create(ref, strategy);
         square_mtx->copy_from(gen_mtx<Vec>(mtx_size[0], mtx_size[0], 1));
+        expected = gen_mtx<Vec>(mtx_size[0], num_vectors, 1);
+        y = gen_mtx<Vec>(mtx_size[1], num_vectors, 1);
         alpha = gko::initialize<Vec>({2.0}, ref);
         beta = gko::initialize<Vec>({-1.0}, ref);
-        dmtx = Mtx::create(dpcpp);
+        dmtx = Mtx::create(dpcpp, strategy);
         dmtx->copy_from(mtx.get());
-        complex_dmtx = ComplexMtx::create(dpcpp);
-        complex_dmtx->copy_from(complex_mtx.get());
-        square_dmtx = Mtx::create(dpcpp);
+        square_dmtx = Mtx::create(dpcpp, strategy);
         square_dmtx->copy_from(square_mtx.get());
+        dresult = Vec::create(dpcpp);
+        dresult->copy_from(expected.get());
+        dy = Vec::create(dpcpp);
+        dy->copy_from(y.get());
         dalpha = Vec::create(dpcpp);
         dalpha->copy_from(alpha.get());
         dbeta = Vec::create(dpcpp);
         dbeta->copy_from(beta.get());
-    }
 
-    struct matrix_pair {
-        std::unique_ptr<Mtx> ref;
-        std::unique_ptr<Mtx> dpcpp;
-    };
+        std::vector<int> tmp(mtx->get_size()[0], 0);
+        auto rng = std::default_random_engine{};
+        std::iota(tmp.begin(), tmp.end(), 0);
+        std::shuffle(tmp.begin(), tmp.end(), rng);
+        std::vector<int> tmp2(mtx->get_size()[1], 0);
+        std::iota(tmp2.begin(), tmp2.end(), 0);
+        std::shuffle(tmp2.begin(), tmp2.end(), rng);
+        rpermute_idxs = std::make_unique<Arr>(ref, tmp.begin(), tmp.end());
+        cpermute_idxs = std::make_unique<Arr>(ref, tmp2.begin(), tmp2.end());
+    }
 
-    matrix_pair gen_unsorted_mtx()
+    void set_up_apply_complex_data(
+        std::shared_ptr<ComplexMtx::strategy_type> strategy)
     {
-        constexpr int min_nnz_per_row{2};
-        auto local_mtx_ref =
-            gen_mtx<Mtx>(mtx_size[0], mtx_size[1], min_nnz_per_row);
-        gko::test::unsort_matrix(gko::lend(local_mtx_ref), rand_engine);
-
-        auto local_mtx_dpcpp = Mtx::create(dpcpp);
-        local_mtx_dpcpp->copy_from(local_mtx_ref.get());
+        complex_mtx = ComplexMtx::create(ref, strategy);
+        complex_mtx->copy_from(
+            gen_mtx<ComplexVec>(mtx_size[0], mtx_size[1], 1));
+        complex_dmtx = ComplexMtx::create(dpcpp, strategy);
+        complex_dmtx->copy_from(complex_mtx.get());
+    }
 
-        return {std::move(local_mtx_ref), std::move(local_mtx_dpcpp)};
+    void unsort_mtx()
+    {
+        gko::test::unsort_matrix(mtx.get(), rand_engine);
+        dmtx->copy_from(mtx.get());
     }
 
     std::shared_ptr<gko::ReferenceExecutor> ref;
@@ -150,104 +157,295 @@ class Csr : public ::testing::Test {
     std::unique_ptr<Mtx> mtx;
     std::unique_ptr<ComplexMtx> complex_mtx;
     std::unique_ptr<Mtx> square_mtx;
+    std::unique_ptr<Vec> expected;
+    std::unique_ptr<Vec> y;
     std::unique_ptr<Vec> alpha;
     std::unique_ptr<Vec> beta;
 
     std::unique_ptr<Mtx> dmtx;
     std::unique_ptr<ComplexMtx> complex_dmtx;
     std::unique_ptr<Mtx> square_dmtx;
+    std::unique_ptr<Vec> dresult;
+    std::unique_ptr<Vec> dy;
     std::unique_ptr<Vec> dalpha;
     std::unique_ptr<Vec> dbeta;
+    std::unique_ptr<Arr> rpermute_idxs;
+    std::unique_ptr<Arr> cpermute_idxs;
 };
 
 
-TEST_F(Csr, AdvancedApplyToCsrMatrixIsEquivalentToRef)
+TEST_F(Csr, StrategyAfterCopyIsEquivalentToRef)
 {
-    set_up_apply_data();
-    auto trans = mtx->transpose();
-    auto d_trans = gko::clone(dpcpp, trans);
+    set_up_apply_data(std::make_shared<Mtx::load_balance>(dpcpp));
 
-    mtx->apply(alpha.get(), trans.get(), beta.get(), square_mtx.get());
-    dmtx->apply(dalpha.get(), d_trans.get(), dbeta.get(), square_dmtx.get());
+    ASSERT_EQ(mtx->get_strategy()->get_name(),
+              dmtx->get_strategy()->get_name());
+}
 
-    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
-    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
+
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithLoadBalance)
+{
+    set_up_apply_data(std::make_shared<Mtx::load_balance>(dpcpp));
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
-TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRef)
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithLoadBalanceUnsorted)
 {
-    set_up_apply_data();
-    auto trans = mtx->transpose();
-    auto d_trans = gko::clone(dpcpp, trans);
+    set_up_apply_data(std::make_shared<Mtx::load_balance>(dpcpp));
+    unsort_mtx();
 
-    mtx->apply(trans.get(), square_mtx.get());
-    dmtx->apply(d_trans.get(), square_dmtx.get());
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
-    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithLoadBalance)
+{
+    set_up_apply_data(std::make_shared<Mtx::load_balance>(dpcpp));
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithCusparse)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithCusparseUnsorted)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    unsort_mtx();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithCusparse)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
-TEST_F(Csr, SimpleApplyToSparseCsrMatrixIsEquivalentToRef)
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithMergePath)
 {
-    set_up_apply_data();
-    auto mtx2 =
-        gen_mtx<Mtx>(mtx->get_size()[1], square_mtx->get_size()[1], 0, 10);
-    auto dmtx2 = Mtx::create(dpcpp, mtx2->get_size());
-    dmtx2->copy_from(mtx2.get());
+    set_up_apply_data(std::make_shared<Mtx::merge_path>());
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithMergePathUnsorted)
+{
+    set_up_apply_data(std::make_shared<Mtx::merge_path>());
+    unsort_mtx();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
 
-    mtx->apply(mtx2.get(), square_mtx.get());
-    dmtx->apply(dmtx2.get(), square_dmtx.get());
 
+TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithMergePath)
+{
+    set_up_apply_data(std::make_shared<Mtx::merge_path>());
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithClassical)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithClassicalUnsorted)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+    unsort_mtx();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithClassical)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithAutomatical)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>(dpcpp));
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, SimpleApplyToDenseMatrixIsEquivalentToRefWithLoadBalance)
+{
+    set_up_apply_data(std::make_shared<Mtx::load_balance>(dpcpp), 3);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithLoadBalance)
+{
+    set_up_apply_data(std::make_shared<Mtx::load_balance>(dpcpp), 3);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, SimpleApplyToDenseMatrixIsEquivalentToRefWithClassical)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>(), 3);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithClassical)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>(), 3);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, SimpleApplyToDenseMatrixIsEquivalentToRefWithMergePath)
+{
+    set_up_apply_data(std::make_shared<Mtx::merge_path>(), 3);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithMergePath)
+{
+    set_up_apply_data(std::make_shared<Mtx::merge_path>(), 3);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Csr, AdvancedApplyToCsrMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>());
+    auto trans = mtx->transpose();
+    auto d_trans = dmtx->transpose();
+
+    mtx->apply(alpha.get(), trans.get(), beta.get(), square_mtx.get());
+    dmtx->apply(dalpha.get(), d_trans.get(), dbeta.get(), square_dmtx.get());
+
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
     GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
     ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
 }
 
 
-TEST_F(Csr, SimpleApplySparseToSparseCsrMatrixIsEquivalentToRef)
+TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRef)
 {
-    set_up_apply_data();
-    auto mtx1 = gen_mtx<Mtx>(mtx->get_size()[0], mtx->get_size()[1], 0, 10);
-    auto mtx2 =
-        gen_mtx<Mtx>(mtx->get_size()[1], square_mtx->get_size()[1], 0, 10);
-    auto dmtx1 = Mtx::create(dpcpp, mtx1->get_size());
-    auto dmtx2 = Mtx::create(dpcpp, mtx2->get_size());
-    dmtx1->copy_from(mtx1.get());
-    dmtx2->copy_from(mtx2.get());
+    set_up_apply_data(std::make_shared<Mtx::automatical>());
+    auto trans = mtx->transpose();
+    auto d_trans = dmtx->transpose();
 
-    mtx1->apply(mtx2.get(), square_mtx.get());
-    dmtx1->apply(dmtx2.get(), square_dmtx.get());
+    mtx->apply(trans.get(), square_mtx.get());
+    dmtx->apply(d_trans.get(), square_dmtx.get());
 
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
     GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
     ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
 }
 
 
-TEST_F(Csr, SimpleApplyToEmptyCsrMatrixIsEquivalentToRef)
+TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRefUnsorted)
 {
-    set_up_apply_data();
-    auto mtx2 =
-        gen_mtx<Mtx>(mtx->get_size()[1], square_mtx->get_size()[1], 0, 0);
-    auto dmtx2 = Mtx::create(dpcpp, mtx2->get_size());
-    dmtx2->copy_from(mtx2.get());
+    set_up_apply_data(std::make_shared<Mtx::automatical>());
+    auto trans = mtx->transpose();
+    auto d_trans = dmtx->transpose();
+    gko::test::unsort_matrix(static_cast<Mtx *>(dmtx.get()), rand_engine);
+    gko::test::unsort_matrix(static_cast<Mtx *>(d_trans.get()), rand_engine);
 
-    mtx->apply(mtx2.get(), square_mtx.get());
-    dmtx->apply(dmtx2.get(), square_dmtx.get());
+    mtx->apply(trans.get(), square_mtx.get());
+    dmtx->apply(d_trans.get(), square_dmtx.get());
 
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
     GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<value_type>::value);
     ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
 }
 
 
 TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_data(std::make_shared<Mtx::automatical>());
     auto a = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
     auto b = gen_mtx<Mtx>(mtx_size[0], mtx_size[1], 0);
     auto da = Mtx::create(dpcpp);
@@ -261,15 +459,49 @@ TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
     a->apply(alpha.get(), id.get(), beta.get(), b.get());
     da->apply(dalpha.get(), did.get(), dbeta.get(), db.get());
 
-    GKO_ASSERT_MTX_NEAR(b, db, r<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(b, db, 1e-14);
     GKO_ASSERT_MTX_EQ_SPARSITY(b, db);
     ASSERT_TRUE(db->is_sorted_by_column_index());
 }
 
 
+TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>());
+    auto complex_b = gen_mtx<ComplexVec>(this->mtx_size[1], 3, 1);
+    auto dcomplex_b = ComplexVec::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(this->mtx_size[0], 3, 1);
+    auto dcomplex_x = ComplexVec::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+
+TEST_F(Csr, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>());
+    auto complex_b = gen_mtx<ComplexVec>(this->mtx_size[1], 3, 1);
+    auto dcomplex_b = ComplexVec::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(this->mtx_size[0], 3, 1);
+    auto dcomplex_x = ComplexVec::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+
 TEST_F(Csr, TransposeIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_data(std::make_shared<Mtx::automatical>(dpcpp));
 
     auto trans = gko::as<Mtx>(mtx->transpose());
     auto d_trans = gko::as<Mtx>(dmtx->transpose());
@@ -281,7 +513,7 @@ TEST_F(Csr, TransposeIsEquivalentToRef)
 
 TEST_F(Csr, ConjugateTransposeIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(dpcpp));
 
     auto trans = gko::as<ComplexMtx>(complex_mtx->conj_transpose());
     auto d_trans = gko::as<ComplexMtx>(complex_dmtx->conj_transpose());
@@ -291,9 +523,308 @@ TEST_F(Csr, ConjugateTransposeIsEquivalentToRef)
 }
 
 
+TEST_F(Csr, ConvertToDenseIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    auto dense_mtx = gko::matrix::Dense<>::create(ref);
+    auto ddense_mtx = gko::matrix::Dense<>::create(dpcpp);
+
+    mtx->convert_to(dense_mtx.get());
+    dmtx->convert_to(ddense_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, MoveToDenseIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    auto dense_mtx = gko::matrix::Dense<>::create(ref);
+    auto ddense_mtx = gko::matrix::Dense<>::create(dpcpp);
+
+    mtx->move_to(dense_mtx.get());
+    dmtx->move_to(ddense_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, ConvertToEllIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    auto ell_mtx = gko::matrix::Ell<>::create(ref);
+    auto dell_mtx = gko::matrix::Ell<>::create(dpcpp);
+
+    mtx->convert_to(ell_mtx.get());
+    dmtx->convert_to(dell_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(ell_mtx.get(), dell_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, MoveToEllIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    auto ell_mtx = gko::matrix::Ell<>::create(ref);
+    auto dell_mtx = gko::matrix::Ell<>::create(dpcpp);
+
+    mtx->move_to(ell_mtx.get());
+    dmtx->move_to(dell_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(ell_mtx.get(), dell_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, ConvertToSparsityCsrIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    auto sparsity_mtx = gko::matrix::SparsityCsr<>::create(ref);
+    auto d_sparsity_mtx = gko::matrix::SparsityCsr<>::create(dpcpp);
+
+    mtx->convert_to(sparsity_mtx.get());
+    dmtx->convert_to(d_sparsity_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(sparsity_mtx.get(), d_sparsity_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, MoveToSparsityCsrIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    auto sparsity_mtx = gko::matrix::SparsityCsr<>::create(ref);
+    auto d_sparsity_mtx = gko::matrix::SparsityCsr<>::create(dpcpp);
+
+    mtx->move_to(sparsity_mtx.get());
+    dmtx->move_to(d_sparsity_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(sparsity_mtx.get(), d_sparsity_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, CalculateMaxNnzPerRowIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    gko::size_type max_nnz_per_row;
+    gko::size_type dmax_nnz_per_row;
+
+    gko::kernels::reference::csr::calculate_max_nnz_per_row(ref, mtx.get(),
+                                                            &max_nnz_per_row);
+    gko::kernels::dpcpp::csr::calculate_max_nnz_per_row(dpcpp, dmtx.get(),
+                                                        &dmax_nnz_per_row);
+
+    ASSERT_EQ(max_nnz_per_row, dmax_nnz_per_row);
+}
+
+
+TEST_F(Csr, ConvertToCooIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    auto coo_mtx = gko::matrix::Coo<>::create(ref);
+    auto dcoo_mtx = gko::matrix::Coo<>::create(dpcpp);
+
+    mtx->convert_to(coo_mtx.get());
+    dmtx->convert_to(dcoo_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, MoveToCooIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    auto coo_mtx = gko::matrix::Coo<>::create(ref);
+    auto dcoo_mtx = gko::matrix::Coo<>::create(dpcpp);
+
+    mtx->move_to(coo_mtx.get());
+    dmtx->move_to(dcoo_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, ConvertToSellpIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
+    auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+
+    mtx->convert_to(sellp_mtx.get());
+    dmtx->convert_to(dsellp_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(sellp_mtx.get(), dsellp_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, MoveToSellpIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
+    auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+
+    mtx->move_to(sellp_mtx.get());
+    dmtx->move_to(dsellp_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(sellp_mtx.get(), dsellp_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, ConvertsEmptyToSellp)
+{
+    auto dempty_mtx = Mtx::create(dpcpp);
+    auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+
+    dempty_mtx->convert_to(dsellp_mtx.get());
+
+    ASSERT_EQ(dpcpp->copy_val_to_host(dsellp_mtx->get_const_slice_sets()), 0);
+    ASSERT_FALSE(dsellp_mtx->get_size());
+}
+
+
+TEST_F(Csr, CalculateTotalColsIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    gko::size_type total_cols;
+    gko::size_type dtotal_cols;
+
+    gko::kernels::reference::csr::calculate_total_cols(
+        ref, mtx.get(), &total_cols, 2, gko::matrix::default_slice_size);
+    gko::kernels::dpcpp::csr::calculate_total_cols(
+        dpcpp, dmtx.get(), &dtotal_cols, 2, gko::matrix::default_slice_size);
+
+    ASSERT_EQ(total_cols, dtotal_cols);
+}
+
+
+TEST_F(Csr, CalculatesNonzerosPerRow)
+{
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    gko::Array<gko::size_type> row_nnz(ref, mtx->get_size()[0]);
+    gko::Array<gko::size_type> drow_nnz(dpcpp, dmtx->get_size()[0]);
+
+    gko::kernels::reference::csr::calculate_nonzeros_per_row(ref, mtx.get(),
+                                                             &row_nnz);
+    gko::kernels::dpcpp::csr::calculate_nonzeros_per_row(dpcpp, dmtx.get(),
+                                                         &drow_nnz);
+
+    GKO_ASSERT_ARRAY_EQ(row_nnz, drow_nnz);
+}
+
+
+TEST_F(Csr, ConvertToHybridIsEquivalentToRef)
+{
+    using Hybrid_type = gko::matrix::Hybrid<>;
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    auto hybrid_mtx = Hybrid_type::create(
+        ref, std::make_shared<Hybrid_type::column_limit>(2));
+    auto dhybrid_mtx = Hybrid_type::create(
+        dpcpp, std::make_shared<Hybrid_type::column_limit>(2));
+
+    mtx->convert_to(hybrid_mtx.get());
+    dmtx->convert_to(dhybrid_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, MoveToHybridIsEquivalentToRef)
+{
+    using Hybrid_type = gko::matrix::Hybrid<>;
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    auto hybrid_mtx = Hybrid_type::create(
+        ref, std::make_shared<Hybrid_type::column_limit>(2));
+    auto dhybrid_mtx = Hybrid_type::create(
+        dpcpp, std::make_shared<Hybrid_type::column_limit>(2));
+
+    mtx->move_to(hybrid_mtx.get());
+    dmtx->move_to(dhybrid_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Csr, IsPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+
+    auto permuted = gko::as<Mtx>(square_mtx->permute(rpermute_idxs.get()));
+    auto dpermuted = gko::as<Mtx>(square_dmtx->permute(rpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(permuted, dpermuted);
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
+TEST_F(Csr, IsInversePermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+
+    auto permuted =
+        gko::as<Mtx>(square_mtx->inverse_permute(rpermute_idxs.get()));
+    auto dpermuted =
+        gko::as<Mtx>(square_dmtx->inverse_permute(rpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(permuted, dpermuted);
+    GKO_ASSERT_MTX_NEAR(permuted, dpermuted, 0);
+}
+
+
+TEST_F(Csr, IsRowPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+
+    auto r_permute = gko::as<Mtx>(mtx->row_permute(rpermute_idxs.get()));
+    auto dr_permute = gko::as<Mtx>(dmtx->row_permute(rpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(r_permute, dr_permute);
+    GKO_ASSERT_MTX_NEAR(r_permute, dr_permute, 0);
+}
+
+
+TEST_F(Csr, IsColPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+
+    auto c_permute = gko::as<Mtx>(mtx->column_permute(cpermute_idxs.get()));
+    auto dc_permute = gko::as<Mtx>(dmtx->column_permute(cpermute_idxs.get()));
+
+    ASSERT_TRUE(dc_permute->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_EQ_SPARSITY(c_permute, dc_permute);
+    GKO_ASSERT_MTX_NEAR(c_permute, dc_permute, 0);
+}
+
+
+TEST_F(Csr, IsInverseRowPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+
+    auto inverse_r_permute =
+        gko::as<Mtx>(mtx->inverse_row_permute(rpermute_idxs.get()));
+    auto d_inverse_r_permute =
+        gko::as<Mtx>(dmtx->inverse_row_permute(rpermute_idxs.get()));
+
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse_r_permute, d_inverse_r_permute);
+    GKO_ASSERT_MTX_NEAR(inverse_r_permute, d_inverse_r_permute, 0);
+}
+
+
+TEST_F(Csr, IsInverseColPermutable)
+{
+    set_up_apply_data(std::make_shared<Mtx::classical>());
+
+    auto inverse_c_permute =
+        gko::as<Mtx>(mtx->inverse_column_permute(cpermute_idxs.get()));
+    auto d_inverse_c_permute =
+        gko::as<Mtx>(dmtx->inverse_column_permute(cpermute_idxs.get()));
+
+    ASSERT_TRUE(d_inverse_c_permute->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_EQ_SPARSITY(inverse_c_permute, d_inverse_c_permute);
+    GKO_ASSERT_MTX_NEAR(inverse_c_permute, d_inverse_c_permute, 0);
+}
+
+
 TEST_F(Csr, RecognizeSortedMatrixIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_data(std::make_shared<Mtx::automatical>());
     bool is_sorted_dpcpp{};
     bool is_sorted_ref{};
 
@@ -306,12 +837,13 @@ TEST_F(Csr, RecognizeSortedMatrixIsEquivalentToRef)
 
 TEST_F(Csr, RecognizeUnsortedMatrixIsEquivalentToRef)
 {
-    auto uns_mtx = gen_unsorted_mtx();
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    unsort_mtx();
     bool is_sorted_dpcpp{};
     bool is_sorted_ref{};
 
-    is_sorted_ref = uns_mtx.ref->is_sorted_by_column_index();
-    is_sorted_dpcpp = uns_mtx.dpcpp->is_sorted_by_column_index();
+    is_sorted_ref = mtx->is_sorted_by_column_index();
+    is_sorted_dpcpp = dmtx->is_sorted_by_column_index();
 
     ASSERT_EQ(is_sorted_ref, is_sorted_dpcpp);
 }
@@ -319,29 +851,106 @@ TEST_F(Csr, RecognizeUnsortedMatrixIsEquivalentToRef)
 
 TEST_F(Csr, SortSortedMatrixIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_data(std::make_shared<Mtx::automatical>());
 
     mtx->sort_by_column_index();
     dmtx->sort_by_column_index();
 
     // Values must be unchanged, therefore, tolerance is `0`
     GKO_ASSERT_MTX_NEAR(mtx, dmtx, 0);
-    ASSERT_TRUE(mtx->is_sorted_by_column_index());
-    ASSERT_TRUE(dmtx->is_sorted_by_column_index());
 }
 
 
 TEST_F(Csr, SortUnsortedMatrixIsEquivalentToRef)
 {
-    auto uns_mtx = gen_unsorted_mtx();
+    set_up_apply_data(std::make_shared<Mtx::sparselib>());
+    unsort_mtx();
 
-    uns_mtx.ref->sort_by_column_index();
-    uns_mtx.dpcpp->sort_by_column_index();
+    mtx->sort_by_column_index();
+    dmtx->sort_by_column_index();
 
     // Values must be unchanged, therefore, tolerance is `0`
-    GKO_ASSERT_MTX_NEAR(uns_mtx.ref, uns_mtx.dpcpp, 0);
-    ASSERT_TRUE(uns_mtx.ref->is_sorted_by_column_index());
-    ASSERT_TRUE(uns_mtx.dpcpp->is_sorted_by_column_index());
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 0);
+}
+
+
+TEST_F(Csr, OneAutomaticalWorksWithDifferentMatrices)
+{
+    auto automatical = std::make_shared<Mtx::automatical>();
+    auto row_len_limit = std::max(automatical->nvidia_row_len_limit,
+                                  automatical->amd_row_len_limit);
+    auto load_balance_mtx = Mtx::create(ref);
+    auto classical_mtx = Mtx::create(ref);
+    load_balance_mtx->copy_from(
+        gen_mtx<Vec>(1, row_len_limit + 1000, row_len_limit + 1));
+    classical_mtx->copy_from(gen_mtx<Vec>(50, 50, 1));
+    auto load_balance_mtx_d = Mtx::create(dpcpp);
+    auto classical_mtx_d = Mtx::create(dpcpp);
+    load_balance_mtx_d->copy_from(load_balance_mtx.get());
+    classical_mtx_d->copy_from(classical_mtx.get());
+
+    load_balance_mtx_d->set_strategy(automatical);
+    classical_mtx_d->set_strategy(automatical);
+
+    EXPECT_EQ("load_balance", load_balance_mtx_d->get_strategy()->get_name());
+    EXPECT_EQ("classical", classical_mtx_d->get_strategy()->get_name());
+    ASSERT_NE(load_balance_mtx_d->get_strategy().get(),
+              classical_mtx_d->get_strategy().get());
+}
+
+
+TEST_F(Csr, ExtractDiagonalIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>());
+
+    auto diag = mtx->extract_diagonal();
+    auto ddiag = dmtx->extract_diagonal();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
+TEST_F(Csr, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>(dpcpp));
+
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
+
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
+}
+
+
+TEST_F(Csr, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(std::make_shared<Mtx::automatical>(dpcpp));
+
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+}
+
+
+TEST_F(Csr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
+{
+    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(dpcpp));
+
+    complex_mtx->compute_absolute_inplace();
+    complex_dmtx->compute_absolute_inplace();
+
+    GKO_ASSERT_MTX_NEAR(complex_mtx, complex_dmtx, 1e-14);
+}
+
+
+TEST_F(Csr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
+{
+    set_up_apply_complex_data(std::make_shared<ComplexMtx::automatical>(dpcpp));
+
+    auto abs_mtx = complex_mtx->compute_absolute();
+    auto dabs_mtx = complex_dmtx->compute_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
 }
 
 
diff --git a/dpcpp/test/matrix/diagonal_kernels.cpp b/dpcpp/test/matrix/diagonal_kernels.cpp
new file mode 100644
index 00000000000..fc041aea4e1
--- /dev/null
+++ b/dpcpp/test/matrix/diagonal_kernels.cpp
@@ -0,0 +1,278 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include <vector>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+
+
+#include "core/matrix/diagonal_kernels.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class Diagonal : public ::testing::Test {
+protected:
+    using ValueType = double;
+    using ComplexValueType = std::complex<double>;
+    using Csr = gko::matrix::Csr<ValueType>;
+    using Diag = gko::matrix::Diagonal<ValueType>;
+    using Dense = gko::matrix::Dense<ValueType>;
+    using ComplexDiag = gko::matrix::Diagonal<ComplexValueType>;
+
+    Diagonal()
+#ifdef GINKGO_FAST_TESTS
+        : mtx_size(152, 231),
+#else
+        : mtx_size(532, 231),
+#endif
+          rand_engine(42)
+    {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        dpcpp = gko::DpcppExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (dpcpp != nullptr) {
+            ASSERT_NO_THROW(dpcpp->synchronize());
+        }
+    }
+
+    template <typename MtxType>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row)
+    {
+        return gko::test::generate_random_matrix<MtxType>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(min_nnz_row, num_cols),
+            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
+    }
+
+    std::unique_ptr<Diag> gen_diag(int size)
+    {
+        auto diag = Diag::create(ref, size);
+        auto vals = diag->get_values();
+        auto value_dist = std::normal_distribution<>(0.0, 1.0);
+        for (int i = 0; i < size; i++) {
+            vals[i] = gko::test::detail::get_rand_value<ValueType>(value_dist,
+                                                                   rand_engine);
+        }
+        return diag;
+    }
+
+    std::unique_ptr<ComplexDiag> gen_cdiag(int size)
+    {
+        auto cdiag = ComplexDiag::create(ref, size);
+        auto vals = cdiag->get_values();
+        auto value_dist = std::normal_distribution<>(0.0, 1.0);
+        for (int i = 0; i < size; i++) {
+            vals[i] = ComplexValueType{
+                gko::test::detail::get_rand_value<ComplexValueType>(
+                    value_dist, rand_engine)};
+        }
+        return cdiag;
+    }
+
+    void set_up_apply_data()
+    {
+        diag = gen_diag(mtx_size[0]);
+        ddiag = Diag::create(dpcpp);
+        ddiag->copy_from(diag.get());
+        dense1 = gen_mtx<Dense>(mtx_size[0], mtx_size[1], mtx_size[1]);
+        dense2 = gen_mtx<Dense>(mtx_size[1], mtx_size[0], mtx_size[0]);
+        denseexpected1 = gen_mtx<Dense>(mtx_size[0], mtx_size[1], mtx_size[1]);
+        denseexpected2 = gen_mtx<Dense>(mtx_size[1], mtx_size[0], mtx_size[0]);
+        ddense1 = Dense::create(dpcpp);
+        ddense1->copy_from(dense1.get());
+        ddense2 = Dense::create(dpcpp);
+        ddense2->copy_from(dense2.get());
+        denseresult1 = Dense::create(dpcpp);
+        denseresult1->copy_from(denseexpected1.get());
+        denseresult2 = Dense::create(dpcpp);
+        denseresult2->copy_from(denseexpected2.get());
+        csr1 = gen_mtx<Csr>(mtx_size[0], mtx_size[1], 1);
+        csr2 = gen_mtx<Csr>(mtx_size[1], mtx_size[0], 1);
+        csrexpected1 = gen_mtx<Csr>(mtx_size[0], mtx_size[1], 1);
+        csrexpected2 = gen_mtx<Csr>(mtx_size[1], mtx_size[0], 1);
+        dcsr1 = Csr::create(dpcpp);
+        dcsr1->copy_from(csr1.get());
+        dcsr2 = Csr::create(dpcpp);
+        dcsr2->copy_from(csr2.get());
+        csrresult1 = Csr::create(dpcpp);
+        csrresult1->copy_from(csrexpected1.get());
+        csrresult2 = Csr::create(dpcpp);
+        csrresult2->copy_from(csrexpected2.get());
+    }
+
+    void set_up_complex_data()
+    {
+        cdiag = gen_cdiag(mtx_size[0]);
+        dcdiag = ComplexDiag::create(dpcpp);
+        dcdiag->copy_from(cdiag.get());
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
+
+    const gko::dim<2> mtx_size;
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Diag> diag;
+    std::unique_ptr<Diag> ddiag;
+    std::unique_ptr<ComplexDiag> cdiag;
+    std::unique_ptr<ComplexDiag> dcdiag;
+
+    std::unique_ptr<Dense> dense1;
+    std::unique_ptr<Dense> dense2;
+    std::unique_ptr<Dense> denseexpected1;
+    std::unique_ptr<Dense> denseexpected2;
+    std::unique_ptr<Dense> denseresult1;
+    std::unique_ptr<Dense> denseresult2;
+    std::unique_ptr<Dense> ddense1;
+    std::unique_ptr<Dense> ddense2;
+    std::unique_ptr<Csr> csr1;
+    std::unique_ptr<Csr> csr2;
+    std::unique_ptr<Csr> dcsr1;
+    std::unique_ptr<Csr> dcsr2;
+    std::unique_ptr<Csr> csrexpected1;
+    std::unique_ptr<Csr> csrexpected2;
+    std::unique_ptr<Csr> csrresult1;
+    std::unique_ptr<Csr> csrresult2;
+};
+
+
+TEST_F(Diagonal, ApplyToDenseIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    diag->apply(dense1.get(), denseexpected1.get());
+    ddiag->apply(ddense1.get(), denseresult1.get());
+
+    GKO_ASSERT_MTX_NEAR(denseexpected1, denseresult1, 1e-14);
+}
+
+
+TEST_F(Diagonal, RightApplyToDenseIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    diag->rapply(dense2.get(), denseexpected2.get());
+    ddiag->rapply(ddense2.get(), denseresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(denseexpected2, denseresult2, 1e-14);
+}
+
+
+TEST_F(Diagonal, ApplyToCsrIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    diag->apply(csr1.get(), csrexpected1.get());
+    ddiag->apply(dcsr1.get(), csrresult1.get());
+
+    GKO_ASSERT_MTX_NEAR(csrexpected1, csrresult1, 1e-14);
+}
+
+
+TEST_F(Diagonal, RightApplyToCsrIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    diag->rapply(csr2.get(), csrexpected2.get());
+    ddiag->rapply(dcsr2.get(), csrresult2.get());
+
+    GKO_ASSERT_MTX_NEAR(csrexpected2, csrresult2, 1e-14);
+}
+
+
+TEST_F(Diagonal, ConvertToCsrIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    diag->convert_to(csr1.get());
+    ddiag->convert_to(dcsr1.get());
+
+    GKO_ASSERT_MTX_NEAR(csr1, dcsr1, 0);
+}
+
+
+TEST_F(Diagonal, ConjTransposeIsEquivalentToRef)
+{
+    set_up_complex_data();
+
+    auto trans = cdiag->conj_transpose();
+    auto trans_diag = static_cast<ComplexDiag *>(trans.get());
+    auto dtrans = dcdiag->conj_transpose();
+    auto dtrans_diag = static_cast<ComplexDiag *>(dtrans.get());
+
+    GKO_ASSERT_MTX_NEAR(trans_diag, dtrans_diag, 0);
+}
+
+
+TEST_F(Diagonal, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    diag->compute_absolute_inplace();
+    ddiag->compute_absolute_inplace();
+
+    GKO_ASSERT_MTX_NEAR(diag, ddiag, 1e-14);
+}
+
+
+TEST_F(Diagonal, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto abs_diag = diag->compute_absolute();
+    auto dabs_diag = ddiag->compute_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_diag, dabs_diag, 1e-14);
+}
+
+
+}  // namespace
diff --git a/dpcpp/test/matrix/fbcsr_kernels.cpp b/dpcpp/test/matrix/fbcsr_kernels.cpp
new file mode 100644
index 00000000000..d64fe1c075b
--- /dev/null
+++ b/dpcpp/test/matrix/fbcsr_kernels.cpp
@@ -0,0 +1,90 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include "core/test/matrix/fbcsr_sample.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class Fbcsr : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Fbcsr<>;
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        dpcpp = gko::DpcppExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (dpcpp != nullptr) {
+            ASSERT_NO_THROW(dpcpp->synchronize());
+        }
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
+
+    std::unique_ptr<Mtx> mtx;
+};
+
+
+TEST_F(Fbcsr, CanWriteFromMatrixOnDevice)
+{
+    using value_type = Mtx::value_type;
+    using index_type = Mtx::index_type;
+    using MatData = gko::matrix_data<value_type, index_type>;
+    gko::testing::FbcsrSample<value_type, index_type> sample(ref);
+    auto refmat = sample.generate_fbcsr();
+    auto dpcppmat = Mtx::create(dpcpp);
+    dpcppmat->copy_from(gko::lend(refmat));
+
+    MatData refdata;
+    MatData dpcppdata;
+    refmat->write(refdata);
+    dpcppmat->write(dpcppdata);
+
+    ASSERT_TRUE(refdata.nonzeros == dpcppdata.nonzeros);
+}
+
+
+}  // namespace
diff --git a/dpcpp/test/matrix/hybrid_kernels.cpp b/dpcpp/test/matrix/hybrid_kernels.cpp
new file mode 100644
index 00000000000..04782587360
--- /dev/null
+++ b/dpcpp/test/matrix/hybrid_kernels.cpp
@@ -0,0 +1,297 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+
+
+#include "core/matrix/hybrid_kernels.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class Hybrid : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Hybrid<>;
+    using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
+
+    Hybrid() : rand_engine(42) {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        dpcpp = gko::DpcppExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (dpcpp != nullptr) {
+            ASSERT_NO_THROW(dpcpp->synchronize());
+        }
+    }
+
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols,
+                                     int min_nnz_row)
+    {
+        return gko::test::generate_random_matrix<MtxType>(
+            num_rows, num_cols,
+            std::uniform_int_distribution<>(min_nnz_row, num_cols),
+            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    void set_up_apply_data(int num_vectors = 1,
+                           std::shared_ptr<Mtx::strategy_type> strategy =
+                               std::make_shared<Mtx::automatic>())
+    {
+        mtx = Mtx::create(ref, strategy);
+        mtx->copy_from(gen_mtx(532, 231, 1));
+        expected = gen_mtx(532, num_vectors, 1);
+        y = gen_mtx(231, num_vectors, 1);
+        alpha = gko::initialize<Vec>({2.0}, ref);
+        beta = gko::initialize<Vec>({-1.0}, ref);
+        dmtx = Mtx::create(dpcpp, strategy);
+        dmtx->copy_from(mtx.get());
+        dresult = Vec::create(dpcpp);
+        dresult->copy_from(expected.get());
+        dy = Vec::create(dpcpp);
+        dy->copy_from(y.get());
+        dalpha = Vec::create(dpcpp);
+        dalpha->copy_from(alpha.get());
+        dbeta = Vec::create(dpcpp);
+        dbeta->copy_from(beta.get());
+    }
+
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
+
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Mtx> mtx;
+    std::unique_ptr<Vec> expected;
+    std::unique_ptr<Vec> y;
+    std::unique_ptr<Vec> alpha;
+    std::unique_ptr<Vec> beta;
+
+    std::unique_ptr<Mtx> dmtx;
+    std::unique_ptr<Vec> dresult;
+    std::unique_ptr<Vec> dy;
+    std::unique_ptr<Vec> dalpha;
+    std::unique_ptr<Vec> dbeta;
+};
+
+
+TEST_F(Hybrid, SubMatrixExecutorAfterCopyIsEquivalentToExcutor)
+{
+    set_up_apply_data();
+
+    auto coo_mtx = dmtx->get_coo();
+    auto ell_mtx = dmtx->get_ell();
+
+    ASSERT_EQ(coo_mtx->get_executor(), dpcpp);
+    ASSERT_EQ(ell_mtx->get_executor(), dpcpp);
+    ASSERT_EQ(dmtx->get_executor(), dpcpp);
+}
+
+
+TEST_F(Hybrid, SimpleApplyIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Hybrid, AdvancedApplyIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Hybrid, SimpleApplyToDenseMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(3);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Hybrid, AdvancedApplyToDenseMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(3);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+}
+
+
+TEST_F(Hybrid, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+
+TEST_F(Hybrid, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_data();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3, 1);
+    auto dcomplex_b = ComplexVec::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3, 1);
+    auto dcomplex_x = ComplexVec::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+
+TEST_F(Hybrid, CountNonzerosIsEquivalentToRef)
+{
+    set_up_apply_data();
+    gko::size_type nonzeros;
+    gko::size_type dnonzeros;
+
+    gko::kernels::reference::hybrid::count_nonzeros(ref, mtx.get(), &nonzeros);
+    gko::kernels::dpcpp::hybrid::count_nonzeros(dpcpp, dmtx.get(), &dnonzeros);
+
+    ASSERT_EQ(nonzeros, dnonzeros);
+}
+
+
+TEST_F(Hybrid, ConvertToCsrIsEquivalentToRef)
+{
+    set_up_apply_data(1, std::make_shared<Mtx::column_limit>(2));
+    auto csr_mtx = gko::matrix::Csr<>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<>::create(dpcpp);
+
+    mtx->convert_to(csr_mtx.get());
+    dmtx->convert_to(dcsr_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Hybrid, MoveToCsrIsEquivalentToRef)
+{
+    set_up_apply_data(1, std::make_shared<Mtx::column_limit>(2));
+    auto csr_mtx = gko::matrix::Csr<>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<>::create(dpcpp);
+
+    mtx->move_to(csr_mtx.get());
+    dmtx->move_to(dcsr_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Hybrid, ExtractDiagonalIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    auto diag = mtx->extract_diagonal();
+    auto ddiag = dmtx->extract_diagonal();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
+TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data();
+
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
+
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
+}
+
+
+TEST_F(Hybrid, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_data(1, std::make_shared<Mtx::column_limit>(2));
+    using AbsMtx = gko::remove_complex<Mtx>;
+
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
+    auto abs_strategy = gko::as<AbsMtx::column_limit>(abs_mtx->get_strategy());
+    auto dabs_strategy =
+        gko::as<AbsMtx::column_limit>(dabs_mtx->get_strategy());
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+    GKO_ASSERT_EQ(abs_strategy->get_num_columns(),
+                  dabs_strategy->get_num_columns());
+    GKO_ASSERT_EQ(abs_strategy->get_num_columns(), 2);
+}
+
+
+}  // namespace
diff --git a/dpcpp/test/matrix/sellp_kernels.cpp b/dpcpp/test/matrix/sellp_kernels.cpp
new file mode 100644
index 00000000000..ee8be3091fa
--- /dev/null
+++ b/dpcpp/test/matrix/sellp_kernels.cpp
@@ -0,0 +1,407 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <random>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+#include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
+#include <ginkgo/core/matrix/dense.hpp>
+#include <ginkgo/core/matrix/diagonal.hpp>
+
+
+#include "core/matrix/sellp_kernels.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class Sellp : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Sellp<>;
+    using Vec = gko::matrix::Dense<>;
+    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
+
+    Sellp() : rand_engine(42) {}
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        dpcpp = gko::DpcppExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (dpcpp != nullptr) {
+            ASSERT_NO_THROW(dpcpp->synchronize());
+        }
+    }
+
+    template <typename MtxType = Vec>
+    std::unique_ptr<MtxType> gen_mtx(int num_rows, int num_cols)
+    {
+        return gko::test::generate_random_matrix<MtxType>(
+            num_rows, num_cols, std::uniform_int_distribution<>(1, num_cols),
+            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+    }
+
+    void set_up_apply_vector(
+        int slice_size = gko::matrix::default_slice_size,
+        int stride_factor = gko::matrix::default_stride_factor,
+        int total_cols = 0)
+    {
+        mtx = Mtx::create(ref);
+        mtx->copy_from(gen_mtx(532, 231));
+        expected = gen_mtx(532, 1);
+        y = gen_mtx(231, 1);
+        alpha = gko::initialize<Vec>({2.0}, ref);
+        beta = gko::initialize<Vec>({-1.0}, ref);
+        dmtx = Mtx::create(dpcpp);
+        dmtx->copy_from(mtx.get());
+        dresult = Vec::create(dpcpp);
+        dresult->copy_from(expected.get());
+        dy = Vec::create(dpcpp);
+        dy->copy_from(y.get());
+        dalpha = Vec::create(dpcpp);
+        dalpha->copy_from(alpha.get());
+        dbeta = Vec::create(dpcpp);
+        dbeta->copy_from(beta.get());
+    }
+
+    void set_up_apply_matrix(
+        int slice_size = gko::matrix::default_slice_size,
+        int stride_factor = gko::matrix::default_stride_factor,
+        int total_cols = 0)
+    {
+        mtx = Mtx::create(ref);
+        mtx->copy_from(gen_mtx(532, 231));
+        empty = Mtx::create(ref);
+        expected = gen_mtx(532, 64);
+        y = gen_mtx(231, 64);
+        alpha = gko::initialize<Vec>({2.0}, ref);
+        beta = gko::initialize<Vec>({-1.0}, ref);
+        dmtx = Mtx::create(dpcpp);
+        dmtx->copy_from(mtx.get());
+        dempty = Mtx::create(dpcpp);
+        dresult = Vec::create(dpcpp);
+        dresult->copy_from(expected.get());
+        dy = Vec::create(dpcpp);
+        dy->copy_from(y.get());
+        dalpha = Vec::create(dpcpp);
+        dalpha->copy_from(alpha.get());
+        dbeta = Vec::create(dpcpp);
+        dbeta->copy_from(beta.get());
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::DpcppExecutor> dpcpp;
+
+    std::ranlux48 rand_engine;
+
+    std::unique_ptr<Mtx> mtx;
+    std::unique_ptr<Mtx> empty;
+    std::unique_ptr<Vec> expected;
+    std::unique_ptr<Vec> y;
+    std::unique_ptr<Vec> alpha;
+    std::unique_ptr<Vec> beta;
+
+    std::unique_ptr<Mtx> dmtx;
+    std::unique_ptr<Mtx> dempty;
+    std::unique_ptr<Vec> dresult;
+    std::unique_ptr<Vec> dy;
+    std::unique_ptr<Vec> dalpha;
+    std::unique_ptr<Vec> dbeta;
+};
+
+
+TEST_F(Sellp, SimpleApplyIsEquivalentToRef)
+{
+    set_up_apply_vector();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+}
+
+
+TEST_F(Sellp, AdvancedApplyIsEquivalentToRef)
+{
+    set_up_apply_vector();
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+}
+
+
+TEST_F(Sellp, SimpleApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
+{
+    set_up_apply_vector(32, 2);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+}
+
+
+TEST_F(Sellp, AdvancedApplyWithSliceSizeAndStrideFActorIsEquivalentToRef)
+{
+    set_up_apply_vector(32, 2);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+}
+
+
+TEST_F(Sellp, SimpleApplyMultipleRHSIsEquivalentToRef)
+{
+    set_up_apply_matrix();
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+}
+
+
+TEST_F(Sellp, AdvancedApplyMultipleRHSIsEquivalentToRef)
+{
+    set_up_apply_matrix();
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+}
+
+
+TEST_F(Sellp,
+       SimpleApplyMultipleRHSWithSliceSizeAndStrideFactorIsEquivalentToRef)
+{
+    set_up_apply_matrix(32, 2);
+
+    mtx->apply(y.get(), expected.get());
+    dmtx->apply(dy.get(), dresult.get());
+
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+}
+
+
+TEST_F(Sellp,
+       AdvancedApplyMultipleRHSWithSliceSizeAndStrideFActorIsEquivalentToRef)
+{
+    set_up_apply_matrix(32, 2);
+
+    mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
+    dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
+
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+}
+
+
+TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_matrix();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(complex_b.get(), complex_x.get());
+    dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+
+TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
+{
+    set_up_apply_matrix();
+    auto complex_b = gen_mtx<ComplexVec>(231, 3);
+    auto dcomplex_b = ComplexVec::create(dpcpp);
+    dcomplex_b->copy_from(complex_b.get());
+    auto complex_x = gen_mtx<ComplexVec>(532, 3);
+    auto dcomplex_x = ComplexVec::create(dpcpp);
+    dcomplex_x->copy_from(complex_x.get());
+
+    mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
+    dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
+
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+}
+
+
+TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
+{
+    set_up_apply_matrix();
+
+    auto dense_mtx = gko::matrix::Dense<>::create(ref);
+    auto ddense_mtx = gko::matrix::Dense<>::create(dpcpp);
+
+    mtx->convert_to(dense_mtx.get());
+    dmtx->convert_to(ddense_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Sellp, ConvertToCsrIsEquivalentToRef)
+{
+    set_up_apply_matrix();
+
+    auto csr_mtx = gko::matrix::Csr<>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<>::create(dpcpp);
+
+    mtx->convert_to(csr_mtx.get());
+    dmtx->convert_to(dcsr_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), 1e-14);
+}
+
+
+TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
+{
+    set_up_apply_matrix();
+
+    auto dense_mtx = gko::matrix::Dense<>::create(ref);
+    auto ddense_mtx = gko::matrix::Dense<>::create(dpcpp);
+
+    empty->convert_to(dense_mtx.get());
+    dempty->convert_to(ddense_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 0);
+}
+
+
+TEST_F(Sellp, ConvertEmptyToCsrIsEquivalentToRef)
+{
+    set_up_apply_matrix();
+
+    auto csr_mtx = gko::matrix::Csr<>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<>::create(dpcpp);
+
+    empty->convert_to(csr_mtx.get());
+    dempty->convert_to(dcsr_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), 0);
+}
+
+
+TEST_F(Sellp, CountNonzerosIsEquivalentToRef)
+{
+    set_up_apply_matrix();
+
+    gko::size_type nnz;
+    gko::size_type dnnz;
+
+    gko::kernels::reference::sellp::count_nonzeros(ref, mtx.get(), &nnz);
+    gko::kernels::dpcpp::sellp::count_nonzeros(dpcpp, dmtx.get(), &dnnz);
+
+    ASSERT_EQ(nnz, dnnz);
+}
+
+
+TEST_F(Sellp, ExtractDiagonalIsEquivalentToRef)
+{
+    set_up_apply_matrix();
+
+    auto diag = mtx->extract_diagonal();
+    auto ddiag = dmtx->extract_diagonal();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
+TEST_F(Sellp, ExtractDiagonalWithSliceSizeAndStrideFactorIsEquivalentToRef)
+{
+    set_up_apply_matrix(32, 2);
+
+    auto diag = mtx->extract_diagonal();
+    auto ddiag = dmtx->extract_diagonal();
+
+    GKO_ASSERT_MTX_NEAR(diag.get(), ddiag.get(), 0);
+}
+
+
+TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_matrix(32, 2);
+
+    mtx->compute_absolute_inplace();
+    dmtx->compute_absolute_inplace();
+
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
+}
+
+
+TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
+{
+    set_up_apply_matrix(32, 2);
+
+    auto abs_mtx = mtx->compute_absolute();
+    auto dabs_mtx = dmtx->compute_absolute();
+
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+}
+
+
+}  // namespace

From 40c620b30a5db07f3de6bfa042fb59e7701ba15a Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 22 Jul 2021 11:46:01 +0200
Subject: [PATCH 0975/1008] auto components

---
 dpcpp/components/format_conversion.dp.hpp |  31 +-
 dpcpp/components/intrinsics.dp.hpp        |  98 +++++++
 dpcpp/components/merging.dp.hpp           | 338 ++++++++++++++++++++++
 dpcpp/components/segment_scan.dp.hpp      |  86 ++++++
 4 files changed, 551 insertions(+), 2 deletions(-)
 create mode 100644 dpcpp/components/intrinsics.dp.hpp
 create mode 100644 dpcpp/components/merging.dp.hpp
 create mode 100644 dpcpp/components/segment_scan.dp.hpp

diff --git a/dpcpp/components/format_conversion.dp.hpp b/dpcpp/components/format_conversion.dp.hpp
index 270824d80e4..16624130b34 100644
--- a/dpcpp/components/format_conversion.dp.hpp
+++ b/dpcpp/components/format_conversion.dp.hpp
@@ -37,8 +37,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <CL/sycl.hpp>
 
 
+#include <ginkgo/config.hpp>
 #include <ginkgo/core/base/executor.hpp>
-#include <ginkgo/core/base/types.hpp>
+
+
+#include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
+#include "dpcpp/components/thread_ids.dp.hpp"
+
+
+#ifdef GINKGO_BENCHMARK_ENABLE_TUNING
+#include "benchmark/utils/tuning_variables.hpp"
+#endif  // GINKGO_BENCHMARK_ENABLE_TUNING
 
 
 #include "dpcpp/base/dim3.dp.hpp"
@@ -88,6 +98,7 @@ void convert_row_idxs_to_ptrs(dim3 grid, dim3 block,
 
 }  // namespace kernel
 
+}  // namespace kernel
 
 namespace host_kernel {
 
@@ -102,8 +113,24 @@ template <size_type subwarp_size = config::warp_size>
 size_type calculate_nwarps(std::shared_ptr<const DpcppExecutor> exec,
                            const size_type nnz)
 {
-    size_type nwarps_in_dpcpp = exec->get_num_computing_units() * 7;
+    size_type warps_per_sm =
+        exec->get_num_warps_per_sm() * config::warp_size / subwarp_size;
+    size_type nwarps_in_dpcpp = exec->get_num_multiprocessor() * warps_per_sm;
     size_type multiple = 8;
+    if (nnz >= 2e8) {
+        multiple = 2048;
+    } else if (nnz >= 2e7) {
+        multiple = 512;
+    } else if (nnz >= 2e6) {
+        multiple = 128;
+    } else if (nnz >= 2e5) {
+        multiple = 32;
+    }
+#ifdef GINKGO_BENCHMARK_ENABLE_TUNING
+    if (_tuning_flag) {
+        multiple = _tuned_value;
+    }
+#endif  // GINKGO_BENCHMARK_ENABLE_TUNING
     return std::min(multiple * nwarps_in_dpcpp,
                     size_type(ceildiv(nnz, config::warp_size)));
 }
diff --git a/dpcpp/components/intrinsics.dp.hpp b/dpcpp/components/intrinsics.dp.hpp
new file mode 100644
index 00000000000..291147eac2d
--- /dev/null
+++ b/dpcpp/components/intrinsics.dp.hpp
@@ -0,0 +1,98 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_COMPONENTS_INTRINSICS_DP_HPP_
+#define GKO_DPCPP_COMPONENTS_INTRINSICS_DP_HPP_
+
+
+#include <CL/sycl.hpp>
+
+
+#include <ginkgo/core/base/types.hpp>
+
+
+#include "dpcpp/base/dpct.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+/**
+ * @internal
+ * Returns the number of set bits in the given mask.
+ */
+__dpct_inline__ int popcnt(uint32 mask) { return sycl::popcount(mask); }
+
+/** @copydoc popcnt */
+__dpct_inline__ int popcnt(uint64 mask) { return sycl::popcount(mask); }
+
+
+/**
+ * @internal
+ * Returns the (1-based!) index of the first set bit in the given mask,
+ * starting from the least significant bit.
+ */
+/*
+DPCT1004:0: Compatible DPC++ code could not be generated.
+*/
+__dpct_inline__ int ffs(uint32 mask) { return __ffs(mask); }
+
+/** @copydoc ffs */
+__dpct_inline__ int ffs(uint64 mask)
+{
+    // the cast is necessary, as the overloads defined by HIP are ambiguous
+    /*
+    DPCT1004:1: Compatible DPC++ code could not be generated.
+    */
+    return __ffsll(static_cast<unsigned long long int>(mask));
+}
+
+
+/**
+ * @internal
+ * Returns the number of zero bits before the first set bit in the given mask,
+ * starting from the most significant bit.
+ */
+__dpct_inline__ int clz(uint32 mask) { return sycl::clz((int)mask); }
+
+/** @copydoc clz */
+__dpct_inline__ int clz(uint64 mask) { return sycl::clz((long long)mask); }
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_DPCPP_COMPONENTS_INTRINSICS_DP_HPP_
diff --git a/dpcpp/components/merging.dp.hpp b/dpcpp/components/merging.dp.hpp
new file mode 100644
index 00000000000..ecef5e4e549
--- /dev/null
+++ b/dpcpp/components/merging.dp.hpp
@@ -0,0 +1,338 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_COMPONENTS_MERGING_DP_HPP_
+#define GKO_DPCPP_COMPONENTS_MERGING_DP_HPP_
+
+
+#include <dpcpp/base/math.hpp>
+#include <dpcpp/components/searching.dp.hpp>
+
+
+#include <CL/sycl.hpp>
+
+
+#include "core/base/utils.hpp"
+#include "dpcpp/base/dpct.hpp"
+#include "dpcpp/components/intrinsics.dp.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+namespace detail {
+
+
+/**
+ * @internal
+ * The result from the @ref group_merge_step function.
+ */
+template <typename ValueType>
+struct merge_result {
+    /** The element of a being merged in the current thread. */
+    ValueType a_val;
+    /** The element of b being merged in the current thread. */
+    ValueType b_val;
+    /** The index from a that is being merged in the current thread. */
+    int a_idx;
+    /** The index from b that is being merged in the current thread. */
+    int b_idx;
+    /** The number of elements from a that have been merged in total. */
+    int a_advance;
+    /** The number of elements from b that have been merged in total. */
+    int b_advance;
+};
+
+}  // namespace detail
+
+
+/**
+ * @internal
+ * Warp-parallel merge algorithm that merges the first `warp_size` elements from
+ * two ranges, where each warp stores a single element from each range.
+ * It assumes that the elements are sorted in ascending order, i.e. for i < j,
+ * the value of `a` at thread i is smaller or equal to the value at thread j,
+ * and the same holds for `b`.
+ *
+ * This implementation is based on ideas from  Green et al.,
+ * "GPU merge path: a GPU merging algorithm", but uses random-access warp
+ * shuffles instead of shared-memory to exchange values of a and b.
+ *
+ * @param a      the element from the first range
+ * @param b      the element from the second range
+ * @param size   the number of elements in the output range
+ * @param group  the cooperative group that executes the merge
+ * @return  a structure containing the merge result distributed over the group.
+ */
+template <int group_size, typename ValueType, typename Group>
+__dpct_inline__ detail::merge_result<ValueType> group_merge_step(ValueType a,
+                                                                 ValueType b,
+                                                                 Group group)
+{
+    // thread i takes care of ith element of the merged sequence
+    auto i = int(group.thread_rank());
+
+    // we want to find the smallest index `x` such that a[x] >= b[i - x - 1]
+    // or `i` if no such index exists
+    //
+    // if x = i then c[0...i - 1] = a[0...i - 1]
+    //     => merge a[i] with b[0]
+    // if x = 0 then c[0...i - 1] = b[0...i - 1]
+    //     => merge a[0] with b[i]
+    // otherwise c[0...i - 1] contains a[0...x - 1] and b[0...i - x - 1]
+    //   because the minimality of `x` implies
+    //   b[i - x] >= a[x - 1]
+    //   and a[x] >= a[0...x - 1], b[0...i - x - 1]
+    //     => merge a[x] with b[i - x]
+    auto minx = synchronous_fixed_binary_search<group_size>([&](int x) {
+        auto a_remote = group.shfl(a, x);
+        auto b_remote = group.shfl(b, max(i - x - 1, 0));
+        return a_remote >= b_remote || x >= i;
+    });
+
+    auto a_idx = minx;
+    auto b_idx = max(i - minx, 0);
+    auto a_val = group.shfl(a, a_idx);
+    auto b_val = group.shfl(b, b_idx);
+    auto cmp = a_val < b_val;
+    auto a_advance = popcnt(group.ballot(cmp));
+    auto b_advance = int(group.size()) - a_advance;
+
+    return {a_val, b_val, a_idx, b_idx, a_advance, b_advance};
+}
+
+
+/**
+ * @internal
+ * Warp-parallel merge algorithm that merges two sorted ranges of arbitrary
+ * size. `merge_fn` will be called for each merged element.
+ *
+ * @param a       the first range
+ * @param a_size  the size of the first range
+ * @param b       the second range
+ * @param b_size  the size of the second range
+ * @param group   the group that executes the merge
+ * @param merge_fn  the callback that is being called for each merged element.
+ *                  It takes six parameters:
+ *                  `IndexType a_idx, ValueType a_val, IndexType b_idx,
+ *                   ValueType b_val, IndexType c_index, bool valid`.
+ *                  `*_val` and `*_idx` are the values resp. the indices of the
+ *                  values from a/b being compared at output index `c_index`.
+ *                  `valid` specifies if the current thread has to merge an
+ *                  element (this is necessary for shfl and ballot operations).
+ *                  It must return `false` on all threads of the group iff the
+ *                  merge shouldn't be continued.
+ */
+template <int group_size, typename ValueType, typename IndexType,
+          typename Group, typename Callback>
+__dpct_inline__ void group_merge(const ValueType *__restrict__ a,
+                                 IndexType a_size,
+                                 const ValueType *__restrict__ b,
+                                 IndexType b_size, Group group,
+                                 Callback merge_fn)
+{
+    auto c_size = a_size + b_size;
+    IndexType a_begin{};
+    IndexType b_begin{};
+    auto lane = static_cast<IndexType>(group.thread_rank());
+    auto sentinel = device_numeric_limits<IndexType>::max;
+    auto a_cur = checked_load(a, a_begin + lane, a_size, sentinel);
+    auto b_cur = checked_load(b, b_begin + lane, b_size, sentinel);
+    for (IndexType c_begin{}; c_begin < c_size; c_begin += group_size) {
+        auto merge_result = group_merge_step<group_size>(a_cur, b_cur, group);
+        auto valid = c_begin + lane < c_size;
+        auto cont = merge_fn(merge_result.a_idx + a_begin, merge_result.a_val,
+                             merge_result.b_idx + b_begin, merge_result.b_val,
+                             c_begin + lane, valid);
+        if (!group.any(cont && valid)) {
+            break;
+        }
+        auto a_advance = merge_result.a_advance;
+        auto b_advance = merge_result.b_advance;
+        a_begin += a_advance;
+        b_begin += b_advance;
+
+        // shuffle the unmerged elements to the front
+        a_cur = group.shfl_down(a_cur, a_advance);
+        b_cur = group.shfl_down(b_cur, b_advance);
+        /*
+         * To optimize memory access, we load the new elements for `a` and `b`
+         * with a single load instruction:
+         * the lower part of the group loads new elements for `a`
+         * the upper part of the group loads new elements for `b`
+         * `load_lane` is the part-local lane idx
+         * The elements for `a` have to be shuffled up afterwards.
+         */
+        auto load_a = lane < a_advance;
+        auto load_lane = load_a ? lane : lane - a_advance;
+        auto load_source = load_a ? a : b;
+        auto load_begin = load_a ? a_begin + b_advance : b_begin + a_advance;
+        auto load_size = load_a ? a_size : b_size;
+
+        auto load_idx = load_begin + load_lane;
+        auto loaded = checked_load(load_source, load_idx, load_size, sentinel);
+        // shuffle the `a` values to the end of the warp
+        auto lower_loaded = group.shfl_up(loaded, b_advance);
+        a_cur = lane < b_advance ? a_cur : lower_loaded;
+        b_cur = lane < a_advance ? b_cur : loaded;
+    }
+}
+
+
+/**
+ * @internal
+ * Warp-parallel merge algorithm that reports matching elements from two sorted
+ * ranges of arbitrary size. `merge_fn` will be called for each pair of matching
+ * element.
+ *
+ * @param a       the first range
+ * @param a_size  the size of the first range
+ * @param b       the second range
+ * @param b_size  the size of the second range
+ * @param group   the group that executes the merge
+ * @param match_fn  the callback that is being called for each matching pair.
+ *                  It takes five parameters:
+ *                  `ValueType val, IndexType a_idx, IndexType b_idx,
+ *                   lane_mask_type match_mask, bool valid`.
+ *                  `val` is the matching element, `*_idx` are the indices of
+ *                  the matching values from a and b, match_mask is a lane mask
+ *                  that is 1 for every subwarp lane that found a match.
+ *                  `valid` is true iff there is actually a match.
+ *                  (necessary for warp-synchronous operations)
+ */
+template <int group_size, typename IndexType, typename ValueType,
+          typename Group, typename Callback>
+__dpct_inline__ void group_match(const ValueType *__restrict__ a,
+                                 IndexType a_size,
+                                 const ValueType *__restrict__ b,
+                                 IndexType b_size, Group group,
+                                 Callback match_fn)
+{
+    group_merge<group_size>(
+        a, a_size, b, b_size, group,
+        [&](IndexType a_idx, ValueType a_val, IndexType b_idx, ValueType b_val,
+            IndexType, bool valid) {
+            auto matchmask = group.ballot(a_val == b_val && valid);
+            match_fn(a_val, a_idx, b_idx, matchmask, a_val == b_val && valid);
+            return a_idx < a_size && b_idx < b_size;
+        });
+}
+
+
+/**
+ * @internal
+ * Sequential merge algorithm that merges two sorted ranges of arbitrary
+ * size. `merge_fn` will be called for each merged element.
+ *
+ * @param a  the first range
+ * @param a_size the size of the first range
+ * @param b  the second range
+ * @param b_size the size of the second range
+ * @param merge_fn  the callback that will be called for each merge step.
+ *                  It takes five parameters:
+ *                  `IndexType a_idx, ValueType a_val,
+ *                   IndexType b_idx, ValueType b_val, IndexType c_idx`.
+ *                  `*_val` and `*_idx` are the values resp. the indices of
+ *                  the values from a/b being compared in step `c_idx`.
+ *                  It must return `false` iff the merge should stop.
+ */
+template <typename ValueType, typename IndexType, typename Callback>
+__dpct_inline__ void sequential_merge(const ValueType *__restrict__ a,
+                                      IndexType a_size,
+                                      const ValueType *__restrict__ b,
+                                      IndexType b_size, Callback merge_fn)
+{
+    auto c_size = a_size + b_size;
+    IndexType a_begin{};
+    IndexType b_begin{};
+    auto sentinel = device_numeric_limits<IndexType>::max;
+    auto a_cur = checked_load(a, a_begin, a_size, sentinel);
+    auto b_cur = checked_load(b, b_begin, b_size, sentinel);
+    for (IndexType c_begin{}; c_begin < c_size; c_begin++) {
+        auto cont = merge_fn(a_begin, a_cur, b_begin, b_cur, c_begin);
+        if (!cont) {
+            break;
+        }
+        auto a_advance = a_cur < b_cur;
+        auto b_advance = !a_advance;
+        a_begin += a_advance;
+        b_begin += b_advance;
+
+        auto load = a_advance ? a : b;
+        auto load_size = a_advance ? a_size : b_size;
+        auto load_idx = a_advance ? a_begin : b_begin;
+        auto loaded = checked_load(load, load_idx, load_size, sentinel);
+        a_cur = a_advance ? loaded : a_cur;
+        b_cur = b_advance ? loaded : b_cur;
+    }
+}
+
+
+/**
+ * @internal
+ * Sequential algorithm that finds matching elements in two sorted ranges of
+ * arbitrary size. `merge_fn` will be called for each pair of matching
+ * elements.
+ *
+ * @param a  the first range
+ * @param a_size the size of the first range
+ * @param b  the second range
+ * @param b_size the size of the second range
+ * @param match_fn  the callback that is being called for each match.
+ *                  It takes three parameters:
+ *                  `ValueType val, IndexType a_idx, IndexType b_idx`.
+ *                  `val` is the matching element, `*_idx` are the
+ *                  indices of the matching values from a and b.
+ */
+template <typename IndexType, typename ValueType, typename Callback>
+__dpct_inline__ void sequential_match(const ValueType *a, IndexType a_size,
+                                      const ValueType *b, IndexType b_size,
+                                      Callback match_fn)
+{
+    sequential_merge(a, a_size, b, b_size,
+                     [&](IndexType a_idx, ValueType a_val, IndexType b_idx,
+                         ValueType b_val, IndexType) {
+                         if (a_val == b_val) {
+                             match_fn(a_val, a_idx, b_idx);
+                         }
+                         return a_idx < a_size && b_idx < b_size;
+                     });
+}
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_DPCPP_COMPONENTS_MERGING_DP_HPP_
diff --git a/dpcpp/components/segment_scan.dp.hpp b/dpcpp/components/segment_scan.dp.hpp
new file mode 100644
index 00000000000..b1ca309f4b6
--- /dev/null
+++ b/dpcpp/components/segment_scan.dp.hpp
@@ -0,0 +1,86 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_DPCPP_COMPONENTS_SEGMENT_SCAN_DP_HPP_
+#define GKO_DPCPP_COMPONENTS_SEGMENT_SCAN_DP_HPP_
+
+
+#include <CL/sycl.hpp>
+
+
+#include "dpcpp/base/dpct.hpp"
+#include "dpcpp/components/cooperative_groups.dp.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace dpcpp {
+
+
+/**
+ * @internal
+ *
+ * Compute a segement scan using add operation (+) of a subwarp. Each segment
+ * performs suffix sum. Works on the source array and returns whether the thread
+ * is the first element of its segment with same `ind`.
+ */
+template <unsigned subwarp_size, typename ValueType, typename IndexType>
+__dpct_inline__ bool segment_scan(
+    const group::thread_block_tile<subwarp_size> &group, const IndexType ind,
+    ValueType *__restrict__ val, sycl::nd_item<3> item_ct1)
+{
+    bool head = true;
+#pragma unroll
+    for (int i = 1; i < subwarp_size; i <<= 1) {
+        const IndexType add_ind = group.shfl_up(ind, i);
+        ValueType add_val = zero<ValueType>();
+        if (add_ind == ind && item_ct1.get_local_id(2) >= i) {
+            add_val = *val;
+            if (i == 1) {
+                head = false;
+            }
+        }
+        add_val = group.shfl_down(add_val, i);
+        if (item_ct1.get_local_id(2) < subwarp_size - i) {
+            *val += add_val;
+        }
+    }
+    return head;
+}
+
+
+}  // namespace dpcpp
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_DPCPP_COMPONENTS_SEGMENT_SCAN_DP_HPP_

From fb6509a6f75c00fffd8cfa2be787219d7bbb9a10 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 22 Jul 2021 11:48:19 +0200
Subject: [PATCH 0976/1008] format matrix

---
 dpcpp/matrix/coo_kernels.dp.cpp    | 2 +-
 dpcpp/matrix/csr_kernels.dp.cpp    | 6 +++---
 dpcpp/matrix/hybrid_kernels.dp.cpp | 4 +---
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/dpcpp/matrix/coo_kernels.dp.cpp b/dpcpp/matrix/coo_kernels.dp.cpp
index 593752c452b..3225f0a6164 100644
--- a/dpcpp/matrix/coo_kernels.dp.cpp
+++ b/dpcpp/matrix/coo_kernels.dp.cpp
@@ -34,7 +34,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <dpcpp/base/math.hpp>
-#include <dpcpp/components/segment_scan.dp.hpp>
 
 
 #include <CL/sycl.hpp>
@@ -54,6 +53,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "dpcpp/components/atomic.dp.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/format_conversion.dp.hpp"
+#include "dpcpp/components/segment_scan.dp.hpp"
 #include "dpcpp/components/thread_ids.dp.hpp"
 
 
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index 3cc17ae6d98..0062dc169fb 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <algorithm>
 #include <dpcpp/base/math.hpp>
 #include <dpcpp/base/pointer_mode_guard.hpp>
-#include <dpcpp/components/intrinsics.dp.hpp>
-#include <dpcpp/components/merging.dp.hpp>
-#include <dpcpp/components/segment_scan.dp.hpp>
 
 
 #include <CL/sycl.hpp>
@@ -62,7 +59,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "dpcpp/base/onemkl_bindings.hpp"
 #include "dpcpp/components/atomic.dp.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
+#include "dpcpp/components/intrinsics.dp.hpp"
+#include "dpcpp/components/merging.dp.hpp"
 #include "dpcpp/components/reduction.dp.hpp"
+#include "dpcpp/components/segment_scan.dp.hpp"
 #include "dpcpp/components/thread_ids.dp.hpp"
 #include "dpcpp/components/uninitialized_array.hpp"
 
diff --git a/dpcpp/matrix/hybrid_kernels.dp.cpp b/dpcpp/matrix/hybrid_kernels.dp.cpp
index 25d453a623d..a8a92eee7bb 100644
--- a/dpcpp/matrix/hybrid_kernels.dp.cpp
+++ b/dpcpp/matrix/hybrid_kernels.dp.cpp
@@ -30,9 +30,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include <dpcpp/components/segment_scan.dp.hpp>
-
-
 #include <CL/sycl.hpp>
 
 
@@ -50,6 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/format_conversion.dp.hpp"
 #include "dpcpp/components/reduction.dp.hpp"
+#include "dpcpp/components/segment_scan.dp.hpp"
 #include "dpcpp/components/thread_ids.dp.hpp"
 
 

From 0f04274bd5744d129f91bc6f5abe46617a250c89 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 22 Jul 2021 12:29:51 +0200
Subject: [PATCH 0977/1008] fix matrix format

---
 dpcpp/matrix/coo_kernels.dp.cpp          |  12 +-
 dpcpp/matrix/csr_kernels.dp.cpp          | 608 +++--------------------
 dpcpp/matrix/diagonal_kernels.dp.cpp     |   9 +-
 dpcpp/matrix/hybrid_kernels.dp.cpp       |   5 +-
 dpcpp/matrix/sellp_kernels.dp.cpp        |   8 +-
 dpcpp/matrix/sparsity_csr_kernels.dp.cpp |   3 +
 6 files changed, 102 insertions(+), 543 deletions(-)

diff --git a/dpcpp/matrix/coo_kernels.dp.cpp b/dpcpp/matrix/coo_kernels.dp.cpp
index 3225f0a6164..3a48635fef0 100644
--- a/dpcpp/matrix/coo_kernels.dp.cpp
+++ b/dpcpp/matrix/coo_kernels.dp.cpp
@@ -33,9 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/coo_kernels.hpp"
 
 
-#include <dpcpp/base/math.hpp>
-
-
 #include <CL/sycl.hpp>
 
 
@@ -73,7 +70,7 @@ namespace dpcpp {
 namespace coo {
 
 
-constexpr int default_block_size = 512;
+constexpr int default_block_size = 256;
 constexpr int warps_in_block = 4;
 constexpr int spmv_block_size = warps_in_block * config::warp_size;
 
@@ -378,7 +375,6 @@ void convert_row_idxs_to_ptrs(const IndexType *__restrict__ idxs,
     }
 }
 
-template <typename IndexType>
 void convert_row_idxs_to_ptrs(dim3 grid, dim3 block,
                               gko::size_type dynamic_shared_memory,
                               sycl::queue *stream, const IndexType *idxs,
@@ -394,6 +390,12 @@ void convert_row_idxs_to_ptrs(dim3 grid, dim3 block,
     });
 }
 
+template void convert_row_idxs_to_ptrs(dim3, dim3, size_t, sycl::queue *,
+                                       const int32 *idxs, size_type, int32 *,
+                                       size_type);
+template void convert_row_idxs_to_ptrs(dim3, dim3, size_t, sycl::queue *,
+                                       const int64 *idxs, size_type, int64 *,
+                                       size_type);
 
 template <typename ValueType>
 void initialize_zero_dense(size_type num_rows, size_type num_cols,
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index 0062dc169fb..3c110f303f6 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -30,12 +30,14 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include "core/matrix/csr_kernels.hpp"
+
+
 #include <algorithm>
-#include <dpcpp/base/math.hpp>
-#include <dpcpp/base/pointer_mode_guard.hpp>
 
 
 #include <CL/sycl.hpp>
+#include <oneapi/mkl.hpp>
 
 
 #include <ginkgo/core/base/array.hpp>
@@ -78,7 +80,7 @@ namespace dpcpp {
 namespace csr {
 
 
-constexpr int default_block_size = 512;
+constexpr int default_block_size = 256;
 constexpr int warps_in_block = 4;
 constexpr int spmv_block_size = warps_in_block * config::warp_size;
 constexpr int wsize = config::warp_size;
@@ -89,13 +91,11 @@ constexpr int classical_overweight = 32;
  * A compile-time list of the number items per threads for which spmv kernel
  * should be compiled.
  */
-using compiled_kernels = syn::value_list<int, 3, 4, 6, 7, 8, 12, 14>;
+using compiled_kernels = syn::value_list<int, 6>;
 
-using classical_kernels =
-    syn::value_list<int, config::warp_size, 32, 16, 8, 4, 2, 1>;
+using classical_kernels = syn::value_list<int, config::warp_size, 16, 8, 1>;
 
-using spgeam_kernels =
-    syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
+using spgeam_kernels = syn::value_list<int, 8, 16>;
 
 
 namespace kernel {
@@ -532,14 +532,15 @@ void abstract_merge_path_spmv(dim3 grid, dim3 block,
     stream->submit([&](sycl::handler &cgh) {
         sycl::accessor<IndexType, 1, sycl::access_mode::read_write,
                        sycl::access::target::local>
-            shared_row_ptrs_acc_ct1(sycl::range<1>(block_items), cgh);
+            shared_row_ptrs_acc_ct1(
+                sycl::range<1>(spmv_block_size * items_per_thread), cgh);
 
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 abstract_merge_path_spmv<items_per_thread>(
                     num_rows, val, col_idxs, row_ptrs, srow, b, b_stride, c,
                     c_stride, row_out, val_out, item_ct1,
-                    (IndexType *)shared_row_ptrs_acc_ct1.get_pointer());
+                    shared_row_ptrs_acc_ct1.get_pointer().get());
             });
     });
 }
@@ -577,7 +578,8 @@ void abstract_merge_path_spmv(
     stream->submit([&](sycl::handler &cgh) {
         sycl::accessor<IndexType, 1, sycl::access_mode::read_write,
                        sycl::access::target::local>
-            shared_row_ptrs_acc_ct1(sycl::range<1>(block_items), cgh);
+            shared_row_ptrs_acc_ct1(
+                sycl::range<1>(spmv_block_size * items_per_thread), cgh);
 
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
@@ -624,7 +626,8 @@ void abstract_reduce(dim3 grid, dim3 block,
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 abstract_reduce(nwarps, last_val, last_row, c, c_stride,
-                                item_ct1, tmp_val_acc_ct1.get_pointer().get());
+                                item_ct1, tmp_ind_acc_ct1.get_pointer().get(),
+                                tmp_val_acc_ct1.get_pointer().get());
             });
     });
 }
@@ -667,7 +670,8 @@ void abstract_reduce(dim3 grid, dim3 block,
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 abstract_reduce(nwarps, last_val, last_row, alpha, c, c_stride,
-                                item_ct1, tmp_val_acc_ct1.get_pointer().get());
+                                item_ct1, tmp_ind_acc_ct1.get_pointer().get(),
+                                tmp_val_acc_ct1.get_pointer().get());
             });
     });
 }
@@ -697,7 +701,7 @@ void device_classical_spmv(const size_type num_rows,
              ind += subwarp_size) {
             temp_val += val[ind] * b[col_idxs[ind] * b_stride + column_id];
         }
-        auto subwarp_result = reduce(
+        auto subwarp_result = ::gko::kernels::dpcpp::reduce(
             subwarp_tile, temp_val,
             [](const ValueType &a, const ValueType &b) { return a + b; });
         if (subid == 0) {
@@ -1062,7 +1066,7 @@ void calculate_slice_lengths(size_type num_rows, size_type slice_size,
 
         auto warp_tile = group::tiled_partition<warp_size>(
             group::this_thread_block(item_ct1));
-        auto warp_result = reduce(
+        auto warp_result = ::gko::kernels::dpcpp::reduce(
             warp_tile, thread_result,
             [](const size_type &a, const size_type &b) { return max(a, b); });
 
@@ -1243,7 +1247,7 @@ void reduce_max_nnz_per_slice(size_type num_rows, size_type slice_size,
                 max(thread_result, nnz_per_row[warpid * slice_size + i]);
         }
     }
-    auto warp_result = reduce(
+    auto warp_result = ::gko::kernels::dpcpp::reduce(
         warp_tile, thread_result,
         [](const size_type &a, const size_type &b) { return max(a, b); });
 
@@ -1273,7 +1277,7 @@ void reduce_total_cols(size_type num_slices,
                        size_type *__restrict__ result,
                        sycl::nd_item<3> item_ct1, size_type *block_result)
 {
-    reduce_array(num_slices, max_nnz_per_slice, block_result,
+    reduce_array(num_slices, max_nnz_per_slice, block_result, item_ct1,
                  [](const size_type &x, const size_type &y) { return x + y; });
 
     if (item_ct1.get_local_id(2) == 0) {
@@ -1306,7 +1310,7 @@ void reduce_max_nnz(size_type size, const size_type *__restrict__ nnz_per_row,
                     size_type *block_max)
 {
     reduce_array(
-        size, nnz_per_row, block_max,
+        size, nnz_per_row, block_max, item_ct1,
         [](const size_type &x, const size_type &y) { return max(x, y); });
 
     if (item_ct1.get_local_id(2) == 0) {
@@ -1531,10 +1535,6 @@ void conjugate_kernel(size_type num_nonzeros, ValueType *__restrict__ val,
     const auto tidx = thread::get_thread_id_flat(item_ct1);
 
     if (tidx < num_nonzeros) {
-        /*
-        DPCT1007:26: Migration of this CUDA API is not supported by the Intel(R)
-        DPC++ Compatibility Tool.
-        */
         val[tidx] = conj(val[tidx]);
     }
 }
@@ -1898,32 +1898,7 @@ GKO_ENABLE_IMPLEMENTATION_SELECTION(select_merge_path_spmv, merge_path_spmv);
 template <typename ValueType, typename IndexType>
 int compute_items_per_thread(std::shared_ptr<const DpcppExecutor> exec)
 {
-    const int version =
-        (exec->get_major_version() << 4) + exec->get_minor_version();
-    // The num_item is decided to make the occupancy 100%
-    // TODO: Extend this list when new GPU is released
-    //       Tune this parameter
-    // 128 threads/block the number of items per threads
-    // 3.0 3.5: 6
-    // 3.7: 14
-    // 5.0, 5.3, 6.0, 6.2: 8
-    // 5.2, 6.1, 7.0: 12
     int num_item = 6;
-    switch (version) {
-    case 0x50:
-    case 0x53:
-    case 0x60:
-    case 0x62:
-        num_item = 8;
-        break;
-    case 0x52:
-    case 0x61:
-    case 0x70:
-        num_item = 12;
-        break;
-    case 0x37:
-        num_item = 14;
-    }
     // Ensure that the following is satisfied:
     // sizeof(IndexType) + sizeof(ValueType)
     // <= items_per_thread * sizeof(IndexType)
@@ -1943,8 +1918,9 @@ void classical_spmv(syn::value_list<int, subwarp_size>,
                     const matrix::Dense<ValueType> *alpha = nullptr,
                     const matrix::Dense<ValueType> *beta = nullptr)
 {
-    const auto nwarps = exec->get_num_warps_per_sm() *
-                        exec->get_num_multiprocessor() * classical_overweight;
+    const auto threads_per_cu = 7;
+    const auto nwarps =
+        exec->get_num_computing_units() * threads_per_cu * classical_overweight;
     const auto gridx =
         std::min(ceildiv(a->get_size()[0], spmv_block_size / subwarp_size),
                  int64(nwarps / warps_in_block));
@@ -2028,59 +2004,30 @@ void spmv(std::shared_ptr<const DpcppExecutor> exec,
             syn::value_list<int>(), syn::type_list<>(), exec, a, b, c);
     } else if (a->get_strategy()->get_name() == "sparselib" ||
                a->get_strategy()->get_name() == "cusparse") {
-        if (cusparse::is_supported<ValueType, IndexType>::value) {
-            // TODO: add implementation for int64 and multiple RHS
-            auto handle = exec->get_cusparse_handle();
-            {
-                cusparse::pointer_mode_guard pm_guard(handle);
-                const auto alpha = one<ValueType>();
-                const auto beta = zero<ValueType>();
-                // TODO: add implementation for int64 and multiple RHS
-                if (b->get_stride() != 1 || c->get_stride() != 1)
-                    GKO_NOT_IMPLEMENTED;
-
-#if defined(DPCPP_VERSION) && (DPCPP_VERSION < 11000)
-                auto descr = cusparse::create_mat_descr();
-                auto row_ptrs = a->get_const_row_ptrs();
-                auto col_idxs = a->get_const_col_idxs();
-                cusparse::spmv(handle, oneapi::mkl::transpose::nontrans,
-                               a->get_size()[0], a->get_size()[1],
-                               a->get_num_stored_elements(), &alpha, descr,
-                               a->get_const_values(), row_ptrs, col_idxs,
-                               b->get_const_values(), &beta, c->get_values());
-
-                cusparse::destroy(descr);
-#else  // DPCPP_VERSION >= 11000
-                cusparseOperation_t trans = CUSPARSE_OPERATION_NON_TRANSPOSE;
-                cusparseSpMVAlg_t alg = CUSPARSE_CSRMV_ALG1;
-                auto row_ptrs =
-                    const_cast<IndexType *>(a->get_const_row_ptrs());
-                auto col_idxs =
-                    const_cast<IndexType *>(a->get_const_col_idxs());
-                auto values = const_cast<ValueType *>(a->get_const_values());
-                auto mat = cusparse::create_csr(
-                    a->get_size()[0], a->get_size()[1],
-                    a->get_num_stored_elements(), row_ptrs, col_idxs, values);
-                auto b_val = const_cast<ValueType *>(b->get_const_values());
-                auto c_val = c->get_values();
-                auto vecb =
-                    cusparse::create_dnvec(b->get_num_stored_elements(), b_val);
-                auto vecc =
-                    cusparse::create_dnvec(c->get_num_stored_elements(), c_val);
-                size_type buffer_size = 0;
-                cusparse::spmv_buffersize<ValueType>(handle, trans, &alpha, mat,
-                                                     vecb, &beta, vecc, alg,
-                                                     &buffer_size);
-
-                gko::Array<char> buffer_array(exec, buffer_size);
-                auto buffer = buffer_array.get_data();
-                cusparse::spmv<ValueType>(handle, trans, &alpha, mat, vecb,
-                                          &beta, vecc, alg, buffer);
-                cusparse::destroy(vecb);
-                cusparse::destroy(vecc);
-                cusparse::destroy(mat);
-#endif
+        if (!is_complex<ValueType>()) {
+            oneapi::mkl::sparse::matrix_handle_t mat_handle;
+            oneapi::mkl::sparse::init_matrix_handle(&mat_handle);
+            oneapi::mkl::sparse::set_csr_data(
+                mat_handle, IndexType(a->get_size()[0]),
+                IndexType(a->get_size()[1]), oneapi::mkl::index_base::zero,
+                const_cast<IndexType *>(a->get_const_row_ptrs()),
+                const_cast<IndexType *>(a->get_const_col_idxs()),
+                const_cast<ValueType *>(a->get_const_values()));
+            if (b->get_size()[1] == 1 && b->get_stride() == 1) {
+                oneapi::mkl::sparse::gemv(
+                    *exec->get_queue(), oneapi::mkl::transpose::nontrans,
+                    one<ValueType>(), mat_handle,
+                    const_cast<ValueType *>(b->get_const_values()),
+                    zero<ValueType>(), c->get_values());
+            } else {
+                oneapi::mkl::sparse::gemm(
+                    *exec->get_queue(), oneapi::mkl::transpose::nontrans,
+                    one<ValueType>(), mat_handle,
+                    const_cast<ValueType *>(b->get_const_values()),
+                    b->get_size()[1], b->get_stride(), zero<ValueType>(),
+                    c->get_values(), c->get_stride());
             }
+            oneapi::mkl::sparse::release_matrix_handle(&mat_handle);
         } else {
             GKO_NOT_IMPLEMENTED;
         }
@@ -2121,52 +2068,32 @@ void advanced_spmv(std::shared_ptr<const DpcppExecutor> exec,
         }
     } else if (a->get_strategy()->get_name() == "sparselib" ||
                a->get_strategy()->get_name() == "cusparse") {
-        if (cusparse::is_supported<ValueType, IndexType>::value) {
-            // TODO: add implementation for int64 and multiple RHS
-            if (b->get_stride() != 1 || c->get_stride() != 1)
-                GKO_NOT_IMPLEMENTED;
-
-#if defined(DPCPP_VERSION) && (DPCPP_VERSION < 11000)
-            auto descr = cusparse::create_mat_descr();
-            auto row_ptrs = a->get_const_row_ptrs();
-            auto col_idxs = a->get_const_col_idxs();
-            cusparse::spmv(exec->get_cusparse_handle(),
-                           oneapi::mkl::transpose::nontrans, a->get_size()[0],
-                           a->get_size()[1], a->get_num_stored_elements(),
-                           alpha->get_const_values(), descr,
-                           a->get_const_values(), row_ptrs, col_idxs,
-                           b->get_const_values(), beta->get_const_values(),
-                           c->get_values());
-
-            cusparse::destroy(descr);
-#else  // DPCPP_VERSION >= 11000
-            cusparseOperation_t trans = CUSPARSE_OPERATION_NON_TRANSPOSE;
-            cusparseSpMVAlg_t alg = CUSPARSE_CSRMV_ALG1;
-            auto row_ptrs = const_cast<IndexType *>(a->get_const_row_ptrs());
-            auto col_idxs = const_cast<IndexType *>(a->get_const_col_idxs());
-            auto values = const_cast<ValueType *>(a->get_const_values());
-            auto mat = cusparse::create_csr(a->get_size()[0], a->get_size()[1],
-                                            a->get_num_stored_elements(),
-                                            row_ptrs, col_idxs, values);
-            auto b_val = const_cast<ValueType *>(b->get_const_values());
-            auto c_val = c->get_values();
-            auto vecb =
-                cusparse::create_dnvec(b->get_num_stored_elements(), b_val);
-            auto vecc =
-                cusparse::create_dnvec(c->get_num_stored_elements(), c_val);
-            size_type buffer_size = 0;
-            cusparse::spmv_buffersize<ValueType>(
-                exec->get_cusparse_handle(), trans, alpha->get_const_values(),
-                mat, vecb, beta->get_const_values(), vecc, alg, &buffer_size);
-            gko::Array<char> buffer_array(exec, buffer_size);
-            auto buffer = buffer_array.get_data();
-            cusparse::spmv<ValueType>(
-                exec->get_cusparse_handle(), trans, alpha->get_const_values(),
-                mat, vecb, beta->get_const_values(), vecc, alg, buffer);
-            cusparse::destroy(vecb);
-            cusparse::destroy(vecc);
-            cusparse::destroy(mat);
-#endif
+        if (!is_complex<ValueType>()) {
+            oneapi::mkl::sparse::matrix_handle_t mat_handle;
+            oneapi::mkl::sparse::init_matrix_handle(&mat_handle);
+            oneapi::mkl::sparse::set_csr_data(
+                mat_handle, IndexType(a->get_size()[0]),
+                IndexType(a->get_size()[1]), oneapi::mkl::index_base::zero,
+                const_cast<IndexType *>(a->get_const_row_ptrs()),
+                const_cast<IndexType *>(a->get_const_col_idxs()),
+                const_cast<ValueType *>(a->get_const_values()));
+            if (b->get_size()[1] == 1 && b->get_stride() == 1) {
+                oneapi::mkl::sparse::gemv(
+                    *exec->get_queue(), oneapi::mkl::transpose::nontrans,
+                    exec->copy_val_to_host(alpha->get_const_values()),
+                    mat_handle, const_cast<ValueType *>(b->get_const_values()),
+                    exec->copy_val_to_host(beta->get_const_values()),
+                    c->get_values());
+            } else {
+                oneapi::mkl::sparse::gemm(
+                    *exec->get_queue(), oneapi::mkl::transpose::nontrans,
+                    exec->copy_val_to_host(alpha->get_const_values()),
+                    mat_handle, const_cast<ValueType *>(b->get_const_values()),
+                    b->get_size()[1], b->get_stride(),
+                    exec->copy_val_to_host(beta->get_const_values()),
+                    c->get_values(), c->get_stride());
+            }
+            oneapi::mkl::sparse::release_matrix_handle(&mat_handle);
         } else {
             GKO_NOT_IMPLEMENTED;
         }
@@ -2215,124 +2142,7 @@ void spgemm(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Csr<ValueType, IndexType> *b,
             matrix::Csr<ValueType, IndexType> *c)
 {
-    auto a_nnz = IndexType(a->get_num_stored_elements());
-    auto a_vals = a->get_const_values();
-    auto a_row_ptrs = a->get_const_row_ptrs();
-    auto a_col_idxs = a->get_const_col_idxs();
-    auto b_vals = b->get_const_values();
-    auto b_row_ptrs = b->get_const_row_ptrs();
-    auto b_col_idxs = b->get_const_col_idxs();
-    auto c_row_ptrs = c->get_row_ptrs();
-
-    if (cusparse::is_supported<ValueType, IndexType>::value) {
-        auto handle = exec->get_cusparse_handle();
-        cusparse::pointer_mode_guard pm_guard(handle);
-
-        auto alpha = one<ValueType>();
-        auto a_nnz = static_cast<IndexType>(a->get_num_stored_elements());
-        auto b_nnz = static_cast<IndexType>(b->get_num_stored_elements());
-        auto null_value = static_cast<ValueType *>(nullptr);
-        auto null_index = static_cast<IndexType *>(nullptr);
-        auto zero_nnz = IndexType{};
-        auto m = IndexType(a->get_size()[0]);
-        auto n = IndexType(b->get_size()[1]);
-        auto k = IndexType(a->get_size()[1]);
-        matrix::CsrBuilder<ValueType, IndexType> c_builder{c};
-        auto &c_col_idxs_array = c_builder.get_col_idx_array();
-        auto &c_vals_array = c_builder.get_value_array();
-
-#if defined(DPCPP_VERSION) && (DPCPP_VERSION < 11000)
-        auto a_descr = cusparse::create_mat_descr();
-        auto b_descr = cusparse::create_mat_descr();
-        auto c_descr = cusparse::create_mat_descr();
-        auto d_descr = cusparse::create_mat_descr();
-        auto info = cusparse::create_spgemm_info();
-        // allocate buffer
-        size_type buffer_size{};
-        cusparse::spgemm_buffer_size(
-            handle, m, n, k, &alpha, a_descr, a_nnz, a_row_ptrs, a_col_idxs,
-            b_descr, b_nnz, b_row_ptrs, b_col_idxs, null_value, d_descr,
-            zero_nnz, null_index, null_index, info, buffer_size);
-        Array<char> buffer_array(exec, buffer_size);
-        auto buffer = buffer_array.get_data();
-
-        // count nnz
-        IndexType c_nnz{};
-        cusparse::spgemm_nnz(handle, m, n, k, a_descr, a_nnz, a_row_ptrs,
-                             a_col_idxs, b_descr, b_nnz, b_row_ptrs, b_col_idxs,
-                             d_descr, zero_nnz, null_index, null_index, c_descr,
-                             c_row_ptrs, &c_nnz, info, buffer);
-
-        // accumulate non-zeros
-        c_col_idxs_array.resize_and_reset(c_nnz);
-        c_vals_array.resize_and_reset(c_nnz);
-        auto c_col_idxs = c_col_idxs_array.get_data();
-        auto c_vals = c_vals_array.get_data();
-        cusparse::spgemm(handle, m, n, k, &alpha, a_descr, a_nnz, a_vals,
-                         a_row_ptrs, a_col_idxs, b_descr, b_nnz, b_vals,
-                         b_row_ptrs, b_col_idxs, null_value, d_descr, zero_nnz,
-                         null_value, null_index, null_index, c_descr, c_vals,
-                         c_row_ptrs, c_col_idxs, info, buffer);
-
-        cusparse::destroy(info);
-        cusparse::destroy(d_descr);
-        cusparse::destroy(c_descr);
-        cusparse::destroy(b_descr);
-        cusparse::destroy(a_descr);
-
-#else   // DPCPP_VERSION >= 11000
-        const auto beta = zero<ValueType>();
-        auto spgemm_descr = cusparse::create_spgemm_descr();
-        auto a_descr = cusparse::create_csr(m, k, a_nnz,
-                                            const_cast<IndexType *>(a_row_ptrs),
-                                            const_cast<IndexType *>(a_col_idxs),
-                                            const_cast<ValueType *>(a_vals));
-        auto b_descr = cusparse::create_csr(k, n, b_nnz,
-                                            const_cast<IndexType *>(b_row_ptrs),
-                                            const_cast<IndexType *>(b_col_idxs),
-                                            const_cast<ValueType *>(b_vals));
-        auto c_descr = cusparse::create_csr(m, n, zero_nnz, null_index,
-                                            null_index, null_value);
-
-        // estimate work
-        size_type buffer1_size{};
-        cusparse::spgemm_work_estimation(handle, &alpha, a_descr, b_descr,
-                                         &beta, c_descr, spgemm_descr,
-                                         buffer1_size, nullptr);
-        Array<char> buffer1{exec, buffer1_size};
-        cusparse::spgemm_work_estimation(handle, &alpha, a_descr, b_descr,
-                                         &beta, c_descr, spgemm_descr,
-                                         buffer1_size, buffer1.get_data());
-
-        // compute spgemm
-        size_type buffer2_size{};
-        cusparse::spgemm_compute(handle, &alpha, a_descr, b_descr, &beta,
-                                 c_descr, spgemm_descr, buffer1.get_data(),
-                                 buffer2_size, nullptr);
-        Array<char> buffer2{exec, buffer2_size};
-        cusparse::spgemm_compute(handle, &alpha, a_descr, b_descr, &beta,
-                                 c_descr, spgemm_descr, buffer1.get_data(),
-                                 buffer2_size, buffer2.get_data());
-
-        // copy data to result
-        auto c_nnz = cusparse::sparse_matrix_nnz(c_descr);
-        c_col_idxs_array.resize_and_reset(c_nnz);
-        c_vals_array.resize_and_reset(c_nnz);
-        cusparse::csr_set_pointers(c_descr, c_row_ptrs,
-                                   c_col_idxs_array.get_data(),
-                                   c_vals_array.get_data());
-
-        cusparse::spgemm_copy(handle, &alpha, a_descr, b_descr, &beta, c_descr,
-                              spgemm_descr);
-
-        cusparse::destroy(c_descr);
-        cusparse::destroy(b_descr);
-        cusparse::destroy(a_descr);
-        cusparse::destroy(spgemm_descr);
-#endif  // DPCPP_VERSION >= 11000
-    } else {
-        GKO_NOT_IMPLEMENTED;
-    }
+    GKO_NOT_IMPLEMENTED;
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEMM_KERNEL);
@@ -2389,142 +2199,7 @@ void advanced_spgemm(std::shared_ptr<const DpcppExecutor> exec,
                      const matrix::Csr<ValueType, IndexType> *d,
                      matrix::Csr<ValueType, IndexType> *c)
 {
-    if (cusparse::is_supported<ValueType, IndexType>::value) {
-        auto handle = exec->get_cusparse_handle();
-        cusparse::pointer_mode_guard pm_guard(handle);
-
-        auto valpha = exec->copy_val_to_host(alpha->get_const_values());
-        auto a_nnz = IndexType(a->get_num_stored_elements());
-        auto a_vals = a->get_const_values();
-        auto a_row_ptrs = a->get_const_row_ptrs();
-        auto a_col_idxs = a->get_const_col_idxs();
-        auto b_nnz = IndexType(b->get_num_stored_elements());
-        auto b_vals = b->get_const_values();
-        auto b_row_ptrs = b->get_const_row_ptrs();
-        auto b_col_idxs = b->get_const_col_idxs();
-        auto vbeta = exec->copy_val_to_host(beta->get_const_values());
-        auto d_nnz = IndexType(d->get_num_stored_elements());
-        auto d_vals = d->get_const_values();
-        auto d_row_ptrs = d->get_const_row_ptrs();
-        auto d_col_idxs = d->get_const_col_idxs();
-        auto m = IndexType(a->get_size()[0]);
-        auto n = IndexType(b->get_size()[1]);
-        auto k = IndexType(a->get_size()[1]);
-        auto c_row_ptrs = c->get_row_ptrs();
-
-#if defined(DPCPP_VERSION) && (DPCPP_VERSION < 11000)
-        matrix::CsrBuilder<ValueType, IndexType> c_builder{c};
-        auto &c_col_idxs_array = c_builder.get_col_idx_array();
-        auto &c_vals_array = c_builder.get_value_array();
-        auto a_descr = cusparse::create_mat_descr();
-        auto b_descr = cusparse::create_mat_descr();
-        auto c_descr = cusparse::create_mat_descr();
-        auto d_descr = cusparse::create_mat_descr();
-        auto info = cusparse::create_spgemm_info();
-        // allocate buffer
-        size_type buffer_size{};
-        cusparse::spgemm_buffer_size(
-            handle, m, n, k, &valpha, a_descr, a_nnz, a_row_ptrs, a_col_idxs,
-            b_descr, b_nnz, b_row_ptrs, b_col_idxs, &vbeta, d_descr, d_nnz,
-            d_row_ptrs, d_col_idxs, info, buffer_size);
-        Array<char> buffer_array(exec, buffer_size);
-        auto buffer = buffer_array.get_data();
-
-        // count nnz
-        IndexType c_nnz{};
-        cusparse::spgemm_nnz(handle, m, n, k, a_descr, a_nnz, a_row_ptrs,
-                             a_col_idxs, b_descr, b_nnz, b_row_ptrs, b_col_idxs,
-                             d_descr, d_nnz, d_row_ptrs, d_col_idxs, c_descr,
-                             c_row_ptrs, &c_nnz, info, buffer);
-
-        // accumulate non-zeros
-        c_col_idxs_array.resize_and_reset(c_nnz);
-        c_vals_array.resize_and_reset(c_nnz);
-        auto c_col_idxs = c_col_idxs_array.get_data();
-        auto c_vals = c_vals_array.get_data();
-        cusparse::spgemm(handle, m, n, k, &valpha, a_descr, a_nnz, a_vals,
-                         a_row_ptrs, a_col_idxs, b_descr, b_nnz, b_vals,
-                         b_row_ptrs, b_col_idxs, &vbeta, d_descr, d_nnz, d_vals,
-                         d_row_ptrs, d_col_idxs, c_descr, c_vals, c_row_ptrs,
-                         c_col_idxs, info, buffer);
-
-        cusparse::destroy(info);
-        cusparse::destroy(d_descr);
-        cusparse::destroy(c_descr);
-        cusparse::destroy(b_descr);
-        cusparse::destroy(a_descr);
-#else   // DPCPP_VERSION >= 11000
-        auto null_value = static_cast<ValueType *>(nullptr);
-        auto null_index = static_cast<IndexType *>(nullptr);
-        auto one_val = one<ValueType>();
-        auto zero_val = zero<ValueType>();
-        auto zero_nnz = IndexType{};
-        auto spgemm_descr = cusparse::create_spgemm_descr();
-        auto a_descr = cusparse::create_csr(m, k, a_nnz,
-                                            const_cast<IndexType *>(a_row_ptrs),
-                                            const_cast<IndexType *>(a_col_idxs),
-                                            const_cast<ValueType *>(a_vals));
-        auto b_descr = cusparse::create_csr(k, n, b_nnz,
-                                            const_cast<IndexType *>(b_row_ptrs),
-                                            const_cast<IndexType *>(b_col_idxs),
-                                            const_cast<ValueType *>(b_vals));
-        auto c_descr = cusparse::create_csr(m, n, zero_nnz, null_index,
-                                            null_index, null_value);
-
-        // estimate work
-        size_type buffer1_size{};
-        cusparse::spgemm_work_estimation(handle, &one_val, a_descr, b_descr,
-                                         &zero_val, c_descr, spgemm_descr,
-                                         buffer1_size, nullptr);
-        Array<char> buffer1{exec, buffer1_size};
-        cusparse::spgemm_work_estimation(handle, &one_val, a_descr, b_descr,
-                                         &zero_val, c_descr, spgemm_descr,
-                                         buffer1_size, buffer1.get_data());
-
-        // compute spgemm
-        size_type buffer2_size{};
-        cusparse::spgemm_compute(handle, &one_val, a_descr, b_descr, &zero_val,
-                                 c_descr, spgemm_descr, buffer1.get_data(),
-                                 buffer2_size, nullptr);
-        Array<char> buffer2{exec, buffer2_size};
-        cusparse::spgemm_compute(handle, &one_val, a_descr, b_descr, &zero_val,
-                                 c_descr, spgemm_descr, buffer1.get_data(),
-                                 buffer2_size, buffer2.get_data());
-
-        // write result to temporary storage
-        auto c_tmp_nnz = cusparse::sparse_matrix_nnz(c_descr);
-        Array<IndexType> c_tmp_row_ptrs_array(exec, m + 1);
-        Array<IndexType> c_tmp_col_idxs_array(exec, c_tmp_nnz);
-        Array<ValueType> c_tmp_vals_array(exec, c_tmp_nnz);
-        cusparse::csr_set_pointers(c_descr, c_tmp_row_ptrs_array.get_data(),
-                                   c_tmp_col_idxs_array.get_data(),
-                                   c_tmp_vals_array.get_data());
-
-        cusparse::spgemm_copy(handle, &one_val, a_descr, b_descr, &zero_val,
-                              c_descr, spgemm_descr);
-
-        cusparse::destroy(c_descr);
-        cusparse::destroy(b_descr);
-        cusparse::destroy(a_descr);
-        cusparse::destroy(spgemm_descr);
-
-        auto spgeam_total_nnz = c_tmp_nnz + d->get_num_stored_elements();
-        auto nnz_per_row = spgeam_total_nnz / m;
-        select_spgeam(
-            spgeam_kernels(),
-            [&](int compiled_subwarp_size) {
-                return compiled_subwarp_size >= nnz_per_row ||
-                       compiled_subwarp_size == config::warp_size;
-            },
-            syn::value_list<int>(), syn::type_list<>(), exec,
-            alpha->get_const_values(), c_tmp_row_ptrs_array.get_const_data(),
-            c_tmp_col_idxs_array.get_const_data(),
-            c_tmp_vals_array.get_const_data(), beta->get_const_values(),
-            d_row_ptrs, d_col_idxs, d_vals, c);
-#endif  // DPCPP_VERSION >= 11000
-    } else {
-        GKO_NOT_IMPLEMENTED;
-    }
+    GKO_NOT_IMPLEMENTED;
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -2766,44 +2441,7 @@ void transpose(std::shared_ptr<const DpcppExecutor> exec,
                const matrix::Csr<ValueType, IndexType> *orig,
                matrix::Csr<ValueType, IndexType> *trans)
 {
-    if (cusparse::is_supported<ValueType, IndexType>::value) {
-#if defined(DPCPP_VERSION) && (DPCPP_VERSION < 11000)
-        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
-        oneapi::mkl::index_base idxBase = oneapi::mkl::index_base::zero;
-
-        cusparse::transpose(
-            exec->get_cusparse_handle(), orig->get_size()[0],
-            orig->get_size()[1], orig->get_num_stored_elements(),
-            orig->get_const_values(), orig->get_const_row_ptrs(),
-            orig->get_const_col_idxs(), trans->get_values(),
-            trans->get_row_ptrs(), trans->get_col_idxs(), copyValues, idxBase);
-#else  // DPCPP_VERSION >= 11000
-        dpcppDataType_t cu_value =
-            gko::kernels::dpcpp::dpcpp_data_type<ValueType>();
-        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
-        cusparseIndexBase_t idxBase = CUSPARSE_INDEX_BASE_ZERO;
-        cusparseCsr2CscAlg_t alg = CUSPARSE_CSR2CSC_ALG1;
-        size_type buffer_size = 0;
-        cusparse::transpose_buffersize(
-            exec->get_cusparse_handle(), orig->get_size()[0],
-            orig->get_size()[1], orig->get_num_stored_elements(),
-            orig->get_const_values(), orig->get_const_row_ptrs(),
-            orig->get_const_col_idxs(), trans->get_values(),
-            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value, copyValues,
-            idxBase, alg, &buffer_size);
-        Array<char> buffer_array(exec, buffer_size);
-        auto buffer = buffer_array.get_data();
-        cusparse::transpose(
-            exec->get_cusparse_handle(), orig->get_size()[0],
-            orig->get_size()[1], orig->get_num_stored_elements(),
-            orig->get_const_values(), orig->get_const_row_ptrs(),
-            orig->get_const_col_idxs(), trans->get_values(),
-            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value, copyValues,
-            idxBase, alg, buffer);
-#endif
-    } else {
-        GKO_NOT_IMPLEMENTED;
-    }
+    GKO_NOT_IMPLEMENTED;
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_TRANSPOSE_KERNEL);
@@ -2814,51 +2452,7 @@ void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Csr<ValueType, IndexType> *orig,
                     matrix::Csr<ValueType, IndexType> *trans)
 {
-    if (cusparse::is_supported<ValueType, IndexType>::value) {
-        const dim3 block_size(default_block_size, 1, 1);
-        const dim3 grid_size(
-            ceildiv(trans->get_num_stored_elements(), block_size.x), 1, 1);
-
-#if defined(DPCPP_VERSION) && (DPCPP_VERSION < 11000)
-        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
-        oneapi::mkl::index_base idxBase = oneapi::mkl::index_base::zero;
-
-        cusparse::transpose(
-            exec->get_cusparse_handle(), orig->get_size()[0],
-            orig->get_size()[1], orig->get_num_stored_elements(),
-            orig->get_const_values(), orig->get_const_row_ptrs(),
-            orig->get_const_col_idxs(), trans->get_values(),
-            trans->get_row_ptrs(), trans->get_col_idxs(), copyValues, idxBase);
-#else  // DPCPP_VERSION >= 11000
-        dpcppDataType_t cu_value =
-            gko::kernels::dpcpp::dpcpp_data_type<ValueType>();
-        cusparseAction_t copyValues = CUSPARSE_ACTION_NUMERIC;
-        cusparseIndexBase_t idxBase = CUSPARSE_INDEX_BASE_ZERO;
-        cusparseCsr2CscAlg_t alg = CUSPARSE_CSR2CSC_ALG1;
-        size_type buffer_size = 0;
-        cusparse::transpose_buffersize(
-            exec->get_cusparse_handle(), orig->get_size()[0],
-            orig->get_size()[1], orig->get_num_stored_elements(),
-            orig->get_const_values(), orig->get_const_row_ptrs(),
-            orig->get_const_col_idxs(), trans->get_values(),
-            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value, copyValues,
-            idxBase, alg, &buffer_size);
-        Array<char> buffer_array(exec, buffer_size);
-        auto buffer = buffer_array.get_data();
-        cusparse::transpose(
-            exec->get_cusparse_handle(), orig->get_size()[0],
-            orig->get_size()[1], orig->get_num_stored_elements(),
-            orig->get_const_values(), orig->get_const_row_ptrs(),
-            orig->get_const_col_idxs(), trans->get_values(),
-            trans->get_row_ptrs(), trans->get_col_idxs(), cu_value, copyValues,
-            idxBase, alg, buffer);
-#endif
-
-        conjugate_kernel(grid_size, block_size, 0, exec->get_queue(),
-                         trans->get_num_stored_elements(), trans->get_values());
-    } else {
-        GKO_NOT_IMPLEMENTED;
-    }
+    GKO_NOT_IMPLEMENTED;
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -3069,51 +2663,7 @@ template <typename ValueType, typename IndexType>
 void sort_by_column_index(std::shared_ptr<const DpcppExecutor> exec,
                           matrix::Csr<ValueType, IndexType> *to_sort)
 {
-    if (cusparse::is_supported<ValueType, IndexType>::value) {
-        auto handle = exec->get_cusparse_handle();
-        auto descr = cusparse::create_mat_descr();
-        auto m = IndexType(to_sort->get_size()[0]);
-        auto n = IndexType(to_sort->get_size()[1]);
-        auto nnz = IndexType(to_sort->get_num_stored_elements());
-        auto row_ptrs = to_sort->get_const_row_ptrs();
-        auto col_idxs = to_sort->get_col_idxs();
-        auto vals = to_sort->get_values();
-
-        // copy values
-        Array<ValueType> tmp_vals_array(exec, nnz);
-        exec->copy(nnz, vals, tmp_vals_array.get_data());
-        auto tmp_vals = tmp_vals_array.get_const_data();
-
-        // init identity permutation
-        Array<IndexType> permutation_array(exec, nnz);
-        auto permutation = permutation_array.get_data();
-        cusparse::create_identity_permutation(handle, nnz, permutation);
-
-        // allocate buffer
-        size_type buffer_size{};
-        cusparse::csrsort_buffer_size(handle, m, n, nnz, row_ptrs, col_idxs,
-                                      buffer_size);
-        Array<char> buffer_array{exec, buffer_size};
-        auto buffer = buffer_array.get_data();
-
-        // sort column indices
-        cusparse::csrsort(handle, m, n, nnz, descr, row_ptrs, col_idxs,
-                          permutation, buffer);
-
-        // sort values
-#if defined(DPCPP_VERSION) && (DPCPP_VERSION < 11000)
-        cusparse::gather(handle, nnz, tmp_vals, vals, permutation);
-#else  // DPCPP_VERSION >= 11000
-        auto val_vec = cusparse::create_spvec(nnz, nnz, permutation, vals);
-        auto tmp_vec =
-            cusparse::create_dnvec(nnz, const_cast<ValueType *>(tmp_vals));
-        cusparse::gather(handle, tmp_vec, val_vec);
-#endif
-
-        cusparse::destroy(descr);
-    } else {
-        GKO_NOT_IMPLEMENTED;
-    }
+    GKO_NOT_IMPLEMENTED;
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
diff --git a/dpcpp/matrix/diagonal_kernels.dp.cpp b/dpcpp/matrix/diagonal_kernels.dp.cpp
index 8bd2b937a35..737283291d2 100644
--- a/dpcpp/matrix/diagonal_kernels.dp.cpp
+++ b/dpcpp/matrix/diagonal_kernels.dp.cpp
@@ -30,6 +30,9 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include "core/matrix/diagonal_kernels.hpp"
+
+
 #include <CL/sycl.hpp>
 
 
@@ -54,7 +57,7 @@ namespace dpcpp {
 namespace diagonal {
 
 
-constexpr auto default_block_size = 512;
+constexpr auto default_block_size = 256;
 
 
 namespace kernel {
@@ -254,10 +257,6 @@ void conj_transpose(size_type size, const ValueType *__restrict__ orig_values,
         return;
     }
 
-    /*
-    DPCT1007:5: Migration of this CUDA API is not supported by the Intel(R)
-    DPC++ Compatibility Tool.
-    */
     trans_values[tidx] = conj(orig_values[tidx]);
 }
 
diff --git a/dpcpp/matrix/hybrid_kernels.dp.cpp b/dpcpp/matrix/hybrid_kernels.dp.cpp
index a8a92eee7bb..5f2b4ab906d 100644
--- a/dpcpp/matrix/hybrid_kernels.dp.cpp
+++ b/dpcpp/matrix/hybrid_kernels.dp.cpp
@@ -30,6 +30,9 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include "core/matrix/hybrid_kernels.hpp"
+
+
 #include <CL/sycl.hpp>
 
 
@@ -62,7 +65,7 @@ namespace dpcpp {
 namespace hybrid {
 
 
-constexpr int default_block_size = 512;
+constexpr int default_block_size = 256;
 constexpr int warps_in_block = 4;
 
 
diff --git a/dpcpp/matrix/sellp_kernels.dp.cpp b/dpcpp/matrix/sellp_kernels.dp.cpp
index 0cc5973975b..7a9a218224b 100644
--- a/dpcpp/matrix/sellp_kernels.dp.cpp
+++ b/dpcpp/matrix/sellp_kernels.dp.cpp
@@ -30,6 +30,9 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include "core/matrix/sellp_kernels.hpp"
+
+
 #include <CL/sycl.hpp>
 
 
@@ -43,7 +46,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/prefix_sum.hpp"
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
-#include "dpcpp/base/onemkl_bindings.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/reduction.dp.hpp"
 #include "dpcpp/components/thread_ids.dp.hpp"
@@ -60,7 +62,7 @@ namespace dpcpp {
 namespace sellp {
 
 
-constexpr auto default_block_size = 512;
+constexpr auto default_block_size = 256;
 
 
 namespace {
@@ -275,7 +277,7 @@ void count_nnz_per_row(size_type num_rows, size_type slice_size,
                 part_result += 1;
             }
         }
-        result[row_idx] = reduce(
+        result[row_idx] = ::gko::kernels::dpcpp::reduce(
             warp_tile, part_result,
             [](const size_type &a, const size_type &b) { return a + b; });
     }
diff --git a/dpcpp/matrix/sparsity_csr_kernels.dp.cpp b/dpcpp/matrix/sparsity_csr_kernels.dp.cpp
index 9d0afae20b7..7dc935e83a5 100644
--- a/dpcpp/matrix/sparsity_csr_kernels.dp.cpp
+++ b/dpcpp/matrix/sparsity_csr_kernels.dp.cpp
@@ -30,6 +30,9 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include "core/matrix/sparsity_csr_kernels.hpp"
+
+
 #include <ginkgo/core/base/exception_helpers.hpp>
 
 

From 63910f14bca99c8c90cc38a0a88584860014c2b2 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 22 Jul 2021 23:03:14 +0200
Subject: [PATCH 0978/1008] component fix

---
 dpcpp/components/format_conversion.dp.hpp | 18 ++++--------------
 dpcpp/components/intrinsics.dp.hpp        | 14 ++++----------
 dpcpp/components/segment_scan.dp.hpp      |  7 ++++---
 3 files changed, 12 insertions(+), 27 deletions(-)

diff --git a/dpcpp/components/format_conversion.dp.hpp b/dpcpp/components/format_conversion.dp.hpp
index 16624130b34..4eb5d7fcf4c 100644
--- a/dpcpp/components/format_conversion.dp.hpp
+++ b/dpcpp/components/format_conversion.dp.hpp
@@ -37,7 +37,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <CL/sycl.hpp>
 
 
-#include <ginkgo/config.hpp>
 #include <ginkgo/core/base/executor.hpp>
 
 
@@ -51,11 +50,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif  // GINKGO_BENCHMARK_ENABLE_TUNING
 
 
-#include "dpcpp/base/dim3.dp.hpp"
-#include "dpcpp/components/cooperative_groups.dp.hpp"
-#include "dpcpp/components/thread_ids.dp.hpp"
-
-
 namespace gko {
 namespace kernels {
 namespace dpcpp {
@@ -75,6 +69,7 @@ void count_nnz_per_row(dim3 grid, dim3 block, size_type dynamic_shared_memory,
                        const ValueType *values, IndexType *result);
 
 
+
 }  // namespace kernel
 }  // namespace ell
 
@@ -96,6 +91,7 @@ void convert_row_idxs_to_ptrs(dim3 grid, dim3 block,
                               size_type length);
 
 
+
 }  // namespace kernel
 
 }  // namespace kernel
@@ -113,17 +109,11 @@ template <size_type subwarp_size = config::warp_size>
 size_type calculate_nwarps(std::shared_ptr<const DpcppExecutor> exec,
                            const size_type nnz)
 {
-    size_type warps_per_sm =
-        exec->get_num_warps_per_sm() * config::warp_size / subwarp_size;
-    size_type nwarps_in_dpcpp = exec->get_num_multiprocessor() * warps_per_sm;
+    size_type nwarps_in_dpcpp = exec->get_num_computing_units() * 7;
     size_type multiple = 8;
     if (nnz >= 2e8) {
-        multiple = 2048;
+        multiple = 256;
     } else if (nnz >= 2e7) {
-        multiple = 512;
-    } else if (nnz >= 2e6) {
-        multiple = 128;
-    } else if (nnz >= 2e5) {
         multiple = 32;
     }
 #ifdef GINKGO_BENCHMARK_ENABLE_TUNING
diff --git a/dpcpp/components/intrinsics.dp.hpp b/dpcpp/components/intrinsics.dp.hpp
index 291147eac2d..68f936e09cb 100644
--- a/dpcpp/components/intrinsics.dp.hpp
+++ b/dpcpp/components/intrinsics.dp.hpp
@@ -63,19 +63,13 @@ __dpct_inline__ int popcnt(uint64 mask) { return sycl::popcount(mask); }
  * Returns the (1-based!) index of the first set bit in the given mask,
  * starting from the least significant bit.
  */
-/*
-DPCT1004:0: Compatible DPC++ code could not be generated.
-*/
-__dpct_inline__ int ffs(uint32 mask) { return __ffs(mask); }
+__dpct_inline__ int ffs(uint32 mask) { return __builtin_ffs(mask); }
 
 /** @copydoc ffs */
 __dpct_inline__ int ffs(uint64 mask)
 {
     // the cast is necessary, as the overloads defined by HIP are ambiguous
-    /*
-    DPCT1004:1: Compatible DPC++ code could not be generated.
-    */
-    return __ffsll(static_cast<unsigned long long int>(mask));
+    return __builtin_ffsll(static_cast<unsigned long long int>(mask));
 }
 
 
@@ -84,10 +78,10 @@ __dpct_inline__ int ffs(uint64 mask)
  * Returns the number of zero bits before the first set bit in the given mask,
  * starting from the most significant bit.
  */
-__dpct_inline__ int clz(uint32 mask) { return sycl::clz((int)mask); }
+__dpct_inline__ int clz(uint32 mask) { return __builtin_clz(mask); }
 
 /** @copydoc clz */
-__dpct_inline__ int clz(uint64 mask) { return sycl::clz((long long)mask); }
+__dpct_inline__ int clz(uint64 mask) { return __builtin_clzll(mask); }
 
 
 }  // namespace dpcpp
diff --git a/dpcpp/components/segment_scan.dp.hpp b/dpcpp/components/segment_scan.dp.hpp
index b1ca309f4b6..0985bc09739 100644
--- a/dpcpp/components/segment_scan.dp.hpp
+++ b/dpcpp/components/segment_scan.dp.hpp
@@ -37,6 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <CL/sycl.hpp>
 
 
+#include "dpcpp/base/dim3.dp.hpp"
 #include "dpcpp/base/dpct.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 
@@ -56,21 +57,21 @@ namespace dpcpp {
 template <unsigned subwarp_size, typename ValueType, typename IndexType>
 __dpct_inline__ bool segment_scan(
     const group::thread_block_tile<subwarp_size> &group, const IndexType ind,
-    ValueType *__restrict__ val, sycl::nd_item<3> item_ct1)
+    ValueType *__restrict__ val)
 {
     bool head = true;
 #pragma unroll
     for (int i = 1; i < subwarp_size; i <<= 1) {
         const IndexType add_ind = group.shfl_up(ind, i);
         ValueType add_val = zero<ValueType>();
-        if (add_ind == ind && item_ct1.get_local_id(2) >= i) {
+        if (add_ind == ind && group.thread_rank() >= i) {
             add_val = *val;
             if (i == 1) {
                 head = false;
             }
         }
         add_val = group.shfl_down(add_val, i);
-        if (item_ct1.get_local_id(2) < subwarp_size - i) {
+        if (group.thread_rank() < subwarp_size - i) {
             *val += add_val;
         }
     }

From 0c935d53e3d72d799c1ff7bacc67be24082969c3 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 22 Jul 2021 23:45:39 +0200
Subject: [PATCH 0979/1008] fix

---
 dpcpp/components/format_conversion.dp.hpp |   3 -
 dpcpp/matrix/coo_kernels.dp.cpp           |   2 +-
 dpcpp/matrix/csr_kernels.dp.cpp           | 781 ++++++++++++++++------
 dpcpp/matrix/ell_kernels.dp.cpp           |   9 +
 dpcpp/matrix/fbcsr_kernels.dp.cpp         |   3 +
 dpcpp/test/matrix/CMakeLists.txt          |   6 +-
 dpcpp/test/matrix/coo_kernels.cpp         |  69 +-
 dpcpp/test/matrix/csr_kernels.cpp         | 149 +++--
 dpcpp/test/matrix/diagonal_kernels.cpp    |  29 +-
 dpcpp/test/matrix/fbcsr_kernels.cpp       |   9 +-
 dpcpp/test/matrix/hybrid_kernels.cpp      |  43 +-
 dpcpp/test/matrix/sellp_kernels.cpp       |  59 +-
 include/ginkgo/core/matrix/csr.hpp        |  62 +-
 13 files changed, 840 insertions(+), 384 deletions(-)

diff --git a/dpcpp/components/format_conversion.dp.hpp b/dpcpp/components/format_conversion.dp.hpp
index 4eb5d7fcf4c..281677f4b98 100644
--- a/dpcpp/components/format_conversion.dp.hpp
+++ b/dpcpp/components/format_conversion.dp.hpp
@@ -69,7 +69,6 @@ void count_nnz_per_row(dim3 grid, dim3 block, size_type dynamic_shared_memory,
                        const ValueType *values, IndexType *result);
 
 
-
 }  // namespace kernel
 }  // namespace ell
 
@@ -91,10 +90,8 @@ void convert_row_idxs_to_ptrs(dim3 grid, dim3 block,
                               size_type length);
 
 
-
 }  // namespace kernel
 
-}  // namespace kernel
 
 namespace host_kernel {
 
diff --git a/dpcpp/matrix/coo_kernels.dp.cpp b/dpcpp/matrix/coo_kernels.dp.cpp
index 3a48635fef0..8f22e4a67c8 100644
--- a/dpcpp/matrix/coo_kernels.dp.cpp
+++ b/dpcpp/matrix/coo_kernels.dp.cpp
@@ -46,7 +46,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/dense_kernels.hpp"
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
-#include "dpcpp/base/onemkl_bindings.hpp"
 #include "dpcpp/components/atomic.dp.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/format_conversion.dp.hpp"
@@ -375,6 +374,7 @@ void convert_row_idxs_to_ptrs(const IndexType *__restrict__ idxs,
     }
 }
 
+template <typename IndexType>
 void convert_row_idxs_to_ptrs(dim3 grid, dim3 block,
                               gko::size_type dynamic_shared_memory,
                               sycl::queue *stream, const IndexType *idxs,
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index 3c110f303f6..a69604f8341 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -50,6 +50,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/sellp.hpp>
 
 
+#include "core/base/utils.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/components/prefix_sum.hpp"
 #include "core/matrix/csr_builder.hpp"
@@ -58,11 +59,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
 #include "dpcpp/base/dpct.hpp"
-#include "dpcpp/base/onemkl_bindings.hpp"
 #include "dpcpp/components/atomic.dp.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
-#include "dpcpp/components/intrinsics.dp.hpp"
-#include "dpcpp/components/merging.dp.hpp"
 #include "dpcpp/components/reduction.dp.hpp"
 #include "dpcpp/components/segment_scan.dp.hpp"
 #include "dpcpp/components/thread_ids.dp.hpp"
@@ -95,8 +93,6 @@ using compiled_kernels = syn::value_list<int, 6>;
 
 using classical_kernels = syn::value_list<int, config::warp_size, 16, 8, 1>;
 
-using spgeam_kernels = syn::value_list<int, 8, 16>;
-
 
 namespace kernel {
 
@@ -540,7 +536,8 @@ void abstract_merge_path_spmv(dim3 grid, dim3 block,
                 abstract_merge_path_spmv<items_per_thread>(
                     num_rows, val, col_idxs, row_ptrs, srow, b, b_stride, c,
                     c_stride, row_out, val_out, item_ct1,
-                    shared_row_ptrs_acc_ct1.get_pointer().get());
+                    static_cast<IndexType *>(
+                        shared_row_ptrs_acc_ct1.get_pointer()));
             });
     });
 }
@@ -586,7 +583,8 @@ void abstract_merge_path_spmv(
                 abstract_merge_path_spmv<items_per_thread>(
                     num_rows, alpha, val, col_idxs, row_ptrs, srow, b, b_stride,
                     beta, c, c_stride, row_out, val_out, item_ct1,
-                    (IndexType *)shared_row_ptrs_acc_ct1.get_pointer());
+                    static_cast<IndexType *>(
+                        shared_row_ptrs_acc_ct1.get_pointer()));
             });
     });
 }
@@ -785,145 +783,6 @@ void abstract_classical_spmv(dim3 grid, dim3 block,
 }
 
 
-template <int subwarp_size, typename IndexType>
-void spgeam_nnz(const IndexType *__restrict__ a_row_ptrs,
-                const IndexType *__restrict__ a_col_idxs,
-                const IndexType *__restrict__ b_row_ptrs,
-                const IndexType *__restrict__ b_col_idxs, IndexType num_rows,
-                IndexType *__restrict__ nnz, sycl::nd_item<3> item_ct1)
-{
-    const auto row =
-        thread::get_subwarp_id_flat<subwarp_size, IndexType>(item_ct1);
-    auto subwarp = group::tiled_partition<subwarp_size>(
-        group::this_thread_block(item_ct1));
-    if (row >= num_rows) {
-        return;
-    }
-
-    const auto a_begin = a_row_ptrs[row];
-    const auto b_begin = b_row_ptrs[row];
-    const auto a_size = a_row_ptrs[row + 1] - a_begin;
-    const auto b_size = b_row_ptrs[row + 1] - b_begin;
-    IndexType count{};
-    group_merge<subwarp_size>(
-        a_col_idxs + a_begin, a_size, b_col_idxs + b_begin, b_size, subwarp,
-        [&](IndexType, IndexType a_col, IndexType, IndexType b_col, IndexType,
-            bool valid) {
-            count += popcnt(subwarp.ballot(a_col != b_col && valid));
-            return true;
-        });
-
-    if (subwarp.thread_rank() == 0) {
-        nnz[row] = count;
-    }
-}
-
-template <int subwarp_size, typename IndexType>
-void spgeam_nnz(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                sycl::queue *stream, const IndexType *a_row_ptrs,
-                const IndexType *a_col_idxs, const IndexType *b_row_ptrs,
-                const IndexType *b_col_idxs, IndexType num_rows, IndexType *nnz)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                spgeam_nnz<subwarp_size>(a_row_ptrs, a_col_idxs, b_row_ptrs,
-                                         b_col_idxs, num_rows, nnz, item_ct1);
-            });
-    });
-}
-
-
-template <int subwarp_size, typename ValueType, typename IndexType>
-void spgeam(const ValueType *__restrict__ palpha,
-            const IndexType *__restrict__ a_row_ptrs,
-            const IndexType *__restrict__ a_col_idxs,
-            const ValueType *__restrict__ a_vals,
-            const ValueType *__restrict__ pbeta,
-            const IndexType *__restrict__ b_row_ptrs,
-            const IndexType *__restrict__ b_col_idxs,
-            const ValueType *__restrict__ b_vals, IndexType num_rows,
-            const IndexType *__restrict__ c_row_ptrs,
-            IndexType *__restrict__ c_col_idxs, ValueType *__restrict__ c_vals,
-            sycl::nd_item<3> item_ct1)
-{
-    const auto row =
-        thread::get_subwarp_id_flat<subwarp_size, IndexType>(item_ct1);
-    auto subwarp = group::tiled_partition<subwarp_size>(
-        group::this_thread_block(item_ct1));
-    if (row >= num_rows) {
-        return;
-    }
-
-    const auto alpha = palpha[0];
-    const auto beta = pbeta[0];
-    const auto lane = static_cast<IndexType>(subwarp.thread_rank());
-    constexpr auto lanemask_full =
-        ~config::lane_mask_type{} >> (config::warp_size - subwarp_size);
-    const auto lanemask_eq = config::lane_mask_type{1} << lane;
-    const auto lanemask_lt = lanemask_eq - 1;
-
-    const auto a_begin = a_row_ptrs[row];
-    const auto b_begin = b_row_ptrs[row];
-    const auto a_size = a_row_ptrs[row + 1] - a_begin;
-    const auto b_size = b_row_ptrs[row + 1] - b_begin;
-    auto c_begin = c_row_ptrs[row];
-    bool skip_first{};
-    group_merge<subwarp_size>(
-        a_col_idxs + a_begin, a_size, b_col_idxs + b_begin, b_size, subwarp,
-        [&](IndexType a_nz, IndexType a_col, IndexType b_nz, IndexType b_col,
-            IndexType, bool valid) {
-            auto c_col = min(a_col, b_col);
-            auto equal_mask = subwarp.ballot(a_col == b_col && valid);
-            // check if the elements in the previous merge step are
-            // equal
-            auto prev_equal_mask = equal_mask << 1 | skip_first;
-            // store the highest bit for the next group_merge_step
-            skip_first = bool(equal_mask >> (subwarp_size - 1));
-            auto prev_equal = bool(prev_equal_mask & lanemask_eq);
-            // only output an entry if the previous cols weren't equal.
-            // if they were equal, they were both handled in the
-            // previous step
-            if (valid && !prev_equal) {
-                auto c_ofs = popcnt(~prev_equal_mask & lanemask_lt);
-                c_col_idxs[c_begin + c_ofs] = c_col;
-                auto a_val =
-                    a_col <= b_col ? a_vals[a_nz + a_begin] : zero<ValueType>();
-                auto b_val =
-                    b_col <= a_col ? b_vals[b_nz + b_begin] : zero<ValueType>();
-                c_vals[c_begin + c_ofs] = alpha * a_val + beta * b_val;
-            }
-            // advance by the number of merged elements
-            // in theory, we would need to mask by `valid`, but this
-            // would only be false somwhere in the last iteration, where
-            // we don't need the value of c_begin afterwards, anyways.
-            c_begin += popcnt(~prev_equal_mask & lanemask_full);
-            return true;
-        });
-}
-
-template <int subwarp_size, typename ValueType, typename IndexType>
-void spgeam(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-            sycl::queue *stream, const ValueType *palpha,
-            const IndexType *a_row_ptrs, const IndexType *a_col_idxs,
-            const ValueType *a_vals, const ValueType *pbeta,
-            const IndexType *b_row_ptrs, const IndexType *b_col_idxs,
-            const ValueType *b_vals, IndexType num_rows,
-            const IndexType *c_row_ptrs, IndexType *c_col_idxs,
-            ValueType *c_vals)
-{
-    stream->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                spgeam<subwarp_size>(palpha, a_row_ptrs, a_col_idxs, a_vals,
-                                     pbeta, b_row_ptrs, b_col_idxs, b_vals,
-                                     num_rows, c_row_ptrs, c_col_idxs, c_vals,
-                                     item_ct1);
-            });
-    });
-}
-
-
 template <typename IndexType>
 void convert_row_ptrs_to_idxs(size_type num_rows,
                               const IndexType *__restrict__ ptrs,
@@ -2136,58 +1995,265 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_ADVANCED_SPMV_KERNEL);
 
 
+namespace {
+
+
+/**
+ * @internal
+ *
+ * Entry in a heap storing a column index and associated non-zero index
+ * (and row end) from a matrix.
+ *
+ * @tparam ValueType  The value type for matrices.
+ * @tparam IndexType  The index type for matrices.
+ */
 template <typename ValueType, typename IndexType>
-void spgemm(std::shared_ptr<const DpcppExecutor> exec,
-            const matrix::Csr<ValueType, IndexType> *a,
-            const matrix::Csr<ValueType, IndexType> *b,
-            matrix::Csr<ValueType, IndexType> *c)
-{
-    GKO_NOT_IMPLEMENTED;
+struct col_heap_element {
+    using value_type = ValueType;
+    using index_type = IndexType;
+
+    IndexType idx;
+    IndexType end;
+    IndexType col;
+
+    ValueType val() const { return zero<ValueType>(); }
+
+    col_heap_element(IndexType idx, IndexType end, IndexType col, ValueType)
+        : idx{idx}, end{end}, col{col}
+    {}
+};
+
+
+/**
+ * @internal
+ *
+ * Entry in a heap storing an entry (value and column index) and associated
+ * non-zero index (and row end) from a matrix.
+ *
+ * @tparam ValueType  The value type for matrices.
+ * @tparam IndexType  The index type for matrices.
+ */
+template <typename ValueType, typename IndexType>
+struct val_heap_element {
+    using value_type = ValueType;
+    using index_type = IndexType;
+
+    IndexType idx;
+    IndexType end;
+    IndexType col;
+    ValueType val_;
+
+    ValueType val() const { return val_; }
+
+    val_heap_element(IndexType idx, IndexType end, IndexType col, ValueType val)
+        : idx{idx}, end{end}, col{col}, val_{val}
+    {}
+};
+
+
+/**
+ * @internal
+ *
+ * Restores the binary heap condition downwards from a given index.
+ *
+ * The heap condition is: col(child) >= col(parent)
+ *
+ * @param heap  a pointer to the array containing the heap elements.
+ * @param idx  the index of the starting heap node that potentially
+ *             violates the heap condition.
+ * @param size  the number of elements in the heap.
+ * @tparam HeapElement  the element type in the heap. See col_heap_element and
+ *                      val_heap_element
+ */
+template <typename HeapElement>
+void sift_down(HeapElement *heap, typename HeapElement::index_type idx,
+               typename HeapElement::index_type size)
+{
+    auto curcol = heap[idx].col;
+    while (idx * 2 + 1 < size) {
+        auto lchild = idx * 2 + 1;
+        auto rchild = min(lchild + 1, size - 1);
+        auto lcol = heap[lchild].col;
+        auto rcol = heap[rchild].col;
+        auto mincol = min(lcol, rcol);
+        if (mincol >= curcol) {
+            break;
+        }
+        auto minchild = lcol == mincol ? lchild : rchild;
+        std::swap(heap[minchild], heap[idx]);
+        idx = minchild;
+    }
 }
 
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEMM_KERNEL);
 
+/**
+ * @internal
+ *
+ * Generic SpGEMM implementation for a single output row of A * B using binary
+ * heap-based multiway merging.
+ *
+ * @param row  The row for which to compute the SpGEMM
+ * @param a  The input matrix A
+ * @param b  The input matrix B (its column indices must be sorted within each
+ *           row!)
+ * @param heap  The heap to use for this implementation. It must have as many
+ *              entries as the input row has non-zeros.
+ * @param init_cb  function to initialize the state for a single row. Its return
+ *                 value will be updated by subsequent calls of other callbacks,
+ *                 and then returned by this function. Its signature must be
+ *                 compatible with `return_type state = init_cb(row)`.
+ * @param step_cb  function that will be called for each accumulation from an
+ *                 entry of B into the output state. Its signature must be
+ *                 compatible with `step_cb(value, column, state)`.
+ * @param col_cb  function that will be called once for each output column after
+ *                all accumulations into it are completed. Its signature must be
+ *                compatible with `col_cb(column, state)`.
+ * @return the value initialized by init_cb and updated by step_cb and col_cb
+ * @note If the columns of B are not sorted, the output may have duplicate
+ *       column entries.
+ *
+ * @tparam HeapElement  the heap element type. See col_heap_element and
+ *                      val_heap_element
+ * @tparam InitCallback  functor type for init_cb
+ * @tparam StepCallback  functor type for step_cb
+ * @tparam ColCallback  functor type for col_cb
+ */
+template <typename HeapElement, typename InitCallback, typename StepCallback,
+          typename ColCallback>
+auto spgemm_multiway_merge(size_type row,
+                           const typename HeapElement::index_type *a_row_ptrs,
+                           const typename HeapElement::index_type *a_cols,
+                           const typename HeapElement::value_type *a_vals,
+                           const typename HeapElement::index_type *b_row_ptrs,
+                           const typename HeapElement::index_type *b_cols,
+                           const typename HeapElement::value_type *b_vals,
+                           HeapElement *heap, InitCallback init_cb,
+                           StepCallback step_cb, ColCallback col_cb)
+    -> decltype(init_cb(0))
+{
+    auto a_begin = a_row_ptrs[row];
+    auto a_end = a_row_ptrs[row + 1];
+
+    using index_type = typename HeapElement::index_type;
+    constexpr auto sentinel = std::numeric_limits<index_type>::max();
+
+    auto state = init_cb(row);
+
+    // initialize the heap
+    for (auto a_nz = a_begin; a_nz < a_end; ++a_nz) {
+        auto b_row = a_cols[a_nz];
+        auto b_begin = b_row_ptrs[b_row];
+        auto b_end = b_row_ptrs[b_row + 1];
+        heap[a_nz] = {b_begin, b_end,
+                      checked_load(b_cols, b_begin, b_end, sentinel),
+                      a_vals[a_nz]};
+    }
 
-namespace {
+    if (a_begin != a_end) {
+        // make heap:
+        auto a_size = a_end - a_begin;
+        for (auto i = (a_size - 2) / 2; i >= 0; --i) {
+            sift_down(heap + a_begin, i, a_size);
+        }
+        auto &top = heap[a_begin];
+        auto &bot = heap[a_end - 1];
+        auto col = top.col;
+
+        while (top.col != sentinel) {
+            step_cb(b_vals[top.idx] * top.val(), top.col, state);
+            // move to the next element
+            top.idx++;
+            top.col = checked_load(b_cols, top.idx, top.end, sentinel);
+            // restore heap property
+            // pop_heap swaps top and bot, we need to prevent that
+            // so that we do a simple sift_down instead
+            sift_down(heap + a_begin, index_type{}, a_size);
+            if (top.col != col) {
+                col_cb(col, state);
+            }
+            col = top.col;
+        }
+    }
 
+    return state;
+}
 
-template <int subwarp_size, typename ValueType, typename IndexType>
-void spgeam(syn::value_list<int, subwarp_size>,
-            std::shared_ptr<const DefaultExecutor> exec, const ValueType *alpha,
-            const IndexType *a_row_ptrs, const IndexType *a_col_idxs,
-            const ValueType *a_vals, const ValueType *beta,
-            const IndexType *b_row_ptrs, const IndexType *b_col_idxs,
-            const ValueType *b_vals, matrix::Csr<ValueType, IndexType> *c)
-{
-    auto m = static_cast<IndexType>(c->get_size()[0]);
+
+}  // namespace
+
+
+template <typename ValueType, typename IndexType>
+void spgemm(std::shared_ptr<const DpcppExecutor> exec,
+            const matrix::Csr<ValueType, IndexType> *a,
+            const matrix::Csr<ValueType, IndexType> *b,
+            matrix::Csr<ValueType, IndexType> *c)
+{
+    auto num_rows = a->get_size()[0];
+    const auto a_row_ptrs = a->get_const_row_ptrs();
+    const auto a_cols = a->get_const_col_idxs();
+    const auto a_vals = a->get_const_values();
+    const auto b_row_ptrs = b->get_const_row_ptrs();
+    const auto b_cols = b->get_const_col_idxs();
+    const auto b_vals = b->get_const_values();
     auto c_row_ptrs = c->get_row_ptrs();
-    // count nnz for alpha * A + beta * B
-    auto subwarps_per_block = default_block_size / subwarp_size;
-    auto num_blocks = ceildiv(m, subwarps_per_block);
-    kernel::spgeam_nnz<subwarp_size>(num_blocks, default_block_size, 0,
-                                     exec->get_queue(), a_row_ptrs, a_col_idxs,
-                                     b_row_ptrs, b_col_idxs, m, c_row_ptrs);
+    auto queue = exec->get_queue();
+
+    Array<val_heap_element<ValueType, IndexType>> heap_array(
+        exec, a->get_num_stored_elements());
+
+    auto heap = heap_array.get_data();
+    auto col_heap =
+        reinterpret_cast<col_heap_element<ValueType, IndexType> *>(heap);
+
+    // first sweep: count nnz for each row
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto a_row = static_cast<size_type>(idx[0]);
+            c_row_ptrs[a_row] = spgemm_multiway_merge(
+                a_row, a_row_ptrs, a_cols, a_vals, b_row_ptrs, b_cols, b_vals,
+                col_heap, [](size_type) { return IndexType{}; },
+                [](ValueType, IndexType, IndexType &) {},
+                [](IndexType, IndexType &nnz) { nnz++; });
+        });
+    });
 
     // build row pointers
-    components::prefix_sum(exec, c_row_ptrs, m + 1);
+    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
 
-    // accumulate non-zeros for alpha * A + beta * B
+    // second sweep: accumulate non-zeros
+    const auto new_nnz = exec->copy_val_to_host(c_row_ptrs + num_rows);
     matrix::CsrBuilder<ValueType, IndexType> c_builder{c};
-    auto c_nnz = exec->copy_val_to_host(c_row_ptrs + m);
-    c_builder.get_col_idx_array().resize_and_reset(c_nnz);
-    c_builder.get_value_array().resize_and_reset(c_nnz);
-    auto c_col_idxs = c->get_col_idxs();
-    auto c_vals = c->get_values();
-    kernel::spgeam<subwarp_size>(
-        num_blocks, default_block_size, 0, exec->get_queue(), alpha, a_row_ptrs,
-        a_col_idxs, a_vals, beta, b_row_ptrs, b_col_idxs, b_vals, m, c_row_ptrs,
-        c_col_idxs, c_vals);
+    auto &c_col_idxs_array = c_builder.get_col_idx_array();
+    auto &c_vals_array = c_builder.get_value_array();
+    c_col_idxs_array.resize_and_reset(new_nnz);
+    c_vals_array.resize_and_reset(new_nnz);
+    auto c_col_idxs = c_col_idxs_array.get_data();
+    auto c_vals = c_vals_array.get_data();
+
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto a_row = static_cast<size_type>(idx[0]);
+            spgemm_multiway_merge(
+                a_row, a_row_ptrs, a_cols, a_vals, b_row_ptrs, b_cols, b_vals,
+                heap,
+                [&](size_type row) {
+                    return std::make_pair(zero<ValueType>(), c_row_ptrs[row]);
+                },
+                [](ValueType val, IndexType,
+                   std::pair<ValueType, IndexType> &state) {
+                    state.first += val;
+                },
+                [&](IndexType col, std::pair<ValueType, IndexType> &state) {
+                    c_col_idxs[state.second] = col;
+                    c_vals[state.second] = state.first;
+                    state.first = zero<ValueType>();
+                    state.second++;
+                });
+        });
+    });
 }
 
-GKO_ENABLE_IMPLEMENTATION_SELECTION(select_spgeam, spgeam);
-
-
-}  // namespace
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEMM_KERNEL);
 
 
 template <typename ValueType, typename IndexType>
@@ -2199,7 +2265,125 @@ void advanced_spgemm(std::shared_ptr<const DpcppExecutor> exec,
                      const matrix::Csr<ValueType, IndexType> *d,
                      matrix::Csr<ValueType, IndexType> *c)
 {
-    GKO_NOT_IMPLEMENTED;
+    auto num_rows = a->get_size()[0];
+    const auto a_row_ptrs = a->get_const_row_ptrs();
+    const auto a_cols = a->get_const_col_idxs();
+    const auto a_vals = a->get_const_values();
+    const auto b_row_ptrs = b->get_const_row_ptrs();
+    const auto b_cols = b->get_const_col_idxs();
+    const auto b_vals = b->get_const_values();
+    const auto d_row_ptrs = d->get_const_row_ptrs();
+    const auto d_cols = d->get_const_col_idxs();
+    const auto d_vals = d->get_const_values();
+    auto c_row_ptrs = c->get_row_ptrs();
+    const auto alpha_vals = alpha->get_const_values();
+    const auto beta_vals = beta->get_const_values();
+    constexpr auto sentinel = std::numeric_limits<IndexType>::max();
+    auto queue = exec->get_queue();
+
+    // first sweep: count nnz for each row
+
+    Array<val_heap_element<ValueType, IndexType>> heap_array(
+        exec, a->get_num_stored_elements());
+
+    auto heap = heap_array.get_data();
+    auto col_heap =
+        reinterpret_cast<col_heap_element<ValueType, IndexType> *>(heap);
+
+    // first sweep: count nnz for each row
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto a_row = static_cast<size_type>(idx[0]);
+            auto d_nz = d_row_ptrs[a_row];
+            const auto d_end = d_row_ptrs[a_row + 1];
+            auto d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+            c_row_ptrs[a_row] = spgemm_multiway_merge(
+                a_row, a_row_ptrs, a_cols, a_vals, b_row_ptrs, b_cols, b_vals,
+                col_heap, [](size_type row) { return IndexType{}; },
+                [](ValueType, IndexType, IndexType &) {},
+                [&](IndexType col, IndexType &nnz) {
+                    // skip smaller elements from d
+                    while (d_col <= col) {
+                        d_nz++;
+                        nnz += d_col != col;
+                        d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+                    }
+                    nnz++;
+                });
+            // handle the remaining columns from d
+            c_row_ptrs[a_row] += d_end - d_nz;
+        });
+    });
+
+    // build row pointers
+    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
+
+    // second sweep: accumulate non-zeros
+    const auto new_nnz = exec->copy_val_to_host(c_row_ptrs + num_rows);
+    matrix::CsrBuilder<ValueType, IndexType> c_builder{c};
+    auto &c_col_idxs_array = c_builder.get_col_idx_array();
+    auto &c_vals_array = c_builder.get_value_array();
+    c_col_idxs_array.resize_and_reset(new_nnz);
+    c_vals_array.resize_and_reset(new_nnz);
+
+    auto c_col_idxs = c_col_idxs_array.get_data();
+    auto c_vals = c_vals_array.get_data();
+
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto a_row = static_cast<size_type>(idx[0]);
+            auto d_nz = d_row_ptrs[a_row];
+            const auto d_end = d_row_ptrs[a_row + 1];
+            auto d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+            auto d_val = checked_load(d_vals, d_nz, d_end, zero<ValueType>());
+            const auto valpha = alpha_vals[0];
+            const auto vbeta = beta_vals[0];
+            auto c_nz =
+                spgemm_multiway_merge(
+                    a_row, a_row_ptrs, a_cols, a_vals, b_row_ptrs, b_cols,
+                    b_vals, heap,
+                    [&](size_type row) {
+                        return std::make_pair(zero<ValueType>(),
+                                              c_row_ptrs[row]);
+                    },
+                    [](ValueType val, IndexType,
+                       std::pair<ValueType, IndexType> &state) {
+                        state.first += val;
+                    },
+                    [&](IndexType col, std::pair<ValueType, IndexType> &state) {
+                        // handle smaller elements from d
+                        ValueType part_d_val{};
+                        while (d_col <= col) {
+                            if (d_col == col) {
+                                part_d_val = d_val;
+                            } else {
+                                c_col_idxs[state.second] = d_col;
+                                c_vals[state.second] = vbeta * d_val;
+                                state.second++;
+                            }
+                            d_nz++;
+                            d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+                            d_val = checked_load(d_vals, d_nz, d_end,
+                                                 zero<ValueType>());
+                        }
+                        c_col_idxs[state.second] = col;
+                        c_vals[state.second] =
+                            vbeta * part_d_val + valpha * state.first;
+                        state.first = zero<ValueType>();
+                        state.second++;
+                    })
+                    .second;
+            // handle remaining elements from d
+            while (d_col < sentinel) {
+                c_col_idxs[c_nz] = d_col;
+                c_vals[c_nz] = vbeta * d_val;
+                c_nz++;
+                d_nz++;
+                d_col = checked_load(d_cols, d_nz, d_end, sentinel);
+                d_val = checked_load(d_vals, d_nz, d_end, zero<ValueType>());
+            }
+        });
+    });
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -2207,27 +2391,85 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 
 
 template <typename ValueType, typename IndexType>
-void spgeam(std::shared_ptr<const DefaultExecutor> exec,
+void spgeam(std::shared_ptr<const DpcppExecutor> exec,
             const matrix::Dense<ValueType> *alpha,
             const matrix::Csr<ValueType, IndexType> *a,
             const matrix::Dense<ValueType> *beta,
             const matrix::Csr<ValueType, IndexType> *b,
             matrix::Csr<ValueType, IndexType> *c)
 {
-    auto total_nnz =
-        a->get_num_stored_elements() + b->get_num_stored_elements();
-    auto nnz_per_row = total_nnz / a->get_size()[0];
-    select_spgeam(
-        spgeam_kernels(),
-        [&](int compiled_subwarp_size) {
-            return compiled_subwarp_size >= nnz_per_row ||
-                   compiled_subwarp_size == config::warp_size;
-        },
-        syn::value_list<int>(), syn::type_list<>(), exec,
-        alpha->get_const_values(), a->get_const_row_ptrs(),
-        a->get_const_col_idxs(), a->get_const_values(),
-        beta->get_const_values(), b->get_const_row_ptrs(),
-        b->get_const_col_idxs(), b->get_const_values(), c);
+    constexpr auto sentinel = std::numeric_limits<IndexType>::max();
+    const auto num_rows = a->get_size()[0];
+    const auto a_row_ptrs = a->get_const_row_ptrs();
+    const auto a_cols = a->get_const_col_idxs();
+    const auto b_row_ptrs = b->get_const_row_ptrs();
+    const auto b_cols = b->get_const_col_idxs();
+    auto c_row_ptrs = c->get_row_ptrs();
+    auto queue = exec->get_queue();
+
+    // count number of non-zeros per row
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto row = static_cast<size_type>(idx[0]);
+            auto a_idx = a_row_ptrs[row];
+            const auto a_end = a_row_ptrs[row + 1];
+            auto b_idx = b_row_ptrs[row];
+            const auto b_end = b_row_ptrs[row + 1];
+            IndexType row_nnz{};
+            while (a_idx < a_end || b_idx < b_end) {
+                const auto a_col = checked_load(a_cols, a_idx, a_end, sentinel);
+                const auto b_col = checked_load(b_cols, b_idx, b_end, sentinel);
+                row_nnz++;
+                a_idx += (a_col <= b_col) ? 1 : 0;
+                b_idx += (b_col <= a_col) ? 1 : 0;
+            }
+            c_row_ptrs[row] = row_nnz;
+        });
+    });
+
+    components::prefix_sum(exec, c_row_ptrs, num_rows + 1);
+
+    // second sweep: accumulate non-zeros
+    const auto new_nnz = exec->copy_val_to_host(c_row_ptrs + num_rows);
+    matrix::CsrBuilder<ValueType, IndexType> c_builder{c};
+    auto &c_col_idxs_array = c_builder.get_col_idx_array();
+    auto &c_vals_array = c_builder.get_value_array();
+    c_col_idxs_array.resize_and_reset(new_nnz);
+    c_vals_array.resize_and_reset(new_nnz);
+    auto c_cols = c_col_idxs_array.get_data();
+    auto c_vals = c_vals_array.get_data();
+
+    const auto a_vals = a->get_const_values();
+    const auto b_vals = b->get_const_values();
+    const auto alpha_vals = alpha->get_const_values();
+    const auto beta_vals = beta->get_const_values();
+
+    // count number of non-zeros per row
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto row = static_cast<size_type>(idx[0]);
+            auto a_idx = a_row_ptrs[row];
+            const auto a_end = a_row_ptrs[row + 1];
+            auto b_idx = b_row_ptrs[row];
+            const auto b_end = b_row_ptrs[row + 1];
+            const auto alpha = alpha_vals[0];
+            const auto beta = beta_vals[0];
+            auto c_nz = c_row_ptrs[row];
+            while (a_idx < a_end || b_idx < b_end) {
+                const auto a_col = checked_load(a_cols, a_idx, a_end, sentinel);
+                const auto b_col = checked_load(b_cols, b_idx, b_end, sentinel);
+                const bool use_a = a_col <= b_col;
+                const bool use_b = b_col <= a_col;
+                const auto a_val = use_a ? a_vals[a_idx] : zero<ValueType>();
+                const auto b_val = use_b ? b_vals[b_idx] : zero<ValueType>();
+                c_cols[c_nz] = std::min(a_col, b_col);
+                c_vals[c_nz] = alpha * a_val + beta * b_val;
+                c_nz++;
+                a_idx += use_a ? 1 : 0;
+                b_idx += use_b ? 1 : 0;
+            }
+        });
+    });
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEAM_KERNEL);
@@ -2436,12 +2678,63 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CALCULATE_TOTAL_COLS_KERNEL);
 
 
+template <bool conjugate, typename ValueType, typename IndexType>
+void generic_transpose(std::shared_ptr<const DpcppExecutor> exec,
+                       const matrix::Csr<ValueType, IndexType> *orig,
+                       matrix::Csr<ValueType, IndexType> *trans)
+{
+    const auto num_rows = orig->get_size()[0];
+    const auto num_cols = orig->get_size()[1];
+    auto queue = exec->get_queue();
+    const auto row_ptrs = orig->get_const_row_ptrs();
+    const auto cols = orig->get_const_col_idxs();
+    const auto vals = orig->get_const_values();
+
+    Array<IndexType> counts{exec, num_cols + 1};
+    auto tmp_counts = counts.get_data();
+    auto out_row_ptrs = trans->get_row_ptrs();
+    auto out_cols = trans->get_col_idxs();
+    auto out_vals = trans->get_values();
+    components::fill_array(exec, tmp_counts, num_cols, IndexType{});
+
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto row = static_cast<size_type>(idx[0]);
+            const auto begin = row_ptrs[row];
+            const auto end = row_ptrs[row + 1];
+            for (auto i = begin; i < end; i++) {
+                atomic_fetch_add(tmp_counts + cols[i], IndexType{1});
+            }
+        });
+    });
+
+    components::prefix_sum(exec, tmp_counts, num_cols + 1);
+    exec->copy(num_cols + 1, tmp_counts, out_row_ptrs);
+
+    queue->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto row = static_cast<size_type>(idx[0]);
+            const auto begin = row_ptrs[row];
+            const auto end = row_ptrs[row + 1];
+            for (auto i = begin; i < end; i++) {
+                auto out_nz =
+                    atomic_fetch_add(tmp_counts + cols[i], IndexType{1});
+                out_cols[out_nz] = row;
+                out_vals[out_nz] = conjugate ? conj(vals[i]) : vals[i];
+            }
+        });
+    });
+
+    sort_by_column_index(exec, trans);
+}
+
+
 template <typename ValueType, typename IndexType>
 void transpose(std::shared_ptr<const DpcppExecutor> exec,
                const matrix::Csr<ValueType, IndexType> *orig,
                matrix::Csr<ValueType, IndexType> *trans)
 {
-    GKO_NOT_IMPLEMENTED;
+    generic_transpose<false>(exec, orig, trans);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_TRANSPOSE_KERNEL);
@@ -2452,13 +2745,12 @@ void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Csr<ValueType, IndexType> *orig,
                     matrix::Csr<ValueType, IndexType> *trans)
 {
-    GKO_NOT_IMPLEMENTED;
+    generic_transpose<true>(exec, orig, trans);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONJ_TRANSPOSE_KERNEL);
 
-
 template <typename IndexType>
 void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
                         size_type size, const IndexType *permutation_indices,
@@ -2663,7 +2955,54 @@ template <typename ValueType, typename IndexType>
 void sort_by_column_index(std::shared_ptr<const DpcppExecutor> exec,
                           matrix::Csr<ValueType, IndexType> *to_sort)
 {
-    GKO_NOT_IMPLEMENTED;
+    const auto num_rows = to_sort->get_size()[0];
+    const auto row_ptrs = to_sort->get_const_row_ptrs();
+    auto cols = to_sort->get_col_idxs();
+    auto vals = to_sort->get_values();
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto row = static_cast<size_type>(idx[0]);
+            const auto begin = row_ptrs[row];
+            auto size = row_ptrs[row + 1] - begin;
+            if (size <= 1) {
+                return;
+            }
+            auto swap = [&](IndexType i, IndexType j) {
+                std::swap(cols[i + begin], cols[j + begin]);
+                std::swap(vals[i + begin], vals[j + begin]);
+            };
+            auto lchild = [](IndexType i) { return 2 * i + 1; };
+            auto rchild = [](IndexType i) { return 2 * i + 2; };
+            auto parent = [](IndexType i) { return (i - 1) / 2; };
+            auto sift_down = [&](IndexType i) {
+                const auto col = cols[i + begin];
+                while (lchild(i) < size) {
+                    const auto lcol = cols[lchild(i) + begin];
+                    // -1 as sentinel, since we are building a max heap
+                    const auto rcol = checked_load(cols + begin, rchild(i),
+                                                   size, IndexType{-1});
+                    if (col >= std::max(lcol, rcol)) {
+                        return;
+                    }
+                    const auto maxchild = lcol > rcol ? lchild(i) : rchild(i);
+                    swap(i, maxchild);
+                    i = maxchild;
+                }
+            };
+            // heapify / sift_down for max-heap
+            for (auto i = (size - 2) / 2; i >= 0; i--) {
+                sift_down(i);
+            }
+            // heapsort: swap maximum to the end, shrink heap
+            swap(0, size - 1);
+            size--;
+            for (; size > 1; size--) {
+                // restore heap property and repeat
+                sift_down(0);
+                swap(0, size - 1);
+            }
+        });
+    });
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
@@ -2675,18 +3014,28 @@ void is_sorted_by_column_index(
     std::shared_ptr<const DpcppExecutor> exec,
     const matrix::Csr<ValueType, IndexType> *to_check, bool *is_sorted)
 {
-    *is_sorted = true;
-    auto cpu_array = Array<bool>::view(exec->get_master(), 1, is_sorted);
-    auto gpu_array = Array<bool>{exec, cpu_array};
-    auto block_size = default_block_size;
-    auto num_rows = static_cast<IndexType>(to_check->get_size()[0]);
-    auto num_blocks = ceildiv(num_rows, block_size);
-    kernel::check_unsorted(num_blocks, block_size, 0, exec->get_queue(),
-                           to_check->get_const_row_ptrs(),
-                           to_check->get_const_col_idxs(), num_rows,
-                           gpu_array.get_data());
-    cpu_array = gpu_array;
-}
+    Array<bool> is_sorted_device_array{exec, {true}};
+    const auto num_rows = to_check->get_size()[0];
+    const auto row_ptrs = to_check->get_const_row_ptrs();
+    const auto cols = to_check->get_const_col_idxs();
+    auto is_sorted_device = is_sorted_device_array.get_data();
+    exec->get_queue()->submit([&](sycl::handler &cgh) {
+        cgh.parallel_for(sycl::range<1>{num_rows}, [=](sycl::id<1> idx) {
+            const auto row = static_cast<size_type>(idx[0]);
+            const auto begin = row_ptrs[row];
+            const auto end = row_ptrs[row + 1];
+            if (*is_sorted_device) {
+                for (auto i = begin; i < end - 1; i++) {
+                    if (cols[i] > cols[i + 1]) {
+                        *is_sorted_device = false;
+                        break;
+                    }
+                }
+            }
+        });
+    });
+    *is_sorted = exec->copy_val_to_host(is_sorted_device);
+};
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_IS_SORTED_BY_COLUMN_INDEX);
diff --git a/dpcpp/matrix/ell_kernels.dp.cpp b/dpcpp/matrix/ell_kernels.dp.cpp
index e69c3402730..e021b3edc22 100644
--- a/dpcpp/matrix/ell_kernels.dp.cpp
+++ b/dpcpp/matrix/ell_kernels.dp.cpp
@@ -406,6 +406,15 @@ void count_nnz_per_row(dim3 grid, dim3 block, size_type dynamic_shared_memory,
 }
 
 
+#define GKO_ELL_COUNT_NNZ_PER_ROW(ValueType, IndexType)                        \
+    void count_nnz_per_row(dim3, dim3, gko::size_type, sycl::queue *,          \
+                           size_type, size_type, size_type, const ValueType *, \
+                           IndexType *)
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_ELL_COUNT_NNZ_PER_ROW);
+
+#undef GKO_ELL_COUNT_NNZ_PER_ROW
+
 template <typename ValueType, typename IndexType>
 void fill_in_csr(size_type num_rows, size_type max_nnz_per_row,
                  size_type stride, const ValueType *__restrict__ source_values,
diff --git a/dpcpp/matrix/fbcsr_kernels.dp.cpp b/dpcpp/matrix/fbcsr_kernels.dp.cpp
index e8846f7229c..bc61e5c6985 100644
--- a/dpcpp/matrix/fbcsr_kernels.dp.cpp
+++ b/dpcpp/matrix/fbcsr_kernels.dp.cpp
@@ -30,6 +30,9 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
+#include "core/matrix/fbcsr_kernels.hpp"
+
+
 #include <algorithm>
 
 
diff --git a/dpcpp/test/matrix/CMakeLists.txt b/dpcpp/test/matrix/CMakeLists.txt
index e905c257493..f36f9c5606e 100644
--- a/dpcpp/test/matrix/CMakeLists.txt
+++ b/dpcpp/test/matrix/CMakeLists.txt
@@ -1,3 +1,7 @@
+ginkgo_create_test(coo_kernels)
 ginkgo_create_test(csr_kernels)
 ginkgo_create_test(dense_kernels)
-ginkgo_create_test(ell_kernels)
+ginkgo_create_test(diagonal_kernels)
+ginkgo_create_test(fbcsr_kernels)
+ginkgo_create_test(hybrid_kernels)
+ginkgo_create_test(sellp_kernels)
diff --git a/dpcpp/test/matrix/coo_kernels.cpp b/dpcpp/test/matrix/coo_kernels.cpp
index 6dc8e2729e3..06073dd56eb 100644
--- a/dpcpp/test/matrix/coo_kernels.cpp
+++ b/dpcpp/test/matrix/coo_kernels.cpp
@@ -57,15 +57,20 @@ namespace {
 
 class Coo : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Coo<>;
-    using Vec = gko::matrix::Dense<>;
-    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
+#if GINKGO_DPCPP_SINGLE_MODE
+    using vtype = float;
+#else
+    using vtype = double;
+#endif  // GINKGO_DPCPP_SINGLE_MODE
+    using Mtx = gko::matrix::Coo<vtype>;
+    using Vec = gko::matrix::Dense<vtype>;
+    using ComplexVec = gko::matrix::Dense<std::complex<vtype>>;
 
     Coo() : rand_engine(42) {}
 
     void SetUp()
     {
-        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices("all"), 0);
         ref = gko::ReferenceExecutor::create();
         dpcpp = gko::DpcppExecutor::create(0, ref);
     }
@@ -82,7 +87,7 @@ class Coo : public ::testing::Test {
     {
         return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols, std::uniform_int_distribution<>(1, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+            std::normal_distribution<vtype>(-1.0, 1.0), rand_engine, ref);
     }
 
     void set_up_apply_data(int num_vectors = 1)
@@ -137,7 +142,7 @@ TEST_F(Coo, SimpleApplyIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -147,14 +152,14 @@ TEST_F(Coo, SimpleApplyDoesntOverwritePadding)
     auto dresult_padded =
         Vec::create(dpcpp, dresult->get_size(), dresult->get_stride() + 1);
     dresult_padded->copy_from(dresult.get());
-    double padding_val{1234.0};
+    vtype padding_val{1234.0};
     dpcpp->copy_from(dpcpp->get_master().get(), 1, &padding_val,
                      dresult_padded->get_values() + 1);
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult_padded.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult_padded, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult_padded, expected, r<vtype>::value);
     ASSERT_EQ(dpcpp->copy_val_to_host(dresult_padded->get_values() + 1),
               1234.0);
 }
@@ -168,7 +173,7 @@ TEST_F(Coo, SimpleApplyIsEquivalentToRefUnsorted)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -179,7 +184,7 @@ TEST_F(Coo, AdvancedApplyIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -189,14 +194,14 @@ TEST_F(Coo, AdvancedApplyDoesntOverwritePadding)
     auto dresult_padded =
         Vec::create(dpcpp, dresult->get_size(), dresult->get_stride() + 1);
     dresult_padded->copy_from(dresult.get());
-    double padding_val{1234.0};
+    vtype padding_val{1234.0};
     dpcpp->copy_from(dpcpp->get_master().get(), 1, &padding_val,
                      dresult_padded->get_values() + 1);
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult_padded.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult_padded, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult_padded, expected, r<vtype>::value);
     ASSERT_EQ(dpcpp->copy_val_to_host(dresult_padded->get_values() + 1),
               1234.0);
 }
@@ -209,7 +214,7 @@ TEST_F(Coo, SimpleApplyAddIsEquivalentToRef)
     mtx->apply2(y.get(), expected.get());
     dmtx->apply2(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -220,7 +225,7 @@ TEST_F(Coo, AdvancedApplyAddIsEquivalentToRef)
     mtx->apply2(alpha.get(), y.get(), expected.get());
     dmtx->apply2(dalpha.get(), dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -231,7 +236,7 @@ TEST_F(Coo, SimpleApplyToDenseMatrixIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -242,7 +247,7 @@ TEST_F(Coo, AdvancedApplyToDenseMatrixIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -253,7 +258,7 @@ TEST_F(Coo, SimpleApplyAddToDenseMatrixIsEquivalentToRef)
     mtx->apply2(y.get(), expected.get());
     dmtx->apply2(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -264,7 +269,7 @@ TEST_F(Coo, SimpleApplyAddToLargeDenseMatrixIsEquivalentToRef)
     mtx->apply2(y.get(), expected.get());
     dmtx->apply2(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -275,7 +280,7 @@ TEST_F(Coo, AdvancedApplyAddToDenseMatrixIsEquivalentToRef)
     mtx->apply2(alpha.get(), y.get(), expected.get());
     dmtx->apply2(dalpha.get(), dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -286,7 +291,7 @@ TEST_F(Coo, AdvancedApplyAddToLargeDenseMatrixIsEquivalentToRef)
     mtx->apply2(y.get(), expected.get());
     dmtx->apply2(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -303,7 +308,7 @@ TEST_F(Coo, ApplyToComplexIsEquivalentToRef)
     mtx->apply(complex_b.get(), complex_x.get());
     dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
 
-    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r<vtype>::value);
 }
 
 
@@ -320,7 +325,7 @@ TEST_F(Coo, AdvancedApplyToComplexIsEquivalentToRef)
     mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
     dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
 
-    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r<vtype>::value);
 }
 
 
@@ -337,35 +342,35 @@ TEST_F(Coo, ApplyAddToComplexIsEquivalentToRef)
     mtx->apply2(alpha.get(), complex_b.get(), complex_x.get());
     dmtx->apply2(dalpha.get(), dcomplex_b.get(), dcomplex_x.get());
 
-    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r<vtype>::value);
 }
 
 
 TEST_F(Coo, ConvertToDenseIsEquivalentToRef)
 {
     set_up_apply_data();
-    auto dense_mtx = gko::matrix::Dense<>::create(ref);
-    auto ddense_mtx = gko::matrix::Dense<>::create(dpcpp);
+    auto dense_mtx = gko::matrix::Dense<vtype>::create(ref);
+    auto ddense_mtx = gko::matrix::Dense<vtype>::create(dpcpp);
 
     mtx->convert_to(dense_mtx.get());
     dmtx->convert_to(ddense_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Coo, ConvertToCsrIsEquivalentToRef)
 {
     set_up_apply_data();
-    auto dense_mtx = gko::matrix::Dense<>::create(ref);
-    auto csr_mtx = gko::matrix::Csr<>::create(ref);
-    auto dcsr_mtx = gko::matrix::Csr<>::create(dpcpp);
+    auto dense_mtx = gko::matrix::Dense<vtype>::create(ref);
+    auto csr_mtx = gko::matrix::Csr<vtype>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<vtype>::create(dpcpp);
 
     mtx->convert_to(dense_mtx.get());
     dense_mtx->convert_to(csr_mtx.get());
     dmtx->convert_to(dcsr_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), r<vtype>::value);
 }
 
 
@@ -387,7 +392,7 @@ TEST_F(Coo, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, r<vtype>::value);
 }
 
 
@@ -398,7 +403,7 @@ TEST_F(Coo, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, r<vtype>::value);
 }
 
 
diff --git a/dpcpp/test/matrix/csr_kernels.cpp b/dpcpp/test/matrix/csr_kernels.cpp
index 3270725fbe4..a5e93e0462a 100644
--- a/dpcpp/test/matrix/csr_kernels.cpp
+++ b/dpcpp/test/matrix/csr_kernels.cpp
@@ -58,11 +58,16 @@ namespace {
 
 class Csr : public ::testing::Test {
 protected:
+#if GINKGO_DPCPP_SINGLE_MODE
+    using vtype = float;
+#else
+    using vtype = double;
+#endif  // GINKGO_DPCPP_SINGLE_MODE
     using Arr = gko::Array<int>;
-    using Vec = gko::matrix::Dense<>;
-    using Mtx = gko::matrix::Csr<>;
-    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
-    using ComplexMtx = gko::matrix::Csr<std::complex<double>>;
+    using Vec = gko::matrix::Dense<vtype>;
+    using Mtx = gko::matrix::Csr<vtype>;
+    using ComplexVec = gko::matrix::Dense<std::complex<vtype>>;
+    using ComplexMtx = gko::matrix::Csr<std::complex<vtype>>;
 
     Csr()
 #ifdef GINKGO_FAST_TESTS
@@ -75,7 +80,7 @@ class Csr : public ::testing::Test {
 
     void SetUp()
     {
-        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices("all"), 0);
         ref = gko::ReferenceExecutor::create();
         dpcpp = gko::DpcppExecutor::create(0, ref);
     }
@@ -94,7 +99,7 @@ class Csr : public ::testing::Test {
         return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(min_nnz_row, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+            std::normal_distribution<vtype>(-1.0, 1.0), rand_engine, ref);
     }
 
     void set_up_apply_data(std::shared_ptr<Mtx::strategy_type> strategy,
@@ -190,7 +195,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithLoadBalance)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -202,7 +207,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithLoadBalanceUnsorted)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -213,7 +218,7 @@ TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithLoadBalance)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -224,7 +229,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithCusparse)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -236,7 +241,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithCusparseUnsorted)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -247,7 +252,7 @@ TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithCusparse)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -258,7 +263,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithMergePath)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -270,7 +275,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithMergePathUnsorted)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -281,7 +286,7 @@ TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithMergePath)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -292,7 +297,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithClassical)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -304,7 +309,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithClassicalUnsorted)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -315,7 +320,7 @@ TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithClassical)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -326,7 +331,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithAutomatical)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -337,7 +342,7 @@ TEST_F(Csr, SimpleApplyToDenseMatrixIsEquivalentToRefWithLoadBalance)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -348,7 +353,7 @@ TEST_F(Csr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithLoadBalance)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -359,7 +364,7 @@ TEST_F(Csr, SimpleApplyToDenseMatrixIsEquivalentToRefWithClassical)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -370,7 +375,7 @@ TEST_F(Csr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithClassical)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -381,7 +386,7 @@ TEST_F(Csr, SimpleApplyToDenseMatrixIsEquivalentToRefWithMergePath)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -392,7 +397,7 @@ TEST_F(Csr, AdvancedApplyToDenseMatrixIsEquivalentToRefWithMergePath)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -405,7 +410,7 @@ TEST_F(Csr, AdvancedApplyToCsrMatrixIsEquivalentToRef)
     mtx->apply(alpha.get(), trans.get(), beta.get(), square_mtx.get());
     dmtx->apply(dalpha.get(), d_trans.get(), dbeta.get(), square_dmtx.get());
 
-    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<vtype>::value);
     GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
     ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
 }
@@ -420,12 +425,13 @@ TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRef)
     mtx->apply(trans.get(), square_mtx.get());
     dmtx->apply(d_trans.get(), square_dmtx.get());
 
-    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<vtype>::value);
     GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
     ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
 }
 
 
+/*
 TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRefUnsorted)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>());
@@ -437,10 +443,11 @@ TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRefUnsorted)
     mtx->apply(trans.get(), square_mtx.get());
     dmtx->apply(d_trans.get(), square_dmtx.get());
 
-    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<vtype>::value);
     GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
     ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
 }
+*/
 
 
 TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
@@ -459,7 +466,7 @@ TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
     a->apply(alpha.get(), id.get(), beta.get(), b.get());
     da->apply(dalpha.get(), did.get(), dbeta.get(), db.get());
 
-    GKO_ASSERT_MTX_NEAR(b, db, 1e-14);
+    GKO_ASSERT_MTX_NEAR(b, db, r<vtype>::value);
     GKO_ASSERT_MTX_EQ_SPARSITY(b, db);
     ASSERT_TRUE(db->is_sorted_by_column_index());
 }
@@ -478,7 +485,7 @@ TEST_F(Csr, ApplyToComplexIsEquivalentToRef)
     mtx->apply(complex_b.get(), complex_x.get());
     dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
 
-    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r<vtype>::value);
 }
 
 
@@ -495,7 +502,7 @@ TEST_F(Csr, AdvancedApplyToComplexIsEquivalentToRef)
     mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
     dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
 
-    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r<vtype>::value);
 }
 
 
@@ -526,78 +533,80 @@ TEST_F(Csr, ConjugateTransposeIsEquivalentToRef)
 TEST_F(Csr, ConvertToDenseIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
-    auto dense_mtx = gko::matrix::Dense<>::create(ref);
-    auto ddense_mtx = gko::matrix::Dense<>::create(dpcpp);
+    auto dense_mtx = gko::matrix::Dense<vtype>::create(ref);
+    auto ddense_mtx = gko::matrix::Dense<vtype>::create(dpcpp);
 
     mtx->convert_to(dense_mtx.get());
     dmtx->convert_to(ddense_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Csr, MoveToDenseIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
-    auto dense_mtx = gko::matrix::Dense<>::create(ref);
-    auto ddense_mtx = gko::matrix::Dense<>::create(dpcpp);
+    auto dense_mtx = gko::matrix::Dense<vtype>::create(ref);
+    auto ddense_mtx = gko::matrix::Dense<vtype>::create(dpcpp);
 
     mtx->move_to(dense_mtx.get());
     dmtx->move_to(ddense_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Csr, ConvertToEllIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
-    auto ell_mtx = gko::matrix::Ell<>::create(ref);
-    auto dell_mtx = gko::matrix::Ell<>::create(dpcpp);
+    auto ell_mtx = gko::matrix::Ell<vtype>::create(ref);
+    auto dell_mtx = gko::matrix::Ell<vtype>::create(dpcpp);
 
     mtx->convert_to(ell_mtx.get());
     dmtx->convert_to(dell_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(ell_mtx.get(), dell_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(ell_mtx.get(), dell_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Csr, MoveToEllIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
-    auto ell_mtx = gko::matrix::Ell<>::create(ref);
-    auto dell_mtx = gko::matrix::Ell<>::create(dpcpp);
+    auto ell_mtx = gko::matrix::Ell<vtype>::create(ref);
+    auto dell_mtx = gko::matrix::Ell<vtype>::create(dpcpp);
 
     mtx->move_to(ell_mtx.get());
     dmtx->move_to(dell_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(ell_mtx.get(), dell_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(ell_mtx.get(), dell_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Csr, ConvertToSparsityCsrIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
-    auto sparsity_mtx = gko::matrix::SparsityCsr<>::create(ref);
-    auto d_sparsity_mtx = gko::matrix::SparsityCsr<>::create(dpcpp);
+    auto sparsity_mtx = gko::matrix::SparsityCsr<vtype>::create(ref);
+    auto d_sparsity_mtx = gko::matrix::SparsityCsr<vtype>::create(dpcpp);
 
     mtx->convert_to(sparsity_mtx.get());
     dmtx->convert_to(d_sparsity_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(sparsity_mtx.get(), d_sparsity_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(sparsity_mtx.get(), d_sparsity_mtx.get(),
+                        r<vtype>::value);
 }
 
 
 TEST_F(Csr, MoveToSparsityCsrIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
-    auto sparsity_mtx = gko::matrix::SparsityCsr<>::create(ref);
-    auto d_sparsity_mtx = gko::matrix::SparsityCsr<>::create(dpcpp);
+    auto sparsity_mtx = gko::matrix::SparsityCsr<vtype>::create(ref);
+    auto d_sparsity_mtx = gko::matrix::SparsityCsr<vtype>::create(dpcpp);
 
     mtx->move_to(sparsity_mtx.get());
     dmtx->move_to(d_sparsity_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(sparsity_mtx.get(), d_sparsity_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(sparsity_mtx.get(), d_sparsity_mtx.get(),
+                        r<vtype>::value);
 }
 
 
@@ -619,59 +628,59 @@ TEST_F(Csr, CalculateMaxNnzPerRowIsEquivalentToRef)
 TEST_F(Csr, ConvertToCooIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
-    auto coo_mtx = gko::matrix::Coo<>::create(ref);
-    auto dcoo_mtx = gko::matrix::Coo<>::create(dpcpp);
+    auto coo_mtx = gko::matrix::Coo<vtype>::create(ref);
+    auto dcoo_mtx = gko::matrix::Coo<vtype>::create(dpcpp);
 
     mtx->convert_to(coo_mtx.get());
     dmtx->convert_to(dcoo_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Csr, MoveToCooIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
-    auto coo_mtx = gko::matrix::Coo<>::create(ref);
-    auto dcoo_mtx = gko::matrix::Coo<>::create(dpcpp);
+    auto coo_mtx = gko::matrix::Coo<vtype>::create(ref);
+    auto dcoo_mtx = gko::matrix::Coo<vtype>::create(dpcpp);
 
     mtx->move_to(coo_mtx.get());
     dmtx->move_to(dcoo_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(coo_mtx.get(), dcoo_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Csr, ConvertToSellpIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
-    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
-    auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+    auto sellp_mtx = gko::matrix::Sellp<vtype>::create(ref);
+    auto dsellp_mtx = gko::matrix::Sellp<vtype>::create(dpcpp);
 
     mtx->convert_to(sellp_mtx.get());
     dmtx->convert_to(dsellp_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(sellp_mtx.get(), dsellp_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(sellp_mtx.get(), dsellp_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Csr, MoveToSellpIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
-    auto sellp_mtx = gko::matrix::Sellp<>::create(ref);
-    auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+    auto sellp_mtx = gko::matrix::Sellp<vtype>::create(ref);
+    auto dsellp_mtx = gko::matrix::Sellp<vtype>::create(dpcpp);
 
     mtx->move_to(sellp_mtx.get());
     dmtx->move_to(dsellp_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(sellp_mtx.get(), dsellp_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(sellp_mtx.get(), dsellp_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Csr, ConvertsEmptyToSellp)
 {
     auto dempty_mtx = Mtx::create(dpcpp);
-    auto dsellp_mtx = gko::matrix::Sellp<>::create(dpcpp);
+    auto dsellp_mtx = gko::matrix::Sellp<vtype>::create(dpcpp);
 
     dempty_mtx->convert_to(dsellp_mtx.get());
 
@@ -712,7 +721,7 @@ TEST_F(Csr, CalculatesNonzerosPerRow)
 
 TEST_F(Csr, ConvertToHybridIsEquivalentToRef)
 {
-    using Hybrid_type = gko::matrix::Hybrid<>;
+    using Hybrid_type = gko::matrix::Hybrid<vtype>;
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
     auto hybrid_mtx = Hybrid_type::create(
         ref, std::make_shared<Hybrid_type::column_limit>(2));
@@ -722,13 +731,13 @@ TEST_F(Csr, ConvertToHybridIsEquivalentToRef)
     mtx->convert_to(hybrid_mtx.get());
     dmtx->convert_to(dhybrid_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Csr, MoveToHybridIsEquivalentToRef)
 {
-    using Hybrid_type = gko::matrix::Hybrid<>;
+    using Hybrid_type = gko::matrix::Hybrid<vtype>;
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
     auto hybrid_mtx = Hybrid_type::create(
         ref, std::make_shared<Hybrid_type::column_limit>(2));
@@ -738,7 +747,7 @@ TEST_F(Csr, MoveToHybridIsEquivalentToRef)
     mtx->move_to(hybrid_mtx.get());
     dmtx->move_to(dhybrid_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(hybrid_mtx.get(), dhybrid_mtx.get(), r<vtype>::value);
 }
 
 
@@ -917,7 +926,7 @@ TEST_F(Csr, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, r<vtype>::value);
 }
 
 
@@ -928,7 +937,7 @@ TEST_F(Csr, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, r<vtype>::value);
 }
 
 
@@ -939,7 +948,7 @@ TEST_F(Csr, InplaceAbsoluteComplexMatrixIsEquivalentToRef)
     complex_mtx->compute_absolute_inplace();
     complex_dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(complex_mtx, complex_dmtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(complex_mtx, complex_dmtx, r<vtype>::value);
 }
 
 
@@ -950,7 +959,7 @@ TEST_F(Csr, OutplaceAbsoluteComplexMatrixIsEquivalentToRef)
     auto abs_mtx = complex_mtx->compute_absolute();
     auto dabs_mtx = complex_dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, r<vtype>::value);
 }
 
 
diff --git a/dpcpp/test/matrix/diagonal_kernels.cpp b/dpcpp/test/matrix/diagonal_kernels.cpp
index fc041aea4e1..352c5e40823 100644
--- a/dpcpp/test/matrix/diagonal_kernels.cpp
+++ b/dpcpp/test/matrix/diagonal_kernels.cpp
@@ -52,8 +52,13 @@ namespace {
 
 class Diagonal : public ::testing::Test {
 protected:
-    using ValueType = double;
-    using ComplexValueType = std::complex<double>;
+#if GINKGO_DPCPP_SINGLE_MODE
+    using vtype = float;
+#else
+    using vtype = double;
+#endif  // GINKGO_DPCPP_SINGLE_MODE
+    using ValueType = vtype;
+    using ComplexValueType = std::complex<vtype>;
     using Csr = gko::matrix::Csr<ValueType>;
     using Diag = gko::matrix::Diagonal<ValueType>;
     using Dense = gko::matrix::Dense<ValueType>;
@@ -70,7 +75,7 @@ class Diagonal : public ::testing::Test {
 
     void SetUp()
     {
-        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices("all"), 0);
         ref = gko::ReferenceExecutor::create();
         dpcpp = gko::DpcppExecutor::create(0, ref);
     }
@@ -89,14 +94,14 @@ class Diagonal : public ::testing::Test {
         return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(min_nnz_row, num_cols),
-            std::normal_distribution<>(0.0, 1.0), rand_engine, ref);
+            std::normal_distribution<vtype>(0.0, 1.0), rand_engine, ref);
     }
 
     std::unique_ptr<Diag> gen_diag(int size)
     {
         auto diag = Diag::create(ref, size);
         auto vals = diag->get_values();
-        auto value_dist = std::normal_distribution<>(0.0, 1.0);
+        auto value_dist = std::normal_distribution<vtype>(0.0, 1.0);
         for (int i = 0; i < size; i++) {
             vals[i] = gko::test::detail::get_rand_value<ValueType>(value_dist,
                                                                    rand_engine);
@@ -108,7 +113,7 @@ class Diagonal : public ::testing::Test {
     {
         auto cdiag = ComplexDiag::create(ref, size);
         auto vals = cdiag->get_values();
-        auto value_dist = std::normal_distribution<>(0.0, 1.0);
+        auto value_dist = std::normal_distribution<vtype>(0.0, 1.0);
         for (int i = 0; i < size; i++) {
             vals[i] = ComplexValueType{
                 gko::test::detail::get_rand_value<ComplexValueType>(
@@ -192,7 +197,7 @@ TEST_F(Diagonal, ApplyToDenseIsEquivalentToRef)
     diag->apply(dense1.get(), denseexpected1.get());
     ddiag->apply(ddense1.get(), denseresult1.get());
 
-    GKO_ASSERT_MTX_NEAR(denseexpected1, denseresult1, 1e-14);
+    GKO_ASSERT_MTX_NEAR(denseexpected1, denseresult1, r<vtype>::value);
 }
 
 
@@ -203,7 +208,7 @@ TEST_F(Diagonal, RightApplyToDenseIsEquivalentToRef)
     diag->rapply(dense2.get(), denseexpected2.get());
     ddiag->rapply(ddense2.get(), denseresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(denseexpected2, denseresult2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(denseexpected2, denseresult2, r<vtype>::value);
 }
 
 
@@ -214,7 +219,7 @@ TEST_F(Diagonal, ApplyToCsrIsEquivalentToRef)
     diag->apply(csr1.get(), csrexpected1.get());
     ddiag->apply(dcsr1.get(), csrresult1.get());
 
-    GKO_ASSERT_MTX_NEAR(csrexpected1, csrresult1, 1e-14);
+    GKO_ASSERT_MTX_NEAR(csrexpected1, csrresult1, r<vtype>::value);
 }
 
 
@@ -225,7 +230,7 @@ TEST_F(Diagonal, RightApplyToCsrIsEquivalentToRef)
     diag->rapply(csr2.get(), csrexpected2.get());
     ddiag->rapply(dcsr2.get(), csrresult2.get());
 
-    GKO_ASSERT_MTX_NEAR(csrexpected2, csrresult2, 1e-14);
+    GKO_ASSERT_MTX_NEAR(csrexpected2, csrresult2, r<vtype>::value);
 }
 
 
@@ -260,7 +265,7 @@ TEST_F(Diagonal, InplaceAbsoluteMatrixIsEquivalentToRef)
     diag->compute_absolute_inplace();
     ddiag->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(diag, ddiag, 1e-14);
+    GKO_ASSERT_MTX_NEAR(diag, ddiag, r<vtype>::value);
 }
 
 
@@ -271,7 +276,7 @@ TEST_F(Diagonal, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_diag = diag->compute_absolute();
     auto dabs_diag = ddiag->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_diag, dabs_diag, 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_diag, dabs_diag, r<vtype>::value);
 }
 
 
diff --git a/dpcpp/test/matrix/fbcsr_kernels.cpp b/dpcpp/test/matrix/fbcsr_kernels.cpp
index d64fe1c075b..c67de961cd7 100644
--- a/dpcpp/test/matrix/fbcsr_kernels.cpp
+++ b/dpcpp/test/matrix/fbcsr_kernels.cpp
@@ -45,11 +45,16 @@ namespace {
 
 class Fbcsr : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Fbcsr<>;
+#if GINKGO_DPCPP_SINGLE_MODE
+    using vtype = float;
+#else
+    using vtype = double;
+#endif  // GINKGO_DPCPP_SINGLE_MODE
+    using Mtx = gko::matrix::Fbcsr<vtype>;
 
     void SetUp()
     {
-        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices("all"), 0);
         ref = gko::ReferenceExecutor::create();
         dpcpp = gko::DpcppExecutor::create(0, ref);
     }
diff --git a/dpcpp/test/matrix/hybrid_kernels.cpp b/dpcpp/test/matrix/hybrid_kernels.cpp
index 04782587360..016c9c9e608 100644
--- a/dpcpp/test/matrix/hybrid_kernels.cpp
+++ b/dpcpp/test/matrix/hybrid_kernels.cpp
@@ -52,15 +52,20 @@ namespace {
 
 class Hybrid : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Hybrid<>;
-    using Vec = gko::matrix::Dense<>;
-    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
+#if GINKGO_DPCPP_SINGLE_MODE
+    using vtype = float;
+#else
+    using vtype = double;
+#endif  // GINKGO_DPCPP_SINGLE_MODE
+    using Mtx = gko::matrix::Hybrid<vtype>;
+    using Vec = gko::matrix::Dense<vtype>;
+    using ComplexVec = gko::matrix::Dense<std::complex<vtype>>;
 
     Hybrid() : rand_engine(42) {}
 
     void SetUp()
     {
-        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices("all"), 0);
         ref = gko::ReferenceExecutor::create();
         dpcpp = gko::DpcppExecutor::create(0, ref);
     }
@@ -79,7 +84,7 @@ class Hybrid : public ::testing::Test {
         return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols,
             std::uniform_int_distribution<>(min_nnz_row, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+            std::normal_distribution<vtype>(-1.0, 1.0), rand_engine, ref);
     }
 
     void set_up_apply_data(int num_vectors = 1,
@@ -144,7 +149,7 @@ TEST_F(Hybrid, SimpleApplyIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -155,7 +160,7 @@ TEST_F(Hybrid, AdvancedApplyIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -166,7 +171,7 @@ TEST_F(Hybrid, SimpleApplyToDenseMatrixIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -177,7 +182,7 @@ TEST_F(Hybrid, AdvancedApplyToDenseMatrixIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -194,7 +199,7 @@ TEST_F(Hybrid, ApplyToComplexIsEquivalentToRef)
     mtx->apply(complex_b.get(), complex_x.get());
     dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
 
-    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r<vtype>::value);
 }
 
 
@@ -211,7 +216,7 @@ TEST_F(Hybrid, AdvancedApplyToComplexIsEquivalentToRef)
     mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
     dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
 
-    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r<vtype>::value);
 }
 
 
@@ -231,26 +236,26 @@ TEST_F(Hybrid, CountNonzerosIsEquivalentToRef)
 TEST_F(Hybrid, ConvertToCsrIsEquivalentToRef)
 {
     set_up_apply_data(1, std::make_shared<Mtx::column_limit>(2));
-    auto csr_mtx = gko::matrix::Csr<>::create(ref);
-    auto dcsr_mtx = gko::matrix::Csr<>::create(dpcpp);
+    auto csr_mtx = gko::matrix::Csr<vtype>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<vtype>::create(dpcpp);
 
     mtx->convert_to(csr_mtx.get());
     dmtx->convert_to(dcsr_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), r<vtype>::value);
 }
 
 
 TEST_F(Hybrid, MoveToCsrIsEquivalentToRef)
 {
     set_up_apply_data(1, std::make_shared<Mtx::column_limit>(2));
-    auto csr_mtx = gko::matrix::Csr<>::create(ref);
-    auto dcsr_mtx = gko::matrix::Csr<>::create(dpcpp);
+    auto csr_mtx = gko::matrix::Csr<vtype>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<vtype>::create(dpcpp);
 
     mtx->move_to(csr_mtx.get());
     dmtx->move_to(dcsr_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), r<vtype>::value);
 }
 
 
@@ -272,7 +277,7 @@ TEST_F(Hybrid, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, r<vtype>::value);
 }
 
 
@@ -287,7 +292,7 @@ TEST_F(Hybrid, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto dabs_strategy =
         gko::as<AbsMtx::column_limit>(dabs_mtx->get_strategy());
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, r<vtype>::value);
     GKO_ASSERT_EQ(abs_strategy->get_num_columns(),
                   dabs_strategy->get_num_columns());
     GKO_ASSERT_EQ(abs_strategy->get_num_columns(), 2);
diff --git a/dpcpp/test/matrix/sellp_kernels.cpp b/dpcpp/test/matrix/sellp_kernels.cpp
index ee8be3091fa..6e385ab13fe 100644
--- a/dpcpp/test/matrix/sellp_kernels.cpp
+++ b/dpcpp/test/matrix/sellp_kernels.cpp
@@ -53,15 +53,20 @@ namespace {
 
 class Sellp : public ::testing::Test {
 protected:
-    using Mtx = gko::matrix::Sellp<>;
-    using Vec = gko::matrix::Dense<>;
-    using ComplexVec = gko::matrix::Dense<std::complex<double>>;
+#if GINKGO_DPCPP_SINGLE_MODE
+    using vtype = float;
+#else
+    using vtype = double;
+#endif  // GINKGO_DPCPP_SINGLE_MODE
+    using Mtx = gko::matrix::Sellp<vtype>;
+    using Vec = gko::matrix::Dense<vtype>;
+    using ComplexVec = gko::matrix::Dense<std::complex<vtype>>;
 
     Sellp() : rand_engine(42) {}
 
     void SetUp()
     {
-        ASSERT_GT(gko::DpcppExecutor::get_num_devices(), 0);
+        ASSERT_GT(gko::DpcppExecutor::get_num_devices("all"), 0);
         ref = gko::ReferenceExecutor::create();
         dpcpp = gko::DpcppExecutor::create(0, ref);
     }
@@ -78,7 +83,7 @@ class Sellp : public ::testing::Test {
     {
         return gko::test::generate_random_matrix<MtxType>(
             num_rows, num_cols, std::uniform_int_distribution<>(1, num_cols),
-            std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
+            std::normal_distribution<vtype>(-1.0, 1.0), rand_engine, ref);
     }
 
     void set_up_apply_vector(
@@ -159,7 +164,7 @@ TEST_F(Sellp, SimpleApplyIsEquivalentToRef)
 
     auto result = Vec::create(ref);
     result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
 }
 
 
@@ -172,7 +177,7 @@ TEST_F(Sellp, AdvancedApplyIsEquivalentToRef)
 
     auto result = Vec::create(ref);
     result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
 }
 
 
@@ -185,7 +190,7 @@ TEST_F(Sellp, SimpleApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
 
     auto result = Vec::create(ref);
     result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
 }
 
 
@@ -198,7 +203,7 @@ TEST_F(Sellp, AdvancedApplyWithSliceSizeAndStrideFActorIsEquivalentToRef)
 
     auto result = Vec::create(ref);
     result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
 }
 
 
@@ -211,7 +216,7 @@ TEST_F(Sellp, SimpleApplyMultipleRHSIsEquivalentToRef)
 
     auto result = Vec::create(ref);
     result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
 }
 
 
@@ -224,7 +229,7 @@ TEST_F(Sellp, AdvancedApplyMultipleRHSIsEquivalentToRef)
 
     auto result = Vec::create(ref);
     result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
 }
 
 
@@ -238,7 +243,7 @@ TEST_F(Sellp,
 
     auto result = Vec::create(ref);
     result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
 }
 
 
@@ -252,7 +257,7 @@ TEST_F(Sellp,
 
     auto result = Vec::create(ref);
     result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
 }
 
 
@@ -269,7 +274,7 @@ TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
     mtx->apply(complex_b.get(), complex_x.get());
     dmtx->apply(dcomplex_b.get(), dcomplex_x.get());
 
-    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r<vtype>::value);
 }
 
 
@@ -286,7 +291,7 @@ TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
     mtx->apply(alpha.get(), complex_b.get(), beta.get(), complex_x.get());
     dmtx->apply(dalpha.get(), dcomplex_b.get(), dbeta.get(), dcomplex_x.get());
 
-    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r<vtype>::value);
 }
 
 
@@ -294,13 +299,13 @@ TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 {
     set_up_apply_matrix();
 
-    auto dense_mtx = gko::matrix::Dense<>::create(ref);
-    auto ddense_mtx = gko::matrix::Dense<>::create(dpcpp);
+    auto dense_mtx = gko::matrix::Dense<vtype>::create(ref);
+    auto ddense_mtx = gko::matrix::Dense<vtype>::create(dpcpp);
 
     mtx->convert_to(dense_mtx.get());
     dmtx->convert_to(ddense_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), r<vtype>::value);
 }
 
 
@@ -308,13 +313,13 @@ TEST_F(Sellp, ConvertToCsrIsEquivalentToRef)
 {
     set_up_apply_matrix();
 
-    auto csr_mtx = gko::matrix::Csr<>::create(ref);
-    auto dcsr_mtx = gko::matrix::Csr<>::create(dpcpp);
+    auto csr_mtx = gko::matrix::Csr<vtype>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<vtype>::create(dpcpp);
 
     mtx->convert_to(csr_mtx.get());
     dmtx->convert_to(dcsr_mtx.get());
 
-    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), 1e-14);
+    GKO_ASSERT_MTX_NEAR(csr_mtx.get(), dcsr_mtx.get(), r<vtype>::value);
 }
 
 
@@ -322,8 +327,8 @@ TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 {
     set_up_apply_matrix();
 
-    auto dense_mtx = gko::matrix::Dense<>::create(ref);
-    auto ddense_mtx = gko::matrix::Dense<>::create(dpcpp);
+    auto dense_mtx = gko::matrix::Dense<vtype>::create(ref);
+    auto ddense_mtx = gko::matrix::Dense<vtype>::create(dpcpp);
 
     empty->convert_to(dense_mtx.get());
     dempty->convert_to(ddense_mtx.get());
@@ -336,8 +341,8 @@ TEST_F(Sellp, ConvertEmptyToCsrIsEquivalentToRef)
 {
     set_up_apply_matrix();
 
-    auto csr_mtx = gko::matrix::Csr<>::create(ref);
-    auto dcsr_mtx = gko::matrix::Csr<>::create(dpcpp);
+    auto csr_mtx = gko::matrix::Csr<vtype>::create(ref);
+    auto dcsr_mtx = gko::matrix::Csr<vtype>::create(dpcpp);
 
     empty->convert_to(csr_mtx.get());
     dempty->convert_to(dcsr_mtx.get());
@@ -389,7 +394,7 @@ TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
 
-    GKO_ASSERT_MTX_NEAR(mtx, dmtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(mtx, dmtx, r<vtype>::value);
 }
 
 
@@ -400,7 +405,7 @@ TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
 
-    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, 1e-14);
+    GKO_ASSERT_MTX_NEAR(abs_mtx, dabs_mtx, r<vtype>::value);
 }
 
 
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index eded7da0434..2c41ced1104 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -365,6 +365,15 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
             : load_balance(exec->get_num_warps(), exec->get_warp_size(), false)
         {}
 
+        /**
+         * Creates a load_balance strategy with DPCPP executor.
+         *
+         * @param exec the HIP executor
+         */
+        load_balance(std::shared_ptr<const DpcppExecutor> exec)
+            : load_balance(exec->get_num_computing_units() * 7, 16, false)
+        {}
+
         /**
          * Creates a load_balance strategy with specified parameters
          *
@@ -448,7 +457,14 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
                 } else if (nnz >= static_cast<int64_t>(2e5)) {
                     multiple = 32;
                 }
-
+                if (!cuda_strategy_) {
+                    multiple = 8;
+                    if (nnz >= static_cast<int64_t>(2e8)) {
+                        multiple = 256;
+                    } else if (nnz >= static_cast<int64_t>(2e7)) {
+                        multiple = 32;
+                    }
+                }
 #if GINKGO_HIP_PLATFORM_HCC
                 if (!cuda_strategy_) {
                     multiple = 8;
@@ -493,6 +509,12 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
         /* Use imbalance strategy when the matrix has more more than 1e8 on AMD
          * hardware */
         const index_type amd_nnz_limit{static_cast<index_type>(1e8)};
+        /* Use imbalance strategy when the maximum number of nonzero per row is
+         * more than 25600 on Intel hardware. */
+        const index_type intel_row_len_limit = 25600;
+        /* Use imbalance strategy when the matrix has more more than 3e8 on
+         * Intel hardware */
+        const index_type intel_nnz_limit{static_cast<index_type>(3e8)};
 
         /**
          * Creates an automatical strategy.
@@ -520,6 +542,15 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
             : automatical(exec->get_num_warps(), exec->get_warp_size(), false)
         {}
 
+        /**
+         * Creates an automatical strategy with Dpcpp executor.
+         *
+         * @param exec the Dpcpp executor
+         */
+        automatical(std::shared_ptr<const DpcppExecutor> exec)
+            : automatical(exec->get_num_computing_units() * 7, 32, false)
+        {}
+
         /**
          * Creates an automatical strategy with specified parameters
          *
@@ -554,6 +585,10 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
                 row_len_limit = amd_row_len_limit;
             }
 #endif  // GINKGO_HIP_PLATFORM_HCC
+            if (!cuda_strategy_) {
+                nnz_limit = intel_nnz_limit;
+                row_len_limit = intel_row_len_limit;
+            }
             auto host_mtx_exec = mtx_row_ptrs.get_executor()->get_master();
             const bool is_mtx_on_host{host_mtx_exec ==
                                       mtx_row_ptrs.get_executor()};
@@ -940,6 +975,8 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
             auto cuda_exec =
                 std::dynamic_pointer_cast<const CudaExecutor>(rexec);
             auto hip_exec = std::dynamic_pointer_cast<const HipExecutor>(rexec);
+            auto dpcpp_exec =
+                std::dynamic_pointer_cast<const DpcppExecutor>(rexec);
             auto lb = dynamic_cast<load_balance *>(strat);
             if (cuda_exec) {
                 if (lb) {
@@ -959,6 +996,15 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
                     new_strat = std::make_shared<typename CsrType::automatical>(
                         hip_exec);
                 }
+            } else if (dpcpp_exec) {
+                if (lb) {
+                    new_strat =
+                        std::make_shared<typename CsrType::load_balance>(
+                            dpcpp_exec);
+                } else {
+                    new_strat = std::make_shared<typename CsrType::automatical>(
+                        dpcpp_exec);
+                }
             } else {
                 // Try to preserve this executor's configuration
                 auto this_cuda_exec =
@@ -967,6 +1013,9 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
                 auto this_hip_exec =
                     std::dynamic_pointer_cast<const HipExecutor>(
                         this->get_executor());
+                auto this_dpcpp_exec =
+                    std::dynamic_pointer_cast<const DpcppExecutor>(
+                        this->get_executor());
                 if (this_cuda_exec) {
                     if (lb) {
                         new_strat =
@@ -987,6 +1036,16 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
                             std::make_shared<typename CsrType::automatical>(
                                 this_hip_exec);
                     }
+                } else if (this_dpcpp_exec) {
+                    if (lb) {
+                        new_strat =
+                            std::make_shared<typename CsrType::load_balance>(
+                                this_dpcpp_exec);
+                    } else {
+                        new_strat =
+                            std::make_shared<typename CsrType::automatical>(
+                                this_dpcpp_exec);
+                    }
                 } else {
                     // We had a load balance or automatical strategy from a non
                     // HIP or Cuda executor and are moving to a non HIP or Cuda
@@ -1020,6 +1079,7 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
     Array<index_type> row_ptrs_;
     Array<index_type> srow_;
     std::shared_ptr<strategy_type> strategy_;
+    // oneapi::mkl::sparse::matrix_handle_t mat_handle_;
 };
 
 

From cfc19da8b2fa732e92552c1acdab9c91e32ae5ba Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 26 Jul 2021 12:43:33 +0200
Subject: [PATCH 0980/1008] del csr apply unsorted csr test, note csr lb arg

---
 cuda/test/matrix/csr_kernels.cpp   | 17 -----------------
 dpcpp/test/matrix/csr_kernels.cpp  | 19 -------------------
 include/ginkgo/core/matrix/csr.hpp |  5 ++++-
 3 files changed, 4 insertions(+), 37 deletions(-)

diff --git a/cuda/test/matrix/csr_kernels.cpp b/cuda/test/matrix/csr_kernels.cpp
index a523612270a..24beac46605 100644
--- a/cuda/test/matrix/csr_kernels.cpp
+++ b/cuda/test/matrix/csr_kernels.cpp
@@ -429,23 +429,6 @@ TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRef)
 }
 
 
-TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRefUnsorted)
-{
-    set_up_apply_data(std::make_shared<Mtx::automatical>());
-    auto trans = mtx->transpose();
-    auto d_trans = dmtx->transpose();
-    gko::test::unsort_matrix(static_cast<Mtx *>(dmtx.get()), rand_engine);
-    gko::test::unsort_matrix(static_cast<Mtx *>(d_trans.get()), rand_engine);
-
-    mtx->apply(trans.get(), square_mtx.get());
-    dmtx->apply(d_trans.get(), square_dmtx.get());
-
-    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, 1e-14);
-    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
-}
-
-
 TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>());
diff --git a/dpcpp/test/matrix/csr_kernels.cpp b/dpcpp/test/matrix/csr_kernels.cpp
index a5e93e0462a..89dfd31d628 100644
--- a/dpcpp/test/matrix/csr_kernels.cpp
+++ b/dpcpp/test/matrix/csr_kernels.cpp
@@ -431,25 +431,6 @@ TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRef)
 }
 
 
-/*
-TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRefUnsorted)
-{
-    set_up_apply_data(std::make_shared<Mtx::automatical>());
-    auto trans = mtx->transpose();
-    auto d_trans = dmtx->transpose();
-    gko::test::unsort_matrix(static_cast<Mtx *>(dmtx.get()), rand_engine);
-    gko::test::unsort_matrix(static_cast<Mtx *>(d_trans.get()), rand_engine);
-
-    mtx->apply(trans.get(), square_mtx.get());
-    dmtx->apply(d_trans.get(), square_dmtx.get());
-
-    GKO_ASSERT_MTX_NEAR(square_dmtx, square_mtx, r<vtype>::value);
-    GKO_ASSERT_MTX_EQ_SPARSITY(square_dmtx, square_mtx);
-    ASSERT_TRUE(square_dmtx->is_sorted_by_column_index());
-}
-*/
-
-
 TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef)
 {
     set_up_apply_data(std::make_shared<Mtx::automatical>());
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index 2c41ced1104..f7627b54053 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -368,7 +368,10 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
         /**
          * Creates a load_balance strategy with DPCPP executor.
          *
-         * @param exec the HIP executor
+         * @param exec the DPCPP executor
+         *
+         * @note TODO: porting - we hardcode the subgroup size is 16 and the
+         *             number of threads in a SIMD unit is 7
          */
         load_balance(std::shared_ptr<const DpcppExecutor> exec)
             : load_balance(exec->get_num_computing_units() * 7, 16, false)

From 6d5b46625560f064212865aca7b37e418d00f5ca Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 29 Jul 2021 15:12:36 +0200
Subject: [PATCH 0981/1008] fix hip, keep consistence, rename cusparse->onemkl
 also relax a error bound for ell small mtx test

---
 dev_tools/oneapi/add_host_function.sh |   2 +-
 dpcpp/matrix/coo_kernels.dp.cpp       |  57 +++---
 dpcpp/matrix/csr_kernels.dp.cpp       | 238 +++++++++++++-------------
 dpcpp/matrix/diagonal_kernels.dp.cpp  |  43 +++--
 dpcpp/matrix/hybrid_kernels.dp.cpp    |  19 +-
 dpcpp/matrix/sellp_kernels.dp.cpp     |  53 +++---
 dpcpp/test/matrix/csr_kernels.cpp     |   6 +-
 dpcpp/test/matrix/ell_kernels.cpp     |   2 +-
 include/ginkgo/core/matrix/csr.hpp    |  14 +-
 9 files changed, 213 insertions(+), 221 deletions(-)

diff --git a/dev_tools/oneapi/add_host_function.sh b/dev_tools/oneapi/add_host_function.sh
index 9d6438915e1..845684537d4 100755
--- a/dev_tools/oneapi/add_host_function.sh
+++ b/dev_tools/oneapi/add_host_function.sh
@@ -10,7 +10,7 @@
 # add_host_function will add another host function with the same template and calling the cuda call
 # ```
 # template <int config = 1, int info, typename ValueType>
-# void kernel_AUTOHOSTFUNC(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory, cudaStream_t stream, ValueType a) {
+# void kernel_AUTOHOSTFUNC(dim3 grid, dim3 block, size_type dynamic_shared_memory, cudaStream_t stream, ValueType a) {
 #     /*KEEP*/kernel<config, info><<<grid, block, dynamic_shared_memory, stream>>>(a);
 # }
 # ```
diff --git a/dpcpp/matrix/coo_kernels.dp.cpp b/dpcpp/matrix/coo_kernels.dp.cpp
index 8f22e4a67c8..a6150a75681 100644
--- a/dpcpp/matrix/coo_kernels.dp.cpp
+++ b/dpcpp/matrix/coo_kernels.dp.cpp
@@ -167,14 +167,14 @@ void abstract_spmv(const size_type nnz, const size_type num_lines,
 }
 
 template <typename ValueType, typename IndexType>
-void abstract_spmv(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                   sycl::queue *stream, const size_type nnz,
+void abstract_spmv(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                   sycl::queue *queue, const size_type nnz,
                    const size_type num_lines, const ValueType *val,
                    const IndexType *col, const IndexType *row,
                    const ValueType *b, const size_type b_stride, ValueType *c,
                    const size_type c_stride)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              abstract_spmv(nnz, num_lines, val, col, row, b,
@@ -202,15 +202,15 @@ void abstract_spmv(const size_type nnz, const size_type num_lines,
 }
 
 template <typename ValueType, typename IndexType>
-void abstract_spmv(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                   sycl::queue *stream, const size_type nnz,
+void abstract_spmv(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                   sycl::queue *queue, const size_type nnz,
                    const size_type num_lines, const ValueType *alpha,
                    const ValueType *val, const IndexType *col,
                    const IndexType *row, const ValueType *b,
                    const size_type b_stride, ValueType *c,
                    const size_type c_stride)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              abstract_spmv(nnz, num_lines, alpha, val, col, row,
@@ -292,15 +292,15 @@ void abstract_spmm(const size_type nnz, const size_type num_elems,
 }
 
 template <typename ValueType, typename IndexType>
-void abstract_spmm(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                   sycl::queue *stream, const size_type nnz,
+void abstract_spmm(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                   sycl::queue *queue, const size_type nnz,
                    const size_type num_elems, const ValueType *val,
                    const IndexType *col, const IndexType *row,
                    const size_type num_cols, const ValueType *b,
                    const size_type b_stride, ValueType *c,
                    const size_type c_stride)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 abstract_spmm(nnz, num_elems, val, col, row, num_cols, b,
@@ -328,15 +328,15 @@ void abstract_spmm(const size_type nnz, const size_type num_elems,
 }
 
 template <typename ValueType, typename IndexType>
-void abstract_spmm(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                   sycl::queue *stream, const size_type nnz,
+void abstract_spmm(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                   sycl::queue *queue, const size_type nnz,
                    const size_type num_elems, const ValueType *alpha,
                    const ValueType *val, const IndexType *col,
                    const IndexType *row, const size_type num_cols,
                    const ValueType *b, const size_type b_stride, ValueType *c,
                    const size_type c_stride)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 abstract_spmm(nnz, num_elems, alpha, val, col, row, num_cols, b,
@@ -376,12 +376,12 @@ void convert_row_idxs_to_ptrs(const IndexType *__restrict__ idxs,
 
 template <typename IndexType>
 void convert_row_idxs_to_ptrs(dim3 grid, dim3 block,
-                              gko::size_type dynamic_shared_memory,
-                              sycl::queue *stream, const IndexType *idxs,
+                              size_type dynamic_shared_memory,
+                              sycl::queue *queue, const IndexType *idxs,
                               size_type num_nonzeros, IndexType *ptrs,
                               size_type length)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              convert_row_idxs_to_ptrs(idxs, num_nonzeros, ptrs,
@@ -415,12 +415,11 @@ void initialize_zero_dense(size_type num_rows, size_type num_cols,
 
 template <typename ValueType>
 void initialize_zero_dense(dim3 grid, dim3 block,
-                           gko::size_type dynamic_shared_memory,
-                           sycl::queue *stream, size_type num_rows,
-                           size_type num_cols, size_type stride,
-                           ValueType *result)
+                           size_type dynamic_shared_memory, sycl::queue *queue,
+                           size_type num_rows, size_type num_cols,
+                           size_type stride, ValueType *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              initialize_zero_dense(num_rows, num_cols, stride,
@@ -443,12 +442,12 @@ void fill_in_dense(size_type nnz, const IndexType *__restrict__ row_idxs,
 }
 
 template <typename ValueType, typename IndexType>
-void fill_in_dense(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                   sycl::queue *stream, size_type nnz,
-                   const IndexType *row_idxs, const IndexType *col_idxs,
-                   const ValueType *values, size_type stride, ValueType *result)
+void fill_in_dense(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                   sycl::queue *queue, size_type nnz, const IndexType *row_idxs,
+                   const IndexType *col_idxs, const ValueType *values,
+                   size_type stride, ValueType *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              fill_in_dense(nnz, row_idxs, col_idxs, values,
@@ -474,13 +473,13 @@ void extract_diagonal(size_type nnz, const ValueType *__restrict__ orig_values,
 }
 
 template <typename ValueType, typename IndexType>
-void extract_diagonal(dim3 grid, dim3 block,
-                      gko::size_type dynamic_shared_memory, sycl::queue *stream,
-                      size_type nnz, const ValueType *orig_values,
+void extract_diagonal(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                      sycl::queue *queue, size_type nnz,
+                      const ValueType *orig_values,
                       const IndexType *orig_row_idxs,
                       const IndexType *orig_col_idxs, ValueType *diag)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              extract_diagonal(nnz, orig_values, orig_row_idxs,
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index a69604f8341..b8fbb6cadd9 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -273,15 +273,15 @@ void abstract_spmv(const IndexType nwarps, const IndexType num_rows,
 }
 
 template <typename ValueType, typename IndexType>
-void abstract_spmv(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                   sycl::queue *stream, const IndexType nwarps,
+void abstract_spmv(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                   sycl::queue *queue, const IndexType nwarps,
                    const IndexType num_rows, const ValueType *val,
                    const IndexType *col_idxs, const IndexType *row_ptrs,
                    const IndexType *srow, const ValueType *b,
                    const size_type b_stride, ValueType *c,
                    const size_type c_stride)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 abstract_spmv(nwarps, num_rows, val, col_idxs, row_ptrs, srow,
@@ -311,15 +311,15 @@ void abstract_spmv(const IndexType nwarps, const IndexType num_rows,
 }
 
 template <typename ValueType, typename IndexType>
-void abstract_spmv(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                   sycl::queue *stream, const IndexType nwarps,
+void abstract_spmv(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                   sycl::queue *queue, const IndexType nwarps,
                    const IndexType num_rows, const ValueType *alpha,
                    const ValueType *val, const IndexType *col_idxs,
                    const IndexType *row_ptrs, const IndexType *srow,
                    const ValueType *b, const size_type b_stride, ValueType *c,
                    const size_type c_stride)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 abstract_spmv(nwarps, num_rows, alpha, val, col_idxs, row_ptrs,
@@ -340,10 +340,10 @@ void set_zero(const size_type nnz, ValueType *__restrict__ val,
 }
 
 template <typename ValueType>
-void set_zero(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-              sycl::queue *stream, const size_type nnz, ValueType *val)
+void set_zero(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+              sycl::queue *queue, const size_type nnz, ValueType *val)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block),
             [=](sycl::nd_item<3> item_ct1) { set_zero(nnz, val, item_ct1); });
@@ -517,15 +517,15 @@ void abstract_merge_path_spmv(
 
 template <int items_per_thread, typename ValueType, typename IndexType>
 void abstract_merge_path_spmv(dim3 grid, dim3 block,
-                              gko::size_type dynamic_shared_memory,
-                              sycl::queue *stream, const IndexType num_rows,
+                              size_type dynamic_shared_memory,
+                              sycl::queue *queue, const IndexType num_rows,
                               const ValueType *val, const IndexType *col_idxs,
                               const IndexType *row_ptrs, const IndexType *srow,
                               const ValueType *b, const size_type b_stride,
                               ValueType *c, const size_type c_stride,
                               IndexType *row_out, ValueType *val_out)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<IndexType, 1, sycl::access_mode::read_write,
                        sycl::access::target::local>
             shared_row_ptrs_acc_ct1(
@@ -565,14 +565,14 @@ void abstract_merge_path_spmv(
 
 template <int items_per_thread, typename ValueType, typename IndexType>
 void abstract_merge_path_spmv(
-    dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-    sycl::queue *stream, const IndexType num_rows, const ValueType *alpha,
-    const ValueType *val, const IndexType *col_idxs, const IndexType *row_ptrs,
-    const IndexType *srow, const ValueType *b, const size_type b_stride,
-    const ValueType *beta, ValueType *c, const size_type c_stride,
-    IndexType *row_out, ValueType *val_out)
-{
-    stream->submit([&](sycl::handler &cgh) {
+    dim3 grid, dim3 block, size_type dynamic_shared_memory, sycl::queue *queue,
+    const IndexType num_rows, const ValueType *alpha, const ValueType *val,
+    const IndexType *col_idxs, const IndexType *row_ptrs, const IndexType *srow,
+    const ValueType *b, const size_type b_stride, const ValueType *beta,
+    ValueType *c, const size_type c_stride, IndexType *row_out,
+    ValueType *val_out)
+{
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<IndexType, 1, sycl::access_mode::read_write,
                        sycl::access::target::local>
             shared_row_ptrs_acc_ct1(
@@ -605,13 +605,12 @@ void abstract_reduce(const IndexType nwarps,
 }
 
 template <typename ValueType, typename IndexType>
-void abstract_reduce(dim3 grid, dim3 block,
-                     gko::size_type dynamic_shared_memory, sycl::queue *stream,
-                     const IndexType nwarps, const ValueType *last_val,
-                     const IndexType *last_row, ValueType *c,
-                     const size_type c_stride)
+void abstract_reduce(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                     sycl::queue *queue, const IndexType nwarps,
+                     const ValueType *last_val, const IndexType *last_row,
+                     ValueType *c, const size_type c_stride)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<IndexType, spmv_block_size>, 0,
                        sycl::access_mode::read_write,
                        sycl::access::target::local>
@@ -649,13 +648,13 @@ void abstract_reduce(const IndexType nwarps,
 }
 
 template <typename ValueType, typename IndexType>
-void abstract_reduce(dim3 grid, dim3 block,
-                     gko::size_type dynamic_shared_memory, sycl::queue *stream,
-                     const IndexType nwarps, const ValueType *last_val,
-                     const IndexType *last_row, const ValueType *alpha,
-                     ValueType *c, const size_type c_stride)
+void abstract_reduce(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                     sycl::queue *queue, const IndexType nwarps,
+                     const ValueType *last_val, const IndexType *last_row,
+                     const ValueType *alpha, ValueType *c,
+                     const size_type c_stride)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<UninitializedArray<IndexType, spmv_block_size>, 0,
                        sycl::access_mode::read_write,
                        sycl::access::target::local>
@@ -675,6 +674,13 @@ void abstract_reduce(dim3 grid, dim3 block,
 }
 
 
+#ifdef __SYCL_DEVICE_ONLY__
+#define CONSTANT __attribute__((opencl_constant))
+#else
+#define CONSTANT
+#endif
+
+
 template <size_type subwarp_size, typename ValueType, typename IndexType,
           typename Closure>
 void device_classical_spmv(const size_type num_rows,
@@ -702,6 +708,7 @@ void device_classical_spmv(const size_type num_rows,
         auto subwarp_result = ::gko::kernels::dpcpp::reduce(
             subwarp_tile, temp_val,
             [](const ValueType &a, const ValueType &b) { return a + b; });
+        item_ct1.barrier();
         if (subid == 0) {
             c[row * c_stride + column_id] =
                 scale(subwarp_result, c[row * c_stride + column_id]);
@@ -725,14 +732,14 @@ void abstract_classical_spmv(
 
 template <size_type subwarp_size, typename ValueType, typename IndexType>
 void abstract_classical_spmv(dim3 grid, dim3 block,
-                             gko::size_type dynamic_shared_memory,
-                             sycl::queue *stream, const size_type num_rows,
+                             size_type dynamic_shared_memory,
+                             sycl::queue *queue, const size_type num_rows,
                              const ValueType *val, const IndexType *col_idxs,
                              const IndexType *row_ptrs, const ValueType *b,
                              const size_type b_stride, ValueType *c,
                              const size_type c_stride)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 abstract_classical_spmv<subwarp_size>(num_rows, val, col_idxs,
@@ -764,15 +771,15 @@ void abstract_classical_spmv(
 
 template <size_type subwarp_size, typename ValueType, typename IndexType>
 void abstract_classical_spmv(dim3 grid, dim3 block,
-                             gko::size_type dynamic_shared_memory,
-                             sycl::queue *stream, const size_type num_rows,
+                             size_type dynamic_shared_memory,
+                             sycl::queue *queue, const size_type num_rows,
                              const ValueType *alpha, const ValueType *val,
                              const IndexType *col_idxs,
                              const IndexType *row_ptrs, const ValueType *b,
                              const size_type b_stride, const ValueType *beta,
                              ValueType *c, const size_type c_stride)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              abstract_classical_spmv<subwarp_size>(
@@ -799,11 +806,11 @@ void convert_row_ptrs_to_idxs(size_type num_rows,
 
 template <typename IndexType>
 void convert_row_ptrs_to_idxs(dim3 grid, dim3 block,
-                              gko::size_type dynamic_shared_memory,
-                              sycl::queue *stream, size_type num_rows,
+                              size_type dynamic_shared_memory,
+                              sycl::queue *queue, size_type num_rows,
                               const IndexType *ptrs, IndexType *idxs)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 convert_row_ptrs_to_idxs(num_rows, ptrs, idxs, item_ct1);
@@ -830,12 +837,11 @@ void initialize_zero_dense(size_type num_rows, size_type num_cols,
 
 template <typename ValueType>
 void initialize_zero_dense(dim3 grid, dim3 block,
-                           gko::size_type dynamic_shared_memory,
-                           sycl::queue *stream, size_type num_rows,
-                           size_type num_cols, size_type stride,
-                           ValueType *result)
+                           size_type dynamic_shared_memory, sycl::queue *queue,
+                           size_type num_rows, size_type num_cols,
+                           size_type stride, ValueType *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              initialize_zero_dense(num_rows, num_cols, stride,
@@ -860,12 +866,12 @@ void fill_in_dense(size_type num_rows, const IndexType *__restrict__ row_ptrs,
 }
 
 template <typename ValueType, typename IndexType>
-void fill_in_dense(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                   sycl::queue *stream, size_type num_rows,
+void fill_in_dense(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                   sycl::queue *queue, size_type num_rows,
                    const IndexType *row_ptrs, const IndexType *col_idxs,
                    const ValueType *values, size_type stride, ValueType *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              fill_in_dense(num_rows, row_ptrs, col_idxs, values,
@@ -889,11 +895,11 @@ void calculate_nnz_per_row(size_type num_rows,
 
 template <typename IndexType>
 void calculate_nnz_per_row(dim3 grid, dim3 block,
-                           gko::size_type dynamic_shared_memory,
-                           sycl::queue *stream, size_type num_rows,
-                           const IndexType *row_ptrs, size_type *nnz_per_row)
+                           size_type dynamic_shared_memory, sycl::queue *queue,
+                           size_type num_rows, const IndexType *row_ptrs,
+                           size_type *nnz_per_row)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              calculate_nnz_per_row(num_rows, row_ptrs,
@@ -939,13 +945,13 @@ void calculate_slice_lengths(size_type num_rows, size_type slice_size,
 }
 
 void calculate_slice_lengths(dim3 grid, dim3 block,
-                             gko::size_type dynamic_shared_memory,
-                             sycl::queue *stream, size_type num_rows,
+                             size_type dynamic_shared_memory,
+                             sycl::queue *queue, size_type num_rows,
                              size_type slice_size, size_type stride_factor,
                              const size_type *nnz_per_row,
                              size_type *slice_lengths, size_type *slice_sets)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 calculate_slice_lengths(num_rows, slice_size, stride_factor,
@@ -991,15 +997,15 @@ void fill_in_sellp(size_type num_rows, size_type slice_size,
 }
 
 template <typename ValueType, typename IndexType>
-void fill_in_sellp(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                   sycl::queue *stream, size_type num_rows,
-                   size_type slice_size, const ValueType *source_values,
+void fill_in_sellp(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                   sycl::queue *queue, size_type num_rows, size_type slice_size,
+                   const ValueType *source_values,
                    const IndexType *source_row_ptrs,
                    const IndexType *source_col_idxs, size_type *slice_lengths,
                    size_type *slice_sets, IndexType *result_col_idxs,
                    ValueType *result_values)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 fill_in_sellp(num_rows, slice_size, source_values,
@@ -1026,13 +1032,12 @@ void initialize_zero_ell(size_type max_nnz_per_row, size_type stride,
 }
 
 template <typename ValueType, typename IndexType>
-void initialize_zero_ell(dim3 grid, dim3 block,
-                         gko::size_type dynamic_shared_memory,
-                         sycl::queue *stream, size_type max_nnz_per_row,
+void initialize_zero_ell(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                         sycl::queue *queue, size_type max_nnz_per_row,
                          size_type stride, ValueType *values,
                          IndexType *col_idxs)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              initialize_zero_ell(max_nnz_per_row, stride,
@@ -1068,14 +1073,14 @@ void fill_in_ell(size_type num_rows, size_type stride,
 }
 
 template <typename ValueType, typename IndexType>
-void fill_in_ell(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                 sycl::queue *stream, size_type num_rows, size_type stride,
+void fill_in_ell(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                 sycl::queue *queue, size_type num_rows, size_type stride,
                  const ValueType *source_values,
                  const IndexType *source_row_ptrs,
                  const IndexType *source_col_idxs, ValueType *result_values,
                  IndexType *result_col_idxs)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 fill_in_ell(num_rows, stride, source_values, source_row_ptrs,
@@ -1116,12 +1121,12 @@ void reduce_max_nnz_per_slice(size_type num_rows, size_type slice_size,
 }
 
 void reduce_max_nnz_per_slice(dim3 grid, dim3 block,
-                              gko::size_type dynamic_shared_memory,
-                              sycl::queue *stream, size_type num_rows,
+                              size_type dynamic_shared_memory,
+                              sycl::queue *queue, size_type num_rows,
                               size_type slice_size, size_type stride_factor,
                               const size_type *nnz_per_row, size_type *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 reduce_max_nnz_per_slice(num_rows, slice_size, stride_factor,
@@ -1144,12 +1149,11 @@ void reduce_total_cols(size_type num_slices,
     }
 }
 
-void reduce_total_cols(dim3 grid, dim3 block,
-                       gko::size_type dynamic_shared_memory,
-                       sycl::queue *stream, size_type num_slices,
+void reduce_total_cols(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                       sycl::queue *queue, size_type num_slices,
                        const size_type *max_nnz_per_slice, size_type *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<size_type, 1, sycl::access_mode::read_write,
                        sycl::access::target::local>
             block_result_acc_ct1(sycl::range<1>(512 /*default_block_size*/),
@@ -1177,11 +1181,11 @@ void reduce_max_nnz(size_type size, const size_type *__restrict__ nnz_per_row,
     }
 }
 
-void reduce_max_nnz(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                    sycl::queue *stream, size_type size,
+void reduce_max_nnz(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                    sycl::queue *queue, size_type size,
                     const size_type *nnz_per_row, size_type *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<size_type, 1, sycl::access_mode::read_write,
                        sycl::access::target::local>
             block_max_acc_ct1(sycl::range<1>(512 /*default_block_size*/), cgh);
@@ -1212,13 +1216,13 @@ void calculate_hybrid_coo_row_nnz(size_type num_rows,
 
 template <typename IndexType>
 void calculate_hybrid_coo_row_nnz(dim3 grid, dim3 block,
-                                  gko::size_type dynamic_shared_memory,
-                                  sycl::queue *stream, size_type num_rows,
+                                  size_type dynamic_shared_memory,
+                                  sycl::queue *queue, size_type num_rows,
                                   size_type ell_max_nnz_per_row,
                                   IndexType *csr_row_idxs,
                                   size_type *coo_row_nnz)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 calculate_hybrid_coo_row_nnz(num_rows, ell_max_nnz_per_row,
@@ -1268,8 +1272,8 @@ void fill_in_hybrid(size_type num_rows, size_type stride,
 }
 
 template <typename ValueType, typename IndexType>
-void fill_in_hybrid(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                    sycl::queue *stream, size_type num_rows, size_type stride,
+void fill_in_hybrid(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                    sycl::queue *queue, size_type num_rows, size_type stride,
                     size_type ell_max_nnz_per_row,
                     const ValueType *source_values,
                     const IndexType *source_row_ptrs,
@@ -1278,7 +1282,7 @@ void fill_in_hybrid(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
                     IndexType *result_ell_col, ValueType *result_coo_val,
                     IndexType *result_coo_col, IndexType *result_coo_row)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 fill_in_hybrid(num_rows, stride, ell_max_nnz_per_row,
@@ -1320,11 +1324,11 @@ void check_unsorted(const IndexType *__restrict__ row_ptrs,
 }
 
 template <typename IndexType>
-void check_unsorted(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                    sycl::queue *stream, const IndexType *row_ptrs,
+void check_unsorted(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                    sycl::queue *queue, const IndexType *row_ptrs,
                     const IndexType *col_idxs, IndexType num_rows, bool *flag)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<bool, 0, sycl::access_mode::read_write,
                        sycl::access::target::local>
             sh_flag_acc_ct1(cgh);
@@ -1364,14 +1368,13 @@ void extract_diagonal(size_type diag_size, size_type nnz,
 }
 
 template <typename ValueType, typename IndexType>
-void extract_diagonal(dim3 grid, dim3 block,
-                      gko::size_type dynamic_shared_memory, sycl::queue *stream,
-                      size_type diag_size, size_type nnz,
+void extract_diagonal(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                      sycl::queue *queue, size_type diag_size, size_type nnz,
                       const ValueType *orig_values,
                       const IndexType *orig_row_ptrs,
                       const IndexType *orig_col_idxs, ValueType *diag)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 extract_diagonal(diag_size, nnz, orig_values, orig_row_ptrs,
@@ -1399,11 +1402,11 @@ void conjugate_kernel(size_type num_nonzeros, ValueType *__restrict__ val,
 }
 
 template <typename ValueType>
-void conjugate_kernel(dim3 grid, dim3 block,
-                      gko::size_type dynamic_shared_memory, sycl::queue *stream,
-                      size_type num_nonzeros, ValueType *val)
+void conjugate_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                      sycl::queue *queue, size_type num_nonzeros,
+                      ValueType *val)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              conjugate_kernel(num_nonzeros, val, item_ct1);
@@ -1430,12 +1433,11 @@ void inv_permutation_kernel(size_type size,
 
 template <typename IndexType>
 void inv_permutation_kernel(dim3 grid, dim3 block,
-                            gko::size_type dynamic_shared_memory,
-                            sycl::queue *stream, size_type size,
-                            const IndexType *permutation,
+                            size_type dynamic_shared_memory, sycl::queue *queue,
+                            size_type size, const IndexType *permutation,
                             IndexType *inv_permutation)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              inv_permutation_kernel(size, permutation,
@@ -1467,15 +1469,14 @@ void col_permute_kernel(size_type num_rows, size_type num_nonzeros,
 }
 
 template <typename ValueType, typename IndexType>
-void col_permute_kernel(dim3 grid, dim3 block,
-                        gko::size_type dynamic_shared_memory,
-                        sycl::queue *stream, size_type num_rows,
+void col_permute_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                        sycl::queue *queue, size_type num_rows,
                         size_type num_nonzeros, const IndexType *permutation,
                         const IndexType *in_row_ptrs, const IndexType *in_cols,
                         const ValueType *in_vals, IndexType *out_row_ptrs,
                         IndexType *out_cols, ValueType *out_vals)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 col_permute_kernel(num_rows, num_nonzeros, permutation,
@@ -1504,12 +1505,11 @@ void row_ptr_permute_kernel(size_type num_rows,
 
 template <typename IndexType>
 void row_ptr_permute_kernel(dim3 grid, dim3 block,
-                            gko::size_type dynamic_shared_memory,
-                            sycl::queue *stream, size_type num_rows,
-                            const IndexType *permutation,
+                            size_type dynamic_shared_memory, sycl::queue *queue,
+                            size_type num_rows, const IndexType *permutation,
                             const IndexType *in_row_ptrs, IndexType *out_nnz)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 row_ptr_permute_kernel(num_rows, permutation, in_row_ptrs,
@@ -1537,13 +1537,13 @@ void inv_row_ptr_permute_kernel(size_type num_rows,
 
 template <typename IndexType>
 void inv_row_ptr_permute_kernel(dim3 grid, dim3 block,
-                                gko::size_type dynamic_shared_memory,
-                                sycl::queue *stream, size_type num_rows,
+                                size_type dynamic_shared_memory,
+                                sycl::queue *queue, size_type num_rows,
                                 const IndexType *permutation,
                                 const IndexType *in_row_ptrs,
                                 IndexType *out_nnz)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 inv_row_ptr_permute_kernel(num_rows, permutation, in_row_ptrs,
@@ -1581,15 +1581,14 @@ void row_permute_kernel(size_type num_rows,
 }
 
 template <int subwarp_size, typename ValueType, typename IndexType>
-void row_permute_kernel(dim3 grid, dim3 block,
-                        gko::size_type dynamic_shared_memory,
-                        sycl::queue *stream, size_type num_rows,
+void row_permute_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                        sycl::queue *queue, size_type num_rows,
                         const IndexType *permutation,
                         const IndexType *in_row_ptrs, const IndexType *in_cols,
                         const ValueType *in_vals, const IndexType *out_row_ptrs,
                         IndexType *out_cols, ValueType *out_vals)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 row_permute_kernel<subwarp_size>(
@@ -1629,15 +1628,14 @@ void inv_row_permute_kernel(size_type num_rows,
 
 template <int subwarp_size, typename ValueType, typename IndexType>
 void inv_row_permute_kernel(dim3 grid, dim3 block,
-                            gko::size_type dynamic_shared_memory,
-                            sycl::queue *stream, size_type num_rows,
-                            const IndexType *permutation,
+                            size_type dynamic_shared_memory, sycl::queue *queue,
+                            size_type num_rows, const IndexType *permutation,
                             const IndexType *in_row_ptrs,
                             const IndexType *in_cols, const ValueType *in_vals,
                             const IndexType *out_row_ptrs, IndexType *out_cols,
                             ValueType *out_vals)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 inv_row_permute_kernel<subwarp_size>(
@@ -1677,15 +1675,15 @@ void inv_symm_permute_kernel(size_type num_rows,
 
 template <int subwarp_size, typename ValueType, typename IndexType>
 void inv_symm_permute_kernel(dim3 grid, dim3 block,
-                             gko::size_type dynamic_shared_memory,
-                             sycl::queue *stream, size_type num_rows,
+                             size_type dynamic_shared_memory,
+                             sycl::queue *queue, size_type num_rows,
                              const IndexType *permutation,
                              const IndexType *in_row_ptrs,
                              const IndexType *in_cols, const ValueType *in_vals,
                              const IndexType *out_row_ptrs, IndexType *out_cols,
                              ValueType *out_vals)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 inv_symm_permute_kernel<subwarp_size>(
diff --git a/dpcpp/matrix/diagonal_kernels.dp.cpp b/dpcpp/matrix/diagonal_kernels.dp.cpp
index 737283291d2..866df7ec315 100644
--- a/dpcpp/matrix/diagonal_kernels.dp.cpp
+++ b/dpcpp/matrix/diagonal_kernels.dp.cpp
@@ -82,13 +82,13 @@ void apply_to_dense(size_type num_rows, size_type num_cols,
 }
 
 template <typename ValueType>
-void apply_to_dense(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                    sycl::queue *stream, size_type num_rows, size_type num_cols,
+void apply_to_dense(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                    sycl::queue *queue, size_type num_rows, size_type num_cols,
                     const ValueType *diag, size_type source_stride,
                     const ValueType *source_values, size_type result_stride,
                     ValueType *result_values)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 apply_to_dense(num_rows, num_cols, diag, source_stride,
@@ -120,14 +120,13 @@ void right_apply_to_dense(size_type num_rows, size_type num_cols,
 
 template <typename ValueType>
 void right_apply_to_dense(dim3 grid, dim3 block,
-                          gko::size_type dynamic_shared_memory,
-                          sycl::queue *stream, size_type num_rows,
-                          size_type num_cols, const ValueType *diag,
-                          size_type source_stride,
+                          size_type dynamic_shared_memory, sycl::queue *queue,
+                          size_type num_rows, size_type num_cols,
+                          const ValueType *diag, size_type source_stride,
                           const ValueType *source_values,
                           size_type result_stride, ValueType *result_values)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 right_apply_to_dense(num_rows, num_cols, diag, source_stride,
@@ -163,12 +162,11 @@ void apply_to_csr(size_type num_rows, const ValueType *__restrict__ diag,
 }
 
 template <typename ValueType, typename IndexType>
-void apply_to_csr(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                  sycl::queue *stream, size_type num_rows,
-                  const ValueType *diag, const IndexType *row_ptrs,
-                  ValueType *result_values)
+void apply_to_csr(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                  sycl::queue *queue, size_type num_rows, const ValueType *diag,
+                  const IndexType *row_ptrs, ValueType *result_values)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 apply_to_csr(num_rows, diag, row_ptrs, result_values, item_ct1);
@@ -193,13 +191,12 @@ void right_apply_to_csr(size_type num_nnz, const ValueType *__restrict__ diag,
 }
 
 template <typename ValueType, typename IndexType>
-void right_apply_to_csr(dim3 grid, dim3 block,
-                        gko::size_type dynamic_shared_memory,
-                        sycl::queue *stream, size_type num_nnz,
+void right_apply_to_csr(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                        sycl::queue *queue, size_type num_nnz,
                         const ValueType *diag, const IndexType *col_idxs,
                         ValueType *result_values)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              right_apply_to_csr(num_nnz, diag, col_idxs,
@@ -231,12 +228,12 @@ void convert_to_csr(size_type size, const ValueType *__restrict__ diag_values,
 }
 
 template <typename ValueType, typename IndexType>
-void convert_to_csr(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                    sycl::queue *stream, size_type size,
+void convert_to_csr(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                    sycl::queue *queue, size_type size,
                     const ValueType *diag_values, IndexType *row_ptrs,
                     IndexType *col_idxs, ValueType *csr_values)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              convert_to_csr(size, diag_values, row_ptrs,
@@ -261,11 +258,11 @@ void conj_transpose(size_type size, const ValueType *__restrict__ orig_values,
 }
 
 template <typename ValueType>
-void conj_transpose(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                    sycl::queue *stream, size_type size,
+void conj_transpose(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                    sycl::queue *queue, size_type size,
                     const ValueType *orig_values, ValueType *trans_values)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 conj_transpose(size, orig_values, trans_values, item_ct1);
diff --git a/dpcpp/matrix/hybrid_kernels.dp.cpp b/dpcpp/matrix/hybrid_kernels.dp.cpp
index 5f2b4ab906d..3a5b0f05173 100644
--- a/dpcpp/matrix/hybrid_kernels.dp.cpp
+++ b/dpcpp/matrix/hybrid_kernels.dp.cpp
@@ -136,13 +136,12 @@ void count_coo_row_nnz(const size_type nnz, const size_type num_lines,
 
 template <int subwarp_size = config::warp_size, typename ValueType,
           typename IndexType>
-void count_coo_row_nnz(dim3 grid, dim3 block,
-                       gko::size_type dynamic_shared_memory,
-                       sycl::queue *stream, const size_type nnz,
+void count_coo_row_nnz(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                       sycl::queue *queue, const size_type nnz,
                        const size_type num_lines, const ValueType *val,
                        const IndexType *row, IndexType *nnz_per_row)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 count_coo_row_nnz<subwarp_size>(nnz, num_lines, val, row,
@@ -187,15 +186,15 @@ void fill_in_csr(size_type num_rows, size_type max_nnz_per_row,
 }
 
 template <typename ValueType, typename IndexType>
-void fill_in_csr(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                 sycl::queue *stream, size_type num_rows,
+void fill_in_csr(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                 sycl::queue *queue, size_type num_rows,
                  size_type max_nnz_per_row, size_type stride,
                  const ValueType *ell_val, const IndexType *ell_col,
                  const ValueType *coo_val, const IndexType *coo_col,
                  const IndexType *coo_offset, IndexType *result_row_ptrs,
                  IndexType *result_col_idxs, ValueType *result_values)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 fill_in_csr(num_rows, max_nnz_per_row, stride, ell_val, ell_col,
@@ -217,11 +216,11 @@ void add(size_type num, ValueType1 *__restrict__ val1,
 }
 
 template <typename ValueType1, typename ValueType2>
-void add(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-         sycl::queue *stream, size_type num, ValueType1 *val1,
+void add(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+         sycl::queue *queue, size_type num, ValueType1 *val1,
          const ValueType2 *val2)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block),
             [=](sycl::nd_item<3> item_ct1) { add(num, val1, val2, item_ct1); });
diff --git a/dpcpp/matrix/sellp_kernels.dp.cpp b/dpcpp/matrix/sellp_kernels.dp.cpp
index 7a9a218224b..f1d6ba433d9 100644
--- a/dpcpp/matrix/sellp_kernels.dp.cpp
+++ b/dpcpp/matrix/sellp_kernels.dp.cpp
@@ -96,14 +96,14 @@ void spmv_kernel(size_type num_rows, size_type num_right_hand_sides,
 }
 
 template <typename ValueType, typename IndexType>
-void spmv_kernel(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                 sycl::queue *stream, size_type num_rows,
+void spmv_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                 sycl::queue *queue, size_type num_rows,
                  size_type num_right_hand_sides, size_type b_stride,
                  size_type c_stride, const size_type *slice_lengths,
                  const size_type *slice_sets, const ValueType *a,
                  const IndexType *col, const ValueType *b, ValueType *c)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 spmv_kernel(num_rows, num_right_hand_sides, b_stride, c_stride,
@@ -145,16 +145,16 @@ void advanced_spmv_kernel(size_type num_rows, size_type num_right_hand_sides,
 
 template <typename ValueType, typename IndexType>
 void advanced_spmv_kernel(dim3 grid, dim3 block,
-                          gko::size_type dynamic_shared_memory,
-                          sycl::queue *stream, size_type num_rows,
-                          size_type num_right_hand_sides, size_type b_stride,
-                          size_type c_stride, const size_type *slice_lengths,
+                          size_type dynamic_shared_memory, sycl::queue *queue,
+                          size_type num_rows, size_type num_right_hand_sides,
+                          size_type b_stride, size_type c_stride,
+                          const size_type *slice_lengths,
                           const size_type *slice_sets, const ValueType *alpha,
                           const ValueType *a, const IndexType *col,
                           const ValueType *b, const ValueType *beta,
                           ValueType *c)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 advanced_spmv_kernel(num_rows, num_right_hand_sides, b_stride,
@@ -189,12 +189,11 @@ void initialize_zero_dense(size_type num_rows, size_type num_cols,
 
 template <typename ValueType>
 void initialize_zero_dense(dim3 grid, dim3 block,
-                           gko::size_type dynamic_shared_memory,
-                           sycl::queue *stream, size_type num_rows,
-                           size_type num_cols, size_type stride,
-                           ValueType *result)
+                           size_type dynamic_shared_memory, sycl::queue *queue,
+                           size_type num_rows, size_type num_cols,
+                           size_type stride, ValueType *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              initialize_zero_dense(num_rows, num_cols, stride,
@@ -233,14 +232,14 @@ void fill_in_dense(size_type num_rows, size_type num_cols, size_type stride,
 }
 
 template <unsigned int threads_per_row, typename ValueType, typename IndexType>
-void fill_in_dense(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                   sycl::queue *stream, size_type num_rows, size_type num_cols,
+void fill_in_dense(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                   sycl::queue *queue, size_type num_rows, size_type num_cols,
                    size_type stride, size_type slice_size,
                    const size_type *slice_lengths, const size_type *slice_sets,
                    const IndexType *col_idxs, const ValueType *values,
                    ValueType *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 fill_in_dense<threads_per_row>(
@@ -284,13 +283,12 @@ void count_nnz_per_row(size_type num_rows, size_type slice_size,
 }
 
 template <typename ValueType, typename IndexType>
-void count_nnz_per_row(dim3 grid, dim3 block,
-                       gko::size_type dynamic_shared_memory,
-                       sycl::queue *stream, size_type num_rows,
+void count_nnz_per_row(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                       sycl::queue *queue, size_type num_rows,
                        size_type slice_size, const size_type *slice_sets,
                        const ValueType *values, IndexType *result)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
                              count_nnz_per_row(num_rows, slice_size, slice_sets,
@@ -330,14 +328,14 @@ void fill_in_csr(size_type num_rows, size_type slice_size,
 }
 
 template <typename ValueType, typename IndexType>
-void fill_in_csr(dim3 grid, dim3 block, gko::size_type dynamic_shared_memory,
-                 sycl::queue *stream, size_type num_rows, size_type slice_size,
+void fill_in_csr(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                 sycl::queue *queue, size_type num_rows, size_type slice_size,
                  const size_type *source_slice_sets,
                  const IndexType *source_col_idxs,
                  const ValueType *source_values, IndexType *result_row_ptrs,
                  IndexType *result_col_idxs, ValueType *result_values)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 fill_in_csr(num_rows, slice_size, source_slice_sets,
@@ -382,14 +380,13 @@ void extract_diagonal(size_type diag_size, size_type slice_size,
 }
 
 template <typename ValueType, typename IndexType>
-void extract_diagonal(dim3 grid, dim3 block,
-                      gko::size_type dynamic_shared_memory, sycl::queue *stream,
-                      size_type diag_size, size_type slice_size,
-                      const size_type *orig_slice_sets,
+void extract_diagonal(dim3 grid, dim3 block, size_type dynamic_shared_memory,
+                      sycl::queue *queue, size_type diag_size,
+                      size_type slice_size, const size_type *orig_slice_sets,
                       const ValueType *orig_values,
                       const IndexType *orig_col_idxs, ValueType *diag)
 {
-    stream->submit([&](sycl::handler &cgh) {
+    queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 extract_diagonal(diag_size, slice_size, orig_slice_sets,
diff --git a/dpcpp/test/matrix/csr_kernels.cpp b/dpcpp/test/matrix/csr_kernels.cpp
index 89dfd31d628..e76669e7136 100644
--- a/dpcpp/test/matrix/csr_kernels.cpp
+++ b/dpcpp/test/matrix/csr_kernels.cpp
@@ -222,7 +222,7 @@ TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithLoadBalance)
 }
 
 
-TEST_F(Csr, SimpleApplyIsEquivalentToRefWithCusparse)
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithOnemkl)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
 
@@ -233,7 +233,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithCusparse)
 }
 
 
-TEST_F(Csr, SimpleApplyIsEquivalentToRefWithCusparseUnsorted)
+TEST_F(Csr, SimpleApplyIsEquivalentToRefWithOnemklUnsorted)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
     unsort_mtx();
@@ -245,7 +245,7 @@ TEST_F(Csr, SimpleApplyIsEquivalentToRefWithCusparseUnsorted)
 }
 
 
-TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithCusparse)
+TEST_F(Csr, AdvancedApplyIsEquivalentToRefWithOnemkl)
 {
     set_up_apply_data(std::make_shared<Mtx::sparselib>());
 
diff --git a/dpcpp/test/matrix/ell_kernels.cpp b/dpcpp/test/matrix/ell_kernels.cpp
index bf38fb885be..3b735aa686e 100644
--- a/dpcpp/test/matrix/ell_kernels.cpp
+++ b/dpcpp/test/matrix/ell_kernels.cpp
@@ -492,7 +492,7 @@ TEST_F(Ell, SimpleApplyOnSmallMatrixIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 5 * r<vtype>::value);
 }
 
 
diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index f7627b54053..01a37608db1 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -549,9 +549,12 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
          * Creates an automatical strategy with Dpcpp executor.
          *
          * @param exec the Dpcpp executor
+         *
+         * @note TODO: porting - we hardcode the subgroup size is 16 and the
+         *             number of threads in a SIMD unit is 7
          */
         automatical(std::shared_ptr<const DpcppExecutor> exec)
-            : automatical(exec->get_num_computing_units() * 7, 32, false)
+            : automatical(exec->get_num_computing_units() * 7, 16, false)
         {}
 
         /**
@@ -582,16 +585,16 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
             // <row_len_limit>, use load_balance otherwise use classical
             index_type nnz_limit = nvidia_nnz_limit;
             index_type row_len_limit = nvidia_row_len_limit;
+            if (!cuda_strategy_) {
+                nnz_limit = intel_nnz_limit;
+                row_len_limit = intel_row_len_limit;
+            }
 #if GINKGO_HIP_PLATFORM_HCC
             if (!cuda_strategy_) {
                 nnz_limit = amd_nnz_limit;
                 row_len_limit = amd_row_len_limit;
             }
 #endif  // GINKGO_HIP_PLATFORM_HCC
-            if (!cuda_strategy_) {
-                nnz_limit = intel_nnz_limit;
-                row_len_limit = intel_row_len_limit;
-            }
             auto host_mtx_exec = mtx_row_ptrs.get_executor()->get_master();
             const bool is_mtx_on_host{host_mtx_exec ==
                                       mtx_row_ptrs.get_executor()};
@@ -1082,7 +1085,6 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
     Array<index_type> row_ptrs_;
     Array<index_type> srow_;
     std::shared_ptr<strategy_type> strategy_;
-    // oneapi::mkl::sparse::matrix_handle_t mat_handle_;
 };
 
 

From 5d92179feee7e87d0ea96074a1ee04f8e5021617 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 30 Jul 2021 18:14:57 +0200
Subject: [PATCH 0982/1008] review update - move some kernels to common - use
 GKO_ENABLE_DEFAULT_HOST when possible - use expilict type when intial declare
 - delete unused function
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 .../matrix/coo_kernels.cpp                    |  69 ++--
 common/matrix/coo_kernels.hpp.inc             |  16 -
 common/matrix/csr_kernels.hpp.inc             |  15 +-
 common/matrix/diagonal_kernels.cpp            | 153 ++++++++
 common/matrix/diagonal_kernels.hpp.inc        |  86 -----
 core/preconditioner/isai_kernels.hpp          |   2 +-
 core/test/matrix/fbcsr_sample.hpp             |   2 +-
 cuda/CMakeLists.txt                           |   2 +
 cuda/factorization/par_ic_kernels.cu          |   2 +-
 cuda/factorization/par_ict_kernels.cu         |   2 +-
 cuda/factorization/par_ilut_filter_kernel.cu  |   2 +-
 cuda/factorization/par_ilut_select_common.cuh |   4 +-
 cuda/factorization/par_ilut_spgeam_kernel.cu  |   2 +-
 cuda/factorization/par_ilut_sweep_kernel.cu   |   2 +-
 cuda/matrix/coo_kernels.cu                    |  23 --
 cuda/matrix/csr_kernels.cu                    |   1 -
 cuda/matrix/dense_kernels.cu                  |  14 +-
 cuda/matrix/diagonal_kernels.cu               | 114 +-----
 cuda/matrix/sellp_kernels.cu                  |   2 +-
 cuda/test/components/sorting_kernels.cu       |   4 +-
 cuda/test/matrix/sellp_kernels.cpp            |  70 ++--
 dpcpp/CMakeLists.txt                          |   2 +
 dpcpp/components/format_conversion.dp.hpp     |   3 +-
 dpcpp/components/merging.dp.hpp               | 338 -----------------
 dpcpp/components/segment_scan.dp.hpp          |  14 +-
 dpcpp/matrix/coo_kernels.dp.cpp               | 164 +-------
 dpcpp/matrix/csr_kernels.dp.cpp               | 359 ++----------------
 dpcpp/matrix/dense_kernels.dp.cpp             |   8 +-
 dpcpp/matrix/diagonal_kernels.dp.cpp          | 297 +--------------
 dpcpp/matrix/ell_kernels.dp.cpp               |  74 +---
 dpcpp/matrix/hybrid_kernels.dp.cpp            |  32 +-
 dpcpp/matrix/sellp_kernels.dp.cpp             | 105 +----
 dpcpp/test/matrix/CMakeLists.txt              |   1 +
 dpcpp/test/matrix/sellp_kernels.cpp           |  76 ++--
 .../custom-matrix-format/stencil_kernel.cu    |   2 +-
 hip/CMakeLists.txt                            |   2 +
 hip/factorization/par_ic_kernels.hip.cpp      |   2 +-
 hip/factorization/par_ict_kernels.hip.cpp     |   2 +-
 .../par_ilut_filter_kernel.hip.cpp            |   2 +-
 .../par_ilut_select_common.hip.hpp            |   4 +-
 .../par_ilut_spgeam_kernel.hip.cpp            |   2 +-
 .../par_ilut_sweep_kernel.hip.cpp             |   2 +-
 hip/matrix/coo_kernels.hip.cpp                |  24 --
 hip/matrix/csr_kernels.hip.cpp                |   1 -
 hip/matrix/dense_kernels.hip.cpp              |  14 +-
 hip/matrix/diagonal_kernels.hip.cpp           | 117 +-----
 hip/matrix/sellp_kernels.hip.cpp              |   2 +-
 hip/test/components/sorting_kernels.hip.cpp   |   4 +-
 hip/test/matrix/sellp_kernels.hip.cpp         |  70 ++--
 omp/CMakeLists.txt                            |   2 +
 omp/matrix/coo_kernels.cpp                    |  24 --
 omp/matrix/diagonal_kernels.cpp               | 105 -----
 omp/test/matrix/sellp_kernels.cpp             | 134 ++++---
 53 files changed, 470 insertions(+), 2105 deletions(-)
 rename dpcpp/components/intrinsics.dp.hpp => common/matrix/coo_kernels.cpp (58%)
 create mode 100644 common/matrix/diagonal_kernels.cpp
 delete mode 100644 dpcpp/components/merging.dp.hpp

diff --git a/dpcpp/components/intrinsics.dp.hpp b/common/matrix/coo_kernels.cpp
similarity index 58%
rename from dpcpp/components/intrinsics.dp.hpp
rename to common/matrix/coo_kernels.cpp
index 68f936e09cb..63fbf2c9ec1 100644
--- a/dpcpp/components/intrinsics.dp.hpp
+++ b/common/matrix/coo_kernels.cpp
@@ -30,63 +30,48 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_DPCPP_COMPONENTS_INTRINSICS_DP_HPP_
-#define GKO_DPCPP_COMPONENTS_INTRINSICS_DP_HPP_
+#include "core/matrix/coo_kernels.hpp"
 
 
-#include <CL/sycl.hpp>
+#include <ginkgo/core/base/math.hpp>
 
 
-#include <ginkgo/core/base/types.hpp>
-
-
-#include "dpcpp/base/dpct.hpp"
+#include "common/base/kernel_launch.hpp"
 
 
 namespace gko {
 namespace kernels {
-namespace dpcpp {
-
-
+namespace GKO_DEVICE_NAMESPACE {
 /**
- * @internal
- * Returns the number of set bits in the given mask.
+ * @brief The Coo matrix format namespace.
+ *
+ * @ingroup coo
  */
-__dpct_inline__ int popcnt(uint32 mask) { return sycl::popcount(mask); }
+namespace coo {
 
-/** @copydoc popcnt */
-__dpct_inline__ int popcnt(uint64 mask) { return sycl::popcount(mask); }
-
-
-/**
- * @internal
- * Returns the (1-based!) index of the first set bit in the given mask,
- * starting from the least significant bit.
- */
-__dpct_inline__ int ffs(uint32 mask) { return __builtin_ffs(mask); }
 
-/** @copydoc ffs */
-__dpct_inline__ int ffs(uint64 mask)
+template <typename ValueType, typename IndexType>
+void extract_diagonal(std::shared_ptr<const DefaultExecutor> exec,
+                      const matrix::Coo<ValueType, IndexType> *orig,
+                      matrix::Diagonal<ValueType> *diag)
 {
-    // the cast is necessary, as the overloads defined by HIP are ambiguous
-    return __builtin_ffsll(static_cast<unsigned long long int>(mask));
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto tidx, auto orig_values, auto orig_row_idxs,
+                      auto orig_col_idxs, auto diag) {
+            if (orig_row_idxs[tidx] == orig_col_idxs[tidx]) {
+                diag[orig_row_idxs[tidx]] = orig_values[tidx];
+            }
+        },
+        orig->get_num_stored_elements(), orig->get_const_values(),
+        orig->get_const_row_idxs(), orig->get_const_col_idxs(),
+        diag->get_values());
 }
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_COO_EXTRACT_DIAGONAL_KERNEL);
 
 
-/**
- * @internal
- * Returns the number of zero bits before the first set bit in the given mask,
- * starting from the most significant bit.
- */
-__dpct_inline__ int clz(uint32 mask) { return __builtin_clz(mask); }
-
-/** @copydoc clz */
-__dpct_inline__ int clz(uint64 mask) { return __builtin_clzll(mask); }
-
-
-}  // namespace dpcpp
+}  // namespace coo
+}  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
 }  // namespace gko
-
-
-#endif  // GKO_DPCPP_COMPONENTS_INTRINSICS_DP_HPP_
diff --git a/common/matrix/coo_kernels.hpp.inc b/common/matrix/coo_kernels.hpp.inc
index 489a65729f3..22d8191ab8a 100644
--- a/common/matrix/coo_kernels.hpp.inc
+++ b/common/matrix/coo_kernels.hpp.inc
@@ -272,20 +272,4 @@ __global__ __launch_bounds__(default_block_size) void fill_in_dense(
 }
 
 
-template <typename ValueType, typename IndexType>
-__global__ __launch_bounds__(default_block_size) void extract_diagonal(
-    size_type nnz, const ValueType *__restrict__ orig_values,
-    const IndexType *__restrict__ orig_row_idxs,
-    const IndexType *__restrict__ orig_col_idxs, ValueType *__restrict__ diag)
-{
-    const auto tidx = thread::get_thread_id_flat();
-
-    if (tidx < nnz) {
-        if (orig_row_idxs[tidx] == orig_col_idxs[tidx]) {
-            diag[orig_row_idxs[tidx]] = orig_values[tidx];
-        }
-    }
-}
-
-
 }  // namespace kernel
diff --git a/common/matrix/csr_kernels.hpp.inc b/common/matrix/csr_kernels.hpp.inc
index fb88d4f9ab2..c2761fa1ab3 100644
--- a/common/matrix/csr_kernels.hpp.inc
+++ b/common/matrix/csr_kernels.hpp.inc
@@ -76,8 +76,9 @@ __device__ __forceinline__ void find_next_row(
         if (ind >= *row_end) {
             *row = row_predict;
             *row_end = row_predict_end;
-            for (; ind >= *row_end; *row_end = row_ptr[++*row + 1])
-                ;
+            while (ind >= *row_end) {
+                *row_end = row_ptr[++*row + 1];
+            }
         }
 
     } else {
@@ -140,8 +141,8 @@ template <typename IndexType>
 __device__ __forceinline__ IndexType get_warp_start_idx(
     const IndexType nwarps, const IndexType nnz, const IndexType warp_idx)
 {
-    const long long cache_lines = ceildivT<IndexType>(nnz, wsize);
-    return (warp_idx * cache_lines / nwarps) * wsize;
+    const long long cache_lines = ceildivT<IndexType>(nnz, config::warp_size);
+    return (warp_idx * cache_lines / nwarps) * config::warp_size;
 }
 
 
@@ -160,6 +161,7 @@ __device__ __forceinline__ void spmv_kernel(
     }
     const IndexType data_size = row_ptrs[num_rows];
     const IndexType start = get_warp_start_idx(nwarps, data_size, warp_idx);
+    constexpr IndexType wsize = config::warp_size;
     const IndexType end =
         min(get_warp_start_idx(nwarps, data_size, warp_idx + 1),
             ceildivT<IndexType>(data_size, wsize) * wsize);
@@ -359,8 +361,7 @@ __device__ void merge_path_spmv(
     tmp_val[threadIdx.x] = value;
     tmp_ind[threadIdx.x] = row_i;
     group::this_thread_block().sync();
-    bool last = block_segment_scan_reverse(static_cast<IndexType *>(tmp_ind),
-                                           static_cast<ValueType *>(tmp_val));
+    bool last = block_segment_scan_reverse(tmp_ind, tmp_val);
     if (threadIdx.x == spmv_block_size - 1) {
         row_out[blockIdx.x] = min(end_x, num_rows - 1);
         val_out[blockIdx.x] = tmp_val[threadIdx.x];
@@ -1088,4 +1089,4 @@ __global__ __launch_bounds__(default_block_size) void inv_symm_permute_kernel(
         out_cols[out_begin + i] = permutation[in_cols[in_begin + i]];
         out_vals[out_begin + i] = in_vals[in_begin + i];
     }
-}
\ No newline at end of file
+}
diff --git a/common/matrix/diagonal_kernels.cpp b/common/matrix/diagonal_kernels.cpp
new file mode 100644
index 00000000000..31b6b2c11c2
--- /dev/null
+++ b/common/matrix/diagonal_kernels.cpp
@@ -0,0 +1,153 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/diagonal_kernels.hpp"
+
+
+#include <ginkgo/core/base/math.hpp>
+
+
+#include "common/base/kernel_launch.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace GKO_DEVICE_NAMESPACE {
+/**
+ * @brief The Diagonal matrix format namespace.
+ *
+ * @ingroup diagonal
+ */
+namespace diagonal {
+
+
+template <typename ValueType>
+void apply_to_dense(std::shared_ptr<const DefaultExecutor> exec,
+                    const matrix::Diagonal<ValueType> *a,
+                    const matrix::Dense<ValueType> *b,
+                    matrix::Dense<ValueType> *c)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto diag, auto source, auto result) {
+            result(row, col) = source(row, col) * diag[row];
+        },
+        b->get_size(), a->get_const_values(), b, c);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_APPLY_TO_DENSE_KERNEL);
+
+
+template <typename ValueType>
+void right_apply_to_dense(std::shared_ptr<const DefaultExecutor> exec,
+                          const matrix::Diagonal<ValueType> *a,
+                          const matrix::Dense<ValueType> *b,
+                          matrix::Dense<ValueType> *c)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto diag, auto source, auto result) {
+            result(row, col) = source(row, col) * diag[col];
+        },
+        b->get_size(), a->get_const_values(), b, c);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_DENSE_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void right_apply_to_csr(std::shared_ptr<const DefaultExecutor> exec,
+                        const matrix::Diagonal<ValueType> *a,
+                        const matrix::Csr<ValueType, IndexType> *b,
+                        matrix::Csr<ValueType, IndexType> *c)
+{
+    // TODO: combine copy and diag apply together
+    c->copy_from(b);
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto tidx, auto diag, auto result_values, auto col_idxs) {
+            result_values[tidx] *= diag[col_idxs[tidx]];
+        },
+        c->get_num_stored_elements(), a->get_const_values(), c->get_values(),
+        c->get_const_col_idxs());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_CSR_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void convert_to_csr(std::shared_ptr<const DefaultExecutor> exec,
+                    const matrix::Diagonal<ValueType> *source,
+                    matrix::Csr<ValueType, IndexType> *result)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto tidx, auto size, auto diag_values, auto row_ptrs,
+                      auto col_idxs, auto csr_values) {
+            row_ptrs[tidx] = tidx;
+            col_idxs[tidx] = tidx;
+            csr_values[tidx] = diag_values[tidx];
+            if (tidx == size - 1) {
+                row_ptrs[size] = size;
+            }
+        },
+        source->get_size()[0], source->get_size()[0],
+        source->get_const_values(), result->get_row_ptrs(),
+        result->get_col_idxs(), result->get_values());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DIAGONAL_CONVERT_TO_CSR_KERNEL);
+
+
+template <typename ValueType>
+void conj_transpose(std::shared_ptr<const DefaultExecutor> exec,
+                    const matrix::Diagonal<ValueType> *orig,
+                    matrix::Diagonal<ValueType> *trans)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto tidx, auto orig_values, auto trans_values) {
+            trans_values[tidx] = conj(orig_values[tidx]);
+        },
+        orig->get_size()[0], orig->get_const_values(), trans->get_values());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_CONJ_TRANSPOSE_KERNEL);
+
+
+}  // namespace diagonal
+}  // namespace GKO_DEVICE_NAMESPACE
+}  // namespace kernels
+}  // namespace gko
diff --git a/common/matrix/diagonal_kernels.hpp.inc b/common/matrix/diagonal_kernels.hpp.inc
index fc1d288df8e..9a3736766e6 100644
--- a/common/matrix/diagonal_kernels.hpp.inc
+++ b/common/matrix/diagonal_kernels.hpp.inc
@@ -33,40 +33,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace kernel {
 
 
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void apply_to_dense(
-    size_type num_rows, size_type num_cols, const ValueType *__restrict__ diag,
-    size_type source_stride, const ValueType *__restrict__ source_values,
-    size_type result_stride, ValueType *__restrict__ result_values)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto row = tidx / num_cols;
-    const auto col = tidx % num_cols;
-
-    if (row < num_rows) {
-        result_values[row * result_stride + col] =
-            source_values[row * source_stride + col] * diag[row];
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void right_apply_to_dense(
-    size_type num_rows, size_type num_cols, const ValueType *__restrict__ diag,
-    size_type source_stride, const ValueType *__restrict__ source_values,
-    size_type result_stride, ValueType *__restrict__ result_values)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto row = tidx / num_cols;
-    const auto col = tidx % num_cols;
-
-    if (row < num_rows) {
-        result_values[row * result_stride + col] =
-            source_values[row * source_stride + col] * diag[col];
-    }
-}
-
-
 template <typename ValueType, typename IndexType>
 __global__ __launch_bounds__(default_block_size) void apply_to_csr(
     size_type num_rows, const ValueType *__restrict__ diag,
@@ -92,56 +58,4 @@ __global__ __launch_bounds__(default_block_size) void apply_to_csr(
 }
 
 
-template <typename ValueType, typename IndexType>
-__global__ __launch_bounds__(default_block_size) void right_apply_to_csr(
-    size_type num_nnz, const ValueType *__restrict__ diag,
-    const IndexType *__restrict__ col_idxs,
-    ValueType *__restrict__ result_values)
-{
-    const auto tidx = thread::get_thread_id_flat();
-
-    if (tidx >= num_nnz) {
-        return;
-    }
-
-    result_values[tidx] *= diag[col_idxs[tidx]];
-}
-
-
-template <typename ValueType, typename IndexType>
-__global__ __launch_bounds__(default_block_size) void convert_to_csr(
-    size_type size, const ValueType *__restrict__ diag_values,
-    IndexType *__restrict__ row_ptrs, IndexType *__restrict__ col_idxs,
-    ValueType *__restrict__ csr_values)
-{
-    const auto tidx = thread::get_thread_id_flat();
-
-    if (tidx >= size) {
-        return;
-    }
-    if (tidx == 0) {
-        row_ptrs[size] = size;
-    }
-
-    row_ptrs[tidx] = tidx;
-    col_idxs[tidx] = tidx;
-    csr_values[tidx] = diag_values[tidx];
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void conj_transpose(
-    size_type size, const ValueType *__restrict__ orig_values,
-    ValueType *__restrict__ trans_values)
-{
-    const auto tidx = thread::get_thread_id_flat();
-
-    if (tidx >= size) {
-        return;
-    }
-
-    trans_values[tidx] = conj(orig_values[tidx]);
-}
-
-
 }  // namespace kernel
diff --git a/core/preconditioner/isai_kernels.hpp b/core/preconditioner/isai_kernels.hpp
index f85a74910da..bdf6360f1f4 100644
--- a/core/preconditioner/isai_kernels.hpp
+++ b/core/preconditioner/isai_kernels.hpp
@@ -83,7 +83,7 @@ namespace kernels {
         size_type e_end)
 
 #define GKO_DECLARE_ALL_AS_TEMPLATES                                        \
-    constexpr auto row_size_limit = 32;                                     \
+    constexpr int row_size_limit = 32;                                      \
     template <typename ValueType, typename IndexType>                       \
     GKO_DECLARE_ISAI_GENERATE_TRI_INVERSE_KERNEL(ValueType, IndexType);     \
     template <typename ValueType, typename IndexType>                       \
diff --git a/core/test/matrix/fbcsr_sample.hpp b/core/test/matrix/fbcsr_sample.hpp
index ac6007e789c..9e6ff969045 100644
--- a/core/test/matrix/fbcsr_sample.hpp
+++ b/core/test/matrix/fbcsr_sample.hpp
@@ -51,7 +51,7 @@ namespace gko {
 namespace testing {
 
 
-constexpr auto fbcsr_test_offset = 0.000011118888;
+constexpr double fbcsr_test_offset = 0.000011118888;
 
 
 /** Generates the same sample block CSR matrix in different formats
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 2e25435e5c5..d40c7177c5b 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -67,7 +67,9 @@ find_library(CURAND curand
 add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_cuda_device> "")
 set(GKO_CUDA_COMMON_SOURCES
     ../common/components/precision_conversion.cpp
+    ../common/matrix/coo_kernels.cpp
     ../common/matrix/dense_kernels.cpp
+    ../common/matrix/diagonal_kernels.cpp
     ../common/solver/bicg_kernels.cpp
     ../common/solver/bicgstab_kernels.cpp
     ../common/solver/cg_kernels.cpp
diff --git a/cuda/factorization/par_ic_kernels.cu b/cuda/factorization/par_ic_kernels.cu
index 35f49c40436..51e624fc2dc 100644
--- a/cuda/factorization/par_ic_kernels.cu
+++ b/cuda/factorization/par_ic_kernels.cu
@@ -54,7 +54,7 @@ namespace cuda {
 namespace par_ic_factorization {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 // subwarp sizes for all warp-parallel kernels (sweep)
diff --git a/cuda/factorization/par_ict_kernels.cu b/cuda/factorization/par_ict_kernels.cu
index 512b34dfee7..f7354147cb7 100644
--- a/cuda/factorization/par_ict_kernels.cu
+++ b/cuda/factorization/par_ict_kernels.cu
@@ -65,7 +65,7 @@ namespace cuda {
 namespace par_ict_factorization {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 // subwarp sizes for all warp-parallel kernels (filter, add_candidates)
diff --git a/cuda/factorization/par_ilut_filter_kernel.cu b/cuda/factorization/par_ilut_filter_kernel.cu
index 661eaf7680d..a3c9d33fece 100644
--- a/cuda/factorization/par_ilut_filter_kernel.cu
+++ b/cuda/factorization/par_ilut_filter_kernel.cu
@@ -64,7 +64,7 @@ namespace cuda {
 namespace par_ilut_factorization {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 // subwarp sizes for filter kernels
diff --git a/cuda/factorization/par_ilut_select_common.cuh b/cuda/factorization/par_ilut_select_common.cuh
index f664d41193f..fe2ddef7401 100644
--- a/cuda/factorization/par_ilut_select_common.cuh
+++ b/cuda/factorization/par_ilut_select_common.cuh
@@ -45,8 +45,8 @@ namespace cuda {
 namespace par_ilut_factorization {
 
 
-constexpr auto default_block_size = 512;
-constexpr auto items_per_thread = 16;
+constexpr int default_block_size = 512;
+constexpr int items_per_thread = 16;
 
 
 template <typename ValueType, typename IndexType>
diff --git a/cuda/factorization/par_ilut_spgeam_kernel.cu b/cuda/factorization/par_ilut_spgeam_kernel.cu
index 29071e3c798..7a3c9e5d575 100644
--- a/cuda/factorization/par_ilut_spgeam_kernel.cu
+++ b/cuda/factorization/par_ilut_spgeam_kernel.cu
@@ -65,7 +65,7 @@ namespace cuda {
 namespace par_ilut_factorization {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 // subwarp sizes for add_candidates kernels
diff --git a/cuda/factorization/par_ilut_sweep_kernel.cu b/cuda/factorization/par_ilut_sweep_kernel.cu
index 118903a2a3c..d50b1a8caad 100644
--- a/cuda/factorization/par_ilut_sweep_kernel.cu
+++ b/cuda/factorization/par_ilut_sweep_kernel.cu
@@ -65,7 +65,7 @@ namespace cuda {
 namespace par_ilut_factorization {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 // subwarp sizes for all warp-parallel kernels (filter, add_candidates)
diff --git a/cuda/matrix/coo_kernels.cu b/cuda/matrix/coo_kernels.cu
index 7ad48c76b3f..db637ea0e24 100644
--- a/cuda/matrix/coo_kernels.cu
+++ b/cuda/matrix/coo_kernels.cu
@@ -243,29 +243,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_CONVERT_TO_DENSE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void extract_diagonal(std::shared_ptr<const CudaExecutor> exec,
-                      const matrix::Coo<ValueType, IndexType> *orig,
-                      matrix::Diagonal<ValueType> *diag)
-{
-    const auto nnz = orig->get_num_stored_elements();
-    const auto diag_size = diag->get_size()[0];
-    const auto num_blocks = ceildiv(nnz, default_block_size);
-
-    const auto orig_values = orig->get_const_values();
-    const auto orig_row_idxs = orig->get_const_row_idxs();
-    const auto orig_col_idxs = orig->get_const_col_idxs();
-    auto diag_values = diag->get_values();
-
-    kernel::extract_diagonal<<<num_blocks, default_block_size>>>(
-        nnz, as_cuda_type(orig_values), as_cuda_type(orig_row_idxs),
-        as_cuda_type(orig_col_idxs), as_cuda_type(diag_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_COO_EXTRACT_DIAGONAL_KERNEL);
-
-
 }  // namespace coo
 }  // namespace cuda
 }  // namespace kernels
diff --git a/cuda/matrix/csr_kernels.cu b/cuda/matrix/csr_kernels.cu
index da53054727c..7c32d3043ec 100644
--- a/cuda/matrix/csr_kernels.cu
+++ b/cuda/matrix/csr_kernels.cu
@@ -80,7 +80,6 @@ namespace csr {
 constexpr int default_block_size = 512;
 constexpr int warps_in_block = 4;
 constexpr int spmv_block_size = warps_in_block * config::warp_size;
-constexpr int wsize = config::warp_size;
 constexpr int classical_overweight = 32;
 
 
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index 1f6e5a2ab16..607bfedc4da 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -64,7 +64,7 @@ namespace cuda {
 namespace dense {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 #include "common/matrix/dense_kernels.hpp.inc"
@@ -135,8 +135,8 @@ void compute_dot(std::shared_ptr<const CudaExecutor> exec,
         // TODO: these are tuning parameters obtained experimentally, once
         // we decide how to handle this uniformly, they should be modified
         // appropriately
-        constexpr auto work_per_thread = 32;
-        constexpr auto block_size = 1024;
+        constexpr int work_per_thread = 32;
+        constexpr int block_size = 1024;
 
         constexpr auto work_per_block = work_per_thread * block_size;
         const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
@@ -178,8 +178,8 @@ void compute_conj_dot(std::shared_ptr<const CudaExecutor> exec,
         // TODO: these are tuning parameters obtained experimentally, once
         // we decide how to handle this uniformly, they should be modified
         // appropriately
-        constexpr auto work_per_thread = 32;
-        constexpr auto block_size = 1024;
+        constexpr int work_per_thread = 32;
+        constexpr int block_size = 1024;
 
         constexpr auto work_per_block = work_per_thread * block_size;
         const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
@@ -220,8 +220,8 @@ void compute_norm2(std::shared_ptr<const CudaExecutor> exec,
         // TODO: these are tuning parameters obtained experimentally, once
         // we decide how to handle this uniformly, they should be modified
         // appropriately
-        constexpr auto work_per_thread = 32;
-        constexpr auto block_size = 1024;
+        constexpr int work_per_thread = 32;
+        constexpr int block_size = 1024;
 
         constexpr auto work_per_block = work_per_thread * block_size;
         const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
diff --git a/cuda/matrix/diagonal_kernels.cu b/cuda/matrix/diagonal_kernels.cu
index 9b0e174b982..ea5835a6a92 100644
--- a/cuda/matrix/diagonal_kernels.cu
+++ b/cuda/matrix/diagonal_kernels.cu
@@ -54,63 +54,12 @@ namespace cuda {
 namespace diagonal {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 #include "common/matrix/diagonal_kernels.hpp.inc"
 
 
-template <typename ValueType>
-void apply_to_dense(std::shared_ptr<const CudaExecutor> exec,
-                    const matrix::Diagonal<ValueType> *a,
-                    const matrix::Dense<ValueType> *b,
-                    matrix::Dense<ValueType> *c)
-{
-    const auto b_size = b->get_size();
-    const auto num_rows = b_size[0];
-    const auto num_cols = b_size[1];
-    const auto b_stride = b->get_stride();
-    const auto c_stride = c->get_stride();
-    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
-
-    const auto diag_values = a->get_const_values();
-    const auto b_values = b->get_const_values();
-    auto c_values = c->get_values();
-
-    kernel::apply_to_dense<<<grid_dim, default_block_size>>>(
-        num_rows, num_cols, as_cuda_type(diag_values), b_stride,
-        as_cuda_type(b_values), c_stride, as_cuda_type(c_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_APPLY_TO_DENSE_KERNEL);
-
-
-template <typename ValueType>
-void right_apply_to_dense(std::shared_ptr<const CudaExecutor> exec,
-                          const matrix::Diagonal<ValueType> *a,
-                          const matrix::Dense<ValueType> *b,
-                          matrix::Dense<ValueType> *c)
-{
-    const auto b_size = b->get_size();
-    const auto num_rows = b_size[0];
-    const auto num_cols = b_size[1];
-    const auto b_stride = b->get_stride();
-    const auto c_stride = c->get_stride();
-    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
-
-    const auto diag_values = a->get_const_values();
-    const auto b_values = b->get_const_values();
-    auto c_values = c->get_values();
-
-    kernel::right_apply_to_dense<<<grid_dim, default_block_size>>>(
-        num_rows, num_cols, as_cuda_type(diag_values), b_stride,
-        as_cuda_type(b_values), c_stride, as_cuda_type(c_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_DENSE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void apply_to_csr(std::shared_ptr<const CudaExecutor> exec,
                   const matrix::Diagonal<ValueType> *a,
@@ -134,67 +83,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DIAGONAL_APPLY_TO_CSR_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void right_apply_to_csr(std::shared_ptr<const CudaExecutor> exec,
-                        const matrix::Diagonal<ValueType> *a,
-                        const matrix::Csr<ValueType, IndexType> *b,
-                        matrix::Csr<ValueType, IndexType> *c)
-{
-    const auto num_nnz = b->get_num_stored_elements();
-    const auto diag_values = a->get_const_values();
-    c->copy_from(b);
-    auto csr_values = c->get_values();
-    const auto csr_col_idxs = c->get_const_col_idxs();
-
-    const auto grid_dim = ceildiv(num_nnz, default_block_size);
-    kernel::right_apply_to_csr<<<grid_dim, default_block_size>>>(
-        num_nnz, as_cuda_type(diag_values), as_cuda_type(csr_col_idxs),
-        as_cuda_type(csr_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_CSR_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void convert_to_csr(std::shared_ptr<const CudaExecutor> exec,
-                    const matrix::Diagonal<ValueType> *source,
-                    matrix::Csr<ValueType, IndexType> *result)
-{
-    const auto size = source->get_size()[0];
-    const auto grid_dim = ceildiv(size, default_block_size);
-
-    const auto diag_values = source->get_const_values();
-    auto row_ptrs = result->get_row_ptrs();
-    auto col_idxs = result->get_col_idxs();
-    auto csr_values = result->get_values();
-
-    kernel::convert_to_csr<<<grid_dim, default_block_size>>>(
-        size, as_cuda_type(diag_values), as_cuda_type(row_ptrs),
-        as_cuda_type(col_idxs), as_cuda_type(csr_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DIAGONAL_CONVERT_TO_CSR_KERNEL);
-
-
-template <typename ValueType>
-void conj_transpose(std::shared_ptr<const CudaExecutor> exec,
-                    const matrix::Diagonal<ValueType> *orig,
-                    matrix::Diagonal<ValueType> *trans)
-{
-    const auto size = orig->get_size()[0];
-    const auto grid_dim = ceildiv(size, default_block_size);
-    const auto orig_values = orig->get_const_values();
-    auto trans_values = trans->get_values();
-
-    kernel::conj_transpose<<<grid_dim, default_block_size>>>(
-        size, as_cuda_type(orig_values), as_cuda_type(trans_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_CONJ_TRANSPOSE_KERNEL);
-
-
 }  // namespace diagonal
 }  // namespace cuda
 }  // namespace kernels
diff --git a/cuda/matrix/sellp_kernels.cu b/cuda/matrix/sellp_kernels.cu
index 997b9a5ed9a..6ec2902a618 100644
--- a/cuda/matrix/sellp_kernels.cu
+++ b/cuda/matrix/sellp_kernels.cu
@@ -59,7 +59,7 @@ namespace cuda {
 namespace sellp {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 #include "common/matrix/sellp_kernels.hpp.inc"
diff --git a/cuda/test/components/sorting_kernels.cu b/cuda/test/components/sorting_kernels.cu
index e2b7abc51d7..6bcf98fdeca 100644
--- a/cuda/test/components/sorting_kernels.cu
+++ b/cuda/test/components/sorting_kernels.cu
@@ -54,8 +54,8 @@ using gko::kernels::cuda::bitonic_sort;
 using gko::kernels::cuda::config;
 
 
-constexpr auto num_elements = 2048;
-constexpr auto num_local = 4;
+constexpr int num_elements = 2048;
+constexpr int num_local = 4;
 constexpr auto num_threads = num_elements / num_local;
 
 
diff --git a/cuda/test/matrix/sellp_kernels.cpp b/cuda/test/matrix/sellp_kernels.cpp
index 2935377014a..071270fde71 100644
--- a/cuda/test/matrix/sellp_kernels.cpp
+++ b/cuda/test/matrix/sellp_kernels.cpp
@@ -84,39 +84,15 @@ class Sellp : public ::testing::Test {
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
     }
 
-    void set_up_apply_vector(
-        int slice_size = gko::matrix::default_slice_size,
-        int stride_factor = gko::matrix::default_stride_factor,
-        int total_cols = 0)
-    {
-        mtx = Mtx::create(ref);
-        mtx->copy_from(gen_mtx(532, 231));
-        expected = gen_mtx(532, 1);
-        y = gen_mtx(231, 1);
-        alpha = gko::initialize<Vec>({2.0}, ref);
-        beta = gko::initialize<Vec>({-1.0}, ref);
-        dmtx = Mtx::create(cuda);
-        dmtx->copy_from(mtx.get());
-        dresult = Vec::create(cuda);
-        dresult->copy_from(expected.get());
-        dy = Vec::create(cuda);
-        dy->copy_from(y.get());
-        dalpha = Vec::create(cuda);
-        dalpha->copy_from(alpha.get());
-        dbeta = Vec::create(cuda);
-        dbeta->copy_from(beta.get());
-    }
-
     void set_up_apply_matrix(
-        int slice_size = gko::matrix::default_slice_size,
-        int stride_factor = gko::matrix::default_stride_factor,
-        int total_cols = 0)
+        int total_cols = 1, int slice_size = gko::matrix::default_slice_size,
+        int stride_factor = gko::matrix::default_stride_factor)
     {
         mtx = Mtx::create(ref);
         mtx->copy_from(gen_mtx(532, 231));
         empty = Mtx::create(ref);
-        expected = gen_mtx(532, 64);
-        y = gen_mtx(231, 64);
+        expected = gen_mtx(532, total_cols);
+        y = gen_mtx(231, total_cols);
         alpha = gko::initialize<Vec>({2.0}, ref);
         beta = gko::initialize<Vec>({-1.0}, ref);
         dmtx = Mtx::create(cuda);
@@ -155,7 +131,7 @@ class Sellp : public ::testing::Test {
 
 TEST_F(Sellp, SimpleApplyIsEquivalentToRef)
 {
-    set_up_apply_vector();
+    set_up_apply_matrix();
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -168,7 +144,7 @@ TEST_F(Sellp, SimpleApplyIsEquivalentToRef)
 
 TEST_F(Sellp, AdvancedApplyIsEquivalentToRef)
 {
-    set_up_apply_vector();
+    set_up_apply_matrix();
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
@@ -181,7 +157,7 @@ TEST_F(Sellp, AdvancedApplyIsEquivalentToRef)
 
 TEST_F(Sellp, SimpleApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
 {
-    set_up_apply_vector(32, 2);
+    set_up_apply_matrix(1, 32, 2);
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -194,7 +170,7 @@ TEST_F(Sellp, SimpleApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
 
 TEST_F(Sellp, AdvancedApplyWithSliceSizeAndStrideFActorIsEquivalentToRef)
 {
-    set_up_apply_vector(32, 2);
+    set_up_apply_matrix(1, 32, 2);
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
@@ -207,7 +183,8 @@ TEST_F(Sellp, AdvancedApplyWithSliceSizeAndStrideFActorIsEquivalentToRef)
 
 TEST_F(Sellp, SimpleApplyMultipleRHSIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -220,7 +197,8 @@ TEST_F(Sellp, SimpleApplyMultipleRHSIsEquivalentToRef)
 
 TEST_F(Sellp, AdvancedApplyMultipleRHSIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
@@ -261,7 +239,8 @@ TEST_F(Sellp,
 
 TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(cuda);
     dcomplex_b->copy_from(complex_b.get());
@@ -278,7 +257,8 @@ TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 
 TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(cuda);
     dcomplex_b->copy_from(complex_b.get());
@@ -295,7 +275,7 @@ TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 
 TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
 
     auto dense_mtx = gko::matrix::Dense<>::create(ref);
     auto ddense_mtx = gko::matrix::Dense<>::create(cuda);
@@ -309,7 +289,7 @@ TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 
 TEST_F(Sellp, ConvertToCsrIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
 
     auto csr_mtx = gko::matrix::Csr<>::create(ref);
     auto dcsr_mtx = gko::matrix::Csr<>::create(cuda);
@@ -323,7 +303,7 @@ TEST_F(Sellp, ConvertToCsrIsEquivalentToRef)
 
 TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
 
     auto dense_mtx = gko::matrix::Dense<>::create(ref);
     auto ddense_mtx = gko::matrix::Dense<>::create(cuda);
@@ -337,7 +317,7 @@ TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 
 TEST_F(Sellp, ConvertEmptyToCsrIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
 
     auto csr_mtx = gko::matrix::Csr<>::create(ref);
     auto dcsr_mtx = gko::matrix::Csr<>::create(cuda);
@@ -351,7 +331,7 @@ TEST_F(Sellp, ConvertEmptyToCsrIsEquivalentToRef)
 
 TEST_F(Sellp, CountNonzerosIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
 
     gko::size_type nnz;
     gko::size_type dnnz;
@@ -365,7 +345,7 @@ TEST_F(Sellp, CountNonzerosIsEquivalentToRef)
 
 TEST_F(Sellp, ExtractDiagonalIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
 
     auto diag = mtx->extract_diagonal();
     auto ddiag = dmtx->extract_diagonal();
@@ -376,7 +356,7 @@ TEST_F(Sellp, ExtractDiagonalIsEquivalentToRef)
 
 TEST_F(Sellp, ExtractDiagonalWithSliceSizeAndStrideFactorIsEquivalentToRef)
 {
-    set_up_apply_matrix(32, 2);
+    set_up_apply_matrix(64, 32, 2);
 
     auto diag = mtx->extract_diagonal();
     auto ddiag = dmtx->extract_diagonal();
@@ -387,7 +367,7 @@ TEST_F(Sellp, ExtractDiagonalWithSliceSizeAndStrideFactorIsEquivalentToRef)
 
 TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
-    set_up_apply_matrix(32, 2);
+    set_up_apply_matrix(64, 32, 2);
 
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
@@ -398,7 +378,7 @@ TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
 
 TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
-    set_up_apply_matrix(32, 2);
+    set_up_apply_matrix(64, 32, 2);
 
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 401be197820..d7b6b1d2465 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -48,7 +48,9 @@ target_sources(ginkgo_dpcpp
     stop/criterion_kernels.dp.cpp
     stop/residual_norm_kernels.dp.cpp
     ../common/components/precision_conversion.cpp
+    ../common/matrix/coo_kernels.cpp
     ../common/matrix/dense_kernels.cpp
+    ../common/matrix/diagonal_kernels.cpp
     ../common/solver/bicg_kernels.cpp
     ../common/solver/bicgstab_kernels.cpp
     ../common/solver/cg_kernels.cpp
diff --git a/dpcpp/components/format_conversion.dp.hpp b/dpcpp/components/format_conversion.dp.hpp
index 281677f4b98..9a2410c3f2e 100644
--- a/dpcpp/components/format_conversion.dp.hpp
+++ b/dpcpp/components/format_conversion.dp.hpp
@@ -38,6 +38,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/types.hpp>
 
 
 #include "dpcpp/base/dim3.dp.hpp"
@@ -119,7 +120,7 @@ size_type calculate_nwarps(std::shared_ptr<const DpcppExecutor> exec,
     }
 #endif  // GINKGO_BENCHMARK_ENABLE_TUNING
     return std::min(multiple * nwarps_in_dpcpp,
-                    size_type(ceildiv(nnz, config::warp_size)));
+                    size_type(ceildiv(nnz, subwarp_size)));
 }
 
 
diff --git a/dpcpp/components/merging.dp.hpp b/dpcpp/components/merging.dp.hpp
deleted file mode 100644
index ecef5e4e549..00000000000
--- a/dpcpp/components/merging.dp.hpp
+++ /dev/null
@@ -1,338 +0,0 @@
-/*******************************<GINKGO LICENSE>******************************
-Copyright (c) 2017-2021, the Ginkgo authors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-******************************<GINKGO LICENSE>*******************************/
-
-#ifndef GKO_DPCPP_COMPONENTS_MERGING_DP_HPP_
-#define GKO_DPCPP_COMPONENTS_MERGING_DP_HPP_
-
-
-#include <dpcpp/base/math.hpp>
-#include <dpcpp/components/searching.dp.hpp>
-
-
-#include <CL/sycl.hpp>
-
-
-#include "core/base/utils.hpp"
-#include "dpcpp/base/dpct.hpp"
-#include "dpcpp/components/intrinsics.dp.hpp"
-
-
-namespace gko {
-namespace kernels {
-namespace dpcpp {
-
-
-namespace detail {
-
-
-/**
- * @internal
- * The result from the @ref group_merge_step function.
- */
-template <typename ValueType>
-struct merge_result {
-    /** The element of a being merged in the current thread. */
-    ValueType a_val;
-    /** The element of b being merged in the current thread. */
-    ValueType b_val;
-    /** The index from a that is being merged in the current thread. */
-    int a_idx;
-    /** The index from b that is being merged in the current thread. */
-    int b_idx;
-    /** The number of elements from a that have been merged in total. */
-    int a_advance;
-    /** The number of elements from b that have been merged in total. */
-    int b_advance;
-};
-
-}  // namespace detail
-
-
-/**
- * @internal
- * Warp-parallel merge algorithm that merges the first `warp_size` elements from
- * two ranges, where each warp stores a single element from each range.
- * It assumes that the elements are sorted in ascending order, i.e. for i < j,
- * the value of `a` at thread i is smaller or equal to the value at thread j,
- * and the same holds for `b`.
- *
- * This implementation is based on ideas from  Green et al.,
- * "GPU merge path: a GPU merging algorithm", but uses random-access warp
- * shuffles instead of shared-memory to exchange values of a and b.
- *
- * @param a      the element from the first range
- * @param b      the element from the second range
- * @param size   the number of elements in the output range
- * @param group  the cooperative group that executes the merge
- * @return  a structure containing the merge result distributed over the group.
- */
-template <int group_size, typename ValueType, typename Group>
-__dpct_inline__ detail::merge_result<ValueType> group_merge_step(ValueType a,
-                                                                 ValueType b,
-                                                                 Group group)
-{
-    // thread i takes care of ith element of the merged sequence
-    auto i = int(group.thread_rank());
-
-    // we want to find the smallest index `x` such that a[x] >= b[i - x - 1]
-    // or `i` if no such index exists
-    //
-    // if x = i then c[0...i - 1] = a[0...i - 1]
-    //     => merge a[i] with b[0]
-    // if x = 0 then c[0...i - 1] = b[0...i - 1]
-    //     => merge a[0] with b[i]
-    // otherwise c[0...i - 1] contains a[0...x - 1] and b[0...i - x - 1]
-    //   because the minimality of `x` implies
-    //   b[i - x] >= a[x - 1]
-    //   and a[x] >= a[0...x - 1], b[0...i - x - 1]
-    //     => merge a[x] with b[i - x]
-    auto minx = synchronous_fixed_binary_search<group_size>([&](int x) {
-        auto a_remote = group.shfl(a, x);
-        auto b_remote = group.shfl(b, max(i - x - 1, 0));
-        return a_remote >= b_remote || x >= i;
-    });
-
-    auto a_idx = minx;
-    auto b_idx = max(i - minx, 0);
-    auto a_val = group.shfl(a, a_idx);
-    auto b_val = group.shfl(b, b_idx);
-    auto cmp = a_val < b_val;
-    auto a_advance = popcnt(group.ballot(cmp));
-    auto b_advance = int(group.size()) - a_advance;
-
-    return {a_val, b_val, a_idx, b_idx, a_advance, b_advance};
-}
-
-
-/**
- * @internal
- * Warp-parallel merge algorithm that merges two sorted ranges of arbitrary
- * size. `merge_fn` will be called for each merged element.
- *
- * @param a       the first range
- * @param a_size  the size of the first range
- * @param b       the second range
- * @param b_size  the size of the second range
- * @param group   the group that executes the merge
- * @param merge_fn  the callback that is being called for each merged element.
- *                  It takes six parameters:
- *                  `IndexType a_idx, ValueType a_val, IndexType b_idx,
- *                   ValueType b_val, IndexType c_index, bool valid`.
- *                  `*_val` and `*_idx` are the values resp. the indices of the
- *                  values from a/b being compared at output index `c_index`.
- *                  `valid` specifies if the current thread has to merge an
- *                  element (this is necessary for shfl and ballot operations).
- *                  It must return `false` on all threads of the group iff the
- *                  merge shouldn't be continued.
- */
-template <int group_size, typename ValueType, typename IndexType,
-          typename Group, typename Callback>
-__dpct_inline__ void group_merge(const ValueType *__restrict__ a,
-                                 IndexType a_size,
-                                 const ValueType *__restrict__ b,
-                                 IndexType b_size, Group group,
-                                 Callback merge_fn)
-{
-    auto c_size = a_size + b_size;
-    IndexType a_begin{};
-    IndexType b_begin{};
-    auto lane = static_cast<IndexType>(group.thread_rank());
-    auto sentinel = device_numeric_limits<IndexType>::max;
-    auto a_cur = checked_load(a, a_begin + lane, a_size, sentinel);
-    auto b_cur = checked_load(b, b_begin + lane, b_size, sentinel);
-    for (IndexType c_begin{}; c_begin < c_size; c_begin += group_size) {
-        auto merge_result = group_merge_step<group_size>(a_cur, b_cur, group);
-        auto valid = c_begin + lane < c_size;
-        auto cont = merge_fn(merge_result.a_idx + a_begin, merge_result.a_val,
-                             merge_result.b_idx + b_begin, merge_result.b_val,
-                             c_begin + lane, valid);
-        if (!group.any(cont && valid)) {
-            break;
-        }
-        auto a_advance = merge_result.a_advance;
-        auto b_advance = merge_result.b_advance;
-        a_begin += a_advance;
-        b_begin += b_advance;
-
-        // shuffle the unmerged elements to the front
-        a_cur = group.shfl_down(a_cur, a_advance);
-        b_cur = group.shfl_down(b_cur, b_advance);
-        /*
-         * To optimize memory access, we load the new elements for `a` and `b`
-         * with a single load instruction:
-         * the lower part of the group loads new elements for `a`
-         * the upper part of the group loads new elements for `b`
-         * `load_lane` is the part-local lane idx
-         * The elements for `a` have to be shuffled up afterwards.
-         */
-        auto load_a = lane < a_advance;
-        auto load_lane = load_a ? lane : lane - a_advance;
-        auto load_source = load_a ? a : b;
-        auto load_begin = load_a ? a_begin + b_advance : b_begin + a_advance;
-        auto load_size = load_a ? a_size : b_size;
-
-        auto load_idx = load_begin + load_lane;
-        auto loaded = checked_load(load_source, load_idx, load_size, sentinel);
-        // shuffle the `a` values to the end of the warp
-        auto lower_loaded = group.shfl_up(loaded, b_advance);
-        a_cur = lane < b_advance ? a_cur : lower_loaded;
-        b_cur = lane < a_advance ? b_cur : loaded;
-    }
-}
-
-
-/**
- * @internal
- * Warp-parallel merge algorithm that reports matching elements from two sorted
- * ranges of arbitrary size. `merge_fn` will be called for each pair of matching
- * element.
- *
- * @param a       the first range
- * @param a_size  the size of the first range
- * @param b       the second range
- * @param b_size  the size of the second range
- * @param group   the group that executes the merge
- * @param match_fn  the callback that is being called for each matching pair.
- *                  It takes five parameters:
- *                  `ValueType val, IndexType a_idx, IndexType b_idx,
- *                   lane_mask_type match_mask, bool valid`.
- *                  `val` is the matching element, `*_idx` are the indices of
- *                  the matching values from a and b, match_mask is a lane mask
- *                  that is 1 for every subwarp lane that found a match.
- *                  `valid` is true iff there is actually a match.
- *                  (necessary for warp-synchronous operations)
- */
-template <int group_size, typename IndexType, typename ValueType,
-          typename Group, typename Callback>
-__dpct_inline__ void group_match(const ValueType *__restrict__ a,
-                                 IndexType a_size,
-                                 const ValueType *__restrict__ b,
-                                 IndexType b_size, Group group,
-                                 Callback match_fn)
-{
-    group_merge<group_size>(
-        a, a_size, b, b_size, group,
-        [&](IndexType a_idx, ValueType a_val, IndexType b_idx, ValueType b_val,
-            IndexType, bool valid) {
-            auto matchmask = group.ballot(a_val == b_val && valid);
-            match_fn(a_val, a_idx, b_idx, matchmask, a_val == b_val && valid);
-            return a_idx < a_size && b_idx < b_size;
-        });
-}
-
-
-/**
- * @internal
- * Sequential merge algorithm that merges two sorted ranges of arbitrary
- * size. `merge_fn` will be called for each merged element.
- *
- * @param a  the first range
- * @param a_size the size of the first range
- * @param b  the second range
- * @param b_size the size of the second range
- * @param merge_fn  the callback that will be called for each merge step.
- *                  It takes five parameters:
- *                  `IndexType a_idx, ValueType a_val,
- *                   IndexType b_idx, ValueType b_val, IndexType c_idx`.
- *                  `*_val` and `*_idx` are the values resp. the indices of
- *                  the values from a/b being compared in step `c_idx`.
- *                  It must return `false` iff the merge should stop.
- */
-template <typename ValueType, typename IndexType, typename Callback>
-__dpct_inline__ void sequential_merge(const ValueType *__restrict__ a,
-                                      IndexType a_size,
-                                      const ValueType *__restrict__ b,
-                                      IndexType b_size, Callback merge_fn)
-{
-    auto c_size = a_size + b_size;
-    IndexType a_begin{};
-    IndexType b_begin{};
-    auto sentinel = device_numeric_limits<IndexType>::max;
-    auto a_cur = checked_load(a, a_begin, a_size, sentinel);
-    auto b_cur = checked_load(b, b_begin, b_size, sentinel);
-    for (IndexType c_begin{}; c_begin < c_size; c_begin++) {
-        auto cont = merge_fn(a_begin, a_cur, b_begin, b_cur, c_begin);
-        if (!cont) {
-            break;
-        }
-        auto a_advance = a_cur < b_cur;
-        auto b_advance = !a_advance;
-        a_begin += a_advance;
-        b_begin += b_advance;
-
-        auto load = a_advance ? a : b;
-        auto load_size = a_advance ? a_size : b_size;
-        auto load_idx = a_advance ? a_begin : b_begin;
-        auto loaded = checked_load(load, load_idx, load_size, sentinel);
-        a_cur = a_advance ? loaded : a_cur;
-        b_cur = b_advance ? loaded : b_cur;
-    }
-}
-
-
-/**
- * @internal
- * Sequential algorithm that finds matching elements in two sorted ranges of
- * arbitrary size. `merge_fn` will be called for each pair of matching
- * elements.
- *
- * @param a  the first range
- * @param a_size the size of the first range
- * @param b  the second range
- * @param b_size the size of the second range
- * @param match_fn  the callback that is being called for each match.
- *                  It takes three parameters:
- *                  `ValueType val, IndexType a_idx, IndexType b_idx`.
- *                  `val` is the matching element, `*_idx` are the
- *                  indices of the matching values from a and b.
- */
-template <typename IndexType, typename ValueType, typename Callback>
-__dpct_inline__ void sequential_match(const ValueType *a, IndexType a_size,
-                                      const ValueType *b, IndexType b_size,
-                                      Callback match_fn)
-{
-    sequential_merge(a, a_size, b, b_size,
-                     [&](IndexType a_idx, ValueType a_val, IndexType b_idx,
-                         ValueType b_val, IndexType) {
-                         if (a_val == b_val) {
-                             match_fn(a_val, a_idx, b_idx);
-                         }
-                         return a_idx < a_size && b_idx < b_size;
-                     });
-}
-
-}  // namespace dpcpp
-}  // namespace kernels
-}  // namespace gko
-
-
-#endif  // GKO_DPCPP_COMPONENTS_MERGING_DP_HPP_
diff --git a/dpcpp/components/segment_scan.dp.hpp b/dpcpp/components/segment_scan.dp.hpp
index 0985bc09739..09dc3e4e20f 100644
--- a/dpcpp/components/segment_scan.dp.hpp
+++ b/dpcpp/components/segment_scan.dp.hpp
@@ -50,18 +50,18 @@ namespace dpcpp {
 /**
  * @internal
  *
- * Compute a segement scan using add operation (+) of a subwarp. Each segment
- * performs suffix sum. Works on the source array and returns whether the thread
- * is the first element of its segment with same `ind`.
+ * Compute a segement scan using add operation (+) of a subgroup_size. Each
+ * segment performs suffix sum. Works on the source array and returns whether
+ * the thread is the first element of its segment with same `ind`.
  */
-template <unsigned subwarp_size, typename ValueType, typename IndexType>
+template <unsigned subgroup_size, typename ValueType, typename IndexType>
 __dpct_inline__ bool segment_scan(
-    const group::thread_block_tile<subwarp_size> &group, const IndexType ind,
+    const group::thread_block_tile<subgroup_size> &group, const IndexType ind,
     ValueType *__restrict__ val)
 {
     bool head = true;
 #pragma unroll
-    for (int i = 1; i < subwarp_size; i <<= 1) {
+    for (int i = 1; i < subgroup_size; i <<= 1) {
         const IndexType add_ind = group.shfl_up(ind, i);
         ValueType add_val = zero<ValueType>();
         if (add_ind == ind && group.thread_rank() >= i) {
@@ -71,7 +71,7 @@ __dpct_inline__ bool segment_scan(
             }
         }
         add_val = group.shfl_down(add_val, i);
-        if (group.thread_rank() < subwarp_size - i) {
+        if (group.thread_rank() < subgroup_size - i) {
             *val += add_val;
         }
     }
diff --git a/dpcpp/matrix/coo_kernels.dp.cpp b/dpcpp/matrix/coo_kernels.dp.cpp
index a6150a75681..63b317e7fec 100644
--- a/dpcpp/matrix/coo_kernels.dp.cpp
+++ b/dpcpp/matrix/coo_kernels.dp.cpp
@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/dense_kernels.hpp"
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/base/helper.hpp"
 #include "dpcpp/components/atomic.dp.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/format_conversion.dp.hpp"
@@ -166,24 +167,6 @@ void abstract_spmv(const size_type nnz, const size_type num_lines,
         [](const ValueType &x) { return x; }, item_ct1);
 }
 
-template <typename ValueType, typename IndexType>
-void abstract_spmv(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                   sycl::queue *queue, const size_type nnz,
-                   const size_type num_lines, const ValueType *val,
-                   const IndexType *col, const IndexType *row,
-                   const ValueType *b, const size_type b_stride, ValueType *c,
-                   const size_type c_stride)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             abstract_spmv(nnz, num_lines, val, col, row, b,
-                                           b_stride, c, c_stride, item_ct1);
-                         });
-    });
-}
-
-
 template <typename ValueType, typename IndexType>
 void abstract_spmv(const size_type nnz, const size_type num_lines,
                    const ValueType *__restrict__ alpha,
@@ -201,23 +184,7 @@ void abstract_spmv(const size_type nnz, const size_type num_lines,
         item_ct1);
 }
 
-template <typename ValueType, typename IndexType>
-void abstract_spmv(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                   sycl::queue *queue, const size_type nnz,
-                   const size_type num_lines, const ValueType *alpha,
-                   const ValueType *val, const IndexType *col,
-                   const IndexType *row, const ValueType *b,
-                   const size_type b_stride, ValueType *c,
-                   const size_type c_stride)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             abstract_spmv(nnz, num_lines, alpha, val, col, row,
-                                           b, b_stride, c, c_stride, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(abstract_spmv, abstract_spmv);
 
 
 /**
@@ -291,25 +258,6 @@ void abstract_spmm(const size_type nnz, const size_type num_elems,
         [](const ValueType &x) { return x; }, item_ct1);
 }
 
-template <typename ValueType, typename IndexType>
-void abstract_spmm(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                   sycl::queue *queue, const size_type nnz,
-                   const size_type num_elems, const ValueType *val,
-                   const IndexType *col, const IndexType *row,
-                   const size_type num_cols, const ValueType *b,
-                   const size_type b_stride, ValueType *c,
-                   const size_type c_stride)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                abstract_spmm(nnz, num_elems, val, col, row, num_cols, b,
-                              b_stride, c, c_stride, item_ct1);
-            });
-    });
-}
-
-
 template <typename ValueType, typename IndexType>
 void abstract_spmm(const size_type nnz, const size_type num_elems,
                    const ValueType *__restrict__ alpha,
@@ -327,23 +275,7 @@ void abstract_spmm(const size_type nnz, const size_type num_elems,
         item_ct1);
 }
 
-template <typename ValueType, typename IndexType>
-void abstract_spmm(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                   sycl::queue *queue, const size_type nnz,
-                   const size_type num_elems, const ValueType *alpha,
-                   const ValueType *val, const IndexType *col,
-                   const IndexType *row, const size_type num_cols,
-                   const ValueType *b, const size_type b_stride, ValueType *c,
-                   const size_type c_stride)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                abstract_spmm(nnz, num_elems, alpha, val, col, row, num_cols, b,
-                              b_stride, c, c_stride, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(abstract_spmm, abstract_spmm);
 
 
 }  // namespace
@@ -374,6 +306,7 @@ void convert_row_idxs_to_ptrs(const IndexType *__restrict__ idxs,
     }
 }
 
+// can not use GKO_ENABLE_DEFAULT_HOST, otherwise we can not inistantiate it.
 template <typename IndexType>
 void convert_row_idxs_to_ptrs(dim3 grid, dim3 block,
                               size_type dynamic_shared_memory,
@@ -390,10 +323,10 @@ void convert_row_idxs_to_ptrs(dim3 grid, dim3 block,
     });
 }
 
-template void convert_row_idxs_to_ptrs(dim3, dim3, size_t, sycl::queue *,
+template void convert_row_idxs_to_ptrs(dim3, dim3, size_type, sycl::queue *,
                                        const int32 *idxs, size_type, int32 *,
                                        size_type);
-template void convert_row_idxs_to_ptrs(dim3, dim3, size_t, sycl::queue *,
+template void convert_row_idxs_to_ptrs(dim3, dim3, size_type, sycl::queue *,
                                        const int64 *idxs, size_type, int64 *,
                                        size_type);
 
@@ -413,20 +346,7 @@ void initialize_zero_dense(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename ValueType>
-void initialize_zero_dense(dim3 grid, dim3 block,
-                           size_type dynamic_shared_memory, sycl::queue *queue,
-                           size_type num_rows, size_type num_cols,
-                           size_type stride, ValueType *result)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             initialize_zero_dense(num_rows, num_cols, stride,
-                                                   result, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(initialize_zero_dense, initialize_zero_dense);
 
 
 template <typename ValueType, typename IndexType>
@@ -441,52 +361,7 @@ void fill_in_dense(size_type nnz, const IndexType *__restrict__ row_idxs,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void fill_in_dense(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                   sycl::queue *queue, size_type nnz, const IndexType *row_idxs,
-                   const IndexType *col_idxs, const ValueType *values,
-                   size_type stride, ValueType *result)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             fill_in_dense(nnz, row_idxs, col_idxs, values,
-                                           stride, result, item_ct1);
-                         });
-    });
-}
-
-
-template <typename ValueType, typename IndexType>
-void extract_diagonal(size_type nnz, const ValueType *__restrict__ orig_values,
-                      const IndexType *__restrict__ orig_row_idxs,
-                      const IndexType *__restrict__ orig_col_idxs,
-                      ValueType *__restrict__ diag, sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-
-    if (tidx < nnz) {
-        if (orig_row_idxs[tidx] == orig_col_idxs[tidx]) {
-            diag[orig_row_idxs[tidx]] = orig_values[tidx];
-        }
-    }
-}
-
-template <typename ValueType, typename IndexType>
-void extract_diagonal(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                      sycl::queue *queue, size_type nnz,
-                      const ValueType *orig_values,
-                      const IndexType *orig_row_idxs,
-                      const IndexType *orig_col_idxs, ValueType *diag)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             extract_diagonal(nnz, orig_values, orig_row_idxs,
-                                              orig_col_idxs, diag, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(fill_in_dense, fill_in_dense);
 
 
 }  // namespace kernel
@@ -659,29 +534,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_CONVERT_TO_DENSE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void extract_diagonal(std::shared_ptr<const DpcppExecutor> exec,
-                      const matrix::Coo<ValueType, IndexType> *orig,
-                      matrix::Diagonal<ValueType> *diag)
-{
-    const auto nnz = orig->get_num_stored_elements();
-    const auto diag_size = diag->get_size()[0];
-    const auto num_blocks = ceildiv(nnz, default_block_size);
-
-    const auto orig_values = orig->get_const_values();
-    const auto orig_row_idxs = orig->get_const_row_idxs();
-    const auto orig_col_idxs = orig->get_const_col_idxs();
-    auto diag_values = diag->get_values();
-
-    kernel::extract_diagonal(num_blocks, default_block_size, 0,
-                             exec->get_queue(), nnz, orig_values, orig_row_idxs,
-                             orig_col_idxs, diag_values);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_COO_EXTRACT_DIAGONAL_KERNEL);
-
-
 }  // namespace coo
 }  // namespace dpcpp
 }  // namespace kernels
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index b8fbb6cadd9..089bc2238b9 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -59,6 +59,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
 #include "dpcpp/base/dpct.hpp"
+#include "dpcpp/base/helper.hpp"
 #include "dpcpp/components/atomic.dp.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/reduction.dp.hpp"
@@ -81,7 +82,6 @@ namespace csr {
 constexpr int default_block_size = 256;
 constexpr int warps_in_block = 4;
 constexpr int spmv_block_size = warps_in_block * config::warp_size;
-constexpr int wsize = config::warp_size;
 constexpr int classical_overweight = 32;
 
 
@@ -142,8 +142,9 @@ __dpct_inline__ void find_next_row(
         if (ind >= *row_end) {
             *row = row_predict;
             *row_end = row_predict_end;
-            for (; ind >= *row_end; *row_end = row_ptr[++*row + 1])
-                ;
+            while (ind >= *row_end) {
+                *row_end = row_ptr[++*row + 1];
+            }
         }
 
     } else {
@@ -207,8 +208,8 @@ __dpct_inline__ IndexType get_warp_start_idx(const IndexType nwarps,
                                              const IndexType nnz,
                                              const IndexType warp_idx)
 {
-    const long long cache_lines = ceildivT<IndexType>(nnz, wsize);
-    return (warp_idx * cache_lines / nwarps) * wsize;
+    const long long cache_lines = ceildivT<IndexType>(nnz, config::warp_size);
+    return (warp_idx * cache_lines / nwarps) * config::warp_size;
 }
 
 
@@ -229,6 +230,7 @@ __dpct_inline__ void spmv_kernel(
     }
     const IndexType data_size = row_ptrs[num_rows];
     const IndexType start = get_warp_start_idx(nwarps, data_size, warp_idx);
+    constexpr IndexType wsize = config::warp_size;
     const IndexType end =
         min(get_warp_start_idx(nwarps, data_size, warp_idx + 1),
             ceildivT<IndexType>(data_size, wsize) * wsize);
@@ -272,25 +274,6 @@ void abstract_spmv(const IndexType nwarps, const IndexType num_rows,
         c_stride, [](const ValueType &x) { return x; }, item_ct1);
 }
 
-template <typename ValueType, typename IndexType>
-void abstract_spmv(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                   sycl::queue *queue, const IndexType nwarps,
-                   const IndexType num_rows, const ValueType *val,
-                   const IndexType *col_idxs, const IndexType *row_ptrs,
-                   const IndexType *srow, const ValueType *b,
-                   const size_type b_stride, ValueType *c,
-                   const size_type c_stride)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                abstract_spmv(nwarps, num_rows, val, col_idxs, row_ptrs, srow,
-                              b, b_stride, c, c_stride, item_ct1);
-            });
-    });
-}
-
-
 template <typename ValueType, typename IndexType>
 void abstract_spmv(const IndexType nwarps, const IndexType num_rows,
                    const ValueType *__restrict__ alpha,
@@ -310,23 +293,7 @@ void abstract_spmv(const IndexType nwarps, const IndexType num_rows,
         item_ct1);
 }
 
-template <typename ValueType, typename IndexType>
-void abstract_spmv(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                   sycl::queue *queue, const IndexType nwarps,
-                   const IndexType num_rows, const ValueType *alpha,
-                   const ValueType *val, const IndexType *col_idxs,
-                   const IndexType *row_ptrs, const IndexType *srow,
-                   const ValueType *b, const size_type b_stride, ValueType *c,
-                   const size_type c_stride)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                abstract_spmv(nwarps, num_rows, alpha, val, col_idxs, row_ptrs,
-                              srow, b, b_stride, c, c_stride, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(abstract_spmv, abstract_spmv);
 
 
 template <typename ValueType>
@@ -339,16 +306,7 @@ void set_zero(const size_type nnz, ValueType *__restrict__ val,
     }
 }
 
-template <typename ValueType>
-void set_zero(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-              sycl::queue *queue, const size_type nnz, ValueType *val)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block),
-            [=](sycl::nd_item<3> item_ct1) { set_zero(nnz, val, item_ct1); });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(set_zero, set_zero);
 
 
 template <typename IndexType>
@@ -487,9 +445,7 @@ void merge_path_spmv(const IndexType num_rows,
     tmp_val[item_ct1.get_local_id(2)] = value;
     tmp_ind[item_ct1.get_local_id(2)] = row_i;
     group::this_thread_block(item_ct1).sync();
-    bool last =
-        block_segment_scan_reverse(static_cast<IndexType *>(tmp_ind),
-                                   static_cast<ValueType *>(tmp_val), item_ct1);
+    bool last = block_segment_scan_reverse(tmp_ind, tmp_val, item_ct1);
     if (item_ct1.get_local_id(2) == spmv_block_size - 1) {
         row_out[item_ct1.get_group(2)] = min(end_x, num_rows - 1);
         val_out[item_ct1.get_group(2)] = tmp_val[item_ct1.get_local_id(2)];
@@ -674,13 +630,6 @@ void abstract_reduce(dim3 grid, dim3 block, size_type dynamic_shared_memory,
 }
 
 
-#ifdef __SYCL_DEVICE_ONLY__
-#define CONSTANT __attribute__((opencl_constant))
-#else
-#define CONSTANT
-#endif
-
-
 template <size_type subwarp_size, typename ValueType, typename IndexType,
           typename Closure>
 void device_classical_spmv(const size_type num_rows,
@@ -708,6 +657,7 @@ void device_classical_spmv(const size_type num_rows,
         auto subwarp_result = ::gko::kernels::dpcpp::reduce(
             subwarp_tile, temp_val,
             [](const ValueType &a, const ValueType &b) { return a + b; });
+        // TODO: check the barrier
         item_ct1.barrier();
         if (subid == 0) {
             c[row * c_stride + column_id] =
@@ -804,19 +754,7 @@ void convert_row_ptrs_to_idxs(size_type num_rows,
     }
 }
 
-template <typename IndexType>
-void convert_row_ptrs_to_idxs(dim3 grid, dim3 block,
-                              size_type dynamic_shared_memory,
-                              sycl::queue *queue, size_type num_rows,
-                              const IndexType *ptrs, IndexType *idxs)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                convert_row_ptrs_to_idxs(num_rows, ptrs, idxs, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(convert_row_ptrs_to_idxs, convert_row_ptrs_to_idxs);
 
 
 template <typename ValueType>
@@ -835,20 +773,7 @@ void initialize_zero_dense(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename ValueType>
-void initialize_zero_dense(dim3 grid, dim3 block,
-                           size_type dynamic_shared_memory, sycl::queue *queue,
-                           size_type num_rows, size_type num_cols,
-                           size_type stride, ValueType *result)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             initialize_zero_dense(num_rows, num_cols, stride,
-                                                   result, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(initialize_zero_dense, initialize_zero_dense);
 
 
 template <typename ValueType, typename IndexType>
@@ -865,20 +790,7 @@ void fill_in_dense(size_type num_rows, const IndexType *__restrict__ row_ptrs,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void fill_in_dense(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                   sycl::queue *queue, size_type num_rows,
-                   const IndexType *row_ptrs, const IndexType *col_idxs,
-                   const ValueType *values, size_type stride, ValueType *result)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             fill_in_dense(num_rows, row_ptrs, col_idxs, values,
-                                           stride, result, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(fill_in_dense, fill_in_dense);
 
 
 template <typename IndexType>
@@ -893,20 +805,7 @@ void calculate_nnz_per_row(size_type num_rows,
     }
 }
 
-template <typename IndexType>
-void calculate_nnz_per_row(dim3 grid, dim3 block,
-                           size_type dynamic_shared_memory, sycl::queue *queue,
-                           size_type num_rows, const IndexType *row_ptrs,
-                           size_type *nnz_per_row)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             calculate_nnz_per_row(num_rows, row_ptrs,
-                                                   nnz_per_row, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(calculate_nnz_per_row, calculate_nnz_per_row);
 
 
 void calculate_slice_lengths(size_type num_rows, size_type slice_size,
@@ -944,22 +843,7 @@ void calculate_slice_lengths(size_type num_rows, size_type slice_size,
     }
 }
 
-void calculate_slice_lengths(dim3 grid, dim3 block,
-                             size_type dynamic_shared_memory,
-                             sycl::queue *queue, size_type num_rows,
-                             size_type slice_size, size_type stride_factor,
-                             const size_type *nnz_per_row,
-                             size_type *slice_lengths, size_type *slice_sets)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                calculate_slice_lengths(num_rows, slice_size, stride_factor,
-                                        nnz_per_row, slice_lengths, slice_sets,
-                                        item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(calculate_slice_lengths, calculate_slice_lengths);
 
 
 template <typename ValueType, typename IndexType>
@@ -996,25 +880,7 @@ void fill_in_sellp(size_type num_rows, size_type slice_size,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void fill_in_sellp(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                   sycl::queue *queue, size_type num_rows, size_type slice_size,
-                   const ValueType *source_values,
-                   const IndexType *source_row_ptrs,
-                   const IndexType *source_col_idxs, size_type *slice_lengths,
-                   size_type *slice_sets, IndexType *result_col_idxs,
-                   ValueType *result_values)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                fill_in_sellp(num_rows, slice_size, source_values,
-                              source_row_ptrs, source_col_idxs, slice_lengths,
-                              slice_sets, result_col_idxs, result_values,
-                              item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(fill_in_sellp, fill_in_sellp);
 
 
 template <typename ValueType, typename IndexType>
@@ -1031,20 +897,7 @@ void initialize_zero_ell(size_type max_nnz_per_row, size_type stride,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void initialize_zero_ell(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                         sycl::queue *queue, size_type max_nnz_per_row,
-                         size_type stride, ValueType *values,
-                         IndexType *col_idxs)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             initialize_zero_ell(max_nnz_per_row, stride,
-                                                 values, col_idxs, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(initialize_zero_ell, initialize_zero_ell);
 
 
 template <typename ValueType, typename IndexType>
@@ -1072,23 +925,7 @@ void fill_in_ell(size_type num_rows, size_type stride,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void fill_in_ell(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                 sycl::queue *queue, size_type num_rows, size_type stride,
-                 const ValueType *source_values,
-                 const IndexType *source_row_ptrs,
-                 const IndexType *source_col_idxs, ValueType *result_values,
-                 IndexType *result_col_idxs)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                fill_in_ell(num_rows, stride, source_values, source_row_ptrs,
-                            source_col_idxs, result_values, result_col_idxs,
-                            item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(fill_in_ell, fill_in_ell);
 
 
 void reduce_max_nnz_per_slice(size_type num_rows, size_type slice_size,
@@ -1120,20 +957,7 @@ void reduce_max_nnz_per_slice(size_type num_rows, size_type slice_size,
     }
 }
 
-void reduce_max_nnz_per_slice(dim3 grid, dim3 block,
-                              size_type dynamic_shared_memory,
-                              sycl::queue *queue, size_type num_rows,
-                              size_type slice_size, size_type stride_factor,
-                              const size_type *nnz_per_row, size_type *result)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                reduce_max_nnz_per_slice(num_rows, slice_size, stride_factor,
-                                         nnz_per_row, result, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(reduce_max_nnz_per_slice, reduce_max_nnz_per_slice);
 
 
 void reduce_total_cols(size_type num_slices,
@@ -1156,8 +980,7 @@ void reduce_total_cols(dim3 grid, dim3 block, size_type dynamic_shared_memory,
     queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<size_type, 1, sycl::access_mode::read_write,
                        sycl::access::target::local>
-            block_result_acc_ct1(sycl::range<1>(512 /*default_block_size*/),
-                                 cgh);
+            block_result_acc_ct1(sycl::range<1>(default_block_size), cgh);
 
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
@@ -1188,7 +1011,7 @@ void reduce_max_nnz(dim3 grid, dim3 block, size_type dynamic_shared_memory,
     queue->submit([&](sycl::handler &cgh) {
         sycl::accessor<size_type, 1, sycl::access_mode::read_write,
                        sycl::access::target::local>
-            block_max_acc_ct1(sycl::range<1>(512 /*default_block_size*/), cgh);
+            block_max_acc_ct1(sycl::range<1>(default_block_size), cgh);
 
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
@@ -1214,23 +1037,8 @@ void calculate_hybrid_coo_row_nnz(size_type num_rows,
     }
 }
 
-template <typename IndexType>
-void calculate_hybrid_coo_row_nnz(dim3 grid, dim3 block,
-                                  size_type dynamic_shared_memory,
-                                  sycl::queue *queue, size_type num_rows,
-                                  size_type ell_max_nnz_per_row,
-                                  IndexType *csr_row_idxs,
-                                  size_type *coo_row_nnz)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                calculate_hybrid_coo_row_nnz(num_rows, ell_max_nnz_per_row,
-                                             csr_row_idxs, coo_row_nnz,
-                                             item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(calculate_hybrid_coo_row_nnz,
+                        calculate_hybrid_coo_row_nnz);
 
 
 template <typename ValueType, typename IndexType>
@@ -1271,28 +1079,7 @@ void fill_in_hybrid(size_type num_rows, size_type stride,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void fill_in_hybrid(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                    sycl::queue *queue, size_type num_rows, size_type stride,
-                    size_type ell_max_nnz_per_row,
-                    const ValueType *source_values,
-                    const IndexType *source_row_ptrs,
-                    const IndexType *source_col_idxs,
-                    const size_type *coo_offset, ValueType *result_ell_val,
-                    IndexType *result_ell_col, ValueType *result_coo_val,
-                    IndexType *result_coo_col, IndexType *result_coo_row)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                fill_in_hybrid(num_rows, stride, ell_max_nnz_per_row,
-                               source_values, source_row_ptrs, source_col_idxs,
-                               coo_offset, result_ell_val, result_ell_col,
-                               result_coo_val, result_coo_col, result_coo_row,
-                               item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(fill_in_hybrid, fill_in_hybrid);
 
 
 template <typename IndexType>
@@ -1367,21 +1154,7 @@ void extract_diagonal(size_type diag_size, size_type nnz,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void extract_diagonal(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                      sycl::queue *queue, size_type diag_size, size_type nnz,
-                      const ValueType *orig_values,
-                      const IndexType *orig_row_ptrs,
-                      const IndexType *orig_col_idxs, ValueType *diag)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                extract_diagonal(diag_size, nnz, orig_values, orig_row_ptrs,
-                                 orig_col_idxs, diag, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(extract_diagonal, extract_diagonal);
 
 
 }  // namespace kernel
@@ -1401,18 +1174,7 @@ void conjugate_kernel(size_type num_nonzeros, ValueType *__restrict__ val,
     }
 }
 
-template <typename ValueType>
-void conjugate_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                      sycl::queue *queue, size_type num_nonzeros,
-                      ValueType *val)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             conjugate_kernel(num_nonzeros, val, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(conjugate_kernel, conjugate_kernel);
 
 
 }  //  namespace
@@ -1431,20 +1193,7 @@ void inv_permutation_kernel(size_type size,
     inv_permutation[permutation[tid]] = tid;
 }
 
-template <typename IndexType>
-void inv_permutation_kernel(dim3 grid, dim3 block,
-                            size_type dynamic_shared_memory, sycl::queue *queue,
-                            size_type size, const IndexType *permutation,
-                            IndexType *inv_permutation)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             inv_permutation_kernel(size, permutation,
-                                                    inv_permutation, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(inv_permutation_kernel, inv_permutation_kernel);
 
 
 template <typename ValueType, typename IndexType>
@@ -1468,23 +1217,7 @@ void col_permute_kernel(size_type num_rows, size_type num_nonzeros,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void col_permute_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                        sycl::queue *queue, size_type num_rows,
-                        size_type num_nonzeros, const IndexType *permutation,
-                        const IndexType *in_row_ptrs, const IndexType *in_cols,
-                        const ValueType *in_vals, IndexType *out_row_ptrs,
-                        IndexType *out_cols, ValueType *out_vals)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                col_permute_kernel(num_rows, num_nonzeros, permutation,
-                                   in_row_ptrs, in_cols, in_vals, out_row_ptrs,
-                                   out_cols, out_vals, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(col_permute_kernel, col_permute_kernel);
 
 
 template <typename IndexType>
@@ -1503,20 +1236,7 @@ void row_ptr_permute_kernel(size_type num_rows,
     out_nnz[out_row] = in_row_ptrs[in_row + 1] - in_row_ptrs[in_row];
 }
 
-template <typename IndexType>
-void row_ptr_permute_kernel(dim3 grid, dim3 block,
-                            size_type dynamic_shared_memory, sycl::queue *queue,
-                            size_type num_rows, const IndexType *permutation,
-                            const IndexType *in_row_ptrs, IndexType *out_nnz)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                row_ptr_permute_kernel(num_rows, permutation, in_row_ptrs,
-                                       out_nnz, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(row_ptr_permute_kernel, row_ptr_permute_kernel);
 
 
 template <typename IndexType>
@@ -1535,22 +1255,7 @@ void inv_row_ptr_permute_kernel(size_type num_rows,
     out_nnz[out_row] = in_row_ptrs[in_row + 1] - in_row_ptrs[in_row];
 }
 
-template <typename IndexType>
-void inv_row_ptr_permute_kernel(dim3 grid, dim3 block,
-                                size_type dynamic_shared_memory,
-                                sycl::queue *queue, size_type num_rows,
-                                const IndexType *permutation,
-                                const IndexType *in_row_ptrs,
-                                IndexType *out_nnz)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                inv_row_ptr_permute_kernel(num_rows, permutation, in_row_ptrs,
-                                           out_nnz, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(inv_row_ptr_permute_kernel, inv_row_ptr_permute_kernel);
 
 
 template <int subwarp_size, typename ValueType, typename IndexType>
@@ -1775,7 +1480,7 @@ void classical_spmv(syn::value_list<int, subwarp_size>,
                     const matrix::Dense<ValueType> *alpha = nullptr,
                     const matrix::Dense<ValueType> *beta = nullptr)
 {
-    const auto threads_per_cu = 7;
+    constexpr int threads_per_cu = 7;
     const auto nwarps =
         exec->get_num_computing_units() * threads_per_cu * classical_overweight;
     const auto gridx =
diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp
index ea2f24c990e..5fefd83f4c7 100644
--- a/dpcpp/matrix/dense_kernels.dp.cpp
+++ b/dpcpp/matrix/dense_kernels.dp.cpp
@@ -78,7 +78,7 @@ constexpr auto kcfg_1d_list =
 constexpr auto subgroup_list =
     syn::value_list<std::uint32_t, 64, 32, 16, 8, 4>();
 constexpr auto kcfg_1d_array = syn::as_array(kcfg_1d_list);
-constexpr auto default_block_size = 256;
+constexpr int default_block_size = 256;
 
 
 namespace kernel {
@@ -828,7 +828,7 @@ void compute_dot(std::shared_ptr<const DpcppExecutor> exec,
         // TODO: these are tuning parameters obtained experimentally, once
         // we decide how to handle this uniformly, they should be modified
         // appropriately
-        constexpr auto work_per_thread = 32;
+        constexpr int work_per_thread = 32;
         auto queue = exec->get_queue();
         constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
         const std::uint32_t cfg =
@@ -875,7 +875,7 @@ void compute_conj_dot(std::shared_ptr<const DpcppExecutor> exec,
         // TODO: these are tuning parameters obtained experimentally, once
         // we decide how to handle this uniformly, they should be modified
         // appropriately
-        constexpr auto work_per_thread = 32;
+        constexpr int work_per_thread = 32;
         auto queue = exec->get_queue();
         constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
         const std::uint32_t cfg =
@@ -920,7 +920,7 @@ void compute_norm2(std::shared_ptr<const DpcppExecutor> exec,
         // TODO: these are tuning parameters obtained experimentally, once
         // we decide how to handle this uniformly, they should be modified
         // appropriately
-        constexpr auto work_per_thread = 32;
+        constexpr int work_per_thread = 32;
         auto queue = exec->get_queue();
         constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
         const std::uint32_t cfg =
diff --git a/dpcpp/matrix/diagonal_kernels.dp.cpp b/dpcpp/matrix/diagonal_kernels.dp.cpp
index 866df7ec315..1aae9a393a7 100644
--- a/dpcpp/matrix/diagonal_kernels.dp.cpp
+++ b/dpcpp/matrix/diagonal_kernels.dp.cpp
@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/base/helper.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/thread_ids.dp.hpp"
 
@@ -57,86 +58,12 @@ namespace dpcpp {
 namespace diagonal {
 
 
-constexpr auto default_block_size = 256;
+constexpr int default_block_size = 256;
 
 
 namespace kernel {
 
 
-template <typename ValueType>
-void apply_to_dense(size_type num_rows, size_type num_cols,
-                    const ValueType *__restrict__ diag, size_type source_stride,
-                    const ValueType *__restrict__ source_values,
-                    size_type result_stride,
-                    ValueType *__restrict__ result_values,
-                    sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-    const auto row = tidx / num_cols;
-    const auto col = tidx % num_cols;
-
-    if (row < num_rows) {
-        result_values[row * result_stride + col] =
-            source_values[row * source_stride + col] * diag[row];
-    }
-}
-
-template <typename ValueType>
-void apply_to_dense(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                    sycl::queue *queue, size_type num_rows, size_type num_cols,
-                    const ValueType *diag, size_type source_stride,
-                    const ValueType *source_values, size_type result_stride,
-                    ValueType *result_values)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                apply_to_dense(num_rows, num_cols, diag, source_stride,
-                               source_values, result_stride, result_values,
-                               item_ct1);
-            });
-    });
-}
-
-
-template <typename ValueType>
-void right_apply_to_dense(size_type num_rows, size_type num_cols,
-                          const ValueType *__restrict__ diag,
-                          size_type source_stride,
-                          const ValueType *__restrict__ source_values,
-                          size_type result_stride,
-                          ValueType *__restrict__ result_values,
-                          sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-    const auto row = tidx / num_cols;
-    const auto col = tidx % num_cols;
-
-    if (row < num_rows) {
-        result_values[row * result_stride + col] =
-            source_values[row * source_stride + col] * diag[col];
-    }
-}
-
-template <typename ValueType>
-void right_apply_to_dense(dim3 grid, dim3 block,
-                          size_type dynamic_shared_memory, sycl::queue *queue,
-                          size_type num_rows, size_type num_cols,
-                          const ValueType *diag, size_type source_stride,
-                          const ValueType *source_values,
-                          size_type result_stride, ValueType *result_values)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                right_apply_to_dense(num_rows, num_cols, diag, source_stride,
-                                     source_values, result_stride,
-                                     result_values, item_ct1);
-            });
-    });
-}
-
-
 template <typename ValueType, typename IndexType>
 void apply_to_csr(size_type num_rows, const ValueType *__restrict__ diag,
                   const IndexType *__restrict__ row_ptrs,
@@ -161,170 +88,12 @@ void apply_to_csr(size_type num_rows, const ValueType *__restrict__ diag,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void apply_to_csr(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                  sycl::queue *queue, size_type num_rows, const ValueType *diag,
-                  const IndexType *row_ptrs, ValueType *result_values)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                apply_to_csr(num_rows, diag, row_ptrs, result_values, item_ct1);
-            });
-    });
-}
-
-
-template <typename ValueType, typename IndexType>
-void right_apply_to_csr(size_type num_nnz, const ValueType *__restrict__ diag,
-                        const IndexType *__restrict__ col_idxs,
-                        ValueType *__restrict__ result_values,
-                        sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-
-    if (tidx >= num_nnz) {
-        return;
-    }
-
-    result_values[tidx] *= diag[col_idxs[tidx]];
-}
-
-template <typename ValueType, typename IndexType>
-void right_apply_to_csr(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                        sycl::queue *queue, size_type num_nnz,
-                        const ValueType *diag, const IndexType *col_idxs,
-                        ValueType *result_values)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             right_apply_to_csr(num_nnz, diag, col_idxs,
-                                                result_values, item_ct1);
-                         });
-    });
-}
-
-
-template <typename ValueType, typename IndexType>
-void convert_to_csr(size_type size, const ValueType *__restrict__ diag_values,
-                    IndexType *__restrict__ row_ptrs,
-                    IndexType *__restrict__ col_idxs,
-                    ValueType *__restrict__ csr_values,
-                    sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-
-    if (tidx >= size) {
-        return;
-    }
-    if (tidx == 0) {
-        row_ptrs[size] = size;
-    }
-
-    row_ptrs[tidx] = tidx;
-    col_idxs[tidx] = tidx;
-    csr_values[tidx] = diag_values[tidx];
-}
-
-template <typename ValueType, typename IndexType>
-void convert_to_csr(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                    sycl::queue *queue, size_type size,
-                    const ValueType *diag_values, IndexType *row_ptrs,
-                    IndexType *col_idxs, ValueType *csr_values)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             convert_to_csr(size, diag_values, row_ptrs,
-                                            col_idxs, csr_values, item_ct1);
-                         });
-    });
-}
-
-
-template <typename ValueType>
-void conj_transpose(size_type size, const ValueType *__restrict__ orig_values,
-                    ValueType *__restrict__ trans_values,
-                    sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-
-    if (tidx >= size) {
-        return;
-    }
-
-    trans_values[tidx] = conj(orig_values[tidx]);
-}
-
-template <typename ValueType>
-void conj_transpose(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                    sycl::queue *queue, size_type size,
-                    const ValueType *orig_values, ValueType *trans_values)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                conj_transpose(size, orig_values, trans_values, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(apply_to_csr, apply_to_csr);
 
 
 }  // namespace kernel
 
 
-template <typename ValueType>
-void apply_to_dense(std::shared_ptr<const DpcppExecutor> exec,
-                    const matrix::Diagonal<ValueType> *a,
-                    const matrix::Dense<ValueType> *b,
-                    matrix::Dense<ValueType> *c)
-{
-    const auto b_size = b->get_size();
-    const auto num_rows = b_size[0];
-    const auto num_cols = b_size[1];
-    const auto b_stride = b->get_stride();
-    const auto c_stride = c->get_stride();
-    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
-
-    const auto diag_values = a->get_const_values();
-    const auto b_values = b->get_const_values();
-    auto c_values = c->get_values();
-
-    kernel::apply_to_dense(grid_dim, default_block_size, 0, exec->get_queue(),
-                           num_rows, num_cols, diag_values, b_stride, b_values,
-                           c_stride, c_values);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_APPLY_TO_DENSE_KERNEL);
-
-
-template <typename ValueType>
-void right_apply_to_dense(std::shared_ptr<const DpcppExecutor> exec,
-                          const matrix::Diagonal<ValueType> *a,
-                          const matrix::Dense<ValueType> *b,
-                          matrix::Dense<ValueType> *c)
-{
-    const auto b_size = b->get_size();
-    const auto num_rows = b_size[0];
-    const auto num_cols = b_size[1];
-    const auto b_stride = b->get_stride();
-    const auto c_stride = c->get_stride();
-    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
-
-    const auto diag_values = a->get_const_values();
-    const auto b_values = b->get_const_values();
-    auto c_values = c->get_values();
-
-    kernel::right_apply_to_dense(
-        grid_dim, default_block_size, 0, exec->get_queue(), num_rows, num_cols,
-        diag_values, b_stride, b_values, c_stride, c_values);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_DENSE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void apply_to_csr(std::shared_ptr<const DpcppExecutor> exec,
                   const matrix::Diagonal<ValueType> *a,
@@ -347,66 +116,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DIAGONAL_APPLY_TO_CSR_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void right_apply_to_csr(std::shared_ptr<const DpcppExecutor> exec,
-                        const matrix::Diagonal<ValueType> *a,
-                        const matrix::Csr<ValueType, IndexType> *b,
-                        matrix::Csr<ValueType, IndexType> *c)
-{
-    const auto num_nnz = b->get_num_stored_elements();
-    const auto diag_values = a->get_const_values();
-    c->copy_from(b);
-    auto csr_values = c->get_values();
-    const auto csr_col_idxs = c->get_const_col_idxs();
-
-    const auto grid_dim = ceildiv(num_nnz, default_block_size);
-    kernel::right_apply_to_csr(grid_dim, default_block_size, 0,
-                               exec->get_queue(), num_nnz, diag_values,
-                               csr_col_idxs, csr_values);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_CSR_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void convert_to_csr(std::shared_ptr<const DpcppExecutor> exec,
-                    const matrix::Diagonal<ValueType> *source,
-                    matrix::Csr<ValueType, IndexType> *result)
-{
-    const auto size = source->get_size()[0];
-    const auto grid_dim = ceildiv(size, default_block_size);
-
-    const auto diag_values = source->get_const_values();
-    auto row_ptrs = result->get_row_ptrs();
-    auto col_idxs = result->get_col_idxs();
-    auto csr_values = result->get_values();
-
-    kernel::convert_to_csr(grid_dim, default_block_size, 0, exec->get_queue(),
-                           size, diag_values, row_ptrs, col_idxs, csr_values);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DIAGONAL_CONVERT_TO_CSR_KERNEL);
-
-
-template <typename ValueType>
-void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
-                    const matrix::Diagonal<ValueType> *orig,
-                    matrix::Diagonal<ValueType> *trans)
-{
-    const auto size = orig->get_size()[0];
-    const auto grid_dim = ceildiv(size, default_block_size);
-    const auto orig_values = orig->get_const_values();
-    auto trans_values = trans->get_values();
-
-    kernel::conj_transpose(grid_dim, default_block_size, 0, exec->get_queue(),
-                           size, orig_values, trans_values);
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_CONJ_TRANSPOSE_KERNEL);
-
-
 }  // namespace diagonal
 }  // namespace dpcpp
 }  // namespace kernels
diff --git a/dpcpp/matrix/ell_kernels.dp.cpp b/dpcpp/matrix/ell_kernels.dp.cpp
index e021b3edc22..34ea5e06dc5 100644
--- a/dpcpp/matrix/ell_kernels.dp.cpp
+++ b/dpcpp/matrix/ell_kernels.dp.cpp
@@ -54,6 +54,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/synthesizer/implementation_selection.hpp"
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/base/helper.hpp"
 #include "dpcpp/components/atomic.dp.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/format_conversion.dp.hpp"
@@ -315,20 +316,7 @@ void initialize_zero_dense(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename ValueType>
-void initialize_zero_dense(dim3 grid, dim3 block,
-                           size_type dynamic_shared_memory, sycl::queue *queue,
-                           size_type num_rows, size_type num_cols,
-                           size_type stride, ValueType *result)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             initialize_zero_dense(num_rows, num_cols, stride,
-                                                   result, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(initialize_zero_dense, initialize_zero_dense);
 
 
 template <typename ValueType, typename IndexType>
@@ -348,21 +336,7 @@ void fill_in_dense(size_type num_rows, size_type nnz, size_type source_stride,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void fill_in_dense(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                   sycl::queue *queue, size_type num_rows, size_type nnz,
-                   size_type source_stride, const IndexType *col_idxs,
-                   const ValueType *values, size_type result_stride,
-                   ValueType *result)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                fill_in_dense(num_rows, nnz, source_stride, col_idxs, values,
-                              result_stride, result, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(fill_in_dense, fill_in_dense);
 
 
 template <typename ValueType, typename IndexType>
@@ -405,16 +379,16 @@ void count_nnz_per_row(dim3 grid, dim3 block, size_type dynamic_shared_memory,
     });
 }
 
-
-#define GKO_ELL_COUNT_NNZ_PER_ROW(ValueType, IndexType)                        \
-    void count_nnz_per_row(dim3, dim3, gko::size_type, sycl::queue *,          \
-                           size_type, size_type, size_type, const ValueType *, \
+#define GKO_ELL_COUNT_NNZ_PER_ROW(ValueType, IndexType)                     \
+    void count_nnz_per_row(dim3, dim3, size_type, sycl::queue *, size_type, \
+                           size_type, size_type, const ValueType *,         \
                            IndexType *)
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_ELL_COUNT_NNZ_PER_ROW);
 
 #undef GKO_ELL_COUNT_NNZ_PER_ROW
 
+
 template <typename ValueType, typename IndexType>
 void fill_in_csr(size_type num_rows, size_type max_nnz_per_row,
                  size_type stride, const ValueType *__restrict__ source_values,
@@ -439,23 +413,7 @@ void fill_in_csr(size_type num_rows, size_type max_nnz_per_row,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void fill_in_csr(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                 sycl::queue *queue, size_type num_rows,
-                 size_type max_nnz_per_row, size_type stride,
-                 const ValueType *source_values,
-                 const IndexType *source_col_idxs, IndexType *result_row_ptrs,
-                 IndexType *result_col_idxs, ValueType *result_values)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                fill_in_csr(num_rows, max_nnz_per_row, stride, source_values,
-                            source_col_idxs, result_row_ptrs, result_col_idxs,
-                            result_values, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(fill_in_csr, fill_in_csr);
 
 
 template <typename ValueType, typename IndexType>
@@ -478,21 +436,7 @@ void extract_diagonal(size_type diag_size, size_type max_nnz_per_row,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void extract_diagonal(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                      sycl::queue *queue, size_type diag_size,
-                      size_type max_nnz_per_row, size_type orig_stride,
-                      const ValueType *orig_values,
-                      const IndexType *orig_col_idxs, ValueType *diag)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                extract_diagonal(diag_size, max_nnz_per_row, orig_stride,
-                                 orig_values, orig_col_idxs, diag, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(extract_diagonal, extract_diagonal);
 
 
 }  // namespace kernel
diff --git a/dpcpp/matrix/hybrid_kernels.dp.cpp b/dpcpp/matrix/hybrid_kernels.dp.cpp
index 3a5b0f05173..2d0684d0cba 100644
--- a/dpcpp/matrix/hybrid_kernels.dp.cpp
+++ b/dpcpp/matrix/hybrid_kernels.dp.cpp
@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/matrix/ell_kernels.hpp"
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/base/helper.hpp"
 #include "dpcpp/components/atomic.dp.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/format_conversion.dp.hpp"
@@ -185,24 +186,7 @@ void fill_in_csr(size_type num_rows, size_type max_nnz_per_row,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void fill_in_csr(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                 sycl::queue *queue, size_type num_rows,
-                 size_type max_nnz_per_row, size_type stride,
-                 const ValueType *ell_val, const IndexType *ell_col,
-                 const ValueType *coo_val, const IndexType *coo_col,
-                 const IndexType *coo_offset, IndexType *result_row_ptrs,
-                 IndexType *result_col_idxs, ValueType *result_values)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                fill_in_csr(num_rows, max_nnz_per_row, stride, ell_val, ell_col,
-                            coo_val, coo_col, coo_offset, result_row_ptrs,
-                            result_col_idxs, result_values, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(fill_in_csr, fill_in_csr);
 
 
 template <typename ValueType1, typename ValueType2>
@@ -215,17 +199,7 @@ void add(size_type num, ValueType1 *__restrict__ val1,
     }
 }
 
-template <typename ValueType1, typename ValueType2>
-void add(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-         sycl::queue *queue, size_type num, ValueType1 *val1,
-         const ValueType2 *val2)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block),
-            [=](sycl::nd_item<3> item_ct1) { add(num, val1, val2, item_ct1); });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(add, add);
 
 
 }  // namespace kernel
diff --git a/dpcpp/matrix/sellp_kernels.dp.cpp b/dpcpp/matrix/sellp_kernels.dp.cpp
index f1d6ba433d9..cd20591f48d 100644
--- a/dpcpp/matrix/sellp_kernels.dp.cpp
+++ b/dpcpp/matrix/sellp_kernels.dp.cpp
@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/prefix_sum.hpp"
 #include "dpcpp/base/config.hpp"
 #include "dpcpp/base/dim3.dp.hpp"
+#include "dpcpp/base/helper.hpp"
 #include "dpcpp/components/cooperative_groups.dp.hpp"
 #include "dpcpp/components/reduction.dp.hpp"
 #include "dpcpp/components/thread_ids.dp.hpp"
@@ -62,7 +63,7 @@ namespace dpcpp {
 namespace sellp {
 
 
-constexpr auto default_block_size = 256;
+constexpr int default_block_size = 256;
 
 
 namespace {
@@ -95,22 +96,7 @@ void spmv_kernel(size_type num_rows, size_type num_right_hand_sides,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void spmv_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                 sycl::queue *queue, size_type num_rows,
-                 size_type num_right_hand_sides, size_type b_stride,
-                 size_type c_stride, const size_type *slice_lengths,
-                 const size_type *slice_sets, const ValueType *a,
-                 const IndexType *col, const ValueType *b, ValueType *c)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                spmv_kernel(num_rows, num_right_hand_sides, b_stride, c_stride,
-                            slice_lengths, slice_sets, a, col, b, c, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(spmv_kernel, spmv_kernel);
 
 
 template <typename ValueType, typename IndexType>
@@ -143,26 +129,7 @@ void advanced_spmv_kernel(size_type num_rows, size_type num_right_hand_sides,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void advanced_spmv_kernel(dim3 grid, dim3 block,
-                          size_type dynamic_shared_memory, sycl::queue *queue,
-                          size_type num_rows, size_type num_right_hand_sides,
-                          size_type b_stride, size_type c_stride,
-                          const size_type *slice_lengths,
-                          const size_type *slice_sets, const ValueType *alpha,
-                          const ValueType *a, const IndexType *col,
-                          const ValueType *b, const ValueType *beta,
-                          ValueType *c)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                advanced_spmv_kernel(num_rows, num_right_hand_sides, b_stride,
-                                     c_stride, slice_lengths, slice_sets, alpha,
-                                     a, col, b, beta, c, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(advanced_spmv_kernel, advanced_spmv_kernel);
 
 
 }  // namespace
@@ -187,20 +154,7 @@ void initialize_zero_dense(size_type num_rows, size_type num_cols,
     }
 }
 
-template <typename ValueType>
-void initialize_zero_dense(dim3 grid, dim3 block,
-                           size_type dynamic_shared_memory, sycl::queue *queue,
-                           size_type num_rows, size_type num_cols,
-                           size_type stride, ValueType *result)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             initialize_zero_dense(num_rows, num_cols, stride,
-                                                   result, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(initialize_zero_dense, initialize_zero_dense);
 
 
 template <unsigned int threads_per_row, typename ValueType, typename IndexType>
@@ -282,20 +236,7 @@ void count_nnz_per_row(size_type num_rows, size_type slice_size,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void count_nnz_per_row(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                       sycl::queue *queue, size_type num_rows,
-                       size_type slice_size, const size_type *slice_sets,
-                       const ValueType *values, IndexType *result)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(sycl_nd_range(grid, block),
-                         [=](sycl::nd_item<3> item_ct1) {
-                             count_nnz_per_row(num_rows, slice_size, slice_sets,
-                                               values, result, item_ct1);
-                         });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(count_nnz_per_row, count_nnz_per_row);
 
 
 template <typename ValueType, typename IndexType>
@@ -327,23 +268,7 @@ void fill_in_csr(size_type num_rows, size_type slice_size,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void fill_in_csr(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                 sycl::queue *queue, size_type num_rows, size_type slice_size,
-                 const size_type *source_slice_sets,
-                 const IndexType *source_col_idxs,
-                 const ValueType *source_values, IndexType *result_row_ptrs,
-                 IndexType *result_col_idxs, ValueType *result_values)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                fill_in_csr(num_rows, slice_size, source_slice_sets,
-                            source_col_idxs, source_values, result_row_ptrs,
-                            result_col_idxs, result_values, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(fill_in_csr, fill_in_csr);
 
 
 template <typename ValueType, typename IndexType>
@@ -379,21 +304,7 @@ void extract_diagonal(size_type diag_size, size_type slice_size,
     }
 }
 
-template <typename ValueType, typename IndexType>
-void extract_diagonal(dim3 grid, dim3 block, size_type dynamic_shared_memory,
-                      sycl::queue *queue, size_type diag_size,
-                      size_type slice_size, const size_type *orig_slice_sets,
-                      const ValueType *orig_values,
-                      const IndexType *orig_col_idxs, ValueType *diag)
-{
-    queue->submit([&](sycl::handler &cgh) {
-        cgh.parallel_for(
-            sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                extract_diagonal(diag_size, slice_size, orig_slice_sets,
-                                 orig_values, orig_col_idxs, diag, item_ct1);
-            });
-    });
-}
+GKO_ENABLE_DEFAULT_HOST(extract_diagonal, extract_diagonal);
 
 
 }  // namespace kernel
diff --git a/dpcpp/test/matrix/CMakeLists.txt b/dpcpp/test/matrix/CMakeLists.txt
index f36f9c5606e..5be841b3d00 100644
--- a/dpcpp/test/matrix/CMakeLists.txt
+++ b/dpcpp/test/matrix/CMakeLists.txt
@@ -2,6 +2,7 @@ ginkgo_create_test(coo_kernels)
 ginkgo_create_test(csr_kernels)
 ginkgo_create_test(dense_kernels)
 ginkgo_create_test(diagonal_kernels)
+ginkgo_create_test(ell_kernels)
 ginkgo_create_test(fbcsr_kernels)
 ginkgo_create_test(hybrid_kernels)
 ginkgo_create_test(sellp_kernels)
diff --git a/dpcpp/test/matrix/sellp_kernels.cpp b/dpcpp/test/matrix/sellp_kernels.cpp
index 6e385ab13fe..91e7ae9d5c8 100644
--- a/dpcpp/test/matrix/sellp_kernels.cpp
+++ b/dpcpp/test/matrix/sellp_kernels.cpp
@@ -86,39 +86,15 @@ class Sellp : public ::testing::Test {
             std::normal_distribution<vtype>(-1.0, 1.0), rand_engine, ref);
     }
 
-    void set_up_apply_vector(
-        int slice_size = gko::matrix::default_slice_size,
-        int stride_factor = gko::matrix::default_stride_factor,
-        int total_cols = 0)
-    {
-        mtx = Mtx::create(ref);
-        mtx->copy_from(gen_mtx(532, 231));
-        expected = gen_mtx(532, 1);
-        y = gen_mtx(231, 1);
-        alpha = gko::initialize<Vec>({2.0}, ref);
-        beta = gko::initialize<Vec>({-1.0}, ref);
-        dmtx = Mtx::create(dpcpp);
-        dmtx->copy_from(mtx.get());
-        dresult = Vec::create(dpcpp);
-        dresult->copy_from(expected.get());
-        dy = Vec::create(dpcpp);
-        dy->copy_from(y.get());
-        dalpha = Vec::create(dpcpp);
-        dalpha->copy_from(alpha.get());
-        dbeta = Vec::create(dpcpp);
-        dbeta->copy_from(beta.get());
-    }
-
     void set_up_apply_matrix(
-        int slice_size = gko::matrix::default_slice_size,
-        int stride_factor = gko::matrix::default_stride_factor,
-        int total_cols = 0)
+        int total_cols = 1, int slice_size = gko::matrix::default_slice_size,
+        int stride_factor = gko::matrix::default_stride_factor)
     {
         mtx = Mtx::create(ref);
         mtx->copy_from(gen_mtx(532, 231));
         empty = Mtx::create(ref);
-        expected = gen_mtx(532, 64);
-        y = gen_mtx(231, 64);
+        expected = gen_mtx(532, total_cols);
+        y = gen_mtx(231, total_cols);
         alpha = gko::initialize<Vec>({2.0}, ref);
         beta = gko::initialize<Vec>({-1.0}, ref);
         dmtx = Mtx::create(dpcpp);
@@ -157,7 +133,7 @@ class Sellp : public ::testing::Test {
 
 TEST_F(Sellp, SimpleApplyIsEquivalentToRef)
 {
-    set_up_apply_vector();
+    set_up_apply_matrix();
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -170,7 +146,7 @@ TEST_F(Sellp, SimpleApplyIsEquivalentToRef)
 
 TEST_F(Sellp, AdvancedApplyIsEquivalentToRef)
 {
-    set_up_apply_vector();
+    set_up_apply_matrix();
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
@@ -183,7 +159,7 @@ TEST_F(Sellp, AdvancedApplyIsEquivalentToRef)
 
 TEST_F(Sellp, SimpleApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
 {
-    set_up_apply_vector(32, 2);
+    set_up_apply_matrix(1, 32, 2);
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -196,7 +172,7 @@ TEST_F(Sellp, SimpleApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
 
 TEST_F(Sellp, AdvancedApplyWithSliceSizeAndStrideFActorIsEquivalentToRef)
 {
-    set_up_apply_vector(32, 2);
+    set_up_apply_matrix(1, 32, 2);
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
@@ -209,7 +185,8 @@ TEST_F(Sellp, AdvancedApplyWithSliceSizeAndStrideFActorIsEquivalentToRef)
 
 TEST_F(Sellp, SimpleApplyMultipleRHSIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -222,7 +199,8 @@ TEST_F(Sellp, SimpleApplyMultipleRHSIsEquivalentToRef)
 
 TEST_F(Sellp, AdvancedApplyMultipleRHSIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
@@ -263,7 +241,8 @@ TEST_F(Sellp,
 
 TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(dpcpp);
     dcomplex_b->copy_from(complex_b.get());
@@ -280,7 +259,8 @@ TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 
 TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(dpcpp);
     dcomplex_b->copy_from(complex_b.get());
@@ -297,7 +277,8 @@ TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 
 TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
 
     auto dense_mtx = gko::matrix::Dense<vtype>::create(ref);
     auto ddense_mtx = gko::matrix::Dense<vtype>::create(dpcpp);
@@ -311,7 +292,8 @@ TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 
 TEST_F(Sellp, ConvertToCsrIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
 
     auto csr_mtx = gko::matrix::Csr<vtype>::create(ref);
     auto dcsr_mtx = gko::matrix::Csr<vtype>::create(dpcpp);
@@ -325,7 +307,8 @@ TEST_F(Sellp, ConvertToCsrIsEquivalentToRef)
 
 TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
 
     auto dense_mtx = gko::matrix::Dense<vtype>::create(ref);
     auto ddense_mtx = gko::matrix::Dense<vtype>::create(dpcpp);
@@ -339,7 +322,8 @@ TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 
 TEST_F(Sellp, ConvertEmptyToCsrIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
 
     auto csr_mtx = gko::matrix::Csr<vtype>::create(ref);
     auto dcsr_mtx = gko::matrix::Csr<vtype>::create(dpcpp);
@@ -353,7 +337,8 @@ TEST_F(Sellp, ConvertEmptyToCsrIsEquivalentToRef)
 
 TEST_F(Sellp, CountNonzerosIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
 
     gko::size_type nnz;
     gko::size_type dnnz;
@@ -367,7 +352,8 @@ TEST_F(Sellp, CountNonzerosIsEquivalentToRef)
 
 TEST_F(Sellp, ExtractDiagonalIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
 
     auto diag = mtx->extract_diagonal();
     auto ddiag = dmtx->extract_diagonal();
@@ -378,7 +364,7 @@ TEST_F(Sellp, ExtractDiagonalIsEquivalentToRef)
 
 TEST_F(Sellp, ExtractDiagonalWithSliceSizeAndStrideFactorIsEquivalentToRef)
 {
-    set_up_apply_matrix(32, 2);
+    set_up_apply_matrix(64, 32, 2);
 
     auto diag = mtx->extract_diagonal();
     auto ddiag = dmtx->extract_diagonal();
@@ -389,7 +375,7 @@ TEST_F(Sellp, ExtractDiagonalWithSliceSizeAndStrideFactorIsEquivalentToRef)
 
 TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
-    set_up_apply_matrix(32, 2);
+    set_up_apply_matrix(64, 32, 2);
 
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
@@ -400,7 +386,7 @@ TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
 
 TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
-    set_up_apply_matrix(32, 2);
+    set_up_apply_matrix(64, 32, 2);
 
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
diff --git a/examples/custom-matrix-format/stencil_kernel.cu b/examples/custom-matrix-format/stencil_kernel.cu
index 3f926e31ecf..47cd2540def 100644
--- a/examples/custom-matrix-format/stencil_kernel.cu
+++ b/examples/custom-matrix-format/stencil_kernel.cu
@@ -75,7 +75,7 @@ template <typename ValueType>
 void stencil_kernel(std::size_t size, const ValueType *coefs,
                     const ValueType *b, ValueType *x)
 {
-    constexpr auto block_size = 512;
+    constexpr int block_size = 512;
     const auto grid_size = (size + block_size - 1) / block_size;
     stencil_kernel_impl<<<grid_size, block_size>>>(size, coefs, b, x);
 }
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 61870d03aa7..dc63148940c 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -186,7 +186,9 @@ set(GINKGO_HIP_SOURCES
     stop/criterion_kernels.hip.cpp
     stop/residual_norm_kernels.hip.cpp
     ../common/components/precision_conversion.cpp
+    ../common/matrix/coo_kernels.cpp
     ../common/matrix/dense_kernels.cpp
+    ../common/matrix/diagonal_kernels.cpp
     ../common/solver/bicg_kernels.cpp
     ../common/solver/bicgstab_kernels.cpp
     ../common/solver/cg_kernels.cpp
diff --git a/hip/factorization/par_ic_kernels.hip.cpp b/hip/factorization/par_ic_kernels.hip.cpp
index 5365083d63c..4e242119f9e 100644
--- a/hip/factorization/par_ic_kernels.hip.cpp
+++ b/hip/factorization/par_ic_kernels.hip.cpp
@@ -54,7 +54,7 @@ namespace hip {
 namespace par_ic_factorization {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 // subwarp sizes for all warp-parallel kernels (sweep)
diff --git a/hip/factorization/par_ict_kernels.hip.cpp b/hip/factorization/par_ict_kernels.hip.cpp
index 456f0677e37..15c175166cc 100644
--- a/hip/factorization/par_ict_kernels.hip.cpp
+++ b/hip/factorization/par_ict_kernels.hip.cpp
@@ -68,7 +68,7 @@ namespace hip {
 namespace par_ict_factorization {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 // subwarp sizes for all warp-parallel kernels (filter, add_candidates)
diff --git a/hip/factorization/par_ilut_filter_kernel.hip.cpp b/hip/factorization/par_ilut_filter_kernel.hip.cpp
index bb9d784c6d1..4898d6ec149 100644
--- a/hip/factorization/par_ilut_filter_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_filter_kernel.hip.cpp
@@ -67,7 +67,7 @@ namespace hip {
 namespace par_ilut_factorization {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 // subwarp sizes for filter kernels
diff --git a/hip/factorization/par_ilut_select_common.hip.hpp b/hip/factorization/par_ilut_select_common.hip.hpp
index fc3a5b9bb88..38f653abf6e 100644
--- a/hip/factorization/par_ilut_select_common.hip.hpp
+++ b/hip/factorization/par_ilut_select_common.hip.hpp
@@ -45,8 +45,8 @@ namespace hip {
 namespace par_ilut_factorization {
 
 
-constexpr auto default_block_size = 512;
-constexpr auto items_per_thread = 16;
+constexpr int default_block_size = 512;
+constexpr int items_per_thread = 16;
 
 
 template <typename ValueType, typename IndexType>
diff --git a/hip/factorization/par_ilut_spgeam_kernel.hip.cpp b/hip/factorization/par_ilut_spgeam_kernel.hip.cpp
index b4ccb7f2904..b18b6fface6 100644
--- a/hip/factorization/par_ilut_spgeam_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_spgeam_kernel.hip.cpp
@@ -68,7 +68,7 @@ namespace hip {
 namespace par_ilut_factorization {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 // subwarp sizes for add_candidates kernels
diff --git a/hip/factorization/par_ilut_sweep_kernel.hip.cpp b/hip/factorization/par_ilut_sweep_kernel.hip.cpp
index 9c803cbfb5b..712dc038937 100644
--- a/hip/factorization/par_ilut_sweep_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_sweep_kernel.hip.cpp
@@ -68,7 +68,7 @@ namespace hip {
 namespace par_ilut_factorization {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 // subwarp sizes for all warp-parallel kernels (filter, add_candidates)
diff --git a/hip/matrix/coo_kernels.hip.cpp b/hip/matrix/coo_kernels.hip.cpp
index c3c1a654f66..06b37134a1f 100644
--- a/hip/matrix/coo_kernels.hip.cpp
+++ b/hip/matrix/coo_kernels.hip.cpp
@@ -255,30 +255,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_CONVERT_TO_DENSE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void extract_diagonal(std::shared_ptr<const HipExecutor> exec,
-                      const matrix::Coo<ValueType, IndexType> *orig,
-                      matrix::Diagonal<ValueType> *diag)
-{
-    const auto nnz = orig->get_num_stored_elements();
-    const auto diag_size = diag->get_size()[0];
-    const auto num_blocks = ceildiv(nnz, default_block_size);
-
-    const auto orig_values = orig->get_const_values();
-    const auto orig_row_idxs = orig->get_const_row_idxs();
-    const auto orig_col_idxs = orig->get_const_col_idxs();
-    auto diag_values = diag->get_values();
-
-    hipLaunchKernelGGL(kernel::extract_diagonal, dim3(num_blocks),
-                       dim3(default_block_size), 0, 0, nnz,
-                       as_hip_type(orig_values), as_hip_type(orig_row_idxs),
-                       as_hip_type(orig_col_idxs), as_hip_type(diag_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_COO_EXTRACT_DIAGONAL_KERNEL);
-
-
 }  // namespace coo
 }  // namespace hip
 }  // namespace kernels
diff --git a/hip/matrix/csr_kernels.hip.cpp b/hip/matrix/csr_kernels.hip.cpp
index d296ece80e5..b58ef80fea3 100644
--- a/hip/matrix/csr_kernels.hip.cpp
+++ b/hip/matrix/csr_kernels.hip.cpp
@@ -83,7 +83,6 @@ namespace csr {
 constexpr int default_block_size = 512;
 constexpr int warps_in_block = 4;
 constexpr int spmv_block_size = warps_in_block * config::warp_size;
-constexpr int wsize = config::warp_size;
 constexpr int classical_overweight = 32;
 
 
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index 4aebe4b9376..f437bc6dddc 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -67,7 +67,7 @@ namespace hip {
 namespace dense {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 #include "common/matrix/dense_kernels.hpp.inc"
@@ -138,8 +138,8 @@ void compute_dot(std::shared_ptr<const HipExecutor> exec,
         // TODO: these are tuning parameters obtained experimentally, once
         // we decide how to handle this uniformly, they should be modified
         // appropriately
-        constexpr auto work_per_thread = 32;
-        constexpr auto block_size = 1024;
+        constexpr int work_per_thread = 32;
+        constexpr int block_size = 1024;
 
         constexpr auto work_per_block = work_per_thread * block_size;
         const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
@@ -185,8 +185,8 @@ void compute_conj_dot(std::shared_ptr<const HipExecutor> exec,
         // TODO: these are tuning parameters obtained experimentally, once
         // we decide how to handle this uniformly, they should be modified
         // appropriately
-        constexpr auto work_per_thread = 32;
-        constexpr auto block_size = 1024;
+        constexpr int work_per_thread = 32;
+        constexpr int block_size = 1024;
 
         constexpr auto work_per_block = work_per_thread * block_size;
         const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
@@ -230,8 +230,8 @@ void compute_norm2(std::shared_ptr<const HipExecutor> exec,
         // TODO: these are tuning parameters obtained experimentally, once
         // we decide how to handle this uniformly, they should be modified
         // appropriately
-        constexpr auto work_per_thread = 32;
-        constexpr auto block_size = 1024;
+        constexpr int work_per_thread = 32;
+        constexpr int block_size = 1024;
 
         constexpr auto work_per_block = work_per_thread * block_size;
         const dim3 grid_dim = ceildiv(x->get_size()[0], work_per_block);
diff --git a/hip/matrix/diagonal_kernels.hip.cpp b/hip/matrix/diagonal_kernels.hip.cpp
index ea965ef3e46..8718fe34a91 100644
--- a/hip/matrix/diagonal_kernels.hip.cpp
+++ b/hip/matrix/diagonal_kernels.hip.cpp
@@ -57,65 +57,12 @@ namespace hip {
 namespace diagonal {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 #include "common/matrix/diagonal_kernels.hpp.inc"
 
 
-template <typename ValueType>
-void apply_to_dense(std::shared_ptr<const HipExecutor> exec,
-                    const matrix::Diagonal<ValueType> *a,
-                    const matrix::Dense<ValueType> *b,
-                    matrix::Dense<ValueType> *c)
-{
-    const auto b_size = b->get_size();
-    const auto num_rows = b_size[0];
-    const auto num_cols = b_size[1];
-    const auto b_stride = b->get_stride();
-    const auto c_stride = c->get_stride();
-    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
-
-    const auto diag_values = a->get_const_values();
-    const auto b_values = b->get_const_values();
-    auto c_values = c->get_values();
-
-    hipLaunchKernelGGL(kernel::apply_to_dense, dim3(grid_dim),
-                       dim3(default_block_size), 0, 0, num_rows, num_cols,
-                       as_hip_type(diag_values), b_stride,
-                       as_hip_type(b_values), c_stride, as_hip_type(c_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_APPLY_TO_DENSE_KERNEL);
-
-
-template <typename ValueType>
-void right_apply_to_dense(std::shared_ptr<const HipExecutor> exec,
-                          const matrix::Diagonal<ValueType> *a,
-                          const matrix::Dense<ValueType> *b,
-                          matrix::Dense<ValueType> *c)
-{
-    const auto b_size = b->get_size();
-    const auto num_rows = b_size[0];
-    const auto num_cols = b_size[1];
-    const auto b_stride = b->get_stride();
-    const auto c_stride = c->get_stride();
-    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
-
-    const auto diag_values = a->get_const_values();
-    const auto b_values = b->get_const_values();
-    auto c_values = c->get_values();
-
-    hipLaunchKernelGGL(kernel::right_apply_to_dense, dim3(grid_dim),
-                       dim3(default_block_size), 0, 0, num_rows, num_cols,
-                       as_hip_type(diag_values), b_stride,
-                       as_hip_type(b_values), c_stride, as_hip_type(c_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_DENSE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void apply_to_csr(std::shared_ptr<const HipExecutor> exec,
                   const matrix::Diagonal<ValueType> *a,
@@ -139,68 +86,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DIAGONAL_APPLY_TO_CSR_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void right_apply_to_csr(std::shared_ptr<const HipExecutor> exec,
-                        const matrix::Diagonal<ValueType> *a,
-                        const matrix::Csr<ValueType, IndexType> *b,
-                        matrix::Csr<ValueType, IndexType> *c)
-{
-    const auto num_nnz = b->get_num_stored_elements();
-    const auto diag_values = a->get_const_values();
-    c->copy_from(b);
-    auto csr_values = c->get_values();
-    const auto csr_col_idxs = c->get_const_col_idxs();
-
-    const auto grid_dim = ceildiv(num_nnz, default_block_size);
-    hipLaunchKernelGGL(kernel::right_apply_to_csr, grid_dim, default_block_size,
-                       0, 0, num_nnz, as_hip_type(diag_values),
-                       as_hip_type(csr_col_idxs), as_hip_type(csr_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_CSR_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void convert_to_csr(std::shared_ptr<const HipExecutor> exec,
-                    const matrix::Diagonal<ValueType> *source,
-                    matrix::Csr<ValueType, IndexType> *result)
-{
-    const auto size = source->get_size()[0];
-    const auto grid_dim = ceildiv(size, default_block_size);
-
-    const auto diag_values = source->get_const_values();
-    auto row_ptrs = result->get_row_ptrs();
-    auto col_idxs = result->get_col_idxs();
-    auto csr_values = result->get_values();
-
-    hipLaunchKernelGGL(kernel::convert_to_csr, grid_dim, default_block_size, 0,
-                       0, size, as_hip_type(diag_values), as_hip_type(row_ptrs),
-                       as_hip_type(col_idxs), as_hip_type(csr_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DIAGONAL_CONVERT_TO_CSR_KERNEL);
-
-
-template <typename ValueType>
-void conj_transpose(std::shared_ptr<const HipExecutor> exec,
-                    const matrix::Diagonal<ValueType> *orig,
-                    matrix::Diagonal<ValueType> *trans)
-{
-    const auto size = orig->get_size()[0];
-    const auto grid_dim = ceildiv(size, default_block_size);
-    const auto orig_values = orig->get_const_values();
-    auto trans_values = trans->get_values();
-
-    hipLaunchKernelGGL(kernel::conj_transpose, grid_dim, default_block_size, 0,
-                       0, size, as_hip_type(orig_values),
-                       as_hip_type(trans_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_CONJ_TRANSPOSE_KERNEL);
-
-
 }  // namespace diagonal
 }  // namespace hip
 }  // namespace kernels
diff --git a/hip/matrix/sellp_kernels.hip.cpp b/hip/matrix/sellp_kernels.hip.cpp
index 10178606a2a..3f9b583404f 100644
--- a/hip/matrix/sellp_kernels.hip.cpp
+++ b/hip/matrix/sellp_kernels.hip.cpp
@@ -62,7 +62,7 @@ namespace hip {
 namespace sellp {
 
 
-constexpr auto default_block_size = 512;
+constexpr int default_block_size = 512;
 
 
 #include "common/matrix/sellp_kernels.hpp.inc"
diff --git a/hip/test/components/sorting_kernels.hip.cpp b/hip/test/components/sorting_kernels.hip.cpp
index 40e1aa473b1..ed65b1ea798 100644
--- a/hip/test/components/sorting_kernels.hip.cpp
+++ b/hip/test/components/sorting_kernels.hip.cpp
@@ -54,8 +54,8 @@ using gko::kernels::hip::bitonic_sort;
 using gko::kernels::hip::config;
 
 
-constexpr auto num_elements = 2048;
-constexpr auto num_local = 4;
+constexpr int num_elements = 2048;
+constexpr int num_local = 4;
 constexpr auto num_threads = num_elements / num_local;
 
 
diff --git a/hip/test/matrix/sellp_kernels.hip.cpp b/hip/test/matrix/sellp_kernels.hip.cpp
index 869930a539e..89dee5466e4 100644
--- a/hip/test/matrix/sellp_kernels.hip.cpp
+++ b/hip/test/matrix/sellp_kernels.hip.cpp
@@ -84,39 +84,15 @@ class Sellp : public ::testing::Test {
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
     }
 
-    void set_up_apply_vector(
-        int slice_size = gko::matrix::default_slice_size,
-        int stride_factor = gko::matrix::default_stride_factor,
-        int total_cols = 0)
-    {
-        mtx = Mtx::create(ref);
-        mtx->copy_from(gen_mtx(532, 231));
-        expected = gen_mtx(532, 1);
-        y = gen_mtx(231, 1);
-        alpha = gko::initialize<Vec>({2.0}, ref);
-        beta = gko::initialize<Vec>({-1.0}, ref);
-        dmtx = Mtx::create(hip);
-        dmtx->copy_from(mtx.get());
-        dresult = Vec::create(hip);
-        dresult->copy_from(expected.get());
-        dy = Vec::create(hip);
-        dy->copy_from(y.get());
-        dalpha = Vec::create(hip);
-        dalpha->copy_from(alpha.get());
-        dbeta = Vec::create(hip);
-        dbeta->copy_from(beta.get());
-    }
-
     void set_up_apply_matrix(
-        int slice_size = gko::matrix::default_slice_size,
-        int stride_factor = gko::matrix::default_stride_factor,
-        int total_cols = 0)
+        int total_cols = 1, int slice_size = gko::matrix::default_slice_size,
+        int stride_factor = gko::matrix::default_stride_factor)
     {
         mtx = Mtx::create(ref);
         mtx->copy_from(gen_mtx(532, 231));
         empty = Mtx::create(ref);
-        expected = gen_mtx(532, 64);
-        y = gen_mtx(231, 64);
+        expected = gen_mtx(532, total_cols);
+        y = gen_mtx(231, total_cols);
         alpha = gko::initialize<Vec>({2.0}, ref);
         beta = gko::initialize<Vec>({-1.0}, ref);
         dmtx = Mtx::create(hip);
@@ -155,7 +131,7 @@ class Sellp : public ::testing::Test {
 
 TEST_F(Sellp, SimpleApplyIsEquivalentToRef)
 {
-    set_up_apply_vector();
+    set_up_apply_matrix();
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -168,7 +144,7 @@ TEST_F(Sellp, SimpleApplyIsEquivalentToRef)
 
 TEST_F(Sellp, AdvancedApplyIsEquivalentToRef)
 {
-    set_up_apply_vector();
+    set_up_apply_matrix();
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
@@ -181,7 +157,7 @@ TEST_F(Sellp, AdvancedApplyIsEquivalentToRef)
 
 TEST_F(Sellp, SimpleApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
 {
-    set_up_apply_vector(32, 2);
+    set_up_apply_matrix(1, 32, 2);
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -194,7 +170,7 @@ TEST_F(Sellp, SimpleApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
 
 TEST_F(Sellp, AdvancedApplyWithSliceSizeAndStrideFActorIsEquivalentToRef)
 {
-    set_up_apply_vector(32, 2);
+    set_up_apply_matrix(1, 32, 2);
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
@@ -207,7 +183,8 @@ TEST_F(Sellp, AdvancedApplyWithSliceSizeAndStrideFActorIsEquivalentToRef)
 
 TEST_F(Sellp, SimpleApplyMultipleRHSIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
@@ -220,7 +197,8 @@ TEST_F(Sellp, SimpleApplyMultipleRHSIsEquivalentToRef)
 
 TEST_F(Sellp, AdvancedApplyMultipleRHSIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
@@ -261,7 +239,8 @@ TEST_F(Sellp,
 
 TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(hip);
     dcomplex_b->copy_from(complex_b.get());
@@ -278,7 +257,8 @@ TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 
 TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
+
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(hip);
     dcomplex_b->copy_from(complex_b.get());
@@ -295,7 +275,7 @@ TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 
 TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
 
     auto dense_mtx = gko::matrix::Dense<>::create(ref);
     auto ddense_mtx = gko::matrix::Dense<>::create(hip);
@@ -309,7 +289,7 @@ TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 
 TEST_F(Sellp, ConvertToCsrIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
 
     auto csr_mtx = gko::matrix::Csr<>::create(ref);
     auto dcsr_mtx = gko::matrix::Csr<>::create(hip);
@@ -323,7 +303,7 @@ TEST_F(Sellp, ConvertToCsrIsEquivalentToRef)
 
 TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
 
     auto dense_mtx = gko::matrix::Dense<>::create(ref);
     auto ddense_mtx = gko::matrix::Dense<>::create(hip);
@@ -337,7 +317,7 @@ TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 
 TEST_F(Sellp, ConvertEmptyToCsrIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
 
     auto csr_mtx = gko::matrix::Csr<>::create(ref);
     auto dcsr_mtx = gko::matrix::Csr<>::create(hip);
@@ -351,7 +331,7 @@ TEST_F(Sellp, ConvertEmptyToCsrIsEquivalentToRef)
 
 TEST_F(Sellp, CountNonzerosIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
 
     gko::size_type nnz;
     gko::size_type dnnz;
@@ -365,7 +345,7 @@ TEST_F(Sellp, CountNonzerosIsEquivalentToRef)
 
 TEST_F(Sellp, ExtractDiagonalIsEquivalentToRef)
 {
-    set_up_apply_matrix();
+    set_up_apply_matrix(64);
 
     auto diag = mtx->extract_diagonal();
     auto ddiag = dmtx->extract_diagonal();
@@ -376,7 +356,7 @@ TEST_F(Sellp, ExtractDiagonalIsEquivalentToRef)
 
 TEST_F(Sellp, ExtractDiagonalWithSliceSizeAndStrideFactorIsEquivalentToRef)
 {
-    set_up_apply_matrix(32, 2);
+    set_up_apply_matrix(64, 32, 2);
 
     auto diag = mtx->extract_diagonal();
     auto ddiag = dmtx->extract_diagonal();
@@ -387,7 +367,7 @@ TEST_F(Sellp, ExtractDiagonalWithSliceSizeAndStrideFactorIsEquivalentToRef)
 
 TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
-    set_up_apply_matrix(32, 2);
+    set_up_apply_matrix(64, 32, 2);
 
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
@@ -398,7 +378,7 @@ TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
 
 TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
-    set_up_apply_matrix(32, 2);
+    set_up_apply_matrix(64, 32, 2);
 
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index c58709bc904..a9e38d8f6e9 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -35,7 +35,9 @@ target_sources(ginkgo_omp
     stop/criterion_kernels.cpp
     stop/residual_norm_kernels.cpp
     ../common/components/precision_conversion.cpp
+    ../common/matrix/coo_kernels.cpp
     ../common/matrix/dense_kernels.cpp
+    ../common/matrix/diagonal_kernels.cpp
     ../common/solver/bicg_kernels.cpp
     ../common/solver/bicgstab_kernels.cpp
     ../common/solver/cg_kernels.cpp
diff --git a/omp/matrix/coo_kernels.cpp b/omp/matrix/coo_kernels.cpp
index 1f9055dc129..303415405b3 100644
--- a/omp/matrix/coo_kernels.cpp
+++ b/omp/matrix/coo_kernels.cpp
@@ -255,30 +255,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_CONVERT_TO_DENSE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void extract_diagonal(std::shared_ptr<const OmpExecutor> exec,
-                      const matrix::Coo<ValueType, IndexType> *orig,
-                      matrix::Diagonal<ValueType> *diag)
-{
-    const auto row_idxs = orig->get_const_row_idxs();
-    const auto col_idxs = orig->get_const_col_idxs();
-    const auto values = orig->get_const_values();
-    const auto diag_size = diag->get_size()[0];
-    const auto nnz = orig->get_num_stored_elements();
-    auto diag_values = diag->get_values();
-
-#pragma omp parallel for
-    for (size_type idx = 0; idx < nnz; idx++) {
-        if (row_idxs[idx] == col_idxs[idx]) {
-            diag_values[row_idxs[idx]] = values[idx];
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_COO_EXTRACT_DIAGONAL_KERNEL);
-
-
 }  // namespace coo
 }  // namespace omp
 }  // namespace kernels
diff --git a/omp/matrix/diagonal_kernels.cpp b/omp/matrix/diagonal_kernels.cpp
index 35985dc0e18..4f5c814e7fd 100644
--- a/omp/matrix/diagonal_kernels.cpp
+++ b/omp/matrix/diagonal_kernels.cpp
@@ -51,44 +51,6 @@ namespace omp {
 namespace diagonal {
 
 
-template <typename ValueType>
-void apply_to_dense(std::shared_ptr<const OmpExecutor> exec,
-                    const matrix::Diagonal<ValueType> *a,
-                    const matrix::Dense<ValueType> *b,
-                    matrix::Dense<ValueType> *c)
-{
-    const auto diag_values = a->get_const_values();
-#pragma omp parallel for
-    for (size_type row = 0; row < a->get_size()[0]; row++) {
-        const auto scal = diag_values[row];
-        for (size_type col = 0; col < b->get_size()[1]; col++) {
-            c->at(row, col) = b->at(row, col) * scal;
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_APPLY_TO_DENSE_KERNEL);
-
-
-template <typename ValueType>
-void right_apply_to_dense(std::shared_ptr<const OmpExecutor> exec,
-                          const matrix::Diagonal<ValueType> *a,
-                          const matrix::Dense<ValueType> *b,
-                          matrix::Dense<ValueType> *c)
-{
-    const auto diag_values = a->get_const_values();
-#pragma omp parallel for
-    for (size_type row = 0; row < b->get_size()[0]; row++) {
-        for (size_type col = 0; col < a->get_size()[1]; col++) {
-            c->at(row, col) = b->at(row, col) * diag_values[col];
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_DENSE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void apply_to_csr(std::shared_ptr<const OmpExecutor> exec,
                   const matrix::Diagonal<ValueType> *a,
@@ -114,73 +76,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_DIAGONAL_APPLY_TO_CSR_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void right_apply_to_csr(std::shared_ptr<const OmpExecutor> exec,
-                        const matrix::Diagonal<ValueType> *a,
-                        const matrix::Csr<ValueType, IndexType> *b,
-                        matrix::Csr<ValueType, IndexType> *c)
-{
-    const auto diag_values = a->get_const_values();
-    c->copy_from(b);
-    auto csr_values = c->get_values();
-    const auto csr_row_ptrs = c->get_const_row_ptrs();
-    const auto csr_col_idxs = c->get_const_col_idxs();
-
-#pragma omp parallel for
-    for (size_type row = 0; row < c->get_size()[0]; row++) {
-        for (size_type idx = csr_row_ptrs[row]; idx < csr_row_ptrs[row + 1];
-             idx++) {
-            csr_values[idx] *= diag_values[csr_col_idxs[idx]];
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DIAGONAL_RIGHT_APPLY_TO_CSR_KERNEL);
-
-
-template <typename ValueType, typename IndexType>
-void convert_to_csr(std::shared_ptr<const OmpExecutor> exec,
-                    const matrix::Diagonal<ValueType> *source,
-                    matrix::Csr<ValueType, IndexType> *result)
-{
-    const auto size = source->get_size()[0];
-    auto row_ptrs = result->get_row_ptrs();
-    auto col_idxs = result->get_col_idxs();
-    auto csr_values = result->get_values();
-    const auto diag_values = source->get_const_values();
-
-#pragma omp parallel for
-    for (size_type i = 0; i < size; i++) {
-        row_ptrs[i] = i;
-        col_idxs[i] = i;
-        csr_values[i] = diag_values[i];
-    }
-    row_ptrs[size] = size;
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_DIAGONAL_CONVERT_TO_CSR_KERNEL);
-
-
-template <typename ValueType>
-void conj_transpose(std::shared_ptr<const OmpExecutor> exec,
-                    const matrix::Diagonal<ValueType> *orig,
-                    matrix::Diagonal<ValueType> *trans)
-{
-    const auto size = orig->get_size()[0];
-    const auto orig_values = orig->get_const_values();
-    auto trans_values = trans->get_values();
-
-#pragma omp parallel for
-    for (size_type i = 0; i < size; i++) {
-        trans_values[i] = conj(orig_values[i]);
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DIAGONAL_CONJ_TRANSPOSE_KERNEL);
-
-
 }  // namespace diagonal
 }  // namespace omp
 }  // namespace kernels
diff --git a/omp/test/matrix/sellp_kernels.cpp b/omp/test/matrix/sellp_kernels.cpp
index a48ea35d9b9..6820f8b8b99 100644
--- a/omp/test/matrix/sellp_kernels.cpp
+++ b/omp/test/matrix/sellp_kernels.cpp
@@ -42,10 +42,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/matrix/csr.hpp>
 #include <ginkgo/core/matrix/dense.hpp>
 #include <ginkgo/core/matrix/diagonal.hpp>
 
 
+#include "core/matrix/sellp_kernels.hpp"
 #include "core/test/utils.hpp"
 
 
@@ -81,20 +83,20 @@ class Sellp : public ::testing::Test {
             std::normal_distribution<>(-1.0, 1.0), rand_engine, ref);
     }
 
-    void set_up_apply_data(
-        int slice_size = gko::matrix::default_slice_size,
-        int stride_factor = gko::matrix::default_stride_factor,
-        int total_cols = 0, int num_vectors = 1)
+    void set_up_apply_matrix(
+        int total_cols = 1, int slice_size = gko::matrix::default_slice_size,
+        int stride_factor = gko::matrix::default_stride_factor)
     {
-        mtx = Mtx::create(ref, gko::dim<2>{}, slice_size, stride_factor,
-                          total_cols);
+        mtx = Mtx::create(ref);
         mtx->copy_from(gen_mtx(532, 231));
-        expected = gen_mtx(532, num_vectors);
-        y = gen_mtx(231, num_vectors);
+        empty = Mtx::create(ref);
+        expected = gen_mtx(532, total_cols);
+        y = gen_mtx(231, total_cols);
         alpha = gko::initialize<Vec>({2.0}, ref);
         beta = gko::initialize<Vec>({-1.0}, ref);
         dmtx = Mtx::create(omp);
         dmtx->copy_from(mtx.get());
+        dempty = Mtx::create(omp);
         dresult = Vec::create(omp);
         dresult->copy_from(expected.get());
         dy = Vec::create(omp);
@@ -111,12 +113,14 @@ class Sellp : public ::testing::Test {
     std::ranlux48 rand_engine;
 
     std::unique_ptr<Mtx> mtx;
+    std::unique_ptr<Mtx> empty;
     std::unique_ptr<Vec> expected;
     std::unique_ptr<Vec> y;
     std::unique_ptr<Vec> alpha;
     std::unique_ptr<Vec> beta;
 
     std::unique_ptr<Mtx> dmtx;
+    std::unique_ptr<Mtx> dempty;
     std::unique_ptr<Vec> dresult;
     std::unique_ptr<Vec> dy;
     std::unique_ptr<Vec> dalpha;
@@ -126,99 +130,116 @@ class Sellp : public ::testing::Test {
 
 TEST_F(Sellp, SimpleApplyIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_matrix();
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
 }
 
 
 TEST_F(Sellp, AdvancedApplyIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_matrix();
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
 }
 
 
 TEST_F(Sellp, SimpleApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
 {
-    set_up_apply_data(32, 4, 0);
+    set_up_apply_matrix(1, 32, 2);
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
 }
 
 
 TEST_F(Sellp, AdvancedApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
 {
-    set_up_apply_data(32, 4, 0);
+    set_up_apply_matrix(1, 32, 2);
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
 }
 
 
-TEST_F(Sellp, SimpleApplyToDenseMatrixIsEquivalentToRef)
+TEST_F(Sellp, SimpleApplyMultipleRHSIsEquivalentToRef)
 {
-    set_up_apply_data(gko::matrix::default_slice_size,
-                      gko::matrix::default_stride_factor, 0, 3);
+    set_up_apply_matrix(64);
+
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
 }
 
 
-TEST_F(Sellp, AdvancedApplyToDenseMatrixIsEquivalentToRef)
+TEST_F(Sellp, AdvancedApplyMultipleRHSIsEquivalentToRef)
 {
-    set_up_apply_data(gko::matrix::default_slice_size,
-                      gko::matrix::default_stride_factor, 0, 3);
+    set_up_apply_matrix(64);
+
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
 }
 
 
 TEST_F(Sellp,
-       SimpleApplyWithSliceSizeAndStrideFactorToDenseMatrixIsEquivalentToRef)
+       SimpleApplyMultipleRHSWithSliceSizeAndStrideFactorIsEquivalentToRef)
 {
-    set_up_apply_data(32, 4, 0, 3);
+    set_up_apply_matrix(32, 2);
 
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
 }
 
 
 TEST_F(Sellp,
-       AdvancedApplyWithSliceSizeAndStrideFactorToDenseMatrixIsEquivalentToRef)
+       AdvancedApplyMultipleRHSWithSliceSizeAndStrideFActorIsEquivalentToRef)
 {
-    set_up_apply_data(32, 4, 0, 3);
+    set_up_apply_matrix(32, 2);
 
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
+    auto result = Vec::create(ref);
+    result->copy_from(dresult.get());
+    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
 }
 
 
 TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_matrix(64);
+
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(omp);
     dcomplex_b->copy_from(complex_b.get());
@@ -235,7 +256,8 @@ TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 
 TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_matrix(64);
+
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(omp);
     dcomplex_b->copy_from(complex_b.get());
@@ -252,37 +274,35 @@ TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 
 TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 {
-    auto rmtx = Mtx::create(ref);
-    rmtx->copy_from(gen_mtx(532, 231));
-    auto omtx = Mtx::create(omp);
-    omtx->copy_from(rmtx.get());
-    auto drmtx = Vec::create(ref);
-    auto domtx = Vec::create(omp);
-
-    rmtx->convert_to(drmtx.get());
-    omtx->convert_to(domtx.get());
-    GKO_ASSERT_MTX_NEAR(drmtx, domtx, 1e-14);
+    set_up_apply_matrix(64);
+
+    auto dense_mtx = gko::matrix::Dense<>::create(ref);
+    auto ddense_mtx = gko::matrix::Dense<>::create(omp);
+
+    mtx->convert_to(dense_mtx.get());
+    dmtx->convert_to(ddense_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 1e-14);
 }
 
 
-TEST_F(Sellp, MoveToDenseIsEquivalentToRef)
+TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 {
-    auto rmtx = Mtx::create(ref);
-    rmtx->copy_from(gen_mtx(532, 231));
-    auto omtx = Mtx::create(omp);
-    omtx->copy_from(rmtx.get());
-    auto drmtx = Vec::create(ref);
-    auto domtx = Vec::create(omp);
-
-    rmtx->move_to(drmtx.get());
-    omtx->move_to(domtx.get());
-    GKO_ASSERT_MTX_NEAR(drmtx, domtx, 1e-14);
+    set_up_apply_matrix(64);
+
+    auto dense_mtx = gko::matrix::Dense<>::create(ref);
+    auto ddense_mtx = gko::matrix::Dense<>::create(omp);
+
+    empty->convert_to(dense_mtx.get());
+    dempty->convert_to(ddense_mtx.get());
+
+    GKO_ASSERT_MTX_NEAR(dense_mtx.get(), ddense_mtx.get(), 0);
 }
 
 
 TEST_F(Sellp, ExtractDiagonalIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_matrix(64);
 
     auto diag = mtx->extract_diagonal();
     auto ddiag = dmtx->extract_diagonal();
@@ -293,7 +313,7 @@ TEST_F(Sellp, ExtractDiagonalIsEquivalentToRef)
 
 TEST_F(Sellp, ExtractDiagonalWithSliceSizeAndStrideFactorIsEquivalentToRef)
 {
-    set_up_apply_data(32, 4, 0, 3);
+    set_up_apply_matrix(64, 32, 2);
 
     auto diag = mtx->extract_diagonal();
     auto ddiag = dmtx->extract_diagonal();
@@ -304,7 +324,7 @@ TEST_F(Sellp, ExtractDiagonalWithSliceSizeAndStrideFactorIsEquivalentToRef)
 
 TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_matrix(64, 32, 2);
 
     mtx->compute_absolute_inplace();
     dmtx->compute_absolute_inplace();
@@ -315,7 +335,7 @@ TEST_F(Sellp, InplaceAbsoluteMatrixIsEquivalentToRef)
 
 TEST_F(Sellp, OutplaceAbsoluteMatrixIsEquivalentToRef)
 {
-    set_up_apply_data();
+    set_up_apply_matrix(64, 32, 2);
 
     auto abs_mtx = mtx->compute_absolute();
     auto dabs_mtx = dmtx->compute_absolute();

From f918087cc9d384f0332f20ed7be08d74002c6db1 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Tue, 3 Aug 2021 11:52:11 +0200
Subject: [PATCH 0983/1008] delete EnableLinOp convert_to due to impl in dense

---
 include/ginkgo/core/matrix/dense.hpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp
index 8e76a036ba3..9ec0a4b8692 100644
--- a/include/ginkgo/core/matrix/dense.hpp
+++ b/include/ginkgo/core/matrix/dense.hpp
@@ -132,8 +132,6 @@ class Dense
     friend class Dense<to_complex<ValueType>>;
 
 public:
-    using EnableLinOp<Dense>::convert_to;
-    using EnableLinOp<Dense>::move_to;
     using ReadableFromMatrixData<ValueType, int32>::read;
     using ReadableFromMatrixData<ValueType, int64>::read;
 

From b8786edac390c86e4391bf0de690776fe0365725 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 4 Aug 2021 23:52:41 +0200
Subject: [PATCH 0984/1008] keep consist, fix exec test, collect csr common -
 use UA reference - del no_exit on exec due to issue on devcloud. it still
 exits if encounter issue, so we do not miss it - collect some of csr into
 common - refine format
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
Co-authored-by: Terry Cojean <terry.cojean@kit.edu>
Co-authored-by: Thomas Grützmacher <thomas.gruetzmacher@kit.edu>
Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 common/matrix/coo_kernels.cpp             |   1 +
 common/matrix/csr_kernels.cpp             | 108 ++++++++++
 common/matrix/csr_kernels.hpp.inc         |  44 ----
 cuda/CMakeLists.txt                       |   1 +
 cuda/matrix/csr_kernels.cu                |  33 ---
 cuda/test/matrix/fbcsr_kernels.cpp        |   2 +-
 cuda/test/matrix/sellp_kernels.cpp        |  41 +---
 dpcpp/CMakeLists.txt                      |   1 +
 dpcpp/components/format_conversion.dp.hpp |   7 +-
 dpcpp/components/prefix_sum.dp.hpp        |  32 +--
 dpcpp/components/reduction.dp.hpp         |   2 +-
 dpcpp/matrix/coo_kernels.dp.cpp           |  18 +-
 dpcpp/matrix/csr_kernels.dp.cpp           | 233 ++++++----------------
 dpcpp/matrix/ell_kernels.dp.cpp           |  20 +-
 dpcpp/matrix/hybrid_kernels.dp.cpp        |  24 +--
 dpcpp/test/base/executor.dp.cpp           |  37 ++--
 dpcpp/test/matrix/fbcsr_kernels.cpp       |   2 +-
 dpcpp/test/matrix/sellp_kernels.cpp       |  47 +----
 hip/CMakeLists.txt                        |   1 +
 hip/matrix/csr_kernels.hip.cpp            |  35 ----
 hip/test/matrix/CMakeLists.txt            |   1 +
 hip/test/matrix/fbcsr_kernels.hip.cpp     |  93 +++++++++
 hip/test/matrix/sellp_kernels.hip.cpp     |  40 +---
 omp/CMakeLists.txt                        |   1 +
 omp/matrix/csr_kernels.cpp                |  43 ----
 omp/matrix/sparsity_csr_kernels.cpp       |   2 +-
 omp/test/matrix/CMakeLists.txt            |   1 +
 omp/test/matrix/fbcsr_kernels.cpp         |  92 +++++++++
 omp/test/matrix/sellp_kernels.cpp         |  38 +---
 reference/matrix/sparsity_csr_kernels.cpp |   2 +-
 30 files changed, 470 insertions(+), 532 deletions(-)
 create mode 100644 common/matrix/csr_kernels.cpp
 create mode 100644 hip/test/matrix/fbcsr_kernels.hip.cpp
 create mode 100644 omp/test/matrix/fbcsr_kernels.cpp

diff --git a/common/matrix/coo_kernels.cpp b/common/matrix/coo_kernels.cpp
index 63fbf2c9ec1..ecb5c397743 100644
--- a/common/matrix/coo_kernels.cpp
+++ b/common/matrix/coo_kernels.cpp
@@ -67,6 +67,7 @@ void extract_diagonal(std::shared_ptr<const DefaultExecutor> exec,
         orig->get_const_row_idxs(), orig->get_const_col_idxs(),
         diag->get_values());
 }
+
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_COO_EXTRACT_DIAGONAL_KERNEL);
 
diff --git a/common/matrix/csr_kernels.cpp b/common/matrix/csr_kernels.cpp
new file mode 100644
index 00000000000..619a06d8dab
--- /dev/null
+++ b/common/matrix/csr_kernels.cpp
@@ -0,0 +1,108 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/matrix/csr_kernels.hpp"
+
+
+#include <algorithm>
+
+
+#include <ginkgo/core/base/math.hpp>
+
+
+#include "common/base/kernel_launch.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace GKO_DEVICE_NAMESPACE {
+/**
+ * @brief The Csr matrix format namespace.
+ *
+ * @ingroup csr
+ */
+namespace csr {
+
+
+template <typename IndexType>
+void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
+                        size_type size, const IndexType *permutation_indices,
+                        IndexType *inv_permutation)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto tid, auto permutation, auto inv_permutation) {
+            inv_permutation[permutation[tid]] = tid;
+        },
+        size, permutation_indices, inv_permutation);
+}
+
+GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
+
+
+template <typename ValueType, typename IndexType>
+void inverse_column_permute(std::shared_ptr<const DefaultExecutor> exec,
+                            const IndexType *perm,
+                            const matrix::Csr<ValueType, IndexType> *orig,
+                            matrix::Csr<ValueType, IndexType> *column_permuted)
+{
+    auto num_rows = orig->get_size()[0];
+    auto nnz = orig->get_num_stored_elements();
+    auto size = std::max(num_rows, nnz);
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto tid, auto num_rows, auto num_nonzeros,
+                      auto permutation, auto in_row_ptrs, auto in_col_idxs,
+                      auto in_vals, auto out_row_ptrs, auto out_col_idxs,
+                      auto out_vals) {
+            if (tid < num_nonzeros) {
+                out_col_idxs[tid] = permutation[in_col_idxs[tid]];
+                out_vals[tid] = in_vals[tid];
+            }
+            if (tid <= num_rows) {
+                out_row_ptrs[tid] = in_row_ptrs[tid];
+            }
+        },
+        size, num_rows, nnz, perm, orig->get_const_row_ptrs(),
+        orig->get_const_col_idxs(), orig->get_const_values(),
+        column_permuted->get_row_ptrs(), column_permuted->get_col_idxs(),
+        column_permuted->get_values());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
+
+
+}  // namespace csr
+}  // namespace GKO_DEVICE_NAMESPACE
+}  // namespace kernels
+}  // namespace gko
diff --git a/common/matrix/csr_kernels.hpp.inc b/common/matrix/csr_kernels.hpp.inc
index c2761fa1ab3..49c3a138e90 100644
--- a/common/matrix/csr_kernels.hpp.inc
+++ b/common/matrix/csr_kernels.hpp.inc
@@ -220,17 +220,6 @@ __global__ __launch_bounds__(spmv_block_size) void abstract_spmv(
 }
 
 
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void set_zero(
-    const size_type nnz, ValueType *__restrict__ val)
-{
-    const auto ind = thread::get_thread_id_flat();
-    if (ind < nnz) {
-        val[ind] = zero<ValueType>();
-    }
-}
-
-
 template <typename IndexType>
 __forceinline__ __device__ void merge_path_search(
     const IndexType diagonal, const IndexType a_len, const IndexType b_len,
@@ -949,39 +938,6 @@ __global__ __launch_bounds__(default_block_size) void conjugate_kernel(
 }  //  namespace
 
 
-template <typename IndexType>
-__global__ __launch_bounds__(default_block_size) void inv_permutation_kernel(
-    size_type size, const IndexType *__restrict__ permutation,
-    IndexType *__restrict__ inv_permutation)
-{
-    auto tid = thread::get_thread_id_flat();
-    if (tid >= size) {
-        return;
-    }
-    inv_permutation[permutation[tid]] = tid;
-}
-
-
-template <typename ValueType, typename IndexType>
-__global__ __launch_bounds__(default_block_size) void col_permute_kernel(
-    size_type num_rows, size_type num_nonzeros,
-    const IndexType *__restrict__ permutation,
-    const IndexType *__restrict__ in_row_ptrs,
-    const IndexType *__restrict__ in_cols,
-    const ValueType *__restrict__ in_vals, IndexType *__restrict__ out_row_ptrs,
-    IndexType *__restrict__ out_cols, ValueType *__restrict__ out_vals)
-{
-    auto tid = thread::get_thread_id_flat();
-    if (tid < num_nonzeros) {
-        out_cols[tid] = permutation[in_cols[tid]];
-        out_vals[tid] = in_vals[tid];
-    }
-    if (tid <= num_rows) {
-        out_row_ptrs[tid] = in_row_ptrs[tid];
-    }
-}
-
-
 template <typename IndexType>
 __global__ __launch_bounds__(default_block_size) void row_ptr_permute_kernel(
     size_type num_rows, const IndexType *__restrict__ permutation,
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index d40c7177c5b..df512019d88 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -68,6 +68,7 @@ add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_cuda_device> "")
 set(GKO_CUDA_COMMON_SOURCES
     ../common/components/precision_conversion.cpp
     ../common/matrix/coo_kernels.cpp
+    ../common/matrix/csr_kernels.cpp
     ../common/matrix/dense_kernels.cpp
     ../common/matrix/diagonal_kernels.cpp
     ../common/solver/bicg_kernels.cpp
diff --git a/cuda/matrix/csr_kernels.cu b/cuda/matrix/csr_kernels.cu
index 7c32d3043ec..d59914993ab 100644
--- a/cuda/matrix/csr_kernels.cu
+++ b/cuda/matrix/csr_kernels.cu
@@ -1142,19 +1142,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename IndexType>
-void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
-                        size_type size, const IndexType *permutation_indices,
-                        IndexType *inv_permutation)
-{
-    auto num_blocks = ceildiv(size, default_block_size);
-    inv_permutation_kernel<<<num_blocks, default_block_size>>>(
-        size, permutation_indices, inv_permutation);
-}
-
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void inv_symm_permute(std::shared_ptr<const CudaExecutor> exec,
                       const IndexType *perm,
@@ -1232,26 +1219,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(std::shared_ptr<const CudaExecutor> exec,
-                            const IndexType *perm,
-                            const matrix::Csr<ValueType, IndexType> *orig,
-                            matrix::Csr<ValueType, IndexType> *column_permuted)
-{
-    auto num_rows = orig->get_size()[0];
-    auto nnz = orig->get_num_stored_elements();
-    auto num_blocks = ceildiv(std::max(num_rows, nnz), default_block_size);
-    col_permute_kernel<<<num_blocks, default_block_size>>>(
-        num_rows, nnz, perm, orig->get_const_row_ptrs(),
-        orig->get_const_col_idxs(), as_cuda_type(orig->get_const_values()),
-        column_permuted->get_row_ptrs(), column_permuted->get_col_idxs(),
-        as_cuda_type(column_permuted->get_values()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void calculate_max_nnz_per_row(std::shared_ptr<const CudaExecutor> exec,
                                const matrix::Csr<ValueType, IndexType> *source,
diff --git a/cuda/test/matrix/fbcsr_kernels.cpp b/cuda/test/matrix/fbcsr_kernels.cpp
index 02bfa3adfe1..6b273de668d 100644
--- a/cuda/test/matrix/fbcsr_kernels.cpp
+++ b/cuda/test/matrix/fbcsr_kernels.cpp
@@ -80,9 +80,9 @@ TEST_F(Fbcsr, CanWriteFromMatrixOnDevice)
     auto refmat = sample.generate_fbcsr();
     auto cudamat = Mtx::create(cuda);
     cudamat->copy_from(gko::lend(refmat));
-
     MatData refdata;
     MatData cudadata;
+
     refmat->write(refdata);
     cudamat->write(cudadata);
 
diff --git a/cuda/test/matrix/sellp_kernels.cpp b/cuda/test/matrix/sellp_kernels.cpp
index 071270fde71..706fedccd3f 100644
--- a/cuda/test/matrix/sellp_kernels.cpp
+++ b/cuda/test/matrix/sellp_kernels.cpp
@@ -136,9 +136,7 @@ TEST_F(Sellp, SimpleApplyIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -149,9 +147,7 @@ TEST_F(Sellp, AdvancedApplyIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -162,9 +158,7 @@ TEST_F(Sellp, SimpleApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -175,9 +169,7 @@ TEST_F(Sellp, AdvancedApplyWithSliceSizeAndStrideFActorIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -185,13 +177,10 @@ TEST_F(Sellp, SimpleApplyMultipleRHSIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
 
-
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -199,13 +188,10 @@ TEST_F(Sellp, AdvancedApplyMultipleRHSIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
 
-
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -217,9 +203,7 @@ TEST_F(Sellp,
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -231,16 +215,13 @@ TEST_F(Sellp,
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
 TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(cuda);
     dcomplex_b->copy_from(complex_b.get());
@@ -258,7 +239,6 @@ TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(cuda);
     dcomplex_b->copy_from(complex_b.get());
@@ -276,7 +256,6 @@ TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto dense_mtx = gko::matrix::Dense<>::create(ref);
     auto ddense_mtx = gko::matrix::Dense<>::create(cuda);
 
@@ -290,7 +269,6 @@ TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 TEST_F(Sellp, ConvertToCsrIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto csr_mtx = gko::matrix::Csr<>::create(ref);
     auto dcsr_mtx = gko::matrix::Csr<>::create(cuda);
 
@@ -304,7 +282,6 @@ TEST_F(Sellp, ConvertToCsrIsEquivalentToRef)
 TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto dense_mtx = gko::matrix::Dense<>::create(ref);
     auto ddense_mtx = gko::matrix::Dense<>::create(cuda);
 
@@ -318,7 +295,6 @@ TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 TEST_F(Sellp, ConvertEmptyToCsrIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto csr_mtx = gko::matrix::Csr<>::create(ref);
     auto dcsr_mtx = gko::matrix::Csr<>::create(cuda);
 
@@ -332,7 +308,6 @@ TEST_F(Sellp, ConvertEmptyToCsrIsEquivalentToRef)
 TEST_F(Sellp, CountNonzerosIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     gko::size_type nnz;
     gko::size_type dnnz;
 
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index d7b6b1d2465..f167bac3742 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -49,6 +49,7 @@ target_sources(ginkgo_dpcpp
     stop/residual_norm_kernels.dp.cpp
     ../common/components/precision_conversion.cpp
     ../common/matrix/coo_kernels.cpp
+    ../common/matrix/csr_kernels.cpp
     ../common/matrix/dense_kernels.cpp
     ../common/matrix/diagonal_kernels.cpp
     ../common/solver/bicg_kernels.cpp
diff --git a/dpcpp/components/format_conversion.dp.hpp b/dpcpp/components/format_conversion.dp.hpp
index 9a2410c3f2e..99df6f02a4a 100644
--- a/dpcpp/components/format_conversion.dp.hpp
+++ b/dpcpp/components/format_conversion.dp.hpp
@@ -34,6 +34,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_DPCPP_COMPONENTS_FORMAT_CONVERSION_DP_HPP_
 
 
+#include <algorithm>
+
+
 #include <CL/sycl.hpp>
 
 
@@ -103,7 +106,7 @@ namespace host_kernel {
  * It calculates the number of warps used in Coo Spmv depending on the GPU
  * architecture and the number of stored elements.
  */
-template <size_type subwarp_size = config::warp_size>
+template <size_type subgroup_size = config::warp_size>
 size_type calculate_nwarps(std::shared_ptr<const DpcppExecutor> exec,
                            const size_type nnz)
 {
@@ -120,7 +123,7 @@ size_type calculate_nwarps(std::shared_ptr<const DpcppExecutor> exec,
     }
 #endif  // GINKGO_BENCHMARK_ENABLE_TUNING
     return std::min(multiple * nwarps_in_dpcpp,
-                    size_type(ceildiv(nnz, subwarp_size)));
+                    size_type(ceildiv(nnz, subgroup_size)));
 }
 
 
diff --git a/dpcpp/components/prefix_sum.dp.hpp b/dpcpp/components/prefix_sum.dp.hpp
index 0eaf5da3f76..3b5e1c317dd 100644
--- a/dpcpp/components/prefix_sum.dp.hpp
+++ b/dpcpp/components/prefix_sum.dp.hpp
@@ -53,59 +53,63 @@ namespace kernels {
 namespace dpcpp {
 
 
+// TODO: porting - some functions name still use subwarp
+
+
 /**
  * @internal
- * Computes the prefix sum and total sum of `element` over a subwarp.
+ * Computes the prefix sum and total sum of `element` over a subgroup.
  *
  * @param element     the element over which we compute the prefix sum.
  * @param prefix_sum  will be set to the sum of all `element`s from lower
  *                    lanes, plus the local `element` if `inclusive` is `true`.
- * @param total_sum   will be set to the total sum of `element` in this subwarp.
- * @param subwarp     the cooperative group representing the subwarp.
+ * @param total_sum   will be set to the total sum of `element` in this
+ * subgroup.
+ * @param subgroup     the cooperative group representing the subgroup.
  *
  * @tparam inclusive  if this is true, the computed prefix sum will be
  *                    inclusive, otherwise it will be exclusive.
  *
  * @note For this function to work on architectures with independent thread
- * scheduling, all threads of the subwarp have to execute it.
+ * scheduling, all threads of the subgroup have to execute it.
  */
 template <bool inclusive, typename ValueType, typename Group>
 __dpct_inline__ void subwarp_prefix_sum(ValueType element,
                                         ValueType &prefix_sum,
-                                        ValueType &total_sum, Group subwarp)
+                                        ValueType &total_sum, Group subgroup)
 {
     prefix_sum = inclusive ? element : zero<ValueType>();
     total_sum = element;
 #pragma unroll
     // hypercube prefix sum
-    for (int step = 1; step < subwarp.size(); step *= 2) {
-        auto neighbor = subwarp.shfl_xor(total_sum, step);
+    for (int step = 1; step < subgroup.size(); step *= 2) {
+        auto neighbor = subgroup.shfl_xor(total_sum, step);
         total_sum += neighbor;
-        prefix_sum += bool(subwarp.thread_rank() & step) ? neighbor : 0;
+        prefix_sum += bool(subgroup.thread_rank() & step) ? neighbor : 0;
     }
 }
 
 /**
  * @internal
- * Computes the prefix sum of `element` over a subwarp.
+ * Computes the prefix sum of `element` over a subgroup.
  *
  * @param element     the element over which we compute the prefix sum.
  * @param prefix_sum  will be set to the sum of all `element`s from lower
  *                    lanes, plus the local `element` if `inclusive` is `true`.
- * @param subwarp     the cooperative group representing the subwarp.
+ * @param subgroup     the cooperative group representing the subgroup.
  *
  * @tparam inclusive  if this is true, the computed prefix sum will be
  *                    inclusive, otherwise it will be exclusive.
  *
- * @note All threads of the subwarp have to execute this function for it to work
- *       (and not dead-lock on newer architectures).
+ * @note All threads of the subgroup have to execute this function for it to
+ * work (and not dead-lock on newer architectures).
  */
 template <bool inclusive, typename ValueType, typename Group>
 __dpct_inline__ void subwarp_prefix_sum(ValueType element,
-                                        ValueType &prefix_sum, Group subwarp)
+                                        ValueType &prefix_sum, Group subgroup)
 {
     ValueType tmp{};
-    subwarp_prefix_sum<inclusive>(element, prefix_sum, tmp, subwarp);
+    subwarp_prefix_sum<inclusive>(element, prefix_sum, tmp, subgroup);
 }
 
 
diff --git a/dpcpp/components/reduction.dp.hpp b/dpcpp/components/reduction.dp.hpp
index 0d47da1490b..0cf6e49e068 100644
--- a/dpcpp/components/reduction.dp.hpp
+++ b/dpcpp/components/reduction.dp.hpp
@@ -81,7 +81,7 @@ constexpr auto kcfg_1d_array = as_array(kcfg_1d_list);
  * @note The function is guaranteed to return the correct value on all threads
  *       only if `reduce_op` is commutative (in addition to being associative).
  *       Otherwise, the correct value is returned only to the thread with
- *       subwarp index 0.
+ *       subgroup index 0.
  */
 template <
     typename Group, typename ValueType, typename Operator,
diff --git a/dpcpp/matrix/coo_kernels.dp.cpp b/dpcpp/matrix/coo_kernels.dp.cpp
index 63b317e7fec..561a0cddf8f 100644
--- a/dpcpp/matrix/coo_kernels.dp.cpp
+++ b/dpcpp/matrix/coo_kernels.dp.cpp
@@ -96,7 +96,7 @@ namespace {
  * @tparam IndexType  type of matrix indexes stored in the structure
  * @tparam Closure  type of the function used to write the result
  */
-template <int subwarp_size = config::warp_size, typename ValueType,
+template <int subgroup_size = config::warp_size, typename ValueType,
           typename IndexType, typename Closure>
 void spmv_kernel(const size_type nnz, const size_type num_lines,
                  const ValueType *__restrict__ val,
@@ -114,23 +114,23 @@ void spmv_kernel(const size_type nnz, const size_type num_lines,
         item_ct1.get_local_id(1) * item_ct1.get_local_range().get(2) *
             num_lines;
     const auto column_id = item_ct1.get_group(1);
-    size_type num = (nnz > start) * ceildiv(nnz - start, subwarp_size);
+    size_type num = (nnz > start) * ceildiv(nnz - start, subgroup_size);
     num = min(num, num_lines);
     const IndexType ind_start = start + item_ct1.get_local_id(2);
-    const IndexType ind_end = ind_start + (num - 1) * subwarp_size;
+    const IndexType ind_end = ind_start + (num - 1) * subgroup_size;
     IndexType ind = ind_start;
     IndexType curr_row = (ind < nnz) ? row[ind] : 0;
-    const auto tile_block = group::tiled_partition<subwarp_size>(
+    const auto tile_block = group::tiled_partition<subgroup_size>(
         group::this_thread_block(item_ct1));
-    for (; ind < ind_end; ind += subwarp_size) {
+    for (; ind < ind_end; ind += subgroup_size) {
         temp_val += (ind < nnz) ? val[ind] * b[col[ind] * b_stride + column_id]
                                 : zero<ValueType>();
-        auto next_row =
-            (ind + subwarp_size < nnz) ? row[ind + subwarp_size] : row[nnz - 1];
+        auto next_row = (ind + subgroup_size < nnz) ? row[ind + subgroup_size]
+                                                    : row[nnz - 1];
         // segmented scan
         if (tile_block.any(curr_row != next_row)) {
             bool is_first_in_segment =
-                segment_scan<subwarp_size>(tile_block, curr_row, &temp_val);
+                segment_scan<subgroup_size>(tile_block, curr_row, &temp_val);
             if (is_first_in_segment) {
                 atomic_add(&(c[curr_row * c_stride + column_id]),
                            scale(temp_val));
@@ -145,7 +145,7 @@ void spmv_kernel(const size_type nnz, const size_type num_lines,
                                 : zero<ValueType>();
         // segmented scan
         bool is_first_in_segment =
-            segment_scan<subwarp_size>(tile_block, curr_row, &temp_val);
+            segment_scan<subgroup_size>(tile_block, curr_row, &temp_val);
         if (is_first_in_segment) {
             atomic_add(&(c[curr_row * c_stride + column_id]), scale(temp_val));
         }
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index 089bc2238b9..ecb4d2e3e49 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -154,10 +154,10 @@ __dpct_inline__ void find_next_row(
 }
 
 
-template <unsigned subwarp_size, typename ValueType, typename IndexType,
+template <unsigned subgroup_size, typename ValueType, typename IndexType,
           typename Closure>
 __dpct_inline__ void warp_atomic_add(
-    const group::thread_block_tile<subwarp_size> &group, bool force_write,
+    const group::thread_block_tile<subgroup_size> &group, bool force_write,
     ValueType *__restrict__ val, const IndexType row, ValueType *__restrict__ c,
     const size_type c_stride, const IndexType column_id, Closure scale)
 {
@@ -172,10 +172,10 @@ __dpct_inline__ void warp_atomic_add(
 }
 
 
-template <bool last, unsigned subwarp_size, typename ValueType,
+template <bool last, unsigned subgroup_size, typename ValueType,
           typename IndexType, typename Closure>
 __dpct_inline__ void process_window(
-    const group::thread_block_tile<subwarp_size> &group,
+    const group::thread_block_tile<subgroup_size> &group,
     const IndexType num_rows, const IndexType data_size, const IndexType ind,
     IndexType *__restrict__ row, IndexType *__restrict__ row_end,
     IndexType *__restrict__ nrow, IndexType *__restrict__ nrow_end,
@@ -192,8 +192,8 @@ __dpct_inline__ void process_window(
     if (group.any(curr_row != *row)) {
         warp_atomic_add(group, curr_row != *row, temp_val, curr_row, c,
                         c_stride, column_id, scale);
-        *nrow = group.shfl(*row, subwarp_size - 1);
-        *nrow_end = group.shfl(*row_end, subwarp_size - 1);
+        *nrow = group.shfl(*row, subgroup_size - 1);
+        *nrow_end = group.shfl(*row_end, subgroup_size - 1);
     }
 
     if (!last || ind < data_size) {
@@ -296,19 +296,6 @@ void abstract_spmv(const IndexType nwarps, const IndexType num_rows,
 GKO_ENABLE_DEFAULT_HOST(abstract_spmv, abstract_spmv);
 
 
-template <typename ValueType>
-void set_zero(const size_type nnz, ValueType *__restrict__ val,
-              sycl::nd_item<3> item_ct1)
-{
-    const auto ind = thread::get_thread_id_flat(item_ct1);
-    if (ind < nnz) {
-        val[ind] = zero<ValueType>();
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(set_zero, set_zero);
-
-
 template <typename IndexType>
 __dpct_inline__ void merge_path_search(
     const IndexType diagonal, const IndexType a_len, const IndexType b_len,
@@ -337,8 +324,8 @@ void merge_path_reduce(const IndexType nwarps,
                        const IndexType *__restrict__ last_row,
                        ValueType *__restrict__ c, const size_type c_stride,
                        Alpha_op alpha_op, sycl::nd_item<3> item_ct1,
-                       UninitializedArray<IndexType, spmv_block_size> *tmp_ind,
-                       UninitializedArray<ValueType, spmv_block_size> *tmp_val)
+                       UninitializedArray<IndexType, spmv_block_size> &tmp_ind,
+                       UninitializedArray<ValueType, spmv_block_size> &tmp_val)
 {
     const IndexType cache_lines = ceildivT<IndexType>(nwarps, spmv_block_size);
     const IndexType tid = item_ct1.get_local_id(2);
@@ -361,15 +348,15 @@ void merge_path_reduce(const IndexType nwarps,
     }
 
 
-    (*tmp_val)[item_ct1.get_local_id(2)] = value;
-    (*tmp_ind)[item_ct1.get_local_id(2)] = row;
+    tmp_val[item_ct1.get_local_id(2)] = value;
+    tmp_ind[item_ct1.get_local_id(2)] = row;
     group::this_thread_block(item_ct1).sync();
-    bool last = block_segment_scan_reverse(static_cast<IndexType *>((*tmp_ind)),
-                                           static_cast<ValueType *>((*tmp_val)),
-                                           item_ct1);
+    bool last =
+        block_segment_scan_reverse(static_cast<IndexType *>(tmp_ind),
+                                   static_cast<ValueType *>(tmp_val), item_ct1);
     group::this_thread_block(item_ct1).sync();
     if (last) {
-        c[row * c_stride] += alpha_op((*tmp_val)[item_ct1.get_local_id(2)]);
+        c[row * c_stride] += alpha_op(tmp_val[item_ct1.get_local_id(2)]);
     }
 }
 
@@ -552,8 +539,8 @@ void abstract_reduce(const IndexType nwarps,
                      const IndexType *__restrict__ last_row,
                      ValueType *__restrict__ c, const size_type c_stride,
                      sycl::nd_item<3> item_ct1,
-                     UninitializedArray<IndexType, spmv_block_size> *tmp_ind,
-                     UninitializedArray<ValueType, spmv_block_size> *tmp_val)
+                     UninitializedArray<IndexType, spmv_block_size> &tmp_ind,
+                     UninitializedArray<ValueType, spmv_block_size> &tmp_val)
 {
     merge_path_reduce(
         nwarps, last_val, last_row, c, c_stride, [](ValueType &x) { return x; },
@@ -579,8 +566,8 @@ void abstract_reduce(dim3 grid, dim3 block, size_type dynamic_shared_memory,
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 abstract_reduce(nwarps, last_val, last_row, c, c_stride,
-                                item_ct1, tmp_ind_acc_ct1.get_pointer().get(),
-                                tmp_val_acc_ct1.get_pointer().get());
+                                item_ct1, *tmp_ind_acc_ct1.get_pointer(),
+                                *tmp_val_acc_ct1.get_pointer());
             });
     });
 }
@@ -593,8 +580,8 @@ void abstract_reduce(const IndexType nwarps,
                      const ValueType *__restrict__ alpha,
                      ValueType *__restrict__ c, const size_type c_stride,
                      sycl::nd_item<3> item_ct1,
-                     UninitializedArray<IndexType, spmv_block_size> *tmp_ind,
-                     UninitializedArray<ValueType, spmv_block_size> *tmp_val)
+                     UninitializedArray<IndexType, spmv_block_size> &tmp_ind,
+                     UninitializedArray<ValueType, spmv_block_size> &tmp_val)
 {
     const auto alpha_val = alpha[0];
     merge_path_reduce(
@@ -623,14 +610,14 @@ void abstract_reduce(dim3 grid, dim3 block, size_type dynamic_shared_memory,
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
                 abstract_reduce(nwarps, last_val, last_row, alpha, c, c_stride,
-                                item_ct1, tmp_ind_acc_ct1.get_pointer().get(),
-                                tmp_val_acc_ct1.get_pointer().get());
+                                item_ct1, *tmp_ind_acc_ct1.get_pointer(),
+                                *tmp_val_acc_ct1.get_pointer());
             });
     });
 }
 
 
-template <size_type subwarp_size, typename ValueType, typename IndexType,
+template <size_type subgroup_size, typename ValueType, typename IndexType,
           typename Closure>
 void device_classical_spmv(const size_type num_rows,
                            const ValueType *__restrict__ val,
@@ -641,33 +628,33 @@ void device_classical_spmv(const size_type num_rows,
                            const size_type c_stride, Closure scale,
                            sycl::nd_item<3> item_ct1)
 {
-    auto subwarp_tile = group::tiled_partition<subwarp_size>(
+    auto subgroup_tile = group::tiled_partition<subgroup_size>(
         group::this_thread_block(item_ct1));
-    const auto subrow = thread::get_subwarp_num_flat<subwarp_size>(item_ct1);
-    const auto subid = subwarp_tile.thread_rank();
+    const auto subrow = thread::get_subwarp_num_flat<subgroup_size>(item_ct1);
+    const auto subid = subgroup_tile.thread_rank();
     const auto column_id = item_ct1.get_group(1);
-    auto row = thread::get_subwarp_id_flat<subwarp_size>(item_ct1);
+    auto row = thread::get_subwarp_id_flat<subgroup_size>(item_ct1);
     for (; row < num_rows; row += subrow) {
         const auto ind_end = row_ptrs[row + 1];
         ValueType temp_val = zero<ValueType>();
         for (auto ind = row_ptrs[row] + subid; ind < ind_end;
-             ind += subwarp_size) {
+             ind += subgroup_size) {
             temp_val += val[ind] * b[col_idxs[ind] * b_stride + column_id];
         }
-        auto subwarp_result = ::gko::kernels::dpcpp::reduce(
-            subwarp_tile, temp_val,
+        auto subgroup_result = ::gko::kernels::dpcpp::reduce(
+            subgroup_tile, temp_val,
             [](const ValueType &a, const ValueType &b) { return a + b; });
         // TODO: check the barrier
         item_ct1.barrier();
         if (subid == 0) {
             c[row * c_stride + column_id] =
-                scale(subwarp_result, c[row * c_stride + column_id]);
+                scale(subgroup_result, c[row * c_stride + column_id]);
         }
     }
 }
 
 
-template <size_type subwarp_size, typename ValueType, typename IndexType>
+template <size_type subgroup_size, typename ValueType, typename IndexType>
 void abstract_classical_spmv(
     const size_type num_rows, const ValueType *__restrict__ val,
     const IndexType *__restrict__ col_idxs,
@@ -675,12 +662,12 @@ void abstract_classical_spmv(
     const size_type b_stride, ValueType *__restrict__ c,
     const size_type c_stride, sycl::nd_item<3> item_ct1)
 {
-    device_classical_spmv<subwarp_size>(
+    device_classical_spmv<subgroup_size>(
         num_rows, val, col_idxs, row_ptrs, b, b_stride, c, c_stride,
         [](const ValueType &x, const ValueType &y) { return x; }, item_ct1);
 }
 
-template <size_type subwarp_size, typename ValueType, typename IndexType>
+template <size_type subgroup_size, typename ValueType, typename IndexType>
 void abstract_classical_spmv(dim3 grid, dim3 block,
                              size_type dynamic_shared_memory,
                              sycl::queue *queue, const size_type num_rows,
@@ -692,15 +679,15 @@ void abstract_classical_spmv(dim3 grid, dim3 block,
     queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                abstract_classical_spmv<subwarp_size>(num_rows, val, col_idxs,
-                                                      row_ptrs, b, b_stride, c,
-                                                      c_stride, item_ct1);
+                abstract_classical_spmv<subgroup_size>(num_rows, val, col_idxs,
+                                                       row_ptrs, b, b_stride, c,
+                                                       c_stride, item_ct1);
             });
     });
 }
 
 
-template <size_type subwarp_size, typename ValueType, typename IndexType>
+template <size_type subgroup_size, typename ValueType, typename IndexType>
 void abstract_classical_spmv(
     const size_type num_rows, const ValueType *__restrict__ alpha,
     const ValueType *__restrict__ val, const IndexType *__restrict__ col_idxs,
@@ -711,7 +698,7 @@ void abstract_classical_spmv(
 {
     const auto alpha_val = alpha[0];
     const auto beta_val = beta[0];
-    device_classical_spmv<subwarp_size>(
+    device_classical_spmv<subgroup_size>(
         num_rows, val, col_idxs, row_ptrs, b, b_stride, c, c_stride,
         [&alpha_val, &beta_val](const ValueType &x, const ValueType &y) {
             return alpha_val * x + beta_val * y;
@@ -719,7 +706,7 @@ void abstract_classical_spmv(
         item_ct1);
 }
 
-template <size_type subwarp_size, typename ValueType, typename IndexType>
+template <size_type subgroup_size, typename ValueType, typename IndexType>
 void abstract_classical_spmv(dim3 grid, dim3 block,
                              size_type dynamic_shared_memory,
                              sycl::queue *queue, const size_type num_rows,
@@ -732,7 +719,7 @@ void abstract_classical_spmv(dim3 grid, dim3 block,
     queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(sycl_nd_range(grid, block),
                          [=](sycl::nd_item<3> item_ct1) {
-                             abstract_classical_spmv<subwarp_size>(
+                             abstract_classical_spmv<subgroup_size>(
                                  num_rows, alpha, val, col_idxs, row_ptrs, b,
                                  b_stride, beta, c, c_stride, item_ct1);
                          });
@@ -1160,66 +1147,6 @@ GKO_ENABLE_DEFAULT_HOST(extract_diagonal, extract_diagonal);
 }  // namespace kernel
 
 
-namespace {
-
-
-template <typename ValueType>
-void conjugate_kernel(size_type num_nonzeros, ValueType *__restrict__ val,
-                      sycl::nd_item<3> item_ct1)
-{
-    const auto tidx = thread::get_thread_id_flat(item_ct1);
-
-    if (tidx < num_nonzeros) {
-        val[tidx] = conj(val[tidx]);
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(conjugate_kernel, conjugate_kernel);
-
-
-}  //  namespace
-
-
-template <typename IndexType>
-void inv_permutation_kernel(size_type size,
-                            const IndexType *__restrict__ permutation,
-                            IndexType *__restrict__ inv_permutation,
-                            sycl::nd_item<3> item_ct1)
-{
-    auto tid = thread::get_thread_id_flat(item_ct1);
-    if (tid >= size) {
-        return;
-    }
-    inv_permutation[permutation[tid]] = tid;
-}
-
-GKO_ENABLE_DEFAULT_HOST(inv_permutation_kernel, inv_permutation_kernel);
-
-
-template <typename ValueType, typename IndexType>
-void col_permute_kernel(size_type num_rows, size_type num_nonzeros,
-                        const IndexType *__restrict__ permutation,
-                        const IndexType *__restrict__ in_row_ptrs,
-                        const IndexType *__restrict__ in_cols,
-                        const ValueType *__restrict__ in_vals,
-                        IndexType *__restrict__ out_row_ptrs,
-                        IndexType *__restrict__ out_cols,
-                        ValueType *__restrict__ out_vals,
-                        sycl::nd_item<3> item_ct1)
-{
-    auto tid = thread::get_thread_id_flat(item_ct1);
-    if (tid < num_nonzeros) {
-        out_cols[tid] = permutation[in_cols[tid]];
-        out_vals[tid] = in_vals[tid];
-    }
-    if (tid <= num_rows) {
-        out_row_ptrs[tid] = in_row_ptrs[tid];
-    }
-}
-
-GKO_ENABLE_DEFAULT_HOST(col_permute_kernel, col_permute_kernel);
-
-
 template <typename IndexType>
 void row_ptr_permute_kernel(size_type num_rows,
                             const IndexType *__restrict__ permutation,
@@ -1258,7 +1185,7 @@ void inv_row_ptr_permute_kernel(size_type num_rows,
 GKO_ENABLE_DEFAULT_HOST(inv_row_ptr_permute_kernel, inv_row_ptr_permute_kernel);
 
 
-template <int subwarp_size, typename ValueType, typename IndexType>
+template <int subgroup_size, typename ValueType, typename IndexType>
 void row_permute_kernel(size_type num_rows,
                         const IndexType *__restrict__ permutation,
                         const IndexType *__restrict__ in_row_ptrs,
@@ -1269,23 +1196,23 @@ void row_permute_kernel(size_type num_rows,
                         ValueType *__restrict__ out_vals,
                         sycl::nd_item<3> item_ct1)
 {
-    auto tid = thread::get_subwarp_id_flat<subwarp_size>(item_ct1);
+    auto tid = thread::get_subwarp_id_flat<subgroup_size>(item_ct1);
     if (tid >= num_rows) {
         return;
     }
-    auto lane = item_ct1.get_local_id(2) % subwarp_size;
+    auto lane = item_ct1.get_local_id(2) % subgroup_size;
     auto in_row = permutation[tid];
     auto out_row = tid;
     auto in_begin = in_row_ptrs[in_row];
     auto in_size = in_row_ptrs[in_row + 1] - in_begin;
     auto out_begin = out_row_ptrs[out_row];
-    for (IndexType i = lane; i < in_size; i += subwarp_size) {
+    for (IndexType i = lane; i < in_size; i += subgroup_size) {
         out_cols[out_begin + i] = in_cols[in_begin + i];
         out_vals[out_begin + i] = in_vals[in_begin + i];
     }
 }
 
-template <int subwarp_size, typename ValueType, typename IndexType>
+template <int subgroup_size, typename ValueType, typename IndexType>
 void row_permute_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
                         sycl::queue *queue, size_type num_rows,
                         const IndexType *permutation,
@@ -1296,7 +1223,7 @@ void row_permute_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
     queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                row_permute_kernel<subwarp_size>(
+                row_permute_kernel<subgroup_size>(
                     num_rows, permutation, in_row_ptrs, in_cols, in_vals,
                     out_row_ptrs, out_cols, out_vals, item_ct1);
             });
@@ -1304,7 +1231,7 @@ void row_permute_kernel(dim3 grid, dim3 block, size_type dynamic_shared_memory,
 }
 
 
-template <int subwarp_size, typename ValueType, typename IndexType>
+template <int subgroup_size, typename ValueType, typename IndexType>
 void inv_row_permute_kernel(size_type num_rows,
                             const IndexType *__restrict__ permutation,
                             const IndexType *__restrict__ in_row_ptrs,
@@ -1315,23 +1242,23 @@ void inv_row_permute_kernel(size_type num_rows,
                             ValueType *__restrict__ out_vals,
                             sycl::nd_item<3> item_ct1)
 {
-    auto tid = thread::get_subwarp_id_flat<subwarp_size>(item_ct1);
+    auto tid = thread::get_subwarp_id_flat<subgroup_size>(item_ct1);
     if (tid >= num_rows) {
         return;
     }
-    auto lane = item_ct1.get_local_id(2) % subwarp_size;
+    auto lane = item_ct1.get_local_id(2) % subgroup_size;
     auto in_row = tid;
     auto out_row = permutation[tid];
     auto in_begin = in_row_ptrs[in_row];
     auto in_size = in_row_ptrs[in_row + 1] - in_begin;
     auto out_begin = out_row_ptrs[out_row];
-    for (IndexType i = lane; i < in_size; i += subwarp_size) {
+    for (IndexType i = lane; i < in_size; i += subgroup_size) {
         out_cols[out_begin + i] = in_cols[in_begin + i];
         out_vals[out_begin + i] = in_vals[in_begin + i];
     }
 }
 
-template <int subwarp_size, typename ValueType, typename IndexType>
+template <int subgroup_size, typename ValueType, typename IndexType>
 void inv_row_permute_kernel(dim3 grid, dim3 block,
                             size_type dynamic_shared_memory, sycl::queue *queue,
                             size_type num_rows, const IndexType *permutation,
@@ -1343,7 +1270,7 @@ void inv_row_permute_kernel(dim3 grid, dim3 block,
     queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                inv_row_permute_kernel<subwarp_size>(
+                inv_row_permute_kernel<subgroup_size>(
                     num_rows, permutation, in_row_ptrs, in_cols, in_vals,
                     out_row_ptrs, out_cols, out_vals, item_ct1);
             });
@@ -1351,7 +1278,7 @@ void inv_row_permute_kernel(dim3 grid, dim3 block,
 }
 
 
-template <int subwarp_size, typename ValueType, typename IndexType>
+template <int subgroup_size, typename ValueType, typename IndexType>
 void inv_symm_permute_kernel(size_type num_rows,
                              const IndexType *__restrict__ permutation,
                              const IndexType *__restrict__ in_row_ptrs,
@@ -1362,23 +1289,23 @@ void inv_symm_permute_kernel(size_type num_rows,
                              ValueType *__restrict__ out_vals,
                              sycl::nd_item<3> item_ct1)
 {
-    auto tid = thread::get_subwarp_id_flat<subwarp_size>(item_ct1);
+    auto tid = thread::get_subwarp_id_flat<subgroup_size>(item_ct1);
     if (tid >= num_rows) {
         return;
     }
-    auto lane = item_ct1.get_local_id(2) % subwarp_size;
+    auto lane = item_ct1.get_local_id(2) % subgroup_size;
     auto in_row = tid;
     auto out_row = permutation[tid];
     auto in_begin = in_row_ptrs[in_row];
     auto in_size = in_row_ptrs[in_row + 1] - in_begin;
     auto out_begin = out_row_ptrs[out_row];
-    for (IndexType i = lane; i < in_size; i += subwarp_size) {
+    for (IndexType i = lane; i < in_size; i += subgroup_size) {
         out_cols[out_begin + i] = permutation[in_cols[in_begin + i]];
         out_vals[out_begin + i] = in_vals[in_begin + i];
     }
 }
 
-template <int subwarp_size, typename ValueType, typename IndexType>
+template <int subgroup_size, typename ValueType, typename IndexType>
 void inv_symm_permute_kernel(dim3 grid, dim3 block,
                              size_type dynamic_shared_memory,
                              sycl::queue *queue, size_type num_rows,
@@ -1391,7 +1318,7 @@ void inv_symm_permute_kernel(dim3 grid, dim3 block,
     queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                inv_symm_permute_kernel<subwarp_size>(
+                inv_symm_permute_kernel<subgroup_size>(
                     num_rows, permutation, in_row_ptrs, in_cols, in_vals,
                     out_row_ptrs, out_cols, out_vals, item_ct1);
             });
@@ -1471,8 +1398,8 @@ int compute_items_per_thread(std::shared_ptr<const DpcppExecutor> exec)
 }
 
 
-template <int subwarp_size, typename ValueType, typename IndexType>
-void classical_spmv(syn::value_list<int, subwarp_size>,
+template <int subgroup_size, typename ValueType, typename IndexType>
+void classical_spmv(syn::value_list<int, subgroup_size>,
                     std::shared_ptr<const DpcppExecutor> exec,
                     const matrix::Csr<ValueType, IndexType> *a,
                     const matrix::Dense<ValueType> *b,
@@ -1484,20 +1411,20 @@ void classical_spmv(syn::value_list<int, subwarp_size>,
     const auto nwarps =
         exec->get_num_computing_units() * threads_per_cu * classical_overweight;
     const auto gridx =
-        std::min(ceildiv(a->get_size()[0], spmv_block_size / subwarp_size),
+        std::min(ceildiv(a->get_size()[0], spmv_block_size / subgroup_size),
                  int64(nwarps / warps_in_block));
     const dim3 grid(gridx, b->get_size()[1]);
     const dim3 block(spmv_block_size);
 
     if (alpha == nullptr && beta == nullptr) {
-        kernel::abstract_classical_spmv<subwarp_size>(
+        kernel::abstract_classical_spmv<subgroup_size>(
             grid, block, 0, exec->get_queue(), a->get_size()[0],
             a->get_const_values(), a->get_const_col_idxs(),
             a->get_const_row_ptrs(), b->get_const_values(), b->get_stride(),
             c->get_values(), c->get_stride());
 
     } else if (alpha != nullptr && beta != nullptr) {
-        kernel::abstract_classical_spmv<subwarp_size>(
+        kernel::abstract_classical_spmv<subgroup_size>(
             grid, block, 0, exec->get_queue(), a->get_size()[0],
             alpha->get_const_values(), a->get_const_values(),
             a->get_const_col_idxs(), a->get_const_row_ptrs(),
@@ -2454,18 +2381,6 @@ void conj_transpose(std::shared_ptr<const DpcppExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONJ_TRANSPOSE_KERNEL);
 
-template <typename IndexType>
-void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
-                        size_type size, const IndexType *permutation_indices,
-                        IndexType *inv_permutation)
-{
-    auto num_blocks = ceildiv(size, default_block_size);
-    inv_permutation_kernel(num_blocks, default_block_size, 0, exec->get_queue(),
-                           size, permutation_indices, inv_permutation);
-}
-
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
-
 
 template <typename ValueType, typename IndexType>
 void inv_symm_permute(std::shared_ptr<const DpcppExecutor> exec,
@@ -2542,26 +2457,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(std::shared_ptr<const DpcppExecutor> exec,
-                            const IndexType *perm,
-                            const matrix::Csr<ValueType, IndexType> *orig,
-                            matrix::Csr<ValueType, IndexType> *column_permuted)
-{
-    auto num_rows = orig->get_size()[0];
-    auto nnz = orig->get_num_stored_elements();
-    auto num_blocks = ceildiv(std::max(num_rows, nnz), default_block_size);
-    col_permute_kernel(
-        num_blocks, default_block_size, 0, exec->get_queue(), num_rows, nnz,
-        perm, orig->get_const_row_ptrs(), orig->get_const_col_idxs(),
-        orig->get_const_values(), column_permuted->get_row_ptrs(),
-        column_permuted->get_col_idxs(), column_permuted->get_values());
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void calculate_max_nnz_per_row(std::shared_ptr<const DpcppExecutor> exec,
                                const matrix::Csr<ValueType, IndexType> *source,
diff --git a/dpcpp/matrix/ell_kernels.dp.cpp b/dpcpp/matrix/ell_kernels.dp.cpp
index 34ea5e06dc5..7525c85eae9 100644
--- a/dpcpp/matrix/ell_kernels.dp.cpp
+++ b/dpcpp/matrix/ell_kernels.dp.cpp
@@ -121,7 +121,7 @@ void spmv_kernel(
     acc::range<b_accessor> b, OutputValueType *__restrict__ c,
     const size_type c_stride, Closure op, sycl::nd_item<3> item_ct1,
     UninitializedArray<OutputValueType,
-                       default_block_size / num_thread_per_worker> *storage)
+                       default_block_size / num_thread_per_worker> &storage)
 {
     const auto tidx = thread::get_thread_id_flat(item_ct1);
     const decltype(tidx) column_id = item_ct1.get_group(1);
@@ -150,7 +150,7 @@ void spmv_kernel(
         const auto step_size = num_worker_per_row * num_thread_per_worker;
 
         if (runnable && idx_in_worker == 0) {
-            (*storage)[item_ct1.get_local_id(2)] = 0;
+            storage[item_ct1.get_local_id(2)] = 0;
         }
 
         item_ct1.barrier(sycl::access::fence_space::local_space);
@@ -167,8 +167,8 @@ void spmv_kernel(
                     temp += val(ind) * b(col_idx, column_id);
                 }
             }
-            atomic_add<atomic::local_space>(
-                &(*storage)[item_ct1.get_local_id(2)], temp);
+            atomic_add<atomic::local_space>(&storage[item_ct1.get_local_id(2)],
+                                            temp);
         }
 
         item_ct1.barrier(sycl::access::fence_space::local_space);
@@ -176,9 +176,9 @@ void spmv_kernel(
             const auto c_ind = x * c_stride + column_id;
             if (atomic) {
                 atomic_add(&(c[c_ind]),
-                           op((*storage)[item_ct1.get_local_id(2)], c[c_ind]));
+                           op(storage[item_ct1.get_local_id(2)], c[c_ind]));
             } else {
-                c[c_ind] = op((*storage)[item_ct1.get_local_id(2)], c[c_ind]);
+                c[c_ind] = op(storage[item_ct1.get_local_id(2)], c[c_ind]);
             }
         }
     }
@@ -194,7 +194,7 @@ void spmv(
     acc::range<b_accessor> b, OutputValueType *__restrict__ c,
     const size_type c_stride, sycl::nd_item<3> item_ct1,
     UninitializedArray<OutputValueType,
-                       default_block_size / num_thread_per_worker> *storage)
+                       default_block_size / num_thread_per_worker> &storage)
 {
     spmv_kernel<num_thread_per_worker, atomic>(
         num_rows, num_worker_per_row, val, col, stride,
@@ -224,7 +224,7 @@ void spmv(dim3 grid, dim3 block, size_type dynamic_shared_memory,
                              spmv<num_thread_per_worker, atomic>(
                                  num_rows, num_worker_per_row, val, col, stride,
                                  num_stored_elements_per_row, b, c, c_stride,
-                                 item_ct1, storage_acc_ct1.get_pointer().get());
+                                 item_ct1, *storage_acc_ct1.get_pointer());
                          });
     });
 }
@@ -240,7 +240,7 @@ void spmv(
     const OutputValueType *__restrict__ beta, OutputValueType *__restrict__ c,
     const size_type c_stride, sycl::nd_item<3> item_ct1,
     UninitializedArray<OutputValueType,
-                       default_block_size / num_thread_per_worker> *storage)
+                       default_block_size / num_thread_per_worker> &storage)
 {
     const OutputValueType alpha_val = alpha(0);
     const OutputValueType beta_val = beta[0];
@@ -291,7 +291,7 @@ void spmv(dim3 grid, dim3 block, size_type dynamic_shared_memory,
                 spmv<num_thread_per_worker, atomic>(
                     num_rows, num_worker_per_row, alpha, val, col, stride,
                     num_stored_elements_per_row, b, beta, c, c_stride, item_ct1,
-                    storage_acc_ct1.get_pointer().get());
+                    *storage_acc_ct1.get_pointer());
             });
     });
 }
diff --git a/dpcpp/matrix/hybrid_kernels.dp.cpp b/dpcpp/matrix/hybrid_kernels.dp.cpp
index 2d0684d0cba..13fa02a3331 100644
--- a/dpcpp/matrix/hybrid_kernels.dp.cpp
+++ b/dpcpp/matrix/hybrid_kernels.dp.cpp
@@ -84,7 +84,7 @@ namespace kernel {
  * @param row  the row index array of the matrix
  * @param nnz_per_row  the output nonzeros per row
  */
-template <int subwarp_size = config::warp_size, typename ValueType,
+template <int subgroup_size = config::warp_size, typename ValueType,
           typename IndexType>
 void count_coo_row_nnz(const size_type nnz, const size_type num_lines,
                        const ValueType *__restrict__ val,
@@ -99,22 +99,22 @@ void count_coo_row_nnz(const size_type nnz, const size_type num_lines,
             num_lines +
         item_ct1.get_local_id(1) * item_ct1.get_local_range().get(2) *
             num_lines;
-    size_type num = (nnz > start) * ceildiv(nnz - start, subwarp_size);
+    size_type num = (nnz > start) * ceildiv(nnz - start, subgroup_size);
     num = min(num, num_lines);
     const IndexType ind_start = start + item_ct1.get_local_id(2);
-    const IndexType ind_end = ind_start + (num - 1) * subwarp_size;
+    const IndexType ind_end = ind_start + (num - 1) * subgroup_size;
     IndexType ind = ind_start;
     IndexType curr_row = (ind < nnz) ? row[ind] : 0;
-    const auto tile_block = group::tiled_partition<subwarp_size>(
+    const auto tile_block = group::tiled_partition<subgroup_size>(
         group::this_thread_block(item_ct1));
-    for (; ind < ind_end; ind += subwarp_size) {
+    for (; ind < ind_end; ind += subgroup_size) {
         temp_val += ind < nnz && val[ind] != zero<ValueType>();
-        auto next_row =
-            (ind + subwarp_size < nnz) ? row[ind + subwarp_size] : row[nnz - 1];
+        auto next_row = (ind + subgroup_size < nnz) ? row[ind + subgroup_size]
+                                                    : row[nnz - 1];
         // segmented scan
         if (tile_block.any(curr_row != next_row)) {
             bool is_first_in_segment =
-                segment_scan<subwarp_size>(tile_block, curr_row, &temp_val);
+                segment_scan<subgroup_size>(tile_block, curr_row, &temp_val);
             if (is_first_in_segment) {
                 atomic_add(&(nnz_per_row[curr_row]), temp_val);
             }
@@ -128,14 +128,14 @@ void count_coo_row_nnz(const size_type nnz, const size_type num_lines,
         // segmented scan
 
         bool is_first_in_segment =
-            segment_scan<subwarp_size>(tile_block, curr_row, &temp_val);
+            segment_scan<subgroup_size>(tile_block, curr_row, &temp_val);
         if (is_first_in_segment) {
             atomic_add(&(nnz_per_row[curr_row]), temp_val);
         }
     }
 }
 
-template <int subwarp_size = config::warp_size, typename ValueType,
+template <int subgroup_size = config::warp_size, typename ValueType,
           typename IndexType>
 void count_coo_row_nnz(dim3 grid, dim3 block, size_type dynamic_shared_memory,
                        sycl::queue *queue, const size_type nnz,
@@ -145,8 +145,8 @@ void count_coo_row_nnz(dim3 grid, dim3 block, size_type dynamic_shared_memory,
     queue->submit([&](sycl::handler &cgh) {
         cgh.parallel_for(
             sycl_nd_range(grid, block), [=](sycl::nd_item<3> item_ct1) {
-                count_coo_row_nnz<subwarp_size>(nnz, num_lines, val, row,
-                                                nnz_per_row, item_ct1);
+                count_coo_row_nnz<subgroup_size>(nnz, num_lines, val, row,
+                                                 nnz_per_row, item_ct1);
             });
     });
 }
diff --git a/dpcpp/test/base/executor.dp.cpp b/dpcpp/test/base/executor.dp.cpp
index 830543349d5..93f52a7d1f1 100644
--- a/dpcpp/test/base/executor.dp.cpp
+++ b/dpcpp/test/base/executor.dp.cpp
@@ -269,30 +269,23 @@ TEST_F(DpcppExecutor, Synchronizes)
 }
 
 
-#define GTEST_ASSERT_NO_EXIT(statement) \
-    ASSERT_EXIT({ {statement} exit(0); }, ::testing::ExitedWithCode(0), "")
-
-
 TEST_F(DpcppExecutor, FreeAfterKernel)
 {
-    testing::FLAGS_gtest_death_test_style = "threadsafe";
-    GTEST_ASSERT_NO_EXIT({
-        size_t length = 10000;
-        auto dpcpp =
-            gko::DpcppExecutor::create(0, gko::ReferenceExecutor::create());
-        {
-            gko::Array<float> x(dpcpp, length);
-            gko::Array<float> y(dpcpp, length);
-            auto x_val = x.get_data();
-            auto y_val = y.get_data();
-            dpcpp->get_queue()->submit([&](sycl::handler &cgh) {
-                cgh.parallel_for(sycl::range<1>{length},
-                                 [=](sycl::id<1> i) { y_val[i] += x_val[i]; });
-            });
-        }
-        // to ensure everything on queue is finished.
-        dpcpp->synchronize();
-    });
+    size_t length = 10000;
+    auto dpcpp =
+        gko::DpcppExecutor::create(0, gko::ReferenceExecutor::create());
+    {
+        gko::Array<float> x(dpcpp, length);
+        gko::Array<float> y(dpcpp, length);
+        auto x_val = x.get_data();
+        auto y_val = y.get_data();
+        dpcpp->get_queue()->submit([&](sycl::handler &cgh) {
+            cgh.parallel_for(sycl::range<1>{length},
+                             [=](sycl::id<1> i) { y_val[i] += x_val[i]; });
+        });
+    }
+    // to ensure everything on queue is finished.
+    dpcpp->synchronize();
 }
 
 
diff --git a/dpcpp/test/matrix/fbcsr_kernels.cpp b/dpcpp/test/matrix/fbcsr_kernels.cpp
index c67de961cd7..5b2dcf19628 100644
--- a/dpcpp/test/matrix/fbcsr_kernels.cpp
+++ b/dpcpp/test/matrix/fbcsr_kernels.cpp
@@ -82,9 +82,9 @@ TEST_F(Fbcsr, CanWriteFromMatrixOnDevice)
     auto refmat = sample.generate_fbcsr();
     auto dpcppmat = Mtx::create(dpcpp);
     dpcppmat->copy_from(gko::lend(refmat));
-
     MatData refdata;
     MatData dpcppdata;
+
     refmat->write(refdata);
     dpcppmat->write(dpcppdata);
 
diff --git a/dpcpp/test/matrix/sellp_kernels.cpp b/dpcpp/test/matrix/sellp_kernels.cpp
index 91e7ae9d5c8..4fd072227bd 100644
--- a/dpcpp/test/matrix/sellp_kernels.cpp
+++ b/dpcpp/test/matrix/sellp_kernels.cpp
@@ -138,9 +138,7 @@ TEST_F(Sellp, SimpleApplyIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -151,9 +149,7 @@ TEST_F(Sellp, AdvancedApplyIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -164,9 +160,7 @@ TEST_F(Sellp, SimpleApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -177,9 +171,7 @@ TEST_F(Sellp, AdvancedApplyWithSliceSizeAndStrideFActorIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -187,13 +179,10 @@ TEST_F(Sellp, SimpleApplyMultipleRHSIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
 
-
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -201,13 +190,10 @@ TEST_F(Sellp, AdvancedApplyMultipleRHSIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
 
-
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -219,9 +205,7 @@ TEST_F(Sellp,
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
@@ -233,16 +217,13 @@ TEST_F(Sellp,
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, r<vtype>::value);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, r<vtype>::value);
 }
 
 
 TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(dpcpp);
     dcomplex_b->copy_from(complex_b.get());
@@ -260,7 +241,6 @@ TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(dpcpp);
     dcomplex_b->copy_from(complex_b.get());
@@ -278,8 +258,6 @@ TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
-
     auto dense_mtx = gko::matrix::Dense<vtype>::create(ref);
     auto ddense_mtx = gko::matrix::Dense<vtype>::create(dpcpp);
 
@@ -293,8 +271,6 @@ TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 TEST_F(Sellp, ConvertToCsrIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
-
     auto csr_mtx = gko::matrix::Csr<vtype>::create(ref);
     auto dcsr_mtx = gko::matrix::Csr<vtype>::create(dpcpp);
 
@@ -308,8 +284,6 @@ TEST_F(Sellp, ConvertToCsrIsEquivalentToRef)
 TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
-
     auto dense_mtx = gko::matrix::Dense<vtype>::create(ref);
     auto ddense_mtx = gko::matrix::Dense<vtype>::create(dpcpp);
 
@@ -323,8 +297,6 @@ TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 TEST_F(Sellp, ConvertEmptyToCsrIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
-
     auto csr_mtx = gko::matrix::Csr<vtype>::create(ref);
     auto dcsr_mtx = gko::matrix::Csr<vtype>::create(dpcpp);
 
@@ -338,8 +310,6 @@ TEST_F(Sellp, ConvertEmptyToCsrIsEquivalentToRef)
 TEST_F(Sellp, CountNonzerosIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
-
     gko::size_type nnz;
     gko::size_type dnnz;
 
@@ -354,7 +324,6 @@ TEST_F(Sellp, ExtractDiagonalIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
 
-
     auto diag = mtx->extract_diagonal();
     auto ddiag = dmtx->extract_diagonal();
 
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index dc63148940c..0778c41a39e 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -187,6 +187,7 @@ set(GINKGO_HIP_SOURCES
     stop/residual_norm_kernels.hip.cpp
     ../common/components/precision_conversion.cpp
     ../common/matrix/coo_kernels.cpp
+    ../common/matrix/csr_kernels.cpp
     ../common/matrix/dense_kernels.cpp
     ../common/matrix/diagonal_kernels.cpp
     ../common/solver/bicg_kernels.cpp
diff --git a/hip/matrix/csr_kernels.hip.cpp b/hip/matrix/csr_kernels.hip.cpp
index b58ef80fea3..318d9e60857 100644
--- a/hip/matrix/csr_kernels.hip.cpp
+++ b/hip/matrix/csr_kernels.hip.cpp
@@ -966,20 +966,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename IndexType>
-void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
-                        size_type size, const IndexType *permutation_indices,
-                        IndexType *inv_permutation)
-{
-    auto num_blocks = ceildiv(size, default_block_size);
-    hipLaunchKernelGGL(HIP_KERNEL_NAME(inv_permutation_kernel), num_blocks,
-                       default_block_size, 0, 0, size, permutation_indices,
-                       inv_permutation);
-}
-
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void inv_symm_permute(std::shared_ptr<const HipExecutor> exec,
                       const IndexType *perm,
@@ -1060,27 +1046,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(std::shared_ptr<const HipExecutor> exec,
-                            const IndexType *perm,
-                            const matrix::Csr<ValueType, IndexType> *orig,
-                            matrix::Csr<ValueType, IndexType> *column_permuted)
-{
-    auto num_rows = orig->get_size()[0];
-    auto nnz = orig->get_num_stored_elements();
-    auto num_blocks = ceildiv(std::max(num_rows, nnz), default_block_size);
-    hipLaunchKernelGGL(
-        HIP_KERNEL_NAME(col_permute_kernel), num_blocks, default_block_size, 0,
-        0, num_rows, nnz, perm, orig->get_const_row_ptrs(),
-        orig->get_const_col_idxs(), as_hip_type(orig->get_const_values()),
-        column_permuted->get_row_ptrs(), column_permuted->get_col_idxs(),
-        as_hip_type(column_permuted->get_values()));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void calculate_max_nnz_per_row(std::shared_ptr<const HipExecutor> exec,
                                const matrix::Csr<ValueType, IndexType> *source,
diff --git a/hip/test/matrix/CMakeLists.txt b/hip/test/matrix/CMakeLists.txt
index 94e92f08f5c..4a32b5272f7 100644
--- a/hip/test/matrix/CMakeLists.txt
+++ b/hip/test/matrix/CMakeLists.txt
@@ -3,5 +3,6 @@ ginkgo_create_hip_test(csr_kernels)
 ginkgo_create_hip_test(dense_kernels)
 ginkgo_create_hip_test(diagonal_kernels)
 ginkgo_create_hip_test(ell_kernels)
+ginkgo_create_hip_test(fbcsr_kernels)
 ginkgo_create_hip_test(hybrid_kernels)
 ginkgo_create_hip_test(sellp_kernels)
diff --git a/hip/test/matrix/fbcsr_kernels.hip.cpp b/hip/test/matrix/fbcsr_kernels.hip.cpp
new file mode 100644
index 00000000000..30c11fba2b8
--- /dev/null
+++ b/hip/test/matrix/fbcsr_kernels.hip.cpp
@@ -0,0 +1,93 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/matrix/fbcsr.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include "core/test/matrix/fbcsr_sample.hpp"
+#include "hip/test/utils.hip.hpp"
+
+
+namespace {
+
+
+class Fbcsr : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Fbcsr<>;
+
+    void SetUp()
+    {
+        ASSERT_GT(gko::HipExecutor::get_num_devices(), 0);
+        ref = gko::ReferenceExecutor::create();
+        hip = gko::HipExecutor::create(0, ref);
+    }
+
+    void TearDown()
+    {
+        if (hip != nullptr) {
+            ASSERT_NO_THROW(hip->synchronize());
+        }
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::HipExecutor> hip;
+
+    std::unique_ptr<Mtx> mtx;
+};
+
+
+TEST_F(Fbcsr, CanWriteFromMatrixOnDevice)
+{
+    using value_type = Mtx::value_type;
+    using index_type = Mtx::index_type;
+    using MatData = gko::matrix_data<value_type, index_type>;
+    gko::testing::FbcsrSample<value_type, index_type> sample(ref);
+    auto refmat = sample.generate_fbcsr();
+    auto hipmat = Mtx::create(hip);
+    hipmat->copy_from(gko::lend(refmat));
+    MatData refdata;
+    MatData hipdata;
+
+    refmat->write(refdata);
+    hipmat->write(hipdata);
+
+    ASSERT_TRUE(refdata.nonzeros == hipdata.nonzeros);
+}
+
+
+}  // namespace
diff --git a/hip/test/matrix/sellp_kernels.hip.cpp b/hip/test/matrix/sellp_kernels.hip.cpp
index 89dee5466e4..5df39fc5356 100644
--- a/hip/test/matrix/sellp_kernels.hip.cpp
+++ b/hip/test/matrix/sellp_kernels.hip.cpp
@@ -136,9 +136,7 @@ TEST_F(Sellp, SimpleApplyIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -149,9 +147,7 @@ TEST_F(Sellp, AdvancedApplyIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -162,9 +158,7 @@ TEST_F(Sellp, SimpleApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -175,9 +169,7 @@ TEST_F(Sellp, AdvancedApplyWithSliceSizeAndStrideFActorIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -185,13 +177,10 @@ TEST_F(Sellp, SimpleApplyMultipleRHSIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
 
-
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -199,13 +188,10 @@ TEST_F(Sellp, AdvancedApplyMultipleRHSIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
 
-
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -217,9 +203,7 @@ TEST_F(Sellp,
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -231,16 +215,13 @@ TEST_F(Sellp,
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
 TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(hip);
     dcomplex_b->copy_from(complex_b.get());
@@ -258,7 +239,6 @@ TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(hip);
     dcomplex_b->copy_from(complex_b.get());
@@ -276,7 +256,6 @@ TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto dense_mtx = gko::matrix::Dense<>::create(ref);
     auto ddense_mtx = gko::matrix::Dense<>::create(hip);
 
@@ -304,7 +283,6 @@ TEST_F(Sellp, ConvertToCsrIsEquivalentToRef)
 TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto dense_mtx = gko::matrix::Dense<>::create(ref);
     auto ddense_mtx = gko::matrix::Dense<>::create(hip);
 
@@ -318,7 +296,6 @@ TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 TEST_F(Sellp, ConvertEmptyToCsrIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto csr_mtx = gko::matrix::Csr<>::create(ref);
     auto dcsr_mtx = gko::matrix::Csr<>::create(hip);
 
@@ -332,7 +309,6 @@ TEST_F(Sellp, ConvertEmptyToCsrIsEquivalentToRef)
 TEST_F(Sellp, CountNonzerosIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     gko::size_type nnz;
     gko::size_type dnnz;
 
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index a9e38d8f6e9..faf49154f34 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -36,6 +36,7 @@ target_sources(ginkgo_omp
     stop/residual_norm_kernels.cpp
     ../common/components/precision_conversion.cpp
     ../common/matrix/coo_kernels.cpp
+    ../common/matrix/csr_kernels.cpp
     ../common/matrix/dense_kernels.cpp
     ../common/matrix/diagonal_kernels.cpp
     ../common/solver/bicg_kernels.cpp
diff --git a/omp/matrix/csr_kernels.cpp b/omp/matrix/csr_kernels.cpp
index 4b56fe27048..8b0e9f5a81d 100644
--- a/omp/matrix/csr_kernels.cpp
+++ b/omp/matrix/csr_kernels.cpp
@@ -789,20 +789,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_CONVERT_TO_HYBRID_KERNEL);
 
 
-template <typename IndexType>
-void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
-                        size_type size, const IndexType *permutation_indices,
-                        IndexType *inv_permutation)
-{
-#pragma omp parallel for
-    for (IndexType i = 0; i < static_cast<IndexType>(size); ++i) {
-        inv_permutation[permutation_indices[i]] = i;
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_INDEX_TYPE(GKO_DECLARE_INVERT_PERMUTATION_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void inv_symm_permute(std::shared_ptr<const DefaultExecutor> exec,
                       const IndexType *perm,
@@ -919,35 +905,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_CSR_INVERSE_ROW_PERMUTE_KERNEL);
 
 
-template <typename ValueType, typename IndexType>
-void inverse_column_permute(std::shared_ptr<const OmpExecutor> exec,
-                            const IndexType *perm,
-                            const matrix::Csr<ValueType, IndexType> *orig,
-                            matrix::Csr<ValueType, IndexType> *column_permuted)
-{
-    auto orig_row_ptrs = orig->get_const_row_ptrs();
-    auto orig_col_idxs = orig->get_const_col_idxs();
-    auto orig_vals = orig->get_const_values();
-    auto cp_row_ptrs = column_permuted->get_row_ptrs();
-    auto cp_col_idxs = column_permuted->get_col_idxs();
-    auto cp_vals = column_permuted->get_values();
-    size_type num_rows = orig->get_size()[0];
-
-#pragma omp parallel for
-    for (size_type row = 0; row < num_rows; ++row) {
-        cp_row_ptrs[row] = orig_row_ptrs[row];
-        for (auto k = orig_row_ptrs[row]; k < orig_row_ptrs[row + 1]; ++k) {
-            cp_col_idxs[k] = perm[orig_col_idxs[k]];
-            cp_vals[k] = orig_vals[k];
-        }
-    }
-    cp_row_ptrs[num_rows] = orig_row_ptrs[num_rows];
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
-    GKO_DECLARE_CSR_INVERSE_COLUMN_PERMUTE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void calculate_nonzeros_per_row(std::shared_ptr<const OmpExecutor> exec,
                                 const matrix::Csr<ValueType, IndexType> *source,
diff --git a/omp/matrix/sparsity_csr_kernels.cpp b/omp/matrix/sparsity_csr_kernels.cpp
index 903e4c43414..fee834f188e 100644
--- a/omp/matrix/sparsity_csr_kernels.cpp
+++ b/omp/matrix/sparsity_csr_kernels.cpp
@@ -54,7 +54,7 @@ namespace gko {
 namespace kernels {
 namespace omp {
 /**
- * @brief The SparsityCsr pattern format namespace.
+ * @brief The Compressed sparse row matrix format namespace.
  *
  * @ingroup sparsity
  */
diff --git a/omp/test/matrix/CMakeLists.txt b/omp/test/matrix/CMakeLists.txt
index 0c9fd00eb30..ff8ed2d0118 100644
--- a/omp/test/matrix/CMakeLists.txt
+++ b/omp/test/matrix/CMakeLists.txt
@@ -3,6 +3,7 @@ ginkgo_create_test(csr_kernels)
 ginkgo_create_test(dense_kernels)
 ginkgo_create_test(diagonal_kernels)
 ginkgo_create_test(ell_kernels)
+ginkgo_create_test(fbcsr_kernels)
 ginkgo_create_test(hybrid_kernels)
 ginkgo_create_test(sellp_kernels)
 ginkgo_create_test(sparsity_csr_kernels)
diff --git a/omp/test/matrix/fbcsr_kernels.cpp b/omp/test/matrix/fbcsr_kernels.cpp
new file mode 100644
index 00000000000..43837790006
--- /dev/null
+++ b/omp/test/matrix/fbcsr_kernels.cpp
@@ -0,0 +1,92 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/matrix/fbcsr.hpp>
+
+
+#include <gtest/gtest.h>
+
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include "core/test/matrix/fbcsr_sample.hpp"
+#include "core/test/utils.hpp"
+
+
+namespace {
+
+
+class Fbcsr : public ::testing::Test {
+protected:
+    using Mtx = gko::matrix::Fbcsr<>;
+
+    void SetUp()
+    {
+        ref = gko::ReferenceExecutor::create();
+        omp = gko::OmpExecutor::create();
+    }
+
+    void TearDown()
+    {
+        if (omp != nullptr) {
+            ASSERT_NO_THROW(omp->synchronize());
+        }
+    }
+
+    std::shared_ptr<gko::ReferenceExecutor> ref;
+    std::shared_ptr<const gko::OmpExecutor> omp;
+
+    std::unique_ptr<Mtx> mtx;
+};
+
+
+TEST_F(Fbcsr, CanWriteFromMatrixOnDevice)
+{
+    using value_type = Mtx::value_type;
+    using index_type = Mtx::index_type;
+    using MatData = gko::matrix_data<value_type, index_type>;
+    gko::testing::FbcsrSample<value_type, index_type> sample(ref);
+    auto refmat = sample.generate_fbcsr();
+    auto ompmat = Mtx::create(omp);
+    ompmat->copy_from(gko::lend(refmat));
+    MatData refdata;
+    MatData ompdata;
+
+    refmat->write(refdata);
+    ompmat->write(ompdata);
+
+    ASSERT_TRUE(refdata.nonzeros == ompdata.nonzeros);
+}
+
+
+}  // namespace
diff --git a/omp/test/matrix/sellp_kernels.cpp b/omp/test/matrix/sellp_kernels.cpp
index 6820f8b8b99..081f75708ae 100644
--- a/omp/test/matrix/sellp_kernels.cpp
+++ b/omp/test/matrix/sellp_kernels.cpp
@@ -135,9 +135,7 @@ TEST_F(Sellp, SimpleApplyIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -148,9 +146,7 @@ TEST_F(Sellp, AdvancedApplyIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -161,9 +157,7 @@ TEST_F(Sellp, SimpleApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -174,9 +168,7 @@ TEST_F(Sellp, AdvancedApplyWithSliceSizeAndStrideFactorIsEquivalentToRef)
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -184,13 +176,10 @@ TEST_F(Sellp, SimpleApplyMultipleRHSIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
 
-
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -198,13 +187,10 @@ TEST_F(Sellp, AdvancedApplyMultipleRHSIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
 
-
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -216,9 +202,7 @@ TEST_F(Sellp,
     mtx->apply(y.get(), expected.get());
     dmtx->apply(dy.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
@@ -230,16 +214,13 @@ TEST_F(Sellp,
     mtx->apply(alpha.get(), y.get(), beta.get(), expected.get());
     dmtx->apply(dalpha.get(), dy.get(), dbeta.get(), dresult.get());
 
-    auto result = Vec::create(ref);
-    result->copy_from(dresult.get());
-    GKO_ASSERT_MTX_NEAR(result, expected, 1e-14);
+    GKO_ASSERT_MTX_NEAR(dresult, expected, 1e-14);
 }
 
 
 TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(omp);
     dcomplex_b->copy_from(complex_b.get());
@@ -257,7 +238,6 @@ TEST_F(Sellp, ApplyToComplexIsEquivalentToRef)
 TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto complex_b = gen_mtx<ComplexVec>(231, 3);
     auto dcomplex_b = ComplexVec::create(omp);
     dcomplex_b->copy_from(complex_b.get());
@@ -275,7 +255,6 @@ TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef)
 TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto dense_mtx = gko::matrix::Dense<>::create(ref);
     auto ddense_mtx = gko::matrix::Dense<>::create(omp);
 
@@ -289,7 +268,6 @@ TEST_F(Sellp, ConvertToDenseIsEquivalentToRef)
 TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef)
 {
     set_up_apply_matrix(64);
-
     auto dense_mtx = gko::matrix::Dense<>::create(ref);
     auto ddense_mtx = gko::matrix::Dense<>::create(omp);
 
diff --git a/reference/matrix/sparsity_csr_kernels.cpp b/reference/matrix/sparsity_csr_kernels.cpp
index 0150fec8fcf..f70af45a18d 100644
--- a/reference/matrix/sparsity_csr_kernels.cpp
+++ b/reference/matrix/sparsity_csr_kernels.cpp
@@ -51,7 +51,7 @@ namespace gko {
 namespace kernels {
 namespace reference {
 /**
- * @brief The SparsityCsr pattern format namespace.
+ * @brief The Compressed sparse row matrix format namespace.
  * @ref SparsityCsr
  * @ingroup sparsity
  */

From acff91c530fd511d53bb2a9c46a76488865e6b0f Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Thu, 5 Aug 2021 14:36:40 +0200
Subject: [PATCH 0985/1008] add strategy_name for csr strategy

---
 include/ginkgo/core/matrix/csr.hpp | 55 +++++++++++++++++-------------
 1 file changed, 31 insertions(+), 24 deletions(-)

diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp
index 01a37608db1..7bf13f459fd 100644
--- a/include/ginkgo/core/matrix/csr.hpp
+++ b/include/ginkgo/core/matrix/csr.hpp
@@ -374,7 +374,8 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
          *             number of threads in a SIMD unit is 7
          */
         load_balance(std::shared_ptr<const DpcppExecutor> exec)
-            : load_balance(exec->get_num_computing_units() * 7, 16, false)
+            : load_balance(exec->get_num_computing_units() * 7, 16, false,
+                           "intel")
         {}
 
         /**
@@ -389,11 +390,13 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
          *       parameters which is replaced during the conversion.
          */
         load_balance(int64_t nwarps, int warp_size = 32,
-                     bool cuda_strategy = true)
+                     bool cuda_strategy = true,
+                     std::string strategy_name = "none")
             : strategy_type("load_balance"),
               nwarps_(nwarps),
               warp_size_(warp_size),
-              cuda_strategy_(cuda_strategy)
+              cuda_strategy_(cuda_strategy),
+              strategy_name_(strategy_name)
         {}
 
         void process(const Array<index_type> &mtx_row_ptrs,
@@ -460,7 +463,7 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
                 } else if (nnz >= static_cast<int64_t>(2e5)) {
                     multiple = 32;
                 }
-                if (!cuda_strategy_) {
+                if (strategy_name_ == "intel") {
                     multiple = 8;
                     if (nnz >= static_cast<int64_t>(2e8)) {
                         multiple = 256;
@@ -488,14 +491,15 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
 
         std::shared_ptr<strategy_type> copy() override
         {
-            return std::make_shared<load_balance>(nwarps_, warp_size_,
-                                                  cuda_strategy_);
+            return std::make_shared<load_balance>(
+                nwarps_, warp_size_, cuda_strategy_, strategy_name_);
         }
 
     private:
         int64_t nwarps_;
         int warp_size_;
         bool cuda_strategy_;
+        std::string strategy_name_;
     };
 
     class automatical : public strategy_type {
@@ -512,13 +516,8 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
         /* Use imbalance strategy when the matrix has more more than 1e8 on AMD
          * hardware */
         const index_type amd_nnz_limit{static_cast<index_type>(1e8)};
-        /* Use imbalance strategy when the maximum number of nonzero per row is
-         * more than 25600 on Intel hardware. */
-        const index_type intel_row_len_limit = 25600;
-        /* Use imbalance strategy when the matrix has more more than 3e8 on
-         * Intel hardware */
-        const index_type intel_nnz_limit{static_cast<index_type>(3e8)};
 
+    public:
         /**
          * Creates an automatical strategy.
          */
@@ -554,7 +553,8 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
          *             number of threads in a SIMD unit is 7
          */
         automatical(std::shared_ptr<const DpcppExecutor> exec)
-            : automatical(exec->get_num_computing_units() * 7, 16, false)
+            : automatical(exec->get_num_computing_units() * 7, 16, false,
+                          "intel")
         {}
 
         /**
@@ -569,11 +569,13 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
          *       parameters which is replaced during the conversion.
          */
         automatical(int64_t nwarps, int warp_size = 32,
-                    bool cuda_strategy = true)
+                    bool cuda_strategy = true,
+                    std::string strategy_name = "none")
             : strategy_type("automatical"),
               nwarps_(nwarps),
               warp_size_(warp_size),
               cuda_strategy_(cuda_strategy),
+              strategy_name_(strategy_name),
               max_length_per_row_(0)
         {}
 
@@ -585,9 +587,13 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
             // <row_len_limit>, use load_balance otherwise use classical
             index_type nnz_limit = nvidia_nnz_limit;
             index_type row_len_limit = nvidia_row_len_limit;
-            if (!cuda_strategy_) {
-                nnz_limit = intel_nnz_limit;
-                row_len_limit = intel_row_len_limit;
+            if (strategy_name_ == "intel") {
+                /* Use imbalance strategy when the maximum number of nonzero per
+                 * row is more than 25600 on Intel hardware. */
+                nnz_limit = 25600;
+                /* Use imbalance strategy when the matrix has more more than 3e8
+                 * on Intel hardware */
+                row_len_limit = 3e8;
             }
 #if GINKGO_HIP_PLATFORM_HCC
             if (!cuda_strategy_) {
@@ -609,7 +615,7 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
             const auto num_rows = mtx_row_ptrs.get_num_elems() - 1;
             if (row_ptrs[num_rows] > nnz_limit) {
                 load_balance actual_strategy(nwarps_, warp_size_,
-                                             cuda_strategy_);
+                                             cuda_strategy_, strategy_name_);
                 if (is_mtx_on_host) {
                     actual_strategy.process(mtx_row_ptrs, mtx_srow);
                 } else {
@@ -622,8 +628,8 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
                     maxnum = std::max(maxnum, row_ptrs[i + 1] - row_ptrs[i]);
                 }
                 if (maxnum > row_len_limit) {
-                    load_balance actual_strategy(nwarps_, warp_size_,
-                                                 cuda_strategy_);
+                    load_balance actual_strategy(
+                        nwarps_, warp_size_, cuda_strategy_, strategy_name_);
                     if (is_mtx_on_host) {
                         actual_strategy.process(mtx_row_ptrs, mtx_srow);
                     } else {
@@ -648,8 +654,8 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
 
         int64_t clac_size(const int64_t nnz) override
         {
-            return std::make_shared<load_balance>(nwarps_, warp_size_,
-                                                  cuda_strategy_)
+            return std::make_shared<load_balance>(
+                       nwarps_, warp_size_, cuda_strategy_, strategy_name_)
                 ->clac_size(nnz);
         }
 
@@ -660,14 +666,15 @@ class Csr : public EnableLinOp<Csr<ValueType, IndexType>>,
 
         std::shared_ptr<strategy_type> copy() override
         {
-            return std::make_shared<automatical>(nwarps_, warp_size_,
-                                                 cuda_strategy_);
+            return std::make_shared<automatical>(
+                nwarps_, warp_size_, cuda_strategy_, strategy_name_);
         }
 
     private:
         int64_t nwarps_;
         int warp_size_;
         bool cuda_strategy_;
+        std::string strategy_name_;
         index_type max_length_per_row_;
     };
 

From 7137a6f475804018596d7b2b5d32403caa0af9fe Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Fri, 6 Aug 2021 17:22:27 +0200
Subject: [PATCH 0986/1008] use subgroup_tile sync, relax idr errorbound

---
 dpcpp/components/reduction.dp.hpp | 7 ++++---
 dpcpp/matrix/csr_kernels.dp.cpp   | 8 ++++----
 dpcpp/test/solver/idr_kernels.cpp | 8 ++++----
 3 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/dpcpp/components/reduction.dp.hpp b/dpcpp/components/reduction.dp.hpp
index 0cf6e49e068..e7f7c8f5ab6 100644
--- a/dpcpp/components/reduction.dp.hpp
+++ b/dpcpp/components/reduction.dp.hpp
@@ -139,8 +139,8 @@ __dpct_inline__ int choose_pivot(const Group &group, ValueType local_data,
  * array.
  */
 template <
-    unsigned int sg_size = 32, typename Group, typename ValueType,
-    typename Operator,
+    unsigned int sg_size = config::warp_size, typename Group,
+    typename ValueType, typename Operator,
     typename = std::enable_if_t<group::is_synchronizable_group<Group>::value>>
 void reduce(const Group &__restrict__ group, ValueType *__restrict__ data,
             Operator reduce_op = Operator{})
@@ -174,7 +174,8 @@ void reduce(const Group &__restrict__ group, ValueType *__restrict__ data,
  * `source` of any size. Has to be called a second time on `result` to reduce
  * an array larger than `block_size`.
  */
-template <unsigned int sg_size = 32, typename Operator, typename ValueType>
+template <unsigned int sg_size = config::warp_size, typename Operator,
+          typename ValueType>
 void reduce_array(size_type size, const ValueType *__restrict__ source,
                   ValueType *__restrict__ result, sycl::nd_item<3> item_ct1,
                   Operator reduce_op = Operator{})
diff --git a/dpcpp/matrix/csr_kernels.dp.cpp b/dpcpp/matrix/csr_kernels.dp.cpp
index ecb4d2e3e49..3bbd03de871 100644
--- a/dpcpp/matrix/csr_kernels.dp.cpp
+++ b/dpcpp/matrix/csr_kernels.dp.cpp
@@ -645,7 +645,7 @@ void device_classical_spmv(const size_type num_rows,
             subgroup_tile, temp_val,
             [](const ValueType &a, const ValueType &b) { return a + b; });
         // TODO: check the barrier
-        item_ct1.barrier();
+        subgroup_tile.sync();
         if (subid == 0) {
             c[row * c_stride + column_id] =
                 scale(subgroup_result, c[row * c_stride + column_id]);
@@ -1408,11 +1408,12 @@ void classical_spmv(syn::value_list<int, subgroup_size>,
                     const matrix::Dense<ValueType> *beta = nullptr)
 {
     constexpr int threads_per_cu = 7;
-    const auto nwarps =
+    const auto num_subgroup =
         exec->get_num_computing_units() * threads_per_cu * classical_overweight;
+    const auto nsg_in_group = spmv_block_size / subgroup_size;
     const auto gridx =
         std::min(ceildiv(a->get_size()[0], spmv_block_size / subgroup_size),
-                 int64(nwarps / warps_in_block));
+                 int64(num_subgroup / nsg_in_group));
     const dim3 grid(gridx, b->get_size()[1]);
     const dim3 block(spmv_block_size);
 
@@ -1422,7 +1423,6 @@ void classical_spmv(syn::value_list<int, subgroup_size>,
             a->get_const_values(), a->get_const_col_idxs(),
             a->get_const_row_ptrs(), b->get_const_values(), b->get_stride(),
             c->get_values(), c->get_stride());
-
     } else if (alpha != nullptr && beta != nullptr) {
         kernel::abstract_classical_spmv<subgroup_size>(
             grid, block, 0, exec->get_queue(), a->get_size()[0],
diff --git a/dpcpp/test/solver/idr_kernels.cpp b/dpcpp/test/solver/idr_kernels.cpp
index ef5afb132b1..954806069af 100644
--- a/dpcpp/test/solver/idr_kernels.cpp
+++ b/dpcpp/test/solver/idr_kernels.cpp
@@ -273,12 +273,12 @@ TEST_F(Idr, IdrStep3IsEquivalentToRef)
         d_f.get(), d_alpha.get(), d_r.get(), d_x.get(), d_stop_status.get());
 
     GKO_ASSERT_MTX_NEAR(g, d_g, 2 * rr<value_type>::value);
-    GKO_ASSERT_MTX_NEAR(v, d_v, rr<value_type>::value);
-    GKO_ASSERT_MTX_NEAR(u, d_u, rr<value_type>::value);
-    GKO_ASSERT_MTX_NEAR(m, d_m, rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(v, d_v, 2 * rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(u, d_u, 2 * rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(m, d_m, 2 * rr<value_type>::value);
     GKO_ASSERT_MTX_NEAR(f, d_f, 13 * rr<value_type>::value);
     GKO_ASSERT_MTX_NEAR(r, d_r, 2 * rr<value_type>::value);
-    GKO_ASSERT_MTX_NEAR(x, d_x, rr<value_type>::value);
+    GKO_ASSERT_MTX_NEAR(x, d_x, 2 * rr<value_type>::value);
 }
 
 

From bdbd08522bc9055724f6df52f54634078cbb1b2e Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 5 Aug 2021 17:08:02 +0200
Subject: [PATCH 0987/1008] Use invert diagonal instead.

---
 common/preconditioner/jacobi_kernels.hpp.inc |  4 +---
 omp/preconditioner/jacobi_kernels.cpp        |  5 +----
 reference/preconditioner/jacobi_kernels.cpp  | 10 ++--------
 3 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/common/preconditioner/jacobi_kernels.hpp.inc b/common/preconditioner/jacobi_kernels.hpp.inc
index 61678280d56..87d9a9be7a7 100644
--- a/common/preconditioner/jacobi_kernels.hpp.inc
+++ b/common/preconditioner/jacobi_kernels.hpp.inc
@@ -231,11 +231,9 @@ __global__ __launch_bounds__(default_block_size) void scalar_apply(
     const auto col = tidx % num_cols;
 
     if (row < num_rows) {
-        auto diag_val =
-            diag[row] == zero<ValueType>() ? one<ValueType>() : diag[row];
         result_values[row * result_stride + col] =
             result_values[row * result_stride + col] * beta[0] +
-            alpha[0] * source_values[row * source_stride + col] / diag_val;
+            alpha[0] * source_values[row * source_stride + col] * diag[row];
     }
 }
 
diff --git a/omp/preconditioner/jacobi_kernels.cpp b/omp/preconditioner/jacobi_kernels.cpp
index 605ba433578..aa8d5c4bd83 100644
--- a/omp/preconditioner/jacobi_kernels.cpp
+++ b/omp/preconditioner/jacobi_kernels.cpp
@@ -707,11 +707,8 @@ void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
 #pragma omp parallel for
     for (size_type i = 0; i < x->get_size()[0]; ++i) {
         for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            auto diag_val = diag.get_const_data()[i] == zero<ValueType>()
-                                ? one<ValueType>()
-                                : diag.get_const_data()[i];
             x->at(i, j) = beta->at(0) * x->at(i, j) +
-                          alpha->at(0) * b->at(i, j) / diag_val;
+                          alpha->at(0) * b->at(i, j) * diag.get_const_data()[i];
         }
     }
 }
diff --git a/reference/preconditioner/jacobi_kernels.cpp b/reference/preconditioner/jacobi_kernels.cpp
index a176b01e4bf..a3472c6d70d 100644
--- a/reference/preconditioner/jacobi_kernels.cpp
+++ b/reference/preconditioner/jacobi_kernels.cpp
@@ -570,11 +570,8 @@ void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
 {
     for (size_type i = 0; i < x->get_size()[0]; ++i) {
         for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            auto diag_val = diag.get_const_data()[i] == zero<ValueType>()
-                                ? one<ValueType>()
-                                : diag.get_const_data()[i];
             x->at(i, j) = beta->at(0) * x->at(i, j) +
-                          alpha->at(0) * b->at(i, j) / diag_val;
+                          alpha->at(0) * b->at(i, j) * diag.get_const_data()[i];
         }
     }
 }
@@ -590,10 +587,7 @@ void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
 {
     for (size_type i = 0; i < x->get_size()[0]; ++i) {
         for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            auto diag_val = diag.get_const_data()[i] == zero<ValueType>()
-                                ? one<ValueType>()
-                                : diag.get_const_data()[i];
-            x->at(i, j) = b->at(i, j) / diag_val;
+            x->at(i, j) = b->at(i, j) * diag.get_const_data()[i];
         }
     }
 }

From 4b6b36b09a9d97a29e682b2aa24809cabff605ba Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 5 Aug 2021 17:09:27 +0200
Subject: [PATCH 0988/1008] Add convert to Dense kernel

---
 core/device_hooks/common_kernels.inc.cpp    | 11 ++++++
 core/preconditioner/jacobi.cpp              | 17 ++++++--
 core/preconditioner/jacobi_kernels.hpp      | 14 +++++++
 cuda/preconditioner/jacobi_kernels.cu       | 19 +++++++++
 dpcpp/preconditioner/jacobi_kernels.dp.cpp  | 19 +++++++++
 hip/preconditioner/jacobi_kernels.hip.cpp   | 19 +++++++++
 omp/preconditioner/jacobi_kernels.cpp       | 43 +++++++++++++++++++--
 reference/preconditioner/jacobi_kernels.cpp | 37 ++++++++++++++++++
 reference/test/preconditioner/jacobi.cpp    | 30 +++++++++++---
 9 files changed, 196 insertions(+), 13 deletions(-)

diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 0c5f46f6c1f..9283147b5d7 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -1119,6 +1119,17 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONJ_TRANSPOSE_KERNEL);
 
+template <typename ValueType>
+GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
+
+template <typename ValueType>
+GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
+
 template <typename ValueType, typename IndexType>
 GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index da830f011a7..7da08d85f64 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -63,9 +63,12 @@ GKO_REGISTER_OPERATION(apply, jacobi::apply);
 GKO_REGISTER_OPERATION(scalar_apply, jacobi::scalar_apply);
 GKO_REGISTER_OPERATION(find_blocks, jacobi::find_blocks);
 GKO_REGISTER_OPERATION(generate, jacobi::generate);
+GKO_REGISTER_OPERATION(invert_diagonal, jacobi::invert_diagonal);
 GKO_REGISTER_OPERATION(transpose_jacobi, jacobi::transpose_jacobi);
 GKO_REGISTER_OPERATION(conj_transpose_jacobi, jacobi::conj_transpose_jacobi);
 GKO_REGISTER_OPERATION(convert_to_dense, jacobi::convert_to_dense);
+GKO_REGISTER_OPERATION(scalar_convert_to_dense,
+                       jacobi::scalar_convert_to_dense);
 GKO_REGISTER_OPERATION(initialize_precisions, jacobi::initialize_precisions);
 
 
@@ -119,10 +122,15 @@ void Jacobi<ValueType, IndexType>::convert_to(
 {
     auto exec = this->get_executor();
     auto tmp = matrix::Dense<ValueType>::create(exec, this->get_size());
-    exec->run(jacobi::make_convert_to_dense(
-        num_blocks_, parameters_.storage_optimization.block_wise,
-        parameters_.block_pointers, blocks_, storage_scheme_, tmp->get_values(),
-        tmp->get_stride()));
+    if (parameters_.max_block_size == 1) {
+        exec->run(jacobi::make_scalar_convert_to_dense(
+            blocks_, tmp->get_values(), tmp->get_size(), tmp->get_stride()));
+    } else {
+        exec->run(jacobi::make_convert_to_dense(
+            num_blocks_, parameters_.storage_optimization.block_wise,
+            parameters_.block_pointers, blocks_, storage_scheme_,
+            tmp->get_values(), tmp->get_stride()));
+    }
     tmp->move_to(result);
 }
 
@@ -244,6 +252,7 @@ void Jacobi<ValueType, IndexType>::generate(const LinOp *system_matrix,
         auto temp = Array<ValueType>::view(diag_vt->get_executor(),
                                            diag_vt->get_size()[0],
                                            diag_vt->get_values());
+        exec->run(jacobi::make_invert_diagonal(temp));
         this->blocks_ = temp;
         this->num_blocks_ = diag_vt->get_size()[0];
     } else {
diff --git a/core/preconditioner/jacobi_kernels.hpp b/core/preconditioner/jacobi_kernels.hpp
index 22cfa512bb8..dab9584b2ed 100644
--- a/core/preconditioner/jacobi_kernels.hpp
+++ b/core/preconditioner/jacobi_kernels.hpp
@@ -62,6 +62,10 @@ namespace kernels {
         Array<precision_reduction> &block_precisions,                     \
         const Array<IndexType> &block_pointers, Array<ValueType> &blocks)
 
+#define GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL(ValueType)          \
+    void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec, \
+                         Array<ValueType> &diag)
+
 #define GKO_DECLARE_JACOBI_APPLY_KERNEL(ValueType, IndexType)                  \
     void apply(                                                                \
         std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks,     \
@@ -120,6 +124,12 @@ namespace kernels {
             &storage_scheme,                                               \
         Array<ValueType> &out_blocks)
 
+#define GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL(ValueType) \
+    void scalar_convert_to_dense(                                    \
+        std::shared_ptr<const DefaultExecutor> exec,                 \
+        const Array<ValueType> &blocks, ValueType *result_values,    \
+        const gko::dim<2> &mat_size, size_type result_stride)
+
 #define GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType)   \
     void convert_to_dense(                                                 \
         std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks, \
@@ -141,6 +151,8 @@ namespace kernels {
     template <typename ValueType, typename IndexType>                 \
     GKO_DECLARE_JACOBI_GENERATE_KERNEL(ValueType, IndexType);         \
     template <typename ValueType>                                     \
+    GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL(ValueType);             \
+    template <typename ValueType>                                     \
     GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL(ValueType);                \
     template <typename ValueType, typename IndexType>                 \
     GKO_DECLARE_JACOBI_APPLY_KERNEL(ValueType, IndexType);            \
@@ -152,6 +164,8 @@ namespace kernels {
     GKO_DECLARE_JACOBI_TRANSPOSE_KERNEL(ValueType, IndexType);        \
     template <typename ValueType, typename IndexType>                 \
     GKO_DECLARE_JACOBI_CONJ_TRANSPOSE_KERNEL(ValueType, IndexType);   \
+    template <typename ValueType>                                     \
+    GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL(ValueType);     \
     template <typename ValueType, typename IndexType>                 \
     GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType); \
     GKO_DECLARE_JACOBI_INITIALIZE_PRECISIONS_KERNEL()
diff --git a/cuda/preconditioner/jacobi_kernels.cu b/cuda/preconditioner/jacobi_kernels.cu
index bb9b918f5d9..ba64ae2ab8c 100644
--- a/cuda/preconditioner/jacobi_kernels.cu
+++ b/cuda/preconditioner/jacobi_kernels.cu
@@ -230,6 +230,25 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONJ_TRANSPOSE_KERNEL);
 
 
+template <typename ValueType>
+void scalar_convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,
+                             const Array<ValueType> &blocks,
+                             ValueType *result_values,
+                             const gko::dim<2> &mat_size,
+                             size_type result_stride) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
+
+
+template <typename ValueType>
+void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
+                     Array<ValueType> &diag)
+{}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void convert_to_dense(
     std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks,
diff --git a/dpcpp/preconditioner/jacobi_kernels.dp.cpp b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
index 3b7f1efb90f..ec42cff2f31 100644
--- a/dpcpp/preconditioner/jacobi_kernels.dp.cpp
+++ b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
@@ -267,6 +267,25 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONJ_TRANSPOSE_KERNEL);
 
 
+template <typename ValueType>
+void scalar_convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,
+                             const Array<ValueType> &blocks,
+                             ValueType *result_values,
+                             const gko::dim<2> &mat_size,
+                             size_type result_stride) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
+
+
+template <typename ValueType>
+void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
+                     Array<ValueType> &diag)
+{}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void convert_to_dense(
     std::shared_ptr<const DpcppExecutor> exec, size_type num_blocks,
diff --git a/hip/preconditioner/jacobi_kernels.hip.cpp b/hip/preconditioner/jacobi_kernels.hip.cpp
index 57d513774b5..079ff2912c6 100644
--- a/hip/preconditioner/jacobi_kernels.hip.cpp
+++ b/hip/preconditioner/jacobi_kernels.hip.cpp
@@ -244,6 +244,25 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONJ_TRANSPOSE_KERNEL);
 
 
+template <typename ValueType>
+void scalar_convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,
+                             const Array<ValueType> &blocks,
+                             ValueType *result_values,
+                             const gko::dim<2> &mat_size,
+                             size_type result_stride) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
+
+
+template <typename ValueType>
+void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
+                     Array<ValueType> &diag)
+{}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void convert_to_dense(
     std::shared_ptr<const HipExecutor> exec, size_type num_blocks,
diff --git a/omp/preconditioner/jacobi_kernels.cpp b/omp/preconditioner/jacobi_kernels.cpp
index aa8d5c4bd83..05ded483c3f 100644
--- a/omp/preconditioner/jacobi_kernels.cpp
+++ b/omp/preconditioner/jacobi_kernels.cpp
@@ -696,6 +696,43 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL);
 
 
+template <typename ValueType>
+void scalar_convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,
+                             const Array<ValueType> &blocks,
+                             ValueType *result_values,
+                             const gko::dim<2> &matrix_size,
+                             size_type result_stride)
+{
+    for (size_type i = 0; i < matrix_size[0]; ++i) {
+        for (size_type j = 0; j < matrix_size[1]; ++j) {
+            result_values[i * result_stride + j] = zero<ValueType>();
+            if (i == j) {
+                result_values[i * result_stride + j] =
+                    blocks.get_const_data()[i];
+            }
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
+
+
+template <typename ValueType>
+void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
+                     Array<ValueType> &diag)
+{
+    for (size_type i = 0; i < diag.get_num_elems(); ++i) {
+        auto diag_val = diag.get_const_data()[i] == zero<ValueType>()
+                            ? one<ValueType>()
+                            : diag.get_const_data()[i];
+        diag.get_data()[i] = static_cast<ValueType>(1.0) / diag_val;
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
+
+
 template <typename ValueType>
 void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
                   const Array<ValueType> &diag,
@@ -725,10 +762,8 @@ void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
 #pragma omp parallel for
     for (size_type i = 0; i < x->get_size()[0]; ++i) {
         for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            auto diag_val = diag.get_const_data()[i] == zero<ValueType>()
-                                ? one<ValueType>()
-                                : diag.get_const_data()[i];
-            x->at(i, j) = b->at(i, j) / diag_val;
+            auto diag_val = diag.get_const_data()[i];
+            x->at(i, j) = b->at(i, j) * diag_val;
         }
     }
 }
diff --git a/reference/preconditioner/jacobi_kernels.cpp b/reference/preconditioner/jacobi_kernels.cpp
index a3472c6d70d..916c3c006d6 100644
--- a/reference/preconditioner/jacobi_kernels.cpp
+++ b/reference/preconditioner/jacobi_kernels.cpp
@@ -596,6 +596,21 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL);
 
 
+template <typename ValueType>
+void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
+                     Array<ValueType> &diag)
+{
+    for (size_type i = 0; i < diag.get_num_elems(); ++i) {
+        auto diag_val = diag.get_const_data()[i] == zero<ValueType>()
+                            ? one<ValueType>()
+                            : diag.get_const_data()[i];
+        diag.get_data()[i] = static_cast<ValueType>(1.0) / diag_val;
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void transpose_jacobi(
     std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks,
@@ -668,6 +683,28 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONJ_TRANSPOSE_KERNEL);
 
 
+template <typename ValueType>
+void scalar_convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,
+                             const Array<ValueType> &blocks,
+                             ValueType *result_values,
+                             const gko::dim<2> &matrix_size,
+                             size_type result_stride)
+{
+    for (size_type i = 0; i < matrix_size[0]; ++i) {
+        for (size_type j = 0; j < matrix_size[1]; ++j) {
+            result_values[i * result_stride + j] = zero<ValueType>();
+            if (i == j) {
+                result_values[i * result_stride + j] =
+                    blocks.get_const_data()[i];
+            }
+        }
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
+
+
 template <typename ValueType, typename IndexType>
 void convert_to_dense(
     std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks,
diff --git a/reference/test/preconditioner/jacobi.cpp b/reference/test/preconditioner/jacobi.cpp
index 8f79e8976fb..9f63ebb1bf6 100644
--- a/reference/test/preconditioner/jacobi.cpp
+++ b/reference/test/preconditioner/jacobi.cpp
@@ -310,6 +310,26 @@ TYPED_TEST(Jacobi, CanBeClearedWithAdaptivePrecision)
 }
 
 
+TYPED_TEST(Jacobi, ScalarJacobiConvertsToDense)
+{
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using Bj = typename TestFixture::Bj;
+    gko::matrix_data<value_type, index_type> data;
+    auto csr = gko::share(
+        gko::matrix::Csr<value_type, index_type>::create(this->exec));
+    csr->copy_from(gko::lend(this->mtx));
+    auto scalar_j = this->scalar_j_factory->generate(csr);
+
+    auto dense_j = gko::matrix::Dense<value_type>::create(this->exec);
+    dense_j->copy_from(scalar_j.get());
+    auto dense_j_val = dense_j->get_const_values();
+    auto j_val = scalar_j->get_blocks();
+
+    EXPECT_EQ(dense_j_val[0], j_val[0]);
+}
+
+
 #define GKO_EXPECT_NONZERO_NEAR(first, second, tol) \
     EXPECT_EQ(first.row, second.row);               \
     EXPECT_EQ(first.column, second.column);         \
@@ -393,11 +413,11 @@ TYPED_TEST(Jacobi, ScalarJacobiGeneratesOnDifferentPrecision)
 
     ASSERT_NO_THROW(bj = this->scalar_j_factory->generate(csr));
     ASSERT_EQ(bj->get_num_blocks(), 5u);
-    ASSERT_EQ(bj->get_blocks()[0], value_type{4});
-    ASSERT_EQ(bj->get_blocks()[1], value_type{4});
-    ASSERT_EQ(bj->get_blocks()[2], value_type{4});
-    ASSERT_EQ(bj->get_blocks()[3], value_type{4});
-    ASSERT_EQ(bj->get_blocks()[4], value_type{4});
+    ASSERT_EQ(bj->get_blocks()[0], value_type{0.25});
+    ASSERT_EQ(bj->get_blocks()[1], value_type{0.25});
+    ASSERT_EQ(bj->get_blocks()[2], value_type{0.25});
+    ASSERT_EQ(bj->get_blocks()[3], value_type{0.25});
+    ASSERT_EQ(bj->get_blocks()[4], value_type{0.25});
 }
 
 

From 0b53587d27ec22431e8496299b479948ff958f38 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 5 Aug 2021 17:09:42 +0200
Subject: [PATCH 0989/1008] Update Jacobi write for scalar.

---
 core/preconditioner/jacobi.cpp           | 55 +++++++++++++++---------
 reference/test/preconditioner/jacobi.cpp | 25 +++++++++++
 2 files changed, 60 insertions(+), 20 deletions(-)

diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index 7da08d85f64..0ca3491a612 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -149,29 +149,44 @@ void Jacobi<ValueType, IndexType>::write(mat_data &data) const
         make_temporary_clone(this->get_executor()->get_master(), this);
     data = {local_clone->get_size(), {}};
 
-    const auto ptrs = local_clone->parameters_.block_pointers.get_const_data();
-    for (size_type block = 0; block < local_clone->get_num_blocks(); ++block) {
-        const auto scheme = local_clone->get_storage_scheme();
-        const auto group_data = local_clone->blocks_.get_const_data() +
-                                scheme.get_group_offset(block);
-        const auto block_size = ptrs[block + 1] - ptrs[block];
-        const auto precisions = local_clone->parameters_.storage_optimization
-                                    .block_wise.get_const_data();
-        const auto prec =
-            precisions ? precisions[block] : precision_reduction();
-        GKO_PRECONDITIONER_JACOBI_RESOLVE_PRECISION(ValueType, prec, {
-            const auto block_data =
-                reinterpret_cast<const resolved_precision *>(group_data) +
-                scheme.get_block_offset(block);
-            for (IndexType row = 0; row < block_size; ++row) {
-                for (IndexType col = 0; col < block_size; ++col) {
+    if (parameters_.max_block_size == 1) {
+        for (IndexType row = 0; row < data.size[0]; ++row) {
+            for (IndexType col = 0; col < data.size[1]; ++col) {
+                if (row == col) {
                     data.nonzeros.emplace_back(
-                        ptrs[block] + row, ptrs[block] + col,
-                        static_cast<ValueType>(
-                            block_data[row + col * scheme.get_stride()]));
+                        row, col,
+                        static_cast<ValueType>(local_clone->get_blocks()[row]));
                 }
             }
-        });
+        }
+    } else {
+        const auto ptrs =
+            local_clone->parameters_.block_pointers.get_const_data();
+        for (size_type block = 0; block < local_clone->get_num_blocks();
+             ++block) {
+            const auto scheme = local_clone->get_storage_scheme();
+            const auto group_data = local_clone->blocks_.get_const_data() +
+                                    scheme.get_group_offset(block);
+            const auto block_size = ptrs[block + 1] - ptrs[block];
+            const auto precisions =
+                local_clone->parameters_.storage_optimization.block_wise
+                    .get_const_data();
+            const auto prec =
+                precisions ? precisions[block] : precision_reduction();
+            GKO_PRECONDITIONER_JACOBI_RESOLVE_PRECISION(ValueType, prec, {
+                const auto block_data =
+                    reinterpret_cast<const resolved_precision *>(group_data) +
+                    scheme.get_block_offset(block);
+                for (IndexType row = 0; row < block_size; ++row) {
+                    for (IndexType col = 0; col < block_size; ++col) {
+                        data.nonzeros.emplace_back(
+                            ptrs[block] + row, ptrs[block] + col,
+                            static_cast<ValueType>(
+                                block_data[row + col * scheme.get_stride()]));
+                    }
+                }
+            });
+        }
     }
 }
 
diff --git a/reference/test/preconditioner/jacobi.cpp b/reference/test/preconditioner/jacobi.cpp
index 9f63ebb1bf6..969eda75215 100644
--- a/reference/test/preconditioner/jacobi.cpp
+++ b/reference/test/preconditioner/jacobi.cpp
@@ -371,6 +371,31 @@ TYPED_TEST(Jacobi, GeneratesCorrectMatrixData)
 }
 
 
+TYPED_TEST(Jacobi, ScalarJacobiGeneratesCorrectMatrixData)
+{
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using Bj = typename TestFixture::Bj;
+    gko::matrix_data<value_type, index_type> data;
+    using tpl = typename decltype(data)::nonzero_type;
+    auto csr = gko::share(
+        gko::matrix::Csr<value_type, index_type>::create(this->exec));
+    csr->copy_from(gko::lend(this->mtx));
+    auto scalar_j = this->scalar_j_factory->generate(csr);
+
+    scalar_j->write(data);
+
+    auto tol = r<value_type>::value;
+    ASSERT_EQ(data.size, gko::dim<2>{5});
+    ASSERT_EQ(data.nonzeros.size(), 5);
+    GKO_EXPECT_NONZERO_NEAR(data.nonzeros[0], tpl(0, 0, 1 / 4.0), tol);
+    GKO_EXPECT_NONZERO_NEAR(data.nonzeros[1], tpl(1, 1, 1 / 4.0), tol);
+    GKO_EXPECT_NONZERO_NEAR(data.nonzeros[2], tpl(2, 2, 1 / 4.0), tol);
+    GKO_EXPECT_NONZERO_NEAR(data.nonzeros[3], tpl(3, 3, 1 / 4.0), tol);
+    GKO_EXPECT_NONZERO_NEAR(data.nonzeros[4], tpl(4, 4, 1 / 4.0), tol);
+}
+
+
 TYPED_TEST(Jacobi, GeneratesCorrectMatrixDataWithAdaptivePrecision)
 {
     using value_type = typename TestFixture::value_type;

From 3ce29d555b5418744f8a2ad29ba96e7036ec8f80 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 5 Aug 2021 18:01:43 +0200
Subject: [PATCH 0990/1008] Scalar Jacobi transpose

---
 core/preconditioner/jacobi.cpp           | 13 ++++++----
 reference/test/preconditioner/jacobi.cpp | 30 ++++++++++++++++++++++--
 2 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index 0ca3491a612..d669b467318 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -203,10 +203,15 @@ std::unique_ptr<LinOp> Jacobi<ValueType, IndexType>::transpose() const
     res->blocks_.resize_and_reset(blocks_.get_num_elems());
     res->conditioning_ = conditioning_;
     res->parameters_ = parameters_;
-    this->get_executor()->run(jacobi::make_transpose_jacobi(
-        num_blocks_, parameters_.max_block_size,
-        parameters_.storage_optimization.block_wise, parameters_.block_pointers,
-        blocks_, storage_scheme_, res->blocks_));
+    if (parameters_.max_block_size == 1) {
+        res->blocks_ = blocks_;
+    } else {
+        this->get_executor()->run(jacobi::make_transpose_jacobi(
+            num_blocks_, parameters_.max_block_size,
+            parameters_.storage_optimization.block_wise,
+            parameters_.block_pointers, blocks_, storage_scheme_,
+            res->blocks_));
+    }
 
     return std::move(res);
 }
diff --git a/reference/test/preconditioner/jacobi.cpp b/reference/test/preconditioner/jacobi.cpp
index 969eda75215..3150709b491 100644
--- a/reference/test/preconditioner/jacobi.cpp
+++ b/reference/test/preconditioner/jacobi.cpp
@@ -323,13 +323,39 @@ TYPED_TEST(Jacobi, ScalarJacobiConvertsToDense)
 
     auto dense_j = gko::matrix::Dense<value_type>::create(this->exec);
     dense_j->copy_from(scalar_j.get());
-    auto dense_j_val = dense_j->get_const_values();
     auto j_val = scalar_j->get_blocks();
 
-    EXPECT_EQ(dense_j_val[0], j_val[0]);
+    EXPECT_EQ(dense_j->at(0, 0), j_val[0]);
+    EXPECT_EQ(dense_j->at(1, 1), j_val[1]);
+    EXPECT_EQ(dense_j->at(2, 2), j_val[2]);
+    EXPECT_EQ(dense_j->at(3, 3), j_val[3]);
+    EXPECT_EQ(dense_j->at(4, 4), j_val[4]);
 }
 
 
+TYPED_TEST(Jacobi, ScalarJacobiCanBeTransposed)
+{
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using Bj = typename TestFixture::Bj;
+    gko::matrix_data<value_type, index_type> data;
+    auto csr = gko::share(
+        gko::matrix::Csr<value_type, index_type>::create(this->exec));
+    csr->copy_from(gko::lend(this->mtx));
+    auto scalar_j = this->scalar_j_factory->generate(csr);
+
+    auto dense_j = gko::matrix::Dense<value_type>::create(this->exec);
+    auto t_j = scalar_j->transpose();
+    auto trans_j = gko::as<Bj>(t_j.get())->get_blocks();
+    auto scal_j = scalar_j->get_blocks();
+
+    EXPECT_EQ(trans_j[0], scal_j[0]);
+    EXPECT_EQ(trans_j[1], scal_j[1]);
+    EXPECT_EQ(trans_j[2], scal_j[2]);
+    EXPECT_EQ(trans_j[3], scal_j[3]);
+    EXPECT_EQ(trans_j[4], scal_j[4]);
+}
+
 #define GKO_EXPECT_NONZERO_NEAR(first, second, tol) \
     EXPECT_EQ(first.row, second.row);               \
     EXPECT_EQ(first.column, second.column);         \

From 249b076c81a5f201bd948519f51fc9e4152d741e Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 5 Aug 2021 18:02:20 +0200
Subject: [PATCH 0991/1008] Scalar Jacobi conj_transpose

---
 core/preconditioner/jacobi.cpp           | 13 ++++++++----
 reference/test/preconditioner/jacobi.cpp | 25 ++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index d669b467318..959993e034f 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -229,10 +229,15 @@ std::unique_ptr<LinOp> Jacobi<ValueType, IndexType>::conj_transpose() const
     res->blocks_.resize_and_reset(blocks_.get_num_elems());
     res->conditioning_ = conditioning_;
     res->parameters_ = parameters_;
-    this->get_executor()->run(jacobi::make_conj_transpose_jacobi(
-        num_blocks_, parameters_.max_block_size,
-        parameters_.storage_optimization.block_wise, parameters_.block_pointers,
-        blocks_, storage_scheme_, res->blocks_));
+    if (parameters_.max_block_size == 1) {
+        res->blocks_ = blocks_;
+    } else {
+        this->get_executor()->run(jacobi::make_conj_transpose_jacobi(
+            num_blocks_, parameters_.max_block_size,
+            parameters_.storage_optimization.block_wise,
+            parameters_.block_pointers, blocks_, storage_scheme_,
+            res->blocks_));
+    }
 
     return std::move(res);
 }
diff --git a/reference/test/preconditioner/jacobi.cpp b/reference/test/preconditioner/jacobi.cpp
index 3150709b491..17745028b5f 100644
--- a/reference/test/preconditioner/jacobi.cpp
+++ b/reference/test/preconditioner/jacobi.cpp
@@ -356,6 +356,31 @@ TYPED_TEST(Jacobi, ScalarJacobiCanBeTransposed)
     EXPECT_EQ(trans_j[4], scal_j[4]);
 }
 
+
+TYPED_TEST(Jacobi, ScalarJacobiCanBeConjTransposed)
+{
+    using value_type = typename TestFixture::value_type;
+    using index_type = typename TestFixture::index_type;
+    using Bj = typename TestFixture::Bj;
+    gko::matrix_data<value_type, index_type> data;
+    auto csr = gko::share(
+        gko::matrix::Csr<value_type, index_type>::create(this->exec));
+    csr->copy_from(gko::lend(this->mtx));
+    auto scalar_j = this->scalar_j_factory->generate(csr);
+
+    auto dense_j = gko::matrix::Dense<value_type>::create(this->exec);
+    auto t_j = scalar_j->transpose();
+    auto trans_j = gko::as<Bj>(t_j.get())->get_blocks();
+    auto scal_j = scalar_j->get_blocks();
+
+    EXPECT_EQ(trans_j[0], gko::conj(scal_j[0]));
+    EXPECT_EQ(trans_j[1], gko::conj(scal_j[1]));
+    EXPECT_EQ(trans_j[2], gko::conj(scal_j[2]));
+    EXPECT_EQ(trans_j[3], gko::conj(scal_j[3]));
+    EXPECT_EQ(trans_j[4], gko::conj(scal_j[4]));
+}
+
+
 #define GKO_EXPECT_NONZERO_NEAR(first, second, tol) \
     EXPECT_EQ(first.row, second.row);               \
     EXPECT_EQ(first.column, second.column);         \

From 7aeea4056cabb74a576f1eaa0bc4f7f6c0934d43 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Thu, 5 Aug 2021 23:55:37 +0200
Subject: [PATCH 0992/1008] Add conj transpose kernel and update inv kernel

---
 common/preconditioner/jacobi_kernels.hpp.inc | 30 ++++++++++++++++--
 core/device_hooks/common_kernels.inc.cpp     |  5 +++
 core/preconditioner/jacobi.cpp               |  8 +++--
 core/preconditioner/jacobi_kernels.hpp       | 10 +++++-
 cuda/preconditioner/jacobi_kernels.cu        | 30 ++++++++++++++++--
 dpcpp/preconditioner/jacobi_kernels.dp.cpp   | 12 ++++++--
 hip/preconditioner/jacobi_kernels.hip.cpp    | 32 ++++++++++++++++++--
 omp/preconditioner/jacobi_kernels.cpp        | 23 +++++++++++---
 reference/preconditioner/jacobi_kernels.cpp  | 20 +++++++++---
 9 files changed, 147 insertions(+), 23 deletions(-)

diff --git a/common/preconditioner/jacobi_kernels.hpp.inc b/common/preconditioner/jacobi_kernels.hpp.inc
index 87d9a9be7a7..14337a92881 100644
--- a/common/preconditioner/jacobi_kernels.hpp.inc
+++ b/common/preconditioner/jacobi_kernels.hpp.inc
@@ -238,6 +238,32 @@ __global__ __launch_bounds__(default_block_size) void scalar_apply(
 }
 
 
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void conj_diagonal(
+    size_type num_elems, const ValueType *__restrict__ diag,
+    ValueType *__restrict__ conj_diag)
+{
+    const auto tidx = thread::get_thread_id_flat();
+
+    if (tidx < num_elems) {
+        conj_diag[tidx] = gko::conj(diag[tidx]);
+    }
+}
+
+
+template <typename ValueType>
+__global__ __launch_bounds__(default_block_size) void invert_diagonal(
+    size_type num_elems, const ValueType *__restrict__ diag,
+    ValueType *__restrict__ inv_diag)
+{
+    const auto tidx = thread::get_thread_id_flat();
+
+    if (tidx < num_elems) {
+        inv_diag[tidx] = ValueType(1.0) / diag[tidx];
+    }
+}
+
+
 template <typename ValueType>
 __global__ __launch_bounds__(default_block_size) void simple_scalar_apply(
     size_type num_rows, size_type num_cols, const ValueType *__restrict__ diag,
@@ -249,10 +275,8 @@ __global__ __launch_bounds__(default_block_size) void simple_scalar_apply(
     const auto col = tidx % num_cols;
 
     if (row < num_rows) {
-        auto diag_val =
-            diag[row] == zero<ValueType>() ? one<ValueType>() : diag[row];
         result_values[row * result_stride + col] =
-            source_values[row * source_stride + col] / diag_val;
+            source_values[row * source_stride + col] * diag[row];
     }
 }
 
diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp
index 9283147b5d7..b9400e90b3f 100644
--- a/core/device_hooks/common_kernels.inc.cpp
+++ b/core/device_hooks/common_kernels.inc.cpp
@@ -1125,6 +1125,11 @@ GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
 
+template <typename ValueType>
+GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL(ValueType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL);
+
 template <typename ValueType>
 GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index 959993e034f..e939470fdea 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -63,6 +63,7 @@ GKO_REGISTER_OPERATION(apply, jacobi::apply);
 GKO_REGISTER_OPERATION(scalar_apply, jacobi::scalar_apply);
 GKO_REGISTER_OPERATION(find_blocks, jacobi::find_blocks);
 GKO_REGISTER_OPERATION(generate, jacobi::generate);
+GKO_REGISTER_OPERATION(scalar_conj, jacobi::scalar_conj);
 GKO_REGISTER_OPERATION(invert_diagonal, jacobi::invert_diagonal);
 GKO_REGISTER_OPERATION(transpose_jacobi, jacobi::transpose_jacobi);
 GKO_REGISTER_OPERATION(conj_transpose_jacobi, jacobi::conj_transpose_jacobi);
@@ -230,7 +231,8 @@ std::unique_ptr<LinOp> Jacobi<ValueType, IndexType>::conj_transpose() const
     res->conditioning_ = conditioning_;
     res->parameters_ = parameters_;
     if (parameters_.max_block_size == 1) {
-        res->blocks_ = blocks_;
+        this->get_executor()->run(
+            jacobi::make_scalar_conj(this->blocks_, res->blocks_));
     } else {
         this->get_executor()->run(jacobi::make_conj_transpose_jacobi(
             num_blocks_, parameters_.max_block_size,
@@ -277,8 +279,8 @@ void Jacobi<ValueType, IndexType>::generate(const LinOp *system_matrix,
         auto temp = Array<ValueType>::view(diag_vt->get_executor(),
                                            diag_vt->get_size()[0],
                                            diag_vt->get_values());
-        exec->run(jacobi::make_invert_diagonal(temp));
-        this->blocks_ = temp;
+        this->blocks_ = Array<ValueType>(exec, temp.get_num_elems());
+        exec->run(jacobi::make_invert_diagonal(temp, this->blocks_));
         this->num_blocks_ = diag_vt->get_size()[0];
     } else {
         auto csr_mtx = convert_to_with_sorting<csr_type>(exec, system_matrix,
diff --git a/core/preconditioner/jacobi_kernels.hpp b/core/preconditioner/jacobi_kernels.hpp
index dab9584b2ed..2960e175de4 100644
--- a/core/preconditioner/jacobi_kernels.hpp
+++ b/core/preconditioner/jacobi_kernels.hpp
@@ -62,9 +62,15 @@ namespace kernels {
         Array<precision_reduction> &block_precisions,                     \
         const Array<IndexType> &block_pointers, Array<ValueType> &blocks)
 
+#define GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL(ValueType)          \
+    void scalar_conj(std::shared_ptr<const DefaultExecutor> exec, \
+                     const Array<ValueType> &diag,                \
+                     Array<ValueType> &conj_diag)
+
 #define GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL(ValueType)          \
     void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec, \
-                         Array<ValueType> &diag)
+                         const Array<ValueType> &diag,                \
+                         Array<ValueType> &inv_diag)
 
 #define GKO_DECLARE_JACOBI_APPLY_KERNEL(ValueType, IndexType)                  \
     void apply(                                                                \
@@ -151,6 +157,8 @@ namespace kernels {
     template <typename ValueType, typename IndexType>                 \
     GKO_DECLARE_JACOBI_GENERATE_KERNEL(ValueType, IndexType);         \
     template <typename ValueType>                                     \
+    GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL(ValueType);                 \
+    template <typename ValueType>                                     \
     GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL(ValueType);             \
     template <typename ValueType>                                     \
     GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL(ValueType);                \
diff --git a/cuda/preconditioner/jacobi_kernels.cu b/cuda/preconditioner/jacobi_kernels.cu
index ba64ae2ab8c..91daccbfb36 100644
--- a/cuda/preconditioner/jacobi_kernels.cu
+++ b/cuda/preconditioner/jacobi_kernels.cu
@@ -241,10 +241,36 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
 
 
+template <typename ValueType>
+void scalar_conj(std::shared_ptr<const DefaultExecutor> exec,
+                 const Array<ValueType> &diag, Array<ValueType> &conj_diag)
+{
+    const auto num_elems = diag.get_num_elems();
+    const auto grid_dim = ceildiv(num_elems, default_block_size);
+
+    const auto diag_values = diag.get_const_data();
+    auto conj_diag_values = conj_diag.get_data();
+
+    kernel::conj_diagonal<<<grid_dim, default_block_size>>>(
+        num_elems, as_cuda_type(diag_values), as_cuda_type(conj_diag_values));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL);
+
+
 template <typename ValueType>
 void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
-                     Array<ValueType> &diag)
-{}
+                     const Array<ValueType> &diag, Array<ValueType> &inv_diag)
+{
+    const auto num_elems = diag.get_num_elems();
+    const auto grid_dim = ceildiv(num_elems, default_block_size);
+
+    const auto diag_values = diag.get_const_data();
+    auto inv_diag_values = inv_diag.get_data();
+
+    kernel::invert_diagonal<<<grid_dim, default_block_size>>>(
+        num_elems, as_cuda_type(diag_values), as_cuda_type(inv_diag_values));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
 
diff --git a/dpcpp/preconditioner/jacobi_kernels.dp.cpp b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
index ec42cff2f31..ccf49d380e9 100644
--- a/dpcpp/preconditioner/jacobi_kernels.dp.cpp
+++ b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
@@ -278,10 +278,18 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
 
 
+template <typename ValueType>
+void scalar_conj(std::shared_ptr<const DefaultExecutor> exec,
+                 const Array<ValueType> &diag,
+                 Array<ValueType> &conj_diag) GKO_NOT_IMPLEMENTED;
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL);
+
+
 template <typename ValueType>
 void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
-                     Array<ValueType> &diag)
-{}
+                     const Array<ValueType> &diag,
+                     Array<ValueType> &inv_diag) GKO_NOT_IMPLEMENTED;
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
 
diff --git a/hip/preconditioner/jacobi_kernels.hip.cpp b/hip/preconditioner/jacobi_kernels.hip.cpp
index 079ff2912c6..f47ea47538d 100644
--- a/hip/preconditioner/jacobi_kernels.hip.cpp
+++ b/hip/preconditioner/jacobi_kernels.hip.cpp
@@ -255,10 +255,38 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
 
 
+template <typename ValueType>
+void scalar_conj(std::shared_ptr<const DefaultExecutor> exec,
+                 const Array<ValueType> &diag, Array<ValueType> &conj_diag)
+{
+    const auto num_elems = diag.get_num_elems();
+    const auto grid_dim = ceildiv(num_elems, default_block_size);
+
+    const auto diag_values = diag.get_const_data();
+    auto conj_diag_values = conj_diag.get_data();
+
+    hipLaunchKernelGGL(kernel::conj_diagonal, dim3(grid_dim),
+                       dim3(default_block_size), 0, 0, num_elems,
+                       as_hip_type(diag_values), as_hip_type(conj_diag_values));
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL);
+
+
 template <typename ValueType>
 void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
-                     Array<ValueType> &diag)
-{}
+                     const Array<ValueType> &diag, Array<ValueType> &inv_diag)
+{
+    const auto num_elems = diag.get_num_elems();
+    const auto grid_dim = ceildiv(num_elems, default_block_size);
+
+    const auto diag_values = diag.get_const_data();
+    auto inv_diag_values = inv_diag.get_data();
+
+    hipLaunchKernelGGL(kernel::invert_diagonal, dim3(grid_dim),
+                       dim3(default_block_size), 0, 0, num_elems,
+                       as_hip_type(diag_values), as_hip_type(inv_diag_values));
+}
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
 
diff --git a/omp/preconditioner/jacobi_kernels.cpp b/omp/preconditioner/jacobi_kernels.cpp
index 05ded483c3f..b17f3ba25c3 100644
--- a/omp/preconditioner/jacobi_kernels.cpp
+++ b/omp/preconditioner/jacobi_kernels.cpp
@@ -703,6 +703,7 @@ void scalar_convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,
                              const gko::dim<2> &matrix_size,
                              size_type result_stride)
 {
+#pragma omp parallel for
     for (size_type i = 0; i < matrix_size[0]; ++i) {
         for (size_type j = 0; j < matrix_size[1]; ++j) {
             result_values[i * result_stride + j] = zero<ValueType>();
@@ -718,15 +719,27 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
 
 
+template <typename ValueType>
+void scalar_conj(std::shared_ptr<const DefaultExecutor> exec,
+                 const Array<ValueType> &diag, Array<ValueType> &conj_diag)
+{
+#pragma omp parallel for
+    for (size_type i = 0; i < diag.get_num_elems(); ++i) {
+        conj_diag.get_data()[i] = gko::conj(diag.get_const_data()[i]);
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL);
+
+
 template <typename ValueType>
 void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
-                     Array<ValueType> &diag)
+                     const Array<ValueType> &diag, Array<ValueType> &inv_diag)
 {
+#pragma omp parallel for
     for (size_type i = 0; i < diag.get_num_elems(); ++i) {
-        auto diag_val = diag.get_const_data()[i] == zero<ValueType>()
-                            ? one<ValueType>()
-                            : diag.get_const_data()[i];
-        diag.get_data()[i] = static_cast<ValueType>(1.0) / diag_val;
+        inv_diag.get_data()[i] =
+            static_cast<ValueType>(1.0) / diag.get_const_data()[i];
     }
 }
 
diff --git a/reference/preconditioner/jacobi_kernels.cpp b/reference/preconditioner/jacobi_kernels.cpp
index 916c3c006d6..85fc575967d 100644
--- a/reference/preconditioner/jacobi_kernels.cpp
+++ b/reference/preconditioner/jacobi_kernels.cpp
@@ -596,15 +596,25 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL);
 
 
+template <typename ValueType>
+void scalar_conj(std::shared_ptr<const DefaultExecutor> exec,
+                 const Array<ValueType> &diag, Array<ValueType> &conj_diag)
+{
+    for (size_type i = 0; i < diag.get_num_elems(); ++i) {
+        conj_diag.get_data()[i] = gko::conj(diag.get_const_data()[i]);
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL);
+
+
 template <typename ValueType>
 void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
-                     Array<ValueType> &diag)
+                     const Array<ValueType> &diag, Array<ValueType> &inv_diag)
 {
     for (size_type i = 0; i < diag.get_num_elems(); ++i) {
-        auto diag_val = diag.get_const_data()[i] == zero<ValueType>()
-                            ? one<ValueType>()
-                            : diag.get_const_data()[i];
-        diag.get_data()[i] = static_cast<ValueType>(1.0) / diag_val;
+        inv_diag.get_data()[i] =
+            static_cast<ValueType>(1.0) / diag.get_const_data()[i];
     }
 }
 

From 3c8a356ea6c9cb16555f4286e910bbb32ebf6bfe Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Fri, 6 Aug 2021 08:45:04 +0200
Subject: [PATCH 0993/1008] Review update.

---
 common/preconditioner/jacobi_kernels.hpp.inc |  2 +-
 core/preconditioner/jacobi.cpp               | 10 +++-------
 omp/preconditioner/jacobi_kernels.cpp        |  2 +-
 reference/preconditioner/jacobi_kernels.cpp  |  2 +-
 4 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/common/preconditioner/jacobi_kernels.hpp.inc b/common/preconditioner/jacobi_kernels.hpp.inc
index 14337a92881..34390e97ae6 100644
--- a/common/preconditioner/jacobi_kernels.hpp.inc
+++ b/common/preconditioner/jacobi_kernels.hpp.inc
@@ -246,7 +246,7 @@ __global__ __launch_bounds__(default_block_size) void conj_diagonal(
     const auto tidx = thread::get_thread_id_flat();
 
     if (tidx < num_elems) {
-        conj_diag[tidx] = gko::conj(diag[tidx]);
+        conj_diag[tidx] = conj(diag[tidx]);
     }
 }
 
diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index e939470fdea..d52d2959f6d 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -152,13 +152,9 @@ void Jacobi<ValueType, IndexType>::write(mat_data &data) const
 
     if (parameters_.max_block_size == 1) {
         for (IndexType row = 0; row < data.size[0]; ++row) {
-            for (IndexType col = 0; col < data.size[1]; ++col) {
-                if (row == col) {
-                    data.nonzeros.emplace_back(
-                        row, col,
-                        static_cast<ValueType>(local_clone->get_blocks()[row]));
-                }
-            }
+            data.nonzeros.emplace_back(
+                row, row,
+                static_cast<ValueType>(local_clone->get_blocks()[row]));
         }
     } else {
         const auto ptrs =
diff --git a/omp/preconditioner/jacobi_kernels.cpp b/omp/preconditioner/jacobi_kernels.cpp
index b17f3ba25c3..f9e8c71fa44 100644
--- a/omp/preconditioner/jacobi_kernels.cpp
+++ b/omp/preconditioner/jacobi_kernels.cpp
@@ -725,7 +725,7 @@ void scalar_conj(std::shared_ptr<const DefaultExecutor> exec,
 {
 #pragma omp parallel for
     for (size_type i = 0; i < diag.get_num_elems(); ++i) {
-        conj_diag.get_data()[i] = gko::conj(diag.get_const_data()[i]);
+        conj_diag.get_data()[i] = conj(diag.get_const_data()[i]);
     }
 }
 
diff --git a/reference/preconditioner/jacobi_kernels.cpp b/reference/preconditioner/jacobi_kernels.cpp
index 85fc575967d..9799329f773 100644
--- a/reference/preconditioner/jacobi_kernels.cpp
+++ b/reference/preconditioner/jacobi_kernels.cpp
@@ -601,7 +601,7 @@ void scalar_conj(std::shared_ptr<const DefaultExecutor> exec,
                  const Array<ValueType> &diag, Array<ValueType> &conj_diag)
 {
     for (size_type i = 0; i < diag.get_num_elems(); ++i) {
-        conj_diag.get_data()[i] = gko::conj(diag.get_const_data()[i]);
+        conj_diag.get_data()[i] = conj(diag.get_const_data()[i]);
     }
 }
 

From ed2550b42a32af4c2dd3f6c4033e15835640f03b Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Fri, 6 Aug 2021 10:13:30 +0200
Subject: [PATCH 0994/1008] Use simple kernels for conj and invert_diag

---
 common/preconditioner/jacobi_kernels.cpp     | 88 ++++++++++++++++++++
 common/preconditioner/jacobi_kernels.hpp.inc | 13 ---
 cuda/CMakeLists.txt                          |  1 +
 cuda/preconditioner/jacobi_kernels.cu        | 34 --------
 dpcpp/CMakeLists.txt                         |  1 +
 dpcpp/preconditioner/jacobi_kernels.dp.cpp   | 16 ----
 hip/CMakeLists.txt                           |  1 +
 hip/preconditioner/jacobi_kernels.hip.cpp    | 36 --------
 omp/CMakeLists.txt                           |  1 +
 omp/preconditioner/jacobi_kernels.cpp        | 27 ------
 reference/test/preconditioner/jacobi.cpp     | 39 ++++++---
 11 files changed, 120 insertions(+), 137 deletions(-)
 create mode 100644 common/preconditioner/jacobi_kernels.cpp

diff --git a/common/preconditioner/jacobi_kernels.cpp b/common/preconditioner/jacobi_kernels.cpp
new file mode 100644
index 00000000000..f4a2b1a3320
--- /dev/null
+++ b/common/preconditioner/jacobi_kernels.cpp
@@ -0,0 +1,88 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/preconditioner/jacobi_kernels.hpp"
+
+
+#include <ginkgo/core/base/math.hpp>
+
+
+#include "common/base/kernel_launch_solver.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace GKO_DEVICE_NAMESPACE {
+/**
+ * @brief The Jacobi preconditioner namespace.
+ *
+ * @ingroup jacobi
+ */
+namespace jacobi {
+
+
+template <typename ValueType>
+void scalar_conj(std::shared_ptr<const DefaultExecutor> exec,
+                 const Array<ValueType> &diag, Array<ValueType> &conj_diag)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto elem, auto diag, auto conj_diag) {
+            conj_diag[elem] = conj(diag[elem]);
+        },
+        diag.get_num_elems(), diag.get_const_data(), conj_diag.get_data());
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL);
+
+
+template <typename ValueType>
+void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
+                     const Array<ValueType> &diag, Array<ValueType> &inv_diag)
+{
+    auto one_val = one<ValueType>();
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto elem, auto diag, auto inv_diag, auto one_val) {
+            inv_diag[elem] = one_val / diag[elem];
+        },
+        diag.get_num_elems(), diag.get_const_data(), inv_diag.get_data(),
+        one_val);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
+
+
+}  // namespace jacobi
+}  // namespace GKO_DEVICE_NAMESPACE
+}  // namespace kernels
+}  // namespace gko
diff --git a/common/preconditioner/jacobi_kernels.hpp.inc b/common/preconditioner/jacobi_kernels.hpp.inc
index 34390e97ae6..704c4870c23 100644
--- a/common/preconditioner/jacobi_kernels.hpp.inc
+++ b/common/preconditioner/jacobi_kernels.hpp.inc
@@ -238,19 +238,6 @@ __global__ __launch_bounds__(default_block_size) void scalar_apply(
 }
 
 
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void conj_diagonal(
-    size_type num_elems, const ValueType *__restrict__ diag,
-    ValueType *__restrict__ conj_diag)
-{
-    const auto tidx = thread::get_thread_id_flat();
-
-    if (tidx < num_elems) {
-        conj_diag[tidx] = conj(diag[tidx]);
-    }
-}
-
-
 template <typename ValueType>
 __global__ __launch_bounds__(default_block_size) void invert_diagonal(
     size_type num_elems, const ValueType *__restrict__ diag,
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index df512019d88..602dea8ee24 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -71,6 +71,7 @@ set(GKO_CUDA_COMMON_SOURCES
     ../common/matrix/csr_kernels.cpp
     ../common/matrix/dense_kernels.cpp
     ../common/matrix/diagonal_kernels.cpp
+    ../common/preconditioner/jacobi_kernels.cpp
     ../common/solver/bicg_kernels.cpp
     ../common/solver/bicgstab_kernels.cpp
     ../common/solver/cg_kernels.cpp
diff --git a/cuda/preconditioner/jacobi_kernels.cu b/cuda/preconditioner/jacobi_kernels.cu
index 91daccbfb36..75b5f797d4e 100644
--- a/cuda/preconditioner/jacobi_kernels.cu
+++ b/cuda/preconditioner/jacobi_kernels.cu
@@ -241,40 +241,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
 
 
-template <typename ValueType>
-void scalar_conj(std::shared_ptr<const DefaultExecutor> exec,
-                 const Array<ValueType> &diag, Array<ValueType> &conj_diag)
-{
-    const auto num_elems = diag.get_num_elems();
-    const auto grid_dim = ceildiv(num_elems, default_block_size);
-
-    const auto diag_values = diag.get_const_data();
-    auto conj_diag_values = conj_diag.get_data();
-
-    kernel::conj_diagonal<<<grid_dim, default_block_size>>>(
-        num_elems, as_cuda_type(diag_values), as_cuda_type(conj_diag_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL);
-
-
-template <typename ValueType>
-void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
-                     const Array<ValueType> &diag, Array<ValueType> &inv_diag)
-{
-    const auto num_elems = diag.get_num_elems();
-    const auto grid_dim = ceildiv(num_elems, default_block_size);
-
-    const auto diag_values = diag.get_const_data();
-    auto inv_diag_values = inv_diag.get_data();
-
-    kernel::invert_diagonal<<<grid_dim, default_block_size>>>(
-        num_elems, as_cuda_type(diag_values), as_cuda_type(inv_diag_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void convert_to_dense(
     std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks,
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index f167bac3742..92ac330364b 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -52,6 +52,7 @@ target_sources(ginkgo_dpcpp
     ../common/matrix/csr_kernels.cpp
     ../common/matrix/dense_kernels.cpp
     ../common/matrix/diagonal_kernels.cpp
+    ../common/preconditioner/jacobi_kernels.cpp
     ../common/solver/bicg_kernels.cpp
     ../common/solver/bicgstab_kernels.cpp
     ../common/solver/cg_kernels.cpp
diff --git a/dpcpp/preconditioner/jacobi_kernels.dp.cpp b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
index ccf49d380e9..4f1c9c6cc0f 100644
--- a/dpcpp/preconditioner/jacobi_kernels.dp.cpp
+++ b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
@@ -278,22 +278,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
 
 
-template <typename ValueType>
-void scalar_conj(std::shared_ptr<const DefaultExecutor> exec,
-                 const Array<ValueType> &diag,
-                 Array<ValueType> &conj_diag) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL);
-
-
-template <typename ValueType>
-void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
-                     const Array<ValueType> &diag,
-                     Array<ValueType> &inv_diag) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void convert_to_dense(
     std::shared_ptr<const DpcppExecutor> exec, size_type num_blocks,
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 0778c41a39e..0bae0c057fc 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -190,6 +190,7 @@ set(GINKGO_HIP_SOURCES
     ../common/matrix/csr_kernels.cpp
     ../common/matrix/dense_kernels.cpp
     ../common/matrix/diagonal_kernels.cpp
+    ../common/preconditioner/jacobi_kernels.cpp
     ../common/solver/bicg_kernels.cpp
     ../common/solver/bicgstab_kernels.cpp
     ../common/solver/cg_kernels.cpp
diff --git a/hip/preconditioner/jacobi_kernels.hip.cpp b/hip/preconditioner/jacobi_kernels.hip.cpp
index f47ea47538d..0be2a3db67b 100644
--- a/hip/preconditioner/jacobi_kernels.hip.cpp
+++ b/hip/preconditioner/jacobi_kernels.hip.cpp
@@ -255,42 +255,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
 
 
-template <typename ValueType>
-void scalar_conj(std::shared_ptr<const DefaultExecutor> exec,
-                 const Array<ValueType> &diag, Array<ValueType> &conj_diag)
-{
-    const auto num_elems = diag.get_num_elems();
-    const auto grid_dim = ceildiv(num_elems, default_block_size);
-
-    const auto diag_values = diag.get_const_data();
-    auto conj_diag_values = conj_diag.get_data();
-
-    hipLaunchKernelGGL(kernel::conj_diagonal, dim3(grid_dim),
-                       dim3(default_block_size), 0, 0, num_elems,
-                       as_hip_type(diag_values), as_hip_type(conj_diag_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL);
-
-
-template <typename ValueType>
-void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
-                     const Array<ValueType> &diag, Array<ValueType> &inv_diag)
-{
-    const auto num_elems = diag.get_num_elems();
-    const auto grid_dim = ceildiv(num_elems, default_block_size);
-
-    const auto diag_values = diag.get_const_data();
-    auto inv_diag_values = inv_diag.get_data();
-
-    hipLaunchKernelGGL(kernel::invert_diagonal, dim3(grid_dim),
-                       dim3(default_block_size), 0, 0, num_elems,
-                       as_hip_type(diag_values), as_hip_type(inv_diag_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void convert_to_dense(
     std::shared_ptr<const HipExecutor> exec, size_type num_blocks,
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index faf49154f34..f2bc90284b1 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -39,6 +39,7 @@ target_sources(ginkgo_omp
     ../common/matrix/csr_kernels.cpp
     ../common/matrix/dense_kernels.cpp
     ../common/matrix/diagonal_kernels.cpp
+    ../common/preconditioner/jacobi_kernels.cpp
     ../common/solver/bicg_kernels.cpp
     ../common/solver/bicgstab_kernels.cpp
     ../common/solver/cg_kernels.cpp
diff --git a/omp/preconditioner/jacobi_kernels.cpp b/omp/preconditioner/jacobi_kernels.cpp
index f9e8c71fa44..a49ffe8aa49 100644
--- a/omp/preconditioner/jacobi_kernels.cpp
+++ b/omp/preconditioner/jacobi_kernels.cpp
@@ -719,33 +719,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
     GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
 
 
-template <typename ValueType>
-void scalar_conj(std::shared_ptr<const DefaultExecutor> exec,
-                 const Array<ValueType> &diag, Array<ValueType> &conj_diag)
-{
-#pragma omp parallel for
-    for (size_type i = 0; i < diag.get_num_elems(); ++i) {
-        conj_diag.get_data()[i] = conj(diag.get_const_data()[i]);
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL);
-
-
-template <typename ValueType>
-void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
-                     const Array<ValueType> &diag, Array<ValueType> &inv_diag)
-{
-#pragma omp parallel for
-    for (size_type i = 0; i < diag.get_num_elems(); ++i) {
-        inv_diag.get_data()[i] =
-            static_cast<ValueType>(1.0) / diag.get_const_data()[i];
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
-
-
 template <typename ValueType>
 void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
                   const Array<ValueType> &diag,
diff --git a/reference/test/preconditioner/jacobi.cpp b/reference/test/preconditioner/jacobi.cpp
index 17745028b5f..390a575290e 100644
--- a/reference/test/preconditioner/jacobi.cpp
+++ b/reference/test/preconditioner/jacobi.cpp
@@ -357,19 +357,36 @@ TYPED_TEST(Jacobi, ScalarJacobiCanBeTransposed)
 }
 
 
-TYPED_TEST(Jacobi, ScalarJacobiCanBeConjTransposed)
+template <typename T>
+void init_array(T *arr, std::initializer_list<T> vals)
 {
-    using value_type = typename TestFixture::value_type;
-    using index_type = typename TestFixture::index_type;
-    using Bj = typename TestFixture::Bj;
-    gko::matrix_data<value_type, index_type> data;
-    auto csr = gko::share(
-        gko::matrix::Csr<value_type, index_type>::create(this->exec));
-    csr->copy_from(gko::lend(this->mtx));
-    auto scalar_j = this->scalar_j_factory->generate(csr);
+    for (auto elem : vals) {
+        *(arr++) = elem;
+    }
+}
 
-    auto dense_j = gko::matrix::Dense<value_type>::create(this->exec);
-    auto t_j = scalar_j->transpose();
+
+TEST(Jacobi, ScalarJacobiCanBeConjTransposed)
+{
+    using value_type = std::complex<double>;
+    using vt = value_type;
+    using index_type = int;
+    using Bj = gko::preconditioner::Jacobi<value_type, index_type>;
+    gko::matrix_data<value_type, index_type> data;
+    using Mtx = gko::matrix::Csr<value_type, index_type>;
+    auto exec = gko::ReferenceExecutor::create();
+    auto csr = gko::share(Mtx::create(exec, gko::dim<2>(5, 5), 13));
+    auto scalar_j_factory = Bj::build().with_max_block_size(1u).on(exec);
+    init_array<index_type>(csr->get_row_ptrs(), {0, 3, 5, 7, 10, 13});
+    init_array<index_type>(csr->get_col_idxs(),
+                           {0, 1, 4, 0, 1, 2, 3, 2, 3, 4, 0, 3, 4});
+    init_array<value_type>(
+        csr->get_values(),
+        {vt(4.0, 1), vt(-2.0), vt(-2.0), vt(-1.0), vt(4.0, -1), vt(4.0),
+         vt(-2.0), vt(-1.0), vt(4.0), vt(-2.0), vt(-1.0), vt(-1.0), vt(4.0)});
+    auto scalar_j = scalar_j_factory->generate(csr);
+
+    auto t_j = scalar_j->conj_transpose();
     auto trans_j = gko::as<Bj>(t_j.get())->get_blocks();
     auto scal_j = scalar_j->get_blocks();
 

From dff2f3e7e1a91f04d743712d3d96e71d070eab90 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Fri, 6 Aug 2021 10:49:20 +0200
Subject: [PATCH 0995/1008] Use simple kernels for scalar apply, dense convert

---
 common/preconditioner/jacobi_kernels.cpp     | 77 +++++++++++++++++++-
 common/preconditioner/jacobi_kernels.hpp.inc | 56 --------------
 cuda/preconditioner/jacobi_kernels.cu        | 66 -----------------
 dpcpp/preconditioner/jacobi_kernels.dp.cpp   | 32 --------
 hip/preconditioner/jacobi_kernels.hip.cpp    | 67 -----------------
 omp/preconditioner/jacobi_kernels.cpp        | 62 ----------------
 6 files changed, 76 insertions(+), 284 deletions(-)

diff --git a/common/preconditioner/jacobi_kernels.cpp b/common/preconditioner/jacobi_kernels.cpp
index f4a2b1a3320..85d4bce3ff7 100644
--- a/common/preconditioner/jacobi_kernels.cpp
+++ b/common/preconditioner/jacobi_kernels.cpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "common/base/kernel_launch_solver.hpp"
+#include "common/base/kernel_launch.hpp"
 
 
 namespace gko {
@@ -82,6 +82,81 @@ void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
 
 
+template <typename ValueType>
+void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
+                  const Array<ValueType> &diag,
+                  const matrix::Dense<ValueType> *alpha,
+                  const matrix::Dense<ValueType> *b,
+                  const matrix::Dense<ValueType> *beta,
+                  matrix::Dense<ValueType> *x)
+{
+    if (alpha->get_size()[1] > 1) {
+        run_kernel(
+            exec,
+            [] GKO_KERNEL(auto row, auto col, auto diag, auto alpha, auto b,
+                          auto beta, auto x) {
+                x(row, col) = beta[col] * x(row, col) +
+                              alpha[col] * b(row, col) * diag[row];
+            },
+            x->get_size(), diag.get_const_data(), alpha->get_const_values(), b,
+            beta->get_const_values(), x);
+    } else {
+        run_kernel(
+            exec,
+            [] GKO_KERNEL(auto row, auto col, auto diag, auto alpha, auto b,
+                          auto beta, auto x) {
+                x(row, col) =
+                    beta[0] * x(row, col) + alpha[0] * b(row, col) * diag[row];
+            },
+            x->get_size(), diag.get_const_data(), alpha->get_const_values(), b,
+            beta->get_const_values(), x);
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
+
+
+template <typename ValueType>
+void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
+                         const Array<ValueType> &diag,
+                         const matrix::Dense<ValueType> *b,
+                         matrix::Dense<ValueType> *x)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto diag, auto b, auto x) {
+            x(row, col) = b(row, col) * diag[row];
+        },
+        x->get_size(), diag.get_const_data(), b, x);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL);
+
+
+template <typename ValueType>
+void scalar_convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,
+                             const Array<ValueType> &blocks,
+                             ValueType *result_values,
+                             const gko::dim<2> &matrix_size,
+                             size_type result_stride)
+{
+    run_kernel(
+        exec,
+        [] GKO_KERNEL(auto row, auto col, auto stride, auto diag,
+                      auto result_values) {
+            result_values[row * stride + col] = zero<ValueType>();
+            if (row == col) {
+                result_values[row * stride + col] = diag[row];
+            }
+        },
+        matrix_size, result_stride, blocks.get_const_data(), result_values);
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
+    GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
+
+
 }  // namespace jacobi
 }  // namespace GKO_DEVICE_NAMESPACE
 }  // namespace kernels
diff --git a/common/preconditioner/jacobi_kernels.hpp.inc b/common/preconditioner/jacobi_kernels.hpp.inc
index 704c4870c23..c3fa889b210 100644
--- a/common/preconditioner/jacobi_kernels.hpp.inc
+++ b/common/preconditioner/jacobi_kernels.hpp.inc
@@ -213,59 +213,3 @@ __launch_bounds__(warps_per_block *config::warp_size) adaptive_transpose_jacobi(
             });
     }
 }
-
-
-namespace kernel {
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void scalar_apply(
-    size_type num_rows, size_type num_cols, const ValueType *__restrict__ diag,
-    const ValueType *__restrict__ alpha, size_type source_stride,
-    const ValueType *__restrict__ source_values,
-    const ValueType *__restrict__ beta, size_type result_stride,
-    ValueType *__restrict__ result_values)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto row = tidx / num_cols;
-    const auto col = tidx % num_cols;
-
-    if (row < num_rows) {
-        result_values[row * result_stride + col] =
-            result_values[row * result_stride + col] * beta[0] +
-            alpha[0] * source_values[row * source_stride + col] * diag[row];
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void invert_diagonal(
-    size_type num_elems, const ValueType *__restrict__ diag,
-    ValueType *__restrict__ inv_diag)
-{
-    const auto tidx = thread::get_thread_id_flat();
-
-    if (tidx < num_elems) {
-        inv_diag[tidx] = ValueType(1.0) / diag[tidx];
-    }
-}
-
-
-template <typename ValueType>
-__global__ __launch_bounds__(default_block_size) void simple_scalar_apply(
-    size_type num_rows, size_type num_cols, const ValueType *__restrict__ diag,
-    size_type source_stride, const ValueType *__restrict__ source_values,
-    size_type result_stride, ValueType *__restrict__ result_values)
-{
-    const auto tidx = thread::get_thread_id_flat();
-    const auto row = tidx / num_cols;
-    const auto col = tidx % num_cols;
-
-    if (row < num_rows) {
-        result_values[row * result_stride + col] =
-            source_values[row * source_stride + col] * diag[row];
-    }
-}
-
-
-}  // namespace kernel
diff --git a/cuda/preconditioner/jacobi_kernels.cu b/cuda/preconditioner/jacobi_kernels.cu
index 75b5f797d4e..1ff609791c7 100644
--- a/cuda/preconditioner/jacobi_kernels.cu
+++ b/cuda/preconditioner/jacobi_kernels.cu
@@ -230,17 +230,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename ValueType>
-void scalar_convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,
-                             const Array<ValueType> &blocks,
-                             ValueType *result_values,
-                             const gko::dim<2> &mat_size,
-                             size_type result_stride) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void convert_to_dense(
     std::shared_ptr<const DefaultExecutor> exec, size_type num_blocks,
@@ -254,61 +243,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL);
 
 
-template <typename ValueType>
-void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                  const Array<ValueType> &diag,
-                  const matrix::Dense<ValueType> *alpha,
-                  const matrix::Dense<ValueType> *b,
-                  const matrix::Dense<ValueType> *beta,
-                  matrix::Dense<ValueType> *x)
-{
-    const auto b_size = b->get_size();
-    const auto num_rows = b_size[0];
-    const auto num_cols = b_size[1];
-    const auto b_stride = b->get_stride();
-    const auto x_stride = x->get_stride();
-    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
-
-    const auto b_values = b->get_const_values();
-    const auto diag_values = diag.get_const_data();
-    auto x_values = x->get_values();
-
-    kernel::scalar_apply<<<grid_dim, default_block_size>>>(
-        num_rows, num_cols, as_cuda_type(diag_values),
-        as_cuda_type(alpha->get_const_values()), b_stride,
-        as_cuda_type(b_values), as_cuda_type(beta->get_const_values()),
-        x_stride, as_cuda_type(x_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
-
-
-template <typename ValueType>
-void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                         const Array<ValueType> &diag,
-                         const matrix::Dense<ValueType> *b,
-                         matrix::Dense<ValueType> *x)
-{
-    const auto b_size = b->get_size();
-    const auto num_rows = b_size[0];
-    const auto num_cols = b_size[1];
-    const auto b_stride = b->get_stride();
-    const auto x_stride = x->get_stride();
-    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
-
-    const auto b_values = b->get_const_values();
-    const auto diag_values = diag.get_const_data();
-    auto x_values = x->get_values();
-
-    kernel::simple_scalar_apply<<<grid_dim, default_block_size>>>(
-        num_rows, num_cols, as_cuda_type(diag_values), b_stride,
-        as_cuda_type(b_values), x_stride, as_cuda_type(x_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL);
-
-
 }  // namespace jacobi
 }  // namespace cuda
 }  // namespace kernels
diff --git a/dpcpp/preconditioner/jacobi_kernels.dp.cpp b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
index 4f1c9c6cc0f..623e2c121ec 100644
--- a/dpcpp/preconditioner/jacobi_kernels.dp.cpp
+++ b/dpcpp/preconditioner/jacobi_kernels.dp.cpp
@@ -267,17 +267,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename ValueType>
-void scalar_convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,
-                             const Array<ValueType> &blocks,
-                             ValueType *result_values,
-                             const gko::dim<2> &mat_size,
-                             size_type result_stride) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void convert_to_dense(
     std::shared_ptr<const DpcppExecutor> exec, size_type num_blocks,
@@ -291,27 +280,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL);
 
 
-template <typename ValueType>
-void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                  const Array<ValueType> &diag,
-                  const matrix::Dense<ValueType> *alpha,
-                  const matrix::Dense<ValueType> *b,
-                  const matrix::Dense<ValueType> *beta,
-                  matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
-
-
-template <typename ValueType>
-void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                         const Array<ValueType> &diag,
-                         const matrix::Dense<ValueType> *b,
-                         matrix::Dense<ValueType> *x) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL);
-
-
 }  // namespace jacobi
 }  // namespace dpcpp
 }  // namespace kernels
diff --git a/hip/preconditioner/jacobi_kernels.hip.cpp b/hip/preconditioner/jacobi_kernels.hip.cpp
index 0be2a3db67b..425820ead4e 100644
--- a/hip/preconditioner/jacobi_kernels.hip.cpp
+++ b/hip/preconditioner/jacobi_kernels.hip.cpp
@@ -244,17 +244,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONJ_TRANSPOSE_KERNEL);
 
 
-template <typename ValueType>
-void scalar_convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,
-                             const Array<ValueType> &blocks,
-                             ValueType *result_values,
-                             const gko::dim<2> &mat_size,
-                             size_type result_stride) GKO_NOT_IMPLEMENTED;
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
-
-
 template <typename ValueType, typename IndexType>
 void convert_to_dense(
     std::shared_ptr<const HipExecutor> exec, size_type num_blocks,
@@ -268,62 +257,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL);
 
 
-template <typename ValueType>
-void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                  const Array<ValueType> &diag,
-                  const matrix::Dense<ValueType> *alpha,
-                  const matrix::Dense<ValueType> *b,
-                  const matrix::Dense<ValueType> *beta,
-                  matrix::Dense<ValueType> *x)
-{
-    const auto b_size = b->get_size();
-    const auto num_rows = b_size[0];
-    const auto num_cols = b_size[1];
-    const auto b_stride = b->get_stride();
-    const auto x_stride = x->get_stride();
-    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
-
-    const auto b_values = b->get_const_values();
-    const auto diag_values = diag.get_const_data();
-    auto x_values = x->get_values();
-
-    hipLaunchKernelGGL(
-        kernel::scalar_apply, dim3(grid_dim), dim3(default_block_size), 0, 0,
-        num_rows, num_cols, as_hip_type(diag_values),
-        as_hip_type(alpha->get_const_values()), b_stride, as_hip_type(b_values),
-        as_hip_type(beta->get_const_values()), x_stride, as_hip_type(x_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
-
-
-template <typename ValueType>
-void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                         const Array<ValueType> &diag,
-                         const matrix::Dense<ValueType> *b,
-                         matrix::Dense<ValueType> *x)
-{
-    const auto b_size = b->get_size();
-    const auto num_rows = b_size[0];
-    const auto num_cols = b_size[1];
-    const auto b_stride = b->get_stride();
-    const auto x_stride = x->get_stride();
-    const auto grid_dim = ceildiv(num_rows * num_cols, default_block_size);
-
-    const auto b_values = b->get_const_values();
-    const auto diag_values = diag.get_const_data();
-    auto x_values = x->get_values();
-
-    hipLaunchKernelGGL(kernel::simple_scalar_apply, dim3(grid_dim),
-                       dim3(default_block_size), 0, 0, num_rows, num_cols,
-                       as_hip_type(diag_values), b_stride,
-                       as_hip_type(b_values), x_stride, as_hip_type(x_values));
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL);
-
-
 }  // namespace jacobi
 }  // namespace hip
 }  // namespace kernels
diff --git a/omp/preconditioner/jacobi_kernels.cpp b/omp/preconditioner/jacobi_kernels.cpp
index a49ffe8aa49..1b0c0c6c1a5 100644
--- a/omp/preconditioner/jacobi_kernels.cpp
+++ b/omp/preconditioner/jacobi_kernels.cpp
@@ -696,68 +696,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
     GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL);
 
 
-template <typename ValueType>
-void scalar_convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,
-                             const Array<ValueType> &blocks,
-                             ValueType *result_values,
-                             const gko::dim<2> &matrix_size,
-                             size_type result_stride)
-{
-#pragma omp parallel for
-    for (size_type i = 0; i < matrix_size[0]; ++i) {
-        for (size_type j = 0; j < matrix_size[1]; ++j) {
-            result_values[i * result_stride + j] = zero<ValueType>();
-            if (i == j) {
-                result_values[i * result_stride + j] =
-                    blocks.get_const_data()[i];
-            }
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL);
-
-
-template <typename ValueType>
-void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                  const Array<ValueType> &diag,
-                  const matrix::Dense<ValueType> *alpha,
-                  const matrix::Dense<ValueType> *b,
-                  const matrix::Dense<ValueType> *beta,
-                  matrix::Dense<ValueType> *x)
-{
-#pragma omp parallel for
-    for (size_type i = 0; i < x->get_size()[0]; ++i) {
-        for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            x->at(i, j) = beta->at(0) * x->at(i, j) +
-                          alpha->at(0) * b->at(i, j) * diag.get_const_data()[i];
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_APPLY_KERNEL);
-
-
-template <typename ValueType>
-void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
-                         const Array<ValueType> &diag,
-                         const matrix::Dense<ValueType> *b,
-                         matrix::Dense<ValueType> *x)
-{
-#pragma omp parallel for
-    for (size_type i = 0; i < x->get_size()[0]; ++i) {
-        for (size_type j = 0; j < x->get_size()[1]; ++j) {
-            auto diag_val = diag.get_const_data()[i];
-            x->at(i, j) = b->at(i, j) * diag_val;
-        }
-    }
-}
-
-GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
-    GKO_DECLARE_JACOBI_SIMPLE_SCALAR_APPLY_KERNEL);
-
-
 }  // namespace jacobi
 }  // namespace omp
 }  // namespace kernels

From 1da5a27886b59ba2eea8101cb48c7f6275762146 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Fri, 6 Aug 2021 12:00:38 +0200
Subject: [PATCH 0996/1008] Review update.

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
Co-authored-by: Yu-Hsiang Tsai <yhmtsai@gmail.com>
---
 common/preconditioner/jacobi_kernels.cpp    | 29 ++++++++----------
 core/preconditioner/jacobi.cpp              |  3 +-
 core/preconditioner/jacobi_kernels.hpp      |  9 +++---
 reference/preconditioner/jacobi_kernels.cpp | 16 +++++-----
 reference/test/preconditioner/jacobi.cpp    | 34 ++++++++++++++-------
 5 files changed, 48 insertions(+), 43 deletions(-)

diff --git a/common/preconditioner/jacobi_kernels.cpp b/common/preconditioner/jacobi_kernels.cpp
index 85d4bce3ff7..442a1b47809 100644
--- a/common/preconditioner/jacobi_kernels.cpp
+++ b/common/preconditioner/jacobi_kernels.cpp
@@ -59,7 +59,7 @@ void scalar_conj(std::shared_ptr<const DefaultExecutor> exec,
         [] GKO_KERNEL(auto elem, auto diag, auto conj_diag) {
             conj_diag[elem] = conj(diag[elem]);
         },
-        diag.get_num_elems(), diag.get_const_data(), conj_diag.get_data());
+        diag.get_num_elems(), diag, conj_diag);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_SCALAR_CONJ_KERNEL);
@@ -69,14 +69,12 @@ template <typename ValueType>
 void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
                      const Array<ValueType> &diag, Array<ValueType> &inv_diag)
 {
-    auto one_val = one<ValueType>();
     run_kernel(
         exec,
-        [] GKO_KERNEL(auto elem, auto diag, auto inv_diag, auto one_val) {
-            inv_diag[elem] = one_val / diag[elem];
+        [] GKO_KERNEL(auto elem, auto diag, auto inv_diag) {
+            inv_diag[elem] = safe_divide(one(diag[elem]), diag[elem]);
         },
-        diag.get_num_elems(), diag.get_const_data(), inv_diag.get_data(),
-        one_val);
+        diag.get_num_elems(), diag, inv_diag);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_JACOBI_INVERT_DIAGONAL_KERNEL);
@@ -98,7 +96,7 @@ void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
                 x(row, col) = beta[col] * x(row, col) +
                               alpha[col] * b(row, col) * diag[row];
             },
-            x->get_size(), diag.get_const_data(), alpha->get_const_values(), b,
+            x->get_size(), diag, alpha->get_const_values(), b,
             beta->get_const_values(), x);
     } else {
         run_kernel(
@@ -108,7 +106,7 @@ void scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
                 x(row, col) =
                     beta[0] * x(row, col) + alpha[0] * b(row, col) * diag[row];
             },
-            x->get_size(), diag.get_const_data(), alpha->get_const_values(), b,
+            x->get_size(), diag, alpha->get_const_values(), b,
             beta->get_const_values(), x);
     }
 }
@@ -127,7 +125,7 @@ void simple_scalar_apply(std::shared_ptr<const DefaultExecutor> exec,
         [] GKO_KERNEL(auto row, auto col, auto diag, auto b, auto x) {
             x(row, col) = b(row, col) * diag[row];
         },
-        x->get_size(), diag.get_const_data(), b, x);
+        x->get_size(), diag, b, x);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
@@ -137,20 +135,17 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
 template <typename ValueType>
 void scalar_convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,
                              const Array<ValueType> &blocks,
-                             ValueType *result_values,
-                             const gko::dim<2> &matrix_size,
-                             size_type result_stride)
+                             matrix::Dense<ValueType> *result)
 {
     run_kernel(
         exec,
-        [] GKO_KERNEL(auto row, auto col, auto stride, auto diag,
-                      auto result_values) {
-            result_values[row * stride + col] = zero<ValueType>();
+        [] GKO_KERNEL(auto row, auto col, auto diag, auto result) {
+            result(row, col) = zero(diag[row]);
             if (row == col) {
-                result_values[row * stride + col] = diag[row];
+                result(row, col) = diag[row];
             }
         },
-        matrix_size, result_stride, blocks.get_const_data(), result_values);
+        result->get_size(), blocks, result);
 }
 
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(
diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp
index d52d2959f6d..d62d97ec0b5 100644
--- a/core/preconditioner/jacobi.cpp
+++ b/core/preconditioner/jacobi.cpp
@@ -124,8 +124,7 @@ void Jacobi<ValueType, IndexType>::convert_to(
     auto exec = this->get_executor();
     auto tmp = matrix::Dense<ValueType>::create(exec, this->get_size());
     if (parameters_.max_block_size == 1) {
-        exec->run(jacobi::make_scalar_convert_to_dense(
-            blocks_, tmp->get_values(), tmp->get_size(), tmp->get_stride()));
+        exec->run(jacobi::make_scalar_convert_to_dense(blocks_, tmp.get()));
     } else {
         exec->run(jacobi::make_convert_to_dense(
             num_blocks_, parameters_.storage_optimization.block_wise,
diff --git a/core/preconditioner/jacobi_kernels.hpp b/core/preconditioner/jacobi_kernels.hpp
index 2960e175de4..7745438abda 100644
--- a/core/preconditioner/jacobi_kernels.hpp
+++ b/core/preconditioner/jacobi_kernels.hpp
@@ -130,11 +130,10 @@ namespace kernels {
             &storage_scheme,                                               \
         Array<ValueType> &out_blocks)
 
-#define GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL(ValueType) \
-    void scalar_convert_to_dense(                                    \
-        std::shared_ptr<const DefaultExecutor> exec,                 \
-        const Array<ValueType> &blocks, ValueType *result_values,    \
-        const gko::dim<2> &mat_size, size_type result_stride)
+#define GKO_DECLARE_JACOBI_SCALAR_CONVERT_TO_DENSE_KERNEL(ValueType)          \
+    void scalar_convert_to_dense(std::shared_ptr<const DefaultExecutor> exec, \
+                                 const Array<ValueType> &blocks,              \
+                                 matrix::Dense<ValueType> *result)
 
 #define GKO_DECLARE_JACOBI_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType)   \
     void convert_to_dense(                                                 \
diff --git a/reference/preconditioner/jacobi_kernels.cpp b/reference/preconditioner/jacobi_kernels.cpp
index 9799329f773..e8abd8b8958 100644
--- a/reference/preconditioner/jacobi_kernels.cpp
+++ b/reference/preconditioner/jacobi_kernels.cpp
@@ -613,8 +613,10 @@ void invert_diagonal(std::shared_ptr<const DefaultExecutor> exec,
                      const Array<ValueType> &diag, Array<ValueType> &inv_diag)
 {
     for (size_type i = 0; i < diag.get_num_elems(); ++i) {
-        inv_diag.get_data()[i] =
-            static_cast<ValueType>(1.0) / diag.get_const_data()[i];
+        auto diag_val = diag.get_const_data()[i] == zero<ValueType>()
+                            ? one<ValueType>()
+                            : diag.get_const_data()[i];
+        inv_diag.get_data()[i] = one<ValueType>() / diag_val;
     }
 }
 
@@ -696,16 +698,14 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
 template <typename ValueType>
 void scalar_convert_to_dense(std::shared_ptr<const DefaultExecutor> exec,
                              const Array<ValueType> &blocks,
-                             ValueType *result_values,
-                             const gko::dim<2> &matrix_size,
-                             size_type result_stride)
+                             matrix::Dense<ValueType> *result)
 {
+    auto matrix_size = result->get_size();
     for (size_type i = 0; i < matrix_size[0]; ++i) {
         for (size_type j = 0; j < matrix_size[1]; ++j) {
-            result_values[i * result_stride + j] = zero<ValueType>();
+            result->at(i, j) = zero<ValueType>();
             if (i == j) {
-                result_values[i * result_stride + j] =
-                    blocks.get_const_data()[i];
+                result->at(i, j) = blocks.get_const_data()[i];
             }
         }
     }
diff --git a/reference/test/preconditioner/jacobi.cpp b/reference/test/preconditioner/jacobi.cpp
index 390a575290e..330195cee6a 100644
--- a/reference/test/preconditioner/jacobi.cpp
+++ b/reference/test/preconditioner/jacobi.cpp
@@ -107,9 +107,7 @@ class Jacobi : public ::testing::Test {
     template <typename T>
     void init_array(T *arr, std::initializer_list<T> vals)
     {
-        for (auto elem : vals) {
-            *(arr++) = elem;
-        }
+        std::copy(std::begin(vals), std::end(vals), arr);
     }
 
     template <typename ValueType>
@@ -325,11 +323,15 @@ TYPED_TEST(Jacobi, ScalarJacobiConvertsToDense)
     dense_j->copy_from(scalar_j.get());
     auto j_val = scalar_j->get_blocks();
 
-    EXPECT_EQ(dense_j->at(0, 0), j_val[0]);
-    EXPECT_EQ(dense_j->at(1, 1), j_val[1]);
-    EXPECT_EQ(dense_j->at(2, 2), j_val[2]);
-    EXPECT_EQ(dense_j->at(3, 3), j_val[3]);
-    EXPECT_EQ(dense_j->at(4, 4), j_val[4]);
+    for (auto i = 0; i < dense_j->get_size()[0]; ++i) {
+        for (auto j = 0; j < dense_j->get_size()[1]; ++j) {
+            if (i == j) {
+                EXPECT_EQ(dense_j->at(i, j), j_val[j]);
+            } else {
+                EXPECT_EQ(dense_j->at(i, j), value_type{0.0});
+            }
+        }
+    }
 }
 
 
@@ -349,6 +351,12 @@ TYPED_TEST(Jacobi, ScalarJacobiCanBeTransposed)
     auto trans_j = gko::as<Bj>(t_j.get())->get_blocks();
     auto scal_j = scalar_j->get_blocks();
 
+    ASSERT_EQ(scalar_j->get_size(), gko::dim<2>(5, 5));
+    ASSERT_EQ(scalar_j->get_num_stored_elements(), 5);
+    ASSERT_EQ(scalar_j->get_parameters().max_block_size, 1);
+    ASSERT_EQ(scalar_j->get_num_blocks(), 5);
+    ASSERT_EQ(scalar_j->get_parameters().block_pointers.get_const_data(),
+              nullptr);
     EXPECT_EQ(trans_j[0], scal_j[0]);
     EXPECT_EQ(trans_j[1], scal_j[1]);
     EXPECT_EQ(trans_j[2], scal_j[2]);
@@ -360,9 +368,7 @@ TYPED_TEST(Jacobi, ScalarJacobiCanBeTransposed)
 template <typename T>
 void init_array(T *arr, std::initializer_list<T> vals)
 {
-    for (auto elem : vals) {
-        *(arr++) = elem;
-    }
+    std::copy(std::begin(vals), std::end(vals), arr);
 }
 
 
@@ -390,6 +396,12 @@ TEST(Jacobi, ScalarJacobiCanBeConjTransposed)
     auto trans_j = gko::as<Bj>(t_j.get())->get_blocks();
     auto scal_j = scalar_j->get_blocks();
 
+    ASSERT_EQ(scalar_j->get_size(), gko::dim<2>(5, 5));
+    ASSERT_EQ(scalar_j->get_num_stored_elements(), 5);
+    ASSERT_EQ(scalar_j->get_parameters().max_block_size, 1);
+    ASSERT_EQ(scalar_j->get_num_blocks(), 5);
+    ASSERT_EQ(scalar_j->get_parameters().block_pointers.get_const_data(),
+              nullptr);
     EXPECT_EQ(trans_j[0], gko::conj(scal_j[0]));
     EXPECT_EQ(trans_j[1], gko::conj(scal_j[1]));
     EXPECT_EQ(trans_j[2], gko::conj(scal_j[2]));

From 4e91906ccdf2ddcff54237ccfced7b04f2a63c94 Mon Sep 17 00:00:00 2001
From: Pratik Nayak <pratikvn@protonmail.com>
Date: Fri, 6 Aug 2021 13:53:07 +0200
Subject: [PATCH 0997/1008] Update doc

---
 include/ginkgo/core/preconditioner/jacobi.hpp | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/include/ginkgo/core/preconditioner/jacobi.hpp b/include/ginkgo/core/preconditioner/jacobi.hpp
index dd05296abde..76e571baf10 100644
--- a/include/ginkgo/core/preconditioner/jacobi.hpp
+++ b/include/ginkgo/core/preconditioner/jacobi.hpp
@@ -200,6 +200,10 @@ struct block_interleaved_storage_scheme {
  * @note When using the adaptive variant, there may be a trade-off in terms of
  *       slightly longer preconditioner generation due to extra work required to
  *       detect the optimal precision of the blocks.
+ * @note When the max_block_size is set to 1, specialized  kernels are used,
+ *       both for generation (inverting the diagonals) and application (diagonal
+ *       scaling) to reduce the overhead involved in the usual (adaptive) block
+ *       case.
  *
  * @ingroup jacobi
  * @ingroup precond
@@ -300,7 +304,10 @@ class Jacobi : public EnableLinOp<Jacobi<ValueType, IndexType>>,
         /**
          * Maximal size of diagonal blocks.
          *
-         * @note This value has to be between 1 and 32 (NVIDIA)/64 (AMD).
+         * @note This value has to be between 1 and 32 (NVIDIA)/64 (AMD). For
+         * efficiency, when the max_block_size is set to 1, specialized kernels
+         * are used and the additional objects (block_ptrs etc) are set to null
+         * values.
          */
         uint32 GKO_FACTORY_PARAMETER_SCALAR(max_block_size, 32u);
 

From 7475a0f8b92b7713e7b2702ccbeb7f3f6ae7ebb2 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Fri, 6 Aug 2021 12:15:45 +0200
Subject: [PATCH 0998/1008] Split common kernel implementations into cuda_hip
 and unified

---
 common/{ => cuda_hip}/base/executor.hpp.inc   |  0
 common/{ => cuda_hip}/base/math.hpp.inc       |  0
 .../components/absolute_array.hpp.inc         |  0
 .../{ => cuda_hip}/components/atomic.hpp.inc  |  0
 .../diagonal_block_manipulation.hpp.inc       |  0
 .../components/fill_array.hpp.inc             |  0
 .../components/intrinsics.hpp.inc             |  0
 .../{ => cuda_hip}/components/merging.hpp.inc |  0
 .../components/prefix_sum.hpp.inc             |  0
 .../components/reduction.hpp.inc              |  0
 .../components/searching.hpp.inc              |  0
 .../components/segment_scan.hpp.inc           |  0
 .../{ => cuda_hip}/components/sorting.hpp.inc |  0
 .../components/thread_ids.hpp.inc             |  0
 .../components/uninitialized_array.hpp.inc    |  0
 .../components/warp_blas.hpp.inc              |  0
 .../factorization_kernels.hpp.inc             |  0
 .../factorization/par_ic_kernels.hpp.inc      |  0
 .../par_ict_spgeam_kernels.hpp.inc            |  0
 .../par_ict_sweep_kernels.hpp.inc             |  0
 .../factorization/par_ilu_kernels.hpp.inc     |  0
 .../par_ilut_filter_kernels.hpp.inc           |  0
 .../par_ilut_select_kernels.hpp.inc           |  0
 .../par_ilut_spgeam_kernels.hpp.inc           |  0
 .../par_ilut_sweep_kernels.hpp.inc            |  0
 .../{ => cuda_hip}/matrix/coo_kernels.hpp.inc |  0
 .../{ => cuda_hip}/matrix/csr_kernels.hpp.inc |  0
 .../matrix/dense_kernels.hpp.inc              |  0
 .../matrix/diagonal_kernels.hpp.inc           |  0
 .../{ => cuda_hip}/matrix/ell_kernels.hpp.inc |  0
 .../matrix/hybrid_kernels.hpp.inc             |  0
 .../matrix/sellp_kernels.hpp.inc              |  0
 .../multigrid/amgx_pgm_kernels.hpp.inc        |  0
 .../preconditioner/isai_kernels.hpp.inc       |  0
 .../jacobi_advanced_apply_kernel.hpp.inc      |  0
 .../jacobi_generate_kernel.hpp.inc            |  0
 .../preconditioner/jacobi_kernels.hpp.inc     |  0
 .../jacobi_simple_apply_kernel.hpp.inc        |  0
 .../solver/cb_gmres_kernels.hpp.inc           |  0
 .../solver/common_gmres_kernels.hpp.inc       |  0
 .../solver/gmres_kernels.hpp.inc              |  0
 .../{ => cuda_hip}/solver/idr_kernels.hpp.inc |  0
 common/{ => unified}/base/kernel_launch.hpp   |  0
 .../base/kernel_launch_solver.hpp             |  2 +-
 .../components/precision_conversion.cpp       |  2 +-
 common/{ => unified}/matrix/coo_kernels.cpp   |  2 +-
 common/{ => unified}/matrix/csr_kernels.cpp   |  2 +-
 common/{ => unified}/matrix/dense_kernels.cpp |  2 +-
 .../{ => unified}/matrix/diagonal_kernels.cpp |  2 +-
 .../preconditioner/jacobi_kernels.cpp         |  2 +-
 common/{ => unified}/solver/bicg_kernels.cpp  |  2 +-
 .../{ => unified}/solver/bicgstab_kernels.cpp |  2 +-
 common/{ => unified}/solver/cg_kernels.cpp    |  2 +-
 common/{ => unified}/solver/cgs_kernels.cpp   |  2 +-
 common/{ => unified}/solver/fcg_kernels.cpp   |  2 +-
 common/{ => unified}/solver/ir_kernels.cpp    |  2 +-
 cuda/CMakeLists.txt                           | 24 +++++++++----------
 cuda/base/executor.cpp                        |  2 +-
 cuda/base/kernel_launch.cuh                   |  2 +-
 cuda/base/kernel_launch_solver.cuh            |  2 +-
 cuda/base/math.hpp                            |  2 +-
 cuda/components/absolute_array.cu             |  2 +-
 cuda/components/atomic.cuh                    |  2 +-
 .../diagonal_block_manipulation.cuh           |  2 +-
 cuda/components/fill_array.cu                 |  2 +-
 cuda/components/intrinsics.cuh                |  2 +-
 cuda/components/merging.cuh                   |  2 +-
 cuda/components/prefix_sum.cuh                |  2 +-
 cuda/components/reduction.cuh                 |  2 +-
 cuda/components/searching.cuh                 |  2 +-
 cuda/components/segment_scan.cuh              |  2 +-
 cuda/components/sorting.cuh                   |  2 +-
 cuda/components/thread_ids.cuh                |  2 +-
 cuda/components/uninitialized_array.hpp       |  2 +-
 cuda/components/warp_blas.cuh                 |  2 +-
 cuda/factorization/factorization_kernels.cu   |  2 +-
 cuda/factorization/par_ic_kernels.cu          |  2 +-
 cuda/factorization/par_ict_kernels.cu         |  4 ++--
 cuda/factorization/par_ilu_kernels.cu         |  2 +-
 .../par_ilut_approx_filter_kernel.cu          |  4 ++--
 cuda/factorization/par_ilut_filter_kernel.cu  |  2 +-
 cuda/factorization/par_ilut_select_common.cu  |  2 +-
 cuda/factorization/par_ilut_select_kernel.cu  |  2 +-
 cuda/factorization/par_ilut_spgeam_kernel.cu  |  2 +-
 cuda/factorization/par_ilut_sweep_kernel.cu   |  2 +-
 cuda/matrix/coo_kernels.cu                    |  2 +-
 cuda/matrix/csr_kernels.cu                    |  2 +-
 cuda/matrix/dense_kernels.cu                  |  2 +-
 cuda/matrix/diagonal_kernels.cu               |  2 +-
 cuda/matrix/ell_kernels.cu                    |  2 +-
 cuda/matrix/hybrid_kernels.cu                 |  2 +-
 cuda/matrix/sellp_kernels.cu                  |  2 +-
 cuda/multigrid/amgx_pgm_kernels.cu            |  2 +-
 cuda/preconditioner/isai_kernels.cu           |  6 ++---
 .../jacobi_advanced_apply_kernel.cu           |  2 +-
 cuda/preconditioner/jacobi_generate_kernel.cu |  2 +-
 cuda/preconditioner/jacobi_kernels.cu         |  2 +-
 .../jacobi_simple_apply_kernel.cu             |  2 +-
 cuda/solver/cb_gmres_kernels.cu               |  2 +-
 cuda/solver/gmres_kernels.cu                  |  2 +-
 cuda/solver/idr_kernels.cu                    |  2 +-
 cuda/test/base/cuda_executor.cu               |  2 +-
 cuda/test/base/kernel_launch.cu               |  4 ++--
 dev_tools/scripts/config                      |  6 ++---
 dpcpp/CMakeLists.txt                          | 24 +++++++++----------
 dpcpp/base/kernel_launch.dp.hpp               |  2 +-
 dpcpp/base/kernel_launch_solver.dp.hpp        |  2 +-
 dpcpp/test/base/kernel_launch.dp.cpp          |  4 ++--
 hip/CMakeLists.txt                            | 24 +++++++++----------
 hip/base/executor.hip.cpp                     |  2 +-
 hip/base/kernel_launch.hip.hpp                |  2 +-
 hip/base/kernel_launch_solver.hip.hpp         |  2 +-
 hip/base/math.hip.hpp                         |  2 +-
 hip/components/absolute_array.hip.cpp         |  2 +-
 hip/components/atomic.hip.hpp                 |  2 +-
 .../diagonal_block_manipulation.hip.hpp       |  2 +-
 hip/components/fill_array.hip.cpp             |  2 +-
 hip/components/intrinsics.hip.hpp             |  2 +-
 hip/components/merging.hip.hpp                |  2 +-
 hip/components/prefix_sum.hip.hpp             |  2 +-
 hip/components/reduction.hip.hpp              |  2 +-
 hip/components/searching.hip.hpp              |  2 +-
 hip/components/segment_scan.hip.hpp           |  2 +-
 hip/components/sorting.hip.hpp                |  2 +-
 hip/components/thread_ids.hip.hpp             |  2 +-
 hip/components/uninitialized_array.hip.hpp    |  2 +-
 hip/components/warp_blas.hip.hpp              |  2 +-
 .../factorization_kernels.hip.cpp             |  2 +-
 hip/factorization/par_ic_kernels.hip.cpp      |  2 +-
 hip/factorization/par_ict_kernels.hip.cpp     |  4 ++--
 hip/factorization/par_ilu_kernels.hip.cpp     |  2 +-
 .../par_ilut_approx_filter_kernel.hip.cpp     |  4 ++--
 .../par_ilut_filter_kernel.hip.cpp            |  2 +-
 .../par_ilut_select_common.hip.cpp            |  2 +-
 .../par_ilut_select_kernel.hip.cpp            |  2 +-
 .../par_ilut_spgeam_kernel.hip.cpp            |  2 +-
 .../par_ilut_sweep_kernel.hip.cpp             |  2 +-
 hip/matrix/coo_kernels.hip.cpp                |  2 +-
 hip/matrix/csr_kernels.hip.cpp                |  2 +-
 hip/matrix/dense_kernels.hip.cpp              |  2 +-
 hip/matrix/diagonal_kernels.hip.cpp           |  2 +-
 hip/matrix/ell_kernels.hip.cpp                |  2 +-
 hip/matrix/hybrid_kernels.hip.cpp             |  2 +-
 hip/matrix/sellp_kernels.hip.cpp              |  2 +-
 hip/multigrid/amgx_pgm_kernels.hip.cpp        |  2 +-
 hip/preconditioner/isai_kernels.hip.cpp       |  6 ++---
 .../jacobi_advanced_apply_kernel.hip.cpp      |  2 +-
 .../jacobi_generate_kernel.hip.cpp            |  2 +-
 hip/preconditioner/jacobi_kernels.hip.cpp     |  2 +-
 .../jacobi_simple_apply_kernel.hip.cpp        |  2 +-
 hip/solver/cb_gmres_kernels.hip.cpp           |  2 +-
 hip/solver/gmres_kernels.hip.cpp              |  2 +-
 hip/solver/idr_kernels.hip.cpp                |  2 +-
 hip/test/base/hip_executor.hip.cpp            |  2 +-
 hip/test/base/kernel_launch.hip.cpp           |  4 ++--
 omp/CMakeLists.txt                            | 24 +++++++++----------
 omp/base/kernel_launch.hpp                    |  2 +-
 omp/base/kernel_launch_solver.hpp             |  2 +-
 omp/test/base/kernel_launch.cpp               |  4 ++--
 159 files changed, 174 insertions(+), 174 deletions(-)
 rename common/{ => cuda_hip}/base/executor.hpp.inc (100%)
 rename common/{ => cuda_hip}/base/math.hpp.inc (100%)
 rename common/{ => cuda_hip}/components/absolute_array.hpp.inc (100%)
 rename common/{ => cuda_hip}/components/atomic.hpp.inc (100%)
 rename common/{ => cuda_hip}/components/diagonal_block_manipulation.hpp.inc (100%)
 rename common/{ => cuda_hip}/components/fill_array.hpp.inc (100%)
 rename common/{ => cuda_hip}/components/intrinsics.hpp.inc (100%)
 rename common/{ => cuda_hip}/components/merging.hpp.inc (100%)
 rename common/{ => cuda_hip}/components/prefix_sum.hpp.inc (100%)
 rename common/{ => cuda_hip}/components/reduction.hpp.inc (100%)
 rename common/{ => cuda_hip}/components/searching.hpp.inc (100%)
 rename common/{ => cuda_hip}/components/segment_scan.hpp.inc (100%)
 rename common/{ => cuda_hip}/components/sorting.hpp.inc (100%)
 rename common/{ => cuda_hip}/components/thread_ids.hpp.inc (100%)
 rename common/{ => cuda_hip}/components/uninitialized_array.hpp.inc (100%)
 rename common/{ => cuda_hip}/components/warp_blas.hpp.inc (100%)
 rename common/{ => cuda_hip}/factorization/factorization_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/factorization/par_ic_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/factorization/par_ict_spgeam_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/factorization/par_ict_sweep_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/factorization/par_ilu_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/factorization/par_ilut_filter_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/factorization/par_ilut_select_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/factorization/par_ilut_spgeam_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/factorization/par_ilut_sweep_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/matrix/coo_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/matrix/csr_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/matrix/dense_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/matrix/diagonal_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/matrix/ell_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/matrix/hybrid_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/matrix/sellp_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/multigrid/amgx_pgm_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/preconditioner/isai_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/preconditioner/jacobi_advanced_apply_kernel.hpp.inc (100%)
 rename common/{ => cuda_hip}/preconditioner/jacobi_generate_kernel.hpp.inc (100%)
 rename common/{ => cuda_hip}/preconditioner/jacobi_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/preconditioner/jacobi_simple_apply_kernel.hpp.inc (100%)
 rename common/{ => cuda_hip}/solver/cb_gmres_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/solver/common_gmres_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/solver/gmres_kernels.hpp.inc (100%)
 rename common/{ => cuda_hip}/solver/idr_kernels.hpp.inc (100%)
 rename common/{ => unified}/base/kernel_launch.hpp (100%)
 rename common/{ => unified}/base/kernel_launch_solver.hpp (98%)
 rename common/{ => unified}/components/precision_conversion.cpp (97%)
 rename common/{ => unified}/matrix/coo_kernels.cpp (98%)
 rename common/{ => unified}/matrix/csr_kernels.cpp (98%)
 rename common/{ => unified}/matrix/dense_kernels.cpp (99%)
 rename common/{ => unified}/matrix/diagonal_kernels.cpp (99%)
 rename common/{ => unified}/preconditioner/jacobi_kernels.cpp (99%)
 rename common/{ => unified}/solver/bicg_kernels.cpp (99%)
 rename common/{ => unified}/solver/bicgstab_kernels.cpp (99%)
 rename common/{ => unified}/solver/cg_kernels.cpp (98%)
 rename common/{ => unified}/solver/cgs_kernels.cpp (99%)
 rename common/{ => unified}/solver/fcg_kernels.cpp (98%)
 rename common/{ => unified}/solver/ir_kernels.cpp (97%)

diff --git a/common/base/executor.hpp.inc b/common/cuda_hip/base/executor.hpp.inc
similarity index 100%
rename from common/base/executor.hpp.inc
rename to common/cuda_hip/base/executor.hpp.inc
diff --git a/common/base/math.hpp.inc b/common/cuda_hip/base/math.hpp.inc
similarity index 100%
rename from common/base/math.hpp.inc
rename to common/cuda_hip/base/math.hpp.inc
diff --git a/common/components/absolute_array.hpp.inc b/common/cuda_hip/components/absolute_array.hpp.inc
similarity index 100%
rename from common/components/absolute_array.hpp.inc
rename to common/cuda_hip/components/absolute_array.hpp.inc
diff --git a/common/components/atomic.hpp.inc b/common/cuda_hip/components/atomic.hpp.inc
similarity index 100%
rename from common/components/atomic.hpp.inc
rename to common/cuda_hip/components/atomic.hpp.inc
diff --git a/common/components/diagonal_block_manipulation.hpp.inc b/common/cuda_hip/components/diagonal_block_manipulation.hpp.inc
similarity index 100%
rename from common/components/diagonal_block_manipulation.hpp.inc
rename to common/cuda_hip/components/diagonal_block_manipulation.hpp.inc
diff --git a/common/components/fill_array.hpp.inc b/common/cuda_hip/components/fill_array.hpp.inc
similarity index 100%
rename from common/components/fill_array.hpp.inc
rename to common/cuda_hip/components/fill_array.hpp.inc
diff --git a/common/components/intrinsics.hpp.inc b/common/cuda_hip/components/intrinsics.hpp.inc
similarity index 100%
rename from common/components/intrinsics.hpp.inc
rename to common/cuda_hip/components/intrinsics.hpp.inc
diff --git a/common/components/merging.hpp.inc b/common/cuda_hip/components/merging.hpp.inc
similarity index 100%
rename from common/components/merging.hpp.inc
rename to common/cuda_hip/components/merging.hpp.inc
diff --git a/common/components/prefix_sum.hpp.inc b/common/cuda_hip/components/prefix_sum.hpp.inc
similarity index 100%
rename from common/components/prefix_sum.hpp.inc
rename to common/cuda_hip/components/prefix_sum.hpp.inc
diff --git a/common/components/reduction.hpp.inc b/common/cuda_hip/components/reduction.hpp.inc
similarity index 100%
rename from common/components/reduction.hpp.inc
rename to common/cuda_hip/components/reduction.hpp.inc
diff --git a/common/components/searching.hpp.inc b/common/cuda_hip/components/searching.hpp.inc
similarity index 100%
rename from common/components/searching.hpp.inc
rename to common/cuda_hip/components/searching.hpp.inc
diff --git a/common/components/segment_scan.hpp.inc b/common/cuda_hip/components/segment_scan.hpp.inc
similarity index 100%
rename from common/components/segment_scan.hpp.inc
rename to common/cuda_hip/components/segment_scan.hpp.inc
diff --git a/common/components/sorting.hpp.inc b/common/cuda_hip/components/sorting.hpp.inc
similarity index 100%
rename from common/components/sorting.hpp.inc
rename to common/cuda_hip/components/sorting.hpp.inc
diff --git a/common/components/thread_ids.hpp.inc b/common/cuda_hip/components/thread_ids.hpp.inc
similarity index 100%
rename from common/components/thread_ids.hpp.inc
rename to common/cuda_hip/components/thread_ids.hpp.inc
diff --git a/common/components/uninitialized_array.hpp.inc b/common/cuda_hip/components/uninitialized_array.hpp.inc
similarity index 100%
rename from common/components/uninitialized_array.hpp.inc
rename to common/cuda_hip/components/uninitialized_array.hpp.inc
diff --git a/common/components/warp_blas.hpp.inc b/common/cuda_hip/components/warp_blas.hpp.inc
similarity index 100%
rename from common/components/warp_blas.hpp.inc
rename to common/cuda_hip/components/warp_blas.hpp.inc
diff --git a/common/factorization/factorization_kernels.hpp.inc b/common/cuda_hip/factorization/factorization_kernels.hpp.inc
similarity index 100%
rename from common/factorization/factorization_kernels.hpp.inc
rename to common/cuda_hip/factorization/factorization_kernels.hpp.inc
diff --git a/common/factorization/par_ic_kernels.hpp.inc b/common/cuda_hip/factorization/par_ic_kernels.hpp.inc
similarity index 100%
rename from common/factorization/par_ic_kernels.hpp.inc
rename to common/cuda_hip/factorization/par_ic_kernels.hpp.inc
diff --git a/common/factorization/par_ict_spgeam_kernels.hpp.inc b/common/cuda_hip/factorization/par_ict_spgeam_kernels.hpp.inc
similarity index 100%
rename from common/factorization/par_ict_spgeam_kernels.hpp.inc
rename to common/cuda_hip/factorization/par_ict_spgeam_kernels.hpp.inc
diff --git a/common/factorization/par_ict_sweep_kernels.hpp.inc b/common/cuda_hip/factorization/par_ict_sweep_kernels.hpp.inc
similarity index 100%
rename from common/factorization/par_ict_sweep_kernels.hpp.inc
rename to common/cuda_hip/factorization/par_ict_sweep_kernels.hpp.inc
diff --git a/common/factorization/par_ilu_kernels.hpp.inc b/common/cuda_hip/factorization/par_ilu_kernels.hpp.inc
similarity index 100%
rename from common/factorization/par_ilu_kernels.hpp.inc
rename to common/cuda_hip/factorization/par_ilu_kernels.hpp.inc
diff --git a/common/factorization/par_ilut_filter_kernels.hpp.inc b/common/cuda_hip/factorization/par_ilut_filter_kernels.hpp.inc
similarity index 100%
rename from common/factorization/par_ilut_filter_kernels.hpp.inc
rename to common/cuda_hip/factorization/par_ilut_filter_kernels.hpp.inc
diff --git a/common/factorization/par_ilut_select_kernels.hpp.inc b/common/cuda_hip/factorization/par_ilut_select_kernels.hpp.inc
similarity index 100%
rename from common/factorization/par_ilut_select_kernels.hpp.inc
rename to common/cuda_hip/factorization/par_ilut_select_kernels.hpp.inc
diff --git a/common/factorization/par_ilut_spgeam_kernels.hpp.inc b/common/cuda_hip/factorization/par_ilut_spgeam_kernels.hpp.inc
similarity index 100%
rename from common/factorization/par_ilut_spgeam_kernels.hpp.inc
rename to common/cuda_hip/factorization/par_ilut_spgeam_kernels.hpp.inc
diff --git a/common/factorization/par_ilut_sweep_kernels.hpp.inc b/common/cuda_hip/factorization/par_ilut_sweep_kernels.hpp.inc
similarity index 100%
rename from common/factorization/par_ilut_sweep_kernels.hpp.inc
rename to common/cuda_hip/factorization/par_ilut_sweep_kernels.hpp.inc
diff --git a/common/matrix/coo_kernels.hpp.inc b/common/cuda_hip/matrix/coo_kernels.hpp.inc
similarity index 100%
rename from common/matrix/coo_kernels.hpp.inc
rename to common/cuda_hip/matrix/coo_kernels.hpp.inc
diff --git a/common/matrix/csr_kernels.hpp.inc b/common/cuda_hip/matrix/csr_kernels.hpp.inc
similarity index 100%
rename from common/matrix/csr_kernels.hpp.inc
rename to common/cuda_hip/matrix/csr_kernels.hpp.inc
diff --git a/common/matrix/dense_kernels.hpp.inc b/common/cuda_hip/matrix/dense_kernels.hpp.inc
similarity index 100%
rename from common/matrix/dense_kernels.hpp.inc
rename to common/cuda_hip/matrix/dense_kernels.hpp.inc
diff --git a/common/matrix/diagonal_kernels.hpp.inc b/common/cuda_hip/matrix/diagonal_kernels.hpp.inc
similarity index 100%
rename from common/matrix/diagonal_kernels.hpp.inc
rename to common/cuda_hip/matrix/diagonal_kernels.hpp.inc
diff --git a/common/matrix/ell_kernels.hpp.inc b/common/cuda_hip/matrix/ell_kernels.hpp.inc
similarity index 100%
rename from common/matrix/ell_kernels.hpp.inc
rename to common/cuda_hip/matrix/ell_kernels.hpp.inc
diff --git a/common/matrix/hybrid_kernels.hpp.inc b/common/cuda_hip/matrix/hybrid_kernels.hpp.inc
similarity index 100%
rename from common/matrix/hybrid_kernels.hpp.inc
rename to common/cuda_hip/matrix/hybrid_kernels.hpp.inc
diff --git a/common/matrix/sellp_kernels.hpp.inc b/common/cuda_hip/matrix/sellp_kernels.hpp.inc
similarity index 100%
rename from common/matrix/sellp_kernels.hpp.inc
rename to common/cuda_hip/matrix/sellp_kernels.hpp.inc
diff --git a/common/multigrid/amgx_pgm_kernels.hpp.inc b/common/cuda_hip/multigrid/amgx_pgm_kernels.hpp.inc
similarity index 100%
rename from common/multigrid/amgx_pgm_kernels.hpp.inc
rename to common/cuda_hip/multigrid/amgx_pgm_kernels.hpp.inc
diff --git a/common/preconditioner/isai_kernels.hpp.inc b/common/cuda_hip/preconditioner/isai_kernels.hpp.inc
similarity index 100%
rename from common/preconditioner/isai_kernels.hpp.inc
rename to common/cuda_hip/preconditioner/isai_kernels.hpp.inc
diff --git a/common/preconditioner/jacobi_advanced_apply_kernel.hpp.inc b/common/cuda_hip/preconditioner/jacobi_advanced_apply_kernel.hpp.inc
similarity index 100%
rename from common/preconditioner/jacobi_advanced_apply_kernel.hpp.inc
rename to common/cuda_hip/preconditioner/jacobi_advanced_apply_kernel.hpp.inc
diff --git a/common/preconditioner/jacobi_generate_kernel.hpp.inc b/common/cuda_hip/preconditioner/jacobi_generate_kernel.hpp.inc
similarity index 100%
rename from common/preconditioner/jacobi_generate_kernel.hpp.inc
rename to common/cuda_hip/preconditioner/jacobi_generate_kernel.hpp.inc
diff --git a/common/preconditioner/jacobi_kernels.hpp.inc b/common/cuda_hip/preconditioner/jacobi_kernels.hpp.inc
similarity index 100%
rename from common/preconditioner/jacobi_kernels.hpp.inc
rename to common/cuda_hip/preconditioner/jacobi_kernels.hpp.inc
diff --git a/common/preconditioner/jacobi_simple_apply_kernel.hpp.inc b/common/cuda_hip/preconditioner/jacobi_simple_apply_kernel.hpp.inc
similarity index 100%
rename from common/preconditioner/jacobi_simple_apply_kernel.hpp.inc
rename to common/cuda_hip/preconditioner/jacobi_simple_apply_kernel.hpp.inc
diff --git a/common/solver/cb_gmres_kernels.hpp.inc b/common/cuda_hip/solver/cb_gmres_kernels.hpp.inc
similarity index 100%
rename from common/solver/cb_gmres_kernels.hpp.inc
rename to common/cuda_hip/solver/cb_gmres_kernels.hpp.inc
diff --git a/common/solver/common_gmres_kernels.hpp.inc b/common/cuda_hip/solver/common_gmres_kernels.hpp.inc
similarity index 100%
rename from common/solver/common_gmres_kernels.hpp.inc
rename to common/cuda_hip/solver/common_gmres_kernels.hpp.inc
diff --git a/common/solver/gmres_kernels.hpp.inc b/common/cuda_hip/solver/gmres_kernels.hpp.inc
similarity index 100%
rename from common/solver/gmres_kernels.hpp.inc
rename to common/cuda_hip/solver/gmres_kernels.hpp.inc
diff --git a/common/solver/idr_kernels.hpp.inc b/common/cuda_hip/solver/idr_kernels.hpp.inc
similarity index 100%
rename from common/solver/idr_kernels.hpp.inc
rename to common/cuda_hip/solver/idr_kernels.hpp.inc
diff --git a/common/base/kernel_launch.hpp b/common/unified/base/kernel_launch.hpp
similarity index 100%
rename from common/base/kernel_launch.hpp
rename to common/unified/base/kernel_launch.hpp
diff --git a/common/base/kernel_launch_solver.hpp b/common/unified/base/kernel_launch_solver.hpp
similarity index 98%
rename from common/base/kernel_launch_solver.hpp
rename to common/unified/base/kernel_launch_solver.hpp
index b67813c1e1a..9dcc404c911 100644
--- a/common/base/kernel_launch_solver.hpp
+++ b/common/unified/base/kernel_launch_solver.hpp
@@ -34,7 +34,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
 
 
-#include "common/base/kernel_launch.hpp"
+#include "common/unified/base/kernel_launch.hpp"
 
 
 namespace gko {
diff --git a/common/components/precision_conversion.cpp b/common/unified/components/precision_conversion.cpp
similarity index 97%
rename from common/components/precision_conversion.cpp
rename to common/unified/components/precision_conversion.cpp
index 89ec288e6af..09ad03125da 100644
--- a/common/components/precision_conversion.cpp
+++ b/common/unified/components/precision_conversion.cpp
@@ -33,7 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/components/precision_conversion.hpp"
 
 
-#include "common/base/kernel_launch.hpp"
+#include "common/unified/base/kernel_launch.hpp"
 
 
 namespace gko {
diff --git a/common/matrix/coo_kernels.cpp b/common/unified/matrix/coo_kernels.cpp
similarity index 98%
rename from common/matrix/coo_kernels.cpp
rename to common/unified/matrix/coo_kernels.cpp
index ecb5c397743..095bbdf0e65 100644
--- a/common/matrix/coo_kernels.cpp
+++ b/common/unified/matrix/coo_kernels.cpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "common/base/kernel_launch.hpp"
+#include "common/unified/base/kernel_launch.hpp"
 
 
 namespace gko {
diff --git a/common/matrix/csr_kernels.cpp b/common/unified/matrix/csr_kernels.cpp
similarity index 98%
rename from common/matrix/csr_kernels.cpp
rename to common/unified/matrix/csr_kernels.cpp
index 619a06d8dab..5134e0b88ee 100644
--- a/common/matrix/csr_kernels.cpp
+++ b/common/unified/matrix/csr_kernels.cpp
@@ -39,7 +39,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "common/base/kernel_launch.hpp"
+#include "common/unified/base/kernel_launch.hpp"
 
 
 namespace gko {
diff --git a/common/matrix/dense_kernels.cpp b/common/unified/matrix/dense_kernels.cpp
similarity index 99%
rename from common/matrix/dense_kernels.cpp
rename to common/unified/matrix/dense_kernels.cpp
index 8c2fd00a8c2..034e465014a 100644
--- a/common/matrix/dense_kernels.cpp
+++ b/common/unified/matrix/dense_kernels.cpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "common/base/kernel_launch.hpp"
+#include "common/unified/base/kernel_launch.hpp"
 
 
 namespace gko {
diff --git a/common/matrix/diagonal_kernels.cpp b/common/unified/matrix/diagonal_kernels.cpp
similarity index 99%
rename from common/matrix/diagonal_kernels.cpp
rename to common/unified/matrix/diagonal_kernels.cpp
index 31b6b2c11c2..e39bd40e207 100644
--- a/common/matrix/diagonal_kernels.cpp
+++ b/common/unified/matrix/diagonal_kernels.cpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "common/base/kernel_launch.hpp"
+#include "common/unified/base/kernel_launch.hpp"
 
 
 namespace gko {
diff --git a/common/preconditioner/jacobi_kernels.cpp b/common/unified/preconditioner/jacobi_kernels.cpp
similarity index 99%
rename from common/preconditioner/jacobi_kernels.cpp
rename to common/unified/preconditioner/jacobi_kernels.cpp
index 442a1b47809..b7160a97617 100644
--- a/common/preconditioner/jacobi_kernels.cpp
+++ b/common/unified/preconditioner/jacobi_kernels.cpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "common/base/kernel_launch.hpp"
+#include "common/unified/base/kernel_launch.hpp"
 
 
 namespace gko {
diff --git a/common/solver/bicg_kernels.cpp b/common/unified/solver/bicg_kernels.cpp
similarity index 99%
rename from common/solver/bicg_kernels.cpp
rename to common/unified/solver/bicg_kernels.cpp
index 11c3feda57b..3f646b93819 100644
--- a/common/solver/bicg_kernels.cpp
+++ b/common/unified/solver/bicg_kernels.cpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "common/base/kernel_launch_solver.hpp"
+#include "common/unified/base/kernel_launch_solver.hpp"
 
 
 namespace gko {
diff --git a/common/solver/bicgstab_kernels.cpp b/common/unified/solver/bicgstab_kernels.cpp
similarity index 99%
rename from common/solver/bicgstab_kernels.cpp
rename to common/unified/solver/bicgstab_kernels.cpp
index c9263e018dc..a1d98b139e5 100644
--- a/common/solver/bicgstab_kernels.cpp
+++ b/common/unified/solver/bicgstab_kernels.cpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "common/base/kernel_launch_solver.hpp"
+#include "common/unified/base/kernel_launch_solver.hpp"
 
 
 namespace gko {
diff --git a/common/solver/cg_kernels.cpp b/common/unified/solver/cg_kernels.cpp
similarity index 98%
rename from common/solver/cg_kernels.cpp
rename to common/unified/solver/cg_kernels.cpp
index f62a444ab1a..f47390aa3df 100644
--- a/common/solver/cg_kernels.cpp
+++ b/common/unified/solver/cg_kernels.cpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "common/base/kernel_launch_solver.hpp"
+#include "common/unified/base/kernel_launch_solver.hpp"
 
 
 namespace gko {
diff --git a/common/solver/cgs_kernels.cpp b/common/unified/solver/cgs_kernels.cpp
similarity index 99%
rename from common/solver/cgs_kernels.cpp
rename to common/unified/solver/cgs_kernels.cpp
index b46731aeadf..948c52864f2 100644
--- a/common/solver/cgs_kernels.cpp
+++ b/common/unified/solver/cgs_kernels.cpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "common/base/kernel_launch_solver.hpp"
+#include "common/unified/base/kernel_launch_solver.hpp"
 
 
 namespace gko {
diff --git a/common/solver/fcg_kernels.cpp b/common/unified/solver/fcg_kernels.cpp
similarity index 98%
rename from common/solver/fcg_kernels.cpp
rename to common/unified/solver/fcg_kernels.cpp
index 66b26d946f0..ba9587e350d 100644
--- a/common/solver/fcg_kernels.cpp
+++ b/common/unified/solver/fcg_kernels.cpp
@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/math.hpp>
 
 
-#include "common/base/kernel_launch_solver.hpp"
+#include "common/unified/base/kernel_launch_solver.hpp"
 
 
 namespace gko {
diff --git a/common/solver/ir_kernels.cpp b/common/unified/solver/ir_kernels.cpp
similarity index 97%
rename from common/solver/ir_kernels.cpp
rename to common/unified/solver/ir_kernels.cpp
index f30fb066588..3ba7c957ee3 100644
--- a/common/solver/ir_kernels.cpp
+++ b/common/unified/solver/ir_kernels.cpp
@@ -33,7 +33,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "core/solver/ir_kernels.hpp"
 
 
-#include "common/base/kernel_launch.hpp"
+#include "common/unified/base/kernel_launch.hpp"
 
 
 namespace gko {
diff --git a/cuda/CMakeLists.txt b/cuda/CMakeLists.txt
index 602dea8ee24..7fa32610a40 100644
--- a/cuda/CMakeLists.txt
+++ b/cuda/CMakeLists.txt
@@ -66,18 +66,18 @@ find_library(CURAND curand
 
 add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_cuda_device> "")
 set(GKO_CUDA_COMMON_SOURCES
-    ../common/components/precision_conversion.cpp
-    ../common/matrix/coo_kernels.cpp
-    ../common/matrix/csr_kernels.cpp
-    ../common/matrix/dense_kernels.cpp
-    ../common/matrix/diagonal_kernels.cpp
-    ../common/preconditioner/jacobi_kernels.cpp
-    ../common/solver/bicg_kernels.cpp
-    ../common/solver/bicgstab_kernels.cpp
-    ../common/solver/cg_kernels.cpp
-    ../common/solver/cgs_kernels.cpp
-    ../common/solver/fcg_kernels.cpp
-    ../common/solver/ir_kernels.cpp
+    ../common/unified/components/precision_conversion.cpp
+    ../common/unified/matrix/coo_kernels.cpp
+    ../common/unified/matrix/csr_kernels.cpp
+    ../common/unified/matrix/dense_kernels.cpp
+    ../common/unified/matrix/diagonal_kernels.cpp
+    ../common/unified/preconditioner/jacobi_kernels.cpp
+    ../common/unified/solver/bicg_kernels.cpp
+    ../common/unified/solver/bicgstab_kernels.cpp
+    ../common/unified/solver/cg_kernels.cpp
+    ../common/unified/solver/cgs_kernels.cpp
+    ../common/unified/solver/fcg_kernels.cpp
+    ../common/unified/solver/ir_kernels.cpp
     )
 target_sources(ginkgo_cuda
     PRIVATE
diff --git a/cuda/base/executor.cpp b/cuda/base/executor.cpp
index 1c35af4d390..b256cb93a2c 100644
--- a/cuda/base/executor.cpp
+++ b/cuda/base/executor.cpp
@@ -53,7 +53,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace gko {
 
 
-#include "common/base/executor.hpp.inc"
+#include "common/cuda_hip/base/executor.hpp.inc"
 
 
 std::shared_ptr<CudaExecutor> CudaExecutor::create(
diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index 905dbf5d0b7..23d92be6dd3 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -31,7 +31,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
-#error "This file can only be used from inside common/base/kernel_launch.hpp"
+#error "This file can only be used from inside common/unified/base/kernel_launch.hpp"
 #endif
 
 
diff --git a/cuda/base/kernel_launch_solver.cuh b/cuda/base/kernel_launch_solver.cuh
index 2424e6a4fa2..ab67ed31db6 100644
--- a/cuda/base/kernel_launch_solver.cuh
+++ b/cuda/base/kernel_launch_solver.cuh
@@ -32,7 +32,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
 #error \
-    "This file can only be used from inside common/base/kernel_launch_solver.hpp"
+    "This file can only be used from inside common/unified/base/kernel_launch_solver.hpp"
 #endif
 
 
diff --git a/cuda/base/math.hpp b/cuda/base/math.hpp
index 1a894d3bdc2..e56904ceb45 100644
--- a/cuda/base/math.hpp
+++ b/cuda/base/math.hpp
@@ -43,7 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace gko {
 
 
-#include "common/base/math.hpp.inc"
+#include "common/cuda_hip/base/math.hpp.inc"
 
 
 }  // namespace gko
diff --git a/cuda/components/absolute_array.cu b/cuda/components/absolute_array.cu
index 09ca759cb1d..2eb62b229a7 100644
--- a/cuda/components/absolute_array.cu
+++ b/cuda/components/absolute_array.cu
@@ -46,7 +46,7 @@ namespace components {
 constexpr int default_block_size = 512;
 
 
-#include "common/components/absolute_array.hpp.inc"
+#include "common/cuda_hip/components/absolute_array.hpp.inc"
 
 
 template <typename ValueType>
diff --git a/cuda/components/atomic.cuh b/cuda/components/atomic.cuh
index 56f757f0dba..243a3ab507f 100644
--- a/cuda/components/atomic.cuh
+++ b/cuda/components/atomic.cuh
@@ -46,7 +46,7 @@ namespace kernels {
 namespace cuda {
 
 
-#include "common/components/atomic.hpp.inc"
+#include "common/cuda_hip/components/atomic.hpp.inc"
 
 
 /**
diff --git a/cuda/components/diagonal_block_manipulation.cuh b/cuda/components/diagonal_block_manipulation.cuh
index c4ee79be397..1dfcc0d3312 100644
--- a/cuda/components/diagonal_block_manipulation.cuh
+++ b/cuda/components/diagonal_block_manipulation.cuh
@@ -48,7 +48,7 @@ namespace cuda {
 namespace csr {
 
 
-#include "common/components/diagonal_block_manipulation.hpp.inc"
+#include "common/cuda_hip/components/diagonal_block_manipulation.hpp.inc"
 
 
 }  // namespace csr
diff --git a/cuda/components/fill_array.cu b/cuda/components/fill_array.cu
index 6098399064f..fd5b69b54bc 100644
--- a/cuda/components/fill_array.cu
+++ b/cuda/components/fill_array.cu
@@ -46,7 +46,7 @@ namespace components {
 constexpr int default_block_size = 512;
 
 
-#include "common/components/fill_array.hpp.inc"
+#include "common/cuda_hip/components/fill_array.hpp.inc"
 
 
 template <typename ValueType>
diff --git a/cuda/components/intrinsics.cuh b/cuda/components/intrinsics.cuh
index b2bd0643271..e8ac3323550 100644
--- a/cuda/components/intrinsics.cuh
+++ b/cuda/components/intrinsics.cuh
@@ -42,7 +42,7 @@ namespace kernels {
 namespace cuda {
 
 
-#include "common/components/intrinsics.hpp.inc"
+#include "common/cuda_hip/components/intrinsics.hpp.inc"
 
 
 }  // namespace cuda
diff --git a/cuda/components/merging.cuh b/cuda/components/merging.cuh
index a4cbd3e87e4..1a6c1c7d965 100644
--- a/cuda/components/merging.cuh
+++ b/cuda/components/merging.cuh
@@ -45,7 +45,7 @@ namespace kernels {
 namespace cuda {
 
 
-#include "common/components/merging.hpp.inc"
+#include "common/cuda_hip/components/merging.hpp.inc"
 
 
 }  // namespace cuda
diff --git a/cuda/components/prefix_sum.cuh b/cuda/components/prefix_sum.cuh
index 7f7827c79dc..d761d459a90 100644
--- a/cuda/components/prefix_sum.cuh
+++ b/cuda/components/prefix_sum.cuh
@@ -48,7 +48,7 @@ namespace kernels {
 namespace cuda {
 
 
-#include "common/components/prefix_sum.hpp.inc"
+#include "common/cuda_hip/components/prefix_sum.hpp.inc"
 
 
 }  // namespace cuda
diff --git a/cuda/components/reduction.cuh b/cuda/components/reduction.cuh
index 49394cf60ba..9aace4fe6bd 100644
--- a/cuda/components/reduction.cuh
+++ b/cuda/components/reduction.cuh
@@ -56,7 +56,7 @@ namespace cuda {
 constexpr int default_block_size = 512;
 
 
-#include "common/components/reduction.hpp.inc"
+#include "common/cuda_hip/components/reduction.hpp.inc"
 
 
 /**
diff --git a/cuda/components/searching.cuh b/cuda/components/searching.cuh
index d6c5fbf6140..2f7812075b2 100644
--- a/cuda/components/searching.cuh
+++ b/cuda/components/searching.cuh
@@ -43,7 +43,7 @@ namespace kernels {
 namespace cuda {
 
 
-#include "common/components/searching.hpp.inc"
+#include "common/cuda_hip/components/searching.hpp.inc"
 
 
 }  // namespace cuda
diff --git a/cuda/components/segment_scan.cuh b/cuda/components/segment_scan.cuh
index 1aa8c5b56a6..a6ea7955d55 100644
--- a/cuda/components/segment_scan.cuh
+++ b/cuda/components/segment_scan.cuh
@@ -42,7 +42,7 @@ namespace kernels {
 namespace cuda {
 
 
-#include "common/components/segment_scan.hpp.inc"
+#include "common/cuda_hip/components/segment_scan.hpp.inc"
 
 
 }  // namespace cuda
diff --git a/cuda/components/sorting.cuh b/cuda/components/sorting.cuh
index da9cfac8c94..c3f274fc5af 100644
--- a/cuda/components/sorting.cuh
+++ b/cuda/components/sorting.cuh
@@ -43,7 +43,7 @@ namespace kernels {
 namespace cuda {
 
 
-#include "common/components/sorting.hpp.inc"
+#include "common/cuda_hip/components/sorting.hpp.inc"
 
 
 }  // namespace cuda
diff --git a/cuda/components/thread_ids.cuh b/cuda/components/thread_ids.cuh
index 3133e4cbe59..3a5b93c8ee4 100644
--- a/cuda/components/thread_ids.cuh
+++ b/cuda/components/thread_ids.cuh
@@ -48,7 +48,7 @@ namespace cuda {
 namespace thread {
 
 
-#include "common/components/thread_ids.hpp.inc"
+#include "common/cuda_hip/components/thread_ids.hpp.inc"
 
 
 }  // namespace thread
diff --git a/cuda/components/uninitialized_array.hpp b/cuda/components/uninitialized_array.hpp
index 84bd021b85b..0300a5e6556 100644
--- a/cuda/components/uninitialized_array.hpp
+++ b/cuda/components/uninitialized_array.hpp
@@ -42,7 +42,7 @@ namespace kernels {
 namespace cuda {
 
 
-#include "common/components/uninitialized_array.hpp.inc"
+#include "common/cuda_hip/components/uninitialized_array.hpp.inc"
 
 
 }  // namespace cuda
diff --git a/cuda/components/warp_blas.cuh b/cuda/components/warp_blas.cuh
index f3e1287b850..cf8e0e4c951 100644
--- a/cuda/components/warp_blas.cuh
+++ b/cuda/components/warp_blas.cuh
@@ -50,7 +50,7 @@ namespace kernels {
 namespace cuda {
 
 
-#include "common/components/warp_blas.hpp.inc"
+#include "common/cuda_hip/components/warp_blas.hpp.inc"
 
 
 }  // namespace cuda
diff --git a/cuda/factorization/factorization_kernels.cu b/cuda/factorization/factorization_kernels.cu
index d98aeaf544f..065d59f892a 100644
--- a/cuda/factorization/factorization_kernels.cu
+++ b/cuda/factorization/factorization_kernels.cu
@@ -60,7 +60,7 @@ namespace factorization {
 constexpr int default_block_size{512};
 
 
-#include "common/factorization/factorization_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/factorization_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/cuda/factorization/par_ic_kernels.cu b/cuda/factorization/par_ic_kernels.cu
index 51e624fc2dc..957eca09932 100644
--- a/cuda/factorization/par_ic_kernels.cu
+++ b/cuda/factorization/par_ic_kernels.cu
@@ -62,7 +62,7 @@ using compiled_kernels =
     syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
 
 
-#include "common/factorization/par_ic_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ic_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/cuda/factorization/par_ict_kernels.cu b/cuda/factorization/par_ict_kernels.cu
index f7354147cb7..fe44da01632 100644
--- a/cuda/factorization/par_ict_kernels.cu
+++ b/cuda/factorization/par_ict_kernels.cu
@@ -73,8 +73,8 @@ using compiled_kernels =
     syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
 
 
-#include "common/factorization/par_ict_spgeam_kernels.hpp.inc"
-#include "common/factorization/par_ict_sweep_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ict_spgeam_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ict_sweep_kernels.hpp.inc"
 
 
 namespace {
diff --git a/cuda/factorization/par_ilu_kernels.cu b/cuda/factorization/par_ilu_kernels.cu
index 9d26f587a7d..80a0125bae2 100644
--- a/cuda/factorization/par_ilu_kernels.cu
+++ b/cuda/factorization/par_ilu_kernels.cu
@@ -55,7 +55,7 @@ namespace par_ilu_factorization {
 constexpr int default_block_size{512};
 
 
-#include "common/factorization/par_ilu_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilu_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/cuda/factorization/par_ilut_approx_filter_kernel.cu b/cuda/factorization/par_ilut_approx_filter_kernel.cu
index 1e909c6d584..1a286c98447 100644
--- a/cuda/factorization/par_ilut_approx_filter_kernel.cu
+++ b/cuda/factorization/par_ilut_approx_filter_kernel.cu
@@ -76,8 +76,8 @@ using compiled_kernels =
     syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
 
 
-#include "common/factorization/par_ilut_filter_kernels.hpp.inc"
-#include "common/factorization/par_ilut_select_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilut_filter_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilut_select_kernels.hpp.inc"
 
 
 template <int subwarp_size, typename ValueType, typename IndexType>
diff --git a/cuda/factorization/par_ilut_filter_kernel.cu b/cuda/factorization/par_ilut_filter_kernel.cu
index a3c9d33fece..db11cb6feb9 100644
--- a/cuda/factorization/par_ilut_filter_kernel.cu
+++ b/cuda/factorization/par_ilut_filter_kernel.cu
@@ -72,7 +72,7 @@ using compiled_kernels =
     syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
 
 
-#include "common/factorization/par_ilut_filter_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilut_filter_kernels.hpp.inc"
 
 
 namespace {
diff --git a/cuda/factorization/par_ilut_select_common.cu b/cuda/factorization/par_ilut_select_common.cu
index 1373015325d..1d2a94883c4 100644
--- a/cuda/factorization/par_ilut_select_common.cu
+++ b/cuda/factorization/par_ilut_select_common.cu
@@ -55,7 +55,7 @@ namespace cuda {
 namespace par_ilut_factorization {
 
 
-#include "common/factorization/par_ilut_select_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilut_select_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/cuda/factorization/par_ilut_select_kernel.cu b/cuda/factorization/par_ilut_select_kernel.cu
index a2278a6c3ab..a471e55958a 100644
--- a/cuda/factorization/par_ilut_select_kernel.cu
+++ b/cuda/factorization/par_ilut_select_kernel.cu
@@ -63,7 +63,7 @@ namespace cuda {
 namespace par_ilut_factorization {
 
 
-#include "common/factorization/par_ilut_select_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilut_select_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/cuda/factorization/par_ilut_spgeam_kernel.cu b/cuda/factorization/par_ilut_spgeam_kernel.cu
index 7a3c9e5d575..074acc92084 100644
--- a/cuda/factorization/par_ilut_spgeam_kernel.cu
+++ b/cuda/factorization/par_ilut_spgeam_kernel.cu
@@ -73,7 +73,7 @@ using compiled_kernels =
     syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
 
 
-#include "common/factorization/par_ilut_spgeam_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilut_spgeam_kernels.hpp.inc"
 
 
 namespace {
diff --git a/cuda/factorization/par_ilut_sweep_kernel.cu b/cuda/factorization/par_ilut_sweep_kernel.cu
index d50b1a8caad..f5674942059 100644
--- a/cuda/factorization/par_ilut_sweep_kernel.cu
+++ b/cuda/factorization/par_ilut_sweep_kernel.cu
@@ -73,7 +73,7 @@ using compiled_kernels =
     syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
 
 
-#include "common/factorization/par_ilut_sweep_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilut_sweep_kernels.hpp.inc"
 
 
 namespace {
diff --git a/cuda/matrix/coo_kernels.cu b/cuda/matrix/coo_kernels.cu
index db637ea0e24..f41a3551296 100644
--- a/cuda/matrix/coo_kernels.cu
+++ b/cuda/matrix/coo_kernels.cu
@@ -73,7 +73,7 @@ constexpr int warps_in_block = 4;
 constexpr int spmv_block_size = warps_in_block * config::warp_size;
 
 
-#include "common/matrix/coo_kernels.hpp.inc"
+#include "common/cuda_hip/matrix/coo_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/cuda/matrix/csr_kernels.cu b/cuda/matrix/csr_kernels.cu
index d59914993ab..c6b41ebd03b 100644
--- a/cuda/matrix/csr_kernels.cu
+++ b/cuda/matrix/csr_kernels.cu
@@ -96,7 +96,7 @@ using spgeam_kernels =
     syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
 
 
-#include "common/matrix/csr_kernels.hpp.inc"
+#include "common/cuda_hip/matrix/csr_kernels.hpp.inc"
 
 
 namespace host_kernel {
diff --git a/cuda/matrix/dense_kernels.cu b/cuda/matrix/dense_kernels.cu
index 607bfedc4da..477bdb89e54 100644
--- a/cuda/matrix/dense_kernels.cu
+++ b/cuda/matrix/dense_kernels.cu
@@ -67,7 +67,7 @@ namespace dense {
 constexpr int default_block_size = 512;
 
 
-#include "common/matrix/dense_kernels.hpp.inc"
+#include "common/cuda_hip/matrix/dense_kernels.hpp.inc"
 
 
 template <typename ValueType>
diff --git a/cuda/matrix/diagonal_kernels.cu b/cuda/matrix/diagonal_kernels.cu
index ea5835a6a92..bea495fdc2f 100644
--- a/cuda/matrix/diagonal_kernels.cu
+++ b/cuda/matrix/diagonal_kernels.cu
@@ -57,7 +57,7 @@ namespace diagonal {
 constexpr int default_block_size = 512;
 
 
-#include "common/matrix/diagonal_kernels.hpp.inc"
+#include "common/cuda_hip/matrix/diagonal_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/cuda/matrix/ell_kernels.cu b/cuda/matrix/ell_kernels.cu
index 953ce4be8d7..fe0cb2a2c44 100644
--- a/cuda/matrix/ell_kernels.cu
+++ b/cuda/matrix/ell_kernels.cu
@@ -104,7 +104,7 @@ constexpr int max_thread_per_worker = 32;
 using compiled_kernels = syn::value_list<int, 0, 1, 2, 4, 8, 16, 32>;
 
 
-#include "common/matrix/ell_kernels.hpp.inc"
+#include "common/cuda_hip/matrix/ell_kernels.hpp.inc"
 
 
 namespace {
diff --git a/cuda/matrix/hybrid_kernels.cu b/cuda/matrix/hybrid_kernels.cu
index 153146aa62d..b699aa52e32 100644
--- a/cuda/matrix/hybrid_kernels.cu
+++ b/cuda/matrix/hybrid_kernels.cu
@@ -66,7 +66,7 @@ constexpr int default_block_size = 512;
 constexpr int warps_in_block = 4;
 
 
-#include "common/matrix/hybrid_kernels.hpp.inc"
+#include "common/cuda_hip/matrix/hybrid_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/cuda/matrix/sellp_kernels.cu b/cuda/matrix/sellp_kernels.cu
index 6ec2902a618..a53ae60be82 100644
--- a/cuda/matrix/sellp_kernels.cu
+++ b/cuda/matrix/sellp_kernels.cu
@@ -62,7 +62,7 @@ namespace sellp {
 constexpr int default_block_size = 512;
 
 
-#include "common/matrix/sellp_kernels.hpp.inc"
+#include "common/cuda_hip/matrix/sellp_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/cuda/multigrid/amgx_pgm_kernels.cu b/cuda/multigrid/amgx_pgm_kernels.cu
index 15d9940b22e..8c811a638a5 100644
--- a/cuda/multigrid/amgx_pgm_kernels.cu
+++ b/cuda/multigrid/amgx_pgm_kernels.cu
@@ -72,7 +72,7 @@ namespace amgx_pgm {
 constexpr int default_block_size = 512;
 
 
-#include "common/multigrid/amgx_pgm_kernels.hpp.inc"
+#include "common/cuda_hip/multigrid/amgx_pgm_kernels.hpp.inc"
 
 
 template <typename IndexType>
diff --git a/cuda/preconditioner/isai_kernels.cu b/cuda/preconditioner/isai_kernels.cu
index 527dd79f1d4..87732ea6dad 100644
--- a/cuda/preconditioner/isai_kernels.cu
+++ b/cuda/preconditioner/isai_kernels.cu
@@ -66,9 +66,9 @@ constexpr int subwarps_per_block{2};
 constexpr int default_block_size{subwarps_per_block * subwarp_size};
 
 
-#include "common/components/atomic.hpp.inc"
-#include "common/components/warp_blas.hpp.inc"
-#include "common/preconditioner/isai_kernels.hpp.inc"
+#include "common/cuda_hip/components/atomic.hpp.inc"
+#include "common/cuda_hip/components/warp_blas.hpp.inc"
+#include "common/cuda_hip/preconditioner/isai_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/cuda/preconditioner/jacobi_advanced_apply_kernel.cu b/cuda/preconditioner/jacobi_advanced_apply_kernel.cu
index 5bb602e7b95..7ed975868ef 100644
--- a/cuda/preconditioner/jacobi_advanced_apply_kernel.cu
+++ b/cuda/preconditioner/jacobi_advanced_apply_kernel.cu
@@ -60,7 +60,7 @@ namespace cuda {
 namespace jacobi {
 
 
-#include "common/preconditioner/jacobi_advanced_apply_kernel.hpp.inc"
+#include "common/cuda_hip/preconditioner/jacobi_advanced_apply_kernel.hpp.inc"
 
 
 namespace {
diff --git a/cuda/preconditioner/jacobi_generate_kernel.cu b/cuda/preconditioner/jacobi_generate_kernel.cu
index 7b7113bef4d..b897bdc90d9 100644
--- a/cuda/preconditioner/jacobi_generate_kernel.cu
+++ b/cuda/preconditioner/jacobi_generate_kernel.cu
@@ -63,7 +63,7 @@ namespace cuda {
 namespace jacobi {
 
 
-#include "common/preconditioner/jacobi_generate_kernel.hpp.inc"
+#include "common/cuda_hip/preconditioner/jacobi_generate_kernel.hpp.inc"
 
 
 namespace {
diff --git a/cuda/preconditioner/jacobi_kernels.cu b/cuda/preconditioner/jacobi_kernels.cu
index 1ff609791c7..66b8b7a13eb 100644
--- a/cuda/preconditioner/jacobi_kernels.cu
+++ b/cuda/preconditioner/jacobi_kernels.cu
@@ -68,7 +68,7 @@ constexpr int default_grid_size = 32 * 32 * 128;
 constexpr int default_block_size = 512;
 
 
-#include "common/preconditioner/jacobi_kernels.hpp.inc"
+#include "common/cuda_hip/preconditioner/jacobi_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/cuda/preconditioner/jacobi_simple_apply_kernel.cu b/cuda/preconditioner/jacobi_simple_apply_kernel.cu
index d55ab89070e..216af08cca7 100644
--- a/cuda/preconditioner/jacobi_simple_apply_kernel.cu
+++ b/cuda/preconditioner/jacobi_simple_apply_kernel.cu
@@ -60,7 +60,7 @@ namespace cuda {
 namespace jacobi {
 
 
-#include "common/preconditioner/jacobi_simple_apply_kernel.hpp.inc"
+#include "common/cuda_hip/preconditioner/jacobi_simple_apply_kernel.hpp.inc"
 
 
 namespace {
diff --git a/cuda/solver/cb_gmres_kernels.cu b/cuda/solver/cb_gmres_kernels.cu
index 7c8b138a1a6..f0c4cb61aa6 100644
--- a/cuda/solver/cb_gmres_kernels.cu
+++ b/cuda/solver/cb_gmres_kernels.cu
@@ -74,7 +74,7 @@ constexpr int default_dot_dim = 32;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
-#include "common/solver/cb_gmres_kernels.hpp.inc"
+#include "common/cuda_hip/solver/cb_gmres_kernels.hpp.inc"
 
 
 // Specialization, so the Accessor can use the same function as regular pointers
diff --git a/cuda/solver/gmres_kernels.cu b/cuda/solver/gmres_kernels.cu
index bacabd0e6c5..a401e8fd4d0 100644
--- a/cuda/solver/gmres_kernels.cu
+++ b/cuda/solver/gmres_kernels.cu
@@ -73,7 +73,7 @@ constexpr int default_dot_dim = 32;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
-#include "common/solver/gmres_kernels.hpp.inc"
+#include "common/cuda_hip/solver/gmres_kernels.hpp.inc"
 
 
 template <typename ValueType>
diff --git a/cuda/solver/idr_kernels.cu b/cuda/solver/idr_kernels.cu
index 13dd40cd762..e05ddfd5317 100644
--- a/cuda/solver/idr_kernels.cu
+++ b/cuda/solver/idr_kernels.cu
@@ -69,7 +69,7 @@ constexpr int default_dot_dim = 32;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
-#include "common/solver/idr_kernels.hpp.inc"
+#include "common/cuda_hip/solver/idr_kernels.hpp.inc"
 
 
 namespace {
diff --git a/cuda/test/base/cuda_executor.cu b/cuda/test/base/cuda_executor.cu
index c53f9497f62..032054a1ca8 100644
--- a/cuda/test/base/cuda_executor.cu
+++ b/cuda/test/base/cuda_executor.cu
@@ -43,7 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 
-#include "common/base/executor.hpp.inc"
+#include "common/cuda_hip/base/executor.hpp.inc"
 #include "cuda/test/utils.hpp"
 
 
diff --git a/cuda/test/base/kernel_launch.cu b/cuda/test/base/kernel_launch.cu
index 18244836674..d983085d974 100644
--- a/cuda/test/base/kernel_launch.cu
+++ b/cuda/test/base/kernel_launch.cu
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "common/base/kernel_launch.hpp"
+#include "common/unified/base/kernel_launch.hpp"
 
 
 #include <memory>
@@ -46,7 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
-#include "common/base/kernel_launch_solver.hpp"
+#include "common/unified/base/kernel_launch_solver.hpp"
 #include "core/test/utils.hpp"
 
 
diff --git a/dev_tools/scripts/config b/dev_tools/scripts/config
index 37656df0872..bb1d9d1810f 100644
--- a/dev_tools/scripts/config
+++ b/dev_tools/scripts/config
@@ -23,11 +23,11 @@
 - "(cuda|hip|dpcpp)/preconditioner/jacobi_"
     - FixInclude: "core/preconditioner/jacobi_kernels.hpp"
 - "(cuda|hip|dpcpp|omp)/base/kernel_launch\."
-    - FixInclude: "common/base/kernel_launch.hpp"
+    - FixInclude: "common/unified/base/kernel_launch.hpp"
 - "(cuda|hip|dpcpp|omp)/test/base/kernel_launch\."
-    - FixInclude: "common/base/kernel_launch.hpp"
+    - FixInclude: "common/unified/base/kernel_launch.hpp"
 - "(cuda|hip|dpcpp|omp)/base/kernel_launch_solver\."
-    - FixInclude: "common/base/kernel_launch_solver.hpp"
+    - FixInclude: "common/unified/base/kernel_launch_solver.hpp"
 - "core/test/base/(extended_float|iterator_factory)"
     - RemoveTest: "true"
 - "core/test/base/allocator"
diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt
index 92ac330364b..5f5aa2a4a87 100644
--- a/dpcpp/CMakeLists.txt
+++ b/dpcpp/CMakeLists.txt
@@ -47,18 +47,18 @@ target_sources(ginkgo_dpcpp
     solver/upper_trs_kernels.dp.cpp
     stop/criterion_kernels.dp.cpp
     stop/residual_norm_kernels.dp.cpp
-    ../common/components/precision_conversion.cpp
-    ../common/matrix/coo_kernels.cpp
-    ../common/matrix/csr_kernels.cpp
-    ../common/matrix/dense_kernels.cpp
-    ../common/matrix/diagonal_kernels.cpp
-    ../common/preconditioner/jacobi_kernels.cpp
-    ../common/solver/bicg_kernels.cpp
-    ../common/solver/bicgstab_kernels.cpp
-    ../common/solver/cg_kernels.cpp
-    ../common/solver/cgs_kernels.cpp
-    ../common/solver/fcg_kernels.cpp
-    ../common/solver/ir_kernels.cpp
+    ../common/unified/components/precision_conversion.cpp
+    ../common/unified/matrix/coo_kernels.cpp
+    ../common/unified/matrix/csr_kernels.cpp
+    ../common/unified/matrix/dense_kernels.cpp
+    ../common/unified/matrix/diagonal_kernels.cpp
+    ../common/unified/preconditioner/jacobi_kernels.cpp
+    ../common/unified/solver/bicg_kernels.cpp
+    ../common/unified/solver/bicgstab_kernels.cpp
+    ../common/unified/solver/cg_kernels.cpp
+    ../common/unified/solver/cgs_kernels.cpp
+    ../common/unified/solver/fcg_kernels.cpp
+    ../common/unified/solver/ir_kernels.cpp
     )
 
 ginkgo_compile_features(ginkgo_dpcpp)
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
index df41eee5e0f..2c779c44375 100644
--- a/dpcpp/base/kernel_launch.dp.hpp
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -31,7 +31,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
-#error "This file can only be used from inside common/base/kernel_launch.hpp"
+#error "This file can only be used from inside common/unified/base/kernel_launch.hpp"
 #endif
 
 
diff --git a/dpcpp/base/kernel_launch_solver.dp.hpp b/dpcpp/base/kernel_launch_solver.dp.hpp
index 81c46b1f503..ac256168d72 100644
--- a/dpcpp/base/kernel_launch_solver.dp.hpp
+++ b/dpcpp/base/kernel_launch_solver.dp.hpp
@@ -32,7 +32,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
 #error \
-    "This file can only be used from inside common/base/kernel_launch_solver.hpp"
+    "This file can only be used from inside common/unified/base/kernel_launch_solver.hpp"
 #endif
 
 
diff --git a/dpcpp/test/base/kernel_launch.dp.cpp b/dpcpp/test/base/kernel_launch.dp.cpp
index c148721fb48..25a36d3a29a 100644
--- a/dpcpp/test/base/kernel_launch.dp.cpp
+++ b/dpcpp/test/base/kernel_launch.dp.cpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "common/base/kernel_launch.hpp"
+#include "common/unified/base/kernel_launch.hpp"
 
 
 #include <memory>
@@ -46,7 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
-#include "common/base/kernel_launch_solver.hpp"
+#include "common/unified/base/kernel_launch_solver.hpp"
 #include "core/test/utils.hpp"
 
 
diff --git a/hip/CMakeLists.txt b/hip/CMakeLists.txt
index 0bae0c057fc..e173605fde2 100644
--- a/hip/CMakeLists.txt
+++ b/hip/CMakeLists.txt
@@ -185,18 +185,18 @@ set(GINKGO_HIP_SOURCES
     solver/upper_trs_kernels.hip.cpp
     stop/criterion_kernels.hip.cpp
     stop/residual_norm_kernels.hip.cpp
-    ../common/components/precision_conversion.cpp
-    ../common/matrix/coo_kernels.cpp
-    ../common/matrix/csr_kernels.cpp
-    ../common/matrix/dense_kernels.cpp
-    ../common/matrix/diagonal_kernels.cpp
-    ../common/preconditioner/jacobi_kernels.cpp
-    ../common/solver/bicg_kernels.cpp
-    ../common/solver/bicgstab_kernels.cpp
-    ../common/solver/cg_kernels.cpp
-    ../common/solver/cgs_kernels.cpp
-    ../common/solver/fcg_kernels.cpp
-    ../common/solver/ir_kernels.cpp
+    ../common/unified/components/precision_conversion.cpp
+    ../common/unified/matrix/coo_kernels.cpp
+    ../common/unified/matrix/csr_kernels.cpp
+    ../common/unified/matrix/dense_kernels.cpp
+    ../common/unified/matrix/diagonal_kernels.cpp
+    ../common/unified/preconditioner/jacobi_kernels.cpp
+    ../common/unified/solver/bicg_kernels.cpp
+    ../common/unified/solver/bicgstab_kernels.cpp
+    ../common/unified/solver/cg_kernels.cpp
+    ../common/unified/solver/cgs_kernels.cpp
+    ../common/unified/solver/fcg_kernels.cpp
+    ../common/unified/solver/ir_kernels.cpp
     )
 
 set(GINKGO_HIP_NVCC_ARCH "")
diff --git a/hip/base/executor.hip.cpp b/hip/base/executor.hip.cpp
index 3875ad95910..fa24c1eb929 100644
--- a/hip/base/executor.hip.cpp
+++ b/hip/base/executor.hip.cpp
@@ -53,7 +53,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace gko {
 
 
-#include "common/base/executor.hpp.inc"
+#include "common/cuda_hip/base/executor.hpp.inc"
 
 
 #if (GINKGO_HIP_PLATFORM_NVCC == 1)
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index 1267fe69695..043f092dd56 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -31,7 +31,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
-#error "This file can only be used from inside common/base/kernel_launch.hpp"
+#error "This file can only be used from inside common/unified/base/kernel_launch.hpp"
 #endif
 
 
diff --git a/hip/base/kernel_launch_solver.hip.hpp b/hip/base/kernel_launch_solver.hip.hpp
index f2e117655bc..e0dffa5112c 100644
--- a/hip/base/kernel_launch_solver.hip.hpp
+++ b/hip/base/kernel_launch_solver.hip.hpp
@@ -32,7 +32,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
 #error \
-    "This file can only be used from inside common/base/kernel_launch_solver.hpp"
+    "This file can only be used from inside common/unified/base/kernel_launch_solver.hpp"
 #endif
 
 
diff --git a/hip/base/math.hip.hpp b/hip/base/math.hip.hpp
index 2dae1f6b67f..2cd504dc43e 100644
--- a/hip/base/math.hip.hpp
+++ b/hip/base/math.hip.hpp
@@ -43,7 +43,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace gko {
 
 
-#include "common/base/math.hpp.inc"
+#include "common/cuda_hip/base/math.hpp.inc"
 
 
 }  // namespace gko
diff --git a/hip/components/absolute_array.hip.cpp b/hip/components/absolute_array.hip.cpp
index 3a4409444ef..f5e4e9637a0 100644
--- a/hip/components/absolute_array.hip.cpp
+++ b/hip/components/absolute_array.hip.cpp
@@ -49,7 +49,7 @@ namespace components {
 constexpr int default_block_size = 512;
 
 
-#include "common/components/absolute_array.hpp.inc"
+#include "common/cuda_hip/components/absolute_array.hpp.inc"
 
 
 template <typename ValueType>
diff --git a/hip/components/atomic.hip.hpp b/hip/components/atomic.hip.hpp
index a3e3e324f79..a6508e87f8c 100644
--- a/hip/components/atomic.hip.hpp
+++ b/hip/components/atomic.hip.hpp
@@ -46,7 +46,7 @@ namespace kernels {
 namespace hip {
 
 
-#include "common/components/atomic.hpp.inc"
+#include "common/cuda_hip/components/atomic.hpp.inc"
 
 
 /**
diff --git a/hip/components/diagonal_block_manipulation.hip.hpp b/hip/components/diagonal_block_manipulation.hip.hpp
index 73f9715a487..8237edcba7d 100644
--- a/hip/components/diagonal_block_manipulation.hip.hpp
+++ b/hip/components/diagonal_block_manipulation.hip.hpp
@@ -48,7 +48,7 @@ namespace hip {
 namespace csr {
 
 
-#include "common/components/diagonal_block_manipulation.hpp.inc"
+#include "common/cuda_hip/components/diagonal_block_manipulation.hpp.inc"
 
 
 }  // namespace csr
diff --git a/hip/components/fill_array.hip.cpp b/hip/components/fill_array.hip.cpp
index 936c4324c86..590a0953b1d 100644
--- a/hip/components/fill_array.hip.cpp
+++ b/hip/components/fill_array.hip.cpp
@@ -51,7 +51,7 @@ namespace components {
 constexpr int default_block_size = 512;
 
 
-#include "common/components/fill_array.hpp.inc"
+#include "common/cuda_hip/components/fill_array.hpp.inc"
 
 
 template <typename ValueType>
diff --git a/hip/components/intrinsics.hip.hpp b/hip/components/intrinsics.hip.hpp
index 4a5f3751a4c..d1d2d13f6bb 100644
--- a/hip/components/intrinsics.hip.hpp
+++ b/hip/components/intrinsics.hip.hpp
@@ -42,7 +42,7 @@ namespace kernels {
 namespace hip {
 
 
-#include "common/components/intrinsics.hpp.inc"
+#include "common/cuda_hip/components/intrinsics.hpp.inc"
 
 
 }  // namespace hip
diff --git a/hip/components/merging.hip.hpp b/hip/components/merging.hip.hpp
index 42e5b32ec71..e3ad7fb1685 100644
--- a/hip/components/merging.hip.hpp
+++ b/hip/components/merging.hip.hpp
@@ -45,7 +45,7 @@ namespace kernels {
 namespace hip {
 
 
-#include "common/components/merging.hpp.inc"
+#include "common/cuda_hip/components/merging.hpp.inc"
 
 
 }  // namespace hip
diff --git a/hip/components/prefix_sum.hip.hpp b/hip/components/prefix_sum.hip.hpp
index 295311bbbb4..fbb87e2fa69 100644
--- a/hip/components/prefix_sum.hip.hpp
+++ b/hip/components/prefix_sum.hip.hpp
@@ -48,7 +48,7 @@ namespace kernels {
 namespace hip {
 
 
-#include "common/components/prefix_sum.hpp.inc"
+#include "common/cuda_hip/components/prefix_sum.hpp.inc"
 
 
 }  // namespace hip
diff --git a/hip/components/reduction.hip.hpp b/hip/components/reduction.hip.hpp
index e5fcb5745a3..f1c9c8d965f 100644
--- a/hip/components/reduction.hip.hpp
+++ b/hip/components/reduction.hip.hpp
@@ -58,7 +58,7 @@ namespace hip {
 constexpr int default_block_size = 512;
 
 
-#include "common/components/reduction.hpp.inc"
+#include "common/cuda_hip/components/reduction.hpp.inc"
 
 
 /**
diff --git a/hip/components/searching.hip.hpp b/hip/components/searching.hip.hpp
index fd26e759794..11c348f7a6f 100644
--- a/hip/components/searching.hip.hpp
+++ b/hip/components/searching.hip.hpp
@@ -43,7 +43,7 @@ namespace kernels {
 namespace hip {
 
 
-#include "common/components/searching.hpp.inc"
+#include "common/cuda_hip/components/searching.hpp.inc"
 
 
 }  // namespace hip
diff --git a/hip/components/segment_scan.hip.hpp b/hip/components/segment_scan.hip.hpp
index 397a78b4556..743ea26d692 100644
--- a/hip/components/segment_scan.hip.hpp
+++ b/hip/components/segment_scan.hip.hpp
@@ -42,7 +42,7 @@ namespace kernels {
 namespace hip {
 
 
-#include "common/components/segment_scan.hpp.inc"
+#include "common/cuda_hip/components/segment_scan.hpp.inc"
 
 
 }  // namespace hip
diff --git a/hip/components/sorting.hip.hpp b/hip/components/sorting.hip.hpp
index 5ebd4c495a1..7c7fc4404da 100644
--- a/hip/components/sorting.hip.hpp
+++ b/hip/components/sorting.hip.hpp
@@ -43,7 +43,7 @@ namespace kernels {
 namespace hip {
 
 
-#include "common/components/sorting.hpp.inc"
+#include "common/cuda_hip/components/sorting.hpp.inc"
 
 
 }  // namespace hip
diff --git a/hip/components/thread_ids.hip.hpp b/hip/components/thread_ids.hip.hpp
index 59eaeb7abbd..79c774e3ec9 100644
--- a/hip/components/thread_ids.hip.hpp
+++ b/hip/components/thread_ids.hip.hpp
@@ -48,7 +48,7 @@ namespace hip {
 namespace thread {
 
 
-#include "common/components/thread_ids.hpp.inc"
+#include "common/cuda_hip/components/thread_ids.hpp.inc"
 
 
 }  // namespace thread
diff --git a/hip/components/uninitialized_array.hip.hpp b/hip/components/uninitialized_array.hip.hpp
index 45d55d7af63..80b5cc74676 100644
--- a/hip/components/uninitialized_array.hip.hpp
+++ b/hip/components/uninitialized_array.hip.hpp
@@ -42,7 +42,7 @@ namespace kernels {
 namespace hip {
 
 
-#include "common/components/uninitialized_array.hpp.inc"
+#include "common/cuda_hip/components/uninitialized_array.hpp.inc"
 
 
 }  // namespace hip
diff --git a/hip/components/warp_blas.hip.hpp b/hip/components/warp_blas.hip.hpp
index 0a341d8a98c..4611e505376 100644
--- a/hip/components/warp_blas.hip.hpp
+++ b/hip/components/warp_blas.hip.hpp
@@ -50,7 +50,7 @@ namespace kernels {
 namespace hip {
 
 
-#include "common/components/warp_blas.hpp.inc"
+#include "common/cuda_hip/components/warp_blas.hpp.inc"
 
 
 }  // namespace hip
diff --git a/hip/factorization/factorization_kernels.hip.cpp b/hip/factorization/factorization_kernels.hip.cpp
index 9e53d54e16d..ec4ab4806ee 100644
--- a/hip/factorization/factorization_kernels.hip.cpp
+++ b/hip/factorization/factorization_kernels.hip.cpp
@@ -62,7 +62,7 @@ namespace factorization {
 constexpr int default_block_size{512};
 
 
-#include "common/factorization/factorization_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/factorization_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/hip/factorization/par_ic_kernels.hip.cpp b/hip/factorization/par_ic_kernels.hip.cpp
index 4e242119f9e..17e556046e2 100644
--- a/hip/factorization/par_ic_kernels.hip.cpp
+++ b/hip/factorization/par_ic_kernels.hip.cpp
@@ -62,7 +62,7 @@ using compiled_kernels =
     syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
 
 
-#include "common/factorization/par_ic_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ic_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/hip/factorization/par_ict_kernels.hip.cpp b/hip/factorization/par_ict_kernels.hip.cpp
index 15c175166cc..a67c29d10a7 100644
--- a/hip/factorization/par_ict_kernels.hip.cpp
+++ b/hip/factorization/par_ict_kernels.hip.cpp
@@ -76,8 +76,8 @@ using compiled_kernels =
     syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
 
 
-#include "common/factorization/par_ict_spgeam_kernels.hpp.inc"
-#include "common/factorization/par_ict_sweep_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ict_spgeam_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ict_sweep_kernels.hpp.inc"
 
 
 namespace {
diff --git a/hip/factorization/par_ilu_kernels.hip.cpp b/hip/factorization/par_ilu_kernels.hip.cpp
index 081ac0fe317..b414874930f 100644
--- a/hip/factorization/par_ilu_kernels.hip.cpp
+++ b/hip/factorization/par_ilu_kernels.hip.cpp
@@ -59,7 +59,7 @@ namespace par_ilu_factorization {
 constexpr int default_block_size{512};
 
 
-#include "common/factorization/par_ilu_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilu_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/hip/factorization/par_ilut_approx_filter_kernel.hip.cpp b/hip/factorization/par_ilut_approx_filter_kernel.hip.cpp
index dde375f41c6..319fdf4210c 100644
--- a/hip/factorization/par_ilut_approx_filter_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_approx_filter_kernel.hip.cpp
@@ -79,8 +79,8 @@ using compiled_kernels =
     syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
 
 
-#include "common/factorization/par_ilut_filter_kernels.hpp.inc"
-#include "common/factorization/par_ilut_select_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilut_filter_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilut_select_kernels.hpp.inc"
 
 
 template <int subwarp_size, typename ValueType, typename IndexType>
diff --git a/hip/factorization/par_ilut_filter_kernel.hip.cpp b/hip/factorization/par_ilut_filter_kernel.hip.cpp
index 4898d6ec149..e8997ebeb40 100644
--- a/hip/factorization/par_ilut_filter_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_filter_kernel.hip.cpp
@@ -75,7 +75,7 @@ using compiled_kernels =
     syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
 
 
-#include "common/factorization/par_ilut_filter_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilut_filter_kernels.hpp.inc"
 
 
 namespace {
diff --git a/hip/factorization/par_ilut_select_common.hip.cpp b/hip/factorization/par_ilut_select_common.hip.cpp
index d9df5ce853c..4e453270d78 100644
--- a/hip/factorization/par_ilut_select_common.hip.cpp
+++ b/hip/factorization/par_ilut_select_common.hip.cpp
@@ -61,7 +61,7 @@ namespace hip {
 namespace par_ilut_factorization {
 
 
-#include "common/factorization/par_ilut_select_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilut_select_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/hip/factorization/par_ilut_select_kernel.hip.cpp b/hip/factorization/par_ilut_select_kernel.hip.cpp
index d75d08063d5..00adc9fcba1 100644
--- a/hip/factorization/par_ilut_select_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_select_kernel.hip.cpp
@@ -66,7 +66,7 @@ namespace hip {
 namespace par_ilut_factorization {
 
 
-#include "common/factorization/par_ilut_select_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilut_select_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/hip/factorization/par_ilut_spgeam_kernel.hip.cpp b/hip/factorization/par_ilut_spgeam_kernel.hip.cpp
index b18b6fface6..d7815de1d2a 100644
--- a/hip/factorization/par_ilut_spgeam_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_spgeam_kernel.hip.cpp
@@ -76,7 +76,7 @@ using compiled_kernels =
     syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
 
 
-#include "common/factorization/par_ilut_spgeam_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilut_spgeam_kernels.hpp.inc"
 
 
 namespace {
diff --git a/hip/factorization/par_ilut_sweep_kernel.hip.cpp b/hip/factorization/par_ilut_sweep_kernel.hip.cpp
index 712dc038937..b00a225a6b4 100644
--- a/hip/factorization/par_ilut_sweep_kernel.hip.cpp
+++ b/hip/factorization/par_ilut_sweep_kernel.hip.cpp
@@ -76,7 +76,7 @@ using compiled_kernels =
     syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
 
 
-#include "common/factorization/par_ilut_sweep_kernels.hpp.inc"
+#include "common/cuda_hip/factorization/par_ilut_sweep_kernels.hpp.inc"
 
 
 namespace {
diff --git a/hip/matrix/coo_kernels.hip.cpp b/hip/matrix/coo_kernels.hip.cpp
index 06b37134a1f..b1318230c7a 100644
--- a/hip/matrix/coo_kernels.hip.cpp
+++ b/hip/matrix/coo_kernels.hip.cpp
@@ -76,7 +76,7 @@ constexpr int warps_in_block = 4;
 constexpr int spmv_block_size = warps_in_block * config::warp_size;
 
 
-#include "common/matrix/coo_kernels.hpp.inc"
+#include "common/cuda_hip/matrix/coo_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/hip/matrix/csr_kernels.hip.cpp b/hip/matrix/csr_kernels.hip.cpp
index 318d9e60857..b2a712e1924 100644
--- a/hip/matrix/csr_kernels.hip.cpp
+++ b/hip/matrix/csr_kernels.hip.cpp
@@ -99,7 +99,7 @@ using spgeam_kernels =
     syn::value_list<int, 1, 2, 4, 8, 16, 32, config::warp_size>;
 
 
-#include "common/matrix/csr_kernels.hpp.inc"
+#include "common/cuda_hip/matrix/csr_kernels.hpp.inc"
 
 
 namespace host_kernel {
diff --git a/hip/matrix/dense_kernels.hip.cpp b/hip/matrix/dense_kernels.hip.cpp
index f437bc6dddc..02ff05b5b3f 100644
--- a/hip/matrix/dense_kernels.hip.cpp
+++ b/hip/matrix/dense_kernels.hip.cpp
@@ -70,7 +70,7 @@ namespace dense {
 constexpr int default_block_size = 512;
 
 
-#include "common/matrix/dense_kernels.hpp.inc"
+#include "common/cuda_hip/matrix/dense_kernels.hpp.inc"
 
 
 template <typename ValueType>
diff --git a/hip/matrix/diagonal_kernels.hip.cpp b/hip/matrix/diagonal_kernels.hip.cpp
index 8718fe34a91..a75d4aa6669 100644
--- a/hip/matrix/diagonal_kernels.hip.cpp
+++ b/hip/matrix/diagonal_kernels.hip.cpp
@@ -60,7 +60,7 @@ namespace diagonal {
 constexpr int default_block_size = 512;
 
 
-#include "common/matrix/diagonal_kernels.hpp.inc"
+#include "common/cuda_hip/matrix/diagonal_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/hip/matrix/ell_kernels.hip.cpp b/hip/matrix/ell_kernels.hip.cpp
index b1ad7bf4662..27ced862d93 100644
--- a/hip/matrix/ell_kernels.hip.cpp
+++ b/hip/matrix/ell_kernels.hip.cpp
@@ -107,7 +107,7 @@ constexpr int max_thread_per_worker = 32;
 using compiled_kernels = syn::value_list<int, 0, 1, 2, 4, 8, 16, 32>;
 
 
-#include "common/matrix/ell_kernels.hpp.inc"
+#include "common/cuda_hip/matrix/ell_kernels.hpp.inc"
 
 
 namespace {
diff --git a/hip/matrix/hybrid_kernels.hip.cpp b/hip/matrix/hybrid_kernels.hip.cpp
index 20c4a540a1b..1d7f46ea743 100644
--- a/hip/matrix/hybrid_kernels.hip.cpp
+++ b/hip/matrix/hybrid_kernels.hip.cpp
@@ -69,7 +69,7 @@ constexpr int default_block_size = 512;
 constexpr int warps_in_block = 4;
 
 
-#include "common/matrix/hybrid_kernels.hpp.inc"
+#include "common/cuda_hip/matrix/hybrid_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/hip/matrix/sellp_kernels.hip.cpp b/hip/matrix/sellp_kernels.hip.cpp
index 3f9b583404f..52f4f38e798 100644
--- a/hip/matrix/sellp_kernels.hip.cpp
+++ b/hip/matrix/sellp_kernels.hip.cpp
@@ -65,7 +65,7 @@ namespace sellp {
 constexpr int default_block_size = 512;
 
 
-#include "common/matrix/sellp_kernels.hpp.inc"
+#include "common/cuda_hip/matrix/sellp_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/hip/multigrid/amgx_pgm_kernels.hip.cpp b/hip/multigrid/amgx_pgm_kernels.hip.cpp
index d7f4685b785..ddc9dd90333 100644
--- a/hip/multigrid/amgx_pgm_kernels.hip.cpp
+++ b/hip/multigrid/amgx_pgm_kernels.hip.cpp
@@ -72,7 +72,7 @@ namespace amgx_pgm {
 constexpr int default_block_size = 512;
 
 
-#include "common/multigrid/amgx_pgm_kernels.hpp.inc"
+#include "common/cuda_hip/multigrid/amgx_pgm_kernels.hpp.inc"
 
 
 template <typename IndexType>
diff --git a/hip/preconditioner/isai_kernels.hip.cpp b/hip/preconditioner/isai_kernels.hip.cpp
index 5b1c9ffa07a..3a96cacfecb 100644
--- a/hip/preconditioner/isai_kernels.hip.cpp
+++ b/hip/preconditioner/isai_kernels.hip.cpp
@@ -69,9 +69,9 @@ constexpr int subwarps_per_block{2};
 constexpr int default_block_size{subwarps_per_block * subwarp_size};
 
 
-#include "common/components/atomic.hpp.inc"
-#include "common/components/warp_blas.hpp.inc"
-#include "common/preconditioner/isai_kernels.hpp.inc"
+#include "common/cuda_hip/components/atomic.hpp.inc"
+#include "common/cuda_hip/components/warp_blas.hpp.inc"
+#include "common/cuda_hip/preconditioner/isai_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/hip/preconditioner/jacobi_advanced_apply_kernel.hip.cpp b/hip/preconditioner/jacobi_advanced_apply_kernel.hip.cpp
index 67d98906ea9..6ac6717447a 100644
--- a/hip/preconditioner/jacobi_advanced_apply_kernel.hip.cpp
+++ b/hip/preconditioner/jacobi_advanced_apply_kernel.hip.cpp
@@ -63,7 +63,7 @@ namespace hip {
 namespace jacobi {
 
 
-#include "common/preconditioner/jacobi_advanced_apply_kernel.hpp.inc"
+#include "common/cuda_hip/preconditioner/jacobi_advanced_apply_kernel.hpp.inc"
 
 
 namespace {
diff --git a/hip/preconditioner/jacobi_generate_kernel.hip.cpp b/hip/preconditioner/jacobi_generate_kernel.hip.cpp
index facf1916de3..6f0e4789f3e 100644
--- a/hip/preconditioner/jacobi_generate_kernel.hip.cpp
+++ b/hip/preconditioner/jacobi_generate_kernel.hip.cpp
@@ -66,7 +66,7 @@ namespace hip {
 namespace jacobi {
 
 
-#include "common/preconditioner/jacobi_generate_kernel.hpp.inc"
+#include "common/cuda_hip/preconditioner/jacobi_generate_kernel.hpp.inc"
 
 
 namespace {
diff --git a/hip/preconditioner/jacobi_kernels.hip.cpp b/hip/preconditioner/jacobi_kernels.hip.cpp
index 425820ead4e..f6f4445326e 100644
--- a/hip/preconditioner/jacobi_kernels.hip.cpp
+++ b/hip/preconditioner/jacobi_kernels.hip.cpp
@@ -74,7 +74,7 @@ constexpr int default_grid_size = 32 * 32 * 128;
 
 constexpr int default_block_size = 512;
 
-#include "common/preconditioner/jacobi_kernels.hpp.inc"
+#include "common/cuda_hip/preconditioner/jacobi_kernels.hpp.inc"
 
 
 template <typename ValueType, typename IndexType>
diff --git a/hip/preconditioner/jacobi_simple_apply_kernel.hip.cpp b/hip/preconditioner/jacobi_simple_apply_kernel.hip.cpp
index 608e33d5f26..0adbedc7473 100644
--- a/hip/preconditioner/jacobi_simple_apply_kernel.hip.cpp
+++ b/hip/preconditioner/jacobi_simple_apply_kernel.hip.cpp
@@ -63,7 +63,7 @@ namespace hip {
 namespace jacobi {
 
 
-#include "common/preconditioner/jacobi_simple_apply_kernel.hpp.inc"
+#include "common/cuda_hip/preconditioner/jacobi_simple_apply_kernel.hpp.inc"
 
 
 namespace {
diff --git a/hip/solver/cb_gmres_kernels.hip.cpp b/hip/solver/cb_gmres_kernels.hip.cpp
index 510009665c0..23bf0546cd6 100644
--- a/hip/solver/cb_gmres_kernels.hip.cpp
+++ b/hip/solver/cb_gmres_kernels.hip.cpp
@@ -76,7 +76,7 @@ constexpr int default_dot_dim = 32;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
-#include "common/solver/cb_gmres_kernels.hpp.inc"
+#include "common/cuda_hip/solver/cb_gmres_kernels.hpp.inc"
 
 
 // Specialization, so the Accessor can use the same function as regular pointers
diff --git a/hip/solver/gmres_kernels.hip.cpp b/hip/solver/gmres_kernels.hip.cpp
index d9d0034b915..38c6477e516 100644
--- a/hip/solver/gmres_kernels.hip.cpp
+++ b/hip/solver/gmres_kernels.hip.cpp
@@ -76,7 +76,7 @@ constexpr int default_dot_dim = 32;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
-#include "common/solver/gmres_kernels.hpp.inc"
+#include "common/cuda_hip/solver/gmres_kernels.hpp.inc"
 
 
 template <typename ValueType>
diff --git a/hip/solver/idr_kernels.hip.cpp b/hip/solver/idr_kernels.hip.cpp
index a467504e95c..1e9cb66aa2b 100644
--- a/hip/solver/idr_kernels.hip.cpp
+++ b/hip/solver/idr_kernels.hip.cpp
@@ -71,7 +71,7 @@ constexpr int default_dot_dim = 32;
 constexpr int default_dot_size = default_dot_dim * default_dot_dim;
 
 
-#include "common/solver/idr_kernels.hpp.inc"
+#include "common/cuda_hip/solver/idr_kernels.hpp.inc"
 
 
 namespace {
diff --git a/hip/test/base/hip_executor.hip.cpp b/hip/test/base/hip_executor.hip.cpp
index 387300526a7..c789004ddb8 100644
--- a/hip/test/base/hip_executor.hip.cpp
+++ b/hip/test/base/hip_executor.hip.cpp
@@ -49,7 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/exception_helpers.hpp>
 
-#include "common/base/executor.hpp.inc"
+#include "common/cuda_hip/base/executor.hpp.inc"
 #include "hip/test/utils.hip.hpp"
 
 
diff --git a/hip/test/base/kernel_launch.hip.cpp b/hip/test/base/kernel_launch.hip.cpp
index 90073a00554..261c552056f 100644
--- a/hip/test/base/kernel_launch.hip.cpp
+++ b/hip/test/base/kernel_launch.hip.cpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "common/base/kernel_launch.hpp"
+#include "common/unified/base/kernel_launch.hpp"
 
 
 #include <memory>
@@ -46,7 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
-#include "common/base/kernel_launch_solver.hpp"
+#include "common/unified/base/kernel_launch_solver.hpp"
 #include "core/test/utils.hpp"
 
 
diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt
index f2bc90284b1..bd5429a6c58 100644
--- a/omp/CMakeLists.txt
+++ b/omp/CMakeLists.txt
@@ -34,18 +34,18 @@ target_sources(ginkgo_omp
     solver/upper_trs_kernels.cpp
     stop/criterion_kernels.cpp
     stop/residual_norm_kernels.cpp
-    ../common/components/precision_conversion.cpp
-    ../common/matrix/coo_kernels.cpp
-    ../common/matrix/csr_kernels.cpp
-    ../common/matrix/dense_kernels.cpp
-    ../common/matrix/diagonal_kernels.cpp
-    ../common/preconditioner/jacobi_kernels.cpp
-    ../common/solver/bicg_kernels.cpp
-    ../common/solver/bicgstab_kernels.cpp
-    ../common/solver/cg_kernels.cpp
-    ../common/solver/cgs_kernels.cpp
-    ../common/solver/fcg_kernels.cpp
-    ../common/solver/ir_kernels.cpp
+    ../common/unified/components/precision_conversion.cpp
+    ../common/unified/matrix/coo_kernels.cpp
+    ../common/unified/matrix/csr_kernels.cpp
+    ../common/unified/matrix/dense_kernels.cpp
+    ../common/unified/matrix/diagonal_kernels.cpp
+    ../common/unified/preconditioner/jacobi_kernels.cpp
+    ../common/unified/solver/bicg_kernels.cpp
+    ../common/unified/solver/bicgstab_kernels.cpp
+    ../common/unified/solver/cg_kernels.cpp
+    ../common/unified/solver/cgs_kernels.cpp
+    ../common/unified/solver/fcg_kernels.cpp
+    ../common/unified/solver/ir_kernels.cpp
     )
 
 ginkgo_compile_features(ginkgo_omp)
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index b8540b382b2..b6b27b0fd81 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -31,7 +31,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
-#error "This file can only be used from inside common/base/kernel_launch.hpp"
+#error "This file can only be used from inside common/unified/base/kernel_launch.hpp"
 #endif
 
 
diff --git a/omp/base/kernel_launch_solver.hpp b/omp/base/kernel_launch_solver.hpp
index 69e20ac4faa..5857ae8e7a4 100644
--- a/omp/base/kernel_launch_solver.hpp
+++ b/omp/base/kernel_launch_solver.hpp
@@ -32,7 +32,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
 #error \
-    "This file can only be used from inside common/base/kernel_launch_solver.hpp"
+    "This file can only be used from inside common/unified/base/kernel_launch_solver.hpp"
 #endif
 
 
diff --git a/omp/test/base/kernel_launch.cpp b/omp/test/base/kernel_launch.cpp
index 6e8c5fc3f9e..cb9ae347dea 100644
--- a/omp/test/base/kernel_launch.cpp
+++ b/omp/test/base/kernel_launch.cpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#include "common/base/kernel_launch.hpp"
+#include "common/unified/base/kernel_launch.hpp"
 
 
 #include <memory>
@@ -46,7 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/matrix/dense.hpp>
 
 
-#include "common/base/kernel_launch_solver.hpp"
+#include "common/unified/base/kernel_launch_solver.hpp"
 #include "core/test/utils.hpp"
 
 

From d717c11ec07895461428cebd18fdb99b226c1524 Mon Sep 17 00:00:00 2001
From: ginkgo-bot <ginkgo.library@gmail.com>
Date: Fri, 6 Aug 2021 11:43:57 +0000
Subject: [PATCH 0999/1008] Format files

Co-authored-by: Terry Cojean <tcojean@users.noreply.github.com>
---
 cuda/base/kernel_launch.cuh     | 3 ++-
 dpcpp/base/kernel_launch.dp.hpp | 3 ++-
 hip/base/kernel_launch.hip.hpp  | 3 ++-
 omp/base/kernel_launch.hpp      | 3 ++-
 4 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index 23d92be6dd3..1e6a69dd57a 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -31,7 +31,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
-#error "This file can only be used from inside common/unified/base/kernel_launch.hpp"
+#error \
+    "This file can only be used from inside common/unified/base/kernel_launch.hpp"
 #endif
 
 
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
index 2c779c44375..a377ee29cfc 100644
--- a/dpcpp/base/kernel_launch.dp.hpp
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -31,7 +31,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
-#error "This file can only be used from inside common/unified/base/kernel_launch.hpp"
+#error \
+    "This file can only be used from inside common/unified/base/kernel_launch.hpp"
 #endif
 
 
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index 043f092dd56..df40dbbb74a 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -31,7 +31,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
-#error "This file can only be used from inside common/unified/base/kernel_launch.hpp"
+#error \
+    "This file can only be used from inside common/unified/base/kernel_launch.hpp"
 #endif
 
 
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index b6b27b0fd81..485a08952ac 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -31,7 +31,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
 #ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
-#error "This file can only be used from inside common/unified/base/kernel_launch.hpp"
+#error \
+    "This file can only be used from inside common/unified/base/kernel_launch.hpp"
 #endif
 
 

From 1f9c5cd7ab12034e82083aeb878d012a30766417 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Fri, 6 Aug 2021 15:18:22 +0200
Subject: [PATCH 1000/1008] Review Updates.

Co-authored-by: Tobias Ribizel <ribizel@kit.edu>
---
 common/unified/base/kernel_launch.hpp        | 6 +++---
 common/unified/base/kernel_launch_solver.hpp | 6 +++---
 cuda/base/kernel_launch.cuh                  | 2 +-
 cuda/base/kernel_launch_solver.cuh           | 2 +-
 dpcpp/base/kernel_launch.dp.hpp              | 2 +-
 dpcpp/base/kernel_launch_solver.dp.hpp       | 2 +-
 hip/base/kernel_launch.hip.hpp               | 2 +-
 hip/base/kernel_launch_solver.hip.hpp        | 2 +-
 omp/base/kernel_launch.hpp                   | 2 +-
 omp/base/kernel_launch_solver.hpp            | 2 +-
 10 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/common/unified/base/kernel_launch.hpp b/common/unified/base/kernel_launch.hpp
index 837f7f9e65a..5e5e1914476 100644
--- a/common/unified/base/kernel_launch.hpp
+++ b/common/unified/base/kernel_launch.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
-#define GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
+#ifndef GKO_COMMON_UNIFIED_BASE_KERNEL_LAUNCH_HPP_
+#define GKO_COMMON_UNIFIED_BASE_KERNEL_LAUNCH_HPP_
 
 
 #include <type_traits>
@@ -280,4 +280,4 @@ typename to_device_type_impl<T>::type map_to_device(T &&param)
 #endif
 
 
-#endif  // GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
+#endif  // GKO_COMMON_UNIFIED_BASE_KERNEL_LAUNCH_HPP_
diff --git a/common/unified/base/kernel_launch_solver.hpp b/common/unified/base/kernel_launch_solver.hpp
index 9dcc404c911..0f859631919 100644
--- a/common/unified/base/kernel_launch_solver.hpp
+++ b/common/unified/base/kernel_launch_solver.hpp
@@ -30,8 +30,8 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
-#define GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
+#ifndef GKO_COMMON_UNIFIED_BASE_KERNEL_LAUNCH_SOLVER_HPP_
+#define GKO_COMMON_UNIFIED_BASE_KERNEL_LAUNCH_SOLVER_HPP_
 
 
 #include "common/unified/base/kernel_launch.hpp"
@@ -146,4 +146,4 @@ const device_type<ValueType> *row_vector(const matrix::Dense<ValueType> *mtx)
 #endif
 
 
-#endif  // GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
+#endif  // GKO_COMMON_UNIFIED_BASE_KERNEL_LAUNCH_SOLVER_HPP_
diff --git a/cuda/base/kernel_launch.cuh b/cuda/base/kernel_launch.cuh
index 1e6a69dd57a..50650057ca6 100644
--- a/cuda/base/kernel_launch.cuh
+++ b/cuda/base/kernel_launch.cuh
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
+#ifndef GKO_COMMON_UNIFIED_BASE_KERNEL_LAUNCH_HPP_
 #error \
     "This file can only be used from inside common/unified/base/kernel_launch.hpp"
 #endif
diff --git a/cuda/base/kernel_launch_solver.cuh b/cuda/base/kernel_launch_solver.cuh
index ab67ed31db6..0dcfe21aaa6 100644
--- a/cuda/base/kernel_launch_solver.cuh
+++ b/cuda/base/kernel_launch_solver.cuh
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
+#ifndef GKO_COMMON_UNIFIED_BASE_KERNEL_LAUNCH_SOLVER_HPP_
 #error \
     "This file can only be used from inside common/unified/base/kernel_launch_solver.hpp"
 #endif
diff --git a/dpcpp/base/kernel_launch.dp.hpp b/dpcpp/base/kernel_launch.dp.hpp
index a377ee29cfc..5e9d505ec52 100644
--- a/dpcpp/base/kernel_launch.dp.hpp
+++ b/dpcpp/base/kernel_launch.dp.hpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
+#ifndef GKO_COMMON_UNIFIED_BASE_KERNEL_LAUNCH_HPP_
 #error \
     "This file can only be used from inside common/unified/base/kernel_launch.hpp"
 #endif
diff --git a/dpcpp/base/kernel_launch_solver.dp.hpp b/dpcpp/base/kernel_launch_solver.dp.hpp
index ac256168d72..ea0a9ea7f89 100644
--- a/dpcpp/base/kernel_launch_solver.dp.hpp
+++ b/dpcpp/base/kernel_launch_solver.dp.hpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
+#ifndef GKO_COMMON_UNIFIED_BASE_KERNEL_LAUNCH_SOLVER_HPP_
 #error \
     "This file can only be used from inside common/unified/base/kernel_launch_solver.hpp"
 #endif
diff --git a/hip/base/kernel_launch.hip.hpp b/hip/base/kernel_launch.hip.hpp
index df40dbbb74a..5e18cb9c645 100644
--- a/hip/base/kernel_launch.hip.hpp
+++ b/hip/base/kernel_launch.hip.hpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
+#ifndef GKO_COMMON_UNIFIED_BASE_KERNEL_LAUNCH_HPP_
 #error \
     "This file can only be used from inside common/unified/base/kernel_launch.hpp"
 #endif
diff --git a/hip/base/kernel_launch_solver.hip.hpp b/hip/base/kernel_launch_solver.hip.hpp
index e0dffa5112c..ed8c610d5b2 100644
--- a/hip/base/kernel_launch_solver.hip.hpp
+++ b/hip/base/kernel_launch_solver.hip.hpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
+#ifndef GKO_COMMON_UNIFIED_BASE_KERNEL_LAUNCH_SOLVER_HPP_
 #error \
     "This file can only be used from inside common/unified/base/kernel_launch_solver.hpp"
 #endif
diff --git a/omp/base/kernel_launch.hpp b/omp/base/kernel_launch.hpp
index 485a08952ac..ae7a3ab4f94 100644
--- a/omp/base/kernel_launch.hpp
+++ b/omp/base/kernel_launch.hpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_HPP_
+#ifndef GKO_COMMON_UNIFIED_BASE_KERNEL_LAUNCH_HPP_
 #error \
     "This file can only be used from inside common/unified/base/kernel_launch.hpp"
 #endif
diff --git a/omp/base/kernel_launch_solver.hpp b/omp/base/kernel_launch_solver.hpp
index 5857ae8e7a4..b3791dd245c 100644
--- a/omp/base/kernel_launch_solver.hpp
+++ b/omp/base/kernel_launch_solver.hpp
@@ -30,7 +30,7 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ******************************<GINKGO LICENSE>*******************************/
 
-#ifndef GKO_COMMON_BASE_KERNEL_LAUNCH_SOLVER_HPP_
+#ifndef GKO_COMMON_UNIFIED_BASE_KERNEL_LAUNCH_SOLVER_HPP_
 #error \
     "This file can only be used from inside common/unified/base/kernel_launch_solver.hpp"
 #endif

From 5caa4fdaad43946658037e1a89b0581a44f50daa Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 9 Aug 2021 13:00:24 +0200
Subject: [PATCH 1001/1008] add gitlab option to ignore idr test result
 (cancelled)

---
 .gitlab-ci.yml                        | 21 ++++++++++++++++++++-
 cuda/test/solver/idr_kernels.cpp      |  1 +
 reference/test/solver/idr_kernels.cpp |  1 +
 3 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2cfb0faf44f..ec17fea0944 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -120,7 +120,22 @@ include:
     - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT} install
     - |
         (( $(ctest -N | tail -1 | sed 's/Total Tests: //') != 0 )) || exit 1
-    - ctest -V
+    - |
+        if [[ "${IGNORE_IDR}" == "TRUE" ]]; then
+          echo "some error may be ignored"
+          ctest -V -O test.log || true
+          num_fail=$( grep '(Failed)' test.log | wc -l )
+          ignored_num_fail=$( grep 'reference/test/solver/idr_kernels (Failed)' test.log | wc -l )
+          if [[ ${num_fail} -eq ${ignored_num_fail} ]]; then
+            echo "the error is ignored"
+            exit 3
+          else
+            echo "the error is not ignored"
+            exit 1
+          fi
+        else
+          ctest -V
+        fi
     - ninja test_install
     - pushd test/test_install
     - ninja install
@@ -152,6 +167,8 @@ include:
   dependencies: []
   except:
       - schedules
+  allow_failure:
+    exit_codes: 3
 
 status_pending:
   stage: init-status
@@ -325,6 +342,7 @@ build/cuda92/gcc/all/release/shared:
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
+    IGNORE_IDR: "TRUE"
     CUDA_ARCH: 61
 
 # cuda 10.0 and friends
@@ -679,6 +697,7 @@ build/nocuda-nomixed/gcc/omp/release/shared:
     BUILD_OMP: "ON"
     BUILD_TYPE: "Release"
     MIXED_PRECISION: "OFF"
+    IGNORE_IDR: "TRUE"
 
 build/nocuda-nomixed/clang/omp/debug/static:
   <<: *default_build_with_test
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index cc075f9191d..a4957c6bb55 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -207,6 +207,7 @@ class Idr : public ::testing::Test {
 
 TEST_F(Idr, IdrInitializeIsEquivalentToRef)
 {
+    EXPECT_EQ(1, 0);
     initialize_data();
 
     gko::kernels::reference::idr::initialize(ref, nrhs, m.get(), p.get(), true,
diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp
index 59c76b8c5c6..47b30a5d928 100644
--- a/reference/test/solver/idr_kernels.cpp
+++ b/reference/test/solver/idr_kernels.cpp
@@ -100,6 +100,7 @@ TYPED_TEST_SUITE(Idr, gko::test::ValueTypes);
 
 TYPED_TEST(Idr, SolvesDenseSystem)
 {
+    GTEST_ASSERT_EQ(1, 0);
     using Mtx = typename TestFixture::Mtx;
     using value_type = typename TestFixture::value_type;
     auto solver = this->idr_factory->generate(this->mtx);

From a3c481407ba9d2d610e88518fb97fd7a6163ebd8 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Mon, 9 Aug 2021 16:29:50 +0200
Subject: [PATCH 1002/1008] skip idr test when the queue is on gpu

---
 .gitlab-ci.yml                        | 21 +--------------------
 cuda/test/solver/idr_kernels.cpp      |  1 -
 dpcpp/test/solver/idr_kernels.cpp     |  4 ++++
 reference/test/solver/idr_kernels.cpp |  1 -
 4 files changed, 5 insertions(+), 22 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ec17fea0944..2cfb0faf44f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -120,22 +120,7 @@ include:
     - ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT} install
     - |
         (( $(ctest -N | tail -1 | sed 's/Total Tests: //') != 0 )) || exit 1
-    - |
-        if [[ "${IGNORE_IDR}" == "TRUE" ]]; then
-          echo "some error may be ignored"
-          ctest -V -O test.log || true
-          num_fail=$( grep '(Failed)' test.log | wc -l )
-          ignored_num_fail=$( grep 'reference/test/solver/idr_kernels (Failed)' test.log | wc -l )
-          if [[ ${num_fail} -eq ${ignored_num_fail} ]]; then
-            echo "the error is ignored"
-            exit 3
-          else
-            echo "the error is not ignored"
-            exit 1
-          fi
-        else
-          ctest -V
-        fi
+    - ctest -V
     - ninja test_install
     - pushd test/test_install
     - ninja install
@@ -167,8 +152,6 @@ include:
   dependencies: []
   except:
       - schedules
-  allow_failure:
-    exit_codes: 3
 
 status_pending:
   stage: init-status
@@ -342,7 +325,6 @@ build/cuda92/gcc/all/release/shared:
     BUILD_CUDA: "ON"
     BUILD_HIP: "ON"
     BUILD_TYPE: "Release"
-    IGNORE_IDR: "TRUE"
     CUDA_ARCH: 61
 
 # cuda 10.0 and friends
@@ -697,7 +679,6 @@ build/nocuda-nomixed/gcc/omp/release/shared:
     BUILD_OMP: "ON"
     BUILD_TYPE: "Release"
     MIXED_PRECISION: "OFF"
-    IGNORE_IDR: "TRUE"
 
 build/nocuda-nomixed/clang/omp/debug/static:
   <<: *default_build_with_test
diff --git a/cuda/test/solver/idr_kernels.cpp b/cuda/test/solver/idr_kernels.cpp
index a4957c6bb55..cc075f9191d 100644
--- a/cuda/test/solver/idr_kernels.cpp
+++ b/cuda/test/solver/idr_kernels.cpp
@@ -207,7 +207,6 @@ class Idr : public ::testing::Test {
 
 TEST_F(Idr, IdrInitializeIsEquivalentToRef)
 {
-    EXPECT_EQ(1, 0);
     initialize_data();
 
     gko::kernels::reference::idr::initialize(ref, nrhs, m.get(), p.get(), true,
diff --git a/dpcpp/test/solver/idr_kernels.cpp b/dpcpp/test/solver/idr_kernels.cpp
index 954806069af..9b655f558be 100644
--- a/dpcpp/test/solver/idr_kernels.cpp
+++ b/dpcpp/test/solver/idr_kernels.cpp
@@ -300,6 +300,10 @@ TEST_F(Idr, IdrComputeOmegaIsEquivalentToRef)
 
 TEST_F(Idr, IdrIterationOneRHSIsEquivalentToRef)
 {
+    if (dpcpp->get_queue()->get_device().is_gpu()) {
+        GTEST_SKIP() << "skip the test because oneMKL GEMM on gpu may give NaN "
+                        "(under investigation)";
+    }
     initialize_data(123, 1);
     auto ref_solver = ref_idr_factory->generate(mtx);
     auto dpcpp_solver = dpcpp_idr_factory->generate(d_mtx);
diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp
index 47b30a5d928..59c76b8c5c6 100644
--- a/reference/test/solver/idr_kernels.cpp
+++ b/reference/test/solver/idr_kernels.cpp
@@ -100,7 +100,6 @@ TYPED_TEST_SUITE(Idr, gko::test::ValueTypes);
 
 TYPED_TEST(Idr, SolvesDenseSystem)
 {
-    GTEST_ASSERT_EQ(1, 0);
     using Mtx = typename TestFixture::Mtx;
     using value_type = typename TestFixture::value_type;
     auto solver = this->idr_factory->generate(this->mtx);

From 192ef945eac24d8b52af1da922c1de552cb8cd43 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 18 Aug 2021 01:13:15 +0200
Subject: [PATCH 1003/1008] match_edge issue on isolated item

---
 reference/test/multigrid/amgx_pgm_kernels.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/reference/test/multigrid/amgx_pgm_kernels.cpp b/reference/test/multigrid/amgx_pgm_kernels.cpp
index c0270312d6c..89efb56c890 100644
--- a/reference/test/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/test/multigrid/amgx_pgm_kernels.cpp
@@ -304,7 +304,8 @@ TYPED_TEST(AmgxPgm, MatchEdge)
     snb_val[1] = 0;
     snb_val[2] = 0;
     snb_val[3] = 1;
-    snb_val[4] = 2;
+    // isolated item
+    snb_val[4] = 4;
 
     gko::kernels::reference::amgx_pgm::match_edge(this->exec, snb, agg);
 
@@ -312,7 +313,8 @@ TYPED_TEST(AmgxPgm, MatchEdge)
     ASSERT_EQ(agg_val[1], -1);
     ASSERT_EQ(agg_val[2], 0);
     ASSERT_EQ(agg_val[3], -1);
-    ASSERT_EQ(agg_val[4], -1);
+    // isolated item should be self aggregation
+    ASSERT_EQ(agg_val[4], 4);
 }
 
 

From 722c517e2b981c3d803e0600c854748384d56920 Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 18 Aug 2021 01:23:05 +0200
Subject: [PATCH 1004/1008] fix ref issue, raise issue device test
 additionally, switch order of RAP product

---
 core/multigrid/amgx_pgm.cpp              | 18 ++++++++++--------
 cuda/preconditioner/jacobi_kernels.cu    |  2 --
 cuda/test/multigrid/amgx_pgm_kernels.cpp |  7 +++++--
 hip/test/multigrid/amgx_pgm_kernels.cpp  |  7 +++++--
 omp/test/multigrid/amgx_pgm_kernels.cpp  |  7 +++++--
 reference/multigrid/amgx_pgm_kernels.cpp |  2 +-
 6 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/core/multigrid/amgx_pgm.cpp b/core/multigrid/amgx_pgm.cpp
index 83421932016..e38d10d27c3 100644
--- a/core/multigrid/amgx_pgm.cpp
+++ b/core/multigrid/amgx_pgm.cpp
@@ -93,8 +93,8 @@ void AmgxPgm<ValueType, IndexType>::generate()
     // Initial agg = -1
     exec->run(amgx_pgm::make_fill_array(agg_.get_data(), agg_.get_num_elems(),
                                         -one<IndexType>()));
-    IndexType num_unagg{0};
-    IndexType num_unagg_prev{0};
+    IndexType num_unagg = num_rows;
+    IndexType num_unagg_prev = num_rows;
     // TODO: if mtx is a hermitian matrix, weight_mtx = abs(mtx)
     // compute weight_mtx = (abs(mtx) + abs(mtx'))/2;
     auto abs_mtx = amgxpgm_op->compute_absolute();
@@ -128,9 +128,11 @@ void AmgxPgm<ValueType, IndexType>::generate()
         // copy the agg to intermediate_agg
         intermediate_agg = agg_;
     }
-    // Assign all left points
-    exec->run(amgx_pgm::make_assign_to_exist_agg(weight_mtx.get(), diag.get(),
-                                                 agg_, intermediate_agg));
+    if (num_unagg != 0) {
+        // Assign all left points
+        exec->run(amgx_pgm::make_assign_to_exist_agg(
+            weight_mtx.get(), diag.get(), agg_, intermediate_agg));
+    }
     IndexType num_agg = 0;
     // Renumber the index
     exec->run(amgx_pgm::make_renumber(agg_, &num_agg));
@@ -153,9 +155,9 @@ void AmgxPgm<ValueType, IndexType>::generate()
     // TODO: use less memory footprint to improve it
     auto coarse_matrix =
         share(matrix_type::create(exec, gko::dim<2>{coarse_dim, coarse_dim}));
-    auto tmp = matrix_type::create(exec, gko::dim<2>{coarse_dim, fine_dim});
-    restrict_op->apply(amgxpgm_op, tmp.get());
-    tmp->apply(prolong_op.get(), coarse_matrix.get());
+    auto tmp = matrix_type::create(exec, gko::dim<2>{fine_dim, coarse_dim});
+    amgxpgm_op->apply(prolong_op.get(), tmp.get());
+    restrict_op->apply(tmp.get(), coarse_matrix.get());
 
     this->set_multigrid_level(prolong_op, coarse_matrix, restrict_op);
 }
diff --git a/cuda/preconditioner/jacobi_kernels.cu b/cuda/preconditioner/jacobi_kernels.cu
index 66b8b7a13eb..77303eb16d7 100644
--- a/cuda/preconditioner/jacobi_kernels.cu
+++ b/cuda/preconditioner/jacobi_kernels.cu
@@ -65,8 +65,6 @@ constexpr int default_num_warps = 32;
 // current GPUs have at most 84 SMs)
 constexpr int default_grid_size = 32 * 32 * 128;
 
-constexpr int default_block_size = 512;
-
 
 #include "common/cuda_hip/preconditioner/jacobi_kernels.hpp.inc"
 
diff --git a/cuda/test/multigrid/amgx_pgm_kernels.cpp b/cuda/test/multigrid/amgx_pgm_kernels.cpp
index 07672359615..493192c8630 100644
--- a/cuda/test/multigrid/amgx_pgm_kernels.cpp
+++ b/cuda/test/multigrid/amgx_pgm_kernels.cpp
@@ -127,8 +127,11 @@ class AmgxPgm : public ::testing::Test {
         int nrhs = 3;
 
         agg = gen_agg_array(m, n);
-        unfinished_agg = gen_array(m, -1, n - 1);
-        strongest_neighbor = gen_array(m, 0, n - 1);
+        // only use 0 ~ n-2 and ensure the end isolated and not yet finished
+        unfinished_agg = gen_array(m, -1, n - 2);
+        unfinished_agg.get_data()[n - 1] = -1;
+        strongest_neighbor = gen_array(m, 0, n - 2);
+        strongest_neighbor.get_data()[n - 1] = n - 1;
         coarse_vector = gen_mtx(n, nrhs);
         fine_vector = gen_mtx(m, nrhs);
         auto weight = gen_mtx(m, m);
diff --git a/hip/test/multigrid/amgx_pgm_kernels.cpp b/hip/test/multigrid/amgx_pgm_kernels.cpp
index 879eae1876e..923e8e0765a 100644
--- a/hip/test/multigrid/amgx_pgm_kernels.cpp
+++ b/hip/test/multigrid/amgx_pgm_kernels.cpp
@@ -126,8 +126,11 @@ class AmgxPgm : public ::testing::Test {
         int nrhs = 3;
 
         agg = gen_agg_array(m, n);
-        unfinished_agg = gen_array(m, -1, n - 1);
-        strongest_neighbor = gen_array(m, 0, n - 1);
+        // only use 0 ~ n-2 and ensure the end isolated and not yet finished
+        unfinished_agg = gen_array(m, -1, n - 2);
+        unfinished_agg.get_data()[n - 1] = -1;
+        strongest_neighbor = gen_array(m, 0, n - 2);
+        strongest_neighbor.get_data()[n - 1] = n - 1;
         coarse_vector = gen_mtx(n, nrhs);
         fine_vector = gen_mtx(m, nrhs);
         auto weight = gen_mtx(m, m);
diff --git a/omp/test/multigrid/amgx_pgm_kernels.cpp b/omp/test/multigrid/amgx_pgm_kernels.cpp
index 59b40657703..68fdd9ca893 100644
--- a/omp/test/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/test/multigrid/amgx_pgm_kernels.cpp
@@ -117,8 +117,11 @@ class AmgxPgm : public ::testing::Test {
         int nrhs = 3;
 
         agg = gen_agg_array(m, n);
-        unfinished_agg = gen_array(m, -1, n - 1);
-        strongest_neighbor = gen_array(m, 0, n - 1);
+        // only use 0 ~ n-2 and ensure the end isolated and not yet finished
+        unfinished_agg = gen_array(m, -1, n - 2);
+        unfinished_agg.get_data()[n - 1] = -1;
+        strongest_neighbor = gen_array(m, 0, n - 2);
+        strongest_neighbor.get_data()[n - 1] = n - 1;
         coarse_vector = gen_mtx(n, nrhs);
         fine_vector = gen_mtx(m, nrhs);
         auto weight = gen_mtx(m, m);
diff --git a/reference/multigrid/amgx_pgm_kernels.cpp b/reference/multigrid/amgx_pgm_kernels.cpp
index 5ad1ff8d108..bd696164c4f 100644
--- a/reference/multigrid/amgx_pgm_kernels.cpp
+++ b/reference/multigrid/amgx_pgm_kernels.cpp
@@ -74,7 +74,7 @@ void match_edge(std::shared_ptr<const ReferenceExecutor> exec,
             auto neighbor = strongest_neighbor_vals[i];
             // i < neighbor always holds when neighbor is not -1
             if (neighbor != -1 && strongest_neighbor_vals[neighbor] == i &&
-                i < neighbor) {
+                i <= neighbor) {
                 // Use the smaller index as agg point
                 agg_vals[i] = i;
                 agg_vals[neighbor] = i;

From de2cd71b2efb59da8e7526c052fcfa180da778aa Mon Sep 17 00:00:00 2001
From: "Yuhsiang M. Tsai" <yhmtsai@gmail.com>
Date: Wed, 18 Aug 2021 11:53:32 +0200
Subject: [PATCH 1005/1008] fix device issue, use new GKO_FACTORY_PARAMETER_*

---
 common/cuda_hip/multigrid/amgx_pgm_kernels.hpp.inc | 2 +-
 hip/preconditioner/jacobi_kernels.hip.cpp          | 1 -
 include/ginkgo/core/multigrid/amgx_pgm.hpp         | 6 +++---
 include/ginkgo/core/multigrid/multigrid_level.hpp  | 5 +++++
 omp/multigrid/amgx_pgm_kernels.cpp                 | 2 +-
 5 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/common/cuda_hip/multigrid/amgx_pgm_kernels.hpp.inc b/common/cuda_hip/multigrid/amgx_pgm_kernels.hpp.inc
index 540478ba427..f03fc4a59b3 100644
--- a/common/cuda_hip/multigrid/amgx_pgm_kernels.hpp.inc
+++ b/common/cuda_hip/multigrid/amgx_pgm_kernels.hpp.inc
@@ -47,7 +47,7 @@ __global__ __launch_bounds__(default_block_size) void match_edge_kernel(
     }
     auto neighbor = strongest_neighbor_vals[tidx];
     if (neighbor != -1 && strongest_neighbor_vals[neighbor] == tidx &&
-        tidx < neighbor) {
+        tidx <= neighbor) {
         // Use the smaller index as agg point
         agg_vals[tidx] = tidx;
         agg_vals[neighbor] = tidx;
diff --git a/hip/preconditioner/jacobi_kernels.hip.cpp b/hip/preconditioner/jacobi_kernels.hip.cpp
index f6f4445326e..40e3aff69c5 100644
--- a/hip/preconditioner/jacobi_kernels.hip.cpp
+++ b/hip/preconditioner/jacobi_kernels.hip.cpp
@@ -72,7 +72,6 @@ constexpr int default_num_warps = 32;
 // current GPUs have at most 84 SMs)
 constexpr int default_grid_size = 32 * 32 * 128;
 
-constexpr int default_block_size = 512;
 
 #include "common/cuda_hip/preconditioner/jacobi_kernels.hpp.inc"
 
diff --git a/include/ginkgo/core/multigrid/amgx_pgm.hpp b/include/ginkgo/core/multigrid/amgx_pgm.hpp
index 5c5b36ebf7c..4afc049aa60 100644
--- a/include/ginkgo/core/multigrid/amgx_pgm.hpp
+++ b/include/ginkgo/core/multigrid/amgx_pgm.hpp
@@ -121,7 +121,7 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
          * NVIDIA AMGX Reference Manual (October 2017, API Version 2,
          * https://github.com/NVIDIA/AMGX/blob/main/doc/AMGX_Reference.pdf).
          */
-        unsigned GKO_FACTORY_PARAMETER(max_iterations, 15u);
+        unsigned GKO_FACTORY_PARAMETER_SCALAR(max_iterations, 15u);
 
         /**
          * The maximum ratio of unassigned number, which is valid in the
@@ -129,7 +129,7 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
          * Reference Manual (October 2017, API Version 2,
          * https://github.com/NVIDIA/AMGX/blob/main/doc/AMGX_Reference.pdf).
          */
-        double GKO_FACTORY_PARAMETER(max_unassigned_ratio, 0.05);
+        double GKO_FACTORY_PARAMETER_SCALAR(max_unassigned_ratio, 0.05);
 
         /**
          * Use the deterministic assign_to_exist_agg method or not.
@@ -138,7 +138,7 @@ class AmgxPgm : public EnableLinOp<AmgxPgm<ValueType, IndexType>>,
          * from the same matrix. Otherwise, the aggregated group might be
          * different depending on the execution ordering.
          */
-        bool GKO_FACTORY_PARAMETER(deterministic, false);
+        bool GKO_FACTORY_PARAMETER_SCALAR(deterministic, false);
     };
     GKO_ENABLE_LIN_OP_FACTORY(AmgxPgm, parameters, Factory);
     GKO_ENABLE_BUILD_METHOD(Factory);
diff --git a/include/ginkgo/core/multigrid/multigrid_level.hpp b/include/ginkgo/core/multigrid/multigrid_level.hpp
index e86e51bc53b..b170b01f0a7 100644
--- a/include/ginkgo/core/multigrid/multigrid_level.hpp
+++ b/include/ginkgo/core/multigrid/multigrid_level.hpp
@@ -46,6 +46,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 namespace gko {
+/**
+ * @brief The multigrid components namespace.
+ *
+ * @ingroup gko
+ */
 namespace multigrid {
 
 
diff --git a/omp/multigrid/amgx_pgm_kernels.cpp b/omp/multigrid/amgx_pgm_kernels.cpp
index 07e84630d85..94c7f9b983d 100644
--- a/omp/multigrid/amgx_pgm_kernels.cpp
+++ b/omp/multigrid/amgx_pgm_kernels.cpp
@@ -78,7 +78,7 @@ void match_edge(std::shared_ptr<const OmpExecutor> exec,
         if (agg_vals[i] == -1) {
             auto neighbor = strongest_neighbor_vals[i];
             if (neighbor != -1 && strongest_neighbor_vals[neighbor] == i &&
-                i < neighbor) {
+                i <= neighbor) {
                 // Use the smaller index as agg point
                 agg_vals[i] = i;
                 agg_vals[neighbor] = i;

From 75780659ed87cb2b295e166cc4b47d9340094fd3 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 10 Aug 2021 10:51:48 +0200
Subject: [PATCH 1006/1008] Update Changelog, Readme

---
 CHANGELOG.md | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++
 README.md    |   1 +
 2 files changed, 134 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c88066c7ebe..59c68a8b8eb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,139 @@ commits. For a comprehensive list, use the following command:
 git log --first-parent
 ```
 
+## Version 1.4.0
+
+The Ginkgo team is proud to announce the new Ginkgo minor release 1.4.0. This
+release brings most of the Ginkgo functionality to the Intel DPC++ ecosystem
+which enables Intel-GPU execution. The only Ginkgo features which have not been
+ported yet are some preconditioners. Ginkgo's mixed-precision support is greatly
+enhanced thanks to:
+1. The new Accessor concept, which allows writing kernels featuring on-the-fly
+memory compression, among other features. The accessor can be used as
+header-only, see the [accessor BLAS benchmarks repository](https://github.com/ginkgo-project/accessor-BLAS/tree/develop) as a usage example.
+2. All LinOps now transparently support mixed-precision execution. By default,
+this is done through a temporary copy which may have a performance impact but
+already allows mixed-precision research.
+
+Native mixed-precision ELL kernels are implemented which do not see this cost.
+The accessor is also leveraged in a new CB-GMRES solver which allows for
+performance improvements by compressing the Krylov basis vectors. Many other
+features have been added to Ginkgo, such as reordering support, a new IDR
+solver, Incomplete Cholesky preconditioner, matrix assembly support (only CPU
+for now), machine topology information, and more!
+
+Supported systems and requirements:
++ For all platforms, cmake 3.13+
++ C++14 compliant compiler
++ Linux and MacOS
+  + gcc: 5.3+, 6.3+, 7.3+, all versions after 8.1+
+  + clang: 3.9+
+  + Intel compiler: 2018+
+  + Apple LLVM: 8.0+
+  + CUDA module: CUDA 9.0+
+  + HIP module: ROCm 3.5+
+  + DPC++ module: Intel OneAPI 2021.3. Set the CXX compiler to `dpcpp`.
++ Windows
+  + MinGW and Cygwin: gcc 5.3+, 6.3+, 7.3+, all versions after 8.1+
+  + Microsoft Visual Studio: VS 2019
+  + CUDA module: CUDA 9.0+, Microsoft Visual Studio
+  + OpenMP module: MinGW or Cygwin.
+
+
+Algorithm and important feature additions:
++ Add a new DPC++ Executor for SYCL execution and other base utilities
+  [#648](https://github.com/ginkgo-project/ginkgo/pull/648), [#661](https://github.com/ginkgo-project/ginkgo/pull/661), [#757](https://github.com/ginkgo-project/ginkgo/pull/757), [#832](https://github.com/ginkgo-project/ginkgo/pull/832)
++ Port matrix formats, solvers and related kernels to DPC++. For some kernels,
+  also make use of a shared kernel implementation for all executors (except
+  Reference). [#710](https://github.com/ginkgo-project/ginkgo/pull/710), [#799](https://github.com/ginkgo-project/ginkgo/pull/799), [#779](https://github.com/ginkgo-project/ginkgo/pull/779), [#733](https://github.com/ginkgo-project/ginkgo/pull/733), [#844](https://github.com/ginkgo-project/ginkgo/pull/844), [#843](https://github.com/ginkgo-project/ginkgo/pull/843), [#789](https://github.com/ginkgo-project/ginkgo/pull/789), [#845](https://github.com/ginkgo-project/ginkgo/pull/845), [#849](https://github.com/ginkgo-project/ginkgo/pull/849), [#855](https://github.com/ginkgo-project/ginkgo/pull/855), [#856](https://github.com/ginkgo-project/ginkgo/pull/856)
++ Add accessors which allow multi-precision kernels, among other things.
+  [#643](https://github.com/ginkgo-project/ginkgo/pull/643), [#708](https://github.com/ginkgo-project/ginkgo/pull/708)
++ Add support for mixed precision operations through apply in all LinOps. [#677](https://github.com/ginkgo-project/ginkgo/pull/677)
++ Add incomplete Cholesky factorizations and preconditioners as well as some
+  improvements to ILU. [#672](https://github.com/ginkgo-project/ginkgo/pull/672), [#837](https://github.com/ginkgo-project/ginkgo/pull/837), [#846](https://github.com/ginkgo-project/ginkgo/pull/846)
++ Add an AMGX implementation and kernels on all devices but DPC++.
+  [#528](https://github.com/ginkgo-project/ginkgo/pull/528), [#695](https://github.com/ginkgo-project/ginkgo/pull/695), [#860](https://github.com/ginkgo-project/ginkgo/pull/860)
++ Add a new mixed-precision capability solver, Compressed Basis GMRES
+  (CB-GMRES). [#693](https://github.com/ginkgo-project/ginkgo/pull/693), [#763](https://github.com/ginkgo-project/ginkgo/pull/763)
++ Add the IDR(s) solver. [#620](https://github.com/ginkgo-project/ginkgo/pull/620)
++ Add a new fixed-size block CSR matrix format (for the Reference executor).
+  [#671](https://github.com/ginkgo-project/ginkgo/pull/671), [#730](https://github.com/ginkgo-project/ginkgo/pull/730)
++ Add native mixed-precision support to the ELL format. [#717](https://github.com/ginkgo-project/ginkgo/pull/717), [#780](https://github.com/ginkgo-project/ginkgo/pull/780)
++ Add Reverse Cuthill-McKee reordering [#500](https://github.com/ginkgo-project/ginkgo/pull/500), [#649](https://github.com/ginkgo-project/ginkgo/pull/649)
++ Add matrix assembly support on CPUs. [#644](https://github.com/ginkgo-project/ginkgo/pull/644)
+
+Other additions:
++ Add possibility to apply real matrices to complex vectors.
+  [#655](https://github.com/ginkgo-project/ginkgo/pull/655), [#636](https://github.com/ginkgo-project/ginkgo/pull/636), [#658](https://github.com/ginkgo-project/ginkgo/pull/658)
++ Add symmetric permutation and improve existing permutations.
+  [#684](https://github.com/ginkgo-project/ginkgo/pull/684), [#657](https://github.com/ginkgo-project/ginkgo/pull/657), [#663](https://github.com/ginkgo-project/ginkgo/pull/663)
++ Add a MachineTopology class with HWLOC support [#554](https://github.com/ginkgo-project/ginkgo/pull/554), [#697](https://github.com/ginkgo-project/ginkgo/pull/697)
++ Add an implicit residual norm criterion. [#702](https://github.com/ginkgo-project/ginkgo/pull/702), [#714](https://github.com/ginkgo-project/ginkgo/pull/714),  [#818](https://github.com/ginkgo-project/ginkgo/pull/818), [#850](https://github.com/ginkgo-project/ginkgo/pull/850)
++ Add an heat equation example. [#698](https://github.com/ginkgo-project/ginkgo/pull/698), [#706](https://github.com/ginkgo-project/ginkgo/pull/706)
++ Add ccache support in CMake and CI. [#725](https://github.com/ginkgo-project/ginkgo/pull/725), [#739](https://github.com/ginkgo-project/ginkgo/pull/739)
++ Allow tuning and benchmarking variables non intrusively. [#692](https://github.com/ginkgo-project/ginkgo/pull/692)
++ Add triangular solver benchmark [#664](https://github.com/ginkgo-project/ginkgo/pull/664)
++ Add benchmarks for BLAS operations [#772](https://github.com/ginkgo-project/ginkgo/pull/772), [#829](https://github.com/ginkgo-project/ginkgo/pull/829)
++ Add support for different precisions and consistent index types in benchmarks.
+  [#675](https://github.com/ginkgo-project/ginkgo/pull/675), [#828](https://github.com/ginkgo-project/ginkgo/pull/828)
++ Add a Github bot system to facilitate development and PR management.
+  [#667](https://github.com/ginkgo-project/ginkgo/pull/667), [#674](https://github.com/ginkgo-project/ginkgo/pull/674), [#689](https://github.com/ginkgo-project/ginkgo/pull/689), [#853](https://github.com/ginkgo-project/ginkgo/pull/853)
++ Add Intel (DPC++) CI support and enable CI on HPC systems. [#736](https://github.com/ginkgo-project/ginkgo/pull/736), [#751](https://github.com/ginkgo-project/ginkgo/pull/751), [#781](https://github.com/ginkgo-project/ginkgo/pull/781)
++ Add ssh debugging for Github Actions CI. [#749](https://github.com/ginkgo-project/ginkgo/pull/749)
++ Add pipeline segmentation for better CI speed. [#737](https://github.com/ginkgo-project/ginkgo/pull/737)
+
+
+Changes:
++ Extends ISAI from triangular to general and spd matrices. [#690](https://github.com/ginkgo-project/ginkgo/pull/690)
++ Add a Scalar Jacobi specialization and kernels. [#808](https://github.com/ginkgo-project/ginkgo/pull/808), [#834](https://github.com/ginkgo-project/ginkgo/pull/834), [#854](https://github.com/ginkgo-project/ginkgo/pull/854)
++ Add implicit residual log for solvers and benchmarks. [#707](https://github.com/ginkgo-project/ginkgo/pull/707)
++ Change handling of the conjugate in the dense dot product. [#755](https://github.com/ginkgo-project/ginkgo/pull/755)
++ Improved Dense stride handling. [#774](https://github.com/ginkgo-project/ginkgo/pull/774)
++ Multiple improvements to the OpenMP kernels performance, including COO,
+an exclusive prefix sum, and more. [#703](https://github.com/ginkgo-project/ginkgo/pull/703), [#765](https://github.com/ginkgo-project/ginkgo/pull/765), [#740](https://github.com/ginkgo-project/ginkgo/pull/740)
++ Allow specialization of submatrix and other dense creation functions in solvers. [#718](https://github.com/ginkgo-project/ginkgo/pull/718)
++ Improved Identity constructor and treatment of rectangular matrices. [#646](https://github.com/ginkgo-project/ginkgo/pull/646)
++ Allow CUDA/HIP executors to select allocation mode. [#758](https://github.com/ginkgo-project/ginkgo/pull/758)
++ Check if executors share the same memory. [#670](https://github.com/ginkgo-project/ginkgo/pull/670)
++ Improve test install and smoke testing support. [#721](https://github.com/ginkgo-project/ginkgo/pull/721)
++ Update the JOSS paper citation and add publications in the documentation.
+  [#629](https://github.com/ginkgo-project/ginkgo/pull/629), [#724](https://github.com/ginkgo-project/ginkgo/pull/724)
++ Improve the version output. [#806](https://github.com/ginkgo-project/ginkgo/pull/806)
++ Add some utilities for dim and span. [#821](https://github.com/ginkgo-project/ginkgo/pull/821)
++ Improved solver and preconditioner benchmarks. [#660](https://github.com/ginkgo-project/ginkgo/pull/660)
++ Improve benchmark timing and output. [#669](https://github.com/ginkgo-project/ginkgo/pull/669), [#791](https://github.com/ginkgo-project/ginkgo/pull/791), [#801](https://github.com/ginkgo-project/ginkgo/pull/801), [#812](https://github.com/ginkgo-project/ginkgo/pull/812)
+
+
+Fixes:
++ Sorting fix for the Jacobi preconditioner. [#659](https://github.com/ginkgo-project/ginkgo/pull/659)
++ Also log the first residual norm in CGS [#735](https://github.com/ginkgo-project/ginkgo/pull/735)
++ Fix BiCG and HIP CSR to work with complex matrices. [#651](https://github.com/ginkgo-project/ginkgo/pull/651)
++ Fix Coo SpMV on strided vectors. [#807](https://github.com/ginkgo-project/ginkgo/pull/807)
++ Fix segfault of extract_diagonal, add short-and-fat test. [#769](https://github.com/ginkgo-project/ginkgo/pull/769)
++ Fix device_reset issue by moving counter/mutex to device. [#810](https://github.com/ginkgo-project/ginkgo/pull/810)
++ Fix `EnableLogging` superclass. [#841](https://github.com/ginkgo-project/ginkgo/pull/841)
++ Support ROCm 4.1.x and breaking HIP_PLATFORM changes. [#726](https://github.com/ginkgo-project/ginkgo/pull/726)
++ Decreased test size for a few device tests. [#742](https://github.com/ginkgo-project/ginkgo/pull/742)
++ Fix multiple issues with our CMake HIP and RPATH setup.
+  [#712](https://github.com/ginkgo-project/ginkgo/pull/712), [#745](https://github.com/ginkgo-project/ginkgo/pull/745), [#709](https://github.com/ginkgo-project/ginkgo/pull/709)
++ Cleanup our CMake installation step. [#713](https://github.com/ginkgo-project/ginkgo/pull/713)
++ Various simplification and fixes to the Windows CMake setup. [#720](https://github.com/ginkgo-project/ginkgo/pull/720), [#785](https://github.com/ginkgo-project/ginkgo/pull/785)
++ Simplify third-party integration. [#786](https://github.com/ginkgo-project/ginkgo/pull/786)
++ Improve Ginkgo device arch flags management. [#696](https://github.com/ginkgo-project/ginkgo/pull/696)
++ Other fixes and improvements to the CMake setup.
+  [#685](https://github.com/ginkgo-project/ginkgo/pull/685), [#792](https://github.com/ginkgo-project/ginkgo/pull/792), [#705](https://github.com/ginkgo-project/ginkgo/pull/705), [#836](https://github.com/ginkgo-project/ginkgo/pull/836)
++ Clarification of dense norm documentation [#784](https://github.com/ginkgo-project/ginkgo/pull/784)
++ Various development tools fixes and improvements [#738](https://github.com/ginkgo-project/ginkgo/pull/738), [#830](https://github.com/ginkgo-project/ginkgo/pull/830), [#840](https://github.com/ginkgo-project/ginkgo/pull/840)
++ Make multiple operators/constructors explicit. [#650](https://github.com/ginkgo-project/ginkgo/pull/650), [#761](https://github.com/ginkgo-project/ginkgo/pull/761)
++ Fix some issues, memory leaks and warnings found by MSVC.
+  [#666](https://github.com/ginkgo-project/ginkgo/pull/666), [#731](https://github.com/ginkgo-project/ginkgo/pull/731)
++ Improved solver memory estimates and consistent iteration counts [#691](https://github.com/ginkgo-project/ginkgo/pull/691)
++ Various logger improvements and fixes [#728](https://github.com/ginkgo-project/ginkgo/pull/728), [#743](https://github.com/ginkgo-project/ginkgo/pull/743), [#754](https://github.com/ginkgo-project/ginkgo/pull/754)
++ Fix for ForwardIterator requirements in iterator_factory. [#665](https://github.com/ginkgo-project/ginkgo/pull/665)
++ Various benchmark fixes. [#647](https://github.com/ginkgo-project/ginkgo/pull/647), [#673](https://github.com/ginkgo-project/ginkgo/pull/673), [#722](https://github.com/ginkgo-project/ginkgo/pull/722)
++ Various CI fixes and improvements. [#642](https://github.com/ginkgo-project/ginkgo/pull/642), [#641](https://github.com/ginkgo-project/ginkgo/pull/641), [#795](https://github.com/ginkgo-project/ginkgo/pull/795), [#783](https://github.com/ginkgo-project/ginkgo/pull/783), [#793](https://github.com/ginkgo-project/ginkgo/pull/793), [#852](https://github.com/ginkgo-project/ginkgo/pull/852)
+
+
 ## Version 1.3.0
 
 The Ginkgo team is proud to announce the new minor release of Ginkgo version
diff --git a/README.md b/README.md
index 16f8894d106..9520d248af1 100644
--- a/README.md
+++ b/README.md
@@ -62,6 +62,7 @@ The Ginkgo DPC++ module has the following __additional__ requirements:
 
 * _OneAPI 2021.3+_
 * Set `dpcpp` as the `CMAKE_CXX_COMPILER`
+* `C++17` is used to compile this module, while the rest of Ginkgo is compiled using `c++14`.
 * The following oneAPI packages should be available:
     * oneMKL
     * oneDPL

From 44538ed114b54d1affcd86daeda167a091f955d4 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Tue, 10 Aug 2021 11:00:05 +0200
Subject: [PATCH 1007/1008] Bump third party libraries

---
 third_party/gflags/CMakeLists.txt    | 2 +-
 third_party/gtest/CMakeLists.txt     | 4 ++--
 third_party/hwloc/CMakeLists.txt     | 4 ++--
 third_party/rapidjson/CMakeLists.txt | 4 ++--
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/third_party/gflags/CMakeLists.txt b/third_party/gflags/CMakeLists.txt
index 7eea739001f..c34c2993a42 100644
--- a/third_party/gflags/CMakeLists.txt
+++ b/third_party/gflags/CMakeLists.txt
@@ -3,7 +3,7 @@ include(FetchContent)
 FetchContent_Declare(
     gflags
     GIT_REPOSITORY https://github.com/gflags/gflags.git
-    GIT_TAG        f7388c6655e699f777a5a74a3c9880b9cfaabe59
+    GIT_TAG        827c769e5fc98e0f2a34c47cef953cc6328abced
 )
 # need to set the variables in CACHE due to CMP0077
 set(GFLAGS_BUILD_TESTING OFF CACHE INTERNAL "")
diff --git a/third_party/gtest/CMakeLists.txt b/third_party/gtest/CMakeLists.txt
index 25f340ce122..23bfc3f4a62 100644
--- a/third_party/gtest/CMakeLists.txt
+++ b/third_party/gtest/CMakeLists.txt
@@ -3,7 +3,7 @@ include(FetchContent)
 FetchContent_Declare(
     googletest
     GIT_REPOSITORY https://github.com/google/googletest.git
-    GIT_TAG        6a7ed316a5cdc07b6d26362c90770787513822d4
+    GIT_TAG        e2239ee6043f73722e7aa812a459f54a28552929
 )
 # need to set the variables in CACHE due to CMP0077
 set(gtest_disable_pthreads ON CACHE INTERNAL "")
@@ -22,4 +22,4 @@ set_target_properties(gtest gtest_main PROPERTIES
     LIBRARY_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}")
 # by default, the outdated targets are not being exported
 add_library(GTest::Main ALIAS gtest_main)
-add_library(GTest::GTest ALIAS gtest)
\ No newline at end of file
+add_library(GTest::GTest ALIAS gtest)
diff --git a/third_party/hwloc/CMakeLists.txt b/third_party/hwloc/CMakeLists.txt
index c4d82d5e654..a47e9a84877 100644
--- a/third_party/hwloc/CMakeLists.txt
+++ b/third_party/hwloc/CMakeLists.txt
@@ -1,7 +1,7 @@
 message(STATUS "Configuring and building HWLOC")
 set(TPL_HWLOC_PATH "${PROJECT_BINARY_DIR}/third_party/hwloc")
-ginkgo_load_and_configure_package(hwloc_external "https://download.open-mpi.org/release/hwloc/v2.3/hwloc-2.3.0.tar.gz"
-    "SHA1=69d8e2f088d365cf5e86d04f049c6b1e09e2b9c3"
+ginkgo_load_and_configure_package(hwloc_external "https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz"
+    "SHA1=b94950e8958e1125ca75ecac0bc0259ee3d108c4"
     "${TPL_HWLOC_PATH}/src/configure" "--disable-nvml" "--disable-cuda" "--disable-rsmi"
     )
 
diff --git a/third_party/rapidjson/CMakeLists.txt b/third_party/rapidjson/CMakeLists.txt
index c272a8b41a0..db8ad449fe6 100644
--- a/third_party/rapidjson/CMakeLists.txt
+++ b/third_party/rapidjson/CMakeLists.txt
@@ -3,7 +3,7 @@ include(FetchContent)
 FetchContent_Declare(
     rapidjson
     GIT_REPOSITORY https://github.com/Tencent/rapidjson.git
-    GIT_TAG        88bd956d66d348f478bceebfdadb8e26c6844695
+    GIT_TAG        48fbd8cd202ca54031fe799db2ad44ffa8e77c13
 )
 FetchContent_GetProperties(rapidjson)
 if(NOT rapidjson_POPULATED)
@@ -11,4 +11,4 @@ if(NOT rapidjson_POPULATED)
 endif()
 set(RapidJSON_INCLUDE_DIR "${rapidjson_SOURCE_DIR}/include")
 add_library(rapidjson INTERFACE)
-set_target_properties(rapidjson PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${RapidJSON_INCLUDE_DIR}")
\ No newline at end of file
+set_target_properties(rapidjson PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${RapidJSON_INCLUDE_DIR}")

From 555f65e8340462077206a34aed649e4f8dfd2f23 Mon Sep 17 00:00:00 2001
From: Terry Cojean <terry.cojean@kit.edu>
Date: Wed, 11 Aug 2021 12:06:26 +0200
Subject: [PATCH 1008/1008] Review updates.

Co-authored-by: Yu-Hsiang Tsai <yhmtsai@gmail.com>
Co-authored-by: Aditya Kashi <aditya.kashi@kit.edu>
Co-authored-by: Pratik Nayak <pratikvn@protonmail.com>
---
 CHANGELOG.md                     | 18 +++++++++++-------
 README.md                        |  2 +-
 third_party/gtest/CMakeLists.txt |  4 ++--
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 59c68a8b8eb..238f8921921 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,9 +10,10 @@ git log --first-parent
 
 The Ginkgo team is proud to announce the new Ginkgo minor release 1.4.0. This
 release brings most of the Ginkgo functionality to the Intel DPC++ ecosystem
-which enables Intel-GPU execution. The only Ginkgo features which have not been
-ported yet are some preconditioners. Ginkgo's mixed-precision support is greatly
-enhanced thanks to:
+which enables Intel-GPU and CPU execution. The only Ginkgo features which have
+not been ported yet are some preconditioners.
+
+Ginkgo's mixed-precision support is greatly enhanced thanks to:
 1. The new Accessor concept, which allows writing kernels featuring on-the-fly
 memory compression, among other features. The accessor can be used as
 header-only, see the [accessor BLAS benchmarks repository](https://github.com/ginkgo-project/accessor-BLAS/tree/develop) as a usage example.
@@ -66,14 +67,18 @@ Algorithm and important feature additions:
 + Add native mixed-precision support to the ELL format. [#717](https://github.com/ginkgo-project/ginkgo/pull/717), [#780](https://github.com/ginkgo-project/ginkgo/pull/780)
 + Add Reverse Cuthill-McKee reordering [#500](https://github.com/ginkgo-project/ginkgo/pull/500), [#649](https://github.com/ginkgo-project/ginkgo/pull/649)
 + Add matrix assembly support on CPUs. [#644](https://github.com/ginkgo-project/ginkgo/pull/644)
++ Extends ISAI from triangular to general and spd matrices. [#690](https://github.com/ginkgo-project/ginkgo/pull/690)
 
 Other additions:
 + Add possibility to apply real matrices to complex vectors.
-  [#655](https://github.com/ginkgo-project/ginkgo/pull/655), [#636](https://github.com/ginkgo-project/ginkgo/pull/636), [#658](https://github.com/ginkgo-project/ginkgo/pull/658)
+  [#655](https://github.com/ginkgo-project/ginkgo/pull/655), [#658](https://github.com/ginkgo-project/ginkgo/pull/658)
++ Add functions to compute the absolute of a matrix format. [#636](https://github.com/ginkgo-project/ginkgo/pull/636)
 + Add symmetric permutation and improve existing permutations.
   [#684](https://github.com/ginkgo-project/ginkgo/pull/684), [#657](https://github.com/ginkgo-project/ginkgo/pull/657), [#663](https://github.com/ginkgo-project/ginkgo/pull/663)
 + Add a MachineTopology class with HWLOC support [#554](https://github.com/ginkgo-project/ginkgo/pull/554), [#697](https://github.com/ginkgo-project/ginkgo/pull/697)
-+ Add an implicit residual norm criterion. [#702](https://github.com/ginkgo-project/ginkgo/pull/702), [#714](https://github.com/ginkgo-project/ginkgo/pull/714),  [#818](https://github.com/ginkgo-project/ginkgo/pull/818), [#850](https://github.com/ginkgo-project/ginkgo/pull/850)
++ Add an implicit residual norm criterion. [#702](https://github.com/ginkgo-project/ginkgo/pull/702), [#818](https://github.com/ginkgo-project/ginkgo/pull/818), [#850](https://github.com/ginkgo-project/ginkgo/pull/850)
++ Row-major accessor is generalized to more than 2 dimensions and a new
+  "block column-major" accessor has been added. [#707](https://github.com/ginkgo-project/ginkgo/pull/707)
 + Add an heat equation example. [#698](https://github.com/ginkgo-project/ginkgo/pull/698), [#706](https://github.com/ginkgo-project/ginkgo/pull/706)
 + Add ccache support in CMake and CI. [#725](https://github.com/ginkgo-project/ginkgo/pull/725), [#739](https://github.com/ginkgo-project/ginkgo/pull/739)
 + Allow tuning and benchmarking variables non intrusively. [#692](https://github.com/ginkgo-project/ginkgo/pull/692)
@@ -89,9 +94,8 @@ Other additions:
 
 
 Changes:
-+ Extends ISAI from triangular to general and spd matrices. [#690](https://github.com/ginkgo-project/ginkgo/pull/690)
 + Add a Scalar Jacobi specialization and kernels. [#808](https://github.com/ginkgo-project/ginkgo/pull/808), [#834](https://github.com/ginkgo-project/ginkgo/pull/834), [#854](https://github.com/ginkgo-project/ginkgo/pull/854)
-+ Add implicit residual log for solvers and benchmarks. [#707](https://github.com/ginkgo-project/ginkgo/pull/707)
++ Add implicit residual log for solvers and benchmarks. [#714](https://github.com/ginkgo-project/ginkgo/pull/714)
 + Change handling of the conjugate in the dense dot product. [#755](https://github.com/ginkgo-project/ginkgo/pull/755)
 + Improved Dense stride handling. [#774](https://github.com/ginkgo-project/ginkgo/pull/774)
 + Multiple improvements to the OpenMP kernels performance, including COO,
diff --git a/README.md b/README.md
index 9520d248af1..de30f8d31b5 100644
--- a/README.md
+++ b/README.md
@@ -62,7 +62,7 @@ The Ginkgo DPC++ module has the following __additional__ requirements:
 
 * _OneAPI 2021.3+_
 * Set `dpcpp` as the `CMAKE_CXX_COMPILER`
-* `C++17` is used to compile this module, while the rest of Ginkgo is compiled using `c++14`.
+* `c++17` is used to compile this module, while the rest of Ginkgo is compiled using `c++14`.
 * The following oneAPI packages should be available:
     * oneMKL
     * oneDPL
diff --git a/third_party/gtest/CMakeLists.txt b/third_party/gtest/CMakeLists.txt
index 23bfc3f4a62..25f340ce122 100644
--- a/third_party/gtest/CMakeLists.txt
+++ b/third_party/gtest/CMakeLists.txt
@@ -3,7 +3,7 @@ include(FetchContent)
 FetchContent_Declare(
     googletest
     GIT_REPOSITORY https://github.com/google/googletest.git
-    GIT_TAG        e2239ee6043f73722e7aa812a459f54a28552929
+    GIT_TAG        6a7ed316a5cdc07b6d26362c90770787513822d4
 )
 # need to set the variables in CACHE due to CMP0077
 set(gtest_disable_pthreads ON CACHE INTERNAL "")
@@ -22,4 +22,4 @@ set_target_properties(gtest gtest_main PROPERTIES
     LIBRARY_OUTPUT_DIRECTORY "${GINKGO_LIBRARY_PATH}")
 # by default, the outdated targets are not being exported
 add_library(GTest::Main ALIAS gtest_main)
-add_library(GTest::GTest ALIAS gtest)
+add_library(GTest::GTest ALIAS gtest)
\ No newline at end of file